From 0128b9505c90be6acc22750c4df310667d879d00 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Fri, 24 Jul 2020 13:02:33 +0100
Subject: [PATCH 0001/1035] Revert rG5dd566b7c7b78bd- "PassManager.h - remove
 unnecessary Function.h/Module.h includes. NFCI."

This reverts commit 5dd566b7c7b78bd385418c72d63c79895be9ae97.

Causing some buildbot failures that I'm not seeing on MSVC builds.
---
 llvm/include/llvm/Analysis/DemandedBits.h                 | 2 --
 llvm/include/llvm/Analysis/DominanceFrontier.h            | 1 -
 llvm/include/llvm/Analysis/LazyValueInfo.h                | 1 -
 llvm/include/llvm/Analysis/OptimizationRemarkEmitter.h    | 2 +-
 llvm/include/llvm/Analysis/ScalarEvolution.h              | 1 -
 llvm/include/llvm/IR/PassManager.h                        | 7 ++-----
 llvm/include/llvm/Transforms/IPO/HotColdSplitting.h       | 1 -
 .../llvm/Transforms/Scalar/AlignmentFromAssumptions.h     | 2 --
 llvm/include/llvm/Transforms/Scalar/ConstantHoisting.h    | 3 ---
 llvm/include/llvm/Transforms/Scalar/Reassociate.h         | 2 +-
 llvm/include/llvm/Transforms/Utils/Debugify.h             | 1 -
 llvm/include/llvm/Transforms/Vectorize/LoopVectorize.h    | 1 -
 llvm/lib/Analysis/CallPrinter.cpp                         | 8 +++-----
 llvm/lib/Analysis/LoopPass.cpp                            | 2 --
 llvm/lib/CodeGen/AsmPrinter/DebugHandlerBase.cpp          | 1 -
 llvm/lib/CodeGen/AsmPrinter/WinCFGuard.cpp                | 3 +--
 llvm/lib/CodeGen/CFGuardLongjmp.cpp                       | 1 -
 llvm/lib/CodeGen/MachineSizeOpts.cpp                      | 3 +--
 llvm/lib/CodeGen/RegAllocBase.cpp                         | 1 -
 llvm/lib/CodeGen/WinEHPrepare.cpp                         | 1 -
 llvm/lib/IR/PassManager.cpp                               | 2 --
 llvm/lib/Linker/IRMover.cpp                               | 2 --
 llvm/lib/Target/BPF/BTFDebug.cpp                          | 1 -
 llvm/lib/Target/TargetMachine.cpp                         | 2 --
 llvm/lib/Transforms/Instrumentation/CGProfile.cpp         | 3 ++-
 llvm/lib/Transforms/Scalar/ConstantHoisting.cpp           | 1 -
 llvm/lib/Transforms/Scalar/StructurizeCFG.cpp             | 1 -
 llvm/lib/Transforms/Utils/Debugify.cpp                    | 1 +
 llvm/lib/Transforms/Utils/DemoteRegToStack.cpp            | 4 +---
 llvm/lib/Transforms/Utils/LowerSwitch.cpp                 | 1 -
 llvm/unittests/Analysis/AssumeBundleQueriesTest.cpp       | 7 +++----
 llvm/unittests/Transforms/Utils/BasicBlockUtilsTest.cpp   | 1 -
 32 files changed, 16 insertions(+), 54 deletions(-)

diff --git a/llvm/include/llvm/Analysis/DemandedBits.h b/llvm/include/llvm/Analysis/DemandedBits.h
index ebea1d3554cb4..04db3eb57c18e 100644
--- a/llvm/include/llvm/Analysis/DemandedBits.h
+++ b/llvm/include/llvm/Analysis/DemandedBits.h
@@ -36,8 +36,6 @@ class Function;
 class Instruction;
 struct KnownBits;
 class raw_ostream;
-class Use;
-class Value;
 
 class DemandedBits {
 public:
diff --git a/llvm/include/llvm/Analysis/DominanceFrontier.h b/llvm/include/llvm/Analysis/DominanceFrontier.h
index e1c6c947d420c..f67929c997f93 100644
--- a/llvm/include/llvm/Analysis/DominanceFrontier.h
+++ b/llvm/include/llvm/Analysis/DominanceFrontier.h
@@ -30,7 +30,6 @@
 
 namespace llvm {
 
-class BasicBlock;
 class Function;
 class raw_ostream;
 
diff --git a/llvm/include/llvm/Analysis/LazyValueInfo.h b/llvm/include/llvm/Analysis/LazyValueInfo.h
index b02a034a62b10..1bc88235273ee 100644
--- a/llvm/include/llvm/Analysis/LazyValueInfo.h
+++ b/llvm/include/llvm/Analysis/LazyValueInfo.h
@@ -19,7 +19,6 @@
 
 namespace llvm {
   class AssumptionCache;
-  class BasicBlock;
   class Constant;
   class ConstantRange;
   class DataLayout;
diff --git a/llvm/include/llvm/Analysis/OptimizationRemarkEmitter.h b/llvm/include/llvm/Analysis/OptimizationRemarkEmitter.h
index b89ea496cd8a3..ab97d5b8504e1 100644
--- a/llvm/include/llvm/Analysis/OptimizationRemarkEmitter.h
+++ b/llvm/include/llvm/Analysis/OptimizationRemarkEmitter.h
@@ -17,11 +17,11 @@
 #include "llvm/ADT/Optional.h"
 #include "llvm/Analysis/BlockFrequencyInfo.h"
 #include "llvm/IR/DiagnosticInfo.h"
-#include "llvm/IR/Function.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/Pass.h"
 
 namespace llvm {
+class Function;
 class Value;
 
 /// The optimization diagnostic interface.
diff --git a/llvm/include/llvm/Analysis/ScalarEvolution.h b/llvm/include/llvm/Analysis/ScalarEvolution.h
index 6b96eaa01f059..81c5fc9325884 100644
--- a/llvm/include/llvm/Analysis/ScalarEvolution.h
+++ b/llvm/include/llvm/Analysis/ScalarEvolution.h
@@ -35,7 +35,6 @@
 #include "llvm/IR/Function.h"
 #include "llvm/IR/InstrTypes.h"
 #include "llvm/IR/Instructions.h"
-#include "llvm/IR/Module.h"
 #include "llvm/IR/Operator.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/IR/ValueHandle.h"
diff --git a/llvm/include/llvm/IR/PassManager.h b/llvm/include/llvm/IR/PassManager.h
index 88d4059ebd4eb..c2f535037a56d 100644
--- a/llvm/include/llvm/IR/PassManager.h
+++ b/llvm/include/llvm/IR/PassManager.h
@@ -39,16 +39,16 @@
 
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/SmallPtrSet.h"
-#include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/TinyPtrVector.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/Module.h"
 #include "llvm/IR/PassInstrumentation.h"
 #include "llvm/IR/PassManagerInternal.h"
 #include "llvm/Pass.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/TimeProfiler.h"
 #include "llvm/Support/TypeName.h"
-#include "llvm/Support/raw_ostream.h"
 #include <algorithm>
 #include <cassert>
 #include <cstring>
@@ -62,9 +62,6 @@
 
 namespace llvm {
 
-class Function;
-class Module;
-
 /// A special type used by analysis passes to provide an address that
 /// identifies that particular analysis pass type.
 ///
diff --git a/llvm/include/llvm/Transforms/IPO/HotColdSplitting.h b/llvm/include/llvm/Transforms/IPO/HotColdSplitting.h
index 93d304f78e15e..8c3049fbaac46 100644
--- a/llvm/include/llvm/Transforms/IPO/HotColdSplitting.h
+++ b/llvm/include/llvm/Transforms/IPO/HotColdSplitting.h
@@ -16,7 +16,6 @@
 
 namespace llvm {
 
-class BasicBlock;
 class Module;
 class ProfileSummaryInfo;
 class BlockFrequencyInfo;
diff --git a/llvm/include/llvm/Transforms/Scalar/AlignmentFromAssumptions.h b/llvm/include/llvm/Transforms/Scalar/AlignmentFromAssumptions.h
index 41d29064d0490..be119b8ab8552 100644
--- a/llvm/include/llvm/Transforms/Scalar/AlignmentFromAssumptions.h
+++ b/llvm/include/llvm/Transforms/Scalar/AlignmentFromAssumptions.h
@@ -22,11 +22,9 @@
 namespace llvm {
 
 class AssumptionCache;
-class CallInst;
 class DominatorTree;
 class ScalarEvolution;
 class SCEV;
-class Value;
 
 struct AlignmentFromAssumptionsPass
     : public PassInfoMixin<AlignmentFromAssumptionsPass> {
diff --git a/llvm/include/llvm/Transforms/Scalar/ConstantHoisting.h b/llvm/include/llvm/Transforms/Scalar/ConstantHoisting.h
index 91ace385c8df4..26d4a2476a86f 100644
--- a/llvm/include/llvm/Transforms/Scalar/ConstantHoisting.h
+++ b/llvm/include/llvm/Transforms/Scalar/ConstantHoisting.h
@@ -53,15 +53,12 @@ class BlockFrequencyInfo;
 class Constant;
 class ConstantInt;
 class ConstantExpr;
-class DataLayout;
 class DominatorTree;
 class Function;
 class GlobalVariable;
 class Instruction;
-class LLVMContext;
 class ProfileSummaryInfo;
 class TargetTransformInfo;
-class Type;
 
 /// A private "module" namespace for types and utilities used by
 /// ConstantHoisting. These are implementation details and should not be used by
diff --git a/llvm/include/llvm/Transforms/Scalar/Reassociate.h b/llvm/include/llvm/Transforms/Scalar/Reassociate.h
index a20b6fd2866a3..28794d27325ad 100644
--- a/llvm/include/llvm/Transforms/Scalar/Reassociate.h
+++ b/llvm/include/llvm/Transforms/Scalar/Reassociate.h
@@ -25,7 +25,6 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/PostOrderIterator.h"
 #include "llvm/ADT/SetVector.h"
-#include "llvm/IR/Instruction.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/IR/ValueHandle.h"
 #include <deque>
@@ -36,6 +35,7 @@ class APInt;
 class BasicBlock;
 class BinaryOperator;
 class Function;
+class Instruction;
 class IRBuilderBase;
 class Value;
 
diff --git a/llvm/include/llvm/Transforms/Utils/Debugify.h b/llvm/include/llvm/Transforms/Utils/Debugify.h
index a7e311a6d74cd..1b9d43b775e75 100644
--- a/llvm/include/llvm/Transforms/Utils/Debugify.h
+++ b/llvm/include/llvm/Transforms/Utils/Debugify.h
@@ -18,7 +18,6 @@
 #include "llvm/Bitcode/BitcodeWriterPass.h"
 #include "llvm/IR/IRPrintingPasses.h"
 #include "llvm/IR/LegacyPassManager.h"
-#include "llvm/IR/Module.h"
 #include "llvm/IR/PassManager.h"
 
 namespace llvm {
diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorize.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorize.h
index f739708179dbf..ecb44a7b1518d 100644
--- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorize.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorize.h
@@ -68,7 +68,6 @@ class BlockFrequencyInfo;
 class DemandedBits;
 class DominatorTree;
 class Function;
-class Instruction;
 class Loop;
 class LoopAccessInfo;
 class LoopInfo;
diff --git a/llvm/lib/Analysis/CallPrinter.cpp b/llvm/lib/Analysis/CallPrinter.cpp
index 7c04b11179df8..bb447411ec472 100644
--- a/llvm/lib/Analysis/CallPrinter.cpp
+++ b/llvm/lib/Analysis/CallPrinter.cpp
@@ -14,17 +14,15 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Analysis/CallPrinter.h"
-#include "llvm/ADT/DenseMap.h"
-#include "llvm/ADT/SmallSet.h"
 #include "llvm/Analysis/BlockFrequencyInfo.h"
 #include "llvm/Analysis/BranchProbabilityInfo.h"
 #include "llvm/Analysis/CallGraph.h"
 #include "llvm/Analysis/DOTGraphTraitsPass.h"
 #include "llvm/Analysis/HeatUtils.h"
-#include "llvm/IR/Function.h"
-#include "llvm/IR/Module.h"
-#include "llvm/InitializePasses.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/SmallSet.h"
 
 using namespace llvm;
 
diff --git a/llvm/lib/Analysis/LoopPass.cpp b/llvm/lib/Analysis/LoopPass.cpp
index 0e253a23b0da9..520f06003dd22 100644
--- a/llvm/lib/Analysis/LoopPass.cpp
+++ b/llvm/lib/Analysis/LoopPass.cpp
@@ -17,7 +17,6 @@
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/IRPrintingPasses.h"
 #include "llvm/IR/LLVMContext.h"
-#include "llvm/IR/Module.h"
 #include "llvm/IR/OptBisect.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/IR/PassTimingInfo.h"
@@ -26,7 +25,6 @@
 #include "llvm/Support/TimeProfiler.h"
 #include "llvm/Support/Timer.h"
 #include "llvm/Support/raw_ostream.h"
-
 using namespace llvm;
 
 #define DEBUG_TYPE "loop-pass-manager"
diff --git a/llvm/lib/CodeGen/AsmPrinter/DebugHandlerBase.cpp b/llvm/lib/CodeGen/AsmPrinter/DebugHandlerBase.cpp
index 638acb5e80451..a46de83e555cc 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DebugHandlerBase.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DebugHandlerBase.cpp
@@ -20,7 +20,6 @@
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/TargetSubtargetInfo.h"
 #include "llvm/IR/DebugInfo.h"
-#include "llvm/IR/Function.h"
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/Support/CommandLine.h"
 
diff --git a/llvm/lib/CodeGen/AsmPrinter/WinCFGuard.cpp b/llvm/lib/CodeGen/AsmPrinter/WinCFGuard.cpp
index d5f85f8c01e7e..914308d9147e2 100644
--- a/llvm/lib/CodeGen/AsmPrinter/WinCFGuard.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/WinCFGuard.cpp
@@ -17,9 +17,8 @@
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/MachineOperand.h"
 #include "llvm/IR/Constants.h"
-#include "llvm/IR/Instructions.h"
 #include "llvm/IR/Metadata.h"
-#include "llvm/IR/Module.h"
+#include "llvm/IR/Instructions.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCObjectFileInfo.h"
 #include "llvm/MC/MCStreamer.h"
diff --git a/llvm/lib/CodeGen/CFGuardLongjmp.cpp b/llvm/lib/CodeGen/CFGuardLongjmp.cpp
index b5d88a7432b17..c3bf938551110 100644
--- a/llvm/lib/CodeGen/CFGuardLongjmp.cpp
+++ b/llvm/lib/CodeGen/CFGuardLongjmp.cpp
@@ -21,7 +21,6 @@
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/MachineOperand.h"
 #include "llvm/CodeGen/Passes.h"
-#include "llvm/IR/Module.h"
 #include "llvm/InitializePasses.h"
 
 using namespace llvm;
diff --git a/llvm/lib/CodeGen/MachineSizeOpts.cpp b/llvm/lib/CodeGen/MachineSizeOpts.cpp
index e19862eae5752..584d43b420044 100644
--- a/llvm/lib/CodeGen/MachineSizeOpts.cpp
+++ b/llvm/lib/CodeGen/MachineSizeOpts.cpp
@@ -12,10 +12,9 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/CodeGen/MachineSizeOpts.h"
-#include "llvm/Analysis/ProfileSummaryInfo.h"
 #include "llvm/CodeGen/MBFIWrapper.h"
+#include "llvm/Analysis/ProfileSummaryInfo.h"
 #include "llvm/CodeGen/MachineBlockFrequencyInfo.h"
-#include "llvm/IR/Function.h"
 
 using namespace llvm;
 
diff --git a/llvm/lib/CodeGen/RegAllocBase.cpp b/llvm/lib/CodeGen/RegAllocBase.cpp
index 71647f0abeeb8..d228268536724 100644
--- a/llvm/lib/CodeGen/RegAllocBase.cpp
+++ b/llvm/lib/CodeGen/RegAllocBase.cpp
@@ -23,7 +23,6 @@
 #include "llvm/CodeGen/Spiller.h"
 #include "llvm/CodeGen/TargetRegisterInfo.h"
 #include "llvm/CodeGen/VirtRegMap.h"
-#include "llvm/IR/Module.h"
 #include "llvm/Pass.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
diff --git a/llvm/lib/CodeGen/WinEHPrepare.cpp b/llvm/lib/CodeGen/WinEHPrepare.cpp
index bbad3beb0b8b8..5a25234ba850b 100644
--- a/llvm/lib/CodeGen/WinEHPrepare.cpp
+++ b/llvm/lib/CodeGen/WinEHPrepare.cpp
@@ -24,7 +24,6 @@
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/Passes.h"
 #include "llvm/CodeGen/WinEHFuncInfo.h"
-#include "llvm/IR/Module.h"
 #include "llvm/IR/Verifier.h"
 #include "llvm/InitializePasses.h"
 #include "llvm/MC/MCSymbol.h"
diff --git a/llvm/lib/IR/PassManager.cpp b/llvm/lib/IR/PassManager.cpp
index 104d3a5234388..624827ff8cd94 100644
--- a/llvm/lib/IR/PassManager.cpp
+++ b/llvm/lib/IR/PassManager.cpp
@@ -8,9 +8,7 @@
 
 #include "llvm/IR/PassManager.h"
 #include "llvm/ADT/STLExtras.h"
-#include "llvm/IR/Function.h"
 #include "llvm/IR/LLVMContext.h"
-#include "llvm/IR/Module.h"
 #include "llvm/IR/PassManagerImpl.h"
 
 using namespace llvm;
diff --git a/llvm/lib/Linker/IRMover.cpp b/llvm/lib/Linker/IRMover.cpp
index 203ad85a528b2..055689b16e8f4 100644
--- a/llvm/lib/Linker/IRMover.cpp
+++ b/llvm/lib/Linker/IRMover.cpp
@@ -16,12 +16,10 @@
 #include "llvm/IR/DiagnosticPrinter.h"
 #include "llvm/IR/GVMaterializer.h"
 #include "llvm/IR/Intrinsics.h"
-#include "llvm/IR/Module.h"
 #include "llvm/IR/TypeFinder.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Transforms/Utils/Cloning.h"
 #include <utility>
-
 using namespace llvm;
 
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/BPF/BTFDebug.cpp b/llvm/lib/Target/BPF/BTFDebug.cpp
index a8b5ea55c8653..13999d800a800 100644
--- a/llvm/lib/Target/BPF/BTFDebug.cpp
+++ b/llvm/lib/Target/BPF/BTFDebug.cpp
@@ -17,7 +17,6 @@
 #include "llvm/BinaryFormat/ELF.h"
 #include "llvm/CodeGen/AsmPrinter.h"
 #include "llvm/CodeGen/MachineModuleInfo.h"
-#include "llvm/IR/Module.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCObjectFileInfo.h"
 #include "llvm/MC/MCSectionELF.h"
diff --git a/llvm/lib/Target/TargetMachine.cpp b/llvm/lib/Target/TargetMachine.cpp
index 730bf49cf46f2..074e9fde79e6b 100644
--- a/llvm/lib/Target/TargetMachine.cpp
+++ b/llvm/lib/Target/TargetMachine.cpp
@@ -18,7 +18,6 @@
 #include "llvm/IR/GlobalVariable.h"
 #include "llvm/IR/LegacyPassManager.h"
 #include "llvm/IR/Mangler.h"
-#include "llvm/IR/Module.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCInstrInfo.h"
@@ -26,7 +25,6 @@
 #include "llvm/MC/MCTargetOptions.h"
 #include "llvm/MC/SectionKind.h"
 #include "llvm/Target/TargetLoweringObjectFile.h"
-
 using namespace llvm;
 
 //---------------------------------------------------------------------------
diff --git a/llvm/lib/Transforms/Instrumentation/CGProfile.cpp b/llvm/lib/Transforms/Instrumentation/CGProfile.cpp
index b143217ae0dc3..0cc0d9b07387b 100644
--- a/llvm/lib/Transforms/Instrumentation/CGProfile.cpp
+++ b/llvm/lib/Transforms/Instrumentation/CGProfile.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Transforms/Instrumentation/CGProfile.h"
+
 #include "llvm/ADT/MapVector.h"
 #include "llvm/Analysis/BlockFrequencyInfo.h"
 #include "llvm/Analysis/LazyBlockFrequencyInfo.h"
@@ -14,11 +15,11 @@
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/MDBuilder.h"
-#include "llvm/IR/Module.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/InitializePasses.h"
 #include "llvm/ProfileData/InstrProf.h"
 #include "llvm/Transforms/Instrumentation.h"
+
 #include <array>
 
 using namespace llvm;
diff --git a/llvm/lib/Transforms/Scalar/ConstantHoisting.cpp b/llvm/lib/Transforms/Scalar/ConstantHoisting.cpp
index 9cace63fd0b86..7c14b69d658dc 100644
--- a/llvm/lib/Transforms/Scalar/ConstantHoisting.cpp
+++ b/llvm/lib/Transforms/Scalar/ConstantHoisting.cpp
@@ -52,7 +52,6 @@
 #include "llvm/IR/Instruction.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/IntrinsicInst.h"
-#include "llvm/IR/Module.h"
 #include "llvm/IR/Value.h"
 #include "llvm/InitializePasses.h"
 #include "llvm/Pass.h"
diff --git a/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp b/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp
index 0e53b27cc4a10..c20e57b02c1a5 100644
--- a/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp
+++ b/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp
@@ -28,7 +28,6 @@
 #include "llvm/IR/Instruction.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/Metadata.h"
-#include "llvm/IR/Module.h"
 #include "llvm/IR/PatternMatch.h"
 #include "llvm/IR/Type.h"
 #include "llvm/IR/Use.h"
diff --git a/llvm/lib/Transforms/Utils/Debugify.cpp b/llvm/lib/Transforms/Utils/Debugify.cpp
index ef156514fa0cd..8f98d81a3d797 100644
--- a/llvm/lib/Transforms/Utils/Debugify.cpp
+++ b/llvm/lib/Transforms/Utils/Debugify.cpp
@@ -19,6 +19,7 @@
 #include "llvm/IR/InstIterator.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/Module.h"
 #include "llvm/Pass.h"
 #include "llvm/Support/CommandLine.h"
 
diff --git a/llvm/lib/Transforms/Utils/DemoteRegToStack.cpp b/llvm/lib/Transforms/Utils/DemoteRegToStack.cpp
index 865a22a3369dd..5f53d794fe8a1 100644
--- a/llvm/lib/Transforms/Utils/DemoteRegToStack.cpp
+++ b/llvm/lib/Transforms/Utils/DemoteRegToStack.cpp
@@ -8,13 +8,11 @@
 
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/Analysis/CFG.h"
+#include "llvm/Transforms/Utils/Local.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/Instructions.h"
-#include "llvm/IR/Module.h"
 #include "llvm/IR/Type.h"
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
-#include "llvm/Transforms/Utils/Local.h"
-
 using namespace llvm;
 
 /// DemoteRegToStack - This function takes a virtual register computed by an
diff --git a/llvm/lib/Transforms/Utils/LowerSwitch.cpp b/llvm/lib/Transforms/Utils/LowerSwitch.cpp
index eb5c670625651..34e836d9660f3 100644
--- a/llvm/lib/Transforms/Utils/LowerSwitch.cpp
+++ b/llvm/lib/Transforms/Utils/LowerSwitch.cpp
@@ -26,7 +26,6 @@
 #include "llvm/IR/Function.h"
 #include "llvm/IR/InstrTypes.h"
 #include "llvm/IR/Instructions.h"
-#include "llvm/IR/Module.h"
 #include "llvm/IR/Value.h"
 #include "llvm/InitializePasses.h"
 #include "llvm/Pass.h"
diff --git a/llvm/unittests/Analysis/AssumeBundleQueriesTest.cpp b/llvm/unittests/Analysis/AssumeBundleQueriesTest.cpp
index f885e56bf4eaf..d35a77fa379be 100644
--- a/llvm/unittests/Analysis/AssumeBundleQueriesTest.cpp
+++ b/llvm/unittests/Analysis/AssumeBundleQueriesTest.cpp
@@ -6,15 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "llvm/Analysis/AssumeBundleQueries.h"
 #include "llvm/Analysis/AssumptionCache.h"
+#include "llvm/Analysis/AssumeBundleQueries.h"
 #include "llvm/AsmParser/Parser.h"
-#include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/LLVMContext.h"
-#include "llvm/IR/Module.h"
-#include "llvm/Support/CommandLine.h"
+#include "llvm/IR/IntrinsicInst.h"
 #include "llvm/Support/Regex.h"
 #include "llvm/Support/SourceMgr.h"
+#include "llvm/Support/CommandLine.h"
 #include "llvm/Transforms/Utils/AssumeBundleBuilder.h"
 #include "gtest/gtest.h"
 #include <random>
diff --git a/llvm/unittests/Transforms/Utils/BasicBlockUtilsTest.cpp b/llvm/unittests/Transforms/Utils/BasicBlockUtilsTest.cpp
index ac8661bafaa13..e810b66b3b7c2 100644
--- a/llvm/unittests/Transforms/Utils/BasicBlockUtilsTest.cpp
+++ b/llvm/unittests/Transforms/Utils/BasicBlockUtilsTest.cpp
@@ -15,7 +15,6 @@
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/LLVMContext.h"
-#include "llvm/IR/Module.h"
 #include "llvm/Support/SourceMgr.h"
 #include "gtest/gtest.h"
 

From 0ed660f9e5740b46bc10773167cb1e08b6a86389 Mon Sep 17 00:00:00 2001
From: Nico Weber <thakis@chromium.org>
Date: Fri, 24 Jul 2020 08:29:36 -0400
Subject: [PATCH 0002/1035] [gn build] (manually) port 228f8d89

---
 llvm/utils/gn/secondary/llvm/include/llvm/Config/BUILD.gn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/utils/gn/secondary/llvm/include/llvm/Config/BUILD.gn b/llvm/utils/gn/secondary/llvm/include/llvm/Config/BUILD.gn
index 81da62eff1743..8fe8a46bf5832 100644
--- a/llvm/utils/gn/secondary/llvm/include/llvm/Config/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/include/llvm/Config/BUILD.gn
@@ -124,7 +124,6 @@ write_cmake_config("config") {
     "RETSIGTYPE=void",
     "LLVM_GISEL_COV_ENABLED=",
     "LLVM_GISEL_COV_PREFIX=",
-    "LLVM_HAVE_TF_AOT=",
     "LLVM_WITH_Z3=",
 
     # FIXME: Set to 1 on mac once the 10.14 SDK is in common use.
@@ -326,6 +325,7 @@ write_cmake_config("llvm-config") {
     "LLVM_ENABLE_DUMP=",
     "LLVM_DEFAULT_TARGET_TRIPLE=$llvm_target_triple",
     "LLVM_HAS_ATOMICS=1",
+    "LLVM_HAVE_TF_AOT=",
     "LLVM_HAVE_TF_API=",
     "LLVM_HOST_TRIPLE=$llvm_current_triple",
     "LLVM_NATIVE_ARCH=$native_target",

From 6371a0a00edb2c56363f7d494a2fac9b6bcaee3c Mon Sep 17 00:00:00 2001
From: Djordje Todorovic <djordje.todorovic@syrmia.com>
Date: Fri, 24 Jul 2020 14:32:25 +0200
Subject: [PATCH 0003/1035] [DWARF][EntryValues] Emit GNU extensions in the
 case of DWARF 4 + SCE

Emit DWARF 5 call-site symbols even though DWARF 4 is set,
only in the case of LLDB tuning.

This patch addresses PR46643.

Differential Revision: https://reviews.llvm.org/D83463
---
 .../CodeGen/AsmPrinter/DwarfCompileUnit.cpp   |  2 +-
 .../MIR/X86/call-site-gnu-vs-dwarf5-attrs.mir | 73 ++++++++++---------
 2 files changed, 40 insertions(+), 35 deletions(-)

diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
index ece6665e99f6c..2de6569767f69 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
@@ -990,7 +990,7 @@ void DwarfCompileUnit::constructAbstractSubprogramScopeDIE(
 }
 
 bool DwarfCompileUnit::useGNUAnalogForDwarf5Feature() const {
-  return DD->getDwarfVersion() == 4 && DD->tuneForGDB();
+  return DD->getDwarfVersion() == 4 && !DD->tuneForLLDB();
 }
 
 dwarf::Tag DwarfCompileUnit::getDwarf5OrGNUTag(dwarf::Tag Tag) const {
diff --git a/llvm/test/DebugInfo/MIR/X86/call-site-gnu-vs-dwarf5-attrs.mir b/llvm/test/DebugInfo/MIR/X86/call-site-gnu-vs-dwarf5-attrs.mir
index fdbebd306f37c..bde717e3c9da5 100644
--- a/llvm/test/DebugInfo/MIR/X86/call-site-gnu-vs-dwarf5-attrs.mir
+++ b/llvm/test/DebugInfo/MIR/X86/call-site-gnu-vs-dwarf5-attrs.mir
@@ -1,48 +1,54 @@
-# Test the call site encoding in DWARF5 vs GNU extensions.
-#
-# === DWARF4, tune for gdb ===
+## Test the call site encoding in DWARF5 vs GNU extensions.
+
+## === DWARF4, tune for gdb ===
 # RUN: llc -emit-call-site-info -dwarf-version 4 -debugger-tune=gdb -filetype=obj \
 # RUN:     -mtriple=x86_64-unknown-unknown -start-after=machineverifier -o - %s  \
 # RUN:     | llvm-dwarfdump - | FileCheck %s -check-prefixes=CHECK-GNU -implicit-check-not=DW_AT_call
-#
+
 # === DWARF5, tune for gdb ===
 # RUN: llc -dwarf-version 5 -debugger-tune=gdb -emit-call-site-info -filetype=obj \
 # RUN:     -mtriple=x86_64-unknown-unknown -start-after=machineverifier -o - %s  \
 # RUN:     | llvm-dwarfdump - | FileCheck %s -check-prefixes=CHECK-DWARF5 -implicit-check-not=DW_AT_call
-#
-# === DWARF4, tune for lldb ===
+
+## === DWARF4, tune for lldb ===
 # RUN: llc -dwarf-version 4 -debugger-tune=lldb -emit-call-site-info -filetype=obj \
 # RUN:     -mtriple=x86_64-unknown-unknown -start-after=machineverifier -o - %s   \
 # RUN:     | llvm-dwarfdump - | FileCheck %s -check-prefixes=CHECK-DWARF5 -implicit-check-not=DW_AT_call
-#
-# === DWARF5, tune for lldb ===
+
+## === DWARF5, tune for lldb ===
 # RUN: llc -dwarf-version 5 -debugger-tune=lldb -emit-call-site-info -filetype=obj \
 # RUN:     -mtriple=x86_64-unknown-unknown -start-after=machineverifier -o - %s   \
 # RUN:     | llvm-dwarfdump - | FileCheck %s -check-prefixes=CHECK-DWARF5 -implicit-check-not=DW_AT_call
-#
+
+## === DWARF4, tune for sce ===
+# RUN: llc -emit-call-site-info -dwarf-version 4 -filetype=obj -debugger-tune=sce \
+# RUN:     -debug-entry-values -mtriple=x86_64-unknown-unknown \
+# RUN:     -start-after=machineverifier -o - %s | llvm-dwarfdump - | FileCheck %s -check-prefixes=CHECK-GNU
+
+## === DWARF5, tune for sce ===
 # RUN: llc -emit-call-site-info -dwarf-version 5 -filetype=obj -debugger-tune=sce \
 # RUN:     -debug-entry-values -mtriple=x86_64-unknown-unknown \
 # RUN:     -start-after=machineverifier -o - %s | llvm-dwarfdump - | FileCheck %s -check-prefixes=CHECK-DWARF5
-#
-# This is based on the following reproducer:
-#
-# extern void fn();
-# extern void fn2(int x);
-# extern int fn3();
-#
-# int fn1(int (*fn4) ()) {
-#   fn();
-#   fn2(5);
-#
-#   int x = (*fn4)();
-#   if (!x)
-#     return fn3();
-#   else
-#     return -1;
-# }
-#
-# Check GNU extensions:
-#
+
+## This is based on the following reproducer:
+##
+## extern void fn();
+## extern void fn2(int x);
+## extern int fn3();
+##
+## int fn1(int (*fn4) ()) {
+##   fn();
+##   fn2(5);
+##
+##   int x = (*fn4)();
+##   if (!x)
+##     return fn3();
+##   else
+##     return -1;
+## }
+
+## Check GNU extensions:
+
 # CHECK-GNU:        DW_TAG_subprogram
 # CHECK-GNU:          DW_AT_GNU_all_call_sites    (true)
 # CHECK-GNU:        DW_TAG_GNU_call_site
@@ -58,10 +64,9 @@
 # CHECK-GNU-NEXT:     DW_AT_abstract_origin
 # CHECK-GNU-NEXT:     DW_AT_GNU_tail_call
 # CHECK-GNU-NEXT:     DW_AT_low_pc
-#
-#
-# Check DWARF 5:
-#
+
+## Check DWARF 5:
+
 # CHECK-DWARF5:        DW_TAG_subprogram
 # CHECK-DWARF5:          DW_AT_call_all_calls    (true)
 # CHECK-DWARF5:        DW_TAG_call_site
@@ -80,7 +85,7 @@
 # CHECK-DWARF5-NEXT:     DW_AT_call_origin
 # CHECK-DWARF5-NEXT:     DW_AT_call_tail_call
 # CHECK-DWARF5-NEXT:     DW_AT_call_pc
-#
+
 --- |
   ; ModuleID = 'call-site-attrs.c'
   source_filename = "call-site-attrs.c"

From 2ad56119f5dc6c6af2b8ddfd9fc8c6460a7507c8 Mon Sep 17 00:00:00 2001
From: Nico Weber <thakis@chromium.org>
Date: Fri, 24 Jul 2020 08:37:47 -0400
Subject: [PATCH 0004/1035] [gn build] (manually) port 10b1b4a23

---
 llvm/utils/gn/secondary/clang/test/BUILD.gn        |  4 ++--
 llvm/utils/gn/secondary/lld/test/BUILD.gn          |  4 ++--
 .../gn/secondary/llvm/include/llvm/Config/BUILD.gn | 14 ++------------
 llvm/utils/gn/secondary/llvm/test/BUILD.gn         |  4 ++--
 4 files changed, 8 insertions(+), 18 deletions(-)

diff --git a/llvm/utils/gn/secondary/clang/test/BUILD.gn b/llvm/utils/gn/secondary/clang/test/BUILD.gn
index 56feaae9a149d..a0680d9848235 100644
--- a/llvm/utils/gn/secondary/clang/test/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/test/BUILD.gn
@@ -79,9 +79,9 @@ write_lit_config("lit_site_cfg") {
   }
 
   if (llvm_enable_zlib) {
-    extra_values += [ "HAVE_LIBZ=1" ]
+    extra_values += [ "LLVM_ENABLE_ZLIB=1" ]
   } else {
-    extra_values += [ "HAVE_LIBZ=0" ]  # Must be 0.
+    extra_values += [ "LLVM_ENABLE_ZLIB=0" ]  # Must be 0.
   }
 
   if (host_cpu == "x64") {
diff --git a/llvm/utils/gn/secondary/lld/test/BUILD.gn b/llvm/utils/gn/secondary/lld/test/BUILD.gn
index dac50890d4ac9..a6fb457cff6ac 100644
--- a/llvm/utils/gn/secondary/lld/test/BUILD.gn
+++ b/llvm/utils/gn/secondary/lld/test/BUILD.gn
@@ -49,9 +49,9 @@ write_lit_cfg("lit_site_cfg") {
   }
 
   if (llvm_enable_zlib) {
-    extra_values += [ "HAVE_LIBZ=1" ]
+    extra_values += [ "LLVM_ENABLE_ZLIB=1" ]
   } else {
-    extra_values += [ "HAVE_LIBZ=0" ]  # Must be 0.
+    extra_values += [ "LLVM_ENABLE_ZLIB=0" ]  # Must be 0.
   }
 
   if (current_cpu == "x64" || current_cpu == "arm64" ||
diff --git a/llvm/utils/gn/secondary/llvm/include/llvm/Config/BUILD.gn b/llvm/utils/gn/secondary/llvm/include/llvm/Config/BUILD.gn
index 8fe8a46bf5832..72b5796cd9aaa 100644
--- a/llvm/utils/gn/secondary/llvm/include/llvm/Config/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/include/llvm/Config/BUILD.gn
@@ -295,20 +295,10 @@ write_cmake_config("config") {
     values += [ "LLVM_ENABLE_DIA_SDK=" ]
   }
 
-  # FIXME: Once https://reviews.llvm.org/D79219 is in, remove the two
-  # redundant HAVE_ variables.
   if (llvm_enable_zlib) {
-    values += [
-      "HAVE_LIBZ=1",
-      "HAVE_ZLIB_H=1",
-      "LLVM_ENABLE_ZLIB=1",
-    ]
+    values += [ "LLVM_ENABLE_ZLIB=1" ]
   } else {
-    values += [
-      "HAVE_LIBZ=",
-      "HAVE_ZLIB_H=",
-      "LLVM_ENABLE_ZLIB=",
-    ]
+    values += [ "LLVM_ENABLE_ZLIB=" ]
   }
 
   if (llvm_enable_libxml2) {
diff --git a/llvm/utils/gn/secondary/llvm/test/BUILD.gn b/llvm/utils/gn/secondary/llvm/test/BUILD.gn
index 3e72ef3c3a446..3ad1d0ba4f443 100644
--- a/llvm/utils/gn/secondary/llvm/test/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/test/BUILD.gn
@@ -174,9 +174,9 @@ write_lit_config("lit_site_cfg") {
   }
 
   if (llvm_enable_zlib) {
-    extra_values += [ "HAVE_LIBZ=1" ]
+    extra_values += [ "LLVM_ENABLE_ZLIB=1" ]
   } else {
-    extra_values += [ "HAVE_LIBZ=0" ]  # Must be 0.
+    extra_values += [ "LLVM_ENABLE_ZLIB=0" ]  # Must be 0.
   }
 }
 

From 0bbaacc8cae0373d4500c4e3f6f128d21f9033b7 Mon Sep 17 00:00:00 2001
From: Nico Weber <thakis@chromium.org>
Date: Fri, 24 Jul 2020 08:48:14 -0400
Subject: [PATCH 0005/1035] [gn build] (manually) port 10b1b4a23 more

---
 llvm/utils/gn/secondary/compiler-rt/test/BUILD.gn | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/llvm/utils/gn/secondary/compiler-rt/test/BUILD.gn b/llvm/utils/gn/secondary/compiler-rt/test/BUILD.gn
index 97007eecd1eb9..5f8058699d729 100644
--- a/llvm/utils/gn/secondary/compiler-rt/test/BUILD.gn
+++ b/llvm/utils/gn/secondary/compiler-rt/test/BUILD.gn
@@ -86,8 +86,8 @@ write_cmake_config("lit_common_configured") {
   }
 
   if (llvm_enable_zlib) {
-    values += [ "HAVE_LIBZ=1" ]
+    values += [ "LLVM_ENABLE_ZLIB=1" ]
   } else {
-    values += [ "HAVE_LIBZ=0" ]
+    values += [ "LLVM_ENABLE_ZLIB=0" ]
   }
 }

From 5c98631391fa1c7315a687da853a776955d1531e Mon Sep 17 00:00:00 2001
From: George Mitenkov <georgemitenk0v@gmail.com>
Date: Fri, 24 Jul 2020 15:01:08 +0300
Subject: [PATCH 0006/1035] [MLIR][SPIRVToLLVM] Conversion of SPIR-V variable
 op

The patch introduces the conversion pattern for function-level
`spv.Variable`. It is modelled as `llvm.alloca` op. If initialized, then
additional store instruction is used. Note that there is no initialization
for arrays and structs since constants of these types are not supported in
LLVM dialect yet. Also, at the moment initialisation is only possible via
`spv.constant` (since `spv.GlobalVariable` conversion is not implemented
yet).

The input code has some scoping is not taken into account and will be
addressed in a different patch.

Reviewed By: ftynse

Differential Revision: https://reviews.llvm.org/D84224
---
 .../SPIRVToLLVM/ConvertSPIRVToLLVM.cpp        | 42 ++++++++++++++++
 .../SPIRVToLLVM/memory-ops-to-llvm.mlir       | 49 +++++++++++++++++++
 2 files changed, 91 insertions(+)
 create mode 100644 mlir/test/Conversion/SPIRVToLLVM/memory-ops-to-llvm.mlir

diff --git a/mlir/lib/Conversion/SPIRVToLLVM/ConvertSPIRVToLLVM.cpp b/mlir/lib/Conversion/SPIRVToLLVM/ConvertSPIRVToLLVM.cpp
index 15aaa7947a497..cb4898c346a65 100644
--- a/mlir/lib/Conversion/SPIRVToLLVM/ConvertSPIRVToLLVM.cpp
+++ b/mlir/lib/Conversion/SPIRVToLLVM/ConvertSPIRVToLLVM.cpp
@@ -177,6 +177,14 @@ static Type convertStructTypePacked(spirv::StructType type,
                                      /*isPacked=*/true);
 }
 
+/// Creates LLVM dialect constant with the given value.
+static Value createI32ConstantOf(Location loc, PatternRewriter &rewriter,
+                                 LLVMTypeConverter &converter, unsigned value) {
+  return rewriter.create<LLVM::ConstantOp>(
+      loc, LLVM::LLVMType::getInt32Ty(converter.getDialect()),
+      rewriter.getIntegerAttr(rewriter.getI32Type(), value));
+}
+
 //===----------------------------------------------------------------------===//
 // Type conversion
 //===----------------------------------------------------------------------===//
@@ -733,6 +741,37 @@ class ShiftPattern : public SPIRVToLLVMConversion<SPIRVOp> {
   }
 };
 
+class VariablePattern : public SPIRVToLLVMConversion<spirv::VariableOp> {
+public:
+  using SPIRVToLLVMConversion<spirv::VariableOp>::SPIRVToLLVMConversion;
+
+  LogicalResult
+  matchAndRewrite(spirv::VariableOp varOp, ArrayRef<Value> operands,
+                  ConversionPatternRewriter &rewriter) const override {
+    auto srcType = varOp.getType();
+    // Initialization is supported for scalars and vectors only.
+    auto pointerTo = srcType.cast<spirv::PointerType>().getPointeeType();
+    auto init = varOp.initializer();
+    if (init && !pointerTo.isIntOrFloat() && !pointerTo.isa<VectorType>())
+      return failure();
+
+    auto dstType = typeConverter.convertType(srcType);
+    if (!dstType)
+      return failure();
+
+    Location loc = varOp.getLoc();
+    Value size = createI32ConstantOf(loc, rewriter, typeConverter, 1);
+    if (!init) {
+      rewriter.replaceOpWithNewOp<LLVM::AllocaOp>(varOp, dstType, size);
+      return success();
+    }
+    Value allocated = rewriter.create<LLVM::AllocaOp>(loc, dstType, size);
+    rewriter.create<LLVM::StoreOp>(loc, init, allocated);
+    rewriter.replaceOp(varOp, allocated);
+    return success();
+  }
+};
+
 //===----------------------------------------------------------------------===//
 // FuncOp conversion
 //===----------------------------------------------------------------------===//
@@ -933,6 +972,9 @@ void mlir::populateSPIRVToLLVMConversionPatterns(
       IComparePattern<spirv::LogicalNotEqualOp, LLVM::ICmpPredicate::ne>,
       NotPattern<spirv::LogicalNotOp>,
 
+      // Memory ops
+      VariablePattern,
+
       // Miscellaneous ops
       DirectConversionPattern<spirv::SelectOp, LLVM::SelectOp>,
       DirectConversionPattern<spirv::UndefOp, LLVM::UndefOp>,
diff --git a/mlir/test/Conversion/SPIRVToLLVM/memory-ops-to-llvm.mlir b/mlir/test/Conversion/SPIRVToLLVM/memory-ops-to-llvm.mlir
new file mode 100644
index 0000000000000..4c549ab8d619c
--- /dev/null
+++ b/mlir/test/Conversion/SPIRVToLLVM/memory-ops-to-llvm.mlir
@@ -0,0 +1,49 @@
+// RUN: mlir-opt -convert-spirv-to-llvm %s | FileCheck %s
+
+//===----------------------------------------------------------------------===//
+// spv.Variable
+//===----------------------------------------------------------------------===//
+
+func @variable_scalar() {
+	// CHECK: %[[SIZE1:.*]] = llvm.mlir.constant(1 : i32) : !llvm.i32
+  // CHECK: %{{.*}} = llvm.alloca %[[SIZE1]] x !llvm.float : (!llvm.i32) -> !llvm<"float*">
+	%0 = spv.Variable : !spv.ptr<f32, Function>
+	// CHECK: %[[SIZE2:.*]] = llvm.mlir.constant(1 : i32) : !llvm.i32
+  // CHECK: %{{.*}} = llvm.alloca %[[SIZE2]] x !llvm.i8 : (!llvm.i32) -> !llvm<"i8*">
+	%1 = spv.Variable : !spv.ptr<i8, Function>
+  return
+}
+
+func @variable_scalar_with_initialization() {
+	// CHECK: %[[VALUE:.*]] = llvm.mlir.constant(0 : i64) : !llvm.i64
+	// CHECK: %[[SIZE:.*]] = llvm.mlir.constant(1 : i32) : !llvm.i32
+  // CHECK: %[[ALLOCATED:.*]] = llvm.alloca %[[SIZE]] x !llvm.i64 : (!llvm.i32) -> !llvm<"i64*">
+	// CHECK: llvm.store %[[VALUE]], %[[ALLOCATED]] : !llvm<"i64*">
+	%c = spv.constant 0 : i64
+	%0 = spv.Variable init(%c) : !spv.ptr<i64, Function>
+  return
+}
+
+func @variable_vector() {
+	// CHECK: %[[SIZE:.*]] = llvm.mlir.constant(1 : i32) : !llvm.i32
+	// CHECK: %{{.*}} = llvm.alloca  %[[SIZE]] x !llvm<"<3 x float>"> : (!llvm.i32) -> !llvm<"<3 x float>*">
+	%0 = spv.Variable : !spv.ptr<vector<3xf32>, Function>
+	return
+}
+
+func @variable_vector_with_initialization() {
+	// CHECK: %[[VALUE:.*]] = llvm.mlir.constant(dense<false> : vector<3xi1>) : !llvm<"<3 x i1>">
+	// CHECK: %[[SIZE:.*]] = llvm.mlir.constant(1 : i32) : !llvm.i32
+  // CHECK: %[[ALLOCATED:.*]] = llvm.alloca %[[SIZE]] x !llvm<"<3 x i1>"> : (!llvm.i32) -> !llvm<"<3 x i1>*">
+	// CHECK: llvm.store %[[VALUE]], %[[ALLOCATED]] : !llvm<"<3 x i1>*">
+	%c = spv.constant dense<false> : vector<3xi1>
+	%0 = spv.Variable init(%c) : !spv.ptr<vector<3xi1>, Function>
+  return
+}
+
+func @variable_array() {
+	// CHECK: %[[SIZE:.*]] = llvm.mlir.constant(1 : i32) : !llvm.i32
+	// CHECK: %{{.*}} = llvm.alloca %[[SIZE]] x !llvm<"[10 x i32]"> : (!llvm.i32) -> !llvm<"[10 x i32]*">
+	%0 = spv.Variable : !spv.ptr<!spv.array<10 x i32>, Function>
+	return
+}

From cf428778128fed5eacee884964af53bf4a9f74f2 Mon Sep 17 00:00:00 2001
From: Yitzhak Mandelbaum <yitzhakm@google.com>
Date: Tue, 21 Jul 2020 20:09:49 +0000
Subject: [PATCH 0007/1035] [libTooling] Add assorted `EditGenerator`
 combinators.

Summary:
This patch adds various combinators that help in constructing `EditGenerator`s:
   * `noEdits`
   * `ifBound`, specialized to `ASTEdit`
   * `flatten` and `flattenVector` which allow for easy construction from a set
     of sub edits.
   * `shrinkTo`, which generates edits to shrink a given range to another that
     it encloses.

Reviewers: asoffer, gribozavr2

Subscribers: cfe-commits

Tags: #clang

Differential Revision: https://reviews.llvm.org/D84310
---
 .../clang/Tooling/Transformer/MatchConsumer.h |   6 -
 .../clang/Tooling/Transformer/RewriteRule.h   |  43 ++++++-
 clang/lib/Tooling/Transformer/RewriteRule.cpp |  18 +++
 clang/unittests/Tooling/TransformerTest.cpp   | 120 ++++++++++++++++++
 4 files changed, 180 insertions(+), 7 deletions(-)

diff --git a/clang/include/clang/Tooling/Transformer/MatchConsumer.h b/clang/include/clang/Tooling/Transformer/MatchConsumer.h
index f407ffce3d252..cb0a5f684b7d6 100644
--- a/clang/include/clang/Tooling/Transformer/MatchConsumer.h
+++ b/clang/include/clang/Tooling/Transformer/MatchConsumer.h
@@ -99,11 +99,5 @@ llvm::Expected<T> MatchComputation<T>::eval(
   return Output;
 }
 } // namespace transformer
-
-namespace tooling {
-// DEPRECATED: Temporary alias supporting client migration to the `transformer`
-// namespace.
-using transformer::ifBound;
-} // namespace tooling
 } // namespace clang
 #endif // LLVM_CLANG_TOOLING_TRANSFORMER_MATCH_CONSUMER_H_
diff --git a/clang/include/clang/Tooling/Transformer/RewriteRule.h b/clang/include/clang/Tooling/Transformer/RewriteRule.h
index 1be5727364608..c22a2da81fe60 100644
--- a/clang/include/clang/Tooling/Transformer/RewriteRule.h
+++ b/clang/include/clang/Tooling/Transformer/RewriteRule.h
@@ -107,9 +107,42 @@ struct ASTEdit {
 /// clients.  We recommend use of the \c AtomicChange or \c Replacements classes
 /// for assistance in detecting such conflicts.
 EditGenerator editList(llvm::SmallVector<ASTEdit, 1> Edits);
-// Convenience form of `editList` for a single edit.
+/// Convenience form of `editList` for a single edit.
 EditGenerator edit(ASTEdit);
 
+/// Convenience generator for a no-op edit generator.
+inline EditGenerator noEdits() { return editList({}); }
+
+/// Convenience version of `ifBound` specialized to `ASTEdit`.
+inline EditGenerator ifBound(std::string ID, ASTEdit TrueEdit,
+                             ASTEdit FalseEdit) {
+  return ifBound(std::move(ID), edit(std::move(TrueEdit)),
+                 edit(std::move(FalseEdit)));
+}
+
+/// Convenience version of `ifBound` that has no "False" branch. If the node is
+/// not bound, then no edits are produced.
+inline EditGenerator ifBound(std::string ID, ASTEdit TrueEdit) {
+  return ifBound(std::move(ID), edit(std::move(TrueEdit)), noEdits());
+}
+
+/// Flattens a list of generators into a single generator whose elements are the
+/// concatenation of the results of the argument generators.
+EditGenerator flattenVector(SmallVector<EditGenerator, 2> Generators);
+
+namespace detail {
+/// Convenience function to construct an \c EditGenerator. Overloaded for common
+/// cases so that user doesn't need to specify which factory function to
+/// use. This pattern gives benefits similar to implicit constructors, while
+/// maintaing a higher degree of explicitness.
+inline EditGenerator injectEdits(ASTEdit E) { return edit(std::move(E)); }
+inline EditGenerator injectEdits(EditGenerator G) { return G; }
+} // namespace detail
+
+template <typename... Ts> EditGenerator flatten(Ts &&...Edits) {
+  return flattenVector({detail::injectEdits(std::forward<Ts>(Edits))...});
+}
+
 /// Format of the path in an include directive -- angle brackets or quotes.
 enum class IncludeFormat {
   Quoted,
@@ -291,6 +324,14 @@ inline ASTEdit withMetadata(ASTEdit Edit, Callable Metadata) {
   return Edit;
 }
 
+/// Assuming that the inner range is enclosed by the outer range, creates
+/// precision edits to remove the parts of the outer range that are not included
+/// in the inner range.
+inline EditGenerator shrinkTo(RangeSelector outer, RangeSelector inner) {
+  return editList({remove(enclose(before(outer), before(inner))),
+                   remove(enclose(after(inner), after(outer)))});
+}
+
 /// The following three functions are a low-level part of the RewriteRule
 /// API. We expose them for use in implementing the fixtures that interpret
 /// RewriteRule, like Transformer and TransfomerTidy, or for more advanced
diff --git a/clang/lib/Tooling/Transformer/RewriteRule.cpp b/clang/lib/Tooling/Transformer/RewriteRule.cpp
index a212a868c81d8..c145895af7ab6 100644
--- a/clang/lib/Tooling/Transformer/RewriteRule.cpp
+++ b/clang/lib/Tooling/Transformer/RewriteRule.cpp
@@ -68,6 +68,24 @@ EditGenerator transformer::edit(ASTEdit Edit) {
   };
 }
 
+EditGenerator
+transformer::flattenVector(SmallVector<EditGenerator, 2> Generators) {
+  if (Generators.size() == 1)
+    return std::move(Generators[0]);
+  return
+      [Gs = std::move(Generators)](
+          const MatchResult &Result) -> llvm::Expected<SmallVector<Edit, 1>> {
+        SmallVector<Edit, 1> AllEdits;
+        for (const auto &G : Gs) {
+          llvm::Expected<SmallVector<Edit, 1>> Edits = G(Result);
+          if (!Edits)
+            return Edits.takeError();
+          AllEdits.append(Edits->begin(), Edits->end());
+        }
+        return AllEdits;
+      };
+}
+
 ASTEdit transformer::changeTo(RangeSelector Target, TextGenerator Replacement) {
   ASTEdit E;
   E.TargetRange = std::move(Target);
diff --git a/clang/unittests/Tooling/TransformerTest.cpp b/clang/unittests/Tooling/TransformerTest.cpp
index 59b334b0ea5a4..1a68eb1d172a3 100644
--- a/clang/unittests/Tooling/TransformerTest.cpp
+++ b/clang/unittests/Tooling/TransformerTest.cpp
@@ -10,6 +10,7 @@
 #include "clang/ASTMatchers/ASTMatchers.h"
 #include "clang/Tooling/Tooling.h"
 #include "clang/Tooling/Transformer/RangeSelector.h"
+#include "clang/Tooling/Transformer/RewriteRule.h"
 #include "clang/Tooling/Transformer/Stencil.h"
 #include "llvm/Support/Errc.h"
 #include "llvm/Support/Error.h"
@@ -378,6 +379,41 @@ TEST_F(TransformerTest, NodePartMemberMultiToken) {
            Input, Expected);
 }
 
+TEST_F(TransformerTest, NoEdits) {
+  using transformer::noEdits;
+  std::string Input = "int f(int x) { return x; }";
+  testRule(makeRule(returnStmt().bind("return"), noEdits()), Input, Input);
+}
+
+TEST_F(TransformerTest, IfBound2Args) {
+  using transformer::ifBound;
+  std::string Input = "int f(int x) { return x; }";
+  std::string Expected = "int f(int x) { CHANGE; }";
+  testRule(makeRule(returnStmt().bind("return"),
+                    ifBound("return", changeTo(cat("CHANGE;")))),
+           Input, Expected);
+}
+
+TEST_F(TransformerTest, IfBound3Args) {
+  using transformer::ifBound;
+  std::string Input = "int f(int x) { return x; }";
+  std::string Expected = "int f(int x) { CHANGE; }";
+  testRule(makeRule(returnStmt().bind("return"),
+                    ifBound("nothing", changeTo(cat("ERROR")),
+                            changeTo(cat("CHANGE;")))),
+           Input, Expected);
+}
+
+TEST_F(TransformerTest, ShrinkTo) {
+  using transformer::shrinkTo;
+  std::string Input = "int f(int x) { return x; }";
+  std::string Expected = "return x;";
+  testRule(makeRule(functionDecl(hasDescendant(returnStmt().bind("return")))
+                        .bind("function"),
+                    shrinkTo(node("function"), node("return"))),
+           Input, Expected);
+}
+
 TEST_F(TransformerTest, InsertBeforeEdit) {
   std::string Input = R"cc(
     int f() {
@@ -497,6 +533,90 @@ TEST_F(TransformerTest, MultiChange) {
       Input, Expected);
 }
 
+TEST_F(TransformerTest, EditList) {
+  using clang::transformer::editList;
+  std::string Input = R"cc(
+    void foo() {
+      if (10 > 1.0)
+        log(1) << "oh no!";
+      else
+        log(0) << "ok";
+    }
+  )cc";
+  std::string Expected = R"(
+    void foo() {
+      if (true) { /* then */ }
+      else { /* else */ }
+    }
+  )";
+
+  StringRef C = "C", T = "T", E = "E";
+  testRule(makeRule(ifStmt(hasCondition(expr().bind(C)),
+                           hasThen(stmt().bind(T)), hasElse(stmt().bind(E))),
+                    editList({changeTo(node(std::string(C)), cat("true")),
+                              changeTo(statement(std::string(T)),
+                                       cat("{ /* then */ }")),
+                              changeTo(statement(std::string(E)),
+                                       cat("{ /* else */ }"))})),
+           Input, Expected);
+}
+
+TEST_F(TransformerTest, Flatten) {
+  using clang::transformer::editList;
+  std::string Input = R"cc(
+    void foo() {
+      if (10 > 1.0)
+        log(1) << "oh no!";
+      else
+        log(0) << "ok";
+    }
+  )cc";
+  std::string Expected = R"(
+    void foo() {
+      if (true) { /* then */ }
+      else { /* else */ }
+    }
+  )";
+
+  StringRef C = "C", T = "T", E = "E";
+  testRule(
+      makeRule(
+          ifStmt(hasCondition(expr().bind(C)), hasThen(stmt().bind(T)),
+                 hasElse(stmt().bind(E))),
+          flatten(changeTo(node(std::string(C)), cat("true")),
+                  changeTo(statement(std::string(T)), cat("{ /* then */ }")),
+                  changeTo(statement(std::string(E)), cat("{ /* else */ }")))),
+      Input, Expected);
+}
+
+TEST_F(TransformerTest, FlattenWithMixedArgs) {
+  using clang::transformer::editList;
+  std::string Input = R"cc(
+    void foo() {
+      if (10 > 1.0)
+        log(1) << "oh no!";
+      else
+        log(0) << "ok";
+    }
+  )cc";
+  std::string Expected = R"(
+    void foo() {
+      if (true) { /* then */ }
+      else { /* else */ }
+    }
+  )";
+
+  StringRef C = "C", T = "T", E = "E";
+  testRule(makeRule(ifStmt(hasCondition(expr().bind(C)),
+                           hasThen(stmt().bind(T)), hasElse(stmt().bind(E))),
+                    flatten(changeTo(node(std::string(C)), cat("true")),
+                            edit(changeTo(statement(std::string(T)),
+                                          cat("{ /* then */ }"))),
+                            editList({changeTo(statement(std::string(E)),
+                                               cat("{ /* else */ }"))}))),
+           Input, Expected);
+}
+
 TEST_F(TransformerTest, OrderedRuleUnrelated) {
   StringRef Flag = "flag";
   RewriteRule FlagRule = makeRule(

From 783a351785c14b7c2eb9f65bd40d37be11cbf38b Mon Sep 17 00:00:00 2001
From: Frederik Gossen <frgossen@google.com>
Date: Fri, 24 Jul 2020 13:24:23 +0000
Subject: [PATCH 0008/1035] [MLIR][Shape] Allow `shape.mul` to operate in
 indices

Differential Revision: https://reviews.llvm.org/D84437
---
 .../include/mlir/Dialect/Shape/IR/ShapeOps.td | 23 +++++++----
 mlir/lib/Dialect/Shape/IR/Shape.cpp           | 39 ++++++++++++++-----
 .../Shape/Transforms/ShapeToShapeLowering.cpp |  4 +-
 .../ShapeToStandard/shape-to-standard.mlir    | 15 +++++--
 mlir/test/Dialect/Shape/invalid.mlir          | 26 +++++++++++++
 mlir/test/Dialect/Shape/ops.mlir              | 13 +++++--
 6 files changed, 94 insertions(+), 26 deletions(-)

diff --git a/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td b/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td
index 32d6ebafff321..425cf917283be 100644
--- a/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td
+++ b/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td
@@ -307,18 +307,25 @@ def Shape_JoinOp : Shape_Op<"join", [Commutative]> {
   let results = (outs Shape_ShapeOrSizeType:$result);
 }
 
-def Shape_MulOp : Shape_Op<"mul", [Commutative, SameOperandsAndResultType]> {
-  let summary = "Multiplication of sizes";
+def Shape_MulOp : Shape_Op<"mul", [Commutative, NoSideEffect]> {
+  let summary = "Multiplication of sizes and indices";
   let description = [{
-    Multiplies two valid sizes as follows:
-    - lhs * rhs = unknown if either lhs or rhs unknown;
-    - lhs * rhs = (int)lhs * (int)rhs if both known;
+    Multiplies two sizes or indices. If either operand is an error it will be
+    propagated to the result. The operands can be of type `size` or `index`. If
+    at least one of the operands can hold an error, i.e. if it is of type `size`,
+    then also the result must be of type `size`. If error propagation is not
+    possible because both operands are of type `index` then the result must also
+    be of type `index`.
   }];
 
-  let arguments = (ins Shape_SizeType:$lhs, Shape_SizeType:$rhs);
-  let results = (outs Shape_SizeType:$result);
+  let arguments = (ins Shape_SizeOrIndexType:$lhs, Shape_SizeOrIndexType:$rhs);
+  let results = (outs Shape_SizeOrIndexType:$result);
 
-  let assemblyFormat = "$lhs `,` $rhs attr-dict";
+  let assemblyFormat = [{
+    $lhs `,` $rhs `:` type($lhs) `,` type($rhs) `->` type($result) attr-dict
+  }];
+
+  let verifier = [{ return ::verify(*this); }];
 }
 
 def Shape_NumElementsOp : Shape_Op<"num_elements", [NoSideEffect]> {
diff --git a/mlir/lib/Dialect/Shape/IR/Shape.cpp b/mlir/lib/Dialect/Shape/IR/Shape.cpp
index 3bdc5cc39a7bd..2f641300c4917 100644
--- a/mlir/lib/Dialect/Shape/IR/Shape.cpp
+++ b/mlir/lib/Dialect/Shape/IR/Shape.cpp
@@ -28,6 +28,13 @@ static RankedTensorType getExtentTensorType(MLIRContext *ctx) {
   return RankedTensorType::get({ShapedType::kDynamicSize}, IndexType::get(ctx));
 }
 
+static bool isErrorPropagationPossible(ArrayRef<Type> operandTypes) {
+  for (Type ty : operandTypes)
+    if (ty.isa<SizeType>() || ty.isa<ShapeType>() || ty.isa<ValueShapeType>())
+      return true;
+  return false;
+}
+
 ShapeDialect::ShapeDialect(MLIRContext *context)
     : Dialect(getDialectNamespace(), context) {
   addOperations<
@@ -539,9 +546,7 @@ static LogicalResult verify(GetExtentOp op) {
   Type shapeTy = op.shape().getType();
   Type dimTy = op.dim().getType();
   Type extentTy = op.extent().getType();
-  bool errorPropagationPossible =
-      shapeTy.isa<ShapeType>() || dimTy.isa<SizeType>();
-  if (errorPropagationPossible) {
+  if (isErrorPropagationPossible({shapeTy, dimTy})) {
     if (!extentTy.isa<SizeType>())
       op.emitError()
           << "if at least one of the operands can hold error values then the "
@@ -593,9 +598,8 @@ void GetExtentOp::build(OpBuilder &builder, OperationState &result, Value shape,
 //===----------------------------------------------------------------------===//
 
 static LogicalResult verify(shape::RankOp op) {
-  Type argTy = op.shape().getType();
-  Type resultTy = op.rank().getType();
-  if (argTy.isa<ShapeType>() && !resultTy.isa<SizeType>())
+  if (op.shape().getType().isa<ShapeType>() &&
+      !op.rank().getType().isa<SizeType>())
     return op.emitOpError()
            << "if operand is of type `shape` then the result must be of type "
               "`size` to propagate potential errors";
@@ -672,6 +676,25 @@ OpFoldResult NumElementsOp::fold(ArrayRef<Attribute> operands) {
   return builder.getIndexAttr(product.getLimitedValue());
 }
 
+//===----------------------------------------------------------------------===//
+// MulOp
+//===----------------------------------------------------------------------===//
+
+static LogicalResult verify(MulOp op) {
+  Type resultTy = op.result().getType();
+  if (isErrorPropagationPossible({op.lhs().getType(), op.rhs().getType()})) {
+    if (!resultTy.isa<SizeType>())
+      return op.emitOpError()
+             << "if at least one of the operands can hold error values then "
+                "the result must be of type `size` to propagate them";
+  } else {
+    if (resultTy.isa<SizeType>())
+      return op.emitError() << "if none of the operands can hold error values "
+                               "then the result must be of type `index`";
+  }
+  return success();
+}
+
 //===----------------------------------------------------------------------===//
 // ShapeOfOp
 //===----------------------------------------------------------------------===//
@@ -685,15 +708,13 @@ OpFoldResult ShapeOfOp::fold(ArrayRef<Attribute>) {
 }
 
 static LogicalResult verify(ShapeOfOp op) {
-  Type argTy = op.arg().getType();
   Type resultTy = op.result().getType();
-  if (argTy.isa<ValueShapeType>()) {
+  if (isErrorPropagationPossible(op.arg().getType())) {
     if (!resultTy.isa<ShapeType>())
       return op.emitOpError()
              << "if operand is of type `value_shape` then the result must be "
                 "of type `shape` to propagate potential error shapes";
   } else {
-    assert(argTy.isa<ShapedType>());
     if (resultTy != getExtentTensorType(op.getContext()))
       return op.emitOpError() << "if operand is a shaped type then the result "
                                  "must be an extent tensor";
diff --git a/mlir/lib/Dialect/Shape/Transforms/ShapeToShapeLowering.cpp b/mlir/lib/Dialect/Shape/Transforms/ShapeToShapeLowering.cpp
index 467f3d33ce231..bb2b03b8ec081 100644
--- a/mlir/lib/Dialect/Shape/Transforms/ShapeToShapeLowering.cpp
+++ b/mlir/lib/Dialect/Shape/Transforms/ShapeToShapeLowering.cpp
@@ -38,8 +38,8 @@ NumElementsOpConverter::matchAndRewrite(NumElementsOp op,
   // Generate reduce operator.
   Block *body = reduce.getBody();
   OpBuilder b = OpBuilder::atBlockEnd(body);
-  Value product =
-      b.create<MulOp>(loc, body->getArgument(1), body->getArgument(2));
+  Value product = b.create<MulOp>(loc, b.getType<SizeType>(),
+                                  body->getArgument(1), body->getArgument(2));
   b.create<YieldOp>(loc, product);
 
   rewriter.replaceOp(op, reduce.result());
diff --git a/mlir/test/Conversion/ShapeToStandard/shape-to-standard.mlir b/mlir/test/Conversion/ShapeToStandard/shape-to-standard.mlir
index 908acabe5345d..8236c6f279755 100644
--- a/mlir/test/Conversion/ShapeToStandard/shape-to-standard.mlir
+++ b/mlir/test/Conversion/ShapeToStandard/shape-to-standard.mlir
@@ -24,10 +24,19 @@ func @shape_id(%shape : !shape.shape) -> !shape.shape {
 // CHECK-LABEL: @binary_ops
 // CHECK-SAME: (%[[LHS:.*]]: index, %[[RHS:.*]]: index)
 func @binary_ops(%lhs : !shape.size, %rhs : !shape.size) {
+  // CHECK: addi %[[LHS]], %[[RHS]] : index
   %sum = "shape.add"(%lhs, %rhs) : (!shape.size, !shape.size) -> !shape.size
-  // CHECK-NEXT: addi %[[LHS]], %[[RHS]] : index
-  %product = shape.mul %lhs, %rhs
-  // CHECK-NEXT: muli %[[LHS]], %[[RHS]] : index
+  return
+}
+
+// -----
+
+// Lower binary ops.
+// CHECK-LABEL: @binary_ops
+// CHECK-SAME: (%[[LHS:.*]]: index, %[[RHS:.*]]: index)
+func @binary_ops(%lhs : index, %rhs : index) {
+  // CHECK: muli %[[LHS]], %[[RHS]] : index
+  %product = shape.mul %lhs, %rhs : index, index -> index
   return
 }
 
diff --git a/mlir/test/Dialect/Shape/invalid.mlir b/mlir/test/Dialect/Shape/invalid.mlir
index d7e9e40ed3f2e..b4900e491fb82 100644
--- a/mlir/test/Dialect/Shape/invalid.mlir
+++ b/mlir/test/Dialect/Shape/invalid.mlir
@@ -6,6 +6,7 @@ func @reduce_op_args_num_mismatch(%shape : !shape.shape, %init : !shape.size) {
     ^bb0(%index: index, %dim: !shape.size):
       shape.yield %dim : !shape.size
   }
+  return
 }
 
 // -----
@@ -18,6 +19,7 @@ func @reduce_op_arg0_wrong_type(%shape : !shape.shape, %init : !shape.size) {
           : (!shape.size, !shape.size) -> !shape.size
       shape.yield %new_acc : !shape.size
   }
+  return
 }
 
 // -----
@@ -28,6 +30,7 @@ func @reduce_op_arg1_wrong_type(%shape : !shape.shape, %init : !shape.size) {
     ^bb0(%index: index, %dim: f32, %lci: !shape.size):
       shape.yield
   }
+  return
 }
 
 // -----
@@ -38,6 +41,7 @@ func @reduce_op_arg1_wrong_type(%shape : tensor<?xindex>, %init : index) {
     ^bb0(%index: index, %dim: f32, %lci: index):
       shape.yield
   }
+  return
 }
 
 // -----
@@ -48,6 +52,7 @@ func @reduce_op_init_type_mismatch(%shape : !shape.shape, %init : f32) {
     ^bb0(%index: index, %dim: !shape.size, %lci: !shape.size):
       shape.yield
   }
+  return
 }
 
 // -----
@@ -58,6 +63,7 @@ func @yield_op_args_num_mismatch(%shape : !shape.shape, %init : !shape.size) {
     ^bb0(%index: index, %dim: !shape.size, %lci: !shape.size):
       shape.yield %dim, %dim : !shape.size, !shape.size
   }
+  return
 }
 
 // -----
@@ -69,6 +75,7 @@ func @yield_op_type_mismatch(%shape : !shape.shape, %init : !shape.size) {
       %c0 = constant 1 : index
       shape.yield %c0 : index
   }
+  return
 }
 
 // -----
@@ -85,6 +92,7 @@ func @shape_of(%value_arg : !shape.value_shape,
                %shaped_arg : tensor<?x3x4xf32>) {
   // expected-error@+1 {{if operand is of type `value_shape` then the result must be of type `shape` to propagate potential error shapes}}
   %0 = shape.shape_of %value_arg : !shape.value_shape -> tensor<?xindex>
+  return
 }
 
 // -----
@@ -93,6 +101,7 @@ func @shape_of(%value_arg : !shape.value_shape,
                %shaped_arg : tensor<?x3x4xf32>) {
   // expected-error@+1 {{if operand is a shaped type then the result must be an extent tensor}}
   %1 = shape.shape_of %shaped_arg : tensor<?x3x4xf32> -> !shape.shape
+  return
 }
 
 // -----
@@ -100,6 +109,7 @@ func @shape_of(%value_arg : !shape.value_shape,
 func @rank(%arg : !shape.shape) {
   // expected-error@+1 {{if operand is of type `shape` then the result must be of type `size` to propagate potential errors}}
   %0 = shape.rank %arg : !shape.shape -> index
+  return
 }
 
 // -----
@@ -120,3 +130,19 @@ func @get_extent_error_possible(%arg : tensor<?xindex>) -> index {
   return %result : index
 }
 
+// -----
+
+func @mul_error_free(%arg : index) -> !shape.size {
+  // expected-error@+1 {{if none of the operands can hold error values then the result must be of type `index`}}
+  %result = shape.mul %arg, %arg : index, index -> !shape.size
+  return %result : !shape.size
+}
+
+// -----
+
+func @mul_error_possible(%lhs : !shape.size, %rhs : index) -> index {
+  // expected-error@+1 {{if at least one of the operands can hold error values then the result must be of type `size` to propagate them}}
+  %result = shape.mul %lhs, %rhs : !shape.size, index -> index
+  return %result : index
+}
+
diff --git a/mlir/test/Dialect/Shape/ops.mlir b/mlir/test/Dialect/Shape/ops.mlir
index b6b839251a884..3a0cb7781ec72 100644
--- a/mlir/test/Dialect/Shape/ops.mlir
+++ b/mlir/test/Dialect/Shape/ops.mlir
@@ -9,6 +9,7 @@ func @shape_num_elements(%shape : !shape.shape) -> !shape.size {
   %num_elements = shape.reduce(%shape, %init) : !shape.shape -> !shape.size {
     ^bb0(%index : index, %extent : !shape.size, %acc : !shape.size):
       %acc_next = shape.mul %acc, %extent
+          : !shape.size, !shape.size -> !shape.size
       shape.yield %acc_next : !shape.size
   }
   return %num_elements : !shape.size
@@ -19,7 +20,7 @@ func @extent_tensor_num_elements(%shape : tensor<?xindex>) -> index {
   %init = constant 1 : index
   %num_elements = shape.reduce(%shape, %init) : tensor<?xindex> -> index {
     ^bb0(%index : index, %extent : index, %acc : index):
-      %acc_next = muli %acc, %extent : index
+      %acc_next = shape.mul %acc, %extent : index, index -> index
       shape.yield %acc_next : index
   }
   return %num_elements : index
@@ -110,9 +111,13 @@ func @broadcastable_on_extent_tensors(%lhs : tensor<?xindex>,
   return
 }
 
-func @test_mul(%lhs: !shape.size, %rhs: !shape.size) -> !shape.size {
-  %product = shape.mul %lhs, %rhs
-  return %product: !shape.size
+func @mul(%size_arg : !shape.size, %index_arg : index) {
+  %size_prod = shape.mul %size_arg, %size_arg
+      : !shape.size, !shape.size -> !shape.size
+  %index_prod = shape.mul %index_arg, %index_arg : index, index -> index
+  %mixed_prod = shape.mul %size_arg, %index_arg
+      : !shape.size, index -> !shape.size
+  return
 }
 
 func @const_size() {

From 670ae4b6da874270aa0cd8ab32120c17b2eadb95 Mon Sep 17 00:00:00 2001
From: Frederik Gossen <frgossen@google.com>
Date: Fri, 24 Jul 2020 13:29:51 +0000
Subject: [PATCH 0009/1035] [MLIR][Shape] Fold `shape.mul`

Implement constant folding for `shape.mul`.

Differential Revision: https://reviews.llvm.org/D84438
---
 .../include/mlir/Dialect/Shape/IR/ShapeOps.td |  1 +
 mlir/lib/Dialect/Shape/IR/Shape.cpp           | 12 ++++++
 mlir/test/Dialect/Shape/canonicalize.mlir     | 40 +++++++++++++++++++
 3 files changed, 53 insertions(+)

diff --git a/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td b/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td
index 425cf917283be..797dc0bc0cb6a 100644
--- a/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td
+++ b/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td
@@ -326,6 +326,7 @@ def Shape_MulOp : Shape_Op<"mul", [Commutative, NoSideEffect]> {
   }];
 
   let verifier = [{ return ::verify(*this); }];
+  let hasFolder = 1;
 }
 
 def Shape_NumElementsOp : Shape_Op<"num_elements", [NoSideEffect]> {
diff --git a/mlir/lib/Dialect/Shape/IR/Shape.cpp b/mlir/lib/Dialect/Shape/IR/Shape.cpp
index 2f641300c4917..d2b0dbdedb052 100644
--- a/mlir/lib/Dialect/Shape/IR/Shape.cpp
+++ b/mlir/lib/Dialect/Shape/IR/Shape.cpp
@@ -695,6 +695,18 @@ static LogicalResult verify(MulOp op) {
   return success();
 }
 
+OpFoldResult MulOp::fold(ArrayRef<Attribute> operands) {
+  auto lhs = operands[0].dyn_cast_or_null<IntegerAttr>();
+  if (!lhs)
+    return nullptr;
+  auto rhs = operands[1].dyn_cast_or_null<IntegerAttr>();
+  if (!rhs)
+    return nullptr;
+  APInt folded = lhs.getValue() * rhs.getValue();
+  Type indexTy = IndexType::get(getContext());
+  return IntegerAttr::get(indexTy, folded);
+}
+
 //===----------------------------------------------------------------------===//
 // ShapeOfOp
 //===----------------------------------------------------------------------===//
diff --git a/mlir/test/Dialect/Shape/canonicalize.mlir b/mlir/test/Dialect/Shape/canonicalize.mlir
index b4dca5e3c2bf5..577656a0b362a 100644
--- a/mlir/test/Dialect/Shape/canonicalize.mlir
+++ b/mlir/test/Dialect/Shape/canonicalize.mlir
@@ -734,3 +734,43 @@ func @shape_eq_do_not_fold(%a : !shape.shape) -> i1 {
   %result = shape.shape_eq %a, %b : !shape.shape, !shape.shape
   return %result : i1
 }
+
+// -----
+
+// Fold `mul` for constant sizes.
+// CHECK-LABEL: @fold_mul_size
+func @fold_mul_size() -> !shape.size {
+  // CHECK: %[[RESULT:.*]] = shape.const_size 6
+  // CHECK: return %[[RESULT]] : !shape.size
+  %c2 = shape.const_size 2
+  %c3 = shape.const_size 3
+  %result = shape.mul %c2, %c3 : !shape.size, !shape.size -> !shape.size
+  return %result : !shape.size
+}
+
+// -----
+
+// Fold `mul` for constant indices.
+// CHECK-LABEL: @fold_mul_index
+func @fold_mul_index() -> index {
+  // CHECK: %[[RESULT:.*]] = constant 6 : index
+  // CHECK: return %[[RESULT]] : index
+  %c2 = constant 2 : index
+  %c3 = constant 3 : index
+  %result = shape.mul %c2, %c3 : index, index -> index
+  return %result : index
+}
+
+// -----
+
+// Fold `mul` for mixed constants.
+// CHECK-LABEL: @fold_mul_mixed
+func @fold_mul_mixed() -> !shape.size {
+  // CHECK: %[[RESULT:.*]] = shape.const_size 6
+  // CHECK: return %[[RESULT]] : !shape.size
+  %c2 = shape.const_size 2
+  %c3 = constant 3 : index
+  %result = shape.mul %c2, %c3 : !shape.size, index -> !shape.size
+  return %result : !shape.size
+}
+

From 8be0371eb79fb25ba22846410e1a31c88b4c89c2 Mon Sep 17 00:00:00 2001
From: George Mitenkov <georgemitenk0v@gmail.com>
Date: Fri, 24 Jul 2020 15:55:07 +0300
Subject: [PATCH 0010/1035] [MLIR][SPIRVToLLVM] Conversion of load and store
 SPIR-V ops

This patch introduces conversion pattern for `spv.Store` and `spv.Load`.
Only op with `Function` Storage Class is supported at the moment
because `spv.GlobalVariable` has not been introduced yet. If the op
has memory access attribute, then there are the following cases.
If the access is `Aligned`, add alignment to the op builder. Otherwise
the conversion fails as other cases are not supported yet because they
require additional attributes for `llvm.store`/`llvm.load` ops: e.g.
`volatile` and `nontemporal`.

Reviewed By: antiagainst

Differential Revision: https://reviews.llvm.org/D84236
---
 .../SPIRVToLLVM/ConvertSPIRVToLLVM.cpp        | 45 +++++++++++++++++
 .../SPIRVToLLVM/memory-ops-to-llvm.mlir       | 50 +++++++++++++++++++
 2 files changed, 95 insertions(+)

diff --git a/mlir/lib/Conversion/SPIRVToLLVM/ConvertSPIRVToLLVM.cpp b/mlir/lib/Conversion/SPIRVToLLVM/ConvertSPIRVToLLVM.cpp
index cb4898c346a65..12aa6573ef379 100644
--- a/mlir/lib/Conversion/SPIRVToLLVM/ConvertSPIRVToLLVM.cpp
+++ b/mlir/lib/Conversion/SPIRVToLLVM/ConvertSPIRVToLLVM.cpp
@@ -185,6 +185,25 @@ static Value createI32ConstantOf(Location loc, PatternRewriter &rewriter,
       rewriter.getIntegerAttr(rewriter.getI32Type(), value));
 }
 
+/// Utility for `spv.Load` and `spv.Store` conversion.
+static LogicalResult replaceWithLoadOrStore(Operation *op,
+                                            ConversionPatternRewriter &rewriter,
+                                            LLVMTypeConverter &typeConverter,
+                                            unsigned alignment) {
+  if (auto loadOp = dyn_cast<spirv::LoadOp>(op)) {
+    auto dstType = typeConverter.convertType(loadOp.getType());
+    if (!dstType)
+      return failure();
+    rewriter.replaceOpWithNewOp<LLVM::LoadOp>(loadOp, dstType, loadOp.ptr(),
+                                              alignment);
+    return success();
+  }
+  auto storeOp = cast<spirv::StoreOp>(op);
+  rewriter.replaceOpWithNewOp<LLVM::StoreOp>(storeOp, storeOp.value(),
+                                             storeOp.ptr(), alignment);
+  return success();
+}
+
 //===----------------------------------------------------------------------===//
 // Type conversion
 //===----------------------------------------------------------------------===//
@@ -566,6 +585,31 @@ class IComparePattern : public SPIRVToLLVMConversion<SPIRVOp> {
   }
 };
 
+/// Converts `spv.Load` and `spv.Store` to LLVM dialect.
+template <typename SPIRVop>
+class LoadStorePattern : public SPIRVToLLVMConversion<SPIRVop> {
+public:
+  using SPIRVToLLVMConversion<SPIRVop>::SPIRVToLLVMConversion;
+
+  LogicalResult
+  matchAndRewrite(SPIRVop op, ArrayRef<Value> operands,
+                  ConversionPatternRewriter &rewriter) const override {
+    if (op.memory_access().hasValue() &&
+        op.memory_access().getValue() != spirv::MemoryAccess::None) {
+      auto memoryAccess = op.memory_access().getValue();
+      if (memoryAccess == spirv::MemoryAccess::Aligned) {
+        unsigned alignment = op.alignment().getValue().getZExtValue();
+        replaceWithLoadOrStore(op, rewriter, this->typeConverter, alignment);
+        return success();
+      }
+      // There is no support of other memory access attributes.
+      return failure();
+    }
+    replaceWithLoadOrStore(op, rewriter, this->typeConverter, 0);
+    return success();
+  }
+};
+
 /// Converts `spv.Not` and `spv.LogicalNot` into LLVM dialect.
 template <typename SPIRVOp>
 class NotPattern : public SPIRVToLLVMConversion<SPIRVOp> {
@@ -973,6 +1017,7 @@ void mlir::populateSPIRVToLLVMConversionPatterns(
       NotPattern<spirv::LogicalNotOp>,
 
       // Memory ops
+      LoadStorePattern<spirv::LoadOp>, LoadStorePattern<spirv::StoreOp>,
       VariablePattern,
 
       // Miscellaneous ops
diff --git a/mlir/test/Conversion/SPIRVToLLVM/memory-ops-to-llvm.mlir b/mlir/test/Conversion/SPIRVToLLVM/memory-ops-to-llvm.mlir
index 4c549ab8d619c..362c702e8d3c4 100644
--- a/mlir/test/Conversion/SPIRVToLLVM/memory-ops-to-llvm.mlir
+++ b/mlir/test/Conversion/SPIRVToLLVM/memory-ops-to-llvm.mlir
@@ -1,5 +1,55 @@
 // RUN: mlir-opt -convert-spirv-to-llvm %s | FileCheck %s
 
+//===----------------------------------------------------------------------===//
+// spv.Load
+//===----------------------------------------------------------------------===//
+
+func @load() {
+  %0 = spv.Variable : !spv.ptr<f32, Function>
+  //  CHECK: %{{.*}} = llvm.load %{{.*}} : !llvm<"float*">
+  %1 = spv.Load "Function" %0 : f32
+  return
+}
+
+func @load_none() {
+  %0 = spv.Variable : !spv.ptr<f32, Function>
+  //  CHECK: %{{.*}} = llvm.load %{{.*}} : !llvm<"float*">
+  %1 = spv.Load "Function" %0 ["None"] : f32
+  return
+}
+
+func @load_with_alignment() {
+  %0 = spv.Variable : !spv.ptr<f32, Function>
+  // CHECK: %{{.*}} = llvm.load %{{.*}} {alignment = 4 : i64} : !llvm<"float*">
+  %1 = spv.Load "Function" %0 ["Aligned", 4] : f32
+  return
+}
+
+//===----------------------------------------------------------------------===//
+// spv.Store
+//===----------------------------------------------------------------------===//
+
+func @store(%arg0 : f32) -> () {
+  %0 = spv.Variable : !spv.ptr<f32, Function>
+  // CHECK: llvm.store %{{.*}}, %{{.*}} : !llvm<"float*">
+  spv.Store "Function" %0, %arg0 : f32
+  return
+}
+
+func @store_composite(%arg0 : !spv.struct<f64>) -> () {
+  %0 = spv.Variable : !spv.ptr<!spv.struct<f64>, Function>
+  // CHECK: llvm.store %{{.*}}, %{{.*}} : !llvm<"<{ double }>*">
+  spv.Store "Function" %0, %arg0 : !spv.struct<f64>
+  return
+}
+
+func @store_with_alignment(%arg0 : f32) -> () {
+  %0 = spv.Variable : !spv.ptr<f32, Function>
+  // CHECK: llvm.store %{{.*}}, %{{.*}} {alignment = 4 : i64} : !llvm<"float*">
+  spv.Store "Function" %0, %arg0 ["Aligned", 4] : f32
+  return
+}
+
 //===----------------------------------------------------------------------===//
 // spv.Variable
 //===----------------------------------------------------------------------===//

From bb099c87abbfb78b2bdda395ea3f04d2a77c3082 Mon Sep 17 00:00:00 2001
From: David Truby <david.truby@arm.com>
Date: Mon, 20 Jul 2020 12:11:26 +0100
Subject: [PATCH 0011/1035] [openmp] Don't copy exports into the source folder
 by default.

Additionally fix the copy if enabled on multi-config targets.

Summary:
This changes the copy command for libomp.so to use the output of the target as
the source of the copy, rather than trying to find it based on
${LIBOMP_LIBRARY_DIR}, which appears to be incorrect in multi-config generator
builds.

Reviewers: jdoerfert

Subscribers: mgorny, yaxunl, guansong, sstefan1, openmp-commits

Tags: #openmp

Differential Revision: https://reviews.llvm.org/D84148
---
 openmp/runtime/CMakeLists.txt            | 2 +-
 openmp/runtime/cmake/LibompExports.cmake | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/openmp/runtime/CMakeLists.txt b/openmp/runtime/CMakeLists.txt
index 40a0fb9afee1d..e24528e9842fa 100644
--- a/openmp/runtime/CMakeLists.txt
+++ b/openmp/runtime/CMakeLists.txt
@@ -114,7 +114,7 @@ set(LIBOMP_FFLAGS "" CACHE STRING
 # Turning this to FALSE aids parallel builds to not interfere with each other.
 # Currently, the testsuite module expects the just built OpenMP library to be located inside the exports/
 # directory.  TODO: have testsuite run under llvm-lit directly.  We can then get rid of copying to exports/
-set(LIBOMP_COPY_EXPORTS TRUE CACHE STRING
+set(LIBOMP_COPY_EXPORTS FALSE CACHE STRING
   "Should exports be copied into source exports/ directory?")
 
 # HWLOC-support
diff --git a/openmp/runtime/cmake/LibompExports.cmake b/openmp/runtime/cmake/LibompExports.cmake
index f98de2631b831..96dab9f4a4657 100644
--- a/openmp/runtime/cmake/LibompExports.cmake
+++ b/openmp/runtime/cmake/LibompExports.cmake
@@ -78,7 +78,7 @@ if(NOT LIBOMP_OUTPUT_DIRECTORY)
 endif()
 add_custom_command(TARGET omp POST_BUILD
   COMMAND ${CMAKE_COMMAND} -E make_directory ${LIBOMP_EXPORTS_LIB_DIR}
-  COMMAND ${CMAKE_COMMAND} -E copy ${LIBOMP_OUTPUT_DIRECTORY}/${LIBOMP_LIB_FILE} ${LIBOMP_EXPORTS_LIB_DIR}
+  COMMAND ${CMAKE_COMMAND} -E copy $<TARGET_FILE:omp> ${LIBOMP_EXPORTS_LIB_DIR}
 )
 
 # Copy Windows import library into exports/ directory post build

From 6b8948922c59c9f786996387b41d8e29ed3e7a8a Mon Sep 17 00:00:00 2001
From: Dmitry Preobrazhensky <dmitry.preobrazhensky@amd.com>
Date: Fri, 24 Jul 2020 16:39:42 +0300
Subject: [PATCH 0012/1035] [AMDGPU][MC] Added support of SP3 syntax for MTBUF
 format modifier

Currently supported LLVM MTBUF syntax is shown below. It is not compatible with SP3.

    op     dst, addr, rsrc, FORMAT, soffset

This change adds support for SP3 syntax:

    op     dst, addr, rsrc, soffset SP3FORMAT

In addition to being compatible with SP3, this syntax allows using symbolic names for data, numeric and unified formats. Below is a list of added syntax variants.

format:<expression>
format:[<numeric-format-name>,<data-format-name>]
format:[<data-format-name>,<numeric-format-name>]
format:[<data-format-name>]
format:[<numeric-format-name>]
format:[<unified-format-name>]

The last syntax variant is supported for GFX10 only.

See llvm bug 37738

Reviewers: arsenm, rampitec, vpykhtin

Differential Revision: https://reviews.llvm.org/D84026
---
 .../AMDGPU/AsmParser/AMDGPUAsmParser.cpp      | 189 ++++++-
 llvm/lib/Target/AMDGPU/BUFInstructions.td     |   1 +
 .../AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp |  47 +-
 .../AMDGPU/MCTargetDesc/AMDGPUInstPrinter.h   |   2 +
 llvm/lib/Target/AMDGPU/SIDefines.h            | 146 ++++-
 .../Target/AMDGPU/Utils/AMDGPUAsmUtils.cpp    | 245 +++++++++
 llvm/lib/Target/AMDGPU/Utils/AMDGPUAsmUtils.h |  14 +
 .../Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp    |  81 +++
 llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h |  24 +
 .../llvm.amdgcn.raw.tbuffer.load.d16.ll       |  18 +-
 .../AMDGPU/llvm.amdgcn.raw.tbuffer.load.ll    |  48 +-
 .../llvm.amdgcn.raw.tbuffer.store.d16.ll      |  16 +-
 .../AMDGPU/llvm.amdgcn.raw.tbuffer.store.ll   |  36 +-
 .../llvm.amdgcn.struct.tbuffer.load.d16.ll    |  16 +-
 .../AMDGPU/llvm.amdgcn.struct.tbuffer.load.ll |  56 +-
 .../llvm.amdgcn.struct.tbuffer.store.d16.ll   |  16 +-
 .../llvm.amdgcn.struct.tbuffer.store.ll       |  52 +-
 .../AMDGPU/llvm.amdgcn.tbuffer.load.d16.ll    |  10 +-
 .../llvm.amdgcn.tbuffer.load.dwordx3.ll       |  12 +-
 .../AMDGPU/llvm.amdgcn.tbuffer.load.ll        |  28 +-
 .../AMDGPU/llvm.amdgcn.tbuffer.store.d16.ll   |  10 +-
 .../llvm.amdgcn.tbuffer.store.dwordx3.ll      |   6 +-
 .../AMDGPU/llvm.amdgcn.tbuffer.store.ll       |  26 +-
 llvm/test/MC/AMDGPU/buf-fmt-d16-packed.s      |  16 +-
 llvm/test/MC/AMDGPU/buf-fmt-d16-unpacked.s    |  16 +-
 llvm/test/MC/AMDGPU/mtbuf-gfx10.s             | 501 ++++++++++++++++--
 llvm/test/MC/AMDGPU/mtbuf.s                   | 326 ++++++++++--
 .../AMDGPU/buf_fmt_packed_d16.txt             |  16 +-
 .../AMDGPU/buf_fmt_unpacked_d16.txt           |  16 +-
 .../MC/Disassembler/AMDGPU/mtbuf_gfx10.txt    | 106 ++--
 llvm/test/MC/Disassembler/AMDGPU/mtbuf_vi.txt |  81 ++-
 31 files changed, 1823 insertions(+), 354 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
index 9f3a6ffc35e6f..2833875e438cd 100644
--- a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
+++ b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
@@ -689,6 +689,11 @@ class AMDGPUOperand : public MCParsedAsmOperand {
     return Imm.Val;
   }
 
+  void setImm(int64_t Val) {
+    assert(isImm());
+    Imm.Val = Val;
+  }
+
   ImmTy getImmTy() const {
     assert(isImm());
     return Imm.Type;
@@ -1297,8 +1302,13 @@ class AMDGPUAsmParser : public MCTargetAsmParser {
   OperandMatchResultTy parseVReg32OrOff(OperandVector &Operands);
   OperandMatchResultTy parseDfmtNfmt(int64_t &Format);
   OperandMatchResultTy parseUfmt(int64_t &Format);
+  OperandMatchResultTy parseSymbolicSplitFormat(StringRef FormatStr, SMLoc Loc, int64_t &Format);
+  OperandMatchResultTy parseSymbolicUnifiedFormat(StringRef FormatStr, SMLoc Loc, int64_t &Format);
   OperandMatchResultTy parseFORMAT(OperandVector &Operands);
+  OperandMatchResultTy parseSymbolicOrNumericFormat(int64_t &Format);
+  OperandMatchResultTy parseNumericFormat(int64_t &Format);
   bool tryParseFmt(const char *Pref, int64_t MaxVal, int64_t &Val);
+  bool matchDfmtNfmt(int64_t &Dfmt, int64_t &Nfmt, StringRef FormatStr, SMLoc Loc);
 
   void cvtDSOffset01(MCInst &Inst, const OperandVector &Operands);
   void cvtDS(MCInst &Inst, const OperandVector &Operands) { cvtDSImpl(Inst, Operands, false); }
@@ -1367,6 +1377,8 @@ class AMDGPUAsmParser : public MCTargetAsmParser {
   bool trySkipToken(const AsmToken::TokenKind Kind);
   bool skipToken(const AsmToken::TokenKind Kind, const StringRef ErrMsg);
   bool parseString(StringRef &Val, const StringRef ErrMsg = "expected a string");
+  bool parseId(StringRef &Val, const StringRef ErrMsg);
+
   void peekTokens(MutableArrayRef<AsmToken> Tokens);
   AsmToken::TokenKind getTokenKind() const;
   bool parseExpr(int64_t &Imm);
@@ -4926,8 +4938,8 @@ AMDGPUAsmParser::parseDfmtNfmt(int64_t &Format) {
   if (Dfmt == DFMT_UNDEF && Nfmt == NFMT_UNDEF)
     return MatchOperand_NoMatch;
 
-  Dfmt = (Dfmt == DFMT_UNDEF)? DFMT_DEFAULT : Dfmt;
-  Nfmt = (Nfmt == NFMT_UNDEF)? NFMT_DEFAULT : Nfmt;
+  Dfmt = (Dfmt == DFMT_UNDEF) ? DFMT_DEFAULT : Dfmt;
+  Nfmt = (Nfmt == NFMT_UNDEF) ? NFMT_DEFAULT : Nfmt;
 
   Format = encodeDfmtNfmt(Dfmt, Nfmt);
   return MatchOperand_Success;
@@ -4949,20 +4961,177 @@ AMDGPUAsmParser::parseUfmt(int64_t &Format) {
   return MatchOperand_Success;
 }
 
+bool AMDGPUAsmParser::matchDfmtNfmt(int64_t &Dfmt,
+                                    int64_t &Nfmt,
+                                    StringRef FormatStr,
+                                    SMLoc Loc) {
+  using namespace llvm::AMDGPU::MTBUFFormat;
+  int64_t Format;
+
+  Format = getDfmt(FormatStr);
+  if (Format != DFMT_UNDEF) {
+    Dfmt = Format;
+    return true;
+  }
+
+  Format = getNfmt(FormatStr, getSTI());
+  if (Format != NFMT_UNDEF) {
+    Nfmt = Format;
+    return true;
+  }
+
+  Error(Loc, "unsupported format");
+  return false;
+}
+
+OperandMatchResultTy
+AMDGPUAsmParser::parseSymbolicSplitFormat(StringRef FormatStr,
+                                          SMLoc FormatLoc,
+                                          int64_t &Format) {
+  using namespace llvm::AMDGPU::MTBUFFormat;
+
+  int64_t Dfmt = DFMT_UNDEF;
+  int64_t Nfmt = NFMT_UNDEF;
+  if (!matchDfmtNfmt(Dfmt, Nfmt, FormatStr, FormatLoc))
+    return MatchOperand_ParseFail;
+
+  if (trySkipToken(AsmToken::Comma)) {
+    StringRef Str;
+    SMLoc Loc = getLoc();
+    if (!parseId(Str, "expected a format string") ||
+        !matchDfmtNfmt(Dfmt, Nfmt, Str, Loc)) {
+      return MatchOperand_ParseFail;
+    }
+    if (Dfmt == DFMT_UNDEF) {
+      Error(Loc, "duplicate numeric format");
+    } else if (Nfmt == NFMT_UNDEF){
+      Error(Loc, "duplicate data format");
+    }
+  }
+
+  Dfmt = (Dfmt == DFMT_UNDEF) ? DFMT_DEFAULT : Dfmt;
+  Nfmt = (Nfmt == NFMT_UNDEF) ? NFMT_DEFAULT : Nfmt;
+
+  if (isGFX10()) {
+    auto Ufmt = convertDfmtNfmt2Ufmt(Dfmt, Nfmt);
+    if (Ufmt == UFMT_UNDEF)
+      Error(FormatLoc, "unsupported format");
+    Format = Ufmt;
+  } else {
+    Format = encodeDfmtNfmt(Dfmt, Nfmt);
+  }
+
+  return MatchOperand_Success;
+}
+
+OperandMatchResultTy
+AMDGPUAsmParser::parseSymbolicUnifiedFormat(StringRef FormatStr,
+                                            SMLoc Loc,
+                                            int64_t &Format) {
+  using namespace llvm::AMDGPU::MTBUFFormat;
+
+  auto Id = getUnifiedFormat(FormatStr);
+  if (Id == UFMT_UNDEF)
+    return MatchOperand_NoMatch;
+
+  if (!isGFX10()) {
+    Error(Loc, "unified format is not supported on this GPU");
+    return MatchOperand_ParseFail;
+  }
+
+  Format = Id;
+  return MatchOperand_Success;
+}
+
+OperandMatchResultTy
+AMDGPUAsmParser::parseNumericFormat(int64_t &Format) {
+  using namespace llvm::AMDGPU::MTBUFFormat;
+  SMLoc Loc = getLoc();
+
+  if (!parseExpr(Format))
+    return MatchOperand_ParseFail;
+  if (!isValidFormatEncoding(Format, getSTI())) {
+    Error(Loc, "out of range format");
+    return MatchOperand_ParseFail;
+  }
+
+  return MatchOperand_Success;
+}
+
+OperandMatchResultTy
+AMDGPUAsmParser::parseSymbolicOrNumericFormat(int64_t &Format) {
+  using namespace llvm::AMDGPU::MTBUFFormat;
+
+  if (!trySkipId("format", AsmToken::Colon))
+    return MatchOperand_NoMatch;
+
+  if (trySkipToken(AsmToken::LBrac)) {
+    StringRef FormatStr;
+    SMLoc Loc = getLoc();
+    if (!parseId(FormatStr, "expected a format string"))
+      return MatchOperand_ParseFail;
+
+    auto Res = parseSymbolicUnifiedFormat(FormatStr, Loc, Format);
+    if (Res == MatchOperand_NoMatch)
+      Res = parseSymbolicSplitFormat(FormatStr, Loc, Format);
+    if (Res != MatchOperand_Success)
+      return Res;
+
+    skipToken(AsmToken::RBrac, "expected a closing square bracket");
+    return MatchOperand_Success;
+  }
+
+  return parseNumericFormat(Format);
+}
+
 OperandMatchResultTy
 AMDGPUAsmParser::parseFORMAT(OperandVector &Operands) {
   using namespace llvm::AMDGPU::MTBUFFormat;
 
-  int64_t Format = isGFX10() ? UFMT_DEFAULT : DFMT_NFMT_DEFAULT;
+  int64_t Format = getDefaultFormatEncoding(getSTI());
   OperandMatchResultTy Res;
   SMLoc Loc = getLoc();
 
+  // Parse legacy format syntax.
   Res = isGFX10() ? parseUfmt(Format) : parseDfmtNfmt(Format);
   if (Res == MatchOperand_ParseFail)
     return Res;
 
+  bool FormatFound = (Res == MatchOperand_Success);
+
   Operands.push_back(
     AMDGPUOperand::CreateImm(this, Format, Loc, AMDGPUOperand::ImmTyFORMAT));
+
+  if (FormatFound)
+    trySkipToken(AsmToken::Comma);
+
+  if (isToken(AsmToken::EndOfStatement)) {
+    // We are expecting an soffset operand,
+    // but let matcher handle the error.
+    return MatchOperand_Success;
+  }
+
+  // Parse soffset.
+  Res = parseRegOrImm(Operands);
+  if (Res != MatchOperand_Success)
+    return Res;
+
+  trySkipToken(AsmToken::Comma);
+
+  if (!FormatFound) {
+    if (parseSymbolicOrNumericFormat(Format) == MatchOperand_Success) {
+      auto Size = Operands.size();
+      AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands[Size - 2]);
+      assert(Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyFORMAT);
+      Op.setImm(Format);
+    }
+    return MatchOperand_Success;
+  }
+
+  if (isId("format") && peekToken().is(AsmToken::Colon)) {
+    Error(getLoc(), "duplicate format");
+    return MatchOperand_ParseFail;
+  }
   return MatchOperand_Success;
 }
 
@@ -5616,6 +5785,18 @@ AMDGPUAsmParser::parseString(StringRef &Val, const StringRef ErrMsg) {
   }
 }
 
+bool
+AMDGPUAsmParser::parseId(StringRef &Val, const StringRef ErrMsg) {
+  if (isToken(AsmToken::Identifier)) {
+    Val = getTokenStr();
+    lex();
+    return true;
+  } else {
+    Error(getLoc(), ErrMsg);
+    return false;
+  }
+}
+
 AsmToken
 AMDGPUAsmParser::getToken() const {
   return Parser.getTok();
@@ -5623,7 +5804,7 @@ AMDGPUAsmParser::getToken() const {
 
 AsmToken
 AMDGPUAsmParser::peekToken() {
-  return getLexer().peekTok();
+  return isToken(AsmToken::EndOfStatement) ? getToken() : getLexer().peekTok();
 }
 
 void
diff --git a/llvm/lib/Target/AMDGPU/BUFInstructions.td b/llvm/lib/Target/AMDGPU/BUFInstructions.td
index 370e9db9e83e9..d175edd93c642 100644
--- a/llvm/lib/Target/AMDGPU/BUFInstructions.td
+++ b/llvm/lib/Target/AMDGPU/BUFInstructions.td
@@ -114,6 +114,7 @@ class MTBUF_Real <MTBUF_Pseudo ps> :
   let isCodeGenOnly = 0;
 
   // copy relevant pseudo op flags
+  let UseNamedOperandTable = ps.UseNamedOperandTable;
   let SubtargetPredicate = ps.SubtargetPredicate;
   let AsmMatchConverter  = ps.AsmMatchConverter;
   let Constraints        = ps.Constraints;
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp
index e44c0194e811b..7a54f49c18b08 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp
@@ -299,23 +299,48 @@ void AMDGPUInstPrinter::printExpVM(const MCInst *MI, unsigned OpNo,
 void AMDGPUInstPrinter::printFORMAT(const MCInst *MI, unsigned OpNo,
                                     const MCSubtargetInfo &STI,
                                     raw_ostream &O) {
+}
+
+void AMDGPUInstPrinter::printSymbolicFormat(const MCInst *MI,
+                                            const MCSubtargetInfo &STI,
+                                            raw_ostream &O) {
   using namespace llvm::AMDGPU::MTBUFFormat;
 
+  int OpNo =
+    AMDGPU::getNamedOperandIdx(MI->getOpcode(), AMDGPU::OpName::format);
+  assert(OpNo != -1);
+
   unsigned Val = MI->getOperand(OpNo).getImm();
   if (AMDGPU::isGFX10(STI)) {
     if (Val == UFMT_DEFAULT)
       return;
-    O << " format:" << Val;
+    if (isValidUnifiedFormat(Val)) {
+      O << " format:[" << getUnifiedFormatName(Val) << ']';
+    } else {
+      O << " format:" << Val;
+    }
   } else {
     if (Val == DFMT_NFMT_DEFAULT)
       return;
-    unsigned Dfmt;
-    unsigned Nfmt;
-    decodeDfmtNfmt(Val, Dfmt, Nfmt);
-    O << " dfmt:" << Dfmt;
-    O << ", nfmt:" << Nfmt;
+    if (isValidDfmtNfmt(Val, STI)) {
+      unsigned Dfmt;
+      unsigned Nfmt;
+      decodeDfmtNfmt(Val, Dfmt, Nfmt);
+      O << " format:[";
+      if (Dfmt != DFMT_DEFAULT) {
+        O << getDfmtName(Dfmt);
+        if (Nfmt != NFMT_DEFAULT) {
+          O << ',';
+        }
+      }
+      if (Nfmt != NFMT_DEFAULT) {
+        O << getNfmtName(Nfmt, STI);
+      }
+      O << ']';
+    } else {
+      O << " format:" << Val;
+    }
   }
-  O << ',';
 }
 
 void AMDGPUInstPrinter::printRegOperand(unsigned RegNo, raw_ostream &O,
@@ -682,6 +707,14 @@ void AMDGPUInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
       printDefaultVccOperand(OpNo, STI, O);
     break;
   }
+
+  if (Desc.TSFlags & SIInstrFlags::MTBUF) {
+    int SOffsetIdx =
+      AMDGPU::getNamedOperandIdx(MI->getOpcode(), AMDGPU::OpName::soffset);
+    assert(SOffsetIdx != -1);
+    if ((int)OpNo == SOffsetIdx)
+      printSymbolicFormat(MI, STI, O);
+  }
 }
 
 void AMDGPUInstPrinter::printOperandAndFPInputMods(const MCInst *MI,
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.h b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.h
index 78a66a78ec341..ed45c5309ea25 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.h
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.h
@@ -99,6 +99,8 @@ class AMDGPUInstPrinter : public MCInstPrinter {
                   const MCSubtargetInfo &STI, raw_ostream &O);
   void printFORMAT(const MCInst *MI, unsigned OpNo,
                    const MCSubtargetInfo &STI, raw_ostream &O);
+  void printSymbolicFormat(const MCInst *MI,
+                           const MCSubtargetInfo &STI, raw_ostream &O);
 
   void printRegOperand(unsigned RegNo, raw_ostream &O);
   void printVOPDst(const MCInst *MI, unsigned OpNo, const MCSubtargetInfo &STI,
diff --git a/llvm/lib/Target/AMDGPU/SIDefines.h b/llvm/lib/Target/AMDGPU/SIDefines.h
index 9c9dd66a4a79c..58d77f3b224b5 100644
--- a/llvm/lib/Target/AMDGPU/SIDefines.h
+++ b/llvm/lib/Target/AMDGPU/SIDefines.h
@@ -394,27 +394,56 @@ enum ModeRegisterMasks : uint32_t {
 
 namespace MTBUFFormat {
 
-enum DataFormat {
-  DFMT_MAX = 15,
+enum DataFormat : int64_t {
+  DFMT_INVALID = 0,
+  DFMT_8,
+  DFMT_16,
+  DFMT_8_8,
+  DFMT_32,
+  DFMT_16_16,
+  DFMT_10_11_11,
+  DFMT_11_11_10,
+  DFMT_10_10_10_2,
+  DFMT_2_10_10_10,
+  DFMT_8_8_8_8,
+  DFMT_32_32,
+  DFMT_16_16_16_16,
+  DFMT_32_32_32,
+  DFMT_32_32_32_32,
+  DFMT_RESERVED_15,
+
+  DFMT_MIN = DFMT_INVALID,
+  DFMT_MAX = DFMT_RESERVED_15,
 
   DFMT_UNDEF = -1,
-  DFMT_DEFAULT = 1,
+  DFMT_DEFAULT = DFMT_8,
 
   DFMT_SHIFT = 0,
-  DFMT_MASK = DFMT_MAX
+  DFMT_MASK = 0xF
 };
 
-enum NumFormat {
-  NFMT_MAX = 7,
+enum NumFormat : int64_t {
+  NFMT_UNORM = 0,
+  NFMT_SNORM,
+  NFMT_USCALED,
+  NFMT_SSCALED,
+  NFMT_UINT,
+  NFMT_SINT,
+  NFMT_RESERVED_6,                    // VI and GFX9
+  NFMT_SNORM_OGL = NFMT_RESERVED_6,   // SI and CI only
+  NFMT_FLOAT,
+
+  NFMT_MIN = NFMT_UNORM,
+  NFMT_MAX = NFMT_FLOAT,
 
   NFMT_UNDEF = -1,
-  NFMT_DEFAULT = 0,
+  NFMT_DEFAULT = NFMT_UNORM,
 
   NFMT_SHIFT = 4,
-  NFMT_MASK = NFMT_MAX
+  NFMT_MASK = 7
 };
 
-enum MergedFormat {
+enum MergedFormat : int64_t {
   DFMT_NFMT_UNDEF = -1,
   DFMT_NFMT_DEFAULT = ((DFMT_DEFAULT & DFMT_MASK) << DFMT_SHIFT) |
                       ((NFMT_DEFAULT & NFMT_MASK) << NFMT_SHIFT),
@@ -425,11 +454,106 @@ enum MergedFormat {
   DFMT_NFMT_MAX = DFMT_NFMT_MASK
 };
 
-enum UnifiedFormat {
+enum UnifiedFormat : int64_t {
+  UFMT_INVALID = 0,
+
+  UFMT_8_UNORM,
+  UFMT_8_SNORM,
+  UFMT_8_USCALED,
+  UFMT_8_SSCALED,
+  UFMT_8_UINT,
+  UFMT_8_SINT,
+
+  UFMT_16_UNORM,
+  UFMT_16_SNORM,
+  UFMT_16_USCALED,
+  UFMT_16_SSCALED,
+  UFMT_16_UINT,
+  UFMT_16_SINT,
+  UFMT_16_FLOAT,
+
+  UFMT_8_8_UNORM,
+  UFMT_8_8_SNORM,
+  UFMT_8_8_USCALED,
+  UFMT_8_8_SSCALED,
+  UFMT_8_8_UINT,
+  UFMT_8_8_SINT,
+
+  UFMT_32_UINT,
+  UFMT_32_SINT,
+  UFMT_32_FLOAT,
+
+  UFMT_16_16_UNORM,
+  UFMT_16_16_SNORM,
+  UFMT_16_16_USCALED,
+  UFMT_16_16_SSCALED,
+  UFMT_16_16_UINT,
+  UFMT_16_16_SINT,
+  UFMT_16_16_FLOAT,
+
+  UFMT_10_11_11_UNORM,
+  UFMT_10_11_11_SNORM,
+  UFMT_10_11_11_USCALED,
+  UFMT_10_11_11_SSCALED,
+  UFMT_10_11_11_UINT,
+  UFMT_10_11_11_SINT,
+  UFMT_10_11_11_FLOAT,
+
+  UFMT_11_11_10_UNORM,
+  UFMT_11_11_10_SNORM,
+  UFMT_11_11_10_USCALED,
+  UFMT_11_11_10_SSCALED,
+  UFMT_11_11_10_UINT,
+  UFMT_11_11_10_SINT,
+  UFMT_11_11_10_FLOAT,
+
+  UFMT_10_10_10_2_UNORM,
+  UFMT_10_10_10_2_SNORM,
+  UFMT_10_10_10_2_USCALED,
+  UFMT_10_10_10_2_SSCALED,
+  UFMT_10_10_10_2_UINT,
+  UFMT_10_10_10_2_SINT,
+
+  UFMT_2_10_10_10_UNORM,
+  UFMT_2_10_10_10_SNORM,
+  UFMT_2_10_10_10_USCALED,
+  UFMT_2_10_10_10_SSCALED,
+  UFMT_2_10_10_10_UINT,
+  UFMT_2_10_10_10_SINT,
+
+  UFMT_8_8_8_8_UNORM,
+  UFMT_8_8_8_8_SNORM,
+  UFMT_8_8_8_8_USCALED,
+  UFMT_8_8_8_8_SSCALED,
+  UFMT_8_8_8_8_UINT,
+  UFMT_8_8_8_8_SINT,
+
+  UFMT_32_32_UINT,
+  UFMT_32_32_SINT,
+  UFMT_32_32_FLOAT,
+
+  UFMT_16_16_16_16_UNORM,
+  UFMT_16_16_16_16_SNORM,
+  UFMT_16_16_16_16_USCALED,
+  UFMT_16_16_16_16_SSCALED,
+  UFMT_16_16_16_16_UINT,
+  UFMT_16_16_16_16_SINT,
+  UFMT_16_16_16_16_FLOAT,
+
+  UFMT_32_32_32_UINT,
+  UFMT_32_32_32_SINT,
+  UFMT_32_32_32_FLOAT,
+  UFMT_32_32_32_32_UINT,
+  UFMT_32_32_32_32_SINT,
+  UFMT_32_32_32_32_FLOAT,
+
+  UFMT_FIRST = UFMT_INVALID,
+  UFMT_LAST = UFMT_32_32_32_32_FLOAT,
+
   UFMT_MAX = 127,
 
   UFMT_UNDEF = -1,
-  UFMT_DEFAULT = 1
+  UFMT_DEFAULT = UFMT_8_UNORM
 };
 
 } // namespace MTBUFFormat
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUAsmUtils.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUAsmUtils.cpp
index 5819a621f55d6..1d645bda4b3fa 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUAsmUtils.cpp
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUAsmUtils.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 #include "AMDGPUAsmUtils.h"
+#include "SIDefines.h"
 
 namespace llvm {
 namespace AMDGPU {
@@ -87,6 +88,250 @@ const char* const IdSymbolic[] = {
 
 } // namespace Hwreg
 
+namespace MTBUFFormat {
+
+StringLiteral const DfmtSymbolic[] = {
+  "BUF_DATA_FORMAT_INVALID",
+  "BUF_DATA_FORMAT_8",
+  "BUF_DATA_FORMAT_16",
+  "BUF_DATA_FORMAT_8_8",
+  "BUF_DATA_FORMAT_32",
+  "BUF_DATA_FORMAT_16_16",
+  "BUF_DATA_FORMAT_10_11_11",
+  "BUF_DATA_FORMAT_11_11_10",
+  "BUF_DATA_FORMAT_10_10_10_2",
+  "BUF_DATA_FORMAT_2_10_10_10",
+  "BUF_DATA_FORMAT_8_8_8_8",
+  "BUF_DATA_FORMAT_32_32",
+  "BUF_DATA_FORMAT_16_16_16_16",
+  "BUF_DATA_FORMAT_32_32_32",
+  "BUF_DATA_FORMAT_32_32_32_32",
+  "BUF_DATA_FORMAT_RESERVED_15"
+};
+
+StringLiteral const NfmtSymbolicGFX10[] = {
+  "BUF_NUM_FORMAT_UNORM",
+  "BUF_NUM_FORMAT_SNORM",
+  "BUF_NUM_FORMAT_USCALED",
+  "BUF_NUM_FORMAT_SSCALED",
+  "BUF_NUM_FORMAT_UINT",
+  "BUF_NUM_FORMAT_SINT",
+  "",
+  "BUF_NUM_FORMAT_FLOAT"
+};
+
+StringLiteral const NfmtSymbolicSICI[] = {
+  "BUF_NUM_FORMAT_UNORM",
+  "BUF_NUM_FORMAT_SNORM",
+  "BUF_NUM_FORMAT_USCALED",
+  "BUF_NUM_FORMAT_SSCALED",
+  "BUF_NUM_FORMAT_UINT",
+  "BUF_NUM_FORMAT_SINT",
+  "BUF_NUM_FORMAT_SNORM_OGL",
+  "BUF_NUM_FORMAT_FLOAT"
+};
+
+StringLiteral const NfmtSymbolicVI[] = {    // VI and GFX9
+  "BUF_NUM_FORMAT_UNORM",
+  "BUF_NUM_FORMAT_SNORM",
+  "BUF_NUM_FORMAT_USCALED",
+  "BUF_NUM_FORMAT_SSCALED",
+  "BUF_NUM_FORMAT_UINT",
+  "BUF_NUM_FORMAT_SINT",
+  "BUF_NUM_FORMAT_RESERVED_6",
+  "BUF_NUM_FORMAT_FLOAT"
+};
+
+StringLiteral const UfmtSymbolic[] = {
+  "BUF_FMT_INVALID",
+
+  "BUF_FMT_8_UNORM",
+  "BUF_FMT_8_SNORM",
+  "BUF_FMT_8_USCALED",
+  "BUF_FMT_8_SSCALED",
+  "BUF_FMT_8_UINT",
+  "BUF_FMT_8_SINT",
+
+  "BUF_FMT_16_UNORM",
+  "BUF_FMT_16_SNORM",
+  "BUF_FMT_16_USCALED",
+  "BUF_FMT_16_SSCALED",
+  "BUF_FMT_16_UINT",
+  "BUF_FMT_16_SINT",
+  "BUF_FMT_16_FLOAT",
+
+  "BUF_FMT_8_8_UNORM",
+  "BUF_FMT_8_8_SNORM",
+  "BUF_FMT_8_8_USCALED",
+  "BUF_FMT_8_8_SSCALED",
+  "BUF_FMT_8_8_UINT",
+  "BUF_FMT_8_8_SINT",
+
+  "BUF_FMT_32_UINT",
+  "BUF_FMT_32_SINT",
+  "BUF_FMT_32_FLOAT",
+
+  "BUF_FMT_16_16_UNORM",
+  "BUF_FMT_16_16_SNORM",
+  "BUF_FMT_16_16_USCALED",
+  "BUF_FMT_16_16_SSCALED",
+  "BUF_FMT_16_16_UINT",
+  "BUF_FMT_16_16_SINT",
+  "BUF_FMT_16_16_FLOAT",
+
+  "BUF_FMT_10_11_11_UNORM",
+  "BUF_FMT_10_11_11_SNORM",
+  "BUF_FMT_10_11_11_USCALED",
+  "BUF_FMT_10_11_11_SSCALED",
+  "BUF_FMT_10_11_11_UINT",
+  "BUF_FMT_10_11_11_SINT",
+  "BUF_FMT_10_11_11_FLOAT",
+
+  "BUF_FMT_11_11_10_UNORM",
+  "BUF_FMT_11_11_10_SNORM",
+  "BUF_FMT_11_11_10_USCALED",
+  "BUF_FMT_11_11_10_SSCALED",
+  "BUF_FMT_11_11_10_UINT",
+  "BUF_FMT_11_11_10_SINT",
+  "BUF_FMT_11_11_10_FLOAT",
+
+  "BUF_FMT_10_10_10_2_UNORM",
+  "BUF_FMT_10_10_10_2_SNORM",
+  "BUF_FMT_10_10_10_2_USCALED",
+  "BUF_FMT_10_10_10_2_SSCALED",
+  "BUF_FMT_10_10_10_2_UINT",
+  "BUF_FMT_10_10_10_2_SINT",
+
+  "BUF_FMT_2_10_10_10_UNORM",
+  "BUF_FMT_2_10_10_10_SNORM",
+  "BUF_FMT_2_10_10_10_USCALED",
+  "BUF_FMT_2_10_10_10_SSCALED",
+  "BUF_FMT_2_10_10_10_UINT",
+  "BUF_FMT_2_10_10_10_SINT",
+
+  "BUF_FMT_8_8_8_8_UNORM",
+  "BUF_FMT_8_8_8_8_SNORM",
+  "BUF_FMT_8_8_8_8_USCALED",
+  "BUF_FMT_8_8_8_8_SSCALED",
+  "BUF_FMT_8_8_8_8_UINT",
+  "BUF_FMT_8_8_8_8_SINT",
+
+  "BUF_FMT_32_32_UINT",
+  "BUF_FMT_32_32_SINT",
+  "BUF_FMT_32_32_FLOAT",
+
+  "BUF_FMT_16_16_16_16_UNORM",
+  "BUF_FMT_16_16_16_16_SNORM",
+  "BUF_FMT_16_16_16_16_USCALED",
+  "BUF_FMT_16_16_16_16_SSCALED",
+  "BUF_FMT_16_16_16_16_UINT",
+  "BUF_FMT_16_16_16_16_SINT",
+  "BUF_FMT_16_16_16_16_FLOAT",
+
+  "BUF_FMT_32_32_32_UINT",
+  "BUF_FMT_32_32_32_SINT",
+  "BUF_FMT_32_32_32_FLOAT",
+  "BUF_FMT_32_32_32_32_UINT",
+  "BUF_FMT_32_32_32_32_SINT",
+  "BUF_FMT_32_32_32_32_FLOAT"
+};
+
+unsigned const DfmtNfmt2UFmt[] = {
+  DFMT_INVALID     | (NFMT_UNORM   << NFMT_SHIFT),
+
+  DFMT_8           | (NFMT_UNORM   << NFMT_SHIFT),
+  DFMT_8           | (NFMT_SNORM   << NFMT_SHIFT),
+  DFMT_8           | (NFMT_USCALED << NFMT_SHIFT),
+  DFMT_8           | (NFMT_SSCALED << NFMT_SHIFT),
+  DFMT_8           | (NFMT_UINT    << NFMT_SHIFT),
+  DFMT_8           | (NFMT_SINT    << NFMT_SHIFT),
+
+  DFMT_16          | (NFMT_UNORM   << NFMT_SHIFT),
+  DFMT_16          | (NFMT_SNORM   << NFMT_SHIFT),
+  DFMT_16          | (NFMT_USCALED << NFMT_SHIFT),
+  DFMT_16          | (NFMT_SSCALED << NFMT_SHIFT),
+  DFMT_16          | (NFMT_UINT    << NFMT_SHIFT),
+  DFMT_16          | (NFMT_SINT    << NFMT_SHIFT),
+  DFMT_16          | (NFMT_FLOAT   << NFMT_SHIFT),
+
+  DFMT_8_8         | (NFMT_UNORM   << NFMT_SHIFT),
+  DFMT_8_8         | (NFMT_SNORM   << NFMT_SHIFT),
+  DFMT_8_8         | (NFMT_USCALED << NFMT_SHIFT),
+  DFMT_8_8         | (NFMT_SSCALED << NFMT_SHIFT),
+  DFMT_8_8         | (NFMT_UINT    << NFMT_SHIFT),
+  DFMT_8_8         | (NFMT_SINT    << NFMT_SHIFT),
+
+  DFMT_32          | (NFMT_UINT    << NFMT_SHIFT),
+  DFMT_32          | (NFMT_SINT    << NFMT_SHIFT),
+  DFMT_32          | (NFMT_FLOAT   << NFMT_SHIFT),
+
+  DFMT_16_16       | (NFMT_UNORM   << NFMT_SHIFT),
+  DFMT_16_16       | (NFMT_SNORM   << NFMT_SHIFT),
+  DFMT_16_16       | (NFMT_USCALED << NFMT_SHIFT),
+  DFMT_16_16       | (NFMT_SSCALED << NFMT_SHIFT),
+  DFMT_16_16       | (NFMT_UINT    << NFMT_SHIFT),
+  DFMT_16_16       | (NFMT_SINT    << NFMT_SHIFT),
+  DFMT_16_16       | (NFMT_FLOAT   << NFMT_SHIFT),
+
+  DFMT_10_11_11    | (NFMT_UNORM   << NFMT_SHIFT),
+  DFMT_10_11_11    | (NFMT_SNORM   << NFMT_SHIFT),
+  DFMT_10_11_11    | (NFMT_USCALED << NFMT_SHIFT),
+  DFMT_10_11_11    | (NFMT_SSCALED << NFMT_SHIFT),
+  DFMT_10_11_11    | (NFMT_UINT    << NFMT_SHIFT),
+  DFMT_10_11_11    | (NFMT_SINT    << NFMT_SHIFT),
+  DFMT_10_11_11    | (NFMT_FLOAT   << NFMT_SHIFT),
+
+  DFMT_11_11_10    | (NFMT_UNORM   << NFMT_SHIFT),
+  DFMT_11_11_10    | (NFMT_SNORM   << NFMT_SHIFT),
+  DFMT_11_11_10    | (NFMT_USCALED << NFMT_SHIFT),
+  DFMT_11_11_10    | (NFMT_SSCALED << NFMT_SHIFT),
+  DFMT_11_11_10    | (NFMT_UINT    << NFMT_SHIFT),
+  DFMT_11_11_10    | (NFMT_SINT    << NFMT_SHIFT),
+  DFMT_11_11_10    | (NFMT_FLOAT   << NFMT_SHIFT),
+
+  DFMT_10_10_10_2  | (NFMT_UNORM   << NFMT_SHIFT),
+  DFMT_10_10_10_2  | (NFMT_SNORM   << NFMT_SHIFT),
+  DFMT_10_10_10_2  | (NFMT_USCALED << NFMT_SHIFT),
+  DFMT_10_10_10_2  | (NFMT_SSCALED << NFMT_SHIFT),
+  DFMT_10_10_10_2  | (NFMT_UINT    << NFMT_SHIFT),
+  DFMT_10_10_10_2  | (NFMT_SINT    << NFMT_SHIFT),
+
+  DFMT_2_10_10_10  | (NFMT_UNORM   << NFMT_SHIFT),
+  DFMT_2_10_10_10  | (NFMT_SNORM   << NFMT_SHIFT),
+  DFMT_2_10_10_10  | (NFMT_USCALED << NFMT_SHIFT),
+  DFMT_2_10_10_10  | (NFMT_SSCALED << NFMT_SHIFT),
+  DFMT_2_10_10_10  | (NFMT_UINT    << NFMT_SHIFT),
+  DFMT_2_10_10_10  | (NFMT_SINT    << NFMT_SHIFT),
+
+  DFMT_8_8_8_8     | (NFMT_UNORM   << NFMT_SHIFT),
+  DFMT_8_8_8_8     | (NFMT_SNORM   << NFMT_SHIFT),
+  DFMT_8_8_8_8     | (NFMT_USCALED << NFMT_SHIFT),
+  DFMT_8_8_8_8     | (NFMT_SSCALED << NFMT_SHIFT),
+  DFMT_8_8_8_8     | (NFMT_UINT    << NFMT_SHIFT),
+  DFMT_8_8_8_8     | (NFMT_SINT    << NFMT_SHIFT),
+
+  DFMT_32_32       | (NFMT_UINT    << NFMT_SHIFT),
+  DFMT_32_32       | (NFMT_SINT    << NFMT_SHIFT),
+  DFMT_32_32       | (NFMT_FLOAT   << NFMT_SHIFT),
+
+  DFMT_16_16_16_16 | (NFMT_UNORM   << NFMT_SHIFT),
+  DFMT_16_16_16_16 | (NFMT_SNORM   << NFMT_SHIFT),
+  DFMT_16_16_16_16 | (NFMT_USCALED << NFMT_SHIFT),
+  DFMT_16_16_16_16 | (NFMT_SSCALED << NFMT_SHIFT),
+  DFMT_16_16_16_16 | (NFMT_UINT    << NFMT_SHIFT),
+  DFMT_16_16_16_16 | (NFMT_SINT    << NFMT_SHIFT),
+  DFMT_16_16_16_16 | (NFMT_FLOAT   << NFMT_SHIFT),
+
+  DFMT_32_32_32    | (NFMT_UINT    << NFMT_SHIFT),
+  DFMT_32_32_32    | (NFMT_SINT    << NFMT_SHIFT),
+  DFMT_32_32_32    | (NFMT_FLOAT   << NFMT_SHIFT),
+  DFMT_32_32_32_32 | (NFMT_UINT    << NFMT_SHIFT),
+  DFMT_32_32_32_32 | (NFMT_SINT    << NFMT_SHIFT),
+  DFMT_32_32_32_32 | (NFMT_FLOAT   << NFMT_SHIFT)
+};
+
+} // namespace MTBUFFormat
+
 namespace Swizzle {
 
 // This must be in sync with llvm::AMDGPU::Swizzle::Id enum members, see SIDefines.h.
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUAsmUtils.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUAsmUtils.h
index cd91c5f6edd56..a6053735d7956 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUAsmUtils.h
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUAsmUtils.h
@@ -9,8 +9,11 @@
 #ifndef LLVM_LIB_TARGET_AMDGPU_UTILS_AMDGPUASMUTILS_H
 #define LLVM_LIB_TARGET_AMDGPU_UTILS_AMDGPUASMUTILS_H
 
+#include "llvm/ADT/StringRef.h"
+
 namespace llvm {
 namespace AMDGPU {
+
 namespace SendMsg { // Symbolic names for the sendmsg(...) syntax.
 
 extern const char* const IdSymbolic[];
@@ -25,6 +28,17 @@ extern const char* const IdSymbolic[];
 
 } // namespace Hwreg
 
+namespace MTBUFFormat {
+
+extern StringLiteral const DfmtSymbolic[];
+extern StringLiteral const NfmtSymbolicGFX10[];
+extern StringLiteral const NfmtSymbolicSICI[];
+extern StringLiteral const NfmtSymbolicVI[];
+extern StringLiteral const UfmtSymbolic[];
+extern unsigned const DfmtNfmt2UFmt[];
+
+} // namespace MTBUFFormat
+
 namespace Swizzle { // Symbolic names for the swizzle(...) syntax.
 
 extern const char* const IdSymbolic[];
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
index b89e34e4c99c1..497a04e25e628 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
@@ -789,6 +789,52 @@ void decodeHwreg(unsigned Val, unsigned &Id, unsigned &Offset, unsigned &Width)
 
 namespace MTBUFFormat {
 
+int64_t getDfmt(const StringRef Name) {
+  for (int Id = DFMT_MIN; Id <= DFMT_MAX; ++Id) {
+    if (Name == DfmtSymbolic[Id])
+      return Id;
+  }
+  return DFMT_UNDEF;
+}
+
+StringRef getDfmtName(unsigned Id) {
+  assert(Id <= DFMT_MAX);
+  return DfmtSymbolic[Id];
+}
+
+static StringLiteral const *getNfmtLookupTable(const MCSubtargetInfo &STI) {
+  if (isSI(STI) || isCI(STI))
+    return NfmtSymbolicSICI;
+  if (isVI(STI) || isGFX9(STI))
+    return NfmtSymbolicVI;
+  return NfmtSymbolicGFX10;
+}
+
+int64_t getNfmt(const StringRef Name, const MCSubtargetInfo &STI) {
+  auto lookupTable = getNfmtLookupTable(STI);
+  for (int Id = NFMT_MIN; Id <= NFMT_MAX; ++Id) {
+    if (Name == lookupTable[Id])
+      return Id;
+  }
+  return NFMT_UNDEF;
+}
+
+StringRef getNfmtName(unsigned Id, const MCSubtargetInfo &STI) {
+  assert(Id <= NFMT_MAX);
+  return getNfmtLookupTable(STI)[Id];
+}
+
+bool isValidDfmtNfmt(unsigned Id, const MCSubtargetInfo &STI) {
+  unsigned Dfmt;
+  unsigned Nfmt;
+  decodeDfmtNfmt(Id, Dfmt, Nfmt);
+  return isValidNfmt(Nfmt, STI);
+}
+
+bool isValidNfmt(unsigned Id, const MCSubtargetInfo &STI) {
+  return !getNfmtName(Id, STI).empty();
+}
+
 int64_t encodeDfmtNfmt(unsigned Dfmt, unsigned Nfmt) {
   return (Dfmt << DFMT_SHIFT) | (Nfmt << NFMT_SHIFT);
 }
@@ -798,6 +844,41 @@ void decodeDfmtNfmt(unsigned Format, unsigned &Dfmt, unsigned &Nfmt) {
   Nfmt = (Format >> NFMT_SHIFT) & NFMT_MASK;
 }
 
+int64_t getUnifiedFormat(const StringRef Name) {
+  for (int Id = UFMT_FIRST; Id <= UFMT_LAST; ++Id) {
+    if (Name == UfmtSymbolic[Id])
+      return Id;
+  }
+  return UFMT_UNDEF;
+}
+
+StringRef getUnifiedFormatName(unsigned Id) {
+  return isValidUnifiedFormat(Id) ? UfmtSymbolic[Id] : "";
+}
+
+bool isValidUnifiedFormat(unsigned Id) {
+  return Id <= UFMT_LAST;
+}
+
+int64_t convertDfmtNfmt2Ufmt(unsigned Dfmt, unsigned Nfmt) {
+  int64_t Fmt = encodeDfmtNfmt(Dfmt, Nfmt);
+  for (int Id = UFMT_FIRST; Id <= UFMT_LAST; ++Id) {
+    if (Fmt == DfmtNfmt2UFmt[Id])
+      return Id;
+  }
+  return UFMT_UNDEF;
+}
+
+bool isValidFormatEncoding(unsigned Val, const MCSubtargetInfo &STI) {
+  return isGFX10(STI) ? (Val <= UFMT_MAX) : (Val <= DFMT_NFMT_MAX);
+}
+
+unsigned getDefaultFormatEncoding(const MCSubtargetInfo &STI) {
+  if (isGFX10(STI))
+    return UFMT_DEFAULT;
+  return DFMT_NFMT_DEFAULT;
+}
+
 } // namespace MTBUFFormat
 
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
index a9ea05755a676..d250cc2ec03db 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
@@ -489,6 +489,30 @@ int64_t encodeDfmtNfmt(unsigned Dfmt, unsigned Nfmt);
 
 void decodeDfmtNfmt(unsigned Format, unsigned &Dfmt, unsigned &Nfmt);
 
+int64_t getDfmt(const StringRef Name);
+
+StringRef getDfmtName(unsigned Id);
+
+int64_t getNfmt(const StringRef Name, const MCSubtargetInfo &STI);
+
+StringRef getNfmtName(unsigned Id, const MCSubtargetInfo &STI);
+
+bool isValidDfmtNfmt(unsigned Val, const MCSubtargetInfo &STI);
+
+bool isValidNfmt(unsigned Val, const MCSubtargetInfo &STI);
+
+int64_t getUnifiedFormat(const StringRef Name);
+
+StringRef getUnifiedFormatName(unsigned Id);
+
+bool isValidUnifiedFormat(unsigned Val);
+
+int64_t convertDfmtNfmt2Ufmt(unsigned Dfmt, unsigned Nfmt);
+
+bool isValidFormatEncoding(unsigned Val, const MCSubtargetInfo &STI);
+
+unsigned getDefaultFormatEncoding(const MCSubtargetInfo &STI);
+
 } // namespace MTBUFFormat
 
 namespace SendMsg {
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.tbuffer.load.d16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.tbuffer.load.d16.ll
index 0d3e3b7efa258..db7949f540964 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.tbuffer.load.d16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.tbuffer.load.d16.ll
@@ -4,8 +4,8 @@
 ; RUN: llc < %s -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs | FileCheck -enable-var-scope -check-prefixes=GCN,PACKED,GFX10,GFX10-PACKED %s
 
 ; GCN-LABEL: {{^}}tbuffer_load_d16_x:
-; PREGFX10: tbuffer_load_format_d16_x v{{[0-9]+}}, off, s[{{[0-9]+:[0-9]+}}], dfmt:6, nfmt:1, 0
-; GFX10: tbuffer_load_format_d16_x v{{[0-9]+}}, off, s[{{[0-9]+:[0-9]+}}], format:22, 0
+; PREGFX10: tbuffer_load_format_d16_x v{{[0-9]+}}, off, s[{{[0-9]+:[0-9]+}}], 0 format:[BUF_DATA_FORMAT_10_11_11,BUF_NUM_FORMAT_SNORM]
+; GFX10: tbuffer_load_format_d16_x v{{[0-9]+}}, off, s[{{[0-9]+:[0-9]+}}], 0 format:[BUF_FMT_32_FLOAT]
 define amdgpu_ps half @tbuffer_load_d16_x(<4 x i32> inreg %rsrc) {
 main_body:
   %data = call half @llvm.amdgcn.raw.tbuffer.load.f16(<4 x i32> %rsrc, i32 0, i32 0, i32 22, i32 0)
@@ -13,11 +13,11 @@ main_body:
 }
 
 ; GCN-LABEL: {{^}}tbuffer_load_d16_xy:
-; PREGFX10-UNPACKED: tbuffer_load_format_d16_xy v{{\[}}{{[0-9]+}}:[[HI:[0-9]+]]{{\]}}, off, s[{{[0-9]+:[0-9]+}}], dfmt:6, nfmt:1, 0
+; PREGFX10-UNPACKED: tbuffer_load_format_d16_xy v{{\[}}{{[0-9]+}}:[[HI:[0-9]+]]{{\]}}, off, s[{{[0-9]+:[0-9]+}}], 0 format:[BUF_DATA_FORMAT_10_11_11,BUF_NUM_FORMAT_SNORM]
 ; UNPACKED: v_mov_b32_e32 v{{[0-9]+}}, v[[HI]]
 
-; PREGFX10-PACKED: tbuffer_load_format_d16_xy v[[FULL:[0-9]+]], off, s[{{[0-9]+:[0-9]+}}], dfmt:6, nfmt:1, 0
-; GFX10-PACKED: tbuffer_load_format_d16_xy v[[FULL:[0-9]+]], off, s[{{[0-9]+:[0-9]+}}], format:22, 0
+; PREGFX10-PACKED: tbuffer_load_format_d16_xy v[[FULL:[0-9]+]], off, s[{{[0-9]+:[0-9]+}}], 0 format:[BUF_DATA_FORMAT_10_11_11,BUF_NUM_FORMAT_SNORM]
+; GFX10-PACKED: tbuffer_load_format_d16_xy v[[FULL:[0-9]+]], off, s[{{[0-9]+:[0-9]+}}], 0 format:[BUF_FMT_32_FLOAT]
 ; PACKED: v_lshrrev_b32_e32 v{{[0-9]+}}, 16, v[[FULL]]
 define amdgpu_ps half @tbuffer_load_d16_xy(<4 x i32> inreg %rsrc) {
 main_body:
@@ -27,12 +27,12 @@ main_body:
 }
 
 ; GCN-LABEL: {{^}}tbuffer_load_d16_xyzw:
-; PREGFX10-UNPACKED: tbuffer_load_format_d16_xyzw v{{\[}}{{[0-9]+}}:[[HI:[0-9]+]]{{\]}}, off, s[{{[0-9]+:[0-9]+}}], dfmt:6, nfmt:1, 0
-; GFX10-UNPACKED: tbuffer_load_format_d16_xyzw v{{\[}}{{[0-9]+}}:[[HI:[0-9]+]]{{\]}}, off, s[{{[0-9]+:[0-9]+}}], format:22, 0
+; PREGFX10-UNPACKED: tbuffer_load_format_d16_xyzw v{{\[}}{{[0-9]+}}:[[HI:[0-9]+]]{{\]}}, off, s[{{[0-9]+:[0-9]+}}], 0 format:[BUF_DATA_FORMAT_10_11_11,BUF_NUM_FORMAT_SNORM]
+; GFX10-UNPACKED: tbuffer_load_format_d16_xyzw v{{\[}}{{[0-9]+}}:[[HI:[0-9]+]]{{\]}}, off, s[{{[0-9]+:[0-9]+}}], 0 format:[BUF_FMT_32_FLOAT]
 ; UNPACKED: v_mov_b32_e32 v{{[0-9]+}}, v[[HI]]
 
-; PREGFX10-PACKED: tbuffer_load_format_d16_xyzw v{{\[}}{{[0-9]+}}:[[HI:[0-9]+]]{{\]}}, off, s[{{[0-9]+:[0-9]+}}], dfmt:6, nfmt:1, 0
-; GFX10-PACKED: tbuffer_load_format_d16_xyzw v{{\[}}{{[0-9]+}}:[[HI:[0-9]+]]{{\]}}, off, s[{{[0-9]+:[0-9]+}}], format:22, 0
+; PREGFX10-PACKED: tbuffer_load_format_d16_xyzw v{{\[}}{{[0-9]+}}:[[HI:[0-9]+]]{{\]}}, off, s[{{[0-9]+:[0-9]+}}], 0 format:[BUF_DATA_FORMAT_10_11_11,BUF_NUM_FORMAT_SNORM]
+; GFX10-PACKED: tbuffer_load_format_d16_xyzw v{{\[}}{{[0-9]+}}:[[HI:[0-9]+]]{{\]}}, off, s[{{[0-9]+:[0-9]+}}], 0 format:[BUF_FMT_32_FLOAT]
 ; PACKED: v_lshrrev_b32_e32 v{{[0-9]+}}, 16, v[[HI]]
 define amdgpu_ps half @tbuffer_load_d16_xyzw(<4 x i32> inreg %rsrc) {
 main_body:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.tbuffer.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.tbuffer.load.ll
index 1aff4b9bc7e2f..462c9dcf1a239 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.tbuffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.tbuffer.load.ll
@@ -3,14 +3,14 @@
 ;RUN: llc < %s -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs | FileCheck -check-prefix=GCN -check-prefix=GFX10 %s
 
 ; GCN-LABEL: {{^}}tbuffer_load:
-; PREGFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, dfmt:14, nfmt:4, 0
-; PREGFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, dfmt:15, nfmt:3, 0 glc
-; PREGFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, dfmt:6, nfmt:1, 0 slc
-; PREGFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, dfmt:6, nfmt:1, 0 glc
-; GFX10-DAG: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, format:78, 0
-; GFX10-DAG: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, format:63, 0 glc
-; GFX10-DAG: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, format:22, 0 slc
-; GFX10-DAG: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, format:22, 0 glc dlc
+; PREGFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 format:[BUF_DATA_FORMAT_32_32_32_32,BUF_NUM_FORMAT_UINT]
+; PREGFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] glc
+; PREGFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 format:[BUF_DATA_FORMAT_10_11_11,BUF_NUM_FORMAT_SNORM] slc
+; PREGFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 format:[BUF_DATA_FORMAT_10_11_11,BUF_NUM_FORMAT_SNORM] glc
+; GFX10-DAG: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 format:78
+; GFX10-DAG: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 format:[BUF_FMT_32_32_SINT] glc
+; GFX10-DAG: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 format:[BUF_FMT_32_FLOAT] slc
+; GFX10-DAG: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 format:[BUF_FMT_32_FLOAT] glc dlc
 ; GCN: s_waitcnt
 define amdgpu_vs {<4 x float>, <4 x float>, <4 x float>, <4 x float>} @tbuffer_load(<4 x i32> inreg) {
 main_body:
@@ -29,8 +29,8 @@ main_body:
 }
 
 ; GCN-LABEL: {{^}}tbuffer_load_immoffs:
-; PREGFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, dfmt:14, nfmt:4, 0 offset:42
-; GFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, format:78, 0 offset:42
+; PREGFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 format:[BUF_DATA_FORMAT_32_32_32_32,BUF_NUM_FORMAT_UINT] offset:42
+; GFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 format:78 offset:42
 define amdgpu_vs <4 x float> @tbuffer_load_immoffs(<4 x i32> inreg) {
 main_body:
     %vdata   = call <4 x i32> @llvm.amdgcn.raw.tbuffer.load.v4i32(<4 x i32> %0, i32 42, i32 0, i32 78, i32 0)
@@ -39,12 +39,12 @@ main_body:
 }
 
 ; GCN-LABEL: {{^}}tbuffer_load_immoffs_large
-; PREGFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, dfmt:15, nfmt:2, 61 offset:4095
-; PREGFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, dfmt:14, nfmt:3, {{s[0-9]+}} offset:73
-; PREGFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, dfmt:13, nfmt:4, {{s[0-9]+}} offset:1
-; GFX10-DAG: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, format:47, 61 offset:4095
-; GFX10-DAG: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, format:62, {{s[0-9]+}} offset:73
-; GFX10-DAG: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, format:77, {{s[0-9]+}} offset:1
+; PREGFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, 61 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] offset:4095
+; PREGFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, {{s[0-9]+}} format:[BUF_DATA_FORMAT_32_32_32_32,BUF_NUM_FORMAT_SSCALED] offset:73
+; PREGFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, {{s[0-9]+}} format:[BUF_DATA_FORMAT_32_32_32,BUF_NUM_FORMAT_UINT] offset:1
+; GFX10-DAG: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, 61 format:[BUF_FMT_10_10_10_2_SSCALED] offset:4095
+; GFX10-DAG: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, {{s[0-9]+}} format:[BUF_FMT_32_32_UINT] offset:73
+; GFX10-DAG: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, {{s[0-9]+}} format:[BUF_FMT_32_32_32_32_FLOAT] offset:1
 ; GCN: s_waitcnt
 define amdgpu_vs {<4 x float>, <4 x float>, <4 x float>} @tbuffer_load_immoffs_large(<4 x i32> inreg, i32 inreg %soffs) {
     %vdata     = call <4 x i32> @llvm.amdgcn.raw.tbuffer.load.v4i32(<4 x i32> %0, i32 4095, i32 61, i32 47, i32 0)
@@ -60,8 +60,8 @@ define amdgpu_vs {<4 x float>, <4 x float>, <4 x float>} @tbuffer_load_immoffs_l
 }
 
 ; GCN-LABEL: {{^}}tbuffer_load_ofs:
-; PREGFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, dfmt:14, nfmt:4, 0 offen
-; GFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, format:78, 0 offen
+; PREGFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, 0 format:[BUF_DATA_FORMAT_32_32_32_32,BUF_NUM_FORMAT_UINT] offen
+; GFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, 0 format:78 offen
 define amdgpu_vs <4 x float> @tbuffer_load_ofs(<4 x i32> inreg, i32 %voffs) {
 main_body:
     %vdata   = call <4 x i32> @llvm.amdgcn.raw.tbuffer.load.v4i32(<4 x i32> %0, i32 %voffs, i32 0, i32 78, i32 0)
@@ -70,8 +70,8 @@ main_body:
 }
 
 ; GCN-LABEL: {{^}}tbuffer_load_ofs_imm:
-; PREGFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, dfmt:14, nfmt:4, 0 offen offset:52
-; GFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, format:78, 0 offen offset:52
+; PREGFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, 0 format:[BUF_DATA_FORMAT_32_32_32_32,BUF_NUM_FORMAT_UINT] offen offset:52
+; GFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, 0 format:78 offen offset:52
 define amdgpu_vs <4 x float> @tbuffer_load_ofs_imm(<4 x i32> inreg, i32 %voffs) {
 main_body:
     %ofs = add i32 %voffs, 52
@@ -81,8 +81,8 @@ main_body:
 }
 
 ; GCN-LABEL: {{^}}buffer_load_xy:
-; PREGFX10: tbuffer_load_format_xy {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, dfmt:13, nfmt:4, 0
-; GFX10: tbuffer_load_format_xy {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, format:77, 0
+; PREGFX10: tbuffer_load_format_xy {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 format:[BUF_DATA_FORMAT_32_32_32,BUF_NUM_FORMAT_UINT]
+; GFX10: tbuffer_load_format_xy {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 format:[BUF_FMT_32_32_32_32_FLOAT]
 define amdgpu_vs <2 x float> @buffer_load_xy(<4 x i32> inreg %rsrc) {
     %vdata = call <2 x i32> @llvm.amdgcn.raw.tbuffer.load.v2i32(<4 x i32> %rsrc, i32 0, i32 0, i32 77, i32 0)
     %vdata.f = bitcast <2 x i32> %vdata to <2 x float>
@@ -90,8 +90,8 @@ define amdgpu_vs <2 x float> @buffer_load_xy(<4 x i32> inreg %rsrc) {
 }
 
 ; GCN-LABEL: {{^}}buffer_load_x:
-; PREGFX10: tbuffer_load_format_x {{v[0-9]+}}, off, {{s\[[0-9]+:[0-9]+\]}}, dfmt:13, nfmt:4, 0
-; GFX10: tbuffer_load_format_x {{v[0-9]+}}, off, {{s\[[0-9]+:[0-9]+\]}}, format:77, 0
+; PREGFX10: tbuffer_load_format_x {{v[0-9]+}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 format:[BUF_DATA_FORMAT_32_32_32,BUF_NUM_FORMAT_UINT]
+; GFX10: tbuffer_load_format_x {{v[0-9]+}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 format:[BUF_FMT_32_32_32_32_FLOAT]
 define amdgpu_vs float @buffer_load_x(<4 x i32> inreg %rsrc) {
     %vdata = call i32 @llvm.amdgcn.raw.tbuffer.load.i32(<4 x i32> %rsrc, i32 0, i32 0, i32 77, i32 0)
     %vdata.f = bitcast i32 %vdata to float
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.tbuffer.store.d16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.tbuffer.store.d16.ll
index b337714f27b1c..5041cf3197342 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.tbuffer.store.d16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.tbuffer.store.d16.ll
@@ -8,8 +8,8 @@
 ; GCN-DAG: s_load_dwordx4
 ; GCN-DAG: s_load_dword s[[S_LO:[0-9]+]]
 ; GCN-DAG: v_mov_b32_e32 v[[V_LO:[0-9]+]], s[[S_LO]]
-; PREGFX10: tbuffer_store_format_d16_x v[[V_LO]], off, s[{{[0-9]+:[0-9]+}}],  dfmt:1,  nfmt:2, 0
-; GFX10: tbuffer_store_format_d16_x v[[V_LO]], off, s[{{[0-9]+:[0-9]+}}],  format:33, 0
+; PREGFX10: tbuffer_store_format_d16_x v[[V_LO]], off, s[{{[0-9]+:[0-9]+}}], 0 format:[BUF_NUM_FORMAT_USCALED]
+; GFX10: tbuffer_store_format_d16_x v[[V_LO]], off, s[{{[0-9]+:[0-9]+}}], 0 format:[BUF_FMT_10_11_11_SSCALED]
 define amdgpu_kernel void @tbuffer_store_d16_x(<4 x i32> %rsrc, half %data) {
 main_body:
   call void @llvm.amdgcn.raw.tbuffer.store.f16(half %data, <4 x i32> %rsrc, i32 0, i32 0, i32 33, i32 0)
@@ -22,10 +22,10 @@ main_body:
 ; UNPACKED-DAG: s_and_b32 [[MASKED:s[0-9]+]], [[S_DATA]], 0xffff{{$}}
 ; UNPACKED-DAG: v_mov_b32_e32 v[[V_LO:[0-9]+]], [[MASKED]]
 ; UNPACKED-DAG: v_mov_b32_e32 v[[V_HI:[0-9]+]], [[SHR]]
-; PREGFX10-UNPACKED: tbuffer_store_format_d16_xy v{{\[}}[[V_LO]]:[[V_HI]]{{\]}}, off, s[{{[0-9]+:[0-9]+}}],  dfmt:1,  nfmt:2, 0
+; PREGFX10-UNPACKED: tbuffer_store_format_d16_xy v{{\[}}[[V_LO]]:[[V_HI]]{{\]}}, off, s[{{[0-9]+:[0-9]+}}], 0 format:[BUF_NUM_FORMAT_USCALED]
 
-; PREGFX10-PACKED: tbuffer_store_format_d16_xy v{{[0-9]+}}, off, s[{{[0-9]+:[0-9]+}}],  dfmt:1,  nfmt:2, 0
-; GFX10-PACKED: tbuffer_store_format_d16_xy v{{[0-9]+}}, off, s[{{[0-9]+:[0-9]+}}],  format:33, 0
+; PREGFX10-PACKED: tbuffer_store_format_d16_xy v{{[0-9]+}}, off, s[{{[0-9]+:[0-9]+}}], 0 format:[BUF_NUM_FORMAT_USCALED]
+; GFX10-PACKED: tbuffer_store_format_d16_xy v{{[0-9]+}}, off, s[{{[0-9]+:[0-9]+}}], 0 format:[BUF_FMT_10_11_11_SSCALED]
 define amdgpu_kernel void @tbuffer_store_d16_xy(<4 x i32> %rsrc, <2 x half> %data) {
 main_body:
   call void @llvm.amdgcn.raw.tbuffer.store.v2f16(<2 x half> %data, <4 x i32> %rsrc, i32 0, i32 0, i32 33, i32 0)
@@ -43,13 +43,13 @@ main_body:
 
 ; UNPACKED-DAG: v_mov_b32_e32 v[[LO:[0-9]+]], [[MASKED0]]
 ; UNPACKED-DAG: v_mov_b32_e32 v[[HI:[0-9]+]], [[SHR1]]
-; PREGFX10-UNPACKED: tbuffer_store_format_d16_xyzw v{{\[}}[[LO]]:[[HI]]{{\]}}, off, s[{{[0-9]+:[0-9]+}}],  dfmt:1,  nfmt:2, 0
+; PREGFX10-UNPACKED: tbuffer_store_format_d16_xyzw v{{\[}}[[LO]]:[[HI]]{{\]}}, off, s[{{[0-9]+:[0-9]+}}], 0 format:[BUF_NUM_FORMAT_USCALED]
 
 
 ; PACKED-DAG: v_mov_b32_e32 v[[LO:[0-9]+]], s[[S_DATA_0]]
 ; PACKED-DAG: v_mov_b32_e32 v[[HI:[0-9]+]], s[[S_DATA_1]]
-; PREGFX10-PACKED: tbuffer_store_format_d16_xyzw v{{\[}}[[LO]]:[[HI]]{{\]}}, off, s[{{[0-9]+:[0-9]+}}],  dfmt:1,  nfmt:2, 0
-; GFX10-PACKED: tbuffer_store_format_d16_xyzw v{{\[}}[[LO]]:[[HI]]{{\]}}, off, s[{{[0-9]+:[0-9]+}}],  format:33, 0
+; PREGFX10-PACKED: tbuffer_store_format_d16_xyzw v{{\[}}[[LO]]:[[HI]]{{\]}}, off, s[{{[0-9]+:[0-9]+}}], 0 format:[BUF_NUM_FORMAT_USCALED]
+; GFX10-PACKED: tbuffer_store_format_d16_xyzw v{{\[}}[[LO]]:[[HI]]{{\]}}, off, s[{{[0-9]+:[0-9]+}}], 0 format:[BUF_FMT_10_11_11_SSCALED]
 define amdgpu_kernel void @tbuffer_store_d16_xyzw(<4 x i32> %rsrc, <4 x half> %data) {
 main_body:
   call void @llvm.amdgcn.raw.tbuffer.store.v4f16(<4 x half> %data, <4 x i32> %rsrc, i32 0, i32 0, i32 33, i32 0)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.tbuffer.store.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.tbuffer.store.ll
index 9c88de9c4f88e..6bc79150b8af5 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.tbuffer.store.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.tbuffer.store.ll
@@ -3,14 +3,14 @@
 ;RUN: llc < %s -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs | FileCheck -check-prefixes=GCN,GFX10 %s
 
 ; GCN-LABEL: {{^}}tbuffer_store:
-; PREGFX10: tbuffer_store_format_xyzw v[0:3], off, s[0:3], dfmt:12, nfmt:2, 0
-; PREGFX10: tbuffer_store_format_xyzw v[4:7], off, s[0:3], dfmt:13, nfmt:3, 0 glc
-; PREGFX10: tbuffer_store_format_xyzw v[8:11], off, s[0:3], dfmt:14, nfmt:4, 0 slc
-; PREGFX10: tbuffer_store_format_xyzw v[8:11], off, s[0:3], dfmt:14, nfmt:4, 0 glc
-; GFX10: tbuffer_store_format_xyzw v[0:3], off, s[0:3], format:44, 0
-; GFX10: tbuffer_store_format_xyzw v[4:7], off, s[0:3], format:61, 0 glc
-; GFX10: tbuffer_store_format_xyzw v[8:11], off, s[0:3], format:78, 0 slc
-; GFX10: tbuffer_store_format_xyzw v[8:11], off, s[0:3], format:78, 0 glc dlc
+; PREGFX10: tbuffer_store_format_xyzw v[0:3], off, s[0:3], 0 format:[BUF_DATA_FORMAT_16_16_16_16,BUF_NUM_FORMAT_USCALED]
+; PREGFX10: tbuffer_store_format_xyzw v[4:7], off, s[0:3], 0 format:[BUF_DATA_FORMAT_32_32_32,BUF_NUM_FORMAT_SSCALED] glc
+; PREGFX10: tbuffer_store_format_xyzw v[8:11], off, s[0:3], 0 format:[BUF_DATA_FORMAT_32_32_32_32,BUF_NUM_FORMAT_UINT] slc
+; PREGFX10: tbuffer_store_format_xyzw v[8:11], off, s[0:3], 0 format:[BUF_DATA_FORMAT_32_32_32_32,BUF_NUM_FORMAT_UINT] glc
+; GFX10: tbuffer_store_format_xyzw v[0:3], off, s[0:3], 0 format:[BUF_FMT_10_10_10_2_UNORM]
+; GFX10: tbuffer_store_format_xyzw v[4:7], off, s[0:3], 0 format:[BUF_FMT_8_8_8_8_SINT] glc
+; GFX10: tbuffer_store_format_xyzw v[8:11], off, s[0:3], 0 format:78 slc
+; GFX10: tbuffer_store_format_xyzw v[8:11], off, s[0:3], 0 format:78 glc dlc
 define amdgpu_ps void @tbuffer_store(<4 x i32> inreg, <4 x float>, <4 x float>, <4 x float>) {
 main_body:
   %in1 = bitcast <4 x float> %1 to <4 x i32>
@@ -24,8 +24,8 @@ main_body:
 }
 
 ; GCN-LABEL: {{^}}tbuffer_store_immoffs:
-; PREGFX10: tbuffer_store_format_xyzw v[0:3], off, s[0:3], dfmt:5, nfmt:7, 0 offset:42
-; GFX10: tbuffer_store_format_xyzw v[0:3], off, s[0:3], format:117, 0 offset:42
+; PREGFX10: tbuffer_store_format_xyzw v[0:3], off, s[0:3], 0 format:[BUF_DATA_FORMAT_16_16,BUF_NUM_FORMAT_FLOAT] offset:42
+; GFX10: tbuffer_store_format_xyzw v[0:3], off, s[0:3], 0 format:117 offset:42
 define amdgpu_ps void @tbuffer_store_immoffs(<4 x i32> inreg, <4 x float>) {
 main_body:
   %in1 = bitcast <4 x float> %1 to <4 x i32>
@@ -34,8 +34,8 @@ main_body:
 }
 
 ; GCN-LABEL: {{^}}tbuffer_store_scalar_and_imm_offs:
-; PREGFX10: tbuffer_store_format_xyzw v[0:3], off, s[0:3], dfmt:5, nfmt:7, {{s[0-9]+}} offset:42
-; GFX10: tbuffer_store_format_xyzw v[0:3], off, s[0:3], format:117, {{s[0-9]+}} offset:42
+; PREGFX10: tbuffer_store_format_xyzw v[0:3], off, s[0:3], {{s[0-9]+}} format:[BUF_DATA_FORMAT_16_16,BUF_NUM_FORMAT_FLOAT] offset:42
+; GFX10: tbuffer_store_format_xyzw v[0:3], off, s[0:3], {{s[0-9]+}} format:117 offset:42
 define amdgpu_ps void @tbuffer_store_scalar_and_imm_offs(<4 x i32> inreg, <4 x float> %vdata, i32 inreg %soffset) {
 main_body:
   %in1 = bitcast <4 x float> %vdata to <4 x i32>
@@ -44,8 +44,8 @@ main_body:
 }
 
 ; GCN-LABEL: {{^}}buffer_store_ofs:
-; PREGFX10: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], dfmt:3, nfmt:7, 0 offen
-; GFX10: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], format:115, 0 offen
+; PREGFX10: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_DATA_FORMAT_8_8,BUF_NUM_FORMAT_FLOAT] offen
+; GFX10: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:115 offen
 define amdgpu_ps void @buffer_store_ofs(<4 x i32> inreg, <4 x float> %vdata, i32 %voffset) {
 main_body:
   %in1 = bitcast <4 x float> %vdata to <4 x i32>
@@ -54,8 +54,8 @@ main_body:
 }
 
 ; GCN-LABEL: {{^}}buffer_store_x1:
-; PREGFX10: tbuffer_store_format_x v0, off, s[0:3], dfmt:13, nfmt:7, 0
-; GFX10: tbuffer_store_format_x v0, off, s[0:3], format:125, 0
+; PREGFX10: tbuffer_store_format_x v0, off, s[0:3], 0 format:[BUF_DATA_FORMAT_32_32_32,BUF_NUM_FORMAT_FLOAT]
+; GFX10: tbuffer_store_format_x v0, off, s[0:3], 0 format:125
 define amdgpu_ps void @buffer_store_x1(<4 x i32> inreg %rsrc, float %data) {
 main_body:
   %data.i = bitcast float %data to i32
@@ -64,8 +64,8 @@ main_body:
 }
 
 ; GCN-LABEL: {{^}}buffer_store_x2:
-; PREGFX10: tbuffer_store_format_xy v[0:1], off, s[0:3], dfmt:1, nfmt:2, 0
-; GFX10: tbuffer_store_format_xy v[0:1], off, s[0:3], format:33, 0
+; PREGFX10: tbuffer_store_format_xy v[0:1], off, s[0:3], 0 format:[BUF_NUM_FORMAT_USCALED]
+; GFX10: tbuffer_store_format_xy v[0:1], off, s[0:3], 0 format:[BUF_FMT_10_11_11_SSCALED]
 define amdgpu_ps void @buffer_store_x2(<4 x i32> inreg %rsrc, <2 x float> %data) {
 main_body:
   %data.i = bitcast <2 x float> %data to <2 x i32>
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.tbuffer.load.d16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.tbuffer.load.d16.ll
index 36aa5012a17c9..2fd21a10564d4 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.tbuffer.load.d16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.tbuffer.load.d16.ll
@@ -5,8 +5,8 @@
 
 ; GCN-LABEL: {{^}}tbuffer_load_d16_x:
 ; GCN: v_mov_b32_e32 [[ZEROREG:v[0-9]+]], 0
-; PREGFX10: tbuffer_load_format_d16_x v{{[0-9]+}}, [[ZEROREG]], s[{{[0-9]+:[0-9]+}}],  dfmt:6,  nfmt:1, 0 idxen
-; GFX10: tbuffer_load_format_d16_x v{{[0-9]+}}, [[ZEROREG]], s[{{[0-9]+:[0-9]+}}],  format:22, 0 idxen
+; PREGFX10: tbuffer_load_format_d16_x v{{[0-9]+}}, [[ZEROREG]], s[{{[0-9]+:[0-9]+}}], 0 format:[BUF_DATA_FORMAT_10_11_11,BUF_NUM_FORMAT_SNORM] idxen
+; GFX10: tbuffer_load_format_d16_x v{{[0-9]+}}, [[ZEROREG]], s[{{[0-9]+:[0-9]+}}], 0 format:[BUF_FMT_32_FLOAT] idxen
 define amdgpu_ps half @tbuffer_load_d16_x(<4 x i32> inreg %rsrc) {
 main_body:
   %data = call half @llvm.amdgcn.struct.tbuffer.load.f16(<4 x i32> %rsrc, i32 0, i32 0, i32 0, i32 22, i32 0)
@@ -15,11 +15,11 @@ main_body:
 
 ; GCN-LABEL: {{^}}tbuffer_load_d16_xy:
 ; GCN: v_mov_b32_e32 [[ZEROREG:v[0-9]+]], 0
-; PREGFX10-UNPACKED: tbuffer_load_format_d16_xy v{{\[}}{{[0-9]+}}:[[HI:[0-9]+]]{{\]}}, [[ZEROREG]], s[{{[0-9]+:[0-9]+}}],  dfmt:6,  nfmt:1, 0 idxen
+; PREGFX10-UNPACKED: tbuffer_load_format_d16_xy v{{\[}}{{[0-9]+}}:[[HI:[0-9]+]]{{\]}}, [[ZEROREG]], s[{{[0-9]+:[0-9]+}}], 0 format:[BUF_DATA_FORMAT_10_11_11,BUF_NUM_FORMAT_SNORM] idxen
 ; PREGFX10-UNPACKED: v_mov_b32_e32 v{{[0-9]+}}, v[[HI]]
 
-; PREGFX10-PACKED: tbuffer_load_format_d16_xy v[[FULL:[0-9]+]], [[ZEROREG]], s[{{[0-9]+:[0-9]+}}],  dfmt:6,  nfmt:1, 0 idxen
-; GFX10-PACKED: tbuffer_load_format_d16_xy v[[FULL:[0-9]+]], [[ZEROREG]], s[{{[0-9]+:[0-9]+}}],  format:22, 0 idxen
+; PREGFX10-PACKED: tbuffer_load_format_d16_xy v[[FULL:[0-9]+]], [[ZEROREG]], s[{{[0-9]+:[0-9]+}}], 0 format:[BUF_DATA_FORMAT_10_11_11,BUF_NUM_FORMAT_SNORM] idxen
+; GFX10-PACKED: tbuffer_load_format_d16_xy v[[FULL:[0-9]+]], [[ZEROREG]], s[{{[0-9]+:[0-9]+}}], 0 format:[BUF_FMT_32_FLOAT] idxen
 ; PACKED: v_lshrrev_b32_e32 v{{[0-9]+}}, 16, v[[FULL]]
 define amdgpu_ps half @tbuffer_load_d16_xy(<4 x i32> inreg %rsrc) {
 main_body:
@@ -30,11 +30,11 @@ main_body:
 
 ; GCN-LABEL: {{^}}tbuffer_load_d16_xyzw:
 ; GCN: v_mov_b32_e32 [[ZEROREG:v[0-9]+]], 0
-; PREGFX10-UNPACKED: tbuffer_load_format_d16_xyzw v{{\[}}{{[0-9]+}}:[[HI:[0-9]+]]{{\]}}, [[ZEROREG]], s[{{[0-9]+:[0-9]+}}],  dfmt:6,  nfmt:1, 0 idxen
+; PREGFX10-UNPACKED: tbuffer_load_format_d16_xyzw v{{\[}}{{[0-9]+}}:[[HI:[0-9]+]]{{\]}}, [[ZEROREG]], s[{{[0-9]+:[0-9]+}}], 0 format:[BUF_DATA_FORMAT_10_11_11,BUF_NUM_FORMAT_SNORM] idxen
 ; PREGFX10-UNPACKED: v_mov_b32_e32 v{{[0-9]+}}, v[[HI]]
 
-; PREGFX10-PACKED: tbuffer_load_format_d16_xyzw v{{\[}}{{[0-9]+}}:[[HI:[0-9]+]]{{\]}}, [[ZEROREG]], s[{{[0-9]+:[0-9]+}}],  dfmt:6,  nfmt:1, 0 idxen
-; GFX10-PACKED: tbuffer_load_format_d16_xyzw v{{\[}}{{[0-9]+}}:[[HI:[0-9]+]]{{\]}}, [[ZEROREG]], s[{{[0-9]+:[0-9]+}}],  format:22, 0 idxen
+; PREGFX10-PACKED: tbuffer_load_format_d16_xyzw v{{\[}}{{[0-9]+}}:[[HI:[0-9]+]]{{\]}}, [[ZEROREG]], s[{{[0-9]+:[0-9]+}}], 0 format:[BUF_DATA_FORMAT_10_11_11,BUF_NUM_FORMAT_SNORM] idxen
+; GFX10-PACKED: tbuffer_load_format_d16_xyzw v{{\[}}{{[0-9]+}}:[[HI:[0-9]+]]{{\]}}, [[ZEROREG]], s[{{[0-9]+:[0-9]+}}], 0 format:[BUF_FMT_32_FLOAT] idxen
 ; PACKED: v_lshrrev_b32_e32 v{{[0-9]+}}, 16, v[[HI]]
 define amdgpu_ps half @tbuffer_load_d16_xyzw(<4 x i32> inreg %rsrc) {
 main_body:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.tbuffer.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.tbuffer.load.ll
index 1150347650c8d..ac18761534ea2 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.tbuffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.tbuffer.load.ll
@@ -4,14 +4,14 @@
 
 ; GCN-LABEL: {{^}}tbuffer_load:
 ; GCN: v_mov_b32_e32 [[ZEROREG:v[0-9]+]], 0
-; PREGFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, [[ZEROREG]], {{s\[[0-9]+:[0-9]+\]}}, dfmt:14, nfmt:4, 0 idxen
-; PREGFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, [[ZEROREG]], {{s\[[0-9]+:[0-9]+\]}}, dfmt:15, nfmt:3, 0 idxen glc
-; PREGFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, [[ZEROREG]], {{s\[[0-9]+:[0-9]+\]}}, dfmt:6, nfmt:1, 0 idxen slc
-; PREGFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, [[ZEROREG]], {{s\[[0-9]+:[0-9]+\]}}, dfmt:6, nfmt:1, 0 idxen glc
-; GFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, [[ZEROREG]], {{s\[[0-9]+:[0-9]+\]}}, format:78, 0 idxen
-; GFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, [[ZEROREG]], {{s\[[0-9]+:[0-9]+\]}}, format:63, 0 idxen glc
-; GFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, [[ZEROREG]], {{s\[[0-9]+:[0-9]+\]}}, format:22, 0 idxen slc
-; GFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, [[ZEROREG]], {{s\[[0-9]+:[0-9]+\]}}, format:22, 0 idxen glc dlc
+; PREGFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, [[ZEROREG]], {{s\[[0-9]+:[0-9]+\]}}, 0 format:[BUF_DATA_FORMAT_32_32_32_32,BUF_NUM_FORMAT_UINT] idxen
+; PREGFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, [[ZEROREG]], {{s\[[0-9]+:[0-9]+\]}}, 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] idxen glc
+; PREGFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, [[ZEROREG]], {{s\[[0-9]+:[0-9]+\]}}, 0 format:[BUF_DATA_FORMAT_10_11_11,BUF_NUM_FORMAT_SNORM] idxen slc
+; PREGFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, [[ZEROREG]], {{s\[[0-9]+:[0-9]+\]}}, 0 format:[BUF_DATA_FORMAT_10_11_11,BUF_NUM_FORMAT_SNORM] idxen glc
+; GFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, [[ZEROREG]], {{s\[[0-9]+:[0-9]+\]}}, 0 format:78 idxen
+; GFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, [[ZEROREG]], {{s\[[0-9]+:[0-9]+\]}}, 0 format:[BUF_FMT_32_32_SINT] idxen glc
+; GFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, [[ZEROREG]], {{s\[[0-9]+:[0-9]+\]}}, 0 format:[BUF_FMT_32_FLOAT] idxen slc
+; GFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, [[ZEROREG]], {{s\[[0-9]+:[0-9]+\]}}, 0 format:[BUF_FMT_32_FLOAT] idxen glc dlc
 ; GCN: s_waitcnt
 define amdgpu_vs {<4 x float>, <4 x float>, <4 x float>, <4 x float>} @tbuffer_load(<4 x i32> inreg) {
 main_body:
@@ -31,8 +31,8 @@ main_body:
 
 ; GCN-LABEL: {{^}}tbuffer_load_immoffs:
 ; GCN: v_mov_b32_e32 [[ZEROREG:v[0-9]+]], 0
-; PREGFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, [[ZEROREG]], {{s\[[0-9]+:[0-9]+\]}}, dfmt:14, nfmt:4, 0 idxen offset:42
-; GFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, [[ZEROREG]], {{s\[[0-9]+:[0-9]+\]}}, format:78, 0 idxen offset:42
+; PREGFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, [[ZEROREG]], {{s\[[0-9]+:[0-9]+\]}}, 0 format:[BUF_DATA_FORMAT_32_32_32_32,BUF_NUM_FORMAT_UINT] idxen offset:42
+; GFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, [[ZEROREG]], {{s\[[0-9]+:[0-9]+\]}}, 0 format:78 idxen offset:42
 define amdgpu_vs <4 x float> @tbuffer_load_immoffs(<4 x i32> inreg) {
 main_body:
     %vdata   = call <4 x i32> @llvm.amdgcn.struct.tbuffer.load.v4i32(<4 x i32> %0, i32 0, i32 42, i32 0, i32 78, i32 0)
@@ -42,12 +42,12 @@ main_body:
 
 ; GCN-LABEL: {{^}}tbuffer_load_immoffs_large
 ; GCN: v_mov_b32_e32 [[ZEROREG:v[0-9]+]], 0
-; PREGFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, [[ZEROREG]], {{s\[[0-9]+:[0-9]+\]}}, dfmt:15, nfmt:2, 61 idxen offset:4095
-; PREGFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, [[ZEROREG]], {{s\[[0-9]+:[0-9]+\]}}, dfmt:14, nfmt:3, {{s[0-9]+}} idxen offset:73
-; PREGFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, [[ZEROREG]], {{s\[[0-9]+:[0-9]+\]}}, dfmt:13, nfmt:4, {{s[0-9]+}} idxen offset:1
-; GFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, [[ZEROREG]], {{s\[[0-9]+:[0-9]+\]}}, format:47, 61 idxen offset:4095
-; GFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, [[ZEROREG]], {{s\[[0-9]+:[0-9]+\]}}, format:62, {{s[0-9]+}} idxen offset:73
-; GFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, [[ZEROREG]], {{s\[[0-9]+:[0-9]+\]}}, format:77, {{s[0-9]+}} idxen offset:1
+; PREGFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, [[ZEROREG]], {{s\[[0-9]+:[0-9]+\]}}, 61 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] idxen offset:4095
+; PREGFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, [[ZEROREG]], {{s\[[0-9]+:[0-9]+\]}}, {{s[0-9]+}} format:[BUF_DATA_FORMAT_32_32_32_32,BUF_NUM_FORMAT_SSCALED] idxen offset:73
+; PREGFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, [[ZEROREG]], {{s\[[0-9]+:[0-9]+\]}}, {{s[0-9]+}} format:[BUF_DATA_FORMAT_32_32_32,BUF_NUM_FORMAT_UINT] idxen offset:1
+; GFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, [[ZEROREG]], {{s\[[0-9]+:[0-9]+\]}}, 61 format:[BUF_FMT_10_10_10_2_SSCALED] idxen offset:4095
+; GFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, [[ZEROREG]], {{s\[[0-9]+:[0-9]+\]}}, {{s[0-9]+}} format:[BUF_FMT_32_32_UINT] idxen offset:73
+; GFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, [[ZEROREG]], {{s\[[0-9]+:[0-9]+\]}}, {{s[0-9]+}} format:[BUF_FMT_32_32_32_32_FLOAT] idxen offset:1
 ; GCN: s_waitcnt
 define amdgpu_vs {<4 x float>, <4 x float>, <4 x float>} @tbuffer_load_immoffs_large(<4 x i32> inreg, i32 inreg %soffs) {
     %vdata     = call <4 x i32> @llvm.amdgcn.struct.tbuffer.load.v4i32(<4 x i32> %0, i32 0, i32 4095, i32 61, i32 47, i32 0)
@@ -63,8 +63,8 @@ define amdgpu_vs {<4 x float>, <4 x float>, <4 x float>} @tbuffer_load_immoffs_l
 }
 
 ; GCN-LABEL: {{^}}tbuffer_load_idx:
-; PREGFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, dfmt:14, nfmt:4, 0 idxen
-; GFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, format:78, 0 idxen
+; PREGFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, 0 format:[BUF_DATA_FORMAT_32_32_32_32,BUF_NUM_FORMAT_UINT] idxen
+; GFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, 0 format:78 idxen
 define amdgpu_vs <4 x float> @tbuffer_load_idx(<4 x i32> inreg, i32 %vindex) {
 main_body:
     %vdata   = call <4 x i32> @llvm.amdgcn.struct.tbuffer.load.v4i32(<4 x i32> %0, i32 %vindex, i32 0, i32 0, i32 78, i32 0)
@@ -73,8 +73,8 @@ main_body:
 }
 
 ; GCN-LABEL: {{^}}tbuffer_load_ofs:
-; PREGFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, dfmt:14, nfmt:4, 0 idxen offen
-; GFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, format:78, 0 idxen offen
+; PREGFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 format:[BUF_DATA_FORMAT_32_32_32_32,BUF_NUM_FORMAT_UINT] idxen offen
+; GFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 format:78 idxen offen
 define amdgpu_vs <4 x float> @tbuffer_load_ofs(<4 x i32> inreg, i32 %voffs) {
 main_body:
     %vdata   = call <4 x i32> @llvm.amdgcn.struct.tbuffer.load.v4i32(<4 x i32> %0, i32 0, i32 %voffs, i32 0, i32 78, i32 0)
@@ -83,8 +83,8 @@ main_body:
 }
 
 ; GCN-LABEL: {{^}}tbuffer_load_ofs_imm:
-; PREGFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, dfmt:14, nfmt:4, 0 idxen offen offset:52
-; GFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, format:78, 0 idxen offen offset:52
+; PREGFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 format:[BUF_DATA_FORMAT_32_32_32_32,BUF_NUM_FORMAT_UINT] idxen offen offset:52
+; GFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 format:78 idxen offen offset:52
 define amdgpu_vs <4 x float> @tbuffer_load_ofs_imm(<4 x i32> inreg, i32 %voffs) {
 main_body:
     %ofs = add i32 %voffs, 52
@@ -94,8 +94,8 @@ main_body:
 }
 
 ; GCN-LABEL: {{^}}tbuffer_load_both:
-; PREGFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, dfmt:14, nfmt:4, 0 idxen offen
-; GFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, format:78, 0 idxen offen
+; PREGFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 format:[BUF_DATA_FORMAT_32_32_32_32,BUF_NUM_FORMAT_UINT] idxen offen
+; GFX10: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 format:78 idxen offen
 define amdgpu_vs <4 x float> @tbuffer_load_both(<4 x i32> inreg, i32 %vindex, i32 %voffs) {
 main_body:
     %vdata   = call <4 x i32> @llvm.amdgcn.struct.tbuffer.load.v4i32(<4 x i32> %0, i32 %vindex, i32 %voffs, i32 0, i32 78, i32 0)
@@ -105,8 +105,8 @@ main_body:
 
 
 ; GCN-LABEL: {{^}}buffer_load_xy:
-; PREGFX10: tbuffer_load_format_xy {{v\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, dfmt:13, nfmt:4, 0 idxen
-; GFX10: tbuffer_load_format_xy {{v\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, format:77, 0 idxen
+; PREGFX10: tbuffer_load_format_xy {{v\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, 0 format:[BUF_DATA_FORMAT_32_32_32,BUF_NUM_FORMAT_UINT] idxen
+; GFX10: tbuffer_load_format_xy {{v\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, 0 format:[BUF_FMT_32_32_32_32_FLOAT] idxen
 define amdgpu_vs <2 x float> @buffer_load_xy(<4 x i32> inreg %rsrc) {
     %vdata = call <2 x i32> @llvm.amdgcn.struct.tbuffer.load.v2i32(<4 x i32> %rsrc, i32 0, i32 0, i32 0, i32 77, i32 0)
     %vdata.f = bitcast <2 x i32> %vdata to <2 x float>
@@ -114,8 +114,8 @@ define amdgpu_vs <2 x float> @buffer_load_xy(<4 x i32> inreg %rsrc) {
 }
 
 ; GCN-LABEL: {{^}}buffer_load_x:
-; PREGFX10: tbuffer_load_format_x {{v[0-9]+}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, dfmt:13, nfmt:4, 0 idxen
-; GFX10: tbuffer_load_format_x {{v[0-9]+}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, format:77, 0 idxen
+; PREGFX10: tbuffer_load_format_x {{v[0-9]+}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, 0 format:[BUF_DATA_FORMAT_32_32_32,BUF_NUM_FORMAT_UINT] idxen
+; GFX10: tbuffer_load_format_x {{v[0-9]+}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, 0 format:[BUF_FMT_32_32_32_32_FLOAT] idxen
 define amdgpu_vs float @buffer_load_x(<4 x i32> inreg %rsrc) {
     %vdata = call i32 @llvm.amdgcn.struct.tbuffer.load.i32(<4 x i32> %rsrc, i32 0, i32 0, i32 0, i32 77, i32 0)
     %vdata.f = bitcast i32 %vdata to float
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.tbuffer.store.d16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.tbuffer.store.d16.ll
index 519eef4cb5b7c..ca78b29cc8f53 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.tbuffer.store.d16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.tbuffer.store.d16.ll
@@ -8,8 +8,8 @@
 ; GCN-DAG: s_load_dwordx4
 ; GCN-DAG: s_load_dword{{[x0-2]*}} s{{\[}}[[S_LO:[0-9]+]]
 ; GCN-DAG: v_mov_b32_e32 v[[V_LO:[0-9]+]], s[[S_LO]]
-; PREGFX10: tbuffer_store_format_d16_x v[[V_LO]], v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], dfmt:1, nfmt:2, 0 idxen
-; GFX10: tbuffer_store_format_d16_x v[[V_LO]], v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], format:33, 0 idxen
+; PREGFX10: tbuffer_store_format_d16_x v[[V_LO]], v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], 0 format:[BUF_NUM_FORMAT_USCALED] idxen
+; GFX10: tbuffer_store_format_d16_x v[[V_LO]], v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], 0 format:[BUF_FMT_10_11_11_SSCALED] idxen
 define amdgpu_kernel void @tbuffer_store_d16_x(<4 x i32> %rsrc, half %data, i32 %vindex) {
 main_body:
   call void @llvm.amdgcn.struct.tbuffer.store.f16(half %data, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 0, i32 33, i32 0)
@@ -22,10 +22,10 @@ main_body:
 ; UNPACKED-DAG: s_and_b32 [[MASKED:s[0-9]+]], [[S_DATA]], 0xffff{{$}}
 ; UNPACKED-DAG: v_mov_b32_e32 v[[V_LO:[0-9]+]], [[MASKED]]
 ; UNPACKED-DAG: v_mov_b32_e32 v[[V_HI:[0-9]+]], [[SHR]]
-; PREGFX10-UNPACKED: tbuffer_store_format_d16_xy v{{\[}}[[V_LO]]:[[V_HI]]{{\]}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], dfmt:1, nfmt:2, 0 idxen
+; PREGFX10-UNPACKED: tbuffer_store_format_d16_xy v{{\[}}[[V_LO]]:[[V_HI]]{{\]}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], 0 format:[BUF_NUM_FORMAT_USCALED] idxen
 
-; PREGFX10-PACKED: tbuffer_store_format_d16_xy v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], dfmt:1, nfmt:2, 0 idxen
-; GFX10-PACKED: tbuffer_store_format_d16_xy v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], format:33, 0 idxen
+; PREGFX10-PACKED: tbuffer_store_format_d16_xy v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], 0 format:[BUF_NUM_FORMAT_USCALED] idxen
+; GFX10-PACKED: tbuffer_store_format_d16_xy v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], 0 format:[BUF_FMT_10_11_11_SSCALED] idxen
 define amdgpu_kernel void @tbuffer_store_d16_xy(<4 x i32> %rsrc, <2 x half> %data, i32 %vindex) {
 main_body:
   call void @llvm.amdgcn.struct.tbuffer.store.v2f16(<2 x half> %data, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 0, i32 33, i32 0)
@@ -43,12 +43,12 @@ main_body:
 
 ; UNPACKED-DAG: v_mov_b32_e32 v[[LO:[0-9]+]], [[MASKED0]]
 ; UNPACKED-DAG: v_mov_b32_e32 v[[HI:[0-9]+]], [[SHR1]]
-; PREGFX10-UNPACKED: tbuffer_store_format_d16_xyzw v{{\[}}[[LO]]:[[HI]]{{\]}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], dfmt:1, nfmt:2, 0 idxen
+; PREGFX10-UNPACKED: tbuffer_store_format_d16_xyzw v{{\[}}[[LO]]:[[HI]]{{\]}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], 0 format:[BUF_NUM_FORMAT_USCALED] idxen
 
 ; PACKED-DAG: v_mov_b32_e32 v[[LO:[0-9]+]], s[[S_DATA_0]]
 ; PACKED-DAG: v_mov_b32_e32 v[[HI:[0-9]+]], s[[S_DATA_1]]
-; PREGFX10-PACKED: tbuffer_store_format_d16_xyzw v{{\[}}[[LO]]:[[HI]]{{\]}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], dfmt:1, nfmt:2, 0 idxen
-; GFX10-PACKED: tbuffer_store_format_d16_xyzw v{{\[}}[[LO]]:[[HI]]{{\]}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], format:33, 0 idxen
+; PREGFX10-PACKED: tbuffer_store_format_d16_xyzw v{{\[}}[[LO]]:[[HI]]{{\]}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], 0 format:[BUF_NUM_FORMAT_USCALED] idxen
+; GFX10-PACKED: tbuffer_store_format_d16_xyzw v{{\[}}[[LO]]:[[HI]]{{\]}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], 0 format:[BUF_FMT_10_11_11_SSCALED] idxen
 define amdgpu_kernel void @tbuffer_store_d16_xyzw(<4 x i32> %rsrc, <4 x half> %data, i32 %vindex) {
 main_body:
   call void @llvm.amdgcn.struct.tbuffer.store.v4f16(<4 x half> %data, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 0, i32 33, i32 0)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.tbuffer.store.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.tbuffer.store.ll
index 254ad80c2bd9b..24247d320c88a 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.tbuffer.store.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.tbuffer.store.ll
@@ -4,14 +4,14 @@
 
 ; GCN-LABEL: {{^}}tbuffer_store:
 ; GCN: v_mov_b32_e32 [[ZEROREG:v[0-9]+]], 0
-; PREGFX10: tbuffer_store_format_xyzw v[0:3], [[ZEROREG]], s[0:3], dfmt:12, nfmt:2, 0 idxen
-; PREGFX10: tbuffer_store_format_xyzw v[4:7], [[ZEROREG]], s[0:3], dfmt:13, nfmt:3, 0 idxen glc
-; PREGFX10: tbuffer_store_format_xyzw v[8:11], [[ZEROREG]], s[0:3], dfmt:14, nfmt:4, 0 idxen slc
-; PREGFX10: tbuffer_store_format_xyzw v[8:11], [[ZEROREG]], s[0:3], dfmt:14, nfmt:4, 0 idxen glc
-; GFX10: tbuffer_store_format_xyzw v[0:3], [[ZEROREG]], s[0:3], format:44, 0 idxen
-; GFX10: tbuffer_store_format_xyzw v[4:7], [[ZEROREG]], s[0:3], format:61, 0 idxen glc
-; GFX10: tbuffer_store_format_xyzw v[8:11], [[ZEROREG]], s[0:3], format:78, 0 idxen slc
-; GFX10: tbuffer_store_format_xyzw v[8:11], [[ZEROREG]], s[0:3], format:78, 0 idxen glc dlc
+; PREGFX10: tbuffer_store_format_xyzw v[0:3], [[ZEROREG]], s[0:3], 0 format:[BUF_DATA_FORMAT_16_16_16_16,BUF_NUM_FORMAT_USCALED] idxen
+; PREGFX10: tbuffer_store_format_xyzw v[4:7], [[ZEROREG]], s[0:3], 0 format:[BUF_DATA_FORMAT_32_32_32,BUF_NUM_FORMAT_SSCALED] idxen glc
+; PREGFX10: tbuffer_store_format_xyzw v[8:11], [[ZEROREG]], s[0:3], 0 format:[BUF_DATA_FORMAT_32_32_32_32,BUF_NUM_FORMAT_UINT] idxen slc
+; PREGFX10: tbuffer_store_format_xyzw v[8:11], [[ZEROREG]], s[0:3], 0 format:[BUF_DATA_FORMAT_32_32_32_32,BUF_NUM_FORMAT_UINT] idxen glc
+; GFX10: tbuffer_store_format_xyzw v[0:3], [[ZEROREG]], s[0:3], 0 format:[BUF_FMT_10_10_10_2_UNORM] idxen
+; GFX10: tbuffer_store_format_xyzw v[4:7], [[ZEROREG]], s[0:3], 0 format:[BUF_FMT_8_8_8_8_SINT] idxen glc
+; GFX10: tbuffer_store_format_xyzw v[8:11], [[ZEROREG]], s[0:3], 0 format:78 idxen slc
+; GFX10: tbuffer_store_format_xyzw v[8:11], [[ZEROREG]], s[0:3], 0 format:78 idxen glc dlc
 define amdgpu_ps void @tbuffer_store(<4 x i32> inreg, <4 x float>, <4 x float>, <4 x float>) {
 main_body:
   %in1 = bitcast <4 x float> %1 to <4 x i32>
@@ -26,8 +26,8 @@ main_body:
 
 ; GCN-LABEL: {{^}}tbuffer_store_immoffs:
 ; GCN: v_mov_b32_e32 [[ZEROREG:v[0-9]+]], 0
-; PREGFX10: tbuffer_store_format_xyzw v[0:3], [[ZEROREG]], s[0:3], dfmt:5, nfmt:7, 0 idxen offset:42
-; GFX10: tbuffer_store_format_xyzw v[0:3], [[ZEROREG]], s[0:3], format:117, 0 idxen offset:42
+; PREGFX10: tbuffer_store_format_xyzw v[0:3], [[ZEROREG]], s[0:3], 0 format:[BUF_DATA_FORMAT_16_16,BUF_NUM_FORMAT_FLOAT] idxen offset:42
+; GFX10: tbuffer_store_format_xyzw v[0:3], [[ZEROREG]], s[0:3], 0 format:117 idxen offset:42
 define amdgpu_ps void @tbuffer_store_immoffs(<4 x i32> inreg, <4 x float>) {
 main_body:
   %in1 = bitcast <4 x float> %1 to <4 x i32>
@@ -37,8 +37,8 @@ main_body:
 
 ; GCN-LABEL: {{^}}tbuffer_store_scalar_and_imm_offs:
 ; GCN: v_mov_b32_e32 [[ZEROREG:v[0-9]+]], 0
-; PREGFX10: tbuffer_store_format_xyzw v[0:3], [[ZEROREG]], s[0:3], dfmt:5, nfmt:7, {{s[0-9]+}} idxen offset:42
-; GFX10: tbuffer_store_format_xyzw v[0:3], [[ZEROREG]], s[0:3], format:117, {{s[0-9]+}} idxen offset:42
+; PREGFX10: tbuffer_store_format_xyzw v[0:3], [[ZEROREG]], s[0:3], {{s[0-9]+}} format:[BUF_DATA_FORMAT_16_16,BUF_NUM_FORMAT_FLOAT] idxen offset:42
+; GFX10: tbuffer_store_format_xyzw v[0:3], [[ZEROREG]], s[0:3], {{s[0-9]+}} format:117 idxen offset:42
 define amdgpu_ps void @tbuffer_store_scalar_and_imm_offs(<4 x i32> inreg, <4 x float> %vdata, i32 inreg %soffset) {
 main_body:
   %in1 = bitcast <4 x float> %vdata to <4 x i32>
@@ -47,8 +47,8 @@ main_body:
 }
 
 ; GCN-LABEL: {{^}}buffer_store_idx:
-; PREGFX10: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], dfmt:15, nfmt:2, 0 idxen
-; GFX10: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], format:47, 0 idxen
+; PREGFX10: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] idxen
+; GFX10: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_FMT_10_10_10_2_SSCALED] idxen
 define amdgpu_ps void @buffer_store_idx(<4 x i32> inreg, <4 x float> %vdata, i32 %vindex) {
 main_body:
   %in1 = bitcast <4 x float> %vdata to <4 x i32>
@@ -57,8 +57,8 @@ main_body:
 }
 
 ; GCN-LABEL: {{^}}buffer_store_ofs:
-; PREGFX10: tbuffer_store_format_xyzw v[0:3], {{v\[[0-9]+:[0-9]+\]}}, s[0:3], dfmt:3, nfmt:7, 0 idxen offen
-; GFX10: tbuffer_store_format_xyzw v[0:3], {{v\[[0-9]+:[0-9]+\]}}, s[0:3], format:115, 0 idxen offen
+; PREGFX10: tbuffer_store_format_xyzw v[0:3], {{v\[[0-9]+:[0-9]+\]}}, s[0:3], 0 format:[BUF_DATA_FORMAT_8_8,BUF_NUM_FORMAT_FLOAT] idxen offen
+; GFX10: tbuffer_store_format_xyzw v[0:3], {{v\[[0-9]+:[0-9]+\]}}, s[0:3], 0 format:115 idxen offen
 define amdgpu_ps void @buffer_store_ofs(<4 x i32> inreg, <4 x float> %vdata, i32 %voffset) {
 main_body:
   %in1 = bitcast <4 x float> %vdata to <4 x i32>
@@ -67,8 +67,8 @@ main_body:
 }
 
 ; GCN-LABEL: {{^}}buffer_store_both:
-; PREGFX10: tbuffer_store_format_xyzw v[0:3], v[4:5], s[0:3], dfmt:6, nfmt:4, 0 idxen offen
-; GFX10: tbuffer_store_format_xyzw v[0:3], v[4:5], s[0:3], format:70, 0 idxen offen
+; PREGFX10: tbuffer_store_format_xyzw v[0:3], v[4:5], s[0:3], 0 format:[BUF_DATA_FORMAT_10_11_11,BUF_NUM_FORMAT_UINT] idxen offen
+; GFX10: tbuffer_store_format_xyzw v[0:3], v[4:5], s[0:3], 0 format:[BUF_FMT_16_16_16_16_SINT] idxen offen
 define amdgpu_ps void @buffer_store_both(<4 x i32> inreg, <4 x float> %vdata, i32 %vindex, i32 %voffset) {
 main_body:
   %in1 = bitcast <4 x float> %vdata to <4 x i32>
@@ -79,13 +79,13 @@ main_body:
 ; Ideally, the register allocator would avoid the wait here
 ;
 ; GCN-LABEL: {{^}}buffer_store_wait:
-; PREGFX10: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], dfmt:15, nfmt:3, 0 idxen
-; GFX10: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], format:63, 0 idxen
+; PREGFX10: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] idxen
+; GFX10: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_FMT_32_32_SINT] idxen
 ; VERDE: s_waitcnt expcnt(0)
 ; GCN: buffer_load_format_xyzw v[0:3], v5, s[0:3], 0 idxen
 ; GCN: s_waitcnt vmcnt(0)
-; PREGFX10: tbuffer_store_format_xyzw v[0:3], v6, s[0:3], dfmt:14, nfmt:2, 0 idxen
-; GFX10: tbuffer_store_format_xyzw v[0:3], v6, s[0:3], format:46, 0 idxen
+; PREGFX10: tbuffer_store_format_xyzw v[0:3], v6, s[0:3], 0 format:[BUF_DATA_FORMAT_32_32_32_32,BUF_NUM_FORMAT_USCALED] idxen
+; GFX10: tbuffer_store_format_xyzw v[0:3], v6, s[0:3], 0 format:[BUF_FMT_10_10_10_2_USCALED] idxen
 define amdgpu_ps void @buffer_store_wait(<4 x i32> inreg, <4 x float> %vdata, i32 %vindex.1, i32 %vindex.2, i32 %vindex.3) {
 main_body:
   %in1 = bitcast <4 x float> %vdata to <4 x i32>
@@ -97,8 +97,8 @@ main_body:
 }
 
 ; GCN-LABEL: {{^}}buffer_store_x1:
-; PREGFX10: tbuffer_store_format_x v0, v1, s[0:3], dfmt:13, nfmt:7, 0 idxen
-; GFX10: tbuffer_store_format_x v0, v1, s[0:3], format:125, 0 idxen
+; PREGFX10: tbuffer_store_format_x v0, v1, s[0:3], 0 format:[BUF_DATA_FORMAT_32_32_32,BUF_NUM_FORMAT_FLOAT] idxen
+; GFX10: tbuffer_store_format_x v0, v1, s[0:3], 0 format:125 idxen
 define amdgpu_ps void @buffer_store_x1(<4 x i32> inreg %rsrc, float %data, i32 %vindex) {
 main_body:
   %data.i = bitcast float %data to i32
@@ -107,8 +107,8 @@ main_body:
 }
 
 ; GCN-LABEL: {{^}}buffer_store_x2:
-; PREGFX10: tbuffer_store_format_xy v[0:1], v2, s[0:3], dfmt:1, nfmt:2, 0 idxen
-; GFX10: tbuffer_store_format_xy v[0:1], v2, s[0:3], format:33, 0 idxen
+; PREGFX10: tbuffer_store_format_xy v[0:1], v2, s[0:3], 0 format:[BUF_NUM_FORMAT_USCALED] idxen
+; GFX10: tbuffer_store_format_xy v[0:1], v2, s[0:3], 0 format:[BUF_FMT_10_11_11_SSCALED] idxen
 define amdgpu_ps void @buffer_store_x2(<4 x i32> inreg %rsrc, <2 x float> %data, i32 %vindex) {
 main_body:
   %data.i = bitcast <2 x float> %data to <2 x i32>
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.load.d16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.load.d16.ll
index 8850acae1b9cc..205cc5f78d335 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.load.d16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.load.d16.ll
@@ -3,7 +3,7 @@
 ; RUN: llc < %s -march=amdgcn -mcpu=gfx900 -verify-machineinstrs | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=PACKED %s
 
 ; GCN-LABEL: {{^}}tbuffer_load_d16_x:
-; GCN: tbuffer_load_format_d16_x v{{[0-9]+}}, off, s[{{[0-9]+:[0-9]+}}],  dfmt:6,  nfmt:1, 0
+; GCN: tbuffer_load_format_d16_x v{{[0-9]+}}, off, s[{{[0-9]+:[0-9]+}}], 0 format:[BUF_DATA_FORMAT_10_11_11,BUF_NUM_FORMAT_SNORM]
 define amdgpu_ps half @tbuffer_load_d16_x(<4 x i32> inreg %rsrc) {
 main_body:
   %data = call half @llvm.amdgcn.tbuffer.load.f16(<4 x i32> %rsrc, i32 0, i32 0, i32 0, i32 0, i32 6, i32 1, i1 0, i1 0)
@@ -11,10 +11,10 @@ main_body:
 }
 
 ; GCN-LABEL: {{^}}tbuffer_load_d16_xy:
-; UNPACKED: tbuffer_load_format_d16_xy v{{\[}}{{[0-9]+}}:[[HI:[0-9]+]]{{\]}}, off, s[{{[0-9]+:[0-9]+}}],  dfmt:6,  nfmt:1, 0
+; UNPACKED: tbuffer_load_format_d16_xy v{{\[}}{{[0-9]+}}:[[HI:[0-9]+]]{{\]}}, off, s[{{[0-9]+:[0-9]+}}], 0 format:[BUF_DATA_FORMAT_10_11_11,BUF_NUM_FORMAT_SNORM]
 ; UNPACKED: v_mov_b32_e32 v{{[0-9]+}}, v[[HI]]
 
-; PACKED: tbuffer_load_format_d16_xy v[[FULL:[0-9]+]], off, s[{{[0-9]+:[0-9]+}}],  dfmt:6,  nfmt:1, 0
+; PACKED: tbuffer_load_format_d16_xy v[[FULL:[0-9]+]], off, s[{{[0-9]+:[0-9]+}}], 0 format:[BUF_DATA_FORMAT_10_11_11,BUF_NUM_FORMAT_SNORM]
 ; PACKED: v_lshrrev_b32_e32 v{{[0-9]+}}, 16, v[[FULL]]
 define amdgpu_ps half @tbuffer_load_d16_xy(<4 x i32> inreg %rsrc) {
 main_body:
@@ -24,10 +24,10 @@ main_body:
 }
 
 ; GCN-LABEL: {{^}}tbuffer_load_d16_xyzw:
-; UNPACKED: tbuffer_load_format_d16_xyzw v{{\[}}{{[0-9]+}}:[[HI:[0-9]+]]{{\]}}, off, s[{{[0-9]+:[0-9]+}}],  dfmt:6,  nfmt:1, 0
+; UNPACKED: tbuffer_load_format_d16_xyzw v{{\[}}{{[0-9]+}}:[[HI:[0-9]+]]{{\]}}, off, s[{{[0-9]+:[0-9]+}}], 0 format:[BUF_DATA_FORMAT_10_11_11,BUF_NUM_FORMAT_SNORM]
 ; UNPACKED: v_mov_b32_e32 v{{[0-9]+}}, v[[HI]]
 
-; PACKED: tbuffer_load_format_d16_xyzw v{{\[}}{{[0-9]+}}:[[HI:[0-9]+]]{{\]}}, off, s[{{[0-9]+:[0-9]+}}],  dfmt:6,  nfmt:1, 0
+; PACKED: tbuffer_load_format_d16_xyzw v{{\[}}{{[0-9]+}}:[[HI:[0-9]+]]{{\]}}, off, s[{{[0-9]+:[0-9]+}}], 0 format:[BUF_DATA_FORMAT_10_11_11,BUF_NUM_FORMAT_SNORM]
 ; PACKED: v_lshrrev_b32_e32 v{{[0-9]+}}, 16, v[[HI]]
 define amdgpu_ps half @tbuffer_load_d16_xyzw(<4 x i32> inreg %rsrc) {
 main_body:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.load.dwordx3.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.load.dwordx3.ll
index 0193d97398910..b04f3c09729a0 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.load.dwordx3.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.load.dwordx3.ll
@@ -2,8 +2,8 @@
 ;RUN: llc < %s -march=amdgcn -mcpu=gfx700 -verify-machineinstrs | FileCheck %s -check-prefixes=GCN,GCNX3
 
 ; GCN-LABEL: {{^}}tbuffer_raw_load_immoffs_x3:
-; SI: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, dfmt:14, nfmt:4, 0 offset:42
-; GCNX3: tbuffer_load_format_xyz {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, dfmt:14, nfmt:4, 0 offset:42
+; SI: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 format:[BUF_DATA_FORMAT_32_32_32_32,BUF_NUM_FORMAT_UINT] offset:42
+; GCNX3: tbuffer_load_format_xyz {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 format:[BUF_DATA_FORMAT_32_32_32_32,BUF_NUM_FORMAT_UINT] offset:42
 define amdgpu_vs <3 x float> @tbuffer_raw_load_immoffs_x3(<4 x i32> inreg) {
 main_body:
     %vdata   = call <3 x i32> @llvm.amdgcn.raw.tbuffer.load.v3i32(<4 x i32> %0, i32 42, i32 0, i32 78, i32 0)
@@ -14,8 +14,8 @@ main_body:
 
 ; GCN-LABEL: {{^}}tbuffer_struct_load_immoffs_x3:
 ; GCN: v_mov_b32_e32 [[ZEROREG:v[0-9]+]], 0
-; SI: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, [[ZEROREG]], {{s\[[0-9]+:[0-9]+\]}}, dfmt:14, nfmt:4, 0 idxen offset:42
-; GCNX3: tbuffer_load_format_xyz {{v\[[0-9]+:[0-9]+\]}}, [[ZEROREG]], {{s\[[0-9]+:[0-9]+\]}}, dfmt:14, nfmt:4, 0 idxen offset:42
+; SI: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, [[ZEROREG]], {{s\[[0-9]+:[0-9]+\]}}, 0 format:[BUF_DATA_FORMAT_32_32_32_32,BUF_NUM_FORMAT_UINT] idxen offset:42
+; GCNX3: tbuffer_load_format_xyz {{v\[[0-9]+:[0-9]+\]}}, [[ZEROREG]], {{s\[[0-9]+:[0-9]+\]}}, 0 format:[BUF_DATA_FORMAT_32_32_32_32,BUF_NUM_FORMAT_UINT] idxen offset:42
 define amdgpu_vs <3 x float> @tbuffer_struct_load_immoffs_x3(<4 x i32> inreg) {
 main_body:
     %vdata   = call <3 x i32> @llvm.amdgcn.struct.tbuffer.load.v3i32(<4 x i32> %0, i32 0, i32 42, i32 0, i32 78, i32 0)
@@ -25,8 +25,8 @@ main_body:
 
 
 ; GCN-LABEL: {{^}}tbuffer_load_format_immoffs_x3:
-; SI: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, dfmt:14, nfmt:4, 0 offset:42
-; GCNX3: tbuffer_load_format_xyz {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, dfmt:14, nfmt:4, 0 offset:42
+; SI: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 format:[BUF_DATA_FORMAT_32_32_32_32,BUF_NUM_FORMAT_UINT] offset:42
+; GCNX3: tbuffer_load_format_xyz {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 format:[BUF_DATA_FORMAT_32_32_32_32,BUF_NUM_FORMAT_UINT] offset:42
 define amdgpu_vs <3 x float> @tbuffer_load_format_immoffs_x3(<4 x i32> inreg) {
 main_body:
     %vdata   = call <3 x i32> @llvm.amdgcn.tbuffer.load.v3i32(<4 x i32> %0, i32 0, i32 0, i32 0, i32 42, i32 14, i32 4, i1 0, i1 0)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.load.ll
index 712ee7ad1e5cb..aca9e0aaf9a26 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.load.ll
@@ -2,10 +2,10 @@
 ;RUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck -check-prefix=GCN %s
 
 ; GCN-LABEL: {{^}}tbuffer_load:
-; GCN: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, dfmt:14, nfmt:4, 0
-; GCN: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, dfmt:15, nfmt:3, 0 glc
-; GCN: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, dfmt:6, nfmt:1, 0 slc
-; GCN: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, dfmt:6, nfmt:1, 0
+; GCN: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 format:[BUF_DATA_FORMAT_32_32_32_32,BUF_NUM_FORMAT_UINT]
+; GCN: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] glc
+; GCN: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 format:[BUF_DATA_FORMAT_10_11_11,BUF_NUM_FORMAT_SNORM] slc
+; GCN: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 format:[BUF_DATA_FORMAT_10_11_11,BUF_NUM_FORMAT_SNORM]
 ; GCN: s_waitcnt
 define amdgpu_vs {<4 x float>, <4 x float>, <4 x float>, <4 x float>} @tbuffer_load(<4 x i32> inreg) {
 main_body:
@@ -24,7 +24,7 @@ main_body:
 }
 
 ; GCN-LABEL: {{^}}tbuffer_load_immoffs:
-; GCN: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, dfmt:14, nfmt:4, 0 offset:42
+; GCN: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 format:[BUF_DATA_FORMAT_32_32_32_32,BUF_NUM_FORMAT_UINT] offset:42
 define amdgpu_vs <4 x float> @tbuffer_load_immoffs(<4 x i32> inreg) {
 main_body:
     %vdata   = call <4 x i32> @llvm.amdgcn.tbuffer.load.v4i32(<4 x i32> %0, i32 0, i32 0, i32 0, i32 42, i32 14, i32 4, i1 0, i1 0)
@@ -33,9 +33,9 @@ main_body:
 }
 
 ; GCN-LABEL: {{^}}tbuffer_load_immoffs_large
-; GCN: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, dfmt:15, nfmt:2, 61 offset:4095
-; GCN: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, dfmt:14, nfmt:3, {{s[0-9]+}} offset:73
-; GCN: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, dfmt:13, nfmt:4, {{s[0-9]+}} offset:1
+; GCN: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, 61 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] offset:4095
+; GCN: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, {{s[0-9]+}} format:[BUF_DATA_FORMAT_32_32_32_32,BUF_NUM_FORMAT_SSCALED] offset:73
+; GCN: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, {{s[0-9]+}} format:[BUF_DATA_FORMAT_32_32_32,BUF_NUM_FORMAT_UINT] offset:1
 ; GCN: s_waitcnt
 define amdgpu_vs {<4 x float>, <4 x float>, <4 x float>} @tbuffer_load_immoffs_large(<4 x i32> inreg, i32 inreg %soffs) {
     %vdata     = call <4 x i32> @llvm.amdgcn.tbuffer.load.v4i32(<4 x i32> %0, i32 0, i32 0, i32 61, i32 4095, i32 15, i32 2, i1 0, i1 0)
@@ -51,7 +51,7 @@ define amdgpu_vs {<4 x float>, <4 x float>, <4 x float>} @tbuffer_load_immoffs_l
 }
 
 ; GCN-LABEL: {{^}}tbuffer_load_idx:
-; GCN: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, dfmt:14, nfmt:4, 0 idxen
+; GCN: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, 0 format:[BUF_DATA_FORMAT_32_32_32_32,BUF_NUM_FORMAT_UINT] idxen
 define amdgpu_vs <4 x float> @tbuffer_load_idx(<4 x i32> inreg, i32 %vindex) {
 main_body:
     %vdata   = call <4 x i32> @llvm.amdgcn.tbuffer.load.v4i32(<4 x i32> %0, i32 %vindex, i32 0, i32 0, i32 0, i32 14, i32 4, i1 0, i1 0)
@@ -60,7 +60,7 @@ main_body:
 }
 
 ; GCN-LABEL: {{^}}tbuffer_load_ofs:
-; GCN: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, dfmt:14, nfmt:4, 0 offen
+; GCN: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, 0 format:[BUF_DATA_FORMAT_32_32_32_32,BUF_NUM_FORMAT_UINT] offen
 define amdgpu_vs <4 x float> @tbuffer_load_ofs(<4 x i32> inreg, i32 %voffs) {
 main_body:
     %vdata   = call <4 x i32> @llvm.amdgcn.tbuffer.load.v4i32(<4 x i32> %0, i32 0, i32 %voffs, i32 0, i32 0, i32 14, i32 4, i1 0, i1 0)
@@ -69,7 +69,7 @@ main_body:
 }
 
 ; GCN-LABEL: {{^}}tbuffer_load_ofs_imm:
-; GCN: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, dfmt:14, nfmt:4, 0 offen offset:52
+; GCN: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, 0 format:[BUF_DATA_FORMAT_32_32_32_32,BUF_NUM_FORMAT_UINT] offen offset:52
 define amdgpu_vs <4 x float> @tbuffer_load_ofs_imm(<4 x i32> inreg, i32 %voffs) {
 main_body:
     %vdata   = call <4 x i32> @llvm.amdgcn.tbuffer.load.v4i32(<4 x i32> %0, i32 0, i32 %voffs, i32 0, i32 52, i32 14, i32 4, i1 0, i1 0)
@@ -78,7 +78,7 @@ main_body:
 }
 
 ; GCN-LABEL: {{^}}tbuffer_load_both:
-; GCN: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, dfmt:14, nfmt:4, 0 idxen offen
+; GCN: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 format:[BUF_DATA_FORMAT_32_32_32_32,BUF_NUM_FORMAT_UINT] idxen offen
 define amdgpu_vs <4 x float> @tbuffer_load_both(<4 x i32> inreg, i32 %vindex, i32 %voffs) {
 main_body:
     %vdata   = call <4 x i32> @llvm.amdgcn.tbuffer.load.v4i32(<4 x i32> %0, i32 %vindex, i32 %voffs, i32 0, i32 0, i32 14, i32 4, i1 0, i1 0)
@@ -88,7 +88,7 @@ main_body:
 
 
 ; GCN-LABEL: {{^}}buffer_load_xy:
-; GCN: tbuffer_load_format_xy {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, dfmt:13, nfmt:4, 0
+; GCN: tbuffer_load_format_xy {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 format:[BUF_DATA_FORMAT_32_32_32,BUF_NUM_FORMAT_UINT]
 define amdgpu_vs <2 x float> @buffer_load_xy(<4 x i32> inreg %rsrc) {
     %vdata = call <2 x i32> @llvm.amdgcn.tbuffer.load.v2i32(<4 x i32> %rsrc, i32 0, i32 0, i32 0, i32 0, i32 13, i32 4, i1 0, i1 0)
     %vdata.f = bitcast <2 x i32> %vdata to <2 x float>
@@ -96,7 +96,7 @@ define amdgpu_vs <2 x float> @buffer_load_xy(<4 x i32> inreg %rsrc) {
 }
 
 ; GCN-LABEL: {{^}}buffer_load_x:
-; GCN: tbuffer_load_format_x {{v[0-9]+}}, off, {{s\[[0-9]+:[0-9]+\]}}, dfmt:13, nfmt:4, 0
+; GCN: tbuffer_load_format_x {{v[0-9]+}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 format:[BUF_DATA_FORMAT_32_32_32,BUF_NUM_FORMAT_UINT]
 define amdgpu_vs float @buffer_load_x(<4 x i32> inreg %rsrc) {
     %vdata = call i32 @llvm.amdgcn.tbuffer.load.i32(<4 x i32> %rsrc, i32 0, i32 0, i32 0, i32 0, i32 13, i32 4, i1 0, i1 0)
     %vdata.f = bitcast i32 %vdata to float
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.store.d16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.store.d16.ll
index 85aaee3dd2e38..4dd76a3a632dc 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.store.d16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.store.d16.ll
@@ -6,7 +6,7 @@
 ; GCN-LABEL: {{^}}tbuffer_store_d16_x:
 ; GCN: s_load_dword s[[S_LO:[0-9]+]]
 ; GCN: v_mov_b32_e32 v[[V_LO:[0-9]+]], s[[S_LO]]
-; GCN: tbuffer_store_format_d16_x v[[V_LO]], v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}],  dfmt:1,  nfmt:2, 0 idxen
+; GCN: tbuffer_store_format_d16_x v[[V_LO]], v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], 0 format:[BUF_NUM_FORMAT_USCALED] idxen
 define amdgpu_kernel void @tbuffer_store_d16_x(<4 x i32> %rsrc, [8 x i32], half %data, [8 x i32], i32 %vindex) {
 main_body:
   call void @llvm.amdgcn.tbuffer.store.f16(half %data, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 0, i32 0, i32 1, i32 2, i1 0, i1 0)
@@ -19,9 +19,9 @@ main_body:
 ; UNPACKED-DAG: s_and_b32 [[MASKED:s[0-9]+]], [[S_DATA]], 0xffff{{$}}
 ; UNPACKED-DAG: v_mov_b32_e32 v[[V_LO:[0-9]+]], [[MASKED]]
 ; UNPACKED-DAG: v_mov_b32_e32 v[[V_HI:[0-9]+]], [[SHR]]
-; UNPACKED: tbuffer_store_format_d16_xy v{{\[}}[[V_LO]]:[[V_HI]]{{\]}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}],  dfmt:1,  nfmt:2, 0 idxen
+; UNPACKED: tbuffer_store_format_d16_xy v{{\[}}[[V_LO]]:[[V_HI]]{{\]}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], 0 format:[BUF_NUM_FORMAT_USCALED] idxen
 
-; PACKED: tbuffer_store_format_d16_xy v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}],  dfmt:1,  nfmt:2, 0 idxen
+; PACKED: tbuffer_store_format_d16_xy v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], 0 format:[BUF_NUM_FORMAT_USCALED] idxen
 define amdgpu_kernel void @tbuffer_store_d16_xy(<4 x i32> %rsrc, <2 x half> %data, i32 %vindex) {
 main_body:
   call void @llvm.amdgcn.tbuffer.store.v2f16(<2 x half> %data, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 0, i32 0, i32 1, i32 2, i1 0, i1 0)
@@ -39,11 +39,11 @@ main_body:
 
 ; UNPACKED-DAG: v_mov_b32_e32 v[[LO:[0-9]+]], [[MASKED0]]
 ; UNPACKED-DAG: v_mov_b32_e32 v[[HI:[0-9]+]], [[SHR1]]
-; UNPACKED: tbuffer_store_format_d16_xyzw v{{\[}}[[LO]]:[[HI]]{{\]}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}],  dfmt:1,  nfmt:2, 0 idxen
+; UNPACKED: tbuffer_store_format_d16_xyzw v{{\[}}[[LO]]:[[HI]]{{\]}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], 0 format:[BUF_NUM_FORMAT_USCALED] idxen
 
 ; PACKED-DAG: v_mov_b32_e32 v[[LO:[0-9]+]], s[[S_DATA_0]]
 ; PACKED-DAG: v_mov_b32_e32 v[[HI:[0-9]+]], s[[S_DATA_1]]
-; PACKED: tbuffer_store_format_d16_xyzw v{{\[}}[[LO]]:[[HI]]{{\]}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}],  dfmt:1,  nfmt:2, 0 idxen
+; PACKED: tbuffer_store_format_d16_xyzw v{{\[}}[[LO]]:[[HI]]{{\]}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], 0 format:[BUF_NUM_FORMAT_USCALED] idxen
 define amdgpu_kernel void @tbuffer_store_d16_xyzw(<4 x i32> %rsrc, <4 x half> %data, i32 %vindex) {
 main_body:
   call void @llvm.amdgcn.tbuffer.store.v4f16(<4 x half> %data, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 0, i32 0, i32 1, i32 2, i1 0, i1 0)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.store.dwordx3.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.store.dwordx3.ll
index a39614c1cf052..dc5abf418131c 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.store.dwordx3.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.store.dwordx3.ll
@@ -1,7 +1,7 @@
 ;RUN: llc < %s -march=amdgcn -mcpu=gfx700 -verify-machineinstrs | FileCheck %s -check-prefixes=GCN
 
 ; GCN-LABEL: {{^}}tbuffer_raw_store_immoffs_x3:
-; GCN: tbuffer_store_format_xyz v[0:2], off, s[0:3], dfmt:5, nfmt:7, 0 offset:42
+; GCN: tbuffer_store_format_xyz v[0:2], off, s[0:3], 0 format:[BUF_DATA_FORMAT_16_16,BUF_NUM_FORMAT_FLOAT] offset:42
 define amdgpu_ps void @tbuffer_raw_store_immoffs_x3(<4 x i32> inreg, <3 x float>) {
 main_body:
   %in1 = bitcast <3 x float> %1 to <3 x i32>
@@ -12,7 +12,7 @@ main_body:
 
 ; GCN-LABEL: {{^}}tbuffer_struct_store_immoffs_x3:
 ; GCN: v_mov_b32_e32 [[ZEROREG:v[0-9]+]], 0
-; GCN: tbuffer_store_format_xyz v[0:2], [[ZEROREG]], s[0:3], dfmt:5, nfmt:7, 0 idxen offset:42
+; GCN: tbuffer_store_format_xyz v[0:2], [[ZEROREG]], s[0:3], 0 format:[BUF_DATA_FORMAT_16_16,BUF_NUM_FORMAT_FLOAT] idxen offset:42
 define amdgpu_ps void @tbuffer_struct_store_immoffs_x3(<4 x i32> inreg, <3 x float>) {
 main_body:
   %in1 = bitcast <3 x float> %1 to <3 x i32>
@@ -21,7 +21,7 @@ main_body:
 }
 
 ; GCN-LABEL: {{^}}tbuffer_store_immoffs_x3:
-; GCN: tbuffer_store_format_xyz v[0:2], off, s[0:3], dfmt:5, nfmt:7, 0 offset:42
+; GCN: tbuffer_store_format_xyz v[0:2], off, s[0:3], 0 format:[BUF_DATA_FORMAT_16_16,BUF_NUM_FORMAT_FLOAT] offset:42
 define amdgpu_ps void @tbuffer_store_immoffs_x3(<4 x i32> inreg, <3 x float>) {
 main_body:
   %in1 = bitcast <3 x float> %1 to <3 x i32>
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.store.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.store.ll
index 4caa8081530ba..ed31ea3c40fea 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.store.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.store.ll
@@ -2,10 +2,10 @@
 ;RUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck -check-prefix=GCN %s
 
 ; GCN-LABEL: {{^}}tbuffer_store:
-; GCN: tbuffer_store_format_xyzw v[0:3], off, s[0:3], dfmt:12, nfmt:2, 0
-; GCN: tbuffer_store_format_xyzw v[4:7], off, s[0:3], dfmt:13, nfmt:3, 0 glc
-; GCN: tbuffer_store_format_xyzw v[8:11], off, s[0:3], dfmt:14, nfmt:4, 0 slc
-; GCN: tbuffer_store_format_xyzw v[8:11], off, s[0:3], dfmt:14, nfmt:4, 0
+; GCN: tbuffer_store_format_xyzw v[0:3], off, s[0:3], 0 format:[BUF_DATA_FORMAT_16_16_16_16,BUF_NUM_FORMAT_USCALED]
+; GCN: tbuffer_store_format_xyzw v[4:7], off, s[0:3], 0 format:[BUF_DATA_FORMAT_32_32_32,BUF_NUM_FORMAT_SSCALED] glc
+; GCN: tbuffer_store_format_xyzw v[8:11], off, s[0:3], 0 format:[BUF_DATA_FORMAT_32_32_32_32,BUF_NUM_FORMAT_UINT] slc
+; GCN: tbuffer_store_format_xyzw v[8:11], off, s[0:3], 0 format:[BUF_DATA_FORMAT_32_32_32_32,BUF_NUM_FORMAT_UINT]
 define amdgpu_ps void @tbuffer_store(<4 x i32> inreg, <4 x float>, <4 x float>, <4 x float>) {
 main_body:
   %in1 = bitcast <4 x float> %1 to <4 x i32>
@@ -19,7 +19,7 @@ main_body:
 }
 
 ; GCN-LABEL: {{^}}tbuffer_store_immoffs:
-; GCN: tbuffer_store_format_xyzw v[0:3], off, s[0:3], dfmt:5, nfmt:7, 0 offset:42
+; GCN: tbuffer_store_format_xyzw v[0:3], off, s[0:3], 0 format:[BUF_DATA_FORMAT_16_16,BUF_NUM_FORMAT_FLOAT] offset:42
 define amdgpu_ps void @tbuffer_store_immoffs(<4 x i32> inreg, <4 x float>) {
 main_body:
   %in1 = bitcast <4 x float> %1 to <4 x i32>
@@ -28,7 +28,7 @@ main_body:
 }
 
 ; GCN-LABEL: {{^}}tbuffer_store_scalar_and_imm_offs:
-; GCN: tbuffer_store_format_xyzw v[0:3], off, s[0:3], dfmt:5, nfmt:7, {{s[0-9]+}} offset:42
+; GCN: tbuffer_store_format_xyzw v[0:3], off, s[0:3], {{s[0-9]+}} format:[BUF_DATA_FORMAT_16_16,BUF_NUM_FORMAT_FLOAT] offset:42
 define amdgpu_ps void @tbuffer_store_scalar_and_imm_offs(<4 x i32> inreg, <4 x float> %vdata, i32 inreg %soffset) {
 main_body:
   %in1 = bitcast <4 x float> %vdata to <4 x i32>
@@ -37,7 +37,7 @@ main_body:
 }
 
 ; GCN-LABEL: {{^}}buffer_store_idx:
-; GCN: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], dfmt:15, nfmt:2, 0 idxen
+; GCN: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] idxen
 define amdgpu_ps void @buffer_store_idx(<4 x i32> inreg, <4 x float> %vdata, i32 %vindex) {
 main_body:
   %in1 = bitcast <4 x float> %vdata to <4 x i32>
@@ -46,7 +46,7 @@ main_body:
 }
 
 ; GCN-LABEL: {{^}}buffer_store_ofs:
-; GCN: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], dfmt:3, nfmt:7, 0 offen
+; GCN: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_DATA_FORMAT_8_8,BUF_NUM_FORMAT_FLOAT] offen
 define amdgpu_ps void @buffer_store_ofs(<4 x i32> inreg, <4 x float> %vdata, i32 %voffset) {
 main_body:
   %in1 = bitcast <4 x float> %vdata to <4 x i32>
@@ -55,7 +55,7 @@ main_body:
 }
 
 ; GCN-LABEL: {{^}}buffer_store_both:
-; GCN: tbuffer_store_format_xyzw v[0:3], v[4:5], s[0:3], dfmt:6, nfmt:4, 0 idxen offen
+; GCN: tbuffer_store_format_xyzw v[0:3], v[4:5], s[0:3], 0 format:[BUF_DATA_FORMAT_10_11_11,BUF_NUM_FORMAT_UINT] idxen offen
 define amdgpu_ps void @buffer_store_both(<4 x i32> inreg, <4 x float> %vdata, i32 %vindex, i32 %voffset) {
 main_body:
   %in1 = bitcast <4 x float> %vdata to <4 x i32>
@@ -66,11 +66,11 @@ main_body:
 ; Ideally, the register allocator would avoid the wait here
 ;
 ; GCN-LABEL: {{^}}buffer_store_wait:
-; GCN: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], dfmt:15, nfmt:3, 0 idxen
+; GCN: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] idxen
 ; VERDE: s_waitcnt expcnt(0)
 ; GCN: buffer_load_format_xyzw v[0:3], v5, s[0:3], 0 idxen
 ; GCN: s_waitcnt vmcnt(0)
-; GCN: tbuffer_store_format_xyzw v[0:3], v6, s[0:3], dfmt:14, nfmt:2, 0 idxen
+; GCN: tbuffer_store_format_xyzw v[0:3], v6, s[0:3], 0 format:[BUF_DATA_FORMAT_32_32_32_32,BUF_NUM_FORMAT_USCALED] idxen
 define amdgpu_ps void @buffer_store_wait(<4 x i32> inreg, <4 x float> %vdata, i32 %vindex.1, i32 %vindex.2, i32 %vindex.3) {
 main_body:
   %in1 = bitcast <4 x float> %vdata to <4 x i32>
@@ -82,7 +82,7 @@ main_body:
 }
 
 ; GCN-LABEL: {{^}}buffer_store_x1:
-; GCN: tbuffer_store_format_x v0, v1, s[0:3], dfmt:13, nfmt:7, 0 idxen
+; GCN: tbuffer_store_format_x v0, v1, s[0:3], 0 format:[BUF_DATA_FORMAT_32_32_32,BUF_NUM_FORMAT_FLOAT] idxen
 define amdgpu_ps void @buffer_store_x1(<4 x i32> inreg %rsrc, float %data, i32 %vindex) {
 main_body:
   %data.i = bitcast float %data to i32
@@ -91,7 +91,7 @@ main_body:
 }
 
 ; GCN-LABEL: {{^}}buffer_store_x2:
-; GCN: tbuffer_store_format_xy v[0:1], v2, s[0:3], dfmt:1, nfmt:2, 0 idxen
+; GCN: tbuffer_store_format_xy v[0:1], v2, s[0:3], 0 format:[BUF_NUM_FORMAT_USCALED] idxen
 define amdgpu_ps void @buffer_store_x2(<4 x i32> inreg %rsrc, <2 x float> %data, i32 %vindex) {
 main_body:
   %data.i = bitcast <2 x float> %data to <2 x i32>
diff --git a/llvm/test/MC/AMDGPU/buf-fmt-d16-packed.s b/llvm/test/MC/AMDGPU/buf-fmt-d16-packed.s
index 108e709e29e0b..196dcada2ebea 100644
--- a/llvm/test/MC/AMDGPU/buf-fmt-d16-packed.s
+++ b/llvm/test/MC/AMDGPU/buf-fmt-d16-packed.s
@@ -44,31 +44,31 @@ buffer_store_format_d16_xyzw v[1:2], off, s[4:7], s1
 //===----------------------------------------------------------------------===//
 
 tbuffer_load_format_d16_x v1, off, s[4:7], dfmt:15, nfmt:2, s1
-// PACKED: tbuffer_load_format_d16_x v1, off, s[4:7],  dfmt:15,  nfmt:2, s1 ; encoding: [0x00,0x00,0x7c,0xe9,0x00,0x01,0x01,0x01]
+// PACKED: tbuffer_load_format_d16_x v1, off, s[4:7], s1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] ; encoding: [0x00,0x00,0x7c,0xe9,0x00,0x01,0x01,0x01]
 
 tbuffer_load_format_d16_xy v1, off, s[4:7], dfmt:15, nfmt:2, s1
-// PACKED: tbuffer_load_format_d16_xy v1, off, s[4:7],  dfmt:15,  nfmt:2, s1 ; encoding: [0x00,0x80,0x7c,0xe9,0x00,0x01,0x01,0x01]
+// PACKED: tbuffer_load_format_d16_xy v1, off, s[4:7], s1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] ; encoding: [0x00,0x80,0x7c,0xe9,0x00,0x01,0x01,0x01]
 // UNPACKED-ERR: error: instruction not supported on this GPU
 
 tbuffer_load_format_d16_xyz v[1:2], off, s[4:7], dfmt:15, nfmt:2, s1
-// PACKED: tbuffer_load_format_d16_xyz v[1:2], off, s[4:7],  dfmt:15,  nfmt:2, s1 ; encoding: [0x00,0x00,0x7d,0xe9,0x00,0x01,0x01,0x01]
+// PACKED: tbuffer_load_format_d16_xyz v[1:2], off, s[4:7], s1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] ; encoding: [0x00,0x00,0x7d,0xe9,0x00,0x01,0x01,0x01]
 // UNPACKED-ERR: error: instruction not supported on this GPU
 
 tbuffer_load_format_d16_xyzw v[1:2], off, s[4:7], dfmt:15, nfmt:2, s1
-// PACKED: tbuffer_load_format_d16_xyzw v[1:2], off, s[4:7],  dfmt:15,  nfmt:2, s1 ; encoding: [0x00,0x80,0x7d,0xe9,0x00,0x01,0x01,0x01]
+// PACKED: tbuffer_load_format_d16_xyzw v[1:2], off, s[4:7], s1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] ; encoding: [0x00,0x80,0x7d,0xe9,0x00,0x01,0x01,0x01]
 // UNPACKED-ERR: error: instruction not supported on this GPU
 
 tbuffer_store_format_d16_x v1, off, s[4:7], dfmt:15, nfmt:2, s1
-// PACKED: tbuffer_store_format_d16_x v1, off, s[4:7],  dfmt:15,  nfmt:2, s1 ; encoding: [0x00,0x00,0x7e,0xe9,0x00,0x01,0x01,0x01]
+// PACKED: tbuffer_store_format_d16_x v1, off, s[4:7], s1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] ; encoding: [0x00,0x00,0x7e,0xe9,0x00,0x01,0x01,0x01]
 
 tbuffer_store_format_d16_xy v1, off, s[4:7], dfmt:15, nfmt:2, s1
-// PACKED: tbuffer_store_format_d16_xy v1, off, s[4:7],  dfmt:15,  nfmt:2, s1 ; encoding: [0x00,0x80,0x7e,0xe9,0x00,0x01,0x01,0x01]
+// PACKED: tbuffer_store_format_d16_xy v1, off, s[4:7], s1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] ; encoding: [0x00,0x80,0x7e,0xe9,0x00,0x01,0x01,0x01]
 // UNPACKED-ERR: error: instruction not supported on this GPU
 
 tbuffer_store_format_d16_xyz v[1:2], off, s[4:7], dfmt:15, nfmt:2, s1
-// PACKED: tbuffer_store_format_d16_xyz v[1:2], off, s[4:7],  dfmt:15,  nfmt:2, s1 ; encoding: [0x00,0x00,0x7f,0xe9,0x00,0x01,0x01,0x01]
+// PACKED: tbuffer_store_format_d16_xyz v[1:2], off, s[4:7], s1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] ; encoding: [0x00,0x00,0x7f,0xe9,0x00,0x01,0x01,0x01]
 // UNPACKED-ERR: error: instruction not supported on this GPU
 
 tbuffer_store_format_d16_xyzw v[1:2], off, s[4:7], dfmt:15, nfmt:2, s1
-// PACKED: tbuffer_store_format_d16_xyzw v[1:2], off, s[4:7],  dfmt:15,  nfmt:2, s1 ; encoding: [0x00,0x80,0x7f,0xe9,0x00,0x01,0x01,0x01]
+// PACKED: tbuffer_store_format_d16_xyzw v[1:2], off, s[4:7], s1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] ; encoding: [0x00,0x80,0x7f,0xe9,0x00,0x01,0x01,0x01]
 // UNPACKED-ERR: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/buf-fmt-d16-unpacked.s b/llvm/test/MC/AMDGPU/buf-fmt-d16-unpacked.s
index 8db9f491e035a..15cfb225b8b55 100644
--- a/llvm/test/MC/AMDGPU/buf-fmt-d16-unpacked.s
+++ b/llvm/test/MC/AMDGPU/buf-fmt-d16-unpacked.s
@@ -43,31 +43,31 @@ buffer_store_format_d16_xyzw v[1:4], off, s[4:7], s1
 //===----------------------------------------------------------------------===//
 
 tbuffer_load_format_d16_x v1, off, s[4:7], dfmt:15, nfmt:2, s1
-// UNPACKED: tbuffer_load_format_d16_x v1, off, s[4:7],  dfmt:15,  nfmt:2, s1 ; encoding: [0x00,0x00,0x7c,0xe9,0x00,0x01,0x01,0x01]
+// UNPACKED: tbuffer_load_format_d16_x v1, off, s[4:7], s1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] ; encoding: [0x00,0x00,0x7c,0xe9,0x00,0x01,0x01,0x01]
 
 tbuffer_load_format_d16_xy v[1:2], off, s[4:7], dfmt:15, nfmt:2, s1
-// UNPACKED: tbuffer_load_format_d16_xy v[1:2], off, s[4:7],  dfmt:15,  nfmt:2, s1 ; encoding: [0x00,0x80,0x7c,0xe9,0x00,0x01,0x01,0x01]
+// UNPACKED: tbuffer_load_format_d16_xy v[1:2], off, s[4:7], s1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] ; encoding: [0x00,0x80,0x7c,0xe9,0x00,0x01,0x01,0x01]
 // PACKED-ERR: error: instruction not supported on this GPU
 
 tbuffer_load_format_d16_xyz v[1:3], off, s[4:7], dfmt:15, nfmt:2, s1
-// UNPACKED: tbuffer_load_format_d16_xyz v[1:3], off, s[4:7],  dfmt:15,  nfmt:2, s1 ; encoding: [0x00,0x00,0x7d,0xe9,0x00,0x01,0x01,0x01]
+// UNPACKED: tbuffer_load_format_d16_xyz v[1:3], off, s[4:7], s1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] ; encoding: [0x00,0x00,0x7d,0xe9,0x00,0x01,0x01,0x01]
 // PACKED-ERR: error: instruction not supported on this GPU
 
 tbuffer_load_format_d16_xyzw v[1:4], off, s[4:7], dfmt:15, nfmt:2, s1
-// UNPACKED: tbuffer_load_format_d16_xyzw v[1:4], off, s[4:7],  dfmt:15,  nfmt:2, s1 ; encoding: [0x00,0x80,0x7d,0xe9,0x00,0x01,0x01,0x01]
+// UNPACKED: tbuffer_load_format_d16_xyzw v[1:4], off, s[4:7], s1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] ; encoding: [0x00,0x80,0x7d,0xe9,0x00,0x01,0x01,0x01]
 // PACKED-ERR: error: instruction not supported on this GPU
 
 tbuffer_store_format_d16_x v1, off, s[4:7], dfmt:15, nfmt:2, s1
-// UNPACKED: tbuffer_store_format_d16_x v1, off, s[4:7],  dfmt:15,  nfmt:2, s1 ; encoding: [0x00,0x00,0x7e,0xe9,0x00,0x01,0x01,0x01]
+// UNPACKED: tbuffer_store_format_d16_x v1, off, s[4:7], s1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] ; encoding: [0x00,0x00,0x7e,0xe9,0x00,0x01,0x01,0x01]
 
 tbuffer_store_format_d16_xy v[1:2], off, s[4:7], dfmt:15, nfmt:2, s1
-// UNPACKED: tbuffer_store_format_d16_xy v[1:2], off, s[4:7],  dfmt:15,  nfmt:2, s1 ; encoding: [0x00,0x80,0x7e,0xe9,0x00,0x01,0x01,0x01]
+// UNPACKED: tbuffer_store_format_d16_xy v[1:2], off, s[4:7], s1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] ; encoding: [0x00,0x80,0x7e,0xe9,0x00,0x01,0x01,0x01]
 // PACKED-ERR: error: instruction not supported on this GPU
 
 tbuffer_store_format_d16_xyz v[1:3], off, s[4:7], dfmt:15, nfmt:2, s1
-// UNPACKED: tbuffer_store_format_d16_xyz v[1:3], off, s[4:7],  dfmt:15,  nfmt:2, s1 ; encoding: [0x00,0x00,0x7f,0xe9,0x00,0x01,0x01,0x01]
+// UNPACKED: tbuffer_store_format_d16_xyz v[1:3], off, s[4:7], s1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] ; encoding: [0x00,0x00,0x7f,0xe9,0x00,0x01,0x01,0x01]
 // PACKED-ERR: error: instruction not supported on this GPU
 
 tbuffer_store_format_d16_xyzw v[1:4], off, s[4:7], dfmt:15, nfmt:2, s1
-// UNPACKED: tbuffer_store_format_d16_xyzw v[1:4], off, s[4:7],  dfmt:15,  nfmt:2, s1 ; encoding: [0x00,0x80,0x7f,0xe9,0x00,0x01,0x01,0x01]
+// UNPACKED: tbuffer_store_format_d16_xyzw v[1:4], off, s[4:7], s1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] ; encoding: [0x00,0x80,0x7f,0xe9,0x00,0x01,0x01,0x01]
 // PACKED-ERR: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/mtbuf-gfx10.s b/llvm/test/MC/AMDGPU/mtbuf-gfx10.s
index 40f082d02ebce..8ea86e7de9657 100644
--- a/llvm/test/MC/AMDGPU/mtbuf-gfx10.s
+++ b/llvm/test/MC/AMDGPU/mtbuf-gfx10.s
@@ -1,80 +1,116 @@
 // RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1010 -show-encoding %s | FileCheck -check-prefix=GCN -check-prefix=GFX10 %s
 // RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1010 -show-encoding %s 2>&1 | FileCheck -check-prefix=GFX10-ERR %s
 
-// GFX10: tbuffer_load_format_d16_x v0, off, s[0:3], format:22, 0 ; encoding: [0x00,0x00,0xb0,0xe8,0x00,0x00,0x20,0x80]
+//===----------------------------------------------------------------------===//
+// Positive tests for legacy format syntax.
+//===----------------------------------------------------------------------===//
+
+// GFX10: tbuffer_load_format_d16_x v0, off, s[0:3], 0 format:[BUF_FMT_32_FLOAT] ; encoding: [0x00,0x00,0xb0,0xe8,0x00,0x00,0x20,0x80]
 tbuffer_load_format_d16_x v0, off, s[0:3], format:22, 0
-// GFX10: tbuffer_load_format_d16_xy v0, off, s[0:3], format:22, 0 ; encoding: [0x00,0x00,0xb1,0xe8,0x00,0x00,0x20,0x80]
+
+// GFX10: tbuffer_load_format_d16_xy v0, off, s[0:3], 0 format:[BUF_FMT_32_FLOAT] ; encoding: [0x00,0x00,0xb1,0xe8,0x00,0x00,0x20,0x80]
 tbuffer_load_format_d16_xy v0, off, s[0:3], format:22, 0
-// GFX10: tbuffer_load_format_d16_xyzw v[0:1], off, s[0:3], format:22, 0 ; encoding: [0x00,0x00,0xb3,0xe8,0x00,0x00,0x20,0x80]
+
+// GFX10: tbuffer_load_format_d16_xyzw v[0:1], off, s[0:3], 0 format:[BUF_FMT_32_FLOAT] ; encoding: [0x00,0x00,0xb3,0xe8,0x00,0x00,0x20,0x80]
 tbuffer_load_format_d16_xyzw v[0:1], off, s[0:3], format:22, 0
-// GFX10: tbuffer_load_format_xyzw v[0:3], off, s[0:3], format:78, 0 ; encoding: [0x00,0x00,0x73,0xea,0x00,0x00,0x00,0x80]
+
+// GFX10: tbuffer_load_format_xyzw v[0:3], off, s[0:3], 0 format:78 ; encoding: [0x00,0x00,0x73,0xea,0x00,0x00,0x00,0x80]
 tbuffer_load_format_xyzw v[0:3], off, s[0:3], format:78, 0
-// GFX10: tbuffer_load_format_xyzw v[8:11], off, s[0:3], format:22, 0 slc ; encoding: [0x00,0x00,0xb3,0xe8,0x00,0x08,0x40,0x80]
+
+// GFX10: tbuffer_load_format_xyzw v[8:11], off, s[0:3], 0 format:[BUF_FMT_32_FLOAT] slc ; encoding: [0x00,0x00,0xb3,0xe8,0x00,0x08,0x40,0x80]
 tbuffer_load_format_xyzw v[8:11], off, s[0:3], format:22, 0 slc
-// GFX10: tbuffer_load_format_xyzw v[4:7], off, s[0:3], format:63, 0 glc ; encoding: [0x00,0x40,0xfb,0xe9,0x00,0x04,0x00,0x80]
+
+// GFX10: tbuffer_load_format_xyzw v[4:7], off, s[0:3], 0 format:[BUF_FMT_32_32_SINT] glc ; encoding: [0x00,0x40,0xfb,0xe9,0x00,0x04,0x00,0x80]
 tbuffer_load_format_xyzw v[4:7], off, s[0:3], format:63, 0 glc
-// GFX10: tbuffer_load_format_xyzw v[12:15], off, s[0:3], format:23, 0 glc dlc ; encoding: [0x00,0xc0,0xbb,0xe8,0x00,0x0c,0x00,0x80]
+
+// GFX10: tbuffer_load_format_xyzw v[12:15], off, s[0:3], 0 format:[BUF_FMT_16_16_UNORM] glc dlc ; encoding: [0x00,0xc0,0xbb,0xe8,0x00,0x0c,0x00,0x80]
 tbuffer_load_format_xyzw v[12:15], off, s[0:3], format:23, 0 glc dlc
-// GFX10: tbuffer_load_format_xyzw v[0:3], off, s[0:3], format:78, 0 offset:42 ; encoding: [0x2a,0x00,0x73,0xea,0x00,0x00,0x00,0x80]
+
+// GFX10: tbuffer_load_format_xyzw v[0:3], off, s[0:3], 0 format:78 offset:42 ; encoding: [0x2a,0x00,0x73,0xea,0x00,0x00,0x00,0x80]
 tbuffer_load_format_xyzw v[0:3], off, s[0:3], format:78, 0 offset:42
-// GFX10: tbuffer_load_format_xyzw v[4:7], off, s[0:3], format:62, s4 offset:73 ; encoding: [0x49,0x00,0xf3,0xe9,0x00,0x04,0x00,0x04]
+
+// GFX10: tbuffer_load_format_xyzw v[4:7], off, s[0:3], s4 format:[BUF_FMT_32_32_UINT] offset:73 ; encoding: [0x49,0x00,0xf3,0xe9,0x00,0x04,0x00,0x04]
 tbuffer_load_format_xyzw v[4:7], off, s[0:3], format:62, s4 offset:73
-// GFX10: tbuffer_load_format_xyzw v[0:3], off, s[0:3], format:47, 61 offset:4095 ; encoding: [0xff,0x0f,0x7b,0xe9,0x00,0x00,0x00,0xbd]
+
+// GFX10: tbuffer_load_format_xyzw v[0:3], off, s[0:3], 61 format:[BUF_FMT_10_10_10_2_SSCALED] offset:4095 ; encoding: [0xff,0x0f,0x7b,0xe9,0x00,0x00,0x00,0xbd]
 tbuffer_load_format_xyzw v[0:3], off, s[0:3], format:47, 61 offset:4095
-// GFX10: tbuffer_load_format_xyzw v[8:11], off, s[0:3], format:77, s4 offset:1 ; encoding: [0x01,0x00,0x6b,0xea,0x00,0x08,0x00,0x04]
+
+// GFX10: tbuffer_load_format_xyzw v[8:11], off, s[0:3], s4 format:[BUF_FMT_32_32_32_32_FLOAT] offset:1 ; encoding: [0x01,0x00,0x6b,0xea,0x00,0x08,0x00,0x04]
 tbuffer_load_format_xyzw v[8:11], off, s[0:3], format:77, s4 offset:1
-// GFX10: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], format:78, 0 idxen ; encoding: [0x00,0x20,0x73,0xea,0x00,0x00,0x00,0x80]
+
+// GFX10: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:78 idxen ; encoding: [0x00,0x20,0x73,0xea,0x00,0x00,0x00,0x80]
 tbuffer_load_format_xyzw v[0:3], v0, s[0:3], format:78, 0 idxen
-// GFX10: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], format:78, 0 offen ; encoding: [0x00,0x10,0x73,0xea,0x00,0x00,0x00,0x80]
+
+// GFX10: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:78 offen ; encoding: [0x00,0x10,0x73,0xea,0x00,0x00,0x00,0x80]
 tbuffer_load_format_xyzw v[0:3], v0, s[0:3], format:78, 0 offen
-// GFX10: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], format:78, 0 offen offset:52 ; encoding: [0x34,0x10,0x73,0xea,0x00,0x00,0x00,0x80]
+
+// GFX10: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:78 offen offset:52 ; encoding: [0x34,0x10,0x73,0xea,0x00,0x00,0x00,0x80]
 tbuffer_load_format_xyzw v[0:3], v0, s[0:3], format:78, 0 offen offset:52
-// GFX10: tbuffer_load_format_xyzw v[0:3], v[0:1], s[0:3], format:78, 0 idxen offen ; encoding: [0x00,0x30,0x73,0xea,0x00,0x00,0x00,0x80]
+
+// GFX10: tbuffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 format:78 idxen offen ; encoding: [0x00,0x30,0x73,0xea,0x00,0x00,0x00,0x80]
 tbuffer_load_format_xyzw v[0:3], v[0:1], s[0:3], format:78, 0 idxen offen
-// GFX10: tbuffer_load_format_xy v[0:1], off, s[0:3], format:77, 0 ; encoding: [0x00,0x00,0x69,0xea,0x00,0x00,0x00,0x80]
+
+// GFX10: tbuffer_load_format_xy v[0:1], off, s[0:3], 0 format:[BUF_FMT_32_32_32_32_FLOAT] ; encoding: [0x00,0x00,0x69,0xea,0x00,0x00,0x00,0x80]
 tbuffer_load_format_xy v[0:1], off, s[0:3], format:77, 0
-// GFX10: tbuffer_load_format_x v0, off, s[0:3], format:77, 0 ; encoding: [0x00,0x00,0x68,0xea,0x00,0x00,0x00,0x80]
+
+// GFX10: tbuffer_load_format_x v0, off, s[0:3], 0 format:[BUF_FMT_32_32_32_32_FLOAT] ; encoding: [0x00,0x00,0x68,0xea,0x00,0x00,0x00,0x80]
 tbuffer_load_format_x v0, off, s[0:3], format:77, 0
-// GFX10: tbuffer_store_format_d16_x v0, v1, s[4:7], format:33, 0 idxen ; encoding: [0x00,0x20,0x0c,0xe9,0x01,0x00,0x21,0x80]
+
+// GFX10: tbuffer_store_format_d16_x v0, v1, s[4:7], 0 format:[BUF_FMT_10_11_11_SSCALED] idxen ; encoding: [0x00,0x20,0x0c,0xe9,0x01,0x00,0x21,0x80]
 tbuffer_store_format_d16_x v0, v1, s[4:7], format:33, 0 idxen
-// GFX10: tbuffer_store_format_d16_xy v0, v1, s[4:7], format:33, 0 idxen ; encoding: [0x00,0x20,0x0d,0xe9,0x01,0x00,0x21,0x80]
+
+// GFX10: tbuffer_store_format_d16_xy v0, v1, s[4:7], 0 format:[BUF_FMT_10_11_11_SSCALED] idxen ; encoding: [0x00,0x20,0x0d,0xe9,0x01,0x00,0x21,0x80]
 tbuffer_store_format_d16_xy v0, v1, s[4:7], format:33, 0 idxen
-// GFX10: tbuffer_store_format_d16_xyzw v[0:1], v2, s[4:7], format:33, 0 idxen ; encoding: [0x00,0x20,0x0f,0xe9,0x02,0x00,0x21,0x80]
+
+// GFX10: tbuffer_store_format_d16_xyzw v[0:1], v2, s[4:7], 0 format:[BUF_FMT_10_11_11_SSCALED] idxen ; encoding: [0x00,0x20,0x0f,0xe9,0x02,0x00,0x21,0x80]
 tbuffer_store_format_d16_xyzw v[0:1], v2, s[4:7], format:33, 0 idxen
-// GFX10: tbuffer_store_format_xyzw v[0:3], off, s[0:3], format:44, 0 ; encoding: [0x00,0x00,0x67,0xe9,0x00,0x00,0x00,0x80]
+
+// GFX10: tbuffer_store_format_xyzw v[0:3], off, s[0:3], 0 format:[BUF_FMT_10_10_10_2_UNORM] ; encoding: [0x00,0x00,0x67,0xe9,0x00,0x00,0x00,0x80]
 tbuffer_store_format_xyzw v[0:3], off, s[0:3], format:44, 0
-// GFX10: tbuffer_store_format_xyzw v[4:7], off, s[0:3], format:61, 0 glc ; encoding: [0x00,0x40,0xef,0xe9,0x00,0x04,0x00,0x80]
+
+// GFX10: tbuffer_store_format_xyzw v[4:7], off, s[0:3], 0 format:[BUF_FMT_8_8_8_8_SINT] glc ; encoding: [0x00,0x40,0xef,0xe9,0x00,0x04,0x00,0x80]
 tbuffer_store_format_xyzw v[4:7], off, s[0:3], format:61, 0 glc
-// GFX10: tbuffer_store_format_xyzw v[8:11], off, s[0:3], format:78, 0 slc ; encoding: [0x00,0x00,0x77,0xea,0x00,0x08,0x40,0x80]
+
+// GFX10: tbuffer_store_format_xyzw v[8:11], off, s[0:3], 0 format:78 slc ; encoding: [0x00,0x00,0x77,0xea,0x00,0x08,0x40,0x80]
 tbuffer_store_format_xyzw v[8:11], off, s[0:3], format:78, 0 slc
-// GFX10: tbuffer_store_format_xyzw v[8:11], off, s[0:3], format:78, 0 ; encoding: [0x00,0x00,0x77,0xea,0x00,0x08,0x00,0x80]
+
+// GFX10: tbuffer_store_format_xyzw v[8:11], off, s[0:3], 0 format:78 ; encoding: [0x00,0x00,0x77,0xea,0x00,0x08,0x00,0x80]
 tbuffer_store_format_xyzw v[8:11], off, s[0:3], format:78, 0
-// GFX10: tbuffer_store_format_xyzw v[0:3], off, s[0:3], format:117, 0 offset:42 ; encoding: [0x2a,0x00,0xaf,0xeb,0x00,0x00,0x00,0x80]
+
+// GFX10: tbuffer_store_format_xyzw v[0:3], off, s[0:3], 0 format:117 offset:42 ; encoding: [0x2a,0x00,0xaf,0xeb,0x00,0x00,0x00,0x80]
 tbuffer_store_format_xyzw v[0:3], off, s[0:3], format:117, 0 offset:42
-// GFX10: tbuffer_store_format_xyzw v[0:3], off, s[0:3], format:117, s4 offset:42 ; encoding: [0x2a,0x00,0xaf,0xeb,0x00,0x00,0x00,0x04]
+
+// GFX10: tbuffer_store_format_xyzw v[0:3], off, s[0:3], s4 format:117 offset:42 ; encoding: [0x2a,0x00,0xaf,0xeb,0x00,0x00,0x00,0x04]
 tbuffer_store_format_xyzw v[0:3], off, s[0:3], format:117, s4 offset:42
-// GFX10: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], format:47, 0 idxen ; encoding: [0x00,0x20,0x7f,0xe9,0x04,0x00,0x00,0x80]
+
+// GFX10: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_FMT_10_10_10_2_SSCALED] idxen ; encoding: [0x00,0x20,0x7f,0xe9,0x04,0x00,0x00,0x80]
 tbuffer_store_format_xyzw v[0:3], v4, s[0:3], format:47, 0 idxen
-// GFX10: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], format:115, 0 offen ; encoding: [0x00,0x10,0x9f,0xeb,0x04,0x00,0x00,0x80]
+
+// GFX10: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:115 offen ; encoding: [0x00,0x10,0x9f,0xeb,0x04,0x00,0x00,0x80]
 tbuffer_store_format_xyzw v[0:3], v4, s[0:3], format:115, 0 offen
-// GFX10: tbuffer_store_format_xyzw v[0:3], v[4:5], s[0:3], format:70, 0 idxen offen ; encoding: [0x00,0x30,0x37,0xea,0x04,0x00,0x00,0x80]
+
+// GFX10: tbuffer_store_format_xyzw v[0:3], v[4:5], s[0:3], 0 format:[BUF_FMT_16_16_16_16_SINT] idxen offen ; encoding: [0x00,0x30,0x37,0xea,0x04,0x00,0x00,0x80]
 tbuffer_store_format_xyzw v[0:3], v[4:5], s[0:3], format:70, 0 idxen offen
-// GFX10: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], format:63, 0 idxen ; encoding: [0x00,0x20,0xff,0xe9,0x04,0x00,0x00,0x80]
+
+// GFX10: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_FMT_32_32_SINT] idxen ; encoding: [0x00,0x20,0xff,0xe9,0x04,0x00,0x00,0x80]
 tbuffer_store_format_xyzw v[0:3], v4, s[0:3], format:63, 0 idxen
-// GFX10: tbuffer_store_format_xyzw v[0:3], v6, s[0:3], format:46, 0 idxen ; encoding: [0x00,0x20,0x77,0xe9,0x06,0x00,0x00,0x80]
+
+// GFX10: tbuffer_store_format_xyzw v[0:3], v6, s[0:3], 0 format:[BUF_FMT_10_10_10_2_USCALED] idxen ; encoding: [0x00,0x20,0x77,0xe9,0x06,0x00,0x00,0x80]
 tbuffer_store_format_xyzw v[0:3], v6, s[0:3], format:46, 0 idxen
-// GFX10: tbuffer_store_format_x v0, v1, s[0:3], format:125, 0 idxen ; encoding: [0x00,0x20,0xec,0xeb,0x01,0x00,0x00,0x80]
+
+// GFX10: tbuffer_store_format_x v0, v1, s[0:3], 0 format:125 idxen ; encoding: [0x00,0x20,0xec,0xeb,0x01,0x00,0x00,0x80]
 tbuffer_store_format_x v0, v1, s[0:3], format:125, 0 idxen
-// GFX10: tbuffer_store_format_xy v[0:1], v2, s[0:3], format:33, 0 idxen ; encoding: [0x00,0x20,0x0d,0xe9,0x02,0x00,0x00,0x80]
+
+// GFX10: tbuffer_store_format_xy v[0:1], v2, s[0:3], 0 format:[BUF_FMT_10_11_11_SSCALED] idxen ; encoding: [0x00,0x20,0x0d,0xe9,0x02,0x00,0x00,0x80]
 tbuffer_store_format_xy v[0:1], v2, s[0:3], format:33, 0 idxen
 
-// GFX10: tbuffer_store_format_x v0, v1, s[0:3], format:127, 0 idxen ; encoding: [0x00,0x20,0xfc,0xeb,0x01,0x00,0x00,0x80]
+// GFX10: tbuffer_store_format_x v0, v1, s[0:3], 0 format:127 idxen ; encoding: [0x00,0x20,0xfc,0xeb,0x01,0x00,0x00,0x80]
 tbuffer_store_format_x v0, v1, s[0:3], format:127, 0 idxen
 
-// GFX10: tbuffer_store_format_x v0, v1, s[0:3], format:127, 0 idxen ; encoding: [0x00,0x20,0xfc,0xeb,0x01,0x00,0x00,0x80]
+// GFX10: tbuffer_store_format_x v0, v1, s[0:3], 0 format:127 idxen ; encoding: [0x00,0x20,0xfc,0xeb,0x01,0x00,0x00,0x80]
 tbuffer_store_format_x v0, v1, s[0:3] format:127 0 idxen
 
-// GFX10: tbuffer_store_format_x v0, v1, s[0:3], format:0, s0 idxen ; encoding: [0x00,0x20,0x04,0xe8,0x01,0x00,0x00,0x00]
+// GFX10: tbuffer_store_format_x v0, v1, s[0:3], s0 format:[BUF_FMT_INVALID] idxen ; encoding: [0x00,0x20,0x04,0xe8,0x01,0x00,0x00,0x00]
 tbuffer_store_format_x v0, v1, s[0:3] format:0 s0 idxen
 
 // GFX10: tbuffer_store_format_x v0, v1, s[0:3], s0 idxen ; encoding: [0x00,0x20,0x0c,0xe8,0x01,0x00,0x00,0x00]
@@ -86,8 +122,15 @@ tbuffer_store_format_x v0, v1, s[0:3], 0 idxen
 // GFX10: tbuffer_load_format_d16_x v0, off, s[0:3], s0 ; encoding: [0x00,0x00,0x08,0xe8,0x00,0x00,0x20,0x00]
 tbuffer_load_format_d16_x v0, off, s[0:3] s0
 
+MAX_FORMAT=127
+// GFX10: tbuffer_store_format_x v0, v1, s[0:3], s0 format:[BUF_FMT_INVALID] idxen ; encoding: [0x00,0x20,0x04,0xe8,0x01,0x00,0x00,0x00]
+tbuffer_store_format_x v0, v1, s[0:3] format:0 s0 idxen
+
+// GFX10: tbuffer_store_format_x v0, v1, s[0:3], s0 format:[BUF_FMT_INVALID] idxen ; encoding: [0x00,0x20,0x04,0xe8,0x01,0x00,0x00,0x00]
+tbuffer_store_format_x v0, v1, s[0:3] format:0 s0 idxen
+
 //===----------------------------------------------------------------------===//
-// Errors handling.
+// Negative tests for legacy format syntax.
 //===----------------------------------------------------------------------===//
 
 // GFX10-ERR: error: out of range format
@@ -111,5 +154,387 @@ tbuffer_load_format_d16_x v0, off, s[0:3], format:1,, s0
 // GFX10-ERR: error: unknown token in expression
 tbuffer_load_format_d16_x v0, off, s[0:3], format:1:, s0
 
-// GFX10-ERR: error: not a valid operand
+// GFX10-ERR: error: unknown token in expression
 tbuffer_load_format_d16_x v0, off, s[0:3],, format:1, s0
+
+//===----------------------------------------------------------------------===//
+// Positive tests for symbolic MTBUF format.
+//===----------------------------------------------------------------------===//
+
+// Format may be specified in numeric form (min value).
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:0 idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_INVALID] idxen ; encoding: [0x00,0x20,0x07,0xe8,0x01,0x01,0x01,0x00]
+
+// Format may be specified in numeric form (max value).
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:127 idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:127 idxen ; encoding: [0x00,0x20,0xff,0xeb,0x01,0x01,0x01,0x00]
+
+// Format may be specified in numeric form (first unsupported value).
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:78 idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:78 idxen ; encoding: [0x00,0x20,0x77,0xea,0x01,0x01,0x01,0x00]
+
+// Format may be specified as an expression.
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:(2 + 3 * 16) idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_2_10_10_10_UNORM] idxen ; encoding: [0x00,0x20,0x97,0xe9,0x01,0x01,0x01,0x00]
+
+// format may be specified as a list of dfmt, nfmt:
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_DATA_FORMAT_8,BUF_NUM_FORMAT_UNORM] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 idxen ; encoding: [0x00,0x20,0x0f,0xe8,0x01,0x01,0x01,0x00]
+
+// nfmt and dfmt can be in either order:
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_NUM_FORMAT_SNORM, BUF_DATA_FORMAT_16] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_16_SNORM] idxen ; encoding: [0x00,0x20,0x47,0xe8,0x01,0x01,0x01,0x00]
+
+// nfmt may be omitted:
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[ BUF_DATA_FORMAT_8_8 ] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_8_8_UNORM] idxen ; encoding: [0x00,0x20,0x77,0xe8,0x01,0x01,0x01,0x00]
+
+// dfmt may be omitted:
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_NUM_FORMAT_USCALED] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_8_USCALED] idxen ; encoding: [0x00,0x20,0x1f,0xe8,0x01,0x01,0x01,0x00]
+
+// Check dfmt formats
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_DATA_FORMAT_16_16] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_16_16_UNORM] idxen ; encoding: [0x00,0x20,0xbf,0xe8,0x01,0x01,0x01,0x00]
+
+// Check dfmt formats
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_DATA_FORMAT_10_11_11] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_10_11_11_UNORM] idxen ; encoding: [0x00,0x20,0xf7,0xe8,0x01,0x01,0x01,0x00]
+
+// Check dfmt formats
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_DATA_FORMAT_11_11_10] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_11_11_10_UNORM] idxen ; encoding: [0x00,0x20,0x2f,0xe9,0x01,0x01,0x01,0x00]
+
+// Check dfmt formats
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_DATA_FORMAT_10_10_10_2] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_10_10_10_2_UNORM] idxen ; encoding: [0x00,0x20,0x67,0xe9,0x01,0x01,0x01,0x00]
+
+// Check dfmt formats
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_DATA_FORMAT_2_10_10_10] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_2_10_10_10_UNORM] idxen ; encoding: [0x00,0x20,0x97,0xe9,0x01,0x01,0x01,0x00]
+
+// Check dfmt formats
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_DATA_FORMAT_8_8_8_8] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_8_8_8_8_UNORM] idxen ; encoding: [0x00,0x20,0xc7,0xe9,0x01,0x01,0x01,0x00]
+
+// Check dfmt formats
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_DATA_FORMAT_16_16_16_16] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_16_16_16_16_UNORM] idxen ; encoding: [0x00,0x20,0x0f,0xea,0x01,0x01,0x01,0x00]
+
+// Check dfmt formats
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_DATA_FORMAT_INVALID] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_INVALID] idxen ; encoding: [0x00,0x20,0x07,0xe8,0x01,0x01,0x01,0x00]
+
+// Check nfmt formats
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_NUM_FORMAT_SSCALED] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_8_SSCALED] idxen ; encoding: [0x00,0x20,0x27,0xe8,0x01,0x01,0x01,0x00]
+
+// Check nfmt formats
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_NUM_FORMAT_UINT] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_8_UINT] idxen ; encoding: [0x00,0x20,0x2f,0xe8,0x01,0x01,0x01,0x00]
+
+// Check nfmt formats
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_NUM_FORMAT_SINT] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_8_SINT] idxen ; encoding: [0x00,0x20,0x37,0xe8,0x01,0x01,0x01,0x00]
+
+//===----------------------------------------------------------------------===//
+// Negative tests for symbolic format errors handling.
+//===----------------------------------------------------------------------===//
+
+// Unknown format specifier
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_DATA_FORMAT] idxen
+// GFX10-ERR: error: unsupported format
+
+// Valid but unsupported format specifier (SNORM_OGL is supported for SI/CI only)
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_NUM_FORMAT_SNORM_OGL] idxen
+// GFX10-ERR: error: unsupported format
+
+// Valid but unsupported format specifier (RESERVED_6 is supported for VI/GFX9 only)
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_NUM_FORMAT_RESERVED_6] idxen
+// GFX10-ERR: error: unsupported format
+
+// Unsupported format
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_DATA_FORMAT_32] idxen
+// GFX10-ERR: error: unsupported format
+
+// Unsupported format
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_DATA_FORMAT_32_32] idxen
+// GFX10-ERR: error: unsupported format
+
+// Unsupported format
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_DATA_FORMAT_32_32_32] idxen
+// GFX10-ERR: error: unsupported format
+
+// Unsupported format
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_DATA_FORMAT_32_32_32_32] idxen
+// GFX10-ERR: error: unsupported format
+
+// Unsupported format
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_DATA_FORMAT_32_32_32_32, BUF_NUM_FORMAT_UNORM] idxen
+// GFX10-ERR: error: unsupported format
+
+// Unsupported format
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_DATA_FORMAT_RESERVED_15] idxen
+// GFX10-ERR: error: unsupported format
+
+// Unsupported format
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_NUM_FORMAT_FLOAT] idxen
+// GFX10-ERR: error: unsupported format
+
+// Unsupported format
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_DATA_FORMAT_8_8, BUF_NUM_FORMAT_FLOAT] idxen
+// GFX10-ERR: error: unsupported format
+
+//===----------------------------------------------------------------------===//
+// Positive tests for unified MTBUF format (GFX10+).
+//===----------------------------------------------------------------------===//
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_INVALID] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_INVALID] idxen ; encoding: [0x00,0x20,0x07,0xe8,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_8_UNORM] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 idxen ; encoding: [0x00,0x20,0x0f,0xe8,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_8_SNORM] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_8_SNORM] idxen ; encoding: [0x00,0x20,0x17,0xe8,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_8_USCALED] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_8_USCALED] idxen ; encoding: [0x00,0x20,0x1f,0xe8,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_8_SSCALED] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_8_SSCALED] idxen ; encoding: [0x00,0x20,0x27,0xe8,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_8_UINT] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_8_UINT] idxen ; encoding: [0x00,0x20,0x2f,0xe8,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_8_SINT] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_8_SINT] idxen ; encoding: [0x00,0x20,0x37,0xe8,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_16_UNORM] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_16_UNORM] idxen ; encoding: [0x00,0x20,0x3f,0xe8,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_16_SNORM] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_16_SNORM] idxen ; encoding: [0x00,0x20,0x47,0xe8,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_16_USCALED] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_16_USCALED] idxen ; encoding: [0x00,0x20,0x4f,0xe8,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_16_SSCALED] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_16_SSCALED] idxen ; encoding: [0x00,0x20,0x57,0xe8,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_16_UINT] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_16_UINT] idxen ; encoding: [0x00,0x20,0x5f,0xe8,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_16_SINT] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_16_SINT] idxen ; encoding: [0x00,0x20,0x67,0xe8,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_16_FLOAT] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_16_FLOAT] idxen ; encoding: [0x00,0x20,0x6f,0xe8,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_8_8_UNORM] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_8_8_UNORM] idxen ; encoding: [0x00,0x20,0x77,0xe8,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_8_8_SNORM] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_8_8_SNORM] idxen ; encoding: [0x00,0x20,0x7f,0xe8,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_8_8_USCALED] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_8_8_USCALED] idxen ; encoding: [0x00,0x20,0x87,0xe8,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_8_8_SSCALED] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_8_8_SSCALED] idxen ; encoding: [0x00,0x20,0x8f,0xe8,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_8_8_UINT] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_8_8_UINT] idxen ; encoding: [0x00,0x20,0x97,0xe8,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_8_8_SINT] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_8_8_SINT] idxen ; encoding: [0x00,0x20,0x9f,0xe8,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_32_UINT] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_32_UINT] idxen ; encoding: [0x00,0x20,0xa7,0xe8,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_32_SINT] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_32_SINT] idxen ; encoding: [0x00,0x20,0xaf,0xe8,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_32_FLOAT] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_32_FLOAT] idxen ; encoding: [0x00,0x20,0xb7,0xe8,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_16_16_UNORM] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_16_16_UNORM] idxen ; encoding: [0x00,0x20,0xbf,0xe8,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_16_16_SNORM] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_16_16_SNORM] idxen ; encoding: [0x00,0x20,0xc7,0xe8,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_16_16_USCALED] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_16_16_USCALED] idxen ; encoding: [0x00,0x20,0xcf,0xe8,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_16_16_SSCALED] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_16_16_SSCALED] idxen ; encoding: [0x00,0x20,0xd7,0xe8,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_16_16_UINT] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_16_16_UINT] idxen ; encoding: [0x00,0x20,0xdf,0xe8,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_16_16_SINT] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_16_16_SINT] idxen ; encoding: [0x00,0x20,0xe7,0xe8,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_16_16_FLOAT] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_16_16_FLOAT] idxen ; encoding: [0x00,0x20,0xef,0xe8,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_10_11_11_UNORM] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_10_11_11_UNORM] idxen ; encoding: [0x00,0x20,0xf7,0xe8,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_10_11_11_SNORM] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_10_11_11_SNORM] idxen ; encoding: [0x00,0x20,0xff,0xe8,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_10_11_11_USCALED] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_10_11_11_USCALED] idxen ; encoding: [0x00,0x20,0x07,0xe9,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_10_11_11_SSCALED] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_10_11_11_SSCALED] idxen ; encoding: [0x00,0x20,0x0f,0xe9,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_10_11_11_UINT] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_10_11_11_UINT] idxen ; encoding: [0x00,0x20,0x17,0xe9,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_10_11_11_SINT] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_10_11_11_SINT] idxen ; encoding: [0x00,0x20,0x1f,0xe9,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_10_11_11_FLOAT] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_10_11_11_FLOAT] idxen ; encoding: [0x00,0x20,0x27,0xe9,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_11_11_10_UNORM] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_11_11_10_UNORM] idxen ; encoding: [0x00,0x20,0x2f,0xe9,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_11_11_10_SNORM] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_11_11_10_SNORM] idxen ; encoding: [0x00,0x20,0x37,0xe9,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_11_11_10_USCALED] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_11_11_10_USCALED] idxen ; encoding: [0x00,0x20,0x3f,0xe9,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_11_11_10_SSCALED] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_11_11_10_SSCALED] idxen ; encoding: [0x00,0x20,0x47,0xe9,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_11_11_10_UINT] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_11_11_10_UINT] idxen ; encoding: [0x00,0x20,0x4f,0xe9,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_11_11_10_SINT] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_11_11_10_SINT] idxen ; encoding: [0x00,0x20,0x57,0xe9,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_11_11_10_FLOAT] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_11_11_10_FLOAT] idxen ; encoding: [0x00,0x20,0x5f,0xe9,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_10_10_10_2_UNORM] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_10_10_10_2_UNORM] idxen ; encoding: [0x00,0x20,0x67,0xe9,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_10_10_10_2_SNORM] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_10_10_10_2_SNORM] idxen ; encoding: [0x00,0x20,0x6f,0xe9,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_10_10_10_2_USCALED] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_10_10_10_2_USCALED] idxen ; encoding: [0x00,0x20,0x77,0xe9,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_10_10_10_2_SSCALED] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_10_10_10_2_SSCALED] idxen ; encoding: [0x00,0x20,0x7f,0xe9,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_10_10_10_2_UINT] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_10_10_10_2_UINT] idxen ; encoding: [0x00,0x20,0x87,0xe9,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_10_10_10_2_SINT] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_10_10_10_2_SINT] idxen ; encoding: [0x00,0x20,0x8f,0xe9,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_2_10_10_10_UNORM] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_2_10_10_10_UNORM] idxen ; encoding: [0x00,0x20,0x97,0xe9,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_2_10_10_10_SNORM] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_2_10_10_10_SNORM] idxen ; encoding: [0x00,0x20,0x9f,0xe9,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_2_10_10_10_USCALED] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_2_10_10_10_USCALED] idxen ; encoding: [0x00,0x20,0xa7,0xe9,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_2_10_10_10_SSCALED] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_2_10_10_10_SSCALED] idxen ; encoding: [0x00,0x20,0xaf,0xe9,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_2_10_10_10_UINT] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_2_10_10_10_UINT] idxen ; encoding: [0x00,0x20,0xb7,0xe9,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_2_10_10_10_SINT] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_2_10_10_10_SINT] idxen ; encoding: [0x00,0x20,0xbf,0xe9,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_8_8_8_8_UNORM] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_8_8_8_8_UNORM] idxen ; encoding: [0x00,0x20,0xc7,0xe9,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_8_8_8_8_SNORM] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_8_8_8_8_SNORM] idxen ; encoding: [0x00,0x20,0xcf,0xe9,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_8_8_8_8_USCALED] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_8_8_8_8_USCALED] idxen ; encoding: [0x00,0x20,0xd7,0xe9,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_8_8_8_8_SSCALED] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_8_8_8_8_SSCALED] idxen ; encoding: [0x00,0x20,0xdf,0xe9,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_8_8_8_8_UINT] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_8_8_8_8_UINT] idxen ; encoding: [0x00,0x20,0xe7,0xe9,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_8_8_8_8_SINT] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_8_8_8_8_SINT] idxen ; encoding: [0x00,0x20,0xef,0xe9,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_32_32_UINT] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_32_32_UINT] idxen ; encoding: [0x00,0x20,0xf7,0xe9,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_32_32_SINT] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_32_32_SINT] idxen ; encoding: [0x00,0x20,0xff,0xe9,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_32_32_FLOAT] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_32_32_FLOAT] idxen ; encoding: [0x00,0x20,0x07,0xea,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_16_16_16_16_UNORM] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_16_16_16_16_UNORM] idxen ; encoding: [0x00,0x20,0x0f,0xea,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_16_16_16_16_SNORM] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_16_16_16_16_SNORM] idxen ; encoding: [0x00,0x20,0x17,0xea,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_16_16_16_16_USCALED] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_16_16_16_16_USCALED] idxen ; encoding: [0x00,0x20,0x1f,0xea,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_16_16_16_16_SSCALED] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_16_16_16_16_SSCALED] idxen ; encoding: [0x00,0x20,0x27,0xea,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_16_16_16_16_UINT] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_16_16_16_16_UINT] idxen ; encoding: [0x00,0x20,0x2f,0xea,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_16_16_16_16_SINT] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_16_16_16_16_SINT] idxen ; encoding: [0x00,0x20,0x37,0xea,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_16_16_16_16_FLOAT] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_16_16_16_16_FLOAT] idxen ; encoding: [0x00,0x20,0x3f,0xea,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_32_32_32_UINT] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_32_32_32_UINT] idxen ; encoding: [0x00,0x20,0x47,0xea,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_32_32_32_SINT] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_32_32_32_SINT] idxen ; encoding: [0x00,0x20,0x4f,0xea,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_32_32_32_FLOAT] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_32_32_32_FLOAT] idxen ; encoding: [0x00,0x20,0x57,0xea,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_32_32_32_32_UINT] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_32_32_32_32_UINT] idxen ; encoding: [0x00,0x20,0x5f,0xea,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_32_32_32_32_SINT] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_32_32_32_32_SINT] idxen ; encoding: [0x00,0x20,0x67,0xea,0x01,0x01,0x01,0x00]
+
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_32_32_32_32_FLOAT] idxen
+// GFX10: tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_32_32_32_32_FLOAT] idxen ; encoding: [0x00,0x20,0x6f,0xea,0x01,0x01,0x01,0x00]
+
+//===----------------------------------------------------------------------===//
+// Negative tests for unified MTBUF format (GFX10+).
+//===----------------------------------------------------------------------===//
+
+// Excessive commas
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_8_SNORM,] idxen
+// GFX10-ERR: error: expected a closing square bracket
+
+// Duplicate format
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_8_SNORM,BUF_FMT_8_SNORM] idxen
+// GFX10-ERR: error: expected a closing square bracket
+
+// Duplicate format
+tbuffer_store_format_xyzw v[1:4], v1, s[4:7], s0 format:[BUF_FMT_8_SNORM,BUF_DATA_FORMAT_8] idxen
+// GFX10-ERR: error: expected a closing square bracket
diff --git a/llvm/test/MC/AMDGPU/mtbuf.s b/llvm/test/MC/AMDGPU/mtbuf.s
index 9d207ff326060..f7fdd29bb83b8 100644
--- a/llvm/test/MC/AMDGPU/mtbuf.s
+++ b/llvm/test/MC/AMDGPU/mtbuf.s
@@ -2,60 +2,60 @@
 // RUN: not llvm-mc -arch=amdgcn -mcpu=bonaire -show-encoding %s | FileCheck -check-prefix=GCN -check-prefix=CI -check-prefix=SICI %s
 // RUN: not llvm-mc -arch=amdgcn -mcpu=tonga -show-encoding %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
 
-// RUN: not llvm-mc -arch=amdgcn -mcpu=tahiti -show-encoding %s  2>&1 | FileCheck -check-prefix=GCN-ERR %s
-// RUN: not llvm-mc -arch=amdgcn -mcpu=bonaire -show-encoding %s 2>&1 | FileCheck -check-prefix=GCN-ERR %s
-// RUN: not llvm-mc -arch=amdgcn -mcpu=tonga -show-encoding %s   2>&1 | FileCheck -check-prefix=GCN-ERR %s
+// RUN: not llvm-mc -arch=amdgcn -mcpu=tahiti -show-encoding %s  2>&1 | FileCheck -check-prefixes=GCN-ERR,SICI-ERR %s
+// RUN: not llvm-mc -arch=amdgcn -mcpu=bonaire -show-encoding %s 2>&1 | FileCheck -check-prefixes=GCN-ERR,SICI-ERR %s
+// RUN: not llvm-mc -arch=amdgcn -mcpu=tonga -show-encoding %s   2>&1 | FileCheck -check-prefixes=GCN-ERR,VI-ERR %s
 
 //===----------------------------------------------------------------------===//
-// Test for dfmt and nfmt (tbuffer only)
+// Positive tests for legacy dfmt/nfmt syntax.
 //===----------------------------------------------------------------------===//
 
 tbuffer_load_format_x v1, off, s[4:7], dfmt:15, nfmt:2, s1
-// SICI: tbuffer_load_format_x v1, off, s[4:7],  dfmt:15,  nfmt:2, s1 ; encoding: [0x00,0x00,0x78,0xe9,0x00,0x01,0x01,0x01]
-// VI:   tbuffer_load_format_x v1, off, s[4:7],  dfmt:15,  nfmt:2, s1 ; encoding: [0x00,0x00,0x78,0xe9,0x00,0x01,0x01,0x01]
+// SICI: tbuffer_load_format_x v1, off, s[4:7], s1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] ; encoding: [0x00,0x00,0x78,0xe9,0x00,0x01,0x01,0x01]
+// VI:   tbuffer_load_format_x v1, off, s[4:7], s1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] ; encoding: [0x00,0x00,0x78,0xe9,0x00,0x01,0x01,0x01]
 
 tbuffer_load_format_xy v[1:2], off, s[4:7], dfmt:15, nfmt:2, s1
-// SICI: tbuffer_load_format_xy v[1:2], off, s[4:7],  dfmt:15,  nfmt:2, s1 ; encoding: [0x00,0x00,0x79,0xe9,0x00,0x01,0x01,0x01]
-// VI:   tbuffer_load_format_xy v[1:2], off, s[4:7],  dfmt:15,  nfmt:2, s1 ; encoding: [0x00,0x80,0x78,0xe9,0x00,0x01,0x01,0x01]
+// SICI: tbuffer_load_format_xy v[1:2], off, s[4:7], s1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] ; encoding: [0x00,0x00,0x79,0xe9,0x00,0x01,0x01,0x01]
+// VI:   tbuffer_load_format_xy v[1:2], off, s[4:7], s1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] ; encoding: [0x00,0x80,0x78,0xe9,0x00,0x01,0x01,0x01]
 
 tbuffer_load_format_xyz v[1:3], off, s[4:7], dfmt:15, nfmt:2, s1
-// SICI: tbuffer_load_format_xyz v[1:3], off, s[4:7],  dfmt:15,  nfmt:2, s1 ; encoding: [0x00,0x00,0x7a,0xe9,0x00,0x01,0x01,0x01]
-// VI:   tbuffer_load_format_xyz v[1:3], off, s[4:7],  dfmt:15,  nfmt:2, s1 ; encoding: [0x00,0x00,0x79,0xe9,0x00,0x01,0x01,0x01]
+// SICI: tbuffer_load_format_xyz v[1:3], off, s[4:7], s1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] ; encoding: [0x00,0x00,0x7a,0xe9,0x00,0x01,0x01,0x01]
+// VI:   tbuffer_load_format_xyz v[1:3], off, s[4:7], s1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] ; encoding: [0x00,0x00,0x79,0xe9,0x00,0x01,0x01,0x01]
 
 tbuffer_load_format_xyzw v[1:4], off, s[4:7], dfmt:15, nfmt:2, s1
-// SICI: tbuffer_load_format_xyzw v[1:4], off, s[4:7],  dfmt:15,  nfmt:2, s1 ; encoding: [0x00,0x00,0x7b,0xe9,0x00,0x01,0x01,0x01]
-// VI:   tbuffer_load_format_xyzw v[1:4], off, s[4:7],  dfmt:15,  nfmt:2, s1 ; encoding: [0x00,0x80,0x79,0xe9,0x00,0x01,0x01,0x01]
+// SICI: tbuffer_load_format_xyzw v[1:4], off, s[4:7], s1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] ; encoding: [0x00,0x00,0x7b,0xe9,0x00,0x01,0x01,0x01]
+// VI:   tbuffer_load_format_xyzw v[1:4], off, s[4:7], s1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] ; encoding: [0x00,0x80,0x79,0xe9,0x00,0x01,0x01,0x01]
 
 tbuffer_store_format_x v1, off, s[4:7], dfmt:15, nfmt:2, s1
-// SICI: tbuffer_store_format_x v1, off, s[4:7],  dfmt:15,  nfmt:2, s1 ; encoding: [0x00,0x00,0x7c,0xe9,0x00,0x01,0x01,0x01]
-// VI:   tbuffer_store_format_x v1, off, s[4:7],  dfmt:15,  nfmt:2, s1 ; encoding: [0x00,0x00,0x7a,0xe9,0x00,0x01,0x01,0x01]
+// SICI: tbuffer_store_format_x v1, off, s[4:7], s1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] ; encoding: [0x00,0x00,0x7c,0xe9,0x00,0x01,0x01,0x01]
+// VI:   tbuffer_store_format_x v1, off, s[4:7], s1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] ; encoding: [0x00,0x00,0x7a,0xe9,0x00,0x01,0x01,0x01]
 
 tbuffer_store_format_xy v[1:2], off, s[4:7], dfmt:15, nfmt:2, s1
-// SICI: tbuffer_store_format_xy v[1:2], off, s[4:7],  dfmt:15,  nfmt:2, s1 ; encoding: [0x00,0x00,0x7d,0xe9,0x00,0x01,0x01,0x01]
-// VI:   tbuffer_store_format_xy v[1:2], off, s[4:7],  dfmt:15,  nfmt:2, s1 ; encoding: [0x00,0x80,0x7a,0xe9,0x00,0x01,0x01,0x01]
+// SICI: tbuffer_store_format_xy v[1:2], off, s[4:7], s1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] ; encoding: [0x00,0x00,0x7d,0xe9,0x00,0x01,0x01,0x01]
+// VI:   tbuffer_store_format_xy v[1:2], off, s[4:7], s1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] ; encoding: [0x00,0x80,0x7a,0xe9,0x00,0x01,0x01,0x01]
 
 tbuffer_store_format_xyzw v[1:4], off, s[4:7], dfmt:15, nfmt:2, s1
-// SICI: tbuffer_store_format_xyzw v[1:4], off, s[4:7],  dfmt:15,  nfmt:2, s1 ; encoding: [0x00,0x00,0x7f,0xe9,0x00,0x01,0x01,0x01]
-// VI:   tbuffer_store_format_xyzw v[1:4], off, s[4:7],  dfmt:15,  nfmt:2, s1 ; encoding: [0x00,0x80,0x7b,0xe9,0x00,0x01,0x01,0x01]
+// SICI: tbuffer_store_format_xyzw v[1:4], off, s[4:7], s1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] ; encoding: [0x00,0x00,0x7f,0xe9,0x00,0x01,0x01,0x01]
+// VI:   tbuffer_store_format_xyzw v[1:4], off, s[4:7], s1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] ; encoding: [0x00,0x80,0x7b,0xe9,0x00,0x01,0x01,0x01]
 
 tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], dfmt:15, nfmt:2, ttmp1
-// SICI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7],  dfmt:15,  nfmt:2, ttmp1 ; encoding: [0x00,0x00,0x7f,0xe9,0x00,0x01,0x1d,0x71]
-// VI:   tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], dfmt:15,  nfmt:2, ttmp1 ; encoding: [0x00,0x80,0x7b,0xe9,0x00,0x01,0x1d,0x71]
+// SICI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], ttmp1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] ; encoding: [0x00,0x00,0x7f,0xe9,0x00,0x01,0x1d,0x71]
+// VI:   tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], ttmp1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] ; encoding: [0x00,0x80,0x7b,0xe9,0x00,0x01,0x1d,0x71]
 
 // nfmt is optional:
 tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], dfmt:15, ttmp1
-// SICI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7],  dfmt:15,  nfmt:0, ttmp1 ; encoding: [0x00,0x00,0x7f,0xe8,0x00,0x01,0x1d,0x71]
-// VI:   tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], dfmt:15,  nfmt:0, ttmp1 ; encoding: [0x00,0x80,0x7b,0xe8,0x00,0x01,0x1d,0x71]
+// SICI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], ttmp1 format:[BUF_DATA_FORMAT_RESERVED_15] ; encoding: [0x00,0x00,0x7f,0xe8,0x00,0x01,0x1d,0x71]
+// VI:   tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], ttmp1 format:[BUF_DATA_FORMAT_RESERVED_15] ; encoding: [0x00,0x80,0x7b,0xe8,0x00,0x01,0x1d,0x71]
 
 // dfmt is optional:
 tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], nfmt:2, ttmp1
-// SICI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], dfmt:1, nfmt:2, ttmp1 ; encoding: [0x00,0x00,0x0f,0xe9,0x00,0x01,0x1d,0x71]
-// VI:   tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], dfmt:1, nfmt:2, ttmp1 ; encoding: [0x00,0x80,0x0b,0xe9,0x00,0x01,0x1d,0x71]
+// SICI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], ttmp1 format:[BUF_NUM_FORMAT_USCALED] ; encoding: [0x00,0x00,0x0f,0xe9,0x00,0x01,0x1d,0x71]
+// VI:   tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], ttmp1 format:[BUF_NUM_FORMAT_USCALED] ; encoding: [0x00,0x80,0x0b,0xe9,0x00,0x01,0x1d,0x71]
 
 // nfmt and dfmt can be in either order:
 tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], nfmt:2, dfmt:15, ttmp1
-// SICI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7],  dfmt:15,  nfmt:2, ttmp1 ; encoding: [0x00,0x00,0x7f,0xe9,0x00,0x01,0x1d,0x71]
-// VI:   tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], dfmt:15,  nfmt:2, ttmp1 ; encoding: [0x00,0x80,0x7b,0xe9,0x00,0x01,0x1d,0x71]
+// SICI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], ttmp1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] ; encoding: [0x00,0x00,0x7f,0xe9,0x00,0x01,0x1d,0x71]
+// VI:   tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], ttmp1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] ; encoding: [0x00,0x80,0x7b,0xe9,0x00,0x01,0x1d,0x71]
 
 // nfmt and dfmt may be omitted:
 tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], ttmp1
@@ -64,13 +64,13 @@ tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], ttmp1
 
 // Check dfmt/nfmt min values
 tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], dfmt:0, nfmt:0, ttmp1
-// SICI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], dfmt:0, nfmt:0, ttmp1 ; encoding: [0x00,0x00,0x07,0xe8,0x00,0x01,0x1d,0x71]
-// VI:   tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], dfmt:0, nfmt:0, ttmp1 ; encoding: [0x00,0x80,0x03,0xe8,0x00,0x01,0x1d,0x71]
+// SICI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], ttmp1 format:[BUF_DATA_FORMAT_INVALID] ; encoding: [0x00,0x00,0x07,0xe8,0x00,0x01,0x1d,0x71]
+// VI:   tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], ttmp1 format:[BUF_DATA_FORMAT_INVALID] ; encoding: [0x00,0x80,0x03,0xe8,0x00,0x01,0x1d,0x71]
 
 // Check dfmt/nfmt max values
 tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], dfmt:15, nfmt:7, ttmp1
-// SICI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], dfmt:15, nfmt:7, ttmp1 ; encoding: [0x00,0x00,0xff,0xeb,0x00,0x01,0x1d,0x71]
-// VI:   tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], dfmt:15, nfmt:7, ttmp1 ; encoding: [0x00,0x80,0xfb,0xeb,0x00,0x01,0x1d,0x71]
+// SICI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], ttmp1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_FLOAT] ; encoding: [0x00,0x00,0xff,0xeb,0x00,0x01,0x1d,0x71]
+// VI:   tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], ttmp1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_FLOAT] ; encoding: [0x00,0x80,0xfb,0xeb,0x00,0x01,0x1d,0x71]
 
 // Check default dfmt/nfmt values
 tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], dfmt:1, nfmt:0, ttmp1
@@ -79,11 +79,18 @@ tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], dfmt:1, nfmt:0, ttmp1
 
 // Check that comma separators are optional
 tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7] dfmt:15 nfmt:7 ttmp1
-// SICI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], dfmt:15, nfmt:7, ttmp1 ; encoding: [0x00,0x00,0xff,0xeb,0x00,0x01,0x1d,0x71]
-// VI:   tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], dfmt:15, nfmt:7, ttmp1 ; encoding: [0x00,0x80,0xfb,0xeb,0x00,0x01,0x1d,0x71]
+// SICI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], ttmp1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_FLOAT] ; encoding: [0x00,0x00,0xff,0xeb,0x00,0x01,0x1d,0x71]
+// VI:   tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], ttmp1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_FLOAT] ; encoding: [0x00,0x80,0xfb,0xeb,0x00,0x01,0x1d,0x71]
+
+dfmt=15
+nfmt=7
+// Check expressions with dfmt
+tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7] dfmt:-1+dfmt+1 nfmt:nfmt ttmp1
+// SICI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], ttmp1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_FLOAT] ; encoding: [0x00,0x00,0xff,0xeb,0x00,0x01,0x1d,0x71]
+// VI:   tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], ttmp1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_FLOAT] ; encoding: [0x00,0x80,0xfb,0xeb,0x00,0x01,0x1d,0x71]
 
 //===----------------------------------------------------------------------===//
-// Errors handling.
+// Negative tests for legacy dfmt/nfmt syntax.
 //===----------------------------------------------------------------------===//
 
 tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7] dfmt:-1 nfmt:1 s0
@@ -102,7 +109,7 @@ tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7]
 // GCN-ERR: error: too few operands for instruction
 
 tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7],, dfmt:1 nfmt:1 s0
-// GCN-ERR: error: not a valid operand
+// GCN-ERR: error: unknown token in expression
 
 tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7] dfmt:1,, nfmt:1 s0
 // GCN-ERR: error: unknown token in expression
@@ -111,19 +118,262 @@ tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7] dfmt:1 nfmt:1,, s0
 // GCN-ERR: error: unknown token in expression
 
 tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7] dfmt:1 dfmt:1 s0
-// GCN-ERR: error: not a valid operand
+// GCN-ERR: error: invalid operand for instruction
 
 tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7] nfmt:1 nfmt:1 s0
-// GCN-ERR: error: not a valid operand
+// GCN-ERR: error: invalid operand for instruction
 
 tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7] dfmt:1 nfmt:1 dfmt:1 s0
-// GCN-ERR: error: not a valid operand
+// GCN-ERR: error: invalid operand for instruction
 
 tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7] nfmt:1 dfmt:1 nfmt:1 s0
-// GCN-ERR: error: not a valid operand
+// GCN-ERR: error: invalid operand for instruction
 
 tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7] dfmt:1: nfmt:1 s0
 // GCN-ERR: error: unknown token in expression
 
 tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7] dfmt:1 nfmt:1: s0
 // GCN-ERR: error: unknown token in expression
+
+//===----------------------------------------------------------------------===//
+// Tests for symbolic MTBUF format
+//===----------------------------------------------------------------------===//
+
+// Format may be specified in numeric form (min value).
+tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:0
+// SICI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_DATA_FORMAT_INVALID] ; encoding: [0x00,0x00,0x07,0xe8,0x00,0x01,0x1d,0x00]
+// VI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_DATA_FORMAT_INVALID] ; encoding: [0x00,0x80,0x03,0xe8,0x00,0x01,0x1d,0x00]
+
+// Format may be specified in numeric form (max value).
+tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:127
+// SICI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_FLOAT] ; encoding: [0x00,0x00,0xff,0xeb,0x00,0x01,0x1d,0x00]
+// VI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_FLOAT] ; encoding: [0x00,0x80,0xfb,0xeb,0x00,0x01,0x1d,0x00]
+
+// Format may be specified as an expression.
+tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:(2 + 3 * 16)
+// SICI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_DATA_FORMAT_16,BUF_NUM_FORMAT_SSCALED] ; encoding: [0x00,0x00,0x97,0xe9,0x00,0x01,0x1d,0x00]
+// VI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_DATA_FORMAT_16,BUF_NUM_FORMAT_SSCALED] ; encoding: [0x00,0x80,0x93,0xe9,0x00,0x01,0x1d,0x00]
+
+// format may be specified as a list of dfmt, nfmt:
+tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_DATA_FORMAT_8,BUF_NUM_FORMAT_UNORM]
+// SICI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 ; encoding: [0x00,0x00,0x0f,0xe8,0x00,0x01,0x1d,0x00]
+// VI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 ; encoding: [0x00,0x80,0x0b,0xe8,0x00,0x01,0x1d,0x00]
+
+// nfmt and dfmt can be in either order:
+tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_NUM_FORMAT_SNORM, BUF_DATA_FORMAT_16]
+// SICI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_DATA_FORMAT_16,BUF_NUM_FORMAT_SNORM] ; encoding: [0x00,0x00,0x97,0xe8,0x00,0x01,0x1d,0x00]
+// VI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_DATA_FORMAT_16,BUF_NUM_FORMAT_SNORM] ; encoding: [0x00,0x80,0x93,0xe8,0x00,0x01,0x1d,0x00]
+
+// nfmt may be omitted:
+tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[ BUF_DATA_FORMAT_8_8 ]
+// SICI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_DATA_FORMAT_8_8] ; encoding: [0x00,0x00,0x1f,0xe8,0x00,0x01,0x1d,0x00]
+// VI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_DATA_FORMAT_8_8] ; encoding: [0x00,0x80,0x1b,0xe8,0x00,0x01,0x1d,0x00]
+
+// dfmt may be omitted:
+tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_NUM_FORMAT_USCALED]
+// SICI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_NUM_FORMAT_USCALED] ; encoding: [0x00,0x00,0x0f,0xe9,0x00,0x01,0x1d,0x00]
+// VI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_NUM_FORMAT_USCALED] ; encoding: [0x00,0x80,0x0b,0xe9,0x00,0x01,0x1d,0x00]
+
+// Check dfmt formats
+tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_DATA_FORMAT_32]
+// SICI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_DATA_FORMAT_32] ; encoding: [0x00,0x00,0x27,0xe8,0x00,0x01,0x1d,0x00]
+// VI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_DATA_FORMAT_32] ; encoding: [0x00,0x80,0x23,0xe8,0x00,0x01,0x1d,0x00]
+
+// Check dfmt formats
+tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_DATA_FORMAT_16_16]
+// SICI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_DATA_FORMAT_16_16] ; encoding: [0x00,0x00,0x2f,0xe8,0x00,0x01,0x1d,0x00]
+// VI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_DATA_FORMAT_16_16] ; encoding: [0x00,0x80,0x2b,0xe8,0x00,0x01,0x1d,0x00]
+
+// Check dfmt formats
+tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_DATA_FORMAT_10_11_11]
+// SICI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_DATA_FORMAT_10_11_11] ; encoding: [0x00,0x00,0x37,0xe8,0x00,0x01,0x1d,0x00]
+// VI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_DATA_FORMAT_10_11_11] ; encoding: [0x00,0x80,0x33,0xe8,0x00,0x01,0x1d,0x00]
+
+// Check dfmt formats
+tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_DATA_FORMAT_11_11_10]
+// SICI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_DATA_FORMAT_11_11_10] ; encoding: [0x00,0x00,0x3f,0xe8,0x00,0x01,0x1d,0x00]
+// VI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_DATA_FORMAT_11_11_10] ; encoding: [0x00,0x80,0x3b,0xe8,0x00,0x01,0x1d,0x00]
+
+// Check dfmt formats
+tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_DATA_FORMAT_10_10_10_2]
+// SICI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_DATA_FORMAT_10_10_10_2] ; encoding: [0x00,0x00,0x47,0xe8,0x00,0x01,0x1d,0x00]
+// VI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_DATA_FORMAT_10_10_10_2] ; encoding: [0x00,0x80,0x43,0xe8,0x00,0x01,0x1d,0x00]
+
+// Check dfmt formats
+tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_DATA_FORMAT_2_10_10_10]
+// SICI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_DATA_FORMAT_2_10_10_10] ; encoding: [0x00,0x00,0x4f,0xe8,0x00,0x01,0x1d,0x00]
+// VI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_DATA_FORMAT_2_10_10_10] ; encoding: [0x00,0x80,0x4b,0xe8,0x00,0x01,0x1d,0x00]
+
+// Check dfmt formats
+tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_DATA_FORMAT_8_8_8_8]
+// SICI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_DATA_FORMAT_8_8_8_8] ; encoding: [0x00,0x00,0x57,0xe8,0x00,0x01,0x1d,0x00]
+// VI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_DATA_FORMAT_8_8_8_8] ; encoding: [0x00,0x80,0x53,0xe8,0x00,0x01,0x1d,0x00]
+
+// Check dfmt formats
+tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_DATA_FORMAT_32_32]
+// SICI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_DATA_FORMAT_32_32] ; encoding: [0x00,0x00,0x5f,0xe8,0x00,0x01,0x1d,0x00]
+// VI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_DATA_FORMAT_32_32] ; encoding: [0x00,0x80,0x5b,0xe8,0x00,0x01,0x1d,0x00]
+
+// Check dfmt formats
+tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_DATA_FORMAT_16_16_16_16]
+// SICI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_DATA_FORMAT_16_16_16_16] ; encoding: [0x00,0x00,0x67,0xe8,0x00,0x01,0x1d,0x00]
+// VI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_DATA_FORMAT_16_16_16_16] ; encoding: [0x00,0x80,0x63,0xe8,0x00,0x01,0x1d,0x00]
+
+// Check dfmt formats
+tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_DATA_FORMAT_32_32_32]
+// SICI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_DATA_FORMAT_32_32_32] ; encoding: [0x00,0x00,0x6f,0xe8,0x00,0x01,0x1d,0x00]
+// VI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_DATA_FORMAT_32_32_32] ; encoding: [0x00,0x80,0x6b,0xe8,0x00,0x01,0x1d,0x00]
+
+// Check dfmt formats
+tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_DATA_FORMAT_32_32_32_32]
+// SICI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_DATA_FORMAT_32_32_32_32] ; encoding: [0x00,0x00,0x77,0xe8,0x00,0x01,0x1d,0x00]
+// VI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_DATA_FORMAT_32_32_32_32] ; encoding: [0x00,0x80,0x73,0xe8,0x00,0x01,0x1d,0x00]
+
+// Check dfmt formats
+tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_DATA_FORMAT_RESERVED_15]
+// SICI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_DATA_FORMAT_RESERVED_15] ; encoding: [0x00,0x00,0x7f,0xe8,0x00,0x01,0x1d,0x00]
+// VI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_DATA_FORMAT_RESERVED_15] ; encoding: [0x00,0x80,0x7b,0xe8,0x00,0x01,0x1d,0x00]
+
+// Check dfmt formats
+tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_DATA_FORMAT_INVALID]
+// SICI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_DATA_FORMAT_INVALID] ; encoding: [0x00,0x00,0x07,0xe8,0x00,0x01,0x1d,0x00]
+// VI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_DATA_FORMAT_INVALID] ; encoding: [0x00,0x80,0x03,0xe8,0x00,0x01,0x1d,0x00]
+
+// Check nfmt formats
+tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_NUM_FORMAT_SSCALED]
+// SICI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_NUM_FORMAT_SSCALED] ; encoding: [0x00,0x00,0x8f,0xe9,0x00,0x01,0x1d,0x00]
+// VI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_NUM_FORMAT_SSCALED] ; encoding: [0x00,0x80,0x8b,0xe9,0x00,0x01,0x1d,0x00]
+
+// Check nfmt formats
+tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_NUM_FORMAT_UINT]
+// SICI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_NUM_FORMAT_UINT] ; encoding: [0x00,0x00,0x0f,0xea,0x00,0x01,0x1d,0x00]
+// VI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_NUM_FORMAT_UINT] ; encoding: [0x00,0x80,0x0b,0xea,0x00,0x01,0x1d,0x00]
+
+// Check nfmt formats
+tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_NUM_FORMAT_SINT]
+// SICI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_NUM_FORMAT_SINT] ; encoding: [0x00,0x00,0x8f,0xea,0x00,0x01,0x1d,0x00]
+// VI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_NUM_FORMAT_SINT] ; encoding: [0x00,0x80,0x8b,0xea,0x00,0x01,0x1d,0x00]
+
+// Check nfmt formats
+tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_NUM_FORMAT_FLOAT]
+// SICI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_NUM_FORMAT_FLOAT] ; encoding: [0x00,0x00,0x8f,0xeb,0x00,0x01,0x1d,0x00]
+// VI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_NUM_FORMAT_FLOAT] ; encoding: [0x00,0x80,0x8b,0xeb,0x00,0x01,0x1d,0x00]
+
+// Check optional comma separators
+tbuffer_store_format_xyzw v[1:4], v1, ttmp[4:7], s0, format:[BUF_DATA_FORMAT_32,BUF_NUM_FORMAT_FLOAT], idxen
+// SICI: tbuffer_store_format_xyzw v[1:4], v1, ttmp[4:7], s0 format:[BUF_DATA_FORMAT_32,BUF_NUM_FORMAT_FLOAT] idxen ; encoding: [0x00,0x20,0xa7,0xeb,0x01,0x01,0x1d,0x00]
+// VI: tbuffer_store_format_xyzw v[1:4], v1, ttmp[4:7], s0 format:[BUF_DATA_FORMAT_32,BUF_NUM_FORMAT_FLOAT] idxen ; encoding: [0x00,0xa0,0xa3,0xeb,0x01,0x01,0x1d,0x00]
+
+// Check offen and offset
+tbuffer_store_format_xyzw v[1:4], v1, ttmp[4:7], s0, format:[BUF_DATA_FORMAT_32,BUF_NUM_FORMAT_FLOAT] offen offset:52
+// SICI: tbuffer_store_format_xyzw v[1:4], v1, ttmp[4:7], s0 format:[BUF_DATA_FORMAT_32,BUF_NUM_FORMAT_FLOAT] offen offset:52 ; encoding: [0x34,0x10,0xa7,0xeb,0x01,0x01,0x1d,0x00]
+// VI: tbuffer_store_format_xyzw v[1:4], v1, ttmp[4:7], s0 format:[BUF_DATA_FORMAT_32,BUF_NUM_FORMAT_FLOAT] offen offset:52 ; encoding: [0x34,0x90,0xa3,0xeb,0x01,0x01,0x1d,0x00]
+
+// Check idxen and offen
+tbuffer_store_format_xyzw v[1:4], v[1:2], ttmp[4:7], s0, format:[BUF_DATA_FORMAT_32,BUF_NUM_FORMAT_FLOAT] idxen offen offset:52
+// SICI: tbuffer_store_format_xyzw v[1:4], v[1:2], ttmp[4:7], s0 format:[BUF_DATA_FORMAT_32,BUF_NUM_FORMAT_FLOAT] idxen offen offset:52 ; encoding: [0x34,0x30,0xa7,0xeb,0x01,0x01,0x1d,0x00]
+// VI: tbuffer_store_format_xyzw v[1:4], v[1:2], ttmp[4:7], s0 format:[BUF_DATA_FORMAT_32,BUF_NUM_FORMAT_FLOAT] idxen offen offset:52 ; encoding: [0x34,0xb0,0xa3,0xeb,0x01,0x01,0x1d,0x00]
+
+// Check addr64
+tbuffer_store_format_xyzw v[1:4], v[1:2], ttmp[4:7], s0, format:[BUF_DATA_FORMAT_32,BUF_NUM_FORMAT_FLOAT] addr64
+// SICI: tbuffer_store_format_xyzw v[1:4], v[1:2], ttmp[4:7], s0 format:[BUF_DATA_FORMAT_32,BUF_NUM_FORMAT_FLOAT] addr64 ; encoding: [0x00,0x80,0xa7,0xeb,0x01,0x01,0x1d,0x00]
+// VI-ERR: error: instruction not supported on this GPU
+
+//===----------------------------------------------------------------------===//
+// Tests for symbolic format errors handling
+//===----------------------------------------------------------------------===//
+
+// Missing soffset
+tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], format:[BUF_DATA_FORMAT_32]
+// GCN-ERR: error: not a valid operand.
+
+// Invalid soffset
+tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s[255] format:[BUF_NUM_FORMAT_FLOAT]
+// GCN-ERR: error: not a valid operand.
+
+// Both legacy and symbolic formats are specified
+tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], dfmt:1 s0 format:[BUF_NUM_FORMAT_FLOAT]
+// GCN-ERR: error: duplicate format
+
+// Missing format number
+tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format: offset:52
+// GCN-ERR: error: expected absolute expression
+
+// Invalid number
+tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:-1
+// GCN-ERR: error: out of range format
+
+// Invalid number
+tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:128
+// GCN-ERR: error: out of range format
+
+MAXVAL=127
+// Invalid expression
+tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:MAXVAL+1
+// GCN-ERR: error: out of range format
+
+// Empty list
+tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[]
+// GCN-ERR: error: expected a format string
+
+// More than 2 format specifiers
+tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_DATA_FORMAT_32,BUF_NUM_FORMAT_FLOAT,BUF_DATA_FORMAT_8]
+// GCN-ERR: error: expected a closing square bracket
+
+// More than 2 format specifiers
+tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_NUM_FORMAT_UINT,BUF_DATA_FORMAT_32,BUF_NUM_FORMAT_FLOAT]
+// GCN-ERR: error: expected a closing square bracket
+
+// Missing brackets
+tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:BUF_NUM_FORMAT_UINT
+// GCN-ERR: error: expected absolute expression
+
+// Unpaired brackets
+tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_NUM_FORMAT_UINT
+// GCN-ERR: error: expected a closing square bracket
+
+// Unpaired brackets
+tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:BUF_NUM_FORMAT_UINT]
+// GCN-ERR: error: expected absolute expression
+
+// Missing comma
+tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_NUM_FORMAT_UINT BUF_DATA_FORMAT_32]
+// GCN-ERR: error: expected a closing square bracket
+
+// Duplicate dfmt
+tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_DATA_FORMAT_32,BUF_DATA_FORMAT_32]
+// GCN-ERR: error: duplicate data format
+
+// Duplicate dfmt
+tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_DATA_FORMAT_32,BUF_DATA_FORMAT_8]
+// GCN-ERR: error: duplicate data format
+
+// Duplicate nfmt
+tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_NUM_FORMAT_UINT,BUF_NUM_FORMAT_FLOAT]
+// GCN-ERR: error: duplicate numeric format
+
+// Unknown format specifier
+tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_DATA_FORMAT]
+// GCN-ERR: error: unsupported format
+
+// Valid but unsupported format specifier (SNORM_OGL is supported for SI/CI only)
+tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_NUM_FORMAT_SNORM_OGL]
+// SICI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_NUM_FORMAT_SNORM_OGL] ; encoding: [0x00,0x00,0x0f,0xeb,0x00,0x01,0x1d,0x00]
+// VI-ERR: error: unsupported format
+
+// Valid but unsupported format specifier (RESERVED_6 is supported for VI/GFX9 only)
+tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_NUM_FORMAT_RESERVED_6]
+// SICI-ERR: error: unsupported format
+// VI: tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_NUM_FORMAT_RESERVED_6] ; encoding: [0x00,0x80,0x0b,0xeb,0x00,0x01,0x1d,0x00]
+
+// Excessive commas
+tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7],, s0 format:[BUF_DATA_FORMAT_8]
+// GCN-ERR: error: unknown token in expression
+
+// Excessive commas
+tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0,, format:[BUF_DATA_FORMAT_8]
+// GCN-ERR: error: not a valid operand.
+
+// Excessive commas
+tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_DATA_FORMAT_8],, offset:52
+// GCN-ERR: error: unknown token in expression
diff --git a/llvm/test/MC/Disassembler/AMDGPU/buf_fmt_packed_d16.txt b/llvm/test/MC/Disassembler/AMDGPU/buf_fmt_packed_d16.txt
index 495e78f691e83..a7599f29fa0a6 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/buf_fmt_packed_d16.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/buf_fmt_packed_d16.txt
@@ -25,26 +25,26 @@
 # PACKED: buffer_store_format_d16_xyzw v[1:2], off, s[4:7], s1 ; encoding: [0x00,0x00,0x3c,0xe0,0x00,0x01,0x01,0x01]
 0x00,0x00,0x3c,0xe0,0x00,0x01,0x01,0x01
 
-# PACKED: tbuffer_load_format_d16_x v1, off, s[4:7],  dfmt:15,  nfmt:2, s1 ; encoding: [0x00,0x00,0x7c,0xe9,0x00,0x01,0x01,0x01]
+# PACKED: tbuffer_load_format_d16_x v1, off, s[4:7], s1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] ; encoding: [0x00,0x00,0x7c,0xe9,0x00,0x01,0x01,0x01]
 0x00,0x00,0x7c,0xe9,0x00,0x01,0x01,0x01
 
-# PACKED: tbuffer_load_format_d16_xy v1, off, s[4:7],  dfmt:15,  nfmt:2, s1 ; encoding: [0x00,0x80,0x7c,0xe9,0x00,0x01,0x01,0x01]
+# PACKED: tbuffer_load_format_d16_xy v1, off, s[4:7], s1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] ; encoding: [0x00,0x80,0x7c,0xe9,0x00,0x01,0x01,0x01]
 0x00,0x80,0x7c,0xe9,0x00,0x01,0x01,0x01
 
-# PACKED: tbuffer_load_format_d16_xyz v[1:2], off, s[4:7],  dfmt:15,  nfmt:2, s1 ; encoding: [0x00,0x00,0x7d,0xe9,0x00,0x01,0x01,0x01]
+# PACKED: tbuffer_load_format_d16_xyz v[1:2], off, s[4:7], s1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] ; encoding: [0x00,0x00,0x7d,0xe9,0x00,0x01,0x01,0x01]
 0x00,0x00,0x7d,0xe9,0x00,0x01,0x01,0x01
 
-# PACKED: tbuffer_load_format_d16_xyzw v[1:2], off, s[4:7],  dfmt:15,  nfmt:2, s1 ; encoding: [0x00,0x80,0x7d,0xe9,0x00,0x01,0x01,0x01]
+# PACKED: tbuffer_load_format_d16_xyzw v[1:2], off, s[4:7], s1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] ; encoding: [0x00,0x80,0x7d,0xe9,0x00,0x01,0x01,0x01]
 0x00,0x80,0x7d,0xe9,0x00,0x01,0x01,0x01
 
-# PACKED: tbuffer_store_format_d16_x v1, off, s[4:7],  dfmt:15,  nfmt:2, s1 ; encoding: [0x00,0x00,0x7e,0xe9,0x00,0x01,0x01,0x01]
+# PACKED: tbuffer_store_format_d16_x v1, off, s[4:7], s1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] ; encoding: [0x00,0x00,0x7e,0xe9,0x00,0x01,0x01,0x01]
 0x00,0x00,0x7e,0xe9,0x00,0x01,0x01,0x01
 
-# PACKED: tbuffer_store_format_d16_xy v1, off, s[4:7],  dfmt:15,  nfmt:2, s1 ; encoding: [0x00,0x80,0x7e,0xe9,0x00,0x01,0x01,0x01]
+# PACKED: tbuffer_store_format_d16_xy v1, off, s[4:7], s1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] ; encoding: [0x00,0x80,0x7e,0xe9,0x00,0x01,0x01,0x01]
 0x00,0x80,0x7e,0xe9,0x00,0x01,0x01,0x01
 
-# PACKED: tbuffer_store_format_d16_xyz v[1:2], off, s[4:7],  dfmt:15,  nfmt:2, s1 ; encoding: [0x00,0x00,0x7f,0xe9,0x00,0x01,0x01,0x01]
+# PACKED: tbuffer_store_format_d16_xyz v[1:2], off, s[4:7], s1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] ; encoding: [0x00,0x00,0x7f,0xe9,0x00,0x01,0x01,0x01]
 0x00,0x00,0x7f,0xe9,0x00,0x01,0x01,0x01
 
-# PACKED: tbuffer_store_format_d16_xyzw v[1:2], off, s[4:7],  dfmt:15,  nfmt:2, s1 ; encoding: [0x00,0x80,0x7f,0xe9,0x00,0x01,0x01,0x01]
+# PACKED: tbuffer_store_format_d16_xyzw v[1:2], off, s[4:7], s1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] ; encoding: [0x00,0x80,0x7f,0xe9,0x00,0x01,0x01,0x01]
 0x00,0x80,0x7f,0xe9,0x00,0x01,0x01,0x01
diff --git a/llvm/test/MC/Disassembler/AMDGPU/buf_fmt_unpacked_d16.txt b/llvm/test/MC/Disassembler/AMDGPU/buf_fmt_unpacked_d16.txt
index 2df262bdceb05..9c78e97c8e3ca 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/buf_fmt_unpacked_d16.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/buf_fmt_unpacked_d16.txt
@@ -25,26 +25,26 @@
 # UNPACKED: buffer_store_format_d16_xyzw v[1:4], off, s[4:7], s1 ; encoding: [0x00,0x00,0x3c,0xe0,0x00,0x01,0x01,0x01]
 0x00,0x00,0x3c,0xe0,0x00,0x01,0x01,0x01
 
-# UNPACKED: tbuffer_load_format_d16_x v1, off, s[4:7],  dfmt:15,  nfmt:2, s1 ; encoding: [0x00,0x00,0x7c,0xe9,0x00,0x01,0x01,0x01]
+# UNPACKED: tbuffer_load_format_d16_x v1, off, s[4:7], s1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] ; encoding: [0x00,0x00,0x7c,0xe9,0x00,0x01,0x01,0x01]
 0x00,0x00,0x7c,0xe9,0x00,0x01,0x01,0x01
 
-# UNPACKED: tbuffer_load_format_d16_xy v[1:2], off, s[4:7],  dfmt:15,  nfmt:2, s1 ; encoding: [0x00,0x80,0x7c,0xe9,0x00,0x01,0x01,0x01]
+# UNPACKED: tbuffer_load_format_d16_xy v[1:2], off, s[4:7], s1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] ; encoding: [0x00,0x80,0x7c,0xe9,0x00,0x01,0x01,0x01]
 0x00,0x80,0x7c,0xe9,0x00,0x01,0x01,0x01
 
-# UNPACKED: tbuffer_load_format_d16_xyz v[1:3], off, s[4:7],  dfmt:15,  nfmt:2, s1 ; encoding: [0x00,0x00,0x7d,0xe9,0x00,0x01,0x01,0x01]
+# UNPACKED: tbuffer_load_format_d16_xyz v[1:3], off, s[4:7], s1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] ; encoding: [0x00,0x00,0x7d,0xe9,0x00,0x01,0x01,0x01]
 0x00,0x00,0x7d,0xe9,0x00,0x01,0x01,0x01
 
-# UNPACKED: tbuffer_load_format_d16_xyzw v[1:4], off, s[4:7],  dfmt:15,  nfmt:2, s1 ; encoding: [0x00,0x80,0x7d,0xe9,0x00,0x01,0x01,0x01]
+# UNPACKED: tbuffer_load_format_d16_xyzw v[1:4], off, s[4:7], s1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] ; encoding: [0x00,0x80,0x7d,0xe9,0x00,0x01,0x01,0x01]
 0x00,0x80,0x7d,0xe9,0x00,0x01,0x01,0x01
 
-# UNPACKED: tbuffer_store_format_d16_x v1, off, s[4:7],  dfmt:15,  nfmt:2, s1 ; encoding: [0x00,0x00,0x7e,0xe9,0x00,0x01,0x01,0x01]
+# UNPACKED: tbuffer_store_format_d16_x v1, off, s[4:7], s1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] ; encoding: [0x00,0x00,0x7e,0xe9,0x00,0x01,0x01,0x01]
 0x00,0x00,0x7e,0xe9,0x00,0x01,0x01,0x01
 
-# UNPACKED: tbuffer_store_format_d16_xy v[1:2], off, s[4:7],  dfmt:15,  nfmt:2, s1 ; encoding: [0x00,0x80,0x7e,0xe9,0x00,0x01,0x01,0x01]
+# UNPACKED: tbuffer_store_format_d16_xy v[1:2], off, s[4:7], s1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] ; encoding: [0x00,0x80,0x7e,0xe9,0x00,0x01,0x01,0x01]
 0x00,0x80,0x7e,0xe9,0x00,0x01,0x01,0x01
 
-# UNPACKED: tbuffer_store_format_d16_xyz v[1:3], off, s[4:7],  dfmt:15,  nfmt:2, s1 ; encoding: [0x00,0x00,0x7f,0xe9,0x00,0x01,0x01,0x01]
+# UNPACKED: tbuffer_store_format_d16_xyz v[1:3], off, s[4:7], s1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] ; encoding: [0x00,0x00,0x7f,0xe9,0x00,0x01,0x01,0x01]
 0x00,0x00,0x7f,0xe9,0x00,0x01,0x01,0x01
 
-# UNPACKED: tbuffer_store_format_d16_xyzw v[1:4], off, s[4:7],  dfmt:15,  nfmt:2, s1 ; encoding: [0x00,0x80,0x7f,0xe9,0x00,0x01,0x01,0x01]
+# UNPACKED: tbuffer_store_format_d16_xyzw v[1:4], off, s[4:7], s1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] ; encoding: [0x00,0x80,0x7f,0xe9,0x00,0x01,0x01,0x01]
 0x00,0x80,0x7f,0xe9,0x00,0x01,0x01,0x01
diff --git a/llvm/test/MC/Disassembler/AMDGPU/mtbuf_gfx10.txt b/llvm/test/MC/Disassembler/AMDGPU/mtbuf_gfx10.txt
index 10d92d9990dc0..4007c5f0dea70 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/mtbuf_gfx10.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/mtbuf_gfx10.txt
@@ -1,79 +1,111 @@
 # RUN: llvm-mc -arch=amdgcn -mcpu=gfx1010 -disassemble -show-encoding < %s | FileCheck %s -check-prefix=GFX10
 
-# GFX10: tbuffer_load_format_d16_x v0, off, s[0:3], format:22, 0
+# GFX10: tbuffer_load_format_d16_x v0, off, s[0:3], 0 format:[BUF_FMT_32_FLOAT]
 0x00,0x00,0xb0,0xe8,0x00,0x00,0x20,0x80
-# GFX10: tbuffer_load_format_d16_xy v0, off, s[0:3], format:22, 0
+
+# GFX10: tbuffer_load_format_d16_xy v0, off, s[0:3], 0 format:[BUF_FMT_32_FLOAT]
 0x00,0x00,0xb1,0xe8,0x00,0x00,0x20,0x80
-# GFX10: tbuffer_load_format_d16_xyzw v[0:1], off, s[0:3], format:22, 0
+
+# GFX10: tbuffer_load_format_d16_xyzw v[0:1], off, s[0:3], 0 format:[BUF_FMT_32_FLOAT]
 0x00,0x00,0xb3,0xe8,0x00,0x00,0x20,0x80
-# GFX10: tbuffer_load_format_xyzw v[0:3], off, s[0:3], format:78, 0
+
+# GFX10: tbuffer_load_format_xyzw v[0:3], off, s[0:3], 0 format:78
 0x00,0x00,0x73,0xea,0x00,0x00,0x00,0x80
-# GFX10: tbuffer_load_format_xyzw v[8:11], off, s[0:3], format:22, 0 slc
+
+# GFX10: tbuffer_load_format_xyzw v[8:11], off, s[0:3], 0 format:[BUF_FMT_32_FLOAT] slc
 0x00,0x00,0xb3,0xe8,0x00,0x08,0x40,0x80
-# GFX10: tbuffer_load_format_xyzw v[4:7], off, s[0:3], format:63, 0 glc
+
+# GFX10: tbuffer_load_format_xyzw v[4:7], off, s[0:3], 0 format:[BUF_FMT_32_32_SINT] glc
 0x00,0x40,0xfb,0xe9,0x00,0x04,0x00,0x80
-# GFX10: tbuffer_load_format_xyzw v[12:15], off, s[0:3], format:23, 0 glc dlc
+
+# GFX10: tbuffer_load_format_xyzw v[12:15], off, s[0:3], 0 format:[BUF_FMT_16_16_UNORM] glc dlc
 0x00,0xc0,0xbb,0xe8,0x00,0x0c,0x00,0x80
-# GFX10: tbuffer_load_format_xyzw v[0:3], off, s[0:3], format:78, 0 offset:42
+
+# GFX10: tbuffer_load_format_xyzw v[0:3], off, s[0:3], 0 format:78 offset:42
 0x2a,0x00,0x73,0xea,0x00,0x00,0x00,0x80
-# GFX10: tbuffer_load_format_xyzw v[4:7], off, s[0:3], format:62, s4 offset:73
+
+# GFX10: tbuffer_load_format_xyzw v[4:7], off, s[0:3], s4 format:[BUF_FMT_32_32_UINT] offset:73
 0x49,0x00,0xf3,0xe9,0x00,0x04,0x00,0x04
-# GFX10: tbuffer_load_format_xyzw v[0:3], off, s[0:3], format:47, 61 offset:4095
+
+# GFX10: tbuffer_load_format_xyzw v[0:3], off, s[0:3], 61 format:[BUF_FMT_10_10_10_2_SSCALED] offset:4095
 0xff,0x0f,0x7b,0xe9,0x00,0x00,0x00,0xbd
-# GFX10: tbuffer_load_format_xyzw v[8:11], off, s[0:3], format:77, s4 offset:1
+
+# GFX10: tbuffer_load_format_xyzw v[8:11], off, s[0:3], s4 format:[BUF_FMT_32_32_32_32_FLOAT] offset:1
 0x01,0x00,0x6b,0xea,0x00,0x08,0x00,0x04
-# GFX10: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], format:78, 0 idxen
+
+# GFX10: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:78 idxen
 0x00,0x20,0x73,0xea,0x00,0x00,0x00,0x80
-# GFX10: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], format:78, 0 offen
+
+# GFX10: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:78 offen
 0x00,0x10,0x73,0xea,0x00,0x00,0x00,0x80
-# GFX10: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], format:78, 0 offen offset:52
+
+# GFX10: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:78 offen offset:52
 0x34,0x10,0x73,0xea,0x00,0x00,0x00,0x80
-# GFX10: tbuffer_load_format_xyzw v[0:3], v[0:1], s[0:3], format:78, 0 idxen offen
+
+# GFX10: tbuffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 format:78 idxen offen
 0x00,0x30,0x73,0xea,0x00,0x00,0x00,0x80
-# GFX10: tbuffer_load_format_xy v[0:1], off, s[0:3], format:77, 0
+
+# GFX10: tbuffer_load_format_xy v[0:1], off, s[0:3], 0 format:[BUF_FMT_32_32_32_32_FLOAT]
 0x00,0x00,0x69,0xea,0x00,0x00,0x00,0x80
-# GFX10: tbuffer_load_format_x v0, off, s[0:3], format:77, 0
+
+# GFX10: tbuffer_load_format_x v0, off, s[0:3], 0 format:[BUF_FMT_32_32_32_32_FLOAT]
 0x00,0x00,0x68,0xea,0x00,0x00,0x00,0x80
-# GFX10: tbuffer_store_format_d16_x v0, v1, s[4:7], format:33, 0 idxen
+
+# GFX10: tbuffer_store_format_d16_x v0, v1, s[4:7], 0 format:[BUF_FMT_10_11_11_SSCALED] idxen
 0x00,0x20,0x0c,0xe9,0x01,0x00,0x21,0x80
-# GFX10: tbuffer_store_format_d16_xy v0, v1, s[4:7], format:33, 0 idxen
+
+# GFX10: tbuffer_store_format_d16_xy v0, v1, s[4:7], 0 format:[BUF_FMT_10_11_11_SSCALED] idxen
 0x00,0x20,0x0d,0xe9,0x01,0x00,0x21,0x80
-# GFX10: tbuffer_store_format_d16_xyzw v[0:1], v2, s[4:7], format:33, 0 idxen
+
+# GFX10: tbuffer_store_format_d16_xyzw v[0:1], v2, s[4:7], 0 format:[BUF_FMT_10_11_11_SSCALED] idxen
 0x00,0x20,0x0f,0xe9,0x02,0x00,0x21,0x80
-# GFX10: tbuffer_store_format_xyzw v[0:3], off, s[0:3], format:44, 0
+
+# GFX10: tbuffer_store_format_xyzw v[0:3], off, s[0:3], 0 format:[BUF_FMT_10_10_10_2_UNORM]
 0x00,0x00,0x67,0xe9,0x00,0x00,0x00,0x80
-# GFX10: tbuffer_store_format_xyzw v[4:7], off, s[0:3], format:61, 0 glc
+
+# GFX10: tbuffer_store_format_xyzw v[4:7], off, s[0:3], 0 format:[BUF_FMT_8_8_8_8_SINT] glc
 0x00,0x40,0xef,0xe9,0x00,0x04,0x00,0x80
-# GFX10: tbuffer_store_format_xyzw v[8:11], off, s[0:3], format:78, 0 slc
+
+# GFX10: tbuffer_store_format_xyzw v[8:11], off, s[0:3], 0 format:78 slc
 0x00,0x00,0x77,0xea,0x00,0x08,0x40,0x80
-# GFX10: tbuffer_store_format_xyzw v[8:11], off, s[0:3], format:78, 0
+
+# GFX10: tbuffer_store_format_xyzw v[8:11], off, s[0:3], 0 format:78
 0x00,0x00,0x77,0xea,0x00,0x08,0x00,0x80
-# GFX10: tbuffer_store_format_xyzw v[0:3], off, s[0:3], format:117, 0 offset:42
+
+# GFX10: tbuffer_store_format_xyzw v[0:3], off, s[0:3], 0 format:117 offset:42
 0x2a,0x00,0xaf,0xeb,0x00,0x00,0x00,0x80
-# GFX10: tbuffer_store_format_xyzw v[0:3], off, s[0:3], format:117, s4 offset:42
+
+# GFX10: tbuffer_store_format_xyzw v[0:3], off, s[0:3], s4 format:117 offset:42
 0x2a,0x00,0xaf,0xeb,0x00,0x00,0x00,0x04
-# GFX10: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], format:47, 0 idxen
+
+# GFX10: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_FMT_10_10_10_2_SSCALED] idxen
 0x00,0x20,0x7f,0xe9,0x04,0x00,0x00,0x80
-# GFX10: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], format:115, 0 offen
+
+# GFX10: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:115 offen
 0x00,0x10,0x9f,0xeb,0x04,0x00,0x00,0x80
-# GFX10: tbuffer_store_format_xyzw v[0:3], v[4:5], s[0:3], format:70, 0 idxen offen
+
+# GFX10: tbuffer_store_format_xyzw v[0:3], v[4:5], s[0:3], 0 format:[BUF_FMT_16_16_16_16_SINT] idxen offen
 0x00,0x30,0x37,0xea,0x04,0x00,0x00,0x80
-# GFX10: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], format:63, 0 idxen
+
+# GFX10: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_FMT_32_32_SINT] idxen
 0x00,0x20,0xff,0xe9,0x04,0x00,0x00,0x80
-# GFX10: tbuffer_store_format_xyzw v[0:3], v6, s[0:3], format:46, 0 idxen
+
+# GFX10: tbuffer_store_format_xyzw v[0:3], v6, s[0:3], 0 format:[BUF_FMT_10_10_10_2_USCALED] idxen
 0x00,0x20,0x77,0xe9,0x06,0x00,0x00,0x80
-# GFX10: tbuffer_store_format_x v0, v1, s[0:3], format:125, 0 idxen
+
+# GFX10: tbuffer_store_format_x v0, v1, s[0:3], 0 format:125 idxen
 0x00,0x20,0xec,0xeb,0x01,0x00,0x00,0x80
-# GFX10: tbuffer_store_format_xy v[0:1], v2, s[0:3], format:33, 0 idxen
+
+# GFX10: tbuffer_store_format_xy v[0:1], v2, s[0:3], 0 format:[BUF_FMT_10_11_11_SSCALED] idxen
 0x00,0x20,0x0d,0xe9,0x02,0x00,0x00,0x80
 
-# GFX10: tbuffer_store_format_x v0, v1, s[0:3], format:127, 0 idxen ; encoding: [0x00,0x20,0xfc,0xeb,0x01,0x00,0x00,0x80]
+# GFX10: tbuffer_store_format_x v0, v1, s[0:3], 0 format:127 idxen ; encoding: [0x00,0x20,0xfc,0xeb,0x01,0x00,0x00,0x80]
 0x00,0x20,0xfc,0xeb,0x01,0x00,0x00,0x80
 
-# GFX10: tbuffer_store_format_x v0, v1, s[0:3], format:0, 0 idxen ; encoding: [0x00,0x20,0x04,0xe8,0x01,0x00,0x00,0x80]
+# GFX10: tbuffer_store_format_x v0, v1, s[0:3], 0 format:[BUF_FMT_INVALID] idxen ; encoding: [0x00,0x20,0x04,0xe8,0x01,0x00,0x00,0x80]
 0x00,0x20,0x04,0xe8,0x01,0x00,0x00,0x80
 
-# GFX10: tbuffer_load_format_d16_x v0, off, s[0:3], format:0, s0 ; encoding: [0x00,0x00,0x00,0xe8,0x00,0x00,0x20,0x00]
+# GFX10: tbuffer_load_format_d16_x v0, off, s[0:3], s0 format:[BUF_FMT_INVALID] ; encoding: [0x00,0x00,0x00,0xe8,0x00,0x00,0x20,0x00]
 0x00,0x00,0x00,0xe8,0x00,0x00,0x20,0x00
 
 # GFX10: tbuffer_store_format_x v0, v1, s[0:3], 0 idxen ; encoding: [0x00,0x20,0x0c,0xe8,0x01,0x00,0x00,0x80]
@@ -82,5 +114,5 @@
 # GFX10: tbuffer_load_format_d16_x v0, off, s[0:3], s0 ; encoding: [0x00,0x00,0x08,0xe8,0x00,0x00,0x20,0x00]
 0x00,0x00,0x08,0xe8,0x00,0x00,0x20,0x00
 
-# GFX10: tbuffer_store_format_x v0, v1, s[0:3], format:2, s0 idxen ; encoding: [0x00,0x20,0x14,0xe8,0x01,0x00,0x00,0x00]
+# GFX10: tbuffer_store_format_x v0, v1, s[0:3], s0 format:[BUF_FMT_8_SNORM] idxen ; encoding: [0x00,0x20,0x14,0xe8,0x01,0x00,0x00,0x00]
 0x00,0x20,0x14,0xe8,0x01,0x00,0x00,0x00
diff --git a/llvm/test/MC/Disassembler/AMDGPU/mtbuf_vi.txt b/llvm/test/MC/Disassembler/AMDGPU/mtbuf_vi.txt
index 35f9d3bfd18f4..d32545f7f48c9 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/mtbuf_vi.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/mtbuf_vi.txt
@@ -1,40 +1,97 @@
 # RUN: llvm-mc -arch=amdgcn -mcpu=tonga -disassemble -show-encoding < %s | FileCheck %s -check-prefix=VI
 
-# VI:   tbuffer_load_format_x v1, off, s[4:7],  dfmt:15,  nfmt:2, s1 ; encoding: [0x00,0x00,0x78,0xe9,0x00,0x01,0x01,0x01]
+# VI:   tbuffer_load_format_x v1, off, s[4:7], s1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] ; encoding: [0x00,0x00,0x78,0xe9,0x00,0x01,0x01,0x01]
 0x00 0x00 0x78 0xe9 0x00 0x01 0x01 0x01
 
-# VI:   tbuffer_load_format_xy v[1:2], off, s[4:7],  dfmt:15,  nfmt:2, s1 ; encoding: [0x00,0x80,0x78,0xe9,0x00,0x01,0x01,0x01]
+# VI:   tbuffer_load_format_xy v[1:2], off, s[4:7], s1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] ; encoding: [0x00,0x80,0x78,0xe9,0x00,0x01,0x01,0x01]
 0x00 0x80 0x78 0xe9 0x00 0x01 0x01 0x01
 
-# VI:   tbuffer_load_format_xyzw v[1:4], off, s[4:7],  dfmt:15,  nfmt:2, s1 ; encoding: [0x00,0x80,0x79,0xe9,0x00,0x01,0x01,0x01]
+# VI:   tbuffer_load_format_xyzw v[1:4], off, s[4:7], s1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] ; encoding: [0x00,0x80,0x79,0xe9,0x00,0x01,0x01,0x01]
 0x00 0x80 0x79 0xe9 0x00 0x01 0x01 0x01
 
-# VI:   tbuffer_store_format_x v1, off, s[4:7],  dfmt:15,  nfmt:2, s1 ; encoding: [0x00,0x00,0x7a,0xe9,0x00,0x01,0x01,0x01]
+# VI:   tbuffer_store_format_x v1, off, s[4:7], s1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] ; encoding: [0x00,0x00,0x7a,0xe9,0x00,0x01,0x01,0x01]
 0x00 0x00 0x7a 0xe9 0x00 0x01 0x01 0x01
 
-# VI:   tbuffer_store_format_xy v[1:2], off, s[4:7],  dfmt:15,  nfmt:2, s1 ; encoding: [0x00,0x80,0x7a,0xe9,0x00,0x01,0x01,0x01]
+# VI:   tbuffer_store_format_xy v[1:2], off, s[4:7], s1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] ; encoding: [0x00,0x80,0x7a,0xe9,0x00,0x01,0x01,0x01]
 0x00 0x80 0x7a 0xe9 0x00 0x01 0x01 0x01
 
-# VI:   tbuffer_store_format_xyzw v[1:4], off, s[4:7],  dfmt:15,  nfmt:2, s1 ; encoding: [0x00,0x80,0x7b,0xe9,0x00,0x01,0x01,0x01]
+# VI:   tbuffer_store_format_xyzw v[1:4], off, s[4:7], s1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] ; encoding: [0x00,0x80,0x7b,0xe9,0x00,0x01,0x01,0x01]
 0x00 0x80 0x7b 0xe9 0x00 0x01 0x01 0x01
 
-# VI:   tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7],  dfmt:15,  nfmt:2, ttmp1 ; encoding: [0x00,0x80,0x7b,0xe9,0x00,0x01,0x1d,0x71]
+# VI:   tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], ttmp1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_USCALED] ; encoding: [0x00,0x80,0x7b,0xe9,0x00,0x01,0x1d,0x71]
 0x00 0x80 0x7b 0xe9 0x00 0x01 0x1d 0x71
 
-# VI:   tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], dfmt:0, nfmt:2, ttmp1 ; encoding: [0x00,0x80,0x03,0xe9,0x00,0x01,0x1d,0x71]
+# VI:   tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], ttmp1 format:[BUF_DATA_FORMAT_INVALID,BUF_NUM_FORMAT_USCALED] ; encoding: [0x00,0x80,0x03,0xe9,0x00,0x01,0x1d,0x71]
 0x00,0x80,0x03,0xe9,0x00,0x01,0x1d,0x71
 
-# VI:   tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], dfmt:15, nfmt:0, ttmp1 ; encoding: [0x00,0x80,0x7b,0xe8,0x00,0x01,0x1d,0x71]
+# VI:   tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], ttmp1 format:[BUF_DATA_FORMAT_RESERVED_15] ; encoding: [0x00,0x80,0x7b,0xe8,0x00,0x01,0x1d,0x71]
 0x00,0x80,0x7b,0xe8,0x00,0x01,0x1d,0x71
 
 # VI:   tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], ttmp1 ; encoding: [0x00,0x80,0x0b,0xe8,0x00,0x01,0x1d,0x71]
 0x00,0x80,0x0b,0xe8,0x00,0x01,0x1d,0x71
 
-# VI:   tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], dfmt:0, nfmt:0, ttmp1 ; encoding: [0x00,0x80,0x03,0xe8,0x00,0x01,0x1d,0x71]
+# VI:   tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], ttmp1 format:[BUF_DATA_FORMAT_INVALID] ; encoding: [0x00,0x80,0x03,0xe8,0x00,0x01,0x1d,0x71]
 0x00,0x80,0x03,0xe8,0x00,0x01,0x1d,0x71
 
-# VI:   tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], dfmt:1, nfmt:1, ttmp1 ; encoding: [0x00,0x80,0x8b,0xe8,0x00,0x01,0x1d,0x71]
+# VI:   tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], ttmp1 format:[BUF_NUM_FORMAT_SNORM] ; encoding: [0x00,0x80,0x8b,0xe8,0x00,0x01,0x1d,0x71]
 0x00,0x80,0x8b,0xe8,0x00,0x01,0x1d,0x71
 
-# VI:   tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], dfmt:15, nfmt:7, ttmp1 ; encoding: [0x00,0x80,0xfb,0xeb,0x00,0x01,0x1d,0x71]
+# VI:   tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], ttmp1 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_FLOAT] ; encoding: [0x00,0x80,0xfb,0xeb,0x00,0x01,0x1d,0x71]
 0x00,0x80,0xfb,0xeb,0x00,0x01,0x1d,0x71
+
+# VI:   tbuffer_load_format_xy v[1:2], off, s[4:7], s1 format:[BUF_DATA_FORMAT_INVALID] ; encoding: [0x00,0x80,0x00,0xe8,0x00,0x01,0x01,0x01]
+0x00 0x80 0x00 0xe8 0x00 0x01 0x01 0x01
+
+# VI:   tbuffer_load_format_xyzw v[1:4], off, s[4:7], s1 format:[BUF_NUM_FORMAT_SNORM] ; encoding: [0x00,0x80,0x89,0xe8,0x00,0x01,0x01,0x01]
+0x00 0x80 0x89 0xe8 0x00 0x01 0x01 0x01
+
+# VI:   tbuffer_store_format_x v1, off, s[4:7], s1 format:[BUF_DATA_FORMAT_16,BUF_NUM_FORMAT_FLOAT] ; encoding: [0x00,0x00,0x92,0xeb,0x00,0x01,0x01,0x01]
+0x00 0x00 0x92 0xeb 0x00 0x01 0x01 0x01
+
+# VI:   tbuffer_store_format_xy v[1:2], off, s[4:7], s1 format:[BUF_DATA_FORMAT_32,BUF_NUM_FORMAT_RESERVED_6] ; encoding: [0x00,0x80,0x22,0xeb,0x00,0x01,0x01,0x01]
+0x00 0x80 0x22 0xeb 0x00 0x01 0x01 0x01
+
+# VI:   tbuffer_store_format_xyzw v[1:4], off, s[4:7], s1 format:[BUF_DATA_FORMAT_8_8,BUF_NUM_FORMAT_SSCALED] ; encoding: [0x00,0x80,0x9b,0xe9,0x00,0x01,0x01,0x01]
+0x00 0x80 0x9b 0xe9 0x00 0x01 0x01 0x01
+
+# VI:   tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], ttmp1 format:[BUF_DATA_FORMAT_16_16,BUF_NUM_FORMAT_UINT] ; encoding: [0x00,0x80,0x2b,0xea,0x00,0x01,0x1d,0x71]
+0x00 0x80 0x2b 0xea 0x00 0x01 0x1d 0x71
+
+# VI:   tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], ttmp1 format:[BUF_DATA_FORMAT_32_32,BUF_NUM_FORMAT_SINT] ; encoding: [0x00,0x80,0xdb,0xea,0x00,0x01,0x1d,0x71]
+0x00,0x80,0xdb,0xea,0x00,0x01,0x1d,0x71
+
+# VI:   tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], ttmp1 format:[BUF_DATA_FORMAT_10_11_11] ; encoding: [0x00,0x80,0x33,0xe8,0x00,0x01,0x1d,0x71]
+0x00,0x80,0x33,0xe8,0x00,0x01,0x1d,0x71
+
+# VI:   tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], ttmp1 ; encoding: [0x00,0x80,0x0b,0xe8,0x00,0x01,0x1d,0x71]
+0x00,0x80,0x0b,0xe8,0x00,0x01,0x1d,0x71
+
+# VI:   tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], ttmp1 format:[BUF_DATA_FORMAT_16_16_16_16] ; encoding: [0x00,0x80,0x63,0xe8,0x00,0x01,0x1d,0x71]
+0x00,0x80,0x63,0xe8,0x00,0x01,0x1d,0x71
+
+# VI:   tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], ttmp1 format:[BUF_NUM_FORMAT_SNORM] ; encoding: [0x00,0x80,0x8b,0xe8,0x00,0x01,0x1d,0x71]
+0x00,0x80,0x8b,0xe8,0x00,0x01,0x1d,0x71
+
+# VI:   tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], ttmp1 format:[BUF_DATA_FORMAT_11_11_10,BUF_NUM_FORMAT_FLOAT] ; encoding: [0x00,0x80,0xbb,0xeb,0x00,0x01,0x1d,0x71]
+0x00,0x80,0xbb,0xeb,0x00,0x01,0x1d,0x71
+
+# VI:   tbuffer_store_format_xyzw v[1:4], v1, ttmp[4:7], s0 format:[BUF_DATA_FORMAT_10_10_10_2,BUF_NUM_FORMAT_FLOAT] idxen
+0x00,0xa0,0xc3,0xeb,0x01,0x01,0x1d,0x00
+
+# VI:   tbuffer_store_format_xyzw v[1:4], v1, ttmp[4:7], s0 format:[BUF_DATA_FORMAT_2_10_10_10,BUF_NUM_FORMAT_FLOAT] offen offset:52
+0x34,0x90,0xcb,0xeb,0x01,0x01,0x1d,0x00
+
+# VI:   tbuffer_store_format_xyzw v[1:4], v[1:2], ttmp[4:7], s0 format:[BUF_DATA_FORMAT_8_8_8_8,BUF_NUM_FORMAT_FLOAT] idxen offen offset:52
+0x34,0xb0,0xd3,0xeb,0x01,0x01,0x1d,0x00
+
+# VI:   tbuffer_store_format_xyzw v[1:4], v[1:2], ttmp[4:7], s0 format:[BUF_DATA_FORMAT_32_32_32,BUF_NUM_FORMAT_FLOAT] idxen offen offset:52
+0x34,0xb0,0xeb,0xeb,0x01,0x01,0x1d,0x00
+
+# VI:   tbuffer_store_format_xyzw v[1:4], v[1:2], ttmp[4:7], s0 format:[BUF_DATA_FORMAT_32_32_32_32,BUF_NUM_FORMAT_FLOAT] idxen offen offset:52
+0x34,0xb0,0xf3,0xeb,0x01,0x01,0x1d,0x00
+
+# VI:   tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_NUM_FORMAT_RESERVED_6]
+0x00,0x80,0x0b,0xeb,0x00,0x01,0x1d,0x00
+
+# VI:   tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:[BUF_NUM_FORMAT_FLOAT]
+0x00,0x80,0x8b,0xeb,0x00,0x01,0x1d,0x00

From 4ef2e594d5be2e0e6d4446c8082b15466bc7ffcb Mon Sep 17 00:00:00 2001
From: David Truby <david.truby@arm.com>
Date: Thu, 16 Jul 2020 14:15:07 +0100
Subject: [PATCH 0013/1035] [flang] Run non-gtest unit tests with lit.

Summary:
As a corrollary, these tests are now run as part of the check-flang
target.

Reviewers: sscalpone

Subscribers: mgorny, delcypher, llvm-commits

Tags: #llvm

Differential Revision: https://reviews.llvm.org/D83946
---
 flang/CMakeLists.txt                       |  1 -
 flang/test/CMakeLists.txt                  |  7 ++
 flang/test/NonGtestUnit/lit.cfg.py         | 16 +++++
 flang/test/NonGtestUnit/lit.site.cfg.py.in | 27 +++++++
 flang/unittests/CMakeLists.txt             |  8 +++
 flang/unittests/Decimal/CMakeLists.txt     |  9 +--
 flang/unittests/Evaluate/CMakeLists.txt    | 82 ++++------------------
 flang/unittests/Runtime/CMakeLists.txt     | 41 ++---------
 llvm/utils/lit/lit/formats/__init__.py     |  3 +-
 llvm/utils/lit/lit/formats/base.py         | 17 +++++
 10 files changed, 97 insertions(+), 114 deletions(-)
 create mode 100644 flang/test/NonGtestUnit/lit.cfg.py
 create mode 100644 flang/test/NonGtestUnit/lit.site.cfg.py.in

diff --git a/flang/CMakeLists.txt b/flang/CMakeLists.txt
index 9dd6281d410bf..f1aaa5c6473fe 100644
--- a/flang/CMakeLists.txt
+++ b/flang/CMakeLists.txt
@@ -380,7 +380,6 @@ endif()
 add_subdirectory(runtime)
 
 if (FLANG_INCLUDE_TESTS)
-  enable_testing()
   add_subdirectory(test)
   if (FLANG_GTEST_AVAIL)
     add_subdirectory(unittests)
diff --git a/flang/test/CMakeLists.txt b/flang/test/CMakeLists.txt
index ad47fff8c4250..7da1d94d84c4e 100644
--- a/flang/test/CMakeLists.txt
+++ b/flang/test/CMakeLists.txt
@@ -19,6 +19,13 @@ configure_lit_site_cfg(
   ${CMAKE_CURRENT_SOURCE_DIR}/Unit/lit.cfg.py
 )
 
+configure_lit_site_cfg(
+  ${CMAKE_CURRENT_SOURCE_DIR}/NonGtestUnit/lit.site.cfg.py.in
+  ${CMAKE_CURRENT_BINARY_DIR}/NonGtestUnit/lit.site.cfg.py
+  MAIN_CONFIG
+  ${CMAKE_CURRENT_SOURCE_DIR}/NonGtestUnit/lit.cfg.py
+)
+
 set(FLANG_TEST_PARAMS
   flang_site_config=${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg.py)
 
diff --git a/flang/test/NonGtestUnit/lit.cfg.py b/flang/test/NonGtestUnit/lit.cfg.py
new file mode 100644
index 0000000000000..7f53f861bc65c
--- /dev/null
+++ b/flang/test/NonGtestUnit/lit.cfg.py
@@ -0,0 +1,16 @@
+import os
+
+import lit.Test
+
+config.name = 'flang-OldUnit'
+
+config.suffixes = [".test"]
+
+config.test_source_root = os.path.join(config.flang_obj_root, 'unittests')
+config.test_exec_root = config.test_source_root
+
+config.test_format = lit.formats.ExecutableTest()
+
+path = os.path.pathsep.join((config.flang_libs_dir, config.llvm_libs_dir,
+                              config.environment.get('LD_LIBRARY_PATH','')))
+config.environment['LD_LIBRARY_PATH'] = path
diff --git a/flang/test/NonGtestUnit/lit.site.cfg.py.in b/flang/test/NonGtestUnit/lit.site.cfg.py.in
new file mode 100644
index 0000000000000..3218fe0b5ce3d
--- /dev/null
+++ b/flang/test/NonGtestUnit/lit.site.cfg.py.in
@@ -0,0 +1,27 @@
+@LIT_SITE_CFG_IN_HEADER@
+
+config.llvm_src_root = "@LLVM_SOURCE_DIR@"
+config.llvm_obj_root = "@LLVM_BINARY_DIR@"
+config.llvm_tools_dir = "@LLVM_TOOLS_DIR@"
+config.llvm_libs_dir = "@LLVM_LIBS_DIR@"
+config.llvm_build_mode = "@LLVM_BUILD_MODE@"
+config.lit_tools_dir = "@LLVM_LIT_TOOLS_DIR@"
+config.flang_obj_root = "@FLANG_BINARY_DIR@"
+config.flang_src_root = "@FLANG_SOURCE_DIR@"
+config.flang_libs_dir = "@LLVM_LIBRARY_OUTPUT_INTDIR@"
+config.flang_tools_dir = "@LLVM_RUNTIME_OUTPUT_INTDIR@"
+config.target_triple = "@TARGET_TRIPLE@"
+config.python_executable = "@Python3_EXECUTABLE@"
+
+# Support substitution of the tools and libs dirs with user parameters. This is
+# used when we can't determine the tool dir at configuration time.
+try:
+    config.llvm_tools_dir = config.llvm_tools_dir % lit_config.params
+    config.llvm_libs_dir = config.llvm_libs_dir % lit_config.params
+    config.llvm_build_mode = config.llvm_build_mode % lit_config.params
+except KeyError as e:
+    key, = e.args
+    lit_config.fatal("unable to find %r parameter, use '--param=%s=VALUE'" % (key,key))
+
+# Let the main config do the real work.
+lit_config.load_config(config, "@FLANG_SOURCE_DIR@/test/NonGtestUnit/lit.cfg.py")
diff --git a/flang/unittests/CMakeLists.txt b/flang/unittests/CMakeLists.txt
index 21da59f3afcbf..a30f0edaec615 100644
--- a/flang/unittests/CMakeLists.txt
+++ b/flang/unittests/CMakeLists.txt
@@ -9,6 +9,14 @@ if (CXX_SUPPORTS_SUGGEST_OVERRIDE_FLAG)
   add_compile_options("-Wno-suggest-override")
 endif()
 
+function(add_flang_nongtest_unittest test_name)
+  add_executable(${test_name}.test ${test_name}.cpp)
+
+  target_link_libraries(${test_name}.test ${ARGN})
+
+  add_dependencies(FlangUnitTests ${test_name}.test)
+endfunction()
+
 add_subdirectory(Optimizer)
 add_subdirectory(Decimal)
 add_subdirectory(Evaluate)
diff --git a/flang/unittests/Decimal/CMakeLists.txt b/flang/unittests/Decimal/CMakeLists.txt
index f26aca5d0e9b6..112b02f9029f9 100644
--- a/flang/unittests/Decimal/CMakeLists.txt
+++ b/flang/unittests/Decimal/CMakeLists.txt
@@ -1,13 +1,10 @@
 set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
-add_executable(quick-sanity-test
-  quick-sanity-test.cpp
-)
-
-target_link_libraries(quick-sanity-test
+add_flang_nongtest_unittest(quick-sanity-test
   FortranDecimal
   LLVMSupport
 )
 
+# This test is not run by default as it takes a long time to execute
 add_executable(thorough-test
   thorough-test.cpp
 )
@@ -16,5 +13,3 @@ target_link_libraries(thorough-test
   FortranDecimal
   LLVMSupport
 )
-
-add_test(NAME Sanity COMMAND quick-sanity-test)
diff --git a/flang/unittests/Evaluate/CMakeLists.txt b/flang/unittests/Evaluate/CMakeLists.txt
index 54676b0b5a0ad..c56789b598ce7 100644
--- a/flang/unittests/Evaluate/CMakeLists.txt
+++ b/flang/unittests/Evaluate/CMakeLists.txt
@@ -8,43 +8,22 @@ target_link_libraries(FortranEvaluateTesting
   LLVMSupport
 )
 
-add_executable(leading-zero-bit-count-test
-  leading-zero-bit-count.cpp
-)
-
-target_link_libraries(leading-zero-bit-count-test
+add_flang_nongtest_unittest(leading-zero-bit-count
   FortranEvaluateTesting
   LLVMSupport
 )
 
-add_executable(bit-population-count-test
-  bit-population-count.cpp
-)
-
-target_link_libraries(bit-population-count-test
+add_flang_nongtest_unittest(bit-population-count
   FortranEvaluateTesting
   LLVMSupport
 )
 
-add_executable(uint128-test
-  uint128.cpp
-)
-
-target_link_libraries(uint128-test
+add_flang_nongtest_unittest(uint128
   FortranEvaluateTesting
   LLVMSupport
 )
 
-# These routines live in lib/Common but we test them here.
-add_test(UINT128 uint128-test)
-add_test(Leadz leading-zero-bit-count-test)
-add_test(PopPar bit-population-count-test)
-
-add_executable(expression-test
-  expression.cpp
-)
-
-target_link_libraries(expression-test
+add_flang_nongtest_unittest(expression
   FortranCommon
   FortranEvaluateTesting
   FortranEvaluate
@@ -53,22 +32,14 @@ target_link_libraries(expression-test
   LLVMSupport
 )
 
-add_executable(integer-test
-  integer.cpp
-)
-
-target_link_libraries(integer-test
+add_flang_nongtest_unittest(integer
   FortranEvaluateTesting
   FortranEvaluate
   FortranSemantics
   LLVMSupport
 )
 
-add_executable(intrinsics-test
-  intrinsics.cpp
-)
-
-target_link_libraries(intrinsics-test
+add_flang_nongtest_unittest(intrinsics
   FortranCommon
   FortranEvaluateTesting
   FortranEvaluate
@@ -79,11 +50,7 @@ target_link_libraries(intrinsics-test
   LLVMSupport
 )
 
-add_executable(logical-test
-  logical.cpp
-)
-
-target_link_libraries(logical-test
+add_flang_nongtest_unittest(logical
   FortranEvaluateTesting
   FortranEvaluate
   FortranSemantics
@@ -96,24 +63,16 @@ target_link_libraries(logical-test
 # C++ exceptions are enabled for this test.
 set(LLVM_REQUIRES_EH ON)
 set(LLVM_REQUIRES_RTTI ON)
-add_executable(real-test
-  real.cpp
-)
-llvm_update_compile_flags(real-test)
-
-target_link_libraries(real-test
+add_flang_nongtest_unittest(real
   FortranEvaluateTesting
   FortranEvaluate
   FortranDecimal
   FortranSemantics
   LLVMSupport
 )
+llvm_update_compile_flags(real.test)
 
-add_executable(reshape-test
-  reshape.cpp
-)
-
-target_link_libraries(reshape-test
+add_flang_nongtest_unittest(reshape
   FortranEvaluateTesting
   FortranSemantics
   FortranEvaluate
@@ -121,11 +80,7 @@ target_link_libraries(reshape-test
   LLVMSupport
 )
 
-add_executable(ISO-Fortran-binding-test
-  ISO-Fortran-binding.cpp
-)
-
-target_link_libraries(ISO-Fortran-binding-test
+add_flang_nongtest_unittest(ISO-Fortran-binding
   FortranEvaluateTesting
   FortranEvaluate
   FortranSemantics
@@ -133,23 +88,10 @@ target_link_libraries(ISO-Fortran-binding-test
   LLVMSupport
 )
 
-add_executable(folding-test
-  folding.cpp
-)
-
-target_link_libraries(folding-test
+add_flang_nongtest_unittest(folding
   FortranCommon
   FortranEvaluateTesting
   FortranEvaluate
   FortranSemantics
   LLVMSupport
 )
-
-add_test(Expression expression-test)
-add_test(Integer integer-test)
-add_test(Intrinsics intrinsics-test)
-add_test(Logical logical-test)
-add_test(Real real-test)
-add_test(RESHAPE reshape-test)
-add_test(ISO-binding ISO-Fortran-binding-test)
-add_test(folding folding-test)
diff --git a/flang/unittests/Runtime/CMakeLists.txt b/flang/unittests/Runtime/CMakeLists.txt
index b13c84e88a028..041b631f59d6b 100644
--- a/flang/unittests/Runtime/CMakeLists.txt
+++ b/flang/unittests/Runtime/CMakeLists.txt
@@ -13,30 +13,19 @@ target_link_libraries(RuntimeTesting
   LLVMSupport
 )
 
-add_executable(format-test
-  format.cpp
-)
-
-target_link_libraries(format-test
+add_flang_nongtest_unittest(format
   RuntimeTesting
   FortranRuntime
   LLVMSupport
 )
 
-add_test(NAME Format COMMAND format-test)
-
-add_executable(hello-world
-  hello.cpp
-)
-
-target_link_libraries(hello-world
+add_flang_nongtest_unittest(hello
   RuntimeTesting
   FortranRuntime
   LLVMSupport
 )
 
-add_test(NAME HelloWorld COMMAND hello-world)
-
+# This test is not run by default as it requires input.
 add_executable(external-hello-world
   external-hello.cpp
 )
@@ -46,38 +35,20 @@ target_link_libraries(external-hello-world
   LLVMSupport
 )
 
-add_executable(external-io
-  external-io.cpp
-)
-
-target_link_libraries(external-io
+add_flang_nongtest_unittest(external-io
   RuntimeTesting
   FortranRuntime
   LLVMSupport
 )
 
-add_test(NAME ExternalIO COMMAND external-io)
-
-add_executable(list-input-test
-  list-input.cpp
-)
-
-target_link_libraries(list-input-test
+add_flang_nongtest_unittest(list-input
   RuntimeTesting
   FortranRuntime
   LLVMSupport
 )
 
-add_test(NAME ListInput COMMAND list-input-test)
-
-add_executable(character-test
-  character.cpp
-)
-
-target_link_libraries(character-test
+add_flang_nongtest_unittest(character
   RuntimeTesting
   FortranRuntime
   LLVMSupport
 )
-
-add_test(NAME CharacterTest COMMAND character-test)
diff --git a/llvm/utils/lit/lit/formats/__init__.py b/llvm/utils/lit/lit/formats/__init__.py
index 3ff46e93ead2e..7a357657670f3 100644
--- a/llvm/utils/lit/lit/formats/__init__.py
+++ b/llvm/utils/lit/lit/formats/__init__.py
@@ -1,7 +1,8 @@
 from lit.formats.base import (  # noqa: F401
     TestFormat,
     FileBasedTest,
-    OneCommandPerFileTest
+    OneCommandPerFileTest,
+    ExecutableTest
 )
 
 from lit.formats.googletest import GoogleTest  # noqa: F401
diff --git a/llvm/utils/lit/lit/formats/base.py b/llvm/utils/lit/lit/formats/base.py
index 6721d17e334e6..b44a606e76a82 100644
--- a/llvm/utils/lit/lit/formats/base.py
+++ b/llvm/utils/lit/lit/formats/base.py
@@ -115,3 +115,20 @@ def execute(self, test, litConfig):
         report += """Output:\n--\n%s--""" % diags
 
         return lit.Test.FAIL, report
+
+
+###
+
+# Check exit code of a simple executable with no input
+class ExecutableTest(FileBasedTest):
+    def execute(self, test, litConfig):
+        if test.config.unsupported:
+            return lit.Test.UNSUPPORTED
+
+        out, err, exitCode = lit.util.executeCommand(test.getSourcePath())
+
+        if not exitCode:
+            return lit.Test.PASS, ''
+
+        return lit.Test.FAIL, out+err
+

From c332a984aefc6f8a6b44fd3687a5bbce3f8f035c Mon Sep 17 00:00:00 2001
From: Yitzhak Mandelbaum <yitzhakm@google.com>
Date: Fri, 24 Jul 2020 13:27:51 +0000
Subject: [PATCH 0014/1035] [libTooling] Add an `EditGenerator` that applies a
 rule throughout a bound node.

The new combinator, `rewriteDescendants`, applies a rewrite rule to all
descendants of a specified bound node.  That rewrite rule can refer to nodes
bound by the parent, both in the matcher and in the edits.

Reviewed By: gribozavr2

Differential Revision: https://reviews.llvm.org/D84409
---
 .../clang/Tooling/Transformer/RewriteRule.h   |  17 +++
 clang/lib/Tooling/Transformer/RewriteRule.cpp | 137 +++++++++++++++++-
 clang/lib/Tooling/Transformer/Transformer.cpp |   7 +-
 clang/unittests/Tooling/TransformerTest.cpp   | 118 ++++++++++++++-
 4 files changed, 269 insertions(+), 10 deletions(-)

diff --git a/clang/include/clang/Tooling/Transformer/RewriteRule.h b/clang/include/clang/Tooling/Transformer/RewriteRule.h
index c22a2da81fe60..2a26d32817dde 100644
--- a/clang/include/clang/Tooling/Transformer/RewriteRule.h
+++ b/clang/include/clang/Tooling/Transformer/RewriteRule.h
@@ -332,6 +332,23 @@ inline EditGenerator shrinkTo(RangeSelector outer, RangeSelector inner) {
                    remove(enclose(after(inner), after(outer)))});
 }
 
+/// Applies `Rule` to all descendants of the node bound to `NodeId`. `Rule` can
+/// refer to nodes bound by the calling rule. `Rule` is not applied to the node
+/// itself.
+///
+/// For example,
+/// ```
+/// auto InlineX =
+///     makeRule(declRefExpr(to(varDecl(hasName("x")))), changeTo(cat("3")));
+/// makeRule(functionDecl(hasName("f"), hasBody(stmt().bind("body"))).bind("f"),
+///          flatten(
+///            changeTo(name("f"), cat("newName")),
+///            rewriteDescendants("body", InlineX)));
+/// ```
+/// Here, we find the function `f`, change its name to `newName` and change all
+/// appearances of `x` in its body to `3`.
+EditGenerator rewriteDescendants(std::string NodeId, RewriteRule Rule);
+
 /// The following three functions are a low-level part of the RewriteRule
 /// API. We expose them for use in implementing the fixtures that interpret
 /// RewriteRule, like Transformer and TransfomerTidy, or for more advanced
diff --git a/clang/lib/Tooling/Transformer/RewriteRule.cpp b/clang/lib/Tooling/Transformer/RewriteRule.cpp
index c145895af7ab6..ce773b59a7e7e 100644
--- a/clang/lib/Tooling/Transformer/RewriteRule.cpp
+++ b/clang/lib/Tooling/Transformer/RewriteRule.cpp
@@ -7,6 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/Tooling/Transformer/RewriteRule.h"
+#include "clang/AST/ASTTypeTraits.h"
+#include "clang/AST/Stmt.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
 #include "clang/ASTMatchers/ASTMatchers.h"
 #include "clang/Basic/SourceLocation.h"
@@ -115,15 +117,144 @@ ASTEdit transformer::remove(RangeSelector S) {
   return change(std::move(S), std::make_shared<SimpleTextGenerator>(""));
 }
 
-RewriteRule transformer::makeRule(ast_matchers::internal::DynTypedMatcher M,
-                                  EditGenerator Edits,
+RewriteRule transformer::makeRule(DynTypedMatcher M, EditGenerator Edits,
                                   TextGenerator Explanation) {
   return RewriteRule{{RewriteRule::Case{
       std::move(M), std::move(Edits), std::move(Explanation), {}}}};
 }
 
+namespace {
+
+/// Unconditionally binds the given node set before trying `InnerMatcher` and
+/// keeps the bound nodes on a successful match.
+template <typename T>
+class BindingsMatcher : public ast_matchers::internal::MatcherInterface<T> {
+  ast_matchers::BoundNodes Nodes;
+  const ast_matchers::internal::Matcher<T> InnerMatcher;
+
+public:
+  explicit BindingsMatcher(ast_matchers::BoundNodes Nodes,
+                           ast_matchers::internal::Matcher<T> InnerMatcher)
+      : Nodes(std::move(Nodes)), InnerMatcher(std::move(InnerMatcher)) {}
+
+  bool matches(
+      const T &Node, ast_matchers::internal::ASTMatchFinder *Finder,
+      ast_matchers::internal::BoundNodesTreeBuilder *Builder) const override {
+    ast_matchers::internal::BoundNodesTreeBuilder Result(*Builder);
+    for (const auto &N : Nodes.getMap())
+      Result.setBinding(N.first, N.second);
+    if (InnerMatcher.matches(Node, Finder, &Result)) {
+      *Builder = std::move(Result);
+      return true;
+    }
+    return false;
+  }
+};
+
+/// Matches nodes of type T that have at least one descendant node for which the
+/// given inner matcher matches.  Will match for each descendant node that
+/// matches.  Based on ForEachDescendantMatcher, but takes a dynamic matcher,
+/// instead of a static one, because it is used by RewriteRule, which carries
+/// (only top-level) dynamic matchers.
+template <typename T>
+class DynamicForEachDescendantMatcher
+    : public ast_matchers::internal::MatcherInterface<T> {
+  const DynTypedMatcher DescendantMatcher;
+
+public:
+  explicit DynamicForEachDescendantMatcher(DynTypedMatcher DescendantMatcher)
+      : DescendantMatcher(std::move(DescendantMatcher)) {}
+
+  bool matches(
+      const T &Node, ast_matchers::internal::ASTMatchFinder *Finder,
+      ast_matchers::internal::BoundNodesTreeBuilder *Builder) const override {
+    return Finder->matchesDescendantOf(
+        Node, this->DescendantMatcher, Builder,
+        ast_matchers::internal::ASTMatchFinder::BK_All);
+  }
+};
+
+template <typename T>
+ast_matchers::internal::Matcher<T>
+forEachDescendantDynamically(ast_matchers::BoundNodes Nodes,
+                             DynTypedMatcher M) {
+  return ast_matchers::internal::makeMatcher(new BindingsMatcher<T>(
+      std::move(Nodes),
+      ast_matchers::internal::makeMatcher(
+          new DynamicForEachDescendantMatcher<T>(std::move(M)))));
+}
+
+class ApplyRuleCallback : public MatchFinder::MatchCallback {
+public:
+  ApplyRuleCallback(RewriteRule Rule) : Rule(std::move(Rule)) {}
+
+  template <typename T>
+  void registerMatchers(const ast_matchers::BoundNodes &Nodes,
+                        MatchFinder *MF) {
+    for (auto &Matcher : transformer::detail::buildMatchers(Rule))
+      MF->addMatcher(forEachDescendantDynamically<T>(Nodes, Matcher), this);
+  }
+
+  void run(const MatchFinder::MatchResult &Result) override {
+    if (!Edits)
+      return;
+    transformer::RewriteRule::Case Case =
+        transformer::detail::findSelectedCase(Result, Rule);
+    auto Transformations = Case.Edits(Result);
+    if (!Transformations) {
+      Edits = Transformations.takeError();
+      return;
+    }
+    Edits->append(Transformations->begin(), Transformations->end());
+  }
+
+  RewriteRule Rule;
+
+  // Initialize to a non-error state.
+  Expected<SmallVector<Edit, 1>> Edits = SmallVector<Edit, 1>();
+};
+} // namespace
+
+template <typename T>
+llvm::Expected<SmallVector<clang::transformer::Edit, 1>>
+rewriteDescendantsImpl(const T &Node, RewriteRule Rule,
+                       const MatchResult &Result) {
+  ApplyRuleCallback Callback(std::move(Rule));
+  MatchFinder Finder;
+  Callback.registerMatchers<T>(Result.Nodes, &Finder);
+  Finder.match(Node, *Result.Context);
+  return std::move(Callback.Edits);
+}
+
+EditGenerator transformer::rewriteDescendants(std::string NodeId,
+                                              RewriteRule Rule) {
+  // FIXME: warn or return error if `Rule` contains any `AddedIncludes`, since
+  // these will be dropped.
+  return [NodeId = std::move(NodeId),
+          Rule = std::move(Rule)](const MatchResult &Result)
+             -> llvm::Expected<SmallVector<clang::transformer::Edit, 1>> {
+    const ast_matchers::BoundNodes::IDToNodeMap &NodesMap =
+        Result.Nodes.getMap();
+    auto It = NodesMap.find(NodeId);
+    if (It == NodesMap.end())
+      return llvm::make_error<llvm::StringError>(llvm::errc::invalid_argument,
+                                                 "ID not bound: " + NodeId);
+    if (auto *Node = It->second.get<Decl>())
+      return rewriteDescendantsImpl(*Node, std::move(Rule), Result);
+    if (auto *Node = It->second.get<Stmt>())
+      return rewriteDescendantsImpl(*Node, std::move(Rule), Result);
+    if (auto *Node = It->second.get<TypeLoc>())
+      return rewriteDescendantsImpl(*Node, std::move(Rule), Result);
+
+    return llvm::make_error<llvm::StringError>(
+        llvm::errc::invalid_argument,
+        "type unsupported for recursive rewriting, ID=\"" + NodeId +
+            "\", Kind=" + It->second.getNodeKind().asStringRef());
+  };
+}
+
 void transformer::addInclude(RewriteRule &Rule, StringRef Header,
-                         IncludeFormat Format) {
+                             IncludeFormat Format) {
   for (auto &Case : Rule.Cases)
     Case.AddedIncludes.emplace_back(Header.str(), Format);
 }
diff --git a/clang/lib/Tooling/Transformer/Transformer.cpp b/clang/lib/Tooling/Transformer/Transformer.cpp
index e8fc00c4e953f..5b5be7a396db4 100644
--- a/clang/lib/Tooling/Transformer/Transformer.cpp
+++ b/clang/lib/Tooling/Transformer/Transformer.cpp
@@ -38,13 +38,8 @@ void Transformer::run(const MatchFinder::MatchResult &Result) {
     return;
   }
 
-  if (Transformations->empty()) {
-    // No rewrite applied (but no error encountered either).
-    transformer::detail::getRuleMatchLoc(Result).print(
-        llvm::errs() << "note: skipping match at loc ", *Result.SourceManager);
-    llvm::errs() << "\n";
+  if (Transformations->empty())
     return;
-  }
 
   // Group the transformations, by file, into AtomicChanges, each anchored by
   // the location of the first change in that file.
diff --git a/clang/unittests/Tooling/TransformerTest.cpp b/clang/unittests/Tooling/TransformerTest.cpp
index 1a68eb1d172a3..77fd380410b27 100644
--- a/clang/unittests/Tooling/TransformerTest.cpp
+++ b/clang/unittests/Tooling/TransformerTest.cpp
@@ -114,7 +114,9 @@ class ClangRefactoringTestBase : public testing::Test {
       if (C) {
         Changes.push_back(std::move(*C));
       } else {
-        consumeError(C.takeError());
+        // FIXME: stash this error rather then printing.
+        llvm::errs() << "Error generating changes: "
+                     << llvm::toString(C.takeError()) << "\n";
         ++ErrorCount;
       }
     };
@@ -414,6 +416,120 @@ TEST_F(TransformerTest, ShrinkTo) {
            Input, Expected);
 }
 
+// Rewrite various Stmts inside a Decl.
+TEST_F(TransformerTest, RewriteDescendantsDeclChangeStmt) {
+  std::string Input =
+      "int f(int x) { int y = x; { int z = x * x; } return x; }";
+  std::string Expected =
+      "int f(int x) { int y = 3; { int z = 3 * 3; } return 3; }";
+  auto InlineX =
+      makeRule(declRefExpr(to(varDecl(hasName("x")))), changeTo(cat("3")));
+  testRule(makeRule(functionDecl(hasName("f")).bind("fun"),
+                    rewriteDescendants("fun", InlineX)),
+           Input, Expected);
+}
+
+// Rewrite various TypeLocs inside a Decl.
+TEST_F(TransformerTest, RewriteDescendantsDeclChangeTypeLoc) {
+  std::string Input = "int f(int *x) { return *x; }";
+  std::string Expected = "char f(char *x) { return *x; }";
+  auto IntToChar = makeRule(typeLoc(loc(qualType(isInteger(), builtinType()))),
+                            changeTo(cat("char")));
+  testRule(makeRule(functionDecl(hasName("f")).bind("fun"),
+                    rewriteDescendants("fun", IntToChar)),
+           Input, Expected);
+}
+
+TEST_F(TransformerTest, RewriteDescendantsStmt) {
+  // Add an unrelated definition to the header that also has a variable named
+  // "x", to test that the rewrite is limited to the scope we intend.
+  appendToHeader(R"cc(int g(int x) { return x; })cc");
+  std::string Input =
+      "int f(int x) { int y = x; { int z = x * x; } return x; }";
+  std::string Expected =
+      "int f(int x) { int y = 3; { int z = 3 * 3; } return 3; }";
+  auto InlineX =
+      makeRule(declRefExpr(to(varDecl(hasName("x")))), changeTo(cat("3")));
+  testRule(makeRule(functionDecl(hasName("f"), hasBody(stmt().bind("body"))),
+                    rewriteDescendants("body", InlineX)),
+           Input, Expected);
+}
+
+TEST_F(TransformerTest, RewriteDescendantsStmtWithAdditionalChange) {
+  std::string Input =
+      "int f(int x) { int y = x; { int z = x * x; } return x; }";
+  std::string Expected =
+      "int newName(int x) { int y = 3; { int z = 3 * 3; } return 3; }";
+  auto InlineX =
+      makeRule(declRefExpr(to(varDecl(hasName("x")))), changeTo(cat("3")));
+  testRule(
+      makeRule(
+          functionDecl(hasName("f"), hasBody(stmt().bind("body"))).bind("f"),
+          flatten(changeTo(name("f"), cat("newName")),
+                  rewriteDescendants("body", InlineX))),
+      Input, Expected);
+}
+
+TEST_F(TransformerTest, RewriteDescendantsTypeLoc) {
+  std::string Input = "int f(int *x) { return *x; }";
+  std::string Expected = "int f(char *x) { return *x; }";
+  auto IntToChar =
+      makeRule(typeLoc(loc(qualType(isInteger(), builtinType()))).bind("loc"),
+               changeTo(cat("char")));
+  testRule(
+      makeRule(functionDecl(hasName("f"),
+                            hasParameter(0, varDecl(hasTypeLoc(
+                                                typeLoc().bind("parmType"))))),
+               rewriteDescendants("parmType", IntToChar)),
+      Input, Expected);
+}
+
+TEST_F(TransformerTest, RewriteDescendantsReferToParentBinding) {
+  std::string Input =
+      "int f(int p) { int y = p; { int z = p * p; } return p; }";
+  std::string Expected =
+      "int f(int p) { int y = 3; { int z = 3 * 3; } return 3; }";
+  std::string VarId = "var";
+  auto InlineVar = makeRule(declRefExpr(to(varDecl(equalsBoundNode(VarId)))),
+                            changeTo(cat("3")));
+  testRule(makeRule(functionDecl(hasName("f"),
+                                 hasParameter(0, varDecl().bind(VarId)))
+                        .bind("fun"),
+                    rewriteDescendants("fun", InlineVar)),
+           Input, Expected);
+}
+
+TEST_F(TransformerTest, RewriteDescendantsUnboundNode) {
+  std::string Input =
+      "int f(int x) { int y = x; { int z = x * x; } return x; }";
+  auto InlineX =
+      makeRule(declRefExpr(to(varDecl(hasName("x")))), changeTo(cat("3")));
+  Transformer T(makeRule(functionDecl(hasName("f")),
+                         rewriteDescendants("UNBOUND", InlineX)),
+                consumer());
+  T.registerMatchers(&MatchFinder);
+  EXPECT_FALSE(rewrite(Input));
+  EXPECT_THAT(Changes, IsEmpty());
+  EXPECT_EQ(ErrorCount, 1);
+}
+
+TEST_F(TransformerTest, RewriteDescendantsInvalidNodeType) {
+  std::string Input =
+      "int f(int x) { int y = x; { int z = x * x; } return x; }";
+  auto IntToChar =
+      makeRule(qualType(isInteger(), builtinType()), changeTo(cat("char")));
+  Transformer T(
+      makeRule(functionDecl(
+                   hasName("f"),
+                   hasParameter(0, varDecl(hasType(qualType().bind("type"))))),
+               rewriteDescendants("type", IntToChar)),
+      consumer());
+  T.registerMatchers(&MatchFinder);
+  EXPECT_FALSE(rewrite(Input));
+  EXPECT_THAT(Changes, IsEmpty());
+  EXPECT_EQ(ErrorCount, 1);
+}
+
 TEST_F(TransformerTest, InsertBeforeEdit) {
   std::string Input = R"cc(
     int f() {

From 9840208db6980f690d09b209e6ad6d57133ec5e5 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev@hotmail.com>
Date: Wed, 15 Jul 2020 17:32:02 -0400
Subject: [PATCH 0015/1035] [OPENMP] Fix PR46730: Fix compiler crash on
 taskloop over constructible loop counters.

Summary:
If the variable is constrcutible, its copy is created by calling a
constructor. Such variables are duplicated and thus, must be captured.

Reviewers: jdoerfert

Subscribers: yaxunl, guansong, cfe-commits, sstefan1, caomhin

Tags: #clang

Differential Revision: https://reviews.llvm.org/D83909
---
 clang/lib/Sema/SemaOpenMP.cpp          |  6 +++++-
 clang/test/OpenMP/taskloop_codegen.cpp | 16 ++++++++++++++++
 2 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index 51609e37e20ca..0192df3bd170d 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -2244,7 +2244,11 @@ OpenMPClauseKind Sema::isOpenMPPrivateDecl(ValueDecl *D, unsigned Level,
           [](OpenMPDirectiveKind K) { return isOpenMPTaskingDirective(K); },
           Level)) {
     bool IsTriviallyCopyable =
-        D->getType().getNonReferenceType().isTriviallyCopyableType(Context);
+        D->getType().getNonReferenceType().isTriviallyCopyableType(Context) &&
+        !D->getType()
+             .getNonReferenceType()
+             .getCanonicalType()
+             ->getAsCXXRecordDecl();
     OpenMPDirectiveKind DKind = DSAStack->getDirective(Level);
     SmallVector<OpenMPDirectiveKind, 4> CaptureRegions;
     getOpenMPCaptureRegions(CaptureRegions, DKind);
diff --git a/clang/test/OpenMP/taskloop_codegen.cpp b/clang/test/OpenMP/taskloop_codegen.cpp
index 55e43ff3a1152..7402c2ad65eba 100644
--- a/clang/test/OpenMP/taskloop_codegen.cpp
+++ b/clang/test/OpenMP/taskloop_codegen.cpp
@@ -229,4 +229,20 @@ struct S {
 // CHECK: br label %
 // CHECK: ret i32 0
 
+class St {
+public:
+  operator int();
+  St &operator+=(int);
+};
+
+// CHECK-LABEL: taskloop_with_class
+void taskloop_with_class() {
+  St s1;
+  // CHECK: [[TD:%.+]] = call i8* @__kmpc_omp_task_alloc(%struct.ident_t* @{{.+}}, i32 [[GTID:%.+]], i32 1, i64 88, i64 8, i32 (i32, i8*)* bitcast (i32 (i32, [[TD_TYPE:%.+]]*)* @{{.+}} to i32 (i32, i8*)*))
+  // CHECK: call void @__kmpc_taskloop(%struct.ident_t* @{{.+}}, i32 [[GTID]], i8* [[TD]], i32 1, i64* %{{.+}}, i64* %{{.+}}, i64 %{{.+}}, i32 1, i32 0, i64 0, i8* bitcast (void ([[TD_TYPE]]*, [[TD_TYPE]]*, i32)* @{{.+}} to i8*))
+#pragma omp taskloop
+  for (St s = St(); s < s1; s += 1) {
+  }
+}
+
 #endif

From 77b61177d7d4c4fe8714f8828123f626f4549be1 Mon Sep 17 00:00:00 2001
From: diggerlin <digger.llvm@gmail.com>
Date: Fri, 24 Jul 2020 11:28:17 -0400
Subject: [PATCH 0016/1035] [AIX] remove -u from the clang when invoke aix as
 assembler

SUMMARY:

since we add .extern directive for external symbol, the -u option for aix as do not need any more.

Reviewers:  Jason liu

Differential Revision: https://reviews.llvm.org/D84356
---
 clang/lib/Driver/ToolChains/AIX.cpp | 6 ------
 clang/test/Driver/aix-as.c          | 7 -------
 2 files changed, 13 deletions(-)

diff --git a/clang/lib/Driver/ToolChains/AIX.cpp b/clang/lib/Driver/ToolChains/AIX.cpp
index ac5544eedb00b..f9d8e18d6fd01 100644
--- a/clang/lib/Driver/ToolChains/AIX.cpp
+++ b/clang/lib/Driver/ToolChains/AIX.cpp
@@ -44,12 +44,6 @@ void aix::Assembler::ConstructJob(Compilation &C, const JobAction &JA,
     CmdArgs.push_back("-a64");
   }
 
-  // Accept an undefined symbol as an extern so that an error message is not
-  // displayed. Otherwise, undefined symbols are flagged with error messages.
-  // FIXME: This should be removed when the assembly generation from the
-  // compiler is able to write externs properly.
-  CmdArgs.push_back("-u");
-
   // Accept any mixture of instructions.
   // On Power for AIX and Linux, this behaviour matches that of GCC for both the
   // user-provided assembler source case and the compiler-produced assembler
diff --git a/clang/test/Driver/aix-as.c b/clang/test/Driver/aix-as.c
index cb3053f5acd31..aa8c610359037 100644
--- a/clang/test/Driver/aix-as.c
+++ b/clang/test/Driver/aix-as.c
@@ -9,7 +9,6 @@
 // CHECK-AS32: {{.*}}clang{{(.exe)?}}" "-cc1" "-triple" "powerpc-ibm-aix7.1.0.0"
 // CHECK-AS32: "{{.*}}as{{(.exe)?}}" 
 // CHECK-AS32: "-a32" 
-// CHECK-AS32: "-u" 
 // CHECK-AS32: "-many" 
 
 // Check powerpc64-ibm-aix7.1.0.0, 64-bit.
@@ -20,7 +19,6 @@
 // CHECK-AS64: {{.*}}clang{{(.exe)?}}" "-cc1" "-triple" "powerpc64-ibm-aix7.1.0.0"
 // CHECK-AS64: "{{.*}}as{{(.exe)?}}" 
 // CHECK-AS64: "-a64" 
-// CHECK-AS64: "-u" 
 // CHECK-AS64: "-many"
 
 // Check powerpc-ibm-aix7.1.0.0, 32-bit. -Xassembler <arg> option. 
@@ -32,7 +30,6 @@
 // CHECK-AS32-Xassembler: {{.*}}clang{{(.exe)?}}" "-cc1" "-triple" "powerpc-ibm-aix7.1.0.0"
 // CHECK-AS32-Xassembler: "{{.*}}as{{(.exe)?}}" 
 // CHECK-AS32-Xassembler: "-a32" 
-// CHECK-AS32-Xassembler: "-u" 
 // CHECK-AS32-Xassembler: "-many"
 // CHECK-AS32-Xassembler: "-w"
 
@@ -45,7 +42,6 @@
 // CHECK-AS64-Wa: {{.*}}clang{{(.exe)?}}" "-cc1" "-triple" "powerpc64-ibm-aix7.1.0.0"
 // CHECK-AS64-Wa: "{{.*}}as{{(.exe)?}}" 
 // CHECK-AS64-Wa: "-a64" 
-// CHECK-AS64-Wa: "-u" 
 // CHECK-AS64-Wa: "-many"
 // CHECK-AS64-Wa: "-v"
 // CHECK-AS64-Wa: "-w"
@@ -60,13 +56,10 @@
 // CHECK-AS32-MultiInput-NOT: warning:
 // CHECK-AS32-MultiInput: "{{.*}}as{{(.exe)?}}"
 // CHECK-AS32-MultiInput: "-a32"
-// CHECK-AS32-MultiInput: "-u"
 // CHECK-AS32-MultiInput: "-many"
 // CHECK-AS32-MultiInput: "{{.*}}as{{(.exe)?}}"
 // CHECK-AS32-MultiInput: "-a32"
-// CHECK-AS32-MultiInput: "-u"
 // CHECK-AS32-MultiInput: "-many"
 // CHECK-AS32-MultiInput: "{{.*}}as{{(.exe)?}}"
 // CHECK-AS32-MultiInput: "-a32"
-// CHECK-AS32-MultiInput: "-u"
 // CHECK-AS32-MultiInput: "-many"

From 5934df0c9abe94fc450fbcf0ceca21cf838840e9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicolai=20H=C3=A4hnle?= <nicolai.haehnle@amd.com>
Date: Fri, 24 Jul 2020 18:18:09 +0200
Subject: [PATCH 0017/1035] MachineBasicBlock: add printName method

Common up some existing MBB name printing logic into a single place.
Note that basic block dumping now prints the same set of attributes as
the MIRPrinter.

Change-Id: I8f022bbd922e831bc96d63143d7472c03282530b

Differential Revision: https://reviews.llvm.org/D83253
---
 llvm/include/llvm/CodeGen/MachineBasicBlock.h |   8 ++
 llvm/lib/CodeGen/MIRPrinter.cpp               |  56 +-------
 llvm/lib/CodeGen/MachineBasicBlock.cpp        | 126 +++++++++++++-----
 3 files changed, 104 insertions(+), 86 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/MachineBasicBlock.h b/llvm/include/llvm/CodeGen/MachineBasicBlock.h
index d6cb7211cf70e..0360e706cbc4a 100644
--- a/llvm/include/llvm/CodeGen/MachineBasicBlock.h
+++ b/llvm/include/llvm/CodeGen/MachineBasicBlock.h
@@ -872,6 +872,14 @@ class MachineBasicBlock
   void print(raw_ostream &OS, ModuleSlotTracker &MST,
              const SlotIndexes * = nullptr, bool IsStandalone = true) const;
 
+  enum PrintNameFlag {
+    PrintNameIr = (1 << 0), ///< Add IR name where available
+    PrintNameAttributes = (1 << 1), ///< Print attributes
+  };
+
+  void printName(raw_ostream &os, unsigned printNameFlags = PrintNameIr,
+                 ModuleSlotTracker *moduleSlotTracker = nullptr) const;
+
   // Printing method used by LoopInfo.
   void printAsOperand(raw_ostream &OS, bool PrintType = true) const;
 
diff --git a/llvm/lib/CodeGen/MIRPrinter.cpp b/llvm/lib/CodeGen/MIRPrinter.cpp
index fa23df6288e99..dde0dc456c05f 100644
--- a/llvm/lib/CodeGen/MIRPrinter.cpp
+++ b/llvm/lib/CodeGen/MIRPrinter.cpp
@@ -608,58 +608,10 @@ bool MIPrinter::canPredictSuccessors(const MachineBasicBlock &MBB) const {
 
 void MIPrinter::print(const MachineBasicBlock &MBB) {
   assert(MBB.getNumber() >= 0 && "Invalid MBB number");
-  OS << "bb." << MBB.getNumber();
-  bool HasAttributes = false;
-  if (const auto *BB = MBB.getBasicBlock()) {
-    if (BB->hasName()) {
-      OS << "." << BB->getName();
-    } else {
-      HasAttributes = true;
-      OS << " (";
-      int Slot = MST.getLocalSlot(BB);
-      if (Slot == -1)
-        OS << "<ir-block badref>";
-      else
-        OS << (Twine("%ir-block.") + Twine(Slot)).str();
-    }
-  }
-  if (MBB.hasAddressTaken()) {
-    OS << (HasAttributes ? ", " : " (");
-    OS << "address-taken";
-    HasAttributes = true;
-  }
-  if (MBB.isEHPad()) {
-    OS << (HasAttributes ? ", " : " (");
-    OS << "landing-pad";
-    HasAttributes = true;
-  }
-  if (MBB.isEHFuncletEntry()) {
-    OS << (HasAttributes ? ", " : " (");
-    OS << "ehfunclet-entry";
-    HasAttributes = true;
-  }
-  if (MBB.getAlignment() != Align(1)) {
-    OS << (HasAttributes ? ", " : " (");
-    OS << "align " << MBB.getAlignment().value();
-    HasAttributes = true;
-  }
-  if (MBB.getSectionID() != MBBSectionID(0)) {
-    OS << (HasAttributes ? ", " : " (");
-    OS << "bbsections ";
-    switch (MBB.getSectionID().Type) {
-    case MBBSectionID::SectionType::Exception:
-      OS << "Exception";
-      break;
-    case MBBSectionID::SectionType::Cold:
-      OS << "Cold";
-      break;
-    default:
-      OS << MBB.getSectionID().Number;
-    }
-    HasAttributes = true;
-  }
-  if (HasAttributes)
-    OS << ")";
+  MBB.printName(OS,
+                MachineBasicBlock::PrintNameIr |
+                    MachineBasicBlock::PrintNameAttributes,
+                &MST);
   OS << ":\n";
 
   bool HasLineAttributes = false;
diff --git a/llvm/lib/CodeGen/MachineBasicBlock.cpp b/llvm/lib/CodeGen/MachineBasicBlock.cpp
index 2d4b60435d962..626c04074a617 100644
--- a/llvm/lib/CodeGen/MachineBasicBlock.cpp
+++ b/llvm/lib/CodeGen/MachineBasicBlock.cpp
@@ -338,39 +338,7 @@ void MachineBasicBlock::print(raw_ostream &OS, ModuleSlotTracker &MST,
   if (Indexes && PrintSlotIndexes)
     OS << Indexes->getMBBStartIdx(this) << '\t';
 
-  OS << "bb." << getNumber();
-  bool HasAttributes = false;
-  if (const auto *BB = getBasicBlock()) {
-    if (BB->hasName()) {
-      OS << "." << BB->getName();
-    } else {
-      HasAttributes = true;
-      OS << " (";
-      int Slot = MST.getLocalSlot(BB);
-      if (Slot == -1)
-        OS << "<ir-block badref>";
-      else
-        OS << (Twine("%ir-block.") + Twine(Slot)).str();
-    }
-  }
-
-  if (hasAddressTaken()) {
-    OS << (HasAttributes ? ", " : " (");
-    OS << "address-taken";
-    HasAttributes = true;
-  }
-  if (isEHPad()) {
-    OS << (HasAttributes ? ", " : " (");
-    OS << "landing-pad";
-    HasAttributes = true;
-  }
-  if (getAlignment() != Align(1)) {
-    OS << (HasAttributes ? ", " : " (");
-    OS << "align " << Log2(getAlignment());
-    HasAttributes = true;
-  }
-  if (HasAttributes)
-    OS << ")";
+  printName(OS, PrintNameIr | PrintNameAttributes, &MST);
   OS << ":\n";
 
   const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
@@ -478,9 +446,99 @@ void MachineBasicBlock::print(raw_ostream &OS, ModuleSlotTracker &MST,
   }
 }
 
+/// Print the basic block's name as:
+///
+///    bb.{number}[.{ir-name}] [(attributes...)]
+///
+/// The {ir-name} is only printed when the \ref PrintNameIr flag is passed
+/// (which is the default). If the IR block has no name, it is identified
+/// numerically using the attribute syntax as "(%ir-block.{ir-slot})".
+///
+/// When the \ref PrintNameAttributes flag is passed, additional attributes
+/// of the block are printed when set.
+///
+/// \param printNameFlags Combination of \ref PrintNameFlag flags indicating
+///                       the parts to print.
+/// \param moduleSlotTracker Optional ModuleSlotTracker. This method will
+///                          incorporate its own tracker when necessary to
+///                          determine the block's IR name.
+void MachineBasicBlock::printName(raw_ostream &os, unsigned printNameFlags,
+                                  ModuleSlotTracker *moduleSlotTracker) const {
+  os << "bb." << getNumber();
+  bool hasAttributes = false;
+
+  if (printNameFlags & PrintNameIr) {
+    if (const auto *bb = getBasicBlock()) {
+      if (bb->hasName()) {
+        os << '.' << bb->getName();
+      } else {
+        hasAttributes = true;
+        os << " (";
+
+        int slot = -1;
+
+        if (moduleSlotTracker) {
+          slot = moduleSlotTracker->getLocalSlot(bb);
+        } else if (bb->getParent()) {
+          ModuleSlotTracker tmpTracker(bb->getModule(), false);
+          tmpTracker.incorporateFunction(*bb->getParent());
+          slot = tmpTracker.getLocalSlot(bb);
+        }
+
+        if (slot == -1)
+          os << "<ir-block badref>";
+        else
+          os << (Twine("%ir-block.") + Twine(slot)).str();
+      }
+    }
+  }
+
+  if (printNameFlags & PrintNameAttributes) {
+    if (hasAddressTaken()) {
+      os << (hasAttributes ? ", " : " (");
+      os << "address-taken";
+      hasAttributes = true;
+    }
+    if (isEHPad()) {
+      os << (hasAttributes ? ", " : " (");
+      os << "landing-pad";
+      hasAttributes = true;
+    }
+    if (isEHFuncletEntry()) {
+      os << (hasAttributes ? ", " : " (");
+      os << "ehfunclet-entry";
+      hasAttributes = true;
+    }
+    if (getAlignment() != Align(1)) {
+      os << (hasAttributes ? ", " : " (");
+      os << "align " << getAlignment().value();
+      hasAttributes = true;
+    }
+    if (getSectionID() != MBBSectionID(0)) {
+      os << (hasAttributes ? ", " : " (");
+      os << "bbsections ";
+      switch (getSectionID().Type) {
+      case MBBSectionID::SectionType::Exception:
+        os << "Exception";
+        break;
+      case MBBSectionID::SectionType::Cold:
+        os << "Cold";
+        break;
+      default:
+        os << getSectionID().Number;
+      }
+      hasAttributes = true;
+    }
+  }
+
+  if (hasAttributes)
+    os << ')';
+}
+
 void MachineBasicBlock::printAsOperand(raw_ostream &OS,
                                        bool /*PrintType*/) const {
-  OS << "%bb." << getNumber();
+  OS << '%';
+  printName(OS, 0);
 }
 
 void MachineBasicBlock::removeLiveIn(MCPhysReg Reg, LaneBitmask LaneMask) {

From 22c16360dd00230987fee5f6f3c57f8071144cc1 Mon Sep 17 00:00:00 2001
From: Fred Riss <friss@apple.com>
Date: Fri, 17 Jul 2020 17:59:15 -0700
Subject: [PATCH 0018/1035] [lldb/ObjectFileMachO] Correctly account for
 resolver symbols

Summary:
The resolver addresses stored in the dyld trie are relative to the base
of the __TEXT segment. This is usually 0 in a dylib, so this was never
noticed, but it is not 0 for most dylibs integrated in the shared cache.
As we started using the shared cache images recently as symbol source,
this causes LLDB to fail to resolve symbols which go through a runtime
resolver.

Reviewers: jasonmolenda, jingham

Subscribers: lldb-commits

Tags: #lldb

Differential Revision: https://reviews.llvm.org/D84083
---
 lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp | 2 ++
 lldb/test/API/macosx/indirect_symbol/Makefile             | 3 ++-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp b/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp
index ab1a6a8bb5f3e..338c798e6cef6 100644
--- a/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp
+++ b/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp
@@ -1990,6 +1990,8 @@ static bool ParseTrieEntries(DataExtractor &data, lldb::offset_t offset,
       if (e.entry.flags & EXPORT_SYMBOL_FLAGS_STUB_AND_RESOLVER) {
         e.entry.other = data.GetULEB128(&offset);
         uint64_t resolver_addr = e.entry.other;
+        if (text_seg_base_addr != LLDB_INVALID_ADDRESS)
+          resolver_addr += text_seg_base_addr;
         if (is_arm)
           resolver_addr &= THUMB_ADDRESS_BIT_MASK;
         resolver_addresses.insert(resolver_addr);
diff --git a/lldb/test/API/macosx/indirect_symbol/Makefile b/lldb/test/API/macosx/indirect_symbol/Makefile
index 929ed58f75757..9069302b39c4f 100644
--- a/lldb/test/API/macosx/indirect_symbol/Makefile
+++ b/lldb/test/API/macosx/indirect_symbol/Makefile
@@ -8,7 +8,8 @@ include Makefile.rules
 
 build-libindirect: indirect.c
 	$(MAKE) -f $(MAKEFILE_RULES) \
-		DYLIB_C_SOURCES=indirect.c DYLIB_NAME=indirect DYLIB_ONLY=YES
+		DYLIB_C_SOURCES=indirect.c DYLIB_NAME=indirect DYLIB_ONLY=YES \
+		LD_EXTRAS="-Wl,-image_base,0x200000000"
 
 build-libreepxoprt: reexport.c
 	$(MAKE) -f $(MAKEFILE_RULES) \

From 1c7c69c795b22d73d038dd9b49de921cfd9d3468 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Fri, 24 Jul 2020 17:28:01 +0100
Subject: [PATCH 0019/1035] [ValueTracking] Check for ConstantExpr before using
 recursive helpers.

Make sure we do not call
constainsConstantExpression/containsUndefElement on ConstantExpression,
which is not supported.

In particular, containsUndefElement/constainsConstantExpression are only
supported on constants which are supported by getAggregateElement.

Unfortunately there's no convenient way to check if a constant supports
getAggregateElement, so just check for non-constantexpressions with
vector type. Other users of those functions do so too.

Reviewers: spatel, nikic, craig.topper, lebedev.ri, jdoerfert, aqjune

Reviewed By: jdoerfert

Differential Revision: https://reviews.llvm.org/D84512
---
 llvm/lib/Analysis/ValueTracking.cpp           |  4 ++--
 .../constexpr-vector-constainsundef-crash.ll  | 23 +++++++++++++++++++
 2 files changed, 25 insertions(+), 2 deletions(-)
 create mode 100644 llvm/test/Transforms/GVN/constexpr-vector-constainsundef-crash.ll

diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index 0ab2a1350af31..116916a9be2d2 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -4794,8 +4794,8 @@ bool llvm::isGuaranteedNotToBeUndefOrPoison(const Value *V,
         isa<ConstantPointerNull>(C) || isa<Function>(C))
       return true;
 
-    if (C->getType()->isVectorTy())
-      return !C->containsUndefElement() && !C->containsConstantExpression();
+    if (C->getType()->isVectorTy() && !isa<ConstantExpr>(C))
+      return !C->containsConstantExpression() && !C->containsUndefElement();
   }
 
   // Strip cast operations from a pointer value.
diff --git a/llvm/test/Transforms/GVN/constexpr-vector-constainsundef-crash.ll b/llvm/test/Transforms/GVN/constexpr-vector-constainsundef-crash.ll
new file mode 100644
index 0000000000000..b99cade66daaa
--- /dev/null
+++ b/llvm/test/Transforms/GVN/constexpr-vector-constainsundef-crash.ll
@@ -0,0 +1,23 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -gvn -S %s | FileCheck %s
+
+; Reduced test case from
+; https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=24278
+
+; Make sure we do not crash when dealing with a vector constant expression.
+define <4 x i64*> @test(i64* %ptr) {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <4 x i64*> getelementptr (i64, i64* null, <4 x i64> <i64 0, i64 0, i64 0, i64 -128>)
+;
+entry:
+  %B9 = sdiv i16 -32768, 256
+  %L3 = load i64, i64* %ptr, align 4
+  %B3 = sub i16 0, %B9
+  %0 = insertelement <4 x i16> undef, i16 %B3, i32 3
+  %1 = sub <4 x i16> zeroinitializer, %0
+  %2 = sext <4 x i16> %1 to <4 x i32>
+  %3 = getelementptr inbounds i64, i64* null, <4 x i32> %2
+  %I6 = insertelement <4 x i64*> %3, i64* undef, i64 %L3
+  ret <4 x i64*> %I6
+}

From 8a4878cc116c7ba2e1031d63e108610920c5ef5c Mon Sep 17 00:00:00 2001
From: Dokyung Song <dokyungs@google.com>
Date: Fri, 24 Jul 2020 16:14:42 +0000
Subject: [PATCH 0020/1035] [libFuzzer] Disable noasan-memcmp64.test and
 bcmp.test on Windows.

Summary: This patch disables (i) noasan-memcmp64.test on Windows as libFuzzer's interceptors are only supported on Linux for now, and (ii) bcmp.test as on Windows bcmp is not available in strings.h.

Reviewers: morehouse, hctim, kcc

Subscribers: #sanitizers

Tags: #sanitizers

Differential Revision: https://reviews.llvm.org/D84536
---
 compiler-rt/test/fuzzer/bcmp.test            | 2 +-
 compiler-rt/test/fuzzer/memcmp64.test        | 4 ----
 compiler-rt/test/fuzzer/noasan-memcmp64.test | 6 ++++++
 3 files changed, 7 insertions(+), 5 deletions(-)
 create mode 100644 compiler-rt/test/fuzzer/noasan-memcmp64.test

diff --git a/compiler-rt/test/fuzzer/bcmp.test b/compiler-rt/test/fuzzer/bcmp.test
index 37ee6bedd4ee1..5bbbe9845beb2 100644
--- a/compiler-rt/test/fuzzer/bcmp.test
+++ b/compiler-rt/test/fuzzer/bcmp.test
@@ -1,4 +1,4 @@
-UNSUPPORTED: freebsd
+UNSUPPORTED: freebsd, windows
 RUN: %cpp_compiler -DMEMCMP=bcmp %S/MemcmpTest.cpp -o %t
 RUN: not %run %t -seed=1 -runs=10000000 2>&1 | FileCheck %s
 CHECK: BINGO
diff --git a/compiler-rt/test/fuzzer/memcmp64.test b/compiler-rt/test/fuzzer/memcmp64.test
index feb81d1e4fb28..24d14bf73bbf4 100644
--- a/compiler-rt/test/fuzzer/memcmp64.test
+++ b/compiler-rt/test/fuzzer/memcmp64.test
@@ -1,8 +1,4 @@
 UNSUPPORTED: freebsd
 RUN: %cpp_compiler %S/Memcmp64BytesTest.cpp -o %t-Memcmp64BytesTest
 RUN: not %run %t-Memcmp64BytesTest        -seed=1 -runs=1000000   2>&1 | FileCheck %s
-
-RUN: %cpp_compiler -fno-sanitize=address -fno-builtin-memcmp %S/Memcmp64BytesTest.cpp -o %t-NoAsanMemcmp64BytesTest
-RUN: not %run %t-NoAsanMemcmp64BytesTest  -seed=1 -runs=1000000   2>&1 | FileCheck %s
-
 CHECK: BINGO
diff --git a/compiler-rt/test/fuzzer/noasan-memcmp64.test b/compiler-rt/test/fuzzer/noasan-memcmp64.test
new file mode 100644
index 0000000000000..a6b8f88594d03
--- /dev/null
+++ b/compiler-rt/test/fuzzer/noasan-memcmp64.test
@@ -0,0 +1,6 @@
+UNSUPPORTED: darwin, freebsd, windows
+
+RUN: %cpp_compiler -fno-sanitize=address -fno-builtin-memcmp %S/Memcmp64BytesTest.cpp -o %t-NoAsanMemcmp64BytesTest
+RUN: not %run %t-NoAsanMemcmp64BytesTest -seed=1 -runs=1000000   2>&1 | FileCheck %s
+
+CHECK: BINGO

From 58d84eb534252747115b358c890a1b79c65d4ad4 Mon Sep 17 00:00:00 2001
From: Adrian Prantl <aprantl@apple.com>
Date: Fri, 24 Jul 2020 09:43:15 -0700
Subject: [PATCH 0021/1035] debugserver: Support ios simulator load command
 disambiguation in qProcessInfo

This patch basically moves the disambiguation code from a place where
it was complicated to implement straight to where the load command is
parsed, which has the neat side affect of actually supporting all call
sites!

rdar://problem/66011909

Differential Revision: https://reviews.llvm.org/D84480
---
 .../macosx/simulator/TestSimulatorPlatform.py | 20 ++++++--
 lldb/tools/debugserver/source/DNB.cpp         |  5 +-
 .../debugserver/source/MacOSX/MachProcess.h   |  3 --
 .../debugserver/source/MacOSX/MachProcess.mm  | 51 ++++++++-----------
 4 files changed, 41 insertions(+), 38 deletions(-)

diff --git a/lldb/test/API/macosx/simulator/TestSimulatorPlatform.py b/lldb/test/API/macosx/simulator/TestSimulatorPlatform.py
index 6e67fdc879bef..824cb9eee295c 100644
--- a/lldb/test/API/macosx/simulator/TestSimulatorPlatform.py
+++ b/lldb/test/API/macosx/simulator/TestSimulatorPlatform.py
@@ -27,18 +27,28 @@ def check_debugserver(self, log, expected_platform, expected_version):
         """scan the debugserver packet log"""
         logfile = open(log, "r")
         dylib_info = None
-        response = False
+        process_info_ostype = None
+        expect_dylib_info_response = False
+        expect_process_info_response = False
         for line in logfile:
-            if response:
+            if expect_dylib_info_response:
                 while line[0] != '$':
                     line = line[1:]
                 line = line[1:]
                 # Unescape '}'.
                 dylib_info = json.loads(line.replace('}]','}')[:-4])
-                response = False
+                expect_dylib_info_response = False
             if 'send packet: $jGetLoadedDynamicLibrariesInfos:{' in line:
-                response = True
-
+                expect_dylib_info_response = True
+            if expect_process_info_response:
+                for pair in line.split(';'):
+                    keyval = pair.split(':')
+                    if len(keyval) == 2 and keyval[0] == 'ostype':
+                        process_info_ostype = keyval[1]
+            if 'send packet: $qProcessInfo#' in line:
+                expect_process_info_response = True
+
+        self.assertEquals(process_info_ostype, expected_platform)
         self.assertTrue(dylib_info)
         aout_info = None
         for image in dylib_info['images']:
diff --git a/lldb/tools/debugserver/source/DNB.cpp b/lldb/tools/debugserver/source/DNB.cpp
index af13a8f8208ba..0830ea36a91ae 100644
--- a/lldb/tools/debugserver/source/DNB.cpp
+++ b/lldb/tools/debugserver/source/DNB.cpp
@@ -1393,7 +1393,10 @@ const char *DNBGetDeploymentInfo(nub_process_t pid,
                                  uint32_t& patch_version) {
   MachProcessSP procSP;
   if (GetProcessSP(pid, procSP)) {
-    // FIXME: This doesn't correct for older ios simulator and macCatalyst.
+    // FIXME: This doesn't return the correct result when xctest (a
+    // macOS binary) is loaded with the macCatalyst dyld platform
+    // override. The image info corrects for this, but qProcessInfo
+    // will return what is in the binary.
     auto info = procSP->GetDeploymentInfo(lc, load_command_address);
     major_version = info.major_version;
     minor_version = info.minor_version;
diff --git a/lldb/tools/debugserver/source/MacOSX/MachProcess.h b/lldb/tools/debugserver/source/MacOSX/MachProcess.h
index c749dd8426c5d..9d712390ac2ac 100644
--- a/lldb/tools/debugserver/source/MacOSX/MachProcess.h
+++ b/lldb/tools/debugserver/source/MacOSX/MachProcess.h
@@ -236,9 +236,6 @@ class MachProcess {
     operator bool() { return platform > 0; }
     /// The Mach-O platform type;
     unsigned char platform = 0;
-    /// Pre-LC_BUILD_VERSION files don't disambiguate between ios and ios
-    /// simulator.
-    bool maybe_simulator = false;
     uint32_t major_version = 0;
     uint32_t minor_version = 0;
     uint32_t patch_version = 0;
diff --git a/lldb/tools/debugserver/source/MacOSX/MachProcess.mm b/lldb/tools/debugserver/source/MacOSX/MachProcess.mm
index 8a35f605daa38..10eaf38ea4357 100644
--- a/lldb/tools/debugserver/source/MacOSX/MachProcess.mm
+++ b/lldb/tools/debugserver/source/MacOSX/MachProcess.mm
@@ -617,7 +617,28 @@ static bool FBSAddEventDataToOptions(NSMutableDictionary *options,
     info.major_version = vers_cmd.version >> 16;
     info.minor_version = (vers_cmd.version >> 8) & 0xffu;
     info.patch_version = vers_cmd.version & 0xffu;
-    info.maybe_simulator = true;
+
+    // Disambiguate legacy simulator platforms.
+#if (defined(__x86_64__) || defined(__i386__))
+    // If we are running on Intel macOS, it is safe to assume this is
+    // really a back-deploying simulator binary.
+    switch (info.platform) {
+    case PLATFORM_IOS:
+      info.platform = PLATFORM_IOSSIMULATOR;
+      break;
+    case PLATFORM_TVOS:
+      info.platform = PLATFORM_TVOSSIMULATOR;
+      break;
+    case PLATFORM_WATCHOS:
+      info.platform = PLATFORM_WATCHOSSIMULATOR;
+      break;
+    }
+#else
+    // On an Apple Silicon macOS host, there is no ambiguity. The only
+    // binaries that use legacy load commands are back-deploying
+    // native iOS binaries. All simulator binaries use the newer,
+    // unambiguous LC_BUILD_VERSION load commands.
+#endif
   };
   switch (cmd) {
   case LC_VERSION_MIN_IPHONEOS:
@@ -778,34 +799,6 @@ static bool FBSAddEventDataToOptions(NSMutableDictionary *options,
         uuid_copy(inf.uuid, uuidcmd.uuid);
     }
     if (DeploymentInfo deployment_info = GetDeploymentInfo(lc, load_cmds_p)) {
-      // Simulator support. If the platform is ambiguous, use the dyld info.
-      if (deployment_info.maybe_simulator) {
-        if (deployment_info.maybe_simulator) {
-#if (defined(__x86_64__) || defined(__i386__))
-          // If dyld doesn't return a platform, use a heuristic.
-          // If we are running on Intel macOS, it is safe to assume
-          // this is really a back-deploying simulator binary.
-          switch (deployment_info.platform) {
-          case PLATFORM_IOS:
-            deployment_info.platform = PLATFORM_IOSSIMULATOR;
-            break;
-          case PLATFORM_TVOS:
-            deployment_info.platform = PLATFORM_TVOSSIMULATOR;
-            break;
-          case PLATFORM_WATCHOS:
-            deployment_info.platform = PLATFORM_WATCHOSSIMULATOR;
-            break;
-          }
-#else
-          // On an Apple Silicon macOS host, there is no
-          // ambiguity. The only binaries that use legacy load
-          // commands are back-deploying native iOS binaries. All
-          // simulator binaries use the newer, unambiguous
-          // LC_BUILD_VERSION load commands.
-          deployment_info.maybe_simulator = false;
-#endif
-        }
-      }
       const char *lc_platform = GetPlatformString(deployment_info.platform);
       // macCatalyst support.
       //

From f26aeab1b9f8fc7261ab8d31693570d6a56afa38 Mon Sep 17 00:00:00 2001
From: biplmish <biplmish@in.ibm.com>
Date: Fri, 24 Jul 2020 04:30:04 -0500
Subject: [PATCH 0022/1035] [test commit] Add my name to the CREDITS.TXT

Add my name to the CREDITS.TXT

This is my test commit. (NFC)

Differential Revision: https://reviews.llvm.org/D84488
---
 llvm/CREDITS.TXT | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/llvm/CREDITS.TXT b/llvm/CREDITS.TXT
index e998fa6bb257d..461b95bdcd8b5 100644
--- a/llvm/CREDITS.TXT
+++ b/llvm/CREDITS.TXT
@@ -537,3 +537,7 @@ D: PowerPC Backend Developer
 N: Djordje Todorovic
 E: djordje.todorovic@rt-rk.com
 D: Debug Information
+
+N: Biplob Mishra
+E: biplmish@in.ibm.com
+D: PowerPC Analysis

From 805e6bcf22f30e37b47cc40b7eb14ae891a277a2 Mon Sep 17 00:00:00 2001
From: Meera Nakrani <meera.nakrani@arm.com>
Date: Fri, 24 Jul 2020 17:22:56 +0000
Subject: [PATCH 0023/1035] Test Commit

Test commit - added whitespace in ARMInstrMVE.td
---
 llvm/lib/Target/ARM/ARMInstrMVE.td | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Target/ARM/ARMInstrMVE.td b/llvm/lib/Target/ARM/ARMInstrMVE.td
index dc701412b5950..fa04a82b0ee26 100644
--- a/llvm/lib/Target/ARM/ARMInstrMVE.td
+++ b/llvm/lib/Target/ARM/ARMInstrMVE.td
@@ -3773,7 +3773,7 @@ multiclass MVE_VABD_fp_m<MVEVectorVTInfo VTI>
   : MVE_VABDT_fp_m<VTI, int_arm_mve_vabd, int_arm_mve_abd_predicated>;
 
 defm MVE_VABDf32 : MVE_VABD_fp_m<MVE_v4f32>;
-defm MVE_VABDf16 : MVE_VABD_fp_m<MVE_v8f16>;
+defm MVE_VABDf16 : MVE_VABD_fp_m<MVE_v8f16>; 
 
 class MVE_VCVT_fix<string suffix, bit fsi, bit U, bit op,
                    Operand imm_operand_type>

From 3319d05630cdac66412a122efd75dfab073f5093 Mon Sep 17 00:00:00 2001
From: Roman Lebedev <lebedev.ri@gmail.com>
Date: Fri, 24 Jul 2020 20:31:35 +0300
Subject: [PATCH 0024/1035] [NFC][GVN] Improve loadpre-missed-opportunity.ll
 test again thanks to @fhahn

---
 .../GVN/loadpre-missed-opportunity.ll         | 77 +++++++++++--------
 1 file changed, 47 insertions(+), 30 deletions(-)

diff --git a/llvm/test/Transforms/GVN/loadpre-missed-opportunity.ll b/llvm/test/Transforms/GVN/loadpre-missed-opportunity.ll
index dae22d18334b8..1c967f48d546c 100644
--- a/llvm/test/Transforms/GVN/loadpre-missed-opportunity.ll
+++ b/llvm/test/Transforms/GVN/loadpre-missed-opportunity.ll
@@ -1,25 +1,33 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -gvn -S | FileCheck %s
 
-define void @loadpre_opportunity(i32** %arg, i1 %arg1, i1 %arg2, i1 %arg3) {
+define i32 @loadpre_opportunity(i32** %arg, i1 %arg1, i1 %arg2, i1 %arg3) {
 ; CHECK-LABEL: @loadpre_opportunity(
 ; CHECK-NEXT:  bb:
-; CHECK-NEXT:    br label [[BB9:%.*]]
-; CHECK:       bb6:
-; CHECK-NEXT:    br label [[BB9]]
-; CHECK:       bb9:
-; CHECK-NEXT:    br i1 [[ARG1:%.*]], label [[BB6:%.*]], label [[BB10:%.*]]
-; CHECK:       bb10:
-; CHECK-NEXT:    call void @somecall()
-; CHECK-NEXT:    br i1 [[ARG2:%.*]], label [[BB12:%.*]], label [[BB15:%.*]]
-; CHECK:       bb12:
-; CHECK-NEXT:    br label [[BB13:%.*]]
+; CHECK-NEXT:    [[I:%.*]] = load i32*, i32** [[ARG:%.*]], align 8
+; CHECK-NEXT:    [[I6:%.*]] = call i32 @use(i32* [[I]])
+; CHECK-NEXT:    br label [[BB11:%.*]]
+; CHECK:       bb7:
+; CHECK-NEXT:    [[I8:%.*]] = load i32*, i32** [[ARG]], align 8
+; CHECK-NEXT:    [[I10:%.*]] = call i32 @use(i32* [[I8]])
+; CHECK-NEXT:    br label [[BB11]]
+; CHECK:       bb11:
+; CHECK-NEXT:    [[I12:%.*]] = phi i32 [ [[I6]], [[BB:%.*]] ], [ [[I10]], [[BB7:%.*]] ]
+; CHECK-NEXT:    br i1 [[ARG1:%.*]], label [[BB7]], label [[BB13:%.*]]
 ; CHECK:       bb13:
-; CHECK-NEXT:    br i1 [[ARG3:%.*]], label [[BB14:%.*]], label [[BB13]]
+; CHECK-NEXT:    call void @somecall()
+; CHECK-NEXT:    br i1 [[ARG2:%.*]], label [[BB14:%.*]], label [[BB17:%.*]]
 ; CHECK:       bb14:
-; CHECK-NEXT:    br label [[BB15]]
+; CHECK-NEXT:    br label [[BB15:%.*]]
 ; CHECK:       bb15:
-; CHECK-NEXT:    br label [[BB6]]
+; CHECK-NEXT:    br i1 [[ARG3:%.*]], label [[BB16:%.*]], label [[BB15]]
+; CHECK:       bb16:
+; CHECK-NEXT:    br label [[BB17]]
+; CHECK:       bb17:
+; CHECK-NEXT:    [[I18:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[I18]], label [[BB7]], label [[BB19:%.*]]
+; CHECK:       bb19:
+; CHECK-NEXT:    ret i32 [[I12]]
 ;
 bb:
   %i = load i32*, i32** %arg, align 8
@@ -27,31 +35,40 @@ bb:
   br label %bb5
 
 bb5:
-  br label %bb9
-
-bb6:
-  %i7 = load i32*, i32** %arg, align 8
-  %i8 = getelementptr inbounds i32, i32* %i7, i64 0
-  br label %bb9
+  %i6 = call i32 @use(i32* %i4)
+  br label %bb11
 
-bb9:
-  br i1 %arg1, label %bb6, label %bb10
-
-bb10:
-  call void @somecall()
-  br i1 %arg2, label %bb12, label %bb15
+bb7:
+  %i8 = load i32*, i32** %arg, align 8
+  %i9 = getelementptr inbounds i32, i32* %i8, i64 0
+  %i10 = call i32 @use(i32* %i9)
+  br label %bb11
 
-bb12:
-  br label %bb13
+bb11:
+  %i12 = phi i32 [ %i6, %bb5 ], [ %i10, %bb7 ]
+  br i1 %arg1, label %bb7, label %bb13
 
 bb13:
-  br i1 %arg3, label %bb14, label %bb13
+  call void @somecall()
+  br i1 %arg2, label %bb14, label %bb17
 
 bb14:
   br label %bb15
 
 bb15:
-  br label %bb6
+  br i1 %arg3, label %bb16, label %bb15
+
+bb16:
+  br label %bb17
+
+bb17:
+  %i18 = call i1 @cond()
+  br i1 %i18, label %bb7, label %bb19
+
+bb19:
+  ret i32 %i12
 }
 
 declare void @somecall()
+declare i32 @use(i32*) readnone
+declare i1 @cond() readnone

From 998334da2b1536e7c8f11c560770c8d4cfacb354 Mon Sep 17 00:00:00 2001
From: Kostya Kortchinsky <kostyak@google.com>
Date: Thu, 16 Jul 2020 16:13:04 -0700
Subject: [PATCH 0025/1035] [scudo][standalone] Change the release loop for
 efficiency purposes

Summary:
On 32-b, the release algo loops multiple times over the freelist for a size
class, which lead to a decrease in performance when there were a lot of free
blocks.

This changes the release functions to loop only once over the freelist, at the
cost of using a little bit more memory for the release process: instead of
working on one region at a time, we pass the whole memory area covered by all
the regions for a given size class, and work on sub-areas of `RegionSize` in
this large area. For 64-b, we just have 1 sub-area encompassing the whole
region. Of course, not all the sub-areas within that large memory area will
belong to the class id we are working on, but those will just be left untouched
(which will not add to the RSS during the release process).

Reviewers: pcc, cferris, hctim, eugenis

Subscribers: llvm-commits, #sanitizers

Tags: #sanitizers

Differential Revision: https://reviews.llvm.org/D83993
---
 compiler-rt/lib/scudo/standalone/primary32.h  |  45 ++++---
 compiler-rt/lib/scudo/standalone/primary64.h  |   2 +-
 compiler-rt/lib/scudo/standalone/release.h    | 119 +++++++++++-------
 .../scudo/standalone/tests/release_test.cpp   |  27 ++--
 4 files changed, 114 insertions(+), 79 deletions(-)

diff --git a/compiler-rt/lib/scudo/standalone/primary32.h b/compiler-rt/lib/scudo/standalone/primary32.h
index 2ee0f6c600ab2..321cf92fae30e 100644
--- a/compiler-rt/lib/scudo/standalone/primary32.h
+++ b/compiler-rt/lib/scudo/standalone/primary32.h
@@ -467,28 +467,33 @@ class SizeClassAllocator32 {
       }
     }
 
-    // TODO(kostyak): currently not ideal as we loop over all regions and
-    // iterate multiple times over the same freelist if a ClassId spans multiple
-    // regions. But it will have to do for now.
-    uptr TotalReleasedBytes = 0;
-    const uptr MaxSize = (RegionSize / BlockSize) * BlockSize;
+    DCHECK_GT(MinRegionIndex, 0U);
+    uptr First = 0;
     for (uptr I = MinRegionIndex; I <= MaxRegionIndex; I++) {
       if (PossibleRegions[I] - 1U == ClassId) {
-        const uptr Region = I * RegionSize;
-        // If the region is the one currently associated to the size-class, we
-        // only need to release up to CurrentRegionAllocated, MaxSize otherwise.
-        const uptr Size = (Region == Sci->CurrentRegion)
-                              ? Sci->CurrentRegionAllocated
-                              : MaxSize;
-        ReleaseRecorder Recorder(Region);
-        releaseFreeMemoryToOS(Sci->FreeList, Region, Size, BlockSize,
-                              &Recorder);
-        if (Recorder.getReleasedRangesCount() > 0) {
-          Sci->ReleaseInfo.PushedBlocksAtLastRelease = Sci->Stats.PushedBlocks;
-          Sci->ReleaseInfo.RangesReleased += Recorder.getReleasedRangesCount();
-          Sci->ReleaseInfo.LastReleasedBytes = Recorder.getReleasedBytes();
-          TotalReleasedBytes += Sci->ReleaseInfo.LastReleasedBytes;
-        }
+        First = I;
+        break;
+      }
+    }
+    uptr Last = 0;
+    for (uptr I = MaxRegionIndex; I >= MinRegionIndex; I--) {
+      if (PossibleRegions[I] - 1U == ClassId) {
+        Last = I;
+        break;
+      }
+    }
+    uptr TotalReleasedBytes = 0;
+    if (First && Last) {
+      const uptr Base = First * RegionSize;
+      const uptr NumberOfRegions = Last - First + 1U;
+      ReleaseRecorder Recorder(Base);
+      releaseFreeMemoryToOS(Sci->FreeList, Base, RegionSize, NumberOfRegions,
+                            BlockSize, &Recorder);
+      if (Recorder.getReleasedRangesCount() > 0) {
+        Sci->ReleaseInfo.PushedBlocksAtLastRelease = Sci->Stats.PushedBlocks;
+        Sci->ReleaseInfo.RangesReleased += Recorder.getReleasedRangesCount();
+        Sci->ReleaseInfo.LastReleasedBytes = Recorder.getReleasedBytes();
+        TotalReleasedBytes += Sci->ReleaseInfo.LastReleasedBytes;
       }
     }
     Sci->ReleaseInfo.LastReleaseAtNs = getMonotonicTime();
diff --git a/compiler-rt/lib/scudo/standalone/primary64.h b/compiler-rt/lib/scudo/standalone/primary64.h
index 01e674bf3fba5..e37dc4951f238 100644
--- a/compiler-rt/lib/scudo/standalone/primary64.h
+++ b/compiler-rt/lib/scudo/standalone/primary64.h
@@ -481,7 +481,7 @@ class SizeClassAllocator64 {
 
     ReleaseRecorder Recorder(Region->RegionBeg, &Region->Data);
     releaseFreeMemoryToOS(Region->FreeList, Region->RegionBeg,
-                          Region->AllocatedUser, BlockSize, &Recorder);
+                          Region->AllocatedUser, 1U, BlockSize, &Recorder);
 
     if (Recorder.getReleasedRangesCount() > 0) {
       Region->ReleaseInfo.PushedBlocksAtLastRelease =
diff --git a/compiler-rt/lib/scudo/standalone/release.h b/compiler-rt/lib/scudo/standalone/release.h
index fd55ea24132e6..748e1c0011530 100644
--- a/compiler-rt/lib/scudo/standalone/release.h
+++ b/compiler-rt/lib/scudo/standalone/release.h
@@ -49,7 +49,10 @@ class ReleaseRecorder {
 // incremented past MaxValue.
 class PackedCounterArray {
 public:
-  PackedCounterArray(uptr NumCounters, uptr MaxValue) : N(NumCounters) {
+  PackedCounterArray(uptr NumberOfRegions, uptr CountersPerRegion,
+                     uptr MaxValue)
+      : Regions(NumberOfRegions), NumCounters(CountersPerRegion) {
+    CHECK_GT(Regions, 0);
     CHECK_GT(NumCounters, 0);
     CHECK_GT(MaxValue, 0);
     constexpr uptr MaxCounterBits = sizeof(*Buffer) * 8UL;
@@ -66,9 +69,10 @@ class PackedCounterArray {
     PackingRatioLog = getLog2(PackingRatio);
     BitOffsetMask = PackingRatio - 1;
 
-    BufferSize = (roundUpTo(N, static_cast<uptr>(1U) << PackingRatioLog) >>
-                  PackingRatioLog) *
-                 sizeof(*Buffer);
+    SizePerRegion =
+        roundUpTo(NumCounters, static_cast<uptr>(1U) << PackingRatioLog) >>
+        PackingRatioLog;
+    BufferSize = SizePerRegion * sizeof(*Buffer) * Regions;
     if (BufferSize <= (StaticBufferCount * sizeof(Buffer[0])) &&
         Mutex.tryLock()) {
       Buffer = &StaticBuffer[0];
@@ -89,41 +93,45 @@ class PackedCounterArray {
 
   bool isAllocated() const { return !!Buffer; }
 
-  uptr getCount() const { return N; }
+  uptr getCount() const { return NumCounters; }
 
-  uptr get(uptr I) const {
-    DCHECK_LT(I, N);
+  uptr get(uptr Region, uptr I) const {
+    DCHECK_LT(Region, Regions);
+    DCHECK_LT(I, NumCounters);
     const uptr Index = I >> PackingRatioLog;
     const uptr BitOffset = (I & BitOffsetMask) << CounterSizeBitsLog;
-    return (Buffer[Index] >> BitOffset) & CounterMask;
+    return (Buffer[Region * SizePerRegion + Index] >> BitOffset) & CounterMask;
   }
 
-  void inc(uptr I) const {
-    DCHECK_LT(get(I), CounterMask);
+  void inc(uptr Region, uptr I) const {
+    DCHECK_LT(get(Region, I), CounterMask);
     const uptr Index = I >> PackingRatioLog;
     const uptr BitOffset = (I & BitOffsetMask) << CounterSizeBitsLog;
     DCHECK_LT(BitOffset, SCUDO_WORDSIZE);
-    Buffer[Index] += static_cast<uptr>(1U) << BitOffset;
+    Buffer[Region * SizePerRegion + Index] += static_cast<uptr>(1U)
+                                              << BitOffset;
   }
 
-  void incRange(uptr From, uptr To) const {
+  void incRange(uptr Region, uptr From, uptr To) const {
     DCHECK_LE(From, To);
-    const uptr Top = Min(To + 1, N);
+    const uptr Top = Min(To + 1, NumCounters);
     for (uptr I = From; I < Top; I++)
-      inc(I);
+      inc(Region, I);
   }
 
   uptr getBufferSize() const { return BufferSize; }
 
-  static const uptr StaticBufferCount = 1024U;
+  static const uptr StaticBufferCount = 2048U;
 
 private:
-  const uptr N;
+  const uptr Regions;
+  const uptr NumCounters;
   uptr CounterSizeBitsLog;
   uptr CounterMask;
   uptr PackingRatioLog;
   uptr BitOffsetMask;
 
+  uptr SizePerRegion;
   uptr BufferSize;
   uptr *Buffer;
 
@@ -169,7 +177,8 @@ template <class ReleaseRecorderT> class FreePagesRangeTracker {
 template <class TransferBatchT, class ReleaseRecorderT>
 NOINLINE void
 releaseFreeMemoryToOS(const IntrusiveList<TransferBatchT> &FreeList, uptr Base,
-                      uptr Size, uptr BlockSize, ReleaseRecorderT *Recorder) {
+                      uptr RegionSize, uptr NumberOfRegions, uptr BlockSize,
+                      ReleaseRecorderT *Recorder) {
   const uptr PageSize = getPageSizeCached();
 
   // Figure out the number of chunks per page and whether we can take a fast
@@ -206,13 +215,15 @@ releaseFreeMemoryToOS(const IntrusiveList<TransferBatchT> &FreeList, uptr Base,
     }
   }
 
-  const uptr PagesCount = roundUpTo(Size, PageSize) / PageSize;
-  PackedCounterArray Counters(PagesCount, FullPagesBlockCountMax);
+  const uptr PagesCount = roundUpTo(RegionSize, PageSize) / PageSize;
+  PackedCounterArray Counters(NumberOfRegions, PagesCount,
+                              FullPagesBlockCountMax);
   if (!Counters.isAllocated())
     return;
 
   const uptr PageSizeLog = getLog2(PageSize);
-  const uptr RoundedSize = PagesCount << PageSizeLog;
+  const uptr RoundedRegionSize = PagesCount << PageSizeLog;
+  const uptr RoundedSize = NumberOfRegions * RoundedRegionSize;
 
   // Iterate over free chunks and count how many free chunks affect each
   // allocated page.
@@ -228,14 +239,17 @@ releaseFreeMemoryToOS(const IntrusiveList<TransferBatchT> &FreeList, uptr Base,
       for (u32 I = IsTransferBatch ? 1 : 0; I < It.getCount(); I++) {
         const uptr P = reinterpret_cast<uptr>(It.get(I)) - Base;
         // This takes care of P < Base and P >= Base + RoundedSize.
-        if (P < RoundedSize)
-          Counters.inc(P >> PageSizeLog);
+        if (P < RoundedSize) {
+          const uptr RegionIndex = NumberOfRegions == 1U ? 0 : P / RegionSize;
+          const uptr PInRegion = P - RegionIndex * RegionSize;
+          Counters.inc(RegionIndex, PInRegion >> PageSizeLog);
+        }
       }
     }
-    for (uptr P = Size; P < RoundedSize; P += BlockSize)
-      Counters.inc(P >> PageSizeLog);
   } else {
     // In all other cases chunks might affect more than one page.
+    DCHECK_GE(RegionSize, BlockSize);
+    const uptr LastBlockInRegion = ((RegionSize / BlockSize) - 1U) * BlockSize;
     for (const auto &It : FreeList) {
       // See TransferBatch comment above.
       const bool IsTransferBatch =
@@ -244,13 +258,24 @@ releaseFreeMemoryToOS(const IntrusiveList<TransferBatchT> &FreeList, uptr Base,
       for (u32 I = IsTransferBatch ? 1 : 0; I < It.getCount(); I++) {
         const uptr P = reinterpret_cast<uptr>(It.get(I)) - Base;
         // This takes care of P < Base and P >= Base + RoundedSize.
-        if (P < RoundedSize)
-          Counters.incRange(P >> PageSizeLog,
-                            (P + BlockSize - 1) >> PageSizeLog);
+        if (P < RoundedSize) {
+          const uptr RegionIndex = NumberOfRegions == 1U ? 0 : P / RegionSize;
+          uptr PInRegion = P - RegionIndex * RegionSize;
+          Counters.incRange(RegionIndex, PInRegion >> PageSizeLog,
+                            (PInRegion + BlockSize - 1) >> PageSizeLog);
+          // The last block in a region might straddle a page, so if it's
+          // free, we mark the following "pretend" memory block(s) as free.
+          if (PInRegion == LastBlockInRegion) {
+            PInRegion += BlockSize;
+            while (PInRegion < RoundedRegionSize) {
+              Counters.incRange(RegionIndex, PInRegion >> PageSizeLog,
+                                (PInRegion + BlockSize - 1) >> PageSizeLog);
+              PInRegion += BlockSize;
+            }
+          }
+        }
       }
     }
-    for (uptr P = Size; P < RoundedSize; P += BlockSize)
-      Counters.incRange(P >> PageSizeLog, (P + BlockSize - 1) >> PageSizeLog);
   }
 
   // Iterate over pages detecting ranges of pages with chunk Counters equal
@@ -258,8 +283,10 @@ releaseFreeMemoryToOS(const IntrusiveList<TransferBatchT> &FreeList, uptr Base,
   FreePagesRangeTracker<ReleaseRecorderT> RangeTracker(Recorder);
   if (SameBlockCountPerPage) {
     // Fast path, every page has the same number of chunks affecting it.
-    for (uptr I = 0; I < Counters.getCount(); I++)
-      RangeTracker.processNextPage(Counters.get(I) == FullPagesBlockCountMax);
+    for (uptr I = 0; I < NumberOfRegions; I++)
+      for (uptr J = 0; J < PagesCount; J++)
+        RangeTracker.processNextPage(Counters.get(I, J) ==
+                                     FullPagesBlockCountMax);
   } else {
     // Slow path, go through the pages keeping count how many chunks affect
     // each page.
@@ -270,23 +297,25 @@ releaseFreeMemoryToOS(const IntrusiveList<TransferBatchT> &FreeList, uptr Base,
     // except the first and the last one) and then the last chunk size, adding
     // up the number of chunks on the current page and checking on every step
     // whether the page boundary was crossed.
-    uptr PrevPageBoundary = 0;
-    uptr CurrentBoundary = 0;
-    for (uptr I = 0; I < Counters.getCount(); I++) {
-      const uptr PageBoundary = PrevPageBoundary + PageSize;
-      uptr BlocksPerPage = Pn;
-      if (CurrentBoundary < PageBoundary) {
-        if (CurrentBoundary > PrevPageBoundary)
-          BlocksPerPage++;
-        CurrentBoundary += Pnc;
+    for (uptr I = 0; I < NumberOfRegions; I++) {
+      uptr PrevPageBoundary = 0;
+      uptr CurrentBoundary = 0;
+      for (uptr J = 0; J < PagesCount; J++) {
+        const uptr PageBoundary = PrevPageBoundary + PageSize;
+        uptr BlocksPerPage = Pn;
         if (CurrentBoundary < PageBoundary) {
-          BlocksPerPage++;
-          CurrentBoundary += BlockSize;
+          if (CurrentBoundary > PrevPageBoundary)
+            BlocksPerPage++;
+          CurrentBoundary += Pnc;
+          if (CurrentBoundary < PageBoundary) {
+            BlocksPerPage++;
+            CurrentBoundary += BlockSize;
+          }
         }
-      }
-      PrevPageBoundary = PageBoundary;
+        PrevPageBoundary = PageBoundary;
 
-      RangeTracker.processNextPage(Counters.get(I) == BlocksPerPage);
+        RangeTracker.processNextPage(Counters.get(I, J) == BlocksPerPage);
+      }
     }
   }
   RangeTracker.finish();
diff --git a/compiler-rt/lib/scudo/standalone/tests/release_test.cpp b/compiler-rt/lib/scudo/standalone/tests/release_test.cpp
index a7478f47479d5..a693b97f80da6 100644
--- a/compiler-rt/lib/scudo/standalone/tests/release_test.cpp
+++ b/compiler-rt/lib/scudo/standalone/tests/release_test.cpp
@@ -21,14 +21,14 @@
 TEST(ScudoReleaseTest, PackedCounterArray) {
   for (scudo::uptr I = 0; I < SCUDO_WORDSIZE; I++) {
     // Various valid counter's max values packed into one word.
-    scudo::PackedCounterArray Counters2N(1, 1UL << I);
+    scudo::PackedCounterArray Counters2N(1U, 1U, 1UL << I);
     EXPECT_EQ(sizeof(scudo::uptr), Counters2N.getBufferSize());
     // Check the "all bit set" values too.
-    scudo::PackedCounterArray Counters2N1_1(1, ~0UL >> I);
+    scudo::PackedCounterArray Counters2N1_1(1U, 1U, ~0UL >> I);
     EXPECT_EQ(sizeof(scudo::uptr), Counters2N1_1.getBufferSize());
     // Verify the packing ratio, the counter is Expected to be packed into the
     // closest power of 2 bits.
-    scudo::PackedCounterArray Counters(SCUDO_WORDSIZE, 1UL << I);
+    scudo::PackedCounterArray Counters(1U, SCUDO_WORDSIZE, 1UL << I);
     EXPECT_EQ(sizeof(scudo::uptr) * scudo::roundUpToPowerOfTwo(I + 1),
               Counters.getBufferSize());
   }
@@ -38,19 +38,20 @@ TEST(ScudoReleaseTest, PackedCounterArray) {
     // Make sure counters request one memory page for the buffer.
     const scudo::uptr NumCounters =
         (scudo::getPageSizeCached() / 8) * (SCUDO_WORDSIZE >> I);
-    scudo::PackedCounterArray Counters(NumCounters, 1UL << ((1UL << I) - 1));
-    Counters.inc(0);
+    scudo::PackedCounterArray Counters(1U, NumCounters,
+                                       1UL << ((1UL << I) - 1));
+    Counters.inc(0U, 0U);
     for (scudo::uptr C = 1; C < NumCounters - 1; C++) {
-      EXPECT_EQ(0UL, Counters.get(C));
-      Counters.inc(C);
-      EXPECT_EQ(1UL, Counters.get(C - 1));
+      EXPECT_EQ(0UL, Counters.get(0U, C));
+      Counters.inc(0U, C);
+      EXPECT_EQ(1UL, Counters.get(0U, C - 1));
     }
-    EXPECT_EQ(0UL, Counters.get(NumCounters - 1));
-    Counters.inc(NumCounters - 1);
+    EXPECT_EQ(0UL, Counters.get(0U, NumCounters - 1));
+    Counters.inc(0U, NumCounters - 1);
     if (I > 0) {
-      Counters.incRange(0, NumCounters - 1);
+      Counters.incRange(0u, 0U, NumCounters - 1);
       for (scudo::uptr C = 0; C < NumCounters; C++)
-        EXPECT_EQ(2UL, Counters.get(C));
+        EXPECT_EQ(2UL, Counters.get(0U, C));
     }
   }
 }
@@ -190,7 +191,7 @@ template <class SizeClassMap> void testReleaseFreeMemoryToOS() {
 
     // Release the memory.
     ReleasedPagesRecorder Recorder;
-    releaseFreeMemoryToOS(FreeList, 0, MaxBlocks * BlockSize, BlockSize,
+    releaseFreeMemoryToOS(FreeList, 0, MaxBlocks * BlockSize, 1U, BlockSize,
                           &Recorder);
 
     // Verify that there are no released pages touched by used chunks and all

From db37937a4738f3a697804d8b2244baec076867a2 Mon Sep 17 00:00:00 2001
From: Meera Nakrani <meera.nakrani@arm.com>
Date: Fri, 24 Jul 2020 17:46:25 +0000
Subject: [PATCH 0026/1035] [ARM] Added additional patterns to VABD instruction

Added extra patterns to VABD instruction so it is selected in place of VSUB and VABS. Added corresponding regression test too.

Differential Revision: https://reviews.llvm.org/D84500
---
 llvm/lib/Target/ARM/ARMInstrMVE.td   |  7 ++++
 llvm/test/CodeGen/Thumb2/mve-vabd.ll | 63 ++++++++++++++++++++++++++++
 2 files changed, 70 insertions(+)
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-vabd.ll

diff --git a/llvm/lib/Target/ARM/ARMInstrMVE.td b/llvm/lib/Target/ARM/ARMInstrMVE.td
index fa04a82b0ee26..b082ca4d1c903 100644
--- a/llvm/lib/Target/ARM/ARMInstrMVE.td
+++ b/llvm/lib/Target/ARM/ARMInstrMVE.td
@@ -3775,6 +3775,13 @@ multiclass MVE_VABD_fp_m<MVEVectorVTInfo VTI>
 defm MVE_VABDf32 : MVE_VABD_fp_m<MVE_v4f32>;
 defm MVE_VABDf16 : MVE_VABD_fp_m<MVE_v8f16>; 
 
+let Predicates = [HasMVEFloat] in {
+  def : Pat<(v8f16 (fabs (fsub (v8f16 MQPR:$Qm), (v8f16 MQPR:$Qn)))),
+        (MVE_VABDf16 MQPR:$Qm, MQPR:$Qn)>;
+  def : Pat<(v4f32 (fabs (fsub (v4f32 MQPR:$Qm), (v4f32 MQPR:$Qn)))),
+        (MVE_VABDf32 MQPR:$Qm, MQPR:$Qn)>;
+}
+
 class MVE_VCVT_fix<string suffix, bit fsi, bit U, bit op,
                    Operand imm_operand_type>
   : MVE_float<"vcvt", suffix,
diff --git a/llvm/test/CodeGen/Thumb2/mve-vabd.ll b/llvm/test/CodeGen/Thumb2/mve-vabd.ll
new file mode 100644
index 0000000000000..3bbf2fc23a1cf
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-vabd.ll
@@ -0,0 +1,63 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve %s -o - | FileCheck %s  --check-prefix=CHECK --check-prefix=CHECK-MVE
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve.fp %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-MVEFP
+
+define arm_aapcs_vfpcc void @vabd_v4f32(<4 x float> %x, <4 x float> %y, <4 x float>* %z) {
+; CHECK-MVE-LABEL: vabd_v4f32
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:  .save {r4, r5, r6, r7, lr}
+; CHECK-MVE-NEXT:  push {r4, r5, r6, r7, lr}
+; CHECK-MVE-NEXT:  .pad #4
+; CHECK-MVE-NEXT:  sub sp, #4
+; CHECK-MVE-NEXT:  .vsave {d8, d9, d10, d11}
+; CHECK-MVE-NEXT:  vpush {d8, d9, d10, d11}
+; CHECK-MVE-NEXT:  vmov q4, q1
+; CHECK-MVE-NEXT:  vmov q5, q0
+; CHECK-MVE-NEXT:  mov r4, r0
+; CHECK-MVE-NEXT:  vmov r0, s20
+; CHECK-MVE-NEXT:  vmov r1, s16
+; CHECK-MVE-NEXT:  bl __aeabi_fsub
+; CHECK-MVE-NEXT:  mov r5, r0
+; CHECK-MVE-NEXT:  vmov r0, s21
+
+; CHECK-MVEFP-LABEL: vabd_v4f32
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:  vabd.f32 q0, q0, q1
+; CHECK-MVEFP-NEXT:  vstrw.32 q0, [r0]
+; CHECK-MVEFP-NEXT:  bx lr
+entry:
+  %0 = fsub <4 x float> %x, %y
+  %1 = call <4 x float> @llvm.fabs.v4f32(<4 x float> %0)
+  store <4 x float> %1, <4 x float>* %z, align 4
+  ret void
+}
+
+define arm_aapcs_vfpcc void @vabd_v8f16(<8 x half> %x, <8 x half> %y, <8 x half>* %z) {
+; CHECK-MVE-LABEL: vabd_v8f16
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:  .save {r4, r5, r6, lr}
+; CHECK-MVE-NEXT:  push {r4, r5, r6, lr}
+; CHECK-MVE-NEXT:  .vsave {d8, d9, d10, d11, d12, d13}
+; CHECK-MVE-NEXT:  vpush {d8, d9, d10, d11, d12, d13}
+; CHECK-MVE-NEXT:  mov r4, r0
+; CHECK-MVE-NEXT:  vmov.u16 r0, q1[1]
+; CHECK-MVE-NEXT:  vmov q5, q1
+; CHECK-MVE-NEXT:  vmov q4, q0
+; CHECK-MVE-NEXT:   bl __aeabi_h2f
+; CHECK-MVE-NEXT:  mov r5, r0
+; CHECK-MVE-NEXT:  vmov.u16 r0, q4[1]
+
+; CHECK-MVEFP-LABEL: vabd_v8f16
+; CHECK-MVEFP:       @ %bb.0: @ %entry
+; CHECK-MVEFP-NEXT:  vabd.f16 q0, q0, q1
+; CHECK-MVEFP-NEXT:  vstrw.32 q0, [r0]
+; CHECK-MVEFP-NEXT:  bx lr
+entry:
+  %0 = fsub <8 x half> %x, %y
+  %1 = call <8 x half> @llvm.fabs.v8f16(<8 x half> %0)
+  store <8 x half> %1, <8 x half>* %z
+  ret void
+}
+
+declare <4 x float> @llvm.fabs.v4f32(<4 x float>)
+declare <8 x half> @llvm.fabs.v8f16(<8 x half>)

From 8158f0cefe72ea7efbf41fdf4a7bfa392f6a8a2e Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@intel.com>
Date: Fri, 24 Jul 2020 10:47:46 -0700
Subject: [PATCH 0027/1035] [X86] Use X86_MC::ParseX86Triple to add mode
 features to feature string in X86Subtarget::initSubtargetFeatures.

Remove mode flags from constructor and remove calls to
ToggleFeature for the mode bits.

By adding them to the feature string we handle initializing the
mode member variables in X86Subtarget and the feature bits in
MCSubtargetInfo in one shot.
---
 llvm/lib/Target/X86/X86Subtarget.cpp | 35 ++++++++--------------------
 llvm/lib/Target/X86/X86Subtarget.h   |  6 ++---
 2 files changed, 13 insertions(+), 28 deletions(-)

diff --git a/llvm/lib/Target/X86/X86Subtarget.cpp b/llvm/lib/Target/X86/X86Subtarget.cpp
index ff2a94bcac8e2..51665255ec06b 100644
--- a/llvm/lib/Target/X86/X86Subtarget.cpp
+++ b/llvm/lib/Target/X86/X86Subtarget.cpp
@@ -231,15 +231,16 @@ void X86Subtarget::initSubtargetFeatures(StringRef CPU, StringRef FS) {
   if (CPU.empty())
     CPU = "generic";
 
-  std::string FullFS = std::string(FS);
-  if (In64BitMode) {
-    // SSE2 should default to enabled in 64-bit mode, but can be turned off
-    // explicitly.
-    if (!FullFS.empty())
-      FullFS = "+sse2," + FullFS;
-    else
-      FullFS = "+sse2";
-  }
+  std::string FullFS = X86_MC::ParseX86Triple(TargetTriple);
+  assert(!FullFS.empty() && "Failed to parse X86 triple");
+
+  // SSE2 should default to enabled in 64-bit mode, but can be turned off
+  // explicitly.
+  if (TargetTriple.isArch64Bit())
+    FullFS += ",+sse2";
+
+  if (!FS.empty())
+    FullFS = (Twine(FullFS) + "," + FS).str();
 
   // Parse features string and set the CPU.
   ParseSubtargetFeatures(CPU, FullFS);
@@ -251,17 +252,6 @@ void X86Subtarget::initSubtargetFeatures(StringRef CPU, StringRef FS) {
   if (hasSSE42() || hasSSE4A())
     IsUAMem16Slow = false;
 
-  // It's important to keep the MCSubtargetInfo feature bits in sync with
-  // target data structure which is shared with MC code emitter, etc.
-  if (In64BitMode)
-    ToggleFeature(X86::Mode64Bit);
-  else if (In32BitMode)
-    ToggleFeature(X86::Mode32Bit);
-  else if (In16BitMode)
-    ToggleFeature(X86::Mode16Bit);
-  else
-    llvm_unreachable("Not 16-bit, 32-bit or 64-bit mode!");
-
   LLVM_DEBUG(dbgs() << "Subtarget features: SSELevel " << X86SSELevel
                     << ", 3DNowLevel " << X863DNowLevel << ", 64bit "
                     << HasX86_64 << "\n");
@@ -312,11 +302,6 @@ X86Subtarget::X86Subtarget(const Triple &TT, StringRef CPU, StringRef FS,
       TM(TM), TargetTriple(TT), StackAlignOverride(StackAlignOverride),
       PreferVectorWidthOverride(PreferVectorWidthOverride),
       RequiredVectorWidth(RequiredVectorWidth),
-      In64BitMode(TargetTriple.getArch() == Triple::x86_64),
-      In32BitMode(TargetTriple.getArch() == Triple::x86 &&
-                  TargetTriple.getEnvironment() != Triple::CODE16),
-      In16BitMode(TargetTriple.getArch() == Triple::x86 &&
-                  TargetTriple.getEnvironment() == Triple::CODE16),
       InstrInfo(initializeSubtargetDependencies(CPU, FS)), TLInfo(TM, *this),
       FrameLowering(*this, getStackAlignment()) {
   // Determine the PICStyle based on the target selected.
diff --git a/llvm/lib/Target/X86/X86Subtarget.h b/llvm/lib/Target/X86/X86Subtarget.h
index 6f6823dea8ffd..54d7fbef7f94a 100644
--- a/llvm/lib/Target/X86/X86Subtarget.h
+++ b/llvm/lib/Target/X86/X86Subtarget.h
@@ -498,13 +498,13 @@ class X86Subtarget final : public X86GenSubtargetInfo {
   unsigned RequiredVectorWidth;
 
   /// True if compiling for 64-bit, false for 16-bit or 32-bit.
-  bool In64BitMode;
+  bool In64BitMode = false;
 
   /// True if compiling for 32-bit, false for 16-bit or 64-bit.
-  bool In32BitMode;
+  bool In32BitMode = false;
 
   /// True if compiling for 16-bit, false for 32-bit or 64-bit.
-  bool In16BitMode;
+  bool In16BitMode = false;
 
   /// Contains the Overhead of gather\scatter instructions
   int GatherOverhead = 1024;

From 809600d6642773f71245f76995dab355effc73af Mon Sep 17 00:00:00 2001
From: Francesco Petrogalli <francesco.petrogalli@arm.com>
Date: Tue, 7 Jul 2020 19:03:13 +0000
Subject: [PATCH 0028/1035] [llvm][sve] Reg + Imm addressing mode for ld1ro.

Reviewers: kmclaughlin, efriedma, sdesmalen

Subscribers: tschuett, hiraditya, psnobl, llvm-commits

Tags: #llvm

Differential Revision: https://reviews.llvm.org/D83357
---
 .../Target/AArch64/AArch64ISelLowering.cpp    |   3 +
 .../lib/Target/AArch64/AArch64InstrFormats.td |   5 +-
 llvm/lib/Target/AArch64/SVEInstrFormats.td    |  10 +-
 ...ntrinsics-ld1ro-addressing-mode-reg-imm.ll | 174 ++++++++++++++++++
 4 files changed, 188 insertions(+), 4 deletions(-)
 create mode 100644 llvm/test/CodeGen/AArch64/sve-intrinsics-ld1ro-addressing-mode-reg-imm.ll

diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index af77171e92849..cb5530077fdd7 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -12347,6 +12347,9 @@ static SDValue performLD1ReplicateCombine(SDNode *N, SelectionDAG &DAG) {
                 "Unsupported opcode.");
   SDLoc DL(N);
   EVT VT = N->getValueType(0);
+  if (VT == MVT::nxv8bf16 &&
+      !static_cast<const AArch64Subtarget &>(DAG.getSubtarget()).hasBF16())
+    return SDValue();
 
   EVT LoadVT = VT;
   if (VT.isFloatingPoint())
diff --git a/llvm/lib/Target/AArch64/AArch64InstrFormats.td b/llvm/lib/Target/AArch64/AArch64InstrFormats.td
index 6df7970f4d82b..4f4ba692c2db4 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrFormats.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrFormats.td
@@ -495,6 +495,9 @@ def SImmS4XForm : SDNodeXForm<imm, [{
 def SImmS16XForm : SDNodeXForm<imm, [{
   return CurDAG->getTargetConstant(N->getSExtValue() / 16, SDLoc(N), MVT::i64);
 }]>;
+def SImmS32XForm : SDNodeXForm<imm, [{
+  return CurDAG->getTargetConstant(N->getSExtValue() / 32, SDLoc(N), MVT::i64);
+}]>;
 
 // simm6sN predicate - True if the immediate is a multiple of N in the range
 // [-32 * N, 31 * N].
@@ -546,7 +549,7 @@ def simm4s16 : Operand<i64>, ImmLeaf<i64,
   let DecoderMethod = "DecodeSImm<4>";
 }
 def simm4s32 : Operand<i64>, ImmLeaf<i64,
-[{ return Imm >=-256  && Imm <= 224 && (Imm % 32) == 0x0; }]> {
+[{ return Imm >=-256  && Imm <= 224 && (Imm % 32) == 0x0; }], SImmS32XForm> {
   let PrintMethod = "printImmScale<32>";
   let ParserMatchClass = SImm4s32Operand;
   let DecoderMethod = "DecodeSImm<4>";
diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td
index ee36ac0168003..1f067908de6b4 100644
--- a/llvm/lib/Target/AArch64/SVEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td
@@ -7728,9 +7728,13 @@ multiclass sve_mem_ldor_si<bits<2> sz, string asm, RegisterOperand listty,
                   (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s32:$imm4), 0>;
 
   // Base addressing mode
-  def : Pat<(Ty (Ld1ro (PredTy PPR3bAny:$gp), GPR64sp:$base)),
-            (!cast<Instruction>(NAME) PPR3bAny:$gp, GPR64sp:$base, (i64 0))>;
-
+  def : Pat<(Ty (Ld1ro (PredTy PPR3bAny:$Pg), GPR64sp:$base)),
+            (!cast<Instruction>(NAME) PPR3bAny:$Pg, GPR64sp:$base, (i64 0))>;
+  let AddedComplexity = 2 in {
+    // Reg + Imm addressing mode
+    def : Pat<(Ty (Ld1ro (PredTy PPR3bAny:$Pg), (add GPR64:$base, (i64 simm4s32:$imm)))),
+              (!cast<Instruction>(NAME) $Pg, $base, simm4s32:$imm)>;
+  }
 }
 
 class sve_mem_ldor_ss<bits<2> sz, string asm, RegisterOperand VecList,
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-ld1ro-addressing-mode-reg-imm.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-ld1ro-addressing-mode-reg-imm.ll
new file mode 100644
index 0000000000000..e7edfc9d6bdd6
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-ld1ro-addressing-mode-reg-imm.ll
@@ -0,0 +1,174 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+f64mm -asm-verbose=0 < %s 2>%t | FileCheck %s
+; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t
+
+; WARN-NOT: warning
+
+;
+; LD1ROB
+;
+
+define <vscale x 16 x i8> @ld1rob_i8(<vscale x 16 x i1> %pg, i8* %a) nounwind {
+; CHECK-LABEL: ld1rob_i8:
+; CHECK-NEXT:  ld1rob { z0.b }, p0/z, [x0, #32]
+; CHECK-NEXT:  ret
+  %base = getelementptr i8, i8* %a, i64 32
+  %load = call <vscale x 16 x i8> @llvm.aarch64.sve.ld1ro.nxv16i8(<vscale x 16 x i1> %pg, i8* %base)
+  ret <vscale x 16 x i8> %load
+}
+
+;
+; LD1ROH
+;
+
+define <vscale x 8 x i16> @ld1roh_i16(<vscale x 8 x i1> %pg, i16* %a) nounwind {
+; CHECK-LABEL: ld1roh_i16:
+; CHECK-NEXT:  ld1roh { z0.h }, p0/z, [x0, #64]
+; CHECK-NEXT:  ret
+  %base = getelementptr i16, i16* %a, i64 32
+  %load = call <vscale x 8 x i16> @llvm.aarch64.sve.ld1ro.nxv8i16(<vscale x 8 x i1> %pg, i16* %base)
+  ret <vscale x 8 x i16> %load
+}
+
+define <vscale x 8 x half> @ld1roh_f16(<vscale x 8 x i1> %pg, half* %a) nounwind {
+; CHECK-LABEL: ld1roh_f16:
+; CHECK-NEXT:  ld1roh { z0.h }, p0/z, [x0, #64]
+; CHECK-NEXT:  ret
+  %base = getelementptr half, half* %a, i64 32
+  %load = call <vscale x 8 x half> @llvm.aarch64.sve.ld1ro.nxv8f16(<vscale x 8 x i1> %pg, half* %base)
+  ret <vscale x 8 x half> %load
+}
+
+define <vscale x 8 x bfloat> @ld1roh_bf16(<vscale x 8 x i1> %pg, bfloat* %a) nounwind #0 {
+; CHECK-LABEL: ld1roh_bf16:
+; CHECK-NEXT:  ld1roh { z0.h }, p0/z, [x0, #64]
+; CHECK-NEXT:  ret
+  %base = getelementptr bfloat, bfloat* %a, i64 32
+  %load = call <vscale x 8 x bfloat> @llvm.aarch64.sve.ld1ro.nxv8bf16(<vscale x 8 x i1> %pg, bfloat* %base)
+  ret <vscale x 8 x bfloat> %load
+}
+
+;
+; LD1ROW
+;
+
+define<vscale x 4 x i32> @ld1row_i32(<vscale x 4 x i1> %pg, i32* %a) nounwind {
+; CHECK-LABEL: ld1row_i32:
+; CHECK-NEXT:  ld1row { z0.s }, p0/z, [x0, #128]
+; CHECK-NEXT:  ret
+  %base = getelementptr i32, i32* %a, i64 32
+  %load = call <vscale x 4 x i32> @llvm.aarch64.sve.ld1ro.nxv4i32(<vscale x 4 x i1> %pg, i32* %base)
+  ret <vscale x 4 x i32> %load
+}
+
+define<vscale x 4 x float> @ld1row_f32(<vscale x 4 x i1> %pg, float* %a) nounwind {
+; CHECK-LABEL: ld1row_f32:
+; CHECK-NEXT:  ld1row { z0.s }, p0/z, [x0, #128]
+; CHECK-NEXT:  ret
+  %base = getelementptr float, float* %a, i64 32
+  %load = call <vscale x 4 x float> @llvm.aarch64.sve.ld1ro.nxv4f32(<vscale x 4 x i1> %pg, float* %base)
+  ret <vscale x 4 x float> %load
+}
+
+;
+; LD1ROD
+;
+
+define <vscale x 2 x i64> @ld1rod_i64(<vscale x 2 x i1> %pg, i64* %a) nounwind {
+; CHECK-LABEL: ld1rod_i64:
+; CHECK-NEXT:  ld1rod { z0.d }, p0/z, [x0, #-64]
+; CHECK-NEXT:  ret
+  %base = getelementptr i64, i64* %a, i64 -8
+  %load = call <vscale x 2 x i64> @llvm.aarch64.sve.ld1ro.nxv2i64(<vscale x 2 x i1> %pg, i64* %base)
+  ret <vscale x 2 x i64> %load
+}
+
+define <vscale x 2 x double> @ld1rod_f64(<vscale x 2 x i1> %pg, double* %a) nounwind {
+; CHECK-LABEL: ld1rod_f64:
+; CHECK-NEXT:  ld1rod { z0.d }, p0/z, [x0, #-128]
+; CHECK-NEXT:  ret
+  %base = getelementptr double, double* %a, i64 -16
+  %load = call <vscale x 2 x double> @llvm.aarch64.sve.ld1ro.nxv2f64(<vscale x 2 x i1> %pg, double* %base)
+  ret <vscale x 2 x double> %load
+}
+
+
+;;;;;;;;;;;;;;
+; range checks: immediate must be a multiple of 32 in the range -256, ..., 224
+
+; lower bound
+define <vscale x 16 x i8> @ld1rob_i8_lower_bound(<vscale x 16 x i1> %pg, i8* %a) nounwind {
+; CHECK-LABEL: ld1rob_i8_lower_bound:
+; CHECK-NEXT:  ld1rob { z0.b }, p0/z, [x0, #-256]
+; CHECK-NEXT:  ret
+  %base = getelementptr i8, i8* %a, i64 -256
+  %load = call <vscale x 16 x i8> @llvm.aarch64.sve.ld1ro.nxv16i8(<vscale x 16 x i1> %pg, i8* %base)
+  ret <vscale x 16 x i8> %load
+}
+
+; below lower bound
+define <vscale x 8 x i16> @ld1roh_i16_below_lower_bound(<vscale x 8 x i1> %pg, i16* %a) nounwind {
+; CHECK-LABEL: ld1roh_i16_below_lower_bound:
+; CHECK-NEXT:  sub     x[[BASE:[0-9]+]], x0, #258
+; CHECK-NEXT:  ld1roh { z0.h }, p0/z, [x[[BASE]]]
+; CHECK-NEXT:  ret
+  %base = getelementptr i16, i16* %a, i64 -129
+  %load = call <vscale x 8 x i16> @llvm.aarch64.sve.ld1ro.nxv8i16(<vscale x 8 x i1> %pg, i16* %base)
+  ret <vscale x 8 x i16> %load
+}
+
+define <vscale x 16 x i8> @ld1rob_i8_below_lower_bound_01(<vscale x 16 x i1> %pg, i8* %a) nounwind {
+; CHECK-LABEL: ld1rob_i8_below_lower_bound_01:
+; CHECK-NEXT:  mov x[[OFFSET:[0-9]+]], #-257
+; CHECK-NEXT:  ld1rob { z0.b }, p0/z, [x0, x[[OFFSET]]]
+; CHECK-NEXT:  ret
+  %base = getelementptr i8, i8* %a, i64 -257
+  %load = call <vscale x 16 x i8> @llvm.aarch64.sve.ld1ro.nxv16i8(<vscale x 16 x i1> %pg, i8* %base)
+  ret <vscale x 16 x i8> %load
+}
+
+; not a multiple of 32
+define<vscale x 4 x i32> @ld1row_i32_not_multiple(<vscale x 4 x i1> %pg, i32* %a) nounwind {
+; CHECK-LABEL: ld1row_i32_not_multiple:
+; CHECK-NEXT:  add x[[BASE:[0-9]+]], x0, #12
+; CHECK-NEXT:  ld1row { z0.s }, p0/z, [x[[BASE]]]
+; CHECK-NEXT:  ret
+  %base = getelementptr i32, i32* %a, i64 3
+  %load = call <vscale x 4 x i32> @llvm.aarch64.sve.ld1ro.nxv4i32(<vscale x 4 x i1> %pg, i32* %base)
+  ret <vscale x 4 x i32> %load
+}
+
+; upper bound
+define <vscale x 2 x i64> @ld1rod_i64_upper_bound(<vscale x 2 x i1> %pg, i64* %a) nounwind {
+; CHECK-LABEL: ld1rod_i64_upper_bound:
+; CHECK-NEXT:  ld1rod { z0.d }, p0/z, [x0, #224]
+; CHECK-NEXT:  ret
+  %base = getelementptr i64, i64* %a, i64 28
+  %load = call <vscale x 2 x i64> @llvm.aarch64.sve.ld1ro.nxv2i64(<vscale x 2 x i1> %pg, i64* %base)
+  ret <vscale x 2 x i64> %load
+}
+
+define <vscale x 16 x i8> @ld1rob_i8_beyond_upper_bound(<vscale x 16 x i1> %pg, i8* %a) nounwind {
+; CHECK-LABEL: ld1rob_i8_beyond_upper_bound:
+; CHECK-NEXT:  mov w[[OFFSET:[0-9]+]], #225
+; CHECK-NEXT:  ld1rob { z0.b }, p0/z, [x0, x[[OFFSET]]]
+; CHECK-NEXT:  ret
+  %base = getelementptr i8, i8* %a, i64 225
+  %load = call <vscale x 16 x i8> @llvm.aarch64.sve.ld1ro.nxv16i8(<vscale x 16 x i1> %pg, i8* %base)
+  ret <vscale x 16 x i8> %load
+}
+
+declare <vscale x 16 x i8> @llvm.aarch64.sve.ld1ro.nxv16i8(<vscale x 16 x i1>, i8*)
+
+declare <vscale x 8 x i16> @llvm.aarch64.sve.ld1ro.nxv8i16(<vscale x 8 x i1>, i16*)
+declare <vscale x 8 x half> @llvm.aarch64.sve.ld1ro.nxv8f16(<vscale x 8 x i1>, half*)
+declare <vscale x 8 x bfloat> @llvm.aarch64.sve.ld1ro.nxv8bf16(<vscale x 8 x i1>, bfloat*)
+
+declare <vscale x 4 x i32> @llvm.aarch64.sve.ld1ro.nxv4i32(<vscale x 4 x i1>, i32*)
+declare <vscale x 4 x float> @llvm.aarch64.sve.ld1ro.nxv4f32(<vscale x 4 x i1>, float*)
+
+declare <vscale x 2 x i64> @llvm.aarch64.sve.ld1ro.nxv2i64(<vscale x 2 x i1>, i64*)
+declare <vscale x 2 x double> @llvm.aarch64.sve.ld1ro.nxv2f64(<vscale x 2 x i1>, double*)
+
+
+; +bf16 is required for the bfloat version.
+attributes #0 = { "target-features"="+sve,+f64mm,+bf16" }

From 945ed22f3397f52469618cd8a94207665f25bebd Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@intel.com>
Date: Fri, 24 Jul 2020 11:10:28 -0700
Subject: [PATCH 0029/1035] [X86] Move the implicit enabling of sse2 for 64-bit
 mode from X86Subtarget::initSubtargetFeatures to X86_MC::ParseX86Triple.

ParseX86Triple already checks for 64-bit mode and produces a
static string. We can just add +sse2 to the end of that static
string. This avoids a potential reallocation when appending it
to the std::string at runtime.

This is a slight change to the behavior of tools that only use
MC layer which weren't implicitly enabling sse2 before, but will
now. I don't think we check for sse2 explicitly in any MC layer
components so this shouldn't matter in practice. And if it did
matter the new behavior is more correct.
---
 llvm/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.cpp | 6 ++++--
 llvm/lib/Target/X86/X86Subtarget.cpp                 | 5 -----
 2 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.cpp
index 8a478354cb168..8679bafa088e1 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.cpp
@@ -44,8 +44,10 @@ using namespace llvm;
 
 std::string X86_MC::ParseX86Triple(const Triple &TT) {
   std::string FS;
-  if (TT.getArch() == Triple::x86_64)
-    FS = "+64bit-mode,-32bit-mode,-16bit-mode";
+  // SSE2 should default to enabled in 64-bit mode, but can be turned off
+  // explicitly.
+  if (TT.isArch64Bit())
+    FS = "+64bit-mode,-32bit-mode,-16bit-mode,+sse2";
   else if (TT.getEnvironment() != Triple::CODE16)
     FS = "-64bit-mode,+32bit-mode,-16bit-mode";
   else
diff --git a/llvm/lib/Target/X86/X86Subtarget.cpp b/llvm/lib/Target/X86/X86Subtarget.cpp
index 51665255ec06b..07e913e139111 100644
--- a/llvm/lib/Target/X86/X86Subtarget.cpp
+++ b/llvm/lib/Target/X86/X86Subtarget.cpp
@@ -234,11 +234,6 @@ void X86Subtarget::initSubtargetFeatures(StringRef CPU, StringRef FS) {
   std::string FullFS = X86_MC::ParseX86Triple(TargetTriple);
   assert(!FullFS.empty() && "Failed to parse X86 triple");
 
-  // SSE2 should default to enabled in 64-bit mode, but can be turned off
-  // explicitly.
-  if (TargetTriple.isArch64Bit())
-    FullFS += ",+sse2";
-
   if (!FS.empty())
     FullFS = (Twine(FullFS) + "," + FS).str();
 

From 4a577c3a22c4ae388adca821a91552296e0d2653 Mon Sep 17 00:00:00 2001
From: madhur13490 <Madhur.Amilkanthwar@amd.com>
Date: Thu, 23 Jul 2020 09:48:03 +0000
Subject: [PATCH 0030/1035] [AMDGPU] Fix incorrect arch assert while setting up
 FlatScratchInit

Reviewers: arsenm, foad, rampitec, scott.linder

Reviewed By: arsenm

Subscribers: kzhuravl, jvesely, wdng, nhaehnle, yaxunl, dstuttard, tpr, t-tye, hiraditya, kerbowa, llvm-commits

Tags: #llvm

Differential Revision: https://reviews.llvm.org/D84391
---
 llvm/lib/Target/AMDGPU/SIFrameLowering.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp b/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
index a2e802009d098..a5b04570655a4 100644
--- a/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
@@ -274,6 +274,7 @@ void SIFrameLowering::emitEntryFunctionFlatScratchInit(
       return;
     }
 
+    // For GFX9.
     BuildMI(MBB, I, DL, TII->get(AMDGPU::S_ADD_U32), AMDGPU::FLAT_SCR_LO)
       .addReg(FlatScrInitLo)
       .addReg(ScratchWaveOffsetReg);
@@ -284,7 +285,7 @@ void SIFrameLowering::emitEntryFunctionFlatScratchInit(
     return;
   }
 
-  assert(ST.getGeneration() < AMDGPUSubtarget::GFX10);
+  assert(ST.getGeneration() < AMDGPUSubtarget::GFX9);
 
   // Copy the size in bytes.
   BuildMI(MBB, I, DL, TII->get(AMDGPU::COPY), AMDGPU::FLAT_SCR_LO)

From 5a9630b7774dbacb7a0bdba068c1b26c231558bc Mon Sep 17 00:00:00 2001
From: cgyurgyik <gyurgyikcp@gmail.com>
Date: Fri, 24 Jul 2020 14:31:27 -0400
Subject: [PATCH 0031/1035] [libc] Adds implementation for memrchr.

Reviewed By: sivachandra
Differential Revision: https://reviews.llvm.org/D84469
---
 libc/config/linux/aarch64/entrypoints.txt |   1 +
 libc/config/linux/api.td                  |   3 +-
 libc/config/linux/x86_64/entrypoints.txt  |   1 +
 libc/spec/gnu_ext.td                      |  16 ++-
 libc/src/string/CMakeLists.txt            |   8 ++
 libc/src/string/memrchr.cpp               |  26 +++++
 libc/src/string/memrchr.h                 |  20 ++++
 libc/test/src/string/CMakeLists.txt       |  10 ++
 libc/test/src/string/memrchr_test.cpp     | 114 ++++++++++++++++++++++
 9 files changed, 197 insertions(+), 2 deletions(-)
 create mode 100644 libc/src/string/memrchr.cpp
 create mode 100644 libc/src/string/memrchr.h
 create mode 100644 libc/test/src/string/memrchr_test.cpp

diff --git a/libc/config/linux/aarch64/entrypoints.txt b/libc/config/linux/aarch64/entrypoints.txt
index 6a1ff3bd64a9b..b287a72d779b9 100644
--- a/libc/config/linux/aarch64/entrypoints.txt
+++ b/libc/config/linux/aarch64/entrypoints.txt
@@ -13,6 +13,7 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.string.strchr
     libc.src.string.strstr
     libc.src.string.strnlen
+    libc.src.string.memrchr
 )
 
 set(TARGET_LIBM_ENTRYPOINTS
diff --git a/libc/config/linux/api.td b/libc/config/linux/api.td
index 1ec1a024f85d0..5f7a858d5fa31 100644
--- a/libc/config/linux/api.td
+++ b/libc/config/linux/api.td
@@ -216,7 +216,8 @@ def StringAPI : PublicAPI<"string.h"> {
     "strtok",
     "strerror",
     "strlen",
-    "strnlen"
+    "strnlen",
+    "memrchr"
   ];
 
   let TypeDeclarations = [
diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt
index b20f58c451847..db53005304896 100644
--- a/libc/config/linux/x86_64/entrypoints.txt
+++ b/libc/config/linux/x86_64/entrypoints.txt
@@ -31,6 +31,7 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.string.strchr
     libc.src.string.strstr
     libc.src.string.strnlen
+    libc.src.string.memrchr
 
     # sys/mman.h entrypoints
     libc.src.sys.mman.mmap
diff --git a/libc/spec/gnu_ext.td b/libc/spec/gnu_ext.td
index 7ac99783bc470..d85c562d9256a 100644
--- a/libc/spec/gnu_ext.td
+++ b/libc/spec/gnu_ext.td
@@ -12,8 +12,22 @@ def GnuExtensions : StandardSpec<"GNUExtensions"> {
         >,
       ]
   >;
+  
+  HeaderSpec String = HeaderSpec<
+      "string.h",
+      [], // Macros
+      [], // Types
+      [], // Enumerations
+      [ 
+        FunctionSpec<
+            "memrchr",
+            RetValSpec<VoidPtr>,
+            [ArgSpec<VoidPtr>, ArgSpec<IntType>, ArgSpec<SizeTType>]
+        >,
+      ]
+  >;
 
   let Headers = [
-    Math,
+    Math, String,
   ];
 }
diff --git a/libc/src/string/CMakeLists.txt b/libc/src/string/CMakeLists.txt
index 8bd7c1c045cf4..99450d5564593 100644
--- a/libc/src/string/CMakeLists.txt
+++ b/libc/src/string/CMakeLists.txt
@@ -78,6 +78,14 @@ add_entrypoint_object(
     .memchr
 )
 
+add_entrypoint_object(
+  memrchr
+  SRCS
+    memrchr.cpp
+  HDRS
+    memrchr.h
+)
+
 # Helper to define a function with multiple implementations
 # - Computes flags to satisfy required/rejected features and arch,
 # - Declares an entry point,
diff --git a/libc/src/string/memrchr.cpp b/libc/src/string/memrchr.cpp
new file mode 100644
index 0000000000000..81b034505202e
--- /dev/null
+++ b/libc/src/string/memrchr.cpp
@@ -0,0 +1,26 @@
+//===-- Implementation of memrchr -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/string/memrchr.h"
+#include "src/__support/common.h"
+#include <stddef.h>
+
+namespace __llvm_libc {
+
+void *LLVM_LIBC_ENTRYPOINT(memrchr)(const void *src, int c, size_t n) {
+  const unsigned char *str = reinterpret_cast<const unsigned char *>(src);
+  const unsigned char ch = c;
+  for (; n != 0; --n) {
+    const unsigned char *s = str + n - 1;
+    if (*s == ch)
+      return const_cast<unsigned char *>(s);
+  }
+  return nullptr;
+}
+
+} // namespace __llvm_libc
diff --git a/libc/src/string/memrchr.h b/libc/src/string/memrchr.h
new file mode 100644
index 0000000000000..8f43577e331fe
--- /dev/null
+++ b/libc/src/string/memrchr.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for memrchr -----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_STRING_MEMRCHR_H
+#define LLVM_LIBC_SRC_STRING_MEMRCHR_H
+
+#include <stddef.h>
+
+namespace __llvm_libc {
+
+void *memrchr(const void *src, int c, size_t n);
+
+} // namespace __llvm_libc
+
+#endif // LLVM_LIBC_SRC_STRING_MEMRCHR_H
diff --git a/libc/test/src/string/CMakeLists.txt b/libc/test/src/string/CMakeLists.txt
index be43cc912b5a3..a116effef2719 100644
--- a/libc/test/src/string/CMakeLists.txt
+++ b/libc/test/src/string/CMakeLists.txt
@@ -82,6 +82,16 @@ add_libc_unittest(
     libc.src.string.strnlen
 )
 
+add_libc_unittest(
+  memrchr_test
+  SUITE
+    libc_string_unittests
+  SRCS
+    memrchr_test.cpp
+  DEPENDS
+    libc.src.string.memrchr
+)
+
 # Tests all implementations that can run on the host.
 function(add_libc_multi_impl_test name)
   get_property(fq_implementations GLOBAL PROPERTY ${name}_implementations)
diff --git a/libc/test/src/string/memrchr_test.cpp b/libc/test/src/string/memrchr_test.cpp
new file mode 100644
index 0000000000000..5f5f7a0d01828
--- /dev/null
+++ b/libc/test/src/string/memrchr_test.cpp
@@ -0,0 +1,114 @@
+//===-- Unittests for memrchr ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/string/memrchr.h"
+#include "utils/UnitTest/Test.h"
+#include <stddef.h>
+
+// A helper function that calls memrchr and abstracts away the explicit cast for
+// readability purposes.
+const char *call_memrchr(const void *src, int c, size_t size) {
+  return reinterpret_cast<const char *>(__llvm_libc::memrchr(src, c, size));
+}
+
+TEST(MemRChrTest, FindsCharacterAfterNullTerminator) {
+  // memrchr should continue searching after a null terminator.
+  const size_t size = 6;
+  const unsigned char src[size] = {'a', '\0', 'b', 'c', 'd', '\0'};
+  // Should return 'b', 'c', 'd', '\0' even when after null terminator.
+  ASSERT_STREQ(call_memrchr(src, 'b', size), "bcd");
+}
+
+TEST(MemRChrTest, FindsCharacterInNonNullTerminatedCollection) {
+  const size_t size = 3;
+  const unsigned char src[size] = {'a', 'b', 'c'};
+  // Should return 'b', 'c'.
+  const char *ret = call_memrchr(src, 'b', size);
+  ASSERT_EQ(ret[0], 'b');
+  ASSERT_EQ(ret[1], 'c');
+}
+
+TEST(MemRChrTest, FindsFirstCharacter) {
+  const size_t size = 6;
+  const unsigned char src[size] = {'a', 'b', 'c', 'd', 'e', '\0'};
+  // Should return original array since 'a' is the first character.
+  ASSERT_STREQ(call_memrchr(src, 'a', size), "abcde");
+}
+
+TEST(MemRChrTest, FindsMiddleCharacter) {
+  const size_t size = 6;
+  const unsigned char src[size] = {'a', 'b', 'c', 'd', 'e', '\0'};
+  // Should return characters after (and including) 'c'.
+  ASSERT_STREQ(call_memrchr(src, 'c', size), "cde");
+}
+
+TEST(MemRChrTest, FindsLastCharacterThatIsNotNullTerminator) {
+  const size_t size = 6;
+  const unsigned char src[size] = {'a', 'b', 'c', 'd', 'e', '\0'};
+  // Should return 'e' and null-terminator.
+  ASSERT_STREQ(call_memrchr(src, 'e', size), "e");
+}
+
+TEST(MemRChrTest, FindsNullTerminator) {
+  const size_t size = 6;
+  const unsigned char src[size] = {'a', 'b', 'c', 'd', 'e', '\0'};
+  // Should return null terminator.
+  ASSERT_STREQ(call_memrchr(src, '\0', size), "");
+}
+
+TEST(MemRChrTest, CharacterNotWithinStringShouldReturnNullptr) {
+  const size_t size = 4;
+  const unsigned char src[size] = {'1', '2', '3', '?'};
+  // Since 'z' is not within 'characters', should return nullptr.
+  ASSERT_STREQ(call_memrchr(src, 'z', size), nullptr);
+}
+
+TEST(MemRChrTest, CharacterNotWithinSizeShouldReturnNullptr) {
+  const unsigned char src[5] = {'1', '2', '3', '4', '\0'};
+  // Since '4' is not within the first 2 characters, this should return nullptr.
+  const size_t size = 2;
+  ASSERT_STREQ(call_memrchr(src, '4', size), nullptr);
+}
+
+TEST(MemRChrTest, ShouldFindLastOfDuplicates) {
+  size_t size = 12; // 11 characters + null terminator.
+  const char *dups = "abc1def1ghi";
+  // 1 is duplicated in 'dups', but it should find the last copy.
+  ASSERT_STREQ(call_memrchr(dups, '1', size), "1ghi");
+
+  const char *repeated = "XXXXX";
+  size = 6; // 5 characters + null terminator.
+  // Should return the last X with the null terminator.
+  ASSERT_STREQ(call_memrchr(repeated, 'X', size), "X");
+}
+
+TEST(MemRChrTest, EmptyStringShouldOnlyMatchNullTerminator) {
+  const size_t size = 1; // Null terminator.
+  const char *empty_string = "";
+  // Null terminator should match.
+  ASSERT_STREQ(call_memrchr(empty_string, '\0', size), "");
+  // All other characters should not match.
+  ASSERT_STREQ(call_memrchr(empty_string, 'A', size), nullptr);
+  ASSERT_STREQ(call_memrchr(empty_string, '9', size), nullptr);
+  ASSERT_STREQ(call_memrchr(empty_string, '?', size), nullptr);
+}
+
+TEST(MemRChrTest, SignedCharacterFound) {
+  char c = -1;
+  const size_t size = 1;
+  char src[size] = {c};
+  const char *actual = call_memrchr(src, c, size);
+  // Should find the last character 'c'.
+  ASSERT_EQ(actual[0], c);
+}
+
+TEST(MemRChrTest, ZeroLengthShouldReturnNullptr) {
+  const unsigned char src[4] = {'a', 'b', 'c', '\0'};
+  // This will iterate over exactly zero characters, so should return nullptr.
+  ASSERT_STREQ(call_memrchr(src, 'd', 0), nullptr);
+}

From 43f09110402d382b9ce7b41d1a24b117d7941695 Mon Sep 17 00:00:00 2001
From: Valentin Clement <clementval@gmail.com>
Date: Fri, 24 Jul 2020 15:06:30 -0400
Subject: [PATCH 0032/1035] [openmp] Clean up OMPKinds.def remove OMP_DIRECTIVE

This patch removes the OMP_DIRECTIVE definition from OMPKinds.def since they
are now defined in OMP.td and OMP_DIRECTIVE is not used anymore in the code.

Reviewed By: jdenny

Differential Revision: https://reviews.llvm.org/D84329
---
 .../include/llvm/Frontend/OpenMP/OMPKinds.def | 94 +------------------
 1 file changed, 3 insertions(+), 91 deletions(-)

diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def b/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
index 4f2fcb8af5d1d..7771dcd72d6a9 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
@@ -10,100 +10,12 @@
 /// This file defines the list of supported OpenMP directives, clauses, runtime
 /// calls, and other things that need to be listed in enums.
 ///
+/// This file is under transition to OMP.td with TableGen code generation.
+///
 //===----------------------------------------------------------------------===//
 
 /// OpenMP Directives and combined directives
-///
-///{
-
-#ifndef OMP_DIRECTIVE
-#define OMP_DIRECTIVE(Enum, Str)
-#endif
-
-#define __OMP_DIRECTIVE_EXT(Name, Str) OMP_DIRECTIVE(OMPD_##Name, Str)
-#define __OMP_DIRECTIVE(Name) __OMP_DIRECTIVE_EXT(Name, #Name)
-
-__OMP_DIRECTIVE(threadprivate)
-__OMP_DIRECTIVE(parallel)
-__OMP_DIRECTIVE(task)
-__OMP_DIRECTIVE(simd)
-__OMP_DIRECTIVE(for)
-__OMP_DIRECTIVE(sections)
-__OMP_DIRECTIVE(section)
-__OMP_DIRECTIVE(single)
-__OMP_DIRECTIVE(master)
-__OMP_DIRECTIVE(critical)
-__OMP_DIRECTIVE(taskyield)
-__OMP_DIRECTIVE(barrier)
-__OMP_DIRECTIVE(taskwait)
-__OMP_DIRECTIVE(taskgroup)
-__OMP_DIRECTIVE(flush)
-__OMP_DIRECTIVE(ordered)
-__OMP_DIRECTIVE(atomic)
-__OMP_DIRECTIVE(target)
-__OMP_DIRECTIVE(teams)
-__OMP_DIRECTIVE(cancel)
-__OMP_DIRECTIVE(requires)
-__OMP_DIRECTIVE_EXT(target_data, "target data")
-__OMP_DIRECTIVE_EXT(target_enter_data, "target enter data")
-__OMP_DIRECTIVE_EXT(target_exit_data, "target exit data")
-__OMP_DIRECTIVE_EXT(target_parallel, "target parallel")
-__OMP_DIRECTIVE_EXT(target_parallel_for, "target parallel for")
-__OMP_DIRECTIVE_EXT(target_update, "target update")
-__OMP_DIRECTIVE_EXT(parallel_for, "parallel for")
-__OMP_DIRECTIVE_EXT(parallel_for_simd, "parallel for simd")
-__OMP_DIRECTIVE_EXT(parallel_master, "parallel master")
-__OMP_DIRECTIVE_EXT(parallel_sections, "parallel sections")
-__OMP_DIRECTIVE_EXT(for_simd, "for simd")
-__OMP_DIRECTIVE_EXT(cancellation_point, "cancellation point")
-__OMP_DIRECTIVE_EXT(declare_reduction, "declare reduction")
-__OMP_DIRECTIVE_EXT(declare_mapper, "declare mapper")
-__OMP_DIRECTIVE_EXT(declare_simd, "declare simd")
-__OMP_DIRECTIVE(taskloop)
-__OMP_DIRECTIVE_EXT(taskloop_simd, "taskloop simd")
-__OMP_DIRECTIVE(distribute)
-__OMP_DIRECTIVE_EXT(declare_target, "declare target")
-__OMP_DIRECTIVE_EXT(end_declare_target, "end declare target")
-__OMP_DIRECTIVE_EXT(distribute_parallel_for, "distribute parallel for")
-__OMP_DIRECTIVE_EXT(distribute_parallel_for_simd,
-                    "distribute parallel for simd")
-__OMP_DIRECTIVE_EXT(distribute_simd, "distribute simd")
-__OMP_DIRECTIVE_EXT(target_parallel_for_simd, "target parallel for simd")
-__OMP_DIRECTIVE_EXT(target_simd, "target simd")
-__OMP_DIRECTIVE_EXT(teams_distribute, "teams distribute")
-__OMP_DIRECTIVE_EXT(teams_distribute_simd, "teams distribute simd")
-__OMP_DIRECTIVE_EXT(teams_distribute_parallel_for_simd,
-                    "teams distribute parallel for simd")
-__OMP_DIRECTIVE_EXT(teams_distribute_parallel_for,
-                    "teams distribute parallel for")
-__OMP_DIRECTIVE_EXT(target_teams, "target teams")
-__OMP_DIRECTIVE_EXT(target_teams_distribute, "target teams distribute")
-__OMP_DIRECTIVE_EXT(target_teams_distribute_parallel_for,
-                    "target teams distribute parallel for")
-__OMP_DIRECTIVE_EXT(target_teams_distribute_parallel_for_simd,
-                    "target teams distribute parallel for simd")
-__OMP_DIRECTIVE_EXT(target_teams_distribute_simd,
-                    "target teams distribute simd")
-__OMP_DIRECTIVE(allocate)
-__OMP_DIRECTIVE_EXT(declare_variant, "declare variant")
-__OMP_DIRECTIVE_EXT(master_taskloop, "master taskloop")
-__OMP_DIRECTIVE_EXT(parallel_master_taskloop, "parallel master taskloop")
-__OMP_DIRECTIVE_EXT(master_taskloop_simd, "master taskloop simd")
-__OMP_DIRECTIVE_EXT(parallel_master_taskloop_simd,
-                    "parallel master taskloop simd")
-__OMP_DIRECTIVE(depobj)
-__OMP_DIRECTIVE(scan)
-__OMP_DIRECTIVE_EXT(begin_declare_variant, "begin declare variant")
-__OMP_DIRECTIVE_EXT(end_declare_variant, "end declare variant")
-
-// Has to be the last because Clang implicitly expects it to be.
-__OMP_DIRECTIVE(unknown)
-
-#undef __OMP_DIRECTIVE_EXT
-#undef __OMP_DIRECTIVE
-#undef OMP_DIRECTIVE
-
-///}
+/// - Moved to OMP.td
 
 /// OpenMP Clauses
 ///

From 0b339c069266b7a4f3c82f80067d74620cbe19c4 Mon Sep 17 00:00:00 2001
From: Jonas Devlieghere <jonas@devlieghere.com>
Date: Fri, 24 Jul 2020 12:09:36 -0700
Subject: [PATCH 0033/1035] [lldb] Inform every language runtime of the
 modified modules

When a process is notified that modules got loaded, currently only
existing language runtimes are given a chance to deal with that. This
means that if the runtime for a given language wasn't needed before it
won't be informed of the module chance.

This is wrong because the module change might be what triggers the need
for a certain runtime. Instead, we should give the language runtime for
every supported language a chance to deal with the modified modules.

Differential revision: https://reviews.llvm.org/D84475
---
 lldb/source/Target/Process.cpp | 40 ++++++++++------------------------
 1 file changed, 12 insertions(+), 28 deletions(-)

diff --git a/lldb/source/Target/Process.cpp b/lldb/source/Target/Process.cpp
index b7694ec43f341..3776a90e546ae 100644
--- a/lldb/source/Target/Process.cpp
+++ b/lldb/source/Target/Process.cpp
@@ -5757,41 +5757,25 @@ addr_t Process::ResolveIndirectFunction(const Address *address, Status &error) {
 }
 
 void Process::ModulesDidLoad(ModuleList &module_list) {
+  // Inform the system runtime of the modified modules.
   SystemRuntime *sys_runtime = GetSystemRuntime();
-  if (sys_runtime) {
+  if (sys_runtime)
     sys_runtime->ModulesDidLoad(module_list);
-  }
 
   GetJITLoaders().ModulesDidLoad(module_list);
 
-  // Give runtimes a chance to be created.
+  // Give the instrumentation runtimes a chance to be created before informing
+  // them of the modified modules.
   InstrumentationRuntime::ModulesDidLoad(module_list, this,
                                          m_instrumentation_runtimes);
+  for (auto &runtime : m_instrumentation_runtimes)
+    runtime.second->ModulesDidLoad(module_list);
 
-  // Tell runtimes about new modules.
-  for (auto pos = m_instrumentation_runtimes.begin();
-       pos != m_instrumentation_runtimes.end(); ++pos) {
-    InstrumentationRuntimeSP runtime = pos->second;
-    runtime->ModulesDidLoad(module_list);
-  }
-
-  // Let any language runtimes we have already created know about the modules
-  // that loaded.
-
-  // Iterate over a copy of this language runtime list in case the language
-  // runtime ModulesDidLoad somehow causes the language runtime to be
-  // unloaded.
-  {
-    std::lock_guard<std::recursive_mutex> guard(m_language_runtimes_mutex);
-    LanguageRuntimeCollection language_runtimes(m_language_runtimes);
-    for (const auto &pair : language_runtimes) {
-      // We must check language_runtime_sp to make sure it is not nullptr as we
-      // might cache the fact that we didn't have a language runtime for a
-      // language.
-      LanguageRuntimeSP language_runtime_sp = pair.second;
-      if (language_runtime_sp)
-        language_runtime_sp->ModulesDidLoad(module_list);
-    }
+  // Give the language runtimes a chance to be created before informing them of
+  // the modified modules.
+  for (const lldb::LanguageType lang_type : Language::GetSupportedLanguages()) {
+    if (LanguageRuntime *runtime = GetLanguageRuntime(lang_type))
+      runtime->ModulesDidLoad(module_list);
   }
 
   // If we don't have an operating system plug-in, try to load one since
@@ -5799,7 +5783,7 @@ void Process::ModulesDidLoad(ModuleList &module_list) {
   if (!m_os_up)
     LoadOperatingSystemPlugin(false);
 
-  // Give structured-data plugins a chance to see the modified modules.
+  // Inform the structured-data plugins of the modified modules.
   for (auto pair : m_structured_data_plugin_map) {
     if (pair.second)
       pair.second->ModulesDidLoad(*this, module_list);

From 9bb6ce78bec79295a5ad2f0ef96bfde4cb3f714c Mon Sep 17 00:00:00 2001
From: Arthur Eubanks <aeubanks@google.com>
Date: Fri, 24 Jul 2020 11:54:13 -0700
Subject: [PATCH 0034/1035] Rename scoped-noalias -> scoped-noalias-aa

Summary: To match NewPM name. Also the new name is clearer and more consistent.

Subscribers: jvesely, nhaehnle, hiraditya, asbirlea, kerbowa, llvm-commits

Tags: #llvm

Differential Revision: https://reviews.llvm.org/D84542
---
 llvm/lib/Analysis/ScopedNoAliasAA.cpp                          | 2 +-
 llvm/test/Analysis/ScopedNoAliasAA/basic-domains.ll            | 2 +-
 llvm/test/Analysis/ScopedNoAliasAA/basic.ll                    | 2 +-
 llvm/test/Analysis/ScopedNoAliasAA/basic2.ll                   | 2 +-
 llvm/test/Transforms/GVN/noalias.ll                            | 2 +-
 llvm/test/Transforms/LICM/dropped-tbaa.ll                      | 2 +-
 llvm/test/Transforms/LICM/pr42969.ll                           | 2 +-
 llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/aa-metadata.ll | 2 +-
 llvm/test/Transforms/LoopVectorize/noalias-md-licm.ll          | 2 +-
 llvm/test/Transforms/LoopVectorize/noalias-md.ll               | 2 +-
 llvm/test/Transforms/LoopVersioning/noalias-version-twice.ll   | 2 +-
 llvm/test/Transforms/MemCpyOpt/load-store-to-memcpy.ll         | 2 +-
 llvm/test/Transforms/NewGVN/noalias.ll                         | 2 +-
 13 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/llvm/lib/Analysis/ScopedNoAliasAA.cpp b/llvm/lib/Analysis/ScopedNoAliasAA.cpp
index 8928678d6ab21..5e2aaab050afe 100644
--- a/llvm/lib/Analysis/ScopedNoAliasAA.cpp
+++ b/llvm/lib/Analysis/ScopedNoAliasAA.cpp
@@ -185,7 +185,7 @@ ScopedNoAliasAAResult ScopedNoAliasAA::run(Function &F,
 
 char ScopedNoAliasAAWrapperPass::ID = 0;
 
-INITIALIZE_PASS(ScopedNoAliasAAWrapperPass, "scoped-noalias",
+INITIALIZE_PASS(ScopedNoAliasAAWrapperPass, "scoped-noalias-aa",
                 "Scoped NoAlias Alias Analysis", false, true)
 
 ImmutablePass *llvm::createScopedNoAliasAAWrapperPass() {
diff --git a/llvm/test/Analysis/ScopedNoAliasAA/basic-domains.ll b/llvm/test/Analysis/ScopedNoAliasAA/basic-domains.ll
index 5c66f03b87020..099a7aeb8a76a 100644
--- a/llvm/test/Analysis/ScopedNoAliasAA/basic-domains.ll
+++ b/llvm/test/Analysis/ScopedNoAliasAA/basic-domains.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basic-aa -scoped-noalias -aa-eval -evaluate-aa-metadata -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -basic-aa -scoped-noalias-aa -aa-eval -evaluate-aa-metadata -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
diff --git a/llvm/test/Analysis/ScopedNoAliasAA/basic.ll b/llvm/test/Analysis/ScopedNoAliasAA/basic.ll
index 9ca06a6826bdb..92cc0ccc32d3c 100644
--- a/llvm/test/Analysis/ScopedNoAliasAA/basic.ll
+++ b/llvm/test/Analysis/ScopedNoAliasAA/basic.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basic-aa -scoped-noalias -aa-eval -evaluate-aa-metadata -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -basic-aa -scoped-noalias-aa -aa-eval -evaluate-aa-metadata -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=basic-aa,scoped-noalias-aa -passes=aa-eval -evaluate-aa-metadata -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/Analysis/ScopedNoAliasAA/basic2.ll b/llvm/test/Analysis/ScopedNoAliasAA/basic2.ll
index e0c16b1991157..8275cc0439e00 100644
--- a/llvm/test/Analysis/ScopedNoAliasAA/basic2.ll
+++ b/llvm/test/Analysis/ScopedNoAliasAA/basic2.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basic-aa -scoped-noalias -aa-eval -evaluate-aa-metadata -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -basic-aa -scoped-noalias-aa -aa-eval -evaluate-aa-metadata -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
diff --git a/llvm/test/Transforms/GVN/noalias.ll b/llvm/test/Transforms/GVN/noalias.ll
index 69c21f110b5ea..9cfcc9bbd9d4c 100644
--- a/llvm/test/Transforms/GVN/noalias.ll
+++ b/llvm/test/Transforms/GVN/noalias.ll
@@ -1,4 +1,4 @@
-; RUN: opt -scoped-noalias -basic-aa -gvn -S < %s | FileCheck %s
+; RUN: opt -scoped-noalias-aa -basic-aa -gvn -S < %s | FileCheck %s
 
 define i32 @test1(i32* %p, i32* %q) {
 ; CHECK-LABEL: @test1(i32* %p, i32* %q)
diff --git a/llvm/test/Transforms/LICM/dropped-tbaa.ll b/llvm/test/Transforms/LICM/dropped-tbaa.ll
index 7d37ca55c1880..3297cc0b305a5 100644
--- a/llvm/test/Transforms/LICM/dropped-tbaa.ll
+++ b/llvm/test/Transforms/LICM/dropped-tbaa.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -scoped-noalias -tbaa -licm -S | FileCheck %s
+; RUN: opt < %s -scoped-noalias-aa -tbaa -licm -S | FileCheck %s
 
 ; This test case case is generated from the following C code with -fstrict-aliasing,
 ; and after passing through -inline -mem2reg -loop-rotate -instcombine
diff --git a/llvm/test/Transforms/LICM/pr42969.ll b/llvm/test/Transforms/LICM/pr42969.ll
index 7fa36f0cfe77d..3cb82dc8095a6 100644
--- a/llvm/test/Transforms/LICM/pr42969.ll
+++ b/llvm/test/Transforms/LICM/pr42969.ll
@@ -1,4 +1,4 @@
-; RUN: opt %s -S -scoped-noalias -enable-mssa-loop-dependency=true -licm | FileCheck %s
+; RUN: opt %s -S -scoped-noalias-aa -enable-mssa-loop-dependency=true -licm | FileCheck %s
 
 define i16 @main(i1 %a_b_mayalias, i16* %a, i16* %b) {
 ; CHECK:       scalar.body:
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/aa-metadata.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/aa-metadata.ll
index 8a0641ffe0d52..debf607e10759 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/aa-metadata.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/aa-metadata.ll
@@ -1,4 +1,4 @@
-; RUN: opt -mtriple=amdgcn-amd-amdhsa -basic-aa -scoped-noalias -load-store-vectorizer -S -o - %s | FileCheck -check-prefix=SCOPE -check-prefix=ALL %s
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -basic-aa -scoped-noalias-aa -load-store-vectorizer -S -o - %s | FileCheck -check-prefix=SCOPE -check-prefix=ALL %s
 ; RUN: opt -mtriple=amdgcn-amd-amdhsa -basic-aa -load-store-vectorizer -S -o - %s | FileCheck -check-prefix=NOSCOPE -check-prefix=ALL %s
 
 target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
diff --git a/llvm/test/Transforms/LoopVectorize/noalias-md-licm.ll b/llvm/test/Transforms/LoopVectorize/noalias-md-licm.ll
index d54a9aefe73a1..c343bb8dd6a32 100644
--- a/llvm/test/Transforms/LoopVectorize/noalias-md-licm.ll
+++ b/llvm/test/Transforms/LoopVectorize/noalias-md-licm.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basic-aa -scoped-noalias -loop-vectorize -licm -force-vector-width=2 \
+; RUN: opt -basic-aa -scoped-noalias-aa -loop-vectorize -licm -force-vector-width=2 \
 ; RUN:     -force-vector-interleave=1 -S < %s | FileCheck %s
 
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/LoopVectorize/noalias-md.ll b/llvm/test/Transforms/LoopVectorize/noalias-md.ll
index b141998163aef..27192de5576fc 100644
--- a/llvm/test/Transforms/LoopVectorize/noalias-md.ll
+++ b/llvm/test/Transforms/LoopVectorize/noalias-md.ll
@@ -1,7 +1,7 @@
 ; RUN: opt -basic-aa -loop-vectorize -force-vector-width=2 \
 ; RUN:     -force-vector-interleave=1 -S < %s \
 ; RUN:     | FileCheck %s -check-prefix=BOTH -check-prefix=LV
-; RUN: opt -basic-aa -scoped-noalias -loop-vectorize -dse -force-vector-width=2 \
+; RUN: opt -basic-aa -scoped-noalias-aa -loop-vectorize -dse -force-vector-width=2 \
 ; RUN:     -force-vector-interleave=1 -S < %s \
 ; RUN:     | FileCheck %s -check-prefix=BOTH -check-prefix=DSE
 
diff --git a/llvm/test/Transforms/LoopVersioning/noalias-version-twice.ll b/llvm/test/Transforms/LoopVersioning/noalias-version-twice.ll
index 89e898a90c178..4420b71d1f452 100644
--- a/llvm/test/Transforms/LoopVersioning/noalias-version-twice.ll
+++ b/llvm/test/Transforms/LoopVersioning/noalias-version-twice.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basic-aa -loop-distribute -enable-loop-distribute -loop-simplify -scoped-noalias \
+; RUN: opt -basic-aa -loop-distribute -enable-loop-distribute -loop-simplify -scoped-noalias-aa \
 ; RUN:     -loop-versioning -S < %s | FileCheck %s
 
 ; Test the metadata generated when versioning an already versioned loop.  Here
diff --git a/llvm/test/Transforms/MemCpyOpt/load-store-to-memcpy.ll b/llvm/test/Transforms/MemCpyOpt/load-store-to-memcpy.ll
index 81043d4f46ddf..c3f7a11272815 100644
--- a/llvm/test/Transforms/MemCpyOpt/load-store-to-memcpy.ll
+++ b/llvm/test/Transforms/MemCpyOpt/load-store-to-memcpy.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -basic-aa -scoped-noalias -memcpyopt -S %s | FileCheck %s
+; RUN: opt -basic-aa -scoped-noalias-aa -memcpyopt -S %s | FileCheck %s
 
 %T = type { i8, i32 }
 
diff --git a/llvm/test/Transforms/NewGVN/noalias.ll b/llvm/test/Transforms/NewGVN/noalias.ll
index c5f23bfad89a5..23fc95d4ff8dc 100644
--- a/llvm/test/Transforms/NewGVN/noalias.ll
+++ b/llvm/test/Transforms/NewGVN/noalias.ll
@@ -1,4 +1,4 @@
-; RUN: opt -scoped-noalias -basic-aa -newgvn -S < %s | FileCheck %s
+; RUN: opt -scoped-noalias-aa -basic-aa -newgvn -S < %s | FileCheck %s
 
 define i32 @test1(i32* %p, i32* %q) {
 ; CHECK-LABEL: @test1(i32* %p, i32* %q)

From 4d09ed953b5b8c70d9ca0aeaed8f26a237b612c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Martin=20Storsj=C3=B6?= <martin@martin.st>
Date: Fri, 24 Jul 2020 00:05:55 +0300
Subject: [PATCH 0035/1035] [llvm-lib] Support adding short import library
 objects with llvm-lib

This fixes PR 42837.

Differential Revision: https://reviews.llvm.org/D84465
---
 llvm/lib/ToolDrivers/llvm-lib/LibDriver.cpp |  6 ++++--
 llvm/test/tools/llvm-lib/implibs.test       | 12 ++++++++++++
 llvm/test/tools/llvm-lib/invalid.test       |  2 +-
 3 files changed, 17 insertions(+), 3 deletions(-)
 create mode 100644 llvm/test/tools/llvm-lib/implibs.test

diff --git a/llvm/lib/ToolDrivers/llvm-lib/LibDriver.cpp b/llvm/lib/ToolDrivers/llvm-lib/LibDriver.cpp
index c409012554245..cd39428b9c38e 100644
--- a/llvm/lib/ToolDrivers/llvm-lib/LibDriver.cpp
+++ b/llvm/lib/ToolDrivers/llvm-lib/LibDriver.cpp
@@ -191,9 +191,11 @@ static void appendFile(std::vector<NewArchiveMember> &Members,
   file_magic Magic = identify_magic(MB.getBuffer());
 
   if (Magic != file_magic::coff_object && Magic != file_magic::bitcode &&
-      Magic != file_magic::archive && Magic != file_magic::windows_resource) {
+      Magic != file_magic::archive && Magic != file_magic::windows_resource &&
+      Magic != file_magic::coff_import_library) {
     llvm::errs() << MB.getBufferIdentifier()
-                 << ": not a COFF object, bitcode, archive or resource file\n";
+                 << ": not a COFF object, bitcode, archive, import library or "
+                    "resource file\n";
     exit(1);
   }
 
diff --git a/llvm/test/tools/llvm-lib/implibs.test b/llvm/test/tools/llvm-lib/implibs.test
new file mode 100644
index 0000000000000..ebff4bb4608f2
--- /dev/null
+++ b/llvm/test/tools/llvm-lib/implibs.test
@@ -0,0 +1,12 @@
+Test that import libraries (and the members thereof) can be added to another
+static library.
+
+RUN: rm -rf %t
+RUN: mkdir -p %t
+
+RUN: echo -e "EXPORTS\nMyFunc" > %t/lib.def
+RUN: llvm-dlltool -m i386:x86-64 -l %t/lib.lib -d %t/lib.def -D lib.dll
+RUN: llvm-lib -out:%t/newlib.lib %t/lib.lib
+
+RUN: llvm-ar t %t/newlib.lib | FileCheck %s
+CHECK: lib.dll
diff --git a/llvm/test/tools/llvm-lib/invalid.test b/llvm/test/tools/llvm-lib/invalid.test
index 57266400cdc87..a4b06a03358b0 100644
--- a/llvm/test/tools/llvm-lib/invalid.test
+++ b/llvm/test/tools/llvm-lib/invalid.test
@@ -1,2 +1,2 @@
 RUN: not llvm-lib %S/Inputs/cl-gl.obj 2>&1 | FileCheck %s
-CHECK: not a COFF object, bitcode, archive or resource file
+CHECK: not a COFF object, bitcode, archive, import library or resource file

From 9e81d8bbf19d72fca3d87b7334c613d1aa2a5795 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Martin=20Storsj=C3=B6?= <martin@martin.st>
Date: Tue, 21 Jul 2020 23:39:37 +0300
Subject: [PATCH 0036/1035] [MC] [COFF] Make sure that weak external symbols
 are undefined symbols

For comdats (e.g. caused by -ffunction-sections), Section is already
set here; make sure it's null, for the weak external symbol to be undefined.

This fixes PR46779.

Differential Revision: https://reviews.llvm.org/D84507
---
 llvm/lib/MC/WinCOFFObjectWriter.cpp |  1 +
 llvm/test/MC/COFF/weak-comdat.s     | 34 +++++++++++++++++++++++++++++
 2 files changed, 35 insertions(+)
 create mode 100644 llvm/test/MC/COFF/weak-comdat.s

diff --git a/llvm/lib/MC/WinCOFFObjectWriter.cpp b/llvm/lib/MC/WinCOFFObjectWriter.cpp
index 4796ef531054b..8e7bf1eb01697 100644
--- a/llvm/lib/MC/WinCOFFObjectWriter.cpp
+++ b/llvm/lib/MC/WinCOFFObjectWriter.cpp
@@ -375,6 +375,7 @@ void WinCOFFObjectWriter::DefineSymbol(const MCSymbol &MCSym,
   COFFSymbol *Local = nullptr;
   if (cast<MCSymbolCOFF>(MCSym).isWeakExternal()) {
     Sym->Data.StorageClass = COFF::IMAGE_SYM_CLASS_WEAK_EXTERNAL;
+    Sym->Section = nullptr;
 
     COFFSymbol *WeakDefault = getLinkedSymbol(MCSym);
     if (!WeakDefault) {
diff --git a/llvm/test/MC/COFF/weak-comdat.s b/llvm/test/MC/COFF/weak-comdat.s
new file mode 100644
index 0000000000000..8605da6b521db
--- /dev/null
+++ b/llvm/test/MC/COFF/weak-comdat.s
@@ -0,0 +1,34 @@
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-win32 %s -o %t.o
+// RUN: llvm-readobj --symbols %t.o | FileCheck %s
+
+// Test that the weak symbol is properly undefined, while originally being
+// the leader symbol for a comdat. (This can easily happen if building with
+// -ffunction-sections).
+
+        .section .text$func,"xr",one_only,func
+        .weak   func
+func:
+        ret
+
+// CHECK:       Symbol {
+// CHECK:         Name: func
+// CHECK-NEXT:    Value: 0
+// CHECK-NEXT:    Section: IMAGE_SYM_UNDEFINED (0)
+// CHECK-NEXT:    BaseType: Null (0x0)
+// CHECK-NEXT:    ComplexType: Null (0x0)
+// CHECK-NEXT:    StorageClass: WeakExternal (0x69)
+// CHECK-NEXT:    AuxSymbolCount: 1
+// CHECK-NEXT:    AuxWeakExternal {
+// CHECK-NEXT:      Linked: .weak.func.default (10)
+// CHECK-NEXT:      Search: Alias (0x3)
+// CHECK-NEXT:    }
+// CHECK-NEXT:  }
+// CHECK-NEXT:  Symbol {
+// CHECK-NEXT:    Name: .weak.func.default
+// CHECK-NEXT:    Value: 0
+// CHECK-NEXT:    Section: .text$func (4)
+// CHECK-NEXT:    BaseType: Null (0x0)
+// CHECK-NEXT:    ComplexType: Null (0x0)
+// CHECK-NEXT:    StorageClass: External (0x2)
+// CHECK-NEXT:    AuxSymbolCount: 0
+// CHECK-NEXT:  }

From 032b78a0762bee129f33e4255ada6d374aa70c71 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kirst=C3=B3f=20Umann?= <dkszelethus@gmail.com>
Date: Fri, 24 Jul 2020 19:10:50 +0200
Subject: [PATCH 0037/1035] [analyzer] Revert the accidental commit of D82122

Was accidentally squished into
rGb6cbe6cb0399d4671e5384dcc326af56bc6bd122. The assert fires on the code
snippet included in this commit.

More discussion can be found in https://reviews.llvm.org/D82598.
---
 clang/lib/StaticAnalyzer/Core/Environment.cpp |  12 +--
 clang/test/Analysis/live-stmts.mm             | 101 ++++++++++++++++++
 2 files changed, 104 insertions(+), 9 deletions(-)
 create mode 100644 clang/test/Analysis/live-stmts.mm

diff --git a/clang/lib/StaticAnalyzer/Core/Environment.cpp b/clang/lib/StaticAnalyzer/Core/Environment.cpp
index 9e6d79bb7dcc9..1ccf4c6104a65 100644
--- a/clang/lib/StaticAnalyzer/Core/Environment.cpp
+++ b/clang/lib/StaticAnalyzer/Core/Environment.cpp
@@ -183,18 +183,12 @@ EnvironmentManager::removeDeadBindings(Environment Env,
              F.getTreeFactory());
 
   // Iterate over the block-expr bindings.
-  for (Environment::iterator I = Env.begin(), E = Env.end(); I != E; ++I) {
+  for (Environment::iterator I = Env.begin(), E = Env.end();
+       I != E; ++I) {
     const EnvironmentEntry &BlkExpr = I.getKey();
     const SVal &X = I.getData();
 
-    const bool IsBlkExprLive =
-        SymReaper.isLive(BlkExpr.getStmt(), BlkExpr.getLocationContext());
-
-    assert((isa<Expr>(BlkExpr.getStmt()) || !IsBlkExprLive) &&
-           "Only Exprs can be live, LivenessAnalysis argues about the liveness "
-           "of *values*!");
-
-    if (IsBlkExprLive) {
+    if (SymReaper.isLive(BlkExpr.getStmt(), BlkExpr.getLocationContext())) {
       // Copy the binding to the new map.
       EBMapRef = EBMapRef.add(BlkExpr, X);
 
diff --git a/clang/test/Analysis/live-stmts.mm b/clang/test/Analysis/live-stmts.mm
new file mode 100644
index 0000000000000..a6ddd03ca5d85
--- /dev/null
+++ b/clang/test/Analysis/live-stmts.mm
@@ -0,0 +1,101 @@
+// RUN: %clang_analyze_cc1 -w -fblocks %s \
+// RUN:   -analyzer-checker=debug.DumpLiveStmts \
+// RUN:   2>&1 | FileCheck %s
+
+@interface Item
+// ...
+@end
+
+@interface Collection
+// ...
+@end
+
+typedef void (^Blk)();
+
+struct RAII {
+  Blk blk;
+
+public:
+  RAII(Blk blk): blk(blk) {}
+
+// CHECK: [ B0 (live statements at block exit) ]
+// CHECK-EMPTY:
+// CHECK-EMPTY:
+// CHECK-NEXT: [ B1 (live statements at block exit) ]
+// CHECK-EMPTY:
+// CHECK-EMPTY:
+// CHECK-NEXT: [ B2 (live statements at block exit) ]
+// CHECK-EMPTY:
+// CHECK-EMPTY:
+
+  ~RAII() { blk(); }
+
+// CHECK-NEXT: [ B0 (live statements at block exit) ]
+// CHECK-EMPTY:
+// CHECK-EMPTY:
+// CHECK-NEXT: [ B1 (live statements at block exit) ]
+// CHECK-EMPTY:
+// CHECK-EMPTY:
+// CHECK-NEXT: [ B2 (live statements at block exit) ]
+// CHECK-EMPTY:
+// CHECK-EMPTY:
+};
+
+void foo(Collection *coll) {
+  RAII raii(^{});
+  for (Item *item in coll) {}
+}
+// CHECK-NEXT: [ B0 (live statements at block exit) ]
+// CHECK-EMPTY:
+// CHECK-EMPTY:
+// CHECK-NEXT: [ B1 (live statements at block exit) ]
+// CHECK-EMPTY:
+// CHECK-EMPTY:
+// CHECK-NEXT: [ B2 (live statements at block exit) ]
+// CHECK-EMPTY:
+// CHECK-NEXT: DeclStmt {{.*}}
+// CHECK-NEXT: `-VarDecl {{.*}}  item 'Item *'
+// CHECK-EMPTY:
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'Collection *' <LValueToRValue>
+// CHECK-NEXT: `-DeclRefExpr {{.*}} 'Collection *' lvalue ParmVar {{.*}} 'coll' 'Collection *'
+// CHECK-EMPTY:
+// CHECK-NEXT: CompoundStmt {{.*}}
+// CHECK-EMPTY:
+// CHECK-EMPTY:
+// CHECK-NEXT: [ B3 (live statements at block exit) ]
+// CHECK-EMPTY:
+// CHECK-NEXT: DeclStmt {{.*}}
+// CHECK-NEXT: `-VarDecl {{.*}}  item 'Item *'
+// CHECK-EMPTY:
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'Collection *' <LValueToRValue>
+// CHECK-NEXT: `-DeclRefExpr {{.*}} 'Collection *' lvalue ParmVar {{.*}} 'coll' 'Collection *'
+// CHECK-EMPTY:
+// CHECK-NEXT: CompoundStmt {{.*}}
+// CHECK-EMPTY:
+// CHECK-EMPTY:
+// CHECK-NEXT: [ B4 (live statements at block exit) ]
+// CHECK-EMPTY:
+// CHECK-NEXT: DeclStmt {{.*}}
+// CHECK-NEXT: `-VarDecl {{.*}}  item 'Item *'
+// CHECK-EMPTY:
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'Collection *' <LValueToRValue>
+// CHECK-NEXT: `-DeclRefExpr {{.*}} 'Collection *' lvalue ParmVar {{.*}} 'coll' 'Collection *'
+// CHECK-EMPTY:
+// CHECK-NEXT: CompoundStmt {{.*}}
+// CHECK-EMPTY:
+// CHECK-EMPTY:
+// CHECK-NEXT: [ B5 (live statements at block exit) ]
+// CHECK-EMPTY:
+// CHECK-NEXT: DeclStmt {{.*}}
+// CHECK-NEXT: `-VarDecl {{.*}}  item 'Item *'
+// CHECK-EMPTY:
+// CHECK-NEXT: CompoundStmt {{.*}}
+// CHECK-EMPTY:
+// CHECK-EMPTY:
+// CHECK-NEXT: [ B0 (live statements at block exit) ]
+// CHECK-EMPTY:
+// CHECK-EMPTY:
+// CHECK-NEXT: [ B1 (live statements at block exit) ]
+// CHECK-EMPTY:
+// CHECK-EMPTY:
+

From 7d076e19e31a2a32e357cbdcf0183f88fe1fb0fb Mon Sep 17 00:00:00 2001
From: Nemanja Ivanovic <nemanja.i.ibm@gmail.com>
Date: Fri, 24 Jul 2020 15:38:27 -0400
Subject: [PATCH 0038/1035] [PowerPC] Fix computation of offset for
 load-and-splat for permuted loads

Unfortunately this is another regression from my canonicalization patch
(1fed131660b2). The patch contained two implicit assumptions:
1. That we would have a permuted load only if we are loading a partial vector
2. That a partial vector load would necessarily be as wide as the splat

However, assumption 2 is not correct since it is possible to do a wider
load and only splat a half of it. This patch corrects this assumption by
simply checking if the load is permuted and adjusting the offset if it is.
---
 llvm/lib/Target/PowerPC/PPCISelLowering.cpp   | 26 ++++--
 .../PowerPC/canonical-merge-shuffles.ll       | 88 +++++++++++++++++++
 2 files changed, 106 insertions(+), 8 deletions(-)

diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index edc23b2673f38..c2ba7195509a1 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -9126,13 +9126,15 @@ SDValue PPCTargetLowering::LowerBITCAST(SDValue Op, SelectionDAG &DAG) const {
                      Op0.getOperand(1));
 }
 
-static const SDValue *getNormalLoadInput(const SDValue &Op) {
+static const SDValue *getNormalLoadInput(const SDValue &Op, bool &IsPermuted) {
   const SDValue *InputLoad = &Op;
   if (InputLoad->getOpcode() == ISD::BITCAST)
     InputLoad = &InputLoad->getOperand(0);
   if (InputLoad->getOpcode() == ISD::SCALAR_TO_VECTOR ||
-      InputLoad->getOpcode() == PPCISD::SCALAR_TO_VECTOR_PERMUTED)
+      InputLoad->getOpcode() == PPCISD::SCALAR_TO_VECTOR_PERMUTED) {
+    IsPermuted = InputLoad->getOpcode() == PPCISD::SCALAR_TO_VECTOR_PERMUTED;
     InputLoad = &InputLoad->getOperand(0);
+  }
   if (InputLoad->getOpcode() != ISD::LOAD)
     return nullptr;
   LoadSDNode *LD = cast<LoadSDNode>(*InputLoad);
@@ -9304,7 +9306,8 @@ SDValue PPCTargetLowering::LowerBUILD_VECTOR(SDValue Op,
 
   if (!BVNIsConstantSplat || SplatBitSize > 32) {
 
-    const SDValue *InputLoad = getNormalLoadInput(Op.getOperand(0));
+    bool IsPermutedLoad = false;
+    const SDValue *InputLoad = getNormalLoadInput(Op.getOperand(0), IsPermutedLoad);
     // Handle load-and-splat patterns as we have instructions that will do this
     // in one go.
     if (InputLoad && DAG.isSplatValue(Op, true)) {
@@ -9927,7 +9930,8 @@ SDValue PPCTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
   // If this is a load-and-splat, we can do that with a single instruction
   // in some cases. However if the load has multiple uses, we don't want to
   // combine it because that will just produce multiple loads.
-  const SDValue *InputLoad = getNormalLoadInput(V1);
+  bool IsPermutedLoad = false;
+  const SDValue *InputLoad = getNormalLoadInput(V1, IsPermutedLoad);
   if (InputLoad && Subtarget.hasVSX() && V2.isUndef() &&
       (PPC::isSplatShuffleMask(SVOp, 4) || PPC::isSplatShuffleMask(SVOp, 8)) &&
       InputLoad->hasOneUse()) {
@@ -9935,6 +9939,16 @@ SDValue PPCTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
     int SplatIdx =
       PPC::getSplatIdxForPPCMnemonics(SVOp, IsFourByte ? 4 : 8, DAG);
 
+    // The splat index for permuted loads will be in the left half of the vector
+    // which is strictly wider than the loaded value by 8 bytes. So we need to
+    // adjust the splat index to point to the correct address in memory.
+    if (IsPermutedLoad) {
+      assert(isLittleEndian && "Unexpected permuted load on big endian target");
+      SplatIdx += IsFourByte ? 2 : 1;
+      assert(SplatIdx < IsFourByte ? 4 : 2 &&
+             "Splat of a value outside of the loaded memory");
+    }
+
     LoadSDNode *LD = cast<LoadSDNode>(*InputLoad);
     // For 4-byte load-and-splat, we need Power9.
     if ((IsFourByte && Subtarget.hasP9Vector()) || !IsFourByte) {
@@ -9944,10 +9958,6 @@ SDValue PPCTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
       else
         Offset = isLittleEndian ? (1 - SplatIdx) * 8 : SplatIdx * 8;
 
-      // If we are loading a partial vector, it does not make sense to adjust
-      // the base pointer. This happens with (splat (s_to_v_permuted (ld))).
-      if (LD->getMemoryVT().getSizeInBits() == (IsFourByte ? 32 : 64))
-        Offset = 0;
       SDValue BasePtr = LD->getBasePtr();
       if (Offset != 0)
         BasePtr = DAG.getNode(ISD::ADD, dl, getPointerTy(DAG.getDataLayout()),
diff --git a/llvm/test/CodeGen/PowerPC/canonical-merge-shuffles.ll b/llvm/test/CodeGen/PowerPC/canonical-merge-shuffles.ll
index 11bc2bae9871f..cdd04b33318ea 100644
--- a/llvm/test/CodeGen/PowerPC/canonical-merge-shuffles.ll
+++ b/llvm/test/CodeGen/PowerPC/canonical-merge-shuffles.ll
@@ -446,5 +446,93 @@ entry:
   ret <16 x i8> %shuffle
 }
 
+define dso_local <4 x i32> @testSplat4Low(<8 x i8>* nocapture readonly %ptr) local_unnamed_addr #0 {
+; CHECK-P8-LABEL: testSplat4Low:
+; CHECK-P8:       # %bb.0: # %entry
+; CHECK-P8-NEXT:    ld r3, 0(r3)
+; CHECK-P8-NEXT:    mtfprd f0, r3
+; CHECK-P8-NEXT:    xxspltw v2, vs0, 0
+; CHECK-P8-NEXT:    blr
+;
+; CHECK-P9-LABEL: testSplat4Low:
+; CHECK-P9:       # %bb.0: # %entry
+; CHECK-P9-NEXT:    addi r3, r3, 4
+; CHECK-P9-NEXT:    lxvwsx v2, 0, r3
+; CHECK-P9-NEXT:    blr
+;
+; CHECK-NOVSX-LABEL: testSplat4Low:
+; CHECK-NOVSX:       # %bb.0: # %entry
+; CHECK-NOVSX-NEXT:    ld r3, 0(r3)
+; CHECK-NOVSX-NEXT:    addi r4, r1, -16
+; CHECK-NOVSX-NEXT:    std r3, -16(r1)
+; CHECK-NOVSX-NEXT:    lvx v2, 0, r4
+; CHECK-NOVSX-NEXT:    vspltw v2, v2, 2
+; CHECK-NOVSX-NEXT:    blr
+entry:
+  %0 = load <8 x i8>, <8 x i8>* %ptr, align 8
+  %vecinit18 = shufflevector <8 x i8> %0, <8 x i8> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %1 = bitcast <16 x i8> %vecinit18 to <4 x i32>
+  ret <4 x i32> %1
+}
+
+; Function Attrs: norecurse nounwind readonly
+define dso_local <4 x i32> @testSplat4hi(<8 x i8>* nocapture readonly %ptr) local_unnamed_addr #0 {
+; CHECK-P8-LABEL: testSplat4hi:
+; CHECK-P8:       # %bb.0: # %entry
+; CHECK-P8-NEXT:    ld r3, 0(r3)
+; CHECK-P8-NEXT:    mtfprd f0, r3
+; CHECK-P8-NEXT:    xxspltw v2, vs0, 1
+; CHECK-P8-NEXT:    blr
+;
+; CHECK-P9-LABEL: testSplat4hi:
+; CHECK-P9:       # %bb.0: # %entry
+; CHECK-P9-NEXT:    lxvwsx v2, 0, r3
+; CHECK-P9-NEXT:    blr
+;
+; CHECK-NOVSX-LABEL: testSplat4hi:
+; CHECK-NOVSX:       # %bb.0: # %entry
+; CHECK-NOVSX-NEXT:    ld r3, 0(r3)
+; CHECK-NOVSX-NEXT:    addi r4, r1, -16
+; CHECK-NOVSX-NEXT:    std r3, -16(r1)
+; CHECK-NOVSX-NEXT:    lvx v2, 0, r4
+; CHECK-NOVSX-NEXT:    vspltw v2, v2, 3
+; CHECK-NOVSX-NEXT:    blr
+entry:
+  %0 = load <8 x i8>, <8 x i8>* %ptr, align 8
+  %vecinit22 = shufflevector <8 x i8> %0, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %1 = bitcast <16 x i8> %vecinit22 to <4 x i32>
+  ret <4 x i32> %1
+}
+
+; Function Attrs: norecurse nounwind readonly
+define dso_local <2 x i64> @testSplat8(<8 x i8>* nocapture readonly %ptr) local_unnamed_addr #0 {
+; CHECK-P8-LABEL: testSplat8:
+; CHECK-P8:       # %bb.0: # %entry
+; CHECK-P8-NEXT:    lxvdsx v2, 0, r3
+; CHECK-P8-NEXT:    blr
+;
+; CHECK-P9-LABEL: testSplat8:
+; CHECK-P9:       # %bb.0: # %entry
+; CHECK-P9-NEXT:    lxvdsx v2, 0, r3
+; CHECK-P9-NEXT:    blr
+;
+; CHECK-NOVSX-LABEL: testSplat8:
+; CHECK-NOVSX:       # %bb.0: # %entry
+; CHECK-NOVSX-NEXT:    ld r3, 0(r3)
+; CHECK-NOVSX-NEXT:    addis r4, r2, .LCPI19_0@toc@ha
+; CHECK-NOVSX-NEXT:    addi r4, r4, .LCPI19_0@toc@l
+; CHECK-NOVSX-NEXT:    lvx v2, 0, r4
+; CHECK-NOVSX-NEXT:    std r3, -16(r1)
+; CHECK-NOVSX-NEXT:    addi r3, r1, -16
+; CHECK-NOVSX-NEXT:    lvx v3, 0, r3
+; CHECK-NOVSX-NEXT:    vperm v2, v3, v3, v2
+; CHECK-NOVSX-NEXT:    blr
+entry:
+  %0 = load <8 x i8>, <8 x i8>* %ptr, align 8
+  %vecinit30 = shufflevector <8 x i8> %0, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %1 = bitcast <16 x i8> %vecinit30 to <2 x i64>
+  ret <2 x i64> %1
+}
+
 declare double @dummy() local_unnamed_addr
 attributes #0 = { nounwind }

From 679158e662aa247282b8eea4c2d60b33204171fb Mon Sep 17 00:00:00 2001
From: Jon Chesterfield <jonathanchesterfield@gmail.com>
Date: Fri, 24 Jul 2020 20:50:25 +0100
Subject: [PATCH 0039/1035] Make hip math headers easier to use from C

Summary:
Make hip math headers easier to use from C

Motivation is a step towards using the hip math headers to implement math.h
for openmp, which needs to work with C as well as C++. NFC for C++ code.

Reviewers: yaxunl, jdoerfert

Reviewed By: yaxunl

Subscribers: sstefan1, cfe-commits

Tags: #clang

Differential Revision: https://reviews.llvm.org/D84476
---
 clang/lib/Headers/__clang_hip_libdevice_declares.h |  6 +++++-
 clang/lib/Headers/__clang_hip_math.h               | 10 ++++++++--
 2 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/clang/lib/Headers/__clang_hip_libdevice_declares.h b/clang/lib/Headers/__clang_hip_libdevice_declares.h
index 7110404434405..2cf9cc7f1eb65 100644
--- a/clang/lib/Headers/__clang_hip_libdevice_declares.h
+++ b/clang/lib/Headers/__clang_hip_libdevice_declares.h
@@ -10,7 +10,9 @@
 #ifndef __CLANG_HIP_LIBDEVICE_DECLARES_H__
 #define __CLANG_HIP_LIBDEVICE_DECLARES_H__
 
+#ifdef __cplusplus
 extern "C" {
+#endif
 
 // BEGIN FLOAT
 __device__ __attribute__((const)) float __ocml_acos_f32(float);
@@ -316,7 +318,7 @@ __device__ __attribute__((pure)) __2f16 __ocml_log2_2f16(__2f16);
 __device__ inline __2f16
 __llvm_amdgcn_rcp_2f16(__2f16 __x) // Not currently exposed by ROCDL.
 {
-  return __2f16{__llvm_amdgcn_rcp_f16(__x.x), __llvm_amdgcn_rcp_f16(__x.y)};
+  return (__2f16){__llvm_amdgcn_rcp_f16(__x.x), __llvm_amdgcn_rcp_f16(__x.y)};
 }
 __device__ __attribute__((const)) __2f16 __ocml_rint_2f16(__2f16);
 __device__ __attribute__((const)) __2f16 __ocml_rsqrt_2f16(__2f16);
@@ -325,6 +327,8 @@ __device__ __attribute__((const)) __2f16 __ocml_sqrt_2f16(__2f16);
 __device__ __attribute__((const)) __2f16 __ocml_trunc_2f16(__2f16);
 __device__ __attribute__((const)) __2f16 __ocml_pown_2f16(__2f16, __2i16);
 
+#ifdef __cplusplus
 } // extern "C"
+#endif
 
 #endif // __CLANG_HIP_LIBDEVICE_DECLARES_H__
diff --git a/clang/lib/Headers/__clang_hip_math.h b/clang/lib/Headers/__clang_hip_math.h
index 47d3c17175591..f9ca9bf606fb7 100644
--- a/clang/lib/Headers/__clang_hip_math.h
+++ b/clang/lib/Headers/__clang_hip_math.h
@@ -95,8 +95,10 @@ inline uint64_t __make_mantissa(const char *__tagp) {
 }
 
 // BEGIN FLOAT
+#ifdef __cplusplus
 __DEVICE__
 inline float abs(float __x) { return __ocml_fabs_f32(__x); }
+#endif
 __DEVICE__
 inline float acosf(float __x) { return __ocml_acos_f32(__x); }
 __DEVICE__
@@ -251,7 +253,7 @@ inline float nanf(const char *__tagp) {
       uint32_t sign : 1;
     } bits;
 
-    static_assert(sizeof(float) == sizeof(ieee_float), "");
+    static_assert(sizeof(float) == sizeof(struct ieee_float), "");
   } __tmp;
 
   __tmp.bits.sign = 0u;
@@ -553,8 +555,10 @@ inline float __tanf(float __x) { return __ocml_tan_f32(__x); }
 // END FLOAT
 
 // BEGIN DOUBLE
+#ifdef __cplusplus
 __DEVICE__
 inline double abs(double __x) { return __ocml_fabs_f64(__x); }
+#endif
 __DEVICE__
 inline double acos(double __x) { return __ocml_acos_f64(__x); }
 __DEVICE__
@@ -712,7 +716,7 @@ inline double nan(const char *__tagp) {
       uint32_t exponent : 11;
       uint32_t sign : 1;
     } bits;
-    static_assert(sizeof(double) == sizeof(ieee_double), "");
+    static_assert(sizeof(double) == sizeof(struct ieee_double), "");
   } __tmp;
 
   __tmp.bits.sign = 0u;
@@ -1178,6 +1182,7 @@ __host__ inline static int max(int __arg1, int __arg2) {
   return std::max(__arg1, __arg2);
 }
 
+#ifdef __cplusplus
 __DEVICE__
 inline float pow(float __base, int __iexp) { return powif(__base, __iexp); }
 
@@ -1188,6 +1193,7 @@ __DEVICE__
 inline _Float16 pow(_Float16 __base, int __iexp) {
   return __ocml_pown_f16(__base, __iexp);
 }
+#endif
 
 #pragma pop_macro("__DEF_FUN1")
 #pragma pop_macro("__DEF_FUN2")

From 51eeeb477fad47d4885f3fe226aa4941470dae26 Mon Sep 17 00:00:00 2001
From: cgyurgyik <gyurgyikcp@gmail.com>
Date: Fri, 24 Jul 2020 15:40:19 -0400
Subject: [PATCH 0040/1035] [libc] [Obvious] Place entrypoints, specs
 alphabetically.

---
 libc/config/linux/aarch64/entrypoints.txt | 10 +++++-----
 libc/config/linux/api.td                  | 24 +++++++++++------------
 libc/config/linux/x86_64/entrypoints.txt  | 12 ++++++------
 3 files changed, 23 insertions(+), 23 deletions(-)

diff --git a/libc/config/linux/aarch64/entrypoints.txt b/libc/config/linux/aarch64/entrypoints.txt
index b287a72d779b9..8cdc6ca68621f 100644
--- a/libc/config/linux/aarch64/entrypoints.txt
+++ b/libc/config/linux/aarch64/entrypoints.txt
@@ -4,16 +4,16 @@ set(TARGET_LIBC_ENTRYPOINTS
 
     # string.h entrypoints
     libc.src.string.bzero
+    libc.src.string.memchr
     libc.src.string.memcpy
     libc.src.string.memset
-    libc.src.string.strcpy
+    libc.src.string.memrchr
     libc.src.string.strcat
-    libc.src.string.strlen
-    libc.src.string.memchr
     libc.src.string.strchr
-    libc.src.string.strstr
+    libc.src.string.strcpy
+    libc.src.string.strlen
     libc.src.string.strnlen
-    libc.src.string.memrchr
+    libc.src.string.strstr
 )
 
 set(TARGET_LIBM_ENTRYPOINTS
diff --git a/libc/config/linux/api.td b/libc/config/linux/api.td
index 5f7a858d5fa31..3abdf9f55b07a 100644
--- a/libc/config/linux/api.td
+++ b/libc/config/linux/api.td
@@ -194,30 +194,30 @@ def MathAPI : PublicAPI<"math.h"> {
 def StringAPI : PublicAPI<"string.h"> {
   let Functions = [
     "bzero",
+    "memchr",
+    "memcmp",
     "memcpy",
     "memmove",
-    "memcmp",
-    "memchr",
+    "memrchr",
     "memset",
-    "strcpy",
-    "strncpy",
     "strcat",
-    "strncat",
+    "strchr",
     "strcmp",
     "strcoll",
+    "strcpy",
+    "strcspn",  
+    "strerror",
+    "strlen",
+    "strncat",
     "strncmp",
-    "strxfrm",
-    "strchr",
-    "strcspn",
+    "strncpy",
+    "strnlen",
     "strpbrk",
     "strrchr",
     "strspn",
     "strstr",
     "strtok",
-    "strerror",
-    "strlen",
-    "strnlen",
-    "memrchr"
+    "strxfrm",   
   ];
 
   let TypeDeclarations = [
diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt
index db53005304896..16dd702d0eb56 100644
--- a/libc/config/linux/x86_64/entrypoints.txt
+++ b/libc/config/linux/x86_64/entrypoints.txt
@@ -21,17 +21,17 @@ set(TARGET_LIBC_ENTRYPOINTS
 
     # string.h entrypoints
     libc.src.string.bzero
+    libc.src.string.memchr
     libc.src.string.memcpy
+    libc.src.string.memrchr
     libc.src.string.memset
-    libc.src.string.strcpy
     libc.src.string.strcat
-    libc.src.string.strlen
-    libc.src.string.strcmp
-    libc.src.string.memchr
     libc.src.string.strchr
-    libc.src.string.strstr
+    libc.src.string.strcmp
+    libc.src.string.strcpy
+    libc.src.string.strlen
     libc.src.string.strnlen
-    libc.src.string.memrchr
+    libc.src.string.strstr
 
     # sys/mman.h entrypoints
     libc.src.sys.mman.mmap

From 1e77b3af125e4d1ba79e5d502b959d49f3addef9 Mon Sep 17 00:00:00 2001
From: Gui Andrade <guiand@google.com>
Date: Fri, 24 Jul 2020 20:00:02 +0000
Subject: [PATCH 0041/1035] [MSAN] Allow inserting array checks

Flattens arrays by ORing together all their elements.

Differential Revision: https://reviews.llvm.org/D84446
---
 .../Instrumentation/MemorySanitizer.cpp       | 79 +++++++++++--------
 .../MemorySanitizer/check-array.ll            | 19 +++++
 2 files changed, 65 insertions(+), 33 deletions(-)
 create mode 100644 llvm/test/Instrumentation/MemorySanitizer/check-array.ll

diff --git a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
index f4f62a31d89ee..427abde4277d4 100644
--- a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
@@ -1149,36 +1149,30 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     const DataLayout &DL = F.getParent()->getDataLayout();
     const Align OriginAlignment = std::max(kMinOriginAlignment, Alignment);
     unsigned StoreSize = DL.getTypeStoreSize(Shadow->getType());
-    if (Shadow->getType()->isArrayTy()) {
-      paintOrigin(IRB, updateOrigin(Origin, IRB), OriginPtr, StoreSize,
-                  OriginAlignment);
-    } else {
-      Value *ConvertedShadow = convertShadowToScalar(Shadow, IRB);
-      if (auto *ConstantShadow = dyn_cast<Constant>(ConvertedShadow)) {
-        if (ClCheckConstantShadow && !ConstantShadow->isZeroValue())
-          paintOrigin(IRB, updateOrigin(Origin, IRB), OriginPtr, StoreSize,
-                      OriginAlignment);
-        return;
-      }
-
-      unsigned TypeSizeInBits =
-          DL.getTypeSizeInBits(ConvertedShadow->getType());
-      unsigned SizeIndex = TypeSizeToSizeIndex(TypeSizeInBits);
-      if (AsCall && SizeIndex < kNumberOfAccessSizes && !MS.CompileKernel) {
-        FunctionCallee Fn = MS.MaybeStoreOriginFn[SizeIndex];
-        Value *ConvertedShadow2 = IRB.CreateZExt(
-            ConvertedShadow, IRB.getIntNTy(8 * (1 << SizeIndex)));
-        IRB.CreateCall(Fn, {ConvertedShadow2,
-                            IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy()),
-                            Origin});
-      } else {
-        Value *Cmp = convertToBool(ConvertedShadow, IRB, "_mscmp");
-        Instruction *CheckTerm = SplitBlockAndInsertIfThen(
-            Cmp, &*IRB.GetInsertPoint(), false, MS.OriginStoreWeights);
-        IRBuilder<> IRBNew(CheckTerm);
-        paintOrigin(IRBNew, updateOrigin(Origin, IRBNew), OriginPtr, StoreSize,
+    Value *ConvertedShadow = convertShadowToScalar(Shadow, IRB);
+    if (auto *ConstantShadow = dyn_cast<Constant>(ConvertedShadow)) {
+      if (ClCheckConstantShadow && !ConstantShadow->isZeroValue())
+        paintOrigin(IRB, updateOrigin(Origin, IRB), OriginPtr, StoreSize,
                     OriginAlignment);
-      }
+      return;
+    }
+
+    unsigned TypeSizeInBits = DL.getTypeSizeInBits(ConvertedShadow->getType());
+    unsigned SizeIndex = TypeSizeToSizeIndex(TypeSizeInBits);
+    if (AsCall && SizeIndex < kNumberOfAccessSizes && !MS.CompileKernel) {
+      FunctionCallee Fn = MS.MaybeStoreOriginFn[SizeIndex];
+      Value *ConvertedShadow2 =
+          IRB.CreateZExt(ConvertedShadow, IRB.getIntNTy(8 * (1 << SizeIndex)));
+      IRB.CreateCall(Fn,
+                     {ConvertedShadow2,
+                      IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy()), Origin});
+    } else {
+      Value *Cmp = convertToBool(ConvertedShadow, IRB, "_mscmp");
+      Instruction *CheckTerm = SplitBlockAndInsertIfThen(
+          Cmp, &*IRB.GetInsertPoint(), false, MS.OriginStoreWeights);
+      IRBuilder<> IRBNew(CheckTerm);
+      paintOrigin(IRBNew, updateOrigin(Origin, IRBNew), OriginPtr, StoreSize,
+                  OriginAlignment);
     }
   }
 
@@ -1410,12 +1404,31 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     return Aggregator;
   }
 
+  // Extract combined shadow of array elements
+  Value *collapseArrayShadow(ArrayType *Array, Value *Shadow,
+                             IRBuilder<> &IRB) {
+    if (!Array->getNumElements())
+      return IRB.getIntN(/* width */ 1, /* value */ 0);
+
+    Value *FirstItem = IRB.CreateExtractValue(Shadow, 0);
+    Value *Aggregator = convertShadowToScalar(FirstItem, IRB);
+
+    for (unsigned Idx = 1; Idx < Array->getNumElements(); Idx++) {
+      Value *ShadowItem = IRB.CreateExtractValue(Shadow, Idx);
+      Value *ShadowInner = convertShadowToScalar(ShadowItem, IRB);
+      Aggregator = IRB.CreateOr(Aggregator, ShadowInner);
+    }
+    return Aggregator;
+  }
+
   /// Convert a shadow value to it's flattened variant. The resulting
   /// shadow may not necessarily have the same bit width as the input
   /// value, but it will always be comparable to zero.
   Value *convertShadowToScalar(Value *V, IRBuilder<> &IRB) {
     if (StructType *Struct = dyn_cast<StructType>(V->getType()))
       return collapseStructShadow(Struct, V, IRB);
+    if (ArrayType *Array = dyn_cast<ArrayType>(V->getType()))
+      return collapseArrayShadow(Array, V, IRB);
     Type *Ty = V->getType();
     Type *NoVecTy = getShadowTyNoVec(Ty);
     if (Ty == NoVecTy) return V;
@@ -1765,10 +1778,10 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     if (!InsertChecks) return;
 #ifndef NDEBUG
     Type *ShadowTy = Shadow->getType();
-    assert(
-        (isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy) ||
-         isa<StructType>(ShadowTy)) &&
-        "Can only insert checks for integer, vector, and struct shadow types");
+    assert((isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy) ||
+            isa<StructType>(ShadowTy) || isa<ArrayType>(ShadowTy)) &&
+           "Can only insert checks for integer, vector, and aggregate shadow "
+           "types");
 #endif
     InstrumentationList.push_back(
         ShadowOriginAndInsertPoint(Shadow, Origin, OrigIns));
diff --git a/llvm/test/Instrumentation/MemorySanitizer/check-array.ll b/llvm/test/Instrumentation/MemorySanitizer/check-array.ll
new file mode 100644
index 0000000000000..6d1e517a186d4
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/check-array.ll
@@ -0,0 +1,19 @@
+; RUN: opt < %s -msan-eager-checks -msan-check-access-address=0 -msan-track-origins=1 -S -passes='module(msan-module),function(msan)' 2>&1 | \
+; RUN:   FileCheck -allow-deprecated-dag-overlap -check-prefixes=CHECK,CHECK-ORIGINS %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define noundef [2 x i24] @check_array([2 x i24]* %p) sanitize_memory {
+; CHECK: @check_array([2 x i24]* [[P:%.*]])
+; CHECK: [[O:%.*]] = load [2 x i24], [2 x i24]* [[P]]
+  %o = load [2 x i24], [2 x i24]* %p
+; CHECK: [[FIELD0:%.+]] = extractvalue [2 x i24] %_msld, 0
+; CHECK: [[FIELD1:%.+]] = extractvalue [2 x i24] %_msld, 1
+; CHECK: [[F1_OR:%.+]] = or i24 [[FIELD0]], [[FIELD1]]
+; CHECK: %_mscmp = icmp ne i24 [[F1_OR]], 0
+; CHECK: br i1 %_mscmp
+; CHECK: call void @__msan_warning
+; CHECK: ret [2 x i24] [[O]]
+  ret [2 x i24] %o
+}

From 0db2934b0fa9e00ac98e2cb168adba96f6bcd0da Mon Sep 17 00:00:00 2001
From: shafik <syaghmour@apple.com>
Date: Fri, 24 Jul 2020 13:11:59 -0700
Subject: [PATCH 0042/1035] [ASTImporter] Modify ImportDefiniton for
 ObjCInterfaceDecl so that we always the ImportDeclContext one we start the
 definition

Once we start the definition of an ObjCInterfaceDecl we won't attempt to ImportDeclContext
later on. Unlike RecordDecl case which uses DefinitionCompleter to force completeDefinition
we don't seem to have a similar mechanism for ObjCInterfaceDecl.

This fix was needed due to a bug we see in LLDB expression parsing where an initial expression
cause an ObjCInterfaceDecl to be defined and subsequent expressions during import do not call
ImportDeclContext and we can end up in a situation where ivars are imported out of order and not all ivars are imported.

Differential Revision: https://reviews.llvm.org/D83972
---
 clang/lib/AST/ASTImporter.cpp                            | 9 ++++-----
 .../API/lang/objc/bitfield_ivars/TestBitfieldIvars.py    | 5 ++---
 2 files changed, 6 insertions(+), 8 deletions(-)

diff --git a/clang/lib/AST/ASTImporter.cpp b/clang/lib/AST/ASTImporter.cpp
index fcfaba625a722..e0bca8f08bb41 100644
--- a/clang/lib/AST/ASTImporter.cpp
+++ b/clang/lib/AST/ASTImporter.cpp
@@ -4758,11 +4758,10 @@ Error ASTNodeImporter::ImportDefinition(
       return ToImplOrErr.takeError();
   }
 
-  if (shouldForceImportDeclContext(Kind)) {
-    // Import all of the members of this class.
-    if (Error Err = ImportDeclContext(From, /*ForceImport=*/true))
-      return Err;
-  }
+  // Import all of the members of this class.
+  if (Error Err = ImportDeclContext(From, /*ForceImport=*/true))
+    return Err;
+
   return Error::success();
 }
 
diff --git a/lldb/test/API/lang/objc/bitfield_ivars/TestBitfieldIvars.py b/lldb/test/API/lang/objc/bitfield_ivars/TestBitfieldIvars.py
index 6118854131024..4154bb144b350 100644
--- a/lldb/test/API/lang/objc/bitfield_ivars/TestBitfieldIvars.py
+++ b/lldb/test/API/lang/objc/bitfield_ivars/TestBitfieldIvars.py
@@ -14,9 +14,8 @@ def test(self):
         lldbutil.run_to_source_breakpoint(self, "// break here", lldb.SBFileSpec("main.m"))
 
         self.expect_expr("chb->hb->field1", result_type="unsigned int", result_value="0")
-
-        ## FIXME field2 should have a value of 1
-        self.expect("expr chb->hb->field2", matching=False, substrs = ["= 1"]) # this must happen second
+        ## This should happen second
+        self.expect_expr("chb->hb->field2", result_type="unsigned int", result_value="1")
 
         self.expect_expr("hb2->field1", result_type="unsigned int", result_value="10")
         self.expect_expr("hb2->field2", result_type="unsigned int", result_value="3")

From dfa267a61c2b797b8fe9c345ee94742d496b39c6 Mon Sep 17 00:00:00 2001
From: Jacques Pienaar <jpienaar@google.com>
Date: Fri, 24 Jul 2020 13:15:53 -0700
Subject: [PATCH 0043/1035] [mlir][shape] Fix missing dependency

---
 mlir/lib/Dialect/Shape/IR/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/mlir/lib/Dialect/Shape/IR/CMakeLists.txt b/mlir/lib/Dialect/Shape/IR/CMakeLists.txt
index 84e085fbafdfa..e39f1c770f29f 100644
--- a/mlir/lib/Dialect/Shape/IR/CMakeLists.txt
+++ b/mlir/lib/Dialect/Shape/IR/CMakeLists.txt
@@ -17,4 +17,5 @@ add_mlir_dialect_library(MLIRShape
   MLIRInferTypeOpInterface
   MLIRIR
   MLIRSideEffectInterfaces
+  MLIRStandardOps
   )

From 9c87466c3960145eb5baa2881d4239de1dcfbc6f Mon Sep 17 00:00:00 2001
From: Johannes Doerfert <johannes@jdoerfert.de>
Date: Thu, 16 Jul 2020 16:55:09 -0500
Subject: [PATCH 0044/1035] [OpenMP] Use `abort` not `error` for fatal runtime
 exceptions

See PR46515 for the rational but generally, we want to *really* abort
not gracefully shut down.

Reviewed By: grokos, ABataev

Differential Revision: https://reviews.llvm.org/D83963
---
 openmp/libomptarget/src/private.h | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/openmp/libomptarget/src/private.h b/openmp/libomptarget/src/private.h
index 7772175cdca2e..ba2f161c7927a 100644
--- a/openmp/libomptarget/src/private.h
+++ b/openmp/libomptarget/src/private.h
@@ -93,17 +93,17 @@ typedef int (*TargetDataFuncPtrTy)(DeviceTy &, int32_t, void **, void **,
     fprintf(stderr, "Libomptarget message: " _str "\n", __VA_ARGS__);          \
   } while (0)
 
-#define FATAL_MESSAGE0(_num, _str)                                    \
-  do {                                                                \
-    fprintf(stderr, "Libomptarget fatal error %d: %s\n", _num, _str); \
-    exit(1);                                                          \
+#define FATAL_MESSAGE0(_num, _str)                                             \
+  do {                                                                         \
+    fprintf(stderr, "Libomptarget fatal error %d: %s\n", _num, _str);          \
+    abort();                                                                   \
   } while (0)
 
-#define FATAL_MESSAGE(_num, _str, ...)                              \
-  do {                                                              \
-    fprintf(stderr, "Libomptarget fatal error %d:" _str "\n", _num, \
-            __VA_ARGS__);                                           \
-    exit(1);                                                        \
+#define FATAL_MESSAGE(_num, _str, ...)                                         \
+  do {                                                                         \
+    fprintf(stderr, "Libomptarget fatal error %d:" _str "\n", _num,            \
+            __VA_ARGS__);                                                      \
+    abort();                                                                   \
   } while (0)
 
 // Implemented in libomp, they are called from within __tgt_* functions.

From ce2d69b5577314ee97504876381273bbcddca3c5 Mon Sep 17 00:00:00 2001
From: Johannes Doerfert <johannes@jdoerfert.de>
Date: Fri, 24 Jul 2020 13:10:55 -0500
Subject: [PATCH 0045/1035] [SROA][Mem2Reg] Do not crash on alloca +
 addrspacecast

SROA knows that it can look through addrspacecast but
PromoteMemoryToRegister did not handle them. This caused an assertion
error for the test case, exposed while running
`Transforms/PhaseOrdering/inlining-alignment-assumptions.ll` with D83978
applied.

Reviewed By: arsenm

Differential Revision: https://reviews.llvm.org/D84085
---
 .../Utils/PromoteMemoryToRegister.cpp         |  3 +++
 .../Transforms/Mem2Reg/alloca_addrspace.ll    | 19 +++++++++++++++++++
 .../Transforms/SROA/alloca-address-space.ll   | 13 +++++++++++++
 3 files changed, 35 insertions(+)
 create mode 100644 llvm/test/Transforms/Mem2Reg/alloca_addrspace.ll

diff --git a/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp b/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
index c7e9c919ec471..f079f81a6e8f5 100644
--- a/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
+++ b/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
@@ -95,6 +95,9 @@ bool llvm::isAllocaPromotable(const AllocaInst *AI) {
         return false;
       if (!onlyUsedByLifetimeMarkers(GEPI))
         return false;
+    } else if (const AddrSpaceCastInst *ASCI = dyn_cast<AddrSpaceCastInst>(U)) {
+      if (!onlyUsedByLifetimeMarkers(ASCI))
+        return false;
     } else {
       return false;
     }
diff --git a/llvm/test/Transforms/Mem2Reg/alloca_addrspace.ll b/llvm/test/Transforms/Mem2Reg/alloca_addrspace.ll
new file mode 100644
index 0000000000000..95db828fd117e
--- /dev/null
+++ b/llvm/test/Transforms/Mem2Reg/alloca_addrspace.ll
@@ -0,0 +1,19 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mem2reg -S | FileCheck %s
+; RUN: opt < %s -passes=mem2reg -S | FileCheck %s
+
+; Do not crash ;)
+
+target datalayout = "e-p:64:64-p5:32:32-A5"
+
+define amdgpu_kernel void @addressspace_alloca() {
+; CHECK-LABEL: @addressspace_alloca(
+; CHECK-NEXT:    ret void
+;
+  %alloca = alloca i8, align 8, addrspace(5)
+  %cast = addrspacecast i8 addrspace(5)* %alloca to i8*
+  call void @llvm.lifetime.start.p0i8(i64 2, i8* %cast)
+  ret void
+}
+
+declare void @llvm.lifetime.start.p0i8(i64 %size, i8* nocapture %ptr)
diff --git a/llvm/test/Transforms/SROA/alloca-address-space.ll b/llvm/test/Transforms/SROA/alloca-address-space.ll
index c8e22f5b25d8a..2ec59e66b9da6 100644
--- a/llvm/test/Transforms/SROA/alloca-address-space.ll
+++ b/llvm/test/Transforms/SROA/alloca-address-space.ll
@@ -1,4 +1,5 @@
 ; RUN: opt < %s -sroa -S | FileCheck %s
+; RUN: opt < %s -passes=sroa -S | FileCheck %s
 target datalayout = "e-p:64:64:64-p1:16:16:16-p2:32:32-p3:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-n8:16:32:64-A2"
 
 declare void @llvm.memcpy.p2i8.p2i8.i32(i8 addrspace(2)* nocapture, i8 addrspace(2)* nocapture readonly, i32, i1)
@@ -127,3 +128,15 @@ define void @test_load_store_diff_addr_space([2 x float] addrspace(1)* %complex1
   store i64 %v2, i64 addrspace(1)* %p2
   ret void
 }
+
+define void @addressspace_alloca_lifetime() {
+; CHECK-LABEL: @addressspace_alloca_lifetime(
+; CHECK-NEXT:    ret void
+;
+  %alloca = alloca i8, align 8, addrspace(2)
+  %cast = addrspacecast i8 addrspace(2)* %alloca to i8*
+  call void @llvm.lifetime.start.p0i8(i64 2, i8* %cast)
+  ret void
+}
+
+declare void @llvm.lifetime.start.p0i8(i64 %size, i8* nocapture %ptr)

From ce8928f2e4e5e9d192f2c603b37d32da92214ab1 Mon Sep 17 00:00:00 2001
From: Johannes Doerfert <johannes@jdoerfert.de>
Date: Fri, 24 Jul 2020 13:11:19 -0500
Subject: [PATCH 0046/1035] [Mem2Reg] Teach promote to register about droppable
 instructions

This is the first of two patches to address PR46753. We basically allow
mem2reg to promote allocas that are used in doppable instructions, for
now that means `llvm.assume`. The uses of the alloca (or a bitcast or
zero offset GEP from there) are replaced by `undef` in the droppable
instructions.

Reviewed By: Tyker

Differential Revision: https://reviews.llvm.org/D83976
---
 llvm/include/llvm/Analysis/ValueTracking.h    |  4 +
 llvm/include/llvm/IR/Value.h                  |  3 +
 llvm/lib/Analysis/ValueTracking.cpp           | 25 ++++--
 llvm/lib/IR/Value.cpp                         |  4 +
 .../Utils/PromoteMemoryToRegister.cpp         | 46 +++++++---
 .../Transforms/Mem2Reg/ignore-droppable.ll    | 85 +++++++++++++++++++
 6 files changed, 149 insertions(+), 18 deletions(-)
 create mode 100644 llvm/test/Transforms/Mem2Reg/ignore-droppable.ll

diff --git a/llvm/include/llvm/Analysis/ValueTracking.h b/llvm/include/llvm/Analysis/ValueTracking.h
index ce5aea2e8d34d..4d81bb692b47c 100644
--- a/llvm/include/llvm/Analysis/ValueTracking.h
+++ b/llvm/include/llvm/Analysis/ValueTracking.h
@@ -416,6 +416,10 @@ class Value;
   /// Return true if the only users of this pointer are lifetime markers.
   bool onlyUsedByLifetimeMarkers(const Value *V);
 
+  /// Return true if the only users of this pointer are lifetime markers or
+  /// droppable instructions.
+  bool onlyUsedByLifetimeMarkersOrDroppableInsts(const Value *V);
+
   /// Return true if speculation of the given load must be suppressed to avoid
   /// ordering or interfering with an active sanitizer.  If not suppressed,
   /// dereferenceability and alignment must be proven separately.  Note: This
diff --git a/llvm/include/llvm/IR/Value.h b/llvm/include/llvm/IR/Value.h
index 04ca682746264..fa706a1b64086 100644
--- a/llvm/include/llvm/IR/Value.h
+++ b/llvm/include/llvm/IR/Value.h
@@ -470,6 +470,9 @@ class Value {
   void dropDroppableUses(llvm::function_ref<bool(const Use *)> ShouldDrop =
                              [](const Use *) { return true; });
 
+  /// Remove every use of \p User that can safely be removed.
+  void dropDroppableUsesByUser(const User &Usr);
+
   /// Check if this value is used in the specified basic block.
   bool isUsedInBasicBlock(const BasicBlock *BB) const;
 
diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index 116916a9be2d2..271200f7030a2 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -4302,18 +4302,33 @@ bool llvm::getUnderlyingObjectsForCodeGen(const Value *V,
   return true;
 }
 
-/// Return true if the only users of this pointer are lifetime markers.
-bool llvm::onlyUsedByLifetimeMarkers(const Value *V) {
+static bool onlyUsedByLifetimeMarkersOrDroppableInstsHelper(
+    const Value *V, bool AllowLifetime, bool AllowDroppable) {
   for (const User *U : V->users()) {
     const IntrinsicInst *II = dyn_cast<IntrinsicInst>(U);
-    if (!II) return false;
-
-    if (!II->isLifetimeStartOrEnd())
+    if (!II)
       return false;
+
+    if (AllowLifetime && II->isLifetimeStartOrEnd())
+      continue;
+
+    if (AllowDroppable && II->isDroppable())
+      continue;
+
+    return false;
   }
   return true;
 }
 
+bool llvm::onlyUsedByLifetimeMarkers(const Value *V) {
+  return onlyUsedByLifetimeMarkersOrDroppableInstsHelper(
+      V, /* AllowLifetime */ true, /* AllowDroppable */ false);
+}
+bool llvm::onlyUsedByLifetimeMarkersOrDroppableInsts(const Value *V) {
+  return onlyUsedByLifetimeMarkersOrDroppableInstsHelper(
+      V, /* AllowLifetime */ true, /* AllowDroppable */ true);
+}
+
 bool llvm::mustSuppressSpeculation(const LoadInst &LI) {
   if (!LI.isUnordered())
     return true;
diff --git a/llvm/lib/IR/Value.cpp b/llvm/lib/IR/Value.cpp
index efb8d53e8964b..8c1f9c5a3b36f 100644
--- a/llvm/lib/IR/Value.cpp
+++ b/llvm/lib/IR/Value.cpp
@@ -192,6 +192,10 @@ void Value::dropDroppableUses(
   }
 }
 
+void Value::dropDroppableUsesByUser(const User &Usr) {
+  dropDroppableUses([&](const Use *U) { return U->getUser() == &Usr; });
+}
+
 bool Value::isUsedInBasicBlock(const BasicBlock *BB) const {
   // This can be computed either by scanning the instructions in BB, or by
   // scanning the use list of this Value. Both lists can be very long, but
diff --git a/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp b/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
index f079f81a6e8f5..33904e54ac237 100644
--- a/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
+++ b/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
@@ -62,10 +62,6 @@ STATISTIC(NumDeadAlloca,    "Number of dead alloca's removed");
 STATISTIC(NumPHIInsert,     "Number of PHI nodes inserted");
 
 bool llvm::isAllocaPromotable(const AllocaInst *AI) {
-  // FIXME: If the memory unit is of pointer or integer type, we can permit
-  // assignments to subsections of the memory unit.
-  unsigned AS = AI->getType()->getAddressSpace();
-
   // Only allow direct and non-volatile loads and stores...
   for (const User *U : AI->users()) {
     if (const LoadInst *LI = dyn_cast<LoadInst>(U)) {
@@ -81,19 +77,15 @@ bool llvm::isAllocaPromotable(const AllocaInst *AI) {
       if (SI->isVolatile())
         return false;
     } else if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(U)) {
-      if (!II->isLifetimeStartOrEnd())
+      if (!II->isLifetimeStartOrEnd() && !II->isDroppable())
         return false;
     } else if (const BitCastInst *BCI = dyn_cast<BitCastInst>(U)) {
-      if (BCI->getType() != Type::getInt8PtrTy(U->getContext(), AS))
-        return false;
-      if (!onlyUsedByLifetimeMarkers(BCI))
+      if (!onlyUsedByLifetimeMarkersOrDroppableInsts(BCI))
         return false;
     } else if (const GetElementPtrInst *GEPI = dyn_cast<GetElementPtrInst>(U)) {
-      if (GEPI->getType() != Type::getInt8PtrTy(U->getContext(), AS))
-        return false;
       if (!GEPI->hasAllZeroIndices())
         return false;
-      if (!onlyUsedByLifetimeMarkers(GEPI))
+      if (!onlyUsedByLifetimeMarkersOrDroppableInsts(GEPI))
         return false;
     } else if (const AddrSpaceCastInst *ASCI = dyn_cast<AddrSpaceCastInst>(U)) {
       if (!onlyUsedByLifetimeMarkers(ASCI))
@@ -315,16 +307,38 @@ static void addAssumeNonNull(AssumptionCache *AC, LoadInst *LI) {
   AC->registerAssumption(CI);
 }
 
-static void removeLifetimeIntrinsicUsers(AllocaInst *AI) {
+static void removeIntrinsicUsers(AllocaInst *AI) {
   // Knowing that this alloca is promotable, we know that it's safe to kill all
   // instructions except for load and store.
 
+  // Helper to drop the uses of \p I in \p UserI.
+  auto DropUsesIn = [](Instruction *UserI, Instruction *I,
+                       Instruction::user_iterator &UI,
+                       const Instruction::user_iterator &UE) {
+    // TODO For now we forget assumed information, this can be improved.
+    assert(isa<IntrinsicInst>(UserI) &&
+           cast<IntrinsicInst>(UserI)->getIntrinsicID() == Intrinsic::assume &&
+           "Expected assume");
+
+    // Skip ahead if User has multiple uses of I.
+    while (UI != UE && *UI == UserI)
+      ++UI;
+
+    I->dropDroppableUsesByUser(*UserI);
+  };
+
   for (auto UI = AI->user_begin(), UE = AI->user_end(); UI != UE;) {
     Instruction *I = cast<Instruction>(*UI);
     ++UI;
     if (isa<LoadInst>(I) || isa<StoreInst>(I))
       continue;
 
+    // Drop the use of AI in droppable instructions.
+    if (I->isDroppable()) {
+      DropUsesIn(I, AI, UI, UE);
+      continue;
+    }
+
     if (!I->getType()->isVoidTy()) {
       // The only users of this bitcast/GEP instruction are lifetime intrinsics.
       // Follow the use/def chain to erase them now instead of leaving it for
@@ -332,6 +346,12 @@ static void removeLifetimeIntrinsicUsers(AllocaInst *AI) {
       for (auto UUI = I->user_begin(), UUE = I->user_end(); UUI != UUE;) {
         Instruction *Inst = cast<Instruction>(*UUI);
         ++UUI;
+
+        // Drop the use of I in droppable instructions.
+        if (Inst->isDroppable()) {
+          DropUsesIn(Inst, I, UUI, UUE);
+          continue;
+        }
         Inst->eraseFromParent();
       }
     }
@@ -547,7 +567,7 @@ void PromoteMem2Reg::run() {
     assert(AI->getParent()->getParent() == &F &&
            "All allocas should be in the same function, which is same as DF!");
 
-    removeLifetimeIntrinsicUsers(AI);
+    removeIntrinsicUsers(AI);
 
     if (AI->use_empty()) {
       // If there are no uses of the alloca, just delete it now.
diff --git a/llvm/test/Transforms/Mem2Reg/ignore-droppable.ll b/llvm/test/Transforms/Mem2Reg/ignore-droppable.ll
new file mode 100644
index 0000000000000..ecad226e1d0f0
--- /dev/null
+++ b/llvm/test/Transforms/Mem2Reg/ignore-droppable.ll
@@ -0,0 +1,85 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -mem2reg -S -o - < %s | FileCheck %s
+; RUN: opt -passes=mem2reg -S -o - < %s | FileCheck %s
+
+declare void @llvm.assume(i1)
+declare void @llvm.lifetime.start.p0i8(i64 %size, i8* nocapture %ptr)
+declare void @llvm.lifetime.end.p0i8(i64 %size, i8* nocapture %ptr)
+
+define void @positive_assume_uses(i32* %arg) {
+; CHECK-LABEL: @positive_assume_uses(
+; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "nonnull"(i32* [[ARG:%.*]]), "ignore"(i32* undef, i64 2) ]
+; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "ignore"(i32* undef, i64 8), "nonnull"(i32* [[ARG]]) ]
+; CHECK-NEXT:    ret void
+;
+  %A = alloca i32
+  call void @llvm.assume(i1 true) ["nonnull"(i32* %arg), "align"(i32* %A, i64 2)]
+  store i32 1, i32* %A
+  call void @llvm.assume(i1 true) ["align"(i32* %A, i64 8), "nonnull"(i32* %arg)]
+  ret void
+}
+
+define void @negative_assume_condition_use() {
+; CHECK-LABEL: @negative_assume_condition_use(
+; CHECK-NEXT:    [[A:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[B:%.*]] = bitcast i32* [[A]] to i8*
+; CHECK-NEXT:    [[CND:%.*]] = icmp eq i8* [[B]], null
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CND]])
+; CHECK-NEXT:    store i32 1, i32* [[A]], align 4
+; CHECK-NEXT:    ret void
+;
+  %A = alloca i32
+  %B = bitcast i32* %A to i8*
+  %cnd = icmp eq i8* %B, null
+  call void @llvm.assume(i1 %cnd)
+  store i32 1, i32* %A
+  ret void
+}
+
+define void @positive_multiple_assume_uses() {
+; CHECK-LABEL: @positive_multiple_assume_uses(
+; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "ignore"({ i8, i16 }* undef, i64 8), "ignore"({ i8, i16 }* undef, i64 16) ]
+; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "ignore"({ i8, i16 }* undef), "ignore"({ i8, i16 }* undef, i64 2) ]
+; CHECK-NEXT:    ret void
+;
+  %A = alloca {i8, i16}
+  call void @llvm.assume(i1 true) ["align"({i8, i16}* %A, i64 8), "align"({i8, i16}* %A, i64 16)]
+  store {i8, i16} zeroinitializer, {i8, i16}* %A
+  call void @llvm.assume(i1 true) ["nonnull"({i8, i16}* %A), "align"({i8, i16}* %A, i64 2)]
+  ret void
+}
+
+define void @positive_gep_assume_uses() {
+; CHECK-LABEL: @positive_gep_assume_uses(
+; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "ignore"(i8* undef, i64 8), "ignore"(i8* undef, i64 16) ]
+; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "ignore"(i8* undef), "ignore"(i8* undef, i64 2) ]
+; CHECK-NEXT:    ret void
+;
+  %A = alloca {i8, i16}
+  %B = getelementptr {i8, i16}, {i8, i16}* %A, i32 0, i32 0
+  call void @llvm.lifetime.start.p0i8(i64 2, i8* %B)
+  call void @llvm.assume(i1 true) ["align"(i8* %B, i64 8), "align"(i8* %B, i64 16)]
+  store {i8, i16} zeroinitializer, {i8, i16}* %A
+  call void @llvm.lifetime.end.p0i8(i64 2, i8* %B)
+  call void @llvm.assume(i1 true) ["nonnull"(i8* %B), "align"(i8* %B, i64 2)]
+  ret void
+}
+
+define void @positive_mixed_assume_uses() {
+; CHECK-LABEL: @positive_mixed_assume_uses(
+; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "ignore"(i8* undef), "ignore"(i8* undef, i64 8), "ignore"(i8* undef, i64 16) ]
+; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "ignore"(i8* undef), "ignore"(i8* undef, i64 2), "ignore"(i8* undef) ]
+; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "ignore"(i32* undef), "ignore"(i32* undef, i64 2), "ignore"(i8* undef) ]
+; CHECK-NEXT:    ret void
+;
+  %A = alloca i8
+  %B = getelementptr i8, i8* %A, i32 0
+  %C = bitcast i8* %A to i32*
+  call void @llvm.lifetime.start.p0i8(i64 2, i8* %B)
+  call void @llvm.assume(i1 true) ["nonnull"(i8* %B), "align"(i8* %A, i64 8), "align"(i8* %B, i64 16)]
+  store i8 1, i8* %A
+  call void @llvm.lifetime.end.p0i8(i64 2, i8* %B)
+  call void @llvm.assume(i1 true) ["nonnull"(i8* %B), "align"(i8* %A, i64 2), "nonnull"(i8* %A)]
+  call void @llvm.assume(i1 true) ["nonnull"(i32* %C), "align"(i32* %C, i64 2), "nonnull"(i8* %A)]
+  ret void
+}

From aa09db495a9b40de456e196c0deb6b98324779b9 Mon Sep 17 00:00:00 2001
From: Johannes Doerfert <johannes@jdoerfert.de>
Date: Fri, 24 Jul 2020 14:06:27 -0500
Subject: [PATCH 0047/1035] [SROA] Teach promote to register about droppable
 instructions

This is the second of two patches to address PR46753. We basically allow
SROA to promote allocas that are used in doppable instructions, for
now that means `llvm.assume`. The (transitive) uses are replaced by
`undef` in the droppable instructions.

See also D83976.

Reviewed By: Tyker

Differential Revision: https://reviews.llvm.org/D83978
---
 llvm/lib/Transforms/Scalar/SROA.cpp           | 21 +++--
 llvm/test/Transforms/SROA/ignore-droppable.ll | 88 +++++++++++++++++++
 2 files changed, 104 insertions(+), 5 deletions(-)
 create mode 100644 llvm/test/Transforms/SROA/ignore-droppable.ll

diff --git a/llvm/lib/Transforms/Scalar/SROA.cpp b/llvm/lib/Transforms/Scalar/SROA.cpp
index 89f324deef9fd..8eb2853347bdb 100644
--- a/llvm/lib/Transforms/Scalar/SROA.cpp
+++ b/llvm/lib/Transforms/Scalar/SROA.cpp
@@ -920,6 +920,9 @@ class AllocaSlices::SliceBuilder : public PtrUseVisitor<SliceBuilder> {
   // FIXME: What about debug intrinsics? This matches old behavior, but
   // doesn't make sense.
   void visitIntrinsicInst(IntrinsicInst &II) {
+    if (II.isDroppable())
+      return;
+
     if (!IsOffsetKnown)
       return PI.setAborted(&II);
 
@@ -1825,7 +1828,7 @@ static bool isVectorPromotionViableForSlice(Partition &P, const Slice &S,
     if (!S.isSplittable())
       return false; // Skip any unsplittable intrinsics.
   } else if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(U->getUser())) {
-    if (!II->isLifetimeStartOrEnd())
+    if (!II->isLifetimeStartOrEnd() && !II->isDroppable())
       return false;
   } else if (U->get()->getType()->getPointerElementType()->isStructTy()) {
     // Disable vector promotion when there are loads or stores of an FCA.
@@ -2058,7 +2061,7 @@ static bool isIntegerWideningViableForSlice(const Slice &S,
     if (!S.isSplittable())
       return false; // Skip any unsplittable intrinsics.
   } else if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(U->getUser())) {
-    if (!II->isLifetimeStartOrEnd())
+    if (!II->isLifetimeStartOrEnd() && !II->isDroppable())
       return false;
   } else {
     return false;
@@ -2778,7 +2781,7 @@ class llvm::sroa::AllocaSliceRewriter
 
     Type *AllocaTy = NewAI.getAllocatedType();
     Type *ScalarTy = AllocaTy->getScalarType();
-    
+
     const bool CanContinue = [&]() {
       if (VecTy || IntTy)
         return true;
@@ -3074,13 +3077,21 @@ class llvm::sroa::AllocaSliceRewriter
   }
 
   bool visitIntrinsicInst(IntrinsicInst &II) {
-    assert(II.isLifetimeStartOrEnd());
+    assert((II.isLifetimeStartOrEnd() || II.isDroppable()) &&
+           "Unexpected intrinsic!");
     LLVM_DEBUG(dbgs() << "    original: " << II << "\n");
-    assert(II.getArgOperand(1) == OldPtr);
 
     // Record this instruction for deletion.
     Pass.DeadInsts.insert(&II);
 
+    if (II.isDroppable()) {
+      assert(II.getIntrinsicID() == Intrinsic::assume && "Expected assume");
+      // TODO For now we forget assumed information, this can be improved.
+      OldPtr->dropDroppableUsesByUser(II);
+      return true;
+    }
+
+    assert(II.getArgOperand(1) == OldPtr);
     // Lifetime intrinsics are only promotable if they cover the whole alloca.
     // Therefore, we drop lifetime intrinsics which don't cover the whole
     // alloca.
diff --git a/llvm/test/Transforms/SROA/ignore-droppable.ll b/llvm/test/Transforms/SROA/ignore-droppable.ll
new file mode 100644
index 0000000000000..85f3d8a5f39f4
--- /dev/null
+++ b/llvm/test/Transforms/SROA/ignore-droppable.ll
@@ -0,0 +1,88 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -sroa -S -o - < %s | FileCheck %s
+; RUN: opt -passes=sroa -S -o - < %s | FileCheck %s
+
+declare void @llvm.assume(i1)
+declare void @llvm.lifetime.start.p0i8(i64 %size, i8* nocapture %ptr)
+declare void @llvm.lifetime.end.p0i8(i64 %size, i8* nocapture %ptr)
+
+define void @positive_assume_uses(i32* %arg) {
+; CHECK-LABEL: @positive_assume_uses(
+; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "nonnull"(i32* [[ARG:%.*]]), "ignore"(i32* undef, i64 2) ]
+; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "ignore"(i32* undef, i64 8), "nonnull"(i32* [[ARG]]) ]
+; CHECK-NEXT:    ret void
+;
+  %A = alloca i32
+  call void @llvm.assume(i1 true) ["nonnull"(i32* %arg), "align"(i32* %A, i64 2)]
+  store i32 1, i32* %A
+  call void @llvm.assume(i1 true) ["align"(i32* %A, i64 8), "nonnull"(i32* %arg)]
+  ret void
+}
+
+define void @negative_assume_condition_use() {
+; CHECK-LABEL: @negative_assume_condition_use(
+; CHECK-NEXT:    [[A:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[B:%.*]] = bitcast i32* [[A]] to i8*
+; CHECK-NEXT:    [[CND:%.*]] = icmp eq i8* [[B]], null
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CND]])
+; CHECK-NEXT:    store i32 1, i32* [[A]], align 4
+; CHECK-NEXT:    ret void
+;
+  %A = alloca i32
+  %B = bitcast i32* %A to i8*
+  %cnd = icmp eq i8* %B, null
+  call void @llvm.assume(i1 %cnd)
+  store i32 1, i32* %A
+  ret void
+}
+
+define void @positive_multiple_assume_uses() {
+; CHECK-LABEL: @positive_multiple_assume_uses(
+; CHECK-NEXT:    [[A:%.*]] = alloca { i8, i16 }, align 8
+; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "align"({ i8, i16 }* [[A]], i64 8), "align"({ i8, i16 }* [[A]], i64 16) ]
+; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "nonnull"({ i8, i16 }* [[A]]), "align"({ i8, i16 }* [[A]], i64 2) ]
+; CHECK-NEXT:    ret void
+;
+  %A = alloca {i8, i16}
+  call void @llvm.assume(i1 true) ["align"({i8, i16}* %A, i64 8), "align"({i8, i16}* %A, i64 16)]
+  store {i8, i16} zeroinitializer, {i8, i16}* %A
+  call void @llvm.assume(i1 true) ["nonnull"({i8, i16}* %A), "align"({i8, i16}* %A, i64 2)]
+  ret void
+}
+
+define void @positive_gep_assume_uses() {
+; CHECK-LABEL: @positive_gep_assume_uses(
+; CHECK-NEXT:    [[A:%.*]] = alloca { i8, i16 }, align 8
+; CHECK-NEXT:    [[B:%.*]] = getelementptr { i8, i16 }, { i8, i16 }* [[A]], i32 0, i32 0
+; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "align"(i8* [[B]], i64 8), "align"(i8* [[B]], i64 16) ]
+; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "nonnull"(i8* [[B]]), "align"(i8* [[B]], i64 2) ]
+; CHECK-NEXT:    ret void
+;
+  %A = alloca {i8, i16}
+  %B = getelementptr {i8, i16}, {i8, i16}* %A, i32 0, i32 0
+  call void @llvm.lifetime.start.p0i8(i64 2, i8* %B)
+  call void @llvm.assume(i1 true) ["align"(i8* %B, i64 8), "align"(i8* %B, i64 16)]
+  store {i8, i16} zeroinitializer, {i8, i16}* %A
+  call void @llvm.lifetime.end.p0i8(i64 2, i8* %B)
+  call void @llvm.assume(i1 true) ["nonnull"(i8* %B), "align"(i8* %B, i64 2)]
+  ret void
+}
+
+define void @positive_mixed_assume_uses() {
+; CHECK-LABEL: @positive_mixed_assume_uses(
+; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "ignore"(i8* undef), "ignore"(i8* undef, i64 8), "ignore"(i8* undef, i64 16) ]
+; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "ignore"(i8* undef), "ignore"(i8* undef, i64 2), "ignore"(i8* undef) ]
+; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "ignore"(i32* undef), "ignore"(i32* undef, i64 2), "ignore"(i8* undef) ]
+; CHECK-NEXT:    ret void
+;
+  %A = alloca i8
+  %B = getelementptr i8, i8* %A, i32 0
+  %C = bitcast i8* %A to i32*
+  call void @llvm.lifetime.start.p0i8(i64 2, i8* %B)
+  call void @llvm.assume(i1 true) ["nonnull"(i8* %B), "align"(i8* %A, i64 8), "align"(i8* %B, i64 16)]
+  store i8 1, i8* %A
+  call void @llvm.lifetime.end.p0i8(i64 2, i8* %B)
+  call void @llvm.assume(i1 true) ["nonnull"(i8* %B), "align"(i8* %A, i64 2), "nonnull"(i8* %A)]
+  call void @llvm.assume(i1 true) ["nonnull"(i32* %C), "align"(i32* %C, i64 2), "nonnull"(i8* %A)]
+  ret void
+}

From c02aa53ecb25189bfdecd852a251e1c17ed0ee24 Mon Sep 17 00:00:00 2001
From: Eli Friedman <efriedma@quicinc.com>
Date: Thu, 23 Jul 2020 12:52:46 -0700
Subject: [PATCH 0048/1035] [AArch64][SVE] Add "fast" fcmp operations.

dacf8d3 added support for most fcmp operations, but there are some extra
variations I hadn't considered: SelectionDAG supports float comparisons
that are neither ordered nor unordered. Add support for the missing
operations.

Differential Revision: https://reviews.llvm.org/D84460
---
 .../Target/AArch64/AArch64ISelLowering.cpp    |  2 +
 .../lib/Target/AArch64/AArch64SVEInstrInfo.td | 21 +++++--
 llvm/test/CodeGen/AArch64/sve-fcmp.ll         | 55 +++++++++++++++++++
 3 files changed, 74 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index cb5530077fdd7..f080abd8e6271 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -230,7 +230,9 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
            MVT::nxv2f64 }) {
       setCondCodeAction(ISD::SETO, VT, Expand);
       setCondCodeAction(ISD::SETOLT, VT, Expand);
+      setCondCodeAction(ISD::SETLT, VT, Expand);
       setCondCodeAction(ISD::SETOLE, VT, Expand);
+      setCondCodeAction(ISD::SETLE, VT, Expand);
       setCondCodeAction(ISD::SETULT, VT, Expand);
       setCondCodeAction(ISD::SETULE, VT, Expand);
       setCondCodeAction(ISD::SETUGE, VT, Expand);
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index dc501a9536b9a..7c39268a4441f 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -210,6 +210,19 @@ def index_vector : SDNode<"AArch64ISD::INDEX_VECTOR", SDT_IndexVector, []>;
 
 def reinterpret_cast : SDNode<"AArch64ISD::REINTERPRET_CAST", SDTUnaryOp>;
 
+def setoge_or_setge : PatFrags<(ops node:$lhs, node:$rhs),
+                               [(setoge node:$lhs, node:$rhs),
+                                (setge node:$lhs, node:$rhs)]>;
+def setogt_or_setgt : PatFrags<(ops node:$lhs, node:$rhs),
+                                [(setogt node:$lhs, node:$rhs),
+                                 (setgt node:$lhs, node:$rhs)]>;
+def setoeq_or_seteq : PatFrags<(ops node:$lhs, node:$rhs),
+                                [(setoeq node:$lhs, node:$rhs),
+                                 (seteq node:$lhs, node:$rhs)]>;
+def setone_or_setne : PatFrags<(ops node:$lhs, node:$rhs),
+                                [(setone node:$lhs, node:$rhs),
+                                 (setne node:$lhs, node:$rhs)]>;
+
 let Predicates = [HasSVE] in {
   defm RDFFR_PPz  : sve_int_rdffr_pred<0b0, "rdffr", int_aarch64_sve_rdffr_z>;
   def  RDFFRS_PPz : sve_int_rdffr_pred<0b1, "rdffrs">;
@@ -1172,10 +1185,10 @@ multiclass sve_prefetch<SDPatternOperator prefetch, ValueType PredTy, Instructio
   defm CMPLO_PPzZI : sve_int_ucmp_vi<0b10, "cmplo", SETULT, SETUGT>;
   defm CMPLS_PPzZI : sve_int_ucmp_vi<0b11, "cmpls", SETULE, SETUGE>;
 
-  defm FCMGE_PPzZZ : sve_fp_3op_p_pd_cc<0b000, "fcmge", int_aarch64_sve_fcmpge, setoge>;
-  defm FCMGT_PPzZZ : sve_fp_3op_p_pd_cc<0b001, "fcmgt", int_aarch64_sve_fcmpgt, setogt>;
-  defm FCMEQ_PPzZZ : sve_fp_3op_p_pd_cc<0b010, "fcmeq", int_aarch64_sve_fcmpeq, setoeq>;
-  defm FCMNE_PPzZZ : sve_fp_3op_p_pd_cc<0b011, "fcmne", int_aarch64_sve_fcmpne, setone>;
+  defm FCMGE_PPzZZ : sve_fp_3op_p_pd_cc<0b000, "fcmge", int_aarch64_sve_fcmpge, setoge_or_setge>;
+  defm FCMGT_PPzZZ : sve_fp_3op_p_pd_cc<0b001, "fcmgt", int_aarch64_sve_fcmpgt, setogt_or_setgt>;
+  defm FCMEQ_PPzZZ : sve_fp_3op_p_pd_cc<0b010, "fcmeq", int_aarch64_sve_fcmpeq, setoeq_or_seteq>;
+  defm FCMNE_PPzZZ : sve_fp_3op_p_pd_cc<0b011, "fcmne", int_aarch64_sve_fcmpne, setone_or_setne>;
   defm FCMUO_PPzZZ : sve_fp_3op_p_pd_cc<0b100, "fcmuo", int_aarch64_sve_fcmpuo, setuo>;
   defm FACGE_PPzZZ : sve_fp_3op_p_pd<0b101, "facge", int_aarch64_sve_facge>;
   defm FACGT_PPzZZ : sve_fp_3op_p_pd<0b111, "facgt", int_aarch64_sve_facgt>;
diff --git a/llvm/test/CodeGen/AArch64/sve-fcmp.ll b/llvm/test/CodeGen/AArch64/sve-fcmp.ll
index 86fff734f1883..3bb3627e23939 100644
--- a/llvm/test/CodeGen/AArch64/sve-fcmp.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fcmp.ll
@@ -257,3 +257,58 @@ define <vscale x 4 x i32> @oeq_4f32_zext(<vscale x 4 x float> %x, <vscale x 4 x
   %r = zext <vscale x 4 x i1> %y to <vscale x 4 x i32>
   ret <vscale x 4 x i32> %r
 }
+
+define <vscale x 4 x i1> @eq_fast(<vscale x 4 x float> %x, <vscale x 4 x float> %x2) {
+; CHECK-LABEL: eq_fast:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    fcmeq p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT:    ret
+  %y = fcmp fast oeq <vscale x 4 x float> %x, %x2
+  ret <vscale x 4 x i1> %y
+}
+define <vscale x 4 x i1> @gt_fast(<vscale x 4 x float> %x, <vscale x 4 x float> %x2) {
+; CHECK-LABEL: gt_fast:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    fcmgt p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT:    ret
+  %y = fcmp fast ogt <vscale x 4 x float> %x, %x2
+  ret <vscale x 4 x i1> %y
+}
+define <vscale x 4 x i1> @ge_fast(<vscale x 4 x float> %x, <vscale x 4 x float> %x2) {
+; CHECK-LABEL: ge_fast:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    fcmge p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT:    ret
+  %y = fcmp fast oge <vscale x 4 x float> %x, %x2
+  ret <vscale x 4 x i1> %y
+}
+define <vscale x 4 x i1> @lt_fast(<vscale x 4 x float> %x, <vscale x 4 x float> %x2) {
+; CHECK-LABEL: lt_fast:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    fcmgt p0.s, p0/z, z1.s, z0.s
+; CHECK-NEXT:    ret
+  %y = fcmp fast olt <vscale x 4 x float> %x, %x2
+  ret <vscale x 4 x i1> %y
+}
+define <vscale x 4 x i1> @le_fast(<vscale x 4 x float> %x, <vscale x 4 x float> %x2) {
+; CHECK-LABEL: le_fast:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    fcmge p0.s, p0/z, z1.s, z0.s
+; CHECK-NEXT:    ret
+  %y = fcmp fast ole <vscale x 4 x float> %x, %x2
+  ret <vscale x 4 x i1> %y
+}
+define <vscale x 4 x i1> @ne_fast(<vscale x 4 x float> %x, <vscale x 4 x float> %x2) {
+; CHECK-LABEL: ne_fast:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    fcmne p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT:    ret
+  %y = fcmp fast one <vscale x 4 x float> %x, %x2
+  ret <vscale x 4 x i1> %y
+}

From 986e3af53bfe591e88a1ae4f82ea1cc0a15819a3 Mon Sep 17 00:00:00 2001
From: Yifan Shen <aelitashen@fb.com>
Date: Fri, 24 Jul 2020 12:45:41 -0700
Subject: [PATCH 0049/1035] Add Debug Info Size to Symbol Status

Summary: If a module has debug info, the size of debug symbol will be displayed after the Symbols Loaded Message for each module in the VScode modules view.{F12335461}

Reviewers: wallace, clayborg

Reviewed By: wallace, clayborg

Subscribers: cfe-commits, aprantl, lldb-commits

Tags: #lldb, #clang

Differential Revision: https://reviews.llvm.org/D83731
---
 clang/tools/clang-format/git-clang-format     | 585 ------------------
 .../tools/lldb-vscode/lldbvscode_testcase.py  |   9 +-
 .../API/tools/lldb-vscode/module/Makefile     |  10 +-
 .../lldb-vscode/module/TestVSCode_module.py   |  67 +-
 lldb/tools/lldb-vscode/JSONUtils.cpp          |  61 +-
 5 files changed, 121 insertions(+), 611 deletions(-)

diff --git a/clang/tools/clang-format/git-clang-format b/clang/tools/clang-format/git-clang-format
index f3cd585e7f4a0..e69de29bb2d1d 100755
--- a/clang/tools/clang-format/git-clang-format
+++ b/clang/tools/clang-format/git-clang-format
@@ -1,585 +0,0 @@
-#!/usr/bin/env python
-#
-#===- git-clang-format - ClangFormat Git Integration ---------*- python -*--===#
-#
-# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-# See https://llvm.org/LICENSE.txt for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#
-#===------------------------------------------------------------------------===#
-
-r"""
-clang-format git integration
-============================
-
-This file provides a clang-format integration for git. Put it somewhere in your
-path and ensure that it is executable. Then, "git clang-format" will invoke
-clang-format on the changes in current files or a specific commit.
-
-For further details, run:
-git clang-format -h
-
-Requires Python 2.7 or Python 3
-"""
-
-from __future__ import absolute_import, division, print_function
-import argparse
-import collections
-import contextlib
-import errno
-import os
-import re
-import subprocess
-import sys
-
-usage = 'git clang-format [OPTIONS] [<commit>] [<commit>] [--] [<file>...]'
-
-desc = '''
-If zero or one commits are given, run clang-format on all lines that differ
-between the working directory and <commit>, which defaults to HEAD.  Changes are
-only applied to the working directory.
-
-If two commits are given (requires --diff), run clang-format on all lines in the
-second <commit> that differ from the first <commit>.
-
-The following git-config settings set the default of the corresponding option:
-  clangFormat.binary
-  clangFormat.commit
-  clangFormat.extensions
-  clangFormat.style
-'''
-
-# Name of the temporary index file in which save the output of clang-format.
-# This file is created within the .git directory.
-temp_index_basename = 'clang-format-index'
-
-
-Range = collections.namedtuple('Range', 'start, count')
-
-
-def main():
-  config = load_git_config()
-
-  # In order to keep '--' yet allow options after positionals, we need to
-  # check for '--' ourselves.  (Setting nargs='*' throws away the '--', while
-  # nargs=argparse.REMAINDER disallows options after positionals.)
-  argv = sys.argv[1:]
-  try:
-    idx = argv.index('--')
-  except ValueError:
-    dash_dash = []
-  else:
-    dash_dash = argv[idx:]
-    argv = argv[:idx]
-
-  default_extensions = ','.join([
-      # From clang/lib/Frontend/FrontendOptions.cpp, all lower case
-      'c', 'h',  # C
-      'm',  # ObjC
-      'mm',  # ObjC++
-      'cc', 'cp', 'cpp', 'c++', 'cxx', 'hh', 'hpp', 'hxx',  # C++
-      'cu',  # CUDA
-      # Other languages that clang-format supports
-      'proto', 'protodevel',  # Protocol Buffers
-      'java',  # Java
-      'js',  # JavaScript
-      'ts',  # TypeScript
-      'cs',  # C Sharp
-      ])
-
-  p = argparse.ArgumentParser(
-    usage=usage, formatter_class=argparse.RawDescriptionHelpFormatter,
-    description=desc)
-  p.add_argument('--binary',
-                 default=config.get('clangformat.binary', 'clang-format'),
-                 help='path to clang-format'),
-  p.add_argument('--commit',
-                 default=config.get('clangformat.commit', 'HEAD'),
-                 help='default commit to use if none is specified'),
-  p.add_argument('--diff', action='store_true',
-                 help='print a diff instead of applying the changes')
-  p.add_argument('--extensions',
-                 default=config.get('clangformat.extensions',
-                                    default_extensions),
-                 help=('comma-separated list of file extensions to format, '
-                       'excluding the period and case-insensitive')),
-  p.add_argument('-f', '--force', action='store_true',
-                 help='allow changes to unstaged files')
-  p.add_argument('-p', '--patch', action='store_true',
-                 help='select hunks interactively')
-  p.add_argument('-q', '--quiet', action='count', default=0,
-                 help='print less information')
-  p.add_argument('--style',
-                 default=config.get('clangformat.style', None),
-                 help='passed to clang-format'),
-  p.add_argument('-v', '--verbose', action='count', default=0,
-                 help='print extra information')
-  # We gather all the remaining positional arguments into 'args' since we need
-  # to use some heuristics to determine whether or not <commit> was present.
-  # However, to print pretty messages, we make use of metavar and help.
-  p.add_argument('args', nargs='*', metavar='<commit>',
-                 help='revision from which to compute the diff')
-  p.add_argument('ignored', nargs='*', metavar='<file>...',
-                 help='if specified, only consider differences in these files')
-  opts = p.parse_args(argv)
-
-  opts.verbose -= opts.quiet
-  del opts.quiet
-
-  commits, files = interpret_args(opts.args, dash_dash, opts.commit)
-  if len(commits) > 1:
-    if not opts.diff:
-      die('--diff is required when two commits are given')
-  else:
-    if len(commits) > 2:
-      die('at most two commits allowed; %d given' % len(commits))
-  changed_lines = compute_diff_and_extract_lines(commits, files)
-  if opts.verbose >= 1:
-    ignored_files = set(changed_lines)
-  filter_by_extension(changed_lines, opts.extensions.lower().split(','))
-  if opts.verbose >= 1:
-    ignored_files.difference_update(changed_lines)
-    if ignored_files:
-      print('Ignoring changes in the following files (wrong extension):')
-      for filename in ignored_files:
-        print('    %s' % filename)
-    if changed_lines:
-      print('Running clang-format on the following files:')
-      for filename in changed_lines:
-        print('    %s' % filename)
-  if not changed_lines:
-    print('no modified files to format')
-    return
-  # The computed diff outputs absolute paths, so we must cd before accessing
-  # those files.
-  cd_to_toplevel()
-  if len(commits) > 1:
-    old_tree = commits[1]
-    new_tree = run_clang_format_and_save_to_tree(changed_lines,
-                                                 revision=commits[1],
-                                                 binary=opts.binary,
-                                                 style=opts.style)
-  else:
-    old_tree = create_tree_from_workdir(changed_lines)
-    new_tree = run_clang_format_and_save_to_tree(changed_lines,
-                                                 binary=opts.binary,
-                                                 style=opts.style)
-  if opts.verbose >= 1:
-    print('old tree: %s' % old_tree)
-    print('new tree: %s' % new_tree)
-  if old_tree == new_tree:
-    if opts.verbose >= 0:
-      print('clang-format did not modify any files')
-  elif opts.diff:
-    print_diff(old_tree, new_tree)
-  else:
-    changed_files = apply_changes(old_tree, new_tree, force=opts.force,
-                                  patch_mode=opts.patch)
-    if (opts.verbose >= 0 and not opts.patch) or opts.verbose >= 1:
-      print('changed files:')
-      for filename in changed_files:
-        print('    %s' % filename)
-
-
-def load_git_config(non_string_options=None):
-  """Return the git configuration as a dictionary.
-
-  All options are assumed to be strings unless in `non_string_options`, in which
-  is a dictionary mapping option name (in lower case) to either "--bool" or
-  "--int"."""
-  if non_string_options is None:
-    non_string_options = {}
-  out = {}
-  for entry in run('git', 'config', '--list', '--null').split('\0'):
-    if entry:
-      if '\n' in entry:
-        name, value = entry.split('\n', 1)
-      else:
-        # A setting with no '=' ('\n' with --null) is implicitly 'true'
-        name = entry
-        value = 'true'
-      if name in non_string_options:
-        value = run('git', 'config', non_string_options[name], name)
-      out[name] = value
-  return out
-
-
-def interpret_args(args, dash_dash, default_commit):
-  """Interpret `args` as "[commits] [--] [files]" and return (commits, files).
-
-  It is assumed that "--" and everything that follows has been removed from
-  args and placed in `dash_dash`.
-
-  If "--" is present (i.e., `dash_dash` is non-empty), the arguments to its
-  left (if present) are taken as commits.  Otherwise, the arguments are checked
-  from left to right if they are commits or files.  If commits are not given,
-  a list with `default_commit` is used."""
-  if dash_dash:
-    if len(args) == 0:
-      commits = [default_commit]
-    else:
-      commits = args
-    for commit in commits:
-      object_type = get_object_type(commit)
-      if object_type not in ('commit', 'tag'):
-        if object_type is None:
-          die("'%s' is not a commit" % commit)
-        else:
-          die("'%s' is a %s, but a commit was expected" % (commit, object_type))
-    files = dash_dash[1:]
-  elif args:
-    commits = []
-    while args:
-      if not disambiguate_revision(args[0]):
-        break
-      commits.append(args.pop(0))
-    if not commits:
-      commits = [default_commit]
-    files = args
-  else:
-    commits = [default_commit]
-    files = []
-  return commits, files
-
-
-def disambiguate_revision(value):
-  """Returns True if `value` is a revision, False if it is a file, or dies."""
-  # If `value` is ambiguous (neither a commit nor a file), the following
-  # command will die with an appropriate error message.
-  run('git', 'rev-parse', value, verbose=False)
-  object_type = get_object_type(value)
-  if object_type is None:
-    return False
-  if object_type in ('commit', 'tag'):
-    return True
-  die('`%s` is a %s, but a commit or filename was expected' %
-      (value, object_type))
-
-
-def get_object_type(value):
-  """Returns a string description of an object's type, or None if it is not
-  a valid git object."""
-  cmd = ['git', 'cat-file', '-t', value]
-  p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-  stdout, stderr = p.communicate()
-  if p.returncode != 0:
-    return None
-  return convert_string(stdout.strip())
-
-
-def compute_diff_and_extract_lines(commits, files):
-  """Calls compute_diff() followed by extract_lines()."""
-  diff_process = compute_diff(commits, files)
-  changed_lines = extract_lines(diff_process.stdout)
-  diff_process.stdout.close()
-  diff_process.wait()
-  if diff_process.returncode != 0:
-    # Assume error was already printed to stderr.
-    sys.exit(2)
-  return changed_lines
-
-
-def compute_diff(commits, files):
-  """Return a subprocess object producing the diff from `commits`.
-
-  The return value's `stdin` file object will produce a patch with the
-  differences between the working directory and the first commit if a single
-  one was specified, or the difference between both specified commits, filtered
-  on `files` (if non-empty).  Zero context lines are used in the patch."""
-  git_tool = 'diff-index'
-  if len(commits) > 1:
-    git_tool = 'diff-tree'
-  cmd = ['git', git_tool, '-p', '-U0'] + commits + ['--']
-  cmd.extend(files)
-  p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
-  p.stdin.close()
-  return p
-
-
-def extract_lines(patch_file):
-  """Extract the changed lines in `patch_file`.
-
-  The return value is a dictionary mapping filename to a list of (start_line,
-  line_count) pairs.
-
-  The input must have been produced with ``-U0``, meaning unidiff format with
-  zero lines of context.  The return value is a dict mapping filename to a
-  list of line `Range`s."""
-  matches = {}
-  for line in patch_file:
-    line = convert_string(line)
-    match = re.search(r'^\+\+\+\ [^/]+/(.*)', line)
-    if match:
-      filename = match.group(1).rstrip('\r\n')
-    match = re.search(r'^@@ -[0-9,]+ \+(\d+)(,(\d+))?', line)
-    if match:
-      start_line = int(match.group(1))
-      line_count = 1
-      if match.group(3):
-        line_count = int(match.group(3))
-      if line_count > 0:
-        matches.setdefault(filename, []).append(Range(start_line, line_count))
-  return matches
-
-
-def filter_by_extension(dictionary, allowed_extensions):
-  """Delete every key in `dictionary` that doesn't have an allowed extension.
-
-  `allowed_extensions` must be a collection of lowercase file extensions,
-  excluding the period."""
-  allowed_extensions = frozenset(allowed_extensions)
-  for filename in list(dictionary.keys()):
-    base_ext = filename.rsplit('.', 1)
-    if len(base_ext) == 1 and '' in allowed_extensions:
-        continue
-    if len(base_ext) == 1 or base_ext[1].lower() not in allowed_extensions:
-      del dictionary[filename]
-
-
-def cd_to_toplevel():
-  """Change to the top level of the git repository."""
-  toplevel = run('git', 'rev-parse', '--show-toplevel')
-  os.chdir(toplevel)
-
-
-def create_tree_from_workdir(filenames):
-  """Create a new git tree with the given files from the working directory.
-
-  Returns the object ID (SHA-1) of the created tree."""
-  return create_tree(filenames, '--stdin')
-
-
-def run_clang_format_and_save_to_tree(changed_lines, revision=None,
-                                      binary='clang-format', style=None):
-  """Run clang-format on each file and save the result to a git tree.
-
-  Returns the object ID (SHA-1) of the created tree."""
-  def iteritems(container):
-      try:
-          return container.iteritems() # Python 2
-      except AttributeError:
-          return container.items() # Python 3
-  def index_info_generator():
-    for filename, line_ranges in iteritems(changed_lines):
-      if revision:
-        git_metadata_cmd = ['git', 'ls-tree',
-                            '%s:%s' % (revision, os.path.dirname(filename)),
-                            os.path.basename(filename)]
-        git_metadata = subprocess.Popen(git_metadata_cmd, stdin=subprocess.PIPE,
-                                        stdout=subprocess.PIPE)
-        stdout = git_metadata.communicate()[0]
-        mode = oct(int(stdout.split()[0], 8))
-      else:
-        mode = oct(os.stat(filename).st_mode)
-      # Adjust python3 octal format so that it matches what git expects
-      if mode.startswith('0o'):
-          mode = '0' + mode[2:]
-      blob_id = clang_format_to_blob(filename, line_ranges,
-                                     revision=revision,
-                                     binary=binary,
-                                     style=style)
-      yield '%s %s\t%s' % (mode, blob_id, filename)
-  return create_tree(index_info_generator(), '--index-info')
-
-
-def create_tree(input_lines, mode):
-  """Create a tree object from the given input.
-
-  If mode is '--stdin', it must be a list of filenames.  If mode is
-  '--index-info' is must be a list of values suitable for "git update-index
-  --index-info", such as "<mode> <SP> <sha1> <TAB> <filename>".  Any other mode
-  is invalid."""
-  assert mode in ('--stdin', '--index-info')
-  cmd = ['git', 'update-index', '--add', '-z', mode]
-  with temporary_index_file():
-    p = subprocess.Popen(cmd, stdin=subprocess.PIPE)
-    for line in input_lines:
-      p.stdin.write(to_bytes('%s\0' % line))
-    p.stdin.close()
-    if p.wait() != 0:
-      die('`%s` failed' % ' '.join(cmd))
-    tree_id = run('git', 'write-tree')
-    return tree_id
-
-
-def clang_format_to_blob(filename, line_ranges, revision=None,
-                         binary='clang-format', style=None):
-  """Run clang-format on the given file and save the result to a git blob.
-
-  Runs on the file in `revision` if not None, or on the file in the working
-  directory if `revision` is None.
-
-  Returns the object ID (SHA-1) of the created blob."""
-  clang_format_cmd = [binary]
-  if style:
-    clang_format_cmd.extend(['-style='+style])
-  clang_format_cmd.extend([
-      '-lines=%s:%s' % (start_line, start_line+line_count-1)
-      for start_line, line_count in line_ranges])
-  if revision:
-    clang_format_cmd.extend(['-assume-filename='+filename])
-    git_show_cmd = ['git', 'cat-file', 'blob', '%s:%s' % (revision, filename)]
-    git_show = subprocess.Popen(git_show_cmd, stdin=subprocess.PIPE,
-                                stdout=subprocess.PIPE)
-    git_show.stdin.close()
-    clang_format_stdin = git_show.stdout
-  else:
-    clang_format_cmd.extend([filename])
-    git_show = None
-    clang_format_stdin = subprocess.PIPE
-  try:
-    clang_format = subprocess.Popen(clang_format_cmd, stdin=clang_format_stdin,
-                                    stdout=subprocess.PIPE)
-    if clang_format_stdin == subprocess.PIPE:
-      clang_format_stdin = clang_format.stdin
-  except OSError as e:
-    if e.errno == errno.ENOENT:
-      die('cannot find executable "%s"' % binary)
-    else:
-      raise
-  clang_format_stdin.close()
-  hash_object_cmd = ['git', 'hash-object', '-w', '--path='+filename, '--stdin']
-  hash_object = subprocess.Popen(hash_object_cmd, stdin=clang_format.stdout,
-                                 stdout=subprocess.PIPE)
-  clang_format.stdout.close()
-  stdout = hash_object.communicate()[0]
-  if hash_object.returncode != 0:
-    die('`%s` failed' % ' '.join(hash_object_cmd))
-  if clang_format.wait() != 0:
-    die('`%s` failed' % ' '.join(clang_format_cmd))
-  if git_show and git_show.wait() != 0:
-    die('`%s` failed' % ' '.join(git_show_cmd))
-  return convert_string(stdout).rstrip('\r\n')
-
-
-@contextlib.contextmanager
-def temporary_index_file(tree=None):
-  """Context manager for setting GIT_INDEX_FILE to a temporary file and deleting
-  the file afterward."""
-  index_path = create_temporary_index(tree)
-  old_index_path = os.environ.get('GIT_INDEX_FILE')
-  os.environ['GIT_INDEX_FILE'] = index_path
-  try:
-    yield
-  finally:
-    if old_index_path is None:
-      del os.environ['GIT_INDEX_FILE']
-    else:
-      os.environ['GIT_INDEX_FILE'] = old_index_path
-    os.remove(index_path)
-
-
-def create_temporary_index(tree=None):
-  """Create a temporary index file and return the created file's path.
-
-  If `tree` is not None, use that as the tree to read in.  Otherwise, an
-  empty index is created."""
-  gitdir = run('git', 'rev-parse', '--git-dir')
-  path = os.path.join(gitdir, temp_index_basename)
-  if tree is None:
-    tree = '--empty'
-  run('git', 'read-tree', '--index-output='+path, tree)
-  return path
-
-
-def print_diff(old_tree, new_tree):
-  """Print the diff between the two trees to stdout."""
-  # We use the porcelain 'diff' and not plumbing 'diff-tree' because the output
-  # is expected to be viewed by the user, and only the former does nice things
-  # like color and pagination.
-  #
-  # We also only print modified files since `new_tree` only contains the files
-  # that were modified, so unmodified files would show as deleted without the
-  # filter.
-  subprocess.check_call(['git', 'diff', '--diff-filter=M', old_tree, new_tree,
-                         '--'])
-
-
-def apply_changes(old_tree, new_tree, force=False, patch_mode=False):
-  """Apply the changes in `new_tree` to the working directory.
-
-  Bails if there are local changes in those files and not `force`.  If
-  `patch_mode`, runs `git checkout --patch` to select hunks interactively."""
-  changed_files = run('git', 'diff-tree', '--diff-filter=M', '-r', '-z',
-                      '--name-only', old_tree,
-                      new_tree).rstrip('\0').split('\0')
-  if not force:
-    unstaged_files = run('git', 'diff-files', '--name-status', *changed_files)
-    if unstaged_files:
-      print('The following files would be modified but '
-                'have unstaged changes:', file=sys.stderr)
-      print(unstaged_files, file=sys.stderr)
-      print('Please commit, stage, or stash them first.', file=sys.stderr)
-      sys.exit(2)
-  if patch_mode:
-    # In patch mode, we could just as well create an index from the new tree
-    # and checkout from that, but then the user will be presented with a
-    # message saying "Discard ... from worktree".  Instead, we use the old
-    # tree as the index and checkout from new_tree, which gives the slightly
-    # better message, "Apply ... to index and worktree".  This is not quite
-    # right, since it won't be applied to the user's index, but oh well.
-    with temporary_index_file(old_tree):
-      subprocess.check_call(['git', 'checkout', '--patch', new_tree])
-    index_tree = old_tree
-  else:
-    with temporary_index_file(new_tree):
-      run('git', 'checkout-index', '-a', '-f')
-  return changed_files
-
-
-def run(*args, **kwargs):
-  stdin = kwargs.pop('stdin', '')
-  verbose = kwargs.pop('verbose', True)
-  strip = kwargs.pop('strip', True)
-  for name in kwargs:
-    raise TypeError("run() got an unexpected keyword argument '%s'" % name)
-  p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
-                       stdin=subprocess.PIPE)
-  stdout, stderr = p.communicate(input=stdin)
-
-  stdout = convert_string(stdout)
-  stderr = convert_string(stderr)
-
-  if p.returncode == 0:
-    if stderr:
-      if verbose:
-        print('`%s` printed to stderr:' % ' '.join(args), file=sys.stderr)
-      print(stderr.rstrip(), file=sys.stderr)
-    if strip:
-      stdout = stdout.rstrip('\r\n')
-    return stdout
-  if verbose:
-    print('`%s` returned %s' % (' '.join(args), p.returncode), file=sys.stderr)
-  if stderr:
-    print(stderr.rstrip(), file=sys.stderr)
-  sys.exit(2)
-
-
-def die(message):
-  print('error:', message, file=sys.stderr)
-  sys.exit(2)
-
-
-def to_bytes(str_input):
-    # Encode to UTF-8 to get binary data.
-    if isinstance(str_input, bytes):
-        return str_input
-    return str_input.encode('utf-8')
-
-
-def to_string(bytes_input):
-    if isinstance(bytes_input, str):
-        return bytes_input
-    return bytes_input.encode('utf-8')
-
-
-def convert_string(bytes_input):
-    try:
-        return to_string(bytes_input.decode('utf-8'))
-    except AttributeError: # 'str' object has no attribute 'decode'.
-        return str(bytes_input)
-    except UnicodeError:
-        return str(bytes_input)
-
-if __name__ == '__main__':
-  main()
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/lldbvscode_testcase.py b/lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/lldbvscode_testcase.py
index c1b33c220b4bd..fa5a9c0db1ebd 100644
--- a/lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/lldbvscode_testcase.py
+++ b/lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/lldbvscode_testcase.py
@@ -53,11 +53,12 @@ def set_function_breakpoints(self, functions, condition=None,
             breakpoint_ids.append('%i' % (breakpoint['id']))
         return breakpoint_ids
 
-    def waitUntil(self, condition):
-        while True:
-            if condition():
-                break
+    def waitUntil(self, condition_callback):
+        for _ in range(20):
+            if condition_callback():
+                return True
             time.sleep(0.5)
+        return False
 
     def verify_breakpoint_hit(self, breakpoint_ids):
         '''Wait for the process we are debugging to stop, and verify we hit
diff --git a/lldb/test/API/tools/lldb-vscode/module/Makefile b/lldb/test/API/tools/lldb-vscode/module/Makefile
index 1fb944b138937..b30baf48b972e 100644
--- a/lldb/test/API/tools/lldb-vscode/module/Makefile
+++ b/lldb/test/API/tools/lldb-vscode/module/Makefile
@@ -2,12 +2,16 @@ DYLIB_NAME := foo
 DYLIB_CXX_SOURCES := foo.cpp
 CXX_SOURCES := main.cpp
 
-all: a.out.stripped
+LD_EXTRAS := -Wl,-rpath "-Wl,$(shell pwd)"
+USE_LIBDL :=1
 
 include Makefile.rules
 
-a.out.stripped: a.out.dSYM
+all: a.out.stripped
+
+a.out.stripped:
 	strip -o a.out.stripped a.out
+
 ifneq "$(CODESIGN)" ""
 	$(CODESIGN) -fs - a.out.stripped
-endif
+endif
\ No newline at end of file
diff --git a/lldb/test/API/tools/lldb-vscode/module/TestVSCode_module.py b/lldb/test/API/tools/lldb-vscode/module/TestVSCode_module.py
index 40c4145b38e36..a16430fccae1d 100644
--- a/lldb/test/API/tools/lldb-vscode/module/TestVSCode_module.py
+++ b/lldb/test/API/tools/lldb-vscode/module/TestVSCode_module.py
@@ -10,56 +10,93 @@
 from lldbsuite.test.lldbtest import *
 from lldbsuite.test import lldbutil
 import lldbvscode_testcase
+import re
 
 class TestVSCode_module(lldbvscode_testcase.VSCodeTestCaseBase):
 
     mydir = TestBase.compute_mydir(__file__)
 
-
-    @skipIfWindows
-    @skipUnlessDarwin
-    @skipIfRemote
-    def test_modules_event(self):
+    def run_test(self, symbol_basename, expect_debug_info_size):
         program_basename = "a.out.stripped"
-        program= self.getBuildArtifact(program_basename)
+        program = self.getBuildArtifact(program_basename)
         self.build_and_launch(program)
         functions = ['foo']
         breakpoint_ids = self.set_function_breakpoints(functions)
-        self.assertEquals(len(breakpoint_ids), len(functions),
-                        'expect one breakpoint')
+        self.assertEquals(len(breakpoint_ids), len(functions), 'expect one breakpoint')
         self.continue_to_breakpoints(breakpoint_ids)
         active_modules = self.vscode.get_active_modules()
-        self.assertIn(program_basename, active_modules, '%s module is in active modules' % (program_basename))
         program_module = active_modules[program_basename]
+        self.assertIn(program_basename, active_modules, '%s module is in active modules' % (program_basename))
         self.assertIn('name', program_module, 'make sure name is in module')
         self.assertEqual(program_basename, program_module['name'])
         self.assertIn('path', program_module, 'make sure path is in module')
         self.assertEqual(program, program_module['path'])
         self.assertTrue('symbolFilePath' not in program_module, 'Make sure a.out.stripped has no debug info')
         self.assertEqual('Symbols not found.', program_module['symbolStatus'])
-        symbol_path = self.getBuildArtifact("a.out")
-        self.vscode.request_evaluate('`%s' % ('target symbols add -s "%s" "%s"' % (program, symbol_path)))
+        symbols_path = self.getBuildArtifact(symbol_basename)
+        self.vscode.request_evaluate('`%s' % ('target symbols add -s "%s" "%s"' % (program, symbols_path)))
 
         def checkSymbolsLoaded():
             active_modules = self.vscode.get_active_modules()
             program_module = active_modules[program_basename]
             return 'Symbols loaded.' == program_module['symbolStatus']
-        self.waitUntil(checkSymbolsLoaded)
 
+        def checkSymbolsLoadedWithSize():
+            active_modules = self.vscode.get_active_modules()
+            program_module = active_modules[program_basename]
+            symbolsStatus = program_module['symbolStatus']
+            symbol_regex = re.compile(r"Symbols loaded. \([0-9]+(\.[0-9]*)?[KMG]?B\)")
+            return symbol_regex.match(program_module['symbolStatus'])
+                
+        if expect_debug_info_size:
+            self.waitUntil(checkSymbolsLoadedWithSize)
+        else:
+            self.waitUntil(checkSymbolsLoaded)
         active_modules = self.vscode.get_active_modules()
         program_module = active_modules[program_basename]
         self.assertEqual(program_basename, program_module['name'])
         self.assertEqual(program, program_module['path'])
-        self.assertEqual('Symbols loaded.', program_module['symbolStatus'])
         self.assertIn('symbolFilePath', program_module)
-        self.assertEqual(symbol_path, program_module['symbolFilePath'])
+        self.assertIn(symbols_path, program_module['symbolFilePath'])
         self.assertIn('addressRange', program_module)
 
+    @skipIfWindows
+    @skipUnlessDarwin
+    @skipIfRemote  
+    #TODO: Update the Makefile so that this test runs on Linux
+    def test_module_event(self):
+        '''
+            Mac or linux.
+
+            On mac, if we load a.out as our symbol file, we will use DWARF with .o files and we will
+            have debug symbols, but we won't see any debug info size because all of the DWARF
+            sections are in .o files.
+
+            On other platforms, we expect a.out to have debug info, so we will expect a size.
+            expect_debug_info_size = platform.system() != 'Darwin'
+            return self.run_test("a.out", expect_debug_info_size)
+        '''
+        expect_debug_info_size = platform.system() != 'Darwin'
+        return self.run_test("a.out", expect_debug_info_size)
+
+    @skipIfWindows
+    @skipUnlessDarwin
+    @skipIfRemote    
+    def test_module_event_dsym(self):
+        '''
+            Darwin only test with dSYM file.
+
+            On mac, if we load a.out.dSYM as our symbol file, we will have debug symbols and we
+            will have DWARF sections added to the module, so we will expect a size.
+            return self.run_test("a.out.dSYM", True)
+        '''
+        return self.run_test("a.out.dSYM", True)
+
     @skipIfWindows
     @skipUnlessDarwin
     @skipIfRemote
     def test_compile_units(self):
-        program= self.getBuildArtifact("a.out")
+        program = self.getBuildArtifact("a.out")
         self.build_and_launch(program)
         source = "main.cpp"
         main_source_path = self.getSourcePath(source)
diff --git a/lldb/tools/lldb-vscode/JSONUtils.cpp b/lldb/tools/lldb-vscode/JSONUtils.cpp
index 1ebaa5c377121..f6cdcf5a46cfc 100644
--- a/lldb/tools/lldb-vscode/JSONUtils.cpp
+++ b/lldb/tools/lldb-vscode/JSONUtils.cpp
@@ -7,6 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #include <algorithm>
+#include <iomanip>
+#include <sstream>
 
 #include "llvm/ADT/Optional.h"
 #include "llvm/Support/FormatAdapters.h"
@@ -327,6 +329,50 @@ llvm::json::Value CreateBreakpoint(lldb::SBBreakpoint &bp,
   return llvm::json::Value(std::move(object));
 }
 
+static uint64_t GetDebugInfoSizeInSection(lldb::SBSection section) {
+  uint64_t debug_info_size = 0;
+  llvm::StringRef section_name(section.GetName());
+  if (section_name.startswith(".debug") || section_name.startswith("__debug") ||
+      section_name.startswith(".apple") || section_name.startswith("__apple"))
+    debug_info_size += section.GetFileByteSize();
+  size_t num_sub_sections = section.GetNumSubSections();
+  for (size_t i = 0; i < num_sub_sections; i++) {
+    debug_info_size +=
+        GetDebugInfoSizeInSection(section.GetSubSectionAtIndex(i));
+  }
+  return debug_info_size;
+}
+
+static uint64_t GetDebugInfoSize(lldb::SBModule module) {
+  uint64_t debug_info_size = 0;
+  size_t num_sections = module.GetNumSections();
+  for (size_t i = 0; i < num_sections; i++) {
+    debug_info_size += GetDebugInfoSizeInSection(module.GetSectionAtIndex(i));
+  }
+  return debug_info_size;
+}
+
+static std::string ConvertDebugInfoSizeToString(uint64_t debug_info) {
+  std::ostringstream oss;
+  oss << " (";
+  oss << std::fixed << std::setprecision(1);
+
+  if (debug_info < 1024) {
+    oss << debug_info << "B";
+  } else if (debug_info < 1024 * 1024) {
+    double kb = double(debug_info) / 1024.0;
+    oss << kb << "KB";
+  } else if (debug_info < 1024 * 1024 * 1024) {
+    double mb = double(debug_info) / (1024.0 * 1024.0);
+    oss << mb << "MB";
+  } else {
+    double gb = double(debug_info) / (1024.0 * 1024.0 * 1024.0);
+    oss << gb << "GB";
+    ;
+  }
+  oss << ")";
+  return oss.str();
+}
 llvm::json::Value CreateModule(lldb::SBModule &module) {
   llvm::json::Object object;
   if (!module.IsValid())
@@ -339,9 +385,15 @@ llvm::json::Value CreateModule(lldb::SBModule &module) {
   std::string module_path(module_path_arr);
   object.try_emplace("path", module_path);
   if (module.GetNumCompileUnits() > 0) {
-    object.try_emplace("symbolStatus", "Symbols loaded.");
+    std::string symbol_str = "Symbols loaded.";
+    uint64_t debug_info = GetDebugInfoSize(module);
+    if (debug_info > 0) {
+      symbol_str += ConvertDebugInfoSizeToString(debug_info);
+    }
+    object.try_emplace("symbolStatus", symbol_str);
     char symbol_path_arr[PATH_MAX];
-    module.GetSymbolFileSpec().GetPath(symbol_path_arr, sizeof(symbol_path_arr));
+    module.GetSymbolFileSpec().GetPath(symbol_path_arr,
+                                       sizeof(symbol_path_arr));
     std::string symbol_path(symbol_path_arr);
     object.try_emplace("symbolFilePath", symbol_path);
   } else {
@@ -352,8 +404,9 @@ llvm::json::Value CreateModule(lldb::SBModule &module) {
   object.try_emplace("addressRange", loaded_addr);
   std::string version_str;
   uint32_t version_nums[3];
-  uint32_t num_versions = module.GetVersion(version_nums, sizeof(version_nums)/sizeof(uint32_t));
-  for (uint32_t i=0; i<num_versions; ++i) {
+  uint32_t num_versions =
+      module.GetVersion(version_nums, sizeof(version_nums) / sizeof(uint32_t));
+  for (uint32_t i = 0; i < num_versions; ++i) {
     if (!version_str.empty())
       version_str += ".";
     version_str += std::to_string(version_nums[i]);

From 313b60742af1f567ef7d8e26eb5b1015a7b5c78f Mon Sep 17 00:00:00 2001
From: Walter Erquinigo <wallace@fb.com>
Date: Fri, 24 Jul 2020 13:28:29 -0700
Subject: [PATCH 0050/1035] Revert "Add Debug Info Size to Symbol Status"

This reverts commit 986e3af53bfe591e88a1ae4f82ea1cc0a15819a3.

It incorrectly deleted clang/tools/clang-format/git-clang-format
---
 clang/tools/clang-format/git-clang-format     | 585 ++++++++++++++++++
 .../tools/lldb-vscode/lldbvscode_testcase.py  |   9 +-
 .../API/tools/lldb-vscode/module/Makefile     |  10 +-
 .../lldb-vscode/module/TestVSCode_module.py   |  67 +-
 lldb/tools/lldb-vscode/JSONUtils.cpp          |  61 +-
 5 files changed, 611 insertions(+), 121 deletions(-)

diff --git a/clang/tools/clang-format/git-clang-format b/clang/tools/clang-format/git-clang-format
index e69de29bb2d1d..f3cd585e7f4a0 100755
--- a/clang/tools/clang-format/git-clang-format
+++ b/clang/tools/clang-format/git-clang-format
@@ -0,0 +1,585 @@
+#!/usr/bin/env python
+#
+#===- git-clang-format - ClangFormat Git Integration ---------*- python -*--===#
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+#===------------------------------------------------------------------------===#
+
+r"""
+clang-format git integration
+============================
+
+This file provides a clang-format integration for git. Put it somewhere in your
+path and ensure that it is executable. Then, "git clang-format" will invoke
+clang-format on the changes in current files or a specific commit.
+
+For further details, run:
+git clang-format -h
+
+Requires Python 2.7 or Python 3
+"""
+
+from __future__ import absolute_import, division, print_function
+import argparse
+import collections
+import contextlib
+import errno
+import os
+import re
+import subprocess
+import sys
+
+usage = 'git clang-format [OPTIONS] [<commit>] [<commit>] [--] [<file>...]'
+
+desc = '''
+If zero or one commits are given, run clang-format on all lines that differ
+between the working directory and <commit>, which defaults to HEAD.  Changes are
+only applied to the working directory.
+
+If two commits are given (requires --diff), run clang-format on all lines in the
+second <commit> that differ from the first <commit>.
+
+The following git-config settings set the default of the corresponding option:
+  clangFormat.binary
+  clangFormat.commit
+  clangFormat.extensions
+  clangFormat.style
+'''
+
+# Name of the temporary index file in which save the output of clang-format.
+# This file is created within the .git directory.
+temp_index_basename = 'clang-format-index'
+
+
+Range = collections.namedtuple('Range', 'start, count')
+
+
+def main():
+  config = load_git_config()
+
+  # In order to keep '--' yet allow options after positionals, we need to
+  # check for '--' ourselves.  (Setting nargs='*' throws away the '--', while
+  # nargs=argparse.REMAINDER disallows options after positionals.)
+  argv = sys.argv[1:]
+  try:
+    idx = argv.index('--')
+  except ValueError:
+    dash_dash = []
+  else:
+    dash_dash = argv[idx:]
+    argv = argv[:idx]
+
+  default_extensions = ','.join([
+      # From clang/lib/Frontend/FrontendOptions.cpp, all lower case
+      'c', 'h',  # C
+      'm',  # ObjC
+      'mm',  # ObjC++
+      'cc', 'cp', 'cpp', 'c++', 'cxx', 'hh', 'hpp', 'hxx',  # C++
+      'cu',  # CUDA
+      # Other languages that clang-format supports
+      'proto', 'protodevel',  # Protocol Buffers
+      'java',  # Java
+      'js',  # JavaScript
+      'ts',  # TypeScript
+      'cs',  # C Sharp
+      ])
+
+  p = argparse.ArgumentParser(
+    usage=usage, formatter_class=argparse.RawDescriptionHelpFormatter,
+    description=desc)
+  p.add_argument('--binary',
+                 default=config.get('clangformat.binary', 'clang-format'),
+                 help='path to clang-format'),
+  p.add_argument('--commit',
+                 default=config.get('clangformat.commit', 'HEAD'),
+                 help='default commit to use if none is specified'),
+  p.add_argument('--diff', action='store_true',
+                 help='print a diff instead of applying the changes')
+  p.add_argument('--extensions',
+                 default=config.get('clangformat.extensions',
+                                    default_extensions),
+                 help=('comma-separated list of file extensions to format, '
+                       'excluding the period and case-insensitive')),
+  p.add_argument('-f', '--force', action='store_true',
+                 help='allow changes to unstaged files')
+  p.add_argument('-p', '--patch', action='store_true',
+                 help='select hunks interactively')
+  p.add_argument('-q', '--quiet', action='count', default=0,
+                 help='print less information')
+  p.add_argument('--style',
+                 default=config.get('clangformat.style', None),
+                 help='passed to clang-format'),
+  p.add_argument('-v', '--verbose', action='count', default=0,
+                 help='print extra information')
+  # We gather all the remaining positional arguments into 'args' since we need
+  # to use some heuristics to determine whether or not <commit> was present.
+  # However, to print pretty messages, we make use of metavar and help.
+  p.add_argument('args', nargs='*', metavar='<commit>',
+                 help='revision from which to compute the diff')
+  p.add_argument('ignored', nargs='*', metavar='<file>...',
+                 help='if specified, only consider differences in these files')
+  opts = p.parse_args(argv)
+
+  opts.verbose -= opts.quiet
+  del opts.quiet
+
+  commits, files = interpret_args(opts.args, dash_dash, opts.commit)
+  if len(commits) > 1:
+    if not opts.diff:
+      die('--diff is required when two commits are given')
+  else:
+    if len(commits) > 2:
+      die('at most two commits allowed; %d given' % len(commits))
+  changed_lines = compute_diff_and_extract_lines(commits, files)
+  if opts.verbose >= 1:
+    ignored_files = set(changed_lines)
+  filter_by_extension(changed_lines, opts.extensions.lower().split(','))
+  if opts.verbose >= 1:
+    ignored_files.difference_update(changed_lines)
+    if ignored_files:
+      print('Ignoring changes in the following files (wrong extension):')
+      for filename in ignored_files:
+        print('    %s' % filename)
+    if changed_lines:
+      print('Running clang-format on the following files:')
+      for filename in changed_lines:
+        print('    %s' % filename)
+  if not changed_lines:
+    print('no modified files to format')
+    return
+  # The computed diff outputs absolute paths, so we must cd before accessing
+  # those files.
+  cd_to_toplevel()
+  if len(commits) > 1:
+    old_tree = commits[1]
+    new_tree = run_clang_format_and_save_to_tree(changed_lines,
+                                                 revision=commits[1],
+                                                 binary=opts.binary,
+                                                 style=opts.style)
+  else:
+    old_tree = create_tree_from_workdir(changed_lines)
+    new_tree = run_clang_format_and_save_to_tree(changed_lines,
+                                                 binary=opts.binary,
+                                                 style=opts.style)
+  if opts.verbose >= 1:
+    print('old tree: %s' % old_tree)
+    print('new tree: %s' % new_tree)
+  if old_tree == new_tree:
+    if opts.verbose >= 0:
+      print('clang-format did not modify any files')
+  elif opts.diff:
+    print_diff(old_tree, new_tree)
+  else:
+    changed_files = apply_changes(old_tree, new_tree, force=opts.force,
+                                  patch_mode=opts.patch)
+    if (opts.verbose >= 0 and not opts.patch) or opts.verbose >= 1:
+      print('changed files:')
+      for filename in changed_files:
+        print('    %s' % filename)
+
+
+def load_git_config(non_string_options=None):
+  """Return the git configuration as a dictionary.
+
+  All options are assumed to be strings unless in `non_string_options`, in which
+  is a dictionary mapping option name (in lower case) to either "--bool" or
+  "--int"."""
+  if non_string_options is None:
+    non_string_options = {}
+  out = {}
+  for entry in run('git', 'config', '--list', '--null').split('\0'):
+    if entry:
+      if '\n' in entry:
+        name, value = entry.split('\n', 1)
+      else:
+        # A setting with no '=' ('\n' with --null) is implicitly 'true'
+        name = entry
+        value = 'true'
+      if name in non_string_options:
+        value = run('git', 'config', non_string_options[name], name)
+      out[name] = value
+  return out
+
+
+def interpret_args(args, dash_dash, default_commit):
+  """Interpret `args` as "[commits] [--] [files]" and return (commits, files).
+
+  It is assumed that "--" and everything that follows has been removed from
+  args and placed in `dash_dash`.
+
+  If "--" is present (i.e., `dash_dash` is non-empty), the arguments to its
+  left (if present) are taken as commits.  Otherwise, the arguments are checked
+  from left to right if they are commits or files.  If commits are not given,
+  a list with `default_commit` is used."""
+  if dash_dash:
+    if len(args) == 0:
+      commits = [default_commit]
+    else:
+      commits = args
+    for commit in commits:
+      object_type = get_object_type(commit)
+      if object_type not in ('commit', 'tag'):
+        if object_type is None:
+          die("'%s' is not a commit" % commit)
+        else:
+          die("'%s' is a %s, but a commit was expected" % (commit, object_type))
+    files = dash_dash[1:]
+  elif args:
+    commits = []
+    while args:
+      if not disambiguate_revision(args[0]):
+        break
+      commits.append(args.pop(0))
+    if not commits:
+      commits = [default_commit]
+    files = args
+  else:
+    commits = [default_commit]
+    files = []
+  return commits, files
+
+
+def disambiguate_revision(value):
+  """Returns True if `value` is a revision, False if it is a file, or dies."""
+  # If `value` is ambiguous (neither a commit nor a file), the following
+  # command will die with an appropriate error message.
+  run('git', 'rev-parse', value, verbose=False)
+  object_type = get_object_type(value)
+  if object_type is None:
+    return False
+  if object_type in ('commit', 'tag'):
+    return True
+  die('`%s` is a %s, but a commit or filename was expected' %
+      (value, object_type))
+
+
+def get_object_type(value):
+  """Returns a string description of an object's type, or None if it is not
+  a valid git object."""
+  cmd = ['git', 'cat-file', '-t', value]
+  p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+  stdout, stderr = p.communicate()
+  if p.returncode != 0:
+    return None
+  return convert_string(stdout.strip())
+
+
+def compute_diff_and_extract_lines(commits, files):
+  """Calls compute_diff() followed by extract_lines()."""
+  diff_process = compute_diff(commits, files)
+  changed_lines = extract_lines(diff_process.stdout)
+  diff_process.stdout.close()
+  diff_process.wait()
+  if diff_process.returncode != 0:
+    # Assume error was already printed to stderr.
+    sys.exit(2)
+  return changed_lines
+
+
+def compute_diff(commits, files):
+  """Return a subprocess object producing the diff from `commits`.
+
+  The return value's `stdin` file object will produce a patch with the
+  differences between the working directory and the first commit if a single
+  one was specified, or the difference between both specified commits, filtered
+  on `files` (if non-empty).  Zero context lines are used in the patch."""
+  git_tool = 'diff-index'
+  if len(commits) > 1:
+    git_tool = 'diff-tree'
+  cmd = ['git', git_tool, '-p', '-U0'] + commits + ['--']
+  cmd.extend(files)
+  p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+  p.stdin.close()
+  return p
+
+
+def extract_lines(patch_file):
+  """Extract the changed lines in `patch_file`.
+
+  The return value is a dictionary mapping filename to a list of (start_line,
+  line_count) pairs.
+
+  The input must have been produced with ``-U0``, meaning unidiff format with
+  zero lines of context.  The return value is a dict mapping filename to a
+  list of line `Range`s."""
+  matches = {}
+  for line in patch_file:
+    line = convert_string(line)
+    match = re.search(r'^\+\+\+\ [^/]+/(.*)', line)
+    if match:
+      filename = match.group(1).rstrip('\r\n')
+    match = re.search(r'^@@ -[0-9,]+ \+(\d+)(,(\d+))?', line)
+    if match:
+      start_line = int(match.group(1))
+      line_count = 1
+      if match.group(3):
+        line_count = int(match.group(3))
+      if line_count > 0:
+        matches.setdefault(filename, []).append(Range(start_line, line_count))
+  return matches
+
+
+def filter_by_extension(dictionary, allowed_extensions):
+  """Delete every key in `dictionary` that doesn't have an allowed extension.
+
+  `allowed_extensions` must be a collection of lowercase file extensions,
+  excluding the period."""
+  allowed_extensions = frozenset(allowed_extensions)
+  for filename in list(dictionary.keys()):
+    base_ext = filename.rsplit('.', 1)
+    if len(base_ext) == 1 and '' in allowed_extensions:
+        continue
+    if len(base_ext) == 1 or base_ext[1].lower() not in allowed_extensions:
+      del dictionary[filename]
+
+
+def cd_to_toplevel():
+  """Change to the top level of the git repository."""
+  toplevel = run('git', 'rev-parse', '--show-toplevel')
+  os.chdir(toplevel)
+
+
+def create_tree_from_workdir(filenames):
+  """Create a new git tree with the given files from the working directory.
+
+  Returns the object ID (SHA-1) of the created tree."""
+  return create_tree(filenames, '--stdin')
+
+
+def run_clang_format_and_save_to_tree(changed_lines, revision=None,
+                                      binary='clang-format', style=None):
+  """Run clang-format on each file and save the result to a git tree.
+
+  Returns the object ID (SHA-1) of the created tree."""
+  def iteritems(container):
+      try:
+          return container.iteritems() # Python 2
+      except AttributeError:
+          return container.items() # Python 3
+  def index_info_generator():
+    for filename, line_ranges in iteritems(changed_lines):
+      if revision:
+        git_metadata_cmd = ['git', 'ls-tree',
+                            '%s:%s' % (revision, os.path.dirname(filename)),
+                            os.path.basename(filename)]
+        git_metadata = subprocess.Popen(git_metadata_cmd, stdin=subprocess.PIPE,
+                                        stdout=subprocess.PIPE)
+        stdout = git_metadata.communicate()[0]
+        mode = oct(int(stdout.split()[0], 8))
+      else:
+        mode = oct(os.stat(filename).st_mode)
+      # Adjust python3 octal format so that it matches what git expects
+      if mode.startswith('0o'):
+          mode = '0' + mode[2:]
+      blob_id = clang_format_to_blob(filename, line_ranges,
+                                     revision=revision,
+                                     binary=binary,
+                                     style=style)
+      yield '%s %s\t%s' % (mode, blob_id, filename)
+  return create_tree(index_info_generator(), '--index-info')
+
+
+def create_tree(input_lines, mode):
+  """Create a tree object from the given input.
+
+  If mode is '--stdin', it must be a list of filenames.  If mode is
+  '--index-info' is must be a list of values suitable for "git update-index
+  --index-info", such as "<mode> <SP> <sha1> <TAB> <filename>".  Any other mode
+  is invalid."""
+  assert mode in ('--stdin', '--index-info')
+  cmd = ['git', 'update-index', '--add', '-z', mode]
+  with temporary_index_file():
+    p = subprocess.Popen(cmd, stdin=subprocess.PIPE)
+    for line in input_lines:
+      p.stdin.write(to_bytes('%s\0' % line))
+    p.stdin.close()
+    if p.wait() != 0:
+      die('`%s` failed' % ' '.join(cmd))
+    tree_id = run('git', 'write-tree')
+    return tree_id
+
+
+def clang_format_to_blob(filename, line_ranges, revision=None,
+                         binary='clang-format', style=None):
+  """Run clang-format on the given file and save the result to a git blob.
+
+  Runs on the file in `revision` if not None, or on the file in the working
+  directory if `revision` is None.
+
+  Returns the object ID (SHA-1) of the created blob."""
+  clang_format_cmd = [binary]
+  if style:
+    clang_format_cmd.extend(['-style='+style])
+  clang_format_cmd.extend([
+      '-lines=%s:%s' % (start_line, start_line+line_count-1)
+      for start_line, line_count in line_ranges])
+  if revision:
+    clang_format_cmd.extend(['-assume-filename='+filename])
+    git_show_cmd = ['git', 'cat-file', 'blob', '%s:%s' % (revision, filename)]
+    git_show = subprocess.Popen(git_show_cmd, stdin=subprocess.PIPE,
+                                stdout=subprocess.PIPE)
+    git_show.stdin.close()
+    clang_format_stdin = git_show.stdout
+  else:
+    clang_format_cmd.extend([filename])
+    git_show = None
+    clang_format_stdin = subprocess.PIPE
+  try:
+    clang_format = subprocess.Popen(clang_format_cmd, stdin=clang_format_stdin,
+                                    stdout=subprocess.PIPE)
+    if clang_format_stdin == subprocess.PIPE:
+      clang_format_stdin = clang_format.stdin
+  except OSError as e:
+    if e.errno == errno.ENOENT:
+      die('cannot find executable "%s"' % binary)
+    else:
+      raise
+  clang_format_stdin.close()
+  hash_object_cmd = ['git', 'hash-object', '-w', '--path='+filename, '--stdin']
+  hash_object = subprocess.Popen(hash_object_cmd, stdin=clang_format.stdout,
+                                 stdout=subprocess.PIPE)
+  clang_format.stdout.close()
+  stdout = hash_object.communicate()[0]
+  if hash_object.returncode != 0:
+    die('`%s` failed' % ' '.join(hash_object_cmd))
+  if clang_format.wait() != 0:
+    die('`%s` failed' % ' '.join(clang_format_cmd))
+  if git_show and git_show.wait() != 0:
+    die('`%s` failed' % ' '.join(git_show_cmd))
+  return convert_string(stdout).rstrip('\r\n')
+
+
+@contextlib.contextmanager
+def temporary_index_file(tree=None):
+  """Context manager for setting GIT_INDEX_FILE to a temporary file and deleting
+  the file afterward."""
+  index_path = create_temporary_index(tree)
+  old_index_path = os.environ.get('GIT_INDEX_FILE')
+  os.environ['GIT_INDEX_FILE'] = index_path
+  try:
+    yield
+  finally:
+    if old_index_path is None:
+      del os.environ['GIT_INDEX_FILE']
+    else:
+      os.environ['GIT_INDEX_FILE'] = old_index_path
+    os.remove(index_path)
+
+
+def create_temporary_index(tree=None):
+  """Create a temporary index file and return the created file's path.
+
+  If `tree` is not None, use that as the tree to read in.  Otherwise, an
+  empty index is created."""
+  gitdir = run('git', 'rev-parse', '--git-dir')
+  path = os.path.join(gitdir, temp_index_basename)
+  if tree is None:
+    tree = '--empty'
+  run('git', 'read-tree', '--index-output='+path, tree)
+  return path
+
+
+def print_diff(old_tree, new_tree):
+  """Print the diff between the two trees to stdout."""
+  # We use the porcelain 'diff' and not plumbing 'diff-tree' because the output
+  # is expected to be viewed by the user, and only the former does nice things
+  # like color and pagination.
+  #
+  # We also only print modified files since `new_tree` only contains the files
+  # that were modified, so unmodified files would show as deleted without the
+  # filter.
+  subprocess.check_call(['git', 'diff', '--diff-filter=M', old_tree, new_tree,
+                         '--'])
+
+
+def apply_changes(old_tree, new_tree, force=False, patch_mode=False):
+  """Apply the changes in `new_tree` to the working directory.
+
+  Bails if there are local changes in those files and not `force`.  If
+  `patch_mode`, runs `git checkout --patch` to select hunks interactively."""
+  changed_files = run('git', 'diff-tree', '--diff-filter=M', '-r', '-z',
+                      '--name-only', old_tree,
+                      new_tree).rstrip('\0').split('\0')
+  if not force:
+    unstaged_files = run('git', 'diff-files', '--name-status', *changed_files)
+    if unstaged_files:
+      print('The following files would be modified but '
+                'have unstaged changes:', file=sys.stderr)
+      print(unstaged_files, file=sys.stderr)
+      print('Please commit, stage, or stash them first.', file=sys.stderr)
+      sys.exit(2)
+  if patch_mode:
+    # In patch mode, we could just as well create an index from the new tree
+    # and checkout from that, but then the user will be presented with a
+    # message saying "Discard ... from worktree".  Instead, we use the old
+    # tree as the index and checkout from new_tree, which gives the slightly
+    # better message, "Apply ... to index and worktree".  This is not quite
+    # right, since it won't be applied to the user's index, but oh well.
+    with temporary_index_file(old_tree):
+      subprocess.check_call(['git', 'checkout', '--patch', new_tree])
+    index_tree = old_tree
+  else:
+    with temporary_index_file(new_tree):
+      run('git', 'checkout-index', '-a', '-f')
+  return changed_files
+
+
+def run(*args, **kwargs):
+  stdin = kwargs.pop('stdin', '')
+  verbose = kwargs.pop('verbose', True)
+  strip = kwargs.pop('strip', True)
+  for name in kwargs:
+    raise TypeError("run() got an unexpected keyword argument '%s'" % name)
+  p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                       stdin=subprocess.PIPE)
+  stdout, stderr = p.communicate(input=stdin)
+
+  stdout = convert_string(stdout)
+  stderr = convert_string(stderr)
+
+  if p.returncode == 0:
+    if stderr:
+      if verbose:
+        print('`%s` printed to stderr:' % ' '.join(args), file=sys.stderr)
+      print(stderr.rstrip(), file=sys.stderr)
+    if strip:
+      stdout = stdout.rstrip('\r\n')
+    return stdout
+  if verbose:
+    print('`%s` returned %s' % (' '.join(args), p.returncode), file=sys.stderr)
+  if stderr:
+    print(stderr.rstrip(), file=sys.stderr)
+  sys.exit(2)
+
+
+def die(message):
+  print('error:', message, file=sys.stderr)
+  sys.exit(2)
+
+
+def to_bytes(str_input):
+    # Encode to UTF-8 to get binary data.
+    if isinstance(str_input, bytes):
+        return str_input
+    return str_input.encode('utf-8')
+
+
+def to_string(bytes_input):
+    if isinstance(bytes_input, str):
+        return bytes_input
+    return bytes_input.encode('utf-8')
+
+
+def convert_string(bytes_input):
+    try:
+        return to_string(bytes_input.decode('utf-8'))
+    except AttributeError: # 'str' object has no attribute 'decode'.
+        return str(bytes_input)
+    except UnicodeError:
+        return str(bytes_input)
+
+if __name__ == '__main__':
+  main()
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/lldbvscode_testcase.py b/lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/lldbvscode_testcase.py
index fa5a9c0db1ebd..c1b33c220b4bd 100644
--- a/lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/lldbvscode_testcase.py
+++ b/lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/lldbvscode_testcase.py
@@ -53,12 +53,11 @@ def set_function_breakpoints(self, functions, condition=None,
             breakpoint_ids.append('%i' % (breakpoint['id']))
         return breakpoint_ids
 
-    def waitUntil(self, condition_callback):
-        for _ in range(20):
-            if condition_callback():
-                return True
+    def waitUntil(self, condition):
+        while True:
+            if condition():
+                break
             time.sleep(0.5)
-        return False
 
     def verify_breakpoint_hit(self, breakpoint_ids):
         '''Wait for the process we are debugging to stop, and verify we hit
diff --git a/lldb/test/API/tools/lldb-vscode/module/Makefile b/lldb/test/API/tools/lldb-vscode/module/Makefile
index b30baf48b972e..1fb944b138937 100644
--- a/lldb/test/API/tools/lldb-vscode/module/Makefile
+++ b/lldb/test/API/tools/lldb-vscode/module/Makefile
@@ -2,16 +2,12 @@ DYLIB_NAME := foo
 DYLIB_CXX_SOURCES := foo.cpp
 CXX_SOURCES := main.cpp
 
-LD_EXTRAS := -Wl,-rpath "-Wl,$(shell pwd)"
-USE_LIBDL :=1
+all: a.out.stripped
 
 include Makefile.rules
 
-all: a.out.stripped
-
-a.out.stripped:
+a.out.stripped: a.out.dSYM
 	strip -o a.out.stripped a.out
-
 ifneq "$(CODESIGN)" ""
 	$(CODESIGN) -fs - a.out.stripped
-endif
\ No newline at end of file
+endif
diff --git a/lldb/test/API/tools/lldb-vscode/module/TestVSCode_module.py b/lldb/test/API/tools/lldb-vscode/module/TestVSCode_module.py
index a16430fccae1d..40c4145b38e36 100644
--- a/lldb/test/API/tools/lldb-vscode/module/TestVSCode_module.py
+++ b/lldb/test/API/tools/lldb-vscode/module/TestVSCode_module.py
@@ -10,93 +10,56 @@
 from lldbsuite.test.lldbtest import *
 from lldbsuite.test import lldbutil
 import lldbvscode_testcase
-import re
 
 class TestVSCode_module(lldbvscode_testcase.VSCodeTestCaseBase):
 
     mydir = TestBase.compute_mydir(__file__)
 
-    def run_test(self, symbol_basename, expect_debug_info_size):
+
+    @skipIfWindows
+    @skipUnlessDarwin
+    @skipIfRemote
+    def test_modules_event(self):
         program_basename = "a.out.stripped"
-        program = self.getBuildArtifact(program_basename)
+        program= self.getBuildArtifact(program_basename)
         self.build_and_launch(program)
         functions = ['foo']
         breakpoint_ids = self.set_function_breakpoints(functions)
-        self.assertEquals(len(breakpoint_ids), len(functions), 'expect one breakpoint')
+        self.assertEquals(len(breakpoint_ids), len(functions),
+                        'expect one breakpoint')
         self.continue_to_breakpoints(breakpoint_ids)
         active_modules = self.vscode.get_active_modules()
-        program_module = active_modules[program_basename]
         self.assertIn(program_basename, active_modules, '%s module is in active modules' % (program_basename))
+        program_module = active_modules[program_basename]
         self.assertIn('name', program_module, 'make sure name is in module')
         self.assertEqual(program_basename, program_module['name'])
         self.assertIn('path', program_module, 'make sure path is in module')
         self.assertEqual(program, program_module['path'])
         self.assertTrue('symbolFilePath' not in program_module, 'Make sure a.out.stripped has no debug info')
         self.assertEqual('Symbols not found.', program_module['symbolStatus'])
-        symbols_path = self.getBuildArtifact(symbol_basename)
-        self.vscode.request_evaluate('`%s' % ('target symbols add -s "%s" "%s"' % (program, symbols_path)))
+        symbol_path = self.getBuildArtifact("a.out")
+        self.vscode.request_evaluate('`%s' % ('target symbols add -s "%s" "%s"' % (program, symbol_path)))
 
         def checkSymbolsLoaded():
             active_modules = self.vscode.get_active_modules()
             program_module = active_modules[program_basename]
             return 'Symbols loaded.' == program_module['symbolStatus']
+        self.waitUntil(checkSymbolsLoaded)
 
-        def checkSymbolsLoadedWithSize():
-            active_modules = self.vscode.get_active_modules()
-            program_module = active_modules[program_basename]
-            symbolsStatus = program_module['symbolStatus']
-            symbol_regex = re.compile(r"Symbols loaded. \([0-9]+(\.[0-9]*)?[KMG]?B\)")
-            return symbol_regex.match(program_module['symbolStatus'])
-                
-        if expect_debug_info_size:
-            self.waitUntil(checkSymbolsLoadedWithSize)
-        else:
-            self.waitUntil(checkSymbolsLoaded)
         active_modules = self.vscode.get_active_modules()
         program_module = active_modules[program_basename]
         self.assertEqual(program_basename, program_module['name'])
         self.assertEqual(program, program_module['path'])
+        self.assertEqual('Symbols loaded.', program_module['symbolStatus'])
         self.assertIn('symbolFilePath', program_module)
-        self.assertIn(symbols_path, program_module['symbolFilePath'])
+        self.assertEqual(symbol_path, program_module['symbolFilePath'])
         self.assertIn('addressRange', program_module)
 
-    @skipIfWindows
-    @skipUnlessDarwin
-    @skipIfRemote  
-    #TODO: Update the Makefile so that this test runs on Linux
-    def test_module_event(self):
-        '''
-            Mac or linux.
-
-            On mac, if we load a.out as our symbol file, we will use DWARF with .o files and we will
-            have debug symbols, but we won't see any debug info size because all of the DWARF
-            sections are in .o files.
-
-            On other platforms, we expect a.out to have debug info, so we will expect a size.
-            expect_debug_info_size = platform.system() != 'Darwin'
-            return self.run_test("a.out", expect_debug_info_size)
-        '''
-        expect_debug_info_size = platform.system() != 'Darwin'
-        return self.run_test("a.out", expect_debug_info_size)
-
-    @skipIfWindows
-    @skipUnlessDarwin
-    @skipIfRemote    
-    def test_module_event_dsym(self):
-        '''
-            Darwin only test with dSYM file.
-
-            On mac, if we load a.out.dSYM as our symbol file, we will have debug symbols and we
-            will have DWARF sections added to the module, so we will expect a size.
-            return self.run_test("a.out.dSYM", True)
-        '''
-        return self.run_test("a.out.dSYM", True)
-
     @skipIfWindows
     @skipUnlessDarwin
     @skipIfRemote
     def test_compile_units(self):
-        program = self.getBuildArtifact("a.out")
+        program= self.getBuildArtifact("a.out")
         self.build_and_launch(program)
         source = "main.cpp"
         main_source_path = self.getSourcePath(source)
diff --git a/lldb/tools/lldb-vscode/JSONUtils.cpp b/lldb/tools/lldb-vscode/JSONUtils.cpp
index f6cdcf5a46cfc..1ebaa5c377121 100644
--- a/lldb/tools/lldb-vscode/JSONUtils.cpp
+++ b/lldb/tools/lldb-vscode/JSONUtils.cpp
@@ -7,8 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 #include <algorithm>
-#include <iomanip>
-#include <sstream>
 
 #include "llvm/ADT/Optional.h"
 #include "llvm/Support/FormatAdapters.h"
@@ -329,50 +327,6 @@ llvm::json::Value CreateBreakpoint(lldb::SBBreakpoint &bp,
   return llvm::json::Value(std::move(object));
 }
 
-static uint64_t GetDebugInfoSizeInSection(lldb::SBSection section) {
-  uint64_t debug_info_size = 0;
-  llvm::StringRef section_name(section.GetName());
-  if (section_name.startswith(".debug") || section_name.startswith("__debug") ||
-      section_name.startswith(".apple") || section_name.startswith("__apple"))
-    debug_info_size += section.GetFileByteSize();
-  size_t num_sub_sections = section.GetNumSubSections();
-  for (size_t i = 0; i < num_sub_sections; i++) {
-    debug_info_size +=
-        GetDebugInfoSizeInSection(section.GetSubSectionAtIndex(i));
-  }
-  return debug_info_size;
-}
-
-static uint64_t GetDebugInfoSize(lldb::SBModule module) {
-  uint64_t debug_info_size = 0;
-  size_t num_sections = module.GetNumSections();
-  for (size_t i = 0; i < num_sections; i++) {
-    debug_info_size += GetDebugInfoSizeInSection(module.GetSectionAtIndex(i));
-  }
-  return debug_info_size;
-}
-
-static std::string ConvertDebugInfoSizeToString(uint64_t debug_info) {
-  std::ostringstream oss;
-  oss << " (";
-  oss << std::fixed << std::setprecision(1);
-
-  if (debug_info < 1024) {
-    oss << debug_info << "B";
-  } else if (debug_info < 1024 * 1024) {
-    double kb = double(debug_info) / 1024.0;
-    oss << kb << "KB";
-  } else if (debug_info < 1024 * 1024 * 1024) {
-    double mb = double(debug_info) / (1024.0 * 1024.0);
-    oss << mb << "MB";
-  } else {
-    double gb = double(debug_info) / (1024.0 * 1024.0 * 1024.0);
-    oss << gb << "GB";
-    ;
-  }
-  oss << ")";
-  return oss.str();
-}
 llvm::json::Value CreateModule(lldb::SBModule &module) {
   llvm::json::Object object;
   if (!module.IsValid())
@@ -385,15 +339,9 @@ llvm::json::Value CreateModule(lldb::SBModule &module) {
   std::string module_path(module_path_arr);
   object.try_emplace("path", module_path);
   if (module.GetNumCompileUnits() > 0) {
-    std::string symbol_str = "Symbols loaded.";
-    uint64_t debug_info = GetDebugInfoSize(module);
-    if (debug_info > 0) {
-      symbol_str += ConvertDebugInfoSizeToString(debug_info);
-    }
-    object.try_emplace("symbolStatus", symbol_str);
+    object.try_emplace("symbolStatus", "Symbols loaded.");
     char symbol_path_arr[PATH_MAX];
-    module.GetSymbolFileSpec().GetPath(symbol_path_arr,
-                                       sizeof(symbol_path_arr));
+    module.GetSymbolFileSpec().GetPath(symbol_path_arr, sizeof(symbol_path_arr));
     std::string symbol_path(symbol_path_arr);
     object.try_emplace("symbolFilePath", symbol_path);
   } else {
@@ -404,9 +352,8 @@ llvm::json::Value CreateModule(lldb::SBModule &module) {
   object.try_emplace("addressRange", loaded_addr);
   std::string version_str;
   uint32_t version_nums[3];
-  uint32_t num_versions =
-      module.GetVersion(version_nums, sizeof(version_nums) / sizeof(uint32_t));
-  for (uint32_t i = 0; i < num_versions; ++i) {
+  uint32_t num_versions = module.GetVersion(version_nums, sizeof(version_nums)/sizeof(uint32_t));
+  for (uint32_t i=0; i<num_versions; ++i) {
     if (!version_str.empty())
       version_str += ".";
     version_str += std::to_string(version_nums[i]);

From 2c1bea88a5f8334015455aae8b842fdd87f7e34a Mon Sep 17 00:00:00 2001
From: Yifan Shen <aelitashen@fb.com>
Date: Fri, 24 Jul 2020 13:30:04 -0700
Subject: [PATCH 0051/1035] Add Debug Info Size to Symbol Status

If a module has debug info, the size of debug symbol will be displayed after the Symbols Loaded Message for each module in the VScode modules view.{F12335461}

Reviewed By: wallace, clayborg

Differential Revision: https://reviews.llvm.org/D83731
---
 .../tools/lldb-vscode/lldbvscode_testcase.py  |  9 +--
 .../API/tools/lldb-vscode/module/Makefile     | 10 ++-
 .../lldb-vscode/module/TestVSCode_module.py   | 67 ++++++++++++++-----
 lldb/tools/lldb-vscode/JSONUtils.cpp          | 61 +++++++++++++++--
 4 files changed, 121 insertions(+), 26 deletions(-)

diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/lldbvscode_testcase.py b/lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/lldbvscode_testcase.py
index c1b33c220b4bd..fa5a9c0db1ebd 100644
--- a/lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/lldbvscode_testcase.py
+++ b/lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/lldbvscode_testcase.py
@@ -53,11 +53,12 @@ def set_function_breakpoints(self, functions, condition=None,
             breakpoint_ids.append('%i' % (breakpoint['id']))
         return breakpoint_ids
 
-    def waitUntil(self, condition):
-        while True:
-            if condition():
-                break
+    def waitUntil(self, condition_callback):
+        for _ in range(20):
+            if condition_callback():
+                return True
             time.sleep(0.5)
+        return False
 
     def verify_breakpoint_hit(self, breakpoint_ids):
         '''Wait for the process we are debugging to stop, and verify we hit
diff --git a/lldb/test/API/tools/lldb-vscode/module/Makefile b/lldb/test/API/tools/lldb-vscode/module/Makefile
index 1fb944b138937..b30baf48b972e 100644
--- a/lldb/test/API/tools/lldb-vscode/module/Makefile
+++ b/lldb/test/API/tools/lldb-vscode/module/Makefile
@@ -2,12 +2,16 @@ DYLIB_NAME := foo
 DYLIB_CXX_SOURCES := foo.cpp
 CXX_SOURCES := main.cpp
 
-all: a.out.stripped
+LD_EXTRAS := -Wl,-rpath "-Wl,$(shell pwd)"
+USE_LIBDL :=1
 
 include Makefile.rules
 
-a.out.stripped: a.out.dSYM
+all: a.out.stripped
+
+a.out.stripped:
 	strip -o a.out.stripped a.out
+
 ifneq "$(CODESIGN)" ""
 	$(CODESIGN) -fs - a.out.stripped
-endif
+endif
\ No newline at end of file
diff --git a/lldb/test/API/tools/lldb-vscode/module/TestVSCode_module.py b/lldb/test/API/tools/lldb-vscode/module/TestVSCode_module.py
index 40c4145b38e36..a16430fccae1d 100644
--- a/lldb/test/API/tools/lldb-vscode/module/TestVSCode_module.py
+++ b/lldb/test/API/tools/lldb-vscode/module/TestVSCode_module.py
@@ -10,56 +10,93 @@
 from lldbsuite.test.lldbtest import *
 from lldbsuite.test import lldbutil
 import lldbvscode_testcase
+import re
 
 class TestVSCode_module(lldbvscode_testcase.VSCodeTestCaseBase):
 
     mydir = TestBase.compute_mydir(__file__)
 
-
-    @skipIfWindows
-    @skipUnlessDarwin
-    @skipIfRemote
-    def test_modules_event(self):
+    def run_test(self, symbol_basename, expect_debug_info_size):
         program_basename = "a.out.stripped"
-        program= self.getBuildArtifact(program_basename)
+        program = self.getBuildArtifact(program_basename)
         self.build_and_launch(program)
         functions = ['foo']
         breakpoint_ids = self.set_function_breakpoints(functions)
-        self.assertEquals(len(breakpoint_ids), len(functions),
-                        'expect one breakpoint')
+        self.assertEquals(len(breakpoint_ids), len(functions), 'expect one breakpoint')
         self.continue_to_breakpoints(breakpoint_ids)
         active_modules = self.vscode.get_active_modules()
-        self.assertIn(program_basename, active_modules, '%s module is in active modules' % (program_basename))
         program_module = active_modules[program_basename]
+        self.assertIn(program_basename, active_modules, '%s module is in active modules' % (program_basename))
         self.assertIn('name', program_module, 'make sure name is in module')
         self.assertEqual(program_basename, program_module['name'])
         self.assertIn('path', program_module, 'make sure path is in module')
         self.assertEqual(program, program_module['path'])
         self.assertTrue('symbolFilePath' not in program_module, 'Make sure a.out.stripped has no debug info')
         self.assertEqual('Symbols not found.', program_module['symbolStatus'])
-        symbol_path = self.getBuildArtifact("a.out")
-        self.vscode.request_evaluate('`%s' % ('target symbols add -s "%s" "%s"' % (program, symbol_path)))
+        symbols_path = self.getBuildArtifact(symbol_basename)
+        self.vscode.request_evaluate('`%s' % ('target symbols add -s "%s" "%s"' % (program, symbols_path)))
 
         def checkSymbolsLoaded():
             active_modules = self.vscode.get_active_modules()
             program_module = active_modules[program_basename]
             return 'Symbols loaded.' == program_module['symbolStatus']
-        self.waitUntil(checkSymbolsLoaded)
 
+        def checkSymbolsLoadedWithSize():
+            active_modules = self.vscode.get_active_modules()
+            program_module = active_modules[program_basename]
+            symbolsStatus = program_module['symbolStatus']
+            symbol_regex = re.compile(r"Symbols loaded. \([0-9]+(\.[0-9]*)?[KMG]?B\)")
+            return symbol_regex.match(program_module['symbolStatus'])
+                
+        if expect_debug_info_size:
+            self.waitUntil(checkSymbolsLoadedWithSize)
+        else:
+            self.waitUntil(checkSymbolsLoaded)
         active_modules = self.vscode.get_active_modules()
         program_module = active_modules[program_basename]
         self.assertEqual(program_basename, program_module['name'])
         self.assertEqual(program, program_module['path'])
-        self.assertEqual('Symbols loaded.', program_module['symbolStatus'])
         self.assertIn('symbolFilePath', program_module)
-        self.assertEqual(symbol_path, program_module['symbolFilePath'])
+        self.assertIn(symbols_path, program_module['symbolFilePath'])
         self.assertIn('addressRange', program_module)
 
+    @skipIfWindows
+    @skipUnlessDarwin
+    @skipIfRemote  
+    #TODO: Update the Makefile so that this test runs on Linux
+    def test_module_event(self):
+        '''
+            Mac or linux.
+
+            On mac, if we load a.out as our symbol file, we will use DWARF with .o files and we will
+            have debug symbols, but we won't see any debug info size because all of the DWARF
+            sections are in .o files.
+
+            On other platforms, we expect a.out to have debug info, so we will expect a size.
+            expect_debug_info_size = platform.system() != 'Darwin'
+            return self.run_test("a.out", expect_debug_info_size)
+        '''
+        expect_debug_info_size = platform.system() != 'Darwin'
+        return self.run_test("a.out", expect_debug_info_size)
+
+    @skipIfWindows
+    @skipUnlessDarwin
+    @skipIfRemote    
+    def test_module_event_dsym(self):
+        '''
+            Darwin only test with dSYM file.
+
+            On mac, if we load a.out.dSYM as our symbol file, we will have debug symbols and we
+            will have DWARF sections added to the module, so we will expect a size.
+            return self.run_test("a.out.dSYM", True)
+        '''
+        return self.run_test("a.out.dSYM", True)
+
     @skipIfWindows
     @skipUnlessDarwin
     @skipIfRemote
     def test_compile_units(self):
-        program= self.getBuildArtifact("a.out")
+        program = self.getBuildArtifact("a.out")
         self.build_and_launch(program)
         source = "main.cpp"
         main_source_path = self.getSourcePath(source)
diff --git a/lldb/tools/lldb-vscode/JSONUtils.cpp b/lldb/tools/lldb-vscode/JSONUtils.cpp
index 1ebaa5c377121..f6cdcf5a46cfc 100644
--- a/lldb/tools/lldb-vscode/JSONUtils.cpp
+++ b/lldb/tools/lldb-vscode/JSONUtils.cpp
@@ -7,6 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #include <algorithm>
+#include <iomanip>
+#include <sstream>
 
 #include "llvm/ADT/Optional.h"
 #include "llvm/Support/FormatAdapters.h"
@@ -327,6 +329,50 @@ llvm::json::Value CreateBreakpoint(lldb::SBBreakpoint &bp,
   return llvm::json::Value(std::move(object));
 }
 
+static uint64_t GetDebugInfoSizeInSection(lldb::SBSection section) {
+  uint64_t debug_info_size = 0;
+  llvm::StringRef section_name(section.GetName());
+  if (section_name.startswith(".debug") || section_name.startswith("__debug") ||
+      section_name.startswith(".apple") || section_name.startswith("__apple"))
+    debug_info_size += section.GetFileByteSize();
+  size_t num_sub_sections = section.GetNumSubSections();
+  for (size_t i = 0; i < num_sub_sections; i++) {
+    debug_info_size +=
+        GetDebugInfoSizeInSection(section.GetSubSectionAtIndex(i));
+  }
+  return debug_info_size;
+}
+
+static uint64_t GetDebugInfoSize(lldb::SBModule module) {
+  uint64_t debug_info_size = 0;
+  size_t num_sections = module.GetNumSections();
+  for (size_t i = 0; i < num_sections; i++) {
+    debug_info_size += GetDebugInfoSizeInSection(module.GetSectionAtIndex(i));
+  }
+  return debug_info_size;
+}
+
+static std::string ConvertDebugInfoSizeToString(uint64_t debug_info) {
+  std::ostringstream oss;
+  oss << " (";
+  oss << std::fixed << std::setprecision(1);
+
+  if (debug_info < 1024) {
+    oss << debug_info << "B";
+  } else if (debug_info < 1024 * 1024) {
+    double kb = double(debug_info) / 1024.0;
+    oss << kb << "KB";
+  } else if (debug_info < 1024 * 1024 * 1024) {
+    double mb = double(debug_info) / (1024.0 * 1024.0);
+    oss << mb << "MB";
+  } else {
+    double gb = double(debug_info) / (1024.0 * 1024.0 * 1024.0);
+    oss << gb << "GB";
+    ;
+  }
+  oss << ")";
+  return oss.str();
+}
 llvm::json::Value CreateModule(lldb::SBModule &module) {
   llvm::json::Object object;
   if (!module.IsValid())
@@ -339,9 +385,15 @@ llvm::json::Value CreateModule(lldb::SBModule &module) {
   std::string module_path(module_path_arr);
   object.try_emplace("path", module_path);
   if (module.GetNumCompileUnits() > 0) {
-    object.try_emplace("symbolStatus", "Symbols loaded.");
+    std::string symbol_str = "Symbols loaded.";
+    uint64_t debug_info = GetDebugInfoSize(module);
+    if (debug_info > 0) {
+      symbol_str += ConvertDebugInfoSizeToString(debug_info);
+    }
+    object.try_emplace("symbolStatus", symbol_str);
     char symbol_path_arr[PATH_MAX];
-    module.GetSymbolFileSpec().GetPath(symbol_path_arr, sizeof(symbol_path_arr));
+    module.GetSymbolFileSpec().GetPath(symbol_path_arr,
+                                       sizeof(symbol_path_arr));
     std::string symbol_path(symbol_path_arr);
     object.try_emplace("symbolFilePath", symbol_path);
   } else {
@@ -352,8 +404,9 @@ llvm::json::Value CreateModule(lldb::SBModule &module) {
   object.try_emplace("addressRange", loaded_addr);
   std::string version_str;
   uint32_t version_nums[3];
-  uint32_t num_versions = module.GetVersion(version_nums, sizeof(version_nums)/sizeof(uint32_t));
-  for (uint32_t i=0; i<num_versions; ++i) {
+  uint32_t num_versions =
+      module.GetVersion(version_nums, sizeof(version_nums) / sizeof(uint32_t));
+  for (uint32_t i = 0; i < num_versions; ++i) {
     if (!version_str.empty())
       version_str += ".";
     version_str += std::to_string(version_nums[i]);

From c86f56e32e724c6018e579bb2bc11e667c96fc96 Mon Sep 17 00:00:00 2001
From: Petr Hosek <phosek@google.com>
Date: Fri, 24 Jul 2020 13:36:13 -0700
Subject: [PATCH 0052/1035] [CMake] Find zlib when building lldb as standalone

This addresses the issue introduced by 10b1b4a.
---
 lldb/cmake/modules/LLDBStandalone.cmake | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/lldb/cmake/modules/LLDBStandalone.cmake b/lldb/cmake/modules/LLDBStandalone.cmake
index 752113bcc6c41..edd2b34ec8655 100644
--- a/lldb/cmake/modules/LLDBStandalone.cmake
+++ b/lldb/cmake/modules/LLDBStandalone.cmake
@@ -73,6 +73,11 @@ endif()
 # We append the directory in which LLVMConfig.cmake lives. We expect LLVM's
 # CMake modules to be in that directory as well.
 list(APPEND CMAKE_MODULE_PATH "${LLVM_DIR}")
+
+if(LLVM_ENABLE_ZLIB)
+  find_package(ZLIB)
+endif()
+
 include(AddLLVM)
 include(TableGen)
 include(HandleLLVMOptions)

From 2bd72abef0f24d9f7da13af3a43a25409159a87e Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Wed, 22 Jul 2020 21:24:21 -0400
Subject: [PATCH 0053/1035] AMDGPU: Skip other terminators before inserting
 s_cbranch_exec[n]z

PHIElimination/createPHISourceCopy inserts non-branch terminators
after the control flow pseudo if a successor phi reads a register
defined by the control flow pseudo. If this happens, we need to split
the expansion of the control flow pseudo to ensure all the branches
are after all of the other mask management instructions.

GlobalISel hit this in testscases that happened to be tail
duplicated. The original testcase still does not work, since the same
problem appears to be present in a later pass.
---
 llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp |  24 +-
 .../lower-control-flow-other-terminators.mir  | 246 ++++++++++++++++++
 .../AMDGPU/si-if-lower-user-terminators.mir   |  75 ------
 3 files changed, 269 insertions(+), 76 deletions(-)
 create mode 100644 llvm/test/CodeGen/AMDGPU/lower-control-flow-other-terminators.mir
 delete mode 100644 llvm/test/CodeGen/AMDGPU/si-if-lower-user-terminators.mir

diff --git a/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp b/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp
index 36d52ac3ee891..140e1f0e122a9 100644
--- a/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp
+++ b/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp
@@ -122,6 +122,19 @@ class SILowerControlFlow : public MachineFunctionPass {
   skipIgnoreExecInstsTrivialSucc(MachineBasicBlock &MBB,
                                  MachineBasicBlock::iterator It) const;
 
+  /// Find the insertion point for a new conditional branch.
+  MachineBasicBlock::iterator
+  skipToUncondBrOrEnd(MachineBasicBlock &MBB,
+                      MachineBasicBlock::iterator I) const {
+    assert(I->isTerminator());
+
+    // FIXME: What if we had multiple pre-existing conditional branches?
+    MachineBasicBlock::iterator End = MBB.end();
+    while (I != End && !I->isUnconditionalBranch())
+      ++I;
+    return I;
+  }
+
   // Remove redundant SI_END_CF instructions.
   void optimizeEndCf();
 
@@ -275,6 +288,10 @@ void SILowerControlFlow::emitIf(MachineInstr &MI) {
     BuildMI(MBB, I, DL, TII->get(MovTermOpc), Exec)
     .addReg(Tmp, RegState::Kill);
 
+  // Skip ahead to the unconditional branch in case there are other terminators
+  // present.
+  I = skipToUncondBrOrEnd(MBB, I);
+
   // Insert the S_CBRANCH_EXECZ instruction which will be optimized later
   // during SIRemoveShortExecBranches.
   MachineInstr *NewBr = BuildMI(MBB, I, DL, TII->get(AMDGPU::S_CBRANCH_EXECZ))
@@ -353,6 +370,10 @@ void SILowerControlFlow::emitElse(MachineInstr &MI) {
     .addReg(Exec)
     .addReg(DstReg);
 
+  // Skip ahead to the unconditional branch in case there are other terminators
+  // present.
+  ElsePt = skipToUncondBrOrEnd(MBB, ElsePt);
+
   MachineInstr *Branch =
       BuildMI(MBB, ElsePt, DL, TII->get(AMDGPU::S_CBRANCH_EXECZ))
           .addMBB(DestBB);
@@ -435,8 +456,9 @@ void SILowerControlFlow::emitLoop(MachineInstr &MI) {
           .addReg(Exec)
           .add(MI.getOperand(0));
 
+  auto BranchPt = skipToUncondBrOrEnd(MBB, MI.getIterator());
   MachineInstr *Branch =
-      BuildMI(MBB, &MI, DL, TII->get(AMDGPU::S_CBRANCH_EXECNZ))
+      BuildMI(MBB, BranchPt, DL, TII->get(AMDGPU::S_CBRANCH_EXECNZ))
           .add(MI.getOperand(1));
 
   if (LIS) {
diff --git a/llvm/test/CodeGen/AMDGPU/lower-control-flow-other-terminators.mir b/llvm/test/CodeGen/AMDGPU/lower-control-flow-other-terminators.mir
new file mode 100644
index 0000000000000..08e6f1a067ac5
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/lower-control-flow-other-terminators.mir
@@ -0,0 +1,246 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs -run-pass=si-lower-control-flow -o - %s | FileCheck %s
+
+# Test si-lower-control-flow insertion points when other terminator
+# instructions are present besides the control flow pseudo and a
+# branch.
+
+
+# There's another terminator instruction between SI_IF and
+# S_BRANCH. The S_CBRANCH_EXECZ should be inserted immediately before
+# S_BRANCH.
+---
+name: other_terminator_sbranch_after_si_if
+tracksRegLiveness: true
+body:             |
+  ; CHECK-LABEL: name: other_terminator_sbranch_after_si_if
+  ; CHECK: bb.0:
+  ; CHECK:   successors: %bb.2(0x40000000), %bb.1(0x40000000)
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $sgpr4_sgpr5
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY killed $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_64_xexec = COPY $sgpr4_sgpr5
+  ; CHECK:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 0, [[COPY]], implicit $exec
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_64 = COPY $exec, implicit-def $exec
+  ; CHECK:   [[S_AND_B64_:%[0-9]+]]:sreg_64 = S_AND_B64 [[COPY2]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
+  ; CHECK:   [[S_XOR_B64_:%[0-9]+]]:sreg_64_xexec = S_XOR_B64 [[S_AND_B64_]], [[COPY2]], implicit-def dead $scc
+  ; CHECK:   $exec = S_MOV_B64_term killed [[S_AND_B64_]]
+  ; CHECK:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term killed [[COPY1]], implicit $exec
+  ; CHECK:   S_CBRANCH_EXECZ %bb.1, implicit $exec
+  ; CHECK:   S_BRANCH %bb.2
+  ; CHECK: bb.1:
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   S_BRANCH %bb.2
+  ; CHECK: bb.2:
+  ; CHECK:   S_ENDPGM 0, implicit [[S_MOV_B64_term]]
+  bb.0:
+    successors: %bb.2, %bb.1
+    liveins: $vgpr0, $vgpr1, $sgpr4_sgpr5
+
+    %0:vgpr_32 = COPY killed $vgpr0
+    %1:sreg_64_xexec = COPY $sgpr4_sgpr5
+    %2:sreg_64_xexec = V_CMP_EQ_U32_e64 0, %0, implicit $exec
+    %3:sreg_64_xexec = SI_IF %2, %bb.1, implicit-def $exec, implicit-def dead $scc, implicit $exec
+    %4:sreg_64_xexec = S_MOV_B64_term killed %1, implicit $exec
+    S_BRANCH %bb.2
+
+  bb.1:
+    S_BRANCH %bb.2
+
+  bb.2:
+    S_ENDPGM 0, implicit %4
+
+...
+
+# S_CBRANCH_EXECZ should be inserted after the other terminator
+---
+name: other_terminator_fallthrough_after_si_if
+tracksRegLiveness: true
+body:             |
+  ; CHECK-LABEL: name: other_terminator_fallthrough_after_si_if
+  ; CHECK: bb.0:
+  ; CHECK:   successors: %bb.2(0x40000000), %bb.1(0x40000000)
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $sgpr4_sgpr5
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY killed $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_64_xexec = COPY $sgpr4_sgpr5
+  ; CHECK:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 0, [[COPY]], implicit $exec
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_64 = COPY $exec, implicit-def $exec
+  ; CHECK:   [[S_AND_B64_:%[0-9]+]]:sreg_64 = S_AND_B64 [[COPY2]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
+  ; CHECK:   [[S_XOR_B64_:%[0-9]+]]:sreg_64_xexec = S_XOR_B64 [[S_AND_B64_]], [[COPY2]], implicit-def dead $scc
+  ; CHECK:   $exec = S_MOV_B64_term killed [[S_AND_B64_]]
+  ; CHECK:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term killed [[COPY1]], implicit $exec
+  ; CHECK:   S_CBRANCH_EXECZ %bb.2, implicit $exec
+  ; CHECK: bb.1:
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   S_BRANCH %bb.2
+  ; CHECK: bb.2:
+  ; CHECK:   S_ENDPGM 0, implicit [[S_MOV_B64_term]]
+  bb.0:
+    successors: %bb.2, %bb.1
+    liveins: $vgpr0, $vgpr1, $sgpr4_sgpr5
+
+    %0:vgpr_32 = COPY killed $vgpr0
+    %1:sreg_64_xexec = COPY $sgpr4_sgpr5
+    %2:sreg_64_xexec = V_CMP_EQ_U32_e64 0, %0, implicit $exec
+    %3:sreg_64_xexec = SI_IF %2, %bb.2, implicit-def $exec, implicit-def dead $scc, implicit $exec
+    %4:sreg_64_xexec = S_MOV_B64_term killed %1, implicit $exec
+
+  bb.1:
+    S_BRANCH %bb.2
+
+  bb.2:
+    S_ENDPGM 0, implicit %4
+
+...
+
+---
+name: other_terminator_sbranch_after_si_else
+tracksRegLiveness: true
+body:             |
+  ; CHECK-LABEL: name: other_terminator_sbranch_after_si_else
+  ; CHECK: bb.0:
+  ; CHECK:   successors: %bb.2(0x40000000), %bb.1(0x40000000)
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $sgpr4_sgpr5
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_64 = COPY %2
+  ; CHECK:   [[S_OR_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_OR_SAVEEXEC_B64 [[COPY]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY killed $vgpr0
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_64_xexec = COPY $sgpr4_sgpr5
+  ; CHECK:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 0, [[COPY1]], implicit $exec
+  ; CHECK:   $exec = S_XOR_B64_term $exec, [[S_OR_SAVEEXEC_B64_]], implicit-def $scc
+  ; CHECK:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term killed [[COPY2]], implicit $exec
+  ; CHECK:   S_CBRANCH_EXECZ %bb.1, implicit $exec
+  ; CHECK:   S_BRANCH %bb.2
+  ; CHECK: bb.1:
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   S_BRANCH %bb.2
+  ; CHECK: bb.2:
+  ; CHECK:   S_ENDPGM 0, implicit [[S_MOV_B64_term]]
+  bb.0:
+    successors: %bb.2, %bb.1
+    liveins: $vgpr0, $vgpr1, $sgpr4_sgpr5
+
+    %0:vgpr_32 = COPY killed $vgpr0
+    %1:sreg_64_xexec = COPY $sgpr4_sgpr5
+    %2:sreg_64_xexec = V_CMP_EQ_U32_e64 0, %0, implicit $exec
+    %3:sreg_64_xexec = SI_ELSE %2, %bb.1, 0, implicit-def $exec, implicit-def dead $scc, implicit $exec
+    %4:sreg_64_xexec = S_MOV_B64_term killed %1, implicit $exec
+    S_BRANCH %bb.2
+
+  bb.1:
+    S_BRANCH %bb.2
+
+  bb.2:
+    S_ENDPGM 0, implicit %4
+
+...
+
+---
+name: other_terminator_sbranch_after_si_loop
+tracksRegLiveness: true
+body:             |
+  ; CHECK-LABEL: name: other_terminator_sbranch_after_si_loop
+  ; CHECK: bb.0:
+  ; CHECK:   successors: %bb.2(0x40000000), %bb.1(0x40000000)
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $sgpr4_sgpr5
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY killed $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_64_xexec = COPY $sgpr4_sgpr5
+  ; CHECK:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 0, [[COPY]], implicit $exec
+  ; CHECK:   $exec = S_ANDN2_B64_term $exec, [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; CHECK:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term killed [[COPY1]], implicit $exec
+  ; CHECK:   S_CBRANCH_EXECNZ %bb.1, implicit $exec
+  ; CHECK:   S_BRANCH %bb.2
+  ; CHECK: bb.1:
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   S_BRANCH %bb.2
+  ; CHECK: bb.2:
+  ; CHECK:   S_ENDPGM 0, implicit [[S_MOV_B64_term]]
+  bb.0:
+    successors: %bb.2, %bb.1
+    liveins: $vgpr0, $vgpr1, $sgpr4_sgpr5
+
+    %0:vgpr_32 = COPY killed $vgpr0
+    %1:sreg_64_xexec = COPY $sgpr4_sgpr5
+    %2:sreg_64_xexec = V_CMP_EQ_U32_e64 0, %0, implicit $exec
+    SI_LOOP %2, %bb.1, implicit-def $exec, implicit-def dead $scc, implicit $exec
+    %4:sreg_64_xexec = S_MOV_B64_term killed %1, implicit $exec
+    S_BRANCH %bb.2
+
+  bb.1:
+    S_BRANCH %bb.2
+
+  bb.2:
+    S_ENDPGM 0, implicit %4
+
+...
+
+# The save exec result register of SI_IF is used by other terminators
+# inserted to behave as a lowered phi. The output register of SI_IF
+# was ignored, and the def was removed, so the S_MOV_B64_term uses
+# would fail the verifier.
+
+---
+name:            si_if_use
+alignment:       1
+legalized:       true
+regBankSelected: true
+selected:        true
+tracksRegLiveness: true
+body:             |
+  ; CHECK-LABEL: name: si_if_use
+  ; CHECK: bb.0:
+  ; CHECK:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY killed $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY killed $vgpr1
+  ; CHECK:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 killed [[COPY]], killed [[COPY1]], implicit $exec
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_64 = COPY $exec, implicit-def $exec
+  ; CHECK:   [[S_AND_B64_:%[0-9]+]]:sreg_64 = S_AND_B64 [[COPY2]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
+  ; CHECK:   [[S_XOR_B64_:%[0-9]+]]:sreg_64_xexec = S_XOR_B64 [[S_AND_B64_]], [[COPY2]], implicit-def dead $scc
+  ; CHECK:   $exec = S_MOV_B64_term killed [[S_AND_B64_]]
+  ; CHECK:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term [[S_XOR_B64_]], implicit $exec
+  ; CHECK:   [[S_MOV_B64_term1:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term [[S_XOR_B64_]], implicit $exec
+  ; CHECK:   S_CBRANCH_EXECZ %bb.1, implicit $exec
+  ; CHECK:   S_BRANCH %bb.2
+  ; CHECK: bb.1:
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_64_xexec = COPY [[S_MOV_B64_term1]]
+  ; CHECK:   dead %7:vgpr_32 = GLOBAL_LOAD_DWORD undef %8:vreg_64, 0, 0, 0, 0, implicit $exec :: (volatile load 4, addrspace 1)
+  ; CHECK:   [[COPY4:%[0-9]+]]:sreg_64_xexec = COPY [[COPY3]]
+  ; CHECK: bb.2:
+  ; CHECK:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   [[COPY5:%[0-9]+]]:sreg_64_xexec = COPY [[COPY4]]
+  ; CHECK:   $exec = S_OR_B64 $exec, killed [[COPY5]], implicit-def $scc
+  ; CHECK:   S_SLEEP 1
+  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_64 = COPY $exec, implicit-def $exec
+  ; CHECK:   [[S_AND_B64_1:%[0-9]+]]:sreg_64 = S_AND_B64 [[COPY6]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
+  ; CHECK:   [[S_XOR_B64_1:%[0-9]+]]:sreg_64_xexec = S_XOR_B64 [[S_AND_B64_1]], [[COPY6]], implicit-def dead $scc
+  ; CHECK:   $exec = S_MOV_B64_term killed [[S_AND_B64_1]]
+  ; CHECK:   [[S_MOV_B64_term1:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term [[S_XOR_B64_1]], implicit $exec
+  ; CHECK:   [[S_MOV_B64_term2:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term [[S_XOR_B64_1]], implicit $exec
+  ; CHECK:   S_CBRANCH_EXECZ %bb.1, implicit $exec
+  ; CHECK:   S_BRANCH %bb.2
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $sgpr30_sgpr31
+
+    %0:vgpr_32 = COPY killed $vgpr0
+    %1:vgpr_32 = COPY killed $vgpr1
+    %3:sreg_64_xexec = V_CMP_EQ_U32_e64 killed %0, killed %1, implicit $exec
+    %10:sreg_64_xexec = SI_IF %3, %bb.1, implicit-def $exec, implicit-def dead $scc, implicit $exec
+    %14:sreg_64_xexec = S_MOV_B64_term %10, implicit $exec
+    %13:sreg_64_xexec = S_MOV_B64_term %10, implicit $exec
+    S_BRANCH %bb.2
+
+  bb.1:
+    %11:sreg_64_xexec = COPY %13
+    dead %6:vgpr_32 = GLOBAL_LOAD_DWORD undef %8:vreg_64, 0, 0, 0, 0, implicit $exec :: (volatile load 4, addrspace 1)
+    %14:sreg_64_xexec = COPY %11
+
+  bb.2:
+    %12:sreg_64_xexec = COPY %14
+    SI_END_CF killed %12, implicit-def $exec, implicit-def dead $scc, implicit $exec
+    S_SLEEP 1
+    %9:sreg_64_xexec = SI_IF %3, %bb.1, implicit-def $exec, implicit-def dead $scc, implicit $exec
+    %14:sreg_64_xexec = S_MOV_B64_term %9, implicit $exec
+    %13:sreg_64_xexec = S_MOV_B64_term %9, implicit $exec
+    S_BRANCH %bb.2
+
+...
diff --git a/llvm/test/CodeGen/AMDGPU/si-if-lower-user-terminators.mir b/llvm/test/CodeGen/AMDGPU/si-if-lower-user-terminators.mir
deleted file mode 100644
index 5850a3b27bce8..0000000000000
--- a/llvm/test/CodeGen/AMDGPU/si-if-lower-user-terminators.mir
+++ /dev/null
@@ -1,75 +0,0 @@
-# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -run-pass=si-lower-control-flow -verify-machineinstrs -o - %s | FileCheck %s
-
-# The save exec result register of SI_IF is used by other terminators
-# inserted to behave as a lowered phi. The output register of SI_IF
-# was ignored, and the def was removed, so the S_MOV_B64_term uses
-# would fail the verifier.
-
----
-name:            si_if_use
-alignment:       1
-legalized:       true
-regBankSelected: true
-selected:        true
-tracksRegLiveness: true
-body:             |
-  ; CHECK-LABEL: name: si_if_use
-  ; CHECK: bb.0:
-  ; CHECK:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
-  ; CHECK:   liveins: $vgpr0, $vgpr1, $sgpr30_sgpr31
-  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY killed $vgpr0
-  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY killed $vgpr1
-  ; CHECK:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 killed [[COPY]], killed [[COPY1]], implicit $exec
-  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_64 = COPY $exec, implicit-def $exec
-  ; CHECK:   [[S_AND_B64_:%[0-9]+]]:sreg_64 = S_AND_B64 [[COPY2]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
-  ; CHECK:   [[S_XOR_B64_:%[0-9]+]]:sreg_64_xexec = S_XOR_B64 [[S_AND_B64_]], [[COPY2]], implicit-def dead $scc
-  ; CHECK:   $exec = S_MOV_B64_term killed [[S_AND_B64_]]
-  ; CHECK:   S_CBRANCH_EXECZ %bb.1, implicit $exec
-  ; CHECK:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term [[S_XOR_B64_]], implicit $exec
-  ; CHECK:   [[S_MOV_B64_term1:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term [[S_XOR_B64_]], implicit $exec
-  ; CHECK:   S_BRANCH %bb.2
-  ; CHECK: bb.1:
-  ; CHECK:   successors: %bb.2(0x80000000)
-  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_64_xexec = COPY [[S_MOV_B64_term1]]
-  ; CHECK:   dead %7:vgpr_32 = GLOBAL_LOAD_DWORD undef %8:vreg_64, 0, 0, 0, 0, implicit $exec :: (volatile load 4, addrspace 1)
-  ; CHECK:   [[COPY4:%[0-9]+]]:sreg_64_xexec = COPY [[COPY3]]
-  ; CHECK: bb.2:
-  ; CHECK:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
-  ; CHECK:   [[COPY5:%[0-9]+]]:sreg_64_xexec = COPY [[COPY4]]
-  ; CHECK:   $exec = S_OR_B64 $exec, killed [[COPY5]], implicit-def $scc
-  ; CHECK:   S_SLEEP 1
-  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_64 = COPY $exec, implicit-def $exec
-  ; CHECK:   [[S_AND_B64_1:%[0-9]+]]:sreg_64 = S_AND_B64 [[COPY6]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
-  ; CHECK:   [[S_XOR_B64_1:%[0-9]+]]:sreg_64_xexec = S_XOR_B64 [[S_AND_B64_1]], [[COPY6]], implicit-def dead $scc
-  ; CHECK:   $exec = S_MOV_B64_term killed [[S_AND_B64_1]]
-  ; CHECK:   S_CBRANCH_EXECZ %bb.1, implicit $exec
-  ; CHECK:   [[S_MOV_B64_term1:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term [[S_XOR_B64_1]], implicit $exec
-  ; CHECK:   [[S_MOV_B64_term2:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term [[S_XOR_B64_1]], implicit $exec
-  ; CHECK:   S_BRANCH %bb.2
-  bb.0:
-    liveins: $vgpr0, $vgpr1, $sgpr30_sgpr31
-
-    %0:vgpr_32 = COPY killed $vgpr0
-    %1:vgpr_32 = COPY killed $vgpr1
-    %3:sreg_64_xexec = V_CMP_EQ_U32_e64 killed %0, killed %1, implicit $exec
-    %10:sreg_64_xexec = SI_IF %3, %bb.1, implicit-def $exec, implicit-def dead $scc, implicit $exec
-    %14:sreg_64_xexec = S_MOV_B64_term %10, implicit $exec
-    %13:sreg_64_xexec = S_MOV_B64_term %10, implicit $exec
-    S_BRANCH %bb.2
-
-  bb.1:
-    %11:sreg_64_xexec = COPY %13
-    dead %6:vgpr_32 = GLOBAL_LOAD_DWORD undef %8:vreg_64, 0, 0, 0, 0, implicit $exec :: (volatile load 4, addrspace 1)
-    %14:sreg_64_xexec = COPY %11
-
-  bb.2:
-    %12:sreg_64_xexec = COPY %14
-    SI_END_CF killed %12, implicit-def $exec, implicit-def dead $scc, implicit $exec
-    S_SLEEP 1
-    %9:sreg_64_xexec = SI_IF %3, %bb.1, implicit-def $exec, implicit-def dead $scc, implicit $exec
-    %14:sreg_64_xexec = S_MOV_B64_term %9, implicit $exec
-    %13:sreg_64_xexec = S_MOV_B64_term %9, implicit $exec
-    S_BRANCH %bb.2
-
-...

From 4dc3014c51fda2a3189318c4ae54c4da9cfc6a0e Mon Sep 17 00:00:00 2001
From: Jon Roelofs <jonathan_roelofs@apple.com>
Date: Fri, 24 Jul 2020 14:54:17 -0600
Subject: [PATCH 0054/1035] [compiler-rt][fuzzer] Disable bcmp.test on darwin

It broke one of the buildbots:

http://lab.llvm.org:8080/green/job/clang-stage1-RA/13026/console
---
 compiler-rt/test/fuzzer/bcmp.test | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/compiler-rt/test/fuzzer/bcmp.test b/compiler-rt/test/fuzzer/bcmp.test
index 5bbbe9845beb2..8c1e532b16aba 100644
--- a/compiler-rt/test/fuzzer/bcmp.test
+++ b/compiler-rt/test/fuzzer/bcmp.test
@@ -1,4 +1,4 @@
-UNSUPPORTED: freebsd, windows
+UNSUPPORTED: darwin, freebsd, windows
 RUN: %cpp_compiler -DMEMCMP=bcmp %S/MemcmpTest.cpp -o %t
 RUN: not %run %t -seed=1 -runs=10000000 2>&1 | FileCheck %s
 CHECK: BINGO

From 3554cf4f382c7f18beb1265f5199651470c438d8 Mon Sep 17 00:00:00 2001
From: Jinsong Ji <jji@us.ibm.com>
Date: Fri, 24 Jul 2020 20:55:52 +0000
Subject: [PATCH 0055/1035] [compiler-rt][CMake] Remove unused -stdlib when
 passing -nostdinc++

We added -nostdinc++ to clang_rt.profile in https://reviews.llvm.org/D84205.
This will cause warnings when building with LLVM_ENABLE_LIBCXX,
and failure if with Werror on.

This patch is to fix it by removing unused -stdlib,
similar to what we have done in https://reviews.llvm.org/D42238.

Reviewed By: phosek

Differential Revision: https://reviews.llvm.org/D84543
---
 compiler-rt/lib/profile/CMakeLists.txt | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/compiler-rt/lib/profile/CMakeLists.txt b/compiler-rt/lib/profile/CMakeLists.txt
index 5ff0e10182b4d..a968009f9ea90 100644
--- a/compiler-rt/lib/profile/CMakeLists.txt
+++ b/compiler-rt/lib/profile/CMakeLists.txt
@@ -113,6 +113,8 @@ endif()
 
 # We don't use the C++ Standard Library here, so avoid including it by mistake.
 append_list_if(COMPILER_RT_HAS_NOSTDINCXX_FLAG -nostdinc++ EXTRA_FLAGS)
+# Remove -stdlib= which is unused when passing -nostdinc++.
+string(REGEX REPLACE "-stdlib=[a-zA-Z+]*" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
 
 # This appears to be a C-only warning banning the use of locals in aggregate
 # initializers. All other compilers accept this, though.

From 8bf4c1f4fb257774f66c8cda07adc6c5e8668326 Mon Sep 17 00:00:00 2001
From: Alina Sbirlea <asbirlea@google.com>
Date: Thu, 9 Apr 2020 18:29:40 -0700
Subject: [PATCH 0056/1035] Reapply "[DomTree] Replace ChildrenGetter with
 GraphTraits over GraphDiff."

This is the part of the patch that's moving the Updates to a CFGDiff
object. Splitting off from the clean-up work merging the two branches when BUI is null.

Differential Revision: https://reviews.llvm.org/D77341
---
 llvm/include/llvm/IR/Dominators.h             |   9 +-
 llvm/include/llvm/Support/CFGDiff.h           |  42 ++++-
 llvm/include/llvm/Support/GenericDomTree.h    |  11 +-
 .../llvm/Support/GenericDomTreeConstruction.h | 170 +++++-------------
 llvm/lib/IR/Dominators.cpp                    |   5 +-
 5 files changed, 98 insertions(+), 139 deletions(-)

diff --git a/llvm/include/llvm/IR/Dominators.h b/llvm/include/llvm/IR/Dominators.h
index 71595cb15df48..7b34309cb1319 100644
--- a/llvm/include/llvm/IR/Dominators.h
+++ b/llvm/include/llvm/IR/Dominators.h
@@ -44,6 +44,9 @@ using BBPostDomTree = PostDomTreeBase<BasicBlock>;
 
 using BBUpdates = ArrayRef<llvm::cfg::Update<BasicBlock *>>;
 
+using BBDomTreeGraphDiff = GraphDiff<BasicBlock *, false>;
+using BBPostDomTreeGraphDiff = GraphDiff<BasicBlock *, true>;
+
 extern template void Calculate<BBDomTree>(BBDomTree &DT);
 extern template void CalculateWithUpdates<BBDomTree>(BBDomTree &DT,
                                                      BBUpdates U);
@@ -62,8 +65,10 @@ extern template void DeleteEdge<BBPostDomTree>(BBPostDomTree &DT,
                                                BasicBlock *From,
                                                BasicBlock *To);
 
-extern template void ApplyUpdates<BBDomTree>(BBDomTree &DT, BBUpdates);
-extern template void ApplyUpdates<BBPostDomTree>(BBPostDomTree &DT, BBUpdates);
+extern template void ApplyUpdates<BBDomTree>(BBDomTree &DT,
+                                             BBDomTreeGraphDiff &);
+extern template void ApplyUpdates<BBPostDomTree>(BBPostDomTree &DT,
+                                                 BBPostDomTreeGraphDiff &);
 
 extern template bool Verify<BBDomTree>(const BBDomTree &DT,
                                        BBDomTree::VerificationLevel VL);
diff --git a/llvm/include/llvm/Support/CFGDiff.h b/llvm/include/llvm/Support/CFGDiff.h
index 94734ce70e02c..269984872bfac 100644
--- a/llvm/include/llvm/Support/CFGDiff.h
+++ b/llvm/include/llvm/Support/CFGDiff.h
@@ -70,6 +70,23 @@
 
 namespace llvm {
 
+namespace detail {
+template <typename Range>
+auto reverse_if_helper(Range &&R, std::integral_constant<bool, false>) {
+  return std::forward<Range>(R);
+}
+
+template <typename Range>
+auto reverse_if_helper(Range &&R, std::integral_constant<bool, true>) {
+  return llvm::reverse(std::forward<Range>(R));
+}
+
+template <bool B, typename Range> auto reverse_if(Range &&R) {
+  return reverse_if_helper(std::forward<Range>(R),
+                           std::integral_constant<bool, B>{});
+}
+} // namespace detail
+
 // GraphDiff defines a CFG snapshot: given a set of Update<NodePtr>, provide
 // utilities to skip edges marked as deleted and return a set of edges marked as
 // newly inserted. The current diff treats the CFG as a graph rather than a
@@ -113,8 +130,7 @@ template <typename NodePtr, bool InverseGraph = false> class GraphDiff {
   GraphDiff() : UpdatedAreReverseApplied(false) {}
   GraphDiff(ArrayRef<cfg::Update<NodePtr>> Updates,
             bool ReverseApplyUpdates = false) {
-    cfg::LegalizeUpdates<NodePtr>(Updates, LegalizedUpdates, InverseGraph,
-                                  /*ReverseResultOrder=*/true);
+    cfg::LegalizeUpdates<NodePtr>(Updates, LegalizedUpdates, InverseGraph);
     // The legalized updates are stored in reverse so we can pop_back when doing
     // incremental updates.
     for (auto U : LegalizedUpdates) {
@@ -174,6 +190,25 @@ template <typename NodePtr, bool InverseGraph = false> class GraphDiff {
     return make_range(It->second.begin(), It->second.end());
   }
 
+  using VectRet = SmallVector<NodePtr, 8>;
+
+  template <bool InverseEdge> VectRet getChildren(NodePtr N) const {
+    using DirectedNodeT =
+        std::conditional_t<InverseEdge, Inverse<NodePtr>, NodePtr>;
+    auto R = children<DirectedNodeT>(N);
+    auto CurrentCFGChildren = detail::reverse_if<!InverseEdge>(R);
+
+    VectRet UpdatedCFGChildren;
+    for (auto Child : CurrentCFGChildren)
+      if (Child && !ignoreChild(N, Child, InverseEdge))
+        UpdatedCFGChildren.push_back(Child);
+
+    auto AddedCFGChildren = getAddedChildren(N, InverseEdge);
+    UpdatedCFGChildren.insert(UpdatedCFGChildren.end(),
+                              AddedCFGChildren.begin(), AddedCFGChildren.end());
+    return UpdatedCFGChildren;
+  }
+
   void print(raw_ostream &OS) const {
     OS << "===== GraphDiff: CFG edge changes to create a CFG snapshot. \n"
           "===== (Note: notion of children/inverse_children depends on "
@@ -210,9 +245,10 @@ struct CFGViewChildren {
 
     // filter iterator init:
     auto R = make_range(GT::child_begin(N.second), GT::child_end(N.second));
+    auto RR = detail::reverse_if<!InverseEdge>(R);
     // This lambda is copied into the iterators and persists to callers, ensure
     // captures are by value or otherwise have sufficient lifetime.
-    auto First = make_filter_range(makeChildRange(R, N.first), [N](NodeRef C) {
+    auto First = make_filter_range(makeChildRange(RR, N.first), [N](NodeRef C) {
       return !C.first->ignoreChild(N.second, C.second, InverseEdge);
     });
 
diff --git a/llvm/include/llvm/Support/GenericDomTree.h b/llvm/include/llvm/Support/GenericDomTree.h
index 10e591a69d369..325365824b3b9 100644
--- a/llvm/include/llvm/Support/GenericDomTree.h
+++ b/llvm/include/llvm/Support/GenericDomTree.h
@@ -28,6 +28,7 @@
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/Support/CFGDiff.h"
 #include "llvm/Support/CFGUpdate.h"
 #include "llvm/Support/raw_ostream.h"
 #include <algorithm>
@@ -211,7 +212,8 @@ void DeleteEdge(DomTreeT &DT, typename DomTreeT::NodePtr From,
 
 template <typename DomTreeT>
 void ApplyUpdates(DomTreeT &DT,
-                  ArrayRef<typename DomTreeT::UpdateType> Updates);
+                  GraphDiff<typename DomTreeT::NodePtr,
+                            DomTreeT::IsPostDominator> &PreViewCFG);
 
 template <typename DomTreeT>
 bool Verify(const DomTreeT &DT, typename DomTreeT::VerificationLevel VL);
@@ -535,10 +537,13 @@ class DominatorTreeBase {
   /// The type of updates is the same for DomTreeBase<T> and PostDomTreeBase<T>
   /// with the same template parameter T.
   ///
-  /// \param Updates An unordered sequence of updates to perform.
+  /// \param Updates An unordered sequence of updates to perform. The current
+  /// CFG and the reverse of these updates provides the pre-view of the CFG.
   ///
   void applyUpdates(ArrayRef<UpdateType> Updates) {
-    DomTreeBuilder::ApplyUpdates(*this, Updates);
+    GraphDiff<NodePtr, IsPostDominator> PreViewCFG(
+        Updates, /*ReverseApplyUpdates=*/true);
+    DomTreeBuilder::ApplyUpdates(*this, PreViewCFG);
   }
 
   /// Inform the dominator tree about a CFG edge insertion and update the tree.
diff --git a/llvm/include/llvm/Support/GenericDomTreeConstruction.h b/llvm/include/llvm/Support/GenericDomTreeConstruction.h
index 464de4e2b3ba1..709276ab7a29f 100644
--- a/llvm/include/llvm/Support/GenericDomTreeConstruction.h
+++ b/llvm/include/llvm/Support/GenericDomTreeConstruction.h
@@ -58,6 +58,7 @@ struct SemiNCAInfo {
   using TreeNodePtr = DomTreeNodeBase<NodeT> *;
   using RootsT = decltype(DomTreeT::Roots);
   static constexpr bool IsPostDom = DomTreeT::IsPostDominator;
+  using GraphDiffT = GraphDiff<NodePtr, IsPostDom>;
 
   // Information record used by Semi-NCA during tree construction.
   struct InfoRec {
@@ -77,28 +78,27 @@ struct SemiNCAInfo {
   using UpdateT = typename DomTreeT::UpdateType;
   using UpdateKind = typename DomTreeT::UpdateKind;
   struct BatchUpdateInfo {
-    SmallVector<UpdateT, 4> Updates;
-    using NodePtrAndKind = PointerIntPair<NodePtr, 1, UpdateKind>;
-
-    // In order to be able to walk a CFG that is out of sync with the CFG
-    // DominatorTree last knew about, use the list of updates to reconstruct
-    // previous CFG versions of the current CFG. For each node, we store a set
-    // of its virtually added/deleted future successors and predecessors.
-    // Note that these children are from the future relative to what the
-    // DominatorTree knows about -- using them to gets us some snapshot of the
-    // CFG from the past (relative to the state of the CFG).
-    DenseMap<NodePtr, SmallVector<NodePtrAndKind, 4>> FutureSuccessors;
-    DenseMap<NodePtr, SmallVector<NodePtrAndKind, 4>> FuturePredecessors;
+    // Note: Updates inside PreViewCFG are aleady legalized.
+    BatchUpdateInfo(GraphDiffT &PreViewCFG)
+        : PreViewCFG(PreViewCFG),
+          NumLegalized(PreViewCFG.getNumLegalizedUpdates()) {}
+
     // Remembers if the whole tree was recalculated at some point during the
     // current batch update.
     bool IsRecalculated = false;
+    GraphDiffT &PreViewCFG;
+    const size_t NumLegalized;
   };
 
   BatchUpdateInfo *BatchUpdates;
   using BatchUpdatePtr = BatchUpdateInfo *;
+  std::unique_ptr<GraphDiffT> EmptyGD;
 
   // If BUI is a nullptr, then there's no batch update in progress.
-  SemiNCAInfo(BatchUpdatePtr BUI) : BatchUpdates(BUI) {}
+  SemiNCAInfo(BatchUpdatePtr BUI) : BatchUpdates(BUI) {
+    if (!BatchUpdates)
+      EmptyGD = std::make_unique<GraphDiffT>();
+  }
 
   void clear() {
     NumToNode = {nullptr}; // Restore to initial state with a dummy start node.
@@ -107,8 +107,7 @@ struct SemiNCAInfo {
     // in progress, we need this information to continue it.
   }
 
-  template <bool Inverse>
-  struct ChildrenGetter {
+  template <bool Inversed> struct ChildrenGetter {
     using ResultTy = SmallVector<NodePtr, 8>;
 
     static ResultTy Get(NodePtr N, std::integral_constant<bool, false>) {
@@ -121,50 +120,16 @@ struct SemiNCAInfo {
       return ResultTy(IChildren.begin(), IChildren.end());
     }
 
-    using Tag = std::integral_constant<bool, Inverse>;
+    using Tag = std::integral_constant<bool, Inversed>;
 
     // The function below is the core part of the batch updater. It allows the
     // Depth Based Search algorithm to perform incremental updates in lockstep
     // with updates to the CFG. We emulated lockstep CFG updates by getting its
     // next snapshots by reverse-applying future updates.
     static ResultTy Get(NodePtr N, BatchUpdatePtr BUI) {
-      ResultTy Res = Get(N, Tag());
-      // If there's no batch update in progress, simply return node's children.
-      if (!BUI) return Res;
-
-      // CFG children are actually its *most current* children, and we have to
-      // reverse-apply the future updates to get the node's children at the
-      // point in time the update was performed.
-      auto &FutureChildren = (Inverse != IsPostDom) ? BUI->FuturePredecessors
-                                                    : BUI->FutureSuccessors;
-      auto FCIt = FutureChildren.find(N);
-      if (FCIt == FutureChildren.end()) return Res;
-
-      for (auto ChildAndKind : FCIt->second) {
-        const NodePtr Child = ChildAndKind.getPointer();
-        const UpdateKind UK = ChildAndKind.getInt();
-
-        // Reverse-apply the future update.
-        if (UK == UpdateKind::Insert) {
-          // If there's an insertion in the future, it means that the edge must
-          // exist in the current CFG, but was not present in it before.
-          assert(llvm::find(Res, Child) != Res.end()
-                 && "Expected child not found in the CFG");
-          Res.erase(std::remove(Res.begin(), Res.end(), Child), Res.end());
-          LLVM_DEBUG(dbgs() << "\tHiding edge " << BlockNamePrinter(N) << " -> "
-                            << BlockNamePrinter(Child) << "\n");
-        } else {
-          // If there's an deletion in the future, it means that the edge cannot
-          // exist in the current CFG, but existed in it before.
-          assert(llvm::find(Res, Child) == Res.end() &&
-                 "Unexpected child found in the CFG");
-          LLVM_DEBUG(dbgs() << "\tShowing virtual edge " << BlockNamePrinter(N)
-                            << " -> " << BlockNamePrinter(Child) << "\n");
-          Res.push_back(Child);
-        }
-      }
-
-      return Res;
+      if (!BUI)
+        return Get(N, Tag());
+      return BUI->PreViewCFG.template getChildren<Inversed>(N);
     }
   };
 
@@ -1005,15 +970,14 @@ struct SemiNCAInfo {
                                const TreeNodePtr TN) {
     LLVM_DEBUG(dbgs() << "IsReachableFromIDom " << BlockNamePrinter(TN)
                       << "\n");
-    for (const NodePtr Pred :
-         ChildrenGetter<!IsPostDom>::Get(TN->getBlock(), BUI)) {
+    auto TNB = TN->getBlock();
+    for (const NodePtr Pred : ChildrenGetter<!IsPostDom>::Get(TNB, BUI)) {
       LLVM_DEBUG(dbgs() << "\tPred " << BlockNamePrinter(Pred) << "\n");
       if (!DT.getNode(Pred)) continue;
 
-      const NodePtr Support =
-          DT.findNearestCommonDominator(TN->getBlock(), Pred);
+      const NodePtr Support = DT.findNearestCommonDominator(TNB, Pred);
       LLVM_DEBUG(dbgs() << "\tSupport " << BlockNamePrinter(Support) << "\n");
-      if (Support != TN->getBlock()) {
+      if (Support != TNB) {
         LLVM_DEBUG(dbgs() << "\t" << BlockNamePrinter(TN)
                           << " is reachable from support "
                           << BlockNamePrinter(Support) << "\n");
@@ -1144,53 +1108,23 @@ struct SemiNCAInfo {
   //===--------------------- DomTree Batch Updater --------------------------===
   //~~
 
-  static void ApplyUpdates(DomTreeT &DT, ArrayRef<UpdateT> Updates) {
-    const size_t NumUpdates = Updates.size();
+  static void ApplyUpdates(DomTreeT &DT, GraphDiffT &PreViewCFG) {
+    const size_t NumUpdates = PreViewCFG.getNumLegalizedUpdates();
     if (NumUpdates == 0)
       return;
 
     // Take the fast path for a single update and avoid running the batch update
     // machinery.
     if (NumUpdates == 1) {
-      const auto &Update = Updates.front();
+      UpdateT Update = PreViewCFG.popUpdateForIncrementalUpdates();
       if (Update.getKind() == UpdateKind::Insert)
-        DT.insertEdge(Update.getFrom(), Update.getTo());
+        InsertEdge(DT, /*BUI=*/nullptr, Update.getFrom(), Update.getTo());
       else
-        DT.deleteEdge(Update.getFrom(), Update.getTo());
-
+        DeleteEdge(DT, /*BUI=*/nullptr, Update.getFrom(), Update.getTo());
       return;
     }
 
-    BatchUpdateInfo BUI;
-    LLVM_DEBUG(dbgs() << "Legalizing " << BUI.Updates.size() << " updates\n");
-    cfg::LegalizeUpdates<NodePtr>(Updates, BUI.Updates, IsPostDom);
-
-    const size_t NumLegalized = BUI.Updates.size();
-    BUI.FutureSuccessors.reserve(NumLegalized);
-    BUI.FuturePredecessors.reserve(NumLegalized);
-
-    // Use the legalized future updates to initialize future successors and
-    // predecessors. Note that these sets will only decrease size over time, as
-    // the next CFG snapshots slowly approach the actual (current) CFG.
-    for (UpdateT &U : BUI.Updates) {
-      BUI.FutureSuccessors[U.getFrom()].push_back({U.getTo(), U.getKind()});
-      BUI.FuturePredecessors[U.getTo()].push_back({U.getFrom(), U.getKind()});
-    }
-
-#if 0
-    // FIXME: The LLVM_DEBUG macro only plays well with a modular
-    // build of LLVM when the header is marked as textual, but doing
-    // so causes redefinition errors.
-    LLVM_DEBUG(dbgs() << "About to apply " << NumLegalized << " updates\n");
-    LLVM_DEBUG(if (NumLegalized < 32) for (const auto &U
-                                           : reverse(BUI.Updates)) {
-      dbgs() << "\t";
-      U.dump();
-      dbgs() << "\n";
-    });
-    LLVM_DEBUG(dbgs() << "\n");
-#endif
-
+    BatchUpdateInfo BUI(PreViewCFG);
     // Recalculate the DominatorTree when the number of updates
     // exceeds a threshold, which usually makes direct updating slower than
     // recalculation. We select this threshold proportional to the
@@ -1200,21 +1134,21 @@ struct SemiNCAInfo {
 
     // Make unittests of the incremental algorithm work
     if (DT.DomTreeNodes.size() <= 100) {
-      if (NumLegalized > DT.DomTreeNodes.size())
+      if (BUI.NumLegalized > DT.DomTreeNodes.size())
         CalculateFromScratch(DT, &BUI);
-    } else if (NumLegalized > DT.DomTreeNodes.size() / 40)
+    } else if (BUI.NumLegalized > DT.DomTreeNodes.size() / 40)
       CalculateFromScratch(DT, &BUI);
 
     // If the DominatorTree was recalculated at some point, stop the batch
     // updates. Full recalculations ignore batch updates and look at the actual
     // CFG.
-    for (size_t i = 0; i < NumLegalized && !BUI.IsRecalculated; ++i)
+    for (size_t i = 0; i < BUI.NumLegalized && !BUI.IsRecalculated; ++i)
       ApplyNextUpdate(DT, BUI);
   }
 
   static void ApplyNextUpdate(DomTreeT &DT, BatchUpdateInfo &BUI) {
-    assert(!BUI.Updates.empty() && "No updates to apply!");
-    UpdateT CurrentUpdate = BUI.Updates.pop_back_val();
+    // Popping the next update, will move the PreViewCFG to the next snapshot.
+    UpdateT CurrentUpdate = BUI.PreViewCFG.popUpdateForIncrementalUpdates();
 #if 0
     // FIXME: The LLVM_DEBUG macro only plays well with a modular
     // build of LLVM when the header is marked as textual, but doing
@@ -1223,21 +1157,6 @@ struct SemiNCAInfo {
     LLVM_DEBUG(CurrentUpdate.dump(); dbgs() << "\n");
 #endif
 
-    // Move to the next snapshot of the CFG by removing the reverse-applied
-    // current update. Since updates are performed in the same order they are
-    // legalized it's sufficient to pop the last item here.
-    auto &FS = BUI.FutureSuccessors[CurrentUpdate.getFrom()];
-    assert(FS.back().getPointer() == CurrentUpdate.getTo() &&
-           FS.back().getInt() == CurrentUpdate.getKind());
-    FS.pop_back();
-    if (FS.empty()) BUI.FutureSuccessors.erase(CurrentUpdate.getFrom());
-
-    auto &FP = BUI.FuturePredecessors[CurrentUpdate.getTo()];
-    assert(FP.back().getPointer() == CurrentUpdate.getFrom() &&
-           FP.back().getInt() == CurrentUpdate.getKind());
-    FP.pop_back();
-    if (FP.empty()) BUI.FuturePredecessors.erase(CurrentUpdate.getTo());
-
     if (CurrentUpdate.getKind() == UpdateKind::Insert)
       InsertEdge(DT, &BUI, CurrentUpdate.getFrom(), CurrentUpdate.getTo());
     else
@@ -1596,19 +1515,11 @@ void Calculate(DomTreeT &DT) {
 template <typename DomTreeT>
 void CalculateWithUpdates(DomTreeT &DT,
                           ArrayRef<typename DomTreeT::UpdateType> Updates) {
-  // TODO: Move BUI creation in common method, reuse in ApplyUpdates.
-  typename SemiNCAInfo<DomTreeT>::BatchUpdateInfo BUI;
-  LLVM_DEBUG(dbgs() << "Legalizing " << BUI.Updates.size() << " updates\n");
-  cfg::LegalizeUpdates<typename DomTreeT::NodePtr>(Updates, BUI.Updates,
-                                                   DomTreeT::IsPostDominator);
-  const size_t NumLegalized = BUI.Updates.size();
-  BUI.FutureSuccessors.reserve(NumLegalized);
-  BUI.FuturePredecessors.reserve(NumLegalized);
-  for (auto &U : BUI.Updates) {
-    BUI.FutureSuccessors[U.getFrom()].push_back({U.getTo(), U.getKind()});
-    BUI.FuturePredecessors[U.getTo()].push_back({U.getFrom(), U.getKind()});
-  }
-
+  // FIXME: Updated to use the PreViewCFG and behave the same as until now.
+  // This behavior is however incorrect; this actually needs the PostViewCFG.
+  GraphDiff<typename DomTreeT::NodePtr, DomTreeT::IsPostDominator> PreViewCFG(
+      Updates, /*ReverseApplyUpdates=*/true);
+  typename SemiNCAInfo<DomTreeT>::BatchUpdateInfo BUI(PreViewCFG);
   SemiNCAInfo<DomTreeT>::CalculateFromScratch(DT, &BUI);
 }
 
@@ -1628,8 +1539,9 @@ void DeleteEdge(DomTreeT &DT, typename DomTreeT::NodePtr From,
 
 template <class DomTreeT>
 void ApplyUpdates(DomTreeT &DT,
-                  ArrayRef<typename DomTreeT::UpdateType> Updates) {
-  SemiNCAInfo<DomTreeT>::ApplyUpdates(DT, Updates);
+                  GraphDiff<typename DomTreeT::NodePtr,
+                            DomTreeT::IsPostDominator> &PreViewCFG) {
+  SemiNCAInfo<DomTreeT>::ApplyUpdates(DT, PreViewCFG);
 }
 
 template <class DomTreeT>
diff --git a/llvm/lib/IR/Dominators.cpp b/llvm/lib/IR/Dominators.cpp
index bb1cc347dcb12..f0b169ea1337f 100644
--- a/llvm/lib/IR/Dominators.cpp
+++ b/llvm/lib/IR/Dominators.cpp
@@ -90,9 +90,10 @@ template void llvm::DomTreeBuilder::DeleteEdge<DomTreeBuilder::BBPostDomTree>(
     DomTreeBuilder::BBPostDomTree &DT, BasicBlock *From, BasicBlock *To);
 
 template void llvm::DomTreeBuilder::ApplyUpdates<DomTreeBuilder::BBDomTree>(
-    DomTreeBuilder::BBDomTree &DT, DomTreeBuilder::BBUpdates);
+    DomTreeBuilder::BBDomTree &DT, DomTreeBuilder::BBDomTreeGraphDiff &);
 template void llvm::DomTreeBuilder::ApplyUpdates<DomTreeBuilder::BBPostDomTree>(
-    DomTreeBuilder::BBPostDomTree &DT, DomTreeBuilder::BBUpdates);
+    DomTreeBuilder::BBPostDomTree &DT,
+    DomTreeBuilder::BBPostDomTreeGraphDiff &);
 
 template bool llvm::DomTreeBuilder::Verify<DomTreeBuilder::BBDomTree>(
     const DomTreeBuilder::BBDomTree &DT,

From 31d58858425f6021d380eff879dd8983e25a5715 Mon Sep 17 00:00:00 2001
From: Jez Ng <jezng@fb.com>
Date: Fri, 24 Jul 2020 15:55:25 -0700
Subject: [PATCH 0057/1035] [lld-macho] Partial support for weak definitions

This diff adds support for weak definitions, though it doesn't handle weak
symbols in dylibs quite correctly -- we need to emit binding opcodes for them
in the weak binding section rather than the lazy binding section.

What *is* covered in this diff:

1. Reading the weak flag from symbol table / export trie, and writing it to the
   export trie
2. Refining the symbol table's rules for choosing one symbol definition over
   another. Wrote a few dozen test cases to make sure we were matching ld64's
   behavior.

We can now link basic C++ programs.

Reviewed By: #lld-macho, compnerd

Differential Revision: https://reviews.llvm.org/D83532
---
 lld/MachO/Arch/X86_64.cpp                     |  1 +
 lld/MachO/ExportTrie.cpp                      | 14 +--
 lld/MachO/InputFiles.cpp                      | 16 ++--
 lld/MachO/SymbolTable.cpp                     | 28 ++++--
 lld/MachO/SymbolTable.h                       | 11 ++-
 lld/MachO/Symbols.h                           | 25 ++++--
 lld/MachO/SyntheticSections.cpp               |  3 +-
 lld/test/MachO/weak-definition-direct-fetch.s | 90 +++++++++++++++++++
 .../MachO/weak-definition-indirect-fetch.s    | 42 +++++++++
 lld/test/MachO/weak-definition-order.s        | 36 ++++++++
 lld/test/MachO/weak-definition-over-dysym.s   | 39 ++++++++
 11 files changed, 275 insertions(+), 30 deletions(-)
 create mode 100644 lld/test/MachO/weak-definition-direct-fetch.s
 create mode 100644 lld/test/MachO/weak-definition-indirect-fetch.s
 create mode 100644 lld/test/MachO/weak-definition-order.s
 create mode 100644 lld/test/MachO/weak-definition-over-dysym.s

diff --git a/lld/MachO/Arch/X86_64.cpp b/lld/MachO/Arch/X86_64.cpp
index 36f686ca2f1d8..458dad805b4a2 100644
--- a/lld/MachO/Arch/X86_64.cpp
+++ b/lld/MachO/Arch/X86_64.cpp
@@ -218,6 +218,7 @@ void X86_64::prepareSymbolRelocation(lld::macho::Symbol &sym,
     in.got->addEntry(sym);
     break;
   case X86_64_RELOC_BRANCH: {
+    // TODO: weak dysyms should go into the weak binding section instead
     if (auto *dysym = dyn_cast<DylibSymbol>(&sym))
       in.stubs->addEntry(*dysym);
     break;
diff --git a/lld/MachO/ExportTrie.cpp b/lld/MachO/ExportTrie.cpp
index 7cc81bcfd5f18..993a552435325 100644
--- a/lld/MachO/ExportTrie.cpp
+++ b/lld/MachO/ExportTrie.cpp
@@ -59,6 +59,10 @@ struct Edge {
 
 struct ExportInfo {
   uint64_t address;
+  uint8_t flags;
+  explicit ExportInfo(const Symbol &sym)
+      : address(sym.getVA()),
+        flags(sym.isWeakDef() ? EXPORT_SYMBOL_FLAGS_WEAK_DEFINITION : 0) {}
   // TODO: Add proper support for re-exports & stub-and-resolver flags.
 };
 
@@ -83,9 +87,8 @@ bool TrieNode::updateOffset(size_t &nextOffset) {
   // node.
   size_t nodeSize;
   if (info) {
-    uint64_t flags = 0;
     uint32_t terminalSize =
-        getULEB128Size(flags) + getULEB128Size(info->address);
+        getULEB128Size(info->flags) + getULEB128Size(info->address);
     // Overall node size so far is the uleb128 size of the length of the symbol
     // info + the symbol info itself.
     nodeSize = terminalSize + getULEB128Size(terminalSize);
@@ -110,11 +113,10 @@ void TrieNode::writeTo(uint8_t *buf) const {
   buf += offset;
   if (info) {
     // TrieNodes with Symbol info: size, flags address
-    uint64_t flags = 0; // TODO: emit proper flags
     uint32_t terminalSize =
-        getULEB128Size(flags) + getULEB128Size(info->address);
+        getULEB128Size(info->flags) + getULEB128Size(info->address);
     buf += encodeULEB128(terminalSize, buf);
-    buf += encodeULEB128(flags, buf);
+    buf += encodeULEB128(info->flags, buf);
     buf += encodeULEB128(info->address, buf);
   } else {
     // TrieNode with no Symbol info.
@@ -194,7 +196,7 @@ void TrieBuilder::sortAndBuild(MutableArrayRef<const Symbol *> vec,
 
   if (isTerminal) {
     assert(j - i == 1); // no duplicate symbols
-    node->info = {pivotSymbol->getVA()};
+    node->info = ExportInfo(*pivotSymbol);
   } else {
     // This is the tail-call-optimized version of the following:
     // sortAndBuild(vec.slice(i, j - i), node, lastPos, pos + 1);
diff --git a/lld/MachO/InputFiles.cpp b/lld/MachO/InputFiles.cpp
index 46fe82f988222..f1afc187aca23 100644
--- a/lld/MachO/InputFiles.cpp
+++ b/lld/MachO/InputFiles.cpp
@@ -228,10 +228,9 @@ void InputFile::parseSymbols(ArrayRef<structs::nlist_64> nList,
     StringRef name = strtab + sym.n_strx;
     if (sym.n_type & N_EXT)
       // Global defined symbol
-      return symtab->addDefined(name, isec, value);
-    else
-      // Local defined symbol
-      return make<Defined>(name, isec, value);
+      return symtab->addDefined(name, isec, value, sym.n_desc & N_WEAK_DEF);
+    // Local defined symbol
+    return make<Defined>(name, isec, value, sym.n_desc & N_WEAK_DEF);
   };
 
   for (size_t i = 0, n = nList.size(); i < n; ++i) {
@@ -351,7 +350,9 @@ DylibFile::DylibFile(MemoryBufferRef mb, DylibFile *umbrella)
     auto *c = reinterpret_cast<const dyld_info_command *>(cmd);
     parseTrie(buf + c->export_off, c->export_size,
               [&](const Twine &name, uint64_t flags) {
-                symbols.push_back(symtab->addDylib(saver.save(name), umbrella));
+                bool isWeakDef = flags & EXPORT_SYMBOL_FLAGS_WEAK_DEFINITION;
+                symbols.push_back(
+                    symtab->addDylib(saver.save(name), umbrella, isWeakDef));
               });
   } else {
     error("LC_DYLD_INFO_ONLY not found in " + getName());
@@ -390,10 +391,11 @@ DylibFile::DylibFile(std::shared_ptr<llvm::MachO::InterfaceFile> interface,
 
   dylibName = saver.save(interface->getInstallName());
   // TODO(compnerd) filter out symbols based on the target platform
+  // TODO: handle weak defs
   for (const auto symbol : interface->symbols())
     if (symbol->getArchitectures().has(config->arch))
-      symbols.push_back(
-          symtab->addDylib(saver.save(symbol->getName()), umbrella));
+      symbols.push_back(symtab->addDylib(saver.save(symbol->getName()),
+                                         umbrella, /*isWeakDef=*/false));
   // TODO(compnerd) properly represent the hierarchy of the documents as it is
   // in theory possible to have re-exported dylibs from re-exported dylibs which
   // should be parent'ed to the child.
diff --git a/lld/MachO/SymbolTable.cpp b/lld/MachO/SymbolTable.cpp
index 80e870d79890c..061642d73f441 100644
--- a/lld/MachO/SymbolTable.cpp
+++ b/lld/MachO/SymbolTable.cpp
@@ -37,15 +37,23 @@ std::pair<Symbol *, bool> SymbolTable::insert(StringRef name) {
 }
 
 Symbol *SymbolTable::addDefined(StringRef name, InputSection *isec,
-                                uint32_t value) {
+                                uint32_t value, bool isWeakDef) {
   Symbol *s;
   bool wasInserted;
   std::tie(s, wasInserted) = insert(name);
 
-  if (!wasInserted && isa<Defined>(s))
-    error("duplicate symbol: " + name);
-
-  replaceSymbol<Defined>(s, name, isec, value);
+  if (!wasInserted) {
+    if (auto *defined = dyn_cast<Defined>(s)) {
+      if (isWeakDef)
+        return s;
+      if (!defined->isWeakDef())
+        error("duplicate symbol: " + name);
+    }
+    // Defined symbols take priority over other types of symbols, so in case
+    // of a name conflict, we fall through to the replaceSymbol() call below.
+  }
+
+  replaceSymbol<Defined>(s, name, isec, value, isWeakDef);
   return s;
 }
 
@@ -61,13 +69,15 @@ Symbol *SymbolTable::addUndefined(StringRef name) {
   return s;
 }
 
-Symbol *SymbolTable::addDylib(StringRef name, DylibFile *file) {
+Symbol *SymbolTable::addDylib(StringRef name, DylibFile *file, bool isWeakDef) {
   Symbol *s;
   bool wasInserted;
   std::tie(s, wasInserted) = insert(name);
 
-  if (wasInserted || isa<Undefined>(s))
-    replaceSymbol<DylibSymbol>(s, file, name);
+  if (wasInserted || isa<Undefined>(s) ||
+      (isa<DylibSymbol>(s) && !isWeakDef && s->isWeakDef()))
+    replaceSymbol<DylibSymbol>(s, file, name, isWeakDef);
+
   return s;
 }
 
@@ -79,7 +89,7 @@ Symbol *SymbolTable::addLazy(StringRef name, ArchiveFile *file,
 
   if (wasInserted)
     replaceSymbol<LazySymbol>(s, file, sym);
-  else if (isa<Undefined>(s))
+  else if (isa<Undefined>(s) || (isa<DylibSymbol>(s) && s->isWeakDef()))
     file->fetch(sym);
   return s;
 }
diff --git a/lld/MachO/SymbolTable.h b/lld/MachO/SymbolTable.h
index 2379008db56da..088b0e97c8406 100644
--- a/lld/MachO/SymbolTable.h
+++ b/lld/MachO/SymbolTable.h
@@ -22,13 +22,20 @@ class DylibFile;
 class InputSection;
 class Symbol;
 
+/*
+ * Note that the SymbolTable handles name collisions by calling
+ * replaceSymbol(), which does an in-place update of the Symbol via `placement
+ * new`. Therefore, there is no need to update any relocations that hold
+ * pointers the "old" Symbol -- they will automatically point to the new one.
+ */
 class SymbolTable {
 public:
-  Symbol *addDefined(StringRef name, InputSection *isec, uint32_t value);
+  Symbol *addDefined(StringRef name, InputSection *isec, uint32_t value,
+                     bool isWeakDef);
 
   Symbol *addUndefined(StringRef name);
 
-  Symbol *addDylib(StringRef name, DylibFile *file);
+  Symbol *addDylib(StringRef name, DylibFile *file, bool isWeakDef);
 
   Symbol *addLazy(StringRef name, ArchiveFile *file,
                   const llvm::object::Archive::Symbol &sym);
diff --git a/lld/MachO/Symbols.h b/lld/MachO/Symbols.h
index 63748ee483245..2dcccd03a8d02 100644
--- a/lld/MachO/Symbols.h
+++ b/lld/MachO/Symbols.h
@@ -39,6 +39,8 @@ class Symbol {
     LazyKind,
   };
 
+  virtual ~Symbol() {}
+
   Kind kind() const { return static_cast<Kind>(symbolKind); }
 
   StringRef getName() const { return {name.data, name.size}; }
@@ -47,6 +49,8 @@ class Symbol {
 
   uint64_t getFileOffset() const;
 
+  virtual bool isWeakDef() const { llvm_unreachable("cannot be weak"); }
+
   uint32_t gotIndex = UINT32_MAX;
 
 protected:
@@ -58,13 +62,19 @@ class Symbol {
 
 class Defined : public Symbol {
 public:
-  Defined(StringRefZ name, InputSection *isec, uint32_t value)
-      : Symbol(DefinedKind, name), isec(isec), value(value) {}
+  Defined(StringRefZ name, InputSection *isec, uint32_t value, bool isWeakDef)
+      : Symbol(DefinedKind, name), isec(isec), value(value),
+        weakDef(isWeakDef) {}
+
+  bool isWeakDef() const override { return weakDef; }
+
+  static bool classof(const Symbol *s) { return s->kind() == DefinedKind; }
 
   InputSection *isec;
   uint32_t value;
 
-  static bool classof(const Symbol *s) { return s->kind() == DefinedKind; }
+private:
+  const bool weakDef;
 };
 
 class Undefined : public Symbol {
@@ -76,14 +86,19 @@ class Undefined : public Symbol {
 
 class DylibSymbol : public Symbol {
 public:
-  DylibSymbol(DylibFile *file, StringRefZ name)
-      : Symbol(DylibKind, name), file(file) {}
+  DylibSymbol(DylibFile *file, StringRefZ name, bool isWeakDef)
+      : Symbol(DylibKind, name), file(file), weakDef(isWeakDef) {}
+
+  bool isWeakDef() const override { return weakDef; }
 
   static bool classof(const Symbol *s) { return s->kind() == DylibKind; }
 
   DylibFile *file;
   uint32_t stubsIndex = UINT32_MAX;
   uint32_t lazyBindOffset = UINT32_MAX;
+
+private:
+  const bool weakDef;
 };
 
 class LazySymbol : public Symbol {
diff --git a/lld/MachO/SyntheticSections.cpp b/lld/MachO/SyntheticSections.cpp
index cc0d5a93c40de..a2d8bf42e9edf 100644
--- a/lld/MachO/SyntheticSections.cpp
+++ b/lld/MachO/SyntheticSections.cpp
@@ -264,7 +264,8 @@ void StubHelperSection::setup() {
   in.got->addEntry(*stubBinder);
 
   inputSections.push_back(in.imageLoaderCache);
-  symtab->addDefined("__dyld_private", in.imageLoaderCache, 0);
+  symtab->addDefined("__dyld_private", in.imageLoaderCache, 0,
+                     /*isWeakDef=*/false);
 }
 
 ImageLoaderCacheSection::ImageLoaderCacheSection() {
diff --git a/lld/test/MachO/weak-definition-direct-fetch.s b/lld/test/MachO/weak-definition-direct-fetch.s
new file mode 100644
index 0000000000000..04c022e9c086d
--- /dev/null
+++ b/lld/test/MachO/weak-definition-direct-fetch.s
@@ -0,0 +1,90 @@
+# REQUIRES: x86
+# RUN: mkdir -p %t
+
+## This test exercises the various possible combinations of weak and non-weak
+## symbols that get referenced directly by a relocation in an object file.
+
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %s -o %t/test.o
+# RUN: echo ".globl _foo; .section __TEXT,nonweak; _foo:" | llvm-mc -filetype=obj -triple=x86_64-apple-darwin -o %t/foo.o
+# RUN: echo ".globl _foo; .weak_definition _foo; .section __TEXT,weak; _foo:" | llvm-mc -filetype=obj -triple=x86_64-apple-darwin -o %t/weakfoo.o
+
+# RUN: lld -flavor darwinnew -dylib -install_name \
+# RUN:   @executable_path/libfoo.dylib %t/foo.o -o %t/libfoo.dylib
+# RUN: lld -flavor darwinnew -dylib -install_name \
+# RUN:   @executable_path/libweakfoo.dylib %t/weakfoo.o -o %t/libweakfoo.dylib
+
+# RUN: llvm-objdump --macho --exports-trie %t/libweakfoo.dylib | FileCheck %s --check-prefix WEAK-DYLIB-CHECK
+# WEAK-DYLIB-CHECK: _foo [weak_def]
+
+## Make sure we are using the export trie and not the symbol table when linking
+## against these dylibs.
+# RUN: llvm-strip %t/libfoo.dylib
+# RUN: llvm-strip %t/libweakfoo.dylib
+# RUN: llvm-nm %t/libfoo.dylib 2>&1 | FileCheck %s --check-prefix=NOSYM
+# RUN: llvm-nm %t/libweakfoo.dylib 2>&1 | FileCheck %s --check-prefix=NOSYM
+# NOSYM: no symbols
+
+# RUN: rm -f %t/foo.a
+# RUN: llvm-ar --format=darwin rcs %t/foo.a %t/foo.o
+# RUN: rm -f %t/weakfoo.a
+# RUN: llvm-ar --format=darwin rcs %t/weakfoo.a %t/weakfoo.o
+
+## End of input file setup. The following lines check which symbol "wins" when
+## there are multiple definitions.
+
+# PREFER-NONWEAK-DYLIB:  __DATA __la_symbol_ptr 0x{{[0-9a-f]+}} libfoo _foo
+# PREFER-WEAK-OBJECT:    O __TEXT,weak _foo
+# PREFER-NONWEAK-OBJECT: O __TEXT,nonweak _foo
+
+## First, we test the cases where the symbols are of the same type (both from a
+## dylib, or both from an archive, etc.)
+##
+## For dylibs and object files, the non-weak symbol always wins. But the weak
+## flag has no effect when we are dealing with two archive symbols.
+
+# RUN: lld -flavor darwinnew -L%S/Inputs/MacOSX.sdk/usr/lib -lSystem -o %t/weak-nonweak-dylibs -Z -L%t -lweakfoo -lfoo %t/test.o
+# RUN: llvm-objdump --macho --lazy-bind --syms %t/weak-nonweak-dylibs | FileCheck %s --check-prefix=PREFER-NONWEAK-DYLIB
+# RUN: lld -flavor darwinnew -L%S/Inputs/MacOSX.sdk/usr/lib -lSystem -o %t/nonweak-weak-dylibs -Z -L%t -lfoo -lweakfoo %t/test.o
+# RUN: llvm-objdump --macho --lazy-bind --syms %t/nonweak-weak-dylibs | FileCheck %s --check-prefix=PREFER-NONWEAK-DYLIB
+
+# RUN: lld -flavor darwinnew -L%S/Inputs/MacOSX.sdk/usr/lib -lSystem -o %t/weak-nonweak-objs -Z -L%t %t/weakfoo.o %t/foo.o %t/test.o
+# RUN: llvm-objdump --macho --lazy-bind --syms %t/weak-nonweak-objs | FileCheck %s --check-prefix=PREFER-NONWEAK-OBJECT
+# RUN: lld -flavor darwinnew -L%S/Inputs/MacOSX.sdk/usr/lib -lSystem -o %t/nonweak-weak-objs -Z -L%t %t/foo.o %t/weakfoo.o %t/test.o
+# RUN: llvm-objdump --macho --lazy-bind --syms %t/nonweak-weak-objs | FileCheck %s --check-prefix=PREFER-NONWEAK-OBJECT
+
+# RUN: lld -flavor darwinnew -L%S/Inputs/MacOSX.sdk/usr/lib -lSystem -o %t/weak-nonweak-archives -Z -L%t %t/weakfoo.a %t/foo.a %t/test.o
+# RUN: llvm-objdump --macho --lazy-bind --syms %t/weak-nonweak-archives | FileCheck %s --check-prefix=PREFER-WEAK-OBJECT
+# RUN: lld -flavor darwinnew -L%S/Inputs/MacOSX.sdk/usr/lib -lSystem -o %t/nonweak-weak-archives -Z -L%t %t/foo.a %t/weakfoo.a %t/test.o
+# RUN: llvm-objdump --macho --lazy-bind --syms %t/nonweak-weak-archives | FileCheck %s --check-prefix=PREFER-NONWEAK-OBJECT
+
+## The remaining lines test symbol pairs of different types.
+
+# RUN: lld -flavor darwinnew -L%S/Inputs/MacOSX.sdk/usr/lib -lSystem -o %t/weak-dylib-weak-ar -Z -L%t -lweakfoo %t/weakfoo.a %t/test.o
+# RUN: llvm-objdump --macho --lazy-bind --syms %t/weak-dylib-weak-ar | FileCheck %s --check-prefix=PREFER-WEAK-OBJECT
+# RUN: lld -flavor darwinnew -L%S/Inputs/MacOSX.sdk/usr/lib -lSystem -o %t/weak-ar-weak-dylib -Z -L%t %t/weakfoo.a -lweakfoo %t/test.o
+# RUN: llvm-objdump --macho --lazy-bind --syms %t/weak-ar-weak-dylib | FileCheck %s --check-prefix=PREFER-WEAK-OBJECT
+
+# RUN: lld -flavor darwinnew -L%S/Inputs/MacOSX.sdk/usr/lib -lSystem -o %t/weak-ar-nonweak-dylib -Z -L%t %t/weakfoo.a -lfoo %t/test.o
+# RUN: llvm-objdump --macho --lazy-bind --syms %t/weak-ar-nonweak-dylib | FileCheck %s --check-prefix=PREFER-WEAK-OBJECT
+# RUN: lld -flavor darwinnew -L%S/Inputs/MacOSX.sdk/usr/lib -lSystem -o %t/nonweak-dylib-weak-ar -Z -L%t -lfoo %t/weakfoo.a %t/test.o
+# RUN: llvm-objdump --macho --lazy-bind --syms %t/nonweak-dylib-weak-ar | FileCheck %s --check-prefix=PREFER-NONWEAK-DYLIB
+
+# RUN: lld -flavor darwinnew -L%S/Inputs/MacOSX.sdk/usr/lib -lSystem -o %t/weak-dylib-weak-obj -Z -L%t -lweakfoo %t/weakfoo.o %t/test.o
+# RUN: llvm-objdump --macho --lazy-bind --syms %t/weak-dylib-weak-obj | FileCheck %s --check-prefix=PREFER-WEAK-OBJECT
+# RUN: lld -flavor darwinnew -L%S/Inputs/MacOSX.sdk/usr/lib -lSystem -o %t/weak-obj-weak-dylib -Z -L%t %t/weakfoo.o -lweakfoo %t/test.o
+# RUN: llvm-objdump --macho --lazy-bind --syms %t/weak-obj-weak-dylib | FileCheck %s --check-prefix=PREFER-WEAK-OBJECT
+
+# RUN: lld -flavor darwinnew -L%S/Inputs/MacOSX.sdk/usr/lib -lSystem -o %t/weak-obj-nonweak-dylib -Z -L%t %t/weakfoo.o -lfoo %t/test.o
+# RUN: llvm-objdump --macho --lazy-bind --syms %t/weak-obj-nonweak-dylib | FileCheck %s --check-prefix=PREFER-WEAK-OBJECT
+# RUN: lld -flavor darwinnew -L%S/Inputs/MacOSX.sdk/usr/lib -lSystem -o %t/nonweak-dylib-weak-obj -Z -L%t -lfoo %t/weakfoo.o %t/test.o
+# RUN: llvm-objdump --macho --lazy-bind --syms %t/nonweak-dylib-weak-obj | FileCheck %s --check-prefix=PREFER-WEAK-OBJECT
+
+# RUN: lld -flavor darwinnew -L%S/Inputs/MacOSX.sdk/usr/lib -lSystem -o %t/weak-obj-nonweak-ar -Z -L%t %t/weakfoo.o %t/foo.a %t/test.o
+# RUN: llvm-objdump --macho --lazy-bind --syms %t/weak-obj-nonweak-ar | FileCheck %s --check-prefix=PREFER-WEAK-OBJECT
+# RUN: lld -flavor darwinnew -L%S/Inputs/MacOSX.sdk/usr/lib -lSystem -o %t/nonweak-ar-weak-obj -Z -L%t %t/foo.a %t/weakfoo.o %t/test.o
+# RUN: llvm-objdump --macho --lazy-bind --syms %t/nonweak-ar-weak-obj | FileCheck %s --check-prefix=PREFER-WEAK-OBJECT
+
+.globl _main
+_main:
+  callq _foo
+  ret
diff --git a/lld/test/MachO/weak-definition-indirect-fetch.s b/lld/test/MachO/weak-definition-indirect-fetch.s
new file mode 100644
index 0000000000000..d22e0a370d5e0
--- /dev/null
+++ b/lld/test/MachO/weak-definition-indirect-fetch.s
@@ -0,0 +1,42 @@
+# REQUIRES: x86
+# RUN: mkdir -p %t
+
+## This tests examines the effect of .weak_definition on symbols in an archive
+## that are not referenced directly, but which are still loaded due to some
+## other symbol in the archive member being referenced.
+##
+## In this particular test, _foo isn't referenced directly, but both archives
+## will be fetched when linking against the main test file due to its references
+## to _bar and _baz.
+
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %s -o %t/test.o
+# RUN: echo ".globl _foo, _bar; .section __TEXT,nonweak; _bar: _foo:" | llvm-mc -filetype=obj -triple=x86_64-apple-darwin -o %t/foo.o
+# RUN: echo ".globl _foo, _baz; .weak_definition _foo; .section __TEXT,weak; _baz: _foo:" | llvm-mc -filetype=obj -triple=x86_64-apple-darwin -o %t/weakfoo.o
+
+# RUN: rm -f %t/foo.a
+# RUN: llvm-ar --format=darwin rcs %t/foo.a %t/foo.o
+# RUN: rm -f %t/weakfoo.a
+# RUN: llvm-ar --format=darwin rcs %t/weakfoo.a %t/weakfoo.o
+
+# PREFER-NONWEAK-OBJECT: O __TEXT,nonweak _foo
+
+# RUN: lld -flavor darwinnew -L%S/Inputs/MacOSX.sdk/usr/lib -lSystem -o %t/weak-nonweak-archives -Z -L%t %t/weakfoo.a %t/foo.a %t/test.o
+# RUN: llvm-objdump --syms %t/weak-nonweak-archives | FileCheck %s --check-prefix=PREFER-NONWEAK-OBJECT
+# RUN: lld -flavor darwinnew -L%S/Inputs/MacOSX.sdk/usr/lib -lSystem -o %t/nonweak-weak-archives -Z -L%t %t/foo.a %t/weakfoo.a %t/test.o
+# RUN: llvm-objdump --syms %t/nonweak-weak-archives | FileCheck %s --check-prefix=PREFER-NONWEAK-OBJECT
+
+# RUN: lld -flavor darwinnew -L%S/Inputs/MacOSX.sdk/usr/lib -lSystem -o %t/weak-nonweak-objs -Z -L%t %t/weakfoo.o %t/foo.o %t/test.o
+# RUN: llvm-objdump --syms %t/weak-nonweak-objs | FileCheck %s --check-prefix=PREFER-NONWEAK-OBJECT
+# RUN: lld -flavor darwinnew -L%S/Inputs/MacOSX.sdk/usr/lib -lSystem -o %t/nonweak-weak-objs -Z -L%t %t/foo.o %t/weakfoo.o %t/test.o
+# RUN: llvm-objdump --syms %t/nonweak-weak-objs | FileCheck %s --check-prefix=PREFER-NONWEAK-OBJECT
+
+# RUN: lld -flavor darwinnew -L%S/Inputs/MacOSX.sdk/usr/lib -lSystem -o %t/weak-obj-nonweak-ar -Z -L%t %t/weakfoo.o %t/foo.a %t/test.o
+# RUN: llvm-objdump --syms %t/weak-obj-nonweak-ar | FileCheck %s --check-prefix=PREFER-NONWEAK-OBJECT
+# RUN: lld -flavor darwinnew -L%S/Inputs/MacOSX.sdk/usr/lib -lSystem -o %t/nonweak-ar-weak-obj -Z -L%t %t/foo.a %t/weakfoo.o %t/test.o
+# RUN: llvm-objdump --syms %t/nonweak-ar-weak-obj | FileCheck %s --check-prefix=PREFER-NONWEAK-OBJECT
+
+.globl _main
+_main:
+  callq _bar
+  callq _baz
+  ret
diff --git a/lld/test/MachO/weak-definition-order.s b/lld/test/MachO/weak-definition-order.s
new file mode 100644
index 0000000000000..6770a5f76b391
--- /dev/null
+++ b/lld/test/MachO/weak-definition-order.s
@@ -0,0 +1,36 @@
+# REQUIRES: x86
+# RUN: mkdir -p %t
+
+## This test demonstrates that when we have two weak symbols of the same type,
+## we pick the one whose containing file appears earlier in the command-line
+## invocation.
+
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %s -o %t/test.o
+# RUN: echo ".globl _foo; .weak_definition _foo; .section __TEXT,weak1; _foo:" | llvm-mc -filetype=obj -triple=x86_64-apple-darwin -o %t/weak1.o
+# RUN: echo ".globl _foo; .weak_definition _foo; .section __TEXT,weak2; _foo:" | llvm-mc -filetype=obj -triple=x86_64-apple-darwin -o %t/weak2.o
+
+# RUN: lld -flavor darwinnew -L%S/Inputs/MacOSX.sdk/usr/lib -lSystem -o %t/obj12 -Z -L%t %t/weak1.o %t/weak2.o %t/test.o
+# RUN: llvm-objdump --syms %t/obj12 | FileCheck %s --check-prefix=WEAK1
+# RUN: lld -flavor darwinnew -L%S/Inputs/MacOSX.sdk/usr/lib -lSystem -o %t/obj21 -Z -L%t %t/weak2.o %t/weak1.o %t/test.o
+# RUN: llvm-objdump --syms %t/obj21 | FileCheck %s --check-prefix=WEAK2
+
+# WEAK1: O __TEXT,weak1 _foo
+# WEAK2: O __TEXT,weak2 _foo
+
+# RUN: lld -flavor darwinnew -dylib -install_name \
+# RUN:   @executable_path/libweak1.dylib %t/weak1.o -o %t/libweak1.dylib
+# RUN: lld -flavor darwinnew -dylib -install_name \
+# RUN:   @executable_path/libweak2.dylib %t/weak2.o -o %t/libweak2.dylib
+
+# RUN: lld -flavor darwinnew -L%S/Inputs/MacOSX.sdk/usr/lib -lSystem -o %t/dylib12 -Z -L%t -lweak1 -lweak2 %t/test.o
+# RUN: llvm-objdump --macho --lazy-bind %t/dylib12 | FileCheck %s --check-prefix=DYLIB1
+# RUN: lld -flavor darwinnew -L%S/Inputs/MacOSX.sdk/usr/lib -lSystem -o %t/dylib21 -Z -L%t -lweak2 -lweak1 %t/test.o
+# RUN: llvm-objdump --macho --lazy-bind %t/dylib21 | FileCheck %s --check-prefix=DYLIB2
+## TODO: these should really be in the weak binding section, not the lazy binding section
+# DYLIB1: __DATA   __la_symbol_ptr    0x{{[0-9a-f]*}} libweak1         _foo
+# DYLIB2: __DATA   __la_symbol_ptr    0x{{[0-9a-f]*}} libweak2         _foo
+
+.globl _main
+_main:
+  callq _foo
+  ret
diff --git a/lld/test/MachO/weak-definition-over-dysym.s b/lld/test/MachO/weak-definition-over-dysym.s
new file mode 100644
index 0000000000000..e3cf030b7149f
--- /dev/null
+++ b/lld/test/MachO/weak-definition-over-dysym.s
@@ -0,0 +1,39 @@
+# REQUIRES: x86
+# RUN: mkdir -p %t
+
+## This test demonstrates that when an archive file is fetched, its symbols
+## always override any conflicting dylib symbols, regardless of any weak
+## definition flags.
+
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %s -o %t/test.o
+# RUN: echo ".globl _foo; _foo:" | llvm-mc -filetype=obj -triple=x86_64-apple-darwin -o %t/libfoo.o
+# RUN: lld -flavor darwinnew -dylib -install_name \
+# RUN:   @executable_path/libfoo.dylib %t/libfoo.o -o %t/libfoo.dylib
+
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %s -o %t/test.o
+# RUN: echo ".globl _foo, _bar; .section __TEXT,nonweak; _bar: _foo:" | llvm-mc -filetype=obj -triple=x86_64-apple-darwin -o %t/foo.o
+# RUN: echo ".globl _foo, _bar; .weak_definition _foo; .section __TEXT,weak; _bar: _foo:" | llvm-mc -filetype=obj -triple=x86_64-apple-darwin -o %t/weakfoo.o
+
+# RUN: rm -f %t/foo.a
+# RUN: llvm-ar --format=darwin rcs %t/foo.a %t/foo.o
+# RUN: rm -f %t/weakfoo.a
+# RUN: llvm-ar --format=darwin rcs %t/weakfoo.a %t/weakfoo.o
+
+# PREFER-WEAK-OBJECT: O __TEXT,weak _foo
+# PREFER-NONWEAK-OBJECT: O __TEXT,nonweak _foo
+
+# RUN: lld -flavor darwinnew -L%S/Inputs/MacOSX.sdk/usr/lib -lSystem -o %t/nonweak-dylib-weak-ar -Z -L%t -lfoo %t/weakfoo.a %t/test.o
+# RUN: llvm-objdump --macho --lazy-bind --syms %t/nonweak-dylib-weak-ar | FileCheck %s --check-prefix=PREFER-WEAK-OBJECT
+# RUN: lld -flavor darwinnew -L%S/Inputs/MacOSX.sdk/usr/lib -lSystem -o %t/weak-ar-nonweak-dylib -Z -L%t %t/weakfoo.a -lfoo %t/test.o
+# RUN: llvm-objdump --macho --lazy-bind --syms %t/weak-ar-nonweak-dylib | FileCheck %s --check-prefix=PREFER-WEAK-OBJECT
+
+# RUN: lld -flavor darwinnew -L%S/Inputs/MacOSX.sdk/usr/lib -lSystem -o %t/weak-obj-nonweak-dylib -Z -L%t %t/weakfoo.o -lfoo %t/test.o
+# RUN: llvm-objdump --macho --lazy-bind --syms %t/weak-obj-nonweak-dylib | FileCheck %s --check-prefix=PREFER-WEAK-OBJECT
+# RUN: lld -flavor darwinnew -L%S/Inputs/MacOSX.sdk/usr/lib -lSystem -o %t/nonweak-dylib-weak-obj -Z -L%t -lfoo %t/weakfoo.o %t/test.o
+# RUN: llvm-objdump --macho --lazy-bind --syms %t/nonweak-dylib-weak-obj | FileCheck %s --check-prefix=PREFER-WEAK-OBJECT
+
+.globl _main
+_main:
+  callq _foo
+  callq _bar
+  ret

From 06a0dd2467d5c4726699eea3589a6444e00a2eef Mon Sep 17 00:00:00 2001
From: Jez Ng <jezng@fb.com>
Date: Fri, 24 Jul 2020 15:55:14 -0700
Subject: [PATCH 0058/1035] [lld-macho] Ignore -dependency_info and its
 argument

XCode passes in this flag, which we do not yet implement. Skip
over the argument for now so we can at least successfully parse the
linker invocation.

Reviewed By: #lld-macho, compnerd

Differential Revision: https://reviews.llvm.org/D84485
---
 lld/MachO/Options.td              | 9 +++++----
 lld/test/MachO/silent-ignore.test | 1 +
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/lld/MachO/Options.td b/lld/MachO/Options.td
index 1e42542b9ac47..7d81ca7c14270 100644
--- a/lld/MachO/Options.td
+++ b/lld/MachO/Options.td
@@ -456,6 +456,11 @@ def map : Separate<["-"], "map">,
      HelpText<"Writes all symbols and their addresses to <path>">,
      Flags<[HelpHidden]>,
      Group<grp_introspect>;
+def dependency_info : Separate<["-"], "dependency_info">,
+     MetaVarName<"<path>">,
+     HelpText<"Dump dependency info">,
+     Flags<[HelpHidden]>,
+     Group<grp_introspect>;
 
 def grp_symtab : OptionGroup<"symtab">, HelpText<"SYMBOL TABLE OPTIMIZATIONS">;
 
@@ -1098,10 +1103,6 @@ def demangle : Flag<["-"], "demangle">,
      HelpText<"This option is undocumented in ld64">,
      Flags<[HelpHidden]>,
      Group<grp_undocumented>;
-def dependency_info : Flag<["-"], "dependency_info">,
-     HelpText<"This option is undocumented in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_undocumented>;
 def dyld_env : Flag<["-"], "dyld_env">,
      HelpText<"This option is undocumented in ld64">,
      Flags<[HelpHidden]>,
diff --git a/lld/test/MachO/silent-ignore.test b/lld/test/MachO/silent-ignore.test
index ae68dd8fe81ff..0d2086386fa8f 100644
--- a/lld/test/MachO/silent-ignore.test
+++ b/lld/test/MachO/silent-ignore.test
@@ -4,6 +4,7 @@ RUN:   -dynamic \
 RUN:   -no_deduplicate \
 RUN:   -lto_library /lib/foo \
 RUN:   -macosx_version_min 0 \
+RUN:   -dependency_info /path/to/dependency_info.dat \
 RUN:   -syslibroot /path/to/MacOSX.platform/Developer/SDKs/MacOSX.sdk
 RUN: not lld -flavor darwinnew -v --not-an-ignored-argument 2>&1 | FileCheck %s
 CHECK: error: unknown argument: --not-an-ignored-argument

From 99996213ebd9655bb0488387f906d5438bcd37f8 Mon Sep 17 00:00:00 2001
From: Jonas Devlieghere <jonas@devlieghere.com>
Date: Fri, 24 Jul 2020 15:10:05 -0700
Subject: [PATCH 0059/1035] [lldb] Don't wrap and release raw pointer in
 unique_ptr (NFC)

---
 lldb/source/Target/LanguageRuntime.cpp | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/lldb/source/Target/LanguageRuntime.cpp b/lldb/source/Target/LanguageRuntime.cpp
index 58ad70c2b9028..f8143839bb64c 100644
--- a/lldb/source/Target/LanguageRuntime.cpp
+++ b/lldb/source/Target/LanguageRuntime.cpp
@@ -202,20 +202,15 @@ class ExceptionBreakpointResolver : public BreakpointResolver {
 
 LanguageRuntime *LanguageRuntime::FindPlugin(Process *process,
                                              lldb::LanguageType language) {
-  std::unique_ptr<LanguageRuntime> language_runtime_up;
   LanguageRuntimeCreateInstance create_callback;
-
   for (uint32_t idx = 0;
        (create_callback =
             PluginManager::GetLanguageRuntimeCreateCallbackAtIndex(idx)) !=
        nullptr;
        ++idx) {
-    language_runtime_up.reset(create_callback(process, language));
-
-    if (language_runtime_up)
-      return language_runtime_up.release();
+    if (LanguageRuntime *runtime = create_callback(process, language))
+      return runtime;
   }
-
   return nullptr;
 }
 

From 34d4c8a53e569b1b83a0672015a19f8ca9bb3c35 Mon Sep 17 00:00:00 2001
From: Jonas Devlieghere <jonas@devlieghere.com>
Date: Fri, 24 Jul 2020 16:20:55 -0700
Subject: [PATCH 0060/1035] [lldb] Have LanguageRuntime and SystemRuntime share
 a base class (NFC)

LangaugeRuntime and SystemRuntime now both inherit from Runtime.
---
 lldb/include/lldb/Target/LanguageRuntime.h | 12 ++------
 lldb/include/lldb/Target/Runtime.h         | 33 ++++++++++++++++++++++
 lldb/include/lldb/Target/SystemRuntime.h   | 10 +++----
 lldb/source/Target/LanguageRuntime.cpp     |  2 +-
 lldb/source/Target/SystemRuntime.cpp       |  6 ++--
 5 files changed, 43 insertions(+), 20 deletions(-)
 create mode 100644 lldb/include/lldb/Target/Runtime.h

diff --git a/lldb/include/lldb/Target/LanguageRuntime.h b/lldb/include/lldb/Target/LanguageRuntime.h
index b0b9b919911a1..da3cb9702392d 100644
--- a/lldb/include/lldb/Target/LanguageRuntime.h
+++ b/lldb/include/lldb/Target/LanguageRuntime.h
@@ -18,6 +18,7 @@
 #include "lldb/Expression/LLVMUserExpression.h"
 #include "lldb/Symbol/DeclVendor.h"
 #include "lldb/Target/ExecutionContextScope.h"
+#include "lldb/Target/Runtime.h"
 #include "lldb/lldb-private.h"
 #include "lldb/lldb-public.h"
 
@@ -56,7 +57,7 @@ class ExceptionSearchFilter : public SearchFilter {
   void UpdateModuleListIfNeeded();
 };
 
-class LanguageRuntime : public PluginInterface {
+class LanguageRuntime : public Runtime, public PluginInterface {
 public:
   ~LanguageRuntime() override;
 
@@ -127,10 +128,6 @@ class LanguageRuntime : public PluginInterface {
     return lldb::ThreadSP();
   }
 
-  Process *GetProcess() { return m_process; }
-
-  Target &GetTargetRef() { return m_process->GetTarget(); }
-
   virtual DeclVendor *GetDeclVendor() { return nullptr; }
 
   virtual lldb::BreakpointResolverSP
@@ -159,7 +156,7 @@ class LanguageRuntime : public PluginInterface {
     return llvm::None;
   }
 
-  virtual void ModulesDidLoad(const ModuleList &module_list) {}
+  virtual void ModulesDidLoad(const ModuleList &module_list) override {}
 
   // Called by ClangExpressionParser::PrepareForExecution to query for any
   // custom LLVM IR passes that need to be run before an expression is
@@ -179,10 +176,7 @@ class LanguageRuntime : public PluginInterface {
   static char ID;
 
 protected:
-  // Classes that inherit from LanguageRuntime can see and modify these
-
   LanguageRuntime(Process *process);
-  Process *m_process;
 
 private:
   LanguageRuntime(const LanguageRuntime &) = delete;
diff --git a/lldb/include/lldb/Target/Runtime.h b/lldb/include/lldb/Target/Runtime.h
new file mode 100644
index 0000000000000..06f0b610e40ba
--- /dev/null
+++ b/lldb/include/lldb/Target/Runtime.h
@@ -0,0 +1,33 @@
+//===-- Runtime.h -----------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLDB_TARGET_RUNTIME_H
+#define LLDB_TARGET_RUNTIME_H
+
+#include "lldb/Target/Process.h"
+
+namespace lldb_private {
+class Runtime {
+public:
+  Runtime(Process *process) : m_process(process) {}
+  virtual ~Runtime() = default;
+  Runtime(const Runtime &) = delete;
+  const Runtime &operator=(const Runtime &) = delete;
+
+  Process *GetProcess() { return m_process; }
+  Target &GetTargetRef() { return m_process->GetTarget(); }
+
+  /// Called when modules have been loaded in the process.
+  virtual void ModulesDidLoad(const ModuleList &module_list) = 0;
+
+protected:
+  Process *m_process;
+};
+} // namespace lldb_private
+
+#endif // LLDB_TARGET_RUNTIME_H
diff --git a/lldb/include/lldb/Target/SystemRuntime.h b/lldb/include/lldb/Target/SystemRuntime.h
index 4f07d7ab52e5b..0ec0793e95f9b 100644
--- a/lldb/include/lldb/Target/SystemRuntime.h
+++ b/lldb/include/lldb/Target/SystemRuntime.h
@@ -15,6 +15,7 @@
 #include "lldb/Core/PluginInterface.h"
 #include "lldb/Target/QueueItem.h"
 #include "lldb/Target/QueueList.h"
+#include "lldb/Target/Runtime.h"
 #include "lldb/Utility/ConstString.h"
 #include "lldb/Utility/StructuredData.h"
 #include "lldb/lldb-private.h"
@@ -39,7 +40,7 @@ namespace lldb_private {
 /// can be asked to provide that information.
 ///
 
-class SystemRuntime : public PluginInterface {
+class SystemRuntime : public Runtime, public PluginInterface {
 public:
   /// Find a system runtime plugin for a given process.
   ///
@@ -52,7 +53,7 @@ class SystemRuntime : public PluginInterface {
   static SystemRuntime *FindPlugin(Process *process);
 
   /// Construct with a process.
-  SystemRuntime(lldb_private::Process *process);
+  SystemRuntime(Process *process);
 
   /// Destructor.
   ///
@@ -76,7 +77,7 @@ class SystemRuntime : public PluginInterface {
   ///
   /// Allow the SystemRuntime plugin to enable logging features in the system
   /// runtime libraries.
-  virtual void ModulesDidLoad(lldb_private::ModuleList &module_list);
+  virtual void ModulesDidLoad(const ModuleList &module_list) override;
 
   /// Called before detaching from a process.
   ///
@@ -294,9 +295,6 @@ class SystemRuntime : public PluginInterface {
   }
 
 protected:
-  // Member variables.
-  Process *m_process;
-
   std::vector<ConstString> m_types;
 
 private:
diff --git a/lldb/source/Target/LanguageRuntime.cpp b/lldb/source/Target/LanguageRuntime.cpp
index f8143839bb64c..0bbb9660f7412 100644
--- a/lldb/source/Target/LanguageRuntime.cpp
+++ b/lldb/source/Target/LanguageRuntime.cpp
@@ -214,7 +214,7 @@ LanguageRuntime *LanguageRuntime::FindPlugin(Process *process,
   return nullptr;
 }
 
-LanguageRuntime::LanguageRuntime(Process *process) : m_process(process) {}
+LanguageRuntime::LanguageRuntime(Process *process) : Runtime(process) {}
 
 LanguageRuntime::~LanguageRuntime() = default;
 
diff --git a/lldb/source/Target/SystemRuntime.cpp b/lldb/source/Target/SystemRuntime.cpp
index cd3d8ba2c7b09..6d8a2ef55225f 100644
--- a/lldb/source/Target/SystemRuntime.cpp
+++ b/lldb/source/Target/SystemRuntime.cpp
@@ -27,9 +27,7 @@ SystemRuntime *SystemRuntime::FindPlugin(Process *process) {
   return nullptr;
 }
 
-// SystemRuntime constructor
-SystemRuntime::SystemRuntime(Process *process)
-    : m_process(process), m_types() {}
+SystemRuntime::SystemRuntime(Process *process) : Runtime(process), m_types() {}
 
 SystemRuntime::~SystemRuntime() = default;
 
@@ -39,7 +37,7 @@ void SystemRuntime::DidLaunch() {}
 
 void SystemRuntime::Detach() {}
 
-void SystemRuntime::ModulesDidLoad(ModuleList &module_list) {}
+void SystemRuntime::ModulesDidLoad(const ModuleList &module_list) {}
 
 const std::vector<ConstString> &SystemRuntime::GetExtendedBacktraceTypes() {
   return m_types;

From f320f83f3ac2d9b04fcab9975031fe8afef20253 Mon Sep 17 00:00:00 2001
From: Amara Emerson <amara@apple.com>
Date: Fri, 24 Jul 2020 16:43:55 -0700
Subject: [PATCH 0061/1035] [AArch64][GlobalISel] Promote G_UITOFP vector
 operands to same elt size as result.

Fixes legalization failures.
---
 .../AArch64/GISel/AArch64LegalizerInfo.cpp    |  2 +-
 .../AArch64/GlobalISel/legalize-itofp.mir     | 37 +++++++++++++++++++
 .../GlobalISel/legalizer-info-validation.mir  |  8 ++--
 3 files changed, 42 insertions(+), 5 deletions(-)

diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index 2eaec0b970fa6..bbceb0e169039 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -399,7 +399,7 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
   getActionDefinitionsBuilder({G_SITOFP, G_UITOFP})
       .legalForCartesianProduct({s32, s64, v2s64, v4s32, v2s32})
       .clampScalar(1, s32, s64)
-      .widenScalarToNextPow2(1)
+      .minScalarSameAs(1, 0)
       .clampScalar(0, s32, s64)
       .widenScalarToNextPow2(0);
 
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-itofp.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-itofp.mir
index e0a20e030710e..e348c0e454b80 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-itofp.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-itofp.mir
@@ -24,6 +24,9 @@
   define void @test_sitofp_v4s32() { ret void }
   define void @test_uitofp_v4s32() { ret void }
 
+  define void @test_uitofp_v2s64_v2i1() { ret void }
+  define void @test_sitofp_v2s64_v2i1() { ret void }
+
   define void @test_sitofp_s32_s16() { ret void }
   define void @test_uitofp_s32_s16() { ret void }
 ...
@@ -239,6 +242,40 @@ body: |
     $q0 = COPY %1
 ...
 
+---
+name:            test_uitofp_v2s64_v2i1
+body: |
+  bb.0:
+    liveins: $q0
+    ; CHECK-LABEL: name: test_uitofp_v2s64_v2i1
+    ; CHECK: [[DEF:%[0-9]+]]:_(s1) = G_IMPLICIT_DEF
+    ; CHECK: [[COPY:%[0-9]+]]:_(s1) = COPY [[DEF]](s1)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s1>) = G_BUILD_VECTOR [[DEF]](s1), [[COPY]](s1)
+    ; CHECK: [[ZEXT:%[0-9]+]]:_(<2 x s64>) = G_ZEXT [[BUILD_VECTOR]](<2 x s1>)
+    ; CHECK: [[UITOFP:%[0-9]+]]:_(<2 x s64>) = G_UITOFP [[ZEXT]](<2 x s64>)
+    ; CHECK: $q0 = COPY [[UITOFP]](<2 x s64>)
+    %0:_(<2 x s1>) = G_IMPLICIT_DEF
+    %1:_(<2 x s64>) = G_UITOFP %0
+    $q0 = COPY %1
+...
+
+---
+name:            test_sitofp_v2s64_v2i1
+body: |
+  bb.0:
+    liveins: $q0
+    ; CHECK-LABEL: name: test_sitofp_v2s64_v2i1
+    ; CHECK: [[DEF:%[0-9]+]]:_(s1) = G_IMPLICIT_DEF
+    ; CHECK: [[COPY:%[0-9]+]]:_(s1) = COPY [[DEF]](s1)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s1>) = G_BUILD_VECTOR [[DEF]](s1), [[COPY]](s1)
+    ; CHECK: [[SEXT:%[0-9]+]]:_(<2 x s64>) = G_SEXT [[BUILD_VECTOR]](<2 x s1>)
+    ; CHECK: [[SITOFP:%[0-9]+]]:_(<2 x s64>) = G_SITOFP [[SEXT]](<2 x s64>)
+    ; CHECK: $q0 = COPY [[SITOFP]](<2 x s64>)
+    %0:_(<2 x s1>) = G_IMPLICIT_DEF
+    %1:_(<2 x s64>) = G_SITOFP %0
+    $q0 = COPY %1
+...
+
 ---
 name:            test_sitofp_s32_s16
 body: |
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
index dcd310fba7e9e..320ec99a51892 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
@@ -407,12 +407,12 @@
 # DEBUG-NEXT: .. the first uncovered type index: 2, OK
 # DEBUG-NEXT: .. the first uncovered imm index: 0, OK
 # DEBUG-NEXT: G_SITOFP (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
-# DEBUG-NEXT: .. the first uncovered type index: 2, OK
-# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: G_UITOFP (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
 # DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
-# DEBUG-NEXT: .. the first uncovered type index: 2, OK
-# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: G_FABS (opcode {{[0-9]+}}): 1 type index, 0 imm indices
 # DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
 # DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected

From 4b53072ee526ea41ef918f2e4505d9314bd7fa56 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Sun, 19 Jul 2020 13:09:48 -0400
Subject: [PATCH 0062/1035] GlobalISel: Define mulfix/divfix opcodes

The full expansion involves the funnel shifts, which depend on another
patch to expand those.
---
 .../llvm/CodeGen/GlobalISel/IRTranslator.h    |   2 +
 llvm/include/llvm/Support/TargetOpcodes.def   |  24 +++
 llvm/include/llvm/Target/GenericOpcodes.td    |  71 +++++++++
 .../Target/GlobalISel/SelectionDAGCompat.td   |   8 +
 llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp  |  26 ++++
 .../irtranslator-fixed-point-intrinsics.ll    | 142 ++++++++++++++++++
 .../GlobalISel/legalizer-info-validation.mir  |  24 +++
 7 files changed, 297 insertions(+)
 create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-fixed-point-intrinsics.ll

diff --git a/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h b/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h
index 751ab67c4e973..928743a6cbd7d 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h
@@ -223,6 +223,8 @@ class IRTranslator : public MachineFunctionPass {
 
   bool translateOverflowIntrinsic(const CallInst &CI, unsigned Op,
                                   MachineIRBuilder &MIRBuilder);
+  bool translateFixedPointIntrinsic(unsigned Op, const CallInst &CI,
+                                    MachineIRBuilder &MIRBuilder);
 
   /// Helper function for translateSimpleIntrinsic.
   /// \return The generic opcode for \p IntrinsicID if \p IntrinsicID is a
diff --git a/llvm/include/llvm/Support/TargetOpcodes.def b/llvm/include/llvm/Support/TargetOpcodes.def
index f61b3f9acb3aa..5eb3398562ba2 100644
--- a/llvm/include/llvm/Support/TargetOpcodes.def
+++ b/llvm/include/llvm/Support/TargetOpcodes.def
@@ -469,6 +469,30 @@ HANDLE_TARGET_OPCODE(G_USUBSAT)
 /// Generic saturating signed subtraction.
 HANDLE_TARGET_OPCODE(G_SSUBSAT)
 
+// Perform signed fixed point multiplication
+HANDLE_TARGET_OPCODE(G_SMULFIX)
+
+// Perform unsigned fixed point multiplication
+HANDLE_TARGET_OPCODE(G_UMULFIX)
+
+// Perform signed, saturating fixed point multiplication
+HANDLE_TARGET_OPCODE(G_SMULFIXSAT)
+
+// Perform unsigned, saturating fixed point multiplication
+HANDLE_TARGET_OPCODE(G_UMULFIXSAT)
+
+// Perform signed fixed point division
+HANDLE_TARGET_OPCODE(G_SDIVFIX)
+
+// Perform unsigned fixed point division
+HANDLE_TARGET_OPCODE(G_UDIVFIX)
+
+// Perform signed, saturating fixed point division
+HANDLE_TARGET_OPCODE(G_SDIVFIXSAT)
+
+// Perform unsigned, saturating fixed point division
+HANDLE_TARGET_OPCODE(G_UDIVFIXSAT)
+
 /// Generic FP addition.
 HANDLE_TARGET_OPCODE(G_FADD)
 
diff --git a/llvm/include/llvm/Target/GenericOpcodes.td b/llvm/include/llvm/Target/GenericOpcodes.td
index 03795c0050a2c..5b04713d40e81 100644
--- a/llvm/include/llvm/Target/GenericOpcodes.td
+++ b/llvm/include/llvm/Target/GenericOpcodes.td
@@ -545,6 +545,77 @@ def G_SSUBSAT : GenericInstruction {
   let isCommutable = 0;
 }
 
+/// RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point
+/// multiplication on 2 integers with the same width and scale. SCALE
+/// represents the scale of both operands as fixed point numbers. This
+/// SCALE parameter must be a constant integer. A scale of zero is
+/// effectively performing multiplication on 2 integers.
+def G_SMULFIX : GenericInstruction {
+  let OutOperandList = (outs type0:$dst);
+  let InOperandList = (ins type0:$src0, type0:$src1, untyped_imm_0:$scale);
+  let hasSideEffects = 0;
+  let isCommutable = 1;
+}
+
+def G_UMULFIX : GenericInstruction {
+  let OutOperandList = (outs type0:$dst);
+  let InOperandList = (ins type0:$src0, type0:$src1, untyped_imm_0:$scale);
+  let hasSideEffects = 0;
+  let isCommutable = 1;
+}
+
+/// Same as the corresponding unsaturated fixed point instructions, but the
+/// result is clamped between the min and max values representable by the
+/// bits of the first 2 operands.
+def G_SMULFIXSAT : GenericInstruction {
+  let OutOperandList = (outs type0:$dst);
+  let InOperandList = (ins type0:$src0, type0:$src1, untyped_imm_0:$scale);
+  let hasSideEffects = 0;
+  let isCommutable = 1;
+}
+
+def G_UMULFIXSAT : GenericInstruction {
+  let OutOperandList = (outs type0:$dst);
+  let InOperandList = (ins type0:$src0, type0:$src1, untyped_imm_0:$scale);
+  let hasSideEffects = 0;
+  let isCommutable = 1;
+}
+
+/// RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on
+/// 2 integers with the same width and scale. SCALE represents the scale
+/// of both operands as fixed point numbers. This SCALE parameter must be a
+/// constant integer.
+def G_SDIVFIX : GenericInstruction {
+  let OutOperandList = (outs type0:$dst);
+  let InOperandList = (ins type0:$src0, type0:$src1, untyped_imm_0:$scale);
+  let hasSideEffects = 0;
+  let isCommutable = 0;
+}
+
+def G_UDIVFIX : GenericInstruction {
+  let OutOperandList = (outs type0:$dst);
+  let InOperandList = (ins type0:$src0, type0:$src1, untyped_imm_0:$scale);
+  let hasSideEffects = 0;
+  let isCommutable = 0;
+}
+
+/// Same as the corresponding unsaturated fixed point instructions,
+/// but the result is clamped between the min and max values
+/// representable by the bits of the first 2 operands.
+def G_SDIVFIXSAT : GenericInstruction {
+  let OutOperandList = (outs type0:$dst);
+  let InOperandList = (ins type0:$src0, type0:$src1, untyped_imm_0:$scale);
+  let hasSideEffects = 0;
+  let isCommutable = 0;
+}
+
+def G_UDIVFIXSAT : GenericInstruction {
+  let OutOperandList = (outs type0:$dst);
+  let InOperandList = (ins type0:$src0, type0:$src1, untyped_imm_0:$scale);
+  let hasSideEffects = 0;
+  let isCommutable = 0;
+}
+
 //------------------------------------------------------------------------------
 // Floating Point Unary Ops.
 //------------------------------------------------------------------------------
diff --git a/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td b/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td
index 150834e65b2dc..14718556810f5 100644
--- a/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td
+++ b/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td
@@ -71,6 +71,14 @@ def : GINodeEquiv<G_SADDSAT, saddsat>;
 def : GINodeEquiv<G_UADDSAT, uaddsat>;
 def : GINodeEquiv<G_SSUBSAT, ssubsat>;
 def : GINodeEquiv<G_USUBSAT, usubsat>;
+def : GINodeEquiv<G_SMULFIX, smulfix>;
+def : GINodeEquiv<G_UMULFIX, umulfix>;
+def : GINodeEquiv<G_SMULFIXSAT, smulfixsat>;
+def : GINodeEquiv<G_UMULFIXSAT, umulfixsat>;
+def : GINodeEquiv<G_SDIVFIX, sdivfix>;
+def : GINodeEquiv<G_UDIVFIX, udivfix>;
+def : GINodeEquiv<G_SDIVFIXSAT, sdivfixsat>;
+def : GINodeEquiv<G_UDIVFIXSAT, udivfixsat>;
 def : GINodeEquiv<G_SELECT, select>;
 def : GINodeEquiv<G_FNEG, fneg>;
 def : GINodeEquiv<G_FPEXT, fpextend>;
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index aa898d5a61896..e6b3d51bf1ec4 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -1214,6 +1214,16 @@ bool IRTranslator::translateOverflowIntrinsic(const CallInst &CI, unsigned Op,
   return true;
 }
 
+bool IRTranslator::translateFixedPointIntrinsic(unsigned Op, const CallInst &CI,
+                                                MachineIRBuilder &MIRBuilder) {
+  Register Dst = getOrCreateVReg(CI);
+  Register Src0 = getOrCreateVReg(*CI.getOperand(0));
+  Register Src1 = getOrCreateVReg(*CI.getOperand(1));
+  uint64_t Scale = cast<ConstantInt>(CI.getOperand(2))->getZExtValue();
+  MIRBuilder.buildInstr(Op, {Dst}, { Src0, Src1, Scale });
+  return true;
+}
+
 unsigned IRTranslator::getSimpleIntrinsicOpcode(Intrinsic::ID ID) {
   switch (ID) {
     default:
@@ -1494,6 +1504,22 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
     return translateBinaryOp(TargetOpcode::G_SMIN, CI, MIRBuilder);
   case Intrinsic::smax:
     return translateBinaryOp(TargetOpcode::G_SMAX, CI, MIRBuilder);
+  case Intrinsic::smul_fix:
+    return translateFixedPointIntrinsic(TargetOpcode::G_SMULFIX, CI, MIRBuilder);
+  case Intrinsic::umul_fix:
+    return translateFixedPointIntrinsic(TargetOpcode::G_UMULFIX, CI, MIRBuilder);
+  case Intrinsic::smul_fix_sat:
+    return translateFixedPointIntrinsic(TargetOpcode::G_SMULFIXSAT, CI, MIRBuilder);
+  case Intrinsic::umul_fix_sat:
+    return translateFixedPointIntrinsic(TargetOpcode::G_UMULFIXSAT, CI, MIRBuilder);
+  case Intrinsic::sdiv_fix:
+    return translateFixedPointIntrinsic(TargetOpcode::G_SDIVFIX, CI, MIRBuilder);
+  case Intrinsic::udiv_fix:
+    return translateFixedPointIntrinsic(TargetOpcode::G_UDIVFIX, CI, MIRBuilder);
+  case Intrinsic::sdiv_fix_sat:
+    return translateFixedPointIntrinsic(TargetOpcode::G_SDIVFIXSAT, CI, MIRBuilder);
+  case Intrinsic::udiv_fix_sat:
+    return translateFixedPointIntrinsic(TargetOpcode::G_UDIVFIXSAT, CI, MIRBuilder);
   case Intrinsic::fmuladd: {
     const TargetMachine &TM = MF->getTarget();
     const TargetLowering &TLI = *MF->getSubtarget().getTargetLowering();
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-fixed-point-intrinsics.ll b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-fixed-point-intrinsics.ll
new file mode 100644
index 0000000000000..c70b93e50ded8
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-fixed-point-intrinsics.ll
@@ -0,0 +1,142 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -global-isel -stop-after=irtranslator -mtriple=aarch64-- -verify-machineinstrs -o - %s | FileCheck %s
+
+define i16 @smul_fix(i16 %arg0, i16 %arg1) {
+  ; CHECK-LABEL: name: smul_fix
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $w0, $w1
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $w0
+  ; CHECK:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $w1
+  ; CHECK:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
+  ; CHECK:   [[SMULFIX:%[0-9]+]]:_(s16) = G_SMULFIX [[TRUNC]], [[TRUNC1]], 7
+  ; CHECK:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[SMULFIX]](s16)
+  ; CHECK:   $w0 = COPY [[ANYEXT]](s32)
+  ; CHECK:   RET_ReallyLR implicit $w0
+  %res = call i16 @llvm.smul.fix.i16(i16 %arg0, i16 %arg1, i32 7)
+  ret i16 %res
+}
+
+define i16 @umul_fix(i16 %arg0, i16 %arg1) {
+  ; CHECK-LABEL: name: umul_fix
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $w0, $w1
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $w0
+  ; CHECK:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $w1
+  ; CHECK:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
+  ; CHECK:   [[UMULFIX:%[0-9]+]]:_(s16) = G_UMULFIX [[TRUNC]], [[TRUNC1]], 7
+  ; CHECK:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[UMULFIX]](s16)
+  ; CHECK:   $w0 = COPY [[ANYEXT]](s32)
+  ; CHECK:   RET_ReallyLR implicit $w0
+  %res = call i16 @llvm.umul.fix.i16(i16 %arg0, i16 %arg1, i32 7)
+  ret i16 %res
+}
+
+define i16 @smul_fix_sat(i16 %arg0, i16 %arg1) {
+  ; CHECK-LABEL: name: smul_fix_sat
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $w0, $w1
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $w0
+  ; CHECK:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $w1
+  ; CHECK:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
+  ; CHECK:   [[SMULFIXSAT:%[0-9]+]]:_(s16) = G_SMULFIXSAT [[TRUNC]], [[TRUNC1]], 7
+  ; CHECK:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[SMULFIXSAT]](s16)
+  ; CHECK:   $w0 = COPY [[ANYEXT]](s32)
+  ; CHECK:   RET_ReallyLR implicit $w0
+  %res = call i16 @llvm.smul.fix.sat.i16(i16 %arg0, i16 %arg1, i32 7)
+  ret i16 %res
+}
+
+define i16 @umul_fix_sat(i16 %arg0, i16 %arg1) {
+  ; CHECK-LABEL: name: umul_fix_sat
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $w0, $w1
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $w0
+  ; CHECK:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $w1
+  ; CHECK:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
+  ; CHECK:   [[UMULFIXSAT:%[0-9]+]]:_(s16) = G_UMULFIXSAT [[TRUNC]], [[TRUNC1]], 7
+  ; CHECK:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[UMULFIXSAT]](s16)
+  ; CHECK:   $w0 = COPY [[ANYEXT]](s32)
+  ; CHECK:   RET_ReallyLR implicit $w0
+  %res = call i16 @llvm.umul.fix.sat.i16(i16 %arg0, i16 %arg1, i32 7)
+  ret i16 %res
+}
+
+define i16 @sdiv_fix(i16 %arg0, i16 %arg1) {
+  ; CHECK-LABEL: name: sdiv_fix
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $w0, $w1
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $w0
+  ; CHECK:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $w1
+  ; CHECK:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
+  ; CHECK:   [[SDIVFIX:%[0-9]+]]:_(s16) = G_SDIVFIX [[TRUNC]], [[TRUNC1]], 7
+  ; CHECK:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[SDIVFIX]](s16)
+  ; CHECK:   $w0 = COPY [[ANYEXT]](s32)
+  ; CHECK:   RET_ReallyLR implicit $w0
+  %res = call i16 @llvm.sdiv.fix.i16(i16 %arg0, i16 %arg1, i32 7)
+  ret i16 %res
+}
+
+define i16 @udiv_fix(i16 %arg0, i16 %arg1) {
+  ; CHECK-LABEL: name: udiv_fix
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $w0, $w1
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $w0
+  ; CHECK:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $w1
+  ; CHECK:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
+  ; CHECK:   [[UDIVFIX:%[0-9]+]]:_(s16) = G_UDIVFIX [[TRUNC]], [[TRUNC1]], 7
+  ; CHECK:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[UDIVFIX]](s16)
+  ; CHECK:   $w0 = COPY [[ANYEXT]](s32)
+  ; CHECK:   RET_ReallyLR implicit $w0
+  %res = call i16 @llvm.udiv.fix.i16(i16 %arg0, i16 %arg1, i32 7)
+  ret i16 %res
+}
+
+define i16 @sdiv_fix_sat(i16 %arg0, i16 %arg1) {
+  ; CHECK-LABEL: name: sdiv_fix_sat
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $w0, $w1
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $w0
+  ; CHECK:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $w1
+  ; CHECK:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
+  ; CHECK:   [[SDIVFIXSAT:%[0-9]+]]:_(s16) = G_SDIVFIXSAT [[TRUNC]], [[TRUNC1]], 7
+  ; CHECK:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[SDIVFIXSAT]](s16)
+  ; CHECK:   $w0 = COPY [[ANYEXT]](s32)
+  ; CHECK:   RET_ReallyLR implicit $w0
+  %res = call i16 @llvm.sdiv.fix.sat.i16(i16 %arg0, i16 %arg1, i32 7)
+  ret i16 %res
+}
+
+define i16 @udiv_fix_sat(i16 %arg0, i16 %arg1) {
+  ; CHECK-LABEL: name: udiv_fix_sat
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $w0, $w1
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $w0
+  ; CHECK:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $w1
+  ; CHECK:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
+  ; CHECK:   [[UDIVFIXSAT:%[0-9]+]]:_(s16) = G_UDIVFIXSAT [[TRUNC]], [[TRUNC1]], 7
+  ; CHECK:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[UDIVFIXSAT]](s16)
+  ; CHECK:   $w0 = COPY [[ANYEXT]](s32)
+  ; CHECK:   RET_ReallyLR implicit $w0
+  %res = call i16 @llvm.udiv.fix.sat.i16(i16 %arg0, i16 %arg1, i32 7)
+  ret i16 %res
+}
+
+declare i16 @llvm.smul.fix.i16(i16, i16, i32 immarg) #0
+declare i16 @llvm.umul.fix.i16(i16, i16, i32 immarg) #0
+declare i16 @llvm.smul.fix.sat.i16(i16, i16, i32 immarg) #0
+declare i16 @llvm.umul.fix.sat.i16(i16, i16, i32 immarg) #0
+declare i16 @llvm.sdiv.fix.i16(i16, i16, i32 immarg) #1
+declare i16 @llvm.udiv.fix.i16(i16, i16, i32 immarg) #1
+declare i16 @llvm.sdiv.fix.sat.i16(i16, i16, i32 immarg) #1
+declare i16 @llvm.udiv.fix.sat.i16(i16, i16, i32 immarg) #1
+
+attributes #0 = { nounwind readnone speculatable willreturn }
+attributes #1 = { nounwind readnone }
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
index 320ec99a51892..b21f66e53fd48 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
@@ -337,6 +337,30 @@
 # DEBUG-NEXT: G_SSUBSAT (opcode {{[0-9]+}}): 1 type index, 0 imm indices
 # DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: G_SMULFIX (opcode {{[0-9]+}}): 1 type index, 1 imm index
+# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: G_UMULFIX (opcode {{[0-9]+}}): 1 type index, 1 imm index
+# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: G_SMULFIXSAT (opcode {{[0-9]+}}): 1 type index, 1 imm index
+# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: G_UMULFIXSAT (opcode {{[0-9]+}}): 1 type index, 1 imm index
+# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: G_SDIVFIX (opcode {{[0-9]+}}): 1 type index, 1 imm index
+# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: G_UDIVFIX (opcode {{[0-9]+}}): 1 type index, 1 imm index
+# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: G_SDIVFIXSAT (opcode {{[0-9]+}}): 1 type index, 1 imm index
+# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: G_UDIVFIXSAT (opcode {{[0-9]+}}): 1 type index, 1 imm index
+# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
 # DEBUG-NEXT: G_FADD (opcode {{[0-9]+}}): 1 type index, 0 imm indices
 # DEBUG-NEXT: .. the first uncovered type index: 1, OK
 # DEBUG-NEXT: .. the first uncovered imm index: 0, OK

From 6fdc6f6c7d34af60c45c405f448370a684ef6f2a Mon Sep 17 00:00:00 2001
From: Rong Xu <xur@google.com>
Date: Fri, 24 Jul 2020 17:13:58 -0700
Subject: [PATCH 0063/1035] [PGO][InstrProf] Do not promote count if the exit
 blocks contains ret instruction

Skip profile count promotion if any of the ExitBlocks contains a ret
instruction. This is to prevent dumping of incomplete profile -- if the
the loop is a long running loop and dump is called in the middle
of the loop, the result profile is incomplete.

ExitBlocks containing a ret instruction is an indication of a long running
loop -- early exit to error handling code.

Differential Revision: https://reviews.llvm.org/D84379
---
 .../Instrumentation/PGOInstrumentation.cpp          | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
index 67bf264e8eeaf..c4a43abaa53cc 100644
--- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
+++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
@@ -1131,11 +1131,18 @@ bool PGOUseFunc::setInstrumentedCounts(
   if (NumCounters != CountFromProfile.size()) {
     return false;
   }
+  auto *FuncEntry = &*F.begin();
+
   // Set the profile count to the Instrumented BBs.
   uint32_t I = 0;
   for (BasicBlock *InstrBB : InstrumentBBs) {
     uint64_t CountValue = CountFromProfile[I++];
     UseBBInfo &Info = getBBInfo(InstrBB);
+    // If we reach here, we know that we have some nonzero count
+    // values in this function. The entry count should not be 0.
+    // Fix it if necessary.
+    if (InstrBB == FuncEntry && CountValue == 0)
+      CountValue = 1;
     Info.setBBInfoCount(CountValue);
   }
   ProfileCountSize = CountFromProfile.size();
@@ -1326,7 +1333,6 @@ void PGOUseFunc::populateCounters() {
   }
 #endif
   uint64_t FuncEntryCount = getBBInfo(&*F.begin()).CountValue;
-  F.setEntryCount(ProfileCount(FuncEntryCount, Function::PCT_Real));
   uint64_t FuncMaxCount = FuncEntryCount;
   for (auto &BB : F) {
     auto BI = findBBInfo(&BB);
@@ -1334,6 +1340,11 @@ void PGOUseFunc::populateCounters() {
       continue;
     FuncMaxCount = std::max(FuncMaxCount, BI->CountValue);
   }
+
+  // Fix the obviously inconsistent entry count.
+  if (FuncMaxCount > 0 && FuncEntryCount == 0)
+    FuncEntryCount = 1;
+  F.setEntryCount(ProfileCount(FuncEntryCount, Function::PCT_Real));
   markFunctionAttributes(FuncEntryCount, FuncMaxCount);
 
   // Now annotate select instructions

From fcc55c0952f4e4b30326badd6fb11fa06be2b6fd Mon Sep 17 00:00:00 2001
From: Jessica Paquette <jpaquette@apple.com>
Date: Fri, 24 Jul 2020 16:57:37 -0700
Subject: [PATCH 0064/1035] [AArch64][GlobalISel] Use wzr/xzr for 16 and 32 bit
 stores of zero

We weren't performing this optimization on 16 and 32 bit stores. SDAG happily
does this though.

e.g. https://godbolt.org/z/cWocKr

This saves about 0.2% in code size on CTMark at -O3.

Differential Revision: https://reviews.llvm.org/D84568
---
 .../GISel/AArch64InstructionSelector.cpp      | 15 ++++++---
 .../AArch64/GlobalISel/select-store.mir       | 33 +++++++++++++++++++
 2 files changed, 44 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
index eb6a4aa3d8266..0b0de09fff29e 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
@@ -2306,10 +2306,17 @@ bool AArch64InstructionSelector::select(MachineInstr &I) {
     // If we're storing a 0, use WZR/XZR.
     if (auto CVal = getConstantVRegVal(ValReg, MRI)) {
       if (*CVal == 0 && Opcode == TargetOpcode::G_STORE) {
-        if (I.getOpcode() == AArch64::STRWui)
-          I.getOperand(0).setReg(AArch64::WZR);
-        else if (I.getOpcode() == AArch64::STRXui)
-          I.getOperand(0).setReg(AArch64::XZR);
+        unsigned Opc = I.getOpcode();
+        switch(Opc) {
+          case AArch64::STRWui:
+          case AArch64::STRHHui:
+          case AArch64::STRBBui:
+            I.getOperand(0).setReg(AArch64::WZR);
+            break;
+          case AArch64::STRXui:
+            I.getOperand(0).setReg(AArch64::XZR);
+            break;
+        }
       }
     }
 
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/select-store.mir b/llvm/test/CodeGen/AArch64/GlobalISel/select-store.mir
index b72b8e5d0d700..d60a34ef6631a 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/select-store.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/select-store.mir
@@ -11,6 +11,8 @@
 
   define void @store_zero_s64_gpr(i64* %addr) { ret void }
   define void @store_zero_s32_gpr(i32* %addr) { ret void }
+  define void @store_zero_s16(i32* %addr) { ret void }
+  define void @store_zero_s8(i32* %addr) { ret void }
 
   define void @store_fi_s64_gpr() {
     %ptr0 = alloca i64
@@ -176,6 +178,37 @@ body:             |
 
 ...
 
+---
+name:            store_zero_s16
+legalized:       true
+regBankSelected: true
+body:             |
+  bb.0:
+    liveins: $x0
+    ; CHECK-LABEL: name: store_zero_s16
+    ; CHECK: [[COPY:%[0-9]+]]:gpr64sp = COPY $x0
+    ; CHECK: STRHHui $wzr, [[COPY]], 0 :: (store 2)
+    %0:gpr(p0) = COPY $x0
+    %1:gpr(s16) = G_CONSTANT i16 0
+    G_STORE %1(s16), %0(p0) :: (store 2)
+
+...
+
+---
+name:            store_zero_s8
+legalized:       true
+regBankSelected: true
+body:             |
+  bb.0:
+    liveins: $x0
+    ; CHECK-LABEL: name: store_zero_s8
+    ; CHECK: [[COPY:%[0-9]+]]:gpr64sp = COPY $x0
+    ; CHECK: STRBBui $wzr, [[COPY]], 0 :: (store 1)
+    %0:gpr(p0) = COPY $x0
+    %1:gpr(s8) = G_CONSTANT i8 0
+    G_STORE %1(s8), %0(p0) :: (store 1)
+...
+
 ---
 name:            store_fi_s64_gpr
 legalized:       true

From 74790a5dde9ae01b7e96bea0b2596ef37b5325bd Mon Sep 17 00:00:00 2001
From: Amy Kwan <amy.kwan1@ibm.com>
Date: Fri, 24 Jul 2020 17:41:50 -0500
Subject: [PATCH 0065/1035] [PowerPC] Implement Truncate and Store VSX Vector
 Builtins

This patch implements the `vec_xst_trunc` function in altivec.h in  order to
utilize the Store VSX Vector Rightmost [byte | half | word | doubleword] Indexed
instructions introduced in Power10.

Differential Revision: https://reviews.llvm.org/D82467
---
 clang/lib/Headers/altivec.h                   |  52 +++++
 clang/test/CodeGen/builtins-ppc-p10vector.c   |  48 ++++
 llvm/lib/Target/PowerPC/PPCInstrPrefix.td     |  11 +
 .../CodeGen/PowerPC/builtins-ppc-p10vsx.ll    | 205 +++++++++++++++++-
 4 files changed, 315 insertions(+), 1 deletion(-)

diff --git a/clang/lib/Headers/altivec.h b/clang/lib/Headers/altivec.h
index ac5f43836316e..4e25ec118072b 100644
--- a/clang/lib/Headers/altivec.h
+++ b/clang/lib/Headers/altivec.h
@@ -16597,6 +16597,58 @@ static inline __ATTRS_o_ai void vec_xst(vector unsigned __int128 __vec,
 }
 #endif
 
+/* vec_xst_trunc */
+
+#if defined(__POWER10_VECTOR__) && defined(__VSX__)
+static inline __ATTRS_o_ai void vec_xst_trunc(vector signed __int128 __vec,
+                                              signed long long __offset,
+                                              signed char *__ptr) {
+  *(__ptr + __offset) = (signed char)__vec[0];
+}
+
+static inline __ATTRS_o_ai void vec_xst_trunc(vector unsigned __int128 __vec,
+                                              signed long long __offset,
+                                              unsigned char *__ptr) {
+  *(__ptr + __offset) = (unsigned char)__vec[0];
+}
+
+static inline __ATTRS_o_ai void vec_xst_trunc(vector signed __int128 __vec,
+                                              signed long long __offset,
+                                              signed short *__ptr) {
+  *(__ptr + __offset) = (signed short)__vec[0];
+}
+
+static inline __ATTRS_o_ai void vec_xst_trunc(vector unsigned __int128 __vec,
+                                              signed long long __offset,
+                                              unsigned short *__ptr) {
+  *(__ptr + __offset) = (unsigned short)__vec[0];
+}
+
+static inline __ATTRS_o_ai void vec_xst_trunc(vector signed __int128 __vec,
+                                              signed long long __offset,
+                                              signed int *__ptr) {
+  *(__ptr + __offset) = (signed int)__vec[0];
+}
+
+static inline __ATTRS_o_ai void vec_xst_trunc(vector unsigned __int128 __vec,
+                                              signed long long __offset,
+                                              unsigned int *__ptr) {
+  *(__ptr + __offset) = (unsigned int)__vec[0];
+}
+
+static inline __ATTRS_o_ai void vec_xst_trunc(vector signed __int128 __vec,
+                                              signed long long __offset,
+                                              signed long long *__ptr) {
+  *(__ptr + __offset) = (signed long long)__vec[0];
+}
+
+static inline __ATTRS_o_ai void vec_xst_trunc(vector unsigned __int128 __vec,
+                                              signed long long __offset,
+                                              unsigned long long *__ptr) {
+  *(__ptr + __offset) = (unsigned long long)__vec[0];
+}
+#endif
+
 /* vec_xst_be */
 
 #ifdef __LITTLE_ENDIAN__
diff --git a/clang/test/CodeGen/builtins-ppc-p10vector.c b/clang/test/CodeGen/builtins-ppc-p10vector.c
index 6f38ac77ee242..2182a19f2452d 100644
--- a/clang/test/CodeGen/builtins-ppc-p10vector.c
+++ b/clang/test/CodeGen/builtins-ppc-p10vector.c
@@ -582,6 +582,54 @@ vector float test_vec_vec_splati_ins_f(void) {
   return vec_splati_ins(vfa, 0, 1.0f);
 }
 
+void test_vec_xst_trunc_sc(vector signed __int128 __a, signed long long __b,
+                           signed char *__c) {
+  // CHECK: store i8 %{{.+}}, i8* %{{.+}}, align 1
+  vec_xst_trunc(__a, __b, __c);
+}
+
+void test_vec_xst_trunc_uc(vector unsigned __int128 __a, signed long long __b,
+                           unsigned char *__c) {
+  // CHECK: store i8 %{{.+}}, i8* %{{.+}}, align 1
+  vec_xst_trunc(__a, __b, __c);
+}
+
+void test_vec_xst_trunc_ss(vector signed __int128 __a, signed long long __b,
+                           signed short *__c) {
+  // CHECK: store i16 %{{.+}}, i16* %{{.+}}, align 2
+  vec_xst_trunc(__a, __b, __c);
+}
+
+void test_vec_xst_trunc_us(vector unsigned __int128 __a, signed long long __b,
+                           unsigned short *__c) {
+  // CHECK: store i16 %{{.+}}, i16* %{{.+}}, align 2
+  vec_xst_trunc(__a, __b, __c);
+}
+
+void test_vec_xst_trunc_si(vector signed __int128 __a, signed long long __b,
+                           signed int *__c) {
+  // CHECK: store i32 %{{.+}}, i32* %{{.+}}, align 4
+  vec_xst_trunc(__a, __b, __c);
+}
+
+void test_vec_xst_trunc_ui(vector unsigned __int128 __a, signed long long __b,
+                           unsigned int *__c) {
+  // CHECK: store i32 %{{.+}}, i32* %{{.+}}, align 4
+  vec_xst_trunc(__a, __b, __c);
+}
+
+void test_vec_xst_trunc_sll(vector signed __int128 __a, signed long long __b,
+                            signed long long *__c) {
+  // CHECK: store i64 %{{.+}}, i64* %{{.+}}, align 8
+  vec_xst_trunc(__a, __b, __c);
+}
+
+void test_vec_xst_trunc_ull(vector unsigned __int128 __a, signed long long __b,
+                            unsigned long long *__c) {
+  // CHECK: store i64 %{{.+}}, i64* %{{.+}}, align 8
+  vec_xst_trunc(__a, __b, __c);
+}
+
 int test_vec_test_lsbb_all_ones(void) {
   // CHECK: @llvm.ppc.vsx.xvtlsbb(<16 x i8> %{{.+}}, i32 1
   // CHECK-NEXT: ret i32
diff --git a/llvm/lib/Target/PowerPC/PPCInstrPrefix.td b/llvm/lib/Target/PowerPC/PPCInstrPrefix.td
index b468a8f318ee3..4e048ee9930e9 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrPrefix.td
+++ b/llvm/lib/Target/PowerPC/PPCInstrPrefix.td
@@ -1100,6 +1100,17 @@ let Predicates = [IsISA3_1] in {
             (EXTRACT_SUBREG (XVTLSBB (COPY_TO_REGCLASS $XB, VSRC)), sub_eq)>;
 }
 
+let AddedComplexity = 400, Predicates = [IsISA3_1] in {
+  def : Pat<(truncstorei8 (i32 (vector_extract v16i8:$rS, 0)), xoaddr:$src),
+            (STXVRBX (COPY_TO_REGCLASS $rS, VSRC), xoaddr:$src)>;
+  def : Pat<(truncstorei16 (i32 (vector_extract v8i16:$rS, 0)), xoaddr:$src),
+            (STXVRHX (COPY_TO_REGCLASS $rS, VSRC), xoaddr:$src)>;
+  def : Pat<(store (i32 (vector_extract v4i32:$rS, 0)), xoaddr:$src),
+            (STXVRWX (COPY_TO_REGCLASS $rS, VSRC), xoaddr:$src)>;
+  def : Pat<(store (i64 (vector_extract v2i64:$rS, 0)), xoaddr:$src),
+            (STXVRDX (COPY_TO_REGCLASS $rS, VSRC), xoaddr:$src)>;
+}
+
 let AddedComplexity = 400, Predicates = [PrefixInstrs] in {
  def : Pat<(v4i32 (build_vector i32immNonAllOneNonZero:$A,
                                 i32immNonAllOneNonZero:$A,
diff --git a/llvm/test/CodeGen/PowerPC/builtins-ppc-p10vsx.ll b/llvm/test/CodeGen/PowerPC/builtins-ppc-p10vsx.ll
index 2ac1b2b7514bc..faddb5b4cc7f0 100644
--- a/llvm/test/CodeGen/PowerPC/builtins-ppc-p10vsx.ll
+++ b/llvm/test/CodeGen/PowerPC/builtins-ppc-p10vsx.ll
@@ -2,9 +2,12 @@
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu \
 ; RUN:   -mcpu=pwr10 -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr < %s | \
 ; RUN:   FileCheck %s
-; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O0 \
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu \
 ; RUN:   -mcpu=pwr10 -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr < %s | \
 ; RUN:   FileCheck %s
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O0 \
+; RUN:   -mcpu=pwr10 -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr < %s | \
+; RUN:   FileCheck %s --check-prefix=CHECK-O0
 
 ; These test cases aims to test the builtins for the Power10 VSX vector
 ; instructions introduced in ISA 3.1.
@@ -19,6 +22,14 @@ define signext i32 @test_vec_test_lsbb_all_ones(<16 x i8> %vuca) {
 ; CHECK-NEXT:    srwi r3, r3, 31
 ; CHECK-NEXT:    extsw r3, r3
 ; CHECK-NEXT:    blr
+;
+; CHECK-O0-LABEL: test_vec_test_lsbb_all_ones:
+; CHECK-O0:       # %bb.0: # %entry
+; CHECK-O0-NEXT:    xvtlsbb cr0, v2
+; CHECK-O0-NEXT:    mfocrf r3, 128
+; CHECK-O0-NEXT:    srwi r3, r3, 31
+; CHECK-O0-NEXT:    extsw r3, r3
+; CHECK-O0-NEXT:    blr
 entry:
   %0 = tail call i32 @llvm.ppc.vsx.xvtlsbb(<16 x i8> %vuca, i32 1)
   ret i32 %0
@@ -32,7 +43,199 @@ define signext i32 @test_vec_test_lsbb_all_zeros(<16 x i8> %vuca) {
 ; CHECK-NEXT:    rlwinm r3, r3, 3, 31, 31
 ; CHECK-NEXT:    extsw r3, r3
 ; CHECK-NEXT:    blr
+;
+; CHECK-O0-LABEL: test_vec_test_lsbb_all_zeros:
+; CHECK-O0:       # %bb.0: # %entry
+; CHECK-O0-NEXT:    xvtlsbb cr0, v2
+; CHECK-O0-NEXT:    mfocrf r3, 128
+; CHECK-O0-NEXT:    rlwinm r3, r3, 3, 31, 31
+; CHECK-O0-NEXT:    extsw r3, r3
+; CHECK-O0-NEXT:    blr
 entry:
   %0 = tail call i32 @llvm.ppc.vsx.xvtlsbb(<16 x i8> %vuca, i32 0)
   ret i32 %0
 }
+
+define void @vec_xst_trunc_sc(<1 x i128> %__vec, i64 %__offset, i8* nocapture %__ptr) {
+; CHECK-LABEL: vec_xst_trunc_sc:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    stxvrbx v2, r6, r5
+; CHECK-NEXT:    blr
+;
+; CHECK-O0-LABEL: vec_xst_trunc_sc:
+; CHECK-O0:       # %bb.0: # %entry
+; CHECK-O0-NEXT:    li r3, 0
+; CHECK-O0-NEXT:    vextubrx r3, r3, v2
+; CHECK-O0-NEXT:    # kill: def $r3 killed $r3 killed $x3
+; CHECK-O0-NEXT:    add r4, r6, r5
+; CHECK-O0-NEXT:    stb r3, 0(r4)
+; CHECK-O0-NEXT:    blr
+entry:
+  %0 = bitcast <1 x i128> %__vec to <16 x i8>
+  %conv = extractelement <16 x i8> %0, i32 0
+  %add.ptr = getelementptr inbounds i8, i8* %__ptr, i64 %__offset
+  store i8 %conv, i8* %add.ptr, align 1
+  ret void
+}
+
+define void @vec_xst_trunc_uc(<1 x i128> %__vec, i64 %__offset, i8* nocapture %__ptr) {
+; CHECK-LABEL: vec_xst_trunc_uc:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    stxvrbx v2, r6, r5
+; CHECK-NEXT:    blr
+;
+; CHECK-O0-LABEL: vec_xst_trunc_uc:
+; CHECK-O0:       # %bb.0: # %entry
+; CHECK-O0-NEXT:    li r3, 0
+; CHECK-O0-NEXT:    vextubrx r3, r3, v2
+; CHECK-O0-NEXT:    # kill: def $r3 killed $r3 killed $x3
+; CHECK-O0-NEXT:    add r4, r6, r5
+; CHECK-O0-NEXT:    stb r3, 0(r4)
+; CHECK-O0-NEXT:    blr
+entry:
+  %0 = bitcast <1 x i128> %__vec to <16 x i8>
+  %conv = extractelement <16 x i8> %0, i32 0
+  %add.ptr = getelementptr inbounds i8, i8* %__ptr, i64 %__offset
+  store i8 %conv, i8* %add.ptr, align 1
+  ret void
+}
+
+define void @vec_xst_trunc_ss(<1 x i128> %__vec, i64 %__offset, i16* nocapture %__ptr) {
+; CHECK-LABEL: vec_xst_trunc_ss:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sldi r3, r5, 1
+; CHECK-NEXT:    stxvrhx v2, r6, r3
+; CHECK-NEXT:    blr
+;
+; CHECK-O0-LABEL: vec_xst_trunc_ss:
+; CHECK-O0:       # %bb.0: # %entry
+; CHECK-O0-NEXT:    li r3, 0
+; CHECK-O0-NEXT:    vextuhrx r3, r3, v2
+; CHECK-O0-NEXT:    # kill: def $r3 killed $r3 killed $x3
+; CHECK-O0-NEXT:    sldi r4, r5, 1
+; CHECK-O0-NEXT:    add r4, r6, r4
+; CHECK-O0-NEXT:    sth r3, 0(r4)
+; CHECK-O0-NEXT:    blr
+entry:
+  %0 = bitcast <1 x i128> %__vec to <8 x i16>
+  %conv = extractelement <8 x i16> %0, i32 0
+  %add.ptr = getelementptr inbounds i16, i16* %__ptr, i64 %__offset
+  store i16 %conv, i16* %add.ptr, align 2
+  ret void
+}
+
+define void @vec_xst_trunc_us(<1 x i128> %__vec, i64 %__offset, i16* nocapture %__ptr) {
+; CHECK-LABEL: vec_xst_trunc_us:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sldi r3, r5, 1
+; CHECK-NEXT:    stxvrhx v2, r6, r3
+; CHECK-NEXT:    blr
+;
+; CHECK-O0-LABEL: vec_xst_trunc_us:
+; CHECK-O0:       # %bb.0: # %entry
+; CHECK-O0-NEXT:    li r3, 0
+; CHECK-O0-NEXT:    vextuhrx r3, r3, v2
+; CHECK-O0-NEXT:    # kill: def $r3 killed $r3 killed $x3
+; CHECK-O0-NEXT:    sldi r4, r5, 1
+; CHECK-O0-NEXT:    add r4, r6, r4
+; CHECK-O0-NEXT:    sth r3, 0(r4)
+; CHECK-O0-NEXT:    blr
+entry:
+  %0 = bitcast <1 x i128> %__vec to <8 x i16>
+  %conv = extractelement <8 x i16> %0, i32 0
+  %add.ptr = getelementptr inbounds i16, i16* %__ptr, i64 %__offset
+  store i16 %conv, i16* %add.ptr, align 2
+  ret void
+}
+
+define void @vec_xst_trunc_si(<1 x i128> %__vec, i64 %__offset, i32* nocapture %__ptr) {
+; CHECK-LABEL: vec_xst_trunc_si:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sldi r3, r5, 2
+; CHECK-NEXT:    stxvrwx v2, r6, r3
+; CHECK-NEXT:    blr
+;
+; CHECK-O0-LABEL: vec_xst_trunc_si:
+; CHECK-O0:       # %bb.0: # %entry
+; CHECK-O0-NEXT:    li r3, 0
+; CHECK-O0-NEXT:    vextuwrx r3, r3, v2
+; CHECK-O0-NEXT:    # kill: def $r3 killed $r3 killed $x3
+; CHECK-O0-NEXT:    sldi r4, r5, 2
+; CHECK-O0-NEXT:    add r4, r6, r4
+; CHECK-O0-NEXT:    stw r3, 0(r4)
+; CHECK-O0-NEXT:    blr
+entry:
+  %0 = bitcast <1 x i128> %__vec to <4 x i32>
+  %conv = extractelement <4 x i32> %0, i32 0
+  %add.ptr = getelementptr inbounds i32, i32* %__ptr, i64 %__offset
+  store i32 %conv, i32* %add.ptr, align 4
+  ret void
+}
+
+define void @vec_xst_trunc_ui(<1 x i128> %__vec, i64 %__offset, i32* nocapture %__ptr) {
+; CHECK-LABEL: vec_xst_trunc_ui:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sldi r3, r5, 2
+; CHECK-NEXT:    stxvrwx v2, r6, r3
+; CHECK-NEXT:    blr
+;
+; CHECK-O0-LABEL: vec_xst_trunc_ui:
+; CHECK-O0:       # %bb.0: # %entry
+; CHECK-O0-NEXT:    li r3, 0
+; CHECK-O0-NEXT:    vextuwrx r3, r3, v2
+; CHECK-O0-NEXT:    # kill: def $r3 killed $r3 killed $x3
+; CHECK-O0-NEXT:    sldi r4, r5, 2
+; CHECK-O0-NEXT:    add r4, r6, r4
+; CHECK-O0-NEXT:    stw r3, 0(r4)
+; CHECK-O0-NEXT:    blr
+entry:
+  %0 = bitcast <1 x i128> %__vec to <4 x i32>
+  %conv = extractelement <4 x i32> %0, i32 0
+  %add.ptr = getelementptr inbounds i32, i32* %__ptr, i64 %__offset
+  store i32 %conv, i32* %add.ptr, align 4
+  ret void
+}
+
+define void @vec_xst_trunc_sll(<1 x i128> %__vec, i64 %__offset, i64* nocapture %__ptr)  {
+; CHECK-LABEL: vec_xst_trunc_sll:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sldi r3, r5, 3
+; CHECK-NEXT:    stxvrdx v2, r6, r3
+; CHECK-NEXT:    blr
+;
+; CHECK-O0-LABEL: vec_xst_trunc_sll:
+; CHECK-O0:       # %bb.0: # %entry
+; CHECK-O0-NEXT:    mfvsrld r3, v2
+; CHECK-O0-NEXT:    sldi r4, r5, 3
+; CHECK-O0-NEXT:    add r4, r6, r4
+; CHECK-O0-NEXT:    std r3, 0(r4)
+; CHECK-O0-NEXT:    blr
+entry:
+  %0 = bitcast <1 x i128> %__vec to <2 x i64>
+  %conv = extractelement <2 x i64> %0, i32 0
+  %add.ptr = getelementptr inbounds i64, i64* %__ptr, i64 %__offset
+  store i64 %conv, i64* %add.ptr, align 8
+  ret void
+}
+
+define void @vec_xst_trunc_ull(<1 x i128> %__vec, i64 %__offset, i64* nocapture %__ptr)  {
+; CHECK-LABEL: vec_xst_trunc_ull:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sldi r3, r5, 3
+; CHECK-NEXT:    stxvrdx v2, r6, r3
+; CHECK-NEXT:    blr
+;
+; CHECK-O0-LABEL: vec_xst_trunc_ull:
+; CHECK-O0:       # %bb.0: # %entry
+; CHECK-O0-NEXT:    mfvsrld r3, v2
+; CHECK-O0-NEXT:    sldi r4, r5, 3
+; CHECK-O0-NEXT:    add r4, r6, r4
+; CHECK-O0-NEXT:    std r3, 0(r4)
+; CHECK-O0-NEXT:    blr
+entry:
+  %0 = bitcast <1 x i128> %__vec to <2 x i64>
+  %conv = extractelement <2 x i64> %0, i32 0
+  %add.ptr = getelementptr inbounds i64, i64* %__ptr, i64 %__offset
+  store i64 %conv, i64* %add.ptr, align 8
+  ret void
+}

From 867ef4472d8e57384c929e4f06b74d1ac8883a99 Mon Sep 17 00:00:00 2001
From: Rong Xu <xur@google.com>
Date: Fri, 24 Jul 2020 17:16:25 -0700
Subject: [PATCH 0066/1035] [PGO][InstrProf] Do not promote count if the exit
 blocks contains ret instruction

Forgot including the tests in the commit 6fdc6f6c7d34af60c4.
---
 .../Inputs/fix_entry_count.proftext           |  8 +++++
 .../Transforms/PGOProfile/fix_entry_count.ll  | 34 +++++++++++++++++++
 2 files changed, 42 insertions(+)
 create mode 100644 llvm/test/Transforms/PGOProfile/Inputs/fix_entry_count.proftext
 create mode 100644 llvm/test/Transforms/PGOProfile/fix_entry_count.ll

diff --git a/llvm/test/Transforms/PGOProfile/Inputs/fix_entry_count.proftext b/llvm/test/Transforms/PGOProfile/Inputs/fix_entry_count.proftext
new file mode 100644
index 0000000000000..de4771beaacc2
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/Inputs/fix_entry_count.proftext
@@ -0,0 +1,8 @@
+:ir
+:entry_first
+test_simple_for
+34137660316
+2
+0
+96
+
diff --git a/llvm/test/Transforms/PGOProfile/fix_entry_count.ll b/llvm/test/Transforms/PGOProfile/fix_entry_count.ll
new file mode 100644
index 0000000000000..5923bc923a6a8
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/fix_entry_count.ll
@@ -0,0 +1,34 @@
+; RUN: llvm-profdata merge %S/Inputs/fix_entry_count.proftext -o %t.profdata
+; RUN: opt < %s -pgo-instr-use -pgo-instrument-entry=true -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
+; RUN: opt < %s -passes=pgo-instr-use -pgo-instrument-entry=true -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i32 @test_simple_for(i32 %n) {
+; USE: define i32 @test_simple_for(i32 %n)
+; USE-SAME: !prof ![[ENTRY_COUNT:[0-9]*]]
+entry:
+  br label %for.cond
+
+for.cond:
+  %i = phi i32 [ 0, %entry ], [ %inc1, %for.inc ]
+  %sum = phi i32 [ 1, %entry ], [ %inc, %for.inc ]
+  %cmp = icmp slt i32 %i, %n
+  br i1 %cmp, label %for.body, label %for.end
+; USE: br i1 %cmp, label %for.body, label %for.end
+; USE-SAME: !prof ![[BW_FOR_COND:[0-9]+]]
+
+for.body:
+  %inc = add nsw i32 %sum, 1
+  br label %for.inc
+
+for.inc:
+  %inc1 = add nsw i32 %i, 1
+  br label %for.cond
+
+for.end:
+  ret i32 %sum
+}
+; USE: ![[ENTRY_COUNT]] = !{!"function_entry_count", i64 1}
+; USE: ![[BW_FOR_COND]] = !{!"branch_weights", i32 96, i32 1}

From dcf1bca0de3b2690c017c545466234198573b4dd Mon Sep 17 00:00:00 2001
From: Rong Xu <xur@google.com>
Date: Fri, 24 Jul 2020 17:33:49 -0700
Subject: [PATCH 0067/1035] Revert "[PGO][InstrProf] Do not promote count if
 the exit blocks contains ret instruction"

This reverts commit 867ef4472d8e57384c929e4f06b74d1ac8883a99.
---
 .../Inputs/fix_entry_count.proftext           |  8 -----
 .../Transforms/PGOProfile/fix_entry_count.ll  | 34 -------------------
 2 files changed, 42 deletions(-)
 delete mode 100644 llvm/test/Transforms/PGOProfile/Inputs/fix_entry_count.proftext
 delete mode 100644 llvm/test/Transforms/PGOProfile/fix_entry_count.ll

diff --git a/llvm/test/Transforms/PGOProfile/Inputs/fix_entry_count.proftext b/llvm/test/Transforms/PGOProfile/Inputs/fix_entry_count.proftext
deleted file mode 100644
index de4771beaacc2..0000000000000
--- a/llvm/test/Transforms/PGOProfile/Inputs/fix_entry_count.proftext
+++ /dev/null
@@ -1,8 +0,0 @@
-:ir
-:entry_first
-test_simple_for
-34137660316
-2
-0
-96
-
diff --git a/llvm/test/Transforms/PGOProfile/fix_entry_count.ll b/llvm/test/Transforms/PGOProfile/fix_entry_count.ll
deleted file mode 100644
index 5923bc923a6a8..0000000000000
--- a/llvm/test/Transforms/PGOProfile/fix_entry_count.ll
+++ /dev/null
@@ -1,34 +0,0 @@
-; RUN: llvm-profdata merge %S/Inputs/fix_entry_count.proftext -o %t.profdata
-; RUN: opt < %s -pgo-instr-use -pgo-instrument-entry=true -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
-; RUN: opt < %s -passes=pgo-instr-use -pgo-instrument-entry=true -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
-
-target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
-target triple = "x86_64-unknown-linux-gnu"
-
-define i32 @test_simple_for(i32 %n) {
-; USE: define i32 @test_simple_for(i32 %n)
-; USE-SAME: !prof ![[ENTRY_COUNT:[0-9]*]]
-entry:
-  br label %for.cond
-
-for.cond:
-  %i = phi i32 [ 0, %entry ], [ %inc1, %for.inc ]
-  %sum = phi i32 [ 1, %entry ], [ %inc, %for.inc ]
-  %cmp = icmp slt i32 %i, %n
-  br i1 %cmp, label %for.body, label %for.end
-; USE: br i1 %cmp, label %for.body, label %for.end
-; USE-SAME: !prof ![[BW_FOR_COND:[0-9]+]]
-
-for.body:
-  %inc = add nsw i32 %sum, 1
-  br label %for.inc
-
-for.inc:
-  %inc1 = add nsw i32 %i, 1
-  br label %for.cond
-
-for.end:
-  ret i32 %sum
-}
-; USE: ![[ENTRY_COUNT]] = !{!"function_entry_count", i64 1}
-; USE: ![[BW_FOR_COND]] = !{!"branch_weights", i32 96, i32 1}

From 5546c2ab42d474dde9b490035ed69e96989f22c5 Mon Sep 17 00:00:00 2001
From: Rong Xu <xur@google.com>
Date: Fri, 24 Jul 2020 17:35:44 -0700
Subject: [PATCH 0068/1035] Revert "[PGO][InstrProf] Do not promote count if
 the exit blocks contains ret instruction"

This reverts commit 6fdc6f6c7d34af60c45c405f448370a684ef6f2a.
---
 .../Instrumentation/PGOInstrumentation.cpp          | 13 +------------
 1 file changed, 1 insertion(+), 12 deletions(-)

diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
index c4a43abaa53cc..67bf264e8eeaf 100644
--- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
+++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
@@ -1131,18 +1131,11 @@ bool PGOUseFunc::setInstrumentedCounts(
   if (NumCounters != CountFromProfile.size()) {
     return false;
   }
-  auto *FuncEntry = &*F.begin();
-
   // Set the profile count to the Instrumented BBs.
   uint32_t I = 0;
   for (BasicBlock *InstrBB : InstrumentBBs) {
     uint64_t CountValue = CountFromProfile[I++];
     UseBBInfo &Info = getBBInfo(InstrBB);
-    // If we reach here, we know that we have some nonzero count
-    // values in this function. The entry count should not be 0.
-    // Fix it if necessary.
-    if (InstrBB == FuncEntry && CountValue == 0)
-      CountValue = 1;
     Info.setBBInfoCount(CountValue);
   }
   ProfileCountSize = CountFromProfile.size();
@@ -1333,6 +1326,7 @@ void PGOUseFunc::populateCounters() {
   }
 #endif
   uint64_t FuncEntryCount = getBBInfo(&*F.begin()).CountValue;
+  F.setEntryCount(ProfileCount(FuncEntryCount, Function::PCT_Real));
   uint64_t FuncMaxCount = FuncEntryCount;
   for (auto &BB : F) {
     auto BI = findBBInfo(&BB);
@@ -1340,11 +1334,6 @@ void PGOUseFunc::populateCounters() {
       continue;
     FuncMaxCount = std::max(FuncMaxCount, BI->CountValue);
   }
-
-  // Fix the obviously inconsistent entry count.
-  if (FuncMaxCount > 0 && FuncEntryCount == 0)
-    FuncEntryCount = 1;
-  F.setEntryCount(ProfileCount(FuncEntryCount, Function::PCT_Real));
   markFunctionAttributes(FuncEntryCount, FuncMaxCount);
 
   // Now annotate select instructions

From 31bd15c562449954d8211c067fc38b1907d60615 Mon Sep 17 00:00:00 2001
From: Rong Xu <xur@google.com>
Date: Fri, 24 Jul 2020 17:38:31 -0700
Subject: [PATCH 0069/1035] [PGO][InstrProf] Do not promote count if the exit
 blocks contains ret instruction

Skip profile count promotion if any of the ExitBlocks contains a ret
instruction. This is to prevent dumping of incomplete profile -- if the
the loop is a long running loop and dump is called in the middle
of the loop, the result profile is incomplete.

ExitBlocks containing a ret instruction is an indication of a long running
loop -- early exit to error handling code.

Differential Revision: https://reviews.llvm.org/D84379
---
 .../Instrumentation/InstrProfiling.cpp        | 16 ++++++++
 .../Transforms/PGOProfile/counter_promo.ll    | 10 ++---
 .../PGOProfile/not_promote_ret_exit.ll        | 38 +++++++++++++++++++
 3 files changed, 59 insertions(+), 5 deletions(-)
 create mode 100644 llvm/test/Transforms/PGOProfile/not_promote_ret_exit.ll

diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
index 7b03bbfcdfe4b..8279716002864 100644
--- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
+++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
@@ -150,6 +150,10 @@ cl::opt<bool> IterativeCounterPromotion(
     cl::ZeroOrMore, "iterative-counter-promotion", cl::init(true),
     cl::desc("Allow counter promotion across the whole loop nest."));
 
+cl::opt<bool> SkipRetExitBlock(
+    cl::ZeroOrMore, "skip-ret-exit-block", cl::init(true),
+    cl::desc("Suppress counter promotion if exit blocks contain ret."));
+
 class InstrProfilingLegacyPass : public ModulePass {
   InstrProfiling InstrProf;
 
@@ -272,6 +276,18 @@ class PGOCounterPromoter {
     // Skip 'infinite' loops:
     if (ExitBlocks.size() == 0)
       return false;
+
+    // Skip if any of the ExitBlocks contains a ret instruction.
+    // This is to prevent dumping of incomplete profile -- if the
+    // the loop is a long running loop and dump is called in the middle
+    // of the loop, the result profile is incomplete.
+    // FIXME: add other heuristics to detect long running loops.
+    if (SkipRetExitBlock) {
+      for (auto BB : ExitBlocks)
+        if (dyn_cast<ReturnInst>(BB->getTerminator()) != nullptr)
+          return false;
+    }
+
     unsigned MaxProm = getMaxNumOfPromotionsInLoop(&L);
     if (MaxProm == 0)
       return false;
diff --git a/llvm/test/Transforms/PGOProfile/counter_promo.ll b/llvm/test/Transforms/PGOProfile/counter_promo.ll
index 812d0fefaa79e..55930c1be1660 100644
--- a/llvm/test/Transforms/PGOProfile/counter_promo.ll
+++ b/llvm/test/Transforms/PGOProfile/counter_promo.ll
@@ -1,12 +1,12 @@
-; RUN: opt < %s -pgo-instr-gen -instrprof -do-counter-promotion=true -S | FileCheck --check-prefix=PROMO --check-prefix=NONATOMIC_PROMO %s
-; RUN: opt < %s --passes=pgo-instr-gen,instrprof -do-counter-promotion=true -S | FileCheck --check-prefix=PROMO --check-prefix=NONATOMIC_PROMO %s 
-; RUN: opt < %s -pgo-instr-gen -instrprof -do-counter-promotion=true -atomic-counter-update-promoted -S | FileCheck --check-prefix=PROMO --check-prefix=ATOMIC_PROMO %s
-; RUN: opt < %s --passes=pgo-instr-gen,instrprof -do-counter-promotion=true -atomic-counter-update-promoted -S | FileCheck --check-prefix=PROMO --check-prefix=ATOMIC_PROMO %s 
+; RUN: opt < %s -pgo-instr-gen -instrprof -do-counter-promotion=true -skip-ret-exit-block=0 -S | FileCheck --check-prefix=PROMO --check-prefix=NONATOMIC_PROMO %s
+; RUN: opt < %s --passes=pgo-instr-gen,instrprof -do-counter-promotion=true -skip-ret-exit-block=0 -S | FileCheck --check-prefix=PROMO --check-prefix=NONATOMIC_PROMO %s
+; RUN: opt < %s -pgo-instr-gen -instrprof -do-counter-promotion=true -atomic-counter-update-promoted -skip-ret-exit-block=0 -S | FileCheck --check-prefix=PROMO --check-prefix=ATOMIC_PROMO %s
+; RUN: opt < %s --passes=pgo-instr-gen,instrprof -do-counter-promotion=true -atomic-counter-update-promoted -skip-ret-exit-block=0 -S | FileCheck --check-prefix=PROMO --check-prefix=ATOMIC_PROMO %s
 
 define void @foo(i32 %n, i32 %N) {
 ; PROMO-LABEL: @foo
 ; PROMO: {{.*}} = load {{.*}} @__profc_foo{{.*}} 3)
-; PROMO-NEXT: add 
+; PROMO-NEXT: add
 ; PROMO-NEXT: store {{.*}}@__profc_foo{{.*}}3)
 bb:
   %tmp = add nsw i32 %n, 1
diff --git a/llvm/test/Transforms/PGOProfile/not_promote_ret_exit.ll b/llvm/test/Transforms/PGOProfile/not_promote_ret_exit.ll
new file mode 100644
index 0000000000000..25d9784e23b1f
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/not_promote_ret_exit.ll
@@ -0,0 +1,38 @@
+; RUN: opt < %s -instrprof -S -o - -do-counter-promotion=1  -skip-ret-exit-block=1 | FileCheck %s --check-prefixes=CHECK,SKIP
+; RUN: opt < %s -instrprof -S -o - -do-counter-promotion=1  -skip-ret-exit-block=0 | FileCheck %s --check-prefixes=CHECK,NOTSKIP
+
+$__llvm_profile_raw_version = comdat any
+
+@bar = dso_local local_unnamed_addr global i32 0, align 4
+@__llvm_profile_raw_version = constant i64 72057594037927941, comdat
+@__profn_foo = private constant [3 x i8] c"foo"
+
+define dso_local void @foo(i32 %n)  {
+entry:
+  call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_foo, i32 0, i32 0), i64 29212902728, i32 2, i32 1)
+  br label %for.cond
+
+for.cond:
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %if.end ]
+  %cmp = icmp slt i32 %i.0, %n
+  %0 = load i32, i32* @bar, align 4
+  %tobool.not = icmp eq i32 %0, 0
+  %or.cond = and i1 %cmp, %tobool.not
+  br i1 %or.cond, label %if.end, label %cleanup
+
+if.end:
+  call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_foo, i32 0, i32 0), i64 29212902728, i32 2, i32 0)
+  call void (...) @bar2()
+  %inc = add nuw nsw i32 %i.0, 1
+  br label %for.cond
+
+cleanup:
+; CHECK: cleanup:
+; SKIP-NOT:  %pgocount.promoted
+; NOTSKIP:  %pgocount.promoted
+  ret void
+}
+
+declare dso_local void @bar2(...)
+
+declare void @llvm.instrprof.increment(i8*, i64, i32, i32)

From 1dd39b1133136c58847d9f2d3c73d2ad8e76e25d Mon Sep 17 00:00:00 2001
From: Rong Xu <xur@google.com>
Date: Fri, 24 Jul 2020 17:39:55 -0700
Subject: [PATCH 0070/1035] [PGO] Fix incorrect function entry count

Function entry count might be zero after the profile counts reset and
before reentry to the function.

Zero profile entry count is very bad as the profile count from BFI will
be wrong.

A simple fix is to set the profile entry count to 1 if there are
non-zero profile counts in this function.

Differential Revision: https://reviews.llvm.org/D84378
---
 .../Instrumentation/PGOInstrumentation.cpp    | 13 ++++++-
 .../Inputs/fix_entry_count.proftext           |  8 +++++
 .../Transforms/PGOProfile/fix_entry_count.ll  | 34 +++++++++++++++++++
 3 files changed, 54 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/Transforms/PGOProfile/Inputs/fix_entry_count.proftext
 create mode 100644 llvm/test/Transforms/PGOProfile/fix_entry_count.ll

diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
index 67bf264e8eeaf..c4a43abaa53cc 100644
--- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
+++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
@@ -1131,11 +1131,18 @@ bool PGOUseFunc::setInstrumentedCounts(
   if (NumCounters != CountFromProfile.size()) {
     return false;
   }
+  auto *FuncEntry = &*F.begin();
+
   // Set the profile count to the Instrumented BBs.
   uint32_t I = 0;
   for (BasicBlock *InstrBB : InstrumentBBs) {
     uint64_t CountValue = CountFromProfile[I++];
     UseBBInfo &Info = getBBInfo(InstrBB);
+    // If we reach here, we know that we have some nonzero count
+    // values in this function. The entry count should not be 0.
+    // Fix it if necessary.
+    if (InstrBB == FuncEntry && CountValue == 0)
+      CountValue = 1;
     Info.setBBInfoCount(CountValue);
   }
   ProfileCountSize = CountFromProfile.size();
@@ -1326,7 +1333,6 @@ void PGOUseFunc::populateCounters() {
   }
 #endif
   uint64_t FuncEntryCount = getBBInfo(&*F.begin()).CountValue;
-  F.setEntryCount(ProfileCount(FuncEntryCount, Function::PCT_Real));
   uint64_t FuncMaxCount = FuncEntryCount;
   for (auto &BB : F) {
     auto BI = findBBInfo(&BB);
@@ -1334,6 +1340,11 @@ void PGOUseFunc::populateCounters() {
       continue;
     FuncMaxCount = std::max(FuncMaxCount, BI->CountValue);
   }
+
+  // Fix the obviously inconsistent entry count.
+  if (FuncMaxCount > 0 && FuncEntryCount == 0)
+    FuncEntryCount = 1;
+  F.setEntryCount(ProfileCount(FuncEntryCount, Function::PCT_Real));
   markFunctionAttributes(FuncEntryCount, FuncMaxCount);
 
   // Now annotate select instructions
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/fix_entry_count.proftext b/llvm/test/Transforms/PGOProfile/Inputs/fix_entry_count.proftext
new file mode 100644
index 0000000000000..de4771beaacc2
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/Inputs/fix_entry_count.proftext
@@ -0,0 +1,8 @@
+:ir
+:entry_first
+test_simple_for
+34137660316
+2
+0
+96
+
diff --git a/llvm/test/Transforms/PGOProfile/fix_entry_count.ll b/llvm/test/Transforms/PGOProfile/fix_entry_count.ll
new file mode 100644
index 0000000000000..5923bc923a6a8
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/fix_entry_count.ll
@@ -0,0 +1,34 @@
+; RUN: llvm-profdata merge %S/Inputs/fix_entry_count.proftext -o %t.profdata
+; RUN: opt < %s -pgo-instr-use -pgo-instrument-entry=true -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
+; RUN: opt < %s -passes=pgo-instr-use -pgo-instrument-entry=true -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i32 @test_simple_for(i32 %n) {
+; USE: define i32 @test_simple_for(i32 %n)
+; USE-SAME: !prof ![[ENTRY_COUNT:[0-9]*]]
+entry:
+  br label %for.cond
+
+for.cond:
+  %i = phi i32 [ 0, %entry ], [ %inc1, %for.inc ]
+  %sum = phi i32 [ 1, %entry ], [ %inc, %for.inc ]
+  %cmp = icmp slt i32 %i, %n
+  br i1 %cmp, label %for.body, label %for.end
+; USE: br i1 %cmp, label %for.body, label %for.end
+; USE-SAME: !prof ![[BW_FOR_COND:[0-9]+]]
+
+for.body:
+  %inc = add nsw i32 %sum, 1
+  br label %for.inc
+
+for.inc:
+  %inc1 = add nsw i32 %i, 1
+  br label %for.cond
+
+for.end:
+  ret i32 %sum
+}
+; USE: ![[ENTRY_COUNT]] = !{!"function_entry_count", i64 1}
+; USE: ![[BW_FOR_COND]] = !{!"branch_weights", i32 96, i32 1}

From e937840dbdce20f2ab7ca4dcd9f04c3fd89e56e3 Mon Sep 17 00:00:00 2001
From: Adrian Prantl <aprantl@apple.com>
Date: Fri, 24 Jul 2020 17:59:28 -0700
Subject: [PATCH 0071/1035] Upstream macCatalyst support in ArchSpec and
 associated unit tests.

---
 lldb/source/Utility/ArchSpec.cpp        | 36 ++++++++++++++++++++++---
 lldb/unittests/Utility/ArchSpecTest.cpp | 34 +++++++++++++++++++++++
 2 files changed, 66 insertions(+), 4 deletions(-)

diff --git a/lldb/source/Utility/ArchSpec.cpp b/lldb/source/Utility/ArchSpec.cpp
index f220f4e30b29e..a77ae8633070e 100644
--- a/lldb/source/Utility/ArchSpec.cpp
+++ b/lldb/source/Utility/ArchSpec.cpp
@@ -846,6 +846,15 @@ bool ArchSpec::ContainsOnlyArch(const llvm::Triple &normalized_triple) {
 }
 
 void ArchSpec::MergeFrom(const ArchSpec &other) {
+  // ios-macabi always wins over macosx.
+  if ((GetTriple().getOS() == llvm::Triple::MacOSX ||
+       GetTriple().getOS() == llvm::Triple::UnknownOS) &&
+      other.GetTriple().getOS() == llvm::Triple::IOS &&
+      other.GetTriple().getEnvironment() == llvm::Triple::MacABI) {
+    (*this) = other;
+    return;
+  }
+
   if (!TripleVendorWasSpecified() && other.TripleVendorWasSpecified())
     GetTriple().setVendor(other.GetTriple().getVendor());
   if (!TripleOSWasSpecified() && other.TripleOSWasSpecified())
@@ -1031,6 +1040,22 @@ bool ArchSpec::IsEqualTo(const ArchSpec &rhs, bool exact_match) const {
 
     const llvm::Triple::OSType lhs_triple_os = lhs_triple.getOS();
     const llvm::Triple::OSType rhs_triple_os = rhs_triple.getOS();
+    const llvm::Triple::EnvironmentType lhs_triple_env =
+      lhs_triple.getEnvironment();
+    const llvm::Triple::EnvironmentType rhs_triple_env =
+      rhs_triple.getEnvironment();
+
+    if (!exact_match) {
+      // x86_64-apple-ios-macabi, x86_64-apple-macosx are compatible, no match.
+      if ((lhs_triple_os == llvm::Triple::IOS &&
+           lhs_triple_env == llvm::Triple::MacABI &&
+           rhs_triple_os == llvm::Triple::MacOSX) ||
+          (lhs_triple_os == llvm::Triple::MacOSX &&
+           rhs_triple_os == llvm::Triple::IOS &&
+           rhs_triple_env == llvm::Triple::MacABI))
+        return true;
+    }
+
     if (lhs_triple_os != rhs_triple_os) {
       const bool rhs_os_specified = rhs.TripleOSWasSpecified();
       const bool lhs_os_specified = TripleOSWasSpecified();
@@ -1045,10 +1070,13 @@ bool ArchSpec::IsEqualTo(const ArchSpec &rhs, bool exact_match) const {
         return false;
     }
 
-    const llvm::Triple::EnvironmentType lhs_triple_env =
-        lhs_triple.getEnvironment();
-    const llvm::Triple::EnvironmentType rhs_triple_env =
-        rhs_triple.getEnvironment();
+    // x86_64-apple-ios-macabi and x86_64-apple-ios are not compatible.
+    if (lhs_triple_os == llvm::Triple::IOS &&
+        rhs_triple_os == llvm::Triple::IOS &&
+        (lhs_triple_env == llvm::Triple::MacABI ||
+         rhs_triple_env == llvm::Triple::MacABI) &&
+        lhs_triple_env != rhs_triple_env)
+      return false;
 
     return IsCompatibleEnvironment(lhs_triple_env, rhs_triple_env);
   }
diff --git a/lldb/unittests/Utility/ArchSpecTest.cpp b/lldb/unittests/Utility/ArchSpecTest.cpp
index 4e8e2f3c34d71..a8f43ed7dc7c9 100644
--- a/lldb/unittests/Utility/ArchSpecTest.cpp
+++ b/lldb/unittests/Utility/ArchSpecTest.cpp
@@ -328,6 +328,40 @@ TEST(ArchSpecTest, Compatibility) {
     ASSERT_TRUE(A.IsExactMatch(B));
     ASSERT_TRUE(A.IsCompatibleMatch(B));
   }
+  {
+    ArchSpec A("x86_64");
+    ArchSpec B("x86_64-apple-ios12.0.0-macabi");
+    // FIXME: The exact match also looks unintuitive.
+    ASSERT_TRUE(A.IsExactMatch(B));
+    ASSERT_TRUE(A.IsCompatibleMatch(B));
+  }
+  {
+    ArchSpec A("x86_64-apple-ios12.0.0");
+    ArchSpec B("x86_64-apple-ios12.0.0-macabi");
+    ASSERT_FALSE(A.IsExactMatch(B));
+    ASSERT_FALSE(A.IsCompatibleMatch(B));
+  }
+  {
+    ArchSpec A("x86_64-apple-macosx10.14.2");
+    ArchSpec B("x86_64-apple-ios12.0.0-macabi");
+    ASSERT_FALSE(A.IsExactMatch(B));
+    ASSERT_TRUE(A.IsCompatibleMatch(B));
+  }
+  {
+    ArchSpec A("x86_64-apple-macosx10.14.2");
+    ArchSpec B("x86_64-apple-ios12.0.0-macabi");
+    // ios-macabi wins.
+    A.MergeFrom(B);
+    ASSERT_TRUE(A.IsExactMatch(B));
+  }
+  {
+    ArchSpec A("x86_64-apple-macosx10.14.2");
+    ArchSpec B("x86_64-apple-ios12.0.0-macabi");
+    ArchSpec C(B);
+    // ios-macabi wins.
+    B.MergeFrom(A);
+    ASSERT_TRUE(B.IsExactMatch(C));
+  }
 }
 
 TEST(ArchSpecTest, OperatorBool) {

From 739cd2638b12c37d6bf867d68cd86d18bc5e2e42 Mon Sep 17 00:00:00 2001
From: Amy Kwan <amy.kwan1@ibm.com>
Date: Fri, 24 Jul 2020 20:57:57 -0500
Subject: [PATCH 0072/1035] [PowerPC] Exploit the High Order Vector Multiply
 Instructions on Power10

This patch aims to exploit the following vector multiply high instructions on Power10.
vmulhsw VRT, VRA, VRB
vmulhsd VRT, VRA, VRB
vmulhuw VRT, VRA, VRB
vmulhud VRT, VRA, VRB

Differential Revision: https://reviews.llvm.org/D82584
---
 llvm/lib/Target/PowerPC/PPCISelLowering.cpp   |  4 ++
 llvm/lib/Target/PowerPC/PPCInstrPrefix.td     | 12 ++--
 .../CodeGen/PowerPC/p10-vector-multiply.ll    | 59 +++++++++++++++++++
 3 files changed, 71 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index c2ba7195509a1..8c28ead9f6041 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -811,6 +811,10 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM,
 
     if (Subtarget.isISA3_1()) {
       setOperationAction(ISD::MUL, MVT::v2i64, Legal);
+      setOperationAction(ISD::MULHS, MVT::v2i64, Legal);
+      setOperationAction(ISD::MULHU, MVT::v2i64, Legal);
+      setOperationAction(ISD::MULHS, MVT::v4i32, Legal);
+      setOperationAction(ISD::MULHU, MVT::v4i32, Legal);
       setOperationAction(ISD::UDIV, MVT::v2i64, Legal);
       setOperationAction(ISD::SDIV, MVT::v2i64, Legal);
       setOperationAction(ISD::UDIV, MVT::v4i32, Legal);
diff --git a/llvm/lib/Target/PowerPC/PPCInstrPrefix.td b/llvm/lib/Target/PowerPC/PPCInstrPrefix.td
index 4e048ee9930e9..22839e697381b 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrPrefix.td
+++ b/llvm/lib/Target/PowerPC/PPCInstrPrefix.td
@@ -976,13 +976,17 @@ let Predicates = [IsISA3_1] in {
                         "vmulld $vD, $vA, $vB", IIC_VecGeneral,
                         [(set v2i64:$vD, (mul v2i64:$vA, v2i64:$vB))]>;
   def VMULHSW : VXForm_1<905, (outs vrrc:$vD), (ins vrrc:$vA, vrrc:$vB),
-                         "vmulhsw $vD, $vA, $vB", IIC_VecGeneral, []>;
+                         "vmulhsw $vD, $vA, $vB", IIC_VecGeneral,
+                         [(set v4i32:$vD, (mulhs v4i32:$vA, v4i32:$vB))]>;
   def VMULHUW : VXForm_1<649, (outs vrrc:$vD), (ins vrrc:$vA, vrrc:$vB),
-                         "vmulhuw $vD, $vA, $vB", IIC_VecGeneral, []>;
+                         "vmulhuw $vD, $vA, $vB", IIC_VecGeneral,
+                         [(set v4i32:$vD, (mulhu v4i32:$vA, v4i32:$vB))]>;
   def VMULHSD : VXForm_1<969, (outs vrrc:$vD), (ins vrrc:$vA, vrrc:$vB),
-                         "vmulhsd $vD, $vA, $vB", IIC_VecGeneral, []>;
+                         "vmulhsd $vD, $vA, $vB", IIC_VecGeneral,
+                         [(set v2i64:$vD, (mulhs v2i64:$vA, v2i64:$vB))]>;
   def VMULHUD : VXForm_1<713, (outs vrrc:$vD), (ins vrrc:$vA, vrrc:$vB),
-                         "vmulhud $vD, $vA, $vB", IIC_VecGeneral, []>;
+                         "vmulhud $vD, $vA, $vB", IIC_VecGeneral,
+                         [(set v2i64:$vD, (mulhu v2i64:$vA, v2i64:$vB))]>;
   def VMODSW : VXForm_1<1931, (outs vrrc:$vD), (ins vrrc:$vA, vrrc:$vB),
                         "vmodsw $vD, $vA, $vB", IIC_VecGeneral,
                         [(set v4i32:$vD, (srem v4i32:$vA, v4i32:$vB))]>;
diff --git a/llvm/test/CodeGen/PowerPC/p10-vector-multiply.ll b/llvm/test/CodeGen/PowerPC/p10-vector-multiply.ll
index e8f77574f66cc..75c6d8c24038e 100644
--- a/llvm/test/CodeGen/PowerPC/p10-vector-multiply.ll
+++ b/llvm/test/CodeGen/PowerPC/p10-vector-multiply.ll
@@ -7,6 +7,9 @@
 ; RUN:   FileCheck %s
 
 ; This test case aims to test the vector multiply instructions on Power10.
+; This includes the low order and high order versions of vector multiply.
+; The low order version operates on doublewords, whereas the high order version
+; operates on signed and unsigned words and doublewords.
 
 define <2 x i64> @test_vmulld(<2 x i64> %a, <2 x i64> %b) {
 ; CHECK-LABEL: test_vmulld:
@@ -17,3 +20,59 @@ entry:
   %mul = mul <2 x i64> %b, %a
   ret <2 x i64> %mul
 }
+
+define <2 x i64> @test_vmulhsd(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: test_vmulhsd:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmulhsd v2, v3, v2
+; CHECK-NEXT:    blr
+entry:
+  %0 = sext <2 x i64> %a to <2 x i128>
+  %1 = sext <2 x i64> %b to <2 x i128>
+  %mul = mul <2 x i128> %1, %0
+  %shr = lshr <2 x i128> %mul, <i128 64, i128 64>
+  %tr = trunc <2 x i128> %shr to <2 x i64>
+  ret <2 x i64> %tr
+}
+
+define <2 x i64> @test_vmulhud(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: test_vmulhud:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmulhud v2, v3, v2
+; CHECK-NEXT:    blr
+entry:
+  %0 = zext <2 x i64> %a to <2 x i128>
+  %1 = zext <2 x i64> %b to <2 x i128>
+  %mul = mul <2 x i128> %1, %0
+  %shr = lshr <2 x i128> %mul, <i128 64, i128 64>
+  %tr = trunc <2 x i128> %shr to <2 x i64>
+  ret <2 x i64> %tr
+}
+
+define <4 x i32> @test_vmulhsw(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vmulhsw:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmulhsw v2, v3, v2
+; CHECK-NEXT:    blr
+entry:
+  %0 = sext <4 x i32> %a to <4 x i64>
+  %1 = sext <4 x i32> %b to <4 x i64>
+  %mul = mul <4 x i64> %1, %0
+  %shr = lshr <4 x i64> %mul, <i64 32, i64 32, i64 32, i64 32>
+  %tr = trunc <4 x i64> %shr to <4 x i32>
+  ret <4 x i32> %tr
+}
+
+define <4 x i32> @test_vmulhuw(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vmulhuw:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmulhuw v2, v3, v2
+; CHECK-NEXT:    blr
+entry:
+  %0 = zext <4 x i32> %a to <4 x i64>
+  %1 = zext <4 x i32> %b to <4 x i64>
+  %mul = mul <4 x i64> %1, %0
+  %shr = lshr <4 x i64> %mul, <i64 32, i64 32, i64 32, i64 32>
+  %tr = trunc <4 x i64> %shr to <4 x i32>
+  ret <4 x i32> %tr
+}

From 33d9c4109ac234bcf17501ba16880ce80622cc9c Mon Sep 17 00:00:00 2001
From: Kuba Mracek <mracek@apple.com>
Date: Fri, 24 Jul 2020 20:14:00 -0700
Subject: [PATCH 0073/1035] [tsan] Allow TSan in the Clang driver for Apple
 Silicon Macs

Differential Revision: https://reviews.llvm.org/D84082
---
 clang/lib/Driver/ToolChains/Darwin.cpp | 6 +++---
 clang/test/Driver/fsanitize.c          | 4 ++++
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/clang/lib/Driver/ToolChains/Darwin.cpp b/clang/lib/Driver/ToolChains/Darwin.cpp
index f910c88fa9674..325dcb7df545c 100644
--- a/clang/lib/Driver/ToolChains/Darwin.cpp
+++ b/clang/lib/Driver/ToolChains/Darwin.cpp
@@ -2714,6 +2714,7 @@ void Darwin::CheckObjCARC() const {
 
 SanitizerMask Darwin::getSupportedSanitizers() const {
   const bool IsX86_64 = getTriple().getArch() == llvm::Triple::x86_64;
+  const bool IsAArch64 = getTriple().getArch() == llvm::Triple::aarch64;
   SanitizerMask Res = ToolChain::getSupportedSanitizers();
   Res |= SanitizerKind::Address;
   Res |= SanitizerKind::PointerCompare;
@@ -2731,9 +2732,8 @@ SanitizerMask Darwin::getSupportedSanitizers() const {
       && !(isTargetIPhoneOS() && isIPhoneOSVersionLT(5, 0)))
     Res |= SanitizerKind::Vptr;
 
-  if (isTargetMacOS()) {
-    if (IsX86_64)
-      Res |= SanitizerKind::Thread;
+  if ((IsX86_64 || IsAArch64) && isTargetMacOS()) {
+    Res |= SanitizerKind::Thread;
   } else if (isTargetIOSSimulator() || isTargetTvOSSimulator()) {
     if (IsX86_64)
       Res |= SanitizerKind::Thread;
diff --git a/clang/test/Driver/fsanitize.c b/clang/test/Driver/fsanitize.c
index 7340bfb35e40c..cfefd3fb632cb 100644
--- a/clang/test/Driver/fsanitize.c
+++ b/clang/test/Driver/fsanitize.c
@@ -458,6 +458,10 @@
 
 // RUN: %clang -target x86_64-apple-darwin -fsanitize=thread %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-TSAN-X86-64-DARWIN
 // CHECK-TSAN-X86-64-DARWIN-NOT: unsupported option
+// RUN: %clang -target x86_64-apple-macos -fsanitize=thread %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-TSAN-X86-64-MACOS
+// CHECK-TSAN-X86-64-MACOS-NOT: unsupported option
+// RUN: %clang -target arm64-apple-macos -fsanitize=thread %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-TSAN-ARM64-MACOS
+// CHECK-TSAN-ARM64-MACOS-NOT: unsupported option
 
 // RUN: %clang -target x86_64-apple-iossimulator -fsanitize=thread %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-TSAN-X86-64-IOSSIMULATOR
 // CHECK-TSAN-X86-64-IOSSIMULATOR-NOT: unsupported option

From 604e33e83a55c854db5fb9594e667f114423bb58 Mon Sep 17 00:00:00 2001
From: Jessica Paquette <jpaquette@apple.com>
Date: Fri, 24 Jul 2020 18:14:41 -0700
Subject: [PATCH 0074/1035] [AArch64][GlobalISel] Look through constants when
 selection stores of 0

Very minor code size improvements (hits 8 times in Bullet at -O3), but still
something.

Also very minor NFC change to make sure we only search for a 0 constant when
selecting a store. Before, we'd do this for loads as well.

Differential Revision: https://reviews.llvm.org/D84573
---
 .../GISel/AArch64InstructionSelector.cpp      | 25 +++++++++++--------
 .../AArch64/GlobalISel/select-store.mir       | 17 +++++++++++++
 2 files changed, 31 insertions(+), 11 deletions(-)

diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
index 0b0de09fff29e..6de5e5ea5b955 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
@@ -2304,18 +2304,21 @@ bool AArch64InstructionSelector::select(MachineInstr &I) {
     I.addOperand(MachineOperand::CreateImm(Offset));
 
     // If we're storing a 0, use WZR/XZR.
-    if (auto CVal = getConstantVRegVal(ValReg, MRI)) {
-      if (*CVal == 0 && Opcode == TargetOpcode::G_STORE) {
+    if (Opcode == TargetOpcode::G_STORE) {
+      auto CVal = getConstantVRegValWithLookThrough(
+          ValReg, MRI, /*LookThroughInstrs = */ true,
+          /*HandleFConstants = */ false);
+      if (CVal && CVal->Value == 0) {
         unsigned Opc = I.getOpcode();
-        switch(Opc) {
-          case AArch64::STRWui:
-          case AArch64::STRHHui:
-          case AArch64::STRBBui:
-            I.getOperand(0).setReg(AArch64::WZR);
-            break;
-          case AArch64::STRXui:
-            I.getOperand(0).setReg(AArch64::XZR);
-            break;
+        switch (Opc) {
+        case AArch64::STRWui:
+        case AArch64::STRHHui:
+        case AArch64::STRBBui:
+          I.getOperand(0).setReg(AArch64::WZR);
+          break;
+        case AArch64::STRXui:
+          I.getOperand(0).setReg(AArch64::XZR);
+          break;
         }
       }
     }
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/select-store.mir b/llvm/test/CodeGen/AArch64/GlobalISel/select-store.mir
index d60a34ef6631a..db355dfc151f5 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/select-store.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/select-store.mir
@@ -13,6 +13,7 @@
   define void @store_zero_s32_gpr(i32* %addr) { ret void }
   define void @store_zero_s16(i32* %addr) { ret void }
   define void @store_zero_s8(i32* %addr) { ret void }
+  define void @store_zero_look_through_cst(i32* %addr) { ret void }
 
   define void @store_fi_s64_gpr() {
     %ptr0 = alloca i64
@@ -209,6 +210,22 @@ body:             |
     G_STORE %1(s8), %0(p0) :: (store 1)
 ...
 
+---
+name:            store_zero_look_through_cst
+legalized:       true
+regBankSelected: true
+body:             |
+  bb.0:
+    liveins: $x0
+    ; CHECK-LABEL: name: store_zero_look_through_cst
+    ; CHECK: [[COPY:%[0-9]+]]:gpr64sp = COPY $x0
+    ; CHECK: STRXui $xzr, [[COPY]], 0 :: (store 8 into %ir.addr)
+    %0:gpr(p0) = COPY $x0
+    %1:gpr(s32) = G_CONSTANT i32 0
+    %2:gpr(s64) = G_ZEXT %1
+    G_STORE  %2, %0 :: (store 8 into %ir.addr)
+...
+
 ---
 name:            store_fi_s64_gpr
 legalized:       true

From 6c25fc35e096790a216fa5ddeda8c514c7536818 Mon Sep 17 00:00:00 2001
From: Nathan James <n.james93@hotmail.co.uk>
Date: Sat, 25 Jul 2020 10:37:33 +0100
Subject: [PATCH 0075/1035] [ADT] Add a range-based version of std::move

Adds a range-based version of `std::move`, the version that moves a range, not the one that creates r-value references.

Reviewed By: dblaikie, gamesh411

Differential Revision: https://reviews.llvm.org/D83902
---
 llvm/include/llvm/ADT/STLExtras.h    |  7 ++++
 llvm/unittests/ADT/STLExtrasTest.cpp | 52 ++++++++++++++++++++++++++++
 2 files changed, 59 insertions(+)

diff --git a/llvm/include/llvm/ADT/STLExtras.h b/llvm/include/llvm/ADT/STLExtras.h
index 2e7a097174763..92eea4e83f693 100644
--- a/llvm/include/llvm/ADT/STLExtras.h
+++ b/llvm/include/llvm/ADT/STLExtras.h
@@ -1535,6 +1535,13 @@ OutputIt copy(R &&Range, OutputIt Out) {
   return std::copy(adl_begin(Range), adl_end(Range), Out);
 }
 
+/// Provide wrappers to std::move which take ranges instead of having to
+/// pass begin/end explicitly.
+template <typename R, typename OutputIt>
+OutputIt move(R &&Range, OutputIt Out) {
+  return std::move(adl_begin(Range), adl_end(Range), Out);
+}
+
 /// Wrapper function around std::find to detect if an element exists
 /// in a container.
 template <typename R, typename E>
diff --git a/llvm/unittests/ADT/STLExtrasTest.cpp b/llvm/unittests/ADT/STLExtrasTest.cpp
index 1d275d1db9094..f09d5986c0090 100644
--- a/llvm/unittests/ADT/STLExtrasTest.cpp
+++ b/llvm/unittests/ADT/STLExtrasTest.cpp
@@ -568,4 +568,56 @@ TEST(STLExtras, hasNItemsOrLess) {
   EXPECT_FALSE(
       hasNItemsOrLess(V3.begin(), V3.end(), 2, [](int x) { return x < 10; }));
 }
+
+TEST(STLExtras, MoveRange) {
+  class Foo {
+    bool A;
+
+  public:
+    Foo() : A(true) {}
+    Foo(const Foo &) = delete;
+    Foo(Foo &&Other) : A(Other.A) { Other.A = false; }
+    Foo &operator=(const Foo &) = delete;
+    Foo &operator=(Foo &&Other) {
+      if (this != &Other) {
+        A = Other.A;
+        Other.A = false;
+      }
+      return *this;
+    }
+    operator bool() const { return A; }
+  };
+
+  constexpr size_t ItemCount = 4;
+  SmallVector<Foo, ItemCount> V1, V2, V3, V4;
+  auto HasVal = [](const Foo &Item) { return static_cast<bool>(Item); };
+  auto Build = [&] {
+    SmallVector<Foo, ItemCount> Foos;
+    Foos.resize(ItemCount);
+    return Foos;
+  };
+
+  V1.resize(ItemCount);
+  EXPECT_TRUE(llvm::all_of(V1, HasVal));
+
+  llvm::move(V1, std::back_inserter(V2));
+
+  // Ensure input container is same size, but its contents were moved out.
+  EXPECT_EQ(V1.size(), ItemCount);
+  EXPECT_TRUE(llvm::none_of(V1, HasVal));
+
+  // Ensure output container has the contents of the input container.
+  EXPECT_EQ(V2.size(), ItemCount);
+  EXPECT_TRUE(llvm::all_of(V2, HasVal));
+
+  llvm::move(std::move(V2), std::back_inserter(V3));
+
+  EXPECT_TRUE(llvm::none_of(V2, HasVal));
+  EXPECT_EQ(V3.size(), ItemCount);
+  EXPECT_TRUE(llvm::all_of(V3, HasVal));
+
+  llvm::move(Build(), std::back_inserter(V4));
+  EXPECT_EQ(V4.size(), ItemCount);
+  EXPECT_TRUE(llvm::all_of(V4, HasVal));
+}
 } // namespace

From 66998ae59f4e334b285f735bb2d927976cdd323c Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Sat, 25 Jul 2020 10:50:56 +0100
Subject: [PATCH 0076/1035] [X86][SSE] getFauxShuffle - ignore undemanded
 sources for PACKSS/PACKUS faux shuffles

If we don't care about an entire LHS/RHS of the PACK op, then can just treat it the same as undef (we don't care if it saturates) and is safe to treat as a shuffle.

This can happen if we attempt to decode as a faux shuffle before SimplifyDemandedVectorElts has been called on the PACK which should replace the source with UNDEF entirely.
---
 llvm/lib/Target/X86/X86ISelLowering.cpp    |  12 +--
 llvm/test/CodeGen/X86/vector-reduce-mul.ll | 106 +++++++++------------
 2 files changed, 51 insertions(+), 67 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 2e97bef6c106a..4dc68d469bbae 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -7595,19 +7595,19 @@ static bool getFauxShuffleMask(SDValue N, const APInt &DemandedElts,
     APInt EltsLHS, EltsRHS;
     getPackDemandedElts(VT, DemandedElts, EltsLHS, EltsRHS);
 
-    // If we know input saturation won't happen we can treat this
-    // as a truncation shuffle.
+    // If we know input saturation won't happen (or we don't care for particular
+    // lanes), we can treat this as a truncation shuffle.
     if (Opcode == X86ISD::PACKSS) {
-      if ((!N0.isUndef() &&
+      if ((!(N0.isUndef() || EltsLHS.isNullValue()) &&
            DAG.ComputeNumSignBits(N0, EltsLHS, Depth + 1) <= NumBitsPerElt) ||
-          (!N1.isUndef() &&
+          (!(N1.isUndef() || EltsRHS.isNullValue()) &&
            DAG.ComputeNumSignBits(N1, EltsRHS, Depth + 1) <= NumBitsPerElt))
         return false;
     } else {
       APInt ZeroMask = APInt::getHighBitsSet(2 * NumBitsPerElt, NumBitsPerElt);
-      if ((!N0.isUndef() &&
+      if ((!(N0.isUndef() || EltsLHS.isNullValue()) &&
            !DAG.MaskedValueIsZero(N0, ZeroMask, EltsLHS, Depth + 1)) ||
-          (!N1.isUndef() &&
+          (!(N1.isUndef() || EltsRHS.isNullValue()) &&
            !DAG.MaskedValueIsZero(N1, ZeroMask, EltsRHS, Depth + 1)))
         return false;
     }
diff --git a/llvm/test/CodeGen/X86/vector-reduce-mul.ll b/llvm/test/CodeGen/X86/vector-reduce-mul.ll
index 08abd66e8eb77..bafe112c5dfc8 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-mul.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-mul.ll
@@ -1697,9 +1697,8 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ; SSE2-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    pmullw %xmm2, %xmm0
-; SSE2-NEXT:    pand %xmm1, %xmm0
-; SSE2-NEXT:    packuswb %xmm3, %xmm0
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    pand %xmm0, %xmm1
+; SSE2-NEXT:    packuswb %xmm1, %xmm1
 ; SSE2-NEXT:    psrlw $8, %xmm1
 ; SSE2-NEXT:    pmullw %xmm0, %xmm1
 ; SSE2-NEXT:    movd %xmm1, %eax
@@ -1718,17 +1717,16 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ; SSE41-NEXT:    packuswb %xmm3, %xmm0
 ; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; SSE41-NEXT:    pmullw %xmm2, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
-; SSE41-NEXT:    pand %xmm1, %xmm0
-; SSE41-NEXT:    packuswb %xmm3, %xmm0
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; SSE41-NEXT:    pmullw %xmm2, %xmm0
-; SSE41-NEXT:    pand %xmm1, %xmm0
-; SSE41-NEXT:    packuswb %xmm3, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psrlw $8, %xmm1
+; SSE41-NEXT:    pand %xmm0, %xmm1
+; SSE41-NEXT:    packuswb %xmm3, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; SSE41-NEXT:    pmullw %xmm0, %xmm1
-; SSE41-NEXT:    movd %xmm1, %eax
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    psrld $8, %xmm0
+; SSE41-NEXT:    psrlw $8, %xmm0
+; SSE41-NEXT:    pmullw %xmm1, %xmm0
+; SSE41-NEXT:    movd %xmm0, %eax
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    retq
 ;
@@ -1741,10 +1739,8 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX1-NEXT:    vpsrld $8, %xmm0, %xmm1
+; AVX1-NEXT:    vpsrlw $8, %xmm1, %xmm1
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
 ; AVX1-NEXT:    # kill: def $al killed $al killed $eax
@@ -1889,9 +1885,8 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; SSE2-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    pmullw %xmm2, %xmm0
-; SSE2-NEXT:    pand %xmm1, %xmm0
-; SSE2-NEXT:    packuswb %xmm3, %xmm0
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    pand %xmm0, %xmm1
+; SSE2-NEXT:    packuswb %xmm1, %xmm1
 ; SSE2-NEXT:    psrlw $8, %xmm1
 ; SSE2-NEXT:    pmullw %xmm0, %xmm1
 ; SSE2-NEXT:    movd %xmm1, %eax
@@ -1919,16 +1914,15 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; SSE41-NEXT:    packuswb %xmm3, %xmm2
 ; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
 ; SSE41-NEXT:    pmullw %xmm0, %xmm2
+; SSE41-NEXT:    pand %xmm2, %xmm1
+; SSE41-NEXT:    packuswb %xmm3, %xmm1
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    packuswb %xmm3, %xmm2
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
-; SSE41-NEXT:    pmullw %xmm0, %xmm2
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    packuswb %xmm3, %xmm2
-; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; SSE41-NEXT:    pmullw %xmm0, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    psrld $8, %xmm0
 ; SSE41-NEXT:    psrlw $8, %xmm0
-; SSE41-NEXT:    pmullw %xmm2, %xmm0
+; SSE41-NEXT:    pmullw %xmm1, %xmm0
 ; SSE41-NEXT:    movd %xmm0, %eax
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    retq
@@ -1947,10 +1941,8 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX1-NEXT:    vpsrld $8, %xmm0, %xmm1
+; AVX1-NEXT:    vpsrlw $8, %xmm1, %xmm1
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
 ; AVX1-NEXT:    # kill: def $al killed $al killed $eax
@@ -2130,9 +2122,8 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; SSE2-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    pmullw %xmm2, %xmm0
-; SSE2-NEXT:    pand %xmm1, %xmm0
-; SSE2-NEXT:    packuswb %xmm3, %xmm0
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    pand %xmm0, %xmm1
+; SSE2-NEXT:    packuswb %xmm1, %xmm1
 ; SSE2-NEXT:    psrlw $8, %xmm1
 ; SSE2-NEXT:    pmullw %xmm0, %xmm1
 ; SSE2-NEXT:    movd %xmm1, %eax
@@ -2173,14 +2164,13 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; SSE41-NEXT:    packuswb %xmm3, %xmm1
 ; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; SSE41-NEXT:    pmullw %xmm0, %xmm1
+; SSE41-NEXT:    pand %xmm1, %xmm2
+; SSE41-NEXT:    packuswb %xmm3, %xmm2
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
-; SSE41-NEXT:    pand %xmm2, %xmm1
-; SSE41-NEXT:    packuswb %xmm3, %xmm1
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
 ; SSE41-NEXT:    pmullw %xmm0, %xmm1
-; SSE41-NEXT:    pand %xmm2, %xmm1
-; SSE41-NEXT:    packuswb %xmm3, %xmm1
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    psrld $8, %xmm0
 ; SSE41-NEXT:    psrlw $8, %xmm0
 ; SSE41-NEXT:    pmullw %xmm1, %xmm0
 ; SSE41-NEXT:    movd %xmm0, %eax
@@ -2210,10 +2200,8 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX1-NEXT:    vpsrld $8, %xmm0, %xmm1
+; AVX1-NEXT:    vpsrlw $8, %xmm1, %xmm1
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
 ; AVX1-NEXT:    # kill: def $al killed $al killed $eax
@@ -2447,9 +2435,8 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; SSE2-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    pmullw %xmm2, %xmm1
-; SSE2-NEXT:    pand %xmm0, %xmm1
-; SSE2-NEXT:    packuswb %xmm3, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    pand %xmm1, %xmm0
+; SSE2-NEXT:    packuswb %xmm0, %xmm0
 ; SSE2-NEXT:    psrlw $8, %xmm0
 ; SSE2-NEXT:    pmullw %xmm1, %xmm0
 ; SSE2-NEXT:    movd %xmm0, %eax
@@ -2514,17 +2501,16 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; SSE41-NEXT:    packuswb %xmm2, %xmm0
 ; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; SSE41-NEXT:    pmullw %xmm1, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; SSE41-NEXT:    pand %xmm4, %xmm0
-; SSE41-NEXT:    packuswb %xmm2, %xmm0
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; SSE41-NEXT:    pmullw %xmm1, %xmm0
-; SSE41-NEXT:    pand %xmm4, %xmm0
-; SSE41-NEXT:    packuswb %xmm2, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psrlw $8, %xmm1
+; SSE41-NEXT:    pand %xmm0, %xmm4
+; SSE41-NEXT:    packuswb %xmm2, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
 ; SSE41-NEXT:    pmullw %xmm0, %xmm1
-; SSE41-NEXT:    movd %xmm1, %eax
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    psrld $8, %xmm0
+; SSE41-NEXT:    psrlw $8, %xmm0
+; SSE41-NEXT:    pmullw %xmm1, %xmm0
+; SSE41-NEXT:    movd %xmm0, %eax
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    retq
 ;
@@ -2569,10 +2555,8 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX1-NEXT:    vpsrld $8, %xmm0, %xmm1
+; AVX1-NEXT:    vpsrlw $8, %xmm1, %xmm1
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
 ; AVX1-NEXT:    # kill: def $al killed $al killed $eax

From 4363ea61058b67fd6a86e3798ff7c21631270c6d Mon Sep 17 00:00:00 2001
From: Nathan James <n.james93@hotmail.co.uk>
Date: Sat, 25 Jul 2020 11:03:59 +0100
Subject: [PATCH 0077/1035] Fix C2975 error under MSVC

Apparantly a constexpr value isn't a compile time constant under certain versions of MSVC.
---
 llvm/unittests/ADT/STLExtrasTest.cpp | 18 ++++++++----------
 1 file changed, 8 insertions(+), 10 deletions(-)

diff --git a/llvm/unittests/ADT/STLExtrasTest.cpp b/llvm/unittests/ADT/STLExtrasTest.cpp
index f09d5986c0090..aaef46a2cf05e 100644
--- a/llvm/unittests/ADT/STLExtrasTest.cpp
+++ b/llvm/unittests/ADT/STLExtrasTest.cpp
@@ -587,37 +587,35 @@ TEST(STLExtras, MoveRange) {
     }
     operator bool() const { return A; }
   };
-
-  constexpr size_t ItemCount = 4;
-  SmallVector<Foo, ItemCount> V1, V2, V3, V4;
+  SmallVector<Foo, 4U> V1, V2, V3, V4;
   auto HasVal = [](const Foo &Item) { return static_cast<bool>(Item); };
   auto Build = [&] {
-    SmallVector<Foo, ItemCount> Foos;
-    Foos.resize(ItemCount);
+    SmallVector<Foo, 4U> Foos;
+    Foos.resize(4U);
     return Foos;
   };
 
-  V1.resize(ItemCount);
+  V1.resize(4U);
   EXPECT_TRUE(llvm::all_of(V1, HasVal));
 
   llvm::move(V1, std::back_inserter(V2));
 
   // Ensure input container is same size, but its contents were moved out.
-  EXPECT_EQ(V1.size(), ItemCount);
+  EXPECT_EQ(V1.size(), 4U);
   EXPECT_TRUE(llvm::none_of(V1, HasVal));
 
   // Ensure output container has the contents of the input container.
-  EXPECT_EQ(V2.size(), ItemCount);
+  EXPECT_EQ(V2.size(), 4U);
   EXPECT_TRUE(llvm::all_of(V2, HasVal));
 
   llvm::move(std::move(V2), std::back_inserter(V3));
 
   EXPECT_TRUE(llvm::none_of(V2, HasVal));
-  EXPECT_EQ(V3.size(), ItemCount);
+  EXPECT_EQ(V3.size(), 4U);
   EXPECT_TRUE(llvm::all_of(V3, HasVal));
 
   llvm::move(Build(), std::back_inserter(V4));
-  EXPECT_EQ(V4.size(), ItemCount);
+  EXPECT_EQ(V4.size(), 4U);
   EXPECT_TRUE(llvm::all_of(V4, HasVal));
 }
 } // namespace

From 3c1476d26c769cd97a631a129b30c62232ac96b6 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Sat, 25 Jul 2020 11:52:14 +0100
Subject: [PATCH 0078/1035] [IPSCCP] Drop argmemonly after replacing pointer
 argument.

This patch updates IPSCCP to drop argmemonly and
inaccessiblemem_or_argmemonly if it replaces a pointer argument.

Fixes PR46717.

Reviewers: efriedma, davide, nikic, jdoerfert

Reviewed By: efriedma, jdoerfert

Differential Revision: https://reviews.llvm.org/D84432
---
 llvm/lib/Transforms/Scalar/SCCP.cpp           | 30 +++++++++++++++----
 .../Transforms/SCCP/ipscp-drop-argmemonly.ll  | 17 +++++------
 2 files changed, 32 insertions(+), 15 deletions(-)

diff --git a/llvm/lib/Transforms/Scalar/SCCP.cpp b/llvm/lib/Transforms/Scalar/SCCP.cpp
index 32dc14e5ec195..428f9675e088c 100644
--- a/llvm/lib/Transforms/Scalar/SCCP.cpp
+++ b/llvm/lib/Transforms/Scalar/SCCP.cpp
@@ -1911,15 +1911,35 @@ bool llvm::runIPSCCP(
 
     SmallVector<BasicBlock *, 512> BlocksToErase;
 
-    if (Solver.isBlockExecutable(&F.front()))
-      for (Function::arg_iterator AI = F.arg_begin(), E = F.arg_end(); AI != E;
-           ++AI) {
-        if (!AI->use_empty() && tryToReplaceWithConstant(Solver, &*AI)) {
+    if (Solver.isBlockExecutable(&F.front())) {
+      bool ReplacedPointerArg = false;
+      for (Argument &Arg : F.args()) {
+        if (!Arg.use_empty() && tryToReplaceWithConstant(Solver, &Arg)) {
+          ReplacedPointerArg |= Arg.getType()->isPointerTy();
           ++IPNumArgsElimed;
-          continue;
         }
       }
 
+      // If we replaced an argument, the argmemonly and
+      // inaccessiblemem_or_argmemonly attributes do not hold any longer. Remove
+      // them from both the function and callsites.
+      if (ReplacedPointerArg) {
+        SmallVector<Attribute::AttrKind, 2> AttributesToRemove = {
+            Attribute::ArgMemOnly, Attribute::InaccessibleMemOrArgMemOnly};
+        for (auto Attr : AttributesToRemove)
+          F.removeFnAttr(Attr);
+
+        for (User *U : F.users()) {
+          auto *CB = dyn_cast<CallBase>(U);
+          if (!CB || CB->getCalledFunction() != &F)
+            continue;
+
+          for (auto Attr : AttributesToRemove)
+            CB->removeAttribute(AttributeList::FunctionIndex, Attr);
+        }
+      }
+    }
+
     SmallPtrSet<Value *, 32> InsertedValues;
     for (BasicBlock &BB : F) {
       if (!Solver.isBlockExecutable(&BB)) {
diff --git a/llvm/test/Transforms/SCCP/ipscp-drop-argmemonly.ll b/llvm/test/Transforms/SCCP/ipscp-drop-argmemonly.ll
index a110476eb4302..2e3a35779a15c 100644
--- a/llvm/test/Transforms/SCCP/ipscp-drop-argmemonly.ll
+++ b/llvm/test/Transforms/SCCP/ipscp-drop-argmemonly.ll
@@ -11,7 +11,7 @@
 ; Here the pointer argument %arg will be replaced by a constant. We need to
 ; drop argmemonly.
 define internal void @ptrarg.1(i32* %arg, i32 %val) argmemonly nounwind {
-; CHECK: Function Attrs: argmemonly nounwind
+; CHECK: Function Attrs: nounwind
 ; CHECK-LABEL: @ptrarg.1(
 ; CHECK-NEXT:    store i32 10, i32* @g, align 4
 ; CHECK-NEXT:    ret void
@@ -59,7 +59,7 @@ define void @caller.2(i32* %ptr) {
 ; Here the pointer argument %arg will be replaced by a constant. We need to
 ; drop inaccessiblemem_or_argmemonly.
 define internal void @ptrarg.3(i32* %arg, i32 %val) inaccessiblemem_or_argmemonly nounwind {
-; CHECK: Function Attrs: inaccessiblemem_or_argmemonly nounwind
+; CHECK: Function Attrs: nounwind
 ; CHECK-LABEL: @ptrarg.3(
 ; CHECK-NEXT:    store i32 10, i32* @g, align 4
 ; CHECK-NEXT:    ret void
@@ -107,7 +107,7 @@ define void @caller.4(i32* %ptr) {
 ; Here the pointer argument %arg will be replaced by a constant. We need to
 ; drop inaccessiblemem_or_argmemonly.
 define internal void @ptrarg.5(i32* %arg, i32 %val) argmemonly inaccessiblemem_or_argmemonly nounwind {
-; CHECK: Function Attrs: argmemonly inaccessiblemem_or_argmemonly nounwind
+; CHECK: Function Attrs: nounwind
 ; CHECK-LABEL: @ptrarg.5(
 ; CHECK-NEXT:    store i32 10, i32* @g, align 4
 ; CHECK-NEXT:    ret void
@@ -143,9 +143,9 @@ define internal void @ptrarg.6.cs.attributes(i32* %arg, i32 %val) {
 define i32 @caller.6.cs.attributes(i32 %n) {
 ; CHECK-LABEL: @caller.6.cs.attributes(
 ; CHECK-NEXT:    store i32 1, i32* @g, align 4
-; CHECK-NEXT:    tail call void @ptrarg.5(i32* @g, i32 10) [[ARGMEMONLY_INACCESSIBLEMEM_OR_ARGMEMONLY_NOUNWIND:#[0-9]+]]
-; CHECK-NEXT:    tail call void @ptrarg.5(i32* @g, i32 10) [[INACCESSIBLEMEM_OR_ARGMEMONLY_NOUNWIND:#[0-9]+]]
-; CHECK-NEXT:    tail call void @ptrarg.5(i32* @g, i32 10) [[ARGMEMONLY_NOUNWIND:#[0-9]+]]
+; CHECK-NEXT:    tail call void @ptrarg.5(i32* @g, i32 10) [[NOUNWIND:#[0-9]+]]
+; CHECK-NEXT:    tail call void @ptrarg.5(i32* @g, i32 10) [[NOUNWIND:#[0-9]+]]
+; CHECK-NEXT:    tail call void @ptrarg.5(i32* @g, i32 10) [[NOUNWIND:#[0-9]+]]
 ; CHECK-NEXT:    tail call void @ptrarg.5(i32* @g, i32 10) [[NOUNWIND:#[0-9]+]]
 ; CHECK-NEXT:    [[G_VAL:%.*]] = load i32, i32* @g, align 4
 ; CHECK-NEXT:    ret i32 [[G_VAL]]
@@ -159,7 +159,4 @@ define i32 @caller.6.cs.attributes(i32 %n) {
   ret i32 %g.val
 }
 
-; CHECK-DAG: [[ARGMEMONLY_INACCESSIBLEMEM_OR_ARGMEMONLY_NOUNWIND]] = { argmemonly inaccessiblemem_or_argmemonly nounwind }
-; CHECK-DAG: [[INACCESSIBLEMEM_OR_ARGMEMONLY_NOUNWIND]] = { inaccessiblemem_or_argmemonly nounwind }
-; CHECK-DAG: [[ARGMEMONLY_NOUNWIND]] = { argmemonly nounwind }
-; CHECK-DAG: [[NOUNWIND]] = { nounwind }
+; CHECK: [[NOUNWIND]] = { nounwind }

From 18d481cdf9d0e4a8ac5ad05007f34c944327f3d8 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Sat, 25 Jul 2020 11:35:47 +0100
Subject: [PATCH 0079/1035] SymbolRemappingReader.h - pass Twine by reference
 not value. NFCI.

---
 llvm/include/llvm/Support/SymbolRemappingReader.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/include/llvm/Support/SymbolRemappingReader.h b/llvm/include/llvm/Support/SymbolRemappingReader.h
index 2b9ab570eb8b3..820cf9e021920 100644
--- a/llvm/include/llvm/Support/SymbolRemappingReader.h
+++ b/llvm/include/llvm/Support/SymbolRemappingReader.h
@@ -68,7 +68,7 @@ namespace llvm {
 
 class SymbolRemappingParseError : public ErrorInfo<SymbolRemappingParseError> {
 public:
-  SymbolRemappingParseError(StringRef File, int64_t Line, Twine Message)
+  SymbolRemappingParseError(StringRef File, int64_t Line, const Twine &Message)
       : File(File), Line(Line), Message(Message.str()) {}
 
   void log(llvm::raw_ostream &OS) const override {

From 3b21823e4abb501ca0b0de7777493be56a11cb67 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Sat, 25 Jul 2020 12:08:06 +0100
Subject: [PATCH 0080/1035] [X86][SSE] combineX86ShufflesRecursively - move all
 Root node asserts to the same location. NFCI.

Minor tidyup for some upcoming shuffle combine improvements.
---
 llvm/lib/Target/X86/X86ISelLowering.cpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 4dc68d469bbae..24bc264df129f 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -35346,6 +35346,9 @@ static SDValue combineX86ShufflesRecursively(
   assert(RootMask.size() > 0 &&
          (RootMask.size() > 1 || (RootMask[0] == 0 && SrcOpIndex == 0)) &&
          "Illegal shuffle root mask");
+  assert(Root.getSimpleValueType().isVector() &&
+         "Shuffles operate on vector types!");
+  unsigned RootSizeInBits = Root.getSimpleValueType().getSizeInBits();
 
   // Bound the depth of our recursive combine because this is ultimately
   // quadratic in nature.
@@ -35361,9 +35364,6 @@ static SDValue combineX86ShufflesRecursively(
   if (!VT.isVector())
     return SDValue(); // Bail if we hit a non-vector.
 
-  assert(Root.getSimpleValueType().isVector() &&
-         "Shuffles operate on vector types!");
-  unsigned RootSizeInBits = Root.getSimpleValueType().getSizeInBits();
   assert(VT.getSizeInBits() == RootSizeInBits &&
          "Can only combine shuffles of the same vector register size.");
 

From b5e14d78f16960ebb94f0a22e2a8c10f421f8757 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Sat, 25 Jul 2020 12:58:39 +0100
Subject: [PATCH 0081/1035] SimplifyLibCalls - remove unnecessary header and
 forward declaration. NFC.

We include TargetLibraryInfo.h so don't need to forward declare it, and we don't need to include TargetLibraryInfo.h in SimplifyLibCalls.cpp as well.
---
 llvm/include/llvm/Transforms/Utils/SimplifyLibCalls.h | 1 -
 llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp        | 1 -
 2 files changed, 2 deletions(-)

diff --git a/llvm/include/llvm/Transforms/Utils/SimplifyLibCalls.h b/llvm/include/llvm/Transforms/Utils/SimplifyLibCalls.h
index d6ee19365c729..2819a3468766d 100644
--- a/llvm/include/llvm/Transforms/Utils/SimplifyLibCalls.h
+++ b/llvm/include/llvm/Transforms/Utils/SimplifyLibCalls.h
@@ -24,7 +24,6 @@ class CallInst;
 class DataLayout;
 class Instruction;
 class IRBuilderBase;
-class TargetLibraryInfo;
 class Function;
 class OptimizationRemarkEmitter;
 class BlockFrequencyInfo;
diff --git a/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp b/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
index cfcc3454a2102..b03389aec7bf4 100644
--- a/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
@@ -20,7 +20,6 @@
 #include "llvm/Analysis/ConstantFolding.h"
 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
 #include "llvm/Analysis/ProfileSummaryInfo.h"
-#include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/Transforms/Utils/Local.h"
 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/Analysis/CaptureTracking.h"

From ad16e71c953c92dbb4ffe9581c4e31a4cde70ccf Mon Sep 17 00:00:00 2001
From: Nikita Popov <nikita.ppv@gmail.com>
Date: Tue, 21 Jul 2020 21:26:30 +0200
Subject: [PATCH 0082/1035] Reapply [SCCP] Directly remove non-feasible edges

Reapply with DTU update moved after CFG update, which is a
requirement of the API.

-----

Non-feasible control-flow edges are currently removed by replacing
the branch condition with a constant and then calling
ConstantFoldTerminator. This happens in a rather roundabout manner,
by inspecting the users (effectively: predecessors) of unreachable
blocks, and further complicated by the need to explicitly materialize
the condition for "forced" edges. I would like to extend SCCP to
discard switch conditions that are non-feasible based on range
information, but this is incompatible with the current approach
(as there is no single constant we could use.)

Instead, this patch explicitly removes non-feasible edges. It
currently only needs to handle the case where there is a single
feasible edge. The llvm_unreachable() branch will need to be
implemented for the aforementioned switch improvement.

Differential Revision: https://reviews.llvm.org/D84264
---
 llvm/lib/Transforms/Scalar/SCCP.cpp           | 118 +++++++-----------
 .../test/Transforms/SCCP/conditions-ranges.ll |   4 +-
 llvm/test/Transforms/SCCP/domtree-update.ll   |  41 ++++++
 .../Transforms/SCCP/predicateinfo-cond.ll     |   2 +-
 .../SCCP/resolvedundefsin-tracked-fn.ll       |   8 +-
 .../SCCP/switch-constantfold-crash.ll         |  12 +-
 llvm/test/Transforms/SCCP/switch.ll           |   8 +-
 llvm/test/Transforms/SCCP/widening.ll         |  14 +--
 8 files changed, 110 insertions(+), 97 deletions(-)
 create mode 100644 llvm/test/Transforms/SCCP/domtree-update.ll

diff --git a/llvm/lib/Transforms/Scalar/SCCP.cpp b/llvm/lib/Transforms/Scalar/SCCP.cpp
index 428f9675e088c..270524a009596 100644
--- a/llvm/lib/Transforms/Scalar/SCCP.cpp
+++ b/llvm/lib/Transforms/Scalar/SCCP.cpp
@@ -276,7 +276,7 @@ class SCCPSolver : public InstVisitor<SCCPSolver> {
 
   // isEdgeFeasible - Return true if the control flow edge from the 'From' basic
   // block to the 'To' basic block is currently feasible.
-  bool isEdgeFeasible(BasicBlock *From, BasicBlock *To);
+  bool isEdgeFeasible(BasicBlock *From, BasicBlock *To) const;
 
   std::vector<ValueLatticeElement> getStructLatticeValueFor(Value *V) const {
     std::vector<ValueLatticeElement> StructValues;
@@ -705,7 +705,7 @@ void SCCPSolver::getFeasibleSuccessors(Instruction &TI,
 
 // isEdgeFeasible - Return true if the control flow edge from the 'From' basic
 // block to the 'To' basic block is currently feasible.
-bool SCCPSolver::isEdgeFeasible(BasicBlock *From, BasicBlock *To) {
+bool SCCPSolver::isEdgeFeasible(BasicBlock *From, BasicBlock *To) const {
   // Check if we've called markEdgeExecutable on the edge yet. (We could
   // be more aggressive and try to consider edges which haven't been marked
   // yet, but there isn't any need.)
@@ -1807,39 +1807,51 @@ static void findReturnsToZap(Function &F,
   }
 }
 
-// Update the condition for terminators that are branching on indeterminate
-// values, forcing them to use a specific edge.
-static void forceIndeterminateEdge(Instruction* I, SCCPSolver &Solver) {
-  BasicBlock *Dest = nullptr;
-  Constant *C = nullptr;
-  if (SwitchInst *SI = dyn_cast<SwitchInst>(I)) {
-    if (!isa<ConstantInt>(SI->getCondition())) {
-      // Indeterminate switch; use first case value.
-      Dest = SI->case_begin()->getCaseSuccessor();
-      C = SI->case_begin()->getCaseValue();
-    }
-  } else if (BranchInst *BI = dyn_cast<BranchInst>(I)) {
-    if (!isa<ConstantInt>(BI->getCondition())) {
-      // Indeterminate branch; use false.
-      Dest = BI->getSuccessor(1);
-      C = ConstantInt::getFalse(BI->getContext());
-    }
-  } else if (IndirectBrInst *IBR = dyn_cast<IndirectBrInst>(I)) {
-    if (!isa<BlockAddress>(IBR->getAddress()->stripPointerCasts())) {
-      // Indeterminate indirectbr; use successor 0.
-      Dest = IBR->getSuccessor(0);
-      C = BlockAddress::get(IBR->getSuccessor(0));
-    }
-  } else {
-    llvm_unreachable("Unexpected terminator instruction");
+static bool removeNonFeasibleEdges(const SCCPSolver &Solver, BasicBlock *BB,
+                                   DomTreeUpdater &DTU) {
+  SmallPtrSet<BasicBlock *, 8> FeasibleSuccessors;
+  bool HasNonFeasibleEdges = false;
+  for (BasicBlock *Succ : successors(BB)) {
+    if (Solver.isEdgeFeasible(BB, Succ))
+      FeasibleSuccessors.insert(Succ);
+    else
+      HasNonFeasibleEdges = true;
   }
-  if (C) {
-    assert(Solver.isEdgeFeasible(I->getParent(), Dest) &&
-           "Didn't find feasible edge?");
-    (void)Dest;
 
-    I->setOperand(0, C);
+  // All edges feasible, nothing to do.
+  if (!HasNonFeasibleEdges)
+    return false;
+
+  // SCCP can only determine non-feasible edges for br, switch and indirectbr.
+  Instruction *TI = BB->getTerminator();
+  assert((isa<BranchInst>(TI) || isa<SwitchInst>(TI) ||
+          isa<IndirectBrInst>(TI)) &&
+         "Terminator must be a br, switch or indirectbr");
+
+  if (FeasibleSuccessors.size() == 1) {
+    // Replace with an unconditional branch to the only feasible successor.
+    BasicBlock *OnlyFeasibleSuccessor = *FeasibleSuccessors.begin();
+    SmallVector<DominatorTree::UpdateType, 8> Updates;
+    bool HaveSeenOnlyFeasibleSuccessor = false;
+    for (BasicBlock *Succ : successors(BB)) {
+      if (Succ == OnlyFeasibleSuccessor && !HaveSeenOnlyFeasibleSuccessor) {
+        // Don't remove the edge to the only feasible successor the first time
+        // we see it. We still do need to remove any multi-edges to it though.
+        HaveSeenOnlyFeasibleSuccessor = true;
+        continue;
+      }
+
+      Succ->removePredecessor(BB);
+      Updates.push_back({DominatorTree::Delete, BB, Succ});
+    }
+
+    BranchInst::Create(OnlyFeasibleSuccessor, BB);
+    TI->eraseFromParent();
+    DTU.applyUpdatesPermissive(Updates);
+  } else {
+    llvm_unreachable("Either all successors are feasible, or exactly one is");
   }
+  return true;
 }
 
 bool llvm::runIPSCCP(
@@ -1972,45 +1984,11 @@ bool llvm::runIPSCCP(
                                             /*UseLLVMTrap=*/false,
                                             /*PreserveLCSSA=*/false, &DTU);
 
-    // Now that all instructions in the function are constant folded,
-    // use ConstantFoldTerminator to get rid of in-edges, record DT updates and
-    // delete dead BBs.
-    for (BasicBlock *DeadBB : BlocksToErase) {
-      // If there are any PHI nodes in this successor, drop entries for BB now.
-      for (Value::user_iterator UI = DeadBB->user_begin(),
-                                UE = DeadBB->user_end();
-           UI != UE;) {
-        // Grab the user and then increment the iterator early, as the user
-        // will be deleted. Step past all adjacent uses from the same user.
-        auto *I = dyn_cast<Instruction>(*UI);
-        do { ++UI; } while (UI != UE && *UI == I);
-
-        // Ignore blockaddress users; BasicBlock's dtor will handle them.
-        if (!I) continue;
-
-        // If we have forced an edge for an indeterminate value, then force the
-        // terminator to fold to that edge.
-        forceIndeterminateEdge(I, Solver);
-        BasicBlock *InstBB = I->getParent();
-        bool Folded = ConstantFoldTerminator(InstBB,
-                                             /*DeleteDeadConditions=*/false,
-                                             /*TLI=*/nullptr, &DTU);
-        assert(Folded &&
-              "Expect TermInst on constantint or blockaddress to be folded");
-        (void) Folded;
-        // If we folded the terminator to an unconditional branch to another
-        // dead block, replace it with Unreachable, to avoid trying to fold that
-        // branch again.
-        BranchInst *BI = cast<BranchInst>(InstBB->getTerminator());
-        if (BI && BI->isUnconditional() &&
-            !Solver.isBlockExecutable(BI->getSuccessor(0))) {
-          InstBB->getTerminator()->eraseFromParent();
-          new UnreachableInst(InstBB->getContext(), InstBB);
-        }
-      }
-      // Mark dead BB for deletion.
+    for (BasicBlock &BB : F)
+      removeNonFeasibleEdges(Solver, &BB, DTU);
+
+    for (BasicBlock *DeadBB : BlocksToErase)
       DTU.deleteBB(DeadBB);
-    }
 
     for (BasicBlock &BB : F) {
       for (BasicBlock::iterator BI = BB.begin(), E = BB.end(); BI != E;) {
diff --git a/llvm/test/Transforms/SCCP/conditions-ranges.ll b/llvm/test/Transforms/SCCP/conditions-ranges.ll
index 612a38f008fc5..dada59099d81b 100644
--- a/llvm/test/Transforms/SCCP/conditions-ranges.ll
+++ b/llvm/test/Transforms/SCCP/conditions-ranges.ll
@@ -231,12 +231,12 @@ define void @f7_nested_conds(i32* %a, i32 %b) {
 ; CHECK-NEXT:    [[C_1:%.*]] = icmp ne i32 [[A_V]], 0
 ; CHECK-NEXT:    br i1 [[C_1]], label [[TRUE:%.*]], label [[FALSE:%.*]]
 ; CHECK:       false:
-; CHECK-NEXT:    br i1 true, label [[TRUE_2:%.*]], label [[TRUE]]
+; CHECK-NEXT:    br label [[TRUE_2:%.*]]
 ; CHECK:       true.2:
 ; CHECK-NEXT:    call void @use(i1 true)
 ; CHECK-NEXT:    ret void
 ; CHECK:       true:
-; CHECK-NEXT:    store i32 [[B:%.*]], i32* [[A]]
+; CHECK-NEXT:    store i32 [[B:%.*]], i32* [[A]], align 4
 ; CHECK-NEXT:    ret void
 ;
 entry:
diff --git a/llvm/test/Transforms/SCCP/domtree-update.ll b/llvm/test/Transforms/SCCP/domtree-update.ll
new file mode 100644
index 0000000000000..32adbde300e88
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/domtree-update.ll
@@ -0,0 +1,41 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -ipsccp < %s | FileCheck %s
+; RUN: opt -S -passes='ipsccp,function(verify<domtree>)' < %s | FileCheck %s
+
+; DTU should not crash.
+
+define i32 @test() {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    br label [[IF_THEN2:%.*]]
+; CHECK:       if.then2:
+; CHECK-NEXT:    br label [[FOR_INC:%.*]]
+; CHECK:       for.inc:
+; CHECK-NEXT:    unreachable
+;
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry
+  br i1 true, label %if.then2, label %if.else
+
+if.then2:                                         ; preds = %for.body
+  br label %for.inc
+
+if.else:                                          ; preds = %for.body
+  br i1 undef, label %lor.rhs, label %if.then19.critedge
+
+lor.rhs:                                          ; preds = %if.else
+  br i1 undef, label %if.then19, label %for.inc
+
+if.then19.critedge:                               ; preds = %if.else
+  br label %if.then19
+
+if.then19:                                        ; preds = %if.then19.critedge, %lor.rhs
+  unreachable
+
+for.inc:                                          ; preds = %lor.rhs, %if.then2
+  unreachable
+}
diff --git a/llvm/test/Transforms/SCCP/predicateinfo-cond.ll b/llvm/test/Transforms/SCCP/predicateinfo-cond.ll
index 8ed96ec9301f5..1443cc72c2ef8 100644
--- a/llvm/test/Transforms/SCCP/predicateinfo-cond.ll
+++ b/llvm/test/Transforms/SCCP/predicateinfo-cond.ll
@@ -105,7 +105,7 @@ define void @pr46814(i32 %a) {
 ; CHECK-NEXT:    [[C3:%.*]] = and i1 [[C1]], [[C2]]
 ; CHECK-NEXT:    br i1 [[C3]], label [[IF_1:%.*]], label [[EXIT:%.*]]
 ; CHECK:       if.1:
-; CHECK-NEXT:    br i1 true, label [[IF_2:%.*]], label [[EXIT]]
+; CHECK-NEXT:    br label [[IF_2:%.*]]
 ; CHECK:       if.2:
 ; CHECK-NEXT:    br i1 true, label [[EXIT]], label [[EXIT]]
 ; CHECK:       exit:
diff --git a/llvm/test/Transforms/SCCP/resolvedundefsin-tracked-fn.ll b/llvm/test/Transforms/SCCP/resolvedundefsin-tracked-fn.ll
index e1c7b3d5662d0..9e9d1256c4cc0 100644
--- a/llvm/test/Transforms/SCCP/resolvedundefsin-tracked-fn.ll
+++ b/llvm/test/Transforms/SCCP/resolvedundefsin-tracked-fn.ll
@@ -136,13 +136,12 @@ define internal i1 @test2_g(%t1* %h, i32 %i) {
 ; CHECK-LABEL: define {{[^@]+}}@test2_g
 ; CHECK-SAME: (%t1* [[H:%.*]], i32 [[I:%.*]])
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 true, label [[LAND_RHS:%.*]], label [[LAND_END:%.*]]
+; CHECK-NEXT:    br label [[LAND_RHS:%.*]]
 ; CHECK:       land.rhs:
 ; CHECK-NEXT:    [[CALL:%.*]] = call i32 (...) @test2_j()
 ; CHECK-NEXT:    [[TOBOOL1:%.*]] = icmp ne i32 [[CALL]], 0
-; CHECK-NEXT:    br label [[LAND_END]]
+; CHECK-NEXT:    br label [[LAND_END:%.*]]
 ; CHECK:       land.end:
-; CHECK-NEXT:    [[TMP0:%.*]] = phi i1 [ false, [[ENTRY:%.*]] ], [ [[TOBOOL1]], [[LAND_RHS]] ]
 ; CHECK-NEXT:    ret i1 undef
 ;
 entry:
@@ -196,10 +195,9 @@ define internal i32 @test3_k(i8 %h, i32 %i) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = inttoptr i64 [[CONV]] to %t1*
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[PHI:%.*]] = phi i1 [ undef, [[ENTRY:%.*]] ], [ false, [[LOOP]] ]
 ; CHECK-NEXT:    [[CALL:%.*]] = call i1 @test3_g(%t1* [[TMP1]], i32 0)
 ; CHECK-NEXT:    call void @use.1(i1 false)
-; CHECK-NEXT:    br i1 false, label [[LOOP]], label [[EXIT:%.*]]
+; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret i32 undef
 ;
diff --git a/llvm/test/Transforms/SCCP/switch-constantfold-crash.ll b/llvm/test/Transforms/SCCP/switch-constantfold-crash.ll
index 7596a56b81229..17b37f000407c 100644
--- a/llvm/test/Transforms/SCCP/switch-constantfold-crash.ll
+++ b/llvm/test/Transforms/SCCP/switch-constantfold-crash.ll
@@ -5,11 +5,11 @@
 define void @barney() {
 ; CHECK-LABEL: @barney(
 ; CHECK-NEXT:  bb:
-; CHECK-NEXT:    br label %bb9
+; CHECK-NEXT:    br label [[BB9:%.*]]
 ; CHECK:       bb6:
 ; CHECK-NEXT:    unreachable
 ; CHECK:       bb9:
-; CHECK-NEXT:    unreachable
+; CHECK-NEXT:    br label [[BB6:%.*]]
 ;
 bb:
   br label %bb9
@@ -29,9 +29,9 @@ bb9:                                              ; preds = %bb
 define void @blam() {
 ; CHECK-LABEL: @blam(
 ; CHECK-NEXT:  bb:
-; CHECK-NEXT:    br label %bb16
+; CHECK-NEXT:    br label [[BB16:%.*]]
 ; CHECK:       bb16:
-; CHECK-NEXT:    br label %bb38
+; CHECK-NEXT:    br label [[BB38:%.*]]
 ; CHECK:       bb38:
 ; CHECK-NEXT:    unreachable
 ;
@@ -62,9 +62,9 @@ bb38:                                             ; preds = %bb16
 define void @hoge() {
 ; CHECK-LABEL: @hoge(
 ; CHECK-NEXT:  bb:
-; CHECK-NEXT:    br label %bb2
+; CHECK-NEXT:    br label [[BB2:%.*]]
 ; CHECK:       bb2:
-; CHECK-NEXT:    unreachable
+; CHECK-NEXT:    br label [[BB3:%.*]]
 ; CHECK:       bb3:
 ; CHECK-NEXT:    unreachable
 ;
diff --git a/llvm/test/Transforms/SCCP/switch.ll b/llvm/test/Transforms/SCCP/switch.ll
index 5f607a3afd89d..3587587bcb91d 100644
--- a/llvm/test/Transforms/SCCP/switch.ll
+++ b/llvm/test/Transforms/SCCP/switch.ll
@@ -23,15 +23,11 @@ define i32 @test_duplicate_successors_phi(i1 %c, i32 %x) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br i1 [[C:%.*]], label [[SWITCH:%.*]], label [[END:%.*]]
 ; CHECK:       switch:
-; CHECK-NEXT:    switch i32 -1, label [[SWITCH_DEFAULT:%.*]] [
-; CHECK-NEXT:    i32 0, label [[END]]
-; CHECK-NEXT:    i32 1, label [[END]]
-; CHECK-NEXT:    ]
+; CHECK-NEXT:    br label [[SWITCH_DEFAULT:%.*]]
 ; CHECK:       switch.default:
 ; CHECK-NEXT:    ret i32 -1
 ; CHECK:       end:
-; CHECK-NEXT:    [[PHI:%.*]] = phi i32 [ [[X:%.*]], [[ENTRY:%.*]] ], [ 1, [[SWITCH]] ], [ 1, [[SWITCH]] ]
-; CHECK-NEXT:    ret i32 [[PHI]]
+; CHECK-NEXT:    ret i32 [[X:%.*]]
 ;
 entry:
   br i1 %c, label %switch, label %end
diff --git a/llvm/test/Transforms/SCCP/widening.ll b/llvm/test/Transforms/SCCP/widening.ll
index 2703bdb27dff4..23a88c35a93ea 100644
--- a/llvm/test/Transforms/SCCP/widening.ll
+++ b/llvm/test/Transforms/SCCP/widening.ll
@@ -216,11 +216,11 @@ define void @rotated_loop_2(i32 %x) {
 ; IPSCCP:       bb3:
 ; IPSCCP-NEXT:    br label [[EXIT]]
 ; IPSCCP:       exit:
-; IPSCCP-NEXT:    [[P:%.*]] = phi i32 [ 1, [[ENTRY:%.*]] ], [ 3, [[BB1]] ], [ 2, [[BB2]] ], [ 5, [[BB3]] ], [ [[A:%.*]], [[EXIT]] ]
-; IPSCCP-NEXT:    [[A]] = add i32 [[P]], 1
+; IPSCCP-NEXT:    [[P:%.*]] = phi i32 [ 1, [[ENTRY:%.*]] ], [ 3, [[BB1]] ], [ 2, [[BB2]] ], [ 5, [[BB3]] ]
+; IPSCCP-NEXT:    [[A:%.*]] = add i32 [[P]], 1
 ; IPSCCP-NEXT:    call void @use(i1 true)
 ; IPSCCP-NEXT:    call void @use(i1 false)
-; IPSCCP-NEXT:    br i1 false, label [[EXIT]], label [[EXIT_1:%.*]]
+; IPSCCP-NEXT:    br label [[EXIT_1:%.*]]
 ; IPSCCP:       exit.1:
 ; IPSCCP-NEXT:    ret void
 ;
@@ -451,10 +451,10 @@ define void @foo(i64* %arg) {
 ; SCCP-NEXT:    [[TMP7:%.*]] = sub i64 3, [[TMP6]]
 ; SCCP-NEXT:    [[TMP8:%.*]] = shl i64 [[TMP7]], 1
 ; SCCP-NEXT:    [[TMP9:%.*]] = trunc i64 [[TMP8]] to i32
-; SCCP-NEXT:    [[TMP10:%.*]] = zext i32 [[TMP9]] to i64
+; SCCP-NEXT:    [[TMP0:%.*]] = zext i32 [[TMP9]] to i64
 ; SCCP-NEXT:    br label [[BB11:%.*]]
 ; SCCP:       bb11:
-; SCCP-NEXT:    [[TMP12:%.*]] = phi i64 [ [[TMP10]], [[BB4]] ], [ [[TMP17:%.*]], [[BB18:%.*]] ]
+; SCCP-NEXT:    [[TMP12:%.*]] = phi i64 [ [[TMP0]], [[BB4]] ], [ [[TMP17:%.*]], [[BB18:%.*]] ]
 ; SCCP-NEXT:    br label [[BB13:%.*]]
 ; SCCP:       bb13:
 ; SCCP-NEXT:    [[C_1:%.*]] = icmp eq i64 [[TMP12]], 6
@@ -489,10 +489,10 @@ define void @foo(i64* %arg) {
 ; IPSCCP-NEXT:    [[TMP7:%.*]] = sub i64 3, [[TMP6]]
 ; IPSCCP-NEXT:    [[TMP8:%.*]] = shl i64 [[TMP7]], 1
 ; IPSCCP-NEXT:    [[TMP9:%.*]] = trunc i64 [[TMP8]] to i32
-; IPSCCP-NEXT:    [[TMP10:%.*]] = zext i32 [[TMP9]] to i64
+; IPSCCP-NEXT:    [[TMP0:%.*]] = zext i32 [[TMP9]] to i64
 ; IPSCCP-NEXT:    br label [[BB11:%.*]]
 ; IPSCCP:       bb11:
-; IPSCCP-NEXT:    [[TMP12:%.*]] = phi i64 [ [[TMP10]], [[BB4]] ], [ [[TMP17:%.*]], [[BB18:%.*]] ]
+; IPSCCP-NEXT:    [[TMP12:%.*]] = phi i64 [ [[TMP0]], [[BB4]] ], [ [[TMP17:%.*]], [[BB18:%.*]] ]
 ; IPSCCP-NEXT:    br label [[BB13:%.*]]
 ; IPSCCP:       bb13:
 ; IPSCCP-NEXT:    [[C_1:%.*]] = icmp eq i64 [[TMP12]], 6

From 632a89e866f38e8fdfae7785ffa94fb2f302eefc Mon Sep 17 00:00:00 2001
From: Nikita Popov <nikita.ppv@gmail.com>
Date: Sat, 25 Jul 2020 15:10:48 +0200
Subject: [PATCH 0083/1035] [SCCP] Restore the change reporting as well

Reapply 5db5b4bc4394ca247c9eb665e03b851848aa2fbf.
---
 llvm/lib/Transforms/Scalar/SCCP.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Transforms/Scalar/SCCP.cpp b/llvm/lib/Transforms/Scalar/SCCP.cpp
index 270524a009596..c4f5c522e4509 100644
--- a/llvm/lib/Transforms/Scalar/SCCP.cpp
+++ b/llvm/lib/Transforms/Scalar/SCCP.cpp
@@ -1985,7 +1985,7 @@ bool llvm::runIPSCCP(
                                             /*PreserveLCSSA=*/false, &DTU);
 
     for (BasicBlock &BB : F)
-      removeNonFeasibleEdges(Solver, &BB, DTU);
+      MadeChanges |= removeNonFeasibleEdges(Solver, &BB, DTU);
 
     for (BasicBlock *DeadBB : BlocksToErase)
       DTU.deleteBB(DeadBB);

From f4199b8f0bdbea12ff69dbb78336a05894f81a8c Mon Sep 17 00:00:00 2001
From: Nikita Popov <nikita.ppv@gmail.com>
Date: Sat, 25 Jul 2020 16:02:15 +0200
Subject: [PATCH 0084/1035] [SCCP] Add assume non null test (NFC)

---
 llvm/test/Transforms/SCCP/assume.ll | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/llvm/test/Transforms/SCCP/assume.ll b/llvm/test/Transforms/SCCP/assume.ll
index 764c1737c2874..dc827f03c0abe 100644
--- a/llvm/test/Transforms/SCCP/assume.ll
+++ b/llvm/test/Transforms/SCCP/assume.ll
@@ -46,3 +46,30 @@ define void @basic(i32 %v) {
   call void @use(i1 %c8)
   ret void
 }
+
+define void @nonnull(i32* %v) {
+; CHECK-LABEL: @nonnull(
+; CHECK-NEXT:    [[A:%.*]] = icmp ne i32* [[V:%.*]], null
+; CHECK-NEXT:    call void @llvm.assume(i1 [[A]])
+; CHECK-NEXT:    [[C1:%.*]] = icmp eq i32* [[V]], null
+; CHECK-NEXT:    call void @use(i1 [[C1]])
+; CHECK-NEXT:    [[C2:%.*]] = icmp ne i32* [[V]], null
+; CHECK-NEXT:    call void @use(i1 [[C2]])
+; CHECK-NEXT:    [[C3:%.*]] = icmp eq i32* null, [[V]]
+; CHECK-NEXT:    call void @use(i1 [[C3]])
+; CHECK-NEXT:    [[C4:%.*]] = icmp ne i32* null, [[V]]
+; CHECK-NEXT:    call void @use(i1 [[C4]])
+; CHECK-NEXT:    ret void
+;
+  %a = icmp ne i32* %v, null
+  call void @llvm.assume(i1 %a)
+  %c1 = icmp eq i32* %v, null
+  call void @use(i1 %c1)
+  %c2 = icmp ne i32* %v, null
+  call void @use(i1 %c2)
+  %c3 = icmp eq i32* null, %v
+  call void @use(i1 %c3)
+  %c4 = icmp ne i32* null, %v
+  call void @use(i1 %c4)
+  ret void
+}

From 392b969c3203001d076eafc080d6d014827d39ca Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Mon, 15 Jun 2020 20:13:24 -0400
Subject: [PATCH 0085/1035] AMDGPU/GlobalISel: Don't assert on G_INSERT >
 128-bits

Just fallback for now. Really tablegen needs to generate all of the
subregister index handling we need.
---
 .../AMDGPU/AMDGPUInstructionSelector.cpp      |  4 +++
 .../GlobalISel/inst-select-insert.xfail.mir   | 26 ++++++++++++++++---
 2 files changed, 27 insertions(+), 3 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
index a637442002932..4740a58519996 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
@@ -736,6 +736,10 @@ bool AMDGPUInstructionSelector::selectG_INSERT(MachineInstr &I) const {
   if (Offset % 32 != 0 || InsSize % 32 != 0)
     return false;
 
+  // Currently not handled by getSubRegFromChannel.
+  if (InsSize > 128)
+    return false;
+
   unsigned SubReg = TRI.getSubRegFromChannel(Offset / 32, InsSize / 32);
   if (SubReg == AMDGPU::NoSubRegister)
     return false;
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-insert.xfail.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-insert.xfail.mir
index 5e58e8b633ec4..150b341561f97 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-insert.xfail.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-insert.xfail.mir
@@ -1,9 +1,13 @@
 # RUN: llc -march=amdgcn -mcpu=gfx900 -run-pass=instruction-select -global-isel-abort=2 -pass-remarks-missed='gisel*' -verify-machineinstrs -o /dev/null %s 2>&1 | FileCheck -check-prefix=ERR %s
 
-# FIXME: This should not be legal and this test should be deleted
-# ERR: remark: <unknown>:0:0: cannot select: %3:sgpr(<4 x s16>) = G_INSERT %0:sgpr, %2:sgpr(s16), 0 (in function: insert_sgpr_2s16_to_v4s16_offset0)
+# ERR: remark: <unknown>:0:0: cannot select: %3:sgpr(<4 x s16>) = G_INSERT %0:sgpr, %2:sgpr(s16), 0 (in function: insert_sgpr_s16_to_v4s16_offset0)
+# ERR-NEXT: remark: <unknown>:0:0: cannot select: %2:sgpr(<16 x s32>) = G_INSERT %0:sgpr, %1:sgpr(<8 x s32>), 0 (in function: insert_sgpr_v8s32_to_v16s32_offset0)
+# ERR-NOT: remark
+
+# FIXME: This 16-bit insert source should not be legal and this test
+# should be deleted
 ---
-name:            insert_sgpr_2s16_to_v4s16_offset0
+name:            insert_sgpr_s16_to_v4s16_offset0
 legalized:       true
 regBankSelected: true
 
@@ -17,3 +21,19 @@ body: |
     S_ENDPGM 0, implicit %3
 
 ...
+
+# getSubRegFromChannel current does not handle cases > 128-bits
+---
+name:            insert_sgpr_v8s32_to_v16s32_offset0
+legalized:       true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15, $sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23
+    %0:sgpr(<16 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
+    %1:sgpr(<8 x s32>) = COPY $sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23
+    %2:sgpr(<16 x s32>) = G_INSERT %0, %1, 0
+    S_ENDPGM 0, implicit %2
+
+...

From bc79ed7e16003c8550da8710b321d6d5d4243faf Mon Sep 17 00:00:00 2001
From: Nikita Popov <nikita.ppv@gmail.com>
Date: Sat, 25 Jul 2020 16:32:22 +0200
Subject: [PATCH 0086/1035] [LVI] Don't require operand number for range (NFC)

Pass the Value* instead of the operand number, rename I to CxtI.
This makes the function a bit more generally useful.
---
 llvm/lib/Analysis/LazyValueInfo.cpp | 23 +++++++++++------------
 1 file changed, 11 insertions(+), 12 deletions(-)

diff --git a/llvm/lib/Analysis/LazyValueInfo.cpp b/llvm/lib/Analysis/LazyValueInfo.cpp
index f5ffa7286b3b8..34cc81c4bf2a3 100644
--- a/llvm/lib/Analysis/LazyValueInfo.cpp
+++ b/llvm/lib/Analysis/LazyValueInfo.cpp
@@ -388,8 +388,8 @@ class LazyValueInfoImpl {
                                                        BasicBlock *BB);
   Optional<ValueLatticeElement> solveBlockValueSelect(SelectInst *S,
                                                       BasicBlock *BB);
-  Optional<ConstantRange> getRangeForOperand(unsigned Op, Instruction *I,
-                                             BasicBlock *BB);
+  Optional<ConstantRange> getRangeFor(Value *V, Instruction *CxtI,
+                                      BasicBlock *BB);
   Optional<ValueLatticeElement> solveBlockValueBinaryOpImpl(
       Instruction *I, BasicBlock *BB,
       std::function<ConstantRange(const ConstantRange &,
@@ -919,20 +919,19 @@ Optional<ValueLatticeElement> LazyValueInfoImpl::solveBlockValueSelect(
   return Result;
 }
 
-Optional<ConstantRange> LazyValueInfoImpl::getRangeForOperand(unsigned Op,
-                                                              Instruction *I,
-                                                              BasicBlock *BB) {
-  Optional<ValueLatticeElement> OptVal = getBlockValue(I->getOperand(Op), BB);
+Optional<ConstantRange> LazyValueInfoImpl::getRangeFor(Value *V,
+                                                       Instruction *CxtI,
+                                                       BasicBlock *BB) {
+  Optional<ValueLatticeElement> OptVal = getBlockValue(V, BB);
   if (!OptVal)
     return None;
 
   ValueLatticeElement &Val = *OptVal;
-  intersectAssumeOrGuardBlockValueConstantRange(I->getOperand(Op), Val, I);
+  intersectAssumeOrGuardBlockValueConstantRange(V, Val, CxtI);
   if (Val.isConstantRange())
     return Val.getConstantRange();
 
-  const unsigned OperandBitWidth =
-    DL.getTypeSizeInBits(I->getOperand(Op)->getType());
+  const unsigned OperandBitWidth = DL.getTypeSizeInBits(V->getType());
   return ConstantRange::getFull(OperandBitWidth);
 }
 
@@ -962,7 +961,7 @@ Optional<ValueLatticeElement> LazyValueInfoImpl::solveBlockValueCast(
   // Figure out the range of the LHS.  If that fails, we still apply the
   // transfer rule on the full set since we may be able to locally infer
   // interesting facts.
-  Optional<ConstantRange> LHSRes = getRangeForOperand(0, CI, BB);
+  Optional<ConstantRange> LHSRes = getRangeFor(CI->getOperand(0), CI, BB);
   if (!LHSRes.hasValue())
     // More work to do before applying this transfer rule.
     return None;
@@ -985,8 +984,8 @@ Optional<ValueLatticeElement> LazyValueInfoImpl::solveBlockValueBinaryOpImpl(
   // conservative range, but apply the transfer rule anyways.  This
   // lets us pick up facts from expressions like "and i32 (call i32
   // @foo()), 32"
-  Optional<ConstantRange> LHSRes = getRangeForOperand(0, I, BB);
-  Optional<ConstantRange> RHSRes = getRangeForOperand(1, I, BB);
+  Optional<ConstantRange> LHSRes = getRangeFor(I->getOperand(0), I, BB);
+  Optional<ConstantRange> RHSRes = getRangeFor(I->getOperand(1), I, BB);
   if (!LHSRes.hasValue() || !RHSRes.hasValue())
     // More work to do before applying this transfer rule.
     return None;

From c09a10845b429307a38a93799e7520c0e16850fd Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Sat, 25 Jul 2020 15:45:24 +0100
Subject: [PATCH 0087/1035] [X86] Remove stress-scheduledagrrlist.ll.

This test seems to take quite a long time with EXPENSIVE_CHECKS.

Remove it.
---
 llvm/test/CodeGen/X86/stress-scheduledagrrlist.ll | 12 ------------
 1 file changed, 12 deletions(-)
 delete mode 100644 llvm/test/CodeGen/X86/stress-scheduledagrrlist.ll

diff --git a/llvm/test/CodeGen/X86/stress-scheduledagrrlist.ll b/llvm/test/CodeGen/X86/stress-scheduledagrrlist.ll
deleted file mode 100644
index a699134a8c7c5..0000000000000
--- a/llvm/test/CodeGen/X86/stress-scheduledagrrlist.ll
+++ /dev/null
@@ -1,12 +0,0 @@
-; RUN: llc -O0 -mtriple=x86_64-apple-macosx %s -o %t.s
-
-; Stress test for the list scheduler. The store will be expanded to a very
-; large number of stores during isel, stressing ScheduleDAGRRList. It should
-; compiles in a reasonable amount of time. Run with -O0, to disable most other
-; optimizations.
-
-define void @test(i1000000* %ptr) {
-entry:
-  store i1000000 0, i1000000* %ptr, align 4
-  ret void
-}

From 1d9b860fb6a85df33fd52fcacc6a5efb421621bd Mon Sep 17 00:00:00 2001
From: Adrian Prantl <aprantl@apple.com>
Date: Sat, 25 Jul 2020 08:27:21 -0700
Subject: [PATCH 0088/1035] Unify the return value of GetByteSize to an
 llvm::Optional<uint64_t> (NFC-ish)

This cleanup patch unifies all methods called GetByteSize() in the
ValueObject hierarchy to return an optional, like the methods in
CompilerType do. This means fewer magic 0 values, which could fix bugs
down the road in languages where types can have a size of zero, such
as Swift and C (but not C++).

Differential Revision: https://reviews.llvm.org/D84285
---
 lldb/include/lldb/Core/ValueObject.h          |  2 +-
 lldb/include/lldb/Core/ValueObjectCast.h      |  2 +-
 lldb/include/lldb/Core/ValueObjectChild.h     |  2 +-
 .../lldb/Core/ValueObjectConstResult.h        |  4 ++--
 .../lldb/Core/ValueObjectDynamicValue.h       |  2 +-
 lldb/include/lldb/Core/ValueObjectMemory.h    |  2 +-
 lldb/include/lldb/Core/ValueObjectRegister.h  |  4 ++--
 .../lldb/Core/ValueObjectSyntheticFilter.h    |  2 +-
 lldb/include/lldb/Core/ValueObjectVariable.h  |  2 +-
 .../lldb/Expression/ExpressionVariable.h      |  2 +-
 .../lldb/Target/StackFrameRecognizer.h        |  4 +++-
 lldb/source/API/SBValue.cpp                   |  2 +-
 .../Commands/CommandObjectWatchpoint.cpp      |  2 +-
 lldb/source/Core/ValueObject.cpp              | 12 +++++-----
 lldb/source/Core/ValueObjectCast.cpp          |  2 +-
 lldb/source/Core/ValueObjectConstResult.cpp   | 10 ++++----
 lldb/source/Core/ValueObjectDynamicValue.cpp  |  2 +-
 lldb/source/Core/ValueObjectMemory.cpp        |  8 +++----
 lldb/source/Core/ValueObjectRegister.cpp      |  6 +++--
 .../Core/ValueObjectSyntheticFilter.cpp       |  4 +++-
 lldb/source/Core/ValueObjectVariable.cpp      |  6 ++---
 lldb/source/Expression/ExpressionVariable.cpp |  8 +++----
 lldb/source/Expression/Materializer.cpp       | 23 +++++++++++--------
 lldb/source/Target/StackFrame.cpp             |  6 ++---
 24 files changed, 62 insertions(+), 57 deletions(-)

diff --git a/lldb/include/lldb/Core/ValueObject.h b/lldb/include/lldb/Core/ValueObject.h
index 0080368fd9965..a557d69f3ae30 100644
--- a/lldb/include/lldb/Core/ValueObject.h
+++ b/lldb/include/lldb/Core/ValueObject.h
@@ -358,7 +358,7 @@ class ValueObject : public UserID {
   virtual bool CanProvideValue();
 
   // Subclasses must implement the functions below.
-  virtual uint64_t GetByteSize() = 0;
+  virtual llvm::Optional<uint64_t> GetByteSize() = 0;
 
   virtual lldb::ValueType GetValueType() const = 0;
 
diff --git a/lldb/include/lldb/Core/ValueObjectCast.h b/lldb/include/lldb/Core/ValueObjectCast.h
index d91ca6a92be8d..342803f8ca63a 100644
--- a/lldb/include/lldb/Core/ValueObjectCast.h
+++ b/lldb/include/lldb/Core/ValueObjectCast.h
@@ -30,7 +30,7 @@ class ValueObjectCast : public ValueObject {
                                     ConstString name,
                                     const CompilerType &cast_type);
 
-  uint64_t GetByteSize() override;
+  llvm::Optional<uint64_t> GetByteSize() override;
 
   size_t CalculateNumChildren(uint32_t max) override;
 
diff --git a/lldb/include/lldb/Core/ValueObjectChild.h b/lldb/include/lldb/Core/ValueObjectChild.h
index c6f44a29b0591..9a9fd9294261a 100644
--- a/lldb/include/lldb/Core/ValueObjectChild.h
+++ b/lldb/include/lldb/Core/ValueObjectChild.h
@@ -30,7 +30,7 @@ class ValueObjectChild : public ValueObject {
 public:
   ~ValueObjectChild() override;
 
-  uint64_t GetByteSize() override { return m_byte_size; }
+  llvm::Optional<uint64_t> GetByteSize() override { return m_byte_size; }
 
   lldb::offset_t GetByteOffset() override { return m_byte_offset; }
 
diff --git a/lldb/include/lldb/Core/ValueObjectConstResult.h b/lldb/include/lldb/Core/ValueObjectConstResult.h
index 0e868c687e931..8d823baa0b7b4 100644
--- a/lldb/include/lldb/Core/ValueObjectConstResult.h
+++ b/lldb/include/lldb/Core/ValueObjectConstResult.h
@@ -62,7 +62,7 @@ class ValueObjectConstResult : public ValueObject {
   static lldb::ValueObjectSP Create(ExecutionContextScope *exe_scope,
                                     const Status &error);
 
-  uint64_t GetByteSize() override;
+  llvm::Optional<uint64_t> GetByteSize() override;
 
   lldb::ValueType GetValueType() const override;
 
@@ -113,7 +113,7 @@ class ValueObjectConstResult : public ValueObject {
   CompilerType GetCompilerTypeImpl() override;
 
   ConstString m_type_name;
-  uint64_t m_byte_size;
+  llvm::Optional<uint64_t> m_byte_size;
 
   ValueObjectConstResultImpl m_impl;
 
diff --git a/lldb/include/lldb/Core/ValueObjectDynamicValue.h b/lldb/include/lldb/Core/ValueObjectDynamicValue.h
index 9f5304b55e934..2806857339efb 100644
--- a/lldb/include/lldb/Core/ValueObjectDynamicValue.h
+++ b/lldb/include/lldb/Core/ValueObjectDynamicValue.h
@@ -34,7 +34,7 @@ class ValueObjectDynamicValue : public ValueObject {
 public:
   ~ValueObjectDynamicValue() override;
 
-  uint64_t GetByteSize() override;
+  llvm::Optional<uint64_t> GetByteSize() override;
 
   ConstString GetTypeName() override;
 
diff --git a/lldb/include/lldb/Core/ValueObjectMemory.h b/lldb/include/lldb/Core/ValueObjectMemory.h
index d1cd6ae41445d..b5d5e6ecf4c0e 100644
--- a/lldb/include/lldb/Core/ValueObjectMemory.h
+++ b/lldb/include/lldb/Core/ValueObjectMemory.h
@@ -40,7 +40,7 @@ class ValueObjectMemory : public ValueObject {
                                     const Address &address,
                                     const CompilerType &ast_type);
 
-  uint64_t GetByteSize() override;
+  llvm::Optional<uint64_t> GetByteSize() override;
 
   ConstString GetTypeName() override;
 
diff --git a/lldb/include/lldb/Core/ValueObjectRegister.h b/lldb/include/lldb/Core/ValueObjectRegister.h
index 41051d93b707e..3968584ad5185 100644
--- a/lldb/include/lldb/Core/ValueObjectRegister.h
+++ b/lldb/include/lldb/Core/ValueObjectRegister.h
@@ -36,7 +36,7 @@ class ValueObjectRegisterSet : public ValueObject {
                                     lldb::RegisterContextSP &reg_ctx_sp,
                                     uint32_t set_idx);
 
-  uint64_t GetByteSize() override;
+  llvm::Optional<uint64_t> GetByteSize() override;
 
   lldb::ValueType GetValueType() const override {
     return lldb::eValueTypeRegisterSet;
@@ -86,7 +86,7 @@ class ValueObjectRegister : public ValueObject {
                                     lldb::RegisterContextSP &reg_ctx_sp,
                                     uint32_t reg_num);
 
-  uint64_t GetByteSize() override;
+  llvm::Optional<uint64_t> GetByteSize() override;
 
   lldb::ValueType GetValueType() const override {
     return lldb::eValueTypeRegister;
diff --git a/lldb/include/lldb/Core/ValueObjectSyntheticFilter.h b/lldb/include/lldb/Core/ValueObjectSyntheticFilter.h
index cb471657aec9b..41c461ce13f0d 100644
--- a/lldb/include/lldb/Core/ValueObjectSyntheticFilter.h
+++ b/lldb/include/lldb/Core/ValueObjectSyntheticFilter.h
@@ -36,7 +36,7 @@ class ValueObjectSynthetic : public ValueObject {
 public:
   ~ValueObjectSynthetic() override;
 
-  uint64_t GetByteSize() override;
+  llvm::Optional<uint64_t> GetByteSize() override;
 
   ConstString GetTypeName() override;
 
diff --git a/lldb/include/lldb/Core/ValueObjectVariable.h b/lldb/include/lldb/Core/ValueObjectVariable.h
index b7e262574a14d..23fdedbf5a4a6 100644
--- a/lldb/include/lldb/Core/ValueObjectVariable.h
+++ b/lldb/include/lldb/Core/ValueObjectVariable.h
@@ -37,7 +37,7 @@ class ValueObjectVariable : public ValueObject {
   static lldb::ValueObjectSP Create(ExecutionContextScope *exe_scope,
                                     const lldb::VariableSP &var_sp);
 
-  uint64_t GetByteSize() override;
+  llvm::Optional<uint64_t> GetByteSize() override;
 
   ConstString GetTypeName() override;
 
diff --git a/lldb/include/lldb/Expression/ExpressionVariable.h b/lldb/include/lldb/Expression/ExpressionVariable.h
index 60062d212badf..4259e6395da47 100644
--- a/lldb/include/lldb/Expression/ExpressionVariable.h
+++ b/lldb/include/lldb/Expression/ExpressionVariable.h
@@ -32,7 +32,7 @@ class ExpressionVariable
 
   virtual ~ExpressionVariable();
 
-  size_t GetByteSize() { return m_frozen_sp->GetByteSize(); }
+  llvm::Optional<uint64_t> GetByteSize() { return m_frozen_sp->GetByteSize(); }
 
   ConstString GetName() { return m_frozen_sp->GetName(); }
 
diff --git a/lldb/include/lldb/Target/StackFrameRecognizer.h b/lldb/include/lldb/Target/StackFrameRecognizer.h
index 302b56bec907b..baffc890bb065 100644
--- a/lldb/include/lldb/Target/StackFrameRecognizer.h
+++ b/lldb/include/lldb/Target/StackFrameRecognizer.h
@@ -154,7 +154,9 @@ class ValueObjectRecognizerSynthesizedValue : public ValueObject {
     SetName(parent.GetName());
   }
 
-  uint64_t GetByteSize() override { return m_parent->GetByteSize(); }
+  llvm::Optional<uint64_t> GetByteSize() override {
+    return m_parent->GetByteSize();
+  }
   lldb::ValueType GetValueType() const override { return m_type; }
   bool UpdateValue() override {
     if (!m_parent->UpdateValueIfNeeded()) return false;
diff --git a/lldb/source/API/SBValue.cpp b/lldb/source/API/SBValue.cpp
index 7485b0ee1838e..686d1f23a75a8 100644
--- a/lldb/source/API/SBValue.cpp
+++ b/lldb/source/API/SBValue.cpp
@@ -333,7 +333,7 @@ size_t SBValue::GetByteSize() {
   ValueLocker locker;
   lldb::ValueObjectSP value_sp(GetSP(locker));
   if (value_sp) {
-    result = value_sp->GetByteSize();
+    result = value_sp->GetByteSize().getValueOr(0);
   }
 
   return result;
diff --git a/lldb/source/Commands/CommandObjectWatchpoint.cpp b/lldb/source/Commands/CommandObjectWatchpoint.cpp
index ce4662930a7c2..c2a008af79d6f 100644
--- a/lldb/source/Commands/CommandObjectWatchpoint.cpp
+++ b/lldb/source/Commands/CommandObjectWatchpoint.cpp
@@ -905,7 +905,7 @@ corresponding to the byte size of the data type.");
         // We're in business.
         // Find out the size of this variable.
         size = m_option_watchpoint.watch_size == 0
-                   ? valobj_sp->GetByteSize()
+                   ? valobj_sp->GetByteSize().getValueOr(0)
                    : m_option_watchpoint.watch_size;
       }
       compiler_type = valobj_sp->GetCompilerType();
diff --git a/lldb/source/Core/ValueObject.cpp b/lldb/source/Core/ValueObject.cpp
index 3a775b07e5e1f..aedefd0cf0fd9 100644
--- a/lldb/source/Core/ValueObject.cpp
+++ b/lldb/source/Core/ValueObject.cpp
@@ -849,7 +849,7 @@ bool ValueObject::SetData(DataExtractor &data, Status &error) {
   uint64_t count = 0;
   const Encoding encoding = GetCompilerType().GetEncoding(count);
 
-  const size_t byte_size = GetByteSize();
+  const size_t byte_size = GetByteSize().getValueOr(0);
 
   Value::ValueType value_type = m_value.GetValueType();
 
@@ -1524,7 +1524,7 @@ bool ValueObject::SetValueFromCString(const char *value_str, Status &error) {
   uint64_t count = 0;
   const Encoding encoding = GetCompilerType().GetEncoding(count);
 
-  const size_t byte_size = GetByteSize();
+  const size_t byte_size = GetByteSize().getValueOr(0);
 
   Value::ValueType value_type = m_value.GetValueType();
 
@@ -1741,13 +1741,13 @@ ValueObjectSP ValueObject::GetSyntheticBitFieldChild(uint32_t from, uint32_t to,
       uint32_t bit_field_offset = from;
       if (GetDataExtractor().GetByteOrder() == eByteOrderBig)
         bit_field_offset =
-            GetByteSize() * 8 - bit_field_size - bit_field_offset;
+            GetByteSize().getValueOr(0) * 8 - bit_field_size - bit_field_offset;
       // We haven't made a synthetic array member for INDEX yet, so lets make
       // one and cache it for any future reference.
       ValueObjectChild *synthetic_child = new ValueObjectChild(
-          *this, GetCompilerType(), index_const_str, GetByteSize(), 0,
-          bit_field_size, bit_field_offset, false, false, eAddressTypeInvalid,
-          0);
+          *this, GetCompilerType(), index_const_str,
+          GetByteSize().getValueOr(0), 0, bit_field_size, bit_field_offset,
+          false, false, eAddressTypeInvalid, 0);
 
       // Cache the value if we got one back...
       if (synthetic_child) {
diff --git a/lldb/source/Core/ValueObjectCast.cpp b/lldb/source/Core/ValueObjectCast.cpp
index 22e856be539b5..7b6d3591faf44 100644
--- a/lldb/source/Core/ValueObjectCast.cpp
+++ b/lldb/source/Core/ValueObjectCast.cpp
@@ -47,7 +47,7 @@ size_t ValueObjectCast::CalculateNumChildren(uint32_t max) {
   return children_count <= max ? children_count : max;
 }
 
-uint64_t ValueObjectCast::GetByteSize() {
+llvm::Optional<uint64_t> ValueObjectCast::GetByteSize() {
   ExecutionContext exe_ctx(GetExecutionContextRef());
   return m_value.GetValueByteSize(nullptr, &exe_ctx);
 }
diff --git a/lldb/source/Core/ValueObjectConstResult.cpp b/lldb/source/Core/ValueObjectConstResult.cpp
index 8d84f8e62ccc5..fd31ddc676b43 100644
--- a/lldb/source/Core/ValueObjectConstResult.cpp
+++ b/lldb/source/Core/ValueObjectConstResult.cpp
@@ -179,8 +179,7 @@ ValueObjectSP ValueObjectConstResult::Create(ExecutionContextScope *exe_scope,
 ValueObjectConstResult::ValueObjectConstResult(ExecutionContextScope *exe_scope,
                                                ValueObjectManager &manager,
                                                const Status &error)
-    : ValueObject(exe_scope, manager), m_type_name(), m_byte_size(0),
-      m_impl(this) {
+    : ValueObject(exe_scope, manager), m_impl(this) {
   m_error = error;
   SetIsConstant();
 }
@@ -189,8 +188,7 @@ ValueObjectConstResult::ValueObjectConstResult(ExecutionContextScope *exe_scope,
                                                ValueObjectManager &manager,
                                                const Value &value,
                                                ConstString name, Module *module)
-    : ValueObject(exe_scope, manager), m_type_name(), m_byte_size(0),
-      m_impl(this) {
+    : ValueObject(exe_scope, manager), m_impl(this) {
   m_value = value;
   m_name = name;
   ExecutionContext exe_ctx;
@@ -208,9 +206,9 @@ lldb::ValueType ValueObjectConstResult::GetValueType() const {
   return eValueTypeConstResult;
 }
 
-uint64_t ValueObjectConstResult::GetByteSize() {
+llvm::Optional<uint64_t> ValueObjectConstResult::GetByteSize() {
   ExecutionContext exe_ctx(GetExecutionContextRef());
-  if (m_byte_size == 0) {
+  if (!m_byte_size) {
     if (auto size =
         GetCompilerType().GetByteSize(exe_ctx.GetBestExecutionContextScope()))
       SetByteSize(*size);
diff --git a/lldb/source/Core/ValueObjectDynamicValue.cpp b/lldb/source/Core/ValueObjectDynamicValue.cpp
index ca66740cb55d4..1c25b8c85a059 100644
--- a/lldb/source/Core/ValueObjectDynamicValue.cpp
+++ b/lldb/source/Core/ValueObjectDynamicValue.cpp
@@ -98,7 +98,7 @@ size_t ValueObjectDynamicValue::CalculateNumChildren(uint32_t max) {
     return m_parent->GetNumChildren(max);
 }
 
-uint64_t ValueObjectDynamicValue::GetByteSize() {
+llvm::Optional<uint64_t> ValueObjectDynamicValue::GetByteSize() {
   const bool success = UpdateValueIfNeeded(false);
   if (success && m_dynamic_type_info.HasType()) {
     ExecutionContext exe_ctx(GetExecutionContextRef());
diff --git a/lldb/source/Core/ValueObjectMemory.cpp b/lldb/source/Core/ValueObjectMemory.cpp
index 8e7d3ebc93f69..17fade9e5fdc3 100644
--- a/lldb/source/Core/ValueObjectMemory.cpp
+++ b/lldb/source/Core/ValueObjectMemory.cpp
@@ -139,13 +139,11 @@ size_t ValueObjectMemory::CalculateNumChildren(uint32_t max) {
   return child_count <= max ? child_count : max;
 }
 
-uint64_t ValueObjectMemory::GetByteSize() {
+llvm::Optional<uint64_t> ValueObjectMemory::GetByteSize() {
   ExecutionContext exe_ctx(GetExecutionContextRef());
   if (m_type_sp)
-    return m_type_sp->GetByteSize(exe_ctx.GetBestExecutionContextScope())
-        .getValueOr(0);
-  return m_compiler_type.GetByteSize(exe_ctx.GetBestExecutionContextScope())
-      .getValueOr(0);
+    return m_type_sp->GetByteSize(exe_ctx.GetBestExecutionContextScope());
+  return m_compiler_type.GetByteSize(exe_ctx.GetBestExecutionContextScope());
 }
 
 lldb::ValueType ValueObjectMemory::GetValueType() const {
diff --git a/lldb/source/Core/ValueObjectRegister.cpp b/lldb/source/Core/ValueObjectRegister.cpp
index ec87c38fb3679..27461e9cebc41 100644
--- a/lldb/source/Core/ValueObjectRegister.cpp
+++ b/lldb/source/Core/ValueObjectRegister.cpp
@@ -81,7 +81,7 @@ size_t ValueObjectRegisterSet::CalculateNumChildren(uint32_t max) {
   return 0;
 }
 
-uint64_t ValueObjectRegisterSet::GetByteSize() { return 0; }
+llvm::Optional<uint64_t> ValueObjectRegisterSet::GetByteSize() { return 0; }
 
 bool ValueObjectRegisterSet::UpdateValue() {
   m_error.Clear();
@@ -229,7 +229,9 @@ size_t ValueObjectRegister::CalculateNumChildren(uint32_t max) {
   return children_count <= max ? children_count : max;
 }
 
-uint64_t ValueObjectRegister::GetByteSize() { return m_reg_info.byte_size; }
+llvm::Optional<uint64_t> ValueObjectRegister::GetByteSize() {
+  return m_reg_info.byte_size;
+}
 
 bool ValueObjectRegister::UpdateValue() {
   m_error.Clear();
diff --git a/lldb/source/Core/ValueObjectSyntheticFilter.cpp b/lldb/source/Core/ValueObjectSyntheticFilter.cpp
index 32d1e6ab8368c..fb2d32e602cea 100644
--- a/lldb/source/Core/ValueObjectSyntheticFilter.cpp
+++ b/lldb/source/Core/ValueObjectSyntheticFilter.cpp
@@ -121,7 +121,9 @@ bool ValueObjectSynthetic::MightHaveChildren() {
   return (m_might_have_children != eLazyBoolNo);
 }
 
-uint64_t ValueObjectSynthetic::GetByteSize() { return m_parent->GetByteSize(); }
+llvm::Optional<uint64_t> ValueObjectSynthetic::GetByteSize() {
+  return m_parent->GetByteSize();
+}
 
 lldb::ValueType ValueObjectSynthetic::GetValueType() const {
   return m_parent->GetValueType();
diff --git a/lldb/source/Core/ValueObjectVariable.cpp b/lldb/source/Core/ValueObjectVariable.cpp
index 0d1e7b047a0ac..ab67e3038cf0a 100644
--- a/lldb/source/Core/ValueObjectVariable.cpp
+++ b/lldb/source/Core/ValueObjectVariable.cpp
@@ -105,15 +105,15 @@ size_t ValueObjectVariable::CalculateNumChildren(uint32_t max) {
   return child_count <= max ? child_count : max;
 }
 
-uint64_t ValueObjectVariable::GetByteSize() {
+llvm::Optional<uint64_t> ValueObjectVariable::GetByteSize() {
   ExecutionContext exe_ctx(GetExecutionContextRef());
 
   CompilerType type(GetCompilerType());
 
   if (!type.IsValid())
-    return 0;
+    return {};
 
-  return type.GetByteSize(exe_ctx.GetBestExecutionContextScope()).getValueOr(0);
+  return type.GetByteSize(exe_ctx.GetBestExecutionContextScope());
 }
 
 lldb::ValueType ValueObjectVariable::GetValueType() const {
diff --git a/lldb/source/Expression/ExpressionVariable.cpp b/lldb/source/Expression/ExpressionVariable.cpp
index d95f0745cf4ba..8b3dda7b2fe10 100644
--- a/lldb/source/Expression/ExpressionVariable.cpp
+++ b/lldb/source/Expression/ExpressionVariable.cpp
@@ -16,10 +16,10 @@ using namespace lldb_private;
 ExpressionVariable::~ExpressionVariable() {}
 
 uint8_t *ExpressionVariable::GetValueBytes() {
-  const size_t byte_size = m_frozen_sp->GetByteSize();
-  if (byte_size > 0) {
-    if (m_frozen_sp->GetDataExtractor().GetByteSize() < byte_size) {
-      m_frozen_sp->GetValue().ResizeData(byte_size);
+  llvm::Optional<uint64_t> byte_size = m_frozen_sp->GetByteSize();
+  if (byte_size && *byte_size) {
+    if (m_frozen_sp->GetDataExtractor().GetByteSize() < *byte_size) {
+      m_frozen_sp->GetValue().ResizeData(*byte_size);
       m_frozen_sp->GetValue().GetData(m_frozen_sp->GetDataExtractor());
     }
     return const_cast<uint8_t *>(
diff --git a/lldb/source/Expression/Materializer.cpp b/lldb/source/Expression/Materializer.cpp
index 6f8d9b154570a..327e15a26266f 100644
--- a/lldb/source/Expression/Materializer.cpp
+++ b/lldb/source/Expression/Materializer.cpp
@@ -67,7 +67,7 @@ class EntityPersistentVariable : public Materializer::Entity {
     const bool zero_memory = false;
 
     lldb::addr_t mem = map.Malloc(
-        m_persistent_variable_sp->GetByteSize(), 8,
+        m_persistent_variable_sp->GetByteSize().getValueOr(0), 8,
         lldb::ePermissionsReadable | lldb::ePermissionsWritable,
         IRMemoryMap::eAllocationPolicyMirror, zero_memory, allocate_error);
 
@@ -106,7 +106,8 @@ class EntityPersistentVariable : public Materializer::Entity {
     Status write_error;
 
     map.WriteMemory(mem, m_persistent_variable_sp->GetValueBytes(),
-                    m_persistent_variable_sp->GetByteSize(), write_error);
+                    m_persistent_variable_sp->GetByteSize().getValueOr(0),
+                    write_error);
 
     if (!write_error.Success()) {
       err.SetErrorStringWithFormat(
@@ -234,7 +235,7 @@ class EntityPersistentVariable : public Materializer::Entity {
             map.GetBestExecutionContextScope(),
             m_persistent_variable_sp.get()->GetCompilerType(),
             m_persistent_variable_sp->GetName(), location, eAddressTypeLoad,
-            m_persistent_variable_sp->GetByteSize());
+            m_persistent_variable_sp->GetByteSize().getValueOr(0));
 
         if (frame_top != LLDB_INVALID_ADDRESS &&
             frame_bottom != LLDB_INVALID_ADDRESS && location >= frame_bottom &&
@@ -279,7 +280,8 @@ class EntityPersistentVariable : public Materializer::Entity {
         LLDB_LOGF(log, "Dematerializing %s from 0x%" PRIx64 " (size = %llu)",
                   m_persistent_variable_sp->GetName().GetCString(),
                   (uint64_t)mem,
-                  (unsigned long long)m_persistent_variable_sp->GetByteSize());
+                  (unsigned long long)m_persistent_variable_sp->GetByteSize()
+                      .getValueOr(0));
 
         // Read the contents of the spare memory area
 
@@ -288,7 +290,7 @@ class EntityPersistentVariable : public Materializer::Entity {
         Status read_error;
 
         map.ReadMemory(m_persistent_variable_sp->GetValueBytes(), mem,
-                       m_persistent_variable_sp->GetByteSize(), read_error);
+                       m_persistent_variable_sp->GetByteSize().getValueOr(0), read_error);
 
         if (!read_error.Success()) {
           err.SetErrorStringWithFormat(
@@ -369,10 +371,11 @@ class EntityPersistentVariable : public Materializer::Entity {
       if (!err.Success()) {
         dump_stream.Printf("  <could not be read>\n");
       } else {
-        DataBufferHeap data(m_persistent_variable_sp->GetByteSize(), 0);
+        DataBufferHeap data(
+            m_persistent_variable_sp->GetByteSize().getValueOr(0), 0);
 
         map.ReadMemory(data.GetBytes(), target_address,
-                       m_persistent_variable_sp->GetByteSize(), err);
+                       m_persistent_variable_sp->GetByteSize().getValueOr(0), err);
 
         if (!err.Success()) {
           dump_stream.Printf("  <could not be read>\n");
@@ -621,8 +624,8 @@ class EntityVariable : public Materializer::Entity {
 
       Status extract_error;
 
-      map.GetMemoryData(data, m_temporary_allocation, valobj_sp->GetByteSize(),
-                        extract_error);
+      map.GetMemoryData(data, m_temporary_allocation,
+                        valobj_sp->GetByteSize().getValueOr(0), extract_error);
 
       if (!extract_error.Success()) {
         err.SetErrorStringWithFormat("couldn't get the data for variable %s",
@@ -919,7 +922,7 @@ class EntityResultVariable : public Materializer::Entity {
 
     ret->ValueUpdated();
 
-    const size_t pvar_byte_size = ret->GetByteSize();
+    const size_t pvar_byte_size = ret->GetByteSize().getValueOr(0);
     uint8_t *pvar_data = ret->GetValueBytes();
 
     map.ReadMemory(pvar_data, address, pvar_byte_size, read_error);
diff --git a/lldb/source/Target/StackFrame.cpp b/lldb/source/Target/StackFrame.cpp
index 098aed9cd8125..22bca52d7f98a 100644
--- a/lldb/source/Target/StackFrame.cpp
+++ b/lldb/source/Target/StackFrame.cpp
@@ -1408,7 +1408,7 @@ ValueObjectSP GetValueForOffset(StackFrame &frame, ValueObjectSP &parent,
     }
 
     int64_t child_offset = child_sp->GetByteOffset();
-    int64_t child_size = child_sp->GetByteSize();
+    int64_t child_size = child_sp->GetByteSize().getValueOr(0);
 
     if (offset >= child_offset && offset < (child_offset + child_size)) {
       return GetValueForOffset(frame, child_sp, offset - child_offset);
@@ -1441,8 +1441,8 @@ ValueObjectSP GetValueForDereferincingOffset(StackFrame &frame,
   }
 
   if (offset >= 0 && uint64_t(offset) >= pointee->GetByteSize()) {
-    int64_t index = offset / pointee->GetByteSize();
-    offset = offset % pointee->GetByteSize();
+    int64_t index = offset / pointee->GetByteSize().getValueOr(1);
+    offset = offset % pointee->GetByteSize().getValueOr(1);
     const bool can_create = true;
     pointee = base->GetSyntheticArrayMember(index, can_create);
   }

From 136c8f50e96381ee9daf0ac3dbe524ba63d51560 Mon Sep 17 00:00:00 2001
From: Roman Lebedev <lebedev.ri@gmail.com>
Date: Sat, 25 Jul 2020 21:43:36 +0300
Subject: [PATCH 0089/1035] [Reduce] Try turning function definitions into
 declarations first, NFCI-ish

ReduceFunctions could do it, but it also replaces *all* calls with undef,
so if any of undef replacements makes reduction uninteresting,
it won't work.

ReduceBasicBlocks also could do it, but well, it may take many guesses
for all the blocks of a function to happen to be out-of-chunk,
which is not a very efficient way to go about it.

So let's just do this first.
---
 llvm/test/Reduce/remove-function-bodies.ll    | 17 ++++++
 llvm/tools/llvm-reduce/CMakeLists.txt         |  1 +
 llvm/tools/llvm-reduce/DeltaManager.h         |  2 +
 .../deltas/ReduceFunctionBodies.cpp           | 54 +++++++++++++++++++
 .../llvm-reduce/deltas/ReduceFunctionBodies.h | 18 +++++++
 5 files changed, 92 insertions(+)
 create mode 100644 llvm/test/Reduce/remove-function-bodies.ll
 create mode 100644 llvm/tools/llvm-reduce/deltas/ReduceFunctionBodies.cpp
 create mode 100644 llvm/tools/llvm-reduce/deltas/ReduceFunctionBodies.h

diff --git a/llvm/test/Reduce/remove-function-bodies.ll b/llvm/test/Reduce/remove-function-bodies.ll
new file mode 100644
index 0000000000000..c70df0e38f3dd
--- /dev/null
+++ b/llvm/test/Reduce/remove-function-bodies.ll
@@ -0,0 +1,17 @@
+; RUN: llvm-reduce --test FileCheck --test-arg --check-prefixes=CHECK-ALL,CHECK-INTERESTINGNESS --test-arg %s --test-arg --input-file %s -o %t
+; RUN: cat %t | FileCheck --check-prefixes=CHECK-ALL,CHECK-FINAL %s
+
+; CHECK-INTERESTINGNESS: @callee(
+; CHECK-FINAL: declare void @callee()
+define void @callee() {
+  ret void
+}
+
+; CHECK-ALL: define void @caller()
+define void @caller() {
+entry:
+; CHECK-ALL: call void @callee()
+; CHECK-ALL: ret void
+  call void @callee()
+  ret void
+}
diff --git a/llvm/tools/llvm-reduce/CMakeLists.txt b/llvm/tools/llvm-reduce/CMakeLists.txt
index 01b9d0b4afe1a..81b4e95eece59 100644
--- a/llvm/tools/llvm-reduce/CMakeLists.txt
+++ b/llvm/tools/llvm-reduce/CMakeLists.txt
@@ -16,6 +16,7 @@ add_llvm_tool(llvm-reduce
   deltas/ReduceArguments.cpp
   deltas/ReduceAttributes.cpp
   deltas/ReduceBasicBlocks.cpp
+  deltas/ReduceFunctionBodies.cpp
   deltas/ReduceFunctions.cpp
   deltas/ReduceGlobalVars.cpp
   deltas/ReduceInstructions.cpp
diff --git a/llvm/tools/llvm-reduce/DeltaManager.h b/llvm/tools/llvm-reduce/DeltaManager.h
index b1a4ee0df4dbe..83278c88791b0 100644
--- a/llvm/tools/llvm-reduce/DeltaManager.h
+++ b/llvm/tools/llvm-reduce/DeltaManager.h
@@ -16,6 +16,7 @@
 #include "deltas/ReduceArguments.h"
 #include "deltas/ReduceAttributes.h"
 #include "deltas/ReduceBasicBlocks.h"
+#include "deltas/ReduceFunctionBodies.h"
 #include "deltas/ReduceFunctions.h"
 #include "deltas/ReduceGlobalVars.h"
 #include "deltas/ReduceInstructions.h"
@@ -26,6 +27,7 @@ namespace llvm {
 
 // TODO: Add CLI option to run only specified Passes (for unit tests)
 inline void runDeltaPasses(TestRunner &Tester) {
+  reduceFunctionBodiesDeltaPass(Tester);
   reduceFunctionsDeltaPass(Tester);
   reduceBasicBlocksDeltaPass(Tester);
   reduceGlobalsDeltaPass(Tester);
diff --git a/llvm/tools/llvm-reduce/deltas/ReduceFunctionBodies.cpp b/llvm/tools/llvm-reduce/deltas/ReduceFunctionBodies.cpp
new file mode 100644
index 0000000000000..a047d42b50c56
--- /dev/null
+++ b/llvm/tools/llvm-reduce/deltas/ReduceFunctionBodies.cpp
@@ -0,0 +1,54 @@
+//===- ReduceFunctions.cpp - Specialized Delta Pass -----------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements a function which calls the Generic Delta pass in order
+// to reduce function bodies in the provided Module.
+//
+//===----------------------------------------------------------------------===//
+
+#include "ReduceFunctionBodies.h"
+#include "Delta.h"
+
+using namespace llvm;
+
+/// Removes all the bodies of defined functions that aren't inside any of the
+/// desired Chunks.
+static void
+extractFunctionBodiesFromModule(const std::vector<Chunk> &ChunksToKeep,
+                                Module *Program) {
+  Oracle O(ChunksToKeep);
+
+  // Delete out-of-chunk function bodies
+  std::vector<Function *> FuncDefsToReduce;
+  for (auto &F : *Program)
+    if (!F.isDeclaration() && !O.shouldKeep())
+      F.deleteBody();
+}
+
+/// Counts the amount of non-declaration functions and prints their
+/// respective name & index
+static int countFunctionDefinitions(Module *Program) {
+  // TODO: Silence index with --quiet flag
+  errs() << "----------------------------\n";
+  errs() << "Function Definition Index Reference:\n";
+  int FunctionDefinitionCount = 0;
+  for (auto &F : *Program)
+    if (!F.isDeclaration())
+      errs() << "\t" << ++FunctionDefinitionCount << ": " << F.getName()
+             << "\n";
+
+  errs() << "----------------------------\n";
+  return FunctionDefinitionCount;
+}
+
+void llvm::reduceFunctionBodiesDeltaPass(TestRunner &Test) {
+  errs() << "*** Reducing Function Bodies...\n";
+  int Functions = countFunctionDefinitions(Test.getProgram());
+  runDeltaPass(Test, Functions, extractFunctionBodiesFromModule);
+  errs() << "----------------------------\n";
+}
diff --git a/llvm/tools/llvm-reduce/deltas/ReduceFunctionBodies.h b/llvm/tools/llvm-reduce/deltas/ReduceFunctionBodies.h
new file mode 100644
index 0000000000000..8c06c2e4a1a93
--- /dev/null
+++ b/llvm/tools/llvm-reduce/deltas/ReduceFunctionBodies.h
@@ -0,0 +1,18 @@
+//===- ReduceFunctionBodies.h - Specialized Delta Pass --------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements a function which calls the Generic Delta pass in order
+// to reduce function bodies in the provided Module.
+//
+//===----------------------------------------------------------------------===//
+
+#include "Delta.h"
+
+namespace llvm {
+void reduceFunctionBodiesDeltaPass(TestRunner &Test);
+} // namespace llvm

From 48c3228c5cf7ba4605180b9d6b81fa6e575d964d Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
Date: Sat, 25 Jul 2020 18:51:58 +0000
Subject: [PATCH 0090/1035] [gn build] Port 136c8f50e96

---
 llvm/utils/gn/secondary/llvm/tools/llvm-reduce/BUILD.gn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/utils/gn/secondary/llvm/tools/llvm-reduce/BUILD.gn b/llvm/utils/gn/secondary/llvm/tools/llvm-reduce/BUILD.gn
index a8648d73ca0d1..14c38d87fe5f8 100644
--- a/llvm/utils/gn/secondary/llvm/tools/llvm-reduce/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/tools/llvm-reduce/BUILD.gn
@@ -14,6 +14,7 @@ executable("llvm-reduce") {
     "deltas/ReduceArguments.cpp",
     "deltas/ReduceAttributes.cpp",
     "deltas/ReduceBasicBlocks.cpp",
+    "deltas/ReduceFunctionBodies.cpp",
     "deltas/ReduceFunctions.cpp",
     "deltas/ReduceGlobalVars.cpp",
     "deltas/ReduceInstructions.cpp",

From 6a75496836ea14bcfd2f4b59d35a1cad4ac58cee Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Sat, 25 Jul 2020 12:33:18 -0700
Subject: [PATCH 0091/1035] [Driver] Define LinkOption and fix forwarded
 options to GCC for linking

Many driver options are neither 'DriverOption' nor 'LinkerInput'. When gcc is
used for linking, these options get forwarded even if they don't have anything
to do with linking. Among these options, clang-specific ones can cause gcc to
error.

Just use 'OPT_Link_Group' and a new flag 'LinkOption' for options which already
have a group.

gfortran support apparently bit rots (which does not seem to make much sense). XFAIL the test.
---
 clang/include/clang/Driver/Options.h  |  3 ++-
 clang/include/clang/Driver/Options.td | 24 ++++++++++++++----------
 clang/lib/Driver/ToolChains/Gnu.cpp   | 22 +---------------------
 clang/test/Driver/gcc_forward.c       | 17 ++++++++---------
 clang/test/Driver/gfortran.f90        |  1 +
 5 files changed, 26 insertions(+), 41 deletions(-)

diff --git a/clang/include/clang/Driver/Options.h b/clang/include/clang/Driver/Options.h
index 7c5cddd9e8960..9831efda4e580 100644
--- a/clang/include/clang/Driver/Options.h
+++ b/clang/include/clang/Driver/Options.h
@@ -33,7 +33,8 @@ enum ClangFlags {
   CC1Option = (1 << 10),
   CC1AsOption = (1 << 11),
   NoDriverOption = (1 << 12),
-  Ignored = (1 << 13)
+  LinkOption = (1 << 13),
+  Ignored = (1 << 14),
 };
 
 enum ID {
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 700a5c4578f68..b6e31700c0a62 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -52,6 +52,10 @@ def CC1AsOption : OptionFlag;
 // NoDriverOption - This option should not be accepted by the driver.
 def NoDriverOption : OptionFlag;
 
+// If an option affects linking, but has a primary group (so Link_Group cannot
+// be used), add this flag.
+def LinkOption : OptionFlag;
+
 // A short name to show in documentation. The name will be interpreted as rST.
 class DocName<string name> { string DocName = name; }
 
@@ -573,7 +577,7 @@ def config_system_dir_EQ : Joined<["--"], "config-system-dir=">, Flags<[DriverOp
   HelpText<"System directory for configuration files">;
 def config_user_dir_EQ : Joined<["--"], "config-user-dir=">, Flags<[DriverOption, HelpHidden]>,
   HelpText<"User directory for configuration files">;
-def coverage : Flag<["-", "--"], "coverage">, Flags<[CoreOption]>;
+def coverage : Flag<["-", "--"], "coverage">, Group<Link_Group>, Flags<[CoreOption]>;
 def cpp_precomp : Flag<["-"], "cpp-precomp">, Group<clang_ignored_f_Group>;
 def current__version : JoinedOrSeparate<["-"], "current_version">;
 def cxx_isystem : JoinedOrSeparate<["-"], "cxx-isystem">, Group<clang_i_Group>,
@@ -1747,7 +1751,7 @@ def fpass_plugin_EQ : Joined<["-"], "fpass-plugin=">,
   HelpText<"Load pass plugin from a dynamic shared object file (only with new pass manager).">;
 defm preserve_as_comments : OptOutFFlag<"preserve-as-comments", "",
   "Do not preserve comments in inline assembly">;
-def fprofile_arcs : Flag<["-"], "fprofile-arcs">, Group<f_Group>;
+def fprofile_arcs : Flag<["-"], "fprofile-arcs">, Group<f_Group>, Flags<[LinkOption]>;
 def fno_profile_arcs : Flag<["-"], "fno-profile-arcs">, Group<f_Group>;
 def framework : Separate<["-"], "framework">, Flags<[LinkerInput]>;
 def frandom_seed_EQ : Joined<["-"], "frandom-seed=">, Group<clang_ignored_f_Group>;
@@ -2724,7 +2728,7 @@ def nostdinc : Flag<["-"], "nostdinc">, Flags<[CoreOption]>;
 def nostdlibinc : Flag<["-"], "nostdlibinc">;
 def nostdincxx : Flag<["-"], "nostdinc++">, Flags<[CC1Option]>,
   HelpText<"Disable standard #include directories for the C++ standard library">;
-def nostdlib : Flag<["-"], "nostdlib">;
+def nostdlib : Flag<["-"], "nostdlib">, Group<Link_Group>;
 def nostdlibxx : Flag<["-"], "nostdlib++">;
 def object : Flag<["-"], "object">;
 def o : JoinedOrSeparate<["-"], "o">, Flags<[DriverOption, RenderAsInput, CC1Option, CC1AsOption]>,
@@ -2768,15 +2772,15 @@ def pthread : Flag<["-"], "pthread">, Flags<[CC1Option]>,
   HelpText<"Support POSIX threads in generated code">;
 def no_pthread : Flag<["-"], "no-pthread">, Flags<[CC1Option]>;
 def p : Flag<["-"], "p">;
-def pie : Flag<["-"], "pie">;
-def static_pie : Flag<["-"], "static-pie">;
+def pie : Flag<["-"], "pie">, Group<Link_Group>;
+def static_pie : Flag<["-"], "static-pie">, Group<Link_Group>;
 def read__only__relocs : Separate<["-"], "read_only_relocs">;
 def remap : Flag<["-"], "remap">;
 def rewrite_objc : Flag<["-"], "rewrite-objc">, Flags<[DriverOption,CC1Option]>,
   HelpText<"Rewrite Objective-C source to C++">, Group<Action_Group>;
 def rewrite_legacy_objc : Flag<["-"], "rewrite-legacy-objc">, Flags<[DriverOption]>,
   HelpText<"Rewrite Legacy Objective-C source to C++">;
-def rdynamic : Flag<["-"], "rdynamic">;
+def rdynamic : Flag<["-"], "rdynamic">, Group<Link_Group>;
 def resource_dir : Separate<["-"], "resource-dir">,
   Flags<[DriverOption, CC1Option, CoreOption, HelpHidden]>,
   HelpText<"The directory which holds the compiler resource files">;
@@ -2818,13 +2822,13 @@ def segs__read__only__addr : Separate<["-"], "segs_read_only_addr">;
 def segs__read__write__addr : Separate<["-"], "segs_read_write_addr">;
 def segs__read__ : Joined<["-"], "segs_read_">;
 def shared_libgcc : Flag<["-"], "shared-libgcc">;
-def shared : Flag<["-", "--"], "shared">;
+def shared : Flag<["-", "--"], "shared">, Group<Link_Group>;
 def single__module : Flag<["-"], "single_module">;
 def specs_EQ : Joined<["-", "--"], "specs=">;
 def specs : Separate<["-", "--"], "specs">, Flags<[Unsupported]>;
 def static_libgcc : Flag<["-"], "static-libgcc">;
 def static_libstdcxx : Flag<["-"], "static-libstdc++">;
-def static : Flag<["-", "--"], "static">, Flags<[NoArgumentUnused]>;
+def static : Flag<["-", "--"], "static">, Group<Link_Group>, Flags<[NoArgumentUnused]>;
 def std_default_EQ : Joined<["-"], "std-default=">;
 def std_EQ : Joined<["-", "--"], "std=">, Flags<[CC1Option]>,
   Group<CompileOnly_Group>, HelpText<"Language standard to compile for">,
@@ -3283,8 +3287,8 @@ defm : BooleanFFlag<"keep-inline-functions">, Group<clang_ignored_gcc_optimizati
 
 def fprofile_dir : Joined<["-"], "fprofile-dir=">, Group<f_Group>;
 
-def fuse_ld_EQ : Joined<["-"], "fuse-ld=">, Group<f_Group>, Flags<[CoreOption]>;
-def ld_path_EQ : Joined<["--"], "ld-path=">;
+def fuse_ld_EQ : Joined<["-"], "fuse-ld=">, Group<f_Group>, Flags<[CoreOption, LinkOption]>;
+def ld_path_EQ : Joined<["--"], "ld-path=">, Group<Link_Group>;
 
 defm align_labels : BooleanFFlag<"align-labels">, Group<clang_ignored_gcc_optimization_f_Group>;
 def falign_labels_EQ : Joined<["-"], "falign-labels=">, Group<clang_ignored_gcc_optimization_f_Group>;
diff --git a/clang/lib/Driver/ToolChains/Gnu.cpp b/clang/lib/Driver/ToolChains/Gnu.cpp
index 9ea07e44e37ec..1806c14c395dc 100644
--- a/clang/lib/Driver/ToolChains/Gnu.cpp
+++ b/clang/lib/Driver/ToolChains/Gnu.cpp
@@ -38,10 +38,7 @@ using tools::addMultilibFlag;
 using tools::addPathIfExists;
 
 static bool forwardToGCC(const Option &O) {
-  // Don't forward inputs from the original command line.  They are added from
-  // InputInfoList.
-  return O.getKind() != Option::InputClass &&
-         !O.hasFlag(options::DriverOption) && !O.hasFlag(options::LinkerInput);
+  return O.matches(options::OPT_Link_Group) || O.hasFlag(options::LinkOption);
 }
 
 // Switch CPU names not recognized by GNU assembler to a close CPU that it does
@@ -76,23 +73,6 @@ void tools::gcc::Common::ConstructJob(Compilation &C, const JobAction &JA,
       // to get to the assembler.
       A->claim();
 
-      // Don't forward any -g arguments to assembly steps.
-      if (isa<AssembleJobAction>(JA) &&
-          A->getOption().matches(options::OPT_g_Group))
-        continue;
-
-      // Don't forward any -W arguments to assembly and link steps.
-      if ((isa<AssembleJobAction>(JA) || isa<LinkJobAction>(JA)) &&
-          A->getOption().matches(options::OPT_W_Group))
-        continue;
-
-      // Don't forward -mno-unaligned-access since GCC doesn't understand
-      // it and because it doesn't affect the assembly or link steps.
-      if ((isa<AssembleJobAction>(JA) || isa<LinkJobAction>(JA)) &&
-          (A->getOption().matches(options::OPT_munaligned_access) ||
-           A->getOption().matches(options::OPT_mno_unaligned_access)))
-        continue;
-
       A->render(Args, CmdArgs);
     }
   }
diff --git a/clang/test/Driver/gcc_forward.c b/clang/test/Driver/gcc_forward.c
index f75b1c738b05c..9579d0d60d1cf 100644
--- a/clang/test/Driver/gcc_forward.c
+++ b/clang/test/Driver/gcc_forward.c
@@ -1,3 +1,8 @@
+// RUN: %clang -### %s -target aarch64-none-elf \
+// RUN:   --coverage -fuse-ld=lld --ld-path=ld -nostdlib -r -rdynamic -static -static-pie \
+// RUN:   2>&1 | FileCheck --check-prefix=FORWARD %s
+// FORWARD: gcc{{[^"]*}}" "--coverage" "-fuse-ld=lld" "--ld-path=ld" "-nostdlib" "-r" "-rdynamic" "-static" "-static-pie"
+
 // Check that we don't try to forward -Xclang or -mlinker-version to GCC.
 // PR12920 -- Check also we may not forward W_Group options to GCC.
 //
@@ -5,7 +10,7 @@
 // RUN:   %s \
 // RUN:   -Wall -Wdocumentation \
 // RUN:   -Xclang foo-bar \
-// RUN:   -march=x86-64 \
+// RUN:   -pie -march=x86-64 \
 // RUN:   -mlinker-version=10 -### 2> %t
 // RUN: FileCheck < %t %s
 //
@@ -15,13 +20,13 @@
 // CHECK: "-o" "{{[^"]+}}.o"
 //
 // gcc as ld.
-// CHECK: gcc{{[^"]*}}"
+// CHECK: gcc{{[^"]*}}" "-pie"
 // CHECK-NOT: "-mlinker-version=10"
 // CHECK-NOT: "-Xclang"
 // CHECK-NOT: "foo-bar"
 // CHECK-NOT: "-Wall"
 // CHECK-NOT: "-Wdocumentation"
-// CHECK: -march
+// CHECK-NOT: -march
 // CHECK-NOT: "-mlinker-version=10"
 // CHECK-NOT: "-Xclang"
 // CHECK-NOT: "foo-bar"
@@ -34,9 +39,3 @@
 // RUN:   | FileCheck --check-prefix=CHECK-ASM %s
 // CHECK-ASM: as
 // CHECK-ASM-NOT: "-g"
-
-// Check that we're not forwarding -mno-unaligned-access.
-// RUN: %clang -target aarch64-none-elf -mno-unaligned-access %s -### 2>&1 \
-// RUN:   | FileCheck --check-prefix=CHECK-ARM %s
-// CHECK-ARM: gcc{{[^"]*}}"
-// CHECK-ARM-NOT: -mno-unaligned-access
diff --git a/clang/test/Driver/gfortran.f90 b/clang/test/Driver/gfortran.f90
index d2f90b47a3902..6f972cc333ae0 100644
--- a/clang/test/Driver/gfortran.f90
+++ b/clang/test/Driver/gfortran.f90
@@ -1,3 +1,4 @@
+! XFAIL: *
 ! Test that Clang can forward all of the flags which are documented as
 ! being supported by gfortran to GCC when falling back to GCC for
 ! a fortran input file.

From 60a5799e6e8b1206640a2321eab60dcf1b65975d Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@intel.com>
Date: Sat, 25 Jul 2020 12:12:16 -0700
Subject: [PATCH 0092/1035] [X86] Fix intrinsic names in strict fp80 tests to
 use f80 in their names instead of x86_fp80.

The type is called x86_fp80, but when it is printed in the intrinsic
name it should be f80. The parser doesn't seem to care that the
name was wrong.
---
 .../CodeGen/X86/fp80-strict-scalar-cmp.ll     |  60 ++++-----
 llvm/test/CodeGen/X86/fp80-strict-scalar.ll   | 116 +++++++++---------
 2 files changed, 88 insertions(+), 88 deletions(-)

diff --git a/llvm/test/CodeGen/X86/fp80-strict-scalar-cmp.ll b/llvm/test/CodeGen/X86/fp80-strict-scalar-cmp.ll
index 1e38b6744f3c0..a3951f826fe09 100644
--- a/llvm/test/CodeGen/X86/fp80-strict-scalar-cmp.ll
+++ b/llvm/test/CodeGen/X86/fp80-strict-scalar-cmp.ll
@@ -33,7 +33,7 @@ define i32 @test_oeq_q(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 {
 ; X87-64-NEXT:    cmovnel %esi, %eax
 ; X87-64-NEXT:    cmovpl %esi, %eax
 ; X87-64-NEXT:    retq
-  %cond = call i1 @llvm.experimental.constrained.fcmp.x86_fp80(
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f80(
                                                x86_fp80 %f1, x86_fp80 %f2, metadata !"oeq",
                                                metadata !"fpexcept.strict") #0
   %res = select i1 %cond, i32 %a, i32 %b
@@ -70,7 +70,7 @@ define i32 @test_ogt_q(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 {
 ; X87-64-NEXT:    wait
 ; X87-64-NEXT:    cmovbel %esi, %eax
 ; X87-64-NEXT:    retq
-  %cond = call i1 @llvm.experimental.constrained.fcmp.x86_fp80(
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f80(
                                                x86_fp80 %f1, x86_fp80 %f2, metadata !"ogt",
                                                metadata !"fpexcept.strict") #0
   %res = select i1 %cond, i32 %a, i32 %b
@@ -107,7 +107,7 @@ define i32 @test_oge_q(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 {
 ; X87-64-NEXT:    wait
 ; X87-64-NEXT:    cmovbl %esi, %eax
 ; X87-64-NEXT:    retq
-  %cond = call i1 @llvm.experimental.constrained.fcmp.x86_fp80(
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f80(
                                                x86_fp80 %f1, x86_fp80 %f2, metadata !"oge",
                                                metadata !"fpexcept.strict") #0
   %res = select i1 %cond, i32 %a, i32 %b
@@ -144,7 +144,7 @@ define i32 @test_olt_q(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 {
 ; X87-64-NEXT:    wait
 ; X87-64-NEXT:    cmovbel %esi, %eax
 ; X87-64-NEXT:    retq
-  %cond = call i1 @llvm.experimental.constrained.fcmp.x86_fp80(
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f80(
                                                x86_fp80 %f1, x86_fp80 %f2, metadata !"olt",
                                                metadata !"fpexcept.strict") #0
   %res = select i1 %cond, i32 %a, i32 %b
@@ -181,7 +181,7 @@ define i32 @test_ole_q(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 {
 ; X87-64-NEXT:    wait
 ; X87-64-NEXT:    cmovbl %esi, %eax
 ; X87-64-NEXT:    retq
-  %cond = call i1 @llvm.experimental.constrained.fcmp.x86_fp80(
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f80(
                                                x86_fp80 %f1, x86_fp80 %f2, metadata !"ole",
                                                metadata !"fpexcept.strict") #0
   %res = select i1 %cond, i32 %a, i32 %b
@@ -218,7 +218,7 @@ define i32 @test_one_q(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 {
 ; X87-64-NEXT:    wait
 ; X87-64-NEXT:    cmovel %esi, %eax
 ; X87-64-NEXT:    retq
-  %cond = call i1 @llvm.experimental.constrained.fcmp.x86_fp80(
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f80(
                                                x86_fp80 %f1, x86_fp80 %f2, metadata !"one",
                                                metadata !"fpexcept.strict") #0
   %res = select i1 %cond, i32 %a, i32 %b
@@ -255,7 +255,7 @@ define i32 @test_ord_q(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 {
 ; X87-64-NEXT:    wait
 ; X87-64-NEXT:    cmovpl %esi, %eax
 ; X87-64-NEXT:    retq
-  %cond = call i1 @llvm.experimental.constrained.fcmp.x86_fp80(
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f80(
                                                x86_fp80 %f1, x86_fp80 %f2, metadata !"ord",
                                                metadata !"fpexcept.strict") #0
   %res = select i1 %cond, i32 %a, i32 %b
@@ -292,7 +292,7 @@ define i32 @test_ueq_q(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 {
 ; X87-64-NEXT:    wait
 ; X87-64-NEXT:    cmovnel %esi, %eax
 ; X87-64-NEXT:    retq
-  %cond = call i1 @llvm.experimental.constrained.fcmp.x86_fp80(
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f80(
                                                x86_fp80 %f1, x86_fp80 %f2, metadata !"ueq",
                                                metadata !"fpexcept.strict") #0
   %res = select i1 %cond, i32 %a, i32 %b
@@ -329,7 +329,7 @@ define i32 @test_ugt_q(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 {
 ; X87-64-NEXT:    wait
 ; X87-64-NEXT:    cmovael %esi, %eax
 ; X87-64-NEXT:    retq
-  %cond = call i1 @llvm.experimental.constrained.fcmp.x86_fp80(
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f80(
                                                x86_fp80 %f1, x86_fp80 %f2, metadata !"ugt",
                                                metadata !"fpexcept.strict") #0
   %res = select i1 %cond, i32 %a, i32 %b
@@ -366,7 +366,7 @@ define i32 @test_uge_q(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 {
 ; X87-64-NEXT:    wait
 ; X87-64-NEXT:    cmoval %esi, %eax
 ; X87-64-NEXT:    retq
-  %cond = call i1 @llvm.experimental.constrained.fcmp.x86_fp80(
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f80(
                                                x86_fp80 %f1, x86_fp80 %f2, metadata !"uge",
                                                metadata !"fpexcept.strict") #0
   %res = select i1 %cond, i32 %a, i32 %b
@@ -403,7 +403,7 @@ define i32 @test_ult_q(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 {
 ; X87-64-NEXT:    wait
 ; X87-64-NEXT:    cmovael %esi, %eax
 ; X87-64-NEXT:    retq
-  %cond = call i1 @llvm.experimental.constrained.fcmp.x86_fp80(
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f80(
                                                x86_fp80 %f1, x86_fp80 %f2, metadata !"ult",
                                                metadata !"fpexcept.strict") #0
   %res = select i1 %cond, i32 %a, i32 %b
@@ -440,7 +440,7 @@ define i32 @test_ule_q(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 {
 ; X87-64-NEXT:    wait
 ; X87-64-NEXT:    cmoval %esi, %eax
 ; X87-64-NEXT:    retq
-  %cond = call i1 @llvm.experimental.constrained.fcmp.x86_fp80(
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f80(
                                                x86_fp80 %f1, x86_fp80 %f2, metadata !"ule",
                                                metadata !"fpexcept.strict") #0
   %res = select i1 %cond, i32 %a, i32 %b
@@ -478,7 +478,7 @@ define i32 @test_une_q(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 {
 ; X87-64-NEXT:    cmovnel %edi, %eax
 ; X87-64-NEXT:    cmovpl %edi, %eax
 ; X87-64-NEXT:    retq
-  %cond = call i1 @llvm.experimental.constrained.fcmp.x86_fp80(
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f80(
                                                x86_fp80 %f1, x86_fp80 %f2, metadata !"une",
                                                metadata !"fpexcept.strict") #0
   %res = select i1 %cond, i32 %a, i32 %b
@@ -515,7 +515,7 @@ define i32 @test_uno_q(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 {
 ; X87-64-NEXT:    wait
 ; X87-64-NEXT:    cmovnpl %esi, %eax
 ; X87-64-NEXT:    retq
-  %cond = call i1 @llvm.experimental.constrained.fcmp.x86_fp80(
+  %cond = call i1 @llvm.experimental.constrained.fcmp.f80(
                                                x86_fp80 %f1, x86_fp80 %f2, metadata !"uno",
                                                metadata !"fpexcept.strict") #0
   %res = select i1 %cond, i32 %a, i32 %b
@@ -553,7 +553,7 @@ define i32 @test_oeq_s(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 {
 ; X87-64-NEXT:    cmovnel %esi, %eax
 ; X87-64-NEXT:    cmovpl %esi, %eax
 ; X87-64-NEXT:    retq
-  %cond = call i1 @llvm.experimental.constrained.fcmps.x86_fp80(
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f80(
                                                x86_fp80 %f1, x86_fp80 %f2, metadata !"oeq",
                                                metadata !"fpexcept.strict") #0
   %res = select i1 %cond, i32 %a, i32 %b
@@ -590,7 +590,7 @@ define i32 @test_ogt_s(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 {
 ; X87-64-NEXT:    wait
 ; X87-64-NEXT:    cmovbel %esi, %eax
 ; X87-64-NEXT:    retq
-  %cond = call i1 @llvm.experimental.constrained.fcmps.x86_fp80(
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f80(
                                                x86_fp80 %f1, x86_fp80 %f2, metadata !"ogt",
                                                metadata !"fpexcept.strict") #0
   %res = select i1 %cond, i32 %a, i32 %b
@@ -627,7 +627,7 @@ define i32 @test_oge_s(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 {
 ; X87-64-NEXT:    wait
 ; X87-64-NEXT:    cmovbl %esi, %eax
 ; X87-64-NEXT:    retq
-  %cond = call i1 @llvm.experimental.constrained.fcmps.x86_fp80(
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f80(
                                                x86_fp80 %f1, x86_fp80 %f2, metadata !"oge",
                                                metadata !"fpexcept.strict") #0
   %res = select i1 %cond, i32 %a, i32 %b
@@ -664,7 +664,7 @@ define i32 @test_olt_s(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 {
 ; X87-64-NEXT:    wait
 ; X87-64-NEXT:    cmovbel %esi, %eax
 ; X87-64-NEXT:    retq
-  %cond = call i1 @llvm.experimental.constrained.fcmps.x86_fp80(
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f80(
                                                x86_fp80 %f1, x86_fp80 %f2, metadata !"olt",
                                                metadata !"fpexcept.strict") #0
   %res = select i1 %cond, i32 %a, i32 %b
@@ -701,7 +701,7 @@ define i32 @test_ole_s(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 {
 ; X87-64-NEXT:    wait
 ; X87-64-NEXT:    cmovbl %esi, %eax
 ; X87-64-NEXT:    retq
-  %cond = call i1 @llvm.experimental.constrained.fcmps.x86_fp80(
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f80(
                                                x86_fp80 %f1, x86_fp80 %f2, metadata !"ole",
                                                metadata !"fpexcept.strict") #0
   %res = select i1 %cond, i32 %a, i32 %b
@@ -738,7 +738,7 @@ define i32 @test_one_s(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 {
 ; X87-64-NEXT:    wait
 ; X87-64-NEXT:    cmovel %esi, %eax
 ; X87-64-NEXT:    retq
-  %cond = call i1 @llvm.experimental.constrained.fcmps.x86_fp80(
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f80(
                                                x86_fp80 %f1, x86_fp80 %f2, metadata !"one",
                                                metadata !"fpexcept.strict") #0
   %res = select i1 %cond, i32 %a, i32 %b
@@ -775,7 +775,7 @@ define i32 @test_ord_s(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 {
 ; X87-64-NEXT:    wait
 ; X87-64-NEXT:    cmovpl %esi, %eax
 ; X87-64-NEXT:    retq
-  %cond = call i1 @llvm.experimental.constrained.fcmps.x86_fp80(
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f80(
                                                x86_fp80 %f1, x86_fp80 %f2, metadata !"ord",
                                                metadata !"fpexcept.strict") #0
   %res = select i1 %cond, i32 %a, i32 %b
@@ -812,7 +812,7 @@ define i32 @test_ueq_s(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 {
 ; X87-64-NEXT:    wait
 ; X87-64-NEXT:    cmovnel %esi, %eax
 ; X87-64-NEXT:    retq
-  %cond = call i1 @llvm.experimental.constrained.fcmps.x86_fp80(
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f80(
                                                x86_fp80 %f1, x86_fp80 %f2, metadata !"ueq",
                                                metadata !"fpexcept.strict") #0
   %res = select i1 %cond, i32 %a, i32 %b
@@ -849,7 +849,7 @@ define i32 @test_ugt_s(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 {
 ; X87-64-NEXT:    wait
 ; X87-64-NEXT:    cmovael %esi, %eax
 ; X87-64-NEXT:    retq
-  %cond = call i1 @llvm.experimental.constrained.fcmps.x86_fp80(
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f80(
                                                x86_fp80 %f1, x86_fp80 %f2, metadata !"ugt",
                                                metadata !"fpexcept.strict") #0
   %res = select i1 %cond, i32 %a, i32 %b
@@ -886,7 +886,7 @@ define i32 @test_uge_s(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 {
 ; X87-64-NEXT:    wait
 ; X87-64-NEXT:    cmoval %esi, %eax
 ; X87-64-NEXT:    retq
-  %cond = call i1 @llvm.experimental.constrained.fcmps.x86_fp80(
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f80(
                                                x86_fp80 %f1, x86_fp80 %f2, metadata !"uge",
                                                metadata !"fpexcept.strict") #0
   %res = select i1 %cond, i32 %a, i32 %b
@@ -923,7 +923,7 @@ define i32 @test_ult_s(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 {
 ; X87-64-NEXT:    wait
 ; X87-64-NEXT:    cmovael %esi, %eax
 ; X87-64-NEXT:    retq
-  %cond = call i1 @llvm.experimental.constrained.fcmps.x86_fp80(
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f80(
                                                x86_fp80 %f1, x86_fp80 %f2, metadata !"ult",
                                                metadata !"fpexcept.strict") #0
   %res = select i1 %cond, i32 %a, i32 %b
@@ -960,7 +960,7 @@ define i32 @test_ule_s(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 {
 ; X87-64-NEXT:    wait
 ; X87-64-NEXT:    cmoval %esi, %eax
 ; X87-64-NEXT:    retq
-  %cond = call i1 @llvm.experimental.constrained.fcmps.x86_fp80(
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f80(
                                                x86_fp80 %f1, x86_fp80 %f2, metadata !"ule",
                                                metadata !"fpexcept.strict") #0
   %res = select i1 %cond, i32 %a, i32 %b
@@ -998,7 +998,7 @@ define i32 @test_une_s(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 {
 ; X87-64-NEXT:    cmovnel %edi, %eax
 ; X87-64-NEXT:    cmovpl %edi, %eax
 ; X87-64-NEXT:    retq
-  %cond = call i1 @llvm.experimental.constrained.fcmps.x86_fp80(
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f80(
                                                x86_fp80 %f1, x86_fp80 %f2, metadata !"une",
                                                metadata !"fpexcept.strict") #0
   %res = select i1 %cond, i32 %a, i32 %b
@@ -1035,7 +1035,7 @@ define i32 @test_uno_s(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 {
 ; X87-64-NEXT:    wait
 ; X87-64-NEXT:    cmovnpl %esi, %eax
 ; X87-64-NEXT:    retq
-  %cond = call i1 @llvm.experimental.constrained.fcmps.x86_fp80(
+  %cond = call i1 @llvm.experimental.constrained.fcmps.f80(
                                                x86_fp80 %f1, x86_fp80 %f2, metadata !"uno",
                                                metadata !"fpexcept.strict") #0
   %res = select i1 %cond, i32 %a, i32 %b
@@ -1044,5 +1044,5 @@ define i32 @test_uno_s(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 {
 
 attributes #0 = { strictfp }
 
-declare i1 @llvm.experimental.constrained.fcmp.x86_fp80(x86_fp80, x86_fp80, metadata, metadata)
-declare i1 @llvm.experimental.constrained.fcmps.x86_fp80(x86_fp80, x86_fp80, metadata, metadata)
+declare i1 @llvm.experimental.constrained.fcmp.f80(x86_fp80, x86_fp80, metadata, metadata)
+declare i1 @llvm.experimental.constrained.fcmps.f80(x86_fp80, x86_fp80, metadata, metadata)
diff --git a/llvm/test/CodeGen/X86/fp80-strict-scalar.ll b/llvm/test/CodeGen/X86/fp80-strict-scalar.ll
index cf4a51fd6920a..221bebea2957e 100644
--- a/llvm/test/CodeGen/X86/fp80-strict-scalar.ll
+++ b/llvm/test/CodeGen/X86/fp80-strict-scalar.ll
@@ -2,35 +2,35 @@
 ; RUN: llc < %s -mtriple=i686-unknown-unknown -O3 | FileCheck %s --check-prefixes=CHECK,X86
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -O3 | FileCheck %s --check-prefixes=CHECK,X64
 
-declare x86_fp80 @llvm.experimental.constrained.fadd.x86_fp80(x86_fp80, x86_fp80, metadata, metadata)
-declare x86_fp80 @llvm.experimental.constrained.fsub.x86_fp80(x86_fp80, x86_fp80, metadata, metadata)
-declare x86_fp80 @llvm.experimental.constrained.fmul.x86_fp80(x86_fp80, x86_fp80, metadata, metadata)
-declare x86_fp80 @llvm.experimental.constrained.fdiv.x86_fp80(x86_fp80, x86_fp80, metadata, metadata)
-declare x86_fp80 @llvm.experimental.constrained.fpext.x86_fp80.f32(float, metadata)
-declare x86_fp80 @llvm.experimental.constrained.fpext.x86_fp80.f64(double, metadata)
-declare x86_fp80 @llvm.experimental.constrained.sqrt.x86_fp80(x86_fp80, metadata, metadata)
-declare float @llvm.experimental.constrained.fptrunc.f32.x86_fp80(x86_fp80, metadata, metadata)
-declare double @llvm.experimental.constrained.fptrunc.f64.x86_fp80(x86_fp80, metadata, metadata)
-declare i1  @llvm.experimental.constrained.fptosi.i1.x86_fp80(x86_fp80, metadata)
-declare i8  @llvm.experimental.constrained.fptosi.i8.x86_fp80(x86_fp80, metadata)
-declare i16 @llvm.experimental.constrained.fptosi.i16.x86_fp80(x86_fp80, metadata)
-declare i32 @llvm.experimental.constrained.fptosi.i32.x86_fp80(x86_fp80, metadata)
-declare i64 @llvm.experimental.constrained.fptosi.i64.x86_fp80(x86_fp80, metadata)
-declare i1 @llvm.experimental.constrained.fptoui.i1.x86_fp80(x86_fp80, metadata)
-declare i8  @llvm.experimental.constrained.fptoui.i8.x86_fp80(x86_fp80, metadata)
-declare i16 @llvm.experimental.constrained.fptoui.i16.x86_fp80(x86_fp80, metadata)
-declare i32 @llvm.experimental.constrained.fptoui.i32.x86_fp80(x86_fp80, metadata)
-declare i64 @llvm.experimental.constrained.fptoui.i64.x86_fp80(x86_fp80, metadata)
-declare x86_fp80 @llvm.experimental.constrained.sitofp.x86_fp80.i1(i1, metadata, metadata)
-declare x86_fp80 @llvm.experimental.constrained.sitofp.x86_fp80.i8(i8, metadata, metadata)
-declare x86_fp80 @llvm.experimental.constrained.sitofp.x86_fp80.i16(i16, metadata, metadata)
-declare x86_fp80 @llvm.experimental.constrained.sitofp.x86_fp80.i32(i32, metadata, metadata)
-declare x86_fp80 @llvm.experimental.constrained.sitofp.x86_fp80.i64(i64, metadata, metadata)
-declare x86_fp80 @llvm.experimental.constrained.uitofp.x86_fp80.i1(i1, metadata, metadata)
-declare x86_fp80 @llvm.experimental.constrained.uitofp.x86_fp80.i8(i8, metadata, metadata)
-declare x86_fp80 @llvm.experimental.constrained.uitofp.x86_fp80.i16(i16, metadata, metadata)
-declare x86_fp80 @llvm.experimental.constrained.uitofp.x86_fp80.i32(i32, metadata, metadata)
-declare x86_fp80 @llvm.experimental.constrained.uitofp.x86_fp80.i64(i64, metadata, metadata)
+declare x86_fp80 @llvm.experimental.constrained.fadd.f80(x86_fp80, x86_fp80, metadata, metadata)
+declare x86_fp80 @llvm.experimental.constrained.fsub.f80(x86_fp80, x86_fp80, metadata, metadata)
+declare x86_fp80 @llvm.experimental.constrained.fmul.f80(x86_fp80, x86_fp80, metadata, metadata)
+declare x86_fp80 @llvm.experimental.constrained.fdiv.f80(x86_fp80, x86_fp80, metadata, metadata)
+declare x86_fp80 @llvm.experimental.constrained.fpext.f80.f32(float, metadata)
+declare x86_fp80 @llvm.experimental.constrained.fpext.f80.f64(double, metadata)
+declare x86_fp80 @llvm.experimental.constrained.sqrt.f80(x86_fp80, metadata, metadata)
+declare float @llvm.experimental.constrained.fptrunc.f32.f80(x86_fp80, metadata, metadata)
+declare double @llvm.experimental.constrained.fptrunc.f64.f80(x86_fp80, metadata, metadata)
+declare i1  @llvm.experimental.constrained.fptosi.i1.f80(x86_fp80, metadata)
+declare i8  @llvm.experimental.constrained.fptosi.i8.f80(x86_fp80, metadata)
+declare i16 @llvm.experimental.constrained.fptosi.i16.f80(x86_fp80, metadata)
+declare i32 @llvm.experimental.constrained.fptosi.i32.f80(x86_fp80, metadata)
+declare i64 @llvm.experimental.constrained.fptosi.i64.f80(x86_fp80, metadata)
+declare i1 @llvm.experimental.constrained.fptoui.i1.f80(x86_fp80, metadata)
+declare i8  @llvm.experimental.constrained.fptoui.i8.f80(x86_fp80, metadata)
+declare i16 @llvm.experimental.constrained.fptoui.i16.f80(x86_fp80, metadata)
+declare i32 @llvm.experimental.constrained.fptoui.i32.f80(x86_fp80, metadata)
+declare i64 @llvm.experimental.constrained.fptoui.i64.f80(x86_fp80, metadata)
+declare x86_fp80 @llvm.experimental.constrained.sitofp.f80.i1(i1, metadata, metadata)
+declare x86_fp80 @llvm.experimental.constrained.sitofp.f80.i8(i8, metadata, metadata)
+declare x86_fp80 @llvm.experimental.constrained.sitofp.f80.i16(i16, metadata, metadata)
+declare x86_fp80 @llvm.experimental.constrained.sitofp.f80.i32(i32, metadata, metadata)
+declare x86_fp80 @llvm.experimental.constrained.sitofp.f80.i64(i64, metadata, metadata)
+declare x86_fp80 @llvm.experimental.constrained.uitofp.f80.i1(i1, metadata, metadata)
+declare x86_fp80 @llvm.experimental.constrained.uitofp.f80.i8(i8, metadata, metadata)
+declare x86_fp80 @llvm.experimental.constrained.uitofp.f80.i16(i16, metadata, metadata)
+declare x86_fp80 @llvm.experimental.constrained.uitofp.f80.i32(i32, metadata, metadata)
+declare x86_fp80 @llvm.experimental.constrained.uitofp.f80.i64(i64, metadata, metadata)
 
 define x86_fp80 @fadd_fp80(x86_fp80 %a, x86_fp80 %b) nounwind strictfp {
 ; X86-LABEL: fadd_fp80:
@@ -48,7 +48,7 @@ define x86_fp80 @fadd_fp80(x86_fp80 %a, x86_fp80 %b) nounwind strictfp {
 ; X64-NEXT:    faddp %st, %st(1)
 ; X64-NEXT:    wait
 ; X64-NEXT:    retq
-  %ret = call x86_fp80 @llvm.experimental.constrained.fadd.x86_fp80(x86_fp80 %a, x86_fp80 %b,
+  %ret = call x86_fp80 @llvm.experimental.constrained.fadd.f80(x86_fp80 %a, x86_fp80 %b,
                                                                     metadata !"round.dynamic",
                                                                     metadata !"fpexcept.strict") #0
   ret x86_fp80 %ret
@@ -70,7 +70,7 @@ define x86_fp80 @fsub_fp80(x86_fp80 %a, x86_fp80 %b) nounwind strictfp {
 ; X64-NEXT:    fsubp %st, %st(1)
 ; X64-NEXT:    wait
 ; X64-NEXT:    retq
-  %ret = call x86_fp80 @llvm.experimental.constrained.fsub.x86_fp80(x86_fp80 %a, x86_fp80 %b,
+  %ret = call x86_fp80 @llvm.experimental.constrained.fsub.f80(x86_fp80 %a, x86_fp80 %b,
                                                                     metadata !"round.dynamic",
                                                                     metadata !"fpexcept.strict") #0
   ret x86_fp80 %ret
@@ -92,7 +92,7 @@ define x86_fp80 @fmul_fp80(x86_fp80 %a, x86_fp80 %b) nounwind strictfp {
 ; X64-NEXT:    fmulp %st, %st(1)
 ; X64-NEXT:    wait
 ; X64-NEXT:    retq
-  %ret = call x86_fp80 @llvm.experimental.constrained.fmul.x86_fp80(x86_fp80 %a, x86_fp80 %b,
+  %ret = call x86_fp80 @llvm.experimental.constrained.fmul.f80(x86_fp80 %a, x86_fp80 %b,
                                                                     metadata !"round.dynamic",
                                                                     metadata !"fpexcept.strict") #0
   ret x86_fp80 %ret
@@ -114,7 +114,7 @@ define x86_fp80 @fdiv_fp80(x86_fp80 %a, x86_fp80 %b) nounwind strictfp {
 ; X64-NEXT:    fdivp %st, %st(1)
 ; X64-NEXT:    wait
 ; X64-NEXT:    retq
-  %ret = call x86_fp80 @llvm.experimental.constrained.fdiv.x86_fp80(x86_fp80 %a, x86_fp80 %b,
+  %ret = call x86_fp80 @llvm.experimental.constrained.fdiv.f80(x86_fp80 %a, x86_fp80 %b,
                                                                     metadata !"round.dynamic",
                                                                     metadata !"fpexcept.strict") #0
   ret x86_fp80 %ret
@@ -133,7 +133,7 @@ define x86_fp80 @fpext_f32_to_fp80(float %a) nounwind strictfp {
 ; X64-NEXT:    flds -{{[0-9]+}}(%rsp)
 ; X64-NEXT:    wait
 ; X64-NEXT:    retq
-  %ret = call x86_fp80 @llvm.experimental.constrained.fpext.x86_fp80.f32(float %a,
+  %ret = call x86_fp80 @llvm.experimental.constrained.fpext.f80.f32(float %a,
                                                                          metadata !"fpexcept.strict") #0
   ret x86_fp80 %ret
 }
@@ -151,7 +151,7 @@ define x86_fp80 @fpext_f64_to_fp80(double %a) nounwind strictfp {
 ; X64-NEXT:    fldl -{{[0-9]+}}(%rsp)
 ; X64-NEXT:    wait
 ; X64-NEXT:    retq
-  %ret = call x86_fp80 @llvm.experimental.constrained.fpext.x86_fp80.f64(double %a,
+  %ret = call x86_fp80 @llvm.experimental.constrained.fpext.f80.f64(double %a,
                                                                          metadata !"fpexcept.strict") #0
   ret x86_fp80 %ret
 }
@@ -174,7 +174,7 @@ define float @fptrunc_fp80_to_f32(x86_fp80 %a) nounwind strictfp {
 ; X64-NEXT:    wait
 ; X64-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X64-NEXT:    retq
-  %ret = call float @llvm.experimental.constrained.fptrunc.f32.x86_fp80(x86_fp80 %a,
+  %ret = call float @llvm.experimental.constrained.fptrunc.f32.f80(x86_fp80 %a,
                                                                         metadata !"round.dynamic",
                                                                         metadata !"fpexcept.strict") #0
   ret float %ret
@@ -202,7 +202,7 @@ define double @fptrunc_fp80_to_f64(x86_fp80 %a) nounwind strictfp {
 ; X64-NEXT:    wait
 ; X64-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; X64-NEXT:    retq
-  %ret = call double @llvm.experimental.constrained.fptrunc.f64.x86_fp80(x86_fp80 %a,
+  %ret = call double @llvm.experimental.constrained.fptrunc.f64.f80(x86_fp80 %a,
                                                                          metadata !"round.dynamic",
                                                                          metadata !"fpexcept.strict") #0
   ret double %ret
@@ -222,7 +222,7 @@ define x86_fp80 @fsqrt_fp80(x86_fp80 %a) nounwind strictfp {
 ; X64-NEXT:    fsqrt
 ; X64-NEXT:    wait
 ; X64-NEXT:    retq
-  %ret = call x86_fp80 @llvm.experimental.constrained.sqrt.x86_fp80(x86_fp80 %a,
+  %ret = call x86_fp80 @llvm.experimental.constrained.sqrt.f80(x86_fp80 %a,
                                                                     metadata !"round.dynamic",
                                                                     metadata !"fpexcept.strict") #0
   ret x86_fp80 %ret
@@ -260,7 +260,7 @@ define i1 @fp80_to_sint1(x86_fp80 %x) #0 {
 ; X64-NEXT:    fldcw -{{[0-9]+}}(%rsp)
 ; X64-NEXT:    movb -{{[0-9]+}}(%rsp), %al
 ; X64-NEXT:    retq
-  %result = call i1 @llvm.experimental.constrained.fptosi.i1.x86_fp80(x86_fp80 %x,
+  %result = call i1 @llvm.experimental.constrained.fptosi.i1.f80(x86_fp80 %x,
                                                metadata !"fpexcept.strict") #0
   ret i1 %result
 }
@@ -297,7 +297,7 @@ define i8 @fp80_to_sint8(x86_fp80 %x) #0 {
 ; X64-NEXT:    fldcw -{{[0-9]+}}(%rsp)
 ; X64-NEXT:    movb -{{[0-9]+}}(%rsp), %al
 ; X64-NEXT:    retq
-  %result = call i8 @llvm.experimental.constrained.fptosi.i8.x86_fp80(x86_fp80 %x,
+  %result = call i8 @llvm.experimental.constrained.fptosi.i8.f80(x86_fp80 %x,
                                                metadata !"fpexcept.strict") #0
   ret i8 %result
 }
@@ -334,7 +334,7 @@ define i16 @fp80_to_sint16(x86_fp80 %x) #0 {
 ; X64-NEXT:    fldcw -{{[0-9]+}}(%rsp)
 ; X64-NEXT:    movzwl -{{[0-9]+}}(%rsp), %eax
 ; X64-NEXT:    retq
-  %result = call i16 @llvm.experimental.constrained.fptosi.i16.x86_fp80(x86_fp80 %x,
+  %result = call i16 @llvm.experimental.constrained.fptosi.i16.f80(x86_fp80 %x,
                                                metadata !"fpexcept.strict") #0
   ret i16 %result
 }
@@ -372,7 +372,7 @@ define i32 @fp80_to_sint32(x86_fp80 %x) #0 {
 ; X64-NEXT:    movl -{{[0-9]+}}(%rsp), %eax
 ; X64-NEXT:    retq
 entry:
-  %result = call i32 @llvm.experimental.constrained.fptosi.i32.x86_fp80(x86_fp80 %x,
+  %result = call i32 @llvm.experimental.constrained.fptosi.i32.f80(x86_fp80 %x,
                                                metadata !"fpexcept.strict") #0
   ret i32 %result
 }
@@ -416,7 +416,7 @@ define i64 @fp80_to_sint64(x86_fp80 %x) #0 {
 ; X64-NEXT:    fldcw -{{[0-9]+}}(%rsp)
 ; X64-NEXT:    movq -{{[0-9]+}}(%rsp), %rax
 ; X64-NEXT:    retq
-  %result = call i64 @llvm.experimental.constrained.fptosi.i64.x86_fp80(x86_fp80 %x,
+  %result = call i64 @llvm.experimental.constrained.fptosi.i64.f80(x86_fp80 %x,
                                                metadata !"fpexcept.strict") #0
   ret i64 %result
 }
@@ -453,7 +453,7 @@ define i1 @fp80_to_uint1(x86_fp80 %x) #0 {
 ; X64-NEXT:    fldcw -{{[0-9]+}}(%rsp)
 ; X64-NEXT:    movb -{{[0-9]+}}(%rsp), %al
 ; X64-NEXT:    retq
-  %result = call i1 @llvm.experimental.constrained.fptoui.i1.x86_fp80(x86_fp80 %x,
+  %result = call i1 @llvm.experimental.constrained.fptoui.i1.f80(x86_fp80 %x,
                                                metadata !"fpexcept.strict") #0
   ret i1 %result
 }
@@ -490,7 +490,7 @@ define i8 @fp80_to_uint8(x86_fp80 %x) #0 {
 ; X64-NEXT:    fldcw -{{[0-9]+}}(%rsp)
 ; X64-NEXT:    movb -{{[0-9]+}}(%rsp), %al
 ; X64-NEXT:    retq
-  %result = call i8 @llvm.experimental.constrained.fptoui.i8.x86_fp80(x86_fp80 %x,
+  %result = call i8 @llvm.experimental.constrained.fptoui.i8.f80(x86_fp80 %x,
                                                metadata !"fpexcept.strict") #0
   ret i8 %result
 }
@@ -529,7 +529,7 @@ define i16 @fp80_to_uint16(x86_fp80 %x) #0 {
 ; X64-NEXT:    movl -{{[0-9]+}}(%rsp), %eax
 ; X64-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X64-NEXT:    retq
-  %result = call i16 @llvm.experimental.constrained.fptoui.i16.x86_fp80(x86_fp80 %x,
+  %result = call i16 @llvm.experimental.constrained.fptoui.i16.f80(x86_fp80 %x,
                                                metadata !"fpexcept.strict") #0
   ret i16 %result
 }
@@ -572,7 +572,7 @@ define i32 @fp80_to_uint32(x86_fp80 %x) #0 {
 ; X64-NEXT:    fldcw -{{[0-9]+}}(%rsp)
 ; X64-NEXT:    movl -{{[0-9]+}}(%rsp), %eax
 ; X64-NEXT:    retq
-  %result = call i32 @llvm.experimental.constrained.fptoui.i32.x86_fp80(x86_fp80 %x,
+  %result = call i32 @llvm.experimental.constrained.fptoui.i32.f80(x86_fp80 %x,
                                                metadata !"fpexcept.strict") #0
   ret i32 %result
 }
@@ -647,7 +647,7 @@ define i64 @fp80_to_uint64(x86_fp80 %x) #0 {
 ; X64-NEXT:    shlq $63, %rax
 ; X64-NEXT:    xorq -{{[0-9]+}}(%rsp), %rax
 ; X64-NEXT:    retq
-  %result = call i64 @llvm.experimental.constrained.fptoui.i64.x86_fp80(x86_fp80 %x,
+  %result = call i64 @llvm.experimental.constrained.fptoui.i64.f80(x86_fp80 %x,
                                                metadata !"fpexcept.strict") #0
   ret i64 %result
 }
@@ -677,7 +677,7 @@ define x86_fp80 @sint1_to_fp80(i1 %x) #0 {
 ; X64-NEXT:    filds -{{[0-9]+}}(%rsp)
 ; X64-NEXT:    wait
 ; X64-NEXT:    retq
-  %result = call x86_fp80 @llvm.experimental.constrained.sitofp.x86_fp80.i1(i1 %x,
+  %result = call x86_fp80 @llvm.experimental.constrained.sitofp.f80.i1(i1 %x,
                                                metadata !"round.dynamic",
                                                metadata !"fpexcept.strict") #0
   ret x86_fp80 %result
@@ -703,7 +703,7 @@ define x86_fp80 @sint8_to_fp80(i8 %x) #0 {
 ; X64-NEXT:    filds -{{[0-9]+}}(%rsp)
 ; X64-NEXT:    wait
 ; X64-NEXT:    retq
-  %result = call x86_fp80 @llvm.experimental.constrained.sitofp.x86_fp80.i8(i8 %x,
+  %result = call x86_fp80 @llvm.experimental.constrained.sitofp.f80.i8(i8 %x,
                                                metadata !"round.dynamic",
                                                metadata !"fpexcept.strict") #0
   ret x86_fp80 %result
@@ -728,7 +728,7 @@ define x86_fp80 @sint16_to_fp80(i16 %x) #0 {
 ; X64-NEXT:    filds -{{[0-9]+}}(%rsp)
 ; X64-NEXT:    wait
 ; X64-NEXT:    retq
-  %result = call x86_fp80 @llvm.experimental.constrained.sitofp.x86_fp80.i16(i16 %x,
+  %result = call x86_fp80 @llvm.experimental.constrained.sitofp.f80.i16(i16 %x,
                                                metadata !"round.dynamic",
                                                metadata !"fpexcept.strict") #0
   ret x86_fp80 %result
@@ -753,7 +753,7 @@ define x86_fp80 @sint32_to_fp80(i32 %x) #0 {
 ; X64-NEXT:    fildl -{{[0-9]+}}(%rsp)
 ; X64-NEXT:    wait
 ; X64-NEXT:    retq
-  %result = call x86_fp80 @llvm.experimental.constrained.sitofp.x86_fp80.i32(i32 %x,
+  %result = call x86_fp80 @llvm.experimental.constrained.sitofp.f80.i32(i32 %x,
                                                metadata !"round.dynamic",
                                                metadata !"fpexcept.strict") #0
   ret x86_fp80 %result
@@ -772,7 +772,7 @@ define x86_fp80 @sint64_to_fp80(i64 %x) #0 {
 ; X64-NEXT:    fildll -{{[0-9]+}}(%rsp)
 ; X64-NEXT:    wait
 ; X64-NEXT:    retq
-  %result = call x86_fp80 @llvm.experimental.constrained.sitofp.x86_fp80.i64(i64 %x,
+  %result = call x86_fp80 @llvm.experimental.constrained.sitofp.f80.i64(i64 %x,
                                                metadata !"round.dynamic",
                                                metadata !"fpexcept.strict") #0
   ret x86_fp80 %result
@@ -800,7 +800,7 @@ define x86_fp80 @uint1_to_fp80(i1 %x) #0 {
 ; X64-NEXT:    filds -{{[0-9]+}}(%rsp)
 ; X64-NEXT:    wait
 ; X64-NEXT:    retq
-  %result = call x86_fp80 @llvm.experimental.constrained.uitofp.x86_fp80.i1(i1 %x,
+  %result = call x86_fp80 @llvm.experimental.constrained.uitofp.f80.i1(i1 %x,
                                                metadata !"round.dynamic",
                                                metadata !"fpexcept.strict") #0
   ret x86_fp80 %result
@@ -826,7 +826,7 @@ define x86_fp80 @uint8_to_fp80(i8 %x) #0 {
 ; X64-NEXT:    filds -{{[0-9]+}}(%rsp)
 ; X64-NEXT:    wait
 ; X64-NEXT:    retq
-  %result = call x86_fp80 @llvm.experimental.constrained.uitofp.x86_fp80.i8(i8 %x,
+  %result = call x86_fp80 @llvm.experimental.constrained.uitofp.f80.i8(i8 %x,
                                                metadata !"round.dynamic",
                                                metadata !"fpexcept.strict") #0
   ret x86_fp80 %result
@@ -852,7 +852,7 @@ define x86_fp80 @uint16_to_fp80(i16 %x) #0 {
 ; X64-NEXT:    fildl -{{[0-9]+}}(%rsp)
 ; X64-NEXT:    wait
 ; X64-NEXT:    retq
-  %result = call x86_fp80 @llvm.experimental.constrained.uitofp.x86_fp80.i16(i16 %x,
+  %result = call x86_fp80 @llvm.experimental.constrained.uitofp.f80.i16(i16 %x,
                                                metadata !"round.dynamic",
                                                metadata !"fpexcept.strict") #0
   ret x86_fp80 %result
@@ -885,7 +885,7 @@ define x86_fp80 @uint32_to_fp80(i32 %x) #0 {
 ; X64-NEXT:    fildll -{{[0-9]+}}(%rsp)
 ; X64-NEXT:    wait
 ; X64-NEXT:    retq
-  %result = call x86_fp80 @llvm.experimental.constrained.uitofp.x86_fp80.i32(i32 %x,
+  %result = call x86_fp80 @llvm.experimental.constrained.uitofp.f80.i32(i32 %x,
                                                metadata !"round.dynamic",
                                                metadata !"fpexcept.strict") #0
   ret x86_fp80 %result
@@ -924,7 +924,7 @@ define x86_fp80 @uint64_to_fp80(i64 %x) #0 {
 ; X64-NEXT:    fadds {{\.LCPI.*}}(,%rax,4)
 ; X64-NEXT:    wait
 ; X64-NEXT:    retq
-  %result = call x86_fp80 @llvm.experimental.constrained.uitofp.x86_fp80.i64(i64 %x,
+  %result = call x86_fp80 @llvm.experimental.constrained.uitofp.f80.i64(i64 %x,
                                                metadata !"round.dynamic",
                                                metadata !"fpexcept.strict") #0
   ret x86_fp80 %result

From 9182dc78145b9f1505d7fcc34b818f6d8aabcfda Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@intel.com>
Date: Sat, 25 Jul 2020 13:24:58 -0700
Subject: [PATCH 0093/1035] [X86] Add llvm.roundeven test cases. Add f80 tests
 cases for constrained intrinsics that lower to libcalls. NFC

---
 llvm/test/CodeGen/X86/fp-cvt.ll               |   52 +
 llvm/test/CodeGen/X86/fp-roundeven.ll         | 1044 +++++++++++++++++
 .../CodeGen/X86/fp-strict-scalar-round.ll     |  184 +++
 .../test/CodeGen/X86/fp128-libcalls-strict.ll |   42 +
 llvm/test/CodeGen/X86/fp80-strict-libcalls.ll |  657 +++++++++++
 5 files changed, 1979 insertions(+)
 create mode 100644 llvm/test/CodeGen/X86/fp-roundeven.ll
 create mode 100644 llvm/test/CodeGen/X86/fp80-strict-libcalls.ll

diff --git a/llvm/test/CodeGen/X86/fp-cvt.ll b/llvm/test/CodeGen/X86/fp-cvt.ll
index 667c2d414ed1b..cedbfd2e9bff4 100644
--- a/llvm/test/CodeGen/X86/fp-cvt.ll
+++ b/llvm/test/CodeGen/X86/fp-cvt.ll
@@ -1090,3 +1090,55 @@ define x86_fp80 @rint_fp80_ld(x86_fp80 *%a0) nounwind {
 }
 
 declare x86_fp80 @llvm.rint.f80(x86_fp80 %p)
+
+;
+; roundeven
+;
+
+define x86_fp80 @roundeven_fp80(x86_fp80 %a0) nounwind {
+; X86-LABEL: roundeven_fp80:
+; X86:       # %bb.0:
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fstpt (%esp)
+; X86-NEXT:    calll roundevenl
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    retl
+;
+; X64-LABEL: roundeven_fp80:
+; X64:       # %bb.0:
+; X64-NEXT:    subq $24, %rsp
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fstpt (%rsp)
+; X64-NEXT:    callq roundevenl
+; X64-NEXT:    addq $24, %rsp
+; X64-NEXT:    retq
+  %1 = call x86_fp80 @llvm.roundeven.f80(x86_fp80 %a0)
+  ret x86_fp80 %1
+}
+
+define x86_fp80 @roundeven_fp80_ld(x86_fp80 *%a0) nounwind {
+; X86-LABEL: roundeven_fp80_ld:
+; X86:       # %bb.0:
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    fldt (%eax)
+; X86-NEXT:    fstpt (%esp)
+; X86-NEXT:    calll roundevenl
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    retl
+;
+; X64-LABEL: roundeven_fp80_ld:
+; X64:       # %bb.0:
+; X64-NEXT:    subq $24, %rsp
+; X64-NEXT:    fldt (%rdi)
+; X64-NEXT:    fstpt (%rsp)
+; X64-NEXT:    callq roundevenl
+; X64-NEXT:    addq $24, %rsp
+; X64-NEXT:    retq
+  %1 = load x86_fp80, x86_fp80 *%a0
+  %2 = call x86_fp80 @llvm.roundeven.f80(x86_fp80 %1)
+  ret x86_fp80 %2
+}
+
+declare x86_fp80 @llvm.roundeven.f80(x86_fp80 %p)
diff --git a/llvm/test/CodeGen/X86/fp-roundeven.ll b/llvm/test/CodeGen/X86/fp-roundeven.ll
new file mode 100644
index 0000000000000..a3eae0137f3ec
--- /dev/null
+++ b/llvm/test/CodeGen/X86/fp-roundeven.ll
@@ -0,0 +1,1044 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=+sse2 | FileCheck %s --check-prefix=SSE2
+; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=+sse4.1 | FileCheck %s --check-prefixes=SSE41
+; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=+avx | FileCheck %s --check-prefixes=AVX,AVX1
+; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=+avx512f | FileCheck %s --check-prefixes=AVX,AVX512
+
+define float @roundeven_f32(float %x) {
+; SSE2-LABEL: roundeven_f32:
+; SSE2:       ## %bb.0:
+; SSE2-NEXT:    jmp _roundevenf ## TAILCALL
+;
+; SSE41-LABEL: roundeven_f32:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    jmp _roundevenf ## TAILCALL
+;
+; AVX-LABEL: roundeven_f32:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    jmp _roundevenf ## TAILCALL
+  %a = call float @llvm.roundeven.f32(float %x)
+  ret float %a
+}
+
+define double @roundeven_f64(double %x) {
+; SSE2-LABEL: roundeven_f64:
+; SSE2:       ## %bb.0:
+; SSE2-NEXT:    jmp _roundeven ## TAILCALL
+;
+; SSE41-LABEL: roundeven_f64:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    jmp _roundeven ## TAILCALL
+;
+; AVX-LABEL: roundeven_f64:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    jmp _roundeven ## TAILCALL
+  %a = call double @llvm.roundeven.f64(double %x)
+  ret double %a
+}
+
+define <4 x float> @roundeven_v4f32(<4 x float> %x) {
+; SSE2-LABEL: roundeven_v4f32:
+; SSE2:       ## %bb.0:
+; SSE2-NEXT:    subq $56, %rsp
+; SSE2-NEXT:    .cfi_def_cfa_offset 64
+; SSE2-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    callq _roundevenf
+; SSE2-NEXT:    movaps %xmm0, (%rsp) ## 16-byte Spill
+; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
+; SSE2-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; SSE2-NEXT:    callq _roundevenf
+; SSE2-NEXT:    unpcklps (%rsp), %xmm0 ## 16-byte Folded Reload
+; SSE2-NEXT:    ## xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
+; SSE2-NEXT:    movaps %xmm0, (%rsp) ## 16-byte Spill
+; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
+; SSE2-NEXT:    callq _roundevenf
+; SSE2-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE2-NEXT:    callq _roundevenf
+; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
+; SSE2-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT:    unpcklpd (%rsp), %xmm1 ## 16-byte Folded Reload
+; SSE2-NEXT:    ## xmm1 = xmm1[0],mem[0]
+; SSE2-NEXT:    movaps %xmm1, %xmm0
+; SSE2-NEXT:    addq $56, %rsp
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: roundeven_v4f32:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    subq $40, %rsp
+; SSE41-NEXT:    .cfi_def_cfa_offset 48
+; SSE41-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE41-NEXT:    callq _roundevenf
+; SSE41-NEXT:    movaps %xmm0, (%rsp) ## 16-byte Spill
+; SSE41-NEXT:    movshdup {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
+; SSE41-NEXT:    ## xmm0 = mem[1,1,3,3]
+; SSE41-NEXT:    callq _roundevenf
+; SSE41-NEXT:    movaps (%rsp), %xmm1 ## 16-byte Reload
+; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[2,3]
+; SSE41-NEXT:    movaps %xmm1, (%rsp) ## 16-byte Spill
+; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
+; SSE41-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; SSE41-NEXT:    callq _roundevenf
+; SSE41-NEXT:    movaps (%rsp), %xmm1 ## 16-byte Reload
+; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0,1],xmm0[0],xmm1[3]
+; SSE41-NEXT:    movaps %xmm1, (%rsp) ## 16-byte Spill
+; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
+; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE41-NEXT:    callq _roundevenf
+; SSE41-NEXT:    movaps (%rsp), %xmm1 ## 16-byte Reload
+; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm0[0]
+; SSE41-NEXT:    movaps %xmm1, %xmm0
+; SSE41-NEXT:    addq $40, %rsp
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: roundeven_v4f32:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    subq $40, %rsp
+; AVX-NEXT:    .cfi_def_cfa_offset 48
+; AVX-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; AVX-NEXT:    callq _roundevenf
+; AVX-NEXT:    vmovaps %xmm0, (%rsp) ## 16-byte Spill
+; AVX-NEXT:    vmovshdup {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
+; AVX-NEXT:    ## xmm0 = mem[1,1,3,3]
+; AVX-NEXT:    callq _roundevenf
+; AVX-NEXT:    vmovaps (%rsp), %xmm1 ## 16-byte Reload
+; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3]
+; AVX-NEXT:    vmovaps %xmm0, (%rsp) ## 16-byte Spill
+; AVX-NEXT:    vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
+; AVX-NEXT:    ## xmm0 = mem[1,0]
+; AVX-NEXT:    callq _roundevenf
+; AVX-NEXT:    vmovaps (%rsp), %xmm1 ## 16-byte Reload
+; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
+; AVX-NEXT:    vmovaps %xmm0, (%rsp) ## 16-byte Spill
+; AVX-NEXT:    vpermilps $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
+; AVX-NEXT:    ## xmm0 = mem[3,1,2,3]
+; AVX-NEXT:    callq _roundevenf
+; AVX-NEXT:    vmovaps (%rsp), %xmm1 ## 16-byte Reload
+; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
+; AVX-NEXT:    addq $40, %rsp
+; AVX-NEXT:    retq
+  %a = call <4 x float> @llvm.roundeven.v4f32(<4 x float> %x)
+  ret <4 x float> %a
+}
+
+define <2 x double> @roundeven_v2f64(<2 x double> %x) {
+; SSE2-LABEL: roundeven_v2f64:
+; SSE2:       ## %bb.0:
+; SSE2-NEXT:    subq $40, %rsp
+; SSE2-NEXT:    .cfi_def_cfa_offset 48
+; SSE2-NEXT:    movaps %xmm0, (%rsp) ## 16-byte Spill
+; SSE2-NEXT:    callq _roundeven
+; SSE2-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE2-NEXT:    movaps (%rsp), %xmm0 ## 16-byte Reload
+; SSE2-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; SSE2-NEXT:    callq _roundeven
+; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
+; SSE2-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE2-NEXT:    movaps %xmm1, %xmm0
+; SSE2-NEXT:    addq $40, %rsp
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: roundeven_v2f64:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    subq $40, %rsp
+; SSE41-NEXT:    .cfi_def_cfa_offset 48
+; SSE41-NEXT:    movaps %xmm0, (%rsp) ## 16-byte Spill
+; SSE41-NEXT:    callq _roundeven
+; SSE41-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE41-NEXT:    movaps (%rsp), %xmm0 ## 16-byte Reload
+; SSE41-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; SSE41-NEXT:    callq _roundeven
+; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
+; SSE41-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE41-NEXT:    movaps %xmm1, %xmm0
+; SSE41-NEXT:    addq $40, %rsp
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: roundeven_v2f64:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    subq $40, %rsp
+; AVX-NEXT:    .cfi_def_cfa_offset 48
+; AVX-NEXT:    vmovaps %xmm0, (%rsp) ## 16-byte Spill
+; AVX-NEXT:    callq _roundeven
+; AVX-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; AVX-NEXT:    vpermilpd $1, (%rsp), %xmm0 ## 16-byte Folded Reload
+; AVX-NEXT:    ## xmm0 = mem[1,0]
+; AVX-NEXT:    callq _roundeven
+; AVX-NEXT:    vmovapd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
+; AVX-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX-NEXT:    addq $40, %rsp
+; AVX-NEXT:    retq
+  %a = call <2 x double> @llvm.roundeven.v2f64(<2 x double> %x)
+  ret <2 x double> %a
+}
+
+define <8 x float> @roundeven_v8f32(<8 x float> %x) {
+; SSE2-LABEL: roundeven_v8f32:
+; SSE2:       ## %bb.0:
+; SSE2-NEXT:    subq $72, %rsp
+; SSE2-NEXT:    .cfi_def_cfa_offset 80
+; SSE2-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE2-NEXT:    movaps %xmm0, (%rsp) ## 16-byte Spill
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    callq _roundevenf
+; SSE2-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE2-NEXT:    movaps (%rsp), %xmm0 ## 16-byte Reload
+; SSE2-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; SSE2-NEXT:    callq _roundevenf
+; SSE2-NEXT:    unpcklps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
+; SSE2-NEXT:    ## xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
+; SSE2-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE2-NEXT:    movaps (%rsp), %xmm0 ## 16-byte Reload
+; SSE2-NEXT:    callq _roundevenf
+; SSE2-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE2-NEXT:    movaps (%rsp), %xmm0 ## 16-byte Reload
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE2-NEXT:    callq _roundevenf
+; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
+; SSE2-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT:    unpcklpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Folded Reload
+; SSE2-NEXT:    ## xmm1 = xmm1[0],mem[0]
+; SSE2-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    callq _roundevenf
+; SSE2-NEXT:    movaps %xmm0, (%rsp) ## 16-byte Spill
+; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
+; SSE2-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; SSE2-NEXT:    callq _roundevenf
+; SSE2-NEXT:    unpcklps (%rsp), %xmm0 ## 16-byte Folded Reload
+; SSE2-NEXT:    ## xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
+; SSE2-NEXT:    movaps %xmm0, (%rsp) ## 16-byte Spill
+; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
+; SSE2-NEXT:    callq _roundevenf
+; SSE2-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE2-NEXT:    callq _roundevenf
+; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
+; SSE2-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT:    unpcklpd (%rsp), %xmm1 ## 16-byte Folded Reload
+; SSE2-NEXT:    ## xmm1 = xmm1[0],mem[0]
+; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
+; SSE2-NEXT:    addq $72, %rsp
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: roundeven_v8f32:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    subq $56, %rsp
+; SSE41-NEXT:    .cfi_def_cfa_offset 64
+; SSE41-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE41-NEXT:    movaps %xmm0, (%rsp) ## 16-byte Spill
+; SSE41-NEXT:    callq _roundevenf
+; SSE41-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE41-NEXT:    movshdup (%rsp), %xmm0 ## 16-byte Folded Reload
+; SSE41-NEXT:    ## xmm0 = mem[1,1,3,3]
+; SSE41-NEXT:    callq _roundevenf
+; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
+; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[2,3]
+; SSE41-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE41-NEXT:    movaps (%rsp), %xmm0 ## 16-byte Reload
+; SSE41-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; SSE41-NEXT:    callq _roundevenf
+; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
+; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0,1],xmm0[0],xmm1[3]
+; SSE41-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE41-NEXT:    movaps (%rsp), %xmm0 ## 16-byte Reload
+; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE41-NEXT:    callq _roundevenf
+; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
+; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm0[0]
+; SSE41-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
+; SSE41-NEXT:    callq _roundevenf
+; SSE41-NEXT:    movaps %xmm0, (%rsp) ## 16-byte Spill
+; SSE41-NEXT:    movshdup {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
+; SSE41-NEXT:    ## xmm0 = mem[1,1,3,3]
+; SSE41-NEXT:    callq _roundevenf
+; SSE41-NEXT:    movaps (%rsp), %xmm1 ## 16-byte Reload
+; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[2,3]
+; SSE41-NEXT:    movaps %xmm1, (%rsp) ## 16-byte Spill
+; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
+; SSE41-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; SSE41-NEXT:    callq _roundevenf
+; SSE41-NEXT:    movaps (%rsp), %xmm1 ## 16-byte Reload
+; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0,1],xmm0[0],xmm1[3]
+; SSE41-NEXT:    movaps %xmm1, (%rsp) ## 16-byte Spill
+; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
+; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE41-NEXT:    callq _roundevenf
+; SSE41-NEXT:    movaps (%rsp), %xmm1 ## 16-byte Reload
+; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm0[0]
+; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
+; SSE41-NEXT:    addq $56, %rsp
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: roundeven_v8f32:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    subq $88, %rsp
+; AVX-NEXT:    .cfi_def_cfa_offset 96
+; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 32-byte Spill
+; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; AVX-NEXT:    vzeroupper
+; AVX-NEXT:    callq _roundevenf
+; AVX-NEXT:    vmovaps %xmm0, (%rsp) ## 16-byte Spill
+; AVX-NEXT:    vmovshdup {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
+; AVX-NEXT:    ## xmm0 = mem[1,1,3,3]
+; AVX-NEXT:    callq _roundevenf
+; AVX-NEXT:    vmovaps (%rsp), %xmm1 ## 16-byte Reload
+; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3]
+; AVX-NEXT:    vmovaps %xmm0, (%rsp) ## 16-byte Spill
+; AVX-NEXT:    vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
+; AVX-NEXT:    ## xmm0 = mem[1,0]
+; AVX-NEXT:    callq _roundevenf
+; AVX-NEXT:    vmovaps (%rsp), %xmm1 ## 16-byte Reload
+; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
+; AVX-NEXT:    vmovaps %xmm0, (%rsp) ## 16-byte Spill
+; AVX-NEXT:    vpermilps $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
+; AVX-NEXT:    ## xmm0 = mem[3,1,2,3]
+; AVX-NEXT:    callq _roundevenf
+; AVX-NEXT:    vmovaps (%rsp), %xmm1 ## 16-byte Reload
+; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
+; AVX-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 ## 32-byte Reload
+; AVX-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX-NEXT:    vzeroupper
+; AVX-NEXT:    callq _roundevenf
+; AVX-NEXT:    vmovaps %xmm0, (%rsp) ## 16-byte Spill
+; AVX-NEXT:    vmovshdup {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
+; AVX-NEXT:    ## xmm0 = mem[1,1,3,3]
+; AVX-NEXT:    callq _roundevenf
+; AVX-NEXT:    vmovaps (%rsp), %xmm1 ## 16-byte Reload
+; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3]
+; AVX-NEXT:    vmovaps %xmm0, (%rsp) ## 16-byte Spill
+; AVX-NEXT:    vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
+; AVX-NEXT:    ## xmm0 = mem[1,0]
+; AVX-NEXT:    callq _roundevenf
+; AVX-NEXT:    vmovaps (%rsp), %xmm1 ## 16-byte Reload
+; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
+; AVX-NEXT:    vmovaps %xmm0, (%rsp) ## 16-byte Spill
+; AVX-NEXT:    vpermilps $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
+; AVX-NEXT:    ## xmm0 = mem[3,1,2,3]
+; AVX-NEXT:    callq _roundevenf
+; AVX-NEXT:    vmovaps (%rsp), %xmm1 ## 16-byte Reload
+; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
+; AVX-NEXT:    vinsertf128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 ## 16-byte Folded Reload
+; AVX-NEXT:    addq $88, %rsp
+; AVX-NEXT:    retq
+  %a = call <8 x float> @llvm.roundeven.v8f32(<8 x float> %x)
+  ret <8 x float> %a
+}
+
+define <4 x double> @roundeven_v4f64(<4 x double> %x) {
+; SSE2-LABEL: roundeven_v4f64:
+; SSE2:       ## %bb.0:
+; SSE2-NEXT:    subq $56, %rsp
+; SSE2-NEXT:    .cfi_def_cfa_offset 64
+; SSE2-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE2-NEXT:    movaps %xmm0, (%rsp) ## 16-byte Spill
+; SSE2-NEXT:    callq _roundeven
+; SSE2-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE2-NEXT:    movaps (%rsp), %xmm0 ## 16-byte Reload
+; SSE2-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; SSE2-NEXT:    callq _roundeven
+; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
+; SSE2-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE2-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
+; SSE2-NEXT:    callq _roundeven
+; SSE2-NEXT:    movaps %xmm0, (%rsp) ## 16-byte Spill
+; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
+; SSE2-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; SSE2-NEXT:    callq _roundeven
+; SSE2-NEXT:    movaps (%rsp), %xmm1 ## 16-byte Reload
+; SSE2-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
+; SSE2-NEXT:    addq $56, %rsp
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: roundeven_v4f64:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    subq $56, %rsp
+; SSE41-NEXT:    .cfi_def_cfa_offset 64
+; SSE41-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE41-NEXT:    movaps %xmm0, (%rsp) ## 16-byte Spill
+; SSE41-NEXT:    callq _roundeven
+; SSE41-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE41-NEXT:    movaps (%rsp), %xmm0 ## 16-byte Reload
+; SSE41-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; SSE41-NEXT:    callq _roundeven
+; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
+; SSE41-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE41-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
+; SSE41-NEXT:    callq _roundeven
+; SSE41-NEXT:    movaps %xmm0, (%rsp) ## 16-byte Spill
+; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
+; SSE41-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; SSE41-NEXT:    callq _roundeven
+; SSE41-NEXT:    movaps (%rsp), %xmm1 ## 16-byte Reload
+; SSE41-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
+; SSE41-NEXT:    addq $56, %rsp
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: roundeven_v4f64:
+; AVX:       ## %bb.0:
+; AVX-NEXT:    subq $88, %rsp
+; AVX-NEXT:    .cfi_def_cfa_offset 96
+; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 32-byte Spill
+; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX-NEXT:    vmovaps %xmm0, (%rsp) ## 16-byte Spill
+; AVX-NEXT:    vzeroupper
+; AVX-NEXT:    callq _roundeven
+; AVX-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; AVX-NEXT:    vpermilpd $1, (%rsp), %xmm0 ## 16-byte Folded Reload
+; AVX-NEXT:    ## xmm0 = mem[1,0]
+; AVX-NEXT:    callq _roundeven
+; AVX-NEXT:    vmovapd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
+; AVX-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX-NEXT:    vmovapd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 ## 32-byte Reload
+; AVX-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX-NEXT:    vzeroupper
+; AVX-NEXT:    callq _roundeven
+; AVX-NEXT:    vmovaps %xmm0, (%rsp) ## 16-byte Spill
+; AVX-NEXT:    vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
+; AVX-NEXT:    ## xmm0 = mem[1,0]
+; AVX-NEXT:    callq _roundeven
+; AVX-NEXT:    vmovapd (%rsp), %xmm1 ## 16-byte Reload
+; AVX-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX-NEXT:    vinsertf128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 ## 16-byte Folded Reload
+; AVX-NEXT:    addq $88, %rsp
+; AVX-NEXT:    retq
+  %a = call <4 x double> @llvm.roundeven.v4f64(<4 x double> %x)
+  ret <4 x double> %a
+}
+
+define <16 x float> @roundeven_v16f32(<16 x float> %x) {
+; SSE2-LABEL: roundeven_v16f32:
+; SSE2:       ## %bb.0:
+; SSE2-NEXT:    subq $104, %rsp
+; SSE2-NEXT:    .cfi_def_cfa_offset 112
+; SSE2-NEXT:    movaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE2-NEXT:    movaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE2-NEXT:    movaps %xmm1, (%rsp) ## 16-byte Spill
+; SSE2-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    callq _roundevenf
+; SSE2-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
+; SSE2-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; SSE2-NEXT:    callq _roundevenf
+; SSE2-NEXT:    unpcklps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
+; SSE2-NEXT:    ## xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
+; SSE2-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
+; SSE2-NEXT:    callq _roundevenf
+; SSE2-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE2-NEXT:    callq _roundevenf
+; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
+; SSE2-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT:    unpcklpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Folded Reload
+; SSE2-NEXT:    ## xmm1 = xmm1[0],mem[0]
+; SSE2-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE2-NEXT:    movaps (%rsp), %xmm0 ## 16-byte Reload
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    callq _roundevenf
+; SSE2-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE2-NEXT:    movaps (%rsp), %xmm0 ## 16-byte Reload
+; SSE2-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; SSE2-NEXT:    callq _roundevenf
+; SSE2-NEXT:    unpcklps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
+; SSE2-NEXT:    ## xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
+; SSE2-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE2-NEXT:    movaps (%rsp), %xmm0 ## 16-byte Reload
+; SSE2-NEXT:    callq _roundevenf
+; SSE2-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE2-NEXT:    movaps (%rsp), %xmm0 ## 16-byte Reload
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE2-NEXT:    callq _roundevenf
+; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
+; SSE2-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT:    unpcklpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Folded Reload
+; SSE2-NEXT:    ## xmm1 = xmm1[0],mem[0]
+; SSE2-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    callq _roundevenf
+; SSE2-NEXT:    movaps %xmm0, (%rsp) ## 16-byte Spill
+; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
+; SSE2-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; SSE2-NEXT:    callq _roundevenf
+; SSE2-NEXT:    unpcklps (%rsp), %xmm0 ## 16-byte Folded Reload
+; SSE2-NEXT:    ## xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
+; SSE2-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
+; SSE2-NEXT:    callq _roundevenf
+; SSE2-NEXT:    movaps %xmm0, (%rsp) ## 16-byte Spill
+; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE2-NEXT:    callq _roundevenf
+; SSE2-NEXT:    movaps (%rsp), %xmm1 ## 16-byte Reload
+; SSE2-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT:    unpcklpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Folded Reload
+; SSE2-NEXT:    ## xmm1 = xmm1[0],mem[0]
+; SSE2-NEXT:    movaps %xmm1, (%rsp) ## 16-byte Spill
+; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    callq _roundevenf
+; SSE2-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
+; SSE2-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; SSE2-NEXT:    callq _roundevenf
+; SSE2-NEXT:    unpcklps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
+; SSE2-NEXT:    ## xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
+; SSE2-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
+; SSE2-NEXT:    callq _roundevenf
+; SSE2-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE2-NEXT:    callq _roundevenf
+; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 ## 16-byte Reload
+; SSE2-NEXT:    unpcklps {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1]
+; SSE2-NEXT:    unpcklpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 ## 16-byte Folded Reload
+; SSE2-NEXT:    ## xmm3 = xmm3[0],mem[0]
+; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
+; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
+; SSE2-NEXT:    movaps (%rsp), %xmm2 ## 16-byte Reload
+; SSE2-NEXT:    addq $104, %rsp
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: roundeven_v16f32:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    subq $88, %rsp
+; SSE41-NEXT:    .cfi_def_cfa_offset 96
+; SSE41-NEXT:    movaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE41-NEXT:    movaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE41-NEXT:    movaps %xmm1, (%rsp) ## 16-byte Spill
+; SSE41-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE41-NEXT:    callq _roundevenf
+; SSE41-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE41-NEXT:    movshdup {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
+; SSE41-NEXT:    ## xmm0 = mem[1,1,3,3]
+; SSE41-NEXT:    callq _roundevenf
+; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
+; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[2,3]
+; SSE41-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
+; SSE41-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; SSE41-NEXT:    callq _roundevenf
+; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
+; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0,1],xmm0[0],xmm1[3]
+; SSE41-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
+; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE41-NEXT:    callq _roundevenf
+; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
+; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm0[0]
+; SSE41-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE41-NEXT:    movaps (%rsp), %xmm0 ## 16-byte Reload
+; SSE41-NEXT:    callq _roundevenf
+; SSE41-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE41-NEXT:    movshdup (%rsp), %xmm0 ## 16-byte Folded Reload
+; SSE41-NEXT:    ## xmm0 = mem[1,1,3,3]
+; SSE41-NEXT:    callq _roundevenf
+; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
+; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[2,3]
+; SSE41-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE41-NEXT:    movaps (%rsp), %xmm0 ## 16-byte Reload
+; SSE41-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; SSE41-NEXT:    callq _roundevenf
+; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
+; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0,1],xmm0[0],xmm1[3]
+; SSE41-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE41-NEXT:    movaps (%rsp), %xmm0 ## 16-byte Reload
+; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE41-NEXT:    callq _roundevenf
+; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
+; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm0[0]
+; SSE41-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
+; SSE41-NEXT:    callq _roundevenf
+; SSE41-NEXT:    movaps %xmm0, (%rsp) ## 16-byte Spill
+; SSE41-NEXT:    movshdup {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
+; SSE41-NEXT:    ## xmm0 = mem[1,1,3,3]
+; SSE41-NEXT:    callq _roundevenf
+; SSE41-NEXT:    movaps (%rsp), %xmm1 ## 16-byte Reload
+; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[2,3]
+; SSE41-NEXT:    movaps %xmm1, (%rsp) ## 16-byte Spill
+; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
+; SSE41-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; SSE41-NEXT:    callq _roundevenf
+; SSE41-NEXT:    movaps (%rsp), %xmm1 ## 16-byte Reload
+; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0,1],xmm0[0],xmm1[3]
+; SSE41-NEXT:    movaps %xmm1, (%rsp) ## 16-byte Spill
+; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
+; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE41-NEXT:    callq _roundevenf
+; SSE41-NEXT:    movaps (%rsp), %xmm1 ## 16-byte Reload
+; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm0[0]
+; SSE41-NEXT:    movaps %xmm1, (%rsp) ## 16-byte Spill
+; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
+; SSE41-NEXT:    callq _roundevenf
+; SSE41-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE41-NEXT:    movshdup {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
+; SSE41-NEXT:    ## xmm0 = mem[1,1,3,3]
+; SSE41-NEXT:    callq _roundevenf
+; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
+; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[2,3]
+; SSE41-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
+; SSE41-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; SSE41-NEXT:    callq _roundevenf
+; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
+; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0,1],xmm0[0],xmm1[3]
+; SSE41-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
+; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE41-NEXT:    callq _roundevenf
+; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 ## 16-byte Reload
+; SSE41-NEXT:    insertps {{.*#+}} xmm3 = xmm3[0,1,2],xmm0[0]
+; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
+; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
+; SSE41-NEXT:    movaps (%rsp), %xmm2 ## 16-byte Reload
+; SSE41-NEXT:    addq $88, %rsp
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: roundeven_v16f32:
+; AVX1:       ## %bb.0:
+; AVX1-NEXT:    subq $152, %rsp
+; AVX1-NEXT:    .cfi_def_cfa_offset 160
+; AVX1-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) ## 32-byte Spill
+; AVX1-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 32-byte Spill
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    callq _roundevenf
+; AVX1-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; AVX1-NEXT:    vmovshdup {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
+; AVX1-NEXT:    ## xmm0 = mem[1,1,3,3]
+; AVX1-NEXT:    callq _roundevenf
+; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3]
+; AVX1-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; AVX1-NEXT:    vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
+; AVX1-NEXT:    ## xmm0 = mem[1,0]
+; AVX1-NEXT:    callq _roundevenf
+; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
+; AVX1-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; AVX1-NEXT:    vpermilps $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
+; AVX1-NEXT:    ## xmm0 = mem[3,1,2,3]
+; AVX1-NEXT:    callq _roundevenf
+; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
+; AVX1-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; AVX1-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 ## 32-byte Reload
+; AVX1-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    callq _roundevenf
+; AVX1-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; AVX1-NEXT:    vmovshdup {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
+; AVX1-NEXT:    ## xmm0 = mem[1,1,3,3]
+; AVX1-NEXT:    callq _roundevenf
+; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3]
+; AVX1-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; AVX1-NEXT:    vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
+; AVX1-NEXT:    ## xmm0 = mem[1,0]
+; AVX1-NEXT:    callq _roundevenf
+; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
+; AVX1-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; AVX1-NEXT:    vpermilps $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
+; AVX1-NEXT:    ## xmm0 = mem[3,1,2,3]
+; AVX1-NEXT:    callq _roundevenf
+; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
+; AVX1-NEXT:    vinsertf128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 ## 16-byte Folded Reload
+; AVX1-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 32-byte Spill
+; AVX1-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 ## 32-byte Reload
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    callq _roundevenf
+; AVX1-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; AVX1-NEXT:    vmovshdup {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
+; AVX1-NEXT:    ## xmm0 = mem[1,1,3,3]
+; AVX1-NEXT:    callq _roundevenf
+; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3]
+; AVX1-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; AVX1-NEXT:    vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
+; AVX1-NEXT:    ## xmm0 = mem[1,0]
+; AVX1-NEXT:    callq _roundevenf
+; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
+; AVX1-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; AVX1-NEXT:    vpermilps $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
+; AVX1-NEXT:    ## xmm0 = mem[3,1,2,3]
+; AVX1-NEXT:    callq _roundevenf
+; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
+; AVX1-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; AVX1-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 ## 32-byte Reload
+; AVX1-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    callq _roundevenf
+; AVX1-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; AVX1-NEXT:    vmovshdup {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
+; AVX1-NEXT:    ## xmm0 = mem[1,1,3,3]
+; AVX1-NEXT:    callq _roundevenf
+; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3]
+; AVX1-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; AVX1-NEXT:    vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
+; AVX1-NEXT:    ## xmm0 = mem[1,0]
+; AVX1-NEXT:    callq _roundevenf
+; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
+; AVX1-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; AVX1-NEXT:    vpermilps $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
+; AVX1-NEXT:    ## xmm0 = mem[3,1,2,3]
+; AVX1-NEXT:    callq _roundevenf
+; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
+; AVX1-NEXT:    vinsertf128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 ## 16-byte Folded Reload
+; AVX1-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 ## 32-byte Reload
+; AVX1-NEXT:    addq $152, %rsp
+; AVX1-NEXT:    retq
+;
+; AVX512-LABEL: roundeven_v16f32:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    subq $184, %rsp
+; AVX512-NEXT:    .cfi_def_cfa_offset 192
+; AVX512-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 64-byte Spill
+; AVX512-NEXT:    vextractf32x4 $3, %zmm0, %xmm0
+; AVX512-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    callq _roundevenf
+; AVX512-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; AVX512-NEXT:    vmovshdup {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
+; AVX512-NEXT:    ## xmm0 = mem[1,1,3,3]
+; AVX512-NEXT:    callq _roundevenf
+; AVX512-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
+; AVX512-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3]
+; AVX512-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; AVX512-NEXT:    vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
+; AVX512-NEXT:    ## xmm0 = mem[1,0]
+; AVX512-NEXT:    callq _roundevenf
+; AVX512-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
+; AVX512-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
+; AVX512-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; AVX512-NEXT:    vpermilps $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
+; AVX512-NEXT:    ## xmm0 = mem[3,1,2,3]
+; AVX512-NEXT:    callq _roundevenf
+; AVX512-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
+; AVX512-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
+; AVX512-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; AVX512-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 ## 64-byte Reload
+; AVX512-NEXT:    vextractf32x4 $2, %zmm0, %xmm0
+; AVX512-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    callq _roundevenf
+; AVX512-NEXT:    vmovaps %xmm0, (%rsp) ## 16-byte Spill
+; AVX512-NEXT:    vmovshdup {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
+; AVX512-NEXT:    ## xmm0 = mem[1,1,3,3]
+; AVX512-NEXT:    callq _roundevenf
+; AVX512-NEXT:    vmovaps (%rsp), %xmm1 ## 16-byte Reload
+; AVX512-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3]
+; AVX512-NEXT:    vmovaps %xmm0, (%rsp) ## 16-byte Spill
+; AVX512-NEXT:    vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
+; AVX512-NEXT:    ## xmm0 = mem[1,0]
+; AVX512-NEXT:    callq _roundevenf
+; AVX512-NEXT:    vmovaps (%rsp), %xmm1 ## 16-byte Reload
+; AVX512-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
+; AVX512-NEXT:    vmovaps %xmm0, (%rsp) ## 16-byte Spill
+; AVX512-NEXT:    vpermilps $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
+; AVX512-NEXT:    ## xmm0 = mem[3,1,2,3]
+; AVX512-NEXT:    callq _roundevenf
+; AVX512-NEXT:    vmovaps (%rsp), %xmm1 ## 16-byte Reload
+; AVX512-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
+; AVX512-NEXT:    vinsertf128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 ## 16-byte Folded Reload
+; AVX512-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 32-byte Spill
+; AVX512-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 ## 64-byte Reload
+; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX512-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    callq _roundevenf
+; AVX512-NEXT:    vmovaps %xmm0, (%rsp) ## 16-byte Spill
+; AVX512-NEXT:    vmovshdup {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
+; AVX512-NEXT:    ## xmm0 = mem[1,1,3,3]
+; AVX512-NEXT:    callq _roundevenf
+; AVX512-NEXT:    vmovaps (%rsp), %xmm1 ## 16-byte Reload
+; AVX512-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3]
+; AVX512-NEXT:    vmovaps %xmm0, (%rsp) ## 16-byte Spill
+; AVX512-NEXT:    vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
+; AVX512-NEXT:    ## xmm0 = mem[1,0]
+; AVX512-NEXT:    callq _roundevenf
+; AVX512-NEXT:    vmovaps (%rsp), %xmm1 ## 16-byte Reload
+; AVX512-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
+; AVX512-NEXT:    vmovaps %xmm0, (%rsp) ## 16-byte Spill
+; AVX512-NEXT:    vpermilps $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
+; AVX512-NEXT:    ## xmm0 = mem[3,1,2,3]
+; AVX512-NEXT:    callq _roundevenf
+; AVX512-NEXT:    vmovaps (%rsp), %xmm1 ## 16-byte Reload
+; AVX512-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
+; AVX512-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; AVX512-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 ## 64-byte Reload
+; AVX512-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    callq _roundevenf
+; AVX512-NEXT:    vmovaps %xmm0, (%rsp) ## 16-byte Spill
+; AVX512-NEXT:    vmovshdup {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
+; AVX512-NEXT:    ## xmm0 = mem[1,1,3,3]
+; AVX512-NEXT:    callq _roundevenf
+; AVX512-NEXT:    vmovaps (%rsp), %xmm1 ## 16-byte Reload
+; AVX512-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3]
+; AVX512-NEXT:    vmovaps %xmm0, (%rsp) ## 16-byte Spill
+; AVX512-NEXT:    vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
+; AVX512-NEXT:    ## xmm0 = mem[1,0]
+; AVX512-NEXT:    callq _roundevenf
+; AVX512-NEXT:    vmovaps (%rsp), %xmm1 ## 16-byte Reload
+; AVX512-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
+; AVX512-NEXT:    vmovaps %xmm0, (%rsp) ## 16-byte Spill
+; AVX512-NEXT:    vpermilps $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
+; AVX512-NEXT:    ## xmm0 = mem[3,1,2,3]
+; AVX512-NEXT:    callq _roundevenf
+; AVX512-NEXT:    vmovaps (%rsp), %xmm1 ## 16-byte Reload
+; AVX512-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
+; AVX512-NEXT:    vinsertf128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 ## 16-byte Folded Reload
+; AVX512-NEXT:    vinsertf64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 ## 32-byte Folded Reload
+; AVX512-NEXT:    addq $184, %rsp
+; AVX512-NEXT:    retq
+  %a = call <16 x float> @llvm.roundeven.v16f32(<16 x float> %x)
+  ret <16 x float> %a
+}
+
+define <8 x double> @roundeven_v8f64(<8 x double> %x) {
+; SSE2-LABEL: roundeven_v8f64:
+; SSE2:       ## %bb.0:
+; SSE2-NEXT:    subq $88, %rsp
+; SSE2-NEXT:    .cfi_def_cfa_offset 96
+; SSE2-NEXT:    movaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE2-NEXT:    movaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE2-NEXT:    movaps %xmm1, (%rsp) ## 16-byte Spill
+; SSE2-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE2-NEXT:    callq _roundeven
+; SSE2-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
+; SSE2-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; SSE2-NEXT:    callq _roundeven
+; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
+; SSE2-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE2-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE2-NEXT:    movaps (%rsp), %xmm0 ## 16-byte Reload
+; SSE2-NEXT:    callq _roundeven
+; SSE2-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE2-NEXT:    movaps (%rsp), %xmm0 ## 16-byte Reload
+; SSE2-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; SSE2-NEXT:    callq _roundeven
+; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
+; SSE2-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE2-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
+; SSE2-NEXT:    callq _roundeven
+; SSE2-NEXT:    movaps %xmm0, (%rsp) ## 16-byte Spill
+; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
+; SSE2-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; SSE2-NEXT:    callq _roundeven
+; SSE2-NEXT:    movaps (%rsp), %xmm1 ## 16-byte Reload
+; SSE2-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE2-NEXT:    movaps %xmm1, (%rsp) ## 16-byte Spill
+; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
+; SSE2-NEXT:    callq _roundeven
+; SSE2-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
+; SSE2-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; SSE2-NEXT:    callq _roundeven
+; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 ## 16-byte Reload
+; SSE2-NEXT:    movlhps {{.*#+}} xmm3 = xmm3[0],xmm0[0]
+; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
+; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
+; SSE2-NEXT:    movaps (%rsp), %xmm2 ## 16-byte Reload
+; SSE2-NEXT:    addq $88, %rsp
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: roundeven_v8f64:
+; SSE41:       ## %bb.0:
+; SSE41-NEXT:    subq $88, %rsp
+; SSE41-NEXT:    .cfi_def_cfa_offset 96
+; SSE41-NEXT:    movaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE41-NEXT:    movaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE41-NEXT:    movaps %xmm1, (%rsp) ## 16-byte Spill
+; SSE41-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE41-NEXT:    callq _roundeven
+; SSE41-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
+; SSE41-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; SSE41-NEXT:    callq _roundeven
+; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
+; SSE41-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE41-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE41-NEXT:    movaps (%rsp), %xmm0 ## 16-byte Reload
+; SSE41-NEXT:    callq _roundeven
+; SSE41-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE41-NEXT:    movaps (%rsp), %xmm0 ## 16-byte Reload
+; SSE41-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; SSE41-NEXT:    callq _roundeven
+; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
+; SSE41-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE41-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
+; SSE41-NEXT:    callq _roundeven
+; SSE41-NEXT:    movaps %xmm0, (%rsp) ## 16-byte Spill
+; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
+; SSE41-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; SSE41-NEXT:    callq _roundeven
+; SSE41-NEXT:    movaps (%rsp), %xmm1 ## 16-byte Reload
+; SSE41-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE41-NEXT:    movaps %xmm1, (%rsp) ## 16-byte Spill
+; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
+; SSE41-NEXT:    callq _roundeven
+; SSE41-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
+; SSE41-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; SSE41-NEXT:    callq _roundeven
+; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 ## 16-byte Reload
+; SSE41-NEXT:    movlhps {{.*#+}} xmm3 = xmm3[0],xmm0[0]
+; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
+; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
+; SSE41-NEXT:    movaps (%rsp), %xmm2 ## 16-byte Reload
+; SSE41-NEXT:    addq $88, %rsp
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: roundeven_v8f64:
+; AVX1:       ## %bb.0:
+; AVX1-NEXT:    subq $120, %rsp
+; AVX1-NEXT:    .cfi_def_cfa_offset 128
+; AVX1-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) ## 32-byte Spill
+; AVX1-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 32-byte Spill
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vmovaps %xmm0, (%rsp) ## 16-byte Spill
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    callq _roundeven
+; AVX1-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; AVX1-NEXT:    vpermilpd $1, (%rsp), %xmm0 ## 16-byte Folded Reload
+; AVX1-NEXT:    ## xmm0 = mem[1,0]
+; AVX1-NEXT:    callq _roundeven
+; AVX1-NEXT:    vmovapd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
+; AVX1-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX1-NEXT:    vmovapd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; AVX1-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 ## 32-byte Reload
+; AVX1-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    callq _roundeven
+; AVX1-NEXT:    vmovaps %xmm0, (%rsp) ## 16-byte Spill
+; AVX1-NEXT:    vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
+; AVX1-NEXT:    ## xmm0 = mem[1,0]
+; AVX1-NEXT:    callq _roundeven
+; AVX1-NEXT:    vmovapd (%rsp), %xmm1 ## 16-byte Reload
+; AVX1-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX1-NEXT:    vinsertf128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 ## 16-byte Folded Reload
+; AVX1-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 32-byte Spill
+; AVX1-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 ## 32-byte Reload
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vmovaps %xmm0, (%rsp) ## 16-byte Spill
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    callq _roundeven
+; AVX1-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; AVX1-NEXT:    vpermilpd $1, (%rsp), %xmm0 ## 16-byte Folded Reload
+; AVX1-NEXT:    ## xmm0 = mem[1,0]
+; AVX1-NEXT:    callq _roundeven
+; AVX1-NEXT:    vmovapd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
+; AVX1-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX1-NEXT:    vmovapd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; AVX1-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 ## 32-byte Reload
+; AVX1-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    callq _roundeven
+; AVX1-NEXT:    vmovaps %xmm0, (%rsp) ## 16-byte Spill
+; AVX1-NEXT:    vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
+; AVX1-NEXT:    ## xmm0 = mem[1,0]
+; AVX1-NEXT:    callq _roundeven
+; AVX1-NEXT:    vmovapd (%rsp), %xmm1 ## 16-byte Reload
+; AVX1-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX1-NEXT:    vinsertf128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 ## 16-byte Folded Reload
+; AVX1-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 ## 32-byte Reload
+; AVX1-NEXT:    addq $120, %rsp
+; AVX1-NEXT:    retq
+;
+; AVX512-LABEL: roundeven_v8f64:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    subq $184, %rsp
+; AVX512-NEXT:    .cfi_def_cfa_offset 192
+; AVX512-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 64-byte Spill
+; AVX512-NEXT:    vextractf32x4 $3, %zmm0, %xmm0
+; AVX512-NEXT:    vmovaps %xmm0, (%rsp) ## 16-byte Spill
+; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    callq _roundeven
+; AVX512-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; AVX512-NEXT:    vpermilpd $1, (%rsp), %xmm0 ## 16-byte Folded Reload
+; AVX512-NEXT:    ## xmm0 = mem[1,0]
+; AVX512-NEXT:    callq _roundeven
+; AVX512-NEXT:    vmovapd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
+; AVX512-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512-NEXT:    vmovapd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; AVX512-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 ## 64-byte Reload
+; AVX512-NEXT:    vextractf32x4 $2, %zmm0, %xmm0
+; AVX512-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    callq _roundeven
+; AVX512-NEXT:    vmovaps %xmm0, (%rsp) ## 16-byte Spill
+; AVX512-NEXT:    vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
+; AVX512-NEXT:    ## xmm0 = mem[1,0]
+; AVX512-NEXT:    callq _roundeven
+; AVX512-NEXT:    vmovapd (%rsp), %xmm1 ## 16-byte Reload
+; AVX512-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512-NEXT:    vinsertf128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 ## 16-byte Folded Reload
+; AVX512-NEXT:    vmovupd %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 32-byte Spill
+; AVX512-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 ## 64-byte Reload
+; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX512-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    callq _roundeven
+; AVX512-NEXT:    vmovaps %xmm0, (%rsp) ## 16-byte Spill
+; AVX512-NEXT:    vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
+; AVX512-NEXT:    ## xmm0 = mem[1,0]
+; AVX512-NEXT:    callq _roundeven
+; AVX512-NEXT:    vmovapd (%rsp), %xmm1 ## 16-byte Reload
+; AVX512-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512-NEXT:    vmovapd %xmm0, (%rsp) ## 16-byte Spill
+; AVX512-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 ## 64-byte Reload
+; AVX512-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    callq _roundeven
+; AVX512-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
+; AVX512-NEXT:    vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
+; AVX512-NEXT:    ## xmm0 = mem[1,0]
+; AVX512-NEXT:    callq _roundeven
+; AVX512-NEXT:    vmovapd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
+; AVX512-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512-NEXT:    vinsertf128 $1, (%rsp), %ymm0, %ymm0 ## 16-byte Folded Reload
+; AVX512-NEXT:    vinsertf64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 ## 32-byte Folded Reload
+; AVX512-NEXT:    addq $184, %rsp
+; AVX512-NEXT:    retq
+  %a = call <8 x double> @llvm.roundeven.v8f64(<8 x double> %x)
+  ret <8 x double> %a
+}
+
+declare float @llvm.roundeven.f32(float)
+declare double @llvm.roundeven.f64(double)
+declare <4 x float> @llvm.roundeven.v4f32(<4 x float>)
+declare <2 x double> @llvm.roundeven.v2f64(<2 x double>)
+declare <8 x float> @llvm.roundeven.v8f32(<8 x float>)
+declare <4 x double> @llvm.roundeven.v4f64(<4 x double>)
+declare <16 x float> @llvm.roundeven.v16f32(<16 x float>)
+declare <8 x double> @llvm.roundeven.v8f64(<8 x double>)
diff --git a/llvm/test/CodeGen/X86/fp-strict-scalar-round.ll b/llvm/test/CodeGen/X86/fp-strict-scalar-round.ll
index da05e8be432eb..f5a6af9c4d657 100644
--- a/llvm/test/CodeGen/X86/fp-strict-scalar-round.ll
+++ b/llvm/test/CodeGen/X86/fp-strict-scalar-round.ll
@@ -16,6 +16,10 @@ declare float @llvm.experimental.constrained.rint.f32(float, metadata, metadata)
 declare double @llvm.experimental.constrained.rint.f64(double, metadata, metadata)
 declare float @llvm.experimental.constrained.nearbyint.f32(float, metadata, metadata)
 declare double @llvm.experimental.constrained.nearbyint.f64(double, metadata, metadata)
+declare float @llvm.experimental.constrained.round.f32(float, metadata)
+declare double @llvm.experimental.constrained.round.f64(double, metadata)
+declare float @llvm.experimental.constrained.roundeven.f32(float, metadata)
+declare double @llvm.experimental.constrained.roundeven.f64(double, metadata)
 
 define float @fceil32(float %f) #0 {
 ; SSE41-X86-LABEL: fceil32:
@@ -491,4 +495,184 @@ define double @fnearbyintf64(double %f) #0 {
   ret double %res
 }
 
+define float @fround32(float %f) #0 {
+; SSE41-X86-LABEL: fround32:
+; SSE41-X86:       # %bb.0:
+; SSE41-X86-NEXT:    pushl %eax
+; SSE41-X86-NEXT:    .cfi_def_cfa_offset 8
+; SSE41-X86-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE41-X86-NEXT:    movss %xmm0, (%esp)
+; SSE41-X86-NEXT:    calll roundf
+; SSE41-X86-NEXT:    popl %eax
+; SSE41-X86-NEXT:    .cfi_def_cfa_offset 4
+; SSE41-X86-NEXT:    retl
+;
+; SSE41-X64-LABEL: fround32:
+; SSE41-X64:       # %bb.0:
+; SSE41-X64-NEXT:    pushq %rax
+; SSE41-X64-NEXT:    .cfi_def_cfa_offset 16
+; SSE41-X64-NEXT:    callq roundf
+; SSE41-X64-NEXT:    popq %rax
+; SSE41-X64-NEXT:    .cfi_def_cfa_offset 8
+; SSE41-X64-NEXT:    retq
+;
+; AVX-X86-LABEL: fround32:
+; AVX-X86:       # %bb.0:
+; AVX-X86-NEXT:    pushl %eax
+; AVX-X86-NEXT:    .cfi_def_cfa_offset 8
+; AVX-X86-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-X86-NEXT:    vmovss %xmm0, (%esp)
+; AVX-X86-NEXT:    calll roundf
+; AVX-X86-NEXT:    popl %eax
+; AVX-X86-NEXT:    .cfi_def_cfa_offset 4
+; AVX-X86-NEXT:    retl
+;
+; AVX-X64-LABEL: fround32:
+; AVX-X64:       # %bb.0:
+; AVX-X64-NEXT:    pushq %rax
+; AVX-X64-NEXT:    .cfi_def_cfa_offset 16
+; AVX-X64-NEXT:    callq roundf
+; AVX-X64-NEXT:    popq %rax
+; AVX-X64-NEXT:    .cfi_def_cfa_offset 8
+; AVX-X64-NEXT:    retq
+  %res = call float @llvm.experimental.constrained.round.f32(
+                        float %f, metadata !"fpexcept.strict") #0
+  ret float %res
+}
+
+define double @froundf64(double %f) #0 {
+; SSE41-X86-LABEL: froundf64:
+; SSE41-X86:       # %bb.0:
+; SSE41-X86-NEXT:    subl $8, %esp
+; SSE41-X86-NEXT:    .cfi_def_cfa_offset 12
+; SSE41-X86-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE41-X86-NEXT:    movsd %xmm0, (%esp)
+; SSE41-X86-NEXT:    calll round
+; SSE41-X86-NEXT:    addl $8, %esp
+; SSE41-X86-NEXT:    .cfi_def_cfa_offset 4
+; SSE41-X86-NEXT:    retl
+;
+; SSE41-X64-LABEL: froundf64:
+; SSE41-X64:       # %bb.0:
+; SSE41-X64-NEXT:    pushq %rax
+; SSE41-X64-NEXT:    .cfi_def_cfa_offset 16
+; SSE41-X64-NEXT:    callq round
+; SSE41-X64-NEXT:    popq %rax
+; SSE41-X64-NEXT:    .cfi_def_cfa_offset 8
+; SSE41-X64-NEXT:    retq
+;
+; AVX-X86-LABEL: froundf64:
+; AVX-X86:       # %bb.0:
+; AVX-X86-NEXT:    subl $8, %esp
+; AVX-X86-NEXT:    .cfi_def_cfa_offset 12
+; AVX-X86-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-X86-NEXT:    vmovsd %xmm0, (%esp)
+; AVX-X86-NEXT:    calll round
+; AVX-X86-NEXT:    addl $8, %esp
+; AVX-X86-NEXT:    .cfi_def_cfa_offset 4
+; AVX-X86-NEXT:    retl
+;
+; AVX-X64-LABEL: froundf64:
+; AVX-X64:       # %bb.0:
+; AVX-X64-NEXT:    pushq %rax
+; AVX-X64-NEXT:    .cfi_def_cfa_offset 16
+; AVX-X64-NEXT:    callq round
+; AVX-X64-NEXT:    popq %rax
+; AVX-X64-NEXT:    .cfi_def_cfa_offset 8
+; AVX-X64-NEXT:    retq
+  %res = call double @llvm.experimental.constrained.round.f64(
+                        double %f, metadata !"fpexcept.strict") #0
+  ret double %res
+}
+
+define float @froundeven32(float %f) #0 {
+; SSE41-X86-LABEL: froundeven32:
+; SSE41-X86:       # %bb.0:
+; SSE41-X86-NEXT:    pushl %eax
+; SSE41-X86-NEXT:    .cfi_def_cfa_offset 8
+; SSE41-X86-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE41-X86-NEXT:    movss %xmm0, (%esp)
+; SSE41-X86-NEXT:    calll roundevenf
+; SSE41-X86-NEXT:    popl %eax
+; SSE41-X86-NEXT:    .cfi_def_cfa_offset 4
+; SSE41-X86-NEXT:    retl
+;
+; SSE41-X64-LABEL: froundeven32:
+; SSE41-X64:       # %bb.0:
+; SSE41-X64-NEXT:    pushq %rax
+; SSE41-X64-NEXT:    .cfi_def_cfa_offset 16
+; SSE41-X64-NEXT:    callq roundevenf
+; SSE41-X64-NEXT:    popq %rax
+; SSE41-X64-NEXT:    .cfi_def_cfa_offset 8
+; SSE41-X64-NEXT:    retq
+;
+; AVX-X86-LABEL: froundeven32:
+; AVX-X86:       # %bb.0:
+; AVX-X86-NEXT:    pushl %eax
+; AVX-X86-NEXT:    .cfi_def_cfa_offset 8
+; AVX-X86-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-X86-NEXT:    vmovss %xmm0, (%esp)
+; AVX-X86-NEXT:    calll roundevenf
+; AVX-X86-NEXT:    popl %eax
+; AVX-X86-NEXT:    .cfi_def_cfa_offset 4
+; AVX-X86-NEXT:    retl
+;
+; AVX-X64-LABEL: froundeven32:
+; AVX-X64:       # %bb.0:
+; AVX-X64-NEXT:    pushq %rax
+; AVX-X64-NEXT:    .cfi_def_cfa_offset 16
+; AVX-X64-NEXT:    callq roundevenf
+; AVX-X64-NEXT:    popq %rax
+; AVX-X64-NEXT:    .cfi_def_cfa_offset 8
+; AVX-X64-NEXT:    retq
+  %res = call float @llvm.experimental.constrained.roundeven.f32(
+                        float %f, metadata !"fpexcept.strict") #0
+  ret float %res
+}
+
+define double @froundevenf64(double %f) #0 {
+; SSE41-X86-LABEL: froundevenf64:
+; SSE41-X86:       # %bb.0:
+; SSE41-X86-NEXT:    subl $8, %esp
+; SSE41-X86-NEXT:    .cfi_def_cfa_offset 12
+; SSE41-X86-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE41-X86-NEXT:    movsd %xmm0, (%esp)
+; SSE41-X86-NEXT:    calll roundeven
+; SSE41-X86-NEXT:    addl $8, %esp
+; SSE41-X86-NEXT:    .cfi_def_cfa_offset 4
+; SSE41-X86-NEXT:    retl
+;
+; SSE41-X64-LABEL: froundevenf64:
+; SSE41-X64:       # %bb.0:
+; SSE41-X64-NEXT:    pushq %rax
+; SSE41-X64-NEXT:    .cfi_def_cfa_offset 16
+; SSE41-X64-NEXT:    callq roundeven
+; SSE41-X64-NEXT:    popq %rax
+; SSE41-X64-NEXT:    .cfi_def_cfa_offset 8
+; SSE41-X64-NEXT:    retq
+;
+; AVX-X86-LABEL: froundevenf64:
+; AVX-X86:       # %bb.0:
+; AVX-X86-NEXT:    subl $8, %esp
+; AVX-X86-NEXT:    .cfi_def_cfa_offset 12
+; AVX-X86-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-X86-NEXT:    vmovsd %xmm0, (%esp)
+; AVX-X86-NEXT:    calll roundeven
+; AVX-X86-NEXT:    addl $8, %esp
+; AVX-X86-NEXT:    .cfi_def_cfa_offset 4
+; AVX-X86-NEXT:    retl
+;
+; AVX-X64-LABEL: froundevenf64:
+; AVX-X64:       # %bb.0:
+; AVX-X64-NEXT:    pushq %rax
+; AVX-X64-NEXT:    .cfi_def_cfa_offset 16
+; AVX-X64-NEXT:    callq roundeven
+; AVX-X64-NEXT:    popq %rax
+; AVX-X64-NEXT:    .cfi_def_cfa_offset 8
+; AVX-X64-NEXT:    retq
+  %res = call double @llvm.experimental.constrained.roundeven.f64(
+                        double %f, metadata !"fpexcept.strict") #0
+  ret double %res
+}
+
 attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/X86/fp128-libcalls-strict.ll b/llvm/test/CodeGen/X86/fp128-libcalls-strict.ll
index b705c760287ed..d2be7fb68900e 100644
--- a/llvm/test/CodeGen/X86/fp128-libcalls-strict.ll
+++ b/llvm/test/CodeGen/X86/fp128-libcalls-strict.ll
@@ -913,6 +913,47 @@ entry:
   ret fp128 %round
 }
 
+define fp128 @roundeven(fp128 %x) nounwind strictfp {
+; CHECK-LABEL: roundeven:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    callq roundevenl
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    retq
+;
+; X86-LABEL: roundeven:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    calll roundevenl
+; X86-NEXT:    addl $28, %esp
+; X86-NEXT:    movl (%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %edi, 8(%esi)
+; X86-NEXT:    movl %edx, 12(%esi)
+; X86-NEXT:    movl %eax, (%esi)
+; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    retl $4
+entry:
+  %roundeven = call fp128 @llvm.experimental.constrained.roundeven.f128(fp128 %x, metadata !"fpexcept.strict") #0
+  ret fp128 %roundeven
+}
+
 define fp128 @sin(fp128 %x) nounwind strictfp {
 ; CHECK-LABEL: sin:
 ; CHECK:       # %bb.0: # %entry
@@ -1409,6 +1450,7 @@ declare fp128 @llvm.experimental.constrained.pow.f128(fp128, fp128, metadata, me
 declare fp128 @llvm.experimental.constrained.powi.f128(fp128, i32, metadata, metadata)
 declare fp128 @llvm.experimental.constrained.rint.f128(fp128, metadata, metadata)
 declare fp128 @llvm.experimental.constrained.round.f128(fp128, metadata)
+declare fp128 @llvm.experimental.constrained.roundeven.f128(fp128, metadata)
 declare fp128 @llvm.experimental.constrained.sin.f128(fp128, metadata, metadata)
 declare fp128 @llvm.experimental.constrained.sqrt.f128(fp128, metadata, metadata)
 declare fp128 @llvm.experimental.constrained.trunc.f128(fp128, metadata)
diff --git a/llvm/test/CodeGen/X86/fp80-strict-libcalls.ll b/llvm/test/CodeGen/X86/fp80-strict-libcalls.ll
new file mode 100644
index 0000000000000..c199352d14239
--- /dev/null
+++ b/llvm/test/CodeGen/X86/fp80-strict-libcalls.ll
@@ -0,0 +1,657 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-unknown-unknown -O3 | FileCheck %s --check-prefixes=CHECK,X86
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -O3 | FileCheck %s --check-prefixes=CHECK,X64
+
+define x86_fp80 @fma(x86_fp80 %x, x86_fp80 %y, x86_fp80 %z) nounwind strictfp {
+; X86-LABEL: fma:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    subl $36, %esp
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fstpt {{[0-9]+}}(%esp)
+; X86-NEXT:    fstpt {{[0-9]+}}(%esp)
+; X86-NEXT:    fstpt (%esp)
+; X86-NEXT:    wait
+; X86-NEXT:    calll fmal
+; X86-NEXT:    addl $36, %esp
+; X86-NEXT:    retl
+;
+; X64-LABEL: fma:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    subq $56, %rsp
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fstpt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fstpt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fstpt (%rsp)
+; X64-NEXT:    wait
+; X64-NEXT:    callq fmal
+; X64-NEXT:    addq $56, %rsp
+; X64-NEXT:    retq
+entry:
+  %fma = call x86_fp80 @llvm.experimental.constrained.fma.f80(x86_fp80 %x, x86_fp80 %y,  x86_fp80 %z, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret x86_fp80 %fma
+}
+
+define x86_fp80 @frem(x86_fp80 %x, x86_fp80 %y) nounwind strictfp {
+; X86-LABEL: frem:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    subl $24, %esp
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fstpt {{[0-9]+}}(%esp)
+; X86-NEXT:    fstpt (%esp)
+; X86-NEXT:    wait
+; X86-NEXT:    calll fmodl
+; X86-NEXT:    addl $24, %esp
+; X86-NEXT:    retl
+;
+; X64-LABEL: frem:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    subq $40, %rsp
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fstpt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fstpt (%rsp)
+; X64-NEXT:    wait
+; X64-NEXT:    callq fmodl
+; X64-NEXT:    addq $40, %rsp
+; X64-NEXT:    retq
+entry:
+  %div = call x86_fp80 @llvm.experimental.constrained.frem.f80(x86_fp80 %x, x86_fp80 %y,  metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret x86_fp80 %div
+}
+
+define x86_fp80 @ceil(x86_fp80 %x) nounwind strictfp {
+; X86-LABEL: ceil:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fstpt (%esp)
+; X86-NEXT:    wait
+; X86-NEXT:    calll ceill
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    retl
+;
+; X64-LABEL: ceil:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    subq $24, %rsp
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fstpt (%rsp)
+; X64-NEXT:    wait
+; X64-NEXT:    callq ceill
+; X64-NEXT:    addq $24, %rsp
+; X64-NEXT:    retq
+entry:
+  %ceil = call x86_fp80 @llvm.experimental.constrained.ceil.f80(x86_fp80 %x, metadata !"fpexcept.strict") #0
+  ret x86_fp80 %ceil
+}
+
+define x86_fp80 @cos(x86_fp80 %x) nounwind strictfp {
+; X86-LABEL: cos:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fstpt (%esp)
+; X86-NEXT:    wait
+; X86-NEXT:    calll cosl
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    retl
+;
+; X64-LABEL: cos:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    subq $24, %rsp
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fstpt (%rsp)
+; X64-NEXT:    wait
+; X64-NEXT:    callq cosl
+; X64-NEXT:    addq $24, %rsp
+; X64-NEXT:    retq
+entry:
+  %cos = call x86_fp80 @llvm.experimental.constrained.cos.f80(x86_fp80 %x, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret x86_fp80 %cos
+}
+
+define x86_fp80 @exp(x86_fp80 %x) nounwind strictfp {
+; X86-LABEL: exp:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fstpt (%esp)
+; X86-NEXT:    wait
+; X86-NEXT:    calll expl
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    retl
+;
+; X64-LABEL: exp:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    subq $24, %rsp
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fstpt (%rsp)
+; X64-NEXT:    wait
+; X64-NEXT:    callq expl
+; X64-NEXT:    addq $24, %rsp
+; X64-NEXT:    retq
+entry:
+  %exp = call x86_fp80 @llvm.experimental.constrained.exp.f80(x86_fp80 %x, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret x86_fp80 %exp
+}
+
+define x86_fp80 @exp2(x86_fp80 %x) nounwind strictfp {
+; X86-LABEL: exp2:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fstpt (%esp)
+; X86-NEXT:    wait
+; X86-NEXT:    calll exp2l
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    retl
+;
+; X64-LABEL: exp2:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    subq $24, %rsp
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fstpt (%rsp)
+; X64-NEXT:    wait
+; X64-NEXT:    callq exp2l
+; X64-NEXT:    addq $24, %rsp
+; X64-NEXT:    retq
+entry:
+  %exp2 = call x86_fp80 @llvm.experimental.constrained.exp2.f80(x86_fp80 %x, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret x86_fp80 %exp2
+}
+
+define x86_fp80 @floor(x86_fp80 %x) nounwind strictfp {
+; X86-LABEL: floor:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fstpt (%esp)
+; X86-NEXT:    wait
+; X86-NEXT:    calll floorl
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    retl
+;
+; X64-LABEL: floor:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    subq $24, %rsp
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fstpt (%rsp)
+; X64-NEXT:    wait
+; X64-NEXT:    callq floorl
+; X64-NEXT:    addq $24, %rsp
+; X64-NEXT:    retq
+entry:
+  %floor = call x86_fp80 @llvm.experimental.constrained.floor.f80(x86_fp80 %x, metadata !"fpexcept.strict") #0
+  ret x86_fp80 %floor
+}
+
+define x86_fp80 @log(x86_fp80 %x) nounwind strictfp {
+; X86-LABEL: log:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fstpt (%esp)
+; X86-NEXT:    wait
+; X86-NEXT:    calll logl
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    retl
+;
+; X64-LABEL: log:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    subq $24, %rsp
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fstpt (%rsp)
+; X64-NEXT:    wait
+; X64-NEXT:    callq logl
+; X64-NEXT:    addq $24, %rsp
+; X64-NEXT:    retq
+entry:
+  %log = call x86_fp80 @llvm.experimental.constrained.log.f80(x86_fp80 %x, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret x86_fp80 %log
+}
+
+define x86_fp80 @log10(x86_fp80 %x) nounwind strictfp {
+; X86-LABEL: log10:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fstpt (%esp)
+; X86-NEXT:    wait
+; X86-NEXT:    calll log10l
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    retl
+;
+; X64-LABEL: log10:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    subq $24, %rsp
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fstpt (%rsp)
+; X64-NEXT:    wait
+; X64-NEXT:    callq log10l
+; X64-NEXT:    addq $24, %rsp
+; X64-NEXT:    retq
+entry:
+  %log10 = call x86_fp80 @llvm.experimental.constrained.log10.f80(x86_fp80 %x, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret x86_fp80 %log10
+}
+
+define x86_fp80 @log2(x86_fp80 %x) nounwind strictfp {
+; X86-LABEL: log2:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fstpt (%esp)
+; X86-NEXT:    wait
+; X86-NEXT:    calll log2l
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    retl
+;
+; X64-LABEL: log2:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    subq $24, %rsp
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fstpt (%rsp)
+; X64-NEXT:    wait
+; X64-NEXT:    callq log2l
+; X64-NEXT:    addq $24, %rsp
+; X64-NEXT:    retq
+entry:
+  %log2 = call x86_fp80 @llvm.experimental.constrained.log2.f80(x86_fp80 %x, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret x86_fp80 %log2
+}
+
+define x86_fp80 @maxnum(x86_fp80 %x, x86_fp80 %y) nounwind strictfp {
+; X86-LABEL: maxnum:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    subl $24, %esp
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fstpt {{[0-9]+}}(%esp)
+; X86-NEXT:    fstpt (%esp)
+; X86-NEXT:    wait
+; X86-NEXT:    calll fmaxl
+; X86-NEXT:    addl $24, %esp
+; X86-NEXT:    retl
+;
+; X64-LABEL: maxnum:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    subq $40, %rsp
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fstpt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fstpt (%rsp)
+; X64-NEXT:    wait
+; X64-NEXT:    callq fmaxl
+; X64-NEXT:    addq $40, %rsp
+; X64-NEXT:    retq
+entry:
+  %maxnum = call x86_fp80 @llvm.experimental.constrained.maxnum.f80(x86_fp80 %x, x86_fp80 %y, metadata !"fpexcept.strict") #0
+  ret x86_fp80 %maxnum
+}
+
+define x86_fp80 @minnum(x86_fp80 %x, x86_fp80 %y) nounwind strictfp {
+; X86-LABEL: minnum:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    subl $24, %esp
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fstpt {{[0-9]+}}(%esp)
+; X86-NEXT:    fstpt (%esp)
+; X86-NEXT:    wait
+; X86-NEXT:    calll fminl
+; X86-NEXT:    addl $24, %esp
+; X86-NEXT:    retl
+;
+; X64-LABEL: minnum:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    subq $40, %rsp
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fstpt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fstpt (%rsp)
+; X64-NEXT:    wait
+; X64-NEXT:    callq fminl
+; X64-NEXT:    addq $40, %rsp
+; X64-NEXT:    retq
+entry:
+  %minnum = call x86_fp80 @llvm.experimental.constrained.minnum.f80(x86_fp80 %x, x86_fp80 %y, metadata !"fpexcept.strict") #0
+  ret x86_fp80 %minnum
+}
+
+define x86_fp80 @nearbyint(x86_fp80 %x) nounwind strictfp {
+; X86-LABEL: nearbyint:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fstpt (%esp)
+; X86-NEXT:    wait
+; X86-NEXT:    calll nearbyintl
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    retl
+;
+; X64-LABEL: nearbyint:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    subq $24, %rsp
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fstpt (%rsp)
+; X64-NEXT:    wait
+; X64-NEXT:    callq nearbyintl
+; X64-NEXT:    addq $24, %rsp
+; X64-NEXT:    retq
+entry:
+  %nearbyint = call x86_fp80 @llvm.experimental.constrained.nearbyint.f80(x86_fp80 %x, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret x86_fp80 %nearbyint
+}
+
+define x86_fp80 @pow(x86_fp80 %x, x86_fp80 %y) nounwind strictfp {
+; X86-LABEL: pow:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    subl $24, %esp
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fstpt {{[0-9]+}}(%esp)
+; X86-NEXT:    fstpt (%esp)
+; X86-NEXT:    wait
+; X86-NEXT:    calll powl
+; X86-NEXT:    addl $24, %esp
+; X86-NEXT:    retl
+;
+; X64-LABEL: pow:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    subq $40, %rsp
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fstpt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fstpt (%rsp)
+; X64-NEXT:    wait
+; X64-NEXT:    callq powl
+; X64-NEXT:    addq $40, %rsp
+; X64-NEXT:    retq
+entry:
+  %pow = call x86_fp80 @llvm.experimental.constrained.pow.f80(x86_fp80 %x, x86_fp80 %y, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret x86_fp80 %pow
+}
+
+define x86_fp80 @powi(x86_fp80 %x, i32 %y) nounwind strictfp {
+; X86-LABEL: powi:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    subl $16, %esp
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    wait
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT:    fstpt (%esp)
+; X86-NEXT:    wait
+; X86-NEXT:    calll __powixf2
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    retl
+;
+; X64-LABEL: powi:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    subq $24, %rsp
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fstpt (%rsp)
+; X64-NEXT:    wait
+; X64-NEXT:    callq __powixf2
+; X64-NEXT:    addq $24, %rsp
+; X64-NEXT:    retq
+entry:
+  %powi = call x86_fp80 @llvm.experimental.constrained.powi.f80(x86_fp80 %x, i32 %y, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret x86_fp80 %powi
+}
+
+define x86_fp80 @rint(x86_fp80 %x) nounwind strictfp {
+; X86-LABEL: rint:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fstpt (%esp)
+; X86-NEXT:    wait
+; X86-NEXT:    calll rintl
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    retl
+;
+; X64-LABEL: rint:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    subq $24, %rsp
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fstpt (%rsp)
+; X64-NEXT:    wait
+; X64-NEXT:    callq rintl
+; X64-NEXT:    addq $24, %rsp
+; X64-NEXT:    retq
+entry:
+  %rint = call x86_fp80 @llvm.experimental.constrained.rint.f80(x86_fp80 %x, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret x86_fp80 %rint
+}
+
+define x86_fp80 @round(x86_fp80 %x) nounwind strictfp {
+; X86-LABEL: round:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fstpt (%esp)
+; X86-NEXT:    wait
+; X86-NEXT:    calll roundl
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    retl
+;
+; X64-LABEL: round:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    subq $24, %rsp
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fstpt (%rsp)
+; X64-NEXT:    wait
+; X64-NEXT:    callq roundl
+; X64-NEXT:    addq $24, %rsp
+; X64-NEXT:    retq
+entry:
+  %round = call x86_fp80 @llvm.experimental.constrained.round.f80(x86_fp80 %x, metadata !"fpexcept.strict") #0
+  ret x86_fp80 %round
+}
+
+define x86_fp80 @roundeven(x86_fp80 %x) nounwind strictfp {
+; X86-LABEL: roundeven:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fstpt (%esp)
+; X86-NEXT:    wait
+; X86-NEXT:    calll roundevenl
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    retl
+;
+; X64-LABEL: roundeven:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    subq $24, %rsp
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fstpt (%rsp)
+; X64-NEXT:    wait
+; X64-NEXT:    callq roundevenl
+; X64-NEXT:    addq $24, %rsp
+; X64-NEXT:    retq
+entry:
+  %roundeven = call x86_fp80 @llvm.experimental.constrained.roundeven.f80(x86_fp80 %x, metadata !"fpexcept.strict") #0
+  ret x86_fp80 %roundeven
+}
+
+define x86_fp80 @sin(x86_fp80 %x) nounwind strictfp {
+; X86-LABEL: sin:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fstpt (%esp)
+; X86-NEXT:    wait
+; X86-NEXT:    calll sinl
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    retl
+;
+; X64-LABEL: sin:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    subq $24, %rsp
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fstpt (%rsp)
+; X64-NEXT:    wait
+; X64-NEXT:    callq sinl
+; X64-NEXT:    addq $24, %rsp
+; X64-NEXT:    retq
+entry:
+  %sin = call x86_fp80 @llvm.experimental.constrained.sin.f80(x86_fp80 %x, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret x86_fp80 %sin
+}
+
+define x86_fp80 @trunc(x86_fp80 %x) nounwind strictfp {
+; X86-LABEL: trunc:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fstpt (%esp)
+; X86-NEXT:    wait
+; X86-NEXT:    calll truncl
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    retl
+;
+; X64-LABEL: trunc:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    subq $24, %rsp
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fstpt (%rsp)
+; X64-NEXT:    wait
+; X64-NEXT:    callq truncl
+; X64-NEXT:    addq $24, %rsp
+; X64-NEXT:    retq
+entry:
+  %trunc = call x86_fp80 @llvm.experimental.constrained.trunc.f80(x86_fp80 %x, metadata !"fpexcept.strict") #0
+  ret x86_fp80 %trunc
+}
+
+define i32 @lrint(x86_fp80 %x) nounwind strictfp {
+; X86-LABEL: lrint:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fstpt (%esp)
+; X86-NEXT:    wait
+; X86-NEXT:    calll lrintl
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    retl
+;
+; X64-LABEL: lrint:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    subq $24, %rsp
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fstpt (%rsp)
+; X64-NEXT:    wait
+; X64-NEXT:    callq lrintl
+; X64-NEXT:    addq $24, %rsp
+; X64-NEXT:    retq
+entry:
+  %rint = call i32 @llvm.experimental.constrained.lrint.i32.f80(x86_fp80 %x, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret i32 %rint
+}
+
+define i64 @llrint(x86_fp80 %x) nounwind strictfp {
+; X86-LABEL: llrint:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fstpt (%esp)
+; X86-NEXT:    wait
+; X86-NEXT:    calll llrintl
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    retl
+;
+; X64-LABEL: llrint:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    subq $24, %rsp
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fstpt (%rsp)
+; X64-NEXT:    wait
+; X64-NEXT:    callq llrintl
+; X64-NEXT:    addq $24, %rsp
+; X64-NEXT:    retq
+entry:
+  %rint = call i64 @llvm.experimental.constrained.llrint.i64.f80(x86_fp80 %x, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret i64 %rint
+}
+
+define i32 @lround(x86_fp80 %x) nounwind strictfp {
+; X86-LABEL: lround:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fstpt (%esp)
+; X86-NEXT:    wait
+; X86-NEXT:    calll lroundl
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    retl
+;
+; X64-LABEL: lround:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    subq $24, %rsp
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fstpt (%rsp)
+; X64-NEXT:    wait
+; X64-NEXT:    callq lroundl
+; X64-NEXT:    addq $24, %rsp
+; X64-NEXT:    retq
+entry:
+  %round = call i32 @llvm.experimental.constrained.lround.i32.f80(x86_fp80 %x, metadata !"fpexcept.strict") #0
+  ret i32 %round
+}
+
+define i64 @llround(x86_fp80 %x) nounwind strictfp {
+; X86-LABEL: llround:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; X86-NEXT:    fstpt (%esp)
+; X86-NEXT:    wait
+; X86-NEXT:    calll llroundl
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    retl
+;
+; X64-LABEL: llround:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    subq $24, %rsp
+; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NEXT:    fstpt (%rsp)
+; X64-NEXT:    wait
+; X64-NEXT:    callq llroundl
+; X64-NEXT:    addq $24, %rsp
+; X64-NEXT:    retq
+entry:
+  %round = call i64 @llvm.experimental.constrained.llround.i64.f80(x86_fp80 %x, metadata !"fpexcept.strict") #0
+  ret i64 %round
+}
+
+attributes #0 = { strictfp }
+
+declare x86_fp80 @llvm.experimental.constrained.fma.f80(x86_fp80, x86_fp80, x86_fp80, metadata, metadata)
+declare x86_fp80 @llvm.experimental.constrained.frem.f80(x86_fp80, x86_fp80, metadata, metadata)
+declare x86_fp80 @llvm.experimental.constrained.ceil.f80(x86_fp80, metadata)
+declare x86_fp80 @llvm.experimental.constrained.cos.f80(x86_fp80, metadata, metadata)
+declare x86_fp80 @llvm.experimental.constrained.exp.f80(x86_fp80, metadata, metadata)
+declare x86_fp80 @llvm.experimental.constrained.exp2.f80(x86_fp80, metadata, metadata)
+declare x86_fp80 @llvm.experimental.constrained.floor.f80(x86_fp80, metadata)
+declare x86_fp80 @llvm.experimental.constrained.log.f80(x86_fp80, metadata, metadata)
+declare x86_fp80 @llvm.experimental.constrained.log10.f80(x86_fp80, metadata, metadata)
+declare x86_fp80 @llvm.experimental.constrained.log2.f80(x86_fp80, metadata, metadata)
+declare x86_fp80 @llvm.experimental.constrained.maxnum.f80(x86_fp80, x86_fp80, metadata)
+declare x86_fp80 @llvm.experimental.constrained.minnum.f80(x86_fp80, x86_fp80, metadata)
+declare x86_fp80 @llvm.experimental.constrained.nearbyint.f80(x86_fp80, metadata, metadata)
+declare x86_fp80 @llvm.experimental.constrained.pow.f80(x86_fp80, x86_fp80, metadata, metadata)
+declare x86_fp80 @llvm.experimental.constrained.powi.f80(x86_fp80, i32, metadata, metadata)
+declare x86_fp80 @llvm.experimental.constrained.rint.f80(x86_fp80, metadata, metadata)
+declare x86_fp80 @llvm.experimental.constrained.round.f80(x86_fp80, metadata)
+declare x86_fp80 @llvm.experimental.constrained.roundeven.f80(x86_fp80, metadata)
+declare x86_fp80 @llvm.experimental.constrained.sin.f80(x86_fp80, metadata, metadata)
+declare x86_fp80 @llvm.experimental.constrained.trunc.f80(x86_fp80, metadata)
+declare i32 @llvm.experimental.constrained.lrint.i32.f80(x86_fp80, metadata, metadata)
+declare i64 @llvm.experimental.constrained.llrint.i64.f80(x86_fp80, metadata, metadata)
+declare i32 @llvm.experimental.constrained.lround.i32.f80(x86_fp80, metadata)
+declare i64 @llvm.experimental.constrained.llround.i64.f80(x86_fp80, metadata)

From 3da1a9634eb9d92b5ffa2571215c350a9641d07b Mon Sep 17 00:00:00 2001
From: Philip Reames <listmail@philipreames.com>
Date: Sat, 11 Jul 2020 10:50:34 -0700
Subject: [PATCH 0094/1035] [Statepoints] Support lowering gc relocations to
 virtual registers

(Disabled under flag for the moment)

This is part of a larger project wherein we are finally integrating lowering of gc live operands with the register allocator.  Today, we force spill all operands in SelectionDAG.  The code to do so is distinctly non-optimal.  The approach this patch is working towards is to instead lower the relocations directly into the MI form, and let the register allocator pick which ones get spilled and which stack slots they get spilled to.  In terms of performance, the later part is actually more important as it avoids redundant shuffling of values between stack slots.

This particular change adds ISEL support to produce the variadic def STATEPOINT form required by the above.  In particular, the first N are lowered to variadic tied def/use pairs.  So new statepoint looks like this:
reloc1,reloc2,... = STATEPOINT ..., base1, derived1<tied-def0>, base2, derived2<tied-def1>, ...

N is limited by the maximal number of tied registers machine instruction can have (15 at the moment).

The current patch is restricted to handling relocations within a single basic block.  Cross block relocations (e.g. invokes) are handled via the legacy mechanism.  This restriction will be relaxed in future patches.

Patch By: dantrushin
Differential Revision: https://reviews.llvm.org/D81648
---
 .../lib/CodeGen/SelectionDAG/InstrEmitter.cpp |  40 +
 .../SelectionDAG/ScheduleDAGSDNodes.cpp       |   3 +-
 .../SelectionDAG/StatepointLowering.cpp       | 132 ++-
 .../CodeGen/SelectionDAG/StatepointLowering.h |   4 +
 llvm/lib/CodeGen/TargetLoweringBase.cpp       |  12 +-
 llvm/test/CodeGen/X86/statepoint-vreg.ll      | 907 ++++++++++++++++++
 6 files changed, 1074 insertions(+), 24 deletions(-)
 create mode 100644 llvm/test/CodeGen/X86/statepoint-vreg.ll

diff --git a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
index 0e4e99214aa24..ff84fdd62075c 100644
--- a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
@@ -82,6 +82,28 @@ static unsigned countOperands(SDNode *Node, unsigned NumExpUses,
   return N;
 }
 
+/// Return starting index of GC operand list.
+// FIXME: need a better place for this. Put it in StackMaps?
+static unsigned getStatepointGCArgStartIdx(MachineInstr *MI) {
+  assert(MI->getOpcode() == TargetOpcode::STATEPOINT &&
+         "STATEPOINT node expected");
+  unsigned OperIdx = StatepointOpers(MI).getNumDeoptArgsIdx();
+  unsigned NumDeopts = MI->getOperand(OperIdx).getImm();
+  // At this point stack references has not been lowered yet, so they
+  // take single operand.
+  ++OperIdx;
+  while (NumDeopts--) {
+    MachineOperand &MO = MI->getOperand(OperIdx);
+    if (MO.isImm() && MO.getImm() == StackMaps::ConstantOp) {
+      ++OperIdx;
+      assert(MI->getOperand(OperIdx).isImm() &&
+             "Unexpected statepoint operand");
+    }
+    ++OperIdx;
+  }
+  return OperIdx;
+}
+
 /// EmitCopyFromReg - Generate machine code for an CopyFromReg node or an
 /// implicit physical register output.
 void InstrEmitter::
@@ -200,6 +222,8 @@ void InstrEmitter::CreateVirtualRegisters(SDNode *Node,
   bool HasVRegVariadicDefs = !MF->getTarget().usesPhysRegsForValues() &&
                              II.isVariadic() && II.variadicOpsAreDefs();
   unsigned NumVRegs = HasVRegVariadicDefs ? NumResults : II.getNumDefs();
+  if (Node->getMachineOpcode() == TargetOpcode::STATEPOINT)
+    NumVRegs = NumResults;
   for (unsigned i = 0; i < NumVRegs; ++i) {
     // If the specific node value is only used by a CopyToReg and the dest reg
     // is a vreg in the same register class, use the CopyToReg'd destination
@@ -821,6 +845,8 @@ EmitMachineNode(SDNode *Node, bool IsClone, bool IsCloned,
       NumDefs = NumResults;
     }
     ScratchRegs = TLI->getScratchRegisters((CallingConv::ID) CC);
+  } else if (Opc == TargetOpcode::STATEPOINT) {
+    NumDefs = NumResults;
   }
 
   unsigned NumImpUses = 0;
@@ -970,6 +996,20 @@ EmitMachineNode(SDNode *Node, bool IsClone, bool IsCloned,
   if (!UsedRegs.empty() || II.getImplicitDefs() || II.hasOptionalDef())
     MIB->setPhysRegsDeadExcept(UsedRegs, *TRI);
 
+  // STATEPOINT is too 'dynamic' to have meaningful machine description.
+  // We have to manually tie operands.
+  if (Opc == TargetOpcode::STATEPOINT && NumDefs > 0) {
+    assert(!HasPhysRegOuts && "STATEPOINT mishandled");
+    MachineInstr *MI = MIB;
+    unsigned Def = 0;
+    unsigned Use = getStatepointGCArgStartIdx(MI) + 1;
+    while (Def < NumDefs) {
+      if (MI->getOperand(Use).isReg())
+        MI->tieOperands(Def++, Use);
+      Use += 2;
+    }
+  }
+
   // Run post-isel target hook to adjust this instruction if needed.
   if (II.hasPostISelHook())
     TLI->AdjustInstrPostInstrSelection(*MIB, Node);
diff --git a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp
index 17c68f2bf73b3..76fb0340a7a88 100644
--- a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp
@@ -125,8 +125,7 @@ static void CheckForPhysRegDependency(SDNode *Def, SDNode *User, unsigned Op,
     PhysReg = Reg;
   } else if (Def->isMachineOpcode()) {
     const MCInstrDesc &II = TII->get(Def->getMachineOpcode());
-    if (ResNo >= II.getNumDefs() &&
-        II.ImplicitDefs[ResNo - II.getNumDefs()] == Reg)
+    if (ResNo >= II.getNumDefs() && II.hasImplicitDefOfPhysReg(Reg))
       PhysReg = Reg;
   }
 
diff --git a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
index 2cb57c1d1ccc8..b8c4c73bcccee 100644
--- a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
@@ -67,6 +67,10 @@ cl::opt<bool> UseRegistersForDeoptValues(
     "use-registers-for-deopt-values", cl::Hidden, cl::init(false),
     cl::desc("Allow using registers for non pointer deopt args"));
 
+cl::opt<unsigned> MaxRegistersForGCPointers(
+    "max-registers-for-gc-values", cl::Hidden, cl::init(0),
+    cl::desc("Max number of VRegs allowed to pass GC pointer meta args in"));
+
 static void pushStackMapConstant(SmallVectorImpl<SDValue>& Ops,
                                  SelectionDAGBuilder &Builder, uint64_t Value) {
   SDLoc L = Builder.getCurSDLoc();
@@ -86,11 +90,13 @@ void StatepointLoweringState::startNewStatepoint(SelectionDAGBuilder &Builder) {
   // FunctionLoweringInfo.  Also need to ensure used bits get cleared.
   AllocatedStackSlots.clear();
   AllocatedStackSlots.resize(Builder.FuncInfo.StatepointStackSlots.size());
+  DerivedPtrMap.clear();
 }
 
 void StatepointLoweringState::clear() {
   Locations.clear();
   AllocatedStackSlots.clear();
+  DerivedPtrMap.clear();
   assert(PendingGCRelocateCalls.empty() &&
          "cleared before statepoint sequence completed");
 }
@@ -221,7 +227,6 @@ static Optional<int> findPreviousSpillSlot(const Value *Val,
   return None;
 }
 
-
 /// Return true if-and-only-if the given SDValue can be lowered as either a
 /// constant argument or a stack reference.  The key point is that the value
 /// doesn't need to be spilled or tracked as a vreg use.
@@ -242,7 +247,6 @@ static bool willLowerDirectly(SDValue Incoming) {
           Incoming.isUndef());
 }
 
-
 /// Try to find existing copies of the incoming values in stack slots used for
 /// statepoint spilling.  If we can find a spill slot for the incoming value,
 /// mark that slot as allocated, and reuse the same slot for this safepoint.
@@ -388,7 +392,7 @@ spillIncomingStatepointValue(SDValue Incoming, SDValue Chain,
                                  StoreMMO);
 
     MMO = getMachineMemOperand(MF, *cast<FrameIndexSDNode>(Loc));
-    
+
     Builder.StatepointLowering.setLocation(Incoming, Loc);
   }
 
@@ -485,7 +489,9 @@ lowerIncomingStatepointValue(SDValue Incoming, bool RequireSpillSlot,
 /// will be set to the last value spilled (if any were).
 static void
 lowerStatepointMetaArgs(SmallVectorImpl<SDValue> &Ops,
-                        SmallVectorImpl<MachineMemOperand*> &MemRefs,                                    SelectionDAGBuilder::StatepointLoweringInfo &SI,
+                        SmallVectorImpl<MachineMemOperand *> &MemRefs,
+                        DenseMap<SDValue, int> &LowerAsVReg,
+                        SelectionDAGBuilder::StatepointLoweringInfo &SI,
                         SelectionDAGBuilder &Builder) {
   // Lower the deopt and gc arguments for this statepoint.  Layout will be:
   // deopt argument length, deopt arguments.., gc arguments...
@@ -531,6 +537,37 @@ lowerStatepointMetaArgs(SmallVectorImpl<SDValue> &Ops,
   const bool LiveInDeopt =
     SI.StatepointFlags & (uint64_t)StatepointFlags::DeoptLiveIn;
 
+  // Decide which deriver pointers will go on VRegs
+  const unsigned MaxTiedRegs = 15; // Max  number of tied regs MI can have.
+  unsigned MaxVRegPtrs =
+      std::min(MaxTiedRegs, MaxRegistersForGCPointers.getValue());
+  // Use old spill scheme for cross-block relocates.
+  if (SI.StatepointInstr) {
+    const BasicBlock *BB = SI.StatepointInstr->getParent();
+    bool NonLocalReloc =
+        llvm::any_of(SI.GCRelocates, [BB](const GCRelocateInst *R) {
+          return R->getParent() != BB;
+        });
+    if (NonLocalReloc)
+      MaxVRegPtrs = 0;
+  }
+
+  LLVM_DEBUG(dbgs() << "Desiding how to lower GC Pointers:\n");
+  unsigned CurNumVRegs = 0;
+  for (const Value *P : SI.Ptrs) {
+    if (LowerAsVReg.size() == MaxVRegPtrs)
+      break;
+    SDValue PtrSD = Builder.getValue(P);
+    if (willLowerDirectly(PtrSD) || P->getType()->isVectorTy()) {
+      LLVM_DEBUG(dbgs() << "direct/spill "; PtrSD.dump(&Builder.DAG));
+      continue;
+    }
+    LLVM_DEBUG(dbgs() << "vreg "; PtrSD.dump(&Builder.DAG));
+    LowerAsVReg[PtrSD] = CurNumVRegs++;
+  }
+  LLVM_DEBUG(dbgs() << LowerAsVReg.size()
+                    << " derived pointers will go in vregs\n");
+
   auto isGCValue = [&](const Value *V) {
     auto *Ty = V->getType();
     if (!Ty->isPtrOrPtrVectorTy())
@@ -542,7 +579,9 @@ lowerStatepointMetaArgs(SmallVectorImpl<SDValue> &Ops,
   };
 
   auto requireSpillSlot = [&](const Value *V) {
-    return !(LiveInDeopt || UseRegistersForDeoptValues) || isGCValue(V);
+    if (isGCValue(V))
+      return !LowerAsVReg.count(Builder.getValue(V));
+    return !(LiveInDeopt || UseRegistersForDeoptValues);
   };
 
   // Before we actually start lowering (and allocating spill slots for values),
@@ -554,9 +593,14 @@ lowerStatepointMetaArgs(SmallVectorImpl<SDValue> &Ops,
     if (requireSpillSlot(V))
       reservePreviousStackSlotForValue(V, Builder);
   }
+
   for (unsigned i = 0; i < SI.Bases.size(); ++i) {
-    reservePreviousStackSlotForValue(SI.Bases[i], Builder);
-    reservePreviousStackSlotForValue(SI.Ptrs[i], Builder);
+    SDValue SDV = Builder.getValue(SI.Bases[i]);
+    if (!LowerAsVReg.count(SDV))
+      reservePreviousStackSlotForValue(SI.Bases[i], Builder);
+    SDV = Builder.getValue(SI.Ptrs[i]);
+    if (!LowerAsVReg.count(SDV))
+      reservePreviousStackSlotForValue(SI.Ptrs[i], Builder);
   }
 
   // First, prefix the list with the number of unique values to be
@@ -567,6 +611,7 @@ lowerStatepointMetaArgs(SmallVectorImpl<SDValue> &Ops,
 
   // The vm state arguments are lowered in an opaque manner.  We do not know
   // what type of values are contained within.
+  LLVM_DEBUG(dbgs() << "Lowering deopt state\n");
   for (const Value *V : SI.DeoptState) {
     SDValue Incoming;
     // If this is a function argument at a static frame index, generate it as
@@ -578,6 +623,8 @@ lowerStatepointMetaArgs(SmallVectorImpl<SDValue> &Ops,
     }
     if (!Incoming.getNode())
       Incoming = Builder.getValue(V);
+    LLVM_DEBUG(dbgs() << "Value " << *V
+                      << " requireSpillSlot = " << requireSpillSlot(V) << "\n");
     lowerIncomingStatepointValue(Incoming, requireSpillSlot(V), Ops, MemRefs,
                                  Builder);
   }
@@ -588,14 +635,15 @@ lowerStatepointMetaArgs(SmallVectorImpl<SDValue> &Ops,
   // it's (lowered) derived pointer.  i.e
   // (base[0], ptr[0], base[1], ptr[1], ...)
   for (unsigned i = 0; i < SI.Bases.size(); ++i) {
-    const Value *Base = SI.Bases[i];
-    lowerIncomingStatepointValue(Builder.getValue(Base),
-                                 /*RequireSpillSlot*/ true, Ops, MemRefs,
+    bool RequireSpillSlot;
+    SDValue Base = Builder.getValue(SI.Bases[i]);
+    RequireSpillSlot = !LowerAsVReg.count(Base);
+    lowerIncomingStatepointValue(Base, RequireSpillSlot, Ops, MemRefs,
                                  Builder);
 
-    const Value *Ptr = SI.Ptrs[i];
-    lowerIncomingStatepointValue(Builder.getValue(Ptr),
-                                 /*RequireSpillSlot*/ true, Ops, MemRefs,
+    SDValue Derived = Builder.getValue(SI.Ptrs[i]);
+    RequireSpillSlot = !LowerAsVReg.count(Derived);
+    lowerIncomingStatepointValue(Derived, RequireSpillSlot, Ops, MemRefs,
                                  Builder);
   }
 
@@ -630,7 +678,9 @@ lowerStatepointMetaArgs(SmallVectorImpl<SDValue> &Ops,
     SDValue SDV = Builder.getValue(V);
     SDValue Loc = Builder.StatepointLowering.getLocation(SDV);
 
-    if (Loc.getNode()) {
+    if (LowerAsVReg.count(SDV)) {
+      SpillMap[V] = None;
+    } else if (Loc.getNode()) {
       SpillMap[V] = cast<FrameIndexSDNode>(Loc)->getIndex();
     } else {
       // Record value as visited, but not spilled. This is case for allocas
@@ -665,6 +715,7 @@ SDValue SelectionDAGBuilder::LowerAsSTATEPOINT(
   assert(SI.Bases.size() == SI.Ptrs.size() &&
          SI.Ptrs.size() <= SI.GCRelocates.size());
 
+  LLVM_DEBUG(dbgs() << "Lowering statepoint " << *SI.StatepointInstr << "\n");
 #ifndef NDEBUG
   for (auto *Reloc : SI.GCRelocates)
     if (Reloc->getParent() == SI.StatepointInstr->getParent())
@@ -674,7 +725,9 @@ SDValue SelectionDAGBuilder::LowerAsSTATEPOINT(
   // Lower statepoint vmstate and gcstate arguments
   SmallVector<SDValue, 10> LoweredMetaArgs;
   SmallVector<MachineMemOperand*, 16> MemRefs;
-  lowerStatepointMetaArgs(LoweredMetaArgs, MemRefs, SI, *this);
+  // Maps derived pointer SDValue to statepoint result of relocated pointer.
+  DenseMap<SDValue, int> LowerAsVReg;
+  lowerStatepointMetaArgs(LoweredMetaArgs, MemRefs, LowerAsVReg, SI, *this);
 
   // Now that we've emitted the spills, we need to update the root so that the
   // call sequence is ordered correctly.
@@ -788,14 +841,35 @@ SDValue SelectionDAGBuilder::LowerAsSTATEPOINT(
 
   // Compute return values.  Provide a glue output since we consume one as
   // input.  This allows someone else to chain off us as needed.
-  SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
+  SmallVector<EVT, 8> NodeTys;
+  for (auto &Ptr : SI.Ptrs) {
+    SDValue SD = getValue(Ptr);
+    if (LowerAsVReg.count(SD)) {
+      NodeTys.push_back(SD.getValueType());
+    }
+  }
+  LLVM_DEBUG(dbgs() << "Statepoint has " << NodeTys.size() << " results\n");
+  assert(NodeTys.size() == LowerAsVReg.size() && "Inconsistent GC Ptr lowering");
+  NodeTys.push_back(MVT::Other);
+  NodeTys.push_back(MVT::Glue);
 
+  unsigned NumResults = NodeTys.size();
   MachineSDNode *StatepointMCNode =
     DAG.getMachineNode(TargetOpcode::STATEPOINT, getCurSDLoc(), NodeTys, Ops);
   DAG.setNodeMemRefs(StatepointMCNode, MemRefs);
 
   SDNode *SinkNode = StatepointMCNode;
 
+  // Fill mapping from derived pointer to statepoint result denoting its
+  // relocated value.
+  auto &DPtrMap = StatepointLowering.DerivedPtrMap;
+  for (const auto *Relocate : SI.GCRelocates) {
+    Value *Derived = Relocate->getDerivedPtr();
+    SDValue SD = getValue(Derived);
+    if (LowerAsVReg.count(SD))
+      DPtrMap[Derived] = SDValue(StatepointMCNode, LowerAsVReg[SD]);
+  }
+
   // Build the GC_TRANSITION_END node if necessary.
   //
   // See the comment above regarding GC_TRANSITION_START for the layout of
@@ -804,7 +878,7 @@ SDValue SelectionDAGBuilder::LowerAsSTATEPOINT(
     SmallVector<SDValue, 8> TEOps;
 
     // Add chain
-    TEOps.push_back(SDValue(StatepointMCNode, 0));
+    TEOps.push_back(SDValue(StatepointMCNode, NumResults - 2));
 
     // Add GC transition arguments
     for (const Value *V : SI.GCTransitionArgs) {
@@ -814,7 +888,7 @@ SDValue SelectionDAGBuilder::LowerAsSTATEPOINT(
     }
 
     // Add glue
-    TEOps.push_back(SDValue(StatepointMCNode, 1));
+    TEOps.push_back(SDValue(StatepointMCNode, NumResults - 1));
 
     SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
 
@@ -825,7 +899,12 @@ SDValue SelectionDAGBuilder::LowerAsSTATEPOINT(
   }
 
   // Replace original call
-  DAG.ReplaceAllUsesWith(CallNode, SinkNode); // This may update Root
+  // Call: ch,glue = CALL ...
+  // Statepoint: [gc relocates],ch,glue = STATEPOINT ...
+  unsigned NumSinkValues = SinkNode->getNumValues();
+  SDValue StatepointValues[2] = {SDValue(SinkNode, NumSinkValues - 2),
+                                 SDValue(SinkNode, NumSinkValues - 1)};
+  DAG.ReplaceAllUsesWith(CallNode, StatepointValues);
   // Remove original call node
   DAG.DeleteNode(CallNode);
 
@@ -927,7 +1006,7 @@ SelectionDAGBuilder::LowerStatepoint(const GCStatepointInst &I,
     setValue(&I, ReturnValue);
     return;
   }
-  
+
   // Result value will be used in a different basic block so we need to export
   // it now.  Default exporting mechanism will not work here because statepoint
   // call has a different type than the actual call. It means that by default
@@ -1010,12 +1089,13 @@ void SelectionDAGBuilder::visitGCResult(const GCResultInst &CI) {
 }
 
 void SelectionDAGBuilder::visitGCRelocate(const GCRelocateInst &Relocate) {
+  const BasicBlock *StatepointBB = Relocate.getStatepoint()->getParent();
 #ifndef NDEBUG
   // Consistency check
   // We skip this check for relocates not in the same basic block as their
   // statepoint. It would be too expensive to preserve validation info through
   // different basic blocks.
-  if (Relocate.getStatepoint()->getParent() == Relocate.getParent())
+  if (StatepointBB == Relocate.getParent())
     StatepointLowering.relocCallVisited(Relocate);
 
   auto *Ty = Relocate.getType()->getScalarType();
@@ -1033,6 +1113,16 @@ void SelectionDAGBuilder::visitGCRelocate(const GCRelocateInst &Relocate) {
     return;
   }
 
+  // Relocate is local to statepoint block and its pointer was assigned
+  // to VReg. Use corresponding statepoint result.
+  auto &DPtrMap = StatepointLowering.DerivedPtrMap;
+  auto It = DPtrMap.find(DerivedPtr);
+  if (It != DPtrMap.end()) {
+    setValue(&Relocate, It->second);
+    assert(Relocate.getParent() == StatepointBB && "unexpected DPtrMap entry");
+    return;
+  }
+
   auto &SpillMap = FuncInfo.StatepointSpillMaps[Relocate.getStatepoint()];
   auto SlotIt = SpillMap.find(DerivedPtr);
   assert(SlotIt != SpillMap.end() && "Relocating not lowered gc value");
diff --git a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.h b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.h
index 634ef87f3840e..d6c18379c5add 100644
--- a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.h
+++ b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.h
@@ -103,6 +103,10 @@ class StatepointLoweringState {
     return AllocatedStackSlots.test(Offset);
   }
 
+  /// For each statepoint keep mapping from original derived pointer to
+  /// the statepoint node result defining its new value.
+  DenseMap<const Value *, SDValue> DerivedPtrMap;
+
 private:
   /// Maps pre-relocation value (gc pointer directly incoming into statepoint)
   /// into it's location (currently only stack slots)
diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index 2c94c2c62e5f0..db4fcf7494c7f 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -1041,9 +1041,19 @@ TargetLoweringBase::emitPatchPoint(MachineInstr &InitialMI,
   // Inherit previous memory operands.
   MIB.cloneMemRefs(*MI);
 
-  for (auto &MO : MI->operands()) {
+  for (unsigned i = 0; i < MI->getNumOperands(); ++i) {
+    MachineOperand &MO = MI->getOperand(i);
     if (!MO.isFI()) {
+      // Index of Def operand this Use it tied to.
+      // Since Defs are coming before Uses, if Use is tied, then
+      // index of Def must be smaller that index of that Use.
+      // Also, Defs preserve their position in new MI.
+      unsigned TiedTo = i;
+      if (MO.isReg() && MO.isTied())
+        TiedTo = MI->findTiedOperandIdx(i);
       MIB.add(MO);
+      if (TiedTo < i)
+        MIB->tieOperands(TiedTo, MIB->getNumOperands() - 1);
       continue;
     }
 
diff --git a/llvm/test/CodeGen/X86/statepoint-vreg.ll b/llvm/test/CodeGen/X86/statepoint-vreg.ll
new file mode 100644
index 0000000000000..bb86e9e1f1cfb
--- /dev/null
+++ b/llvm/test/CodeGen/X86/statepoint-vreg.ll
@@ -0,0 +1,907 @@
+; This run is to demonstrate what MIR SSA looks like.
+; RUN: llc -max-registers-for-gc-values=4 -stop-after finalize-isel < %s | FileCheck --check-prefix=CHECK-VREG %s
+; This run is to demonstrate register allocator work.
+; RUN: llc -max-registers-for-gc-values=4 -stop-after virtregrewriter < %s | FileCheck --check-prefix=CHECK-PREG %s
+; This run is to demonstrate resulting assembly/stackmaps.
+; NOTE: When D81647 is landed this run line will need to be adjusted!
+; RUN: llc -max-registers-for-gc-values=4 < %s | FileCheck --check-prefix=CHECK-ASM %s
+
+target datalayout = "e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-linux-gnu"
+
+declare i1 @return_i1()
+declare void @func()
+declare void @consume(i32 addrspace(1)*)
+declare void @consume2(i32 addrspace(1)*, i32 addrspace(1)*)
+declare void @consume5(i32 addrspace(1)*, i32 addrspace(1)*, i32 addrspace(1)*, i32 addrspace(1)*, i32 addrspace(1)*)
+declare void @use1(i32 addrspace(1)*, i8 addrspace(1)*)
+
+; test most simple relocate
+define i1 @test_relocate(i32 addrspace(1)* %a) gc "statepoint-example" {
+; CHECK-VREG-LABEL: name:            test_relocate
+; CHECK-VREG:    %0:gr64 = COPY $rdi
+; CHECK-VREG:    %1:gr64 = STATEPOINT 0, 0, 0, @return_i1, 2, 0, 2, 0, 2, 0, %0, %0(tied-def 0), csr_64, implicit-def $rsp, implicit-def $ssp, implicit-def $al
+; CHECK-VREG:    %2:gr8 = COPY $al
+; CHECK-VREG:    $rdi = COPY %1
+; CHECK-VREG:    CALL64pcrel32 @consume, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit-def $rsp, implicit-def $ssp
+
+; CHECK-PREG-LABEL: name:            test_relocate
+; CHECK-PREG:    renamable $rbx = COPY $rdi
+; CHECK-PREG:    renamable $rbx = STATEPOINT 0, 0, 0, @return_i1, 2, 0, 2, 0, 2, 0, killed renamable $rbx, renamable $rbx(tied-def 0), csr_64, implicit-def $rsp, implicit-def $ssp, implicit-def $al
+; CHECK-PREG:    renamable $bpl = COPY killed $al
+; CHECK-PREG:    $rdi = COPY killed renamable $rbx
+; CHECK-PREG:    CALL64pcrel32 @consume, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit-def $rsp, implicit-def $ssp
+
+; CHECK-ASM-LABEL: test_relocate:
+; CHECK-ASM:       # %bb.0:
+; CHECK-ASM-NEXT:	pushq	%rbp
+; CHECK-ASM-NEXT:	.cfi_def_cfa_offset 16
+; CHECK-ASM-NEXT:	pushq	%rbx
+; CHECK-ASM-NEXT:	.cfi_def_cfa_offset 24
+; CHECK-ASM-NEXT:	pushq	%rax
+; CHECK-ASM-NEXT:	.cfi_def_cfa_offset 32
+; CHECK-ASM-NEXT:	.cfi_offset %rbx, -24
+; CHECK-ASM-NEXT:	.cfi_offset %rbp, -16
+; CHECK-ASM-NEXT:	movq	%rdi, %rbx
+; CHECK-ASM-NEXT:	callq	return_i1
+; CHECK-ASM-NEXT:  .Ltmp0:
+; CHECK-ASM-NEXT:	movl	%eax, %ebp
+; CHECK-ASM-NEXT:	movq	%rbx, %rdi
+; CHECK-ASM-NEXT:	callq	consume
+; CHECK-ASM-NEXT:	movl	%ebp, %eax
+; CHECK-ASM-NEXT:	addq	$8, %rsp
+; CHECK-ASM-NEXT:	.cfi_def_cfa_offset 24
+; CHECK-ASM-NEXT:	popq	%rbx
+; CHECK-ASM-NEXT:	.cfi_def_cfa_offset 16
+; CHECK-ASM-NEXT:	popq	%rbp
+; CHECK-ASM-NEXT:	.cfi_def_cfa_offset 8
+; CHECK-ASM-NEXT:	retq
+entry:
+  %safepoint_token = tail call token (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0) ["gc-live" (i32 addrspace(1)* %a)]
+  %rel1 = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 0, i32 0)
+  %res1 = call zeroext i1 @llvm.experimental.gc.result.i1(token %safepoint_token)
+  call void @consume(i32 addrspace(1)* %rel1)
+  ret i1 %res1
+}
+; test pointer variables intermixed with pointer constants
+define void @test_mixed(i32 addrspace(1)* %a, i32 addrspace(1)* %b, i32 addrspace(1)* %c) gc "statepoint-example" {
+; CHECK-VREG-LABEL: name:            test_mixed
+; CHECK-VREG:    %2:gr64 = COPY $rdx
+; CHECK-VREG:    %1:gr64 = COPY $rsi
+; CHECK-VREG:    %0:gr64 = COPY $rdi
+; CHECK-VREG:    %3:gr64, %4:gr64, %5:gr64 = STATEPOINT 0, 0, 0, @func, 2, 0, 2, 0, 2, 0, %2, %2(tied-def 0), 2, 0, 2, 0, %1, %1(tied-def 1), %0, %0(tied-def 2), csr_64
+; CHECK-VREG:    %6:gr32 = MOV32r0 implicit-def dead $eflags
+; CHECK-VREG:    %7:gr64 = SUBREG_TO_REG 0, killed %6, %subreg.sub_32bit
+; CHECK-VREG:    $rdi = COPY %5
+; CHECK-VREG:    $rsi = COPY %7
+; CHECK-VREG:    $rdx = COPY %4
+; CHECK-VREG:    $rcx = COPY %7
+; CHECK-VREG:    $r8 = COPY %3
+; CHECK-VREG:    CALL64pcrel32 @consume5, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit $rsi, implicit $rdx, implicit $rcx, implicit $r8, implicit-def $rsp, implicit-def $ssp
+
+; CHECK-PREG-LABEL: name:            test_mixed
+; CHECK-PREG:    renamable $r14 = COPY $rdx
+; CHECK-PREG:    renamable $r15 = COPY $rsi
+; CHECK-PREG:    renamable $rbx = COPY $rdi
+; CHECK-PREG:    renamable $r14, renamable $r15, renamable $rbx = STATEPOINT 0, 0, 0, @func, 2, 0, 2, 0, 2, 0, killed renamable $r14, renamable $r14(tied-def 0), 2, 0, 2, 0, killed renamable $r15, renamable $r15(tied-def 1), killed renamable $rbx, renamable $rbx(tied-def 2), csr_64, implicit-def $rsp, implicit-def $ssp
+; CHECK-PREG:    $rdi = COPY killed renamable $rbx
+; CHECK-PREG:    dead $esi = MOV32r0 implicit-def dead $eflags, implicit-def $rsi
+; CHECK-PREG:    $rdx = COPY killed renamable $r15
+; CHECK-PREG:    dead $ecx = MOV32r0 implicit-def dead $eflags, implicit-def $rcx
+; CHECK-PREG:    $r8 = COPY killed renamable $r14
+; CHECK-PREG:    CALL64pcrel32 @consume5, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit $rsi, implicit $rdx, implicit killed $rcx, implicit killed $r8, implicit-def $rsp, implicit-def $ssp
+
+; CHECK-ASM-LABEL: test_mixed:
+; CHECK-ASM:        # %bb.0:                                # %entry
+; CHECK-ASM-NEXT:	pushq	%r15
+; CHECK-ASM-NEXT:	.cfi_def_cfa_offset 16
+; CHECK-ASM-NEXT:	pushq	%r14
+; CHECK-ASM-NEXT:	.cfi_def_cfa_offset 24
+; CHECK-ASM-NEXT:	pushq	%rbx
+; CHECK-ASM-NEXT:	.cfi_def_cfa_offset 32
+; CHECK-ASM-NEXT:	.cfi_offset %rbx, -32
+; CHECK-ASM-NEXT:	.cfi_offset %r14, -24
+; CHECK-ASM-NEXT:	.cfi_offset %r15, -16
+; CHECK-ASM-NEXT:	movq	%rdx, %r14
+; CHECK-ASM-NEXT:	movq	%rsi, %r15
+; CHECK-ASM-NEXT:	movq	%rdi, %rbx
+; CHECK-ASM-NEXT:	callq	func
+; CHECK-ASM-NEXT:.Ltmp1:
+; CHECK-ASM-NEXT:	movq	%rbx, %rdi
+; CHECK-ASM-NEXT:	xorl	%esi, %esi
+; CHECK-ASM-NEXT:	movq	%r15, %rdx
+; CHECK-ASM-NEXT:	xorl	%ecx, %ecx
+; CHECK-ASM-NEXT:	movq	%r14, %r8
+; CHECK-ASM-NEXT:	callq	consume5
+; CHECK-ASM-NEXT:	popq	%rbx
+; CHECK-ASM-NEXT:	.cfi_def_cfa_offset 24
+; CHECK-ASM-NEXT:	popq	%r14
+; CHECK-ASM-NEXT:	.cfi_def_cfa_offset 16
+; CHECK-ASM-NEXT:	popq	%r15
+; CHECK-ASM-NEXT:	.cfi_def_cfa_offset 8
+; CHECK-ASM-NEXT:	retq
+entry:
+  %safepoint_token = tail call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0) ["gc-live" (i32 addrspace(1)* %a, i32 addrspace(1)* null, i32 addrspace(1)* %b, i32 addrspace(1)* null, i32 addrspace(1)* %c)]
+  %rel1 = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 0, i32 0)
+  %rel2 = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 1, i32 1)
+  %rel3 = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 2, i32 2)
+  %rel4 = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 3, i32 3)
+  %rel5 = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 4, i32 4)
+  call void @consume5(i32 addrspace(1)* %rel1, i32 addrspace(1)* %rel2, i32 addrspace(1)* %rel3, i32 addrspace(1)* %rel4, i32 addrspace(1)* %rel5)
+  ret void
+}
+
+; same as above, but for alloca
+define i32 addrspace(1)* @test_alloca(i32 addrspace(1)* %ptr) gc "statepoint-example" {
+; CHECK-VREG-LABEL: name:            test_alloca
+; CHECK-VREG:    %0:gr64 = COPY $rdi
+; CHECK-VREG:    MOV64mr %stack.0.alloca, 1, $noreg, 0, $noreg, %0 :: (store 8 into %ir.alloca)
+; CHECK-VREG:    %1:gr64 = STATEPOINT 0, 0, 0, @return_i1, 2, 0, 2, 0, 2, 0, %0, %0(tied-def 0), 0, %stack.0.alloca, 0, csr_64, implicit-def $rsp, implicit-def $ssp, implicit-def $al :: (volatile load store 8 on %stack.0.alloca)
+; CHECK-VREG:    %2:gr8 = COPY $al
+; CHECK-VREG:    %3:gr64 = MOV64rm %stack.0.alloca, 1, $noreg, 0, $noreg :: (dereferenceable load 8 from %ir.alloca)
+; CHECK-VREG:    $rdi = COPY %1
+; CHECK-VREG:    CALL64pcrel32 @consume, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit-def $rsp, implicit-def $ssp
+
+; CHECK-PREG-LABEL: name:            test_alloca
+; CHECK-PREG:    renamable $rbx = COPY $rdi
+; CHECK-PREG:    MOV64mr %stack.0.alloca, 1, $noreg, 0, $noreg, renamable $rbx :: (store 8 into %ir.alloca)
+; CHECK-PREG:    renamable $rbx = STATEPOINT 0, 0, 0, @return_i1, 2, 0, 2, 0, 2, 0, killed renamable $rbx, renamable $rbx(tied-def 0), 0, %stack.0.alloca, 0, csr_64, implicit-def $rsp, implicit-def $ssp, implicit-def dead $al :: (volatile load store 8 on %stack.0.alloca)
+; CHECK-PREG:    renamable $r14 = MOV64rm %stack.0.alloca, 1, $noreg, 0, $noreg :: (dereferenceable load 8 from %ir.alloca)
+; CHECK-PREG:    $rdi = COPY killed renamable $rbx
+; CHECK-PREG:    CALL64pcrel32 @consume, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit-def $rsp, implicit-def $ssp
+
+; CHECK-ASM-LABEL: test_alloca:
+; CHECK-ASM:       # %bb.0: # %entry
+; CHECK-ASM-NEXT:	pushq	%r14
+; CHECK-ASM-NEXT:	.cfi_def_cfa_offset 16
+; CHECK-ASM-NEXT:	pushq	%rbx
+; CHECK-ASM-NEXT:	.cfi_def_cfa_offset 24
+; CHECK-ASM-NEXT:	pushq	%rax
+; CHECK-ASM-NEXT:	.cfi_def_cfa_offset 32
+; CHECK-ASM-NEXT:	.cfi_offset %rbx, -24
+; CHECK-ASM-NEXT:	.cfi_offset %r14, -16
+; CHECK-ASM-NEXT:	movq	%rdi, %rbx
+; CHECK-ASM-NEXT:	movq	%rdi, (%rsp)
+; CHECK-ASM-NEXT:	callq	return_i1
+; CHECK-ASM-NEXT:  .Ltmp2:
+; CHECK-ASM-NEXT:	movq	(%rsp), %r14
+; CHECK-ASM-NEXT:	movq	%rbx, %rdi
+; CHECK-ASM-NEXT:	callq	consume
+; CHECK-ASM-NEXT:	movq	%r14, %rax
+; CHECK-ASM-NEXT:	addq	$8, %rsp
+; CHECK-ASM-NEXT:	.cfi_def_cfa_offset 24
+; CHECK-ASM-NEXT:	popq	%rbx
+; CHECK-ASM-NEXT:	.cfi_def_cfa_offset 16
+; CHECK-ASM-NEXT:	popq	%r14
+; CHECK-ASM-NEXT:	.cfi_def_cfa_offset 8
+; CHECK-ASM-NEXT:	retq
+entry:
+  %alloca = alloca i32 addrspace(1)*, align 8
+  store i32 addrspace(1)* %ptr, i32 addrspace(1)** %alloca
+  %safepoint_token = call token (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0) ["gc-live" (i32 addrspace(1)** %alloca, i32 addrspace(1)* %ptr)]
+  %rel1 = load i32 addrspace(1)*, i32 addrspace(1)** %alloca
+  %rel2 = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 1, i32 1)
+  call void @consume(i32 addrspace(1)* %rel2)
+  ret i32 addrspace(1)* %rel1
+}
+
+; test base != derived
+define void @test_base_derived(i32 addrspace(1)* %base, i32 addrspace(1)* %derived) gc "statepoint-example" {
+; CHECK-VREG-LABEL: name:            test_base_derived
+; CHECK-VREG:    %1:gr64 = COPY $rsi
+; CHECK-VREG:    %0:gr64 = COPY $rdi
+; CHECK-VREG:    MOV64mr %stack.0, 1, $noreg, 0, $noreg, %0 :: (store 8 into %stack.0)
+; CHECK-VREG:    %2:gr64 = STATEPOINT 0, 0, 0, @func, 2, 0, 2, 0, 2, 0, 1, 8, %stack.0, 0, %1(tied-def 0), csr_64, implicit-def $rsp, implicit-def $ssp :: (volatile load store 8 on %stack.0)
+; CHECK-VREG:    $rdi = COPY %2
+; CHECK-VREG:    CALL64pcrel32 @consume, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit-def $rsp, implicit-def $ssp
+
+; CHECK-PREG-LABEL: name:            test_base_derived
+; CHECK-PREG:    renamable $rbx = COPY $rsi
+; CHECK-PREG:    MOV64mr %stack.0, 1, $noreg, 0, $noreg, killed renamable $rdi :: (store 8 into %stack.0)
+; CHECK-PREG:    renamable $rbx = STATEPOINT 0, 0, 0, @func, 2, 0, 2, 0, 2, 0, 1, 8, %stack.0, 0, killed renamable $rbx(tied-def 0), csr_64, implicit-def $rsp, implicit-def $ssp :: (volatile load store 8 on %stack.0)
+; CHECK-PREG:    $rdi = COPY killed renamable $rbx
+; CHECK-PREG:    CALL64pcrel32 @consume, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit-def $rsp, implicit-def $ssp
+
+; CHECK-ASM-LABEL: test_base_derived:
+; CHECK-ASM:       # %bb.0:
+; CHECK-ASM-NEXT:	pushq	%rbx
+; CHECK-ASM-NEXT:	.cfi_def_cfa_offset 16
+; CHECK-ASM-NEXT:	subq	$16, %rsp
+; CHECK-ASM-NEXT:	.cfi_def_cfa_offset 32
+; CHECK-ASM-NEXT:	.cfi_offset %rbx, -16
+; CHECK-ASM-NEXT:	movq	%rsi, %rbx
+; CHECK-ASM-NEXT:	movq	%rdi, 8(%rsp)
+; CHECK-ASM-NEXT:	callq	func
+; CHECK-ASM-NEXT:  .Ltmp3:
+; CHECK-ASM-NEXT:	movq	%rbx, %rdi
+; CHECK-ASM-NEXT:	callq	consume
+; CHECK-ASM-NEXT:	addq	$16, %rsp
+; CHECK-ASM-NEXT:	.cfi_def_cfa_offset 16
+; CHECK-ASM-NEXT:	popq	%rbx
+; CHECK-ASM-NEXT:	.cfi_def_cfa_offset 8
+; CHECK-ASM-NEXT:	retq
+  %safepoint_token = tail call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0) ["gc-live" (i32 addrspace(1)* %base, i32 addrspace(1)* %derived)]
+  %reloc = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 0, i32 1)
+  call void @consume(i32 addrspace(1)* %reloc)
+  ret void
+}
+
+; deopt GC pointer not present in GC args must be spilled
+define void @test_deopt_gcpointer(i32 addrspace(1)* %a, i32 addrspace(1)* %b) gc "statepoint-example" {
+; CHECK-VREG-LABEL: name:            test_deopt_gcpointer
+; CHECK-VREG:    %1:gr64 = COPY $rsi
+; CHECK-VREG:    %0:gr64 = COPY $rdi
+; CHECK-VREG:    MOV64mr %stack.0, 1, $noreg, 0, $noreg, %0 :: (store 8 into %stack.0)
+; CHECK-VREG:    %2:gr64 = STATEPOINT 0, 0, 0, @func, 2, 0, 2, 0, 2, 1, 1, 8, %stack.0, 0, %1, %1(tied-def 0), csr_64, implicit-def $rsp, implicit-def $ssp :: (volatile load store 8 on %stack.0)
+; CHECK-VREG:    $rdi = COPY %2
+; CHECK-VREG:    CALL64pcrel32 @consume, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit-def $rsp, implicit-def $ssp
+; CHECK-VREG:    RET 0
+
+; CHECK-PREG-LABEL: name:            test_deopt_gcpointer
+; CHECK-PREG:    renamable $rbx = COPY $rsi
+; CHECK-PREG:    MOV64mr %stack.0, 1, $noreg, 0, $noreg, killed renamable $rdi :: (store 8 into %stack.0)
+; CHECK-PREG:    renamable $rbx = STATEPOINT 0, 0, 0, @func, 2, 0, 2, 0, 2, 1, 1, 8, %stack.0, 0, killed renamable $rbx, renamable $rbx(tied-def 0), csr_64, implicit-def $rsp, implicit-def $ssp :: (volatile load store 8 on %stack.0)
+; CHECK-PREG:    $rdi = COPY killed renamable $rbx
+; CHECK-PREG:    CALL64pcrel32 @consume, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit-def $rsp, implicit-def $ssp
+
+; CHECK-ASM-LABEL: test_deopt_gcpointer:
+; CHECK-ASM:       # %bb.0:
+; CHECK-ASM-NEXT:	pushq	%rbx
+; CHECK-ASM-NEXT:	.cfi_def_cfa_offset 16
+; CHECK-ASM-NEXT:	subq	$16, %rsp
+; CHECK-ASM-NEXT:	.cfi_def_cfa_offset 32
+; CHECK-ASM-NEXT:	.cfi_offset %rbx, -16
+; CHECK-ASM-NEXT:	movq	%rsi, %rbx
+; CHECK-ASM-NEXT:	movq	%rdi, 8(%rsp)
+; CHECK-ASM-NEXT:	callq	func
+; CHECK-ASM-NEXT:  .Ltmp4:
+; CHECK-ASM-NEXT:	movq	%rbx, %rdi
+; CHECK-ASM-NEXT:	callq	consume
+; CHECK-ASM-NEXT:	addq	$16, %rsp
+; CHECK-ASM-NEXT:	.cfi_def_cfa_offset 16
+; CHECK-ASM-NEXT:	popq	%rbx
+; CHECK-ASM-NEXT:	.cfi_def_cfa_offset 8
+; CHECK-ASM-NEXT:	retq
+  %safepoint_token = tail call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0) ["deopt" (i32 addrspace(1)* %a), "gc-live" (i32 addrspace(1)* %b)]
+  %rel = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 0, i32 0)
+  call void @consume(i32 addrspace(1)* %rel)
+  ret void
+}
+
+;; Two gc.relocates of the same input, should require only a single spill/fill
+define void @test_gcrelocate_uniqueing(i32 addrspace(1)* %ptr) gc "statepoint-example" {
+; CHECK-VREG-LABEL: name:            test_gcrelocate_uniqueing
+; CHECK-VREG:    %0:gr64 = COPY $rdi
+; CHECK-VREG:    %1:gr64 = STATEPOINT 0, 0, 0, @func, 2, 0, 2, 0, 2, 2, %0, 2, 4278124286, %0, %0(tied-def 0), csr_64, implicit-def $rsp, implicit-def $ssp
+; CHECK-VREG:    $rdi = COPY %1
+; CHECK-VREG:    $rsi = COPY %1
+; CHECK-VREG:    CALL64pcrel32 @consume2, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit $rsi, implicit-def $rsp, implicit-def $ssp
+
+; CHECK-PREG-LABEL: name:            test_gcrelocate_uniqueing
+; CHECK-PREG:    renamable $rbx = COPY $rdi
+; CHECK-PREG:    renamable $rbx = STATEPOINT 0, 0, 0, @func, 2, 0, 2, 0, 2, 2, killed renamable $rbx, 2, 4278124286, renamable $rbx, renamable $rbx(tied-def 0), csr_64, implicit-def $rsp, implicit-def $ssp
+; CHECK-PREG:    $rdi = COPY renamable $rbx
+; CHECK-PREG:    $rsi = COPY killed renamable $rbx
+; CHECK-PREG:    CALL64pcrel32 @consume2, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit killed $rsi, implicit-def $rsp, implicit-def $ssp
+
+; CHECK-ASM-LABEL: test_gcrelocate_uniqueing:
+; CHECK-ASM:       # %bb.0:
+; CHECK-ASM-NEXT:	pushq	%rbx
+; CHECK-ASM-NEXT:	.cfi_def_cfa_offset 16
+; CHECK-ASM-NEXT:	.cfi_offset %rbx, -16
+; CHECK-ASM-NEXT:	movq	%rdi, %rbx
+; CHECK-ASM-NEXT:	callq	func
+; CHECK-ASM-NEXT: .Ltmp5:
+; CHECK-ASM-NEXT:	movq	%rbx, %rdi
+; CHECK-ASM-NEXT:	movq	%rbx, %rsi
+; CHECK-ASM-NEXT:	callq	consume2
+; CHECK-ASM-NEXT:	popq	%rbx
+; CHECK-ASM-NEXT:	.cfi_def_cfa_offset 8
+; CHECK-ASM-NEXT:	retq
+  %tok = tail call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0) ["deopt" (i32 addrspace(1)* %ptr, i32 undef), "gc-live" (i32 addrspace(1)* %ptr, i32 addrspace(1)* %ptr)]
+  %a = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %tok, i32 0, i32 0)
+  %b = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %tok, i32 1, i32 1)
+  call void @consume2(i32 addrspace(1)* %a, i32 addrspace(1)* %b)
+  ret void
+}
+
+; Two gc.relocates of a bitcasted pointer should only require a single spill/fill
+define void @test_gcptr_uniqueing(i32 addrspace(1)* %ptr) gc "statepoint-example" {
+; CHECK-VREG-LABEL: name:            test_gcptr_uniqueing
+; CHECK-VREG:    %0:gr64 = COPY $rdi
+; CHECK-VREG:    ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+; CHECK-VREG:    %1:gr64 = STATEPOINT 0, 0, 0, @func, 2, 0, 2, 0, 2, 2, %0, 2, 4278124286, %0, %0(tied-def 0), csr_64, implicit-def $rsp, implicit-def $ssp
+; CHECK-VREG:    ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+; CHECK-VREG:    ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+; CHECK-VREG:    $rdi = COPY %1
+; CHECK-VREG:    $rsi = COPY %1
+; CHECK-VREG:    CALL64pcrel32 @use1, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit $rsi, implicit-def $rsp, implicit-def $ssp
+
+; CHECK-PREG-LABEL: name:            test_gcptr_uniqueing
+; CHECK-PREG:    renamable $rbx = COPY $rdi
+; CHECK-PREG:    renamable $rbx = STATEPOINT 0, 0, 0, @func, 2, 0, 2, 0, 2, 2, killed renamable $rbx, 2, 4278124286, renamable $rbx, renamable $rbx(tied-def 0), csr_64, implicit-def $rsp, implicit-def $ssp
+; CHECK-PREG:    $rdi = COPY renamable $rbx
+; CHECK-PREG:    $rsi = COPY killed renamable $rbx
+; CHECK-PREG:    CALL64pcrel32 @use1, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit killed $rsi, implicit-def $rsp, implicit-def $ssp
+
+; CHECK-ASM-LABEL: test_gcptr_uniqueing:
+; CHECK-ASM:       # %bb.0:
+; CHECK-ASM-NEXT:	pushq	%rbx
+; CHECK-ASM-NEXT:	.cfi_def_cfa_offset 16
+; CHECK-ASM-NEXT:	.cfi_offset %rbx, -16
+; CHECK-ASM-NEXT:	movq	%rdi, %rbx
+; CHECK-ASM-NEXT:	callq	func
+; CHECK-ASM-NEXT:  .Ltmp6:
+; CHECK-ASM-NEXT:	movq	%rbx, %rdi
+; CHECK-ASM-NEXT:	movq	%rbx, %rsi
+; CHECK-ASM-NEXT:	callq	use1
+; CHECK-ASM-NEXT:	popq	%rbx
+; CHECK-ASM-NEXT:	.cfi_def_cfa_offset 8
+; CHECK-ASM-NEXT:	retq
+  %ptr2 = bitcast i32 addrspace(1)* %ptr to i8 addrspace(1)*
+  %tok = tail call token (i64, i32, void ()*, i32, i32, ...)
+      @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0) ["deopt" (i32 addrspace(1)* %ptr, i32 undef), "gc-live" (i32 addrspace(1)* %ptr, i8 addrspace(1)* %ptr2)]
+  %a = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %tok, i32 0, i32 0)
+  %b = call i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %tok, i32 1, i32 1)
+  call void @use1(i32 addrspace(1)* %a, i8 addrspace(1)* %b)
+  ret void
+}
+
+;
+; Cross-basicblock relocates are handled with spilling for now.
+; No need to check post-RA output
+define i1 @test_cross_bb(i32 addrspace(1)* %a, i1 %external_cond) gc "statepoint-example" {
+; CHECK-VREG-LABEL: name:            test_cross_bb
+; CHECK-VREG:  bb.0.entry:
+; CHECK-VREG:         %1:gr32 = COPY $esi
+; CHECK-VREG-NEXT:    %0:gr64 = COPY $rdi
+; CHECK-VREG-NEXT:    %3:gr8 = COPY %1.sub_8bit
+; CHECK-VREG-NEXT:    MOV64mr %stack.0, 1, $noreg, 0, $noreg, %0 :: (store 8 into %stack.0)
+; CHECK-VREG-NEXT:    ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+; CHECK-VREG-NEXT:    STATEPOINT 0, 0, 0, @return_i1, 2, 0, 2, 0, 2, 0, 1, 8, %stack.0, 0, 1, 8, %stack.0, 0, csr_64, implicit-def $rsp, implicit-def $ssp, implicit-def $al :: (volatile load store 8 on %stack.0)
+; CHECK-VREG-NEXT:    ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+; CHECK-VREG-NEXT:    %4:gr8 = COPY $al
+; CHECK-VREG-NEXT:    %2:gr8 = COPY %4
+; CHECK-VREG-NEXT:    TEST8ri killed %3, 1, implicit-def $eflags
+; CHECK-VREG-NEXT:    JCC_1 %bb.2, 4, implicit $eflags
+; CHECK-VREG-NEXT:    JMP_1 %bb.1
+; CHECK-VREG:       bb.1.left:
+; CHECK-VREG-NEXT:    %6:gr64 = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load 8 from %stack.0)
+; CHECK-VREG-NEXT:    ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+; CHECK-VREG-NEXT:    $rdi = COPY %6
+; CHECK-VREG-NEXT:    CALL64pcrel32 @consume, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit-def $rsp, implicit-def $ssp
+; CHECK-VREG-NEXT:    ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+; CHECK-VREG-NEXT:    $al = COPY %2
+; CHECK-VREG-NEXT:    RET 0, $al
+; CHECK-VREG:       bb.2.right:
+; CHECK-VREG-NEXT:    %5:gr8 = MOV8ri 1
+; CHECK-VREG-NEXT:    $al = COPY %5
+; CHECK-VREG-NEXT:    RET 0, $al
+
+entry:
+  %safepoint_token = tail call token (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0) ["gc-live" (i32 addrspace(1)* %a)]
+  br i1 %external_cond, label %left, label %right
+
+left:
+  %call1 = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 0, i32 0)
+  %call2 = call zeroext i1 @llvm.experimental.gc.result.i1(token %safepoint_token)
+  call void @consume(i32 addrspace(1)* %call1)
+  ret i1 %call2
+
+right:
+  ret i1 true
+}
+
+; No need to check post-regalloc output as it is the same
+define i1 @duplicate_reloc() gc "statepoint-example" {
+; CHECK-VREG-LABEL: name:            duplicate_reloc
+; CHECK-VREG:  bb.0.entry:
+; CHECK-VREG:    STATEPOINT 0, 0, 0, @func, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, csr_64, implicit-def $rsp, implicit-def $ssp
+; CHECK-VREG:    STATEPOINT 0, 0, 0, @func, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, csr_64, implicit-def $rsp, implicit-def $ssp
+; CHECK-VREG:    %0:gr8 = MOV8ri 1
+; CHECK-VREG:    $al = COPY %0
+; CHECK-VREG:    RET 0, $al
+
+; CHECK-ASM-LABEL: duplicate_reloc:
+; CHECK-ASM:       # %bb.0: # %entry
+; CHECK-ASM-NEXT:          pushq	%rax
+; CHECK-ASM-NEXT:          .cfi_def_cfa_offset 16
+; CHECK-ASM-NEXT:          callq	func
+; CHECK-ASM-NEXT:  .Ltmp8:
+; CHECK-ASM-NEXT:          callq	func
+; CHECK-ASM-NEXT:  .Ltmp9:
+; CHECK-ASM-NEXT:          movb	$1, %al
+; CHECK-ASM-NEXT:          popq	%rcx
+; CHECK-ASM-NEXT:          .cfi_def_cfa_offset 8
+; CHECK-ASM-NEXT:          retq
+entry:
+  %safepoint_token = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0) ["gc-live" (i32 addrspace(1)* null, i32 addrspace(1)* null)]
+  %base = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 0, i32 0)
+  %derived = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 0, i32 1)
+  %safepoint_token2 = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0) ["gc-live" (i32 addrspace(1)* %base, i32 addrspace(1)* %derived)]
+  %base_reloc = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token2,  i32 0, i32 0)
+  %derived_reloc = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token2,  i32 0, i32 1)
+  %cmp1 = icmp eq i32 addrspace(1)* %base_reloc, null
+  %cmp2 = icmp eq i32 addrspace(1)* %derived_reloc, null
+  %cmp = and i1 %cmp1, %cmp2
+  ret i1 %cmp
+}
+
+; Vectors cannot go in VRegs
+; No need to check post-regalloc output as it is lowered using old scheme
+define <2 x i8 addrspace(1)*> @test_vector(<2 x i8 addrspace(1)*> %obj) gc "statepoint-example" {
+; CHECK-VREG-LABEL: name:            test_vector
+; CHECK-VREG:    %0:vr128 = COPY $xmm0
+; CHECK-VREG:    MOVAPSmr %stack.0, 1, $noreg, 0, $noreg, %0 :: (store 16 into %stack.0)
+; CHECK-VREG:    STATEPOINT 0, 0, 0, @func, 2, 0, 2, 0, 2, 0, 1, 16, %stack.0, 0, 1, 16, %stack.0, 0, csr_64, implicit-def $rsp, implicit-def $ssp :: (volatile load store 16 on %stack.0)
+; CHECK-VREG:    %1:vr128 = MOVAPSrm %stack.0, 1, $noreg, 0, $noreg :: (load 16 from %stack.0)
+; CHECK-VREG:    $xmm0 = COPY %1
+; CHECK-VREG:    RET 0, $xmm0
+
+; CHECK-ASM-LABEL: test_vector:
+; CHECK-ASM:       # %bb.0: # %entry
+; CHECK-ASM-NEXT:          subq	$24, %rsp
+; CHECK-ASM-NEXT:          .cfi_def_cfa_offset 32
+; CHECK-ASM-NEXT:          movaps	%xmm0, (%rsp)
+; CHECK-ASM-NEXT:          callq	func
+; CHECK-ASM-NEXT:  .Ltmp10:
+; CHECK-ASM-NEXT:          movaps	(%rsp), %xmm0
+; CHECK-ASM-NEXT:          addq	$24, %rsp
+; CHECK-ASM-NEXT:          .cfi_def_cfa_offset 8
+; CHECK-ASM-NEXT:          retq
+entry:
+  %safepoint_token = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0) ["gc-live" (<2 x i8 addrspace(1)*> %obj)]
+  %obj.relocated = call coldcc <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token %safepoint_token, i32 0, i32 0) ; (%obj, %obj)
+  ret <2 x i8 addrspace(1)*> %obj.relocated
+}
+
+
+; test limit on amount of vregs
+define void @test_limit(i32 addrspace(1)* %a, i32 addrspace(1)* %b, i32 addrspace(1)* %c, i32 addrspace(1)* %d, i32 addrspace(1)*  %e) gc "statepoint-example" {
+; CHECK-VREG-LABEL: name:            test_limit
+; CHECK-VREG:    %4:gr64 = COPY $r8
+; CHECK-VREG:    %3:gr64 = COPY $rcx
+; CHECK-VREG:    %2:gr64 = COPY $rdx
+; CHECK-VREG:    %1:gr64 = COPY $rsi
+; CHECK-VREG:    %0:gr64 = COPY $rdi
+; CHECK-VREG:    MOV64mr %stack.0, 1, $noreg, 0, $noreg, %0 :: (store 8 into %stack.0)
+; CHECK-VREG:    %5:gr64, %6:gr64, %7:gr64, %8:gr64 = STATEPOINT 0, 0, 0, @func, 2, 0, 2, 0, 2, 0, %4, %4(tied-def 0), %3, %3(tied-def 1), %2, %2(tied-def 2), %1, %1(tied-def 3), 1, 8, %stack.0, 0, 1, 8, %stack.0, 0, csr_64, implicit-def $rsp, implicit-def $ssp :: (volatile load store 8 on %stack.0)
+; CHECK-VREG:    %9:gr64 = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load 8 from %stack.0)
+; CHECK-VREG:    $rdi = COPY %9
+; CHECK-VREG:    $rsi = COPY %8
+; CHECK-VREG:    $rdx = COPY %7
+; CHECK-VREG:    $rcx = COPY %6
+; CHECK-VREG:    $r8 = COPY %5
+; CHECK-VREG:    CALL64pcrel32 @consume5, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit $rsi, implicit $rdx, implicit $rcx, implicit $r8, implicit-def $rsp, implicit-def $ssp
+; CHECK-VREG:    RET 0
+entry:
+  %safepoint_token = tail call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0) ["gc-live" (i32 addrspace(1)* %a, i32 addrspace(1)* %b, i32 addrspace(1)* %c, i32 addrspace(1)* %d, i32 addrspace(1)* %e)]
+  %rel1 = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 0, i32 0)
+  %rel2 = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 1, i32 1)
+  %rel3 = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 2, i32 2)
+  %rel4 = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 3, i32 3)
+  %rel5 = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 4, i32 4)
+  call void @consume5(i32 addrspace(1)* %rel1, i32 addrspace(1)* %rel2, i32 addrspace(1)* %rel3, i32 addrspace(1)* %rel4, i32 addrspace(1)* %rel5)
+  ret void
+}
+
+declare token @llvm.experimental.gc.statepoint.p0f_i1f(i64, i32, i1 ()*, i32, i32, ...)
+declare token @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...)
+declare i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token, i32, i32)
+declare i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token, i32, i32)
+declare <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token, i32, i32)
+declare i1 @llvm.experimental.gc.result.i1(token)
+
+; CHECK-ASM-LABEL: .section .llvm_stackmaps
+; CHECK-ASM-NEXT:  __LLVM_StackMaps:
+; Entry for test_relocate
+; CHECK-ASM:	        .quad	0
+; CHECK-ASM-NEXT:     	.long	.Ltmp0-test_relocate
+; CHECK-ASM-NEXT:	.short	0
+; Num locations
+; CHECK-ASM-NEXT:	.short	5
+; Location 1 Constant 0
+; CHECK-ASM-NEXT:	.byte	4
+; CHECK-ASM-NEXT:	.byte	0
+; CHECK-ASM-NEXT:	.short	8
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.long	0
+; Location 2 Constant 0
+; CHECK-ASM-NEXT:	.byte	4
+; CHECK-ASM-NEXT:	.byte	0
+; CHECK-ASM-NEXT:	.short	8
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.long	0
+; Location 3 Constant 0
+; CHECK-ASM-NEXT:	.byte	4
+; CHECK-ASM-NEXT:	.byte	0
+; CHECK-ASM-NEXT:	.short	8
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.long	0
+; Location 4 Register $rbx
+; CHECK-ASM-NEXT:	.byte	1
+; CHECK-ASM-NEXT:	.byte	0
+; CHECK-ASM-NEXT:	.short	8
+; CHECK-ASM-NEXT:	.short	3
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.long	0
+; Location 5 Register $rbx
+; CHECK-ASM-NEXT:	.byte	1
+; CHECK-ASM-NEXT:	.byte	0
+; CHECK-ASM-NEXT:	.short	8
+; CHECK-ASM-NEXT:	.short	3
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.long	0
+;  Entry for test_mixed
+; CHECK-ASM:     	.quad	0
+; CHECK-ASM-NEXT:	.long	.Ltmp1-test_mixed
+; CHECK-ASM-NEXT:	.short	0
+; Num locations
+; CHECK-ASM-NEXT:	.short	11
+; Location 1 Constant 0
+; CHECK-ASM-NEXT:	.byte	4
+; CHECK-ASM-NEXT:	.byte	0
+; CHECK-ASM-NEXT:	.short	8
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.long	0
+; Location 2 Constant 0
+; CHECK-ASM-NEXT:	.byte	4
+; CHECK-ASM-NEXT:	.byte	0
+; CHECK-ASM-NEXT:	.short	8
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.long	0
+; Location 3 Constant 0
+; CHECK-ASM-NEXT:	.byte	4
+; CHECK-ASM-NEXT:	.byte	0
+; CHECK-ASM-NEXT:	.short	8
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.long	0
+; Location 4 Register $r14
+; CHECK-ASM-NEXT:	.byte	1
+; CHECK-ASM-NEXT:	.byte	0
+; CHECK-ASM-NEXT:	.short	8
+; CHECK-ASM-NEXT:	.short	14
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.long	0
+; Location 5 Register $r14
+; CHECK-ASM-NEXT:	.byte	1
+; CHECK-ASM-NEXT:	.byte	0
+; CHECK-ASM-NEXT:	.short	8
+; CHECK-ASM-NEXT:	.short	14
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.long	0
+; Location 6 Constant 0
+; CHECK-ASM-NEXT:	.byte	4
+; CHECK-ASM-NEXT:	.byte	0
+; CHECK-ASM-NEXT:	.short	8
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.long	0
+; Location 7 Constant 0
+; CHECK-ASM-NEXT:	.byte	4
+; CHECK-ASM-NEXT:	.byte	0
+; CHECK-ASM-NEXT:	.short	8
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.long	0
+; Location 8 Register $r15
+; CHECK-ASM-NEXT:	.byte	1
+; CHECK-ASM-NEXT:	.byte	0
+; CHECK-ASM-NEXT:	.short	8
+; CHECK-ASM-NEXT:	.short	15
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.long	0
+; Location 9 Register $r15
+; CHECK-ASM-NEXT:	.byte	1
+; CHECK-ASM-NEXT:	.byte	0
+; CHECK-ASM-NEXT:	.short	8
+; CHECK-ASM-NEXT:	.short	15
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.long	0
+; Location 10 Register $rbx
+; CHECK-ASM-NEXT:	.byte	1
+; CHECK-ASM-NEXT:	.byte	0
+; CHECK-ASM-NEXT:	.short	8
+; CHECK-ASM-NEXT:	.short	3
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.long	0
+; Location 11 Register $rbx
+; CHECK-ASM-NEXT:	.byte	1
+; CHECK-ASM-NEXT:	.byte	0
+; CHECK-ASM-NEXT:	.short	8
+; CHECK-ASM-NEXT:	.short	3
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.long	0
+; Entry for test_alloca
+; CHECK-ASM:     	.quad	0
+; CHECK-ASM-NEXT:	.long	.Ltmp2-test_alloca
+; CHECK-ASM-NEXT:	.short	0
+; Num locations
+; CHECK-ASM-NEXT:	.short	6
+; Location 1 Constant 0
+; CHECK-ASM-NEXT:	.byte	4
+; CHECK-ASM-NEXT:	.byte	0
+; CHECK-ASM-NEXT:	.short	8
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.long	0
+; Location 2 Constant 0
+; CHECK-ASM-NEXT:	.byte	4
+; CHECK-ASM-NEXT:	.byte	0
+; CHECK-ASM-NEXT:	.short	8
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.long	0
+; Location 3 Constant 0
+; CHECK-ASM-NEXT:	.byte	4
+; CHECK-ASM-NEXT:	.byte	0
+; CHECK-ASM-NEXT:	.short	8
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.long	0
+; Location 4 Register $rbx
+; CHECK-ASM-NEXT:	.byte	1
+; CHECK-ASM-NEXT:	.byte	0
+; CHECK-ASM-NEXT:	.short	8
+; CHECK-ASM-NEXT:	.short	3
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.long	0
+; Location 5 Register $rbx
+; CHECK-ASM-NEXT:	.byte	1
+; CHECK-ASM-NEXT:	.byte	0
+; CHECK-ASM-NEXT:	.short	8
+; CHECK-ASM-NEXT:	.short	3
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.long	0
+; Location 6 Direct $rsp + 0
+; CHECK-ASM-NEXT:	.byte	2
+; CHECK-ASM-NEXT:	.byte	0
+; CHECK-ASM-NEXT:	.short	8
+; CHECK-ASM-NEXT:	.short	7
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.long	0
+; Entry for test_base_derive
+; CHECK-ASM:     	.quad	0
+; CHECK-ASM-NEXT:	.long	.Ltmp3-test_base_derived
+; CHECK-ASM-NEXT:	.short	0
+; Num locations
+; CHECK-ASM-NEXT:	.short	5
+; Location 1 Constant 0
+; CHECK-ASM-NEXT:	.byte	4
+; CHECK-ASM-NEXT:	.byte	0
+; CHECK-ASM-NEXT:	.short	8
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.long	0
+; Location 2 Constant 0
+; CHECK-ASM-NEXT:	.byte	4
+; CHECK-ASM-NEXT:	.byte	0
+; CHECK-ASM-NEXT:	.short	8
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.long	0
+; Location 3 Constant 0
+; CHECK-ASM-NEXT:	.byte	4
+; CHECK-ASM-NEXT:	.byte	0
+; CHECK-ASM-NEXT:	.short	8
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.long	0
+; Location 4 Indirect $rsp + 8
+; CHECK-ASM-NEXT:	.byte	3
+; CHECK-ASM-NEXT:	.byte	0
+; CHECK-ASM-NEXT:	.short	8
+; CHECK-ASM-NEXT:	.short	7
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.long	8
+; Location 5 Register $rbx
+; CHECK-ASM-NEXT:	.byte	1
+; CHECK-ASM-NEXT:	.byte	0
+; CHECK-ASM-NEXT:	.short	8
+; CHECK-ASM-NEXT:	.short	3
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.long	0
+; Entry for test_deopt_gcpointer
+; CHECK-ASM:     	.quad	0
+; CHECK-ASM-NEXT:	.long	.Ltmp4-test_deopt_gcpointer
+; CHECK-ASM-NEXT:	.short	0
+; Num locations
+; CHECK-ASM-NEXT:	.short	6
+; Location 1 Constant 0
+; CHECK-ASM-NEXT:	.byte	4
+; CHECK-ASM-NEXT:	.byte	0
+; CHECK-ASM-NEXT:	.short	8
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.long	0
+; Location 2 Constant 0
+; CHECK-ASM-NEXT:	.byte	4
+; CHECK-ASM-NEXT:	.byte	0
+; CHECK-ASM-NEXT:	.short	8
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.long	0
+; Location 3 Constant 1
+; CHECK-ASM-NEXT:	.byte	4
+; CHECK-ASM-NEXT:	.byte	0
+; CHECK-ASM-NEXT:	.short	8
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.long	1
+; Location 4Indirect $rsp + 8
+; CHECK-ASM-NEXT:	.byte	3
+; CHECK-ASM-NEXT:	.byte	0
+; CHECK-ASM-NEXT:	.short	8
+; CHECK-ASM-NEXT:	.short	7
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.long	8
+; Location 5 Register $rbx
+; CHECK-ASM-NEXT:	.byte	1
+; CHECK-ASM-NEXT:	.byte	0
+; CHECK-ASM-NEXT:	.short	8
+; CHECK-ASM-NEXT:	.short	3
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.long	0
+; Location 6
+; CHECK-ASM-NEXT:	.byte	1
+; CHECK-ASM-NEXT:	.byte	0
+; CHECK-ASM-NEXT:	.short	8
+; CHECK-ASM-NEXT:	.short	3
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.long	0
+; Entry for test_gcrelocate_uniqueing
+; CHECK-ASM:     	.quad	0
+; CHECK-ASM-NEXT:	.long	.Ltmp5-test_gcrelocate_uniqueing
+; CHECK-ASM-NEXT:	.short	0
+; Num locations
+; CHECK-ASM-NEXT:	.short	7
+; Location 1 Constant 0
+; CHECK-ASM-NEXT:	.byte	4
+; CHECK-ASM-NEXT:	.byte	0
+; CHECK-ASM-NEXT:	.short	8
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.long	0
+; Location 2 Constant 0
+; CHECK-ASM-NEXT:	.byte	4
+; CHECK-ASM-NEXT:	.byte	0
+; CHECK-ASM-NEXT:	.short	8
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.long	0
+; Location 3 Constant 2
+; CHECK-ASM-NEXT:	.byte	4
+; CHECK-ASM-NEXT:	.byte	0
+; CHECK-ASM-NEXT:	.short	8
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.long	2
+; Location 4 Register $rbx
+; CHECK-ASM-NEXT:	.byte	1
+; CHECK-ASM-NEXT:	.byte	0
+; CHECK-ASM-NEXT:	.short	8
+; CHECK-ASM-NEXT:	.short	3
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.long	0
+; Location 5 Constant Index 0
+; CHECK-ASM-NEXT:	.byte	5
+; CHECK-ASM-NEXT:	.byte	0
+; CHECK-ASM-NEXT:	.short	8
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.long	0
+; Location 6 Register $rbx
+; CHECK-ASM-NEXT:	.byte	1
+; CHECK-ASM-NEXT:	.byte	0
+; CHECK-ASM-NEXT:	.short	8
+; CHECK-ASM-NEXT:	.short	3
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.long	0
+; Location 7 Register $rbx
+; CHECK-ASM-NEXT:	.byte	1
+; CHECK-ASM-NEXT:	.byte	0
+; CHECK-ASM-NEXT:	.short	8
+; CHECK-ASM-NEXT:	.short	3
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.long	0
+; Entry for test_gcptr_uniqueing
+; CHECK-ASM:     	.long	.Ltmp6-test_gcptr_uniqueing
+; CHECK-ASM-NEXT:	.short	0
+; Num locations
+; CHECK-ASM-NEXT:	.short	7
+; Location 1 Constant 0
+; CHECK-ASM-NEXT:	.byte	4
+; CHECK-ASM-NEXT:	.byte	0
+; CHECK-ASM-NEXT:	.short	8
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.long	0
+; Location 2 Constant 0
+; CHECK-ASM-NEXT:	.byte	4
+; CHECK-ASM-NEXT:	.byte	0
+; CHECK-ASM-NEXT:	.short	8
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.long	0
+; Location 3 Constant 2
+; CHECK-ASM-NEXT:	.byte	4
+; CHECK-ASM-NEXT:	.byte	0
+; CHECK-ASM-NEXT:	.short	8
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.long	2
+; Location 4 Register $rbx
+; CHECK-ASM-NEXT:	.byte	1
+; CHECK-ASM-NEXT:	.byte	0
+; CHECK-ASM-NEXT:	.short	8
+; CHECK-ASM-NEXT:	.short	3
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.long	0
+; Location 5 Constant Index 0
+; CHECK-ASM-NEXT:	.byte	5
+; CHECK-ASM-NEXT:	.byte	0
+; CHECK-ASM-NEXT:	.short	8
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.long	0
+; Location 6 Register $rbx
+; CHECK-ASM-NEXT:	.byte	1
+; CHECK-ASM-NEXT:	.byte	0
+; CHECK-ASM-NEXT:	.short	8
+; CHECK-ASM-NEXT:	.short	3
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.long	0
+; Location 7 Register $rbx
+; CHECK-ASM-NEXT:	.byte	1
+; CHECK-ASM-NEXT:	.byte	0
+; CHECK-ASM-NEXT:	.short	8
+; CHECK-ASM-NEXT:	.short	3
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.long	0
+; Entry for test_cross_bb
+; CHECK-ASM:     	.quad	0
+; CHECK-ASM-NEXT:	.long	.Ltmp7-test_cross_bb
+; CHECK-ASM-NEXT:	.short	0
+; Num locations
+; CHECK-ASM-NEXT:	.short	5
+; Location 1 Constant 0
+; CHECK-ASM-NEXT:	.byte	4
+; CHECK-ASM-NEXT:	.byte	0
+; CHECK-ASM-NEXT:	.short	8
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.long	0
+; Location 2 Constant 0
+; CHECK-ASM-NEXT:	.byte	4
+; CHECK-ASM-NEXT:	.byte	0
+; CHECK-ASM-NEXT:	.short	8
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.long	0
+; Location 3 Constant 0
+; CHECK-ASM-NEXT:	.byte	4
+; CHECK-ASM-NEXT:	.byte	0
+; CHECK-ASM-NEXT:	.short	8
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.long	0
+; Location 4 Indirect $rsp + 0
+; CHECK-ASM-NEXT:	.byte	3
+; CHECK-ASM-NEXT:	.byte	0
+; CHECK-ASM-NEXT:	.short	8
+; CHECK-ASM-NEXT:	.short	7
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.long	0
+; Location 5 Indirect $rsp + 0
+; CHECK-ASM-NEXT:	.byte	3
+; CHECK-ASM-NEXT:	.byte	0
+; CHECK-ASM-NEXT:	.short	8
+; CHECK-ASM-NEXT:	.short	7
+; CHECK-ASM-NEXT:	.short	0
+; CHECK-ASM-NEXT:	.long	0

From 55ced04d6bc13fd0f9396a0cfc393b44378d8784 Mon Sep 17 00:00:00 2001
From: Frederik Gossen <frgossen@google.com>
Date: Sat, 25 Jul 2020 14:39:18 -0700
Subject: [PATCH 0095/1035] [MLIR][Shape] Allow `num_elements` to operate on
 extent tensors

Differential Revision: https://reviews.llvm.org/D84445
---
 mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td | 16 ++++++++--------
 mlir/test/Dialect/Shape/canonicalize.mlir      |  4 ++--
 mlir/test/Dialect/Shape/invalid.mlir           | 16 ++++++++++++++++
 mlir/test/Dialect/Shape/ops.mlir               | 11 +++++++++++
 mlir/test/Dialect/Shape/shape-to-shape.mlir    |  6 +++---
 5 files changed, 40 insertions(+), 13 deletions(-)

diff --git a/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td b/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td
index 797dc0bc0cb6a..abbd8f0931091 100644
--- a/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td
+++ b/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td
@@ -333,19 +333,19 @@ def Shape_NumElementsOp : Shape_Op<"num_elements", [NoSideEffect]> {
   let summary = "Returns the number of elements for a given shape";
   let description = [{
     Returns the number of elements for a given shape which is the product of its
-    dimensions.
-
-    ```mlir
-    %product = shape.mul %lhs, %rhs
-    ```
+    extents. If the argument is of type `shape` then the result will be of type
+    `size` and potential errors will be propagated. Otherwise, if the argument
+    is and extent tensor `tensor<?xindex>` then the result will be of type
+    `index`.
   }];
 
-  let arguments = (ins Shape_ShapeType:$shape);
-  let results = (outs Shape_SizeType:$result);
+  let arguments = (ins Shape_ShapeOrExtentTensorType:$shape);
+  let results = (outs Shape_SizeOrIndexType:$result);
 
-  let assemblyFormat = "$shape attr-dict";
+  let assemblyFormat = "$shape `:` type($shape) `->` type($result) attr-dict";
 
   let hasFolder = 1;
+  let verifier = [{ return ::verifySizeOrIndexOp(*this); }];
 }
 
 def Shape_ReduceOp : Shape_Op<"reduce",
diff --git a/mlir/test/Dialect/Shape/canonicalize.mlir b/mlir/test/Dialect/Shape/canonicalize.mlir
index 577656a0b362a..e147fbeb81ac2 100644
--- a/mlir/test/Dialect/Shape/canonicalize.mlir
+++ b/mlir/test/Dialect/Shape/canonicalize.mlir
@@ -217,7 +217,7 @@ func @num_elements() -> !shape.size {
   // CHECK-NOT: shape.const_shape
   %shape = shape.const_shape [4, 5, 6] : !shape.shape
   // CHECK-NOT: shape.num_elements
-  %num_elements = shape.num_elements %shape
+  %num_elements = shape.num_elements %shape : !shape.shape -> !shape.size
   // CHECK: %[[NUM:.*]] = shape.const_size 120
   // CHECK-NEXT: return %[[NUM]] : !shape.size
   return %num_elements : !shape.size
@@ -229,7 +229,7 @@ func @num_elements() -> !shape.size {
 // CHECK-LABEL: func @nonfoldable_num_elements
 func @nonfoldable_num_elements(%shape : !shape.shape) -> !shape.size {
   // CHECK-NOT: shape.const_{{.*}}
-  %num_elements = shape.num_elements %shape
+  %num_elements = shape.num_elements %shape : !shape.shape -> !shape.size
   return %num_elements : !shape.size
 }
 
diff --git a/mlir/test/Dialect/Shape/invalid.mlir b/mlir/test/Dialect/Shape/invalid.mlir
index b4900e491fb82..0bbd6cec777dc 100644
--- a/mlir/test/Dialect/Shape/invalid.mlir
+++ b/mlir/test/Dialect/Shape/invalid.mlir
@@ -146,3 +146,19 @@ func @mul_error_possible(%lhs : !shape.size, %rhs : index) -> index {
   return %result : index
 }
 
+// -----
+
+func @num_elements_error_free(%arg : tensor<?xindex>) -> !shape.size {
+  // expected-error@+1 {{if none of the operands can hold error values then the result must be of type `index`}}
+  %result = shape.num_elements %arg : tensor<?xindex> -> !shape.size
+  return %result : !shape.size
+}
+
+// -----
+
+func @num_elements_error_possible(%arg : !shape.shape) -> index {
+  // expected-error@+1 {{if at least one of the operands can hold error values then the result must be of type `size` to propagate them}}
+  %result = shape.num_elements %arg : !shape.shape -> index
+  return %result : index
+}
+
diff --git a/mlir/test/Dialect/Shape/ops.mlir b/mlir/test/Dialect/Shape/ops.mlir
index 3a0cb7781ec72..f57826097d34f 100644
--- a/mlir/test/Dialect/Shape/ops.mlir
+++ b/mlir/test/Dialect/Shape/ops.mlir
@@ -195,3 +195,14 @@ func @any() {
   return
 }
 
+func @num_elements_extent_tensor(%arg : tensor<?xindex>) -> index {
+  %result = shape.num_elements %arg : tensor<?xindex> -> index
+  return %result : index
+}
+
+func @num_elements_shape(%arg : !shape.shape) -> !shape.size {
+  %result = shape.num_elements %arg : !shape.shape -> !shape.size
+  return %result : !shape.size
+}
+
+
diff --git a/mlir/test/Dialect/Shape/shape-to-shape.mlir b/mlir/test/Dialect/Shape/shape-to-shape.mlir
index 9a75f0b9ca1bc..d1b00bc12a22c 100644
--- a/mlir/test/Dialect/Shape/shape-to-shape.mlir
+++ b/mlir/test/Dialect/Shape/shape-to-shape.mlir
@@ -1,9 +1,9 @@
 // RUN: mlir-opt -shape-to-shape-lowering -split-input-file %s | FileCheck %s
 
-// CHECK-LABEL: func @num_elements_to_reduce(
-// CHECK-SAME:    [[ARG:%.*]]: !shape.shape) -> !shape.size {
+// CHECK-LABEL: func @num_elements_to_reduce
+// CHECK-SAME:  ([[ARG:%.*]]: !shape.shape) -> !shape.size
 func @num_elements_to_reduce(%shape : !shape.shape) -> !shape.size {
-  %num_elements = shape.num_elements %shape
+  %num_elements = shape.num_elements %shape : !shape.shape -> !shape.size
   return %num_elements : !shape.size
 }
 // CHECK: [[C1:%.*]] = shape.const_size 1

From 7bfecd773968668b17fddf3865b1d611325942a8 Mon Sep 17 00:00:00 2001
From: Jacques Pienaar <jpienaar@google.com>
Date: Sat, 25 Jul 2020 14:47:57 -0700
Subject: [PATCH 0096/1035] Revert "[MLIR][Shape] Allow `num_elements` to
 operate on extent tensors"

This reverts commit 55ced04d6bc13fd0f9396a0cfc393b44378d8784.

Forgot to submit depend change first.
---
 mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td | 16 ++++++++--------
 mlir/test/Dialect/Shape/canonicalize.mlir      |  4 ++--
 mlir/test/Dialect/Shape/invalid.mlir           | 16 ----------------
 mlir/test/Dialect/Shape/ops.mlir               | 11 -----------
 mlir/test/Dialect/Shape/shape-to-shape.mlir    |  6 +++---
 5 files changed, 13 insertions(+), 40 deletions(-)

diff --git a/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td b/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td
index abbd8f0931091..797dc0bc0cb6a 100644
--- a/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td
+++ b/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td
@@ -333,19 +333,19 @@ def Shape_NumElementsOp : Shape_Op<"num_elements", [NoSideEffect]> {
   let summary = "Returns the number of elements for a given shape";
   let description = [{
     Returns the number of elements for a given shape which is the product of its
-    extents. If the argument is of type `shape` then the result will be of type
-    `size` and potential errors will be propagated. Otherwise, if the argument
-    is and extent tensor `tensor<?xindex>` then the result will be of type
-    `index`.
+    dimensions.
+
+    ```mlir
+    %product = shape.mul %lhs, %rhs
+    ```
   }];
 
-  let arguments = (ins Shape_ShapeOrExtentTensorType:$shape);
-  let results = (outs Shape_SizeOrIndexType:$result);
+  let arguments = (ins Shape_ShapeType:$shape);
+  let results = (outs Shape_SizeType:$result);
 
-  let assemblyFormat = "$shape `:` type($shape) `->` type($result) attr-dict";
+  let assemblyFormat = "$shape attr-dict";
 
   let hasFolder = 1;
-  let verifier = [{ return ::verifySizeOrIndexOp(*this); }];
 }
 
 def Shape_ReduceOp : Shape_Op<"reduce",
diff --git a/mlir/test/Dialect/Shape/canonicalize.mlir b/mlir/test/Dialect/Shape/canonicalize.mlir
index e147fbeb81ac2..577656a0b362a 100644
--- a/mlir/test/Dialect/Shape/canonicalize.mlir
+++ b/mlir/test/Dialect/Shape/canonicalize.mlir
@@ -217,7 +217,7 @@ func @num_elements() -> !shape.size {
   // CHECK-NOT: shape.const_shape
   %shape = shape.const_shape [4, 5, 6] : !shape.shape
   // CHECK-NOT: shape.num_elements
-  %num_elements = shape.num_elements %shape : !shape.shape -> !shape.size
+  %num_elements = shape.num_elements %shape
   // CHECK: %[[NUM:.*]] = shape.const_size 120
   // CHECK-NEXT: return %[[NUM]] : !shape.size
   return %num_elements : !shape.size
@@ -229,7 +229,7 @@ func @num_elements() -> !shape.size {
 // CHECK-LABEL: func @nonfoldable_num_elements
 func @nonfoldable_num_elements(%shape : !shape.shape) -> !shape.size {
   // CHECK-NOT: shape.const_{{.*}}
-  %num_elements = shape.num_elements %shape : !shape.shape -> !shape.size
+  %num_elements = shape.num_elements %shape
   return %num_elements : !shape.size
 }
 
diff --git a/mlir/test/Dialect/Shape/invalid.mlir b/mlir/test/Dialect/Shape/invalid.mlir
index 0bbd6cec777dc..b4900e491fb82 100644
--- a/mlir/test/Dialect/Shape/invalid.mlir
+++ b/mlir/test/Dialect/Shape/invalid.mlir
@@ -146,19 +146,3 @@ func @mul_error_possible(%lhs : !shape.size, %rhs : index) -> index {
   return %result : index
 }
 
-// -----
-
-func @num_elements_error_free(%arg : tensor<?xindex>) -> !shape.size {
-  // expected-error@+1 {{if none of the operands can hold error values then the result must be of type `index`}}
-  %result = shape.num_elements %arg : tensor<?xindex> -> !shape.size
-  return %result : !shape.size
-}
-
-// -----
-
-func @num_elements_error_possible(%arg : !shape.shape) -> index {
-  // expected-error@+1 {{if at least one of the operands can hold error values then the result must be of type `size` to propagate them}}
-  %result = shape.num_elements %arg : !shape.shape -> index
-  return %result : index
-}
-
diff --git a/mlir/test/Dialect/Shape/ops.mlir b/mlir/test/Dialect/Shape/ops.mlir
index f57826097d34f..3a0cb7781ec72 100644
--- a/mlir/test/Dialect/Shape/ops.mlir
+++ b/mlir/test/Dialect/Shape/ops.mlir
@@ -195,14 +195,3 @@ func @any() {
   return
 }
 
-func @num_elements_extent_tensor(%arg : tensor<?xindex>) -> index {
-  %result = shape.num_elements %arg : tensor<?xindex> -> index
-  return %result : index
-}
-
-func @num_elements_shape(%arg : !shape.shape) -> !shape.size {
-  %result = shape.num_elements %arg : !shape.shape -> !shape.size
-  return %result : !shape.size
-}
-
-
diff --git a/mlir/test/Dialect/Shape/shape-to-shape.mlir b/mlir/test/Dialect/Shape/shape-to-shape.mlir
index d1b00bc12a22c..9a75f0b9ca1bc 100644
--- a/mlir/test/Dialect/Shape/shape-to-shape.mlir
+++ b/mlir/test/Dialect/Shape/shape-to-shape.mlir
@@ -1,9 +1,9 @@
 // RUN: mlir-opt -shape-to-shape-lowering -split-input-file %s | FileCheck %s
 
-// CHECK-LABEL: func @num_elements_to_reduce
-// CHECK-SAME:  ([[ARG:%.*]]: !shape.shape) -> !shape.size
+// CHECK-LABEL: func @num_elements_to_reduce(
+// CHECK-SAME:    [[ARG:%.*]]: !shape.shape) -> !shape.size {
 func @num_elements_to_reduce(%shape : !shape.shape) -> !shape.size {
-  %num_elements = shape.num_elements %shape : !shape.shape -> !shape.size
+  %num_elements = shape.num_elements %shape
   return %num_elements : !shape.size
 }
 // CHECK: [[C1:%.*]] = shape.const_size 1

From 5142448a5e2aeeffefb3aabdb48f19033025bc09 Mon Sep 17 00:00:00 2001
From: Jacques Pienaar <jpienaar@google.com>
Date: Sat, 25 Jul 2020 14:55:19 -0700
Subject: [PATCH 0097/1035] [MLIR][Shape] Refactor verification

Based on https://reviews.llvm.org/D84439 but less restrictive, else we
don't allow shape_of to be able to produce a ranked output and doesn't
allow for iterative refinement here. We can consider making it more
restrictive later.
---
 .../include/mlir/Dialect/Shape/IR/ShapeOps.td |  8 +-
 mlir/lib/Dialect/Shape/IR/Shape.cpp           | 82 ++++++-------------
 mlir/test/Dialect/Shape/invalid.mlir          | 30 +------
 3 files changed, 31 insertions(+), 89 deletions(-)

diff --git a/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td b/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td
index 797dc0bc0cb6a..8c32faee55f90 100644
--- a/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td
+++ b/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td
@@ -207,7 +207,7 @@ def Shape_RankOp : Shape_Op<"rank", [NoSideEffect]> {
 
   let hasFolder = 1;
   let hasCanonicalizer = 1;
-  let verifier = [{ return ::verify(*this); }];
+  let verifier = [{ return ::verifySizeOrIndexOp(*this); }];
 }
 
 def Shape_ToExtentTensorOp : Shape_Op<"to_extent_tensor", [NoSideEffect]> {
@@ -252,7 +252,7 @@ def Shape_GetExtentOp : Shape_Op<"get_extent", [NoSideEffect]> {
   }];
 
   let hasFolder = 1;
-  let verifier = [{ return ::verify(*this); }];
+  let verifier = [{ return ::verifySizeOrIndexOp(*this); }];
 }
 
 def Shape_IndexToSizeOp : Shape_Op<"index_to_size", [NoSideEffect]> {
@@ -325,7 +325,7 @@ def Shape_MulOp : Shape_Op<"mul", [Commutative, NoSideEffect]> {
     $lhs `,` $rhs `:` type($lhs) `,` type($rhs) `->` type($result) attr-dict
   }];
 
-  let verifier = [{ return ::verify(*this); }];
+  let verifier = [{ return ::verifySizeOrIndexOp(*this); }];
   let hasFolder = 1;
 }
 
@@ -412,7 +412,7 @@ def Shape_ShapeOfOp : Shape_Op<"shape_of", [NoSideEffect]> {
 
   let assemblyFormat = "$arg `:` type($arg) `->` type($result) attr-dict";
 
-  let verifier = [{ return ::verify(*this); }];
+  let verifier = [{ return ::verifyShapeOrExtentTensorOp(*this); }];
   let hasFolder = 1;
 }
 
diff --git a/mlir/lib/Dialect/Shape/IR/Shape.cpp b/mlir/lib/Dialect/Shape/IR/Shape.cpp
index d2b0dbdedb052..104ab46c55813 100644
--- a/mlir/lib/Dialect/Shape/IR/Shape.cpp
+++ b/mlir/lib/Dialect/Shape/IR/Shape.cpp
@@ -28,13 +28,37 @@ static RankedTensorType getExtentTensorType(MLIRContext *ctx) {
   return RankedTensorType::get({ShapedType::kDynamicSize}, IndexType::get(ctx));
 }
 
-static bool isErrorPropagationPossible(ArrayRef<Type> operandTypes) {
+static bool isErrorPropagationPossible(TypeRange operandTypes) {
   for (Type ty : operandTypes)
     if (ty.isa<SizeType>() || ty.isa<ShapeType>() || ty.isa<ValueShapeType>())
       return true;
   return false;
 }
 
+static LogicalResult verifySizeOrIndexOp(Operation *op) {
+  assert(op != nullptr && op->getNumResults() == 1);
+  Type resultTy = op->getResultTypes().front();
+  if (isErrorPropagationPossible(op->getOperandTypes())) {
+    if (!resultTy.isa<SizeType>())
+      return op->emitOpError()
+             << "if at least one of the operands can hold error values then "
+                "the result must be of type `size` to propagate them";
+  }
+  return success();
+}
+
+static LogicalResult verifyShapeOrExtentTensorOp(Operation *op) {
+  assert(op != nullptr && op->getNumResults() == 1);
+  Type resultTy = op->getResultTypes().front();
+  if (isErrorPropagationPossible(op->getOperandTypes())) {
+    if (!resultTy.isa<ShapeType>())
+      return op->emitOpError()
+             << "if at least one of the operands can hold error values then "
+                "the result must be of type `shape` to propagate them";
+  }
+  return success();
+}
+
 ShapeDialect::ShapeDialect(MLIRContext *context)
     : Dialect(getDialectNamespace(), context) {
   addOperations<
@@ -542,23 +566,6 @@ OpFoldResult FromExtentsOp::fold(ArrayRef<Attribute> operands) {
 // GetExtentOp
 //===----------------------------------------------------------------------===//
 
-static LogicalResult verify(GetExtentOp op) {
-  Type shapeTy = op.shape().getType();
-  Type dimTy = op.dim().getType();
-  Type extentTy = op.extent().getType();
-  if (isErrorPropagationPossible({shapeTy, dimTy})) {
-    if (!extentTy.isa<SizeType>())
-      op.emitError()
-          << "if at least one of the operands can hold error values then the "
-             "result must be of type `size` to propagate them";
-  } else {
-    if (extentTy.isa<SizeType>())
-      op.emitError() << "if none of the operands can hold error values then "
-                        "the result must be of type `index`";
-  }
-  return success();
-}
-
 Optional<int64_t> GetExtentOp::getConstantDim() {
   if (auto constSizeOp = dim().getDefiningOp<ConstSizeOp>())
     return constSizeOp.value().getLimitedValue();
@@ -597,15 +604,6 @@ void GetExtentOp::build(OpBuilder &builder, OperationState &result, Value shape,
 // RankOp
 //===----------------------------------------------------------------------===//
 
-static LogicalResult verify(shape::RankOp op) {
-  if (op.shape().getType().isa<ShapeType>() &&
-      !op.rank().getType().isa<SizeType>())
-    return op.emitOpError()
-           << "if operand is of type `shape` then the result must be of type "
-              "`size` to propagate potential errors";
-  return success();
-}
-
 OpFoldResult shape::RankOp::fold(ArrayRef<Attribute> operands) {
   auto shape = operands[0].dyn_cast_or_null<DenseIntElementsAttr>();
   if (!shape)
@@ -680,21 +678,6 @@ OpFoldResult NumElementsOp::fold(ArrayRef<Attribute> operands) {
 // MulOp
 //===----------------------------------------------------------------------===//
 
-static LogicalResult verify(MulOp op) {
-  Type resultTy = op.result().getType();
-  if (isErrorPropagationPossible({op.lhs().getType(), op.rhs().getType()})) {
-    if (!resultTy.isa<SizeType>())
-      return op.emitOpError()
-             << "if at least one of the operands can hold error values then "
-                "the result must be of type `size` to propagate them";
-  } else {
-    if (resultTy.isa<SizeType>())
-      return op.emitError() << "if none of the operands can hold error values "
-                               "then the result must be of type `index`";
-  }
-  return success();
-}
-
 OpFoldResult MulOp::fold(ArrayRef<Attribute> operands) {
   auto lhs = operands[0].dyn_cast_or_null<IntegerAttr>();
   if (!lhs)
@@ -719,21 +702,6 @@ OpFoldResult ShapeOfOp::fold(ArrayRef<Attribute>) {
   return builder.getIndexTensorAttr(type.getShape());
 }
 
-static LogicalResult verify(ShapeOfOp op) {
-  Type resultTy = op.result().getType();
-  if (isErrorPropagationPossible(op.arg().getType())) {
-    if (!resultTy.isa<ShapeType>())
-      return op.emitOpError()
-             << "if operand is of type `value_shape` then the result must be "
-                "of type `shape` to propagate potential error shapes";
-  } else {
-    if (resultTy != getExtentTensorType(op.getContext()))
-      return op.emitOpError() << "if operand is a shaped type then the result "
-                                 "must be an extent tensor";
-  }
-  return success();
-}
-
 //===----------------------------------------------------------------------===//
 // SizeToIndexOp
 //===----------------------------------------------------------------------===//
diff --git a/mlir/test/Dialect/Shape/invalid.mlir b/mlir/test/Dialect/Shape/invalid.mlir
index b4900e491fb82..20f4e877a2a90 100644
--- a/mlir/test/Dialect/Shape/invalid.mlir
+++ b/mlir/test/Dialect/Shape/invalid.mlir
@@ -90,39 +90,21 @@ func @assuming_all_op_too_few_operands() {
 
 func @shape_of(%value_arg : !shape.value_shape,
                %shaped_arg : tensor<?x3x4xf32>) {
-  // expected-error@+1 {{if operand is of type `value_shape` then the result must be of type `shape` to propagate potential error shapes}}
+  // expected-error@+1 {{if at least one of the operands can hold error values then the result must be of type `shape` to propagate them}}
   %0 = shape.shape_of %value_arg : !shape.value_shape -> tensor<?xindex>
   return
 }
 
 // -----
 
-func @shape_of(%value_arg : !shape.value_shape,
-               %shaped_arg : tensor<?x3x4xf32>) {
-  // expected-error@+1 {{if operand is a shaped type then the result must be an extent tensor}}
-  %1 = shape.shape_of %shaped_arg : tensor<?x3x4xf32> -> !shape.shape
-  return
-}
-
-// -----
-
 func @rank(%arg : !shape.shape) {
-  // expected-error@+1 {{if operand is of type `shape` then the result must be of type `size` to propagate potential errors}}
+  // expected-error@+1 {{if at least one of the operands can hold error values then the result must be of type `size` to propagate them}}
   %0 = shape.rank %arg : !shape.shape -> index
   return
 }
 
 // -----
 
-func @get_extent_error_free(%arg : tensor<?xindex>) -> !shape.size {
-  %c0 = constant 0 : index
-  // expected-error@+1 {{if none of the operands can hold error values then the result must be of type `index`}}
-  %result = shape.get_extent %arg, %c0 : tensor<?xindex>, index -> !shape.size
-  return %result : !shape.size
-}
-
-// -----
-
 func @get_extent_error_possible(%arg : tensor<?xindex>) -> index {
   %c0 = shape.const_size 0
   // expected-error@+1 {{if at least one of the operands can hold error values then the result must be of type `size` to propagate them}}
@@ -132,14 +114,6 @@ func @get_extent_error_possible(%arg : tensor<?xindex>) -> index {
 
 // -----
 
-func @mul_error_free(%arg : index) -> !shape.size {
-  // expected-error@+1 {{if none of the operands can hold error values then the result must be of type `index`}}
-  %result = shape.mul %arg, %arg : index, index -> !shape.size
-  return %result : !shape.size
-}
-
-// -----
-
 func @mul_error_possible(%lhs : !shape.size, %rhs : index) -> index {
   // expected-error@+1 {{if at least one of the operands can hold error values then the result must be of type `size` to propagate them}}
   %result = shape.mul %lhs, %rhs : !shape.size, index -> index

From 07f227c0eb8c5628842e7f7aa30001b24b8aede9 Mon Sep 17 00:00:00 2001
From: Frederik Gossen <frgossen@google.com>
Date: Sat, 25 Jul 2020 15:01:21 -0700
Subject: [PATCH 0098/1035] [MLIR][Shape] Allow `num_elements` to operate on
 extent tensors

Re-landing with dependent change landed and error condition relaxed.
Beyond the change to error condition exactly https://reviews.llvm.org/D84445.
---
 mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td | 16 ++++++++--------
 mlir/test/Dialect/Shape/canonicalize.mlir      |  4 ++--
 mlir/test/Dialect/Shape/invalid.mlir           |  8 ++++++++
 mlir/test/Dialect/Shape/ops.mlir               | 11 +++++++++++
 mlir/test/Dialect/Shape/shape-to-shape.mlir    |  6 +++---
 5 files changed, 32 insertions(+), 13 deletions(-)

diff --git a/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td b/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td
index 8c32faee55f90..3c50a4f8b39f4 100644
--- a/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td
+++ b/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td
@@ -333,19 +333,19 @@ def Shape_NumElementsOp : Shape_Op<"num_elements", [NoSideEffect]> {
   let summary = "Returns the number of elements for a given shape";
   let description = [{
     Returns the number of elements for a given shape which is the product of its
-    dimensions.
-
-    ```mlir
-    %product = shape.mul %lhs, %rhs
-    ```
+    extents. If the argument is of type `shape` then the result will be of type
+    `size` and potential errors will be propagated. Otherwise, if the argument
+    is and extent tensor `tensor<?xindex>` then the result will be of type
+    `index`.
   }];
 
-  let arguments = (ins Shape_ShapeType:$shape);
-  let results = (outs Shape_SizeType:$result);
+  let arguments = (ins Shape_ShapeOrExtentTensorType:$shape);
+  let results = (outs Shape_SizeOrIndexType:$result);
 
-  let assemblyFormat = "$shape attr-dict";
+  let assemblyFormat = "$shape `:` type($shape) `->` type($result) attr-dict";
 
   let hasFolder = 1;
+  let verifier = [{ return ::verifySizeOrIndexOp(*this); }];
 }
 
 def Shape_ReduceOp : Shape_Op<"reduce",
diff --git a/mlir/test/Dialect/Shape/canonicalize.mlir b/mlir/test/Dialect/Shape/canonicalize.mlir
index 577656a0b362a..e147fbeb81ac2 100644
--- a/mlir/test/Dialect/Shape/canonicalize.mlir
+++ b/mlir/test/Dialect/Shape/canonicalize.mlir
@@ -217,7 +217,7 @@ func @num_elements() -> !shape.size {
   // CHECK-NOT: shape.const_shape
   %shape = shape.const_shape [4, 5, 6] : !shape.shape
   // CHECK-NOT: shape.num_elements
-  %num_elements = shape.num_elements %shape
+  %num_elements = shape.num_elements %shape : !shape.shape -> !shape.size
   // CHECK: %[[NUM:.*]] = shape.const_size 120
   // CHECK-NEXT: return %[[NUM]] : !shape.size
   return %num_elements : !shape.size
@@ -229,7 +229,7 @@ func @num_elements() -> !shape.size {
 // CHECK-LABEL: func @nonfoldable_num_elements
 func @nonfoldable_num_elements(%shape : !shape.shape) -> !shape.size {
   // CHECK-NOT: shape.const_{{.*}}
-  %num_elements = shape.num_elements %shape
+  %num_elements = shape.num_elements %shape : !shape.shape -> !shape.size
   return %num_elements : !shape.size
 }
 
diff --git a/mlir/test/Dialect/Shape/invalid.mlir b/mlir/test/Dialect/Shape/invalid.mlir
index 20f4e877a2a90..4a45181d4587c 100644
--- a/mlir/test/Dialect/Shape/invalid.mlir
+++ b/mlir/test/Dialect/Shape/invalid.mlir
@@ -120,3 +120,11 @@ func @mul_error_possible(%lhs : !shape.size, %rhs : index) -> index {
   return %result : index
 }
 
+// -----
+
+func @num_elements_error_possible(%arg : !shape.shape) -> index {
+  // expected-error@+1 {{if at least one of the operands can hold error values then the result must be of type `size` to propagate them}}
+  %result = shape.num_elements %arg : !shape.shape -> index
+  return %result : index
+}
+
diff --git a/mlir/test/Dialect/Shape/ops.mlir b/mlir/test/Dialect/Shape/ops.mlir
index 3a0cb7781ec72..f57826097d34f 100644
--- a/mlir/test/Dialect/Shape/ops.mlir
+++ b/mlir/test/Dialect/Shape/ops.mlir
@@ -195,3 +195,14 @@ func @any() {
   return
 }
 
+func @num_elements_extent_tensor(%arg : tensor<?xindex>) -> index {
+  %result = shape.num_elements %arg : tensor<?xindex> -> index
+  return %result : index
+}
+
+func @num_elements_shape(%arg : !shape.shape) -> !shape.size {
+  %result = shape.num_elements %arg : !shape.shape -> !shape.size
+  return %result : !shape.size
+}
+
+
diff --git a/mlir/test/Dialect/Shape/shape-to-shape.mlir b/mlir/test/Dialect/Shape/shape-to-shape.mlir
index 9a75f0b9ca1bc..d1b00bc12a22c 100644
--- a/mlir/test/Dialect/Shape/shape-to-shape.mlir
+++ b/mlir/test/Dialect/Shape/shape-to-shape.mlir
@@ -1,9 +1,9 @@
 // RUN: mlir-opt -shape-to-shape-lowering -split-input-file %s | FileCheck %s
 
-// CHECK-LABEL: func @num_elements_to_reduce(
-// CHECK-SAME:    [[ARG:%.*]]: !shape.shape) -> !shape.size {
+// CHECK-LABEL: func @num_elements_to_reduce
+// CHECK-SAME:  ([[ARG:%.*]]: !shape.shape) -> !shape.size
 func @num_elements_to_reduce(%shape : !shape.shape) -> !shape.size {
-  %num_elements = shape.num_elements %shape
+  %num_elements = shape.num_elements %shape : !shape.shape -> !shape.size
   return %num_elements : !shape.size
 }
 // CHECK: [[C1:%.*]] = shape.const_size 1

From 11d5316afd102871027c103a4b8bfe6c072d4368 Mon Sep 17 00:00:00 2001
From: Lang Hames <lhames@gmail.com>
Date: Fri, 24 Jul 2020 21:17:37 -0700
Subject: [PATCH 0099/1035] [ORC] Don't require PageSize or Triple during
 TargetProcessControl construction

Subclasses will commonly gather that information from a remote during
construction, in which case they won't have meaningful values to pass to
TargetProcessControl's constructor.
---
 .../llvm/ExecutionEngine/Orc/TargetProcessControl.h      | 1 -
 llvm/lib/ExecutionEngine/Orc/TargetProcessControl.cpp    | 9 ++++-----
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/llvm/include/llvm/ExecutionEngine/Orc/TargetProcessControl.h b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcessControl.h
index 37bfa5a45b6c0..887ac94e4d5ce 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/TargetProcessControl.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcessControl.h
@@ -152,7 +152,6 @@ class TargetProcessControl {
   virtual Expected<LookupResult> lookupSymbols(LookupRequest Request) = 0;
 
 protected:
-  TargetProcessControl(Triple TT, unsigned PageSize);
 
   Triple TT;
   unsigned PageSize = 0;
diff --git a/llvm/lib/ExecutionEngine/Orc/TargetProcessControl.cpp b/llvm/lib/ExecutionEngine/Orc/TargetProcessControl.cpp
index ab07d3ad90b11..3f9a938db339a 100644
--- a/llvm/lib/ExecutionEngine/Orc/TargetProcessControl.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/TargetProcessControl.cpp
@@ -19,13 +19,12 @@ namespace orc {
 
 TargetProcessControl::MemoryAccess::~MemoryAccess() {}
 
-TargetProcessControl::TargetProcessControl(Triple TT, unsigned PageSize)
-    : TT(std::move(TT)), PageSize(PageSize) {}
-
 TargetProcessControl::~TargetProcessControl() {}
 
-SelfTargetProcessControl::SelfTargetProcessControl(Triple TT, unsigned PageSize)
-    : TargetProcessControl(std::move(TT), PageSize) {
+SelfTargetProcessControl::SelfTargetProcessControl(Triple TT,
+                                                   unsigned PageSize) {
+  this->TT = std::move(TT);
+  this->PageSize = PageSize;
   this->MemMgr = IPMM.get();
   this->MemAccess = this;
   if (this->TT.isOSBinFormatMachO())

From a01c4ee71cb2bcdd1fd93396af2ed6dc25f5f828 Mon Sep 17 00:00:00 2001
From: Lang Hames <lhames@gmail.com>
Date: Sat, 25 Jul 2020 14:18:52 -0700
Subject: [PATCH 0100/1035] [ORC] Rename TargetProcessControl
 DynamicLibraryHandle and loadLibrary.

The new names, DylibHandle and loadDylib, are more concise and make
clear that these utilities are for loading dynamic libraries, not static
ones.
---
 .../Orc/TPCDynamicLibrarySearchGenerator.h    |  9 +++---
 .../Orc/TargetProcessControl.h                | 28 ++++++++++---------
 .../Orc/TPCDynamicLibrarySearchGenerator.cpp  |  4 +--
 .../Orc/TargetProcessControl.cpp              |  6 ++--
 4 files changed, 24 insertions(+), 23 deletions(-)

diff --git a/llvm/include/llvm/ExecutionEngine/Orc/TPCDynamicLibrarySearchGenerator.h b/llvm/include/llvm/ExecutionEngine/Orc/TPCDynamicLibrarySearchGenerator.h
index 6c95e22a4257d..d35c8abc84a2e 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/TPCDynamicLibrarySearchGenerator.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/TPCDynamicLibrarySearchGenerator.h
@@ -27,10 +27,9 @@ class TPCDynamicLibrarySearchGenerator : public JITDylib::DefinitionGenerator {
   /// If the Allow predicate is given then only symbols matching the predicate
   /// will be searched for. If the predicate is not given then all symbols will
   /// be searched for.
-  TPCDynamicLibrarySearchGenerator(
-      TargetProcessControl &TPC,
-      TargetProcessControl::DynamicLibraryHandle DylibHandle)
-      : TPC(TPC), DylibHandle(DylibHandle) {}
+  TPCDynamicLibrarySearchGenerator(TargetProcessControl &TPC,
+                                   TargetProcessControl::DylibHandle H)
+      : TPC(TPC), H(H) {}
 
   /// Permanently loads the library at the given path and, on success, returns
   /// a DynamicLibrarySearchGenerator that will search it for symbol definitions
@@ -51,7 +50,7 @@ class TPCDynamicLibrarySearchGenerator : public JITDylib::DefinitionGenerator {
 
 private:
   TargetProcessControl &TPC;
-  TargetProcessControl::DynamicLibraryHandle DylibHandle;
+  TargetProcessControl::DylibHandle H;
 };
 
 } // end namespace orc
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/TargetProcessControl.h b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcessControl.h
index 887ac94e4d5ce..e260c64bee512 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/TargetProcessControl.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcessControl.h
@@ -110,15 +110,18 @@ class TargetProcessControl {
     }
   };
 
-  using DynamicLibraryHandle = JITTargetAddress;
-
-  /// Request lookup within a single library.
-  /// If Library is None then the whole target process should be searched.
+  /// A handle for a library opened via loadDylib.
+  ///
+  /// Note that this handle does not necessarily represent a JITDylib: it may
+  /// be a regular dynamic library or shared object (e.g. one opened via a
+  /// dlopen in the target process).
+  using DylibHandle = JITTargetAddress;
+
+  /// Request lookup within the given DylibHandle.
   struct LookupRequestElement {
-    LookupRequestElement(DynamicLibraryHandle Handle,
-                         const SymbolLookupSet &Symbols)
+    LookupRequestElement(DylibHandle Handle, const SymbolLookupSet &Symbols)
         : Handle(Handle), Symbols(Symbols) {}
-    DynamicLibraryHandle Handle;
+    DylibHandle Handle;
     const SymbolLookupSet &Symbols;
   };
 
@@ -140,11 +143,10 @@ class TargetProcessControl {
   /// Return a MemoryAccess object for the target process.
   MemoryAccess &getMemoryAccess() const { return *MemAccess; }
 
-  /// Load the library at the given path. Returns a handle to the loaded
-  /// library. If LibraryPath is null this function will return the global
-  /// handle for the target process.
-  virtual Expected<DynamicLibraryHandle>
-  loadLibrary(const char *LibraryPath) = 0;
+  /// Load the dynamic library at the given path and return a handle to it.
+  /// If LibraryPath is null this function will return the global handle for
+  /// the target process.
+  virtual Expected<DylibHandle> loadDylib(const char *DylibPath) = 0;
 
   /// Search for symbols in the target process.
   /// The result of the lookup is a 2-dimentional array of target addresses
@@ -167,7 +169,7 @@ class SelfTargetProcessControl : public TargetProcessControl,
 
   static Expected<std::unique_ptr<SelfTargetProcessControl>> Create();
 
-  Expected<DynamicLibraryHandle> loadLibrary(const char *LibraryPath) override;
+  Expected<DylibHandle> loadDylib(const char *DylibPath) override;
 
   Expected<LookupResult> lookupSymbols(LookupRequest Request) override;
 
diff --git a/llvm/lib/ExecutionEngine/Orc/TPCDynamicLibrarySearchGenerator.cpp b/llvm/lib/ExecutionEngine/Orc/TPCDynamicLibrarySearchGenerator.cpp
index ea8bde971d1d4..18de5b616eec8 100644
--- a/llvm/lib/ExecutionEngine/Orc/TPCDynamicLibrarySearchGenerator.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/TPCDynamicLibrarySearchGenerator.cpp
@@ -14,7 +14,7 @@ namespace orc {
 Expected<std::unique_ptr<TPCDynamicLibrarySearchGenerator>>
 TPCDynamicLibrarySearchGenerator::Load(TargetProcessControl &TPC,
                                        const char *LibraryPath) {
-  auto Handle = TPC.loadLibrary(LibraryPath);
+  auto Handle = TPC.loadDylib(LibraryPath);
   if (!Handle)
     return Handle.takeError();
 
@@ -30,7 +30,7 @@ Error TPCDynamicLibrarySearchGenerator::tryToGenerate(
 
   SymbolMap NewSymbols;
 
-  TargetProcessControl::LookupRequestElement Request(DylibHandle, Symbols);
+  TargetProcessControl::LookupRequestElement Request(H, Symbols);
   auto Result = TPC.lookupSymbols(Request);
   if (!Result)
     return Result.takeError();
diff --git a/llvm/lib/ExecutionEngine/Orc/TargetProcessControl.cpp b/llvm/lib/ExecutionEngine/Orc/TargetProcessControl.cpp
index 3f9a938db339a..f17f5bf32856f 100644
--- a/llvm/lib/ExecutionEngine/Orc/TargetProcessControl.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/TargetProcessControl.cpp
@@ -42,11 +42,11 @@ SelfTargetProcessControl::Create() {
   return std::make_unique<SelfTargetProcessControl>(std::move(TT), *PageSize);
 }
 
-Expected<TargetProcessControl::DynamicLibraryHandle>
-SelfTargetProcessControl::loadLibrary(const char *LibraryPath) {
+Expected<TargetProcessControl::DylibHandle>
+SelfTargetProcessControl::loadDylib(const char *DylibPath) {
   std::string ErrMsg;
   auto Dylib = std::make_unique<sys::DynamicLibrary>(
-      sys::DynamicLibrary::getPermanentLibrary(LibraryPath, &ErrMsg));
+      sys::DynamicLibrary::getPermanentLibrary(DylibPath, &ErrMsg));
   if (!Dylib->isValid())
     return make_error<StringError>(std::move(ErrMsg), inconvertibleErrorCode());
   DynamicLibraries.push_back(std::move(Dylib));

From af1dd0b1adc40e62bd922673af3741ad66989eee Mon Sep 17 00:00:00 2001
From: Roman Lebedev <lebedev.ri@gmail.com>
Date: Sat, 25 Jul 2020 22:31:05 +0300
Subject: [PATCH 0101/1035] [Reduce] Basic block reduction: do properly handle
 invoke insts (PR46818)

Terminator may have returned value, so we need to replace uses,
and in general handle invoke as a branch inst.

I'm not sure this is the best handling, but IMO poorly reduced
input is much better than crashing reduction tool.
A (previously-crashing!) test added.

Fixes https://bugs.llvm.org/show_bug.cgi?id=46818
---
 llvm/test/Reduce/remove-bbs-unwinded-to.ll    | 39 +++++++++++++++++++
 .../llvm-reduce/deltas/ReduceBasicBlocks.cpp  |  3 +-
 2 files changed, 41 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/Reduce/remove-bbs-unwinded-to.ll

diff --git a/llvm/test/Reduce/remove-bbs-unwinded-to.ll b/llvm/test/Reduce/remove-bbs-unwinded-to.ll
new file mode 100644
index 0000000000000..375678ec99c9d
--- /dev/null
+++ b/llvm/test/Reduce/remove-bbs-unwinded-to.ll
@@ -0,0 +1,39 @@
+; RUN: llvm-reduce --test FileCheck --test-arg --check-prefixes=CHECK-ALL,CHECK-INTERESTINGNESS --test-arg %s --test-arg --input-file %s -o %t
+; RUN: cat %t | FileCheck --check-prefixes=CHECK-ALL,CHECK-FINAL %s
+
+declare i32 @maybe_throwing_callee()
+
+; CHECK-ALL: declare void @did_not_throw(i32)
+declare void @did_not_throw(i32)
+
+declare void @thrown()
+
+; CHECK-ALL: define void @caller()
+define void @caller() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+; CHECK-ALL: bb:
+bb:
+; CHECK-INTERESTINGNESS: label %bb3
+; CHECK-FINAL: br label %bb3
+  %i0 = invoke i32 @maybe_throwing_callee()
+          to label %bb3 unwind label %bb1
+
+bb1:
+  landingpad { i8*, i32 } catch i8* null
+  call void @thrown()
+  br label %bb4
+
+; CHECK-ALL: bb3:
+bb3:
+; CHECK-INTERESTINGNESS: call void @did_not_throw(i32
+; CHECK-FINAL: call void @did_not_throw(i32 undef)
+; CHECK-ALL: br label %bb4
+  call void @did_not_throw(i32 %i0)
+  br label %bb4
+
+; CHECK-ALL: bb4:
+; CHECK-ALL: ret void
+bb4:
+  ret void
+}
+
+declare i32 @__gxx_personality_v0(...)
diff --git a/llvm/tools/llvm-reduce/deltas/ReduceBasicBlocks.cpp b/llvm/tools/llvm-reduce/deltas/ReduceBasicBlocks.cpp
index 002d81a674872..9dee738d4906b 100644
--- a/llvm/tools/llvm-reduce/deltas/ReduceBasicBlocks.cpp
+++ b/llvm/tools/llvm-reduce/deltas/ReduceBasicBlocks.cpp
@@ -36,11 +36,12 @@ static void replaceBranchTerminator(BasicBlock &BB,
   if (ChunkSucessors.size() == Term->getNumSuccessors())
     return;
 
-  bool IsBranch = isa<BranchInst>(Term);
+  bool IsBranch = isa<BranchInst>(Term) || isa<InvokeInst>(Term);
   Value *Address = nullptr;
   if (auto IndBI = dyn_cast<IndirectBrInst>(Term))
     Address = IndBI->getAddress();
 
+  Term->replaceAllUsesWith(UndefValue::get(Term->getType()));
   Term->eraseFromParent();
 
   if (ChunkSucessors.empty()) {

From 9932d74740b19030f12b82ae93a9e61af32f5931 Mon Sep 17 00:00:00 2001
From: Roman Lebedev <lebedev.ri@gmail.com>
Date: Sat, 25 Jul 2020 23:24:13 +0300
Subject: [PATCH 0102/1035] [Reduce] Argument reduction: do properly handle
 invoke insts (PR46819)

replaceFunctionCalls() is very non-exhaustive, it only handles
CallInst's. Which means, by the time we drop old function,
there may still be uses of it lurking around.
Let's instead whack-a-mole them by all by replacing with undef.

I'm not sure this is the best handling, especially for calls, but IMO
poorly reduced input is much better than crashing reduction tool.
A (previously-crashing!) test added.

Fixes https://bugs.llvm.org/show_bug.cgi?id=46819
---
 llvm/test/Reduce/remove-invoked-functions.ll  | 55 +++++++++++++++++++
 .../llvm-reduce/deltas/ReduceArguments.cpp    |  1 +
 2 files changed, 56 insertions(+)
 create mode 100644 llvm/test/Reduce/remove-invoked-functions.ll

diff --git a/llvm/test/Reduce/remove-invoked-functions.ll b/llvm/test/Reduce/remove-invoked-functions.ll
new file mode 100644
index 0000000000000..e4458e662fee3
--- /dev/null
+++ b/llvm/test/Reduce/remove-invoked-functions.ll
@@ -0,0 +1,55 @@
+; RUN: llvm-reduce --test FileCheck --test-arg --check-prefixes=CHECK-ALL,CHECK-INTERESTINGNESS --test-arg %s --test-arg --input-file %s -o %t
+; RUN: cat %t | FileCheck --check-prefixes=CHECK-ALL,CHECK-FINAL %s
+
+; CHECK-INTERESTINGNESS: define i32 @maybe_throwing_callee(
+; CHECK-FINAL: define i32 @maybe_throwing_callee()
+define i32 @maybe_throwing_callee(i32 %arg) {
+; CHECK-ALL: call void @thrown()
+; CHECK-INTERESTINGNESS: ret i32
+; CHECK-FINAL: ret i32 undef
+  call void @thrown()
+  ret i32 %arg
+}
+
+; CHECK-ALL: declare void @did_not_throw(i32)
+declare void @did_not_throw(i32)
+
+; CHECK-ALL: declare void @thrown()
+declare void @thrown()
+
+; CHECK-INTERESTINGNESS: define void @caller(
+; CHECK-FINAL: define void @caller(i32 %arg)
+define void @caller(i32 %arg) personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+; CHECK-ALL: bb:
+bb:
+; CHECK-INTERESTINGNESS: %i0 = invoke i32
+; CHECK-FINAL: %i0 = invoke i32 undef(i32 %arg)
+; CHECK-ALL: to label %bb3 unwind label %bb1
+  %i0 = invoke i32 @maybe_throwing_callee(i32 %arg)
+          to label %bb3 unwind label %bb1
+
+; CHECK-ALL: bb1:
+bb1:
+; CHECK-ALL: landingpad { i8*, i32 }
+; CHECK-ALL: catch i8* null
+; CHECK-ALL: call void @thrown()
+; CHECK-ALL: br label %bb4
+  landingpad { i8*, i32 }
+  catch i8* null
+  call void @thrown()
+  br label %bb4
+
+; CHECK-ALL: bb3:
+bb3:
+; CHECK-ALL: call void @did_not_throw(i32 %i0)
+; CHECK-ALL: br label %bb4
+  call void @did_not_throw(i32 %i0)
+  br label %bb4
+
+; CHECK-ALL: bb4:
+; CHECK-ALL: ret void
+bb4:
+  ret void
+}
+
+declare i32 @__gxx_personality_v0(...)
diff --git a/llvm/tools/llvm-reduce/deltas/ReduceArguments.cpp b/llvm/tools/llvm-reduce/deltas/ReduceArguments.cpp
index e0e1d1c225670..1eafc2c560dea 100644
--- a/llvm/tools/llvm-reduce/deltas/ReduceArguments.cpp
+++ b/llvm/tools/llvm-reduce/deltas/ReduceArguments.cpp
@@ -94,6 +94,7 @@ static void extractArgumentsFromModule(std::vector<Chunk> ChunksToKeep,
     replaceFunctionCalls(*F, *ClonedFunc, ArgIndexesToKeep);
     // Rename Cloned Function to Old's name
     std::string FName = std::string(F->getName());
+    F->replaceAllUsesWith(UndefValue::get(F->getType()));
     F->eraseFromParent();
     ClonedFunc->setName(FName);
   }

From 96d74530c09edd95452295bb4e300ab310a9bb2d Mon Sep 17 00:00:00 2001
From: Roman Lebedev <lebedev.ri@gmail.com>
Date: Sun, 26 Jul 2020 00:56:36 +0300
Subject: [PATCH 0103/1035] [Reduce] Argument reduction: do deal with function
 declarations

We can happily turn function definitions into declarations,
thus obscuring their argument from being elided by this pass.

I don't believe there is a good reason to just ignore declarations.
likely even proper llvm intrinsics ones,
at worst the input becomes uninteresting.

The other question here is that all these transforms are all-or-nothing.
In some cases, should we be treating each use separately?

The main blocker here seemed to be that llvm::CloneFunctionInto()
does `&OldFunc->front()`, which inserts a nullptr into a densemap,
which is not happy about it and asserts.
---
 llvm/lib/Transforms/Utils/CloneFunction.cpp   |  5 ++++
 ...ultiple-use-of-args-in-same-instruction.py | 13 ----------
 ...-use-of-global-vars-in-same-instruction.py | 13 ----------
 .../Reduce/remove-args-from-declaration.ll    | 24 +++++++++++++++++++
 ...ultiple-use-of-args-in-same-instruction.ll |  9 +++----
 ...-use-of-global-vars-in-same-instruction.ll | 12 +++++-----
 .../llvm-reduce/deltas/ReduceArguments.cpp    |  4 ++--
 7 files changed, 42 insertions(+), 38 deletions(-)
 delete mode 100644 llvm/test/Reduce/Inputs/remove-multiple-use-of-args-in-same-instruction.py
 delete mode 100644 llvm/test/Reduce/Inputs/remove-multiple-use-of-global-vars-in-same-instruction.py
 create mode 100644 llvm/test/Reduce/remove-args-from-declaration.ll

diff --git a/llvm/lib/Transforms/Utils/CloneFunction.cpp b/llvm/lib/Transforms/Utils/CloneFunction.cpp
index 788983c156903..957e4028bae77 100644
--- a/llvm/lib/Transforms/Utils/CloneFunction.cpp
+++ b/llvm/lib/Transforms/Utils/CloneFunction.cpp
@@ -147,6 +147,11 @@ void llvm::CloneFunctionInto(Function *NewFunc, const Function *OldFunc,
                      TypeMapper, Materializer));
   }
 
+  // Everything else beyond this point deals with function instructions,
+  // so if we are dealing with a function declaration, we're done.
+  if (OldFunc->isDeclaration())
+    return;
+
   // When we remap instructions, we want to avoid duplicating inlined
   // DISubprograms, so record all subprograms we find as we duplicate
   // instructions and then freeze them in the MD map.
diff --git a/llvm/test/Reduce/Inputs/remove-multiple-use-of-args-in-same-instruction.py b/llvm/test/Reduce/Inputs/remove-multiple-use-of-args-in-same-instruction.py
deleted file mode 100644
index 93fa5c0bc29e7..0000000000000
--- a/llvm/test/Reduce/Inputs/remove-multiple-use-of-args-in-same-instruction.py
+++ /dev/null
@@ -1,13 +0,0 @@
-import sys
-
-FunctionCallPresent = False
-
-input = open(sys.argv[1], "r")
-for line in input:
-  if "call void @use" in line:
-    FunctionCallPresent = True
-
-if FunctionCallPresent:
-  sys.exit(0) # Interesting!
-
-sys.exit(1)
diff --git a/llvm/test/Reduce/Inputs/remove-multiple-use-of-global-vars-in-same-instruction.py b/llvm/test/Reduce/Inputs/remove-multiple-use-of-global-vars-in-same-instruction.py
deleted file mode 100644
index 93fa5c0bc29e7..0000000000000
--- a/llvm/test/Reduce/Inputs/remove-multiple-use-of-global-vars-in-same-instruction.py
+++ /dev/null
@@ -1,13 +0,0 @@
-import sys
-
-FunctionCallPresent = False
-
-input = open(sys.argv[1], "r")
-for line in input:
-  if "call void @use" in line:
-    FunctionCallPresent = True
-
-if FunctionCallPresent:
-  sys.exit(0) # Interesting!
-
-sys.exit(1)
diff --git a/llvm/test/Reduce/remove-args-from-declaration.ll b/llvm/test/Reduce/remove-args-from-declaration.ll
new file mode 100644
index 0000000000000..f476495c57314
--- /dev/null
+++ b/llvm/test/Reduce/remove-args-from-declaration.ll
@@ -0,0 +1,24 @@
+; RUN: llvm-reduce --test FileCheck --test-arg --check-prefixes=CHECK-ALL,CHECK-INTERESTINGNESS --test-arg %s --test-arg --input-file %s -o %t
+; RUN: cat %t | FileCheck --check-prefixes=CHECK-ALL,CHECK-FINAL %s
+
+; CHECK-INTERESTINGNESS-LABEL: @interesting(
+; CHECK-INTERESTINGNESS-SAME: i32
+; CHECK-FINAL: declare void @interesting(i32)
+declare void @interesting(i32 %uninteresting1, i32 %interesting, i32 %uninteresting2)
+
+; CHECK-INTERESTINGNESS-LABEL: @interesting2(
+; CHECK-INTERESTINGNESS-SAME: i32
+; CHECK-FINAL: declare void @interesting2(i32)
+declare void @interesting2(i32 %uninteresting1, i32 %interesting, i32 %uninteresting2)
+
+; CHECK-INTERESTINGNESS-LABEL: @callee(
+; CHECK-INTERESTINGNESS-SAME: i32 %interesting
+; CHECK-FINAL: define void @callee(i32 %interesting) {
+define void @callee(i32 %uninteresting1, i32 %interesting, i32 %uninteresting2) {
+; CHECK-INTERESTINGNESS: call void @interesting2(
+; CHECK-INTERESTINGNESS-SAME: i32 %interesting
+; CHECK-FINAL: call void @interesting2(i32 %interesting)
+  call void @interesting2(i32 %uninteresting1, i32 %interesting, i32 %uninteresting2)
+; CHECK-ALL: ret void
+  ret void
+}
diff --git a/llvm/test/Reduce/remove-multiple-use-of-args-in-same-instruction.ll b/llvm/test/Reduce/remove-multiple-use-of-args-in-same-instruction.ll
index 21a638f1e6bce..cd23d6e616022 100644
--- a/llvm/test/Reduce/remove-multiple-use-of-args-in-same-instruction.ll
+++ b/llvm/test/Reduce/remove-multiple-use-of-args-in-same-instruction.ll
@@ -1,14 +1,15 @@
 ; Test that llvm-reduce can remove uninteresting function arguments from function definitions as well as their calls.
 ;
-; RUN: llvm-reduce --test %python --test-arg %p/Inputs/remove-multiple-use-of-args-in-same-instruction.py %s -o %t
-; RUN: cat %t | FileCheck -implicit-check-not=uninteresting %s
+; RUN: llvm-reduce --test FileCheck --test-arg --check-prefixes=CHECK-ALL,CHECK-INTERESTINGNESS --test-arg %s --test-arg --input-file %s -o %t
+; RUN: cat %t | FileCheck --check-prefixes=CHECK-ALL,CHECK-FINAL %s
 
+; CHECK-ALL: declare void @use(i32, i32, i32)
 declare void @use(i32, i32, i32)
 
-; CHECK-LABEL: @interesting(i32 %uninteresting1, i32 %uninteresting2, i32 %uninteresting3
+; CHECK-ALL: @interesting(i32 %uninteresting1, i32 %uninteresting2, i32 %uninteresting3
 define void @interesting(i32 %uninteresting1, i32 %uninteresting2, i32 %uninteresting3) {
 entry:
-  ; CHECK: call void @use(i32 %uninteresting1, i32 %uninteresting2, i32 %uninteresting3)
+  ; CHECK-ALL: call void @use(i32 %uninteresting1, i32 %uninteresting2, i32 %uninteresting3)
   call void @use(i32 %uninteresting1, i32 %uninteresting2, i32 %uninteresting3)
   call void @use(i32 %uninteresting1, i32 %uninteresting2, i32 %uninteresting3)
   call void @use(i32 %uninteresting1, i32 %uninteresting2, i32 %uninteresting3)
diff --git a/llvm/test/Reduce/remove-multiple-use-of-global-vars-in-same-instruction.ll b/llvm/test/Reduce/remove-multiple-use-of-global-vars-in-same-instruction.ll
index 4400bc818e554..6d62bd2938d7a 100644
--- a/llvm/test/Reduce/remove-multiple-use-of-global-vars-in-same-instruction.ll
+++ b/llvm/test/Reduce/remove-multiple-use-of-global-vars-in-same-instruction.ll
@@ -1,11 +1,11 @@
 ; Test that llvm-reduce can remove uninteresting function arguments from function definitions as well as their calls.
 ;
-; RUN: llvm-reduce --test %python --test-arg %p/Inputs/remove-multiple-use-of-global-vars-in-same-instruction.py %s -o %t
-; RUN: cat %t | FileCheck -implicit-check-not=uninteresting %s
+; RUN: llvm-reduce --test FileCheck --test-arg --check-prefixes=CHECK-ALL,CHECK-INTERESTINGNESS --test-arg %s --test-arg --input-file %s -o %t
+; RUN: cat %t | FileCheck --check-prefixes=CHECK-ALL,CHECK-FINAL %s
 
-; CHECK: @uninteresting1 = global
-; CHECK: @uninteresting2 = global
-; CHECK: @uninteresting3 = global
+; CHECK-ALL: @uninteresting1 = global
+; CHECK-ALL: @uninteresting2 = global
+; CHECK-ALL: @uninteresting3 = global
 @uninteresting1 = global i32 0, align 4
 @uninteresting2 = global i32 0, align 4
 @uninteresting3 = global i32 0, align 4
@@ -15,7 +15,7 @@ declare void @use(i32*, i32*, i32*)
 ; CHECK-LABEL: @interesting()
 define void @interesting() {
 entry:
-  ; CHECK: call void @use(i32* @uninteresting1, i32* @uninteresting2, i32* @uninteresting3)
+  ; CHECK-ALL: call void @use(i32* @uninteresting1, i32* @uninteresting2, i32* @uninteresting3)
   call void @use(i32* @uninteresting1, i32* @uninteresting2, i32* @uninteresting3)
   call void @use(i32* @uninteresting1, i32* @uninteresting2, i32* @uninteresting3)
   call void @use(i32* @uninteresting1, i32* @uninteresting2, i32* @uninteresting3)
diff --git a/llvm/tools/llvm-reduce/deltas/ReduceArguments.cpp b/llvm/tools/llvm-reduce/deltas/ReduceArguments.cpp
index 1eafc2c560dea..9488d71b71c35 100644
--- a/llvm/tools/llvm-reduce/deltas/ReduceArguments.cpp
+++ b/llvm/tools/llvm-reduce/deltas/ReduceArguments.cpp
@@ -48,7 +48,7 @@ static void extractArgumentsFromModule(std::vector<Chunk> ChunksToKeep,
   std::vector<Function *> Funcs;
   // Get inside-chunk arguments, as well as their parent function
   for (auto &F : *Program)
-    if (!F.isDeclaration()) {
+    if (!F.arg_empty()) {
       Funcs.push_back(&F);
       for (auto &A : F.args())
         if (O.shouldKeep())
@@ -108,7 +108,7 @@ static int countArguments(Module *Program) {
   outs() << "Param Index Reference:\n";
   int ArgsCount = 0;
   for (auto &F : *Program)
-    if (!F.isDeclaration() && F.arg_size()) {
+    if (!F.arg_empty()) {
       outs() << "  " << F.getName() << "\n";
       for (auto &A : F.args())
         outs() << "\t" << ++ArgsCount << ": " << A.getName() << "\n";

From c5b23714368eabfb22fcd7f7567cf2a9830c8d8b Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@intel.com>
Date: Sat, 25 Jul 2020 16:36:33 -0700
Subject: [PATCH 0104/1035] [X86] Add masked versions of the VPTERNLOG test
 cases added for D83630. NFC

We don't handle these yet and D83630 won't improve that, but
at least we'll have the tests.
---
 llvm/test/CodeGen/X86/avx512-logic.ll   | 136 ++++++++++++
 llvm/test/CodeGen/X86/avx512vl-logic.ll | 270 ++++++++++++++++++++++++
 2 files changed, 406 insertions(+)

diff --git a/llvm/test/CodeGen/X86/avx512-logic.ll b/llvm/test/CodeGen/X86/avx512-logic.ll
index 88a3b5aea9bd4..30607214f56d5 100644
--- a/llvm/test/CodeGen/X86/avx512-logic.ll
+++ b/llvm/test/CodeGen/X86/avx512-logic.ll
@@ -919,3 +919,139 @@ define <8 x i64> @ternlog_xor_and_mask(<8 x i64> %x, <8 x i64> %y) {
   %b = xor <8 x i64> %a, %y
   ret <8 x i64> %b
 }
+
+define <16 x i32> @ternlog_maskz_or_and_mask(<16 x i32> %x, <16 x i32> %y, <16 x i32> %mask) {
+; KNL-LABEL: ternlog_maskz_or_and_mask:
+; KNL:       ## %bb.0:
+; KNL-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; KNL-NEXT:    vpcmpgtd %zmm2, %zmm3, %k1
+; KNL-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
+; KNL-NEXT:    vpord %zmm1, %zmm0, %zmm0 {%k1} {z}
+; KNL-NEXT:    retq
+;
+; SKX-LABEL: ternlog_maskz_or_and_mask:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    vpmovd2m %zmm2, %k1
+; SKX-NEXT:    vandps {{.*}}(%rip), %zmm0, %zmm0
+; SKX-NEXT:    vorps %zmm1, %zmm0, %zmm0 {%k1} {z}
+; SKX-NEXT:    retq
+  %m = icmp slt <16 x i32> %mask, zeroinitializer
+  %a = and <16 x i32> %x, <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>
+  %b = or <16 x i32> %a, %y
+  %c = select <16 x i1> %m, <16 x i32> %b, <16 x i32> zeroinitializer
+  ret <16 x i32> %c
+}
+
+define <8 x i64> @ternlog_maskz_xor_and_mask(<8 x i64> %x, <8 x i64> %y, <8 x i64> %mask) {
+; KNL-LABEL: ternlog_maskz_xor_and_mask:
+; KNL:       ## %bb.0:
+; KNL-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; KNL-NEXT:    vpcmpgtq %zmm2, %zmm3, %k1
+; KNL-NEXT:    vpandd {{.*}}(%rip), %zmm0, %zmm0
+; KNL-NEXT:    vpxorq %zmm1, %zmm0, %zmm0 {%k1} {z}
+; KNL-NEXT:    retq
+;
+; SKX-LABEL: ternlog_maskz_xor_and_mask:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    vpmovq2m %zmm2, %k1
+; SKX-NEXT:    vandpd {{.*}}(%rip), %zmm0, %zmm0
+; SKX-NEXT:    vxorpd %zmm1, %zmm0, %zmm0 {%k1} {z}
+; SKX-NEXT:    retq
+  %m = icmp slt <8 x i64> %mask, zeroinitializer
+  %a = and <8 x i64> %x, <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>
+  %b = xor <8 x i64> %a, %y
+  %c = select <8 x i1> %m, <8 x i64> %b, <8 x i64> zeroinitializer
+  ret <8 x i64> %c
+}
+
+define <16 x i32> @ternlog_maskx_or_and_mask(<16 x i32> %x, <16 x i32> %y, <16 x i32> %mask) {
+; KNL-LABEL: ternlog_maskx_or_and_mask:
+; KNL:       ## %bb.0:
+; KNL-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; KNL-NEXT:    vpcmpgtd %zmm2, %zmm3, %k1
+; KNL-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm2
+; KNL-NEXT:    vpord %zmm1, %zmm2, %zmm0 {%k1}
+; KNL-NEXT:    retq
+;
+; SKX-LABEL: ternlog_maskx_or_and_mask:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    vpmovd2m %zmm2, %k1
+; SKX-NEXT:    vandps {{.*}}(%rip), %zmm0, %zmm2
+; SKX-NEXT:    vorps %zmm1, %zmm2, %zmm0 {%k1}
+; SKX-NEXT:    retq
+  %m = icmp slt <16 x i32> %mask, zeroinitializer
+  %a = and <16 x i32> %x, <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>
+  %b = or <16 x i32> %a, %y
+  %c = select <16 x i1> %m, <16 x i32> %b, <16 x i32> %x
+  ret <16 x i32> %c
+}
+
+define <16 x i32> @ternlog_masky_or_and_mask(<16 x i32> %x, <16 x i32> %y, <16 x i32> %mask) {
+; KNL-LABEL: ternlog_masky_or_and_mask:
+; KNL:       ## %bb.0:
+; KNL-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; KNL-NEXT:    vpcmpgtd %zmm2, %zmm3, %k1
+; KNL-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
+; KNL-NEXT:    vpord %zmm1, %zmm0, %zmm1 {%k1}
+; KNL-NEXT:    vmovdqa64 %zmm1, %zmm0
+; KNL-NEXT:    retq
+;
+; SKX-LABEL: ternlog_masky_or_and_mask:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    vpmovd2m %zmm2, %k1
+; SKX-NEXT:    vandps {{.*}}(%rip), %zmm0, %zmm0
+; SKX-NEXT:    vorps %zmm1, %zmm0, %zmm1 {%k1}
+; SKX-NEXT:    vmovaps %zmm1, %zmm0
+; SKX-NEXT:    retq
+  %m = icmp slt <16 x i32> %mask, zeroinitializer
+  %a = and <16 x i32> %x, <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>
+  %b = or <16 x i32> %a, %y
+  %c = select <16 x i1> %m, <16 x i32> %b, <16 x i32> %y
+  ret <16 x i32> %c
+}
+
+define <8 x i64> @ternlog_maskx_xor_and_mask(<8 x i64> %x, <8 x i64> %y, <8 x i64> %mask) {
+; KNL-LABEL: ternlog_maskx_xor_and_mask:
+; KNL:       ## %bb.0:
+; KNL-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; KNL-NEXT:    vpcmpgtq %zmm2, %zmm3, %k1
+; KNL-NEXT:    vpandd {{.*}}(%rip), %zmm0, %zmm2
+; KNL-NEXT:    vpxorq %zmm1, %zmm2, %zmm0 {%k1}
+; KNL-NEXT:    retq
+;
+; SKX-LABEL: ternlog_maskx_xor_and_mask:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    vpmovq2m %zmm2, %k1
+; SKX-NEXT:    vandpd {{.*}}(%rip), %zmm0, %zmm2
+; SKX-NEXT:    vxorpd %zmm1, %zmm2, %zmm0 {%k1}
+; SKX-NEXT:    retq
+  %m = icmp slt <8 x i64> %mask, zeroinitializer
+  %a = and <8 x i64> %x, <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>
+  %b = xor <8 x i64> %a, %y
+  %c = select <8 x i1> %m, <8 x i64> %b, <8 x i64> %x
+  ret <8 x i64> %c
+}
+
+define <8 x i64> @ternlog_masky_xor_and_mask(<8 x i64> %x, <8 x i64> %y, <8 x i64> %mask) {
+; KNL-LABEL: ternlog_masky_xor_and_mask:
+; KNL:       ## %bb.0:
+; KNL-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; KNL-NEXT:    vpcmpgtq %zmm2, %zmm3, %k1
+; KNL-NEXT:    vpandd {{.*}}(%rip), %zmm0, %zmm0
+; KNL-NEXT:    vpxorq %zmm1, %zmm0, %zmm1 {%k1}
+; KNL-NEXT:    vmovdqa64 %zmm1, %zmm0
+; KNL-NEXT:    retq
+;
+; SKX-LABEL: ternlog_masky_xor_and_mask:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    vpmovq2m %zmm2, %k1
+; SKX-NEXT:    vandpd {{.*}}(%rip), %zmm0, %zmm0
+; SKX-NEXT:    vxorpd %zmm1, %zmm0, %zmm1 {%k1}
+; SKX-NEXT:    vmovapd %zmm1, %zmm0
+; SKX-NEXT:    retq
+  %m = icmp slt <8 x i64> %mask, zeroinitializer
+  %a = and <8 x i64> %x, <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>
+  %b = xor <8 x i64> %a, %y
+  %c = select <8 x i1> %m, <8 x i64> %b, <8 x i64> %y
+  ret <8 x i64> %c
+}
diff --git a/llvm/test/CodeGen/X86/avx512vl-logic.ll b/llvm/test/CodeGen/X86/avx512vl-logic.ll
index 26d905ebeae77..3f0ce30928478 100644
--- a/llvm/test/CodeGen/X86/avx512vl-logic.ll
+++ b/llvm/test/CodeGen/X86/avx512vl-logic.ll
@@ -1031,3 +1031,273 @@ define <4 x i64> @ternlog_xor_and_mask_ymm(<4 x i64> %x, <4 x i64> %y) {
   %b = xor <4 x i64> %a, %y
   ret <4 x i64> %b
 }
+
+define <4 x i32> @ternlog_maskz_or_and_mask(<4 x i32> %x, <4 x i32> %y, <4 x i32> %z, <4 x i32> %mask) {
+; KNL-LABEL: ternlog_maskz_or_and_mask:
+; KNL:       ## %bb.0:
+; KNL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; KNL-NEXT:    vpcmpgtd %xmm3, %xmm2, %k1
+; KNL-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; KNL-NEXT:    vpord %xmm1, %xmm0, %xmm0 {%k1} {z}
+; KNL-NEXT:    retq
+;
+; SKX-LABEL: ternlog_maskz_or_and_mask:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    vpmovd2m %xmm3, %k1
+; SKX-NEXT:    vandps {{.*}}(%rip), %xmm0, %xmm0
+; SKX-NEXT:    vorps %xmm1, %xmm0, %xmm0 {%k1} {z}
+; SKX-NEXT:    retq
+  %m = icmp slt <4 x i32> %mask, zeroinitializer
+  %a = and <4 x i32> %x, <i32 255, i32 255, i32 255, i32 255>
+  %b = or <4 x i32> %a, %y
+  %c = select <4 x i1> %m, <4 x i32> %b, <4 x i32> zeroinitializer
+  ret <4 x i32> %c
+}
+
+define <8 x i32> @ternlog_maskz_or_and_mask_ymm(<8 x i32> %x, <8 x i32> %y, <8 x i32> %mask) {
+; KNL-LABEL: ternlog_maskz_or_and_mask_ymm:
+; KNL:       ## %bb.0:
+; KNL-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; KNL-NEXT:    vpcmpgtd %ymm2, %ymm3, %k1
+; KNL-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
+; KNL-NEXT:    vpord %ymm1, %ymm0, %ymm0 {%k1} {z}
+; KNL-NEXT:    retq
+;
+; SKX-LABEL: ternlog_maskz_or_and_mask_ymm:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    vpmovd2m %ymm2, %k1
+; SKX-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
+; SKX-NEXT:    vorps %ymm1, %ymm0, %ymm0 {%k1} {z}
+; SKX-NEXT:    retq
+  %m = icmp slt <8 x i32> %mask, zeroinitializer
+  %a = and <8 x i32> %x, <i32 -16777216, i32 -16777216, i32 -16777216, i32 -16777216, i32 -16777216, i32 -16777216, i32 -16777216, i32 -16777216>
+  %b = or <8 x i32> %a, %y
+  %c = select <8 x i1> %m, <8 x i32> %b, <8 x i32> zeroinitializer
+  ret <8 x i32> %c
+}
+
+define <2 x i64> @ternlog_maskz_xor_and_mask(<2 x i64> %x, <2 x i64> %y, <2 x i64> %mask) {
+; KNL-LABEL: ternlog_maskz_xor_and_mask:
+; KNL:       ## %bb.0:
+; KNL-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; KNL-NEXT:    vpcmpgtq %xmm2, %xmm3, %k1
+; KNL-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; KNL-NEXT:    vpxorq %xmm1, %xmm0, %xmm0 {%k1} {z}
+; KNL-NEXT:    retq
+;
+; SKX-LABEL: ternlog_maskz_xor_and_mask:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    vpmovq2m %xmm2, %k1
+; SKX-NEXT:    vandpd {{.*}}(%rip), %xmm0, %xmm0
+; SKX-NEXT:    vxorpd %xmm1, %xmm0, %xmm0 {%k1} {z}
+; SKX-NEXT:    retq
+  %m = icmp slt <2 x i64> %mask, zeroinitializer
+  %a = and <2 x i64> %x, <i64 1099511627775, i64 1099511627775>
+  %b = xor <2 x i64> %a, %y
+  %c = select <2 x i1> %m, <2 x i64> %b, <2 x i64> zeroinitializer
+  ret <2 x i64> %c
+}
+
+define <4 x i64> @ternlog_maskz_xor_and_mask_ymm(<4 x i64> %x, <4 x i64> %y, <4 x i64> %mask) {
+; KNL-LABEL: ternlog_maskz_xor_and_mask_ymm:
+; KNL:       ## %bb.0:
+; KNL-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; KNL-NEXT:    vpcmpgtq %ymm2, %ymm3, %k1
+; KNL-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
+; KNL-NEXT:    vpxorq %ymm1, %ymm0, %ymm0 {%k1} {z}
+; KNL-NEXT:    retq
+;
+; SKX-LABEL: ternlog_maskz_xor_and_mask_ymm:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    vpmovq2m %ymm2, %k1
+; SKX-NEXT:    vandpd {{.*}}(%rip), %ymm0, %ymm0
+; SKX-NEXT:    vxorpd %ymm1, %ymm0, %ymm0 {%k1} {z}
+; SKX-NEXT:    retq
+  %m = icmp slt <4 x i64> %mask, zeroinitializer
+  %a = and <4 x i64> %x, <i64 72057594037927935, i64 72057594037927935, i64 72057594037927935, i64 72057594037927935>
+  %b = xor <4 x i64> %a, %y
+  %c = select <4 x i1> %m, <4 x i64> %b, <4 x i64> zeroinitializer
+  ret <4 x i64> %c
+}
+
+define <4 x i32> @ternlog_maskx_or_and_mask(<4 x i32> %x, <4 x i32> %y, <4 x i32> %z, <4 x i32> %mask) {
+; KNL-LABEL: ternlog_maskx_or_and_mask:
+; KNL:       ## %bb.0:
+; KNL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; KNL-NEXT:    vpcmpgtd %xmm3, %xmm2, %k1
+; KNL-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm2
+; KNL-NEXT:    vpord %xmm1, %xmm2, %xmm0 {%k1}
+; KNL-NEXT:    retq
+;
+; SKX-LABEL: ternlog_maskx_or_and_mask:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    vpmovd2m %xmm3, %k1
+; SKX-NEXT:    vandps {{.*}}(%rip), %xmm0, %xmm2
+; SKX-NEXT:    vorps %xmm1, %xmm2, %xmm0 {%k1}
+; SKX-NEXT:    retq
+  %m = icmp slt <4 x i32> %mask, zeroinitializer
+  %a = and <4 x i32> %x, <i32 255, i32 255, i32 255, i32 255>
+  %b = or <4 x i32> %a, %y
+  %c = select <4 x i1> %m, <4 x i32> %b, <4 x i32> %x
+  ret <4 x i32> %c
+}
+
+define <8 x i32> @ternlog_maskx_or_and_mask_ymm(<8 x i32> %x, <8 x i32> %y, <8 x i32> %mask) {
+; KNL-LABEL: ternlog_maskx_or_and_mask_ymm:
+; KNL:       ## %bb.0:
+; KNL-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; KNL-NEXT:    vpcmpgtd %ymm2, %ymm3, %k1
+; KNL-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm2
+; KNL-NEXT:    vpord %ymm1, %ymm2, %ymm0 {%k1}
+; KNL-NEXT:    retq
+;
+; SKX-LABEL: ternlog_maskx_or_and_mask_ymm:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    vpmovd2m %ymm2, %k1
+; SKX-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm2
+; SKX-NEXT:    vorps %ymm1, %ymm2, %ymm0 {%k1}
+; SKX-NEXT:    retq
+  %m = icmp slt <8 x i32> %mask, zeroinitializer
+  %a = and <8 x i32> %x, <i32 -16777216, i32 -16777216, i32 -16777216, i32 -16777216, i32 -16777216, i32 -16777216, i32 -16777216, i32 -16777216>
+  %b = or <8 x i32> %a, %y
+  %c = select <8 x i1> %m, <8 x i32> %b, <8 x i32> %x
+  ret <8 x i32> %c
+}
+
+define <2 x i64> @ternlog_maskx_xor_and_mask(<2 x i64> %x, <2 x i64> %y, <2 x i64> %mask) {
+; KNL-LABEL: ternlog_maskx_xor_and_mask:
+; KNL:       ## %bb.0:
+; KNL-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; KNL-NEXT:    vpcmpgtq %xmm2, %xmm3, %k1
+; KNL-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm2
+; KNL-NEXT:    vpxorq %xmm1, %xmm2, %xmm0 {%k1}
+; KNL-NEXT:    retq
+;
+; SKX-LABEL: ternlog_maskx_xor_and_mask:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    vpmovq2m %xmm2, %k1
+; SKX-NEXT:    vandpd {{.*}}(%rip), %xmm0, %xmm2
+; SKX-NEXT:    vxorpd %xmm1, %xmm2, %xmm0 {%k1}
+; SKX-NEXT:    retq
+  %m = icmp slt <2 x i64> %mask, zeroinitializer
+  %a = and <2 x i64> %x, <i64 1099511627775, i64 1099511627775>
+  %b = xor <2 x i64> %a, %y
+  %c = select <2 x i1> %m, <2 x i64> %b, <2 x i64> %x
+  ret <2 x i64> %c
+}
+
+define <4 x i64> @ternlog_maskx_xor_and_mask_ymm(<4 x i64> %x, <4 x i64> %y, <4 x i64> %mask) {
+; KNL-LABEL: ternlog_maskx_xor_and_mask_ymm:
+; KNL:       ## %bb.0:
+; KNL-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; KNL-NEXT:    vpcmpgtq %ymm2, %ymm3, %k1
+; KNL-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm2
+; KNL-NEXT:    vpxorq %ymm1, %ymm2, %ymm0 {%k1}
+; KNL-NEXT:    retq
+;
+; SKX-LABEL: ternlog_maskx_xor_and_mask_ymm:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    vpmovq2m %ymm2, %k1
+; SKX-NEXT:    vandpd {{.*}}(%rip), %ymm0, %ymm2
+; SKX-NEXT:    vxorpd %ymm1, %ymm2, %ymm0 {%k1}
+; SKX-NEXT:    retq
+  %m = icmp slt <4 x i64> %mask, zeroinitializer
+  %a = and <4 x i64> %x, <i64 72057594037927935, i64 72057594037927935, i64 72057594037927935, i64 72057594037927935>
+  %b = xor <4 x i64> %a, %y
+  %c = select <4 x i1> %m, <4 x i64> %b, <4 x i64> %x
+  ret <4 x i64> %c
+}
+
+define <4 x i32> @ternlog_masky_or_and_mask(<4 x i32> %x, <4 x i32> %y, <4 x i32> %z, <4 x i32> %mask) {
+; KNL-LABEL: ternlog_masky_or_and_mask:
+; KNL:       ## %bb.0:
+; KNL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; KNL-NEXT:    vpcmpgtd %xmm3, %xmm2, %k1
+; KNL-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; KNL-NEXT:    vpord %xmm1, %xmm0, %xmm1 {%k1}
+; KNL-NEXT:    vmovdqa %xmm1, %xmm0
+; KNL-NEXT:    retq
+;
+; SKX-LABEL: ternlog_masky_or_and_mask:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    vpmovd2m %xmm3, %k1
+; SKX-NEXT:    vandps {{.*}}(%rip), %xmm0, %xmm0
+; SKX-NEXT:    vorps %xmm1, %xmm0, %xmm1 {%k1}
+; SKX-NEXT:    vmovaps %xmm1, %xmm0
+; SKX-NEXT:    retq
+  %m = icmp slt <4 x i32> %mask, zeroinitializer
+  %a = and <4 x i32> %x, <i32 255, i32 255, i32 255, i32 255>
+  %b = or <4 x i32> %a, %y
+  %c = select <4 x i1> %m, <4 x i32> %b, <4 x i32> %y
+  ret <4 x i32> %c
+}
+
+define <8 x i32> @ternlog_masky_or_and_mask_ymm(<8 x i32> %x, <8 x i32> %y, <8 x i32> %mask) {
+; KNL-LABEL: ternlog_masky_or_and_mask_ymm:
+; KNL:       ## %bb.0:
+; KNL-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; KNL-NEXT:    vpcmpgtd %ymm2, %ymm3, %k1
+; KNL-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm2
+; KNL-NEXT:    vpord %ymm1, %ymm2, %ymm0 {%k1}
+; KNL-NEXT:    retq
+;
+; SKX-LABEL: ternlog_masky_or_and_mask_ymm:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    vpmovd2m %ymm2, %k1
+; SKX-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm2
+; SKX-NEXT:    vorps %ymm1, %ymm2, %ymm0 {%k1}
+; SKX-NEXT:    retq
+  %m = icmp slt <8 x i32> %mask, zeroinitializer
+  %a = and <8 x i32> %x, <i32 -16777216, i32 -16777216, i32 -16777216, i32 -16777216, i32 -16777216, i32 -16777216, i32 -16777216, i32 -16777216>
+  %b = or <8 x i32> %a, %y
+  %c = select <8 x i1> %m, <8 x i32> %b, <8 x i32> %x
+  ret <8 x i32> %c
+}
+
+define <2 x i64> @ternlog_masky_xor_and_mask(<2 x i64> %x, <2 x i64> %y, <2 x i64> %mask) {
+; KNL-LABEL: ternlog_masky_xor_and_mask:
+; KNL:       ## %bb.0:
+; KNL-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; KNL-NEXT:    vpcmpgtq %xmm2, %xmm3, %k1
+; KNL-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; KNL-NEXT:    vpxorq %xmm1, %xmm0, %xmm1 {%k1}
+; KNL-NEXT:    vmovdqa %xmm1, %xmm0
+; KNL-NEXT:    retq
+;
+; SKX-LABEL: ternlog_masky_xor_and_mask:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    vpmovq2m %xmm2, %k1
+; SKX-NEXT:    vandpd {{.*}}(%rip), %xmm0, %xmm0
+; SKX-NEXT:    vxorpd %xmm1, %xmm0, %xmm1 {%k1}
+; SKX-NEXT:    vmovapd %xmm1, %xmm0
+; SKX-NEXT:    retq
+  %m = icmp slt <2 x i64> %mask, zeroinitializer
+  %a = and <2 x i64> %x, <i64 1099511627775, i64 1099511627775>
+  %b = xor <2 x i64> %a, %y
+  %c = select <2 x i1> %m, <2 x i64> %b, <2 x i64> %y
+  ret <2 x i64> %c
+}
+
+define <4 x i64> @ternlog_masky_xor_and_mask_ymm(<4 x i64> %x, <4 x i64> %y, <4 x i64> %mask) {
+; KNL-LABEL: ternlog_masky_xor_and_mask_ymm:
+; KNL:       ## %bb.0:
+; KNL-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; KNL-NEXT:    vpcmpgtq %ymm2, %ymm3, %k1
+; KNL-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
+; KNL-NEXT:    vpxorq %ymm1, %ymm0, %ymm1 {%k1}
+; KNL-NEXT:    vmovdqa %ymm1, %ymm0
+; KNL-NEXT:    retq
+;
+; SKX-LABEL: ternlog_masky_xor_and_mask_ymm:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    vpmovq2m %ymm2, %k1
+; SKX-NEXT:    vandpd {{.*}}(%rip), %ymm0, %ymm0
+; SKX-NEXT:    vxorpd %ymm1, %ymm0, %ymm1 {%k1}
+; SKX-NEXT:    vmovapd %ymm1, %ymm0
+; SKX-NEXT:    retq
+  %m = icmp slt <4 x i64> %mask, zeroinitializer
+  %a = and <4 x i64> %x, <i64 72057594037927935, i64 72057594037927935, i64 72057594037927935, i64 72057594037927935>
+  %b = xor <4 x i64> %a, %y
+  %c = select <4 x i1> %m, <4 x i64> %b, <4 x i64> %y
+  ret <4 x i64> %c
+}

From 55dae9c20ce3f1149cf4c773258536f972e411c0 Mon Sep 17 00:00:00 2001
From: Philip Reames <listmail@philipreames.com>
Date: Sat, 25 Jul 2020 16:40:06 -0700
Subject: [PATCH 0105/1035] [Statepoints] Style cleanup after 3da1a963 [NFC]

Just fixing a few minor stylistic issues.
---
 .../SelectionDAG/StatepointLowering.cpp       | 40 +++++++++----------
 1 file changed, 19 insertions(+), 21 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
index b8c4c73bcccee..3063993ba97d8 100644
--- a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
@@ -678,27 +678,24 @@ lowerStatepointMetaArgs(SmallVectorImpl<SDValue> &Ops,
     SDValue SDV = Builder.getValue(V);
     SDValue Loc = Builder.StatepointLowering.getLocation(SDV);
 
-    if (LowerAsVReg.count(SDV)) {
-      SpillMap[V] = None;
-    } else if (Loc.getNode()) {
+    if (Loc.getNode()) {
+      // If this is a value we spilled, remember where for when we visit the
+      // gc.relocate corresponding to this gc.statepoint
       SpillMap[V] = cast<FrameIndexSDNode>(Loc)->getIndex();
     } else {
-      // Record value as visited, but not spilled. This is case for allocas
-      // and constants. For this values we can avoid emitting spill load while
-      // visiting corresponding gc_relocate.
-      // Actually we do not need to record them in this map at all.
-      // We do this only to check that we are not relocating any unvisited
-      // value.
+      // If we didn't spill the value - allocas, constants, and values lowered
+      // as tied vregs - mark them as visited, but not spilled.  Marking them
+      // visited (as opposed to simply missing in the map), allows tighter
+      // assertion checking.  
       SpillMap[V] = None;
 
-      // Default llvm mechanisms for exporting values which are used in
-      // different basic blocks does not work for gc relocates.
-      // Note that it would be incorrect to teach llvm that all relocates are
-      // uses of the corresponding values so that it would automatically
-      // export them. Relocates of the spilled values does not use original
-      // value.
-      if (Relocate->getParent() != StatepointInstr->getParent())
+      // Conservatively export all values used by gc.relocates outside this
+      // block.  This is currently only needed for expressions which don't need
+      // relocation, but will likely be extended for vreg case shortly.
+      if (Relocate->getParent() != StatepointInstr->getParent()) {
         Builder.ExportFromCurrentBlock(V);
+        assert(!LowerAsVReg.count(SDV));
+      }
     }
   }
 }
@@ -844,9 +841,9 @@ SDValue SelectionDAGBuilder::LowerAsSTATEPOINT(
   SmallVector<EVT, 8> NodeTys;
   for (auto &Ptr : SI.Ptrs) {
     SDValue SD = getValue(Ptr);
-    if (LowerAsVReg.count(SD)) {
-      NodeTys.push_back(SD.getValueType());
-    }
+    if (!LowerAsVReg.count(SD))
+      continue;
+    NodeTys.push_back(SD.getValueType());
   }
   LLVM_DEBUG(dbgs() << "Statepoint has " << NodeTys.size() << " results\n");
   assert(NodeTys.size() == LowerAsVReg.size() && "Inconsistent GC Ptr lowering");
@@ -866,8 +863,9 @@ SDValue SelectionDAGBuilder::LowerAsSTATEPOINT(
   for (const auto *Relocate : SI.GCRelocates) {
     Value *Derived = Relocate->getDerivedPtr();
     SDValue SD = getValue(Derived);
-    if (LowerAsVReg.count(SD))
-      DPtrMap[Derived] = SDValue(StatepointMCNode, LowerAsVReg[SD]);
+    if (!LowerAsVReg.count(SD))
+      continue;
+    DPtrMap[Derived] = SDValue(StatepointMCNode, LowerAsVReg[SD]);
   }
 
   // Build the GC_TRANSITION_END node if necessary.

From cdead4f89c0eecf11f50092bc088e3a9c6511825 Mon Sep 17 00:00:00 2001
From: Nemanja Ivanovic <nemanja.i.ibm@gmail.com>
Date: Sat, 25 Jul 2020 20:28:52 -0400
Subject: [PATCH 0106/1035] [PowerPC][NFC] Fix an assert that cannot trip from
 7d076e19e31a

I mixed up the precedence of operators in the assert and thought I
had it right since there was no compiler warning. This just
adds the parentheses in the expression as needed.
---
 llvm/lib/Target/PowerPC/PPCISelLowering.cpp | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index 8c28ead9f6041..ae840a9fa37de 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -9311,7 +9311,8 @@ SDValue PPCTargetLowering::LowerBUILD_VECTOR(SDValue Op,
   if (!BVNIsConstantSplat || SplatBitSize > 32) {
 
     bool IsPermutedLoad = false;
-    const SDValue *InputLoad = getNormalLoadInput(Op.getOperand(0), IsPermutedLoad);
+    const SDValue *InputLoad =
+        getNormalLoadInput(Op.getOperand(0), IsPermutedLoad);
     // Handle load-and-splat patterns as we have instructions that will do this
     // in one go.
     if (InputLoad && DAG.isSplatValue(Op, true)) {
@@ -9949,7 +9950,7 @@ SDValue PPCTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
     if (IsPermutedLoad) {
       assert(isLittleEndian && "Unexpected permuted load on big endian target");
       SplatIdx += IsFourByte ? 2 : 1;
-      assert(SplatIdx < IsFourByte ? 4 : 2 &&
+      assert((SplatIdx < (IsFourByte ? 4 : 2)) &&
              "Splat of a value outside of the loaded memory");
     }
 

From 18975762c1974047baeeacb17879416410012a31 Mon Sep 17 00:00:00 2001
From: Eric Christopher <echristo@gmail.com>
Date: Sat, 25 Jul 2020 18:34:02 -0700
Subject: [PATCH 0107/1035] Fold StatepointBB into checks as it's only used
 from an NDEBUG or ASSERT context fixing an unused variable warning.

---
 llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
index 3063993ba97d8..4d93dec9e1fad 100644
--- a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
@@ -1087,13 +1087,12 @@ void SelectionDAGBuilder::visitGCResult(const GCResultInst &CI) {
 }
 
 void SelectionDAGBuilder::visitGCRelocate(const GCRelocateInst &Relocate) {
-  const BasicBlock *StatepointBB = Relocate.getStatepoint()->getParent();
 #ifndef NDEBUG
   // Consistency check
   // We skip this check for relocates not in the same basic block as their
   // statepoint. It would be too expensive to preserve validation info through
   // different basic blocks.
-  if (StatepointBB == Relocate.getParent())
+  if (Relocate.getStatepoint()->getParent() == Relocate.getParent())
     StatepointLowering.relocCallVisited(Relocate);
 
   auto *Ty = Relocate.getType()->getScalarType();
@@ -1117,7 +1116,8 @@ void SelectionDAGBuilder::visitGCRelocate(const GCRelocateInst &Relocate) {
   auto It = DPtrMap.find(DerivedPtr);
   if (It != DPtrMap.end()) {
     setValue(&Relocate, It->second);
-    assert(Relocate.getParent() == StatepointBB && "unexpected DPtrMap entry");
+    assert(Relocate.getParent() == Relocate.getStatepoint()->getParent() &&
+           "unexpected DPtrMap entry");
     return;
   }
 

From 4b14ef33e81c01632e848e7a67ccc6b11fb4c595 Mon Sep 17 00:00:00 2001
From: Eric Christopher <echristo@gmail.com>
Date: Sat, 25 Jul 2020 18:42:04 -0700
Subject: [PATCH 0108/1035] Temporarily Revert "Unify the return value of
 GetByteSize to an llvm::Optional<uint64_t> (NFC-ish)" as it's causing
 numerous (176) test failures on linux.

This reverts commit 1d9b860fb6a85df33fd52fcacc6a5efb421621bd.
---
 lldb/include/lldb/Core/ValueObject.h          |  2 +-
 lldb/include/lldb/Core/ValueObjectCast.h      |  2 +-
 lldb/include/lldb/Core/ValueObjectChild.h     |  2 +-
 .../lldb/Core/ValueObjectConstResult.h        |  4 ++--
 .../lldb/Core/ValueObjectDynamicValue.h       |  2 +-
 lldb/include/lldb/Core/ValueObjectMemory.h    |  2 +-
 lldb/include/lldb/Core/ValueObjectRegister.h  |  4 ++--
 .../lldb/Core/ValueObjectSyntheticFilter.h    |  2 +-
 lldb/include/lldb/Core/ValueObjectVariable.h  |  2 +-
 .../lldb/Expression/ExpressionVariable.h      |  2 +-
 .../lldb/Target/StackFrameRecognizer.h        |  4 +---
 lldb/source/API/SBValue.cpp                   |  2 +-
 .../Commands/CommandObjectWatchpoint.cpp      |  2 +-
 lldb/source/Core/ValueObject.cpp              | 12 +++++-----
 lldb/source/Core/ValueObjectCast.cpp          |  2 +-
 lldb/source/Core/ValueObjectConstResult.cpp   | 10 ++++----
 lldb/source/Core/ValueObjectDynamicValue.cpp  |  2 +-
 lldb/source/Core/ValueObjectMemory.cpp        |  8 ++++---
 lldb/source/Core/ValueObjectRegister.cpp      |  6 ++---
 .../Core/ValueObjectSyntheticFilter.cpp       |  4 +---
 lldb/source/Core/ValueObjectVariable.cpp      |  6 ++---
 lldb/source/Expression/ExpressionVariable.cpp |  8 +++----
 lldb/source/Expression/Materializer.cpp       | 23 ++++++++-----------
 lldb/source/Target/StackFrame.cpp             |  6 ++---
 24 files changed, 57 insertions(+), 62 deletions(-)

diff --git a/lldb/include/lldb/Core/ValueObject.h b/lldb/include/lldb/Core/ValueObject.h
index a557d69f3ae30..0080368fd9965 100644
--- a/lldb/include/lldb/Core/ValueObject.h
+++ b/lldb/include/lldb/Core/ValueObject.h
@@ -358,7 +358,7 @@ class ValueObject : public UserID {
   virtual bool CanProvideValue();
 
   // Subclasses must implement the functions below.
-  virtual llvm::Optional<uint64_t> GetByteSize() = 0;
+  virtual uint64_t GetByteSize() = 0;
 
   virtual lldb::ValueType GetValueType() const = 0;
 
diff --git a/lldb/include/lldb/Core/ValueObjectCast.h b/lldb/include/lldb/Core/ValueObjectCast.h
index 342803f8ca63a..d91ca6a92be8d 100644
--- a/lldb/include/lldb/Core/ValueObjectCast.h
+++ b/lldb/include/lldb/Core/ValueObjectCast.h
@@ -30,7 +30,7 @@ class ValueObjectCast : public ValueObject {
                                     ConstString name,
                                     const CompilerType &cast_type);
 
-  llvm::Optional<uint64_t> GetByteSize() override;
+  uint64_t GetByteSize() override;
 
   size_t CalculateNumChildren(uint32_t max) override;
 
diff --git a/lldb/include/lldb/Core/ValueObjectChild.h b/lldb/include/lldb/Core/ValueObjectChild.h
index 9a9fd9294261a..c6f44a29b0591 100644
--- a/lldb/include/lldb/Core/ValueObjectChild.h
+++ b/lldb/include/lldb/Core/ValueObjectChild.h
@@ -30,7 +30,7 @@ class ValueObjectChild : public ValueObject {
 public:
   ~ValueObjectChild() override;
 
-  llvm::Optional<uint64_t> GetByteSize() override { return m_byte_size; }
+  uint64_t GetByteSize() override { return m_byte_size; }
 
   lldb::offset_t GetByteOffset() override { return m_byte_offset; }
 
diff --git a/lldb/include/lldb/Core/ValueObjectConstResult.h b/lldb/include/lldb/Core/ValueObjectConstResult.h
index 8d823baa0b7b4..0e868c687e931 100644
--- a/lldb/include/lldb/Core/ValueObjectConstResult.h
+++ b/lldb/include/lldb/Core/ValueObjectConstResult.h
@@ -62,7 +62,7 @@ class ValueObjectConstResult : public ValueObject {
   static lldb::ValueObjectSP Create(ExecutionContextScope *exe_scope,
                                     const Status &error);
 
-  llvm::Optional<uint64_t> GetByteSize() override;
+  uint64_t GetByteSize() override;
 
   lldb::ValueType GetValueType() const override;
 
@@ -113,7 +113,7 @@ class ValueObjectConstResult : public ValueObject {
   CompilerType GetCompilerTypeImpl() override;
 
   ConstString m_type_name;
-  llvm::Optional<uint64_t> m_byte_size;
+  uint64_t m_byte_size;
 
   ValueObjectConstResultImpl m_impl;
 
diff --git a/lldb/include/lldb/Core/ValueObjectDynamicValue.h b/lldb/include/lldb/Core/ValueObjectDynamicValue.h
index 2806857339efb..9f5304b55e934 100644
--- a/lldb/include/lldb/Core/ValueObjectDynamicValue.h
+++ b/lldb/include/lldb/Core/ValueObjectDynamicValue.h
@@ -34,7 +34,7 @@ class ValueObjectDynamicValue : public ValueObject {
 public:
   ~ValueObjectDynamicValue() override;
 
-  llvm::Optional<uint64_t> GetByteSize() override;
+  uint64_t GetByteSize() override;
 
   ConstString GetTypeName() override;
 
diff --git a/lldb/include/lldb/Core/ValueObjectMemory.h b/lldb/include/lldb/Core/ValueObjectMemory.h
index b5d5e6ecf4c0e..d1cd6ae41445d 100644
--- a/lldb/include/lldb/Core/ValueObjectMemory.h
+++ b/lldb/include/lldb/Core/ValueObjectMemory.h
@@ -40,7 +40,7 @@ class ValueObjectMemory : public ValueObject {
                                     const Address &address,
                                     const CompilerType &ast_type);
 
-  llvm::Optional<uint64_t> GetByteSize() override;
+  uint64_t GetByteSize() override;
 
   ConstString GetTypeName() override;
 
diff --git a/lldb/include/lldb/Core/ValueObjectRegister.h b/lldb/include/lldb/Core/ValueObjectRegister.h
index 3968584ad5185..41051d93b707e 100644
--- a/lldb/include/lldb/Core/ValueObjectRegister.h
+++ b/lldb/include/lldb/Core/ValueObjectRegister.h
@@ -36,7 +36,7 @@ class ValueObjectRegisterSet : public ValueObject {
                                     lldb::RegisterContextSP &reg_ctx_sp,
                                     uint32_t set_idx);
 
-  llvm::Optional<uint64_t> GetByteSize() override;
+  uint64_t GetByteSize() override;
 
   lldb::ValueType GetValueType() const override {
     return lldb::eValueTypeRegisterSet;
@@ -86,7 +86,7 @@ class ValueObjectRegister : public ValueObject {
                                     lldb::RegisterContextSP &reg_ctx_sp,
                                     uint32_t reg_num);
 
-  llvm::Optional<uint64_t> GetByteSize() override;
+  uint64_t GetByteSize() override;
 
   lldb::ValueType GetValueType() const override {
     return lldb::eValueTypeRegister;
diff --git a/lldb/include/lldb/Core/ValueObjectSyntheticFilter.h b/lldb/include/lldb/Core/ValueObjectSyntheticFilter.h
index 41c461ce13f0d..cb471657aec9b 100644
--- a/lldb/include/lldb/Core/ValueObjectSyntheticFilter.h
+++ b/lldb/include/lldb/Core/ValueObjectSyntheticFilter.h
@@ -36,7 +36,7 @@ class ValueObjectSynthetic : public ValueObject {
 public:
   ~ValueObjectSynthetic() override;
 
-  llvm::Optional<uint64_t> GetByteSize() override;
+  uint64_t GetByteSize() override;
 
   ConstString GetTypeName() override;
 
diff --git a/lldb/include/lldb/Core/ValueObjectVariable.h b/lldb/include/lldb/Core/ValueObjectVariable.h
index 23fdedbf5a4a6..b7e262574a14d 100644
--- a/lldb/include/lldb/Core/ValueObjectVariable.h
+++ b/lldb/include/lldb/Core/ValueObjectVariable.h
@@ -37,7 +37,7 @@ class ValueObjectVariable : public ValueObject {
   static lldb::ValueObjectSP Create(ExecutionContextScope *exe_scope,
                                     const lldb::VariableSP &var_sp);
 
-  llvm::Optional<uint64_t> GetByteSize() override;
+  uint64_t GetByteSize() override;
 
   ConstString GetTypeName() override;
 
diff --git a/lldb/include/lldb/Expression/ExpressionVariable.h b/lldb/include/lldb/Expression/ExpressionVariable.h
index 4259e6395da47..60062d212badf 100644
--- a/lldb/include/lldb/Expression/ExpressionVariable.h
+++ b/lldb/include/lldb/Expression/ExpressionVariable.h
@@ -32,7 +32,7 @@ class ExpressionVariable
 
   virtual ~ExpressionVariable();
 
-  llvm::Optional<uint64_t> GetByteSize() { return m_frozen_sp->GetByteSize(); }
+  size_t GetByteSize() { return m_frozen_sp->GetByteSize(); }
 
   ConstString GetName() { return m_frozen_sp->GetName(); }
 
diff --git a/lldb/include/lldb/Target/StackFrameRecognizer.h b/lldb/include/lldb/Target/StackFrameRecognizer.h
index baffc890bb065..302b56bec907b 100644
--- a/lldb/include/lldb/Target/StackFrameRecognizer.h
+++ b/lldb/include/lldb/Target/StackFrameRecognizer.h
@@ -154,9 +154,7 @@ class ValueObjectRecognizerSynthesizedValue : public ValueObject {
     SetName(parent.GetName());
   }
 
-  llvm::Optional<uint64_t> GetByteSize() override {
-    return m_parent->GetByteSize();
-  }
+  uint64_t GetByteSize() override { return m_parent->GetByteSize(); }
   lldb::ValueType GetValueType() const override { return m_type; }
   bool UpdateValue() override {
     if (!m_parent->UpdateValueIfNeeded()) return false;
diff --git a/lldb/source/API/SBValue.cpp b/lldb/source/API/SBValue.cpp
index 686d1f23a75a8..7485b0ee1838e 100644
--- a/lldb/source/API/SBValue.cpp
+++ b/lldb/source/API/SBValue.cpp
@@ -333,7 +333,7 @@ size_t SBValue::GetByteSize() {
   ValueLocker locker;
   lldb::ValueObjectSP value_sp(GetSP(locker));
   if (value_sp) {
-    result = value_sp->GetByteSize().getValueOr(0);
+    result = value_sp->GetByteSize();
   }
 
   return result;
diff --git a/lldb/source/Commands/CommandObjectWatchpoint.cpp b/lldb/source/Commands/CommandObjectWatchpoint.cpp
index c2a008af79d6f..ce4662930a7c2 100644
--- a/lldb/source/Commands/CommandObjectWatchpoint.cpp
+++ b/lldb/source/Commands/CommandObjectWatchpoint.cpp
@@ -905,7 +905,7 @@ corresponding to the byte size of the data type.");
         // We're in business.
         // Find out the size of this variable.
         size = m_option_watchpoint.watch_size == 0
-                   ? valobj_sp->GetByteSize().getValueOr(0)
+                   ? valobj_sp->GetByteSize()
                    : m_option_watchpoint.watch_size;
       }
       compiler_type = valobj_sp->GetCompilerType();
diff --git a/lldb/source/Core/ValueObject.cpp b/lldb/source/Core/ValueObject.cpp
index aedefd0cf0fd9..3a775b07e5e1f 100644
--- a/lldb/source/Core/ValueObject.cpp
+++ b/lldb/source/Core/ValueObject.cpp
@@ -849,7 +849,7 @@ bool ValueObject::SetData(DataExtractor &data, Status &error) {
   uint64_t count = 0;
   const Encoding encoding = GetCompilerType().GetEncoding(count);
 
-  const size_t byte_size = GetByteSize().getValueOr(0);
+  const size_t byte_size = GetByteSize();
 
   Value::ValueType value_type = m_value.GetValueType();
 
@@ -1524,7 +1524,7 @@ bool ValueObject::SetValueFromCString(const char *value_str, Status &error) {
   uint64_t count = 0;
   const Encoding encoding = GetCompilerType().GetEncoding(count);
 
-  const size_t byte_size = GetByteSize().getValueOr(0);
+  const size_t byte_size = GetByteSize();
 
   Value::ValueType value_type = m_value.GetValueType();
 
@@ -1741,13 +1741,13 @@ ValueObjectSP ValueObject::GetSyntheticBitFieldChild(uint32_t from, uint32_t to,
       uint32_t bit_field_offset = from;
       if (GetDataExtractor().GetByteOrder() == eByteOrderBig)
         bit_field_offset =
-            GetByteSize().getValueOr(0) * 8 - bit_field_size - bit_field_offset;
+            GetByteSize() * 8 - bit_field_size - bit_field_offset;
       // We haven't made a synthetic array member for INDEX yet, so lets make
       // one and cache it for any future reference.
       ValueObjectChild *synthetic_child = new ValueObjectChild(
-          *this, GetCompilerType(), index_const_str,
-          GetByteSize().getValueOr(0), 0, bit_field_size, bit_field_offset,
-          false, false, eAddressTypeInvalid, 0);
+          *this, GetCompilerType(), index_const_str, GetByteSize(), 0,
+          bit_field_size, bit_field_offset, false, false, eAddressTypeInvalid,
+          0);
 
       // Cache the value if we got one back...
       if (synthetic_child) {
diff --git a/lldb/source/Core/ValueObjectCast.cpp b/lldb/source/Core/ValueObjectCast.cpp
index 7b6d3591faf44..22e856be539b5 100644
--- a/lldb/source/Core/ValueObjectCast.cpp
+++ b/lldb/source/Core/ValueObjectCast.cpp
@@ -47,7 +47,7 @@ size_t ValueObjectCast::CalculateNumChildren(uint32_t max) {
   return children_count <= max ? children_count : max;
 }
 
-llvm::Optional<uint64_t> ValueObjectCast::GetByteSize() {
+uint64_t ValueObjectCast::GetByteSize() {
   ExecutionContext exe_ctx(GetExecutionContextRef());
   return m_value.GetValueByteSize(nullptr, &exe_ctx);
 }
diff --git a/lldb/source/Core/ValueObjectConstResult.cpp b/lldb/source/Core/ValueObjectConstResult.cpp
index fd31ddc676b43..8d84f8e62ccc5 100644
--- a/lldb/source/Core/ValueObjectConstResult.cpp
+++ b/lldb/source/Core/ValueObjectConstResult.cpp
@@ -179,7 +179,8 @@ ValueObjectSP ValueObjectConstResult::Create(ExecutionContextScope *exe_scope,
 ValueObjectConstResult::ValueObjectConstResult(ExecutionContextScope *exe_scope,
                                                ValueObjectManager &manager,
                                                const Status &error)
-    : ValueObject(exe_scope, manager), m_impl(this) {
+    : ValueObject(exe_scope, manager), m_type_name(), m_byte_size(0),
+      m_impl(this) {
   m_error = error;
   SetIsConstant();
 }
@@ -188,7 +189,8 @@ ValueObjectConstResult::ValueObjectConstResult(ExecutionContextScope *exe_scope,
                                                ValueObjectManager &manager,
                                                const Value &value,
                                                ConstString name, Module *module)
-    : ValueObject(exe_scope, manager), m_impl(this) {
+    : ValueObject(exe_scope, manager), m_type_name(), m_byte_size(0),
+      m_impl(this) {
   m_value = value;
   m_name = name;
   ExecutionContext exe_ctx;
@@ -206,9 +208,9 @@ lldb::ValueType ValueObjectConstResult::GetValueType() const {
   return eValueTypeConstResult;
 }
 
-llvm::Optional<uint64_t> ValueObjectConstResult::GetByteSize() {
+uint64_t ValueObjectConstResult::GetByteSize() {
   ExecutionContext exe_ctx(GetExecutionContextRef());
-  if (!m_byte_size) {
+  if (m_byte_size == 0) {
     if (auto size =
         GetCompilerType().GetByteSize(exe_ctx.GetBestExecutionContextScope()))
       SetByteSize(*size);
diff --git a/lldb/source/Core/ValueObjectDynamicValue.cpp b/lldb/source/Core/ValueObjectDynamicValue.cpp
index 1c25b8c85a059..ca66740cb55d4 100644
--- a/lldb/source/Core/ValueObjectDynamicValue.cpp
+++ b/lldb/source/Core/ValueObjectDynamicValue.cpp
@@ -98,7 +98,7 @@ size_t ValueObjectDynamicValue::CalculateNumChildren(uint32_t max) {
     return m_parent->GetNumChildren(max);
 }
 
-llvm::Optional<uint64_t> ValueObjectDynamicValue::GetByteSize() {
+uint64_t ValueObjectDynamicValue::GetByteSize() {
   const bool success = UpdateValueIfNeeded(false);
   if (success && m_dynamic_type_info.HasType()) {
     ExecutionContext exe_ctx(GetExecutionContextRef());
diff --git a/lldb/source/Core/ValueObjectMemory.cpp b/lldb/source/Core/ValueObjectMemory.cpp
index 17fade9e5fdc3..8e7d3ebc93f69 100644
--- a/lldb/source/Core/ValueObjectMemory.cpp
+++ b/lldb/source/Core/ValueObjectMemory.cpp
@@ -139,11 +139,13 @@ size_t ValueObjectMemory::CalculateNumChildren(uint32_t max) {
   return child_count <= max ? child_count : max;
 }
 
-llvm::Optional<uint64_t> ValueObjectMemory::GetByteSize() {
+uint64_t ValueObjectMemory::GetByteSize() {
   ExecutionContext exe_ctx(GetExecutionContextRef());
   if (m_type_sp)
-    return m_type_sp->GetByteSize(exe_ctx.GetBestExecutionContextScope());
-  return m_compiler_type.GetByteSize(exe_ctx.GetBestExecutionContextScope());
+    return m_type_sp->GetByteSize(exe_ctx.GetBestExecutionContextScope())
+        .getValueOr(0);
+  return m_compiler_type.GetByteSize(exe_ctx.GetBestExecutionContextScope())
+      .getValueOr(0);
 }
 
 lldb::ValueType ValueObjectMemory::GetValueType() const {
diff --git a/lldb/source/Core/ValueObjectRegister.cpp b/lldb/source/Core/ValueObjectRegister.cpp
index 27461e9cebc41..ec87c38fb3679 100644
--- a/lldb/source/Core/ValueObjectRegister.cpp
+++ b/lldb/source/Core/ValueObjectRegister.cpp
@@ -81,7 +81,7 @@ size_t ValueObjectRegisterSet::CalculateNumChildren(uint32_t max) {
   return 0;
 }
 
-llvm::Optional<uint64_t> ValueObjectRegisterSet::GetByteSize() { return 0; }
+uint64_t ValueObjectRegisterSet::GetByteSize() { return 0; }
 
 bool ValueObjectRegisterSet::UpdateValue() {
   m_error.Clear();
@@ -229,9 +229,7 @@ size_t ValueObjectRegister::CalculateNumChildren(uint32_t max) {
   return children_count <= max ? children_count : max;
 }
 
-llvm::Optional<uint64_t> ValueObjectRegister::GetByteSize() {
-  return m_reg_info.byte_size;
-}
+uint64_t ValueObjectRegister::GetByteSize() { return m_reg_info.byte_size; }
 
 bool ValueObjectRegister::UpdateValue() {
   m_error.Clear();
diff --git a/lldb/source/Core/ValueObjectSyntheticFilter.cpp b/lldb/source/Core/ValueObjectSyntheticFilter.cpp
index fb2d32e602cea..32d1e6ab8368c 100644
--- a/lldb/source/Core/ValueObjectSyntheticFilter.cpp
+++ b/lldb/source/Core/ValueObjectSyntheticFilter.cpp
@@ -121,9 +121,7 @@ bool ValueObjectSynthetic::MightHaveChildren() {
   return (m_might_have_children != eLazyBoolNo);
 }
 
-llvm::Optional<uint64_t> ValueObjectSynthetic::GetByteSize() {
-  return m_parent->GetByteSize();
-}
+uint64_t ValueObjectSynthetic::GetByteSize() { return m_parent->GetByteSize(); }
 
 lldb::ValueType ValueObjectSynthetic::GetValueType() const {
   return m_parent->GetValueType();
diff --git a/lldb/source/Core/ValueObjectVariable.cpp b/lldb/source/Core/ValueObjectVariable.cpp
index ab67e3038cf0a..0d1e7b047a0ac 100644
--- a/lldb/source/Core/ValueObjectVariable.cpp
+++ b/lldb/source/Core/ValueObjectVariable.cpp
@@ -105,15 +105,15 @@ size_t ValueObjectVariable::CalculateNumChildren(uint32_t max) {
   return child_count <= max ? child_count : max;
 }
 
-llvm::Optional<uint64_t> ValueObjectVariable::GetByteSize() {
+uint64_t ValueObjectVariable::GetByteSize() {
   ExecutionContext exe_ctx(GetExecutionContextRef());
 
   CompilerType type(GetCompilerType());
 
   if (!type.IsValid())
-    return {};
+    return 0;
 
-  return type.GetByteSize(exe_ctx.GetBestExecutionContextScope());
+  return type.GetByteSize(exe_ctx.GetBestExecutionContextScope()).getValueOr(0);
 }
 
 lldb::ValueType ValueObjectVariable::GetValueType() const {
diff --git a/lldb/source/Expression/ExpressionVariable.cpp b/lldb/source/Expression/ExpressionVariable.cpp
index 8b3dda7b2fe10..d95f0745cf4ba 100644
--- a/lldb/source/Expression/ExpressionVariable.cpp
+++ b/lldb/source/Expression/ExpressionVariable.cpp
@@ -16,10 +16,10 @@ using namespace lldb_private;
 ExpressionVariable::~ExpressionVariable() {}
 
 uint8_t *ExpressionVariable::GetValueBytes() {
-  llvm::Optional<uint64_t> byte_size = m_frozen_sp->GetByteSize();
-  if (byte_size && *byte_size) {
-    if (m_frozen_sp->GetDataExtractor().GetByteSize() < *byte_size) {
-      m_frozen_sp->GetValue().ResizeData(*byte_size);
+  const size_t byte_size = m_frozen_sp->GetByteSize();
+  if (byte_size > 0) {
+    if (m_frozen_sp->GetDataExtractor().GetByteSize() < byte_size) {
+      m_frozen_sp->GetValue().ResizeData(byte_size);
       m_frozen_sp->GetValue().GetData(m_frozen_sp->GetDataExtractor());
     }
     return const_cast<uint8_t *>(
diff --git a/lldb/source/Expression/Materializer.cpp b/lldb/source/Expression/Materializer.cpp
index 327e15a26266f..6f8d9b154570a 100644
--- a/lldb/source/Expression/Materializer.cpp
+++ b/lldb/source/Expression/Materializer.cpp
@@ -67,7 +67,7 @@ class EntityPersistentVariable : public Materializer::Entity {
     const bool zero_memory = false;
 
     lldb::addr_t mem = map.Malloc(
-        m_persistent_variable_sp->GetByteSize().getValueOr(0), 8,
+        m_persistent_variable_sp->GetByteSize(), 8,
         lldb::ePermissionsReadable | lldb::ePermissionsWritable,
         IRMemoryMap::eAllocationPolicyMirror, zero_memory, allocate_error);
 
@@ -106,8 +106,7 @@ class EntityPersistentVariable : public Materializer::Entity {
     Status write_error;
 
     map.WriteMemory(mem, m_persistent_variable_sp->GetValueBytes(),
-                    m_persistent_variable_sp->GetByteSize().getValueOr(0),
-                    write_error);
+                    m_persistent_variable_sp->GetByteSize(), write_error);
 
     if (!write_error.Success()) {
       err.SetErrorStringWithFormat(
@@ -235,7 +234,7 @@ class EntityPersistentVariable : public Materializer::Entity {
             map.GetBestExecutionContextScope(),
             m_persistent_variable_sp.get()->GetCompilerType(),
             m_persistent_variable_sp->GetName(), location, eAddressTypeLoad,
-            m_persistent_variable_sp->GetByteSize().getValueOr(0));
+            m_persistent_variable_sp->GetByteSize());
 
         if (frame_top != LLDB_INVALID_ADDRESS &&
             frame_bottom != LLDB_INVALID_ADDRESS && location >= frame_bottom &&
@@ -280,8 +279,7 @@ class EntityPersistentVariable : public Materializer::Entity {
         LLDB_LOGF(log, "Dematerializing %s from 0x%" PRIx64 " (size = %llu)",
                   m_persistent_variable_sp->GetName().GetCString(),
                   (uint64_t)mem,
-                  (unsigned long long)m_persistent_variable_sp->GetByteSize()
-                      .getValueOr(0));
+                  (unsigned long long)m_persistent_variable_sp->GetByteSize());
 
         // Read the contents of the spare memory area
 
@@ -290,7 +288,7 @@ class EntityPersistentVariable : public Materializer::Entity {
         Status read_error;
 
         map.ReadMemory(m_persistent_variable_sp->GetValueBytes(), mem,
-                       m_persistent_variable_sp->GetByteSize().getValueOr(0), read_error);
+                       m_persistent_variable_sp->GetByteSize(), read_error);
 
         if (!read_error.Success()) {
           err.SetErrorStringWithFormat(
@@ -371,11 +369,10 @@ class EntityPersistentVariable : public Materializer::Entity {
       if (!err.Success()) {
         dump_stream.Printf("  <could not be read>\n");
       } else {
-        DataBufferHeap data(
-            m_persistent_variable_sp->GetByteSize().getValueOr(0), 0);
+        DataBufferHeap data(m_persistent_variable_sp->GetByteSize(), 0);
 
         map.ReadMemory(data.GetBytes(), target_address,
-                       m_persistent_variable_sp->GetByteSize().getValueOr(0), err);
+                       m_persistent_variable_sp->GetByteSize(), err);
 
         if (!err.Success()) {
           dump_stream.Printf("  <could not be read>\n");
@@ -624,8 +621,8 @@ class EntityVariable : public Materializer::Entity {
 
       Status extract_error;
 
-      map.GetMemoryData(data, m_temporary_allocation,
-                        valobj_sp->GetByteSize().getValueOr(0), extract_error);
+      map.GetMemoryData(data, m_temporary_allocation, valobj_sp->GetByteSize(),
+                        extract_error);
 
       if (!extract_error.Success()) {
         err.SetErrorStringWithFormat("couldn't get the data for variable %s",
@@ -922,7 +919,7 @@ class EntityResultVariable : public Materializer::Entity {
 
     ret->ValueUpdated();
 
-    const size_t pvar_byte_size = ret->GetByteSize().getValueOr(0);
+    const size_t pvar_byte_size = ret->GetByteSize();
     uint8_t *pvar_data = ret->GetValueBytes();
 
     map.ReadMemory(pvar_data, address, pvar_byte_size, read_error);
diff --git a/lldb/source/Target/StackFrame.cpp b/lldb/source/Target/StackFrame.cpp
index 22bca52d7f98a..098aed9cd8125 100644
--- a/lldb/source/Target/StackFrame.cpp
+++ b/lldb/source/Target/StackFrame.cpp
@@ -1408,7 +1408,7 @@ ValueObjectSP GetValueForOffset(StackFrame &frame, ValueObjectSP &parent,
     }
 
     int64_t child_offset = child_sp->GetByteOffset();
-    int64_t child_size = child_sp->GetByteSize().getValueOr(0);
+    int64_t child_size = child_sp->GetByteSize();
 
     if (offset >= child_offset && offset < (child_offset + child_size)) {
       return GetValueForOffset(frame, child_sp, offset - child_offset);
@@ -1441,8 +1441,8 @@ ValueObjectSP GetValueForDereferincingOffset(StackFrame &frame,
   }
 
   if (offset >= 0 && uint64_t(offset) >= pointee->GetByteSize()) {
-    int64_t index = offset / pointee->GetByteSize().getValueOr(1);
-    offset = offset % pointee->GetByteSize().getValueOr(1);
+    int64_t index = offset / pointee->GetByteSize();
+    offset = offset % pointee->GetByteSize();
     const bool can_create = true;
     pointee = base->GetSyntheticArrayMember(index, can_create);
   }

From 1df8804ce57576ff0c655073df082042cdf970f6 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@intel.com>
Date: Sat, 25 Jul 2020 20:46:42 -0700
Subject: [PATCH 0109/1035] [X86] Replace a use of ProcIntelSLM with
 FeatureFast7ByteNOP.

---
 llvm/lib/Target/X86/X86MCInstLower.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Target/X86/X86MCInstLower.cpp b/llvm/lib/Target/X86/X86MCInstLower.cpp
index 9ce2a4637e2ea..b4db72e150601 100644
--- a/llvm/lib/Target/X86/X86MCInstLower.cpp
+++ b/llvm/lib/Target/X86/X86MCInstLower.cpp
@@ -1083,7 +1083,7 @@ void X86AsmPrinter::LowerTlsAddr(X86MCInstLower &MCInstLowering,
 /// target cpu.  15-bytes is the longest single NOP instruction, but some
 /// platforms can't decode the longest forms efficiently.
 static unsigned maxLongNopLength(const X86Subtarget *Subtarget) {
-  if (Subtarget->getFeatureBits()[X86::ProcIntelSLM])
+  if (Subtarget->getFeatureBits()[X86::FeatureFast7ByteNOP])
     return 7;
   if (Subtarget->getFeatureBits()[X86::FeatureFast15ByteNOP])
     return 15;

From 9162b70e510475a044ee98332c9cdabf094b0e9b Mon Sep 17 00:00:00 2001
From: Changpeng Fang <changpeng.fang@gmail.com>
Date: Sat, 25 Jul 2020 21:20:59 -0700
Subject: [PATCH 0110/1035] DADCombiner: Don't simplify the token factor if the
 node's number of operands already exceeds TokenFactorInlineLimit

Summary:
  In parallelizeChainedStores, a TokenFactor was created with the size greater than 3000.
We found that DAGCombiner::visitTokenFactor will consume a huge amount of time on
such nodes. Since the number of operands already exceeds TokenFactorInlineLimit, we propose
to give up simplification with the consideration of compile time.

Reviewers:
  @spatel, @arsenm

Differential Revision:
  https://reviews.llvm.org/D84204
---
 llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp |  4 ++
 .../AMDGPU/token-factor-inline-limit-test.ll  | 58 +++++++++++++++++++
 2 files changed, 62 insertions(+)
 create mode 100644 llvm/test/CodeGen/AMDGPU/token-factor-inline-limit-test.ll

diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index a026d39600264..8932503b9564b 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -1805,6 +1805,10 @@ SDValue DAGCombiner::visitTokenFactor(SDNode *N) {
   if (OptLevel == CodeGenOpt::None)
     return SDValue();
 
+  // Don't simplify the token factor if the node itself has too many operands.
+  if (N->getNumOperands() > TokenFactorInlineLimit)
+    return SDValue();
+
   // If the sole user is a token factor, we should make sure we have a
   // chance to merge them together. This prevents TF chains from inhibiting
   // optimizations.
diff --git a/llvm/test/CodeGen/AMDGPU/token-factor-inline-limit-test.ll b/llvm/test/CodeGen/AMDGPU/token-factor-inline-limit-test.ll
new file mode 100644
index 0000000000000..88d9eeb8f2667
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/token-factor-inline-limit-test.ll
@@ -0,0 +1,58 @@
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN-TFILD %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -combiner-tokenfactor-inline-limit=7 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN-TFIL7 %s
+
+
+; GCN-LABEL: {{^}}token_factor_inline_limit_test:
+
+; GCN-TFILD: v_mov_b32_e32 [[REG8:v[0-9]+]], 8
+; GCN-TFILD: v_mov_b32_e32 [[REG9:v[0-9]+]], 9
+; GCN-TFILD: v_mov_b32_e32 [[REG10:v[0-9]+]], 10
+; GCN-TFILD: v_mov_b32_e32 [[REG11:v[0-9]+]], 11
+; GCN-TFILD: v_mov_b32_e32 [[REG12:v[0-9]+]], 12
+; GCN-TFILD: buffer_store_dword [[REG8]], {{.*$}}
+; GCN-TFILD: buffer_store_dword [[REG9]], {{.*}} offset:4
+; GCN-TFILD: buffer_store_dword [[REG10]], {{.*}} offset:8
+; GCN-TFILD: buffer_store_dword [[REG11]], {{.*}} offset:12
+; GCN-TFILD: buffer_store_dword [[REG12]], {{.*}} offset:16
+; GCN-TFILD: v_mov_b32_e32 [[REG13:v[0-9]+]], 13
+; GCN-TFILD: v_mov_b32_e32 [[REG14:v[0-9]+]], 14
+; GCN-TFILD: v_mov_b32_e32 [[REG15:v[0-9]+]], 15
+; GCN-TFILD: buffer_store_dword [[REG13]], {{.*}} offset:20
+; GCN-TFILD: buffer_store_dword [[REG14]], {{.*}} offset:24
+; GCN-TFILD: buffer_store_dword [[REG15]], {{.*}} offset:28
+
+; GCN-TFIL7: v_mov_b32_e32 [[REG15:v[0-9]+]], 15
+; GCN-TFIL7: v_mov_b32_e32 [[REG14:v[0-9]+]], 14
+; GCN-TFIL7: v_mov_b32_e32 [[REG13:v[0-9]+]], 13
+; GCN-TFIL7: buffer_store_dword [[REG15]], {{.*}} offset:28
+; GCN-TFIL7: buffer_store_dword [[REG14]], {{.*}} offset:24
+; GCN-TFIL7: buffer_store_dword [[REG13]], {{.*}} offset:20
+; GCN-TFIL7: v_mov_b32_e32 [[REG12:v[0-9]+]], 12
+; GCN-TFIL7: v_mov_b32_e32 [[REG11:v[0-9]+]], 11
+; GCN-TFIL7: v_mov_b32_e32 [[REG10:v[0-9]+]], 10
+; GCN-TFIL7: v_mov_b32_e32 [[REG9:v[0-9]+]], 9
+; GCN-TFIL7: v_mov_b32_e32 [[REG8:v[0-9]+]], 8
+; GCN-TFIL7: buffer_store_dword [[REG12]], {{.*}} offset:16
+; GCN-TFIL7: buffer_store_dword [[REG11]], {{.*}} offset:12
+; GCN-TFIL7: buffer_store_dword [[REG10]], {{.*}} offset:8
+; GCN-TFIL7: buffer_store_dword [[REG9]], {{.*}} offset:4
+; GCN-TFIL7: buffer_store_dword [[REG8]], {{.*$}}
+
+; GCN: v_mov_b32_e32 v31, 7
+; GCN: s_getpc
+define void @token_factor_inline_limit_test() {
+entry:
+  call void @external_void_func_8xv5i32(
+      <5 x i32><i32 0, i32 0, i32 0, i32 0, i32 0>,
+      <5 x i32><i32 1, i32 1, i32 1, i32 1, i32 1>,
+      <5 x i32><i32 2, i32 2, i32 2, i32 2, i32 2>,
+      <5 x i32><i32 3, i32 3, i32 3, i32 3, i32 3>,
+      <5 x i32><i32 4, i32 4, i32 4, i32 4, i32 4>,
+      <5 x i32><i32 5, i32 5, i32 5, i32 5, i32 5>,
+      <5 x i32><i32 6, i32 7, i32 8, i32 9, i32 10>,
+      <5 x i32><i32 11, i32 12, i32 13, i32 14, i32 15>)
+  ret void
+}
+
+declare hidden void @external_void_func_8xv5i32(<5 x i32>, <5 x i32>, <5 x i32>, <5 x i32>,
+                                                <5 x i32>, <5 x i32>, <5 x i32>, <5 x i32>)

From 595d214f47e484ffe517a4294d3ac042d6c7d25d Mon Sep 17 00:00:00 2001
From: Jacques Pienaar <jpienaar@google.com>
Date: Sat, 25 Jul 2020 21:37:15 -0700
Subject: [PATCH 0111/1035] [mlir][shape] Further operand and result type
 generalization

Previous changes generalized some of the operands and results. Complete
a larger group of those to simplify progressive lowering. Also update
some of the declarative asm form due to generalization. Tried to keep it
mostly mechanical.
---
 .../include/mlir/Dialect/Shape/IR/ShapeOps.td | 38 ++++++++++-------
 mlir/lib/Dialect/Shape/IR/Shape.cpp           | 42 +++++++++++++++++++
 .../ShapeToStandard/shape-to-standard.mlir    |  6 +--
 mlir/test/Dialect/Shape/canonicalize.mlir     | 30 ++++++-------
 mlir/test/Dialect/Shape/ops.mlir              | 10 ++---
 5 files changed, 87 insertions(+), 39 deletions(-)

diff --git a/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td b/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td
index 3c50a4f8b39f4..7b676a2b05981 100644
--- a/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td
+++ b/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td
@@ -86,11 +86,12 @@ def Shape_BroadcastOp : Shape_Op<"broadcast", [Commutative]> {
     broadcastable output shape possible for the given inputs.
   }];
 
-  let arguments = (ins Shape_ShapeType:$lhs, Shape_ShapeType:$rhs,
-                   OptionalAttr<StrAttr>:$error);
+  let arguments = (ins Shape_ShapeOrExtentTensorType:$lhs,
+                       Shape_ShapeOrExtentTensorType:$rhs,
+                       OptionalAttr<StrAttr>:$error);
   let results = (outs Shape_ShapeType:$result);
 
-  let assemblyFormat = "$lhs `,` $rhs attr-dict";
+  let assemblyFormat = "$lhs `,` $rhs attr-dict `:` type($lhs) `,` type($rhs)";
 
   let hasFolder = 1;
 }
@@ -220,10 +221,10 @@ def Shape_ToExtentTensorOp : Shape_Op<"to_extent_tensor", [NoSideEffect]> {
     If the shape represents an error, this op's behavior is undefined.
   }];
 
-  let arguments = (ins Shape_ShapeType:$input);
+  let arguments = (ins Shape_ShapeOrExtentTensorType:$input);
   let results = (outs IndexTensor:$result);
 
-  let assemblyFormat = "attr-dict $input `:` type($result)";
+  let assemblyFormat = "attr-dict $input `:` type($input) `->` type($result)";
 
   let hasFolder = 1;
 }
@@ -342,6 +343,10 @@ def Shape_NumElementsOp : Shape_Op<"num_elements", [NoSideEffect]> {
   let arguments = (ins Shape_ShapeOrExtentTensorType:$shape);
   let results = (outs Shape_SizeOrIndexType:$result);
 
+  let builders = [
+    OpBuilder<"OpBuilder &builder, OperationState &result, Value shape">,
+  ];
+
   let assemblyFormat = "$shape `:` type($shape) `->` type($result) attr-dict";
 
   let hasFolder = 1;
@@ -412,23 +417,28 @@ def Shape_ShapeOfOp : Shape_Op<"shape_of", [NoSideEffect]> {
 
   let assemblyFormat = "$arg `:` type($arg) `->` type($result) attr-dict";
 
+  let builders = [
+    OpBuilder<"OpBuilder &builder, OperationState &result, Value arg">
+  ];
+
   let verifier = [{ return ::verifyShapeOrExtentTensorOp(*this); }];
+  let hasCanonicalizer = 1;
   let hasFolder = 1;
 }
 
 def Shape_SizeToIndexOp : Shape_Op<"size_to_index", [NoSideEffect]> {
   let summary = "Casts between index types of the shape and standard dialect";
   let description = [{
-    Converts a `shape.size` to a standard index.
-    This operation and its inverse, `index_to_size`, facilitate index conversion
-    between the standard and the shape dialect.
-    The behavior is undefined for unknown and invalid arguments.
+    Converts a `shape.size` to a standard index. This operation and its
+    inverse, `index_to_size`, facilitate index conversion between the standard
+    and the shape dialect. The behavior is undefined for unknown and invalid
+    arguments.
   }];
 
-  let arguments = (ins Shape_SizeType:$arg);
+  let arguments = (outs Shape_SizeOrIndexType:$arg);
   let results = (outs Index:$result);
 
-  let assemblyFormat = "$arg attr-dict";
+  let assemblyFormat = "$arg attr-dict `:` type($arg)";
 
   let hasFolder = 1;
   let hasCanonicalizer = 1;
@@ -490,7 +500,7 @@ def Shape_SplitAtOp : Shape_Op<"split_at", []> {
     - `index` is in the range [-rank(operand),rank(operand)]
   }];
 
-  let arguments = (ins Shape_ShapeType:$operand, I32:$index);
+  let arguments = (ins Shape_ShapeOrExtentTensorType:$operand, I32:$index);
   let results = (outs Shape_ShapeType:$head, Shape_ShapeType:$tail);
   let hasFolder = 1;
 }
@@ -520,8 +530,7 @@ def Shape_ConcatOp : Shape_Op<"concat", []> {
 
 // TODO: Move the code below and witnesses to a different file.
 def Shape_AnyOp : Shape_Op<"any", [Commutative,
-                                   NoSideEffect,
-                                   SameOperandsAndResultType]> {
+                                   NoSideEffect]> {
   let summary = "Return any combination of the input shapes";
   let description = [{
     This operation takes multiple input shapes or extent tensors and returns
@@ -541,7 +550,6 @@ def Shape_AnyOp : Shape_Op<"any", [Commutative,
   let arguments = (ins Variadic<Shape_ShapeOrExtentTensorType>:$inputs);
   let results = (outs Shape_ShapeOrExtentTensorType:$result);
 
-  let assemblyFormat = "$inputs `:` type($result)  attr-dict";
   let hasFolder = 1;
 }
 
diff --git a/mlir/lib/Dialect/Shape/IR/Shape.cpp b/mlir/lib/Dialect/Shape/IR/Shape.cpp
index 104ab46c55813..4887c87c1e5f1 100644
--- a/mlir/lib/Dialect/Shape/IR/Shape.cpp
+++ b/mlir/lib/Dialect/Shape/IR/Shape.cpp
@@ -674,6 +674,16 @@ OpFoldResult NumElementsOp::fold(ArrayRef<Attribute> operands) {
   return builder.getIndexAttr(product.getLimitedValue());
 }
 
+void NumElementsOp::build(OpBuilder &builder, OperationState &result,
+                          Value shape) {
+  if (shape.getType().isa<ShapedType>()) {
+    auto type = builder.getIndexType();
+    return build(builder, result, type, shape);
+  }
+  auto type = SizeType::get(builder.getContext());
+  return build(builder, result, type, shape);
+}
+
 //===----------------------------------------------------------------------===//
 // MulOp
 //===----------------------------------------------------------------------===//
@@ -702,6 +712,38 @@ OpFoldResult ShapeOfOp::fold(ArrayRef<Attribute>) {
   return builder.getIndexTensorAttr(type.getShape());
 }
 
+void ShapeOfOp::build(OpBuilder &builder, OperationState &result, Value arg) {
+  if (arg.getType().isa<ShapedType>()) {
+    auto type = RankedTensorType::get({ShapedType::kDynamicSize},
+                                      builder.getIndexType());
+    return ShapeOfOp::build(builder, result, type, arg);
+  }
+  auto type = ShapeType::get(builder.getContext());
+  return ShapeOfOp::build(builder, result, type, arg);
+}
+
+namespace {
+struct ShapeOfWithTensor : public OpRewritePattern<shape::ShapeOfOp> {
+  using OpRewritePattern<shape::ShapeOfOp>::OpRewritePattern;
+
+  LogicalResult matchAndRewrite(shape::ShapeOfOp op,
+                                PatternRewriter &rewriter) const override {
+    if (!op.arg().getType().isa<ShapedType>())
+      return failure();
+    if (op.getType().isa<ShapedType>())
+      return failure();
+
+    rewriter.replaceOpWithNewOp<shape::ShapeOfOp>(op.getOperation(), op.arg());
+    return success();
+  }
+};
+} // namespace
+
+void ShapeOfOp::getCanonicalizationPatterns(OwningRewritePatternList &patterns,
+                                            MLIRContext *context) {
+  patterns.insert<ShapeOfWithTensor>(context);
+}
+
 //===----------------------------------------------------------------------===//
 // SizeToIndexOp
 //===----------------------------------------------------------------------===//
diff --git a/mlir/test/Conversion/ShapeToStandard/shape-to-standard.mlir b/mlir/test/Conversion/ShapeToStandard/shape-to-standard.mlir
index 8236c6f279755..9336402d86da4 100644
--- a/mlir/test/Conversion/ShapeToStandard/shape-to-standard.mlir
+++ b/mlir/test/Conversion/ShapeToStandard/shape-to-standard.mlir
@@ -50,7 +50,6 @@ func @shape_of_stat(%arg : tensor<1x2x3xf32>) {
   // CHECK-DAG: %[[C2:.*]] = constant 2 : index
   // CHECK-DAG: %[[C3:.*]] = constant 3 : index
   // CHECK-DAG: %[[SHAPE_UNCASTED:.*]] = tensor_from_elements(%[[C1]], %[[C2]], %[[C3]]) : tensor<3xindex>
-  // CHECK-DAG: %[[SHAPE:.*]] = tensor_cast %[[SHAPE_UNCASTED]] : tensor<3xindex> to tensor<?xindex>
   %shape = shape.shape_of %arg : tensor<1x2x3xf32> -> tensor<?xindex>
   return
 }
@@ -66,7 +65,6 @@ func @shape_of_dyn(%arg : tensor<1x5x?xf32>) {
   // CHECK-DAG: %[[C2:.*]] = constant 2 : index
   // CHECK-DAG: %[[DYN_DIM:.*]] = dim %[[ARG]], %[[C2]] : tensor<1x5x?xf32>
   // CHECK-DAG: %[[SHAPE_UNCASTED:.*]] = tensor_from_elements(%[[C1]], %[[C5]], %[[DYN_DIM]]) : tensor<3xindex>
-  // CHECK-DAG: %[[SHAPE:.*]] = tensor_cast %[[SHAPE_UNCASTED]] : tensor<3xindex> to tensor<?xindex>
   %shape = shape.shape_of %arg : tensor<1x5x?xf32> -> tensor<?xindex>
   return
 }
@@ -120,7 +118,7 @@ func @any_of_three(%a : tensor<?xindex>,
                    %b : tensor<?xindex>,
                    %c : tensor<?xindex>) -> tensor<?xindex> {
   // CHECK: return %[[A]] : tensor<?xindex>
-  %result = shape.any %a, %b, %c : tensor<?xindex>
+  %result = "shape.any"(%a, %b, %c) : (tensor<?xindex>, tensor<?xindex>, tensor<?xindex>) -> tensor<?xindex>
   return %result : tensor<?xindex>
 }
 
@@ -131,7 +129,7 @@ func @any_of_three(%a : tensor<?xindex>,
 // CHECK-SAME:  (%[[A:.*]]: tensor<?xindex>) -> tensor<?xindex>
 func @any_of_one(%a : tensor<?xindex>) -> tensor<?xindex> {
   // CHECK: return %[[A]] : tensor<?xindex>
-  %result = shape.any %a : tensor<?xindex>
+  %result = "shape.any"(%a) : (tensor<?xindex>) -> tensor<?xindex>
   return %result : tensor<?xindex>
 }
 
diff --git a/mlir/test/Dialect/Shape/canonicalize.mlir b/mlir/test/Dialect/Shape/canonicalize.mlir
index e147fbeb81ac2..5fe2ac108a69d 100644
--- a/mlir/test/Dialect/Shape/canonicalize.mlir
+++ b/mlir/test/Dialect/Shape/canonicalize.mlir
@@ -54,7 +54,7 @@ func @f() -> !shape.shape {
   // CHECK: shape.const_shape [7, 2] : !shape.shape
   %0 = shape.const_shape [1, 2] : !shape.shape
   %1 = shape.const_shape [7, 1] : !shape.shape
-  %2 = shape.broadcast %0, %1
+  %2 = shape.broadcast %0, %1 : !shape.shape, !shape.shape
   return %2 : !shape.shape
 }
 
@@ -65,7 +65,7 @@ func @f() -> !shape.shape {
 func @f(%arg0 : !shape.shape) -> !shape.shape {
   // CHECK: return %arg0
   %0 = shape.const_shape [] : !shape.shape
-  %1 = shape.broadcast %arg0, %0
+  %1 = shape.broadcast %arg0, %0 : !shape.shape, !shape.shape
   return %1 : !shape.shape
 }
 
@@ -76,7 +76,7 @@ func @f(%arg0 : !shape.shape) -> !shape.shape {
 func @f(%arg0 : !shape.shape) -> !shape.shape {
   // CHECK: return %arg0
   %0 = shape.const_shape [] : !shape.shape
-  %1 = shape.broadcast %0, %arg0
+  %1 = shape.broadcast %0, %arg0 : !shape.shape, !shape.shape
   return %1 : !shape.shape
 }
 
@@ -89,7 +89,7 @@ func @f() -> !shape.shape {
   // CHECK: return %[[CST]]
   %0 = shape.const_shape [] : !shape.shape
   %1 = shape.const_shape [1, 2, 3] : !shape.shape
-  %2 = shape.broadcast %0, %1
+  %2 = shape.broadcast %0, %1 : !shape.shape, !shape.shape
   return %2 : !shape.shape
 }
 
@@ -101,7 +101,7 @@ func @f() -> !shape.shape {
   // CHECK: shape.broadcast
   %0 = shape.const_shape [2] : !shape.shape
   %1 = shape.const_shape [7] : !shape.shape
-  %2 = shape.broadcast %0, %1
+  %2 = shape.broadcast %0, %1 : !shape.shape, !shape.shape
   return %2 : !shape.shape
 }
 
@@ -124,7 +124,7 @@ func @f() -> !shape.shape {
 func @f() -> tensor<2xindex> {
   // CHECK: constant dense<[0, 1]> : tensor<2xindex>
   %cs = shape.const_shape [0, 1] : !shape.shape
-  %0 = shape.to_extent_tensor %cs : tensor<2xindex>
+  %0 = shape.to_extent_tensor %cs : !shape.shape -> tensor<2xindex>
   return %0 : tensor<2xindex>
 }
 
@@ -159,7 +159,7 @@ func @const_size_to_index() -> index {
   // CHECK-NOT: shape.index_cast
   %cs = shape.const_size 123
   // CHECK: constant 123 : index
-  %ci = shape.size_to_index %cs
+  %ci = shape.size_to_index %cs : !shape.size
   return %ci : index
 }
 
@@ -185,7 +185,7 @@ func @const_index_to_size_to_index() -> index {
   %cs0 = shape.index_to_size %ci0
   // CHECK: %[[CI:.*]] = constant 123 : index
   // CHECK-NEXT: return %[[CI]] : index
-  %ci1 = shape.size_to_index %cs0
+  %ci1 = shape.size_to_index %cs0 : !shape.size
   return %ci1 : index
 }
 
@@ -195,7 +195,7 @@ func @const_index_to_size_to_index() -> index {
 // CHECK-LABEL: func @nonfoldable_size_to_index
 func @nonfoldable_size_to_index(%cs : !shape.size) -> index {
   // CHECK: shape.size_to_index
-  %ci = shape.size_to_index %cs
+  %ci = shape.size_to_index %cs : !shape.size
   return %ci : index
 }
 
@@ -403,7 +403,7 @@ func @f(%arg : !shape.shape) -> !shape.shape {
   // CHECK-NEXT: %[[CS:.*]] = shape.const_shape
   // CHECK-NEXT: return %[[CS]]
   %0 = shape.const_shape [2, 3, 4] : !shape.shape
-  %1 = shape.any %0, %arg : !shape.shape
+  %1 = "shape.any"(%0, %arg) : (!shape.shape, !shape.shape) -> !shape.shape
   return %1 : !shape.shape
 }
 
@@ -415,7 +415,7 @@ func @f(%arg : tensor<?xindex>) -> tensor<?xindex> {
   // CHECK-NEXT: %[[CS:.*]] = shape.const_shape [2, 3, 4] : tensor<?xindex>
   // CHECK-NEXT: return %[[CS]] : tensor<?xindex>
   %0 = shape.const_shape [2, 3, 4] : tensor<?xindex>
-  %1 = shape.any %0, %arg : tensor<?xindex>
+  %1 = "shape.any"(%0, %arg) : (tensor<?xindex>, tensor<?xindex>) -> tensor<?xindex>
   return %1 : tensor<?xindex>
 }
 
@@ -424,9 +424,9 @@ func @f(%arg : tensor<?xindex>) -> tensor<?xindex> {
 // Folding of any with partially constant operands is not yet implemented.
 // CHECK-LABEL: func @f
 func @f(%arg0 : !shape.shape, %arg1 : !shape.shape) -> !shape.shape {
-  // CHECK-NEXT: %[[CS:.*]] = shape.any
+  // CHECK-NEXT: %[[CS:.*]] = "shape.any"
   // CHECK-NEXT: return %[[CS]]
-  %1 = shape.any %arg0, %arg1 : !shape.shape
+  %1 = "shape.any"(%arg0, %arg1) : (!shape.shape, !shape.shape) -> !shape.shape
   return %1 : !shape.shape
 }
 
@@ -619,7 +619,7 @@ func @dont_canonicalize_rank(%arg : tensor<*xf32>) -> index {
 func @index_to_size_to_index(%index : index) -> index {
   // CHECK: return %[[IDX]] : index
   %size = shape.index_to_size %index
-  %result = shape.size_to_index %size
+  %result = shape.size_to_index %size : !shape.size
   return %result : index
 }
 
@@ -630,7 +630,7 @@ func @index_to_size_to_index(%index : index) -> index {
 // CHECK-SAME: (%[[SIZE:.*]]: !shape.size) -> !shape.size
 func @size_to_index_to_size(%size : !shape.size) -> !shape.size {
   // CHECK: return %[[SIZE]] : !shape.size
-  %idx = shape.size_to_index %size
+  %idx = shape.size_to_index %size : !shape.size
   %result = shape.index_to_size %idx
   return %result : !shape.size
 }
diff --git a/mlir/test/Dialect/Shape/ops.mlir b/mlir/test/Dialect/Shape/ops.mlir
index f57826097d34f..87af623fe0f7b 100644
--- a/mlir/test/Dialect/Shape/ops.mlir
+++ b/mlir/test/Dialect/Shape/ops.mlir
@@ -49,7 +49,7 @@ func @test_shape_num_elements_fixed() {
 func @test_broadcast_fixed() {
   %0 = shape.const_shape [10, 1, 57, 92] : !shape.shape
   %1 = shape.const_shape [4, 57, 92] : !shape.shape
-  %2 = shape.broadcast %0, %1
+  %2 = shape.broadcast %0, %1 : !shape.shape, !shape.shape
   %3 = "shape.print"(%2) : (!shape.shape) -> !shape.shape
   return
 }
@@ -99,7 +99,7 @@ func @test_constraints() {
   %w3 = shape.const_witness false
   %w4 = shape.assuming_all %w0, %w1, %w2, %w3
   shape.assuming %w4 -> !shape.shape {
-    %2 = shape.any %0, %1 : !shape.shape
+    %2 = "shape.any"(%0, %1) : (!shape.shape, !shape.shape) -> !shape.shape
     shape.assuming_yield %2 : !shape.shape
   }
   return
@@ -131,7 +131,7 @@ func @const_size() {
 }
 
 func @test_to_extent_tensor(%arg: !shape.shape) -> tensor<3xindex> {
-  %0 = shape.to_extent_tensor %arg : tensor<3xindex>
+  %0 = shape.to_extent_tensor %arg : !shape.shape -> tensor<3xindex>
   return %0 : tensor<3xindex>
 }
 
@@ -188,10 +188,10 @@ func @get_extent_on_mixed_operands(%arg : tensor<?xindex>) -> !shape.size {
 func @any() {
   %0 = shape.const_shape [1, 2, 3] : !shape.shape
   %1 = shape.const_shape [4, 5, 6] : !shape.shape
-  %2 = shape.any %0, %1 : !shape.shape
+  %2 = "shape.any"(%0, %1) : (!shape.shape, !shape.shape) -> !shape.shape
   %3 = shape.const_shape [1, 2, 3] : tensor<?xindex>
   %4 = shape.const_shape [4, 5, 6] : tensor<?xindex>
-  %5 = shape.any %3, %4 : tensor<?xindex>
+  %5 = "shape.any"(%3, %4) : (tensor<?xindex>, tensor<?xindex>) -> tensor<?xindex>
   return
 }
 

From 14c59b4577658655e56671bf2f17cca9c0d952d9 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@intel.com>
Date: Sat, 25 Jul 2020 20:48:46 -0700
Subject: [PATCH 0112/1035] [X86] Remove getProcFamily() method from
 X86Subtarget. NFC

This isn't used and we've decided in the past that a CPU enum
for tuning is not a good idea.
---
 llvm/lib/Target/X86/X86Subtarget.h | 2 --
 1 file changed, 2 deletions(-)

diff --git a/llvm/lib/Target/X86/X86Subtarget.h b/llvm/lib/Target/X86/X86Subtarget.h
index 54d7fbef7f94a..5b5ab4b969aac 100644
--- a/llvm/lib/Target/X86/X86Subtarget.h
+++ b/llvm/lib/Target/X86/X86Subtarget.h
@@ -787,8 +787,6 @@ class X86Subtarget final : public X86GenSubtargetInfo {
 
   bool isXRaySupported() const override { return is64Bit(); }
 
-  X86ProcFamilyEnum getProcFamily() const { return X86ProcFamily; }
-
   /// TODO: to be removed later and replaced with suitable properties
   bool isAtom() const { return X86ProcFamily == IntelAtom; }
   bool isSLM() const { return X86ProcFamily == IntelSLM; }

From 1a1448e6568d9b11f198e510fa9c4cb6b1f4216a Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@intel.com>
Date: Sat, 25 Jul 2020 22:05:46 -0700
Subject: [PATCH 0113/1035] [X86] Merge X86MCInstLowering's maxLongNopLength
 into emitNop and remove check for FeatureNOPL.

The switch in emitNop uses 64-bit registers for nops exceeding
2 bytes. This isn't valid outside 64-bit mode. We could fix this
easily enough, but there are no users that ask for more than 2
bytes outside 64-bit mode.

Inlining the method to make the coupling between the two methods
more explicit.
---
 llvm/lib/Target/X86/X86MCInstLower.cpp | 34 ++++++++++++--------------
 1 file changed, 16 insertions(+), 18 deletions(-)

diff --git a/llvm/lib/Target/X86/X86MCInstLower.cpp b/llvm/lib/Target/X86/X86MCInstLower.cpp
index b4db72e150601..8f3e32727371b 100644
--- a/llvm/lib/Target/X86/X86MCInstLower.cpp
+++ b/llvm/lib/Target/X86/X86MCInstLower.cpp
@@ -1079,29 +1079,27 @@ void X86AsmPrinter::LowerTlsAddr(X86MCInstLower &MCInstLowering,
   }
 }
 
-/// Return the longest nop which can be efficiently decoded for the given
-/// target cpu.  15-bytes is the longest single NOP instruction, but some
-/// platforms can't decode the longest forms efficiently.
-static unsigned maxLongNopLength(const X86Subtarget *Subtarget) {
-  if (Subtarget->getFeatureBits()[X86::FeatureFast7ByteNOP])
-    return 7;
-  if (Subtarget->getFeatureBits()[X86::FeatureFast15ByteNOP])
-    return 15;
-  if (Subtarget->getFeatureBits()[X86::FeatureFast11ByteNOP])
-    return 11;
-  if (Subtarget->getFeatureBits()[X86::FeatureNOPL] || Subtarget->is64Bit())
-    return 10;
-  if (Subtarget->is32Bit())
-    return 2;
-  return 1;
-}
-
 /// Emit the largest nop instruction smaller than or equal to \p NumBytes
 /// bytes.  Return the size of nop emitted.
 static unsigned emitNop(MCStreamer &OS, unsigned NumBytes,
                         const X86Subtarget *Subtarget) {
+  unsigned MaxNopLength = 1;
+  if (Subtarget->is64Bit()) {
+    // FIXME: We can use NOOPL on 32-bit targets with FeatureNOPL, but the
+    // IndexReg/BaseReg below need to be updated.
+    if (Subtarget->hasFeature(X86::FeatureFast7ByteNOP))
+      MaxNopLength = 7;
+    else if (Subtarget->hasFeature(X86::FeatureFast15ByteNOP))
+      MaxNopLength = 15;
+    else if (Subtarget->hasFeature(X86::FeatureFast11ByteNOP))
+      MaxNopLength = 11;
+    else
+      MaxNopLength = 10;
+  } if (Subtarget->is32Bit())
+    MaxNopLength = 2;
+
   // Cap a single nop emission at the profitable value for the target
-  NumBytes = std::min(NumBytes, maxLongNopLength(Subtarget));
+  NumBytes = std::min(NumBytes, MaxNopLength);
 
   unsigned NopSize;
   unsigned Opc, BaseReg, ScaleVal, IndexReg, Displacement, SegmentReg;

From 9b19400004dfee9d07a90aa11d448bade9ee71a2 Mon Sep 17 00:00:00 2001
From: Amara Emerson <aemerson@apple.com>
Date: Sun, 26 Jul 2020 00:46:29 -0700
Subject: [PATCH 0114/1035] [AArch64][GlobalISel] Make <8 x s16> and <16 x s8>
 legal types for G_SHUFFLE_VECTOR and G_IMPLICIT_DEF.

Trivial change, we're still missing support for rev matching for these types
in the combiner.
---
 .../AArch64/GISel/AArch64LegalizerInfo.cpp    |  4 +-
 .../GlobalISel/legalize-shuffle-vector.mir    | 44 +++++++++++++++++++
 .../AArch64/GlobalISel/legalize-undef.mir     | 22 ++++++++++
 llvm/test/CodeGen/AArch64/arm64-rev.ll        | 36 +++++++--------
 4 files changed, 84 insertions(+), 22 deletions(-)

diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index bbceb0e169039..54e368cd7768f 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -62,7 +62,7 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
   }
 
   getActionDefinitionsBuilder({G_IMPLICIT_DEF, G_FREEZE})
-    .legalFor({p0, s1, s8, s16, s32, s64, v2s32, v4s32, v2s64})
+    .legalFor({p0, s1, s8, s16, s32, s64, v2s32, v4s32, v2s64, v16s8, v8s16})
     .clampScalar(0, s1, s64)
     .widenScalarToNextPow2(0, 8)
     .fewerElementsIf(
@@ -596,7 +596,7 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
         // to be the same size as the dest.
         if (DstTy != SrcTy)
           return false;
-        for (auto &Ty : {v2s32, v4s32, v2s64}) {
+        for (auto &Ty : {v2s32, v4s32, v2s64, v16s8, v8s16}) {
           if (DstTy == Ty)
             return true;
         }
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-shuffle-vector.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-shuffle-vector.mir
index 7b4ae3d56ab0f..22d9406ac4f6f 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-shuffle-vector.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-shuffle-vector.mir
@@ -43,6 +43,50 @@ body:             |
     $q0 = COPY %2(<2 x s64>)
     RET_ReallyLR implicit $q0
 
+...
+---
+name:            shuffle_v16i8
+alignment:       4
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $q0, $q1
+
+    ; CHECK-LABEL: name: shuffle_v16i8
+    ; CHECK: liveins: $q0, $q1
+    ; CHECK: [[COPY:%[0-9]+]]:_(<16 x s8>) = COPY $q0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<16 x s8>) = COPY $q1
+    ; CHECK: [[SHUF:%[0-9]+]]:_(<16 x s8>) = G_SHUFFLE_VECTOR [[COPY]](<16 x s8>), [[COPY1]], shufflemask(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)
+    ; CHECK: $q0 = COPY [[SHUF]](<16 x s8>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %0:_(<16 x s8>) = COPY $q0
+    %1:_(<16 x s8>) = COPY $q1
+    %2:_(<16 x s8>) = G_SHUFFLE_VECTOR %0(<16 x s8>), %1, shufflemask(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)
+    $q0 = COPY %2(<16 x s8>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            shuffle_v8i16
+alignment:       4
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $q0, $q1
+
+    ; CHECK-LABEL: name: shuffle_v8i16
+    ; CHECK: liveins: $q0, $q1
+    ; CHECK: [[COPY:%[0-9]+]]:_(<8 x s16>) = COPY $q0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<8 x s16>) = COPY $q1
+    ; CHECK: [[SHUF:%[0-9]+]]:_(<8 x s16>) = G_SHUFFLE_VECTOR [[COPY]](<8 x s16>), [[COPY1]], shufflemask(0, 0, 0, 0, 0, 0, 0, 0)
+    ; CHECK: $q0 = COPY [[SHUF]](<8 x s16>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %0:_(<8 x s16>) = COPY $q0
+    %1:_(<8 x s16>) = COPY $q1
+    %2:_(<8 x s16>) = G_SHUFFLE_VECTOR %0(<8 x s16>), %1, shufflemask(0, 0, 0, 0, 0, 0, 0, 0)
+    $q0 = COPY %2(<8 x s16>)
+    RET_ReallyLR implicit $q0
+
 ...
 ---
 name:            shuffle_1elt_mask
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-undef.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-undef.mir
index 2e7c1cbb33914..984909d342072 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-undef.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-undef.mir
@@ -77,3 +77,25 @@ body: |
     $w0 = COPY %1
     $w1 = COPY %2
 ...
+---
+name: test_implicit_def_v16s8
+body: |
+  bb.0:
+
+    ; CHECK-LABEL: name: test_implicit_def_v16s8
+    ; CHECK: [[DEF:%[0-9]+]]:_(<16 x s8>) = G_IMPLICIT_DEF
+    ; CHECK: $q0 = COPY [[DEF]](<16 x s8>)
+    %0:_(<16 x s8>) = G_IMPLICIT_DEF
+    $q0 = COPY %0
+...
+---
+name: test_implicit_def_v8s16
+body: |
+  bb.0:
+
+    ; CHECK-LABEL: name: test_implicit_def_v8s16
+    ; CHECK: [[DEF:%[0-9]+]]:_(<8 x s16>) = G_IMPLICIT_DEF
+    ; CHECK: $q0 = COPY [[DEF]](<8 x s16>)
+    %0:_(<8 x s16>) = G_IMPLICIT_DEF
+    $q0 = COPY %0
+...
diff --git a/llvm/test/CodeGen/AArch64/arm64-rev.ll b/llvm/test/CodeGen/AArch64/arm64-rev.ll
index 0a6c7a14a3352..5f76f0a1c2714 100644
--- a/llvm/test/CodeGen/AArch64/arm64-rev.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-rev.ll
@@ -426,11 +426,10 @@ define <16 x i8> @test_vrev32Q8(<16 x i8>* %A) nounwind {
 ; CHECK-NEXT:    rev32.16b v0, v0
 ; CHECK-NEXT:    ret
 ;
-; FALLBACK-LABEL: test_vrev32Q8:
-; FALLBACK:       // %bb.0:
-; FALLBACK-NEXT:    ldr q0, [x0]
-; FALLBACK-NEXT:    rev32.16b v0, v0
-; FALLBACK-NEXT:    ret
+; GISEL-LABEL: test_vrev32Q8:
+; GISEL:       // %bb.0:
+; GISEL:         tbl.16b v0, { v0, v1 }, v2
+; GISEL:         ret
 	%tmp1 = load <16 x i8>, <16 x i8>* %A
 	%tmp2 = shufflevector <16 x i8> %tmp1, <16 x i8> undef, <16 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12>
 	ret <16 x i8> %tmp2
@@ -443,11 +442,10 @@ define <8 x i16> @test_vrev32Q16(<8 x i16>* %A) nounwind {
 ; CHECK-NEXT:    rev32.8h v0, v0
 ; CHECK-NEXT:    ret
 ;
-; FALLBACK-LABEL: test_vrev32Q16:
-; FALLBACK:       // %bb.0:
-; FALLBACK-NEXT:    ldr q0, [x0]
-; FALLBACK-NEXT:    rev32.8h v0, v0
-; FALLBACK-NEXT:    ret
+; GISEL-LABEL: test_vrev32Q16:
+; GISEL:       // %bb.0:
+; GISEL:         tbl.16b v0, { v0, v1 }, v2
+; GISEL:         ret
 	%tmp1 = load <8 x i16>, <8 x i16>* %A
 	%tmp2 = shufflevector <8 x i16> %tmp1, <8 x i16> undef, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
 	ret <8 x i16> %tmp2
@@ -477,11 +475,10 @@ define <16 x i8> @test_vrev16Q8(<16 x i8>* %A) nounwind {
 ; CHECK-NEXT:    rev16.16b v0, v0
 ; CHECK-NEXT:    ret
 ;
-; FALLBACK-LABEL: test_vrev16Q8:
-; FALLBACK:       // %bb.0:
-; FALLBACK-NEXT:    ldr q0, [x0]
-; FALLBACK-NEXT:    rev16.16b v0, v0
-; FALLBACK-NEXT:    ret
+; GISEL-LABEL: test_vrev16Q8:
+; GISEL:       // %bb.0:
+; GISEL:         tbl.16b v0, { v0, v1 }, v2
+; GISEL:         ret
 	%tmp1 = load <16 x i8>, <16 x i8>* %A
 	%tmp2 = shufflevector <16 x i8> %tmp1, <16 x i8> undef, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14>
 	ret <16 x i8> %tmp2
@@ -513,11 +510,10 @@ define <8 x i16> @test_vrev32Q16_undef(<8 x i16>* %A) nounwind {
 ; CHECK-NEXT:    rev32.8h v0, v0
 ; CHECK-NEXT:    ret
 ;
-; FALLBACK-LABEL: test_vrev32Q16_undef:
-; FALLBACK:       // %bb.0:
-; FALLBACK-NEXT:    ldr q0, [x0]
-; FALLBACK-NEXT:    rev32.8h v0, v0
-; FALLBACK-NEXT:    ret
+; GISEL-LABEL: test_vrev32Q16_undef:
+; GISEL:       // %bb.0:
+; GISEL:         tbl.16b v0, { v0, v1 }, v2
+; GISEL:         ret
 	%tmp1 = load <8 x i16>, <8 x i16>* %A
 	%tmp2 = shufflevector <8 x i16> %tmp1, <8 x i16> undef, <8 x i32> <i32 undef, i32 0, i32 undef, i32 2, i32 5, i32 4, i32 7, i32 undef>
 	ret <8 x i16> %tmp2

From 98b61112c32e1d2f4a0037698e9ac1d46c23d6c7 Mon Sep 17 00:00:00 2001
From: Alex Richardson <Alexander.Richardson@cl.cam.ac.uk>
Date: Sun, 26 Jul 2020 11:37:47 +0100
Subject: [PATCH 0115/1035] [asan] Mark the strstr test as UNSUPPORTED on
 FreeBSD

 Like Android, FreeBSDs libc calls memchr which causes this test to fail.

Reviewed By: emaste

Differential Revision: https://reviews.llvm.org/D84541
---
 compiler-rt/test/asan/TestCases/strstr_strict.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/compiler-rt/test/asan/TestCases/strstr_strict.c b/compiler-rt/test/asan/TestCases/strstr_strict.c
index 7d2c4ec2f3258..0d9482723f5f6 100644
--- a/compiler-rt/test/asan/TestCases/strstr_strict.c
+++ b/compiler-rt/test/asan/TestCases/strstr_strict.c
@@ -3,8 +3,8 @@
 
 // Newer versions of Android's strstr() uses memchr() internally, which actually
 // does trigger a heap-buffer-overflow (as it tries to find the
-// null-terminator).
-// UNSUPPORTED: android
+// null-terminator). The same applies to FreeBSD.
+// UNSUPPORTED: android, freebsd
 // RUN: %env_asan_opts=strict_string_checks=false %run %t 2>&1
 
 // RUN: %env_asan_opts=strict_string_checks=true not %run %t 2>&1 | FileCheck %s

From 18df607dbeb41451f5842f46e9b7a67baf759d5e Mon Sep 17 00:00:00 2001
From: Alex Richardson <Alexander.Richardson@cl.cam.ac.uk>
Date: Sun, 26 Jul 2020 11:39:22 +0100
Subject: [PATCH 0116/1035] [lit] Don't include tests skipped due to sharding
 in reports

When running multiple shards, don't include skipped tests in the xunit
output since merging the files will result in duplicates.
In our CHERI Jenkins CI, I configured the libc++ tests to run using sharding
(since we are testing using a single-CPU QEMU). We then merge the generated
XUnit xml files to produce a final result, but if the individual XMLs
report tests excluded due to sharding each test is included N times in the
final result. This also makes it difficult to find the tests that were
skipped due to missing REQUIRES: etc.

Reviewed By: yln

Differential Revision: https://reviews.llvm.org/D84235
---
 llvm/utils/lit/lit/main.py           | 6 +++++-
 llvm/utils/lit/lit/reports.py        | 2 +-
 llvm/utils/lit/tests/xunit-output.py | 2 +-
 3 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py
index c47bdede3176c..d94d7280809da 100755
--- a/llvm/utils/lit/lit/main.py
+++ b/llvm/utils/lit/lit/main.py
@@ -80,9 +80,13 @@ def main(builtin_params={}):
                              'error.\n')
             sys.exit(2)
 
+    # When running multiple shards, don't include skipped tests in the xunit
+    # output since merging the files will result in duplicates.
+    tests_for_report = discovered_tests
     if opts.shard:
         (run, shards) = opts.shard
         selected_tests = filter_by_shard(selected_tests, run, shards, lit_config)
+        tests_for_report = selected_tests
         if not selected_tests:
             sys.stderr.write('warning: shard does not contain any tests.  '
                              'Consider decreasing the number of shards.\n')
@@ -102,7 +106,7 @@ def main(builtin_params={}):
     print_results(discovered_tests, elapsed, opts)
 
     for report in opts.reports:
-        report.write_results(discovered_tests, elapsed)
+        report.write_results(tests_for_report, elapsed)
 
     if lit_config.numErrors:
         sys.stderr.write('\n%d error(s) in tests\n' % lit_config.numErrors)
diff --git a/llvm/utils/lit/lit/reports.py b/llvm/utils/lit/lit/reports.py
index 9631d5fe3b358..3ce961b44029e 100755
--- a/llvm/utils/lit/lit/reports.py
+++ b/llvm/utils/lit/lit/reports.py
@@ -127,7 +127,7 @@ def _write_test(self, file, test, suite_name):
     def _get_skip_reason(self, test):
         code = test.result.code
         if code == lit.Test.EXCLUDED:
-            return 'Test not selected (--filter, --max-tests, --run-shard)'
+            return 'Test not selected (--filter, --max-tests)'
         if code == lit.Test.SKIPPED:
             return 'User interrupt'
 
diff --git a/llvm/utils/lit/tests/xunit-output.py b/llvm/utils/lit/tests/xunit-output.py
index 81d8525f33b46..92b693256c69b 100644
--- a/llvm/utils/lit/tests/xunit-output.py
+++ b/llvm/utils/lit/tests/xunit-output.py
@@ -14,7 +14,7 @@
 # CHECK-NEXT:   <failure><![CDATA[& < > ]]]]><![CDATA[> &"]]></failure>
 # CHECK-NEXT: </testcase>
 # CHECK-NEXT: <testcase classname="test-data.test-data" name="excluded.ini" time="{{[0-1]\.[0-9]+}}">
-# CHECK-NEXT:   <skipped message="Test not selected (--filter, --max-tests, --run-shard)"/>
+# CHECK-NEXT:   <skipped message="Test not selected (--filter, --max-tests)"/>
 # CHECK-NEXT: </testcase>
 # CHECK-NEXT: <testcase classname="test-data.test-data" name="missing_feature.ini" time="{{[0-1]\.[0-9]+}}">
 # CHECK-NEXT:   <skipped message="Missing required feature(s): dummy_feature"/>

From ed5a6b9305145fd073d046d6e9814ffb70c5fdbe Mon Sep 17 00:00:00 2001
From: Roman Lebedev <lebedev.ri@gmail.com>
Date: Sun, 26 Jul 2020 14:00:15 +0300
Subject: [PATCH 0117/1035] [NFC][XRay] Account: decouple getStats() interface
 from underlying data structure

It doesn't really need to know where Timings are stored, it just needs
to be able to sort them, so MutableArrayRef is enough.

That uncovers an interesting quirk that it relied on
implicit double->int conversion for calculating percentiles.
---
 llvm/tools/llvm-xray/xray-account.cpp | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/llvm/tools/llvm-xray/xray-account.cpp b/llvm/tools/llvm-xray/xray-account.cpp
index fcac33b23d4d6..21da53f362b09 100644
--- a/llvm/tools/llvm-xray/xray-account.cpp
+++ b/llvm/tools/llvm-xray/xray-account.cpp
@@ -226,7 +226,7 @@ struct ResultRow {
   std::string Function;
 };
 
-ResultRow getStats(std::vector<uint64_t> &Timings) {
+ResultRow getStats(MutableArrayRef<uint64_t> Timings) {
   assert(!Timings.empty());
   ResultRow R;
   R.Sum = std::accumulate(Timings.begin(), Timings.end(), 0.0);
@@ -240,11 +240,13 @@ ResultRow getStats(std::vector<uint64_t> &Timings) {
   R.Median = Timings[MedianOff];
 
   auto Pct90Off = std::floor(Timings.size() * 0.9);
-  std::nth_element(Timings.begin(), Timings.begin() + Pct90Off, Timings.end());
+  std::nth_element(Timings.begin(), Timings.begin() + (uint64_t)Pct90Off,
+                   Timings.end());
   R.Pct90 = Timings[Pct90Off];
 
   auto Pct99Off = std::floor(Timings.size() * 0.99);
-  std::nth_element(Timings.begin(), Timings.begin() + Pct99Off, Timings.end());
+  std::nth_element(Timings.begin(), Timings.begin() + (uint64_t)Pct99Off,
+                   Timings.end());
   R.Pct99 = Timings[Pct99Off];
   return R;
 }

From b1210c059d1ef084ecd275ed0ffb8343ac3cdfad Mon Sep 17 00:00:00 2001
From: Roman Lebedev <lebedev.ri@gmail.com>
Date: Sun, 26 Jul 2020 14:05:00 +0300
Subject: [PATCH 0118/1035] [NFC][XRay] Account: migrate to DenseMap +
 SmallVector, -16% faster on large (3.8G) input

DenseMap is a single allocation underneath, so this is has pretty expected
performance impact on large-ish (3.8G) xray log processing time.
---
 llvm/tools/llvm-xray/xray-account.h | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/llvm/tools/llvm-xray/xray-account.h b/llvm/tools/llvm-xray/xray-account.h
index b63ecc59b71ad..575114d6096a2 100644
--- a/llvm/tools/llvm-xray/xray-account.h
+++ b/llvm/tools/llvm-xray/xray-account.h
@@ -27,12 +27,14 @@ namespace xray {
 
 class LatencyAccountant {
 public:
-  typedef std::map<int32_t, std::vector<uint64_t>> FunctionLatencyMap;
-  typedef std::map<uint32_t, std::pair<uint64_t, uint64_t>>
+  typedef llvm::DenseMap<int32_t, llvm::SmallVector<uint64_t, 0>>
+      FunctionLatencyMap;
+  typedef llvm::DenseMap<uint32_t, std::pair<uint64_t, uint64_t>>
       PerThreadMinMaxTSCMap;
-  typedef std::map<uint8_t, std::pair<uint64_t, uint64_t>> PerCPUMinMaxTSCMap;
-  typedef std::vector<std::pair<int32_t, uint64_t>> FunctionStack;
-  typedef std::map<uint32_t, FunctionStack> PerThreadFunctionStackMap;
+  typedef llvm::DenseMap<uint8_t, std::pair<uint64_t, uint64_t>>
+      PerCPUMinMaxTSCMap;
+  typedef llvm::SmallVector<std::pair<int32_t, uint64_t>, 32> FunctionStack;
+  typedef llvm::DenseMap<uint32_t, FunctionStack> PerThreadFunctionStackMap;
 
 private:
   PerThreadFunctionStackMap PerThreadFunctionStack;

From 02dadab1b459a3ed4f440b2de3ab21eba7bd5a2e Mon Sep 17 00:00:00 2001
From: Juneyoung Lee <aqjune@gmail.com>
Date: Sun, 26 Jul 2020 20:47:19 +0900
Subject: [PATCH 0119/1035] NFC; add an example that subtracts pointers to two
 global vars

---
 llvm/test/Transforms/InstSimplify/freeze.ll | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/llvm/test/Transforms/InstSimplify/freeze.ll b/llvm/test/Transforms/InstSimplify/freeze.ll
index 3c4ca7a1afb95..3aa77854f6020 100644
--- a/llvm/test/Transforms/InstSimplify/freeze.ll
+++ b/llvm/test/Transforms/InstSimplify/freeze.ll
@@ -111,8 +111,7 @@ define <2 x float> @constvector_FP_noopt() {
 }
 
 @g = external global i16, align 1
-
-; Negative test
+@g2 = external global i16, align 1
 
 define float @constant_expr() {
 ; CHECK-LABEL: @constant_expr(
@@ -138,6 +137,18 @@ define i32* @constant_expr3() {
   ret i32* %r
 }
 
+define i64 @ptrdiff() {
+; CHECK-LABEL: @ptrdiff(
+; CHECK-NEXT:    [[R:%.*]] = freeze i64 sub (i64 ptrtoint (i16* @g to i64), i64 ptrtoint (i16* @g2 to i64))
+; CHECK-NEXT:    ret i64 [[R]]
+;
+  %i = ptrtoint i16* @g to i64
+  %i2 = ptrtoint i16* @g2 to i64
+  %diff = sub i64 %i, %i2
+  %r = freeze i64 %diff
+  ret i64 %r
+}
+
 ; Negative test
 
 define <2 x i31> @vector_element_constant_expr() {

From 1b802fe34d716f2a3542fad3e8ba74f77685590d Mon Sep 17 00:00:00 2001
From: Juneyoung Lee <aqjune@gmail.com>
Date: Sun, 26 Jul 2020 21:02:31 +0900
Subject: [PATCH 0120/1035] NFC; add a test for freeze's constprop

---
 llvm/test/Analysis/ConstantFolding/freeze.ll | 30 ++++++++++++++++++++
 1 file changed, 30 insertions(+)
 create mode 100644 llvm/test/Analysis/ConstantFolding/freeze.ll

diff --git a/llvm/test/Analysis/ConstantFolding/freeze.ll b/llvm/test/Analysis/ConstantFolding/freeze.ll
new file mode 100644
index 0000000000000..c2da041308456
--- /dev/null
+++ b/llvm/test/Analysis/ConstantFolding/freeze.ll
@@ -0,0 +1,30 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -constprop -S | FileCheck %s
+
+@g = external global i16, align 1
+@g2 = external global i16, align 1
+
+define i64 @ptrdiff1() {
+; CHECK-LABEL: @ptrdiff1(
+; CHECK-NEXT:    [[R:%.*]] = freeze i64 sub (i64 ptrtoint (i16* @g to i64), i64 ptrtoint (i16* @g2 to i64))
+; CHECK-NEXT:    ret i64 [[R]]
+;
+  %i = ptrtoint i16* @g to i64
+  %i2 = ptrtoint i16* @g2 to i64
+  %diff = sub i64 %i, %i2
+  %r = freeze i64 %diff
+  ret i64 %r
+}
+
+define i64 @ptrdiff2() {
+; CHECK-LABEL: @ptrdiff2(
+; CHECK-NEXT:    [[R:%.*]] = freeze i64 -2
+; CHECK-NEXT:    ret i64 [[R]]
+;
+  %i = ptrtoint i16* @g to i64
+  %gep = getelementptr i16, i16* @g, i64 1
+  %i2 = ptrtoint i16* %gep to i64
+  %diff = sub i64 %i, %i2
+  %r = freeze i64 %diff
+  ret i64 %r
+}

From 9f074214b7a3bc9d88caadbf7bd6116305977a4e Mon Sep 17 00:00:00 2001
From: Juneyoung Lee <aqjune@gmail.com>
Date: Sun, 26 Jul 2020 21:48:51 +0900
Subject: [PATCH 0121/1035] [ValueTracking] Instruction::isBinaryOp should be
 used for constexprs

This is a simple patch that makes canCreateUndefOrPoison use
Instruction::isBinaryOp because BinaryOperator inherits Instruction.

Reviewed By: nikic

Differential Revision: https://reviews.llvm.org/D84596
---
 llvm/lib/Analysis/ValueTracking.cpp         | 2 +-
 llvm/test/Transforms/InstSimplify/freeze.ll | 3 +--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index 271200f7030a2..3bf09b30b2f9c 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -4773,7 +4773,7 @@ static bool canCreateUndefOrPoison(const Operator *Op, bool PoisonOnly) {
     const auto *CE = dyn_cast<ConstantExpr>(Op);
     if (isa<CastInst>(Op) || (CE && CE->isCast()))
       return false;
-    else if (isa<BinaryOperator>(Op))
+    else if (Instruction::isBinaryOp(Opcode))
       return false;
     // Be conservative and return true.
     return true;
diff --git a/llvm/test/Transforms/InstSimplify/freeze.ll b/llvm/test/Transforms/InstSimplify/freeze.ll
index 3aa77854f6020..66ec5d61a3c1c 100644
--- a/llvm/test/Transforms/InstSimplify/freeze.ll
+++ b/llvm/test/Transforms/InstSimplify/freeze.ll
@@ -139,8 +139,7 @@ define i32* @constant_expr3() {
 
 define i64 @ptrdiff() {
 ; CHECK-LABEL: @ptrdiff(
-; CHECK-NEXT:    [[R:%.*]] = freeze i64 sub (i64 ptrtoint (i16* @g to i64), i64 ptrtoint (i16* @g2 to i64))
-; CHECK-NEXT:    ret i64 [[R]]
+; CHECK-NEXT:    ret i64 sub (i64 ptrtoint (i16* @g to i64), i64 ptrtoint (i16* @g2 to i64))
 ;
   %i = ptrtoint i16* @g to i64
   %i2 = ptrtoint i16* @g2 to i64

From 32088f4f7fc734474d1249d288a34894a4e901a9 Mon Sep 17 00:00:00 2001
From: Juneyoung Lee <aqjune@gmail.com>
Date: Sun, 26 Jul 2020 21:54:44 +0900
Subject: [PATCH 0122/1035] [ConstantFolding] Fold freeze if it is never undef
 or poison

This is a simple patch that adds constant folding for freeze
instruction.

IIUC, it isn't needed to update ConstantFold.cpp because there is no freeze
constexpr.

Reviewed By: nikic

Differential Revision: https://reviews.llvm.org/D84597
---
 llvm/lib/Analysis/ConstantFolding.cpp        | 2 ++
 llvm/test/Analysis/ConstantFolding/freeze.ll | 6 ++----
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp
index 794edeb714fd2..7e4293d72a0e5 100644
--- a/llvm/lib/Analysis/ConstantFolding.cpp
+++ b/llvm/lib/Analysis/ConstantFolding.cpp
@@ -1071,6 +1071,8 @@ Constant *ConstantFoldInstOperandsImpl(const Value *InstOrCE, unsigned Opcode,
   default: return nullptr;
   case Instruction::ICmp:
   case Instruction::FCmp: llvm_unreachable("Invalid for compares");
+  case Instruction::Freeze:
+    return isGuaranteedNotToBeUndefOrPoison(Ops[0]) ? Ops[0] : nullptr;
   case Instruction::Call:
     if (auto *F = dyn_cast<Function>(Ops.back())) {
       const auto *Call = cast<CallBase>(InstOrCE);
diff --git a/llvm/test/Analysis/ConstantFolding/freeze.ll b/llvm/test/Analysis/ConstantFolding/freeze.ll
index c2da041308456..4908f71b924ec 100644
--- a/llvm/test/Analysis/ConstantFolding/freeze.ll
+++ b/llvm/test/Analysis/ConstantFolding/freeze.ll
@@ -6,8 +6,7 @@
 
 define i64 @ptrdiff1() {
 ; CHECK-LABEL: @ptrdiff1(
-; CHECK-NEXT:    [[R:%.*]] = freeze i64 sub (i64 ptrtoint (i16* @g to i64), i64 ptrtoint (i16* @g2 to i64))
-; CHECK-NEXT:    ret i64 [[R]]
+; CHECK-NEXT:    ret i64 sub (i64 ptrtoint (i16* @g to i64), i64 ptrtoint (i16* @g2 to i64))
 ;
   %i = ptrtoint i16* @g to i64
   %i2 = ptrtoint i16* @g2 to i64
@@ -18,8 +17,7 @@ define i64 @ptrdiff1() {
 
 define i64 @ptrdiff2() {
 ; CHECK-LABEL: @ptrdiff2(
-; CHECK-NEXT:    [[R:%.*]] = freeze i64 -2
-; CHECK-NEXT:    ret i64 [[R]]
+; CHECK-NEXT:    ret i64 -2
 ;
   %i = ptrtoint i16* @g to i64
   %gep = getelementptr i16, i16* @g, i64 1

From 920e26797451774b29debfaa3faaca2cb18cfb1e Mon Sep 17 00:00:00 2001
From: Juneyoung Lee <aqjune@gmail.com>
Date: Sun, 26 Jul 2020 22:00:01 +0900
Subject: [PATCH 0123/1035] [JumpThreading] Add a test for D84598; NFC

---
 llvm/test/Transforms/JumpThreading/freeze.ll | 94 ++++++++++++++++++++
 1 file changed, 94 insertions(+)
 create mode 100644 llvm/test/Transforms/JumpThreading/freeze.ll

diff --git a/llvm/test/Transforms/JumpThreading/freeze.ll b/llvm/test/Transforms/JumpThreading/freeze.ll
new file mode 100644
index 0000000000000..37d2ad672f789
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/freeze.ll
@@ -0,0 +1,94 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -jump-threading -S < %s | FileCheck %s
+
+declare i32 @f1()
+declare i32 @f2()
+declare void @f3()
+
+define i32 @test1(i1 %cond) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[T1:%.*]], label [[F1:%.*]]
+; CHECK:       T1:
+; CHECK-NEXT:    [[V1:%.*]] = call i32 @f1()
+; CHECK-NEXT:    br label [[MERGE:%.*]]
+; CHECK:       F1:
+; CHECK-NEXT:    [[V2:%.*]] = call i32 @f2()
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       Merge:
+; CHECK-NEXT:    [[A:%.*]] = phi i1 [ true, [[T1]] ], [ false, [[F1]] ]
+; CHECK-NEXT:    [[B:%.*]] = phi i32 [ [[V1]], [[T1]] ], [ [[V2]], [[F1]] ]
+; CHECK-NEXT:    [[A_FR:%.*]] = freeze i1 [[A]]
+; CHECK-NEXT:    br i1 [[A_FR]], label [[T2:%.*]], label [[F2:%.*]]
+; CHECK:       T2:
+; CHECK-NEXT:    call void @f3()
+; CHECK-NEXT:    ret i32 [[B]]
+; CHECK:       F2:
+; CHECK-NEXT:    ret i32 [[B]]
+;
+  br i1 %cond, label %T1, label %F1
+
+T1:
+  %v1 = call i32 @f1()
+  br label %Merge
+
+F1:
+  %v2 = call i32 @f2()
+  br label %Merge
+
+Merge:
+  %A = phi i1 [true, %T1], [false, %F1]
+  %B = phi i32 [%v1, %T1], [%v2, %F1]
+  %A.fr = freeze i1 %A
+  br i1 %A.fr, label %T2, label %F2
+
+T2:
+  call void @f3()
+  ret i32 %B
+
+F2:
+  ret i32 %B
+}
+
+define i32 @test2(i1 %cond, i1 %cond2) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[T1:%.*]], label [[F1:%.*]]
+; CHECK:       T1:
+; CHECK-NEXT:    [[V1:%.*]] = call i32 @f1()
+; CHECK-NEXT:    br label [[MERGE:%.*]]
+; CHECK:       F1:
+; CHECK-NEXT:    [[V2:%.*]] = call i32 @f2()
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       Merge:
+; CHECK-NEXT:    [[A:%.*]] = phi i1 [ true, [[T1]] ], [ [[COND2:%.*]], [[F1]] ]
+; CHECK-NEXT:    [[B:%.*]] = phi i32 [ [[V1]], [[T1]] ], [ [[V2]], [[F1]] ]
+; CHECK-NEXT:    [[A_FR:%.*]] = freeze i1 [[A]]
+; CHECK-NEXT:    br i1 [[A_FR]], label [[T2:%.*]], label [[F2:%.*]]
+; CHECK:       T2:
+; CHECK-NEXT:    call void @f3()
+; CHECK-NEXT:    ret i32 [[B]]
+; CHECK:       F2:
+; CHECK-NEXT:    ret i32 [[B]]
+;
+  br i1 %cond, label %T1, label %F1
+
+T1:
+  %v1 = call i32 @f1()
+  br label %Merge
+
+F1:
+  %v2 = call i32 @f2()
+  br label %Merge
+
+Merge:
+  %A = phi i1 [true, %T1], [%cond2, %F1]
+  %B = phi i32 [%v1, %T1], [%v2, %F1]
+  %A.fr = freeze i1 %A
+  br i1 %A.fr, label %T2, label %F2
+
+T2:
+  call void @f3()
+  ret i32 %B
+
+F2:
+  ret i32 %B
+}

From 912e9e526233c54aa08b082c957499fa9124eece Mon Sep 17 00:00:00 2001
From: Sanjay Patel <spatel@rotateright.com>
Date: Fri, 24 Jul 2020 14:45:50 -0400
Subject: [PATCH 0124/1035] [InstSimplify] add tests for fcmp with infinity
 constant; NFC

---
 .../InstSimplify/floating-point-compare.ll    | 46 +++++++++++++++++++
 1 file changed, 46 insertions(+)

diff --git a/llvm/test/Transforms/InstSimplify/floating-point-compare.ll b/llvm/test/Transforms/InstSimplify/floating-point-compare.ll
index 718a4427e15e3..6ce5c5e67b08a 100644
--- a/llvm/test/Transforms/InstSimplify/floating-point-compare.ll
+++ b/llvm/test/Transforms/InstSimplify/floating-point-compare.ll
@@ -1070,6 +1070,28 @@ define i1 @is_infinite_or_nan(float %x) {
   ret i1 %r
 }
 
+define i1 @is_infinite_or_nan2(float %x) {
+; CHECK-LABEL: @is_infinite_or_nan2(
+; CHECK-NEXT:    [[XABS:%.*]] = call nnan ninf float @llvm.fabs.f32(float [[X:%.*]])
+; CHECK-NEXT:    [[R:%.*]] = fcmp ueq float [[XABS]], 0x7FF0000000000000
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %xabs = call nnan ninf float @llvm.fabs.f32(float %x)
+  %r = fcmp ueq float %xabs, 0x7FF0000000000000
+  ret i1 %r
+}
+
+define <2 x i1> @is_infinite_neg_or_nan(<2 x float> %x) {
+; CHECK-LABEL: @is_infinite_neg_or_nan(
+; CHECK-NEXT:    [[X42:%.*]] = fadd nnan ninf <2 x float> [[X:%.*]], <float 4.200000e+01, float 4.200000e+01>
+; CHECK-NEXT:    [[R:%.*]] = fcmp ueq <2 x float> [[X42]], <float 0xFFF0000000000000, float 0xFFF0000000000000>
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %x42 = fadd nnan ninf <2 x float> %x, <float 42.0, float 42.0>
+  %r = fcmp ueq <2 x float> %x42, <float 0xFFF0000000000000, float 0xFFF0000000000000>
+  ret <2 x i1> %r
+}
+
 define i1 @is_finite_or_nan(i1 %c, double %x) {
 ; CHECK-LABEL: @is_finite_or_nan(
 ; CHECK-NEXT:    ret i1 true
@@ -1101,3 +1123,27 @@ define i1 @is_finite_and_ordered(double %x) {
   %r = fcmp one double %xx, 0x7FF0000000000000
   ret i1 %r
 }
+
+define i1 @is_finite(i1 %c, double %x) {
+; CHECK-LABEL: @is_finite(
+; CHECK-NEXT:    [[XX:%.*]] = fmul nnan ninf double [[X:%.*]], [[X]]
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[C:%.*]], double 4.200000e+01, double [[XX]]
+; CHECK-NEXT:    [[R:%.*]] = fcmp one double [[S]], 0x7FF0000000000000
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %xx = fmul nnan ninf double %x, %x
+  %s = select i1 %c, double 42.0, double %xx
+  %r = fcmp one double %s, 0x7FF0000000000000
+  ret i1 %r
+}
+
+define <2 x i1> @is_finite_commute(<2 x i8> %x) {
+; CHECK-LABEL: @is_finite_commute(
+; CHECK-NEXT:    [[CAST:%.*]] = uitofp <2 x i8> [[X:%.*]] to <2 x float>
+; CHECK-NEXT:    [[R:%.*]] = fcmp one <2 x float> <float 0x7FF0000000000000, float 0x7FF0000000000000>, [[CAST]]
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %cast = uitofp <2 x i8> %x to <2 x float>
+  %r = fcmp one <2 x float> <float 0x7FF0000000000000, float 0x7FF0000000000000>, %cast
+  ret <2 x i1> %r
+}

From b89ae102e6f5ed3760f1ae5788bd76ef8e9d9490 Mon Sep 17 00:00:00 2001
From: Sanjay Patel <spatel@rotateright.com>
Date: Fri, 24 Jul 2020 15:11:02 -0400
Subject: [PATCH 0125/1035] [InstSimplify] fold fcmp using isKnownNeverInfinity
 + isKnownNeverNaN

Follow-up to D84035 / rG7393d7574c09.
This sidesteps a question of FMF/poison on fcmp raised in PR46077:
http://bugs.llvm.org/PR46077

https://alive2.llvm.org/ce/z/TCsyzD
  define i1 @src(float %x) {
  %0:
    %x42 = fadd nnan ninf float %x, 42.000000
    %r = fcmp ueq float %x42, inf
    ret i1 %r
  }
  =>
  define i1 @tgt(float %x) {
  %0:
    ret i1 0
  }
  Transformation seems to be correct!

https://alive2.llvm.org/ce/z/FQaH7a
  define i1 @src(i8 %x) {
  %0:
    %cast = uitofp i8 %x to float
    %r = fcmp one float inf, %cast
    ret i1 %r
  }
  =>
  define i1 @tgt(i8 %x) {
  %0:
    ret i1 1
  }
  Transformation seems to be correct!
---
 llvm/lib/Analysis/InstructionSimplify.cpp       |  8 ++++++++
 .../InstSimplify/floating-point-compare.ll      | 17 ++++-------------
 2 files changed, 12 insertions(+), 13 deletions(-)

diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp
index 49346860f251e..396fc22920cdf 100644
--- a/llvm/lib/Analysis/InstructionSimplify.cpp
+++ b/llvm/lib/Analysis/InstructionSimplify.cpp
@@ -3701,6 +3701,14 @@ static Value *SimplifyFCmpInst(unsigned Predicate, Value *LHS, Value *RHS,
       // LHS != Inf
       if (Pred == FCmpInst::FCMP_UNE && isKnownNeverInfinity(LHS, Q.TLI))
         return getTrue(RetTy);
+      // LHS == Inf || LHS == NaN
+      if (Pred == FCmpInst::FCMP_UEQ && isKnownNeverInfinity(LHS, Q.TLI) &&
+          isKnownNeverNaN(LHS, Q.TLI))
+        return getFalse(RetTy);
+      // LHS != Inf && LHS != NaN
+      if (Pred == FCmpInst::FCMP_ONE && isKnownNeverInfinity(LHS, Q.TLI) &&
+          isKnownNeverNaN(LHS, Q.TLI))
+        return getTrue(RetTy);
     }
     if (C->isNegative() && !C->isNegZero()) {
       assert(!C->isNaN() && "Unexpected NaN constant!");
diff --git a/llvm/test/Transforms/InstSimplify/floating-point-compare.ll b/llvm/test/Transforms/InstSimplify/floating-point-compare.ll
index 6ce5c5e67b08a..d0864e78570b3 100644
--- a/llvm/test/Transforms/InstSimplify/floating-point-compare.ll
+++ b/llvm/test/Transforms/InstSimplify/floating-point-compare.ll
@@ -1072,9 +1072,7 @@ define i1 @is_infinite_or_nan(float %x) {
 
 define i1 @is_infinite_or_nan2(float %x) {
 ; CHECK-LABEL: @is_infinite_or_nan2(
-; CHECK-NEXT:    [[XABS:%.*]] = call nnan ninf float @llvm.fabs.f32(float [[X:%.*]])
-; CHECK-NEXT:    [[R:%.*]] = fcmp ueq float [[XABS]], 0x7FF0000000000000
-; CHECK-NEXT:    ret i1 [[R]]
+; CHECK-NEXT:    ret i1 false
 ;
   %xabs = call nnan ninf float @llvm.fabs.f32(float %x)
   %r = fcmp ueq float %xabs, 0x7FF0000000000000
@@ -1083,9 +1081,7 @@ define i1 @is_infinite_or_nan2(float %x) {
 
 define <2 x i1> @is_infinite_neg_or_nan(<2 x float> %x) {
 ; CHECK-LABEL: @is_infinite_neg_or_nan(
-; CHECK-NEXT:    [[X42:%.*]] = fadd nnan ninf <2 x float> [[X:%.*]], <float 4.200000e+01, float 4.200000e+01>
-; CHECK-NEXT:    [[R:%.*]] = fcmp ueq <2 x float> [[X42]], <float 0xFFF0000000000000, float 0xFFF0000000000000>
-; CHECK-NEXT:    ret <2 x i1> [[R]]
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
 ;
   %x42 = fadd nnan ninf <2 x float> %x, <float 42.0, float 42.0>
   %r = fcmp ueq <2 x float> %x42, <float 0xFFF0000000000000, float 0xFFF0000000000000>
@@ -1126,10 +1122,7 @@ define i1 @is_finite_and_ordered(double %x) {
 
 define i1 @is_finite(i1 %c, double %x) {
 ; CHECK-LABEL: @is_finite(
-; CHECK-NEXT:    [[XX:%.*]] = fmul nnan ninf double [[X:%.*]], [[X]]
-; CHECK-NEXT:    [[S:%.*]] = select i1 [[C:%.*]], double 4.200000e+01, double [[XX]]
-; CHECK-NEXT:    [[R:%.*]] = fcmp one double [[S]], 0x7FF0000000000000
-; CHECK-NEXT:    ret i1 [[R]]
+; CHECK-NEXT:    ret i1 true
 ;
   %xx = fmul nnan ninf double %x, %x
   %s = select i1 %c, double 42.0, double %xx
@@ -1139,9 +1132,7 @@ define i1 @is_finite(i1 %c, double %x) {
 
 define <2 x i1> @is_finite_commute(<2 x i8> %x) {
 ; CHECK-LABEL: @is_finite_commute(
-; CHECK-NEXT:    [[CAST:%.*]] = uitofp <2 x i8> [[X:%.*]] to <2 x float>
-; CHECK-NEXT:    [[R:%.*]] = fcmp one <2 x float> <float 0x7FF0000000000000, float 0x7FF0000000000000>, [[CAST]]
-; CHECK-NEXT:    ret <2 x i1> [[R]]
+; CHECK-NEXT:    ret <2 x i1> <i1 true, i1 true>
 ;
   %cast = uitofp <2 x i8> %x to <2 x float>
   %r = fcmp one <2 x float> <float 0x7FF0000000000000, float 0x7FF0000000000000>, %cast

From c6cf71107af8b8110572e8fe42d4624e1fc83549 Mon Sep 17 00:00:00 2001
From: Sanjay Patel <spatel@rotateright.com>
Date: Sat, 25 Jul 2020 16:40:14 -0400
Subject: [PATCH 0126/1035] [InstSimplify] add tests for min/max intrinsics;
 NFC

---
 .../InstSimplify/maxmin_intrinsics.ll         | 83 +++++++++++++++++++
 1 file changed, 83 insertions(+)
 create mode 100644 llvm/test/Transforms/InstSimplify/maxmin_intrinsics.ll

diff --git a/llvm/test/Transforms/InstSimplify/maxmin_intrinsics.ll b/llvm/test/Transforms/InstSimplify/maxmin_intrinsics.ll
new file mode 100644
index 0000000000000..fe940ef7bc179
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/maxmin_intrinsics.ll
@@ -0,0 +1,83 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+declare i8 @llvm.smax.i8(i8, i8)
+declare <2 x i8> @llvm.smax.v2i8(<2 x i8>, <2 x i8>)
+declare i8 @llvm.smin.i8(i8, i8)
+declare <2 x i8> @llvm.smin.v2i8(<2 x i8>, <2 x i8>)
+declare i8 @llvm.umax.i8(i8, i8)
+declare <2 x i8> @llvm.umax.v2i8(<2 x i8>, <2 x i8>)
+declare i8 @llvm.umin.i8(i8, i8)
+declare <2 x i8> @llvm.umin.v2i8(<2 x i8>, <2 x i8>)
+
+define i8 @smax_maxval(i8 %x) {
+; CHECK-LABEL: @smax_maxval(
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.smax.i8(i8 [[X:%.*]], i8 127)
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %r = call i8 @llvm.smax.i8(i8 %x, i8 127)
+  ret i8 %r
+}
+
+define <2 x i8> @smax_maxval_commute(<2 x i8> %x) {
+; CHECK-LABEL: @smax_maxval_commute(
+; CHECK-NEXT:    [[R:%.*]] = call <2 x i8> @llvm.smax.v2i8(<2 x i8> <i8 127, i8 127>, <2 x i8> [[X:%.*]])
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %r = call <2 x i8> @llvm.smax.v2i8(<2 x i8> <i8 127, i8 127>, <2 x i8> %x)
+  ret <2 x i8> %r
+}
+
+define i8 @smin_minval(i8 %x) {
+; CHECK-LABEL: @smin_minval(
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.smin.i8(i8 -128, i8 [[X:%.*]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %r = call i8 @llvm.smin.i8(i8 -128, i8 %x)
+  ret i8 %r
+}
+
+define <2 x i8> @smin_minval_commute(<2 x i8> %x) {
+; CHECK-LABEL: @smin_minval_commute(
+; CHECK-NEXT:    [[R:%.*]] = call <2 x i8> @llvm.smin.v2i8(<2 x i8> [[X:%.*]], <2 x i8> <i8 -128, i8 -128>)
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %r = call <2 x i8> @llvm.smin.v2i8(<2 x i8> %x, <2 x i8> <i8 -128, i8 -128>)
+  ret <2 x i8> %r
+}
+
+define i8 @umax_maxval(i8 %x) {
+; CHECK-LABEL: @umax_maxval(
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.umax.i8(i8 [[X:%.*]], i8 -1)
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %r = call i8 @llvm.umax.i8(i8 %x, i8 255)
+  ret i8 %r
+}
+
+define <2 x i8> @umax_maxval_commute(<2 x i8> %x) {
+; CHECK-LABEL: @umax_maxval_commute(
+; CHECK-NEXT:    [[R:%.*]] = call <2 x i8> @llvm.umax.v2i8(<2 x i8> <i8 -1, i8 -1>, <2 x i8> [[X:%.*]])
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %r = call <2 x i8> @llvm.umax.v2i8(<2 x i8> <i8 255, i8 255>, <2 x i8> %x)
+  ret <2 x i8> %r
+}
+
+define i8 @umin_minval(i8 %x) {
+; CHECK-LABEL: @umin_minval(
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.umin.i8(i8 0, i8 [[X:%.*]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %r = call i8 @llvm.umin.i8(i8 0, i8 %x)
+  ret i8 %r
+}
+
+define <2 x i8> @umin_minval_commute(<2 x i8> %x) {
+; CHECK-LABEL: @umin_minval_commute(
+; CHECK-NEXT:    [[R:%.*]] = call <2 x i8> @llvm.umin.v2i8(<2 x i8> [[X:%.*]], <2 x i8> zeroinitializer)
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %r = call <2 x i8> @llvm.umin.v2i8(<2 x i8> %x, <2 x i8> zeroinitializer)
+  ret <2 x i8> %r
+}

From b1731da871593a7bd5acad9830dc0797c5e6dec9 Mon Sep 17 00:00:00 2001
From: Xing GUO <higuoxing@gmail.com>
Date: Sun, 26 Jul 2020 16:01:22 +0800
Subject: [PATCH 0127/1035] [DWARFYAML] Rename getUsedSectionNames() to
 getNonEmptySectionNames().

This patch renames getUsedSectionNames() to getNonEmptySectionNames.
NFC.
---
 llvm/include/llvm/ObjectYAML/DWARFYAML.h | 2 +-
 llvm/lib/ObjectYAML/DWARFYAML.cpp        | 2 +-
 llvm/lib/ObjectYAML/ELFEmitter.cpp       | 4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/llvm/include/llvm/ObjectYAML/DWARFYAML.h b/llvm/include/llvm/ObjectYAML/DWARFYAML.h
index 7b18ceebdb1cf..88ac404b21b13 100644
--- a/llvm/include/llvm/ObjectYAML/DWARFYAML.h
+++ b/llvm/include/llvm/ObjectYAML/DWARFYAML.h
@@ -225,7 +225,7 @@ struct Data {
 
   bool isEmpty() const;
 
-  SetVector<StringRef> getUsedSectionNames() const;
+  SetVector<StringRef> getNonEmptySectionNames() const;
 };
 
 } // end namespace DWARFYAML
diff --git a/llvm/lib/ObjectYAML/DWARFYAML.cpp b/llvm/lib/ObjectYAML/DWARFYAML.cpp
index 186ec8ac4bdc3..e5c77bc3721fb 100644
--- a/llvm/lib/ObjectYAML/DWARFYAML.cpp
+++ b/llvm/lib/ObjectYAML/DWARFYAML.cpp
@@ -22,7 +22,7 @@ bool DWARFYAML::Data::isEmpty() const {
          !GNUPubTypes && CompileUnits.empty() && DebugLines.empty();
 }
 
-SetVector<StringRef> DWARFYAML::Data::getUsedSectionNames() const {
+SetVector<StringRef> DWARFYAML::Data::getNonEmptySectionNames() const {
   SetVector<StringRef> SecNames;
   if (!DebugStrings.empty())
     SecNames.insert("debug_str");
diff --git a/llvm/lib/ObjectYAML/ELFEmitter.cpp b/llvm/lib/ObjectYAML/ELFEmitter.cpp
index fc80839064ee0..65b03050c7cdb 100644
--- a/llvm/lib/ObjectYAML/ELFEmitter.cpp
+++ b/llvm/lib/ObjectYAML/ELFEmitter.cpp
@@ -355,7 +355,7 @@ ELFState<ELFT>::ELFState(ELFYAML::Object &D, yaml::ErrorHandler EH)
   if (Doc.Symbols)
     ImplicitSections.push_back(".symtab");
   if (Doc.DWARF)
-    for (StringRef DebugSecName : Doc.DWARF->getUsedSectionNames()) {
+    for (StringRef DebugSecName : Doc.DWARF->getNonEmptySectionNames()) {
       std::string SecName = ("." + DebugSecName).str();
       ImplicitSections.push_back(StringRef(SecName).copy(StringAlloc));
     }
@@ -931,7 +931,7 @@ void ELFState<ELFT>::initStrtabSectionHeader(Elf_Shdr &SHeader, StringRef Name,
 }
 
 static bool shouldEmitDWARF(DWARFYAML::Data &DWARF, StringRef Name) {
-  SetVector<StringRef> DebugSecNames = DWARF.getUsedSectionNames();
+  SetVector<StringRef> DebugSecNames = DWARF.getNonEmptySectionNames();
   return Name.consume_front(".") && DebugSecNames.count(Name);
 }
 

From 4f6502ab3356f51f711b57001ab596f1bc66fd3a Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Sat, 25 Jul 2020 13:21:31 -0400
Subject: [PATCH 0128/1035] AMDGPU/GlobalISel: Replace selection tests for
 G_CONSTANT/G_FCONSTANT

Split into separate tests and make more consistent with the others.
---
 .../GlobalISel/inst-select-constant.mir       | 166 ++++++++++++++----
 .../GlobalISel/inst-select-fconstant.mir      | 159 +++++++++++++++++
 2 files changed, 288 insertions(+), 37 deletions(-)
 create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-fconstant.mir

diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-constant.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-constant.mir
index db37495f052d8..2f2c305cfac79 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-constant.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-constant.mir
@@ -1,55 +1,147 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -march=amdgcn -mcpu=hawaii -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s -check-prefixes=GCN
 
 ---
-
-name:            constant
+name:            constant_v_s32
 legalized:       true
 regBankSelected: true
 tracksRegLiveness: true
 
-
 body: |
   bb.0:
-    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
-    ; GCN-LABEL: name: constant
-    %0:vgpr(p1) = COPY $vgpr0_vgpr1
-    %1:vgpr(p1) = COPY $vgpr2_vgpr3
-
-    ; GCN: %{{[0-9]+}}:sreg_32 = S_MOV_B32 1
-    %2:sreg_32(s32) = G_CONSTANT i32 1
-
-    ; GCN: [[LO0:%[0-9]+]]:sreg_32 = S_MOV_B32 0
-    ; GCN: [[HI0:%[0-9]+]]:sreg_32 = S_MOV_B32 1
-    ; GCN: %{{[0-9]+}}:sreg_64 = REG_SEQUENCE [[LO0]], %subreg.sub0, [[HI0]], %subreg.sub1
-    %3:sgpr(s64) = G_CONSTANT i64 4294967296
-
-    ; GCN: %{{[0-9]+}}:sreg_32 = S_MOV_B32 1065353216
-    %4:sgpr(s32) = G_FCONSTANT float 1.0
+    ; GCN-LABEL: name: constant_v_s32
+    ; GCN: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    ; GCN: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+    ; GCN: [[V_MOV_B32_e32_2:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4294967295, implicit $exec
+    ; GCN: [[V_MOV_B32_e32_3:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4294967242, implicit $exec
+    ; GCN: [[V_MOV_B32_e32_4:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 27, implicit $exec
+    ; GCN: S_ENDPGM 0, implicit [[V_MOV_B32_e32_]], implicit [[V_MOV_B32_e32_1]], implicit [[V_MOV_B32_e32_2]], implicit [[V_MOV_B32_e32_3]], implicit [[V_MOV_B32_e32_4]]
+    %0:vgpr(s32) = G_CONSTANT i32 0
+    %1:vgpr(s32) = G_CONSTANT i32 1
+    %2:vgpr(s32) = G_CONSTANT i32 -1
+    %3:vgpr(s32) = G_CONSTANT i32 -54
+    %4:vgpr(s32) = G_CONSTANT i32 27
+    S_ENDPGM 0, implicit %0 , implicit %1 , implicit %2, implicit %3, implicit %4
+...
 
-    ; GCN: %5:sreg_64 = S_MOV_B64 4607182418800017408
-    %5:sgpr(s64) = G_FCONSTANT double 1.0
+---
+name:            constant_s_s32
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
 
-    ; GCN: [[LO1:%[0-9]+]]:sreg_32 = S_MOV_B32 0
-    ; GCN: [[HI1:%[0-9]+]]:sreg_32 = S_MOV_B32 1076101120
-    ; GCN: %{{[0-9]+}}:sreg_64 = REG_SEQUENCE [[LO1]], %subreg.sub0, [[HI1]], %subreg.sub1
-    %6:sgpr(s64) = G_FCONSTANT double 10.0
+body: |
+  bb.0:
+    ; GCN-LABEL: name: constant_s_s32
+    ; GCN: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; GCN: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 1
+    ; GCN: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 4294967295
+    ; GCN: [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 4294967242
+    ; GCN: [[S_MOV_B32_4:%[0-9]+]]:sreg_32 = S_MOV_B32 27
+    ; GCN: S_ENDPGM 0, implicit [[S_MOV_B32_]], implicit [[S_MOV_B32_1]], implicit [[S_MOV_B32_2]], implicit [[S_MOV_B32_3]], implicit [[S_MOV_B32_4]]
+    %0:sgpr(s32) = G_CONSTANT i32 0
+    %1:sgpr(s32) = G_CONSTANT i32 1
+    %2:sgpr(s32) = G_CONSTANT i32 -1
+    %3:sgpr(s32) = G_CONSTANT i32 -54
+    %4:sgpr(s32) = G_CONSTANT i32 27
+    S_ENDPGM 0, implicit %0 , implicit %1 , implicit %2, implicit %3, implicit %4
+...
 
-    ; GCN: %{{[0-9]+}}:vgpr_32 = V_MOV_B32_e32 1
-    %7:vgpr(s32) = G_CONSTANT i32 1
+# FIXME
+# ---
+# name:            constant_v_s16
+# legalized:       true
+# regBankSelected: true
+# tracksRegLiveness: true
 
-    ; GCN: [[LO2:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0
-    ; GCN: [[HI2:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1
-    ; GCN: %{{[0-9]+}}:vreg_64 = REG_SEQUENCE [[LO2]], %subreg.sub0, [[HI2]], %subreg.sub1
-    %8:vgpr(s64) = G_CONSTANT i64 4294967296
+# body: |
+#   bb.0:
+#     %0:vgpry(s16) = G_CONSTANT i16 0
+#     %1:vgpr(s16) = G_CONSTANT i16 1
+#     %2:vgpr(s16) = G_CONSTANT i16 -1
+#     %3:vgpr(s16) = G_CONSTANT i16 -54
+#     %4:vgpr(s16) = G_CONSTANT i16 27
+#     S_ENDPGM 0, implicit %0 , implicit %1 , implicit %2, implicit %3, implicit %4
+# ...
 
-    ; GCN: %{{[0-9]+}}:vgpr_32 = V_MOV_B32_e32 1065353216
-    %9:vgpr(s32) = G_FCONSTANT float 1.0
+---
+name:            constant_v_s64
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
 
-    ; GCN: [[LO3:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0
-    ; GCN: [[HI3:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1072693248
-    ; GCN: %{{[0-9]+}}:vreg_64 = REG_SEQUENCE [[LO3]], %subreg.sub0, [[HI3]], %subreg.sub1
-    %10:vgpr(s64) = G_FCONSTANT double 1.0
+body: |
+  bb.0:
+    ; GCN-LABEL: name: constant_v_s64
+    ; GCN: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    ; GCN: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    ; GCN: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_MOV_B32_e32_]], %subreg.sub0, [[V_MOV_B32_e32_1]], %subreg.sub1
+    ; GCN: [[V_MOV_B32_e32_2:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+    ; GCN: [[V_MOV_B32_e32_3:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    ; GCN: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_MOV_B32_e32_2]], %subreg.sub0, [[V_MOV_B32_e32_3]], %subreg.sub1
+    ; GCN: [[V_MOV_B32_e32_4:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4294967295, implicit $exec
+    ; GCN: [[V_MOV_B32_e32_5:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -1, implicit $exec
+    ; GCN: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_MOV_B32_e32_4]], %subreg.sub0, [[V_MOV_B32_e32_5]], %subreg.sub1
+    ; GCN: [[V_MOV_B32_e32_6:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4294967242, implicit $exec
+    ; GCN: [[V_MOV_B32_e32_7:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -1, implicit $exec
+    ; GCN: [[REG_SEQUENCE3:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_MOV_B32_e32_6]], %subreg.sub0, [[V_MOV_B32_e32_7]], %subreg.sub1
+    ; GCN: [[V_MOV_B32_e32_8:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 27, implicit $exec
+    ; GCN: [[V_MOV_B32_e32_9:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    ; GCN: [[REG_SEQUENCE4:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_MOV_B32_e32_8]], %subreg.sub0, [[V_MOV_B32_e32_9]], %subreg.sub1
+    ; GCN: [[V_MOV_B32_e32_10:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4294967295, implicit $exec
+    ; GCN: [[V_MOV_B32_e32_11:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    ; GCN: [[REG_SEQUENCE5:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_MOV_B32_e32_10]], %subreg.sub0, [[V_MOV_B32_e32_11]], %subreg.sub1
+    ; GCN: [[V_MOV_B32_e32_12:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    ; GCN: [[V_MOV_B32_e32_13:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+    ; GCN: [[REG_SEQUENCE6:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_MOV_B32_e32_12]], %subreg.sub0, [[V_MOV_B32_e32_13]], %subreg.sub1
+    ; GCN: [[V_MOV_B32_e32_14:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 23255, implicit $exec
+    ; GCN: [[V_MOV_B32_e32_15:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -16, implicit $exec
+    ; GCN: [[REG_SEQUENCE7:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_MOV_B32_e32_14]], %subreg.sub0, [[V_MOV_B32_e32_15]], %subreg.sub1
+    ; GCN: S_ENDPGM 0, implicit [[REG_SEQUENCE]], implicit [[REG_SEQUENCE1]], implicit [[REG_SEQUENCE2]], implicit [[REG_SEQUENCE3]], implicit [[REG_SEQUENCE4]], implicit [[REG_SEQUENCE5]], implicit [[REG_SEQUENCE6]], implicit [[REG_SEQUENCE7]]
+    %0:vgpr(s64) = G_CONSTANT i64 0
+    %1:vgpr(s64) = G_CONSTANT i64 1
+    %2:vgpr(s64) = G_CONSTANT i64 -1
+    %3:vgpr(s64) = G_CONSTANT i64 -54
+    %4:vgpr(s64) = G_CONSTANT i64 27
+    %5:vgpr(s64) = G_CONSTANT i64 4294967295
+    %6:vgpr(s64) = G_CONSTANT i64 4294967296
+    %7:vgpr(s64) = G_CONSTANT i64 18446744004990098135
+    S_ENDPGM 0, implicit %0 , implicit %1 , implicit %2, implicit %3, implicit %4, implicit %5, implicit %6, implicit %7
+...
 
-    S_ENDPGM 0, implicit %2, implicit %4, implicit %5, implicit %6, implicit %8, implicit %3, implicit %5, implicit %7, implicit %9, implicit %10
+---
+name:            constant_s_s64
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
 
+body: |
+  bb.0:
+    ; GCN-LABEL: name: constant_s_s64
+    ; GCN: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+    ; GCN: [[S_MOV_B64_1:%[0-9]+]]:sreg_64 = S_MOV_B64 1
+    ; GCN: [[S_MOV_B64_2:%[0-9]+]]:sreg_64 = S_MOV_B64 -1
+    ; GCN: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4294967242
+    ; GCN: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
+    ; GCN: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+    ; GCN: [[S_MOV_B64_3:%[0-9]+]]:sreg_64 = S_MOV_B64 27
+    ; GCN: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 4294967295
+    ; GCN: [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; GCN: [[REG_SEQUENCE1:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_2]], %subreg.sub0, [[S_MOV_B32_3]], %subreg.sub1
+    ; GCN: [[S_MOV_B32_4:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; GCN: [[S_MOV_B32_5:%[0-9]+]]:sreg_32 = S_MOV_B32 1
+    ; GCN: [[REG_SEQUENCE2:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_4]], %subreg.sub0, [[S_MOV_B32_5]], %subreg.sub1
+    ; GCN: [[S_MOV_B32_6:%[0-9]+]]:sreg_32 = S_MOV_B32 23255
+    ; GCN: [[S_MOV_B32_7:%[0-9]+]]:sreg_32 = S_MOV_B32 -16
+    ; GCN: [[REG_SEQUENCE3:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_6]], %subreg.sub0, [[S_MOV_B32_7]], %subreg.sub1
+    ; GCN: S_ENDPGM 0, implicit [[S_MOV_B64_]], implicit [[S_MOV_B64_1]], implicit [[S_MOV_B64_2]], implicit [[REG_SEQUENCE]], implicit [[S_MOV_B64_3]], implicit [[REG_SEQUENCE1]], implicit [[REG_SEQUENCE2]], implicit [[REG_SEQUENCE3]]
+    %0:sgpr(s64) = G_CONSTANT i64 0
+    %1:sgpr(s64) = G_CONSTANT i64 1
+    %2:sgpr(s64) = G_CONSTANT i64 -1
+    %3:sgpr(s64) = G_CONSTANT i64 -54
+    %4:sgpr(s64) = G_CONSTANT i64 27
+    %5:sgpr(s64) = G_CONSTANT i64 4294967295
+    %6:sgpr(s64) = G_CONSTANT i64 4294967296
+    %7:sgpr(s64) = G_CONSTANT i64 18446744004990098135
+    S_ENDPGM 0, implicit %0 , implicit %1 , implicit %2, implicit %3, implicit %4, implicit %5, implicit %6, implicit %7
 ...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-fconstant.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-fconstant.mir
new file mode 100644
index 0000000000000..9afa4b08c0ecb
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-fconstant.mir
@@ -0,0 +1,159 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=hawaii -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck -check-prefix=GCN %s
+
+---
+name:            fconstant_v_s32
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    ; GCN-LABEL: name: fconstant_v_s32
+    ; GCN: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1065353216, implicit $exec
+    ; GCN: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1090519040, implicit $exec
+    ; GCN: [[V_MOV_B32_e32_2:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1065353216, implicit $exec
+    ; GCN: [[V_MOV_B32_e32_3:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1090519040, implicit $exec
+    ; GCN: S_ENDPGM 0, implicit [[V_MOV_B32_e32_]], implicit [[V_MOV_B32_e32_1]], implicit [[V_MOV_B32_e32_2]], implicit [[V_MOV_B32_e32_3]]
+    %0:vgpr(s32) = G_FCONSTANT float 1.0
+    %1:vgpr(s32) = G_FCONSTANT float 8.0
+    %2:vgpr(s32) = G_FCONSTANT float 1.0
+    %3:vgpr(s32) = G_FCONSTANT float 8.0
+    S_ENDPGM 0, implicit %0 , implicit %1 , implicit %2 , implicit %3
+...
+
+---
+name:            fconstant_s_s32
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    ; GCN-LABEL: name: fconstant_s_s32
+    ; GCN: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 1065353216
+    ; GCN: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 1090519040
+    ; GCN: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 3212836864
+    ; GCN: [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 3238002688
+    ; GCN: $sgpr0 = COPY [[S_MOV_B32_]]
+    ; GCN: $sgpr1 = COPY [[S_MOV_B32_1]]
+    ; GCN: S_ENDPGM 0, implicit [[S_MOV_B32_]], implicit [[S_MOV_B32_1]], implicit [[S_MOV_B32_2]], implicit [[S_MOV_B32_3]]
+    %0:sgpr(s32) = G_FCONSTANT float 1.0
+    %1:sgpr(s32) = G_FCONSTANT float 8.0
+    %2:sgpr(s32) = G_FCONSTANT float -1.0
+    %3:sgpr(s32) = G_FCONSTANT float -8.0
+    $sgpr0 = COPY %0
+    $sgpr1 = COPY %1
+    S_ENDPGM 0, implicit %0 , implicit %1 , implicit %2 , implicit %3
+
+...
+
+---
+name:            fconstant_v_s64
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    ; GCN-LABEL: name: fconstant_v_s64
+    ; GCN: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    ; GCN: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1072693248, implicit $exec
+    ; GCN: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_MOV_B32_e32_]], %subreg.sub0, [[V_MOV_B32_e32_1]], %subreg.sub1
+    ; GCN: [[V_MOV_B32_e32_2:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    ; GCN: [[V_MOV_B32_e32_3:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1075838976, implicit $exec
+    ; GCN: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_MOV_B32_e32_2]], %subreg.sub0, [[V_MOV_B32_e32_3]], %subreg.sub1
+    ; GCN: [[V_MOV_B32_e32_4:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    ; GCN: [[V_MOV_B32_e32_5:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -1073741824, implicit $exec
+    ; GCN: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_MOV_B32_e32_4]], %subreg.sub0, [[V_MOV_B32_e32_5]], %subreg.sub1
+    ; GCN: [[V_MOV_B32_e32_6:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    ; GCN: [[V_MOV_B32_e32_7:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1076101120, implicit $exec
+    ; GCN: [[REG_SEQUENCE3:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_MOV_B32_e32_6]], %subreg.sub0, [[V_MOV_B32_e32_7]], %subreg.sub1
+    ; GCN: $vgpr0_vgpr1 = COPY [[REG_SEQUENCE]]
+    ; GCN: $vgpr2_vgpr3 = COPY [[REG_SEQUENCE1]]
+    ; GCN: S_ENDPGM 0, implicit [[REG_SEQUENCE]], implicit [[REG_SEQUENCE1]], implicit [[REG_SEQUENCE2]], implicit [[REG_SEQUENCE3]]
+    %0:vgpr(s64) = G_FCONSTANT double 1.0
+    %1:vgpr(s64) = G_FCONSTANT double 8.0
+    %2:vgpr(s64) = G_FCONSTANT double -2.0
+    %3:vgpr(s64) = G_FCONSTANT double 10.0
+    $vgpr0_vgpr1 = COPY %0
+    $vgpr2_vgpr3 = COPY %1
+    S_ENDPGM 0, implicit %0 , implicit %1 , implicit %2 , implicit %3
+
+...
+
+---
+name:            fconstant_s_s64
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    ; GCN-LABEL: name: fconstant_s_s64
+    ; GCN: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 4607182418800017408
+    ; GCN: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; GCN: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 1075838976
+    ; GCN: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+    ; GCN: [[S_MOV_B64_1:%[0-9]+]]:sreg_64 = S_MOV_B64 -4611686018427387904
+    ; GCN: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; GCN: [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 -1071382528
+    ; GCN: [[REG_SEQUENCE1:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_2]], %subreg.sub0, [[S_MOV_B32_3]], %subreg.sub1
+    ; GCN: $sgpr0_sgpr1 = COPY [[S_MOV_B64_]]
+    ; GCN: $sgpr2_sgpr3 = COPY [[REG_SEQUENCE]]
+    ; GCN: S_ENDPGM 0, implicit [[S_MOV_B64_]], implicit [[REG_SEQUENCE]], implicit [[S_MOV_B64_1]], implicit [[REG_SEQUENCE1]]
+    %0:sgpr(s64) = G_FCONSTANT double 1.0
+    %1:sgpr(s64) = G_FCONSTANT double 8.0
+    %2:sgpr(s64) = G_FCONSTANT double -2.0
+    %3:sgpr(s64) = G_FCONSTANT double -10.0
+    $sgpr0_sgpr1 = COPY %0
+    $sgpr2_sgpr3 = COPY %1
+    S_ENDPGM 0, implicit %0 , implicit %1 , implicit %2 , implicit %3
+...
+
+---
+name:            fconstant_v_s16
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    ; GCN-LABEL: name: fconstant_v_s16
+    ; GCN: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 15360, implicit $exec
+    ; GCN: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 18432, implicit $exec
+    ; GCN: $vgpr0 = COPY [[V_MOV_B32_e32_]]
+    ; GCN: $vgpr1 = COPY [[V_MOV_B32_e32_1]]
+    %0:vgpr(s16) = G_FCONSTANT half 1.0
+    %1:vgpr(s16) = G_FCONSTANT half 8.0
+    %2:vgpr(s32) = G_ANYEXT %0
+    %3:vgpr(s32) = G_ANYEXT %1
+    $vgpr0 = COPY %2
+    $vgpr1 = COPY %3
+
+...
+
+---
+name:            fconstant_s_s16
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    ; GCN-LABEL: name: fconstant_s_s16
+    ; GCN: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 15360
+    ; GCN: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 18432
+    ; GCN: [[COPY:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+    ; GCN: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
+    ; GCN: $sgpr0 = COPY [[COPY]]
+    ; GCN: $sgpr1 = COPY [[COPY1]]
+    %0:sgpr(s16) = G_FCONSTANT half 1.0
+    %1:sgpr(s16) = G_FCONSTANT half 8.0
+    %2:vgpr(s32) = G_ANYEXT %0
+    %3:vgpr(s32) = G_ANYEXT %1
+    $sgpr0 = COPY %2
+    $sgpr1 = COPY %3
+
+...
+

From 4033aa1467d6ea0cb5289ed9db2f4e3cc015eac1 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Sat, 25 Jul 2020 14:37:29 -0400
Subject: [PATCH 0129/1035] AMDGPU/GlobalISel: Sign extend integer constants

This matches the DAG behavior and fixes immediate folding
---
 .../AMDGPU/AMDGPUInstructionSelector.cpp      |   2 +-
 .../CodeGen/AMDGPU/GlobalISel/add.v2i16.ll    |   8 +-
 .../GlobalISel/divergent-control-flow.ll      |  26 +-
 .../AMDGPU/GlobalISel/inst-select-add.mir     |   2 +-
 .../AMDGPU/GlobalISel/inst-select-and.mir     |  24 +-
 .../GlobalISel/inst-select-constant.mir       |   8 +-
 .../inst-select-extract-vector-elt.mir        |  12 +-
 .../GlobalISel/inst-select-load-local.mir     |   6 +-
 .../GlobalISel/inst-select-load-private.mir   |  28 +-
 .../AMDGPU/GlobalISel/inst-select-or.mir      |  24 +-
 .../AMDGPU/GlobalISel/inst-select-ptrmask.mir |  30 +-
 .../AMDGPU/GlobalISel/inst-select-xor.mir     |  24 +-
 .../GlobalISel/llvm.amdgcn.s.buffer.load.ll   |  26 +-
 .../AMDGPU/GlobalISel/llvm.amdgcn.sbfe.ll     |   8 +-
 .../AMDGPU/GlobalISel/llvm.amdgcn.ubfe.ll     |   8 +-
 .../CodeGen/AMDGPU/GlobalISel/localizer.ll    |   8 +-
 .../test/CodeGen/AMDGPU/GlobalISel/saddsat.ll | 244 +++---
 .../CodeGen/AMDGPU/GlobalISel/sdiv.i32.ll     |   2 +-
 .../CodeGen/AMDGPU/GlobalISel/sdiv.i64.ll     |  17 +-
 .../CodeGen/AMDGPU/GlobalISel/srem.i32.ll     |   2 +-
 .../CodeGen/AMDGPU/GlobalISel/srem.i64.ll     |  15 +-
 .../test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll | 750 +++++++++---------
 .../test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll | 330 ++++----
 .../CodeGen/AMDGPU/GlobalISel/udiv.i32.ll     |   2 +-
 .../CodeGen/AMDGPU/GlobalISel/udiv.i64.ll     |  19 +-
 .../CodeGen/AMDGPU/GlobalISel/urem.i32.ll     |   2 +-
 .../CodeGen/AMDGPU/GlobalISel/urem.i64.ll     |  19 +-
 27 files changed, 796 insertions(+), 850 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
index 4740a58519996..a126ed1daf17f 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
@@ -1952,7 +1952,7 @@ bool AMDGPUInstructionSelector::selectG_CONSTANT(MachineInstr &I) const {
     const APInt &Imm = ImmOp.getFPImm()->getValueAPF().bitcastToAPInt();
     ImmOp.ChangeToImmediate(Imm.getZExtValue());
   } else if (ImmOp.isCImm()) {
-    ImmOp.ChangeToImmediate(ImmOp.getCImm()->getZExtValue());
+    ImmOp.ChangeToImmediate(ImmOp.getCImm()->getSExtValue());
   }
 
   Register DstReg = I.getOperand(0).getReg();
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/add.v2i16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/add.v2i16.ll
index ccf846a933abd..c6c0eb7c4a937 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/add.v2i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/add.v2i16.ll
@@ -91,7 +91,7 @@ define <2 x i16> @v_add_v2i16_neg_inline_imm_splat(<2 x i16> %a) {
 ; GFX9-LABEL: v_add_v2i16_neg_inline_imm_splat:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    s_mov_b32 s4, 0xffffffc0
+; GFX9-NEXT:    s_movk_i32 s4, 0xffc0
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s4
 ; GFX9-NEXT:    v_pk_add_u16 v0, v0, s4
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -99,7 +99,7 @@ define <2 x i16> @v_add_v2i16_neg_inline_imm_splat(<2 x i16> %a) {
 ; GFX8-LABEL: v_add_v2i16_neg_inline_imm_splat:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    s_mov_b32 s4, 0xffc0
+; GFX8-NEXT:    s_movk_i32 s4, 0xffc0
 ; GFX8-NEXT:    v_mov_b32_e32 v2, s4
 ; GFX8-NEXT:    v_add_u16_e32 v1, s4, v0
 ; GFX8-NEXT:    v_add_u16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
@@ -140,7 +140,7 @@ define <2 x i16> @v_add_v2i16_neg_inline_imm_hi(<2 x i16> %a) {
 ; GFX8-LABEL: v_add_v2i16_neg_inline_imm_hi:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v1, 0xffc0
+; GFX8-NEXT:    v_mov_b32_e32 v1, 0xffffffc0
 ; GFX8-NEXT:    v_add_u16_e32 v2, 4, v0
 ; GFX8-NEXT:    v_add_u16_sdwa v0, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
 ; GFX8-NEXT:    v_or_b32_e32 v0, v2, v0
@@ -152,7 +152,7 @@ define <2 x i16> @v_add_v2i16_neg_inline_imm_hi(<2 x i16> %a) {
 define amdgpu_ps i32 @s_add_v2i16_neg_inline_imm_splat(<2 x i16> inreg %a) {
 ; GFX9-LABEL: s_add_v2i16_neg_inline_imm_splat:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_mov_b32 s1, 0xffffffc0
+; GFX9-NEXT:    s_movk_i32 s1, 0xffc0
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s1
 ; GFX9-NEXT:    s_lshr_b32 s2, s0, 16
 ; GFX9-NEXT:    s_lshr_b32 s3, s1, 16
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergent-control-flow.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergent-control-flow.ll
index 2695952bfd193..1f9c3bc60876e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergent-control-flow.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergent-control-flow.ll
@@ -135,24 +135,24 @@ define void @constrained_if_register_class() {
 ; CHECK-NEXT:    s_load_dword s4, s[4:5], 0x0
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_cmp_lg_u32 s4, 0
-; CHECK-NEXT:    s_cselect_b32 s4, 1, 0
-; CHECK-NEXT:    s_xor_b32 s4, s4, 1
-; CHECK-NEXT:    s_and_b32 s4, s4, 1
-; CHECK-NEXT:    s_cmp_lg_u32 s4, 0
+; CHECK-NEXT:    s_cselect_b32 s5, 1, 0
+; CHECK-NEXT:    s_xor_b32 s5, s5, -1
+; CHECK-NEXT:    s_and_b32 s5, s5, 1
+; CHECK-NEXT:    s_mov_b32 s4, -1
+; CHECK-NEXT:    s_cmp_lg_u32 s5, 0
 ; CHECK-NEXT:    s_cbranch_scc0 BB4_6
 ; CHECK-NEXT:  ; %bb.1: ; %bb2
-; CHECK-NEXT:    s_getpc_b64 s[4:5]
-; CHECK-NEXT:    s_add_u32 s4, s4, const.ptr@gotpcrel32@lo+4
-; CHECK-NEXT:    s_addc_u32 s5, s5, const.ptr@gotpcrel32@hi+4
-; CHECK-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
-; CHECK-NEXT:    v_cmp_ne_u32_e64 s[6:7], 0, 1
+; CHECK-NEXT:    s_getpc_b64 s[6:7]
+; CHECK-NEXT:    s_add_u32 s6, s6, const.ptr@gotpcrel32@lo+4
+; CHECK-NEXT:    s_addc_u32 s7, s7, const.ptr@gotpcrel32@hi+4
+; CHECK-NEXT:    s_load_dwordx2 s[6:7], s[6:7], 0x0
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
+; CHECK-NEXT:    s_load_dwordx2 s[6:7], s[6:7], 0x0
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    v_mov_b32_e32 v0, s4
-; CHECK-NEXT:    v_mov_b32_e32 v1, s5
+; CHECK-NEXT:    v_mov_b32_e32 v0, s6
+; CHECK-NEXT:    v_mov_b32_e32 v1, s7
 ; CHECK-NEXT:    flat_load_dword v0, v[0:1]
-; CHECK-NEXT:    s_mov_b32 s4, -1
+; CHECK-NEXT:    v_cmp_ne_u32_e64 s[6:7], 0, 1
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; CHECK-NEXT:    v_cmp_gt_f32_e32 vcc, 1.0, v0
 ; CHECK-NEXT:    s_xor_b64 s[8:9], vcc, s[6:7]
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-add.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-add.mir
index 51a116a944ad6..baed490c07581 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-add.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-add.mir
@@ -94,7 +94,7 @@ body: |
     ; GFX6-LABEL: name: add_neg_inline_const_64_to_sub_s32_v
     ; GFX6: liveins: $vgpr0
     ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; GFX6: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4294967232, implicit $exec
+    ; GFX6: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -64, implicit $exec
     ; GFX6: %2:vgpr_32, dead %3:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
     ; GFX6: S_ENDPGM 0, implicit %2
     ; GFX9-LABEL: name: add_neg_inline_const_64_to_sub_s32_v
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-and.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-and.mir
index ecfb9b618f5ed..81437acbbbc53 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-and.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-and.mir
@@ -20,7 +20,7 @@ body: |
     ; WAVE64: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
     ; WAVE64: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_EQ_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], implicit $exec
     ; WAVE64: [[V_CMP_EQ_U32_e64_1:%[0-9]+]]:sreg_64 = V_CMP_EQ_U32_e64 [[COPY1]], [[V_MOV_B32_e32_]], implicit $exec
-    ; WAVE64: [[S_AND_B64_:%[0-9]+]]:sreg_64 = S_AND_B64 [[V_CMP_EQ_U32_e64_]], [[V_CMP_EQ_U32_e64_1]]
+    ; WAVE64: [[S_AND_B64_:%[0-9]+]]:sreg_64 = S_AND_B64 [[V_CMP_EQ_U32_e64_]], [[V_CMP_EQ_U32_e64_1]], implicit-def dead $scc
     ; WAVE64: S_ENDPGM 0, implicit [[S_AND_B64_]]
     ; WAVE32-LABEL: name: and_s1_vcc_vcc_vcc
     ; WAVE32: liveins: $vgpr0, $vgpr1
@@ -30,7 +30,7 @@ body: |
     ; WAVE32: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
     ; WAVE32: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32 = V_CMP_EQ_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], implicit $exec
     ; WAVE32: [[V_CMP_EQ_U32_e64_1:%[0-9]+]]:sreg_32 = V_CMP_EQ_U32_e64 [[COPY1]], [[V_MOV_B32_e32_]], implicit $exec
-    ; WAVE32: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 [[V_CMP_EQ_U32_e64_]], [[V_CMP_EQ_U32_e64_1]]
+    ; WAVE32: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 [[V_CMP_EQ_U32_e64_]], [[V_CMP_EQ_U32_e64_1]], implicit-def dead $scc
     ; WAVE32: S_ENDPGM 0, implicit [[S_AND_B32_]]
     %0:vgpr(s32) = COPY $vgpr0
     %1:vgpr(s32) = COPY $vgpr1
@@ -386,7 +386,7 @@ body: |
     ; WAVE64: [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_NE_U32_e64 0, [[V_AND_B32_e32_]], implicit $exec
     ; WAVE64: [[V_AND_B32_e32_1:%[0-9]+]]:vgpr_32 = V_AND_B32_e32 1, [[COPY1]], implicit $exec
     ; WAVE64: [[V_CMP_NE_U32_e64_1:%[0-9]+]]:sreg_64 = V_CMP_NE_U32_e64 0, [[V_AND_B32_e32_1]], implicit $exec
-    ; WAVE64: [[S_AND_B64_:%[0-9]+]]:sreg_64 = S_AND_B64 [[V_CMP_NE_U32_e64_]], [[V_CMP_NE_U32_e64_1]]
+    ; WAVE64: [[S_AND_B64_:%[0-9]+]]:sreg_64 = S_AND_B64 [[V_CMP_NE_U32_e64_]], [[V_CMP_NE_U32_e64_1]], implicit-def dead $scc
     ; WAVE64: S_ENDPGM 0, implicit [[S_AND_B64_]]
     ; WAVE32-LABEL: name: and_s1_vcc_copy_to_vcc
     ; WAVE32: liveins: $vgpr0, $vgpr1
@@ -397,7 +397,7 @@ body: |
     ; WAVE32: [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_32 = V_CMP_NE_U32_e64 0, [[V_AND_B32_e32_]], implicit $exec
     ; WAVE32: [[V_AND_B32_e32_1:%[0-9]+]]:vgpr_32 = V_AND_B32_e32 1, [[COPY1]], implicit $exec
     ; WAVE32: [[V_CMP_NE_U32_e64_1:%[0-9]+]]:sreg_32 = V_CMP_NE_U32_e64 0, [[V_AND_B32_e32_1]], implicit $exec
-    ; WAVE32: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 [[V_CMP_NE_U32_e64_]], [[V_CMP_NE_U32_e64_1]]
+    ; WAVE32: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 [[V_CMP_NE_U32_e64_]], [[V_CMP_NE_U32_e64_1]], implicit-def dead $scc
     ; WAVE32: S_ENDPGM 0, implicit [[S_AND_B32_]]
     %0:vgpr(s32) = COPY $vgpr0
     %1:vgpr(s32) = COPY $vgpr1
@@ -425,24 +425,24 @@ body:             |
     ; WAVE64-LABEL: name: copy_select_constrain_vcc_result_reg_wave32
     ; WAVE64: liveins: $vgpr0
     ; WAVE64: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; WAVE64: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 1
+    ; WAVE64: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
     ; WAVE64: [[V_AND_B32_e32_:%[0-9]+]]:vgpr_32 = V_AND_B32_e32 1, [[COPY]], implicit $exec
     ; WAVE64: [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_NE_U32_e64 0, [[V_AND_B32_e32_]], implicit $exec
     ; WAVE64: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 1, [[S_MOV_B32_]], implicit-def $scc
     ; WAVE64: [[V_CMP_NE_U32_e64_1:%[0-9]+]]:sreg_64 = V_CMP_NE_U32_e64 0, [[S_AND_B32_]], implicit $exec
-    ; WAVE64: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_NE_U32_e64_]], [[V_CMP_NE_U32_e64_1]]
+    ; WAVE64: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_NE_U32_e64_]], [[V_CMP_NE_U32_e64_1]], implicit-def dead $scc
     ; WAVE64: [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY [[S_AND_B64_]]
     ; WAVE64: S_ENDPGM 0, implicit [[COPY1]]
     ; WAVE32-LABEL: name: copy_select_constrain_vcc_result_reg_wave32
     ; WAVE32: liveins: $vgpr0
     ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; WAVE32: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 1
+    ; WAVE32: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
     ; WAVE32: [[V_AND_B32_e32_:%[0-9]+]]:vgpr_32 = V_AND_B32_e32 1, [[COPY]], implicit $exec
     ; WAVE32: [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_32 = V_CMP_NE_U32_e64 0, [[V_AND_B32_e32_]], implicit $exec
     ; WAVE32: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 1, [[S_MOV_B32_]], implicit-def $scc
     ; WAVE32: [[V_CMP_NE_U32_e64_1:%[0-9]+]]:sreg_32 = V_CMP_NE_U32_e64 0, [[S_AND_B32_]], implicit $exec
-    ; WAVE32: [[S_AND_B32_1:%[0-9]+]]:sreg_32 = S_AND_B32 [[V_CMP_NE_U32_e64_]], [[V_CMP_NE_U32_e64_1]]
+    ; WAVE32: [[S_AND_B32_1:%[0-9]+]]:sreg_32 = S_AND_B32 [[V_CMP_NE_U32_e64_]], [[V_CMP_NE_U32_e64_1]], implicit-def dead $scc
     ; WAVE32: [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY [[S_AND_B32_1]]
     ; WAVE32: S_ENDPGM 0, implicit [[COPY1]]
     %1:vgpr(s32) = COPY $vgpr0
@@ -471,24 +471,24 @@ body:             |
     ; WAVE64-LABEL: name: copy_select_constrain_vcc_result_reg_wave64
     ; WAVE64: liveins: $vgpr0
     ; WAVE64: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; WAVE64: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 1
+    ; WAVE64: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
     ; WAVE64: [[V_AND_B32_e32_:%[0-9]+]]:vgpr_32 = V_AND_B32_e32 1, [[COPY]], implicit $exec
     ; WAVE64: [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_NE_U32_e64 0, [[V_AND_B32_e32_]], implicit $exec
     ; WAVE64: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 1, [[S_MOV_B32_]], implicit-def $scc
     ; WAVE64: [[V_CMP_NE_U32_e64_1:%[0-9]+]]:sreg_64 = V_CMP_NE_U32_e64 0, [[S_AND_B32_]], implicit $exec
-    ; WAVE64: [[S_AND_B64_:%[0-9]+]]:sreg_64 = S_AND_B64 [[V_CMP_NE_U32_e64_]], [[V_CMP_NE_U32_e64_1]]
+    ; WAVE64: [[S_AND_B64_:%[0-9]+]]:sreg_64 = S_AND_B64 [[V_CMP_NE_U32_e64_]], [[V_CMP_NE_U32_e64_1]], implicit-def dead $scc
     ; WAVE64: [[COPY1:%[0-9]+]]:sreg_64_xexec = COPY [[S_AND_B64_]]
     ; WAVE64: S_ENDPGM 0, implicit [[COPY1]]
     ; WAVE32-LABEL: name: copy_select_constrain_vcc_result_reg_wave64
     ; WAVE32: liveins: $vgpr0
     ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; WAVE32: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 1
+    ; WAVE32: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
     ; WAVE32: [[V_AND_B32_e32_:%[0-9]+]]:vgpr_32 = V_AND_B32_e32 1, [[COPY]], implicit $exec
     ; WAVE32: [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_32 = V_CMP_NE_U32_e64 0, [[V_AND_B32_e32_]], implicit $exec
     ; WAVE32: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 1, [[S_MOV_B32_]], implicit-def $scc
     ; WAVE32: [[V_CMP_NE_U32_e64_1:%[0-9]+]]:sreg_32 = V_CMP_NE_U32_e64 0, [[S_AND_B32_]], implicit $exec
-    ; WAVE32: [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_NE_U32_e64_]], [[V_CMP_NE_U32_e64_1]]
+    ; WAVE32: [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_NE_U32_e64_]], [[V_CMP_NE_U32_e64_1]], implicit-def dead $scc
     ; WAVE32: [[COPY1:%[0-9]+]]:sreg_64_xexec = COPY [[S_AND_B32_1]]
     ; WAVE32: S_ENDPGM 0, implicit [[COPY1]]
     %1:vgpr(s32) = COPY $vgpr0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-constant.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-constant.mir
index 2f2c305cfac79..c8762c0d578eb 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-constant.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-constant.mir
@@ -12,8 +12,8 @@ body: |
     ; GCN-LABEL: name: constant_v_s32
     ; GCN: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
     ; GCN: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
-    ; GCN: [[V_MOV_B32_e32_2:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4294967295, implicit $exec
-    ; GCN: [[V_MOV_B32_e32_3:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4294967242, implicit $exec
+    ; GCN: [[V_MOV_B32_e32_2:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -1, implicit $exec
+    ; GCN: [[V_MOV_B32_e32_3:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -54, implicit $exec
     ; GCN: [[V_MOV_B32_e32_4:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 27, implicit $exec
     ; GCN: S_ENDPGM 0, implicit [[V_MOV_B32_e32_]], implicit [[V_MOV_B32_e32_1]], implicit [[V_MOV_B32_e32_2]], implicit [[V_MOV_B32_e32_3]], implicit [[V_MOV_B32_e32_4]]
     %0:vgpr(s32) = G_CONSTANT i32 0
@@ -35,8 +35,8 @@ body: |
     ; GCN-LABEL: name: constant_s_s32
     ; GCN: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
     ; GCN: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 1
-    ; GCN: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 4294967295
-    ; GCN: [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 4294967242
+    ; GCN: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
+    ; GCN: [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 -54
     ; GCN: [[S_MOV_B32_4:%[0-9]+]]:sreg_32 = S_MOV_B32 27
     ; GCN: S_ENDPGM 0, implicit [[S_MOV_B32_]], implicit [[S_MOV_B32_1]], implicit [[S_MOV_B32_2]], implicit [[S_MOV_B32_3]], implicit [[S_MOV_B32_4]]
     %0:sgpr(s32) = G_CONSTANT i32 0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-extract-vector-elt.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-extract-vector-elt.mir
index da730b0c9fa9a..10e4cbdc14676 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-extract-vector-elt.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-extract-vector-elt.mir
@@ -315,7 +315,7 @@ body: |
     ; MOVREL-LABEL: name: extract_vector_elt_s_s32_v8s32_idx_offset_m1
     ; MOVREL: [[COPY:%[0-9]+]]:sgpr_256 = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     ; MOVREL: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr8
-    ; MOVREL: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4294967295
+    ; MOVREL: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
     ; MOVREL: [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[COPY1]], [[S_MOV_B32_]], implicit-def $scc
     ; MOVREL: $m0 = COPY [[S_ADD_I32_]]
     ; MOVREL: [[S_MOVRELS_B32_:%[0-9]+]]:sreg_32 = S_MOVRELS_B32 [[COPY]].sub0, implicit $m0, implicit [[COPY]]
@@ -323,7 +323,7 @@ body: |
     ; GPRIDX-LABEL: name: extract_vector_elt_s_s32_v8s32_idx_offset_m1
     ; GPRIDX: [[COPY:%[0-9]+]]:sgpr_256 = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     ; GPRIDX: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr8
-    ; GPRIDX: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4294967295
+    ; GPRIDX: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
     ; GPRIDX: [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[COPY1]], [[S_MOV_B32_]], implicit-def $scc
     ; GPRIDX: $m0 = COPY [[S_ADD_I32_]]
     ; GPRIDX: [[S_MOVRELS_B32_:%[0-9]+]]:sreg_32 = S_MOVRELS_B32 [[COPY]].sub0, implicit $m0, implicit [[COPY]]
@@ -468,7 +468,7 @@ body: |
     ; MOVREL-LABEL: name: extract_vector_elt_s_s64_v8s64_idx_offset_m1
     ; MOVREL: [[COPY:%[0-9]+]]:sgpr_512 = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
     ; MOVREL: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr8
-    ; MOVREL: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4294967295
+    ; MOVREL: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
     ; MOVREL: [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[COPY1]], [[S_MOV_B32_]], implicit-def $scc
     ; MOVREL: $m0 = COPY [[S_ADD_I32_]]
     ; MOVREL: [[S_MOVRELS_B64_:%[0-9]+]]:sreg_64 = S_MOVRELS_B64 [[COPY]].sub0_sub1, implicit $m0, implicit [[COPY]]
@@ -476,7 +476,7 @@ body: |
     ; GPRIDX-LABEL: name: extract_vector_elt_s_s64_v8s64_idx_offset_m1
     ; GPRIDX: [[COPY:%[0-9]+]]:sgpr_512 = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
     ; GPRIDX: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr8
-    ; GPRIDX: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4294967295
+    ; GPRIDX: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
     ; GPRIDX: [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[COPY1]], [[S_MOV_B32_]], implicit-def $scc
     ; GPRIDX: $m0 = COPY [[S_ADD_I32_]]
     ; GPRIDX: [[S_MOVRELS_B64_:%[0-9]+]]:sreg_64 = S_MOVRELS_B64 [[COPY]].sub0_sub1, implicit $m0, implicit [[COPY]]
@@ -699,7 +699,7 @@ body: |
     ; MOVREL-LABEL: name: extract_vector_elt_v_s32_v8s32_idx_offset_m1
     ; MOVREL: [[COPY:%[0-9]+]]:vreg_256 = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
     ; MOVREL: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr8
-    ; MOVREL: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4294967295
+    ; MOVREL: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
     ; MOVREL: [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[COPY1]], [[S_MOV_B32_]], implicit-def $scc
     ; MOVREL: $m0 = COPY [[S_ADD_I32_]]
     ; MOVREL: [[V_MOVRELS_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOVRELS_B32_e32 undef [[COPY]].sub0, implicit $m0, implicit $exec, implicit [[COPY]]
@@ -707,7 +707,7 @@ body: |
     ; GPRIDX-LABEL: name: extract_vector_elt_v_s32_v8s32_idx_offset_m1
     ; GPRIDX: [[COPY:%[0-9]+]]:vreg_256 = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
     ; GPRIDX: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr8
-    ; GPRIDX: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4294967295
+    ; GPRIDX: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
     ; GPRIDX: [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[COPY1]], [[S_MOV_B32_]], implicit-def $scc
     ; GPRIDX: S_SET_GPR_IDX_ON [[S_ADD_I32_]], 1, implicit-def $m0, implicit-def $mode, implicit $m0, implicit $mode
     ; GPRIDX: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 undef [[COPY]].sub0, implicit $exec, implicit [[COPY]], implicit $m0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-local.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-local.mir
index a80ad208b5898..45d74ad38e532 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-local.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-local.mir
@@ -681,7 +681,7 @@ body: |
     ; GFX7-LABEL: name: load_local_s32_from_1_gep_m1
     ; GFX7: liveins: $vgpr0
     ; GFX7: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; GFX7: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4294967295, implicit $exec
+    ; GFX7: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -1, implicit $exec
     ; GFX7: %2:vgpr_32, dead %4:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
     ; GFX7: $m0 = S_MOV_B32 -1
     ; GFX7: [[DS_READ_U8_:%[0-9]+]]:vgpr_32 = DS_READ_U8 %2, 0, 0, implicit $m0, implicit $exec :: (load 1, addrspace 3)
@@ -689,14 +689,14 @@ body: |
     ; GFX9-LABEL: name: load_local_s32_from_1_gep_m1
     ; GFX9: liveins: $vgpr0
     ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; GFX9: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4294967295, implicit $exec
+    ; GFX9: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -1, implicit $exec
     ; GFX9: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
     ; GFX9: [[DS_READ_U8_gfx9_:%[0-9]+]]:vgpr_32 = DS_READ_U8_gfx9 [[V_ADD_U32_e64_]], 0, 0, implicit $exec :: (load 1, addrspace 3)
     ; GFX9: $vgpr0 = COPY [[DS_READ_U8_gfx9_]]
     ; GFX6-LABEL: name: load_local_s32_from_1_gep_m1
     ; GFX6: liveins: $vgpr0
     ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; GFX6: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4294967295, implicit $exec
+    ; GFX6: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -1, implicit $exec
     ; GFX6: %2:vgpr_32, dead %4:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
     ; GFX6: $m0 = S_MOV_B32 -1
     ; GFX6: [[DS_READ_U8_:%[0-9]+]]:vgpr_32 = DS_READ_U8 %2, 0, 0, implicit $m0, implicit $exec :: (load 1, addrspace 3)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-private.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-private.mir
index 162dd01de66d1..9f6d10722143d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-private.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-private.mir
@@ -311,14 +311,14 @@ body: |
     ; GFX6-LABEL: name: load_private_s32_from_1_gep_m2047
     ; GFX6: liveins: $vgpr0
     ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; GFX6: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4294965249, implicit $exec
+    ; GFX6: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -2047, implicit $exec
     ; GFX6: %2:vgpr_32, dead %4:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
     ; GFX6: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN %2, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
     ; GFX6: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
     ; GFX9-LABEL: name: load_private_s32_from_1_gep_m2047
     ; GFX9: liveins: $vgpr0
     ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; GFX9: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4294965249, implicit $exec
+    ; GFX9: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -2047, implicit $exec
     ; GFX9: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
     ; GFX9: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[V_ADD_U32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
     ; GFX9: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
@@ -347,14 +347,14 @@ body: |
     ; GFX6-LABEL: name: load_private_s32_from_1_gep_m2048
     ; GFX6: liveins: $vgpr0
     ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; GFX6: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4294965248, implicit $exec
+    ; GFX6: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -2048, implicit $exec
     ; GFX6: %2:vgpr_32, dead %4:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
     ; GFX6: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN %2, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
     ; GFX6: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
     ; GFX9-LABEL: name: load_private_s32_from_1_gep_m2048
     ; GFX9: liveins: $vgpr0
     ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; GFX9: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4294965248, implicit $exec
+    ; GFX9: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -2048, implicit $exec
     ; GFX9: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
     ; GFX9: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[V_ADD_U32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
     ; GFX9: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
@@ -453,14 +453,14 @@ body: |
     ; GFX6-LABEL: name: load_private_s32_from_1_gep_m4095
     ; GFX6: liveins: $vgpr0
     ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; GFX6: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4294963201, implicit $exec
+    ; GFX6: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -4095, implicit $exec
     ; GFX6: %2:vgpr_32, dead %4:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
     ; GFX6: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN %2, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
     ; GFX6: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
     ; GFX9-LABEL: name: load_private_s32_from_1_gep_m4095
     ; GFX9: liveins: $vgpr0
     ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; GFX9: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4294963201, implicit $exec
+    ; GFX9: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -4095, implicit $exec
     ; GFX9: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
     ; GFX9: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[V_ADD_U32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
     ; GFX9: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
@@ -489,14 +489,14 @@ body: |
     ; GFX6-LABEL: name: load_private_s32_from_1_gep_m4096
     ; GFX6: liveins: $vgpr0
     ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; GFX6: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4294963200, implicit $exec
+    ; GFX6: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -4096, implicit $exec
     ; GFX6: %2:vgpr_32, dead %4:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
     ; GFX6: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN %2, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
     ; GFX6: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
     ; GFX9-LABEL: name: load_private_s32_from_1_gep_m4096
     ; GFX9: liveins: $vgpr0
     ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; GFX9: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4294963200, implicit $exec
+    ; GFX9: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -4096, implicit $exec
     ; GFX9: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
     ; GFX9: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[V_ADD_U32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
     ; GFX9: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
@@ -597,14 +597,14 @@ body: |
     ; GFX6-LABEL: name: load_private_s32_from_1_gep_m8191
     ; GFX6: liveins: $vgpr0
     ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; GFX6: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4294959105, implicit $exec
+    ; GFX6: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -8191, implicit $exec
     ; GFX6: %2:vgpr_32, dead %4:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
     ; GFX6: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN %2, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
     ; GFX6: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
     ; GFX9-LABEL: name: load_private_s32_from_1_gep_m8191
     ; GFX9: liveins: $vgpr0
     ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; GFX9: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4294959105, implicit $exec
+    ; GFX9: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -8191, implicit $exec
     ; GFX9: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
     ; GFX9: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[V_ADD_U32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
     ; GFX9: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
@@ -633,14 +633,14 @@ body: |
     ; GFX6-LABEL: name: load_private_s32_from_1_gep_m8192
     ; GFX6: liveins: $vgpr0
     ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; GFX6: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4294959104, implicit $exec
+    ; GFX6: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -8192, implicit $exec
     ; GFX6: %2:vgpr_32, dead %4:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
     ; GFX6: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN %2, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
     ; GFX6: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
     ; GFX9-LABEL: name: load_private_s32_from_1_gep_m8192
     ; GFX9: liveins: $vgpr0
     ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; GFX9: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4294959104, implicit $exec
+    ; GFX9: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -8192, implicit $exec
     ; GFX9: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
     ; GFX9: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[V_ADD_U32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
     ; GFX9: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
@@ -860,11 +860,11 @@ body: |
   bb.0:
 
     ; GFX6-LABEL: name: load_private_s32_from_neg1
-    ; GFX6: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4294967295, implicit $exec
+    ; GFX6: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -1, implicit $exec
     ; GFX6: [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[V_MOV_B32_e32_]], $sgpr0_sgpr1_sgpr2_sgpr3, 0, -1, 0, 0, 0, 0, 0, implicit $exec :: (load 4, addrspace 5)
     ; GFX6: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
     ; GFX9-LABEL: name: load_private_s32_from_neg1
-    ; GFX9: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4294967295, implicit $exec
+    ; GFX9: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -1, implicit $exec
     ; GFX9: [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[V_MOV_B32_e32_]], $sgpr0_sgpr1_sgpr2_sgpr3, 0, -1, 0, 0, 0, 0, 0, implicit $exec :: (load 4, addrspace 5)
     ; GFX9: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
     %0:vgpr(p5) = G_CONSTANT i32 -1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-or.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-or.mir
index 40b7b69f83b7f..7f1f52d2c522a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-or.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-or.mir
@@ -20,7 +20,7 @@ body: |
     ; WAVE64: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
     ; WAVE64: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_EQ_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], implicit $exec
     ; WAVE64: [[V_CMP_EQ_U32_e64_1:%[0-9]+]]:sreg_64 = V_CMP_EQ_U32_e64 [[COPY1]], [[V_MOV_B32_e32_]], implicit $exec
-    ; WAVE64: [[S_OR_B64_:%[0-9]+]]:sreg_64 = S_OR_B64 [[V_CMP_EQ_U32_e64_]], [[V_CMP_EQ_U32_e64_1]]
+    ; WAVE64: [[S_OR_B64_:%[0-9]+]]:sreg_64 = S_OR_B64 [[V_CMP_EQ_U32_e64_]], [[V_CMP_EQ_U32_e64_1]], implicit-def dead $scc
     ; WAVE64: S_ENDPGM 0, implicit [[S_OR_B64_]]
     ; WAVE32-LABEL: name: or_s1_vcc_vcc_vcc
     ; WAVE32: liveins: $vgpr0, $vgpr1
@@ -30,7 +30,7 @@ body: |
     ; WAVE32: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
     ; WAVE32: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32 = V_CMP_EQ_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], implicit $exec
     ; WAVE32: [[V_CMP_EQ_U32_e64_1:%[0-9]+]]:sreg_32 = V_CMP_EQ_U32_e64 [[COPY1]], [[V_MOV_B32_e32_]], implicit $exec
-    ; WAVE32: [[S_OR_B32_:%[0-9]+]]:sreg_32 = S_OR_B32 [[V_CMP_EQ_U32_e64_]], [[V_CMP_EQ_U32_e64_1]]
+    ; WAVE32: [[S_OR_B32_:%[0-9]+]]:sreg_32 = S_OR_B32 [[V_CMP_EQ_U32_e64_]], [[V_CMP_EQ_U32_e64_1]], implicit-def dead $scc
     ; WAVE32: S_ENDPGM 0, implicit [[S_OR_B32_]]
     %0:vgpr(s32) = COPY $vgpr0
     %1:vgpr(s32) = COPY $vgpr1
@@ -386,7 +386,7 @@ body: |
     ; WAVE64: [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_NE_U32_e64 0, [[V_AND_B32_e32_]], implicit $exec
     ; WAVE64: [[V_AND_B32_e32_1:%[0-9]+]]:vgpr_32 = V_AND_B32_e32 1, [[COPY1]], implicit $exec
     ; WAVE64: [[V_CMP_NE_U32_e64_1:%[0-9]+]]:sreg_64 = V_CMP_NE_U32_e64 0, [[V_AND_B32_e32_1]], implicit $exec
-    ; WAVE64: [[S_OR_B64_:%[0-9]+]]:sreg_64 = S_OR_B64 [[V_CMP_NE_U32_e64_]], [[V_CMP_NE_U32_e64_1]]
+    ; WAVE64: [[S_OR_B64_:%[0-9]+]]:sreg_64 = S_OR_B64 [[V_CMP_NE_U32_e64_]], [[V_CMP_NE_U32_e64_1]], implicit-def dead $scc
     ; WAVE64: S_ENDPGM 0, implicit [[S_OR_B64_]]
     ; WAVE32-LABEL: name: or_s1_vcc_copy_to_vcc
     ; WAVE32: liveins: $vgpr0, $vgpr1
@@ -397,7 +397,7 @@ body: |
     ; WAVE32: [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_32 = V_CMP_NE_U32_e64 0, [[V_AND_B32_e32_]], implicit $exec
     ; WAVE32: [[V_AND_B32_e32_1:%[0-9]+]]:vgpr_32 = V_AND_B32_e32 1, [[COPY1]], implicit $exec
     ; WAVE32: [[V_CMP_NE_U32_e64_1:%[0-9]+]]:sreg_32 = V_CMP_NE_U32_e64 0, [[V_AND_B32_e32_1]], implicit $exec
-    ; WAVE32: [[S_OR_B32_:%[0-9]+]]:sreg_32 = S_OR_B32 [[V_CMP_NE_U32_e64_]], [[V_CMP_NE_U32_e64_1]]
+    ; WAVE32: [[S_OR_B32_:%[0-9]+]]:sreg_32 = S_OR_B32 [[V_CMP_NE_U32_e64_]], [[V_CMP_NE_U32_e64_1]], implicit-def dead $scc
     ; WAVE32: S_ENDPGM 0, implicit [[S_OR_B32_]]
     %0:vgpr(s32) = COPY $vgpr0
     %1:vgpr(s32) = COPY $vgpr1
@@ -425,24 +425,24 @@ body:             |
     ; WAVE64-LABEL: name: copy_select_constrain_vcc_result_reg_wave32
     ; WAVE64: liveins: $vgpr0
     ; WAVE64: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; WAVE64: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 1
+    ; WAVE64: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
     ; WAVE64: [[V_AND_B32_e32_:%[0-9]+]]:vgpr_32 = V_AND_B32_e32 1, [[COPY]], implicit $exec
     ; WAVE64: [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_NE_U32_e64 0, [[V_AND_B32_e32_]], implicit $exec
     ; WAVE64: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 1, [[S_MOV_B32_]], implicit-def $scc
     ; WAVE64: [[V_CMP_NE_U32_e64_1:%[0-9]+]]:sreg_64 = V_CMP_NE_U32_e64 0, [[S_AND_B32_]], implicit $exec
-    ; WAVE64: [[S_OR_B64_:%[0-9]+]]:sreg_64_xexec = S_OR_B64 [[V_CMP_NE_U32_e64_]], [[V_CMP_NE_U32_e64_1]]
+    ; WAVE64: [[S_OR_B64_:%[0-9]+]]:sreg_64_xexec = S_OR_B64 [[V_CMP_NE_U32_e64_]], [[V_CMP_NE_U32_e64_1]], implicit-def dead $scc
     ; WAVE64: [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY [[S_OR_B64_]]
     ; WAVE64: S_ENDPGM 0, implicit [[COPY1]]
     ; WAVE32-LABEL: name: copy_select_constrain_vcc_result_reg_wave32
     ; WAVE32: liveins: $vgpr0
     ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; WAVE32: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 1
+    ; WAVE32: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
     ; WAVE32: [[V_AND_B32_e32_:%[0-9]+]]:vgpr_32 = V_AND_B32_e32 1, [[COPY]], implicit $exec
     ; WAVE32: [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_32 = V_CMP_NE_U32_e64 0, [[V_AND_B32_e32_]], implicit $exec
     ; WAVE32: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 1, [[S_MOV_B32_]], implicit-def $scc
     ; WAVE32: [[V_CMP_NE_U32_e64_1:%[0-9]+]]:sreg_32 = V_CMP_NE_U32_e64 0, [[S_AND_B32_]], implicit $exec
-    ; WAVE32: [[S_OR_B32_:%[0-9]+]]:sreg_32 = S_OR_B32 [[V_CMP_NE_U32_e64_]], [[V_CMP_NE_U32_e64_1]]
+    ; WAVE32: [[S_OR_B32_:%[0-9]+]]:sreg_32 = S_OR_B32 [[V_CMP_NE_U32_e64_]], [[V_CMP_NE_U32_e64_1]], implicit-def dead $scc
     ; WAVE32: [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY [[S_OR_B32_]]
     ; WAVE32: S_ENDPGM 0, implicit [[COPY1]]
     %1:vgpr(s32) = COPY $vgpr0
@@ -471,24 +471,24 @@ body:             |
     ; WAVE64-LABEL: name: copy_select_constrain_vcc_result_reg_wave64
     ; WAVE64: liveins: $vgpr0
     ; WAVE64: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; WAVE64: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 1
+    ; WAVE64: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
     ; WAVE64: [[V_AND_B32_e32_:%[0-9]+]]:vgpr_32 = V_AND_B32_e32 1, [[COPY]], implicit $exec
     ; WAVE64: [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_NE_U32_e64 0, [[V_AND_B32_e32_]], implicit $exec
     ; WAVE64: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 1, [[S_MOV_B32_]], implicit-def $scc
     ; WAVE64: [[V_CMP_NE_U32_e64_1:%[0-9]+]]:sreg_64 = V_CMP_NE_U32_e64 0, [[S_AND_B32_]], implicit $exec
-    ; WAVE64: [[S_OR_B64_:%[0-9]+]]:sreg_64 = S_OR_B64 [[V_CMP_NE_U32_e64_]], [[V_CMP_NE_U32_e64_1]]
+    ; WAVE64: [[S_OR_B64_:%[0-9]+]]:sreg_64 = S_OR_B64 [[V_CMP_NE_U32_e64_]], [[V_CMP_NE_U32_e64_1]], implicit-def dead $scc
     ; WAVE64: [[COPY1:%[0-9]+]]:sreg_64_xexec = COPY [[S_OR_B64_]]
     ; WAVE64: S_ENDPGM 0, implicit [[COPY1]]
     ; WAVE32-LABEL: name: copy_select_constrain_vcc_result_reg_wave64
     ; WAVE32: liveins: $vgpr0
     ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; WAVE32: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 1
+    ; WAVE32: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
     ; WAVE32: [[V_AND_B32_e32_:%[0-9]+]]:vgpr_32 = V_AND_B32_e32 1, [[COPY]], implicit $exec
     ; WAVE32: [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_32 = V_CMP_NE_U32_e64 0, [[V_AND_B32_e32_]], implicit $exec
     ; WAVE32: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 1, [[S_MOV_B32_]], implicit-def $scc
     ; WAVE32: [[V_CMP_NE_U32_e64_1:%[0-9]+]]:sreg_32 = V_CMP_NE_U32_e64 0, [[S_AND_B32_]], implicit $exec
-    ; WAVE32: [[S_OR_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_OR_B32 [[V_CMP_NE_U32_e64_]], [[V_CMP_NE_U32_e64_1]]
+    ; WAVE32: [[S_OR_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_OR_B32 [[V_CMP_NE_U32_e64_]], [[V_CMP_NE_U32_e64_1]], implicit-def dead $scc
     ; WAVE32: [[COPY1:%[0-9]+]]:sreg_64_xexec = COPY [[S_OR_B32_]]
     ; WAVE32: S_ENDPGM 0, implicit [[COPY1]]
     %1:vgpr(s32) = COPY $vgpr0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-ptrmask.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-ptrmask.mir
index bbb885c705ed6..a7f875fcdd428 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-ptrmask.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-ptrmask.mir
@@ -33,7 +33,7 @@ body: |
 
     ; CHECK-LABEL: name: ptrmask_p3_s32_sgpr_sgpr_0xf0f0f0f0
     ; CHECK: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
-    ; CHECK: %const:sreg_32 = S_MOV_B32 4042322160
+    ; CHECK: %const:sreg_32 = S_MOV_B32 -252645136
     ; CHECK: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 [[COPY]], %const, implicit-def $scc
     ; CHECK: S_ENDPGM 0, implicit [[S_AND_B32_]]
     %0:sgpr(p3) = COPY $sgpr0
@@ -54,7 +54,7 @@ body: |
 
     ; CHECK-LABEL: name: ptrmask_p3_s32_sgpr_sgpr_0xffffffff
     ; CHECK: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
-    ; CHECK: %const:sreg_32 = S_MOV_B32 4294967295
+    ; CHECK: %const:sreg_32 = S_MOV_B32 -1
     ; CHECK: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 [[COPY]], %const, implicit-def $scc
     ; CHECK: S_ENDPGM 0, implicit [[S_AND_B32_]]
     %0:sgpr(p3) = COPY $sgpr0
@@ -96,7 +96,7 @@ body: |
 
     ; CHECK-LABEL: name: ptrmask_p3_s32_sgpr_sgpr_clearhi1
     ; CHECK: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
-    ; CHECK: %const:sreg_32 = S_MOV_B32 2147483648
+    ; CHECK: %const:sreg_32 = S_MOV_B32 -2147483648
     ; CHECK: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 [[COPY]], %const, implicit-def $scc
     ; CHECK: S_ENDPGM 0, implicit [[S_AND_B32_]]
     %0:sgpr(p3) = COPY $sgpr0
@@ -117,7 +117,7 @@ body: |
 
     ; CHECK-LABEL: name: ptrmask_p3_s32_sgpr_sgpr_clearhi2
     ; CHECK: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
-    ; CHECK: %const:sreg_32 = S_MOV_B32 3221225472
+    ; CHECK: %const:sreg_32 = S_MOV_B32 -1073741824
     ; CHECK: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 [[COPY]], %const, implicit-def $scc
     ; CHECK: S_ENDPGM 0, implicit [[S_AND_B32_]]
     %0:sgpr(p3) = COPY $sgpr0
@@ -138,7 +138,7 @@ body: |
 
     ; CHECK-LABEL: name: ptrmask_p3_s32_sgpr_sgpr_clearlo1
     ; CHECK: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
-    ; CHECK: %const:sreg_32 = S_MOV_B32 4294967294
+    ; CHECK: %const:sreg_32 = S_MOV_B32 -2
     ; CHECK: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 [[COPY]], %const, implicit-def $scc
     ; CHECK: S_ENDPGM 0, implicit [[S_AND_B32_]]
     %0:sgpr(p3) = COPY $sgpr0
@@ -159,7 +159,7 @@ body: |
 
     ; CHECK-LABEL: name: ptrmask_p3_s32_sgpr_sgpr_clearlo2
     ; CHECK: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
-    ; CHECK: %const:sreg_32 = S_MOV_B32 4294967292
+    ; CHECK: %const:sreg_32 = S_MOV_B32 -4
     ; CHECK: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 [[COPY]], %const, implicit-def $scc
     ; CHECK: S_ENDPGM 0, implicit [[S_AND_B32_]]
     %0:sgpr(p3) = COPY $sgpr0
@@ -180,7 +180,7 @@ body: |
 
     ; CHECK-LABEL: name: ptrmask_p3_s32_sgpr_sgpr_clearlo3
     ; CHECK: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
-    ; CHECK: %const:sreg_32 = S_MOV_B32 4294967288
+    ; CHECK: %const:sreg_32 = S_MOV_B32 -8
     ; CHECK: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 [[COPY]], %const, implicit-def $scc
     ; CHECK: S_ENDPGM 0, implicit [[S_AND_B32_]]
     %0:sgpr(p3) = COPY $sgpr0
@@ -201,7 +201,7 @@ body: |
 
     ; CHECK-LABEL: name: ptrmask_p3_s32_sgpr_sgpr_clearlo4
     ; CHECK: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
-    ; CHECK: %const:sreg_32 = S_MOV_B32 4294967280
+    ; CHECK: %const:sreg_32 = S_MOV_B32 -16
     ; CHECK: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 [[COPY]], %const, implicit-def $scc
     ; CHECK: S_ENDPGM 0, implicit [[S_AND_B32_]]
     %0:sgpr(p3) = COPY $sgpr0
@@ -222,7 +222,7 @@ body: |
 
     ; CHECK-LABEL: name: ptrmask_p3_s32_sgpr_sgpr_clearlo29
     ; CHECK: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
-    ; CHECK: %const:sreg_32 = S_MOV_B32 3758096384
+    ; CHECK: %const:sreg_32 = S_MOV_B32 -536870912
     ; CHECK: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 [[COPY]], %const, implicit-def $scc
     ; CHECK: S_ENDPGM 0, implicit [[S_AND_B32_]]
     %0:sgpr(p3) = COPY $sgpr0
@@ -560,7 +560,7 @@ body: |
 
     ; CHECK-LABEL: name: ptrmask_p3_vgpr_vgpr_0xf0f0f0f0
     ; CHECK: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; CHECK: %const:vgpr_32 = V_MOV_B32_e32 4042322160, implicit $exec
+    ; CHECK: %const:vgpr_32 = V_MOV_B32_e32 -252645136, implicit $exec
     ; CHECK: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY]], %const, implicit $exec
     ; CHECK: S_ENDPGM 0, implicit [[V_AND_B32_e64_]]
     %0:vgpr(p3) = COPY $vgpr0
@@ -581,7 +581,7 @@ body: |
 
     ; CHECK-LABEL: name: ptrmask_p3_vgpr_vgpr_clearlo1
     ; CHECK: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; CHECK: %const:vgpr_32 = V_MOV_B32_e32 4294967294, implicit $exec
+    ; CHECK: %const:vgpr_32 = V_MOV_B32_e32 -2, implicit $exec
     ; CHECK: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY]], %const, implicit $exec
     ; CHECK: S_ENDPGM 0, implicit [[V_AND_B32_e64_]]
     %0:vgpr(p3) = COPY $vgpr0
@@ -602,7 +602,7 @@ body: |
 
     ; CHECK-LABEL: name: ptrmask_p3_vgpr_vgpr_clearlo2
     ; CHECK: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; CHECK: %const:vgpr_32 = V_MOV_B32_e32 4294967292, implicit $exec
+    ; CHECK: %const:vgpr_32 = V_MOV_B32_e32 -4, implicit $exec
     ; CHECK: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY]], %const, implicit $exec
     ; CHECK: S_ENDPGM 0, implicit [[V_AND_B32_e64_]]
     %0:vgpr(p3) = COPY $vgpr0
@@ -623,7 +623,7 @@ body: |
 
     ; CHECK-LABEL: name: ptrmask_p3_vgpr_vgpr_clearlo3
     ; CHECK: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; CHECK: %const:vgpr_32 = V_MOV_B32_e32 4294967288, implicit $exec
+    ; CHECK: %const:vgpr_32 = V_MOV_B32_e32 -8, implicit $exec
     ; CHECK: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY]], %const, implicit $exec
     ; CHECK: S_ENDPGM 0, implicit [[V_AND_B32_e64_]]
     %0:vgpr(p3) = COPY $vgpr0
@@ -644,7 +644,7 @@ body: |
 
     ; CHECK-LABEL: name: ptrmask_p3_vgpr_vgpr_clearlo4
     ; CHECK: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; CHECK: %const:vgpr_32 = V_MOV_B32_e32 4294967280, implicit $exec
+    ; CHECK: %const:vgpr_32 = V_MOV_B32_e32 -16, implicit $exec
     ; CHECK: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY]], %const, implicit $exec
     ; CHECK: S_ENDPGM 0, implicit [[V_AND_B32_e64_]]
     %0:vgpr(p3) = COPY $vgpr0
@@ -665,7 +665,7 @@ body: |
 
     ; CHECK-LABEL: name: ptrmask_p3_vgpr_vgpr_clearlo29
     ; CHECK: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; CHECK: %const:vgpr_32 = V_MOV_B32_e32 3758096384, implicit $exec
+    ; CHECK: %const:vgpr_32 = V_MOV_B32_e32 -536870912, implicit $exec
     ; CHECK: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY]], %const, implicit $exec
     ; CHECK: S_ENDPGM 0, implicit [[V_AND_B32_e64_]]
     %0:vgpr(p3) = COPY $vgpr0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-xor.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-xor.mir
index 1c03557cc4953..f923a4c9f02b8 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-xor.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-xor.mir
@@ -20,7 +20,7 @@ body: |
     ; WAVE64: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
     ; WAVE64: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_EQ_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], implicit $exec
     ; WAVE64: [[V_CMP_EQ_U32_e64_1:%[0-9]+]]:sreg_64 = V_CMP_EQ_U32_e64 [[COPY1]], [[V_MOV_B32_e32_]], implicit $exec
-    ; WAVE64: [[S_XOR_B64_:%[0-9]+]]:sreg_64 = S_XOR_B64 [[V_CMP_EQ_U32_e64_]], [[V_CMP_EQ_U32_e64_1]]
+    ; WAVE64: [[S_XOR_B64_:%[0-9]+]]:sreg_64 = S_XOR_B64 [[V_CMP_EQ_U32_e64_]], [[V_CMP_EQ_U32_e64_1]], implicit-def dead $scc
     ; WAVE64: S_ENDPGM 0, implicit [[S_XOR_B64_]]
     ; WAVE32-LABEL: name: xor_s1_vcc_vcc_vcc
     ; WAVE32: liveins: $vgpr0, $vgpr1
@@ -30,7 +30,7 @@ body: |
     ; WAVE32: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
     ; WAVE32: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32 = V_CMP_EQ_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], implicit $exec
     ; WAVE32: [[V_CMP_EQ_U32_e64_1:%[0-9]+]]:sreg_32 = V_CMP_EQ_U32_e64 [[COPY1]], [[V_MOV_B32_e32_]], implicit $exec
-    ; WAVE32: [[S_XOR_B32_:%[0-9]+]]:sreg_32 = S_XOR_B32 [[V_CMP_EQ_U32_e64_]], [[V_CMP_EQ_U32_e64_1]]
+    ; WAVE32: [[S_XOR_B32_:%[0-9]+]]:sreg_32 = S_XOR_B32 [[V_CMP_EQ_U32_e64_]], [[V_CMP_EQ_U32_e64_1]], implicit-def dead $scc
     ; WAVE32: S_ENDPGM 0, implicit [[S_XOR_B32_]]
     %0:vgpr(s32) = COPY $vgpr0
     %1:vgpr(s32) = COPY $vgpr1
@@ -387,7 +387,7 @@ body: |
     ; WAVE64: [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_NE_U32_e64 0, [[V_AND_B32_e32_]], implicit $exec
     ; WAVE64: [[V_AND_B32_e32_1:%[0-9]+]]:vgpr_32 = V_AND_B32_e32 1, [[COPY1]], implicit $exec
     ; WAVE64: [[V_CMP_NE_U32_e64_1:%[0-9]+]]:sreg_64 = V_CMP_NE_U32_e64 0, [[V_AND_B32_e32_1]], implicit $exec
-    ; WAVE64: [[S_XOR_B64_:%[0-9]+]]:sreg_64 = S_XOR_B64 [[V_CMP_NE_U32_e64_]], [[V_CMP_NE_U32_e64_1]]
+    ; WAVE64: [[S_XOR_B64_:%[0-9]+]]:sreg_64 = S_XOR_B64 [[V_CMP_NE_U32_e64_]], [[V_CMP_NE_U32_e64_1]], implicit-def dead $scc
     ; WAVE64: S_ENDPGM 0, implicit [[S_XOR_B64_]]
     ; WAVE32-LABEL: name: xor_s1_vcc_copy_to_vcc
     ; WAVE32: liveins: $vgpr0, $vgpr1
@@ -398,7 +398,7 @@ body: |
     ; WAVE32: [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_32 = V_CMP_NE_U32_e64 0, [[V_AND_B32_e32_]], implicit $exec
     ; WAVE32: [[V_AND_B32_e32_1:%[0-9]+]]:vgpr_32 = V_AND_B32_e32 1, [[COPY1]], implicit $exec
     ; WAVE32: [[V_CMP_NE_U32_e64_1:%[0-9]+]]:sreg_32 = V_CMP_NE_U32_e64 0, [[V_AND_B32_e32_1]], implicit $exec
-    ; WAVE32: [[S_XOR_B32_:%[0-9]+]]:sreg_32 = S_XOR_B32 [[V_CMP_NE_U32_e64_]], [[V_CMP_NE_U32_e64_1]]
+    ; WAVE32: [[S_XOR_B32_:%[0-9]+]]:sreg_32 = S_XOR_B32 [[V_CMP_NE_U32_e64_]], [[V_CMP_NE_U32_e64_1]], implicit-def dead $scc
     ; WAVE32: S_ENDPGM 0, implicit [[S_XOR_B32_]]
     %0:vgpr(s32) = COPY $vgpr0
     %1:vgpr(s32) = COPY $vgpr1
@@ -426,24 +426,24 @@ body:             |
     ; WAVE64-LABEL: name: copy_select_constrain_vcc_result_reg_wave32
     ; WAVE64: liveins: $vgpr0
     ; WAVE64: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; WAVE64: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 1
+    ; WAVE64: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
     ; WAVE64: [[V_AND_B32_e32_:%[0-9]+]]:vgpr_32 = V_AND_B32_e32 1, [[COPY]], implicit $exec
     ; WAVE64: [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_NE_U32_e64 0, [[V_AND_B32_e32_]], implicit $exec
     ; WAVE64: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 1, [[S_MOV_B32_]], implicit-def $scc
     ; WAVE64: [[V_CMP_NE_U32_e64_1:%[0-9]+]]:sreg_64 = V_CMP_NE_U32_e64 0, [[S_AND_B32_]], implicit $exec
-    ; WAVE64: [[S_XOR_B64_:%[0-9]+]]:sreg_64_xexec = S_XOR_B64 [[V_CMP_NE_U32_e64_]], [[V_CMP_NE_U32_e64_1]]
+    ; WAVE64: [[S_XOR_B64_:%[0-9]+]]:sreg_64_xexec = S_XOR_B64 [[V_CMP_NE_U32_e64_]], [[V_CMP_NE_U32_e64_1]], implicit-def dead $scc
     ; WAVE64: [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY [[S_XOR_B64_]]
     ; WAVE64: S_ENDPGM 0, implicit [[COPY1]]
     ; WAVE32-LABEL: name: copy_select_constrain_vcc_result_reg_wave32
     ; WAVE32: liveins: $vgpr0
     ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; WAVE32: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 1
+    ; WAVE32: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
     ; WAVE32: [[V_AND_B32_e32_:%[0-9]+]]:vgpr_32 = V_AND_B32_e32 1, [[COPY]], implicit $exec
     ; WAVE32: [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_32 = V_CMP_NE_U32_e64 0, [[V_AND_B32_e32_]], implicit $exec
     ; WAVE32: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 1, [[S_MOV_B32_]], implicit-def $scc
     ; WAVE32: [[V_CMP_NE_U32_e64_1:%[0-9]+]]:sreg_32 = V_CMP_NE_U32_e64 0, [[S_AND_B32_]], implicit $exec
-    ; WAVE32: [[S_XOR_B32_:%[0-9]+]]:sreg_32 = S_XOR_B32 [[V_CMP_NE_U32_e64_]], [[V_CMP_NE_U32_e64_1]]
+    ; WAVE32: [[S_XOR_B32_:%[0-9]+]]:sreg_32 = S_XOR_B32 [[V_CMP_NE_U32_e64_]], [[V_CMP_NE_U32_e64_1]], implicit-def dead $scc
     ; WAVE32: [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY [[S_XOR_B32_]]
     ; WAVE32: S_ENDPGM 0, implicit [[COPY1]]
     %1:vgpr(s32) = COPY $vgpr0
@@ -472,24 +472,24 @@ body:             |
     ; WAVE64-LABEL: name: copy_select_constrain_vcc_result_reg_wave64
     ; WAVE64: liveins: $vgpr0
     ; WAVE64: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; WAVE64: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 1
+    ; WAVE64: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
     ; WAVE64: [[V_AND_B32_e32_:%[0-9]+]]:vgpr_32 = V_AND_B32_e32 1, [[COPY]], implicit $exec
     ; WAVE64: [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_NE_U32_e64 0, [[V_AND_B32_e32_]], implicit $exec
     ; WAVE64: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 1, [[S_MOV_B32_]], implicit-def $scc
     ; WAVE64: [[V_CMP_NE_U32_e64_1:%[0-9]+]]:sreg_64 = V_CMP_NE_U32_e64 0, [[S_AND_B32_]], implicit $exec
-    ; WAVE64: [[S_XOR_B64_:%[0-9]+]]:sreg_64 = S_XOR_B64 [[V_CMP_NE_U32_e64_]], [[V_CMP_NE_U32_e64_1]]
+    ; WAVE64: [[S_XOR_B64_:%[0-9]+]]:sreg_64 = S_XOR_B64 [[V_CMP_NE_U32_e64_]], [[V_CMP_NE_U32_e64_1]], implicit-def dead $scc
     ; WAVE64: [[COPY1:%[0-9]+]]:sreg_64_xexec = COPY [[S_XOR_B64_]]
     ; WAVE64: S_ENDPGM 0, implicit [[COPY1]]
     ; WAVE32-LABEL: name: copy_select_constrain_vcc_result_reg_wave64
     ; WAVE32: liveins: $vgpr0
     ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; WAVE32: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 1
+    ; WAVE32: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
     ; WAVE32: [[V_AND_B32_e32_:%[0-9]+]]:vgpr_32 = V_AND_B32_e32 1, [[COPY]], implicit $exec
     ; WAVE32: [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_32 = V_CMP_NE_U32_e64 0, [[V_AND_B32_e32_]], implicit $exec
     ; WAVE32: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 1, [[S_MOV_B32_]], implicit-def $scc
     ; WAVE32: [[V_CMP_NE_U32_e64_1:%[0-9]+]]:sreg_32 = V_CMP_NE_U32_e64 0, [[S_AND_B32_]], implicit $exec
-    ; WAVE32: [[S_XOR_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_XOR_B32 [[V_CMP_NE_U32_e64_]], [[V_CMP_NE_U32_e64_1]]
+    ; WAVE32: [[S_XOR_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_XOR_B32 [[V_CMP_NE_U32_e64_]], [[V_CMP_NE_U32_e64_1]], implicit-def dead $scc
     ; WAVE32: [[COPY1:%[0-9]+]]:sreg_64_xexec = COPY [[S_XOR_B32_]]
     ; WAVE32: S_ENDPGM 0, implicit [[COPY1]]
     %1:vgpr(s32) = COPY $vgpr0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.s.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.s.buffer.load.ll
index 66425c27a19fe..aba47890f61cb 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.s.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.s.buffer.load.ll
@@ -973,7 +973,7 @@ define amdgpu_ps i32 @s_buffer_load_i32_offset_neg1(<4 x i32> inreg %desc) {
   ; GFX6:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
   ; GFX6:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
   ; GFX6:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX6:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4294967295
+  ; GFX6:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
   ; GFX6:   [[S_BUFFER_LOAD_DWORD_SGPR:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_SGPR [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0 :: (dereferenceable invariant load 4)
   ; GFX6:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[S_BUFFER_LOAD_DWORD_SGPR]]
   ; GFX6:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
@@ -987,7 +987,7 @@ define amdgpu_ps i32 @s_buffer_load_i32_offset_neg1(<4 x i32> inreg %desc) {
   ; GFX7:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
   ; GFX7:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
   ; GFX7:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX7:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4294967295
+  ; GFX7:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
   ; GFX7:   [[S_BUFFER_LOAD_DWORD_SGPR:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_SGPR [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0 :: (dereferenceable invariant load 4)
   ; GFX7:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[S_BUFFER_LOAD_DWORD_SGPR]]
   ; GFX7:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
@@ -1001,7 +1001,7 @@ define amdgpu_ps i32 @s_buffer_load_i32_offset_neg1(<4 x i32> inreg %desc) {
   ; GFX8:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
   ; GFX8:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
   ; GFX8:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX8:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4294967295
+  ; GFX8:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
   ; GFX8:   [[S_BUFFER_LOAD_DWORD_SGPR:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_SGPR [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0 :: (dereferenceable invariant load 4)
   ; GFX8:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[S_BUFFER_LOAD_DWORD_SGPR]]
   ; GFX8:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
@@ -1020,7 +1020,7 @@ define amdgpu_ps i32 @s_buffer_load_i32_offset_neg4(<4 x i32> inreg %desc) {
   ; GFX6:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
   ; GFX6:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
   ; GFX6:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX6:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4294967292
+  ; GFX6:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 -4
   ; GFX6:   [[S_BUFFER_LOAD_DWORD_SGPR:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_SGPR [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0 :: (dereferenceable invariant load 4)
   ; GFX6:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[S_BUFFER_LOAD_DWORD_SGPR]]
   ; GFX6:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
@@ -1047,7 +1047,7 @@ define amdgpu_ps i32 @s_buffer_load_i32_offset_neg4(<4 x i32> inreg %desc) {
   ; GFX8:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
   ; GFX8:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
   ; GFX8:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX8:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4294967292
+  ; GFX8:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 -4
   ; GFX8:   [[S_BUFFER_LOAD_DWORD_SGPR:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_SGPR [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0 :: (dereferenceable invariant load 4)
   ; GFX8:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[S_BUFFER_LOAD_DWORD_SGPR]]
   ; GFX8:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
@@ -1066,7 +1066,7 @@ define amdgpu_ps i32 @s_buffer_load_i32_offset_neg8(<4 x i32> inreg %desc) {
   ; GFX6:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
   ; GFX6:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
   ; GFX6:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX6:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4294967288
+  ; GFX6:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 -8
   ; GFX6:   [[S_BUFFER_LOAD_DWORD_SGPR:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_SGPR [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0 :: (dereferenceable invariant load 4)
   ; GFX6:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[S_BUFFER_LOAD_DWORD_SGPR]]
   ; GFX6:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
@@ -1093,7 +1093,7 @@ define amdgpu_ps i32 @s_buffer_load_i32_offset_neg8(<4 x i32> inreg %desc) {
   ; GFX8:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
   ; GFX8:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
   ; GFX8:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX8:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4294967288
+  ; GFX8:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 -8
   ; GFX8:   [[S_BUFFER_LOAD_DWORD_SGPR:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_SGPR [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0 :: (dereferenceable invariant load 4)
   ; GFX8:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[S_BUFFER_LOAD_DWORD_SGPR]]
   ; GFX8:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
@@ -1112,7 +1112,7 @@ define amdgpu_ps i32 @s_buffer_load_i32_offset_bit31(<4 x i32> inreg %desc) {
   ; GFX6:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
   ; GFX6:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
   ; GFX6:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX6:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 2147483648
+  ; GFX6:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 -2147483648
   ; GFX6:   [[S_BUFFER_LOAD_DWORD_SGPR:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_SGPR [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0 :: (dereferenceable invariant load 4)
   ; GFX6:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[S_BUFFER_LOAD_DWORD_SGPR]]
   ; GFX6:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
@@ -1139,7 +1139,7 @@ define amdgpu_ps i32 @s_buffer_load_i32_offset_bit31(<4 x i32> inreg %desc) {
   ; GFX8:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
   ; GFX8:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
   ; GFX8:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX8:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 2147483648
+  ; GFX8:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 -2147483648
   ; GFX8:   [[S_BUFFER_LOAD_DWORD_SGPR:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_SGPR [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0 :: (dereferenceable invariant load 4)
   ; GFX8:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[S_BUFFER_LOAD_DWORD_SGPR]]
   ; GFX8:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
@@ -1342,7 +1342,7 @@ define amdgpu_ps i32 @s_buffer_load_i32_offset_neg_bit20(<4 x i32> inreg %desc)
   ; GFX6:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
   ; GFX6:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
   ; GFX6:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX6:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4293918720
+  ; GFX6:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 -1048576
   ; GFX6:   [[S_BUFFER_LOAD_DWORD_SGPR:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_SGPR [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0 :: (dereferenceable invariant load 4)
   ; GFX6:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[S_BUFFER_LOAD_DWORD_SGPR]]
   ; GFX6:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
@@ -1369,7 +1369,7 @@ define amdgpu_ps i32 @s_buffer_load_i32_offset_neg_bit20(<4 x i32> inreg %desc)
   ; GFX8:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
   ; GFX8:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
   ; GFX8:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX8:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4293918720
+  ; GFX8:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 -1048576
   ; GFX8:   [[S_BUFFER_LOAD_DWORD_SGPR:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_SGPR [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0 :: (dereferenceable invariant load 4)
   ; GFX8:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[S_BUFFER_LOAD_DWORD_SGPR]]
   ; GFX8:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
@@ -1433,7 +1433,7 @@ define amdgpu_ps i32 @s_buffer_load_i32_offset_neg_bit19(<4 x i32> inreg %desc)
   ; GFX6:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
   ; GFX6:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
   ; GFX6:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX6:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4294443008
+  ; GFX6:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 -524288
   ; GFX6:   [[S_BUFFER_LOAD_DWORD_SGPR:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_SGPR [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0 :: (dereferenceable invariant load 4)
   ; GFX6:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[S_BUFFER_LOAD_DWORD_SGPR]]
   ; GFX6:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
@@ -1460,7 +1460,7 @@ define amdgpu_ps i32 @s_buffer_load_i32_offset_neg_bit19(<4 x i32> inreg %desc)
   ; GFX8:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
   ; GFX8:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
   ; GFX8:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX8:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4294443008
+  ; GFX8:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 -524288
   ; GFX8:   [[S_BUFFER_LOAD_DWORD_SGPR:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_SGPR [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0 :: (dereferenceable invariant load 4)
   ; GFX8:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[S_BUFFER_LOAD_DWORD_SGPR]]
   ; GFX8:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.sbfe.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.sbfe.ll
index 7d116f8e8925f..e5d26476e9424 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.sbfe.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.sbfe.ll
@@ -414,8 +414,8 @@ define amdgpu_kernel void @bfe_i32_constant_fold_test_4(i32 addrspace(1)* %out)
 ; GFX6-LABEL: bfe_i32_constant_fold_test_4:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
-; GFX6-NEXT:    s_bfe_i32 s3, -1, 0x10000
-; GFX6-NEXT:    v_mov_b32_e32 v0, s3
+; GFX6-NEXT:    s_bfe_i32 s2, -1, 0x10000
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX6-NEXT:    s_mov_b32 s2, -1
 ; GFX6-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
@@ -617,8 +617,8 @@ define amdgpu_kernel void @bfe_i32_constant_fold_test_16(i32 addrspace(1)* %out)
 ; GFX6-LABEL: bfe_i32_constant_fold_test_16:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
-; GFX6-NEXT:    s_bfe_i32 s3, -1, 0x70001
-; GFX6-NEXT:    v_mov_b32_e32 v0, s3
+; GFX6-NEXT:    s_bfe_i32 s2, -1, 0x70001
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX6-NEXT:    s_mov_b32 s2, -1
 ; GFX6-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ubfe.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ubfe.ll
index a5737e8233af3..6fc1cd5753083 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ubfe.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ubfe.ll
@@ -694,8 +694,8 @@ define amdgpu_kernel void @bfe_u32_constant_fold_test_4(i32 addrspace(1)* %out)
 ; GFX6-LABEL: bfe_u32_constant_fold_test_4:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
-; GFX6-NEXT:    s_bfe_u32 s3, -1, 0x10000
-; GFX6-NEXT:    v_mov_b32_e32 v0, s3
+; GFX6-NEXT:    s_bfe_u32 s2, -1, 0x10000
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX6-NEXT:    s_mov_b32 s2, -1
 ; GFX6-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
@@ -897,8 +897,8 @@ define amdgpu_kernel void @bfe_u32_constant_fold_test_16(i32 addrspace(1)* %out)
 ; GFX6-LABEL: bfe_u32_constant_fold_test_16:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
-; GFX6-NEXT:    s_bfe_u32 s3, -1, 0x70001
-; GFX6-NEXT:    v_mov_b32_e32 v0, s3
+; GFX6-NEXT:    s_bfe_u32 s2, -1, 0x70001
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX6-NEXT:    s_mov_b32 s2, -1
 ; GFX6-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/localizer.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/localizer.ll
index 06bf7f7949309..23398f8ecf6e4 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/localizer.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/localizer.ll
@@ -8,9 +8,9 @@ define amdgpu_kernel void @localize_constants(i1 %cond) {
 ; GFX9-LABEL: localize_constants:
 ; GFX9:       ; %bb.0: ; %entry
 ; GFX9-NEXT:    s_load_dword s1, s[4:5], 0x0
-; GFX9-NEXT:    s_mov_b32 s0, 1
+; GFX9-NEXT:    s_mov_b32 s0, -1
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_xor_b32 s1, s1, 1
+; GFX9-NEXT:    s_xor_b32 s1, s1, -1
 ; GFX9-NEXT:    s_and_b32 s1, s1, 1
 ; GFX9-NEXT:    s_cmp_lg_u32 s1, 0
 ; GFX9-NEXT:    s_cbranch_scc0 BB0_2
@@ -83,9 +83,9 @@ define amdgpu_kernel void @localize_globals(i1 %cond) {
 ; GFX9-LABEL: localize_globals:
 ; GFX9:       ; %bb.0: ; %entry
 ; GFX9-NEXT:    s_load_dword s1, s[4:5], 0x0
-; GFX9-NEXT:    s_mov_b32 s0, 1
+; GFX9-NEXT:    s_mov_b32 s0, -1
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_xor_b32 s1, s1, 1
+; GFX9-NEXT:    s_xor_b32 s1, s1, -1
 ; GFX9-NEXT:    s_and_b32 s1, s1, 1
 ; GFX9-NEXT:    s_cmp_lg_u32 s1, 0
 ; GFX9-NEXT:    s_cbranch_scc0 BB1_2
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/saddsat.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/saddsat.ll
index 80ff9bb2b5752..b2e7f1ea326f6 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/saddsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/saddsat.ll
@@ -101,7 +101,7 @@ define amdgpu_ps i7 @s_saddsat_i7(i7 inreg %lhs, i7 inreg %rhs) {
 ; GFX8-NEXT:    s_sub_i32 s5, 0x7fff, s5
 ; GFX8-NEXT:    s_cmp_lt_i32 s3, s4
 ; GFX8-NEXT:    s_cselect_b32 s3, s3, s4
-; GFX8-NEXT:    s_sub_i32 s3, 0x8000, s3
+; GFX8-NEXT:    s_sub_i32 s3, 0xffff8000, s3
 ; GFX8-NEXT:    s_sext_i32_i16 s3, s3
 ; GFX8-NEXT:    s_sext_i32_i16 s1, s1
 ; GFX8-NEXT:    s_cmp_gt_i32 s3, s1
@@ -127,7 +127,7 @@ define amdgpu_ps i7 @s_saddsat_i7(i7 inreg %lhs, i7 inreg %rhs) {
 ; GFX9-NEXT:    s_sub_i32 s5, 0x7fff, s5
 ; GFX9-NEXT:    s_cmp_lt_i32 s3, s4
 ; GFX9-NEXT:    s_cselect_b32 s3, s3, s4
-; GFX9-NEXT:    s_sub_i32 s3, 0x8000, s3
+; GFX9-NEXT:    s_sub_i32 s3, 0xffff8000, s3
 ; GFX9-NEXT:    s_sext_i32_i16 s3, s3
 ; GFX9-NEXT:    s_sext_i32_i16 s1, s1
 ; GFX9-NEXT:    s_cmp_gt_i32 s3, s1
@@ -155,7 +155,7 @@ define amdgpu_ps i7 @s_saddsat_i7(i7 inreg %lhs, i7 inreg %rhs) {
 ; GFX10-NEXT:    s_sub_i32 s5, 0x7fff, s5
 ; GFX10-NEXT:    s_cmp_lt_i32 s3, s4
 ; GFX10-NEXT:    s_cselect_b32 s3, s3, s4
-; GFX10-NEXT:    s_sub_i32 s3, 0x8000, s3
+; GFX10-NEXT:    s_sub_i32 s3, 0xffff8000, s3
 ; GFX10-NEXT:    s_sext_i32_i16 s3, s3
 ; GFX10-NEXT:    s_cmp_gt_i32 s3, s1
 ; GFX10-NEXT:    s_cselect_b32 s1, s3, s1
@@ -268,7 +268,7 @@ define amdgpu_ps i8 @s_saddsat_i8(i8 inreg %lhs, i8 inreg %rhs) {
 ; GFX8-NEXT:    s_sub_i32 s5, 0x7fff, s5
 ; GFX8-NEXT:    s_cmp_lt_i32 s3, s4
 ; GFX8-NEXT:    s_cselect_b32 s3, s3, s4
-; GFX8-NEXT:    s_sub_i32 s3, 0x8000, s3
+; GFX8-NEXT:    s_sub_i32 s3, 0xffff8000, s3
 ; GFX8-NEXT:    s_sext_i32_i16 s3, s3
 ; GFX8-NEXT:    s_sext_i32_i16 s1, s1
 ; GFX8-NEXT:    s_cmp_gt_i32 s3, s1
@@ -294,7 +294,7 @@ define amdgpu_ps i8 @s_saddsat_i8(i8 inreg %lhs, i8 inreg %rhs) {
 ; GFX9-NEXT:    s_sub_i32 s5, 0x7fff, s5
 ; GFX9-NEXT:    s_cmp_lt_i32 s3, s4
 ; GFX9-NEXT:    s_cselect_b32 s3, s3, s4
-; GFX9-NEXT:    s_sub_i32 s3, 0x8000, s3
+; GFX9-NEXT:    s_sub_i32 s3, 0xffff8000, s3
 ; GFX9-NEXT:    s_sext_i32_i16 s3, s3
 ; GFX9-NEXT:    s_sext_i32_i16 s1, s1
 ; GFX9-NEXT:    s_cmp_gt_i32 s3, s1
@@ -322,7 +322,7 @@ define amdgpu_ps i8 @s_saddsat_i8(i8 inreg %lhs, i8 inreg %rhs) {
 ; GFX10-NEXT:    s_sub_i32 s5, 0x7fff, s5
 ; GFX10-NEXT:    s_cmp_lt_i32 s3, s4
 ; GFX10-NEXT:    s_cselect_b32 s3, s3, s4
-; GFX10-NEXT:    s_sub_i32 s3, 0x8000, s3
+; GFX10-NEXT:    s_sub_i32 s3, 0xffff8000, s3
 ; GFX10-NEXT:    s_sext_i32_i16 s3, s3
 ; GFX10-NEXT:    s_cmp_gt_i32 s3, s1
 ; GFX10-NEXT:    s_cselect_b32 s1, s3, s1
@@ -344,7 +344,7 @@ define i16 @v_saddsat_v2i8(i16 %lhs.arg, i16 %rhs.arg) {
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX6-NEXT:    v_lshrrev_b32_e32 v2, 8, v0
 ; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 24, v0
-; GFX6-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX6-NEXT:    s_brev_b32 s5, 1
 ; GFX6-NEXT:    v_min_i32_e32 v5, 0, v0
 ; GFX6-NEXT:    v_lshrrev_b32_e32 v3, 8, v1
 ; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 24, v1
@@ -379,7 +379,7 @@ define i16 @v_saddsat_v2i8(i16 %lhs.arg, i16 %rhs.arg) {
 ; GFX8-NEXT:    v_mov_b32_e32 v2, 8
 ; GFX8-NEXT:    v_lshrrev_b32_sdwa v3, v2, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX8-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; GFX8-NEXT:    s_mov_b32 s5, 0x8000
+; GFX8-NEXT:    s_movk_i32 s5, 0x8000
 ; GFX8-NEXT:    v_min_i16_e32 v5, 0, v0
 ; GFX8-NEXT:    v_lshrrev_b32_sdwa v2, v2, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX8-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
@@ -409,7 +409,7 @@ define i16 @v_saddsat_v2i8(i16 %lhs.arg, i16 %rhs.arg) {
 ; GFX9-NEXT:    s_mov_b32 s4, 8
 ; GFX9-NEXT:    v_lshrrev_b32_sdwa v2, s4, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; GFX9-NEXT:    s_mov_b32 s5, 0x8000
+; GFX9-NEXT:    s_movk_i32 s5, 0x8000
 ; GFX9-NEXT:    v_min_i16_e32 v5, 0, v0
 ; GFX9-NEXT:    v_lshrrev_b32_sdwa v3, s4, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
@@ -440,7 +440,7 @@ define i16 @v_saddsat_v2i8(i16 %lhs.arg, i16 %rhs.arg) {
 ; GFX10-NEXT:    s_mov_b32 s4, 8
 ; GFX10-NEXT:    v_lshlrev_b16_e64 v2, 8, v0
 ; GFX10-NEXT:    v_lshrrev_b32_sdwa v0, s4, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT:    s_mov_b32 s5, 0x8000
+; GFX10-NEXT:    s_movk_i32 s5, 0x8000
 ; GFX10-NEXT:    v_lshrrev_b32_sdwa v3, s4, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX10-NEXT:    v_lshlrev_b16_e64 v1, 8, v1
 ; GFX10-NEXT:    v_min_i16_e64 v4, v2, 0
@@ -483,7 +483,7 @@ define amdgpu_ps i16 @s_saddsat_v2i8(i16 inreg %lhs.arg, i16 inreg %rhs.arg) {
 ; GFX6-NEXT:    s_cselect_b32 s6, s0, 0
 ; GFX6-NEXT:    s_sub_i32 s6, s4, s6
 ; GFX6-NEXT:    s_cmp_lt_i32 s0, 0
-; GFX6-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX6-NEXT:    s_brev_b32 s5, 1
 ; GFX6-NEXT:    s_cselect_b32 s7, s0, 0
 ; GFX6-NEXT:    s_sub_i32 s7, s5, s7
 ; GFX6-NEXT:    s_cmp_gt_i32 s7, s1
@@ -527,7 +527,7 @@ define amdgpu_ps i16 @s_saddsat_v2i8(i16 inreg %lhs.arg, i16 inreg %rhs.arg) {
 ; GFX8-NEXT:    s_cselect_b32 s9, s7, s8
 ; GFX8-NEXT:    s_sub_i32 s9, s5, s9
 ; GFX8-NEXT:    s_cmp_lt_i32 s7, s8
-; GFX8-NEXT:    s_mov_b32 s6, 0x8000
+; GFX8-NEXT:    s_movk_i32 s6, 0x8000
 ; GFX8-NEXT:    s_cselect_b32 s7, s7, s8
 ; GFX8-NEXT:    s_sub_i32 s7, s6, s7
 ; GFX8-NEXT:    s_sext_i32_i16 s7, s7
@@ -582,7 +582,7 @@ define amdgpu_ps i16 @s_saddsat_v2i8(i16 inreg %lhs.arg, i16 inreg %rhs.arg) {
 ; GFX9-NEXT:    s_cselect_b32 s9, s7, s8
 ; GFX9-NEXT:    s_sub_i32 s9, s5, s9
 ; GFX9-NEXT:    s_cmp_lt_i32 s7, s8
-; GFX9-NEXT:    s_mov_b32 s6, 0x8000
+; GFX9-NEXT:    s_movk_i32 s6, 0x8000
 ; GFX9-NEXT:    s_cselect_b32 s7, s7, s8
 ; GFX9-NEXT:    s_sub_i32 s7, s6, s7
 ; GFX9-NEXT:    s_sext_i32_i16 s7, s7
@@ -635,7 +635,7 @@ define amdgpu_ps i16 @s_saddsat_v2i8(i16 inreg %lhs.arg, i16 inreg %rhs.arg) {
 ; GFX10-NEXT:    s_cmp_gt_i32 s5, s6
 ; GFX10-NEXT:    s_movk_i32 s7, 0x7fff
 ; GFX10-NEXT:    s_cselect_b32 s8, s5, s6
-; GFX10-NEXT:    s_mov_b32 s9, 0x8000
+; GFX10-NEXT:    s_movk_i32 s9, 0x8000
 ; GFX10-NEXT:    s_sub_i32 s8, s7, s8
 ; GFX10-NEXT:    s_cmp_lt_i32 s5, s6
 ; GFX10-NEXT:    s_sext_i32_i16 s1, s1
@@ -693,7 +693,7 @@ define i32 @v_saddsat_v4i8(i32 %lhs.arg, i32 %rhs.arg) {
 ; GFX6-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
 ; GFX6-NEXT:    v_lshrrev_b32_e32 v4, 24, v0
 ; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 24, v0
-; GFX6-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX6-NEXT:    s_brev_b32 s5, 1
 ; GFX6-NEXT:    v_min_i32_e32 v10, 0, v0
 ; GFX6-NEXT:    v_lshrrev_b32_e32 v5, 8, v1
 ; GFX6-NEXT:    v_lshrrev_b32_e32 v6, 16, v1
@@ -726,7 +726,7 @@ define i32 @v_saddsat_v4i8(i32 %lhs.arg, i32 %rhs.arg) {
 ; GFX6-NEXT:    v_min_i32_e32 v3, v3, v5
 ; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
 ; GFX6-NEXT:    v_lshlrev_b32_e32 v3, 24, v4
-; GFX6-NEXT:    v_mov_b32_e32 v11, 0x80000000
+; GFX6-NEXT:    v_bfrev_b32_e32 v11, 1
 ; GFX6-NEXT:    v_min_i32_e32 v6, 0, v3
 ; GFX6-NEXT:    v_max_i32_e32 v5, 0, v3
 ; GFX6-NEXT:    v_ashrrev_i32_e32 v1, 24, v1
@@ -760,7 +760,7 @@ define i32 @v_saddsat_v4i8(i32 %lhs.arg, i32 %rhs.arg) {
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v4, 16, v0
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v5, 24, v0
 ; GFX8-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; GFX8-NEXT:    s_mov_b32 s5, 0x8000
+; GFX8-NEXT:    s_movk_i32 s5, 0x8000
 ; GFX8-NEXT:    v_min_i16_e32 v10, 0, v0
 ; GFX8-NEXT:    v_lshrrev_b32_sdwa v2, v2, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v6, 16, v1
@@ -818,7 +818,7 @@ define i32 @v_saddsat_v4i8(i32 %lhs.arg, i32 %rhs.arg) {
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v4, 24, v0
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; GFX9-NEXT:    s_mov_b32 s5, 0x8000
+; GFX9-NEXT:    s_movk_i32 s5, 0x8000
 ; GFX9-NEXT:    v_min_i16_e32 v10, 0, v0
 ; GFX9-NEXT:    v_lshrrev_b32_sdwa v5, s4, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v6, 16, v1
@@ -878,7 +878,7 @@ define i32 @v_saddsat_v4i8(i32 %lhs.arg, i32 %rhs.arg) {
 ; GFX10-NEXT:    v_lshrrev_b32_sdwa v3, s5, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX10-NEXT:    v_min_i16_e64 v8, v4, 0
 ; GFX10-NEXT:    v_lshrrev_b32_sdwa v6, s5, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT:    s_mov_b32 s5, 0x8000
+; GFX10-NEXT:    s_movk_i32 s5, 0x8000
 ; GFX10-NEXT:    v_min_i16_e64 v9, v2, 0
 ; GFX10-NEXT:    v_lshlrev_b16_e64 v7, 8, v1
 ; GFX10-NEXT:    v_sub_nc_u16_e64 v8, s5, v8
@@ -946,7 +946,7 @@ define amdgpu_ps i32 @s_saddsat_v4i8(i32 inreg %lhs.arg, i32 inreg %rhs.arg) {
 ; GFX6-NEXT:    s_cselect_b32 s10, s0, 0
 ; GFX6-NEXT:    s_sub_i32 s10, s8, s10
 ; GFX6-NEXT:    s_cmp_lt_i32 s0, 0
-; GFX6-NEXT:    s_mov_b32 s9, 0x80000000
+; GFX6-NEXT:    s_brev_b32 s9, 1
 ; GFX6-NEXT:    s_cselect_b32 s11, s0, 0
 ; GFX6-NEXT:    s_sub_i32 s11, s9, s11
 ; GFX6-NEXT:    s_cmp_gt_i32 s11, s1
@@ -1028,7 +1028,7 @@ define amdgpu_ps i32 @s_saddsat_v4i8(i32 inreg %lhs.arg, i32 inreg %rhs.arg) {
 ; GFX8-NEXT:    s_cselect_b32 s13, s11, s12
 ; GFX8-NEXT:    s_sub_i32 s13, s9, s13
 ; GFX8-NEXT:    s_cmp_lt_i32 s11, s12
-; GFX8-NEXT:    s_mov_b32 s10, 0x8000
+; GFX8-NEXT:    s_movk_i32 s10, 0x8000
 ; GFX8-NEXT:    s_cselect_b32 s11, s11, s12
 ; GFX8-NEXT:    s_sub_i32 s11, s10, s11
 ; GFX8-NEXT:    s_sext_i32_i16 s11, s11
@@ -1133,7 +1133,7 @@ define amdgpu_ps i32 @s_saddsat_v4i8(i32 inreg %lhs.arg, i32 inreg %rhs.arg) {
 ; GFX9-NEXT:    s_cselect_b32 s13, s11, s12
 ; GFX9-NEXT:    s_sub_i32 s13, s9, s13
 ; GFX9-NEXT:    s_cmp_lt_i32 s11, s12
-; GFX9-NEXT:    s_mov_b32 s10, 0x8000
+; GFX9-NEXT:    s_movk_i32 s10, 0x8000
 ; GFX9-NEXT:    s_cselect_b32 s11, s11, s12
 ; GFX9-NEXT:    s_sub_i32 s11, s10, s11
 ; GFX9-NEXT:    s_sext_i32_i16 s11, s11
@@ -1236,7 +1236,7 @@ define amdgpu_ps i32 @s_saddsat_v4i8(i32 inreg %lhs.arg, i32 inreg %rhs.arg) {
 ; GFX10-NEXT:    s_cmp_gt_i32 s9, s10
 ; GFX10-NEXT:    s_movk_i32 s11, 0x7fff
 ; GFX10-NEXT:    s_cselect_b32 s12, s9, s10
-; GFX10-NEXT:    s_mov_b32 s13, 0x8000
+; GFX10-NEXT:    s_movk_i32 s13, 0x8000
 ; GFX10-NEXT:    s_sub_i32 s12, s11, s12
 ; GFX10-NEXT:    s_cmp_lt_i32 s9, s10
 ; GFX10-NEXT:    s_sext_i32_i16 s1, s1
@@ -1723,7 +1723,7 @@ define <2 x i32> @v_saddsat_v2i32(<2 x i32> %lhs, <2 x i32> %rhs) {
 ; GFX6-LABEL: v_saddsat_v2i32:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX6-NEXT:    s_brev_b32 s5, 1
 ; GFX6-NEXT:    v_min_i32_e32 v5, 0, v0
 ; GFX6-NEXT:    v_sub_i32_e32 v5, vcc, s5, v5
 ; GFX6-NEXT:    s_brev_b32 s4, -2
@@ -1744,7 +1744,7 @@ define <2 x i32> @v_saddsat_v2i32(<2 x i32> %lhs, <2 x i32> %rhs) {
 ; GFX8-LABEL: v_saddsat_v2i32:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX8-NEXT:    s_brev_b32 s5, 1
 ; GFX8-NEXT:    v_min_i32_e32 v5, 0, v0
 ; GFX8-NEXT:    v_sub_u32_e32 v5, vcc, s5, v5
 ; GFX8-NEXT:    s_brev_b32 s4, -2
@@ -1765,7 +1765,7 @@ define <2 x i32> @v_saddsat_v2i32(<2 x i32> %lhs, <2 x i32> %rhs) {
 ; GFX9-LABEL: v_saddsat_v2i32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX9-NEXT:    s_brev_b32 s5, 1
 ; GFX9-NEXT:    v_min_i32_e32 v5, 0, v0
 ; GFX9-NEXT:    v_sub_u32_e32 v5, s5, v5
 ; GFX9-NEXT:    s_brev_b32 s4, -2
@@ -1789,7 +1789,7 @@ define <2 x i32> @v_saddsat_v2i32(<2 x i32> %lhs, <2 x i32> %rhs) {
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_min_i32_e32 v4, 0, v0
 ; GFX10-NEXT:    v_min_i32_e32 v5, 0, v1
-; GFX10-NEXT:    s_mov_b32 s4, 0x80000000
+; GFX10-NEXT:    s_brev_b32 s4, 1
 ; GFX10-NEXT:    v_max_i32_e32 v6, 0, v0
 ; GFX10-NEXT:    v_max_i32_e32 v7, 0, v1
 ; GFX10-NEXT:    v_sub_nc_u32_e32 v4, s4, v4
@@ -1817,7 +1817,7 @@ define amdgpu_ps <2 x i32> @s_saddsat_v2i32(<2 x i32> inreg %lhs, <2 x i32> inre
 ; GFX6-NEXT:    s_cselect_b32 s6, s0, 0
 ; GFX6-NEXT:    s_sub_i32 s6, s4, s6
 ; GFX6-NEXT:    s_cmp_lt_i32 s0, 0
-; GFX6-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX6-NEXT:    s_brev_b32 s5, 1
 ; GFX6-NEXT:    s_cselect_b32 s7, s0, 0
 ; GFX6-NEXT:    s_sub_i32 s7, s5, s7
 ; GFX6-NEXT:    s_cmp_gt_i32 s7, s2
@@ -1845,7 +1845,7 @@ define amdgpu_ps <2 x i32> @s_saddsat_v2i32(<2 x i32> inreg %lhs, <2 x i32> inre
 ; GFX8-NEXT:    s_cselect_b32 s6, s0, 0
 ; GFX8-NEXT:    s_sub_i32 s6, s4, s6
 ; GFX8-NEXT:    s_cmp_lt_i32 s0, 0
-; GFX8-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX8-NEXT:    s_brev_b32 s5, 1
 ; GFX8-NEXT:    s_cselect_b32 s7, s0, 0
 ; GFX8-NEXT:    s_sub_i32 s7, s5, s7
 ; GFX8-NEXT:    s_cmp_gt_i32 s7, s2
@@ -1873,7 +1873,7 @@ define amdgpu_ps <2 x i32> @s_saddsat_v2i32(<2 x i32> inreg %lhs, <2 x i32> inre
 ; GFX9-NEXT:    s_cselect_b32 s6, s0, 0
 ; GFX9-NEXT:    s_sub_i32 s6, s4, s6
 ; GFX9-NEXT:    s_cmp_lt_i32 s0, 0
-; GFX9-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX9-NEXT:    s_brev_b32 s5, 1
 ; GFX9-NEXT:    s_cselect_b32 s7, s0, 0
 ; GFX9-NEXT:    s_sub_i32 s7, s5, s7
 ; GFX9-NEXT:    s_cmp_gt_i32 s7, s2
@@ -1899,7 +1899,7 @@ define amdgpu_ps <2 x i32> @s_saddsat_v2i32(<2 x i32> inreg %lhs, <2 x i32> inre
 ; GFX10-NEXT:    s_cmp_gt_i32 s0, 0
 ; GFX10-NEXT:    s_brev_b32 s4, -2
 ; GFX10-NEXT:    s_cselect_b32 s5, s0, 0
-; GFX10-NEXT:    s_mov_b32 s6, 0x80000000
+; GFX10-NEXT:    s_brev_b32 s6, 1
 ; GFX10-NEXT:    s_sub_i32 s5, s4, s5
 ; GFX10-NEXT:    s_cmp_lt_i32 s0, 0
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
@@ -1930,7 +1930,7 @@ define <3 x i32> @v_saddsat_v3i32(<3 x i32> %lhs, <3 x i32> %rhs) {
 ; GFX6-LABEL: v_saddsat_v3i32:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX6-NEXT:    s_brev_b32 s5, 1
 ; GFX6-NEXT:    v_min_i32_e32 v7, 0, v0
 ; GFX6-NEXT:    v_sub_i32_e32 v7, vcc, s5, v7
 ; GFX6-NEXT:    s_brev_b32 s4, -2
@@ -1958,7 +1958,7 @@ define <3 x i32> @v_saddsat_v3i32(<3 x i32> %lhs, <3 x i32> %rhs) {
 ; GFX8-LABEL: v_saddsat_v3i32:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX8-NEXT:    s_brev_b32 s5, 1
 ; GFX8-NEXT:    v_min_i32_e32 v7, 0, v0
 ; GFX8-NEXT:    v_sub_u32_e32 v7, vcc, s5, v7
 ; GFX8-NEXT:    s_brev_b32 s4, -2
@@ -1986,7 +1986,7 @@ define <3 x i32> @v_saddsat_v3i32(<3 x i32> %lhs, <3 x i32> %rhs) {
 ; GFX9-LABEL: v_saddsat_v3i32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX9-NEXT:    s_brev_b32 s5, 1
 ; GFX9-NEXT:    v_min_i32_e32 v7, 0, v0
 ; GFX9-NEXT:    v_sub_u32_e32 v7, s5, v7
 ; GFX9-NEXT:    s_brev_b32 s4, -2
@@ -2018,7 +2018,7 @@ define <3 x i32> @v_saddsat_v3i32(<3 x i32> %lhs, <3 x i32> %rhs) {
 ; GFX10-NEXT:    v_min_i32_e32 v7, 0, v0
 ; GFX10-NEXT:    v_min_i32_e32 v8, 0, v1
 ; GFX10-NEXT:    v_min_i32_e32 v9, 0, v2
-; GFX10-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX10-NEXT:    s_brev_b32 s5, 1
 ; GFX10-NEXT:    v_max_i32_e32 v6, 0, v0
 ; GFX10-NEXT:    v_sub_nc_u32_e32 v14, s5, v7
 ; GFX10-NEXT:    v_sub_nc_u32_e32 v15, s5, v8
@@ -2052,7 +2052,7 @@ define amdgpu_ps <3 x i32> @s_saddsat_v3i32(<3 x i32> inreg %lhs, <3 x i32> inre
 ; GFX6-NEXT:    s_cselect_b32 s8, s0, 0
 ; GFX6-NEXT:    s_sub_i32 s8, s6, s8
 ; GFX6-NEXT:    s_cmp_lt_i32 s0, 0
-; GFX6-NEXT:    s_mov_b32 s7, 0x80000000
+; GFX6-NEXT:    s_brev_b32 s7, 1
 ; GFX6-NEXT:    s_cselect_b32 s9, s0, 0
 ; GFX6-NEXT:    s_sub_i32 s9, s7, s9
 ; GFX6-NEXT:    s_cmp_gt_i32 s9, s3
@@ -2091,7 +2091,7 @@ define amdgpu_ps <3 x i32> @s_saddsat_v3i32(<3 x i32> inreg %lhs, <3 x i32> inre
 ; GFX8-NEXT:    s_cselect_b32 s8, s0, 0
 ; GFX8-NEXT:    s_sub_i32 s8, s6, s8
 ; GFX8-NEXT:    s_cmp_lt_i32 s0, 0
-; GFX8-NEXT:    s_mov_b32 s7, 0x80000000
+; GFX8-NEXT:    s_brev_b32 s7, 1
 ; GFX8-NEXT:    s_cselect_b32 s9, s0, 0
 ; GFX8-NEXT:    s_sub_i32 s9, s7, s9
 ; GFX8-NEXT:    s_cmp_gt_i32 s9, s3
@@ -2130,7 +2130,7 @@ define amdgpu_ps <3 x i32> @s_saddsat_v3i32(<3 x i32> inreg %lhs, <3 x i32> inre
 ; GFX9-NEXT:    s_cselect_b32 s8, s0, 0
 ; GFX9-NEXT:    s_sub_i32 s8, s6, s8
 ; GFX9-NEXT:    s_cmp_lt_i32 s0, 0
-; GFX9-NEXT:    s_mov_b32 s7, 0x80000000
+; GFX9-NEXT:    s_brev_b32 s7, 1
 ; GFX9-NEXT:    s_cselect_b32 s9, s0, 0
 ; GFX9-NEXT:    s_sub_i32 s9, s7, s9
 ; GFX9-NEXT:    s_cmp_gt_i32 s9, s3
@@ -2167,7 +2167,7 @@ define amdgpu_ps <3 x i32> @s_saddsat_v3i32(<3 x i32> inreg %lhs, <3 x i32> inre
 ; GFX10-NEXT:    s_cmp_gt_i32 s0, 0
 ; GFX10-NEXT:    s_brev_b32 s6, -2
 ; GFX10-NEXT:    s_cselect_b32 s7, s0, 0
-; GFX10-NEXT:    s_mov_b32 s8, 0x80000000
+; GFX10-NEXT:    s_brev_b32 s8, 1
 ; GFX10-NEXT:    s_sub_i32 s7, s6, s7
 ; GFX10-NEXT:    s_cmp_lt_i32 s0, 0
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
@@ -2209,7 +2209,7 @@ define <4 x i32> @v_saddsat_v4i32(<4 x i32> %lhs, <4 x i32> %rhs) {
 ; GFX6-LABEL: v_saddsat_v4i32:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX6-NEXT:    s_brev_b32 s5, 1
 ; GFX6-NEXT:    v_min_i32_e32 v9, 0, v0
 ; GFX6-NEXT:    v_sub_i32_e32 v9, vcc, s5, v9
 ; GFX6-NEXT:    s_brev_b32 s4, -2
@@ -2244,7 +2244,7 @@ define <4 x i32> @v_saddsat_v4i32(<4 x i32> %lhs, <4 x i32> %rhs) {
 ; GFX8-LABEL: v_saddsat_v4i32:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX8-NEXT:    s_brev_b32 s5, 1
 ; GFX8-NEXT:    v_min_i32_e32 v9, 0, v0
 ; GFX8-NEXT:    v_sub_u32_e32 v9, vcc, s5, v9
 ; GFX8-NEXT:    s_brev_b32 s4, -2
@@ -2279,7 +2279,7 @@ define <4 x i32> @v_saddsat_v4i32(<4 x i32> %lhs, <4 x i32> %rhs) {
 ; GFX9-LABEL: v_saddsat_v4i32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX9-NEXT:    s_brev_b32 s5, 1
 ; GFX9-NEXT:    v_min_i32_e32 v9, 0, v0
 ; GFX9-NEXT:    v_sub_u32_e32 v9, s5, v9
 ; GFX9-NEXT:    s_brev_b32 s4, -2
@@ -2316,7 +2316,7 @@ define <4 x i32> @v_saddsat_v4i32(<4 x i32> %lhs, <4 x i32> %rhs) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_min_i32_e32 v8, 0, v0
-; GFX10-NEXT:    s_mov_b32 s4, 0x80000000
+; GFX10-NEXT:    s_brev_b32 s4, 1
 ; GFX10-NEXT:    v_min_i32_e32 v11, 0, v1
 ; GFX10-NEXT:    v_min_i32_e32 v12, 0, v3
 ; GFX10-NEXT:    v_max_i32_e32 v9, 0, v0
@@ -2359,7 +2359,7 @@ define amdgpu_ps <4 x i32> @s_saddsat_v4i32(<4 x i32> inreg %lhs, <4 x i32> inre
 ; GFX6-NEXT:    s_cselect_b32 s10, s0, 0
 ; GFX6-NEXT:    s_sub_i32 s10, s8, s10
 ; GFX6-NEXT:    s_cmp_lt_i32 s0, 0
-; GFX6-NEXT:    s_mov_b32 s9, 0x80000000
+; GFX6-NEXT:    s_brev_b32 s9, 1
 ; GFX6-NEXT:    s_cselect_b32 s11, s0, 0
 ; GFX6-NEXT:    s_sub_i32 s11, s9, s11
 ; GFX6-NEXT:    s_cmp_gt_i32 s11, s4
@@ -2409,7 +2409,7 @@ define amdgpu_ps <4 x i32> @s_saddsat_v4i32(<4 x i32> inreg %lhs, <4 x i32> inre
 ; GFX8-NEXT:    s_cselect_b32 s10, s0, 0
 ; GFX8-NEXT:    s_sub_i32 s10, s8, s10
 ; GFX8-NEXT:    s_cmp_lt_i32 s0, 0
-; GFX8-NEXT:    s_mov_b32 s9, 0x80000000
+; GFX8-NEXT:    s_brev_b32 s9, 1
 ; GFX8-NEXT:    s_cselect_b32 s11, s0, 0
 ; GFX8-NEXT:    s_sub_i32 s11, s9, s11
 ; GFX8-NEXT:    s_cmp_gt_i32 s11, s4
@@ -2459,7 +2459,7 @@ define amdgpu_ps <4 x i32> @s_saddsat_v4i32(<4 x i32> inreg %lhs, <4 x i32> inre
 ; GFX9-NEXT:    s_cselect_b32 s10, s0, 0
 ; GFX9-NEXT:    s_sub_i32 s10, s8, s10
 ; GFX9-NEXT:    s_cmp_lt_i32 s0, 0
-; GFX9-NEXT:    s_mov_b32 s9, 0x80000000
+; GFX9-NEXT:    s_brev_b32 s9, 1
 ; GFX9-NEXT:    s_cselect_b32 s11, s0, 0
 ; GFX9-NEXT:    s_sub_i32 s11, s9, s11
 ; GFX9-NEXT:    s_cmp_gt_i32 s11, s4
@@ -2507,7 +2507,7 @@ define amdgpu_ps <4 x i32> @s_saddsat_v4i32(<4 x i32> inreg %lhs, <4 x i32> inre
 ; GFX10-NEXT:    s_cmp_gt_i32 s0, 0
 ; GFX10-NEXT:    s_brev_b32 s8, -2
 ; GFX10-NEXT:    s_cselect_b32 s9, s0, 0
-; GFX10-NEXT:    s_mov_b32 s10, 0x80000000
+; GFX10-NEXT:    s_brev_b32 s10, 1
 ; GFX10-NEXT:    s_sub_i32 s9, s8, s9
 ; GFX10-NEXT:    s_cmp_lt_i32 s0, 0
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
@@ -2560,7 +2560,7 @@ define <5 x i32> @v_saddsat_v5i32(<5 x i32> %lhs, <5 x i32> %rhs) {
 ; GFX6-LABEL: v_saddsat_v5i32:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX6-NEXT:    s_brev_b32 s5, 1
 ; GFX6-NEXT:    v_min_i32_e32 v12, 0, v0
 ; GFX6-NEXT:    v_sub_i32_e32 v12, vcc, s5, v12
 ; GFX6-NEXT:    s_brev_b32 s4, -2
@@ -2582,7 +2582,7 @@ define <5 x i32> @v_saddsat_v5i32(<5 x i32> %lhs, <5 x i32> %rhs) {
 ; GFX6-NEXT:    v_sub_i32_e32 v5, vcc, s4, v5
 ; GFX6-NEXT:    v_max_i32_e32 v6, v6, v7
 ; GFX6-NEXT:    v_min_i32_e32 v5, v6, v5
-; GFX6-NEXT:    v_mov_b32_e32 v13, 0x80000000
+; GFX6-NEXT:    v_bfrev_b32_e32 v13, 1
 ; GFX6-NEXT:    v_min_i32_e32 v6, 0, v3
 ; GFX6-NEXT:    v_sub_i32_e32 v6, vcc, v13, v6
 ; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v5
@@ -2604,7 +2604,7 @@ define <5 x i32> @v_saddsat_v5i32(<5 x i32> %lhs, <5 x i32> %rhs) {
 ; GFX8-LABEL: v_saddsat_v5i32:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX8-NEXT:    s_brev_b32 s5, 1
 ; GFX8-NEXT:    v_min_i32_e32 v12, 0, v0
 ; GFX8-NEXT:    v_sub_u32_e32 v12, vcc, s5, v12
 ; GFX8-NEXT:    s_brev_b32 s4, -2
@@ -2626,7 +2626,7 @@ define <5 x i32> @v_saddsat_v5i32(<5 x i32> %lhs, <5 x i32> %rhs) {
 ; GFX8-NEXT:    v_sub_u32_e32 v5, vcc, s4, v5
 ; GFX8-NEXT:    v_max_i32_e32 v6, v6, v7
 ; GFX8-NEXT:    v_min_i32_e32 v5, v6, v5
-; GFX8-NEXT:    v_mov_b32_e32 v13, 0x80000000
+; GFX8-NEXT:    v_bfrev_b32_e32 v13, 1
 ; GFX8-NEXT:    v_min_i32_e32 v6, 0, v3
 ; GFX8-NEXT:    v_sub_u32_e32 v6, vcc, v13, v6
 ; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v5
@@ -2648,7 +2648,7 @@ define <5 x i32> @v_saddsat_v5i32(<5 x i32> %lhs, <5 x i32> %rhs) {
 ; GFX9-LABEL: v_saddsat_v5i32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX9-NEXT:    s_brev_b32 s5, 1
 ; GFX9-NEXT:    v_min_i32_e32 v12, 0, v0
 ; GFX9-NEXT:    v_sub_u32_e32 v12, s5, v12
 ; GFX9-NEXT:    s_brev_b32 s4, -2
@@ -2670,7 +2670,7 @@ define <5 x i32> @v_saddsat_v5i32(<5 x i32> %lhs, <5 x i32> %rhs) {
 ; GFX9-NEXT:    v_sub_u32_e32 v5, s4, v5
 ; GFX9-NEXT:    v_max_i32_e32 v6, v6, v7
 ; GFX9-NEXT:    v_min_i32_e32 v5, v6, v5
-; GFX9-NEXT:    v_mov_b32_e32 v13, 0x80000000
+; GFX9-NEXT:    v_bfrev_b32_e32 v13, 1
 ; GFX9-NEXT:    v_min_i32_e32 v6, 0, v3
 ; GFX9-NEXT:    v_sub_u32_e32 v6, v13, v6
 ; GFX9-NEXT:    v_add_u32_e32 v2, v2, v5
@@ -2694,10 +2694,10 @@ define <5 x i32> @v_saddsat_v5i32(<5 x i32> %lhs, <5 x i32> %rhs) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_min_i32_e32 v13, 0, v1
-; GFX10-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX10-NEXT:    s_brev_b32 s5, 1
 ; GFX10-NEXT:    v_min_i32_e32 v10, 0, v0
 ; GFX10-NEXT:    v_min_i32_e32 v16, 0, v2
-; GFX10-NEXT:    v_mov_b32_e32 v15, 0x80000000
+; GFX10-NEXT:    v_bfrev_b32_e32 v15, 1
 ; GFX10-NEXT:    v_sub_nc_u32_e32 v13, s5, v13
 ; GFX10-NEXT:    v_min_i32_e32 v17, 0, v4
 ; GFX10-NEXT:    v_sub_nc_u32_e32 v10, s5, v10
@@ -2746,7 +2746,7 @@ define amdgpu_ps <5 x i32> @s_saddsat_v5i32(<5 x i32> inreg %lhs, <5 x i32> inre
 ; GFX6-NEXT:    s_cselect_b32 s12, s0, 0
 ; GFX6-NEXT:    s_sub_i32 s12, s10, s12
 ; GFX6-NEXT:    s_cmp_lt_i32 s0, 0
-; GFX6-NEXT:    s_mov_b32 s11, 0x80000000
+; GFX6-NEXT:    s_brev_b32 s11, 1
 ; GFX6-NEXT:    s_cselect_b32 s13, s0, 0
 ; GFX6-NEXT:    s_sub_i32 s13, s11, s13
 ; GFX6-NEXT:    s_cmp_gt_i32 s13, s5
@@ -2807,7 +2807,7 @@ define amdgpu_ps <5 x i32> @s_saddsat_v5i32(<5 x i32> inreg %lhs, <5 x i32> inre
 ; GFX8-NEXT:    s_cselect_b32 s12, s0, 0
 ; GFX8-NEXT:    s_sub_i32 s12, s10, s12
 ; GFX8-NEXT:    s_cmp_lt_i32 s0, 0
-; GFX8-NEXT:    s_mov_b32 s11, 0x80000000
+; GFX8-NEXT:    s_brev_b32 s11, 1
 ; GFX8-NEXT:    s_cselect_b32 s13, s0, 0
 ; GFX8-NEXT:    s_sub_i32 s13, s11, s13
 ; GFX8-NEXT:    s_cmp_gt_i32 s13, s5
@@ -2868,7 +2868,7 @@ define amdgpu_ps <5 x i32> @s_saddsat_v5i32(<5 x i32> inreg %lhs, <5 x i32> inre
 ; GFX9-NEXT:    s_cselect_b32 s12, s0, 0
 ; GFX9-NEXT:    s_sub_i32 s12, s10, s12
 ; GFX9-NEXT:    s_cmp_lt_i32 s0, 0
-; GFX9-NEXT:    s_mov_b32 s11, 0x80000000
+; GFX9-NEXT:    s_brev_b32 s11, 1
 ; GFX9-NEXT:    s_cselect_b32 s13, s0, 0
 ; GFX9-NEXT:    s_sub_i32 s13, s11, s13
 ; GFX9-NEXT:    s_cmp_gt_i32 s13, s5
@@ -2927,7 +2927,7 @@ define amdgpu_ps <5 x i32> @s_saddsat_v5i32(<5 x i32> inreg %lhs, <5 x i32> inre
 ; GFX10-NEXT:    s_cmp_gt_i32 s0, 0
 ; GFX10-NEXT:    s_brev_b32 s10, -2
 ; GFX10-NEXT:    s_cselect_b32 s11, s0, 0
-; GFX10-NEXT:    s_mov_b32 s12, 0x80000000
+; GFX10-NEXT:    s_brev_b32 s12, 1
 ; GFX10-NEXT:    s_sub_i32 s11, s10, s11
 ; GFX10-NEXT:    s_cmp_lt_i32 s0, 0
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
@@ -2991,7 +2991,7 @@ define <16 x i32> @v_saddsat_v16i32(<16 x i32> %lhs, <16 x i32> %rhs) {
 ; GFX6-LABEL: v_saddsat_v16i32:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT:    s_mov_b32 s4, 0x80000000
+; GFX6-NEXT:    s_brev_b32 s4, 1
 ; GFX6-NEXT:    v_min_i32_e32 v32, 0, v0
 ; GFX6-NEXT:    v_sub_i32_e32 v32, vcc, s4, v32
 ; GFX6-NEXT:    v_max_i32_e32 v16, v32, v16
@@ -3014,7 +3014,7 @@ define <16 x i32> @v_saddsat_v16i32(<16 x i32> %lhs, <16 x i32> %rhs) {
 ; GFX6-NEXT:    v_sub_i32_e32 v17, vcc, s5, v17
 ; GFX6-NEXT:    v_min_i32_e32 v16, v16, v17
 ; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v16
-; GFX6-NEXT:    v_mov_b32_e32 v16, 0x80000000
+; GFX6-NEXT:    v_bfrev_b32_e32 v16, 1
 ; GFX6-NEXT:    v_min_i32_e32 v17, 0, v3
 ; GFX6-NEXT:    v_sub_i32_e32 v17, vcc, v16, v17
 ; GFX6-NEXT:    v_max_i32_e32 v17, v17, v19
@@ -3112,7 +3112,7 @@ define <16 x i32> @v_saddsat_v16i32(<16 x i32> %lhs, <16 x i32> %rhs) {
 ; GFX8-LABEL: v_saddsat_v16i32:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    s_mov_b32 s4, 0x80000000
+; GFX8-NEXT:    s_brev_b32 s4, 1
 ; GFX8-NEXT:    v_min_i32_e32 v32, 0, v0
 ; GFX8-NEXT:    v_sub_u32_e32 v32, vcc, s4, v32
 ; GFX8-NEXT:    v_max_i32_e32 v16, v32, v16
@@ -3135,7 +3135,7 @@ define <16 x i32> @v_saddsat_v16i32(<16 x i32> %lhs, <16 x i32> %rhs) {
 ; GFX8-NEXT:    v_sub_u32_e32 v17, vcc, s5, v17
 ; GFX8-NEXT:    v_min_i32_e32 v16, v16, v17
 ; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v16
-; GFX8-NEXT:    v_mov_b32_e32 v16, 0x80000000
+; GFX8-NEXT:    v_bfrev_b32_e32 v16, 1
 ; GFX8-NEXT:    v_min_i32_e32 v17, 0, v3
 ; GFX8-NEXT:    v_sub_u32_e32 v17, vcc, v16, v17
 ; GFX8-NEXT:    v_max_i32_e32 v17, v17, v19
@@ -3233,7 +3233,7 @@ define <16 x i32> @v_saddsat_v16i32(<16 x i32> %lhs, <16 x i32> %rhs) {
 ; GFX9-LABEL: v_saddsat_v16i32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    s_mov_b32 s4, 0x80000000
+; GFX9-NEXT:    s_brev_b32 s4, 1
 ; GFX9-NEXT:    v_min_i32_e32 v32, 0, v0
 ; GFX9-NEXT:    v_sub_u32_e32 v32, s4, v32
 ; GFX9-NEXT:    v_max_i32_e32 v16, v32, v16
@@ -3256,7 +3256,7 @@ define <16 x i32> @v_saddsat_v16i32(<16 x i32> %lhs, <16 x i32> %rhs) {
 ; GFX9-NEXT:    v_sub_u32_e32 v17, s5, v17
 ; GFX9-NEXT:    v_min_i32_e32 v16, v16, v17
 ; GFX9-NEXT:    v_add_u32_e32 v2, v2, v16
-; GFX9-NEXT:    v_mov_b32_e32 v16, 0x80000000
+; GFX9-NEXT:    v_bfrev_b32_e32 v16, 1
 ; GFX9-NEXT:    v_min_i32_e32 v17, 0, v3
 ; GFX9-NEXT:    v_sub_u32_e32 v17, v16, v17
 ; GFX9-NEXT:    v_max_i32_e32 v17, v17, v19
@@ -3356,7 +3356,7 @@ define <16 x i32> @v_saddsat_v16i32(<16 x i32> %lhs, <16 x i32> %rhs) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_min_i32_e32 v32, 0, v0
-; GFX10-NEXT:    s_mov_b32 s4, 0x80000000
+; GFX10-NEXT:    s_brev_b32 s4, 1
 ; GFX10-NEXT:    v_max_i32_e32 v33, 0, v0
 ; GFX10-NEXT:    s_brev_b32 s5, -2
 ; GFX10-NEXT:    v_min_i32_e32 v36, 0, v2
@@ -3367,7 +3367,7 @@ define <16 x i32> @v_saddsat_v16i32(<16 x i32> %lhs, <16 x i32> %rhs) {
 ; GFX10-NEXT:    v_sub_nc_u32_e32 v36, s4, v36
 ; GFX10-NEXT:    v_max_i32_e32 v16, v35, v16
 ; GFX10-NEXT:    v_sub_nc_u32_e32 v32, s4, v32
-; GFX10-NEXT:    v_mov_b32_e32 v35, 0x80000000
+; GFX10-NEXT:    v_bfrev_b32_e32 v35, 1
 ; GFX10-NEXT:    v_min_i32_e32 v38, 0, v3
 ; GFX10-NEXT:    v_max_i32_e32 v18, v36, v18
 ; GFX10-NEXT:    v_min_i32_e32 v16, v16, v33
@@ -3485,7 +3485,7 @@ define amdgpu_ps <16 x i32> @s_saddsat_v16i32(<16 x i32> inreg %lhs, <16 x i32>
 ; GFX6-NEXT:    s_cselect_b32 s34, s0, 0
 ; GFX6-NEXT:    s_sub_i32 s34, s32, s34
 ; GFX6-NEXT:    s_cmp_lt_i32 s0, 0
-; GFX6-NEXT:    s_mov_b32 s33, 0x80000000
+; GFX6-NEXT:    s_brev_b32 s33, 1
 ; GFX6-NEXT:    s_cselect_b32 s35, s0, 0
 ; GFX6-NEXT:    s_sub_i32 s35, s33, s35
 ; GFX6-NEXT:    s_cmp_gt_i32 s35, s16
@@ -3667,7 +3667,7 @@ define amdgpu_ps <16 x i32> @s_saddsat_v16i32(<16 x i32> inreg %lhs, <16 x i32>
 ; GFX8-NEXT:    s_cselect_b32 s34, s0, 0
 ; GFX8-NEXT:    s_sub_i32 s34, s32, s34
 ; GFX8-NEXT:    s_cmp_lt_i32 s0, 0
-; GFX8-NEXT:    s_mov_b32 s33, 0x80000000
+; GFX8-NEXT:    s_brev_b32 s33, 1
 ; GFX8-NEXT:    s_cselect_b32 s35, s0, 0
 ; GFX8-NEXT:    s_sub_i32 s35, s33, s35
 ; GFX8-NEXT:    s_cmp_gt_i32 s35, s16
@@ -3849,7 +3849,7 @@ define amdgpu_ps <16 x i32> @s_saddsat_v16i32(<16 x i32> inreg %lhs, <16 x i32>
 ; GFX9-NEXT:    s_cselect_b32 s34, s0, 0
 ; GFX9-NEXT:    s_sub_i32 s34, s32, s34
 ; GFX9-NEXT:    s_cmp_lt_i32 s0, 0
-; GFX9-NEXT:    s_mov_b32 s33, 0x80000000
+; GFX9-NEXT:    s_brev_b32 s33, 1
 ; GFX9-NEXT:    s_cselect_b32 s35, s0, 0
 ; GFX9-NEXT:    s_sub_i32 s35, s33, s35
 ; GFX9-NEXT:    s_cmp_gt_i32 s35, s16
@@ -4029,7 +4029,7 @@ define amdgpu_ps <16 x i32> @s_saddsat_v16i32(<16 x i32> inreg %lhs, <16 x i32>
 ; GFX10-NEXT:    s_cmp_gt_i32 s0, 0
 ; GFX10-NEXT:    s_brev_b32 s32, -2
 ; GFX10-NEXT:    s_cselect_b32 s33, s0, 0
-; GFX10-NEXT:    s_mov_b32 s34, 0x80000000
+; GFX10-NEXT:    s_brev_b32 s34, 1
 ; GFX10-NEXT:    s_sub_i32 s46, s32, s33
 ; GFX10-NEXT:    s_cmp_lt_i32 s0, 0
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
@@ -4295,7 +4295,7 @@ define amdgpu_ps i16 @s_saddsat_i16(i16 inreg %lhs, i16 inreg %rhs) {
 ; GFX8-NEXT:    s_sub_i32 s4, 0x7fff, s4
 ; GFX8-NEXT:    s_cmp_lt_i32 s2, s3
 ; GFX8-NEXT:    s_cselect_b32 s2, s2, s3
-; GFX8-NEXT:    s_sub_i32 s2, 0x8000, s2
+; GFX8-NEXT:    s_sub_i32 s2, 0xffff8000, s2
 ; GFX8-NEXT:    s_sext_i32_i16 s2, s2
 ; GFX8-NEXT:    s_sext_i32_i16 s1, s1
 ; GFX8-NEXT:    s_cmp_gt_i32 s2, s1
@@ -4316,7 +4316,7 @@ define amdgpu_ps i16 @s_saddsat_i16(i16 inreg %lhs, i16 inreg %rhs) {
 ; GFX9-NEXT:    s_sub_i32 s4, 0x7fff, s4
 ; GFX9-NEXT:    s_cmp_lt_i32 s2, s3
 ; GFX9-NEXT:    s_cselect_b32 s2, s2, s3
-; GFX9-NEXT:    s_sub_i32 s2, 0x8000, s2
+; GFX9-NEXT:    s_sub_i32 s2, 0xffff8000, s2
 ; GFX9-NEXT:    s_sext_i32_i16 s2, s2
 ; GFX9-NEXT:    s_sext_i32_i16 s1, s1
 ; GFX9-NEXT:    s_cmp_gt_i32 s2, s1
@@ -4339,7 +4339,7 @@ define amdgpu_ps i16 @s_saddsat_i16(i16 inreg %lhs, i16 inreg %rhs) {
 ; GFX10-NEXT:    s_sub_i32 s4, 0x7fff, s4
 ; GFX10-NEXT:    s_cmp_lt_i32 s3, s2
 ; GFX10-NEXT:    s_cselect_b32 s2, s3, s2
-; GFX10-NEXT:    s_sub_i32 s2, 0x8000, s2
+; GFX10-NEXT:    s_sub_i32 s2, 0xffff8000, s2
 ; GFX10-NEXT:    s_sext_i32_i16 s2, s2
 ; GFX10-NEXT:    s_cmp_gt_i32 s2, s1
 ; GFX10-NEXT:    s_cselect_b32 s1, s2, s1
@@ -4379,7 +4379,7 @@ define amdgpu_ps half @saddsat_i16_sv(i16 inreg %lhs, i16 %rhs) {
 ; GFX8-NEXT:    s_sub_i32 s3, 0x7fff, s3
 ; GFX8-NEXT:    s_cmp_lt_i32 s1, s2
 ; GFX8-NEXT:    s_cselect_b32 s1, s1, s2
-; GFX8-NEXT:    s_sub_i32 s1, 0x8000, s1
+; GFX8-NEXT:    s_sub_i32 s1, 0xffff8000, s1
 ; GFX8-NEXT:    v_max_i16_e32 v0, s1, v0
 ; GFX8-NEXT:    v_min_i16_e32 v0, s3, v0
 ; GFX8-NEXT:    v_add_u16_e32 v0, s0, v0
@@ -4394,7 +4394,7 @@ define amdgpu_ps half @saddsat_i16_sv(i16 inreg %lhs, i16 %rhs) {
 ; GFX9-NEXT:    s_sub_i32 s3, 0x7fff, s3
 ; GFX9-NEXT:    s_cmp_lt_i32 s1, s2
 ; GFX9-NEXT:    s_cselect_b32 s1, s1, s2
-; GFX9-NEXT:    s_sub_i32 s1, 0x8000, s1
+; GFX9-NEXT:    s_sub_i32 s1, 0xffff8000, s1
 ; GFX9-NEXT:    v_max_i16_e32 v0, s1, v0
 ; GFX9-NEXT:    v_min_i16_e32 v0, s3, v0
 ; GFX9-NEXT:    v_add_u16_e32 v0, s0, v0
@@ -4410,7 +4410,7 @@ define amdgpu_ps half @saddsat_i16_sv(i16 inreg %lhs, i16 %rhs) {
 ; GFX10-NEXT:    s_sub_i32 s3, 0x7fff, s3
 ; GFX10-NEXT:    s_cmp_lt_i32 s1, s2
 ; GFX10-NEXT:    s_cselect_b32 s1, s1, s2
-; GFX10-NEXT:    s_sub_i32 s1, 0x8000, s1
+; GFX10-NEXT:    s_sub_i32 s1, 0xffff8000, s1
 ; GFX10-NEXT:    v_max_i16_e64 v0, s1, v0
 ; GFX10-NEXT:    v_min_i16_e64 v0, v0, s3
 ; GFX10-NEXT:    v_add_nc_u16_e64 v0, s0, v0
@@ -4478,7 +4478,7 @@ define <2 x i16> @v_saddsat_v2i16(<2 x i16> %lhs, <2 x i16> %rhs) {
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; GFX6-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX6-NEXT:    s_brev_b32 s5, 1
 ; GFX6-NEXT:    v_min_i32_e32 v5, 0, v0
 ; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX6-NEXT:    v_sub_i32_e32 v5, vcc, s5, v5
@@ -4504,7 +4504,7 @@ define <2 x i16> @v_saddsat_v2i16(<2 x i16> %lhs, <2 x i16> %rhs) {
 ; GFX8-LABEL: v_saddsat_v2i16:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    s_mov_b32 s5, 0x8000
+; GFX8-NEXT:    s_movk_i32 s5, 0x8000
 ; GFX8-NEXT:    v_min_i16_e32 v4, 0, v0
 ; GFX8-NEXT:    v_sub_u16_e32 v4, s5, v4
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
@@ -4527,7 +4527,7 @@ define <2 x i16> @v_saddsat_v2i16(<2 x i16> %lhs, <2 x i16> %rhs) {
 ; GFX9-LABEL: v_saddsat_v2i16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    s_mov_b32 s5, 0xffff8000
+; GFX9-NEXT:    s_movk_i32 s5, 0x8000
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s6, 0, 0
 ; GFX9-NEXT:    s_movk_i32 s4, 0x7fff
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s5, s5, s5
@@ -4546,7 +4546,7 @@ define <2 x i16> @v_saddsat_v2i16(<2 x i16> %lhs, <2 x i16> %rhs) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s4, 0, 0
-; GFX10-NEXT:    s_mov_b32 s5, 0xffff8000
+; GFX10-NEXT:    s_movk_i32 s5, 0x8000
 ; GFX10-NEXT:    v_pk_min_i16 v2, v0, s4
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s5, s5, s5
 ; GFX10-NEXT:    v_pk_max_i16 v3, v0, s4
@@ -4573,7 +4573,7 @@ define amdgpu_ps i32 @s_saddsat_v2i16(<2 x i16> inreg %lhs, <2 x i16> inreg %rhs
 ; GFX6-NEXT:    s_cselect_b32 s6, s0, 0
 ; GFX6-NEXT:    s_sub_i32 s6, s4, s6
 ; GFX6-NEXT:    s_cmp_lt_i32 s0, 0
-; GFX6-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX6-NEXT:    s_brev_b32 s5, 1
 ; GFX6-NEXT:    s_cselect_b32 s7, s0, 0
 ; GFX6-NEXT:    s_sub_i32 s7, s5, s7
 ; GFX6-NEXT:    s_cmp_gt_i32 s7, s2
@@ -4614,7 +4614,7 @@ define amdgpu_ps i32 @s_saddsat_v2i16(<2 x i16> inreg %lhs, <2 x i16> inreg %rhs
 ; GFX8-NEXT:    s_cselect_b32 s8, s6, s7
 ; GFX8-NEXT:    s_sub_i32 s8, s4, s8
 ; GFX8-NEXT:    s_cmp_lt_i32 s6, s7
-; GFX8-NEXT:    s_mov_b32 s5, 0x8000
+; GFX8-NEXT:    s_movk_i32 s5, 0x8000
 ; GFX8-NEXT:    s_cselect_b32 s6, s6, s7
 ; GFX8-NEXT:    s_sub_i32 s6, s5, s6
 ; GFX8-NEXT:    s_sext_i32_i16 s6, s6
@@ -4669,7 +4669,7 @@ define amdgpu_ps i32 @s_saddsat_v2i16(<2 x i16> inreg %lhs, <2 x i16> inreg %rhs
 ; GFX9-NEXT:    s_cmp_lt_i32 s5, s7
 ; GFX9-NEXT:    s_cselect_b32 s5, s5, s7
 ; GFX9-NEXT:    s_cmp_lt_i32 s6, s4
-; GFX9-NEXT:    s_mov_b32 s3, 0xffff8000
+; GFX9-NEXT:    s_movk_i32 s3, 0x8000
 ; GFX9-NEXT:    s_cselect_b32 s4, s6, s4
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s5, s4
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s3, s3
@@ -4726,7 +4726,7 @@ define amdgpu_ps i32 @s_saddsat_v2i16(<2 x i16> inreg %lhs, <2 x i16> inreg %rhs
 ; GFX10-NEXT:    s_cmp_lt_i32 s3, s5
 ; GFX10-NEXT:    s_cselect_b32 s3, s3, s5
 ; GFX10-NEXT:    s_cmp_lt_i32 s4, s2
-; GFX10-NEXT:    s_mov_b32 s5, 0xffff8000
+; GFX10-NEXT:    s_movk_i32 s5, 0x8000
 ; GFX10-NEXT:    s_cselect_b32 s2, s4, s2
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s5, s5
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s3, s2
@@ -4774,7 +4774,7 @@ define amdgpu_ps float @saddsat_v2i16_sv(<2 x i16> inreg %lhs, <2 x i16> %rhs) {
 ; GFX6-NEXT:    s_cselect_b32 s4, s0, 0
 ; GFX6-NEXT:    s_sub_i32 s4, s2, s4
 ; GFX6-NEXT:    s_cmp_lt_i32 s0, 0
-; GFX6-NEXT:    s_mov_b32 s3, 0x80000000
+; GFX6-NEXT:    s_brev_b32 s3, 1
 ; GFX6-NEXT:    s_cselect_b32 s5, s0, 0
 ; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GFX6-NEXT:    s_sub_i32 s5, s3, s5
@@ -4811,7 +4811,7 @@ define amdgpu_ps float @saddsat_v2i16_sv(<2 x i16> inreg %lhs, <2 x i16> %rhs) {
 ; GFX8-NEXT:    s_cselect_b32 s6, s4, s5
 ; GFX8-NEXT:    s_sub_i32 s6, s2, s6
 ; GFX8-NEXT:    s_cmp_lt_i32 s4, s5
-; GFX8-NEXT:    s_mov_b32 s3, 0x8000
+; GFX8-NEXT:    s_movk_i32 s3, 0x8000
 ; GFX8-NEXT:    s_cselect_b32 s4, s4, s5
 ; GFX8-NEXT:    s_sub_i32 s4, s3, s4
 ; GFX8-NEXT:    v_max_i16_e32 v1, s4, v0
@@ -4853,7 +4853,7 @@ define amdgpu_ps float @saddsat_v2i16_sv(<2 x i16> inreg %lhs, <2 x i16> %rhs) {
 ; GFX9-NEXT:    s_cmp_lt_i32 s4, s6
 ; GFX9-NEXT:    s_cselect_b32 s4, s4, s6
 ; GFX9-NEXT:    s_cmp_lt_i32 s5, s3
-; GFX9-NEXT:    s_mov_b32 s2, 0xffff8000
+; GFX9-NEXT:    s_movk_i32 s2, 0x8000
 ; GFX9-NEXT:    s_cselect_b32 s3, s5, s3
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s4, s3
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s2
@@ -4890,7 +4890,7 @@ define amdgpu_ps float @saddsat_v2i16_sv(<2 x i16> inreg %lhs, <2 x i16> %rhs) {
 ; GFX10-NEXT:    s_cmp_lt_i32 s2, s4
 ; GFX10-NEXT:    s_cselect_b32 s2, s2, s4
 ; GFX10-NEXT:    s_cmp_lt_i32 s3, s1
-; GFX10-NEXT:    s_mov_b32 s4, 0xffff8000
+; GFX10-NEXT:    s_movk_i32 s4, 0x8000
 ; GFX10-NEXT:    s_cselect_b32 s1, s3, s1
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s4, s4
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s2, s1
@@ -4913,7 +4913,7 @@ define amdgpu_ps float @saddsat_v2i16_vs(<2 x i16> %lhs, <2 x i16> inreg %rhs) {
 ; GFX6-LABEL: saddsat_v2i16_vs:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; GFX6-NEXT:    s_mov_b32 s3, 0x80000000
+; GFX6-NEXT:    s_brev_b32 s3, 1
 ; GFX6-NEXT:    v_min_i32_e32 v3, 0, v0
 ; GFX6-NEXT:    s_lshl_b32 s0, s0, 16
 ; GFX6-NEXT:    v_sub_i32_e32 v3, vcc, s3, v3
@@ -4943,7 +4943,7 @@ define amdgpu_ps float @saddsat_v2i16_vs(<2 x i16> %lhs, <2 x i16> inreg %rhs) {
 ;
 ; GFX8-LABEL: saddsat_v2i16_vs:
 ; GFX8:       ; %bb.0:
-; GFX8-NEXT:    s_mov_b32 s3, 0x8000
+; GFX8-NEXT:    s_movk_i32 s3, 0x8000
 ; GFX8-NEXT:    v_min_i16_e32 v3, 0, v0
 ; GFX8-NEXT:    v_sub_u16_e32 v3, s3, v3
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
@@ -4966,7 +4966,7 @@ define amdgpu_ps float @saddsat_v2i16_vs(<2 x i16> %lhs, <2 x i16> inreg %rhs) {
 ;
 ; GFX9-LABEL: saddsat_v2i16_vs:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_mov_b32 s2, 0xffff8000
+; GFX9-NEXT:    s_movk_i32 s2, 0x8000
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s3, 0, 0
 ; GFX9-NEXT:    s_movk_i32 s1, 0x7fff
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s2
@@ -4983,7 +4983,7 @@ define amdgpu_ps float @saddsat_v2i16_vs(<2 x i16> %lhs, <2 x i16> inreg %rhs) {
 ; GFX10-LABEL: saddsat_v2i16_vs:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s1, 0, 0
-; GFX10-NEXT:    s_mov_b32 s2, 0xffff8000
+; GFX10-NEXT:    s_movk_i32 s2, 0x8000
 ; GFX10-NEXT:    v_pk_min_i16 v1, v0, s1
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s2, s2
 ; GFX10-NEXT:    v_pk_max_i16 v2, v0, s1
@@ -5017,7 +5017,7 @@ define <2 x float> @v_saddsat_v4i16(<4 x i16> %lhs, <4 x i16> %rhs) {
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; GFX6-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX6-NEXT:    s_brev_b32 s5, 1
 ; GFX6-NEXT:    v_min_i32_e32 v10, 0, v0
 ; GFX6-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
 ; GFX6-NEXT:    v_sub_i32_e32 v10, vcc, s5, v10
@@ -5046,7 +5046,7 @@ define <2 x float> @v_saddsat_v4i16(<4 x i16> %lhs, <4 x i16> %rhs) {
 ; GFX6-NEXT:    v_max_i32_e32 v4, v6, v4
 ; GFX6-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; GFX6-NEXT:    v_min_i32_e32 v4, v4, v5
-; GFX6-NEXT:    v_mov_b32_e32 v11, 0x80000000
+; GFX6-NEXT:    v_bfrev_b32_e32 v11, 1
 ; GFX6-NEXT:    v_min_i32_e32 v6, 0, v3
 ; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
 ; GFX6-NEXT:    v_max_i32_e32 v5, 0, v3
@@ -5074,7 +5074,7 @@ define <2 x float> @v_saddsat_v4i16(<4 x i16> %lhs, <4 x i16> %rhs) {
 ; GFX8-LABEL: v_saddsat_v4i16:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    s_mov_b32 s5, 0x8000
+; GFX8-NEXT:    s_movk_i32 s5, 0x8000
 ; GFX8-NEXT:    v_min_i16_e32 v7, 0, v0
 ; GFX8-NEXT:    v_sub_u16_e32 v7, s5, v7
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v4, 16, v0
@@ -5113,7 +5113,7 @@ define <2 x float> @v_saddsat_v4i16(<4 x i16> %lhs, <4 x i16> %rhs) {
 ; GFX9-LABEL: v_saddsat_v4i16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    s_mov_b32 s5, 0xffff8000
+; GFX9-NEXT:    s_movk_i32 s5, 0x8000
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s6, 0, 0
 ; GFX9-NEXT:    s_movk_i32 s4, 0x7fff
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s5, s5, s5
@@ -5139,7 +5139,7 @@ define <2 x float> @v_saddsat_v4i16(<4 x i16> %lhs, <4 x i16> %rhs) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s5, 0, 0
-; GFX10-NEXT:    s_mov_b32 s6, 0xffff8000
+; GFX10-NEXT:    s_movk_i32 s6, 0x8000
 ; GFX10-NEXT:    v_pk_min_i16 v4, v0, s5
 ; GFX10-NEXT:    v_pk_min_i16 v5, v1, s5
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s6, s6
@@ -5174,7 +5174,7 @@ define amdgpu_ps <2 x i32> @s_saddsat_v4i16(<4 x i16> inreg %lhs, <4 x i16> inre
 ; GFX6-NEXT:    s_cselect_b32 s10, s0, 0
 ; GFX6-NEXT:    s_sub_i32 s10, s8, s10
 ; GFX6-NEXT:    s_cmp_lt_i32 s0, 0
-; GFX6-NEXT:    s_mov_b32 s9, 0x80000000
+; GFX6-NEXT:    s_brev_b32 s9, 1
 ; GFX6-NEXT:    s_cselect_b32 s11, s0, 0
 ; GFX6-NEXT:    s_sub_i32 s11, s9, s11
 ; GFX6-NEXT:    s_cmp_gt_i32 s11, s4
@@ -5249,7 +5249,7 @@ define amdgpu_ps <2 x i32> @s_saddsat_v4i16(<4 x i16> inreg %lhs, <4 x i16> inre
 ; GFX8-NEXT:    s_cselect_b32 s12, s10, s11
 ; GFX8-NEXT:    s_sub_i32 s12, s8, s12
 ; GFX8-NEXT:    s_cmp_lt_i32 s10, s11
-; GFX8-NEXT:    s_mov_b32 s9, 0x8000
+; GFX8-NEXT:    s_movk_i32 s9, 0x8000
 ; GFX8-NEXT:    s_cselect_b32 s10, s10, s11
 ; GFX8-NEXT:    s_sub_i32 s10, s9, s10
 ; GFX8-NEXT:    s_sext_i32_i16 s10, s10
@@ -5340,7 +5340,7 @@ define amdgpu_ps <2 x i32> @s_saddsat_v4i16(<4 x i16> inreg %lhs, <4 x i16> inre
 ; GFX9-NEXT:    s_cmp_lt_i32 s7, s9
 ; GFX9-NEXT:    s_cselect_b32 s7, s7, s9
 ; GFX9-NEXT:    s_cmp_lt_i32 s8, s6
-; GFX9-NEXT:    s_mov_b32 s5, 0xffff8000
+; GFX9-NEXT:    s_movk_i32 s5, 0x8000
 ; GFX9-NEXT:    s_cselect_b32 s8, s8, s6
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s5, s5, s5
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s7, s7, s8
@@ -5431,7 +5431,7 @@ define amdgpu_ps <2 x i32> @s_saddsat_v4i16(<4 x i16> inreg %lhs, <4 x i16> inre
 ; GFX10-NEXT:    s_cmp_gt_i32 s6, s4
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s9, s9, s9
 ; GFX10-NEXT:    s_cselect_b32 s10, s6, s4
-; GFX10-NEXT:    s_mov_b32 s12, 0xffff8000
+; GFX10-NEXT:    s_movk_i32 s12, 0x8000
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s8, s8, s10
 ; GFX10-NEXT:    s_lshr_b32 s10, s9, 16
 ; GFX10-NEXT:    s_lshr_b32 s11, s8, 16
@@ -5538,7 +5538,7 @@ define <3 x float> @v_saddsat_v6i16(<6 x i16> %lhs, <6 x i16> %rhs) {
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; GFX6-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX6-NEXT:    s_brev_b32 s5, 1
 ; GFX6-NEXT:    v_min_i32_e32 v14, 0, v0
 ; GFX6-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
 ; GFX6-NEXT:    v_sub_i32_e32 v14, vcc, s5, v14
@@ -5567,7 +5567,7 @@ define <3 x float> @v_saddsat_v6i16(<6 x i16> %lhs, <6 x i16> %rhs) {
 ; GFX6-NEXT:    v_max_i32_e32 v6, v8, v6
 ; GFX6-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; GFX6-NEXT:    v_min_i32_e32 v6, v6, v7
-; GFX6-NEXT:    v_mov_b32_e32 v15, 0x80000000
+; GFX6-NEXT:    v_bfrev_b32_e32 v15, 1
 ; GFX6-NEXT:    v_min_i32_e32 v8, 0, v3
 ; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v6
 ; GFX6-NEXT:    v_max_i32_e32 v7, 0, v3
@@ -5619,7 +5619,7 @@ define <3 x float> @v_saddsat_v6i16(<6 x i16> %lhs, <6 x i16> %rhs) {
 ; GFX8-LABEL: v_saddsat_v6i16:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    s_mov_b32 s5, 0x8000
+; GFX8-NEXT:    s_movk_i32 s5, 0x8000
 ; GFX8-NEXT:    v_min_i16_e32 v11, 0, v0
 ; GFX8-NEXT:    v_sub_u16_e32 v11, s5, v11
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v6, 16, v0
@@ -5646,7 +5646,7 @@ define <3 x float> @v_saddsat_v6i16(<6 x i16> %lhs, <6 x i16> %rhs) {
 ; GFX8-NEXT:    v_sub_u16_e32 v14, s5, v14
 ; GFX8-NEXT:    v_max_i16_sdwa v4, v14, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
 ; GFX8-NEXT:    v_sub_u16_e32 v13, s4, v13
-; GFX8-NEXT:    v_mov_b32_e32 v12, 0x8000
+; GFX8-NEXT:    v_mov_b32_e32 v12, 0xffff8000
 ; GFX8-NEXT:    v_min_i16_e32 v14, 0, v2
 ; GFX8-NEXT:    v_sub_u16_e32 v14, v12, v14
 ; GFX8-NEXT:    v_min_i16_e32 v4, v4, v13
@@ -5676,7 +5676,7 @@ define <3 x float> @v_saddsat_v6i16(<6 x i16> %lhs, <6 x i16> %rhs) {
 ; GFX9-LABEL: v_saddsat_v6i16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    s_mov_b32 s5, 0xffff8000
+; GFX9-NEXT:    s_movk_i32 s5, 0x8000
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s6, 0, 0
 ; GFX9-NEXT:    s_movk_i32 s4, 0x7fff
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s5, s5, s5
@@ -5709,7 +5709,7 @@ define <3 x float> @v_saddsat_v6i16(<6 x i16> %lhs, <6 x i16> %rhs) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s5, 0, 0
-; GFX10-NEXT:    s_mov_b32 s6, 0xffff8000
+; GFX10-NEXT:    s_movk_i32 s6, 0x8000
 ; GFX10-NEXT:    v_pk_min_i16 v7, v0, s5
 ; GFX10-NEXT:    v_pk_min_i16 v8, v1, s5
 ; GFX10-NEXT:    v_pk_min_i16 v9, v2, s5
@@ -5751,7 +5751,7 @@ define amdgpu_ps <3 x i32> @s_saddsat_v6i16(<6 x i16> inreg %lhs, <6 x i16> inre
 ; GFX6-NEXT:    s_cselect_b32 s14, s0, 0
 ; GFX6-NEXT:    s_sub_i32 s14, s12, s14
 ; GFX6-NEXT:    s_cmp_lt_i32 s0, 0
-; GFX6-NEXT:    s_mov_b32 s13, 0x80000000
+; GFX6-NEXT:    s_brev_b32 s13, 1
 ; GFX6-NEXT:    s_cselect_b32 s15, s0, 0
 ; GFX6-NEXT:    s_sub_i32 s15, s13, s15
 ; GFX6-NEXT:    s_cmp_gt_i32 s15, s6
@@ -5860,7 +5860,7 @@ define amdgpu_ps <3 x i32> @s_saddsat_v6i16(<6 x i16> inreg %lhs, <6 x i16> inre
 ; GFX8-NEXT:    s_cselect_b32 s16, s14, s15
 ; GFX8-NEXT:    s_sub_i32 s16, s12, s16
 ; GFX8-NEXT:    s_cmp_lt_i32 s14, s15
-; GFX8-NEXT:    s_mov_b32 s13, 0x8000
+; GFX8-NEXT:    s_movk_i32 s13, 0x8000
 ; GFX8-NEXT:    s_cselect_b32 s14, s14, s15
 ; GFX8-NEXT:    s_sub_i32 s14, s13, s14
 ; GFX8-NEXT:    s_sext_i32_i16 s14, s14
@@ -5987,7 +5987,7 @@ define amdgpu_ps <3 x i32> @s_saddsat_v6i16(<6 x i16> inreg %lhs, <6 x i16> inre
 ; GFX9-NEXT:    s_cmp_lt_i32 s9, s11
 ; GFX9-NEXT:    s_cselect_b32 s9, s9, s11
 ; GFX9-NEXT:    s_cmp_lt_i32 s10, s8
-; GFX9-NEXT:    s_mov_b32 s7, 0xffff8000
+; GFX9-NEXT:    s_movk_i32 s7, 0x8000
 ; GFX9-NEXT:    s_cselect_b32 s10, s10, s8
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s7, s7, s7
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s9, s9, s10
@@ -6121,7 +6121,7 @@ define amdgpu_ps <3 x i32> @s_saddsat_v6i16(<6 x i16> inreg %lhs, <6 x i16> inre
 ; GFX10-NEXT:    s_cmp_gt_i32 s8, s6
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s11, s11, s11
 ; GFX10-NEXT:    s_cselect_b32 s12, s8, s6
-; GFX10-NEXT:    s_mov_b32 s14, 0xffff8000
+; GFX10-NEXT:    s_movk_i32 s14, 0x8000
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s10, s10, s12
 ; GFX10-NEXT:    s_lshr_b32 s12, s11, 16
 ; GFX10-NEXT:    s_lshr_b32 s13, s10, 16
@@ -6260,7 +6260,7 @@ define <4 x float> @v_saddsat_v8i16(<8 x i16> %lhs, <8 x i16> %rhs) {
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; GFX6-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX6-NEXT:    s_brev_b32 s5, 1
 ; GFX6-NEXT:    v_min_i32_e32 v18, 0, v0
 ; GFX6-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
 ; GFX6-NEXT:    v_sub_i32_e32 v18, vcc, s5, v18
@@ -6289,7 +6289,7 @@ define <4 x float> @v_saddsat_v8i16(<8 x i16> %lhs, <8 x i16> %rhs) {
 ; GFX6-NEXT:    v_max_i32_e32 v8, v10, v8
 ; GFX6-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; GFX6-NEXT:    v_min_i32_e32 v8, v8, v9
-; GFX6-NEXT:    v_mov_b32_e32 v19, 0x80000000
+; GFX6-NEXT:    v_bfrev_b32_e32 v19, 1
 ; GFX6-NEXT:    v_min_i32_e32 v10, 0, v3
 ; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v8
 ; GFX6-NEXT:    v_max_i32_e32 v9, 0, v3
@@ -6365,7 +6365,7 @@ define <4 x float> @v_saddsat_v8i16(<8 x i16> %lhs, <8 x i16> %rhs) {
 ; GFX8-LABEL: v_saddsat_v8i16:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    s_mov_b32 s5, 0x8000
+; GFX8-NEXT:    s_movk_i32 s5, 0x8000
 ; GFX8-NEXT:    v_min_i16_e32 v14, 0, v0
 ; GFX8-NEXT:    v_sub_u16_e32 v14, s5, v14
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v8, 16, v0
@@ -6392,7 +6392,7 @@ define <4 x float> @v_saddsat_v8i16(<8 x i16> %lhs, <8 x i16> %rhs) {
 ; GFX8-NEXT:    v_sub_u16_e32 v17, s5, v17
 ; GFX8-NEXT:    v_max_i16_sdwa v5, v17, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
 ; GFX8-NEXT:    v_sub_u16_e32 v16, s4, v16
-; GFX8-NEXT:    v_mov_b32_e32 v15, 0x8000
+; GFX8-NEXT:    v_mov_b32_e32 v15, 0xffff8000
 ; GFX8-NEXT:    v_min_i16_e32 v17, 0, v2
 ; GFX8-NEXT:    v_sub_u16_e32 v17, v15, v17
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v10, 16, v2
@@ -6438,7 +6438,7 @@ define <4 x float> @v_saddsat_v8i16(<8 x i16> %lhs, <8 x i16> %rhs) {
 ; GFX9-LABEL: v_saddsat_v8i16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    s_mov_b32 s5, 0xffff8000
+; GFX9-NEXT:    s_movk_i32 s5, 0x8000
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s6, 0, 0
 ; GFX9-NEXT:    s_movk_i32 s4, 0x7fff
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s5, s5, s5
@@ -6478,7 +6478,7 @@ define <4 x float> @v_saddsat_v8i16(<8 x i16> %lhs, <8 x i16> %rhs) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s4, 0, 0
-; GFX10-NEXT:    s_mov_b32 s5, 0xffff8000
+; GFX10-NEXT:    s_movk_i32 s5, 0x8000
 ; GFX10-NEXT:    v_pk_min_i16 v8, v0, s4
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s5, s5, s5
 ; GFX10-NEXT:    v_pk_min_i16 v11, v1, s4
@@ -6527,7 +6527,7 @@ define amdgpu_ps <4 x i32> @s_saddsat_v8i16(<8 x i16> inreg %lhs, <8 x i16> inre
 ; GFX6-NEXT:    s_cselect_b32 s18, s0, 0
 ; GFX6-NEXT:    s_sub_i32 s18, s16, s18
 ; GFX6-NEXT:    s_cmp_lt_i32 s0, 0
-; GFX6-NEXT:    s_mov_b32 s17, 0x80000000
+; GFX6-NEXT:    s_brev_b32 s17, 1
 ; GFX6-NEXT:    s_cselect_b32 s19, s0, 0
 ; GFX6-NEXT:    s_sub_i32 s19, s17, s19
 ; GFX6-NEXT:    s_cmp_gt_i32 s19, s8
@@ -6670,7 +6670,7 @@ define amdgpu_ps <4 x i32> @s_saddsat_v8i16(<8 x i16> inreg %lhs, <8 x i16> inre
 ; GFX8-NEXT:    s_cselect_b32 s20, s18, s19
 ; GFX8-NEXT:    s_sub_i32 s20, s16, s20
 ; GFX8-NEXT:    s_cmp_lt_i32 s18, s19
-; GFX8-NEXT:    s_mov_b32 s17, 0x8000
+; GFX8-NEXT:    s_movk_i32 s17, 0x8000
 ; GFX8-NEXT:    s_cselect_b32 s18, s18, s19
 ; GFX8-NEXT:    s_sub_i32 s18, s17, s18
 ; GFX8-NEXT:    s_sext_i32_i16 s18, s18
@@ -6833,7 +6833,7 @@ define amdgpu_ps <4 x i32> @s_saddsat_v8i16(<8 x i16> inreg %lhs, <8 x i16> inre
 ; GFX9-NEXT:    s_cmp_lt_i32 s11, s13
 ; GFX9-NEXT:    s_cselect_b32 s11, s11, s13
 ; GFX9-NEXT:    s_cmp_lt_i32 s12, s10
-; GFX9-NEXT:    s_mov_b32 s9, 0xffff8000
+; GFX9-NEXT:    s_movk_i32 s9, 0x8000
 ; GFX9-NEXT:    s_cselect_b32 s12, s12, s10
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s9, s9, s9
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s11, s11, s12
@@ -7010,7 +7010,7 @@ define amdgpu_ps <4 x i32> @s_saddsat_v8i16(<8 x i16> inreg %lhs, <8 x i16> inre
 ; GFX10-NEXT:    s_cmp_gt_i32 s10, s8
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s13, s13, s13
 ; GFX10-NEXT:    s_cselect_b32 s14, s10, s8
-; GFX10-NEXT:    s_mov_b32 s16, 0xffff8000
+; GFX10-NEXT:    s_movk_i32 s16, 0x8000
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s12, s12, s14
 ; GFX10-NEXT:    s_lshr_b32 s14, s13, 16
 ; GFX10-NEXT:    s_lshr_b32 s15, s12, 16
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i32.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i32.ll
index f68465faf61cd..57737aeb886fa 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i32.ll
@@ -399,7 +399,7 @@ define <2 x i32> @v_sdiv_v2i32_pow2k_denom(<2 x i32> %num) {
 ; CGP-NEXT:    s_movk_i32 s4, 0x1000
 ; CGP-NEXT:    v_ashrrev_i32_e32 v2, 31, v0
 ; CGP-NEXT:    v_mov_b32_e32 v3, 0x1000
-; CGP-NEXT:    s_mov_b32 s5, 0xfffff000
+; CGP-NEXT:    s_movk_i32 s5, 0xf000
 ; CGP-NEXT:    v_mov_b32_e32 v4, 0xfffff000
 ; CGP-NEXT:    v_ashrrev_i32_e32 v5, 31, v1
 ; CGP-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i64.ll
index 1813c33019ae8..f6565fe1b6e24 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i64.ll
@@ -200,13 +200,12 @@ declare i32 @llvm.amdgcn.readfirstlane(i32)
 define amdgpu_ps i64 @s_sdiv_i64(i64 inreg %num, i64 inreg %den) {
 ; CHECK-LABEL: s_sdiv_i64:
 ; CHECK:       ; %bb.0:
-; CHECK-NEXT:    s_or_b64 s[0:1], s[2:3], s[4:5]
-; CHECK-NEXT:    s_mov_b32 s6, 0
-; CHECK-NEXT:    s_mov_b32 s7, -1
-; CHECK-NEXT:    s_and_b64 s[0:1], s[0:1], s[6:7]
-; CHECK-NEXT:    v_cmp_eq_u64_e64 s[6:7], s[0:1], 0
+; CHECK-NEXT:    s_or_b64 s[6:7], s[2:3], s[4:5]
+; CHECK-NEXT:    s_mov_b32 s0, 0
+; CHECK-NEXT:    s_mov_b32 s1, -1
+; CHECK-NEXT:    s_and_b64 s[6:7], s[6:7], s[0:1]
+; CHECK-NEXT:    v_cmp_eq_u64_e64 s[6:7], s[6:7], 0
 ; CHECK-NEXT:    v_cmp_ne_u32_e64 s[8:9], 0, 1
-; CHECK-NEXT:    s_mov_b32 s0, 1
 ; CHECK-NEXT:    s_xor_b64 vcc, s[6:7], s[8:9]
 ; CHECK-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; CHECK-NEXT:    s_cbranch_vccz BB1_2
@@ -354,13 +353,13 @@ define amdgpu_ps i64 @s_sdiv_i64(i64 inreg %num, i64 inreg %den) {
 ; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v1
 ; CHECK-NEXT:    v_cndmask_b32_e32 v1, v4, v3, vcc
 ; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
-; CHECK-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
 ; CHECK-NEXT:    s_xor_b64 s[0:1], s[6:7], s[8:9]
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
 ; CHECK-NEXT:    v_xor_b32_e32 v0, s0, v0
 ; CHECK-NEXT:    v_subrev_i32_e32 v0, vcc, s0, v0
-; CHECK-NEXT:    s_mov_b32 s0, 0
+; CHECK-NEXT:    s_mov_b32 s1, 0
 ; CHECK-NEXT:  BB1_2: ; %Flow
-; CHECK-NEXT:    s_and_b32 s0, s0, 1
+; CHECK-NEXT:    s_and_b32 s0, s1, 1
 ; CHECK-NEXT:    s_cmp_lg_u32 s0, 0
 ; CHECK-NEXT:    s_cbranch_scc0 BB1_4
 ; CHECK-NEXT:  ; %bb.3:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i32.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i32.ll
index 43f79f4b207d0..320d814be8a94 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i32.ll
@@ -369,7 +369,7 @@ define <2 x i32> @v_srem_v2i32_pow2k_denom(<2 x i32> %num) {
 ; CGP-NEXT:    s_movk_i32 s4, 0x1000
 ; CGP-NEXT:    v_ashrrev_i32_e32 v2, 31, v0
 ; CGP-NEXT:    v_mov_b32_e32 v3, 0x1000
-; CGP-NEXT:    s_mov_b32 s5, 0xfffff000
+; CGP-NEXT:    s_movk_i32 s5, 0xf000
 ; CGP-NEXT:    v_mov_b32_e32 v4, 0xfffff000
 ; CGP-NEXT:    v_ashrrev_i32_e32 v5, 31, v1
 ; CGP-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i64.ll
index 438388ebf7136..06d46321a59b6 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i64.ll
@@ -196,13 +196,12 @@ declare i32 @llvm.amdgcn.readfirstlane(i32)
 define amdgpu_ps i64 @s_srem_i64(i64 inreg %num, i64 inreg %den) {
 ; CHECK-LABEL: s_srem_i64:
 ; CHECK:       ; %bb.0:
-; CHECK-NEXT:    s_or_b64 s[0:1], s[2:3], s[4:5]
-; CHECK-NEXT:    s_mov_b32 s6, 0
-; CHECK-NEXT:    s_mov_b32 s7, -1
-; CHECK-NEXT:    s_and_b64 s[0:1], s[0:1], s[6:7]
-; CHECK-NEXT:    v_cmp_eq_u64_e64 s[6:7], s[0:1], 0
+; CHECK-NEXT:    s_or_b64 s[6:7], s[2:3], s[4:5]
+; CHECK-NEXT:    s_mov_b32 s0, 0
+; CHECK-NEXT:    s_mov_b32 s1, -1
+; CHECK-NEXT:    s_and_b64 s[6:7], s[6:7], s[0:1]
+; CHECK-NEXT:    v_cmp_eq_u64_e64 s[6:7], s[6:7], 0
 ; CHECK-NEXT:    v_cmp_ne_u32_e64 s[8:9], 0, 1
-; CHECK-NEXT:    s_mov_b32 s0, 1
 ; CHECK-NEXT:    s_xor_b64 vcc, s[6:7], s[8:9]
 ; CHECK-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; CHECK-NEXT:    s_cbranch_vccz BB1_2
@@ -352,9 +351,9 @@ define amdgpu_ps i64 @s_srem_i64(i64 inreg %num, i64 inreg %den) {
 ; CHECK-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
 ; CHECK-NEXT:    v_xor_b32_e32 v0, s6, v0
 ; CHECK-NEXT:    v_subrev_i32_e32 v0, vcc, s6, v0
-; CHECK-NEXT:    s_mov_b32 s0, 0
+; CHECK-NEXT:    s_mov_b32 s1, 0
 ; CHECK-NEXT:  BB1_2: ; %Flow
-; CHECK-NEXT:    s_and_b32 s0, s0, 1
+; CHECK-NEXT:    s_and_b32 s0, s1, 1
 ; CHECK-NEXT:    s_cmp_lg_u32 s0, 0
 ; CHECK-NEXT:    s_cbranch_scc0 BB1_4
 ; CHECK-NEXT:  ; %bb.3:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll
index a7154131e3c07..3e1778bcb881e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll
@@ -24,9 +24,8 @@ define i7 @v_ssubsat_i7(i7 %lhs, i7 %rhs) {
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    v_lshlrev_b16_e32 v0, 9, v0
-; GFX8-NEXT:    s_mov_b32 s4, 0xffff
-; GFX8-NEXT:    v_max_i16_e32 v2, s4, v0
-; GFX8-NEXT:    v_min_i16_e32 v3, s4, v0
+; GFX8-NEXT:    v_max_i16_e32 v2, -1, v0
+; GFX8-NEXT:    v_min_i16_e32 v3, -1, v0
 ; GFX8-NEXT:    v_lshlrev_b16_e32 v1, 9, v1
 ; GFX8-NEXT:    v_subrev_u16_e32 v2, 0x7fff, v2
 ; GFX8-NEXT:    v_subrev_u16_e32 v3, 0x8000, v3
@@ -40,9 +39,8 @@ define i7 @v_ssubsat_i7(i7 %lhs, i7 %rhs) {
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 9, v0
-; GFX9-NEXT:    s_mov_b32 s4, 0xffff
-; GFX9-NEXT:    v_max_i16_e32 v2, s4, v0
-; GFX9-NEXT:    v_min_i16_e32 v3, s4, v0
+; GFX9-NEXT:    v_max_i16_e32 v2, -1, v0
+; GFX9-NEXT:    v_min_i16_e32 v3, -1, v0
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 9, v1
 ; GFX9-NEXT:    v_subrev_u16_e32 v2, 0x7fff, v2
 ; GFX9-NEXT:    v_subrev_u16_e32 v3, 0x8000, v3
@@ -57,11 +55,10 @@ define i7 @v_ssubsat_i7(i7 %lhs, i7 %rhs) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_lshlrev_b16_e64 v0, 9, v0
-; GFX10-NEXT:    s_mov_b32 s4, 0xffff
 ; GFX10-NEXT:    v_lshlrev_b16_e64 v1, 9, v1
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_max_i16_e64 v2, v0, s4
-; GFX10-NEXT:    v_min_i16_e64 v3, v0, s4
+; GFX10-NEXT:    v_max_i16_e64 v2, v0, -1
+; GFX10-NEXT:    v_min_i16_e64 v3, v0, -1
 ; GFX10-NEXT:    v_sub_nc_u16_e64 v2, v2, 0x7fff
 ; GFX10-NEXT:    v_sub_nc_u16_e64 v3, v3, 0x8000
 ; GFX10-NEXT:    v_max_i16_e64 v1, v2, v1
@@ -98,13 +95,13 @@ define amdgpu_ps i7 @s_ssubsat_i7(i7 inreg %lhs, i7 inreg %rhs) {
 ; GFX8-NEXT:    s_lshl_b32 s0, s0, s2
 ; GFX8-NEXT:    s_lshl_b32 s1, s1, s2
 ; GFX8-NEXT:    s_sext_i32_i16 s3, s0
-; GFX8-NEXT:    s_sext_i32_i16 s4, 0xffff
+; GFX8-NEXT:    s_sext_i32_i16 s4, -1
 ; GFX8-NEXT:    s_cmp_gt_i32 s3, s4
 ; GFX8-NEXT:    s_cselect_b32 s5, s3, s4
 ; GFX8-NEXT:    s_sub_i32 s5, s5, 0x7fff
 ; GFX8-NEXT:    s_cmp_lt_i32 s3, s4
 ; GFX8-NEXT:    s_cselect_b32 s3, s3, s4
-; GFX8-NEXT:    s_sub_i32 s3, s3, 0x8000
+; GFX8-NEXT:    s_sub_i32 s3, s3, 0xffff8000
 ; GFX8-NEXT:    s_sext_i32_i16 s4, s5
 ; GFX8-NEXT:    s_sext_i32_i16 s1, s1
 ; GFX8-NEXT:    s_cmp_gt_i32 s4, s1
@@ -124,13 +121,13 @@ define amdgpu_ps i7 @s_ssubsat_i7(i7 inreg %lhs, i7 inreg %rhs) {
 ; GFX9-NEXT:    s_lshl_b32 s0, s0, s2
 ; GFX9-NEXT:    s_lshl_b32 s1, s1, s2
 ; GFX9-NEXT:    s_sext_i32_i16 s3, s0
-; GFX9-NEXT:    s_sext_i32_i16 s4, 0xffff
+; GFX9-NEXT:    s_sext_i32_i16 s4, -1
 ; GFX9-NEXT:    s_cmp_gt_i32 s3, s4
 ; GFX9-NEXT:    s_cselect_b32 s5, s3, s4
 ; GFX9-NEXT:    s_sub_i32 s5, s5, 0x7fff
 ; GFX9-NEXT:    s_cmp_lt_i32 s3, s4
 ; GFX9-NEXT:    s_cselect_b32 s3, s3, s4
-; GFX9-NEXT:    s_sub_i32 s3, s3, 0x8000
+; GFX9-NEXT:    s_sub_i32 s3, s3, 0xffff8000
 ; GFX9-NEXT:    s_sext_i32_i16 s4, s5
 ; GFX9-NEXT:    s_sext_i32_i16 s1, s1
 ; GFX9-NEXT:    s_cmp_gt_i32 s4, s1
@@ -147,7 +144,7 @@ define amdgpu_ps i7 @s_ssubsat_i7(i7 inreg %lhs, i7 inreg %rhs) {
 ; GFX10-LABEL: s_ssubsat_i7:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_bfe_u32 s2, 9, 0x100000
-; GFX10-NEXT:    s_sext_i32_i16 s4, 0xffff
+; GFX10-NEXT:    s_sext_i32_i16 s4, -1
 ; GFX10-NEXT:    s_lshl_b32 s0, s0, s2
 ; GFX10-NEXT:    s_lshl_b32 s1, s1, s2
 ; GFX10-NEXT:    s_sext_i32_i16 s3, s0
@@ -159,7 +156,7 @@ define amdgpu_ps i7 @s_ssubsat_i7(i7 inreg %lhs, i7 inreg %rhs) {
 ; GFX10-NEXT:    s_cmp_lt_i32 s3, s4
 ; GFX10-NEXT:    s_cselect_b32 s3, s3, s4
 ; GFX10-NEXT:    s_sext_i32_i16 s4, s5
-; GFX10-NEXT:    s_sub_i32 s3, s3, 0x8000
+; GFX10-NEXT:    s_sub_i32 s3, s3, 0xffff8000
 ; GFX10-NEXT:    s_cmp_gt_i32 s4, s1
 ; GFX10-NEXT:    s_sext_i32_i16 s3, s3
 ; GFX10-NEXT:    s_cselect_b32 s1, s4, s1
@@ -194,9 +191,8 @@ define i8 @v_ssubsat_i8(i8 %lhs, i8 %rhs) {
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; GFX8-NEXT:    s_mov_b32 s4, 0xffff
-; GFX8-NEXT:    v_max_i16_e32 v2, s4, v0
-; GFX8-NEXT:    v_min_i16_e32 v3, s4, v0
+; GFX8-NEXT:    v_max_i16_e32 v2, -1, v0
+; GFX8-NEXT:    v_min_i16_e32 v3, -1, v0
 ; GFX8-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
 ; GFX8-NEXT:    v_subrev_u16_e32 v2, 0x7fff, v2
 ; GFX8-NEXT:    v_subrev_u16_e32 v3, 0x8000, v3
@@ -210,9 +206,8 @@ define i8 @v_ssubsat_i8(i8 %lhs, i8 %rhs) {
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; GFX9-NEXT:    s_mov_b32 s4, 0xffff
-; GFX9-NEXT:    v_max_i16_e32 v2, s4, v0
-; GFX9-NEXT:    v_min_i16_e32 v3, s4, v0
+; GFX9-NEXT:    v_max_i16_e32 v2, -1, v0
+; GFX9-NEXT:    v_min_i16_e32 v3, -1, v0
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
 ; GFX9-NEXT:    v_subrev_u16_e32 v2, 0x7fff, v2
 ; GFX9-NEXT:    v_subrev_u16_e32 v3, 0x8000, v3
@@ -227,11 +222,10 @@ define i8 @v_ssubsat_i8(i8 %lhs, i8 %rhs) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_lshlrev_b16_e64 v0, 8, v0
-; GFX10-NEXT:    s_mov_b32 s4, 0xffff
 ; GFX10-NEXT:    v_lshlrev_b16_e64 v1, 8, v1
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_max_i16_e64 v2, v0, s4
-; GFX10-NEXT:    v_min_i16_e64 v3, v0, s4
+; GFX10-NEXT:    v_max_i16_e64 v2, v0, -1
+; GFX10-NEXT:    v_min_i16_e64 v3, v0, -1
 ; GFX10-NEXT:    v_sub_nc_u16_e64 v2, v2, 0x7fff
 ; GFX10-NEXT:    v_sub_nc_u16_e64 v3, v3, 0x8000
 ; GFX10-NEXT:    v_max_i16_e64 v1, v2, v1
@@ -268,13 +262,13 @@ define amdgpu_ps i8 @s_ssubsat_i8(i8 inreg %lhs, i8 inreg %rhs) {
 ; GFX8-NEXT:    s_lshl_b32 s0, s0, s2
 ; GFX8-NEXT:    s_lshl_b32 s1, s1, s2
 ; GFX8-NEXT:    s_sext_i32_i16 s3, s0
-; GFX8-NEXT:    s_sext_i32_i16 s4, 0xffff
+; GFX8-NEXT:    s_sext_i32_i16 s4, -1
 ; GFX8-NEXT:    s_cmp_gt_i32 s3, s4
 ; GFX8-NEXT:    s_cselect_b32 s5, s3, s4
 ; GFX8-NEXT:    s_sub_i32 s5, s5, 0x7fff
 ; GFX8-NEXT:    s_cmp_lt_i32 s3, s4
 ; GFX8-NEXT:    s_cselect_b32 s3, s3, s4
-; GFX8-NEXT:    s_sub_i32 s3, s3, 0x8000
+; GFX8-NEXT:    s_sub_i32 s3, s3, 0xffff8000
 ; GFX8-NEXT:    s_sext_i32_i16 s4, s5
 ; GFX8-NEXT:    s_sext_i32_i16 s1, s1
 ; GFX8-NEXT:    s_cmp_gt_i32 s4, s1
@@ -294,13 +288,13 @@ define amdgpu_ps i8 @s_ssubsat_i8(i8 inreg %lhs, i8 inreg %rhs) {
 ; GFX9-NEXT:    s_lshl_b32 s0, s0, s2
 ; GFX9-NEXT:    s_lshl_b32 s1, s1, s2
 ; GFX9-NEXT:    s_sext_i32_i16 s3, s0
-; GFX9-NEXT:    s_sext_i32_i16 s4, 0xffff
+; GFX9-NEXT:    s_sext_i32_i16 s4, -1
 ; GFX9-NEXT:    s_cmp_gt_i32 s3, s4
 ; GFX9-NEXT:    s_cselect_b32 s5, s3, s4
 ; GFX9-NEXT:    s_sub_i32 s5, s5, 0x7fff
 ; GFX9-NEXT:    s_cmp_lt_i32 s3, s4
 ; GFX9-NEXT:    s_cselect_b32 s3, s3, s4
-; GFX9-NEXT:    s_sub_i32 s3, s3, 0x8000
+; GFX9-NEXT:    s_sub_i32 s3, s3, 0xffff8000
 ; GFX9-NEXT:    s_sext_i32_i16 s4, s5
 ; GFX9-NEXT:    s_sext_i32_i16 s1, s1
 ; GFX9-NEXT:    s_cmp_gt_i32 s4, s1
@@ -317,7 +311,7 @@ define amdgpu_ps i8 @s_ssubsat_i8(i8 inreg %lhs, i8 inreg %rhs) {
 ; GFX10-LABEL: s_ssubsat_i8:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_bfe_u32 s2, 8, 0x100000
-; GFX10-NEXT:    s_sext_i32_i16 s4, 0xffff
+; GFX10-NEXT:    s_sext_i32_i16 s4, -1
 ; GFX10-NEXT:    s_lshl_b32 s0, s0, s2
 ; GFX10-NEXT:    s_lshl_b32 s1, s1, s2
 ; GFX10-NEXT:    s_sext_i32_i16 s3, s0
@@ -329,7 +323,7 @@ define amdgpu_ps i8 @s_ssubsat_i8(i8 inreg %lhs, i8 inreg %rhs) {
 ; GFX10-NEXT:    s_cmp_lt_i32 s3, s4
 ; GFX10-NEXT:    s_cselect_b32 s3, s3, s4
 ; GFX10-NEXT:    s_sext_i32_i16 s4, s5
-; GFX10-NEXT:    s_sub_i32 s3, s3, 0x8000
+; GFX10-NEXT:    s_sub_i32 s3, s3, 0xffff8000
 ; GFX10-NEXT:    s_cmp_gt_i32 s4, s1
 ; GFX10-NEXT:    s_sext_i32_i16 s3, s3
 ; GFX10-NEXT:    s_cselect_b32 s1, s4, s1
@@ -355,7 +349,7 @@ define i16 @v_ssubsat_v2i8(i16 %lhs.arg, i16 %rhs.arg) {
 ; GFX6-NEXT:    v_lshrrev_b32_e32 v3, 8, v1
 ; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 24, v1
 ; GFX6-NEXT:    v_subrev_i32_e32 v4, vcc, s4, v4
-; GFX6-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX6-NEXT:    s_brev_b32 s5, 1
 ; GFX6-NEXT:    v_min_i32_e32 v5, -1, v0
 ; GFX6-NEXT:    v_subrev_i32_e32 v5, vcc, s5, v5
 ; GFX6-NEXT:    v_max_i32_e32 v1, v4, v1
@@ -385,21 +379,20 @@ define i16 @v_ssubsat_v2i8(i16 %lhs.arg, i16 %rhs.arg) {
 ; GFX8-NEXT:    v_mov_b32_e32 v2, 8
 ; GFX8-NEXT:    v_lshrrev_b32_sdwa v3, v2, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX8-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; GFX8-NEXT:    s_mov_b32 s6, 0xffff
 ; GFX8-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX8-NEXT:    v_max_i16_e32 v4, s6, v0
+; GFX8-NEXT:    v_max_i16_e32 v4, -1, v0
 ; GFX8-NEXT:    v_lshrrev_b32_sdwa v2, v2, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX8-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
 ; GFX8-NEXT:    v_subrev_u16_e32 v4, s4, v4
-; GFX8-NEXT:    s_mov_b32 s5, 0x8000
-; GFX8-NEXT:    v_min_i16_e32 v5, s6, v0
+; GFX8-NEXT:    s_movk_i32 s5, 0x8000
+; GFX8-NEXT:    v_min_i16_e32 v5, -1, v0
 ; GFX8-NEXT:    v_max_i16_e32 v1, v4, v1
 ; GFX8-NEXT:    v_subrev_u16_e32 v5, s5, v5
 ; GFX8-NEXT:    v_min_i16_e32 v1, v1, v5
 ; GFX8-NEXT:    v_sub_u16_e32 v0, v0, v1
-; GFX8-NEXT:    v_max_i16_e32 v1, s6, v3
+; GFX8-NEXT:    v_max_i16_e32 v1, -1, v3
 ; GFX8-NEXT:    v_subrev_u16_e32 v1, s4, v1
-; GFX8-NEXT:    v_min_i16_e32 v4, s6, v3
+; GFX8-NEXT:    v_min_i16_e32 v4, -1, v3
 ; GFX8-NEXT:    v_max_i16_e32 v1, v1, v2
 ; GFX8-NEXT:    v_subrev_u16_e32 v4, s5, v4
 ; GFX8-NEXT:    v_min_i16_e32 v1, v1, v4
@@ -416,21 +409,20 @@ define i16 @v_ssubsat_v2i8(i16 %lhs.arg, i16 %rhs.arg) {
 ; GFX9-NEXT:    s_mov_b32 s4, 8
 ; GFX9-NEXT:    v_lshrrev_b32_sdwa v2, s4, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; GFX9-NEXT:    s_mov_b32 s6, 0xffff
 ; GFX9-NEXT:    v_lshrrev_b32_sdwa v3, s4, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX9-NEXT:    v_max_i16_e32 v4, s6, v0
+; GFX9-NEXT:    v_max_i16_e32 v4, -1, v0
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
 ; GFX9-NEXT:    v_subrev_u16_e32 v4, s4, v4
-; GFX9-NEXT:    s_mov_b32 s5, 0x8000
-; GFX9-NEXT:    v_min_i16_e32 v5, s6, v0
+; GFX9-NEXT:    s_movk_i32 s5, 0x8000
+; GFX9-NEXT:    v_min_i16_e32 v5, -1, v0
 ; GFX9-NEXT:    v_max_i16_e32 v1, v4, v1
 ; GFX9-NEXT:    v_subrev_u16_e32 v5, s5, v5
 ; GFX9-NEXT:    v_min_i16_e32 v1, v1, v5
 ; GFX9-NEXT:    v_sub_u16_e32 v0, v0, v1
-; GFX9-NEXT:    v_max_i16_e32 v1, s6, v2
+; GFX9-NEXT:    v_max_i16_e32 v1, -1, v2
 ; GFX9-NEXT:    v_subrev_u16_e32 v1, s4, v1
-; GFX9-NEXT:    v_min_i16_e32 v4, s6, v2
+; GFX9-NEXT:    v_min_i16_e32 v4, -1, v2
 ; GFX9-NEXT:    v_subrev_u16_e32 v4, s5, v4
 ; GFX9-NEXT:    v_max_i16_e32 v1, v1, v3
 ; GFX9-NEXT:    v_min_i16_e32 v1, v1, v4
@@ -448,23 +440,22 @@ define i16 @v_ssubsat_v2i8(i16 %lhs.arg, i16 %rhs.arg) {
 ; GFX10-NEXT:    s_mov_b32 s4, 8
 ; GFX10-NEXT:    v_lshlrev_b16_e64 v2, 8, v0
 ; GFX10-NEXT:    v_lshrrev_b32_sdwa v0, s4, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT:    s_mov_b32 s6, 0xffff
 ; GFX10-NEXT:    s_movk_i32 s5, 0x7fff
 ; GFX10-NEXT:    v_lshrrev_b32_sdwa v3, s4, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT:    v_max_i16_e64 v4, v2, s6
-; GFX10-NEXT:    v_max_i16_e64 v5, v0, s6
 ; GFX10-NEXT:    v_lshlrev_b16_e64 v1, 8, v1
-; GFX10-NEXT:    v_min_i16_e64 v6, v2, s6
-; GFX10-NEXT:    v_min_i16_e64 v7, v0, s6
+; GFX10-NEXT:    v_max_i16_e64 v4, v2, -1
+; GFX10-NEXT:    v_max_i16_e64 v5, v0, -1
+; GFX10-NEXT:    v_min_i16_e64 v6, v2, -1
+; GFX10-NEXT:    v_min_i16_e64 v7, v0, -1
+; GFX10-NEXT:    s_movk_i32 s4, 0x8000
 ; GFX10-NEXT:    v_sub_nc_u16_e64 v4, v4, s5
 ; GFX10-NEXT:    v_sub_nc_u16_e64 v5, v5, s5
-; GFX10-NEXT:    s_mov_b32 s4, 0x8000
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_sub_nc_u16_e64 v6, v6, s4
-; GFX10-NEXT:    v_max_i16_e64 v1, v4, v1
 ; GFX10-NEXT:    v_sub_nc_u16_e64 v7, v7, s4
-; GFX10-NEXT:    v_max_i16_e64 v10, v5, v3
 ; GFX10-NEXT:    s_movk_i32 s4, 0xff
+; GFX10-NEXT:    v_max_i16_e64 v1, v4, v1
+; GFX10-NEXT:    v_max_i16_e64 v10, v5, v3
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_min_i16_e64 v1, v1, v6
 ; GFX10-NEXT:    v_min_i16_e64 v3, v10, v7
 ; GFX10-NEXT:    v_sub_nc_u16_e64 v1, v2, v1
@@ -492,7 +483,7 @@ define amdgpu_ps i16 @s_ssubsat_v2i8(i16 inreg %lhs.arg, i16 inreg %rhs.arg) {
 ; GFX6-NEXT:    s_cselect_b32 s6, s0, -1
 ; GFX6-NEXT:    s_sub_i32 s6, s6, s4
 ; GFX6-NEXT:    s_cmp_lt_i32 s0, -1
-; GFX6-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX6-NEXT:    s_brev_b32 s5, 1
 ; GFX6-NEXT:    s_cselect_b32 s7, s0, -1
 ; GFX6-NEXT:    s_sub_i32 s7, s7, s5
 ; GFX6-NEXT:    s_cmp_gt_i32 s6, s1
@@ -530,13 +521,13 @@ define amdgpu_ps i16 @s_ssubsat_v2i8(i16 inreg %lhs.arg, i16 inreg %rhs.arg) {
 ; GFX8-NEXT:    s_lshr_b32 s3, s1, 8
 ; GFX8-NEXT:    s_lshl_b32 s1, s1, s4
 ; GFX8-NEXT:    s_sext_i32_i16 s7, s0
-; GFX8-NEXT:    s_sext_i32_i16 s8, 0xffff
+; GFX8-NEXT:    s_sext_i32_i16 s8, -1
 ; GFX8-NEXT:    s_cmp_gt_i32 s7, s8
 ; GFX8-NEXT:    s_movk_i32 s5, 0x7fff
 ; GFX8-NEXT:    s_cselect_b32 s9, s7, s8
 ; GFX8-NEXT:    s_sub_i32 s9, s9, s5
 ; GFX8-NEXT:    s_cmp_lt_i32 s7, s8
-; GFX8-NEXT:    s_mov_b32 s6, 0x8000
+; GFX8-NEXT:    s_movk_i32 s6, 0x8000
 ; GFX8-NEXT:    s_cselect_b32 s7, s7, s8
 ; GFX8-NEXT:    s_sub_i32 s7, s7, s6
 ; GFX8-NEXT:    s_sext_i32_i16 s9, s9
@@ -585,13 +576,13 @@ define amdgpu_ps i16 @s_ssubsat_v2i8(i16 inreg %lhs.arg, i16 inreg %rhs.arg) {
 ; GFX9-NEXT:    s_lshr_b32 s3, s1, 8
 ; GFX9-NEXT:    s_lshl_b32 s1, s1, s4
 ; GFX9-NEXT:    s_sext_i32_i16 s7, s0
-; GFX9-NEXT:    s_sext_i32_i16 s8, 0xffff
+; GFX9-NEXT:    s_sext_i32_i16 s8, -1
 ; GFX9-NEXT:    s_cmp_gt_i32 s7, s8
 ; GFX9-NEXT:    s_movk_i32 s5, 0x7fff
 ; GFX9-NEXT:    s_cselect_b32 s9, s7, s8
 ; GFX9-NEXT:    s_sub_i32 s9, s9, s5
 ; GFX9-NEXT:    s_cmp_lt_i32 s7, s8
-; GFX9-NEXT:    s_mov_b32 s6, 0x8000
+; GFX9-NEXT:    s_movk_i32 s6, 0x8000
 ; GFX9-NEXT:    s_cselect_b32 s7, s7, s8
 ; GFX9-NEXT:    s_sub_i32 s7, s7, s6
 ; GFX9-NEXT:    s_sext_i32_i16 s9, s9
@@ -637,14 +628,14 @@ define amdgpu_ps i16 @s_ssubsat_v2i8(i16 inreg %lhs.arg, i16 inreg %rhs.arg) {
 ; GFX10-NEXT:    s_bfe_u32 s2, 8, 0x100000
 ; GFX10-NEXT:    s_lshr_b32 s3, s0, 8
 ; GFX10-NEXT:    s_lshl_b32 s0, s0, s2
-; GFX10-NEXT:    s_sext_i32_i16 s6, 0xffff
+; GFX10-NEXT:    s_sext_i32_i16 s6, -1
 ; GFX10-NEXT:    s_sext_i32_i16 s5, s0
 ; GFX10-NEXT:    s_lshr_b32 s4, s1, 8
 ; GFX10-NEXT:    s_lshl_b32 s1, s1, s2
 ; GFX10-NEXT:    s_cmp_gt_i32 s5, s6
 ; GFX10-NEXT:    s_movk_i32 s7, 0x7fff
 ; GFX10-NEXT:    s_cselect_b32 s8, s5, s6
-; GFX10-NEXT:    s_mov_b32 s9, 0x8000
+; GFX10-NEXT:    s_movk_i32 s9, 0x8000
 ; GFX10-NEXT:    s_sub_i32 s8, s8, s7
 ; GFX10-NEXT:    s_cmp_lt_i32 s5, s6
 ; GFX10-NEXT:    s_sext_i32_i16 s8, s8
@@ -709,7 +700,7 @@ define i32 @v_ssubsat_v4i8(i32 %lhs.arg, i32 %rhs.arg) {
 ; GFX6-NEXT:    v_lshrrev_b32_e32 v7, 24, v1
 ; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 24, v1
 ; GFX6-NEXT:    v_subrev_i32_e32 v8, vcc, s4, v8
-; GFX6-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX6-NEXT:    s_brev_b32 s5, 1
 ; GFX6-NEXT:    v_min_i32_e32 v10, -1, v0
 ; GFX6-NEXT:    v_subrev_i32_e32 v10, vcc, s5, v10
 ; GFX6-NEXT:    v_max_i32_e32 v1, v8, v1
@@ -736,7 +727,7 @@ define i32 @v_ssubsat_v4i8(i32 %lhs.arg, i32 %rhs.arg) {
 ; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, v2, v3
 ; GFX6-NEXT:    v_lshlrev_b32_e32 v3, 24, v4
 ; GFX6-NEXT:    v_max_i32_e32 v5, -1, v3
-; GFX6-NEXT:    v_mov_b32_e32 v11, 0x80000000
+; GFX6-NEXT:    v_bfrev_b32_e32 v11, 1
 ; GFX6-NEXT:    v_min_i32_e32 v6, -1, v3
 ; GFX6-NEXT:    v_ashrrev_i32_e32 v1, 24, v1
 ; GFX6-NEXT:    s_movk_i32 s4, 0xff
@@ -769,43 +760,41 @@ define i32 @v_ssubsat_v4i8(i32 %lhs.arg, i32 %rhs.arg) {
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v4, 16, v0
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v5, 24, v0
 ; GFX8-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; GFX8-NEXT:    s_mov_b32 s6, 0xffff
 ; GFX8-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX8-NEXT:    v_max_i16_e32 v9, s6, v0
+; GFX8-NEXT:    v_max_i16_e32 v8, -1, v0
 ; GFX8-NEXT:    v_lshrrev_b32_sdwa v2, v2, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v6, 16, v1
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v7, 24, v1
 ; GFX8-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
-; GFX8-NEXT:    v_subrev_u16_e32 v9, s4, v9
-; GFX8-NEXT:    s_mov_b32 s5, 0x8000
-; GFX8-NEXT:    v_min_i16_e32 v11, s6, v0
-; GFX8-NEXT:    v_max_i16_e32 v1, v9, v1
-; GFX8-NEXT:    v_subrev_u16_e32 v11, s5, v11
-; GFX8-NEXT:    v_min_i16_e32 v1, v1, v11
+; GFX8-NEXT:    v_subrev_u16_e32 v8, s4, v8
+; GFX8-NEXT:    s_movk_i32 s5, 0x8000
+; GFX8-NEXT:    v_min_i16_e32 v10, -1, v0
+; GFX8-NEXT:    v_max_i16_e32 v1, v8, v1
+; GFX8-NEXT:    v_subrev_u16_e32 v10, s5, v10
+; GFX8-NEXT:    v_min_i16_e32 v1, v1, v10
 ; GFX8-NEXT:    v_sub_u16_e32 v0, v0, v1
-; GFX8-NEXT:    v_max_i16_e32 v1, s6, v3
+; GFX8-NEXT:    v_max_i16_e32 v1, -1, v3
 ; GFX8-NEXT:    v_subrev_u16_e32 v1, s4, v1
-; GFX8-NEXT:    v_min_i16_e32 v9, s6, v3
+; GFX8-NEXT:    v_min_i16_e32 v8, -1, v3
 ; GFX8-NEXT:    v_max_i16_e32 v1, v1, v2
-; GFX8-NEXT:    v_subrev_u16_e32 v9, s5, v9
-; GFX8-NEXT:    v_mov_b32_e32 v8, 0xffff
 ; GFX8-NEXT:    v_lshlrev_b16_e32 v2, 8, v4
-; GFX8-NEXT:    v_min_i16_e32 v1, v1, v9
-; GFX8-NEXT:    v_mov_b32_e32 v10, 0x7fff
-; GFX8-NEXT:    v_max_i16_e32 v4, v2, v8
+; GFX8-NEXT:    v_subrev_u16_e32 v8, s5, v8
+; GFX8-NEXT:    v_min_i16_e32 v1, v1, v8
+; GFX8-NEXT:    v_mov_b32_e32 v9, 0x7fff
+; GFX8-NEXT:    v_max_i16_e32 v4, -1, v2
 ; GFX8-NEXT:    v_sub_u16_e32 v1, v3, v1
 ; GFX8-NEXT:    v_lshlrev_b16_e32 v3, 8, v6
-; GFX8-NEXT:    v_min_i16_e32 v6, v2, v8
-; GFX8-NEXT:    v_sub_u16_e32 v4, v4, v10
+; GFX8-NEXT:    v_min_i16_e32 v6, -1, v2
+; GFX8-NEXT:    v_sub_u16_e32 v4, v4, v9
 ; GFX8-NEXT:    v_max_i16_e32 v3, v4, v3
 ; GFX8-NEXT:    v_subrev_u16_e32 v6, s5, v6
 ; GFX8-NEXT:    v_min_i16_e32 v3, v3, v6
 ; GFX8-NEXT:    v_sub_u16_e32 v2, v2, v3
 ; GFX8-NEXT:    v_lshlrev_b16_e32 v3, 8, v5
-; GFX8-NEXT:    v_max_i16_e32 v5, v3, v8
-; GFX8-NEXT:    v_min_i16_e32 v6, v3, v8
+; GFX8-NEXT:    v_max_i16_e32 v5, -1, v3
+; GFX8-NEXT:    v_min_i16_e32 v6, -1, v3
 ; GFX8-NEXT:    v_lshlrev_b16_e32 v4, 8, v7
-; GFX8-NEXT:    v_sub_u16_e32 v5, v5, v10
+; GFX8-NEXT:    v_sub_u16_e32 v5, v5, v9
 ; GFX8-NEXT:    v_subrev_u16_e32 v6, 0x8000, v6
 ; GFX8-NEXT:    v_max_i16_e32 v4, v5, v4
 ; GFX8-NEXT:    v_min_i16_e32 v4, v4, v6
@@ -829,43 +818,41 @@ define i32 @v_ssubsat_v4i8(i32 %lhs.arg, i32 %rhs.arg) {
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v4, 24, v0
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; GFX9-NEXT:    s_mov_b32 s6, 0xffff
 ; GFX9-NEXT:    v_lshrrev_b32_sdwa v5, s4, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX9-NEXT:    v_max_i16_e32 v9, s6, v0
+; GFX9-NEXT:    v_max_i16_e32 v8, -1, v0
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v6, 16, v1
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v7, 24, v1
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
-; GFX9-NEXT:    v_subrev_u16_e32 v9, s4, v9
-; GFX9-NEXT:    s_mov_b32 s5, 0x8000
-; GFX9-NEXT:    v_min_i16_e32 v11, s6, v0
-; GFX9-NEXT:    v_max_i16_e32 v1, v9, v1
-; GFX9-NEXT:    v_subrev_u16_e32 v11, s5, v11
-; GFX9-NEXT:    v_min_i16_e32 v1, v1, v11
+; GFX9-NEXT:    v_subrev_u16_e32 v8, s4, v8
+; GFX9-NEXT:    s_movk_i32 s5, 0x8000
+; GFX9-NEXT:    v_min_i16_e32 v10, -1, v0
+; GFX9-NEXT:    v_max_i16_e32 v1, v8, v1
+; GFX9-NEXT:    v_subrev_u16_e32 v10, s5, v10
+; GFX9-NEXT:    v_min_i16_e32 v1, v1, v10
 ; GFX9-NEXT:    v_sub_u16_e32 v0, v0, v1
-; GFX9-NEXT:    v_max_i16_e32 v1, s6, v2
+; GFX9-NEXT:    v_max_i16_e32 v1, -1, v2
 ; GFX9-NEXT:    v_subrev_u16_e32 v1, s4, v1
-; GFX9-NEXT:    v_min_i16_e32 v9, s6, v2
-; GFX9-NEXT:    v_subrev_u16_e32 v9, s5, v9
+; GFX9-NEXT:    v_min_i16_e32 v8, -1, v2
+; GFX9-NEXT:    v_subrev_u16_e32 v8, s5, v8
 ; GFX9-NEXT:    v_max_i16_e32 v1, v1, v5
-; GFX9-NEXT:    v_min_i16_e32 v1, v1, v9
+; GFX9-NEXT:    v_min_i16_e32 v1, v1, v8
 ; GFX9-NEXT:    v_sub_u16_e32 v1, v2, v1
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v2, 8, v3
-; GFX9-NEXT:    v_mov_b32_e32 v8, 0xffff
-; GFX9-NEXT:    v_mov_b32_e32 v10, 0x7fff
-; GFX9-NEXT:    v_max_i16_e32 v5, v2, v8
+; GFX9-NEXT:    v_mov_b32_e32 v9, 0x7fff
+; GFX9-NEXT:    v_max_i16_e32 v5, -1, v2
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v3, 8, v6
-; GFX9-NEXT:    v_min_i16_e32 v6, v2, v8
-; GFX9-NEXT:    v_sub_u16_e32 v5, v5, v10
+; GFX9-NEXT:    v_min_i16_e32 v6, -1, v2
+; GFX9-NEXT:    v_sub_u16_e32 v5, v5, v9
 ; GFX9-NEXT:    v_subrev_u16_e32 v6, s5, v6
 ; GFX9-NEXT:    v_max_i16_e32 v3, v5, v3
 ; GFX9-NEXT:    v_min_i16_e32 v3, v3, v6
 ; GFX9-NEXT:    v_sub_u16_e32 v2, v2, v3
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v3, 8, v4
-; GFX9-NEXT:    v_max_i16_e32 v5, v3, v8
-; GFX9-NEXT:    v_min_i16_e32 v6, v3, v8
+; GFX9-NEXT:    v_max_i16_e32 v5, -1, v3
+; GFX9-NEXT:    v_min_i16_e32 v6, -1, v3
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v4, 8, v7
-; GFX9-NEXT:    v_sub_u16_e32 v5, v5, v10
+; GFX9-NEXT:    v_sub_u16_e32 v5, v5, v9
 ; GFX9-NEXT:    s_movk_i32 s4, 0xff
 ; GFX9-NEXT:    v_subrev_u16_e32 v6, 0x8000, v6
 ; GFX9-NEXT:    v_max_i16_e32 v4, v5, v4
@@ -884,59 +871,57 @@ define i32 @v_ssubsat_v4i8(i32 %lhs.arg, i32 %rhs.arg) {
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_lshlrev_b16_e64 v4, 8, v0
 ; GFX10-NEXT:    s_mov_b32 s4, 8
-; GFX10-NEXT:    s_mov_b32 s5, 16
-; GFX10-NEXT:    s_mov_b32 s6, 24
+; GFX10-NEXT:    v_lshlrev_b16_e64 v7, 8, v1
 ; GFX10-NEXT:    v_lshrrev_b32_sdwa v2, s4, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT:    v_lshrrev_b32_sdwa v3, s5, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT:    v_lshrrev_b32_sdwa v19, s6, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT:    v_lshlrev_b16_e64 v0, 8, v0
 ; GFX10-NEXT:    v_lshrrev_b32_sdwa v5, s4, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT:    s_mov_b32 s4, 0xffff
+; GFX10-NEXT:    v_max_i16_e64 v8, v4, -1
+; GFX10-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX10-NEXT:    v_min_i16_e64 v10, v4, -1
+; GFX10-NEXT:    v_max_i16_e64 v9, v2, -1
+; GFX10-NEXT:    s_mov_b32 s5, 16
+; GFX10-NEXT:    v_sub_nc_u16_e64 v8, v8, s4
+; GFX10-NEXT:    v_lshrrev_b32_sdwa v3, s5, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    s_mov_b32 s6, 24
 ; GFX10-NEXT:    v_lshrrev_b32_sdwa v6, s5, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT:    s_movk_i32 s5, 0x7fff
-; GFX10-NEXT:    v_max_i16_e64 v8, v0, s4
-; GFX10-NEXT:    v_lshrrev_b32_sdwa v7, s6, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT:    v_max_i16_e64 v9, v2, s4
-; GFX10-NEXT:    v_lshlrev_b16_e64 v1, 8, v1
-; GFX10-NEXT:    v_min_i16_e64 v10, v0, s4
-; GFX10-NEXT:    v_sub_nc_u16_e64 v8, v8, s5
-; GFX10-NEXT:    s_mov_b32 s6, 0x8000
-; GFX10-NEXT:    v_sub_nc_u16_e64 v15, v9, s5
-; GFX10-NEXT:    v_min_i16_e64 v11, v2, s4
-; GFX10-NEXT:    v_mov_b32_e32 v12, 0xffff
-; GFX10-NEXT:    v_max_i16_e64 v1, v8, v1
-; GFX10-NEXT:    v_sub_nc_u16_e64 v10, v10, s6
+; GFX10-NEXT:    s_movk_i32 s5, 0x8000
+; GFX10-NEXT:    v_sub_nc_u16_e64 v15, v9, s4
+; GFX10-NEXT:    v_lshrrev_b32_sdwa v0, s6, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    v_min_i16_e64 v11, v2, -1
+; GFX10-NEXT:    v_max_i16_e64 v7, v8, v7
+; GFX10-NEXT:    v_sub_nc_u16_e64 v10, v10, s5
 ; GFX10-NEXT:    v_max_i16_e64 v5, v15, v5
-; GFX10-NEXT:    v_sub_nc_u16_e64 v8, v11, s6
 ; GFX10-NEXT:    v_mov_b32_e32 v9, 0x7fff
-; GFX10-NEXT:    v_max_i16_e64 v11, v3, v12
-; GFX10-NEXT:    v_min_i16_e64 v1, v1, v10
-; GFX10-NEXT:    v_max_i16_e64 v10, v19, v12
+; GFX10-NEXT:    v_sub_nc_u16_e64 v8, v11, s5
+; GFX10-NEXT:    v_max_i16_e64 v11, v3, -1
+; GFX10-NEXT:    v_min_i16_e64 v7, v7, v10
+; GFX10-NEXT:    v_max_i16_e64 v10, v0, -1
+; GFX10-NEXT:    v_lshrrev_b32_sdwa v1, s6, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX10-NEXT:    v_min_i16_e64 v5, v5, v8
-; GFX10-NEXT:    v_min_i16_e64 v8, v3, v12
 ; GFX10-NEXT:    v_sub_nc_u16_e64 v11, v11, v9
-; GFX10-NEXT:    v_min_i16_e64 v12, v19, v12
-; GFX10-NEXT:    v_sub_nc_u16_e64 v9, v10, v9
+; GFX10-NEXT:    v_min_i16_e64 v8, v3, -1
+; GFX10-NEXT:    v_sub_nc_u16_e64 v15, v10, v9
+; GFX10-NEXT:    v_min_i16_e64 v12, v0, -1
 ; GFX10-NEXT:    v_sub_nc_u16_e64 v2, v2, v5
-; GFX10-NEXT:    v_sub_nc_u16_e64 v5, v8, s6
 ; GFX10-NEXT:    v_max_i16_e64 v6, v11, v6
+; GFX10-NEXT:    v_sub_nc_u16_e64 v5, v8, s5
+; GFX10-NEXT:    v_max_i16_e64 v1, v15, v1
 ; GFX10-NEXT:    v_sub_nc_u16_e64 v8, v12, 0x8000
-; GFX10-NEXT:    v_max_i16_e64 v7, v9, v7
 ; GFX10-NEXT:    s_movk_i32 s4, 0xff
-; GFX10-NEXT:    v_sub_nc_u16_e64 v0, v0, v1
-; GFX10-NEXT:    v_and_b32_sdwa v1, sext(v2), s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
-; GFX10-NEXT:    v_min_i16_e64 v2, v6, v5
-; GFX10-NEXT:    v_min_i16_e64 v5, v7, v8
+; GFX10-NEXT:    v_sub_nc_u16_e64 v4, v4, v7
+; GFX10-NEXT:    v_and_b32_sdwa v2, sext(v2), s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_min_i16_e64 v5, v6, v5
+; GFX10-NEXT:    v_min_i16_e64 v1, v1, v8
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_ashrrev_i16_e64 v0, 8, v0
-; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
-; GFX10-NEXT:    v_sub_nc_u16_e64 v2, v3, v2
-; GFX10-NEXT:    v_sub_nc_u16_e64 v3, v19, v5
-; GFX10-NEXT:    v_and_or_b32 v0, v0, s4, v1
-; GFX10-NEXT:    v_and_b32_sdwa v1, sext(v2), s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
-; GFX10-NEXT:    v_and_b32_sdwa v2, sext(v3), s4 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
-; GFX10-NEXT:    v_or3_b32 v0, v0, v1, v2
+; GFX10-NEXT:    v_ashrrev_i16_e64 v4, 8, v4
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
+; GFX10-NEXT:    v_sub_nc_u16_e64 v3, v3, v5
+; GFX10-NEXT:    v_sub_nc_u16_e64 v0, v0, v1
+; GFX10-NEXT:    v_and_or_b32 v1, v4, s4, v2
+; GFX10-NEXT:    v_and_b32_sdwa v2, sext(v3), s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_and_b32_sdwa v0, sext(v0), s4 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_or3_b32 v0, v1, v2, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %lhs = bitcast i32 %lhs.arg to <4 x i8>
   %rhs = bitcast i32 %rhs.arg to <4 x i8>
@@ -961,7 +946,7 @@ define amdgpu_ps i32 @s_ssubsat_v4i8(i32 inreg %lhs.arg, i32 inreg %rhs.arg) {
 ; GFX6-NEXT:    s_cselect_b32 s10, s0, -1
 ; GFX6-NEXT:    s_sub_i32 s10, s10, s8
 ; GFX6-NEXT:    s_cmp_lt_i32 s0, -1
-; GFX6-NEXT:    s_mov_b32 s9, 0x80000000
+; GFX6-NEXT:    s_brev_b32 s9, 1
 ; GFX6-NEXT:    s_cselect_b32 s11, s0, -1
 ; GFX6-NEXT:    s_sub_i32 s11, s11, s9
 ; GFX6-NEXT:    s_cmp_gt_i32 s10, s1
@@ -1037,13 +1022,13 @@ define amdgpu_ps i32 @s_ssubsat_v4i8(i32 inreg %lhs.arg, i32 inreg %rhs.arg) {
 ; GFX8-NEXT:    s_lshr_b32 s7, s1, 24
 ; GFX8-NEXT:    s_lshl_b32 s1, s1, s8
 ; GFX8-NEXT:    s_sext_i32_i16 s11, s0
-; GFX8-NEXT:    s_sext_i32_i16 s12, 0xffff
+; GFX8-NEXT:    s_sext_i32_i16 s12, -1
 ; GFX8-NEXT:    s_cmp_gt_i32 s11, s12
 ; GFX8-NEXT:    s_movk_i32 s9, 0x7fff
 ; GFX8-NEXT:    s_cselect_b32 s13, s11, s12
 ; GFX8-NEXT:    s_sub_i32 s13, s13, s9
 ; GFX8-NEXT:    s_cmp_lt_i32 s11, s12
-; GFX8-NEXT:    s_mov_b32 s10, 0x8000
+; GFX8-NEXT:    s_movk_i32 s10, 0x8000
 ; GFX8-NEXT:    s_cselect_b32 s11, s11, s12
 ; GFX8-NEXT:    s_sub_i32 s11, s11, s10
 ; GFX8-NEXT:    s_sext_i32_i16 s13, s13
@@ -1142,13 +1127,13 @@ define amdgpu_ps i32 @s_ssubsat_v4i8(i32 inreg %lhs.arg, i32 inreg %rhs.arg) {
 ; GFX9-NEXT:    s_lshr_b32 s7, s1, 24
 ; GFX9-NEXT:    s_lshl_b32 s1, s1, s8
 ; GFX9-NEXT:    s_sext_i32_i16 s11, s0
-; GFX9-NEXT:    s_sext_i32_i16 s12, 0xffff
+; GFX9-NEXT:    s_sext_i32_i16 s12, -1
 ; GFX9-NEXT:    s_cmp_gt_i32 s11, s12
 ; GFX9-NEXT:    s_movk_i32 s9, 0x7fff
 ; GFX9-NEXT:    s_cselect_b32 s13, s11, s12
 ; GFX9-NEXT:    s_sub_i32 s13, s13, s9
 ; GFX9-NEXT:    s_cmp_lt_i32 s11, s12
-; GFX9-NEXT:    s_mov_b32 s10, 0x8000
+; GFX9-NEXT:    s_movk_i32 s10, 0x8000
 ; GFX9-NEXT:    s_cselect_b32 s11, s11, s12
 ; GFX9-NEXT:    s_sub_i32 s11, s11, s10
 ; GFX9-NEXT:    s_sext_i32_i16 s13, s13
@@ -1242,7 +1227,7 @@ define amdgpu_ps i32 @s_ssubsat_v4i8(i32 inreg %lhs.arg, i32 inreg %rhs.arg) {
 ; GFX10-NEXT:    s_lshr_b32 s3, s0, 16
 ; GFX10-NEXT:    s_lshr_b32 s4, s0, 24
 ; GFX10-NEXT:    s_lshl_b32 s0, s0, s6
-; GFX10-NEXT:    s_sext_i32_i16 s10, 0xffff
+; GFX10-NEXT:    s_sext_i32_i16 s10, -1
 ; GFX10-NEXT:    s_sext_i32_i16 s9, s0
 ; GFX10-NEXT:    s_lshr_b32 s5, s1, 8
 ; GFX10-NEXT:    s_lshr_b32 s7, s1, 16
@@ -1251,7 +1236,7 @@ define amdgpu_ps i32 @s_ssubsat_v4i8(i32 inreg %lhs.arg, i32 inreg %rhs.arg) {
 ; GFX10-NEXT:    s_cmp_gt_i32 s9, s10
 ; GFX10-NEXT:    s_movk_i32 s11, 0x7fff
 ; GFX10-NEXT:    s_cselect_b32 s12, s9, s10
-; GFX10-NEXT:    s_mov_b32 s13, 0x8000
+; GFX10-NEXT:    s_movk_i32 s13, 0x8000
 ; GFX10-NEXT:    s_sub_i32 s12, s12, s11
 ; GFX10-NEXT:    s_cmp_lt_i32 s9, s10
 ; GFX10-NEXT:    s_sext_i32_i16 s12, s12
@@ -1726,7 +1711,7 @@ define <2 x i32> @v_ssubsat_v2i32(<2 x i32> %lhs, <2 x i32> %rhs) {
 ; GFX6-NEXT:    s_brev_b32 s4, -2
 ; GFX6-NEXT:    v_max_i32_e32 v4, -1, v0
 ; GFX6-NEXT:    v_subrev_i32_e32 v4, vcc, s4, v4
-; GFX6-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX6-NEXT:    s_brev_b32 s5, 1
 ; GFX6-NEXT:    v_min_i32_e32 v5, -1, v0
 ; GFX6-NEXT:    v_max_i32_e32 v2, v4, v2
 ; GFX6-NEXT:    v_subrev_i32_e32 v5, vcc, s5, v5
@@ -1747,7 +1732,7 @@ define <2 x i32> @v_ssubsat_v2i32(<2 x i32> %lhs, <2 x i32> %rhs) {
 ; GFX8-NEXT:    s_brev_b32 s4, -2
 ; GFX8-NEXT:    v_max_i32_e32 v4, -1, v0
 ; GFX8-NEXT:    v_subrev_u32_e32 v4, vcc, s4, v4
-; GFX8-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX8-NEXT:    s_brev_b32 s5, 1
 ; GFX8-NEXT:    v_min_i32_e32 v5, -1, v0
 ; GFX8-NEXT:    v_max_i32_e32 v2, v4, v2
 ; GFX8-NEXT:    v_subrev_u32_e32 v5, vcc, s5, v5
@@ -1768,7 +1753,7 @@ define <2 x i32> @v_ssubsat_v2i32(<2 x i32> %lhs, <2 x i32> %rhs) {
 ; GFX9-NEXT:    s_brev_b32 s4, -2
 ; GFX9-NEXT:    v_max_i32_e32 v4, -1, v0
 ; GFX9-NEXT:    v_subrev_u32_e32 v4, s4, v4
-; GFX9-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX9-NEXT:    s_brev_b32 s5, 1
 ; GFX9-NEXT:    v_min_i32_e32 v5, -1, v0
 ; GFX9-NEXT:    v_max_i32_e32 v2, v4, v2
 ; GFX9-NEXT:    v_subrev_u32_e32 v5, s5, v5
@@ -1794,7 +1779,7 @@ define <2 x i32> @v_ssubsat_v2i32(<2 x i32> %lhs, <2 x i32> %rhs) {
 ; GFX10-NEXT:    v_min_i32_e32 v7, -1, v1
 ; GFX10-NEXT:    v_subrev_nc_u32_e32 v4, s4, v4
 ; GFX10-NEXT:    v_subrev_nc_u32_e32 v5, s4, v5
-; GFX10-NEXT:    s_mov_b32 s4, 0x80000000
+; GFX10-NEXT:    s_brev_b32 s4, 1
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_subrev_nc_u32_e32 v6, s4, v6
 ; GFX10-NEXT:    v_max_i32_e32 v11, v4, v2
@@ -1817,7 +1802,7 @@ define amdgpu_ps <2 x i32> @s_ssubsat_v2i32(<2 x i32> inreg %lhs, <2 x i32> inre
 ; GFX6-NEXT:    s_cselect_b32 s6, s0, -1
 ; GFX6-NEXT:    s_sub_i32 s6, s6, s4
 ; GFX6-NEXT:    s_cmp_lt_i32 s0, -1
-; GFX6-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX6-NEXT:    s_brev_b32 s5, 1
 ; GFX6-NEXT:    s_cselect_b32 s7, s0, -1
 ; GFX6-NEXT:    s_sub_i32 s7, s7, s5
 ; GFX6-NEXT:    s_cmp_gt_i32 s6, s2
@@ -1845,7 +1830,7 @@ define amdgpu_ps <2 x i32> @s_ssubsat_v2i32(<2 x i32> inreg %lhs, <2 x i32> inre
 ; GFX8-NEXT:    s_cselect_b32 s6, s0, -1
 ; GFX8-NEXT:    s_sub_i32 s6, s6, s4
 ; GFX8-NEXT:    s_cmp_lt_i32 s0, -1
-; GFX8-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX8-NEXT:    s_brev_b32 s5, 1
 ; GFX8-NEXT:    s_cselect_b32 s7, s0, -1
 ; GFX8-NEXT:    s_sub_i32 s7, s7, s5
 ; GFX8-NEXT:    s_cmp_gt_i32 s6, s2
@@ -1873,7 +1858,7 @@ define amdgpu_ps <2 x i32> @s_ssubsat_v2i32(<2 x i32> inreg %lhs, <2 x i32> inre
 ; GFX9-NEXT:    s_cselect_b32 s6, s0, -1
 ; GFX9-NEXT:    s_sub_i32 s6, s6, s4
 ; GFX9-NEXT:    s_cmp_lt_i32 s0, -1
-; GFX9-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX9-NEXT:    s_brev_b32 s5, 1
 ; GFX9-NEXT:    s_cselect_b32 s7, s0, -1
 ; GFX9-NEXT:    s_sub_i32 s7, s7, s5
 ; GFX9-NEXT:    s_cmp_gt_i32 s6, s2
@@ -1899,7 +1884,7 @@ define amdgpu_ps <2 x i32> @s_ssubsat_v2i32(<2 x i32> inreg %lhs, <2 x i32> inre
 ; GFX10-NEXT:    s_cmp_gt_i32 s0, -1
 ; GFX10-NEXT:    s_brev_b32 s4, -2
 ; GFX10-NEXT:    s_cselect_b32 s5, s0, -1
-; GFX10-NEXT:    s_mov_b32 s6, 0x80000000
+; GFX10-NEXT:    s_brev_b32 s6, 1
 ; GFX10-NEXT:    s_sub_i32 s5, s5, s4
 ; GFX10-NEXT:    s_cmp_lt_i32 s0, -1
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
@@ -1933,7 +1918,7 @@ define <3 x i32> @v_ssubsat_v3i32(<3 x i32> %lhs, <3 x i32> %rhs) {
 ; GFX6-NEXT:    s_brev_b32 s4, -2
 ; GFX6-NEXT:    v_max_i32_e32 v6, -1, v0
 ; GFX6-NEXT:    v_subrev_i32_e32 v6, vcc, s4, v6
-; GFX6-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX6-NEXT:    s_brev_b32 s5, 1
 ; GFX6-NEXT:    v_min_i32_e32 v7, -1, v0
 ; GFX6-NEXT:    v_max_i32_e32 v3, v6, v3
 ; GFX6-NEXT:    v_subrev_i32_e32 v7, vcc, s5, v7
@@ -1961,7 +1946,7 @@ define <3 x i32> @v_ssubsat_v3i32(<3 x i32> %lhs, <3 x i32> %rhs) {
 ; GFX8-NEXT:    s_brev_b32 s4, -2
 ; GFX8-NEXT:    v_max_i32_e32 v6, -1, v0
 ; GFX8-NEXT:    v_subrev_u32_e32 v6, vcc, s4, v6
-; GFX8-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX8-NEXT:    s_brev_b32 s5, 1
 ; GFX8-NEXT:    v_min_i32_e32 v7, -1, v0
 ; GFX8-NEXT:    v_max_i32_e32 v3, v6, v3
 ; GFX8-NEXT:    v_subrev_u32_e32 v7, vcc, s5, v7
@@ -1989,7 +1974,7 @@ define <3 x i32> @v_ssubsat_v3i32(<3 x i32> %lhs, <3 x i32> %rhs) {
 ; GFX9-NEXT:    s_brev_b32 s4, -2
 ; GFX9-NEXT:    v_max_i32_e32 v6, -1, v0
 ; GFX9-NEXT:    v_subrev_u32_e32 v6, s4, v6
-; GFX9-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX9-NEXT:    s_brev_b32 s5, 1
 ; GFX9-NEXT:    v_min_i32_e32 v7, -1, v0
 ; GFX9-NEXT:    v_max_i32_e32 v3, v6, v3
 ; GFX9-NEXT:    v_subrev_u32_e32 v7, s5, v7
@@ -2025,7 +2010,7 @@ define <3 x i32> @v_ssubsat_v3i32(<3 x i32> %lhs, <3 x i32> %rhs) {
 ; GFX10-NEXT:    v_subrev_nc_u32_e32 v19, s4, v9
 ; GFX10-NEXT:    v_min_i32_e32 v10, -1, v1
 ; GFX10-NEXT:    v_min_i32_e32 v11, -1, v2
-; GFX10-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX10-NEXT:    s_brev_b32 s5, 1
 ; GFX10-NEXT:    v_max_i32_e32 v14, v6, v3
 ; GFX10-NEXT:    v_subrev_nc_u32_e32 v7, s5, v7
 ; GFX10-NEXT:    v_subrev_nc_u32_e32 v6, s5, v10
@@ -2052,7 +2037,7 @@ define amdgpu_ps <3 x i32> @s_ssubsat_v3i32(<3 x i32> inreg %lhs, <3 x i32> inre
 ; GFX6-NEXT:    s_cselect_b32 s8, s0, -1
 ; GFX6-NEXT:    s_sub_i32 s8, s8, s6
 ; GFX6-NEXT:    s_cmp_lt_i32 s0, -1
-; GFX6-NEXT:    s_mov_b32 s7, 0x80000000
+; GFX6-NEXT:    s_brev_b32 s7, 1
 ; GFX6-NEXT:    s_cselect_b32 s9, s0, -1
 ; GFX6-NEXT:    s_sub_i32 s9, s9, s7
 ; GFX6-NEXT:    s_cmp_gt_i32 s8, s3
@@ -2091,7 +2076,7 @@ define amdgpu_ps <3 x i32> @s_ssubsat_v3i32(<3 x i32> inreg %lhs, <3 x i32> inre
 ; GFX8-NEXT:    s_cselect_b32 s8, s0, -1
 ; GFX8-NEXT:    s_sub_i32 s8, s8, s6
 ; GFX8-NEXT:    s_cmp_lt_i32 s0, -1
-; GFX8-NEXT:    s_mov_b32 s7, 0x80000000
+; GFX8-NEXT:    s_brev_b32 s7, 1
 ; GFX8-NEXT:    s_cselect_b32 s9, s0, -1
 ; GFX8-NEXT:    s_sub_i32 s9, s9, s7
 ; GFX8-NEXT:    s_cmp_gt_i32 s8, s3
@@ -2130,7 +2115,7 @@ define amdgpu_ps <3 x i32> @s_ssubsat_v3i32(<3 x i32> inreg %lhs, <3 x i32> inre
 ; GFX9-NEXT:    s_cselect_b32 s8, s0, -1
 ; GFX9-NEXT:    s_sub_i32 s8, s8, s6
 ; GFX9-NEXT:    s_cmp_lt_i32 s0, -1
-; GFX9-NEXT:    s_mov_b32 s7, 0x80000000
+; GFX9-NEXT:    s_brev_b32 s7, 1
 ; GFX9-NEXT:    s_cselect_b32 s9, s0, -1
 ; GFX9-NEXT:    s_sub_i32 s9, s9, s7
 ; GFX9-NEXT:    s_cmp_gt_i32 s8, s3
@@ -2167,7 +2152,7 @@ define amdgpu_ps <3 x i32> @s_ssubsat_v3i32(<3 x i32> inreg %lhs, <3 x i32> inre
 ; GFX10-NEXT:    s_cmp_gt_i32 s0, -1
 ; GFX10-NEXT:    s_brev_b32 s6, -2
 ; GFX10-NEXT:    s_cselect_b32 s7, s0, -1
-; GFX10-NEXT:    s_mov_b32 s8, 0x80000000
+; GFX10-NEXT:    s_brev_b32 s8, 1
 ; GFX10-NEXT:    s_sub_i32 s7, s7, s6
 ; GFX10-NEXT:    s_cmp_lt_i32 s0, -1
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
@@ -2212,7 +2197,7 @@ define <4 x i32> @v_ssubsat_v4i32(<4 x i32> %lhs, <4 x i32> %rhs) {
 ; GFX6-NEXT:    s_brev_b32 s4, -2
 ; GFX6-NEXT:    v_max_i32_e32 v8, -1, v0
 ; GFX6-NEXT:    v_subrev_i32_e32 v8, vcc, s4, v8
-; GFX6-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX6-NEXT:    s_brev_b32 s5, 1
 ; GFX6-NEXT:    v_min_i32_e32 v9, -1, v0
 ; GFX6-NEXT:    v_max_i32_e32 v4, v8, v4
 ; GFX6-NEXT:    v_subrev_i32_e32 v9, vcc, s5, v9
@@ -2247,7 +2232,7 @@ define <4 x i32> @v_ssubsat_v4i32(<4 x i32> %lhs, <4 x i32> %rhs) {
 ; GFX8-NEXT:    s_brev_b32 s4, -2
 ; GFX8-NEXT:    v_max_i32_e32 v8, -1, v0
 ; GFX8-NEXT:    v_subrev_u32_e32 v8, vcc, s4, v8
-; GFX8-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX8-NEXT:    s_brev_b32 s5, 1
 ; GFX8-NEXT:    v_min_i32_e32 v9, -1, v0
 ; GFX8-NEXT:    v_max_i32_e32 v4, v8, v4
 ; GFX8-NEXT:    v_subrev_u32_e32 v9, vcc, s5, v9
@@ -2282,7 +2267,7 @@ define <4 x i32> @v_ssubsat_v4i32(<4 x i32> %lhs, <4 x i32> %rhs) {
 ; GFX9-NEXT:    s_brev_b32 s4, -2
 ; GFX9-NEXT:    v_max_i32_e32 v8, -1, v0
 ; GFX9-NEXT:    v_subrev_u32_e32 v8, s4, v8
-; GFX9-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX9-NEXT:    s_brev_b32 s5, 1
 ; GFX9-NEXT:    v_min_i32_e32 v9, -1, v0
 ; GFX9-NEXT:    v_max_i32_e32 v4, v8, v4
 ; GFX9-NEXT:    v_subrev_u32_e32 v9, s5, v9
@@ -2328,7 +2313,7 @@ define <4 x i32> @v_ssubsat_v4i32(<4 x i32> %lhs, <4 x i32> %rhs) {
 ; GFX10-NEXT:    v_min_i32_e32 v13, -1, v2
 ; GFX10-NEXT:    v_subrev_nc_u32_e32 v8, s4, v8
 ; GFX10-NEXT:    v_min_i32_e32 v14, -1, v3
-; GFX10-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX10-NEXT:    s_brev_b32 s5, 1
 ; GFX10-NEXT:    v_max_i32_e32 v4, v15, v4
 ; GFX10-NEXT:    v_subrev_nc_u32_e32 v9, s5, v9
 ; GFX10-NEXT:    v_max_i32_e32 v5, v10, v5
@@ -2359,7 +2344,7 @@ define amdgpu_ps <4 x i32> @s_ssubsat_v4i32(<4 x i32> inreg %lhs, <4 x i32> inre
 ; GFX6-NEXT:    s_cselect_b32 s10, s0, -1
 ; GFX6-NEXT:    s_sub_i32 s10, s10, s8
 ; GFX6-NEXT:    s_cmp_lt_i32 s0, -1
-; GFX6-NEXT:    s_mov_b32 s9, 0x80000000
+; GFX6-NEXT:    s_brev_b32 s9, 1
 ; GFX6-NEXT:    s_cselect_b32 s11, s0, -1
 ; GFX6-NEXT:    s_sub_i32 s11, s11, s9
 ; GFX6-NEXT:    s_cmp_gt_i32 s10, s4
@@ -2409,7 +2394,7 @@ define amdgpu_ps <4 x i32> @s_ssubsat_v4i32(<4 x i32> inreg %lhs, <4 x i32> inre
 ; GFX8-NEXT:    s_cselect_b32 s10, s0, -1
 ; GFX8-NEXT:    s_sub_i32 s10, s10, s8
 ; GFX8-NEXT:    s_cmp_lt_i32 s0, -1
-; GFX8-NEXT:    s_mov_b32 s9, 0x80000000
+; GFX8-NEXT:    s_brev_b32 s9, 1
 ; GFX8-NEXT:    s_cselect_b32 s11, s0, -1
 ; GFX8-NEXT:    s_sub_i32 s11, s11, s9
 ; GFX8-NEXT:    s_cmp_gt_i32 s10, s4
@@ -2459,7 +2444,7 @@ define amdgpu_ps <4 x i32> @s_ssubsat_v4i32(<4 x i32> inreg %lhs, <4 x i32> inre
 ; GFX9-NEXT:    s_cselect_b32 s10, s0, -1
 ; GFX9-NEXT:    s_sub_i32 s10, s10, s8
 ; GFX9-NEXT:    s_cmp_lt_i32 s0, -1
-; GFX9-NEXT:    s_mov_b32 s9, 0x80000000
+; GFX9-NEXT:    s_brev_b32 s9, 1
 ; GFX9-NEXT:    s_cselect_b32 s11, s0, -1
 ; GFX9-NEXT:    s_sub_i32 s11, s11, s9
 ; GFX9-NEXT:    s_cmp_gt_i32 s10, s4
@@ -2507,7 +2492,7 @@ define amdgpu_ps <4 x i32> @s_ssubsat_v4i32(<4 x i32> inreg %lhs, <4 x i32> inre
 ; GFX10-NEXT:    s_cmp_gt_i32 s0, -1
 ; GFX10-NEXT:    s_brev_b32 s8, -2
 ; GFX10-NEXT:    s_cselect_b32 s9, s0, -1
-; GFX10-NEXT:    s_mov_b32 s10, 0x80000000
+; GFX10-NEXT:    s_brev_b32 s10, 1
 ; GFX10-NEXT:    s_sub_i32 s9, s9, s8
 ; GFX10-NEXT:    s_cmp_lt_i32 s0, -1
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
@@ -2563,7 +2548,7 @@ define <5 x i32> @v_ssubsat_v5i32(<5 x i32> %lhs, <5 x i32> %rhs) {
 ; GFX6-NEXT:    s_brev_b32 s4, -2
 ; GFX6-NEXT:    v_max_i32_e32 v10, -1, v0
 ; GFX6-NEXT:    v_subrev_i32_e32 v10, vcc, s4, v10
-; GFX6-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX6-NEXT:    s_brev_b32 s5, 1
 ; GFX6-NEXT:    v_min_i32_e32 v12, -1, v0
 ; GFX6-NEXT:    v_max_i32_e32 v5, v10, v5
 ; GFX6-NEXT:    v_subrev_i32_e32 v12, vcc, s5, v12
@@ -2586,7 +2571,7 @@ define <5 x i32> @v_ssubsat_v5i32(<5 x i32> %lhs, <5 x i32> %rhs) {
 ; GFX6-NEXT:    v_bfrev_b32_e32 v11, -2
 ; GFX6-NEXT:    v_max_i32_e32 v5, -1, v3
 ; GFX6-NEXT:    v_sub_i32_e32 v5, vcc, v5, v11
-; GFX6-NEXT:    v_mov_b32_e32 v13, 0x80000000
+; GFX6-NEXT:    v_bfrev_b32_e32 v13, 1
 ; GFX6-NEXT:    v_min_i32_e32 v6, -1, v3
 ; GFX6-NEXT:    v_sub_i32_e32 v6, vcc, v6, v13
 ; GFX6-NEXT:    v_max_i32_e32 v5, v5, v8
@@ -2607,7 +2592,7 @@ define <5 x i32> @v_ssubsat_v5i32(<5 x i32> %lhs, <5 x i32> %rhs) {
 ; GFX8-NEXT:    s_brev_b32 s4, -2
 ; GFX8-NEXT:    v_max_i32_e32 v10, -1, v0
 ; GFX8-NEXT:    v_subrev_u32_e32 v10, vcc, s4, v10
-; GFX8-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX8-NEXT:    s_brev_b32 s5, 1
 ; GFX8-NEXT:    v_min_i32_e32 v12, -1, v0
 ; GFX8-NEXT:    v_max_i32_e32 v5, v10, v5
 ; GFX8-NEXT:    v_subrev_u32_e32 v12, vcc, s5, v12
@@ -2630,7 +2615,7 @@ define <5 x i32> @v_ssubsat_v5i32(<5 x i32> %lhs, <5 x i32> %rhs) {
 ; GFX8-NEXT:    v_bfrev_b32_e32 v11, -2
 ; GFX8-NEXT:    v_max_i32_e32 v5, -1, v3
 ; GFX8-NEXT:    v_sub_u32_e32 v5, vcc, v5, v11
-; GFX8-NEXT:    v_mov_b32_e32 v13, 0x80000000
+; GFX8-NEXT:    v_bfrev_b32_e32 v13, 1
 ; GFX8-NEXT:    v_min_i32_e32 v6, -1, v3
 ; GFX8-NEXT:    v_sub_u32_e32 v6, vcc, v6, v13
 ; GFX8-NEXT:    v_max_i32_e32 v5, v5, v8
@@ -2651,7 +2636,7 @@ define <5 x i32> @v_ssubsat_v5i32(<5 x i32> %lhs, <5 x i32> %rhs) {
 ; GFX9-NEXT:    s_brev_b32 s4, -2
 ; GFX9-NEXT:    v_max_i32_e32 v10, -1, v0
 ; GFX9-NEXT:    v_subrev_u32_e32 v10, s4, v10
-; GFX9-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX9-NEXT:    s_brev_b32 s5, 1
 ; GFX9-NEXT:    v_min_i32_e32 v12, -1, v0
 ; GFX9-NEXT:    v_max_i32_e32 v5, v10, v5
 ; GFX9-NEXT:    v_subrev_u32_e32 v12, s5, v12
@@ -2674,7 +2659,7 @@ define <5 x i32> @v_ssubsat_v5i32(<5 x i32> %lhs, <5 x i32> %rhs) {
 ; GFX9-NEXT:    v_bfrev_b32_e32 v11, -2
 ; GFX9-NEXT:    v_max_i32_e32 v5, -1, v3
 ; GFX9-NEXT:    v_sub_u32_e32 v5, v5, v11
-; GFX9-NEXT:    v_mov_b32_e32 v13, 0x80000000
+; GFX9-NEXT:    v_bfrev_b32_e32 v13, 1
 ; GFX9-NEXT:    v_min_i32_e32 v6, -1, v3
 ; GFX9-NEXT:    v_sub_u32_e32 v6, v6, v13
 ; GFX9-NEXT:    v_max_i32_e32 v5, v5, v8
@@ -2701,7 +2686,7 @@ define <5 x i32> @v_ssubsat_v5i32(<5 x i32> %lhs, <5 x i32> %rhs) {
 ; GFX10-NEXT:    v_subrev_nc_u32_e32 v10, s4, v10
 ; GFX10-NEXT:    v_subrev_nc_u32_e32 v13, s4, v13
 ; GFX10-NEXT:    v_min_i32_e32 v12, -1, v0
-; GFX10-NEXT:    v_mov_b32_e32 v14, 0x80000000
+; GFX10-NEXT:    v_bfrev_b32_e32 v14, 1
 ; GFX10-NEXT:    v_min_i32_e32 v15, -1, v1
 ; GFX10-NEXT:    v_max_i32_e32 v5, v10, v5
 ; GFX10-NEXT:    v_max_i32_e32 v10, -1, v2
@@ -2713,7 +2698,7 @@ define <5 x i32> @v_ssubsat_v5i32(<5 x i32> %lhs, <5 x i32> %rhs) {
 ; GFX10-NEXT:    v_min_i32_e32 v19, -1, v4
 ; GFX10-NEXT:    v_sub_nc_u32_e32 v13, v13, v11
 ; GFX10-NEXT:    v_sub_nc_u32_e32 v11, v17, v11
-; GFX10-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX10-NEXT:    s_brev_b32 s5, 1
 ; GFX10-NEXT:    v_max_i32_e32 v7, v10, v7
 ; GFX10-NEXT:    v_subrev_nc_u32_e32 v12, s5, v12
 ; GFX10-NEXT:    v_subrev_nc_u32_e32 v15, s5, v15
@@ -2746,7 +2731,7 @@ define amdgpu_ps <5 x i32> @s_ssubsat_v5i32(<5 x i32> inreg %lhs, <5 x i32> inre
 ; GFX6-NEXT:    s_cselect_b32 s12, s0, -1
 ; GFX6-NEXT:    s_sub_i32 s12, s12, s10
 ; GFX6-NEXT:    s_cmp_lt_i32 s0, -1
-; GFX6-NEXT:    s_mov_b32 s11, 0x80000000
+; GFX6-NEXT:    s_brev_b32 s11, 1
 ; GFX6-NEXT:    s_cselect_b32 s13, s0, -1
 ; GFX6-NEXT:    s_sub_i32 s13, s13, s11
 ; GFX6-NEXT:    s_cmp_gt_i32 s12, s5
@@ -2807,7 +2792,7 @@ define amdgpu_ps <5 x i32> @s_ssubsat_v5i32(<5 x i32> inreg %lhs, <5 x i32> inre
 ; GFX8-NEXT:    s_cselect_b32 s12, s0, -1
 ; GFX8-NEXT:    s_sub_i32 s12, s12, s10
 ; GFX8-NEXT:    s_cmp_lt_i32 s0, -1
-; GFX8-NEXT:    s_mov_b32 s11, 0x80000000
+; GFX8-NEXT:    s_brev_b32 s11, 1
 ; GFX8-NEXT:    s_cselect_b32 s13, s0, -1
 ; GFX8-NEXT:    s_sub_i32 s13, s13, s11
 ; GFX8-NEXT:    s_cmp_gt_i32 s12, s5
@@ -2868,7 +2853,7 @@ define amdgpu_ps <5 x i32> @s_ssubsat_v5i32(<5 x i32> inreg %lhs, <5 x i32> inre
 ; GFX9-NEXT:    s_cselect_b32 s12, s0, -1
 ; GFX9-NEXT:    s_sub_i32 s12, s12, s10
 ; GFX9-NEXT:    s_cmp_lt_i32 s0, -1
-; GFX9-NEXT:    s_mov_b32 s11, 0x80000000
+; GFX9-NEXT:    s_brev_b32 s11, 1
 ; GFX9-NEXT:    s_cselect_b32 s13, s0, -1
 ; GFX9-NEXT:    s_sub_i32 s13, s13, s11
 ; GFX9-NEXT:    s_cmp_gt_i32 s12, s5
@@ -2927,7 +2912,7 @@ define amdgpu_ps <5 x i32> @s_ssubsat_v5i32(<5 x i32> inreg %lhs, <5 x i32> inre
 ; GFX10-NEXT:    s_cmp_gt_i32 s0, -1
 ; GFX10-NEXT:    s_brev_b32 s10, -2
 ; GFX10-NEXT:    s_cselect_b32 s11, s0, -1
-; GFX10-NEXT:    s_mov_b32 s12, 0x80000000
+; GFX10-NEXT:    s_brev_b32 s12, 1
 ; GFX10-NEXT:    s_sub_i32 s11, s11, s10
 ; GFX10-NEXT:    s_cmp_lt_i32 s0, -1
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
@@ -2995,7 +2980,7 @@ define <16 x i32> @v_ssubsat_v16i32(<16 x i32> %lhs, <16 x i32> %rhs) {
 ; GFX6-NEXT:    v_max_i32_e32 v32, -1, v0
 ; GFX6-NEXT:    v_subrev_i32_e32 v32, vcc, s4, v32
 ; GFX6-NEXT:    v_max_i32_e32 v16, v32, v16
-; GFX6-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX6-NEXT:    s_brev_b32 s5, 1
 ; GFX6-NEXT:    v_min_i32_e32 v32, -1, v0
 ; GFX6-NEXT:    v_subrev_i32_e32 v32, vcc, s5, v32
 ; GFX6-NEXT:    v_min_i32_e32 v16, v16, v32
@@ -3018,7 +3003,7 @@ define <16 x i32> @v_ssubsat_v16i32(<16 x i32> %lhs, <16 x i32> %rhs) {
 ; GFX6-NEXT:    v_max_i32_e32 v17, -1, v3
 ; GFX6-NEXT:    v_sub_i32_e32 v17, vcc, v17, v16
 ; GFX6-NEXT:    v_max_i32_e32 v17, v17, v19
-; GFX6-NEXT:    v_mov_b32_e32 v18, 0x80000000
+; GFX6-NEXT:    v_bfrev_b32_e32 v18, 1
 ; GFX6-NEXT:    v_min_i32_e32 v19, -1, v3
 ; GFX6-NEXT:    v_sub_i32_e32 v19, vcc, v19, v18
 ; GFX6-NEXT:    v_min_i32_e32 v17, v17, v19
@@ -3116,7 +3101,7 @@ define <16 x i32> @v_ssubsat_v16i32(<16 x i32> %lhs, <16 x i32> %rhs) {
 ; GFX8-NEXT:    v_max_i32_e32 v32, -1, v0
 ; GFX8-NEXT:    v_subrev_u32_e32 v32, vcc, s4, v32
 ; GFX8-NEXT:    v_max_i32_e32 v16, v32, v16
-; GFX8-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX8-NEXT:    s_brev_b32 s5, 1
 ; GFX8-NEXT:    v_min_i32_e32 v32, -1, v0
 ; GFX8-NEXT:    v_subrev_u32_e32 v32, vcc, s5, v32
 ; GFX8-NEXT:    v_min_i32_e32 v16, v16, v32
@@ -3139,7 +3124,7 @@ define <16 x i32> @v_ssubsat_v16i32(<16 x i32> %lhs, <16 x i32> %rhs) {
 ; GFX8-NEXT:    v_max_i32_e32 v17, -1, v3
 ; GFX8-NEXT:    v_sub_u32_e32 v17, vcc, v17, v16
 ; GFX8-NEXT:    v_max_i32_e32 v17, v17, v19
-; GFX8-NEXT:    v_mov_b32_e32 v18, 0x80000000
+; GFX8-NEXT:    v_bfrev_b32_e32 v18, 1
 ; GFX8-NEXT:    v_min_i32_e32 v19, -1, v3
 ; GFX8-NEXT:    v_sub_u32_e32 v19, vcc, v19, v18
 ; GFX8-NEXT:    v_min_i32_e32 v17, v17, v19
@@ -3237,7 +3222,7 @@ define <16 x i32> @v_ssubsat_v16i32(<16 x i32> %lhs, <16 x i32> %rhs) {
 ; GFX9-NEXT:    v_max_i32_e32 v32, -1, v0
 ; GFX9-NEXT:    v_subrev_u32_e32 v32, s4, v32
 ; GFX9-NEXT:    v_max_i32_e32 v16, v32, v16
-; GFX9-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX9-NEXT:    s_brev_b32 s5, 1
 ; GFX9-NEXT:    v_min_i32_e32 v32, -1, v0
 ; GFX9-NEXT:    v_subrev_u32_e32 v32, s5, v32
 ; GFX9-NEXT:    v_min_i32_e32 v16, v16, v32
@@ -3260,7 +3245,7 @@ define <16 x i32> @v_ssubsat_v16i32(<16 x i32> %lhs, <16 x i32> %rhs) {
 ; GFX9-NEXT:    v_max_i32_e32 v17, -1, v3
 ; GFX9-NEXT:    v_sub_u32_e32 v17, v17, v16
 ; GFX9-NEXT:    v_max_i32_e32 v17, v17, v19
-; GFX9-NEXT:    v_mov_b32_e32 v18, 0x80000000
+; GFX9-NEXT:    v_bfrev_b32_e32 v18, 1
 ; GFX9-NEXT:    v_min_i32_e32 v19, -1, v3
 ; GFX9-NEXT:    v_sub_u32_e32 v19, v19, v18
 ; GFX9-NEXT:    v_min_i32_e32 v17, v17, v19
@@ -3358,7 +3343,7 @@ define <16 x i32> @v_ssubsat_v16i32(<16 x i32> %lhs, <16 x i32> %rhs) {
 ; GFX10-NEXT:    v_max_i32_e32 v32, -1, v0
 ; GFX10-NEXT:    s_brev_b32 s4, -2
 ; GFX10-NEXT:    v_min_i32_e32 v33, -1, v0
-; GFX10-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX10-NEXT:    s_brev_b32 s5, 1
 ; GFX10-NEXT:    v_max_i32_e32 v36, -1, v2
 ; GFX10-NEXT:    v_subrev_nc_u32_e32 v35, s4, v32
 ; GFX10-NEXT:    v_max_i32_e32 v32, -1, v1
@@ -3375,7 +3360,7 @@ define <16 x i32> @v_ssubsat_v16i32(<16 x i32> %lhs, <16 x i32> %rhs) {
 ; GFX10-NEXT:    v_max_i32_e32 v38, v32, v17
 ; GFX10-NEXT:    v_max_i32_e32 v17, -1, v4
 ; GFX10-NEXT:    v_sub_nc_u32_e32 v36, v39, v34
-; GFX10-NEXT:    v_mov_b32_e32 v35, 0x80000000
+; GFX10-NEXT:    v_bfrev_b32_e32 v35, 1
 ; GFX10-NEXT:    v_subrev_nc_u32_e32 v32, s5, v33
 ; GFX10-NEXT:    v_sub_nc_u32_e32 v0, v0, v16
 ; GFX10-NEXT:    v_min_i32_e32 v33, -1, v3
@@ -3485,7 +3470,7 @@ define amdgpu_ps <16 x i32> @s_ssubsat_v16i32(<16 x i32> inreg %lhs, <16 x i32>
 ; GFX6-NEXT:    s_cselect_b32 s34, s0, -1
 ; GFX6-NEXT:    s_sub_i32 s34, s34, s32
 ; GFX6-NEXT:    s_cmp_lt_i32 s0, -1
-; GFX6-NEXT:    s_mov_b32 s33, 0x80000000
+; GFX6-NEXT:    s_brev_b32 s33, 1
 ; GFX6-NEXT:    s_cselect_b32 s35, s0, -1
 ; GFX6-NEXT:    s_sub_i32 s35, s35, s33
 ; GFX6-NEXT:    s_cmp_gt_i32 s34, s16
@@ -3667,7 +3652,7 @@ define amdgpu_ps <16 x i32> @s_ssubsat_v16i32(<16 x i32> inreg %lhs, <16 x i32>
 ; GFX8-NEXT:    s_cselect_b32 s34, s0, -1
 ; GFX8-NEXT:    s_sub_i32 s34, s34, s32
 ; GFX8-NEXT:    s_cmp_lt_i32 s0, -1
-; GFX8-NEXT:    s_mov_b32 s33, 0x80000000
+; GFX8-NEXT:    s_brev_b32 s33, 1
 ; GFX8-NEXT:    s_cselect_b32 s35, s0, -1
 ; GFX8-NEXT:    s_sub_i32 s35, s35, s33
 ; GFX8-NEXT:    s_cmp_gt_i32 s34, s16
@@ -3849,7 +3834,7 @@ define amdgpu_ps <16 x i32> @s_ssubsat_v16i32(<16 x i32> inreg %lhs, <16 x i32>
 ; GFX9-NEXT:    s_cselect_b32 s34, s0, -1
 ; GFX9-NEXT:    s_sub_i32 s34, s34, s32
 ; GFX9-NEXT:    s_cmp_lt_i32 s0, -1
-; GFX9-NEXT:    s_mov_b32 s33, 0x80000000
+; GFX9-NEXT:    s_brev_b32 s33, 1
 ; GFX9-NEXT:    s_cselect_b32 s35, s0, -1
 ; GFX9-NEXT:    s_sub_i32 s35, s35, s33
 ; GFX9-NEXT:    s_cmp_gt_i32 s34, s16
@@ -4029,7 +4014,7 @@ define amdgpu_ps <16 x i32> @s_ssubsat_v16i32(<16 x i32> inreg %lhs, <16 x i32>
 ; GFX10-NEXT:    s_cmp_gt_i32 s0, -1
 ; GFX10-NEXT:    s_brev_b32 s46, -2
 ; GFX10-NEXT:    s_cselect_b32 s33, s0, -1
-; GFX10-NEXT:    s_mov_b32 s34, 0x80000000
+; GFX10-NEXT:    s_brev_b32 s34, 1
 ; GFX10-NEXT:    s_sub_i32 s47, s33, s46
 ; GFX10-NEXT:    s_cmp_lt_i32 s0, -1
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
@@ -4229,9 +4214,8 @@ define i16 @v_ssubsat_i16(i16 %lhs, i16 %rhs) {
 ; GFX8-LABEL: v_ssubsat_i16:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    s_mov_b32 s4, 0xffff
-; GFX8-NEXT:    v_max_i16_e32 v2, s4, v0
-; GFX8-NEXT:    v_min_i16_e32 v3, s4, v0
+; GFX8-NEXT:    v_max_i16_e32 v2, -1, v0
+; GFX8-NEXT:    v_min_i16_e32 v3, -1, v0
 ; GFX8-NEXT:    v_subrev_u16_e32 v2, 0x7fff, v2
 ; GFX8-NEXT:    v_subrev_u16_e32 v3, 0x8000, v3
 ; GFX8-NEXT:    v_max_i16_e32 v1, v2, v1
@@ -4242,9 +4226,8 @@ define i16 @v_ssubsat_i16(i16 %lhs, i16 %rhs) {
 ; GFX9-LABEL: v_ssubsat_i16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    s_mov_b32 s4, 0xffff
-; GFX9-NEXT:    v_max_i16_e32 v2, s4, v0
-; GFX9-NEXT:    v_min_i16_e32 v3, s4, v0
+; GFX9-NEXT:    v_max_i16_e32 v2, -1, v0
+; GFX9-NEXT:    v_min_i16_e32 v3, -1, v0
 ; GFX9-NEXT:    v_subrev_u16_e32 v2, 0x7fff, v2
 ; GFX9-NEXT:    v_subrev_u16_e32 v3, 0x8000, v3
 ; GFX9-NEXT:    v_max_i16_e32 v1, v2, v1
@@ -4256,10 +4239,9 @@ define i16 @v_ssubsat_i16(i16 %lhs, i16 %rhs) {
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    s_mov_b32 s4, 0xffff
+; GFX10-NEXT:    v_max_i16_e64 v2, v0, -1
+; GFX10-NEXT:    v_min_i16_e64 v3, v0, -1
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_max_i16_e64 v2, v0, s4
-; GFX10-NEXT:    v_min_i16_e64 v3, v0, s4
 ; GFX10-NEXT:    v_sub_nc_u16_e64 v2, v2, 0x7fff
 ; GFX10-NEXT:    v_sub_nc_u16_e64 v3, v3, 0x8000
 ; GFX10-NEXT:    v_max_i16_e64 v1, v2, v1
@@ -4292,13 +4274,13 @@ define amdgpu_ps i16 @s_ssubsat_i16(i16 inreg %lhs, i16 inreg %rhs) {
 ; GFX8-LABEL: s_ssubsat_i16:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_sext_i32_i16 s2, s0
-; GFX8-NEXT:    s_sext_i32_i16 s3, 0xffff
+; GFX8-NEXT:    s_sext_i32_i16 s3, -1
 ; GFX8-NEXT:    s_cmp_gt_i32 s2, s3
 ; GFX8-NEXT:    s_cselect_b32 s4, s2, s3
 ; GFX8-NEXT:    s_sub_i32 s4, s4, 0x7fff
 ; GFX8-NEXT:    s_cmp_lt_i32 s2, s3
 ; GFX8-NEXT:    s_cselect_b32 s2, s2, s3
-; GFX8-NEXT:    s_sub_i32 s2, s2, 0x8000
+; GFX8-NEXT:    s_sub_i32 s2, s2, 0xffff8000
 ; GFX8-NEXT:    s_sext_i32_i16 s3, s4
 ; GFX8-NEXT:    s_sext_i32_i16 s1, s1
 ; GFX8-NEXT:    s_cmp_gt_i32 s3, s1
@@ -4313,13 +4295,13 @@ define amdgpu_ps i16 @s_ssubsat_i16(i16 inreg %lhs, i16 inreg %rhs) {
 ; GFX9-LABEL: s_ssubsat_i16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_sext_i32_i16 s2, s0
-; GFX9-NEXT:    s_sext_i32_i16 s3, 0xffff
+; GFX9-NEXT:    s_sext_i32_i16 s3, -1
 ; GFX9-NEXT:    s_cmp_gt_i32 s2, s3
 ; GFX9-NEXT:    s_cselect_b32 s4, s2, s3
 ; GFX9-NEXT:    s_sub_i32 s4, s4, 0x7fff
 ; GFX9-NEXT:    s_cmp_lt_i32 s2, s3
 ; GFX9-NEXT:    s_cselect_b32 s2, s2, s3
-; GFX9-NEXT:    s_sub_i32 s2, s2, 0x8000
+; GFX9-NEXT:    s_sub_i32 s2, s2, 0xffff8000
 ; GFX9-NEXT:    s_sext_i32_i16 s3, s4
 ; GFX9-NEXT:    s_sext_i32_i16 s1, s1
 ; GFX9-NEXT:    s_cmp_gt_i32 s3, s1
@@ -4333,7 +4315,7 @@ define amdgpu_ps i16 @s_ssubsat_i16(i16 inreg %lhs, i16 inreg %rhs) {
 ;
 ; GFX10-LABEL: s_ssubsat_i16:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_sext_i32_i16 s2, 0xffff
+; GFX10-NEXT:    s_sext_i32_i16 s2, -1
 ; GFX10-NEXT:    s_sext_i32_i16 s3, s0
 ; GFX10-NEXT:    s_sext_i32_i16 s1, s1
 ; GFX10-NEXT:    s_cmp_gt_i32 s3, s2
@@ -4343,7 +4325,7 @@ define amdgpu_ps i16 @s_ssubsat_i16(i16 inreg %lhs, i16 inreg %rhs) {
 ; GFX10-NEXT:    s_cmp_lt_i32 s3, s2
 ; GFX10-NEXT:    s_cselect_b32 s2, s3, s2
 ; GFX10-NEXT:    s_sext_i32_i16 s3, s4
-; GFX10-NEXT:    s_sub_i32 s2, s2, 0x8000
+; GFX10-NEXT:    s_sub_i32 s2, s2, 0xffff8000
 ; GFX10-NEXT:    s_cmp_gt_i32 s3, s1
 ; GFX10-NEXT:    s_sext_i32_i16 s2, s2
 ; GFX10-NEXT:    s_cselect_b32 s1, s3, s1
@@ -4376,13 +4358,13 @@ define amdgpu_ps half @ssubsat_i16_sv(i16 inreg %lhs, i16 %rhs) {
 ; GFX8-LABEL: ssubsat_i16_sv:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_sext_i32_i16 s1, s0
-; GFX8-NEXT:    s_sext_i32_i16 s2, 0xffff
+; GFX8-NEXT:    s_sext_i32_i16 s2, -1
 ; GFX8-NEXT:    s_cmp_gt_i32 s1, s2
 ; GFX8-NEXT:    s_cselect_b32 s3, s1, s2
 ; GFX8-NEXT:    s_sub_i32 s3, s3, 0x7fff
 ; GFX8-NEXT:    s_cmp_lt_i32 s1, s2
 ; GFX8-NEXT:    s_cselect_b32 s1, s1, s2
-; GFX8-NEXT:    s_sub_i32 s1, s1, 0x8000
+; GFX8-NEXT:    s_sub_i32 s1, s1, 0xffff8000
 ; GFX8-NEXT:    v_max_i16_e32 v0, s3, v0
 ; GFX8-NEXT:    v_min_i16_e32 v0, s1, v0
 ; GFX8-NEXT:    v_sub_u16_e32 v0, s0, v0
@@ -4391,13 +4373,13 @@ define amdgpu_ps half @ssubsat_i16_sv(i16 inreg %lhs, i16 %rhs) {
 ; GFX9-LABEL: ssubsat_i16_sv:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_sext_i32_i16 s1, s0
-; GFX9-NEXT:    s_sext_i32_i16 s2, 0xffff
+; GFX9-NEXT:    s_sext_i32_i16 s2, -1
 ; GFX9-NEXT:    s_cmp_gt_i32 s1, s2
 ; GFX9-NEXT:    s_cselect_b32 s3, s1, s2
 ; GFX9-NEXT:    s_sub_i32 s3, s3, 0x7fff
 ; GFX9-NEXT:    s_cmp_lt_i32 s1, s2
 ; GFX9-NEXT:    s_cselect_b32 s1, s1, s2
-; GFX9-NEXT:    s_sub_i32 s1, s1, 0x8000
+; GFX9-NEXT:    s_sub_i32 s1, s1, 0xffff8000
 ; GFX9-NEXT:    v_max_i16_e32 v0, s3, v0
 ; GFX9-NEXT:    v_min_i16_e32 v0, s1, v0
 ; GFX9-NEXT:    v_sub_u16_e32 v0, s0, v0
@@ -4406,7 +4388,7 @@ define amdgpu_ps half @ssubsat_i16_sv(i16 inreg %lhs, i16 %rhs) {
 ; GFX10-LABEL: ssubsat_i16_sv:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_sext_i32_i16 s1, s0
-; GFX10-NEXT:    s_sext_i32_i16 s2, 0xffff
+; GFX10-NEXT:    s_sext_i32_i16 s2, -1
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_cmp_gt_i32 s1, s2
 ; GFX10-NEXT:    s_cselect_b32 s3, s1, s2
@@ -4414,7 +4396,7 @@ define amdgpu_ps half @ssubsat_i16_sv(i16 inreg %lhs, i16 %rhs) {
 ; GFX10-NEXT:    s_cmp_lt_i32 s1, s2
 ; GFX10-NEXT:    v_max_i16_e64 v0, s3, v0
 ; GFX10-NEXT:    s_cselect_b32 s1, s1, s2
-; GFX10-NEXT:    s_sub_i32 s1, s1, 0x8000
+; GFX10-NEXT:    s_sub_i32 s1, s1, 0xffff8000
 ; GFX10-NEXT:    v_min_i16_e64 v0, v0, s1
 ; GFX10-NEXT:    v_sub_nc_u16_e64 v0, s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -4440,10 +4422,9 @@ define amdgpu_ps half @ssubsat_i16_vs(i16 %lhs, i16 inreg %rhs) {
 ;
 ; GFX8-LABEL: ssubsat_i16_vs:
 ; GFX8:       ; %bb.0:
-; GFX8-NEXT:    s_mov_b32 s1, 0xffff
-; GFX8-NEXT:    v_max_i16_e32 v1, s1, v0
+; GFX8-NEXT:    v_max_i16_e32 v1, -1, v0
 ; GFX8-NEXT:    v_subrev_u16_e32 v1, 0x7fff, v1
-; GFX8-NEXT:    v_min_i16_e32 v2, s1, v0
+; GFX8-NEXT:    v_min_i16_e32 v2, -1, v0
 ; GFX8-NEXT:    v_subrev_u16_e32 v2, 0x8000, v2
 ; GFX8-NEXT:    v_max_i16_e32 v1, s0, v1
 ; GFX8-NEXT:    v_min_i16_e32 v1, v1, v2
@@ -4452,10 +4433,9 @@ define amdgpu_ps half @ssubsat_i16_vs(i16 %lhs, i16 inreg %rhs) {
 ;
 ; GFX9-LABEL: ssubsat_i16_vs:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_mov_b32 s1, 0xffff
-; GFX9-NEXT:    v_max_i16_e32 v1, s1, v0
+; GFX9-NEXT:    v_max_i16_e32 v1, -1, v0
 ; GFX9-NEXT:    v_subrev_u16_e32 v1, 0x7fff, v1
-; GFX9-NEXT:    v_min_i16_e32 v2, s1, v0
+; GFX9-NEXT:    v_min_i16_e32 v2, -1, v0
 ; GFX9-NEXT:    v_subrev_u16_e32 v2, 0x8000, v2
 ; GFX9-NEXT:    v_max_i16_e32 v1, s0, v1
 ; GFX9-NEXT:    v_min_i16_e32 v1, v1, v2
@@ -4464,10 +4444,9 @@ define amdgpu_ps half @ssubsat_i16_vs(i16 %lhs, i16 inreg %rhs) {
 ;
 ; GFX10-LABEL: ssubsat_i16_vs:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_mov_b32 s1, 0xffff
+; GFX10-NEXT:    v_max_i16_e64 v1, v0, -1
+; GFX10-NEXT:    v_min_i16_e64 v2, v0, -1
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_max_i16_e64 v1, v0, s1
-; GFX10-NEXT:    v_min_i16_e64 v2, v0, s1
 ; GFX10-NEXT:    v_sub_nc_u16_e64 v1, v1, 0x7fff
 ; GFX10-NEXT:    v_sub_nc_u16_e64 v2, v2, 0x8000
 ; GFX10-NEXT:    v_max_i16_e64 v1, v1, s0
@@ -4488,7 +4467,7 @@ define <2 x i16> @v_ssubsat_v2i16(<2 x i16> %lhs, <2 x i16> %rhs) {
 ; GFX6-NEXT:    v_max_i32_e32 v4, -1, v0
 ; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX6-NEXT:    v_subrev_i32_e32 v4, vcc, s4, v4
-; GFX6-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX6-NEXT:    s_brev_b32 s5, 1
 ; GFX6-NEXT:    v_min_i32_e32 v5, -1, v0
 ; GFX6-NEXT:    v_subrev_i32_e32 v5, vcc, s5, v5
 ; GFX6-NEXT:    v_max_i32_e32 v2, v4, v2
@@ -4510,18 +4489,17 @@ define <2 x i16> @v_ssubsat_v2i16(<2 x i16> %lhs, <2 x i16> %rhs) {
 ; GFX8-LABEL: v_ssubsat_v2i16:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    s_mov_b32 s6, 0xffff
 ; GFX8-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX8-NEXT:    v_max_i16_e32 v3, s6, v0
+; GFX8-NEXT:    v_max_i16_e32 v3, -1, v0
 ; GFX8-NEXT:    v_subrev_u16_e32 v3, s4, v3
-; GFX8-NEXT:    s_mov_b32 s5, 0x8000
-; GFX8-NEXT:    v_min_i16_e32 v4, s6, v0
+; GFX8-NEXT:    s_movk_i32 s5, 0x8000
+; GFX8-NEXT:    v_min_i16_e32 v4, -1, v0
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
 ; GFX8-NEXT:    v_subrev_u16_e32 v4, s5, v4
 ; GFX8-NEXT:    v_max_i16_e32 v3, v3, v1
 ; GFX8-NEXT:    v_min_i16_e32 v3, v3, v4
-; GFX8-NEXT:    v_max_i16_e32 v4, s6, v2
-; GFX8-NEXT:    v_min_i16_e32 v5, s6, v2
+; GFX8-NEXT:    v_max_i16_e32 v4, -1, v2
+; GFX8-NEXT:    v_min_i16_e32 v5, -1, v2
 ; GFX8-NEXT:    v_subrev_u16_e32 v4, s4, v4
 ; GFX8-NEXT:    v_subrev_u16_e32 v5, s5, v5
 ; GFX8-NEXT:    v_max_i16_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
@@ -4536,7 +4514,7 @@ define <2 x i16> @v_ssubsat_v2i16(<2 x i16> %lhs, <2 x i16> %rhs) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    s_movk_i32 s4, 0x7fff
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s6, -1, -1
-; GFX9-NEXT:    s_mov_b32 s5, 0xffff8000
+; GFX9-NEXT:    s_movk_i32 s5, 0x8000
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s4
 ; GFX9-NEXT:    v_pk_max_i16 v2, v0, s6
 ; GFX9-NEXT:    v_pk_sub_i16 v2, v2, s4
@@ -4557,7 +4535,7 @@ define <2 x i16> @v_ssubsat_v2i16(<2 x i16> %lhs, <2 x i16> %rhs) {
 ; GFX10-NEXT:    v_pk_max_i16 v2, v0, s4
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s5, s5, s5
 ; GFX10-NEXT:    v_pk_min_i16 v3, v0, s4
-; GFX10-NEXT:    s_mov_b32 s6, 0xffff8000
+; GFX10-NEXT:    s_movk_i32 s6, 0x8000
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_pk_sub_i16 v2, v2, s5
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s6, s6
@@ -4580,7 +4558,7 @@ define amdgpu_ps i32 @s_ssubsat_v2i16(<2 x i16> inreg %lhs, <2 x i16> inreg %rhs
 ; GFX6-NEXT:    s_cselect_b32 s6, s0, -1
 ; GFX6-NEXT:    s_sub_i32 s6, s6, s4
 ; GFX6-NEXT:    s_cmp_lt_i32 s0, -1
-; GFX6-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX6-NEXT:    s_brev_b32 s5, 1
 ; GFX6-NEXT:    s_cselect_b32 s7, s0, -1
 ; GFX6-NEXT:    s_sub_i32 s7, s7, s5
 ; GFX6-NEXT:    s_cmp_gt_i32 s6, s2
@@ -4615,13 +4593,13 @@ define amdgpu_ps i32 @s_ssubsat_v2i16(<2 x i16> inreg %lhs, <2 x i16> inreg %rhs
 ; GFX8-NEXT:    s_lshr_b32 s3, s1, 16
 ; GFX8-NEXT:    s_lshr_b32 s2, s0, 16
 ; GFX8-NEXT:    s_sext_i32_i16 s6, s0
-; GFX8-NEXT:    s_sext_i32_i16 s7, 0xffff
+; GFX8-NEXT:    s_sext_i32_i16 s7, -1
 ; GFX8-NEXT:    s_cmp_gt_i32 s6, s7
 ; GFX8-NEXT:    s_movk_i32 s4, 0x7fff
 ; GFX8-NEXT:    s_cselect_b32 s8, s6, s7
 ; GFX8-NEXT:    s_sub_i32 s8, s8, s4
 ; GFX8-NEXT:    s_cmp_lt_i32 s6, s7
-; GFX8-NEXT:    s_mov_b32 s5, 0x8000
+; GFX8-NEXT:    s_movk_i32 s5, 0x8000
 ; GFX8-NEXT:    s_cselect_b32 s6, s6, s7
 ; GFX8-NEXT:    s_sub_i32 s6, s6, s5
 ; GFX8-NEXT:    s_sext_i32_i16 s8, s8
@@ -4676,7 +4654,7 @@ define amdgpu_ps i32 @s_ssubsat_v2i16(<2 x i16> inreg %lhs, <2 x i16> inreg %rhs
 ; GFX9-NEXT:    s_cmp_lt_i32 s5, s7
 ; GFX9-NEXT:    s_cselect_b32 s5, s5, s7
 ; GFX9-NEXT:    s_cmp_lt_i32 s6, s4
-; GFX9-NEXT:    s_mov_b32 s3, 0xffff8000
+; GFX9-NEXT:    s_movk_i32 s3, 0x8000
 ; GFX9-NEXT:    s_cselect_b32 s4, s6, s4
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s5, s4
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s3, s3
@@ -4733,7 +4711,7 @@ define amdgpu_ps i32 @s_ssubsat_v2i16(<2 x i16> inreg %lhs, <2 x i16> inreg %rhs
 ; GFX10-NEXT:    s_cmp_lt_i32 s3, s5
 ; GFX10-NEXT:    s_cselect_b32 s3, s3, s5
 ; GFX10-NEXT:    s_cmp_lt_i32 s4, s2
-; GFX10-NEXT:    s_mov_b32 s5, 0xffff8000
+; GFX10-NEXT:    s_movk_i32 s5, 0x8000
 ; GFX10-NEXT:    s_cselect_b32 s2, s4, s2
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s5, s5
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s3, s2
@@ -4782,7 +4760,7 @@ define amdgpu_ps float @ssubsat_v2i16_sv(<2 x i16> inreg %lhs, <2 x i16> %rhs) {
 ; GFX6-NEXT:    s_sub_i32 s4, s4, s2
 ; GFX6-NEXT:    s_cmp_lt_i32 s0, -1
 ; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; GFX6-NEXT:    s_mov_b32 s3, 0x80000000
+; GFX6-NEXT:    s_brev_b32 s3, 1
 ; GFX6-NEXT:    s_cselect_b32 s5, s0, -1
 ; GFX6-NEXT:    s_sub_i32 s5, s5, s3
 ; GFX6-NEXT:    v_max_i32_e32 v0, s4, v0
@@ -4812,13 +4790,13 @@ define amdgpu_ps float @ssubsat_v2i16_sv(<2 x i16> inreg %lhs, <2 x i16> %rhs) {
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_lshr_b32 s1, s0, 16
 ; GFX8-NEXT:    s_sext_i32_i16 s4, s0
-; GFX8-NEXT:    s_sext_i32_i16 s5, 0xffff
+; GFX8-NEXT:    s_sext_i32_i16 s5, -1
 ; GFX8-NEXT:    s_cmp_gt_i32 s4, s5
 ; GFX8-NEXT:    s_movk_i32 s2, 0x7fff
 ; GFX8-NEXT:    s_cselect_b32 s6, s4, s5
 ; GFX8-NEXT:    s_sub_i32 s6, s6, s2
 ; GFX8-NEXT:    s_cmp_lt_i32 s4, s5
-; GFX8-NEXT:    s_mov_b32 s3, 0x8000
+; GFX8-NEXT:    s_movk_i32 s3, 0x8000
 ; GFX8-NEXT:    s_cselect_b32 s4, s4, s5
 ; GFX8-NEXT:    s_sub_i32 s4, s4, s3
 ; GFX8-NEXT:    v_max_i16_e32 v1, s6, v0
@@ -4860,7 +4838,7 @@ define amdgpu_ps float @ssubsat_v2i16_sv(<2 x i16> inreg %lhs, <2 x i16> %rhs) {
 ; GFX9-NEXT:    s_cmp_lt_i32 s4, s6
 ; GFX9-NEXT:    s_cselect_b32 s4, s4, s6
 ; GFX9-NEXT:    s_cmp_lt_i32 s5, s3
-; GFX9-NEXT:    s_mov_b32 s2, 0xffff8000
+; GFX9-NEXT:    s_movk_i32 s2, 0x8000
 ; GFX9-NEXT:    s_cselect_b32 s3, s5, s3
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s4, s3
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s2
@@ -4897,7 +4875,7 @@ define amdgpu_ps float @ssubsat_v2i16_sv(<2 x i16> inreg %lhs, <2 x i16> %rhs) {
 ; GFX10-NEXT:    s_cmp_lt_i32 s2, s4
 ; GFX10-NEXT:    s_cselect_b32 s2, s2, s4
 ; GFX10-NEXT:    s_cmp_lt_i32 s3, s1
-; GFX10-NEXT:    s_mov_b32 s4, 0xffff8000
+; GFX10-NEXT:    s_movk_i32 s4, 0x8000
 ; GFX10-NEXT:    s_cselect_b32 s1, s3, s1
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s4, s4
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s2, s1
@@ -4924,7 +4902,7 @@ define amdgpu_ps float @ssubsat_v2i16_vs(<2 x i16> %lhs, <2 x i16> inreg %rhs) {
 ; GFX6-NEXT:    v_max_i32_e32 v2, -1, v0
 ; GFX6-NEXT:    s_lshl_b32 s0, s0, 16
 ; GFX6-NEXT:    v_subrev_i32_e32 v2, vcc, s2, v2
-; GFX6-NEXT:    s_mov_b32 s3, 0x80000000
+; GFX6-NEXT:    s_brev_b32 s3, 1
 ; GFX6-NEXT:    v_min_i32_e32 v3, -1, v0
 ; GFX6-NEXT:    v_max_i32_e32 v2, s0, v2
 ; GFX6-NEXT:    v_subrev_i32_e32 v3, vcc, s3, v3
@@ -4950,18 +4928,17 @@ define amdgpu_ps float @ssubsat_v2i16_vs(<2 x i16> %lhs, <2 x i16> inreg %rhs) {
 ;
 ; GFX8-LABEL: ssubsat_v2i16_vs:
 ; GFX8:       ; %bb.0:
-; GFX8-NEXT:    s_mov_b32 s4, 0xffff
 ; GFX8-NEXT:    s_movk_i32 s2, 0x7fff
-; GFX8-NEXT:    v_max_i16_e32 v2, s4, v0
+; GFX8-NEXT:    v_max_i16_e32 v2, -1, v0
 ; GFX8-NEXT:    v_subrev_u16_e32 v2, s2, v2
-; GFX8-NEXT:    s_mov_b32 s3, 0x8000
-; GFX8-NEXT:    v_min_i16_e32 v3, s4, v0
+; GFX8-NEXT:    s_movk_i32 s3, 0x8000
+; GFX8-NEXT:    v_min_i16_e32 v3, -1, v0
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
 ; GFX8-NEXT:    v_subrev_u16_e32 v3, s3, v3
 ; GFX8-NEXT:    v_max_i16_e32 v2, s0, v2
 ; GFX8-NEXT:    v_min_i16_e32 v2, v2, v3
-; GFX8-NEXT:    v_max_i16_e32 v3, s4, v1
-; GFX8-NEXT:    v_min_i16_e32 v4, s4, v1
+; GFX8-NEXT:    v_max_i16_e32 v3, -1, v1
+; GFX8-NEXT:    v_min_i16_e32 v4, -1, v1
 ; GFX8-NEXT:    s_lshr_b32 s1, s0, 16
 ; GFX8-NEXT:    v_subrev_u16_e32 v3, s2, v3
 ; GFX8-NEXT:    v_subrev_u16_e32 v4, s3, v4
@@ -4976,7 +4953,7 @@ define amdgpu_ps float @ssubsat_v2i16_vs(<2 x i16> %lhs, <2 x i16> inreg %rhs) {
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_movk_i32 s1, 0x7fff
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s3, -1, -1
-; GFX9-NEXT:    s_mov_b32 s2, 0xffff8000
+; GFX9-NEXT:    s_movk_i32 s2, 0x8000
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s1
 ; GFX9-NEXT:    v_pk_max_i16 v1, v0, s3
 ; GFX9-NEXT:    v_pk_sub_i16 v1, v1, s1
@@ -4995,7 +4972,7 @@ define amdgpu_ps float @ssubsat_v2i16_vs(<2 x i16> %lhs, <2 x i16> inreg %rhs) {
 ; GFX10-NEXT:    v_pk_max_i16 v1, v0, s1
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s2, s2
 ; GFX10-NEXT:    v_pk_min_i16 v2, v0, s1
-; GFX10-NEXT:    s_mov_b32 s3, 0xffff8000
+; GFX10-NEXT:    s_movk_i32 s3, 0x8000
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_pk_sub_i16 v1, v1, s2
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s3, s3
@@ -5029,7 +5006,7 @@ define <2 x float> @v_ssubsat_v4i16(<4 x i16> %lhs, <4 x i16> %rhs) {
 ; GFX6-NEXT:    v_max_i32_e32 v8, -1, v0
 ; GFX6-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
 ; GFX6-NEXT:    v_subrev_i32_e32 v8, vcc, s4, v8
-; GFX6-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX6-NEXT:    s_brev_b32 s5, 1
 ; GFX6-NEXT:    v_min_i32_e32 v10, -1, v0
 ; GFX6-NEXT:    v_subrev_i32_e32 v10, vcc, s5, v10
 ; GFX6-NEXT:    v_max_i32_e32 v4, v8, v4
@@ -5056,7 +5033,7 @@ define <2 x float> @v_ssubsat_v4i16(<4 x i16> %lhs, <4 x i16> %rhs) {
 ; GFX6-NEXT:    v_min_i32_e32 v4, v4, v6
 ; GFX6-NEXT:    v_max_i32_e32 v5, -1, v3
 ; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, v2, v4
-; GFX6-NEXT:    v_mov_b32_e32 v11, 0x80000000
+; GFX6-NEXT:    v_bfrev_b32_e32 v11, 1
 ; GFX6-NEXT:    v_min_i32_e32 v6, -1, v3
 ; GFX6-NEXT:    v_lshlrev_b32_e32 v4, 16, v7
 ; GFX6-NEXT:    v_sub_i32_e32 v5, vcc, v5, v9
@@ -5082,30 +5059,29 @@ define <2 x float> @v_ssubsat_v4i16(<4 x i16> %lhs, <4 x i16> %rhs) {
 ; GFX8-LABEL: v_ssubsat_v4i16:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    s_mov_b32 s6, 0xffff
 ; GFX8-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX8-NEXT:    v_max_i16_e32 v6, s6, v0
+; GFX8-NEXT:    v_max_i16_e32 v6, -1, v0
 ; GFX8-NEXT:    v_subrev_u16_e32 v6, s4, v6
-; GFX8-NEXT:    s_mov_b32 s5, 0x8000
-; GFX8-NEXT:    v_min_i16_e32 v7, s6, v0
+; GFX8-NEXT:    s_movk_i32 s5, 0x8000
+; GFX8-NEXT:    v_min_i16_e32 v7, -1, v0
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v4, 16, v0
 ; GFX8-NEXT:    v_subrev_u16_e32 v7, s5, v7
 ; GFX8-NEXT:    v_max_i16_e32 v6, v6, v2
 ; GFX8-NEXT:    v_min_i16_e32 v6, v6, v7
-; GFX8-NEXT:    v_max_i16_e32 v7, s6, v4
-; GFX8-NEXT:    v_min_i16_e32 v8, s6, v4
+; GFX8-NEXT:    v_max_i16_e32 v7, -1, v4
+; GFX8-NEXT:    v_min_i16_e32 v8, -1, v4
 ; GFX8-NEXT:    v_subrev_u16_e32 v7, s4, v7
 ; GFX8-NEXT:    v_max_i16_sdwa v2, v7, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT:    v_max_i16_e32 v7, s6, v1
+; GFX8-NEXT:    v_max_i16_e32 v7, -1, v1
 ; GFX8-NEXT:    v_subrev_u16_e32 v8, s5, v8
 ; GFX8-NEXT:    v_min_i16_e32 v2, v2, v8
 ; GFX8-NEXT:    v_subrev_u16_e32 v7, s4, v7
-; GFX8-NEXT:    v_min_i16_e32 v8, s6, v1
+; GFX8-NEXT:    v_min_i16_e32 v8, -1, v1
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
 ; GFX8-NEXT:    v_subrev_u16_e32 v8, s5, v8
 ; GFX8-NEXT:    v_max_i16_e32 v7, v7, v3
 ; GFX8-NEXT:    v_min_i16_e32 v7, v7, v8
-; GFX8-NEXT:    v_max_i16_e32 v8, s6, v5
+; GFX8-NEXT:    v_max_i16_e32 v8, -1, v5
 ; GFX8-NEXT:    v_min_i16_e32 v9, -1, v5
 ; GFX8-NEXT:    v_subrev_u16_e32 v8, s4, v8
 ; GFX8-NEXT:    v_subrev_u16_e32 v9, s5, v9
@@ -5124,7 +5100,7 @@ define <2 x float> @v_ssubsat_v4i16(<4 x i16> %lhs, <4 x i16> %rhs) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    s_movk_i32 s4, 0x7fff
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s6, -1, -1
-; GFX9-NEXT:    s_mov_b32 s5, 0xffff8000
+; GFX9-NEXT:    s_movk_i32 s5, 0x8000
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s4
 ; GFX9-NEXT:    v_pk_max_i16 v4, v0, s6
 ; GFX9-NEXT:    v_pk_sub_i16 v4, v4, s4
@@ -5156,7 +5132,7 @@ define <2 x float> @v_ssubsat_v4i16(<4 x i16> %lhs, <4 x i16> %rhs) {
 ; GFX10-NEXT:    v_pk_min_i16 v7, v1, s5
 ; GFX10-NEXT:    v_pk_sub_i16 v4, v4, s4
 ; GFX10-NEXT:    v_pk_sub_i16 v5, v5, s4
-; GFX10-NEXT:    s_mov_b32 s6, 0xffff8000
+; GFX10-NEXT:    s_movk_i32 s6, 0x8000
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s6, s6
 ; GFX10-NEXT:    v_pk_max_i16 v11, v4, v2
@@ -5183,7 +5159,7 @@ define amdgpu_ps <2 x i32> @s_ssubsat_v4i16(<4 x i16> inreg %lhs, <4 x i16> inre
 ; GFX6-NEXT:    s_cselect_b32 s10, s0, -1
 ; GFX6-NEXT:    s_sub_i32 s10, s10, s8
 ; GFX6-NEXT:    s_cmp_lt_i32 s0, -1
-; GFX6-NEXT:    s_mov_b32 s9, 0x80000000
+; GFX6-NEXT:    s_brev_b32 s9, 1
 ; GFX6-NEXT:    s_cselect_b32 s11, s0, -1
 ; GFX6-NEXT:    s_sub_i32 s11, s11, s9
 ; GFX6-NEXT:    s_cmp_gt_i32 s10, s4
@@ -5252,13 +5228,13 @@ define amdgpu_ps <2 x i32> @s_ssubsat_v4i16(<4 x i16> inreg %lhs, <4 x i16> inre
 ; GFX8-NEXT:    s_lshr_b32 s4, s0, 16
 ; GFX8-NEXT:    s_lshr_b32 s5, s1, 16
 ; GFX8-NEXT:    s_sext_i32_i16 s10, s0
-; GFX8-NEXT:    s_sext_i32_i16 s11, 0xffff
+; GFX8-NEXT:    s_sext_i32_i16 s11, -1
 ; GFX8-NEXT:    s_cmp_gt_i32 s10, s11
 ; GFX8-NEXT:    s_movk_i32 s8, 0x7fff
 ; GFX8-NEXT:    s_cselect_b32 s12, s10, s11
 ; GFX8-NEXT:    s_sub_i32 s12, s12, s8
 ; GFX8-NEXT:    s_cmp_lt_i32 s10, s11
-; GFX8-NEXT:    s_mov_b32 s9, 0x8000
+; GFX8-NEXT:    s_movk_i32 s9, 0x8000
 ; GFX8-NEXT:    s_cselect_b32 s10, s10, s11
 ; GFX8-NEXT:    s_sub_i32 s10, s10, s9
 ; GFX8-NEXT:    s_sext_i32_i16 s12, s12
@@ -5349,7 +5325,7 @@ define amdgpu_ps <2 x i32> @s_ssubsat_v4i16(<4 x i16> inreg %lhs, <4 x i16> inre
 ; GFX9-NEXT:    s_cmp_lt_i32 s7, s9
 ; GFX9-NEXT:    s_cselect_b32 s7, s7, s9
 ; GFX9-NEXT:    s_cmp_lt_i32 s8, s6
-; GFX9-NEXT:    s_mov_b32 s5, 0xffff8000
+; GFX9-NEXT:    s_movk_i32 s5, 0x8000
 ; GFX9-NEXT:    s_cselect_b32 s8, s8, s6
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s7, s7, s8
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s5, s5, s5
@@ -5438,7 +5414,7 @@ define amdgpu_ps <2 x i32> @s_ssubsat_v4i16(<4 x i16> inreg %lhs, <4 x i16> inre
 ; GFX10-NEXT:    s_movk_i32 s10, 0x7fff
 ; GFX10-NEXT:    s_cselect_b32 s8, s5, s7
 ; GFX10-NEXT:    s_cmp_gt_i32 s6, s4
-; GFX10-NEXT:    s_mov_b32 s12, 0xffff8000
+; GFX10-NEXT:    s_movk_i32 s12, 0x8000
 ; GFX10-NEXT:    s_cselect_b32 s9, s6, s4
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s12, s12, s12
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s8, s8, s9
@@ -5551,7 +5527,7 @@ define <3 x float> @v_ssubsat_v6i16(<6 x i16> %lhs, <6 x i16> %rhs) {
 ; GFX6-NEXT:    v_max_i32_e32 v12, -1, v0
 ; GFX6-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
 ; GFX6-NEXT:    v_subrev_i32_e32 v12, vcc, s4, v12
-; GFX6-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX6-NEXT:    s_brev_b32 s5, 1
 ; GFX6-NEXT:    v_min_i32_e32 v14, -1, v0
 ; GFX6-NEXT:    v_subrev_i32_e32 v14, vcc, s5, v14
 ; GFX6-NEXT:    v_max_i32_e32 v6, v12, v6
@@ -5578,7 +5554,7 @@ define <3 x float> @v_ssubsat_v6i16(<6 x i16> %lhs, <6 x i16> %rhs) {
 ; GFX6-NEXT:    v_min_i32_e32 v6, v6, v8
 ; GFX6-NEXT:    v_max_i32_e32 v7, -1, v3
 ; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, v2, v6
-; GFX6-NEXT:    v_mov_b32_e32 v15, 0x80000000
+; GFX6-NEXT:    v_bfrev_b32_e32 v15, 1
 ; GFX6-NEXT:    v_min_i32_e32 v8, -1, v3
 ; GFX6-NEXT:    v_lshlrev_b32_e32 v6, 16, v9
 ; GFX6-NEXT:    v_sub_i32_e32 v7, vcc, v7, v13
@@ -5628,58 +5604,56 @@ define <3 x float> @v_ssubsat_v6i16(<6 x i16> %lhs, <6 x i16> %rhs) {
 ; GFX8-LABEL: v_ssubsat_v6i16:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    s_mov_b32 s6, 0xffff
 ; GFX8-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX8-NEXT:    v_max_i16_e32 v10, s6, v0
-; GFX8-NEXT:    v_subrev_u16_e32 v10, s4, v10
-; GFX8-NEXT:    s_mov_b32 s5, 0x8000
-; GFX8-NEXT:    v_min_i16_e32 v12, s6, v0
+; GFX8-NEXT:    v_max_i16_e32 v9, -1, v0
+; GFX8-NEXT:    v_subrev_u16_e32 v9, s4, v9
+; GFX8-NEXT:    s_movk_i32 s5, 0x8000
+; GFX8-NEXT:    v_min_i16_e32 v11, -1, v0
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v6, 16, v0
-; GFX8-NEXT:    v_subrev_u16_e32 v12, s5, v12
-; GFX8-NEXT:    v_max_i16_e32 v10, v10, v3
-; GFX8-NEXT:    v_min_i16_e32 v10, v10, v12
-; GFX8-NEXT:    v_max_i16_e32 v12, s6, v6
-; GFX8-NEXT:    v_min_i16_e32 v14, s6, v6
-; GFX8-NEXT:    v_subrev_u16_e32 v12, s4, v12
-; GFX8-NEXT:    v_max_i16_sdwa v3, v12, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT:    v_max_i16_e32 v12, s6, v1
-; GFX8-NEXT:    v_subrev_u16_e32 v14, s5, v14
-; GFX8-NEXT:    v_min_i16_e32 v3, v3, v14
-; GFX8-NEXT:    v_subrev_u16_e32 v12, s4, v12
-; GFX8-NEXT:    v_min_i16_e32 v14, s6, v1
+; GFX8-NEXT:    v_subrev_u16_e32 v11, s5, v11
+; GFX8-NEXT:    v_max_i16_e32 v9, v9, v3
+; GFX8-NEXT:    v_min_i16_e32 v9, v9, v11
+; GFX8-NEXT:    v_max_i16_e32 v11, -1, v6
+; GFX8-NEXT:    v_min_i16_e32 v13, -1, v6
+; GFX8-NEXT:    v_subrev_u16_e32 v11, s4, v11
+; GFX8-NEXT:    v_max_i16_sdwa v3, v11, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_max_i16_e32 v11, -1, v1
+; GFX8-NEXT:    v_subrev_u16_e32 v13, s5, v13
+; GFX8-NEXT:    v_min_i16_e32 v3, v3, v13
+; GFX8-NEXT:    v_subrev_u16_e32 v11, s4, v11
+; GFX8-NEXT:    v_min_i16_e32 v13, -1, v1
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v7, 16, v1
+; GFX8-NEXT:    v_subrev_u16_e32 v13, s5, v13
+; GFX8-NEXT:    v_max_i16_e32 v11, v11, v4
+; GFX8-NEXT:    v_min_i16_e32 v11, v11, v13
+; GFX8-NEXT:    v_max_i16_e32 v13, -1, v7
+; GFX8-NEXT:    v_min_i16_e32 v14, -1, v7
+; GFX8-NEXT:    v_subrev_u16_e32 v13, s4, v13
+; GFX8-NEXT:    v_max_i16_sdwa v4, v13, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
 ; GFX8-NEXT:    v_subrev_u16_e32 v14, s5, v14
-; GFX8-NEXT:    v_max_i16_e32 v12, v12, v4
-; GFX8-NEXT:    v_min_i16_e32 v12, v12, v14
-; GFX8-NEXT:    v_max_i16_e32 v14, s6, v7
-; GFX8-NEXT:    v_mov_b32_e32 v9, 0xffff
-; GFX8-NEXT:    v_min_i16_e32 v15, v7, v9
-; GFX8-NEXT:    v_subrev_u16_e32 v14, s4, v14
-; GFX8-NEXT:    v_max_i16_sdwa v4, v14, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT:    v_subrev_u16_e32 v15, s5, v15
-; GFX8-NEXT:    v_mov_b32_e32 v11, 0x7fff
-; GFX8-NEXT:    v_max_i16_e32 v14, v2, v9
-; GFX8-NEXT:    v_sub_u16_e32 v14, v14, v11
-; GFX8-NEXT:    v_min_i16_e32 v4, v4, v15
-; GFX8-NEXT:    v_mov_b32_e32 v13, 0x8000
-; GFX8-NEXT:    v_min_i16_e32 v15, v2, v9
+; GFX8-NEXT:    v_mov_b32_e32 v10, 0x7fff
+; GFX8-NEXT:    v_max_i16_e32 v13, -1, v2
+; GFX8-NEXT:    v_sub_u16_e32 v13, v13, v10
+; GFX8-NEXT:    v_min_i16_e32 v4, v4, v14
+; GFX8-NEXT:    v_mov_b32_e32 v12, 0xffff8000
+; GFX8-NEXT:    v_min_i16_e32 v14, -1, v2
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v8, 16, v2
-; GFX8-NEXT:    v_sub_u16_e32 v15, v15, v13
-; GFX8-NEXT:    v_max_i16_e32 v14, v14, v5
-; GFX8-NEXT:    v_min_i16_e32 v14, v14, v15
-; GFX8-NEXT:    v_max_i16_e32 v15, v8, v9
-; GFX8-NEXT:    v_min_i16_e32 v9, v8, v9
-; GFX8-NEXT:    v_sub_u16_e32 v11, v15, v11
-; GFX8-NEXT:    v_sub_u16_e32 v9, v9, v13
-; GFX8-NEXT:    v_max_i16_sdwa v5, v11, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT:    v_sub_u16_e32 v0, v0, v10
+; GFX8-NEXT:    v_sub_u16_e32 v14, v14, v12
+; GFX8-NEXT:    v_max_i16_e32 v13, v13, v5
+; GFX8-NEXT:    v_min_i16_e32 v13, v13, v14
+; GFX8-NEXT:    v_max_i16_e32 v14, -1, v8
+; GFX8-NEXT:    v_sub_u16_e32 v10, v14, v10
+; GFX8-NEXT:    v_min_i16_e32 v14, -1, v8
+; GFX8-NEXT:    v_sub_u16_e32 v12, v14, v12
+; GFX8-NEXT:    v_max_i16_sdwa v5, v10, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_sub_u16_e32 v0, v0, v9
 ; GFX8-NEXT:    v_sub_u16_sdwa v3, v6, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX8-NEXT:    v_or_b32_e32 v0, v0, v3
-; GFX8-NEXT:    v_min_i16_e32 v5, v5, v9
-; GFX8-NEXT:    v_sub_u16_e32 v1, v1, v12
+; GFX8-NEXT:    v_min_i16_e32 v5, v5, v12
+; GFX8-NEXT:    v_sub_u16_e32 v1, v1, v11
 ; GFX8-NEXT:    v_sub_u16_sdwa v3, v7, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX8-NEXT:    v_or_b32_e32 v1, v1, v3
-; GFX8-NEXT:    v_sub_u16_e32 v2, v2, v14
+; GFX8-NEXT:    v_sub_u16_e32 v2, v2, v13
 ; GFX8-NEXT:    v_sub_u16_sdwa v3, v8, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX8-NEXT:    v_or_b32_e32 v2, v2, v3
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
@@ -5689,7 +5663,7 @@ define <3 x float> @v_ssubsat_v6i16(<6 x i16> %lhs, <6 x i16> %rhs) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    s_movk_i32 s4, 0x7fff
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s6, -1, -1
-; GFX9-NEXT:    s_mov_b32 s5, 0xffff8000
+; GFX9-NEXT:    s_movk_i32 s5, 0x8000
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s4
 ; GFX9-NEXT:    v_pk_max_i16 v6, v0, s6
 ; GFX9-NEXT:    v_pk_sub_i16 v6, v6, s4
@@ -5731,7 +5705,7 @@ define <3 x float> @v_ssubsat_v6i16(<6 x i16> %lhs, <6 x i16> %rhs) {
 ; GFX10-NEXT:    v_pk_sub_i16 v19, v9, s4
 ; GFX10-NEXT:    v_pk_min_i16 v10, v1, s5
 ; GFX10-NEXT:    v_pk_min_i16 v11, v2, s5
-; GFX10-NEXT:    s_mov_b32 s6, 0xffff8000
+; GFX10-NEXT:    s_movk_i32 s6, 0x8000
 ; GFX10-NEXT:    v_pk_max_i16 v14, v6, v3
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s6, s6
 ; GFX10-NEXT:    v_pk_max_i16 v4, v15, v4
@@ -5762,7 +5736,7 @@ define amdgpu_ps <3 x i32> @s_ssubsat_v6i16(<6 x i16> inreg %lhs, <6 x i16> inre
 ; GFX6-NEXT:    s_cselect_b32 s14, s0, -1
 ; GFX6-NEXT:    s_sub_i32 s14, s14, s12
 ; GFX6-NEXT:    s_cmp_lt_i32 s0, -1
-; GFX6-NEXT:    s_mov_b32 s13, 0x80000000
+; GFX6-NEXT:    s_brev_b32 s13, 1
 ; GFX6-NEXT:    s_cselect_b32 s15, s0, -1
 ; GFX6-NEXT:    s_sub_i32 s15, s15, s13
 ; GFX6-NEXT:    s_cmp_gt_i32 s14, s6
@@ -5865,13 +5839,13 @@ define amdgpu_ps <3 x i32> @s_ssubsat_v6i16(<6 x i16> inreg %lhs, <6 x i16> inre
 ; GFX8-NEXT:    s_lshr_b32 s7, s1, 16
 ; GFX8-NEXT:    s_lshr_b32 s8, s2, 16
 ; GFX8-NEXT:    s_sext_i32_i16 s14, s0
-; GFX8-NEXT:    s_sext_i32_i16 s15, 0xffff
+; GFX8-NEXT:    s_sext_i32_i16 s15, -1
 ; GFX8-NEXT:    s_cmp_gt_i32 s14, s15
 ; GFX8-NEXT:    s_movk_i32 s12, 0x7fff
 ; GFX8-NEXT:    s_cselect_b32 s16, s14, s15
 ; GFX8-NEXT:    s_sub_i32 s16, s16, s12
 ; GFX8-NEXT:    s_cmp_lt_i32 s14, s15
-; GFX8-NEXT:    s_mov_b32 s13, 0x8000
+; GFX8-NEXT:    s_movk_i32 s13, 0x8000
 ; GFX8-NEXT:    s_cselect_b32 s14, s14, s15
 ; GFX8-NEXT:    s_sub_i32 s14, s14, s13
 ; GFX8-NEXT:    s_sext_i32_i16 s16, s16
@@ -5998,7 +5972,7 @@ define amdgpu_ps <3 x i32> @s_ssubsat_v6i16(<6 x i16> inreg %lhs, <6 x i16> inre
 ; GFX9-NEXT:    s_cmp_lt_i32 s9, s11
 ; GFX9-NEXT:    s_cselect_b32 s9, s9, s11
 ; GFX9-NEXT:    s_cmp_lt_i32 s10, s8
-; GFX9-NEXT:    s_mov_b32 s7, 0xffff8000
+; GFX9-NEXT:    s_movk_i32 s7, 0x8000
 ; GFX9-NEXT:    s_cselect_b32 s10, s10, s8
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s9, s9, s10
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s7, s7, s7
@@ -6130,7 +6104,7 @@ define amdgpu_ps <3 x i32> @s_ssubsat_v6i16(<6 x i16> inreg %lhs, <6 x i16> inre
 ; GFX10-NEXT:    s_movk_i32 s12, 0x7fff
 ; GFX10-NEXT:    s_cselect_b32 s10, s7, s9
 ; GFX10-NEXT:    s_cmp_gt_i32 s8, s6
-; GFX10-NEXT:    s_mov_b32 s14, 0xffff8000
+; GFX10-NEXT:    s_movk_i32 s14, 0x8000
 ; GFX10-NEXT:    s_cselect_b32 s11, s8, s6
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s14, s14, s14
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s10, s10, s11
@@ -6275,7 +6249,7 @@ define <4 x float> @v_ssubsat_v8i16(<8 x i16> %lhs, <8 x i16> %rhs) {
 ; GFX6-NEXT:    v_max_i32_e32 v16, -1, v0
 ; GFX6-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
 ; GFX6-NEXT:    v_subrev_i32_e32 v16, vcc, s4, v16
-; GFX6-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX6-NEXT:    s_brev_b32 s5, 1
 ; GFX6-NEXT:    v_min_i32_e32 v18, -1, v0
 ; GFX6-NEXT:    v_subrev_i32_e32 v18, vcc, s5, v18
 ; GFX6-NEXT:    v_max_i32_e32 v8, v16, v8
@@ -6302,7 +6276,7 @@ define <4 x float> @v_ssubsat_v8i16(<8 x i16> %lhs, <8 x i16> %rhs) {
 ; GFX6-NEXT:    v_min_i32_e32 v8, v8, v10
 ; GFX6-NEXT:    v_max_i32_e32 v9, -1, v3
 ; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, v2, v8
-; GFX6-NEXT:    v_mov_b32_e32 v19, 0x80000000
+; GFX6-NEXT:    v_bfrev_b32_e32 v19, 1
 ; GFX6-NEXT:    v_min_i32_e32 v10, -1, v3
 ; GFX6-NEXT:    v_lshlrev_b32_e32 v8, 16, v11
 ; GFX6-NEXT:    v_sub_i32_e32 v9, vcc, v9, v17
@@ -6376,74 +6350,72 @@ define <4 x float> @v_ssubsat_v8i16(<8 x i16> %lhs, <8 x i16> %rhs) {
 ; GFX8-LABEL: v_ssubsat_v8i16:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    s_mov_b32 s6, 0xffff
 ; GFX8-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX8-NEXT:    v_max_i16_e32 v13, s6, v0
-; GFX8-NEXT:    v_subrev_u16_e32 v13, s4, v13
-; GFX8-NEXT:    s_mov_b32 s5, 0x8000
-; GFX8-NEXT:    v_min_i16_e32 v15, s6, v0
+; GFX8-NEXT:    v_max_i16_e32 v12, -1, v0
+; GFX8-NEXT:    v_subrev_u16_e32 v12, s4, v12
+; GFX8-NEXT:    s_movk_i32 s5, 0x8000
+; GFX8-NEXT:    v_min_i16_e32 v14, -1, v0
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v8, 16, v0
-; GFX8-NEXT:    v_subrev_u16_e32 v15, s5, v15
-; GFX8-NEXT:    v_max_i16_e32 v13, v13, v4
-; GFX8-NEXT:    v_min_i16_e32 v13, v13, v15
-; GFX8-NEXT:    v_max_i16_e32 v15, s6, v8
-; GFX8-NEXT:    v_min_i16_e32 v17, s6, v8
-; GFX8-NEXT:    v_subrev_u16_e32 v15, s4, v15
-; GFX8-NEXT:    v_max_i16_sdwa v4, v15, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT:    v_max_i16_e32 v15, s6, v1
-; GFX8-NEXT:    v_subrev_u16_e32 v17, s5, v17
-; GFX8-NEXT:    v_min_i16_e32 v4, v4, v17
-; GFX8-NEXT:    v_subrev_u16_e32 v15, s4, v15
-; GFX8-NEXT:    v_min_i16_e32 v17, s6, v1
+; GFX8-NEXT:    v_subrev_u16_e32 v14, s5, v14
+; GFX8-NEXT:    v_max_i16_e32 v12, v12, v4
+; GFX8-NEXT:    v_min_i16_e32 v12, v12, v14
+; GFX8-NEXT:    v_max_i16_e32 v14, -1, v8
+; GFX8-NEXT:    v_min_i16_e32 v16, -1, v8
+; GFX8-NEXT:    v_subrev_u16_e32 v14, s4, v14
+; GFX8-NEXT:    v_max_i16_sdwa v4, v14, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_max_i16_e32 v14, -1, v1
+; GFX8-NEXT:    v_subrev_u16_e32 v16, s5, v16
+; GFX8-NEXT:    v_min_i16_e32 v4, v4, v16
+; GFX8-NEXT:    v_subrev_u16_e32 v14, s4, v14
+; GFX8-NEXT:    v_min_i16_e32 v16, -1, v1
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v9, 16, v1
+; GFX8-NEXT:    v_subrev_u16_e32 v16, s5, v16
+; GFX8-NEXT:    v_max_i16_e32 v14, v14, v5
+; GFX8-NEXT:    v_min_i16_e32 v14, v14, v16
+; GFX8-NEXT:    v_max_i16_e32 v16, -1, v9
+; GFX8-NEXT:    v_min_i16_e32 v17, -1, v9
+; GFX8-NEXT:    v_subrev_u16_e32 v16, s4, v16
+; GFX8-NEXT:    v_max_i16_sdwa v5, v16, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
 ; GFX8-NEXT:    v_subrev_u16_e32 v17, s5, v17
-; GFX8-NEXT:    v_max_i16_e32 v15, v15, v5
-; GFX8-NEXT:    v_min_i16_e32 v15, v15, v17
-; GFX8-NEXT:    v_max_i16_e32 v17, s6, v9
-; GFX8-NEXT:    v_mov_b32_e32 v12, 0xffff
-; GFX8-NEXT:    v_min_i16_e32 v18, v9, v12
-; GFX8-NEXT:    v_subrev_u16_e32 v17, s4, v17
-; GFX8-NEXT:    v_max_i16_sdwa v5, v17, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT:    v_subrev_u16_e32 v18, s5, v18
-; GFX8-NEXT:    v_mov_b32_e32 v14, 0x7fff
-; GFX8-NEXT:    v_max_i16_e32 v17, v2, v12
-; GFX8-NEXT:    v_sub_u16_e32 v17, v17, v14
-; GFX8-NEXT:    v_min_i16_e32 v5, v5, v18
-; GFX8-NEXT:    v_mov_b32_e32 v16, 0x8000
-; GFX8-NEXT:    v_min_i16_e32 v18, v2, v12
+; GFX8-NEXT:    v_mov_b32_e32 v13, 0x7fff
+; GFX8-NEXT:    v_max_i16_e32 v16, -1, v2
+; GFX8-NEXT:    v_sub_u16_e32 v16, v16, v13
+; GFX8-NEXT:    v_min_i16_e32 v5, v5, v17
+; GFX8-NEXT:    v_mov_b32_e32 v15, 0xffff8000
+; GFX8-NEXT:    v_min_i16_e32 v17, -1, v2
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v10, 16, v2
-; GFX8-NEXT:    v_sub_u16_e32 v18, v18, v16
-; GFX8-NEXT:    v_max_i16_e32 v17, v17, v6
-; GFX8-NEXT:    v_min_i16_e32 v17, v17, v18
-; GFX8-NEXT:    v_max_i16_e32 v18, v10, v12
-; GFX8-NEXT:    v_min_i16_e32 v19, v10, v12
-; GFX8-NEXT:    v_sub_u16_e32 v18, v18, v14
-; GFX8-NEXT:    v_max_i16_sdwa v6, v18, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT:    v_max_i16_e32 v18, v3, v12
-; GFX8-NEXT:    v_sub_u16_e32 v19, v19, v16
-; GFX8-NEXT:    v_sub_u16_e32 v18, v18, v14
-; GFX8-NEXT:    v_min_i16_e32 v6, v6, v19
-; GFX8-NEXT:    v_min_i16_e32 v19, v3, v12
+; GFX8-NEXT:    v_sub_u16_e32 v17, v17, v15
+; GFX8-NEXT:    v_max_i16_e32 v16, v16, v6
+; GFX8-NEXT:    v_min_i16_e32 v16, v16, v17
+; GFX8-NEXT:    v_max_i16_e32 v17, -1, v10
+; GFX8-NEXT:    v_min_i16_e32 v18, -1, v10
+; GFX8-NEXT:    v_sub_u16_e32 v17, v17, v13
+; GFX8-NEXT:    v_max_i16_sdwa v6, v17, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_max_i16_e32 v17, -1, v3
+; GFX8-NEXT:    v_sub_u16_e32 v18, v18, v15
+; GFX8-NEXT:    v_sub_u16_e32 v17, v17, v13
+; GFX8-NEXT:    v_min_i16_e32 v6, v6, v18
+; GFX8-NEXT:    v_min_i16_e32 v18, -1, v3
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v11, 16, v3
-; GFX8-NEXT:    v_sub_u16_e32 v19, v19, v16
-; GFX8-NEXT:    v_max_i16_e32 v18, v18, v7
-; GFX8-NEXT:    v_min_i16_e32 v18, v18, v19
-; GFX8-NEXT:    v_max_i16_e32 v19, v11, v12
-; GFX8-NEXT:    v_min_i16_e32 v12, v11, v12
-; GFX8-NEXT:    v_sub_u16_e32 v14, v19, v14
-; GFX8-NEXT:    v_sub_u16_e32 v0, v0, v13
+; GFX8-NEXT:    v_sub_u16_e32 v18, v18, v15
+; GFX8-NEXT:    v_max_i16_e32 v17, v17, v7
+; GFX8-NEXT:    v_min_i16_e32 v17, v17, v18
+; GFX8-NEXT:    v_max_i16_e32 v18, -1, v11
+; GFX8-NEXT:    v_sub_u16_e32 v13, v18, v13
+; GFX8-NEXT:    v_min_i16_e32 v18, -1, v11
+; GFX8-NEXT:    v_sub_u16_e32 v0, v0, v12
 ; GFX8-NEXT:    v_sub_u16_sdwa v4, v8, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX8-NEXT:    v_or_b32_e32 v0, v0, v4
-; GFX8-NEXT:    v_sub_u16_e32 v12, v12, v16
-; GFX8-NEXT:    v_max_i16_sdwa v7, v14, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT:    v_sub_u16_e32 v1, v1, v15
+; GFX8-NEXT:    v_sub_u16_e32 v15, v18, v15
+; GFX8-NEXT:    v_max_i16_sdwa v7, v13, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_sub_u16_e32 v1, v1, v14
 ; GFX8-NEXT:    v_sub_u16_sdwa v4, v9, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX8-NEXT:    v_or_b32_e32 v1, v1, v4
-; GFX8-NEXT:    v_min_i16_e32 v7, v7, v12
-; GFX8-NEXT:    v_sub_u16_e32 v2, v2, v17
+; GFX8-NEXT:    v_min_i16_e32 v7, v7, v15
+; GFX8-NEXT:    v_sub_u16_e32 v2, v2, v16
 ; GFX8-NEXT:    v_sub_u16_sdwa v4, v10, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX8-NEXT:    v_or_b32_e32 v2, v2, v4
-; GFX8-NEXT:    v_sub_u16_e32 v3, v3, v18
+; GFX8-NEXT:    v_sub_u16_e32 v3, v3, v17
 ; GFX8-NEXT:    v_sub_u16_sdwa v4, v11, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX8-NEXT:    v_or_b32_e32 v3, v3, v4
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
@@ -6453,7 +6425,7 @@ define <4 x float> @v_ssubsat_v8i16(<8 x i16> %lhs, <8 x i16> %rhs) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    s_movk_i32 s4, 0x7fff
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s6, -1, -1
-; GFX9-NEXT:    s_mov_b32 s5, 0xffff8000
+; GFX9-NEXT:    s_movk_i32 s5, 0x8000
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s4
 ; GFX9-NEXT:    v_pk_max_i16 v8, v0, s6
 ; GFX9-NEXT:    v_pk_sub_i16 v8, v8, s4
@@ -6505,7 +6477,7 @@ define <4 x float> @v_ssubsat_v8i16(<8 x i16> %lhs, <8 x i16> %rhs) {
 ; GFX10-NEXT:    v_pk_min_i16 v13, v2, s4
 ; GFX10-NEXT:    v_pk_sub_i16 v8, v8, s5
 ; GFX10-NEXT:    v_pk_min_i16 v14, v3, s4
-; GFX10-NEXT:    s_mov_b32 s6, 0xffff8000
+; GFX10-NEXT:    s_movk_i32 s6, 0x8000
 ; GFX10-NEXT:    v_pk_max_i16 v4, v15, v4
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s6, s6
 ; GFX10-NEXT:    v_pk_max_i16 v5, v10, v5
@@ -6540,7 +6512,7 @@ define amdgpu_ps <4 x i32> @s_ssubsat_v8i16(<8 x i16> inreg %lhs, <8 x i16> inre
 ; GFX6-NEXT:    s_cselect_b32 s18, s0, -1
 ; GFX6-NEXT:    s_sub_i32 s18, s18, s16
 ; GFX6-NEXT:    s_cmp_lt_i32 s0, -1
-; GFX6-NEXT:    s_mov_b32 s17, 0x80000000
+; GFX6-NEXT:    s_brev_b32 s17, 1
 ; GFX6-NEXT:    s_cselect_b32 s19, s0, -1
 ; GFX6-NEXT:    s_sub_i32 s19, s19, s17
 ; GFX6-NEXT:    s_cmp_gt_i32 s18, s8
@@ -6677,13 +6649,13 @@ define amdgpu_ps <4 x i32> @s_ssubsat_v8i16(<8 x i16> inreg %lhs, <8 x i16> inre
 ; GFX8-NEXT:    s_lshr_b32 s10, s2, 16
 ; GFX8-NEXT:    s_lshr_b32 s11, s3, 16
 ; GFX8-NEXT:    s_sext_i32_i16 s18, s0
-; GFX8-NEXT:    s_sext_i32_i16 s19, 0xffff
+; GFX8-NEXT:    s_sext_i32_i16 s19, -1
 ; GFX8-NEXT:    s_cmp_gt_i32 s18, s19
 ; GFX8-NEXT:    s_movk_i32 s16, 0x7fff
 ; GFX8-NEXT:    s_cselect_b32 s20, s18, s19
 ; GFX8-NEXT:    s_sub_i32 s20, s20, s16
 ; GFX8-NEXT:    s_cmp_lt_i32 s18, s19
-; GFX8-NEXT:    s_mov_b32 s17, 0x8000
+; GFX8-NEXT:    s_movk_i32 s17, 0x8000
 ; GFX8-NEXT:    s_cselect_b32 s18, s18, s19
 ; GFX8-NEXT:    s_sub_i32 s18, s18, s17
 ; GFX8-NEXT:    s_sext_i32_i16 s20, s20
@@ -6846,7 +6818,7 @@ define amdgpu_ps <4 x i32> @s_ssubsat_v8i16(<8 x i16> inreg %lhs, <8 x i16> inre
 ; GFX9-NEXT:    s_cmp_lt_i32 s11, s13
 ; GFX9-NEXT:    s_cselect_b32 s11, s11, s13
 ; GFX9-NEXT:    s_cmp_lt_i32 s12, s10
-; GFX9-NEXT:    s_mov_b32 s9, 0xffff8000
+; GFX9-NEXT:    s_movk_i32 s9, 0x8000
 ; GFX9-NEXT:    s_cselect_b32 s12, s12, s10
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s11, s11, s12
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s9, s9, s9
@@ -7021,7 +6993,7 @@ define amdgpu_ps <4 x i32> @s_ssubsat_v8i16(<8 x i16> inreg %lhs, <8 x i16> inre
 ; GFX10-NEXT:    s_movk_i32 s14, 0x7fff
 ; GFX10-NEXT:    s_cselect_b32 s12, s9, s11
 ; GFX10-NEXT:    s_cmp_gt_i32 s10, s8
-; GFX10-NEXT:    s_mov_b32 s16, 0xffff8000
+; GFX10-NEXT:    s_movk_i32 s16, 0x8000
 ; GFX10-NEXT:    s_cselect_b32 s13, s10, s8
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s16, s16, s16
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s12, s12, s13
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll
index 413bc71038470..5570309a5be7d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll
@@ -21,7 +21,7 @@ define i7 @v_uaddsat_i7(i7 %lhs, i7 %rhs) {
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    v_lshlrev_b16_e32 v0, 9, v0
 ; GFX8-NEXT:    v_lshlrev_b16_e32 v1, 9, v1
-; GFX8-NEXT:    v_xor_b32_e32 v2, 0xffff, v0
+; GFX8-NEXT:    v_xor_b32_e32 v2, -1, v0
 ; GFX8-NEXT:    v_min_u16_e32 v1, v2, v1
 ; GFX8-NEXT:    v_add_u16_e32 v0, v0, v1
 ; GFX8-NEXT:    v_lshrrev_b16_e32 v0, 9, v0
@@ -32,7 +32,7 @@ define i7 @v_uaddsat_i7(i7 %lhs, i7 %rhs) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 9, v0
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 9, v1
-; GFX9-NEXT:    v_xor_b32_e32 v2, 0xffff, v0
+; GFX9-NEXT:    v_xor_b32_e32 v2, -1, v0
 ; GFX9-NEXT:    v_min_u16_e32 v1, v2, v1
 ; GFX9-NEXT:    v_add_u16_e32 v0, v0, v1
 ; GFX9-NEXT:    v_lshrrev_b16_e32 v0, 9, v0
@@ -45,7 +45,7 @@ define i7 @v_uaddsat_i7(i7 %lhs, i7 %rhs) {
 ; GFX10-NEXT:    v_lshlrev_b16_e64 v0, 9, v0
 ; GFX10-NEXT:    v_lshlrev_b16_e64 v1, 9, v1
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_xor_b32_e32 v2, 0xffff, v0
+; GFX10-NEXT:    v_xor_b32_e32 v2, -1, v0
 ; GFX10-NEXT:    v_min_u16_e64 v1, v2, v1
 ; GFX10-NEXT:    v_add_nc_u16_e64 v0, v0, v1
 ; GFX10-NEXT:    v_lshrrev_b16_e64 v0, 9, v0
@@ -71,7 +71,7 @@ define amdgpu_ps i7 @s_uaddsat_i7(i7 inreg %lhs, i7 inreg %rhs) {
 ; GFX8-NEXT:    s_bfe_u32 s2, 9, 0x100000
 ; GFX8-NEXT:    s_lshl_b32 s0, s0, s2
 ; GFX8-NEXT:    s_lshl_b32 s1, s1, s2
-; GFX8-NEXT:    s_xor_b32 s3, s0, 0xffff
+; GFX8-NEXT:    s_xor_b32 s3, s0, -1
 ; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
 ; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
 ; GFX8-NEXT:    s_cmp_lt_u32 s3, s1
@@ -86,7 +86,7 @@ define amdgpu_ps i7 @s_uaddsat_i7(i7 inreg %lhs, i7 inreg %rhs) {
 ; GFX9-NEXT:    s_bfe_u32 s2, 9, 0x100000
 ; GFX9-NEXT:    s_lshl_b32 s0, s0, s2
 ; GFX9-NEXT:    s_lshl_b32 s1, s1, s2
-; GFX9-NEXT:    s_xor_b32 s3, s0, 0xffff
+; GFX9-NEXT:    s_xor_b32 s3, s0, -1
 ; GFX9-NEXT:    s_bfe_u32 s3, s3, 0x100000
 ; GFX9-NEXT:    s_bfe_u32 s1, s1, 0x100000
 ; GFX9-NEXT:    s_cmp_lt_u32 s3, s1
@@ -102,7 +102,7 @@ define amdgpu_ps i7 @s_uaddsat_i7(i7 inreg %lhs, i7 inreg %rhs) {
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_lshl_b32 s0, s0, s2
 ; GFX10-NEXT:    s_lshl_b32 s1, s1, s2
-; GFX10-NEXT:    s_xor_b32 s3, s0, 0xffff
+; GFX10-NEXT:    s_xor_b32 s3, s0, -1
 ; GFX10-NEXT:    s_bfe_u32 s1, s1, 0x100000
 ; GFX10-NEXT:    s_bfe_u32 s3, s3, 0x100000
 ; GFX10-NEXT:    s_cmp_lt_u32 s3, s1
@@ -132,7 +132,7 @@ define i8 @v_uaddsat_i8(i8 %lhs, i8 %rhs) {
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
 ; GFX8-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
-; GFX8-NEXT:    v_xor_b32_e32 v2, 0xffff, v0
+; GFX8-NEXT:    v_xor_b32_e32 v2, -1, v0
 ; GFX8-NEXT:    v_min_u16_e32 v1, v2, v1
 ; GFX8-NEXT:    v_add_u16_e32 v0, v0, v1
 ; GFX8-NEXT:    v_lshrrev_b16_e32 v0, 8, v0
@@ -143,7 +143,7 @@ define i8 @v_uaddsat_i8(i8 %lhs, i8 %rhs) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
-; GFX9-NEXT:    v_xor_b32_e32 v2, 0xffff, v0
+; GFX9-NEXT:    v_xor_b32_e32 v2, -1, v0
 ; GFX9-NEXT:    v_min_u16_e32 v1, v2, v1
 ; GFX9-NEXT:    v_add_u16_e32 v0, v0, v1
 ; GFX9-NEXT:    v_lshrrev_b16_e32 v0, 8, v0
@@ -156,7 +156,7 @@ define i8 @v_uaddsat_i8(i8 %lhs, i8 %rhs) {
 ; GFX10-NEXT:    v_lshlrev_b16_e64 v0, 8, v0
 ; GFX10-NEXT:    v_lshlrev_b16_e64 v1, 8, v1
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_xor_b32_e32 v2, 0xffff, v0
+; GFX10-NEXT:    v_xor_b32_e32 v2, -1, v0
 ; GFX10-NEXT:    v_min_u16_e64 v1, v2, v1
 ; GFX10-NEXT:    v_add_nc_u16_e64 v0, v0, v1
 ; GFX10-NEXT:    v_lshrrev_b16_e64 v0, 8, v0
@@ -182,7 +182,7 @@ define amdgpu_ps i8 @s_uaddsat_i8(i8 inreg %lhs, i8 inreg %rhs) {
 ; GFX8-NEXT:    s_bfe_u32 s2, 8, 0x100000
 ; GFX8-NEXT:    s_lshl_b32 s0, s0, s2
 ; GFX8-NEXT:    s_lshl_b32 s1, s1, s2
-; GFX8-NEXT:    s_xor_b32 s3, s0, 0xffff
+; GFX8-NEXT:    s_xor_b32 s3, s0, -1
 ; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
 ; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
 ; GFX8-NEXT:    s_cmp_lt_u32 s3, s1
@@ -197,7 +197,7 @@ define amdgpu_ps i8 @s_uaddsat_i8(i8 inreg %lhs, i8 inreg %rhs) {
 ; GFX9-NEXT:    s_bfe_u32 s2, 8, 0x100000
 ; GFX9-NEXT:    s_lshl_b32 s0, s0, s2
 ; GFX9-NEXT:    s_lshl_b32 s1, s1, s2
-; GFX9-NEXT:    s_xor_b32 s3, s0, 0xffff
+; GFX9-NEXT:    s_xor_b32 s3, s0, -1
 ; GFX9-NEXT:    s_bfe_u32 s3, s3, 0x100000
 ; GFX9-NEXT:    s_bfe_u32 s1, s1, 0x100000
 ; GFX9-NEXT:    s_cmp_lt_u32 s3, s1
@@ -213,7 +213,7 @@ define amdgpu_ps i8 @s_uaddsat_i8(i8 inreg %lhs, i8 inreg %rhs) {
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_lshl_b32 s0, s0, s2
 ; GFX10-NEXT:    s_lshl_b32 s1, s1, s2
-; GFX10-NEXT:    s_xor_b32 s3, s0, 0xffff
+; GFX10-NEXT:    s_xor_b32 s3, s0, -1
 ; GFX10-NEXT:    s_bfe_u32 s1, s1, 0x100000
 ; GFX10-NEXT:    s_bfe_u32 s3, s3, 0x100000
 ; GFX10-NEXT:    s_cmp_lt_u32 s3, s1
@@ -257,13 +257,12 @@ define i16 @v_uaddsat_v2i8(i16 %lhs.arg, i16 %rhs.arg) {
 ; GFX8-NEXT:    v_mov_b32_e32 v2, 8
 ; GFX8-NEXT:    v_lshrrev_b32_sdwa v3, v2, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX8-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; GFX8-NEXT:    s_mov_b32 s4, 0xffff
 ; GFX8-NEXT:    v_lshrrev_b32_sdwa v2, v2, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX8-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
-; GFX8-NEXT:    v_xor_b32_e32 v4, s4, v0
+; GFX8-NEXT:    v_xor_b32_e32 v4, -1, v0
 ; GFX8-NEXT:    v_min_u16_e32 v1, v4, v1
 ; GFX8-NEXT:    v_add_u16_e32 v0, v0, v1
-; GFX8-NEXT:    v_xor_b32_e32 v1, s4, v3
+; GFX8-NEXT:    v_xor_b32_e32 v1, -1, v3
 ; GFX8-NEXT:    v_min_u16_e32 v1, v1, v2
 ; GFX8-NEXT:    v_mov_b32_e32 v2, 0xff
 ; GFX8-NEXT:    v_add_u16_e32 v1, v3, v1
@@ -277,14 +276,13 @@ define i16 @v_uaddsat_v2i8(i16 %lhs.arg, i16 %rhs.arg) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    s_mov_b32 s4, 8
 ; GFX9-NEXT:    v_lshrrev_b32_sdwa v2, s4, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    v_lshrrev_b32_sdwa v3, s4, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; GFX9-NEXT:    s_mov_b32 s4, 0xffff
+; GFX9-NEXT:    v_lshrrev_b32_sdwa v3, s4, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
-; GFX9-NEXT:    v_xor_b32_e32 v4, s4, v0
+; GFX9-NEXT:    v_xor_b32_e32 v4, -1, v0
 ; GFX9-NEXT:    v_min_u16_e32 v1, v4, v1
 ; GFX9-NEXT:    v_add_u16_e32 v0, v0, v1
-; GFX9-NEXT:    v_xor_b32_e32 v1, s4, v2
+; GFX9-NEXT:    v_xor_b32_e32 v1, -1, v2
 ; GFX9-NEXT:    v_min_u16_e32 v1, v1, v3
 ; GFX9-NEXT:    s_movk_i32 s4, 0xff
 ; GFX9-NEXT:    v_add_u16_e32 v1, v2, v1
@@ -300,12 +298,11 @@ define i16 @v_uaddsat_v2i8(i16 %lhs.arg, i16 %rhs.arg) {
 ; GFX10-NEXT:    s_mov_b32 s4, 8
 ; GFX10-NEXT:    v_lshlrev_b16_e64 v2, 8, v0
 ; GFX10-NEXT:    v_lshrrev_b32_sdwa v0, s4, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT:    s_mov_b32 s5, 0xffff
 ; GFX10-NEXT:    v_lshrrev_b32_sdwa v3, s4, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX10-NEXT:    v_lshlrev_b16_e64 v1, 8, v1
-; GFX10-NEXT:    v_xor_b32_e32 v4, s5, v2
-; GFX10-NEXT:    v_xor_b32_e32 v5, s5, v0
 ; GFX10-NEXT:    s_movk_i32 s4, 0xff
+; GFX10-NEXT:    v_xor_b32_e32 v4, -1, v2
+; GFX10-NEXT:    v_xor_b32_e32 v5, -1, v0
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_min_u16_e64 v1, v4, v1
 ; GFX10-NEXT:    v_min_u16_e64 v3, v5, v3
@@ -353,19 +350,18 @@ define amdgpu_ps i16 @s_uaddsat_v2i8(i16 inreg %lhs.arg, i16 inreg %rhs.arg) {
 ; GFX8-NEXT:    s_bfe_u32 s4, 8, 0x100000
 ; GFX8-NEXT:    s_lshr_b32 s2, s0, 8
 ; GFX8-NEXT:    s_lshl_b32 s0, s0, s4
-; GFX8-NEXT:    s_mov_b32 s5, 0xffff
-; GFX8-NEXT:    s_xor_b32 s6, s0, s5
+; GFX8-NEXT:    s_xor_b32 s5, s0, -1
 ; GFX8-NEXT:    s_lshr_b32 s3, s1, 8
 ; GFX8-NEXT:    s_lshl_b32 s1, s1, s4
-; GFX8-NEXT:    s_bfe_u32 s6, s6, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s5, s5, 0x100000
 ; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s6, s1
-; GFX8-NEXT:    s_cselect_b32 s1, s6, s1
+; GFX8-NEXT:    s_cmp_lt_u32 s5, s1
+; GFX8-NEXT:    s_cselect_b32 s1, s5, s1
 ; GFX8-NEXT:    s_add_i32 s0, s0, s1
 ; GFX8-NEXT:    s_lshl_b32 s1, s2, s4
 ; GFX8-NEXT:    s_lshl_b32 s2, s3, s4
 ; GFX8-NEXT:    s_bfe_u32 s0, s0, 0x100000
-; GFX8-NEXT:    s_xor_b32 s3, s1, s5
+; GFX8-NEXT:    s_xor_b32 s3, s1, -1
 ; GFX8-NEXT:    s_lshr_b32 s0, s0, s4
 ; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
 ; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
@@ -386,19 +382,18 @@ define amdgpu_ps i16 @s_uaddsat_v2i8(i16 inreg %lhs.arg, i16 inreg %rhs.arg) {
 ; GFX9-NEXT:    s_bfe_u32 s4, 8, 0x100000
 ; GFX9-NEXT:    s_lshr_b32 s2, s0, 8
 ; GFX9-NEXT:    s_lshl_b32 s0, s0, s4
-; GFX9-NEXT:    s_mov_b32 s5, 0xffff
-; GFX9-NEXT:    s_xor_b32 s6, s0, s5
+; GFX9-NEXT:    s_xor_b32 s5, s0, -1
 ; GFX9-NEXT:    s_lshr_b32 s3, s1, 8
 ; GFX9-NEXT:    s_lshl_b32 s1, s1, s4
-; GFX9-NEXT:    s_bfe_u32 s6, s6, 0x100000
+; GFX9-NEXT:    s_bfe_u32 s5, s5, 0x100000
 ; GFX9-NEXT:    s_bfe_u32 s1, s1, 0x100000
-; GFX9-NEXT:    s_cmp_lt_u32 s6, s1
-; GFX9-NEXT:    s_cselect_b32 s1, s6, s1
+; GFX9-NEXT:    s_cmp_lt_u32 s5, s1
+; GFX9-NEXT:    s_cselect_b32 s1, s5, s1
 ; GFX9-NEXT:    s_add_i32 s0, s0, s1
 ; GFX9-NEXT:    s_lshl_b32 s1, s2, s4
 ; GFX9-NEXT:    s_lshl_b32 s2, s3, s4
 ; GFX9-NEXT:    s_bfe_u32 s0, s0, 0x100000
-; GFX9-NEXT:    s_xor_b32 s3, s1, s5
+; GFX9-NEXT:    s_xor_b32 s3, s1, -1
 ; GFX9-NEXT:    s_lshr_b32 s0, s0, s4
 ; GFX9-NEXT:    s_bfe_u32 s3, s3, 0x100000
 ; GFX9-NEXT:    s_bfe_u32 s2, s2, 0x100000
@@ -417,20 +412,19 @@ define amdgpu_ps i16 @s_uaddsat_v2i8(i16 inreg %lhs.arg, i16 inreg %rhs.arg) {
 ; GFX10-LABEL: s_uaddsat_v2i8:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_bfe_u32 s2, 8, 0x100000
-; GFX10-NEXT:    s_mov_b32 s4, 0xffff
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_lshl_b32 s3, s0, s2
-; GFX10-NEXT:    s_lshl_b32 s6, s1, s2
-; GFX10-NEXT:    s_xor_b32 s5, s3, s4
-; GFX10-NEXT:    s_bfe_u32 s6, s6, 0x100000
+; GFX10-NEXT:    s_lshl_b32 s5, s1, s2
+; GFX10-NEXT:    s_xor_b32 s4, s3, -1
 ; GFX10-NEXT:    s_bfe_u32 s5, s5, 0x100000
+; GFX10-NEXT:    s_bfe_u32 s4, s4, 0x100000
 ; GFX10-NEXT:    s_lshr_b32 s0, s0, 8
 ; GFX10-NEXT:    s_lshr_b32 s1, s1, 8
-; GFX10-NEXT:    s_cmp_lt_u32 s5, s6
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cselect_b32 s5, s5, s6
+; GFX10-NEXT:    s_cmp_lt_u32 s4, s5
+; GFX10-NEXT:    s_cselect_b32 s4, s4, s5
 ; GFX10-NEXT:    s_lshl_b32 s0, s0, s2
-; GFX10-NEXT:    s_add_i32 s3, s3, s5
-; GFX10-NEXT:    s_xor_b32 s4, s0, s4
+; GFX10-NEXT:    s_add_i32 s3, s3, s4
+; GFX10-NEXT:    s_xor_b32 s4, s0, -1
 ; GFX10-NEXT:    s_lshl_b32 s1, s1, s2
 ; GFX10-NEXT:    s_bfe_u32 s3, s3, 0x100000
 ; GFX10-NEXT:    s_bfe_u32 s4, s4, 0x100000
@@ -509,25 +503,24 @@ define i32 @v_uaddsat_v4i8(i32 %lhs.arg, i32 %rhs.arg) {
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v4, 16, v0
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v5, 24, v0
 ; GFX8-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; GFX8-NEXT:    s_mov_b32 s4, 0xffff
 ; GFX8-NEXT:    v_lshrrev_b32_sdwa v2, v2, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v6, 16, v1
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v7, 24, v1
 ; GFX8-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
-; GFX8-NEXT:    v_xor_b32_e32 v8, s4, v0
+; GFX8-NEXT:    v_xor_b32_e32 v8, -1, v0
 ; GFX8-NEXT:    v_min_u16_e32 v1, v8, v1
 ; GFX8-NEXT:    v_add_u16_e32 v0, v0, v1
-; GFX8-NEXT:    v_xor_b32_e32 v1, s4, v3
+; GFX8-NEXT:    v_xor_b32_e32 v1, -1, v3
 ; GFX8-NEXT:    v_min_u16_e32 v1, v1, v2
 ; GFX8-NEXT:    v_lshlrev_b16_e32 v2, 8, v4
 ; GFX8-NEXT:    v_add_u16_e32 v1, v3, v1
 ; GFX8-NEXT:    v_lshlrev_b16_e32 v3, 8, v6
-; GFX8-NEXT:    v_xor_b32_e32 v4, s4, v2
+; GFX8-NEXT:    v_xor_b32_e32 v4, -1, v2
 ; GFX8-NEXT:    v_min_u16_e32 v3, v4, v3
 ; GFX8-NEXT:    v_add_u16_e32 v2, v2, v3
 ; GFX8-NEXT:    v_lshlrev_b16_e32 v3, 8, v5
 ; GFX8-NEXT:    v_lshlrev_b16_e32 v4, 8, v7
-; GFX8-NEXT:    v_xor_b32_e32 v5, s4, v3
+; GFX8-NEXT:    v_xor_b32_e32 v5, -1, v3
 ; GFX8-NEXT:    v_min_u16_e32 v4, v5, v4
 ; GFX8-NEXT:    v_add_u16_e32 v3, v3, v4
 ; GFX8-NEXT:    v_mov_b32_e32 v4, 0xff
@@ -546,29 +539,28 @@ define i32 @v_uaddsat_v4i8(i32 %lhs.arg, i32 %rhs.arg) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    s_mov_b32 s4, 8
 ; GFX9-NEXT:    v_lshrrev_b32_sdwa v2, s4, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    v_lshrrev_b32_sdwa v5, s4, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v4, 24, v0
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; GFX9-NEXT:    s_mov_b32 s4, 0xffff
+; GFX9-NEXT:    v_lshrrev_b32_sdwa v5, s4, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v6, 16, v1
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v7, 24, v1
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
-; GFX9-NEXT:    v_xor_b32_e32 v8, s4, v0
+; GFX9-NEXT:    v_xor_b32_e32 v8, -1, v0
 ; GFX9-NEXT:    v_min_u16_e32 v1, v8, v1
 ; GFX9-NEXT:    v_add_u16_e32 v0, v0, v1
-; GFX9-NEXT:    v_xor_b32_e32 v1, s4, v2
+; GFX9-NEXT:    v_xor_b32_e32 v1, -1, v2
 ; GFX9-NEXT:    v_min_u16_e32 v1, v1, v5
 ; GFX9-NEXT:    v_add_u16_e32 v1, v2, v1
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v2, 8, v3
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v3, 8, v6
-; GFX9-NEXT:    v_xor_b32_e32 v5, s4, v2
+; GFX9-NEXT:    v_xor_b32_e32 v5, -1, v2
 ; GFX9-NEXT:    v_min_u16_e32 v3, v5, v3
 ; GFX9-NEXT:    v_add_u16_e32 v2, v2, v3
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v3, 8, v4
-; GFX9-NEXT:    v_xor_b32_e32 v5, s4, v3
-; GFX9-NEXT:    v_lshlrev_b16_e32 v4, 8, v7
 ; GFX9-NEXT:    s_movk_i32 s4, 0xff
+; GFX9-NEXT:    v_lshlrev_b16_e32 v4, 8, v7
+; GFX9-NEXT:    v_xor_b32_e32 v5, -1, v3
 ; GFX9-NEXT:    v_and_b32_sdwa v1, v1, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
 ; GFX9-NEXT:    v_min_u16_e32 v4, v5, v4
 ; GFX9-NEXT:    v_lshrrev_b16_e32 v0, 8, v0
@@ -585,30 +577,29 @@ define i32 @v_uaddsat_v4i8(i32 %lhs.arg, i32 %rhs.arg) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    s_mov_b32 s4, 8
-; GFX10-NEXT:    s_mov_b32 s7, 0xffff
+; GFX10-NEXT:    v_lshlrev_b16_e64 v4, 8, v0
 ; GFX10-NEXT:    v_lshrrev_b32_sdwa v2, s4, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX10-NEXT:    v_lshrrev_b32_sdwa v3, s4, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT:    v_lshlrev_b16_e64 v4, 8, v0
 ; GFX10-NEXT:    s_mov_b32 s5, 16
 ; GFX10-NEXT:    s_mov_b32 s6, 24
-; GFX10-NEXT:    v_xor_b32_e32 v5, s7, v2
 ; GFX10-NEXT:    v_lshrrev_b32_sdwa v6, s5, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    v_xor_b32_e32 v5, -1, v2
 ; GFX10-NEXT:    v_lshrrev_b32_sdwa v0, s6, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX10-NEXT:    v_lshlrev_b16_e64 v7, 8, v1
-; GFX10-NEXT:    v_xor_b32_e32 v8, s7, v4
+; GFX10-NEXT:    v_xor_b32_e32 v8, -1, v4
+; GFX10-NEXT:    v_xor_b32_e32 v11, -1, v6
 ; GFX10-NEXT:    v_min_u16_e64 v3, v5, v3
 ; GFX10-NEXT:    v_lshrrev_b32_sdwa v5, s5, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT:    v_xor_b32_e32 v11, s7, v6
 ; GFX10-NEXT:    v_lshrrev_b32_sdwa v1, s6, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX10-NEXT:    v_min_u16_e64 v7, v8, v7
-; GFX10-NEXT:    v_add_nc_u16_e64 v2, v2, v3
-; GFX10-NEXT:    v_xor_b32_e32 v3, s7, v0
 ; GFX10-NEXT:    s_movk_i32 s4, 0xff
+; GFX10-NEXT:    v_add_nc_u16_e64 v2, v2, v3
+; GFX10-NEXT:    v_xor_b32_e32 v3, -1, v0
 ; GFX10-NEXT:    v_min_u16_e64 v5, v11, v5
 ; GFX10-NEXT:    v_add_nc_u16_e64 v4, v4, v7
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_b32_sdwa v2, v2, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
 ; GFX10-NEXT:    v_min_u16_e64 v1, v3, v1
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_nc_u16_e64 v3, v6, v5
 ; GFX10-NEXT:    v_lshrrev_b16_e64 v4, 8, v4
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
@@ -682,21 +673,20 @@ define amdgpu_ps i32 @s_uaddsat_v4i8(i32 inreg %lhs.arg, i32 inreg %rhs.arg) {
 ; GFX8-NEXT:    s_lshr_b32 s3, s0, 16
 ; GFX8-NEXT:    s_lshr_b32 s4, s0, 24
 ; GFX8-NEXT:    s_lshl_b32 s0, s0, s8
-; GFX8-NEXT:    s_mov_b32 s9, 0xffff
-; GFX8-NEXT:    s_xor_b32 s10, s0, s9
+; GFX8-NEXT:    s_xor_b32 s9, s0, -1
 ; GFX8-NEXT:    s_lshr_b32 s5, s1, 8
 ; GFX8-NEXT:    s_lshr_b32 s6, s1, 16
 ; GFX8-NEXT:    s_lshr_b32 s7, s1, 24
 ; GFX8-NEXT:    s_lshl_b32 s1, s1, s8
-; GFX8-NEXT:    s_bfe_u32 s10, s10, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s9, s9, 0x100000
 ; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s10, s1
-; GFX8-NEXT:    s_cselect_b32 s1, s10, s1
+; GFX8-NEXT:    s_cmp_lt_u32 s9, s1
+; GFX8-NEXT:    s_cselect_b32 s1, s9, s1
 ; GFX8-NEXT:    s_add_i32 s0, s0, s1
 ; GFX8-NEXT:    s_lshl_b32 s1, s2, s8
 ; GFX8-NEXT:    s_lshl_b32 s2, s5, s8
 ; GFX8-NEXT:    s_bfe_u32 s0, s0, 0x100000
-; GFX8-NEXT:    s_xor_b32 s5, s1, s9
+; GFX8-NEXT:    s_xor_b32 s5, s1, -1
 ; GFX8-NEXT:    s_lshr_b32 s0, s0, s8
 ; GFX8-NEXT:    s_bfe_u32 s5, s5, 0x100000
 ; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
@@ -706,7 +696,7 @@ define amdgpu_ps i32 @s_uaddsat_v4i8(i32 inreg %lhs.arg, i32 inreg %rhs.arg) {
 ; GFX8-NEXT:    s_lshl_b32 s2, s3, s8
 ; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
 ; GFX8-NEXT:    s_lshl_b32 s3, s6, s8
-; GFX8-NEXT:    s_xor_b32 s5, s2, s9
+; GFX8-NEXT:    s_xor_b32 s5, s2, -1
 ; GFX8-NEXT:    s_lshr_b32 s1, s1, s8
 ; GFX8-NEXT:    s_bfe_u32 s5, s5, 0x100000
 ; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
@@ -716,7 +706,7 @@ define amdgpu_ps i32 @s_uaddsat_v4i8(i32 inreg %lhs.arg, i32 inreg %rhs.arg) {
 ; GFX8-NEXT:    s_lshl_b32 s3, s4, s8
 ; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
 ; GFX8-NEXT:    s_lshl_b32 s4, s7, s8
-; GFX8-NEXT:    s_xor_b32 s5, s3, s9
+; GFX8-NEXT:    s_xor_b32 s5, s3, -1
 ; GFX8-NEXT:    s_lshr_b32 s2, s2, s8
 ; GFX8-NEXT:    s_bfe_u32 s5, s5, 0x100000
 ; GFX8-NEXT:    s_bfe_u32 s4, s4, 0x100000
@@ -745,21 +735,20 @@ define amdgpu_ps i32 @s_uaddsat_v4i8(i32 inreg %lhs.arg, i32 inreg %rhs.arg) {
 ; GFX9-NEXT:    s_lshr_b32 s3, s0, 16
 ; GFX9-NEXT:    s_lshr_b32 s4, s0, 24
 ; GFX9-NEXT:    s_lshl_b32 s0, s0, s8
-; GFX9-NEXT:    s_mov_b32 s9, 0xffff
-; GFX9-NEXT:    s_xor_b32 s10, s0, s9
+; GFX9-NEXT:    s_xor_b32 s9, s0, -1
 ; GFX9-NEXT:    s_lshr_b32 s5, s1, 8
 ; GFX9-NEXT:    s_lshr_b32 s6, s1, 16
 ; GFX9-NEXT:    s_lshr_b32 s7, s1, 24
 ; GFX9-NEXT:    s_lshl_b32 s1, s1, s8
-; GFX9-NEXT:    s_bfe_u32 s10, s10, 0x100000
+; GFX9-NEXT:    s_bfe_u32 s9, s9, 0x100000
 ; GFX9-NEXT:    s_bfe_u32 s1, s1, 0x100000
-; GFX9-NEXT:    s_cmp_lt_u32 s10, s1
-; GFX9-NEXT:    s_cselect_b32 s1, s10, s1
+; GFX9-NEXT:    s_cmp_lt_u32 s9, s1
+; GFX9-NEXT:    s_cselect_b32 s1, s9, s1
 ; GFX9-NEXT:    s_add_i32 s0, s0, s1
 ; GFX9-NEXT:    s_lshl_b32 s1, s2, s8
 ; GFX9-NEXT:    s_lshl_b32 s2, s5, s8
 ; GFX9-NEXT:    s_bfe_u32 s0, s0, 0x100000
-; GFX9-NEXT:    s_xor_b32 s5, s1, s9
+; GFX9-NEXT:    s_xor_b32 s5, s1, -1
 ; GFX9-NEXT:    s_lshr_b32 s0, s0, s8
 ; GFX9-NEXT:    s_bfe_u32 s5, s5, 0x100000
 ; GFX9-NEXT:    s_bfe_u32 s2, s2, 0x100000
@@ -769,7 +758,7 @@ define amdgpu_ps i32 @s_uaddsat_v4i8(i32 inreg %lhs.arg, i32 inreg %rhs.arg) {
 ; GFX9-NEXT:    s_lshl_b32 s2, s3, s8
 ; GFX9-NEXT:    s_bfe_u32 s1, s1, 0x100000
 ; GFX9-NEXT:    s_lshl_b32 s3, s6, s8
-; GFX9-NEXT:    s_xor_b32 s5, s2, s9
+; GFX9-NEXT:    s_xor_b32 s5, s2, -1
 ; GFX9-NEXT:    s_lshr_b32 s1, s1, s8
 ; GFX9-NEXT:    s_bfe_u32 s5, s5, 0x100000
 ; GFX9-NEXT:    s_bfe_u32 s3, s3, 0x100000
@@ -779,7 +768,7 @@ define amdgpu_ps i32 @s_uaddsat_v4i8(i32 inreg %lhs.arg, i32 inreg %rhs.arg) {
 ; GFX9-NEXT:    s_lshl_b32 s3, s4, s8
 ; GFX9-NEXT:    s_bfe_u32 s2, s2, 0x100000
 ; GFX9-NEXT:    s_lshl_b32 s4, s7, s8
-; GFX9-NEXT:    s_xor_b32 s5, s3, s9
+; GFX9-NEXT:    s_xor_b32 s5, s3, -1
 ; GFX9-NEXT:    s_lshr_b32 s2, s2, s8
 ; GFX9-NEXT:    s_bfe_u32 s5, s5, 0x100000
 ; GFX9-NEXT:    s_bfe_u32 s4, s4, 0x100000
@@ -808,40 +797,39 @@ define amdgpu_ps i32 @s_uaddsat_v4i8(i32 inreg %lhs.arg, i32 inreg %rhs.arg) {
 ; GFX10-NEXT:    s_lshr_b32 s3, s0, 16
 ; GFX10-NEXT:    s_lshr_b32 s4, s0, 24
 ; GFX10-NEXT:    s_lshl_b32 s0, s0, s5
-; GFX10-NEXT:    s_mov_b32 s7, 0xffff
-; GFX10-NEXT:    s_lshl_b32 s10, s1, s5
-; GFX10-NEXT:    s_xor_b32 s9, s0, s7
-; GFX10-NEXT:    s_bfe_u32 s10, s10, 0x100000
+; GFX10-NEXT:    s_lshl_b32 s9, s1, s5
+; GFX10-NEXT:    s_xor_b32 s8, s0, -1
 ; GFX10-NEXT:    s_bfe_u32 s9, s9, 0x100000
+; GFX10-NEXT:    s_bfe_u32 s8, s8, 0x100000
 ; GFX10-NEXT:    s_lshr_b32 s6, s1, 8
-; GFX10-NEXT:    s_lshr_b32 s8, s1, 16
+; GFX10-NEXT:    s_lshr_b32 s7, s1, 16
 ; GFX10-NEXT:    s_lshr_b32 s1, s1, 24
-; GFX10-NEXT:    s_cmp_lt_u32 s9, s10
+; GFX10-NEXT:    s_cmp_lt_u32 s8, s9
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cselect_b32 s9, s9, s10
+; GFX10-NEXT:    s_cselect_b32 s8, s8, s9
 ; GFX10-NEXT:    s_lshl_b32 s2, s2, s5
-; GFX10-NEXT:    s_add_i32 s0, s0, s9
-; GFX10-NEXT:    s_xor_b32 s9, s2, s7
+; GFX10-NEXT:    s_add_i32 s0, s0, s8
+; GFX10-NEXT:    s_xor_b32 s8, s2, -1
 ; GFX10-NEXT:    s_lshl_b32 s6, s6, s5
 ; GFX10-NEXT:    s_bfe_u32 s0, s0, 0x100000
-; GFX10-NEXT:    s_bfe_u32 s9, s9, 0x100000
+; GFX10-NEXT:    s_bfe_u32 s8, s8, 0x100000
 ; GFX10-NEXT:    s_bfe_u32 s6, s6, 0x100000
 ; GFX10-NEXT:    s_lshr_b32 s0, s0, s5
-; GFX10-NEXT:    s_cmp_lt_u32 s9, s6
-; GFX10-NEXT:    s_cselect_b32 s6, s9, s6
+; GFX10-NEXT:    s_cmp_lt_u32 s8, s6
+; GFX10-NEXT:    s_cselect_b32 s6, s8, s6
 ; GFX10-NEXT:    s_lshl_b32 s3, s3, s5
 ; GFX10-NEXT:    s_add_i32 s2, s2, s6
-; GFX10-NEXT:    s_xor_b32 s6, s3, s7
-; GFX10-NEXT:    s_lshl_b32 s8, s8, s5
+; GFX10-NEXT:    s_xor_b32 s6, s3, -1
+; GFX10-NEXT:    s_lshl_b32 s7, s7, s5
 ; GFX10-NEXT:    s_bfe_u32 s2, s2, 0x100000
 ; GFX10-NEXT:    s_bfe_u32 s6, s6, 0x100000
-; GFX10-NEXT:    s_bfe_u32 s8, s8, 0x100000
+; GFX10-NEXT:    s_bfe_u32 s7, s7, 0x100000
 ; GFX10-NEXT:    s_lshr_b32 s2, s2, s5
-; GFX10-NEXT:    s_cmp_lt_u32 s6, s8
-; GFX10-NEXT:    s_cselect_b32 s6, s6, s8
+; GFX10-NEXT:    s_cmp_lt_u32 s6, s7
+; GFX10-NEXT:    s_cselect_b32 s6, s6, s7
 ; GFX10-NEXT:    s_lshl_b32 s4, s4, s5
 ; GFX10-NEXT:    s_add_i32 s3, s3, s6
-; GFX10-NEXT:    s_xor_b32 s6, s4, s7
+; GFX10-NEXT:    s_xor_b32 s6, s4, -1
 ; GFX10-NEXT:    s_lshl_b32 s1, s1, s5
 ; GFX10-NEXT:    s_bfe_u32 s3, s3, 0x100000
 ; GFX10-NEXT:    s_bfe_u32 s6, s6, 0x100000
@@ -2203,7 +2191,7 @@ define i16 @v_uaddsat_i16(i16 %lhs, i16 %rhs) {
 ; GFX8-LABEL: v_uaddsat_i16:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_xor_b32_e32 v2, 0xffff, v0
+; GFX8-NEXT:    v_xor_b32_e32 v2, -1, v0
 ; GFX8-NEXT:    v_min_u16_e32 v1, v2, v1
 ; GFX8-NEXT:    v_add_u16_e32 v0, v0, v1
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
@@ -2211,7 +2199,7 @@ define i16 @v_uaddsat_i16(i16 %lhs, i16 %rhs) {
 ; GFX9-LABEL: v_uaddsat_i16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_xor_b32_e32 v2, 0xffff, v0
+; GFX9-NEXT:    v_xor_b32_e32 v2, -1, v0
 ; GFX9-NEXT:    v_min_u16_e32 v1, v2, v1
 ; GFX9-NEXT:    v_add_u16_e32 v0, v0, v1
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -2220,7 +2208,7 @@ define i16 @v_uaddsat_i16(i16 %lhs, i16 %rhs) {
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_xor_b32_e32 v2, 0xffff, v0
+; GFX10-NEXT:    v_xor_b32_e32 v2, -1, v0
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_min_u16_e64 v1, v2, v1
 ; GFX10-NEXT:    v_add_nc_u16_e64 v0, v0, v1
@@ -2243,7 +2231,7 @@ define amdgpu_ps i16 @s_uaddsat_i16(i16 inreg %lhs, i16 inreg %rhs) {
 ;
 ; GFX8-LABEL: s_uaddsat_i16:
 ; GFX8:       ; %bb.0:
-; GFX8-NEXT:    s_xor_b32 s2, s0, 0xffff
+; GFX8-NEXT:    s_xor_b32 s2, s0, -1
 ; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
 ; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
 ; GFX8-NEXT:    s_cmp_lt_u32 s2, s1
@@ -2253,7 +2241,7 @@ define amdgpu_ps i16 @s_uaddsat_i16(i16 inreg %lhs, i16 inreg %rhs) {
 ;
 ; GFX9-LABEL: s_uaddsat_i16:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_xor_b32 s2, s0, 0xffff
+; GFX9-NEXT:    s_xor_b32 s2, s0, -1
 ; GFX9-NEXT:    s_bfe_u32 s2, s2, 0x100000
 ; GFX9-NEXT:    s_bfe_u32 s1, s1, 0x100000
 ; GFX9-NEXT:    s_cmp_lt_u32 s2, s1
@@ -2263,7 +2251,7 @@ define amdgpu_ps i16 @s_uaddsat_i16(i16 inreg %lhs, i16 inreg %rhs) {
 ;
 ; GFX10-LABEL: s_uaddsat_i16:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_xor_b32 s2, s0, 0xffff
+; GFX10-NEXT:    s_xor_b32 s2, s0, -1
 ; GFX10-NEXT:    s_bfe_u32 s1, s1, 0x100000
 ; GFX10-NEXT:    s_bfe_u32 s2, s2, 0x100000
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
@@ -2288,21 +2276,21 @@ define amdgpu_ps half @uaddsat_i16_sv(i16 inreg %lhs, i16 %rhs) {
 ;
 ; GFX8-LABEL: uaddsat_i16_sv:
 ; GFX8:       ; %bb.0:
-; GFX8-NEXT:    s_xor_b32 s1, s0, 0xffff
+; GFX8-NEXT:    s_xor_b32 s1, s0, -1
 ; GFX8-NEXT:    v_min_u16_e32 v0, s1, v0
 ; GFX8-NEXT:    v_add_u16_e32 v0, s0, v0
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: uaddsat_i16_sv:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_xor_b32 s1, s0, 0xffff
+; GFX9-NEXT:    s_xor_b32 s1, s0, -1
 ; GFX9-NEXT:    v_min_u16_e32 v0, s1, v0
 ; GFX9-NEXT:    v_add_u16_e32 v0, s0, v0
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: uaddsat_i16_sv:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_xor_b32 s1, s0, 0xffff
+; GFX10-NEXT:    s_xor_b32 s1, s0, -1
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_min_u16_e64 v0, s1, v0
 ; GFX10-NEXT:    v_add_nc_u16_e64 v0, s0, v0
@@ -2325,21 +2313,21 @@ define amdgpu_ps half @uaddsat_i16_vs(i16 %lhs, i16 inreg %rhs) {
 ;
 ; GFX8-LABEL: uaddsat_i16_vs:
 ; GFX8:       ; %bb.0:
-; GFX8-NEXT:    v_xor_b32_e32 v1, 0xffff, v0
+; GFX8-NEXT:    v_xor_b32_e32 v1, -1, v0
 ; GFX8-NEXT:    v_min_u16_e32 v1, s0, v1
 ; GFX8-NEXT:    v_add_u16_e32 v0, v0, v1
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: uaddsat_i16_vs:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    v_xor_b32_e32 v1, 0xffff, v0
+; GFX9-NEXT:    v_xor_b32_e32 v1, -1, v0
 ; GFX9-NEXT:    v_min_u16_e32 v1, s0, v1
 ; GFX9-NEXT:    v_add_u16_e32 v0, v0, v1
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: uaddsat_i16_vs:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    v_xor_b32_e32 v1, 0xffff, v0
+; GFX10-NEXT:    v_xor_b32_e32 v1, -1, v0
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_min_u16_e64 v1, v1, s0
 ; GFX10-NEXT:    v_add_nc_u16_e64 v0, v0, v1
@@ -2370,10 +2358,9 @@ define <2 x i16> @v_uaddsat_v2i16(<2 x i16> %lhs, <2 x i16> %rhs) {
 ; GFX8-LABEL: v_uaddsat_v2i16:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    s_mov_b32 s4, 0xffff
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
-; GFX8-NEXT:    v_xor_b32_e32 v3, s4, v0
-; GFX8-NEXT:    v_xor_b32_e32 v4, s4, v2
+; GFX8-NEXT:    v_xor_b32_e32 v3, -1, v0
+; GFX8-NEXT:    v_xor_b32_e32 v4, -1, v2
 ; GFX8-NEXT:    v_min_u16_e32 v3, v3, v1
 ; GFX8-NEXT:    v_min_u16_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
 ; GFX8-NEXT:    v_add_u16_e32 v0, v0, v3
@@ -2430,16 +2417,15 @@ define amdgpu_ps i32 @s_uaddsat_v2i16(<2 x i16> inreg %lhs, <2 x i16> inreg %rhs
 ;
 ; GFX8-LABEL: s_uaddsat_v2i16:
 ; GFX8:       ; %bb.0:
-; GFX8-NEXT:    s_mov_b32 s4, 0xffff
-; GFX8-NEXT:    s_xor_b32 s5, s0, s4
+; GFX8-NEXT:    s_xor_b32 s4, s0, -1
 ; GFX8-NEXT:    s_lshr_b32 s3, s1, 16
 ; GFX8-NEXT:    s_lshr_b32 s2, s0, 16
-; GFX8-NEXT:    s_bfe_u32 s5, s5, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s4, s4, 0x100000
 ; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s5, s1
-; GFX8-NEXT:    s_cselect_b32 s1, s5, s1
+; GFX8-NEXT:    s_cmp_lt_u32 s4, s1
+; GFX8-NEXT:    s_cselect_b32 s1, s4, s1
 ; GFX8-NEXT:    s_add_i32 s0, s0, s1
-; GFX8-NEXT:    s_xor_b32 s1, s2, s4
+; GFX8-NEXT:    s_xor_b32 s1, s2, -1
 ; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
 ; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
 ; GFX8-NEXT:    s_cmp_lt_u32 s1, s3
@@ -2522,12 +2508,11 @@ define amdgpu_ps float @uaddsat_v2i16_sv(<2 x i16> inreg %lhs, <2 x i16> %rhs) {
 ;
 ; GFX8-LABEL: uaddsat_v2i16_sv:
 ; GFX8:       ; %bb.0:
-; GFX8-NEXT:    s_mov_b32 s2, 0xffff
+; GFX8-NEXT:    s_xor_b32 s2, s0, -1
 ; GFX8-NEXT:    s_lshr_b32 s1, s0, 16
-; GFX8-NEXT:    s_xor_b32 s3, s0, s2
-; GFX8-NEXT:    s_xor_b32 s2, s1, s2
+; GFX8-NEXT:    v_min_u16_e32 v1, s2, v0
+; GFX8-NEXT:    s_xor_b32 s2, s1, -1
 ; GFX8-NEXT:    v_mov_b32_e32 v2, s2
-; GFX8-NEXT:    v_min_u16_e32 v1, s3, v0
 ; GFX8-NEXT:    v_min_u16_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
 ; GFX8-NEXT:    v_mov_b32_e32 v2, s1
 ; GFX8-NEXT:    v_add_u16_e32 v1, s0, v1
@@ -2580,11 +2565,10 @@ define amdgpu_ps float @uaddsat_v2i16_vs(<2 x i16> %lhs, <2 x i16> inreg %rhs) {
 ;
 ; GFX8-LABEL: uaddsat_v2i16_vs:
 ; GFX8:       ; %bb.0:
-; GFX8-NEXT:    s_mov_b32 s2, 0xffff
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
-; GFX8-NEXT:    v_xor_b32_e32 v2, s2, v0
+; GFX8-NEXT:    v_xor_b32_e32 v2, -1, v0
 ; GFX8-NEXT:    s_lshr_b32 s1, s0, 16
-; GFX8-NEXT:    v_xor_b32_e32 v3, s2, v1
+; GFX8-NEXT:    v_xor_b32_e32 v3, -1, v1
 ; GFX8-NEXT:    v_min_u16_e32 v2, s0, v2
 ; GFX8-NEXT:    v_min_u16_e32 v3, s1, v3
 ; GFX8-NEXT:    v_add_u16_e32 v0, v0, v2
@@ -2666,15 +2650,14 @@ define <2 x float> @v_uaddsat_v4i16(<4 x i16> %lhs, <4 x i16> %rhs) {
 ; GFX8-LABEL: v_uaddsat_v4i16:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    s_mov_b32 s4, 0xffff
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v4, 16, v0
-; GFX8-NEXT:    v_xor_b32_e32 v6, s4, v0
+; GFX8-NEXT:    v_xor_b32_e32 v6, -1, v0
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
-; GFX8-NEXT:    v_xor_b32_e32 v7, s4, v4
+; GFX8-NEXT:    v_xor_b32_e32 v7, -1, v4
 ; GFX8-NEXT:    v_min_u16_e32 v6, v6, v2
 ; GFX8-NEXT:    v_min_u16_sdwa v2, v7, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT:    v_xor_b32_e32 v7, s4, v1
-; GFX8-NEXT:    v_xor_b32_e32 v8, s4, v5
+; GFX8-NEXT:    v_xor_b32_e32 v7, -1, v1
+; GFX8-NEXT:    v_xor_b32_e32 v8, -1, v5
 ; GFX8-NEXT:    v_min_u16_e32 v7, v7, v3
 ; GFX8-NEXT:    v_min_u16_sdwa v3, v8, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
 ; GFX8-NEXT:    v_add_u16_e32 v0, v0, v6
@@ -2759,30 +2742,29 @@ define amdgpu_ps <2 x i32> @s_uaddsat_v4i16(<4 x i16> inreg %lhs, <4 x i16> inre
 ;
 ; GFX8-LABEL: s_uaddsat_v4i16:
 ; GFX8:       ; %bb.0:
-; GFX8-NEXT:    s_mov_b32 s8, 0xffff
-; GFX8-NEXT:    s_xor_b32 s9, s0, s8
+; GFX8-NEXT:    s_xor_b32 s8, s0, -1
 ; GFX8-NEXT:    s_lshr_b32 s6, s2, 16
 ; GFX8-NEXT:    s_lshr_b32 s4, s0, 16
 ; GFX8-NEXT:    s_lshr_b32 s5, s1, 16
 ; GFX8-NEXT:    s_lshr_b32 s7, s3, 16
-; GFX8-NEXT:    s_bfe_u32 s9, s9, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s8, s8, 0x100000
 ; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s9, s2
-; GFX8-NEXT:    s_cselect_b32 s2, s9, s2
+; GFX8-NEXT:    s_cmp_lt_u32 s8, s2
+; GFX8-NEXT:    s_cselect_b32 s2, s8, s2
 ; GFX8-NEXT:    s_add_i32 s0, s0, s2
-; GFX8-NEXT:    s_xor_b32 s2, s4, s8
+; GFX8-NEXT:    s_xor_b32 s2, s4, -1
 ; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
 ; GFX8-NEXT:    s_bfe_u32 s6, s6, 0x100000
 ; GFX8-NEXT:    s_cmp_lt_u32 s2, s6
 ; GFX8-NEXT:    s_cselect_b32 s2, s2, s6
 ; GFX8-NEXT:    s_add_i32 s4, s4, s2
-; GFX8-NEXT:    s_xor_b32 s2, s1, s8
+; GFX8-NEXT:    s_xor_b32 s2, s1, -1
 ; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
 ; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
 ; GFX8-NEXT:    s_cmp_lt_u32 s2, s3
 ; GFX8-NEXT:    s_cselect_b32 s2, s2, s3
 ; GFX8-NEXT:    s_add_i32 s1, s1, s2
-; GFX8-NEXT:    s_xor_b32 s2, s5, s8
+; GFX8-NEXT:    s_xor_b32 s2, s5, -1
 ; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
 ; GFX8-NEXT:    s_bfe_u32 s3, s7, 0x100000
 ; GFX8-NEXT:    s_cmp_lt_u32 s2, s3
@@ -2944,20 +2926,19 @@ define <3 x float> @v_uaddsat_v6i16(<6 x i16> %lhs, <6 x i16> %rhs) {
 ; GFX8-LABEL: v_uaddsat_v6i16:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    s_mov_b32 s4, 0xffff
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v6, 16, v0
-; GFX8-NEXT:    v_xor_b32_e32 v9, s4, v0
+; GFX8-NEXT:    v_xor_b32_e32 v9, -1, v0
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v7, 16, v1
-; GFX8-NEXT:    v_xor_b32_e32 v10, s4, v6
+; GFX8-NEXT:    v_xor_b32_e32 v10, -1, v6
 ; GFX8-NEXT:    v_min_u16_e32 v9, v9, v3
 ; GFX8-NEXT:    v_min_u16_sdwa v3, v10, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT:    v_xor_b32_e32 v10, s4, v1
+; GFX8-NEXT:    v_xor_b32_e32 v10, -1, v1
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v8, 16, v2
-; GFX8-NEXT:    v_xor_b32_e32 v11, s4, v7
+; GFX8-NEXT:    v_xor_b32_e32 v11, -1, v7
 ; GFX8-NEXT:    v_min_u16_e32 v10, v10, v4
 ; GFX8-NEXT:    v_min_u16_sdwa v4, v11, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT:    v_xor_b32_e32 v11, s4, v2
-; GFX8-NEXT:    v_xor_b32_e32 v12, s4, v8
+; GFX8-NEXT:    v_xor_b32_e32 v11, -1, v2
+; GFX8-NEXT:    v_xor_b32_e32 v12, -1, v8
 ; GFX8-NEXT:    v_add_u16_e32 v0, v0, v9
 ; GFX8-NEXT:    v_add_u16_sdwa v3, v6, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX8-NEXT:    v_min_u16_e32 v11, v11, v5
@@ -3069,44 +3050,43 @@ define amdgpu_ps <3 x i32> @s_uaddsat_v6i16(<6 x i16> inreg %lhs, <6 x i16> inre
 ;
 ; GFX8-LABEL: s_uaddsat_v6i16:
 ; GFX8:       ; %bb.0:
-; GFX8-NEXT:    s_mov_b32 s12, 0xffff
-; GFX8-NEXT:    s_xor_b32 s13, s0, s12
+; GFX8-NEXT:    s_xor_b32 s12, s0, -1
 ; GFX8-NEXT:    s_lshr_b32 s9, s3, 16
 ; GFX8-NEXT:    s_lshr_b32 s6, s0, 16
 ; GFX8-NEXT:    s_lshr_b32 s7, s1, 16
 ; GFX8-NEXT:    s_lshr_b32 s8, s2, 16
 ; GFX8-NEXT:    s_lshr_b32 s10, s4, 16
 ; GFX8-NEXT:    s_lshr_b32 s11, s5, 16
-; GFX8-NEXT:    s_bfe_u32 s13, s13, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s12, s12, 0x100000
 ; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s13, s3
-; GFX8-NEXT:    s_cselect_b32 s3, s13, s3
+; GFX8-NEXT:    s_cmp_lt_u32 s12, s3
+; GFX8-NEXT:    s_cselect_b32 s3, s12, s3
 ; GFX8-NEXT:    s_add_i32 s0, s0, s3
-; GFX8-NEXT:    s_xor_b32 s3, s6, s12
+; GFX8-NEXT:    s_xor_b32 s3, s6, -1
 ; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
 ; GFX8-NEXT:    s_bfe_u32 s9, s9, 0x100000
 ; GFX8-NEXT:    s_cmp_lt_u32 s3, s9
 ; GFX8-NEXT:    s_cselect_b32 s3, s3, s9
 ; GFX8-NEXT:    s_add_i32 s6, s6, s3
-; GFX8-NEXT:    s_xor_b32 s3, s1, s12
+; GFX8-NEXT:    s_xor_b32 s3, s1, -1
 ; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
 ; GFX8-NEXT:    s_bfe_u32 s4, s4, 0x100000
 ; GFX8-NEXT:    s_cmp_lt_u32 s3, s4
 ; GFX8-NEXT:    s_cselect_b32 s3, s3, s4
 ; GFX8-NEXT:    s_add_i32 s1, s1, s3
-; GFX8-NEXT:    s_xor_b32 s3, s7, s12
+; GFX8-NEXT:    s_xor_b32 s3, s7, -1
 ; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
 ; GFX8-NEXT:    s_bfe_u32 s4, s10, 0x100000
 ; GFX8-NEXT:    s_cmp_lt_u32 s3, s4
 ; GFX8-NEXT:    s_cselect_b32 s3, s3, s4
 ; GFX8-NEXT:    s_add_i32 s7, s7, s3
-; GFX8-NEXT:    s_xor_b32 s3, s2, s12
+; GFX8-NEXT:    s_xor_b32 s3, s2, -1
 ; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
 ; GFX8-NEXT:    s_bfe_u32 s4, s5, 0x100000
 ; GFX8-NEXT:    s_cmp_lt_u32 s3, s4
 ; GFX8-NEXT:    s_cselect_b32 s3, s3, s4
 ; GFX8-NEXT:    s_add_i32 s2, s2, s3
-; GFX8-NEXT:    s_xor_b32 s3, s8, s12
+; GFX8-NEXT:    s_xor_b32 s3, s8, -1
 ; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
 ; GFX8-NEXT:    s_bfe_u32 s4, s11, 0x100000
 ; GFX8-NEXT:    s_cmp_lt_u32 s3, s4
@@ -3307,28 +3287,27 @@ define <4 x float> @v_uaddsat_v8i16(<8 x i16> %lhs, <8 x i16> %rhs) {
 ; GFX8-LABEL: v_uaddsat_v8i16:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    s_mov_b32 s4, 0xffff
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v8, 16, v0
-; GFX8-NEXT:    v_xor_b32_e32 v12, s4, v0
+; GFX8-NEXT:    v_xor_b32_e32 v12, -1, v0
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v9, 16, v1
-; GFX8-NEXT:    v_xor_b32_e32 v13, s4, v8
+; GFX8-NEXT:    v_xor_b32_e32 v13, -1, v8
 ; GFX8-NEXT:    v_min_u16_e32 v12, v12, v4
 ; GFX8-NEXT:    v_min_u16_sdwa v4, v13, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT:    v_xor_b32_e32 v13, s4, v1
+; GFX8-NEXT:    v_xor_b32_e32 v13, -1, v1
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v10, 16, v2
-; GFX8-NEXT:    v_xor_b32_e32 v14, s4, v9
+; GFX8-NEXT:    v_xor_b32_e32 v14, -1, v9
 ; GFX8-NEXT:    v_min_u16_e32 v13, v13, v5
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v11, 16, v3
 ; GFX8-NEXT:    v_min_u16_sdwa v5, v14, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT:    v_xor_b32_e32 v14, s4, v2
-; GFX8-NEXT:    v_xor_b32_e32 v15, s4, v10
+; GFX8-NEXT:    v_xor_b32_e32 v14, -1, v2
+; GFX8-NEXT:    v_xor_b32_e32 v15, -1, v10
 ; GFX8-NEXT:    v_add_u16_e32 v0, v0, v12
 ; GFX8-NEXT:    v_add_u16_sdwa v4, v8, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX8-NEXT:    v_min_u16_e32 v14, v14, v6
 ; GFX8-NEXT:    v_min_u16_sdwa v6, v15, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT:    v_xor_b32_e32 v15, s4, v3
+; GFX8-NEXT:    v_xor_b32_e32 v15, -1, v3
 ; GFX8-NEXT:    v_or_b32_e32 v0, v0, v4
-; GFX8-NEXT:    v_xor_b32_e32 v16, s4, v11
+; GFX8-NEXT:    v_xor_b32_e32 v16, -1, v11
 ; GFX8-NEXT:    v_add_u16_e32 v1, v1, v13
 ; GFX8-NEXT:    v_add_u16_sdwa v4, v9, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX8-NEXT:    v_min_u16_e32 v15, v15, v7
@@ -3464,8 +3443,7 @@ define amdgpu_ps <4 x i32> @s_uaddsat_v8i16(<8 x i16> inreg %lhs, <8 x i16> inre
 ;
 ; GFX8-LABEL: s_uaddsat_v8i16:
 ; GFX8:       ; %bb.0:
-; GFX8-NEXT:    s_mov_b32 s16, 0xffff
-; GFX8-NEXT:    s_xor_b32 s17, s0, s16
+; GFX8-NEXT:    s_xor_b32 s16, s0, -1
 ; GFX8-NEXT:    s_lshr_b32 s12, s4, 16
 ; GFX8-NEXT:    s_lshr_b32 s8, s0, 16
 ; GFX8-NEXT:    s_lshr_b32 s9, s1, 16
@@ -3474,48 +3452,48 @@ define amdgpu_ps <4 x i32> @s_uaddsat_v8i16(<8 x i16> inreg %lhs, <8 x i16> inre
 ; GFX8-NEXT:    s_lshr_b32 s13, s5, 16
 ; GFX8-NEXT:    s_lshr_b32 s14, s6, 16
 ; GFX8-NEXT:    s_lshr_b32 s15, s7, 16
-; GFX8-NEXT:    s_bfe_u32 s17, s17, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s16, s16, 0x100000
 ; GFX8-NEXT:    s_bfe_u32 s4, s4, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s17, s4
-; GFX8-NEXT:    s_cselect_b32 s4, s17, s4
+; GFX8-NEXT:    s_cmp_lt_u32 s16, s4
+; GFX8-NEXT:    s_cselect_b32 s4, s16, s4
 ; GFX8-NEXT:    s_add_i32 s0, s0, s4
-; GFX8-NEXT:    s_xor_b32 s4, s8, s16
+; GFX8-NEXT:    s_xor_b32 s4, s8, -1
 ; GFX8-NEXT:    s_bfe_u32 s4, s4, 0x100000
 ; GFX8-NEXT:    s_bfe_u32 s12, s12, 0x100000
 ; GFX8-NEXT:    s_cmp_lt_u32 s4, s12
 ; GFX8-NEXT:    s_cselect_b32 s4, s4, s12
 ; GFX8-NEXT:    s_add_i32 s8, s8, s4
-; GFX8-NEXT:    s_xor_b32 s4, s1, s16
+; GFX8-NEXT:    s_xor_b32 s4, s1, -1
 ; GFX8-NEXT:    s_bfe_u32 s4, s4, 0x100000
 ; GFX8-NEXT:    s_bfe_u32 s5, s5, 0x100000
 ; GFX8-NEXT:    s_cmp_lt_u32 s4, s5
 ; GFX8-NEXT:    s_cselect_b32 s4, s4, s5
 ; GFX8-NEXT:    s_add_i32 s1, s1, s4
-; GFX8-NEXT:    s_xor_b32 s4, s9, s16
+; GFX8-NEXT:    s_xor_b32 s4, s9, -1
 ; GFX8-NEXT:    s_bfe_u32 s4, s4, 0x100000
 ; GFX8-NEXT:    s_bfe_u32 s5, s13, 0x100000
 ; GFX8-NEXT:    s_cmp_lt_u32 s4, s5
 ; GFX8-NEXT:    s_cselect_b32 s4, s4, s5
 ; GFX8-NEXT:    s_add_i32 s9, s9, s4
-; GFX8-NEXT:    s_xor_b32 s4, s2, s16
+; GFX8-NEXT:    s_xor_b32 s4, s2, -1
 ; GFX8-NEXT:    s_bfe_u32 s4, s4, 0x100000
 ; GFX8-NEXT:    s_bfe_u32 s5, s6, 0x100000
 ; GFX8-NEXT:    s_cmp_lt_u32 s4, s5
 ; GFX8-NEXT:    s_cselect_b32 s4, s4, s5
 ; GFX8-NEXT:    s_add_i32 s2, s2, s4
-; GFX8-NEXT:    s_xor_b32 s4, s10, s16
+; GFX8-NEXT:    s_xor_b32 s4, s10, -1
 ; GFX8-NEXT:    s_bfe_u32 s4, s4, 0x100000
 ; GFX8-NEXT:    s_bfe_u32 s5, s14, 0x100000
 ; GFX8-NEXT:    s_cmp_lt_u32 s4, s5
 ; GFX8-NEXT:    s_cselect_b32 s4, s4, s5
 ; GFX8-NEXT:    s_add_i32 s10, s10, s4
-; GFX8-NEXT:    s_xor_b32 s4, s3, s16
+; GFX8-NEXT:    s_xor_b32 s4, s3, -1
 ; GFX8-NEXT:    s_bfe_u32 s4, s4, 0x100000
 ; GFX8-NEXT:    s_bfe_u32 s5, s7, 0x100000
 ; GFX8-NEXT:    s_cmp_lt_u32 s4, s5
 ; GFX8-NEXT:    s_cselect_b32 s4, s4, s5
 ; GFX8-NEXT:    s_add_i32 s3, s3, s4
-; GFX8-NEXT:    s_xor_b32 s4, s11, s16
+; GFX8-NEXT:    s_xor_b32 s4, s11, -1
 ; GFX8-NEXT:    s_bfe_u32 s4, s4, 0x100000
 ; GFX8-NEXT:    s_bfe_u32 s5, s15, 0x100000
 ; GFX8-NEXT:    s_cmp_lt_u32 s4, s5
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/udiv.i32.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/udiv.i32.ll
index 25eafb45f9309..54eebc9205796 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/udiv.i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/udiv.i32.ll
@@ -311,7 +311,7 @@ define <2 x i32> @v_udiv_v2i32_pow2k_denom(<2 x i32> %num) {
 ; CGP-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; CGP-NEXT:    s_movk_i32 s4, 0x1000
 ; CGP-NEXT:    v_mov_b32_e32 v2, 0x1000
-; CGP-NEXT:    s_mov_b32 s5, 0xfffff000
+; CGP-NEXT:    s_movk_i32 s5, 0xf000
 ; CGP-NEXT:    v_mov_b32_e32 v3, 0xfffff000
 ; CGP-NEXT:    v_cvt_f32_u32_e32 v4, s4
 ; CGP-NEXT:    v_cvt_f32_u32_e32 v5, v2
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/udiv.i64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/udiv.i64.ll
index e956af93bc6f1..59b4318012e4c 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/udiv.i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/udiv.i64.ll
@@ -185,14 +185,13 @@ declare i32 @llvm.amdgcn.readfirstlane(i32)
 define amdgpu_ps i64 @s_udiv_i64(i64 inreg %num, i64 inreg %den) {
 ; CHECK-LABEL: s_udiv_i64:
 ; CHECK:       ; %bb.0:
-; CHECK-NEXT:    s_or_b64 s[4:5], s[0:1], s[2:3]
-; CHECK-NEXT:    s_mov_b32 s6, 0
-; CHECK-NEXT:    s_mov_b32 s7, -1
-; CHECK-NEXT:    s_and_b64 s[4:5], s[4:5], s[6:7]
-; CHECK-NEXT:    v_cmp_eq_u64_e64 s[4:5], s[4:5], 0
-; CHECK-NEXT:    v_cmp_ne_u32_e64 s[6:7], 0, 1
-; CHECK-NEXT:    s_xor_b64 vcc, s[4:5], s[6:7]
-; CHECK-NEXT:    s_mov_b32 s4, 1
+; CHECK-NEXT:    s_or_b64 s[6:7], s[0:1], s[2:3]
+; CHECK-NEXT:    s_mov_b32 s4, 0
+; CHECK-NEXT:    s_mov_b32 s5, -1
+; CHECK-NEXT:    s_and_b64 s[6:7], s[6:7], s[4:5]
+; CHECK-NEXT:    v_cmp_eq_u64_e64 s[6:7], s[6:7], 0
+; CHECK-NEXT:    v_cmp_ne_u32_e64 s[8:9], 0, 1
+; CHECK-NEXT:    s_xor_b64 vcc, s[6:7], s[8:9]
 ; CHECK-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; CHECK-NEXT:    s_cbranch_vccz BB1_2
 ; CHECK-NEXT:  ; %bb.1:
@@ -324,9 +323,9 @@ define amdgpu_ps i64 @s_udiv_i64(i64 inreg %num, i64 inreg %den) {
 ; CHECK-NEXT:    v_cndmask_b32_e32 v1, v8, v4, vcc
 ; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
 ; CHECK-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; CHECK-NEXT:    s_mov_b32 s4, 0
+; CHECK-NEXT:    s_mov_b32 s5, 0
 ; CHECK-NEXT:  BB1_2: ; %Flow
-; CHECK-NEXT:    s_and_b32 s1, s4, 1
+; CHECK-NEXT:    s_and_b32 s1, s5, 1
 ; CHECK-NEXT:    s_cmp_lg_u32 s1, 0
 ; CHECK-NEXT:    s_cbranch_scc0 BB1_4
 ; CHECK-NEXT:  ; %bb.3:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i32.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i32.ll
index 68a83a91c62f8..f331deea89e54 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i32.ll
@@ -286,7 +286,7 @@ define <2 x i32> @v_urem_v2i32_pow2k_denom(<2 x i32> %num) {
 ; CGP-NEXT:    s_movk_i32 s4, 0x1000
 ; CGP-NEXT:    v_mov_b32_e32 v2, 0x1000
 ; CGP-NEXT:    s_mov_b32 s5, 0x4f7ffffe
-; CGP-NEXT:    s_mov_b32 s6, 0xfffff000
+; CGP-NEXT:    s_movk_i32 s6, 0xf000
 ; CGP-NEXT:    v_cvt_f32_u32_e32 v3, s4
 ; CGP-NEXT:    v_cvt_f32_u32_e32 v4, v2
 ; CGP-NEXT:    v_rcp_iflag_f32_e32 v3, v3
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i64.ll
index 6b9357043b3ca..a01ba29cc3276 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i64.ll
@@ -182,14 +182,13 @@ declare i32 @llvm.amdgcn.readfirstlane(i32)
 define amdgpu_ps i64 @s_urem_i64(i64 inreg %num, i64 inreg %den) {
 ; CHECK-LABEL: s_urem_i64:
 ; CHECK:       ; %bb.0:
-; CHECK-NEXT:    s_or_b64 s[4:5], s[0:1], s[2:3]
-; CHECK-NEXT:    s_mov_b32 s6, 0
-; CHECK-NEXT:    s_mov_b32 s7, -1
-; CHECK-NEXT:    s_and_b64 s[4:5], s[4:5], s[6:7]
-; CHECK-NEXT:    v_cmp_eq_u64_e64 s[4:5], s[4:5], 0
-; CHECK-NEXT:    v_cmp_ne_u32_e64 s[6:7], 0, 1
-; CHECK-NEXT:    s_xor_b64 vcc, s[4:5], s[6:7]
-; CHECK-NEXT:    s_mov_b32 s4, 1
+; CHECK-NEXT:    s_or_b64 s[6:7], s[0:1], s[2:3]
+; CHECK-NEXT:    s_mov_b32 s4, 0
+; CHECK-NEXT:    s_mov_b32 s5, -1
+; CHECK-NEXT:    s_and_b64 s[6:7], s[6:7], s[4:5]
+; CHECK-NEXT:    v_cmp_eq_u64_e64 s[6:7], s[6:7], 0
+; CHECK-NEXT:    v_cmp_ne_u32_e64 s[8:9], 0, 1
+; CHECK-NEXT:    s_xor_b64 vcc, s[6:7], s[8:9]
 ; CHECK-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; CHECK-NEXT:    s_cbranch_vccz BB1_2
 ; CHECK-NEXT:  ; %bb.1:
@@ -320,9 +319,9 @@ define amdgpu_ps i64 @s_urem_i64(i64 inreg %num, i64 inreg %den) {
 ; CHECK-NEXT:    v_cndmask_b32_e32 v0, v3, v5, vcc
 ; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v1
 ; CHECK-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; CHECK-NEXT:    s_mov_b32 s4, 0
+; CHECK-NEXT:    s_mov_b32 s5, 0
 ; CHECK-NEXT:  BB1_2: ; %Flow
-; CHECK-NEXT:    s_and_b32 s1, s4, 1
+; CHECK-NEXT:    s_and_b32 s1, s5, 1
 ; CHECK-NEXT:    s_cmp_lg_u32 s1, 0
 ; CHECK-NEXT:    s_cbranch_scc0 BB1_4
 ; CHECK-NEXT:  ; %bb.3:

From 61ced4b87a80b65d89b2b84418038efde704b9ed Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Sun, 26 Jul 2020 09:26:48 -0400
Subject: [PATCH 0130/1035] GlobalISel: Handle 'n' inline asm constraint

---
 llvm/lib/CodeGen/GlobalISel/InlineAsmLowering.cpp        | 1 +
 .../CodeGen/AMDGPU/GlobalISel/irtranslator-inline-asm.ll | 9 +++++++++
 2 files changed, 10 insertions(+)

diff --git a/llvm/lib/CodeGen/GlobalISel/InlineAsmLowering.cpp b/llvm/lib/CodeGen/GlobalISel/InlineAsmLowering.cpp
index 502f3cb85726c..7acf9c843235e 100644
--- a/llvm/lib/CodeGen/GlobalISel/InlineAsmLowering.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/InlineAsmLowering.cpp
@@ -658,6 +658,7 @@ bool InlineAsmLowering::lowerAsmOperandForConstraint(
   default:
     return false;
   case 'i': // Simple Integer or Relocatable Constant
+  case 'n': // immediate integer with a known value.
     if (ConstantInt *CI = dyn_cast<ConstantInt>(Val)) {
       assert(CI->getBitWidth() <= 64 &&
              "expected immediate to fit into 64-bits");
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-inline-asm.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-inline-asm.ll
index 07b8dced57132..1315f1bc275eb 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-inline-asm.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-inline-asm.ll
@@ -326,4 +326,13 @@ entry:
   ret i32 %asm1
 }
 
+define amdgpu_kernel void @asm_constraint_n_n()  {
+  ; CHECK-LABEL: name: asm_constraint_n_n
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   INLINEASM &"s_trap ${0:n}", 1 /* sideeffect attdialect */, 13 /* imm */, 10
+  ; CHECK:   S_ENDPGM 0
+  tail call void asm sideeffect "s_trap ${0:n}", "n"(i32 10) #1
+  ret void
+}
+
 !0 = !{i32 70}

From 0481e1ae3c17a8faa6c8e319495a51a76b77b76b Mon Sep 17 00:00:00 2001
From: Sanjay Patel <spatel@rotateright.com>
Date: Sun, 26 Jul 2020 09:33:13 -0400
Subject: [PATCH 0131/1035] [InstSimplify] fold integer min/max intrinsics with
 limit constant

---
 llvm/lib/Analysis/InstructionSimplify.cpp     | 21 ++++++++++++++++
 .../InstSimplify/maxmin_intrinsics.ll         | 24 +++++++------------
 2 files changed, 29 insertions(+), 16 deletions(-)

diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp
index 396fc22920cdf..c920fb3f52bea 100644
--- a/llvm/lib/Analysis/InstructionSimplify.cpp
+++ b/llvm/lib/Analysis/InstructionSimplify.cpp
@@ -5254,6 +5254,27 @@ static Value *simplifyBinaryIntrinsic(Function *F, Value *Op0, Value *Op1,
   Intrinsic::ID IID = F->getIntrinsicID();
   Type *ReturnType = F->getReturnType();
   switch (IID) {
+  case Intrinsic::smax:
+  case Intrinsic::smin:
+  case Intrinsic::umax:
+  case Intrinsic::umin: {
+    // Canonicalize constant operand as Op1.
+    if (isa<Constant>(Op0))
+      std::swap(Op0, Op1);
+
+    // TODO: Allow partial undef vector constants.
+    const APInt *C;
+    if (!match(Op1, m_APInt(C)))
+      break;
+
+    if ((IID == Intrinsic::smax && C->isMaxSignedValue()) ||
+        (IID == Intrinsic::smin && C->isMinSignedValue()) ||
+        (IID == Intrinsic::umax && C->isMaxValue()) ||
+        (IID == Intrinsic::umin && C->isMinValue()))
+      return Op1;
+
+    break;
+  }
   case Intrinsic::usub_with_overflow:
   case Intrinsic::ssub_with_overflow:
     // X - X -> { 0, false }
diff --git a/llvm/test/Transforms/InstSimplify/maxmin_intrinsics.ll b/llvm/test/Transforms/InstSimplify/maxmin_intrinsics.ll
index fe940ef7bc179..99a8656b34aa6 100644
--- a/llvm/test/Transforms/InstSimplify/maxmin_intrinsics.ll
+++ b/llvm/test/Transforms/InstSimplify/maxmin_intrinsics.ll
@@ -12,8 +12,7 @@ declare <2 x i8> @llvm.umin.v2i8(<2 x i8>, <2 x i8>)
 
 define i8 @smax_maxval(i8 %x) {
 ; CHECK-LABEL: @smax_maxval(
-; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.smax.i8(i8 [[X:%.*]], i8 127)
-; CHECK-NEXT:    ret i8 [[R]]
+; CHECK-NEXT:    ret i8 127
 ;
   %r = call i8 @llvm.smax.i8(i8 %x, i8 127)
   ret i8 %r
@@ -21,8 +20,7 @@ define i8 @smax_maxval(i8 %x) {
 
 define <2 x i8> @smax_maxval_commute(<2 x i8> %x) {
 ; CHECK-LABEL: @smax_maxval_commute(
-; CHECK-NEXT:    [[R:%.*]] = call <2 x i8> @llvm.smax.v2i8(<2 x i8> <i8 127, i8 127>, <2 x i8> [[X:%.*]])
-; CHECK-NEXT:    ret <2 x i8> [[R]]
+; CHECK-NEXT:    ret <2 x i8> <i8 127, i8 127>
 ;
   %r = call <2 x i8> @llvm.smax.v2i8(<2 x i8> <i8 127, i8 127>, <2 x i8> %x)
   ret <2 x i8> %r
@@ -30,8 +28,7 @@ define <2 x i8> @smax_maxval_commute(<2 x i8> %x) {
 
 define i8 @smin_minval(i8 %x) {
 ; CHECK-LABEL: @smin_minval(
-; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.smin.i8(i8 -128, i8 [[X:%.*]])
-; CHECK-NEXT:    ret i8 [[R]]
+; CHECK-NEXT:    ret i8 -128
 ;
   %r = call i8 @llvm.smin.i8(i8 -128, i8 %x)
   ret i8 %r
@@ -39,8 +36,7 @@ define i8 @smin_minval(i8 %x) {
 
 define <2 x i8> @smin_minval_commute(<2 x i8> %x) {
 ; CHECK-LABEL: @smin_minval_commute(
-; CHECK-NEXT:    [[R:%.*]] = call <2 x i8> @llvm.smin.v2i8(<2 x i8> [[X:%.*]], <2 x i8> <i8 -128, i8 -128>)
-; CHECK-NEXT:    ret <2 x i8> [[R]]
+; CHECK-NEXT:    ret <2 x i8> <i8 -128, i8 -128>
 ;
   %r = call <2 x i8> @llvm.smin.v2i8(<2 x i8> %x, <2 x i8> <i8 -128, i8 -128>)
   ret <2 x i8> %r
@@ -48,8 +44,7 @@ define <2 x i8> @smin_minval_commute(<2 x i8> %x) {
 
 define i8 @umax_maxval(i8 %x) {
 ; CHECK-LABEL: @umax_maxval(
-; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.umax.i8(i8 [[X:%.*]], i8 -1)
-; CHECK-NEXT:    ret i8 [[R]]
+; CHECK-NEXT:    ret i8 -1
 ;
   %r = call i8 @llvm.umax.i8(i8 %x, i8 255)
   ret i8 %r
@@ -57,8 +52,7 @@ define i8 @umax_maxval(i8 %x) {
 
 define <2 x i8> @umax_maxval_commute(<2 x i8> %x) {
 ; CHECK-LABEL: @umax_maxval_commute(
-; CHECK-NEXT:    [[R:%.*]] = call <2 x i8> @llvm.umax.v2i8(<2 x i8> <i8 -1, i8 -1>, <2 x i8> [[X:%.*]])
-; CHECK-NEXT:    ret <2 x i8> [[R]]
+; CHECK-NEXT:    ret <2 x i8> <i8 -1, i8 -1>
 ;
   %r = call <2 x i8> @llvm.umax.v2i8(<2 x i8> <i8 255, i8 255>, <2 x i8> %x)
   ret <2 x i8> %r
@@ -66,8 +60,7 @@ define <2 x i8> @umax_maxval_commute(<2 x i8> %x) {
 
 define i8 @umin_minval(i8 %x) {
 ; CHECK-LABEL: @umin_minval(
-; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.umin.i8(i8 0, i8 [[X:%.*]])
-; CHECK-NEXT:    ret i8 [[R]]
+; CHECK-NEXT:    ret i8 0
 ;
   %r = call i8 @llvm.umin.i8(i8 0, i8 %x)
   ret i8 %r
@@ -75,8 +68,7 @@ define i8 @umin_minval(i8 %x) {
 
 define <2 x i8> @umin_minval_commute(<2 x i8> %x) {
 ; CHECK-LABEL: @umin_minval_commute(
-; CHECK-NEXT:    [[R:%.*]] = call <2 x i8> @llvm.umin.v2i8(<2 x i8> [[X:%.*]], <2 x i8> zeroinitializer)
-; CHECK-NEXT:    ret <2 x i8> [[R]]
+; CHECK-NEXT:    ret <2 x i8> zeroinitializer
 ;
   %r = call <2 x i8> @llvm.umin.v2i8(<2 x i8> %x, <2 x i8> zeroinitializer)
   ret <2 x i8> %r

From 5819159995657091e4e21e538509b2af210fd48d Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Sat, 18 Jul 2020 15:30:59 -0400
Subject: [PATCH 0132/1035] AMDGPU/GlobalISel: Pack constant
 G_BUILD_VECTOR_TRUNCs when selecting

---
 .../AMDGPU/AMDGPUInstructionSelector.cpp      |   21 +-
 .../CodeGen/AMDGPU/GlobalISel/add.v2i16.ll    |   44 +-
 llvm/test/CodeGen/AMDGPU/GlobalISel/ashr.ll   |    3 +-
 .../inst-select-build-vector-trunc.v2s16.mir  |  270 +++
 .../AMDGPU/GlobalISel/llvm.amdgcn.sdot2.ll    |   46 +-
 .../AMDGPU/GlobalISel/llvm.amdgcn.udot2.ll    |   46 +-
 llvm/test/CodeGen/AMDGPU/GlobalISel/lshr.ll   |    3 +-
 .../test/CodeGen/AMDGPU/GlobalISel/saddsat.ll | 1700 ++++++++---------
 llvm/test/CodeGen/AMDGPU/GlobalISel/shl.ll    |    3 +-
 .../test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll | 1536 +++++++--------
 .../test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll |  426 ++---
 llvm/test/CodeGen/AMDGPU/GlobalISel/xnor.ll   |   10 +-
 12 files changed, 2059 insertions(+), 2049 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
index a126ed1daf17f..8bc597664634a 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
@@ -616,11 +616,6 @@ bool AMDGPUInstructionSelector::selectG_UNMERGE_VALUES(MachineInstr &MI) const {
   return true;
 }
 
-static bool isZero(Register Reg, const MachineRegisterInfo &MRI) {
-  int64_t Val;
-  return mi_match(Reg, MRI, m_ICst(Val)) && Val == 0;
-}
-
 bool AMDGPUInstructionSelector::selectG_BUILD_VECTOR_TRUNC(
   MachineInstr &MI) const {
   if (selectImpl(MI, *CoverageInfo))
@@ -645,6 +640,20 @@ bool AMDGPUInstructionSelector::selectG_BUILD_VECTOR_TRUNC(
   const DebugLoc &DL = MI.getDebugLoc();
   MachineBasicBlock *BB = MI.getParent();
 
+  auto ConstSrc1 = getConstantVRegValWithLookThrough(Src1, *MRI, true, true);
+  if (ConstSrc1) {
+    auto ConstSrc0 = getConstantVRegValWithLookThrough(Src0, *MRI, true, true);
+    if (ConstSrc0) {
+      uint32_t Lo16 = static_cast<uint32_t>(ConstSrc0->Value) & 0xffff;
+      uint32_t Hi16 = static_cast<uint32_t>(ConstSrc1->Value) & 0xffff;
+
+      BuildMI(*BB, &MI, DL, TII.get(AMDGPU::S_MOV_B32), Dst)
+        .addImm(Lo16 | (Hi16 << 16));
+      MI.eraseFromParent();
+      return RBI.constrainGenericRegister(Dst, AMDGPU::SReg_32RegClass, *MRI);
+    }
+  }
+
   // TODO: This should probably be a combine somewhere
   // (build_vector_trunc $src0, undef -> copy $src0
   MachineInstr *Src1Def = getDefIgnoringCopies(Src1, *MRI);
@@ -686,7 +695,7 @@ bool AMDGPUInstructionSelector::selectG_BUILD_VECTOR_TRUNC(
   } else if (Shift1) {
     Opc = AMDGPU::S_PACK_LH_B32_B16;
     MI.getOperand(2).setReg(ShiftSrc1);
-  } else if (Shift0 && isZero(Src1, *MRI)) {
+  } else if (Shift0 && ConstSrc1 && ConstSrc1->Value == 0) {
     // build_vector_trunc (lshr $src0, 16), 0 -> s_lshr_b32 $src0, 16
     auto MIB = BuildMI(*BB, &MI, DL, TII.get(AMDGPU::S_LSHR_B32), Dst)
       .addReg(ShiftSrc0)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/add.v2i16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/add.v2i16.ll
index c6c0eb7c4a937..2205bfe3c71d7 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/add.v2i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/add.v2i16.ll
@@ -91,9 +91,8 @@ define <2 x i16> @v_add_v2i16_neg_inline_imm_splat(<2 x i16> %a) {
 ; GFX9-LABEL: v_add_v2i16_neg_inline_imm_splat:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    s_movk_i32 s4, 0xffc0
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s4
-; GFX9-NEXT:    v_pk_add_u16 v0, v0, s4
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0xffc0ffc0
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, v1
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_add_v2i16_neg_inline_imm_splat:
@@ -113,8 +112,8 @@ define <2 x i16> @v_add_v2i16_neg_inline_imm_lo(<2 x i16> %a) {
 ; GFX9-LABEL: v_add_v2i16_neg_inline_imm_lo:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, 0xffffffc0, 4
-; GFX9-NEXT:    v_pk_add_u16 v0, v0, s4
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0x4ffc0
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, v1
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_add_v2i16_neg_inline_imm_lo:
@@ -133,8 +132,8 @@ define <2 x i16> @v_add_v2i16_neg_inline_imm_hi(<2 x i16> %a) {
 ; GFX9-LABEL: v_add_v2i16_neg_inline_imm_hi:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, 4, 0xffffffc0
-; GFX9-NEXT:    v_pk_add_u16 v0, v0, s4
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0xffc00004
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, v1
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_add_v2i16_neg_inline_imm_hi:
@@ -152,13 +151,10 @@ define <2 x i16> @v_add_v2i16_neg_inline_imm_hi(<2 x i16> %a) {
 define amdgpu_ps i32 @s_add_v2i16_neg_inline_imm_splat(<2 x i16> inreg %a) {
 ; GFX9-LABEL: s_add_v2i16_neg_inline_imm_splat:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_movk_i32 s1, 0xffc0
-; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s1
-; GFX9-NEXT:    s_lshr_b32 s2, s0, 16
-; GFX9-NEXT:    s_lshr_b32 s3, s1, 16
-; GFX9-NEXT:    s_add_i32 s0, s0, s1
-; GFX9-NEXT:    s_add_i32 s2, s2, s3
-; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s2
+; GFX9-NEXT:    s_lshr_b32 s1, s0, 16
+; GFX9-NEXT:    s_add_i32 s0, s0, 0xffc0ffc0
+; GFX9-NEXT:    s_add_i32 s1, s1, 0xffc0
+; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s1
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX8-LABEL: s_add_v2i16_neg_inline_imm_splat:
@@ -182,12 +178,10 @@ define amdgpu_ps i32 @s_add_v2i16_neg_inline_imm_splat(<2 x i16> inreg %a) {
 define amdgpu_ps i32 @s_add_v2i16_neg_inline_imm_lo(<2 x i16> inreg %a) {
 ; GFX9-LABEL: s_add_v2i16_neg_inline_imm_lo:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_pack_ll_b32_b16 s1, 0xffffffc0, 4
-; GFX9-NEXT:    s_lshr_b32 s2, s0, 16
-; GFX9-NEXT:    s_lshr_b32 s3, s1, 16
-; GFX9-NEXT:    s_add_i32 s0, s0, s1
-; GFX9-NEXT:    s_add_i32 s2, s2, s3
-; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s2
+; GFX9-NEXT:    s_lshr_b32 s1, s0, 16
+; GFX9-NEXT:    s_add_i32 s0, s0, 0x4ffc0
+; GFX9-NEXT:    s_add_i32 s1, s1, 4
+; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s1
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX8-LABEL: s_add_v2i16_neg_inline_imm_lo:
@@ -210,12 +204,10 @@ define amdgpu_ps i32 @s_add_v2i16_neg_inline_imm_lo(<2 x i16> inreg %a) {
 define amdgpu_ps i32 @s_add_v2i16_neg_inline_imm_hi(<2 x i16> inreg %a) {
 ; GFX9-LABEL: s_add_v2i16_neg_inline_imm_hi:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_pack_ll_b32_b16 s1, 4, 0xffffffc0
-; GFX9-NEXT:    s_lshr_b32 s2, s0, 16
-; GFX9-NEXT:    s_lshr_b32 s3, s1, 16
-; GFX9-NEXT:    s_add_i32 s0, s0, s1
-; GFX9-NEXT:    s_add_i32 s2, s2, s3
-; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s2
+; GFX9-NEXT:    s_lshr_b32 s1, s0, 16
+; GFX9-NEXT:    s_add_i32 s0, s0, 0xffc00004
+; GFX9-NEXT:    s_add_i32 s1, s1, 0xffc0
+; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s1
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX8-LABEL: s_add_v2i16_neg_inline_imm_hi:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/ashr.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/ashr.ll
index 71ee562f0ecc2..c1896f81ef296 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/ashr.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/ashr.ll
@@ -522,8 +522,7 @@ define <2 x i16> @v_ashr_v2i16_15(<2 x i16> %value) {
 ; GFX9-LABEL: v_ashr_v2i16_15:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, 15, 15
-; GFX9-NEXT:    v_pk_ashrrev_i16 v0, s4, v0
+; GFX9-NEXT:    v_pk_ashrrev_i16 v0, 15, v0 op_sel_hi:[0,1]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %result = ashr <2 x i16> %value, <i16 15, i16 15>
   ret <2 x i16> %result
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-build-vector-trunc.v2s16.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-build-vector-trunc.v2s16.mir
index c380d3c77defc..056ea79a98988 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-build-vector-trunc.v2s16.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-build-vector-trunc.v2s16.mir
@@ -430,3 +430,273 @@ body: |
     %5:sgpr(<2 x s16>) = G_BUILD_VECTOR_TRUNC %3, %4
     S_ENDPGM 0, implicit %5
 ...
+
+---
+name: test_build_vector_trunc_s_v2s16_constant_constant
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+
+    ; GFX9-LABEL: name: test_build_vector_trunc_s_v2s16_constant_constant
+    ; GFX9: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 29884539
+    ; GFX9: S_ENDPGM 0, implicit [[S_MOV_B32_]]
+    %0:sgpr(s32) = G_CONSTANT i32 123
+    %1:sgpr(s32) = G_CONSTANT i32 456
+    %2:sgpr(<2 x s16>) = G_BUILD_VECTOR_TRUNC %0, %1
+    S_ENDPGM 0, implicit %2
+...
+
+---
+name: test_build_vector_trunc_s_v2s16_constant_impdef
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+
+    ; GFX9-LABEL: name: test_build_vector_trunc_s_v2s16_constant_impdef
+    ; GFX9: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 123
+    ; GFX9: S_ENDPGM 0, implicit [[S_MOV_B32_]]
+    %0:sgpr(s32) = G_CONSTANT i32 123
+    %1:sgpr(s32) = G_IMPLICIT_DEF
+    %2:sgpr(<2 x s16>) = G_BUILD_VECTOR_TRUNC %0, %1
+    S_ENDPGM 0, implicit %2
+...
+
+---
+name: test_build_vector_trunc_s_v2s16_impdef_constant
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+
+    ; GFX9-LABEL: name: test_build_vector_trunc_s_v2s16_impdef_constant
+    ; GFX9: [[DEF:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+    ; GFX9: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 123
+    ; GFX9: [[S_PACK_LL_B32_B16_:%[0-9]+]]:sreg_32 = S_PACK_LL_B32_B16 [[DEF]], [[S_MOV_B32_]]
+    ; GFX9: S_ENDPGM 0, implicit [[S_PACK_LL_B32_B16_]]
+    %0:sgpr(s32) = G_IMPLICIT_DEF
+    %1:sgpr(s32) = G_CONSTANT i32 123
+    %2:sgpr(<2 x s16>) = G_BUILD_VECTOR_TRUNC %0, %1
+    S_ENDPGM 0, implicit %2
+...
+
+---
+name: test_build_vector_trunc_s_v2s16_impdef_impdef
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+
+    ; GFX9-LABEL: name: test_build_vector_trunc_s_v2s16_impdef_impdef
+    ; GFX9: [[DEF:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+    ; GFX9: S_ENDPGM 0, implicit [[DEF]]
+    %0:sgpr(s32) = G_IMPLICIT_DEF
+    %1:sgpr(s32) = G_IMPLICIT_DEF
+    %2:sgpr(<2 x s16>) = G_BUILD_VECTOR_TRUNC %0, %1
+    S_ENDPGM 0, implicit %2
+...
+
+---
+name: test_build_vector_trunc_s_v2s16_zext_constant_zext_constant
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+
+    ; GFX9-LABEL: name: test_build_vector_trunc_s_v2s16_zext_constant_zext_constant
+    ; GFX9: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 29884539
+    ; GFX9: S_ENDPGM 0, implicit [[S_MOV_B32_]]
+    %0:sgpr(s16) = G_CONSTANT i16 123
+    %1:sgpr(s16) = G_CONSTANT i16 456
+    %2:sgpr(s32) = G_ZEXT %0
+    %3:sgpr(s32) = G_ZEXT %1
+    %4:sgpr(<2 x s16>) = G_BUILD_VECTOR_TRUNC %2, %3
+    S_ENDPGM 0, implicit %4
+...
+
+---
+name: test_build_vector_trunc_s_v2s16_zext_impdef_zext_constant
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+
+    ; GFX9-LABEL: name: test_build_vector_trunc_s_v2s16_zext_impdef_zext_constant
+    ; GFX9: [[DEF:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+    ; GFX9: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 123
+    ; GFX9: [[S_BFE_U32_:%[0-9]+]]:sreg_32 = S_BFE_U32 [[DEF]], 1048576, implicit-def $scc
+    ; GFX9: [[S_BFE_U32_1:%[0-9]+]]:sreg_32 = S_BFE_U32 [[S_MOV_B32_]], 1048576, implicit-def $scc
+    ; GFX9: [[S_PACK_LL_B32_B16_:%[0-9]+]]:sreg_32 = S_PACK_LL_B32_B16 [[S_BFE_U32_]], [[S_BFE_U32_1]]
+    ; GFX9: S_ENDPGM 0, implicit [[S_PACK_LL_B32_B16_]]
+    %0:sgpr(s16) = G_IMPLICIT_DEF
+    %1:sgpr(s16) = G_CONSTANT i16 123
+    %2:sgpr(s32) = G_ZEXT %0
+    %3:sgpr(s32) = G_ZEXT %1
+    %4:sgpr(<2 x s16>) = G_BUILD_VECTOR_TRUNC %2, %3
+    S_ENDPGM 0, implicit %4
+...
+
+---
+name: test_build_vector_trunc_s_v2s16_sext_constant_sext_constant
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+
+    ; GFX9-LABEL: name: test_build_vector_trunc_s_v2s16_sext_constant_sext_constant
+    ; GFX9: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4294836208
+    ; GFX9: S_ENDPGM 0, implicit [[S_MOV_B32_]]
+    %0:sgpr(s16) = G_CONSTANT i16 -16
+    %1:sgpr(s16) = G_CONSTANT i16 -3
+    %2:sgpr(s32) = G_SEXT %0
+    %3:sgpr(s32) = G_SEXT %1
+    %4:sgpr(<2 x s16>) = G_BUILD_VECTOR_TRUNC %2, %3
+    S_ENDPGM 0, implicit %4
+...
+
+---
+name: test_build_vector_trunc_s_v2s16_anyext_constant_anyext_constant
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+
+    ; GFX9-LABEL: name: test_build_vector_trunc_s_v2s16_anyext_constant_anyext_constant
+    ; GFX9: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 123
+    ; GFX9: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 456
+    ; GFX9: [[S_PACK_LL_B32_B16_:%[0-9]+]]:sreg_32 = S_PACK_LL_B32_B16 [[S_MOV_B32_]], [[S_MOV_B32_1]]
+    ; GFX9: S_ENDPGM 0, implicit [[S_PACK_LL_B32_B16_]]
+    %0:sgpr(s16) = G_CONSTANT i16 123
+    %1:sgpr(s16) = G_CONSTANT i16 456
+    %2:sgpr(s32) = G_ANYEXT %0
+    %3:sgpr(s32) = G_ANYEXT %1
+    %4:sgpr(<2 x s16>) = G_BUILD_VECTOR_TRUNC %2, %3
+    S_ENDPGM 0, implicit %4
+...
+
+---
+name: test_build_vector_trunc_s_v2s16_anyext_impdef_anyext_constant
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+
+    ; GFX9-LABEL: name: test_build_vector_trunc_s_v2s16_anyext_impdef_anyext_constant
+    ; GFX9: [[DEF:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+    ; GFX9: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 123
+    ; GFX9: [[S_PACK_LL_B32_B16_:%[0-9]+]]:sreg_32 = S_PACK_LL_B32_B16 [[DEF]], [[S_MOV_B32_]]
+    ; GFX9: S_ENDPGM 0, implicit [[S_PACK_LL_B32_B16_]]
+    %0:sgpr(s16) = G_IMPLICIT_DEF
+    %1:sgpr(s16) = G_CONSTANT i16 123
+    %2:sgpr(s32) = G_ANYEXT %0
+    %3:sgpr(s32) = G_ANYEXT %1
+    %4:sgpr(<2 x s16>) = G_BUILD_VECTOR_TRUNC %2, %3
+    S_ENDPGM 0, implicit %4
+...
+
+---
+name: test_build_vector_trunc_s_v2s16_var_constant
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0
+
+    ; GFX9-LABEL: name: test_build_vector_trunc_s_v2s16_var_constant
+    ; GFX9: liveins: $sgpr0
+    ; GFX9: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+    ; GFX9: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 456
+    ; GFX9: [[S_PACK_LL_B32_B16_:%[0-9]+]]:sreg_32 = S_PACK_LL_B32_B16 [[COPY]], [[S_MOV_B32_]]
+    ; GFX9: S_ENDPGM 0, implicit [[S_PACK_LL_B32_B16_]]
+    %0:sgpr(s32) = COPY $sgpr0
+    %1:sgpr(s32) = G_CONSTANT i32 456
+    %2:sgpr(<2 x s16>) = G_BUILD_VECTOR_TRUNC %0, %1
+    S_ENDPGM 0, implicit %2
+...
+
+---
+name: test_build_vector_trunc_s_v2s16_constant_var
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0
+
+    ; GFX9-LABEL: name: test_build_vector_trunc_s_v2s16_constant_var
+    ; GFX9: liveins: $sgpr0
+    ; GFX9: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 456
+    ; GFX9: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+    ; GFX9: [[S_PACK_LL_B32_B16_:%[0-9]+]]:sreg_32 = S_PACK_LL_B32_B16 [[S_MOV_B32_]], [[COPY]]
+    ; GFX9: S_ENDPGM 0, implicit [[S_PACK_LL_B32_B16_]]
+    %0:sgpr(s32) = G_CONSTANT i32 456
+    %1:sgpr(s32) = COPY $sgpr0
+    %2:sgpr(<2 x s16>) = G_BUILD_VECTOR_TRUNC %0, %1
+    S_ENDPGM 0, implicit %2
+...
+
+---
+name: test_build_vector_trunc_s_v2s16_var_0
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0
+
+    ; GFX9-LABEL: name: test_build_vector_trunc_s_v2s16_var_0
+    ; GFX9: liveins: $sgpr0
+    ; GFX9: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+    ; GFX9: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; GFX9: [[S_PACK_LL_B32_B16_:%[0-9]+]]:sreg_32 = S_PACK_LL_B32_B16 [[COPY]], [[S_MOV_B32_]]
+    ; GFX9: S_ENDPGM 0, implicit [[S_PACK_LL_B32_B16_]]
+    %0:sgpr(s32) = COPY $sgpr0
+    %1:sgpr(s32) = G_CONSTANT i32 0
+    %2:sgpr(<2 x s16>) = G_BUILD_VECTOR_TRUNC %0, %1
+    S_ENDPGM 0, implicit %2
+...
+
+---
+name: test_build_vector_trunc_s_v2s16_0_var
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0
+
+    ; GFX9-LABEL: name: test_build_vector_trunc_s_v2s16_0_var
+    ; GFX9: liveins: $sgpr0
+    ; GFX9: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; GFX9: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+    ; GFX9: [[S_PACK_LL_B32_B16_:%[0-9]+]]:sreg_32 = S_PACK_LL_B32_B16 [[S_MOV_B32_]], [[COPY]]
+    ; GFX9: S_ENDPGM 0, implicit [[S_PACK_LL_B32_B16_]]
+    %0:sgpr(s32) = G_CONSTANT i32 0
+    %1:sgpr(s32) = COPY $sgpr0
+    %2:sgpr(<2 x s16>) = G_BUILD_VECTOR_TRUNC %0, %1
+    S_ENDPGM 0, implicit %2
+...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.sdot2.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.sdot2.ll
index fdcf0f1515f91..172656f08aefb 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.sdot2.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.sdot2.ll
@@ -82,24 +82,21 @@ define i32 @v_sdot2_inline_literal_a(<2 x i16> %b, i32 %c) {
 ; GFX906-LABEL: v_sdot2_inline_literal_a:
 ; GFX906:       ; %bb.0:
 ; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX906-NEXT:    s_pack_ll_b32_b16 s4, 4, 4
-; GFX906-NEXT:    v_dot2_i32_i16 v0, s4, v0, v1
+; GFX906-NEXT:    v_dot2_i32_i16 v0, 4, v0, v1 op_sel_hi:[0,1,1]
 ; GFX906-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX908-LABEL: v_sdot2_inline_literal_a:
 ; GFX908:       ; %bb.0:
 ; GFX908-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT:    s_pack_ll_b32_b16 s4, 4, 4
-; GFX908-NEXT:    v_dot2_i32_i16 v0, s4, v0, v1
+; GFX908-NEXT:    v_dot2_i32_i16 v0, 4, v0, v1 op_sel_hi:[0,1,1]
 ; GFX908-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_sdot2_inline_literal_a:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, 4, 4
+; GFX10-NEXT:    v_dot2_i32_i16 v0, 4, v0, v1 op_sel_hi:[0,1,1]
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_dot2_i32_i16 v0, s4, v0, v1
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %r = call i32 @llvm.amdgcn.sdot2(<2 x i16> <i16 4, i16 4>, <2 x i16> %b, i32 %c, i1 false)
   ret i32 %r
@@ -109,24 +106,21 @@ define i32 @v_sdot2_inline_literal_b(<2 x i16> %a, i32 %c) {
 ; GFX906-LABEL: v_sdot2_inline_literal_b:
 ; GFX906:       ; %bb.0:
 ; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX906-NEXT:    s_pack_ll_b32_b16 s4, 4, 4
-; GFX906-NEXT:    v_dot2_i32_i16 v0, v0, s4, v1
+; GFX906-NEXT:    v_dot2_i32_i16 v0, v0, 4, v1 op_sel_hi:[1,0,1]
 ; GFX906-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX908-LABEL: v_sdot2_inline_literal_b:
 ; GFX908:       ; %bb.0:
 ; GFX908-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT:    s_pack_ll_b32_b16 s4, 4, 4
-; GFX908-NEXT:    v_dot2_i32_i16 v0, v0, s4, v1
+; GFX908-NEXT:    v_dot2_i32_i16 v0, v0, 4, v1 op_sel_hi:[1,0,1]
 ; GFX908-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_sdot2_inline_literal_b:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, 4, 4
+; GFX10-NEXT:    v_dot2_i32_i16 v0, v0, 4, v1 op_sel_hi:[1,0,1]
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_dot2_i32_i16 v0, v0, s4, v1
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %r = call i32 @llvm.amdgcn.sdot2(<2 x i16> %a, <2 x i16> <i16 4, i16 4>, i32 %c, i1 false)
   ret i32 %r
@@ -136,29 +130,21 @@ define i32 @v_sdot2_inline_literal_a_b(<2 x i16> %a, i32 %c) {
 ; GFX906-LABEL: v_sdot2_inline_literal_a_b:
 ; GFX906:       ; %bb.0:
 ; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX906-NEXT:    s_pack_ll_b32_b16 s5, 4, 4
-; GFX906-NEXT:    s_pack_ll_b32_b16 s4, 8, 8
-; GFX906-NEXT:    v_mov_b32_e32 v0, s5
-; GFX906-NEXT:    v_dot2_i32_i16 v0, s4, v0, v1
+; GFX906-NEXT:    v_dot2_i32_i16 v0, 8, 4, v1 op_sel_hi:[0,0,1]
 ; GFX906-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX908-LABEL: v_sdot2_inline_literal_a_b:
 ; GFX908:       ; %bb.0:
 ; GFX908-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT:    s_pack_ll_b32_b16 s5, 4, 4
-; GFX908-NEXT:    s_pack_ll_b32_b16 s4, 8, 8
-; GFX908-NEXT:    v_mov_b32_e32 v0, s5
-; GFX908-NEXT:    v_dot2_i32_i16 v0, s4, v0, v1
+; GFX908-NEXT:    v_dot2_i32_i16 v0, 8, 4, v1 op_sel_hi:[0,0,1]
 ; GFX908-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_sdot2_inline_literal_a_b:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, 8, 8
-; GFX10-NEXT:    s_pack_ll_b32_b16 s5, 4, 4
+; GFX10-NEXT:    v_dot2_i32_i16 v0, 8, 4, v1 op_sel_hi:[0,0,1]
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_dot2_i32_i16 v0, s4, s5, v1
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %r = call i32 @llvm.amdgcn.sdot2(<2 x i16> <i16 8, i16 8>, <2 x i16> <i16 4, i16 4>, i32 %c, i1 false)
   ret i32 %r
@@ -168,29 +154,21 @@ define i32 @v_sdot2_inline_literal_a_b_c() {
 ; GFX906-LABEL: v_sdot2_inline_literal_a_b_c:
 ; GFX906:       ; %bb.0:
 ; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX906-NEXT:    s_pack_ll_b32_b16 s5, 4, 4
-; GFX906-NEXT:    s_pack_ll_b32_b16 s4, 8, 8
-; GFX906-NEXT:    v_mov_b32_e32 v0, s5
-; GFX906-NEXT:    v_dot2_i32_i16 v0, s4, v0, 8
+; GFX906-NEXT:    v_dot2_i32_i16 v0, 8, 4, 8 op_sel_hi:[0,0,1]
 ; GFX906-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX908-LABEL: v_sdot2_inline_literal_a_b_c:
 ; GFX908:       ; %bb.0:
 ; GFX908-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT:    s_pack_ll_b32_b16 s5, 4, 4
-; GFX908-NEXT:    s_pack_ll_b32_b16 s4, 8, 8
-; GFX908-NEXT:    v_mov_b32_e32 v0, s5
-; GFX908-NEXT:    v_dot2_i32_i16 v0, s4, v0, 8
+; GFX908-NEXT:    v_dot2_i32_i16 v0, 8, 4, 8 op_sel_hi:[0,0,1]
 ; GFX908-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_sdot2_inline_literal_a_b_c:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, 8, 8
-; GFX10-NEXT:    s_pack_ll_b32_b16 s5, 4, 4
+; GFX10-NEXT:    v_dot2_i32_i16 v0, 8, 4, 8 op_sel_hi:[0,0,1]
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_dot2_i32_i16 v0, s4, s5, 8
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %r = call i32 @llvm.amdgcn.sdot2(<2 x i16> <i16 8, i16 8>, <2 x i16> <i16 4, i16 4>, i32 8, i1 false)
   ret i32 %r
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.udot2.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.udot2.ll
index d285ee132cc21..976536c728838 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.udot2.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.udot2.ll
@@ -82,24 +82,21 @@ define i32 @v_udot2_inline_literal_a(<2 x i16> %b, i32 %c) {
 ; GFX906-LABEL: v_udot2_inline_literal_a:
 ; GFX906:       ; %bb.0:
 ; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX906-NEXT:    s_pack_ll_b32_b16 s4, 4, 4
-; GFX906-NEXT:    v_dot2_u32_u16 v0, s4, v0, v1
+; GFX906-NEXT:    v_dot2_u32_u16 v0, 4, v0, v1 op_sel_hi:[0,1,1]
 ; GFX906-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX908-LABEL: v_udot2_inline_literal_a:
 ; GFX908:       ; %bb.0:
 ; GFX908-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT:    s_pack_ll_b32_b16 s4, 4, 4
-; GFX908-NEXT:    v_dot2_u32_u16 v0, s4, v0, v1
+; GFX908-NEXT:    v_dot2_u32_u16 v0, 4, v0, v1 op_sel_hi:[0,1,1]
 ; GFX908-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_udot2_inline_literal_a:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, 4, 4
+; GFX10-NEXT:    v_dot2_u32_u16 v0, 4, v0, v1 op_sel_hi:[0,1,1]
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_dot2_u32_u16 v0, s4, v0, v1
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %r = call i32 @llvm.amdgcn.udot2(<2 x i16> <i16 4, i16 4>, <2 x i16> %b, i32 %c, i1 false)
   ret i32 %r
@@ -109,24 +106,21 @@ define i32 @v_udot2_inline_literal_b(<2 x i16> %a, i32 %c) {
 ; GFX906-LABEL: v_udot2_inline_literal_b:
 ; GFX906:       ; %bb.0:
 ; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX906-NEXT:    s_pack_ll_b32_b16 s4, 4, 4
-; GFX906-NEXT:    v_dot2_u32_u16 v0, v0, s4, v1
+; GFX906-NEXT:    v_dot2_u32_u16 v0, v0, 4, v1 op_sel_hi:[1,0,1]
 ; GFX906-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX908-LABEL: v_udot2_inline_literal_b:
 ; GFX908:       ; %bb.0:
 ; GFX908-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT:    s_pack_ll_b32_b16 s4, 4, 4
-; GFX908-NEXT:    v_dot2_u32_u16 v0, v0, s4, v1
+; GFX908-NEXT:    v_dot2_u32_u16 v0, v0, 4, v1 op_sel_hi:[1,0,1]
 ; GFX908-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_udot2_inline_literal_b:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, 4, 4
+; GFX10-NEXT:    v_dot2_u32_u16 v0, v0, 4, v1 op_sel_hi:[1,0,1]
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_dot2_u32_u16 v0, v0, s4, v1
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %r = call i32 @llvm.amdgcn.udot2(<2 x i16> %a, <2 x i16> <i16 4, i16 4>, i32 %c, i1 false)
   ret i32 %r
@@ -136,29 +130,21 @@ define i32 @v_udot2_inline_literal_a_b(<2 x i16> %a, i32 %c) {
 ; GFX906-LABEL: v_udot2_inline_literal_a_b:
 ; GFX906:       ; %bb.0:
 ; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX906-NEXT:    s_pack_ll_b32_b16 s5, 4, 4
-; GFX906-NEXT:    s_pack_ll_b32_b16 s4, 8, 8
-; GFX906-NEXT:    v_mov_b32_e32 v0, s5
-; GFX906-NEXT:    v_dot2_u32_u16 v0, s4, v0, v1
+; GFX906-NEXT:    v_dot2_u32_u16 v0, 8, 4, v1 op_sel_hi:[0,0,1]
 ; GFX906-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX908-LABEL: v_udot2_inline_literal_a_b:
 ; GFX908:       ; %bb.0:
 ; GFX908-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT:    s_pack_ll_b32_b16 s5, 4, 4
-; GFX908-NEXT:    s_pack_ll_b32_b16 s4, 8, 8
-; GFX908-NEXT:    v_mov_b32_e32 v0, s5
-; GFX908-NEXT:    v_dot2_u32_u16 v0, s4, v0, v1
+; GFX908-NEXT:    v_dot2_u32_u16 v0, 8, 4, v1 op_sel_hi:[0,0,1]
 ; GFX908-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_udot2_inline_literal_a_b:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, 8, 8
-; GFX10-NEXT:    s_pack_ll_b32_b16 s5, 4, 4
+; GFX10-NEXT:    v_dot2_u32_u16 v0, 8, 4, v1 op_sel_hi:[0,0,1]
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_dot2_u32_u16 v0, s4, s5, v1
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %r = call i32 @llvm.amdgcn.udot2(<2 x i16> <i16 8, i16 8>, <2 x i16> <i16 4, i16 4>, i32 %c, i1 false)
   ret i32 %r
@@ -168,29 +154,21 @@ define i32 @v_udot2_inline_literal_a_b_c() {
 ; GFX906-LABEL: v_udot2_inline_literal_a_b_c:
 ; GFX906:       ; %bb.0:
 ; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX906-NEXT:    s_pack_ll_b32_b16 s5, 4, 4
-; GFX906-NEXT:    s_pack_ll_b32_b16 s4, 8, 8
-; GFX906-NEXT:    v_mov_b32_e32 v0, s5
-; GFX906-NEXT:    v_dot2_u32_u16 v0, s4, v0, 8
+; GFX906-NEXT:    v_dot2_u32_u16 v0, 8, 4, 8 op_sel_hi:[0,0,1]
 ; GFX906-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX908-LABEL: v_udot2_inline_literal_a_b_c:
 ; GFX908:       ; %bb.0:
 ; GFX908-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT:    s_pack_ll_b32_b16 s5, 4, 4
-; GFX908-NEXT:    s_pack_ll_b32_b16 s4, 8, 8
-; GFX908-NEXT:    v_mov_b32_e32 v0, s5
-; GFX908-NEXT:    v_dot2_u32_u16 v0, s4, v0, 8
+; GFX908-NEXT:    v_dot2_u32_u16 v0, 8, 4, 8 op_sel_hi:[0,0,1]
 ; GFX908-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_udot2_inline_literal_a_b_c:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, 8, 8
-; GFX10-NEXT:    s_pack_ll_b32_b16 s5, 4, 4
+; GFX10-NEXT:    v_dot2_u32_u16 v0, 8, 4, 8 op_sel_hi:[0,0,1]
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_dot2_u32_u16 v0, s4, s5, 8
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %r = call i32 @llvm.amdgcn.udot2(<2 x i16> <i16 8, i16 8>, <2 x i16> <i16 4, i16 4>, i32 8, i1 false)
   ret i32 %r
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/lshr.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/lshr.ll
index 9d82396bbc364..ea2631cbcb294 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/lshr.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/lshr.ll
@@ -533,8 +533,7 @@ define <2 x i16> @v_lshr_v2i16_15(<2 x i16> %value) {
 ; GFX9-LABEL: v_lshr_v2i16_15:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, 15, 15
-; GFX9-NEXT:    v_pk_lshrrev_b16 v0, s4, v0
+; GFX9-NEXT:    v_pk_lshrrev_b16 v0, 15, v0 op_sel_hi:[0,1]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %result = lshr <2 x i16> %value, <i16 15, i16 15>
   ret <2 x i16> %result
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/saddsat.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/saddsat.ll
index b2e7f1ea326f6..ba672883fa562 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/saddsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/saddsat.ll
@@ -4527,15 +4527,12 @@ define <2 x i16> @v_saddsat_v2i16(<2 x i16> %lhs, <2 x i16> %rhs) {
 ; GFX9-LABEL: v_saddsat_v2i16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    s_movk_i32 s5, 0x8000
-; GFX9-NEXT:    s_pack_ll_b32_b16 s6, 0, 0
-; GFX9-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX9-NEXT:    s_pack_ll_b32_b16 s5, s5, s5
-; GFX9-NEXT:    v_pk_min_i16 v3, v0, s6
-; GFX9-NEXT:    v_pk_sub_i16 v3, s5, v3
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s4
-; GFX9-NEXT:    v_pk_max_i16 v2, v0, s6
-; GFX9-NEXT:    v_pk_sub_i16 v2, s4, v2
+; GFX9-NEXT:    v_pk_max_i16 v2, v0, 0
+; GFX9-NEXT:    v_mov_b32_e32 v3, 0x7fff7fff
+; GFX9-NEXT:    v_pk_sub_i16 v2, v3, v2
+; GFX9-NEXT:    v_pk_min_i16 v3, v0, 0
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0x80008000
+; GFX9-NEXT:    v_pk_sub_i16 v3, v4, v3
 ; GFX9-NEXT:    v_pk_max_i16 v1, v3, v1
 ; GFX9-NEXT:    v_pk_min_i16 v1, v1, v2
 ; GFX9-NEXT:    v_pk_add_u16 v0, v0, v1
@@ -4545,16 +4542,11 @@ define <2 x i16> @v_saddsat_v2i16(<2 x i16> %lhs, <2 x i16> %rhs) {
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, 0, 0
-; GFX10-NEXT:    s_movk_i32 s5, 0x8000
-; GFX10-NEXT:    v_pk_min_i16 v2, v0, s4
-; GFX10-NEXT:    s_pack_ll_b32_b16 s5, s5, s5
-; GFX10-NEXT:    v_pk_max_i16 v3, v0, s4
-; GFX10-NEXT:    s_movk_i32 s6, 0x7fff
+; GFX10-NEXT:    v_pk_min_i16 v2, v0, 0
+; GFX10-NEXT:    v_pk_max_i16 v3, v0, 0
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_pk_sub_i16 v2, s5, v2
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s6, s6
-; GFX10-NEXT:    v_pk_sub_i16 v3, s4, v3
+; GFX10-NEXT:    v_pk_sub_i16 v2, 0x80008000, v2
+; GFX10-NEXT:    v_pk_sub_i16 v3, 0x7fff7fff, v3
 ; GFX10-NEXT:    v_pk_max_i16 v1, v2, v1
 ; GFX10-NEXT:    v_pk_min_i16 v1, v1, v3
 ; GFX10-NEXT:    v_pk_add_u16 v0, v0, v1
@@ -4650,53 +4642,45 @@ define amdgpu_ps i32 @s_saddsat_v2i16(<2 x i16> inreg %lhs, <2 x i16> inreg %rhs
 ;
 ; GFX9-LABEL: s_saddsat_v2i16:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, 0, 0
-; GFX9-NEXT:    s_sext_i32_i16 s7, s4
-; GFX9-NEXT:    s_sext_i32_i16 s5, s0
-; GFX9-NEXT:    s_ashr_i32 s6, s0, 16
-; GFX9-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s5, s7
-; GFX9-NEXT:    s_cselect_b32 s8, s5, s7
-; GFX9-NEXT:    s_cmp_gt_i32 s6, s4
-; GFX9-NEXT:    s_movk_i32 s2, 0x7fff
-; GFX9-NEXT:    s_cselect_b32 s9, s6, s4
-; GFX9-NEXT:    s_pack_ll_b32_b16 s8, s8, s9
-; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s2
-; GFX9-NEXT:    s_lshr_b32 s9, s2, 16
-; GFX9-NEXT:    s_lshr_b32 s10, s8, 16
-; GFX9-NEXT:    s_sub_i32 s2, s2, s8
-; GFX9-NEXT:    s_sub_i32 s8, s9, s10
-; GFX9-NEXT:    s_cmp_lt_i32 s5, s7
-; GFX9-NEXT:    s_cselect_b32 s5, s5, s7
-; GFX9-NEXT:    s_cmp_lt_i32 s6, s4
-; GFX9-NEXT:    s_movk_i32 s3, 0x8000
-; GFX9-NEXT:    s_cselect_b32 s4, s6, s4
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s5, s4
-; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s3, s3
-; GFX9-NEXT:    s_lshr_b32 s5, s3, 16
-; GFX9-NEXT:    s_lshr_b32 s6, s4, 16
-; GFX9-NEXT:    s_sub_i32 s3, s3, s4
-; GFX9-NEXT:    s_sub_i32 s4, s5, s6
-; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s3, s4
-; GFX9-NEXT:    s_sext_i32_i16 s4, s3
-; GFX9-NEXT:    s_sext_i32_i16 s5, s1
-; GFX9-NEXT:    s_ashr_i32 s3, s3, 16
-; GFX9-NEXT:    s_ashr_i32 s1, s1, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s4, s5
-; GFX9-NEXT:    s_cselect_b32 s4, s4, s5
-; GFX9-NEXT:    s_cmp_gt_i32 s3, s1
-; GFX9-NEXT:    s_cselect_b32 s1, s3, s1
-; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s4, s1
-; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s8
-; GFX9-NEXT:    s_sext_i32_i16 s3, s1
-; GFX9-NEXT:    s_sext_i32_i16 s4, s2
-; GFX9-NEXT:    s_ashr_i32 s1, s1, 16
+; GFX9-NEXT:    s_sext_i32_i16 s2, s0
+; GFX9-NEXT:    s_ashr_i32 s3, s0, 16
+; GFX9-NEXT:    s_sext_i32_i16 s4, 0
+; GFX9-NEXT:    s_cmp_gt_i32 s2, s4
+; GFX9-NEXT:    s_cselect_b32 s5, s2, s4
+; GFX9-NEXT:    s_cmp_gt_i32 s3, 0
+; GFX9-NEXT:    s_cselect_b32 s6, s3, 0
+; GFX9-NEXT:    s_pack_ll_b32_b16 s5, s5, s6
+; GFX9-NEXT:    s_lshr_b32 s6, s5, 16
+; GFX9-NEXT:    s_sub_i32 s5, 0x7fff7fff, s5
+; GFX9-NEXT:    s_sub_i32 s6, 0x7fff, s6
+; GFX9-NEXT:    s_cmp_lt_i32 s2, s4
+; GFX9-NEXT:    s_cselect_b32 s2, s2, s4
+; GFX9-NEXT:    s_cmp_lt_i32 s3, 0
+; GFX9-NEXT:    s_cselect_b32 s3, s3, 0
+; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s3
+; GFX9-NEXT:    s_lshr_b32 s3, s2, 16
+; GFX9-NEXT:    s_sub_i32 s2, 0x80008000, s2
+; GFX9-NEXT:    s_sub_i32 s3, 0x8000, s3
+; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s3
+; GFX9-NEXT:    s_sext_i32_i16 s3, s2
+; GFX9-NEXT:    s_sext_i32_i16 s4, s1
 ; GFX9-NEXT:    s_ashr_i32 s2, s2, 16
-; GFX9-NEXT:    s_cmp_lt_i32 s3, s4
+; GFX9-NEXT:    s_ashr_i32 s1, s1, 16
+; GFX9-NEXT:    s_cmp_gt_i32 s3, s4
 ; GFX9-NEXT:    s_cselect_b32 s3, s3, s4
-; GFX9-NEXT:    s_cmp_lt_i32 s1, s2
-; GFX9-NEXT:    s_cselect_b32 s1, s1, s2
+; GFX9-NEXT:    s_cmp_gt_i32 s2, s1
+; GFX9-NEXT:    s_cselect_b32 s1, s2, s1
+; GFX9-NEXT:    s_pack_ll_b32_b16 s5, s5, s6
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s3, s1
+; GFX9-NEXT:    s_sext_i32_i16 s2, s1
+; GFX9-NEXT:    s_ashr_i32 s1, s1, 16
+; GFX9-NEXT:    s_sext_i32_i16 s3, s5
+; GFX9-NEXT:    s_ashr_i32 s4, s5, 16
+; GFX9-NEXT:    s_cmp_lt_i32 s2, s3
+; GFX9-NEXT:    s_cselect_b32 s2, s2, s3
+; GFX9-NEXT:    s_cmp_lt_i32 s1, s4
+; GFX9-NEXT:    s_cselect_b32 s1, s1, s4
+; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s2, s1
 ; GFX9-NEXT:    s_lshr_b32 s2, s0, 16
 ; GFX9-NEXT:    s_lshr_b32 s3, s1, 16
 ; GFX9-NEXT:    s_add_i32 s0, s0, s1
@@ -4706,55 +4690,47 @@ define amdgpu_ps i32 @s_saddsat_v2i16(<2 x i16> inreg %lhs, <2 x i16> inreg %rhs
 ;
 ; GFX10-LABEL: s_saddsat_v2i16:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_pack_ll_b32_b16 s2, 0, 0
-; GFX10-NEXT:    s_sext_i32_i16 s3, s0
-; GFX10-NEXT:    s_sext_i32_i16 s5, s2
+; GFX10-NEXT:    s_sext_i32_i16 s2, s0
+; GFX10-NEXT:    s_sext_i32_i16 s3, 0
 ; GFX10-NEXT:    s_ashr_i32 s4, s0, 16
-; GFX10-NEXT:    s_ashr_i32 s2, s2, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s3, s5
-; GFX10-NEXT:    s_movk_i32 s7, 0x7fff
-; GFX10-NEXT:    s_cselect_b32 s6, s3, s5
-; GFX10-NEXT:    s_cmp_gt_i32 s4, s2
-; GFX10-NEXT:    s_pack_ll_b32_b16 s7, s7, s7
-; GFX10-NEXT:    s_cselect_b32 s8, s4, s2
+; GFX10-NEXT:    s_cmp_gt_i32 s2, s3
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s6, s8
-; GFX10-NEXT:    s_lshr_b32 s8, s7, 16
-; GFX10-NEXT:    s_lshr_b32 s9, s6, 16
-; GFX10-NEXT:    s_sub_i32 s6, s7, s6
-; GFX10-NEXT:    s_sub_i32 s7, s8, s9
-; GFX10-NEXT:    s_cmp_lt_i32 s3, s5
-; GFX10-NEXT:    s_cselect_b32 s3, s3, s5
-; GFX10-NEXT:    s_cmp_lt_i32 s4, s2
-; GFX10-NEXT:    s_movk_i32 s5, 0x8000
-; GFX10-NEXT:    s_cselect_b32 s2, s4, s2
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s5, s5
-; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s3, s2
-; GFX10-NEXT:    s_lshr_b32 s3, s4, 16
-; GFX10-NEXT:    s_lshr_b32 s5, s2, 16
-; GFX10-NEXT:    s_sub_i32 s2, s4, s2
-; GFX10-NEXT:    s_sub_i32 s3, s3, s5
+; GFX10-NEXT:    s_cselect_b32 s5, s2, s3
+; GFX10-NEXT:    s_cmp_gt_i32 s4, 0
+; GFX10-NEXT:    s_cselect_b32 s6, s4, 0
+; GFX10-NEXT:    s_pack_ll_b32_b16 s5, s5, s6
+; GFX10-NEXT:    s_lshr_b32 s6, s5, 16
+; GFX10-NEXT:    s_sub_i32 s5, 0x7fff7fff, s5
+; GFX10-NEXT:    s_sub_i32 s6, 0x7fff, s6
+; GFX10-NEXT:    s_cmp_lt_i32 s2, s3
+; GFX10-NEXT:    s_cselect_b32 s2, s2, s3
+; GFX10-NEXT:    s_cmp_lt_i32 s4, 0
+; GFX10-NEXT:    s_cselect_b32 s3, s4, 0
 ; GFX10-NEXT:    s_sext_i32_i16 s4, s1
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s2, s3
 ; GFX10-NEXT:    s_ashr_i32 s1, s1, 16
+; GFX10-NEXT:    s_lshr_b32 s3, s2, 16
+; GFX10-NEXT:    s_sub_i32 s2, 0x80008000, s2
+; GFX10-NEXT:    s_sub_i32 s3, 0x8000, s3
+; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s2, s3
 ; GFX10-NEXT:    s_sext_i32_i16 s3, s2
 ; GFX10-NEXT:    s_ashr_i32 s2, s2, 16
 ; GFX10-NEXT:    s_cmp_gt_i32 s3, s4
 ; GFX10-NEXT:    s_cselect_b32 s3, s3, s4
 ; GFX10-NEXT:    s_cmp_gt_i32 s2, s1
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s5, s6
 ; GFX10-NEXT:    s_cselect_b32 s1, s2, s1
-; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s6, s7
+; GFX10-NEXT:    s_sext_i32_i16 s2, s4
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s3, s1
-; GFX10-NEXT:    s_sext_i32_i16 s4, s2
-; GFX10-NEXT:    s_sext_i32_i16 s3, s1
+; GFX10-NEXT:    s_ashr_i32 s3, s4, 16
+; GFX10-NEXT:    s_sext_i32_i16 s4, s1
 ; GFX10-NEXT:    s_ashr_i32 s1, s1, 16
-; GFX10-NEXT:    s_ashr_i32 s2, s2, 16
-; GFX10-NEXT:    s_cmp_lt_i32 s3, s4
-; GFX10-NEXT:    s_cselect_b32 s3, s3, s4
-; GFX10-NEXT:    s_cmp_lt_i32 s1, s2
-; GFX10-NEXT:    s_cselect_b32 s1, s1, s2
+; GFX10-NEXT:    s_cmp_lt_i32 s4, s2
+; GFX10-NEXT:    s_cselect_b32 s2, s4, s2
+; GFX10-NEXT:    s_cmp_lt_i32 s1, s3
+; GFX10-NEXT:    s_cselect_b32 s1, s1, s3
+; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s2, s1
 ; GFX10-NEXT:    s_lshr_b32 s2, s0, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s3, s1
 ; GFX10-NEXT:    s_lshr_b32 s3, s1, 16
 ; GFX10-NEXT:    s_add_i32 s0, s0, s1
 ; GFX10-NEXT:    s_add_i32 s2, s2, s3
@@ -4834,73 +4810,57 @@ define amdgpu_ps float @saddsat_v2i16_sv(<2 x i16> inreg %lhs, <2 x i16> %rhs) {
 ;
 ; GFX9-LABEL: saddsat_v2i16_sv:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_pack_ll_b32_b16 s3, 0, 0
-; GFX9-NEXT:    s_sext_i32_i16 s6, s3
-; GFX9-NEXT:    s_sext_i32_i16 s4, s0
-; GFX9-NEXT:    s_ashr_i32 s5, s0, 16
-; GFX9-NEXT:    s_ashr_i32 s3, s3, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s4, s6
-; GFX9-NEXT:    s_cselect_b32 s7, s4, s6
-; GFX9-NEXT:    s_cmp_gt_i32 s5, s3
-; GFX9-NEXT:    s_movk_i32 s1, 0x7fff
-; GFX9-NEXT:    s_cselect_b32 s8, s5, s3
-; GFX9-NEXT:    s_pack_ll_b32_b16 s7, s7, s8
-; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s1
-; GFX9-NEXT:    s_lshr_b32 s8, s1, 16
-; GFX9-NEXT:    s_lshr_b32 s9, s7, 16
-; GFX9-NEXT:    s_sub_i32 s1, s1, s7
-; GFX9-NEXT:    s_sub_i32 s7, s8, s9
-; GFX9-NEXT:    s_cmp_lt_i32 s4, s6
-; GFX9-NEXT:    s_cselect_b32 s4, s4, s6
-; GFX9-NEXT:    s_cmp_lt_i32 s5, s3
-; GFX9-NEXT:    s_movk_i32 s2, 0x8000
-; GFX9-NEXT:    s_cselect_b32 s3, s5, s3
-; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s4, s3
-; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s2
-; GFX9-NEXT:    s_lshr_b32 s4, s2, 16
-; GFX9-NEXT:    s_lshr_b32 s5, s3, 16
-; GFX9-NEXT:    s_sub_i32 s2, s2, s3
-; GFX9-NEXT:    s_sub_i32 s3, s4, s5
-; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s3
-; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s7
-; GFX9-NEXT:    v_pk_max_i16 v0, s2, v0
-; GFX9-NEXT:    v_pk_min_i16 v0, v0, s1
+; GFX9-NEXT:    s_sext_i32_i16 s1, s0
+; GFX9-NEXT:    s_ashr_i32 s2, s0, 16
+; GFX9-NEXT:    s_sext_i32_i16 s3, 0
+; GFX9-NEXT:    s_cmp_gt_i32 s1, s3
+; GFX9-NEXT:    s_cselect_b32 s4, s1, s3
+; GFX9-NEXT:    s_cmp_gt_i32 s2, 0
+; GFX9-NEXT:    s_cselect_b32 s5, s2, 0
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
+; GFX9-NEXT:    s_lshr_b32 s5, s4, 16
+; GFX9-NEXT:    s_sub_i32 s4, 0x7fff7fff, s4
+; GFX9-NEXT:    s_sub_i32 s5, 0x7fff, s5
+; GFX9-NEXT:    s_cmp_lt_i32 s1, s3
+; GFX9-NEXT:    s_cselect_b32 s1, s1, s3
+; GFX9-NEXT:    s_cmp_lt_i32 s2, 0
+; GFX9-NEXT:    s_cselect_b32 s2, s2, 0
+; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s2
+; GFX9-NEXT:    s_lshr_b32 s2, s1, 16
+; GFX9-NEXT:    s_sub_i32 s1, 0x80008000, s1
+; GFX9-NEXT:    s_sub_i32 s2, 0x8000, s2
+; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s2
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
+; GFX9-NEXT:    v_pk_max_i16 v0, s1, v0
+; GFX9-NEXT:    v_pk_min_i16 v0, v0, s4
 ; GFX9-NEXT:    v_pk_add_u16 v0, s0, v0
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: saddsat_v2i16_sv:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_pack_ll_b32_b16 s1, 0, 0
-; GFX10-NEXT:    s_sext_i32_i16 s2, s0
-; GFX10-NEXT:    s_sext_i32_i16 s4, s1
+; GFX10-NEXT:    s_sext_i32_i16 s1, s0
+; GFX10-NEXT:    s_sext_i32_i16 s2, 0
 ; GFX10-NEXT:    s_ashr_i32 s3, s0, 16
-; GFX10-NEXT:    s_ashr_i32 s1, s1, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s2, s4
-; GFX10-NEXT:    s_movk_i32 s6, 0x7fff
-; GFX10-NEXT:    s_cselect_b32 s5, s2, s4
-; GFX10-NEXT:    s_cmp_gt_i32 s3, s1
-; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s6, s6
-; GFX10-NEXT:    s_cselect_b32 s7, s3, s1
+; GFX10-NEXT:    s_cmp_gt_i32 s1, s2
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_pack_ll_b32_b16 s5, s5, s7
-; GFX10-NEXT:    s_lshr_b32 s7, s6, 16
-; GFX10-NEXT:    s_lshr_b32 s8, s5, 16
-; GFX10-NEXT:    s_sub_i32 s5, s6, s5
-; GFX10-NEXT:    s_sub_i32 s6, s7, s8
-; GFX10-NEXT:    s_cmp_lt_i32 s2, s4
-; GFX10-NEXT:    s_cselect_b32 s2, s2, s4
-; GFX10-NEXT:    s_cmp_lt_i32 s3, s1
-; GFX10-NEXT:    s_movk_i32 s4, 0x8000
-; GFX10-NEXT:    s_cselect_b32 s1, s3, s1
-; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s4, s4
-; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s2, s1
-; GFX10-NEXT:    s_lshr_b32 s2, s3, 16
-; GFX10-NEXT:    s_lshr_b32 s4, s1, 16
-; GFX10-NEXT:    s_sub_i32 s1, s3, s1
-; GFX10-NEXT:    s_sub_i32 s2, s2, s4
+; GFX10-NEXT:    s_cselect_b32 s4, s1, s2
+; GFX10-NEXT:    s_cmp_gt_i32 s3, 0
+; GFX10-NEXT:    s_cselect_b32 s5, s3, 0
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
+; GFX10-NEXT:    s_lshr_b32 s5, s4, 16
+; GFX10-NEXT:    s_sub_i32 s4, 0x7fff7fff, s4
+; GFX10-NEXT:    s_sub_i32 s5, 0x7fff, s5
+; GFX10-NEXT:    s_cmp_lt_i32 s1, s2
+; GFX10-NEXT:    s_cselect_b32 s1, s1, s2
+; GFX10-NEXT:    s_cmp_lt_i32 s3, 0
+; GFX10-NEXT:    s_cselect_b32 s2, s3, 0
+; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s1, s2
+; GFX10-NEXT:    s_lshr_b32 s2, s1, 16
+; GFX10-NEXT:    s_sub_i32 s1, 0x80008000, s1
+; GFX10-NEXT:    s_sub_i32 s2, 0x8000, s2
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s1, s2
 ; GFX10-NEXT:    v_pk_max_i16 v0, s1, v0
-; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s5, s6
+; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s4, s5
 ; GFX10-NEXT:    v_pk_min_i16 v0, v0, s1
 ; GFX10-NEXT:    v_pk_add_u16 v0, s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -4966,15 +4926,12 @@ define amdgpu_ps float @saddsat_v2i16_vs(<2 x i16> %lhs, <2 x i16> inreg %rhs) {
 ;
 ; GFX9-LABEL: saddsat_v2i16_vs:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_movk_i32 s2, 0x8000
-; GFX9-NEXT:    s_pack_ll_b32_b16 s3, 0, 0
-; GFX9-NEXT:    s_movk_i32 s1, 0x7fff
-; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s2
-; GFX9-NEXT:    v_pk_min_i16 v2, v0, s3
-; GFX9-NEXT:    v_pk_sub_i16 v2, s2, v2
-; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s1
-; GFX9-NEXT:    v_pk_max_i16 v1, v0, s3
-; GFX9-NEXT:    v_pk_sub_i16 v1, s1, v1
+; GFX9-NEXT:    v_pk_max_i16 v1, v0, 0
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0x7fff7fff
+; GFX9-NEXT:    v_pk_sub_i16 v1, v2, v1
+; GFX9-NEXT:    v_pk_min_i16 v2, v0, 0
+; GFX9-NEXT:    v_mov_b32_e32 v3, 0x80008000
+; GFX9-NEXT:    v_pk_sub_i16 v2, v3, v2
 ; GFX9-NEXT:    v_pk_max_i16 v2, v2, s0
 ; GFX9-NEXT:    v_pk_min_i16 v1, v2, v1
 ; GFX9-NEXT:    v_pk_add_u16 v0, v0, v1
@@ -4982,16 +4939,11 @@ define amdgpu_ps float @saddsat_v2i16_vs(<2 x i16> %lhs, <2 x i16> inreg %rhs) {
 ;
 ; GFX10-LABEL: saddsat_v2i16_vs:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_pack_ll_b32_b16 s1, 0, 0
-; GFX10-NEXT:    s_movk_i32 s2, 0x8000
-; GFX10-NEXT:    v_pk_min_i16 v1, v0, s1
-; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s2, s2
-; GFX10-NEXT:    v_pk_max_i16 v2, v0, s1
-; GFX10-NEXT:    s_movk_i32 s3, 0x7fff
+; GFX10-NEXT:    v_pk_min_i16 v1, v0, 0
+; GFX10-NEXT:    v_pk_max_i16 v2, v0, 0
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_pk_sub_i16 v1, s2, v1
-; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s3, s3
-; GFX10-NEXT:    v_pk_sub_i16 v2, s1, v2
+; GFX10-NEXT:    v_pk_sub_i16 v1, 0x80008000, v1
+; GFX10-NEXT:    v_pk_sub_i16 v2, 0x7fff7fff, v2
 ; GFX10-NEXT:    v_pk_max_i16 v1, v1, s0
 ; GFX10-NEXT:    v_pk_min_i16 v1, v1, v2
 ; GFX10-NEXT:    v_pk_add_u16 v0, v0, v1
@@ -5113,22 +5065,19 @@ define <2 x float> @v_saddsat_v4i16(<4 x i16> %lhs, <4 x i16> %rhs) {
 ; GFX9-LABEL: v_saddsat_v4i16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    s_movk_i32 s5, 0x8000
-; GFX9-NEXT:    s_pack_ll_b32_b16 s6, 0, 0
-; GFX9-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX9-NEXT:    s_pack_ll_b32_b16 s5, s5, s5
-; GFX9-NEXT:    v_pk_min_i16 v5, v0, s6
-; GFX9-NEXT:    v_pk_sub_i16 v5, s5, v5
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s4
-; GFX9-NEXT:    v_pk_max_i16 v4, v0, s6
-; GFX9-NEXT:    v_pk_sub_i16 v4, s4, v4
-; GFX9-NEXT:    v_pk_max_i16 v2, v5, v2
+; GFX9-NEXT:    v_pk_min_i16 v6, v0, 0
+; GFX9-NEXT:    v_mov_b32_e32 v7, 0x80008000
+; GFX9-NEXT:    v_pk_sub_i16 v6, v7, v6
+; GFX9-NEXT:    v_pk_max_i16 v4, v0, 0
+; GFX9-NEXT:    v_mov_b32_e32 v5, 0x7fff7fff
+; GFX9-NEXT:    v_pk_sub_i16 v4, v5, v4
+; GFX9-NEXT:    v_pk_max_i16 v2, v6, v2
 ; GFX9-NEXT:    v_pk_min_i16 v2, v2, v4
-; GFX9-NEXT:    v_pk_min_i16 v4, v1, s6
+; GFX9-NEXT:    v_pk_min_i16 v4, v1, 0
 ; GFX9-NEXT:    v_pk_add_u16 v0, v0, v2
-; GFX9-NEXT:    v_pk_max_i16 v2, v1, s6
-; GFX9-NEXT:    v_pk_sub_i16 v4, s5, v4
-; GFX9-NEXT:    v_pk_sub_i16 v2, s4, v2
+; GFX9-NEXT:    v_pk_max_i16 v2, v1, 0
+; GFX9-NEXT:    v_pk_sub_i16 v4, v7, v4
+; GFX9-NEXT:    v_pk_sub_i16 v2, v5, v2
 ; GFX9-NEXT:    v_pk_max_i16 v3, v4, v3
 ; GFX9-NEXT:    v_pk_min_i16 v2, v3, v2
 ; GFX9-NEXT:    v_pk_add_u16 v1, v1, v2
@@ -5138,24 +5087,19 @@ define <2 x float> @v_saddsat_v4i16(<4 x i16> %lhs, <4 x i16> %rhs) {
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    s_pack_ll_b32_b16 s5, 0, 0
-; GFX10-NEXT:    s_movk_i32 s6, 0x8000
-; GFX10-NEXT:    v_pk_min_i16 v4, v0, s5
-; GFX10-NEXT:    v_pk_min_i16 v5, v1, s5
-; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s6, s6
-; GFX10-NEXT:    v_pk_max_i16 v6, v0, s5
-; GFX10-NEXT:    v_pk_max_i16 v7, v1, s5
-; GFX10-NEXT:    v_pk_sub_i16 v4, s6, v4
-; GFX10-NEXT:    v_pk_sub_i16 v5, s6, v5
-; GFX10-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX10-NEXT:    v_pk_min_i16 v4, v0, 0
+; GFX10-NEXT:    v_pk_min_i16 v5, v1, 0
+; GFX10-NEXT:    v_pk_max_i16 v6, v0, 0
+; GFX10-NEXT:    v_pk_max_i16 v7, v1, 0
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s4, s4
+; GFX10-NEXT:    v_pk_sub_i16 v4, 0x80008000, v4
+; GFX10-NEXT:    v_pk_sub_i16 v5, 0x80008000, v5
+; GFX10-NEXT:    v_pk_sub_i16 v6, 0x7fff7fff, v6
+; GFX10-NEXT:    v_pk_sub_i16 v7, 0x7fff7fff, v7
 ; GFX10-NEXT:    v_pk_max_i16 v11, v4, v2
-; GFX10-NEXT:    v_pk_sub_i16 v6, s4, v6
-; GFX10-NEXT:    v_pk_sub_i16 v4, s4, v7
-; GFX10-NEXT:    v_pk_max_i16 v3, v5, v3
+; GFX10-NEXT:    v_pk_max_i16 v10, v5, v3
 ; GFX10-NEXT:    v_pk_min_i16 v2, v11, v6
-; GFX10-NEXT:    v_pk_min_i16 v3, v3, v4
+; GFX10-NEXT:    v_pk_min_i16 v3, v10, v7
 ; GFX10-NEXT:    v_pk_add_u16 v0, v0, v2
 ; GFX10-NEXT:    v_pk_add_u16 v1, v1, v3
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
@@ -5321,76 +5265,72 @@ define amdgpu_ps <2 x i32> @s_saddsat_v4i16(<4 x i16> inreg %lhs, <4 x i16> inre
 ;
 ; GFX9-LABEL: s_saddsat_v4i16:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_pack_ll_b32_b16 s6, 0, 0
-; GFX9-NEXT:    s_sext_i32_i16 s9, s6
-; GFX9-NEXT:    s_sext_i32_i16 s7, s0
-; GFX9-NEXT:    s_ashr_i32 s8, s0, 16
+; GFX9-NEXT:    s_sext_i32_i16 s6, s0
+; GFX9-NEXT:    s_ashr_i32 s7, s0, 16
+; GFX9-NEXT:    s_sext_i32_i16 s8, 0
+; GFX9-NEXT:    s_cmp_gt_i32 s6, s8
+; GFX9-NEXT:    s_cselect_b32 s9, s6, s8
+; GFX9-NEXT:    s_cmp_gt_i32 s7, 0
+; GFX9-NEXT:    s_cselect_b32 s10, s7, 0
+; GFX9-NEXT:    s_pack_ll_b32_b16 s9, s9, s10
+; GFX9-NEXT:    s_mov_b32 s4, 0x7fff7fff
+; GFX9-NEXT:    s_lshr_b32 s11, s9, 16
+; GFX9-NEXT:    s_movk_i32 s10, 0x7fff
+; GFX9-NEXT:    s_sub_i32 s9, s4, s9
+; GFX9-NEXT:    s_sub_i32 s11, s10, s11
+; GFX9-NEXT:    s_cmp_lt_i32 s6, s8
+; GFX9-NEXT:    s_cselect_b32 s6, s6, s8
+; GFX9-NEXT:    s_cmp_lt_i32 s7, 0
+; GFX9-NEXT:    s_cselect_b32 s7, s7, 0
+; GFX9-NEXT:    s_pack_ll_b32_b16 s6, s6, s7
+; GFX9-NEXT:    s_pack_ll_b32_b16 s9, s9, s11
+; GFX9-NEXT:    s_mov_b32 s5, 0x80008000
+; GFX9-NEXT:    s_lshr_b32 s11, s6, 16
+; GFX9-NEXT:    s_mov_b32 s7, 0x8000
+; GFX9-NEXT:    s_sub_i32 s6, s5, s6
+; GFX9-NEXT:    s_sub_i32 s11, s7, s11
+; GFX9-NEXT:    s_pack_ll_b32_b16 s6, s6, s11
+; GFX9-NEXT:    s_sext_i32_i16 s11, s6
+; GFX9-NEXT:    s_sext_i32_i16 s12, s2
 ; GFX9-NEXT:    s_ashr_i32 s6, s6, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s7, s9
-; GFX9-NEXT:    s_cselect_b32 s10, s7, s9
-; GFX9-NEXT:    s_cmp_gt_i32 s8, s6
-; GFX9-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX9-NEXT:    s_cselect_b32 s11, s8, s6
-; GFX9-NEXT:    s_pack_ll_b32_b16 s10, s10, s11
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s4
-; GFX9-NEXT:    s_lshr_b32 s12, s10, 16
-; GFX9-NEXT:    s_lshr_b32 s11, s4, 16
-; GFX9-NEXT:    s_sub_i32 s10, s4, s10
-; GFX9-NEXT:    s_sub_i32 s12, s11, s12
-; GFX9-NEXT:    s_cmp_lt_i32 s7, s9
-; GFX9-NEXT:    s_cselect_b32 s7, s7, s9
-; GFX9-NEXT:    s_cmp_lt_i32 s8, s6
-; GFX9-NEXT:    s_movk_i32 s5, 0x8000
-; GFX9-NEXT:    s_cselect_b32 s8, s8, s6
-; GFX9-NEXT:    s_pack_ll_b32_b16 s5, s5, s5
-; GFX9-NEXT:    s_pack_ll_b32_b16 s7, s7, s8
-; GFX9-NEXT:    s_pack_ll_b32_b16 s10, s10, s12
-; GFX9-NEXT:    s_lshr_b32 s12, s7, 16
-; GFX9-NEXT:    s_lshr_b32 s8, s5, 16
-; GFX9-NEXT:    s_sub_i32 s7, s5, s7
-; GFX9-NEXT:    s_sub_i32 s12, s8, s12
-; GFX9-NEXT:    s_pack_ll_b32_b16 s7, s7, s12
-; GFX9-NEXT:    s_sext_i32_i16 s12, s7
-; GFX9-NEXT:    s_sext_i32_i16 s13, s2
-; GFX9-NEXT:    s_ashr_i32 s7, s7, 16
 ; GFX9-NEXT:    s_ashr_i32 s2, s2, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s12, s13
-; GFX9-NEXT:    s_cselect_b32 s12, s12, s13
-; GFX9-NEXT:    s_cmp_gt_i32 s7, s2
-; GFX9-NEXT:    s_cselect_b32 s2, s7, s2
-; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s12, s2
-; GFX9-NEXT:    s_sext_i32_i16 s7, s2
-; GFX9-NEXT:    s_sext_i32_i16 s12, s10
+; GFX9-NEXT:    s_cmp_gt_i32 s11, s12
+; GFX9-NEXT:    s_cselect_b32 s11, s11, s12
+; GFX9-NEXT:    s_cmp_gt_i32 s6, s2
+; GFX9-NEXT:    s_cselect_b32 s2, s6, s2
+; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s11, s2
+; GFX9-NEXT:    s_sext_i32_i16 s6, s2
+; GFX9-NEXT:    s_sext_i32_i16 s11, s9
 ; GFX9-NEXT:    s_ashr_i32 s2, s2, 16
-; GFX9-NEXT:    s_ashr_i32 s10, s10, 16
-; GFX9-NEXT:    s_cmp_lt_i32 s7, s12
-; GFX9-NEXT:    s_cselect_b32 s7, s7, s12
-; GFX9-NEXT:    s_cmp_lt_i32 s2, s10
-; GFX9-NEXT:    s_cselect_b32 s2, s2, s10
-; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s7, s2
-; GFX9-NEXT:    s_lshr_b32 s7, s0, 16
-; GFX9-NEXT:    s_lshr_b32 s10, s2, 16
-; GFX9-NEXT:    s_add_i32 s0, s0, s2
-; GFX9-NEXT:    s_add_i32 s7, s7, s10
-; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s7
-; GFX9-NEXT:    s_sext_i32_i16 s2, s1
-; GFX9-NEXT:    s_ashr_i32 s7, s1, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s2, s9
-; GFX9-NEXT:    s_cselect_b32 s10, s2, s9
-; GFX9-NEXT:    s_cmp_gt_i32 s7, s6
-; GFX9-NEXT:    s_cselect_b32 s12, s7, s6
-; GFX9-NEXT:    s_pack_ll_b32_b16 s10, s10, s12
-; GFX9-NEXT:    s_lshr_b32 s12, s10, 16
-; GFX9-NEXT:    s_sub_i32 s4, s4, s10
-; GFX9-NEXT:    s_sub_i32 s10, s11, s12
+; GFX9-NEXT:    s_ashr_i32 s9, s9, 16
+; GFX9-NEXT:    s_cmp_lt_i32 s6, s11
+; GFX9-NEXT:    s_cselect_b32 s6, s6, s11
 ; GFX9-NEXT:    s_cmp_lt_i32 s2, s9
 ; GFX9-NEXT:    s_cselect_b32 s2, s2, s9
-; GFX9-NEXT:    s_cmp_lt_i32 s7, s6
-; GFX9-NEXT:    s_cselect_b32 s6, s7, s6
+; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s6, s2
+; GFX9-NEXT:    s_lshr_b32 s6, s0, 16
+; GFX9-NEXT:    s_lshr_b32 s9, s2, 16
+; GFX9-NEXT:    s_add_i32 s0, s0, s2
+; GFX9-NEXT:    s_add_i32 s6, s6, s9
+; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s6
+; GFX9-NEXT:    s_sext_i32_i16 s2, s1
+; GFX9-NEXT:    s_ashr_i32 s6, s1, 16
+; GFX9-NEXT:    s_cmp_gt_i32 s2, s8
+; GFX9-NEXT:    s_cselect_b32 s9, s2, s8
+; GFX9-NEXT:    s_cmp_gt_i32 s6, 0
+; GFX9-NEXT:    s_cselect_b32 s11, s6, 0
+; GFX9-NEXT:    s_pack_ll_b32_b16 s9, s9, s11
+; GFX9-NEXT:    s_lshr_b32 s11, s9, 16
+; GFX9-NEXT:    s_sub_i32 s4, s4, s9
+; GFX9-NEXT:    s_sub_i32 s9, s10, s11
+; GFX9-NEXT:    s_cmp_lt_i32 s2, s8
+; GFX9-NEXT:    s_cselect_b32 s2, s2, s8
+; GFX9-NEXT:    s_cmp_lt_i32 s6, 0
+; GFX9-NEXT:    s_cselect_b32 s6, s6, 0
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s6
 ; GFX9-NEXT:    s_lshr_b32 s6, s2, 16
 ; GFX9-NEXT:    s_sub_i32 s2, s5, s2
-; GFX9-NEXT:    s_sub_i32 s5, s8, s6
+; GFX9-NEXT:    s_sub_i32 s5, s7, s6
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s5
 ; GFX9-NEXT:    s_sext_i32_i16 s5, s2
 ; GFX9-NEXT:    s_sext_i32_i16 s6, s3
@@ -5401,7 +5341,7 @@ define amdgpu_ps <2 x i32> @s_saddsat_v4i16(<4 x i16> inreg %lhs, <4 x i16> inre
 ; GFX9-NEXT:    s_cmp_gt_i32 s2, s3
 ; GFX9-NEXT:    s_cselect_b32 s2, s2, s3
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s5, s2
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s10
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s9
 ; GFX9-NEXT:    s_sext_i32_i16 s3, s2
 ; GFX9-NEXT:    s_sext_i32_i16 s5, s4
 ; GFX9-NEXT:    s_ashr_i32 s2, s2, 16
@@ -5420,94 +5360,90 @@ define amdgpu_ps <2 x i32> @s_saddsat_v4i16(<4 x i16> inreg %lhs, <4 x i16> inre
 ;
 ; GFX10-LABEL: s_saddsat_v4i16:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, 0, 0
-; GFX10-NEXT:    s_sext_i32_i16 s5, s0
-; GFX10-NEXT:    s_sext_i32_i16 s7, s4
+; GFX10-NEXT:    s_sext_i32_i16 s4, s0
+; GFX10-NEXT:    s_sext_i32_i16 s5, 0
 ; GFX10-NEXT:    s_ashr_i32 s6, s0, 16
-; GFX10-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s5, s7
-; GFX10-NEXT:    s_movk_i32 s9, 0x7fff
-; GFX10-NEXT:    s_cselect_b32 s8, s5, s7
-; GFX10-NEXT:    s_cmp_gt_i32 s6, s4
-; GFX10-NEXT:    s_pack_ll_b32_b16 s9, s9, s9
-; GFX10-NEXT:    s_cselect_b32 s10, s6, s4
-; GFX10-NEXT:    s_movk_i32 s12, 0x8000
-; GFX10-NEXT:    s_pack_ll_b32_b16 s8, s8, s10
-; GFX10-NEXT:    s_lshr_b32 s10, s9, 16
-; GFX10-NEXT:    s_lshr_b32 s11, s8, 16
-; GFX10-NEXT:    s_sub_i32 s8, s9, s8
-; GFX10-NEXT:    s_sub_i32 s11, s10, s11
-; GFX10-NEXT:    s_cmp_lt_i32 s5, s7
-; GFX10-NEXT:    s_pack_ll_b32_b16 s12, s12, s12
-; GFX10-NEXT:    s_cselect_b32 s5, s5, s7
-; GFX10-NEXT:    s_cmp_lt_i32 s6, s4
-; GFX10-NEXT:    s_sext_i32_i16 s14, s2
-; GFX10-NEXT:    s_cselect_b32 s6, s6, s4
-; GFX10-NEXT:    s_ashr_i32 s2, s2, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s5, s5, s6
-; GFX10-NEXT:    s_lshr_b32 s6, s12, 16
-; GFX10-NEXT:    s_lshr_b32 s13, s5, 16
-; GFX10-NEXT:    s_sub_i32 s5, s12, s5
-; GFX10-NEXT:    s_sub_i32 s13, s6, s13
+; GFX10-NEXT:    s_cmp_gt_i32 s4, s5
+; GFX10-NEXT:    s_mov_b32 s9, 0x7fff7fff
+; GFX10-NEXT:    s_cselect_b32 s7, s4, s5
+; GFX10-NEXT:    s_cmp_gt_i32 s6, 0
+; GFX10-NEXT:    s_mov_b32 s11, 0x80008000
+; GFX10-NEXT:    s_cselect_b32 s8, s6, 0
+; GFX10-NEXT:    s_sext_i32_i16 s13, s2
+; GFX10-NEXT:    s_pack_ll_b32_b16 s7, s7, s8
+; GFX10-NEXT:    s_movk_i32 s8, 0x7fff
+; GFX10-NEXT:    s_lshr_b32 s10, s7, 16
+; GFX10-NEXT:    s_sub_i32 s7, s9, s7
+; GFX10-NEXT:    s_sub_i32 s10, s8, s10
+; GFX10-NEXT:    s_cmp_lt_i32 s4, s5
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_pack_ll_b32_b16 s5, s5, s13
-; GFX10-NEXT:    s_sext_i32_i16 s13, s5
-; GFX10-NEXT:    s_ashr_i32 s5, s5, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s13, s14
-; GFX10-NEXT:    s_cselect_b32 s13, s13, s14
-; GFX10-NEXT:    s_cmp_gt_i32 s5, s2
-; GFX10-NEXT:    s_cselect_b32 s2, s5, s2
-; GFX10-NEXT:    s_pack_ll_b32_b16 s5, s8, s11
-; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s13, s2
-; GFX10-NEXT:    s_sext_i32_i16 s11, s5
-; GFX10-NEXT:    s_sext_i32_i16 s8, s2
+; GFX10-NEXT:    s_cselect_b32 s4, s4, s5
+; GFX10-NEXT:    s_cmp_lt_i32 s6, 0
+; GFX10-NEXT:    s_cselect_b32 s6, s6, 0
 ; GFX10-NEXT:    s_ashr_i32 s2, s2, 16
-; GFX10-NEXT:    s_ashr_i32 s5, s5, 16
-; GFX10-NEXT:    s_cmp_lt_i32 s8, s11
-; GFX10-NEXT:    s_cselect_b32 s8, s8, s11
-; GFX10-NEXT:    s_cmp_lt_i32 s2, s5
-; GFX10-NEXT:    s_cselect_b32 s2, s2, s5
-; GFX10-NEXT:    s_lshr_b32 s5, s0, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s8, s2
-; GFX10-NEXT:    s_lshr_b32 s8, s2, 16
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s4, s6
+; GFX10-NEXT:    s_mov_b32 s6, 0x8000
+; GFX10-NEXT:    s_lshr_b32 s12, s4, 16
+; GFX10-NEXT:    s_sub_i32 s4, s11, s4
+; GFX10-NEXT:    s_sub_i32 s12, s6, s12
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s4, s12
+; GFX10-NEXT:    s_sext_i32_i16 s12, s4
+; GFX10-NEXT:    s_ashr_i32 s4, s4, 16
+; GFX10-NEXT:    s_cmp_gt_i32 s12, s13
+; GFX10-NEXT:    s_cselect_b32 s12, s12, s13
+; GFX10-NEXT:    s_cmp_gt_i32 s4, s2
+; GFX10-NEXT:    s_cselect_b32 s2, s4, s2
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s7, s10
+; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s12, s2
+; GFX10-NEXT:    s_sext_i32_i16 s10, s4
+; GFX10-NEXT:    s_sext_i32_i16 s7, s2
+; GFX10-NEXT:    s_ashr_i32 s2, s2, 16
+; GFX10-NEXT:    s_ashr_i32 s4, s4, 16
+; GFX10-NEXT:    s_cmp_lt_i32 s7, s10
+; GFX10-NEXT:    s_cselect_b32 s7, s7, s10
+; GFX10-NEXT:    s_cmp_lt_i32 s2, s4
+; GFX10-NEXT:    s_cselect_b32 s2, s2, s4
+; GFX10-NEXT:    s_sext_i32_i16 s4, s1
+; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s7, s2
+; GFX10-NEXT:    s_lshr_b32 s7, s0, 16
+; GFX10-NEXT:    s_lshr_b32 s10, s2, 16
 ; GFX10-NEXT:    s_add_i32 s0, s0, s2
-; GFX10-NEXT:    s_sext_i32_i16 s2, s1
-; GFX10-NEXT:    s_add_i32 s5, s5, s8
-; GFX10-NEXT:    s_ashr_i32 s8, s1, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s2, s7
-; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s5
-; GFX10-NEXT:    s_cselect_b32 s11, s2, s7
-; GFX10-NEXT:    s_cmp_gt_i32 s8, s4
-; GFX10-NEXT:    s_cselect_b32 s13, s8, s4
-; GFX10-NEXT:    s_pack_ll_b32_b16 s11, s11, s13
-; GFX10-NEXT:    s_lshr_b32 s13, s11, 16
-; GFX10-NEXT:    s_sub_i32 s9, s9, s11
-; GFX10-NEXT:    s_sub_i32 s10, s10, s13
-; GFX10-NEXT:    s_cmp_lt_i32 s2, s7
-; GFX10-NEXT:    s_cselect_b32 s2, s2, s7
-; GFX10-NEXT:    s_cmp_lt_i32 s8, s4
-; GFX10-NEXT:    s_cselect_b32 s4, s8, s4
-; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s2, s4
+; GFX10-NEXT:    s_add_i32 s7, s7, s10
+; GFX10-NEXT:    s_ashr_i32 s2, s1, 16
+; GFX10-NEXT:    s_cmp_gt_i32 s4, s5
+; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s7
+; GFX10-NEXT:    s_cselect_b32 s10, s4, s5
+; GFX10-NEXT:    s_cmp_gt_i32 s2, 0
+; GFX10-NEXT:    s_cselect_b32 s12, s2, 0
+; GFX10-NEXT:    s_pack_ll_b32_b16 s10, s10, s12
+; GFX10-NEXT:    s_lshr_b32 s12, s10, 16
+; GFX10-NEXT:    s_sub_i32 s9, s9, s10
+; GFX10-NEXT:    s_sub_i32 s8, s8, s12
+; GFX10-NEXT:    s_cmp_lt_i32 s4, s5
+; GFX10-NEXT:    s_cselect_b32 s4, s4, s5
+; GFX10-NEXT:    s_cmp_lt_i32 s2, 0
+; GFX10-NEXT:    s_sext_i32_i16 s5, s3
+; GFX10-NEXT:    s_cselect_b32 s2, s2, 0
+; GFX10-NEXT:    s_ashr_i32 s3, s3, 16
+; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s4, s2
 ; GFX10-NEXT:    s_lshr_b32 s4, s2, 16
-; GFX10-NEXT:    s_sub_i32 s2, s12, s2
+; GFX10-NEXT:    s_sub_i32 s2, s11, s2
 ; GFX10-NEXT:    s_sub_i32 s4, s6, s4
-; GFX10-NEXT:    s_sext_i32_i16 s6, s3
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s2, s4
-; GFX10-NEXT:    s_ashr_i32 s3, s3, 16
 ; GFX10-NEXT:    s_sext_i32_i16 s4, s2
 ; GFX10-NEXT:    s_ashr_i32 s2, s2, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s4, s6
-; GFX10-NEXT:    s_cselect_b32 s4, s4, s6
+; GFX10-NEXT:    s_cmp_gt_i32 s4, s5
+; GFX10-NEXT:    s_cselect_b32 s4, s4, s5
 ; GFX10-NEXT:    s_cmp_gt_i32 s2, s3
-; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s9, s10
+; GFX10-NEXT:    s_pack_ll_b32_b16 s5, s9, s8
 ; GFX10-NEXT:    s_cselect_b32 s2, s2, s3
-; GFX10-NEXT:    s_sext_i32_i16 s3, s6
+; GFX10-NEXT:    s_sext_i32_i16 s3, s5
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s4, s2
-; GFX10-NEXT:    s_ashr_i32 s4, s6, 16
-; GFX10-NEXT:    s_sext_i32_i16 s6, s2
+; GFX10-NEXT:    s_ashr_i32 s4, s5, 16
+; GFX10-NEXT:    s_sext_i32_i16 s5, s2
 ; GFX10-NEXT:    s_ashr_i32 s2, s2, 16
-; GFX10-NEXT:    s_cmp_lt_i32 s6, s3
-; GFX10-NEXT:    s_cselect_b32 s3, s6, s3
+; GFX10-NEXT:    s_cmp_lt_i32 s5, s3
+; GFX10-NEXT:    s_cselect_b32 s3, s5, s3
 ; GFX10-NEXT:    s_cmp_lt_i32 s2, s4
 ; GFX10-NEXT:    s_cselect_b32 s2, s2, s4
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s3, s2
@@ -5676,29 +5612,26 @@ define <3 x float> @v_saddsat_v6i16(<6 x i16> %lhs, <6 x i16> %rhs) {
 ; GFX9-LABEL: v_saddsat_v6i16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    s_movk_i32 s5, 0x8000
-; GFX9-NEXT:    s_pack_ll_b32_b16 s6, 0, 0
-; GFX9-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX9-NEXT:    s_pack_ll_b32_b16 s5, s5, s5
-; GFX9-NEXT:    v_pk_min_i16 v7, v0, s6
-; GFX9-NEXT:    v_pk_sub_i16 v7, s5, v7
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s4
-; GFX9-NEXT:    v_pk_max_i16 v6, v0, s6
-; GFX9-NEXT:    v_pk_sub_i16 v6, s4, v6
-; GFX9-NEXT:    v_pk_max_i16 v3, v7, v3
+; GFX9-NEXT:    v_pk_min_i16 v8, v0, 0
+; GFX9-NEXT:    v_mov_b32_e32 v9, 0x80008000
+; GFX9-NEXT:    v_pk_sub_i16 v8, v9, v8
+; GFX9-NEXT:    v_pk_max_i16 v6, v0, 0
+; GFX9-NEXT:    v_mov_b32_e32 v7, 0x7fff7fff
+; GFX9-NEXT:    v_pk_sub_i16 v6, v7, v6
+; GFX9-NEXT:    v_pk_max_i16 v3, v8, v3
 ; GFX9-NEXT:    v_pk_min_i16 v3, v3, v6
-; GFX9-NEXT:    v_pk_min_i16 v6, v1, s6
+; GFX9-NEXT:    v_pk_min_i16 v6, v1, 0
 ; GFX9-NEXT:    v_pk_add_u16 v0, v0, v3
-; GFX9-NEXT:    v_pk_max_i16 v3, v1, s6
-; GFX9-NEXT:    v_pk_sub_i16 v6, s5, v6
-; GFX9-NEXT:    v_pk_sub_i16 v3, s4, v3
+; GFX9-NEXT:    v_pk_max_i16 v3, v1, 0
+; GFX9-NEXT:    v_pk_sub_i16 v6, v9, v6
+; GFX9-NEXT:    v_pk_sub_i16 v3, v7, v3
 ; GFX9-NEXT:    v_pk_max_i16 v4, v6, v4
 ; GFX9-NEXT:    v_pk_min_i16 v3, v4, v3
-; GFX9-NEXT:    v_pk_min_i16 v4, v2, s6
-; GFX9-NEXT:    v_pk_sub_i16 v4, s5, v4
+; GFX9-NEXT:    v_pk_min_i16 v4, v2, 0
+; GFX9-NEXT:    v_pk_sub_i16 v4, v9, v4
 ; GFX9-NEXT:    v_pk_add_u16 v1, v1, v3
-; GFX9-NEXT:    v_pk_max_i16 v3, v2, s6
-; GFX9-NEXT:    v_pk_sub_i16 v3, s4, v3
+; GFX9-NEXT:    v_pk_max_i16 v3, v2, 0
+; GFX9-NEXT:    v_pk_sub_i16 v3, v7, v3
 ; GFX9-NEXT:    v_pk_max_i16 v4, v4, v5
 ; GFX9-NEXT:    v_pk_min_i16 v3, v4, v3
 ; GFX9-NEXT:    v_pk_add_u16 v2, v2, v3
@@ -5708,28 +5641,23 @@ define <3 x float> @v_saddsat_v6i16(<6 x i16> %lhs, <6 x i16> %rhs) {
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    s_pack_ll_b32_b16 s5, 0, 0
-; GFX10-NEXT:    s_movk_i32 s6, 0x8000
-; GFX10-NEXT:    v_pk_min_i16 v7, v0, s5
-; GFX10-NEXT:    v_pk_min_i16 v8, v1, s5
-; GFX10-NEXT:    v_pk_min_i16 v9, v2, s5
-; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s6, s6
-; GFX10-NEXT:    v_pk_max_i16 v6, v0, s5
-; GFX10-NEXT:    v_pk_sub_i16 v14, s6, v7
-; GFX10-NEXT:    v_pk_sub_i16 v15, s6, v8
-; GFX10-NEXT:    v_pk_sub_i16 v19, s6, v9
-; GFX10-NEXT:    v_pk_max_i16 v10, v1, s5
-; GFX10-NEXT:    v_pk_max_i16 v11, v2, s5
-; GFX10-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX10-NEXT:    v_pk_min_i16 v7, v0, 0
+; GFX10-NEXT:    v_pk_min_i16 v8, v1, 0
+; GFX10-NEXT:    v_pk_min_i16 v9, v2, 0
+; GFX10-NEXT:    v_pk_max_i16 v6, v0, 0
+; GFX10-NEXT:    v_pk_max_i16 v10, v1, 0
+; GFX10-NEXT:    v_pk_sub_i16 v14, 0x80008000, v7
+; GFX10-NEXT:    v_pk_sub_i16 v15, 0x80008000, v8
+; GFX10-NEXT:    v_pk_max_i16 v11, v2, 0
+; GFX10-NEXT:    v_pk_sub_i16 v19, 0x80008000, v9
+; GFX10-NEXT:    v_pk_sub_i16 v6, 0x7fff7fff, v6
 ; GFX10-NEXT:    v_pk_max_i16 v3, v14, v3
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s4, s4
+; GFX10-NEXT:    v_pk_sub_i16 v7, 0x7fff7fff, v10
 ; GFX10-NEXT:    v_pk_max_i16 v4, v15, v4
-; GFX10-NEXT:    v_pk_sub_i16 v6, s4, v6
-; GFX10-NEXT:    v_pk_sub_i16 v7, s4, v10
-; GFX10-NEXT:    v_pk_sub_i16 v8, s4, v11
+; GFX10-NEXT:    v_pk_sub_i16 v8, 0x7fff7fff, v11
 ; GFX10-NEXT:    v_pk_max_i16 v5, v19, v5
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_pk_min_i16 v3, v3, v6
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_pk_min_i16 v4, v4, v7
 ; GFX10-NEXT:    v_pk_min_i16 v5, v5, v8
 ; GFX10-NEXT:    v_pk_add_u16 v0, v0, v3
@@ -5968,119 +5896,115 @@ define amdgpu_ps <3 x i32> @s_saddsat_v6i16(<6 x i16> inreg %lhs, <6 x i16> inre
 ;
 ; GFX9-LABEL: s_saddsat_v6i16:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_pack_ll_b32_b16 s8, 0, 0
-; GFX9-NEXT:    s_sext_i32_i16 s11, s8
-; GFX9-NEXT:    s_sext_i32_i16 s9, s0
-; GFX9-NEXT:    s_ashr_i32 s10, s0, 16
+; GFX9-NEXT:    s_sext_i32_i16 s8, s0
+; GFX9-NEXT:    s_ashr_i32 s9, s0, 16
+; GFX9-NEXT:    s_sext_i32_i16 s10, 0
+; GFX9-NEXT:    s_cmp_gt_i32 s8, s10
+; GFX9-NEXT:    s_cselect_b32 s11, s8, s10
+; GFX9-NEXT:    s_cmp_gt_i32 s9, 0
+; GFX9-NEXT:    s_cselect_b32 s12, s9, 0
+; GFX9-NEXT:    s_pack_ll_b32_b16 s11, s11, s12
+; GFX9-NEXT:    s_mov_b32 s6, 0x7fff7fff
+; GFX9-NEXT:    s_lshr_b32 s13, s11, 16
+; GFX9-NEXT:    s_movk_i32 s12, 0x7fff
+; GFX9-NEXT:    s_sub_i32 s11, s6, s11
+; GFX9-NEXT:    s_sub_i32 s13, s12, s13
+; GFX9-NEXT:    s_cmp_lt_i32 s8, s10
+; GFX9-NEXT:    s_cselect_b32 s8, s8, s10
+; GFX9-NEXT:    s_cmp_lt_i32 s9, 0
+; GFX9-NEXT:    s_cselect_b32 s9, s9, 0
+; GFX9-NEXT:    s_pack_ll_b32_b16 s8, s8, s9
+; GFX9-NEXT:    s_pack_ll_b32_b16 s11, s11, s13
+; GFX9-NEXT:    s_mov_b32 s7, 0x80008000
+; GFX9-NEXT:    s_lshr_b32 s13, s8, 16
+; GFX9-NEXT:    s_mov_b32 s9, 0x8000
+; GFX9-NEXT:    s_sub_i32 s8, s7, s8
+; GFX9-NEXT:    s_sub_i32 s13, s9, s13
+; GFX9-NEXT:    s_pack_ll_b32_b16 s8, s8, s13
+; GFX9-NEXT:    s_sext_i32_i16 s13, s8
+; GFX9-NEXT:    s_sext_i32_i16 s14, s3
 ; GFX9-NEXT:    s_ashr_i32 s8, s8, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s9, s11
-; GFX9-NEXT:    s_cselect_b32 s12, s9, s11
-; GFX9-NEXT:    s_cmp_gt_i32 s10, s8
-; GFX9-NEXT:    s_movk_i32 s6, 0x7fff
-; GFX9-NEXT:    s_cselect_b32 s13, s10, s8
-; GFX9-NEXT:    s_pack_ll_b32_b16 s12, s12, s13
-; GFX9-NEXT:    s_pack_ll_b32_b16 s6, s6, s6
-; GFX9-NEXT:    s_lshr_b32 s14, s12, 16
-; GFX9-NEXT:    s_lshr_b32 s13, s6, 16
-; GFX9-NEXT:    s_sub_i32 s12, s6, s12
-; GFX9-NEXT:    s_sub_i32 s14, s13, s14
-; GFX9-NEXT:    s_cmp_lt_i32 s9, s11
-; GFX9-NEXT:    s_cselect_b32 s9, s9, s11
-; GFX9-NEXT:    s_cmp_lt_i32 s10, s8
-; GFX9-NEXT:    s_movk_i32 s7, 0x8000
-; GFX9-NEXT:    s_cselect_b32 s10, s10, s8
-; GFX9-NEXT:    s_pack_ll_b32_b16 s7, s7, s7
-; GFX9-NEXT:    s_pack_ll_b32_b16 s9, s9, s10
-; GFX9-NEXT:    s_pack_ll_b32_b16 s12, s12, s14
-; GFX9-NEXT:    s_lshr_b32 s14, s9, 16
-; GFX9-NEXT:    s_lshr_b32 s10, s7, 16
-; GFX9-NEXT:    s_sub_i32 s9, s7, s9
-; GFX9-NEXT:    s_sub_i32 s14, s10, s14
-; GFX9-NEXT:    s_pack_ll_b32_b16 s9, s9, s14
-; GFX9-NEXT:    s_sext_i32_i16 s14, s9
-; GFX9-NEXT:    s_sext_i32_i16 s15, s3
-; GFX9-NEXT:    s_ashr_i32 s9, s9, 16
 ; GFX9-NEXT:    s_ashr_i32 s3, s3, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s14, s15
-; GFX9-NEXT:    s_cselect_b32 s14, s14, s15
-; GFX9-NEXT:    s_cmp_gt_i32 s9, s3
-; GFX9-NEXT:    s_cselect_b32 s3, s9, s3
-; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s14, s3
-; GFX9-NEXT:    s_sext_i32_i16 s9, s3
-; GFX9-NEXT:    s_sext_i32_i16 s14, s12
+; GFX9-NEXT:    s_cmp_gt_i32 s13, s14
+; GFX9-NEXT:    s_cselect_b32 s13, s13, s14
+; GFX9-NEXT:    s_cmp_gt_i32 s8, s3
+; GFX9-NEXT:    s_cselect_b32 s3, s8, s3
+; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s13, s3
+; GFX9-NEXT:    s_sext_i32_i16 s8, s3
+; GFX9-NEXT:    s_sext_i32_i16 s13, s11
 ; GFX9-NEXT:    s_ashr_i32 s3, s3, 16
-; GFX9-NEXT:    s_ashr_i32 s12, s12, 16
-; GFX9-NEXT:    s_cmp_lt_i32 s9, s14
-; GFX9-NEXT:    s_cselect_b32 s9, s9, s14
-; GFX9-NEXT:    s_cmp_lt_i32 s3, s12
-; GFX9-NEXT:    s_cselect_b32 s3, s3, s12
-; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s9, s3
-; GFX9-NEXT:    s_lshr_b32 s9, s0, 16
-; GFX9-NEXT:    s_lshr_b32 s12, s3, 16
-; GFX9-NEXT:    s_add_i32 s0, s0, s3
-; GFX9-NEXT:    s_add_i32 s9, s9, s12
-; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s9
-; GFX9-NEXT:    s_sext_i32_i16 s3, s1
-; GFX9-NEXT:    s_ashr_i32 s9, s1, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s3, s11
-; GFX9-NEXT:    s_cselect_b32 s12, s3, s11
-; GFX9-NEXT:    s_cmp_gt_i32 s9, s8
-; GFX9-NEXT:    s_cselect_b32 s14, s9, s8
-; GFX9-NEXT:    s_pack_ll_b32_b16 s12, s12, s14
-; GFX9-NEXT:    s_lshr_b32 s14, s12, 16
-; GFX9-NEXT:    s_sub_i32 s12, s6, s12
-; GFX9-NEXT:    s_sub_i32 s14, s13, s14
+; GFX9-NEXT:    s_ashr_i32 s11, s11, 16
+; GFX9-NEXT:    s_cmp_lt_i32 s8, s13
+; GFX9-NEXT:    s_cselect_b32 s8, s8, s13
 ; GFX9-NEXT:    s_cmp_lt_i32 s3, s11
 ; GFX9-NEXT:    s_cselect_b32 s3, s3, s11
-; GFX9-NEXT:    s_cmp_lt_i32 s9, s8
-; GFX9-NEXT:    s_cselect_b32 s9, s9, s8
-; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s3, s9
-; GFX9-NEXT:    s_lshr_b32 s9, s3, 16
+; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s8, s3
+; GFX9-NEXT:    s_lshr_b32 s8, s0, 16
+; GFX9-NEXT:    s_lshr_b32 s11, s3, 16
+; GFX9-NEXT:    s_add_i32 s0, s0, s3
+; GFX9-NEXT:    s_add_i32 s8, s8, s11
+; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s8
+; GFX9-NEXT:    s_sext_i32_i16 s3, s1
+; GFX9-NEXT:    s_ashr_i32 s8, s1, 16
+; GFX9-NEXT:    s_cmp_gt_i32 s3, s10
+; GFX9-NEXT:    s_cselect_b32 s11, s3, s10
+; GFX9-NEXT:    s_cmp_gt_i32 s8, 0
+; GFX9-NEXT:    s_cselect_b32 s13, s8, 0
+; GFX9-NEXT:    s_pack_ll_b32_b16 s11, s11, s13
+; GFX9-NEXT:    s_lshr_b32 s13, s11, 16
+; GFX9-NEXT:    s_sub_i32 s11, s6, s11
+; GFX9-NEXT:    s_sub_i32 s13, s12, s13
+; GFX9-NEXT:    s_cmp_lt_i32 s3, s10
+; GFX9-NEXT:    s_cselect_b32 s3, s3, s10
+; GFX9-NEXT:    s_cmp_lt_i32 s8, 0
+; GFX9-NEXT:    s_cselect_b32 s8, s8, 0
+; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s3, s8
+; GFX9-NEXT:    s_lshr_b32 s8, s3, 16
 ; GFX9-NEXT:    s_sub_i32 s3, s7, s3
-; GFX9-NEXT:    s_sub_i32 s9, s10, s9
-; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s3, s9
-; GFX9-NEXT:    s_pack_ll_b32_b16 s12, s12, s14
-; GFX9-NEXT:    s_sext_i32_i16 s9, s3
-; GFX9-NEXT:    s_sext_i32_i16 s14, s4
+; GFX9-NEXT:    s_sub_i32 s8, s9, s8
+; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s3, s8
+; GFX9-NEXT:    s_pack_ll_b32_b16 s11, s11, s13
+; GFX9-NEXT:    s_sext_i32_i16 s8, s3
+; GFX9-NEXT:    s_sext_i32_i16 s13, s4
 ; GFX9-NEXT:    s_ashr_i32 s3, s3, 16
 ; GFX9-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s9, s14
-; GFX9-NEXT:    s_cselect_b32 s9, s9, s14
+; GFX9-NEXT:    s_cmp_gt_i32 s8, s13
+; GFX9-NEXT:    s_cselect_b32 s8, s8, s13
 ; GFX9-NEXT:    s_cmp_gt_i32 s3, s4
 ; GFX9-NEXT:    s_cselect_b32 s3, s3, s4
-; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s9, s3
+; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s8, s3
 ; GFX9-NEXT:    s_sext_i32_i16 s4, s3
-; GFX9-NEXT:    s_sext_i32_i16 s9, s12
+; GFX9-NEXT:    s_sext_i32_i16 s8, s11
 ; GFX9-NEXT:    s_ashr_i32 s3, s3, 16
-; GFX9-NEXT:    s_ashr_i32 s12, s12, 16
-; GFX9-NEXT:    s_cmp_lt_i32 s4, s9
-; GFX9-NEXT:    s_cselect_b32 s4, s4, s9
-; GFX9-NEXT:    s_cmp_lt_i32 s3, s12
-; GFX9-NEXT:    s_cselect_b32 s3, s3, s12
+; GFX9-NEXT:    s_ashr_i32 s11, s11, 16
+; GFX9-NEXT:    s_cmp_lt_i32 s4, s8
+; GFX9-NEXT:    s_cselect_b32 s4, s4, s8
+; GFX9-NEXT:    s_cmp_lt_i32 s3, s11
+; GFX9-NEXT:    s_cselect_b32 s3, s3, s11
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s4, s3
 ; GFX9-NEXT:    s_lshr_b32 s4, s1, 16
-; GFX9-NEXT:    s_lshr_b32 s9, s3, 16
+; GFX9-NEXT:    s_lshr_b32 s8, s3, 16
 ; GFX9-NEXT:    s_add_i32 s1, s1, s3
-; GFX9-NEXT:    s_add_i32 s4, s4, s9
+; GFX9-NEXT:    s_add_i32 s4, s4, s8
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s4
 ; GFX9-NEXT:    s_sext_i32_i16 s3, s2
 ; GFX9-NEXT:    s_ashr_i32 s4, s2, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s3, s11
-; GFX9-NEXT:    s_cselect_b32 s9, s3, s11
-; GFX9-NEXT:    s_cmp_gt_i32 s4, s8
-; GFX9-NEXT:    s_cselect_b32 s12, s4, s8
-; GFX9-NEXT:    s_pack_ll_b32_b16 s9, s9, s12
-; GFX9-NEXT:    s_lshr_b32 s12, s9, 16
-; GFX9-NEXT:    s_sub_i32 s6, s6, s9
-; GFX9-NEXT:    s_sub_i32 s9, s13, s12
-; GFX9-NEXT:    s_cmp_lt_i32 s3, s11
-; GFX9-NEXT:    s_cselect_b32 s3, s3, s11
-; GFX9-NEXT:    s_cmp_lt_i32 s4, s8
-; GFX9-NEXT:    s_cselect_b32 s4, s4, s8
+; GFX9-NEXT:    s_cmp_gt_i32 s3, s10
+; GFX9-NEXT:    s_cselect_b32 s8, s3, s10
+; GFX9-NEXT:    s_cmp_gt_i32 s4, 0
+; GFX9-NEXT:    s_cselect_b32 s11, s4, 0
+; GFX9-NEXT:    s_pack_ll_b32_b16 s8, s8, s11
+; GFX9-NEXT:    s_lshr_b32 s11, s8, 16
+; GFX9-NEXT:    s_sub_i32 s6, s6, s8
+; GFX9-NEXT:    s_sub_i32 s8, s12, s11
+; GFX9-NEXT:    s_cmp_lt_i32 s3, s10
+; GFX9-NEXT:    s_cselect_b32 s3, s3, s10
+; GFX9-NEXT:    s_cmp_lt_i32 s4, 0
+; GFX9-NEXT:    s_cselect_b32 s4, s4, 0
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s3, s4
 ; GFX9-NEXT:    s_lshr_b32 s4, s3, 16
 ; GFX9-NEXT:    s_sub_i32 s3, s7, s3
-; GFX9-NEXT:    s_sub_i32 s4, s10, s4
+; GFX9-NEXT:    s_sub_i32 s4, s9, s4
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s3, s4
 ; GFX9-NEXT:    s_sext_i32_i16 s4, s3
 ; GFX9-NEXT:    s_sext_i32_i16 s7, s5
@@ -6091,7 +6015,7 @@ define amdgpu_ps <3 x i32> @s_saddsat_v6i16(<6 x i16> inreg %lhs, <6 x i16> inre
 ; GFX9-NEXT:    s_cmp_gt_i32 s3, s5
 ; GFX9-NEXT:    s_cselect_b32 s3, s3, s5
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s4, s3
-; GFX9-NEXT:    s_pack_ll_b32_b16 s6, s6, s9
+; GFX9-NEXT:    s_pack_ll_b32_b16 s6, s6, s8
 ; GFX9-NEXT:    s_sext_i32_i16 s4, s3
 ; GFX9-NEXT:    s_sext_i32_i16 s5, s6
 ; GFX9-NEXT:    s_ashr_i32 s3, s3, 16
@@ -6110,141 +6034,137 @@ define amdgpu_ps <3 x i32> @s_saddsat_v6i16(<6 x i16> inreg %lhs, <6 x i16> inre
 ;
 ; GFX10-LABEL: s_saddsat_v6i16:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_pack_ll_b32_b16 s6, 0, 0
-; GFX10-NEXT:    s_sext_i32_i16 s7, s0
-; GFX10-NEXT:    s_sext_i32_i16 s9, s6
+; GFX10-NEXT:    s_sext_i32_i16 s6, s0
+; GFX10-NEXT:    s_sext_i32_i16 s7, 0
 ; GFX10-NEXT:    s_ashr_i32 s8, s0, 16
-; GFX10-NEXT:    s_ashr_i32 s6, s6, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s7, s9
-; GFX10-NEXT:    s_movk_i32 s11, 0x7fff
-; GFX10-NEXT:    s_cselect_b32 s10, s7, s9
-; GFX10-NEXT:    s_cmp_gt_i32 s8, s6
-; GFX10-NEXT:    s_pack_ll_b32_b16 s11, s11, s11
-; GFX10-NEXT:    s_cselect_b32 s12, s8, s6
-; GFX10-NEXT:    s_movk_i32 s14, 0x8000
-; GFX10-NEXT:    s_pack_ll_b32_b16 s10, s10, s12
-; GFX10-NEXT:    s_lshr_b32 s12, s11, 16
-; GFX10-NEXT:    s_lshr_b32 s13, s10, 16
-; GFX10-NEXT:    s_sub_i32 s10, s11, s10
-; GFX10-NEXT:    s_sub_i32 s13, s12, s13
-; GFX10-NEXT:    s_cmp_lt_i32 s7, s9
-; GFX10-NEXT:    s_pack_ll_b32_b16 s14, s14, s14
-; GFX10-NEXT:    s_cselect_b32 s7, s7, s9
-; GFX10-NEXT:    s_cmp_lt_i32 s8, s6
-; GFX10-NEXT:    s_sext_i32_i16 s16, s3
-; GFX10-NEXT:    s_cselect_b32 s8, s8, s6
-; GFX10-NEXT:    s_ashr_i32 s3, s3, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s7, s7, s8
-; GFX10-NEXT:    s_lshr_b32 s8, s14, 16
-; GFX10-NEXT:    s_lshr_b32 s15, s7, 16
-; GFX10-NEXT:    s_sub_i32 s7, s14, s7
-; GFX10-NEXT:    s_sub_i32 s15, s8, s15
+; GFX10-NEXT:    s_cmp_gt_i32 s6, s7
+; GFX10-NEXT:    s_mov_b32 s11, 0x7fff7fff
+; GFX10-NEXT:    s_cselect_b32 s9, s6, s7
+; GFX10-NEXT:    s_cmp_gt_i32 s8, 0
+; GFX10-NEXT:    s_mov_b32 s13, 0x80008000
+; GFX10-NEXT:    s_cselect_b32 s10, s8, 0
+; GFX10-NEXT:    s_sext_i32_i16 s15, s3
+; GFX10-NEXT:    s_pack_ll_b32_b16 s9, s9, s10
+; GFX10-NEXT:    s_movk_i32 s10, 0x7fff
+; GFX10-NEXT:    s_lshr_b32 s12, s9, 16
+; GFX10-NEXT:    s_sub_i32 s9, s11, s9
+; GFX10-NEXT:    s_sub_i32 s12, s10, s12
+; GFX10-NEXT:    s_cmp_lt_i32 s6, s7
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_pack_ll_b32_b16 s7, s7, s15
-; GFX10-NEXT:    s_sext_i32_i16 s15, s7
-; GFX10-NEXT:    s_ashr_i32 s7, s7, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s15, s16
-; GFX10-NEXT:    s_cselect_b32 s15, s15, s16
-; GFX10-NEXT:    s_cmp_gt_i32 s7, s3
-; GFX10-NEXT:    s_sext_i32_i16 s16, s4
-; GFX10-NEXT:    s_cselect_b32 s3, s7, s3
-; GFX10-NEXT:    s_pack_ll_b32_b16 s7, s10, s13
-; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s15, s3
-; GFX10-NEXT:    s_sext_i32_i16 s13, s7
-; GFX10-NEXT:    s_sext_i32_i16 s10, s3
+; GFX10-NEXT:    s_cselect_b32 s6, s6, s7
+; GFX10-NEXT:    s_cmp_lt_i32 s8, 0
+; GFX10-NEXT:    s_cselect_b32 s8, s8, 0
 ; GFX10-NEXT:    s_ashr_i32 s3, s3, 16
-; GFX10-NEXT:    s_ashr_i32 s7, s7, 16
-; GFX10-NEXT:    s_cmp_lt_i32 s10, s13
-; GFX10-NEXT:    s_cselect_b32 s10, s10, s13
-; GFX10-NEXT:    s_cmp_lt_i32 s3, s7
-; GFX10-NEXT:    s_cselect_b32 s3, s3, s7
-; GFX10-NEXT:    s_lshr_b32 s7, s0, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s10, s3
-; GFX10-NEXT:    s_lshr_b32 s10, s3, 16
+; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s6, s8
+; GFX10-NEXT:    s_mov_b32 s8, 0x8000
+; GFX10-NEXT:    s_lshr_b32 s14, s6, 16
+; GFX10-NEXT:    s_sub_i32 s6, s13, s6
+; GFX10-NEXT:    s_sub_i32 s14, s8, s14
+; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s6, s14
+; GFX10-NEXT:    s_sext_i32_i16 s14, s6
+; GFX10-NEXT:    s_ashr_i32 s6, s6, 16
+; GFX10-NEXT:    s_cmp_gt_i32 s14, s15
+; GFX10-NEXT:    s_cselect_b32 s14, s14, s15
+; GFX10-NEXT:    s_cmp_gt_i32 s6, s3
+; GFX10-NEXT:    s_sext_i32_i16 s15, s4
+; GFX10-NEXT:    s_cselect_b32 s3, s6, s3
+; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s9, s12
+; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s14, s3
+; GFX10-NEXT:    s_sext_i32_i16 s12, s6
+; GFX10-NEXT:    s_sext_i32_i16 s9, s3
+; GFX10-NEXT:    s_ashr_i32 s3, s3, 16
+; GFX10-NEXT:    s_ashr_i32 s6, s6, 16
+; GFX10-NEXT:    s_cmp_lt_i32 s9, s12
+; GFX10-NEXT:    s_cselect_b32 s9, s9, s12
+; GFX10-NEXT:    s_cmp_lt_i32 s3, s6
+; GFX10-NEXT:    s_cselect_b32 s3, s3, s6
+; GFX10-NEXT:    s_lshr_b32 s6, s0, 16
+; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s9, s3
+; GFX10-NEXT:    s_lshr_b32 s9, s3, 16
 ; GFX10-NEXT:    s_add_i32 s0, s0, s3
 ; GFX10-NEXT:    s_sext_i32_i16 s3, s1
-; GFX10-NEXT:    s_add_i32 s7, s7, s10
-; GFX10-NEXT:    s_ashr_i32 s10, s1, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s3, s9
-; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s7
-; GFX10-NEXT:    s_cselect_b32 s13, s3, s9
-; GFX10-NEXT:    s_cmp_gt_i32 s10, s6
-; GFX10-NEXT:    s_cselect_b32 s15, s10, s6
-; GFX10-NEXT:    s_pack_ll_b32_b16 s13, s13, s15
-; GFX10-NEXT:    s_lshr_b32 s15, s13, 16
-; GFX10-NEXT:    s_sub_i32 s13, s11, s13
-; GFX10-NEXT:    s_sub_i32 s15, s12, s15
-; GFX10-NEXT:    s_cmp_lt_i32 s3, s9
-; GFX10-NEXT:    s_cselect_b32 s3, s3, s9
-; GFX10-NEXT:    s_cmp_lt_i32 s10, s6
-; GFX10-NEXT:    s_cselect_b32 s10, s10, s6
+; GFX10-NEXT:    s_add_i32 s6, s6, s9
+; GFX10-NEXT:    s_ashr_i32 s9, s1, 16
+; GFX10-NEXT:    s_cmp_gt_i32 s3, s7
+; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s6
+; GFX10-NEXT:    s_cselect_b32 s12, s3, s7
+; GFX10-NEXT:    s_cmp_gt_i32 s9, 0
+; GFX10-NEXT:    s_cselect_b32 s14, s9, 0
+; GFX10-NEXT:    s_pack_ll_b32_b16 s12, s12, s14
+; GFX10-NEXT:    s_lshr_b32 s14, s12, 16
+; GFX10-NEXT:    s_sub_i32 s12, s11, s12
+; GFX10-NEXT:    s_sub_i32 s14, s10, s14
+; GFX10-NEXT:    s_cmp_lt_i32 s3, s7
+; GFX10-NEXT:    s_pack_ll_b32_b16 s12, s12, s14
+; GFX10-NEXT:    s_cselect_b32 s3, s3, s7
+; GFX10-NEXT:    s_cmp_lt_i32 s9, 0
+; GFX10-NEXT:    s_cselect_b32 s9, s9, 0
 ; GFX10-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s3, s10
-; GFX10-NEXT:    s_lshr_b32 s10, s3, 16
-; GFX10-NEXT:    s_sub_i32 s3, s14, s3
-; GFX10-NEXT:    s_sub_i32 s10, s8, s10
-; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s3, s10
-; GFX10-NEXT:    s_sext_i32_i16 s10, s3
+; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s3, s9
+; GFX10-NEXT:    s_lshr_b32 s9, s3, 16
+; GFX10-NEXT:    s_sub_i32 s3, s13, s3
+; GFX10-NEXT:    s_sub_i32 s9, s8, s9
+; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s3, s9
+; GFX10-NEXT:    s_sext_i32_i16 s9, s3
 ; GFX10-NEXT:    s_ashr_i32 s3, s3, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s10, s16
-; GFX10-NEXT:    s_cselect_b32 s10, s10, s16
+; GFX10-NEXT:    s_cmp_gt_i32 s9, s15
+; GFX10-NEXT:    s_cselect_b32 s9, s9, s15
 ; GFX10-NEXT:    s_cmp_gt_i32 s3, s4
 ; GFX10-NEXT:    s_cselect_b32 s3, s3, s4
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s13, s15
-; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s10, s3
-; GFX10-NEXT:    s_sext_i32_i16 s13, s4
-; GFX10-NEXT:    s_sext_i32_i16 s10, s3
+; GFX10-NEXT:    s_sext_i32_i16 s4, s12
+; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s9, s3
+; GFX10-NEXT:    s_ashr_i32 s9, s12, 16
+; GFX10-NEXT:    s_sext_i32_i16 s12, s3
 ; GFX10-NEXT:    s_ashr_i32 s3, s3, 16
-; GFX10-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX10-NEXT:    s_cmp_lt_i32 s10, s13
-; GFX10-NEXT:    s_cselect_b32 s10, s10, s13
-; GFX10-NEXT:    s_cmp_lt_i32 s3, s4
-; GFX10-NEXT:    s_cselect_b32 s3, s3, s4
-; GFX10-NEXT:    s_sext_i32_i16 s4, s2
-; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s10, s3
-; GFX10-NEXT:    s_lshr_b32 s10, s1, 16
-; GFX10-NEXT:    s_lshr_b32 s13, s3, 16
-; GFX10-NEXT:    s_add_i32 s1, s1, s3
-; GFX10-NEXT:    s_add_i32 s10, s10, s13
-; GFX10-NEXT:    s_ashr_i32 s3, s2, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s4, s9
-; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s1, s10
-; GFX10-NEXT:    s_cselect_b32 s13, s4, s9
-; GFX10-NEXT:    s_cmp_gt_i32 s3, s6
-; GFX10-NEXT:    s_cselect_b32 s15, s3, s6
-; GFX10-NEXT:    s_pack_ll_b32_b16 s13, s13, s15
-; GFX10-NEXT:    s_lshr_b32 s15, s13, 16
-; GFX10-NEXT:    s_sub_i32 s11, s11, s13
-; GFX10-NEXT:    s_sub_i32 s12, s12, s15
-; GFX10-NEXT:    s_cmp_lt_i32 s4, s9
-; GFX10-NEXT:    s_cselect_b32 s4, s4, s9
-; GFX10-NEXT:    s_cmp_lt_i32 s3, s6
-; GFX10-NEXT:    s_cselect_b32 s3, s3, s6
-; GFX10-NEXT:    s_sext_i32_i16 s6, s5
+; GFX10-NEXT:    s_cmp_lt_i32 s12, s4
+; GFX10-NEXT:    s_cselect_b32 s4, s12, s4
+; GFX10-NEXT:    s_cmp_lt_i32 s3, s9
+; GFX10-NEXT:    s_sext_i32_i16 s12, s2
+; GFX10-NEXT:    s_cselect_b32 s3, s3, s9
+; GFX10-NEXT:    s_lshr_b32 s9, s1, 16
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s4, s3
-; GFX10-NEXT:    s_lshr_b32 s4, s3, 16
-; GFX10-NEXT:    s_sub_i32 s3, s14, s3
-; GFX10-NEXT:    s_sub_i32 s4, s8, s4
-; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s3, s4
-; GFX10-NEXT:    s_ashr_i32 s4, s5, 16
+; GFX10-NEXT:    s_ashr_i32 s4, s2, 16
+; GFX10-NEXT:    s_lshr_b32 s14, s3, 16
+; GFX10-NEXT:    s_add_i32 s1, s1, s3
+; GFX10-NEXT:    s_add_i32 s9, s9, s14
+; GFX10-NEXT:    s_cmp_gt_i32 s12, s7
+; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s1, s9
+; GFX10-NEXT:    s_cselect_b32 s3, s12, s7
+; GFX10-NEXT:    s_cmp_gt_i32 s4, 0
+; GFX10-NEXT:    s_cselect_b32 s14, s4, 0
+; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s3, s14
+; GFX10-NEXT:    s_lshr_b32 s14, s3, 16
+; GFX10-NEXT:    s_sub_i32 s3, s11, s3
+; GFX10-NEXT:    s_sub_i32 s10, s10, s14
+; GFX10-NEXT:    s_cmp_lt_i32 s12, s7
+; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s3, s10
+; GFX10-NEXT:    s_cselect_b32 s7, s12, s7
+; GFX10-NEXT:    s_cmp_lt_i32 s4, 0
+; GFX10-NEXT:    s_cselect_b32 s4, s4, 0
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s7, s4
+; GFX10-NEXT:    s_lshr_b32 s7, s4, 16
+; GFX10-NEXT:    s_sub_i32 s4, s13, s4
+; GFX10-NEXT:    s_sub_i32 s7, s8, s7
+; GFX10-NEXT:    s_sext_i32_i16 s8, s5
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s4, s7
+; GFX10-NEXT:    s_ashr_i32 s5, s5, 16
+; GFX10-NEXT:    s_sext_i32_i16 s7, s4
+; GFX10-NEXT:    s_ashr_i32 s4, s4, 16
+; GFX10-NEXT:    s_cmp_gt_i32 s7, s8
+; GFX10-NEXT:    s_cselect_b32 s7, s7, s8
+; GFX10-NEXT:    s_cmp_gt_i32 s4, s5
+; GFX10-NEXT:    s_cselect_b32 s4, s4, s5
 ; GFX10-NEXT:    s_sext_i32_i16 s5, s3
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s7, s4
 ; GFX10-NEXT:    s_ashr_i32 s3, s3, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s5, s6
-; GFX10-NEXT:    s_cselect_b32 s5, s5, s6
-; GFX10-NEXT:    s_cmp_gt_i32 s3, s4
-; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s11, s12
-; GFX10-NEXT:    s_cselect_b32 s3, s3, s4
-; GFX10-NEXT:    s_sext_i32_i16 s4, s6
-; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s5, s3
-; GFX10-NEXT:    s_ashr_i32 s5, s6, 16
-; GFX10-NEXT:    s_sext_i32_i16 s6, s3
-; GFX10-NEXT:    s_ashr_i32 s3, s3, 16
-; GFX10-NEXT:    s_cmp_lt_i32 s6, s4
-; GFX10-NEXT:    s_cselect_b32 s4, s6, s4
-; GFX10-NEXT:    s_cmp_lt_i32 s3, s5
-; GFX10-NEXT:    s_cselect_b32 s3, s3, s5
-; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s4, s3
+; GFX10-NEXT:    s_sext_i32_i16 s7, s4
+; GFX10-NEXT:    s_ashr_i32 s4, s4, 16
+; GFX10-NEXT:    s_cmp_lt_i32 s7, s5
+; GFX10-NEXT:    s_cselect_b32 s5, s7, s5
+; GFX10-NEXT:    s_cmp_lt_i32 s4, s3
+; GFX10-NEXT:    s_cselect_b32 s3, s4, s3
 ; GFX10-NEXT:    s_lshr_b32 s4, s2, 16
+; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s5, s3
 ; GFX10-NEXT:    s_lshr_b32 s5, s3, 16
 ; GFX10-NEXT:    s_add_i32 s2, s2, s3
 ; GFX10-NEXT:    s_add_i32 s4, s4, s5
@@ -6438,36 +6358,33 @@ define <4 x float> @v_saddsat_v8i16(<8 x i16> %lhs, <8 x i16> %rhs) {
 ; GFX9-LABEL: v_saddsat_v8i16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    s_movk_i32 s5, 0x8000
-; GFX9-NEXT:    s_pack_ll_b32_b16 s6, 0, 0
-; GFX9-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX9-NEXT:    s_pack_ll_b32_b16 s5, s5, s5
-; GFX9-NEXT:    v_pk_min_i16 v9, v0, s6
-; GFX9-NEXT:    v_pk_sub_i16 v9, s5, v9
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s4
-; GFX9-NEXT:    v_pk_max_i16 v8, v0, s6
-; GFX9-NEXT:    v_pk_sub_i16 v8, s4, v8
-; GFX9-NEXT:    v_pk_max_i16 v4, v9, v4
+; GFX9-NEXT:    v_pk_min_i16 v10, v0, 0
+; GFX9-NEXT:    v_mov_b32_e32 v11, 0x80008000
+; GFX9-NEXT:    v_pk_sub_i16 v10, v11, v10
+; GFX9-NEXT:    v_pk_max_i16 v8, v0, 0
+; GFX9-NEXT:    v_mov_b32_e32 v9, 0x7fff7fff
+; GFX9-NEXT:    v_pk_sub_i16 v8, v9, v8
+; GFX9-NEXT:    v_pk_max_i16 v4, v10, v4
 ; GFX9-NEXT:    v_pk_min_i16 v4, v4, v8
-; GFX9-NEXT:    v_pk_min_i16 v8, v1, s6
+; GFX9-NEXT:    v_pk_min_i16 v8, v1, 0
 ; GFX9-NEXT:    v_pk_add_u16 v0, v0, v4
-; GFX9-NEXT:    v_pk_max_i16 v4, v1, s6
-; GFX9-NEXT:    v_pk_sub_i16 v8, s5, v8
-; GFX9-NEXT:    v_pk_sub_i16 v4, s4, v4
+; GFX9-NEXT:    v_pk_max_i16 v4, v1, 0
+; GFX9-NEXT:    v_pk_sub_i16 v8, v11, v8
+; GFX9-NEXT:    v_pk_sub_i16 v4, v9, v4
 ; GFX9-NEXT:    v_pk_max_i16 v5, v8, v5
 ; GFX9-NEXT:    v_pk_min_i16 v4, v5, v4
-; GFX9-NEXT:    v_pk_min_i16 v5, v2, s6
-; GFX9-NEXT:    v_pk_sub_i16 v5, s5, v5
+; GFX9-NEXT:    v_pk_min_i16 v5, v2, 0
+; GFX9-NEXT:    v_pk_sub_i16 v5, v11, v5
 ; GFX9-NEXT:    v_pk_add_u16 v1, v1, v4
-; GFX9-NEXT:    v_pk_max_i16 v4, v2, s6
-; GFX9-NEXT:    v_pk_sub_i16 v4, s4, v4
+; GFX9-NEXT:    v_pk_max_i16 v4, v2, 0
+; GFX9-NEXT:    v_pk_sub_i16 v4, v9, v4
 ; GFX9-NEXT:    v_pk_max_i16 v5, v5, v6
 ; GFX9-NEXT:    v_pk_min_i16 v4, v5, v4
-; GFX9-NEXT:    v_pk_min_i16 v5, v3, s6
-; GFX9-NEXT:    v_pk_sub_i16 v5, s5, v5
+; GFX9-NEXT:    v_pk_min_i16 v5, v3, 0
+; GFX9-NEXT:    v_pk_sub_i16 v5, v11, v5
 ; GFX9-NEXT:    v_pk_add_u16 v2, v2, v4
-; GFX9-NEXT:    v_pk_max_i16 v4, v3, s6
-; GFX9-NEXT:    v_pk_sub_i16 v4, s4, v4
+; GFX9-NEXT:    v_pk_max_i16 v4, v3, 0
+; GFX9-NEXT:    v_pk_sub_i16 v4, v9, v4
 ; GFX9-NEXT:    v_pk_max_i16 v5, v5, v7
 ; GFX9-NEXT:    v_pk_min_i16 v4, v5, v4
 ; GFX9-NEXT:    v_pk_add_u16 v3, v3, v4
@@ -6477,30 +6394,25 @@ define <4 x float> @v_saddsat_v8i16(<8 x i16> %lhs, <8 x i16> %rhs) {
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, 0, 0
-; GFX10-NEXT:    s_movk_i32 s5, 0x8000
-; GFX10-NEXT:    v_pk_min_i16 v8, v0, s4
-; GFX10-NEXT:    s_pack_ll_b32_b16 s5, s5, s5
-; GFX10-NEXT:    v_pk_min_i16 v11, v1, s4
-; GFX10-NEXT:    v_pk_min_i16 v12, v3, s4
-; GFX10-NEXT:    v_pk_max_i16 v9, v0, s4
-; GFX10-NEXT:    v_pk_sub_i16 v15, s5, v8
-; GFX10-NEXT:    v_pk_min_i16 v8, v2, s4
-; GFX10-NEXT:    v_pk_sub_i16 v11, s5, v11
-; GFX10-NEXT:    v_pk_sub_i16 v12, s5, v12
-; GFX10-NEXT:    v_pk_max_i16 v10, v1, s4
-; GFX10-NEXT:    v_pk_max_i16 v13, v2, s4
-; GFX10-NEXT:    v_pk_sub_i16 v8, s5, v8
-; GFX10-NEXT:    v_pk_max_i16 v14, v3, s4
-; GFX10-NEXT:    s_movk_i32 s6, 0x7fff
-; GFX10-NEXT:    v_pk_max_i16 v4, v15, v4
-; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s6, s6
+; GFX10-NEXT:    v_pk_min_i16 v8, v0, 0
+; GFX10-NEXT:    v_pk_min_i16 v11, v1, 0
+; GFX10-NEXT:    v_pk_min_i16 v12, v3, 0
+; GFX10-NEXT:    v_pk_max_i16 v9, v0, 0
+; GFX10-NEXT:    v_pk_max_i16 v10, v1, 0
+; GFX10-NEXT:    v_pk_sub_i16 v15, 0x80008000, v8
+; GFX10-NEXT:    v_pk_min_i16 v8, v2, 0
+; GFX10-NEXT:    v_pk_sub_i16 v11, 0x80008000, v11
+; GFX10-NEXT:    v_pk_sub_i16 v12, 0x80008000, v12
+; GFX10-NEXT:    v_pk_max_i16 v13, v2, 0
+; GFX10-NEXT:    v_pk_max_i16 v14, v3, 0
+; GFX10-NEXT:    v_pk_sub_i16 v8, 0x80008000, v8
 ; GFX10-NEXT:    v_pk_max_i16 v5, v11, v5
-; GFX10-NEXT:    v_pk_sub_i16 v9, s6, v9
-; GFX10-NEXT:    v_pk_sub_i16 v10, s6, v10
+; GFX10-NEXT:    v_pk_sub_i16 v10, 0x7fff7fff, v10
+; GFX10-NEXT:    v_pk_sub_i16 v9, 0x7fff7fff, v9
+; GFX10-NEXT:    v_pk_max_i16 v4, v15, v4
 ; GFX10-NEXT:    v_pk_max_i16 v6, v8, v6
-; GFX10-NEXT:    v_pk_sub_i16 v11, s6, v13
-; GFX10-NEXT:    v_pk_sub_i16 v8, s6, v14
+; GFX10-NEXT:    v_pk_sub_i16 v11, 0x7fff7fff, v13
+; GFX10-NEXT:    v_pk_sub_i16 v8, 0x7fff7fff, v14
 ; GFX10-NEXT:    v_pk_max_i16 v7, v12, v7
 ; GFX10-NEXT:    v_pk_min_i16 v15, v4, v9
 ; GFX10-NEXT:    v_pk_min_i16 v19, v5, v10
@@ -6814,138 +6726,134 @@ define amdgpu_ps <4 x i32> @s_saddsat_v8i16(<8 x i16> inreg %lhs, <8 x i16> inre
 ;
 ; GFX9-LABEL: s_saddsat_v8i16:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_pack_ll_b32_b16 s10, 0, 0
-; GFX9-NEXT:    s_sext_i32_i16 s13, s10
-; GFX9-NEXT:    s_sext_i32_i16 s11, s0
-; GFX9-NEXT:    s_ashr_i32 s12, s0, 16
+; GFX9-NEXT:    s_sext_i32_i16 s10, s0
+; GFX9-NEXT:    s_ashr_i32 s11, s0, 16
+; GFX9-NEXT:    s_sext_i32_i16 s12, 0
+; GFX9-NEXT:    s_cmp_gt_i32 s10, s12
+; GFX9-NEXT:    s_cselect_b32 s13, s10, s12
+; GFX9-NEXT:    s_cmp_gt_i32 s11, 0
+; GFX9-NEXT:    s_cselect_b32 s14, s11, 0
+; GFX9-NEXT:    s_pack_ll_b32_b16 s13, s13, s14
+; GFX9-NEXT:    s_mov_b32 s8, 0x7fff7fff
+; GFX9-NEXT:    s_lshr_b32 s15, s13, 16
+; GFX9-NEXT:    s_movk_i32 s14, 0x7fff
+; GFX9-NEXT:    s_sub_i32 s13, s8, s13
+; GFX9-NEXT:    s_sub_i32 s15, s14, s15
+; GFX9-NEXT:    s_cmp_lt_i32 s10, s12
+; GFX9-NEXT:    s_cselect_b32 s10, s10, s12
+; GFX9-NEXT:    s_cmp_lt_i32 s11, 0
+; GFX9-NEXT:    s_cselect_b32 s11, s11, 0
+; GFX9-NEXT:    s_pack_ll_b32_b16 s10, s10, s11
+; GFX9-NEXT:    s_pack_ll_b32_b16 s13, s13, s15
+; GFX9-NEXT:    s_mov_b32 s9, 0x80008000
+; GFX9-NEXT:    s_lshr_b32 s15, s10, 16
+; GFX9-NEXT:    s_mov_b32 s11, 0x8000
+; GFX9-NEXT:    s_sub_i32 s10, s9, s10
+; GFX9-NEXT:    s_sub_i32 s15, s11, s15
+; GFX9-NEXT:    s_pack_ll_b32_b16 s10, s10, s15
+; GFX9-NEXT:    s_sext_i32_i16 s15, s10
+; GFX9-NEXT:    s_sext_i32_i16 s16, s4
 ; GFX9-NEXT:    s_ashr_i32 s10, s10, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s11, s13
-; GFX9-NEXT:    s_cselect_b32 s14, s11, s13
-; GFX9-NEXT:    s_cmp_gt_i32 s12, s10
-; GFX9-NEXT:    s_movk_i32 s8, 0x7fff
-; GFX9-NEXT:    s_cselect_b32 s15, s12, s10
-; GFX9-NEXT:    s_pack_ll_b32_b16 s14, s14, s15
-; GFX9-NEXT:    s_pack_ll_b32_b16 s8, s8, s8
-; GFX9-NEXT:    s_lshr_b32 s16, s14, 16
-; GFX9-NEXT:    s_lshr_b32 s15, s8, 16
-; GFX9-NEXT:    s_sub_i32 s14, s8, s14
-; GFX9-NEXT:    s_sub_i32 s16, s15, s16
-; GFX9-NEXT:    s_cmp_lt_i32 s11, s13
-; GFX9-NEXT:    s_cselect_b32 s11, s11, s13
-; GFX9-NEXT:    s_cmp_lt_i32 s12, s10
-; GFX9-NEXT:    s_movk_i32 s9, 0x8000
-; GFX9-NEXT:    s_cselect_b32 s12, s12, s10
-; GFX9-NEXT:    s_pack_ll_b32_b16 s9, s9, s9
-; GFX9-NEXT:    s_pack_ll_b32_b16 s11, s11, s12
-; GFX9-NEXT:    s_pack_ll_b32_b16 s14, s14, s16
-; GFX9-NEXT:    s_lshr_b32 s16, s11, 16
-; GFX9-NEXT:    s_lshr_b32 s12, s9, 16
-; GFX9-NEXT:    s_sub_i32 s11, s9, s11
-; GFX9-NEXT:    s_sub_i32 s16, s12, s16
-; GFX9-NEXT:    s_pack_ll_b32_b16 s11, s11, s16
-; GFX9-NEXT:    s_sext_i32_i16 s16, s11
-; GFX9-NEXT:    s_sext_i32_i16 s17, s4
-; GFX9-NEXT:    s_ashr_i32 s11, s11, 16
 ; GFX9-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s16, s17
-; GFX9-NEXT:    s_cselect_b32 s16, s16, s17
-; GFX9-NEXT:    s_cmp_gt_i32 s11, s4
-; GFX9-NEXT:    s_cselect_b32 s4, s11, s4
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s16, s4
-; GFX9-NEXT:    s_sext_i32_i16 s11, s4
-; GFX9-NEXT:    s_sext_i32_i16 s16, s14
+; GFX9-NEXT:    s_cmp_gt_i32 s15, s16
+; GFX9-NEXT:    s_cselect_b32 s15, s15, s16
+; GFX9-NEXT:    s_cmp_gt_i32 s10, s4
+; GFX9-NEXT:    s_cselect_b32 s4, s10, s4
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s15, s4
+; GFX9-NEXT:    s_sext_i32_i16 s10, s4
+; GFX9-NEXT:    s_sext_i32_i16 s15, s13
 ; GFX9-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX9-NEXT:    s_ashr_i32 s14, s14, 16
-; GFX9-NEXT:    s_cmp_lt_i32 s11, s16
-; GFX9-NEXT:    s_cselect_b32 s11, s11, s16
-; GFX9-NEXT:    s_cmp_lt_i32 s4, s14
-; GFX9-NEXT:    s_cselect_b32 s4, s4, s14
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s11, s4
-; GFX9-NEXT:    s_lshr_b32 s11, s0, 16
-; GFX9-NEXT:    s_lshr_b32 s14, s4, 16
-; GFX9-NEXT:    s_add_i32 s0, s0, s4
-; GFX9-NEXT:    s_add_i32 s11, s11, s14
-; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s11
-; GFX9-NEXT:    s_sext_i32_i16 s4, s1
-; GFX9-NEXT:    s_ashr_i32 s11, s1, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s4, s13
-; GFX9-NEXT:    s_cselect_b32 s14, s4, s13
-; GFX9-NEXT:    s_cmp_gt_i32 s11, s10
-; GFX9-NEXT:    s_cselect_b32 s16, s11, s10
-; GFX9-NEXT:    s_pack_ll_b32_b16 s14, s14, s16
-; GFX9-NEXT:    s_lshr_b32 s16, s14, 16
-; GFX9-NEXT:    s_sub_i32 s14, s8, s14
-; GFX9-NEXT:    s_sub_i32 s16, s15, s16
+; GFX9-NEXT:    s_ashr_i32 s13, s13, 16
+; GFX9-NEXT:    s_cmp_lt_i32 s10, s15
+; GFX9-NEXT:    s_cselect_b32 s10, s10, s15
 ; GFX9-NEXT:    s_cmp_lt_i32 s4, s13
 ; GFX9-NEXT:    s_cselect_b32 s4, s4, s13
-; GFX9-NEXT:    s_cmp_lt_i32 s11, s10
-; GFX9-NEXT:    s_cselect_b32 s11, s11, s10
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s11
-; GFX9-NEXT:    s_lshr_b32 s11, s4, 16
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s10, s4
+; GFX9-NEXT:    s_lshr_b32 s10, s0, 16
+; GFX9-NEXT:    s_lshr_b32 s13, s4, 16
+; GFX9-NEXT:    s_add_i32 s0, s0, s4
+; GFX9-NEXT:    s_add_i32 s10, s10, s13
+; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s10
+; GFX9-NEXT:    s_sext_i32_i16 s4, s1
+; GFX9-NEXT:    s_ashr_i32 s10, s1, 16
+; GFX9-NEXT:    s_cmp_gt_i32 s4, s12
+; GFX9-NEXT:    s_cselect_b32 s13, s4, s12
+; GFX9-NEXT:    s_cmp_gt_i32 s10, 0
+; GFX9-NEXT:    s_cselect_b32 s15, s10, 0
+; GFX9-NEXT:    s_pack_ll_b32_b16 s13, s13, s15
+; GFX9-NEXT:    s_lshr_b32 s15, s13, 16
+; GFX9-NEXT:    s_sub_i32 s13, s8, s13
+; GFX9-NEXT:    s_sub_i32 s15, s14, s15
+; GFX9-NEXT:    s_cmp_lt_i32 s4, s12
+; GFX9-NEXT:    s_cselect_b32 s4, s4, s12
+; GFX9-NEXT:    s_cmp_lt_i32 s10, 0
+; GFX9-NEXT:    s_cselect_b32 s10, s10, 0
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s10
+; GFX9-NEXT:    s_lshr_b32 s10, s4, 16
 ; GFX9-NEXT:    s_sub_i32 s4, s9, s4
-; GFX9-NEXT:    s_sub_i32 s11, s12, s11
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s11
-; GFX9-NEXT:    s_pack_ll_b32_b16 s14, s14, s16
-; GFX9-NEXT:    s_sext_i32_i16 s11, s4
-; GFX9-NEXT:    s_sext_i32_i16 s16, s5
+; GFX9-NEXT:    s_sub_i32 s10, s11, s10
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s10
+; GFX9-NEXT:    s_pack_ll_b32_b16 s13, s13, s15
+; GFX9-NEXT:    s_sext_i32_i16 s10, s4
+; GFX9-NEXT:    s_sext_i32_i16 s15, s5
 ; GFX9-NEXT:    s_ashr_i32 s4, s4, 16
 ; GFX9-NEXT:    s_ashr_i32 s5, s5, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s11, s16
-; GFX9-NEXT:    s_cselect_b32 s11, s11, s16
+; GFX9-NEXT:    s_cmp_gt_i32 s10, s15
+; GFX9-NEXT:    s_cselect_b32 s10, s10, s15
 ; GFX9-NEXT:    s_cmp_gt_i32 s4, s5
 ; GFX9-NEXT:    s_cselect_b32 s4, s4, s5
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s11, s4
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s10, s4
 ; GFX9-NEXT:    s_sext_i32_i16 s5, s4
-; GFX9-NEXT:    s_sext_i32_i16 s11, s14
+; GFX9-NEXT:    s_sext_i32_i16 s10, s13
 ; GFX9-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX9-NEXT:    s_ashr_i32 s14, s14, 16
-; GFX9-NEXT:    s_cmp_lt_i32 s5, s11
-; GFX9-NEXT:    s_cselect_b32 s5, s5, s11
-; GFX9-NEXT:    s_cmp_lt_i32 s4, s14
-; GFX9-NEXT:    s_cselect_b32 s4, s4, s14
+; GFX9-NEXT:    s_ashr_i32 s13, s13, 16
+; GFX9-NEXT:    s_cmp_lt_i32 s5, s10
+; GFX9-NEXT:    s_cselect_b32 s5, s5, s10
+; GFX9-NEXT:    s_cmp_lt_i32 s4, s13
+; GFX9-NEXT:    s_cselect_b32 s4, s4, s13
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s5, s4
 ; GFX9-NEXT:    s_lshr_b32 s5, s1, 16
-; GFX9-NEXT:    s_lshr_b32 s11, s4, 16
+; GFX9-NEXT:    s_lshr_b32 s10, s4, 16
 ; GFX9-NEXT:    s_add_i32 s1, s1, s4
-; GFX9-NEXT:    s_add_i32 s5, s5, s11
+; GFX9-NEXT:    s_add_i32 s5, s5, s10
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s5
 ; GFX9-NEXT:    s_sext_i32_i16 s4, s2
 ; GFX9-NEXT:    s_ashr_i32 s5, s2, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s4, s13
-; GFX9-NEXT:    s_cselect_b32 s11, s4, s13
-; GFX9-NEXT:    s_cmp_gt_i32 s5, s10
-; GFX9-NEXT:    s_cselect_b32 s14, s5, s10
-; GFX9-NEXT:    s_pack_ll_b32_b16 s11, s11, s14
-; GFX9-NEXT:    s_lshr_b32 s14, s11, 16
-; GFX9-NEXT:    s_sub_i32 s11, s8, s11
-; GFX9-NEXT:    s_sub_i32 s14, s15, s14
-; GFX9-NEXT:    s_cmp_lt_i32 s4, s13
-; GFX9-NEXT:    s_cselect_b32 s4, s4, s13
-; GFX9-NEXT:    s_cmp_lt_i32 s5, s10
-; GFX9-NEXT:    s_cselect_b32 s5, s5, s10
+; GFX9-NEXT:    s_cmp_gt_i32 s4, s12
+; GFX9-NEXT:    s_cselect_b32 s10, s4, s12
+; GFX9-NEXT:    s_cmp_gt_i32 s5, 0
+; GFX9-NEXT:    s_cselect_b32 s13, s5, 0
+; GFX9-NEXT:    s_pack_ll_b32_b16 s10, s10, s13
+; GFX9-NEXT:    s_lshr_b32 s13, s10, 16
+; GFX9-NEXT:    s_sub_i32 s10, s8, s10
+; GFX9-NEXT:    s_sub_i32 s13, s14, s13
+; GFX9-NEXT:    s_cmp_lt_i32 s4, s12
+; GFX9-NEXT:    s_cselect_b32 s4, s4, s12
+; GFX9-NEXT:    s_cmp_lt_i32 s5, 0
+; GFX9-NEXT:    s_cselect_b32 s5, s5, 0
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
 ; GFX9-NEXT:    s_lshr_b32 s5, s4, 16
 ; GFX9-NEXT:    s_sub_i32 s4, s9, s4
-; GFX9-NEXT:    s_sub_i32 s5, s12, s5
+; GFX9-NEXT:    s_sub_i32 s5, s11, s5
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
-; GFX9-NEXT:    s_pack_ll_b32_b16 s11, s11, s14
+; GFX9-NEXT:    s_pack_ll_b32_b16 s10, s10, s13
 ; GFX9-NEXT:    s_sext_i32_i16 s5, s4
-; GFX9-NEXT:    s_sext_i32_i16 s14, s6
+; GFX9-NEXT:    s_sext_i32_i16 s13, s6
 ; GFX9-NEXT:    s_ashr_i32 s4, s4, 16
 ; GFX9-NEXT:    s_ashr_i32 s6, s6, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s5, s14
-; GFX9-NEXT:    s_cselect_b32 s5, s5, s14
+; GFX9-NEXT:    s_cmp_gt_i32 s5, s13
+; GFX9-NEXT:    s_cselect_b32 s5, s5, s13
 ; GFX9-NEXT:    s_cmp_gt_i32 s4, s6
 ; GFX9-NEXT:    s_cselect_b32 s4, s4, s6
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s5, s4
 ; GFX9-NEXT:    s_sext_i32_i16 s5, s4
-; GFX9-NEXT:    s_sext_i32_i16 s6, s11
+; GFX9-NEXT:    s_sext_i32_i16 s6, s10
 ; GFX9-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX9-NEXT:    s_ashr_i32 s11, s11, 16
+; GFX9-NEXT:    s_ashr_i32 s10, s10, 16
 ; GFX9-NEXT:    s_cmp_lt_i32 s5, s6
 ; GFX9-NEXT:    s_cselect_b32 s5, s5, s6
-; GFX9-NEXT:    s_cmp_lt_i32 s4, s11
-; GFX9-NEXT:    s_cselect_b32 s4, s4, s11
+; GFX9-NEXT:    s_cmp_lt_i32 s4, s10
+; GFX9-NEXT:    s_cselect_b32 s4, s4, s10
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s5, s4
 ; GFX9-NEXT:    s_lshr_b32 s5, s2, 16
 ; GFX9-NEXT:    s_lshr_b32 s6, s4, 16
@@ -6954,22 +6862,22 @@ define amdgpu_ps <4 x i32> @s_saddsat_v8i16(<8 x i16> inreg %lhs, <8 x i16> inre
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s5
 ; GFX9-NEXT:    s_sext_i32_i16 s4, s3
 ; GFX9-NEXT:    s_ashr_i32 s5, s3, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s4, s13
-; GFX9-NEXT:    s_cselect_b32 s6, s4, s13
-; GFX9-NEXT:    s_cmp_gt_i32 s5, s10
-; GFX9-NEXT:    s_cselect_b32 s11, s5, s10
-; GFX9-NEXT:    s_pack_ll_b32_b16 s6, s6, s11
-; GFX9-NEXT:    s_lshr_b32 s11, s6, 16
+; GFX9-NEXT:    s_cmp_gt_i32 s4, s12
+; GFX9-NEXT:    s_cselect_b32 s6, s4, s12
+; GFX9-NEXT:    s_cmp_gt_i32 s5, 0
+; GFX9-NEXT:    s_cselect_b32 s10, s5, 0
+; GFX9-NEXT:    s_pack_ll_b32_b16 s6, s6, s10
+; GFX9-NEXT:    s_lshr_b32 s10, s6, 16
 ; GFX9-NEXT:    s_sub_i32 s6, s8, s6
-; GFX9-NEXT:    s_sub_i32 s8, s15, s11
-; GFX9-NEXT:    s_cmp_lt_i32 s4, s13
-; GFX9-NEXT:    s_cselect_b32 s4, s4, s13
-; GFX9-NEXT:    s_cmp_lt_i32 s5, s10
-; GFX9-NEXT:    s_cselect_b32 s5, s5, s10
+; GFX9-NEXT:    s_sub_i32 s8, s14, s10
+; GFX9-NEXT:    s_cmp_lt_i32 s4, s12
+; GFX9-NEXT:    s_cselect_b32 s4, s4, s12
+; GFX9-NEXT:    s_cmp_lt_i32 s5, 0
+; GFX9-NEXT:    s_cselect_b32 s5, s5, 0
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
 ; GFX9-NEXT:    s_lshr_b32 s5, s4, 16
 ; GFX9-NEXT:    s_sub_i32 s4, s9, s4
-; GFX9-NEXT:    s_sub_i32 s5, s12, s5
+; GFX9-NEXT:    s_sub_i32 s5, s11, s5
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s6, s6, s8
 ; GFX9-NEXT:    s_sext_i32_i16 s5, s4
@@ -6999,188 +6907,184 @@ define amdgpu_ps <4 x i32> @s_saddsat_v8i16(<8 x i16> inreg %lhs, <8 x i16> inre
 ;
 ; GFX10-LABEL: s_saddsat_v8i16:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_pack_ll_b32_b16 s8, 0, 0
-; GFX10-NEXT:    s_sext_i32_i16 s9, s0
-; GFX10-NEXT:    s_sext_i32_i16 s11, s8
+; GFX10-NEXT:    s_sext_i32_i16 s8, s0
+; GFX10-NEXT:    s_sext_i32_i16 s9, 0
 ; GFX10-NEXT:    s_ashr_i32 s10, s0, 16
-; GFX10-NEXT:    s_ashr_i32 s8, s8, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s9, s11
-; GFX10-NEXT:    s_movk_i32 s13, 0x7fff
-; GFX10-NEXT:    s_cselect_b32 s12, s9, s11
-; GFX10-NEXT:    s_cmp_gt_i32 s10, s8
-; GFX10-NEXT:    s_pack_ll_b32_b16 s13, s13, s13
-; GFX10-NEXT:    s_cselect_b32 s14, s10, s8
-; GFX10-NEXT:    s_movk_i32 s16, 0x8000
-; GFX10-NEXT:    s_pack_ll_b32_b16 s12, s12, s14
-; GFX10-NEXT:    s_lshr_b32 s14, s13, 16
-; GFX10-NEXT:    s_lshr_b32 s15, s12, 16
-; GFX10-NEXT:    s_sub_i32 s12, s13, s12
-; GFX10-NEXT:    s_sub_i32 s15, s14, s15
-; GFX10-NEXT:    s_cmp_lt_i32 s9, s11
-; GFX10-NEXT:    s_pack_ll_b32_b16 s16, s16, s16
-; GFX10-NEXT:    s_cselect_b32 s9, s9, s11
-; GFX10-NEXT:    s_cmp_lt_i32 s10, s8
-; GFX10-NEXT:    s_sext_i32_i16 s18, s4
-; GFX10-NEXT:    s_cselect_b32 s10, s10, s8
-; GFX10-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s9, s9, s10
-; GFX10-NEXT:    s_lshr_b32 s10, s16, 16
-; GFX10-NEXT:    s_lshr_b32 s17, s9, 16
-; GFX10-NEXT:    s_sub_i32 s9, s16, s9
-; GFX10-NEXT:    s_sub_i32 s17, s10, s17
+; GFX10-NEXT:    s_cmp_gt_i32 s8, s9
+; GFX10-NEXT:    s_mov_b32 s13, 0x7fff7fff
+; GFX10-NEXT:    s_cselect_b32 s11, s8, s9
+; GFX10-NEXT:    s_cmp_gt_i32 s10, 0
+; GFX10-NEXT:    s_mov_b32 s15, 0x80008000
+; GFX10-NEXT:    s_cselect_b32 s12, s10, 0
+; GFX10-NEXT:    s_sext_i32_i16 s17, s4
+; GFX10-NEXT:    s_pack_ll_b32_b16 s11, s11, s12
+; GFX10-NEXT:    s_movk_i32 s12, 0x7fff
+; GFX10-NEXT:    s_lshr_b32 s14, s11, 16
+; GFX10-NEXT:    s_sub_i32 s11, s13, s11
+; GFX10-NEXT:    s_sub_i32 s14, s12, s14
+; GFX10-NEXT:    s_cmp_lt_i32 s8, s9
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_pack_ll_b32_b16 s9, s9, s17
-; GFX10-NEXT:    s_sext_i32_i16 s17, s9
-; GFX10-NEXT:    s_ashr_i32 s9, s9, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s17, s18
-; GFX10-NEXT:    s_cselect_b32 s17, s17, s18
-; GFX10-NEXT:    s_cmp_gt_i32 s9, s4
-; GFX10-NEXT:    s_sext_i32_i16 s18, s5
-; GFX10-NEXT:    s_cselect_b32 s4, s9, s4
-; GFX10-NEXT:    s_pack_ll_b32_b16 s9, s12, s15
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s17, s4
-; GFX10-NEXT:    s_sext_i32_i16 s15, s9
-; GFX10-NEXT:    s_sext_i32_i16 s12, s4
+; GFX10-NEXT:    s_cselect_b32 s8, s8, s9
+; GFX10-NEXT:    s_cmp_lt_i32 s10, 0
+; GFX10-NEXT:    s_cselect_b32 s10, s10, 0
 ; GFX10-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX10-NEXT:    s_ashr_i32 s9, s9, 16
-; GFX10-NEXT:    s_cmp_lt_i32 s12, s15
-; GFX10-NEXT:    s_cselect_b32 s12, s12, s15
-; GFX10-NEXT:    s_cmp_lt_i32 s4, s9
-; GFX10-NEXT:    s_cselect_b32 s4, s4, s9
-; GFX10-NEXT:    s_lshr_b32 s9, s0, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s12, s4
-; GFX10-NEXT:    s_lshr_b32 s12, s4, 16
+; GFX10-NEXT:    s_pack_ll_b32_b16 s8, s8, s10
+; GFX10-NEXT:    s_mov_b32 s10, 0x8000
+; GFX10-NEXT:    s_lshr_b32 s16, s8, 16
+; GFX10-NEXT:    s_sub_i32 s8, s15, s8
+; GFX10-NEXT:    s_sub_i32 s16, s10, s16
+; GFX10-NEXT:    s_pack_ll_b32_b16 s8, s8, s16
+; GFX10-NEXT:    s_sext_i32_i16 s16, s8
+; GFX10-NEXT:    s_ashr_i32 s8, s8, 16
+; GFX10-NEXT:    s_cmp_gt_i32 s16, s17
+; GFX10-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX10-NEXT:    s_cmp_gt_i32 s8, s4
+; GFX10-NEXT:    s_sext_i32_i16 s17, s5
+; GFX10-NEXT:    s_cselect_b32 s4, s8, s4
+; GFX10-NEXT:    s_pack_ll_b32_b16 s8, s11, s14
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s16, s4
+; GFX10-NEXT:    s_sext_i32_i16 s14, s8
+; GFX10-NEXT:    s_sext_i32_i16 s11, s4
+; GFX10-NEXT:    s_ashr_i32 s4, s4, 16
+; GFX10-NEXT:    s_ashr_i32 s8, s8, 16
+; GFX10-NEXT:    s_cmp_lt_i32 s11, s14
+; GFX10-NEXT:    s_cselect_b32 s11, s11, s14
+; GFX10-NEXT:    s_cmp_lt_i32 s4, s8
+; GFX10-NEXT:    s_cselect_b32 s4, s4, s8
+; GFX10-NEXT:    s_lshr_b32 s8, s0, 16
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s11, s4
+; GFX10-NEXT:    s_lshr_b32 s11, s4, 16
 ; GFX10-NEXT:    s_add_i32 s0, s0, s4
 ; GFX10-NEXT:    s_sext_i32_i16 s4, s1
-; GFX10-NEXT:    s_add_i32 s9, s9, s12
-; GFX10-NEXT:    s_ashr_i32 s12, s1, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s4, s11
-; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s9
-; GFX10-NEXT:    s_cselect_b32 s15, s4, s11
-; GFX10-NEXT:    s_cmp_gt_i32 s12, s8
-; GFX10-NEXT:    s_cselect_b32 s17, s12, s8
-; GFX10-NEXT:    s_pack_ll_b32_b16 s15, s15, s17
-; GFX10-NEXT:    s_lshr_b32 s17, s15, 16
-; GFX10-NEXT:    s_sub_i32 s15, s13, s15
-; GFX10-NEXT:    s_sub_i32 s17, s14, s17
-; GFX10-NEXT:    s_cmp_lt_i32 s4, s11
-; GFX10-NEXT:    s_cselect_b32 s4, s4, s11
-; GFX10-NEXT:    s_cmp_lt_i32 s12, s8
-; GFX10-NEXT:    s_cselect_b32 s12, s12, s8
+; GFX10-NEXT:    s_add_i32 s8, s8, s11
+; GFX10-NEXT:    s_ashr_i32 s11, s1, 16
+; GFX10-NEXT:    s_cmp_gt_i32 s4, s9
+; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s8
+; GFX10-NEXT:    s_cselect_b32 s14, s4, s9
+; GFX10-NEXT:    s_cmp_gt_i32 s11, 0
+; GFX10-NEXT:    s_cselect_b32 s16, s11, 0
+; GFX10-NEXT:    s_pack_ll_b32_b16 s14, s14, s16
+; GFX10-NEXT:    s_lshr_b32 s16, s14, 16
+; GFX10-NEXT:    s_sub_i32 s14, s13, s14
+; GFX10-NEXT:    s_sub_i32 s16, s12, s16
+; GFX10-NEXT:    s_cmp_lt_i32 s4, s9
+; GFX10-NEXT:    s_pack_ll_b32_b16 s14, s14, s16
+; GFX10-NEXT:    s_cselect_b32 s4, s4, s9
+; GFX10-NEXT:    s_cmp_lt_i32 s11, 0
+; GFX10-NEXT:    s_cselect_b32 s11, s11, 0
 ; GFX10-NEXT:    s_ashr_i32 s5, s5, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s4, s12
-; GFX10-NEXT:    s_lshr_b32 s12, s4, 16
-; GFX10-NEXT:    s_sub_i32 s4, s16, s4
-; GFX10-NEXT:    s_sub_i32 s12, s10, s12
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s4, s12
-; GFX10-NEXT:    s_sext_i32_i16 s12, s4
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s4, s11
+; GFX10-NEXT:    s_lshr_b32 s11, s4, 16
+; GFX10-NEXT:    s_sub_i32 s4, s15, s4
+; GFX10-NEXT:    s_sub_i32 s11, s10, s11
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s4, s11
+; GFX10-NEXT:    s_sext_i32_i16 s11, s4
 ; GFX10-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s12, s18
-; GFX10-NEXT:    s_cselect_b32 s12, s12, s18
+; GFX10-NEXT:    s_cmp_gt_i32 s11, s17
+; GFX10-NEXT:    s_cselect_b32 s11, s11, s17
 ; GFX10-NEXT:    s_cmp_gt_i32 s4, s5
-; GFX10-NEXT:    s_sext_i32_i16 s18, s6
+; GFX10-NEXT:    s_sext_i32_i16 s17, s6
 ; GFX10-NEXT:    s_cselect_b32 s4, s4, s5
-; GFX10-NEXT:    s_pack_ll_b32_b16 s5, s15, s17
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s12, s4
-; GFX10-NEXT:    s_sext_i32_i16 s15, s5
-; GFX10-NEXT:    s_sext_i32_i16 s12, s4
+; GFX10-NEXT:    s_sext_i32_i16 s5, s14
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s11, s4
+; GFX10-NEXT:    s_ashr_i32 s11, s14, 16
+; GFX10-NEXT:    s_sext_i32_i16 s14, s4
 ; GFX10-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX10-NEXT:    s_ashr_i32 s5, s5, 16
-; GFX10-NEXT:    s_cmp_lt_i32 s12, s15
-; GFX10-NEXT:    s_cselect_b32 s12, s12, s15
-; GFX10-NEXT:    s_cmp_lt_i32 s4, s5
-; GFX10-NEXT:    s_cselect_b32 s4, s4, s5
-; GFX10-NEXT:    s_lshr_b32 s5, s1, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s12, s4
-; GFX10-NEXT:    s_lshr_b32 s12, s4, 16
-; GFX10-NEXT:    s_add_i32 s1, s1, s4
-; GFX10-NEXT:    s_sext_i32_i16 s4, s2
-; GFX10-NEXT:    s_add_i32 s5, s5, s12
-; GFX10-NEXT:    s_ashr_i32 s12, s2, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s4, s11
-; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s1, s5
-; GFX10-NEXT:    s_cselect_b32 s15, s4, s11
-; GFX10-NEXT:    s_cmp_gt_i32 s12, s8
-; GFX10-NEXT:    s_cselect_b32 s17, s12, s8
-; GFX10-NEXT:    s_pack_ll_b32_b16 s15, s15, s17
-; GFX10-NEXT:    s_lshr_b32 s17, s15, 16
-; GFX10-NEXT:    s_sub_i32 s15, s13, s15
-; GFX10-NEXT:    s_sub_i32 s17, s14, s17
+; GFX10-NEXT:    s_cmp_lt_i32 s14, s5
+; GFX10-NEXT:    s_cselect_b32 s5, s14, s5
 ; GFX10-NEXT:    s_cmp_lt_i32 s4, s11
-; GFX10-NEXT:    s_pack_ll_b32_b16 s15, s15, s17
+; GFX10-NEXT:    s_sext_i32_i16 s14, s2
 ; GFX10-NEXT:    s_cselect_b32 s4, s4, s11
-; GFX10-NEXT:    s_cmp_lt_i32 s12, s8
-; GFX10-NEXT:    s_cselect_b32 s12, s12, s8
+; GFX10-NEXT:    s_lshr_b32 s11, s1, 16
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s5, s4
+; GFX10-NEXT:    s_ashr_i32 s5, s2, 16
+; GFX10-NEXT:    s_lshr_b32 s16, s4, 16
+; GFX10-NEXT:    s_add_i32 s1, s1, s4
+; GFX10-NEXT:    s_add_i32 s11, s11, s16
+; GFX10-NEXT:    s_cmp_gt_i32 s14, s9
+; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s1, s11
+; GFX10-NEXT:    s_cselect_b32 s4, s14, s9
+; GFX10-NEXT:    s_cmp_gt_i32 s5, 0
+; GFX10-NEXT:    s_cselect_b32 s16, s5, 0
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s4, s16
+; GFX10-NEXT:    s_lshr_b32 s16, s4, 16
+; GFX10-NEXT:    s_sub_i32 s4, s13, s4
+; GFX10-NEXT:    s_sub_i32 s16, s12, s16
+; GFX10-NEXT:    s_cmp_lt_i32 s14, s9
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s4, s16
+; GFX10-NEXT:    s_cselect_b32 s14, s14, s9
+; GFX10-NEXT:    s_cmp_lt_i32 s5, 0
+; GFX10-NEXT:    s_cselect_b32 s5, s5, 0
 ; GFX10-NEXT:    s_ashr_i32 s6, s6, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s4, s12
-; GFX10-NEXT:    s_lshr_b32 s12, s4, 16
-; GFX10-NEXT:    s_sub_i32 s4, s16, s4
-; GFX10-NEXT:    s_sub_i32 s12, s10, s12
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s4, s12
-; GFX10-NEXT:    s_sext_i32_i16 s12, s4
-; GFX10-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s12, s18
-; GFX10-NEXT:    s_cselect_b32 s12, s12, s18
-; GFX10-NEXT:    s_cmp_gt_i32 s4, s6
-; GFX10-NEXT:    s_cselect_b32 s4, s4, s6
-; GFX10-NEXT:    s_sext_i32_i16 s6, s15
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s12, s4
-; GFX10-NEXT:    s_ashr_i32 s12, s15, 16
-; GFX10-NEXT:    s_sext_i32_i16 s15, s4
+; GFX10-NEXT:    s_pack_ll_b32_b16 s5, s14, s5
+; GFX10-NEXT:    s_lshr_b32 s14, s5, 16
+; GFX10-NEXT:    s_sub_i32 s5, s15, s5
+; GFX10-NEXT:    s_sub_i32 s14, s10, s14
+; GFX10-NEXT:    s_pack_ll_b32_b16 s5, s5, s14
+; GFX10-NEXT:    s_sext_i32_i16 s14, s5
+; GFX10-NEXT:    s_ashr_i32 s5, s5, 16
+; GFX10-NEXT:    s_cmp_gt_i32 s14, s17
+; GFX10-NEXT:    s_cselect_b32 s14, s14, s17
+; GFX10-NEXT:    s_cmp_gt_i32 s5, s6
+; GFX10-NEXT:    s_cselect_b32 s5, s5, s6
+; GFX10-NEXT:    s_sext_i32_i16 s6, s4
+; GFX10-NEXT:    s_pack_ll_b32_b16 s5, s14, s5
 ; GFX10-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX10-NEXT:    s_cmp_lt_i32 s15, s6
-; GFX10-NEXT:    s_cselect_b32 s6, s15, s6
-; GFX10-NEXT:    s_cmp_lt_i32 s4, s12
-; GFX10-NEXT:    s_sext_i32_i16 s15, s3
-; GFX10-NEXT:    s_cselect_b32 s4, s4, s12
-; GFX10-NEXT:    s_lshr_b32 s12, s2, 16
+; GFX10-NEXT:    s_sext_i32_i16 s14, s5
+; GFX10-NEXT:    s_ashr_i32 s5, s5, 16
+; GFX10-NEXT:    s_cmp_lt_i32 s14, s6
+; GFX10-NEXT:    s_cselect_b32 s6, s14, s6
+; GFX10-NEXT:    s_cmp_lt_i32 s5, s4
+; GFX10-NEXT:    s_sext_i32_i16 s14, s3
+; GFX10-NEXT:    s_cselect_b32 s4, s5, s4
+; GFX10-NEXT:    s_lshr_b32 s5, s2, 16
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s6, s4
 ; GFX10-NEXT:    s_ashr_i32 s6, s3, 16
-; GFX10-NEXT:    s_lshr_b32 s17, s4, 16
+; GFX10-NEXT:    s_lshr_b32 s16, s4, 16
 ; GFX10-NEXT:    s_add_i32 s2, s2, s4
-; GFX10-NEXT:    s_add_i32 s12, s12, s17
-; GFX10-NEXT:    s_cmp_gt_i32 s15, s11
-; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s2, s12
-; GFX10-NEXT:    s_cselect_b32 s4, s15, s11
-; GFX10-NEXT:    s_cmp_gt_i32 s6, s8
-; GFX10-NEXT:    s_cselect_b32 s17, s6, s8
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s4, s17
-; GFX10-NEXT:    s_lshr_b32 s17, s4, 16
+; GFX10-NEXT:    s_add_i32 s5, s5, s16
+; GFX10-NEXT:    s_cmp_gt_i32 s14, s9
+; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s2, s5
+; GFX10-NEXT:    s_cselect_b32 s4, s14, s9
+; GFX10-NEXT:    s_cmp_gt_i32 s6, 0
+; GFX10-NEXT:    s_cselect_b32 s16, s6, 0
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s4, s16
+; GFX10-NEXT:    s_lshr_b32 s16, s4, 16
 ; GFX10-NEXT:    s_sub_i32 s4, s13, s4
-; GFX10-NEXT:    s_sub_i32 s13, s14, s17
-; GFX10-NEXT:    s_cmp_lt_i32 s15, s11
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s4, s13
-; GFX10-NEXT:    s_cselect_b32 s11, s15, s11
-; GFX10-NEXT:    s_cmp_lt_i32 s6, s8
-; GFX10-NEXT:    s_cselect_b32 s6, s6, s8
-; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s11, s6
-; GFX10-NEXT:    s_lshr_b32 s8, s6, 16
-; GFX10-NEXT:    s_sub_i32 s6, s16, s6
-; GFX10-NEXT:    s_sub_i32 s8, s10, s8
+; GFX10-NEXT:    s_sub_i32 s12, s12, s16
+; GFX10-NEXT:    s_cmp_lt_i32 s14, s9
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s4, s12
+; GFX10-NEXT:    s_cselect_b32 s9, s14, s9
+; GFX10-NEXT:    s_cmp_lt_i32 s6, 0
+; GFX10-NEXT:    s_cselect_b32 s6, s6, 0
+; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s9, s6
+; GFX10-NEXT:    s_lshr_b32 s9, s6, 16
+; GFX10-NEXT:    s_sub_i32 s6, s15, s6
+; GFX10-NEXT:    s_sub_i32 s9, s10, s9
 ; GFX10-NEXT:    s_sext_i32_i16 s10, s7
-; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s6, s8
+; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s6, s9
 ; GFX10-NEXT:    s_ashr_i32 s7, s7, 16
-; GFX10-NEXT:    s_sext_i32_i16 s8, s6
+; GFX10-NEXT:    s_sext_i32_i16 s9, s6
 ; GFX10-NEXT:    s_ashr_i32 s6, s6, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s8, s10
-; GFX10-NEXT:    s_cselect_b32 s8, s8, s10
+; GFX10-NEXT:    s_cmp_gt_i32 s9, s10
+; GFX10-NEXT:    s_cselect_b32 s9, s9, s10
 ; GFX10-NEXT:    s_cmp_gt_i32 s6, s7
 ; GFX10-NEXT:    s_cselect_b32 s6, s6, s7
 ; GFX10-NEXT:    s_sext_i32_i16 s7, s4
-; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s8, s6
+; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s9, s6
 ; GFX10-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX10-NEXT:    s_sext_i32_i16 s8, s6
+; GFX10-NEXT:    s_sext_i32_i16 s9, s6
 ; GFX10-NEXT:    s_ashr_i32 s6, s6, 16
-; GFX10-NEXT:    s_cmp_lt_i32 s8, s7
-; GFX10-NEXT:    s_cselect_b32 s7, s8, s7
+; GFX10-NEXT:    s_cmp_lt_i32 s9, s7
+; GFX10-NEXT:    s_cselect_b32 s7, s9, s7
 ; GFX10-NEXT:    s_cmp_lt_i32 s6, s4
 ; GFX10-NEXT:    s_cselect_b32 s4, s6, s4
-; GFX10-NEXT:    s_lshr_b32 s5, s3, 16
+; GFX10-NEXT:    s_lshr_b32 s6, s3, 16
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s7, s4
-; GFX10-NEXT:    s_lshr_b32 s6, s4, 16
+; GFX10-NEXT:    s_lshr_b32 s7, s4, 16
 ; GFX10-NEXT:    s_add_i32 s3, s3, s4
-; GFX10-NEXT:    s_add_i32 s5, s5, s6
-; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s3, s5
+; GFX10-NEXT:    s_add_i32 s6, s6, s7
+; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s3, s6
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> %lhs, <8 x i16> %rhs)
   %cast = bitcast <8 x i16> %result to <4 x i32>
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/shl.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/shl.ll
index ed1fe7af5f365..015f6b5de8b04 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/shl.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/shl.ll
@@ -529,8 +529,7 @@ define <2 x i16> @v_shl_v2i16_15(<2 x i16> %value) {
 ; GFX9-LABEL: v_shl_v2i16_15:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, 15, 15
-; GFX9-NEXT:    v_pk_lshlrev_b16 v0, s4, v0
+; GFX9-NEXT:    v_pk_lshlrev_b16 v0, 15, v0 op_sel_hi:[0,1]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %result = shl <2 x i16> %value, <i16 15, i16 15>
   ret <2 x i16> %result
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll
index 3e1778bcb881e..ac2a75383cba3 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll
@@ -4512,15 +4512,12 @@ define <2 x i16> @v_ssubsat_v2i16(<2 x i16> %lhs, <2 x i16> %rhs) {
 ; GFX9-LABEL: v_ssubsat_v2i16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX9-NEXT:    s_pack_ll_b32_b16 s6, -1, -1
-; GFX9-NEXT:    s_movk_i32 s5, 0x8000
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s4
-; GFX9-NEXT:    v_pk_max_i16 v2, v0, s6
-; GFX9-NEXT:    v_pk_sub_i16 v2, v2, s4
-; GFX9-NEXT:    s_pack_ll_b32_b16 s5, s5, s5
-; GFX9-NEXT:    v_pk_min_i16 v3, v0, s6
-; GFX9-NEXT:    v_pk_sub_i16 v3, v3, s5
+; GFX9-NEXT:    v_pk_max_i16 v2, v0, -1 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_mov_b32_e32 v3, 0x7fff7fff
+; GFX9-NEXT:    v_pk_sub_i16 v2, v2, v3
+; GFX9-NEXT:    v_pk_min_i16 v3, v0, -1 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0x80008000
+; GFX9-NEXT:    v_pk_sub_i16 v3, v3, v4
 ; GFX9-NEXT:    v_pk_max_i16 v1, v2, v1
 ; GFX9-NEXT:    v_pk_min_i16 v1, v1, v3
 ; GFX9-NEXT:    v_pk_sub_i16 v0, v0, v1
@@ -4530,16 +4527,11 @@ define <2 x i16> @v_ssubsat_v2i16(<2 x i16> %lhs, <2 x i16> %rhs) {
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, -1, -1
-; GFX10-NEXT:    s_movk_i32 s5, 0x7fff
-; GFX10-NEXT:    v_pk_max_i16 v2, v0, s4
-; GFX10-NEXT:    s_pack_ll_b32_b16 s5, s5, s5
-; GFX10-NEXT:    v_pk_min_i16 v3, v0, s4
-; GFX10-NEXT:    s_movk_i32 s6, 0x8000
+; GFX10-NEXT:    v_pk_max_i16 v2, v0, -1 op_sel_hi:[1,0]
+; GFX10-NEXT:    v_pk_min_i16 v3, v0, -1 op_sel_hi:[1,0]
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_pk_sub_i16 v2, v2, s5
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s6, s6
-; GFX10-NEXT:    v_pk_sub_i16 v3, v3, s4
+; GFX10-NEXT:    v_pk_sub_i16 v2, v2, 0x7fff7fff
+; GFX10-NEXT:    v_pk_sub_i16 v3, v3, 0x80008000
 ; GFX10-NEXT:    v_pk_max_i16 v1, v2, v1
 ; GFX10-NEXT:    v_pk_min_i16 v1, v1, v3
 ; GFX10-NEXT:    v_pk_sub_i16 v0, v0, v1
@@ -4635,53 +4627,45 @@ define amdgpu_ps i32 @s_ssubsat_v2i16(<2 x i16> inreg %lhs, <2 x i16> inreg %rhs
 ;
 ; GFX9-LABEL: s_ssubsat_v2i16:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, -1, -1
-; GFX9-NEXT:    s_sext_i32_i16 s7, s4
-; GFX9-NEXT:    s_sext_i32_i16 s5, s0
-; GFX9-NEXT:    s_ashr_i32 s6, s0, 16
-; GFX9-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s5, s7
-; GFX9-NEXT:    s_cselect_b32 s8, s5, s7
-; GFX9-NEXT:    s_cmp_gt_i32 s6, s4
-; GFX9-NEXT:    s_movk_i32 s2, 0x7fff
-; GFX9-NEXT:    s_cselect_b32 s9, s6, s4
-; GFX9-NEXT:    s_pack_ll_b32_b16 s8, s8, s9
-; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s2
-; GFX9-NEXT:    s_lshr_b32 s10, s2, 16
-; GFX9-NEXT:    s_lshr_b32 s9, s8, 16
-; GFX9-NEXT:    s_sub_i32 s2, s8, s2
-; GFX9-NEXT:    s_sub_i32 s8, s9, s10
-; GFX9-NEXT:    s_cmp_lt_i32 s5, s7
-; GFX9-NEXT:    s_cselect_b32 s5, s5, s7
-; GFX9-NEXT:    s_cmp_lt_i32 s6, s4
-; GFX9-NEXT:    s_movk_i32 s3, 0x8000
-; GFX9-NEXT:    s_cselect_b32 s4, s6, s4
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s5, s4
-; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s3, s3
-; GFX9-NEXT:    s_lshr_b32 s6, s3, 16
-; GFX9-NEXT:    s_lshr_b32 s5, s4, 16
-; GFX9-NEXT:    s_sub_i32 s3, s4, s3
-; GFX9-NEXT:    s_sub_i32 s4, s5, s6
-; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s8
-; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s3, s4
-; GFX9-NEXT:    s_sext_i32_i16 s4, s2
+; GFX9-NEXT:    s_sext_i32_i16 s2, s0
+; GFX9-NEXT:    s_ashr_i32 s3, s0, 16
+; GFX9-NEXT:    s_sext_i32_i16 s4, -1
+; GFX9-NEXT:    s_cmp_gt_i32 s2, s4
+; GFX9-NEXT:    s_cselect_b32 s5, s2, s4
+; GFX9-NEXT:    s_cmp_gt_i32 s3, -1
+; GFX9-NEXT:    s_cselect_b32 s6, s3, -1
+; GFX9-NEXT:    s_pack_ll_b32_b16 s5, s5, s6
+; GFX9-NEXT:    s_lshr_b32 s6, s5, 16
+; GFX9-NEXT:    s_sub_i32 s5, s5, 0x7fff7fff
+; GFX9-NEXT:    s_sub_i32 s6, s6, 0x7fff
+; GFX9-NEXT:    s_cmp_lt_i32 s2, s4
+; GFX9-NEXT:    s_cselect_b32 s2, s2, s4
+; GFX9-NEXT:    s_cmp_lt_i32 s3, -1
+; GFX9-NEXT:    s_cselect_b32 s3, s3, -1
+; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s3
+; GFX9-NEXT:    s_lshr_b32 s3, s2, 16
+; GFX9-NEXT:    s_pack_ll_b32_b16 s5, s5, s6
+; GFX9-NEXT:    s_sub_i32 s2, s2, 0x80008000
+; GFX9-NEXT:    s_sub_i32 s3, s3, 0x8000
+; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s3
+; GFX9-NEXT:    s_sext_i32_i16 s3, s5
+; GFX9-NEXT:    s_ashr_i32 s4, s5, 16
 ; GFX9-NEXT:    s_sext_i32_i16 s5, s1
-; GFX9-NEXT:    s_ashr_i32 s2, s2, 16
 ; GFX9-NEXT:    s_ashr_i32 s1, s1, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s4, s5
-; GFX9-NEXT:    s_cselect_b32 s4, s4, s5
-; GFX9-NEXT:    s_cmp_gt_i32 s2, s1
-; GFX9-NEXT:    s_cselect_b32 s1, s2, s1
-; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s4, s1
-; GFX9-NEXT:    s_sext_i32_i16 s2, s1
-; GFX9-NEXT:    s_sext_i32_i16 s4, s3
+; GFX9-NEXT:    s_cmp_gt_i32 s3, s5
+; GFX9-NEXT:    s_cselect_b32 s3, s3, s5
+; GFX9-NEXT:    s_cmp_gt_i32 s4, s1
+; GFX9-NEXT:    s_cselect_b32 s1, s4, s1
+; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s3, s1
+; GFX9-NEXT:    s_sext_i32_i16 s3, s1
+; GFX9-NEXT:    s_sext_i32_i16 s4, s2
 ; GFX9-NEXT:    s_ashr_i32 s1, s1, 16
-; GFX9-NEXT:    s_ashr_i32 s3, s3, 16
-; GFX9-NEXT:    s_cmp_lt_i32 s2, s4
-; GFX9-NEXT:    s_cselect_b32 s2, s2, s4
-; GFX9-NEXT:    s_cmp_lt_i32 s1, s3
-; GFX9-NEXT:    s_cselect_b32 s1, s1, s3
-; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s2, s1
+; GFX9-NEXT:    s_ashr_i32 s2, s2, 16
+; GFX9-NEXT:    s_cmp_lt_i32 s3, s4
+; GFX9-NEXT:    s_cselect_b32 s3, s3, s4
+; GFX9-NEXT:    s_cmp_lt_i32 s1, s2
+; GFX9-NEXT:    s_cselect_b32 s1, s1, s2
+; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s3, s1
 ; GFX9-NEXT:    s_lshr_b32 s2, s0, 16
 ; GFX9-NEXT:    s_lshr_b32 s3, s1, 16
 ; GFX9-NEXT:    s_sub_i32 s0, s0, s1
@@ -4691,47 +4675,39 @@ define amdgpu_ps i32 @s_ssubsat_v2i16(<2 x i16> inreg %lhs, <2 x i16> inreg %rhs
 ;
 ; GFX10-LABEL: s_ssubsat_v2i16:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_pack_ll_b32_b16 s2, -1, -1
-; GFX10-NEXT:    s_sext_i32_i16 s3, s0
-; GFX10-NEXT:    s_sext_i32_i16 s5, s2
+; GFX10-NEXT:    s_sext_i32_i16 s2, s0
+; GFX10-NEXT:    s_sext_i32_i16 s3, -1
 ; GFX10-NEXT:    s_ashr_i32 s4, s0, 16
-; GFX10-NEXT:    s_ashr_i32 s2, s2, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s3, s5
-; GFX10-NEXT:    s_movk_i32 s8, 0x7fff
-; GFX10-NEXT:    s_cselect_b32 s6, s3, s5
-; GFX10-NEXT:    s_cmp_gt_i32 s4, s2
+; GFX10-NEXT:    s_cmp_gt_i32 s2, s3
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cselect_b32 s7, s4, s2
-; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s6, s7
-; GFX10-NEXT:    s_pack_ll_b32_b16 s7, s8, s8
-; GFX10-NEXT:    s_lshr_b32 s8, s6, 16
-; GFX10-NEXT:    s_lshr_b32 s9, s7, 16
-; GFX10-NEXT:    s_sub_i32 s6, s6, s7
-; GFX10-NEXT:    s_sub_i32 s7, s8, s9
-; GFX10-NEXT:    s_cmp_lt_i32 s3, s5
-; GFX10-NEXT:    s_cselect_b32 s3, s3, s5
-; GFX10-NEXT:    s_cmp_lt_i32 s4, s2
-; GFX10-NEXT:    s_movk_i32 s5, 0x8000
-; GFX10-NEXT:    s_cselect_b32 s2, s4, s2
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s5, s5
-; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s3, s2
-; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s6, s7
-; GFX10-NEXT:    s_lshr_b32 s5, s2, 16
-; GFX10-NEXT:    s_lshr_b32 s6, s4, 16
-; GFX10-NEXT:    s_sub_i32 s2, s2, s4
-; GFX10-NEXT:    s_sub_i32 s4, s5, s6
-; GFX10-NEXT:    s_sext_i32_i16 s5, s3
+; GFX10-NEXT:    s_cselect_b32 s5, s2, s3
+; GFX10-NEXT:    s_cmp_gt_i32 s4, -1
+; GFX10-NEXT:    s_cselect_b32 s6, s4, -1
+; GFX10-NEXT:    s_pack_ll_b32_b16 s5, s5, s6
 ; GFX10-NEXT:    s_sext_i32_i16 s6, s1
-; GFX10-NEXT:    s_ashr_i32 s3, s3, 16
+; GFX10-NEXT:    s_lshr_b32 s7, s5, 16
+; GFX10-NEXT:    s_sub_i32 s5, s5, 0x7fff7fff
+; GFX10-NEXT:    s_sub_i32 s7, s7, 0x7fff
+; GFX10-NEXT:    s_cmp_lt_i32 s2, s3
+; GFX10-NEXT:    s_pack_ll_b32_b16 s5, s5, s7
+; GFX10-NEXT:    s_cselect_b32 s2, s2, s3
+; GFX10-NEXT:    s_cmp_lt_i32 s4, -1
+; GFX10-NEXT:    s_sext_i32_i16 s3, s5
+; GFX10-NEXT:    s_cselect_b32 s4, s4, -1
+; GFX10-NEXT:    s_ashr_i32 s5, s5, 16
+; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s2, s4
 ; GFX10-NEXT:    s_ashr_i32 s1, s1, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s5, s6
+; GFX10-NEXT:    s_lshr_b32 s4, s2, 16
+; GFX10-NEXT:    s_sub_i32 s2, s2, 0x80008000
+; GFX10-NEXT:    s_sub_i32 s4, s4, 0x8000
+; GFX10-NEXT:    s_cmp_gt_i32 s3, s6
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s2, s4
-; GFX10-NEXT:    s_cselect_b32 s5, s5, s6
-; GFX10-NEXT:    s_cmp_gt_i32 s3, s1
+; GFX10-NEXT:    s_cselect_b32 s3, s3, s6
+; GFX10-NEXT:    s_cmp_gt_i32 s5, s1
 ; GFX10-NEXT:    s_sext_i32_i16 s4, s2
-; GFX10-NEXT:    s_cselect_b32 s1, s3, s1
+; GFX10-NEXT:    s_cselect_b32 s1, s5, s1
 ; GFX10-NEXT:    s_ashr_i32 s2, s2, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s5, s1
+; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s3, s1
 ; GFX10-NEXT:    s_sext_i32_i16 s3, s1
 ; GFX10-NEXT:    s_ashr_i32 s1, s1, 16
 ; GFX10-NEXT:    s_cmp_lt_i32 s3, s4
@@ -4819,72 +4795,56 @@ define amdgpu_ps float @ssubsat_v2i16_sv(<2 x i16> inreg %lhs, <2 x i16> %rhs) {
 ;
 ; GFX9-LABEL: ssubsat_v2i16_sv:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_pack_ll_b32_b16 s3, -1, -1
-; GFX9-NEXT:    s_sext_i32_i16 s6, s3
-; GFX9-NEXT:    s_sext_i32_i16 s4, s0
-; GFX9-NEXT:    s_ashr_i32 s5, s0, 16
-; GFX9-NEXT:    s_ashr_i32 s3, s3, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s4, s6
-; GFX9-NEXT:    s_cselect_b32 s7, s4, s6
-; GFX9-NEXT:    s_cmp_gt_i32 s5, s3
-; GFX9-NEXT:    s_movk_i32 s1, 0x7fff
-; GFX9-NEXT:    s_cselect_b32 s8, s5, s3
-; GFX9-NEXT:    s_pack_ll_b32_b16 s7, s7, s8
-; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s1
-; GFX9-NEXT:    s_lshr_b32 s9, s1, 16
-; GFX9-NEXT:    s_lshr_b32 s8, s7, 16
-; GFX9-NEXT:    s_sub_i32 s1, s7, s1
-; GFX9-NEXT:    s_sub_i32 s7, s8, s9
-; GFX9-NEXT:    s_cmp_lt_i32 s4, s6
-; GFX9-NEXT:    s_cselect_b32 s4, s4, s6
-; GFX9-NEXT:    s_cmp_lt_i32 s5, s3
-; GFX9-NEXT:    s_movk_i32 s2, 0x8000
-; GFX9-NEXT:    s_cselect_b32 s3, s5, s3
-; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s4, s3
-; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s2
-; GFX9-NEXT:    s_lshr_b32 s5, s2, 16
-; GFX9-NEXT:    s_lshr_b32 s4, s3, 16
-; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s7
-; GFX9-NEXT:    s_sub_i32 s2, s3, s2
-; GFX9-NEXT:    s_sub_i32 s3, s4, s5
-; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s3
-; GFX9-NEXT:    v_pk_max_i16 v0, s1, v0
-; GFX9-NEXT:    v_pk_min_i16 v0, v0, s2
+; GFX9-NEXT:    s_sext_i32_i16 s1, s0
+; GFX9-NEXT:    s_ashr_i32 s2, s0, 16
+; GFX9-NEXT:    s_sext_i32_i16 s3, -1
+; GFX9-NEXT:    s_cmp_gt_i32 s1, s3
+; GFX9-NEXT:    s_cselect_b32 s4, s1, s3
+; GFX9-NEXT:    s_cmp_gt_i32 s2, -1
+; GFX9-NEXT:    s_cselect_b32 s5, s2, -1
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
+; GFX9-NEXT:    s_lshr_b32 s5, s4, 16
+; GFX9-NEXT:    s_sub_i32 s4, s4, 0x7fff7fff
+; GFX9-NEXT:    s_sub_i32 s5, s5, 0x7fff
+; GFX9-NEXT:    s_cmp_lt_i32 s1, s3
+; GFX9-NEXT:    s_cselect_b32 s1, s1, s3
+; GFX9-NEXT:    s_cmp_lt_i32 s2, -1
+; GFX9-NEXT:    s_cselect_b32 s2, s2, -1
+; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s2
+; GFX9-NEXT:    s_lshr_b32 s2, s1, 16
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
+; GFX9-NEXT:    s_sub_i32 s1, s1, 0x80008000
+; GFX9-NEXT:    s_sub_i32 s2, s2, 0x8000
+; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s2
+; GFX9-NEXT:    v_pk_max_i16 v0, s4, v0
+; GFX9-NEXT:    v_pk_min_i16 v0, v0, s1
 ; GFX9-NEXT:    v_pk_sub_i16 v0, s0, v0
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: ssubsat_v2i16_sv:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_pack_ll_b32_b16 s1, -1, -1
-; GFX10-NEXT:    s_sext_i32_i16 s2, s0
-; GFX10-NEXT:    s_sext_i32_i16 s4, s1
+; GFX10-NEXT:    s_sext_i32_i16 s1, s0
+; GFX10-NEXT:    s_sext_i32_i16 s2, -1
 ; GFX10-NEXT:    s_ashr_i32 s3, s0, 16
-; GFX10-NEXT:    s_ashr_i32 s1, s1, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s2, s4
-; GFX10-NEXT:    s_movk_i32 s7, 0x7fff
-; GFX10-NEXT:    s_cselect_b32 s5, s2, s4
-; GFX10-NEXT:    s_cmp_gt_i32 s3, s1
+; GFX10-NEXT:    s_cmp_gt_i32 s1, s2
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cselect_b32 s6, s3, s1
-; GFX10-NEXT:    s_pack_ll_b32_b16 s5, s5, s6
-; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s7, s7
-; GFX10-NEXT:    s_lshr_b32 s7, s5, 16
-; GFX10-NEXT:    s_lshr_b32 s8, s6, 16
-; GFX10-NEXT:    s_sub_i32 s5, s5, s6
-; GFX10-NEXT:    s_sub_i32 s6, s7, s8
-; GFX10-NEXT:    s_cmp_lt_i32 s2, s4
-; GFX10-NEXT:    s_cselect_b32 s2, s2, s4
-; GFX10-NEXT:    s_cmp_lt_i32 s3, s1
-; GFX10-NEXT:    s_movk_i32 s4, 0x8000
-; GFX10-NEXT:    s_cselect_b32 s1, s3, s1
-; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s4, s4
-; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s2, s1
-; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s5, s6
-; GFX10-NEXT:    s_lshr_b32 s4, s1, 16
-; GFX10-NEXT:    s_lshr_b32 s5, s3, 16
+; GFX10-NEXT:    s_cselect_b32 s4, s1, s2
+; GFX10-NEXT:    s_cmp_gt_i32 s3, -1
+; GFX10-NEXT:    s_cselect_b32 s5, s3, -1
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
+; GFX10-NEXT:    s_lshr_b32 s5, s4, 16
+; GFX10-NEXT:    s_sub_i32 s4, s4, 0x7fff7fff
+; GFX10-NEXT:    s_sub_i32 s5, s5, 0x7fff
+; GFX10-NEXT:    s_cmp_lt_i32 s1, s2
+; GFX10-NEXT:    s_cselect_b32 s1, s1, s2
+; GFX10-NEXT:    s_cmp_lt_i32 s3, -1
+; GFX10-NEXT:    s_cselect_b32 s2, s3, -1
+; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s1, s2
+; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s4, s5
+; GFX10-NEXT:    s_lshr_b32 s3, s1, 16
 ; GFX10-NEXT:    v_pk_max_i16 v0, s2, v0
-; GFX10-NEXT:    s_sub_i32 s1, s1, s3
-; GFX10-NEXT:    s_sub_i32 s2, s4, s5
+; GFX10-NEXT:    s_sub_i32 s1, s1, 0x80008000
+; GFX10-NEXT:    s_sub_i32 s2, s3, 0x8000
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s1, s2
 ; GFX10-NEXT:    v_pk_min_i16 v0, v0, s1
 ; GFX10-NEXT:    v_pk_sub_i16 v0, s0, v0
@@ -4951,15 +4911,12 @@ define amdgpu_ps float @ssubsat_v2i16_vs(<2 x i16> %lhs, <2 x i16> inreg %rhs) {
 ;
 ; GFX9-LABEL: ssubsat_v2i16_vs:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_movk_i32 s1, 0x7fff
-; GFX9-NEXT:    s_pack_ll_b32_b16 s3, -1, -1
-; GFX9-NEXT:    s_movk_i32 s2, 0x8000
-; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s1
-; GFX9-NEXT:    v_pk_max_i16 v1, v0, s3
-; GFX9-NEXT:    v_pk_sub_i16 v1, v1, s1
-; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s2
-; GFX9-NEXT:    v_pk_min_i16 v2, v0, s3
-; GFX9-NEXT:    v_pk_sub_i16 v2, v2, s2
+; GFX9-NEXT:    v_pk_max_i16 v1, v0, -1 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0x7fff7fff
+; GFX9-NEXT:    v_pk_sub_i16 v1, v1, v2
+; GFX9-NEXT:    v_pk_min_i16 v2, v0, -1 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_mov_b32_e32 v3, 0x80008000
+; GFX9-NEXT:    v_pk_sub_i16 v2, v2, v3
 ; GFX9-NEXT:    v_pk_max_i16 v1, v1, s0
 ; GFX9-NEXT:    v_pk_min_i16 v1, v1, v2
 ; GFX9-NEXT:    v_pk_sub_i16 v0, v0, v1
@@ -4967,16 +4924,11 @@ define amdgpu_ps float @ssubsat_v2i16_vs(<2 x i16> %lhs, <2 x i16> inreg %rhs) {
 ;
 ; GFX10-LABEL: ssubsat_v2i16_vs:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_pack_ll_b32_b16 s1, -1, -1
-; GFX10-NEXT:    s_movk_i32 s2, 0x7fff
-; GFX10-NEXT:    v_pk_max_i16 v1, v0, s1
-; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s2, s2
-; GFX10-NEXT:    v_pk_min_i16 v2, v0, s1
-; GFX10-NEXT:    s_movk_i32 s3, 0x8000
+; GFX10-NEXT:    v_pk_max_i16 v1, v0, -1 op_sel_hi:[1,0]
+; GFX10-NEXT:    v_pk_min_i16 v2, v0, -1 op_sel_hi:[1,0]
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_pk_sub_i16 v1, v1, s2
-; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s3, s3
-; GFX10-NEXT:    v_pk_sub_i16 v2, v2, s1
+; GFX10-NEXT:    v_pk_sub_i16 v1, v1, 0x7fff7fff
+; GFX10-NEXT:    v_pk_sub_i16 v2, v2, 0x80008000
 ; GFX10-NEXT:    v_pk_max_i16 v1, v1, s0
 ; GFX10-NEXT:    v_pk_min_i16 v1, v1, v2
 ; GFX10-NEXT:    v_pk_sub_i16 v0, v0, v1
@@ -5098,22 +5050,19 @@ define <2 x float> @v_ssubsat_v4i16(<4 x i16> %lhs, <4 x i16> %rhs) {
 ; GFX9-LABEL: v_ssubsat_v4i16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX9-NEXT:    s_pack_ll_b32_b16 s6, -1, -1
-; GFX9-NEXT:    s_movk_i32 s5, 0x8000
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s4
-; GFX9-NEXT:    v_pk_max_i16 v4, v0, s6
-; GFX9-NEXT:    v_pk_sub_i16 v4, v4, s4
-; GFX9-NEXT:    s_pack_ll_b32_b16 s5, s5, s5
-; GFX9-NEXT:    v_pk_min_i16 v5, v0, s6
+; GFX9-NEXT:    v_pk_max_i16 v4, v0, -1 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_mov_b32_e32 v5, 0x7fff7fff
+; GFX9-NEXT:    v_pk_sub_i16 v4, v4, v5
+; GFX9-NEXT:    v_pk_min_i16 v6, v0, -1 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_mov_b32_e32 v7, 0x80008000
 ; GFX9-NEXT:    v_pk_max_i16 v2, v4, v2
-; GFX9-NEXT:    v_pk_sub_i16 v5, v5, s5
-; GFX9-NEXT:    v_pk_min_i16 v2, v2, v5
+; GFX9-NEXT:    v_pk_sub_i16 v6, v6, v7
+; GFX9-NEXT:    v_pk_min_i16 v2, v2, v6
 ; GFX9-NEXT:    v_pk_sub_i16 v0, v0, v2
-; GFX9-NEXT:    v_pk_max_i16 v2, v1, s6
-; GFX9-NEXT:    v_pk_sub_i16 v2, v2, s4
-; GFX9-NEXT:    v_pk_min_i16 v4, v1, s6
-; GFX9-NEXT:    v_pk_sub_i16 v4, v4, s5
+; GFX9-NEXT:    v_pk_max_i16 v2, v1, -1 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_sub_i16 v2, v2, v5
+; GFX9-NEXT:    v_pk_min_i16 v4, v1, -1 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_sub_i16 v4, v4, v7
 ; GFX9-NEXT:    v_pk_max_i16 v2, v2, v3
 ; GFX9-NEXT:    v_pk_min_i16 v2, v2, v4
 ; GFX9-NEXT:    v_pk_sub_i16 v1, v1, v2
@@ -5123,24 +5072,19 @@ define <2 x float> @v_ssubsat_v4i16(<4 x i16> %lhs, <4 x i16> %rhs) {
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    s_pack_ll_b32_b16 s5, -1, -1
-; GFX10-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX10-NEXT:    v_pk_max_i16 v4, v0, s5
-; GFX10-NEXT:    v_pk_max_i16 v5, v1, s5
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s4, s4
-; GFX10-NEXT:    v_pk_min_i16 v6, v0, s5
-; GFX10-NEXT:    v_pk_min_i16 v7, v1, s5
-; GFX10-NEXT:    v_pk_sub_i16 v4, v4, s4
-; GFX10-NEXT:    v_pk_sub_i16 v5, v5, s4
-; GFX10-NEXT:    s_movk_i32 s6, 0x8000
+; GFX10-NEXT:    v_pk_max_i16 v4, v0, -1 op_sel_hi:[1,0]
+; GFX10-NEXT:    v_pk_max_i16 v5, v1, -1 op_sel_hi:[1,0]
+; GFX10-NEXT:    v_pk_min_i16 v6, v0, -1 op_sel_hi:[1,0]
+; GFX10-NEXT:    v_pk_min_i16 v7, v1, -1 op_sel_hi:[1,0]
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s6, s6
+; GFX10-NEXT:    v_pk_sub_i16 v4, v4, 0x7fff7fff
+; GFX10-NEXT:    v_pk_sub_i16 v5, v5, 0x7fff7fff
+; GFX10-NEXT:    v_pk_sub_i16 v6, v6, 0x80008000
+; GFX10-NEXT:    v_pk_sub_i16 v7, v7, 0x80008000
 ; GFX10-NEXT:    v_pk_max_i16 v11, v4, v2
-; GFX10-NEXT:    v_pk_sub_i16 v6, v6, s6
-; GFX10-NEXT:    v_pk_sub_i16 v4, v7, s6
-; GFX10-NEXT:    v_pk_max_i16 v3, v5, v3
+; GFX10-NEXT:    v_pk_max_i16 v10, v5, v3
 ; GFX10-NEXT:    v_pk_min_i16 v2, v11, v6
-; GFX10-NEXT:    v_pk_min_i16 v3, v3, v4
+; GFX10-NEXT:    v_pk_min_i16 v3, v10, v7
 ; GFX10-NEXT:    v_pk_sub_i16 v0, v0, v2
 ; GFX10-NEXT:    v_pk_sub_i16 v1, v1, v3
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
@@ -5306,77 +5250,73 @@ define amdgpu_ps <2 x i32> @s_ssubsat_v4i16(<4 x i16> inreg %lhs, <4 x i16> inre
 ;
 ; GFX9-LABEL: s_ssubsat_v4i16:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_pack_ll_b32_b16 s6, -1, -1
+; GFX9-NEXT:    s_sext_i32_i16 s6, s0
+; GFX9-NEXT:    s_ashr_i32 s7, s0, 16
+; GFX9-NEXT:    s_sext_i32_i16 s8, -1
+; GFX9-NEXT:    s_cmp_gt_i32 s6, s8
+; GFX9-NEXT:    s_cselect_b32 s9, s6, s8
+; GFX9-NEXT:    s_cmp_gt_i32 s7, -1
+; GFX9-NEXT:    s_cselect_b32 s10, s7, -1
+; GFX9-NEXT:    s_pack_ll_b32_b16 s9, s9, s10
+; GFX9-NEXT:    s_mov_b32 s4, 0x7fff7fff
+; GFX9-NEXT:    s_lshr_b32 s10, s9, 16
+; GFX9-NEXT:    s_movk_i32 s11, 0x7fff
+; GFX9-NEXT:    s_sub_i32 s9, s9, s4
+; GFX9-NEXT:    s_sub_i32 s10, s10, s11
+; GFX9-NEXT:    s_cmp_lt_i32 s6, s8
+; GFX9-NEXT:    s_cselect_b32 s6, s6, s8
+; GFX9-NEXT:    s_cmp_lt_i32 s7, -1
+; GFX9-NEXT:    s_cselect_b32 s7, s7, -1
+; GFX9-NEXT:    s_pack_ll_b32_b16 s6, s6, s7
+; GFX9-NEXT:    s_pack_ll_b32_b16 s9, s9, s10
+; GFX9-NEXT:    s_mov_b32 s5, 0x80008000
+; GFX9-NEXT:    s_lshr_b32 s7, s6, 16
+; GFX9-NEXT:    s_mov_b32 s10, 0x8000
+; GFX9-NEXT:    s_sub_i32 s6, s6, s5
+; GFX9-NEXT:    s_sub_i32 s7, s7, s10
+; GFX9-NEXT:    s_pack_ll_b32_b16 s6, s6, s7
+; GFX9-NEXT:    s_sext_i32_i16 s7, s9
+; GFX9-NEXT:    s_sext_i32_i16 s12, s2
+; GFX9-NEXT:    s_ashr_i32 s9, s9, 16
+; GFX9-NEXT:    s_ashr_i32 s2, s2, 16
+; GFX9-NEXT:    s_cmp_gt_i32 s7, s12
+; GFX9-NEXT:    s_cselect_b32 s7, s7, s12
+; GFX9-NEXT:    s_cmp_gt_i32 s9, s2
+; GFX9-NEXT:    s_cselect_b32 s2, s9, s2
+; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s7, s2
+; GFX9-NEXT:    s_sext_i32_i16 s7, s2
 ; GFX9-NEXT:    s_sext_i32_i16 s9, s6
-; GFX9-NEXT:    s_sext_i32_i16 s7, s0
-; GFX9-NEXT:    s_ashr_i32 s8, s0, 16
+; GFX9-NEXT:    s_ashr_i32 s2, s2, 16
 ; GFX9-NEXT:    s_ashr_i32 s6, s6, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s7, s9
-; GFX9-NEXT:    s_cselect_b32 s10, s7, s9
-; GFX9-NEXT:    s_cmp_gt_i32 s8, s6
-; GFX9-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX9-NEXT:    s_cselect_b32 s11, s8, s6
-; GFX9-NEXT:    s_pack_ll_b32_b16 s10, s10, s11
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s4
-; GFX9-NEXT:    s_lshr_b32 s11, s10, 16
-; GFX9-NEXT:    s_lshr_b32 s12, s4, 16
-; GFX9-NEXT:    s_sub_i32 s10, s10, s4
-; GFX9-NEXT:    s_sub_i32 s11, s11, s12
 ; GFX9-NEXT:    s_cmp_lt_i32 s7, s9
 ; GFX9-NEXT:    s_cselect_b32 s7, s7, s9
-; GFX9-NEXT:    s_cmp_lt_i32 s8, s6
-; GFX9-NEXT:    s_movk_i32 s5, 0x8000
-; GFX9-NEXT:    s_cselect_b32 s8, s8, s6
-; GFX9-NEXT:    s_pack_ll_b32_b16 s7, s7, s8
-; GFX9-NEXT:    s_pack_ll_b32_b16 s5, s5, s5
-; GFX9-NEXT:    s_pack_ll_b32_b16 s10, s10, s11
-; GFX9-NEXT:    s_lshr_b32 s8, s7, 16
-; GFX9-NEXT:    s_lshr_b32 s11, s5, 16
-; GFX9-NEXT:    s_sub_i32 s7, s7, s5
-; GFX9-NEXT:    s_sub_i32 s8, s8, s11
-; GFX9-NEXT:    s_pack_ll_b32_b16 s7, s7, s8
-; GFX9-NEXT:    s_sext_i32_i16 s8, s10
-; GFX9-NEXT:    s_sext_i32_i16 s13, s2
-; GFX9-NEXT:    s_ashr_i32 s10, s10, 16
-; GFX9-NEXT:    s_ashr_i32 s2, s2, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s8, s13
-; GFX9-NEXT:    s_cselect_b32 s8, s8, s13
-; GFX9-NEXT:    s_cmp_gt_i32 s10, s2
-; GFX9-NEXT:    s_cselect_b32 s2, s10, s2
-; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s8, s2
-; GFX9-NEXT:    s_sext_i32_i16 s8, s2
-; GFX9-NEXT:    s_sext_i32_i16 s10, s7
-; GFX9-NEXT:    s_ashr_i32 s2, s2, 16
-; GFX9-NEXT:    s_ashr_i32 s7, s7, 16
-; GFX9-NEXT:    s_cmp_lt_i32 s8, s10
-; GFX9-NEXT:    s_cselect_b32 s8, s8, s10
-; GFX9-NEXT:    s_cmp_lt_i32 s2, s7
-; GFX9-NEXT:    s_cselect_b32 s2, s2, s7
-; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s8, s2
-; GFX9-NEXT:    s_lshr_b32 s7, s0, 16
-; GFX9-NEXT:    s_lshr_b32 s8, s2, 16
+; GFX9-NEXT:    s_cmp_lt_i32 s2, s6
+; GFX9-NEXT:    s_cselect_b32 s2, s2, s6
+; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s7, s2
+; GFX9-NEXT:    s_lshr_b32 s6, s0, 16
+; GFX9-NEXT:    s_lshr_b32 s7, s2, 16
 ; GFX9-NEXT:    s_sub_i32 s0, s0, s2
-; GFX9-NEXT:    s_sub_i32 s2, s7, s8
+; GFX9-NEXT:    s_sub_i32 s2, s6, s7
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s2
 ; GFX9-NEXT:    s_sext_i32_i16 s2, s1
-; GFX9-NEXT:    s_ashr_i32 s7, s1, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s2, s9
-; GFX9-NEXT:    s_cselect_b32 s8, s2, s9
-; GFX9-NEXT:    s_cmp_gt_i32 s7, s6
-; GFX9-NEXT:    s_cselect_b32 s10, s7, s6
-; GFX9-NEXT:    s_pack_ll_b32_b16 s8, s8, s10
-; GFX9-NEXT:    s_lshr_b32 s10, s8, 16
-; GFX9-NEXT:    s_sub_i32 s4, s8, s4
-; GFX9-NEXT:    s_sub_i32 s8, s10, s12
-; GFX9-NEXT:    s_cmp_lt_i32 s2, s9
-; GFX9-NEXT:    s_cselect_b32 s2, s2, s9
-; GFX9-NEXT:    s_cmp_lt_i32 s7, s6
-; GFX9-NEXT:    s_cselect_b32 s6, s7, s6
+; GFX9-NEXT:    s_ashr_i32 s6, s1, 16
+; GFX9-NEXT:    s_cmp_gt_i32 s2, s8
+; GFX9-NEXT:    s_cselect_b32 s7, s2, s8
+; GFX9-NEXT:    s_cmp_gt_i32 s6, -1
+; GFX9-NEXT:    s_cselect_b32 s9, s6, -1
+; GFX9-NEXT:    s_pack_ll_b32_b16 s7, s7, s9
+; GFX9-NEXT:    s_lshr_b32 s9, s7, 16
+; GFX9-NEXT:    s_sub_i32 s4, s7, s4
+; GFX9-NEXT:    s_sub_i32 s7, s9, s11
+; GFX9-NEXT:    s_cmp_lt_i32 s2, s8
+; GFX9-NEXT:    s_cselect_b32 s2, s2, s8
+; GFX9-NEXT:    s_cmp_lt_i32 s6, -1
+; GFX9-NEXT:    s_cselect_b32 s6, s6, -1
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s6
 ; GFX9-NEXT:    s_lshr_b32 s6, s2, 16
 ; GFX9-NEXT:    s_sub_i32 s2, s2, s5
-; GFX9-NEXT:    s_sub_i32 s5, s6, s11
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s8
+; GFX9-NEXT:    s_sub_i32 s5, s6, s10
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s7
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s5
 ; GFX9-NEXT:    s_sext_i32_i16 s5, s4
 ; GFX9-NEXT:    s_sext_i32_i16 s6, s3
@@ -5405,80 +5345,76 @@ define amdgpu_ps <2 x i32> @s_ssubsat_v4i16(<4 x i16> inreg %lhs, <4 x i16> inre
 ;
 ; GFX10-LABEL: s_ssubsat_v4i16:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, -1, -1
-; GFX10-NEXT:    s_sext_i32_i16 s5, s0
-; GFX10-NEXT:    s_sext_i32_i16 s7, s4
+; GFX10-NEXT:    s_sext_i32_i16 s4, s0
+; GFX10-NEXT:    s_sext_i32_i16 s5, -1
 ; GFX10-NEXT:    s_ashr_i32 s6, s0, 16
-; GFX10-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s5, s7
+; GFX10-NEXT:    s_cmp_gt_i32 s4, s5
 ; GFX10-NEXT:    s_movk_i32 s10, 0x7fff
-; GFX10-NEXT:    s_cselect_b32 s8, s5, s7
-; GFX10-NEXT:    s_cmp_gt_i32 s6, s4
-; GFX10-NEXT:    s_movk_i32 s12, 0x8000
-; GFX10-NEXT:    s_cselect_b32 s9, s6, s4
-; GFX10-NEXT:    s_pack_ll_b32_b16 s12, s12, s12
-; GFX10-NEXT:    s_pack_ll_b32_b16 s8, s8, s9
-; GFX10-NEXT:    s_pack_ll_b32_b16 s9, s10, s10
-; GFX10-NEXT:    s_lshr_b32 s10, s8, 16
-; GFX10-NEXT:    s_lshr_b32 s11, s9, 16
-; GFX10-NEXT:    s_sub_i32 s8, s8, s9
-; GFX10-NEXT:    s_sub_i32 s10, s10, s11
-; GFX10-NEXT:    s_cmp_lt_i32 s5, s7
-; GFX10-NEXT:    s_sext_i32_i16 s14, s2
-; GFX10-NEXT:    s_cselect_b32 s5, s5, s7
-; GFX10-NEXT:    s_cmp_lt_i32 s6, s4
+; GFX10-NEXT:    s_cselect_b32 s7, s4, s5
+; GFX10-NEXT:    s_cmp_gt_i32 s6, -1
+; GFX10-NEXT:    s_mov_b32 s11, 0x80008000
+; GFX10-NEXT:    s_cselect_b32 s8, s6, -1
+; GFX10-NEXT:    s_sext_i32_i16 s13, s2
+; GFX10-NEXT:    s_pack_ll_b32_b16 s7, s7, s8
+; GFX10-NEXT:    s_mov_b32 s8, 0x7fff7fff
+; GFX10-NEXT:    s_lshr_b32 s9, s7, 16
+; GFX10-NEXT:    s_sub_i32 s7, s7, s8
+; GFX10-NEXT:    s_sub_i32 s9, s9, s10
+; GFX10-NEXT:    s_cmp_lt_i32 s4, s5
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cselect_b32 s6, s6, s4
+; GFX10-NEXT:    s_cselect_b32 s4, s4, s5
+; GFX10-NEXT:    s_cmp_lt_i32 s6, -1
+; GFX10-NEXT:    s_cselect_b32 s6, s6, -1
 ; GFX10-NEXT:    s_ashr_i32 s2, s2, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s5, s5, s6
-; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s8, s10
-; GFX10-NEXT:    s_lshr_b32 s8, s5, 16
-; GFX10-NEXT:    s_lshr_b32 s10, s12, 16
-; GFX10-NEXT:    s_sext_i32_i16 s13, s6
-; GFX10-NEXT:    s_sub_i32 s5, s5, s12
-; GFX10-NEXT:    s_sub_i32 s8, s8, s10
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s4, s6
+; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s7, s9
+; GFX10-NEXT:    s_lshr_b32 s7, s4, 16
+; GFX10-NEXT:    s_mov_b32 s9, 0x8000
+; GFX10-NEXT:    s_sext_i32_i16 s12, s6
+; GFX10-NEXT:    s_sub_i32 s4, s4, s11
+; GFX10-NEXT:    s_sub_i32 s7, s7, s9
 ; GFX10-NEXT:    s_ashr_i32 s6, s6, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s13, s14
-; GFX10-NEXT:    s_pack_ll_b32_b16 s5, s5, s8
-; GFX10-NEXT:    s_cselect_b32 s13, s13, s14
+; GFX10-NEXT:    s_cmp_gt_i32 s12, s13
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s4, s7
+; GFX10-NEXT:    s_cselect_b32 s12, s12, s13
 ; GFX10-NEXT:    s_cmp_gt_i32 s6, s2
-; GFX10-NEXT:    s_sext_i32_i16 s8, s5
+; GFX10-NEXT:    s_sext_i32_i16 s7, s4
 ; GFX10-NEXT:    s_cselect_b32 s2, s6, s2
-; GFX10-NEXT:    s_ashr_i32 s5, s5, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s13, s2
+; GFX10-NEXT:    s_ashr_i32 s4, s4, 16
+; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s12, s2
 ; GFX10-NEXT:    s_sext_i32_i16 s6, s2
 ; GFX10-NEXT:    s_ashr_i32 s2, s2, 16
-; GFX10-NEXT:    s_cmp_lt_i32 s6, s8
-; GFX10-NEXT:    s_cselect_b32 s6, s6, s8
-; GFX10-NEXT:    s_cmp_lt_i32 s2, s5
-; GFX10-NEXT:    s_cselect_b32 s2, s2, s5
-; GFX10-NEXT:    s_lshr_b32 s5, s0, 16
+; GFX10-NEXT:    s_cmp_lt_i32 s6, s7
+; GFX10-NEXT:    s_cselect_b32 s6, s6, s7
+; GFX10-NEXT:    s_cmp_lt_i32 s2, s4
+; GFX10-NEXT:    s_cselect_b32 s2, s2, s4
+; GFX10-NEXT:    s_lshr_b32 s4, s0, 16
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s6, s2
 ; GFX10-NEXT:    s_lshr_b32 s6, s2, 16
 ; GFX10-NEXT:    s_sub_i32 s0, s0, s2
-; GFX10-NEXT:    s_sub_i32 s2, s5, s6
-; GFX10-NEXT:    s_sext_i32_i16 s5, s1
+; GFX10-NEXT:    s_sub_i32 s2, s4, s6
+; GFX10-NEXT:    s_sext_i32_i16 s4, s1
 ; GFX10-NEXT:    s_ashr_i32 s6, s1, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s5, s7
+; GFX10-NEXT:    s_cmp_gt_i32 s4, s5
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s2
-; GFX10-NEXT:    s_cselect_b32 s8, s5, s7
-; GFX10-NEXT:    s_cmp_gt_i32 s6, s4
-; GFX10-NEXT:    s_cselect_b32 s13, s6, s4
-; GFX10-NEXT:    s_pack_ll_b32_b16 s8, s8, s13
-; GFX10-NEXT:    s_lshr_b32 s13, s8, 16
-; GFX10-NEXT:    s_sub_i32 s8, s8, s9
-; GFX10-NEXT:    s_sub_i32 s9, s13, s11
-; GFX10-NEXT:    s_cmp_lt_i32 s5, s7
-; GFX10-NEXT:    s_cselect_b32 s5, s5, s7
-; GFX10-NEXT:    s_cmp_lt_i32 s6, s4
-; GFX10-NEXT:    s_cselect_b32 s4, s6, s4
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s5, s4
-; GFX10-NEXT:    s_pack_ll_b32_b16 s5, s8, s9
+; GFX10-NEXT:    s_cselect_b32 s7, s4, s5
+; GFX10-NEXT:    s_cmp_gt_i32 s6, -1
+; GFX10-NEXT:    s_cselect_b32 s12, s6, -1
+; GFX10-NEXT:    s_pack_ll_b32_b16 s7, s7, s12
+; GFX10-NEXT:    s_lshr_b32 s12, s7, 16
+; GFX10-NEXT:    s_sub_i32 s7, s7, s8
+; GFX10-NEXT:    s_sub_i32 s8, s12, s10
+; GFX10-NEXT:    s_cmp_lt_i32 s4, s5
+; GFX10-NEXT:    s_cselect_b32 s4, s4, s5
+; GFX10-NEXT:    s_cmp_lt_i32 s6, -1
+; GFX10-NEXT:    s_cselect_b32 s5, s6, -1
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
+; GFX10-NEXT:    s_pack_ll_b32_b16 s5, s7, s8
 ; GFX10-NEXT:    s_lshr_b32 s6, s4, 16
 ; GFX10-NEXT:    s_sext_i32_i16 s7, s5
 ; GFX10-NEXT:    s_sext_i32_i16 s8, s3
-; GFX10-NEXT:    s_sub_i32 s4, s4, s12
-; GFX10-NEXT:    s_sub_i32 s6, s6, s10
+; GFX10-NEXT:    s_sub_i32 s4, s4, s11
+; GFX10-NEXT:    s_sub_i32 s6, s6, s9
 ; GFX10-NEXT:    s_ashr_i32 s5, s5, 16
 ; GFX10-NEXT:    s_ashr_i32 s3, s3, 16
 ; GFX10-NEXT:    s_cmp_gt_i32 s7, s8
@@ -5661,29 +5597,26 @@ define <3 x float> @v_ssubsat_v6i16(<6 x i16> %lhs, <6 x i16> %rhs) {
 ; GFX9-LABEL: v_ssubsat_v6i16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX9-NEXT:    s_pack_ll_b32_b16 s6, -1, -1
-; GFX9-NEXT:    s_movk_i32 s5, 0x8000
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s4
-; GFX9-NEXT:    v_pk_max_i16 v6, v0, s6
-; GFX9-NEXT:    v_pk_sub_i16 v6, v6, s4
-; GFX9-NEXT:    s_pack_ll_b32_b16 s5, s5, s5
-; GFX9-NEXT:    v_pk_min_i16 v7, v0, s6
+; GFX9-NEXT:    v_pk_max_i16 v6, v0, -1 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_mov_b32_e32 v7, 0x7fff7fff
+; GFX9-NEXT:    v_pk_sub_i16 v6, v6, v7
+; GFX9-NEXT:    v_pk_min_i16 v8, v0, -1 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_mov_b32_e32 v9, 0x80008000
 ; GFX9-NEXT:    v_pk_max_i16 v3, v6, v3
-; GFX9-NEXT:    v_pk_sub_i16 v7, v7, s5
-; GFX9-NEXT:    v_pk_min_i16 v3, v3, v7
+; GFX9-NEXT:    v_pk_sub_i16 v8, v8, v9
+; GFX9-NEXT:    v_pk_min_i16 v3, v3, v8
 ; GFX9-NEXT:    v_pk_sub_i16 v0, v0, v3
-; GFX9-NEXT:    v_pk_max_i16 v3, v1, s6
-; GFX9-NEXT:    v_pk_sub_i16 v3, v3, s4
-; GFX9-NEXT:    v_pk_min_i16 v6, v1, s6
+; GFX9-NEXT:    v_pk_max_i16 v3, v1, -1 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_sub_i16 v3, v3, v7
+; GFX9-NEXT:    v_pk_min_i16 v6, v1, -1 op_sel_hi:[1,0]
 ; GFX9-NEXT:    v_pk_max_i16 v3, v3, v4
-; GFX9-NEXT:    v_pk_sub_i16 v6, v6, s5
+; GFX9-NEXT:    v_pk_sub_i16 v6, v6, v9
 ; GFX9-NEXT:    v_pk_min_i16 v3, v3, v6
 ; GFX9-NEXT:    v_pk_sub_i16 v1, v1, v3
-; GFX9-NEXT:    v_pk_max_i16 v3, v2, s6
-; GFX9-NEXT:    v_pk_sub_i16 v3, v3, s4
-; GFX9-NEXT:    v_pk_min_i16 v4, v2, s6
-; GFX9-NEXT:    v_pk_sub_i16 v4, v4, s5
+; GFX9-NEXT:    v_pk_max_i16 v3, v2, -1 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_sub_i16 v3, v3, v7
+; GFX9-NEXT:    v_pk_min_i16 v4, v2, -1 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_sub_i16 v4, v4, v9
 ; GFX9-NEXT:    v_pk_max_i16 v3, v3, v5
 ; GFX9-NEXT:    v_pk_min_i16 v3, v3, v4
 ; GFX9-NEXT:    v_pk_sub_i16 v2, v2, v3
@@ -5693,28 +5626,23 @@ define <3 x float> @v_ssubsat_v6i16(<6 x i16> %lhs, <6 x i16> %rhs) {
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    s_pack_ll_b32_b16 s5, -1, -1
-; GFX10-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX10-NEXT:    v_pk_max_i16 v6, v0, s5
-; GFX10-NEXT:    v_pk_max_i16 v8, v1, s5
-; GFX10-NEXT:    v_pk_max_i16 v9, v2, s5
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s4, s4
-; GFX10-NEXT:    v_pk_min_i16 v7, v0, s5
-; GFX10-NEXT:    v_pk_sub_i16 v6, v6, s4
-; GFX10-NEXT:    v_pk_sub_i16 v15, v8, s4
-; GFX10-NEXT:    v_pk_sub_i16 v19, v9, s4
-; GFX10-NEXT:    v_pk_min_i16 v10, v1, s5
-; GFX10-NEXT:    v_pk_min_i16 v11, v2, s5
-; GFX10-NEXT:    s_movk_i32 s6, 0x8000
+; GFX10-NEXT:    v_pk_max_i16 v6, v0, -1 op_sel_hi:[1,0]
+; GFX10-NEXT:    v_pk_max_i16 v8, v1, -1 op_sel_hi:[1,0]
+; GFX10-NEXT:    v_pk_max_i16 v9, v2, -1 op_sel_hi:[1,0]
+; GFX10-NEXT:    v_pk_min_i16 v7, v0, -1 op_sel_hi:[1,0]
+; GFX10-NEXT:    v_pk_min_i16 v10, v1, -1 op_sel_hi:[1,0]
+; GFX10-NEXT:    v_pk_sub_i16 v6, v6, 0x7fff7fff
+; GFX10-NEXT:    v_pk_sub_i16 v15, v8, 0x7fff7fff
+; GFX10-NEXT:    v_pk_min_i16 v11, v2, -1 op_sel_hi:[1,0]
+; GFX10-NEXT:    v_pk_sub_i16 v19, v9, 0x7fff7fff
+; GFX10-NEXT:    v_pk_sub_i16 v7, v7, 0x80008000
 ; GFX10-NEXT:    v_pk_max_i16 v14, v6, v3
-; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s6, s6
+; GFX10-NEXT:    v_pk_sub_i16 v6, v10, 0x80008000
 ; GFX10-NEXT:    v_pk_max_i16 v4, v15, v4
-; GFX10-NEXT:    v_pk_sub_i16 v7, v7, s6
-; GFX10-NEXT:    v_pk_sub_i16 v6, v10, s6
-; GFX10-NEXT:    v_pk_sub_i16 v8, v11, s6
+; GFX10-NEXT:    v_pk_sub_i16 v8, v11, 0x80008000
 ; GFX10-NEXT:    v_pk_max_i16 v5, v19, v5
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_pk_min_i16 v3, v14, v7
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_pk_min_i16 v4, v4, v6
 ; GFX10-NEXT:    v_pk_min_i16 v5, v5, v8
 ; GFX10-NEXT:    v_pk_sub_i16 v0, v0, v3
@@ -5953,120 +5881,116 @@ define amdgpu_ps <3 x i32> @s_ssubsat_v6i16(<6 x i16> inreg %lhs, <6 x i16> inre
 ;
 ; GFX9-LABEL: s_ssubsat_v6i16:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_pack_ll_b32_b16 s8, -1, -1
+; GFX9-NEXT:    s_sext_i32_i16 s8, s0
+; GFX9-NEXT:    s_ashr_i32 s9, s0, 16
+; GFX9-NEXT:    s_sext_i32_i16 s10, -1
+; GFX9-NEXT:    s_cmp_gt_i32 s8, s10
+; GFX9-NEXT:    s_cselect_b32 s11, s8, s10
+; GFX9-NEXT:    s_cmp_gt_i32 s9, -1
+; GFX9-NEXT:    s_cselect_b32 s12, s9, -1
+; GFX9-NEXT:    s_pack_ll_b32_b16 s11, s11, s12
+; GFX9-NEXT:    s_mov_b32 s6, 0x7fff7fff
+; GFX9-NEXT:    s_lshr_b32 s12, s11, 16
+; GFX9-NEXT:    s_movk_i32 s13, 0x7fff
+; GFX9-NEXT:    s_sub_i32 s11, s11, s6
+; GFX9-NEXT:    s_sub_i32 s12, s12, s13
+; GFX9-NEXT:    s_cmp_lt_i32 s8, s10
+; GFX9-NEXT:    s_cselect_b32 s8, s8, s10
+; GFX9-NEXT:    s_cmp_lt_i32 s9, -1
+; GFX9-NEXT:    s_cselect_b32 s9, s9, -1
+; GFX9-NEXT:    s_pack_ll_b32_b16 s8, s8, s9
+; GFX9-NEXT:    s_pack_ll_b32_b16 s11, s11, s12
+; GFX9-NEXT:    s_mov_b32 s7, 0x80008000
+; GFX9-NEXT:    s_lshr_b32 s9, s8, 16
+; GFX9-NEXT:    s_mov_b32 s12, 0x8000
+; GFX9-NEXT:    s_sub_i32 s8, s8, s7
+; GFX9-NEXT:    s_sub_i32 s9, s9, s12
+; GFX9-NEXT:    s_pack_ll_b32_b16 s8, s8, s9
+; GFX9-NEXT:    s_sext_i32_i16 s9, s11
+; GFX9-NEXT:    s_sext_i32_i16 s14, s3
+; GFX9-NEXT:    s_ashr_i32 s11, s11, 16
+; GFX9-NEXT:    s_ashr_i32 s3, s3, 16
+; GFX9-NEXT:    s_cmp_gt_i32 s9, s14
+; GFX9-NEXT:    s_cselect_b32 s9, s9, s14
+; GFX9-NEXT:    s_cmp_gt_i32 s11, s3
+; GFX9-NEXT:    s_cselect_b32 s3, s11, s3
+; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s9, s3
+; GFX9-NEXT:    s_sext_i32_i16 s9, s3
 ; GFX9-NEXT:    s_sext_i32_i16 s11, s8
-; GFX9-NEXT:    s_sext_i32_i16 s9, s0
-; GFX9-NEXT:    s_ashr_i32 s10, s0, 16
+; GFX9-NEXT:    s_ashr_i32 s3, s3, 16
 ; GFX9-NEXT:    s_ashr_i32 s8, s8, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s9, s11
-; GFX9-NEXT:    s_cselect_b32 s12, s9, s11
-; GFX9-NEXT:    s_cmp_gt_i32 s10, s8
-; GFX9-NEXT:    s_movk_i32 s6, 0x7fff
-; GFX9-NEXT:    s_cselect_b32 s13, s10, s8
-; GFX9-NEXT:    s_pack_ll_b32_b16 s12, s12, s13
-; GFX9-NEXT:    s_pack_ll_b32_b16 s6, s6, s6
-; GFX9-NEXT:    s_lshr_b32 s13, s12, 16
-; GFX9-NEXT:    s_lshr_b32 s14, s6, 16
-; GFX9-NEXT:    s_sub_i32 s12, s12, s6
-; GFX9-NEXT:    s_sub_i32 s13, s13, s14
 ; GFX9-NEXT:    s_cmp_lt_i32 s9, s11
 ; GFX9-NEXT:    s_cselect_b32 s9, s9, s11
-; GFX9-NEXT:    s_cmp_lt_i32 s10, s8
-; GFX9-NEXT:    s_movk_i32 s7, 0x8000
-; GFX9-NEXT:    s_cselect_b32 s10, s10, s8
-; GFX9-NEXT:    s_pack_ll_b32_b16 s9, s9, s10
-; GFX9-NEXT:    s_pack_ll_b32_b16 s7, s7, s7
-; GFX9-NEXT:    s_pack_ll_b32_b16 s12, s12, s13
-; GFX9-NEXT:    s_lshr_b32 s10, s9, 16
-; GFX9-NEXT:    s_lshr_b32 s13, s7, 16
-; GFX9-NEXT:    s_sub_i32 s9, s9, s7
-; GFX9-NEXT:    s_sub_i32 s10, s10, s13
-; GFX9-NEXT:    s_pack_ll_b32_b16 s9, s9, s10
-; GFX9-NEXT:    s_sext_i32_i16 s10, s12
-; GFX9-NEXT:    s_sext_i32_i16 s15, s3
-; GFX9-NEXT:    s_ashr_i32 s12, s12, 16
-; GFX9-NEXT:    s_ashr_i32 s3, s3, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s10, s15
-; GFX9-NEXT:    s_cselect_b32 s10, s10, s15
-; GFX9-NEXT:    s_cmp_gt_i32 s12, s3
-; GFX9-NEXT:    s_cselect_b32 s3, s12, s3
-; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s10, s3
-; GFX9-NEXT:    s_sext_i32_i16 s10, s3
-; GFX9-NEXT:    s_sext_i32_i16 s12, s9
-; GFX9-NEXT:    s_ashr_i32 s3, s3, 16
-; GFX9-NEXT:    s_ashr_i32 s9, s9, 16
-; GFX9-NEXT:    s_cmp_lt_i32 s10, s12
-; GFX9-NEXT:    s_cselect_b32 s10, s10, s12
-; GFX9-NEXT:    s_cmp_lt_i32 s3, s9
-; GFX9-NEXT:    s_cselect_b32 s3, s3, s9
-; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s10, s3
-; GFX9-NEXT:    s_lshr_b32 s9, s0, 16
-; GFX9-NEXT:    s_lshr_b32 s10, s3, 16
+; GFX9-NEXT:    s_cmp_lt_i32 s3, s8
+; GFX9-NEXT:    s_cselect_b32 s3, s3, s8
+; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s9, s3
+; GFX9-NEXT:    s_lshr_b32 s8, s0, 16
+; GFX9-NEXT:    s_lshr_b32 s9, s3, 16
 ; GFX9-NEXT:    s_sub_i32 s0, s0, s3
-; GFX9-NEXT:    s_sub_i32 s3, s9, s10
+; GFX9-NEXT:    s_sub_i32 s3, s8, s9
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s3
 ; GFX9-NEXT:    s_sext_i32_i16 s3, s1
-; GFX9-NEXT:    s_ashr_i32 s9, s1, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s3, s11
-; GFX9-NEXT:    s_cselect_b32 s10, s3, s11
-; GFX9-NEXT:    s_cmp_gt_i32 s9, s8
-; GFX9-NEXT:    s_cselect_b32 s12, s9, s8
-; GFX9-NEXT:    s_pack_ll_b32_b16 s10, s10, s12
-; GFX9-NEXT:    s_lshr_b32 s12, s10, 16
-; GFX9-NEXT:    s_sub_i32 s10, s10, s6
-; GFX9-NEXT:    s_sub_i32 s12, s12, s14
-; GFX9-NEXT:    s_cmp_lt_i32 s3, s11
-; GFX9-NEXT:    s_cselect_b32 s3, s3, s11
-; GFX9-NEXT:    s_cmp_lt_i32 s9, s8
-; GFX9-NEXT:    s_cselect_b32 s9, s9, s8
-; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s3, s9
-; GFX9-NEXT:    s_lshr_b32 s9, s3, 16
-; GFX9-NEXT:    s_pack_ll_b32_b16 s10, s10, s12
+; GFX9-NEXT:    s_ashr_i32 s8, s1, 16
+; GFX9-NEXT:    s_cmp_gt_i32 s3, s10
+; GFX9-NEXT:    s_cselect_b32 s9, s3, s10
+; GFX9-NEXT:    s_cmp_gt_i32 s8, -1
+; GFX9-NEXT:    s_cselect_b32 s11, s8, -1
+; GFX9-NEXT:    s_pack_ll_b32_b16 s9, s9, s11
+; GFX9-NEXT:    s_lshr_b32 s11, s9, 16
+; GFX9-NEXT:    s_sub_i32 s9, s9, s6
+; GFX9-NEXT:    s_sub_i32 s11, s11, s13
+; GFX9-NEXT:    s_cmp_lt_i32 s3, s10
+; GFX9-NEXT:    s_cselect_b32 s3, s3, s10
+; GFX9-NEXT:    s_cmp_lt_i32 s8, -1
+; GFX9-NEXT:    s_cselect_b32 s8, s8, -1
+; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s3, s8
+; GFX9-NEXT:    s_lshr_b32 s8, s3, 16
+; GFX9-NEXT:    s_pack_ll_b32_b16 s9, s9, s11
 ; GFX9-NEXT:    s_sub_i32 s3, s3, s7
-; GFX9-NEXT:    s_sub_i32 s9, s9, s13
-; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s3, s9
-; GFX9-NEXT:    s_sext_i32_i16 s9, s10
-; GFX9-NEXT:    s_sext_i32_i16 s12, s4
-; GFX9-NEXT:    s_ashr_i32 s10, s10, 16
+; GFX9-NEXT:    s_sub_i32 s8, s8, s12
+; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s3, s8
+; GFX9-NEXT:    s_sext_i32_i16 s8, s9
+; GFX9-NEXT:    s_sext_i32_i16 s11, s4
+; GFX9-NEXT:    s_ashr_i32 s9, s9, 16
 ; GFX9-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s9, s12
-; GFX9-NEXT:    s_cselect_b32 s9, s9, s12
-; GFX9-NEXT:    s_cmp_gt_i32 s10, s4
-; GFX9-NEXT:    s_cselect_b32 s4, s10, s4
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s9, s4
-; GFX9-NEXT:    s_sext_i32_i16 s9, s4
-; GFX9-NEXT:    s_sext_i32_i16 s10, s3
+; GFX9-NEXT:    s_cmp_gt_i32 s8, s11
+; GFX9-NEXT:    s_cselect_b32 s8, s8, s11
+; GFX9-NEXT:    s_cmp_gt_i32 s9, s4
+; GFX9-NEXT:    s_cselect_b32 s4, s9, s4
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s8, s4
+; GFX9-NEXT:    s_sext_i32_i16 s8, s4
+; GFX9-NEXT:    s_sext_i32_i16 s9, s3
 ; GFX9-NEXT:    s_ashr_i32 s4, s4, 16
 ; GFX9-NEXT:    s_ashr_i32 s3, s3, 16
-; GFX9-NEXT:    s_cmp_lt_i32 s9, s10
-; GFX9-NEXT:    s_cselect_b32 s9, s9, s10
+; GFX9-NEXT:    s_cmp_lt_i32 s8, s9
+; GFX9-NEXT:    s_cselect_b32 s8, s8, s9
 ; GFX9-NEXT:    s_cmp_lt_i32 s4, s3
 ; GFX9-NEXT:    s_cselect_b32 s3, s4, s3
-; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s9, s3
+; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s8, s3
 ; GFX9-NEXT:    s_lshr_b32 s4, s1, 16
-; GFX9-NEXT:    s_lshr_b32 s9, s3, 16
+; GFX9-NEXT:    s_lshr_b32 s8, s3, 16
 ; GFX9-NEXT:    s_sub_i32 s1, s1, s3
-; GFX9-NEXT:    s_sub_i32 s3, s4, s9
+; GFX9-NEXT:    s_sub_i32 s3, s4, s8
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s3
 ; GFX9-NEXT:    s_sext_i32_i16 s3, s2
 ; GFX9-NEXT:    s_ashr_i32 s4, s2, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s3, s11
-; GFX9-NEXT:    s_cselect_b32 s9, s3, s11
-; GFX9-NEXT:    s_cmp_gt_i32 s4, s8
-; GFX9-NEXT:    s_cselect_b32 s10, s4, s8
-; GFX9-NEXT:    s_pack_ll_b32_b16 s9, s9, s10
-; GFX9-NEXT:    s_lshr_b32 s10, s9, 16
-; GFX9-NEXT:    s_sub_i32 s6, s9, s6
-; GFX9-NEXT:    s_sub_i32 s9, s10, s14
-; GFX9-NEXT:    s_cmp_lt_i32 s3, s11
-; GFX9-NEXT:    s_cselect_b32 s3, s3, s11
-; GFX9-NEXT:    s_cmp_lt_i32 s4, s8
-; GFX9-NEXT:    s_cselect_b32 s4, s4, s8
+; GFX9-NEXT:    s_cmp_gt_i32 s3, s10
+; GFX9-NEXT:    s_cselect_b32 s8, s3, s10
+; GFX9-NEXT:    s_cmp_gt_i32 s4, -1
+; GFX9-NEXT:    s_cselect_b32 s9, s4, -1
+; GFX9-NEXT:    s_pack_ll_b32_b16 s8, s8, s9
+; GFX9-NEXT:    s_lshr_b32 s9, s8, 16
+; GFX9-NEXT:    s_sub_i32 s6, s8, s6
+; GFX9-NEXT:    s_sub_i32 s8, s9, s13
+; GFX9-NEXT:    s_cmp_lt_i32 s3, s10
+; GFX9-NEXT:    s_cselect_b32 s3, s3, s10
+; GFX9-NEXT:    s_cmp_lt_i32 s4, -1
+; GFX9-NEXT:    s_cselect_b32 s4, s4, -1
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s3, s4
 ; GFX9-NEXT:    s_lshr_b32 s4, s3, 16
 ; GFX9-NEXT:    s_sub_i32 s3, s3, s7
-; GFX9-NEXT:    s_sub_i32 s4, s4, s13
-; GFX9-NEXT:    s_pack_ll_b32_b16 s6, s6, s9
+; GFX9-NEXT:    s_sub_i32 s4, s4, s12
+; GFX9-NEXT:    s_pack_ll_b32_b16 s6, s6, s8
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s3, s4
 ; GFX9-NEXT:    s_sext_i32_i16 s4, s6
 ; GFX9-NEXT:    s_sext_i32_i16 s7, s5
@@ -6095,123 +6019,119 @@ define amdgpu_ps <3 x i32> @s_ssubsat_v6i16(<6 x i16> inreg %lhs, <6 x i16> inre
 ;
 ; GFX10-LABEL: s_ssubsat_v6i16:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_pack_ll_b32_b16 s6, -1, -1
-; GFX10-NEXT:    s_sext_i32_i16 s7, s0
-; GFX10-NEXT:    s_sext_i32_i16 s9, s6
+; GFX10-NEXT:    s_sext_i32_i16 s6, s0
+; GFX10-NEXT:    s_sext_i32_i16 s7, -1
 ; GFX10-NEXT:    s_ashr_i32 s8, s0, 16
-; GFX10-NEXT:    s_ashr_i32 s6, s6, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s7, s9
+; GFX10-NEXT:    s_cmp_gt_i32 s6, s7
 ; GFX10-NEXT:    s_movk_i32 s12, 0x7fff
-; GFX10-NEXT:    s_cselect_b32 s10, s7, s9
-; GFX10-NEXT:    s_cmp_gt_i32 s8, s6
-; GFX10-NEXT:    s_movk_i32 s14, 0x8000
-; GFX10-NEXT:    s_cselect_b32 s11, s8, s6
-; GFX10-NEXT:    s_pack_ll_b32_b16 s14, s14, s14
-; GFX10-NEXT:    s_pack_ll_b32_b16 s10, s10, s11
-; GFX10-NEXT:    s_pack_ll_b32_b16 s11, s12, s12
-; GFX10-NEXT:    s_lshr_b32 s12, s10, 16
-; GFX10-NEXT:    s_lshr_b32 s13, s11, 16
-; GFX10-NEXT:    s_sub_i32 s10, s10, s11
-; GFX10-NEXT:    s_sub_i32 s12, s12, s13
-; GFX10-NEXT:    s_cmp_lt_i32 s7, s9
-; GFX10-NEXT:    s_sext_i32_i16 s16, s3
-; GFX10-NEXT:    s_cselect_b32 s7, s7, s9
-; GFX10-NEXT:    s_cmp_lt_i32 s8, s6
+; GFX10-NEXT:    s_cselect_b32 s9, s6, s7
+; GFX10-NEXT:    s_cmp_gt_i32 s8, -1
+; GFX10-NEXT:    s_mov_b32 s13, 0x80008000
+; GFX10-NEXT:    s_cselect_b32 s10, s8, -1
+; GFX10-NEXT:    s_sext_i32_i16 s15, s3
+; GFX10-NEXT:    s_pack_ll_b32_b16 s9, s9, s10
+; GFX10-NEXT:    s_mov_b32 s10, 0x7fff7fff
+; GFX10-NEXT:    s_lshr_b32 s11, s9, 16
+; GFX10-NEXT:    s_sub_i32 s9, s9, s10
+; GFX10-NEXT:    s_sub_i32 s11, s11, s12
+; GFX10-NEXT:    s_cmp_lt_i32 s6, s7
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cselect_b32 s8, s8, s6
+; GFX10-NEXT:    s_cselect_b32 s6, s6, s7
+; GFX10-NEXT:    s_cmp_lt_i32 s8, -1
+; GFX10-NEXT:    s_cselect_b32 s8, s8, -1
 ; GFX10-NEXT:    s_ashr_i32 s3, s3, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s7, s7, s8
-; GFX10-NEXT:    s_pack_ll_b32_b16 s8, s10, s12
-; GFX10-NEXT:    s_lshr_b32 s10, s7, 16
-; GFX10-NEXT:    s_lshr_b32 s12, s14, 16
-; GFX10-NEXT:    s_sext_i32_i16 s15, s8
-; GFX10-NEXT:    s_sub_i32 s7, s7, s14
-; GFX10-NEXT:    s_sub_i32 s10, s10, s12
+; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s6, s8
+; GFX10-NEXT:    s_pack_ll_b32_b16 s8, s9, s11
+; GFX10-NEXT:    s_lshr_b32 s9, s6, 16
+; GFX10-NEXT:    s_mov_b32 s11, 0x8000
+; GFX10-NEXT:    s_sext_i32_i16 s14, s8
+; GFX10-NEXT:    s_sub_i32 s6, s6, s13
+; GFX10-NEXT:    s_sub_i32 s9, s9, s11
 ; GFX10-NEXT:    s_ashr_i32 s8, s8, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s15, s16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s7, s7, s10
-; GFX10-NEXT:    s_cselect_b32 s15, s15, s16
+; GFX10-NEXT:    s_cmp_gt_i32 s14, s15
+; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s6, s9
+; GFX10-NEXT:    s_cselect_b32 s14, s14, s15
 ; GFX10-NEXT:    s_cmp_gt_i32 s8, s3
-; GFX10-NEXT:    s_sext_i32_i16 s10, s7
+; GFX10-NEXT:    s_sext_i32_i16 s9, s6
 ; GFX10-NEXT:    s_cselect_b32 s3, s8, s3
-; GFX10-NEXT:    s_ashr_i32 s7, s7, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s15, s3
-; GFX10-NEXT:    s_sext_i32_i16 s16, s4
+; GFX10-NEXT:    s_ashr_i32 s6, s6, 16
+; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s14, s3
+; GFX10-NEXT:    s_sext_i32_i16 s15, s4
 ; GFX10-NEXT:    s_sext_i32_i16 s8, s3
 ; GFX10-NEXT:    s_ashr_i32 s3, s3, 16
-; GFX10-NEXT:    s_cmp_lt_i32 s8, s10
-; GFX10-NEXT:    s_cselect_b32 s8, s8, s10
-; GFX10-NEXT:    s_cmp_lt_i32 s3, s7
-; GFX10-NEXT:    s_cselect_b32 s3, s3, s7
-; GFX10-NEXT:    s_lshr_b32 s7, s0, 16
+; GFX10-NEXT:    s_cmp_lt_i32 s8, s9
+; GFX10-NEXT:    s_cselect_b32 s8, s8, s9
+; GFX10-NEXT:    s_cmp_lt_i32 s3, s6
+; GFX10-NEXT:    s_cselect_b32 s3, s3, s6
+; GFX10-NEXT:    s_lshr_b32 s6, s0, 16
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s8, s3
 ; GFX10-NEXT:    s_lshr_b32 s8, s3, 16
 ; GFX10-NEXT:    s_sub_i32 s0, s0, s3
-; GFX10-NEXT:    s_sub_i32 s3, s7, s8
-; GFX10-NEXT:    s_sext_i32_i16 s7, s1
+; GFX10-NEXT:    s_sub_i32 s3, s6, s8
+; GFX10-NEXT:    s_sext_i32_i16 s6, s1
 ; GFX10-NEXT:    s_ashr_i32 s8, s1, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s7, s9
+; GFX10-NEXT:    s_cmp_gt_i32 s6, s7
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s3
-; GFX10-NEXT:    s_cselect_b32 s10, s7, s9
-; GFX10-NEXT:    s_cmp_gt_i32 s8, s6
-; GFX10-NEXT:    s_cselect_b32 s15, s8, s6
-; GFX10-NEXT:    s_pack_ll_b32_b16 s10, s10, s15
-; GFX10-NEXT:    s_lshr_b32 s15, s10, 16
-; GFX10-NEXT:    s_sub_i32 s10, s10, s11
-; GFX10-NEXT:    s_sub_i32 s15, s15, s13
-; GFX10-NEXT:    s_cmp_lt_i32 s7, s9
-; GFX10-NEXT:    s_cselect_b32 s7, s7, s9
-; GFX10-NEXT:    s_cmp_lt_i32 s8, s6
-; GFX10-NEXT:    s_cselect_b32 s8, s8, s6
+; GFX10-NEXT:    s_cselect_b32 s9, s6, s7
+; GFX10-NEXT:    s_cmp_gt_i32 s8, -1
+; GFX10-NEXT:    s_cselect_b32 s14, s8, -1
+; GFX10-NEXT:    s_pack_ll_b32_b16 s9, s9, s14
+; GFX10-NEXT:    s_lshr_b32 s14, s9, 16
+; GFX10-NEXT:    s_sub_i32 s9, s9, s10
+; GFX10-NEXT:    s_sub_i32 s14, s14, s12
+; GFX10-NEXT:    s_cmp_lt_i32 s6, s7
+; GFX10-NEXT:    s_cselect_b32 s6, s6, s7
+; GFX10-NEXT:    s_cmp_lt_i32 s8, -1
+; GFX10-NEXT:    s_cselect_b32 s8, s8, -1
 ; GFX10-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s7, s7, s8
-; GFX10-NEXT:    s_pack_ll_b32_b16 s8, s10, s15
-; GFX10-NEXT:    s_lshr_b32 s10, s7, 16
-; GFX10-NEXT:    s_sext_i32_i16 s15, s8
-; GFX10-NEXT:    s_sub_i32 s7, s7, s14
-; GFX10-NEXT:    s_sub_i32 s10, s10, s12
+; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s6, s8
+; GFX10-NEXT:    s_pack_ll_b32_b16 s8, s9, s14
+; GFX10-NEXT:    s_lshr_b32 s9, s6, 16
+; GFX10-NEXT:    s_sext_i32_i16 s14, s8
+; GFX10-NEXT:    s_sub_i32 s6, s6, s13
+; GFX10-NEXT:    s_sub_i32 s9, s9, s11
 ; GFX10-NEXT:    s_ashr_i32 s8, s8, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s15, s16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s7, s7, s10
-; GFX10-NEXT:    s_cselect_b32 s15, s15, s16
+; GFX10-NEXT:    s_cmp_gt_i32 s14, s15
+; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s6, s9
+; GFX10-NEXT:    s_cselect_b32 s14, s14, s15
 ; GFX10-NEXT:    s_cmp_gt_i32 s8, s4
-; GFX10-NEXT:    s_sext_i32_i16 s10, s7
+; GFX10-NEXT:    s_sext_i32_i16 s9, s6
 ; GFX10-NEXT:    s_cselect_b32 s4, s8, s4
-; GFX10-NEXT:    s_ashr_i32 s7, s7, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s15, s4
+; GFX10-NEXT:    s_ashr_i32 s6, s6, 16
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s14, s4
 ; GFX10-NEXT:    s_sext_i32_i16 s8, s4
 ; GFX10-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX10-NEXT:    s_cmp_lt_i32 s8, s10
-; GFX10-NEXT:    s_cselect_b32 s8, s8, s10
-; GFX10-NEXT:    s_cmp_lt_i32 s4, s7
-; GFX10-NEXT:    s_cselect_b32 s4, s4, s7
-; GFX10-NEXT:    s_lshr_b32 s7, s1, 16
+; GFX10-NEXT:    s_cmp_lt_i32 s8, s9
+; GFX10-NEXT:    s_cselect_b32 s8, s8, s9
+; GFX10-NEXT:    s_cmp_lt_i32 s4, s6
+; GFX10-NEXT:    s_cselect_b32 s4, s4, s6
+; GFX10-NEXT:    s_lshr_b32 s6, s1, 16
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s8, s4
 ; GFX10-NEXT:    s_lshr_b32 s8, s4, 16
 ; GFX10-NEXT:    s_sub_i32 s1, s1, s4
-; GFX10-NEXT:    s_sub_i32 s4, s7, s8
-; GFX10-NEXT:    s_sext_i32_i16 s7, s2
+; GFX10-NEXT:    s_sub_i32 s4, s6, s8
+; GFX10-NEXT:    s_sext_i32_i16 s6, s2
 ; GFX10-NEXT:    s_ashr_i32 s8, s2, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s7, s9
+; GFX10-NEXT:    s_cmp_gt_i32 s6, s7
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s1, s4
-; GFX10-NEXT:    s_cselect_b32 s10, s7, s9
-; GFX10-NEXT:    s_cmp_gt_i32 s8, s6
-; GFX10-NEXT:    s_cselect_b32 s15, s8, s6
-; GFX10-NEXT:    s_pack_ll_b32_b16 s10, s10, s15
-; GFX10-NEXT:    s_lshr_b32 s15, s10, 16
-; GFX10-NEXT:    s_sub_i32 s10, s10, s11
-; GFX10-NEXT:    s_sub_i32 s11, s15, s13
-; GFX10-NEXT:    s_cmp_lt_i32 s7, s9
-; GFX10-NEXT:    s_cselect_b32 s7, s7, s9
-; GFX10-NEXT:    s_cmp_lt_i32 s8, s6
-; GFX10-NEXT:    s_cselect_b32 s6, s8, s6
-; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s7, s6
-; GFX10-NEXT:    s_pack_ll_b32_b16 s7, s10, s11
+; GFX10-NEXT:    s_cselect_b32 s9, s6, s7
+; GFX10-NEXT:    s_cmp_gt_i32 s8, -1
+; GFX10-NEXT:    s_cselect_b32 s14, s8, -1
+; GFX10-NEXT:    s_pack_ll_b32_b16 s9, s9, s14
+; GFX10-NEXT:    s_lshr_b32 s14, s9, 16
+; GFX10-NEXT:    s_sub_i32 s9, s9, s10
+; GFX10-NEXT:    s_sub_i32 s10, s14, s12
+; GFX10-NEXT:    s_cmp_lt_i32 s6, s7
+; GFX10-NEXT:    s_cselect_b32 s6, s6, s7
+; GFX10-NEXT:    s_cmp_lt_i32 s8, -1
+; GFX10-NEXT:    s_cselect_b32 s7, s8, -1
+; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s6, s7
+; GFX10-NEXT:    s_pack_ll_b32_b16 s7, s9, s10
 ; GFX10-NEXT:    s_lshr_b32 s8, s6, 16
 ; GFX10-NEXT:    s_sext_i32_i16 s9, s7
 ; GFX10-NEXT:    s_sext_i32_i16 s10, s5
-; GFX10-NEXT:    s_sub_i32 s6, s6, s14
-; GFX10-NEXT:    s_sub_i32 s8, s8, s12
+; GFX10-NEXT:    s_sub_i32 s6, s6, s13
+; GFX10-NEXT:    s_sub_i32 s8, s8, s11
 ; GFX10-NEXT:    s_ashr_i32 s7, s7, 16
 ; GFX10-NEXT:    s_ashr_i32 s5, s5, 16
 ; GFX10-NEXT:    s_cmp_gt_i32 s9, s10
@@ -6423,36 +6343,33 @@ define <4 x float> @v_ssubsat_v8i16(<8 x i16> %lhs, <8 x i16> %rhs) {
 ; GFX9-LABEL: v_ssubsat_v8i16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX9-NEXT:    s_pack_ll_b32_b16 s6, -1, -1
-; GFX9-NEXT:    s_movk_i32 s5, 0x8000
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s4
-; GFX9-NEXT:    v_pk_max_i16 v8, v0, s6
-; GFX9-NEXT:    v_pk_sub_i16 v8, v8, s4
-; GFX9-NEXT:    s_pack_ll_b32_b16 s5, s5, s5
-; GFX9-NEXT:    v_pk_min_i16 v9, v0, s6
+; GFX9-NEXT:    v_pk_max_i16 v8, v0, -1 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_mov_b32_e32 v9, 0x7fff7fff
+; GFX9-NEXT:    v_pk_sub_i16 v8, v8, v9
+; GFX9-NEXT:    v_pk_min_i16 v10, v0, -1 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_mov_b32_e32 v11, 0x80008000
 ; GFX9-NEXT:    v_pk_max_i16 v4, v8, v4
-; GFX9-NEXT:    v_pk_sub_i16 v9, v9, s5
-; GFX9-NEXT:    v_pk_min_i16 v4, v4, v9
+; GFX9-NEXT:    v_pk_sub_i16 v10, v10, v11
+; GFX9-NEXT:    v_pk_min_i16 v4, v4, v10
 ; GFX9-NEXT:    v_pk_sub_i16 v0, v0, v4
-; GFX9-NEXT:    v_pk_max_i16 v4, v1, s6
-; GFX9-NEXT:    v_pk_sub_i16 v4, v4, s4
-; GFX9-NEXT:    v_pk_min_i16 v8, v1, s6
+; GFX9-NEXT:    v_pk_max_i16 v4, v1, -1 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_sub_i16 v4, v4, v9
+; GFX9-NEXT:    v_pk_min_i16 v8, v1, -1 op_sel_hi:[1,0]
 ; GFX9-NEXT:    v_pk_max_i16 v4, v4, v5
-; GFX9-NEXT:    v_pk_sub_i16 v8, v8, s5
+; GFX9-NEXT:    v_pk_sub_i16 v8, v8, v11
 ; GFX9-NEXT:    v_pk_min_i16 v4, v4, v8
 ; GFX9-NEXT:    v_pk_sub_i16 v1, v1, v4
-; GFX9-NEXT:    v_pk_max_i16 v4, v2, s6
-; GFX9-NEXT:    v_pk_sub_i16 v4, v4, s4
-; GFX9-NEXT:    v_pk_min_i16 v5, v2, s6
-; GFX9-NEXT:    v_pk_sub_i16 v5, v5, s5
+; GFX9-NEXT:    v_pk_max_i16 v4, v2, -1 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_sub_i16 v4, v4, v9
+; GFX9-NEXT:    v_pk_min_i16 v5, v2, -1 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_sub_i16 v5, v5, v11
 ; GFX9-NEXT:    v_pk_max_i16 v4, v4, v6
 ; GFX9-NEXT:    v_pk_min_i16 v4, v4, v5
 ; GFX9-NEXT:    v_pk_sub_i16 v2, v2, v4
-; GFX9-NEXT:    v_pk_max_i16 v4, v3, s6
-; GFX9-NEXT:    v_pk_sub_i16 v4, v4, s4
-; GFX9-NEXT:    v_pk_min_i16 v5, v3, s6
-; GFX9-NEXT:    v_pk_sub_i16 v5, v5, s5
+; GFX9-NEXT:    v_pk_max_i16 v4, v3, -1 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_sub_i16 v4, v4, v9
+; GFX9-NEXT:    v_pk_min_i16 v5, v3, -1 op_sel_hi:[1,0]
+; GFX9-NEXT:    v_pk_sub_i16 v5, v5, v11
 ; GFX9-NEXT:    v_pk_max_i16 v4, v4, v7
 ; GFX9-NEXT:    v_pk_min_i16 v4, v4, v5
 ; GFX9-NEXT:    v_pk_sub_i16 v3, v3, v4
@@ -6462,30 +6379,25 @@ define <4 x float> @v_ssubsat_v8i16(<8 x i16> %lhs, <8 x i16> %rhs) {
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, -1, -1
-; GFX10-NEXT:    s_movk_i32 s5, 0x7fff
-; GFX10-NEXT:    v_pk_max_i16 v8, v0, s4
-; GFX10-NEXT:    s_pack_ll_b32_b16 s5, s5, s5
-; GFX10-NEXT:    v_pk_max_i16 v10, v1, s4
-; GFX10-NEXT:    v_pk_max_i16 v12, v3, s4
-; GFX10-NEXT:    v_pk_min_i16 v9, v0, s4
-; GFX10-NEXT:    v_pk_sub_i16 v15, v8, s5
-; GFX10-NEXT:    v_pk_max_i16 v8, v2, s4
-; GFX10-NEXT:    v_pk_sub_i16 v10, v10, s5
-; GFX10-NEXT:    v_pk_sub_i16 v12, v12, s5
-; GFX10-NEXT:    v_pk_min_i16 v11, v1, s4
-; GFX10-NEXT:    v_pk_min_i16 v13, v2, s4
-; GFX10-NEXT:    v_pk_sub_i16 v8, v8, s5
-; GFX10-NEXT:    v_pk_min_i16 v14, v3, s4
-; GFX10-NEXT:    s_movk_i32 s6, 0x8000
+; GFX10-NEXT:    v_pk_max_i16 v8, v0, -1 op_sel_hi:[1,0]
+; GFX10-NEXT:    v_pk_max_i16 v10, v1, -1 op_sel_hi:[1,0]
+; GFX10-NEXT:    v_pk_max_i16 v12, v3, -1 op_sel_hi:[1,0]
+; GFX10-NEXT:    v_pk_min_i16 v9, v0, -1 op_sel_hi:[1,0]
+; GFX10-NEXT:    v_pk_min_i16 v11, v1, -1 op_sel_hi:[1,0]
+; GFX10-NEXT:    v_pk_sub_i16 v15, v8, 0x7fff7fff
+; GFX10-NEXT:    v_pk_max_i16 v8, v2, -1 op_sel_hi:[1,0]
+; GFX10-NEXT:    v_pk_sub_i16 v10, v10, 0x7fff7fff
+; GFX10-NEXT:    v_pk_sub_i16 v12, v12, 0x7fff7fff
+; GFX10-NEXT:    v_pk_min_i16 v13, v2, -1 op_sel_hi:[1,0]
+; GFX10-NEXT:    v_pk_min_i16 v14, v3, -1 op_sel_hi:[1,0]
+; GFX10-NEXT:    v_pk_sub_i16 v8, v8, 0x7fff7fff
 ; GFX10-NEXT:    v_pk_max_i16 v4, v15, v4
-; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s6, s6
+; GFX10-NEXT:    v_pk_sub_i16 v9, v9, 0x80008000
 ; GFX10-NEXT:    v_pk_max_i16 v5, v10, v5
-; GFX10-NEXT:    v_pk_sub_i16 v11, v11, s6
-; GFX10-NEXT:    v_pk_sub_i16 v9, v9, s6
+; GFX10-NEXT:    v_pk_sub_i16 v11, v11, 0x80008000
 ; GFX10-NEXT:    v_pk_max_i16 v15, v8, v6
-; GFX10-NEXT:    v_pk_sub_i16 v10, v13, s6
-; GFX10-NEXT:    v_pk_sub_i16 v8, v14, s6
+; GFX10-NEXT:    v_pk_sub_i16 v10, v13, 0x80008000
+; GFX10-NEXT:    v_pk_sub_i16 v8, v14, 0x80008000
 ; GFX10-NEXT:    v_pk_max_i16 v7, v12, v7
 ; GFX10-NEXT:    v_pk_min_i16 v19, v4, v9
 ; GFX10-NEXT:    v_pk_min_i16 v11, v5, v11
@@ -6799,136 +6711,132 @@ define amdgpu_ps <4 x i32> @s_ssubsat_v8i16(<8 x i16> inreg %lhs, <8 x i16> inre
 ;
 ; GFX9-LABEL: s_ssubsat_v8i16:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_pack_ll_b32_b16 s10, -1, -1
+; GFX9-NEXT:    s_sext_i32_i16 s10, s0
+; GFX9-NEXT:    s_ashr_i32 s11, s0, 16
+; GFX9-NEXT:    s_sext_i32_i16 s12, -1
+; GFX9-NEXT:    s_cmp_gt_i32 s10, s12
+; GFX9-NEXT:    s_cselect_b32 s13, s10, s12
+; GFX9-NEXT:    s_cmp_gt_i32 s11, -1
+; GFX9-NEXT:    s_cselect_b32 s14, s11, -1
+; GFX9-NEXT:    s_pack_ll_b32_b16 s13, s13, s14
+; GFX9-NEXT:    s_mov_b32 s8, 0x7fff7fff
+; GFX9-NEXT:    s_lshr_b32 s14, s13, 16
+; GFX9-NEXT:    s_movk_i32 s15, 0x7fff
+; GFX9-NEXT:    s_sub_i32 s13, s13, s8
+; GFX9-NEXT:    s_sub_i32 s14, s14, s15
+; GFX9-NEXT:    s_cmp_lt_i32 s10, s12
+; GFX9-NEXT:    s_cselect_b32 s10, s10, s12
+; GFX9-NEXT:    s_cmp_lt_i32 s11, -1
+; GFX9-NEXT:    s_cselect_b32 s11, s11, -1
+; GFX9-NEXT:    s_pack_ll_b32_b16 s10, s10, s11
+; GFX9-NEXT:    s_pack_ll_b32_b16 s13, s13, s14
+; GFX9-NEXT:    s_mov_b32 s9, 0x80008000
+; GFX9-NEXT:    s_lshr_b32 s11, s10, 16
+; GFX9-NEXT:    s_mov_b32 s14, 0x8000
+; GFX9-NEXT:    s_sub_i32 s10, s10, s9
+; GFX9-NEXT:    s_sub_i32 s11, s11, s14
+; GFX9-NEXT:    s_pack_ll_b32_b16 s10, s10, s11
+; GFX9-NEXT:    s_sext_i32_i16 s11, s13
+; GFX9-NEXT:    s_sext_i32_i16 s16, s4
+; GFX9-NEXT:    s_ashr_i32 s13, s13, 16
+; GFX9-NEXT:    s_ashr_i32 s4, s4, 16
+; GFX9-NEXT:    s_cmp_gt_i32 s11, s16
+; GFX9-NEXT:    s_cselect_b32 s11, s11, s16
+; GFX9-NEXT:    s_cmp_gt_i32 s13, s4
+; GFX9-NEXT:    s_cselect_b32 s4, s13, s4
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s11, s4
+; GFX9-NEXT:    s_sext_i32_i16 s11, s4
 ; GFX9-NEXT:    s_sext_i32_i16 s13, s10
-; GFX9-NEXT:    s_sext_i32_i16 s11, s0
-; GFX9-NEXT:    s_ashr_i32 s12, s0, 16
+; GFX9-NEXT:    s_ashr_i32 s4, s4, 16
 ; GFX9-NEXT:    s_ashr_i32 s10, s10, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s11, s13
-; GFX9-NEXT:    s_cselect_b32 s14, s11, s13
-; GFX9-NEXT:    s_cmp_gt_i32 s12, s10
-; GFX9-NEXT:    s_movk_i32 s8, 0x7fff
-; GFX9-NEXT:    s_cselect_b32 s15, s12, s10
-; GFX9-NEXT:    s_pack_ll_b32_b16 s14, s14, s15
-; GFX9-NEXT:    s_pack_ll_b32_b16 s8, s8, s8
-; GFX9-NEXT:    s_lshr_b32 s15, s14, 16
-; GFX9-NEXT:    s_lshr_b32 s16, s8, 16
-; GFX9-NEXT:    s_sub_i32 s14, s14, s8
-; GFX9-NEXT:    s_sub_i32 s15, s15, s16
 ; GFX9-NEXT:    s_cmp_lt_i32 s11, s13
 ; GFX9-NEXT:    s_cselect_b32 s11, s11, s13
-; GFX9-NEXT:    s_cmp_lt_i32 s12, s10
-; GFX9-NEXT:    s_movk_i32 s9, 0x8000
-; GFX9-NEXT:    s_cselect_b32 s12, s12, s10
-; GFX9-NEXT:    s_pack_ll_b32_b16 s11, s11, s12
-; GFX9-NEXT:    s_pack_ll_b32_b16 s9, s9, s9
-; GFX9-NEXT:    s_pack_ll_b32_b16 s14, s14, s15
-; GFX9-NEXT:    s_lshr_b32 s12, s11, 16
-; GFX9-NEXT:    s_lshr_b32 s15, s9, 16
-; GFX9-NEXT:    s_sub_i32 s11, s11, s9
-; GFX9-NEXT:    s_sub_i32 s12, s12, s15
-; GFX9-NEXT:    s_pack_ll_b32_b16 s11, s11, s12
-; GFX9-NEXT:    s_sext_i32_i16 s12, s14
-; GFX9-NEXT:    s_sext_i32_i16 s17, s4
-; GFX9-NEXT:    s_ashr_i32 s14, s14, 16
-; GFX9-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s12, s17
-; GFX9-NEXT:    s_cselect_b32 s12, s12, s17
-; GFX9-NEXT:    s_cmp_gt_i32 s14, s4
-; GFX9-NEXT:    s_cselect_b32 s4, s14, s4
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s12, s4
-; GFX9-NEXT:    s_sext_i32_i16 s12, s4
-; GFX9-NEXT:    s_sext_i32_i16 s14, s11
-; GFX9-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX9-NEXT:    s_ashr_i32 s11, s11, 16
-; GFX9-NEXT:    s_cmp_lt_i32 s12, s14
-; GFX9-NEXT:    s_cselect_b32 s12, s12, s14
-; GFX9-NEXT:    s_cmp_lt_i32 s4, s11
-; GFX9-NEXT:    s_cselect_b32 s4, s4, s11
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s12, s4
-; GFX9-NEXT:    s_lshr_b32 s11, s0, 16
-; GFX9-NEXT:    s_lshr_b32 s12, s4, 16
+; GFX9-NEXT:    s_cmp_lt_i32 s4, s10
+; GFX9-NEXT:    s_cselect_b32 s4, s4, s10
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s11, s4
+; GFX9-NEXT:    s_lshr_b32 s10, s0, 16
+; GFX9-NEXT:    s_lshr_b32 s11, s4, 16
 ; GFX9-NEXT:    s_sub_i32 s0, s0, s4
-; GFX9-NEXT:    s_sub_i32 s4, s11, s12
+; GFX9-NEXT:    s_sub_i32 s4, s10, s11
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s4
 ; GFX9-NEXT:    s_sext_i32_i16 s4, s1
-; GFX9-NEXT:    s_ashr_i32 s11, s1, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s4, s13
-; GFX9-NEXT:    s_cselect_b32 s12, s4, s13
-; GFX9-NEXT:    s_cmp_gt_i32 s11, s10
-; GFX9-NEXT:    s_cselect_b32 s14, s11, s10
-; GFX9-NEXT:    s_pack_ll_b32_b16 s12, s12, s14
-; GFX9-NEXT:    s_lshr_b32 s14, s12, 16
-; GFX9-NEXT:    s_sub_i32 s12, s12, s8
-; GFX9-NEXT:    s_sub_i32 s14, s14, s16
-; GFX9-NEXT:    s_cmp_lt_i32 s4, s13
-; GFX9-NEXT:    s_cselect_b32 s4, s4, s13
-; GFX9-NEXT:    s_cmp_lt_i32 s11, s10
-; GFX9-NEXT:    s_cselect_b32 s11, s11, s10
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s11
-; GFX9-NEXT:    s_lshr_b32 s11, s4, 16
-; GFX9-NEXT:    s_pack_ll_b32_b16 s12, s12, s14
+; GFX9-NEXT:    s_ashr_i32 s10, s1, 16
+; GFX9-NEXT:    s_cmp_gt_i32 s4, s12
+; GFX9-NEXT:    s_cselect_b32 s11, s4, s12
+; GFX9-NEXT:    s_cmp_gt_i32 s10, -1
+; GFX9-NEXT:    s_cselect_b32 s13, s10, -1
+; GFX9-NEXT:    s_pack_ll_b32_b16 s11, s11, s13
+; GFX9-NEXT:    s_lshr_b32 s13, s11, 16
+; GFX9-NEXT:    s_sub_i32 s11, s11, s8
+; GFX9-NEXT:    s_sub_i32 s13, s13, s15
+; GFX9-NEXT:    s_cmp_lt_i32 s4, s12
+; GFX9-NEXT:    s_cselect_b32 s4, s4, s12
+; GFX9-NEXT:    s_cmp_lt_i32 s10, -1
+; GFX9-NEXT:    s_cselect_b32 s10, s10, -1
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s10
+; GFX9-NEXT:    s_lshr_b32 s10, s4, 16
+; GFX9-NEXT:    s_pack_ll_b32_b16 s11, s11, s13
 ; GFX9-NEXT:    s_sub_i32 s4, s4, s9
-; GFX9-NEXT:    s_sub_i32 s11, s11, s15
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s11
-; GFX9-NEXT:    s_sext_i32_i16 s11, s12
-; GFX9-NEXT:    s_sext_i32_i16 s14, s5
-; GFX9-NEXT:    s_ashr_i32 s12, s12, 16
+; GFX9-NEXT:    s_sub_i32 s10, s10, s14
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s10
+; GFX9-NEXT:    s_sext_i32_i16 s10, s11
+; GFX9-NEXT:    s_sext_i32_i16 s13, s5
+; GFX9-NEXT:    s_ashr_i32 s11, s11, 16
 ; GFX9-NEXT:    s_ashr_i32 s5, s5, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s11, s14
-; GFX9-NEXT:    s_cselect_b32 s11, s11, s14
-; GFX9-NEXT:    s_cmp_gt_i32 s12, s5
-; GFX9-NEXT:    s_cselect_b32 s5, s12, s5
-; GFX9-NEXT:    s_pack_ll_b32_b16 s5, s11, s5
-; GFX9-NEXT:    s_sext_i32_i16 s11, s5
-; GFX9-NEXT:    s_sext_i32_i16 s12, s4
+; GFX9-NEXT:    s_cmp_gt_i32 s10, s13
+; GFX9-NEXT:    s_cselect_b32 s10, s10, s13
+; GFX9-NEXT:    s_cmp_gt_i32 s11, s5
+; GFX9-NEXT:    s_cselect_b32 s5, s11, s5
+; GFX9-NEXT:    s_pack_ll_b32_b16 s5, s10, s5
+; GFX9-NEXT:    s_sext_i32_i16 s10, s5
+; GFX9-NEXT:    s_sext_i32_i16 s11, s4
 ; GFX9-NEXT:    s_ashr_i32 s5, s5, 16
 ; GFX9-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX9-NEXT:    s_cmp_lt_i32 s11, s12
-; GFX9-NEXT:    s_cselect_b32 s11, s11, s12
+; GFX9-NEXT:    s_cmp_lt_i32 s10, s11
+; GFX9-NEXT:    s_cselect_b32 s10, s10, s11
 ; GFX9-NEXT:    s_cmp_lt_i32 s5, s4
 ; GFX9-NEXT:    s_cselect_b32 s4, s5, s4
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s11, s4
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s10, s4
 ; GFX9-NEXT:    s_lshr_b32 s5, s1, 16
-; GFX9-NEXT:    s_lshr_b32 s11, s4, 16
+; GFX9-NEXT:    s_lshr_b32 s10, s4, 16
 ; GFX9-NEXT:    s_sub_i32 s1, s1, s4
-; GFX9-NEXT:    s_sub_i32 s4, s5, s11
+; GFX9-NEXT:    s_sub_i32 s4, s5, s10
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s4
 ; GFX9-NEXT:    s_sext_i32_i16 s4, s2
 ; GFX9-NEXT:    s_ashr_i32 s5, s2, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s4, s13
-; GFX9-NEXT:    s_cselect_b32 s11, s4, s13
-; GFX9-NEXT:    s_cmp_gt_i32 s5, s10
-; GFX9-NEXT:    s_cselect_b32 s12, s5, s10
-; GFX9-NEXT:    s_pack_ll_b32_b16 s11, s11, s12
-; GFX9-NEXT:    s_lshr_b32 s12, s11, 16
-; GFX9-NEXT:    s_sub_i32 s11, s11, s8
-; GFX9-NEXT:    s_sub_i32 s12, s12, s16
-; GFX9-NEXT:    s_cmp_lt_i32 s4, s13
-; GFX9-NEXT:    s_cselect_b32 s4, s4, s13
-; GFX9-NEXT:    s_cmp_lt_i32 s5, s10
-; GFX9-NEXT:    s_cselect_b32 s5, s5, s10
+; GFX9-NEXT:    s_cmp_gt_i32 s4, s12
+; GFX9-NEXT:    s_cselect_b32 s10, s4, s12
+; GFX9-NEXT:    s_cmp_gt_i32 s5, -1
+; GFX9-NEXT:    s_cselect_b32 s11, s5, -1
+; GFX9-NEXT:    s_pack_ll_b32_b16 s10, s10, s11
+; GFX9-NEXT:    s_lshr_b32 s11, s10, 16
+; GFX9-NEXT:    s_sub_i32 s10, s10, s8
+; GFX9-NEXT:    s_sub_i32 s11, s11, s15
+; GFX9-NEXT:    s_cmp_lt_i32 s4, s12
+; GFX9-NEXT:    s_cselect_b32 s4, s4, s12
+; GFX9-NEXT:    s_cmp_lt_i32 s5, -1
+; GFX9-NEXT:    s_cselect_b32 s5, s5, -1
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
 ; GFX9-NEXT:    s_lshr_b32 s5, s4, 16
-; GFX9-NEXT:    s_pack_ll_b32_b16 s11, s11, s12
+; GFX9-NEXT:    s_pack_ll_b32_b16 s10, s10, s11
 ; GFX9-NEXT:    s_sub_i32 s4, s4, s9
-; GFX9-NEXT:    s_sub_i32 s5, s5, s15
+; GFX9-NEXT:    s_sub_i32 s5, s5, s14
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
-; GFX9-NEXT:    s_sext_i32_i16 s5, s11
-; GFX9-NEXT:    s_sext_i32_i16 s12, s6
-; GFX9-NEXT:    s_ashr_i32 s11, s11, 16
+; GFX9-NEXT:    s_sext_i32_i16 s5, s10
+; GFX9-NEXT:    s_sext_i32_i16 s11, s6
+; GFX9-NEXT:    s_ashr_i32 s10, s10, 16
 ; GFX9-NEXT:    s_ashr_i32 s6, s6, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s5, s12
-; GFX9-NEXT:    s_cselect_b32 s5, s5, s12
-; GFX9-NEXT:    s_cmp_gt_i32 s11, s6
-; GFX9-NEXT:    s_cselect_b32 s6, s11, s6
+; GFX9-NEXT:    s_cmp_gt_i32 s5, s11
+; GFX9-NEXT:    s_cselect_b32 s5, s5, s11
+; GFX9-NEXT:    s_cmp_gt_i32 s10, s6
+; GFX9-NEXT:    s_cselect_b32 s6, s10, s6
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s5, s5, s6
 ; GFX9-NEXT:    s_sext_i32_i16 s6, s5
-; GFX9-NEXT:    s_sext_i32_i16 s11, s4
+; GFX9-NEXT:    s_sext_i32_i16 s10, s4
 ; GFX9-NEXT:    s_ashr_i32 s5, s5, 16
 ; GFX9-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX9-NEXT:    s_cmp_lt_i32 s6, s11
-; GFX9-NEXT:    s_cselect_b32 s6, s6, s11
+; GFX9-NEXT:    s_cmp_lt_i32 s6, s10
+; GFX9-NEXT:    s_cselect_b32 s6, s6, s10
 ; GFX9-NEXT:    s_cmp_lt_i32 s5, s4
 ; GFX9-NEXT:    s_cselect_b32 s4, s5, s4
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s6, s4
@@ -6939,23 +6847,23 @@ define amdgpu_ps <4 x i32> @s_ssubsat_v8i16(<8 x i16> inreg %lhs, <8 x i16> inre
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s4
 ; GFX9-NEXT:    s_sext_i32_i16 s4, s3
 ; GFX9-NEXT:    s_ashr_i32 s5, s3, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s4, s13
-; GFX9-NEXT:    s_cselect_b32 s6, s4, s13
-; GFX9-NEXT:    s_cmp_gt_i32 s5, s10
-; GFX9-NEXT:    s_cselect_b32 s11, s5, s10
-; GFX9-NEXT:    s_pack_ll_b32_b16 s6, s6, s11
-; GFX9-NEXT:    s_lshr_b32 s11, s6, 16
+; GFX9-NEXT:    s_cmp_gt_i32 s4, s12
+; GFX9-NEXT:    s_cselect_b32 s6, s4, s12
+; GFX9-NEXT:    s_cmp_gt_i32 s5, -1
+; GFX9-NEXT:    s_cselect_b32 s10, s5, -1
+; GFX9-NEXT:    s_pack_ll_b32_b16 s6, s6, s10
+; GFX9-NEXT:    s_lshr_b32 s10, s6, 16
 ; GFX9-NEXT:    s_sub_i32 s6, s6, s8
-; GFX9-NEXT:    s_sub_i32 s8, s11, s16
-; GFX9-NEXT:    s_cmp_lt_i32 s4, s13
-; GFX9-NEXT:    s_cselect_b32 s4, s4, s13
-; GFX9-NEXT:    s_cmp_lt_i32 s5, s10
-; GFX9-NEXT:    s_cselect_b32 s5, s5, s10
+; GFX9-NEXT:    s_sub_i32 s8, s10, s15
+; GFX9-NEXT:    s_cmp_lt_i32 s4, s12
+; GFX9-NEXT:    s_cselect_b32 s4, s4, s12
+; GFX9-NEXT:    s_cmp_lt_i32 s5, -1
+; GFX9-NEXT:    s_cselect_b32 s5, s5, -1
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
 ; GFX9-NEXT:    s_lshr_b32 s5, s4, 16
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s6, s6, s8
 ; GFX9-NEXT:    s_sub_i32 s4, s4, s9
-; GFX9-NEXT:    s_sub_i32 s5, s5, s15
+; GFX9-NEXT:    s_sub_i32 s5, s5, s14
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
 ; GFX9-NEXT:    s_sext_i32_i16 s5, s6
 ; GFX9-NEXT:    s_sext_i32_i16 s8, s7
@@ -6984,166 +6892,162 @@ define amdgpu_ps <4 x i32> @s_ssubsat_v8i16(<8 x i16> inreg %lhs, <8 x i16> inre
 ;
 ; GFX10-LABEL: s_ssubsat_v8i16:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_pack_ll_b32_b16 s8, -1, -1
-; GFX10-NEXT:    s_sext_i32_i16 s9, s0
-; GFX10-NEXT:    s_sext_i32_i16 s11, s8
+; GFX10-NEXT:    s_sext_i32_i16 s8, s0
+; GFX10-NEXT:    s_sext_i32_i16 s9, -1
 ; GFX10-NEXT:    s_ashr_i32 s10, s0, 16
-; GFX10-NEXT:    s_ashr_i32 s8, s8, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s9, s11
+; GFX10-NEXT:    s_cmp_gt_i32 s8, s9
 ; GFX10-NEXT:    s_movk_i32 s14, 0x7fff
-; GFX10-NEXT:    s_cselect_b32 s12, s9, s11
-; GFX10-NEXT:    s_cmp_gt_i32 s10, s8
-; GFX10-NEXT:    s_movk_i32 s16, 0x8000
-; GFX10-NEXT:    s_cselect_b32 s13, s10, s8
-; GFX10-NEXT:    s_pack_ll_b32_b16 s16, s16, s16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s12, s12, s13
-; GFX10-NEXT:    s_pack_ll_b32_b16 s13, s14, s14
-; GFX10-NEXT:    s_lshr_b32 s14, s12, 16
-; GFX10-NEXT:    s_lshr_b32 s15, s13, 16
-; GFX10-NEXT:    s_sub_i32 s12, s12, s13
-; GFX10-NEXT:    s_sub_i32 s14, s14, s15
-; GFX10-NEXT:    s_cmp_lt_i32 s9, s11
-; GFX10-NEXT:    s_sext_i32_i16 s18, s4
-; GFX10-NEXT:    s_cselect_b32 s9, s9, s11
-; GFX10-NEXT:    s_cmp_lt_i32 s10, s8
+; GFX10-NEXT:    s_cselect_b32 s11, s8, s9
+; GFX10-NEXT:    s_cmp_gt_i32 s10, -1
+; GFX10-NEXT:    s_mov_b32 s15, 0x80008000
+; GFX10-NEXT:    s_cselect_b32 s12, s10, -1
+; GFX10-NEXT:    s_sext_i32_i16 s17, s4
+; GFX10-NEXT:    s_pack_ll_b32_b16 s11, s11, s12
+; GFX10-NEXT:    s_mov_b32 s12, 0x7fff7fff
+; GFX10-NEXT:    s_lshr_b32 s13, s11, 16
+; GFX10-NEXT:    s_sub_i32 s11, s11, s12
+; GFX10-NEXT:    s_sub_i32 s13, s13, s14
+; GFX10-NEXT:    s_cmp_lt_i32 s8, s9
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cselect_b32 s10, s10, s8
+; GFX10-NEXT:    s_cselect_b32 s8, s8, s9
+; GFX10-NEXT:    s_cmp_lt_i32 s10, -1
+; GFX10-NEXT:    s_cselect_b32 s10, s10, -1
 ; GFX10-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s9, s9, s10
-; GFX10-NEXT:    s_pack_ll_b32_b16 s10, s12, s14
-; GFX10-NEXT:    s_lshr_b32 s12, s9, 16
-; GFX10-NEXT:    s_lshr_b32 s14, s16, 16
-; GFX10-NEXT:    s_sext_i32_i16 s17, s10
-; GFX10-NEXT:    s_sub_i32 s9, s9, s16
-; GFX10-NEXT:    s_sub_i32 s12, s12, s14
+; GFX10-NEXT:    s_pack_ll_b32_b16 s8, s8, s10
+; GFX10-NEXT:    s_pack_ll_b32_b16 s10, s11, s13
+; GFX10-NEXT:    s_lshr_b32 s11, s8, 16
+; GFX10-NEXT:    s_mov_b32 s13, 0x8000
+; GFX10-NEXT:    s_sext_i32_i16 s16, s10
+; GFX10-NEXT:    s_sub_i32 s8, s8, s15
+; GFX10-NEXT:    s_sub_i32 s11, s11, s13
 ; GFX10-NEXT:    s_ashr_i32 s10, s10, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s17, s18
-; GFX10-NEXT:    s_pack_ll_b32_b16 s9, s9, s12
-; GFX10-NEXT:    s_cselect_b32 s17, s17, s18
+; GFX10-NEXT:    s_cmp_gt_i32 s16, s17
+; GFX10-NEXT:    s_pack_ll_b32_b16 s8, s8, s11
+; GFX10-NEXT:    s_cselect_b32 s16, s16, s17
 ; GFX10-NEXT:    s_cmp_gt_i32 s10, s4
-; GFX10-NEXT:    s_sext_i32_i16 s12, s9
+; GFX10-NEXT:    s_sext_i32_i16 s11, s8
 ; GFX10-NEXT:    s_cselect_b32 s4, s10, s4
-; GFX10-NEXT:    s_ashr_i32 s9, s9, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s17, s4
-; GFX10-NEXT:    s_sext_i32_i16 s18, s5
+; GFX10-NEXT:    s_ashr_i32 s8, s8, 16
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s16, s4
+; GFX10-NEXT:    s_sext_i32_i16 s17, s5
 ; GFX10-NEXT:    s_sext_i32_i16 s10, s4
 ; GFX10-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX10-NEXT:    s_cmp_lt_i32 s10, s12
-; GFX10-NEXT:    s_cselect_b32 s10, s10, s12
-; GFX10-NEXT:    s_cmp_lt_i32 s4, s9
-; GFX10-NEXT:    s_cselect_b32 s4, s4, s9
-; GFX10-NEXT:    s_lshr_b32 s9, s0, 16
+; GFX10-NEXT:    s_cmp_lt_i32 s10, s11
+; GFX10-NEXT:    s_cselect_b32 s10, s10, s11
+; GFX10-NEXT:    s_cmp_lt_i32 s4, s8
+; GFX10-NEXT:    s_cselect_b32 s4, s4, s8
+; GFX10-NEXT:    s_lshr_b32 s8, s0, 16
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s10, s4
 ; GFX10-NEXT:    s_lshr_b32 s10, s4, 16
 ; GFX10-NEXT:    s_sub_i32 s0, s0, s4
-; GFX10-NEXT:    s_sub_i32 s4, s9, s10
-; GFX10-NEXT:    s_sext_i32_i16 s9, s1
+; GFX10-NEXT:    s_sub_i32 s4, s8, s10
+; GFX10-NEXT:    s_sext_i32_i16 s8, s1
 ; GFX10-NEXT:    s_ashr_i32 s10, s1, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s9, s11
+; GFX10-NEXT:    s_cmp_gt_i32 s8, s9
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s4
-; GFX10-NEXT:    s_cselect_b32 s12, s9, s11
-; GFX10-NEXT:    s_cmp_gt_i32 s10, s8
-; GFX10-NEXT:    s_cselect_b32 s17, s10, s8
-; GFX10-NEXT:    s_pack_ll_b32_b16 s12, s12, s17
-; GFX10-NEXT:    s_lshr_b32 s17, s12, 16
-; GFX10-NEXT:    s_sub_i32 s12, s12, s13
-; GFX10-NEXT:    s_sub_i32 s17, s17, s15
-; GFX10-NEXT:    s_cmp_lt_i32 s9, s11
-; GFX10-NEXT:    s_cselect_b32 s9, s9, s11
-; GFX10-NEXT:    s_cmp_lt_i32 s10, s8
-; GFX10-NEXT:    s_cselect_b32 s10, s10, s8
+; GFX10-NEXT:    s_cselect_b32 s11, s8, s9
+; GFX10-NEXT:    s_cmp_gt_i32 s10, -1
+; GFX10-NEXT:    s_cselect_b32 s16, s10, -1
+; GFX10-NEXT:    s_pack_ll_b32_b16 s11, s11, s16
+; GFX10-NEXT:    s_lshr_b32 s16, s11, 16
+; GFX10-NEXT:    s_sub_i32 s11, s11, s12
+; GFX10-NEXT:    s_sub_i32 s16, s16, s14
+; GFX10-NEXT:    s_cmp_lt_i32 s8, s9
+; GFX10-NEXT:    s_cselect_b32 s8, s8, s9
+; GFX10-NEXT:    s_cmp_lt_i32 s10, -1
+; GFX10-NEXT:    s_cselect_b32 s10, s10, -1
 ; GFX10-NEXT:    s_ashr_i32 s5, s5, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s9, s9, s10
-; GFX10-NEXT:    s_pack_ll_b32_b16 s10, s12, s17
-; GFX10-NEXT:    s_lshr_b32 s12, s9, 16
-; GFX10-NEXT:    s_sext_i32_i16 s17, s10
-; GFX10-NEXT:    s_sub_i32 s9, s9, s16
-; GFX10-NEXT:    s_sub_i32 s12, s12, s14
+; GFX10-NEXT:    s_pack_ll_b32_b16 s8, s8, s10
+; GFX10-NEXT:    s_pack_ll_b32_b16 s10, s11, s16
+; GFX10-NEXT:    s_lshr_b32 s11, s8, 16
+; GFX10-NEXT:    s_sext_i32_i16 s16, s10
+; GFX10-NEXT:    s_sub_i32 s8, s8, s15
+; GFX10-NEXT:    s_sub_i32 s11, s11, s13
 ; GFX10-NEXT:    s_ashr_i32 s10, s10, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s17, s18
-; GFX10-NEXT:    s_pack_ll_b32_b16 s9, s9, s12
-; GFX10-NEXT:    s_cselect_b32 s17, s17, s18
+; GFX10-NEXT:    s_cmp_gt_i32 s16, s17
+; GFX10-NEXT:    s_pack_ll_b32_b16 s8, s8, s11
+; GFX10-NEXT:    s_cselect_b32 s16, s16, s17
 ; GFX10-NEXT:    s_cmp_gt_i32 s10, s5
-; GFX10-NEXT:    s_sext_i32_i16 s12, s9
+; GFX10-NEXT:    s_sext_i32_i16 s11, s8
 ; GFX10-NEXT:    s_cselect_b32 s5, s10, s5
-; GFX10-NEXT:    s_ashr_i32 s9, s9, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s5, s17, s5
-; GFX10-NEXT:    s_sext_i32_i16 s18, s6
+; GFX10-NEXT:    s_ashr_i32 s8, s8, 16
+; GFX10-NEXT:    s_pack_ll_b32_b16 s5, s16, s5
+; GFX10-NEXT:    s_sext_i32_i16 s17, s6
 ; GFX10-NEXT:    s_sext_i32_i16 s10, s5
 ; GFX10-NEXT:    s_ashr_i32 s5, s5, 16
-; GFX10-NEXT:    s_cmp_lt_i32 s10, s12
-; GFX10-NEXT:    s_cselect_b32 s10, s10, s12
-; GFX10-NEXT:    s_cmp_lt_i32 s5, s9
-; GFX10-NEXT:    s_cselect_b32 s5, s5, s9
-; GFX10-NEXT:    s_lshr_b32 s9, s1, 16
+; GFX10-NEXT:    s_cmp_lt_i32 s10, s11
+; GFX10-NEXT:    s_cselect_b32 s10, s10, s11
+; GFX10-NEXT:    s_cmp_lt_i32 s5, s8
+; GFX10-NEXT:    s_cselect_b32 s5, s5, s8
+; GFX10-NEXT:    s_lshr_b32 s8, s1, 16
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s5, s10, s5
 ; GFX10-NEXT:    s_lshr_b32 s10, s5, 16
 ; GFX10-NEXT:    s_sub_i32 s1, s1, s5
-; GFX10-NEXT:    s_sub_i32 s5, s9, s10
-; GFX10-NEXT:    s_sext_i32_i16 s9, s2
+; GFX10-NEXT:    s_sub_i32 s5, s8, s10
+; GFX10-NEXT:    s_sext_i32_i16 s8, s2
 ; GFX10-NEXT:    s_ashr_i32 s10, s2, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s9, s11
+; GFX10-NEXT:    s_cmp_gt_i32 s8, s9
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s1, s5
-; GFX10-NEXT:    s_cselect_b32 s12, s9, s11
-; GFX10-NEXT:    s_cmp_gt_i32 s10, s8
-; GFX10-NEXT:    s_cselect_b32 s17, s10, s8
-; GFX10-NEXT:    s_pack_ll_b32_b16 s12, s12, s17
-; GFX10-NEXT:    s_lshr_b32 s17, s12, 16
-; GFX10-NEXT:    s_sub_i32 s12, s12, s13
-; GFX10-NEXT:    s_sub_i32 s17, s17, s15
-; GFX10-NEXT:    s_cmp_lt_i32 s9, s11
-; GFX10-NEXT:    s_cselect_b32 s9, s9, s11
-; GFX10-NEXT:    s_cmp_lt_i32 s10, s8
-; GFX10-NEXT:    s_cselect_b32 s10, s10, s8
+; GFX10-NEXT:    s_cselect_b32 s11, s8, s9
+; GFX10-NEXT:    s_cmp_gt_i32 s10, -1
+; GFX10-NEXT:    s_cselect_b32 s16, s10, -1
+; GFX10-NEXT:    s_pack_ll_b32_b16 s11, s11, s16
+; GFX10-NEXT:    s_lshr_b32 s16, s11, 16
+; GFX10-NEXT:    s_sub_i32 s11, s11, s12
+; GFX10-NEXT:    s_sub_i32 s16, s16, s14
+; GFX10-NEXT:    s_cmp_lt_i32 s8, s9
+; GFX10-NEXT:    s_cselect_b32 s8, s8, s9
+; GFX10-NEXT:    s_cmp_lt_i32 s10, -1
+; GFX10-NEXT:    s_cselect_b32 s10, s10, -1
 ; GFX10-NEXT:    s_ashr_i32 s6, s6, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s9, s9, s10
-; GFX10-NEXT:    s_pack_ll_b32_b16 s10, s12, s17
-; GFX10-NEXT:    s_lshr_b32 s12, s9, 16
-; GFX10-NEXT:    s_sext_i32_i16 s17, s10
-; GFX10-NEXT:    s_sub_i32 s9, s9, s16
-; GFX10-NEXT:    s_sub_i32 s12, s12, s14
+; GFX10-NEXT:    s_pack_ll_b32_b16 s8, s8, s10
+; GFX10-NEXT:    s_pack_ll_b32_b16 s10, s11, s16
+; GFX10-NEXT:    s_lshr_b32 s11, s8, 16
+; GFX10-NEXT:    s_sext_i32_i16 s16, s10
+; GFX10-NEXT:    s_sub_i32 s8, s8, s15
+; GFX10-NEXT:    s_sub_i32 s11, s11, s13
 ; GFX10-NEXT:    s_ashr_i32 s10, s10, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s17, s18
-; GFX10-NEXT:    s_pack_ll_b32_b16 s9, s9, s12
-; GFX10-NEXT:    s_cselect_b32 s17, s17, s18
+; GFX10-NEXT:    s_cmp_gt_i32 s16, s17
+; GFX10-NEXT:    s_pack_ll_b32_b16 s8, s8, s11
+; GFX10-NEXT:    s_cselect_b32 s16, s16, s17
 ; GFX10-NEXT:    s_cmp_gt_i32 s10, s6
-; GFX10-NEXT:    s_sext_i32_i16 s12, s9
+; GFX10-NEXT:    s_sext_i32_i16 s11, s8
 ; GFX10-NEXT:    s_cselect_b32 s6, s10, s6
-; GFX10-NEXT:    s_ashr_i32 s9, s9, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s17, s6
+; GFX10-NEXT:    s_ashr_i32 s8, s8, 16
+; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s16, s6
 ; GFX10-NEXT:    s_sext_i32_i16 s10, s6
 ; GFX10-NEXT:    s_ashr_i32 s6, s6, 16
-; GFX10-NEXT:    s_cmp_lt_i32 s10, s12
-; GFX10-NEXT:    s_cselect_b32 s10, s10, s12
-; GFX10-NEXT:    s_cmp_lt_i32 s6, s9
-; GFX10-NEXT:    s_cselect_b32 s6, s6, s9
-; GFX10-NEXT:    s_lshr_b32 s9, s2, 16
+; GFX10-NEXT:    s_cmp_lt_i32 s10, s11
+; GFX10-NEXT:    s_cselect_b32 s10, s10, s11
+; GFX10-NEXT:    s_cmp_lt_i32 s6, s8
+; GFX10-NEXT:    s_cselect_b32 s6, s6, s8
+; GFX10-NEXT:    s_lshr_b32 s8, s2, 16
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s10, s6
 ; GFX10-NEXT:    s_lshr_b32 s10, s6, 16
 ; GFX10-NEXT:    s_sub_i32 s2, s2, s6
-; GFX10-NEXT:    s_sub_i32 s6, s9, s10
-; GFX10-NEXT:    s_sext_i32_i16 s9, s3
+; GFX10-NEXT:    s_sub_i32 s6, s8, s10
+; GFX10-NEXT:    s_sext_i32_i16 s8, s3
 ; GFX10-NEXT:    s_ashr_i32 s10, s3, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s9, s11
+; GFX10-NEXT:    s_cmp_gt_i32 s8, s9
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s2, s6
-; GFX10-NEXT:    s_cselect_b32 s12, s9, s11
-; GFX10-NEXT:    s_cmp_gt_i32 s10, s8
-; GFX10-NEXT:    s_cselect_b32 s17, s10, s8
-; GFX10-NEXT:    s_pack_ll_b32_b16 s12, s12, s17
-; GFX10-NEXT:    s_lshr_b32 s17, s12, 16
-; GFX10-NEXT:    s_sub_i32 s12, s12, s13
-; GFX10-NEXT:    s_sub_i32 s13, s17, s15
-; GFX10-NEXT:    s_cmp_lt_i32 s9, s11
-; GFX10-NEXT:    s_cselect_b32 s9, s9, s11
-; GFX10-NEXT:    s_cmp_lt_i32 s10, s8
-; GFX10-NEXT:    s_cselect_b32 s8, s10, s8
-; GFX10-NEXT:    s_pack_ll_b32_b16 s8, s9, s8
-; GFX10-NEXT:    s_pack_ll_b32_b16 s9, s12, s13
+; GFX10-NEXT:    s_cselect_b32 s11, s8, s9
+; GFX10-NEXT:    s_cmp_gt_i32 s10, -1
+; GFX10-NEXT:    s_cselect_b32 s16, s10, -1
+; GFX10-NEXT:    s_pack_ll_b32_b16 s11, s11, s16
+; GFX10-NEXT:    s_lshr_b32 s16, s11, 16
+; GFX10-NEXT:    s_sub_i32 s11, s11, s12
+; GFX10-NEXT:    s_sub_i32 s12, s16, s14
+; GFX10-NEXT:    s_cmp_lt_i32 s8, s9
+; GFX10-NEXT:    s_cselect_b32 s8, s8, s9
+; GFX10-NEXT:    s_cmp_lt_i32 s10, -1
+; GFX10-NEXT:    s_cselect_b32 s9, s10, -1
+; GFX10-NEXT:    s_pack_ll_b32_b16 s8, s8, s9
+; GFX10-NEXT:    s_pack_ll_b32_b16 s9, s11, s12
 ; GFX10-NEXT:    s_lshr_b32 s10, s8, 16
 ; GFX10-NEXT:    s_sext_i32_i16 s11, s9
 ; GFX10-NEXT:    s_sext_i32_i16 s12, s7
-; GFX10-NEXT:    s_sub_i32 s8, s8, s16
-; GFX10-NEXT:    s_sub_i32 s10, s10, s14
+; GFX10-NEXT:    s_sub_i32 s8, s8, s15
+; GFX10-NEXT:    s_sub_i32 s10, s10, s13
 ; GFX10-NEXT:    s_ashr_i32 s9, s9, 16
 ; GFX10-NEXT:    s_ashr_i32 s7, s7, 16
 ; GFX10-NEXT:    s_cmp_gt_i32 s11, s12
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll
index 5570309a5be7d..3a742fbcbd919 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll
@@ -2371,8 +2371,7 @@ define <2 x i16> @v_uaddsat_v2i16(<2 x i16> %lhs, <2 x i16> %rhs) {
 ; GFX9-LABEL: v_uaddsat_v2i16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, -1, -1
-; GFX9-NEXT:    v_xor_b32_e32 v2, s4, v0
+; GFX9-NEXT:    v_xor_b32_e32 v2, -1, v0
 ; GFX9-NEXT:    v_pk_min_u16 v1, v2, v1
 ; GFX9-NEXT:    v_pk_add_u16 v0, v0, v1
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -2381,9 +2380,8 @@ define <2 x i16> @v_uaddsat_v2i16(<2 x i16> %lhs, <2 x i16> %rhs) {
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, -1, -1
+; GFX10-NEXT:    v_xor_b32_e32 v2, -1, v0
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_xor_b32_e32 v2, s4, v0
 ; GFX10-NEXT:    v_pk_min_u16 v1, v2, v1
 ; GFX10-NEXT:    v_pk_add_u16 v0, v0, v1
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
@@ -2439,8 +2437,7 @@ define amdgpu_ps i32 @s_uaddsat_v2i16(<2 x i16> inreg %lhs, <2 x i16> inreg %rhs
 ;
 ; GFX9-LABEL: s_uaddsat_v2i16:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_pack_ll_b32_b16 s2, -1, -1
-; GFX9-NEXT:    s_xor_b32 s2, s0, s2
+; GFX9-NEXT:    s_xor_b32 s2, s0, -1
 ; GFX9-NEXT:    s_mov_b32 s4, 0xffff
 ; GFX9-NEXT:    s_lshr_b32 s3, s2, 16
 ; GFX9-NEXT:    s_lshr_b32 s5, s1, 16
@@ -2460,15 +2457,14 @@ define amdgpu_ps i32 @s_uaddsat_v2i16(<2 x i16> inreg %lhs, <2 x i16> inreg %rhs
 ;
 ; GFX10-LABEL: s_uaddsat_v2i16:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_pack_ll_b32_b16 s2, -1, -1
+; GFX10-NEXT:    s_xor_b32 s2, s0, -1
 ; GFX10-NEXT:    s_mov_b32 s3, 0xffff
-; GFX10-NEXT:    s_xor_b32 s2, s0, s2
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_lshr_b32 s4, s2, 16
 ; GFX10-NEXT:    s_and_b32 s2, s2, s3
 ; GFX10-NEXT:    s_and_b32 s3, s1, s3
 ; GFX10-NEXT:    s_lshr_b32 s1, s1, 16
 ; GFX10-NEXT:    s_cmp_lt_u32 s2, s3
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_cselect_b32 s2, s2, s3
 ; GFX10-NEXT:    s_cmp_lt_u32 s4, s1
 ; GFX10-NEXT:    s_cselect_b32 s1, s4, s1
@@ -2522,17 +2518,15 @@ define amdgpu_ps float @uaddsat_v2i16_sv(<2 x i16> inreg %lhs, <2 x i16> %rhs) {
 ;
 ; GFX9-LABEL: uaddsat_v2i16_sv:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_pack_ll_b32_b16 s1, -1, -1
-; GFX9-NEXT:    s_xor_b32 s1, s0, s1
+; GFX9-NEXT:    s_xor_b32 s1, s0, -1
 ; GFX9-NEXT:    v_pk_min_u16 v0, s1, v0
 ; GFX9-NEXT:    v_pk_add_u16 v0, s0, v0
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: uaddsat_v2i16_sv:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_pack_ll_b32_b16 s1, -1, -1
+; GFX10-NEXT:    s_xor_b32 s1, s0, -1
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_xor_b32 s1, s0, s1
 ; GFX10-NEXT:    v_pk_min_u16 v0, s1, v0
 ; GFX10-NEXT:    v_pk_add_u16 v0, s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -2578,17 +2572,15 @@ define amdgpu_ps float @uaddsat_v2i16_vs(<2 x i16> %lhs, <2 x i16> inreg %rhs) {
 ;
 ; GFX9-LABEL: uaddsat_v2i16_vs:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_pack_ll_b32_b16 s1, -1, -1
-; GFX9-NEXT:    v_xor_b32_e32 v1, s1, v0
+; GFX9-NEXT:    v_xor_b32_e32 v1, -1, v0
 ; GFX9-NEXT:    v_pk_min_u16 v1, v1, s0
 ; GFX9-NEXT:    v_pk_add_u16 v0, v0, v1
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: uaddsat_v2i16_vs:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_pack_ll_b32_b16 s1, -1, -1
+; GFX10-NEXT:    v_xor_b32_e32 v1, -1, v0
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_xor_b32_e32 v1, s1, v0
 ; GFX10-NEXT:    v_pk_min_u16 v1, v1, s0
 ; GFX10-NEXT:    v_pk_add_u16 v0, v0, v1
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -2671,11 +2663,10 @@ define <2 x float> @v_uaddsat_v4i16(<4 x i16> %lhs, <4 x i16> %rhs) {
 ; GFX9-LABEL: v_uaddsat_v4i16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, -1, -1
-; GFX9-NEXT:    v_xor_b32_e32 v4, s4, v0
+; GFX9-NEXT:    v_xor_b32_e32 v4, -1, v0
 ; GFX9-NEXT:    v_pk_min_u16 v2, v4, v2
 ; GFX9-NEXT:    v_pk_add_u16 v0, v0, v2
-; GFX9-NEXT:    v_xor_b32_e32 v2, s4, v1
+; GFX9-NEXT:    v_xor_b32_e32 v2, -1, v1
 ; GFX9-NEXT:    v_pk_min_u16 v2, v2, v3
 ; GFX9-NEXT:    v_pk_add_u16 v1, v1, v2
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -2684,10 +2675,9 @@ define <2 x float> @v_uaddsat_v4i16(<4 x i16> %lhs, <4 x i16> %rhs) {
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, -1, -1
+; GFX10-NEXT:    v_xor_b32_e32 v4, -1, v0
+; GFX10-NEXT:    v_xor_b32_e32 v5, -1, v1
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_xor_b32_e32 v4, s4, v0
-; GFX10-NEXT:    v_xor_b32_e32 v5, s4, v1
 ; GFX10-NEXT:    v_pk_min_u16 v2, v4, v2
 ; GFX10-NEXT:    v_pk_min_u16 v3, v5, v3
 ; GFX10-NEXT:    v_pk_add_u16 v0, v0, v2
@@ -2782,28 +2772,27 @@ define amdgpu_ps <2 x i32> @s_uaddsat_v4i16(<4 x i16> inreg %lhs, <4 x i16> inre
 ;
 ; GFX9-LABEL: s_uaddsat_v4i16:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, -1, -1
-; GFX9-NEXT:    s_xor_b32 s5, s0, s4
-; GFX9-NEXT:    s_mov_b32 s7, 0xffff
-; GFX9-NEXT:    s_lshr_b32 s6, s5, 16
-; GFX9-NEXT:    s_lshr_b32 s8, s2, 16
-; GFX9-NEXT:    s_and_b32 s5, s5, s7
-; GFX9-NEXT:    s_and_b32 s2, s2, s7
-; GFX9-NEXT:    s_cmp_lt_u32 s5, s2
-; GFX9-NEXT:    s_cselect_b32 s2, s5, s2
-; GFX9-NEXT:    s_cmp_lt_u32 s6, s8
-; GFX9-NEXT:    s_cselect_b32 s5, s6, s8
-; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s5
-; GFX9-NEXT:    s_lshr_b32 s5, s0, 16
-; GFX9-NEXT:    s_lshr_b32 s6, s2, 16
+; GFX9-NEXT:    s_xor_b32 s4, s0, -1
+; GFX9-NEXT:    s_mov_b32 s6, 0xffff
+; GFX9-NEXT:    s_lshr_b32 s5, s4, 16
+; GFX9-NEXT:    s_lshr_b32 s7, s2, 16
+; GFX9-NEXT:    s_and_b32 s4, s4, s6
+; GFX9-NEXT:    s_and_b32 s2, s2, s6
+; GFX9-NEXT:    s_cmp_lt_u32 s4, s2
+; GFX9-NEXT:    s_cselect_b32 s2, s4, s2
+; GFX9-NEXT:    s_cmp_lt_u32 s5, s7
+; GFX9-NEXT:    s_cselect_b32 s4, s5, s7
+; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s4
+; GFX9-NEXT:    s_lshr_b32 s4, s0, 16
+; GFX9-NEXT:    s_lshr_b32 s5, s2, 16
 ; GFX9-NEXT:    s_add_i32 s0, s0, s2
-; GFX9-NEXT:    s_xor_b32 s2, s1, s4
-; GFX9-NEXT:    s_add_i32 s5, s5, s6
-; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s5
+; GFX9-NEXT:    s_add_i32 s4, s4, s5
+; GFX9-NEXT:    s_xor_b32 s2, s1, -1
+; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s4
 ; GFX9-NEXT:    s_lshr_b32 s4, s2, 16
 ; GFX9-NEXT:    s_lshr_b32 s5, s3, 16
-; GFX9-NEXT:    s_and_b32 s2, s2, s7
-; GFX9-NEXT:    s_and_b32 s3, s3, s7
+; GFX9-NEXT:    s_and_b32 s2, s2, s6
+; GFX9-NEXT:    s_and_b32 s3, s3, s6
 ; GFX9-NEXT:    s_cmp_lt_u32 s2, s3
 ; GFX9-NEXT:    s_cselect_b32 s2, s2, s3
 ; GFX9-NEXT:    s_cmp_lt_u32 s4, s5
@@ -2818,38 +2807,37 @@ define amdgpu_ps <2 x i32> @s_uaddsat_v4i16(<4 x i16> inreg %lhs, <4 x i16> inre
 ;
 ; GFX10-LABEL: s_uaddsat_v4i16:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, -1, -1
-; GFX10-NEXT:    s_mov_b32 s6, 0xffff
-; GFX10-NEXT:    s_xor_b32 s5, s0, s4
-; GFX10-NEXT:    s_and_b32 s8, s2, s6
-; GFX10-NEXT:    s_lshr_b32 s7, s5, 16
-; GFX10-NEXT:    s_and_b32 s5, s5, s6
+; GFX10-NEXT:    s_xor_b32 s4, s0, -1
+; GFX10-NEXT:    s_mov_b32 s5, 0xffff
+; GFX10-NEXT:    s_lshr_b32 s6, s4, 16
+; GFX10-NEXT:    s_and_b32 s7, s2, s5
+; GFX10-NEXT:    s_and_b32 s4, s4, s5
 ; GFX10-NEXT:    s_lshr_b32 s2, s2, 16
-; GFX10-NEXT:    s_cmp_lt_u32 s5, s8
+; GFX10-NEXT:    s_cmp_lt_u32 s4, s7
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cselect_b32 s5, s5, s8
-; GFX10-NEXT:    s_cmp_lt_u32 s7, s2
-; GFX10-NEXT:    s_cselect_b32 s2, s7, s2
-; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s5, s2
-; GFX10-NEXT:    s_lshr_b32 s5, s0, 16
-; GFX10-NEXT:    s_lshr_b32 s7, s2, 16
+; GFX10-NEXT:    s_cselect_b32 s4, s4, s7
+; GFX10-NEXT:    s_cmp_lt_u32 s6, s2
+; GFX10-NEXT:    s_cselect_b32 s2, s6, s2
+; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s4, s2
+; GFX10-NEXT:    s_lshr_b32 s4, s0, 16
+; GFX10-NEXT:    s_lshr_b32 s6, s2, 16
 ; GFX10-NEXT:    s_add_i32 s0, s0, s2
-; GFX10-NEXT:    s_xor_b32 s2, s1, s4
-; GFX10-NEXT:    s_add_i32 s5, s5, s7
-; GFX10-NEXT:    s_lshr_b32 s4, s2, 16
-; GFX10-NEXT:    s_and_b32 s2, s2, s6
-; GFX10-NEXT:    s_and_b32 s6, s3, s6
+; GFX10-NEXT:    s_xor_b32 s2, s1, -1
+; GFX10-NEXT:    s_add_i32 s4, s4, s6
+; GFX10-NEXT:    s_lshr_b32 s6, s2, 16
+; GFX10-NEXT:    s_and_b32 s2, s2, s5
+; GFX10-NEXT:    s_and_b32 s5, s3, s5
 ; GFX10-NEXT:    s_lshr_b32 s3, s3, 16
-; GFX10-NEXT:    s_cmp_lt_u32 s2, s6
-; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s5
-; GFX10-NEXT:    s_cselect_b32 s2, s2, s6
-; GFX10-NEXT:    s_cmp_lt_u32 s4, s3
-; GFX10-NEXT:    s_cselect_b32 s3, s4, s3
+; GFX10-NEXT:    s_cmp_lt_u32 s2, s5
+; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s4
+; GFX10-NEXT:    s_cselect_b32 s2, s2, s5
+; GFX10-NEXT:    s_cmp_lt_u32 s6, s3
+; GFX10-NEXT:    s_cselect_b32 s3, s6, s3
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s2, s3
 ; GFX10-NEXT:    s_lshr_b32 s3, s1, 16
-; GFX10-NEXT:    s_lshr_b32 s4, s2, 16
+; GFX10-NEXT:    s_lshr_b32 s5, s2, 16
 ; GFX10-NEXT:    s_add_i32 s1, s1, s2
-; GFX10-NEXT:    s_add_i32 s3, s3, s4
+; GFX10-NEXT:    s_add_i32 s3, s3, s5
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s1, s3
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <4 x i16> @llvm.uadd.sat.v4i16(<4 x i16> %lhs, <4 x i16> %rhs)
@@ -2955,14 +2943,13 @@ define <3 x float> @v_uaddsat_v6i16(<6 x i16> %lhs, <6 x i16> %rhs) {
 ; GFX9-LABEL: v_uaddsat_v6i16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, -1, -1
-; GFX9-NEXT:    v_xor_b32_e32 v6, s4, v0
+; GFX9-NEXT:    v_xor_b32_e32 v6, -1, v0
 ; GFX9-NEXT:    v_pk_min_u16 v3, v6, v3
 ; GFX9-NEXT:    v_pk_add_u16 v0, v0, v3
-; GFX9-NEXT:    v_xor_b32_e32 v3, s4, v1
+; GFX9-NEXT:    v_xor_b32_e32 v3, -1, v1
 ; GFX9-NEXT:    v_pk_min_u16 v3, v3, v4
 ; GFX9-NEXT:    v_pk_add_u16 v1, v1, v3
-; GFX9-NEXT:    v_xor_b32_e32 v3, s4, v2
+; GFX9-NEXT:    v_xor_b32_e32 v3, -1, v2
 ; GFX9-NEXT:    v_pk_min_u16 v3, v3, v5
 ; GFX9-NEXT:    v_pk_add_u16 v2, v2, v3
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -2971,11 +2958,10 @@ define <3 x float> @v_uaddsat_v6i16(<6 x i16> %lhs, <6 x i16> %rhs) {
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, -1, -1
+; GFX10-NEXT:    v_xor_b32_e32 v6, -1, v0
+; GFX10-NEXT:    v_xor_b32_e32 v7, -1, v1
+; GFX10-NEXT:    v_xor_b32_e32 v8, -1, v2
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_xor_b32_e32 v6, s4, v0
-; GFX10-NEXT:    v_xor_b32_e32 v7, s4, v1
-; GFX10-NEXT:    v_xor_b32_e32 v8, s4, v2
 ; GFX10-NEXT:    v_pk_min_u16 v3, v6, v3
 ; GFX10-NEXT:    v_pk_min_u16 v4, v7, v4
 ; GFX10-NEXT:    v_pk_min_u16 v5, v8, v5
@@ -3108,43 +3094,42 @@ define amdgpu_ps <3 x i32> @s_uaddsat_v6i16(<6 x i16> inreg %lhs, <6 x i16> inre
 ;
 ; GFX9-LABEL: s_uaddsat_v6i16:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_pack_ll_b32_b16 s6, -1, -1
-; GFX9-NEXT:    s_xor_b32 s7, s0, s6
-; GFX9-NEXT:    s_mov_b32 s9, 0xffff
-; GFX9-NEXT:    s_lshr_b32 s8, s7, 16
-; GFX9-NEXT:    s_lshr_b32 s10, s3, 16
-; GFX9-NEXT:    s_and_b32 s7, s7, s9
-; GFX9-NEXT:    s_and_b32 s3, s3, s9
-; GFX9-NEXT:    s_cmp_lt_u32 s7, s3
-; GFX9-NEXT:    s_cselect_b32 s3, s7, s3
-; GFX9-NEXT:    s_cmp_lt_u32 s8, s10
-; GFX9-NEXT:    s_cselect_b32 s7, s8, s10
-; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s3, s7
-; GFX9-NEXT:    s_lshr_b32 s7, s0, 16
-; GFX9-NEXT:    s_lshr_b32 s8, s3, 16
-; GFX9-NEXT:    s_add_i32 s0, s0, s3
-; GFX9-NEXT:    s_add_i32 s7, s7, s8
-; GFX9-NEXT:    s_xor_b32 s3, s1, s6
-; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s7
+; GFX9-NEXT:    s_xor_b32 s6, s0, -1
+; GFX9-NEXT:    s_mov_b32 s8, 0xffff
+; GFX9-NEXT:    s_lshr_b32 s7, s6, 16
+; GFX9-NEXT:    s_lshr_b32 s9, s3, 16
+; GFX9-NEXT:    s_and_b32 s6, s6, s8
+; GFX9-NEXT:    s_and_b32 s3, s3, s8
+; GFX9-NEXT:    s_cmp_lt_u32 s6, s3
+; GFX9-NEXT:    s_cselect_b32 s3, s6, s3
+; GFX9-NEXT:    s_cmp_lt_u32 s7, s9
+; GFX9-NEXT:    s_cselect_b32 s6, s7, s9
+; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s3, s6
+; GFX9-NEXT:    s_lshr_b32 s6, s0, 16
 ; GFX9-NEXT:    s_lshr_b32 s7, s3, 16
-; GFX9-NEXT:    s_lshr_b32 s8, s4, 16
-; GFX9-NEXT:    s_and_b32 s3, s3, s9
-; GFX9-NEXT:    s_and_b32 s4, s4, s9
+; GFX9-NEXT:    s_add_i32 s0, s0, s3
+; GFX9-NEXT:    s_add_i32 s6, s6, s7
+; GFX9-NEXT:    s_xor_b32 s3, s1, -1
+; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s6
+; GFX9-NEXT:    s_lshr_b32 s6, s3, 16
+; GFX9-NEXT:    s_lshr_b32 s7, s4, 16
+; GFX9-NEXT:    s_and_b32 s3, s3, s8
+; GFX9-NEXT:    s_and_b32 s4, s4, s8
 ; GFX9-NEXT:    s_cmp_lt_u32 s3, s4
 ; GFX9-NEXT:    s_cselect_b32 s3, s3, s4
-; GFX9-NEXT:    s_cmp_lt_u32 s7, s8
-; GFX9-NEXT:    s_cselect_b32 s4, s7, s8
+; GFX9-NEXT:    s_cmp_lt_u32 s6, s7
+; GFX9-NEXT:    s_cselect_b32 s4, s6, s7
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s3, s4
 ; GFX9-NEXT:    s_lshr_b32 s4, s1, 16
-; GFX9-NEXT:    s_lshr_b32 s7, s3, 16
+; GFX9-NEXT:    s_lshr_b32 s6, s3, 16
 ; GFX9-NEXT:    s_add_i32 s1, s1, s3
-; GFX9-NEXT:    s_add_i32 s4, s4, s7
-; GFX9-NEXT:    s_xor_b32 s3, s2, s6
+; GFX9-NEXT:    s_add_i32 s4, s4, s6
+; GFX9-NEXT:    s_xor_b32 s3, s2, -1
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s4
 ; GFX9-NEXT:    s_lshr_b32 s4, s3, 16
 ; GFX9-NEXT:    s_lshr_b32 s6, s5, 16
-; GFX9-NEXT:    s_and_b32 s3, s3, s9
-; GFX9-NEXT:    s_and_b32 s5, s5, s9
+; GFX9-NEXT:    s_and_b32 s3, s3, s8
+; GFX9-NEXT:    s_and_b32 s5, s5, s8
 ; GFX9-NEXT:    s_cmp_lt_u32 s3, s5
 ; GFX9-NEXT:    s_cselect_b32 s3, s3, s5
 ; GFX9-NEXT:    s_cmp_lt_u32 s4, s6
@@ -3159,48 +3144,47 @@ define amdgpu_ps <3 x i32> @s_uaddsat_v6i16(<6 x i16> inreg %lhs, <6 x i16> inre
 ;
 ; GFX10-LABEL: s_uaddsat_v6i16:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_pack_ll_b32_b16 s6, -1, -1
-; GFX10-NEXT:    s_mov_b32 s8, 0xffff
-; GFX10-NEXT:    s_xor_b32 s7, s0, s6
-; GFX10-NEXT:    s_and_b32 s10, s3, s8
-; GFX10-NEXT:    s_lshr_b32 s9, s7, 16
-; GFX10-NEXT:    s_and_b32 s7, s7, s8
+; GFX10-NEXT:    s_xor_b32 s6, s0, -1
+; GFX10-NEXT:    s_mov_b32 s7, 0xffff
+; GFX10-NEXT:    s_lshr_b32 s8, s6, 16
+; GFX10-NEXT:    s_and_b32 s9, s3, s7
+; GFX10-NEXT:    s_and_b32 s6, s6, s7
 ; GFX10-NEXT:    s_lshr_b32 s3, s3, 16
-; GFX10-NEXT:    s_cmp_lt_u32 s7, s10
+; GFX10-NEXT:    s_cmp_lt_u32 s6, s9
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cselect_b32 s7, s7, s10
-; GFX10-NEXT:    s_cmp_lt_u32 s9, s3
-; GFX10-NEXT:    s_cselect_b32 s3, s9, s3
-; GFX10-NEXT:    s_and_b32 s10, s4, s8
-; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s7, s3
-; GFX10-NEXT:    s_lshr_b32 s7, s0, 16
-; GFX10-NEXT:    s_lshr_b32 s9, s3, 16
+; GFX10-NEXT:    s_cselect_b32 s6, s6, s9
+; GFX10-NEXT:    s_cmp_lt_u32 s8, s3
+; GFX10-NEXT:    s_cselect_b32 s3, s8, s3
+; GFX10-NEXT:    s_and_b32 s9, s4, s7
+; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s6, s3
+; GFX10-NEXT:    s_lshr_b32 s6, s0, 16
+; GFX10-NEXT:    s_lshr_b32 s8, s3, 16
 ; GFX10-NEXT:    s_add_i32 s0, s0, s3
-; GFX10-NEXT:    s_xor_b32 s3, s1, s6
-; GFX10-NEXT:    s_add_i32 s7, s7, s9
-; GFX10-NEXT:    s_lshr_b32 s9, s3, 16
-; GFX10-NEXT:    s_and_b32 s3, s3, s8
+; GFX10-NEXT:    s_xor_b32 s3, s1, -1
+; GFX10-NEXT:    s_add_i32 s6, s6, s8
+; GFX10-NEXT:    s_lshr_b32 s8, s3, 16
+; GFX10-NEXT:    s_and_b32 s3, s3, s7
 ; GFX10-NEXT:    s_lshr_b32 s4, s4, 16
-; GFX10-NEXT:    s_cmp_lt_u32 s3, s10
-; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s7
-; GFX10-NEXT:    s_cselect_b32 s3, s3, s10
-; GFX10-NEXT:    s_cmp_lt_u32 s9, s4
-; GFX10-NEXT:    s_cselect_b32 s4, s9, s4
+; GFX10-NEXT:    s_cmp_lt_u32 s3, s9
+; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s6
+; GFX10-NEXT:    s_cselect_b32 s3, s3, s9
+; GFX10-NEXT:    s_cmp_lt_u32 s8, s4
+; GFX10-NEXT:    s_cselect_b32 s4, s8, s4
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s3, s4
 ; GFX10-NEXT:    s_lshr_b32 s4, s1, 16
-; GFX10-NEXT:    s_lshr_b32 s9, s3, 16
+; GFX10-NEXT:    s_lshr_b32 s8, s3, 16
 ; GFX10-NEXT:    s_add_i32 s1, s1, s3
-; GFX10-NEXT:    s_xor_b32 s3, s2, s6
-; GFX10-NEXT:    s_add_i32 s4, s4, s9
-; GFX10-NEXT:    s_lshr_b32 s6, s3, 16
-; GFX10-NEXT:    s_and_b32 s3, s3, s8
-; GFX10-NEXT:    s_and_b32 s8, s5, s8
+; GFX10-NEXT:    s_xor_b32 s3, s2, -1
+; GFX10-NEXT:    s_add_i32 s4, s4, s8
+; GFX10-NEXT:    s_lshr_b32 s8, s3, 16
+; GFX10-NEXT:    s_and_b32 s3, s3, s7
+; GFX10-NEXT:    s_and_b32 s7, s5, s7
 ; GFX10-NEXT:    s_lshr_b32 s5, s5, 16
-; GFX10-NEXT:    s_cmp_lt_u32 s3, s8
+; GFX10-NEXT:    s_cmp_lt_u32 s3, s7
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s1, s4
-; GFX10-NEXT:    s_cselect_b32 s3, s3, s8
-; GFX10-NEXT:    s_cmp_lt_u32 s6, s5
-; GFX10-NEXT:    s_cselect_b32 s5, s6, s5
+; GFX10-NEXT:    s_cselect_b32 s3, s3, s7
+; GFX10-NEXT:    s_cmp_lt_u32 s8, s5
+; GFX10-NEXT:    s_cselect_b32 s5, s8, s5
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s3, s5
 ; GFX10-NEXT:    s_lshr_b32 s5, s2, 16
 ; GFX10-NEXT:    s_lshr_b32 s6, s3, 16
@@ -3324,17 +3308,16 @@ define <4 x float> @v_uaddsat_v8i16(<8 x i16> %lhs, <8 x i16> %rhs) {
 ; GFX9-LABEL: v_uaddsat_v8i16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, -1, -1
-; GFX9-NEXT:    v_xor_b32_e32 v8, s4, v0
+; GFX9-NEXT:    v_xor_b32_e32 v8, -1, v0
 ; GFX9-NEXT:    v_pk_min_u16 v4, v8, v4
 ; GFX9-NEXT:    v_pk_add_u16 v0, v0, v4
-; GFX9-NEXT:    v_xor_b32_e32 v4, s4, v1
+; GFX9-NEXT:    v_xor_b32_e32 v4, -1, v1
 ; GFX9-NEXT:    v_pk_min_u16 v4, v4, v5
 ; GFX9-NEXT:    v_pk_add_u16 v1, v1, v4
-; GFX9-NEXT:    v_xor_b32_e32 v4, s4, v2
+; GFX9-NEXT:    v_xor_b32_e32 v4, -1, v2
 ; GFX9-NEXT:    v_pk_min_u16 v4, v4, v6
 ; GFX9-NEXT:    v_pk_add_u16 v2, v2, v4
-; GFX9-NEXT:    v_xor_b32_e32 v4, s4, v3
+; GFX9-NEXT:    v_xor_b32_e32 v4, -1, v3
 ; GFX9-NEXT:    v_pk_min_u16 v4, v4, v7
 ; GFX9-NEXT:    v_pk_add_u16 v3, v3, v4
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -3343,12 +3326,11 @@ define <4 x float> @v_uaddsat_v8i16(<8 x i16> %lhs, <8 x i16> %rhs) {
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, -1, -1
+; GFX10-NEXT:    v_xor_b32_e32 v15, -1, v0
+; GFX10-NEXT:    v_xor_b32_e32 v19, -1, v1
+; GFX10-NEXT:    v_xor_b32_e32 v23, -1, v2
+; GFX10-NEXT:    v_xor_b32_e32 v10, -1, v3
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_xor_b32_e32 v15, s4, v0
-; GFX10-NEXT:    v_xor_b32_e32 v19, s4, v1
-; GFX10-NEXT:    v_xor_b32_e32 v23, s4, v2
-; GFX10-NEXT:    v_xor_b32_e32 v10, s4, v3
 ; GFX10-NEXT:    v_pk_min_u16 v11, v15, v4
 ; GFX10-NEXT:    v_pk_min_u16 v15, v19, v5
 ; GFX10-NEXT:    v_pk_min_u16 v19, v23, v6
@@ -3519,58 +3501,57 @@ define amdgpu_ps <4 x i32> @s_uaddsat_v8i16(<8 x i16> inreg %lhs, <8 x i16> inre
 ;
 ; GFX9-LABEL: s_uaddsat_v8i16:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_pack_ll_b32_b16 s8, -1, -1
-; GFX9-NEXT:    s_xor_b32 s9, s0, s8
-; GFX9-NEXT:    s_mov_b32 s11, 0xffff
-; GFX9-NEXT:    s_lshr_b32 s10, s9, 16
-; GFX9-NEXT:    s_lshr_b32 s12, s4, 16
-; GFX9-NEXT:    s_and_b32 s9, s9, s11
-; GFX9-NEXT:    s_and_b32 s4, s4, s11
-; GFX9-NEXT:    s_cmp_lt_u32 s9, s4
-; GFX9-NEXT:    s_cselect_b32 s4, s9, s4
-; GFX9-NEXT:    s_cmp_lt_u32 s10, s12
-; GFX9-NEXT:    s_cselect_b32 s9, s10, s12
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s9
-; GFX9-NEXT:    s_lshr_b32 s9, s0, 16
-; GFX9-NEXT:    s_lshr_b32 s10, s4, 16
-; GFX9-NEXT:    s_add_i32 s0, s0, s4
-; GFX9-NEXT:    s_add_i32 s9, s9, s10
-; GFX9-NEXT:    s_xor_b32 s4, s1, s8
-; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s9
+; GFX9-NEXT:    s_xor_b32 s8, s0, -1
+; GFX9-NEXT:    s_mov_b32 s10, 0xffff
+; GFX9-NEXT:    s_lshr_b32 s9, s8, 16
+; GFX9-NEXT:    s_lshr_b32 s11, s4, 16
+; GFX9-NEXT:    s_and_b32 s8, s8, s10
+; GFX9-NEXT:    s_and_b32 s4, s4, s10
+; GFX9-NEXT:    s_cmp_lt_u32 s8, s4
+; GFX9-NEXT:    s_cselect_b32 s4, s8, s4
+; GFX9-NEXT:    s_cmp_lt_u32 s9, s11
+; GFX9-NEXT:    s_cselect_b32 s8, s9, s11
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s8
+; GFX9-NEXT:    s_lshr_b32 s8, s0, 16
 ; GFX9-NEXT:    s_lshr_b32 s9, s4, 16
-; GFX9-NEXT:    s_lshr_b32 s10, s5, 16
-; GFX9-NEXT:    s_and_b32 s4, s4, s11
-; GFX9-NEXT:    s_and_b32 s5, s5, s11
+; GFX9-NEXT:    s_add_i32 s0, s0, s4
+; GFX9-NEXT:    s_add_i32 s8, s8, s9
+; GFX9-NEXT:    s_xor_b32 s4, s1, -1
+; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s8
+; GFX9-NEXT:    s_lshr_b32 s8, s4, 16
+; GFX9-NEXT:    s_lshr_b32 s9, s5, 16
+; GFX9-NEXT:    s_and_b32 s4, s4, s10
+; GFX9-NEXT:    s_and_b32 s5, s5, s10
 ; GFX9-NEXT:    s_cmp_lt_u32 s4, s5
 ; GFX9-NEXT:    s_cselect_b32 s4, s4, s5
-; GFX9-NEXT:    s_cmp_lt_u32 s9, s10
-; GFX9-NEXT:    s_cselect_b32 s5, s9, s10
+; GFX9-NEXT:    s_cmp_lt_u32 s8, s9
+; GFX9-NEXT:    s_cselect_b32 s5, s8, s9
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
 ; GFX9-NEXT:    s_lshr_b32 s5, s1, 16
-; GFX9-NEXT:    s_lshr_b32 s9, s4, 16
+; GFX9-NEXT:    s_lshr_b32 s8, s4, 16
 ; GFX9-NEXT:    s_add_i32 s1, s1, s4
-; GFX9-NEXT:    s_add_i32 s5, s5, s9
-; GFX9-NEXT:    s_xor_b32 s4, s2, s8
+; GFX9-NEXT:    s_add_i32 s5, s5, s8
+; GFX9-NEXT:    s_xor_b32 s4, s2, -1
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s5
 ; GFX9-NEXT:    s_lshr_b32 s5, s4, 16
-; GFX9-NEXT:    s_lshr_b32 s9, s6, 16
-; GFX9-NEXT:    s_and_b32 s4, s4, s11
-; GFX9-NEXT:    s_and_b32 s6, s6, s11
+; GFX9-NEXT:    s_lshr_b32 s8, s6, 16
+; GFX9-NEXT:    s_and_b32 s4, s4, s10
+; GFX9-NEXT:    s_and_b32 s6, s6, s10
 ; GFX9-NEXT:    s_cmp_lt_u32 s4, s6
 ; GFX9-NEXT:    s_cselect_b32 s4, s4, s6
-; GFX9-NEXT:    s_cmp_lt_u32 s5, s9
-; GFX9-NEXT:    s_cselect_b32 s5, s5, s9
+; GFX9-NEXT:    s_cmp_lt_u32 s5, s8
+; GFX9-NEXT:    s_cselect_b32 s5, s5, s8
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
 ; GFX9-NEXT:    s_lshr_b32 s5, s2, 16
 ; GFX9-NEXT:    s_lshr_b32 s6, s4, 16
 ; GFX9-NEXT:    s_add_i32 s2, s2, s4
 ; GFX9-NEXT:    s_add_i32 s5, s5, s6
-; GFX9-NEXT:    s_xor_b32 s4, s3, s8
+; GFX9-NEXT:    s_xor_b32 s4, s3, -1
 ; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s5
 ; GFX9-NEXT:    s_lshr_b32 s5, s4, 16
 ; GFX9-NEXT:    s_lshr_b32 s6, s7, 16
-; GFX9-NEXT:    s_and_b32 s4, s4, s11
-; GFX9-NEXT:    s_and_b32 s7, s7, s11
+; GFX9-NEXT:    s_and_b32 s4, s4, s10
+; GFX9-NEXT:    s_and_b32 s7, s7, s10
 ; GFX9-NEXT:    s_cmp_lt_u32 s4, s7
 ; GFX9-NEXT:    s_cselect_b32 s4, s4, s7
 ; GFX9-NEXT:    s_cmp_lt_u32 s5, s6
@@ -3585,63 +3566,62 @@ define amdgpu_ps <4 x i32> @s_uaddsat_v8i16(<8 x i16> inreg %lhs, <8 x i16> inre
 ;
 ; GFX10-LABEL: s_uaddsat_v8i16:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_pack_ll_b32_b16 s8, -1, -1
-; GFX10-NEXT:    s_mov_b32 s10, 0xffff
-; GFX10-NEXT:    s_xor_b32 s9, s0, s8
-; GFX10-NEXT:    s_and_b32 s12, s4, s10
-; GFX10-NEXT:    s_lshr_b32 s11, s9, 16
-; GFX10-NEXT:    s_and_b32 s9, s9, s10
+; GFX10-NEXT:    s_xor_b32 s8, s0, -1
+; GFX10-NEXT:    s_mov_b32 s9, 0xffff
+; GFX10-NEXT:    s_lshr_b32 s10, s8, 16
+; GFX10-NEXT:    s_and_b32 s11, s4, s9
+; GFX10-NEXT:    s_and_b32 s8, s8, s9
 ; GFX10-NEXT:    s_lshr_b32 s4, s4, 16
-; GFX10-NEXT:    s_cmp_lt_u32 s9, s12
+; GFX10-NEXT:    s_cmp_lt_u32 s8, s11
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cselect_b32 s9, s9, s12
-; GFX10-NEXT:    s_cmp_lt_u32 s11, s4
-; GFX10-NEXT:    s_cselect_b32 s4, s11, s4
-; GFX10-NEXT:    s_and_b32 s12, s5, s10
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s9, s4
-; GFX10-NEXT:    s_lshr_b32 s9, s0, 16
-; GFX10-NEXT:    s_lshr_b32 s11, s4, 16
+; GFX10-NEXT:    s_cselect_b32 s8, s8, s11
+; GFX10-NEXT:    s_cmp_lt_u32 s10, s4
+; GFX10-NEXT:    s_cselect_b32 s4, s10, s4
+; GFX10-NEXT:    s_and_b32 s11, s5, s9
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s8, s4
+; GFX10-NEXT:    s_lshr_b32 s8, s0, 16
+; GFX10-NEXT:    s_lshr_b32 s10, s4, 16
 ; GFX10-NEXT:    s_add_i32 s0, s0, s4
-; GFX10-NEXT:    s_xor_b32 s4, s1, s8
-; GFX10-NEXT:    s_add_i32 s9, s9, s11
-; GFX10-NEXT:    s_lshr_b32 s11, s4, 16
-; GFX10-NEXT:    s_and_b32 s4, s4, s10
+; GFX10-NEXT:    s_xor_b32 s4, s1, -1
+; GFX10-NEXT:    s_add_i32 s8, s8, s10
+; GFX10-NEXT:    s_lshr_b32 s10, s4, 16
+; GFX10-NEXT:    s_and_b32 s4, s4, s9
 ; GFX10-NEXT:    s_lshr_b32 s5, s5, 16
-; GFX10-NEXT:    s_cmp_lt_u32 s4, s12
-; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s9
-; GFX10-NEXT:    s_cselect_b32 s4, s4, s12
-; GFX10-NEXT:    s_cmp_lt_u32 s11, s5
-; GFX10-NEXT:    s_cselect_b32 s5, s11, s5
-; GFX10-NEXT:    s_and_b32 s12, s6, s10
+; GFX10-NEXT:    s_cmp_lt_u32 s4, s11
+; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s8
+; GFX10-NEXT:    s_cselect_b32 s4, s4, s11
+; GFX10-NEXT:    s_cmp_lt_u32 s10, s5
+; GFX10-NEXT:    s_cselect_b32 s5, s10, s5
+; GFX10-NEXT:    s_and_b32 s11, s6, s9
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
 ; GFX10-NEXT:    s_lshr_b32 s5, s1, 16
-; GFX10-NEXT:    s_lshr_b32 s11, s4, 16
+; GFX10-NEXT:    s_lshr_b32 s10, s4, 16
 ; GFX10-NEXT:    s_add_i32 s1, s1, s4
-; GFX10-NEXT:    s_xor_b32 s4, s2, s8
-; GFX10-NEXT:    s_add_i32 s5, s5, s11
-; GFX10-NEXT:    s_lshr_b32 s11, s4, 16
-; GFX10-NEXT:    s_and_b32 s4, s4, s10
+; GFX10-NEXT:    s_xor_b32 s4, s2, -1
+; GFX10-NEXT:    s_add_i32 s5, s5, s10
+; GFX10-NEXT:    s_lshr_b32 s10, s4, 16
+; GFX10-NEXT:    s_and_b32 s4, s4, s9
 ; GFX10-NEXT:    s_lshr_b32 s6, s6, 16
-; GFX10-NEXT:    s_cmp_lt_u32 s4, s12
+; GFX10-NEXT:    s_cmp_lt_u32 s4, s11
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s1, s5
-; GFX10-NEXT:    s_cselect_b32 s4, s4, s12
-; GFX10-NEXT:    s_cmp_lt_u32 s11, s6
-; GFX10-NEXT:    s_cselect_b32 s6, s11, s6
+; GFX10-NEXT:    s_cselect_b32 s4, s4, s11
+; GFX10-NEXT:    s_cmp_lt_u32 s10, s6
+; GFX10-NEXT:    s_cselect_b32 s6, s10, s6
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s4, s6
 ; GFX10-NEXT:    s_lshr_b32 s6, s2, 16
-; GFX10-NEXT:    s_lshr_b32 s11, s4, 16
+; GFX10-NEXT:    s_lshr_b32 s10, s4, 16
 ; GFX10-NEXT:    s_add_i32 s2, s2, s4
-; GFX10-NEXT:    s_xor_b32 s4, s3, s8
-; GFX10-NEXT:    s_add_i32 s6, s6, s11
-; GFX10-NEXT:    s_lshr_b32 s8, s4, 16
-; GFX10-NEXT:    s_and_b32 s4, s4, s10
-; GFX10-NEXT:    s_and_b32 s10, s7, s10
+; GFX10-NEXT:    s_xor_b32 s4, s3, -1
+; GFX10-NEXT:    s_add_i32 s6, s6, s10
+; GFX10-NEXT:    s_lshr_b32 s10, s4, 16
+; GFX10-NEXT:    s_and_b32 s4, s4, s9
+; GFX10-NEXT:    s_and_b32 s9, s7, s9
 ; GFX10-NEXT:    s_lshr_b32 s7, s7, 16
-; GFX10-NEXT:    s_cmp_lt_u32 s4, s10
+; GFX10-NEXT:    s_cmp_lt_u32 s4, s9
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s2, s6
-; GFX10-NEXT:    s_cselect_b32 s4, s4, s10
-; GFX10-NEXT:    s_cmp_lt_u32 s8, s7
-; GFX10-NEXT:    s_cselect_b32 s7, s8, s7
+; GFX10-NEXT:    s_cselect_b32 s4, s4, s9
+; GFX10-NEXT:    s_cmp_lt_u32 s10, s7
+; GFX10-NEXT:    s_cselect_b32 s7, s10, s7
 ; GFX10-NEXT:    s_lshr_b32 s5, s3, 16
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s4, s7
 ; GFX10-NEXT:    s_lshr_b32 s7, s4, 16
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/xnor.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/xnor.ll
index 2512aaaeb082c..474f6655bda2c 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/xnor.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/xnor.ll
@@ -50,16 +50,14 @@ define amdgpu_ps i32 @scalar_xnor_v2i16_one_use(<2 x i16> inreg %a, <2 x i16> in
 ;
 ; GFX900-LABEL: scalar_xnor_v2i16_one_use:
 ; GFX900:       ; %bb.0: ; %entry
-; GFX900-NEXT:    s_pack_ll_b32_b16 s2, -1, -1
 ; GFX900-NEXT:    s_xor_b32 s0, s0, s1
-; GFX900-NEXT:    s_xor_b32 s0, s0, s2
+; GFX900-NEXT:    s_xor_b32 s0, s0, -1
 ; GFX900-NEXT:    ; return to shader part epilog
 ;
 ; GFX906-LABEL: scalar_xnor_v2i16_one_use:
 ; GFX906:       ; %bb.0: ; %entry
-; GFX906-NEXT:    s_pack_ll_b32_b16 s2, -1, -1
 ; GFX906-NEXT:    s_xor_b32 s0, s0, s1
-; GFX906-NEXT:    s_xor_b32 s0, s0, s2
+; GFX906-NEXT:    s_xor_b32 s0, s0, -1
 ; GFX906-NEXT:    ; return to shader part epilog
 entry:
   %xor = xor <2 x i16> %a, %b
@@ -150,7 +148,7 @@ define amdgpu_ps i64 @scalar_xnor_v4i16_one_use(<4 x i16> inreg %a, <4 x i16> in
 ;
 ; GFX900-LABEL: scalar_xnor_v4i16_one_use:
 ; GFX900:       ; %bb.0:
-; GFX900-NEXT:    s_pack_ll_b32_b16 s4, -1, -1
+; GFX900-NEXT:    s_mov_b32 s4, -1
 ; GFX900-NEXT:    s_mov_b32 s5, s4
 ; GFX900-NEXT:    s_xor_b64 s[0:1], s[0:1], s[2:3]
 ; GFX900-NEXT:    s_xor_b64 s[0:1], s[0:1], s[4:5]
@@ -158,7 +156,7 @@ define amdgpu_ps i64 @scalar_xnor_v4i16_one_use(<4 x i16> inreg %a, <4 x i16> in
 ;
 ; GFX906-LABEL: scalar_xnor_v4i16_one_use:
 ; GFX906:       ; %bb.0:
-; GFX906-NEXT:    s_pack_ll_b32_b16 s4, -1, -1
+; GFX906-NEXT:    s_mov_b32 s4, -1
 ; GFX906-NEXT:    s_mov_b32 s5, s4
 ; GFX906-NEXT:    s_xor_b64 s[0:1], s[0:1], s[2:3]
 ; GFX906-NEXT:    s_xor_b64 s[0:1], s[0:1], s[4:5]

From 6f961a1e7ec2373dceda74a1a49f93b009db1b72 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Sat, 25 Jul 2020 15:41:58 -0400
Subject: [PATCH 0133/1035] AMDGPU/GlobalISel: Legalize GDS atomics

I noticed these don't use the _gfx9, non-m0 reading variants but not
sure if that's a bug or not. It's the same in the DAG.
---
 .../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp |   5 +-
 .../inst-select-atomic-cmpxchg-region.mir     | 173 ++++++++++++++++++
 .../inst-select-atomicrmw-fadd-region.mir     | 116 ++++++++++++
 .../inst-select-atomicrmw-xchg-region.mir     |  83 +++++++++
 4 files changed, 375 insertions(+), 2 deletions(-)
 create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-atomic-cmpxchg-region.mir
 create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-atomicrmw-fadd-region.mir
 create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-atomicrmw-xchg-region.mir

diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 80633bbc6179d..f72db8a61aab8 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -1196,14 +1196,15 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
      G_ATOMICRMW_MAX, G_ATOMICRMW_MIN, G_ATOMICRMW_UMAX,
      G_ATOMICRMW_UMIN})
     .legalFor({{S32, GlobalPtr}, {S32, LocalPtr},
-               {S64, GlobalPtr}, {S64, LocalPtr}});
+               {S64, GlobalPtr}, {S64, LocalPtr},
+               {S32, RegionPtr}, {S64, RegionPtr}});
   if (ST.hasFlatAddressSpace()) {
     Atomics.legalFor({{S32, FlatPtr}, {S64, FlatPtr}});
   }
 
   if (ST.hasLDSFPAtomics()) {
     getActionDefinitionsBuilder(G_ATOMICRMW_FADD)
-      .legalFor({{S32, LocalPtr}});
+      .legalFor({{S32, LocalPtr}, {S32, RegionPtr}});
   }
 
   // BUFFER/FLAT_ATOMIC_CMP_SWAP on GCN GPUs needs input marshalling, and output
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-atomic-cmpxchg-region.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-atomic-cmpxchg-region.mir
new file mode 100644
index 0000000000000..834380d617912
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-atomic-cmpxchg-region.mir
@@ -0,0 +1,173 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=tahiti -run-pass=instruction-select -verify-machineinstrs  -global-isel-abort=0 -o - %s | FileCheck -check-prefix=GFX6 %s
+# RUN: llc -march=amdgcn -mcpu=hawaii -run-pass=instruction-select -verify-machineinstrs  -global-isel-abort=0 -o - %s | FileCheck -check-prefix=GFX7 %s
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=instruction-select -verify-machineinstrs  -global-isel-abort=0 -o - %s | FileCheck -check-prefix=GFX7 %s
+# RUN: llc -march=amdgcn -mcpu=gfx900 -run-pass=instruction-select -verify-machineinstrs -global-isel-abort=0 -o - %s | FileCheck -check-prefix=GFX9 %s
+# RUN: llc -march=amdgcn -mcpu=gfx1010 -run-pass=instruction-select -verify-machineinstrs -global-isel-abort=0 -o - %s | FileCheck -check-prefix=GFX9 %s
+
+
+---
+name:            atomic_cmpxchg_s32_region
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2
+
+    ; GFX6-LABEL: name: atomic_cmpxchg_s32_region
+    ; GFX6: liveins: $vgpr0, $vgpr1, $vgpr2
+    ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX6: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX6: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+    ; GFX6: $m0 = S_MOV_B32 -1
+    ; GFX6: [[DS_CMPST_RTN_B32_:%[0-9]+]]:vgpr_32 = DS_CMPST_RTN_B32 [[COPY]], [[COPY1]], [[COPY2]], 0, 1, implicit $m0, implicit $exec :: (load store seq_cst 4, addrspace 2)
+    ; GFX6: $vgpr0 = COPY [[DS_CMPST_RTN_B32_]]
+    ; GFX7-LABEL: name: atomic_cmpxchg_s32_region
+    ; GFX7: liveins: $vgpr0, $vgpr1, $vgpr2
+    ; GFX7: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX7: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX7: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+    ; GFX7: $m0 = S_MOV_B32 -1
+    ; GFX7: [[DS_CMPST_RTN_B32_:%[0-9]+]]:vgpr_32 = DS_CMPST_RTN_B32 [[COPY]], [[COPY1]], [[COPY2]], 0, 1, implicit $m0, implicit $exec :: (load store seq_cst 4, addrspace 2)
+    ; GFX7: $vgpr0 = COPY [[DS_CMPST_RTN_B32_]]
+    ; GFX9-LABEL: name: atomic_cmpxchg_s32_region
+    ; GFX9: liveins: $vgpr0, $vgpr1, $vgpr2
+    ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX9: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+    ; GFX9: [[DS_CMPST_RTN_B32_:%[0-9]+]]:vgpr_32 = DS_CMPST_RTN_B32 [[COPY]], [[COPY1]], [[COPY2]], 0, 1, implicit $m0, implicit $exec :: (load store seq_cst 4, addrspace 2)
+    ; GFX9: $vgpr0 = COPY [[DS_CMPST_RTN_B32_]]
+    %0:vgpr(p2) = COPY $vgpr0
+    %1:vgpr(s32) = COPY $vgpr1
+    %2:vgpr(s32) = COPY $vgpr2
+    %3:vgpr(s32) = G_ATOMIC_CMPXCHG %0, %1, %2 :: (load store seq_cst 4, addrspace 2)
+    $vgpr0 = COPY %3
+
+...
+
+---
+name:            atomic_cmpxchg_s32_region_gep4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2
+
+    ; GFX6-LABEL: name: atomic_cmpxchg_s32_region_gep4
+    ; GFX6: liveins: $vgpr0, $vgpr1, $vgpr2
+    ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX6: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX6: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+    ; GFX6: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4, implicit $exec
+    ; GFX6: %4:vgpr_32, dead %6:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
+    ; GFX6: $m0 = S_MOV_B32 -1
+    ; GFX6: [[DS_CMPST_RTN_B32_:%[0-9]+]]:vgpr_32 = DS_CMPST_RTN_B32 %4, [[COPY1]], [[COPY2]], 0, 1, implicit $m0, implicit $exec :: (load store seq_cst 4, addrspace 2)
+    ; GFX6: $vgpr0 = COPY [[DS_CMPST_RTN_B32_]]
+    ; GFX7-LABEL: name: atomic_cmpxchg_s32_region_gep4
+    ; GFX7: liveins: $vgpr0, $vgpr1, $vgpr2
+    ; GFX7: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX7: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX7: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+    ; GFX7: $m0 = S_MOV_B32 -1
+    ; GFX7: [[DS_CMPST_RTN_B32_:%[0-9]+]]:vgpr_32 = DS_CMPST_RTN_B32 [[COPY]], [[COPY1]], [[COPY2]], 4, 1, implicit $m0, implicit $exec :: (load store seq_cst 4, addrspace 2)
+    ; GFX7: $vgpr0 = COPY [[DS_CMPST_RTN_B32_]]
+    ; GFX9-LABEL: name: atomic_cmpxchg_s32_region_gep4
+    ; GFX9: liveins: $vgpr0, $vgpr1, $vgpr2
+    ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX9: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+    ; GFX9: [[DS_CMPST_RTN_B32_:%[0-9]+]]:vgpr_32 = DS_CMPST_RTN_B32 [[COPY]], [[COPY1]], [[COPY2]], 4, 1, implicit $m0, implicit $exec :: (load store seq_cst 4, addrspace 2)
+    ; GFX9: $vgpr0 = COPY [[DS_CMPST_RTN_B32_]]
+    %0:vgpr(p2) = COPY $vgpr0
+    %1:vgpr(s32) = COPY $vgpr1
+    %2:vgpr(s32) = COPY $vgpr2
+    %3:vgpr(s32) = G_CONSTANT i32 4
+    %4:vgpr(p2) = G_PTR_ADD %0, %3
+    %5:vgpr(s32) = G_ATOMIC_CMPXCHG %4, %1, %2 :: (load store seq_cst 4, addrspace 2)
+    $vgpr0 = COPY %5
+
+...
+
+---
+name:            atomic_cmpxchg_s64_region
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1_vgpr2, $vgpr3_vgpr4
+
+    ; GFX6-LABEL: name: atomic_cmpxchg_s64_region
+    ; GFX6: liveins: $vgpr0, $vgpr1_vgpr2, $vgpr3_vgpr4
+    ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX6: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr1_vgpr2
+    ; GFX6: [[COPY2:%[0-9]+]]:vreg_64 = COPY $vgpr3_vgpr4
+    ; GFX6: $m0 = S_MOV_B32 -1
+    ; GFX6: [[DS_CMPST_RTN_B64_:%[0-9]+]]:vreg_64 = DS_CMPST_RTN_B64 [[COPY]], [[COPY1]], [[COPY2]], 0, 1, implicit $m0, implicit $exec :: (load store seq_cst 8, addrspace 2)
+    ; GFX6: $vgpr0_vgpr1 = COPY [[DS_CMPST_RTN_B64_]]
+    ; GFX7-LABEL: name: atomic_cmpxchg_s64_region
+    ; GFX7: liveins: $vgpr0, $vgpr1_vgpr2, $vgpr3_vgpr4
+    ; GFX7: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX7: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr1_vgpr2
+    ; GFX7: [[COPY2:%[0-9]+]]:vreg_64 = COPY $vgpr3_vgpr4
+    ; GFX7: $m0 = S_MOV_B32 -1
+    ; GFX7: [[DS_CMPST_RTN_B64_:%[0-9]+]]:vreg_64 = DS_CMPST_RTN_B64 [[COPY]], [[COPY1]], [[COPY2]], 0, 1, implicit $m0, implicit $exec :: (load store seq_cst 8, addrspace 2)
+    ; GFX7: $vgpr0_vgpr1 = COPY [[DS_CMPST_RTN_B64_]]
+    ; GFX9-LABEL: name: atomic_cmpxchg_s64_region
+    ; GFX9: liveins: $vgpr0, $vgpr1_vgpr2, $vgpr3_vgpr4
+    ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr1_vgpr2
+    ; GFX9: [[COPY2:%[0-9]+]]:vreg_64 = COPY $vgpr3_vgpr4
+    ; GFX9: [[DS_CMPST_RTN_B64_:%[0-9]+]]:vreg_64 = DS_CMPST_RTN_B64 [[COPY]], [[COPY1]], [[COPY2]], 0, 1, implicit $m0, implicit $exec :: (load store seq_cst 8, addrspace 2)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[DS_CMPST_RTN_B64_]]
+    %0:vgpr(p2) = COPY $vgpr0
+    %1:vgpr(s64) = COPY $vgpr1_vgpr2
+    %2:vgpr(s64) = COPY $vgpr3_vgpr4
+    %3:vgpr(s64) = G_ATOMIC_CMPXCHG %0, %1, %2 :: (load store seq_cst 8, addrspace 2)
+    $vgpr0_vgpr1 = COPY %3
+
+...
+
+---
+name:            atomic_cmpxchg_s64_region_gep4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1_vgpr2, $vgpr3_vgpr4
+
+    ; GFX6-LABEL: name: atomic_cmpxchg_s64_region_gep4
+    ; GFX6: liveins: $vgpr0, $vgpr1_vgpr2, $vgpr3_vgpr4
+    ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX6: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr1_vgpr2
+    ; GFX6: [[COPY2:%[0-9]+]]:vreg_64 = COPY $vgpr3_vgpr4
+    ; GFX6: $m0 = S_MOV_B32 -1
+    ; GFX6: [[DS_CMPST_RTN_B64_:%[0-9]+]]:vreg_64 = DS_CMPST_RTN_B64 [[COPY]], [[COPY1]], [[COPY2]], 0, 1, implicit $m0, implicit $exec :: (load store seq_cst 8, addrspace 2)
+    ; GFX6: $vgpr0_vgpr1 = COPY [[DS_CMPST_RTN_B64_]]
+    ; GFX7-LABEL: name: atomic_cmpxchg_s64_region_gep4
+    ; GFX7: liveins: $vgpr0, $vgpr1_vgpr2, $vgpr3_vgpr4
+    ; GFX7: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX7: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr1_vgpr2
+    ; GFX7: [[COPY2:%[0-9]+]]:vreg_64 = COPY $vgpr3_vgpr4
+    ; GFX7: $m0 = S_MOV_B32 -1
+    ; GFX7: [[DS_CMPST_RTN_B64_:%[0-9]+]]:vreg_64 = DS_CMPST_RTN_B64 [[COPY]], [[COPY1]], [[COPY2]], 0, 1, implicit $m0, implicit $exec :: (load store seq_cst 8, addrspace 2)
+    ; GFX7: $vgpr0_vgpr1 = COPY [[DS_CMPST_RTN_B64_]]
+    ; GFX9-LABEL: name: atomic_cmpxchg_s64_region_gep4
+    ; GFX9: liveins: $vgpr0, $vgpr1_vgpr2, $vgpr3_vgpr4
+    ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr1_vgpr2
+    ; GFX9: [[COPY2:%[0-9]+]]:vreg_64 = COPY $vgpr3_vgpr4
+    ; GFX9: [[DS_CMPST_RTN_B64_:%[0-9]+]]:vreg_64 = DS_CMPST_RTN_B64 [[COPY]], [[COPY1]], [[COPY2]], 0, 1, implicit $m0, implicit $exec :: (load store seq_cst 8, addrspace 2)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[DS_CMPST_RTN_B64_]]
+    %0:vgpr(p2) = COPY $vgpr0
+    %1:vgpr(s64) = COPY $vgpr1_vgpr2
+    %2:vgpr(s64) = COPY $vgpr3_vgpr4
+    %3:vgpr(s32) = G_CONSTANT i32 4
+    %4:vgpr(p2) = G_PTR_ADD %0, %3
+    %5:vgpr(s64) = G_ATOMIC_CMPXCHG %0, %1, %2 :: (load store seq_cst 8, addrspace 2)
+    $vgpr0_vgpr1 = COPY %5
+
+...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-atomicrmw-fadd-region.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-atomicrmw-fadd-region.mir
new file mode 100644
index 0000000000000..26619a1753eb5
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-atomicrmw-fadd-region.mir
@@ -0,0 +1,116 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=instruction-select -verify-machineinstrs -global-isel-abort=0 -o - %s | FileCheck -check-prefix=GFX8 %s
+# RUN: llc -march=amdgcn -mcpu=gfx900 -run-pass=instruction-select -verify-machineinstrs -global-isel-abort=0 -o - %s | FileCheck -check-prefix=GFX9 %s
+# RUN: llc -march=amdgcn -mcpu=gfx1010 -run-pass=instruction-select -verify-machineinstrs -global-isel-abort=0 -o - %s | FileCheck -check-prefix=GFX9 %s
+
+# GFX6/7 selection should fail.
+# RUN: llc -march=amdgcn -mcpu=tahiti -run-pass=instruction-select -verify-machineinstrs -global-isel-abort=0 -disable-gisel-legality-check -o - %s | FileCheck -check-prefix=GFX6 %s
+# RUN: llc -march=amdgcn -mcpu=hawaii -run-pass=instruction-select -verify-machineinstrs -global-isel-abort=0 -disable-gisel-legality-check -o - %s | FileCheck -check-prefix=GFX6 %s
+
+---
+name:            atomicrmw_fadd_s32_region
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; GFX8-LABEL: name: atomicrmw_fadd_s32_region
+    ; GFX8: liveins: $vgpr0, $vgpr1
+    ; GFX8: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX8: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX8: $m0 = S_MOV_B32 -1
+    ; GFX8: [[DS_ADD_RTN_F32_:%[0-9]+]]:vgpr_32 = DS_ADD_RTN_F32 [[COPY]], [[COPY1]], 0, 1, implicit $m0, implicit $exec :: (load store seq_cst 4, addrspace 2)
+    ; GFX8: $vgpr0 = COPY [[DS_ADD_RTN_F32_]]
+    ; GFX9-LABEL: name: atomicrmw_fadd_s32_region
+    ; GFX9: liveins: $vgpr0, $vgpr1
+    ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX9: [[DS_ADD_RTN_F32_:%[0-9]+]]:vgpr_32 = DS_ADD_RTN_F32 [[COPY]], [[COPY1]], 0, 1, implicit $m0, implicit $exec :: (load store seq_cst 4, addrspace 2)
+    ; GFX9: $vgpr0 = COPY [[DS_ADD_RTN_F32_]]
+    ; GFX6-LABEL: name: atomicrmw_fadd_s32_region
+    ; GFX6: liveins: $vgpr0, $vgpr1
+    ; GFX6: [[COPY:%[0-9]+]]:vgpr(p2) = COPY $vgpr0
+    ; GFX6: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; GFX6: $m0 = S_MOV_B32 -1
+    ; GFX6: [[ATOMICRMW_FADD:%[0-9]+]]:vgpr_32(s32) = G_ATOMICRMW_FADD [[COPY]](p2), [[COPY1]] :: (load store seq_cst 4, addrspace 2)
+    ; GFX6: $vgpr0 = COPY [[ATOMICRMW_FADD]](s32)
+    %0:vgpr(p2) = COPY $vgpr0
+    %1:vgpr(s32) = COPY $vgpr1
+    %2:vgpr(s32) = G_ATOMICRMW_FADD %0(p2), %1 :: (load store seq_cst 4, addrspace 2)
+    $vgpr0 = COPY %2
+
+...
+
+---
+name:            atomicrmw_fadd_s32_region_noret
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; GFX8-LABEL: name: atomicrmw_fadd_s32_region_noret
+    ; GFX8: liveins: $vgpr0, $vgpr1
+    ; GFX8: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX8: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX8: $m0 = S_MOV_B32 -1
+    ; GFX8: [[DS_ADD_RTN_F32_:%[0-9]+]]:vgpr_32 = DS_ADD_RTN_F32 [[COPY]], [[COPY1]], 0, 1, implicit $m0, implicit $exec :: (load store seq_cst 4, addrspace 2)
+    ; GFX9-LABEL: name: atomicrmw_fadd_s32_region_noret
+    ; GFX9: liveins: $vgpr0, $vgpr1
+    ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX9: [[DS_ADD_RTN_F32_:%[0-9]+]]:vgpr_32 = DS_ADD_RTN_F32 [[COPY]], [[COPY1]], 0, 1, implicit $m0, implicit $exec :: (load store seq_cst 4, addrspace 2)
+    ; GFX6-LABEL: name: atomicrmw_fadd_s32_region_noret
+    ; GFX6: liveins: $vgpr0, $vgpr1
+    ; GFX6: [[COPY:%[0-9]+]]:vgpr(p2) = COPY $vgpr0
+    ; GFX6: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; GFX6: $m0 = S_MOV_B32 -1
+    ; GFX6: [[ATOMICRMW_FADD:%[0-9]+]]:vgpr(s32) = G_ATOMICRMW_FADD [[COPY]](p2), [[COPY1]] :: (load store seq_cst 4, addrspace 2)
+    %0:vgpr(p2) = COPY $vgpr0
+    %1:vgpr(s32) = COPY $vgpr1
+    %2:vgpr(s32) = G_ATOMICRMW_FADD %0(p2), %1 :: (load store seq_cst 4, addrspace 2)
+
+...
+
+---
+name:            atomicrmw_fadd_s32_region_gep4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; GFX8-LABEL: name: atomicrmw_fadd_s32_region_gep4
+    ; GFX8: liveins: $vgpr0, $vgpr1
+    ; GFX8: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX8: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX8: $m0 = S_MOV_B32 -1
+    ; GFX8: [[DS_ADD_RTN_F32_:%[0-9]+]]:vgpr_32 = DS_ADD_RTN_F32 [[COPY]], [[COPY1]], 4, 1, implicit $m0, implicit $exec :: (load store seq_cst 4, addrspace 2)
+    ; GFX8: $vgpr0 = COPY [[DS_ADD_RTN_F32_]]
+    ; GFX9-LABEL: name: atomicrmw_fadd_s32_region_gep4
+    ; GFX9: liveins: $vgpr0, $vgpr1
+    ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX9: [[DS_ADD_RTN_F32_:%[0-9]+]]:vgpr_32 = DS_ADD_RTN_F32 [[COPY]], [[COPY1]], 4, 1, implicit $m0, implicit $exec :: (load store seq_cst 4, addrspace 2)
+    ; GFX9: $vgpr0 = COPY [[DS_ADD_RTN_F32_]]
+    ; GFX6-LABEL: name: atomicrmw_fadd_s32_region_gep4
+    ; GFX6: liveins: $vgpr0, $vgpr1
+    ; GFX6: [[COPY:%[0-9]+]]:vgpr(p2) = COPY $vgpr0
+    ; GFX6: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; GFX6: [[C:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 4
+    ; GFX6: [[PTR_ADD:%[0-9]+]]:vgpr(p2) = G_PTR_ADD [[COPY]], [[C]](s32)
+    ; GFX6: $m0 = S_MOV_B32 -1
+    ; GFX6: [[ATOMICRMW_FADD:%[0-9]+]]:vgpr_32(s32) = G_ATOMICRMW_FADD [[PTR_ADD]](p2), [[COPY1]] :: (load store seq_cst 4, addrspace 2)
+    ; GFX6: $vgpr0 = COPY [[ATOMICRMW_FADD]](s32)
+    %0:vgpr(p2) = COPY $vgpr0
+    %1:vgpr(s32) = COPY $vgpr1
+    %2:vgpr(s32) = G_CONSTANT i32 4
+    %3:vgpr(p2) = G_PTR_ADD %0, %2
+    %4:vgpr(s32) = G_ATOMICRMW_FADD %3(p2), %1 :: (load store seq_cst 4, addrspace 2)
+    $vgpr0 = COPY %4
+
+...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-atomicrmw-xchg-region.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-atomicrmw-xchg-region.mir
new file mode 100644
index 0000000000000..0552551f35105
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-atomicrmw-xchg-region.mir
@@ -0,0 +1,83 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=tahiti -run-pass=instruction-select -verify-machineinstrs  -global-isel-abort=0 -o - %s | FileCheck -check-prefix=GFX6 %s
+# RUN: llc -march=amdgcn -mcpu=hawaii -run-pass=instruction-select -verify-machineinstrs  -global-isel-abort=0 -o - %s | FileCheck -check-prefix=GFX7 %s
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=instruction-select -verify-machineinstrs  -global-isel-abort=0 -o - %s | FileCheck -check-prefix=GFX7 %s
+# RUN: llc -march=amdgcn -mcpu=gfx900 -run-pass=instruction-select -verify-machineinstrs -global-isel-abort=0 -o - %s | FileCheck -check-prefix=GFX9 %s
+# RUN: llc -march=amdgcn -mcpu=gfx1010 -run-pass=instruction-select -verify-machineinstrs -global-isel-abort=0 -o - %s | FileCheck -check-prefix=GFX9 %s
+
+
+---
+name:            atomicrmw_xchg_s32_region
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; GFX6-LABEL: name: atomicrmw_xchg_s32_region
+    ; GFX6: liveins: $vgpr0, $vgpr1
+    ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX6: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX6: $m0 = S_MOV_B32 -1
+    ; GFX6: [[DS_WRXCHG_RTN_B32_:%[0-9]+]]:vgpr_32 = DS_WRXCHG_RTN_B32 [[COPY]], [[COPY1]], 0, 1, implicit $m0, implicit $exec :: (load store seq_cst 4, addrspace 2)
+    ; GFX6: $vgpr0 = COPY [[DS_WRXCHG_RTN_B32_]]
+    ; GFX7-LABEL: name: atomicrmw_xchg_s32_region
+    ; GFX7: liveins: $vgpr0, $vgpr1
+    ; GFX7: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX7: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX7: $m0 = S_MOV_B32 -1
+    ; GFX7: [[DS_WRXCHG_RTN_B32_:%[0-9]+]]:vgpr_32 = DS_WRXCHG_RTN_B32 [[COPY]], [[COPY1]], 0, 1, implicit $m0, implicit $exec :: (load store seq_cst 4, addrspace 2)
+    ; GFX7: $vgpr0 = COPY [[DS_WRXCHG_RTN_B32_]]
+    ; GFX9-LABEL: name: atomicrmw_xchg_s32_region
+    ; GFX9: liveins: $vgpr0, $vgpr1
+    ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX9: [[DS_WRXCHG_RTN_B32_:%[0-9]+]]:vgpr_32 = DS_WRXCHG_RTN_B32 [[COPY]], [[COPY1]], 0, 1, implicit $m0, implicit $exec :: (load store seq_cst 4, addrspace 2)
+    ; GFX9: $vgpr0 = COPY [[DS_WRXCHG_RTN_B32_]]
+    %0:vgpr(p2) = COPY $vgpr0
+    %1:vgpr(s32) = COPY $vgpr1
+    %2:vgpr(s32) = G_ATOMICRMW_XCHG %0(p2), %1 :: (load store seq_cst 4, addrspace 2)
+    $vgpr0 = COPY %2
+
+...
+
+---
+name:            atomicrmw_xchg_s32_region_gep4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; GFX6-LABEL: name: atomicrmw_xchg_s32_region_gep4
+    ; GFX6: liveins: $vgpr0, $vgpr1
+    ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX6: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX6: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4, implicit $exec
+    ; GFX6: %3:vgpr_32, dead %5:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
+    ; GFX6: $m0 = S_MOV_B32 -1
+    ; GFX6: [[DS_WRXCHG_RTN_B32_:%[0-9]+]]:vgpr_32 = DS_WRXCHG_RTN_B32 %3, [[COPY1]], 0, 1, implicit $m0, implicit $exec :: (load store seq_cst 4, addrspace 2)
+    ; GFX6: $vgpr0 = COPY [[DS_WRXCHG_RTN_B32_]]
+    ; GFX7-LABEL: name: atomicrmw_xchg_s32_region_gep4
+    ; GFX7: liveins: $vgpr0, $vgpr1
+    ; GFX7: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX7: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX7: $m0 = S_MOV_B32 -1
+    ; GFX7: [[DS_WRXCHG_RTN_B32_:%[0-9]+]]:vgpr_32 = DS_WRXCHG_RTN_B32 [[COPY]], [[COPY1]], 4, 1, implicit $m0, implicit $exec :: (load store seq_cst 4, addrspace 2)
+    ; GFX7: $vgpr0 = COPY [[DS_WRXCHG_RTN_B32_]]
+    ; GFX9-LABEL: name: atomicrmw_xchg_s32_region_gep4
+    ; GFX9: liveins: $vgpr0, $vgpr1
+    ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX9: [[DS_WRXCHG_RTN_B32_:%[0-9]+]]:vgpr_32 = DS_WRXCHG_RTN_B32 [[COPY]], [[COPY1]], 4, 1, implicit $m0, implicit $exec :: (load store seq_cst 4, addrspace 2)
+    ; GFX9: $vgpr0 = COPY [[DS_WRXCHG_RTN_B32_]]
+    %0:vgpr(p2) = COPY $vgpr0
+    %1:vgpr(s32) = COPY $vgpr1
+    %2:vgpr(s32) = G_CONSTANT i32 4
+    %3:vgpr(p2) = G_PTR_ADD %0, %2
+    %4:vgpr(s32) = G_ATOMICRMW_XCHG %3(p2), %1 :: (load store seq_cst 4, addrspace 2)
+    $vgpr0 = COPY %4
+
+...

From bcf5184a68d1d851895692bae6eed16a74b519db Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Sat, 25 Jul 2020 17:22:22 -0400
Subject: [PATCH 0134/1035] AMDGPU/GlobalISel: Make sure <2 x s1> phis are
 scalarized

---
 .../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp |  3 +-
 .../AMDGPU/GlobalISel/legalize-phi.mir        | 91 +++++++++++++++++++
 2 files changed, 93 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index f72db8a61aab8..c21414d59ba0c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -415,11 +415,12 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
     .legalFor(AllS64Vectors)
     .legalFor(AddrSpaces64)
     .legalFor(AddrSpaces32)
+    .legalIf(isPointer(0))
     .clampScalar(0, S32, S256)
     .widenScalarToNextPow2(0, 32)
     .clampMaxNumElements(0, S32, 16)
     .moreElementsIf(isSmallOddVector(0), oneMoreElement(0))
-    .legalIf(isPointer(0));
+    .scalarize(0);
 
   if (ST.hasVOP3PInsts()) {
     assert(ST.hasIntClamp() && "all targets with VOP3P should support clamp");
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-phi.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-phi.mir
index 81408b79b11f7..9a91d908bb7b0 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-phi.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-phi.mir
@@ -1547,3 +1547,94 @@ body: |
     S_SETPC_B64 undef $sgpr30_sgpr31
 
 ...
+---
+name: test_phi_v2s1
+tracksRegLiveness: true
+
+body: |
+  ; CHECK-LABEL: name: test_phi_v2s1
+  ; CHECK: bb.0:
+  ; CHECK:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; CHECK:   [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; CHECK:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
+  ; CHECK:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; CHECK:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
+  ; CHECK:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
+  ; CHECK:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+  ; CHECK:   [[COPY4:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
+  ; CHECK:   [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C2]]
+  ; CHECK:   [[COPY5:%[0-9]+]]:_(s32) = COPY [[BITCAST1]](s32)
+  ; CHECK:   [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY5]], [[C2]]
+  ; CHECK:   [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[AND]](s32), [[AND1]]
+  ; CHECK:   [[COPY6:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; CHECK:   [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY6]], [[C2]]
+  ; CHECK:   [[COPY7:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+  ; CHECK:   [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY7]], [[C2]]
+  ; CHECK:   [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[AND2]](s32), [[AND3]]
+  ; CHECK:   [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[COPY3]](s32), [[C]]
+  ; CHECK:   G_BRCOND [[ICMP2]](s1), %bb.1
+  ; CHECK:   G_BR %bb.2
+  ; CHECK: bb.1:
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
+  ; CHECK:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; CHECK:   [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C3]](s32)
+  ; CHECK:   [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY2]](<2 x s16>)
+  ; CHECK:   [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C3]](s32)
+  ; CHECK:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+  ; CHECK:   [[COPY8:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
+  ; CHECK:   [[AND4:%[0-9]+]]:_(s32) = G_AND [[COPY8]], [[C4]]
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(s32) = COPY [[BITCAST3]](s32)
+  ; CHECK:   [[AND5:%[0-9]+]]:_(s32) = G_AND [[COPY9]], [[C4]]
+  ; CHECK:   [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[AND4]](s32), [[AND5]]
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
+  ; CHECK:   [[AND6:%[0-9]+]]:_(s32) = G_AND [[COPY10]], [[C4]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(s32) = COPY [[LSHR3]](s32)
+  ; CHECK:   [[AND7:%[0-9]+]]:_(s32) = G_AND [[COPY11]], [[C4]]
+  ; CHECK:   [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[AND6]](s32), [[AND7]]
+  ; CHECK:   G_BR %bb.2
+  ; CHECK: bb.2:
+  ; CHECK:   [[PHI:%[0-9]+]]:_(s1) = G_PHI [[ICMP]](s1), %bb.0, [[ICMP3]](s1), %bb.1
+  ; CHECK:   [[PHI1:%[0-9]+]]:_(s1) = G_PHI [[ICMP1]](s1), %bb.0, [[ICMP4]](s1), %bb.1
+  ; CHECK:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[PHI]](s1)
+  ; CHECK:   [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[PHI1]](s1)
+  ; CHECK:   [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(s32) = COPY [[ANYEXT]](s32)
+  ; CHECK:   [[AND8:%[0-9]+]]:_(s32) = G_AND [[COPY12]], [[C5]]
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s32) = COPY [[ANYEXT1]](s32)
+  ; CHECK:   [[AND9:%[0-9]+]]:_(s32) = G_AND [[COPY13]], [[C5]]
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[AND8]](s32), [[AND9]](s32)
+  ; CHECK:   $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+  ; CHECK:   S_SETPC_B64 undef $sgpr30_sgpr31
+  bb.0:
+    successors: %bb.1, %bb.2
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+
+    %0:_(<2 x s16>) = COPY $vgpr0
+    %1:_(<2 x s16>) = COPY $vgpr1
+    %2:_(<2 x s16>) = COPY $vgpr2
+    %3:_(s32) = COPY $vgpr1
+    %4:_(s32) = G_CONSTANT i32 0
+    %5:_(<2 x s1>) = G_ICMP intpred(eq), %0, %1
+    %6:_(s1) = G_ICMP intpred(eq), %3, %4
+    G_BRCOND %6, %bb.1
+    G_BR %bb.2
+
+  bb.1:
+    successors: %bb.2
+
+    %7:_(<2 x s1>) = G_ICMP intpred(ne), %0, %2
+    G_BR %bb.2
+
+  bb.2:
+    %8:_(<2 x s1>) = G_PHI %5, %bb.0, %7, %bb.1
+    %9:_(<2 x s32>) = G_ZEXT %8
+    $vgpr0_vgpr1 = COPY %9
+    S_SETPC_B64 undef $sgpr30_sgpr31
+...

From 7c09c173a294698d06bcd1cd8c95fd331a31ccdc Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Sat, 25 Jul 2020 11:14:27 -0400
Subject: [PATCH 0135/1035] AMDGPU/GlobalISel: Reorder G_CONSTANT legality
 rules

The legal cases should be the first rules.
---
 llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index c21414d59ba0c..c2200f4b2d616 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -525,9 +525,9 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
   getActionDefinitionsBuilder(G_CONSTANT)
     .legalFor({S1, S32, S64, S16, GlobalPtr,
                LocalPtr, ConstantPtr, PrivatePtr, FlatPtr })
+    .legalIf(isPointer(0))
     .clampScalar(0, S32, S64)
-    .widenScalarToNextPow2(0)
-    .legalIf(isPointer(0));
+    .widenScalarToNextPow2(0);
 
   getActionDefinitionsBuilder(G_FCONSTANT)
     .legalFor({S32, S64, S16})

From 3e8bb7a00027c97fcf0025c9e068593d8c0ed71d Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Sat, 25 Jul 2020 10:47:33 -0400
Subject: [PATCH 0136/1035] GlobalISel: Handle fewerElementsVector for
 G_PTR_ADD

---
 .../CodeGen/GlobalISel/LegalizerHelper.cpp    |  1 +
 .../AMDGPU/GlobalISel/legalize-ptr-add.mir    | 42 +++++++++++++++++++
 2 files changed, 43 insertions(+)

diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index 5dcb5b3271d8a..49d0dceb34d37 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -3469,6 +3469,7 @@ LegalizerHelper::fewerElementsVector(MachineInstr &MI, unsigned TypeIdx,
   case G_ADD:
   case G_SUB:
   case G_MUL:
+  case G_PTR_ADD:
   case G_SMULH:
   case G_UMULH:
   case G_FADD:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ptr-add.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ptr-add.mir
index 709e9a24493c6..1d76ddc61f4b6 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ptr-add.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ptr-add.mir
@@ -143,3 +143,45 @@ body: |
     %2:_(p999) = G_PTR_ADD %0, %1
     $vgpr0_vgpr1 = COPY %2
 ...
+
+---
+name: test_gep_v2p1_v2i64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+
+    ; CHECK-LABEL: name: test_gep_v2p1_v2i64
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x p1>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+    ; CHECK: [[UV:%[0-9]+]]:_(p1), [[UV1:%[0-9]+]]:_(p1) = G_UNMERGE_VALUES [[COPY]](<2 x p1>)
+    ; CHECK: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY1]](<2 x s64>)
+    ; CHECK: [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[UV]], [[UV2]](s64)
+    ; CHECK: [[PTR_ADD1:%[0-9]+]]:_(p1) = G_PTR_ADD [[UV1]], [[UV3]](s64)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x p1>) = G_BUILD_VECTOR [[PTR_ADD]](p1), [[PTR_ADD1]](p1)
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x p1>)
+    %0:_(<2 x p1>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+    %2:_(<2 x p1>) = G_PTR_ADD %0, %1
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %2
+...
+
+---
+name: test_gep_v2p3_v2s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+
+    ; CHECK-LABEL: name: test_gep_v2p3_v2s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x p3>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    ; CHECK: [[UV:%[0-9]+]]:_(p3), [[UV1:%[0-9]+]]:_(p3) = G_UNMERGE_VALUES [[COPY]](<2 x p3>)
+    ; CHECK: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; CHECK: [[PTR_ADD:%[0-9]+]]:_(p3) = G_PTR_ADD [[UV]], [[UV2]](s32)
+    ; CHECK: [[PTR_ADD1:%[0-9]+]]:_(p3) = G_PTR_ADD [[UV1]], [[UV3]](s32)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x p3>) = G_BUILD_VECTOR [[PTR_ADD]](p3), [[PTR_ADD1]](p3)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x p3>)
+    %0:_(<2 x p3>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    %2:_(<2 x p3>) = G_PTR_ADD %0, %1
+    $vgpr0_vgpr1 = COPY %2
+...

From f6176f8a5f02d7c3ee6d3b70c46045cfdce230d4 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Sat, 25 Jul 2020 11:00:35 -0400
Subject: [PATCH 0137/1035] GlobalISel: Handle G_PTR_ADD in narrowScalar

---
 .../CodeGen/GlobalISel/LegalizerHelper.cpp    |   1 +
 .../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp |   6 +-
 .../AMDGPU/GlobalISel/legalize-ptr-add.mir    | 192 ++++++++++++++++++
 3 files changed, 196 insertions(+), 3 deletions(-)

diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index 49d0dceb34d37..441e5ca6d2098 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -1204,6 +1204,7 @@ LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalar(MachineInstr &MI,
     MI.eraseFromParent();
     return Legalized;
   }
+  case TargetOpcode::G_PTR_ADD:
   case TargetOpcode::G_PTRMASK: {
     if (TypeIdx != 1)
       return UnableToLegalize;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index c2200f4b2d616..673c5fc1e840c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -733,10 +733,10 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
       .scalarize(0);
   }
 
-  // FIXME: Clamp offset operand.
   getActionDefinitionsBuilder(G_PTR_ADD)
-    .legalIf(isPointer(0))
-    .scalarize(0);
+    .legalIf(all(isPointer(0), sameSize(0, 1)))
+    .scalarize(0)
+    .scalarSameSizeAs(1, 0);
 
   getActionDefinitionsBuilder(G_PTRMASK)
     .legalIf(all(sameSize(0, 1), typeInSet(1, {S64, S32})))
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ptr-add.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ptr-add.mir
index 1d76ddc61f4b6..e014376446a94 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ptr-add.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ptr-add.mir
@@ -185,3 +185,195 @@ body: |
     %2:_(<2 x p3>) = G_PTR_ADD %0, %1
     $vgpr0_vgpr1 = COPY %2
 ...
+
+---
+name: test_gep_global_s16_idx
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2
+
+    ; CHECK-LABEL: name: test_gep_global_s16_idx
+    ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[COPY1]](s32)
+    ; CHECK: [[SEXT_INREG:%[0-9]+]]:_(s64) = G_SEXT_INREG [[ANYEXT]], 16
+    ; CHECK: [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[SEXT_INREG]](s64)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[PTR_ADD]](p1)
+    %0:_(p1) = COPY $vgpr0_vgpr1
+    %1:_(s32) = COPY $vgpr2
+    %2:_(s16) = G_TRUNC %1
+    %3:_(p1) = G_PTR_ADD %0, %2
+    $vgpr0_vgpr1 = COPY %3
+...
+
+---
+name: test_gep_global_s32_idx
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2
+
+    ; CHECK-LABEL: name: test_gep_global_s32_idx
+    ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; CHECK: [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[COPY1]](s32)
+    ; CHECK: [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[SEXT]](s64)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[PTR_ADD]](p1)
+    %0:_(p1) = COPY $vgpr0_vgpr1
+    %1:_(s32) = COPY $vgpr2
+    %2:_(p1) = G_PTR_ADD %0, %1
+    $vgpr0_vgpr1 = COPY %2
+...
+
+---
+name: test_gep_global_s96_idx
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3_vgpr4
+
+    ; CHECK-LABEL: name: test_gep_global_s96_idx
+    ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s96) = COPY $vgpr2_vgpr3_vgpr4
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(s64) = G_TRUNC [[COPY1]](s96)
+    ; CHECK: [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[TRUNC]](s64)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[PTR_ADD]](p1)
+    %0:_(p1) = COPY $vgpr0_vgpr1
+    %1:_(s96) = COPY $vgpr2_vgpr3_vgpr4
+    %2:_(p1) = G_PTR_ADD %0, %1
+    $vgpr0_vgpr1 = COPY %2
+...
+
+---
+name: test_gep_local_i16_idx
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; CHECK-LABEL: name: test_gep_local_i16_idx
+    ; CHECK: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY2]], 16
+    ; CHECK: [[PTR_ADD:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[SEXT_INREG]](s32)
+    ; CHECK: $vgpr0 = COPY [[PTR_ADD]](p3)
+    %0:_(p3) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s16) = G_TRUNC %1
+    %3:_(p3) = G_PTR_ADD %0, %2
+    $vgpr0 = COPY %3
+...
+
+---
+name: test_gep_local_i64_idx
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1_vgpr2
+
+    ; CHECK-LABEL: name: test_gep_local_i64_idx
+    ; CHECK: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr1_vgpr2
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
+    ; CHECK: [[PTR_ADD:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[TRUNC]](s32)
+    ; CHECK: $vgpr0 = COPY [[PTR_ADD]](p3)
+    %0:_(p3) = COPY $vgpr0
+    %1:_(s64) = COPY $vgpr1_vgpr2
+    %2:_(p3) = G_PTR_ADD %0, %1
+    $vgpr0 = COPY %2
+...
+
+---
+name: test_gep_v2p1_v2i32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5
+
+    ; CHECK-LABEL: name: test_gep_v2p1_v2i32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x p1>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr4_vgpr5
+    ; CHECK: [[UV:%[0-9]+]]:_(p1), [[UV1:%[0-9]+]]:_(p1) = G_UNMERGE_VALUES [[COPY]](<2 x p1>)
+    ; CHECK: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; CHECK: [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[UV2]](s32)
+    ; CHECK: [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[UV]], [[SEXT]](s64)
+    ; CHECK: [[SEXT1:%[0-9]+]]:_(s64) = G_SEXT [[UV3]](s32)
+    ; CHECK: [[PTR_ADD1:%[0-9]+]]:_(p1) = G_PTR_ADD [[UV1]], [[SEXT1]](s64)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x p1>) = G_BUILD_VECTOR [[PTR_ADD]](p1), [[PTR_ADD1]](p1)
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x p1>)
+    %0:_(<2 x p1>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(<2 x s32>) = COPY $vgpr4_vgpr5
+    %2:_(<2 x p1>) = G_PTR_ADD %0, %1
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %2
+...
+
+---
+name: test_gep_v2p1_v2i96
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6, $vgpr7_vgpr8_vgpr9
+
+    ; CHECK-LABEL: name: test_gep_v2p1_v2i96
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x p1>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s96) = COPY $vgpr4_vgpr5_vgpr6
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s96) = COPY $vgpr7_vgpr8_vgpr9
+    ; CHECK: [[UV:%[0-9]+]]:_(p1), [[UV1:%[0-9]+]]:_(p1) = G_UNMERGE_VALUES [[COPY]](<2 x p1>)
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(s64) = G_TRUNC [[COPY1]](s96)
+    ; CHECK: [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[UV]], [[TRUNC]](s64)
+    ; CHECK: [[TRUNC1:%[0-9]+]]:_(s64) = G_TRUNC [[COPY2]](s96)
+    ; CHECK: [[PTR_ADD1:%[0-9]+]]:_(p1) = G_PTR_ADD [[UV1]], [[TRUNC1]](s64)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x p1>) = G_BUILD_VECTOR [[PTR_ADD]](p1), [[PTR_ADD1]](p1)
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x p1>)
+    %0:_(<2 x p1>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(s96) = COPY $vgpr4_vgpr5_vgpr6
+    %2:_(s96) = COPY $vgpr7_vgpr8_vgpr9
+    %3:_(<2 x s96>) = G_BUILD_VECTOR %1, %2
+    %4:_(<2 x p1>) = G_PTR_ADD %0, %3
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %4
+...
+
+---
+name: test_gep_v2p3_v2s16
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2
+
+    ; CHECK-LABEL: name: test_gep_v2p3_v2s16
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x p3>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr2
+    ; CHECK: [[UV:%[0-9]+]]:_(p3), [[UV1:%[0-9]+]]:_(p3) = G_UNMERGE_VALUES [[COPY]](<2 x p3>)
+    ; CHECK: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CHECK: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
+    ; CHECK: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY2]], 16
+    ; CHECK: [[PTR_ADD:%[0-9]+]]:_(p3) = G_PTR_ADD [[UV]], [[SEXT_INREG]](s32)
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; CHECK: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY3]], 16
+    ; CHECK: [[PTR_ADD1:%[0-9]+]]:_(p3) = G_PTR_ADD [[UV1]], [[SEXT_INREG1]](s32)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x p3>) = G_BUILD_VECTOR [[PTR_ADD]](p3), [[PTR_ADD1]](p3)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x p3>)
+    %0:_(<2 x p3>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s16>) = COPY $vgpr2
+    %2:_(<2 x p3>) = G_PTR_ADD %0, %1
+    $vgpr0_vgpr1 = COPY %2
+...
+
+---
+name: test_gep_v2p3_v2s64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3_vgpr4_vgpr5
+
+    ; CHECK-LABEL: name: test_gep_v2p3_v2s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x p3>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
+    ; CHECK: [[UV:%[0-9]+]]:_(p3), [[UV1:%[0-9]+]]:_(p3) = G_UNMERGE_VALUES [[COPY]](<2 x p3>)
+    ; CHECK: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY1]](<2 x s64>)
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[UV2]](s64)
+    ; CHECK: [[PTR_ADD:%[0-9]+]]:_(p3) = G_PTR_ADD [[UV]], [[TRUNC]](s32)
+    ; CHECK: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[UV3]](s64)
+    ; CHECK: [[PTR_ADD1:%[0-9]+]]:_(p3) = G_PTR_ADD [[UV1]], [[TRUNC1]](s32)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x p3>) = G_BUILD_VECTOR [[PTR_ADD]](p3), [[PTR_ADD1]](p3)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x p3>)
+    %0:_(<2 x p3>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s64>) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
+    %2:_(<2 x p3>) = G_PTR_ADD %0, %1
+    $vgpr0_vgpr1 = COPY %2
+...

From d35e2c101d22770a4fd5e387f6ae29bc94437426 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Sat, 25 Jul 2020 11:56:33 -0400
Subject: [PATCH 0138/1035] AMDGPU/GlobalISel: Fix not constraining
 ds_append/consume operands

---
 llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp |  7 +++++--
 llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.append.ll    | 11 +++++++++++
 llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.consume.ll   | 11 +++++++++++
 3 files changed, 27 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
index 8bc597664634a..777c8c6c2ee69 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
@@ -1313,12 +1313,15 @@ bool AMDGPUInstructionSelector::selectDSAppendConsume(MachineInstr &MI,
 
   BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0)
     .addReg(PtrBase);
-  BuildMI(*MBB, &MI, DL, TII.get(Opc), MI.getOperand(0).getReg())
+  if (!RBI.constrainGenericRegister(PtrBase, AMDGPU::SReg_32RegClass, *MRI))
+    return false;
+
+  auto MIB = BuildMI(*MBB, &MI, DL, TII.get(Opc), MI.getOperand(0).getReg())
     .addImm(Offset)
     .addImm(IsGDS ? -1 : 0)
     .cloneMemRefs(MI);
   MI.eraseFromParent();
-  return true;
+  return constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI);
 }
 
 static bool parseTexFail(uint64_t TexFailCtrl, bool &TFE, bool &LWE,
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.append.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.append.ll
index ce1551e44e51b..2da96c4480608 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.append.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.append.ll
@@ -133,6 +133,17 @@ define amdgpu_kernel void @ds_append_lds_m0_restore(i32 addrspace(3)* %lds, i32
   ret void
 }
 
+; Make sure this selects successfully with no use. The result register needs to be constrained.
+; GCN-LABEL: {{^}}ds_append_lds_no_use:
+; GCN: s_load_dword [[PTR:s[0-9]+]]
+; GCN: s_mov_b32 m0, [[PTR]]
+; GCN: ds_append [[RESULT:v[0-9]+]] offset:65532{{$}}
+define amdgpu_kernel void @ds_append_lds_no_use(i32 addrspace(3)* %lds, i32 addrspace(1)* %out) #0 {
+  %gep = getelementptr inbounds i32, i32 addrspace(3)* %lds, i32 16383
+  %val = call i32 @llvm.amdgcn.ds.append.p3i32(i32 addrspace(3)* %gep, i1 false)
+  ret void
+}
+
 declare i32 @llvm.amdgcn.ds.append.p3i32(i32 addrspace(3)* nocapture, i1 immarg) #1
 declare i32 @llvm.amdgcn.ds.append.p2i32(i32 addrspace(2)* nocapture, i1 immarg) #1
 
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.consume.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.consume.ll
index 175c0cf7760ac..40f20bc795222 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.consume.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.consume.ll
@@ -127,6 +127,17 @@ define amdgpu_kernel void @ds_consume_lds_m0_restore(i32 addrspace(3)* %lds, i32
   ret void
 }
 
+; Make sure this selects successfully with no use. The result register needs to be constrained.
+; GCN-LABEL: {{^}}ds_consume_lds_no_use:
+; GCN: s_load_dword [[PTR:s[0-9]+]]
+; GCN: s_mov_b32 m0, [[PTR]]
+; GCN: ds_consume [[RESULT:v[0-9]+]] offset:65532{{$}}
+define amdgpu_kernel void @ds_consume_lds_no_use(i32 addrspace(3)* %lds, i32 addrspace(1)* %out) #0 {
+  %gep = getelementptr inbounds i32, i32 addrspace(3)* %lds, i32 16383
+  %val = call i32 @llvm.amdgcn.ds.consume.p3i32(i32 addrspace(3)* %gep, i1 false)
+  ret void
+}
+
 declare i32 @llvm.amdgcn.ds.consume.p3i32(i32 addrspace(3)* nocapture, i1 immarg) #1
 declare i32 @llvm.amdgcn.ds.consume.p2i32(i32 addrspace(2)* nocapture, i1 immarg) #1
 

From d135744c34dc7a6315f1d2d65a969a2791a97534 Mon Sep 17 00:00:00 2001
From: Vincent Zhao <vincentzhaorz@gmail.com>
Date: Sun, 26 Jul 2020 20:10:07 +0530
Subject: [PATCH 0139/1035] [MLIR][Affine] Add test for non-hyperrectangular
 loop tiling

This diff provides a concrete test case for the error that will be raised when the iteration space is non hyper-rectangular.

The corresponding emission method for this error message has been changed as well.

Differential Revision: https://reviews.llvm.org/D84531
---
 mlir/lib/Dialect/Affine/Transforms/LoopTiling.cpp |  5 ++---
 .../Dialect/Affine/loop-tiling-unsupported.mlir   | 15 +++++++++++++++
 2 files changed, 17 insertions(+), 3 deletions(-)
 create mode 100644 mlir/test/Dialect/Affine/loop-tiling-unsupported.mlir

diff --git a/mlir/lib/Dialect/Affine/Transforms/LoopTiling.cpp b/mlir/lib/Dialect/Affine/Transforms/LoopTiling.cpp
index ddb00bdd8f0e5..d9b2b6ac30438 100644
--- a/mlir/lib/Dialect/Affine/Transforms/LoopTiling.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/LoopTiling.cpp
@@ -218,9 +218,8 @@ mlir::tilePerfectlyNested(MutableArrayRef<AffineForOp> input,
   FlatAffineConstraints cst;
   getIndexSet(input, &cst);
   if (!cst.isHyperRectangular(0, width)) {
-    llvm::dbgs() << "tiled code generation unimplemented for the "
-                    "non-hyperrectangular case, op:"
-                 << *rootAffineForOp << "\n";
+    rootAffineForOp.emitError("tiled code generation unimplemented for the "
+                              "non-hyperrectangular case");
     return failure();
   }
 
diff --git a/mlir/test/Dialect/Affine/loop-tiling-unsupported.mlir b/mlir/test/Dialect/Affine/loop-tiling-unsupported.mlir
new file mode 100644
index 0000000000000..cb558848b21cd
--- /dev/null
+++ b/mlir/test/Dialect/Affine/loop-tiling-unsupported.mlir
@@ -0,0 +1,15 @@
+// RUN: mlir-opt %s -affine-loop-tile="tile-size=32" -split-input-file -verify-diagnostics
+
+// -----
+
+#ub = affine_map<(d0)[s0] -> (d0, s0)>
+func @non_hyperrect_loop() {
+  %N = constant 128 : index
+  // expected-error@+1 {{tiled code generation unimplemented for the non-hyperrectangular case}}
+  affine.for %i = 0 to %N {
+    affine.for %j = 0 to min #ub(%i)[%N] {
+      affine.yield
+    }
+  }
+  return
+}

From 17eafe0841d6e523d410771c8d4de99d5881c59d Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Sun, 26 Jul 2020 16:03:53 +0100
Subject: [PATCH 0140/1035] [X86][SSE] lowerV2I64Shuffle - use undef elements
 in PSHUFD mask widening

If we lower a v2i64 shuffle to PSHUFD, we currently clamp undef elements to 0, (elements 0,1 of the v4i32) which can result in the shuffle referencing more elements of the source vector than expected, affecting later shuffle combines and KnownBits/SimplifyDemanded calls.

By ensuring we widen the undef mask element we allow getV4X86ShuffleImm8 to use inline elements as the default, which are more likely to fold.
---
 llvm/lib/Target/X86/X86ISelLowering.cpp       |   7 +-
 llvm/test/CodeGen/X86/avg.ll                  |  32 ++--
 llvm/test/CodeGen/X86/avx-cvt.ll              |   2 +-
 llvm/test/CodeGen/X86/avx512-hadd-hsub.ll     |   8 +-
 .../X86/avx512-intrinsics-fast-isel.ll        |  32 ++--
 llvm/test/CodeGen/X86/buildvec-extract.ll     |   6 +-
 llvm/test/CodeGen/X86/cast-vsel.ll            |  14 +-
 .../X86/clear_upper_vector_element_bits.ll    |   4 +-
 llvm/test/CodeGen/X86/combine-movmsk-avx.ll   |   4 +-
 llvm/test/CodeGen/X86/combine-sdiv.ll         |   6 +-
 llvm/test/CodeGen/X86/combine-shl.ll          |   6 +-
 llvm/test/CodeGen/X86/combine-sra.ll          |   4 +-
 llvm/test/CodeGen/X86/combine-srl.ll          |   2 +-
 llvm/test/CodeGen/X86/combine-udiv.ll         |  12 +-
 llvm/test/CodeGen/X86/combine-urem.ll         |   2 +-
 .../X86/div-rem-pair-recomposition-signed.ll  |  16 +-
 .../div-rem-pair-recomposition-unsigned.ll    |  16 +-
 llvm/test/CodeGen/X86/extract-store.ll        |   2 +-
 llvm/test/CodeGen/X86/extractelement-index.ll |   6 +-
 llvm/test/CodeGen/X86/extractelement-load.ll  |  38 ++--
 llvm/test/CodeGen/X86/gather-addresses.ll     |   8 +-
 llvm/test/CodeGen/X86/haddsub-2.ll            |  36 ++--
 ...st-and-by-const-from-lshr-in-eqcmp-zero.ll |   6 +-
 .../test/CodeGen/X86/horizontal-reduce-add.ll |  42 ++---
 .../CodeGen/X86/horizontal-reduce-smax.ll     | 140 +++++++-------
 .../CodeGen/X86/horizontal-reduce-smin.ll     | 140 +++++++-------
 .../CodeGen/X86/horizontal-reduce-umax.ll     | 142 +++++++-------
 .../CodeGen/X86/horizontal-reduce-umin.ll     | 142 +++++++-------
 llvm/test/CodeGen/X86/i128-add.ll             |   2 +-
 llvm/test/CodeGen/X86/inline-asm-x-i128.ll    |   2 +-
 llvm/test/CodeGen/X86/known-bits-vector.ll    |   4 +-
 .../test/CodeGen/X86/known-signbits-vector.ll |   6 +-
 llvm/test/CodeGen/X86/madd.ll                 | 110 +++++------
 llvm/test/CodeGen/X86/masked_compressstore.ll |  16 +-
 llvm/test/CodeGen/X86/masked_gather.ll        | 110 +++++------
 llvm/test/CodeGen/X86/masked_load.ll          |  16 +-
 llvm/test/CodeGen/X86/masked_store.ll         |  16 +-
 llvm/test/CodeGen/X86/masked_store_trunc.ll   |   6 +-
 .../CodeGen/X86/masked_store_trunc_ssat.ll    |   6 +-
 .../CodeGen/X86/masked_store_trunc_usat.ll    |   6 +-
 .../X86/merge-consecutive-stores-nt.ll        |  16 +-
 .../CodeGen/X86/min-legal-vector-width.ll     |  16 +-
 llvm/test/CodeGen/X86/nontemporal-2.ll        |   4 +-
 llvm/test/CodeGen/X86/oddshuffles.ll          |  22 +--
 llvm/test/CodeGen/X86/phaddsub-extract.ll     |  76 ++++----
 llvm/test/CodeGen/X86/pmul.ll                 |  10 +-
 llvm/test/CodeGen/X86/pmulh.ll                | 102 +++++-----
 llvm/test/CodeGen/X86/pr15267.ll              |   2 +-
 llvm/test/CodeGen/X86/pr39733.ll              |   2 +-
 llvm/test/CodeGen/X86/pr42452.ll              |   2 +-
 llvm/test/CodeGen/X86/pr42905.ll              |   2 +-
 llvm/test/CodeGen/X86/pr44976.ll              |   2 +-
 llvm/test/CodeGen/X86/pr45378.ll              |   2 +-
 llvm/test/CodeGen/X86/pr46189.ll              |   2 +-
 llvm/test/CodeGen/X86/pr46455.ll              |   2 +-
 .../CodeGen/X86/prefer-avx256-mask-shuffle.ll |   2 +-
 llvm/test/CodeGen/X86/psubus.ll               |  18 +-
 llvm/test/CodeGen/X86/sad.ll                  |  60 +++---
 llvm/test/CodeGen/X86/sdiv_fix.ll             |  12 +-
 llvm/test/CodeGen/X86/sdiv_fix_sat.ll         |  24 +--
 llvm/test/CodeGen/X86/setcc-wide-types.ll     |  48 ++---
 llvm/test/CodeGen/X86/shrink_vmul.ll          |  16 +-
 llvm/test/CodeGen/X86/slow-pmulld.ll          |  80 ++++----
 llvm/test/CodeGen/X86/smul_fix_sat.ll         |   8 +-
 .../CodeGen/X86/split-extend-vector-inreg.ll  |   2 +-
 llvm/test/CodeGen/X86/split-vector-rem.ll     |  16 +-
 .../CodeGen/X86/sse-intrinsics-fast-isel.ll   |   4 +-
 llvm/test/CodeGen/X86/sse41.ll                |  36 ++--
 llvm/test/CodeGen/X86/trunc-subvector.ll      |   4 +-
 llvm/test/CodeGen/X86/udiv_fix.ll             |   8 +-
 llvm/test/CodeGen/X86/udiv_fix_sat.ll         |   8 +-
 llvm/test/CodeGen/X86/uint_to_fp-3.ll         |   4 +-
 llvm/test/CodeGen/X86/umul_fix_sat.ll         |   8 +-
 .../CodeGen/X86/urem-seteq-vec-nonsplat.ll    |   4 +-
 llvm/test/CodeGen/X86/var-permute-128.ll      |  12 +-
 .../CodeGen/X86/vec-strict-inttofp-128.ll     |  34 ++--
 .../CodeGen/X86/vec-strict-inttofp-256.ll     |   2 +-
 llvm/test/CodeGen/X86/vec_cast2.ll            |   2 +-
 llvm/test/CodeGen/X86/vec_int_to_fp.ll        | 120 ++++++------
 llvm/test/CodeGen/X86/vec_saddo.ll            |  28 +--
 llvm/test/CodeGen/X86/vec_smulo.ll            | 136 +++++++-------
 llvm/test/CodeGen/X86/vec_ssubo.ll            |  28 +--
 llvm/test/CodeGen/X86/vec_uaddo.ll            |  28 +--
 llvm/test/CodeGen/X86/vec_umulo.ll            |  78 ++++----
 llvm/test/CodeGen/X86/vec_usubo.ll            |  28 +--
 .../X86/vector-constrained-fp-intrinsics.ll   |  30 +--
 llvm/test/CodeGen/X86/vector-fshl-128.ll      |  24 +--
 llvm/test/CodeGen/X86/vector-fshl-256.ll      |   8 +-
 llvm/test/CodeGen/X86/vector-fshl-rot-128.ll  |  20 +-
 llvm/test/CodeGen/X86/vector-fshl-rot-256.ll  |  10 +-
 llvm/test/CodeGen/X86/vector-fshr-128.ll      |  24 +--
 llvm/test/CodeGen/X86/vector-fshr-256.ll      |   8 +-
 llvm/test/CodeGen/X86/vector-fshr-rot-128.ll  |  20 +-
 llvm/test/CodeGen/X86/vector-fshr-rot-256.ll  |  10 +-
 llvm/test/CodeGen/X86/vector-idiv-sdiv-128.ll |  20 +-
 llvm/test/CodeGen/X86/vector-idiv-sdiv-256.ll |  16 +-
 llvm/test/CodeGen/X86/vector-idiv-udiv-128.ll |   4 +-
 llvm/test/CodeGen/X86/vector-pcmp.ll          |   2 +-
 llvm/test/CodeGen/X86/vector-reduce-add.ll    | 132 ++++++-------
 .../test/CodeGen/X86/vector-reduce-and-cmp.ll | 112 +++++------
 llvm/test/CodeGen/X86/vector-reduce-and.ll    | 112 +++++------
 llvm/test/CodeGen/X86/vector-reduce-mul.ll    | 176 +++++++++---------
 llvm/test/CodeGen/X86/vector-reduce-or-cmp.ll |   2 +-
 llvm/test/CodeGen/X86/vector-reduce-or.ll     | 112 +++++------
 llvm/test/CodeGen/X86/vector-reduce-smax.ll   | 108 +++++------
 llvm/test/CodeGen/X86/vector-reduce-smin.ll   | 108 +++++------
 llvm/test/CodeGen/X86/vector-reduce-umax.ll   | 108 +++++------
 llvm/test/CodeGen/X86/vector-reduce-umin.ll   | 108 +++++------
 llvm/test/CodeGen/X86/vector-reduce-xor.ll    | 112 +++++------
 llvm/test/CodeGen/X86/vector-rem.ll           |   8 +-
 llvm/test/CodeGen/X86/vector-rotate-128.ll    |  18 +-
 llvm/test/CodeGen/X86/vector-rotate-256.ll    |   8 +-
 llvm/test/CodeGen/X86/vector-sext.ll          | 130 ++++++-------
 .../test/CodeGen/X86/vector-shift-ashr-128.ll |  14 +-
 .../test/CodeGen/X86/vector-shift-ashr-256.ll |   8 +-
 .../CodeGen/X86/vector-shift-ashr-sub128.ll   |   6 +-
 .../test/CodeGen/X86/vector-shift-lshr-128.ll |  14 +-
 .../test/CodeGen/X86/vector-shift-lshr-256.ll |   8 +-
 .../CodeGen/X86/vector-shift-lshr-sub128.ll   |   6 +-
 llvm/test/CodeGen/X86/vector-shift-shl-128.ll |   8 +-
 llvm/test/CodeGen/X86/vector-shift-shl-256.ll |   8 +-
 .../test/CodeGen/X86/vector-shuffle-128-v8.ll |  22 +--
 .../CodeGen/X86/vector-shuffle-256-v16.ll     |  20 +-
 .../CodeGen/X86/vector-shuffle-256-v32.ll     |   4 +-
 .../test/CodeGen/X86/vector-shuffle-256-v8.ll |   2 +-
 .../CodeGen/X86/vector-shuffle-512-v32.ll     |   2 +-
 .../CodeGen/X86/vector-shuffle-512-v64.ll     |   4 +-
 .../CodeGen/X86/vector-shuffle-combining.ll   |   2 +-
 llvm/test/CodeGen/X86/vector-shuffle-sse4a.ll |   4 +-
 llvm/test/CodeGen/X86/vector-zext.ll          |  62 +++---
 llvm/test/CodeGen/X86/vsel-cmp-load.ll        |   2 +-
 llvm/test/CodeGen/X86/vselect-avx.ll          |   2 +-
 llvm/test/CodeGen/X86/vselect-pcmp.ll         |   4 +-
 llvm/test/CodeGen/X86/vshift-4.ll             |   4 +-
 llvm/test/CodeGen/X86/widen_conv-4.ll         |   4 +-
 .../X86/x86-setcc-int-to-fp-combine.ll        |   2 +-
 llvm/test/CodeGen/X86/xor.ll                  |   2 +-
 137 files changed, 2028 insertions(+), 2031 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 24bc264df129f..7c134a8c7cb92 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -13723,9 +13723,10 @@ static SDValue lowerV2I64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
     // onward this has a single fast instruction with no scary immediates.
     // We have to map the mask as it is actually a v4i32 shuffle instruction.
     V1 = DAG.getBitcast(MVT::v4i32, V1);
-    int WidenedMask[4] = {
-        std::max(Mask[0], 0) * 2, std::max(Mask[0], 0) * 2 + 1,
-        std::max(Mask[1], 0) * 2, std::max(Mask[1], 0) * 2 + 1};
+    int WidenedMask[4] = {Mask[0] < 0 ? -1 : (Mask[0] * 2),
+                          Mask[0] < 0 ? -1 : ((Mask[0] * 2) + 1),
+                          Mask[1] < 0 ? -1 : (Mask[1] * 2),
+                          Mask[1] < 0 ? -1 : ((Mask[1] * 2) + 1)};
     return DAG.getBitcast(
         MVT::v2i64,
         DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, V1,
diff --git a/llvm/test/CodeGen/X86/avg.ll b/llvm/test/CodeGen/X86/avg.ll
index d2638a1681e85..1411318d8176d 100644
--- a/llvm/test/CodeGen/X86/avg.ll
+++ b/llvm/test/CodeGen/X86/avg.ll
@@ -160,9 +160,9 @@ define void @avg_v24i8(<24 x i8>* %a, <24 x i8>* %b) nounwind {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vmovdqa (%rdi), %xmm0
 ; AVX1-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[3,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[3,3,2,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero
@@ -172,10 +172,10 @@ define void @avg_v24i8(<24 x i8>* %a, <24 x i8>* %b) nounwind {
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm8 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; AVX1-NEXT:    vmovdqa (%rsi), %xmm6
 ; AVX1-NEXT:    vmovdqa 16(%rsi), %xmm7
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm6[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm6[2,3,2,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm6[3,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm6[3,3,2,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
 ; AVX1-NEXT:    vpaddd %xmm2, %xmm3, %xmm2
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm6[1,1,2,3]
@@ -454,24 +454,24 @@ define void @avg_v48i8(<48 x i8>* %a, <48 x i8>* %b) nounwind {
 ; AVX1-NEXT:    vmovdqa (%rdi), %xmm0
 ; AVX1-NEXT:    vmovdqa 16(%rdi), %xmm4
 ; AVX1-NEXT:    vmovdqa 32(%rdi), %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm5 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[3,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[3,3,2,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm6 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,2,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm7 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm15 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[3,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[3,3,2,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm11 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm14 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm13 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm4[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm4[2,3,2,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; AVX1-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm4[3,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm4[3,3,2,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; AVX1-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm4[1,1,2,3]
@@ -482,10 +482,10 @@ define void @avg_v48i8(<48 x i8>* %a, <48 x i8>* %b) nounwind {
 ; AVX1-NEXT:    vmovdqa (%rsi), %xmm0
 ; AVX1-NEXT:    vmovdqa 16(%rsi), %xmm4
 ; AVX1-NEXT:    vmovdqa 32(%rsi), %xmm3
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm3[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
 ; AVX1-NEXT:    vpaddd %xmm2, %xmm5, %xmm12
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm3[3,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm3[3,3,2,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm5 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero
 ; AVX1-NEXT:    vpaddd %xmm5, %xmm6, %xmm10
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm3[1,1,2,3]
@@ -493,10 +493,10 @@ define void @avg_v48i8(<48 x i8>* %a, <48 x i8>* %b) nounwind {
 ; AVX1-NEXT:    vpaddd %xmm6, %xmm7, %xmm9
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
 ; AVX1-NEXT:    vpaddd %xmm3, %xmm1, %xmm8
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
 ; AVX1-NEXT:    vpaddd %xmm3, %xmm15, %xmm15
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm0[3,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm0[3,3,2,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm7 = xmm7[0],zero,zero,zero,xmm7[1],zero,zero,zero,xmm7[2],zero,zero,zero,xmm7[3],zero,zero,zero
 ; AVX1-NEXT:    vpaddd %xmm7, %xmm11, %xmm7
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
@@ -504,10 +504,10 @@ define void @avg_v48i8(<48 x i8>* %a, <48 x i8>* %b) nounwind {
 ; AVX1-NEXT:    vpaddd %xmm2, %xmm14, %xmm14
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; AVX1-NEXT:    vpaddd %xmm0, %xmm13, %xmm13
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm4[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm4[2,3,2,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm5 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero
 ; AVX1-NEXT:    vpaddd {{[-0-9]+}}(%r{{[sb]}}p), %xmm5, %xmm5 # 16-byte Folded Reload
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm4[3,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm4[3,3,2,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm6 = xmm6[0],zero,zero,zero,xmm6[1],zero,zero,zero,xmm6[2],zero,zero,zero,xmm6[3],zero,zero,zero
 ; AVX1-NEXT:    vpaddd {{[-0-9]+}}(%r{{[sb]}}p), %xmm6, %xmm6 # 16-byte Folded Reload
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm4[1,1,2,3]
diff --git a/llvm/test/CodeGen/X86/avx-cvt.ll b/llvm/test/CodeGen/X86/avx-cvt.ll
index 653a88edd26a0..b3fa8ac9aeeda 100644
--- a/llvm/test/CodeGen/X86/avx-cvt.ll
+++ b/llvm/test/CodeGen/X86/avx-cvt.ll
@@ -33,7 +33,7 @@ define <8 x float> @sitofp02(<8 x i16> %a) {
 ; AVX-LABEL: sitofp02:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpmovsxwd %xmm0, %xmm1
-; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpmovsxwd %xmm0, %xmm0
 ; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX-NEXT:    vcvtdq2ps %ymm0, %ymm0
diff --git a/llvm/test/CodeGen/X86/avx512-hadd-hsub.ll b/llvm/test/CodeGen/X86/avx512-hadd-hsub.ll
index 1fd3e15c3e019..b504646336def 100644
--- a/llvm/test/CodeGen/X86/avx512-hadd-hsub.ll
+++ b/llvm/test/CodeGen/X86/avx512-hadd-hsub.ll
@@ -5,7 +5,7 @@
 define i32 @hadd_16(<16 x i32> %x225) {
 ; KNL-LABEL: hadd_16:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; KNL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; KNL-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; KNL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; KNL-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -14,7 +14,7 @@ define i32 @hadd_16(<16 x i32> %x225) {
 ;
 ; SKX-LABEL: hadd_16:
 ; SKX:       # %bb.0:
-; SKX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SKX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SKX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; SKX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SKX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -32,7 +32,7 @@ define i32 @hadd_16(<16 x i32> %x225) {
 define i32 @hsub_16(<16 x i32> %x225) {
 ; KNL-LABEL: hsub_16:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; KNL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; KNL-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; KNL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; KNL-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
@@ -41,7 +41,7 @@ define i32 @hsub_16(<16 x i32> %x225) {
 ;
 ; SKX-LABEL: hsub_16:
 ; SKX:       # %bb.0:
-; SKX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SKX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SKX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; SKX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SKX-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
diff --git a/llvm/test/CodeGen/X86/avx512-intrinsics-fast-isel.ll b/llvm/test/CodeGen/X86/avx512-intrinsics-fast-isel.ll
index f115f9a6ef382..650bbe23b86e5 100644
--- a/llvm/test/CodeGen/X86/avx512-intrinsics-fast-isel.ll
+++ b/llvm/test/CodeGen/X86/avx512-intrinsics-fast-isel.ll
@@ -6550,7 +6550,7 @@ define i64 @test_mm512_reduce_add_epi64(<8 x i64> %__W) {
 ; X86-NEXT:    vpaddq %ymm1, %ymm0, %ymm0
 ; X86-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X86-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
-; X86-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
 ; X86-NEXT:    vmovd %xmm0, %eax
 ; X86-NEXT:    vpextrd $1, %xmm0, %edx
@@ -6563,7 +6563,7 @@ define i64 @test_mm512_reduce_add_epi64(<8 x i64> %__W) {
 ; X64-NEXT:    vpaddq %ymm1, %ymm0, %ymm0
 ; X64-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X64-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
-; X64-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
 ; X64-NEXT:    vmovq %xmm0, %rax
 ; X64-NEXT:    vzeroupper
@@ -6602,7 +6602,7 @@ define i64 @test_mm512_reduce_mul_epi64(<8 x i64> %__W) {
 ; X86-NEXT:    vpsllq $32, %xmm2, %xmm2
 ; X86-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
 ; X86-NEXT:    vpaddq %xmm2, %xmm0, %xmm0
-; X86-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-NEXT:    vpsrldq {{.*#+}} xmm2 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; X86-NEXT:    vpmuludq %xmm0, %xmm2, %xmm2
 ; X86-NEXT:    vpsrlq $32, %xmm0, %xmm3
@@ -6636,7 +6636,7 @@ define i64 @test_mm512_reduce_mul_epi64(<8 x i64> %__W) {
 ; X64-NEXT:    vpsllq $32, %xmm2, %xmm2
 ; X64-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
 ; X64-NEXT:    vpaddq %xmm2, %xmm0, %xmm0
-; X64-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-NEXT:    vpsrldq {{.*#+}} xmm2 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; X64-NEXT:    vpmuludq %xmm0, %xmm2, %xmm2
 ; X64-NEXT:    vpsrlq $32, %xmm0, %xmm3
@@ -6668,7 +6668,7 @@ define i64 @test_mm512_reduce_or_epi64(<8 x i64> %__W) {
 ; X86-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; X86-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X86-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; X86-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-NEXT:    vpor %xmm0, %xmm1, %xmm0
 ; X86-NEXT:    vmovd %xmm0, %eax
 ; X86-NEXT:    vpextrd $1, %xmm0, %edx
@@ -6681,7 +6681,7 @@ define i64 @test_mm512_reduce_or_epi64(<8 x i64> %__W) {
 ; X64-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; X64-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X64-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; X64-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-NEXT:    vpor %xmm0, %xmm1, %xmm0
 ; X64-NEXT:    vmovq %xmm0, %rax
 ; X64-NEXT:    vzeroupper
@@ -6706,7 +6706,7 @@ define i64 @test_mm512_reduce_and_epi64(<8 x i64> %__W) {
 ; X86-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; X86-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X86-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; X86-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-NEXT:    vpand %xmm0, %xmm1, %xmm0
 ; X86-NEXT:    vmovd %xmm0, %eax
 ; X86-NEXT:    vpextrd $1, %xmm0, %edx
@@ -6719,7 +6719,7 @@ define i64 @test_mm512_reduce_and_epi64(<8 x i64> %__W) {
 ; X64-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; X64-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X64-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; X64-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-NEXT:    vpand %xmm0, %xmm1, %xmm0
 ; X64-NEXT:    vmovq %xmm0, %rax
 ; X64-NEXT:    vzeroupper
@@ -6747,7 +6747,7 @@ define i64 @test_mm512_mask_reduce_add_epi64(i8 zeroext %__M, <8 x i64> %__W) {
 ; X86-NEXT:    vpaddq %ymm1, %ymm0, %ymm0
 ; X86-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X86-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
-; X86-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
 ; X86-NEXT:    vmovd %xmm0, %eax
 ; X86-NEXT:    vpextrd $1, %xmm0, %edx
@@ -6762,7 +6762,7 @@ define i64 @test_mm512_mask_reduce_add_epi64(i8 zeroext %__M, <8 x i64> %__W) {
 ; X64-NEXT:    vpaddq %ymm1, %ymm0, %ymm0
 ; X64-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X64-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
-; X64-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
 ; X64-NEXT:    vmovq %xmm0, %rax
 ; X64-NEXT:    vzeroupper
@@ -6807,7 +6807,7 @@ define i64 @test_mm512_mask_reduce_mul_epi64(i8 zeroext %__M, <8 x i64> %__W) {
 ; X86-NEXT:    vpsllq $32, %xmm2, %xmm2
 ; X86-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
 ; X86-NEXT:    vpaddq %xmm2, %xmm0, %xmm0
-; X86-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-NEXT:    vpsrldq {{.*#+}} xmm2 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; X86-NEXT:    vpmuludq %xmm0, %xmm2, %xmm2
 ; X86-NEXT:    vpsrlq $32, %xmm0, %xmm3
@@ -6844,7 +6844,7 @@ define i64 @test_mm512_mask_reduce_mul_epi64(i8 zeroext %__M, <8 x i64> %__W) {
 ; X64-NEXT:    vpsllq $32, %xmm2, %xmm2
 ; X64-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
 ; X64-NEXT:    vpaddq %xmm2, %xmm0, %xmm0
-; X64-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-NEXT:    vpsrldq {{.*#+}} xmm2 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; X64-NEXT:    vpmuludq %xmm0, %xmm2, %xmm2
 ; X64-NEXT:    vpsrlq $32, %xmm0, %xmm3
@@ -6882,7 +6882,7 @@ define i64 @test_mm512_mask_reduce_and_epi64(i8 zeroext %__M, <8 x i64> %__W) {
 ; X86-NEXT:    vpand %ymm0, %ymm1, %ymm0
 ; X86-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X86-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; X86-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-NEXT:    vpand %xmm0, %xmm1, %xmm0
 ; X86-NEXT:    vmovd %xmm0, %eax
 ; X86-NEXT:    vpextrd $1, %xmm0, %edx
@@ -6898,7 +6898,7 @@ define i64 @test_mm512_mask_reduce_and_epi64(i8 zeroext %__M, <8 x i64> %__W) {
 ; X64-NEXT:    vpand %ymm0, %ymm1, %ymm0
 ; X64-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X64-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; X64-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-NEXT:    vpand %xmm0, %xmm1, %xmm0
 ; X64-NEXT:    vmovq %xmm0, %rax
 ; X64-NEXT:    vzeroupper
@@ -6928,7 +6928,7 @@ define i64 @test_mm512_mask_reduce_or_epi64(i8 zeroext %__M, <8 x i64> %__W) {
 ; X86-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; X86-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X86-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; X86-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-NEXT:    vpor %xmm0, %xmm1, %xmm0
 ; X86-NEXT:    vmovd %xmm0, %eax
 ; X86-NEXT:    vpextrd $1, %xmm0, %edx
@@ -6943,7 +6943,7 @@ define i64 @test_mm512_mask_reduce_or_epi64(i8 zeroext %__M, <8 x i64> %__W) {
 ; X64-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; X64-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X64-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; X64-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-NEXT:    vpor %xmm0, %xmm1, %xmm0
 ; X64-NEXT:    vmovq %xmm0, %rax
 ; X64-NEXT:    vzeroupper
diff --git a/llvm/test/CodeGen/X86/buildvec-extract.ll b/llvm/test/CodeGen/X86/buildvec-extract.ll
index 863ab4dee1238..2195526f94c39 100644
--- a/llvm/test/CodeGen/X86/buildvec-extract.ll
+++ b/llvm/test/CodeGen/X86/buildvec-extract.ll
@@ -114,21 +114,21 @@ define <2 x i64> @extract2_i32_zext_insert0_i64_undef(<4 x i32> %x) {
 define <2 x i64> @extract2_i32_zext_insert0_i64_zero(<4 x i32> %x) {
 ; SSE2-LABEL: extract2_i32_zext_insert0_i64_zero:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
 ; SSE2-NEXT:    movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: extract2_i32_zext_insert0_i64_zero:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
 ; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3,4,5,6,7]
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: extract2_i32_zext_insert0_i64_zero:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
 ; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/cast-vsel.ll b/llvm/test/CodeGen/X86/cast-vsel.ll
index 2e044548404eb..6e9e4fd00636d 100644
--- a/llvm/test/CodeGen/X86/cast-vsel.ll
+++ b/llvm/test/CodeGen/X86/cast-vsel.ll
@@ -31,7 +31,7 @@ define <8 x i32> @sext(<8 x float> %a, <8 x float> %b, <8 x i16> %c, <8 x i16> %
 ; SSE41-NEXT:    packssdw %xmm1, %xmm0
 ; SSE41-NEXT:    pblendvb %xmm0, %xmm4, %xmm5
 ; SSE41-NEXT:    pmovsxwd %xmm5, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm5[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm5[2,3,2,3]
 ; SSE41-NEXT:    pmovsxwd %xmm1, %xmm1
 ; SSE41-NEXT:    retq
 ;
@@ -39,9 +39,9 @@ define <8 x i32> @sext(<8 x float> %a, <8 x float> %b, <8 x i16> %c, <8 x i16> %
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vcmpltps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm2[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm2[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxwd %xmm4, %xmm4
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm3[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm3[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxwd %xmm5, %xmm5
 ; AVX1-NEXT:    vblendvps %xmm1, %xmm4, %xmm5, %xmm1
 ; AVX1-NEXT:    vpmovsxwd %xmm2, %xmm2
@@ -87,7 +87,7 @@ define <8 x i32> @zext(<8 x float> %a, <8 x float> %b, <8 x i16> %c, <8 x i16> %
 ; SSE41-NEXT:    packssdw %xmm1, %xmm0
 ; SSE41-NEXT:    pblendvb %xmm0, %xmm4, %xmm5
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm5[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm5[2,3,2,3]
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
 ; SSE41-NEXT:    retq
 ;
@@ -459,7 +459,7 @@ define void @example24(i16 signext %x, i16 signext %y) nounwind {
 ; SSE41-NEXT:    packssdw %xmm3, %xmm0
 ; SSE41-NEXT:    movdqa %xmm2, %xmm3
 ; SSE41-NEXT:    pblendvb %xmm0, %xmm1, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]
 ; SSE41-NEXT:    pmovsxwd %xmm0, %xmm0
 ; SSE41-NEXT:    pmovsxwd %xmm3, %xmm3
 ; SSE41-NEXT:    movdqa %xmm3, dj+4096(%rax)
@@ -480,9 +480,9 @@ define void @example24(i16 signext %x, i16 signext %y) nounwind {
 ; AVX1-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; AVX1-NEXT:    vpmovsxwd %xmm2, %xmm0
 ; AVX1-NEXT:    vpmovsxwd %xmm3, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxwd %xmm2, %xmm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxwd %xmm3, %xmm3
 ; AVX1-NEXT:    .p2align 4, 0x90
 ; AVX1-NEXT:  .LBB6_1: # %vector.body
diff --git a/llvm/test/CodeGen/X86/clear_upper_vector_element_bits.ll b/llvm/test/CodeGen/X86/clear_upper_vector_element_bits.ll
index b96f44ec3073f..16a993316d7e5 100644
--- a/llvm/test/CodeGen/X86/clear_upper_vector_element_bits.ll
+++ b/llvm/test/CodeGen/X86/clear_upper_vector_element_bits.ll
@@ -696,7 +696,7 @@ define <16 x i8> @_clearupper16xi8b(<16 x i8>) nounwind {
 ; SSE2-LABEL: _clearupper16xi8b:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pushq %rbx
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm1, %r10
 ; SSE2-NEXT:    movq %r10, %r8
 ; SSE2-NEXT:    shrq $56, %r8
@@ -878,7 +878,7 @@ define <32 x i8> @_clearupper32xi8b(<32 x i8>) nounwind {
 ; SSE2-LABEL: _clearupper32xi8b:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pushq %rbx
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm2, %r10
 ; SSE2-NEXT:    movq %r10, %r8
 ; SSE2-NEXT:    shrq $56, %r8
diff --git a/llvm/test/CodeGen/X86/combine-movmsk-avx.ll b/llvm/test/CodeGen/X86/combine-movmsk-avx.ll
index b18c7246104d4..17d01e1d3362c 100644
--- a/llvm/test/CodeGen/X86/combine-movmsk-avx.ll
+++ b/llvm/test/CodeGen/X86/combine-movmsk-avx.ll
@@ -93,7 +93,7 @@ define i32 @movmskps_sext_v4i64(<4 x i32> %a0)  {
 ; AVX1-LABEL: movmskps_sext_v4i64:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    vmovmskpd %ymm0, %eax
@@ -116,7 +116,7 @@ define i32 @movmskps_sext_v8i32(<8 x i16> %a0)  {
 ; AVX1-LABEL: movmskps_sext_v8i32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    vmovmskps %ymm0, %eax
diff --git a/llvm/test/CodeGen/X86/combine-sdiv.ll b/llvm/test/CodeGen/X86/combine-sdiv.ll
index 1a52ebfc6cd90..ce411b5e8f06b 100644
--- a/llvm/test/CodeGen/X86/combine-sdiv.ll
+++ b/llvm/test/CodeGen/X86/combine-sdiv.ll
@@ -2959,7 +2959,7 @@ define <16 x i8> @pr38658(<16 x i8> %x) {
 ;
 ; SSE41-LABEL: pr38658:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovsxbw %xmm1, %xmm2
 ; SSE41-NEXT:    pmullw {{.*}}(%rip), %xmm2
 ; SSE41-NEXT:    psrlw $8, %xmm2
@@ -2984,7 +2984,7 @@ define <16 x i8> @pr38658(<16 x i8> %x) {
 ;
 ; AVX1-LABEL: pr38658:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbw %xmm1, %xmm1
 ; AVX1-NEXT:    vpmullw {{.*}}(%rip), %xmm1, %xmm1
 ; AVX1-NEXT:    vpsrlw $8, %xmm1, %xmm1
@@ -3058,7 +3058,7 @@ define <16 x i8> @pr38658(<16 x i8> %x) {
 ;
 ; XOP-LABEL: pr38658:
 ; XOP:       # %bb.0:
-; XOP-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; XOP-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; XOP-NEXT:    vpmovsxbw %xmm1, %xmm1
 ; XOP-NEXT:    vpmullw {{.*}}(%rip), %xmm1, %xmm1
 ; XOP-NEXT:    vpxor %xmm2, %xmm2, %xmm2
diff --git a/llvm/test/CodeGen/X86/combine-shl.ll b/llvm/test/CodeGen/X86/combine-shl.ll
index a6950873daf55..383d1866aa1d9 100644
--- a/llvm/test/CodeGen/X86/combine-shl.ll
+++ b/llvm/test/CodeGen/X86/combine-shl.ll
@@ -311,7 +311,7 @@ define <8 x i32> @combine_vec_shl_ext_shl2(<8 x i16> %x) {
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pmovsxwd %xmm0, %xmm2
 ; SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovsxwd %xmm0, %xmm1
 ; SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm1
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
@@ -344,7 +344,7 @@ define <8 x i32> @combine_vec_shl_zext_lshr0(<8 x i16> %x) {
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pand {{.*}}(%rip), %xmm0
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; SSE41-NEXT:    retq
@@ -377,7 +377,7 @@ define <8 x i32> @combine_vec_shl_zext_lshr1(<8 x i16> %x) {
 ; SSE41-NEXT:    pmulhuw {{.*}}(%rip), %xmm0
 ; SSE41-NEXT:    pmullw {{.*}}(%rip), %xmm0
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; SSE41-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/combine-sra.ll b/llvm/test/CodeGen/X86/combine-sra.ll
index e2f3f2dc7523f..28a73cdb6a41e 100644
--- a/llvm/test/CodeGen/X86/combine-sra.ll
+++ b/llvm/test/CodeGen/X86/combine-sra.ll
@@ -152,7 +152,7 @@ define <4 x i32> @combine_vec_ashr_trunc_and(<4 x i32> %x, <4 x i64> %y) {
 ; SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm1[2,3,3,3,4,5,6,7]
 ; SSE-NEXT:    movdqa %xmm0, %xmm3
 ; SSE-NEXT:    psrad %xmm2, %xmm3
-; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; SSE-NEXT:    pshuflw {{.*#+}} xmm4 = xmm2[2,3,3,3,4,5,6,7]
 ; SSE-NEXT:    movdqa %xmm0, %xmm5
 ; SSE-NEXT:    psrad %xmm4, %xmm5
@@ -272,7 +272,7 @@ define <4 x i32> @combine_vec_ashr_positive(<4 x i32> %x, <4 x i32> %y) {
 ; SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm1[2,3,3,3,4,5,6,7]
 ; SSE-NEXT:    movdqa %xmm0, %xmm3
 ; SSE-NEXT:    psrld %xmm2, %xmm3
-; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; SSE-NEXT:    pshuflw {{.*#+}} xmm4 = xmm2[2,3,3,3,4,5,6,7]
 ; SSE-NEXT:    movdqa %xmm0, %xmm5
 ; SSE-NEXT:    psrld %xmm4, %xmm5
diff --git a/llvm/test/CodeGen/X86/combine-srl.ll b/llvm/test/CodeGen/X86/combine-srl.ll
index 36fbdaf537025..2e886defafd4c 100644
--- a/llvm/test/CodeGen/X86/combine-srl.ll
+++ b/llvm/test/CodeGen/X86/combine-srl.ll
@@ -400,7 +400,7 @@ define <4 x i32> @combine_vec_lshr_trunc_and(<4 x i32> %x, <4 x i64> %y) {
 ; SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm1[2,3,3,3,4,5,6,7]
 ; SSE-NEXT:    movdqa %xmm0, %xmm3
 ; SSE-NEXT:    psrld %xmm2, %xmm3
-; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; SSE-NEXT:    pshuflw {{.*#+}} xmm4 = xmm2[2,3,3,3,4,5,6,7]
 ; SSE-NEXT:    movdqa %xmm0, %xmm5
 ; SSE-NEXT:    psrld %xmm4, %xmm5
diff --git a/llvm/test/CodeGen/X86/combine-udiv.ll b/llvm/test/CodeGen/X86/combine-udiv.ll
index e6d7aac926162..c44342d00357a 100644
--- a/llvm/test/CodeGen/X86/combine-udiv.ll
+++ b/llvm/test/CodeGen/X86/combine-udiv.ll
@@ -233,7 +233,7 @@ define <4 x i32> @combine_vec_udiv_by_pow2c(<4 x i32> %x, <4 x i32> %y) {
 ; SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; SSE2-NEXT:    psrld %xmm4, %xmm2
 ; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm1[2,3,3,3,4,5,6,7]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm4
 ; SSE2-NEXT:    psrld %xmm3, %xmm4
@@ -249,7 +249,7 @@ define <4 x i32> @combine_vec_udiv_by_pow2c(<4 x i32> %x, <4 x i32> %y) {
 ; SSE41-NEXT:    pshuflw {{.*#+}} xmm2 = xmm1[2,3,3,3,4,5,6,7]
 ; SSE41-NEXT:    movdqa %xmm0, %xmm3
 ; SSE41-NEXT:    psrld %xmm2, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; SSE41-NEXT:    pshuflw {{.*#+}} xmm4 = xmm2[2,3,3,3,4,5,6,7]
 ; SSE41-NEXT:    movdqa %xmm0, %xmm5
 ; SSE41-NEXT:    psrld %xmm4, %xmm5
@@ -307,7 +307,7 @@ define <4 x i32> @combine_vec_udiv_by_shl_pow2a(<4 x i32> %x, <4 x i32> %y) {
 ; SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; SSE2-NEXT:    psrld %xmm4, %xmm2
 ; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm1[2,3,3,3,4,5,6,7]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm4
 ; SSE2-NEXT:    psrld %xmm3, %xmm4
@@ -324,7 +324,7 @@ define <4 x i32> @combine_vec_udiv_by_shl_pow2a(<4 x i32> %x, <4 x i32> %y) {
 ; SSE41-NEXT:    pshuflw {{.*#+}} xmm2 = xmm1[2,3,3,3,4,5,6,7]
 ; SSE41-NEXT:    movdqa %xmm0, %xmm3
 ; SSE41-NEXT:    psrld %xmm2, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; SSE41-NEXT:    pshuflw {{.*#+}} xmm4 = xmm2[2,3,3,3,4,5,6,7]
 ; SSE41-NEXT:    movdqa %xmm0, %xmm5
 ; SSE41-NEXT:    psrld %xmm4, %xmm5
@@ -384,7 +384,7 @@ define <4 x i32> @combine_vec_udiv_by_shl_pow2b(<4 x i32> %x, <4 x i32> %y) {
 ; SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; SSE2-NEXT:    psrld %xmm4, %xmm2
 ; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm1[2,3,3,3,4,5,6,7]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm4
 ; SSE2-NEXT:    psrld %xmm3, %xmm4
@@ -401,7 +401,7 @@ define <4 x i32> @combine_vec_udiv_by_shl_pow2b(<4 x i32> %x, <4 x i32> %y) {
 ; SSE41-NEXT:    pshuflw {{.*#+}} xmm2 = xmm1[2,3,3,3,4,5,6,7]
 ; SSE41-NEXT:    movdqa %xmm0, %xmm3
 ; SSE41-NEXT:    psrld %xmm2, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; SSE41-NEXT:    pshuflw {{.*#+}} xmm4 = xmm2[2,3,3,3,4,5,6,7]
 ; SSE41-NEXT:    movdqa %xmm0, %xmm5
 ; SSE41-NEXT:    psrld %xmm4, %xmm5
diff --git a/llvm/test/CodeGen/X86/combine-urem.ll b/llvm/test/CodeGen/X86/combine-urem.ll
index b21ed8ec60cef..cd0b21d02969c 100644
--- a/llvm/test/CodeGen/X86/combine-urem.ll
+++ b/llvm/test/CodeGen/X86/combine-urem.ll
@@ -213,7 +213,7 @@ define <4 x i32> @combine_vec_urem_by_pow2d(<4 x i32> %x, <4 x i32> %y) {
 ; SSE-NEXT:    movdqa {{.*#+}} xmm3 = [2147483648,2147483648,2147483648,2147483648]
 ; SSE-NEXT:    movdqa %xmm3, %xmm4
 ; SSE-NEXT:    psrld %xmm2, %xmm4
-; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; SSE-NEXT:    pshuflw {{.*#+}} xmm5 = xmm2[2,3,3,3,4,5,6,7]
 ; SSE-NEXT:    movdqa %xmm3, %xmm6
 ; SSE-NEXT:    psrld %xmm5, %xmm6
diff --git a/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll b/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll
index 753aee85f319c..59101503b5a93 100644
--- a/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll
+++ b/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll
@@ -565,9 +565,9 @@ define <4 x i32> @vector_i128_i32(<4 x i32> %x, <4 x i32> %y, <4 x i32>* %divdst
 ; X86-NEXT:    cltd
 ; X86-NEXT:    idivl %esi
 ; X86-NEXT:    movd %eax, %xmm2
-; X86-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,0,1]
+; X86-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
 ; X86-NEXT:    movd %xmm3, %eax
-; X86-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,0,1]
+; X86-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
 ; X86-NEXT:    movd %xmm3, %esi
 ; X86-NEXT:    cltd
 ; X86-NEXT:    idivl %esi
@@ -608,9 +608,9 @@ define <4 x i32> @vector_i128_i32(<4 x i32> %x, <4 x i32> %y, <4 x i32>* %divdst
 ; X64-NEXT:    cltd
 ; X64-NEXT:    idivl %ecx
 ; X64-NEXT:    movd %eax, %xmm2
-; X64-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,0,1]
+; X64-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
 ; X64-NEXT:    movd %xmm3, %eax
-; X64-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,0,1]
+; X64-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
 ; X64-NEXT:    movd %xmm3, %ecx
 ; X64-NEXT:    cltd
 ; X64-NEXT:    idivl %ecx
@@ -657,11 +657,11 @@ define <2 x i64> @vector_i128_i64(<2 x i64> %x, <2 x i64> %y, <2 x i64>* %divdst
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,1,2,3]
 ; X86-NEXT:    movd %xmm2, {{[0-9]+}}(%esp)
-; X86-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; X86-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; X86-NEXT:    movd %xmm2, {{[0-9]+}}(%esp)
 ; X86-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3]
 ; X86-NEXT:    movd %xmm1, {{[0-9]+}}(%esp)
-; X86-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-NEXT:    movd %xmm1, (%esp)
 ; X86-NEXT:    calll __divdi3
 ; X86-NEXT:    movdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm1 # 16-byte Reload
@@ -707,9 +707,9 @@ define <2 x i64> @vector_i128_i64(<2 x i64> %x, <2 x i64> %y, <2 x i64>* %divdst
 ; X64-NEXT:    cqto
 ; X64-NEXT:    idivq %rcx
 ; X64-NEXT:    movq %rax, %xmm2
-; X64-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,0,1]
+; X64-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
 ; X64-NEXT:    movq %xmm3, %rax
-; X64-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,0,1]
+; X64-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
 ; X64-NEXT:    movq %xmm3, %rcx
 ; X64-NEXT:    cqto
 ; X64-NEXT:    idivq %rcx
diff --git a/llvm/test/CodeGen/X86/div-rem-pair-recomposition-unsigned.ll b/llvm/test/CodeGen/X86/div-rem-pair-recomposition-unsigned.ll
index 43fc23c836a9b..d787f91ababba 100644
--- a/llvm/test/CodeGen/X86/div-rem-pair-recomposition-unsigned.ll
+++ b/llvm/test/CodeGen/X86/div-rem-pair-recomposition-unsigned.ll
@@ -565,9 +565,9 @@ define <4 x i32> @vector_i128_i32(<4 x i32> %x, <4 x i32> %y, <4 x i32>* %divdst
 ; X86-NEXT:    xorl %edx, %edx
 ; X86-NEXT:    divl %esi
 ; X86-NEXT:    movd %eax, %xmm2
-; X86-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,0,1]
+; X86-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
 ; X86-NEXT:    movd %xmm3, %eax
-; X86-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,0,1]
+; X86-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
 ; X86-NEXT:    movd %xmm3, %esi
 ; X86-NEXT:    xorl %edx, %edx
 ; X86-NEXT:    divl %esi
@@ -608,9 +608,9 @@ define <4 x i32> @vector_i128_i32(<4 x i32> %x, <4 x i32> %y, <4 x i32>* %divdst
 ; X64-NEXT:    xorl %edx, %edx
 ; X64-NEXT:    divl %ecx
 ; X64-NEXT:    movd %eax, %xmm2
-; X64-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,0,1]
+; X64-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
 ; X64-NEXT:    movd %xmm3, %eax
-; X64-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,0,1]
+; X64-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
 ; X64-NEXT:    movd %xmm3, %ecx
 ; X64-NEXT:    xorl %edx, %edx
 ; X64-NEXT:    divl %ecx
@@ -657,11 +657,11 @@ define <2 x i64> @vector_i128_i64(<2 x i64> %x, <2 x i64> %y, <2 x i64>* %divdst
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,1,2,3]
 ; X86-NEXT:    movd %xmm2, {{[0-9]+}}(%esp)
-; X86-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; X86-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; X86-NEXT:    movd %xmm2, {{[0-9]+}}(%esp)
 ; X86-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3]
 ; X86-NEXT:    movd %xmm1, {{[0-9]+}}(%esp)
-; X86-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-NEXT:    movd %xmm1, (%esp)
 ; X86-NEXT:    calll __udivdi3
 ; X86-NEXT:    movdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm1 # 16-byte Reload
@@ -707,9 +707,9 @@ define <2 x i64> @vector_i128_i64(<2 x i64> %x, <2 x i64> %y, <2 x i64>* %divdst
 ; X64-NEXT:    xorl %edx, %edx
 ; X64-NEXT:    divq %rcx
 ; X64-NEXT:    movq %rax, %xmm2
-; X64-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,0,1]
+; X64-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
 ; X64-NEXT:    movq %xmm3, %rax
-; X64-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,0,1]
+; X64-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
 ; X64-NEXT:    movq %xmm3, %rcx
 ; X64-NEXT:    xorl %edx, %edx
 ; X64-NEXT:    divq %rcx
diff --git a/llvm/test/CodeGen/X86/extract-store.ll b/llvm/test/CodeGen/X86/extract-store.ll
index c29fac61af3fe..d50c2ed920753 100644
--- a/llvm/test/CodeGen/X86/extract-store.ll
+++ b/llvm/test/CodeGen/X86/extract-store.ll
@@ -314,7 +314,7 @@ define void @extract_i64_1(i64* nocapture %dst, <2 x i64> %foo) nounwind {
 ;
 ; SSE2-X64-LABEL: extract_i64_1:
 ; SSE2-X64:       # %bb.0:
-; SSE2-X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE2-X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE2-X64-NEXT:    movq %xmm0, (%rdi)
 ; SSE2-X64-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/extractelement-index.ll b/llvm/test/CodeGen/X86/extractelement-index.ll
index cf06f8dcb13e1..f2e01e93361ef 100644
--- a/llvm/test/CodeGen/X86/extractelement-index.ll
+++ b/llvm/test/CodeGen/X86/extractelement-index.ll
@@ -351,7 +351,7 @@ define i64 @extractelement_v2i64_0(<2 x i64> %a, i256 %i) nounwind {
 define i64 @extractelement_v2i64_1(<2 x i64> %a, i256 %i) nounwind {
 ; SSE2-LABEL: extractelement_v2i64_1:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm0, %rax
 ; SSE2-NEXT:    retq
 ;
@@ -371,7 +371,7 @@ define i64 @extractelement_v2i64_1(<2 x i64> %a, i256 %i) nounwind {
 define i64 @extractelement_v4i64_1(<4 x i64> %a, i256 %i) nounwind {
 ; SSE2-LABEL: extractelement_v4i64_1:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm0, %rax
 ; SSE2-NEXT:    retq
 ;
@@ -392,7 +392,7 @@ define i64 @extractelement_v4i64_1(<4 x i64> %a, i256 %i) nounwind {
 define i64 @extractelement_v4i64_3(<4 x i64> %a, i256 %i) nounwind {
 ; SSE2-LABEL: extractelement_v4i64_3:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm0, %rax
 ; SSE2-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/extractelement-load.ll b/llvm/test/CodeGen/X86/extractelement-load.ll
index 94628c70d989e..b694859b757c0 100644
--- a/llvm/test/CodeGen/X86/extractelement-load.ll
+++ b/llvm/test/CodeGen/X86/extractelement-load.ll
@@ -10,13 +10,13 @@ define i32 @t(<2 x i64>* %val) nounwind  {
 ; X32-SSE2-LABEL: t:
 ; X32-SSE2:       # %bb.0:
 ; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE2-NEXT:    pshufd $78, (%eax), %xmm0 # xmm0 = mem[2,3,0,1]
+; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = mem[2,3,2,3]
 ; X32-SSE2-NEXT:    movd %xmm0, %eax
 ; X32-SSE2-NEXT:    retl
 ;
 ; X64-SSSE3-LABEL: t:
 ; X64-SSSE3:       # %bb.0:
-; X64-SSSE3-NEXT:    pshufd $78, (%rdi), %xmm0 # xmm0 = mem[2,3,0,1]
+; X64-SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = mem[2,3,2,3]
 ; X64-SSSE3-NEXT:    movd %xmm0, %eax
 ; X64-SSSE3-NEXT:    retq
 ;
@@ -60,13 +60,13 @@ define void @t3(<2 x double>* %a0) {
 ;
 ; X64-SSSE3-LABEL: t3:
 ; X64-SSSE3:       # %bb.0: # %bb
-; X64-SSSE3-NEXT:    movsd 8(%rdi), %xmm0 # xmm0 = mem[0],zero
+; X64-SSSE3-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; X64-SSSE3-NEXT:    movsd %xmm0, (%rax)
 ; X64-SSSE3-NEXT:    retq
 ;
 ; X64-AVX-LABEL: t3:
 ; X64-AVX:       # %bb.0: # %bb
-; X64-AVX-NEXT:    vmovsd 8(%rdi), %xmm0 # xmm0 = mem[0],zero
+; X64-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; X64-AVX-NEXT:    vmovsd %xmm0, (%rax)
 ; X64-AVX-NEXT:    retq
 bb:
@@ -139,7 +139,7 @@ define float @t6(<8 x float> *%a0) {
 ; X32-SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
 ; X32-SSE2-NEXT:    xorps %xmm1, %xmm1
 ; X32-SSE2-NEXT:    cmpeqss %xmm0, %xmm1
-; X32-SSE2-NEXT:    movss {{\.LCPI.*}}, %xmm2 # xmm2 = mem[0],zero,zero,zero
+; X32-SSE2-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
 ; X32-SSE2-NEXT:    andps %xmm1, %xmm2
 ; X32-SSE2-NEXT:    andnps %xmm0, %xmm1
 ; X32-SSE2-NEXT:    orps %xmm2, %xmm1
@@ -151,10 +151,10 @@ define float @t6(<8 x float> *%a0) {
 ;
 ; X64-SSSE3-LABEL: t6:
 ; X64-SSSE3:       # %bb.0:
-; X64-SSSE3-NEXT:    movshdup (%rdi), %xmm1 # xmm1 = mem[1,1,3,3]
+; X64-SSSE3-NEXT:    movshdup {{.*#+}} xmm1 = mem[1,1,3,3]
 ; X64-SSSE3-NEXT:    xorps %xmm0, %xmm0
 ; X64-SSSE3-NEXT:    cmpeqss %xmm1, %xmm0
-; X64-SSSE3-NEXT:    movss {{.*}}(%rip), %xmm2 # xmm2 = mem[0],zero,zero,zero
+; X64-SSSE3-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
 ; X64-SSSE3-NEXT:    andps %xmm0, %xmm2
 ; X64-SSSE3-NEXT:    andnps %xmm1, %xmm0
 ; X64-SSSE3-NEXT:    orps %xmm2, %xmm0
@@ -162,10 +162,10 @@ define float @t6(<8 x float> *%a0) {
 ;
 ; X64-AVX-LABEL: t6:
 ; X64-AVX:       # %bb.0:
-; X64-AVX-NEXT:    vmovss 4(%rdi), %xmm0 # xmm0 = mem[0],zero,zero,zero
+; X64-AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X64-AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
 ; X64-AVX-NEXT:    vcmpeqss %xmm1, %xmm0, %xmm1
-; X64-AVX-NEXT:    vmovss {{.*}}(%rip), %xmm2 # xmm2 = mem[0],zero,zero,zero
+; X64-AVX-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
 ; X64-AVX-NEXT:    vblendvps %xmm1, %xmm2, %xmm0, %xmm0
 ; X64-AVX-NEXT:    retq
   %vecload = load <8 x float>, <8 x float>* %a0, align 32
@@ -184,7 +184,7 @@ define void @PR43971(<8 x float> *%a0, float *%a1) {
 ; X32-SSE2-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
 ; X32-SSE2-NEXT:    xorps %xmm1, %xmm1
 ; X32-SSE2-NEXT:    cmpltss %xmm0, %xmm1
-; X32-SSE2-NEXT:    movss (%eax), %xmm2 # xmm2 = mem[0],zero,zero,zero
+; X32-SSE2-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
 ; X32-SSE2-NEXT:    andps %xmm1, %xmm2
 ; X32-SSE2-NEXT:    andnps %xmm0, %xmm1
 ; X32-SSE2-NEXT:    orps %xmm2, %xmm1
@@ -193,10 +193,10 @@ define void @PR43971(<8 x float> *%a0, float *%a1) {
 ;
 ; X64-SSSE3-LABEL: PR43971:
 ; X64-SSSE3:       # %bb.0: # %entry
-; X64-SSSE3-NEXT:    movss 24(%rdi), %xmm0 # xmm0 = mem[0],zero,zero,zero
+; X64-SSSE3-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X64-SSSE3-NEXT:    xorps %xmm1, %xmm1
 ; X64-SSSE3-NEXT:    cmpltss %xmm0, %xmm1
-; X64-SSSE3-NEXT:    movss (%rsi), %xmm2 # xmm2 = mem[0],zero,zero,zero
+; X64-SSSE3-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
 ; X64-SSSE3-NEXT:    andps %xmm1, %xmm2
 ; X64-SSSE3-NEXT:    andnps %xmm0, %xmm1
 ; X64-SSSE3-NEXT:    orps %xmm2, %xmm1
@@ -205,10 +205,10 @@ define void @PR43971(<8 x float> *%a0, float *%a1) {
 ;
 ; X64-AVX-LABEL: PR43971:
 ; X64-AVX:       # %bb.0: # %entry
-; X64-AVX-NEXT:    vmovss 24(%rdi), %xmm0 # xmm0 = mem[0],zero,zero,zero
+; X64-AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X64-AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
 ; X64-AVX-NEXT:    vcmpltss %xmm0, %xmm1, %xmm1
-; X64-AVX-NEXT:    vmovss (%rsi), %xmm2 # xmm2 = mem[0],zero,zero,zero
+; X64-AVX-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
 ; X64-AVX-NEXT:    vblendvps %xmm1, %xmm2, %xmm0, %xmm0
 ; X64-AVX-NEXT:    vmovss %xmm0, (%rsi)
 ; X64-AVX-NEXT:    retq
@@ -231,7 +231,7 @@ define float @PR43971_1(<8 x float> *%a0) nounwind {
 ; X32-SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
 ; X32-SSE2-NEXT:    xorps %xmm1, %xmm1
 ; X32-SSE2-NEXT:    cmpeqss %xmm0, %xmm1
-; X32-SSE2-NEXT:    movss {{\.LCPI.*}}, %xmm2 # xmm2 = mem[0],zero,zero,zero
+; X32-SSE2-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
 ; X32-SSE2-NEXT:    andps %xmm1, %xmm2
 ; X32-SSE2-NEXT:    andnps %xmm0, %xmm1
 ; X32-SSE2-NEXT:    orps %xmm2, %xmm1
@@ -242,10 +242,10 @@ define float @PR43971_1(<8 x float> *%a0) nounwind {
 ;
 ; X64-SSSE3-LABEL: PR43971_1:
 ; X64-SSSE3:       # %bb.0: # %entry
-; X64-SSSE3-NEXT:    movshdup (%rdi), %xmm1 # xmm1 = mem[1,1,3,3]
+; X64-SSSE3-NEXT:    movshdup {{.*#+}} xmm1 = mem[1,1,3,3]
 ; X64-SSSE3-NEXT:    xorps %xmm0, %xmm0
 ; X64-SSSE3-NEXT:    cmpeqss %xmm1, %xmm0
-; X64-SSSE3-NEXT:    movss {{.*}}(%rip), %xmm2 # xmm2 = mem[0],zero,zero,zero
+; X64-SSSE3-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
 ; X64-SSSE3-NEXT:    andps %xmm0, %xmm2
 ; X64-SSSE3-NEXT:    andnps %xmm1, %xmm0
 ; X64-SSSE3-NEXT:    orps %xmm2, %xmm0
@@ -253,10 +253,10 @@ define float @PR43971_1(<8 x float> *%a0) nounwind {
 ;
 ; X64-AVX-LABEL: PR43971_1:
 ; X64-AVX:       # %bb.0: # %entry
-; X64-AVX-NEXT:    vmovss 4(%rdi), %xmm0 # xmm0 = mem[0],zero,zero,zero
+; X64-AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X64-AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
 ; X64-AVX-NEXT:    vcmpeqss %xmm1, %xmm0, %xmm1
-; X64-AVX-NEXT:    vmovss {{.*}}(%rip), %xmm2 # xmm2 = mem[0],zero,zero,zero
+; X64-AVX-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
 ; X64-AVX-NEXT:    vblendvps %xmm1, %xmm2, %xmm0, %xmm0
 ; X64-AVX-NEXT:    retq
 entry:
diff --git a/llvm/test/CodeGen/X86/gather-addresses.ll b/llvm/test/CodeGen/X86/gather-addresses.ll
index 1cd85e6e582c4..00f84a6e4b159 100644
--- a/llvm/test/CodeGen/X86/gather-addresses.ll
+++ b/llvm/test/CodeGen/X86/gather-addresses.ll
@@ -17,7 +17,7 @@ define <4 x double> @foo(double* %p, <4 x i32>* %i, <4 x i32>* %h) nounwind {
 ; LIN-SSE2-NEXT:    movd %xmm0, %eax
 ; LIN-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; LIN-SSE2-NEXT:    movd %xmm1, %ecx
-; LIN-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; LIN-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; LIN-SSE2-NEXT:    movd %xmm1, %edx
 ; LIN-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
 ; LIN-SSE2-NEXT:    movd %xmm0, %esi
@@ -56,7 +56,7 @@ define <4 x double> @foo(double* %p, <4 x i32>* %i, <4 x i32>* %h) nounwind {
 ; WIN-SSE2-NEXT:    movd %xmm0, %r8d
 ; WIN-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; WIN-SSE2-NEXT:    movd %xmm1, %r9d
-; WIN-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; WIN-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; WIN-SSE2-NEXT:    movd %xmm1, %r10d
 ; WIN-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
 ; WIN-SSE2-NEXT:    movd %xmm0, %edx
@@ -141,7 +141,7 @@ define <4 x i64> @old(double* %p, <4 x i32>* %i, <4 x i32>* %h, i64 %f) nounwind
 ; LIN-SSE2-NEXT:    movd %xmm0, %eax
 ; LIN-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; LIN-SSE2-NEXT:    movd %xmm1, %edx
-; LIN-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; LIN-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; LIN-SSE2-NEXT:    movd %xmm1, %esi
 ; LIN-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
 ; LIN-SSE2-NEXT:    movd %xmm0, %edi
@@ -184,7 +184,7 @@ define <4 x i64> @old(double* %p, <4 x i32>* %i, <4 x i32>* %h, i64 %f) nounwind
 ; WIN-SSE2-NEXT:    movd %xmm0, %eax
 ; WIN-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; WIN-SSE2-NEXT:    movd %xmm1, %ecx
-; WIN-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; WIN-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; WIN-SSE2-NEXT:    movd %xmm1, %r8d
 ; WIN-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
 ; WIN-SSE2-NEXT:    movd %xmm0, %edx
diff --git a/llvm/test/CodeGen/X86/haddsub-2.ll b/llvm/test/CodeGen/X86/haddsub-2.ll
index c4d470a6cd69e..e36c0479448ea 100644
--- a/llvm/test/CodeGen/X86/haddsub-2.ll
+++ b/llvm/test/CodeGen/X86/haddsub-2.ll
@@ -127,7 +127,7 @@ define <4 x i32> @phadd_d_test1(<4 x i32> %A, <4 x i32> %B) {
 ; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
 ; SSE3-NEXT:    movd %xmm2, %ecx
 ; SSE3-NEXT:    addl %eax, %ecx
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; SSE3-NEXT:    movd %xmm2, %eax
 ; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
 ; SSE3-NEXT:    movd %xmm0, %edx
@@ -136,7 +136,7 @@ define <4 x i32> @phadd_d_test1(<4 x i32> %A, <4 x i32> %B) {
 ; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE3-NEXT:    movd %xmm0, %esi
 ; SSE3-NEXT:    addl %eax, %esi
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE3-NEXT:    movd %xmm0, %eax
 ; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[3,1,2,3]
 ; SSE3-NEXT:    movd %xmm0, %edi
@@ -181,7 +181,7 @@ define <4 x i32> @phadd_d_test1(<4 x i32> %A, <4 x i32> %B) {
 define <4 x i32> @phadd_d_test2(<4 x i32> %A, <4 x i32> %B) {
 ; SSE3-LABEL: phadd_d_test2:
 ; SSE3:       # %bb.0:
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; SSE3-NEXT:    movd %xmm2, %eax
 ; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,1,2,3]
 ; SSE3-NEXT:    movd %xmm2, %ecx
@@ -192,7 +192,7 @@ define <4 x i32> @phadd_d_test2(<4 x i32> %A, <4 x i32> %B) {
 ; SSE3-NEXT:    addl %eax, %edx
 ; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[3,1,2,3]
 ; SSE3-NEXT:    movd %xmm0, %eax
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE3-NEXT:    movd %xmm0, %esi
 ; SSE3-NEXT:    addl %eax, %esi
 ; SSE3-NEXT:    movd %esi, %xmm0
@@ -243,7 +243,7 @@ define <4 x i32> @phsub_d_test1(<4 x i32> %A, <4 x i32> %B) {
 ; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
 ; SSE3-NEXT:    movd %xmm2, %ecx
 ; SSE3-NEXT:    subl %ecx, %eax
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; SSE3-NEXT:    movd %xmm2, %ecx
 ; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
 ; SSE3-NEXT:    movd %xmm0, %edx
@@ -252,7 +252,7 @@ define <4 x i32> @phsub_d_test1(<4 x i32> %A, <4 x i32> %B) {
 ; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE3-NEXT:    movd %xmm0, %esi
 ; SSE3-NEXT:    subl %esi, %edx
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE3-NEXT:    movd %xmm0, %esi
 ; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[3,1,2,3]
 ; SSE3-NEXT:    movd %xmm0, %edi
@@ -297,7 +297,7 @@ define <4 x i32> @phsub_d_test1(<4 x i32> %A, <4 x i32> %B) {
 define <4 x i32> @phsub_d_test2(<4 x i32> %A, <4 x i32> %B) {
 ; SSE3-LABEL: phsub_d_test2:
 ; SSE3:       # %bb.0:
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; SSE3-NEXT:    movd %xmm2, %eax
 ; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,1,2,3]
 ; SSE3-NEXT:    movd %xmm2, %ecx
@@ -306,7 +306,7 @@ define <4 x i32> @phsub_d_test2(<4 x i32> %A, <4 x i32> %B) {
 ; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
 ; SSE3-NEXT:    movd %xmm0, %edx
 ; SSE3-NEXT:    subl %edx, %ecx
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE3-NEXT:    movd %xmm0, %edx
 ; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[3,1,2,3]
 ; SSE3-NEXT:    movd %xmm0, %esi
@@ -513,7 +513,7 @@ define <8 x i32> @avx2_vphadd_d_test(<8 x i32> %A, <8 x i32> %B) {
 ; SSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,2,3]
 ; SSE3-NEXT:    movd %xmm4, %r8d
 ; SSE3-NEXT:    addl %ecx, %r8d
-; SSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[2,3,0,1]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
 ; SSE3-NEXT:    movd %xmm4, %edx
 ; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
 ; SSE3-NEXT:    movd %xmm0, %r9d
@@ -522,7 +522,7 @@ define <8 x i32> @avx2_vphadd_d_test(<8 x i32> %A, <8 x i32> %B) {
 ; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE3-NEXT:    movd %xmm0, %esi
 ; SSE3-NEXT:    addl %edx, %esi
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE3-NEXT:    movd %xmm0, %edx
 ; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[3,1,2,3]
 ; SSE3-NEXT:    movd %xmm0, %edi
@@ -531,7 +531,7 @@ define <8 x i32> @avx2_vphadd_d_test(<8 x i32> %A, <8 x i32> %B) {
 ; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
 ; SSE3-NEXT:    movd %xmm0, %r10d
 ; SSE3-NEXT:    addl %eax, %r10d
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE3-NEXT:    movd %xmm0, %eax
 ; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[3,1,2,3]
 ; SSE3-NEXT:    movd %xmm0, %ecx
@@ -540,7 +540,7 @@ define <8 x i32> @avx2_vphadd_d_test(<8 x i32> %A, <8 x i32> %B) {
 ; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,2,3]
 ; SSE3-NEXT:    movd %xmm0, %edx
 ; SSE3-NEXT:    addl %eax, %edx
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[2,3,0,1]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]
 ; SSE3-NEXT:    movd %xmm0, %r11d
 ; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[3,1,2,3]
 ; SSE3-NEXT:    movd %xmm0, %eax
@@ -819,7 +819,7 @@ define <4 x i32> @not_a_hsub_1(<4 x i32> %A, <4 x i32> %B) {
 ; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
 ; SSE-NEXT:    movd %xmm2, %ecx
 ; SSE-NEXT:    subl %ecx, %eax
-; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; SSE-NEXT:    movd %xmm2, %ecx
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
 ; SSE-NEXT:    movd %xmm0, %edx
@@ -830,7 +830,7 @@ define <4 x i32> @not_a_hsub_1(<4 x i32> %A, <4 x i32> %B) {
 ; SSE-NEXT:    subl %esi, %edx
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[3,1,2,3]
 ; SSE-NEXT:    movd %xmm0, %esi
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE-NEXT:    movd %xmm0, %edi
 ; SSE-NEXT:    subl %edi, %esi
 ; SSE-NEXT:    movd %esi, %xmm0
@@ -1133,7 +1133,7 @@ define <8 x i32> @avx2_hadd_d(<8 x i32> %a, <8 x i32> %b) {
 ; SSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,2,3]
 ; SSE3-NEXT:    movd %xmm4, %r8d
 ; SSE3-NEXT:    addl %ecx, %r8d
-; SSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[2,3,0,1]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
 ; SSE3-NEXT:    movd %xmm4, %edx
 ; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
 ; SSE3-NEXT:    movd %xmm0, %r9d
@@ -1142,7 +1142,7 @@ define <8 x i32> @avx2_hadd_d(<8 x i32> %a, <8 x i32> %b) {
 ; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
 ; SSE3-NEXT:    movd %xmm0, %esi
 ; SSE3-NEXT:    addl %edx, %esi
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE3-NEXT:    movd %xmm0, %edx
 ; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[3,1,2,3]
 ; SSE3-NEXT:    movd %xmm0, %edi
@@ -1151,7 +1151,7 @@ define <8 x i32> @avx2_hadd_d(<8 x i32> %a, <8 x i32> %b) {
 ; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE3-NEXT:    movd %xmm0, %r10d
 ; SSE3-NEXT:    addl %eax, %r10d
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE3-NEXT:    movd %xmm0, %eax
 ; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[3,1,2,3]
 ; SSE3-NEXT:    movd %xmm0, %ecx
@@ -1160,7 +1160,7 @@ define <8 x i32> @avx2_hadd_d(<8 x i32> %a, <8 x i32> %b) {
 ; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,2,3]
 ; SSE3-NEXT:    movd %xmm0, %edx
 ; SSE3-NEXT:    addl %eax, %edx
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[2,3,0,1]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]
 ; SSE3-NEXT:    movd %xmm0, %r11d
 ; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[3,1,2,3]
 ; SSE3-NEXT:    movd %xmm0, %eax
diff --git a/llvm/test/CodeGen/X86/hoist-and-by-const-from-lshr-in-eqcmp-zero.ll b/llvm/test/CodeGen/X86/hoist-and-by-const-from-lshr-in-eqcmp-zero.ll
index 6663459f49d56..740e4b291f5f9 100644
--- a/llvm/test/CodeGen/X86/hoist-and-by-const-from-lshr-in-eqcmp-zero.ll
+++ b/llvm/test/CodeGen/X86/hoist-and-by-const-from-lshr-in-eqcmp-zero.ll
@@ -554,7 +554,7 @@ define <4 x i1> @vec_4xi32_nonsplat_eq(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; SSE2-NEXT:    movdqa %xmm3, %xmm5
 ; SSE2-NEXT:    psrld %xmm2, %xmm5
 ; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm4[0]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm1[2,3,3,3,4,5,6,7]
 ; SSE2-NEXT:    movdqa %xmm3, %xmm4
 ; SSE2-NEXT:    psrld %xmm2, %xmm4
@@ -640,7 +640,7 @@ define <4 x i1> @vec_4xi32_nonsplat_undef1_eq(<4 x i32> %x, <4 x i32> %y) nounwi
 ; SSE2-NEXT:    movdqa %xmm3, %xmm5
 ; SSE2-NEXT:    psrld %xmm2, %xmm5
 ; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm4[0]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm1[2,3,3,3,4,5,6,7]
 ; SSE2-NEXT:    movdqa %xmm3, %xmm4
 ; SSE2-NEXT:    psrld %xmm2, %xmm4
@@ -677,7 +677,7 @@ define <4 x i1> @vec_4xi32_nonsplat_undef2_eq(<4 x i32> %x, <4 x i32> %y) nounwi
 ; SSE2-NEXT:    movdqa %xmm3, %xmm5
 ; SSE2-NEXT:    psrld %xmm2, %xmm5
 ; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm4[0]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm1[2,3,3,3,4,5,6,7]
 ; SSE2-NEXT:    movdqa %xmm3, %xmm4
 ; SSE2-NEXT:    psrld %xmm2, %xmm4
diff --git a/llvm/test/CodeGen/X86/horizontal-reduce-add.ll b/llvm/test/CodeGen/X86/horizontal-reduce-add.ll
index 64d8de9aead78..dab7785c85cc1 100644
--- a/llvm/test/CodeGen/X86/horizontal-reduce-add.ll
+++ b/llvm/test/CodeGen/X86/horizontal-reduce-add.ll
@@ -11,7 +11,7 @@
 define i32 @PR37890_v4i32(<4 x i32> %a)  {
 ; SSE2-LABEL: PR37890_v4i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE2-NEXT:    paddd %xmm1, %xmm0
@@ -20,7 +20,7 @@ define i32 @PR37890_v4i32(<4 x i32> %a)  {
 ;
 ; SSSE3-SLOW-LABEL: PR37890_v4i32:
 ; SSSE3-SLOW:       # %bb.0:
-; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSSE3-SLOW-NEXT:    paddd %xmm0, %xmm1
 ; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSSE3-SLOW-NEXT:    paddd %xmm1, %xmm0
@@ -36,7 +36,7 @@ define i32 @PR37890_v4i32(<4 x i32> %a)  {
 ;
 ; AVX1-SLOW-LABEL: PR37890_v4i32:
 ; AVX1-SLOW:       # %bb.0:
-; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -52,7 +52,7 @@ define i32 @PR37890_v4i32(<4 x i32> %a)  {
 ;
 ; AVX2-LABEL: PR37890_v4i32:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -70,7 +70,7 @@ define i32 @PR37890_v4i32(<4 x i32> %a)  {
 define i16 @PR37890_v8i16(<8 x i16> %a)  {
 ; SSE2-LABEL: PR37890_v8i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    paddw %xmm0, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE2-NEXT:    paddw %xmm1, %xmm0
@@ -83,7 +83,7 @@ define i16 @PR37890_v8i16(<8 x i16> %a)  {
 ;
 ; SSSE3-SLOW-LABEL: PR37890_v8i16:
 ; SSSE3-SLOW:       # %bb.0:
-; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSSE3-SLOW-NEXT:    paddw %xmm0, %xmm1
 ; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSSE3-SLOW-NEXT:    paddw %xmm1, %xmm0
@@ -105,7 +105,7 @@ define i16 @PR37890_v8i16(<8 x i16> %a)  {
 ;
 ; AVX1-SLOW-LABEL: PR37890_v8i16:
 ; AVX1-SLOW:       # %bb.0:
-; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-SLOW-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-SLOW-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
@@ -126,7 +126,7 @@ define i16 @PR37890_v8i16(<8 x i16> %a)  {
 ;
 ; AVX2-LABEL: PR37890_v8i16:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
@@ -151,7 +151,7 @@ define i32 @PR37890_v8i32(<8 x i32> %a)  {
 ; SSE2-LABEL: PR37890_v8i32:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    paddd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE2-NEXT:    paddd %xmm1, %xmm0
@@ -161,7 +161,7 @@ define i32 @PR37890_v8i32(<8 x i32> %a)  {
 ; SSSE3-SLOW-LABEL: PR37890_v8i32:
 ; SSSE3-SLOW:       # %bb.0:
 ; SSSE3-SLOW-NEXT:    paddd %xmm1, %xmm0
-; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSSE3-SLOW-NEXT:    paddd %xmm0, %xmm1
 ; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSSE3-SLOW-NEXT:    paddd %xmm1, %xmm0
@@ -180,7 +180,7 @@ define i32 @PR37890_v8i32(<8 x i32> %a)  {
 ; AVX1-SLOW:       # %bb.0:
 ; AVX1-SLOW-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -202,7 +202,7 @@ define i32 @PR37890_v8i32(<8 x i32> %a)  {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -225,7 +225,7 @@ define i16 @PR37890_v16i16(<16 x i16> %a)  {
 ; SSE2-LABEL: PR37890_v16i16:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    paddw %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    paddw %xmm0, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE2-NEXT:    paddw %xmm1, %xmm0
@@ -239,7 +239,7 @@ define i16 @PR37890_v16i16(<16 x i16> %a)  {
 ; SSSE3-SLOW-LABEL: PR37890_v16i16:
 ; SSSE3-SLOW:       # %bb.0:
 ; SSSE3-SLOW-NEXT:    paddw %xmm1, %xmm0
-; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSSE3-SLOW-NEXT:    paddw %xmm0, %xmm1
 ; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSSE3-SLOW-NEXT:    paddw %xmm1, %xmm0
@@ -264,7 +264,7 @@ define i16 @PR37890_v16i16(<16 x i16> %a)  {
 ; AVX1-SLOW:       # %bb.0:
 ; AVX1-SLOW-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-SLOW-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
-; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-SLOW-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-SLOW-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
@@ -291,7 +291,7 @@ define i16 @PR37890_v16i16(<16 x i16> %a)  {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
@@ -322,7 +322,7 @@ define i32 @PR37890_v16i32(<16 x i32> %a)  {
 ; SSE2-NEXT:    paddd %xmm3, %xmm1
 ; SSE2-NEXT:    paddd %xmm2, %xmm1
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    paddd %xmm1, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
@@ -334,7 +334,7 @@ define i32 @PR37890_v16i32(<16 x i32> %a)  {
 ; SSSE3-SLOW-NEXT:    paddd %xmm3, %xmm1
 ; SSSE3-SLOW-NEXT:    paddd %xmm2, %xmm1
 ; SSSE3-SLOW-NEXT:    paddd %xmm0, %xmm1
-; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSSE3-SLOW-NEXT:    paddd %xmm1, %xmm0
 ; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSSE3-SLOW-NEXT:    paddd %xmm0, %xmm1
@@ -346,7 +346,7 @@ define i32 @PR37890_v16i32(<16 x i32> %a)  {
 ; SSSE3-FAST-NEXT:    paddd %xmm3, %xmm1
 ; SSSE3-FAST-NEXT:    paddd %xmm2, %xmm1
 ; SSSE3-FAST-NEXT:    paddd %xmm0, %xmm1
-; SSSE3-FAST-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSSE3-FAST-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSSE3-FAST-NEXT:    paddd %xmm1, %xmm0
 ; SSSE3-FAST-NEXT:    phaddd %xmm0, %xmm0
 ; SSSE3-FAST-NEXT:    movd %xmm0, %eax
@@ -359,7 +359,7 @@ define i32 @PR37890_v16i32(<16 x i32> %a)  {
 ; AVX1-SLOW-NEXT:    vpaddd %xmm2, %xmm3, %xmm2
 ; AVX1-SLOW-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
 ; AVX1-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -385,7 +385,7 @@ define i32 @PR37890_v16i32(<16 x i32> %a)  {
 ; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
diff --git a/llvm/test/CodeGen/X86/horizontal-reduce-smax.ll b/llvm/test/CodeGen/X86/horizontal-reduce-smax.ll
index dc8f60248c672..a17b1db2c1783 100644
--- a/llvm/test/CodeGen/X86/horizontal-reduce-smax.ll
+++ b/llvm/test/CodeGen/X86/horizontal-reduce-smax.ll
@@ -16,7 +16,7 @@
 define i64 @test_reduce_v2i64(<2 x i64> %a0) {
 ; X86-SSE2-LABEL: test_reduce_v2i64:
 ; X86-SSE2:       ## %bb.0:
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,0,2147483648,0]
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm3
 ; X86-SSE2-NEXT:    pxor %xmm2, %xmm3
@@ -40,7 +40,7 @@ define i64 @test_reduce_v2i64(<2 x i64> %a0) {
 ; X86-SSE42-LABEL: test_reduce_v2i64:
 ; X86-SSE42:       ## %bb.0:
 ; X86-SSE42-NEXT:    movdqa %xmm0, %xmm1
-; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; X86-SSE42-NEXT:    pcmpgtq %xmm2, %xmm0
 ; X86-SSE42-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
 ; X86-SSE42-NEXT:    movd %xmm2, %eax
@@ -49,7 +49,7 @@ define i64 @test_reduce_v2i64(<2 x i64> %a0) {
 ;
 ; X86-AVX-LABEL: test_reduce_v2i64:
 ; X86-AVX:       ## %bb.0:
-; X86-AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-AVX-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm2
 ; X86-AVX-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
 ; X86-AVX-NEXT:    vmovd %xmm0, %eax
@@ -58,7 +58,7 @@ define i64 @test_reduce_v2i64(<2 x i64> %a0) {
 ;
 ; X64-SSE2-LABEL: test_reduce_v2i64:
 ; X64-SSE2:       ## %bb.0:
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
 ; X64-SSE2-NEXT:    movdqa %xmm0, %xmm3
 ; X64-SSE2-NEXT:    pxor %xmm2, %xmm3
@@ -80,7 +80,7 @@ define i64 @test_reduce_v2i64(<2 x i64> %a0) {
 ; X64-SSE42-LABEL: test_reduce_v2i64:
 ; X64-SSE42:       ## %bb.0:
 ; X64-SSE42-NEXT:    movdqa %xmm0, %xmm1
-; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; X64-SSE42-NEXT:    pcmpgtq %xmm2, %xmm0
 ; X64-SSE42-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
 ; X64-SSE42-NEXT:    movq %xmm2, %rax
@@ -88,7 +88,7 @@ define i64 @test_reduce_v2i64(<2 x i64> %a0) {
 ;
 ; X64-AVX1-LABEL: test_reduce_v2i64:
 ; X64-AVX1:       ## %bb.0:
-; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX1-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm2
 ; X64-AVX1-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
 ; X64-AVX1-NEXT:    vmovq %xmm0, %rax
@@ -96,7 +96,7 @@ define i64 @test_reduce_v2i64(<2 x i64> %a0) {
 ;
 ; X64-AVX2-LABEL: test_reduce_v2i64:
 ; X64-AVX2:       ## %bb.0:
-; X64-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX2-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm2
 ; X64-AVX2-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
 ; X64-AVX2-NEXT:    vmovq %xmm0, %rax
@@ -104,7 +104,7 @@ define i64 @test_reduce_v2i64(<2 x i64> %a0) {
 ;
 ; X64-AVX512-LABEL: test_reduce_v2i64:
 ; X64-AVX512:       ## %bb.0:
-; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX512-NEXT:    vpmaxsq %xmm1, %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vmovq %xmm0, %rax
 ; X64-AVX512-NEXT:    retq
@@ -118,7 +118,7 @@ define i64 @test_reduce_v2i64(<2 x i64> %a0) {
 define i32 @test_reduce_v4i32(<4 x i32> %a0) {
 ; X86-SSE2-LABEL: test_reduce_v4i32:
 ; X86-SSE2:       ## %bb.0:
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; X86-SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
 ; X86-SSE2-NEXT:    pand %xmm2, %xmm0
@@ -135,7 +135,7 @@ define i32 @test_reduce_v4i32(<4 x i32> %a0) {
 ;
 ; X86-SSE42-LABEL: test_reduce_v4i32:
 ; X86-SSE42:       ## %bb.0:
-; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE42-NEXT:    pmaxsd %xmm0, %xmm1
 ; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; X86-SSE42-NEXT:    pmaxsd %xmm1, %xmm0
@@ -144,7 +144,7 @@ define i32 @test_reduce_v4i32(<4 x i32> %a0) {
 ;
 ; X86-AVX-LABEL: test_reduce_v4i32:
 ; X86-AVX:       ## %bb.0:
-; X86-AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-AVX-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X86-AVX-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
@@ -153,7 +153,7 @@ define i32 @test_reduce_v4i32(<4 x i32> %a0) {
 ;
 ; X64-SSE2-LABEL: test_reduce_v4i32:
 ; X64-SSE2:       ## %bb.0:
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; X64-SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
 ; X64-SSE2-NEXT:    pand %xmm2, %xmm0
@@ -170,7 +170,7 @@ define i32 @test_reduce_v4i32(<4 x i32> %a0) {
 ;
 ; X64-SSE42-LABEL: test_reduce_v4i32:
 ; X64-SSE42:       ## %bb.0:
-; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE42-NEXT:    pmaxsd %xmm0, %xmm1
 ; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; X64-SSE42-NEXT:    pmaxsd %xmm1, %xmm0
@@ -179,7 +179,7 @@ define i32 @test_reduce_v4i32(<4 x i32> %a0) {
 ;
 ; X64-AVX-LABEL: test_reduce_v4i32:
 ; X64-AVX:       ## %bb.0:
-; X64-AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; X64-AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X64-AVX-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
@@ -198,7 +198,7 @@ define i32 @test_reduce_v4i32(<4 x i32> %a0) {
 define i16 @test_reduce_v8i16(<8 x i16> %a0) {
 ; X86-SSE2-LABEL: test_reduce_v8i16:
 ; X86-SSE2:       ## %bb.0:
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    pmaxsw %xmm0, %xmm1
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; X86-SSE2-NEXT:    pmaxsw %xmm1, %xmm0
@@ -229,7 +229,7 @@ define i16 @test_reduce_v8i16(<8 x i16> %a0) {
 ;
 ; X64-SSE2-LABEL: test_reduce_v8i16:
 ; X64-SSE2:       ## %bb.0:
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    pmaxsw %xmm0, %xmm1
 ; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; X64-SSE2-NEXT:    pmaxsw %xmm1, %xmm0
@@ -273,7 +273,7 @@ define i16 @test_reduce_v8i16(<8 x i16> %a0) {
 define i8 @test_reduce_v16i8(<16 x i8> %a0) {
 ; X86-SSE2-LABEL: test_reduce_v16i8:
 ; X86-SSE2:       ## %bb.0:
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; X86-SSE2-NEXT:    pcmpgtb %xmm1, %xmm2
 ; X86-SSE2-NEXT:    pand %xmm2, %xmm0
@@ -328,7 +328,7 @@ define i8 @test_reduce_v16i8(<16 x i8> %a0) {
 ;
 ; X64-SSE2-LABEL: test_reduce_v16i8:
 ; X64-SSE2:       ## %bb.0:
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; X64-SSE2-NEXT:    pcmpgtb %xmm1, %xmm2
 ; X64-SSE2-NEXT:    pand %xmm2, %xmm0
@@ -419,7 +419,7 @@ define i64 @test_reduce_v4i64(<4 x i64> %a0) {
 ; X86-SSE2-NEXT:    pand %xmm4, %xmm0
 ; X86-SSE2-NEXT:    pandn %xmm1, %xmm4
 ; X86-SSE2-NEXT:    por %xmm0, %xmm4
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[2,3,2,3]
 ; X86-SSE2-NEXT:    movdqa %xmm4, %xmm1
 ; X86-SSE2-NEXT:    pxor %xmm2, %xmm1
 ; X86-SSE2-NEXT:    pxor %xmm0, %xmm2
@@ -444,7 +444,7 @@ define i64 @test_reduce_v4i64(<4 x i64> %a0) {
 ; X86-SSE42-NEXT:    movdqa %xmm0, %xmm2
 ; X86-SSE42-NEXT:    pcmpgtq %xmm1, %xmm0
 ; X86-SSE42-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
-; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; X86-SSE42-NEXT:    movdqa %xmm1, %xmm0
 ; X86-SSE42-NEXT:    pcmpgtq %xmm2, %xmm0
 ; X86-SSE42-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
@@ -457,7 +457,7 @@ define i64 @test_reduce_v4i64(<4 x i64> %a0) {
 ; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; X86-AVX1-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm2
 ; X86-AVX1-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
-; X86-AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-AVX1-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm2
 ; X86-AVX1-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
 ; X86-AVX1-NEXT:    vmovd %xmm0, %eax
@@ -470,7 +470,7 @@ define i64 @test_reduce_v4i64(<4 x i64> %a0) {
 ; X86-AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X86-AVX2-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm2
 ; X86-AVX2-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
-; X86-AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-AVX2-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm2
 ; X86-AVX2-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
 ; X86-AVX2-NEXT:    vmovd %xmm0, %eax
@@ -496,7 +496,7 @@ define i64 @test_reduce_v4i64(<4 x i64> %a0) {
 ; X64-SSE2-NEXT:    pand %xmm4, %xmm0
 ; X64-SSE2-NEXT:    pandn %xmm1, %xmm4
 ; X64-SSE2-NEXT:    por %xmm0, %xmm4
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[2,3,2,3]
 ; X64-SSE2-NEXT:    movdqa %xmm4, %xmm1
 ; X64-SSE2-NEXT:    pxor %xmm2, %xmm1
 ; X64-SSE2-NEXT:    pxor %xmm0, %xmm2
@@ -519,7 +519,7 @@ define i64 @test_reduce_v4i64(<4 x i64> %a0) {
 ; X64-SSE42-NEXT:    movdqa %xmm0, %xmm2
 ; X64-SSE42-NEXT:    pcmpgtq %xmm1, %xmm0
 ; X64-SSE42-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
-; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; X64-SSE42-NEXT:    movdqa %xmm1, %xmm0
 ; X64-SSE42-NEXT:    pcmpgtq %xmm2, %xmm0
 ; X64-SSE42-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
@@ -531,7 +531,7 @@ define i64 @test_reduce_v4i64(<4 x i64> %a0) {
 ; X64-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; X64-AVX1-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm2
 ; X64-AVX1-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
-; X64-AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX1-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm2
 ; X64-AVX1-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
 ; X64-AVX1-NEXT:    vmovq %xmm0, %rax
@@ -543,7 +543,7 @@ define i64 @test_reduce_v4i64(<4 x i64> %a0) {
 ; X64-AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X64-AVX2-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm2
 ; X64-AVX2-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
-; X64-AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX2-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm2
 ; X64-AVX2-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
 ; X64-AVX2-NEXT:    vmovq %xmm0, %rax
@@ -554,7 +554,7 @@ define i64 @test_reduce_v4i64(<4 x i64> %a0) {
 ; X64-AVX512:       ## %bb.0:
 ; X64-AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X64-AVX512-NEXT:    vpmaxsq %xmm1, %xmm0, %xmm0
-; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX512-NEXT:    vpmaxsq %xmm1, %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vmovq %xmm0, %rax
 ; X64-AVX512-NEXT:    vzeroupper
@@ -577,7 +577,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X86-SSE2-NEXT:    pand %xmm2, %xmm0
 ; X86-SSE2-NEXT:    pandn %xmm1, %xmm2
 ; X86-SSE2-NEXT:    por %xmm0, %xmm2
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; X86-SSE2-NEXT:    movdqa %xmm2, %xmm1
 ; X86-SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
 ; X86-SSE2-NEXT:    pand %xmm1, %xmm2
@@ -595,7 +595,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X86-SSE42-LABEL: test_reduce_v8i32:
 ; X86-SSE42:       ## %bb.0:
 ; X86-SSE42-NEXT:    pmaxsd %xmm1, %xmm0
-; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE42-NEXT:    pmaxsd %xmm0, %xmm1
 ; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; X86-SSE42-NEXT:    pmaxsd %xmm1, %xmm0
@@ -606,7 +606,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X86-AVX1:       ## %bb.0:
 ; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; X86-AVX1-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
-; X86-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-AVX1-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X86-AVX1-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
@@ -618,7 +618,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X86-AVX2:       ## %bb.0:
 ; X86-AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X86-AVX2-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
-; X86-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-AVX2-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; X86-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X86-AVX2-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
@@ -633,7 +633,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X64-SSE2-NEXT:    pand %xmm2, %xmm0
 ; X64-SSE2-NEXT:    pandn %xmm1, %xmm2
 ; X64-SSE2-NEXT:    por %xmm0, %xmm2
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; X64-SSE2-NEXT:    movdqa %xmm2, %xmm1
 ; X64-SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
 ; X64-SSE2-NEXT:    pand %xmm1, %xmm2
@@ -651,7 +651,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X64-SSE42-LABEL: test_reduce_v8i32:
 ; X64-SSE42:       ## %bb.0:
 ; X64-SSE42-NEXT:    pmaxsd %xmm1, %xmm0
-; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE42-NEXT:    pmaxsd %xmm0, %xmm1
 ; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; X64-SSE42-NEXT:    pmaxsd %xmm1, %xmm0
@@ -662,7 +662,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X64-AVX1:       ## %bb.0:
 ; X64-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; X64-AVX1-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX1-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X64-AVX1-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
@@ -674,7 +674,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X64-AVX2:       ## %bb.0:
 ; X64-AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X64-AVX2-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
-; X64-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX2-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; X64-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X64-AVX2-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
@@ -686,7 +686,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X64-AVX512:       ## %bb.0:
 ; X64-AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X64-AVX512-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
-; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX512-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X64-AVX512-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
@@ -710,7 +710,7 @@ define i16 @test_reduce_v16i16(<16 x i16> %a0) {
 ; X86-SSE2-LABEL: test_reduce_v16i16:
 ; X86-SSE2:       ## %bb.0:
 ; X86-SSE2-NEXT:    pmaxsw %xmm1, %xmm0
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    pmaxsw %xmm0, %xmm1
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; X86-SSE2-NEXT:    pmaxsw %xmm1, %xmm0
@@ -758,7 +758,7 @@ define i16 @test_reduce_v16i16(<16 x i16> %a0) {
 ; X64-SSE2-LABEL: test_reduce_v16i16:
 ; X64-SSE2:       ## %bb.0:
 ; X64-SSE2-NEXT:    pmaxsw %xmm1, %xmm0
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    pmaxsw %xmm0, %xmm1
 ; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; X64-SSE2-NEXT:    pmaxsw %xmm1, %xmm0
@@ -838,7 +838,7 @@ define i8 @test_reduce_v32i8(<32 x i8> %a0) {
 ; X86-SSE2-NEXT:    pand %xmm2, %xmm0
 ; X86-SSE2-NEXT:    pandn %xmm1, %xmm2
 ; X86-SSE2-NEXT:    por %xmm0, %xmm2
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; X86-SSE2-NEXT:    movdqa %xmm2, %xmm1
 ; X86-SSE2-NEXT:    pcmpgtb %xmm0, %xmm1
 ; X86-SSE2-NEXT:    pand %xmm1, %xmm2
@@ -916,7 +916,7 @@ define i8 @test_reduce_v32i8(<32 x i8> %a0) {
 ; X64-SSE2-NEXT:    pand %xmm2, %xmm0
 ; X64-SSE2-NEXT:    pandn %xmm1, %xmm2
 ; X64-SSE2-NEXT:    por %xmm0, %xmm2
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; X64-SSE2-NEXT:    movdqa %xmm2, %xmm1
 ; X64-SSE2-NEXT:    pcmpgtb %xmm0, %xmm1
 ; X64-SSE2-NEXT:    pand %xmm1, %xmm2
@@ -1072,7 +1072,7 @@ define i64 @test_reduce_v8i64(<8 x i64> %a0) {
 ; X86-SSE2-NEXT:    pand %xmm1, %xmm5
 ; X86-SSE2-NEXT:    pandn %xmm0, %xmm1
 ; X86-SSE2-NEXT:    por %xmm5, %xmm1
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; X86-SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; X86-SSE2-NEXT:    pxor %xmm4, %xmm2
 ; X86-SSE2-NEXT:    pxor %xmm0, %xmm4
@@ -1104,7 +1104,7 @@ define i64 @test_reduce_v8i64(<8 x i64> %a0) {
 ; X86-SSE42-NEXT:    movapd %xmm2, %xmm0
 ; X86-SSE42-NEXT:    pcmpgtq %xmm3, %xmm0
 ; X86-SSE42-NEXT:    blendvpd %xmm0, %xmm2, %xmm3
-; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[2,3,0,1]
+; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[2,3,2,3]
 ; X86-SSE42-NEXT:    movdqa %xmm3, %xmm0
 ; X86-SSE42-NEXT:    pcmpgtq %xmm1, %xmm0
 ; X86-SSE42-NEXT:    blendvpd %xmm0, %xmm3, %xmm1
@@ -1122,7 +1122,7 @@ define i64 @test_reduce_v8i64(<8 x i64> %a0) {
 ; X86-AVX1-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
 ; X86-AVX1-NEXT:    vpcmpgtq %xmm3, %xmm0, %xmm1
 ; X86-AVX1-NEXT:    vblendvpd %xmm1, %xmm0, %xmm3, %xmm0
-; X86-AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-AVX1-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm2
 ; X86-AVX1-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
 ; X86-AVX1-NEXT:    vmovd %xmm0, %eax
@@ -1137,7 +1137,7 @@ define i64 @test_reduce_v8i64(<8 x i64> %a0) {
 ; X86-AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; X86-AVX2-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm2
 ; X86-AVX2-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
-; X86-AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-AVX2-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm2
 ; X86-AVX2-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
 ; X86-AVX2-NEXT:    vmovd %xmm0, %eax
@@ -1193,7 +1193,7 @@ define i64 @test_reduce_v8i64(<8 x i64> %a0) {
 ; X64-SSE2-NEXT:    pand %xmm1, %xmm5
 ; X64-SSE2-NEXT:    pandn %xmm2, %xmm1
 ; X64-SSE2-NEXT:    por %xmm5, %xmm1
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; X64-SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; X64-SSE2-NEXT:    pxor %xmm4, %xmm2
 ; X64-SSE2-NEXT:    pxor %xmm0, %xmm4
@@ -1223,7 +1223,7 @@ define i64 @test_reduce_v8i64(<8 x i64> %a0) {
 ; X64-SSE42-NEXT:    movapd %xmm2, %xmm0
 ; X64-SSE42-NEXT:    pcmpgtq %xmm3, %xmm0
 ; X64-SSE42-NEXT:    blendvpd %xmm0, %xmm2, %xmm3
-; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[2,3,0,1]
+; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[2,3,2,3]
 ; X64-SSE42-NEXT:    movdqa %xmm3, %xmm0
 ; X64-SSE42-NEXT:    pcmpgtq %xmm1, %xmm0
 ; X64-SSE42-NEXT:    blendvpd %xmm0, %xmm3, %xmm1
@@ -1240,7 +1240,7 @@ define i64 @test_reduce_v8i64(<8 x i64> %a0) {
 ; X64-AVX1-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
 ; X64-AVX1-NEXT:    vpcmpgtq %xmm3, %xmm0, %xmm1
 ; X64-AVX1-NEXT:    vblendvpd %xmm1, %xmm0, %xmm3, %xmm0
-; X64-AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX1-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm2
 ; X64-AVX1-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
 ; X64-AVX1-NEXT:    vmovq %xmm0, %rax
@@ -1254,7 +1254,7 @@ define i64 @test_reduce_v8i64(<8 x i64> %a0) {
 ; X64-AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; X64-AVX2-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm2
 ; X64-AVX2-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
-; X64-AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX2-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm2
 ; X64-AVX2-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
 ; X64-AVX2-NEXT:    vmovq %xmm0, %rax
@@ -1267,7 +1267,7 @@ define i64 @test_reduce_v8i64(<8 x i64> %a0) {
 ; X64-AVX512-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
 ; X64-AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X64-AVX512-NEXT:    vpmaxsq %xmm1, %xmm0, %xmm0
-; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX512-NEXT:    vpmaxsq %xmm1, %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vmovq %xmm0, %rax
 ; X64-AVX512-NEXT:    vzeroupper
@@ -1303,7 +1303,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X86-SSE2-NEXT:    pand %xmm0, %xmm1
 ; X86-SSE2-NEXT:    pandn %xmm4, %xmm0
 ; X86-SSE2-NEXT:    por %xmm1, %xmm0
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; X86-SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
 ; X86-SSE2-NEXT:    pand %xmm2, %xmm0
@@ -1323,7 +1323,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X86-SSE42-NEXT:    pmaxsd %xmm3, %xmm1
 ; X86-SSE42-NEXT:    pmaxsd %xmm2, %xmm1
 ; X86-SSE42-NEXT:    pmaxsd %xmm0, %xmm1
-; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; X86-SSE42-NEXT:    pmaxsd %xmm1, %xmm0
 ; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X86-SSE42-NEXT:    pmaxsd %xmm0, %xmm1
@@ -1337,7 +1337,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X86-AVX1-NEXT:    vpmaxsd %xmm2, %xmm3, %xmm2
 ; X86-AVX1-NEXT:    vpmaxsd %xmm2, %xmm1, %xmm1
 ; X86-AVX1-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
-; X86-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-AVX1-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X86-AVX1-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
@@ -1350,7 +1350,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X86-AVX2-NEXT:    vpmaxsd %ymm1, %ymm0, %ymm0
 ; X86-AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X86-AVX2-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
-; X86-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-AVX2-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; X86-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X86-AVX2-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
@@ -1375,7 +1375,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X64-SSE2-NEXT:    pand %xmm0, %xmm1
 ; X64-SSE2-NEXT:    pandn %xmm4, %xmm0
 ; X64-SSE2-NEXT:    por %xmm1, %xmm0
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; X64-SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
 ; X64-SSE2-NEXT:    pand %xmm2, %xmm0
@@ -1395,7 +1395,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X64-SSE42-NEXT:    pmaxsd %xmm3, %xmm1
 ; X64-SSE42-NEXT:    pmaxsd %xmm2, %xmm1
 ; X64-SSE42-NEXT:    pmaxsd %xmm0, %xmm1
-; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; X64-SSE42-NEXT:    pmaxsd %xmm1, %xmm0
 ; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X64-SSE42-NEXT:    pmaxsd %xmm0, %xmm1
@@ -1409,7 +1409,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X64-AVX1-NEXT:    vpmaxsd %xmm2, %xmm3, %xmm2
 ; X64-AVX1-NEXT:    vpmaxsd %xmm2, %xmm1, %xmm1
 ; X64-AVX1-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX1-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X64-AVX1-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
@@ -1422,7 +1422,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X64-AVX2-NEXT:    vpmaxsd %ymm1, %ymm0, %ymm0
 ; X64-AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X64-AVX2-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
-; X64-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX2-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; X64-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X64-AVX2-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
@@ -1436,7 +1436,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X64-AVX512-NEXT:    vpmaxsd %zmm1, %zmm0, %zmm0
 ; X64-AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X64-AVX512-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
-; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX512-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X64-AVX512-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
@@ -1465,7 +1465,7 @@ define i16 @test_reduce_v32i16(<32 x i16> %a0) {
 ; X86-SSE2-NEXT:    pmaxsw %xmm3, %xmm1
 ; X86-SSE2-NEXT:    pmaxsw %xmm2, %xmm1
 ; X86-SSE2-NEXT:    pmaxsw %xmm0, %xmm1
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; X86-SSE2-NEXT:    pmaxsw %xmm1, %xmm0
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X86-SSE2-NEXT:    pmaxsw %xmm0, %xmm1
@@ -1521,7 +1521,7 @@ define i16 @test_reduce_v32i16(<32 x i16> %a0) {
 ; X64-SSE2-NEXT:    pmaxsw %xmm3, %xmm1
 ; X64-SSE2-NEXT:    pmaxsw %xmm2, %xmm1
 ; X64-SSE2-NEXT:    pmaxsw %xmm0, %xmm1
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; X64-SSE2-NEXT:    pmaxsw %xmm1, %xmm0
 ; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X64-SSE2-NEXT:    pmaxsw %xmm0, %xmm1
@@ -1622,7 +1622,7 @@ define i8 @test_reduce_v64i8(<64 x i8> %a0) {
 ; X86-SSE2-NEXT:    pand %xmm0, %xmm1
 ; X86-SSE2-NEXT:    pandn %xmm4, %xmm0
 ; X86-SSE2-NEXT:    por %xmm1, %xmm0
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; X86-SSE2-NEXT:    pcmpgtb %xmm1, %xmm2
 ; X86-SSE2-NEXT:    pand %xmm2, %xmm0
@@ -1716,7 +1716,7 @@ define i8 @test_reduce_v64i8(<64 x i8> %a0) {
 ; X64-SSE2-NEXT:    pand %xmm0, %xmm1
 ; X64-SSE2-NEXT:    pandn %xmm4, %xmm0
 ; X64-SSE2-NEXT:    por %xmm1, %xmm0
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; X64-SSE2-NEXT:    pcmpgtb %xmm1, %xmm2
 ; X64-SSE2-NEXT:    pand %xmm2, %xmm0
@@ -1837,7 +1837,7 @@ define i8 @test_reduce_v64i8(<64 x i8> %a0) {
 define i16 @test_reduce_v16i16_v8i16(<16 x i16> %a0) {
 ; X86-SSE2-LABEL: test_reduce_v16i16_v8i16:
 ; X86-SSE2:       ## %bb.0:
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    pmaxsw %xmm0, %xmm1
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; X86-SSE2-NEXT:    pmaxsw %xmm1, %xmm0
@@ -1869,7 +1869,7 @@ define i16 @test_reduce_v16i16_v8i16(<16 x i16> %a0) {
 ;
 ; X64-SSE2-LABEL: test_reduce_v16i16_v8i16:
 ; X64-SSE2:       ## %bb.0:
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    pmaxsw %xmm0, %xmm1
 ; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; X64-SSE2-NEXT:    pmaxsw %xmm1, %xmm0
@@ -1914,7 +1914,7 @@ define i16 @test_reduce_v16i16_v8i16(<16 x i16> %a0) {
 define i16 @test_reduce_v32i16_v8i16(<32 x i16> %a0) {
 ; X86-SSE2-LABEL: test_reduce_v32i16_v8i16:
 ; X86-SSE2:       ## %bb.0:
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    pmaxsw %xmm0, %xmm1
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; X86-SSE2-NEXT:    pmaxsw %xmm1, %xmm0
@@ -1946,7 +1946,7 @@ define i16 @test_reduce_v32i16_v8i16(<32 x i16> %a0) {
 ;
 ; X64-SSE2-LABEL: test_reduce_v32i16_v8i16:
 ; X64-SSE2:       ## %bb.0:
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    pmaxsw %xmm0, %xmm1
 ; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; X64-SSE2-NEXT:    pmaxsw %xmm1, %xmm0
@@ -1991,7 +1991,7 @@ define i16 @test_reduce_v32i16_v8i16(<32 x i16> %a0) {
 define i8 @test_reduce_v32i8_v16i8(<32 x i8> %a0) {
 ; X86-SSE2-LABEL: test_reduce_v32i8_v16i8:
 ; X86-SSE2:       ## %bb.0:
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; X86-SSE2-NEXT:    pcmpgtb %xmm1, %xmm2
 ; X86-SSE2-NEXT:    pand %xmm2, %xmm0
@@ -2047,7 +2047,7 @@ define i8 @test_reduce_v32i8_v16i8(<32 x i8> %a0) {
 ;
 ; X64-SSE2-LABEL: test_reduce_v32i8_v16i8:
 ; X64-SSE2:       ## %bb.0:
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; X64-SSE2-NEXT:    pcmpgtb %xmm1, %xmm2
 ; X64-SSE2-NEXT:    pand %xmm2, %xmm0
@@ -2119,7 +2119,7 @@ define i8 @test_reduce_v32i8_v16i8(<32 x i8> %a0) {
 define i8 @test_reduce_v64i8_v16i8(<64 x i8> %a0) {
 ; X86-SSE2-LABEL: test_reduce_v64i8_v16i8:
 ; X86-SSE2:       ## %bb.0:
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; X86-SSE2-NEXT:    pcmpgtb %xmm1, %xmm2
 ; X86-SSE2-NEXT:    pand %xmm2, %xmm0
@@ -2175,7 +2175,7 @@ define i8 @test_reduce_v64i8_v16i8(<64 x i8> %a0) {
 ;
 ; X64-SSE2-LABEL: test_reduce_v64i8_v16i8:
 ; X64-SSE2:       ## %bb.0:
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; X64-SSE2-NEXT:    pcmpgtb %xmm1, %xmm2
 ; X64-SSE2-NEXT:    pand %xmm2, %xmm0
diff --git a/llvm/test/CodeGen/X86/horizontal-reduce-smin.ll b/llvm/test/CodeGen/X86/horizontal-reduce-smin.ll
index 987da0f68082c..17c3a9fd4a010 100644
--- a/llvm/test/CodeGen/X86/horizontal-reduce-smin.ll
+++ b/llvm/test/CodeGen/X86/horizontal-reduce-smin.ll
@@ -16,7 +16,7 @@
 define i64 @test_reduce_v2i64(<2 x i64> %a0) {
 ; X86-SSE2-LABEL: test_reduce_v2i64:
 ; X86-SSE2:       ## %bb.0:
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,0,2147483648,0]
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm3
 ; X86-SSE2-NEXT:    pxor %xmm2, %xmm3
@@ -40,7 +40,7 @@ define i64 @test_reduce_v2i64(<2 x i64> %a0) {
 ; X86-SSE42-LABEL: test_reduce_v2i64:
 ; X86-SSE42:       ## %bb.0:
 ; X86-SSE42-NEXT:    movdqa %xmm0, %xmm1
-; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; X86-SSE42-NEXT:    movdqa %xmm2, %xmm0
 ; X86-SSE42-NEXT:    pcmpgtq %xmm1, %xmm0
 ; X86-SSE42-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
@@ -50,7 +50,7 @@ define i64 @test_reduce_v2i64(<2 x i64> %a0) {
 ;
 ; X86-AVX-LABEL: test_reduce_v2i64:
 ; X86-AVX:       ## %bb.0:
-; X86-AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-AVX-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
 ; X86-AVX-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
 ; X86-AVX-NEXT:    vmovd %xmm0, %eax
@@ -59,7 +59,7 @@ define i64 @test_reduce_v2i64(<2 x i64> %a0) {
 ;
 ; X64-SSE2-LABEL: test_reduce_v2i64:
 ; X64-SSE2:       ## %bb.0:
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
 ; X64-SSE2-NEXT:    movdqa %xmm0, %xmm3
 ; X64-SSE2-NEXT:    pxor %xmm2, %xmm3
@@ -81,7 +81,7 @@ define i64 @test_reduce_v2i64(<2 x i64> %a0) {
 ; X64-SSE42-LABEL: test_reduce_v2i64:
 ; X64-SSE42:       ## %bb.0:
 ; X64-SSE42-NEXT:    movdqa %xmm0, %xmm1
-; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; X64-SSE42-NEXT:    movdqa %xmm2, %xmm0
 ; X64-SSE42-NEXT:    pcmpgtq %xmm1, %xmm0
 ; X64-SSE42-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
@@ -90,7 +90,7 @@ define i64 @test_reduce_v2i64(<2 x i64> %a0) {
 ;
 ; X64-AVX1-LABEL: test_reduce_v2i64:
 ; X64-AVX1:       ## %bb.0:
-; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX1-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
 ; X64-AVX1-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
 ; X64-AVX1-NEXT:    vmovq %xmm0, %rax
@@ -98,7 +98,7 @@ define i64 @test_reduce_v2i64(<2 x i64> %a0) {
 ;
 ; X64-AVX2-LABEL: test_reduce_v2i64:
 ; X64-AVX2:       ## %bb.0:
-; X64-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX2-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
 ; X64-AVX2-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
 ; X64-AVX2-NEXT:    vmovq %xmm0, %rax
@@ -106,7 +106,7 @@ define i64 @test_reduce_v2i64(<2 x i64> %a0) {
 ;
 ; X64-AVX512-LABEL: test_reduce_v2i64:
 ; X64-AVX512:       ## %bb.0:
-; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX512-NEXT:    vpminsq %xmm1, %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vmovq %xmm0, %rax
 ; X64-AVX512-NEXT:    retq
@@ -120,7 +120,7 @@ define i64 @test_reduce_v2i64(<2 x i64> %a0) {
 define i32 @test_reduce_v4i32(<4 x i32> %a0) {
 ; X86-SSE2-LABEL: test_reduce_v4i32:
 ; X86-SSE2:       ## %bb.0:
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; X86-SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
 ; X86-SSE2-NEXT:    pand %xmm2, %xmm0
@@ -137,7 +137,7 @@ define i32 @test_reduce_v4i32(<4 x i32> %a0) {
 ;
 ; X86-SSE42-LABEL: test_reduce_v4i32:
 ; X86-SSE42:       ## %bb.0:
-; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE42-NEXT:    pminsd %xmm0, %xmm1
 ; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; X86-SSE42-NEXT:    pminsd %xmm1, %xmm0
@@ -146,7 +146,7 @@ define i32 @test_reduce_v4i32(<4 x i32> %a0) {
 ;
 ; X86-AVX-LABEL: test_reduce_v4i32:
 ; X86-AVX:       ## %bb.0:
-; X86-AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-AVX-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X86-AVX-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
@@ -155,7 +155,7 @@ define i32 @test_reduce_v4i32(<4 x i32> %a0) {
 ;
 ; X64-SSE2-LABEL: test_reduce_v4i32:
 ; X64-SSE2:       ## %bb.0:
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; X64-SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
 ; X64-SSE2-NEXT:    pand %xmm2, %xmm0
@@ -172,7 +172,7 @@ define i32 @test_reduce_v4i32(<4 x i32> %a0) {
 ;
 ; X64-SSE42-LABEL: test_reduce_v4i32:
 ; X64-SSE42:       ## %bb.0:
-; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE42-NEXT:    pminsd %xmm0, %xmm1
 ; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; X64-SSE42-NEXT:    pminsd %xmm1, %xmm0
@@ -181,7 +181,7 @@ define i32 @test_reduce_v4i32(<4 x i32> %a0) {
 ;
 ; X64-AVX-LABEL: test_reduce_v4i32:
 ; X64-AVX:       ## %bb.0:
-; X64-AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; X64-AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X64-AVX-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
@@ -200,7 +200,7 @@ define i32 @test_reduce_v4i32(<4 x i32> %a0) {
 define i16 @test_reduce_v8i16(<8 x i16> %a0) {
 ; X86-SSE2-LABEL: test_reduce_v8i16:
 ; X86-SSE2:       ## %bb.0:
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    pminsw %xmm0, %xmm1
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; X86-SSE2-NEXT:    pminsw %xmm1, %xmm0
@@ -231,7 +231,7 @@ define i16 @test_reduce_v8i16(<8 x i16> %a0) {
 ;
 ; X64-SSE2-LABEL: test_reduce_v8i16:
 ; X64-SSE2:       ## %bb.0:
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    pminsw %xmm0, %xmm1
 ; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; X64-SSE2-NEXT:    pminsw %xmm1, %xmm0
@@ -275,7 +275,7 @@ define i16 @test_reduce_v8i16(<8 x i16> %a0) {
 define i8 @test_reduce_v16i8(<16 x i8> %a0) {
 ; X86-SSE2-LABEL: test_reduce_v16i8:
 ; X86-SSE2:       ## %bb.0:
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; X86-SSE2-NEXT:    pcmpgtb %xmm0, %xmm2
 ; X86-SSE2-NEXT:    pand %xmm2, %xmm0
@@ -330,7 +330,7 @@ define i8 @test_reduce_v16i8(<16 x i8> %a0) {
 ;
 ; X64-SSE2-LABEL: test_reduce_v16i8:
 ; X64-SSE2:       ## %bb.0:
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; X64-SSE2-NEXT:    pcmpgtb %xmm0, %xmm2
 ; X64-SSE2-NEXT:    pand %xmm2, %xmm0
@@ -421,7 +421,7 @@ define i64 @test_reduce_v4i64(<4 x i64> %a0) {
 ; X86-SSE2-NEXT:    pand %xmm4, %xmm0
 ; X86-SSE2-NEXT:    pandn %xmm1, %xmm4
 ; X86-SSE2-NEXT:    por %xmm0, %xmm4
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[2,3,2,3]
 ; X86-SSE2-NEXT:    movdqa %xmm4, %xmm1
 ; X86-SSE2-NEXT:    pxor %xmm2, %xmm1
 ; X86-SSE2-NEXT:    pxor %xmm0, %xmm2
@@ -447,7 +447,7 @@ define i64 @test_reduce_v4i64(<4 x i64> %a0) {
 ; X86-SSE42-NEXT:    movdqa %xmm1, %xmm0
 ; X86-SSE42-NEXT:    pcmpgtq %xmm2, %xmm0
 ; X86-SSE42-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
-; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; X86-SSE42-NEXT:    movdqa %xmm2, %xmm0
 ; X86-SSE42-NEXT:    pcmpgtq %xmm1, %xmm0
 ; X86-SSE42-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
@@ -460,7 +460,7 @@ define i64 @test_reduce_v4i64(<4 x i64> %a0) {
 ; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; X86-AVX1-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
 ; X86-AVX1-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
-; X86-AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-AVX1-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
 ; X86-AVX1-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
 ; X86-AVX1-NEXT:    vmovd %xmm0, %eax
@@ -473,7 +473,7 @@ define i64 @test_reduce_v4i64(<4 x i64> %a0) {
 ; X86-AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X86-AVX2-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
 ; X86-AVX2-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
-; X86-AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-AVX2-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
 ; X86-AVX2-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
 ; X86-AVX2-NEXT:    vmovd %xmm0, %eax
@@ -499,7 +499,7 @@ define i64 @test_reduce_v4i64(<4 x i64> %a0) {
 ; X64-SSE2-NEXT:    pand %xmm4, %xmm0
 ; X64-SSE2-NEXT:    pandn %xmm1, %xmm4
 ; X64-SSE2-NEXT:    por %xmm0, %xmm4
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[2,3,2,3]
 ; X64-SSE2-NEXT:    movdqa %xmm4, %xmm1
 ; X64-SSE2-NEXT:    pxor %xmm2, %xmm1
 ; X64-SSE2-NEXT:    pxor %xmm0, %xmm2
@@ -523,7 +523,7 @@ define i64 @test_reduce_v4i64(<4 x i64> %a0) {
 ; X64-SSE42-NEXT:    movdqa %xmm1, %xmm0
 ; X64-SSE42-NEXT:    pcmpgtq %xmm2, %xmm0
 ; X64-SSE42-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
-; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; X64-SSE42-NEXT:    movdqa %xmm2, %xmm0
 ; X64-SSE42-NEXT:    pcmpgtq %xmm1, %xmm0
 ; X64-SSE42-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
@@ -535,7 +535,7 @@ define i64 @test_reduce_v4i64(<4 x i64> %a0) {
 ; X64-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; X64-AVX1-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
 ; X64-AVX1-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
-; X64-AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX1-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
 ; X64-AVX1-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
 ; X64-AVX1-NEXT:    vmovq %xmm0, %rax
@@ -547,7 +547,7 @@ define i64 @test_reduce_v4i64(<4 x i64> %a0) {
 ; X64-AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X64-AVX2-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
 ; X64-AVX2-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
-; X64-AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX2-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
 ; X64-AVX2-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
 ; X64-AVX2-NEXT:    vmovq %xmm0, %rax
@@ -558,7 +558,7 @@ define i64 @test_reduce_v4i64(<4 x i64> %a0) {
 ; X64-AVX512:       ## %bb.0:
 ; X64-AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X64-AVX512-NEXT:    vpminsq %xmm1, %xmm0, %xmm0
-; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX512-NEXT:    vpminsq %xmm1, %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vmovq %xmm0, %rax
 ; X64-AVX512-NEXT:    vzeroupper
@@ -581,7 +581,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X86-SSE2-NEXT:    pand %xmm2, %xmm0
 ; X86-SSE2-NEXT:    pandn %xmm1, %xmm2
 ; X86-SSE2-NEXT:    por %xmm0, %xmm2
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X86-SSE2-NEXT:    pcmpgtd %xmm2, %xmm1
 ; X86-SSE2-NEXT:    pand %xmm1, %xmm2
@@ -599,7 +599,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X86-SSE42-LABEL: test_reduce_v8i32:
 ; X86-SSE42:       ## %bb.0:
 ; X86-SSE42-NEXT:    pminsd %xmm1, %xmm0
-; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE42-NEXT:    pminsd %xmm0, %xmm1
 ; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; X86-SSE42-NEXT:    pminsd %xmm1, %xmm0
@@ -610,7 +610,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X86-AVX1:       ## %bb.0:
 ; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; X86-AVX1-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
-; X86-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-AVX1-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X86-AVX1-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
@@ -622,7 +622,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X86-AVX2:       ## %bb.0:
 ; X86-AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X86-AVX2-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
-; X86-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-AVX2-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; X86-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X86-AVX2-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
@@ -637,7 +637,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X64-SSE2-NEXT:    pand %xmm2, %xmm0
 ; X64-SSE2-NEXT:    pandn %xmm1, %xmm2
 ; X64-SSE2-NEXT:    por %xmm0, %xmm2
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; X64-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X64-SSE2-NEXT:    pcmpgtd %xmm2, %xmm1
 ; X64-SSE2-NEXT:    pand %xmm1, %xmm2
@@ -655,7 +655,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X64-SSE42-LABEL: test_reduce_v8i32:
 ; X64-SSE42:       ## %bb.0:
 ; X64-SSE42-NEXT:    pminsd %xmm1, %xmm0
-; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE42-NEXT:    pminsd %xmm0, %xmm1
 ; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; X64-SSE42-NEXT:    pminsd %xmm1, %xmm0
@@ -666,7 +666,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X64-AVX1:       ## %bb.0:
 ; X64-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; X64-AVX1-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX1-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X64-AVX1-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
@@ -678,7 +678,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X64-AVX2:       ## %bb.0:
 ; X64-AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X64-AVX2-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
-; X64-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX2-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; X64-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X64-AVX2-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
@@ -690,7 +690,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X64-AVX512:       ## %bb.0:
 ; X64-AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X64-AVX512-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
-; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX512-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X64-AVX512-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
@@ -714,7 +714,7 @@ define i16 @test_reduce_v16i16(<16 x i16> %a0) {
 ; X86-SSE2-LABEL: test_reduce_v16i16:
 ; X86-SSE2:       ## %bb.0:
 ; X86-SSE2-NEXT:    pminsw %xmm1, %xmm0
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    pminsw %xmm0, %xmm1
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; X86-SSE2-NEXT:    pminsw %xmm1, %xmm0
@@ -762,7 +762,7 @@ define i16 @test_reduce_v16i16(<16 x i16> %a0) {
 ; X64-SSE2-LABEL: test_reduce_v16i16:
 ; X64-SSE2:       ## %bb.0:
 ; X64-SSE2-NEXT:    pminsw %xmm1, %xmm0
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    pminsw %xmm0, %xmm1
 ; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; X64-SSE2-NEXT:    pminsw %xmm1, %xmm0
@@ -842,7 +842,7 @@ define i8 @test_reduce_v32i8(<32 x i8> %a0) {
 ; X86-SSE2-NEXT:    pand %xmm2, %xmm0
 ; X86-SSE2-NEXT:    pandn %xmm1, %xmm2
 ; X86-SSE2-NEXT:    por %xmm0, %xmm2
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X86-SSE2-NEXT:    pcmpgtb %xmm2, %xmm1
 ; X86-SSE2-NEXT:    pand %xmm1, %xmm2
@@ -920,7 +920,7 @@ define i8 @test_reduce_v32i8(<32 x i8> %a0) {
 ; X64-SSE2-NEXT:    pand %xmm2, %xmm0
 ; X64-SSE2-NEXT:    pandn %xmm1, %xmm2
 ; X64-SSE2-NEXT:    por %xmm0, %xmm2
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; X64-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X64-SSE2-NEXT:    pcmpgtb %xmm2, %xmm1
 ; X64-SSE2-NEXT:    pand %xmm1, %xmm2
@@ -1076,7 +1076,7 @@ define i64 @test_reduce_v8i64(<8 x i64> %a0) {
 ; X86-SSE2-NEXT:    pand %xmm0, %xmm1
 ; X86-SSE2-NEXT:    pandn %xmm5, %xmm0
 ; X86-SSE2-NEXT:    por %xmm1, %xmm0
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; X86-SSE2-NEXT:    pxor %xmm4, %xmm2
 ; X86-SSE2-NEXT:    pxor %xmm1, %xmm4
@@ -1108,7 +1108,7 @@ define i64 @test_reduce_v8i64(<8 x i64> %a0) {
 ; X86-SSE42-NEXT:    movapd %xmm3, %xmm0
 ; X86-SSE42-NEXT:    pcmpgtq %xmm2, %xmm0
 ; X86-SSE42-NEXT:    blendvpd %xmm0, %xmm2, %xmm3
-; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[2,3,0,1]
+; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[2,3,2,3]
 ; X86-SSE42-NEXT:    movdqa %xmm1, %xmm0
 ; X86-SSE42-NEXT:    pcmpgtq %xmm3, %xmm0
 ; X86-SSE42-NEXT:    blendvpd %xmm0, %xmm3, %xmm1
@@ -1126,7 +1126,7 @@ define i64 @test_reduce_v8i64(<8 x i64> %a0) {
 ; X86-AVX1-NEXT:    vblendvpd %xmm4, %xmm2, %xmm3, %xmm1
 ; X86-AVX1-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
 ; X86-AVX1-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
-; X86-AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-AVX1-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
 ; X86-AVX1-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
 ; X86-AVX1-NEXT:    vmovd %xmm0, %eax
@@ -1141,7 +1141,7 @@ define i64 @test_reduce_v8i64(<8 x i64> %a0) {
 ; X86-AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; X86-AVX2-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
 ; X86-AVX2-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
-; X86-AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-AVX2-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
 ; X86-AVX2-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
 ; X86-AVX2-NEXT:    vmovd %xmm0, %eax
@@ -1197,7 +1197,7 @@ define i64 @test_reduce_v8i64(<8 x i64> %a0) {
 ; X64-SSE2-NEXT:    pand %xmm1, %xmm3
 ; X64-SSE2-NEXT:    pandn %xmm5, %xmm1
 ; X64-SSE2-NEXT:    por %xmm3, %xmm1
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; X64-SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; X64-SSE2-NEXT:    pxor %xmm4, %xmm2
 ; X64-SSE2-NEXT:    pxor %xmm0, %xmm4
@@ -1227,7 +1227,7 @@ define i64 @test_reduce_v8i64(<8 x i64> %a0) {
 ; X64-SSE42-NEXT:    movapd %xmm3, %xmm0
 ; X64-SSE42-NEXT:    pcmpgtq %xmm2, %xmm0
 ; X64-SSE42-NEXT:    blendvpd %xmm0, %xmm2, %xmm3
-; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[2,3,0,1]
+; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[2,3,2,3]
 ; X64-SSE42-NEXT:    movdqa %xmm1, %xmm0
 ; X64-SSE42-NEXT:    pcmpgtq %xmm3, %xmm0
 ; X64-SSE42-NEXT:    blendvpd %xmm0, %xmm3, %xmm1
@@ -1244,7 +1244,7 @@ define i64 @test_reduce_v8i64(<8 x i64> %a0) {
 ; X64-AVX1-NEXT:    vblendvpd %xmm4, %xmm2, %xmm3, %xmm1
 ; X64-AVX1-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
 ; X64-AVX1-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
-; X64-AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX1-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
 ; X64-AVX1-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
 ; X64-AVX1-NEXT:    vmovq %xmm0, %rax
@@ -1258,7 +1258,7 @@ define i64 @test_reduce_v8i64(<8 x i64> %a0) {
 ; X64-AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; X64-AVX2-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
 ; X64-AVX2-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
-; X64-AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX2-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
 ; X64-AVX2-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
 ; X64-AVX2-NEXT:    vmovq %xmm0, %rax
@@ -1271,7 +1271,7 @@ define i64 @test_reduce_v8i64(<8 x i64> %a0) {
 ; X64-AVX512-NEXT:    vpminsq %zmm1, %zmm0, %zmm0
 ; X64-AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X64-AVX512-NEXT:    vpminsq %xmm1, %xmm0, %xmm0
-; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX512-NEXT:    vpminsq %xmm1, %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vmovq %xmm0, %rax
 ; X64-AVX512-NEXT:    vzeroupper
@@ -1307,7 +1307,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X86-SSE2-NEXT:    pand %xmm1, %xmm4
 ; X86-SSE2-NEXT:    pandn %xmm0, %xmm1
 ; X86-SSE2-NEXT:    por %xmm4, %xmm1
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; X86-SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
 ; X86-SSE2-NEXT:    pand %xmm2, %xmm1
@@ -1327,7 +1327,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X86-SSE42-NEXT:    pminsd %xmm3, %xmm1
 ; X86-SSE42-NEXT:    pminsd %xmm2, %xmm1
 ; X86-SSE42-NEXT:    pminsd %xmm0, %xmm1
-; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; X86-SSE42-NEXT:    pminsd %xmm1, %xmm0
 ; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X86-SSE42-NEXT:    pminsd %xmm0, %xmm1
@@ -1341,7 +1341,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X86-AVX1-NEXT:    vpminsd %xmm2, %xmm3, %xmm2
 ; X86-AVX1-NEXT:    vpminsd %xmm2, %xmm1, %xmm1
 ; X86-AVX1-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
-; X86-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-AVX1-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X86-AVX1-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
@@ -1354,7 +1354,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X86-AVX2-NEXT:    vpminsd %ymm1, %ymm0, %ymm0
 ; X86-AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X86-AVX2-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
-; X86-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-AVX2-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; X86-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X86-AVX2-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
@@ -1379,7 +1379,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X64-SSE2-NEXT:    pand %xmm1, %xmm4
 ; X64-SSE2-NEXT:    pandn %xmm0, %xmm1
 ; X64-SSE2-NEXT:    por %xmm4, %xmm1
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; X64-SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; X64-SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
 ; X64-SSE2-NEXT:    pand %xmm2, %xmm1
@@ -1399,7 +1399,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X64-SSE42-NEXT:    pminsd %xmm3, %xmm1
 ; X64-SSE42-NEXT:    pminsd %xmm2, %xmm1
 ; X64-SSE42-NEXT:    pminsd %xmm0, %xmm1
-; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; X64-SSE42-NEXT:    pminsd %xmm1, %xmm0
 ; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X64-SSE42-NEXT:    pminsd %xmm0, %xmm1
@@ -1413,7 +1413,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X64-AVX1-NEXT:    vpminsd %xmm2, %xmm3, %xmm2
 ; X64-AVX1-NEXT:    vpminsd %xmm2, %xmm1, %xmm1
 ; X64-AVX1-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX1-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X64-AVX1-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
@@ -1426,7 +1426,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X64-AVX2-NEXT:    vpminsd %ymm1, %ymm0, %ymm0
 ; X64-AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X64-AVX2-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
-; X64-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX2-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; X64-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X64-AVX2-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
@@ -1440,7 +1440,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X64-AVX512-NEXT:    vpminsd %zmm1, %zmm0, %zmm0
 ; X64-AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X64-AVX512-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
-; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX512-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X64-AVX512-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
@@ -1469,7 +1469,7 @@ define i16 @test_reduce_v32i16(<32 x i16> %a0) {
 ; X86-SSE2-NEXT:    pminsw %xmm3, %xmm1
 ; X86-SSE2-NEXT:    pminsw %xmm2, %xmm1
 ; X86-SSE2-NEXT:    pminsw %xmm0, %xmm1
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; X86-SSE2-NEXT:    pminsw %xmm1, %xmm0
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X86-SSE2-NEXT:    pminsw %xmm0, %xmm1
@@ -1525,7 +1525,7 @@ define i16 @test_reduce_v32i16(<32 x i16> %a0) {
 ; X64-SSE2-NEXT:    pminsw %xmm3, %xmm1
 ; X64-SSE2-NEXT:    pminsw %xmm2, %xmm1
 ; X64-SSE2-NEXT:    pminsw %xmm0, %xmm1
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; X64-SSE2-NEXT:    pminsw %xmm1, %xmm0
 ; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X64-SSE2-NEXT:    pminsw %xmm0, %xmm1
@@ -1626,7 +1626,7 @@ define i8 @test_reduce_v64i8(<64 x i8> %a0) {
 ; X86-SSE2-NEXT:    pand %xmm1, %xmm4
 ; X86-SSE2-NEXT:    pandn %xmm0, %xmm1
 ; X86-SSE2-NEXT:    por %xmm4, %xmm1
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; X86-SSE2-NEXT:    pcmpgtb %xmm1, %xmm2
 ; X86-SSE2-NEXT:    pand %xmm2, %xmm1
@@ -1720,7 +1720,7 @@ define i8 @test_reduce_v64i8(<64 x i8> %a0) {
 ; X64-SSE2-NEXT:    pand %xmm1, %xmm4
 ; X64-SSE2-NEXT:    pandn %xmm0, %xmm1
 ; X64-SSE2-NEXT:    por %xmm4, %xmm1
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; X64-SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; X64-SSE2-NEXT:    pcmpgtb %xmm1, %xmm2
 ; X64-SSE2-NEXT:    pand %xmm2, %xmm1
@@ -1841,7 +1841,7 @@ define i8 @test_reduce_v64i8(<64 x i8> %a0) {
 define i16 @test_reduce_v16i16_v8i16(<16 x i16> %a0) {
 ; X86-SSE2-LABEL: test_reduce_v16i16_v8i16:
 ; X86-SSE2:       ## %bb.0:
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    pminsw %xmm0, %xmm1
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; X86-SSE2-NEXT:    pminsw %xmm1, %xmm0
@@ -1873,7 +1873,7 @@ define i16 @test_reduce_v16i16_v8i16(<16 x i16> %a0) {
 ;
 ; X64-SSE2-LABEL: test_reduce_v16i16_v8i16:
 ; X64-SSE2:       ## %bb.0:
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    pminsw %xmm0, %xmm1
 ; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; X64-SSE2-NEXT:    pminsw %xmm1, %xmm0
@@ -1918,7 +1918,7 @@ define i16 @test_reduce_v16i16_v8i16(<16 x i16> %a0) {
 define i16 @test_reduce_v32i16_v8i16(<32 x i16> %a0) {
 ; X86-SSE2-LABEL: test_reduce_v32i16_v8i16:
 ; X86-SSE2:       ## %bb.0:
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    pminsw %xmm0, %xmm1
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; X86-SSE2-NEXT:    pminsw %xmm1, %xmm0
@@ -1950,7 +1950,7 @@ define i16 @test_reduce_v32i16_v8i16(<32 x i16> %a0) {
 ;
 ; X64-SSE2-LABEL: test_reduce_v32i16_v8i16:
 ; X64-SSE2:       ## %bb.0:
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    pminsw %xmm0, %xmm1
 ; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; X64-SSE2-NEXT:    pminsw %xmm1, %xmm0
@@ -1995,7 +1995,7 @@ define i16 @test_reduce_v32i16_v8i16(<32 x i16> %a0) {
 define i8 @test_reduce_v32i8_v16i8(<32 x i8> %a0) {
 ; X86-SSE2-LABEL: test_reduce_v32i8_v16i8:
 ; X86-SSE2:       ## %bb.0:
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; X86-SSE2-NEXT:    pcmpgtb %xmm0, %xmm2
 ; X86-SSE2-NEXT:    pand %xmm2, %xmm0
@@ -2051,7 +2051,7 @@ define i8 @test_reduce_v32i8_v16i8(<32 x i8> %a0) {
 ;
 ; X64-SSE2-LABEL: test_reduce_v32i8_v16i8:
 ; X64-SSE2:       ## %bb.0:
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; X64-SSE2-NEXT:    pcmpgtb %xmm0, %xmm2
 ; X64-SSE2-NEXT:    pand %xmm2, %xmm0
@@ -2123,7 +2123,7 @@ define i8 @test_reduce_v32i8_v16i8(<32 x i8> %a0) {
 define i8 @test_reduce_v64i8_v16i8(<64 x i8> %a0) {
 ; X86-SSE2-LABEL: test_reduce_v64i8_v16i8:
 ; X86-SSE2:       ## %bb.0:
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; X86-SSE2-NEXT:    pcmpgtb %xmm0, %xmm2
 ; X86-SSE2-NEXT:    pand %xmm2, %xmm0
@@ -2179,7 +2179,7 @@ define i8 @test_reduce_v64i8_v16i8(<64 x i8> %a0) {
 ;
 ; X64-SSE2-LABEL: test_reduce_v64i8_v16i8:
 ; X64-SSE2:       ## %bb.0:
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; X64-SSE2-NEXT:    pcmpgtb %xmm0, %xmm2
 ; X64-SSE2-NEXT:    pand %xmm2, %xmm0
diff --git a/llvm/test/CodeGen/X86/horizontal-reduce-umax.ll b/llvm/test/CodeGen/X86/horizontal-reduce-umax.ll
index 6e59cd046cb0a..c69551aa3d883 100644
--- a/llvm/test/CodeGen/X86/horizontal-reduce-umax.ll
+++ b/llvm/test/CodeGen/X86/horizontal-reduce-umax.ll
@@ -16,7 +16,7 @@
 define i64 @test_reduce_v2i64(<2 x i64> %a0) {
 ; X86-SSE2-LABEL: test_reduce_v2i64:
 ; X86-SSE2:       ## %bb.0:
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm3
 ; X86-SSE2-NEXT:    pxor %xmm2, %xmm3
@@ -40,7 +40,7 @@ define i64 @test_reduce_v2i64(<2 x i64> %a0) {
 ; X86-SSE42-LABEL: test_reduce_v2i64:
 ; X86-SSE42:       ## %bb.0:
 ; X86-SSE42-NEXT:    movdqa %xmm0, %xmm1
-; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; X86-SSE42-NEXT:    movdqa {{.*#+}} xmm3 = [0,2147483648,0,2147483648]
 ; X86-SSE42-NEXT:    pxor %xmm3, %xmm0
 ; X86-SSE42-NEXT:    pxor %xmm2, %xmm3
@@ -52,7 +52,7 @@ define i64 @test_reduce_v2i64(<2 x i64> %a0) {
 ;
 ; X86-AVX1-LABEL: test_reduce_v2i64:
 ; X86-AVX1:       ## %bb.0:
-; X86-AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-AVX1-NEXT:    vmovddup {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
 ; X86-AVX1-NEXT:    ## xmm2 = mem[0,0]
 ; X86-AVX1-NEXT:    vxorps %xmm2, %xmm0, %xmm3
@@ -65,7 +65,7 @@ define i64 @test_reduce_v2i64(<2 x i64> %a0) {
 ;
 ; X86-AVX2-LABEL: test_reduce_v2i64:
 ; X86-AVX2:       ## %bb.0:
-; X86-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-AVX2-NEXT:    vpbroadcastq {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
 ; X86-AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm3
 ; X86-AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm2
@@ -77,7 +77,7 @@ define i64 @test_reduce_v2i64(<2 x i64> %a0) {
 ;
 ; X64-SSE2-LABEL: test_reduce_v2i64:
 ; X64-SSE2:       ## %bb.0:
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [9223372039002259456,9223372039002259456]
 ; X64-SSE2-NEXT:    movdqa %xmm0, %xmm3
 ; X64-SSE2-NEXT:    pxor %xmm2, %xmm3
@@ -99,7 +99,7 @@ define i64 @test_reduce_v2i64(<2 x i64> %a0) {
 ; X64-SSE42-LABEL: test_reduce_v2i64:
 ; X64-SSE42:       ## %bb.0:
 ; X64-SSE42-NEXT:    movdqa %xmm0, %xmm1
-; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; X64-SSE42-NEXT:    movdqa {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808]
 ; X64-SSE42-NEXT:    pxor %xmm3, %xmm0
 ; X64-SSE42-NEXT:    pxor %xmm2, %xmm3
@@ -110,7 +110,7 @@ define i64 @test_reduce_v2i64(<2 x i64> %a0) {
 ;
 ; X64-AVX1-LABEL: test_reduce_v2i64:
 ; X64-AVX1:       ## %bb.0:
-; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
 ; X64-AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm3
 ; X64-AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm2
@@ -121,7 +121,7 @@ define i64 @test_reduce_v2i64(<2 x i64> %a0) {
 ;
 ; X64-AVX2-LABEL: test_reduce_v2i64:
 ; X64-AVX2:       ## %bb.0:
-; X64-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
 ; X64-AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm3
 ; X64-AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm2
@@ -132,7 +132,7 @@ define i64 @test_reduce_v2i64(<2 x i64> %a0) {
 ;
 ; X64-AVX512-LABEL: test_reduce_v2i64:
 ; X64-AVX512:       ## %bb.0:
-; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX512-NEXT:    vpmaxuq %xmm1, %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vmovq %xmm0, %rax
 ; X64-AVX512-NEXT:    retq
@@ -146,7 +146,7 @@ define i64 @test_reduce_v2i64(<2 x i64> %a0) {
 define i32 @test_reduce_v4i32(<4 x i32> %a0) {
 ; X86-SSE2-LABEL: test_reduce_v4i32:
 ; X86-SSE2:       ## %bb.0:
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm3
 ; X86-SSE2-NEXT:    pxor %xmm2, %xmm3
@@ -169,7 +169,7 @@ define i32 @test_reduce_v4i32(<4 x i32> %a0) {
 ;
 ; X86-SSE42-LABEL: test_reduce_v4i32:
 ; X86-SSE42:       ## %bb.0:
-; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE42-NEXT:    pmaxud %xmm0, %xmm1
 ; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; X86-SSE42-NEXT:    pmaxud %xmm1, %xmm0
@@ -178,7 +178,7 @@ define i32 @test_reduce_v4i32(<4 x i32> %a0) {
 ;
 ; X86-AVX-LABEL: test_reduce_v4i32:
 ; X86-AVX:       ## %bb.0:
-; X86-AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-AVX-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X86-AVX-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
@@ -187,7 +187,7 @@ define i32 @test_reduce_v4i32(<4 x i32> %a0) {
 ;
 ; X64-SSE2-LABEL: test_reduce_v4i32:
 ; X64-SSE2:       ## %bb.0:
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]
 ; X64-SSE2-NEXT:    movdqa %xmm0, %xmm3
 ; X64-SSE2-NEXT:    pxor %xmm2, %xmm3
@@ -210,7 +210,7 @@ define i32 @test_reduce_v4i32(<4 x i32> %a0) {
 ;
 ; X64-SSE42-LABEL: test_reduce_v4i32:
 ; X64-SSE42:       ## %bb.0:
-; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE42-NEXT:    pmaxud %xmm0, %xmm1
 ; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; X64-SSE42-NEXT:    pmaxud %xmm1, %xmm0
@@ -219,7 +219,7 @@ define i32 @test_reduce_v4i32(<4 x i32> %a0) {
 ;
 ; X64-AVX-LABEL: test_reduce_v4i32:
 ; X64-AVX:       ## %bb.0:
-; X64-AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; X64-AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X64-AVX-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
@@ -238,7 +238,7 @@ define i32 @test_reduce_v4i32(<4 x i32> %a0) {
 define i16 @test_reduce_v8i16(<8 x i16> %a0) {
 ; X86-SSE2-LABEL: test_reduce_v8i16:
 ; X86-SSE2:       ## %bb.0:
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [32768,32768,32768,32768,32768,32768,32768,32768]
 ; X86-SSE2-NEXT:    pxor %xmm2, %xmm0
 ; X86-SSE2-NEXT:    pxor %xmm2, %xmm1
@@ -275,7 +275,7 @@ define i16 @test_reduce_v8i16(<8 x i16> %a0) {
 ;
 ; X64-SSE2-LABEL: test_reduce_v8i16:
 ; X64-SSE2:       ## %bb.0:
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [32768,32768,32768,32768,32768,32768,32768,32768]
 ; X64-SSE2-NEXT:    pxor %xmm2, %xmm0
 ; X64-SSE2-NEXT:    pxor %xmm2, %xmm1
@@ -344,7 +344,7 @@ define i16 @test_reduce_v8i16(<8 x i16> %a0) {
 define i8 @test_reduce_v16i8(<16 x i8> %a0) {
 ; X86-SSE2-LABEL: test_reduce_v16i8:
 ; X86-SSE2:       ## %bb.0:
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    pmaxub %xmm0, %xmm1
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; X86-SSE2-NEXT:    pmaxub %xmm1, %xmm0
@@ -385,7 +385,7 @@ define i8 @test_reduce_v16i8(<16 x i8> %a0) {
 ;
 ; X64-SSE2-LABEL: test_reduce_v16i8:
 ; X64-SSE2:       ## %bb.0:
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    pmaxub %xmm0, %xmm1
 ; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; X64-SSE2-NEXT:    pmaxub %xmm1, %xmm0
@@ -485,7 +485,7 @@ define i64 @test_reduce_v4i64(<4 x i64> %a0) {
 ; X86-SSE2-NEXT:    pand %xmm4, %xmm0
 ; X86-SSE2-NEXT:    pandn %xmm1, %xmm4
 ; X86-SSE2-NEXT:    por %xmm0, %xmm4
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[2,3,2,3]
 ; X86-SSE2-NEXT:    movdqa %xmm4, %xmm1
 ; X86-SSE2-NEXT:    pxor %xmm2, %xmm1
 ; X86-SSE2-NEXT:    pxor %xmm0, %xmm2
@@ -514,7 +514,7 @@ define i64 @test_reduce_v4i64(<4 x i64> %a0) {
 ; X86-SSE42-NEXT:    pxor %xmm3, %xmm0
 ; X86-SSE42-NEXT:    pcmpgtq %xmm4, %xmm0
 ; X86-SSE42-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
-; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; X86-SSE42-NEXT:    movdqa %xmm1, %xmm0
 ; X86-SSE42-NEXT:    pxor %xmm3, %xmm0
 ; X86-SSE42-NEXT:    pxor %xmm2, %xmm3
@@ -533,7 +533,7 @@ define i64 @test_reduce_v4i64(<4 x i64> %a0) {
 ; X86-AVX1-NEXT:    vxorps %xmm2, %xmm0, %xmm4
 ; X86-AVX1-NEXT:    vpcmpgtq %xmm3, %xmm4, %xmm3
 ; X86-AVX1-NEXT:    vblendvpd %xmm3, %xmm0, %xmm1, %xmm0
-; X86-AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-AVX1-NEXT:    vxorpd %xmm2, %xmm0, %xmm3
 ; X86-AVX1-NEXT:    vxorpd %xmm2, %xmm1, %xmm2
 ; X86-AVX1-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm2
@@ -551,7 +551,7 @@ define i64 @test_reduce_v4i64(<4 x i64> %a0) {
 ; X86-AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm4
 ; X86-AVX2-NEXT:    vpcmpgtq %xmm3, %xmm4, %xmm3
 ; X86-AVX2-NEXT:    vblendvpd %xmm3, %xmm0, %xmm1, %xmm0
-; X86-AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-AVX2-NEXT:    vxorpd %xmm2, %xmm0, %xmm3
 ; X86-AVX2-NEXT:    vxorpd %xmm2, %xmm1, %xmm2
 ; X86-AVX2-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm2
@@ -579,7 +579,7 @@ define i64 @test_reduce_v4i64(<4 x i64> %a0) {
 ; X64-SSE2-NEXT:    pand %xmm4, %xmm0
 ; X64-SSE2-NEXT:    pandn %xmm1, %xmm4
 ; X64-SSE2-NEXT:    por %xmm0, %xmm4
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[2,3,2,3]
 ; X64-SSE2-NEXT:    movdqa %xmm4, %xmm1
 ; X64-SSE2-NEXT:    pxor %xmm2, %xmm1
 ; X64-SSE2-NEXT:    pxor %xmm0, %xmm2
@@ -606,7 +606,7 @@ define i64 @test_reduce_v4i64(<4 x i64> %a0) {
 ; X64-SSE42-NEXT:    pxor %xmm3, %xmm0
 ; X64-SSE42-NEXT:    pcmpgtq %xmm4, %xmm0
 ; X64-SSE42-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
-; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; X64-SSE42-NEXT:    movdqa %xmm1, %xmm0
 ; X64-SSE42-NEXT:    pxor %xmm3, %xmm0
 ; X64-SSE42-NEXT:    pxor %xmm2, %xmm3
@@ -623,7 +623,7 @@ define i64 @test_reduce_v4i64(<4 x i64> %a0) {
 ; X64-AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm4
 ; X64-AVX1-NEXT:    vpcmpgtq %xmm3, %xmm4, %xmm3
 ; X64-AVX1-NEXT:    vblendvpd %xmm3, %xmm0, %xmm1, %xmm0
-; X64-AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX1-NEXT:    vxorpd %xmm2, %xmm0, %xmm3
 ; X64-AVX1-NEXT:    vxorpd %xmm2, %xmm1, %xmm2
 ; X64-AVX1-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm2
@@ -640,7 +640,7 @@ define i64 @test_reduce_v4i64(<4 x i64> %a0) {
 ; X64-AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm4
 ; X64-AVX2-NEXT:    vpcmpgtq %xmm3, %xmm4, %xmm3
 ; X64-AVX2-NEXT:    vblendvpd %xmm3, %xmm0, %xmm1, %xmm0
-; X64-AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX2-NEXT:    vxorpd %xmm2, %xmm0, %xmm3
 ; X64-AVX2-NEXT:    vxorpd %xmm2, %xmm1, %xmm2
 ; X64-AVX2-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm2
@@ -653,7 +653,7 @@ define i64 @test_reduce_v4i64(<4 x i64> %a0) {
 ; X64-AVX512:       ## %bb.0:
 ; X64-AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X64-AVX512-NEXT:    vpmaxuq %xmm1, %xmm0, %xmm0
-; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX512-NEXT:    vpmaxuq %xmm1, %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vmovq %xmm0, %rax
 ; X64-AVX512-NEXT:    vzeroupper
@@ -680,7 +680,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X86-SSE2-NEXT:    pand %xmm4, %xmm0
 ; X86-SSE2-NEXT:    pandn %xmm1, %xmm4
 ; X86-SSE2-NEXT:    por %xmm0, %xmm4
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[2,3,2,3]
 ; X86-SSE2-NEXT:    movdqa %xmm4, %xmm1
 ; X86-SSE2-NEXT:    pxor %xmm2, %xmm1
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm3
@@ -703,7 +703,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X86-SSE42-LABEL: test_reduce_v8i32:
 ; X86-SSE42:       ## %bb.0:
 ; X86-SSE42-NEXT:    pmaxud %xmm1, %xmm0
-; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE42-NEXT:    pmaxud %xmm0, %xmm1
 ; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; X86-SSE42-NEXT:    pmaxud %xmm1, %xmm0
@@ -714,7 +714,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X86-AVX1:       ## %bb.0:
 ; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; X86-AVX1-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
-; X86-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-AVX1-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X86-AVX1-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
@@ -726,7 +726,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X86-AVX2:       ## %bb.0:
 ; X86-AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X86-AVX2-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
-; X86-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-AVX2-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; X86-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X86-AVX2-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
@@ -745,7 +745,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X64-SSE2-NEXT:    pand %xmm4, %xmm0
 ; X64-SSE2-NEXT:    pandn %xmm1, %xmm4
 ; X64-SSE2-NEXT:    por %xmm0, %xmm4
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[2,3,2,3]
 ; X64-SSE2-NEXT:    movdqa %xmm4, %xmm1
 ; X64-SSE2-NEXT:    pxor %xmm2, %xmm1
 ; X64-SSE2-NEXT:    movdqa %xmm0, %xmm3
@@ -768,7 +768,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X64-SSE42-LABEL: test_reduce_v8i32:
 ; X64-SSE42:       ## %bb.0:
 ; X64-SSE42-NEXT:    pmaxud %xmm1, %xmm0
-; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE42-NEXT:    pmaxud %xmm0, %xmm1
 ; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; X64-SSE42-NEXT:    pmaxud %xmm1, %xmm0
@@ -779,7 +779,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X64-AVX1:       ## %bb.0:
 ; X64-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; X64-AVX1-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX1-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X64-AVX1-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
@@ -791,7 +791,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X64-AVX2:       ## %bb.0:
 ; X64-AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X64-AVX2-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
-; X64-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX2-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; X64-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X64-AVX2-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
@@ -803,7 +803,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X64-AVX512:       ## %bb.0:
 ; X64-AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X64-AVX512-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
-; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX512-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X64-AVX512-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
@@ -830,7 +830,7 @@ define i16 @test_reduce_v16i16(<16 x i16> %a0) {
 ; X86-SSE2-NEXT:    pxor %xmm2, %xmm1
 ; X86-SSE2-NEXT:    pxor %xmm2, %xmm0
 ; X86-SSE2-NEXT:    pmaxsw %xmm1, %xmm0
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    pmaxsw %xmm0, %xmm1
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; X86-SSE2-NEXT:    pmaxsw %xmm1, %xmm0
@@ -885,7 +885,7 @@ define i16 @test_reduce_v16i16(<16 x i16> %a0) {
 ; X64-SSE2-NEXT:    pxor %xmm2, %xmm1
 ; X64-SSE2-NEXT:    pxor %xmm2, %xmm0
 ; X64-SSE2-NEXT:    pmaxsw %xmm1, %xmm0
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    pmaxsw %xmm0, %xmm1
 ; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; X64-SSE2-NEXT:    pmaxsw %xmm1, %xmm0
@@ -965,7 +965,7 @@ define i8 @test_reduce_v32i8(<32 x i8> %a0) {
 ; X86-SSE2-LABEL: test_reduce_v32i8:
 ; X86-SSE2:       ## %bb.0:
 ; X86-SSE2-NEXT:    pmaxub %xmm1, %xmm0
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    pmaxub %xmm0, %xmm1
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; X86-SSE2-NEXT:    pmaxub %xmm1, %xmm0
@@ -1026,7 +1026,7 @@ define i8 @test_reduce_v32i8(<32 x i8> %a0) {
 ; X64-SSE2-LABEL: test_reduce_v32i8:
 ; X64-SSE2:       ## %bb.0:
 ; X64-SSE2-NEXT:    pmaxub %xmm1, %xmm0
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    pmaxub %xmm0, %xmm1
 ; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; X64-SSE2-NEXT:    pmaxub %xmm1, %xmm0
@@ -1169,7 +1169,7 @@ define i64 @test_reduce_v8i64(<8 x i64> %a0) {
 ; X86-SSE2-NEXT:    pand %xmm1, %xmm5
 ; X86-SSE2-NEXT:    pandn %xmm0, %xmm1
 ; X86-SSE2-NEXT:    por %xmm5, %xmm1
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; X86-SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; X86-SSE2-NEXT:    pxor %xmm4, %xmm2
 ; X86-SSE2-NEXT:    pxor %xmm0, %xmm4
@@ -1210,7 +1210,7 @@ define i64 @test_reduce_v8i64(<8 x i64> %a0) {
 ; X86-SSE42-NEXT:    xorpd %xmm5, %xmm0
 ; X86-SSE42-NEXT:    pcmpgtq %xmm1, %xmm0
 ; X86-SSE42-NEXT:    blendvpd %xmm0, %xmm2, %xmm3
-; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[2,3,0,1]
+; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[2,3,2,3]
 ; X86-SSE42-NEXT:    movdqa %xmm3, %xmm0
 ; X86-SSE42-NEXT:    pxor %xmm5, %xmm0
 ; X86-SSE42-NEXT:    pxor %xmm1, %xmm5
@@ -1238,7 +1238,7 @@ define i64 @test_reduce_v8i64(<8 x i64> %a0) {
 ; X86-AVX1-NEXT:    vxorpd %xmm2, %xmm0, %xmm1
 ; X86-AVX1-NEXT:    vpcmpgtq %xmm5, %xmm1, %xmm1
 ; X86-AVX1-NEXT:    vblendvpd %xmm1, %xmm0, %xmm4, %xmm0
-; X86-AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-AVX1-NEXT:    vxorpd %xmm2, %xmm0, %xmm3
 ; X86-AVX1-NEXT:    vxorpd %xmm2, %xmm1, %xmm2
 ; X86-AVX1-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm2
@@ -1260,7 +1260,7 @@ define i64 @test_reduce_v8i64(<8 x i64> %a0) {
 ; X86-AVX2-NEXT:    vxorpd %xmm2, %xmm0, %xmm4
 ; X86-AVX2-NEXT:    vpcmpgtq %xmm3, %xmm4, %xmm3
 ; X86-AVX2-NEXT:    vblendvpd %xmm3, %xmm0, %xmm1, %xmm0
-; X86-AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-AVX2-NEXT:    vxorpd %xmm2, %xmm0, %xmm3
 ; X86-AVX2-NEXT:    vxorpd %xmm2, %xmm1, %xmm2
 ; X86-AVX2-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm2
@@ -1318,7 +1318,7 @@ define i64 @test_reduce_v8i64(<8 x i64> %a0) {
 ; X64-SSE2-NEXT:    pand %xmm1, %xmm5
 ; X64-SSE2-NEXT:    pandn %xmm2, %xmm1
 ; X64-SSE2-NEXT:    por %xmm5, %xmm1
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; X64-SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; X64-SSE2-NEXT:    pxor %xmm4, %xmm2
 ; X64-SSE2-NEXT:    pxor %xmm0, %xmm4
@@ -1357,7 +1357,7 @@ define i64 @test_reduce_v8i64(<8 x i64> %a0) {
 ; X64-SSE42-NEXT:    xorpd %xmm5, %xmm0
 ; X64-SSE42-NEXT:    pcmpgtq %xmm1, %xmm0
 ; X64-SSE42-NEXT:    blendvpd %xmm0, %xmm2, %xmm3
-; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[2,3,0,1]
+; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[2,3,2,3]
 ; X64-SSE42-NEXT:    movdqa %xmm3, %xmm0
 ; X64-SSE42-NEXT:    pxor %xmm5, %xmm0
 ; X64-SSE42-NEXT:    pxor %xmm1, %xmm5
@@ -1383,7 +1383,7 @@ define i64 @test_reduce_v8i64(<8 x i64> %a0) {
 ; X64-AVX1-NEXT:    vxorpd %xmm2, %xmm0, %xmm1
 ; X64-AVX1-NEXT:    vpcmpgtq %xmm5, %xmm1, %xmm1
 ; X64-AVX1-NEXT:    vblendvpd %xmm1, %xmm0, %xmm4, %xmm0
-; X64-AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX1-NEXT:    vxorpd %xmm2, %xmm0, %xmm3
 ; X64-AVX1-NEXT:    vxorpd %xmm2, %xmm1, %xmm2
 ; X64-AVX1-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm2
@@ -1404,7 +1404,7 @@ define i64 @test_reduce_v8i64(<8 x i64> %a0) {
 ; X64-AVX2-NEXT:    vxorpd %xmm2, %xmm0, %xmm4
 ; X64-AVX2-NEXT:    vpcmpgtq %xmm3, %xmm4, %xmm3
 ; X64-AVX2-NEXT:    vblendvpd %xmm3, %xmm0, %xmm1, %xmm0
-; X64-AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX2-NEXT:    vxorpd %xmm2, %xmm0, %xmm3
 ; X64-AVX2-NEXT:    vxorpd %xmm2, %xmm1, %xmm2
 ; X64-AVX2-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm2
@@ -1419,7 +1419,7 @@ define i64 @test_reduce_v8i64(<8 x i64> %a0) {
 ; X64-AVX512-NEXT:    vpmaxuq %zmm1, %zmm0, %zmm0
 ; X64-AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X64-AVX512-NEXT:    vpmaxuq %xmm1, %xmm0, %xmm0
-; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX512-NEXT:    vpmaxuq %xmm1, %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vmovq %xmm0, %rax
 ; X64-AVX512-NEXT:    vzeroupper
@@ -1465,7 +1465,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X86-SSE2-NEXT:    pand %xmm1, %xmm5
 ; X86-SSE2-NEXT:    pandn %xmm2, %xmm1
 ; X86-SSE2-NEXT:    por %xmm5, %xmm1
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; X86-SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; X86-SSE2-NEXT:    pxor %xmm4, %xmm2
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm3
@@ -1490,7 +1490,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X86-SSE42-NEXT:    pmaxud %xmm3, %xmm1
 ; X86-SSE42-NEXT:    pmaxud %xmm2, %xmm1
 ; X86-SSE42-NEXT:    pmaxud %xmm0, %xmm1
-; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; X86-SSE42-NEXT:    pmaxud %xmm1, %xmm0
 ; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X86-SSE42-NEXT:    pmaxud %xmm0, %xmm1
@@ -1504,7 +1504,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X86-AVX1-NEXT:    vpmaxud %xmm2, %xmm3, %xmm2
 ; X86-AVX1-NEXT:    vpmaxud %xmm2, %xmm1, %xmm1
 ; X86-AVX1-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
-; X86-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-AVX1-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X86-AVX1-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
@@ -1517,7 +1517,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X86-AVX2-NEXT:    vpmaxud %ymm1, %ymm0, %ymm0
 ; X86-AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X86-AVX2-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
-; X86-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-AVX2-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; X86-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X86-AVX2-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
@@ -1552,7 +1552,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X64-SSE2-NEXT:    pand %xmm1, %xmm5
 ; X64-SSE2-NEXT:    pandn %xmm2, %xmm1
 ; X64-SSE2-NEXT:    por %xmm5, %xmm1
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; X64-SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; X64-SSE2-NEXT:    pxor %xmm4, %xmm2
 ; X64-SSE2-NEXT:    movdqa %xmm0, %xmm3
@@ -1577,7 +1577,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X64-SSE42-NEXT:    pmaxud %xmm3, %xmm1
 ; X64-SSE42-NEXT:    pmaxud %xmm2, %xmm1
 ; X64-SSE42-NEXT:    pmaxud %xmm0, %xmm1
-; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; X64-SSE42-NEXT:    pmaxud %xmm1, %xmm0
 ; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X64-SSE42-NEXT:    pmaxud %xmm0, %xmm1
@@ -1591,7 +1591,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X64-AVX1-NEXT:    vpmaxud %xmm2, %xmm3, %xmm2
 ; X64-AVX1-NEXT:    vpmaxud %xmm2, %xmm1, %xmm1
 ; X64-AVX1-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX1-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X64-AVX1-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
@@ -1604,7 +1604,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X64-AVX2-NEXT:    vpmaxud %ymm1, %ymm0, %ymm0
 ; X64-AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X64-AVX2-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
-; X64-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX2-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; X64-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X64-AVX2-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
@@ -1618,7 +1618,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X64-AVX512-NEXT:    vpmaxud %zmm1, %zmm0, %zmm0
 ; X64-AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X64-AVX512-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
-; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX512-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X64-AVX512-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
@@ -1652,7 +1652,7 @@ define i16 @test_reduce_v32i16(<32 x i16> %a0) {
 ; X86-SSE2-NEXT:    pmaxsw %xmm1, %xmm2
 ; X86-SSE2-NEXT:    pxor %xmm4, %xmm0
 ; X86-SSE2-NEXT:    pmaxsw %xmm2, %xmm0
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    pmaxsw %xmm0, %xmm1
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; X86-SSE2-NEXT:    pmaxsw %xmm1, %xmm0
@@ -1717,7 +1717,7 @@ define i16 @test_reduce_v32i16(<32 x i16> %a0) {
 ; X64-SSE2-NEXT:    pmaxsw %xmm1, %xmm2
 ; X64-SSE2-NEXT:    pxor %xmm4, %xmm0
 ; X64-SSE2-NEXT:    pmaxsw %xmm2, %xmm0
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    pmaxsw %xmm0, %xmm1
 ; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; X64-SSE2-NEXT:    pmaxsw %xmm1, %xmm0
@@ -1810,7 +1810,7 @@ define i8 @test_reduce_v64i8(<64 x i8> %a0) {
 ; X86-SSE2-NEXT:    pmaxub %xmm3, %xmm1
 ; X86-SSE2-NEXT:    pmaxub %xmm2, %xmm1
 ; X86-SSE2-NEXT:    pmaxub %xmm0, %xmm1
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; X86-SSE2-NEXT:    pmaxub %xmm1, %xmm0
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X86-SSE2-NEXT:    pmaxub %xmm0, %xmm1
@@ -1879,7 +1879,7 @@ define i8 @test_reduce_v64i8(<64 x i8> %a0) {
 ; X64-SSE2-NEXT:    pmaxub %xmm3, %xmm1
 ; X64-SSE2-NEXT:    pmaxub %xmm2, %xmm1
 ; X64-SSE2-NEXT:    pmaxub %xmm0, %xmm1
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; X64-SSE2-NEXT:    pmaxub %xmm1, %xmm0
 ; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X64-SSE2-NEXT:    pmaxub %xmm0, %xmm1
@@ -1987,7 +1987,7 @@ define i8 @test_reduce_v64i8(<64 x i8> %a0) {
 define i16 @test_reduce_v16i16_v8i16(<16 x i16> %a0) {
 ; X86-SSE2-LABEL: test_reduce_v16i16_v8i16:
 ; X86-SSE2:       ## %bb.0:
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [32768,32768,32768,32768,32768,32768,32768,32768]
 ; X86-SSE2-NEXT:    pxor %xmm2, %xmm0
 ; X86-SSE2-NEXT:    pxor %xmm2, %xmm1
@@ -2025,7 +2025,7 @@ define i16 @test_reduce_v16i16_v8i16(<16 x i16> %a0) {
 ;
 ; X64-SSE2-LABEL: test_reduce_v16i16_v8i16:
 ; X64-SSE2:       ## %bb.0:
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [32768,32768,32768,32768,32768,32768,32768,32768]
 ; X64-SSE2-NEXT:    pxor %xmm2, %xmm0
 ; X64-SSE2-NEXT:    pxor %xmm2, %xmm1
@@ -2097,7 +2097,7 @@ define i16 @test_reduce_v16i16_v8i16(<16 x i16> %a0) {
 define i16 @test_reduce_v32i16_v8i16(<32 x i16> %a0) {
 ; X86-SSE2-LABEL: test_reduce_v32i16_v8i16:
 ; X86-SSE2:       ## %bb.0:
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [32768,32768,32768,32768,32768,32768,32768,32768]
 ; X86-SSE2-NEXT:    pxor %xmm2, %xmm0
 ; X86-SSE2-NEXT:    pxor %xmm2, %xmm1
@@ -2135,7 +2135,7 @@ define i16 @test_reduce_v32i16_v8i16(<32 x i16> %a0) {
 ;
 ; X64-SSE2-LABEL: test_reduce_v32i16_v8i16:
 ; X64-SSE2:       ## %bb.0:
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [32768,32768,32768,32768,32768,32768,32768,32768]
 ; X64-SSE2-NEXT:    pxor %xmm2, %xmm0
 ; X64-SSE2-NEXT:    pxor %xmm2, %xmm1
@@ -2207,7 +2207,7 @@ define i16 @test_reduce_v32i16_v8i16(<32 x i16> %a0) {
 define i8 @test_reduce_v32i8_v16i8(<32 x i8> %a0) {
 ; X86-SSE2-LABEL: test_reduce_v32i8_v16i8:
 ; X86-SSE2:       ## %bb.0:
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    pmaxub %xmm0, %xmm1
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; X86-SSE2-NEXT:    pmaxub %xmm1, %xmm0
@@ -2249,7 +2249,7 @@ define i8 @test_reduce_v32i8_v16i8(<32 x i8> %a0) {
 ;
 ; X64-SSE2-LABEL: test_reduce_v32i8_v16i8:
 ; X64-SSE2:       ## %bb.0:
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    pmaxub %xmm0, %xmm1
 ; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; X64-SSE2-NEXT:    pmaxub %xmm1, %xmm0
@@ -2332,7 +2332,7 @@ define i8 @test_reduce_v32i8_v16i8(<32 x i8> %a0) {
 define i8 @test_reduce_v64i8_v16i8(<64 x i8> %a0) {
 ; X86-SSE2-LABEL: test_reduce_v64i8_v16i8:
 ; X86-SSE2:       ## %bb.0:
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    pmaxub %xmm0, %xmm1
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; X86-SSE2-NEXT:    pmaxub %xmm1, %xmm0
@@ -2374,7 +2374,7 @@ define i8 @test_reduce_v64i8_v16i8(<64 x i8> %a0) {
 ;
 ; X64-SSE2-LABEL: test_reduce_v64i8_v16i8:
 ; X64-SSE2:       ## %bb.0:
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    pmaxub %xmm0, %xmm1
 ; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; X64-SSE2-NEXT:    pmaxub %xmm1, %xmm0
diff --git a/llvm/test/CodeGen/X86/horizontal-reduce-umin.ll b/llvm/test/CodeGen/X86/horizontal-reduce-umin.ll
index 413b5f2ac4aa5..5f33520200d24 100644
--- a/llvm/test/CodeGen/X86/horizontal-reduce-umin.ll
+++ b/llvm/test/CodeGen/X86/horizontal-reduce-umin.ll
@@ -16,7 +16,7 @@
 define i64 @test_reduce_v2i64(<2 x i64> %a0) {
 ; X86-SSE2-LABEL: test_reduce_v2i64:
 ; X86-SSE2:       ## %bb.0:
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm3
 ; X86-SSE2-NEXT:    pxor %xmm2, %xmm3
@@ -40,7 +40,7 @@ define i64 @test_reduce_v2i64(<2 x i64> %a0) {
 ; X86-SSE42-LABEL: test_reduce_v2i64:
 ; X86-SSE42:       ## %bb.0:
 ; X86-SSE42-NEXT:    movdqa %xmm0, %xmm1
-; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; X86-SSE42-NEXT:    movdqa {{.*#+}} xmm0 = [0,2147483648,0,2147483648]
 ; X86-SSE42-NEXT:    movdqa %xmm1, %xmm3
 ; X86-SSE42-NEXT:    pxor %xmm0, %xmm3
@@ -53,7 +53,7 @@ define i64 @test_reduce_v2i64(<2 x i64> %a0) {
 ;
 ; X86-AVX1-LABEL: test_reduce_v2i64:
 ; X86-AVX1:       ## %bb.0:
-; X86-AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-AVX1-NEXT:    vmovddup {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
 ; X86-AVX1-NEXT:    ## xmm2 = mem[0,0]
 ; X86-AVX1-NEXT:    vxorps %xmm2, %xmm0, %xmm3
@@ -66,7 +66,7 @@ define i64 @test_reduce_v2i64(<2 x i64> %a0) {
 ;
 ; X86-AVX2-LABEL: test_reduce_v2i64:
 ; X86-AVX2:       ## %bb.0:
-; X86-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-AVX2-NEXT:    vpbroadcastq {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
 ; X86-AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm3
 ; X86-AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm2
@@ -78,7 +78,7 @@ define i64 @test_reduce_v2i64(<2 x i64> %a0) {
 ;
 ; X64-SSE2-LABEL: test_reduce_v2i64:
 ; X64-SSE2:       ## %bb.0:
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [9223372039002259456,9223372039002259456]
 ; X64-SSE2-NEXT:    movdqa %xmm0, %xmm3
 ; X64-SSE2-NEXT:    pxor %xmm2, %xmm3
@@ -100,7 +100,7 @@ define i64 @test_reduce_v2i64(<2 x i64> %a0) {
 ; X64-SSE42-LABEL: test_reduce_v2i64:
 ; X64-SSE42:       ## %bb.0:
 ; X64-SSE42-NEXT:    movdqa %xmm0, %xmm1
-; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; X64-SSE42-NEXT:    movdqa {{.*#+}} xmm0 = [9223372036854775808,9223372036854775808]
 ; X64-SSE42-NEXT:    movdqa %xmm1, %xmm3
 ; X64-SSE42-NEXT:    pxor %xmm0, %xmm3
@@ -112,7 +112,7 @@ define i64 @test_reduce_v2i64(<2 x i64> %a0) {
 ;
 ; X64-AVX1-LABEL: test_reduce_v2i64:
 ; X64-AVX1:       ## %bb.0:
-; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
 ; X64-AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm3
 ; X64-AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm2
@@ -123,7 +123,7 @@ define i64 @test_reduce_v2i64(<2 x i64> %a0) {
 ;
 ; X64-AVX2-LABEL: test_reduce_v2i64:
 ; X64-AVX2:       ## %bb.0:
-; X64-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
 ; X64-AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm3
 ; X64-AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm2
@@ -134,7 +134,7 @@ define i64 @test_reduce_v2i64(<2 x i64> %a0) {
 ;
 ; X64-AVX512-LABEL: test_reduce_v2i64:
 ; X64-AVX512:       ## %bb.0:
-; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX512-NEXT:    vpminuq %xmm1, %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vmovq %xmm0, %rax
 ; X64-AVX512-NEXT:    retq
@@ -148,7 +148,7 @@ define i64 @test_reduce_v2i64(<2 x i64> %a0) {
 define i32 @test_reduce_v4i32(<4 x i32> %a0) {
 ; X86-SSE2-LABEL: test_reduce_v4i32:
 ; X86-SSE2:       ## %bb.0:
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm3
 ; X86-SSE2-NEXT:    pxor %xmm2, %xmm3
@@ -171,7 +171,7 @@ define i32 @test_reduce_v4i32(<4 x i32> %a0) {
 ;
 ; X86-SSE42-LABEL: test_reduce_v4i32:
 ; X86-SSE42:       ## %bb.0:
-; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE42-NEXT:    pminud %xmm0, %xmm1
 ; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; X86-SSE42-NEXT:    pminud %xmm1, %xmm0
@@ -180,7 +180,7 @@ define i32 @test_reduce_v4i32(<4 x i32> %a0) {
 ;
 ; X86-AVX-LABEL: test_reduce_v4i32:
 ; X86-AVX:       ## %bb.0:
-; X86-AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-AVX-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X86-AVX-NEXT:    vpminud %xmm1, %xmm0, %xmm0
@@ -189,7 +189,7 @@ define i32 @test_reduce_v4i32(<4 x i32> %a0) {
 ;
 ; X64-SSE2-LABEL: test_reduce_v4i32:
 ; X64-SSE2:       ## %bb.0:
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]
 ; X64-SSE2-NEXT:    movdqa %xmm0, %xmm3
 ; X64-SSE2-NEXT:    pxor %xmm2, %xmm3
@@ -212,7 +212,7 @@ define i32 @test_reduce_v4i32(<4 x i32> %a0) {
 ;
 ; X64-SSE42-LABEL: test_reduce_v4i32:
 ; X64-SSE42:       ## %bb.0:
-; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE42-NEXT:    pminud %xmm0, %xmm1
 ; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; X64-SSE42-NEXT:    pminud %xmm1, %xmm0
@@ -221,7 +221,7 @@ define i32 @test_reduce_v4i32(<4 x i32> %a0) {
 ;
 ; X64-AVX-LABEL: test_reduce_v4i32:
 ; X64-AVX:       ## %bb.0:
-; X64-AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; X64-AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X64-AVX-NEXT:    vpminud %xmm1, %xmm0, %xmm0
@@ -240,7 +240,7 @@ define i32 @test_reduce_v4i32(<4 x i32> %a0) {
 define i16 @test_reduce_v8i16(<8 x i16> %a0) {
 ; X86-SSE2-LABEL: test_reduce_v8i16:
 ; X86-SSE2:       ## %bb.0:
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [32768,32768,32768,32768,32768,32768,32768,32768]
 ; X86-SSE2-NEXT:    pxor %xmm2, %xmm0
 ; X86-SSE2-NEXT:    pxor %xmm2, %xmm1
@@ -271,7 +271,7 @@ define i16 @test_reduce_v8i16(<8 x i16> %a0) {
 ;
 ; X64-SSE2-LABEL: test_reduce_v8i16:
 ; X64-SSE2:       ## %bb.0:
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [32768,32768,32768,32768,32768,32768,32768,32768]
 ; X64-SSE2-NEXT:    pxor %xmm2, %xmm0
 ; X64-SSE2-NEXT:    pxor %xmm2, %xmm1
@@ -315,7 +315,7 @@ define i16 @test_reduce_v8i16(<8 x i16> %a0) {
 define i8 @test_reduce_v16i8(<16 x i8> %a0) {
 ; X86-SSE2-LABEL: test_reduce_v16i8:
 ; X86-SSE2:       ## %bb.0:
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    pminub %xmm0, %xmm1
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; X86-SSE2-NEXT:    pminub %xmm1, %xmm0
@@ -350,7 +350,7 @@ define i8 @test_reduce_v16i8(<16 x i8> %a0) {
 ;
 ; X64-SSE2-LABEL: test_reduce_v16i8:
 ; X64-SSE2:       ## %bb.0:
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    pminub %xmm0, %xmm1
 ; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; X64-SSE2-NEXT:    pminub %xmm1, %xmm0
@@ -421,7 +421,7 @@ define i64 @test_reduce_v4i64(<4 x i64> %a0) {
 ; X86-SSE2-NEXT:    pand %xmm4, %xmm0
 ; X86-SSE2-NEXT:    pandn %xmm1, %xmm4
 ; X86-SSE2-NEXT:    por %xmm0, %xmm4
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[2,3,2,3]
 ; X86-SSE2-NEXT:    movdqa %xmm4, %xmm1
 ; X86-SSE2-NEXT:    pxor %xmm2, %xmm1
 ; X86-SSE2-NEXT:    pxor %xmm0, %xmm2
@@ -451,7 +451,7 @@ define i64 @test_reduce_v4i64(<4 x i64> %a0) {
 ; X86-SSE42-NEXT:    pxor %xmm3, %xmm0
 ; X86-SSE42-NEXT:    pcmpgtq %xmm4, %xmm0
 ; X86-SSE42-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
-; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; X86-SSE42-NEXT:    movdqa %xmm1, %xmm0
 ; X86-SSE42-NEXT:    pxor %xmm3, %xmm0
 ; X86-SSE42-NEXT:    pxor %xmm2, %xmm3
@@ -471,7 +471,7 @@ define i64 @test_reduce_v4i64(<4 x i64> %a0) {
 ; X86-AVX1-NEXT:    vxorps %xmm1, %xmm3, %xmm4
 ; X86-AVX1-NEXT:    vpcmpgtq %xmm2, %xmm4, %xmm2
 ; X86-AVX1-NEXT:    vblendvpd %xmm2, %xmm0, %xmm3, %xmm0
-; X86-AVX1-NEXT:    vpermilps {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; X86-AVX1-NEXT:    vpermilps {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; X86-AVX1-NEXT:    vxorpd %xmm1, %xmm0, %xmm3
 ; X86-AVX1-NEXT:    vxorpd %xmm1, %xmm2, %xmm1
 ; X86-AVX1-NEXT:    vpcmpgtq %xmm3, %xmm1, %xmm1
@@ -489,7 +489,7 @@ define i64 @test_reduce_v4i64(<4 x i64> %a0) {
 ; X86-AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm4
 ; X86-AVX2-NEXT:    vpcmpgtq %xmm3, %xmm4, %xmm3
 ; X86-AVX2-NEXT:    vblendvpd %xmm3, %xmm0, %xmm1, %xmm0
-; X86-AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-AVX2-NEXT:    vxorpd %xmm2, %xmm0, %xmm3
 ; X86-AVX2-NEXT:    vxorpd %xmm2, %xmm1, %xmm2
 ; X86-AVX2-NEXT:    vpcmpgtq %xmm3, %xmm2, %xmm2
@@ -517,7 +517,7 @@ define i64 @test_reduce_v4i64(<4 x i64> %a0) {
 ; X64-SSE2-NEXT:    pand %xmm4, %xmm0
 ; X64-SSE2-NEXT:    pandn %xmm1, %xmm4
 ; X64-SSE2-NEXT:    por %xmm0, %xmm4
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[2,3,2,3]
 ; X64-SSE2-NEXT:    movdqa %xmm4, %xmm1
 ; X64-SSE2-NEXT:    pxor %xmm2, %xmm1
 ; X64-SSE2-NEXT:    pxor %xmm0, %xmm2
@@ -545,7 +545,7 @@ define i64 @test_reduce_v4i64(<4 x i64> %a0) {
 ; X64-SSE42-NEXT:    pxor %xmm3, %xmm0
 ; X64-SSE42-NEXT:    pcmpgtq %xmm4, %xmm0
 ; X64-SSE42-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
-; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; X64-SSE42-NEXT:    movdqa %xmm1, %xmm0
 ; X64-SSE42-NEXT:    pxor %xmm3, %xmm0
 ; X64-SSE42-NEXT:    pxor %xmm2, %xmm3
@@ -563,7 +563,7 @@ define i64 @test_reduce_v4i64(<4 x i64> %a0) {
 ; X64-AVX1-NEXT:    vpxor %xmm1, %xmm3, %xmm4
 ; X64-AVX1-NEXT:    vpcmpgtq %xmm2, %xmm4, %xmm2
 ; X64-AVX1-NEXT:    vblendvpd %xmm2, %xmm0, %xmm3, %xmm0
-; X64-AVX1-NEXT:    vpermilps {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; X64-AVX1-NEXT:    vpermilps {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; X64-AVX1-NEXT:    vxorpd %xmm1, %xmm0, %xmm3
 ; X64-AVX1-NEXT:    vxorpd %xmm1, %xmm2, %xmm1
 ; X64-AVX1-NEXT:    vpcmpgtq %xmm3, %xmm1, %xmm1
@@ -580,7 +580,7 @@ define i64 @test_reduce_v4i64(<4 x i64> %a0) {
 ; X64-AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm4
 ; X64-AVX2-NEXT:    vpcmpgtq %xmm3, %xmm4, %xmm3
 ; X64-AVX2-NEXT:    vblendvpd %xmm3, %xmm0, %xmm1, %xmm0
-; X64-AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX2-NEXT:    vxorpd %xmm2, %xmm0, %xmm3
 ; X64-AVX2-NEXT:    vxorpd %xmm2, %xmm1, %xmm2
 ; X64-AVX2-NEXT:    vpcmpgtq %xmm3, %xmm2, %xmm2
@@ -593,7 +593,7 @@ define i64 @test_reduce_v4i64(<4 x i64> %a0) {
 ; X64-AVX512:       ## %bb.0:
 ; X64-AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X64-AVX512-NEXT:    vpminuq %xmm1, %xmm0, %xmm0
-; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX512-NEXT:    vpminuq %xmm1, %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vmovq %xmm0, %rax
 ; X64-AVX512-NEXT:    vzeroupper
@@ -620,7 +620,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X86-SSE2-NEXT:    pand %xmm4, %xmm0
 ; X86-SSE2-NEXT:    pandn %xmm1, %xmm4
 ; X86-SSE2-NEXT:    por %xmm0, %xmm4
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[2,3,2,3]
 ; X86-SSE2-NEXT:    movdqa %xmm4, %xmm1
 ; X86-SSE2-NEXT:    pxor %xmm2, %xmm1
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm3
@@ -643,7 +643,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X86-SSE42-LABEL: test_reduce_v8i32:
 ; X86-SSE42:       ## %bb.0:
 ; X86-SSE42-NEXT:    pminud %xmm1, %xmm0
-; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE42-NEXT:    pminud %xmm0, %xmm1
 ; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; X86-SSE42-NEXT:    pminud %xmm1, %xmm0
@@ -654,7 +654,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X86-AVX1:       ## %bb.0:
 ; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; X86-AVX1-NEXT:    vpminud %xmm1, %xmm0, %xmm0
-; X86-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-AVX1-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X86-AVX1-NEXT:    vpminud %xmm1, %xmm0, %xmm0
@@ -666,7 +666,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X86-AVX2:       ## %bb.0:
 ; X86-AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X86-AVX2-NEXT:    vpminud %xmm1, %xmm0, %xmm0
-; X86-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-AVX2-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; X86-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X86-AVX2-NEXT:    vpminud %xmm1, %xmm0, %xmm0
@@ -685,7 +685,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X64-SSE2-NEXT:    pand %xmm4, %xmm0
 ; X64-SSE2-NEXT:    pandn %xmm1, %xmm4
 ; X64-SSE2-NEXT:    por %xmm0, %xmm4
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[2,3,2,3]
 ; X64-SSE2-NEXT:    movdqa %xmm4, %xmm1
 ; X64-SSE2-NEXT:    pxor %xmm2, %xmm1
 ; X64-SSE2-NEXT:    movdqa %xmm0, %xmm3
@@ -708,7 +708,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X64-SSE42-LABEL: test_reduce_v8i32:
 ; X64-SSE42:       ## %bb.0:
 ; X64-SSE42-NEXT:    pminud %xmm1, %xmm0
-; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE42-NEXT:    pminud %xmm0, %xmm1
 ; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; X64-SSE42-NEXT:    pminud %xmm1, %xmm0
@@ -719,7 +719,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X64-AVX1:       ## %bb.0:
 ; X64-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; X64-AVX1-NEXT:    vpminud %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX1-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X64-AVX1-NEXT:    vpminud %xmm1, %xmm0, %xmm0
@@ -731,7 +731,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X64-AVX2:       ## %bb.0:
 ; X64-AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X64-AVX2-NEXT:    vpminud %xmm1, %xmm0, %xmm0
-; X64-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX2-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; X64-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X64-AVX2-NEXT:    vpminud %xmm1, %xmm0, %xmm0
@@ -743,7 +743,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X64-AVX512:       ## %bb.0:
 ; X64-AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X64-AVX512-NEXT:    vpminud %xmm1, %xmm0, %xmm0
-; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX512-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X64-AVX512-NEXT:    vpminud %xmm1, %xmm0, %xmm0
@@ -770,7 +770,7 @@ define i16 @test_reduce_v16i16(<16 x i16> %a0) {
 ; X86-SSE2-NEXT:    pxor %xmm2, %xmm1
 ; X86-SSE2-NEXT:    pxor %xmm2, %xmm0
 ; X86-SSE2-NEXT:    pminsw %xmm1, %xmm0
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    pminsw %xmm0, %xmm1
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; X86-SSE2-NEXT:    pminsw %xmm1, %xmm0
@@ -816,7 +816,7 @@ define i16 @test_reduce_v16i16(<16 x i16> %a0) {
 ; X64-SSE2-NEXT:    pxor %xmm2, %xmm1
 ; X64-SSE2-NEXT:    pxor %xmm2, %xmm0
 ; X64-SSE2-NEXT:    pminsw %xmm1, %xmm0
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    pminsw %xmm0, %xmm1
 ; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; X64-SSE2-NEXT:    pminsw %xmm1, %xmm0
@@ -885,7 +885,7 @@ define i8 @test_reduce_v32i8(<32 x i8> %a0) {
 ; X86-SSE2-LABEL: test_reduce_v32i8:
 ; X86-SSE2:       ## %bb.0:
 ; X86-SSE2-NEXT:    pminub %xmm1, %xmm0
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    pminub %xmm0, %xmm1
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; X86-SSE2-NEXT:    pminub %xmm1, %xmm0
@@ -937,7 +937,7 @@ define i8 @test_reduce_v32i8(<32 x i8> %a0) {
 ; X64-SSE2-LABEL: test_reduce_v32i8:
 ; X64-SSE2:       ## %bb.0:
 ; X64-SSE2-NEXT:    pminub %xmm1, %xmm0
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    pminub %xmm0, %xmm1
 ; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; X64-SSE2-NEXT:    pminub %xmm1, %xmm0
@@ -1069,7 +1069,7 @@ define i64 @test_reduce_v8i64(<8 x i64> %a0) {
 ; X86-SSE2-NEXT:    pand %xmm0, %xmm1
 ; X86-SSE2-NEXT:    pandn %xmm5, %xmm0
 ; X86-SSE2-NEXT:    por %xmm1, %xmm0
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; X86-SSE2-NEXT:    pxor %xmm4, %xmm2
 ; X86-SSE2-NEXT:    pxor %xmm1, %xmm4
@@ -1111,7 +1111,7 @@ define i64 @test_reduce_v8i64(<8 x i64> %a0) {
 ; X86-SSE42-NEXT:    xorpd %xmm4, %xmm0
 ; X86-SSE42-NEXT:    pcmpgtq %xmm1, %xmm0
 ; X86-SSE42-NEXT:    blendvpd %xmm0, %xmm2, %xmm3
-; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[2,3,0,1]
+; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[2,3,2,3]
 ; X86-SSE42-NEXT:    movdqa %xmm3, %xmm0
 ; X86-SSE42-NEXT:    pxor %xmm4, %xmm0
 ; X86-SSE42-NEXT:    pxor %xmm1, %xmm4
@@ -1140,7 +1140,7 @@ define i64 @test_reduce_v8i64(<8 x i64> %a0) {
 ; X86-AVX1-NEXT:    vxorpd %xmm3, %xmm2, %xmm4
 ; X86-AVX1-NEXT:    vpcmpgtq %xmm1, %xmm4, %xmm1
 ; X86-AVX1-NEXT:    vblendvpd %xmm1, %xmm0, %xmm2, %xmm0
-; X86-AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-AVX1-NEXT:    vxorpd %xmm3, %xmm0, %xmm2
 ; X86-AVX1-NEXT:    vxorpd %xmm3, %xmm1, %xmm3
 ; X86-AVX1-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm2
@@ -1162,7 +1162,7 @@ define i64 @test_reduce_v8i64(<8 x i64> %a0) {
 ; X86-AVX2-NEXT:    vxorpd %xmm2, %xmm1, %xmm4
 ; X86-AVX2-NEXT:    vpcmpgtq %xmm3, %xmm4, %xmm3
 ; X86-AVX2-NEXT:    vblendvpd %xmm3, %xmm0, %xmm1, %xmm0
-; X86-AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-AVX2-NEXT:    vxorpd %xmm2, %xmm0, %xmm3
 ; X86-AVX2-NEXT:    vxorpd %xmm2, %xmm1, %xmm2
 ; X86-AVX2-NEXT:    vpcmpgtq %xmm3, %xmm2, %xmm2
@@ -1220,7 +1220,7 @@ define i64 @test_reduce_v8i64(<8 x i64> %a0) {
 ; X64-SSE2-NEXT:    pand %xmm1, %xmm3
 ; X64-SSE2-NEXT:    pandn %xmm5, %xmm1
 ; X64-SSE2-NEXT:    por %xmm3, %xmm1
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; X64-SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; X64-SSE2-NEXT:    pxor %xmm4, %xmm2
 ; X64-SSE2-NEXT:    pxor %xmm0, %xmm4
@@ -1260,7 +1260,7 @@ define i64 @test_reduce_v8i64(<8 x i64> %a0) {
 ; X64-SSE42-NEXT:    xorpd %xmm4, %xmm0
 ; X64-SSE42-NEXT:    pcmpgtq %xmm1, %xmm0
 ; X64-SSE42-NEXT:    blendvpd %xmm0, %xmm2, %xmm3
-; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[2,3,0,1]
+; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[2,3,2,3]
 ; X64-SSE42-NEXT:    movdqa %xmm3, %xmm0
 ; X64-SSE42-NEXT:    pxor %xmm4, %xmm0
 ; X64-SSE42-NEXT:    pxor %xmm1, %xmm4
@@ -1287,7 +1287,7 @@ define i64 @test_reduce_v8i64(<8 x i64> %a0) {
 ; X64-AVX1-NEXT:    vxorpd %xmm3, %xmm2, %xmm4
 ; X64-AVX1-NEXT:    vpcmpgtq %xmm1, %xmm4, %xmm1
 ; X64-AVX1-NEXT:    vblendvpd %xmm1, %xmm0, %xmm2, %xmm0
-; X64-AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX1-NEXT:    vxorpd %xmm3, %xmm0, %xmm2
 ; X64-AVX1-NEXT:    vxorpd %xmm3, %xmm1, %xmm3
 ; X64-AVX1-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm2
@@ -1308,7 +1308,7 @@ define i64 @test_reduce_v8i64(<8 x i64> %a0) {
 ; X64-AVX2-NEXT:    vxorpd %xmm2, %xmm1, %xmm4
 ; X64-AVX2-NEXT:    vpcmpgtq %xmm3, %xmm4, %xmm3
 ; X64-AVX2-NEXT:    vblendvpd %xmm3, %xmm0, %xmm1, %xmm0
-; X64-AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX2-NEXT:    vxorpd %xmm2, %xmm0, %xmm3
 ; X64-AVX2-NEXT:    vxorpd %xmm2, %xmm1, %xmm2
 ; X64-AVX2-NEXT:    vpcmpgtq %xmm3, %xmm2, %xmm2
@@ -1323,7 +1323,7 @@ define i64 @test_reduce_v8i64(<8 x i64> %a0) {
 ; X64-AVX512-NEXT:    vpminuq %zmm1, %zmm0, %zmm0
 ; X64-AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X64-AVX512-NEXT:    vpminuq %xmm1, %xmm0, %xmm0
-; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX512-NEXT:    vpminuq %xmm1, %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vmovq %xmm0, %rax
 ; X64-AVX512-NEXT:    vzeroupper
@@ -1369,7 +1369,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X86-SSE2-NEXT:    pand %xmm1, %xmm3
 ; X86-SSE2-NEXT:    pandn %xmm6, %xmm1
 ; X86-SSE2-NEXT:    por %xmm3, %xmm1
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; X86-SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; X86-SSE2-NEXT:    pxor %xmm4, %xmm2
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm3
@@ -1394,7 +1394,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X86-SSE42-NEXT:    pminud %xmm3, %xmm1
 ; X86-SSE42-NEXT:    pminud %xmm2, %xmm1
 ; X86-SSE42-NEXT:    pminud %xmm0, %xmm1
-; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; X86-SSE42-NEXT:    pminud %xmm1, %xmm0
 ; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X86-SSE42-NEXT:    pminud %xmm0, %xmm1
@@ -1408,7 +1408,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X86-AVX1-NEXT:    vpminud %xmm2, %xmm3, %xmm2
 ; X86-AVX1-NEXT:    vpminud %xmm2, %xmm1, %xmm1
 ; X86-AVX1-NEXT:    vpminud %xmm1, %xmm0, %xmm0
-; X86-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-AVX1-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X86-AVX1-NEXT:    vpminud %xmm1, %xmm0, %xmm0
@@ -1421,7 +1421,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X86-AVX2-NEXT:    vpminud %ymm1, %ymm0, %ymm0
 ; X86-AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X86-AVX2-NEXT:    vpminud %xmm1, %xmm0, %xmm0
-; X86-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-AVX2-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; X86-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X86-AVX2-NEXT:    vpminud %xmm1, %xmm0, %xmm0
@@ -1456,7 +1456,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X64-SSE2-NEXT:    pand %xmm1, %xmm3
 ; X64-SSE2-NEXT:    pandn %xmm6, %xmm1
 ; X64-SSE2-NEXT:    por %xmm3, %xmm1
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; X64-SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; X64-SSE2-NEXT:    pxor %xmm4, %xmm2
 ; X64-SSE2-NEXT:    movdqa %xmm0, %xmm3
@@ -1481,7 +1481,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X64-SSE42-NEXT:    pminud %xmm3, %xmm1
 ; X64-SSE42-NEXT:    pminud %xmm2, %xmm1
 ; X64-SSE42-NEXT:    pminud %xmm0, %xmm1
-; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; X64-SSE42-NEXT:    pminud %xmm1, %xmm0
 ; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X64-SSE42-NEXT:    pminud %xmm0, %xmm1
@@ -1495,7 +1495,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X64-AVX1-NEXT:    vpminud %xmm2, %xmm3, %xmm2
 ; X64-AVX1-NEXT:    vpminud %xmm2, %xmm1, %xmm1
 ; X64-AVX1-NEXT:    vpminud %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX1-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X64-AVX1-NEXT:    vpminud %xmm1, %xmm0, %xmm0
@@ -1508,7 +1508,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X64-AVX2-NEXT:    vpminud %ymm1, %ymm0, %ymm0
 ; X64-AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X64-AVX2-NEXT:    vpminud %xmm1, %xmm0, %xmm0
-; X64-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX2-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; X64-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X64-AVX2-NEXT:    vpminud %xmm1, %xmm0, %xmm0
@@ -1522,7 +1522,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X64-AVX512-NEXT:    vpminud %zmm1, %zmm0, %zmm0
 ; X64-AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X64-AVX512-NEXT:    vpminud %xmm1, %xmm0, %xmm0
-; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX512-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X64-AVX512-NEXT:    vpminud %xmm1, %xmm0, %xmm0
@@ -1556,7 +1556,7 @@ define i16 @test_reduce_v32i16(<32 x i16> %a0) {
 ; X86-SSE2-NEXT:    pminsw %xmm1, %xmm2
 ; X86-SSE2-NEXT:    pxor %xmm4, %xmm0
 ; X86-SSE2-NEXT:    pminsw %xmm2, %xmm0
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    pminsw %xmm0, %xmm1
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; X86-SSE2-NEXT:    pminsw %xmm1, %xmm0
@@ -1612,7 +1612,7 @@ define i16 @test_reduce_v32i16(<32 x i16> %a0) {
 ; X64-SSE2-NEXT:    pminsw %xmm1, %xmm2
 ; X64-SSE2-NEXT:    pxor %xmm4, %xmm0
 ; X64-SSE2-NEXT:    pminsw %xmm2, %xmm0
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    pminsw %xmm0, %xmm1
 ; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; X64-SSE2-NEXT:    pminsw %xmm1, %xmm0
@@ -1694,7 +1694,7 @@ define i8 @test_reduce_v64i8(<64 x i8> %a0) {
 ; X86-SSE2-NEXT:    pminub %xmm3, %xmm1
 ; X86-SSE2-NEXT:    pminub %xmm2, %xmm1
 ; X86-SSE2-NEXT:    pminub %xmm0, %xmm1
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; X86-SSE2-NEXT:    pminub %xmm1, %xmm0
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X86-SSE2-NEXT:    pminub %xmm0, %xmm1
@@ -1754,7 +1754,7 @@ define i8 @test_reduce_v64i8(<64 x i8> %a0) {
 ; X64-SSE2-NEXT:    pminub %xmm3, %xmm1
 ; X64-SSE2-NEXT:    pminub %xmm2, %xmm1
 ; X64-SSE2-NEXT:    pminub %xmm0, %xmm1
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; X64-SSE2-NEXT:    pminub %xmm1, %xmm0
 ; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X64-SSE2-NEXT:    pminub %xmm0, %xmm1
@@ -1851,7 +1851,7 @@ define i8 @test_reduce_v64i8(<64 x i8> %a0) {
 define i16 @test_reduce_v16i16_v8i16(<16 x i16> %a0) {
 ; X86-SSE2-LABEL: test_reduce_v16i16_v8i16:
 ; X86-SSE2:       ## %bb.0:
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [32768,32768,32768,32768,32768,32768,32768,32768]
 ; X86-SSE2-NEXT:    pxor %xmm2, %xmm0
 ; X86-SSE2-NEXT:    pxor %xmm2, %xmm1
@@ -1883,7 +1883,7 @@ define i16 @test_reduce_v16i16_v8i16(<16 x i16> %a0) {
 ;
 ; X64-SSE2-LABEL: test_reduce_v16i16_v8i16:
 ; X64-SSE2:       ## %bb.0:
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [32768,32768,32768,32768,32768,32768,32768,32768]
 ; X64-SSE2-NEXT:    pxor %xmm2, %xmm0
 ; X64-SSE2-NEXT:    pxor %xmm2, %xmm1
@@ -1928,7 +1928,7 @@ define i16 @test_reduce_v16i16_v8i16(<16 x i16> %a0) {
 define i16 @test_reduce_v32i16_v8i16(<32 x i16> %a0) {
 ; X86-SSE2-LABEL: test_reduce_v32i16_v8i16:
 ; X86-SSE2:       ## %bb.0:
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [32768,32768,32768,32768,32768,32768,32768,32768]
 ; X86-SSE2-NEXT:    pxor %xmm2, %xmm0
 ; X86-SSE2-NEXT:    pxor %xmm2, %xmm1
@@ -1960,7 +1960,7 @@ define i16 @test_reduce_v32i16_v8i16(<32 x i16> %a0) {
 ;
 ; X64-SSE2-LABEL: test_reduce_v32i16_v8i16:
 ; X64-SSE2:       ## %bb.0:
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [32768,32768,32768,32768,32768,32768,32768,32768]
 ; X64-SSE2-NEXT:    pxor %xmm2, %xmm0
 ; X64-SSE2-NEXT:    pxor %xmm2, %xmm1
@@ -2005,7 +2005,7 @@ define i16 @test_reduce_v32i16_v8i16(<32 x i16> %a0) {
 define i8 @test_reduce_v32i8_v16i8(<32 x i8> %a0) {
 ; X86-SSE2-LABEL: test_reduce_v32i8_v16i8:
 ; X86-SSE2:       ## %bb.0:
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    pminub %xmm0, %xmm1
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; X86-SSE2-NEXT:    pminub %xmm1, %xmm0
@@ -2041,7 +2041,7 @@ define i8 @test_reduce_v32i8_v16i8(<32 x i8> %a0) {
 ;
 ; X64-SSE2-LABEL: test_reduce_v32i8_v16i8:
 ; X64-SSE2:       ## %bb.0:
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    pminub %xmm0, %xmm1
 ; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; X64-SSE2-NEXT:    pminub %xmm1, %xmm0
@@ -2093,7 +2093,7 @@ define i8 @test_reduce_v32i8_v16i8(<32 x i8> %a0) {
 define i8 @test_reduce_v64i8_v16i8(<64 x i8> %a0) {
 ; X86-SSE2-LABEL: test_reduce_v64i8_v16i8:
 ; X86-SSE2:       ## %bb.0:
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    pminub %xmm0, %xmm1
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; X86-SSE2-NEXT:    pminub %xmm1, %xmm0
@@ -2129,7 +2129,7 @@ define i8 @test_reduce_v64i8_v16i8(<64 x i8> %a0) {
 ;
 ; X64-SSE2-LABEL: test_reduce_v64i8_v16i8:
 ; X64-SSE2:       ## %bb.0:
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    pminub %xmm0, %xmm1
 ; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; X64-SSE2-NEXT:    pminub %xmm1, %xmm0
diff --git a/llvm/test/CodeGen/X86/i128-add.ll b/llvm/test/CodeGen/X86/i128-add.ll
index 8e75c4a575ee4..d128d75e64573 100644
--- a/llvm/test/CodeGen/X86/i128-add.ll
+++ b/llvm/test/CodeGen/X86/i128-add.ll
@@ -77,7 +77,7 @@ define <1 x i128> @add_v1i128(<1 x i128> %x, <1 x i128> %y) nounwind {
 ; X64-NEXT:    movq %rax, %xmm0
 ; X64-NEXT:    movq %rsi, %xmm1
 ; X64-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; X64-NEXT:    movq %xmm0, %rdx
 ; X64-NEXT:    addq $1, %rax
 ; X64-NEXT:    adcq $0, %rdx
diff --git a/llvm/test/CodeGen/X86/inline-asm-x-i128.ll b/llvm/test/CodeGen/X86/inline-asm-x-i128.ll
index 552ce96a53a51..7aee1d175494e 100644
--- a/llvm/test/CodeGen/X86/inline-asm-x-i128.ll
+++ b/llvm/test/CodeGen/X86/inline-asm-x-i128.ll
@@ -16,7 +16,7 @@ define { i64, i64 } @foo(i64 %0, i64 %1) {
 ; CHECK-NEXT:    movdqa %xmm1, %xmm0
 ; CHECK-NEXT:    #NO_APP
 ; CHECK-NEXT:    movq %xmm0, %rax
-; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; CHECK-NEXT:    punpckhqdq {{.*#+}} xmm0 = xmm0[1,1]
 ; CHECK-NEXT:    movq %xmm0, %rdx
 ; CHECK-NEXT:    retq
   %3 = zext i64 %1 to i128
diff --git a/llvm/test/CodeGen/X86/known-bits-vector.ll b/llvm/test/CodeGen/X86/known-bits-vector.ll
index a1606e93e2e23..3b6912a9d9461 100644
--- a/llvm/test/CodeGen/X86/known-bits-vector.ll
+++ b/llvm/test/CodeGen/X86/known-bits-vector.ll
@@ -116,14 +116,14 @@ define <4 x i32> @knownbits_mask_shuffle_shuffle_undef_sext(<8 x i16> %a0) nounw
 ; X32-LABEL: knownbits_mask_shuffle_shuffle_undef_sext:
 ; X32:       # %bb.0:
 ; X32-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
-; X32-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; X32-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; X32-NEXT:    vpmovsxwd %xmm0, %xmm0
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: knownbits_mask_shuffle_shuffle_undef_sext:
 ; X64:       # %bb.0:
 ; X64-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; X64-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; X64-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; X64-NEXT:    vpmovsxwd %xmm0, %xmm0
 ; X64-NEXT:    retq
   %1 = and <8 x i16> %a0, <i16 -1, i16 -1, i16 -1, i16 -1, i16 15, i16 15, i16 15, i16 15>
diff --git a/llvm/test/CodeGen/X86/known-signbits-vector.ll b/llvm/test/CodeGen/X86/known-signbits-vector.ll
index b18b8079fd236..6c4d0a919ef0f 100644
--- a/llvm/test/CodeGen/X86/known-signbits-vector.ll
+++ b/llvm/test/CodeGen/X86/known-signbits-vector.ll
@@ -252,7 +252,7 @@ define <4 x double> @signbits_sext_shuffle_sitofp(<4 x i32> %a0, <4 x i64> %a1)
 ; X86-LABEL: signbits_sext_shuffle_sitofp:
 ; X86:       # %bb.0:
 ; X86-NEXT:    vpmovsxdq %xmm0, %xmm1
-; X86-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; X86-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; X86-NEXT:    vpmovsxdq %xmm0, %xmm0
 ; X86-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; X86-NEXT:    vpermilpd {{.*#+}} ymm0 = ymm0[1,0,3,2]
@@ -264,7 +264,7 @@ define <4 x double> @signbits_sext_shuffle_sitofp(<4 x i32> %a0, <4 x i64> %a1)
 ; X64-AVX1-LABEL: signbits_sext_shuffle_sitofp:
 ; X64-AVX1:       # %bb.0:
 ; X64-AVX1-NEXT:    vpmovsxdq %xmm0, %xmm1
-; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; X64-AVX1-NEXT:    vpmovsxdq %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; X64-AVX1-NEXT:    vpermilpd {{.*#+}} ymm0 = ymm0[1,0,3,2]
@@ -478,7 +478,7 @@ define <4 x float> @signbits_ashr_sext_select_shuffle_sitofp(<4 x i64> %a0, <4 x
 ; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
 ; X64-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1],xmm5[2,3],xmm2[4,5],xmm5[6,7]
 ; X64-AVX1-NEXT:    vpmovsxdq %xmm3, %xmm5
-; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,3,0,1]
+; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]
 ; X64-AVX1-NEXT:    vpmovsxdq %xmm3, %xmm3
 ; X64-AVX1-NEXT:    vpcmpeqq %xmm1, %xmm0, %xmm6
 ; X64-AVX1-NEXT:    vblendvpd %xmm6, %xmm4, %xmm5, %xmm4
diff --git a/llvm/test/CodeGen/X86/madd.ll b/llvm/test/CodeGen/X86/madd.ll
index 7bde3facc735e..93097e2b98fb7 100644
--- a/llvm/test/CodeGen/X86/madd.ll
+++ b/llvm/test/CodeGen/X86/madd.ll
@@ -25,7 +25,7 @@ define i32 @_Z10test_shortPsS_i_128(i16* nocapture readonly, i16* nocapture read
 ; SSE2-NEXT:    cmpq %rcx, %rax
 ; SSE2-NEXT:    jne .LBB0_1
 ; SSE2-NEXT:  # %bb.2: # %middle.block
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE2-NEXT:    paddd %xmm1, %xmm0
@@ -48,7 +48,7 @@ define i32 @_Z10test_shortPsS_i_128(i16* nocapture readonly, i16* nocapture read
 ; AVX-NEXT:    cmpq %rcx, %rax
 ; AVX-NEXT:    jne .LBB0_1
 ; AVX-NEXT:  # %bb.2: # %middle.block
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -103,7 +103,7 @@ define i32 @_Z10test_shortPsS_i_256(i16* nocapture readonly, i16* nocapture read
 ; SSE2-NEXT:    jne .LBB1_1
 ; SSE2-NEXT:  # %bb.2: # %middle.block
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    paddd %xmm1, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
@@ -128,7 +128,7 @@ define i32 @_Z10test_shortPsS_i_256(i16* nocapture readonly, i16* nocapture read
 ; AVX1-NEXT:  # %bb.2: # %middle.block
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -153,7 +153,7 @@ define i32 @_Z10test_shortPsS_i_256(i16* nocapture readonly, i16* nocapture read
 ; AVX256-NEXT:  # %bb.2: # %middle.block
 ; AVX256-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX256-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX256-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX256-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX256-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX256-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX256-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -218,7 +218,7 @@ define i32 @_Z10test_shortPsS_i_512(i16* nocapture readonly, i16* nocapture read
 ; SSE2-NEXT:    paddd %xmm0, %xmm2
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
 ; SSE2-NEXT:    paddd %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    paddd %xmm1, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
@@ -251,7 +251,7 @@ define i32 @_Z10test_shortPsS_i_512(i16* nocapture readonly, i16* nocapture read
 ; AVX1-NEXT:    vpaddd %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vpaddd %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -278,7 +278,7 @@ define i32 @_Z10test_shortPsS_i_512(i16* nocapture readonly, i16* nocapture read
 ; AVX2-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -305,7 +305,7 @@ define i32 @_Z10test_shortPsS_i_512(i16* nocapture readonly, i16* nocapture read
 ; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -386,7 +386,7 @@ define i32 @_Z10test_shortPsS_i_1024(i16* nocapture readonly, i16* nocapture rea
 ; SSE2-NEXT:    paddd %xmm8, %xmm1
 ; SSE2-NEXT:    paddd %xmm3, %xmm1
 ; SSE2-NEXT:    paddd %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    paddd %xmm1, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
@@ -433,7 +433,7 @@ define i32 @_Z10test_shortPsS_i_1024(i16* nocapture readonly, i16* nocapture rea
 ; AVX1-NEXT:    vpaddd %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vpaddd %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -466,7 +466,7 @@ define i32 @_Z10test_shortPsS_i_1024(i16* nocapture readonly, i16* nocapture rea
 ; AVX2-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -498,7 +498,7 @@ define i32 @_Z10test_shortPsS_i_1024(i16* nocapture readonly, i16* nocapture rea
 ; AVX512F-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512F-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX512F-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512F-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512F-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512F-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512F-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -527,7 +527,7 @@ define i32 @_Z10test_shortPsS_i_1024(i16* nocapture readonly, i16* nocapture rea
 ; AVX512BW-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512BW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512BW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512BW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -593,7 +593,7 @@ define i32 @_Z9test_charPcS_i_128(i8* nocapture readonly, i8* nocapture readonly
 ; SSE2-NEXT:    cmpq %rcx, %rax
 ; SSE2-NEXT:    jne .LBB4_1
 ; SSE2-NEXT:  # %bb.2: # %middle.block
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE2-NEXT:    paddd %xmm1, %xmm0
@@ -616,7 +616,7 @@ define i32 @_Z9test_charPcS_i_128(i8* nocapture readonly, i8* nocapture readonly
 ; AVX-NEXT:    cmpq %rcx, %rax
 ; AVX-NEXT:    jne .LBB4_1
 ; AVX-NEXT:  # %bb.2: # %middle.block
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -675,7 +675,7 @@ define i32 @_Z9test_charPcS_i_256(i8* nocapture readonly, i8* nocapture readonly
 ; SSE2-NEXT:    jne .LBB5_1
 ; SSE2-NEXT:  # %bb.2: # %middle.block
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    paddd %xmm1, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
@@ -701,7 +701,7 @@ define i32 @_Z9test_charPcS_i_256(i8* nocapture readonly, i8* nocapture readonly
 ; AVX1-NEXT:  # %bb.2: # %middle.block
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -727,7 +727,7 @@ define i32 @_Z9test_charPcS_i_256(i8* nocapture readonly, i8* nocapture readonly
 ; AVX256-NEXT:  # %bb.2: # %middle.block
 ; AVX256-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX256-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX256-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX256-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX256-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX256-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX256-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -798,7 +798,7 @@ define i32 @_Z9test_charPcS_i_512(i8* nocapture readonly, i8* nocapture readonly
 ; SSE2-NEXT:    paddd %xmm0, %xmm2
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
 ; SSE2-NEXT:    paddd %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    paddd %xmm1, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
@@ -833,7 +833,7 @@ define i32 @_Z9test_charPcS_i_512(i8* nocapture readonly, i8* nocapture readonly
 ; AVX1-NEXT:    vpaddd %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vpaddd %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -861,7 +861,7 @@ define i32 @_Z9test_charPcS_i_512(i8* nocapture readonly, i8* nocapture readonly
 ; AVX2-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -889,7 +889,7 @@ define i32 @_Z9test_charPcS_i_512(i8* nocapture readonly, i8* nocapture readonly
 ; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -982,7 +982,7 @@ define i32 @_Z9test_charPcS_i_1024(i8* nocapture readonly, i8* nocapture readonl
 ; SSE2-NEXT:    paddd %xmm8, %xmm1
 ; SSE2-NEXT:    paddd %xmm3, %xmm1
 ; SSE2-NEXT:    paddd %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    paddd %xmm1, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
@@ -1033,7 +1033,7 @@ define i32 @_Z9test_charPcS_i_1024(i8* nocapture readonly, i8* nocapture readonl
 ; AVX1-NEXT:    vpaddd %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vpaddd %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -1068,7 +1068,7 @@ define i32 @_Z9test_charPcS_i_1024(i8* nocapture readonly, i8* nocapture readonl
 ; AVX2-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -1102,7 +1102,7 @@ define i32 @_Z9test_charPcS_i_1024(i8* nocapture readonly, i8* nocapture readonl
 ; AVX512F-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512F-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX512F-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512F-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512F-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512F-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512F-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -1132,7 +1132,7 @@ define i32 @_Z9test_charPcS_i_1024(i8* nocapture readonly, i8* nocapture readonl
 ; AVX512BW-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512BW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512BW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512BW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -1195,7 +1195,7 @@ define i32 @test_unsigned_short_128(i16* nocapture readonly, i16* nocapture read
 ; SSE2-NEXT:    cmpq %rcx, %rax
 ; SSE2-NEXT:    jne .LBB8_1
 ; SSE2-NEXT:  # %bb.2: # %middle.block
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE2-NEXT:    paddd %xmm1, %xmm0
@@ -1218,7 +1218,7 @@ define i32 @test_unsigned_short_128(i16* nocapture readonly, i16* nocapture read
 ; AVX-NEXT:    cmpq %rcx, %rax
 ; AVX-NEXT:    jne .LBB8_1
 ; AVX-NEXT:  # %bb.2: # %middle.block
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -1279,7 +1279,7 @@ define i32 @test_unsigned_short_256(i16* nocapture readonly, i16* nocapture read
 ; SSE2-NEXT:    jne .LBB9_1
 ; SSE2-NEXT:  # %bb.2: # %middle.block
 ; SSE2-NEXT:    paddd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE2-NEXT:    paddd %xmm1, %xmm0
@@ -1310,7 +1310,7 @@ define i32 @test_unsigned_short_256(i16* nocapture readonly, i16* nocapture read
 ; AVX1-NEXT:  # %bb.2: # %middle.block
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -1336,7 +1336,7 @@ define i32 @test_unsigned_short_256(i16* nocapture readonly, i16* nocapture read
 ; AVX256-NEXT:  # %bb.2: # %middle.block
 ; AVX256-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX256-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX256-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX256-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX256-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX256-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX256-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -1414,7 +1414,7 @@ define i32 @test_unsigned_short_512(i16* nocapture readonly, i16* nocapture read
 ; SSE2-NEXT:    paddd %xmm3, %xmm0
 ; SSE2-NEXT:    paddd %xmm2, %xmm1
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    paddd %xmm1, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
@@ -1459,7 +1459,7 @@ define i32 @test_unsigned_short_512(i16* nocapture readonly, i16* nocapture read
 ; AVX1-NEXT:    vpaddd %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -1491,7 +1491,7 @@ define i32 @test_unsigned_short_512(i16* nocapture readonly, i16* nocapture read
 ; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -1519,7 +1519,7 @@ define i32 @test_unsigned_short_512(i16* nocapture readonly, i16* nocapture read
 ; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -1627,7 +1627,7 @@ define i32 @test_unsigned_short_1024(i16* nocapture readonly, i16* nocapture rea
 ; SSE2-NEXT:    paddd %xmm5, %xmm9
 ; SSE2-NEXT:    paddd %xmm10, %xmm9
 ; SSE2-NEXT:    paddd %xmm8, %xmm9
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm9[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm9[2,3,2,3]
 ; SSE2-NEXT:    paddd %xmm9, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
@@ -1700,7 +1700,7 @@ define i32 @test_unsigned_short_1024(i16* nocapture readonly, i16* nocapture rea
 ; AVX1-NEXT:    vpaddd %xmm0, %xmm9, %xmm0
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpaddd %xmm0, %xmm8, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -1744,7 +1744,7 @@ define i32 @test_unsigned_short_1024(i16* nocapture readonly, i16* nocapture rea
 ; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -1778,7 +1778,7 @@ define i32 @test_unsigned_short_1024(i16* nocapture readonly, i16* nocapture rea
 ; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -2074,7 +2074,7 @@ define <4 x i32> @pmaddwd_negative2(<8 x i16> %A) {
 ;
 ; AVX1-LABEL: pmaddwd_negative2:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxwd %xmm1, %xmm1
 ; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm0
 ; AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
@@ -2647,7 +2647,7 @@ define i32 @madd_double_reduction(<8 x i16>* %arg, <8 x i16>* %arg1, <8 x i16>*
 ; SSE2-NEXT:    movdqu (%rcx), %xmm2
 ; SSE2-NEXT:    pmaddwd %xmm0, %xmm2
 ; SSE2-NEXT:    paddd %xmm1, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE2-NEXT:    paddd %xmm2, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
@@ -2661,7 +2661,7 @@ define i32 @madd_double_reduction(<8 x i16>* %arg, <8 x i16>* %arg1, <8 x i16>*
 ; AVX-NEXT:    vmovdqu (%rdx), %xmm1
 ; AVX-NEXT:    vpmaddwd (%rcx), %xmm1, %xmm1
 ; AVX-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -2708,7 +2708,7 @@ define i32 @madd_quad_reduction(<8 x i16>* %arg, <8 x i16>* %arg1, <8 x i16>* %a
 ; SSE2-NEXT:    pmaddwd %xmm0, %xmm1
 ; SSE2-NEXT:    paddd %xmm3, %xmm1
 ; SSE2-NEXT:    paddd %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    paddd %xmm1, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
@@ -2730,7 +2730,7 @@ define i32 @madd_quad_reduction(<8 x i16>* %arg, <8 x i16>* %arg1, <8 x i16>* %a
 ; AVX-NEXT:    vpmaddwd (%r10), %xmm2, %xmm2
 ; AVX-NEXT:    vpaddd %xmm2, %xmm0, %xmm0
 ; AVX-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -2798,13 +2798,13 @@ define i64 @sum_and_sum_of_squares(i8* %a, i32 %n) {
 ; SSE2-NEXT:    jne .LBB33_1
 ; SSE2-NEXT:  # %bb.2: # %middle.block
 ; SSE2-NEXT:    paddd %xmm3, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
 ; SSE2-NEXT:    paddd %xmm2, %xmm3
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,1,2,3]
 ; SSE2-NEXT:    paddd %xmm3, %xmm2
 ; SSE2-NEXT:    movd %xmm2, %ecx
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    paddd %xmm1, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
@@ -2839,14 +2839,14 @@ define i64 @sum_and_sum_of_squares(i8* %a, i32 %n) {
 ; AVX1-NEXT:  # %bb.2: # %middle.block
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
 ; AVX1-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; AVX1-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,2,3]
 ; AVX1-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vmovd %xmm1, %ecx
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -2874,14 +2874,14 @@ define i64 @sum_and_sum_of_squares(i8* %a, i32 %n) {
 ; AVX256-NEXT:  # %bb.2: # %middle.block
 ; AVX256-NEXT:    vextracti128 $1, %ymm1, %xmm2
 ; AVX256-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
-; AVX256-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; AVX256-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; AVX256-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
 ; AVX256-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,2,3]
 ; AVX256-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
 ; AVX256-NEXT:    vmovd %xmm1, %ecx
 ; AVX256-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX256-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX256-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX256-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX256-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX256-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX256-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -2953,7 +2953,7 @@ define i32 @sum_of_square_differences(i8* %a, i8* %b, i32 %n) {
 ; SSE2-NEXT:    jne .LBB34_1
 ; SSE2-NEXT:  # %bb.2: # %middle.block
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    paddd %xmm1, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
@@ -2980,7 +2980,7 @@ define i32 @sum_of_square_differences(i8* %a, i8* %b, i32 %n) {
 ; AVX1-NEXT:  # %bb.2: # %middle.block
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -3007,7 +3007,7 @@ define i32 @sum_of_square_differences(i8* %a, i8* %b, i32 %n) {
 ; AVX256-NEXT:  # %bb.2: # %middle.block
 ; AVX256-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX256-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX256-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX256-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX256-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX256-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX256-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
diff --git a/llvm/test/CodeGen/X86/masked_compressstore.ll b/llvm/test/CodeGen/X86/masked_compressstore.ll
index 24ae0c77af2fe..c93543ddda7ae 100644
--- a/llvm/test/CodeGen/X86/masked_compressstore.ll
+++ b/llvm/test/CodeGen/X86/masked_compressstore.ll
@@ -2478,7 +2478,7 @@ define void @compressstore_v2i64_v2i1(i64* %base, <2 x i64> %V, <2 x i1> %mask)
 ; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je LBB7_4
 ; SSE2-NEXT:  LBB7_3: ## %cond.store1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm0, (%rdi)
 ; SSE2-NEXT:    retq
 ;
@@ -2574,7 +2574,7 @@ define void @compressstore_v4i64_v4i1(i64* %base, <4 x i64> %V, <4 x i1> %mask)
 ; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je LBB8_4
 ; SSE2-NEXT:  LBB8_3: ## %cond.store1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm0, (%rdi)
 ; SSE2-NEXT:    addq $8, %rdi
 ; SSE2-NEXT:    testb $4, %al
@@ -2585,7 +2585,7 @@ define void @compressstore_v4i64_v4i1(i64* %base, <4 x i64> %V, <4 x i1> %mask)
 ; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je LBB8_8
 ; SSE2-NEXT:  LBB8_7: ## %cond.store7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm0, (%rdi)
 ; SSE2-NEXT:    retq
 ;
@@ -2762,7 +2762,7 @@ define void @compressstore_v8i64_v8i1(i64* %base, <8 x i64> %V, <8 x i1> %mask)
 ; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je LBB9_4
 ; SSE2-NEXT:  LBB9_3: ## %cond.store1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm0, (%rdi)
 ; SSE2-NEXT:    addq $8, %rdi
 ; SSE2-NEXT:    testb $4, %al
@@ -2773,7 +2773,7 @@ define void @compressstore_v8i64_v8i1(i64* %base, <8 x i64> %V, <8 x i1> %mask)
 ; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je LBB9_8
 ; SSE2-NEXT:  LBB9_7: ## %cond.store7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm0, (%rdi)
 ; SSE2-NEXT:    addq $8, %rdi
 ; SSE2-NEXT:    testb $16, %al
@@ -2784,7 +2784,7 @@ define void @compressstore_v8i64_v8i1(i64* %base, <8 x i64> %V, <8 x i1> %mask)
 ; SSE2-NEXT:    testb $32, %al
 ; SSE2-NEXT:    je LBB9_12
 ; SSE2-NEXT:  LBB9_11: ## %cond.store13
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm0, (%rdi)
 ; SSE2-NEXT:    addq $8, %rdi
 ; SSE2-NEXT:    testb $64, %al
@@ -2795,7 +2795,7 @@ define void @compressstore_v8i64_v8i1(i64* %base, <8 x i64> %V, <8 x i1> %mask)
 ; SSE2-NEXT:    testb $-128, %al
 ; SSE2-NEXT:    je LBB9_16
 ; SSE2-NEXT:  LBB9_15: ## %cond.store19
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm0, (%rdi)
 ; SSE2-NEXT:    retq
 ;
@@ -3068,7 +3068,7 @@ define void @compressstore_v4i32_v4i32(i32* %base, <4 x i32> %V, <4 x i32> %trig
 ; SSE2-NEXT:    testb $4, %al
 ; SSE2-NEXT:    je LBB10_6
 ; SSE2-NEXT:  LBB10_5: ## %cond.store4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    movd %xmm1, (%rdi)
 ; SSE2-NEXT:    addq $4, %rdi
 ; SSE2-NEXT:    testb $8, %al
diff --git a/llvm/test/CodeGen/X86/masked_gather.ll b/llvm/test/CodeGen/X86/masked_gather.ll
index dbd95213a60de..c3020b5f467c6 100644
--- a/llvm/test/CodeGen/X86/masked_gather.ll
+++ b/llvm/test/CodeGen/X86/masked_gather.ll
@@ -34,23 +34,23 @@ define <4 x float> @gather_v4f32_ptr_v4i32(<4 x float*> %ptr, <4 x i32> %trigger
 ; SSE-NEXT:    retq
 ; SSE-NEXT:  .LBB0_1: # %cond.load
 ; SSE-NEXT:    movq %xmm0, %rcx
-; SSE-NEXT:    movd (%rcx), %xmm2 # xmm2 = mem[0],zero,zero,zero
+; SSE-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
 ; SSE-NEXT:    pblendw {{.*#+}} xmm3 = xmm2[0,1],xmm3[2,3,4,5,6,7]
 ; SSE-NEXT:    testb $2, %al
 ; SSE-NEXT:    je .LBB0_4
 ; SSE-NEXT:  .LBB0_3: # %cond.load1
 ; SSE-NEXT:    pextrq $1, %xmm0, %rcx
-; SSE-NEXT:    insertps $16, (%rcx), %xmm3 # xmm3 = xmm3[0],mem[0],xmm3[2,3]
+; SSE-NEXT:    insertps {{.*#+}} xmm3 = xmm3[0],mem[0],xmm3[2,3]
 ; SSE-NEXT:    testb $4, %al
 ; SSE-NEXT:    je .LBB0_6
 ; SSE-NEXT:  .LBB0_5: # %cond.load4
 ; SSE-NEXT:    movq %xmm1, %rcx
-; SSE-NEXT:    insertps $32, (%rcx), %xmm3 # xmm3 = xmm3[0,1],mem[0],xmm3[3]
+; SSE-NEXT:    insertps {{.*#+}} xmm3 = xmm3[0,1],mem[0],xmm3[3]
 ; SSE-NEXT:    testb $8, %al
 ; SSE-NEXT:    je .LBB0_8
 ; SSE-NEXT:  .LBB0_7: # %cond.load7
 ; SSE-NEXT:    pextrq $1, %xmm1, %rax
-; SSE-NEXT:    insertps $48, (%rax), %xmm3 # xmm3 = xmm3[0,1,2],mem[0]
+; SSE-NEXT:    insertps {{.*#+}} xmm3 = xmm3[0,1,2],mem[0]
 ; SSE-NEXT:    movaps %xmm3, %xmm0
 ; SSE-NEXT:    retq
 ;
@@ -63,14 +63,14 @@ define <4 x float> @gather_v4f32_ptr_v4i32(<4 x float*> %ptr, <4 x i32> %trigger
 ; AVX1-NEXT:    je .LBB0_2
 ; AVX1-NEXT:  # %bb.1: # %cond.load
 ; AVX1-NEXT:    vmovq %xmm0, %rcx
-; AVX1-NEXT:    vmovss (%rcx), %xmm1 # xmm1 = mem[0],zero,zero,zero
+; AVX1-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; AVX1-NEXT:    vblendps {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
 ; AVX1-NEXT:  .LBB0_2: # %else
 ; AVX1-NEXT:    testb $2, %al
 ; AVX1-NEXT:    je .LBB0_4
 ; AVX1-NEXT:  # %bb.3: # %cond.load1
 ; AVX1-NEXT:    vpextrq $1, %xmm0, %rcx
-; AVX1-NEXT:    vinsertps $16, (%rcx), %xmm2, %xmm2 # xmm2 = xmm2[0],mem[0],xmm2[2,3]
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0],mem[0],xmm2[2,3]
 ; AVX1-NEXT:  .LBB0_4: # %else2
 ; AVX1-NEXT:    testb $4, %al
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
@@ -84,12 +84,12 @@ define <4 x float> @gather_v4f32_ptr_v4i32(<4 x float*> %ptr, <4 x i32> %trigger
 ; AVX1-NEXT:    retq
 ; AVX1-NEXT:  .LBB0_5: # %cond.load4
 ; AVX1-NEXT:    vmovq %xmm0, %rcx
-; AVX1-NEXT:    vinsertps $32, (%rcx), %xmm2, %xmm2 # xmm2 = xmm2[0,1],mem[0],xmm2[3]
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1],mem[0],xmm2[3]
 ; AVX1-NEXT:    testb $8, %al
 ; AVX1-NEXT:    je .LBB0_8
 ; AVX1-NEXT:  .LBB0_7: # %cond.load7
 ; AVX1-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX1-NEXT:    vinsertps $48, (%rax), %xmm2, %xmm2 # xmm2 = xmm2[0,1,2],mem[0]
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],mem[0]
 ; AVX1-NEXT:    vmovaps %xmm2, %xmm0
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -103,14 +103,14 @@ define <4 x float> @gather_v4f32_ptr_v4i32(<4 x float*> %ptr, <4 x i32> %trigger
 ; AVX2-NEXT:    je .LBB0_2
 ; AVX2-NEXT:  # %bb.1: # %cond.load
 ; AVX2-NEXT:    vmovq %xmm0, %rcx
-; AVX2-NEXT:    vmovss (%rcx), %xmm1 # xmm1 = mem[0],zero,zero,zero
+; AVX2-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; AVX2-NEXT:    vblendps {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
 ; AVX2-NEXT:  .LBB0_2: # %else
 ; AVX2-NEXT:    testb $2, %al
 ; AVX2-NEXT:    je .LBB0_4
 ; AVX2-NEXT:  # %bb.3: # %cond.load1
 ; AVX2-NEXT:    vpextrq $1, %xmm0, %rcx
-; AVX2-NEXT:    vinsertps $16, (%rcx), %xmm2, %xmm2 # xmm2 = xmm2[0],mem[0],xmm2[2,3]
+; AVX2-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0],mem[0],xmm2[2,3]
 ; AVX2-NEXT:  .LBB0_4: # %else2
 ; AVX2-NEXT:    testb $4, %al
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
@@ -124,12 +124,12 @@ define <4 x float> @gather_v4f32_ptr_v4i32(<4 x float*> %ptr, <4 x i32> %trigger
 ; AVX2-NEXT:    retq
 ; AVX2-NEXT:  .LBB0_5: # %cond.load4
 ; AVX2-NEXT:    vmovq %xmm0, %rcx
-; AVX2-NEXT:    vinsertps $32, (%rcx), %xmm2, %xmm2 # xmm2 = xmm2[0,1],mem[0],xmm2[3]
+; AVX2-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1],mem[0],xmm2[3]
 ; AVX2-NEXT:    testb $8, %al
 ; AVX2-NEXT:    je .LBB0_8
 ; AVX2-NEXT:  .LBB0_7: # %cond.load7
 ; AVX2-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX2-NEXT:    vinsertps $48, (%rax), %xmm2, %xmm2 # xmm2 = xmm2[0,1,2],mem[0]
+; AVX2-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],mem[0]
 ; AVX2-NEXT:    vmovaps %xmm2, %xmm0
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -159,7 +159,7 @@ define <4 x float> @gather_v4f32_v4i32_v4i32(float* %base, <4 x i32> %idx, <4 x
 ; SSE-NEXT:    pmovsxdq %xmm0, %xmm4
 ; SSE-NEXT:    psllq $2, %xmm4
 ; SSE-NEXT:    paddq %xmm3, %xmm4
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE-NEXT:    pmovsxdq %xmm0, %xmm0
 ; SSE-NEXT:    pxor %xmm5, %xmm5
 ; SSE-NEXT:    pcmpeqd %xmm1, %xmm5
@@ -168,7 +168,7 @@ define <4 x float> @gather_v4f32_v4i32_v4i32(float* %base, <4 x i32> %idx, <4 x
 ; SSE-NEXT:    je .LBB1_2
 ; SSE-NEXT:  # %bb.1: # %cond.load
 ; SSE-NEXT:    movq %xmm4, %rcx
-; SSE-NEXT:    movd (%rcx), %xmm1 # xmm1 = mem[0],zero,zero,zero
+; SSE-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; SSE-NEXT:    pblendw {{.*#+}} xmm2 = xmm1[0,1],xmm2[2,3,4,5,6,7]
 ; SSE-NEXT:  .LBB1_2: # %else
 ; SSE-NEXT:    psllq $2, %xmm0
@@ -176,7 +176,7 @@ define <4 x float> @gather_v4f32_v4i32_v4i32(float* %base, <4 x i32> %idx, <4 x
 ; SSE-NEXT:    je .LBB1_4
 ; SSE-NEXT:  # %bb.3: # %cond.load1
 ; SSE-NEXT:    pextrq $1, %xmm4, %rcx
-; SSE-NEXT:    insertps $16, (%rcx), %xmm2 # xmm2 = xmm2[0],mem[0],xmm2[2,3]
+; SSE-NEXT:    insertps {{.*#+}} xmm2 = xmm2[0],mem[0],xmm2[2,3]
 ; SSE-NEXT:  .LBB1_4: # %else2
 ; SSE-NEXT:    paddq %xmm0, %xmm3
 ; SSE-NEXT:    testb $4, %al
@@ -189,12 +189,12 @@ define <4 x float> @gather_v4f32_v4i32_v4i32(float* %base, <4 x i32> %idx, <4 x
 ; SSE-NEXT:    retq
 ; SSE-NEXT:  .LBB1_5: # %cond.load4
 ; SSE-NEXT:    movq %xmm3, %rcx
-; SSE-NEXT:    insertps $32, (%rcx), %xmm2 # xmm2 = xmm2[0,1],mem[0],xmm2[3]
+; SSE-NEXT:    insertps {{.*#+}} xmm2 = xmm2[0,1],mem[0],xmm2[3]
 ; SSE-NEXT:    testb $8, %al
 ; SSE-NEXT:    je .LBB1_8
 ; SSE-NEXT:  .LBB1_7: # %cond.load7
 ; SSE-NEXT:    pextrq $1, %xmm3, %rax
-; SSE-NEXT:    insertps $48, (%rax), %xmm2 # xmm2 = xmm2[0,1,2],mem[0]
+; SSE-NEXT:    insertps {{.*#+}} xmm2 = xmm2[0,1,2],mem[0]
 ; SSE-NEXT:    movaps %xmm2, %xmm0
 ; SSE-NEXT:    retq
 ;
@@ -202,7 +202,7 @@ define <4 x float> @gather_v4f32_v4i32_v4i32(float* %base, <4 x i32> %idx, <4 x
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vmovq %rdi, %xmm3
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[0,1,0,1]
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxdq %xmm4, %xmm4
 ; AVX1-NEXT:    vpsllq $2, %xmm4, %xmm4
 ; AVX1-NEXT:    vpaddq %xmm4, %xmm3, %xmm4
@@ -217,14 +217,14 @@ define <4 x float> @gather_v4f32_v4i32_v4i32(float* %base, <4 x i32> %idx, <4 x
 ; AVX1-NEXT:    je .LBB1_2
 ; AVX1-NEXT:  # %bb.1: # %cond.load
 ; AVX1-NEXT:    vmovq %xmm0, %rcx
-; AVX1-NEXT:    vmovss (%rcx), %xmm1 # xmm1 = mem[0],zero,zero,zero
+; AVX1-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; AVX1-NEXT:    vblendps {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
 ; AVX1-NEXT:  .LBB1_2: # %else
 ; AVX1-NEXT:    testb $2, %al
 ; AVX1-NEXT:    je .LBB1_4
 ; AVX1-NEXT:  # %bb.3: # %cond.load1
 ; AVX1-NEXT:    vpextrq $1, %xmm0, %rcx
-; AVX1-NEXT:    vinsertps $16, (%rcx), %xmm2, %xmm2 # xmm2 = xmm2[0],mem[0],xmm2[2,3]
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0],mem[0],xmm2[2,3]
 ; AVX1-NEXT:  .LBB1_4: # %else2
 ; AVX1-NEXT:    testb $4, %al
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
@@ -238,12 +238,12 @@ define <4 x float> @gather_v4f32_v4i32_v4i32(float* %base, <4 x i32> %idx, <4 x
 ; AVX1-NEXT:    retq
 ; AVX1-NEXT:  .LBB1_5: # %cond.load4
 ; AVX1-NEXT:    vmovq %xmm0, %rcx
-; AVX1-NEXT:    vinsertps $32, (%rcx), %xmm2, %xmm2 # xmm2 = xmm2[0,1],mem[0],xmm2[3]
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1],mem[0],xmm2[3]
 ; AVX1-NEXT:    testb $8, %al
 ; AVX1-NEXT:    je .LBB1_8
 ; AVX1-NEXT:  .LBB1_7: # %cond.load7
 ; AVX1-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX1-NEXT:    vinsertps $48, (%rax), %xmm2, %xmm2 # xmm2 = xmm2[0,1,2],mem[0]
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],mem[0]
 ; AVX1-NEXT:    vmovaps %xmm2, %xmm0
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -262,14 +262,14 @@ define <4 x float> @gather_v4f32_v4i32_v4i32(float* %base, <4 x i32> %idx, <4 x
 ; AVX2-NEXT:    je .LBB1_2
 ; AVX2-NEXT:  # %bb.1: # %cond.load
 ; AVX2-NEXT:    vmovq %xmm0, %rcx
-; AVX2-NEXT:    vmovss (%rcx), %xmm1 # xmm1 = mem[0],zero,zero,zero
+; AVX2-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; AVX2-NEXT:    vblendps {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
 ; AVX2-NEXT:  .LBB1_2: # %else
 ; AVX2-NEXT:    testb $2, %al
 ; AVX2-NEXT:    je .LBB1_4
 ; AVX2-NEXT:  # %bb.3: # %cond.load1
 ; AVX2-NEXT:    vpextrq $1, %xmm0, %rcx
-; AVX2-NEXT:    vinsertps $16, (%rcx), %xmm2, %xmm2 # xmm2 = xmm2[0],mem[0],xmm2[2,3]
+; AVX2-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0],mem[0],xmm2[2,3]
 ; AVX2-NEXT:  .LBB1_4: # %else2
 ; AVX2-NEXT:    testb $4, %al
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
@@ -283,12 +283,12 @@ define <4 x float> @gather_v4f32_v4i32_v4i32(float* %base, <4 x i32> %idx, <4 x
 ; AVX2-NEXT:    retq
 ; AVX2-NEXT:  .LBB1_5: # %cond.load4
 ; AVX2-NEXT:    vmovq %xmm0, %rcx
-; AVX2-NEXT:    vinsertps $32, (%rcx), %xmm2, %xmm2 # xmm2 = xmm2[0,1],mem[0],xmm2[3]
+; AVX2-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1],mem[0],xmm2[3]
 ; AVX2-NEXT:    testb $8, %al
 ; AVX2-NEXT:    je .LBB1_8
 ; AVX2-NEXT:  .LBB1_7: # %cond.load7
 ; AVX2-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX2-NEXT:    vinsertps $48, (%rax), %xmm2, %xmm2 # xmm2 = xmm2[0,1,2],mem[0]
+; AVX2-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],mem[0]
 ; AVX2-NEXT:    vmovaps %xmm2, %xmm0
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -328,7 +328,7 @@ define <4 x float> @gather_v4f32_v4i64_v4i32(float* %base, <4 x i64> %idx, <4 x
 ; SSE-NEXT:    je .LBB2_2
 ; SSE-NEXT:  # %bb.1: # %cond.load
 ; SSE-NEXT:    movq %xmm0, %rcx
-; SSE-NEXT:    movd (%rcx), %xmm2 # xmm2 = mem[0],zero,zero,zero
+; SSE-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
 ; SSE-NEXT:    pblendw {{.*#+}} xmm3 = xmm2[0,1],xmm3[2,3,4,5,6,7]
 ; SSE-NEXT:  .LBB2_2: # %else
 ; SSE-NEXT:    psllq $2, %xmm1
@@ -336,7 +336,7 @@ define <4 x float> @gather_v4f32_v4i64_v4i32(float* %base, <4 x i64> %idx, <4 x
 ; SSE-NEXT:    je .LBB2_4
 ; SSE-NEXT:  # %bb.3: # %cond.load1
 ; SSE-NEXT:    pextrq $1, %xmm0, %rcx
-; SSE-NEXT:    insertps $16, (%rcx), %xmm3 # xmm3 = xmm3[0],mem[0],xmm3[2,3]
+; SSE-NEXT:    insertps {{.*#+}} xmm3 = xmm3[0],mem[0],xmm3[2,3]
 ; SSE-NEXT:  .LBB2_4: # %else2
 ; SSE-NEXT:    paddq %xmm1, %xmm4
 ; SSE-NEXT:    testb $4, %al
@@ -349,12 +349,12 @@ define <4 x float> @gather_v4f32_v4i64_v4i32(float* %base, <4 x i64> %idx, <4 x
 ; SSE-NEXT:    retq
 ; SSE-NEXT:  .LBB2_5: # %cond.load4
 ; SSE-NEXT:    movq %xmm4, %rcx
-; SSE-NEXT:    insertps $32, (%rcx), %xmm3 # xmm3 = xmm3[0,1],mem[0],xmm3[3]
+; SSE-NEXT:    insertps {{.*#+}} xmm3 = xmm3[0,1],mem[0],xmm3[3]
 ; SSE-NEXT:    testb $8, %al
 ; SSE-NEXT:    je .LBB2_8
 ; SSE-NEXT:  .LBB2_7: # %cond.load7
 ; SSE-NEXT:    pextrq $1, %xmm4, %rax
-; SSE-NEXT:    insertps $48, (%rax), %xmm3 # xmm3 = xmm3[0,1,2],mem[0]
+; SSE-NEXT:    insertps {{.*#+}} xmm3 = xmm3[0,1,2],mem[0]
 ; SSE-NEXT:    movaps %xmm3, %xmm0
 ; SSE-NEXT:    retq
 ;
@@ -375,14 +375,14 @@ define <4 x float> @gather_v4f32_v4i64_v4i32(float* %base, <4 x i64> %idx, <4 x
 ; AVX1-NEXT:    je .LBB2_2
 ; AVX1-NEXT:  # %bb.1: # %cond.load
 ; AVX1-NEXT:    vmovq %xmm0, %rcx
-; AVX1-NEXT:    vmovss (%rcx), %xmm1 # xmm1 = mem[0],zero,zero,zero
+; AVX1-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; AVX1-NEXT:    vblendps {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
 ; AVX1-NEXT:  .LBB2_2: # %else
 ; AVX1-NEXT:    testb $2, %al
 ; AVX1-NEXT:    je .LBB2_4
 ; AVX1-NEXT:  # %bb.3: # %cond.load1
 ; AVX1-NEXT:    vpextrq $1, %xmm0, %rcx
-; AVX1-NEXT:    vinsertps $16, (%rcx), %xmm2, %xmm2 # xmm2 = xmm2[0],mem[0],xmm2[2,3]
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0],mem[0],xmm2[2,3]
 ; AVX1-NEXT:  .LBB2_4: # %else2
 ; AVX1-NEXT:    testb $4, %al
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
@@ -396,12 +396,12 @@ define <4 x float> @gather_v4f32_v4i64_v4i32(float* %base, <4 x i64> %idx, <4 x
 ; AVX1-NEXT:    retq
 ; AVX1-NEXT:  .LBB2_5: # %cond.load4
 ; AVX1-NEXT:    vmovq %xmm0, %rcx
-; AVX1-NEXT:    vinsertps $32, (%rcx), %xmm2, %xmm2 # xmm2 = xmm2[0,1],mem[0],xmm2[3]
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1],mem[0],xmm2[3]
 ; AVX1-NEXT:    testb $8, %al
 ; AVX1-NEXT:    je .LBB2_8
 ; AVX1-NEXT:  .LBB2_7: # %cond.load7
 ; AVX1-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX1-NEXT:    vinsertps $48, (%rax), %xmm2, %xmm2 # xmm2 = xmm2[0,1,2],mem[0]
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],mem[0]
 ; AVX1-NEXT:    vmovaps %xmm2, %xmm0
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -419,14 +419,14 @@ define <4 x float> @gather_v4f32_v4i64_v4i32(float* %base, <4 x i64> %idx, <4 x
 ; AVX2-NEXT:    je .LBB2_2
 ; AVX2-NEXT:  # %bb.1: # %cond.load
 ; AVX2-NEXT:    vmovq %xmm0, %rcx
-; AVX2-NEXT:    vmovss (%rcx), %xmm1 # xmm1 = mem[0],zero,zero,zero
+; AVX2-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; AVX2-NEXT:    vblendps {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
 ; AVX2-NEXT:  .LBB2_2: # %else
 ; AVX2-NEXT:    testb $2, %al
 ; AVX2-NEXT:    je .LBB2_4
 ; AVX2-NEXT:  # %bb.3: # %cond.load1
 ; AVX2-NEXT:    vpextrq $1, %xmm0, %rcx
-; AVX2-NEXT:    vinsertps $16, (%rcx), %xmm2, %xmm2 # xmm2 = xmm2[0],mem[0],xmm2[2,3]
+; AVX2-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0],mem[0],xmm2[2,3]
 ; AVX2-NEXT:  .LBB2_4: # %else2
 ; AVX2-NEXT:    testb $4, %al
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
@@ -440,12 +440,12 @@ define <4 x float> @gather_v4f32_v4i64_v4i32(float* %base, <4 x i64> %idx, <4 x
 ; AVX2-NEXT:    retq
 ; AVX2-NEXT:  .LBB2_5: # %cond.load4
 ; AVX2-NEXT:    vmovq %xmm0, %rcx
-; AVX2-NEXT:    vinsertps $32, (%rcx), %xmm2, %xmm2 # xmm2 = xmm2[0,1],mem[0],xmm2[3]
+; AVX2-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1],mem[0],xmm2[3]
 ; AVX2-NEXT:    testb $8, %al
 ; AVX2-NEXT:    je .LBB2_8
 ; AVX2-NEXT:  .LBB2_7: # %cond.load7
 ; AVX2-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX2-NEXT:    vinsertps $48, (%rax), %xmm2, %xmm2 # xmm2 = xmm2[0,1,2],mem[0]
+; AVX2-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],mem[0]
 ; AVX2-NEXT:    vmovaps %xmm2, %xmm0
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -480,7 +480,7 @@ define <16 x i8> @gather_v16i8_v16i32_v16i8(i8* %base, <16 x i32> %idx, <16 x i8
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    movq %rdi, %xmm6
 ; SSE-NEXT:    pshufd {{.*#+}} xmm8 = xmm6[0,1,0,1]
-; SSE-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[2,3,2,3]
 ; SSE-NEXT:    pmovsxdq %xmm0, %xmm0
 ; SSE-NEXT:    paddq %xmm8, %xmm0
 ; SSE-NEXT:    pxor %xmm6, %xmm6
@@ -513,7 +513,7 @@ define <16 x i8> @gather_v16i8_v16i32_v16i8(i8* %base, <16 x i32> %idx, <16 x i8
 ; SSE-NEXT:    pextrq $1, %xmm4, %rcx
 ; SSE-NEXT:    pinsrb $3, (%rcx), %xmm5
 ; SSE-NEXT:  .LBB3_8: # %else8
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE-NEXT:    paddq %xmm8, %xmm0
 ; SSE-NEXT:    testb $16, %al
 ; SSE-NEXT:    je .LBB3_10
@@ -542,7 +542,7 @@ define <16 x i8> @gather_v16i8_v16i32_v16i8(i8* %base, <16 x i32> %idx, <16 x i8
 ; SSE-NEXT:    pextrq $1, %xmm1, %rcx
 ; SSE-NEXT:    pinsrb $7, (%rcx), %xmm5
 ; SSE-NEXT:  .LBB3_16: # %else20
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]
 ; SSE-NEXT:    paddq %xmm8, %xmm0
 ; SSE-NEXT:    testl $256, %eax # imm = 0x100
 ; SSE-NEXT:    je .LBB3_18
@@ -571,7 +571,7 @@ define <16 x i8> @gather_v16i8_v16i32_v16i8(i8* %base, <16 x i32> %idx, <16 x i8
 ; SSE-NEXT:    pextrq $1, %xmm1, %rcx
 ; SSE-NEXT:    pinsrb $11, (%rcx), %xmm5
 ; SSE-NEXT:  .LBB3_24: # %else32
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[2,3,2,3]
 ; SSE-NEXT:    paddq %xmm8, %xmm0
 ; SSE-NEXT:    testl $4096, %eax # imm = 0x1000
 ; SSE-NEXT:    je .LBB3_26
@@ -611,7 +611,7 @@ define <16 x i8> @gather_v16i8_v16i32_v16i8(i8* %base, <16 x i32> %idx, <16 x i8
 ; AVX1-NEXT:    vmovq %rdi, %xmm4
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[0,1,0,1]
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm5
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxdq %xmm6, %xmm6
 ; AVX1-NEXT:    vpaddq %xmm6, %xmm4, %xmm6
 ; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm0
@@ -626,7 +626,7 @@ define <16 x i8> @gather_v16i8_v16i32_v16i8(i8* %base, <16 x i32> %idx, <16 x i8
 ; AVX1-NEXT:    vmovq %xmm0, %rcx
 ; AVX1-NEXT:    vpinsrb $0, (%rcx), %xmm3, %xmm3
 ; AVX1-NEXT:  .LBB3_2: # %else
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm5[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm5[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxdq %xmm5, %xmm6
 ; AVX1-NEXT:    testb $2, %al
 ; AVX1-NEXT:    je .LBB3_4
@@ -657,7 +657,7 @@ define <16 x i8> @gather_v16i8_v16i32_v16i8(i8* %base, <16 x i32> %idx, <16 x i8
 ; AVX1-NEXT:    vmovq %xmm0, %rcx
 ; AVX1-NEXT:    vpinsrb $4, (%rcx), %xmm3, %xmm3
 ; AVX1-NEXT:  .LBB3_10: # %else11
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxdq %xmm1, %xmm6
 ; AVX1-NEXT:    testb $32, %al
 ; AVX1-NEXT:    je .LBB3_12
@@ -689,7 +689,7 @@ define <16 x i8> @gather_v16i8_v16i32_v16i8(i8* %base, <16 x i32> %idx, <16 x i8
 ; AVX1-NEXT:    vmovq %xmm0, %rcx
 ; AVX1-NEXT:    vpinsrb $8, (%rcx), %xmm3, %xmm3
 ; AVX1-NEXT:  .LBB3_18: # %else23
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxdq %xmm1, %xmm1
 ; AVX1-NEXT:    testl $512, %eax # imm = 0x200
 ; AVX1-NEXT:    je .LBB3_20
@@ -1040,7 +1040,7 @@ define <8 x i32> @gather_v8i32_v8i32(<8 x i32> %trigger) {
 ; SSE-NEXT:    je .LBB4_1
 ; SSE-NEXT:  # %bb.2: # %cond.load
 ; SSE-NEXT:    movq %xmm5, %rcx
-; SSE-NEXT:    movd (%rcx), %xmm0 # xmm0 = mem[0],zero,zero,zero
+; SSE-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; SSE-NEXT:    testb $2, %al
 ; SSE-NEXT:    jne .LBB4_4
 ; SSE-NEXT:    jmp .LBB4_5
@@ -1105,7 +1105,7 @@ define <8 x i32> @gather_v8i32_v8i32(<8 x i32> %trigger) {
 ; SSE-NEXT:    je .LBB4_19
 ; SSE-NEXT:  # %bb.20: # %cond.load23
 ; SSE-NEXT:    movq %xmm4, %rcx
-; SSE-NEXT:    movd (%rcx), %xmm5 # xmm5 = mem[0],zero,zero,zero
+; SSE-NEXT:    movd {{.*#+}} xmm5 = mem[0],zero,zero,zero
 ; SSE-NEXT:    testb $2, %al
 ; SSE-NEXT:    jne .LBB4_22
 ; SSE-NEXT:    jmp .LBB4_23
@@ -1174,7 +1174,7 @@ define <8 x i32> @gather_v8i32_v8i32(<8 x i32> %trigger) {
 ; SSE-NEXT:    je .LBB4_37
 ; SSE-NEXT:  # %bb.38: # %cond.load72
 ; SSE-NEXT:    movq %xmm4, %rcx
-; SSE-NEXT:    movd (%rcx), %xmm2 # xmm2 = mem[0],zero,zero,zero
+; SSE-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
 ; SSE-NEXT:    testb $2, %al
 ; SSE-NEXT:    jne .LBB4_40
 ; SSE-NEXT:    jmp .LBB4_41
@@ -1260,7 +1260,7 @@ define <8 x i32> @gather_v8i32_v8i32(<8 x i32> %trigger) {
 ; AVX1-NEXT:    je .LBB4_2
 ; AVX1-NEXT:  # %bb.1: # %cond.load
 ; AVX1-NEXT:    vmovq %xmm3, %rdx
-; AVX1-NEXT:    vmovd (%rdx), %xmm1 # xmm1 = mem[0],zero,zero,zero
+; AVX1-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; AVX1-NEXT:  .LBB4_2: # %else
 ; AVX1-NEXT:    testb $2, %al
 ; AVX1-NEXT:    je .LBB4_4
@@ -1334,7 +1334,7 @@ define <8 x i32> @gather_v8i32_v8i32(<8 x i32> %trigger) {
 ; AVX1-NEXT:    je .LBB4_18
 ; AVX1-NEXT:  # %bb.17: # %cond.load23
 ; AVX1-NEXT:    vmovq %xmm7, %rcx
-; AVX1-NEXT:    vmovd (%rcx), %xmm4 # xmm4 = mem[0],zero,zero,zero
+; AVX1-NEXT:    vmovd {{.*#+}} xmm4 = mem[0],zero,zero,zero
 ; AVX1-NEXT:  .LBB4_18: # %else27
 ; AVX1-NEXT:    testb $2, %al
 ; AVX1-NEXT:    je .LBB4_20
@@ -1405,7 +1405,7 @@ define <8 x i32> @gather_v8i32_v8i32(<8 x i32> %trigger) {
 ; AVX1-NEXT:    je .LBB4_34
 ; AVX1-NEXT:  # %bb.33: # %cond.load72
 ; AVX1-NEXT:    vmovq %xmm7, %rcx
-; AVX1-NEXT:    vmovd (%rcx), %xmm0 # xmm0 = mem[0],zero,zero,zero
+; AVX1-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; AVX1-NEXT:  .LBB4_34: # %else76
 ; AVX1-NEXT:    testb $2, %al
 ; AVX1-NEXT:    je .LBB4_36
@@ -1491,7 +1491,7 @@ define <8 x i32> @gather_v8i32_v8i32(<8 x i32> %trigger) {
 ; AVX2-NEXT:    je .LBB4_2
 ; AVX2-NEXT:  # %bb.1: # %cond.load
 ; AVX2-NEXT:    vmovq %xmm3, %rcx
-; AVX2-NEXT:    vmovd (%rcx), %xmm1 # xmm1 = mem[0],zero,zero,zero
+; AVX2-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; AVX2-NEXT:  .LBB4_2: # %else
 ; AVX2-NEXT:    testb $2, %al
 ; AVX2-NEXT:    je .LBB4_4
@@ -1534,7 +1534,7 @@ define <8 x i32> @gather_v8i32_v8i32(<8 x i32> %trigger) {
 ; AVX2-NEXT:    je .LBB4_18
 ; AVX2-NEXT:  # %bb.17: # %cond.load23
 ; AVX2-NEXT:    vmovq %xmm3, %rcx
-; AVX2-NEXT:    vmovd (%rcx), %xmm2 # xmm2 = mem[0],zero,zero,zero
+; AVX2-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
 ; AVX2-NEXT:  .LBB4_18: # %else27
 ; AVX2-NEXT:    testb $2, %al
 ; AVX2-NEXT:    je .LBB4_20
@@ -1678,7 +1678,7 @@ define <8 x i32> @gather_v8i32_v8i32(<8 x i32> %trigger) {
 ; AVX2-NEXT:    jmp .LBB4_32
 ; AVX2-NEXT:  .LBB4_33: # %cond.load72
 ; AVX2-NEXT:    vmovq %xmm3, %rcx
-; AVX2-NEXT:    vmovd (%rcx), %xmm0 # xmm0 = mem[0],zero,zero,zero
+; AVX2-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; AVX2-NEXT:    testb $2, %al
 ; AVX2-NEXT:    je .LBB4_36
 ; AVX2-NEXT:  .LBB4_35: # %cond.load78
diff --git a/llvm/test/CodeGen/X86/masked_load.ll b/llvm/test/CodeGen/X86/masked_load.ll
index a14d537bc25da..e8dc7412eef8b 100644
--- a/llvm/test/CodeGen/X86/masked_load.ll
+++ b/llvm/test/CodeGen/X86/masked_load.ll
@@ -153,7 +153,7 @@ define <4 x double> @load_v4f64_v4i32(<4 x i32> %trigger, <4 x double>* %addr, <
 ; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
 ; AVX1-NEXT:    vmaskmovpd (%rdi), %ymm0, %ymm2
@@ -233,7 +233,7 @@ define <4 x double> @load_v4f64_v4i32_zero(<4 x i32> %trigger, <4 x double>* %ad
 ; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    vmaskmovpd (%rdi), %ymm0, %ymm0
@@ -458,12 +458,12 @@ define <8 x double> @load_v8f64_v8i16(<8 x i16> %trigger, <8 x double>* %addr, <
 ;
 ; AVX1-LABEL: load_v8f64_v8i16:
 ; AVX1:       ## %bb.0:
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
 ; AVX1-NEXT:    vpcmpeqw %xmm4, %xmm3, %xmm3
 ; AVX1-NEXT:    vpmovsxwd %xmm3, %xmm3
 ; AVX1-NEXT:    vpmovsxdq %xmm3, %xmm5
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxdq %xmm3, %xmm3
 ; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm5, %ymm3
 ; AVX1-NEXT:    vpcmpeqw %xmm4, %xmm0, %xmm0
@@ -479,7 +479,7 @@ define <8 x double> @load_v8f64_v8i16(<8 x i16> %trigger, <8 x double>* %addr, <
 ;
 ; AVX2-LABEL: load_v8f64_v8i16:
 ; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
 ; AVX2-NEXT:    vpcmpeqw %xmm4, %xmm3, %xmm3
 ; AVX2-NEXT:    vpmovsxwd %xmm3, %xmm3
@@ -1778,12 +1778,12 @@ define <8 x i64> @load_v8i64_v8i16(<8 x i16> %trigger, <8 x i64>* %addr, <8 x i6
 ;
 ; AVX1-LABEL: load_v8i64_v8i16:
 ; AVX1:       ## %bb.0:
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
 ; AVX1-NEXT:    vpcmpeqw %xmm4, %xmm3, %xmm3
 ; AVX1-NEXT:    vpmovsxwd %xmm3, %xmm3
 ; AVX1-NEXT:    vpmovsxdq %xmm3, %xmm5
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxdq %xmm3, %xmm3
 ; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm5, %ymm3
 ; AVX1-NEXT:    vpcmpeqw %xmm4, %xmm0, %xmm0
@@ -1799,7 +1799,7 @@ define <8 x i64> @load_v8i64_v8i16(<8 x i16> %trigger, <8 x i64>* %addr, <8 x i6
 ;
 ; AVX2-LABEL: load_v8i64_v8i16:
 ; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
 ; AVX2-NEXT:    vpcmpeqw %xmm4, %xmm3, %xmm3
 ; AVX2-NEXT:    vpmovsxwd %xmm3, %xmm3
diff --git a/llvm/test/CodeGen/X86/masked_store.ll b/llvm/test/CodeGen/X86/masked_store.ll
index 29678e8495c22..389281726d273 100644
--- a/llvm/test/CodeGen/X86/masked_store.ll
+++ b/llvm/test/CodeGen/X86/masked_store.ll
@@ -838,7 +838,7 @@ define void @store_v2i64_v2i64(<2 x i64> %trigger, <2 x i64>* %addr, <2 x i64> %
 ; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je LBB7_4
 ; SSE2-NEXT:  LBB7_3: ## %cond.store1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm0, 8(%rdi)
 ; SSE2-NEXT:    retq
 ;
@@ -922,7 +922,7 @@ define void @store_v4i64_v4i64(<4 x i64> %trigger, <4 x i64>* %addr, <4 x i64> %
 ; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je LBB8_4
 ; SSE2-NEXT:  LBB8_3: ## %cond.store1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm0, 8(%rdi)
 ; SSE2-NEXT:    testb $4, %al
 ; SSE2-NEXT:    je LBB8_6
@@ -931,7 +931,7 @@ define void @store_v4i64_v4i64(<4 x i64> %trigger, <4 x i64>* %addr, <4 x i64> %
 ; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je LBB8_8
 ; SSE2-NEXT:  LBB8_7: ## %cond.store5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm0, 24(%rdi)
 ; SSE2-NEXT:    retq
 ;
@@ -1158,7 +1158,7 @@ define void @store_v4i32_v4i32(<4 x i32> %trigger, <4 x i32>* %addr, <4 x i32> %
 ; SSE2-NEXT:    testb $4, %al
 ; SSE2-NEXT:    je LBB11_6
 ; SSE2-NEXT:  LBB11_5: ## %cond.store3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    movd %xmm0, 8(%rdi)
 ; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je LBB11_8
@@ -1280,7 +1280,7 @@ define void @store_v8i32_v8i32(<8 x i32> %trigger, <8 x i32>* %addr, <8 x i32> %
 ; SSE2-NEXT:    testb $4, %al
 ; SSE2-NEXT:    je LBB12_6
 ; SSE2-NEXT:  LBB12_5: ## %cond.store3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE2-NEXT:    movd %xmm0, 8(%rdi)
 ; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je LBB12_8
@@ -1299,7 +1299,7 @@ define void @store_v8i32_v8i32(<8 x i32> %trigger, <8 x i32>* %addr, <8 x i32> %
 ; SSE2-NEXT:    testb $64, %al
 ; SSE2-NEXT:    je LBB12_14
 ; SSE2-NEXT:  LBB12_13: ## %cond.store11
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]
 ; SSE2-NEXT:    movd %xmm0, 24(%rdi)
 ; SSE2-NEXT:    testb $-128, %al
 ; SSE2-NEXT:    je LBB12_16
@@ -4674,7 +4674,7 @@ define void @masked_store_bool_mask_demand_trunc_sext(<4 x double> %x, <4 x doub
 ; AVX1:       ## %bb.0:
 ; AVX1-NEXT:    vpslld $31, %xmm1, %xmm1
 ; AVX1-NEXT:    vpmovsxdq %xmm1, %xmm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxdq %xmm1, %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
 ; AVX1-NEXT:    vmaskmovpd %ymm0, %ymm1, (%rdi)
@@ -4853,7 +4853,7 @@ define void @widen_masked_store(<3 x i32> %v, <3 x i32>* %p, <3 x i1> %mask) {
 ; SSE2-NEXT:    testb $4, %cl
 ; SSE2-NEXT:    je LBB25_6
 ; SSE2-NEXT:  LBB25_5: ## %cond.store3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    movd %xmm0, 8(%rdi)
 ; SSE2-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/masked_store_trunc.ll b/llvm/test/CodeGen/X86/masked_store_trunc.ll
index 341a349911180..640e145c20231 100644
--- a/llvm/test/CodeGen/X86/masked_store_trunc.ll
+++ b/llvm/test/CodeGen/X86/masked_store_trunc.ll
@@ -60,7 +60,7 @@ define void @truncstore_v8i64_v8i32(<8 x i64> %x, <8 x i32>* %p, <8 x i32> %mask
 ; SSE2-NEXT:    testb $4, %al
 ; SSE2-NEXT:    je .LBB0_6
 ; SSE2-NEXT:  .LBB0_5: # %cond.store3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    movd %xmm1, 8(%rdi)
 ; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    jne .LBB0_7
@@ -75,7 +75,7 @@ define void @truncstore_v8i64_v8i32(<8 x i64> %x, <8 x i32>* %p, <8 x i32> %mask
 ; SSE2-NEXT:    testb $64, %al
 ; SSE2-NEXT:    je .LBB0_14
 ; SSE2-NEXT:  .LBB0_13: # %cond.store11
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE2-NEXT:    movd %xmm0, 24(%rdi)
 ; SSE2-NEXT:    testb $-128, %al
 ; SSE2-NEXT:    je .LBB0_16
@@ -1030,7 +1030,7 @@ define void @truncstore_v4i64_v4i32(<4 x i64> %x, <4 x i32>* %p, <4 x i32> %mask
 ; SSE2-NEXT:    testb $4, %al
 ; SSE2-NEXT:    je .LBB3_6
 ; SSE2-NEXT:  .LBB3_5: # %cond.store3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    movd %xmm1, 8(%rdi)
 ; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je .LBB3_8
diff --git a/llvm/test/CodeGen/X86/masked_store_trunc_ssat.ll b/llvm/test/CodeGen/X86/masked_store_trunc_ssat.ll
index c120684c42610..17c113f098eb4 100644
--- a/llvm/test/CodeGen/X86/masked_store_trunc_ssat.ll
+++ b/llvm/test/CodeGen/X86/masked_store_trunc_ssat.ll
@@ -141,7 +141,7 @@ define void @truncstore_v8i64_v8i32(<8 x i64> %x, <8 x i32>* %p, <8 x i32> %mask
 ; SSE2-NEXT:    testb $4, %al
 ; SSE2-NEXT:    je .LBB0_6
 ; SSE2-NEXT:  # %bb.5: # %cond.store3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    movd %xmm4, 8(%rdi)
 ; SSE2-NEXT:  .LBB0_6: # %else4
 ; SSE2-NEXT:    por %xmm3, %xmm2
@@ -176,7 +176,7 @@ define void @truncstore_v8i64_v8i32(<8 x i64> %x, <8 x i32>* %p, <8 x i32> %mask
 ; SSE2-NEXT:    testb $64, %al
 ; SSE2-NEXT:    je .LBB0_14
 ; SSE2-NEXT:  .LBB0_13: # %cond.store11
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm13[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm13[2,3,2,3]
 ; SSE2-NEXT:    movd %xmm0, 24(%rdi)
 ; SSE2-NEXT:    testb $-128, %al
 ; SSE2-NEXT:    je .LBB0_16
@@ -1579,7 +1579,7 @@ define void @truncstore_v4i64_v4i32(<4 x i64> %x, <4 x i32>* %p, <4 x i32> %mask
 ; SSE2-NEXT:    testb $4, %al
 ; SSE2-NEXT:    je .LBB3_6
 ; SSE2-NEXT:  .LBB3_5: # %cond.store3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    movd %xmm1, 8(%rdi)
 ; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je .LBB3_8
diff --git a/llvm/test/CodeGen/X86/masked_store_trunc_usat.ll b/llvm/test/CodeGen/X86/masked_store_trunc_usat.ll
index 0160733732d70..2ca9ebb0d5c6d 100644
--- a/llvm/test/CodeGen/X86/masked_store_trunc_usat.ll
+++ b/llvm/test/CodeGen/X86/masked_store_trunc_usat.ll
@@ -87,7 +87,7 @@ define void @truncstore_v8i64_v8i32(<8 x i64> %x, <8 x i32>* %p, <8 x i32> %mask
 ; SSE2-NEXT:    testb $4, %al
 ; SSE2-NEXT:    je .LBB0_6
 ; SSE2-NEXT:  # %bb.5: # %cond.store3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm13[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm13[2,3,2,3]
 ; SSE2-NEXT:    movd %xmm1, 8(%rdi)
 ; SSE2-NEXT:  .LBB0_6: # %else4
 ; SSE2-NEXT:    por %xmm0, %xmm3
@@ -122,7 +122,7 @@ define void @truncstore_v8i64_v8i32(<8 x i64> %x, <8 x i32>* %p, <8 x i32> %mask
 ; SSE2-NEXT:    testb $64, %al
 ; SSE2-NEXT:    je .LBB0_14
 ; SSE2-NEXT:  .LBB0_13: # %cond.store11
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE2-NEXT:    movd %xmm0, 24(%rdi)
 ; SSE2-NEXT:    testb $-128, %al
 ; SSE2-NEXT:    je .LBB0_16
@@ -1351,7 +1351,7 @@ define void @truncstore_v4i64_v4i32(<4 x i64> %x, <4 x i32>* %p, <4 x i32> %mask
 ; SSE2-NEXT:    testb $4, %al
 ; SSE2-NEXT:    je .LBB3_6
 ; SSE2-NEXT:  .LBB3_5: # %cond.store3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    movd %xmm0, 8(%rdi)
 ; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je .LBB3_8
diff --git a/llvm/test/CodeGen/X86/merge-consecutive-stores-nt.ll b/llvm/test/CodeGen/X86/merge-consecutive-stores-nt.ll
index 8b3756a1fa241..6dcc47b9a65c4 100644
--- a/llvm/test/CodeGen/X86/merge-consecutive-stores-nt.ll
+++ b/llvm/test/CodeGen/X86/merge-consecutive-stores-nt.ll
@@ -309,7 +309,7 @@ define void @merge_2_v4f32_align1_ntstore(<4 x float>* %a0, <4 x float>* %a1) no
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,1,2,3]
 ; X86-SSE2-NEXT:    movd %xmm2, %ecx
 ; X86-SSE2-NEXT:    movntil %ecx, 12(%eax)
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    movd %xmm2, %ecx
 ; X86-SSE2-NEXT:    movntil %ecx, 8(%eax)
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
@@ -320,7 +320,7 @@ define void @merge_2_v4f32_align1_ntstore(<4 x float>* %a0, <4 x float>* %a1) no
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[3,1,2,3]
 ; X86-SSE2-NEXT:    movd %xmm0, %ecx
 ; X86-SSE2-NEXT:    movntil %ecx, 28(%eax)
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; X86-SSE2-NEXT:    movd %xmm0, %ecx
 ; X86-SSE2-NEXT:    movntil %ecx, 24(%eax)
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
@@ -348,12 +348,12 @@ define void @merge_2_v4f32_align1_ntstore(<4 x float>* %a0, <4 x float>* %a1) no
 ; X64-SSE2-NEXT:    movdqu 16(%rdi), %xmm1
 ; X64-SSE2-NEXT:    movq %xmm0, %rax
 ; X64-SSE2-NEXT:    movntiq %rax, (%rsi)
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    movq %xmm0, %rax
 ; X64-SSE2-NEXT:    movntiq %rax, 8(%rsi)
 ; X64-SSE2-NEXT:    movq %xmm1, %rax
 ; X64-SSE2-NEXT:    movntiq %rax, 16(%rsi)
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; X64-SSE2-NEXT:    movq %xmm0, %rax
 ; X64-SSE2-NEXT:    movntiq %rax, 24(%rsi)
 ; X64-SSE2-NEXT:    retq
@@ -422,7 +422,7 @@ define void @merge_2_v4f32_align1(<4 x float>* %a0, <4 x float>* %a1) nounwind {
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,1,2,3]
 ; X86-SSE2-NEXT:    movd %xmm2, %ecx
 ; X86-SSE2-NEXT:    movntil %ecx, 12(%eax)
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    movd %xmm2, %ecx
 ; X86-SSE2-NEXT:    movntil %ecx, 8(%eax)
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
@@ -433,7 +433,7 @@ define void @merge_2_v4f32_align1(<4 x float>* %a0, <4 x float>* %a1) nounwind {
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[3,1,2,3]
 ; X86-SSE2-NEXT:    movd %xmm0, %ecx
 ; X86-SSE2-NEXT:    movntil %ecx, 28(%eax)
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; X86-SSE2-NEXT:    movd %xmm0, %ecx
 ; X86-SSE2-NEXT:    movntil %ecx, 24(%eax)
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
@@ -461,12 +461,12 @@ define void @merge_2_v4f32_align1(<4 x float>* %a0, <4 x float>* %a1) nounwind {
 ; X64-SSE2-NEXT:    movdqu 16(%rdi), %xmm1
 ; X64-SSE2-NEXT:    movq %xmm0, %rax
 ; X64-SSE2-NEXT:    movntiq %rax, (%rsi)
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    movq %xmm0, %rax
 ; X64-SSE2-NEXT:    movntiq %rax, 8(%rsi)
 ; X64-SSE2-NEXT:    movq %xmm1, %rax
 ; X64-SSE2-NEXT:    movntiq %rax, 16(%rsi)
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; X64-SSE2-NEXT:    movq %xmm0, %rax
 ; X64-SSE2-NEXT:    movntiq %rax, 24(%rsi)
 ; X64-SSE2-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/min-legal-vector-width.ll b/llvm/test/CodeGen/X86/min-legal-vector-width.ll
index 2980a1665db14..666df4a1b960d 100644
--- a/llvm/test/CodeGen/X86/min-legal-vector-width.ll
+++ b/llvm/test/CodeGen/X86/min-legal-vector-width.ll
@@ -197,7 +197,7 @@ define i32 @_Z9test_charPcS_i_256(i8* nocapture readonly, i8* nocapture readonly
 ; CHECK-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
 ; CHECK-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -263,7 +263,7 @@ define i32 @_Z9test_charPcS_i_512(i8* nocapture readonly, i8* nocapture readonly
 ; CHECK-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
 ; CHECK-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -327,7 +327,7 @@ define i32 @sad_16i8_256() "min-legal-vector-width"="256" {
 ; CHECK-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
 ; CHECK-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -388,7 +388,7 @@ define i32 @sad_16i8_512() "min-legal-vector-width"="512" {
 ; CHECK-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
 ; CHECK-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -955,10 +955,10 @@ define void @zext_v16i8_v16i64(<16 x i8> %x, <16 x i64>* %y) nounwind "min-legal
 ; CHECK-LABEL: zext_v16i8_v16i64:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
-; CHECK-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; CHECK-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; CHECK-NEXT:    vpmovzxwq {{.*#+}} ymm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
 ; CHECK-NEXT:    vextracti128 $1, %ymm1, %xmm1
-; CHECK-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[2,3,0,1]
+; CHECK-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
 ; CHECK-NEXT:    vpmovzxwq {{.*#+}} ymm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
 ; CHECK-NEXT:    vpmovzxwq {{.*#+}} ymm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; CHECK-NEXT:    vpmovzxbq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero
@@ -977,10 +977,10 @@ define void @sext_v16i8_v16i64(<16 x i8> %x, <16 x i64>* %y) nounwind "min-legal
 ; CHECK-LABEL: sext_v16i8_v16i64:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vpmovsxbw %xmm0, %ymm0
-; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; CHECK-NEXT:    vpmovsxwq %xmm1, %ymm1
 ; CHECK-NEXT:    vextracti128 $1, %ymm0, %xmm2
-; CHECK-NEXT:    vpshufd {{.*#+}} xmm3 = xmm2[2,3,0,1]
+; CHECK-NEXT:    vpshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
 ; CHECK-NEXT:    vpmovsxwq %xmm3, %ymm3
 ; CHECK-NEXT:    vpmovsxwq %xmm0, %ymm0
 ; CHECK-NEXT:    vpmovsxwq %xmm2, %ymm2
diff --git a/llvm/test/CodeGen/X86/nontemporal-2.ll b/llvm/test/CodeGen/X86/nontemporal-2.ll
index c83c675e78890..4e83a7ce7231b 100644
--- a/llvm/test/CodeGen/X86/nontemporal-2.ll
+++ b/llvm/test/CodeGen/X86/nontemporal-2.ll
@@ -595,14 +595,14 @@ define void @test_extract_f64(<2 x double> %arg, double* %dst) {
 define void @test_extract_i64(<2 x i64> %arg, i64* %dst) {
 ; SSE2-LABEL: test_extract_i64:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm0, %rax
 ; SSE2-NEXT:    movntiq %rax, (%rdi)
 ; SSE2-NEXT:    retq
 ;
 ; SSE4A-LABEL: test_extract_i64:
 ; SSE4A:       # %bb.0:
-; SSE4A-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE4A-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE4A-NEXT:    movq %xmm0, %rax
 ; SSE4A-NEXT:    movntiq %rax, (%rdi)
 ; SSE4A-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/oddshuffles.ll b/llvm/test/CodeGen/X86/oddshuffles.ll
index 2f74c830221b0..d24fd3f024d49 100644
--- a/llvm/test/CodeGen/X86/oddshuffles.ll
+++ b/llvm/test/CodeGen/X86/oddshuffles.ll
@@ -9,7 +9,7 @@
 define void @v3i64(<2 x i64> %a, <2 x i64> %b, <3 x i64>* %p) nounwind {
 ; SSE2-LABEL: v3i64:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; SSE2-NEXT:    movq %xmm2, 16(%rdi)
 ; SSE2-NEXT:    movdqa %xmm0, (%rdi)
@@ -285,7 +285,7 @@ define void @v7i16(<4 x i16> %a, <4 x i16> %b, <7 x i16>* %p) nounwind {
 ; SSE2-NEXT:    pshufhw {{.*#+}} xmm1 = xmm0[0,1,2,3,5,6,4,7]
 ; SSE2-NEXT:    movw %ax, 12(%rdi)
 ; SSE2-NEXT:    movq %xmm0, (%rdi)
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    movd %xmm0, 8(%rdi)
 ; SSE2-NEXT:    retq
 ;
@@ -391,7 +391,7 @@ define void @v12i8(<8 x i8> %a, <8 x i8> %b, <12 x i8>* %p) nounwind {
 ; SSE2-NEXT:    pandn %xmm1, %xmm2
 ; SSE2-NEXT:    por %xmm0, %xmm2
 ; SSE2-NEXT:    movq %xmm2, (%rdi)
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE2-NEXT:    movd %xmm0, 8(%rdi)
 ; SSE2-NEXT:    retq
 ;
@@ -539,7 +539,7 @@ define void @v12i32(<8 x i32> %a, <8 x i32> %b, <12 x i32>* %p) nounwind {
 ; SSE42-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,2,2]
 ; SSE42-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1],xmm2[2,3],xmm4[4,5,6,7]
 ; SSE42-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1,2,3],xmm0[4,5],xmm4[6,7]
-; SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
 ; SSE42-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm0[2,3],xmm2[4,5,6,7]
 ; SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,2,3,3]
@@ -637,7 +637,7 @@ define void @pr29025(<4 x i8> %a, <4 x i8> %b, <4 x i8> %c, <12 x i8> *%p) nounw
 ; SSE2-NEXT:    pandn %xmm2, %xmm1
 ; SSE2-NEXT:    por %xmm0, %xmm1
 ; SSE2-NEXT:    movq %xmm1, (%rdi)
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    movd %xmm0, 8(%rdi)
 ; SSE2-NEXT:    retq
 ;
@@ -1202,7 +1202,7 @@ define void @interleave_24i32_out(<24 x i32>* %p, <8 x i32>* %q1, <8 x i32>* %q2
 ; SSE2-NEXT:    movups 32(%rdi), %xmm10
 ; SSE2-NEXT:    movups 48(%rdi), %xmm12
 ; SSE2-NEXT:    movdqa %xmm0, %xmm11
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,0],xmm6[0,0]
 ; SSE2-NEXT:    movaps %xmm0, %xmm1
 ; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,2],xmm6[3,3]
@@ -1215,7 +1215,7 @@ define void @interleave_24i32_out(<24 x i32>* %p, <8 x i32>* %q1, <8 x i32>* %q2
 ; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,2],xmm4[3,3]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,2,3]
 ; SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[2,0],xmm8[1,0]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm12[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm12[2,3,2,3]
 ; SSE2-NEXT:    shufps {{.*#+}} xmm12 = xmm12[0,3],xmm4[0,2]
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]
 ; SSE2-NEXT:    shufps {{.*#+}} xmm5 = xmm5[0,1],xmm8[0,3]
@@ -1243,12 +1243,12 @@ define void @interleave_24i32_out(<24 x i32>* %p, <8 x i32>* %q1, <8 x i32>* %q2
 ; SSE42-NEXT:    movdqu 48(%rdi), %xmm5
 ; SSE42-NEXT:    movdqa %xmm2, %xmm6
 ; SSE42-NEXT:    pblendw {{.*#+}} xmm6 = xmm6[0,1],xmm4[2,3],xmm6[4,5,6,7]
-; SSE42-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[2,3,0,1]
+; SSE42-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[2,3,2,3]
 ; SSE42-NEXT:    shufps {{.*#+}} xmm4 = xmm4[0,3],xmm2[2,3]
 ; SSE42-NEXT:    insertps {{.*#+}} xmm4 = xmm4[0,1,2],xmm10[1]
 ; SSE42-NEXT:    movdqa %xmm9, %xmm1
 ; SSE42-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm5[2,3],xmm1[4,5,6,7]
-; SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[2,3,0,1]
+; SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[2,3,2,3]
 ; SSE42-NEXT:    shufps {{.*#+}} xmm5 = xmm5[0,3],xmm9[2,3]
 ; SSE42-NEXT:    insertps {{.*#+}} xmm5 = xmm5[0,1,2],xmm8[1]
 ; SSE42-NEXT:    pshufd {{.*#+}} xmm3 = xmm10[0,1,2,2]
@@ -1476,7 +1476,7 @@ define void @interleave_24i32_in(<24 x i32>* %p, <8 x i32>* %q1, <8 x i32>* %q2,
 ; SSE42-NEXT:    pblendw {{.*#+}} xmm7 = xmm7[0,1],xmm1[2,3],xmm7[4,5,6,7]
 ; SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[0,1,0,1]
 ; SSE42-NEXT:    pblendw {{.*#+}} xmm1 = xmm7[0,1,2,3],xmm1[4,5],xmm7[6,7]
-; SSE42-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[2,3,0,1]
+; SSE42-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[2,3,2,3]
 ; SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[2,3,2,3]
 ; SSE42-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm7[2,3],xmm0[4,5,6,7]
 ; SSE42-NEXT:    pshufd {{.*#+}} xmm7 = xmm5[2,2,3,3]
@@ -1489,7 +1489,7 @@ define void @interleave_24i32_in(<24 x i32>* %p, <8 x i32>* %q1, <8 x i32>* %q2,
 ; SSE42-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1],xmm5[2,3],xmm4[4,5,6,7]
 ; SSE42-NEXT:    pshufd {{.*#+}} xmm5 = xmm6[0,1,0,1]
 ; SSE42-NEXT:    pblendw {{.*#+}} xmm5 = xmm4[0,1,2,3],xmm5[4,5],xmm4[6,7]
-; SSE42-NEXT:    pshufd {{.*#+}} xmm4 = xmm8[2,3,0,1]
+; SSE42-NEXT:    pshufd {{.*#+}} xmm4 = xmm8[2,3,2,3]
 ; SSE42-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[2,3,2,3]
 ; SSE42-NEXT:    pblendw {{.*#+}} xmm6 = xmm6[0,1],xmm4[2,3],xmm6[4,5,6,7]
 ; SSE42-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[2,2,3,3]
diff --git a/llvm/test/CodeGen/X86/phaddsub-extract.ll b/llvm/test/CodeGen/X86/phaddsub-extract.ll
index dd258c5f424a4..7f3eb0898cc86 100644
--- a/llvm/test/CodeGen/X86/phaddsub-extract.ll
+++ b/llvm/test/CodeGen/X86/phaddsub-extract.ll
@@ -46,7 +46,7 @@ define i32 @extract_extract01_v4i32_add_i32(<4 x i32> %x) {
 define i32 @extract_extract23_v4i32_add_i32(<4 x i32> %x) {
 ; SSE3-SLOW-LABEL: extract_extract23_v4i32_add_i32:
 ; SSE3-SLOW:       # %bb.0:
-; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE3-SLOW-NEXT:    movd %xmm1, %ecx
 ; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
 ; SSE3-SLOW-NEXT:    movd %xmm0, %eax
@@ -114,7 +114,7 @@ define i32 @extract_extract01_v4i32_add_i32_commute(<4 x i32> %x) {
 define i32 @extract_extract23_v4i32_add_i32_commute(<4 x i32> %x) {
 ; SSE3-SLOW-LABEL: extract_extract23_v4i32_add_i32_commute:
 ; SSE3-SLOW:       # %bb.0:
-; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE3-SLOW-NEXT:    movd %xmm1, %ecx
 ; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
 ; SSE3-SLOW-NEXT:    movd %xmm0, %eax
@@ -326,7 +326,7 @@ define i32 @extract_extract01_v4i32_sub_i32(<4 x i32> %x) {
 define i32 @extract_extract23_v4i32_sub_i32(<4 x i32> %x) {
 ; SSE3-SLOW-LABEL: extract_extract23_v4i32_sub_i32:
 ; SSE3-SLOW:       # %bb.0:
-; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE3-SLOW-NEXT:    movd %xmm1, %eax
 ; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
 ; SSE3-SLOW-NEXT:    movd %xmm0, %ecx
@@ -382,7 +382,7 @@ define i32 @extract_extract01_v4i32_sub_i32_commute(<4 x i32> %x) {
 define i32 @extract_extract23_v4i32_sub_i32_commute(<4 x i32> %x) {
 ; SSE3-LABEL: extract_extract23_v4i32_sub_i32_commute:
 ; SSE3:       # %bb.0:
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE3-NEXT:    movd %xmm1, %ecx
 ; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
 ; SSE3-NEXT:    movd %xmm0, %eax
@@ -557,7 +557,7 @@ define i32 @extract_extract01_v8i32_add_i32(<8 x i32> %x) {
 define i32 @extract_extract23_v8i32_add_i32(<8 x i32> %x) {
 ; SSE3-SLOW-LABEL: extract_extract23_v8i32_add_i32:
 ; SSE3-SLOW:       # %bb.0:
-; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE3-SLOW-NEXT:    movd %xmm1, %ecx
 ; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
 ; SSE3-SLOW-NEXT:    movd %xmm0, %eax
@@ -594,7 +594,7 @@ define i32 @extract_extract23_v8i32_add_i32(<8 x i32> %x) {
 define i32 @extract_extract67_v8i32_add_i32(<8 x i32> %x) {
 ; SSE3-SLOW-LABEL: extract_extract67_v8i32_add_i32:
 ; SSE3-SLOW:       # %bb.0:
-; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE3-SLOW-NEXT:    movd %xmm0, %ecx
 ; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[3,1,2,3]
 ; SSE3-SLOW-NEXT:    movd %xmm0, %eax
@@ -684,7 +684,7 @@ define i32 @extract_extract01_v8i32_add_i32_commute(<8 x i32> %x) {
 define i32 @extract_extract23_v8i32_add_i32_commute(<8 x i32> %x) {
 ; SSE3-SLOW-LABEL: extract_extract23_v8i32_add_i32_commute:
 ; SSE3-SLOW:       # %bb.0:
-; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE3-SLOW-NEXT:    movd %xmm1, %ecx
 ; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
 ; SSE3-SLOW-NEXT:    movd %xmm0, %eax
@@ -721,7 +721,7 @@ define i32 @extract_extract23_v8i32_add_i32_commute(<8 x i32> %x) {
 define i32 @extract_extract67_v8i32_add_i32_commute(<8 x i32> %x) {
 ; SSE3-SLOW-LABEL: extract_extract67_v8i32_add_i32_commute:
 ; SSE3-SLOW:       # %bb.0:
-; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE3-SLOW-NEXT:    movd %xmm0, %ecx
 ; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[3,1,2,3]
 ; SSE3-SLOW-NEXT:    movd %xmm0, %eax
@@ -1119,7 +1119,7 @@ define i32 @extract_extract01_v8i32_sub_i32(<8 x i32> %x) {
 define i32 @extract_extract23_v8i32_sub_i32(<8 x i32> %x) {
 ; SSE3-SLOW-LABEL: extract_extract23_v8i32_sub_i32:
 ; SSE3-SLOW:       # %bb.0:
-; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE3-SLOW-NEXT:    movd %xmm1, %eax
 ; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
 ; SSE3-SLOW-NEXT:    movd %xmm0, %ecx
@@ -1156,7 +1156,7 @@ define i32 @extract_extract23_v8i32_sub_i32(<8 x i32> %x) {
 define i32 @extract_extract67_v8i32_sub_i32(<8 x i32> %x) {
 ; SSE3-SLOW-LABEL: extract_extract67_v8i32_sub_i32:
 ; SSE3-SLOW:       # %bb.0:
-; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE3-SLOW-NEXT:    movd %xmm0, %eax
 ; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[3,1,2,3]
 ; SSE3-SLOW-NEXT:    movd %xmm0, %ecx
@@ -1672,7 +1672,7 @@ define i32 @extract_extract01_v4i32_add_i32_uses3(<4 x i32> %x, i32* %p1, i32* %
 define i32 @partial_reduction_add_v8i32(<8 x i32> %x) {
 ; SSE3-SLOW-LABEL: partial_reduction_add_v8i32:
 ; SSE3-SLOW:       # %bb.0:
-; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE3-SLOW-NEXT:    paddd %xmm0, %xmm1
 ; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE3-SLOW-NEXT:    paddd %xmm1, %xmm0
@@ -1681,7 +1681,7 @@ define i32 @partial_reduction_add_v8i32(<8 x i32> %x) {
 ;
 ; SSE3-FAST-LABEL: partial_reduction_add_v8i32:
 ; SSE3-FAST:       # %bb.0:
-; SSE3-FAST-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE3-FAST-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE3-FAST-NEXT:    paddd %xmm0, %xmm1
 ; SSE3-FAST-NEXT:    phaddd %xmm1, %xmm1
 ; SSE3-FAST-NEXT:    movd %xmm1, %eax
@@ -1689,7 +1689,7 @@ define i32 @partial_reduction_add_v8i32(<8 x i32> %x) {
 ;
 ; AVX-SLOW-LABEL: partial_reduction_add_v8i32:
 ; AVX-SLOW:       # %bb.0:
-; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -1715,7 +1715,7 @@ define i32 @partial_reduction_add_v8i32(<8 x i32> %x) {
 define i32 @partial_reduction_add_v16i32(<16 x i32> %x) {
 ; SSE3-SLOW-LABEL: partial_reduction_add_v16i32:
 ; SSE3-SLOW:       # %bb.0:
-; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE3-SLOW-NEXT:    paddd %xmm0, %xmm1
 ; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE3-SLOW-NEXT:    paddd %xmm1, %xmm0
@@ -1724,7 +1724,7 @@ define i32 @partial_reduction_add_v16i32(<16 x i32> %x) {
 ;
 ; SSE3-FAST-LABEL: partial_reduction_add_v16i32:
 ; SSE3-FAST:       # %bb.0:
-; SSE3-FAST-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE3-FAST-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE3-FAST-NEXT:    paddd %xmm0, %xmm1
 ; SSE3-FAST-NEXT:    phaddd %xmm1, %xmm1
 ; SSE3-FAST-NEXT:    movd %xmm1, %eax
@@ -1732,7 +1732,7 @@ define i32 @partial_reduction_add_v16i32(<16 x i32> %x) {
 ;
 ; AVX-SLOW-LABEL: partial_reduction_add_v16i32:
 ; AVX-SLOW:       # %bb.0:
-; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -1758,7 +1758,7 @@ define i32 @partial_reduction_add_v16i32(<16 x i32> %x) {
 define i32 @partial_reduction_sub_v8i32(<8 x i32> %x) {
 ; SSE3-SLOW-LABEL: partial_reduction_sub_v8i32:
 ; SSE3-SLOW:       # %bb.0:
-; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE3-SLOW-NEXT:    psubd %xmm1, %xmm0
 ; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE3-SLOW-NEXT:    psubd %xmm1, %xmm0
@@ -1767,7 +1767,7 @@ define i32 @partial_reduction_sub_v8i32(<8 x i32> %x) {
 ;
 ; SSE3-FAST-LABEL: partial_reduction_sub_v8i32:
 ; SSE3-FAST:       # %bb.0:
-; SSE3-FAST-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE3-FAST-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE3-FAST-NEXT:    psubd %xmm1, %xmm0
 ; SSE3-FAST-NEXT:    phsubd %xmm0, %xmm0
 ; SSE3-FAST-NEXT:    movd %xmm0, %eax
@@ -1775,7 +1775,7 @@ define i32 @partial_reduction_sub_v8i32(<8 x i32> %x) {
 ;
 ; AVX-SLOW-LABEL: partial_reduction_sub_v8i32:
 ; AVX-SLOW:       # %bb.0:
-; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-SLOW-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
 ; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX-SLOW-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
@@ -1785,7 +1785,7 @@ define i32 @partial_reduction_sub_v8i32(<8 x i32> %x) {
 ;
 ; AVX-FAST-LABEL: partial_reduction_sub_v8i32:
 ; AVX-FAST:       # %bb.0:
-; AVX-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-FAST-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
 ; AVX-FAST-NEXT:    vphsubd %xmm0, %xmm0, %xmm0
 ; AVX-FAST-NEXT:    vmovd %xmm0, %eax
@@ -1802,7 +1802,7 @@ define i32 @partial_reduction_sub_v8i32(<8 x i32> %x) {
 define i32 @partial_reduction_sub_v16i32(<16 x i32> %x) {
 ; SSE3-SLOW-LABEL: partial_reduction_sub_v16i32:
 ; SSE3-SLOW:       # %bb.0:
-; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE3-SLOW-NEXT:    psubd %xmm1, %xmm0
 ; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE3-SLOW-NEXT:    psubd %xmm1, %xmm0
@@ -1811,7 +1811,7 @@ define i32 @partial_reduction_sub_v16i32(<16 x i32> %x) {
 ;
 ; SSE3-FAST-LABEL: partial_reduction_sub_v16i32:
 ; SSE3-FAST:       # %bb.0:
-; SSE3-FAST-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE3-FAST-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE3-FAST-NEXT:    psubd %xmm1, %xmm0
 ; SSE3-FAST-NEXT:    phsubd %xmm0, %xmm0
 ; SSE3-FAST-NEXT:    movd %xmm0, %eax
@@ -1819,7 +1819,7 @@ define i32 @partial_reduction_sub_v16i32(<16 x i32> %x) {
 ;
 ; AVX-SLOW-LABEL: partial_reduction_sub_v16i32:
 ; AVX-SLOW:       # %bb.0:
-; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-SLOW-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
 ; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX-SLOW-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
@@ -1829,7 +1829,7 @@ define i32 @partial_reduction_sub_v16i32(<16 x i32> %x) {
 ;
 ; AVX1-FAST-LABEL: partial_reduction_sub_v16i32:
 ; AVX1-FAST:       # %bb.0:
-; AVX1-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-FAST-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
 ; AVX1-FAST-NEXT:    vphsubd %xmm0, %xmm0, %xmm0
 ; AVX1-FAST-NEXT:    vmovd %xmm0, %eax
@@ -1838,7 +1838,7 @@ define i32 @partial_reduction_sub_v16i32(<16 x i32> %x) {
 ;
 ; AVX2-FAST-LABEL: partial_reduction_sub_v16i32:
 ; AVX2-FAST:       # %bb.0:
-; AVX2-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-FAST-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
 ; AVX2-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-FAST-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
@@ -1848,7 +1848,7 @@ define i32 @partial_reduction_sub_v16i32(<16 x i32> %x) {
 ;
 ; AVX512-FAST-LABEL: partial_reduction_sub_v16i32:
 ; AVX512-FAST:       # %bb.0:
-; AVX512-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-FAST-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
 ; AVX512-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-FAST-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
@@ -1868,7 +1868,7 @@ define i32 @partial_reduction_sub_v16i32(<16 x i32> %x) {
 define i16 @hadd16_8(<8 x i16> %x223) {
 ; SSE3-SLOW-LABEL: hadd16_8:
 ; SSE3-SLOW:       # %bb.0:
-; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE3-SLOW-NEXT:    paddw %xmm0, %xmm1
 ; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE3-SLOW-NEXT:    paddw %xmm1, %xmm0
@@ -1890,7 +1890,7 @@ define i16 @hadd16_8(<8 x i16> %x223) {
 ;
 ; AVX-SLOW-LABEL: hadd16_8:
 ; AVX-SLOW:       # %bb.0:
-; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-SLOW-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX-SLOW-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
@@ -1921,7 +1921,7 @@ define i16 @hadd16_8(<8 x i16> %x223) {
 define i32 @hadd32_4(<4 x i32> %x225) {
 ; SSE3-SLOW-LABEL: hadd32_4:
 ; SSE3-SLOW:       # %bb.0:
-; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE3-SLOW-NEXT:    paddd %xmm0, %xmm1
 ; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE3-SLOW-NEXT:    paddd %xmm1, %xmm0
@@ -1937,7 +1937,7 @@ define i32 @hadd32_4(<4 x i32> %x225) {
 ;
 ; AVX-SLOW-LABEL: hadd32_4:
 ; AVX-SLOW:       # %bb.0:
-; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -1961,7 +1961,7 @@ define i32 @hadd32_4(<4 x i32> %x225) {
 define i32 @hadd32_8(<8 x i32> %x225) {
 ; SSE3-SLOW-LABEL: hadd32_8:
 ; SSE3-SLOW:       # %bb.0:
-; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE3-SLOW-NEXT:    paddd %xmm0, %xmm1
 ; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE3-SLOW-NEXT:    paddd %xmm1, %xmm0
@@ -1970,7 +1970,7 @@ define i32 @hadd32_8(<8 x i32> %x225) {
 ;
 ; SSE3-FAST-LABEL: hadd32_8:
 ; SSE3-FAST:       # %bb.0:
-; SSE3-FAST-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE3-FAST-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE3-FAST-NEXT:    paddd %xmm0, %xmm1
 ; SSE3-FAST-NEXT:    phaddd %xmm1, %xmm1
 ; SSE3-FAST-NEXT:    movd %xmm1, %eax
@@ -1978,7 +1978,7 @@ define i32 @hadd32_8(<8 x i32> %x225) {
 ;
 ; AVX-SLOW-LABEL: hadd32_8:
 ; AVX-SLOW:       # %bb.0:
-; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -2004,7 +2004,7 @@ define i32 @hadd32_8(<8 x i32> %x225) {
 define i32 @hadd32_16(<16 x i32> %x225) {
 ; SSE3-SLOW-LABEL: hadd32_16:
 ; SSE3-SLOW:       # %bb.0:
-; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE3-SLOW-NEXT:    paddd %xmm0, %xmm1
 ; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE3-SLOW-NEXT:    paddd %xmm1, %xmm0
@@ -2013,7 +2013,7 @@ define i32 @hadd32_16(<16 x i32> %x225) {
 ;
 ; SSE3-FAST-LABEL: hadd32_16:
 ; SSE3-FAST:       # %bb.0:
-; SSE3-FAST-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE3-FAST-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE3-FAST-NEXT:    paddd %xmm0, %xmm1
 ; SSE3-FAST-NEXT:    phaddd %xmm1, %xmm1
 ; SSE3-FAST-NEXT:    movd %xmm1, %eax
@@ -2021,7 +2021,7 @@ define i32 @hadd32_16(<16 x i32> %x225) {
 ;
 ; AVX-SLOW-LABEL: hadd32_16:
 ; AVX-SLOW:       # %bb.0:
-; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -2119,7 +2119,7 @@ define i32 @hadd32_4_pgso(<4 x i32> %x225) !prof !14 {
 define i32 @hadd32_8_optsize(<8 x i32> %x225) optsize {
 ; SSE3-LABEL: hadd32_8_optsize:
 ; SSE3:       # %bb.0:
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE3-NEXT:    paddd %xmm0, %xmm1
 ; SSE3-NEXT:    phaddd %xmm1, %xmm1
 ; SSE3-NEXT:    movd %xmm1, %eax
@@ -2143,7 +2143,7 @@ define i32 @hadd32_8_optsize(<8 x i32> %x225) optsize {
 define i32 @hadd32_16_optsize(<16 x i32> %x225) optsize {
 ; SSE3-LABEL: hadd32_16_optsize:
 ; SSE3:       # %bb.0:
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE3-NEXT:    paddd %xmm0, %xmm1
 ; SSE3-NEXT:    phaddd %xmm1, %xmm1
 ; SSE3-NEXT:    movd %xmm1, %eax
diff --git a/llvm/test/CodeGen/X86/pmul.ll b/llvm/test/CodeGen/X86/pmul.ll
index 30f87e3d9b27b..4285e7b603f80 100644
--- a/llvm/test/CodeGen/X86/pmul.ll
+++ b/llvm/test/CodeGen/X86/pmul.ll
@@ -1083,7 +1083,7 @@ define <4 x i32> @mul_v4i64_zero_upper_left(<4 x i32> %val1, <4 x i64> %val2) {
 ;
 ; SSE41-LABEL: mul_v4i64_zero_upper_left:
 ; SSE41:       # %bb.0: # %entry
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero
 ; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero
 ; SSE41-NEXT:    movdqa %xmm4, %xmm0
@@ -1252,14 +1252,14 @@ define <8 x i64> @mul_v8i64_sext(<8 x i16> %val1, <8 x i32> %val2) {
 ; SSE2-NEXT:    movdqa %xmm0, %xmm11
 ; SSE2-NEXT:    punpckhdq {{.*#+}} xmm11 = xmm11[2],xmm15[2],xmm11[3],xmm15[3]
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm15[0],xmm0[1],xmm15[1]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
 ; SSE2-NEXT:    pxor %xmm14, %xmm14
 ; SSE2-NEXT:    pcmpgtd %xmm3, %xmm14
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm14[0],xmm3[1],xmm14[1]
 ; SSE2-NEXT:    pxor %xmm5, %xmm5
 ; SSE2-NEXT:    pcmpgtd %xmm2, %xmm5
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    pxor %xmm6, %xmm6
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm6
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm6[0],xmm1[1],xmm6[1]
@@ -1306,7 +1306,7 @@ define <8 x i64> @mul_v8i64_sext(<8 x i16> %val1, <8 x i32> %val2) {
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[3,1,2,3]
 ; SSE41-NEXT:    pmovsxwq %xmm3, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovsxwq %xmm3, %xmm5
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,2,3]
 ; SSE41-NEXT:    pmovsxwq %xmm3, %xmm6
@@ -1324,7 +1324,7 @@ define <8 x i64> @mul_v8i64_sext(<8 x i16> %val1, <8 x i32> %val2) {
 ;
 ; AVX2-LABEL: mul_v8i64_sext:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpmovsxwq %xmm2, %ymm2
 ; AVX2-NEXT:    vpmovsxwq %xmm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm3
diff --git a/llvm/test/CodeGen/X86/pmulh.ll b/llvm/test/CodeGen/X86/pmulh.ll
index 14fc182a334a8..31e113f9a003f 100644
--- a/llvm/test/CodeGen/X86/pmulh.ll
+++ b/llvm/test/CodeGen/X86/pmulh.ll
@@ -434,7 +434,7 @@ define <8 x i32> @mulhuw_v8i16_lshr(<8 x i16> %a, <8 x i16> %b) {
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pmulhuw %xmm1, %xmm0
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; SSE41-NEXT:    retq
@@ -467,7 +467,7 @@ define <8 x i32> @mulhsw_v8i16_lshr(<8 x i16> %a, <8 x i16> %b) {
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pmulhw %xmm1, %xmm0
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; SSE41-NEXT:    retq
@@ -498,7 +498,7 @@ define <8 x i32> @mulhsw_v8i16_ashr(<8 x i16> %a, <8 x i16> %b) {
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pmulhw %xmm1, %xmm0
 ; SSE41-NEXT:    pmovsxwd %xmm0, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovsxwd %xmm0, %xmm1
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; SSE41-NEXT:    retq
@@ -538,9 +538,9 @@ define <16 x i32> @mulhuw_v16i16_lshr(<16 x i16> %a, <16 x i16> %b) {
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; SSE41-NEXT:    pmulhuw %xmm3, %xmm1
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm5 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; SSE41-NEXT:    movdqa %xmm4, %xmm0
 ; SSE41-NEXT:    movdqa %xmm5, %xmm1
@@ -589,9 +589,9 @@ define <16 x i32> @mulhsw_v16i16_lshr(<16 x i16> %a, <16 x i16> %b) {
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; SSE41-NEXT:    pmulhw %xmm3, %xmm1
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm5 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; SSE41-NEXT:    movdqa %xmm4, %xmm0
 ; SSE41-NEXT:    movdqa %xmm5, %xmm1
@@ -639,9 +639,9 @@ define <16 x i32> @mulhsw_v16i16_ashr(<16 x i16> %a, <16 x i16> %b) {
 ; SSE41-NEXT:    pmovsxwd %xmm0, %xmm4
 ; SSE41-NEXT:    pmulhw %xmm3, %xmm1
 ; SSE41-NEXT:    pmovsxwd %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovsxwd %xmm0, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE41-NEXT:    pmovsxwd %xmm0, %xmm3
 ; SSE41-NEXT:    movdqa %xmm4, %xmm0
 ; SSE41-NEXT:    movdqa %xmm5, %xmm1
@@ -702,16 +702,16 @@ define <32 x i32> @mulhuw_v32i16_lshr(<32 x i16> %a, <32 x i16> %b) {
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movq %rdi, %rax
 ; SSE41-NEXT:    pmulhuw %xmm4, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero
 ; SSE41-NEXT:    pmulhuw %xmm5, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[2,3,2,3]
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero
 ; SSE41-NEXT:    pmulhuw %xmm6, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[2,3,2,3]
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm6 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero
 ; SSE41-NEXT:    pmulhuw %xmm7, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm3[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm3[2,3,2,3]
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm7 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
@@ -800,16 +800,16 @@ define <32 x i32> @mulhsw_v32i16_lshr(<32 x i16> %a, <32 x i16> %b) {
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movq %rdi, %rax
 ; SSE41-NEXT:    pmulhw %xmm4, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero
 ; SSE41-NEXT:    pmulhw %xmm5, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[2,3,2,3]
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero
 ; SSE41-NEXT:    pmulhw %xmm6, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[2,3,2,3]
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm6 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero
 ; SSE41-NEXT:    pmulhw %xmm7, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm3[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm3[2,3,2,3]
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm7 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
@@ -901,16 +901,16 @@ define <32 x i32> @mulhsw_v32i16_ashr(<32 x i16> %a, <32 x i16> %b) {
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movq %rdi, %rax
 ; SSE41-NEXT:    pmulhw %xmm4, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovsxwd %xmm4, %xmm4
 ; SSE41-NEXT:    pmulhw %xmm5, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[2,3,2,3]
 ; SSE41-NEXT:    pmovsxwd %xmm5, %xmm5
 ; SSE41-NEXT:    pmulhw %xmm6, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[2,3,2,3]
 ; SSE41-NEXT:    pmovsxwd %xmm6, %xmm6
 ; SSE41-NEXT:    pmulhw %xmm7, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm3[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm3[2,3,2,3]
 ; SSE41-NEXT:    pmovsxwd %xmm7, %xmm7
 ; SSE41-NEXT:    pmovsxwd %xmm0, %xmm0
 ; SSE41-NEXT:    pmovsxwd %xmm1, %xmm1
@@ -1026,28 +1026,28 @@ define <64 x i32> @mulhuw_v64i16_lshr(<64 x i16> %a, <64 x i16> %b) {
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movq %rdi, %rax
 ; SSE41-NEXT:    pmulhuw {{[0-9]+}}(%rsp), %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm8 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm8 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm8 = xmm8[0],zero,xmm8[1],zero,xmm8[2],zero,xmm8[3],zero
 ; SSE41-NEXT:    pmulhuw {{[0-9]+}}(%rsp), %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm9 = xmm1[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm9 = xmm1[2,3,2,3]
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm9 = xmm9[0],zero,xmm9[1],zero,xmm9[2],zero,xmm9[3],zero
 ; SSE41-NEXT:    pmulhuw {{[0-9]+}}(%rsp), %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm10 = xmm2[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm10 = xmm2[2,3,2,3]
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm10 = xmm10[0],zero,xmm10[1],zero,xmm10[2],zero,xmm10[3],zero
 ; SSE41-NEXT:    pmulhuw {{[0-9]+}}(%rsp), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm11 = xmm3[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm11 = xmm3[2,3,2,3]
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm11 = xmm11[0],zero,xmm11[1],zero,xmm11[2],zero,xmm11[3],zero
 ; SSE41-NEXT:    pmulhuw {{[0-9]+}}(%rsp), %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm12 = xmm4[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm12 = xmm4[2,3,2,3]
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm12 = xmm12[0],zero,xmm12[1],zero,xmm12[2],zero,xmm12[3],zero
 ; SSE41-NEXT:    pmulhuw {{[0-9]+}}(%rsp), %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm13 = xmm5[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm13 = xmm5[2,3,2,3]
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm13 = xmm13[0],zero,xmm13[1],zero,xmm13[2],zero,xmm13[3],zero
 ; SSE41-NEXT:    pmulhuw {{[0-9]+}}(%rsp), %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm14 = xmm6[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm14 = xmm6[2,3,2,3]
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm14 = xmm14[0],zero,xmm14[1],zero,xmm14[2],zero,xmm14[3],zero
 ; SSE41-NEXT:    pmulhuw {{[0-9]+}}(%rsp), %xmm7
-; SSE41-NEXT:    pshufd {{.*#+}} xmm15 = xmm7[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm15 = xmm7[2,3,2,3]
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm15 = xmm15[0],zero,xmm15[1],zero,xmm15[2],zero,xmm15[3],zero
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
@@ -1204,28 +1204,28 @@ define <64 x i32> @mulhsw_v64i16_lshr(<64 x i16> %a, <64 x i16> %b) {
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movq %rdi, %rax
 ; SSE41-NEXT:    pmulhw {{[0-9]+}}(%rsp), %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm8 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm8 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm8 = xmm8[0],zero,xmm8[1],zero,xmm8[2],zero,xmm8[3],zero
 ; SSE41-NEXT:    pmulhw {{[0-9]+}}(%rsp), %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm9 = xmm1[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm9 = xmm1[2,3,2,3]
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm9 = xmm9[0],zero,xmm9[1],zero,xmm9[2],zero,xmm9[3],zero
 ; SSE41-NEXT:    pmulhw {{[0-9]+}}(%rsp), %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm10 = xmm2[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm10 = xmm2[2,3,2,3]
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm10 = xmm10[0],zero,xmm10[1],zero,xmm10[2],zero,xmm10[3],zero
 ; SSE41-NEXT:    pmulhw {{[0-9]+}}(%rsp), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm11 = xmm3[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm11 = xmm3[2,3,2,3]
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm11 = xmm11[0],zero,xmm11[1],zero,xmm11[2],zero,xmm11[3],zero
 ; SSE41-NEXT:    pmulhw {{[0-9]+}}(%rsp), %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm12 = xmm4[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm12 = xmm4[2,3,2,3]
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm12 = xmm12[0],zero,xmm12[1],zero,xmm12[2],zero,xmm12[3],zero
 ; SSE41-NEXT:    pmulhw {{[0-9]+}}(%rsp), %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm13 = xmm5[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm13 = xmm5[2,3,2,3]
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm13 = xmm13[0],zero,xmm13[1],zero,xmm13[2],zero,xmm13[3],zero
 ; SSE41-NEXT:    pmulhw {{[0-9]+}}(%rsp), %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm14 = xmm6[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm14 = xmm6[2,3,2,3]
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm14 = xmm14[0],zero,xmm14[1],zero,xmm14[2],zero,xmm14[3],zero
 ; SSE41-NEXT:    pmulhw {{[0-9]+}}(%rsp), %xmm7
-; SSE41-NEXT:    pshufd {{.*#+}} xmm15 = xmm7[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm15 = xmm7[2,3,2,3]
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm15 = xmm15[0],zero,xmm15[1],zero,xmm15[2],zero,xmm15[3],zero
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
@@ -1386,28 +1386,28 @@ define <64 x i32> @mulhsw_v64i16_ashr(<64 x i16> %a, <64 x i16> %b) {
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movq %rdi, %rax
 ; SSE41-NEXT:    pmulhw {{[0-9]+}}(%rsp), %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm8 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm8 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovsxwd %xmm8, %xmm8
 ; SSE41-NEXT:    pmulhw {{[0-9]+}}(%rsp), %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm9 = xmm1[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm9 = xmm1[2,3,2,3]
 ; SSE41-NEXT:    pmovsxwd %xmm9, %xmm9
 ; SSE41-NEXT:    pmulhw {{[0-9]+}}(%rsp), %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm10 = xmm2[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm10 = xmm2[2,3,2,3]
 ; SSE41-NEXT:    pmovsxwd %xmm10, %xmm10
 ; SSE41-NEXT:    pmulhw {{[0-9]+}}(%rsp), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm11 = xmm3[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm11 = xmm3[2,3,2,3]
 ; SSE41-NEXT:    pmovsxwd %xmm11, %xmm11
 ; SSE41-NEXT:    pmulhw {{[0-9]+}}(%rsp), %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm12 = xmm4[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm12 = xmm4[2,3,2,3]
 ; SSE41-NEXT:    pmovsxwd %xmm12, %xmm12
 ; SSE41-NEXT:    pmulhw {{[0-9]+}}(%rsp), %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm13 = xmm5[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm13 = xmm5[2,3,2,3]
 ; SSE41-NEXT:    pmovsxwd %xmm13, %xmm13
 ; SSE41-NEXT:    pmulhw {{[0-9]+}}(%rsp), %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm14 = xmm6[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm14 = xmm6[2,3,2,3]
 ; SSE41-NEXT:    pmovsxwd %xmm14, %xmm14
 ; SSE41-NEXT:    pmulhw {{[0-9]+}}(%rsp), %xmm7
-; SSE41-NEXT:    pshufd {{.*#+}} xmm15 = xmm7[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm15 = xmm7[2,3,2,3]
 ; SSE41-NEXT:    pmovsxwd %xmm15, %xmm15
 ; SSE41-NEXT:    pmovsxwd %xmm0, %xmm0
 ; SSE41-NEXT:    pmovsxwd %xmm1, %xmm1
@@ -1541,7 +1541,7 @@ define <8 x i64> @mulhuw_v8i16_lshr_i64(<8 x i16> %a, <8 x i16> %b) {
 ; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm4 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
 ; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm3 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
@@ -1552,7 +1552,7 @@ define <8 x i64> @mulhuw_v8i16_lshr_i64(<8 x i16> %a, <8 x i16> %b) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpmulhuw %xmm1, %xmm0, %xmm1
 ; AVX2-NEXT:    vpmovzxwq {{.*#+}} ymm0 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; AVX2-NEXT:    vpmovzxwq {{.*#+}} ymm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; AVX2-NEXT:    retq
 ;
@@ -1648,7 +1648,7 @@ define <8 x i64> @mulhsw_v8i16_lshr_i64(<8 x i16> %a, <8 x i16> %b) {
 ; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm4 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
 ; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm3 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
@@ -1659,7 +1659,7 @@ define <8 x i64> @mulhsw_v8i16_lshr_i64(<8 x i16> %a, <8 x i16> %b) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpmulhw %xmm1, %xmm0, %xmm1
 ; AVX2-NEXT:    vpmovzxwq {{.*#+}} ymm0 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; AVX2-NEXT:    vpmovzxwq {{.*#+}} ymm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; AVX2-NEXT:    retq
 ;
@@ -1775,7 +1775,7 @@ define <8 x i64> @mulhsw_v8i16_ashr_i64(<8 x i16> %a, <8 x i16> %b) {
 ; SSE41-NEXT:    pmovsxwq %xmm0, %xmm4
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE41-NEXT:    pmovsxwq %xmm1, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovsxwq %xmm2, %xmm2
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
 ; SSE41-NEXT:    pmovsxwq %xmm0, %xmm3
@@ -1786,7 +1786,7 @@ define <8 x i64> @mulhsw_v8i16_ashr_i64(<8 x i16> %a, <8 x i16> %b) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpmulhw %xmm1, %xmm0, %xmm1
 ; AVX2-NEXT:    vpmovsxwq %xmm1, %ymm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; AVX2-NEXT:    vpmovsxwq %xmm1, %ymm1
 ; AVX2-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/pr15267.ll b/llvm/test/CodeGen/X86/pr15267.ll
index 73acb76ce55fc..b8ecadba81c0e 100644
--- a/llvm/test/CodeGen/X86/pr15267.ll
+++ b/llvm/test/CodeGen/X86/pr15267.ll
@@ -75,7 +75,7 @@ define <4 x i64> @test3(<4 x i1>* %in) nounwind {
 ; CHECK-NEXT:    negl %eax
 ; CHECK-NEXT:    vpinsrd $3, %eax, %xmm0, %xmm0
 ; CHECK-NEXT:    vpmovsxdq %xmm0, %xmm1
-; CHECK-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; CHECK-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; CHECK-NEXT:    vpmovsxdq %xmm0, %xmm0
 ; CHECK-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; CHECK-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/pr39733.ll b/llvm/test/CodeGen/X86/pr39733.ll
index 75f9dc51b85eb..31bd5b71d0a6e 100644
--- a/llvm/test/CodeGen/X86/pr39733.ll
+++ b/llvm/test/CodeGen/X86/pr39733.ll
@@ -21,7 +21,7 @@ define void @test55() {
 ; CHECK-NEXT:    vpmovsxwd %xmm0, %xmm1
 ; CHECK-NEXT:    # implicit-def: $ymm2
 ; CHECK-NEXT:    vmovaps %xmm1, %xmm2
-; CHECK-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; CHECK-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; CHECK-NEXT:    vpmovsxwd %xmm0, %xmm0
 ; CHECK-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
 ; CHECK-NEXT:    vmovdqa %ymm0, (%rsp)
diff --git a/llvm/test/CodeGen/X86/pr42452.ll b/llvm/test/CodeGen/X86/pr42452.ll
index f2f0cd2d3ce6b..d3a1dad42bd39 100644
--- a/llvm/test/CodeGen/X86/pr42452.ll
+++ b/llvm/test/CodeGen/X86/pr42452.ll
@@ -8,7 +8,7 @@ define void @foo(i1 %c, <2 x i64> %x) {
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    # kill: def $dil killed $dil killed $edi
 ; CHECK-NEXT:    movq %xmm0, %rax
-; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; CHECK-NEXT:    movq %xmm0, %rcx
 ; CHECK-NEXT:    movb %dil, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
 ; CHECK-NEXT:    movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
diff --git a/llvm/test/CodeGen/X86/pr42905.ll b/llvm/test/CodeGen/X86/pr42905.ll
index 310a173f824e9..6ebe5be45a4f8 100644
--- a/llvm/test/CodeGen/X86/pr42905.ll
+++ b/llvm/test/CodeGen/X86/pr42905.ll
@@ -7,7 +7,7 @@ define <4 x double> @autogen_SD30452(i1 %L230) {
 ; CHECK-NEXT:    movdqa {{.*#+}} xmm1 = [151829,151829]
 ; CHECK-NEXT:    movq %xmm0, %rax
 ; CHECK-NEXT:    cvtsi2sd %rax, %xmm0
-; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; CHECK-NEXT:    movq %xmm2, %rax
 ; CHECK-NEXT:    xorps %xmm2, %xmm2
 ; CHECK-NEXT:    cvtsi2sd %rax, %xmm2
diff --git a/llvm/test/CodeGen/X86/pr44976.ll b/llvm/test/CodeGen/X86/pr44976.ll
index 871937d295508..72ced2b1fa0f4 100644
--- a/llvm/test/CodeGen/X86/pr44976.ll
+++ b/llvm/test/CodeGen/X86/pr44976.ll
@@ -57,7 +57,7 @@ define <3 x i32> @f_29(<12 x i16> %a, <12 x i16> %b) {
 ; CHECK-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
 ; CHECK-NEXT:    shufps {{.*#+}} xmm1 = xmm1[2,3],xmm2[0,1]
 ; CHECK-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1,2,3]
-; CHECK-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[3,3,0,1]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[3,3,2,3]
 ; CHECK-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
 ; CHECK-NEXT:    paddd %xmm3, %xmm0
 ; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,1,3]
diff --git a/llvm/test/CodeGen/X86/pr45378.ll b/llvm/test/CodeGen/X86/pr45378.ll
index 681e06ada65d2..36b2de07bcc83 100644
--- a/llvm/test/CodeGen/X86/pr45378.ll
+++ b/llvm/test/CodeGen/X86/pr45378.ll
@@ -77,7 +77,7 @@ define i1 @parseHeaders2_scalar_and(i64 * %ptr) nounwind {
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movdqu (%rdi), %xmm0
 ; SSE2-NEXT:    movq %xmm0, %rax
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm0, %rcx
 ; SSE2-NEXT:    testq %rcx, %rax
 ; SSE2-NEXT:    sete %al
diff --git a/llvm/test/CodeGen/X86/pr46189.ll b/llvm/test/CodeGen/X86/pr46189.ll
index 558483754c680..97190c10ef7c9 100644
--- a/llvm/test/CodeGen/X86/pr46189.ll
+++ b/llvm/test/CodeGen/X86/pr46189.ll
@@ -21,7 +21,7 @@ define { i64, i64 } @PR46189(double %0, double %1, double %2, double %3, double
 ; SSE-NEXT:    cvttpd2dq %xmm3, %xmm1
 ; SSE-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; SSE-NEXT:    movq %xmm0, %rax
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE-NEXT:    movq %xmm0, %rdx
 ; SSE-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/pr46455.ll b/llvm/test/CodeGen/X86/pr46455.ll
index e5ed94aa54934..c232d548faefd 100644
--- a/llvm/test/CodeGen/X86/pr46455.ll
+++ b/llvm/test/CodeGen/X86/pr46455.ll
@@ -10,7 +10,7 @@ define void @EntryModule(i8** %buffer_table) {
 ; CHECK-NEXT:    vcmpneqps (%rax), %ymm0, %ymm0
 ; CHECK-NEXT:    vpsrld $31, %xmm0, %xmm1
 ; CHECK-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,2,3]
-; CHECK-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[2,3,0,1]
+; CHECK-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
 ; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]
 ; CHECK-NEXT:    vpaddd %xmm1, %xmm3, %xmm1
 ; CHECK-NEXT:    vpsubd %xmm0, %xmm2, %xmm0
diff --git a/llvm/test/CodeGen/X86/prefer-avx256-mask-shuffle.ll b/llvm/test/CodeGen/X86/prefer-avx256-mask-shuffle.ll
index 0706254f4e5c0..bdbc563816864 100644
--- a/llvm/test/CodeGen/X86/prefer-avx256-mask-shuffle.ll
+++ b/llvm/test/CodeGen/X86/prefer-avx256-mask-shuffle.ll
@@ -135,7 +135,7 @@ define <32 x i1> @shuf32i1_3_6_22_12_3_7_7_0_3_6_1_13_3_21_7_0_3_6_22_12_3_7_7_0
 ; AVX256VL-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX256VL-NEXT:    vpmovsxbd %xmm1, %ymm1
 ; AVX256VL-NEXT:    vptestmd %ymm1, %ymm1, %k1
-; AVX256VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX256VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX256VL-NEXT:    vpmovsxbd %xmm1, %ymm1
 ; AVX256VL-NEXT:    vptestmd %ymm1, %ymm1, %k2
 ; AVX256VL-NEXT:    vpmovsxbd %xmm0, %ymm0
diff --git a/llvm/test/CodeGen/X86/psubus.ll b/llvm/test/CodeGen/X86/psubus.ll
index 28ab2e1968dbe..d785b10f9c327 100644
--- a/llvm/test/CodeGen/X86/psubus.ll
+++ b/llvm/test/CodeGen/X86/psubus.ll
@@ -522,7 +522,7 @@ define <8 x i16> @test13(<8 x i16> %x, <8 x i32> %y) nounwind {
 ;
 ; SSE41-LABEL: test13:
 ; SSE41:       # %bb.0: # %vector.ph
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm4 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm5 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; SSE41-NEXT:    movdqa %xmm5, %xmm3
@@ -697,7 +697,7 @@ define <16 x i8> @test14(<16 x i8> %x, <16 x i32> %y) nounwind {
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,1,2,3]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm8 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm5 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm7 = xmm7[0],zero,zero,zero,xmm7[1],zero,zero,zero,xmm7[2],zero,zero,zero,xmm7[3],zero,zero,zero
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm0[3,1,2,3]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm6 = xmm6[0],zero,zero,zero,xmm6[1],zero,zero,zero,xmm6[2],zero,zero,zero,xmm6[3],zero,zero,zero
@@ -735,9 +735,9 @@ define <16 x i8> @test14(<16 x i8> %x, <16 x i32> %y) nounwind {
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm4 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm5 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm0[3,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm0[3,3,2,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm6 = xmm6[0],zero,zero,zero,xmm6[1],zero,zero,zero,xmm6[2],zero,zero,zero,xmm6[3],zero,zero,zero
 ; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm7
 ; AVX1-NEXT:    vpmaxud %xmm6, %xmm7, %xmm6
@@ -772,7 +772,7 @@ define <16 x i8> @test14(<16 x i8> %x, <16 x i32> %y) nounwind {
 ;
 ; AVX2-LABEL: test14:
 ; AVX2:       # %bb.0: # %vector.ph
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero,xmm3[4],zero,zero,zero,xmm3[5],zero,zero,zero,xmm3[6],zero,zero,zero,xmm3[7],zero,zero,zero
 ; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm4 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
 ; AVX2-NEXT:    vpmaxud %ymm4, %ymm1, %ymm4
@@ -873,7 +873,7 @@ define <8 x i16> @test15(<8 x i16> %x, <8 x i32> %y) nounwind {
 ;
 ; SSE41-LABEL: test15:
 ; SSE41:       # %bb.0: # %vector.ph
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; SSE41-NEXT:    movdqa %xmm4, %xmm5
@@ -1005,7 +1005,7 @@ define <8 x i16> @test16(<8 x i16> %x, <8 x i32> %y) nounwind {
 ;
 ; SSE41-LABEL: test16:
 ; SSE41:       # %bb.0: # %vector.ph
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; SSE41-NEXT:    pmaxud %xmm1, %xmm4
@@ -1871,10 +1871,10 @@ define <16 x i16> @psubus_16i32_max(<16 x i16> %x, <16 x i32> %y) nounwind {
 ;
 ; SSE41-LABEL: psubus_16i32_max:
 ; SSE41:       # %bb.0: # %vector.ph
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[2,3,2,3]
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm6 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm7 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; SSE41-NEXT:    pmaxud %xmm2, %xmm0
diff --git a/llvm/test/CodeGen/X86/sad.ll b/llvm/test/CodeGen/X86/sad.ll
index f55a58048e227..a197e795754ad 100644
--- a/llvm/test/CodeGen/X86/sad.ll
+++ b/llvm/test/CodeGen/X86/sad.ll
@@ -27,7 +27,7 @@ define i32 @sad_16i8() nounwind {
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
 ; SSE2-NEXT:    paddd %xmm0, %xmm0
 ; SSE2-NEXT:    paddd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE2-NEXT:    paddd %xmm1, %xmm0
@@ -54,7 +54,7 @@ define i32 @sad_16i8() nounwind {
 ; AVX1-NEXT:    vpaddd %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vpaddd %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -79,7 +79,7 @@ define i32 @sad_16i8() nounwind {
 ; AVX2-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -104,7 +104,7 @@ define i32 @sad_16i8() nounwind {
 ; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -172,7 +172,7 @@ define i32 @sad_32i8() nounwind {
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
 ; SSE2-NEXT:    paddd %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    paddd %xmm1, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
@@ -207,7 +207,7 @@ define i32 @sad_32i8() nounwind {
 ; AVX1-NEXT:    vpaddd %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpaddd %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -234,7 +234,7 @@ define i32 @sad_32i8() nounwind {
 ; AVX2-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -261,7 +261,7 @@ define i32 @sad_32i8() nounwind {
 ; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -346,7 +346,7 @@ define i32 @sad_avx64i8() nounwind {
 ; SSE2-NEXT:    paddd %xmm3, %xmm1
 ; SSE2-NEXT:    paddd %xmm2, %xmm1
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    paddd %xmm1, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
@@ -397,7 +397,7 @@ define i32 @sad_avx64i8() nounwind {
 ; AVX1-NEXT:    vpaddd %xmm2, %xmm6, %xmm2
 ; AVX1-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -431,7 +431,7 @@ define i32 @sad_avx64i8() nounwind {
 ; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -463,7 +463,7 @@ define i32 @sad_avx64i8() nounwind {
 ; AVX512F-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512F-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX512F-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512F-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512F-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512F-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512F-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -492,7 +492,7 @@ define i32 @sad_avx64i8() nounwind {
 ; AVX512BW-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512BW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512BW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512BW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -628,7 +628,7 @@ define i32 @sad_4i8() nounwind {
 ; SSE2-NEXT:    addq $4, %rax
 ; SSE2-NEXT:    jne .LBB4_1
 ; SSE2-NEXT:  # %bb.2: # %middle.block
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE2-NEXT:    paddd %xmm1, %xmm0
@@ -649,7 +649,7 @@ define i32 @sad_4i8() nounwind {
 ; AVX-NEXT:    addq $4, %rax
 ; AVX-NEXT:    jne .LBB4_1
 ; AVX-NEXT:  # %bb.2: # %middle.block
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -760,7 +760,7 @@ define i32 @sad_nonloop_16i8(<16 x i8>* nocapture readonly %p, i64, <16 x i8>* n
 ; SSE2-NEXT:    movdqu (%rdi), %xmm0
 ; SSE2-NEXT:    movdqu (%rdx), %xmm1
 ; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    paddq %xmm1, %xmm0
 ; SSE2-NEXT:    movd %xmm0, %eax
 ; SSE2-NEXT:    retq
@@ -769,7 +769,7 @@ define i32 @sad_nonloop_16i8(<16 x i8>* nocapture readonly %p, i64, <16 x i8>* n
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vmovdqu (%rdi), %xmm0
 ; AVX-NEXT:    vpsadbw (%rdx), %xmm0, %xmm0
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovd %xmm0, %eax
 ; AVX-NEXT:    retq
@@ -803,7 +803,7 @@ define i32 @sad_nonloop_32i8(<32 x i8>* nocapture readonly %p, i64, <32 x i8>* n
 ; SSE2-NEXT:    movdqu 16(%rdi), %xmm0
 ; SSE2-NEXT:    psadbw %xmm1, %xmm0
 ; SSE2-NEXT:    paddq %xmm2, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    paddq %xmm0, %xmm1
 ; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    retq
@@ -815,7 +815,7 @@ define i32 @sad_nonloop_32i8(<32 x i8>* nocapture readonly %p, i64, <32 x i8>* n
 ; AVX1-NEXT:    vpsadbw 16(%rdx), %xmm1, %xmm1
 ; AVX1-NEXT:    vpsadbw (%rdx), %xmm0, %xmm0
 ; AVX1-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
 ; AVX1-NEXT:    retq
@@ -826,7 +826,7 @@ define i32 @sad_nonloop_32i8(<32 x i8>* nocapture readonly %p, i64, <32 x i8>* n
 ; AVX2-NEXT:    vpsadbw (%rdx), %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
 ; AVX2-NEXT:    vzeroupper
@@ -838,7 +838,7 @@ define i32 @sad_nonloop_32i8(<32 x i8>* nocapture readonly %p, i64, <32 x i8>* n
 ; AVX512-NEXT:    vpsadbw (%rdx), %ymm0, %ymm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
 ; AVX512-NEXT:    vzeroupper
@@ -883,7 +883,7 @@ define i32 @sad_nonloop_64i8(<64 x i8>* nocapture readonly %p, i64, <64 x i8>* n
 ; SSE2-NEXT:    paddq %xmm0, %xmm2
 ; SSE2-NEXT:    paddq %xmm1, %xmm2
 ; SSE2-NEXT:    paddq %xmm4, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE2-NEXT:    paddq %xmm2, %xmm0
 ; SSE2-NEXT:    movd %xmm0, %eax
 ; SSE2-NEXT:    retq
@@ -901,7 +901,7 @@ define i32 @sad_nonloop_64i8(<64 x i8>* nocapture readonly %p, i64, <64 x i8>* n
 ; AVX1-NEXT:    vpaddq %xmm1, %xmm2, %xmm1
 ; AVX1-NEXT:    vpsadbw (%rdx), %xmm0, %xmm0
 ; AVX1-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
 ; AVX1-NEXT:    retq
@@ -915,7 +915,7 @@ define i32 @sad_nonloop_64i8(<64 x i8>* nocapture readonly %p, i64, <64 x i8>* n
 ; AVX2-NEXT:    vpaddq %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
 ; AVX2-NEXT:    vzeroupper
@@ -931,7 +931,7 @@ define i32 @sad_nonloop_64i8(<64 x i8>* nocapture readonly %p, i64, <64 x i8>* n
 ; AVX512F-NEXT:    vpaddq %zmm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512F-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
-; AVX512F-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512F-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512F-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
 ; AVX512F-NEXT:    vmovd %xmm0, %eax
 ; AVX512F-NEXT:    vzeroupper
@@ -945,7 +945,7 @@ define i32 @sad_nonloop_64i8(<64 x i8>* nocapture readonly %p, i64, <64 x i8>* n
 ; AVX512BW-NEXT:    vpaddq %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512BW-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512BW-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vmovd %xmm0, %eax
 ; AVX512BW-NEXT:    vzeroupper
@@ -987,7 +987,7 @@ define i32 @sad_unroll_nonzero_initial(<16 x i8>* %arg, <16 x i8>* %arg1, <16 x
 ; SSE2-NEXT:    psadbw %xmm0, %xmm2
 ; SSE2-NEXT:    paddd %xmm1, %xmm2
 ; SSE2-NEXT:    paddd {{.*}}(%rip), %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE2-NEXT:    paddd %xmm2, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
@@ -1002,7 +1002,7 @@ define i32 @sad_unroll_nonzero_initial(<16 x i8>* %arg, <16 x i8>* %arg1, <16 x
 ; AVX-NEXT:    vpsadbw (%rcx), %xmm1, %xmm1
 ; AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpaddd {{.*}}(%rip), %xmm0, %xmm0
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -1051,7 +1051,7 @@ define i32 @sad_double_reduction(<16 x i8>* %arg, <16 x i8>* %arg1, <16 x i8>* %
 ; SSE2-NEXT:    movdqu (%rcx), %xmm2
 ; SSE2-NEXT:    psadbw %xmm0, %xmm2
 ; SSE2-NEXT:    paddd %xmm1, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE2-NEXT:    paddd %xmm2, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
@@ -1065,7 +1065,7 @@ define i32 @sad_double_reduction(<16 x i8>* %arg, <16 x i8>* %arg1, <16 x i8>* %
 ; AVX-NEXT:    vmovdqu (%rdx), %xmm1
 ; AVX-NEXT:    vpsadbw (%rcx), %xmm1, %xmm1
 ; AVX-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
diff --git a/llvm/test/CodeGen/X86/sdiv_fix.ll b/llvm/test/CodeGen/X86/sdiv_fix.ll
index 02400d28c0e2c..3b3a1b57ecd02 100644
--- a/llvm/test/CodeGen/X86/sdiv_fix.ll
+++ b/llvm/test/CodeGen/X86/sdiv_fix.ll
@@ -443,13 +443,13 @@ define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; X64:       # %bb.0:
 ; X64-NEXT:    pxor %xmm2, %xmm2
 ; X64-NEXT:    pcmpgtd %xmm1, %xmm2
-; X64-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,0,1]
+; X64-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
 ; X64-NEXT:    movdqa %xmm1, %xmm4
 ; X64-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
 ; X64-NEXT:    movq %xmm4, %rcx
 ; X64-NEXT:    pxor %xmm2, %xmm2
 ; X64-NEXT:    pcmpgtd %xmm0, %xmm2
-; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
 ; X64-NEXT:    psllq $31, %xmm0
 ; X64-NEXT:    movq %xmm0, %rax
@@ -457,9 +457,9 @@ define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; X64-NEXT:    idivq %rcx
 ; X64-NEXT:    movq %rax, %r8
 ; X64-NEXT:    movq %rdx, %r11
-; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[2,3,0,1]
+; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[2,3,2,3]
 ; X64-NEXT:    movq %xmm2, %rcx
-; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; X64-NEXT:    movq %xmm2, %rax
 ; X64-NEXT:    cqto
 ; X64-NEXT:    idivq %rcx
@@ -478,9 +478,9 @@ define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; X64-NEXT:    idivq %rdi
 ; X64-NEXT:    movq %rax, %r9
 ; X64-NEXT:    movq %rdx, %rdi
-; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,0,1]
+; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]
 ; X64-NEXT:    movq %xmm2, %rsi
-; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; X64-NEXT:    movq %xmm2, %rax
 ; X64-NEXT:    cqto
 ; X64-NEXT:    idivq %rsi
diff --git a/llvm/test/CodeGen/X86/sdiv_fix_sat.ll b/llvm/test/CodeGen/X86/sdiv_fix_sat.ll
index e5348b9febb4d..512488e8f8725 100644
--- a/llvm/test/CodeGen/X86/sdiv_fix_sat.ll
+++ b/llvm/test/CodeGen/X86/sdiv_fix_sat.ll
@@ -651,14 +651,14 @@ define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; X64-NEXT:    cmoveq %rax, %r13
 ; X64-NEXT:    movq %r13, %xmm0
 ; X64-NEXT:    movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; X64-NEXT:    pshufd $78, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
-; X64-NEXT:    # xmm0 = mem[2,3,0,1]
+; X64-NEXT:    pshufd $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; X64-NEXT:    # xmm0 = mem[2,3,2,3]
 ; X64-NEXT:    movq %xmm0, %rbx
 ; X64-NEXT:    movq %rbx, %r13
 ; X64-NEXT:    sarq $63, %r13
 ; X64-NEXT:    shldq $31, %rbx, %r13
-; X64-NEXT:    pshufd $78, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
-; X64-NEXT:    # xmm0 = mem[2,3,0,1]
+; X64-NEXT:    pshufd $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; X64-NEXT:    # xmm0 = mem[2,3,2,3]
 ; X64-NEXT:    movq %xmm0, %rdx
 ; X64-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; X64-NEXT:    movq %rdx, %rbp
@@ -709,8 +709,8 @@ define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; X64-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
 ; X64-NEXT:    psrlq $1, %xmm1
 ; X64-NEXT:    movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; X64-NEXT:    pshufd $78, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
-; X64-NEXT:    # xmm1 = mem[2,3,0,1]
+; X64-NEXT:    pshufd $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
+; X64-NEXT:    # xmm1 = mem[2,3,2,3]
 ; X64-NEXT:    pxor %xmm0, %xmm0
 ; X64-NEXT:    pcmpgtd %xmm1, %xmm0
 ; X64-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
@@ -720,8 +720,8 @@ define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; X64-NEXT:    movq %rbx, %r12
 ; X64-NEXT:    sarq $63, %r12
 ; X64-NEXT:    shldq $31, %rbx, %r12
-; X64-NEXT:    pshufd $78, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
-; X64-NEXT:    # xmm1 = mem[2,3,0,1]
+; X64-NEXT:    pshufd $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
+; X64-NEXT:    # xmm1 = mem[2,3,2,3]
 ; X64-NEXT:    pxor %xmm0, %xmm0
 ; X64-NEXT:    pcmpgtd %xmm1, %xmm0
 ; X64-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
@@ -773,14 +773,14 @@ define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; X64-NEXT:    cmoveq %rax, %r13
 ; X64-NEXT:    movq %r13, %xmm0
 ; X64-NEXT:    movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; X64-NEXT:    pshufd $78, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
-; X64-NEXT:    # xmm0 = mem[2,3,0,1]
+; X64-NEXT:    pshufd $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; X64-NEXT:    # xmm0 = mem[2,3,2,3]
 ; X64-NEXT:    movq %xmm0, %rbx
 ; X64-NEXT:    movq %rbx, %r13
 ; X64-NEXT:    sarq $63, %r13
 ; X64-NEXT:    shldq $31, %rbx, %r13
-; X64-NEXT:    pshufd $78, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
-; X64-NEXT:    # xmm0 = mem[2,3,0,1]
+; X64-NEXT:    pshufd $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; X64-NEXT:    # xmm0 = mem[2,3,2,3]
 ; X64-NEXT:    movq %xmm0, %rdx
 ; X64-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; X64-NEXT:    movq %rdx, %rbp
diff --git a/llvm/test/CodeGen/X86/setcc-wide-types.ll b/llvm/test/CodeGen/X86/setcc-wide-types.ll
index 74a6ae58f9f9f..8a788f41d5cc2 100644
--- a/llvm/test/CodeGen/X86/setcc-wide-types.ll
+++ b/llvm/test/CodeGen/X86/setcc-wide-types.ll
@@ -75,16 +75,16 @@ define i32 @eq_i128(<2 x i64> %x, <2 x i64> %y) {
 define i32 @ne_i256(<4 x i64> %x, <4 x i64> %y) {
 ; SSE2-LABEL: ne_i256:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm4, %rax
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm4, %rcx
 ; SSE2-NEXT:    movq %xmm0, %rdx
 ; SSE2-NEXT:    movq %xmm1, %r8
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm0, %rdi
 ; SSE2-NEXT:    xorq %rax, %rdi
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm0, %rsi
 ; SSE2-NEXT:    xorq %rcx, %rsi
 ; SSE2-NEXT:    orq %rdi, %rsi
@@ -155,16 +155,16 @@ define i32 @ne_i256(<4 x i64> %x, <4 x i64> %y) {
 define i32 @eq_i256(<4 x i64> %x, <4 x i64> %y) {
 ; SSE2-LABEL: eq_i256:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm4, %rax
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm4, %rcx
 ; SSE2-NEXT:    movq %xmm0, %rdx
 ; SSE2-NEXT:    movq %xmm1, %r8
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm0, %rdi
 ; SSE2-NEXT:    xorq %rax, %rdi
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm0, %rsi
 ; SSE2-NEXT:    xorq %rcx, %rsi
 ; SSE2-NEXT:    orq %rdi, %rsi
@@ -235,28 +235,28 @@ define i32 @eq_i256(<4 x i64> %x, <4 x i64> %y) {
 define i32 @ne_i512(<8 x i64> %x, <8 x i64> %y) {
 ; SSE2-LABEL: ne_i512:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm8, %rax
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm2[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm2[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm8, %rcx
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm8, %rdx
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm3[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm3[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm8, %rsi
 ; SSE2-NEXT:    movq %xmm0, %r11
 ; SSE2-NEXT:    movq %xmm2, %r8
 ; SSE2-NEXT:    movq %xmm1, %r9
 ; SSE2-NEXT:    movq %xmm3, %r10
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm0, %rdi
 ; SSE2-NEXT:    xorq %rax, %rdi
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm0, %rax
 ; SSE2-NEXT:    xorq %rcx, %rax
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm0, %rcx
 ; SSE2-NEXT:    xorq %rdx, %rcx
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm0, %rdx
 ; SSE2-NEXT:    xorq %rsi, %rdx
 ; SSE2-NEXT:    orq %rcx, %rdx
@@ -426,28 +426,28 @@ define i32 @ne_i512(<8 x i64> %x, <8 x i64> %y) {
 define i32 @eq_i512(<8 x i64> %x, <8 x i64> %y) {
 ; SSE2-LABEL: eq_i512:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm8, %rax
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm2[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm2[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm8, %rcx
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm8, %rdx
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm3[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm3[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm8, %rsi
 ; SSE2-NEXT:    movq %xmm0, %r11
 ; SSE2-NEXT:    movq %xmm2, %r8
 ; SSE2-NEXT:    movq %xmm1, %r9
 ; SSE2-NEXT:    movq %xmm3, %r10
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm0, %rdi
 ; SSE2-NEXT:    xorq %rax, %rdi
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm0, %rax
 ; SSE2-NEXT:    xorq %rcx, %rax
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm0, %rcx
 ; SSE2-NEXT:    xorq %rdx, %rcx
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm0, %rdx
 ; SSE2-NEXT:    xorq %rsi, %rdx
 ; SSE2-NEXT:    orq %rcx, %rdx
diff --git a/llvm/test/CodeGen/X86/shrink_vmul.ll b/llvm/test/CodeGen/X86/shrink_vmul.ll
index 6a5fab8469fa7..17cfc53266909 100644
--- a/llvm/test/CodeGen/X86/shrink_vmul.ll
+++ b/llvm/test/CodeGen/X86/shrink_vmul.ll
@@ -2094,9 +2094,9 @@ define void @PR34947(<9 x i16>* %p0, <9 x i32>* %p1) nounwind {
 ; X86-SSE-NEXT:    xorl %edx, %edx
 ; X86-SSE-NEXT:    divl %esi
 ; X86-SSE-NEXT:    movd %edx, %xmm3
-; X86-SSE-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[2,3,0,1]
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[2,3,2,3]
 ; X86-SSE-NEXT:    movd %xmm7, %eax
-; X86-SSE-NEXT:    pshufd {{.*#+}} xmm7 = xmm5[2,3,0,1]
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm7 = xmm5[2,3,2,3]
 ; X86-SSE-NEXT:    movd %xmm7, %esi
 ; X86-SSE-NEXT:    xorl %edx, %edx
 ; X86-SSE-NEXT:    divl %esi
@@ -2137,9 +2137,9 @@ define void @PR34947(<9 x i16>* %p0, <9 x i32>* %p1) nounwind {
 ; X86-SSE-NEXT:    xorl %edx, %edx
 ; X86-SSE-NEXT:    divl %esi
 ; X86-SSE-NEXT:    movd %edx, %xmm4
-; X86-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
 ; X86-SSE-NEXT:    movd %xmm2, %eax
-; X86-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; X86-SSE-NEXT:    movd %xmm1, %esi
 ; X86-SSE-NEXT:    xorl %edx, %edx
 ; X86-SSE-NEXT:    divl %esi
@@ -2336,9 +2336,9 @@ define void @PR34947(<9 x i16>* %p0, <9 x i32>* %p1) nounwind {
 ; X64-SSE-NEXT:    xorl %edx, %edx
 ; X64-SSE-NEXT:    divl %ecx
 ; X64-SSE-NEXT:    movd %edx, %xmm3
-; X64-SSE-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[2,3,0,1]
+; X64-SSE-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[2,3,2,3]
 ; X64-SSE-NEXT:    movd %xmm7, %eax
-; X64-SSE-NEXT:    pshufd {{.*#+}} xmm7 = xmm5[2,3,0,1]
+; X64-SSE-NEXT:    pshufd {{.*#+}} xmm7 = xmm5[2,3,2,3]
 ; X64-SSE-NEXT:    movd %xmm7, %ecx
 ; X64-SSE-NEXT:    xorl %edx, %edx
 ; X64-SSE-NEXT:    divl %ecx
@@ -2379,9 +2379,9 @@ define void @PR34947(<9 x i16>* %p0, <9 x i32>* %p1) nounwind {
 ; X64-SSE-NEXT:    xorl %edx, %edx
 ; X64-SSE-NEXT:    divl %ecx
 ; X64-SSE-NEXT:    movd %edx, %xmm4
-; X64-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
+; X64-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
 ; X64-SSE-NEXT:    movd %xmm2, %eax
-; X64-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; X64-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; X64-SSE-NEXT:    movd %xmm1, %ecx
 ; X64-SSE-NEXT:    xorl %edx, %edx
 ; X64-SSE-NEXT:    divl %ecx
diff --git a/llvm/test/CodeGen/X86/slow-pmulld.ll b/llvm/test/CodeGen/X86/slow-pmulld.ll
index 0b79b62f84a11..53deafc9a4b43 100644
--- a/llvm/test/CodeGen/X86/slow-pmulld.ll
+++ b/llvm/test/CodeGen/X86/slow-pmulld.ll
@@ -224,7 +224,7 @@ define <16 x i32> @test_mul_v16i32_v16i8(<16 x i8> %A) {
 ; SLM32:       # %bb.0:
 ; SLM32-NEXT:    movdqa {{.*#+}} xmm2 = [18778,18778,18778,18778,18778,18778,18778,18778]
 ; SLM32-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; SLM32-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SLM32-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SLM32-NEXT:    pmovzxbw {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; SLM32-NEXT:    movdqa %xmm1, %xmm4
 ; SLM32-NEXT:    movdqa %xmm3, %xmm5
@@ -244,7 +244,7 @@ define <16 x i32> @test_mul_v16i32_v16i8(<16 x i8> %A) {
 ; SLM64:       # %bb.0:
 ; SLM64-NEXT:    movdqa {{.*#+}} xmm2 = [18778,18778,18778,18778,18778,18778,18778,18778]
 ; SLM64-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; SLM64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SLM64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SLM64-NEXT:    pmovzxbw {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; SLM64-NEXT:    movdqa %xmm1, %xmm4
 ; SLM64-NEXT:    movdqa %xmm3, %xmm5
@@ -270,7 +270,7 @@ define <16 x i32> @test_mul_v16i32_v16i8(<16 x i8> %A) {
 ; SLOW32-NEXT:    movdqa %xmm1, %xmm4
 ; SLOW32-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
 ; SLOW32-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
-; SLOW32-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SLOW32-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SLOW32-NEXT:    pmovzxbw {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; SLOW32-NEXT:    movdqa %xmm3, %xmm0
 ; SLOW32-NEXT:    pmulhw %xmm2, %xmm0
@@ -291,7 +291,7 @@ define <16 x i32> @test_mul_v16i32_v16i8(<16 x i8> %A) {
 ; SLOW64-NEXT:    movdqa %xmm1, %xmm4
 ; SLOW64-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
 ; SLOW64-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
-; SLOW64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SLOW64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SLOW64-NEXT:    pmovzxbw {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; SLOW64-NEXT:    movdqa %xmm3, %xmm0
 ; SLOW64-NEXT:    pmulhw %xmm2, %xmm0
@@ -306,7 +306,7 @@ define <16 x i32> @test_mul_v16i32_v16i8(<16 x i8> %A) {
 ; SSE4-32:       # %bb.0:
 ; SSE4-32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3]
 ; SSE4-32-NEXT:    pmovzxbd {{.*#+}} xmm3 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
-; SSE4-32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE4-32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE4-32-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; SSE4-32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE4-32-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
@@ -322,7 +322,7 @@ define <16 x i32> @test_mul_v16i32_v16i8(<16 x i8> %A) {
 ; SSE4-64:       # %bb.0:
 ; SSE4-64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3]
 ; SSE4-64-NEXT:    pmovzxbd {{.*#+}} xmm3 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
-; SSE4-64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE4-64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE4-64-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; SSE4-64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE4-64-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
@@ -336,7 +336,7 @@ define <16 x i32> @test_mul_v16i32_v16i8(<16 x i8> %A) {
 ;
 ; AVX2-32-LABEL: test_mul_v16i32_v16i8:
 ; AVX2-32:       # %bb.0:
-; AVX2-32-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-32-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-32-NEXT:    vpmovzxbd {{.*#+}} ymm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero
 ; AVX2-32-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
 ; AVX2-32-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [18778,18778,18778,18778,18778,18778,18778,18778]
@@ -346,7 +346,7 @@ define <16 x i32> @test_mul_v16i32_v16i8(<16 x i8> %A) {
 ;
 ; AVX2-64-LABEL: test_mul_v16i32_v16i8:
 ; AVX2-64:       # %bb.0:
-; AVX2-64-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-64-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-64-NEXT:    vpmovzxbd {{.*#+}} ymm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero
 ; AVX2-64-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
 ; AVX2-64-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [18778,18778,18778,18778,18778,18778,18778,18778]
@@ -494,7 +494,7 @@ define <8 x i32> @test_mul_v8i32_v8i16(<8 x i16> %A) {
 ;
 ; SSE4-32-LABEL: test_mul_v8i32_v8i16:
 ; SSE4-32:       # %bb.0:
-; SSE4-32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE4-32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE4-32-NEXT:    pmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
 ; SSE4-32-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; SSE4-32-NEXT:    movdqa {{.*#+}} xmm2 = [18778,18778,18778,18778]
@@ -504,7 +504,7 @@ define <8 x i32> @test_mul_v8i32_v8i16(<8 x i16> %A) {
 ;
 ; SSE4-64-LABEL: test_mul_v8i32_v8i16:
 ; SSE4-64:       # %bb.0:
-; SSE4-64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE4-64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE4-64-NEXT:    pmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
 ; SSE4-64-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; SSE4-64-NEXT:    movdqa {{.*#+}} xmm2 = [18778,18778,18778,18778]
@@ -609,9 +609,9 @@ define <16 x i32> @test_mul_v16i32_v16i16(<16 x i16> %A) {
 ;
 ; SSE4-32-LABEL: test_mul_v16i32_v16i16:
 ; SSE4-32:       # %bb.0:
-; SSE4-32-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; SSE4-32-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; SSE4-32-NEXT:    pmovzxwd {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
-; SSE4-32-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; SSE4-32-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; SSE4-32-NEXT:    pmovzxwd {{.*#+}} xmm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
 ; SSE4-32-NEXT:    pmovzxwd {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
 ; SSE4-32-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
@@ -625,9 +625,9 @@ define <16 x i32> @test_mul_v16i32_v16i16(<16 x i16> %A) {
 ;
 ; SSE4-64-LABEL: test_mul_v16i32_v16i16:
 ; SSE4-64:       # %bb.0:
-; SSE4-64-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; SSE4-64-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; SSE4-64-NEXT:    pmovzxwd {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
-; SSE4-64-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; SSE4-64-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; SSE4-64-NEXT:    pmovzxwd {{.*#+}} xmm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
 ; SSE4-64-NEXT:    pmovzxwd {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
 ; SSE4-64-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
@@ -880,7 +880,7 @@ define <16 x i32> @test_mul_v16i32_v16i8_minsize(<16 x i8> %A) minsize {
 ; SLM32-NEXT:    movdqa {{.*#+}} xmm5 = [18778,18778,18778,18778]
 ; SLM32-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,2,3]
 ; SLM32-NEXT:    pmovzxbd {{.*#+}} xmm3 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
-; SLM32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SLM32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SLM32-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SLM32-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; SLM32-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero
@@ -896,7 +896,7 @@ define <16 x i32> @test_mul_v16i32_v16i8_minsize(<16 x i8> %A) minsize {
 ; SLM64-NEXT:    movdqa {{.*#+}} xmm5 = [18778,18778,18778,18778]
 ; SLM64-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,2,3]
 ; SLM64-NEXT:    pmovzxbd {{.*#+}} xmm3 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
-; SLM64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SLM64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SLM64-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SLM64-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; SLM64-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero
@@ -910,7 +910,7 @@ define <16 x i32> @test_mul_v16i32_v16i8_minsize(<16 x i8> %A) minsize {
 ; SLOW32:       # %bb.0:
 ; SLOW32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3]
 ; SLOW32-NEXT:    pmovzxbd {{.*#+}} xmm3 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
-; SLOW32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SLOW32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SLOW32-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; SLOW32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SLOW32-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
@@ -926,7 +926,7 @@ define <16 x i32> @test_mul_v16i32_v16i8_minsize(<16 x i8> %A) minsize {
 ; SLOW64:       # %bb.0:
 ; SLOW64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3]
 ; SLOW64-NEXT:    pmovzxbd {{.*#+}} xmm3 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
-; SLOW64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SLOW64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SLOW64-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; SLOW64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SLOW64-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
@@ -942,7 +942,7 @@ define <16 x i32> @test_mul_v16i32_v16i8_minsize(<16 x i8> %A) minsize {
 ; SSE4-32:       # %bb.0:
 ; SSE4-32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3]
 ; SSE4-32-NEXT:    pmovzxbd {{.*#+}} xmm3 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
-; SSE4-32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE4-32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE4-32-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; SSE4-32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE4-32-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
@@ -958,7 +958,7 @@ define <16 x i32> @test_mul_v16i32_v16i8_minsize(<16 x i8> %A) minsize {
 ; SSE4-64:       # %bb.0:
 ; SSE4-64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3]
 ; SSE4-64-NEXT:    pmovzxbd {{.*#+}} xmm3 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
-; SSE4-64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE4-64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE4-64-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; SSE4-64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE4-64-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
@@ -972,7 +972,7 @@ define <16 x i32> @test_mul_v16i32_v16i8_minsize(<16 x i8> %A) minsize {
 ;
 ; AVX2-32-LABEL: test_mul_v16i32_v16i8_minsize:
 ; AVX2-32:       # %bb.0:
-; AVX2-32-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-32-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-32-NEXT:    vpmovzxbd {{.*#+}} ymm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero
 ; AVX2-32-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
 ; AVX2-32-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [18778,18778,18778,18778,18778,18778,18778,18778]
@@ -982,7 +982,7 @@ define <16 x i32> @test_mul_v16i32_v16i8_minsize(<16 x i8> %A) minsize {
 ;
 ; AVX2-64-LABEL: test_mul_v16i32_v16i8_minsize:
 ; AVX2-64:       # %bb.0:
-; AVX2-64-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-64-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-64-NEXT:    vpmovzxbd {{.*#+}} ymm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero
 ; AVX2-64-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
 ; AVX2-64-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [18778,18778,18778,18778,18778,18778,18778,18778]
@@ -1077,7 +1077,7 @@ define <8 x i32> @test_mul_v8i32_v8i16_minsize(<8 x i16> %A) minsize {
 ; SLM32-LABEL: test_mul_v8i32_v8i16_minsize:
 ; SLM32:       # %bb.0:
 ; SLM32-NEXT:    movdqa {{.*#+}} xmm2 = [18778,18778,18778,18778]
-; SLM32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SLM32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SLM32-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; SLM32-NEXT:    pmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
 ; SLM32-NEXT:    pmulld %xmm2, %xmm0
@@ -1087,7 +1087,7 @@ define <8 x i32> @test_mul_v8i32_v8i16_minsize(<8 x i16> %A) minsize {
 ; SLM64-LABEL: test_mul_v8i32_v8i16_minsize:
 ; SLM64:       # %bb.0:
 ; SLM64-NEXT:    movdqa {{.*#+}} xmm2 = [18778,18778,18778,18778]
-; SLM64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SLM64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SLM64-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; SLM64-NEXT:    pmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
 ; SLM64-NEXT:    pmulld %xmm2, %xmm0
@@ -1096,7 +1096,7 @@ define <8 x i32> @test_mul_v8i32_v8i16_minsize(<8 x i16> %A) minsize {
 ;
 ; SLOW32-LABEL: test_mul_v8i32_v8i16_minsize:
 ; SLOW32:       # %bb.0:
-; SLOW32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SLOW32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SLOW32-NEXT:    pmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
 ; SLOW32-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; SLOW32-NEXT:    movdqa {{.*#+}} xmm2 = [18778,18778,18778,18778]
@@ -1106,7 +1106,7 @@ define <8 x i32> @test_mul_v8i32_v8i16_minsize(<8 x i16> %A) minsize {
 ;
 ; SLOW64-LABEL: test_mul_v8i32_v8i16_minsize:
 ; SLOW64:       # %bb.0:
-; SLOW64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SLOW64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SLOW64-NEXT:    pmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
 ; SLOW64-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; SLOW64-NEXT:    movdqa {{.*#+}} xmm2 = [18778,18778,18778,18778]
@@ -1116,7 +1116,7 @@ define <8 x i32> @test_mul_v8i32_v8i16_minsize(<8 x i16> %A) minsize {
 ;
 ; SSE4-32-LABEL: test_mul_v8i32_v8i16_minsize:
 ; SSE4-32:       # %bb.0:
-; SSE4-32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE4-32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE4-32-NEXT:    pmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
 ; SSE4-32-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; SSE4-32-NEXT:    movdqa {{.*#+}} xmm2 = [18778,18778,18778,18778]
@@ -1126,7 +1126,7 @@ define <8 x i32> @test_mul_v8i32_v8i16_minsize(<8 x i16> %A) minsize {
 ;
 ; SSE4-64-LABEL: test_mul_v8i32_v8i16_minsize:
 ; SSE4-64:       # %bb.0:
-; SSE4-64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE4-64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE4-64-NEXT:    pmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
 ; SSE4-64-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; SSE4-64-NEXT:    movdqa {{.*#+}} xmm2 = [18778,18778,18778,18778]
@@ -1155,9 +1155,9 @@ define <8 x i32> @test_mul_v8i32_v8i16_minsize(<8 x i16> %A) minsize {
 define <16 x i32> @test_mul_v16i32_v16i16_minsize(<16 x i16> %A) minsize {
 ; SLM32-LABEL: test_mul_v16i32_v16i16_minsize:
 ; SLM32:       # %bb.0:
-; SLM32-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; SLM32-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; SLM32-NEXT:    pmovzxwd {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
-; SLM32-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; SLM32-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; SLM32-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; SLM32-NEXT:    pmovzxwd {{.*#+}} xmm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
 ; SLM32-NEXT:    pmovzxwd {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
@@ -1171,9 +1171,9 @@ define <16 x i32> @test_mul_v16i32_v16i16_minsize(<16 x i16> %A) minsize {
 ;
 ; SLM64-LABEL: test_mul_v16i32_v16i16_minsize:
 ; SLM64:       # %bb.0:
-; SLM64-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; SLM64-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; SLM64-NEXT:    pmovzxwd {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
-; SLM64-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; SLM64-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; SLM64-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; SLM64-NEXT:    pmovzxwd {{.*#+}} xmm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
 ; SLM64-NEXT:    pmovzxwd {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
@@ -1187,9 +1187,9 @@ define <16 x i32> @test_mul_v16i32_v16i16_minsize(<16 x i16> %A) minsize {
 ;
 ; SLOW32-LABEL: test_mul_v16i32_v16i16_minsize:
 ; SLOW32:       # %bb.0:
-; SLOW32-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; SLOW32-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; SLOW32-NEXT:    pmovzxwd {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
-; SLOW32-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; SLOW32-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; SLOW32-NEXT:    pmovzxwd {{.*#+}} xmm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
 ; SLOW32-NEXT:    pmovzxwd {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
 ; SLOW32-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
@@ -1203,9 +1203,9 @@ define <16 x i32> @test_mul_v16i32_v16i16_minsize(<16 x i16> %A) minsize {
 ;
 ; SLOW64-LABEL: test_mul_v16i32_v16i16_minsize:
 ; SLOW64:       # %bb.0:
-; SLOW64-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; SLOW64-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; SLOW64-NEXT:    pmovzxwd {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
-; SLOW64-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; SLOW64-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; SLOW64-NEXT:    pmovzxwd {{.*#+}} xmm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
 ; SLOW64-NEXT:    pmovzxwd {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
 ; SLOW64-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
@@ -1219,9 +1219,9 @@ define <16 x i32> @test_mul_v16i32_v16i16_minsize(<16 x i16> %A) minsize {
 ;
 ; SSE4-32-LABEL: test_mul_v16i32_v16i16_minsize:
 ; SSE4-32:       # %bb.0:
-; SSE4-32-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; SSE4-32-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; SSE4-32-NEXT:    pmovzxwd {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
-; SSE4-32-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; SSE4-32-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; SSE4-32-NEXT:    pmovzxwd {{.*#+}} xmm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
 ; SSE4-32-NEXT:    pmovzxwd {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
 ; SSE4-32-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
@@ -1235,9 +1235,9 @@ define <16 x i32> @test_mul_v16i32_v16i16_minsize(<16 x i16> %A) minsize {
 ;
 ; SSE4-64-LABEL: test_mul_v16i32_v16i16_minsize:
 ; SSE4-64:       # %bb.0:
-; SSE4-64-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; SSE4-64-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; SSE4-64-NEXT:    pmovzxwd {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
-; SSE4-64-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; SSE4-64-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; SSE4-64-NEXT:    pmovzxwd {{.*#+}} xmm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
 ; SSE4-64-NEXT:    pmovzxwd {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
 ; SSE4-64-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
diff --git a/llvm/test/CodeGen/X86/smul_fix_sat.ll b/llvm/test/CodeGen/X86/smul_fix_sat.ll
index 284f51d7422e6..e0c1b762c1507 100644
--- a/llvm/test/CodeGen/X86/smul_fix_sat.ll
+++ b/llvm/test/CodeGen/X86/smul_fix_sat.ll
@@ -206,10 +206,10 @@ define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; X64-NEXT:    movl $-2147483648, %ecx # imm = 0x80000000
 ; X64-NEXT:    cmovll %ecx, %edx
 ; X64-NEXT:    movd %edx, %xmm2
-; X64-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,0,1]
+; X64-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
 ; X64-NEXT:    movd %xmm3, %edx
 ; X64-NEXT:    movslq %edx, %rdx
-; X64-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,0,1]
+; X64-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
 ; X64-NEXT:    movd %xmm3, %esi
 ; X64-NEXT:    movslq %esi, %rsi
 ; X64-NEXT:    imulq %rdx, %rsi
@@ -476,9 +476,9 @@ define <4 x i32> @vec2(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; X64-NEXT:    addl $2147483647, %edi # imm = 0x7FFFFFFF
 ; X64-NEXT:    imull %edx, %ecx
 ; X64-NEXT:    cmovol %edi, %ecx
-; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; X64-NEXT:    movd %xmm2, %edx
-; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; X64-NEXT:    movd %xmm2, %esi
 ; X64-NEXT:    movl %esi, %edi
 ; X64-NEXT:    imull %edx, %edi
diff --git a/llvm/test/CodeGen/X86/split-extend-vector-inreg.ll b/llvm/test/CodeGen/X86/split-extend-vector-inreg.ll
index 7f0e2cfa44b02..0abac92099055 100644
--- a/llvm/test/CodeGen/X86/split-extend-vector-inreg.ll
+++ b/llvm/test/CodeGen/X86/split-extend-vector-inreg.ll
@@ -5,7 +5,7 @@
 define <4 x i64> @autogen_SD88863() {
 ; CHECK-LABEL: autogen_SD88863:
 ; CHECK:       # %bb.0: # %BB
-; CHECK-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; CHECK-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; CHECK-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
 ; CHECK-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5],ymm1[6,7]
diff --git a/llvm/test/CodeGen/X86/split-vector-rem.ll b/llvm/test/CodeGen/X86/split-vector-rem.ll
index ef03075ac65d6..959c9bd0ff769 100644
--- a/llvm/test/CodeGen/X86/split-vector-rem.ll
+++ b/llvm/test/CodeGen/X86/split-vector-rem.ll
@@ -12,9 +12,9 @@ define <8 x i32> @foo(<8 x i32> %t, <8 x i32> %u) {
 ; CHECK-NEXT:    cltd
 ; CHECK-NEXT:    idivl %ecx
 ; CHECK-NEXT:    movd %edx, %xmm0
-; CHECK-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[2,3,0,1]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[2,3,2,3]
 ; CHECK-NEXT:    movd %xmm5, %eax
-; CHECK-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[2,3,0,1]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[2,3,2,3]
 ; CHECK-NEXT:    movd %xmm5, %ecx
 ; CHECK-NEXT:    cltd
 ; CHECK-NEXT:    idivl %ecx
@@ -41,9 +41,9 @@ define <8 x i32> @foo(<8 x i32> %t, <8 x i32> %u) {
 ; CHECK-NEXT:    cltd
 ; CHECK-NEXT:    idivl %ecx
 ; CHECK-NEXT:    movd %edx, %xmm2
-; CHECK-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[2,3,0,1]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[2,3,2,3]
 ; CHECK-NEXT:    movd %xmm4, %eax
-; CHECK-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[2,3,0,1]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[2,3,2,3]
 ; CHECK-NEXT:    movd %xmm4, %ecx
 ; CHECK-NEXT:    cltd
 ; CHECK-NEXT:    idivl %ecx
@@ -79,9 +79,9 @@ define <8 x i32> @bar(<8 x i32> %t, <8 x i32> %u) {
 ; CHECK-NEXT:    xorl %edx, %edx
 ; CHECK-NEXT:    divl %ecx
 ; CHECK-NEXT:    movd %edx, %xmm0
-; CHECK-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[2,3,0,1]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[2,3,2,3]
 ; CHECK-NEXT:    movd %xmm5, %eax
-; CHECK-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[2,3,0,1]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[2,3,2,3]
 ; CHECK-NEXT:    movd %xmm5, %ecx
 ; CHECK-NEXT:    xorl %edx, %edx
 ; CHECK-NEXT:    divl %ecx
@@ -108,9 +108,9 @@ define <8 x i32> @bar(<8 x i32> %t, <8 x i32> %u) {
 ; CHECK-NEXT:    xorl %edx, %edx
 ; CHECK-NEXT:    divl %ecx
 ; CHECK-NEXT:    movd %edx, %xmm2
-; CHECK-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[2,3,0,1]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[2,3,2,3]
 ; CHECK-NEXT:    movd %xmm4, %eax
-; CHECK-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[2,3,0,1]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[2,3,2,3]
 ; CHECK-NEXT:    movd %xmm4, %ecx
 ; CHECK-NEXT:    xorl %edx, %edx
 ; CHECK-NEXT:    divl %ecx
diff --git a/llvm/test/CodeGen/X86/sse-intrinsics-fast-isel.ll b/llvm/test/CodeGen/X86/sse-intrinsics-fast-isel.ll
index 85400656e2e54..206814cfcf1c7 100644
--- a/llvm/test/CodeGen/X86/sse-intrinsics-fast-isel.ll
+++ b/llvm/test/CodeGen/X86/sse-intrinsics-fast-isel.ll
@@ -2791,8 +2791,8 @@ define void @test_mm_storeh_pi(x86_mmx *%a0, <4 x float> %a1) nounwind {
 ;
 ; X64-SSE2-LABEL: test_mm_storeh_pi:
 ; X64-SSE2:       # %bb.0:
-; X64-SSE2-NEXT:    pshufd $78, %xmm0, %xmm0 # encoding: [0x66,0x0f,0x70,0xc0,0x4e]
-; X64-SSE2-NEXT:    # xmm0 = xmm0[2,3,0,1]
+; X64-SSE2-NEXT:    punpckhqdq %xmm0, %xmm0 # encoding: [0x66,0x0f,0x6d,0xc0]
+; X64-SSE2-NEXT:    # xmm0 = xmm0[1,1]
 ; X64-SSE2-NEXT:    movq %xmm0, %rax # encoding: [0x66,0x48,0x0f,0x7e,0xc0]
 ; X64-SSE2-NEXT:    movq %rax, (%rdi) # encoding: [0x48,0x89,0x07]
 ; X64-SSE2-NEXT:    retq # encoding: [0xc3]
diff --git a/llvm/test/CodeGen/X86/sse41.ll b/llvm/test/CodeGen/X86/sse41.ll
index 460987cd74dfe..e2973ebbab89e 100644
--- a/llvm/test/CodeGen/X86/sse41.ll
+++ b/llvm/test/CodeGen/X86/sse41.ll
@@ -692,24 +692,24 @@ entry:
 define <4 x i32> @insertps_from_shufflevector_i32_2(<4 x i32> %a, <4 x i32> %b) {
 ; SSE-LABEL: insertps_from_shufflevector_i32_2:
 ; SSE:       ## %bb.0: ## %entry
-; SSE-NEXT:    pshufd $78, %xmm1, %xmm1 ## encoding: [0x66,0x0f,0x70,0xc9,0x4e]
-; SSE-NEXT:    ## xmm1 = xmm1[2,3,0,1]
+; SSE-NEXT:    pshufd $238, %xmm1, %xmm1 ## encoding: [0x66,0x0f,0x70,0xc9,0xee]
+; SSE-NEXT:    ## xmm1 = xmm1[2,3,2,3]
 ; SSE-NEXT:    pblendw $12, %xmm1, %xmm0 ## encoding: [0x66,0x0f,0x3a,0x0e,0xc1,0x0c]
 ; SSE-NEXT:    ## xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5,6,7]
 ; SSE-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
 ;
 ; AVX1-LABEL: insertps_from_shufflevector_i32_2:
 ; AVX1:       ## %bb.0: ## %entry
-; AVX1-NEXT:    vpermilps $78, %xmm1, %xmm1 ## encoding: [0xc4,0xe3,0x79,0x04,0xc9,0x4e]
-; AVX1-NEXT:    ## xmm1 = xmm1[2,3,0,1]
+; AVX1-NEXT:    vpermilps $238, %xmm1, %xmm1 ## encoding: [0xc4,0xe3,0x79,0x04,0xc9,0xee]
+; AVX1-NEXT:    ## xmm1 = xmm1[2,3,2,3]
 ; AVX1-NEXT:    vblendps $2, %xmm1, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x0c,0xc1,0x02]
 ; AVX1-NEXT:    ## xmm0 = xmm0[0],xmm1[1],xmm0[2,3]
 ; AVX1-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
 ;
 ; AVX512-LABEL: insertps_from_shufflevector_i32_2:
 ; AVX512:       ## %bb.0: ## %entry
-; AVX512-NEXT:    vpermilps $78, %xmm1, %xmm1 ## EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x04,0xc9,0x4e]
-; AVX512-NEXT:    ## xmm1 = xmm1[2,3,0,1]
+; AVX512-NEXT:    vpermilps $238, %xmm1, %xmm1 ## EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x04,0xc9,0xee]
+; AVX512-NEXT:    ## xmm1 = xmm1[2,3,2,3]
 ; AVX512-NEXT:    vblendps $2, %xmm1, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x0c,0xc1,0x02]
 ; AVX512-NEXT:    ## xmm0 = xmm0[0],xmm1[1],xmm0[2,3]
 ; AVX512-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
@@ -1875,8 +1875,8 @@ define void @insertps_pr20411(<4 x i32> %shuffle109, <4 x i32> %shuffle116, i32*
 ; X86-SSE-LABEL: insertps_pr20411:
 ; X86-SSE:       ## %bb.0:
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x04]
-; X86-SSE-NEXT:    pshufd $78, %xmm1, %xmm1 ## encoding: [0x66,0x0f,0x70,0xc9,0x4e]
-; X86-SSE-NEXT:    ## xmm1 = xmm1[2,3,0,1]
+; X86-SSE-NEXT:    pshufd $238, %xmm1, %xmm1 ## encoding: [0x66,0x0f,0x70,0xc9,0xee]
+; X86-SSE-NEXT:    ## xmm1 = xmm1[2,3,2,3]
 ; X86-SSE-NEXT:    pblendw $243, %xmm0, %xmm1 ## encoding: [0x66,0x0f,0x3a,0x0e,0xc8,0xf3]
 ; X86-SSE-NEXT:    ## xmm1 = xmm0[0,1],xmm1[2,3],xmm0[4,5,6,7]
 ; X86-SSE-NEXT:    movdqu %xmm1, (%eax) ## encoding: [0xf3,0x0f,0x7f,0x08]
@@ -1885,8 +1885,8 @@ define void @insertps_pr20411(<4 x i32> %shuffle109, <4 x i32> %shuffle116, i32*
 ; X86-AVX1-LABEL: insertps_pr20411:
 ; X86-AVX1:       ## %bb.0:
 ; X86-AVX1-NEXT:    movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x04]
-; X86-AVX1-NEXT:    vpermilps $78, %xmm1, %xmm1 ## encoding: [0xc4,0xe3,0x79,0x04,0xc9,0x4e]
-; X86-AVX1-NEXT:    ## xmm1 = xmm1[2,3,0,1]
+; X86-AVX1-NEXT:    vpermilps $238, %xmm1, %xmm1 ## encoding: [0xc4,0xe3,0x79,0x04,0xc9,0xee]
+; X86-AVX1-NEXT:    ## xmm1 = xmm1[2,3,2,3]
 ; X86-AVX1-NEXT:    vblendps $2, %xmm1, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x0c,0xc1,0x02]
 ; X86-AVX1-NEXT:    ## xmm0 = xmm0[0],xmm1[1],xmm0[2,3]
 ; X86-AVX1-NEXT:    vmovups %xmm0, (%eax) ## encoding: [0xc5,0xf8,0x11,0x00]
@@ -1895,8 +1895,8 @@ define void @insertps_pr20411(<4 x i32> %shuffle109, <4 x i32> %shuffle116, i32*
 ; X86-AVX512-LABEL: insertps_pr20411:
 ; X86-AVX512:       ## %bb.0:
 ; X86-AVX512-NEXT:    movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x04]
-; X86-AVX512-NEXT:    vpermilps $78, %xmm1, %xmm1 ## EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x04,0xc9,0x4e]
-; X86-AVX512-NEXT:    ## xmm1 = xmm1[2,3,0,1]
+; X86-AVX512-NEXT:    vpermilps $238, %xmm1, %xmm1 ## EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x04,0xc9,0xee]
+; X86-AVX512-NEXT:    ## xmm1 = xmm1[2,3,2,3]
 ; X86-AVX512-NEXT:    vblendps $2, %xmm1, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x0c,0xc1,0x02]
 ; X86-AVX512-NEXT:    ## xmm0 = xmm0[0],xmm1[1],xmm0[2,3]
 ; X86-AVX512-NEXT:    vmovups %xmm0, (%eax) ## EVEX TO VEX Compression encoding: [0xc5,0xf8,0x11,0x00]
@@ -1904,8 +1904,8 @@ define void @insertps_pr20411(<4 x i32> %shuffle109, <4 x i32> %shuffle116, i32*
 ;
 ; X64-SSE-LABEL: insertps_pr20411:
 ; X64-SSE:       ## %bb.0:
-; X64-SSE-NEXT:    pshufd $78, %xmm1, %xmm1 ## encoding: [0x66,0x0f,0x70,0xc9,0x4e]
-; X64-SSE-NEXT:    ## xmm1 = xmm1[2,3,0,1]
+; X64-SSE-NEXT:    pshufd $238, %xmm1, %xmm1 ## encoding: [0x66,0x0f,0x70,0xc9,0xee]
+; X64-SSE-NEXT:    ## xmm1 = xmm1[2,3,2,3]
 ; X64-SSE-NEXT:    pblendw $243, %xmm0, %xmm1 ## encoding: [0x66,0x0f,0x3a,0x0e,0xc8,0xf3]
 ; X64-SSE-NEXT:    ## xmm1 = xmm0[0,1],xmm1[2,3],xmm0[4,5,6,7]
 ; X64-SSE-NEXT:    movdqu %xmm1, (%rdi) ## encoding: [0xf3,0x0f,0x7f,0x0f]
@@ -1913,8 +1913,8 @@ define void @insertps_pr20411(<4 x i32> %shuffle109, <4 x i32> %shuffle116, i32*
 ;
 ; X64-AVX1-LABEL: insertps_pr20411:
 ; X64-AVX1:       ## %bb.0:
-; X64-AVX1-NEXT:    vpermilps $78, %xmm1, %xmm1 ## encoding: [0xc4,0xe3,0x79,0x04,0xc9,0x4e]
-; X64-AVX1-NEXT:    ## xmm1 = xmm1[2,3,0,1]
+; X64-AVX1-NEXT:    vpermilps $238, %xmm1, %xmm1 ## encoding: [0xc4,0xe3,0x79,0x04,0xc9,0xee]
+; X64-AVX1-NEXT:    ## xmm1 = xmm1[2,3,2,3]
 ; X64-AVX1-NEXT:    vblendps $2, %xmm1, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x0c,0xc1,0x02]
 ; X64-AVX1-NEXT:    ## xmm0 = xmm0[0],xmm1[1],xmm0[2,3]
 ; X64-AVX1-NEXT:    vmovups %xmm0, (%rdi) ## encoding: [0xc5,0xf8,0x11,0x07]
@@ -1922,8 +1922,8 @@ define void @insertps_pr20411(<4 x i32> %shuffle109, <4 x i32> %shuffle116, i32*
 ;
 ; X64-AVX512-LABEL: insertps_pr20411:
 ; X64-AVX512:       ## %bb.0:
-; X64-AVX512-NEXT:    vpermilps $78, %xmm1, %xmm1 ## EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x04,0xc9,0x4e]
-; X64-AVX512-NEXT:    ## xmm1 = xmm1[2,3,0,1]
+; X64-AVX512-NEXT:    vpermilps $238, %xmm1, %xmm1 ## EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x04,0xc9,0xee]
+; X64-AVX512-NEXT:    ## xmm1 = xmm1[2,3,2,3]
 ; X64-AVX512-NEXT:    vblendps $2, %xmm1, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x0c,0xc1,0x02]
 ; X64-AVX512-NEXT:    ## xmm0 = xmm0[0],xmm1[1],xmm0[2,3]
 ; X64-AVX512-NEXT:    vmovups %xmm0, (%rdi) ## EVEX TO VEX Compression encoding: [0xc5,0xf8,0x11,0x07]
diff --git a/llvm/test/CodeGen/X86/trunc-subvector.ll b/llvm/test/CodeGen/X86/trunc-subvector.ll
index 05b2b6608addd..4c7acf60d3080 100644
--- a/llvm/test/CodeGen/X86/trunc-subvector.ll
+++ b/llvm/test/CodeGen/X86/trunc-subvector.ll
@@ -73,7 +73,7 @@ define <2 x i32> @test4(<8 x i32> %v) {
 define <2 x i32> @test5(<8 x i32> %v) {
 ; SSE2-LABEL: test5:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,3,2,2]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; SSE2-NEXT:    retq
 ;
@@ -175,7 +175,7 @@ define <2 x i32> @test9(<8 x i32> %v) {
 define <2 x i32> @test10(<8 x i32> %v) {
 ; SSE2-LABEL: test10:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,3,2,2]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; SSE2-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/udiv_fix.ll b/llvm/test/CodeGen/X86/udiv_fix.ll
index 7540b394babdf..e42e527553f14 100644
--- a/llvm/test/CodeGen/X86/udiv_fix.ll
+++ b/llvm/test/CodeGen/X86/udiv_fix.ll
@@ -248,9 +248,9 @@ define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; X64-NEXT:    xorl %edx, %edx
 ; X64-NEXT:    divq %rcx
 ; X64-NEXT:    movq %rax, %xmm3
-; X64-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[2,3,0,1]
+; X64-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[2,3,2,3]
 ; X64-NEXT:    movq %xmm4, %rcx
-; X64-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[2,3,0,1]
+; X64-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[2,3,2,3]
 ; X64-NEXT:    movq %xmm4, %rax
 ; X64-NEXT:    xorl %edx, %edx
 ; X64-NEXT:    divq %rcx
@@ -264,9 +264,9 @@ define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; X64-NEXT:    xorl %edx, %edx
 ; X64-NEXT:    divq %rcx
 ; X64-NEXT:    movq %rax, %xmm2
-; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; X64-NEXT:    movq %xmm1, %rcx
-; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; X64-NEXT:    movq %xmm0, %rax
 ; X64-NEXT:    xorl %edx, %edx
 ; X64-NEXT:    divq %rcx
diff --git a/llvm/test/CodeGen/X86/udiv_fix_sat.ll b/llvm/test/CodeGen/X86/udiv_fix_sat.ll
index 493515c418caa..f6c8baf2e9238 100644
--- a/llvm/test/CodeGen/X86/udiv_fix_sat.ll
+++ b/llvm/test/CodeGen/X86/udiv_fix_sat.ll
@@ -335,9 +335,9 @@ define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; X64-NEXT:    xorl %edx, %edx
 ; X64-NEXT:    divq %rcx
 ; X64-NEXT:    movq %rax, %xmm7
-; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
+; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
 ; X64-NEXT:    movq %xmm2, %rcx
-; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[2,3,0,1]
+; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[2,3,2,3]
 ; X64-NEXT:    movq %xmm2, %rax
 ; X64-NEXT:    xorl %edx, %edx
 ; X64-NEXT:    divq %rcx
@@ -369,9 +369,9 @@ define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; X64-NEXT:    xorl %edx, %edx
 ; X64-NEXT:    divq %rcx
 ; X64-NEXT:    movq %rax, %xmm3
-; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; X64-NEXT:    movq %xmm1, %rcx
-; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; X64-NEXT:    movq %xmm0, %rax
 ; X64-NEXT:    xorl %edx, %edx
 ; X64-NEXT:    divq %rcx
diff --git a/llvm/test/CodeGen/X86/uint_to_fp-3.ll b/llvm/test/CodeGen/X86/uint_to_fp-3.ll
index 9efd9a5bef5f2..ca46b48b7731f 100644
--- a/llvm/test/CodeGen/X86/uint_to_fp-3.ll
+++ b/llvm/test/CodeGen/X86/uint_to_fp-3.ll
@@ -40,7 +40,7 @@ define <4 x double> @mask_ucvt_4i32_4f64(<4 x i32> %a) {
 ; X32-SSE:       # %bb.0:
 ; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
 ; X32-SSE-NEXT:    cvtdq2pd %xmm0, %xmm2
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; X32-SSE-NEXT:    cvtdq2pd %xmm0, %xmm1
 ; X32-SSE-NEXT:    movaps %xmm2, %xmm0
 ; X32-SSE-NEXT:    retl
@@ -55,7 +55,7 @@ define <4 x double> @mask_ucvt_4i32_4f64(<4 x i32> %a) {
 ; X64-SSE:       # %bb.0:
 ; X64-SSE-NEXT:    pand {{.*}}(%rip), %xmm0
 ; X64-SSE-NEXT:    cvtdq2pd %xmm0, %xmm2
-; X64-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; X64-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; X64-SSE-NEXT:    cvtdq2pd %xmm0, %xmm1
 ; X64-SSE-NEXT:    movaps %xmm2, %xmm0
 ; X64-SSE-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/umul_fix_sat.ll b/llvm/test/CodeGen/X86/umul_fix_sat.ll
index 18b769f32ec94..ce744f93cdfe8 100644
--- a/llvm/test/CodeGen/X86/umul_fix_sat.ll
+++ b/llvm/test/CodeGen/X86/umul_fix_sat.ll
@@ -151,9 +151,9 @@ define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; X64-NEXT:    movl $-1, %eax
 ; X64-NEXT:    cmoval %eax, %ecx
 ; X64-NEXT:    movd %ecx, %xmm2
-; X64-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,0,1]
+; X64-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
 ; X64-NEXT:    movd %xmm3, %ecx
-; X64-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,0,1]
+; X64-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
 ; X64-NEXT:    movd %xmm3, %edx
 ; X64-NEXT:    imulq %rcx, %rdx
 ; X64-NEXT:    movq %rdx, %rcx
@@ -361,9 +361,9 @@ define <4 x i32> @vec2(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; X64-NEXT:    movl $-1, %ecx
 ; X64-NEXT:    cmovol %ecx, %eax
 ; X64-NEXT:    movd %eax, %xmm2
-; X64-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,0,1]
+; X64-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
 ; X64-NEXT:    movd %xmm3, %eax
-; X64-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,0,1]
+; X64-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
 ; X64-NEXT:    movd %xmm3, %edx
 ; X64-NEXT:    mull %edx
 ; X64-NEXT:    cmovol %ecx, %eax
diff --git a/llvm/test/CodeGen/X86/urem-seteq-vec-nonsplat.ll b/llvm/test/CodeGen/X86/urem-seteq-vec-nonsplat.ll
index 9fdb08ba6d4e8..923aaa34f04d0 100644
--- a/llvm/test/CodeGen/X86/urem-seteq-vec-nonsplat.ll
+++ b/llvm/test/CodeGen/X86/urem-seteq-vec-nonsplat.ll
@@ -2411,7 +2411,7 @@ define <4 x i32> @test_urem_odd_allones_and_poweroftwo_and_one(<4 x i32> %X) nou
 ; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
 ; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
 ; CHECK-SSE2-NEXT:    psrld $2, %xmm2
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,3],xmm3[2,1]
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,3],xmm3[2,3]
 ; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [5,4294967295,16,1]
 ; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm2
 ; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
@@ -2516,7 +2516,7 @@ define <4 x i32> @test_urem_even_allones_and_poweroftwo_and_one(<4 x i32> %X) no
 ; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
 ; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
 ; CHECK-SSE2-NEXT:    psrld $2, %xmm2
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,3],xmm3[2,1]
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,3],xmm3[2,3]
 ; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [14,4294967295,16,1]
 ; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm2
 ; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
diff --git a/llvm/test/CodeGen/X86/var-permute-128.ll b/llvm/test/CodeGen/X86/var-permute-128.ll
index 8bc971e79f507..0a057852613ae 100644
--- a/llvm/test/CodeGen/X86/var-permute-128.ll
+++ b/llvm/test/CodeGen/X86/var-permute-128.ll
@@ -16,7 +16,7 @@ define <2 x i64> @var_shuffle_v2i64(<2 x i64> %v, <2 x i64> %indices) nounwind {
 ; SSE3:       # %bb.0:
 ; SSE3-NEXT:    movq %xmm1, %rax
 ; SSE3-NEXT:    andl $1, %eax
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE3-NEXT:    movq %xmm1, %rcx
 ; SSE3-NEXT:    andl $1, %ecx
 ; SSE3-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
@@ -29,7 +29,7 @@ define <2 x i64> @var_shuffle_v2i64(<2 x i64> %v, <2 x i64> %indices) nounwind {
 ; SSSE3:       # %bb.0:
 ; SSSE3-NEXT:    movq %xmm1, %rax
 ; SSSE3-NEXT:    andl $1, %eax
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSSE3-NEXT:    movq %xmm1, %rcx
 ; SSSE3-NEXT:    andl $1, %ecx
 ; SSSE3-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
@@ -69,7 +69,7 @@ define <4 x i32> @var_shuffle_v4i32(<4 x i32> %v, <4 x i32> %indices) nounwind {
 ; SSE3-NEXT:    movd %xmm1, %eax
 ; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,2,3]
 ; SSE3-NEXT:    movd %xmm2, %ecx
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; SSE3-NEXT:    movd %xmm2, %edx
 ; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]
 ; SSE3-NEXT:    movd %xmm1, %esi
@@ -379,7 +379,7 @@ define <2 x double> @var_shuffle_v2f64(<2 x double> %v, <2 x i64> %indices) noun
 ; SSE3:       # %bb.0:
 ; SSE3-NEXT:    movq %xmm1, %rax
 ; SSE3-NEXT:    andl $1, %eax
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE3-NEXT:    movq %xmm1, %rcx
 ; SSE3-NEXT:    andl $1, %ecx
 ; SSE3-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
@@ -391,7 +391,7 @@ define <2 x double> @var_shuffle_v2f64(<2 x double> %v, <2 x i64> %indices) noun
 ; SSSE3:       # %bb.0:
 ; SSSE3-NEXT:    movq %xmm1, %rax
 ; SSSE3-NEXT:    andl $1, %eax
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSSE3-NEXT:    movq %xmm1, %rcx
 ; SSSE3-NEXT:    andl $1, %ecx
 ; SSSE3-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
@@ -430,7 +430,7 @@ define <4 x float> @var_shuffle_v4f32(<4 x float> %v, <4 x i32> %indices) nounwi
 ; SSE3-NEXT:    movd %xmm1, %eax
 ; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,2,3]
 ; SSE3-NEXT:    movd %xmm2, %ecx
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; SSE3-NEXT:    movd %xmm2, %edx
 ; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]
 ; SSE3-NEXT:    movd %xmm1, %esi
diff --git a/llvm/test/CodeGen/X86/vec-strict-inttofp-128.ll b/llvm/test/CodeGen/X86/vec-strict-inttofp-128.ll
index 04692995b7e95..94f7d7eeaf39d 100644
--- a/llvm/test/CodeGen/X86/vec-strict-inttofp-128.ll
+++ b/llvm/test/CodeGen/X86/vec-strict-inttofp-128.ll
@@ -135,7 +135,7 @@ define <2 x float> @sitofp_v2i64_v2f32(<2 x i64> %x) #0 {
 ; SSE-32-NEXT:    andl $-8, %esp
 ; SSE-32-NEXT:    subl $24, %esp
 ; SSE-32-NEXT:    movq %xmm0, {{[0-9]+}}(%esp)
-; SSE-32-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE-32-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE-32-NEXT:    movq %xmm0, {{[0-9]+}}(%esp)
 ; SSE-32-NEXT:    fildll {{[0-9]+}}(%esp)
 ; SSE-32-NEXT:    fstps (%esp)
@@ -154,7 +154,7 @@ define <2 x float> @sitofp_v2i64_v2f32(<2 x i64> %x) #0 {
 ; SSE-64:       # %bb.0:
 ; SSE-64-NEXT:    movq %xmm0, %rax
 ; SSE-64-NEXT:    cvtsi2ss %rax, %xmm1
-; SSE-64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE-64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE-64-NEXT:    movq %xmm0, %rax
 ; SSE-64-NEXT:    xorps %xmm0, %xmm0
 ; SSE-64-NEXT:    cvtsi2ss %rax, %xmm0
@@ -172,7 +172,7 @@ define <2 x float> @sitofp_v2i64_v2f32(<2 x i64> %x) #0 {
 ; SSE41-32-NEXT:    andl $-8, %esp
 ; SSE41-32-NEXT:    subl $24, %esp
 ; SSE41-32-NEXT:    movq %xmm0, {{[0-9]+}}(%esp)
-; SSE41-32-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE41-32-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE41-32-NEXT:    movq %xmm0, {{[0-9]+}}(%esp)
 ; SSE41-32-NEXT:    fildll {{[0-9]+}}(%esp)
 ; SSE41-32-NEXT:    fstps (%esp)
@@ -191,7 +191,7 @@ define <2 x float> @sitofp_v2i64_v2f32(<2 x i64> %x) #0 {
 ; SSE41-64:       # %bb.0:
 ; SSE41-64-NEXT:    movq %xmm0, %rax
 ; SSE41-64-NEXT:    cvtsi2ss %rax, %xmm1
-; SSE41-64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE41-64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE41-64-NEXT:    movq %xmm0, %rax
 ; SSE41-64-NEXT:    xorps %xmm0, %xmm0
 ; SSE41-64-NEXT:    cvtsi2ss %rax, %xmm0
@@ -209,7 +209,7 @@ define <2 x float> @sitofp_v2i64_v2f32(<2 x i64> %x) #0 {
 ; AVX-32-NEXT:    andl $-8, %esp
 ; AVX-32-NEXT:    subl $24, %esp
 ; AVX-32-NEXT:    vmovlps %xmm0, {{[0-9]+}}(%esp)
-; AVX-32-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX-32-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; AVX-32-NEXT:    vmovlps %xmm0, {{[0-9]+}}(%esp)
 ; AVX-32-NEXT:    fildll {{[0-9]+}}(%esp)
 ; AVX-32-NEXT:    fstps (%esp)
@@ -236,7 +236,7 @@ define <2 x float> @sitofp_v2i64_v2f32(<2 x i64> %x) #0 {
 ; AVX512DQ-32:       # %bb.0:
 ; AVX512DQ-32-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; AVX512DQ-32-NEXT:    vcvtqq2ps %zmm0, %ymm1
-; AVX512DQ-32-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX512DQ-32-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; AVX512DQ-32-NEXT:    vcvtqq2ps %zmm0, %ymm0
 ; AVX512DQ-32-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],zero,zero
 ; AVX512DQ-32-NEXT:    vzeroupper
@@ -271,7 +271,7 @@ define <2 x float> @uitofp_v2i64_v2f32(<2 x i64> %x) #0 {
 ; SSE-32-NEXT:    .cfi_def_cfa_register %ebp
 ; SSE-32-NEXT:    andl $-8, %esp
 ; SSE-32-NEXT:    subl $24, %esp
-; SSE-32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE-32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-32-NEXT:    movq %xmm1, {{[0-9]+}}(%esp)
 ; SSE-32-NEXT:    movq %xmm0, {{[0-9]+}}(%esp)
 ; SSE-32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3]
@@ -313,7 +313,7 @@ define <2 x float> @uitofp_v2i64_v2f32(<2 x i64> %x) #0 {
 ; SSE-64-NEXT:  # %bb.1:
 ; SSE-64-NEXT:    addss %xmm0, %xmm0
 ; SSE-64-NEXT:  .LBB3_2:
-; SSE-64-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE-64-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE-64-NEXT:    movq %xmm1, %rax
 ; SSE-64-NEXT:    movq %rax, %rcx
 ; SSE-64-NEXT:    shrq %rcx
@@ -340,7 +340,7 @@ define <2 x float> @uitofp_v2i64_v2f32(<2 x i64> %x) #0 {
 ; SSE41-32-NEXT:    .cfi_def_cfa_register %ebp
 ; SSE41-32-NEXT:    andl $-8, %esp
 ; SSE41-32-NEXT:    subl $24, %esp
-; SSE41-32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE41-32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE41-32-NEXT:    movq %xmm1, {{[0-9]+}}(%esp)
 ; SSE41-32-NEXT:    movq %xmm0, {{[0-9]+}}(%esp)
 ; SSE41-32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3]
@@ -382,7 +382,7 @@ define <2 x float> @uitofp_v2i64_v2f32(<2 x i64> %x) #0 {
 ; SSE41-64-NEXT:  # %bb.1:
 ; SSE41-64-NEXT:    addss %xmm0, %xmm0
 ; SSE41-64-NEXT:  .LBB3_2:
-; SSE41-64-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE41-64-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE41-64-NEXT:    movq %xmm1, %rax
 ; SSE41-64-NEXT:    movq %rax, %rcx
 ; SSE41-64-NEXT:    shrq %rcx
@@ -410,7 +410,7 @@ define <2 x float> @uitofp_v2i64_v2f32(<2 x i64> %x) #0 {
 ; AVX-32-NEXT:    andl $-8, %esp
 ; AVX-32-NEXT:    subl $24, %esp
 ; AVX-32-NEXT:    vmovlps %xmm0, {{[0-9]+}}(%esp)
-; AVX-32-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-32-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-32-NEXT:    vmovlps %xmm1, {{[0-9]+}}(%esp)
 ; AVX-32-NEXT:    vextractps $1, %xmm0, %eax
 ; AVX-32-NEXT:    shrl $31, %eax
@@ -471,7 +471,7 @@ define <2 x float> @uitofp_v2i64_v2f32(<2 x i64> %x) #0 {
 ; AVX512DQ-32:       # %bb.0:
 ; AVX512DQ-32-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; AVX512DQ-32-NEXT:    vcvtuqq2ps %zmm0, %ymm1
-; AVX512DQ-32-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX512DQ-32-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; AVX512DQ-32-NEXT:    vcvtuqq2ps %zmm0, %ymm0
 ; AVX512DQ-32-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],zero,zero
 ; AVX512DQ-32-NEXT:    vzeroupper
@@ -1146,7 +1146,7 @@ define <2 x double> @sitofp_v2i64_v2f64(<2 x i64> %x) #0 {
 ; SSE-32-NEXT:    andl $-8, %esp
 ; SSE-32-NEXT:    subl $32, %esp
 ; SSE-32-NEXT:    movq %xmm0, {{[0-9]+}}(%esp)
-; SSE-32-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE-32-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE-32-NEXT:    movq %xmm0, {{[0-9]+}}(%esp)
 ; SSE-32-NEXT:    fildll {{[0-9]+}}(%esp)
 ; SSE-32-NEXT:    fstpl {{[0-9]+}}(%esp)
@@ -1164,7 +1164,7 @@ define <2 x double> @sitofp_v2i64_v2f64(<2 x i64> %x) #0 {
 ; SSE-64:       # %bb.0:
 ; SSE-64-NEXT:    movq %xmm0, %rax
 ; SSE-64-NEXT:    cvtsi2sd %rax, %xmm1
-; SSE-64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE-64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE-64-NEXT:    movq %xmm0, %rax
 ; SSE-64-NEXT:    xorps %xmm0, %xmm0
 ; SSE-64-NEXT:    cvtsi2sd %rax, %xmm0
@@ -1182,7 +1182,7 @@ define <2 x double> @sitofp_v2i64_v2f64(<2 x i64> %x) #0 {
 ; SSE41-32-NEXT:    andl $-8, %esp
 ; SSE41-32-NEXT:    subl $32, %esp
 ; SSE41-32-NEXT:    movq %xmm0, {{[0-9]+}}(%esp)
-; SSE41-32-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE41-32-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE41-32-NEXT:    movq %xmm0, {{[0-9]+}}(%esp)
 ; SSE41-32-NEXT:    fildll {{[0-9]+}}(%esp)
 ; SSE41-32-NEXT:    fstpl {{[0-9]+}}(%esp)
@@ -1200,7 +1200,7 @@ define <2 x double> @sitofp_v2i64_v2f64(<2 x i64> %x) #0 {
 ; SSE41-64:       # %bb.0:
 ; SSE41-64-NEXT:    movq %xmm0, %rax
 ; SSE41-64-NEXT:    cvtsi2sd %rax, %xmm1
-; SSE41-64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE41-64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE41-64-NEXT:    movq %xmm0, %rax
 ; SSE41-64-NEXT:    xorps %xmm0, %xmm0
 ; SSE41-64-NEXT:    cvtsi2sd %rax, %xmm0
@@ -1218,7 +1218,7 @@ define <2 x double> @sitofp_v2i64_v2f64(<2 x i64> %x) #0 {
 ; AVX-32-NEXT:    andl $-8, %esp
 ; AVX-32-NEXT:    subl $32, %esp
 ; AVX-32-NEXT:    vmovlps %xmm0, {{[0-9]+}}(%esp)
-; AVX-32-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX-32-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; AVX-32-NEXT:    vmovlps %xmm0, {{[0-9]+}}(%esp)
 ; AVX-32-NEXT:    fildll {{[0-9]+}}(%esp)
 ; AVX-32-NEXT:    fstpl {{[0-9]+}}(%esp)
diff --git a/llvm/test/CodeGen/X86/vec-strict-inttofp-256.ll b/llvm/test/CodeGen/X86/vec-strict-inttofp-256.ll
index d97787e36cd85..a5519e68f73a3 100644
--- a/llvm/test/CodeGen/X86/vec-strict-inttofp-256.ll
+++ b/llvm/test/CodeGen/X86/vec-strict-inttofp-256.ll
@@ -281,7 +281,7 @@ define <8 x float> @sitofp_v8i16_v8f32(<8 x i16> %x) #0 {
 ; AVX1-LABEL: sitofp_v8i16_v8f32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    vcvtdq2ps %ymm0, %ymm0
diff --git a/llvm/test/CodeGen/X86/vec_cast2.ll b/llvm/test/CodeGen/X86/vec_cast2.ll
index ed703f1708478..c979050fc1809 100644
--- a/llvm/test/CodeGen/X86/vec_cast2.ll
+++ b/llvm/test/CodeGen/X86/vec_cast2.ll
@@ -18,7 +18,7 @@ define <8 x float> @cvt_v8i16_v8f32(<8 x i16> %src) {
 ; CHECK-LABEL: cvt_v8i16_v8f32:
 ; CHECK:       ## %bb.0:
 ; CHECK-NEXT:    vpmovsxwd %xmm0, %xmm1
-; CHECK-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; CHECK-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; CHECK-NEXT:    vpmovsxwd %xmm0, %xmm0
 ; CHECK-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; CHECK-NEXT:    vcvtdq2ps %ymm0, %ymm0
diff --git a/llvm/test/CodeGen/X86/vec_int_to_fp.ll b/llvm/test/CodeGen/X86/vec_int_to_fp.ll
index 6cb352c3f348f..a413752993b5a 100644
--- a/llvm/test/CodeGen/X86/vec_int_to_fp.ll
+++ b/llvm/test/CodeGen/X86/vec_int_to_fp.ll
@@ -95,7 +95,7 @@ define <2 x double> @sitofp_2i64_to_2f64(<2 x i64> %a) {
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movq %xmm0, %rax
 ; SSE2-NEXT:    cvtsi2sd %rax, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm0, %rax
 ; SSE2-NEXT:    xorps %xmm0, %xmm0
 ; SSE2-NEXT:    cvtsi2sd %rax, %xmm0
@@ -305,14 +305,14 @@ define <4 x double> @sitofp_4i64_to_4f64(<4 x i64> %a) {
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movq %xmm0, %rax
 ; SSE2-NEXT:    cvtsi2sd %rax, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm0, %rax
 ; SSE2-NEXT:    xorps %xmm0, %xmm0
 ; SSE2-NEXT:    cvtsi2sd %rax, %xmm0
 ; SSE2-NEXT:    unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm0[0]
 ; SSE2-NEXT:    movq %xmm1, %rax
 ; SSE2-NEXT:    cvtsi2sd %rax, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm0, %rax
 ; SSE2-NEXT:    xorps %xmm0, %xmm0
 ; SSE2-NEXT:    cvtsi2sd %rax, %xmm0
@@ -421,7 +421,7 @@ define <4 x double> @sitofp_4i32_to_4f64(<4 x i32> %a) {
 ; SSE-LABEL: sitofp_4i32_to_4f64:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    cvtdq2pd %xmm0, %xmm2
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE-NEXT:    cvtdq2pd %xmm0, %xmm1
 ; SSE-NEXT:    movaps %xmm2, %xmm0
 ; SSE-NEXT:    retq
@@ -440,7 +440,7 @@ define <4 x double> @sitofp_4i16_to_4f64(<8 x i16> %a) {
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
 ; SSE2-NEXT:    psrad $16, %xmm1
 ; SSE2-NEXT:    cvtdq2pd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    cvtdq2pd %xmm1, %xmm1
 ; SSE2-NEXT:    retq
 ;
@@ -448,7 +448,7 @@ define <4 x double> @sitofp_4i16_to_4f64(<8 x i16> %a) {
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pmovsxwd %xmm0, %xmm1
 ; SSE41-NEXT:    cvtdq2pd %xmm1, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE41-NEXT:    cvtdq2pd %xmm1, %xmm1
 ; SSE41-NEXT:    retq
 ;
@@ -468,7 +468,7 @@ define <4 x double> @sitofp_8i16_to_4f64(<8 x i16> %a) {
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
 ; SSE2-NEXT:    psrad $16, %xmm1
 ; SSE2-NEXT:    cvtdq2pd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    cvtdq2pd %xmm1, %xmm1
 ; SSE2-NEXT:    retq
 ;
@@ -476,7 +476,7 @@ define <4 x double> @sitofp_8i16_to_4f64(<8 x i16> %a) {
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pmovsxwd %xmm0, %xmm1
 ; SSE41-NEXT:    cvtdq2pd %xmm1, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE41-NEXT:    cvtdq2pd %xmm1, %xmm1
 ; SSE41-NEXT:    retq
 ;
@@ -504,7 +504,7 @@ define <4 x double> @sitofp_4i8_to_4f64(<16 x i8> %a) {
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
 ; SSE2-NEXT:    psrad $24, %xmm1
 ; SSE2-NEXT:    cvtdq2pd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    cvtdq2pd %xmm1, %xmm1
 ; SSE2-NEXT:    retq
 ;
@@ -512,7 +512,7 @@ define <4 x double> @sitofp_4i8_to_4f64(<16 x i8> %a) {
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pmovsxbd %xmm0, %xmm1
 ; SSE41-NEXT:    cvtdq2pd %xmm1, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE41-NEXT:    cvtdq2pd %xmm1, %xmm1
 ; SSE41-NEXT:    retq
 ;
@@ -533,7 +533,7 @@ define <4 x double> @sitofp_16i8_to_4f64(<16 x i8> %a) {
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
 ; SSE2-NEXT:    psrad $24, %xmm1
 ; SSE2-NEXT:    cvtdq2pd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    cvtdq2pd %xmm1, %xmm1
 ; SSE2-NEXT:    retq
 ;
@@ -541,7 +541,7 @@ define <4 x double> @sitofp_16i8_to_4f64(<16 x i8> %a) {
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pmovsxbd %xmm0, %xmm1
 ; SSE41-NEXT:    cvtdq2pd %xmm1, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE41-NEXT:    cvtdq2pd %xmm1, %xmm1
 ; SSE41-NEXT:    retq
 ;
@@ -1015,7 +1015,7 @@ define <4 x double> @uitofp_4i32_to_4f64(<4 x i32> %a) {
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [4.503599627370496E+15,4.503599627370496E+15]
 ; SSE41-NEXT:    por %xmm3, %xmm2
 ; SSE41-NEXT:    subpd %xmm3, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero
 ; SSE41-NEXT:    por %xmm3, %xmm1
 ; SSE41-NEXT:    subpd %xmm3, %xmm1
@@ -1074,7 +1074,7 @@ define <4 x double> @uitofp_4i16_to_4f64(<8 x i16> %a) {
 ; SSE2-NEXT:    pxor %xmm1, %xmm1
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; SSE2-NEXT:    cvtdq2pd %xmm0, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    cvtdq2pd %xmm0, %xmm1
 ; SSE2-NEXT:    movaps %xmm2, %xmm0
 ; SSE2-NEXT:    retq
@@ -1083,7 +1083,7 @@ define <4 x double> @uitofp_4i16_to_4f64(<8 x i16> %a) {
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; SSE41-NEXT:    cvtdq2pd %xmm1, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE41-NEXT:    cvtdq2pd %xmm1, %xmm1
 ; SSE41-NEXT:    retq
 ;
@@ -1103,7 +1103,7 @@ define <4 x double> @uitofp_8i16_to_4f64(<8 x i16> %a) {
 ; SSE2-NEXT:    pxor %xmm1, %xmm1
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; SSE2-NEXT:    cvtdq2pd %xmm0, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    cvtdq2pd %xmm0, %xmm1
 ; SSE2-NEXT:    movaps %xmm2, %xmm0
 ; SSE2-NEXT:    retq
@@ -1112,7 +1112,7 @@ define <4 x double> @uitofp_8i16_to_4f64(<8 x i16> %a) {
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; SSE41-NEXT:    cvtdq2pd %xmm1, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE41-NEXT:    cvtdq2pd %xmm1, %xmm1
 ; SSE41-NEXT:    retq
 ;
@@ -1140,7 +1140,7 @@ define <4 x double> @uitofp_4i8_to_4f64(<16 x i8> %a) {
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; SSE2-NEXT:    cvtdq2pd %xmm0, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    cvtdq2pd %xmm0, %xmm1
 ; SSE2-NEXT:    movaps %xmm2, %xmm0
 ; SSE2-NEXT:    retq
@@ -1149,7 +1149,7 @@ define <4 x double> @uitofp_4i8_to_4f64(<16 x i8> %a) {
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE41-NEXT:    cvtdq2pd %xmm1, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE41-NEXT:    cvtdq2pd %xmm1, %xmm1
 ; SSE41-NEXT:    retq
 ;
@@ -1170,7 +1170,7 @@ define <4 x double> @uitofp_16i8_to_4f64(<16 x i8> %a) {
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; SSE2-NEXT:    cvtdq2pd %xmm0, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    cvtdq2pd %xmm0, %xmm1
 ; SSE2-NEXT:    movaps %xmm2, %xmm0
 ; SSE2-NEXT:    retq
@@ -1179,7 +1179,7 @@ define <4 x double> @uitofp_16i8_to_4f64(<16 x i8> %a) {
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE41-NEXT:    cvtdq2pd %xmm1, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE41-NEXT:    cvtdq2pd %xmm1, %xmm1
 ; SSE41-NEXT:    retq
 ;
@@ -1209,7 +1209,7 @@ define <4 x float> @sitofp_2i64_to_4f32(<2 x i64> %a) {
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movq %xmm0, %rax
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm0, %rax
 ; SSE2-NEXT:    xorps %xmm0, %xmm0
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm0
@@ -1274,7 +1274,7 @@ define <4 x float> @sitofp_2i64_to_4f32(<2 x i64> %a) {
 define <4 x float> @sitofp_2i64_to_4f32_zero(<2 x i64> %a) {
 ; SSE2-LABEL: sitofp_2i64_to_4f32_zero:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm1, %rax
 ; SSE2-NEXT:    xorps %xmm1, %xmm1
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm1
@@ -1345,7 +1345,7 @@ define <4 x float> @sitofp_4i64_to_4f32_undef(<2 x i64> %a) {
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movq %xmm0, %rax
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm0, %rax
 ; SSE2-NEXT:    xorps %xmm0, %xmm0
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm0
@@ -1464,7 +1464,7 @@ define <4 x float> @sitofp_8i16_to_4f32(<8 x i16> %a) {
 ; AVX1-LABEL: sitofp_8i16_to_4f32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    vcvtdq2ps %ymm0, %ymm0
@@ -1568,7 +1568,7 @@ define <4 x float> @sitofp_4i64_to_4f32(<4 x i64> %a) {
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movq %xmm1, %rax
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm1, %rax
 ; SSE2-NEXT:    xorps %xmm1, %xmm1
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm1
@@ -1576,7 +1576,7 @@ define <4 x float> @sitofp_4i64_to_4f32(<4 x i64> %a) {
 ; SSE2-NEXT:    movq %xmm0, %rax
 ; SSE2-NEXT:    xorps %xmm1, %xmm1
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm0, %rax
 ; SSE2-NEXT:    xorps %xmm0, %xmm0
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm0
@@ -1719,7 +1719,7 @@ define <8 x float> @sitofp_8i16_to_8f32(<8 x i16> %a) {
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pmovsxwd %xmm0, %xmm1
 ; SSE41-NEXT:    cvtdq2ps %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovsxwd %xmm0, %xmm0
 ; SSE41-NEXT:    cvtdq2ps %xmm0, %xmm1
 ; SSE41-NEXT:    movaps %xmm2, %xmm0
@@ -1728,7 +1728,7 @@ define <8 x float> @sitofp_8i16_to_8f32(<8 x i16> %a) {
 ; AVX1-LABEL: sitofp_8i16_to_8f32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    vcvtdq2ps %ymm0, %ymm0
@@ -1868,7 +1868,7 @@ define <4 x float> @uitofp_2i64_to_4f32(<2 x i64> %a) {
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm0
 ; SSE2-NEXT:    addss %xmm0, %xmm0
 ; SSE2-NEXT:  .LBB41_3:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm1, %rax
 ; SSE2-NEXT:    testq %rax, %rax
 ; SSE2-NEXT:    js .LBB41_4
@@ -1969,7 +1969,7 @@ define <4 x float> @uitofp_2i64_to_4f32(<2 x i64> %a) {
 define <4 x float> @uitofp_2i64_to_2f32(<2 x i64> %a) {
 ; SSE2-LABEL: uitofp_2i64_to_2f32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm1, %rax
 ; SSE2-NEXT:    testq %rax, %rax
 ; SSE2-NEXT:    js .LBB42_1
@@ -2102,7 +2102,7 @@ define <4 x float> @uitofp_4i64_to_4f32_undef(<2 x i64> %a) {
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm1
 ; SSE2-NEXT:    addss %xmm1, %xmm1
 ; SSE2-NEXT:  .LBB43_3:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm0, %rax
 ; SSE2-NEXT:    testq %rax, %rax
 ; SSE2-NEXT:    js .LBB43_4
@@ -2462,7 +2462,7 @@ define <4 x float> @uitofp_4i64_to_4f32(<4 x i64> %a) {
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm2
 ; SSE2-NEXT:    addss %xmm2, %xmm2
 ; SSE2-NEXT:  .LBB49_3:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm1, %rax
 ; SSE2-NEXT:    testq %rax, %rax
 ; SSE2-NEXT:    js .LBB49_4
@@ -2494,7 +2494,7 @@ define <4 x float> @uitofp_4i64_to_4f32(<4 x i64> %a) {
 ; SSE2-NEXT:    addss %xmm1, %xmm1
 ; SSE2-NEXT:  .LBB49_9:
 ; SSE2-NEXT:    unpcklps {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm0, %rax
 ; SSE2-NEXT:    testq %rax, %rax
 ; SSE2-NEXT:    js .LBB49_10
@@ -2769,7 +2769,7 @@ define <8 x float> @uitofp_8i16_to_8f32(<8 x i16> %a) {
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; SSE41-NEXT:    cvtdq2ps %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; SSE41-NEXT:    cvtdq2ps %xmm0, %xmm1
 ; SSE41-NEXT:    movaps %xmm2, %xmm0
@@ -2906,7 +2906,7 @@ define <2 x double> @sitofp_load_2i64_to_2f64(<2 x i64> *%a) {
 ; SSE2-NEXT:    movdqa (%rdi), %xmm1
 ; SSE2-NEXT:    movq %xmm1, %rax
 ; SSE2-NEXT:    cvtsi2sd %rax, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm1, %rax
 ; SSE2-NEXT:    xorps %xmm1, %xmm1
 ; SSE2-NEXT:    cvtsi2sd %rax, %xmm1
@@ -3103,7 +3103,7 @@ define <4 x double> @sitofp_load_4i64_to_4f64(<4 x i64> *%a) {
 ; SSE2-NEXT:    movdqa 16(%rdi), %xmm2
 ; SSE2-NEXT:    movq %xmm1, %rax
 ; SSE2-NEXT:    cvtsi2sd %rax, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm1, %rax
 ; SSE2-NEXT:    xorps %xmm1, %xmm1
 ; SSE2-NEXT:    cvtsi2sd %rax, %xmm1
@@ -3111,7 +3111,7 @@ define <4 x double> @sitofp_load_4i64_to_4f64(<4 x i64> *%a) {
 ; SSE2-NEXT:    movq %xmm2, %rax
 ; SSE2-NEXT:    xorps %xmm1, %xmm1
 ; SSE2-NEXT:    cvtsi2sd %rax, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm2, %rax
 ; SSE2-NEXT:    xorps %xmm2, %xmm2
 ; SSE2-NEXT:    cvtsi2sd %rax, %xmm2
@@ -3209,7 +3209,7 @@ define <4 x double> @sitofp_load_4i32_to_4f64(<4 x i32> *%a) {
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    movdqa (%rdi), %xmm1
 ; SSE-NEXT:    cvtdq2pd %xmm1, %xmm0
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE-NEXT:    cvtdq2pd %xmm1, %xmm1
 ; SSE-NEXT:    retq
 ;
@@ -3229,7 +3229,7 @@ define <4 x double> @sitofp_load_4i16_to_4f64(<4 x i16> *%a) {
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
 ; SSE2-NEXT:    psrad $16, %xmm1
 ; SSE2-NEXT:    cvtdq2pd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    cvtdq2pd %xmm1, %xmm1
 ; SSE2-NEXT:    retq
 ;
@@ -3237,7 +3237,7 @@ define <4 x double> @sitofp_load_4i16_to_4f64(<4 x i16> *%a) {
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pmovsxwd (%rdi), %xmm1
 ; SSE41-NEXT:    cvtdq2pd %xmm1, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE41-NEXT:    cvtdq2pd %xmm1, %xmm1
 ; SSE41-NEXT:    retq
 ;
@@ -3259,7 +3259,7 @@ define <4 x double> @sitofp_load_4i8_to_4f64(<4 x i8> *%a) {
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
 ; SSE2-NEXT:    psrad $24, %xmm1
 ; SSE2-NEXT:    cvtdq2pd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    cvtdq2pd %xmm1, %xmm1
 ; SSE2-NEXT:    retq
 ;
@@ -3267,7 +3267,7 @@ define <4 x double> @sitofp_load_4i8_to_4f64(<4 x i8> *%a) {
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pmovsxbd (%rdi), %xmm1
 ; SSE41-NEXT:    cvtdq2pd %xmm1, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE41-NEXT:    cvtdq2pd %xmm1, %xmm1
 ; SSE41-NEXT:    retq
 ;
@@ -3770,7 +3770,7 @@ define <4 x double> @uitofp_load_4i32_to_4f64(<4 x i32> *%a) {
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [4.503599627370496E+15,4.503599627370496E+15]
 ; SSE41-NEXT:    por %xmm2, %xmm0
 ; SSE41-NEXT:    subpd %xmm2, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
 ; SSE41-NEXT:    por %xmm2, %xmm1
 ; SSE41-NEXT:    subpd %xmm2, %xmm1
@@ -3831,7 +3831,7 @@ define <4 x double> @uitofp_load_4i16_to_4f64(<4 x i16> *%a) {
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
 ; SSE2-NEXT:    cvtdq2pd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    cvtdq2pd %xmm1, %xmm1
 ; SSE2-NEXT:    retq
 ;
@@ -3839,7 +3839,7 @@ define <4 x double> @uitofp_load_4i16_to_4f64(<4 x i16> *%a) {
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
 ; SSE41-NEXT:    cvtdq2pd %xmm1, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE41-NEXT:    cvtdq2pd %xmm1, %xmm1
 ; SSE41-NEXT:    retq
 ;
@@ -3861,7 +3861,7 @@ define <4 x double> @uitofp_load_4i8_to_4f64(<4 x i8> *%a) {
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
 ; SSE2-NEXT:    cvtdq2pd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    cvtdq2pd %xmm1, %xmm1
 ; SSE2-NEXT:    retq
 ;
@@ -3869,7 +3869,7 @@ define <4 x double> @uitofp_load_4i8_to_4f64(<4 x i8> *%a) {
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
 ; SSE41-NEXT:    cvtdq2pd %xmm1, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE41-NEXT:    cvtdq2pd %xmm1, %xmm1
 ; SSE41-NEXT:    retq
 ;
@@ -3894,7 +3894,7 @@ define <4 x float> @sitofp_load_4i64_to_4f32(<4 x i64> *%a) {
 ; SSE2-NEXT:    movdqa 16(%rdi), %xmm0
 ; SSE2-NEXT:    movq %xmm0, %rax
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm0, %rax
 ; SSE2-NEXT:    xorps %xmm0, %xmm0
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm0
@@ -3902,7 +3902,7 @@ define <4 x float> @sitofp_load_4i64_to_4f32(<4 x i64> *%a) {
 ; SSE2-NEXT:    movq %xmm1, %rax
 ; SSE2-NEXT:    xorps %xmm0, %xmm0
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm1, %rax
 ; SSE2-NEXT:    xorps %xmm1, %xmm1
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm1
@@ -4073,7 +4073,7 @@ define <8 x float> @sitofp_load_8i64_to_8f32(<8 x i64> *%a) {
 ; SSE2-NEXT:    movdqa 48(%rdi), %xmm3
 ; SSE2-NEXT:    movq %xmm0, %rax
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm0, %rax
 ; SSE2-NEXT:    xorps %xmm0, %xmm0
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm0
@@ -4081,7 +4081,7 @@ define <8 x float> @sitofp_load_8i64_to_8f32(<8 x i64> *%a) {
 ; SSE2-NEXT:    movq %xmm1, %rax
 ; SSE2-NEXT:    xorps %xmm0, %xmm0
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm1, %rax
 ; SSE2-NEXT:    xorps %xmm1, %xmm1
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm1
@@ -4090,7 +4090,7 @@ define <8 x float> @sitofp_load_8i64_to_8f32(<8 x i64> *%a) {
 ; SSE2-NEXT:    movq %xmm3, %rax
 ; SSE2-NEXT:    xorps %xmm4, %xmm4
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm1, %rax
 ; SSE2-NEXT:    xorps %xmm1, %xmm1
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm1
@@ -4098,7 +4098,7 @@ define <8 x float> @sitofp_load_8i64_to_8f32(<8 x i64> *%a) {
 ; SSE2-NEXT:    movq %xmm2, %rax
 ; SSE2-NEXT:    xorps %xmm1, %xmm1
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm2, %rax
 ; SSE2-NEXT:    xorps %xmm2, %xmm2
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm2
@@ -4378,7 +4378,7 @@ define <4 x float> @uitofp_load_4i64_to_4f32(<4 x i64> *%a) {
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm1
 ; SSE2-NEXT:    addss %xmm1, %xmm1
 ; SSE2-NEXT:  .LBB83_3:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm0, %rax
 ; SSE2-NEXT:    testq %rax, %rax
 ; SSE2-NEXT:    js .LBB83_4
@@ -4410,7 +4410,7 @@ define <4 x float> @uitofp_load_4i64_to_4f32(<4 x i64> *%a) {
 ; SSE2-NEXT:    addss %xmm0, %xmm0
 ; SSE2-NEXT:  .LBB83_9:
 ; SSE2-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm2, %rax
 ; SSE2-NEXT:    testq %rax, %rax
 ; SSE2-NEXT:    js .LBB83_10
@@ -4729,7 +4729,7 @@ define <8 x float> @uitofp_load_8i64_to_8f32(<8 x i64> *%a) {
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm3
 ; SSE2-NEXT:    addss %xmm3, %xmm3
 ; SSE2-NEXT:  .LBB87_3:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm0, %rax
 ; SSE2-NEXT:    testq %rax, %rax
 ; SSE2-NEXT:    js .LBB87_4
@@ -4760,7 +4760,7 @@ define <8 x float> @uitofp_load_8i64_to_8f32(<8 x i64> *%a) {
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm0
 ; SSE2-NEXT:    addss %xmm0, %xmm0
 ; SSE2-NEXT:  .LBB87_9:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm5, %rax
 ; SSE2-NEXT:    testq %rax, %rax
 ; SSE2-NEXT:    js .LBB87_10
@@ -4791,7 +4791,7 @@ define <8 x float> @uitofp_load_8i64_to_8f32(<8 x i64> *%a) {
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm5
 ; SSE2-NEXT:    addss %xmm5, %xmm5
 ; SSE2-NEXT:  .LBB87_15:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm1, %rax
 ; SSE2-NEXT:    testq %rax, %rax
 ; SSE2-NEXT:    js .LBB87_16
@@ -4826,7 +4826,7 @@ define <8 x float> @uitofp_load_8i64_to_8f32(<8 x i64> *%a) {
 ; SSE2-NEXT:  .LBB87_21:
 ; SSE2-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0],xmm3[0]
 ; SSE2-NEXT:    unpcklps {{.*#+}} xmm5 = xmm5[0],xmm7[0],xmm5[1],xmm7[1]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm2, %rax
 ; SSE2-NEXT:    testq %rax, %rax
 ; SSE2-NEXT:    js .LBB87_22
diff --git a/llvm/test/CodeGen/X86/vec_saddo.ll b/llvm/test/CodeGen/X86/vec_saddo.ll
index dd3a733ab2178..4f071c064e5c8 100644
--- a/llvm/test/CodeGen/X86/vec_saddo.ll
+++ b/llvm/test/CodeGen/X86/vec_saddo.ll
@@ -107,7 +107,7 @@ define <3 x i32> @saddo_v3i32(<3 x i32> %a0, <3 x i32> %a1, <3 x i32>* %p2) noun
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE2-NEXT:    pxor %xmm2, %xmm0
 ; SSE2-NEXT:    movq %xmm1, (%rdi)
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    movd %xmm1, 8(%rdi)
 ; SSE2-NEXT:    retq
 ;
@@ -119,7 +119,7 @@ define <3 x i32> @saddo_v3i32(<3 x i32> %a0, <3 x i32> %a1, <3 x i32>* %p2) noun
 ; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSSE3-NEXT:    pxor %xmm2, %xmm0
 ; SSSE3-NEXT:    movq %xmm1, (%rdi)
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSSE3-NEXT:    movd %xmm1, 8(%rdi)
 ; SSSE3-NEXT:    retq
 ;
@@ -512,13 +512,13 @@ define <16 x i32> @saddo_v16i32(<16 x i32> %a0, <16 x i32> %a1, <16 x i32>* %p2)
 ; AVX1-NEXT:    vpaddd %xmm2, %xmm0, %xmm2
 ; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpxor %xmm0, %xmm5, %xmm0
-; AVX1-NEXT:    vpackssdw %xmm4, %xmm0, %xmm4
-; AVX1-NEXT:    vpacksswb %xmm1, %xmm4, %xmm0
-; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm5
+; AVX1-NEXT:    vpackssdw %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm4
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm5, %ymm0
-; AVX1-NEXT:    vpacksswb %xmm4, %xmm1, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm4, %ymm0
+; AVX1-NEXT:    vpacksswb %xmm1, %xmm1, %xmm1
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm4
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
@@ -644,7 +644,7 @@ define <16 x i32> @saddo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nou
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm1
 ; SSE41-NEXT:    psrad $31, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm2
 ; SSE41-NEXT:    psrad $31, %xmm2
@@ -667,9 +667,9 @@ define <16 x i32> @saddo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nou
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm2
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
 ; AVX1-NEXT:    vmovdqa %xmm3, (%rdi)
@@ -683,7 +683,7 @@ define <16 x i32> @saddo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nou
 ; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm1
 ; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm1
 ; AVX2-NEXT:    vmovdqa %xmm3, (%rdi)
 ; AVX2-NEXT:    retq
@@ -769,7 +769,7 @@ define <8 x i32> @saddo_v8i16(<8 x i16> %a0, <8 x i16> %a1, <8 x i16>* %p2) noun
 ; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
 ; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
@@ -889,7 +889,7 @@ define <4 x i32> @saddo_v4i24(<4 x i24> %a0, <4 x i24> %a1, <4 x i24>* %p2) noun
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[3,1,2,3]
 ; SSE2-NEXT:    movd %xmm1, %ecx
 ; SSE2-NEXT:    movw %cx, 9(%rdi)
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]
 ; SSE2-NEXT:    movd %xmm1, %edx
 ; SSE2-NEXT:    movw %dx, 6(%rdi)
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,2,3]
@@ -924,7 +924,7 @@ define <4 x i32> @saddo_v4i24(<4 x i24> %a0, <4 x i24> %a1, <4 x i24>* %p2) noun
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[3,1,2,3]
 ; SSSE3-NEXT:    movd %xmm1, %ecx
 ; SSSE3-NEXT:    movw %cx, 9(%rdi)
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,3,0,1]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]
 ; SSSE3-NEXT:    movd %xmm1, %edx
 ; SSSE3-NEXT:    movw %dx, 6(%rdi)
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,2,3]
diff --git a/llvm/test/CodeGen/X86/vec_smulo.ll b/llvm/test/CodeGen/X86/vec_smulo.ll
index 5fde07d1269df..b5fefe296d77e 100644
--- a/llvm/test/CodeGen/X86/vec_smulo.ll
+++ b/llvm/test/CodeGen/X86/vec_smulo.ll
@@ -1193,13 +1193,13 @@ define <16 x i32> @smulo_v16i32(<16 x i32> %a0, <16 x i32> %a1, <16 x i32>* %p2)
 ; AVX1-NEXT:    vpsrad $31, %xmm2, %xmm0
 ; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm6, %xmm0
 ; AVX1-NEXT:    vpxor %xmm0, %xmm9, %xmm0
-; AVX1-NEXT:    vpackssdw %xmm5, %xmm0, %xmm5
-; AVX1-NEXT:    vpacksswb %xmm1, %xmm5, %xmm0
-; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm6
+; AVX1-NEXT:    vpackssdw %xmm5, %xmm0, %xmm0
+; AVX1-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm5
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm6, %ymm0
-; AVX1-NEXT:    vpacksswb %xmm5, %xmm1, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm5, %ymm0
+; AVX1-NEXT:    vpacksswb %xmm1, %xmm1, %xmm1
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm5
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
@@ -1376,11 +1376,11 @@ define <16 x i32> @smulo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nou
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; SSE41-NEXT:    pmovsxbw %xmm1, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[2,3,2,3]
 ; SSE41-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; SSE41-NEXT:    pmovsxbw %xmm0, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; SSE41-NEXT:    pmullw %xmm1, %xmm0
 ; SSE41-NEXT:    pmullw %xmm2, %xmm4
@@ -1407,7 +1407,7 @@ define <16 x i32> @smulo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nou
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm1
 ; SSE41-NEXT:    psrad $31, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm2
 ; SSE41-NEXT:    psrad $31, %xmm2
@@ -1436,9 +1436,9 @@ define <16 x i32> @smulo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nou
 ; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm5
 ; AVX1-NEXT:    vpmullw %xmm4, %xmm5, %xmm4
 ; AVX1-NEXT:    vpsrlw $8, %xmm4, %xmm4
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbw %xmm1, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm0
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm0
@@ -1450,9 +1450,9 @@ define <16 x i32> @smulo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nou
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
 ; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm3, %ymm1
 ; AVX1-NEXT:    vmovdqa %xmm2, (%rdi)
@@ -1478,7 +1478,7 @@ define <16 x i32> @smulo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nou
 ; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm1
 ; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm1
 ; AVX2-NEXT:    vmovdqa %xmm3, (%rdi)
 ; AVX2-NEXT:    retq
@@ -1730,11 +1730,11 @@ define <32 x i32> @smulo_v32i8(<32 x i8> %a0, <32 x i8> %a1, <32 x i8>* %p2) nou
 ; SSE41-NEXT:    movq %rdi, %rax
 ; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm10 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
 ; SSE41-NEXT:    pmovsxbw %xmm3, %xmm7
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[2,3,2,3]
 ; SSE41-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm9 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; SSE41-NEXT:    pmovsxbw %xmm1, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[2,3,2,3]
 ; SSE41-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; SSE41-NEXT:    pmullw %xmm3, %xmm1
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm8 = [255,255,255,255,255,255,255,255]
@@ -1754,12 +1754,12 @@ define <32 x i32> @smulo_v32i8(<32 x i8> %a0, <32 x i8> %a1, <32 x i8>* %p2) nou
 ; SSE41-NEXT:    pcmpeqb %xmm1, %xmm4
 ; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm10 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
 ; SSE41-NEXT:    pmovsxbw %xmm2, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[2,3,2,3]
 ; SSE41-NEXT:    movdqa %xmm2, %xmm7
 ; SSE41-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; SSE41-NEXT:    pmovsxbw %xmm0, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; SSE41-NEXT:    pmullw %xmm7, %xmm0
 ; SSE41-NEXT:    pmullw %xmm10, %xmm1
@@ -1783,7 +1783,7 @@ define <32 x i32> @smulo_v32i8(<32 x i8> %a0, <32 x i8> %a1, <32 x i8>* %p2) nou
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm8 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm8
 ; SSE41-NEXT:    psrad $31, %xmm8
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm5 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm5
 ; SSE41-NEXT:    psrad $31, %xmm5
@@ -1795,7 +1795,7 @@ define <32 x i32> @smulo_v32i8(<32 x i8> %a0, <32 x i8> %a1, <32 x i8>* %p2) nou
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm7 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm7
 ; SSE41-NEXT:    psrad $31, %xmm7
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[2,3,2,3]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm3
 ; SSE41-NEXT:    psrad $31, %xmm3
@@ -1841,9 +1841,9 @@ define <32 x i32> @smulo_v32i8(<32 x i8> %a0, <32 x i8> %a1, <32 x i8>* %p2) nou
 ; AVX1-NEXT:    vpmovsxbw %xmm5, %xmm4
 ; AVX1-NEXT:    vpmullw %xmm7, %xmm4, %xmm4
 ; AVX1-NEXT:    vpsrlw $8, %xmm4, %xmm4
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbw %xmm2, %xmm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbw %xmm5, %xmm5
 ; AVX1-NEXT:    vpmullw %xmm2, %xmm5, %xmm2
 ; AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
@@ -1865,9 +1865,9 @@ define <32 x i32> @smulo_v32i8(<32 x i8> %a0, <32 x i8> %a1, <32 x i8>* %p2) nou
 ; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm7
 ; AVX1-NEXT:    vpmullw %xmm6, %xmm7, %xmm6
 ; AVX1-NEXT:    vpsrlw $8, %xmm6, %xmm6
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbw %xmm1, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm0
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm0
@@ -1882,14 +1882,14 @@ define <32 x i32> @smulo_v32i8(<32 x i8> %a0, <32 x i8> %a1, <32 x i8>* %p2) nou
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm3[1,1,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm4, %xmm4
 ; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm2, %ymm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm1[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm1[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm4, %xmm4
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm4, %ymm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm3[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm3[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm4, %xmm4
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[3,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[3,3,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
 ; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm4, %ymm3
 ; AVX1-NEXT:    vmovdqa %xmm9, 16(%rdi)
@@ -1929,9 +1929,9 @@ define <32 x i32> @smulo_v32i8(<32 x i8> %a0, <32 x i8> %a1, <32 x i8>* %p2) nou
 ; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm3
 ; AVX2-NEXT:    vpmovsxbd %xmm3, %ymm2
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm1
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]
 ; AVX2-NEXT:    vpmovsxbd %xmm3, %ymm3
 ; AVX2-NEXT:    vmovdqa %ymm4, (%rdi)
 ; AVX2-NEXT:    retq
@@ -2411,8 +2411,8 @@ define <64 x i32> @smulo_v64i8(<64 x i8> %a0, <64 x i8> %a1, <64 x i8>* %p2) nou
 ; SSE41-NEXT:    pand %xmm10, %xmm9
 ; SSE41-NEXT:    packuswb %xmm11, %xmm9
 ; SSE41-NEXT:    pmovsxbw %xmm3, %xmm8
-; SSE41-NEXT:    pshufd {{.*#+}} xmm11 = xmm7[2,3,0,1]
-; SSE41-NEXT:    pshufd {{.*#+}} xmm13 = xmm3[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm11 = xmm7[2,3,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm13 = xmm3[2,3,2,3]
 ; SSE41-NEXT:    pmullw %xmm12, %xmm8
 ; SSE41-NEXT:    pxor %xmm7, %xmm7
 ; SSE41-NEXT:    pcmpgtb %xmm9, %xmm7
@@ -2436,8 +2436,8 @@ define <64 x i32> @smulo_v64i8(<64 x i8> %a0, <64 x i8> %a1, <64 x i8>* %p2) nou
 ; SSE41-NEXT:    pand %xmm10, %xmm12
 ; SSE41-NEXT:    packuswb %xmm3, %xmm12
 ; SSE41-NEXT:    pmovsxbw %xmm2, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[2,3,0,1]
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[2,3,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
 ; SSE41-NEXT:    pmullw %xmm7, %xmm3
 ; SSE41-NEXT:    pxor %xmm7, %xmm7
 ; SSE41-NEXT:    pcmpgtb %xmm12, %xmm7
@@ -2461,8 +2461,8 @@ define <64 x i32> @smulo_v64i8(<64 x i8> %a0, <64 x i8> %a1, <64 x i8>* %p2) nou
 ; SSE41-NEXT:    pand %xmm10, %xmm11
 ; SSE41-NEXT:    packuswb %xmm7, %xmm11
 ; SSE41-NEXT:    pmovsxbw %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[2,3,0,1]
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[2,3,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE41-NEXT:    pmullw %xmm6, %xmm2
 ; SSE41-NEXT:    pxor %xmm6, %xmm6
 ; SSE41-NEXT:    pcmpgtb %xmm11, %xmm6
@@ -2487,8 +2487,8 @@ define <64 x i32> @smulo_v64i8(<64 x i8> %a0, <64 x i8> %a1, <64 x i8>* %p2) nou
 ; SSE41-NEXT:    packuswb %xmm6, %xmm5
 ; SSE41-NEXT:    pmovsxbw %xmm0, %xmm1
 ; SSE41-NEXT:    pmullw %xmm7, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[2,3,0,1]
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[2,3,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovsxbw %xmm4, %xmm4
 ; SSE41-NEXT:    pmovsxbw %xmm0, %xmm0
 ; SSE41-NEXT:    pmullw %xmm4, %xmm0
@@ -2523,7 +2523,7 @@ define <64 x i32> @smulo_v64i8(<64 x i8> %a0, <64 x i8> %a1, <64 x i8>* %p2) nou
 ; SSE41-NEXT:    pslld $31, %xmm0
 ; SSE41-NEXT:    psrad $31, %xmm0
 ; SSE41-NEXT:    movdqa %xmm0, (%rdi)
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm8[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm8[2,3,2,3]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm0
 ; SSE41-NEXT:    psrad $31, %xmm0
@@ -2538,7 +2538,7 @@ define <64 x i32> @smulo_v64i8(<64 x i8> %a0, <64 x i8> %a1, <64 x i8>* %p2) nou
 ; SSE41-NEXT:    pslld $31, %xmm0
 ; SSE41-NEXT:    psrad $31, %xmm0
 ; SSE41-NEXT:    movdqa %xmm0, 208(%rdi)
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm0
 ; SSE41-NEXT:    psrad $31, %xmm0
@@ -2553,7 +2553,7 @@ define <64 x i32> @smulo_v64i8(<64 x i8> %a0, <64 x i8> %a1, <64 x i8>* %p2) nou
 ; SSE41-NEXT:    pslld $31, %xmm0
 ; SSE41-NEXT:    psrad $31, %xmm0
 ; SSE41-NEXT:    movdqa %xmm0, 144(%rdi)
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm0
 ; SSE41-NEXT:    psrad $31, %xmm0
@@ -2568,7 +2568,7 @@ define <64 x i32> @smulo_v64i8(<64 x i8> %a0, <64 x i8> %a1, <64 x i8>* %p2) nou
 ; SSE41-NEXT:    pslld $31, %xmm0
 ; SSE41-NEXT:    psrad $31, %xmm0
 ; SSE41-NEXT:    movdqa %xmm0, 80(%rdi)
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm0
 ; SSE41-NEXT:    psrad $31, %xmm0
@@ -2606,9 +2606,9 @@ define <64 x i32> @smulo_v64i8(<64 x i8> %a0, <64 x i8> %a1, <64 x i8>* %p2) nou
 ; AVX1-NEXT:    vpmovsxbw %xmm6, %xmm7
 ; AVX1-NEXT:    vpmullw %xmm4, %xmm7, %xmm4
 ; AVX1-NEXT:    vpsrlw $8, %xmm4, %xmm4
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbw %xmm5, %xmm5
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbw %xmm6, %xmm6
 ; AVX1-NEXT:    vpmullw %xmm5, %xmm6, %xmm5
 ; AVX1-NEXT:    vpsrlw $8, %xmm5, %xmm5
@@ -2628,9 +2628,9 @@ define <64 x i32> @smulo_v64i8(<64 x i8> %a0, <64 x i8> %a1, <64 x i8>* %p2) nou
 ; AVX1-NEXT:    vpmovsxbw %xmm1, %xmm7
 ; AVX1-NEXT:    vpmullw %xmm6, %xmm7, %xmm6
 ; AVX1-NEXT:    vpsrlw $8, %xmm6, %xmm6
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbw %xmm3, %xmm3
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbw %xmm1, %xmm1
 ; AVX1-NEXT:    vpmullw %xmm3, %xmm1, %xmm1
 ; AVX1-NEXT:    vpsrlw $8, %xmm1, %xmm1
@@ -2652,9 +2652,9 @@ define <64 x i32> @smulo_v64i8(<64 x i8> %a0, <64 x i8> %a1, <64 x i8>* %p2) nou
 ; AVX1-NEXT:    vpmullw %xmm5, %xmm7, %xmm5
 ; AVX1-NEXT:    vpcmpgtb %xmm1, %xmm11, %xmm7
 ; AVX1-NEXT:    vpsrlw $8, %xmm5, %xmm5
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbw %xmm4, %xmm4
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbw %xmm6, %xmm6
 ; AVX1-NEXT:    vpmullw %xmm4, %xmm6, %xmm4
 ; AVX1-NEXT:    vpsrlw $8, %xmm4, %xmm4
@@ -2672,9 +2672,9 @@ define <64 x i32> @smulo_v64i8(<64 x i8> %a0, <64 x i8> %a1, <64 x i8>* %p2) nou
 ; AVX1-NEXT:    vpmovsxbw %xmm2, %xmm5
 ; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm6
 ; AVX1-NEXT:    vpmullw %xmm5, %xmm6, %xmm5
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbw %xmm2, %xmm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm0
 ; AVX1-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrlw $8, %xmm5, %xmm2
@@ -2699,37 +2699,37 @@ define <64 x i32> @smulo_v64i8(<64 x i8> %a0, <64 x i8> %a1, <64 x i8>* %p2) nou
 ; AVX1-NEXT:    vmovdqa %xmm1, 64(%rdi)
 ; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm1
 ; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm6[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm6[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
 ; AVX1-NEXT:    vmovdqa %xmm1, 224(%rdi)
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm6[3,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm6[3,3,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
 ; AVX1-NEXT:    vmovdqa %xmm1, 240(%rdi)
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm6[1,1,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
 ; AVX1-NEXT:    vmovdqa %xmm1, 208(%rdi)
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm3[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm3[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
 ; AVX1-NEXT:    vmovdqa %xmm1, 160(%rdi)
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm3[3,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm3[3,3,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
 ; AVX1-NEXT:    vmovdqa %xmm1, 176(%rdi)
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm3[1,1,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
 ; AVX1-NEXT:    vmovdqa %xmm1, 144(%rdi)
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm2[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
 ; AVX1-NEXT:    vmovdqa %xmm1, 96(%rdi)
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm2[3,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm2[3,3,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
 ; AVX1-NEXT:    vmovdqa %xmm1, 112(%rdi)
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm2[1,1,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
 ; AVX1-NEXT:    vmovdqa %xmm1, 80(%rdi)
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
 ; AVX1-NEXT:    vmovdqa %xmm1, 32(%rdi)
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[3,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[3,3,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
 ; AVX1-NEXT:    vmovdqa %xmm1, 48(%rdi)
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
@@ -2794,15 +2794,15 @@ define <64 x i32> @smulo_v64i8(<64 x i8> %a0, <64 x i8> %a1, <64 x i8>* %p2) nou
 ; AVX2-NEXT:    vpackuswb %ymm6, %ymm0, %ymm0
 ; AVX2-NEXT:    vpcmpeqb %ymm5, %ymm0, %ymm0
 ; AVX2-NEXT:    vpxor %ymm7, %ymm0, %ymm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpmovsxbd %xmm2, %ymm8
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm5
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm6 = xmm5[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm6 = xmm5[2,3,2,3]
 ; AVX2-NEXT:    vpmovsxbd %xmm6, %ymm6
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm7 = xmm1[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm7 = xmm1[2,3,2,3]
 ; AVX2-NEXT:    vpmovsxbd %xmm7, %ymm7
 ; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm4
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm4[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm4[2,3,2,3]
 ; AVX2-NEXT:    vpmovsxbd %xmm2, %ymm2
 ; AVX2-NEXT:    vpmovsxbd %xmm0, %ymm0
 ; AVX2-NEXT:    vpmovsxbd %xmm5, %ymm5
@@ -2932,7 +2932,7 @@ define <8 x i32> @smulo_v8i16(<8 x i16> %a0, <8 x i16> %a1, <8 x i16>* %p2) noun
 ; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
 ; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
@@ -2971,9 +2971,9 @@ define <8 x i32> @smulo_v8i16(<8 x i16> %a0, <8 x i16> %a1, <8 x i16>* %p2) noun
 define <2 x i32> @smulo_v2i64(<2 x i64> %a0, <2 x i64> %a1, <2 x i64>* %p2) nounwind {
 ; SSE2-LABEL: smulo_v2i64:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm2, %r8
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm2, %rcx
 ; SSE2-NEXT:    movq %xmm1, %rdx
 ; SSE2-NEXT:    movq %xmm0, %rsi
@@ -2996,9 +2996,9 @@ define <2 x i32> @smulo_v2i64(<2 x i64> %a0, <2 x i64> %a1, <2 x i64>* %p2) noun
 ;
 ; SSSE3-LABEL: smulo_v2i64:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; SSSE3-NEXT:    movq %xmm2, %r8
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; SSSE3-NEXT:    movq %xmm2, %rcx
 ; SSSE3-NEXT:    movq %xmm1, %rdx
 ; SSSE3-NEXT:    movq %xmm0, %rsi
@@ -3158,7 +3158,7 @@ define <4 x i32> @smulo_v4i24(<4 x i24> %a0, <4 x i24> %a1, <4 x i24>* %p2) noun
 ; SSE2-NEXT:    por %xmm3, %xmm1
 ; SSE2-NEXT:    movd %xmm0, %eax
 ; SSE2-NEXT:    movw %ax, (%rdi)
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    movd %xmm0, %ecx
 ; SSE2-NEXT:    movw %cx, 6(%rdi)
 ; SSE2-NEXT:    movd %xmm2, %edx
@@ -3213,7 +3213,7 @@ define <4 x i32> @smulo_v4i24(<4 x i24> %a0, <4 x i24> %a1, <4 x i24>* %p2) noun
 ; SSSE3-NEXT:    por %xmm3, %xmm1
 ; SSSE3-NEXT:    movd %xmm0, %eax
 ; SSSE3-NEXT:    movw %ax, (%rdi)
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSSE3-NEXT:    movd %xmm0, %ecx
 ; SSSE3-NEXT:    movw %cx, 6(%rdi)
 ; SSSE3-NEXT:    movd %xmm2, %edx
diff --git a/llvm/test/CodeGen/X86/vec_ssubo.ll b/llvm/test/CodeGen/X86/vec_ssubo.ll
index 8ab9367c32f86..eb12f0dbcbff2 100644
--- a/llvm/test/CodeGen/X86/vec_ssubo.ll
+++ b/llvm/test/CodeGen/X86/vec_ssubo.ll
@@ -109,7 +109,7 @@ define <3 x i32> @ssubo_v3i32(<3 x i32> %a0, <3 x i32> %a1, <3 x i32>* %p2) noun
 ; SSE2-NEXT:    pcmpgtd %xmm3, %xmm0
 ; SSE2-NEXT:    pxor %xmm1, %xmm0
 ; SSE2-NEXT:    movq %xmm3, (%rdi)
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[2,3,2,3]
 ; SSE2-NEXT:    movd %xmm1, 8(%rdi)
 ; SSE2-NEXT:    retq
 ;
@@ -122,7 +122,7 @@ define <3 x i32> @ssubo_v3i32(<3 x i32> %a0, <3 x i32> %a1, <3 x i32>* %p2) noun
 ; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
 ; SSSE3-NEXT:    pxor %xmm1, %xmm0
 ; SSSE3-NEXT:    movq %xmm3, (%rdi)
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[2,3,0,1]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[2,3,2,3]
 ; SSSE3-NEXT:    movd %xmm1, 8(%rdi)
 ; SSSE3-NEXT:    retq
 ;
@@ -517,13 +517,13 @@ define <16 x i32> @ssubo_v16i32(<16 x i32> %a0, <16 x i32> %a1, <16 x i32>* %p2)
 ; AVX1-NEXT:    vpsubd %xmm2, %xmm0, %xmm2
 ; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpxor %xmm0, %xmm5, %xmm0
-; AVX1-NEXT:    vpackssdw %xmm4, %xmm0, %xmm4
-; AVX1-NEXT:    vpacksswb %xmm1, %xmm4, %xmm0
-; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm5
+; AVX1-NEXT:    vpackssdw %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm4
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm5, %ymm0
-; AVX1-NEXT:    vpacksswb %xmm4, %xmm1, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm4, %ymm0
+; AVX1-NEXT:    vpacksswb %xmm1, %xmm1, %xmm1
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm4
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
@@ -649,7 +649,7 @@ define <16 x i32> @ssubo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nou
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm1
 ; SSE41-NEXT:    psrad $31, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm2
 ; SSE41-NEXT:    psrad $31, %xmm2
@@ -672,9 +672,9 @@ define <16 x i32> @ssubo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nou
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm2
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
 ; AVX1-NEXT:    vmovdqa %xmm3, (%rdi)
@@ -688,7 +688,7 @@ define <16 x i32> @ssubo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nou
 ; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm1
 ; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm1
 ; AVX2-NEXT:    vmovdqa %xmm3, (%rdi)
 ; AVX2-NEXT:    retq
@@ -774,7 +774,7 @@ define <8 x i32> @ssubo_v8i16(<8 x i16> %a0, <8 x i16> %a1, <8 x i16>* %p2) noun
 ; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
 ; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
@@ -899,7 +899,7 @@ define <4 x i32> @ssubo_v4i24(<4 x i24> %a0, <4 x i24> %a1, <4 x i24>* %p2) noun
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[3,1,2,3]
 ; SSE2-NEXT:    movd %xmm1, %ecx
 ; SSE2-NEXT:    movw %cx, 9(%rdi)
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]
 ; SSE2-NEXT:    movd %xmm1, %edx
 ; SSE2-NEXT:    movw %dx, 6(%rdi)
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,2,3]
@@ -934,7 +934,7 @@ define <4 x i32> @ssubo_v4i24(<4 x i24> %a0, <4 x i24> %a1, <4 x i24>* %p2) noun
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[3,1,2,3]
 ; SSSE3-NEXT:    movd %xmm1, %ecx
 ; SSSE3-NEXT:    movw %cx, 9(%rdi)
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,3,0,1]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]
 ; SSSE3-NEXT:    movd %xmm1, %edx
 ; SSSE3-NEXT:    movw %dx, 6(%rdi)
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,2,3]
diff --git a/llvm/test/CodeGen/X86/vec_uaddo.ll b/llvm/test/CodeGen/X86/vec_uaddo.ll
index 864f0b59f453f..9ff793b6b677f 100644
--- a/llvm/test/CodeGen/X86/vec_uaddo.ll
+++ b/llvm/test/CodeGen/X86/vec_uaddo.ll
@@ -120,7 +120,7 @@ define <3 x i32> @uaddo_v3i32(<3 x i32> %a0, <3 x i32> %a1, <3 x i32>* %p2) noun
 ; SSE2-NEXT:    pxor %xmm1, %xmm2
 ; SSE2-NEXT:    pcmpgtd %xmm2, %xmm0
 ; SSE2-NEXT:    movq %xmm1, (%rdi)
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    movd %xmm1, 8(%rdi)
 ; SSE2-NEXT:    retq
 ;
@@ -132,7 +132,7 @@ define <3 x i32> @uaddo_v3i32(<3 x i32> %a0, <3 x i32> %a1, <3 x i32>* %p2) noun
 ; SSSE3-NEXT:    pxor %xmm1, %xmm2
 ; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm0
 ; SSSE3-NEXT:    movq %xmm1, (%rdi)
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSSE3-NEXT:    movd %xmm1, 8(%rdi)
 ; SSSE3-NEXT:    retq
 ;
@@ -601,13 +601,13 @@ define <16 x i32> @uaddo_v16i32(<16 x i32> %a0, <16 x i32> %a1, <16 x i32>* %p2)
 ; AVX1-NEXT:    vpmaxud %xmm0, %xmm2, %xmm0
 ; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm2, %xmm0
 ; AVX1-NEXT:    vpxor %xmm6, %xmm0, %xmm0
-; AVX1-NEXT:    vpackssdw %xmm7, %xmm0, %xmm6
-; AVX1-NEXT:    vpacksswb %xmm1, %xmm6, %xmm0
-; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm7
+; AVX1-NEXT:    vpackssdw %xmm7, %xmm0, %xmm0
+; AVX1-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm6
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm7, %ymm0
-; AVX1-NEXT:    vpacksswb %xmm6, %xmm1, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm6, %ymm0
+; AVX1-NEXT:    vpacksswb %xmm1, %xmm1, %xmm1
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm6
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
@@ -727,7 +727,7 @@ define <16 x i32> @uaddo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nou
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm4 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm4
 ; SSE41-NEXT:    psrad $31, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm2
 ; SSE41-NEXT:    psrad $31, %xmm2
@@ -750,9 +750,9 @@ define <16 x i32> @uaddo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nou
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
 ; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm3, %ymm1
 ; AVX1-NEXT:    vmovdqa %xmm2, (%rdi)
@@ -766,7 +766,7 @@ define <16 x i32> @uaddo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nou
 ; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm1
 ; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm1
 ; AVX2-NEXT:    vmovdqa %xmm2, (%rdi)
 ; AVX2-NEXT:    retq
@@ -850,7 +850,7 @@ define <8 x i32> @uaddo_v8i16(<8 x i16> %a0, <8 x i16> %a1, <8 x i16>* %p2) noun
 ; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
 ; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
@@ -956,7 +956,7 @@ define <4 x i32> @uaddo_v4i24(<4 x i24> %a0, <4 x i24> %a1, <4 x i24>* %p2) noun
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[3,1,2,3]
 ; SSE2-NEXT:    movd %xmm1, %ecx
 ; SSE2-NEXT:    movw %cx, 9(%rdi)
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]
 ; SSE2-NEXT:    movd %xmm1, %edx
 ; SSE2-NEXT:    movw %dx, 6(%rdi)
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,2,3]
@@ -988,7 +988,7 @@ define <4 x i32> @uaddo_v4i24(<4 x i24> %a0, <4 x i24> %a1, <4 x i24>* %p2) noun
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[3,1,2,3]
 ; SSSE3-NEXT:    movd %xmm1, %ecx
 ; SSSE3-NEXT:    movw %cx, 9(%rdi)
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,3,0,1]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]
 ; SSSE3-NEXT:    movd %xmm1, %edx
 ; SSSE3-NEXT:    movw %dx, 6(%rdi)
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,2,3]
diff --git a/llvm/test/CodeGen/X86/vec_umulo.ll b/llvm/test/CodeGen/X86/vec_umulo.ll
index e08bbc363721c..87fe4922dfcb0 100644
--- a/llvm/test/CodeGen/X86/vec_umulo.ll
+++ b/llvm/test/CodeGen/X86/vec_umulo.ll
@@ -1008,16 +1008,16 @@ define <16 x i32> @umulo_v16i32(<16 x i32> %a0, <16 x i32> %a1, <16 x i32>* %p2)
 ; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm8, %xmm5
 ; AVX1-NEXT:    vpxor %xmm5, %xmm9, %xmm5
 ; AVX1-NEXT:    vpackssdw %xmm13, %xmm5, %xmm5
-; AVX1-NEXT:    vpacksswb %xmm11, %xmm5, %xmm7
+; AVX1-NEXT:    vpacksswb %xmm11, %xmm5, %xmm5
 ; AVX1-NEXT:    vpmulld %xmm2, %xmm0, %xmm2
 ; AVX1-NEXT:    vpmulld %xmm6, %xmm4, %xmm4
 ; AVX1-NEXT:    vpmulld %xmm3, %xmm1, %xmm3
 ; AVX1-NEXT:    vpmulld %xmm10, %xmm12, %xmm6
-; AVX1-NEXT:    vpmovsxbd %xmm7, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm7[1,1,2,3]
+; AVX1-NEXT:    vpmovsxbd %xmm5, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm5[1,1,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX1-NEXT:    vpacksswb %xmm5, %xmm11, %xmm1
+; AVX1-NEXT:    vpacksswb %xmm11, %xmm11, %xmm1
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm5
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
@@ -1217,7 +1217,7 @@ define <16 x i32> @umulo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nou
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm1
 ; SSE41-NEXT:    psrad $31, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm2
 ; SSE41-NEXT:    psrad $31, %xmm2
@@ -1254,9 +1254,9 @@ define <16 x i32> @umulo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nou
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
 ; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm3, %ymm1
 ; AVX1-NEXT:    vmovdqa %xmm2, (%rdi)
@@ -1278,7 +1278,7 @@ define <16 x i32> @umulo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nou
 ; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm1
 ; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm1
 ; AVX2-NEXT:    vmovdqa %xmm2, (%rdi)
 ; AVX2-NEXT:    retq
@@ -1560,7 +1560,7 @@ define <32 x i32> @umulo_v32i8(<32 x i8> %a0, <32 x i8> %a1, <32 x i8>* %p2) nou
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm0
 ; SSE41-NEXT:    psrad $31, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm6[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm6[2,3,2,3]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm1
 ; SSE41-NEXT:    psrad $31, %xmm1
@@ -1572,7 +1572,7 @@ define <32 x i32> @umulo_v32i8(<32 x i8> %a0, <32 x i8> %a1, <32 x i8>* %p2) nou
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm3
 ; SSE41-NEXT:    psrad $31, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[2,3,2,3]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm4
 ; SSE41-NEXT:    psrad $31, %xmm4
@@ -1647,14 +1647,14 @@ define <32 x i32> @umulo_v32i8(<32 x i8> %a0, <32 x i8> %a1, <32 x i8>* %p2) nou
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm6[1,1,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
 ; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm3, %ymm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm6[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm6[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm6[3,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm6[3,3,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm4, %xmm4
 ; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
 ; AVX1-NEXT:    vmovdqa %xmm5, 16(%rdi)
@@ -1689,9 +1689,9 @@ define <32 x i32> @umulo_v32i8(<32 x i8> %a0, <32 x i8> %a1, <32 x i8>* %p2) nou
 ; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm3
 ; AVX2-NEXT:    vpmovsxbd %xmm3, %ymm2
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm1
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]
 ; AVX2-NEXT:    vpmovsxbd %xmm3, %ymm3
 ; AVX2-NEXT:    vmovdqa %ymm4, (%rdi)
 ; AVX2-NEXT:    retq
@@ -2230,7 +2230,7 @@ define <64 x i32> @umulo_v64i8(<64 x i8> %a0, <64 x i8> %a1, <64 x i8>* %p2) nou
 ; SSE41-NEXT:    pslld $31, %xmm0
 ; SSE41-NEXT:    psrad $31, %xmm0
 ; SSE41-NEXT:    movdqa %xmm0, (%rdi)
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm14[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm14[2,3,2,3]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm0
 ; SSE41-NEXT:    psrad $31, %xmm0
@@ -2245,7 +2245,7 @@ define <64 x i32> @umulo_v64i8(<64 x i8> %a0, <64 x i8> %a1, <64 x i8>* %p2) nou
 ; SSE41-NEXT:    pslld $31, %xmm0
 ; SSE41-NEXT:    psrad $31, %xmm0
 ; SSE41-NEXT:    movdqa %xmm0, 208(%rdi)
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm12[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm12[2,3,2,3]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm0
 ; SSE41-NEXT:    psrad $31, %xmm0
@@ -2260,7 +2260,7 @@ define <64 x i32> @umulo_v64i8(<64 x i8> %a0, <64 x i8> %a1, <64 x i8>* %p2) nou
 ; SSE41-NEXT:    pslld $31, %xmm0
 ; SSE41-NEXT:    psrad $31, %xmm0
 ; SSE41-NEXT:    movdqa %xmm0, 144(%rdi)
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm9[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm9[2,3,2,3]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm0
 ; SSE41-NEXT:    psrad $31, %xmm0
@@ -2275,7 +2275,7 @@ define <64 x i32> @umulo_v64i8(<64 x i8> %a0, <64 x i8> %a1, <64 x i8>* %p2) nou
 ; SSE41-NEXT:    pslld $31, %xmm0
 ; SSE41-NEXT:    psrad $31, %xmm0
 ; SSE41-NEXT:    movdqa %xmm0, 80(%rdi)
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm8[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm8[2,3,2,3]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm0
 ; SSE41-NEXT:    psrad $31, %xmm0
@@ -2390,37 +2390,37 @@ define <64 x i32> @umulo_v64i8(<64 x i8> %a0, <64 x i8> %a1, <64 x i8>* %p2) nou
 ; AVX1-NEXT:    vmovdqa %xmm4, 64(%rdi)
 ; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm4
 ; AVX1-NEXT:    vmovdqa %xmm4, (%rdi)
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm3[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm3[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm4, %xmm4
 ; AVX1-NEXT:    vmovdqa %xmm4, 224(%rdi)
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm3[3,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm3[3,3,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm4, %xmm4
 ; AVX1-NEXT:    vmovdqa %xmm4, 240(%rdi)
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
 ; AVX1-NEXT:    vmovdqa %xmm3, 208(%rdi)
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm2[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
 ; AVX1-NEXT:    vmovdqa %xmm3, 160(%rdi)
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm2[3,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm2[3,3,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
 ; AVX1-NEXT:    vmovdqa %xmm3, 176(%rdi)
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm2
 ; AVX1-NEXT:    vmovdqa %xmm2, 144(%rdi)
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm2
 ; AVX1-NEXT:    vmovdqa %xmm2, 96(%rdi)
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[3,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[3,3,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm2
 ; AVX1-NEXT:    vmovdqa %xmm2, 112(%rdi)
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
 ; AVX1-NEXT:    vmovdqa %xmm1, 80(%rdi)
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
 ; AVX1-NEXT:    vmovdqa %xmm1, 32(%rdi)
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[3,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[3,3,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
 ; AVX1-NEXT:    vmovdqa %xmm1, 48(%rdi)
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
@@ -2475,15 +2475,15 @@ define <64 x i32> @umulo_v64i8(<64 x i8> %a0, <64 x i8> %a1, <64 x i8>* %p2) nou
 ; AVX2-NEXT:    vpackuswb %ymm7, %ymm0, %ymm0
 ; AVX2-NEXT:    vpcmpeqb %ymm6, %ymm0, %ymm0
 ; AVX2-NEXT:    vpxor %ymm3, %ymm0, %ymm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpmovsxbd %xmm2, %ymm2
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm3
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm6 = xmm3[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm6 = xmm3[2,3,2,3]
 ; AVX2-NEXT:    vpmovsxbd %xmm6, %ymm6
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm7 = xmm1[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm7 = xmm1[2,3,2,3]
 ; AVX2-NEXT:    vpmovsxbd %xmm7, %ymm7
 ; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm4
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm5 = xmm4[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm5 = xmm4[2,3,2,3]
 ; AVX2-NEXT:    vpmovsxbd %xmm5, %ymm5
 ; AVX2-NEXT:    vpmovsxbd %xmm0, %ymm0
 ; AVX2-NEXT:    vpmovsxbd %xmm3, %ymm3
@@ -2608,7 +2608,7 @@ define <8 x i32> @umulo_v8i16(<8 x i16> %a0, <8 x i16> %a1, <8 x i16>* %p2) noun
 ; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    vmovdqa %xmm2, (%rdi)
@@ -2646,9 +2646,9 @@ define <8 x i32> @umulo_v8i16(<8 x i16> %a0, <8 x i16> %a1, <8 x i16>* %p2) noun
 define <2 x i32> @umulo_v2i64(<2 x i64> %a0, <2 x i64> %a1, <2 x i64>* %p2) nounwind {
 ; SSE2-LABEL: umulo_v2i64:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm2, %r8
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm2, %r10
 ; SSE2-NEXT:    movq %xmm0, %rax
 ; SSE2-NEXT:    movq %xmm1, %rdx
@@ -2672,9 +2672,9 @@ define <2 x i32> @umulo_v2i64(<2 x i64> %a0, <2 x i64> %a1, <2 x i64>* %p2) noun
 ;
 ; SSSE3-LABEL: umulo_v2i64:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; SSSE3-NEXT:    movq %xmm2, %r8
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; SSSE3-NEXT:    movq %xmm2, %r10
 ; SSSE3-NEXT:    movq %xmm0, %rax
 ; SSSE3-NEXT:    movq %xmm1, %rdx
@@ -2829,7 +2829,7 @@ define <4 x i32> @umulo_v4i24(<4 x i24> %a0, <4 x i24> %a1, <4 x i24>* %p2) noun
 ; SSE2-NEXT:    por %xmm3, %xmm0
 ; SSE2-NEXT:    movd %xmm2, %eax
 ; SSE2-NEXT:    movw %ax, (%rdi)
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
 ; SSE2-NEXT:    movd %xmm2, %ecx
 ; SSE2-NEXT:    movw %cx, 6(%rdi)
 ; SSE2-NEXT:    movd %xmm1, %edx
@@ -2873,7 +2873,7 @@ define <4 x i32> @umulo_v4i24(<4 x i24> %a0, <4 x i24> %a1, <4 x i24>* %p2) noun
 ; SSSE3-NEXT:    por %xmm3, %xmm0
 ; SSSE3-NEXT:    movd %xmm2, %eax
 ; SSSE3-NEXT:    movw %ax, (%rdi)
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
 ; SSSE3-NEXT:    movd %xmm2, %ecx
 ; SSSE3-NEXT:    movw %cx, 6(%rdi)
 ; SSSE3-NEXT:    movd %xmm1, %edx
diff --git a/llvm/test/CodeGen/X86/vec_usubo.ll b/llvm/test/CodeGen/X86/vec_usubo.ll
index 5302b3f8913df..155c5591ce11a 100644
--- a/llvm/test/CodeGen/X86/vec_usubo.ll
+++ b/llvm/test/CodeGen/X86/vec_usubo.ll
@@ -126,7 +126,7 @@ define <3 x i32> @usubo_v3i32(<3 x i32> %a0, <3 x i32> %a1, <3 x i32>* %p2) noun
 ; SSE2-NEXT:    pxor %xmm0, %xmm2
 ; SSE2-NEXT:    pcmpgtd %xmm3, %xmm2
 ; SSE2-NEXT:    movq %xmm0, (%rdi)
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    movd %xmm0, 8(%rdi)
 ; SSE2-NEXT:    movdqa %xmm2, %xmm0
 ; SSE2-NEXT:    retq
@@ -140,7 +140,7 @@ define <3 x i32> @usubo_v3i32(<3 x i32> %a0, <3 x i32> %a1, <3 x i32>* %p2) noun
 ; SSSE3-NEXT:    pxor %xmm0, %xmm2
 ; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm2
 ; SSSE3-NEXT:    movq %xmm0, (%rdi)
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSSE3-NEXT:    movd %xmm0, 8(%rdi)
 ; SSSE3-NEXT:    movdqa %xmm2, %xmm0
 ; SSSE3-NEXT:    retq
@@ -644,13 +644,13 @@ define <16 x i32> @usubo_v16i32(<16 x i32> %a0, <16 x i32> %a1, <16 x i32>* %p2)
 ; AVX1-NEXT:    vpminud %xmm0, %xmm2, %xmm0
 ; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm2, %xmm0
 ; AVX1-NEXT:    vpxor %xmm6, %xmm0, %xmm0
-; AVX1-NEXT:    vpackssdw %xmm7, %xmm0, %xmm6
-; AVX1-NEXT:    vpacksswb %xmm1, %xmm6, %xmm0
-; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm7
+; AVX1-NEXT:    vpackssdw %xmm7, %xmm0, %xmm0
+; AVX1-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm6
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm7, %ymm0
-; AVX1-NEXT:    vpacksswb %xmm6, %xmm1, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm6, %ymm0
+; AVX1-NEXT:    vpacksswb %xmm1, %xmm1, %xmm1
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm6
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
@@ -771,7 +771,7 @@ define <16 x i32> @usubo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nou
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm1
 ; SSE41-NEXT:    psrad $31, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm2
 ; SSE41-NEXT:    psrad $31, %xmm2
@@ -793,9 +793,9 @@ define <16 x i32> @usubo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nou
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
 ; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm3, %ymm1
 ; AVX1-NEXT:    vmovdqa %xmm2, (%rdi)
@@ -809,7 +809,7 @@ define <16 x i32> @usubo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nou
 ; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm1
 ; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm1
 ; AVX2-NEXT:    vmovdqa %xmm2, (%rdi)
 ; AVX2-NEXT:    retq
@@ -895,7 +895,7 @@ define <8 x i32> @usubo_v8i16(<8 x i16> %a0, <8 x i16> %a1, <8 x i16>* %p2) noun
 ; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
 ; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
@@ -1003,7 +1003,7 @@ define <4 x i32> @usubo_v4i24(<4 x i24> %a0, <4 x i24> %a1, <4 x i24>* %p2) noun
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[3,1,2,3]
 ; SSE2-NEXT:    movd %xmm1, %ecx
 ; SSE2-NEXT:    movw %cx, 9(%rdi)
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]
 ; SSE2-NEXT:    movd %xmm1, %edx
 ; SSE2-NEXT:    movw %dx, 6(%rdi)
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,2,3]
@@ -1035,7 +1035,7 @@ define <4 x i32> @usubo_v4i24(<4 x i24> %a0, <4 x i24> %a1, <4 x i24>* %p2) noun
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[3,1,2,3]
 ; SSSE3-NEXT:    movd %xmm1, %ecx
 ; SSSE3-NEXT:    movw %cx, 9(%rdi)
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,3,0,1]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]
 ; SSSE3-NEXT:    movd %xmm1, %edx
 ; SSSE3-NEXT:    movw %dx, 6(%rdi)
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,2,3]
diff --git a/llvm/test/CodeGen/X86/vector-constrained-fp-intrinsics.ll b/llvm/test/CodeGen/X86/vector-constrained-fp-intrinsics.ll
index 22e97e7407533..62bc377b9cec3 100644
--- a/llvm/test/CodeGen/X86/vector-constrained-fp-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/vector-constrained-fp-intrinsics.ll
@@ -6296,7 +6296,7 @@ define <2 x double> @constrained_vector_sitofp_v2f64_v2i64(<2 x i64> %x) #0 {
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    movq %xmm0, %rax
 ; CHECK-NEXT:    cvtsi2sd %rax, %xmm1
-; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; CHECK-NEXT:    movq %xmm0, %rax
 ; CHECK-NEXT:    xorps %xmm0, %xmm0
 ; CHECK-NEXT:    cvtsi2sd %rax, %xmm0
@@ -6342,7 +6342,7 @@ define <2 x float> @constrained_vector_sitofp_v2f32_v2i64(<2 x i64> %x) #0 {
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    movq %xmm0, %rax
 ; CHECK-NEXT:    cvtsi2ss %rax, %xmm1
-; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; CHECK-NEXT:    movq %xmm0, %rax
 ; CHECK-NEXT:    xorps %xmm0, %xmm0
 ; CHECK-NEXT:    cvtsi2ss %rax, %xmm0
@@ -6375,7 +6375,7 @@ define <3 x double> @constrained_vector_sitofp_v3f64_v3i32(<3 x i32> %x) #0 {
 ; CHECK-NEXT:    movd %xmm1, %eax
 ; CHECK-NEXT:    xorps %xmm1, %xmm1
 ; CHECK-NEXT:    cvtsi2sd %eax, %xmm1
-; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; CHECK-NEXT:    movd %xmm0, %eax
 ; CHECK-NEXT:    xorps %xmm0, %xmm0
 ; CHECK-NEXT:    cvtsi2sd %eax, %xmm0
@@ -6414,7 +6414,7 @@ define <3 x float> @constrained_vector_sitofp_v3f32_v3i32(<3 x i32> %x) #0 {
 ; CHECK-NEXT:    xorps %xmm2, %xmm2
 ; CHECK-NEXT:    cvtsi2ss %eax, %xmm2
 ; CHECK-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; CHECK-NEXT:    movd %xmm0, %eax
 ; CHECK-NEXT:    xorps %xmm0, %xmm0
 ; CHECK-NEXT:    cvtsi2ss %eax, %xmm0
@@ -6535,7 +6535,7 @@ define <4 x double> @constrained_vector_sitofp_v4f64_v4i32(<4 x i32> %x) #0 {
 ; CHECK-LABEL: constrained_vector_sitofp_v4f64_v4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    cvtdq2pd %xmm0, %xmm2
-; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; CHECK-NEXT:    cvtdq2pd %xmm0, %xmm1
 ; CHECK-NEXT:    movaps %xmm2, %xmm0
 ; CHECK-NEXT:    retq
@@ -6575,14 +6575,14 @@ define <4 x double> @constrained_vector_sitofp_v4f64_v4i64(<4 x i64> %x) #0 {
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    movq %xmm0, %rax
 ; CHECK-NEXT:    cvtsi2sd %rax, %xmm2
-; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; CHECK-NEXT:    movq %xmm0, %rax
 ; CHECK-NEXT:    xorps %xmm0, %xmm0
 ; CHECK-NEXT:    cvtsi2sd %rax, %xmm0
 ; CHECK-NEXT:    unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm0[0]
 ; CHECK-NEXT:    movq %xmm1, %rax
 ; CHECK-NEXT:    cvtsi2sd %rax, %xmm3
-; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; CHECK-NEXT:    movq %xmm0, %rax
 ; CHECK-NEXT:    xorps %xmm0, %xmm0
 ; CHECK-NEXT:    cvtsi2sd %rax, %xmm0
@@ -6642,7 +6642,7 @@ define <4 x float> @constrained_vector_sitofp_v4f32_v4i64(<4 x i64> %x) #0 {
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    movq %xmm1, %rax
 ; CHECK-NEXT:    cvtsi2ss %rax, %xmm2
-; CHECK-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; CHECK-NEXT:    movq %xmm1, %rax
 ; CHECK-NEXT:    xorps %xmm1, %xmm1
 ; CHECK-NEXT:    cvtsi2ss %rax, %xmm1
@@ -6650,7 +6650,7 @@ define <4 x float> @constrained_vector_sitofp_v4f32_v4i64(<4 x i64> %x) #0 {
 ; CHECK-NEXT:    movq %xmm0, %rax
 ; CHECK-NEXT:    xorps %xmm1, %xmm1
 ; CHECK-NEXT:    cvtsi2ss %rax, %xmm1
-; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; CHECK-NEXT:    movq %xmm0, %rax
 ; CHECK-NEXT:    xorps %xmm0, %xmm0
 ; CHECK-NEXT:    cvtsi2ss %rax, %xmm0
@@ -6970,7 +6970,7 @@ define <2 x float> @constrained_vector_uitofp_v2f32_v2i64(<2 x i64> %x) #0 {
 ; CHECK-NEXT:  # %bb.1:
 ; CHECK-NEXT:    addss %xmm0, %xmm0
 ; CHECK-NEXT:  .LBB174_2: # %entry
-; CHECK-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; CHECK-NEXT:    movq %xmm1, %rax
 ; CHECK-NEXT:    movq %rax, %rcx
 ; CHECK-NEXT:    shrq %rcx
@@ -7031,7 +7031,7 @@ define <3 x double> @constrained_vector_uitofp_v3f64_v3i32(<3 x i32> %x) #0 {
 ; CHECK-NEXT:    movd %xmm1, %eax
 ; CHECK-NEXT:    xorps %xmm1, %xmm1
 ; CHECK-NEXT:    cvtsi2sd %rax, %xmm1
-; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; CHECK-NEXT:    movd %xmm0, %eax
 ; CHECK-NEXT:    xorps %xmm0, %xmm0
 ; CHECK-NEXT:    cvtsi2sd %rax, %xmm0
@@ -7082,7 +7082,7 @@ define <3 x float> @constrained_vector_uitofp_v3f32_v3i32(<3 x i32> %x) #0 {
 ; CHECK-NEXT:    xorps %xmm2, %xmm2
 ; CHECK-NEXT:    cvtsi2ss %rax, %xmm2
 ; CHECK-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; CHECK-NEXT:    movd %xmm0, %eax
 ; CHECK-NEXT:    xorps %xmm0, %xmm0
 ; CHECK-NEXT:    cvtsi2ss %rax, %xmm0
@@ -7157,7 +7157,7 @@ define <3 x double> @constrained_vector_uitofp_v3f64_v3i64(<3 x i64> %x) #0 {
 ; AVX1-NEXT:    vsubpd %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vpermilpd {{.*#+}} xmm4 = xmm2[1,0]
 ; AVX1-NEXT:    vaddpd %xmm2, %xmm4, %xmm2
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm4 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm4 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vunpcklps {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
 ; AVX1-NEXT:    vsubpd %xmm3, %xmm4, %xmm4
 ; AVX1-NEXT:    vpermilpd {{.*#+}} xmm5 = xmm4[1,0]
@@ -7458,7 +7458,7 @@ define <4 x float> @constrained_vector_uitofp_v4f32_v4i64(<4 x i64> %x) #0 {
 ; CHECK-NEXT:  # %bb.1:
 ; CHECK-NEXT:    addss %xmm2, %xmm2
 ; CHECK-NEXT:  .LBB182_2: # %entry
-; CHECK-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; CHECK-NEXT:    movq %xmm1, %rax
 ; CHECK-NEXT:    movq %rax, %rcx
 ; CHECK-NEXT:    shrq %rcx
@@ -7487,7 +7487,7 @@ define <4 x float> @constrained_vector_uitofp_v4f32_v4i64(<4 x i64> %x) #0 {
 ; CHECK-NEXT:    addss %xmm1, %xmm1
 ; CHECK-NEXT:  .LBB182_6: # %entry
 ; CHECK-NEXT:    unpcklps {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; CHECK-NEXT:    movq %xmm0, %rax
 ; CHECK-NEXT:    movq %rax, %rcx
 ; CHECK-NEXT:    shrq %rcx
diff --git a/llvm/test/CodeGen/X86/vector-fshl-128.ll b/llvm/test/CodeGen/X86/vector-fshl-128.ll
index d8442048f65ec..0192d1e8137c1 100644
--- a/llvm/test/CodeGen/X86/vector-fshl-128.ll
+++ b/llvm/test/CodeGen/X86/vector-fshl-128.ll
@@ -30,7 +30,7 @@ define <2 x i64> @var_funnnel_v2i64(<2 x i64> %x, <2 x i64> %y, <2 x i64> %amt)
 ; SSE2-NEXT:    pand {{.*}}(%rip), %xmm2
 ; SSE2-NEXT:    movdqa %xmm0, %xmm3
 ; SSE2-NEXT:    psllq %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[2,3,2,3]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm5
 ; SSE2-NEXT:    psllq %xmm4, %xmm5
 ; SSE2-NEXT:    movsd {{.*#+}} xmm5 = xmm3[0],xmm5[1]
@@ -38,7 +38,7 @@ define <2 x i64> @var_funnnel_v2i64(<2 x i64> %x, <2 x i64> %y, <2 x i64> %amt)
 ; SSE2-NEXT:    psubq %xmm2, %xmm3
 ; SSE2-NEXT:    movdqa %xmm1, %xmm4
 ; SSE2-NEXT:    psrlq %xmm3, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]
 ; SSE2-NEXT:    psrlq %xmm3, %xmm1
 ; SSE2-NEXT:    movsd {{.*#+}} xmm1 = xmm4[0],xmm1[1]
 ; SSE2-NEXT:    orpd %xmm5, %xmm1
@@ -56,7 +56,7 @@ define <2 x i64> @var_funnnel_v2i64(<2 x i64> %x, <2 x i64> %y, <2 x i64> %amt)
 ; SSE41-NEXT:    movdqa %xmm0, %xmm3
 ; SSE41-NEXT:    pand {{.*}}(%rip), %xmm2
 ; SSE41-NEXT:    psllq %xmm2, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[2,3,2,3]
 ; SSE41-NEXT:    movdqa %xmm3, %xmm4
 ; SSE41-NEXT:    psllq %xmm5, %xmm4
 ; SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm0[0,1,2,3],xmm4[4,5,6,7]
@@ -64,7 +64,7 @@ define <2 x i64> @var_funnnel_v2i64(<2 x i64> %x, <2 x i64> %y, <2 x i64> %amt)
 ; SSE41-NEXT:    psubq %xmm2, %xmm0
 ; SSE41-NEXT:    movdqa %xmm1, %xmm5
 ; SSE41-NEXT:    psrlq %xmm0, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    psrlq %xmm0, %xmm1
 ; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm5[0,1,2,3],xmm1[4,5,6,7]
 ; SSE41-NEXT:    por %xmm1, %xmm4
@@ -78,13 +78,13 @@ define <2 x i64> @var_funnnel_v2i64(<2 x i64> %x, <2 x i64> %y, <2 x i64> %amt)
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
 ; AVX1-NEXT:    vpsllq %xmm2, %xmm0, %xmm3
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm2[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm2[2,3,2,3]
 ; AVX1-NEXT:    vpsllq %xmm4, %xmm0, %xmm4
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm4[4,5,6,7]
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [64,64]
 ; AVX1-NEXT:    vpsubq %xmm2, %xmm4, %xmm4
 ; AVX1-NEXT:    vpsrlq %xmm4, %xmm1, %xmm5
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[2,3,2,3]
 ; AVX1-NEXT:    vpsrlq %xmm4, %xmm1, %xmm1
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm5[0,1,2,3],xmm1[4,5,6,7]
 ; AVX1-NEXT:    vpor %xmm1, %xmm3, %xmm1
@@ -212,7 +212,7 @@ define <2 x i64> @var_funnnel_v2i64(<2 x i64> %x, <2 x i64> %y, <2 x i64> %amt)
 ; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm2
 ; X32-SSE-NEXT:    movdqa %xmm0, %xmm3
 ; X32-SSE-NEXT:    psllq %xmm2, %xmm3
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[2,3,0,1]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[2,3,2,3]
 ; X32-SSE-NEXT:    movdqa %xmm0, %xmm5
 ; X32-SSE-NEXT:    psllq %xmm4, %xmm5
 ; X32-SSE-NEXT:    movsd {{.*#+}} xmm5 = xmm3[0],xmm5[1]
@@ -220,7 +220,7 @@ define <2 x i64> @var_funnnel_v2i64(<2 x i64> %x, <2 x i64> %y, <2 x i64> %amt)
 ; X32-SSE-NEXT:    psubq %xmm2, %xmm3
 ; X32-SSE-NEXT:    movdqa %xmm1, %xmm4
 ; X32-SSE-NEXT:    psrlq %xmm3, %xmm4
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[2,3,0,1]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]
 ; X32-SSE-NEXT:    psrlq %xmm3, %xmm1
 ; X32-SSE-NEXT:    movsd {{.*#+}} xmm1 = xmm4[0],xmm1[1]
 ; X32-SSE-NEXT:    orpd %xmm5, %xmm1
@@ -249,7 +249,7 @@ define <4 x i32> @var_funnnel_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i32> %amt)
 ; SSE2-NEXT:    movdqa %xmm1, %xmm3
 ; SSE2-NEXT:    psrld %xmm6, %xmm3
 ; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[2,3,2,3]
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm5 = xmm4[2,3,3,3,4,5,6,7]
 ; SSE2-NEXT:    movdqa %xmm1, %xmm6
 ; SSE2-NEXT:    psrld %xmm5, %xmm6
@@ -285,7 +285,7 @@ define <4 x i32> @var_funnnel_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i32> %amt)
 ; SSE41-NEXT:    pshuflw {{.*#+}} xmm4 = xmm0[2,3,3,3,4,5,6,7]
 ; SSE41-NEXT:    movdqa %xmm1, %xmm5
 ; SSE41-NEXT:    psrld %xmm4, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pshuflw {{.*#+}} xmm6 = xmm4[2,3,3,3,4,5,6,7]
 ; SSE41-NEXT:    movdqa %xmm1, %xmm7
 ; SSE41-NEXT:    psrld %xmm6, %xmm7
@@ -465,7 +465,7 @@ define <4 x i32> @var_funnnel_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i32> %amt)
 ; X32-SSE-NEXT:    movdqa %xmm1, %xmm3
 ; X32-SSE-NEXT:    psrld %xmm6, %xmm3
 ; X32-SSE-NEXT:    punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[2,3,0,1]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[2,3,2,3]
 ; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm5 = xmm4[2,3,3,3,4,5,6,7]
 ; X32-SSE-NEXT:    movdqa %xmm1, %xmm6
 ; X32-SSE-NEXT:    psrld %xmm5, %xmm6
@@ -1366,7 +1366,7 @@ define <2 x i64> @splatvar_funnnel_v2i64(<2 x i64> %x, <2 x i64> %y, <2 x i64> %
 ; X32-SSE-NEXT:    psubq %xmm3, %xmm4
 ; X32-SSE-NEXT:    movdqa %xmm1, %xmm3
 ; X32-SSE-NEXT:    psrlq %xmm4, %xmm3
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[2,3,0,1]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[2,3,2,3]
 ; X32-SSE-NEXT:    psrlq %xmm4, %xmm1
 ; X32-SSE-NEXT:    movsd {{.*#+}} xmm1 = xmm3[0],xmm1[1]
 ; X32-SSE-NEXT:    movdqa %xmm0, %xmm3
diff --git a/llvm/test/CodeGen/X86/vector-fshl-256.ll b/llvm/test/CodeGen/X86/vector-fshl-256.ll
index 0688107ed5c0b..0cf4c172412a6 100644
--- a/llvm/test/CodeGen/X86/vector-fshl-256.ll
+++ b/llvm/test/CodeGen/X86/vector-fshl-256.ll
@@ -26,11 +26,11 @@ define <4 x i64> @var_funnnel_v4i64(<4 x i64> %x, <4 x i64> %y, <4 x i64> %amt)
 ; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm2, %ymm2
 ; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm4
 ; AVX1-NEXT:    vpsllq %xmm4, %xmm3, %xmm5
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm4[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm4[2,3,2,3]
 ; AVX1-NEXT:    vpsllq %xmm6, %xmm3, %xmm3
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm5[0,1,2,3],xmm3[4,5,6,7]
 ; AVX1-NEXT:    vpsllq %xmm2, %xmm0, %xmm5
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm2[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm2[2,3,2,3]
 ; AVX1-NEXT:    vpsllq %xmm6, %xmm0, %xmm6
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm5 = xmm5[0,1,2,3],xmm6[4,5,6,7]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm5, %ymm3
@@ -38,12 +38,12 @@ define <4 x i64> @var_funnnel_v4i64(<4 x i64> %x, <4 x i64> %y, <4 x i64> %amt)
 ; AVX1-NEXT:    vpsubq %xmm4, %xmm8, %xmm6
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm7
 ; AVX1-NEXT:    vpsrlq %xmm6, %xmm7, %xmm5
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[2,3,2,3]
 ; AVX1-NEXT:    vpsrlq %xmm6, %xmm7, %xmm6
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm5 = xmm5[0,1,2,3],xmm6[4,5,6,7]
 ; AVX1-NEXT:    vpsubq %xmm2, %xmm8, %xmm6
 ; AVX1-NEXT:    vpsrlq %xmm6, %xmm1, %xmm7
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[2,3,2,3]
 ; AVX1-NEXT:    vpsrlq %xmm6, %xmm1, %xmm1
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm7[0,1,2,3],xmm1[4,5,6,7]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm1, %ymm1
diff --git a/llvm/test/CodeGen/X86/vector-fshl-rot-128.ll b/llvm/test/CodeGen/X86/vector-fshl-rot-128.ll
index c560f99916bed..59bef3a97b1fa 100644
--- a/llvm/test/CodeGen/X86/vector-fshl-rot-128.ll
+++ b/llvm/test/CodeGen/X86/vector-fshl-rot-128.ll
@@ -31,14 +31,14 @@ define <2 x i64> @var_funnnel_v2i64(<2 x i64> %x, <2 x i64> %amt) nounwind {
 ; SSE2-NEXT:    pand %xmm2, %xmm1
 ; SSE2-NEXT:    movdqa %xmm0, %xmm4
 ; SSE2-NEXT:    psllq %xmm1, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm5
 ; SSE2-NEXT:    psllq %xmm1, %xmm5
 ; SSE2-NEXT:    movsd {{.*#+}} xmm5 = xmm4[0],xmm5[1]
 ; SSE2-NEXT:    pand %xmm2, %xmm3
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; SSE2-NEXT:    psrlq %xmm3, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]
 ; SSE2-NEXT:    psrlq %xmm2, %xmm0
 ; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
 ; SSE2-NEXT:    orpd %xmm5, %xmm0
@@ -52,14 +52,14 @@ define <2 x i64> @var_funnnel_v2i64(<2 x i64> %x, <2 x i64> %amt) nounwind {
 ; SSE41-NEXT:    pand %xmm2, %xmm1
 ; SSE41-NEXT:    movdqa %xmm0, %xmm4
 ; SSE41-NEXT:    psllq %xmm1, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE41-NEXT:    movdqa %xmm0, %xmm5
 ; SSE41-NEXT:    psllq %xmm1, %xmm5
 ; SSE41-NEXT:    pblendw {{.*#+}} xmm5 = xmm4[0,1,2,3],xmm5[4,5,6,7]
 ; SSE41-NEXT:    pand %xmm2, %xmm3
 ; SSE41-NEXT:    movdqa %xmm0, %xmm1
 ; SSE41-NEXT:    psrlq %xmm3, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]
 ; SSE41-NEXT:    psrlq %xmm2, %xmm0
 ; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7]
 ; SSE41-NEXT:    por %xmm5, %xmm0
@@ -70,14 +70,14 @@ define <2 x i64> @var_funnnel_v2i64(<2 x i64> %x, <2 x i64> %amt) nounwind {
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [63,63]
 ; AVX1-NEXT:    vpand %xmm2, %xmm1, %xmm3
 ; AVX1-NEXT:    vpsllq %xmm3, %xmm0, %xmm4
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]
 ; AVX1-NEXT:    vpsllq %xmm3, %xmm0, %xmm3
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0,1,2,3],xmm3[4,5,6,7]
 ; AVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
 ; AVX1-NEXT:    vpsubq %xmm1, %xmm4, %xmm1
 ; AVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm2[0,1,2,3],xmm0[4,5,6,7]
 ; AVX1-NEXT:    vpor %xmm0, %xmm3, %xmm0
@@ -136,14 +136,14 @@ define <2 x i64> @var_funnnel_v2i64(<2 x i64> %x, <2 x i64> %amt) nounwind {
 ; X32-SSE-NEXT:    pand %xmm2, %xmm1
 ; X32-SSE-NEXT:    movdqa %xmm0, %xmm4
 ; X32-SSE-NEXT:    psllq %xmm1, %xmm4
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; X32-SSE-NEXT:    movdqa %xmm0, %xmm5
 ; X32-SSE-NEXT:    psllq %xmm1, %xmm5
 ; X32-SSE-NEXT:    movsd {{.*#+}} xmm5 = xmm4[0],xmm5[1]
 ; X32-SSE-NEXT:    pand %xmm2, %xmm3
 ; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
 ; X32-SSE-NEXT:    psrlq %xmm3, %xmm1
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,0,1]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]
 ; X32-SSE-NEXT:    psrlq %xmm2, %xmm0
 ; X32-SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
 ; X32-SSE-NEXT:    orpd %xmm5, %xmm0
@@ -745,14 +745,14 @@ define <2 x i64> @splatvar_funnnel_v2i64(<2 x i64> %x, <2 x i64> %amt) nounwind
 ; X32-SSE-NEXT:    pand %xmm2, %xmm1
 ; X32-SSE-NEXT:    movdqa %xmm0, %xmm4
 ; X32-SSE-NEXT:    psllq %xmm1, %xmm4
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; X32-SSE-NEXT:    movdqa %xmm0, %xmm5
 ; X32-SSE-NEXT:    psllq %xmm1, %xmm5
 ; X32-SSE-NEXT:    movsd {{.*#+}} xmm5 = xmm4[0],xmm5[1]
 ; X32-SSE-NEXT:    pand %xmm2, %xmm3
 ; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
 ; X32-SSE-NEXT:    psrlq %xmm3, %xmm1
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,0,1]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]
 ; X32-SSE-NEXT:    psrlq %xmm2, %xmm0
 ; X32-SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
 ; X32-SSE-NEXT:    orpd %xmm5, %xmm0
diff --git a/llvm/test/CodeGen/X86/vector-fshl-rot-256.ll b/llvm/test/CodeGen/X86/vector-fshl-rot-256.ll
index 655b6e4c25048..fd0e1c7e2f3ab 100644
--- a/llvm/test/CodeGen/X86/vector-fshl-rot-256.ll
+++ b/llvm/test/CodeGen/X86/vector-fshl-rot-256.ll
@@ -24,11 +24,11 @@ define <4 x i64> @var_funnnel_v4i64(<4 x i64> %x, <4 x i64> %amt) nounwind {
 ; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm1, %ymm3
 ; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
 ; AVX1-NEXT:    vpsllq %xmm4, %xmm2, %xmm5
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[2,3,2,3]
 ; AVX1-NEXT:    vpsllq %xmm4, %xmm2, %xmm4
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm4 = xmm5[0,1,2,3],xmm4[4,5,6,7]
 ; AVX1-NEXT:    vpsllq %xmm3, %xmm0, %xmm5
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]
 ; AVX1-NEXT:    vpsllq %xmm3, %xmm0, %xmm3
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm5[0,1,2,3],xmm3[4,5,6,7]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
@@ -38,13 +38,13 @@ define <4 x i64> @var_funnnel_v4i64(<4 x i64> %x, <4 x i64> %amt) nounwind {
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [63,63]
 ; AVX1-NEXT:    vpand %xmm6, %xmm4, %xmm4
 ; AVX1-NEXT:    vpsrlq %xmm4, %xmm2, %xmm7
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[2,3,2,3]
 ; AVX1-NEXT:    vpsrlq %xmm4, %xmm2, %xmm2
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm7[0,1,2,3],xmm2[4,5,6,7]
 ; AVX1-NEXT:    vpsubq %xmm1, %xmm5, %xmm1
 ; AVX1-NEXT:    vpand %xmm6, %xmm1, %xmm1
 ; AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm4
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm4[0,1,2,3],xmm0[4,5,6,7]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
@@ -520,7 +520,7 @@ define <4 x i64> @splatvar_funnnel_v4i64(<4 x i64> %x, <4 x i64> %amt) nounwind
 ; AVX1-NEXT:    vpand %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
 ; AVX1-NEXT:    vpsrlq %xmm2, %xmm4, %xmm5
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm2[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm2[2,3,2,3]
 ; AVX1-NEXT:    vpsrlq %xmm6, %xmm4, %xmm7
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm5 = xmm5[0,1,2,3],xmm7[4,5,6,7]
 ; AVX1-NEXT:    vpsrlq %xmm2, %xmm0, %xmm2
diff --git a/llvm/test/CodeGen/X86/vector-fshr-128.ll b/llvm/test/CodeGen/X86/vector-fshr-128.ll
index b7cc39a32d718..817bca051e0ae 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-128.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-128.ll
@@ -30,7 +30,7 @@ define <2 x i64> @var_funnnel_v2i64(<2 x i64> %x, <2 x i64> %y, <2 x i64> %amt)
 ; SSE2-NEXT:    pand {{.*}}(%rip), %xmm2
 ; SSE2-NEXT:    movdqa %xmm1, %xmm3
 ; SSE2-NEXT:    psrlq %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[2,3,2,3]
 ; SSE2-NEXT:    movdqa %xmm1, %xmm5
 ; SSE2-NEXT:    psrlq %xmm4, %xmm5
 ; SSE2-NEXT:    movsd {{.*#+}} xmm5 = xmm3[0],xmm5[1]
@@ -38,7 +38,7 @@ define <2 x i64> @var_funnnel_v2i64(<2 x i64> %x, <2 x i64> %y, <2 x i64> %amt)
 ; SSE2-NEXT:    psubq %xmm2, %xmm3
 ; SSE2-NEXT:    movdqa %xmm0, %xmm4
 ; SSE2-NEXT:    psllq %xmm3, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]
 ; SSE2-NEXT:    psllq %xmm3, %xmm0
 ; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm4[0],xmm0[1]
 ; SSE2-NEXT:    orpd %xmm5, %xmm0
@@ -58,7 +58,7 @@ define <2 x i64> @var_funnnel_v2i64(<2 x i64> %x, <2 x i64> %y, <2 x i64> %amt)
 ; SSE41-NEXT:    pand {{.*}}(%rip), %xmm2
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
 ; SSE41-NEXT:    psrlq %xmm2, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[2,3,2,3]
 ; SSE41-NEXT:    movdqa %xmm1, %xmm5
 ; SSE41-NEXT:    psrlq %xmm4, %xmm5
 ; SSE41-NEXT:    pblendw {{.*#+}} xmm5 = xmm0[0,1,2,3],xmm5[4,5,6,7]
@@ -66,7 +66,7 @@ define <2 x i64> @var_funnnel_v2i64(<2 x i64> %x, <2 x i64> %y, <2 x i64> %amt)
 ; SSE41-NEXT:    psubq %xmm2, %xmm0
 ; SSE41-NEXT:    movdqa %xmm3, %xmm4
 ; SSE41-NEXT:    psllq %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    psllq %xmm0, %xmm3
 ; SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm4[0,1,2,3],xmm3[4,5,6,7]
 ; SSE41-NEXT:    por %xmm5, %xmm3
@@ -80,13 +80,13 @@ define <2 x i64> @var_funnnel_v2i64(<2 x i64> %x, <2 x i64> %y, <2 x i64> %amt)
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
 ; AVX1-NEXT:    vpsrlq %xmm2, %xmm1, %xmm3
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm2[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm2[2,3,2,3]
 ; AVX1-NEXT:    vpsrlq %xmm4, %xmm1, %xmm4
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm4[4,5,6,7]
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [64,64]
 ; AVX1-NEXT:    vpsubq %xmm2, %xmm4, %xmm4
 ; AVX1-NEXT:    vpsllq %xmm4, %xmm0, %xmm5
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[2,3,2,3]
 ; AVX1-NEXT:    vpsllq %xmm4, %xmm0, %xmm0
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm5[0,1,2,3],xmm0[4,5,6,7]
 ; AVX1-NEXT:    vpor %xmm3, %xmm0, %xmm0
@@ -215,7 +215,7 @@ define <2 x i64> @var_funnnel_v2i64(<2 x i64> %x, <2 x i64> %y, <2 x i64> %amt)
 ; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm2
 ; X32-SSE-NEXT:    movdqa %xmm1, %xmm3
 ; X32-SSE-NEXT:    psrlq %xmm2, %xmm3
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[2,3,0,1]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[2,3,2,3]
 ; X32-SSE-NEXT:    movdqa %xmm1, %xmm5
 ; X32-SSE-NEXT:    psrlq %xmm4, %xmm5
 ; X32-SSE-NEXT:    movsd {{.*#+}} xmm5 = xmm3[0],xmm5[1]
@@ -223,7 +223,7 @@ define <2 x i64> @var_funnnel_v2i64(<2 x i64> %x, <2 x i64> %y, <2 x i64> %amt)
 ; X32-SSE-NEXT:    psubq %xmm2, %xmm3
 ; X32-SSE-NEXT:    movdqa %xmm0, %xmm4
 ; X32-SSE-NEXT:    psllq %xmm3, %xmm4
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[2,3,0,1]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]
 ; X32-SSE-NEXT:    psllq %xmm3, %xmm0
 ; X32-SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm4[0],xmm0[1]
 ; X32-SSE-NEXT:    orpd %xmm5, %xmm0
@@ -251,7 +251,7 @@ define <4 x i32> @var_funnnel_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i32> %amt)
 ; SSE2-NEXT:    movdqa %xmm1, %xmm3
 ; SSE2-NEXT:    psrld %xmm5, %xmm3
 ; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm4[0]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[2,3,2,3]
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm5 = xmm4[2,3,3,3,4,5,6,7]
 ; SSE2-NEXT:    movdqa %xmm1, %xmm6
 ; SSE2-NEXT:    psrld %xmm5, %xmm6
@@ -287,7 +287,7 @@ define <4 x i32> @var_funnnel_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i32> %amt)
 ; SSE41-NEXT:    pshuflw {{.*#+}} xmm0 = xmm2[2,3,3,3,4,5,6,7]
 ; SSE41-NEXT:    movdqa %xmm1, %xmm4
 ; SSE41-NEXT:    psrld %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE41-NEXT:    pshuflw {{.*#+}} xmm5 = xmm0[2,3,3,3,4,5,6,7]
 ; SSE41-NEXT:    movdqa %xmm1, %xmm6
 ; SSE41-NEXT:    psrld %xmm5, %xmm6
@@ -469,7 +469,7 @@ define <4 x i32> @var_funnnel_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i32> %amt)
 ; X32-SSE-NEXT:    movdqa %xmm1, %xmm3
 ; X32-SSE-NEXT:    psrld %xmm5, %xmm3
 ; X32-SSE-NEXT:    punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm4[0]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[2,3,0,1]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[2,3,2,3]
 ; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm5 = xmm4[2,3,3,3,4,5,6,7]
 ; X32-SSE-NEXT:    movdqa %xmm1, %xmm6
 ; X32-SSE-NEXT:    psrld %xmm5, %xmm6
@@ -1380,7 +1380,7 @@ define <2 x i64> @splatvar_funnnel_v2i64(<2 x i64> %x, <2 x i64> %y, <2 x i64> %
 ; X32-SSE-NEXT:    psubq %xmm4, %xmm5
 ; X32-SSE-NEXT:    movdqa %xmm0, %xmm4
 ; X32-SSE-NEXT:    psllq %xmm5, %xmm4
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[2,3,0,1]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[2,3,2,3]
 ; X32-SSE-NEXT:    psllq %xmm5, %xmm0
 ; X32-SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm4[0],xmm0[1]
 ; X32-SSE-NEXT:    movdqa %xmm1, %xmm4
diff --git a/llvm/test/CodeGen/X86/vector-fshr-256.ll b/llvm/test/CodeGen/X86/vector-fshr-256.ll
index e9cb0a0586f09..f0848cfd2e49a 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-256.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-256.ll
@@ -26,11 +26,11 @@ define <4 x i64> @var_funnnel_v4i64(<4 x i64> %x, <4 x i64> %y, <4 x i64> %amt)
 ; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm2, %ymm2
 ; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm4
 ; AVX1-NEXT:    vpsrlq %xmm4, %xmm3, %xmm5
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm4[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm4[2,3,2,3]
 ; AVX1-NEXT:    vpsrlq %xmm6, %xmm3, %xmm3
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm5[0,1,2,3],xmm3[4,5,6,7]
 ; AVX1-NEXT:    vpsrlq %xmm2, %xmm1, %xmm5
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm2[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm2[2,3,2,3]
 ; AVX1-NEXT:    vpsrlq %xmm6, %xmm1, %xmm6
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm5 = xmm5[0,1,2,3],xmm6[4,5,6,7]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm5, %ymm3
@@ -38,12 +38,12 @@ define <4 x i64> @var_funnnel_v4i64(<4 x i64> %x, <4 x i64> %y, <4 x i64> %amt)
 ; AVX1-NEXT:    vpsubq %xmm4, %xmm8, %xmm6
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm7
 ; AVX1-NEXT:    vpsllq %xmm6, %xmm7, %xmm5
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[2,3,2,3]
 ; AVX1-NEXT:    vpsllq %xmm6, %xmm7, %xmm6
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm5 = xmm5[0,1,2,3],xmm6[4,5,6,7]
 ; AVX1-NEXT:    vpsubq %xmm2, %xmm8, %xmm6
 ; AVX1-NEXT:    vpsllq %xmm6, %xmm0, %xmm7
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[2,3,2,3]
 ; AVX1-NEXT:    vpsllq %xmm6, %xmm0, %xmm0
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm7[0,1,2,3],xmm0[4,5,6,7]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm0, %ymm0
diff --git a/llvm/test/CodeGen/X86/vector-fshr-rot-128.ll b/llvm/test/CodeGen/X86/vector-fshr-rot-128.ll
index eadc8544f8afb..8fe7ba9e471a9 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-rot-128.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-rot-128.ll
@@ -31,14 +31,14 @@ define <2 x i64> @var_funnnel_v2i64(<2 x i64> %x, <2 x i64> %amt) nounwind {
 ; SSE2-NEXT:    pand %xmm2, %xmm1
 ; SSE2-NEXT:    movdqa %xmm0, %xmm4
 ; SSE2-NEXT:    psrlq %xmm1, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm5
 ; SSE2-NEXT:    psrlq %xmm1, %xmm5
 ; SSE2-NEXT:    movsd {{.*#+}} xmm5 = xmm4[0],xmm5[1]
 ; SSE2-NEXT:    pand %xmm2, %xmm3
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; SSE2-NEXT:    psllq %xmm3, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]
 ; SSE2-NEXT:    psllq %xmm2, %xmm0
 ; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
 ; SSE2-NEXT:    orpd %xmm5, %xmm0
@@ -52,14 +52,14 @@ define <2 x i64> @var_funnnel_v2i64(<2 x i64> %x, <2 x i64> %amt) nounwind {
 ; SSE41-NEXT:    pand %xmm2, %xmm1
 ; SSE41-NEXT:    movdqa %xmm0, %xmm4
 ; SSE41-NEXT:    psrlq %xmm1, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE41-NEXT:    movdqa %xmm0, %xmm5
 ; SSE41-NEXT:    psrlq %xmm1, %xmm5
 ; SSE41-NEXT:    pblendw {{.*#+}} xmm5 = xmm4[0,1,2,3],xmm5[4,5,6,7]
 ; SSE41-NEXT:    pand %xmm2, %xmm3
 ; SSE41-NEXT:    movdqa %xmm0, %xmm1
 ; SSE41-NEXT:    psllq %xmm3, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]
 ; SSE41-NEXT:    psllq %xmm2, %xmm0
 ; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7]
 ; SSE41-NEXT:    por %xmm5, %xmm0
@@ -70,14 +70,14 @@ define <2 x i64> @var_funnnel_v2i64(<2 x i64> %x, <2 x i64> %amt) nounwind {
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [63,63]
 ; AVX1-NEXT:    vpand %xmm2, %xmm1, %xmm3
 ; AVX1-NEXT:    vpsrlq %xmm3, %xmm0, %xmm4
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]
 ; AVX1-NEXT:    vpsrlq %xmm3, %xmm0, %xmm3
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0,1,2,3],xmm3[4,5,6,7]
 ; AVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
 ; AVX1-NEXT:    vpsubq %xmm1, %xmm4, %xmm1
 ; AVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vpsllq %xmm1, %xmm0, %xmm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; AVX1-NEXT:    vpsllq %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm2[0,1,2,3],xmm0[4,5,6,7]
 ; AVX1-NEXT:    vpor %xmm3, %xmm0, %xmm0
@@ -138,14 +138,14 @@ define <2 x i64> @var_funnnel_v2i64(<2 x i64> %x, <2 x i64> %amt) nounwind {
 ; X32-SSE-NEXT:    pand %xmm2, %xmm1
 ; X32-SSE-NEXT:    movdqa %xmm0, %xmm4
 ; X32-SSE-NEXT:    psrlq %xmm1, %xmm4
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; X32-SSE-NEXT:    movdqa %xmm0, %xmm5
 ; X32-SSE-NEXT:    psrlq %xmm1, %xmm5
 ; X32-SSE-NEXT:    movsd {{.*#+}} xmm5 = xmm4[0],xmm5[1]
 ; X32-SSE-NEXT:    pand %xmm2, %xmm3
 ; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
 ; X32-SSE-NEXT:    psllq %xmm3, %xmm1
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,0,1]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]
 ; X32-SSE-NEXT:    psllq %xmm2, %xmm0
 ; X32-SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
 ; X32-SSE-NEXT:    orpd %xmm5, %xmm0
@@ -789,14 +789,14 @@ define <2 x i64> @splatvar_funnnel_v2i64(<2 x i64> %x, <2 x i64> %amt) nounwind
 ; X32-SSE-NEXT:    pand %xmm2, %xmm1
 ; X32-SSE-NEXT:    movdqa %xmm0, %xmm4
 ; X32-SSE-NEXT:    psrlq %xmm1, %xmm4
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; X32-SSE-NEXT:    movdqa %xmm0, %xmm5
 ; X32-SSE-NEXT:    psrlq %xmm1, %xmm5
 ; X32-SSE-NEXT:    movsd {{.*#+}} xmm5 = xmm4[0],xmm5[1]
 ; X32-SSE-NEXT:    pand %xmm2, %xmm3
 ; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
 ; X32-SSE-NEXT:    psllq %xmm3, %xmm1
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,0,1]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]
 ; X32-SSE-NEXT:    psllq %xmm2, %xmm0
 ; X32-SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
 ; X32-SSE-NEXT:    orpd %xmm5, %xmm0
diff --git a/llvm/test/CodeGen/X86/vector-fshr-rot-256.ll b/llvm/test/CodeGen/X86/vector-fshr-rot-256.ll
index 61c45a118e47a..4e92bfc4f9136 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-rot-256.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-rot-256.ll
@@ -24,11 +24,11 @@ define <4 x i64> @var_funnnel_v4i64(<4 x i64> %x, <4 x i64> %amt) nounwind {
 ; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm1, %ymm3
 ; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
 ; AVX1-NEXT:    vpsrlq %xmm4, %xmm2, %xmm5
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[2,3,2,3]
 ; AVX1-NEXT:    vpsrlq %xmm4, %xmm2, %xmm4
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm4 = xmm5[0,1,2,3],xmm4[4,5,6,7]
 ; AVX1-NEXT:    vpsrlq %xmm3, %xmm0, %xmm5
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]
 ; AVX1-NEXT:    vpsrlq %xmm3, %xmm0, %xmm3
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm5[0,1,2,3],xmm3[4,5,6,7]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
@@ -38,13 +38,13 @@ define <4 x i64> @var_funnnel_v4i64(<4 x i64> %x, <4 x i64> %amt) nounwind {
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [63,63]
 ; AVX1-NEXT:    vpand %xmm6, %xmm4, %xmm4
 ; AVX1-NEXT:    vpsllq %xmm4, %xmm2, %xmm7
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[2,3,2,3]
 ; AVX1-NEXT:    vpsllq %xmm4, %xmm2, %xmm2
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm7[0,1,2,3],xmm2[4,5,6,7]
 ; AVX1-NEXT:    vpsubq %xmm1, %xmm5, %xmm1
 ; AVX1-NEXT:    vpand %xmm6, %xmm1, %xmm1
 ; AVX1-NEXT:    vpsllq %xmm1, %xmm0, %xmm4
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; AVX1-NEXT:    vpsllq %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm4[0,1,2,3],xmm0[4,5,6,7]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
@@ -566,7 +566,7 @@ define <4 x i64> @splatvar_funnnel_v4i64(<4 x i64> %x, <4 x i64> %amt) nounwind
 ; AVX1-NEXT:    vpand %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
 ; AVX1-NEXT:    vpsllq %xmm2, %xmm4, %xmm5
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm2[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm2[2,3,2,3]
 ; AVX1-NEXT:    vpsllq %xmm6, %xmm4, %xmm7
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm5 = xmm5[0,1,2,3],xmm7[4,5,6,7]
 ; AVX1-NEXT:    vpsllq %xmm2, %xmm0, %xmm2
diff --git a/llvm/test/CodeGen/X86/vector-idiv-sdiv-128.ll b/llvm/test/CodeGen/X86/vector-idiv-sdiv-128.ll
index ac8e1998ceb88..22a3895687356 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-sdiv-128.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-sdiv-128.ll
@@ -20,7 +20,7 @@ define <2 x i64> @test_div7_2i64(<2 x i64> %a) nounwind {
 ; SSE2-NEXT:    sarq %rdx
 ; SSE2-NEXT:    addq %rax, %rdx
 ; SSE2-NEXT:    movq %rdx, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm0, %rax
 ; SSE2-NEXT:    imulq %rcx
 ; SSE2-NEXT:    movq %rdx, %rax
@@ -199,7 +199,7 @@ define <16 x i8> @test_div7_16i8(<16 x i8> %a) nounwind {
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [65427,65427,65427,65427,65427,65427,65427,65427]
 ; SSE41-NEXT:    pmullw %xmm2, %xmm1
 ; SSE41-NEXT:    psrlw $8, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovsxbw %xmm3, %xmm3
 ; SSE41-NEXT:    pmullw %xmm2, %xmm3
 ; SSE41-NEXT:    psrlw $8, %xmm3
@@ -223,7 +223,7 @@ define <16 x i8> @test_div7_16i8(<16 x i8> %a) nounwind {
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [65427,65427,65427,65427,65427,65427,65427,65427]
 ; AVX1-NEXT:    vpmullw %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vpsrlw $8, %xmm1, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbw %xmm3, %xmm3
 ; AVX1-NEXT:    vpmullw %xmm2, %xmm3, %xmm2
 ; AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
@@ -327,7 +327,7 @@ define <16 x i8> @test_divconstant_16i8(<16 x i8> %a) nounwind {
 ; SSE41-NEXT:    movdqa %xmm0, %xmm1
 ; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; SSE41-NEXT:    pmovsxbw %xmm0, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
 ; SSE41-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; SSE41-NEXT:    pmullw {{.*}}(%rip), %xmm1
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
@@ -371,7 +371,7 @@ define <16 x i8> @test_divconstant_16i8(<16 x i8> %a) nounwind {
 ; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm2
 ; AVX1-NEXT:    vpmullw {{.*}}(%rip), %xmm2, %xmm2
 ; AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm0
 ; AVX1-NEXT:    vpmullw {{.*}}(%rip), %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm0
@@ -457,7 +457,7 @@ define <2 x i64> @test_rem7_2i64(<2 x i64> %a) nounwind {
 ; SSE2-NEXT:    subq %rax, %rdx
 ; SSE2-NEXT:    addq %rcx, %rdx
 ; SSE2-NEXT:    movq %rdx, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm0, %rcx
 ; SSE2-NEXT:    movq %rcx, %rax
 ; SSE2-NEXT:    imulq %rsi
@@ -674,7 +674,7 @@ define <16 x i8> @test_rem7_16i8(<16 x i8> %a) nounwind {
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [65427,65427,65427,65427,65427,65427,65427,65427]
 ; SSE41-NEXT:    pmullw %xmm2, %xmm1
 ; SSE41-NEXT:    psrlw $8, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovsxbw %xmm3, %xmm3
 ; SSE41-NEXT:    pmullw %xmm2, %xmm3
 ; SSE41-NEXT:    psrlw $8, %xmm3
@@ -702,7 +702,7 @@ define <16 x i8> @test_rem7_16i8(<16 x i8> %a) nounwind {
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [65427,65427,65427,65427,65427,65427,65427,65427]
 ; AVX1-NEXT:    vpmullw %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vpsrlw $8, %xmm1, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbw %xmm3, %xmm3
 ; AVX1-NEXT:    vpmullw %xmm2, %xmm3, %xmm2
 ; AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
@@ -836,7 +836,7 @@ define <16 x i8> @test_remconstant_16i8(<16 x i8> %a) nounwind {
 ; SSE41-NEXT:    pmovsxbw %xmm0, %xmm2
 ; SSE41-NEXT:    pmullw {{.*}}(%rip), %xmm2
 ; SSE41-NEXT:    psrlw $8, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovsxbw %xmm4, %xmm4
 ; SSE41-NEXT:    pmullw {{.*}}(%rip), %xmm4
 ; SSE41-NEXT:    psrlw $8, %xmm4
@@ -879,7 +879,7 @@ define <16 x i8> @test_remconstant_16i8(<16 x i8> %a) nounwind {
 ; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm3
 ; AVX1-NEXT:    vpmullw {{.*}}(%rip), %xmm3, %xmm3
 ; AVX1-NEXT:    vpsrlw $8, %xmm3, %xmm3
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbw %xmm4, %xmm4
 ; AVX1-NEXT:    vpmullw {{.*}}(%rip), %xmm4, %xmm4
 ; AVX1-NEXT:    vpsrlw $8, %xmm4, %xmm4
diff --git a/llvm/test/CodeGen/X86/vector-idiv-sdiv-256.ll b/llvm/test/CodeGen/X86/vector-idiv-sdiv-256.ll
index a9cf1aa80af8d..e06140f988d54 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-sdiv-256.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-sdiv-256.ll
@@ -163,7 +163,7 @@ define <32 x i8> @test_div7_32i8(<32 x i8> %a) nounwind {
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [65427,65427,65427,65427,65427,65427,65427,65427]
 ; AVX1-NEXT:    vpmullw %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm1[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm1[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbw %xmm4, %xmm4
 ; AVX1-NEXT:    vpmullw %xmm3, %xmm4, %xmm4
 ; AVX1-NEXT:    vpsrlw $8, %xmm4, %xmm4
@@ -182,7 +182,7 @@ define <32 x i8> @test_div7_32i8(<32 x i8> %a) nounwind {
 ; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm2
 ; AVX1-NEXT:    vpmullw %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbw %xmm7, %xmm7
 ; AVX1-NEXT:    vpmullw %xmm3, %xmm7, %xmm3
 ; AVX1-NEXT:    vpsrlw $8, %xmm3, %xmm3
@@ -260,7 +260,7 @@ define <32 x i8> @test_divconstant_32i8(<32 x i8> %a) nounwind {
 ; AVX1-NEXT:    vpmovsxbw %xmm2, %xmm4
 ; AVX1-NEXT:    vpmullw {{.*}}(%rip), %xmm4, %xmm4
 ; AVX1-NEXT:    vpsrlw $8, %xmm4, %xmm4
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbw %xmm2, %xmm2
 ; AVX1-NEXT:    vpmullw {{.*}}(%rip), %xmm2, %xmm2
 ; AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
@@ -289,7 +289,7 @@ define <32 x i8> @test_divconstant_32i8(<32 x i8> %a) nounwind {
 ; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm3
 ; AVX1-NEXT:    vpmullw {{.*}}(%rip), %xmm3, %xmm3
 ; AVX1-NEXT:    vpsrlw $8, %xmm3, %xmm3
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm0
 ; AVX1-NEXT:    vpmullw {{.*}}(%rip), %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm0
@@ -574,7 +574,7 @@ define <32 x i8> @test_rem7_32i8(<32 x i8> %a) nounwind {
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [65427,65427,65427,65427,65427,65427,65427,65427]
 ; AVX1-NEXT:    vpmullw %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm1[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm1[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbw %xmm4, %xmm4
 ; AVX1-NEXT:    vpmullw %xmm3, %xmm4, %xmm4
 ; AVX1-NEXT:    vpsrlw $8, %xmm4, %xmm4
@@ -598,7 +598,7 @@ define <32 x i8> @test_rem7_32i8(<32 x i8> %a) nounwind {
 ; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm2
 ; AVX1-NEXT:    vpmullw %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbw %xmm4, %xmm4
 ; AVX1-NEXT:    vpmullw %xmm3, %xmm4, %xmm3
 ; AVX1-NEXT:    vpsrlw $8, %xmm3, %xmm3
@@ -688,7 +688,7 @@ define <32 x i8> @test_remconstant_32i8(<32 x i8> %a) nounwind {
 ; AVX1-NEXT:    vpmovsxbw %xmm2, %xmm4
 ; AVX1-NEXT:    vpmullw {{.*}}(%rip), %xmm4, %xmm4
 ; AVX1-NEXT:    vpsrlw $8, %xmm4, %xmm4
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm2[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm2[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbw %xmm5, %xmm5
 ; AVX1-NEXT:    vpmullw {{.*}}(%rip), %xmm5, %xmm5
 ; AVX1-NEXT:    vpsrlw $8, %xmm5, %xmm5
@@ -725,7 +725,7 @@ define <32 x i8> @test_remconstant_32i8(<32 x i8> %a) nounwind {
 ; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm5
 ; AVX1-NEXT:    vpmullw {{.*}}(%rip), %xmm5, %xmm5
 ; AVX1-NEXT:    vpsrlw $8, %xmm5, %xmm5
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbw %xmm6, %xmm6
 ; AVX1-NEXT:    vpmullw {{.*}}(%rip), %xmm6, %xmm6
 ; AVX1-NEXT:    vpsrlw $8, %xmm6, %xmm6
diff --git a/llvm/test/CodeGen/X86/vector-idiv-udiv-128.ll b/llvm/test/CodeGen/X86/vector-idiv-udiv-128.ll
index 16be83c84fb3c..65131c0e3cb73 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-udiv-128.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-udiv-128.ll
@@ -20,7 +20,7 @@ define <2 x i64> @test_div7_2i64(<2 x i64> %a) nounwind {
 ; SSE2-NEXT:    shrq %rcx
 ; SSE2-NEXT:    addq %rdx, %rcx
 ; SSE2-NEXT:    movq %rcx, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm0, %rcx
 ; SSE2-NEXT:    movq %rcx, %rax
 ; SSE2-NEXT:    mulq %rsi
@@ -445,7 +445,7 @@ define <2 x i64> @test_rem7_2i64(<2 x i64> %a) nounwind {
 ; SSE2-NEXT:    subq %rdx, %rax
 ; SSE2-NEXT:    addq %rcx, %rax
 ; SSE2-NEXT:    movq %rax, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    movq %xmm0, %rcx
 ; SSE2-NEXT:    movq %rcx, %rax
 ; SSE2-NEXT:    mulq %rsi
diff --git a/llvm/test/CodeGen/X86/vector-pcmp.ll b/llvm/test/CodeGen/X86/vector-pcmp.ll
index 2900ce2c8661e..724f6007623db 100644
--- a/llvm/test/CodeGen/X86/vector-pcmp.ll
+++ b/llvm/test/CodeGen/X86/vector-pcmp.ll
@@ -452,7 +452,7 @@ define <8 x i32> @cmpne_knownzeros_zext_v8i16_v8i32(<8 x i16> %x) {
 ; SSE42:       # %bb.0:
 ; SSE42-NEXT:    psrlw $15, %xmm0
 ; SSE42-NEXT:    pmovzxwd {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE42-NEXT:    pmovzxwd {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; SSE42-NEXT:    movdqa %xmm2, %xmm0
 ; SSE42-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/vector-reduce-add.ll b/llvm/test/CodeGen/X86/vector-reduce-add.ll
index e4f785dca2b1a..a00e74fa1cacc 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-add.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-add.ll
@@ -14,21 +14,21 @@
 define i64 @test_v2i64(<2 x i64> %a0) {
 ; SSE-LABEL: test_v2i64:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    paddq %xmm0, %xmm1
 ; SSE-NEXT:    movq %xmm1, %rax
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: test_v2i64:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovq %xmm0, %rax
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: test_v2i64:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovq %xmm0, %rax
 ; AVX512-NEXT:    retq
@@ -40,7 +40,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; SSE-LABEL: test_v4i64:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    paddq %xmm1, %xmm0
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    paddq %xmm0, %xmm1
 ; SSE-NEXT:    movq %xmm1, %rax
 ; SSE-NEXT:    retq
@@ -49,7 +49,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovq %xmm0, %rax
 ; AVX1-NEXT:    vzeroupper
@@ -59,7 +59,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovq %xmm0, %rax
 ; AVX2-NEXT:    vzeroupper
@@ -69,7 +69,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovq %xmm0, %rax
 ; AVX512-NEXT:    vzeroupper
@@ -84,7 +84,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; SSE-NEXT:    paddq %xmm3, %xmm1
 ; SSE-NEXT:    paddq %xmm2, %xmm1
 ; SSE-NEXT:    paddq %xmm0, %xmm1
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE-NEXT:    paddq %xmm1, %xmm0
 ; SSE-NEXT:    movq %xmm0, %rax
 ; SSE-NEXT:    retq
@@ -96,7 +96,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; AVX1-NEXT:    vpaddq %xmm2, %xmm3, %xmm2
 ; AVX1-NEXT:    vpaddq %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovq %xmm0, %rax
 ; AVX1-NEXT:    vzeroupper
@@ -107,7 +107,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; AVX2-NEXT:    vpaddq %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovq %xmm0, %rax
 ; AVX2-NEXT:    vzeroupper
@@ -119,7 +119,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; AVX512-NEXT:    vpaddq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovq %xmm0, %rax
 ; AVX512-NEXT:    vzeroupper
@@ -138,7 +138,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; SSE-NEXT:    paddq %xmm4, %xmm2
 ; SSE-NEXT:    paddq %xmm3, %xmm2
 ; SSE-NEXT:    paddq %xmm0, %xmm2
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE-NEXT:    paddq %xmm2, %xmm0
 ; SSE-NEXT:    movq %xmm0, %rax
 ; SSE-NEXT:    retq
@@ -156,7 +156,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; AVX1-NEXT:    vpaddq %xmm4, %xmm2, %xmm2
 ; AVX1-NEXT:    vpaddq %xmm1, %xmm2, %xmm1
 ; AVX1-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovq %xmm0, %rax
 ; AVX1-NEXT:    vzeroupper
@@ -169,7 +169,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; AVX2-NEXT:    vpaddq %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovq %xmm0, %rax
 ; AVX2-NEXT:    vzeroupper
@@ -182,7 +182,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; AVX512-NEXT:    vpaddq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovq %xmm0, %rax
 ; AVX512-NEXT:    vzeroupper
@@ -236,7 +236,7 @@ define i32 @test_v2i32(<2 x i32> %a0) {
 define i32 @test_v4i32(<4 x i32> %a0) {
 ; SSE-LABEL: test_v4i32:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    paddd %xmm0, %xmm1
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE-NEXT:    paddd %xmm1, %xmm0
@@ -245,7 +245,7 @@ define i32 @test_v4i32(<4 x i32> %a0) {
 ;
 ; AVX1-SLOW-LABEL: test_v4i32:
 ; AVX1-SLOW:       # %bb.0:
-; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -261,7 +261,7 @@ define i32 @test_v4i32(<4 x i32> %a0) {
 ;
 ; AVX2-LABEL: test_v4i32:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -270,7 +270,7 @@ define i32 @test_v4i32(<4 x i32> %a0) {
 ;
 ; AVX512-LABEL: test_v4i32:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -284,7 +284,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; SSE-LABEL: test_v8i32:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    paddd %xmm1, %xmm0
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    paddd %xmm0, %xmm1
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE-NEXT:    paddd %xmm1, %xmm0
@@ -295,7 +295,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX1-SLOW:       # %bb.0:
 ; AVX1-SLOW-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -317,7 +317,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -329,7 +329,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -346,7 +346,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; SSE-NEXT:    paddd %xmm3, %xmm1
 ; SSE-NEXT:    paddd %xmm2, %xmm1
 ; SSE-NEXT:    paddd %xmm0, %xmm1
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE-NEXT:    paddd %xmm1, %xmm0
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE-NEXT:    paddd %xmm0, %xmm1
@@ -360,7 +360,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX1-SLOW-NEXT:    vpaddd %xmm2, %xmm3, %xmm2
 ; AVX1-SLOW-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
 ; AVX1-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -375,7 +375,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX1-FAST-NEXT:    vpaddd %xmm2, %xmm3, %xmm2
 ; AVX1-FAST-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
 ; AVX1-FAST-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-FAST-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX1-FAST-NEXT:    vphaddd %xmm0, %xmm0, %xmm0
 ; AVX1-FAST-NEXT:    vmovd %xmm0, %eax
@@ -387,7 +387,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -401,7 +401,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -422,7 +422,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; SSE-NEXT:    paddd %xmm4, %xmm2
 ; SSE-NEXT:    paddd %xmm3, %xmm2
 ; SSE-NEXT:    paddd %xmm0, %xmm2
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE-NEXT:    paddd %xmm2, %xmm0
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE-NEXT:    paddd %xmm0, %xmm1
@@ -442,7 +442,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX1-SLOW-NEXT:    vpaddd %xmm4, %xmm2, %xmm2
 ; AVX1-SLOW-NEXT:    vpaddd %xmm1, %xmm2, %xmm1
 ; AVX1-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -463,7 +463,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX1-FAST-NEXT:    vpaddd %xmm4, %xmm2, %xmm2
 ; AVX1-FAST-NEXT:    vpaddd %xmm1, %xmm2, %xmm1
 ; AVX1-FAST-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-FAST-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX1-FAST-NEXT:    vphaddd %xmm0, %xmm0, %xmm0
 ; AVX1-FAST-NEXT:    vmovd %xmm0, %eax
@@ -477,7 +477,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -492,7 +492,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
@@ -608,7 +608,7 @@ define i16 @test_v4i16(<4 x i16> %a0) {
 define i16 @test_v8i16(<8 x i16> %a0) {
 ; SSE-LABEL: test_v8i16:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    paddw %xmm0, %xmm1
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE-NEXT:    paddw %xmm1, %xmm0
@@ -621,7 +621,7 @@ define i16 @test_v8i16(<8 x i16> %a0) {
 ;
 ; AVX1-SLOW-LABEL: test_v8i16:
 ; AVX1-SLOW:       # %bb.0:
-; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-SLOW-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-SLOW-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
@@ -642,7 +642,7 @@ define i16 @test_v8i16(<8 x i16> %a0) {
 ;
 ; AVX2-LABEL: test_v8i16:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
@@ -654,7 +654,7 @@ define i16 @test_v8i16(<8 x i16> %a0) {
 ;
 ; AVX512-LABEL: test_v8i16:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
@@ -671,7 +671,7 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; SSE-LABEL: test_v16i16:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    paddw %xmm1, %xmm0
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    paddw %xmm0, %xmm1
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE-NEXT:    paddw %xmm1, %xmm0
@@ -686,7 +686,7 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX1-SLOW:       # %bb.0:
 ; AVX1-SLOW-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-SLOW-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
-; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-SLOW-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-SLOW-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
@@ -713,7 +713,7 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
@@ -728,7 +728,7 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
@@ -748,7 +748,7 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; SSE-NEXT:    paddw %xmm3, %xmm1
 ; SSE-NEXT:    paddw %xmm2, %xmm1
 ; SSE-NEXT:    paddw %xmm0, %xmm1
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE-NEXT:    paddw %xmm1, %xmm0
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE-NEXT:    paddw %xmm0, %xmm1
@@ -766,7 +766,7 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX1-SLOW-NEXT:    vpaddw %xmm2, %xmm3, %xmm2
 ; AVX1-SLOW-NEXT:    vpaddw %xmm2, %xmm1, %xmm1
 ; AVX1-SLOW-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
-; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-SLOW-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-SLOW-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
@@ -784,7 +784,7 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX1-FAST-NEXT:    vpaddw %xmm2, %xmm3, %xmm2
 ; AVX1-FAST-NEXT:    vpaddw %xmm2, %xmm1, %xmm1
 ; AVX1-FAST-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
-; AVX1-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-FAST-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX1-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-FAST-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
@@ -799,7 +799,7 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX2-NEXT:    vpaddw %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
@@ -816,7 +816,7 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX512-NEXT:    vpaddw %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
@@ -840,7 +840,7 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; SSE-NEXT:    paddw %xmm4, %xmm2
 ; SSE-NEXT:    paddw %xmm3, %xmm2
 ; SSE-NEXT:    paddw %xmm0, %xmm2
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE-NEXT:    paddw %xmm2, %xmm0
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE-NEXT:    paddw %xmm0, %xmm1
@@ -864,7 +864,7 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX1-SLOW-NEXT:    vpaddw %xmm4, %xmm2, %xmm2
 ; AVX1-SLOW-NEXT:    vpaddw %xmm1, %xmm2, %xmm1
 ; AVX1-SLOW-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
-; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-SLOW-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-SLOW-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
@@ -888,7 +888,7 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX1-FAST-NEXT:    vpaddw %xmm4, %xmm2, %xmm2
 ; AVX1-FAST-NEXT:    vpaddw %xmm1, %xmm2, %xmm1
 ; AVX1-FAST-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
-; AVX1-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-FAST-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX1-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-FAST-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
@@ -905,7 +905,7 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX2-NEXT:    vpaddw %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
@@ -923,7 +923,7 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX512-NEXT:    vpaddw %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
@@ -1142,7 +1142,7 @@ define i8 @test_v8i8_load(<8 x i8>* %p) {
 define i8 @test_v16i8(<16 x i8> %a0) {
 ; SSE-LABEL: test_v16i8:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    paddb %xmm0, %xmm1
 ; SSE-NEXT:    pxor %xmm0, %xmm0
 ; SSE-NEXT:    psadbw %xmm1, %xmm0
@@ -1152,7 +1152,7 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ;
 ; AVX-LABEL: test_v16i8:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
@@ -1162,7 +1162,7 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ;
 ; AVX512-LABEL: test_v16i8:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX512-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
@@ -1177,7 +1177,7 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; SSE-LABEL: test_v32i8:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    paddb %xmm1, %xmm0
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    paddb %xmm0, %xmm1
 ; SSE-NEXT:    pxor %xmm0, %xmm0
 ; SSE-NEXT:    psadbw %xmm1, %xmm0
@@ -1189,7 +1189,7 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX1-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
@@ -1202,7 +1202,7 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX2-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
@@ -1215,7 +1215,7 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX512-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
@@ -1233,7 +1233,7 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; SSE-NEXT:    paddb %xmm3, %xmm1
 ; SSE-NEXT:    paddb %xmm2, %xmm1
 ; SSE-NEXT:    paddb %xmm0, %xmm1
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE-NEXT:    paddb %xmm1, %xmm0
 ; SSE-NEXT:    pxor %xmm1, %xmm1
 ; SSE-NEXT:    psadbw %xmm0, %xmm1
@@ -1248,7 +1248,7 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX1-NEXT:    vpaddb %xmm2, %xmm3, %xmm2
 ; AVX1-NEXT:    vpaddb %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX1-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
@@ -1262,7 +1262,7 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX2-NEXT:    vpaddb %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX2-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
@@ -1277,7 +1277,7 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX512-NEXT:    vpaddb %ymm1, %ymm0, %ymm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX512-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
@@ -1299,7 +1299,7 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; SSE-NEXT:    paddb %xmm4, %xmm2
 ; SSE-NEXT:    paddb %xmm3, %xmm2
 ; SSE-NEXT:    paddb %xmm0, %xmm2
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE-NEXT:    paddb %xmm2, %xmm0
 ; SSE-NEXT:    pxor %xmm1, %xmm1
 ; SSE-NEXT:    psadbw %xmm0, %xmm1
@@ -1320,7 +1320,7 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX1-NEXT:    vpaddb %xmm1, %xmm2, %xmm1
 ; AVX1-NEXT:    vpaddb %xmm4, %xmm1, %xmm1
 ; AVX1-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX1-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
@@ -1336,7 +1336,7 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX2-NEXT:    vpaddb %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX2-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
@@ -1352,7 +1352,7 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX512-NEXT:    vpaddb %ymm1, %ymm0, %ymm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX512-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
diff --git a/llvm/test/CodeGen/X86/vector-reduce-and-cmp.ll b/llvm/test/CodeGen/X86/vector-reduce-and-cmp.ll
index c94f96958f5b0..17a3d6f46e98c 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-and-cmp.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-and-cmp.ll
@@ -13,7 +13,7 @@
 define i1 @test_v2i64(<2 x i64> %a0) {
 ; SSE-LABEL: test_v2i64:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    pand %xmm0, %xmm1
 ; SSE-NEXT:    movq %xmm1, %rax
 ; SSE-NEXT:    testq %rax, %rax
@@ -22,7 +22,7 @@ define i1 @test_v2i64(<2 x i64> %a0) {
 ;
 ; AVX-LABEL: test_v2i64:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovq %xmm0, %rax
 ; AVX-NEXT:    testq %rax, %rax
@@ -37,7 +37,7 @@ define i1 @test_v4i64(<4 x i64> %a0) {
 ; SSE-LABEL: test_v4i64:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pand %xmm1, %xmm0
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    pand %xmm0, %xmm1
 ; SSE-NEXT:    movq %xmm1, %rax
 ; SSE-NEXT:    testq %rax, %rax
@@ -48,7 +48,7 @@ define i1 @test_v4i64(<4 x i64> %a0) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovq %xmm0, %rax
 ; AVX1-NEXT:    testq %rax, %rax
@@ -60,7 +60,7 @@ define i1 @test_v4i64(<4 x i64> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovq %xmm0, %rax
 ; AVX2-NEXT:    testq %rax, %rax
@@ -72,7 +72,7 @@ define i1 @test_v4i64(<4 x i64> %a0) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovq %xmm0, %rax
 ; AVX512-NEXT:    testq %rax, %rax
@@ -90,7 +90,7 @@ define i1 @test_v8i64(<8 x i64> %a0) {
 ; SSE-NEXT:    pand %xmm3, %xmm1
 ; SSE-NEXT:    pand %xmm2, %xmm1
 ; SSE-NEXT:    pand %xmm0, %xmm1
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE-NEXT:    pand %xmm1, %xmm0
 ; SSE-NEXT:    movq %xmm0, %rax
 ; SSE-NEXT:    testq %rax, %rax
@@ -102,7 +102,7 @@ define i1 @test_v8i64(<8 x i64> %a0) {
 ; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovq %xmm0, %rax
 ; AVX1-NEXT:    testq %rax, %rax
@@ -115,7 +115,7 @@ define i1 @test_v8i64(<8 x i64> %a0) {
 ; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovq %xmm0, %rax
 ; AVX2-NEXT:    testq %rax, %rax
@@ -129,7 +129,7 @@ define i1 @test_v8i64(<8 x i64> %a0) {
 ; AVX512-NEXT:    vpandq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovq %xmm0, %rax
 ; AVX512-NEXT:    testq %rax, %rax
@@ -151,7 +151,7 @@ define i1 @test_v16i64(<16 x i64> %a0) {
 ; SSE-NEXT:    pand %xmm4, %xmm2
 ; SSE-NEXT:    pand %xmm3, %xmm2
 ; SSE-NEXT:    pand %xmm0, %xmm2
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE-NEXT:    pand %xmm2, %xmm0
 ; SSE-NEXT:    movq %xmm0, %rax
 ; SSE-NEXT:    testq %rax, %rax
@@ -165,7 +165,7 @@ define i1 @test_v16i64(<16 x i64> %a0) {
 ; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovq %xmm0, %rax
 ; AVX1-NEXT:    testq %rax, %rax
@@ -180,7 +180,7 @@ define i1 @test_v16i64(<16 x i64> %a0) {
 ; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovq %xmm0, %rax
 ; AVX2-NEXT:    testq %rax, %rax
@@ -195,7 +195,7 @@ define i1 @test_v16i64(<16 x i64> %a0) {
 ; AVX512-NEXT:    vpandq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovq %xmm0, %rax
 ; AVX512-NEXT:    testq %rax, %rax
@@ -237,7 +237,7 @@ define i1 @test_v2i32(<2 x i32> %a0) {
 define i1 @test_v4i32(<4 x i32> %a0) {
 ; SSE-LABEL: test_v4i32:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    pand %xmm0, %xmm1
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE-NEXT:    pand %xmm1, %xmm0
@@ -248,7 +248,7 @@ define i1 @test_v4i32(<4 x i32> %a0) {
 ;
 ; AVX-LABEL: test_v4i32:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -265,7 +265,7 @@ define i1 @test_v8i32(<8 x i32> %a0) {
 ; SSE-LABEL: test_v8i32:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pand %xmm1, %xmm0
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    pand %xmm0, %xmm1
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE-NEXT:    pand %xmm1, %xmm0
@@ -278,7 +278,7 @@ define i1 @test_v8i32(<8 x i32> %a0) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -292,7 +292,7 @@ define i1 @test_v8i32(<8 x i32> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -306,7 +306,7 @@ define i1 @test_v8i32(<8 x i32> %a0) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -326,7 +326,7 @@ define i1 @test_v16i32(<16 x i32> %a0) {
 ; SSE-NEXT:    pand %xmm3, %xmm1
 ; SSE-NEXT:    pand %xmm2, %xmm1
 ; SSE-NEXT:    pand %xmm0, %xmm1
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE-NEXT:    pand %xmm1, %xmm0
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE-NEXT:    pand %xmm0, %xmm1
@@ -340,7 +340,7 @@ define i1 @test_v16i32(<16 x i32> %a0) {
 ; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
@@ -355,7 +355,7 @@ define i1 @test_v16i32(<16 x i32> %a0) {
 ; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -371,7 +371,7 @@ define i1 @test_v16i32(<16 x i32> %a0) {
 ; AVX512-NEXT:    vpandd %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -395,7 +395,7 @@ define i1 @test_v32i32(<32 x i32> %a0) {
 ; SSE-NEXT:    pand %xmm4, %xmm2
 ; SSE-NEXT:    pand %xmm3, %xmm2
 ; SSE-NEXT:    pand %xmm0, %xmm2
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE-NEXT:    pand %xmm2, %xmm0
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE-NEXT:    pand %xmm0, %xmm1
@@ -411,7 +411,7 @@ define i1 @test_v32i32(<32 x i32> %a0) {
 ; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
@@ -428,7 +428,7 @@ define i1 @test_v32i32(<32 x i32> %a0) {
 ; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -445,7 +445,7 @@ define i1 @test_v32i32(<32 x i32> %a0) {
 ; AVX512-NEXT:    vpandd %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -518,7 +518,7 @@ define i1 @test_v4i16(<4 x i16> %a0) {
 define i1 @test_v8i16(<8 x i16> %a0) {
 ; SSE-LABEL: test_v8i16:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    pand %xmm0, %xmm1
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE-NEXT:    pand %xmm1, %xmm0
@@ -532,7 +532,7 @@ define i1 @test_v8i16(<8 x i16> %a0) {
 ;
 ; AVX-LABEL: test_v8i16:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -551,7 +551,7 @@ define i1 @test_v16i16(<16 x i16> %a0) {
 ; SSE-LABEL: test_v16i16:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pand %xmm1, %xmm0
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    pand %xmm0, %xmm1
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE-NEXT:    pand %xmm1, %xmm0
@@ -567,7 +567,7 @@ define i1 @test_v16i16(<16 x i16> %a0) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -583,7 +583,7 @@ define i1 @test_v16i16(<16 x i16> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -599,7 +599,7 @@ define i1 @test_v16i16(<16 x i16> %a0) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -621,7 +621,7 @@ define i1 @test_v32i16(<32 x i16> %a0) {
 ; SSE-NEXT:    pand %xmm3, %xmm1
 ; SSE-NEXT:    pand %xmm2, %xmm1
 ; SSE-NEXT:    pand %xmm0, %xmm1
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE-NEXT:    pand %xmm1, %xmm0
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE-NEXT:    pand %xmm0, %xmm1
@@ -638,7 +638,7 @@ define i1 @test_v32i16(<32 x i16> %a0) {
 ; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
@@ -655,7 +655,7 @@ define i1 @test_v32i16(<32 x i16> %a0) {
 ; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -673,7 +673,7 @@ define i1 @test_v32i16(<32 x i16> %a0) {
 ; AVX512-NEXT:    vpandq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -699,7 +699,7 @@ define i1 @test_v64i16(<64 x i16> %a0) {
 ; SSE-NEXT:    pand %xmm4, %xmm2
 ; SSE-NEXT:    pand %xmm3, %xmm2
 ; SSE-NEXT:    pand %xmm0, %xmm2
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE-NEXT:    pand %xmm2, %xmm0
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE-NEXT:    pand %xmm0, %xmm1
@@ -718,7 +718,7 @@ define i1 @test_v64i16(<64 x i16> %a0) {
 ; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
@@ -737,7 +737,7 @@ define i1 @test_v64i16(<64 x i16> %a0) {
 ; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -756,7 +756,7 @@ define i1 @test_v64i16(<64 x i16> %a0) {
 ; AVX512-NEXT:    vpandq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -865,7 +865,7 @@ define i1 @test_v8i8(<8 x i8> %a0) {
 define i1 @test_v16i8(<16 x i8> %a0) {
 ; SSE-LABEL: test_v16i8:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    pand %xmm0, %xmm1
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE-NEXT:    pand %xmm1, %xmm0
@@ -882,7 +882,7 @@ define i1 @test_v16i8(<16 x i8> %a0) {
 ;
 ; AVX-LABEL: test_v16i8:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -903,7 +903,7 @@ define i1 @test_v32i8(<32 x i8> %a0) {
 ; SSE-LABEL: test_v32i8:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pand %xmm1, %xmm0
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    pand %xmm0, %xmm1
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE-NEXT:    pand %xmm1, %xmm0
@@ -922,7 +922,7 @@ define i1 @test_v32i8(<32 x i8> %a0) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -940,7 +940,7 @@ define i1 @test_v32i8(<32 x i8> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -958,7 +958,7 @@ define i1 @test_v32i8(<32 x i8> %a0) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -982,7 +982,7 @@ define i1 @test_v64i8(<64 x i8> %a0) {
 ; SSE-NEXT:    pand %xmm3, %xmm1
 ; SSE-NEXT:    pand %xmm2, %xmm1
 ; SSE-NEXT:    pand %xmm0, %xmm1
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE-NEXT:    pand %xmm1, %xmm0
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE-NEXT:    pand %xmm0, %xmm1
@@ -1002,7 +1002,7 @@ define i1 @test_v64i8(<64 x i8> %a0) {
 ; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
@@ -1021,7 +1021,7 @@ define i1 @test_v64i8(<64 x i8> %a0) {
 ; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -1041,7 +1041,7 @@ define i1 @test_v64i8(<64 x i8> %a0) {
 ; AVX512-NEXT:    vpandq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -1069,7 +1069,7 @@ define i1 @test_v128i8(<128 x i8> %a0) {
 ; SSE-NEXT:    pand %xmm4, %xmm2
 ; SSE-NEXT:    pand %xmm3, %xmm2
 ; SSE-NEXT:    pand %xmm0, %xmm2
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE-NEXT:    pand %xmm2, %xmm0
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE-NEXT:    pand %xmm0, %xmm1
@@ -1091,7 +1091,7 @@ define i1 @test_v128i8(<128 x i8> %a0) {
 ; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
@@ -1112,7 +1112,7 @@ define i1 @test_v128i8(<128 x i8> %a0) {
 ; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -1133,7 +1133,7 @@ define i1 @test_v128i8(<128 x i8> %a0) {
 ; AVX512-NEXT:    vpandq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
diff --git a/llvm/test/CodeGen/X86/vector-reduce-and.ll b/llvm/test/CodeGen/X86/vector-reduce-and.ll
index 2fc924b1b1257..4a00c22a26701 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-and.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-and.ll
@@ -13,14 +13,14 @@
 define i64 @test_v2i64(<2 x i64> %a0) {
 ; SSE-LABEL: test_v2i64:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    pand %xmm0, %xmm1
 ; SSE-NEXT:    movq %xmm1, %rax
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: test_v2i64:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovq %xmm0, %rax
 ; AVX-NEXT:    retq
@@ -32,7 +32,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; SSE-LABEL: test_v4i64:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pand %xmm1, %xmm0
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    pand %xmm0, %xmm1
 ; SSE-NEXT:    movq %xmm1, %rax
 ; SSE-NEXT:    retq
@@ -41,7 +41,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovq %xmm0, %rax
 ; AVX1-NEXT:    vzeroupper
@@ -51,7 +51,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovq %xmm0, %rax
 ; AVX2-NEXT:    vzeroupper
@@ -61,7 +61,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovq %xmm0, %rax
 ; AVX512-NEXT:    vzeroupper
@@ -76,7 +76,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; SSE-NEXT:    pand %xmm3, %xmm1
 ; SSE-NEXT:    pand %xmm2, %xmm1
 ; SSE-NEXT:    pand %xmm0, %xmm1
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE-NEXT:    pand %xmm1, %xmm0
 ; SSE-NEXT:    movq %xmm0, %rax
 ; SSE-NEXT:    retq
@@ -86,7 +86,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovq %xmm0, %rax
 ; AVX1-NEXT:    vzeroupper
@@ -97,7 +97,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovq %xmm0, %rax
 ; AVX2-NEXT:    vzeroupper
@@ -109,7 +109,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; AVX512-NEXT:    vpandq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovq %xmm0, %rax
 ; AVX512-NEXT:    vzeroupper
@@ -128,7 +128,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; SSE-NEXT:    pand %xmm4, %xmm2
 ; SSE-NEXT:    pand %xmm3, %xmm2
 ; SSE-NEXT:    pand %xmm0, %xmm2
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE-NEXT:    pand %xmm2, %xmm0
 ; SSE-NEXT:    movq %xmm0, %rax
 ; SSE-NEXT:    retq
@@ -140,7 +140,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovq %xmm0, %rax
 ; AVX1-NEXT:    vzeroupper
@@ -153,7 +153,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovq %xmm0, %rax
 ; AVX2-NEXT:    vzeroupper
@@ -166,7 +166,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; AVX512-NEXT:    vpandq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovq %xmm0, %rax
 ; AVX512-NEXT:    vzeroupper
@@ -200,7 +200,7 @@ define i32 @test_v2i32(<2 x i32> %a0) {
 define i32 @test_v4i32(<4 x i32> %a0) {
 ; SSE-LABEL: test_v4i32:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    pand %xmm0, %xmm1
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE-NEXT:    pand %xmm1, %xmm0
@@ -209,7 +209,7 @@ define i32 @test_v4i32(<4 x i32> %a0) {
 ;
 ; AVX-LABEL: test_v4i32:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -223,7 +223,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; SSE-LABEL: test_v8i32:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pand %xmm1, %xmm0
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    pand %xmm0, %xmm1
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE-NEXT:    pand %xmm1, %xmm0
@@ -234,7 +234,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -246,7 +246,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -258,7 +258,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -275,7 +275,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; SSE-NEXT:    pand %xmm3, %xmm1
 ; SSE-NEXT:    pand %xmm2, %xmm1
 ; SSE-NEXT:    pand %xmm0, %xmm1
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE-NEXT:    pand %xmm1, %xmm0
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE-NEXT:    pand %xmm0, %xmm1
@@ -287,7 +287,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
@@ -300,7 +300,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -314,7 +314,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX512-NEXT:    vpandd %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -335,7 +335,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; SSE-NEXT:    pand %xmm4, %xmm2
 ; SSE-NEXT:    pand %xmm3, %xmm2
 ; SSE-NEXT:    pand %xmm0, %xmm2
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE-NEXT:    pand %xmm2, %xmm0
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE-NEXT:    pand %xmm0, %xmm1
@@ -349,7 +349,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
@@ -364,7 +364,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -379,7 +379,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX512-NEXT:    vpandd %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -443,7 +443,7 @@ define i16 @test_v4i16(<4 x i16> %a0) {
 define i16 @test_v8i16(<8 x i16> %a0) {
 ; SSE-LABEL: test_v8i16:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    pand %xmm0, %xmm1
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE-NEXT:    pand %xmm1, %xmm0
@@ -456,7 +456,7 @@ define i16 @test_v8i16(<8 x i16> %a0) {
 ;
 ; AVX-LABEL: test_v8i16:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -473,7 +473,7 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; SSE-LABEL: test_v16i16:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pand %xmm1, %xmm0
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    pand %xmm0, %xmm1
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE-NEXT:    pand %xmm1, %xmm0
@@ -488,7 +488,7 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -503,7 +503,7 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -518,7 +518,7 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -538,7 +538,7 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; SSE-NEXT:    pand %xmm3, %xmm1
 ; SSE-NEXT:    pand %xmm2, %xmm1
 ; SSE-NEXT:    pand %xmm0, %xmm1
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE-NEXT:    pand %xmm1, %xmm0
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE-NEXT:    pand %xmm0, %xmm1
@@ -554,7 +554,7 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
@@ -570,7 +570,7 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -587,7 +587,7 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX512-NEXT:    vpandq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -611,7 +611,7 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; SSE-NEXT:    pand %xmm4, %xmm2
 ; SSE-NEXT:    pand %xmm3, %xmm2
 ; SSE-NEXT:    pand %xmm0, %xmm2
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE-NEXT:    pand %xmm2, %xmm0
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE-NEXT:    pand %xmm0, %xmm1
@@ -629,7 +629,7 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
@@ -647,7 +647,7 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -665,7 +665,7 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX512-NEXT:    vpandq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -763,7 +763,7 @@ define i8 @test_v8i8(<8 x i8> %a0) {
 define i8 @test_v16i8(<16 x i8> %a0) {
 ; SSE-LABEL: test_v16i8:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    pand %xmm0, %xmm1
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE-NEXT:    pand %xmm1, %xmm0
@@ -779,7 +779,7 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ;
 ; AVX-LABEL: test_v16i8:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -798,7 +798,7 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; SSE-LABEL: test_v32i8:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pand %xmm1, %xmm0
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    pand %xmm0, %xmm1
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE-NEXT:    pand %xmm1, %xmm0
@@ -816,7 +816,7 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -833,7 +833,7 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -850,7 +850,7 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -872,7 +872,7 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; SSE-NEXT:    pand %xmm3, %xmm1
 ; SSE-NEXT:    pand %xmm2, %xmm1
 ; SSE-NEXT:    pand %xmm0, %xmm1
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE-NEXT:    pand %xmm1, %xmm0
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE-NEXT:    pand %xmm0, %xmm1
@@ -891,7 +891,7 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
@@ -909,7 +909,7 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -928,7 +928,7 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX512-NEXT:    vpandq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -954,7 +954,7 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; SSE-NEXT:    pand %xmm4, %xmm2
 ; SSE-NEXT:    pand %xmm3, %xmm2
 ; SSE-NEXT:    pand %xmm0, %xmm2
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE-NEXT:    pand %xmm2, %xmm0
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE-NEXT:    pand %xmm0, %xmm1
@@ -975,7 +975,7 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
@@ -995,7 +995,7 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -1015,7 +1015,7 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX512-NEXT:    vpandq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
diff --git a/llvm/test/CodeGen/X86/vector-reduce-mul.ll b/llvm/test/CodeGen/X86/vector-reduce-mul.ll
index bafe112c5dfc8..f7e1a72f9a91f 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-mul.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-mul.ll
@@ -15,7 +15,7 @@
 define i64 @test_v2i64(<2 x i64> %a0) {
 ; SSE-LABEL: test_v2i64:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    movdqa %xmm0, %xmm2
 ; SSE-NEXT:    psrlq $32, %xmm2
 ; SSE-NEXT:    pmuludq %xmm1, %xmm2
@@ -31,7 +31,7 @@ define i64 @test_v2i64(<2 x i64> %a0) {
 ;
 ; AVX-LABEL: test_v2i64:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpsrlq $32, %xmm0, %xmm2
 ; AVX-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2
 ; AVX-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
@@ -45,7 +45,7 @@ define i64 @test_v2i64(<2 x i64> %a0) {
 ;
 ; AVX512BW-LABEL: test_v2i64:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512BW-NEXT:    vpsrlq $32, %xmm0, %xmm2
 ; AVX512BW-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2
 ; AVX512BW-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
@@ -59,7 +59,7 @@ define i64 @test_v2i64(<2 x i64> %a0) {
 ;
 ; AVX512BWVL-LABEL: test_v2i64:
 ; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512BWVL-NEXT:    vpsrlq $32, %xmm0, %xmm2
 ; AVX512BWVL-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2
 ; AVX512BWVL-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
@@ -74,7 +74,7 @@ define i64 @test_v2i64(<2 x i64> %a0) {
 ; AVX512DQ-LABEL: test_v2i64:
 ; AVX512DQ:       # %bb.0:
 ; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512DQ-NEXT:    vpmullq %zmm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    vmovq %xmm0, %rax
 ; AVX512DQ-NEXT:    vzeroupper
@@ -82,7 +82,7 @@ define i64 @test_v2i64(<2 x i64> %a0) {
 ;
 ; AVX512DQVL-LABEL: test_v2i64:
 ; AVX512DQVL:       # %bb.0:
-; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512DQVL-NEXT:    vpmullq %xmm1, %xmm0, %xmm0
 ; AVX512DQVL-NEXT:    vmovq %xmm0, %rax
 ; AVX512DQVL-NEXT:    retq
@@ -103,7 +103,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; SSE-NEXT:    psllq $32, %xmm3
 ; SSE-NEXT:    pmuludq %xmm1, %xmm0
 ; SSE-NEXT:    paddq %xmm3, %xmm0
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    movdqa %xmm0, %xmm2
 ; SSE-NEXT:    psrlq $32, %xmm2
 ; SSE-NEXT:    pmuludq %xmm1, %xmm2
@@ -128,7 +128,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; AVX1-NEXT:    vpsllq $32, %xmm2, %xmm2
 ; AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpaddq %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpsrlq $32, %xmm0, %xmm2
 ; AVX1-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2
 ; AVX1-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
@@ -152,7 +152,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; AVX2-NEXT:    vpsllq $32, %xmm2, %xmm2
 ; AVX2-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpaddq %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpsrlq $32, %xmm0, %xmm2
 ; AVX2-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2
 ; AVX2-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
@@ -176,7 +176,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; AVX512BW-NEXT:    vpsllq $32, %xmm2, %xmm2
 ; AVX512BW-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vpaddq %xmm2, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512BW-NEXT:    vpsrlq $32, %xmm0, %xmm2
 ; AVX512BW-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2
 ; AVX512BW-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
@@ -200,7 +200,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; AVX512BWVL-NEXT:    vpsllq $32, %xmm2, %xmm2
 ; AVX512BWVL-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    vpaddq %xmm2, %xmm0, %xmm0
-; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512BWVL-NEXT:    vpsrlq $32, %xmm0, %xmm2
 ; AVX512BWVL-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2
 ; AVX512BWVL-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
@@ -218,7 +218,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; AVX512DQ-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
 ; AVX512DQ-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512DQ-NEXT:    vpmullq %zmm1, %zmm0, %zmm0
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512DQ-NEXT:    vpmullq %zmm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    vmovq %xmm0, %rax
 ; AVX512DQ-NEXT:    vzeroupper
@@ -228,7 +228,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; AVX512DQVL:       # %bb.0:
 ; AVX512DQVL-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512DQVL-NEXT:    vpmullq %xmm1, %xmm0, %xmm0
-; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512DQVL-NEXT:    vpmullq %xmm1, %xmm0, %xmm0
 ; AVX512DQVL-NEXT:    vmovq %xmm0, %rax
 ; AVX512DQVL-NEXT:    vzeroupper
@@ -270,7 +270,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; SSE-NEXT:    psllq $32, %xmm3
 ; SSE-NEXT:    pmuludq %xmm1, %xmm0
 ; SSE-NEXT:    paddq %xmm3, %xmm0
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    movdqa %xmm0, %xmm2
 ; SSE-NEXT:    psrlq $32, %xmm2
 ; SSE-NEXT:    pmuludq %xmm1, %xmm2
@@ -312,7 +312,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; AVX1-NEXT:    vpsllq $32, %xmm1, %xmm1
 ; AVX1-NEXT:    vpmuludq %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpsrlq $32, %xmm0, %xmm2
 ; AVX1-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2
 ; AVX1-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
@@ -344,7 +344,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; AVX2-NEXT:    vpsllq $32, %xmm2, %xmm2
 ; AVX2-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpaddq %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpsrlq $32, %xmm0, %xmm2
 ; AVX2-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2
 ; AVX2-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
@@ -377,7 +377,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; AVX512BW-NEXT:    vpsllq $32, %xmm2, %xmm2
 ; AVX512BW-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vpaddq %xmm2, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512BW-NEXT:    vpsrlq $32, %xmm0, %xmm2
 ; AVX512BW-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2
 ; AVX512BW-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
@@ -410,7 +410,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; AVX512BWVL-NEXT:    vpsllq $32, %xmm2, %xmm2
 ; AVX512BWVL-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    vpaddq %xmm2, %xmm0, %xmm0
-; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512BWVL-NEXT:    vpsrlq $32, %xmm0, %xmm2
 ; AVX512BWVL-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2
 ; AVX512BWVL-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
@@ -429,7 +429,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; AVX512DQ-NEXT:    vpmullq %zmm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512DQ-NEXT:    vpmullq %zmm1, %zmm0, %zmm0
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512DQ-NEXT:    vpmullq %zmm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    vmovq %xmm0, %rax
 ; AVX512DQ-NEXT:    vzeroupper
@@ -441,7 +441,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; AVX512DQVL-NEXT:    vpmullq %zmm1, %zmm0, %zmm0
 ; AVX512DQVL-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512DQVL-NEXT:    vpmullq %xmm1, %xmm0, %xmm0
-; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512DQVL-NEXT:    vpmullq %xmm1, %xmm0, %xmm0
 ; AVX512DQVL-NEXT:    vmovq %xmm0, %rax
 ; AVX512DQVL-NEXT:    vzeroupper
@@ -523,7 +523,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; SSE-NEXT:    psllq $32, %xmm3
 ; SSE-NEXT:    pmuludq %xmm1, %xmm0
 ; SSE-NEXT:    paddq %xmm3, %xmm0
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    movdqa %xmm0, %xmm2
 ; SSE-NEXT:    psrlq $32, %xmm2
 ; SSE-NEXT:    pmuludq %xmm1, %xmm2
@@ -599,7 +599,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; AVX1-NEXT:    vpsllq $32, %xmm2, %xmm2
 ; AVX1-NEXT:    vpmuludq %xmm0, %xmm1, %xmm0
 ; AVX1-NEXT:    vpaddq %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpsrlq $32, %xmm0, %xmm2
 ; AVX1-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2
 ; AVX1-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
@@ -647,7 +647,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; AVX2-NEXT:    vpsllq $32, %xmm2, %xmm2
 ; AVX2-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpaddq %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpsrlq $32, %xmm0, %xmm2
 ; AVX2-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2
 ; AVX2-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
@@ -688,7 +688,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; AVX512BW-NEXT:    vpsllq $32, %xmm2, %xmm2
 ; AVX512BW-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vpaddq %xmm2, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512BW-NEXT:    vpsrlq $32, %xmm0, %xmm2
 ; AVX512BW-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2
 ; AVX512BW-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
@@ -729,7 +729,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; AVX512BWVL-NEXT:    vpsllq $32, %xmm2, %xmm2
 ; AVX512BWVL-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    vpaddq %xmm2, %xmm0, %xmm0
-; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512BWVL-NEXT:    vpsrlq $32, %xmm0, %xmm2
 ; AVX512BWVL-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2
 ; AVX512BWVL-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
@@ -749,7 +749,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; AVX512DQ-NEXT:    vpmullq %zmm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512DQ-NEXT:    vpmullq %zmm1, %zmm0, %zmm0
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512DQ-NEXT:    vpmullq %zmm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    vmovq %xmm0, %rax
 ; AVX512DQ-NEXT:    vzeroupper
@@ -762,7 +762,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; AVX512DQVL-NEXT:    vpmullq %zmm1, %zmm0, %zmm0
 ; AVX512DQVL-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512DQVL-NEXT:    vpmullq %xmm1, %xmm0, %xmm0
-; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512DQVL-NEXT:    vpmullq %xmm1, %xmm0, %xmm0
 ; AVX512DQVL-NEXT:    vmovq %xmm0, %rax
 ; AVX512DQVL-NEXT:    vzeroupper
@@ -810,7 +810,7 @@ define i32 @test_v2i32(<2 x i32> %a0) {
 define i32 @test_v4i32(<4 x i32> %a0) {
 ; SSE2-LABEL: test_v4i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,1,2,3]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
 ; SSE2-NEXT:    pmuludq %xmm2, %xmm3
@@ -821,7 +821,7 @@ define i32 @test_v4i32(<4 x i32> %a0) {
 ;
 ; SSE41-LABEL: test_v4i32:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmulld %xmm0, %xmm1
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE41-NEXT:    pmulld %xmm1, %xmm0
@@ -830,7 +830,7 @@ define i32 @test_v4i32(<4 x i32> %a0) {
 ;
 ; AVX-LABEL: test_v4i32:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
@@ -839,7 +839,7 @@ define i32 @test_v4i32(<4 x i32> %a0) {
 ;
 ; AVX512-LABEL: test_v4i32:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
@@ -856,7 +856,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
 ; SSE2-NEXT:    pmuludq %xmm2, %xmm3
 ; SSE2-NEXT:    pmuludq %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    pmuludq %xmm0, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[2,0,2,2]
 ; SSE2-NEXT:    pmuludq %xmm3, %xmm0
@@ -867,7 +867,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; SSE41-LABEL: test_v8i32:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pmulld %xmm1, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmulld %xmm0, %xmm1
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE41-NEXT:    pmulld %xmm1, %xmm0
@@ -878,7 +878,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
@@ -890,7 +890,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
@@ -902,7 +902,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
@@ -926,7 +926,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
 ; SSE2-NEXT:    pmuludq %xmm5, %xmm2
 ; SSE2-NEXT:    pmuludq %xmm1, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    pmuludq %xmm0, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,0,2,2]
 ; SSE2-NEXT:    pmuludq %xmm2, %xmm0
@@ -939,7 +939,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; SSE41-NEXT:    pmulld %xmm3, %xmm1
 ; SSE41-NEXT:    pmulld %xmm2, %xmm1
 ; SSE41-NEXT:    pmulld %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE41-NEXT:    pmulld %xmm1, %xmm0
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE41-NEXT:    pmulld %xmm0, %xmm1
@@ -953,7 +953,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX1-NEXT:    vpmulld %xmm2, %xmm3, %xmm2
 ; AVX1-NEXT:    vpmulld %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
@@ -966,7 +966,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX2-NEXT:    vpmulld %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
@@ -980,7 +980,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX512-NEXT:    vpmulld %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
@@ -1016,7 +1016,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; SSE2-NEXT:    pmuludq %xmm5, %xmm1
 ; SSE2-NEXT:    pmuludq %xmm3, %xmm1
 ; SSE2-NEXT:    pmuludq %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    pmuludq %xmm1, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm11[2,0,2,2]
 ; SSE2-NEXT:    pmuludq %xmm11, %xmm1
@@ -1033,7 +1033,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; SSE41-NEXT:    pmulld %xmm4, %xmm2
 ; SSE41-NEXT:    pmulld %xmm3, %xmm2
 ; SSE41-NEXT:    pmulld %xmm0, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE41-NEXT:    pmulld %xmm2, %xmm0
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE41-NEXT:    pmulld %xmm0, %xmm1
@@ -1053,7 +1053,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX1-NEXT:    vpmulld %xmm4, %xmm2, %xmm2
 ; AVX1-NEXT:    vpmulld %xmm1, %xmm2, %xmm1
 ; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
@@ -1068,7 +1068,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX2-NEXT:    vpmulld %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
@@ -1083,7 +1083,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX512-NEXT:    vpmulld %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
@@ -1165,7 +1165,7 @@ define i16 @test_v4i16(<4 x i16> %a0) {
 define i16 @test_v8i16(<8 x i16> %a0) {
 ; SSE-LABEL: test_v8i16:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    pmullw %xmm0, %xmm1
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE-NEXT:    pmullw %xmm1, %xmm0
@@ -1178,7 +1178,7 @@ define i16 @test_v8i16(<8 x i16> %a0) {
 ;
 ; AVX-LABEL: test_v8i16:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -1190,7 +1190,7 @@ define i16 @test_v8i16(<8 x i16> %a0) {
 ;
 ; AVX512-LABEL: test_v8i16:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -1207,7 +1207,7 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; SSE-LABEL: test_v16i16:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pmullw %xmm1, %xmm0
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    pmullw %xmm0, %xmm1
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE-NEXT:    pmullw %xmm1, %xmm0
@@ -1222,7 +1222,7 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -1237,7 +1237,7 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -1252,7 +1252,7 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -1272,7 +1272,7 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; SSE-NEXT:    pmullw %xmm3, %xmm1
 ; SSE-NEXT:    pmullw %xmm2, %xmm1
 ; SSE-NEXT:    pmullw %xmm0, %xmm1
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE-NEXT:    pmullw %xmm1, %xmm0
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE-NEXT:    pmullw %xmm0, %xmm1
@@ -1290,7 +1290,7 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX1-NEXT:    vpmullw %xmm2, %xmm3, %xmm2
 ; AVX1-NEXT:    vpmullw %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -1306,7 +1306,7 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -1323,7 +1323,7 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -1340,7 +1340,7 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
 ; AVX512BWVL-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -1357,7 +1357,7 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX512DQ-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -1374,7 +1374,7 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -1398,7 +1398,7 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; SSE-NEXT:    pmullw %xmm4, %xmm2
 ; SSE-NEXT:    pmullw %xmm3, %xmm2
 ; SSE-NEXT:    pmullw %xmm0, %xmm2
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE-NEXT:    pmullw %xmm2, %xmm0
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE-NEXT:    pmullw %xmm0, %xmm1
@@ -1422,7 +1422,7 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX1-NEXT:    vpmullw %xmm4, %xmm2, %xmm2
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm2, %xmm1
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -1440,7 +1440,7 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -1458,7 +1458,7 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -1476,7 +1476,7 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
 ; AVX512BWVL-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -1496,7 +1496,7 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX512DQ-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -1516,7 +1516,7 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -1634,7 +1634,7 @@ define i8 @test_v8i8(<8 x i8> %a0) {
 ; SSE41-LABEL: test_v8i8:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmullw %xmm0, %xmm1
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE41-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
@@ -1650,7 +1650,7 @@ define i8 @test_v8i8(<8 x i8> %a0) {
 ; AVX-LABEL: test_v8i8:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -1663,7 +1663,7 @@ define i8 @test_v8i8(<8 x i8> %a0) {
 ; AVX512-LABEL: test_v8i8:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -1711,7 +1711,7 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; SSE41-NEXT:    pmullw %xmm1, %xmm0
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pand %xmm1, %xmm0
 ; SSE41-NEXT:    pxor %xmm3, %xmm3
 ; SSE41-NEXT:    packuswb %xmm3, %xmm0
@@ -1735,7 +1735,7 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -1764,7 +1764,7 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ;
 ; AVX512BW-LABEL: test_v16i8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
 ; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
 ; AVX512BW-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
@@ -1786,7 +1786,7 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ;
 ; AVX512BWVL-LABEL: test_v16i8:
 ; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
 ; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
 ; AVX512BWVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
@@ -1808,7 +1808,7 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ;
 ; AVX512DQ-LABEL: test_v16i8:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
 ; AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
 ; AVX512DQ-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
@@ -1833,7 +1833,7 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ;
 ; AVX512DQVL-LABEL: test_v16i8:
 ; AVX512DQVL:       # %bb.0:
-; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512DQVL-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
 ; AVX512DQVL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
 ; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
@@ -1908,7 +1908,7 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; SSE41-NEXT:    packuswb %xmm4, %xmm3
 ; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm2 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
 ; SSE41-NEXT:    pmullw %xmm0, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE41-NEXT:    pand %xmm1, %xmm2
 ; SSE41-NEXT:    pxor %xmm3, %xmm3
 ; SSE41-NEXT:    packuswb %xmm3, %xmm2
@@ -1937,7 +1937,7 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX1-NEXT:    vpmullw %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -1982,7 +1982,7 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
 ; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm1
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
 ; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm1
@@ -2010,7 +2010,7 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
 ; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
 ; AVX512BWVL-NEXT:    vpmovwb %zmm0, %ymm1
-; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
 ; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
 ; AVX512BWVL-NEXT:    vpmovwb %zmm0, %ymm1
@@ -2158,7 +2158,7 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; SSE41-NEXT:    packuswb %xmm3, %xmm1
 ; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; SSE41-NEXT:    pmullw %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE41-NEXT:    pand %xmm2, %xmm1
 ; SSE41-NEXT:    pxor %xmm3, %xmm3
 ; SSE41-NEXT:    packuswb %xmm3, %xmm1
@@ -2196,7 +2196,7 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX1-NEXT:    vpmullw %xmm3, %xmm1, %xmm1
 ; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -2288,10 +2288,8 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX512BWVL-NEXT:    vextracti128 $1, %ymm1, %xmm1
 ; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; AVX512BWVL-NEXT:    vpmullw %xmm3, %xmm2, %xmm2
-; AVX512BWVL-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512BWVL-NEXT:    vpmovwb %ymm2, %xmm2
+; AVX512BWVL-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
 ; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; AVX512BWVL-NEXT:    vpmullw %xmm2, %xmm1, %xmm1
 ; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    vpshufb {{.*#+}} xmm1 = xmm0[8],zero,xmm0[10],zero,xmm0[12],zero,xmm0[14],zero,xmm0[8],zero,xmm0[10],zero,xmm0[12],zero,xmm0[14],zero
@@ -2495,7 +2493,7 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; SSE41-NEXT:    packuswb %xmm0, %xmm2
 ; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
 ; SSE41-NEXT:    pmullw %xmm1, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pand %xmm4, %xmm0
 ; SSE41-NEXT:    pxor %xmm2, %xmm2
 ; SSE41-NEXT:    packuswb %xmm2, %xmm0
@@ -2551,7 +2549,7 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX1-NEXT:    vpmullw %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vpmullw %xmm4, %xmm1, %xmm1
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -2657,10 +2655,8 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX512BWVL-NEXT:    vextracti128 $1, %ymm1, %xmm1
 ; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; AVX512BWVL-NEXT:    vpmullw %xmm3, %xmm0, %xmm0
-; AVX512BWVL-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm0
+; AVX512BWVL-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; AVX512BWVL-NEXT:    vpmullw %xmm0, %xmm1, %xmm0
 ; AVX512BWVL-NEXT:    vpmullw %xmm0, %xmm2, %xmm0
 ; AVX512BWVL-NEXT:    vpshufb {{.*#+}} xmm1 = xmm0[8],zero,xmm0[10],zero,xmm0[12],zero,xmm0[14],zero,xmm0[8],zero,xmm0[10],zero,xmm0[12],zero,xmm0[14],zero
diff --git a/llvm/test/CodeGen/X86/vector-reduce-or-cmp.ll b/llvm/test/CodeGen/X86/vector-reduce-or-cmp.ll
index 8223e6bd23804..e1253975d5a6a 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-or-cmp.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-or-cmp.ll
@@ -823,7 +823,7 @@ define i1 @test_v128i8(<128 x i8> %a0) {
 define i1 @trunc_v2i64(<2 x i64> %a0) {
 ; SSE2-LABEL: trunc_v2i64:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    por %xmm0, %xmm1
 ; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    testw %ax, %ax
diff --git a/llvm/test/CodeGen/X86/vector-reduce-or.ll b/llvm/test/CodeGen/X86/vector-reduce-or.ll
index 14eb3d27d8dff..95bff8e03afae 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-or.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-or.ll
@@ -13,14 +13,14 @@
 define i64 @test_v2i64(<2 x i64> %a0) {
 ; SSE-LABEL: test_v2i64:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    por %xmm0, %xmm1
 ; SSE-NEXT:    movq %xmm1, %rax
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: test_v2i64:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovq %xmm0, %rax
 ; AVX-NEXT:    retq
@@ -32,7 +32,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; SSE-LABEL: test_v4i64:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    por %xmm1, %xmm0
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    por %xmm0, %xmm1
 ; SSE-NEXT:    movq %xmm1, %rax
 ; SSE-NEXT:    retq
@@ -41,7 +41,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovq %xmm0, %rax
 ; AVX1-NEXT:    vzeroupper
@@ -51,7 +51,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovq %xmm0, %rax
 ; AVX2-NEXT:    vzeroupper
@@ -61,7 +61,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovq %xmm0, %rax
 ; AVX512-NEXT:    vzeroupper
@@ -76,7 +76,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; SSE-NEXT:    por %xmm3, %xmm1
 ; SSE-NEXT:    por %xmm2, %xmm1
 ; SSE-NEXT:    por %xmm0, %xmm1
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE-NEXT:    por %xmm1, %xmm0
 ; SSE-NEXT:    movq %xmm0, %rax
 ; SSE-NEXT:    retq
@@ -86,7 +86,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovq %xmm0, %rax
 ; AVX1-NEXT:    vzeroupper
@@ -97,7 +97,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovq %xmm0, %rax
 ; AVX2-NEXT:    vzeroupper
@@ -109,7 +109,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; AVX512-NEXT:    vporq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovq %xmm0, %rax
 ; AVX512-NEXT:    vzeroupper
@@ -128,7 +128,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; SSE-NEXT:    por %xmm4, %xmm2
 ; SSE-NEXT:    por %xmm3, %xmm2
 ; SSE-NEXT:    por %xmm0, %xmm2
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE-NEXT:    por %xmm2, %xmm0
 ; SSE-NEXT:    movq %xmm0, %rax
 ; SSE-NEXT:    retq
@@ -140,7 +140,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovq %xmm0, %rax
 ; AVX1-NEXT:    vzeroupper
@@ -153,7 +153,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovq %xmm0, %rax
 ; AVX2-NEXT:    vzeroupper
@@ -166,7 +166,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; AVX512-NEXT:    vporq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovq %xmm0, %rax
 ; AVX512-NEXT:    vzeroupper
@@ -200,7 +200,7 @@ define i32 @test_v2i32(<2 x i32> %a0) {
 define i32 @test_v4i32(<4 x i32> %a0) {
 ; SSE-LABEL: test_v4i32:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    por %xmm0, %xmm1
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE-NEXT:    por %xmm1, %xmm0
@@ -209,7 +209,7 @@ define i32 @test_v4i32(<4 x i32> %a0) {
 ;
 ; AVX-LABEL: test_v4i32:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
@@ -223,7 +223,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; SSE-LABEL: test_v8i32:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    por %xmm1, %xmm0
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    por %xmm0, %xmm1
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE-NEXT:    por %xmm1, %xmm0
@@ -234,7 +234,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
@@ -246,7 +246,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
@@ -258,7 +258,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
@@ -275,7 +275,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; SSE-NEXT:    por %xmm3, %xmm1
 ; SSE-NEXT:    por %xmm2, %xmm1
 ; SSE-NEXT:    por %xmm0, %xmm1
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE-NEXT:    por %xmm1, %xmm0
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE-NEXT:    por %xmm0, %xmm1
@@ -287,7 +287,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
@@ -300,7 +300,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
@@ -314,7 +314,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX512-NEXT:    vpord %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
@@ -335,7 +335,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; SSE-NEXT:    por %xmm4, %xmm2
 ; SSE-NEXT:    por %xmm3, %xmm2
 ; SSE-NEXT:    por %xmm0, %xmm2
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE-NEXT:    por %xmm2, %xmm0
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE-NEXT:    por %xmm0, %xmm1
@@ -349,7 +349,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
@@ -364,7 +364,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
@@ -379,7 +379,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX512-NEXT:    vpord %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
@@ -443,7 +443,7 @@ define i16 @test_v4i16(<4 x i16> %a0) {
 define i16 @test_v8i16(<8 x i16> %a0) {
 ; SSE-LABEL: test_v8i16:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    por %xmm0, %xmm1
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE-NEXT:    por %xmm1, %xmm0
@@ -456,7 +456,7 @@ define i16 @test_v8i16(<8 x i16> %a0) {
 ;
 ; AVX-LABEL: test_v8i16:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
@@ -473,7 +473,7 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; SSE-LABEL: test_v16i16:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    por %xmm1, %xmm0
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    por %xmm0, %xmm1
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE-NEXT:    por %xmm1, %xmm0
@@ -488,7 +488,7 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
@@ -503,7 +503,7 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
@@ -518,7 +518,7 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
@@ -538,7 +538,7 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; SSE-NEXT:    por %xmm3, %xmm1
 ; SSE-NEXT:    por %xmm2, %xmm1
 ; SSE-NEXT:    por %xmm0, %xmm1
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE-NEXT:    por %xmm1, %xmm0
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE-NEXT:    por %xmm0, %xmm1
@@ -554,7 +554,7 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
@@ -570,7 +570,7 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
@@ -587,7 +587,7 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX512-NEXT:    vporq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
@@ -611,7 +611,7 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; SSE-NEXT:    por %xmm4, %xmm2
 ; SSE-NEXT:    por %xmm3, %xmm2
 ; SSE-NEXT:    por %xmm0, %xmm2
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE-NEXT:    por %xmm2, %xmm0
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE-NEXT:    por %xmm0, %xmm1
@@ -629,7 +629,7 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
@@ -647,7 +647,7 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
@@ -665,7 +665,7 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX512-NEXT:    vporq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
@@ -763,7 +763,7 @@ define i8 @test_v8i8(<8 x i8> %a0) {
 define i8 @test_v16i8(<16 x i8> %a0) {
 ; SSE-LABEL: test_v16i8:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    por %xmm0, %xmm1
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE-NEXT:    por %xmm1, %xmm0
@@ -779,7 +779,7 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ;
 ; AVX-LABEL: test_v16i8:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
@@ -798,7 +798,7 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; SSE-LABEL: test_v32i8:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    por %xmm1, %xmm0
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    por %xmm0, %xmm1
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE-NEXT:    por %xmm1, %xmm0
@@ -816,7 +816,7 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
@@ -833,7 +833,7 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
@@ -850,7 +850,7 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
@@ -872,7 +872,7 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; SSE-NEXT:    por %xmm3, %xmm1
 ; SSE-NEXT:    por %xmm2, %xmm1
 ; SSE-NEXT:    por %xmm0, %xmm1
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE-NEXT:    por %xmm1, %xmm0
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE-NEXT:    por %xmm0, %xmm1
@@ -891,7 +891,7 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
@@ -909,7 +909,7 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
@@ -928,7 +928,7 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX512-NEXT:    vporq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
@@ -954,7 +954,7 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; SSE-NEXT:    por %xmm4, %xmm2
 ; SSE-NEXT:    por %xmm3, %xmm2
 ; SSE-NEXT:    por %xmm0, %xmm2
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE-NEXT:    por %xmm2, %xmm0
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE-NEXT:    por %xmm0, %xmm1
@@ -975,7 +975,7 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
@@ -995,7 +995,7 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
@@ -1015,7 +1015,7 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX512-NEXT:    vporq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
diff --git a/llvm/test/CodeGen/X86/vector-reduce-smax.ll b/llvm/test/CodeGen/X86/vector-reduce-smax.ll
index c0705ab96e039..26bbfed521968 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-smax.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-smax.ll
@@ -14,7 +14,7 @@
 define i64 @test_v2i64(<2 x i64> %a0) {
 ; SSE2-LABEL: test_v2i64:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm3
 ; SSE2-NEXT:    pxor %xmm2, %xmm3
@@ -36,7 +36,7 @@ define i64 @test_v2i64(<2 x i64> %a0) {
 ; SSE41-LABEL: test_v2i64:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [2147483648,2147483648]
 ; SSE41-NEXT:    movdqa %xmm0, %xmm4
 ; SSE41-NEXT:    pxor %xmm3, %xmm4
@@ -54,7 +54,7 @@ define i64 @test_v2i64(<2 x i64> %a0) {
 ; SSE42-LABEL: test_v2i64:
 ; SSE42:       # %bb.0:
 ; SSE42-NEXT:    movdqa %xmm0, %xmm1
-; SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; SSE42-NEXT:    pcmpgtq %xmm2, %xmm0
 ; SSE42-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
 ; SSE42-NEXT:    movq %xmm2, %rax
@@ -62,7 +62,7 @@ define i64 @test_v2i64(<2 x i64> %a0) {
 ;
 ; AVX-LABEL: test_v2i64:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm2
 ; AVX-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    vmovq %xmm0, %rax
@@ -71,7 +71,7 @@ define i64 @test_v2i64(<2 x i64> %a0) {
 ; AVX512BW-LABEL: test_v2i64:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512BW-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vmovq %xmm0, %rax
 ; AVX512BW-NEXT:    vzeroupper
@@ -79,7 +79,7 @@ define i64 @test_v2i64(<2 x i64> %a0) {
 ;
 ; AVX512VL-LABEL: test_v2i64:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512VL-NEXT:    vpmaxsq %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vmovq %xmm0, %rax
 ; AVX512VL-NEXT:    retq
@@ -106,7 +106,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; SSE2-NEXT:    pand %xmm4, %xmm0
 ; SSE2-NEXT:    pandn %xmm1, %xmm4
 ; SSE2-NEXT:    por %xmm0, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[2,3,2,3]
 ; SSE2-NEXT:    movdqa %xmm4, %xmm1
 ; SSE2-NEXT:    pxor %xmm2, %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm2
@@ -139,7 +139,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; SSE41-NEXT:    pand %xmm4, %xmm0
 ; SSE41-NEXT:    por %xmm5, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; SSE41-NEXT:    movdqa %xmm1, %xmm4
 ; SSE41-NEXT:    pxor %xmm3, %xmm4
 ; SSE41-NEXT:    pxor %xmm2, %xmm3
@@ -158,7 +158,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; SSE42-NEXT:    movdqa %xmm0, %xmm2
 ; SSE42-NEXT:    pcmpgtq %xmm1, %xmm0
 ; SSE42-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
-; SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; SSE42-NEXT:    movdqa %xmm1, %xmm0
 ; SSE42-NEXT:    pcmpgtq %xmm2, %xmm0
 ; SSE42-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
@@ -170,7 +170,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm2
 ; AVX1-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm2
 ; AVX1-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
 ; AVX1-NEXT:    vmovq %xmm0, %rax
@@ -182,7 +182,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm2
 ; AVX2-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
-; AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm2
 ; AVX2-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
 ; AVX2-NEXT:    vmovq %xmm0, %rax
@@ -194,7 +194,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
 ; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512BW-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512BW-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vmovq %xmm0, %rax
 ; AVX512BW-NEXT:    vzeroupper
@@ -204,7 +204,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512VL-NEXT:    vpmaxsq %xmm1, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512VL-NEXT:    vpmaxsq %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vmovq %xmm0, %rax
 ; AVX512VL-NEXT:    vzeroupper
@@ -262,7 +262,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; SSE2-NEXT:    pand %xmm1, %xmm5
 ; SSE2-NEXT:    pandn %xmm2, %xmm1
 ; SSE2-NEXT:    por %xmm5, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; SSE2-NEXT:    pxor %xmm4, %xmm2
 ; SSE2-NEXT:    pxor %xmm0, %xmm4
@@ -317,7 +317,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; SSE41-NEXT:    pand %xmm4, %xmm0
 ; SSE41-NEXT:    por %xmm1, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[2,3,2,3]
 ; SSE41-NEXT:    movdqa %xmm3, %xmm2
 ; SSE41-NEXT:    pxor %xmm5, %xmm2
 ; SSE41-NEXT:    pxor %xmm1, %xmm5
@@ -343,7 +343,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; SSE42-NEXT:    movapd %xmm2, %xmm0
 ; SSE42-NEXT:    pcmpgtq %xmm3, %xmm0
 ; SSE42-NEXT:    blendvpd %xmm0, %xmm2, %xmm3
-; SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[2,3,0,1]
+; SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[2,3,2,3]
 ; SSE42-NEXT:    movdqa %xmm3, %xmm0
 ; SSE42-NEXT:    pcmpgtq %xmm1, %xmm0
 ; SSE42-NEXT:    blendvpd %xmm0, %xmm3, %xmm1
@@ -360,7 +360,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; AVX1-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
 ; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm0, %xmm1
 ; AVX1-NEXT:    vblendvpd %xmm1, %xmm0, %xmm3, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm2
 ; AVX1-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
 ; AVX1-NEXT:    vmovq %xmm0, %rax
@@ -374,7 +374,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm2
 ; AVX2-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
-; AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm2
 ; AVX2-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
 ; AVX2-NEXT:    vmovq %xmm0, %rax
@@ -387,7 +387,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; AVX512BW-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512BW-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512BW-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vmovq %xmm0, %rax
 ; AVX512BW-NEXT:    vzeroupper
@@ -399,7 +399,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; AVX512VL-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
 ; AVX512VL-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512VL-NEXT:    vpmaxsq %xmm1, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512VL-NEXT:    vpmaxsq %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vmovq %xmm0, %rax
 ; AVX512VL-NEXT:    vzeroupper
@@ -517,7 +517,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; SSE2-NEXT:    pand %xmm2, %xmm0
 ; SSE2-NEXT:    pandn %xmm3, %xmm2
 ; SSE2-NEXT:    por %xmm0, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE2-NEXT:    movdqa %xmm2, %xmm1
 ; SSE2-NEXT:    pxor %xmm8, %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm8
@@ -616,7 +616,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; SSE41-NEXT:    pand %xmm2, %xmm0
 ; SSE41-NEXT:    por %xmm1, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm6, %xmm7
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm7[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm7[2,3,2,3]
 ; SSE41-NEXT:    movdqa %xmm7, %xmm2
 ; SSE41-NEXT:    pxor %xmm9, %xmm2
 ; SSE41-NEXT:    pxor %xmm1, %xmm9
@@ -654,7 +654,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; SSE42-NEXT:    movapd %xmm6, %xmm0
 ; SSE42-NEXT:    pcmpgtq %xmm7, %xmm0
 ; SSE42-NEXT:    blendvpd %xmm0, %xmm6, %xmm7
-; SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm7[2,3,0,1]
+; SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm7[2,3,2,3]
 ; SSE42-NEXT:    movdqa %xmm7, %xmm0
 ; SSE42-NEXT:    pcmpgtq %xmm1, %xmm0
 ; SSE42-NEXT:    blendvpd %xmm0, %xmm7, %xmm1
@@ -681,7 +681,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; AVX1-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
 ; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm0, %xmm1
 ; AVX1-NEXT:    vblendvpd %xmm1, %xmm0, %xmm3, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm2
 ; AVX1-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
 ; AVX1-NEXT:    vmovq %xmm0, %rax
@@ -699,7 +699,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm2
 ; AVX2-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
-; AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm2
 ; AVX2-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
 ; AVX2-NEXT:    vmovq %xmm0, %rax
@@ -713,7 +713,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; AVX512BW-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512BW-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512BW-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vmovq %xmm0, %rax
 ; AVX512BW-NEXT:    vzeroupper
@@ -726,7 +726,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; AVX512VL-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
 ; AVX512VL-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512VL-NEXT:    vpmaxsq %xmm1, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512VL-NEXT:    vpmaxsq %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vmovq %xmm0, %rax
 ; AVX512VL-NEXT:    vzeroupper
@@ -778,7 +778,7 @@ define i32 @test_v2i32(<2 x i32> %a0) {
 define i32 @test_v4i32(<4 x i32> %a0) {
 ; SSE2-LABEL: test_v4i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
 ; SSE2-NEXT:    pand %xmm2, %xmm0
@@ -795,7 +795,7 @@ define i32 @test_v4i32(<4 x i32> %a0) {
 ;
 ; SSE4-LABEL: test_v4i32:
 ; SSE4:       # %bb.0:
-; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE4-NEXT:    pmaxsd %xmm0, %xmm1
 ; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE4-NEXT:    pmaxsd %xmm1, %xmm0
@@ -804,7 +804,7 @@ define i32 @test_v4i32(<4 x i32> %a0) {
 ;
 ; AVX-LABEL: test_v4i32:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
@@ -813,7 +813,7 @@ define i32 @test_v4i32(<4 x i32> %a0) {
 ;
 ; AVX512-LABEL: test_v4i32:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
@@ -831,7 +831,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; SSE2-NEXT:    pand %xmm2, %xmm0
 ; SSE2-NEXT:    pandn %xmm1, %xmm2
 ; SSE2-NEXT:    por %xmm0, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE2-NEXT:    movdqa %xmm2, %xmm1
 ; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
 ; SSE2-NEXT:    pand %xmm1, %xmm2
@@ -849,7 +849,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; SSE4-LABEL: test_v8i32:
 ; SSE4:       # %bb.0:
 ; SSE4-NEXT:    pmaxsd %xmm1, %xmm0
-; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE4-NEXT:    pmaxsd %xmm0, %xmm1
 ; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE4-NEXT:    pmaxsd %xmm1, %xmm0
@@ -860,7 +860,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
@@ -872,7 +872,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
@@ -884,7 +884,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
@@ -913,7 +913,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; SSE2-NEXT:    pand %xmm0, %xmm1
 ; SSE2-NEXT:    pandn %xmm4, %xmm0
 ; SSE2-NEXT:    por %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
 ; SSE2-NEXT:    pand %xmm2, %xmm0
@@ -933,7 +933,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; SSE4-NEXT:    pmaxsd %xmm3, %xmm1
 ; SSE4-NEXT:    pmaxsd %xmm2, %xmm1
 ; SSE4-NEXT:    pmaxsd %xmm0, %xmm1
-; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE4-NEXT:    pmaxsd %xmm1, %xmm0
 ; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE4-NEXT:    pmaxsd %xmm0, %xmm1
@@ -947,7 +947,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX1-NEXT:    vpmaxsd %xmm2, %xmm3, %xmm2
 ; AVX1-NEXT:    vpmaxsd %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
@@ -960,7 +960,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX2-NEXT:    vpmaxsd %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
@@ -974,7 +974,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX512-NEXT:    vpmaxsd %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
@@ -1023,7 +1023,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; SSE2-NEXT:    pand %xmm2, %xmm0
 ; SSE2-NEXT:    pandn %xmm1, %xmm2
 ; SSE2-NEXT:    por %xmm0, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE2-NEXT:    movdqa %xmm2, %xmm1
 ; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
 ; SSE2-NEXT:    pand %xmm1, %xmm2
@@ -1047,7 +1047,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; SSE4-NEXT:    pmaxsd %xmm4, %xmm2
 ; SSE4-NEXT:    pmaxsd %xmm3, %xmm2
 ; SSE4-NEXT:    pmaxsd %xmm0, %xmm2
-; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE4-NEXT:    pmaxsd %xmm2, %xmm0
 ; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE4-NEXT:    pmaxsd %xmm0, %xmm1
@@ -1067,7 +1067,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX1-NEXT:    vpmaxsd %xmm4, %xmm2, %xmm2
 ; AVX1-NEXT:    vpmaxsd %xmm1, %xmm2, %xmm1
 ; AVX1-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
@@ -1082,7 +1082,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX2-NEXT:    vpmaxsd %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
@@ -1097,7 +1097,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX512-NEXT:    vpmaxsd %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
@@ -1179,7 +1179,7 @@ define i16 @test_v4i16(<4 x i16> %a0) {
 define i16 @test_v8i16(<8 x i16> %a0) {
 ; SSE2-LABEL: test_v8i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    pmaxsw %xmm0, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE2-NEXT:    pmaxsw %xmm1, %xmm0
@@ -1224,7 +1224,7 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; SSE2-LABEL: test_v16i16:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pmaxsw %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    pmaxsw %xmm0, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE2-NEXT:    pmaxsw %xmm1, %xmm0
@@ -1290,7 +1290,7 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; SSE2-NEXT:    pmaxsw %xmm3, %xmm1
 ; SSE2-NEXT:    pmaxsw %xmm2, %xmm1
 ; SSE2-NEXT:    pmaxsw %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    pmaxsw %xmm1, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE2-NEXT:    pmaxsw %xmm0, %xmm1
@@ -1368,7 +1368,7 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; SSE2-NEXT:    pmaxsw %xmm4, %xmm2
 ; SSE2-NEXT:    pmaxsw %xmm3, %xmm2
 ; SSE2-NEXT:    pmaxsw %xmm0, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE2-NEXT:    pmaxsw %xmm2, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE2-NEXT:    pmaxsw %xmm0, %xmm1
@@ -1622,7 +1622,7 @@ define i8 @test_v8i8(<8 x i8> %a0) {
 define i8 @test_v16i8(<16 x i8> %a0) {
 ; SSE2-LABEL: test_v16i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; SSE2-NEXT:    pcmpgtb %xmm1, %xmm2
 ; SSE2-NEXT:    pand %xmm2, %xmm0
@@ -1697,7 +1697,7 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; SSE2-NEXT:    pand %xmm2, %xmm0
 ; SSE2-NEXT:    pandn %xmm1, %xmm2
 ; SSE2-NEXT:    por %xmm0, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE2-NEXT:    movdqa %xmm2, %xmm1
 ; SSE2-NEXT:    pcmpgtb %xmm0, %xmm1
 ; SSE2-NEXT:    pand %xmm1, %xmm2
@@ -1803,7 +1803,7 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; SSE2-NEXT:    pand %xmm0, %xmm1
 ; SSE2-NEXT:    pandn %xmm4, %xmm0
 ; SSE2-NEXT:    por %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; SSE2-NEXT:    pcmpgtb %xmm1, %xmm2
 ; SSE2-NEXT:    pand %xmm2, %xmm0
@@ -1937,7 +1937,7 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; SSE2-NEXT:    pand %xmm2, %xmm0
 ; SSE2-NEXT:    pandn %xmm1, %xmm2
 ; SSE2-NEXT:    por %xmm0, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE2-NEXT:    movdqa %xmm2, %xmm1
 ; SSE2-NEXT:    pcmpgtb %xmm0, %xmm1
 ; SSE2-NEXT:    pand %xmm1, %xmm2
diff --git a/llvm/test/CodeGen/X86/vector-reduce-smin.ll b/llvm/test/CodeGen/X86/vector-reduce-smin.ll
index e8259e147133c..6f561c27d71dc 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-smin.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-smin.ll
@@ -14,7 +14,7 @@
 define i64 @test_v2i64(<2 x i64> %a0) {
 ; SSE2-LABEL: test_v2i64:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm3
 ; SSE2-NEXT:    pxor %xmm2, %xmm3
@@ -36,7 +36,7 @@ define i64 @test_v2i64(<2 x i64> %a0) {
 ; SSE41-LABEL: test_v2i64:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [2147483648,2147483648]
 ; SSE41-NEXT:    pxor %xmm3, %xmm0
 ; SSE41-NEXT:    pxor %xmm2, %xmm3
@@ -53,7 +53,7 @@ define i64 @test_v2i64(<2 x i64> %a0) {
 ; SSE42-LABEL: test_v2i64:
 ; SSE42:       # %bb.0:
 ; SSE42-NEXT:    movdqa %xmm0, %xmm1
-; SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; SSE42-NEXT:    movdqa %xmm2, %xmm0
 ; SSE42-NEXT:    pcmpgtq %xmm1, %xmm0
 ; SSE42-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
@@ -62,7 +62,7 @@ define i64 @test_v2i64(<2 x i64> %a0) {
 ;
 ; AVX-LABEL: test_v2i64:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
 ; AVX-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    vmovq %xmm0, %rax
@@ -71,7 +71,7 @@ define i64 @test_v2i64(<2 x i64> %a0) {
 ; AVX512BW-LABEL: test_v2i64:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512BW-NEXT:    vpminsq %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vmovq %xmm0, %rax
 ; AVX512BW-NEXT:    vzeroupper
@@ -79,7 +79,7 @@ define i64 @test_v2i64(<2 x i64> %a0) {
 ;
 ; AVX512VL-LABEL: test_v2i64:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512VL-NEXT:    vpminsq %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vmovq %xmm0, %rax
 ; AVX512VL-NEXT:    retq
@@ -106,7 +106,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; SSE2-NEXT:    pand %xmm4, %xmm0
 ; SSE2-NEXT:    pandn %xmm1, %xmm4
 ; SSE2-NEXT:    por %xmm0, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[2,3,2,3]
 ; SSE2-NEXT:    movdqa %xmm4, %xmm1
 ; SSE2-NEXT:    pxor %xmm2, %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm2
@@ -138,7 +138,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; SSE41-NEXT:    pand %xmm5, %xmm0
 ; SSE41-NEXT:    por %xmm4, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
 ; SSE41-NEXT:    pxor %xmm3, %xmm0
 ; SSE41-NEXT:    pxor %xmm2, %xmm3
@@ -158,7 +158,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; SSE42-NEXT:    movdqa %xmm1, %xmm0
 ; SSE42-NEXT:    pcmpgtq %xmm2, %xmm0
 ; SSE42-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
-; SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; SSE42-NEXT:    movdqa %xmm2, %xmm0
 ; SSE42-NEXT:    pcmpgtq %xmm1, %xmm0
 ; SSE42-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
@@ -170,7 +170,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
 ; AVX1-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
 ; AVX1-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
 ; AVX1-NEXT:    vmovq %xmm0, %rax
@@ -182,7 +182,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
 ; AVX2-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
-; AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
 ; AVX2-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
 ; AVX2-NEXT:    vmovq %xmm0, %rax
@@ -194,7 +194,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
 ; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512BW-NEXT:    vpminsq %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512BW-NEXT:    vpminsq %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vmovq %xmm0, %rax
 ; AVX512BW-NEXT:    vzeroupper
@@ -204,7 +204,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512VL-NEXT:    vpminsq %xmm1, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512VL-NEXT:    vpminsq %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vmovq %xmm0, %rax
 ; AVX512VL-NEXT:    vzeroupper
@@ -262,7 +262,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; SSE2-NEXT:    pand %xmm1, %xmm3
 ; SSE2-NEXT:    pandn %xmm5, %xmm1
 ; SSE2-NEXT:    por %xmm3, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; SSE2-NEXT:    pxor %xmm4, %xmm2
 ; SSE2-NEXT:    pxor %xmm0, %xmm4
@@ -317,7 +317,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; SSE41-NEXT:    pand %xmm4, %xmm0
 ; SSE41-NEXT:    por %xmm1, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[2,3,2,3]
 ; SSE41-NEXT:    movdqa %xmm3, %xmm0
 ; SSE41-NEXT:    pxor %xmm5, %xmm0
 ; SSE41-NEXT:    pxor %xmm1, %xmm5
@@ -343,7 +343,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; SSE42-NEXT:    movapd %xmm3, %xmm0
 ; SSE42-NEXT:    pcmpgtq %xmm2, %xmm0
 ; SSE42-NEXT:    blendvpd %xmm0, %xmm2, %xmm3
-; SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[2,3,0,1]
+; SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[2,3,2,3]
 ; SSE42-NEXT:    movdqa %xmm1, %xmm0
 ; SSE42-NEXT:    pcmpgtq %xmm3, %xmm0
 ; SSE42-NEXT:    blendvpd %xmm0, %xmm3, %xmm1
@@ -360,7 +360,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; AVX1-NEXT:    vblendvpd %xmm4, %xmm2, %xmm3, %xmm1
 ; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
 ; AVX1-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
 ; AVX1-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
 ; AVX1-NEXT:    vmovq %xmm0, %rax
@@ -374,7 +374,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
 ; AVX2-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
-; AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
 ; AVX2-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
 ; AVX2-NEXT:    vmovq %xmm0, %rax
@@ -387,7 +387,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; AVX512BW-NEXT:    vpminsq %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512BW-NEXT:    vpminsq %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512BW-NEXT:    vpminsq %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vmovq %xmm0, %rax
 ; AVX512BW-NEXT:    vzeroupper
@@ -399,7 +399,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; AVX512VL-NEXT:    vpminsq %zmm1, %zmm0, %zmm0
 ; AVX512VL-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512VL-NEXT:    vpminsq %xmm1, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512VL-NEXT:    vpminsq %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vmovq %xmm0, %rax
 ; AVX512VL-NEXT:    vzeroupper
@@ -517,7 +517,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; SSE2-NEXT:    pand %xmm2, %xmm3
 ; SSE2-NEXT:    pandn %xmm1, %xmm2
 ; SSE2-NEXT:    por %xmm3, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE2-NEXT:    movdqa %xmm2, %xmm1
 ; SSE2-NEXT:    pxor %xmm8, %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm8
@@ -616,7 +616,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; SSE41-NEXT:    pand %xmm2, %xmm0
 ; SSE41-NEXT:    por %xmm1, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm6, %xmm7
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm7[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm7[2,3,2,3]
 ; SSE41-NEXT:    movdqa %xmm7, %xmm0
 ; SSE41-NEXT:    pxor %xmm9, %xmm0
 ; SSE41-NEXT:    pxor %xmm1, %xmm9
@@ -654,7 +654,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; SSE42-NEXT:    movapd %xmm7, %xmm0
 ; SSE42-NEXT:    pcmpgtq %xmm6, %xmm0
 ; SSE42-NEXT:    blendvpd %xmm0, %xmm6, %xmm7
-; SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm7[2,3,0,1]
+; SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm7[2,3,2,3]
 ; SSE42-NEXT:    movdqa %xmm1, %xmm0
 ; SSE42-NEXT:    pcmpgtq %xmm7, %xmm0
 ; SSE42-NEXT:    blendvpd %xmm0, %xmm7, %xmm1
@@ -681,7 +681,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; AVX1-NEXT:    vblendvpd %xmm6, %xmm4, %xmm5, %xmm1
 ; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
 ; AVX1-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
 ; AVX1-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
 ; AVX1-NEXT:    vmovq %xmm0, %rax
@@ -699,7 +699,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
 ; AVX2-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
-; AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
 ; AVX2-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
 ; AVX2-NEXT:    vmovq %xmm0, %rax
@@ -713,7 +713,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; AVX512BW-NEXT:    vpminsq %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512BW-NEXT:    vpminsq %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512BW-NEXT:    vpminsq %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vmovq %xmm0, %rax
 ; AVX512BW-NEXT:    vzeroupper
@@ -726,7 +726,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; AVX512VL-NEXT:    vpminsq %zmm1, %zmm0, %zmm0
 ; AVX512VL-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512VL-NEXT:    vpminsq %xmm1, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512VL-NEXT:    vpminsq %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vmovq %xmm0, %rax
 ; AVX512VL-NEXT:    vzeroupper
@@ -778,7 +778,7 @@ define i32 @test_v2i32(<2 x i32> %a0) {
 define i32 @test_v4i32(<4 x i32> %a0) {
 ; SSE2-LABEL: test_v4i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
 ; SSE2-NEXT:    pand %xmm2, %xmm0
@@ -795,7 +795,7 @@ define i32 @test_v4i32(<4 x i32> %a0) {
 ;
 ; SSE4-LABEL: test_v4i32:
 ; SSE4:       # %bb.0:
-; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE4-NEXT:    pminsd %xmm0, %xmm1
 ; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE4-NEXT:    pminsd %xmm1, %xmm0
@@ -804,7 +804,7 @@ define i32 @test_v4i32(<4 x i32> %a0) {
 ;
 ; AVX-LABEL: test_v4i32:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
@@ -813,7 +813,7 @@ define i32 @test_v4i32(<4 x i32> %a0) {
 ;
 ; AVX512-LABEL: test_v4i32:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
@@ -831,7 +831,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; SSE2-NEXT:    pand %xmm2, %xmm0
 ; SSE2-NEXT:    pandn %xmm1, %xmm2
 ; SSE2-NEXT:    por %xmm0, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; SSE2-NEXT:    pcmpgtd %xmm2, %xmm1
 ; SSE2-NEXT:    pand %xmm1, %xmm2
@@ -849,7 +849,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; SSE4-LABEL: test_v8i32:
 ; SSE4:       # %bb.0:
 ; SSE4-NEXT:    pminsd %xmm1, %xmm0
-; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE4-NEXT:    pminsd %xmm0, %xmm1
 ; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE4-NEXT:    pminsd %xmm1, %xmm0
@@ -860,7 +860,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
@@ -872,7 +872,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
@@ -884,7 +884,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
@@ -913,7 +913,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; SSE2-NEXT:    pand %xmm1, %xmm4
 ; SSE2-NEXT:    pandn %xmm0, %xmm1
 ; SSE2-NEXT:    por %xmm4, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
 ; SSE2-NEXT:    pand %xmm2, %xmm1
@@ -933,7 +933,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; SSE4-NEXT:    pminsd %xmm3, %xmm1
 ; SSE4-NEXT:    pminsd %xmm2, %xmm1
 ; SSE4-NEXT:    pminsd %xmm0, %xmm1
-; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE4-NEXT:    pminsd %xmm1, %xmm0
 ; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE4-NEXT:    pminsd %xmm0, %xmm1
@@ -947,7 +947,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX1-NEXT:    vpminsd %xmm2, %xmm3, %xmm2
 ; AVX1-NEXT:    vpminsd %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
@@ -960,7 +960,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX2-NEXT:    vpminsd %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
@@ -974,7 +974,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX512-NEXT:    vpminsd %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
@@ -1023,7 +1023,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; SSE2-NEXT:    pand %xmm1, %xmm2
 ; SSE2-NEXT:    pandn %xmm0, %xmm1
 ; SSE2-NEXT:    por %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
 ; SSE2-NEXT:    pand %xmm2, %xmm1
@@ -1047,7 +1047,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; SSE4-NEXT:    pminsd %xmm4, %xmm2
 ; SSE4-NEXT:    pminsd %xmm3, %xmm2
 ; SSE4-NEXT:    pminsd %xmm0, %xmm2
-; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE4-NEXT:    pminsd %xmm2, %xmm0
 ; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE4-NEXT:    pminsd %xmm0, %xmm1
@@ -1067,7 +1067,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX1-NEXT:    vpminsd %xmm4, %xmm2, %xmm2
 ; AVX1-NEXT:    vpminsd %xmm1, %xmm2, %xmm1
 ; AVX1-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
@@ -1082,7 +1082,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX2-NEXT:    vpminsd %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
@@ -1097,7 +1097,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX512-NEXT:    vpminsd %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
@@ -1179,7 +1179,7 @@ define i16 @test_v4i16(<4 x i16> %a0) {
 define i16 @test_v8i16(<8 x i16> %a0) {
 ; SSE2-LABEL: test_v8i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    pminsw %xmm0, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE2-NEXT:    pminsw %xmm1, %xmm0
@@ -1224,7 +1224,7 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; SSE2-LABEL: test_v16i16:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pminsw %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    pminsw %xmm0, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE2-NEXT:    pminsw %xmm1, %xmm0
@@ -1290,7 +1290,7 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; SSE2-NEXT:    pminsw %xmm3, %xmm1
 ; SSE2-NEXT:    pminsw %xmm2, %xmm1
 ; SSE2-NEXT:    pminsw %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    pminsw %xmm1, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE2-NEXT:    pminsw %xmm0, %xmm1
@@ -1368,7 +1368,7 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; SSE2-NEXT:    pminsw %xmm4, %xmm2
 ; SSE2-NEXT:    pminsw %xmm3, %xmm2
 ; SSE2-NEXT:    pminsw %xmm0, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE2-NEXT:    pminsw %xmm2, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE2-NEXT:    pminsw %xmm0, %xmm1
@@ -1622,7 +1622,7 @@ define i8 @test_v8i8(<8 x i8> %a0) {
 define i8 @test_v16i8(<16 x i8> %a0) {
 ; SSE2-LABEL: test_v16i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; SSE2-NEXT:    pcmpgtb %xmm0, %xmm2
 ; SSE2-NEXT:    pand %xmm2, %xmm0
@@ -1697,7 +1697,7 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; SSE2-NEXT:    pand %xmm2, %xmm0
 ; SSE2-NEXT:    pandn %xmm1, %xmm2
 ; SSE2-NEXT:    por %xmm0, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; SSE2-NEXT:    pcmpgtb %xmm2, %xmm1
 ; SSE2-NEXT:    pand %xmm1, %xmm2
@@ -1803,7 +1803,7 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; SSE2-NEXT:    pand %xmm1, %xmm4
 ; SSE2-NEXT:    pandn %xmm0, %xmm1
 ; SSE2-NEXT:    por %xmm4, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; SSE2-NEXT:    pcmpgtb %xmm1, %xmm2
 ; SSE2-NEXT:    pand %xmm2, %xmm1
@@ -1937,7 +1937,7 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; SSE2-NEXT:    pand %xmm1, %xmm2
 ; SSE2-NEXT:    pandn %xmm0, %xmm1
 ; SSE2-NEXT:    por %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; SSE2-NEXT:    pcmpgtb %xmm1, %xmm2
 ; SSE2-NEXT:    pand %xmm2, %xmm1
diff --git a/llvm/test/CodeGen/X86/vector-reduce-umax.ll b/llvm/test/CodeGen/X86/vector-reduce-umax.ll
index 75e8133581338..a11fff0a5b5c8 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-umax.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-umax.ll
@@ -14,7 +14,7 @@
 define i64 @test_v2i64(<2 x i64> %a0) {
 ; SSE2-LABEL: test_v2i64:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [9223372039002259456,9223372039002259456]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm3
 ; SSE2-NEXT:    pxor %xmm2, %xmm3
@@ -36,7 +36,7 @@ define i64 @test_v2i64(<2 x i64> %a0) {
 ; SSE41-LABEL: test_v2i64:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [9223372039002259456,9223372039002259456]
 ; SSE41-NEXT:    movdqa %xmm0, %xmm4
 ; SSE41-NEXT:    pxor %xmm3, %xmm4
@@ -54,7 +54,7 @@ define i64 @test_v2i64(<2 x i64> %a0) {
 ; SSE42-LABEL: test_v2i64:
 ; SSE42:       # %bb.0:
 ; SSE42-NEXT:    movdqa %xmm0, %xmm1
-; SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; SSE42-NEXT:    movdqa {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808]
 ; SSE42-NEXT:    pxor %xmm3, %xmm0
 ; SSE42-NEXT:    pxor %xmm2, %xmm3
@@ -65,7 +65,7 @@ define i64 @test_v2i64(<2 x i64> %a0) {
 ;
 ; AVX-LABEL: test_v2i64:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
 ; AVX-NEXT:    vpxor %xmm2, %xmm0, %xmm3
 ; AVX-NEXT:    vpxor %xmm2, %xmm1, %xmm2
@@ -77,7 +77,7 @@ define i64 @test_v2i64(<2 x i64> %a0) {
 ; AVX512BW-LABEL: test_v2i64:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512BW-NEXT:    vpmaxuq %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vmovq %xmm0, %rax
 ; AVX512BW-NEXT:    vzeroupper
@@ -85,7 +85,7 @@ define i64 @test_v2i64(<2 x i64> %a0) {
 ;
 ; AVX512VL-LABEL: test_v2i64:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512VL-NEXT:    vpmaxuq %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vmovq %xmm0, %rax
 ; AVX512VL-NEXT:    retq
@@ -112,7 +112,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; SSE2-NEXT:    pand %xmm4, %xmm0
 ; SSE2-NEXT:    pandn %xmm1, %xmm4
 ; SSE2-NEXT:    por %xmm0, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[2,3,2,3]
 ; SSE2-NEXT:    movdqa %xmm4, %xmm1
 ; SSE2-NEXT:    pxor %xmm2, %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm2
@@ -145,7 +145,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; SSE41-NEXT:    pand %xmm4, %xmm0
 ; SSE41-NEXT:    por %xmm5, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; SSE41-NEXT:    movdqa %xmm1, %xmm4
 ; SSE41-NEXT:    pxor %xmm3, %xmm4
 ; SSE41-NEXT:    pxor %xmm2, %xmm3
@@ -168,7 +168,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; SSE42-NEXT:    pxor %xmm3, %xmm0
 ; SSE42-NEXT:    pcmpgtq %xmm4, %xmm0
 ; SSE42-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
-; SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; SSE42-NEXT:    movdqa %xmm1, %xmm0
 ; SSE42-NEXT:    pxor %xmm3, %xmm0
 ; SSE42-NEXT:    pxor %xmm2, %xmm3
@@ -185,7 +185,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm4
 ; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm4, %xmm3
 ; AVX1-NEXT:    vblendvpd %xmm3, %xmm0, %xmm1, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vxorpd %xmm2, %xmm0, %xmm3
 ; AVX1-NEXT:    vxorpd %xmm2, %xmm1, %xmm2
 ; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm2
@@ -202,7 +202,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm4
 ; AVX2-NEXT:    vpcmpgtq %xmm3, %xmm4, %xmm3
 ; AVX2-NEXT:    vblendvpd %xmm3, %xmm0, %xmm1, %xmm0
-; AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vxorpd %xmm2, %xmm0, %xmm3
 ; AVX2-NEXT:    vxorpd %xmm2, %xmm1, %xmm2
 ; AVX2-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm2
@@ -216,7 +216,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
 ; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512BW-NEXT:    vpmaxuq %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512BW-NEXT:    vpmaxuq %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vmovq %xmm0, %rax
 ; AVX512BW-NEXT:    vzeroupper
@@ -226,7 +226,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512VL-NEXT:    vpmaxuq %xmm1, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512VL-NEXT:    vpmaxuq %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vmovq %xmm0, %rax
 ; AVX512VL-NEXT:    vzeroupper
@@ -284,7 +284,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; SSE2-NEXT:    pand %xmm1, %xmm5
 ; SSE2-NEXT:    pandn %xmm2, %xmm1
 ; SSE2-NEXT:    por %xmm5, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; SSE2-NEXT:    pxor %xmm4, %xmm2
 ; SSE2-NEXT:    pxor %xmm0, %xmm4
@@ -339,7 +339,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; SSE41-NEXT:    pand %xmm4, %xmm0
 ; SSE41-NEXT:    por %xmm1, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[2,3,2,3]
 ; SSE41-NEXT:    movdqa %xmm3, %xmm2
 ; SSE41-NEXT:    pxor %xmm5, %xmm2
 ; SSE41-NEXT:    pxor %xmm1, %xmm5
@@ -374,7 +374,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; SSE42-NEXT:    xorpd %xmm5, %xmm0
 ; SSE42-NEXT:    pcmpgtq %xmm1, %xmm0
 ; SSE42-NEXT:    blendvpd %xmm0, %xmm2, %xmm3
-; SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[2,3,0,1]
+; SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[2,3,2,3]
 ; SSE42-NEXT:    movdqa %xmm3, %xmm0
 ; SSE42-NEXT:    pxor %xmm5, %xmm0
 ; SSE42-NEXT:    pxor %xmm1, %xmm5
@@ -400,7 +400,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; AVX1-NEXT:    vxorpd %xmm2, %xmm0, %xmm1
 ; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm1, %xmm1
 ; AVX1-NEXT:    vblendvpd %xmm1, %xmm0, %xmm4, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vxorpd %xmm2, %xmm0, %xmm3
 ; AVX1-NEXT:    vxorpd %xmm2, %xmm1, %xmm2
 ; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm2
@@ -421,7 +421,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; AVX2-NEXT:    vxorpd %xmm2, %xmm0, %xmm4
 ; AVX2-NEXT:    vpcmpgtq %xmm3, %xmm4, %xmm3
 ; AVX2-NEXT:    vblendvpd %xmm3, %xmm0, %xmm1, %xmm0
-; AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vxorpd %xmm2, %xmm0, %xmm3
 ; AVX2-NEXT:    vxorpd %xmm2, %xmm1, %xmm2
 ; AVX2-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm2
@@ -436,7 +436,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; AVX512BW-NEXT:    vpmaxuq %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512BW-NEXT:    vpmaxuq %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512BW-NEXT:    vpmaxuq %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vmovq %xmm0, %rax
 ; AVX512BW-NEXT:    vzeroupper
@@ -448,7 +448,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; AVX512VL-NEXT:    vpmaxuq %zmm1, %zmm0, %zmm0
 ; AVX512VL-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512VL-NEXT:    vpmaxuq %xmm1, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512VL-NEXT:    vpmaxuq %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vmovq %xmm0, %rax
 ; AVX512VL-NEXT:    vzeroupper
@@ -566,7 +566,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; SSE2-NEXT:    pand %xmm2, %xmm0
 ; SSE2-NEXT:    pandn %xmm3, %xmm2
 ; SSE2-NEXT:    por %xmm0, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE2-NEXT:    movdqa %xmm2, %xmm1
 ; SSE2-NEXT:    pxor %xmm8, %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm8
@@ -665,7 +665,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; SSE41-NEXT:    pand %xmm2, %xmm0
 ; SSE41-NEXT:    por %xmm1, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm6, %xmm7
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm7[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm7[2,3,2,3]
 ; SSE41-NEXT:    movdqa %xmm7, %xmm2
 ; SSE41-NEXT:    pxor %xmm9, %xmm2
 ; SSE41-NEXT:    pxor %xmm1, %xmm9
@@ -725,7 +725,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; SSE42-NEXT:    xorpd %xmm9, %xmm0
 ; SSE42-NEXT:    pcmpgtq %xmm1, %xmm0
 ; SSE42-NEXT:    blendvpd %xmm0, %xmm6, %xmm7
-; SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm7[2,3,0,1]
+; SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm7[2,3,2,3]
 ; SSE42-NEXT:    movdqa %xmm7, %xmm0
 ; SSE42-NEXT:    pxor %xmm9, %xmm0
 ; SSE42-NEXT:    pxor %xmm1, %xmm9
@@ -769,7 +769,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; AVX1-NEXT:    vxorpd %xmm4, %xmm0, %xmm1
 ; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm1, %xmm1
 ; AVX1-NEXT:    vblendvpd %xmm1, %xmm0, %xmm3, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vxorpd %xmm4, %xmm0, %xmm2
 ; AVX1-NEXT:    vxorpd %xmm4, %xmm1, %xmm3
 ; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm2, %xmm2
@@ -798,7 +798,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; AVX2-NEXT:    vxorpd %xmm4, %xmm0, %xmm3
 ; AVX2-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm2
 ; AVX2-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
-; AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vxorpd %xmm4, %xmm0, %xmm2
 ; AVX2-NEXT:    vxorpd %xmm4, %xmm1, %xmm3
 ; AVX2-NEXT:    vpcmpgtq %xmm3, %xmm2, %xmm2
@@ -814,7 +814,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; AVX512BW-NEXT:    vpmaxuq %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512BW-NEXT:    vpmaxuq %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512BW-NEXT:    vpmaxuq %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vmovq %xmm0, %rax
 ; AVX512BW-NEXT:    vzeroupper
@@ -827,7 +827,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; AVX512VL-NEXT:    vpmaxuq %zmm1, %zmm0, %zmm0
 ; AVX512VL-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512VL-NEXT:    vpmaxuq %xmm1, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512VL-NEXT:    vpmaxuq %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vmovq %xmm0, %rax
 ; AVX512VL-NEXT:    vzeroupper
@@ -882,7 +882,7 @@ define i32 @test_v2i32(<2 x i32> %a0) {
 define i32 @test_v4i32(<4 x i32> %a0) {
 ; SSE2-LABEL: test_v4i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm3
 ; SSE2-NEXT:    pxor %xmm2, %xmm3
@@ -905,7 +905,7 @@ define i32 @test_v4i32(<4 x i32> %a0) {
 ;
 ; SSE4-LABEL: test_v4i32:
 ; SSE4:       # %bb.0:
-; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE4-NEXT:    pmaxud %xmm0, %xmm1
 ; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE4-NEXT:    pmaxud %xmm1, %xmm0
@@ -914,7 +914,7 @@ define i32 @test_v4i32(<4 x i32> %a0) {
 ;
 ; AVX-LABEL: test_v4i32:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
@@ -923,7 +923,7 @@ define i32 @test_v4i32(<4 x i32> %a0) {
 ;
 ; AVX512-LABEL: test_v4i32:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
@@ -945,7 +945,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; SSE2-NEXT:    pand %xmm4, %xmm0
 ; SSE2-NEXT:    pandn %xmm1, %xmm4
 ; SSE2-NEXT:    por %xmm0, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[2,3,2,3]
 ; SSE2-NEXT:    movdqa %xmm4, %xmm1
 ; SSE2-NEXT:    pxor %xmm2, %xmm1
 ; SSE2-NEXT:    movdqa %xmm0, %xmm3
@@ -968,7 +968,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; SSE4-LABEL: test_v8i32:
 ; SSE4:       # %bb.0:
 ; SSE4-NEXT:    pmaxud %xmm1, %xmm0
-; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE4-NEXT:    pmaxud %xmm0, %xmm1
 ; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE4-NEXT:    pmaxud %xmm1, %xmm0
@@ -979,7 +979,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
@@ -991,7 +991,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
@@ -1003,7 +1003,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
@@ -1042,7 +1042,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; SSE2-NEXT:    pand %xmm1, %xmm5
 ; SSE2-NEXT:    pandn %xmm2, %xmm1
 ; SSE2-NEXT:    por %xmm5, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; SSE2-NEXT:    pxor %xmm4, %xmm2
 ; SSE2-NEXT:    movdqa %xmm0, %xmm3
@@ -1067,7 +1067,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; SSE4-NEXT:    pmaxud %xmm3, %xmm1
 ; SSE4-NEXT:    pmaxud %xmm2, %xmm1
 ; SSE4-NEXT:    pmaxud %xmm0, %xmm1
-; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE4-NEXT:    pmaxud %xmm1, %xmm0
 ; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE4-NEXT:    pmaxud %xmm0, %xmm1
@@ -1081,7 +1081,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX1-NEXT:    vpmaxud %xmm2, %xmm3, %xmm2
 ; AVX1-NEXT:    vpmaxud %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
@@ -1094,7 +1094,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX2-NEXT:    vpmaxud %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
@@ -1108,7 +1108,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX512-NEXT:    vpmaxud %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
@@ -1179,7 +1179,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; SSE2-NEXT:    pand %xmm2, %xmm0
 ; SSE2-NEXT:    pandn %xmm3, %xmm2
 ; SSE2-NEXT:    por %xmm0, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE2-NEXT:    movdqa %xmm2, %xmm1
 ; SSE2-NEXT:    pxor %xmm8, %xmm1
 ; SSE2-NEXT:    movdqa %xmm0, %xmm3
@@ -1208,7 +1208,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; SSE4-NEXT:    pmaxud %xmm4, %xmm2
 ; SSE4-NEXT:    pmaxud %xmm3, %xmm2
 ; SSE4-NEXT:    pmaxud %xmm0, %xmm2
-; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE4-NEXT:    pmaxud %xmm2, %xmm0
 ; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE4-NEXT:    pmaxud %xmm0, %xmm1
@@ -1228,7 +1228,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX1-NEXT:    vpmaxud %xmm4, %xmm2, %xmm2
 ; AVX1-NEXT:    vpmaxud %xmm1, %xmm2, %xmm1
 ; AVX1-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
@@ -1243,7 +1243,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX2-NEXT:    vpmaxud %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
@@ -1258,7 +1258,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX512-NEXT:    vpmaxud %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
@@ -1368,7 +1368,7 @@ define i16 @test_v4i16(<4 x i16> %a0) {
 define i16 @test_v8i16(<8 x i16> %a0) {
 ; SSE2-LABEL: test_v8i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [32768,32768,32768,32768,32768,32768,32768,32768]
 ; SSE2-NEXT:    pxor %xmm2, %xmm0
 ; SSE2-NEXT:    pxor %xmm2, %xmm1
@@ -1433,7 +1433,7 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; SSE2-NEXT:    pxor %xmm2, %xmm1
 ; SSE2-NEXT:    pxor %xmm2, %xmm0
 ; SSE2-NEXT:    pmaxsw %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    pmaxsw %xmm0, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE2-NEXT:    pmaxsw %xmm1, %xmm0
@@ -1520,7 +1520,7 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; SSE2-NEXT:    pmaxsw %xmm1, %xmm2
 ; SSE2-NEXT:    pxor %xmm4, %xmm0
 ; SSE2-NEXT:    pmaxsw %xmm2, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    pmaxsw %xmm0, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE2-NEXT:    pmaxsw %xmm1, %xmm0
@@ -1625,7 +1625,7 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; SSE2-NEXT:    pmaxsw %xmm5, %xmm1
 ; SSE2-NEXT:    pmaxsw %xmm4, %xmm1
 ; SSE2-NEXT:    pmaxsw %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    pmaxsw %xmm1, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE2-NEXT:    pmaxsw %xmm0, %xmm1
@@ -1839,7 +1839,7 @@ define i8 @test_v8i8(<8 x i8> %a0) {
 define i8 @test_v16i8(<16 x i8> %a0) {
 ; SSE2-LABEL: test_v16i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    pmaxub %xmm0, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE2-NEXT:    pmaxub %xmm1, %xmm0
@@ -1909,7 +1909,7 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; SSE2-LABEL: test_v32i8:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pmaxub %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    pmaxub %xmm0, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE2-NEXT:    pmaxub %xmm1, %xmm0
@@ -2004,7 +2004,7 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; SSE2-NEXT:    pmaxub %xmm3, %xmm1
 ; SSE2-NEXT:    pmaxub %xmm2, %xmm1
 ; SSE2-NEXT:    pmaxub %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    pmaxub %xmm1, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE2-NEXT:    pmaxub %xmm0, %xmm1
@@ -2113,7 +2113,7 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; SSE2-NEXT:    pmaxub %xmm4, %xmm2
 ; SSE2-NEXT:    pmaxub %xmm3, %xmm2
 ; SSE2-NEXT:    pmaxub %xmm0, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE2-NEXT:    pmaxub %xmm2, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE2-NEXT:    pmaxub %xmm0, %xmm1
diff --git a/llvm/test/CodeGen/X86/vector-reduce-umin.ll b/llvm/test/CodeGen/X86/vector-reduce-umin.ll
index 8f6a7266d97b9..9da8d61223efd 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-umin.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-umin.ll
@@ -14,7 +14,7 @@
 define i64 @test_v2i64(<2 x i64> %a0) {
 ; SSE2-LABEL: test_v2i64:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [9223372039002259456,9223372039002259456]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm3
 ; SSE2-NEXT:    pxor %xmm2, %xmm3
@@ -36,7 +36,7 @@ define i64 @test_v2i64(<2 x i64> %a0) {
 ; SSE41-LABEL: test_v2i64:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [9223372039002259456,9223372039002259456]
 ; SSE41-NEXT:    pxor %xmm3, %xmm0
 ; SSE41-NEXT:    pxor %xmm2, %xmm3
@@ -53,7 +53,7 @@ define i64 @test_v2i64(<2 x i64> %a0) {
 ; SSE42-LABEL: test_v2i64:
 ; SSE42:       # %bb.0:
 ; SSE42-NEXT:    movdqa %xmm0, %xmm1
-; SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; SSE42-NEXT:    movdqa {{.*#+}} xmm0 = [9223372036854775808,9223372036854775808]
 ; SSE42-NEXT:    movdqa %xmm1, %xmm3
 ; SSE42-NEXT:    pxor %xmm0, %xmm3
@@ -65,7 +65,7 @@ define i64 @test_v2i64(<2 x i64> %a0) {
 ;
 ; AVX-LABEL: test_v2i64:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
 ; AVX-NEXT:    vpxor %xmm2, %xmm0, %xmm3
 ; AVX-NEXT:    vpxor %xmm2, %xmm1, %xmm2
@@ -77,7 +77,7 @@ define i64 @test_v2i64(<2 x i64> %a0) {
 ; AVX512BW-LABEL: test_v2i64:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512BW-NEXT:    vpminuq %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vmovq %xmm0, %rax
 ; AVX512BW-NEXT:    vzeroupper
@@ -85,7 +85,7 @@ define i64 @test_v2i64(<2 x i64> %a0) {
 ;
 ; AVX512VL-LABEL: test_v2i64:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512VL-NEXT:    vpminuq %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vmovq %xmm0, %rax
 ; AVX512VL-NEXT:    retq
@@ -112,7 +112,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; SSE2-NEXT:    pand %xmm4, %xmm0
 ; SSE2-NEXT:    pandn %xmm1, %xmm4
 ; SSE2-NEXT:    por %xmm0, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[2,3,2,3]
 ; SSE2-NEXT:    movdqa %xmm4, %xmm1
 ; SSE2-NEXT:    pxor %xmm2, %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm2
@@ -144,7 +144,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; SSE41-NEXT:    pand %xmm5, %xmm0
 ; SSE41-NEXT:    por %xmm4, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
 ; SSE41-NEXT:    pxor %xmm3, %xmm0
 ; SSE41-NEXT:    pxor %xmm2, %xmm3
@@ -168,7 +168,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; SSE42-NEXT:    pxor %xmm3, %xmm0
 ; SSE42-NEXT:    pcmpgtq %xmm4, %xmm0
 ; SSE42-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
-; SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; SSE42-NEXT:    movdqa %xmm1, %xmm0
 ; SSE42-NEXT:    pxor %xmm3, %xmm0
 ; SSE42-NEXT:    pxor %xmm2, %xmm3
@@ -186,7 +186,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; AVX1-NEXT:    vpxor %xmm1, %xmm3, %xmm4
 ; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm4, %xmm2
 ; AVX1-NEXT:    vblendvpd %xmm2, %xmm0, %xmm3, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vxorpd %xmm1, %xmm0, %xmm3
 ; AVX1-NEXT:    vxorpd %xmm1, %xmm2, %xmm1
 ; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm1, %xmm1
@@ -203,7 +203,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm4
 ; AVX2-NEXT:    vpcmpgtq %xmm3, %xmm4, %xmm3
 ; AVX2-NEXT:    vblendvpd %xmm3, %xmm0, %xmm1, %xmm0
-; AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vxorpd %xmm2, %xmm0, %xmm3
 ; AVX2-NEXT:    vxorpd %xmm2, %xmm1, %xmm2
 ; AVX2-NEXT:    vpcmpgtq %xmm3, %xmm2, %xmm2
@@ -217,7 +217,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
 ; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512BW-NEXT:    vpminuq %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512BW-NEXT:    vpminuq %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vmovq %xmm0, %rax
 ; AVX512BW-NEXT:    vzeroupper
@@ -227,7 +227,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512VL-NEXT:    vpminuq %xmm1, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512VL-NEXT:    vpminuq %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vmovq %xmm0, %rax
 ; AVX512VL-NEXT:    vzeroupper
@@ -285,7 +285,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; SSE2-NEXT:    pand %xmm1, %xmm3
 ; SSE2-NEXT:    pandn %xmm5, %xmm1
 ; SSE2-NEXT:    por %xmm3, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; SSE2-NEXT:    pxor %xmm4, %xmm2
 ; SSE2-NEXT:    pxor %xmm0, %xmm4
@@ -340,7 +340,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; SSE41-NEXT:    pand %xmm4, %xmm0
 ; SSE41-NEXT:    por %xmm1, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[2,3,2,3]
 ; SSE41-NEXT:    movdqa %xmm3, %xmm0
 ; SSE41-NEXT:    pxor %xmm5, %xmm0
 ; SSE41-NEXT:    pxor %xmm1, %xmm5
@@ -376,7 +376,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; SSE42-NEXT:    xorpd %xmm4, %xmm0
 ; SSE42-NEXT:    pcmpgtq %xmm1, %xmm0
 ; SSE42-NEXT:    blendvpd %xmm0, %xmm2, %xmm3
-; SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[2,3,0,1]
+; SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[2,3,2,3]
 ; SSE42-NEXT:    movdqa %xmm3, %xmm0
 ; SSE42-NEXT:    pxor %xmm4, %xmm0
 ; SSE42-NEXT:    pxor %xmm1, %xmm4
@@ -403,7 +403,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; AVX1-NEXT:    vxorpd %xmm3, %xmm2, %xmm4
 ; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm4, %xmm1
 ; AVX1-NEXT:    vblendvpd %xmm1, %xmm0, %xmm2, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vxorpd %xmm3, %xmm0, %xmm2
 ; AVX1-NEXT:    vxorpd %xmm3, %xmm1, %xmm3
 ; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm2
@@ -424,7 +424,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; AVX2-NEXT:    vxorpd %xmm2, %xmm1, %xmm4
 ; AVX2-NEXT:    vpcmpgtq %xmm3, %xmm4, %xmm3
 ; AVX2-NEXT:    vblendvpd %xmm3, %xmm0, %xmm1, %xmm0
-; AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vxorpd %xmm2, %xmm0, %xmm3
 ; AVX2-NEXT:    vxorpd %xmm2, %xmm1, %xmm2
 ; AVX2-NEXT:    vpcmpgtq %xmm3, %xmm2, %xmm2
@@ -439,7 +439,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; AVX512BW-NEXT:    vpminuq %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512BW-NEXT:    vpminuq %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512BW-NEXT:    vpminuq %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vmovq %xmm0, %rax
 ; AVX512BW-NEXT:    vzeroupper
@@ -451,7 +451,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; AVX512VL-NEXT:    vpminuq %zmm1, %zmm0, %zmm0
 ; AVX512VL-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512VL-NEXT:    vpminuq %xmm1, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512VL-NEXT:    vpminuq %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vmovq %xmm0, %rax
 ; AVX512VL-NEXT:    vzeroupper
@@ -569,7 +569,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; SSE2-NEXT:    pand %xmm2, %xmm3
 ; SSE2-NEXT:    pandn %xmm1, %xmm2
 ; SSE2-NEXT:    por %xmm3, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE2-NEXT:    movdqa %xmm2, %xmm1
 ; SSE2-NEXT:    pxor %xmm8, %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm8
@@ -668,7 +668,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; SSE41-NEXT:    pand %xmm2, %xmm0
 ; SSE41-NEXT:    por %xmm1, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm6, %xmm7
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm7[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm7[2,3,2,3]
 ; SSE41-NEXT:    movdqa %xmm7, %xmm0
 ; SSE41-NEXT:    pxor %xmm9, %xmm0
 ; SSE41-NEXT:    pxor %xmm1, %xmm9
@@ -728,7 +728,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; SSE42-NEXT:    xorpd %xmm8, %xmm0
 ; SSE42-NEXT:    pcmpgtq %xmm1, %xmm0
 ; SSE42-NEXT:    blendvpd %xmm0, %xmm6, %xmm7
-; SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm7[2,3,0,1]
+; SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm7[2,3,2,3]
 ; SSE42-NEXT:    movdqa %xmm7, %xmm0
 ; SSE42-NEXT:    pxor %xmm8, %xmm0
 ; SSE42-NEXT:    pxor %xmm1, %xmm8
@@ -773,7 +773,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; AVX1-NEXT:    vxorpd %xmm4, %xmm2, %xmm3
 ; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm3, %xmm1
 ; AVX1-NEXT:    vblendvpd %xmm1, %xmm0, %xmm2, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vxorpd %xmm4, %xmm0, %xmm2
 ; AVX1-NEXT:    vxorpd %xmm4, %xmm1, %xmm3
 ; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm2
@@ -802,7 +802,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; AVX2-NEXT:    vxorpd %xmm4, %xmm1, %xmm3
 ; AVX2-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm2
 ; AVX2-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
-; AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vxorpd %xmm4, %xmm0, %xmm2
 ; AVX2-NEXT:    vxorpd %xmm4, %xmm1, %xmm3
 ; AVX2-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm2
@@ -818,7 +818,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; AVX512BW-NEXT:    vpminuq %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512BW-NEXT:    vpminuq %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512BW-NEXT:    vpminuq %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vmovq %xmm0, %rax
 ; AVX512BW-NEXT:    vzeroupper
@@ -831,7 +831,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; AVX512VL-NEXT:    vpminuq %zmm1, %zmm0, %zmm0
 ; AVX512VL-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512VL-NEXT:    vpminuq %xmm1, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512VL-NEXT:    vpminuq %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vmovq %xmm0, %rax
 ; AVX512VL-NEXT:    vzeroupper
@@ -886,7 +886,7 @@ define i32 @test_v2i32(<2 x i32> %a0) {
 define i32 @test_v4i32(<4 x i32> %a0) {
 ; SSE2-LABEL: test_v4i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm3
 ; SSE2-NEXT:    pxor %xmm2, %xmm3
@@ -909,7 +909,7 @@ define i32 @test_v4i32(<4 x i32> %a0) {
 ;
 ; SSE4-LABEL: test_v4i32:
 ; SSE4:       # %bb.0:
-; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE4-NEXT:    pminud %xmm0, %xmm1
 ; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE4-NEXT:    pminud %xmm1, %xmm0
@@ -918,7 +918,7 @@ define i32 @test_v4i32(<4 x i32> %a0) {
 ;
 ; AVX-LABEL: test_v4i32:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX-NEXT:    vpminud %xmm1, %xmm0, %xmm0
@@ -927,7 +927,7 @@ define i32 @test_v4i32(<4 x i32> %a0) {
 ;
 ; AVX512-LABEL: test_v4i32:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpminud %xmm1, %xmm0, %xmm0
@@ -949,7 +949,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; SSE2-NEXT:    pand %xmm4, %xmm0
 ; SSE2-NEXT:    pandn %xmm1, %xmm4
 ; SSE2-NEXT:    por %xmm0, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[2,3,2,3]
 ; SSE2-NEXT:    movdqa %xmm4, %xmm1
 ; SSE2-NEXT:    pxor %xmm2, %xmm1
 ; SSE2-NEXT:    movdqa %xmm0, %xmm3
@@ -972,7 +972,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; SSE4-LABEL: test_v8i32:
 ; SSE4:       # %bb.0:
 ; SSE4-NEXT:    pminud %xmm1, %xmm0
-; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE4-NEXT:    pminud %xmm0, %xmm1
 ; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE4-NEXT:    pminud %xmm1, %xmm0
@@ -983,7 +983,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpminud %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpminud %xmm1, %xmm0, %xmm0
@@ -995,7 +995,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpminud %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpminud %xmm1, %xmm0, %xmm0
@@ -1007,7 +1007,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpminud %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpminud %xmm1, %xmm0, %xmm0
@@ -1046,7 +1046,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; SSE2-NEXT:    pand %xmm1, %xmm3
 ; SSE2-NEXT:    pandn %xmm6, %xmm1
 ; SSE2-NEXT:    por %xmm3, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; SSE2-NEXT:    pxor %xmm4, %xmm2
 ; SSE2-NEXT:    movdqa %xmm0, %xmm3
@@ -1071,7 +1071,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; SSE4-NEXT:    pminud %xmm3, %xmm1
 ; SSE4-NEXT:    pminud %xmm2, %xmm1
 ; SSE4-NEXT:    pminud %xmm0, %xmm1
-; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE4-NEXT:    pminud %xmm1, %xmm0
 ; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE4-NEXT:    pminud %xmm0, %xmm1
@@ -1085,7 +1085,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX1-NEXT:    vpminud %xmm2, %xmm3, %xmm2
 ; AVX1-NEXT:    vpminud %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vpminud %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpminud %xmm1, %xmm0, %xmm0
@@ -1098,7 +1098,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX2-NEXT:    vpminud %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpminud %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpminud %xmm1, %xmm0, %xmm0
@@ -1112,7 +1112,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX512-NEXT:    vpminud %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpminud %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpminud %xmm1, %xmm0, %xmm0
@@ -1183,7 +1183,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; SSE2-NEXT:    pand %xmm2, %xmm3
 ; SSE2-NEXT:    pandn %xmm1, %xmm2
 ; SSE2-NEXT:    por %xmm3, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE2-NEXT:    movdqa %xmm2, %xmm1
 ; SSE2-NEXT:    pxor %xmm8, %xmm1
 ; SSE2-NEXT:    movdqa %xmm0, %xmm3
@@ -1212,7 +1212,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; SSE4-NEXT:    pminud %xmm4, %xmm2
 ; SSE4-NEXT:    pminud %xmm3, %xmm2
 ; SSE4-NEXT:    pminud %xmm0, %xmm2
-; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE4-NEXT:    pminud %xmm2, %xmm0
 ; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE4-NEXT:    pminud %xmm0, %xmm1
@@ -1232,7 +1232,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX1-NEXT:    vpminud %xmm4, %xmm2, %xmm2
 ; AVX1-NEXT:    vpminud %xmm1, %xmm2, %xmm1
 ; AVX1-NEXT:    vpminud %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpminud %xmm1, %xmm0, %xmm0
@@ -1247,7 +1247,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX2-NEXT:    vpminud %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpminud %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpminud %xmm1, %xmm0, %xmm0
@@ -1262,7 +1262,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX512-NEXT:    vpminud %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpminud %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpminud %xmm1, %xmm0, %xmm0
@@ -1372,7 +1372,7 @@ define i16 @test_v4i16(<4 x i16> %a0) {
 define i16 @test_v8i16(<8 x i16> %a0) {
 ; SSE2-LABEL: test_v8i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [32768,32768,32768,32768,32768,32768,32768,32768]
 ; SSE2-NEXT:    pxor %xmm2, %xmm0
 ; SSE2-NEXT:    pxor %xmm2, %xmm1
@@ -1418,7 +1418,7 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; SSE2-NEXT:    pxor %xmm2, %xmm1
 ; SSE2-NEXT:    pxor %xmm2, %xmm0
 ; SSE2-NEXT:    pminsw %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    pminsw %xmm0, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE2-NEXT:    pminsw %xmm1, %xmm0
@@ -1482,7 +1482,7 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; SSE2-NEXT:    pminsw %xmm1, %xmm2
 ; SSE2-NEXT:    pxor %xmm4, %xmm0
 ; SSE2-NEXT:    pminsw %xmm2, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    pminsw %xmm0, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE2-NEXT:    pminsw %xmm1, %xmm0
@@ -1562,7 +1562,7 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; SSE2-NEXT:    pminsw %xmm5, %xmm1
 ; SSE2-NEXT:    pminsw %xmm4, %xmm1
 ; SSE2-NEXT:    pminsw %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    pminsw %xmm1, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE2-NEXT:    pminsw %xmm0, %xmm1
@@ -1750,7 +1750,7 @@ define i8 @test_v8i8(<8 x i8> %a0) {
 define i8 @test_v16i8(<16 x i8> %a0) {
 ; SSE2-LABEL: test_v16i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    pminub %xmm0, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE2-NEXT:    pminub %xmm1, %xmm0
@@ -1799,7 +1799,7 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; SSE2-LABEL: test_v32i8:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pminub %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    pminub %xmm0, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE2-NEXT:    pminub %xmm1, %xmm0
@@ -1869,7 +1869,7 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; SSE2-NEXT:    pminub %xmm3, %xmm1
 ; SSE2-NEXT:    pminub %xmm2, %xmm1
 ; SSE2-NEXT:    pminub %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    pminub %xmm1, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE2-NEXT:    pminub %xmm0, %xmm1
@@ -1951,7 +1951,7 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; SSE2-NEXT:    pminub %xmm4, %xmm2
 ; SSE2-NEXT:    pminub %xmm3, %xmm2
 ; SSE2-NEXT:    pminub %xmm0, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE2-NEXT:    pminub %xmm2, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE2-NEXT:    pminub %xmm0, %xmm1
diff --git a/llvm/test/CodeGen/X86/vector-reduce-xor.ll b/llvm/test/CodeGen/X86/vector-reduce-xor.ll
index 35e6db38a584f..2d69190d9d18a 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-xor.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-xor.ll
@@ -13,14 +13,14 @@
 define i64 @test_v2i64(<2 x i64> %a0) {
 ; SSE-LABEL: test_v2i64:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    pxor %xmm0, %xmm1
 ; SSE-NEXT:    movq %xmm1, %rax
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: test_v2i64:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovq %xmm0, %rax
 ; AVX-NEXT:    retq
@@ -32,7 +32,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; SSE-LABEL: test_v4i64:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pxor %xmm1, %xmm0
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    pxor %xmm0, %xmm1
 ; SSE-NEXT:    movq %xmm1, %rax
 ; SSE-NEXT:    retq
@@ -41,7 +41,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovq %xmm0, %rax
 ; AVX1-NEXT:    vzeroupper
@@ -51,7 +51,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovq %xmm0, %rax
 ; AVX2-NEXT:    vzeroupper
@@ -61,7 +61,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovq %xmm0, %rax
 ; AVX512-NEXT:    vzeroupper
@@ -76,7 +76,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; SSE-NEXT:    pxor %xmm3, %xmm1
 ; SSE-NEXT:    pxor %xmm2, %xmm1
 ; SSE-NEXT:    pxor %xmm0, %xmm1
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE-NEXT:    pxor %xmm1, %xmm0
 ; SSE-NEXT:    movq %xmm0, %rax
 ; SSE-NEXT:    retq
@@ -86,7 +86,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovq %xmm0, %rax
 ; AVX1-NEXT:    vzeroupper
@@ -97,7 +97,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovq %xmm0, %rax
 ; AVX2-NEXT:    vzeroupper
@@ -109,7 +109,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; AVX512-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovq %xmm0, %rax
 ; AVX512-NEXT:    vzeroupper
@@ -128,7 +128,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; SSE-NEXT:    pxor %xmm4, %xmm2
 ; SSE-NEXT:    pxor %xmm3, %xmm2
 ; SSE-NEXT:    pxor %xmm0, %xmm2
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE-NEXT:    pxor %xmm2, %xmm0
 ; SSE-NEXT:    movq %xmm0, %rax
 ; SSE-NEXT:    retq
@@ -140,7 +140,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovq %xmm0, %rax
 ; AVX1-NEXT:    vzeroupper
@@ -153,7 +153,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovq %xmm0, %rax
 ; AVX2-NEXT:    vzeroupper
@@ -166,7 +166,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; AVX512-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovq %xmm0, %rax
 ; AVX512-NEXT:    vzeroupper
@@ -200,7 +200,7 @@ define i32 @test_v2i32(<2 x i32> %a0) {
 define i32 @test_v4i32(<4 x i32> %a0) {
 ; SSE-LABEL: test_v4i32:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    pxor %xmm0, %xmm1
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE-NEXT:    pxor %xmm1, %xmm0
@@ -209,7 +209,7 @@ define i32 @test_v4i32(<4 x i32> %a0) {
 ;
 ; AVX-LABEL: test_v4i32:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
@@ -223,7 +223,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; SSE-LABEL: test_v8i32:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pxor %xmm1, %xmm0
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    pxor %xmm0, %xmm1
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE-NEXT:    pxor %xmm1, %xmm0
@@ -234,7 +234,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
@@ -246,7 +246,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
@@ -258,7 +258,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
@@ -275,7 +275,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; SSE-NEXT:    pxor %xmm3, %xmm1
 ; SSE-NEXT:    pxor %xmm2, %xmm1
 ; SSE-NEXT:    pxor %xmm0, %xmm1
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE-NEXT:    pxor %xmm1, %xmm0
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE-NEXT:    pxor %xmm0, %xmm1
@@ -287,7 +287,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
@@ -300,7 +300,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
@@ -314,7 +314,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX512-NEXT:    vpxord %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
@@ -335,7 +335,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; SSE-NEXT:    pxor %xmm4, %xmm2
 ; SSE-NEXT:    pxor %xmm3, %xmm2
 ; SSE-NEXT:    pxor %xmm0, %xmm2
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE-NEXT:    pxor %xmm2, %xmm0
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE-NEXT:    pxor %xmm0, %xmm1
@@ -349,7 +349,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
@@ -364,7 +364,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
@@ -379,7 +379,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX512-NEXT:    vpxord %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
@@ -443,7 +443,7 @@ define i16 @test_v4i16(<4 x i16> %a0) {
 define i16 @test_v8i16(<8 x i16> %a0) {
 ; SSE-LABEL: test_v8i16:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    pxor %xmm0, %xmm1
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE-NEXT:    pxor %xmm1, %xmm0
@@ -456,7 +456,7 @@ define i16 @test_v8i16(<8 x i16> %a0) {
 ;
 ; AVX-LABEL: test_v8i16:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
@@ -473,7 +473,7 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; SSE-LABEL: test_v16i16:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pxor %xmm1, %xmm0
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    pxor %xmm0, %xmm1
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE-NEXT:    pxor %xmm1, %xmm0
@@ -488,7 +488,7 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
@@ -503,7 +503,7 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
@@ -518,7 +518,7 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
@@ -538,7 +538,7 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; SSE-NEXT:    pxor %xmm3, %xmm1
 ; SSE-NEXT:    pxor %xmm2, %xmm1
 ; SSE-NEXT:    pxor %xmm0, %xmm1
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE-NEXT:    pxor %xmm1, %xmm0
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE-NEXT:    pxor %xmm0, %xmm1
@@ -554,7 +554,7 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
@@ -570,7 +570,7 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
@@ -587,7 +587,7 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX512-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
@@ -611,7 +611,7 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; SSE-NEXT:    pxor %xmm4, %xmm2
 ; SSE-NEXT:    pxor %xmm3, %xmm2
 ; SSE-NEXT:    pxor %xmm0, %xmm2
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE-NEXT:    pxor %xmm2, %xmm0
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE-NEXT:    pxor %xmm0, %xmm1
@@ -629,7 +629,7 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
@@ -647,7 +647,7 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
@@ -665,7 +665,7 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX512-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
@@ -763,7 +763,7 @@ define i8 @test_v8i8(<8 x i8> %a0) {
 define i8 @test_v16i8(<16 x i8> %a0) {
 ; SSE-LABEL: test_v16i8:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    pxor %xmm0, %xmm1
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE-NEXT:    pxor %xmm1, %xmm0
@@ -779,7 +779,7 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ;
 ; AVX-LABEL: test_v16i8:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
@@ -798,7 +798,7 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; SSE-LABEL: test_v32i8:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pxor %xmm1, %xmm0
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    pxor %xmm0, %xmm1
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
 ; SSE-NEXT:    pxor %xmm1, %xmm0
@@ -816,7 +816,7 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
@@ -833,7 +833,7 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
@@ -850,7 +850,7 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
@@ -872,7 +872,7 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; SSE-NEXT:    pxor %xmm3, %xmm1
 ; SSE-NEXT:    pxor %xmm2, %xmm1
 ; SSE-NEXT:    pxor %xmm0, %xmm1
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE-NEXT:    pxor %xmm1, %xmm0
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE-NEXT:    pxor %xmm0, %xmm1
@@ -891,7 +891,7 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
@@ -909,7 +909,7 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
@@ -928,7 +928,7 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX512-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
@@ -954,7 +954,7 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; SSE-NEXT:    pxor %xmm4, %xmm2
 ; SSE-NEXT:    pxor %xmm3, %xmm2
 ; SSE-NEXT:    pxor %xmm0, %xmm2
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE-NEXT:    pxor %xmm2, %xmm0
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE-NEXT:    pxor %xmm0, %xmm1
@@ -975,7 +975,7 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
@@ -995,7 +995,7 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
@@ -1015,7 +1015,7 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX512-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
diff --git a/llvm/test/CodeGen/X86/vector-rem.ll b/llvm/test/CodeGen/X86/vector-rem.ll
index 15e1198696659..deaab1c9161b9 100644
--- a/llvm/test/CodeGen/X86/vector-rem.ll
+++ b/llvm/test/CodeGen/X86/vector-rem.ll
@@ -11,9 +11,9 @@ define <4 x i32> @foo(<4 x i32> %t, <4 x i32> %u) nounwind {
 ; CHECK-NEXT:    cltd
 ; CHECK-NEXT:    idivl %ecx
 ; CHECK-NEXT:    movd %edx, %xmm2
-; CHECK-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,0,1]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
 ; CHECK-NEXT:    movd %xmm3, %eax
-; CHECK-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,0,1]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
 ; CHECK-NEXT:    movd %xmm3, %ecx
 ; CHECK-NEXT:    cltd
 ; CHECK-NEXT:    idivl %ecx
@@ -49,9 +49,9 @@ define <4 x i32> @bar(<4 x i32> %t, <4 x i32> %u) nounwind {
 ; CHECK-NEXT:    xorl %edx, %edx
 ; CHECK-NEXT:    divl %ecx
 ; CHECK-NEXT:    movd %edx, %xmm2
-; CHECK-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,0,1]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
 ; CHECK-NEXT:    movd %xmm3, %eax
-; CHECK-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,0,1]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
 ; CHECK-NEXT:    movd %xmm3, %ecx
 ; CHECK-NEXT:    xorl %edx, %edx
 ; CHECK-NEXT:    divl %ecx
diff --git a/llvm/test/CodeGen/X86/vector-rotate-128.ll b/llvm/test/CodeGen/X86/vector-rotate-128.ll
index 67dd15ee87abe..d140fb5c09295 100644
--- a/llvm/test/CodeGen/X86/vector-rotate-128.ll
+++ b/llvm/test/CodeGen/X86/vector-rotate-128.ll
@@ -24,13 +24,13 @@ define <2 x i64> @var_rotate_v2i64(<2 x i64> %a, <2 x i64> %b) nounwind {
 ; SSE2-NEXT:    psubq %xmm1, %xmm2
 ; SSE2-NEXT:    movdqa %xmm0, %xmm3
 ; SSE2-NEXT:    psllq %xmm1, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm4
 ; SSE2-NEXT:    psllq %xmm1, %xmm4
 ; SSE2-NEXT:    movsd {{.*#+}} xmm4 = xmm3[0],xmm4[1]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; SSE2-NEXT:    psrlq %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
 ; SSE2-NEXT:    psrlq %xmm2, %xmm0
 ; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
 ; SSE2-NEXT:    orpd %xmm4, %xmm0
@@ -42,13 +42,13 @@ define <2 x i64> @var_rotate_v2i64(<2 x i64> %a, <2 x i64> %b) nounwind {
 ; SSE41-NEXT:    psubq %xmm1, %xmm2
 ; SSE41-NEXT:    movdqa %xmm0, %xmm3
 ; SSE41-NEXT:    psllq %xmm1, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE41-NEXT:    movdqa %xmm0, %xmm4
 ; SSE41-NEXT:    psllq %xmm1, %xmm4
 ; SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm3[0,1,2,3],xmm4[4,5,6,7]
 ; SSE41-NEXT:    movdqa %xmm0, %xmm1
 ; SSE41-NEXT:    psrlq %xmm2, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
 ; SSE41-NEXT:    psrlq %xmm2, %xmm0
 ; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7]
 ; SSE41-NEXT:    por %xmm4, %xmm0
@@ -59,11 +59,11 @@ define <2 x i64> @var_rotate_v2i64(<2 x i64> %a, <2 x i64> %b) nounwind {
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [64,64]
 ; AVX1-NEXT:    vpsubq %xmm1, %xmm2, %xmm2
 ; AVX1-NEXT:    vpsllq %xmm1, %xmm0, %xmm3
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; AVX1-NEXT:    vpsllq %xmm1, %xmm0, %xmm1
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm3[0,1,2,3],xmm1[4,5,6,7]
 ; AVX1-NEXT:    vpsrlq %xmm2, %xmm0, %xmm3
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
 ; AVX1-NEXT:    vpsrlq %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm3[0,1,2,3],xmm0[4,5,6,7]
 ; AVX1-NEXT:    vpor %xmm0, %xmm1, %xmm0
@@ -117,13 +117,13 @@ define <2 x i64> @var_rotate_v2i64(<2 x i64> %a, <2 x i64> %b) nounwind {
 ; X32-SSE-NEXT:    psubq %xmm1, %xmm2
 ; X32-SSE-NEXT:    movdqa %xmm0, %xmm3
 ; X32-SSE-NEXT:    psllq %xmm1, %xmm3
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; X32-SSE-NEXT:    movdqa %xmm0, %xmm4
 ; X32-SSE-NEXT:    psllq %xmm1, %xmm4
 ; X32-SSE-NEXT:    movsd {{.*#+}} xmm4 = xmm3[0],xmm4[1]
 ; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
 ; X32-SSE-NEXT:    psrlq %xmm2, %xmm1
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
 ; X32-SSE-NEXT:    psrlq %xmm2, %xmm0
 ; X32-SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
 ; X32-SSE-NEXT:    orpd %xmm4, %xmm0
@@ -719,7 +719,7 @@ define <2 x i64> @splatvar_rotate_v2i64(<2 x i64> %a, <2 x i64> %b) nounwind {
 ; X32-SSE-NEXT:    psllq %xmm1, %xmm2
 ; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
 ; X32-SSE-NEXT:    psrlq %xmm3, %xmm1
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[2,3,0,1]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]
 ; X32-SSE-NEXT:    psrlq %xmm3, %xmm0
 ; X32-SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
 ; X32-SSE-NEXT:    orpd %xmm2, %xmm0
diff --git a/llvm/test/CodeGen/X86/vector-rotate-256.ll b/llvm/test/CodeGen/X86/vector-rotate-256.ll
index 31fe575024753..a850ab5ba7822 100644
--- a/llvm/test/CodeGen/X86/vector-rotate-256.ll
+++ b/llvm/test/CodeGen/X86/vector-rotate-256.ll
@@ -21,20 +21,20 @@ define <4 x i64> @var_rotate_v4i64(<4 x i64> %a, <4 x i64> %b) nounwind {
 ; AVX1-NEXT:    vpsubq %xmm4, %xmm2, %xmm2
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm5
 ; AVX1-NEXT:    vpsllq %xmm4, %xmm5, %xmm6
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[2,3,2,3]
 ; AVX1-NEXT:    vpsllq %xmm4, %xmm5, %xmm4
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm4 = xmm6[0,1,2,3],xmm4[4,5,6,7]
 ; AVX1-NEXT:    vpsllq %xmm1, %xmm0, %xmm6
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; AVX1-NEXT:    vpsllq %xmm1, %xmm0, %xmm1
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm6[0,1,2,3],xmm1[4,5,6,7]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm1, %ymm1
 ; AVX1-NEXT:    vpsrlq %xmm2, %xmm5, %xmm4
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
 ; AVX1-NEXT:    vpsrlq %xmm2, %xmm5, %xmm2
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm4[0,1,2,3],xmm2[4,5,6,7]
 ; AVX1-NEXT:    vpsrlq %xmm3, %xmm0, %xmm4
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]
 ; AVX1-NEXT:    vpsrlq %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm4[0,1,2,3],xmm0[4,5,6,7]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
diff --git a/llvm/test/CodeGen/X86/vector-sext.ll b/llvm/test/CodeGen/X86/vector-sext.ll
index d162f5c4a97ab..2a4efccc07c7f 100644
--- a/llvm/test/CodeGen/X86/vector-sext.ll
+++ b/llvm/test/CodeGen/X86/vector-sext.ll
@@ -72,7 +72,7 @@ define <16 x i16> @sext_16i8_to_16i16(<16 x i8> %A) nounwind uwtable readnone ss
 ; SSE41-LABEL: sext_16i8_to_16i16:
 ; SSE41:       # %bb.0: # %entry
 ; SSE41-NEXT:    pmovsxbw %xmm0, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovsxbw %xmm0, %xmm1
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; SSE41-NEXT:    retq
@@ -80,7 +80,7 @@ define <16 x i16> @sext_16i8_to_16i16(<16 x i8> %A) nounwind uwtable readnone ss
 ; AVX1-LABEL: sext_16i8_to_16i16:
 ; AVX1:       # %bb.0: # %entry
 ; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
@@ -107,7 +107,7 @@ define <16 x i16> @sext_16i8_to_16i16(<16 x i8> %A) nounwind uwtable readnone ss
 ; X32-SSE41-LABEL: sext_16i8_to_16i16:
 ; X32-SSE41:       # %bb.0: # %entry
 ; X32-SSE41-NEXT:    pmovsxbw %xmm0, %xmm2
-; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; X32-SSE41-NEXT:    pmovsxbw %xmm0, %xmm1
 ; X32-SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; X32-SSE41-NEXT:    retl
@@ -149,9 +149,9 @@ define <32 x i16> @sext_32i8_to_32i16(<32 x i8> %A) nounwind uwtable readnone ss
 ; SSE41:       # %bb.0: # %entry
 ; SSE41-NEXT:    pmovsxbw %xmm0, %xmm5
 ; SSE41-NEXT:    pmovsxbw %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovsxbw %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE41-NEXT:    pmovsxbw %xmm0, %xmm3
 ; SSE41-NEXT:    movdqa %xmm5, %xmm0
 ; SSE41-NEXT:    movdqa %xmm4, %xmm1
@@ -160,12 +160,12 @@ define <32 x i16> @sext_32i8_to_32i16(<32 x i8> %A) nounwind uwtable readnone ss
 ; AVX1-LABEL: sext_32i8_to_32i16:
 ; AVX1:       # %bb.0: # %entry
 ; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbw %xmm2, %xmm2
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm2
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
 ; AVX1-NEXT:    vmovaps %ymm2, %ymm0
@@ -210,9 +210,9 @@ define <32 x i16> @sext_32i8_to_32i16(<32 x i8> %A) nounwind uwtable readnone ss
 ; X32-SSE41:       # %bb.0: # %entry
 ; X32-SSE41-NEXT:    pmovsxbw %xmm0, %xmm5
 ; X32-SSE41-NEXT:    pmovsxbw %xmm1, %xmm2
-; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; X32-SSE41-NEXT:    pmovsxbw %xmm0, %xmm4
-; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; X32-SSE41-NEXT:    pmovsxbw %xmm0, %xmm3
 ; X32-SSE41-NEXT:    movdqa %xmm5, %xmm0
 ; X32-SSE41-NEXT:    movdqa %xmm4, %xmm1
@@ -367,7 +367,7 @@ define <16 x i32> @sext_16i8_to_16i32(<16 x i8> %A) nounwind uwtable readnone ss
 ; SSE41-NEXT:    pmovsxbd %xmm0, %xmm4
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE41-NEXT:    pmovsxbd %xmm1, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovsxbd %xmm2, %xmm2
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
 ; SSE41-NEXT:    pmovsxbd %xmm0, %xmm3
@@ -380,9 +380,9 @@ define <16 x i32> @sext_16i8_to_16i32(<16 x i8> %A) nounwind uwtable readnone ss
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm2
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,3,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
 ; AVX1-NEXT:    vmovaps %ymm2, %ymm0
@@ -391,7 +391,7 @@ define <16 x i32> @sext_16i8_to_16i32(<16 x i8> %A) nounwind uwtable readnone ss
 ; AVX2-LABEL: sext_16i8_to_16i32:
 ; AVX2:       # %bb.0: # %entry
 ; AVX2-NEXT:    vpmovsxbd %xmm0, %ymm2
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpmovsxbd %xmm0, %ymm1
 ; AVX2-NEXT:    vmovdqa %ymm2, %ymm0
 ; AVX2-NEXT:    retq
@@ -421,7 +421,7 @@ define <16 x i32> @sext_16i8_to_16i32(<16 x i8> %A) nounwind uwtable readnone ss
 ; X32-SSE41-NEXT:    pmovsxbd %xmm0, %xmm4
 ; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X32-SSE41-NEXT:    pmovsxbd %xmm1, %xmm1
-; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; X32-SSE41-NEXT:    pmovsxbd %xmm2, %xmm2
 ; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
 ; X32-SSE41-NEXT:    pmovsxbd %xmm0, %xmm3
@@ -736,7 +736,7 @@ define <8 x i32> @sext_8i16_to_8i32(<8 x i16> %A) nounwind uwtable readnone ssp
 ; SSE41-LABEL: sext_8i16_to_8i32:
 ; SSE41:       # %bb.0: # %entry
 ; SSE41-NEXT:    pmovsxwd %xmm0, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovsxwd %xmm0, %xmm1
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; SSE41-NEXT:    retq
@@ -744,7 +744,7 @@ define <8 x i32> @sext_8i16_to_8i32(<8 x i16> %A) nounwind uwtable readnone ssp
 ; AVX1-LABEL: sext_8i16_to_8i32:
 ; AVX1:       # %bb.0: # %entry
 ; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
@@ -771,7 +771,7 @@ define <8 x i32> @sext_8i16_to_8i32(<8 x i16> %A) nounwind uwtable readnone ssp
 ; X32-SSE41-LABEL: sext_8i16_to_8i32:
 ; X32-SSE41:       # %bb.0: # %entry
 ; X32-SSE41-NEXT:    pmovsxwd %xmm0, %xmm2
-; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; X32-SSE41-NEXT:    pmovsxwd %xmm0, %xmm1
 ; X32-SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; X32-SSE41-NEXT:    retl
@@ -813,9 +813,9 @@ define <16 x i32> @sext_16i16_to_16i32(<16 x i16> %A) nounwind uwtable readnone
 ; SSE41:       # %bb.0: # %entry
 ; SSE41-NEXT:    pmovsxwd %xmm0, %xmm5
 ; SSE41-NEXT:    pmovsxwd %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovsxwd %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE41-NEXT:    pmovsxwd %xmm0, %xmm3
 ; SSE41-NEXT:    movdqa %xmm5, %xmm0
 ; SSE41-NEXT:    movdqa %xmm4, %xmm1
@@ -824,12 +824,12 @@ define <16 x i32> @sext_16i16_to_16i32(<16 x i16> %A) nounwind uwtable readnone
 ; AVX1-LABEL: sext_16i16_to_16i32:
 ; AVX1:       # %bb.0: # %entry
 ; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxwd %xmm2, %xmm2
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm2
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
 ; AVX1-NEXT:    vmovaps %ymm2, %ymm0
@@ -866,9 +866,9 @@ define <16 x i32> @sext_16i16_to_16i32(<16 x i16> %A) nounwind uwtable readnone
 ; X32-SSE41:       # %bb.0: # %entry
 ; X32-SSE41-NEXT:    pmovsxwd %xmm0, %xmm5
 ; X32-SSE41-NEXT:    pmovsxwd %xmm1, %xmm2
-; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; X32-SSE41-NEXT:    pmovsxwd %xmm0, %xmm4
-; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; X32-SSE41-NEXT:    pmovsxwd %xmm0, %xmm3
 ; X32-SSE41-NEXT:    movdqa %xmm5, %xmm0
 ; X32-SSE41-NEXT:    movdqa %xmm4, %xmm1
@@ -1043,7 +1043,7 @@ define <8 x i64> @sext_8i16_to_8i64(<8 x i16> %A) nounwind uwtable readnone ssp
 ; SSE41-NEXT:    pmovsxwq %xmm0, %xmm4
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE41-NEXT:    pmovsxwq %xmm1, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovsxwq %xmm2, %xmm2
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
 ; SSE41-NEXT:    pmovsxwq %xmm0, %xmm3
@@ -1056,9 +1056,9 @@ define <8 x i64> @sext_8i16_to_8i64(<8 x i16> %A) nounwind uwtable readnone ssp
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpmovsxwq %xmm2, %xmm2
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxwq %xmm1, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,3,2,3]
 ; AVX1-NEXT:    vpmovsxwq %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
 ; AVX1-NEXT:    vmovaps %ymm2, %ymm0
@@ -1067,7 +1067,7 @@ define <8 x i64> @sext_8i16_to_8i64(<8 x i16> %A) nounwind uwtable readnone ssp
 ; AVX2-LABEL: sext_8i16_to_8i64:
 ; AVX2:       # %bb.0: # %entry
 ; AVX2-NEXT:    vpmovsxwq %xmm0, %ymm2
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpmovsxwq %xmm0, %ymm1
 ; AVX2-NEXT:    vmovdqa %ymm2, %ymm0
 ; AVX2-NEXT:    retq
@@ -1101,7 +1101,7 @@ define <8 x i64> @sext_8i16_to_8i64(<8 x i16> %A) nounwind uwtable readnone ssp
 ; X32-SSE41-NEXT:    pmovsxwq %xmm0, %xmm4
 ; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; X32-SSE41-NEXT:    pmovsxwq %xmm1, %xmm1
-; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; X32-SSE41-NEXT:    pmovsxwq %xmm2, %xmm2
 ; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
 ; X32-SSE41-NEXT:    pmovsxwq %xmm0, %xmm3
@@ -1160,7 +1160,7 @@ define <4 x i64> @sext_4i32_to_4i64(<4 x i32> %A) nounwind uwtable readnone ssp
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
 ; SSE2-NEXT:    pxor %xmm3, %xmm3
 ; SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
@@ -1171,7 +1171,7 @@ define <4 x i64> @sext_4i32_to_4i64(<4 x i32> %A) nounwind uwtable readnone ssp
 ; SSSE3-NEXT:    pxor %xmm2, %xmm2
 ; SSSE3-NEXT:    pxor %xmm3, %xmm3
 ; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
 ; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
 ; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
@@ -1180,7 +1180,7 @@ define <4 x i64> @sext_4i32_to_4i64(<4 x i32> %A) nounwind uwtable readnone ssp
 ; SSE41-LABEL: sext_4i32_to_4i64:
 ; SSE41:       # %bb.0: # %entry
 ; SSE41-NEXT:    pmovsxdq %xmm0, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovsxdq %xmm0, %xmm1
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; SSE41-NEXT:    retq
@@ -1188,7 +1188,7 @@ define <4 x i64> @sext_4i32_to_4i64(<4 x i32> %A) nounwind uwtable readnone ssp
 ; AVX1-LABEL: sext_4i32_to_4i64:
 ; AVX1:       # %bb.0: # %entry
 ; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
@@ -1208,7 +1208,7 @@ define <4 x i64> @sext_4i32_to_4i64(<4 x i32> %A) nounwind uwtable readnone ssp
 ; X32-SSE2-NEXT:    pxor %xmm2, %xmm2
 ; X32-SSE2-NEXT:    pxor %xmm3, %xmm3
 ; X32-SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
-; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
 ; X32-SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
 ; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
@@ -1217,7 +1217,7 @@ define <4 x i64> @sext_4i32_to_4i64(<4 x i32> %A) nounwind uwtable readnone ssp
 ; X32-SSE41-LABEL: sext_4i32_to_4i64:
 ; X32-SSE41:       # %bb.0: # %entry
 ; X32-SSE41-NEXT:    pmovsxdq %xmm0, %xmm2
-; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; X32-SSE41-NEXT:    pmovsxdq %xmm0, %xmm1
 ; X32-SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; X32-SSE41-NEXT:    retl
@@ -1235,12 +1235,12 @@ define <8 x i64> @sext_8i32_to_8i64(<8 x i32> %A) nounwind uwtable readnone ssp
 ; SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
 ; SSE2-NEXT:    pxor %xmm5, %xmm5
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
 ; SSE2-NEXT:    pxor %xmm3, %xmm3
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
 ; SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
@@ -1254,12 +1254,12 @@ define <8 x i64> @sext_8i32_to_8i64(<8 x i32> %A) nounwind uwtable readnone ssp
 ; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm3
 ; SSSE3-NEXT:    pxor %xmm5, %xmm5
 ; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm5
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
 ; SSSE3-NEXT:    pxor %xmm3, %xmm3
 ; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm3
 ; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[2,3,0,1]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
 ; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
 ; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm4
 ; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
@@ -1269,9 +1269,9 @@ define <8 x i64> @sext_8i32_to_8i64(<8 x i32> %A) nounwind uwtable readnone ssp
 ; SSE41:       # %bb.0: # %entry
 ; SSE41-NEXT:    pmovsxdq %xmm0, %xmm5
 ; SSE41-NEXT:    pmovsxdq %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovsxdq %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE41-NEXT:    pmovsxdq %xmm0, %xmm3
 ; SSE41-NEXT:    movdqa %xmm5, %xmm0
 ; SSE41-NEXT:    movdqa %xmm4, %xmm1
@@ -1280,12 +1280,12 @@ define <8 x i64> @sext_8i32_to_8i64(<8 x i32> %A) nounwind uwtable readnone ssp
 ; AVX1-LABEL: sext_8i32_to_8i64:
 ; AVX1:       # %bb.0: # %entry
 ; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxdq %xmm2, %xmm2
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm2
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
 ; AVX1-NEXT:    vmovaps %ymm2, %ymm0
@@ -1312,12 +1312,12 @@ define <8 x i64> @sext_8i32_to_8i64(<8 x i32> %A) nounwind uwtable readnone ssp
 ; X32-SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
 ; X32-SSE2-NEXT:    pxor %xmm5, %xmm5
 ; X32-SSE2-NEXT:    pcmpgtd %xmm1, %xmm5
-; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
 ; X32-SSE2-NEXT:    pxor %xmm3, %xmm3
 ; X32-SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
 ; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
-; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[2,3,0,1]
+; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
 ; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
 ; X32-SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
 ; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
@@ -1327,9 +1327,9 @@ define <8 x i64> @sext_8i32_to_8i64(<8 x i32> %A) nounwind uwtable readnone ssp
 ; X32-SSE41:       # %bb.0: # %entry
 ; X32-SSE41-NEXT:    pmovsxdq %xmm0, %xmm5
 ; X32-SSE41-NEXT:    pmovsxdq %xmm1, %xmm2
-; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; X32-SSE41-NEXT:    pmovsxdq %xmm0, %xmm4
-; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; X32-SSE41-NEXT:    pmovsxdq %xmm0, %xmm3
 ; X32-SSE41-NEXT:    movdqa %xmm5, %xmm0
 ; X32-SSE41-NEXT:    movdqa %xmm4, %xmm1
@@ -1849,7 +1849,7 @@ define <4 x i64> @load_sext_4i1_to_4i64(<4 x i1> *%ptr) {
 ; AVX1-NEXT:    negl %eax
 ; AVX1-NEXT:    vpinsrd $3, %eax, %xmm0, %xmm0
 ; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
@@ -3046,7 +3046,7 @@ define <4 x i64> @load_sext_4i32_to_4i64(<4 x i32> *%ptr) {
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
 ; SSE2-NEXT:    pxor %xmm3, %xmm3
 ; SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
@@ -3058,7 +3058,7 @@ define <4 x i64> @load_sext_4i32_to_4i64(<4 x i32> *%ptr) {
 ; SSSE3-NEXT:    pxor %xmm2, %xmm2
 ; SSSE3-NEXT:    pxor %xmm3, %xmm3
 ; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
 ; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
 ; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
@@ -3094,7 +3094,7 @@ define <4 x i64> @load_sext_4i32_to_4i64(<4 x i32> *%ptr) {
 ; X32-SSE2-NEXT:    pxor %xmm2, %xmm2
 ; X32-SSE2-NEXT:    pxor %xmm3, %xmm3
 ; X32-SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
-; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
 ; X32-SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
 ; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
@@ -3166,7 +3166,7 @@ define <4 x i64> @sext_4i1_to_4i64(<4 x i1> %mask) {
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
 ; SSE2-NEXT:    pxor %xmm3, %xmm3
 ; SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
@@ -3179,7 +3179,7 @@ define <4 x i64> @sext_4i1_to_4i64(<4 x i1> %mask) {
 ; SSSE3-NEXT:    pxor %xmm2, %xmm2
 ; SSSE3-NEXT:    pxor %xmm3, %xmm3
 ; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
 ; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
 ; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
@@ -3190,7 +3190,7 @@ define <4 x i64> @sext_4i1_to_4i64(<4 x i1> %mask) {
 ; SSE41-NEXT:    pslld $31, %xmm0
 ; SSE41-NEXT:    psrad $31, %xmm0
 ; SSE41-NEXT:    pmovsxdq %xmm0, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovsxdq %xmm0, %xmm1
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; SSE41-NEXT:    retq
@@ -3200,7 +3200,7 @@ define <4 x i64> @sext_4i1_to_4i64(<4 x i1> %mask) {
 ; AVX1-NEXT:    vpslld $31, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrad $31, %xmm0, %xmm0
 ; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
@@ -3226,7 +3226,7 @@ define <4 x i64> @sext_4i1_to_4i64(<4 x i1> %mask) {
 ; X32-SSE2-NEXT:    pxor %xmm2, %xmm2
 ; X32-SSE2-NEXT:    pxor %xmm3, %xmm3
 ; X32-SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
-; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
 ; X32-SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
 ; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
@@ -3237,7 +3237,7 @@ define <4 x i64> @sext_4i1_to_4i64(<4 x i1> %mask) {
 ; X32-SSE41-NEXT:    pslld $31, %xmm0
 ; X32-SSE41-NEXT:    psrad $31, %xmm0
 ; X32-SSE41-NEXT:    pmovsxdq %xmm0, %xmm2
-; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; X32-SSE41-NEXT:    pmovsxdq %xmm0, %xmm1
 ; X32-SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; X32-SSE41-NEXT:    retl
@@ -3737,7 +3737,7 @@ define <8 x i64> @sext_8i6_to_8i64(i32 %x) nounwind uwtable readnone ssp {
 ; SSE41-NEXT:    psrad $26, %xmm1
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]
 ; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
 ; SSE41-NEXT:    psllq $58, %xmm2
 ; SSE41-NEXT:    movdqa %xmm2, %xmm4
@@ -3767,9 +3767,9 @@ define <8 x i64> @sext_8i6_to_8i64(i32 %x) nounwind uwtable readnone ssp {
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,2,3]
 ; AVX1-NEXT:    vpmovsxwq %xmm2, %xmm2
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxwq %xmm2, %xmm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,2,3]
 ; AVX1-NEXT:    vpmovsxwq %xmm1, %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
 ; AVX1-NEXT:    retq
@@ -3782,7 +3782,7 @@ define <8 x i64> @sext_8i6_to_8i64(i32 %x) nounwind uwtable readnone ssp {
 ; AVX2-NEXT:    vpsllw $10, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsraw $10, %xmm0, %xmm1
 ; AVX2-NEXT:    vpmovsxwq %xmm1, %ymm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; AVX2-NEXT:    vpmovsxwq %xmm1, %ymm1
 ; AVX2-NEXT:    retq
 ;
@@ -3861,7 +3861,7 @@ define <8 x i64> @sext_8i6_to_8i64(i32 %x) nounwind uwtable readnone ssp {
 ; X32-SSE41-NEXT:    psrad $26, %xmm1
 ; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; X32-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
-; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,0,1]
+; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]
 ; X32-SSE41-NEXT:    pmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
 ; X32-SSE41-NEXT:    psllq $58, %xmm2
 ; X32-SSE41-NEXT:    movdqa %xmm2, %xmm4
@@ -3916,7 +3916,7 @@ define <8 x i32> @zext_negate_sext(<8 x i8> %x) {
 ; SSE41-NEXT:    pxor %xmm1, %xmm1
 ; SSE41-NEXT:    psubw %xmm0, %xmm1
 ; SSE41-NEXT:    pmovsxwd %xmm1, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE41-NEXT:    pmovsxwd %xmm1, %xmm1
 ; SSE41-NEXT:    retq
 ;
@@ -3962,7 +3962,7 @@ define <8 x i32> @zext_negate_sext(<8 x i8> %x) {
 ; X32-SSE41-NEXT:    pxor %xmm1, %xmm1
 ; X32-SSE41-NEXT:    psubw %xmm0, %xmm1
 ; X32-SSE41-NEXT:    pmovsxwd %xmm1, %xmm0
-; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; X32-SSE41-NEXT:    pmovsxwd %xmm1, %xmm1
 ; X32-SSE41-NEXT:    retl
   %z = zext <8 x i8> %x to <8 x i16>
@@ -4002,7 +4002,7 @@ define <8 x i32> @zext_decremenet_sext(<8 x i8> %x) {
 ; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
 ; SSE41-NEXT:    paddw %xmm0, %xmm1
 ; SSE41-NEXT:    pmovsxwd %xmm1, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE41-NEXT:    pmovsxwd %xmm1, %xmm1
 ; SSE41-NEXT:    retq
 ;
@@ -4049,7 +4049,7 @@ define <8 x i32> @zext_decremenet_sext(<8 x i8> %x) {
 ; X32-SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
 ; X32-SSE41-NEXT:    paddw %xmm0, %xmm1
 ; X32-SSE41-NEXT:    pmovsxwd %xmm1, %xmm0
-; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; X32-SSE41-NEXT:    pmovsxwd %xmm1, %xmm1
 ; X32-SSE41-NEXT:    retl
   %z = zext <8 x i8> %x to <8 x i16>
diff --git a/llvm/test/CodeGen/X86/vector-shift-ashr-128.ll b/llvm/test/CodeGen/X86/vector-shift-ashr-128.ll
index 5764d19f4c7f7..9b1fb29cb0297 100644
--- a/llvm/test/CodeGen/X86/vector-shift-ashr-128.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-ashr-128.ll
@@ -23,7 +23,7 @@ define <2 x i64> @var_shift_v2i64(<2 x i64> %a, <2 x i64> %b) nounwind {
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
 ; SSE2-NEXT:    movdqa %xmm2, %xmm3
 ; SSE2-NEXT:    psrlq %xmm1, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    psrlq %xmm4, %xmm2
 ; SSE2-NEXT:    movsd {{.*#+}} xmm2 = xmm3[0],xmm2[1]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm3
@@ -39,7 +39,7 @@ define <2 x i64> @var_shift_v2i64(<2 x i64> %a, <2 x i64> %b) nounwind {
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
 ; SSE41-NEXT:    movdqa %xmm2, %xmm3
 ; SSE41-NEXT:    psrlq %xmm1, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[2,3,2,3]
 ; SSE41-NEXT:    psrlq %xmm4, %xmm2
 ; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
 ; SSE41-NEXT:    movdqa %xmm0, %xmm3
@@ -54,7 +54,7 @@ define <2 x i64> @var_shift_v2i64(<2 x i64> %a, <2 x i64> %b) nounwind {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
 ; AVX1-NEXT:    vpsrlq %xmm1, %xmm2, %xmm3
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm1[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm1[2,3,2,3]
 ; AVX1-NEXT:    vpsrlq %xmm4, %xmm2, %xmm2
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
 ; AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm1
@@ -99,7 +99,7 @@ define <2 x i64> @var_shift_v2i64(<2 x i64> %a, <2 x i64> %b) nounwind {
 ; X32-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [0,2147483648,0,2147483648]
 ; X32-SSE-NEXT:    movdqa %xmm2, %xmm3
 ; X32-SSE-NEXT:    psrlq %xmm1, %xmm3
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[2,3,0,1]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[2,3,2,3]
 ; X32-SSE-NEXT:    psrlq %xmm4, %xmm2
 ; X32-SSE-NEXT:    movsd {{.*#+}} xmm2 = xmm3[0],xmm2[1]
 ; X32-SSE-NEXT:    movdqa %xmm0, %xmm3
@@ -123,7 +123,7 @@ define <4 x i32> @var_shift_v4i32(<4 x i32> %a, <4 x i32> %b) nounwind {
 ; SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; SSE2-NEXT:    psrad %xmm4, %xmm2
 ; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm1[2,3,3,3,4,5,6,7]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm4
 ; SSE2-NEXT:    psrad %xmm3, %xmm4
@@ -139,7 +139,7 @@ define <4 x i32> @var_shift_v4i32(<4 x i32> %a, <4 x i32> %b) nounwind {
 ; SSE41-NEXT:    pshuflw {{.*#+}} xmm2 = xmm1[2,3,3,3,4,5,6,7]
 ; SSE41-NEXT:    movdqa %xmm0, %xmm3
 ; SSE41-NEXT:    psrad %xmm2, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; SSE41-NEXT:    pshuflw {{.*#+}} xmm4 = xmm2[2,3,3,3,4,5,6,7]
 ; SSE41-NEXT:    movdqa %xmm0, %xmm5
 ; SSE41-NEXT:    psrad %xmm4, %xmm5
@@ -205,7 +205,7 @@ define <4 x i32> @var_shift_v4i32(<4 x i32> %a, <4 x i32> %b) nounwind {
 ; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
 ; X32-SSE-NEXT:    psrad %xmm4, %xmm2
 ; X32-SSE-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm3 = xmm1[2,3,3,3,4,5,6,7]
 ; X32-SSE-NEXT:    movdqa %xmm0, %xmm4
 ; X32-SSE-NEXT:    psrad %xmm3, %xmm4
diff --git a/llvm/test/CodeGen/X86/vector-shift-ashr-256.ll b/llvm/test/CodeGen/X86/vector-shift-ashr-256.ll
index 358f9b8cc4dec..ae9c375eec254 100644
--- a/llvm/test/CodeGen/X86/vector-shift-ashr-256.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-ashr-256.ll
@@ -22,7 +22,7 @@ define <4 x i64> @var_shift_v4i64(<4 x i64> %a, <4 x i64> %b) nounwind {
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808]
 ; AVX1-NEXT:    vpsrlq %xmm2, %xmm3, %xmm4
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm2[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm2[2,3,2,3]
 ; AVX1-NEXT:    vpsrlq %xmm5, %xmm3, %xmm6
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3],xmm6[4,5,6,7]
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm6
@@ -32,7 +32,7 @@ define <4 x i64> @var_shift_v4i64(<4 x i64> %a, <4 x i64> %b) nounwind {
 ; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm2
 ; AVX1-NEXT:    vpsubq %xmm4, %xmm2, %xmm2
 ; AVX1-NEXT:    vpsrlq %xmm1, %xmm3, %xmm4
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm1[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm1[2,3,2,3]
 ; AVX1-NEXT:    vpsrlq %xmm5, %xmm3, %xmm3
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0,1,2,3],xmm3[4,5,6,7]
 ; AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm1
@@ -92,7 +92,7 @@ define <4 x i64> @var_shift_v4i64(<4 x i64> %a, <4 x i64> %b) nounwind {
 ; X32-AVX1-NEXT:    vmovddup {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808]
 ; X32-AVX1-NEXT:    # xmm3 = mem[0,0]
 ; X32-AVX1-NEXT:    vpsrlq %xmm2, %xmm3, %xmm4
-; X32-AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm2[2,3,0,1]
+; X32-AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm2[2,3,2,3]
 ; X32-AVX1-NEXT:    vpsrlq %xmm5, %xmm3, %xmm6
 ; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3],xmm6[4,5,6,7]
 ; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm6
@@ -102,7 +102,7 @@ define <4 x i64> @var_shift_v4i64(<4 x i64> %a, <4 x i64> %b) nounwind {
 ; X32-AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm2
 ; X32-AVX1-NEXT:    vpsubq %xmm4, %xmm2, %xmm2
 ; X32-AVX1-NEXT:    vpsrlq %xmm1, %xmm3, %xmm4
-; X32-AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm1[2,3,0,1]
+; X32-AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm1[2,3,2,3]
 ; X32-AVX1-NEXT:    vpsrlq %xmm5, %xmm3, %xmm3
 ; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0,1,2,3],xmm3[4,5,6,7]
 ; X32-AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm1
diff --git a/llvm/test/CodeGen/X86/vector-shift-ashr-sub128.ll b/llvm/test/CodeGen/X86/vector-shift-ashr-sub128.ll
index 11d118bf31c3d..a994d6610d73d 100644
--- a/llvm/test/CodeGen/X86/vector-shift-ashr-sub128.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-ashr-sub128.ll
@@ -27,7 +27,7 @@ define <2 x i32> @var_shift_v2i32(<2 x i32> %a, <2 x i32> %b) nounwind {
 ; SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; SSE2-NEXT:    psrad %xmm4, %xmm2
 ; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm1[2,3,3,3,4,5,6,7]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm4
 ; SSE2-NEXT:    psrad %xmm3, %xmm4
@@ -43,7 +43,7 @@ define <2 x i32> @var_shift_v2i32(<2 x i32> %a, <2 x i32> %b) nounwind {
 ; SSE41-NEXT:    pshuflw {{.*#+}} xmm2 = xmm1[2,3,3,3,4,5,6,7]
 ; SSE41-NEXT:    movdqa %xmm0, %xmm3
 ; SSE41-NEXT:    psrad %xmm2, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; SSE41-NEXT:    pshuflw {{.*#+}} xmm4 = xmm2[2,3,3,3,4,5,6,7]
 ; SSE41-NEXT:    movdqa %xmm0, %xmm5
 ; SSE41-NEXT:    psrad %xmm4, %xmm5
@@ -109,7 +109,7 @@ define <2 x i32> @var_shift_v2i32(<2 x i32> %a, <2 x i32> %b) nounwind {
 ; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
 ; X32-SSE-NEXT:    psrad %xmm4, %xmm2
 ; X32-SSE-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm3 = xmm1[2,3,3,3,4,5,6,7]
 ; X32-SSE-NEXT:    movdqa %xmm0, %xmm4
 ; X32-SSE-NEXT:    psrad %xmm3, %xmm4
diff --git a/llvm/test/CodeGen/X86/vector-shift-lshr-128.ll b/llvm/test/CodeGen/X86/vector-shift-lshr-128.ll
index 1a2889ab861e2..2e19f753722d3 100644
--- a/llvm/test/CodeGen/X86/vector-shift-lshr-128.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-lshr-128.ll
@@ -22,7 +22,7 @@ define <2 x i64> @var_shift_v2i64(<2 x i64> %a, <2 x i64> %b) nounwind {
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; SSE2-NEXT:    psrlq %xmm1, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    psrlq %xmm1, %xmm0
 ; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm2[0],xmm0[1]
 ; SSE2-NEXT:    retq
@@ -31,7 +31,7 @@ define <2 x i64> @var_shift_v2i64(<2 x i64> %a, <2 x i64> %b) nounwind {
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movdqa %xmm0, %xmm2
 ; SSE41-NEXT:    psrlq %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE41-NEXT:    psrlq %xmm1, %xmm0
 ; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm2[0,1,2,3],xmm0[4,5,6,7]
 ; SSE41-NEXT:    retq
@@ -39,7 +39,7 @@ define <2 x i64> @var_shift_v2i64(<2 x i64> %a, <2 x i64> %b) nounwind {
 ; AVX1-LABEL: var_shift_v2i64:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm2[0,1,2,3],xmm0[4,5,6,7]
 ; AVX1-NEXT:    retq
@@ -75,7 +75,7 @@ define <2 x i64> @var_shift_v2i64(<2 x i64> %a, <2 x i64> %b) nounwind {
 ; X32-SSE:       # %bb.0:
 ; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
 ; X32-SSE-NEXT:    psrlq %xmm1, %xmm2
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; X32-SSE-NEXT:    psrlq %xmm1, %xmm0
 ; X32-SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm2[0],xmm0[1]
 ; X32-SSE-NEXT:    retl
@@ -93,7 +93,7 @@ define <4 x i32> @var_shift_v4i32(<4 x i32> %a, <4 x i32> %b) nounwind {
 ; SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; SSE2-NEXT:    psrld %xmm4, %xmm2
 ; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm1[2,3,3,3,4,5,6,7]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm4
 ; SSE2-NEXT:    psrld %xmm3, %xmm4
@@ -109,7 +109,7 @@ define <4 x i32> @var_shift_v4i32(<4 x i32> %a, <4 x i32> %b) nounwind {
 ; SSE41-NEXT:    pshuflw {{.*#+}} xmm2 = xmm1[2,3,3,3,4,5,6,7]
 ; SSE41-NEXT:    movdqa %xmm0, %xmm3
 ; SSE41-NEXT:    psrld %xmm2, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; SSE41-NEXT:    pshuflw {{.*#+}} xmm4 = xmm2[2,3,3,3,4,5,6,7]
 ; SSE41-NEXT:    movdqa %xmm0, %xmm5
 ; SSE41-NEXT:    psrld %xmm4, %xmm5
@@ -175,7 +175,7 @@ define <4 x i32> @var_shift_v4i32(<4 x i32> %a, <4 x i32> %b) nounwind {
 ; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
 ; X32-SSE-NEXT:    psrld %xmm4, %xmm2
 ; X32-SSE-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm3 = xmm1[2,3,3,3,4,5,6,7]
 ; X32-SSE-NEXT:    movdqa %xmm0, %xmm4
 ; X32-SSE-NEXT:    psrld %xmm3, %xmm4
diff --git a/llvm/test/CodeGen/X86/vector-shift-lshr-256.ll b/llvm/test/CodeGen/X86/vector-shift-lshr-256.ll
index 154c35b51db9f..9119e32bda375 100644
--- a/llvm/test/CodeGen/X86/vector-shift-lshr-256.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-lshr-256.ll
@@ -22,11 +22,11 @@ define <4 x i64> @var_shift_v4i64(<4 x i64> %a, <4 x i64> %b) nounwind {
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
 ; AVX1-NEXT:    vpsrlq %xmm2, %xmm3, %xmm4
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
 ; AVX1-NEXT:    vpsrlq %xmm2, %xmm3, %xmm2
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm4[0,1,2,3],xmm2[4,5,6,7]
 ; AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm3
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm3[0,1,2,3],xmm0[4,5,6,7]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
@@ -69,11 +69,11 @@ define <4 x i64> @var_shift_v4i64(<4 x i64> %a, <4 x i64> %b) nounwind {
 ; X32-AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
 ; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
 ; X32-AVX1-NEXT:    vpsrlq %xmm2, %xmm3, %xmm4
-; X32-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
+; X32-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
 ; X32-AVX1-NEXT:    vpsrlq %xmm2, %xmm3, %xmm2
 ; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm4[0,1,2,3],xmm2[4,5,6,7]
 ; X32-AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm3
-; X32-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; X32-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; X32-AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm0
 ; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm3[0,1,2,3],xmm0[4,5,6,7]
 ; X32-AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
diff --git a/llvm/test/CodeGen/X86/vector-shift-lshr-sub128.ll b/llvm/test/CodeGen/X86/vector-shift-lshr-sub128.ll
index b540421fb3de9..8a843ef652e7f 100644
--- a/llvm/test/CodeGen/X86/vector-shift-lshr-sub128.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-lshr-sub128.ll
@@ -27,7 +27,7 @@ define <2 x i32> @var_shift_v2i32(<2 x i32> %a, <2 x i32> %b) nounwind {
 ; SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; SSE2-NEXT:    psrld %xmm4, %xmm2
 ; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm1[2,3,3,3,4,5,6,7]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm4
 ; SSE2-NEXT:    psrld %xmm3, %xmm4
@@ -43,7 +43,7 @@ define <2 x i32> @var_shift_v2i32(<2 x i32> %a, <2 x i32> %b) nounwind {
 ; SSE41-NEXT:    pshuflw {{.*#+}} xmm2 = xmm1[2,3,3,3,4,5,6,7]
 ; SSE41-NEXT:    movdqa %xmm0, %xmm3
 ; SSE41-NEXT:    psrld %xmm2, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; SSE41-NEXT:    pshuflw {{.*#+}} xmm4 = xmm2[2,3,3,3,4,5,6,7]
 ; SSE41-NEXT:    movdqa %xmm0, %xmm5
 ; SSE41-NEXT:    psrld %xmm4, %xmm5
@@ -109,7 +109,7 @@ define <2 x i32> @var_shift_v2i32(<2 x i32> %a, <2 x i32> %b) nounwind {
 ; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
 ; X32-SSE-NEXT:    psrld %xmm4, %xmm2
 ; X32-SSE-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm3 = xmm1[2,3,3,3,4,5,6,7]
 ; X32-SSE-NEXT:    movdqa %xmm0, %xmm4
 ; X32-SSE-NEXT:    psrld %xmm3, %xmm4
diff --git a/llvm/test/CodeGen/X86/vector-shift-shl-128.ll b/llvm/test/CodeGen/X86/vector-shift-shl-128.ll
index eee4561504764..5f1325aacb4d1 100644
--- a/llvm/test/CodeGen/X86/vector-shift-shl-128.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-shl-128.ll
@@ -22,7 +22,7 @@ define <2 x i64> @var_shift_v2i64(<2 x i64> %a, <2 x i64> %b) nounwind {
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; SSE2-NEXT:    psllq %xmm1, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    psllq %xmm1, %xmm0
 ; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm2[0],xmm0[1]
 ; SSE2-NEXT:    retq
@@ -31,7 +31,7 @@ define <2 x i64> @var_shift_v2i64(<2 x i64> %a, <2 x i64> %b) nounwind {
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movdqa %xmm0, %xmm2
 ; SSE41-NEXT:    psllq %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE41-NEXT:    psllq %xmm1, %xmm0
 ; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm2[0,1,2,3],xmm0[4,5,6,7]
 ; SSE41-NEXT:    retq
@@ -39,7 +39,7 @@ define <2 x i64> @var_shift_v2i64(<2 x i64> %a, <2 x i64> %b) nounwind {
 ; AVX1-LABEL: var_shift_v2i64:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpsllq %xmm1, %xmm0, %xmm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; AVX1-NEXT:    vpsllq %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm2[0,1,2,3],xmm0[4,5,6,7]
 ; AVX1-NEXT:    retq
@@ -73,7 +73,7 @@ define <2 x i64> @var_shift_v2i64(<2 x i64> %a, <2 x i64> %b) nounwind {
 ; X32-SSE:       # %bb.0:
 ; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
 ; X32-SSE-NEXT:    psllq %xmm1, %xmm2
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; X32-SSE-NEXT:    psllq %xmm1, %xmm0
 ; X32-SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm2[0],xmm0[1]
 ; X32-SSE-NEXT:    retl
diff --git a/llvm/test/CodeGen/X86/vector-shift-shl-256.ll b/llvm/test/CodeGen/X86/vector-shift-shl-256.ll
index 56ebce709a8d7..1296fcf8bb902 100644
--- a/llvm/test/CodeGen/X86/vector-shift-shl-256.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-shl-256.ll
@@ -22,11 +22,11 @@ define <4 x i64> @var_shift_v4i64(<4 x i64> %a, <4 x i64> %b) nounwind {
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
 ; AVX1-NEXT:    vpsllq %xmm2, %xmm3, %xmm4
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
 ; AVX1-NEXT:    vpsllq %xmm2, %xmm3, %xmm2
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm4[0,1,2,3],xmm2[4,5,6,7]
 ; AVX1-NEXT:    vpsllq %xmm1, %xmm0, %xmm3
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; AVX1-NEXT:    vpsllq %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm3[0,1,2,3],xmm0[4,5,6,7]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
@@ -66,11 +66,11 @@ define <4 x i64> @var_shift_v4i64(<4 x i64> %a, <4 x i64> %b) nounwind {
 ; X32-AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
 ; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
 ; X32-AVX1-NEXT:    vpsllq %xmm2, %xmm3, %xmm4
-; X32-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
+; X32-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
 ; X32-AVX1-NEXT:    vpsllq %xmm2, %xmm3, %xmm2
 ; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm4[0,1,2,3],xmm2[4,5,6,7]
 ; X32-AVX1-NEXT:    vpsllq %xmm1, %xmm0, %xmm3
-; X32-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; X32-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; X32-AVX1-NEXT:    vpsllq %xmm1, %xmm0, %xmm0
 ; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm3[0,1,2,3],xmm0[4,5,6,7]
 ; X32-AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-128-v8.ll b/llvm/test/CodeGen/X86/vector-shuffle-128-v8.ll
index 04064f1c1970d..a11fc5b7c8ed3 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-128-v8.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-128-v8.ll
@@ -1013,19 +1013,19 @@ define <8 x i16> @shuffle_v8i16_08192a3b(<8 x i16> %a, <8 x i16> %b) {
 define <8 x i16> @shuffle_v8i16_0c1d2e3f(<8 x i16> %a, <8 x i16> %b) {
 ; SSE-LABEL: shuffle_v8i16_0c1d2e3f:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; SSE-NEXT:    retq
 ;
 ; AVX1OR2-LABEL: shuffle_v8i16_0c1d2e3f:
 ; AVX1OR2:       # %bb.0:
-; AVX1OR2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; AVX1OR2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; AVX1OR2-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; AVX1OR2-NEXT:    retq
 ;
 ; AVX512VL-SLOW-LABEL: shuffle_v8i16_0c1d2e3f:
 ; AVX512VL-SLOW:       # %bb.0:
-; AVX512VL-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; AVX512VL-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; AVX512VL-SLOW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; AVX512VL-SLOW-NEXT:    retq
 ;
@@ -1055,19 +1055,19 @@ define <8 x i16> @shuffle_v8i16_4c5d6e7f(<8 x i16> %a, <8 x i16> %b) {
 define <8 x i16> @shuffle_v8i16_48596a7b(<8 x i16> %a, <8 x i16> %b) {
 ; SSE-LABEL: shuffle_v8i16_48596a7b:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; SSE-NEXT:    retq
 ;
 ; AVX1OR2-LABEL: shuffle_v8i16_48596a7b:
 ; AVX1OR2:       # %bb.0:
-; AVX1OR2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX1OR2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; AVX1OR2-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; AVX1OR2-NEXT:    retq
 ;
 ; AVX512VL-SLOW-LABEL: shuffle_v8i16_48596a7b:
 ; AVX512VL-SLOW:       # %bb.0:
-; AVX512VL-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX512VL-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; AVX512VL-SLOW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; AVX512VL-SLOW-NEXT:    retq
 ;
@@ -1513,7 +1513,7 @@ define <8 x i16> @shuffle_v8i16_cde3XXXX(<8 x i16> %a, <8 x i16> %b) {
 ; SSE2-LABEL: shuffle_v8i16_cde3XXXX:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [65535,65535,65535,0,65535,65535,65535,65535]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    pand %xmm2, %xmm1
 ; SSE2-NEXT:    pandn %xmm0, %xmm2
 ; SSE2-NEXT:    por %xmm1, %xmm2
@@ -1529,25 +1529,25 @@ define <8 x i16> @shuffle_v8i16_cde3XXXX(<8 x i16> %a, <8 x i16> %b) {
 ;
 ; SSE41-LABEL: shuffle_v8i16_cde3XXXX:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3],xmm1[4,5,6,7]
 ; SSE41-NEXT:    retq
 ;
 ; AVX1OR2-LABEL: shuffle_v8i16_cde3XXXX:
 ; AVX1OR2:       # %bb.0:
-; AVX1OR2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; AVX1OR2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; AVX1OR2-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3],xmm1[4,5,6,7]
 ; AVX1OR2-NEXT:    retq
 ;
 ; AVX512VL-SLOW-LABEL: shuffle_v8i16_cde3XXXX:
 ; AVX512VL-SLOW:       # %bb.0:
-; AVX512VL-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; AVX512VL-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; AVX512VL-SLOW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3],xmm1[4,5,6,7]
 ; AVX512VL-SLOW-NEXT:    retq
 ;
 ; AVX512VL-FAST-LABEL: shuffle_v8i16_cde3XXXX:
 ; AVX512VL-FAST:       # %bb.0:
-; AVX512VL-FAST-NEXT:    vmovdqa {{.*#+}} xmm2 = [4,5,6,11,0,1,2,3]
+; AVX512VL-FAST-NEXT:    vmovdqa {{.*#+}} xmm2 = [4,5,6,11,4,5,6,7]
 ; AVX512VL-FAST-NEXT:    vpermi2w %xmm0, %xmm1, %xmm2
 ; AVX512VL-FAST-NEXT:    vmovdqa %xmm2, %xmm0
 ; AVX512VL-FAST-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-256-v16.ll b/llvm/test/CodeGen/X86/vector-shuffle-256-v16.ll
index 635d94b9e7239..131c621162a50 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-256-v16.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-256-v16.ll
@@ -3167,7 +3167,7 @@ define <16 x i16> @shuffle_v16i16_28_zz_zz_zz_29_zz_zz_zz_30_zz_zz_zz_31_zz_zz_z
 ; AVX1-LABEL: shuffle_v16i16_28_zz_zz_zz_29_zz_zz_zz_30_zz_zz_zz_31_zz_zz_zz:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
 ; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
@@ -3191,7 +3191,7 @@ define <16 x i16> @shuffle_v16i16_28_zz_zz_zz_29_zz_zz_zz_30_zz_zz_zz_31_zz_zz_z
 ; XOPAVX1-LABEL: shuffle_v16i16_28_zz_zz_zz_29_zz_zz_zz_30_zz_zz_zz_31_zz_zz_zz:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; XOPAVX1-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
 ; XOPAVX1-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
@@ -5064,7 +5064,7 @@ define <16 x i16> @shuffle_v16i16_00_20_01_21_02_22_03_31_08_28_09_29_10_30_11_3
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm3[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm3[2,3,2,3]
 ; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
 ; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
 ; AVX1-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0,1,4,5,8,9,14,15,u,u,u,u,u,u,u,u]
@@ -5090,7 +5090,7 @@ define <16 x i16> @shuffle_v16i16_00_20_01_21_02_22_03_31_08_28_09_29_10_30_11_3
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm3[2,3,0,1]
+; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm3[2,3,2,3]
 ; XOPAVX1-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
 ; XOPAVX1-NEXT:    vpperm {{.*#+}} xmm1 = xmm1[8,9,10,11,12,13],xmm3[14,15],xmm1[u,u,u,u,u,u,u,u]
 ; XOPAVX1-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
@@ -5161,7 +5161,7 @@ define <16 x i16> @shuffle_v16i16_04_16_05_17_06_18_07_27_12_24_13_25_14_26_15_2
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]
 ; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
 ; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
 ; AVX1-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[u,u,u,u,u,u,u,u,0,1,4,5,8,9,14,15]
@@ -5187,7 +5187,7 @@ define <16 x i16> @shuffle_v16i16_04_16_05_17_06_18_07_27_12_24_13_25_14_26_15_2
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
-; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,3,0,1]
+; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]
 ; XOPAVX1-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
 ; XOPAVX1-NEXT:    vpperm {{.*#+}} xmm1 = xmm1[u,u,u,u,u,u,u,u,0,1,2,3,4,5],xmm2[6,7]
 ; XOPAVX1-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
@@ -5752,9 +5752,9 @@ define <16 x i16> @shuffle_v16i16_20_21_22_03_uu_uu_uu_uu_28_29_30_11_uu_uu_uu_u
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2],xmm2[3],xmm3[4,5,6,7]
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3],xmm1[4,5,6,7]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
@@ -5776,9 +5776,9 @@ define <16 x i16> @shuffle_v16i16_20_21_22_03_uu_uu_uu_uu_28_29_30_11_uu_uu_uu_u
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,3,0,1]
+; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]
 ; XOPAVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2],xmm2[3],xmm3[4,5,6,7]
-; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; XOPAVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3],xmm1[4,5,6,7]
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; XOPAVX1-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-256-v32.ll b/llvm/test/CodeGen/X86/vector-shuffle-256-v32.ll
index 04759bf104cb3..0508c6bac2bb1 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-256-v32.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-256-v32.ll
@@ -4053,7 +4053,7 @@ define <32 x i8> @shuffle_v32i8_56_zz_zz_zz_57_zz_zz_zz_58_zz_zz_zz__zz_59_zz_zz
 ; AVX1-LABEL: shuffle_v32i8_56_zz_zz_zz_57_zz_zz_zz_58_zz_zz_zz__zz_59_zz_zz_zz_60_zz_zz_zz_61_zz_zz_zz_62_zz_zz_zz_63_zz_zz_zz:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
@@ -4083,7 +4083,7 @@ define <32 x i8> @shuffle_v32i8_56_zz_zz_zz_57_zz_zz_zz_58_zz_zz_zz__zz_59_zz_zz
 ; XOPAVX1-LABEL: shuffle_v32i8_56_zz_zz_zz_57_zz_zz_zz_58_zz_zz_zz__zz_59_zz_zz_zz_60_zz_zz_zz_61_zz_zz_zz_62_zz_zz_zz_63_zz_zz_zz:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; XOPAVX1-NEXT:    vpmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
 ; XOPAVX1-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-256-v8.ll b/llvm/test/CodeGen/X86/vector-shuffle-256-v8.ll
index 973045696fbd5..4798b4b1d38a2 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-256-v8.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-256-v8.ll
@@ -3216,7 +3216,7 @@ entry:
 define <8 x float> @broadcast_concat_crash(<4 x float> %x, <4 x float> %y, float %z) {
 ; AVX1-LABEL: broadcast_concat_crash:
 ; AVX1:       # %bb.0: # %entry
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm0 = xmm1[3,3,1,1]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm0 = xmm1[3,3,3,3]
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[2,3]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-512-v32.ll b/llvm/test/CodeGen/X86/vector-shuffle-512-v32.ll
index b2d4dc76a10e0..4c8073614d6dd 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-512-v32.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-512-v32.ll
@@ -434,7 +434,7 @@ define <32 x i16> @shuffle_v32i16_32_zz_zz_zz_33_zz_zz_zz_34_zz_zz_zz_35_zz_zz_z
 ; KNL-LABEL: shuffle_v32i16_32_zz_zz_zz_33_zz_zz_zz_34_zz_zz_zz_35_zz_zz_zz_36_zz_zz_zz_37_zz_zz_zz_38_zz_zz_zz_39_zz_zz_zz:
 ; KNL:       ## %bb.0:
 ; KNL-NEXT:    vpmovzxwq {{.*#+}} ymm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; KNL-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; KNL-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; KNL-NEXT:    vpmovzxwq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; KNL-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; KNL-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-512-v64.ll b/llvm/test/CodeGen/X86/vector-shuffle-512-v64.ll
index 3199cc0fa9b9b..4237c4107d477 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-512-v64.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-512-v64.ll
@@ -397,7 +397,7 @@ define <64 x i8> @shuffle_v64i8_64_zz_zz_zz_65_zz_zz_zz_66_zz_zz_zz_67_zz_zz_zz_
 ; AVX512F-LABEL: shuffle_v64i8_64_zz_zz_zz_65_zz_zz_zz_66_zz_zz_zz_67_zz_zz_zz_68_zz_zz_zz_69_zz_zz_zz_70_zz_zz_zz_71_zz_zz_zz_72_zz_zz_zz_73_zz_zz_zz_74_zz_zz_zz_75_zz_zz_zz_76_zz_zz_zz_77_zz_zz_zz_78_zz_zz_zz_79_zz_zz_zz:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpmovzxbd {{.*#+}} ymm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
-; AVX512F-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX512F-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; AVX512F-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
 ; AVX512F-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; AVX512F-NEXT:    retq
@@ -410,7 +410,7 @@ define <64 x i8> @shuffle_v64i8_64_zz_zz_zz_65_zz_zz_zz_66_zz_zz_zz_67_zz_zz_zz_
 ; AVX512DQ-LABEL: shuffle_v64i8_64_zz_zz_zz_65_zz_zz_zz_66_zz_zz_zz_67_zz_zz_zz_68_zz_zz_zz_69_zz_zz_zz_70_zz_zz_zz_71_zz_zz_zz_72_zz_zz_zz_73_zz_zz_zz_74_zz_zz_zz_75_zz_zz_zz_76_zz_zz_zz_77_zz_zz_zz_78_zz_zz_zz_79_zz_zz_zz:
 ; AVX512DQ:       # %bb.0:
 ; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} ymm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; AVX512DQ-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-combining.ll b/llvm/test/CodeGen/X86/vector-shuffle-combining.ll
index 72c1aefb5da7d..622eb0881052e 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-combining.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-combining.ll
@@ -3228,7 +3228,7 @@ define void @PR45604(<32 x i16>* %dst, <8 x i16>* %src) {
 ; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [11,11,11,0,11,11,11,0]
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm3, %ymm1
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-sse4a.ll b/llvm/test/CodeGen/X86/vector-shuffle-sse4a.ll
index 25c44964a5511..6adb6b0c2c0b8 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-sse4a.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-sse4a.ll
@@ -372,14 +372,14 @@ define <16 x i8> @shuffle_8_18_uuuuuuuuuuuuuu(<16 x i8> %a, <16 x i8> %b) {
 ; BTVER1-LABEL: shuffle_8_18_uuuuuuuuuuuuuu:
 ; BTVER1:       # %bb.0:
 ; BTVER1-NEXT:    psrld $16, %xmm1
-; BTVER1-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; BTVER1-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; BTVER1-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
 ; BTVER1-NEXT:    retq
 ;
 ; BTVER2-LABEL: shuffle_8_18_uuuuuuuuuuuuuu:
 ; BTVER2:       # %bb.0:
 ; BTVER2-NEXT:    vpsrld $16, %xmm1, %xmm1
-; BTVER2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; BTVER2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; BTVER2-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
 ; BTVER2-NEXT:    retq
   %1 = shufflevector <16 x i8> %a, <16 x i8> %b, <16 x i32> <i32 8, i32 18, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
diff --git a/llvm/test/CodeGen/X86/vector-zext.ll b/llvm/test/CodeGen/X86/vector-zext.ll
index 42bf2d9eac284..5d065213f925b 100644
--- a/llvm/test/CodeGen/X86/vector-zext.ll
+++ b/llvm/test/CodeGen/X86/vector-zext.ll
@@ -57,7 +57,7 @@ define <16 x i16> @zext_16i8_to_16i16(<16 x i8> %A) {
 ; SSE41-LABEL: zext_16i8_to_16i16:
 ; SSE41:       # %bb.0: # %entry
 ; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; SSE41-NEXT:    retq
@@ -113,9 +113,9 @@ define <32 x i16> @zext_32i8_to_32i16(<32 x i8> %A) {
 ; SSE41:       # %bb.0: # %entry
 ; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm5 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; SSE41-NEXT:    movdqa %xmm5, %xmm0
 ; SSE41-NEXT:    movdqa %xmm4, %xmm1
@@ -277,7 +277,7 @@ define <16 x i32> @zext_16i8_to_16i32(<16 x i8> %A) nounwind uwtable readnone ss
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm4 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm3 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
@@ -290,9 +290,9 @@ define <16 x i32> @zext_16i8_to_16i32(<16 x i8> %A) nounwind uwtable readnone ss
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,3,2,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
 ; AVX1-NEXT:    vmovaps %ymm2, %ymm0
@@ -301,7 +301,7 @@ define <16 x i32> @zext_16i8_to_16i32(<16 x i8> %A) nounwind uwtable readnone ss
 ; AVX2-LABEL: zext_16i8_to_16i32:
 ; AVX2:       # %bb.0: # %entry
 ; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm2 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
 ; AVX2-NEXT:    vmovdqa %ymm2, %ymm0
 ; AVX2-NEXT:    retq
@@ -515,7 +515,7 @@ define <8 x i32> @zext_8i16_to_8i32(<8 x i16> %A) nounwind uwtable readnone ssp
 ; SSE41-LABEL: zext_8i16_to_8i32:
 ; SSE41:       # %bb.0: # %entry
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; SSE41-NEXT:    retq
@@ -571,9 +571,9 @@ define <16 x i32> @zext_16i16_to_16i32(<16 x i16> %A) nounwind uwtable readnone
 ; SSE41:       # %bb.0: # %entry
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm5 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; SSE41-NEXT:    movdqa %xmm5, %xmm0
 ; SSE41-NEXT:    movdqa %xmm4, %xmm1
@@ -727,7 +727,7 @@ define <8 x i64> @zext_8i16_to_8i64(<8 x i16> %A) nounwind uwtable readnone ssp
 ; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm4 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
 ; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm3 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
@@ -740,9 +740,9 @@ define <8 x i64> @zext_8i16_to_8i64(<8 x i16> %A) nounwind uwtable readnone ssp
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,3,2,3]
 ; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
 ; AVX1-NEXT:    vmovaps %ymm2, %ymm0
@@ -751,7 +751,7 @@ define <8 x i64> @zext_8i16_to_8i64(<8 x i16> %A) nounwind uwtable readnone ssp
 ; AVX2-LABEL: zext_8i16_to_8i64:
 ; AVX2:       # %bb.0: # %entry
 ; AVX2-NEXT:    vpmovzxwq {{.*#+}} ymm2 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpmovzxwq {{.*#+}} ymm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; AVX2-NEXT:    vmovdqa %ymm2, %ymm0
 ; AVX2-NEXT:    retq
@@ -813,7 +813,7 @@ define <4 x i64> @zext_4i32_to_4i64(<4 x i32> %A) nounwind uwtable readnone ssp
 ; SSE41-LABEL: zext_4i32_to_4i64:
 ; SSE41:       # %bb.0: # %entry
 ; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; SSE41-NEXT:    retq
@@ -869,9 +869,9 @@ define <8 x i64> @zext_8i32_to_8i64(<8 x i32> %A) nounwind uwtable readnone ssp
 ; SSE41:       # %bb.0: # %entry
 ; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm5 = xmm0[0],zero,xmm0[1],zero
 ; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero
 ; SSE41-NEXT:    movdqa %xmm5, %xmm0
 ; SSE41-NEXT:    movdqa %xmm4, %xmm1
@@ -1885,7 +1885,7 @@ define <4 x i64> @shuf_zext_8i16_to_4i64_offset2(<8 x i16> %A) nounwind uwtable
 ; SSE41:       # %bb.0: # %entry
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; SSE41-NEXT:    retq
@@ -1894,7 +1894,7 @@ define <4 x i64> @shuf_zext_8i16_to_4i64_offset2(<8 x i16> %A) nounwind uwtable
 ; AVX1:       # %bb.0: # %entry
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
@@ -2213,14 +2213,14 @@ define <32 x i32> @zext_32i8_to_32i32(<32 x i8> %x) {
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,2,3]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm5 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[1,1,2,3]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm6 = xmm6[0],zero,zero,zero,xmm6[1],zero,zero,zero,xmm6[2],zero,zero,zero,xmm6[3],zero,zero,zero
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm1[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm1[2,3,2,3]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm7 = xmm7[0],zero,zero,zero,xmm7[1],zero,zero,zero,xmm7[2],zero,zero,zero,xmm7[3],zero,zero,zero
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
@@ -2245,14 +2245,14 @@ define <32 x i32> @zext_32i8_to_32i32(<32 x i8> %x) {
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm3[1,1,2,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,3,2,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm3[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[3,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[3,3,2,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
 ; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm3
 ; AVX1-NEXT:    vmovaps %ymm4, %ymm0
@@ -2263,9 +2263,9 @@ define <32 x i32> @zext_32i8_to_32i32(<32 x i8> %x) {
 ; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm4 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm3
 ; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm2 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero,xmm3[4],zero,zero,zero,xmm3[5],zero,zero,zero,xmm3[6],zero,zero,zero,xmm3[7],zero,zero,zero
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm3[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]
 ; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm3 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
 ; AVX2-NEXT:    vmovdqa %ymm4, %ymm0
 ; AVX2-NEXT:    retq
@@ -2499,7 +2499,7 @@ define <8 x i64> @zext_8i6_to_8i64(i32 %x) nounwind uwtable readnone ssp {
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,2,3]
 ; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
 ; SSE41-NEXT:    pand %xmm4, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,0,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]
 ; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
 ; SSE41-NEXT:    pand %xmm4, %xmm2
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[3,1,2,3]
@@ -2518,9 +2518,9 @@ define <8 x i64> @zext_8i6_to_8i64(i32 %x) nounwind uwtable readnone ssp {
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,2,3]
 ; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,2,3]
 ; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
 ; AVX1-NEXT:    retq
@@ -2532,7 +2532,7 @@ define <8 x i64> @zext_8i6_to_8i64(i32 %x) nounwind uwtable readnone ssp {
 ; AVX2-NEXT:    vpaddw {{.*}}(%rip), %xmm0, %xmm0
 ; AVX2-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm1
 ; AVX2-NEXT:    vpmovzxwq {{.*#+}} ymm0 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; AVX2-NEXT:    vpmovzxwq {{.*#+}} ymm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; AVX2-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/vsel-cmp-load.ll b/llvm/test/CodeGen/X86/vsel-cmp-load.ll
index c809090c8955c..89d1b549182ab 100644
--- a/llvm/test/CodeGen/X86/vsel-cmp-load.ll
+++ b/llvm/test/CodeGen/X86/vsel-cmp-load.ll
@@ -80,7 +80,7 @@ define <16 x i16> @sgt_zero(<16 x i8>* %p, <16 x i16> %x, <16 x i16> %y) {
 ; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
 ; AVX1-NEXT:    vpcmpgtb %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vpmovsxbw %xmm2, %xmm3
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbw %xmm2, %xmm2
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
 ; AVX1-NEXT:    vandnps %ymm1, %ymm2, %ymm1
diff --git a/llvm/test/CodeGen/X86/vselect-avx.ll b/llvm/test/CodeGen/X86/vselect-avx.ll
index 3cd5654771c5a..24849c2b850cd 100644
--- a/llvm/test/CodeGen/X86/vselect-avx.ll
+++ b/llvm/test/CodeGen/X86/vselect-avx.ll
@@ -42,7 +42,7 @@ define void @test2(double** %call1559, i64 %indvars.iv4198, <4 x i1> %tmp1895) {
 ; AVX1:       ## %bb.0: ## %bb
 ; AVX1-NEXT:    vpslld $31, %xmm0, %xmm0
 ; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    movq (%rdi,%rsi,8), %rax
diff --git a/llvm/test/CodeGen/X86/vselect-pcmp.ll b/llvm/test/CodeGen/X86/vselect-pcmp.ll
index 4c56c654defae..42fbdb186357f 100644
--- a/llvm/test/CodeGen/X86/vselect-pcmp.ll
+++ b/llvm/test/CodeGen/X86/vselect-pcmp.ll
@@ -364,7 +364,7 @@ define <4 x double> @signbit_sel_v4f64_small_mask(<4 x double> %x, <4 x double>
 ; AVX1-LABEL: signbit_sel_v4f64_small_mask:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpmovsxdq %xmm2, %xmm3
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxdq %xmm2, %xmm2
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
 ; AVX1-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
@@ -397,7 +397,7 @@ define <4 x double> @signbit_sel_v4f64_small_mask(<4 x double> %x, <4 x double>
 ; XOP-LABEL: signbit_sel_v4f64_small_mask:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vpmovsxdq %xmm2, %xmm3
-; XOP-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
+; XOP-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
 ; XOP-NEXT:    vpmovsxdq %xmm2, %xmm2
 ; XOP-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
 ; XOP-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
diff --git a/llvm/test/CodeGen/X86/vshift-4.ll b/llvm/test/CodeGen/X86/vshift-4.ll
index 9362c41a3dc8a..1131d07b15d35 100644
--- a/llvm/test/CodeGen/X86/vshift-4.ll
+++ b/llvm/test/CodeGen/X86/vshift-4.ll
@@ -32,7 +32,7 @@ define void @shift1b(<2 x i64> %val, <2 x i64>* %dst, <2 x i64> %sh) nounwind {
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-NEXT:    movdqa %xmm0, %xmm2
 ; X32-NEXT:    psllq %xmm1, %xmm2
-; X32-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; X32-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; X32-NEXT:    psllq %xmm1, %xmm0
 ; X32-NEXT:    movsd {{.*#+}} xmm0 = xmm2[0],xmm0[1]
 ; X32-NEXT:    movapd %xmm0, (%eax)
@@ -42,7 +42,7 @@ define void @shift1b(<2 x i64> %val, <2 x i64>* %dst, <2 x i64> %sh) nounwind {
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    movdqa %xmm0, %xmm2
 ; X64-NEXT:    psllq %xmm1, %xmm2
-; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; X64-NEXT:    psllq %xmm1, %xmm0
 ; X64-NEXT:    movsd {{.*#+}} xmm0 = xmm2[0],xmm0[1]
 ; X64-NEXT:    movapd %xmm0, (%rdi)
diff --git a/llvm/test/CodeGen/X86/widen_conv-4.ll b/llvm/test/CodeGen/X86/widen_conv-4.ll
index 9c567fc384157..d5022b24eabe1 100644
--- a/llvm/test/CodeGen/X86/widen_conv-4.ll
+++ b/llvm/test/CodeGen/X86/widen_conv-4.ll
@@ -28,7 +28,7 @@ define void @convert_v7i16_v7f32(<7 x float>* %dst.addr, <7 x i16> %src) nounwin
 ; X86-SSE42-LABEL: convert_v7i16_v7f32:
 ; X86-SSE42:       # %bb.0: # %entry
 ; X86-SSE42-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE42-NEXT:    pmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
 ; X86-SSE42-NEXT:    cvtdq2ps %xmm1, %xmm1
 ; X86-SSE42-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
@@ -55,7 +55,7 @@ define void @convert_v7i16_v7f32(<7 x float>* %dst.addr, <7 x i16> %src) nounwin
 ;
 ; X64-SSE42-LABEL: convert_v7i16_v7f32:
 ; X64-SSE42:       # %bb.0: # %entry
-; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE42-NEXT:    pmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
 ; X64-SSE42-NEXT:    cvtdq2ps %xmm1, %xmm1
 ; X64-SSE42-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
diff --git a/llvm/test/CodeGen/X86/x86-setcc-int-to-fp-combine.ll b/llvm/test/CodeGen/X86/x86-setcc-int-to-fp-combine.ll
index 6f335c00b589f..e258cdf0035a0 100644
--- a/llvm/test/CodeGen/X86/x86-setcc-int-to-fp-combine.ll
+++ b/llvm/test/CodeGen/X86/x86-setcc-int-to-fp-combine.ll
@@ -30,7 +30,7 @@ define void @foo1(<4 x float> %val, <4 x float> %test, <4 x double>* %p) nounwin
 ; CHECK:       ## %bb.0:
 ; CHECK-NEXT:    cmpeqps %xmm1, %xmm0
 ; CHECK-NEXT:    andps {{.*}}(%rip), %xmm0
-; CHECK-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; CHECK-NEXT:    cvtdq2pd %xmm1, %xmm1
 ; CHECK-NEXT:    cvtdq2pd %xmm0, %xmm0
 ; CHECK-NEXT:    movaps %xmm0, (%rdi)
diff --git a/llvm/test/CodeGen/X86/xor.ll b/llvm/test/CodeGen/X86/xor.ll
index e836b00bb6206..e1d6929503f15 100644
--- a/llvm/test/CodeGen/X86/xor.ll
+++ b/llvm/test/CodeGen/X86/xor.ll
@@ -407,7 +407,7 @@ define i32 @PR17487(i1 %tobool) {
 ; X32-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X32-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
 ; X32-NEXT:    pandn {{\.LCPI.*}}, %xmm0
-; X32-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; X32-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; X32-NEXT:    movd %xmm0, %ecx
 ; X32-NEXT:    xorl %eax, %eax
 ; X32-NEXT:    cmpl $1, %ecx

From ca9bfc20f48c82a9f223ec814697714a16d1a22d Mon Sep 17 00:00:00 2001
From: Bruno Ricci <riccibrun@gmail.com>
Date: Sun, 26 Jul 2020 17:10:59 +0100
Subject: [PATCH 0141/1035] [clang][NFC] Remove spurious +x flag on
 SemaConcept.cpp

---
 clang/lib/Sema/SemaConcept.cpp | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 mode change 100755 => 100644 clang/lib/Sema/SemaConcept.cpp

diff --git a/clang/lib/Sema/SemaConcept.cpp b/clang/lib/Sema/SemaConcept.cpp
old mode 100755
new mode 100644

From a2f83d5a07daf7a9b717fff8916c44cd7cc1c678 Mon Sep 17 00:00:00 2001
From: Bruno Ricci <riccibrun@gmail.com>
Date: Sun, 26 Jul 2020 17:20:56 +0100
Subject: [PATCH 0142/1035] [clang][NFC] Add tests for the use of
 NamedDecl::getDeclName in the unused/unneeded diagnostics.

---
 clang/test/SemaCXX/warn-func-not-needed.cpp   | 22 +++--
 clang/test/SemaCXX/warn-member-not-needed.cpp | 12 ++-
 clang/test/SemaCXX/warn-unused-filescoped.cpp | 98 +++++++++++--------
 .../test/SemaCXX/warn-variable-not-needed.cpp |  7 +-
 4 files changed, 89 insertions(+), 50 deletions(-)

diff --git a/clang/test/SemaCXX/warn-func-not-needed.cpp b/clang/test/SemaCXX/warn-func-not-needed.cpp
index 65721f44f5707..5040aaad94601 100644
--- a/clang/test/SemaCXX/warn-func-not-needed.cpp
+++ b/clang/test/SemaCXX/warn-func-not-needed.cpp
@@ -1,13 +1,23 @@
 // RUN: %clang_cc1 -fsyntax-only -verify -Wall %s
 
 namespace test1 {
-  static void f() {} // expected-warning {{is not needed and will not be emitted}}
-  static void f();
-  template <typename T>
-  void foo() {
-    f();
-  }
+static void f() {} // expected-warning {{function 'f' is not needed and will not be emitted}}
+static void f();
+template <typename T>
+void foo() {
+  f();
+}
+}
+
+namespace test1_template {
+template <typename T> static void f() {}
+template <> void f<int>() {} // expected-warning {{function 'f' is not needed and will not be emitted}}
+template <typename T>
+void foo() {
+  f<int>();
+  f<long>();
 }
+} // namespace test1_template
 
 namespace test2 {
   static void f() {}
diff --git a/clang/test/SemaCXX/warn-member-not-needed.cpp b/clang/test/SemaCXX/warn-member-not-needed.cpp
index 61bb3488c6116..95241f4f7fee0 100644
--- a/clang/test/SemaCXX/warn-member-not-needed.cpp
+++ b/clang/test/SemaCXX/warn-member-not-needed.cpp
@@ -1,11 +1,19 @@
-// RUN: %clang_cc1 -fsyntax-only -verify -Wunneeded-member-function %s
+// RUN: %clang_cc1 -fsyntax-only -verify -Wunneeded-member-function -Wno-unused-template %s
 
 namespace {
   class A {
-    void g() {} // expected-warning {{is not needed and will not be emitted}}
+    void g() {} // expected-warning {{member function 'g' is not needed and will not be emitted}}
+    template <typename T> void gt(T) {}
+    template <> void gt<int>(int) {} // expected-warning {{member function 'gt' is not needed and will not be emitted}}
+    template <> void gt(float) {}    // expected-warning {{member function 'gt' is not needed and will not be emitted}}
+
     template <typename T>
     void foo() {
       g();
+      gt(0);
+      gt(0.0f);
+      gt(0.0);
     }
   };
+  template void A::gt(double); // no-warning
 }
diff --git a/clang/test/SemaCXX/warn-unused-filescoped.cpp b/clang/test/SemaCXX/warn-unused-filescoped.cpp
index 7ea398feb2b1e..056543d5eeb08 100644
--- a/clang/test/SemaCXX/warn-unused-filescoped.cpp
+++ b/clang/test/SemaCXX/warn-unused-filescoped.cpp
@@ -1,14 +1,15 @@
-// RUN: %clang_cc1 -fsyntax-only -verify -Wunused -Wunused-template -Wunused-member-function -Wno-unused-local-typedefs -Wno-c++11-extensions -std=c++98 %s
+// RUN: %clang_cc1 -fsyntax-only -verify -Wunused -Wunused-template -Wunused-member-function -Wno-unused-local-typedefs \
+// RUN:            -Wno-c++11-extensions -Wno-c++14-extensions -std=c++98 %s
 // RUN: %clang_cc1 -fsyntax-only -verify -Wunused -Wunused-template -Wunused-member-function -Wno-unused-local-typedefs -std=c++14 %s
 
 #ifdef HEADER
 
-static void headerstatic() {}  // expected-warning{{unused}}
+static void headerstatic() {} // expected-warning{{unused function 'headerstatic'}}
 static inline void headerstaticinline() {}
 
 namespace {
-  void headeranon() {}  // expected-warning{{unused}}
-  inline void headerinlineanon() {}
+void headeranon() {} // expected-warning{{unused function 'headeranon'}}
+inline void headerinlineanon() {}
 }
 
 namespace test7
@@ -43,31 +44,31 @@ namespace pr19713 {
 #define HEADER
 #include "warn-unused-filescoped.cpp"
 
-static void f1(); // expected-warning{{unused}}
+static void f1(); // expected-warning{{unused function 'f1'}}
 
 namespace {
-  void f2();  // expected-warning{{unused}}
+void f2(); // expected-warning{{unused function 'f2'}}
 
-  void f3() { }  // expected-warning{{unused}}
+void f3() {} // expected-warning{{unused function 'f3'}}
 
-  struct S {
-    void m1() { }  // expected-warning{{unused}}
-    void m2();  // expected-warning{{unused}}
-    void m3();
-    S(const S&);
-    void operator=(const S&);
-  };
+struct S {
+  void m1() {} // expected-warning{{unused member function 'm1'}}
+  void m2();   // expected-warning{{unused member function 'm2'}}
+  void m3();
+  S(const S &);
+  void operator=(const S &);
+};
 
   template <typename T>
   struct TS {
     void m();
   };
-  template <> void TS<int>::m() { }  // expected-warning{{unused}}
+  template <> void TS<int>::m() {} // expected-warning{{unused member function 'm'}}
 
   template <typename T>
-  void tf() { }  // expected-warning{{unused}}
-  template <> void tf<int>() { }  // expected-warning{{unused}}
-  
+  void tf() {}                  // expected-warning{{unused function template 'tf'}}
+  template <> void tf<int>() {} // expected-warning{{unused function 'tf'}}
+
   struct VS {
     virtual void vm() { }
   };
@@ -77,27 +78,31 @@ namespace {
   };
 }
 
-void S::m3() { }  // expected-warning{{unused}}
+void S::m3() {} // expected-warning{{unused member function 'm3'}}
 
-static inline void f4() { }  // expected-warning{{unused}}
-const unsigned int cx = 0; // expected-warning{{unused}}
+static inline void f4() {} // expected-warning{{unused function 'f4'}}
+const unsigned int cx = 0; // expected-warning{{unused variable 'cx'}}
 const unsigned int cy = 0;
 int f5() { return cy; }
 
-static int x1;  // expected-warning{{unused}}
+static int x1; // expected-warning{{unused variable 'x1'}}
 
 namespace {
-  int x2;  // expected-warning{{unused}}
-  
-  struct S2 {
-    static int x;  // expected-warning{{unused}}
-  };
+int x2; // expected-warning{{unused variable 'x2'}}
+
+struct S2 {
+  static int x; // expected-warning{{unused variable 'x'}}
+};
 
   template <typename T>
   struct TS2 {
     static int x;
   };
-  template <> int TS2<int>::x;  // expected-warning{{unused}}
+  template <> int TS2<int>::x; // expected-warning{{unused variable 'x'}}
+
+  template <typename T, typename U> int vt = 0; // expected-warning {{unused variable template 'vt'}}
+  template <typename T> int vt<T, void> = 0;
+  template <> int vt<void, void> = 0; // expected-warning {{unused variable 'vt'}}
 }
 
 namespace PR8841 {
@@ -120,17 +125,21 @@ namespace PR8841 {
 namespace test4 {
   namespace { struct A {}; }
 
-  void test(A a); // expected-warning {{unused function}}
+  void test(A a); // expected-warning {{unused function 'test'}}
   extern "C" void test4(A a);
 }
 
 namespace rdar8733476 {
-  static void foo() { } // expected-warning {{not needed and will not be emitted}}
+static void foo() {}                         // expected-warning {{function 'foo' is not needed and will not be emitted}}
+template <typename T> static void foo_t() {} // expected-warning {{unused function template 'foo_t'}}
+template <> void foo_t<int>() {}             // expected-warning {{function 'foo_t' is not needed and will not be emitted}}
 
-  template <int>
-  void bar() {
-    foo();
-  }
+template <int>
+void bar() {
+  foo();
+  foo_t<int>();
+  foo_t<void>();
+}
 }
 
 namespace test5 {
@@ -142,8 +151,15 @@ namespace test5 {
   // FIXME: We should produce warnings for both of these.
   static const int m = n;
   int x = sizeof(m);
-  static const double d = 0.0; // expected-warning{{not needed and will not be emitted}}
+  static const double d = 0.0; // expected-warning{{variable 'd' is not needed and will not be emitted}}
   int y = sizeof(d);
+
+  namespace {
+  // FIXME: Should be "unused variable template 'var_t'" instead.
+  template <typename T> const double var_t = 0; // expected-warning {{unused variable 'var_t'}}
+  template <> const double var_t<int> = 0;      // expected-warning {{variable 'var_t' is not needed and will not be emitted}}
+  int z = sizeof(var_t<int>);                   // expected-warning {{unused variable 'z'}}
+  }                                             // namespace
 }
 
 namespace unused_nested {
@@ -158,9 +174,9 @@ namespace unused_nested {
 
 namespace unused {
   struct {
-    void func() { // expected-warning {{unused member function}}
+    void func() { // expected-warning {{unused member function 'func'}}
     }
-  } x; // expected-warning {{unused variable}}
+  } x; // expected-warning {{unused variable 'x'}}
 }
 
 namespace test6 {
@@ -201,8 +217,8 @@ static void func() {}
 }
 
 namespace test9 {
-template<typename T>
-static void completeRedeclChainForTemplateSpecialization() { } // expected-warning {{unused}}
+template <typename T>
+static void completeRedeclChainForTemplateSpecialization() {} // expected-warning {{unused function template 'completeRedeclChainForTemplateSpecialization'}}
 }
 
 namespace test10 {
@@ -216,8 +232,8 @@ constexpr T pi = T(3.14);
 namespace pr19713 {
 #if __cplusplus >= 201103L
   // FIXME: We should warn on both of these.
-  static constexpr int constexpr3() { return 1; } // expected-warning {{unused}}
-  constexpr int constexpr4() { return 2; }
+static constexpr int constexpr3() { return 1; } // expected-warning {{unused function 'constexpr3'}}
+constexpr int constexpr4() { return 2; }
 #endif
 }
 
diff --git a/clang/test/SemaCXX/warn-variable-not-needed.cpp b/clang/test/SemaCXX/warn-variable-not-needed.cpp
index 0fb0f8151b449..139c2923f4bae 100644
--- a/clang/test/SemaCXX/warn-variable-not-needed.cpp
+++ b/clang/test/SemaCXX/warn-variable-not-needed.cpp
@@ -2,9 +2,14 @@
 
 namespace test1 {
   static int abc = 42; // expected-warning {{variable 'abc' is not needed and will not be emitted}}
+
+  namespace {
+  template <typename T> int abc_template = 0;
+  template <> int abc_template<int> = 0; // expected-warning {{variable 'abc_template' is not needed and will not be emitted}}
+  }                                      // namespace
   template <typename T>
   int foo(void) {
-    return abc;
+    return abc + abc_template<int> + abc_template<long>;
   }
 }
 

From b0512eed1e9dc03dba4ef8cccee73c13d3487565 Mon Sep 17 00:00:00 2001
From: Bruno Ricci <riccibrun@gmail.com>
Date: Sun, 26 Jul 2020 17:24:43 +0100
Subject: [PATCH 0143/1035] [clang][NFC] Add a test for
 __attribute__((flag_enum)) with an unnamed enumeration.

---
 clang/test/Sema/attr-flag-enum.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/clang/test/Sema/attr-flag-enum.c b/clang/test/Sema/attr-flag-enum.c
index ae3e3ad5ab99f..467afd950973d 100644
--- a/clang/test/Sema/attr-flag-enum.c
+++ b/clang/test/Sema/attr-flag-enum.c
@@ -6,6 +6,10 @@ enum __attribute__((flag_enum)) flag {
   ec = 0x8,
 };
 
+enum __attribute__((flag_enum)) {
+  g = 0x7,  // expected-warning {{enumeration value 'g' is out of range of flags in enumeration type ''}}
+};
+
 enum __attribute__((flag_enum)) flag2 {
   ga = 0x1,
   gb = 0x4,

From ab4e1be7ab3f1a1e7b78316b5b1929963e0e3bce Mon Sep 17 00:00:00 2001
From: Juneyoung Lee <aqjune@gmail.com>
Date: Mon, 27 Jul 2020 02:23:51 +0900
Subject: [PATCH 0144/1035] [InstCombine] Add a test for folding freeze into
 phi; NFC

---
 .../test/Transforms/InstCombine/freeze-phi.ll | 171 ++++++++++++++++++
 1 file changed, 171 insertions(+)
 create mode 100644 llvm/test/Transforms/InstCombine/freeze-phi.ll

diff --git a/llvm/test/Transforms/InstCombine/freeze-phi.ll b/llvm/test/Transforms/InstCombine/freeze-phi.ll
new file mode 100644
index 0000000000000..fcda502a0d7f8
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/freeze-phi.ll
@@ -0,0 +1,171 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define i32 @const(i1 %cond) {
+; CHECK-LABEL: @const(
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[A:%.*]], label [[B:%.*]]
+; CHECK:       A:
+; CHECK-NEXT:    br label [[C:%.*]]
+; CHECK:       B:
+; CHECK-NEXT:    br label [[C]]
+; CHECK:       C:
+; CHECK-NEXT:    [[Y:%.*]] = phi i32 [ 0, [[A]] ], [ 1, [[B]] ]
+; CHECK-NEXT:    ret i32 [[Y]]
+;
+  br i1 %cond, label %A, label %B
+A:
+  br label %C
+B:
+  br label %C
+C:
+  %y = phi i32 [0, %A], [1, %B]
+  %y.fr = freeze i32 %y
+  ret i32 %y
+}
+
+define i32 @one(i1 %cond, i32 %x) {
+; CHECK-LABEL: @one(
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[A:%.*]], label [[B:%.*]]
+; CHECK:       A:
+; CHECK-NEXT:    br label [[C:%.*]]
+; CHECK:       B:
+; CHECK-NEXT:    br label [[C]]
+; CHECK:       C:
+; CHECK-NEXT:    [[Y:%.*]] = phi i32 [ 0, [[A]] ], [ [[X:%.*]], [[B]] ]
+; CHECK-NEXT:    [[Y_FR:%.*]] = freeze i32 [[Y]]
+; CHECK-NEXT:    ret i32 [[Y_FR]]
+;
+  br i1 %cond, label %A, label %B
+A:
+  br label %C
+B:
+  br label %C
+C:
+  %y = phi i32 [0, %A], [%x, %B]
+  %y.fr = freeze i32 %y
+  ret i32 %y.fr
+}
+
+define i32 @two(i1 %cond, i32 %x, i32 %x2) {
+; CHECK-LABEL: @two(
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[A:%.*]], label [[B:%.*]]
+; CHECK:       A:
+; CHECK-NEXT:    br label [[C:%.*]]
+; CHECK:       B:
+; CHECK-NEXT:    br label [[C]]
+; CHECK:       C:
+; CHECK-NEXT:    [[Y:%.*]] = phi i32 [ [[X:%.*]], [[A]] ], [ [[X2:%.*]], [[B]] ]
+; CHECK-NEXT:    [[Y_FR:%.*]] = freeze i32 [[Y]]
+; CHECK-NEXT:    ret i32 [[Y_FR]]
+;
+  br i1 %cond, label %A, label %B
+A:
+  br label %C
+B:
+  br label %C
+C:
+  %y = phi i32 [%x, %A], [%x2, %B]
+  %y.fr = freeze i32 %y
+  ret i32 %y.fr
+}
+
+define i32 @two_undef(i8 %cond, i32 %x) {
+; CHECK-LABEL: @two_undef(
+; CHECK-NEXT:    switch i8 [[COND:%.*]], label [[A:%.*]] [
+; CHECK-NEXT:    i8 0, label [[B:%.*]]
+; CHECK-NEXT:    i8 1, label [[C:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       A:
+; CHECK-NEXT:    br label [[D:%.*]]
+; CHECK:       B:
+; CHECK-NEXT:    br label [[D]]
+; CHECK:       C:
+; CHECK-NEXT:    br label [[D]]
+; CHECK:       D:
+; CHECK-NEXT:    [[Y:%.*]] = phi i32 [ undef, [[A]] ], [ [[X:%.*]], [[B]] ], [ 0, [[C]] ]
+; CHECK-NEXT:    [[Y_FR:%.*]] = freeze i32 [[Y]]
+; CHECK-NEXT:    ret i32 [[Y_FR]]
+;
+  switch i8 %cond, label %A [
+  i8 0, label %B
+  i8 1, label %C
+  ]
+A:
+  br label %D
+B:
+  br label %D
+C:
+  br label %D
+D:
+  %y = phi i32 [undef, %A], [%x, %B], [0, %C]
+  %y.fr = freeze i32 %y
+  ret i32 %y.fr
+}
+
+define i32 @one_undef(i8 %cond) {
+; CHECK-LABEL: @one_undef(
+; CHECK-NEXT:    switch i8 [[COND:%.*]], label [[A:%.*]] [
+; CHECK-NEXT:    i8 0, label [[B:%.*]]
+; CHECK-NEXT:    i8 1, label [[C:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       A:
+; CHECK-NEXT:    br label [[D:%.*]]
+; CHECK:       B:
+; CHECK-NEXT:    br label [[D]]
+; CHECK:       C:
+; CHECK-NEXT:    br label [[D]]
+; CHECK:       D:
+; CHECK-NEXT:    [[Y:%.*]] = phi i32 [ undef, [[A]] ], [ 32, [[B]] ], [ 0, [[C]] ]
+; CHECK-NEXT:    [[Y_FR:%.*]] = freeze i32 [[Y]]
+; CHECK-NEXT:    ret i32 [[Y_FR]]
+;
+  switch i8 %cond, label %A [
+  i8 0, label %B
+  i8 1, label %C
+  ]
+A:
+  br label %D
+B:
+  br label %D
+C:
+  br label %D
+D:
+  %y = phi i32 [undef, %A], [32, %B], [0, %C]
+  %y.fr = freeze i32 %y
+  ret i32 %y.fr
+}
+
+@glb = global i8 0
+
+define i32 @one_constexpr(i8 %cond, i32 %x) {
+; CHECK-LABEL: @one_constexpr(
+; CHECK-NEXT:    switch i8 [[COND:%.*]], label [[A:%.*]] [
+; CHECK-NEXT:    i8 0, label [[B:%.*]]
+; CHECK-NEXT:    i8 1, label [[C:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       A:
+; CHECK-NEXT:    br label [[D:%.*]]
+; CHECK:       B:
+; CHECK-NEXT:    br label [[D]]
+; CHECK:       C:
+; CHECK-NEXT:    br label [[D]]
+; CHECK:       D:
+; CHECK-NEXT:    [[Y:%.*]] = phi i32 [ ptrtoint (i8* getelementptr inbounds (i8, i8* @glb, i64 2) to i32), [[A]] ], [ 32, [[B]] ], [ 0, [[C]] ]
+; CHECK-NEXT:    [[Y_FR:%.*]] = freeze i32 [[Y]]
+; CHECK-NEXT:    ret i32 [[Y_FR]]
+;
+  switch i8 %cond, label %A [
+  i8 0, label %B
+  i8 1, label %C
+  ]
+A:
+  br label %D
+B:
+  br label %D
+C:
+  br label %D
+D:
+  %y = phi i32 [ptrtoint (i8* getelementptr inbounds (i8, i8* @glb, i64 2) to i32), %A], [32, %B], [0, %C]
+  %y.fr = freeze i32 %y
+  ret i32 %y.fr
+}

From 1a75d88b3eb4e0cc5494281f24f362ece3d4aa4a Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@intel.com>
Date: Sun, 26 Jul 2020 10:38:34 -0700
Subject: [PATCH 0145/1035] [X86] Move getGatherOverhead/getScatterOverhead
 into X86TargetTransformInfo.

These cost methods don't make much sense in X86Subtarget. Make
them methods in X86's TTI and move the feature checks from the
X86Subtarget constructor into these methods.

Reviewed By: RKSimon

Differential Revision: https://reviews.llvm.org/D84594
---
 llvm/lib/Target/X86/X86Subtarget.cpp          | 11 ---------
 llvm/lib/Target/X86/X86Subtarget.h            |  6 -----
 .../lib/Target/X86/X86TargetTransformInfo.cpp | 24 +++++++++++++++++--
 llvm/lib/Target/X86/X86TargetTransformInfo.h  |  3 +++
 4 files changed, 25 insertions(+), 19 deletions(-)

diff --git a/llvm/lib/Target/X86/X86Subtarget.cpp b/llvm/lib/Target/X86/X86Subtarget.cpp
index 07e913e139111..095aea3a10636 100644
--- a/llvm/lib/Target/X86/X86Subtarget.cpp
+++ b/llvm/lib/Target/X86/X86Subtarget.cpp
@@ -262,17 +262,6 @@ void X86Subtarget::initSubtargetFeatures(StringRef CPU, StringRef FS) {
            isTargetKFreeBSD() || In64BitMode)
     stackAlignment = Align(16);
 
-  // Some CPUs have more overhead for gather. The specified overhead is relative
-  // to the Load operation. "2" is the number provided by Intel architects. This
-  // parameter is used for cost estimation of Gather Op and comparison with
-  // other alternatives.
-  // TODO: Remove the explicit hasAVX512()?, That would mean we would only
-  // enable gather with a -march.
-  if (hasAVX512() || (hasAVX2() && hasFastGather()))
-    GatherOverhead = 2;
-  if (hasAVX512())
-    ScatterOverhead = 2;
-
   // Consume the vector width attribute or apply any target specific limit.
   if (PreferVectorWidthOverride)
     PreferVectorWidth = PreferVectorWidthOverride;
diff --git a/llvm/lib/Target/X86/X86Subtarget.h b/llvm/lib/Target/X86/X86Subtarget.h
index 5b5ab4b969aac..e555dfdd638ac 100644
--- a/llvm/lib/Target/X86/X86Subtarget.h
+++ b/llvm/lib/Target/X86/X86Subtarget.h
@@ -506,10 +506,6 @@ class X86Subtarget final : public X86GenSubtargetInfo {
   /// True if compiling for 16-bit, false for 32-bit or 64-bit.
   bool In16BitMode = false;
 
-  /// Contains the Overhead of gather\scatter instructions
-  int GatherOverhead = 1024;
-  int ScatterOverhead = 1024;
-
   X86SelectionDAGInfo TSInfo;
   // Ordering here is important. X86InstrInfo initializes X86RegisterInfo which
   // X86TargetLowering needs.
@@ -678,8 +674,6 @@ class X86Subtarget final : public X86GenSubtargetInfo {
   bool isPMADDWDSlow() const { return IsPMADDWDSlow; }
   bool isUnalignedMem16Slow() const { return IsUAMem16Slow; }
   bool isUnalignedMem32Slow() const { return IsUAMem32Slow; }
-  int getGatherOverhead() const { return GatherOverhead; }
-  int getScatterOverhead() const { return ScatterOverhead; }
   bool hasSSEUnalignedMem() const { return HasSSEUnalignedMem; }
   bool hasCmpxchg16b() const { return HasCmpxchg16b && is64Bit(); }
   bool useLeaForSP() const { return UseLeaForSP; }
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index cc18e55656ef5..491078fd0542e 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -3848,6 +3848,26 @@ X86TTIImpl::getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind) {
   return CostKind == TTI::TCK_RecipThroughput ? 0 : 1;
 }
 
+int X86TTIImpl::getGatherOverhead() const {
+  // Some CPUs have more overhead for gather. The specified overhead is relative
+  // to the Load operation. "2" is the number provided by Intel architects. This
+  // parameter is used for cost estimation of Gather Op and comparison with
+  // other alternatives.
+  // TODO: Remove the explicit hasAVX512()?, That would mean we would only
+  // enable gather with a -march.
+  if (ST->hasAVX512() || (ST->hasAVX2() && ST->hasFastGather()))
+    return 2;
+
+  return 1024;
+}
+
+int X86TTIImpl::getScatterOverhead() const {
+  if (ST->hasAVX512())
+    return 2;
+
+  return 1024;
+}
+
 // Return an average cost of Gather / Scatter instruction, maybe improved later
 int X86TTIImpl::getGSVectorCost(unsigned Opcode, Type *SrcVTy, const Value *Ptr,
                                 Align Alignment, unsigned AddressSpace) {
@@ -3906,8 +3926,8 @@ int X86TTIImpl::getGSVectorCost(unsigned Opcode, Type *SrcVTy, const Value *Ptr,
   // The gather / scatter cost is given by Intel architects. It is a rough
   // number since we are looking at one instruction in a time.
   const int GSOverhead = (Opcode == Instruction::Load)
-                             ? ST->getGatherOverhead()
-                             : ST->getScatterOverhead();
+                             ? getGatherOverhead()
+                             : getScatterOverhead();
   return GSOverhead + VF * getMemoryOpCost(Opcode, SrcVTy->getScalarType(),
                                            MaybeAlign(Alignment), AddressSpace,
                                            TTI::TCK_RecipThroughput);
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.h b/llvm/lib/Target/X86/X86TargetTransformInfo.h
index ca875fa68523c..5ccaf409790db 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.h
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.h
@@ -244,6 +244,9 @@ class X86TTIImpl : public BasicTTIImplBase<X86TTIImpl> {
   int getGSVectorCost(unsigned Opcode, Type *DataTy, const Value *Ptr,
                       Align Alignment, unsigned AddressSpace);
 
+  int getGatherOverhead() const;
+  int getScatterOverhead() const;
+
   /// @}
 };
 

From 3bbf3e026d3c692966583075ae6d12c4575e9d72 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Hannes=20K=C3=A4ufler?= <hannes.kaeufler@gmail.com>
Date: Sun, 26 Jul 2020 13:59:45 -0400
Subject: [PATCH 0146/1035] Replace comment by private method; NFC.

---
 clang-tools-extra/clang-tidy/utils/HeaderGuard.cpp | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/clang-tools-extra/clang-tidy/utils/HeaderGuard.cpp b/clang-tools-extra/clang-tidy/utils/HeaderGuard.cpp
index 68bb987c1275f..1cae618dfd093 100644
--- a/clang-tools-extra/clang-tidy/utils/HeaderGuard.cpp
+++ b/clang-tools-extra/clang-tidy/utils/HeaderGuard.cpp
@@ -123,12 +123,7 @@ class HeaderGuardPPCallbacks : public PPCallbacks {
 
     // Emit warnings for headers that are missing guards.
     checkGuardlessHeaders();
-
-    // Clear all state.
-    Macros.clear();
-    Files.clear();
-    Ifndefs.clear();
-    EndIfs.clear();
+    clearAllState();
   }
 
   bool wouldFixEndifComment(StringRef FileName, SourceLocation EndIf,
@@ -255,6 +250,13 @@ class HeaderGuardPPCallbacks : public PPCallbacks {
   }
 
 private:
+  void clearAllState() {
+    Macros.clear();
+    Files.clear();
+    Ifndefs.clear();
+    EndIfs.clear();
+  }
+
   std::vector<std::pair<Token, const MacroInfo *>> Macros;
   llvm::StringMap<const FileEntry *> Files;
   std::map<const IdentifierInfo *, std::pair<SourceLocation, SourceLocation>>

From 7454acdf3b7d064ebbf6b8027296f42f504b285a Mon Sep 17 00:00:00 2001
From: Tim Keith <tkeith@nvidia.com>
Date: Sun, 26 Jul 2020 12:13:36 -0700
Subject: [PATCH 0147/1035] [flang] Fix implicit declarations in statement
 functions

If a symbol (that is not a dummy argument) is implicitly declared inside
a statement function, don't create it in the statement function's scope.
Instead, treat statement functions like blocks when finding the inclusive
scope and create the symbol there.

Add a new flag, StmtFunction, to symbols that represent statement functions.

Differential Revision: https://reviews.llvm.org/D84588
---
 flang/include/flang/Semantics/scope.h  |  1 +
 flang/include/flang/Semantics/symbol.h |  1 +
 flang/lib/Semantics/resolve-names.cpp  |  4 +++-
 flang/lib/Semantics/scope.cpp          |  4 ++++
 flang/test/Semantics/symbol16.f90      | 12 +++++++++++-
 5 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/flang/include/flang/Semantics/scope.h b/flang/include/flang/Semantics/scope.h
index a67a008889276..5ebe5f32eb677 100644
--- a/flang/include/flang/Semantics/scope.h
+++ b/flang/include/flang/Semantics/scope.h
@@ -87,6 +87,7 @@ class Scope {
   bool IsModule() const; // only module, not submodule
   bool IsSubmodule() const;
   bool IsDerivedType() const { return kind_ == Kind::DerivedType; }
+  bool IsStmtFunction() const;
   bool IsParameterizedDerivedType() const;
   Symbol *symbol() { return symbol_; }
   const Symbol *symbol() const { return symbol_; }
diff --git a/flang/include/flang/Semantics/symbol.h b/flang/include/flang/Semantics/symbol.h
index 227f951aef5f0..a1fd1baef78d3 100644
--- a/flang/include/flang/Semantics/symbol.h
+++ b/flang/include/flang/Semantics/symbol.h
@@ -482,6 +482,7 @@ class Symbol {
       Error, // an error has been reported on this symbol
       Function, // symbol is a function
       Subroutine, // symbol is a subroutine
+      StmtFunction, // symbol is a statement function (Function is set too)
       Implicit, // symbol is implicitly typed
       ModFile, // symbol came from .mod file
       ParentComp, // symbol is the "parent component" of an extended type
diff --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp
index 7189b48482654..8ea2e8ed9a60a 100644
--- a/flang/lib/Semantics/resolve-names.cpp
+++ b/flang/lib/Semantics/resolve-names.cpp
@@ -2015,7 +2015,8 @@ void ScopeHandler::Say2(const parser::Name &name, MessageFixedText &&msg1,
 
 Scope &ScopeHandler::InclusiveScope() {
   for (auto *scope{&currScope()};; scope = &scope->parent()) {
-    if (scope->kind() != Scope::Kind::Block && !scope->IsDerivedType()) {
+    if (scope->kind() != Scope::Kind::Block && !scope->IsDerivedType() &&
+        !scope->IsStmtFunction()) {
       return *scope;
     }
   }
@@ -2692,6 +2693,7 @@ bool SubprogramVisitor::HandleStmtFunction(const parser::StmtFunctionStmt &x) {
     return true;
   }
   auto &symbol{PushSubprogramScope(name, Symbol::Flag::Function)};
+  symbol.set(Symbol::Flag::StmtFunction);
   EraseSymbol(symbol); // removes symbol added by PushSubprogramScope
   auto &details{symbol.get<SubprogramDetails>()};
   for (const auto &dummyName : std::get<std::list<parser::Name>>(x.t)) {
diff --git a/flang/lib/Semantics/scope.cpp b/flang/lib/Semantics/scope.cpp
index 02637ba2add7f..a2a9e1dbe9e73 100644
--- a/flang/lib/Semantics/scope.cpp
+++ b/flang/lib/Semantics/scope.cpp
@@ -333,6 +333,10 @@ llvm::raw_ostream &operator<<(llvm::raw_ostream &os, const Scope &scope) {
   return os;
 }
 
+bool Scope::IsStmtFunction() const {
+  return symbol_ && symbol_->test(Symbol::Flag::StmtFunction);
+}
+
 bool Scope::IsParameterizedDerivedType() const {
   if (!IsDerivedType()) {
     return false;
diff --git a/flang/test/Semantics/symbol16.f90 b/flang/test/Semantics/symbol16.f90
index ce47134fc3377..4fa6f2b9c0eab 100644
--- a/flang/test/Semantics/symbol16.f90
+++ b/flang/test/Semantics/symbol16.f90
@@ -3,7 +3,7 @@
 
 !DEF: /p1 MainProgram
 program p1
- !DEF: /p1/f (Function) Subprogram INTEGER(4)
+ !DEF: /p1/f (Function, StmtFunction) Subprogram INTEGER(4)
  !DEF: /p1/i ObjectEntity INTEGER(4)
  !DEF: /p1/j ObjectEntity INTEGER(4)
  integer f, i, j
@@ -15,3 +15,13 @@ program p1
  !REF: /p1/f
  j = f(2)
 end program
+
+!DEF: /p2 MainProgram
+program p2
+ !DEF: /p2/f (Function, StmtFunction) Subprogram REAL(4)
+ !DEF: /p2/f/x (Implicit) ObjectEntity REAL(4)
+ !DEF: /p2/y (Implicit) ObjectEntity REAL(4)
+ f(x) = y
+ !REF: /p2/y
+ y = 1.0
+end program

From df12524e6ba02d3eda975de4541f55e151074b07 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@intel.com>
Date: Sun, 26 Jul 2020 10:57:59 -0700
Subject: [PATCH 0148/1035] [X86] Turn X86DAGToDAGISel::tryVPTERNLOG into a
 fully custom instruction selector that can handle bitcasts between logic ops

Previously we just matched the logic ops and replaced with an
X86ISD::VPTERNLOG node that we would send through the normal
pattern match. But that approach couldn't handle a bitcast
between the logic ops. Extending that approach would require us
to peek through the bitcasts and emit new bitcasts to match
the types. Those new bitcasts would then have to be properly
topologically sorted.

This patch instead switches to directly emitting the
MachineSDNode and skips the normal tablegen pattern matching.
We do have to handle load folding and broadcast load folding
ourselves now. Which also means commuting the immediate control.

Reviewed By: RKSimon

Differential Revision: https://reviews.llvm.org/D83630
---
 llvm/lib/Target/X86/X86ISelDAGToDAG.cpp | 151 +++++++++++++++++++++---
 llvm/test/CodeGen/X86/avx512-logic.ll   |  30 ++---
 llvm/test/CodeGen/X86/avx512vl-logic.ll |  12 +-
 3 files changed, 145 insertions(+), 48 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
index 3cd80cb04ab84..4098911dee3b1 100644
--- a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
@@ -3940,30 +3940,39 @@ bool X86DAGToDAGISel::tryVPTERNLOG(SDNode *N) {
   if (!(Subtarget->hasVLX() || NVT.is512BitVector()))
     return false;
 
-  unsigned Opc1 = N->getOpcode();
   SDValue N0 = N->getOperand(0);
   SDValue N1 = N->getOperand(1);
 
-  auto isLogicOp = [](unsigned Opc) {
-    return Opc == ISD::AND || Opc == ISD::OR || Opc == ISD::XOR ||
-           Opc == X86ISD::ANDNP;
+  auto getFoldableLogicOp = [](SDValue Op) {
+    // Peek through single use bitcast.
+    if (Op.getOpcode() == ISD::BITCAST && Op.hasOneUse())
+      Op = Op.getOperand(0);
+
+    if (!Op.hasOneUse())
+      return SDValue();
+
+    unsigned Opc = Op.getOpcode();
+    if (Opc == ISD::AND || Opc == ISD::OR || Opc == ISD::XOR ||
+        Opc == X86ISD::ANDNP)
+      return Op;
+
+    return SDValue();
   };
 
-  SDValue A, B, C;
-  unsigned Opc2;
-  if (isLogicOp(N1.getOpcode()) && N1.hasOneUse()) {
-    Opc2 = N1.getOpcode();
+  SDValue A, FoldableOp;
+  if ((FoldableOp = getFoldableLogicOp(N1))) {
     A = N0;
-    B = N1.getOperand(0);
-    C = N1.getOperand(1);
-  } else if (isLogicOp(N0.getOpcode()) && N0.hasOneUse()) {
-    Opc2 = N0.getOpcode();
+  } else if ((FoldableOp = getFoldableLogicOp(N0))) {
     A = N1;
-    B = N0.getOperand(0);
-    C = N0.getOperand(1);
   } else
     return false;
 
+  SDValue B = FoldableOp.getOperand(0);
+  SDValue C = FoldableOp.getOperand(1);
+
+  unsigned Opc1 = N->getOpcode();
+  unsigned Opc2 = FoldableOp.getOpcode();
+
   uint64_t Imm;
   switch (Opc1) {
   default: llvm_unreachable("Unexpected opcode!");
@@ -3996,11 +4005,117 @@ bool X86DAGToDAGISel::tryVPTERNLOG(SDNode *N) {
     break;
   }
 
+  auto tryFoldLoadOrBCast =
+      [this](SDNode *Root, SDNode *P, SDValue &L, SDValue &Base, SDValue &Scale,
+             SDValue &Index, SDValue &Disp, SDValue &Segment) {
+        if (tryFoldLoad(Root, P, L, Base, Scale, Index, Disp, Segment))
+          return true;
+
+        // Not a load, check for broadcast which may be behind a bitcast.
+        if (L.getOpcode() == ISD::BITCAST && L.hasOneUse()) {
+          P = L.getNode();
+          L = L.getOperand(0);
+        }
+
+        if (L.getOpcode() != X86ISD::VBROADCAST_LOAD)
+          return false;
+
+        // Only 32 and 64 bit broadcasts are supported.
+        auto *MemIntr = cast<MemIntrinsicSDNode>(L);
+        unsigned Size = MemIntr->getMemoryVT().getSizeInBits();
+        if (Size != 32 && Size != 64)
+          return false;
+
+        return tryFoldBroadcast(Root, P, L, Base, Scale, Index, Disp, Segment);
+      };
+
+  bool FoldedLoad = false;
+  SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
+  if (tryFoldLoadOrBCast(N, FoldableOp.getNode(), C, Tmp0, Tmp1, Tmp2, Tmp3,
+                         Tmp4)) {
+    FoldedLoad = true;
+  } else if (tryFoldLoadOrBCast(N, N, A, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
+    FoldedLoad = true;
+    std::swap(A, C);
+    // Swap bits 1/4 and 3/6.
+    uint8_t OldImm = Imm;
+    Imm = OldImm & 0xa5;
+    if (OldImm & 0x02) Imm |= 0x10;
+    if (OldImm & 0x10) Imm |= 0x02;
+    if (OldImm & 0x08) Imm |= 0x40;
+    if (OldImm & 0x40) Imm |= 0x08;
+  } else if (tryFoldLoadOrBCast(N, FoldableOp.getNode(), B, Tmp0, Tmp1, Tmp2,
+                                Tmp3, Tmp4)) {
+    FoldedLoad = true;
+    std::swap(B, C);
+    // Swap bits 1/2 and 5/6.
+    uint8_t OldImm = Imm;
+    Imm = OldImm & 0x99;
+    if (OldImm & 0x02) Imm |= 0x04;
+    if (OldImm & 0x04) Imm |= 0x02;
+    if (OldImm & 0x20) Imm |= 0x40;
+    if (OldImm & 0x40) Imm |= 0x20;
+  }
+
   SDLoc DL(N);
-  SDValue New = CurDAG->getNode(X86ISD::VPTERNLOG, DL, NVT, A, B, C,
-                                CurDAG->getTargetConstant(Imm, DL, MVT::i8));
-  ReplaceNode(N, New.getNode());
-  SelectCode(New.getNode());
+
+  SDValue TImm = CurDAG->getTargetConstant(Imm, DL, MVT::i8);
+
+  MachineSDNode *MNode;
+  if (FoldedLoad) {
+    SDVTList VTs = CurDAG->getVTList(NVT, MVT::Other);
+
+    unsigned Opc;
+    if (C.getOpcode() == X86ISD::VBROADCAST_LOAD) {
+      auto *MemIntr = cast<MemIntrinsicSDNode>(C);
+      unsigned EltSize = MemIntr->getMemoryVT().getSizeInBits();
+      assert((EltSize == 32 || EltSize == 64) && "Unexpected broadcast size!");
+
+      bool UseD = EltSize == 32;
+      if (NVT.is128BitVector())
+        Opc = UseD ? X86::VPTERNLOGDZ128rmbi : X86::VPTERNLOGQZ128rmbi;
+      else if (NVT.is256BitVector())
+        Opc = UseD ? X86::VPTERNLOGDZ256rmbi : X86::VPTERNLOGQZ256rmbi;
+      else if (NVT.is512BitVector())
+        Opc = UseD ? X86::VPTERNLOGDZrmbi : X86::VPTERNLOGQZrmbi;
+      else
+        llvm_unreachable("Unexpected vector size!");
+    } else {
+      bool UseD = NVT.getVectorElementType() == MVT::i32;
+      if (NVT.is128BitVector())
+        Opc = UseD ? X86::VPTERNLOGDZ128rmi : X86::VPTERNLOGQZ128rmi;
+      else if (NVT.is256BitVector())
+        Opc = UseD ? X86::VPTERNLOGDZ256rmi : X86::VPTERNLOGQZ256rmi;
+      else if (NVT.is512BitVector())
+        Opc = UseD ? X86::VPTERNLOGDZrmi : X86::VPTERNLOGQZrmi;
+      else
+        llvm_unreachable("Unexpected vector size!");
+    }
+
+    SDValue Ops[] = {A, B, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, TImm, C.getOperand(0)};
+    MNode = CurDAG->getMachineNode(Opc, DL, VTs, Ops);
+
+    // Update the chain.
+    ReplaceUses(C.getValue(1), SDValue(MNode, 1));
+    // Record the mem-refs
+    CurDAG->setNodeMemRefs(MNode, {cast<MemSDNode>(C)->getMemOperand()});
+  } else {
+    bool UseD = NVT.getVectorElementType() == MVT::i32;
+    unsigned Opc;
+    if (NVT.is128BitVector())
+      Opc = UseD ? X86::VPTERNLOGDZ128rri : X86::VPTERNLOGQZ128rri;
+    else if (NVT.is256BitVector())
+      Opc = UseD ? X86::VPTERNLOGDZ256rri : X86::VPTERNLOGQZ256rri;
+    else if (NVT.is512BitVector())
+      Opc = UseD ? X86::VPTERNLOGDZrri : X86::VPTERNLOGQZrri;
+    else
+      llvm_unreachable("Unexpected vector size!");
+
+    MNode = CurDAG->getMachineNode(Opc, DL, NVT, {A, B, C, TImm});
+  }
+
+  ReplaceUses(SDValue(N, 0), SDValue(MNode, 0));
+  CurDAG->RemoveDeadNode(N);
   return true;
 }
 
diff --git a/llvm/test/CodeGen/X86/avx512-logic.ll b/llvm/test/CodeGen/X86/avx512-logic.ll
index 30607214f56d5..24e58149eb4c2 100644
--- a/llvm/test/CodeGen/X86/avx512-logic.ll
+++ b/llvm/test/CodeGen/X86/avx512-logic.ll
@@ -887,34 +887,20 @@ define <16 x i32> @ternlog_xor_andn(<16 x i32> %x, <16 x i32> %y, <16 x i32> %z)
 }
 
 define <16 x i32> @ternlog_or_and_mask(<16 x i32> %x, <16 x i32> %y) {
-; KNL-LABEL: ternlog_or_and_mask:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
-; KNL-NEXT:    vpord %zmm1, %zmm0, %zmm0
-; KNL-NEXT:    retq
-;
-; SKX-LABEL: ternlog_or_and_mask:
-; SKX:       ## %bb.0:
-; SKX-NEXT:    vandps {{.*}}(%rip), %zmm0, %zmm0
-; SKX-NEXT:    vorps %zmm1, %zmm0, %zmm0
-; SKX-NEXT:    retq
+; ALL-LABEL: ternlog_or_and_mask:
+; ALL:       ## %bb.0:
+; ALL-NEXT:    vpternlogd $236, {{.*}}(%rip), %zmm1, %zmm0
+; ALL-NEXT:    retq
   %a = and <16 x i32> %x, <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>
   %b = or <16 x i32> %a, %y
   ret <16 x i32> %b
 }
 
 define <8 x i64> @ternlog_xor_and_mask(<8 x i64> %x, <8 x i64> %y) {
-; KNL-LABEL: ternlog_xor_and_mask:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    vpandd {{.*}}(%rip), %zmm0, %zmm0
-; KNL-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
-; KNL-NEXT:    retq
-;
-; SKX-LABEL: ternlog_xor_and_mask:
-; SKX:       ## %bb.0:
-; SKX-NEXT:    vandps {{.*}}(%rip), %zmm0, %zmm0
-; SKX-NEXT:    vxorps %zmm1, %zmm0, %zmm0
-; SKX-NEXT:    retq
+; ALL-LABEL: ternlog_xor_and_mask:
+; ALL:       ## %bb.0:
+; ALL-NEXT:    vpternlogq $108, {{.*}}(%rip), %zmm1, %zmm0
+; ALL-NEXT:    retq
   %a = and <8 x i64> %x, <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>
   %b = xor <8 x i64> %a, %y
   ret <8 x i64> %b
diff --git a/llvm/test/CodeGen/X86/avx512vl-logic.ll b/llvm/test/CodeGen/X86/avx512vl-logic.ll
index 3f0ce30928478..13c4c8afb9a8f 100644
--- a/llvm/test/CodeGen/X86/avx512vl-logic.ll
+++ b/llvm/test/CodeGen/X86/avx512vl-logic.ll
@@ -991,8 +991,7 @@ define <4 x i32> @ternlog_xor_andn(<4 x i32> %x, <4 x i32> %y, <4 x i32> %z) {
 define <4 x i32> @ternlog_or_and_mask(<4 x i32> %x, <4 x i32> %y) {
 ; CHECK-LABEL: ternlog_or_and_mask:
 ; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vandps {{.*}}(%rip), %xmm0, %xmm0
-; CHECK-NEXT:    vorps %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    vpternlogd $236, {{.*}}(%rip), %xmm1, %xmm0
 ; CHECK-NEXT:    retq
   %a = and <4 x i32> %x, <i32 255, i32 255, i32 255, i32 255>
   %b = or <4 x i32> %a, %y
@@ -1002,8 +1001,7 @@ define <4 x i32> @ternlog_or_and_mask(<4 x i32> %x, <4 x i32> %y) {
 define <8 x i32> @ternlog_or_and_mask_ymm(<8 x i32> %x, <8 x i32> %y) {
 ; CHECK-LABEL: ternlog_or_and_mask_ymm:
 ; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
-; CHECK-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    vpternlogd $236, {{.*}}(%rip), %ymm1, %ymm0
 ; CHECK-NEXT:    retq
   %a = and <8 x i32> %x, <i32 -16777216, i32 -16777216, i32 -16777216, i32 -16777216, i32 -16777216, i32 -16777216, i32 -16777216, i32 -16777216>
   %b = or <8 x i32> %a, %y
@@ -1013,8 +1011,7 @@ define <8 x i32> @ternlog_or_and_mask_ymm(<8 x i32> %x, <8 x i32> %y) {
 define <2 x i64> @ternlog_xor_and_mask(<2 x i64> %x, <2 x i64> %y) {
 ; CHECK-LABEL: ternlog_xor_and_mask:
 ; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vandps {{.*}}(%rip), %xmm0, %xmm0
-; CHECK-NEXT:    vxorps %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    vpternlogq $108, {{.*}}(%rip), %xmm1, %xmm0
 ; CHECK-NEXT:    retq
   %a = and <2 x i64> %x, <i64 1099511627775, i64 1099511627775>
   %b = xor <2 x i64> %a, %y
@@ -1024,8 +1021,7 @@ define <2 x i64> @ternlog_xor_and_mask(<2 x i64> %x, <2 x i64> %y) {
 define <4 x i64> @ternlog_xor_and_mask_ymm(<4 x i64> %x, <4 x i64> %y) {
 ; CHECK-LABEL: ternlog_xor_and_mask_ymm:
 ; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
-; CHECK-NEXT:    vxorps %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    vpternlogq $108, {{.*}}(%rip), %ymm1, %ymm0
 ; CHECK-NEXT:    retq
   %a = and <4 x i64> %x, <i64 72057594037927935, i64 72057594037927935, i64 72057594037927935, i64 72057594037927935>
   %b = xor <4 x i64> %a, %y

From 9282d04e041c4dd21d3af8463e2cb30964a9272c Mon Sep 17 00:00:00 2001
From: Jez Ng <jezng@fb.com>
Date: Sun, 26 Jul 2020 12:46:46 -0700
Subject: [PATCH 0149/1035] [lld-macho] Support lookup of dylibs in frameworks

Needed for testing Objective-C programs (since e.g. Core
Foundation is a framework)

Reviewed By: #lld-macho, compnerd

Differential Revision: https://reviews.llvm.org/D83925
---
 lld/MachO/Config.h         |  1 -
 lld/MachO/Driver.cpp       | 38 ++++++++++++++++++++++++++++++++++++++
 lld/test/MachO/framework.s | 29 +++++++++++++++++++++++++++++
 3 files changed, 67 insertions(+), 1 deletion(-)
 create mode 100644 lld/test/MachO/framework.s

diff --git a/lld/MachO/Config.h b/lld/MachO/Config.h
index 79812a4335638..c66991b581fc0 100644
--- a/lld/MachO/Config.h
+++ b/lld/MachO/Config.h
@@ -30,7 +30,6 @@ struct Configuration {
   llvm::MachO::Architecture arch;
   llvm::MachO::HeaderFileType outputType;
   std::vector<llvm::StringRef> librarySearchPaths;
-  // TODO: use the framework search paths
   std::vector<llvm::StringRef> frameworkSearchPaths;
   llvm::DenseMap<llvm::StringRef, SymbolPriorityEntry> priorities;
 };
diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp
index 4dfb387e4e622..ee794129e1fc6 100644
--- a/lld/MachO/Driver.cpp
+++ b/lld/MachO/Driver.cpp
@@ -30,6 +30,7 @@
 #include "llvm/Object/Archive.h"
 #include "llvm/Option/ArgList.h"
 #include "llvm/Option/Option.h"
+#include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Host.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Path.h"
@@ -98,6 +99,32 @@ static Optional<std::string> findLibrary(StringRef name) {
   return {};
 }
 
+static Optional<std::string> findFramework(StringRef name) {
+  // TODO: support .tbd files
+  llvm::SmallString<260> symlink;
+  llvm::SmallString<260> location;
+  StringRef suffix;
+  std::tie(name, suffix) = name.split(",");
+  for (StringRef dir : config->frameworkSearchPaths) {
+    symlink = dir;
+    path::append(symlink, name + ".framework", name);
+    // If the symlink fails to resolve, skip to the next search path.
+    // NOTE: we must resolve the symlink before trying the suffixes, because
+    // there are no symlinks for the suffixed paths.
+    if (fs::real_path(symlink, location))
+      continue;
+    if (!suffix.empty()) {
+      llvm::Twine suffixed = location + suffix;
+      if (fs::exists(suffixed))
+        return suffixed.str();
+      // Suffix lookup failed, fall through to the no-suffix case.
+    }
+    if (fs::exists(location))
+      return location.str().str();
+  }
+  return {};
+}
+
 static TargetInfo *createTargetInfo(opt::InputArgList &args) {
   StringRef arch = args.getLastArgValue(OPT_arch, "x86_64");
   config->arch = llvm::MachO::getArchitectureFromName(
@@ -393,13 +420,24 @@ bool macho::link(llvm::ArrayRef<const char *> argsArr, bool canExitEarly,
       error("library not found for -l" + name);
       break;
     }
+    case OPT_framework: {
+      StringRef name = arg->getValue();
+      if (Optional<std::string> path = findFramework(name)) {
+        addFile(*path);
+        break;
+      }
+      error("framework not found for -framework " + name);
+      break;
+    }
     case OPT_platform_version:
       handlePlatformVersion(arg);
       break;
     case OPT_o:
     case OPT_dylib:
     case OPT_e:
+    case OPT_F:
     case OPT_L:
+    case OPT_install_name:
     case OPT_Z:
     case OPT_arch:
       // handled elsewhere
diff --git a/lld/test/MachO/framework.s b/lld/test/MachO/framework.s
new file mode 100644
index 0000000000000..a527970ed6bc9
--- /dev/null
+++ b/lld/test/MachO/framework.s
@@ -0,0 +1,29 @@
+# REQUIRES: x86, shell
+# RUN: mkdir -p %t
+# RUN: echo ".globl _foo; _foo: ret" | llvm-mc -filetype=obj -triple=x86_64-apple-darwin -o %t/foo.o
+# RUN: mkdir -p %t/Foo.framework/Versions/A
+# RUN: lld -flavor darwinnew -L%S/Inputs/MacOSX.sdk/usr/lib -dylib -install_name %t/Foo.framework/Versions/A/Foo %t/foo.o -o %t/Foo.framework/Versions/A/Foo
+# RUN: lld -flavor darwinnew -L%S/Inputs/MacOSX.sdk/usr/lib -dylib -install_name %t/Foo.framework/Versions/A/Foobar %t/foo.o -o %t/Foo.framework/Versions/A/Foobar
+# RUN: ln -sf %t/Foo.framework/Versions/A %t/Foo.framework/Versions/Current
+# RUN: ln -sf %t/Foo.framework/Versions/Current/Foo %t/Foo.framework/Foo
+
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin -o %t/test.o %s
+# RUN: lld -flavor darwinnew -L%S/Inputs/MacOSX.sdk/usr/lib -lSystem -F%t -framework Foo %t/test.o -o %t/test
+# RUN: llvm-objdump --macho --lazy-bind %t/test | FileCheck %s --check-prefix=NOSUFFIX
+# NOSUFFIX: __DATA __la_symbol_ptr 0x{{[0-9a-f]*}} {{.*}}Foo _foo
+
+# RUN: lld -flavor darwinnew -L%S/Inputs/MacOSX.sdk/usr/lib -lSystem -F%t -framework Foo,baz %t/test.o -o %t/test-wrong-suffix
+# RUN: llvm-objdump --macho --lazy-bind %t/test-wrong-suffix | FileCheck %s --check-prefix=NOSUFFIX
+
+# RUN: lld -flavor darwinnew -L%S/Inputs/MacOSX.sdk/usr/lib -lSystem -F%t -framework Foo,bar %t/test.o -o %t/test-suffix
+# RUN: llvm-objdump --macho --lazy-bind %t/test-suffix | FileCheck %s --check-prefix=SUFFIX
+# SUFFIX: __DATA __la_symbol_ptr 0x{{[0-9a-f]*}} {{.*}}Foobar _foo
+
+.globl _main
+.text
+_main:
+  sub $8, %rsp # 16-byte-align the stack; dylld -flavor darwinnew checks for this
+  callq _foo
+  mov $0, %rax
+  add $8, %rsp
+  ret

From 8dc820393219c7ee440b4ec86c9a201301943276 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Martin=20Storsj=C3=B6?= <martin@martin.st>
Date: Sat, 25 Jul 2020 15:01:48 +0300
Subject: [PATCH 0150/1035] [LLD] [COFF] Fix test to properly test all aspects
 of c3b1d730d6. NFC.

Previously, the test could pass with one part of c3b1d730d6 removed.
---
 lld/test/COFF/associative-comdat-mingw-i386.s | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lld/test/COFF/associative-comdat-mingw-i386.s b/lld/test/COFF/associative-comdat-mingw-i386.s
index 3f5e02330d505..8d89478d4eb03 100644
--- a/lld/test/COFF/associative-comdat-mingw-i386.s
+++ b/lld/test/COFF/associative-comdat-mingw-i386.s
@@ -30,7 +30,7 @@ _main:
         .scl            2;
         .type           32;
         .endef
-        .section        .text$foo,"xr",discard,foo
+        .section        .text$foo,"xr",discard,_foo
         .globl          _foo
         .p2align        4
 _foo:

From ff25b2da2ab9049e154cc8b9af06a24f79a74209 Mon Sep 17 00:00:00 2001
From: Valentin Clement <clementval@gmail.com>
Date: Sun, 26 Jul 2020 20:00:49 -0400
Subject: [PATCH 0151/1035] [flang][openacc] Basic name resolution
 infrastructure for OpenACC construct

Reviewed By: tskeith, klausler, ichoyjx

Differential Revision: https://reviews.llvm.org/D83998
---
 flang/include/flang/Semantics/symbol.h       |   7 +
 flang/lib/Semantics/resolve-names.cpp        | 749 +++++++++++++++----
 flang/lib/Semantics/unparse-with-symbols.cpp |   5 +
 flang/test/Semantics/acc-resolve01.f90       |  22 +
 flang/test/Semantics/acc-resolve02.f90       |  17 +
 flang/test/Semantics/acc-symbols01.f90       |  26 +
 flang/test/Semantics/test_symbols.sh         |   5 +-
 7 files changed, 689 insertions(+), 142 deletions(-)
 create mode 100644 flang/test/Semantics/acc-resolve01.f90
 create mode 100644 flang/test/Semantics/acc-resolve02.f90
 create mode 100644 flang/test/Semantics/acc-symbols01.f90

diff --git a/flang/include/flang/Semantics/symbol.h b/flang/include/flang/Semantics/symbol.h
index a1fd1baef78d3..3000a39c3b589 100644
--- a/flang/include/flang/Semantics/symbol.h
+++ b/flang/include/flang/Semantics/symbol.h
@@ -492,6 +492,13 @@ class Symbol {
       LocalityShared, // named in SHARED locality-spec
       InDataStmt, // initialized in a DATA statement
 
+      // OpenACC data-sharing attribute
+      AccPrivate, AccFirstPrivate, AccShared,
+      // OpenACC data-mapping attribute
+      AccCopyIn, AccCopyOut, AccCreate, AccDelete, AccPresent,
+      // OpenACC miscellaneous flags
+      AccCommonBlock, AccThreadPrivate, AccReduction, AccNone, AccPreDetermined,
+
       // OpenMP data-sharing attribute
       OmpShared, OmpPrivate, OmpLinear, OmpFirstPrivate, OmpLastPrivate,
       // OpenMP data-mapping attribute
diff --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp
index 8ea2e8ed9a60a..8d3e97d20521d 100644
--- a/flang/lib/Semantics/resolve-names.cpp
+++ b/flang/lib/Semantics/resolve-names.cpp
@@ -7,6 +7,7 @@
 
 #include "resolve-names.h"
 #include "assignment.h"
+#include "check-acc-structure.h"
 #include "check-omp-structure.h"
 #include "mod-file.h"
 #include "pointer-assignment.h"
@@ -1081,6 +1082,305 @@ class ConstructVisitor : public virtual DeclarationVisitor {
   void PopAssociation();
 };
 
+template <typename T> class DirectiveAttributeVisitor {
+public:
+  explicit DirectiveAttributeVisitor(
+      SemanticsContext &context, ResolveNamesVisitor &resolver)
+      : context_{context}, resolver_{resolver} {}
+
+protected:
+  struct DirContext {
+    DirContext(const parser::CharBlock &source, T d, Scope &s)
+        : directiveSource{source}, directive{d}, scope{s} {}
+    parser::CharBlock directiveSource;
+    T directive;
+    Scope &scope;
+    Symbol::Flag defaultDSA{Symbol::Flag::AccShared}; // TODOACC
+    std::map<const Symbol *, Symbol::Flag> objectWithDSA;
+    bool withinConstruct{false};
+    int64_t associatedLoopLevel{0};
+  };
+
+  DirContext &GetContext() {
+    CHECK(!dirContext_.empty());
+    return dirContext_.back();
+  }
+  void PushContext(const parser::CharBlock &source, T dir) {
+    dirContext_.emplace_back(source, dir, context_.FindScope(source));
+  }
+  void PopContext() { dirContext_.pop_back(); }
+  void SetContextDirectiveSource(parser::CharBlock &dir) {
+    GetContext().directiveSource = dir;
+  }
+  void SetContextDirectiveEnum(T dir) { GetContext().directive = dir; }
+  Scope &currScope() { return GetContext().scope; }
+  void SetContextDefaultDSA(Symbol::Flag flag) {
+    GetContext().defaultDSA = flag;
+  }
+  void AddToContextObjectWithDSA(
+      const Symbol &symbol, Symbol::Flag flag, DirContext &context) {
+    context.objectWithDSA.emplace(&symbol, flag);
+  }
+  void AddToContextObjectWithDSA(const Symbol &symbol, Symbol::Flag flag) {
+    AddToContextObjectWithDSA(symbol, flag, GetContext());
+  }
+  bool IsObjectWithDSA(const Symbol &symbol) {
+    auto it{GetContext().objectWithDSA.find(&symbol)};
+    return it != GetContext().objectWithDSA.end();
+  }
+  void SetContextAssociatedLoopLevel(int64_t level) {
+    GetContext().associatedLoopLevel = level;
+  }
+  Symbol &MakeAssocSymbol(const SourceName &name, Symbol &prev, Scope &scope) {
+    const auto pair{scope.try_emplace(name, Attrs{}, HostAssocDetails{prev})};
+    return *pair.first->second;
+  }
+  Symbol &MakeAssocSymbol(const SourceName &name, Symbol &prev) {
+    return MakeAssocSymbol(name, prev, currScope());
+  }
+  static const parser::Name *GetDesignatorNameIfDataRef(
+      const parser::Designator &designator) {
+    const auto *dataRef{std::get_if<parser::DataRef>(&designator.u)};
+    return dataRef ? std::get_if<parser::Name>(&dataRef->u) : nullptr;
+  }
+  void AddDataSharingAttributeObject(SymbolRef object) {
+    dataSharingAttributeObjects_.insert(object);
+  }
+  void ClearDataSharingAttributeObjects() {
+    dataSharingAttributeObjects_.clear();
+  }
+  bool HasDataSharingAttributeObject(const Symbol &);
+  const parser::Name &GetLoopIndex(const parser::DoConstruct &);
+  const parser::DoConstruct *GetDoConstructIf(
+      const parser::ExecutionPartConstruct &);
+  Symbol *DeclarePrivateAccessEntity(
+      const parser::Name &, Symbol::Flag, Scope &);
+  Symbol *DeclarePrivateAccessEntity(Symbol &, Symbol::Flag, Scope &);
+  Symbol *DeclareOrMarkOtherAccessEntity(const parser::Name &, Symbol::Flag);
+
+  SymbolSet dataSharingAttributeObjects_; // on one directive
+  SemanticsContext &context_;
+  ResolveNamesVisitor &resolver_;
+  std::vector<DirContext> dirContext_; // used as a stack
+};
+
+template <typename T>
+bool DirectiveAttributeVisitor<T>::HasDataSharingAttributeObject(
+    const Symbol &object) {
+  auto it{dataSharingAttributeObjects_.find(object)};
+  return it != dataSharingAttributeObjects_.end();
+}
+
+template <typename T>
+const parser::Name &DirectiveAttributeVisitor<T>::GetLoopIndex(
+    const parser::DoConstruct &x) {
+  auto &loopControl{x.GetLoopControl().value()};
+  using Bounds = parser::LoopControl::Bounds;
+  const Bounds &bounds{std::get<Bounds>(loopControl.u)};
+  return bounds.name.thing;
+}
+
+template <typename T>
+const parser::DoConstruct *DirectiveAttributeVisitor<T>::GetDoConstructIf(
+    const parser::ExecutionPartConstruct &x) {
+  return parser::Unwrap<parser::DoConstruct>(x);
+}
+
+template <typename T>
+Symbol *DirectiveAttributeVisitor<T>::DeclarePrivateAccessEntity(
+    const parser::Name &name, Symbol::Flag flag, Scope &scope) {
+  if (!name.symbol) {
+    return nullptr; // not resolved by Name Resolution step, do nothing
+  }
+  name.symbol = DeclarePrivateAccessEntity(*name.symbol, flag, scope);
+  return name.symbol;
+}
+
+template <typename T>
+Symbol *DirectiveAttributeVisitor<T>::DeclarePrivateAccessEntity(
+    Symbol &object, Symbol::Flag flag, Scope &scope) {
+  if (object.owner() != currScope()) {
+    auto &symbol{MakeAssocSymbol(object.name(), object, scope)};
+    symbol.set(flag);
+    return &symbol;
+  } else {
+    object.set(flag);
+    return &object;
+  }
+}
+
+// Create scopes for OpenACC constructs
+class AccVisitor : public virtual DeclarationVisitor {
+public:
+  void AddAccSourceRange(const parser::CharBlock &);
+
+  static bool NeedsScope(const parser::OpenACCBlockConstruct &);
+
+  bool Pre(const parser::OpenACCBlockConstruct &);
+  void Post(const parser::OpenACCBlockConstruct &);
+  bool Pre(const parser::AccBeginBlockDirective &x) {
+    AddAccSourceRange(x.source);
+    return true;
+  }
+  void Post(const parser::AccBeginBlockDirective &) {
+    messageHandler().set_currStmtSource(std::nullopt);
+  }
+  bool Pre(const parser::AccEndBlockDirective &x) {
+    AddAccSourceRange(x.source);
+    return true;
+  }
+  void Post(const parser::AccEndBlockDirective &) {
+    messageHandler().set_currStmtSource(std::nullopt);
+  }
+  bool Pre(const parser::AccBeginLoopDirective &x) {
+    AddAccSourceRange(x.source);
+    return true;
+  }
+  void Post(const parser::AccBeginLoopDirective &x) {
+    messageHandler().set_currStmtSource(std::nullopt);
+  }
+};
+
+bool AccVisitor::NeedsScope(const parser::OpenACCBlockConstruct &x) {
+  const auto &beginBlockDir{std::get<parser::AccBeginBlockDirective>(x.t)};
+  const auto &beginDir{std::get<parser::AccBlockDirective>(beginBlockDir.t)};
+  switch (beginDir.v) {
+  case llvm::acc::Directive::ACCD_data:
+  case llvm::acc::Directive::ACCD_host_data:
+  case llvm::acc::Directive::ACCD_kernels:
+  case llvm::acc::Directive::ACCD_parallel:
+  case llvm::acc::Directive::ACCD_serial:
+    return true;
+  default:
+    return false;
+  }
+}
+
+void AccVisitor::AddAccSourceRange(const parser::CharBlock &source) {
+  messageHandler().set_currStmtSource(source);
+  currScope().AddSourceRange(source);
+}
+
+bool AccVisitor::Pre(const parser::OpenACCBlockConstruct &x) {
+  if (NeedsScope(x)) {
+    PushScope(Scope::Kind::Block, nullptr);
+  }
+  return true;
+}
+
+void AccVisitor::Post(const parser::OpenACCBlockConstruct &x) {
+  if (NeedsScope(x)) {
+    PopScope();
+  }
+}
+
+class AccAttributeVisitor : DirectiveAttributeVisitor<llvm::acc::Directive> {
+public:
+  explicit AccAttributeVisitor(
+      SemanticsContext &context, ResolveNamesVisitor &resolver)
+      : DirectiveAttributeVisitor(context, resolver) {}
+
+  template <typename A> void Walk(const A &x) { parser::Walk(x, *this); }
+  template <typename A> bool Pre(const A &) { return true; }
+  template <typename A> void Post(const A &) {}
+
+  bool Pre(const parser::SpecificationPart &x) {
+    Walk(std::get<std::list<parser::OpenACCDeclarativeConstruct>>(x.t));
+    return false;
+  }
+
+  bool Pre(const parser::OpenACCBlockConstruct &);
+  void Post(const parser::OpenACCBlockConstruct &) { PopContext(); }
+  bool Pre(const parser::OpenACCCombinedConstruct &);
+  void Post(const parser::OpenACCCombinedConstruct &) { PopContext(); }
+
+  void Post(const parser::AccBeginBlockDirective &) {
+    GetContext().withinConstruct = true;
+  }
+
+  bool Pre(const parser::OpenACCLoopConstruct &);
+  void Post(const parser::OpenACCLoopConstruct &) { PopContext(); }
+  void Post(const parser::AccLoopDirective &) {
+    GetContext().withinConstruct = true;
+  }
+
+  bool Pre(const parser::OpenACCStandaloneConstruct &);
+  void Post(const parser::OpenACCStandaloneConstruct &) { PopContext(); }
+  void Post(const parser::AccStandaloneDirective &) {
+    GetContext().withinConstruct = true;
+  }
+
+  void Post(const parser::AccDefaultClause &);
+
+  bool Pre(const parser::AccClause::Copy &x) {
+    ResolveAccObjectList(x.v, Symbol::Flag::AccCopyIn);
+    ResolveAccObjectList(x.v, Symbol::Flag::AccCopyOut);
+    return false;
+  }
+
+  bool Pre(const parser::AccClause::Create &x) {
+    const auto &objectList{std::get<parser::AccObjectList>(x.v.t)};
+    ResolveAccObjectList(objectList, Symbol::Flag::AccCreate);
+    return false;
+  }
+
+  bool Pre(const parser::AccClause::Copyin &x) {
+    const auto &objectList{std::get<parser::AccObjectList>(x.v.t)};
+    ResolveAccObjectList(objectList, Symbol::Flag::AccCopyIn);
+    return false;
+  }
+
+  bool Pre(const parser::AccClause::Copyout &x) {
+    const auto &objectList{std::get<parser::AccObjectList>(x.v.t)};
+    ResolveAccObjectList(objectList, Symbol::Flag::AccCopyOut);
+    return false;
+  }
+
+  bool Pre(const parser::AccClause::Present &x) {
+    ResolveAccObjectList(x.v, Symbol::Flag::AccPresent);
+    return false;
+  }
+  bool Pre(const parser::AccClause::Private &x) {
+    ResolveAccObjectList(x.v, Symbol::Flag::AccPrivate);
+    return false;
+  }
+  bool Pre(const parser::AccClause::FirstPrivate &x) {
+    ResolveAccObjectList(x.v, Symbol::Flag::AccFirstPrivate);
+    return false;
+  }
+
+  void Post(const parser::Name &);
+
+private:
+  int64_t GetAssociatedLoopLevelFromClauses(const parser::AccClauseList &);
+
+  static constexpr Symbol::Flags dataSharingAttributeFlags{
+      Symbol::Flag::AccShared, Symbol::Flag::AccPrivate,
+      Symbol::Flag::AccPresent, Symbol::Flag::AccFirstPrivate,
+      Symbol::Flag::AccReduction};
+
+  static constexpr Symbol::Flags dataMappingAttributeFlags{
+      Symbol::Flag::AccCreate, Symbol::Flag::AccCopyIn,
+      Symbol::Flag::AccCopyOut, Symbol::Flag::AccDelete};
+
+  static constexpr Symbol::Flags accFlagsRequireNewSymbol{
+      Symbol::Flag::AccPrivate, Symbol::Flag::AccFirstPrivate,
+      Symbol::Flag::AccReduction};
+
+  static constexpr Symbol::Flags accFlagsRequireMark{};
+
+  void PrivatizeAssociatedLoopIndex(const parser::OpenACCLoopConstruct &);
+  void ResolveAccObjectList(const parser::AccObjectList &, Symbol::Flag);
+  void ResolveAccObject(const parser::AccObject &, Symbol::Flag);
+  Symbol *ResolveAcc(const parser::Name &, Symbol::Flag, Scope &);
+  Symbol *ResolveAcc(Symbol &, Symbol::Flag, Scope &);
+  Symbol *ResolveAccCommonBlockName(const parser::Name *);
+  Symbol *DeclareOrMarkOtherAccessEntity(const parser::Name &, Symbol::Flag);
+  Symbol *DeclareOrMarkOtherAccessEntity(Symbol &, Symbol::Flag);
+  void CheckMultipleAppearances(
+      const parser::Name &, const Symbol &, Symbol::Flag);
+};
+
 // Create scopes for OpenMP constructs
 class OmpVisitor : public virtual DeclarationVisitor {
 public:
@@ -1178,11 +1478,11 @@ void OmpVisitor::Post(const parser::OpenMPBlockConstruct &x) {
 }
 
 // Data-sharing and Data-mapping attributes for data-refs in OpenMP construct
-class OmpAttributeVisitor {
+class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
 public:
   explicit OmpAttributeVisitor(
       SemanticsContext &context, ResolveNamesVisitor &resolver)
-      : context_{context}, resolver_{resolver} {}
+      : DirectiveAttributeVisitor(context, resolver) {}
 
   template <typename A> void Walk(const A &x) { parser::Walk(x, *this); }
 
@@ -1235,70 +1535,8 @@ class OmpAttributeVisitor {
   void Post(const parser::Name &);
 
 private:
-  struct OmpContext {
-    OmpContext(
-        const parser::CharBlock &source, llvm::omp::Directive d, Scope &s)
-        : directiveSource{source}, directive{d}, scope{s} {}
-    parser::CharBlock directiveSource;
-    llvm::omp::Directive directive;
-    Scope &scope;
-    // TODO: default DSA is implicitly determined in different ways
-    Symbol::Flag defaultDSA{Symbol::Flag::OmpShared};
-    // variables on Data-sharing attribute clauses
-    std::map<const Symbol *, Symbol::Flag> objectWithDSA;
-    bool withinConstruct{false};
-    std::int64_t associatedLoopLevel{0};
-  };
-  // back() is the top of the stack
-  OmpContext &GetContext() {
-    CHECK(!ompContext_.empty());
-    return ompContext_.back();
-  }
-  void PushContext(const parser::CharBlock &source, llvm::omp::Directive dir) {
-    ompContext_.emplace_back(source, dir, context_.FindScope(source));
-  }
-  void PopContext() { ompContext_.pop_back(); }
-  void SetContextDirectiveSource(parser::CharBlock &dir) {
-    GetContext().directiveSource = dir;
-  }
-  void SetContextDirectiveEnum(llvm::omp::Directive dir) {
-    GetContext().directive = dir;
-  }
-  Scope &currScope() { return GetContext().scope; }
-  void SetContextDefaultDSA(Symbol::Flag flag) {
-    GetContext().defaultDSA = flag;
-  }
-  void AddToContextObjectWithDSA(
-      const Symbol &symbol, Symbol::Flag flag, OmpContext &context) {
-    context.objectWithDSA.emplace(&symbol, flag);
-  }
-  void AddToContextObjectWithDSA(const Symbol &symbol, Symbol::Flag flag) {
-    AddToContextObjectWithDSA(symbol, flag, GetContext());
-  }
-  bool IsObjectWithDSA(const Symbol &symbol) {
-    auto it{GetContext().objectWithDSA.find(&symbol)};
-    return it != GetContext().objectWithDSA.end();
-  }
-
-  void SetContextAssociatedLoopLevel(std::int64_t level) {
-    GetContext().associatedLoopLevel = level;
-  }
   std::int64_t GetAssociatedLoopLevelFromClauses(const parser::OmpClauseList &);
 
-  Symbol &MakeAssocSymbol(const SourceName &name, Symbol &prev, Scope &scope) {
-    const auto pair{scope.try_emplace(name, Attrs{}, HostAssocDetails{prev})};
-    return *pair.first->second;
-  }
-  Symbol &MakeAssocSymbol(const SourceName &name, Symbol &prev) {
-    return MakeAssocSymbol(name, prev, currScope());
-  }
-
-  static const parser::Name *GetDesignatorNameIfDataRef(
-      const parser::Designator &designator) {
-    const auto *dataRef{std::get_if<parser::DataRef>(&designator.u)};
-    return dataRef ? std::get_if<parser::Name>(&dataRef->u) : nullptr;
-  }
-
   static constexpr Symbol::Flags dataSharingAttributeFlags{
       Symbol::Flag::OmpShared, Symbol::Flag::OmpPrivate,
       Symbol::Flag::OmpFirstPrivate, Symbol::Flag::OmpLastPrivate,
@@ -1312,19 +1550,8 @@ class OmpAttributeVisitor {
   static constexpr Symbol::Flags ompFlagsRequireMark{
       Symbol::Flag::OmpThreadprivate};
 
-  void AddDataSharingAttributeObject(SymbolRef object) {
-    dataSharingAttributeObjects_.insert(object);
-  }
-  void ClearDataSharingAttributeObjects() {
-    dataSharingAttributeObjects_.clear();
-  }
-  bool HasDataSharingAttributeObject(const Symbol &);
-
-  const parser::DoConstruct *GetDoConstructIf(
-      const parser::ExecutionPartConstruct &);
   // Predetermined DSA rules
   void PrivatizeAssociatedLoopIndex(const parser::OpenMPLoopConstruct &);
-  const parser::Name &GetLoopIndex(const parser::DoConstruct &);
   void ResolveSeqLoopIndexInParallelOrTaskConstruct(const parser::Name &);
 
   void ResolveOmpObjectList(const parser::OmpObjectList &, Symbol::Flag);
@@ -1332,18 +1559,10 @@ class OmpAttributeVisitor {
   Symbol *ResolveOmp(const parser::Name &, Symbol::Flag, Scope &);
   Symbol *ResolveOmp(Symbol &, Symbol::Flag, Scope &);
   Symbol *ResolveOmpCommonBlockName(const parser::Name *);
-  Symbol *DeclarePrivateAccessEntity(
-      const parser::Name &, Symbol::Flag, Scope &);
-  Symbol *DeclarePrivateAccessEntity(Symbol &, Symbol::Flag, Scope &);
   Symbol *DeclareOrMarkOtherAccessEntity(const parser::Name &, Symbol::Flag);
   Symbol *DeclareOrMarkOtherAccessEntity(Symbol &, Symbol::Flag);
   void CheckMultipleAppearances(
       const parser::Name &, const Symbol &, Symbol::Flag);
-  SymbolSet dataSharingAttributeObjects_; // on one directive
-
-  SemanticsContext &context_;
-  ResolveNamesVisitor &resolver_;
-  std::vector<OmpContext> ompContext_; // used as a stack
 };
 
 // Walk the parse tree and resolve names to symbols.
@@ -1351,8 +1570,11 @@ class ResolveNamesVisitor : public virtual ScopeHandler,
                             public ModuleVisitor,
                             public SubprogramVisitor,
                             public ConstructVisitor,
-                            public OmpVisitor {
+                            public OmpVisitor,
+                            public AccVisitor {
 public:
+  using AccVisitor::Post;
+  using AccVisitor::Pre;
   using ArraySpecVisitor::Post;
   using ConstructVisitor::Post;
   using ConstructVisitor::Pre;
@@ -1450,6 +1672,7 @@ class ResolveNamesVisitor : public virtual ScopeHandler,
   void FinishSpecificationParts(const ProgramTree &);
   void FinishDerivedTypeInstantiation(Scope &);
   void ResolveExecutionParts(const ProgramTree &);
+  void ResolveAccParts(const parser::ProgramUnit &);
   void ResolveOmpParts(const parser::ProgramUnit &);
 };
 
@@ -6275,7 +6498,12 @@ bool ResolveNamesVisitor::Pre(const parser::ProgramUnit &x) {
   inExecutionPart_ = true;
   ResolveExecutionParts(root);
   inExecutionPart_ = false;
-  ResolveOmpParts(x);
+  if (context().IsEnabled(common::LanguageFeature::OpenACC)) {
+    ResolveAccParts(x);
+  }
+  if (context().IsEnabled(common::LanguageFeature::OpenMP)) {
+    ResolveOmpParts(x);
+  }
   return false;
 }
 
@@ -6468,6 +6696,287 @@ class DeferredCheckVisitor {
   bool pushedScope_{false};
 };
 
+bool AccAttributeVisitor::Pre(const parser::OpenACCBlockConstruct &x) {
+  const auto &beginBlockDir{std::get<parser::AccBeginBlockDirective>(x.t)};
+  const auto &blockDir{std::get<parser::AccBlockDirective>(beginBlockDir.t)};
+  switch (blockDir.v) {
+  case llvm::acc::Directive::ACCD_data:
+  case llvm::acc::Directive::ACCD_host_data:
+  case llvm::acc::Directive::ACCD_kernels:
+  case llvm::acc::Directive::ACCD_parallel:
+  case llvm::acc::Directive::ACCD_serial:
+    PushContext(blockDir.source, blockDir.v);
+    break;
+  default:
+    break;
+  }
+  ClearDataSharingAttributeObjects();
+  return true;
+}
+
+bool AccAttributeVisitor::Pre(const parser::OpenACCLoopConstruct &x) {
+  const auto &beginDir{std::get<parser::AccBeginLoopDirective>(x.t)};
+  const auto &loopDir{std::get<parser::AccLoopDirective>(beginDir.t)};
+  const auto &clauseList{std::get<parser::AccClauseList>(beginDir.t)};
+  if (loopDir.v == llvm::acc::Directive::ACCD_loop) {
+    PushContext(loopDir.source, loopDir.v);
+  }
+  ClearDataSharingAttributeObjects();
+  SetContextAssociatedLoopLevel(GetAssociatedLoopLevelFromClauses(clauseList));
+  PrivatizeAssociatedLoopIndex(x);
+  return true;
+}
+
+bool AccAttributeVisitor::Pre(const parser::OpenACCStandaloneConstruct &x) {
+  const auto &standaloneDir{std::get<parser::AccStandaloneDirective>(x.t)};
+  switch (standaloneDir.v) {
+  case llvm::acc::Directive::ACCD_cache:
+  case llvm::acc::Directive::ACCD_enter_data:
+  case llvm::acc::Directive::ACCD_exit_data:
+  case llvm::acc::Directive::ACCD_init:
+  case llvm::acc::Directive::ACCD_set:
+  case llvm::acc::Directive::ACCD_shutdown:
+  case llvm::acc::Directive::ACCD_update:
+    PushContext(standaloneDir.source, standaloneDir.v);
+    break;
+  default:
+    break;
+  }
+  ClearDataSharingAttributeObjects();
+  return true;
+}
+
+bool AccAttributeVisitor::Pre(const parser::OpenACCCombinedConstruct &x) {
+  const auto &beginBlockDir{std::get<parser::AccBeginCombinedDirective>(x.t)};
+  const auto &combinedDir{
+      std::get<parser::AccCombinedDirective>(beginBlockDir.t)};
+  switch (combinedDir.v) {
+  case llvm::acc::Directive::ACCD_kernels_loop:
+  case llvm::acc::Directive::ACCD_parallel_loop:
+  case llvm::acc::Directive::ACCD_serial_loop:
+    PushContext(combinedDir.source, combinedDir.v);
+    break;
+  default:
+    break;
+  }
+  ClearDataSharingAttributeObjects();
+  return true;
+}
+
+int64_t AccAttributeVisitor::GetAssociatedLoopLevelFromClauses(
+    const parser::AccClauseList &x) {
+  int64_t collapseLevel{0};
+  for (const auto &clause : x.v) {
+    if (const auto *collapseClause{
+            std::get_if<parser::AccClause::Collapse>(&clause.u)}) {
+      if (const auto v{evaluate::ToInt64(
+              resolver_.EvaluateIntExpr(collapseClause->v))}) {
+        collapseLevel = *v;
+      }
+    }
+  }
+
+  if (collapseLevel) {
+    return collapseLevel;
+  }
+  return 1; // default is outermost loop
+}
+
+void AccAttributeVisitor::PrivatizeAssociatedLoopIndex(
+    const parser::OpenACCLoopConstruct &x) {
+  int64_t level{GetContext().associatedLoopLevel};
+  if (level <= 0) { // collpase value was negative or 0
+    return;
+  }
+  Symbol::Flag ivDSA{Symbol::Flag::AccPrivate};
+
+  const auto &outer{std::get<std::optional<parser::DoConstruct>>(x.t)};
+  for (const parser::DoConstruct *loop{&*outer}; loop && level > 0; --level) {
+    // go through all the nested do-loops and resolve index variables
+    const parser::Name &iv{GetLoopIndex(*loop)};
+    if (auto *symbol{ResolveAcc(iv, ivDSA, currScope())}) {
+      symbol->set(Symbol::Flag::AccPreDetermined);
+      iv.symbol = symbol; // adjust the symbol within region
+      AddToContextObjectWithDSA(*symbol, ivDSA);
+    }
+
+    const auto &block{std::get<parser::Block>(loop->t)};
+    const auto it{block.begin()};
+    loop = it != block.end() ? GetDoConstructIf(*it) : nullptr;
+  }
+  CHECK(level == 0);
+}
+
+void AccAttributeVisitor::Post(const parser::AccDefaultClause &x) {
+  if (!dirContext_.empty()) {
+    switch (x.v) {
+    case parser::AccDefaultClause::Arg::Present:
+      SetContextDefaultDSA(Symbol::Flag::AccPresent);
+      break;
+    case parser::AccDefaultClause::Arg::None:
+      SetContextDefaultDSA(Symbol::Flag::AccNone);
+      break;
+    }
+  }
+}
+
+// For OpenACC constructs, check all the data-refs within the constructs
+// and adjust the symbol for each Name if necessary
+void AccAttributeVisitor::Post(const parser::Name &name) {
+  auto *symbol{name.symbol};
+  if (symbol && !dirContext_.empty() && GetContext().withinConstruct) {
+    if (!symbol->owner().IsDerivedType() && !symbol->has<ProcEntityDetails>() &&
+        !IsObjectWithDSA(*symbol)) {
+      if (Symbol * found{currScope().FindSymbol(name.source)}) {
+        if (symbol != found) {
+          name.symbol = found; // adjust the symbol within region
+        } else if (GetContext().defaultDSA == Symbol::Flag::AccNone) {
+          // 2.5.14.
+          context_.Say(name.source,
+              "The DEFAULT(NONE) clause requires that '%s' must be listed in "
+              "a data-mapping clause"_err_en_US,
+              symbol->name());
+        }
+      }
+    }
+  } // within OpenACC construct
+}
+
+Symbol *AccAttributeVisitor::ResolveAccCommonBlockName(
+    const parser::Name *name) {
+  if (!name) {
+    return nullptr;
+  } else if (auto *prev{
+                 GetContext().scope.parent().FindCommonBlock(name->source)}) {
+    name->symbol = prev;
+    return prev;
+  } else {
+    return nullptr;
+  }
+}
+
+void AccAttributeVisitor::ResolveAccObjectList(
+    const parser::AccObjectList &accObjectList, Symbol::Flag accFlag) {
+  for (const auto &accObject : accObjectList.v) {
+    ResolveAccObject(accObject, accFlag);
+  }
+}
+
+void AccAttributeVisitor::ResolveAccObject(
+    const parser::AccObject &accObject, Symbol::Flag accFlag) {
+  std::visit(
+      common::visitors{
+          [&](const parser::Designator &designator) {
+            if (const auto *name{GetDesignatorNameIfDataRef(designator)}) {
+              if (auto *symbol{ResolveAcc(*name, accFlag, currScope())}) {
+                AddToContextObjectWithDSA(*symbol, accFlag);
+                if (dataSharingAttributeFlags.test(accFlag)) {
+                  CheckMultipleAppearances(*name, *symbol, accFlag);
+                }
+              }
+            } else if (const auto *designatorName{
+                           resolver_.ResolveDesignator(designator)};
+                       designatorName->symbol) {
+              // Array sections to be changed to substrings as needed
+              if (AnalyzeExpr(context_, designator)) {
+                if (std::holds_alternative<parser::Substring>(designator.u)) {
+                  context_.Say(designator.source,
+                      "Substrings are not allowed on OpenACC "
+                      "directives or clauses"_err_en_US);
+                }
+              }
+              // other checks, more TBD
+              if (const auto *details{designatorName->symbol
+                                          ->detailsIf<ObjectEntityDetails>()}) {
+                if (details->IsArray()) {
+                  // TODO: check Array Sections
+                } else if (designatorName->symbol->owner().IsDerivedType()) {
+                  // TODO: check Structure Component
+                }
+              }
+            }
+          },
+          [&](const parser::Name &name) { // common block
+            if (auto *symbol{ResolveAccCommonBlockName(&name)}) {
+              CheckMultipleAppearances(
+                  name, *symbol, Symbol::Flag::AccCommonBlock);
+              for (auto &object : symbol->get<CommonBlockDetails>().objects()) {
+                if (auto *resolvedObject{
+                        ResolveAcc(*object, accFlag, currScope())}) {
+                  AddToContextObjectWithDSA(*resolvedObject, accFlag);
+                }
+              }
+            } else {
+              context_.Say(name.source,
+                  "COMMON block must be declared in the same scoping unit "
+                  "in which the OpenACC directive or clause appears"_err_en_US);
+            }
+          },
+      },
+      accObject.u);
+}
+
+Symbol *AccAttributeVisitor::ResolveAcc(
+    const parser::Name &name, Symbol::Flag accFlag, Scope &scope) {
+  if (accFlagsRequireNewSymbol.test(accFlag)) {
+    return DeclarePrivateAccessEntity(name, accFlag, scope);
+  } else {
+    return DeclareOrMarkOtherAccessEntity(name, accFlag);
+  }
+}
+
+Symbol *AccAttributeVisitor::ResolveAcc(
+    Symbol &symbol, Symbol::Flag accFlag, Scope &scope) {
+  if (accFlagsRequireNewSymbol.test(accFlag)) {
+    return DeclarePrivateAccessEntity(symbol, accFlag, scope);
+  } else {
+    return DeclareOrMarkOtherAccessEntity(symbol, accFlag);
+  }
+}
+
+Symbol *AccAttributeVisitor::DeclareOrMarkOtherAccessEntity(
+    const parser::Name &name, Symbol::Flag accFlag) {
+  Symbol *prev{currScope().FindSymbol(name.source)};
+  if (!name.symbol || !prev) {
+    return nullptr;
+  } else if (prev != name.symbol) {
+    name.symbol = prev;
+  }
+  return DeclareOrMarkOtherAccessEntity(*prev, accFlag);
+}
+
+Symbol *AccAttributeVisitor::DeclareOrMarkOtherAccessEntity(
+    Symbol &object, Symbol::Flag accFlag) {
+  if (accFlagsRequireMark.test(accFlag)) {
+    object.set(accFlag);
+  }
+  return &object;
+}
+
+static bool WithMultipleAppearancesAccException(
+    const Symbol &symbol, Symbol::Flag flag) {
+  return false; // Place holder
+}
+
+void AccAttributeVisitor::CheckMultipleAppearances(
+    const parser::Name &name, const Symbol &symbol, Symbol::Flag accFlag) {
+  const auto *target{&symbol};
+  if (accFlagsRequireNewSymbol.test(accFlag)) {
+    if (const auto *details{symbol.detailsIf<HostAssocDetails>()}) {
+      target = &details->symbol();
+    }
+  }
+  if (HasDataSharingAttributeObject(*target) &&
+      !WithMultipleAppearancesAccException(symbol, accFlag)) {
+    context_.Say(name.source,
+        "'%s' appears in more than one data-sharing clause "
+        "on the same OpenACC directive"_err_en_US,
+        name.ToString());
+  } else {
+    AddDataSharingAttributeObject(*target);
+  }
+}
+
 bool OmpAttributeVisitor::Pre(const parser::OpenMPBlockConstruct &x) {
   const auto &beginBlockDir{std::get<parser::OmpBeginBlockDirective>(x.t)};
   const auto &beginDir{std::get<parser::OmpBlockDirective>(beginBlockDir.t)};
@@ -6532,19 +7041,11 @@ bool OmpAttributeVisitor::Pre(const parser::OpenMPLoopConstruct &x) {
   return true;
 }
 
-const parser::Name &OmpAttributeVisitor::GetLoopIndex(
-    const parser::DoConstruct &x) {
-  auto &loopControl{x.GetLoopControl().value()};
-  using Bounds = parser::LoopControl::Bounds;
-  const Bounds &bounds{std::get<Bounds>(loopControl.u)};
-  return bounds.name.thing;
-}
-
 void OmpAttributeVisitor::ResolveSeqLoopIndexInParallelOrTaskConstruct(
     const parser::Name &iv) {
-  auto targetIt{ompContext_.rbegin()};
+  auto targetIt{dirContext_.rbegin()};
   for (;; ++targetIt) {
-    if (targetIt == ompContext_.rend()) {
+    if (targetIt == dirContext_.rend()) {
       return;
     }
     if (llvm::omp::parallelSet.test(targetIt->directive) ||
@@ -6556,7 +7057,7 @@ void OmpAttributeVisitor::ResolveSeqLoopIndexInParallelOrTaskConstruct(
     targetIt++;
     symbol->set(Symbol::Flag::OmpPreDetermined);
     iv.symbol = symbol; // adjust the symbol within region
-    for (auto it{ompContext_.rbegin()}; it != targetIt; ++it) {
+    for (auto it{dirContext_.rbegin()}; it != targetIt; ++it) {
       AddToContextObjectWithDSA(*symbol, Symbol::Flag::OmpPrivate, *it);
     }
   }
@@ -6567,7 +7068,7 @@ void OmpAttributeVisitor::ResolveSeqLoopIndexInParallelOrTaskConstruct(
 //     or task generating construct is private in the innermost such
 //     construct that encloses the loop
 bool OmpAttributeVisitor::Pre(const parser::DoConstruct &x) {
-  if (!ompContext_.empty() && GetContext().withinConstruct) {
+  if (!dirContext_.empty() && GetContext().withinConstruct) {
     if (const auto &iv{GetLoopIndex(x)}; iv.symbol) {
       if (!iv.symbol->test(Symbol::Flag::OmpPreDetermined)) {
         ResolveSeqLoopIndexInParallelOrTaskConstruct(iv);
@@ -6579,16 +7080,6 @@ bool OmpAttributeVisitor::Pre(const parser::DoConstruct &x) {
   return true;
 }
 
-const parser::DoConstruct *OmpAttributeVisitor::GetDoConstructIf(
-    const parser::ExecutionPartConstruct &x) {
-  if (auto *y{std::get_if<parser::ExecutableConstruct>(&x.u)}) {
-    if (auto *z{std::get_if<Indirection<parser::DoConstruct>>(&y->u)}) {
-      return &z->value();
-    }
-  }
-  return nullptr;
-}
-
 std::int64_t OmpAttributeVisitor::GetAssociatedLoopLevelFromClauses(
     const parser::OmpClauseList &x) {
   std::int64_t orderedLevel{0};
@@ -6685,7 +7176,7 @@ bool OmpAttributeVisitor::Pre(const parser::OpenMPThreadprivate &x) {
 }
 
 void OmpAttributeVisitor::Post(const parser::OmpDefaultClause &x) {
-  if (!ompContext_.empty()) {
+  if (!dirContext_.empty()) {
     switch (x.v) {
     case parser::OmpDefaultClause::Type::Private:
       SetContextDefaultDSA(Symbol::Flag::OmpPrivate);
@@ -6707,7 +7198,7 @@ void OmpAttributeVisitor::Post(const parser::OmpDefaultClause &x) {
 // and adjust the symbol for each Name if necessary
 void OmpAttributeVisitor::Post(const parser::Name &name) {
   auto *symbol{name.symbol};
-  if (symbol && !ompContext_.empty() && GetContext().withinConstruct) {
+  if (symbol && !dirContext_.empty() && GetContext().withinConstruct) {
     if (!symbol->owner().IsDerivedType() && !symbol->has<ProcEntityDetails>() &&
         !IsObjectWithDSA(*symbol)) {
       // TODO: create a separate function to go through the rules for
@@ -6727,11 +7218,6 @@ void OmpAttributeVisitor::Post(const parser::Name &name) {
   } // within OpenMP construct
 }
 
-bool OmpAttributeVisitor::HasDataSharingAttributeObject(const Symbol &object) {
-  auto it{dataSharingAttributeObjects_.find(object)};
-  return it != dataSharingAttributeObjects_.end();
-}
-
 Symbol *OmpAttributeVisitor::ResolveOmpCommonBlockName(
     const parser::Name *name) {
   if (auto *prev{name
@@ -6826,27 +7312,6 @@ Symbol *OmpAttributeVisitor::ResolveOmp(
   }
 }
 
-Symbol *OmpAttributeVisitor::DeclarePrivateAccessEntity(
-    const parser::Name &name, Symbol::Flag ompFlag, Scope &scope) {
-  if (!name.symbol) {
-    return nullptr; // not resolved by Name Resolution step, do nothing
-  }
-  name.symbol = DeclarePrivateAccessEntity(*name.symbol, ompFlag, scope);
-  return name.symbol;
-}
-
-Symbol *OmpAttributeVisitor::DeclarePrivateAccessEntity(
-    Symbol &object, Symbol::Flag ompFlag, Scope &scope) {
-  if (object.owner() != currScope()) {
-    auto &symbol{MakeAssocSymbol(object.name(), object, scope)};
-    symbol.set(ompFlag);
-    return &symbol;
-  } else {
-    object.set(ompFlag);
-    return &object;
-  }
-}
-
 Symbol *OmpAttributeVisitor::DeclareOrMarkOtherAccessEntity(
     const parser::Name &name, Symbol::Flag ompFlag) {
   Symbol *prev{currScope().FindSymbol(name.source)};
@@ -6866,11 +7331,11 @@ Symbol *OmpAttributeVisitor::DeclareOrMarkOtherAccessEntity(
   return &object;
 }
 
-static bool WithMultipleAppearancesException(
-    const Symbol &symbol, Symbol::Flag ompFlag) {
-  return (ompFlag == Symbol::Flag::OmpFirstPrivate &&
+static bool WithMultipleAppearancesOmpException(
+    const Symbol &symbol, Symbol::Flag flag) {
+  return (flag == Symbol::Flag::OmpFirstPrivate &&
              symbol.test(Symbol::Flag::OmpLastPrivate)) ||
-      (ompFlag == Symbol::Flag::OmpLastPrivate &&
+      (flag == Symbol::Flag::OmpLastPrivate &&
           symbol.test(Symbol::Flag::OmpFirstPrivate));
 }
 
@@ -6883,7 +7348,7 @@ void OmpAttributeVisitor::CheckMultipleAppearances(
     }
   }
   if (HasDataSharingAttributeObject(*target) &&
-      !WithMultipleAppearancesException(symbol, ompFlag)) {
+      !WithMultipleAppearancesOmpException(symbol, ompFlag)) {
     context_.Say(name.source,
         "'%s' appears in more than one data-sharing clause "
         "on the same OpenMP directive"_err_en_US,
@@ -6962,6 +7427,10 @@ void ResolveNamesVisitor::ResolveExecutionParts(const ProgramTree &node) {
   }
 }
 
+void ResolveNamesVisitor::ResolveAccParts(const parser::ProgramUnit &node) {
+  AccAttributeVisitor{context(), *this}.Walk(node);
+}
+
 void ResolveNamesVisitor::ResolveOmpParts(const parser::ProgramUnit &node) {
   OmpAttributeVisitor{context(), *this}.Walk(node);
   if (!context().AnyFatalError()) {
diff --git a/flang/lib/Semantics/unparse-with-symbols.cpp b/flang/lib/Semantics/unparse-with-symbols.cpp
index 44ceb9fe08bb6..67016e85777c7 100644
--- a/flang/lib/Semantics/unparse-with-symbols.cpp
+++ b/flang/lib/Semantics/unparse-with-symbols.cpp
@@ -35,6 +35,11 @@ class SymbolDumpVisitor {
   template <typename T> void Post(const parser::Statement<T> &) {
     currStmt_ = std::nullopt;
   }
+  bool Pre(const parser::AccClause &clause) {
+    currStmt_ = clause.source;
+    return true;
+  }
+  void Post(const parser::AccClause &) { currStmt_ = std::nullopt; }
   bool Pre(const parser::OmpClause &clause) {
     currStmt_ = clause.source;
     return true;
diff --git a/flang/test/Semantics/acc-resolve01.f90 b/flang/test/Semantics/acc-resolve01.f90
new file mode 100644
index 0000000000000..7e904b525926b
--- /dev/null
+++ b/flang/test/Semantics/acc-resolve01.f90
@@ -0,0 +1,22 @@
+! RUN: %S/test_errors.sh %s %t %f18 -fopenacc
+
+! Data-Mapping Attribute Clauses
+! 2.15.14 default Clause
+
+subroutine default_none()
+  integer a(3)
+
+  A = 1
+  B = 2
+  !$acc parallel default(none) private(c)
+  !ERROR: The DEFAULT(NONE) clause requires that 'a' must be listed in a data-mapping clause
+  A(1:2) = 3
+  !ERROR: The DEFAULT(NONE) clause requires that 'b' must be listed in a data-mapping clause
+  B = 4
+  C = 5
+  !$acc end parallel
+end subroutine default_none
+
+program mm
+  call default_none()
+end
\ No newline at end of file
diff --git a/flang/test/Semantics/acc-resolve02.f90 b/flang/test/Semantics/acc-resolve02.f90
new file mode 100644
index 0000000000000..da1a417bba52b
--- /dev/null
+++ b/flang/test/Semantics/acc-resolve02.f90
@@ -0,0 +1,17 @@
+! RUN: %S/test_errors.sh %s %t %f18 -fopenacc
+
+subroutine compute()
+  integer :: a(3), c, i
+
+  a = 1
+  !ERROR: 'c' appears in more than one data-sharing clause on the same OpenACC directive
+  !$acc parallel firstprivate(c) private(c)
+  do i = 1, 3
+    a(i) = c
+  end do
+  !$acc end parallel
+end subroutine compute
+
+program mm
+  call compute()
+end
diff --git a/flang/test/Semantics/acc-symbols01.f90 b/flang/test/Semantics/acc-symbols01.f90
new file mode 100644
index 0000000000000..23d54eb93fbef
--- /dev/null
+++ b/flang/test/Semantics/acc-symbols01.f90
@@ -0,0 +1,26 @@
+! RUN: %S/test_symbols.sh %s %t %f18 -fopenacc
+
+!DEF: /mm MainProgram
+program mm
+  !DEF: /mm/x ObjectEntity REAL(4)
+  !DEF: /mm/y ObjectEntity REAL(4)
+  real x, y
+  !DEF: /mm/a ObjectEntity INTEGER(4)
+  !DEF: /mm/b ObjectEntity INTEGER(4)
+  !DEF: /mm/c ObjectEntity INTEGER(4)
+  !DEF: /mm/i ObjectEntity INTEGER(4)
+  integer a(10), b(10), c(10), i
+  !REF: /mm/b
+  b = 2
+ !$acc parallel present(c) firstprivate(b) private(a)
+ !$acc loop
+  !DEF: /mm/Block1/i (AccPrivate, AccPreDetermined) HostAssoc INTEGER(4)
+  do i=1,10
+   !DEF: /mm/Block1/a (AccPrivate) HostAssoc INTEGER(4)
+   !REF: /mm/Block1/i
+   !DEF: /mm/Block1/b (AccFirstPrivate) HostAssoc INTEGER(4)
+   a(i) = b(i)
+  end do
+ !$acc end parallel
+ end program
+
diff --git a/flang/test/Semantics/test_symbols.sh b/flang/test/Semantics/test_symbols.sh
index d2b3d688a39b8..61ff7fdb1e7bf 100755
--- a/flang/test/Semantics/test_symbols.sh
+++ b/flang/test/Semantics/test_symbols.sh
@@ -16,8 +16,9 @@ diffs=$temp/diffs
 
 # Strip out blank lines and all comments except "!DEF:", "!REF:", and "!$omp"
 sed -e 's/!\([DR]EF:\)/KEEP \1/' -e 's/!\($omp\)/KEEP \1/' \
-  -e 's/!.*//' -e 's/ *$//' -e '/^$/d' -e 's/KEEP \([DR]EF:\)/!\1/' \
-  -e 's/KEEP \($omp\)/!\1/' \
+  -e 's/!\($acc\)/KEEP \1/' -e 's/!.*//' -e 's/ *$//' -e '/^$/d' \
+  -e 's/KEEP \([DR]EF:\)/!\1/' -e 's/KEEP \($omp\)/!\1/' \
+  -e 's/KEEP \($acc\)/!\1/' \
   $src > $src1
 egrep -v '![DR]EF:' $src1 > $src2  # strip out DEF and REF comments
 # compile, inserting comments for symbols:

From 47a40eda178ef808090314e0664dbd3850db83eb Mon Sep 17 00:00:00 2001
From: Lang Hames <lhames@gmail.com>
Date: Sun, 26 Jul 2020 17:33:07 -0700
Subject: [PATCH 0152/1035] [ORC] Remove a redundant call to getTargetMemory.

---
 llvm/lib/ExecutionEngine/Orc/TPCIndirectionUtils.cpp | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/llvm/lib/ExecutionEngine/Orc/TPCIndirectionUtils.cpp b/llvm/lib/ExecutionEngine/Orc/TPCIndirectionUtils.cpp
index b4c21c32310ce..85dc4bec9a122 100644
--- a/llvm/lib/ExecutionEngine/Orc/TPCIndirectionUtils.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/TPCIndirectionUtils.cpp
@@ -297,15 +297,14 @@ TPCIndirectionUtils::writeResolverBlock(JITTargetAddress ReentryFnAddr,
     return Alloc.takeError();
 
   auto WorkingMemory = (*Alloc)->getWorkingMemory(ResolverBlockPermissions);
-  auto TargetAddress = (*Alloc)->getTargetMemory(ResolverBlockPermissions);
-  ABI->writeResolverCode(WorkingMemory.data(), TargetAddress, ReentryFnAddr,
+  ResolverBlockAddr = (*Alloc)->getTargetMemory(ResolverBlockPermissions);
+  ABI->writeResolverCode(WorkingMemory.data(), ResolverBlockAddr, ReentryFnAddr,
                          ReentryCtxAddr);
 
   if (auto Err = (*Alloc)->finalize())
     return std::move(Err);
 
   ResolverBlock = std::move(*Alloc);
-  ResolverBlockAddr = ResolverBlock->getTargetMemory(ResolverBlockPermissions);
   return ResolverBlockAddr;
 }
 

From 194a4beedd1e09ff0de3710ea431d8d3facd59e1 Mon Sep 17 00:00:00 2001
From: Juneyoung Lee <aqjune@gmail.com>
Date: Mon, 27 Jul 2020 09:43:00 +0900
Subject: [PATCH 0153/1035] [InstCombine] Add more tests to freeze-phi.ll; NFC

---
 .../test/Transforms/InstCombine/freeze-phi.ll | 47 ++++++++++++++++++-
 1 file changed, 46 insertions(+), 1 deletion(-)

diff --git a/llvm/test/Transforms/InstCombine/freeze-phi.ll b/llvm/test/Transforms/InstCombine/freeze-phi.ll
index fcda502a0d7f8..430c2d2e8fe66 100644
--- a/llvm/test/Transforms/InstCombine/freeze-phi.ll
+++ b/llvm/test/Transforms/InstCombine/freeze-phi.ll
@@ -20,7 +20,52 @@ B:
 C:
   %y = phi i32 [0, %A], [1, %B]
   %y.fr = freeze i32 %y
-  ret i32 %y
+  ret i32 %y.fr
+}
+
+define <2 x i32> @vec(i1 %cond) {
+; CHECK-LABEL: @vec(
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[A:%.*]], label [[B:%.*]]
+; CHECK:       A:
+; CHECK-NEXT:    br label [[C:%.*]]
+; CHECK:       B:
+; CHECK-NEXT:    br label [[C]]
+; CHECK:       C:
+; CHECK-NEXT:    [[Y:%.*]] = phi <2 x i32> [ <i32 0, i32 1>, [[A]] ], [ <i32 2, i32 3>, [[B]] ]
+; CHECK-NEXT:    ret <2 x i32> [[Y]]
+;
+  br i1 %cond, label %A, label %B
+A:
+  br label %C
+B:
+  br label %C
+C:
+  %y = phi <2 x i32> [<i32 0, i32 1>, %A], [<i32 2, i32 3>, %B]
+  %y.fr = freeze <2 x i32> %y
+  ret <2 x i32> %y.fr
+}
+
+define <2 x i32> @vec_undef(i1 %cond) {
+; CHECK-LABEL: @vec_undef(
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[A:%.*]], label [[B:%.*]]
+; CHECK:       A:
+; CHECK-NEXT:    br label [[C:%.*]]
+; CHECK:       B:
+; CHECK-NEXT:    br label [[C]]
+; CHECK:       C:
+; CHECK-NEXT:    [[Y:%.*]] = phi <2 x i32> [ <i32 0, i32 1>, [[A]] ], [ <i32 2, i32 undef>, [[B]] ]
+; CHECK-NEXT:    [[Y_FR:%.*]] = freeze <2 x i32> [[Y]]
+; CHECK-NEXT:    ret <2 x i32> [[Y_FR]]
+;
+  br i1 %cond, label %A, label %B
+A:
+  br label %C
+B:
+  br label %C
+C:
+  %y = phi <2 x i32> [<i32 0, i32 1>, %A], [<i32 2, i32 undef>, %B]
+  %y.fr = freeze <2 x i32> %y
+  ret <2 x i32> %y.fr
 }
 
 define i32 @one(i1 %cond, i32 %x) {

From a6e9f5264c855dfa5dda3c5940ffdaaf22d7e693 Mon Sep 17 00:00:00 2001
From: QingShan Zhang <qshanz@cn.ibm.com>
Date: Mon, 27 Jul 2020 02:02:40 +0000
Subject: [PATCH 0154/1035] [Scheduling] Improve group algorithm for store
 cluster

Store Addr and Store Addr+8 are clusterable pair. They have memory(ctrl) dependency on different loads.
Current implementation will put these two stores into different group and miss to cluster them.

Reviewed By: evandro

Differential Revision: https://reviews.llvm.org/D84139
---
 llvm/lib/CodeGen/MachineScheduler.cpp         |  8 +++-
 .../CodeGen/AArch64/aarch64-stp-cluster.ll    | 47 +++++++++++++++++++
 2 files changed, 54 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/CodeGen/MachineScheduler.cpp b/llvm/lib/CodeGen/MachineScheduler.cpp
index d9d0a783f8a22..cec7a0c031eb5 100644
--- a/llvm/lib/CodeGen/MachineScheduler.cpp
+++ b/llvm/lib/CodeGen/MachineScheduler.cpp
@@ -1653,7 +1653,13 @@ void BaseMemOpClusterMutation::apply(ScheduleDAGInstrs *DAG) {
 
     unsigned ChainPredID = DAG->SUnits.size();
     for (const SDep &Pred : SU.Preds) {
-      if (Pred.isCtrl() && !Pred.isArtificial()) {
+      // We only want to cluster the mem ops that have the same ctrl(non-data)
+      // pred so that they didn't have ctrl dependency for each other. But for
+      // store instrs, we can still cluster them if the pred is load instr.
+      if ((Pred.isCtrl() &&
+           (IsLoad ||
+            (Pred.getSUnit() && Pred.getSUnit()->getInstr()->mayStore()))) &&
+          !Pred.isArtificial()) {
         ChainPredID = Pred.getSUnit()->NodeNum;
         break;
       }
diff --git a/llvm/test/CodeGen/AArch64/aarch64-stp-cluster.ll b/llvm/test/CodeGen/AArch64/aarch64-stp-cluster.ll
index 5f75b4ef944bb..bfe7e4941da8b 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-stp-cluster.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-stp-cluster.ll
@@ -147,3 +147,50 @@ entry:
   ret i64 %v
 }
 
+; CHECK: ********** MI Scheduling **********
+; CHECK-LABEL: stp_i64_with_ld:%bb.0
+; CHECK:Cluster ld/st SU(5) - SU(10)
+; CHECK:Cluster ld/st SU(15) - SU(20)
+; CHECK:SU(5):   STRXui %7:gpr64, %0:gpr64common, 0 ::
+; CHECK:SU(10):   STRXui %12:gpr64, %0:gpr64common, 1 ::
+; CHECK:SU(15):   STRXui %17:gpr64, %0:gpr64common, 2 ::
+; CHECK:SU(20):   STRXui %22:gpr64, %0:gpr64common, 3 ::
+define void @stp_i64_with_ld(i64* noalias nocapture %a, i64* noalias nocapture readnone %b, i64* noalias nocapture readnone %c) {
+entry:
+  %arrayidx = getelementptr inbounds i64, i64* %a, i64 8
+  %0 = load i64, i64* %arrayidx, align 8
+  %arrayidx3 = getelementptr inbounds i64, i64* %a, i64 16
+  %1 = load i64, i64* %arrayidx3, align 8
+  %mul = mul nsw i64 %1, %0
+  %2 = load i64, i64* %a, align 8
+  %add6 = add nsw i64 %2, %mul
+  store i64 %add6, i64* %a, align 8
+  %arrayidx.1 = getelementptr inbounds i64, i64* %a, i64 9
+  %3 = load i64, i64* %arrayidx.1, align 8
+  %arrayidx3.1 = getelementptr inbounds i64, i64* %a, i64 17
+  %4 = load i64, i64* %arrayidx3.1, align 8
+  %mul.1 = mul nsw i64 %4, %3
+  %arrayidx5.1 = getelementptr inbounds i64, i64* %a, i64 1
+  %5 = load i64, i64* %arrayidx5.1, align 8
+  %add6.1 = add nsw i64 %5, %mul.1
+  store i64 %add6.1, i64* %arrayidx5.1, align 8
+  %arrayidx.2 = getelementptr inbounds i64, i64* %a, i64 10
+  %6 = load i64, i64* %arrayidx.2, align 8
+  %arrayidx3.2 = getelementptr inbounds i64, i64* %a, i64 18
+  %7 = load i64, i64* %arrayidx3.2, align 8
+  %mul.2 = mul nsw i64 %7, %6
+  %arrayidx5.2 = getelementptr inbounds i64, i64* %a, i64 2
+  %8 = load i64, i64* %arrayidx5.2, align 8
+  %add6.2 = add nsw i64 %8, %mul.2
+  store i64 %add6.2, i64* %arrayidx5.2, align 8
+  %arrayidx.3 = getelementptr inbounds i64, i64* %a, i64 11
+  %9 = load i64, i64* %arrayidx.3, align 8
+  %arrayidx3.3 = getelementptr inbounds i64, i64* %a, i64 19
+  %10 = load i64, i64* %arrayidx3.3, align 8
+  %mul.3 = mul nsw i64 %10, %9
+  %arrayidx5.3 = getelementptr inbounds i64, i64* %a, i64 3
+  %11 = load i64, i64* %arrayidx5.3, align 8
+  %add6.3 = add nsw i64 %11, %mul.3
+  store i64 %add6.3, i64* %arrayidx5.3, align 8
+  ret void
+}

From 0eff8b3865ede487bacd605f628891dd028c74bd Mon Sep 17 00:00:00 2001
From: biplmish <biplmish@in.ibm.com>
Date: Sun, 26 Jul 2020 21:23:00 -0500
Subject: [PATCH 0155/1035] [PowerPC] Cleanup p10vector clang test

Remove the duplicate LE test, correct the labels and remove common tests for vec_splat builtin.

Differential Revision: https://reviews.llvm.org/D84382
---
 clang/test/CodeGen/builtins-ppc-p10vector.c | 53 +++++++++------------
 1 file changed, 23 insertions(+), 30 deletions(-)

diff --git a/clang/test/CodeGen/builtins-ppc-p10vector.c b/clang/test/CodeGen/builtins-ppc-p10vector.c
index 2182a19f2452d..e67018b062141 100644
--- a/clang/test/CodeGen/builtins-ppc-p10vector.c
+++ b/clang/test/CodeGen/builtins-ppc-p10vector.c
@@ -1,15 +1,11 @@
 // REQUIRES: powerpc-registered-target
-// RUN: %clang_cc1 -target-feature +vsx -target-feature +altivec \
-// RUN:   -target-cpu pwr10 -triple powerpc64le-unknown-unknown -emit-llvm %s \
-// RUN:   -o - | FileCheck %s
-
-// RUN: %clang_cc1 -target-feature +vsx -target-feature +altivec \
+// RUN: %clang_cc1 -target-feature +vsx \
 // RUN:   -target-cpu pwr10 -triple powerpc64-unknown-unknown -emit-llvm %s \
-// RUN:   -o - | FileCheck %s -check-prefix=CHECK-BE
+// RUN:   -o - | FileCheck %s -check-prefixes=CHECK-BE,CHECK
 
-// RUN: %clang_cc1 -target-feature +vsx -target-feature +altivec \
+// RUN: %clang_cc1 -target-feature +vsx \
 // RUN:   -target-cpu pwr10 -triple powerpc64le-unknown-unknown -emit-llvm %s \
-// RUN:   -o - | FileCheck %s -check-prefix=CHECK-LE
+// RUN:   -o - | FileCheck %s -check-prefixes=CHECK-LE,CHECK
 
 #include <altivec.h>
 
@@ -514,19 +510,16 @@ vector unsigned int test_vec_inserth_uiv(void) {
 }
 
 vector signed int test_vec_vec_splati_si(void) {
-  // CHECK-BE: ret <4 x i32> <i32 -17, i32 -17, i32 -17, i32 -17>
   // CHECK: ret <4 x i32> <i32 -17, i32 -17, i32 -17, i32 -17>
   return vec_splati(-17);
 }
 
 vector unsigned int test_vec_vec_splati_ui(void) {
-  // CHECK-BE: ret <4 x i32> <i32 16, i32 16, i32 16, i32 16>
   // CHECK: ret <4 x i32> <i32 16, i32 16, i32 16, i32 16>
   return vec_splati(16U);
 }
 
 vector float test_vec_vec_splati_f(void) {
-  // CHECK-BE: ret <4 x float> <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>
   // CHECK: ret <4 x float> <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>
   return vec_splati(1.0f);
 }
@@ -536,10 +529,10 @@ vector double test_vec_vec_splatid(void) {
   // CHECK-BE-NEXT: [[T2:%.+]] = insertelement <2 x double> undef, double [[T1:%.+]], i32 0
   // CHECK-BE-NEXT: [[T3:%.+]] = shufflevector <2 x double> [[T2:%.+]], <2 x double> undef, <2 x i32> zeroinitialize
   // CHECK-BE-NEXT: ret <2 x double> [[T3:%.+]]
-  // CHECK: [[T1:%.+]] = fpext float %{{.+}} to double
-  // CHECK-NEXT: [[T2:%.+]] = insertelement <2 x double> undef, double [[T1:%.+]], i32 0
-  // CHECK-NEXT: [[T3:%.+]] = shufflevector <2 x double> [[T2:%.+]], <2 x double> undef, <2 x i32> zeroinitialize
-  // CHECK-NEXT: ret <2 x double> [[T3:%.+]]
+  // CHECK-LE: [[T1:%.+]] = fpext float %{{.+}} to double
+  // CHECK-LE-NEXT: [[T2:%.+]] = insertelement <2 x double> undef, double [[T1:%.+]], i32 0
+  // CHECK-LE-NEXT: [[T3:%.+]] = shufflevector <2 x double> [[T2:%.+]], <2 x double> undef, <2 x i32> zeroinitialize
+  // CHECK-LE-NEXT: ret <2 x double> [[T3:%.+]]
   return vec_splatid(1.0);
 }
 
@@ -548,11 +541,11 @@ vector signed int test_vec_vec_splati_ins_si(void) {
   // CHECK-BE:  [[T1:%.+]] = add i32 2, %{{.+}}
   // CHECK-BE: insertelement <4 x i32> %{{.+}}, i32 %{{.+}}, i32 [[T1]]
   // CHECK-BE: ret <4 x i32>
-  // CHECK:  [[T1:%.+]] = sub i32 1, %{{.+}}
-  // CHECK: insertelement <4 x i32> %{{.+}}, i32 %{{.+}}, i32 [[T1]]
-  // CHECK:  [[T2:%.+]] = sub i32 3, %{{.+}}
-  // CHECK: insertelement <4 x i32> %{{.+}}, i32 %{{.+}}, i32 [[T2]]
-  // CHECK: ret <4 x i32>
+  // CHECK-LE:  [[T1:%.+]] = sub i32 1, %{{.+}}
+  // CHECK-LE: insertelement <4 x i32> %{{.+}}, i32 %{{.+}}, i32 [[T1]]
+  // CHECK-LE:  [[T2:%.+]] = sub i32 3, %{{.+}}
+  // CHECK-LE: insertelement <4 x i32> %{{.+}}, i32 %{{.+}}, i32 [[T2]]
+  // CHECK-LE: ret <4 x i32>
   return vec_splati_ins(vsia, 0, -17);
 }
 
@@ -561,11 +554,11 @@ vector unsigned int test_vec_vec_splati_ins_ui(void) {
   // CHECK-BE:  [[T1:%.+]] = add i32 2, %{{.+}}
   // CHECK-BE: insertelement <4 x i32> %{{.+}}, i32 %{{.+}}, i32 [[T1]]
   // CHECK-BE: ret <4 x i32>
-  // CHECK:  [[T1:%.+]] = sub i32 1, %{{.+}}
-  // CHECK: insertelement <4 x i32> %{{.+}}, i32 %{{.+}}, i32 [[T1]]
-  // CHECK:  [[T2:%.+]] = sub i32 3, %{{.+}}
-  // CHECK: insertelement <4 x i32> %{{.+}}, i32 %{{.+}}, i32 [[T2]]
-  // CHECK: ret <4 x i32>
+  // CHECK-LE:  [[T1:%.+]] = sub i32 1, %{{.+}}
+  // CHECK-LE: insertelement <4 x i32> %{{.+}}, i32 %{{.+}}, i32 [[T1]]
+  // CHECK-LE:  [[T2:%.+]] = sub i32 3, %{{.+}}
+  // CHECK-LE: insertelement <4 x i32> %{{.+}}, i32 %{{.+}}, i32 [[T2]]
+  // CHECK-LE: ret <4 x i32>
   return vec_splati_ins(vuia, 1, 16U);
 }
 
@@ -574,11 +567,11 @@ vector float test_vec_vec_splati_ins_f(void) {
   // CHECK-BE:  [[T1:%.+]] = add i32 2, %{{.+}}
   // CHECK-BE: insertelement <4 x float> %{{.+}}, float %{{.+}}, i32 [[T1]]
   // CHECK-BE: ret <4 x float>
-  // CHECK:  [[T1:%.+]] = sub i32 1, %{{.+}}
-  // CHECK: insertelement <4 x float> %{{.+}}, float %{{.+}}, i32 [[T1]]
-  // CHECK:  [[T2:%.+]] = sub i32 3, %{{.+}}
-  // CHECK: insertelement <4 x float> %{{.+}}, float %{{.+}}, i32 [[T2]]
-  // CHECK: ret <4 x float>
+  // CHECK-LE:  [[T1:%.+]] = sub i32 1, %{{.+}}
+  // CHECK-LE: insertelement <4 x float> %{{.+}}, float %{{.+}}, i32 [[T1]]
+  // CHECK-LE:  [[T2:%.+]] = sub i32 3, %{{.+}}
+  // CHECK-LE: insertelement <4 x float> %{{.+}}, float %{{.+}}, i32 [[T2]]
+  // CHECK-LE: ret <4 x float>
   return vec_splati_ins(vfa, 0, 1.0f);
 }
 

From e97aa5609fa53827c8c09a0fc79075dc834f292d Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Sun, 26 Jul 2020 23:01:28 -0400
Subject: [PATCH 0156/1035] AMDGPU/GlobalISel: Don't assert in LegalizerInfo
 constructor

We don't really need these asserts. The LegalizerInfo is also
overly-aggressivly constructed, even when not in use. It needs to not
assert on dummy targets that have manually specified, unrelated
features.
---
 .../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp |  3 -
 .../CodeGen/AMDGPU/GlobalISel/dummy-target.ll | 91 +++++++++++++++++++
 2 files changed, 91 insertions(+), 3 deletions(-)
 create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/dummy-target.ll

diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 673c5fc1e840c..f1962db35bc01 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -423,7 +423,6 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
     .scalarize(0);
 
   if (ST.hasVOP3PInsts()) {
-    assert(ST.hasIntClamp() && "all targets with VOP3P should support clamp");
     getActionDefinitionsBuilder({G_ADD, G_SUB, G_MUL})
       .legalFor({S32, S16, V2S16})
       .clampScalar(0, S16, S32)
@@ -445,8 +444,6 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
       .scalarize(0)
       .widenScalarToNextPow2(0, 32); // FIXME: min should be 16
 
-    assert(ST.hasIntClamp() && "all targets with 16-bit should support clamp");
-
     // Technically the saturating operations require clamp bit support, but this
     // was introduced at the same time as 16-bit operations.
     getActionDefinitionsBuilder({G_UADDSAT, G_USUBSAT})
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/dummy-target.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/dummy-target.ll
new file mode 100644
index 0000000000000..81f6b8e71254c
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/dummy-target.ll
@@ -0,0 +1,91 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -stop-after=legalizer -o - %s | FileCheck %s
+
+; Make sure legalizer info doesn't assert on dummy targets
+
+define i16 @vop3p_add_i16(i16 %arg0) #0 {
+  ; CHECK-LABEL: name: vop3p_add_i16
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[ADD:%[0-9]+]]:_(s16) = G_ADD [[TRUNC]], [[TRUNC]]
+  ; CHECK:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ADD]](s16)
+  ; CHECK:   $vgpr0 = COPY [[ANYEXT]](s32)
+  ; CHECK:   [[COPY2:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY1]]
+  ; CHECK:   S_SETPC_B64_return [[COPY2]], implicit $vgpr0
+  %add = add i16 %arg0, %arg0
+  ret i16 %add
+}
+
+define <2 x i16> @vop3p_add_v2i16(<2 x i16> %arg0) #0 {
+  ; CHECK-LABEL: name: vop3p_add_v2i16
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
+  ; CHECK:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+  ; CHECK:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; CHECK:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+  ; CHECK:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+  ; CHECK:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
+  ; CHECK:   [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+  ; CHECK:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+  ; CHECK:   [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+  ; CHECK:   [[ADD:%[0-9]+]]:_(s16) = G_ADD [[TRUNC]], [[TRUNC2]]
+  ; CHECK:   [[ADD1:%[0-9]+]]:_(s16) = G_ADD [[TRUNC1]], [[TRUNC3]]
+  ; CHECK:   [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ADD]](s16)
+  ; CHECK:   [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[ADD1]](s16)
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL]]
+  ; CHECK:   [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
+  ; CHECK:   $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
+  ; CHECK:   [[COPY2:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY1]]
+  ; CHECK:   S_SETPC_B64_return [[COPY2]], implicit $vgpr0
+  %add = add <2 x i16> %arg0, %arg0
+  ret <2 x i16> %add
+}
+
+define i16 @halfinsts_add_i16(i16 %arg0) #1 {
+  ; CHECK-LABEL: name: halfinsts_add_i16
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[COPY2]], [[COPY2]]
+  ; CHECK:   [[COPY3:%[0-9]+]]:_(s32) = COPY [[ADD]](s32)
+  ; CHECK:   $vgpr0 = COPY [[COPY3]](s32)
+  ; CHECK:   [[COPY4:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY1]]
+  ; CHECK:   S_SETPC_B64_return [[COPY4]], implicit $vgpr0
+  %add = add i16 %arg0, %arg0
+  ret i16 %add
+}
+
+define <2 x i16> @halfinsts_add_v2i16(<2 x i16> %arg0) #1 {
+  ; CHECK-LABEL: name: halfinsts_add_v2i16
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[COPY4:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[COPY3]], [[COPY4]]
+  ; CHECK:   [[COPY5:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[COPY6:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[COPY5]], [[COPY6]]
+  ; CHECK:   [[COPY7:%[0-9]+]]:_(s32) = COPY [[ADD]](s32)
+  ; CHECK:   [[COPY8:%[0-9]+]]:_(s32) = COPY [[ADD1]](s32)
+  ; CHECK:   $vgpr0 = COPY [[COPY7]](s32)
+  ; CHECK:   $vgpr1 = COPY [[COPY8]](s32)
+  ; CHECK:   [[COPY9:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY2]]
+  ; CHECK:   S_SETPC_B64_return [[COPY9]], implicit $vgpr0, implicit $vgpr1
+  %add = add <2 x i16> %arg0, %arg0
+  ret <2 x i16> %add
+}
+
+attributes #0 = { "target-features"="+vop3p" }
+attributes #0 = { "target-features"="+16-bit-insts" }

From fae221e7ad5a94ed7697c13d169db18d253f5f15 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Sun, 26 Jul 2020 21:14:20 -0700
Subject: [PATCH 0157/1035] [gcov] Simplify/speed up CFG hash calculation

---
 .../Instrumentation/GCOVProfiling.cpp         | 30 ++++++++-----------
 1 file changed, 13 insertions(+), 17 deletions(-)

diff --git a/llvm/lib/Transforms/Instrumentation/GCOVProfiling.cpp b/llvm/lib/Transforms/Instrumentation/GCOVProfiling.cpp
index cd2ea8d5e4edd..53a89f7348de4 100644
--- a/llvm/lib/Transforms/Instrumentation/GCOVProfiling.cpp
+++ b/llvm/lib/Transforms/Instrumentation/GCOVProfiling.cpp
@@ -32,6 +32,7 @@
 #include "llvm/IR/Module.h"
 #include "llvm/InitializePasses.h"
 #include "llvm/Pass.h"
+#include "llvm/Support/CRC.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/FileSystem.h"
@@ -300,15 +301,16 @@ namespace {
       assert(OutEdges.empty());
     }
 
+    uint32_t Number;
+    SmallVector<GCOVBlock *, 4> OutEdges;
+
    private:
     friend class GCOVFunction;
 
     GCOVBlock(GCOVProfiler *P, uint32_t Number)
         : GCOVRecord(P), Number(Number) {}
 
-    uint32_t Number;
     StringMap<GCOVLines> LinesByFile;
-    SmallVector<GCOVBlock *, 4> OutEdges;
   };
 
   // A function has a unique identifier, a checksum (we leave as zero) and a
@@ -347,18 +349,6 @@ namespace {
       return ReturnBlock;
     }
 
-    std::string getEdgeDestinations() {
-      std::string EdgeDestinations;
-      raw_string_ostream EDOS(EdgeDestinations);
-      Function *F = Blocks.begin()->first->getParent();
-      for (BasicBlock &I : *F) {
-        GCOVBlock &Block = getBlock(&I);
-        for (int i = 0, e = Block.OutEdges.size(); i != e; ++i)
-          EDOS << Block.OutEdges[i]->Number;
-      }
-      return EdgeDestinations;
-    }
-
     uint32_t getFuncChecksum() const {
       return FuncChecksum;
     }
@@ -729,7 +719,7 @@ void GCOVProfiler::emitProfileNotes() {
       continue;
     }
 
-    std::string EdgeDestinations;
+    std::vector<uint8_t> EdgeDestinations;
 
     Endian = M->getDataLayout().isLittleEndian() ? support::endianness::little
                                                  : support::endianness::big;
@@ -774,6 +764,11 @@ void GCOVProfiler::emitProfileNotes() {
         } else if (isa<ReturnInst>(TI)) {
           Block.addEdge(Func.getReturnBlock());
         }
+        for (GCOVBlock *Succ : Block.OutEdges) {
+          uint32_t Idx = Succ->Number;
+          do EdgeDestinations.push_back(Idx & 255);
+          while ((Idx >>= 8) > 0);
+        }
 
         for (auto &I : BB) {
           // Debug intrinsic locations correspond to the location of the
@@ -798,12 +793,13 @@ void GCOVProfiler::emitProfileNotes() {
         }
         Line = 0;
       }
-      EdgeDestinations += Func.getEdgeDestinations();
     }
 
     char Tmp[4];
+    JamCRC JC;
+    JC.update(EdgeDestinations);
     os = &out;
-    auto Stamp = static_cast<uint32_t>(hash_value(EdgeDestinations));
+    uint32_t Stamp = JC.getCRC();
     FileChecksums.push_back(Stamp);
     if (Endian == support::endianness::big) {
       out.write("gcno", 4);

From 1c93f09bf34c2f7a9e6b1753cc2edb41609cd544 Mon Sep 17 00:00:00 2001
From: Mehdi Amini <joker.eph@gmail.com>
Date: Mon, 27 Jul 2020 04:50:08 +0000
Subject: [PATCH 0158/1035] Remove declaration of constexpr member kDynamicSize
 in MemRefType

This member is already publicly declared on the base class. The
redundant declaration is mangled differently though and in some
unoptimized build it requires a definition to also exist. However we
have a definition for the base ShapedType class, removing the
declaration here will redirect every use to the base class member
instead.

Differential Revision: https://reviews.llvm.org/D84615
---
 mlir/include/mlir/IR/StandardTypes.h | 1 -
 1 file changed, 1 deletion(-)

diff --git a/mlir/include/mlir/IR/StandardTypes.h b/mlir/include/mlir/IR/StandardTypes.h
index 5380668b59013..1ac24359cbb62 100644
--- a/mlir/include/mlir/IR/StandardTypes.h
+++ b/mlir/include/mlir/IR/StandardTypes.h
@@ -564,7 +564,6 @@ class MemRefType : public Type::TypeBase<MemRefType, BaseMemRefType,
   // TODO: merge these two special values in a single one used everywhere.
   // Unfortunately, uses of `-1` have crept deep into the codebase now and are
   // hard to track.
-  static constexpr int64_t kDynamicSize = -1;
   static int64_t getDynamicStrideOrOffset() {
     return ShapedType::kDynamicStrideOrOffset;
   }

From 825ed2d43d6f3e981ae5d2bbd3b2f4f4fbc2fc39 Mon Sep 17 00:00:00 2001
From: biplmish <biplmish@in.ibm.com>
Date: Sun, 26 Jul 2020 23:56:19 -0500
Subject: [PATCH 0159/1035] [PowerPC] Add Vector Extract Double Instruction
 Definitions and MC tests.

This patch adds the td definitions and asm/disasm tests for the following instructions:

Vector Extract Double Left Index - vextdubvlx, vextduhvlx, vextduwvlx, vextddvlx
Vector Extract Double Right Index - vextdubvrx, vextduhvrx, vextduwvrx, vextddvrx

Differential Revision: https://reviews.llvm.org/D84384
---
 llvm/lib/Target/PowerPC/PPCInstrPrefix.td     | 32 +++++++++++++++++++
 .../PowerPC/ppc64-encoding-ISA31.txt          | 24 ++++++++++++++
 llvm/test/MC/PowerPC/ppc64-encoding-ISA31.s   | 24 ++++++++++++++
 3 files changed, 80 insertions(+)

diff --git a/llvm/lib/Target/PowerPC/PPCInstrPrefix.td b/llvm/lib/Target/PowerPC/PPCInstrPrefix.td
index 22839e697381b..418ef3b377282 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrPrefix.td
+++ b/llvm/lib/Target/PowerPC/PPCInstrPrefix.td
@@ -905,6 +905,38 @@ let Predicates = [IsISA3_1] in {
               [(set v2i64:$vD,
                     (int_ppc_altivec_vinsdrx v2i64:$vDi, i64:$rA, i64:$rB))]>,
               RegConstraint<"$vDi = $vD">, NoEncode<"$vDi">;
+  def VEXTDUBVLX : VAForm_1a<24, (outs vrrc:$vD),
+                             (ins vrrc:$vA, vrrc:$vB, g8rc:$rC),
+                             "vextdubvlx $vD, $vA, $vB, $rC",
+                             IIC_VecGeneral, []>;
+  def VEXTDUBVRX : VAForm_1a<25, (outs vrrc:$vD),
+                             (ins vrrc:$vA, vrrc:$vB, g8rc:$rC),
+                             "vextdubvrx $vD, $vA, $vB, $rC",
+                             IIC_VecGeneral, []>;
+  def VEXTDUHVLX : VAForm_1a<26, (outs vrrc:$vD),
+                             (ins vrrc:$vA, vrrc:$vB, g8rc:$rC),
+                             "vextduhvlx $vD, $vA, $vB, $rC",
+                             IIC_VecGeneral, []>;
+  def VEXTDUHVRX : VAForm_1a<27, (outs vrrc:$vD),
+                             (ins vrrc:$vA, vrrc:$vB, g8rc:$rC),
+                             "vextduhvrx $vD, $vA, $vB, $rC",
+                             IIC_VecGeneral, []>;
+  def VEXTDUWVLX : VAForm_1a<28, (outs vrrc:$vD),
+                             (ins vrrc:$vA, vrrc:$vB, g8rc:$rC),
+                             "vextduwvlx $vD, $vA, $vB, $rC",
+                             IIC_VecGeneral, []>;
+  def VEXTDUWVRX : VAForm_1a<29, (outs vrrc:$vD),
+                             (ins vrrc:$vA, vrrc:$vB, g8rc:$rC),
+                             "vextduwvrx $vD, $vA, $vB, $rC",
+                             IIC_VecGeneral, []>;
+  def VEXTDDVLX : VAForm_1a<30, (outs vrrc:$vD),
+                            (ins vrrc:$vA, vrrc:$vB, g8rc:$rC),
+                            "vextddvlx $vD, $vA, $vB, $rC",
+                            IIC_VecGeneral, []>;
+  def VEXTDDVRX : VAForm_1a<31, (outs vrrc:$vD),
+                            (ins vrrc:$vA, vrrc:$vB, g8rc:$rC),
+                            "vextddvrx $vD, $vA, $vB, $rC",
+                            IIC_VecGeneral, []>;
    def VPDEPD : VXForm_1<1485, (outs vrrc:$vD), (ins vrrc:$vA, vrrc:$vB),
                          "vpdepd $vD, $vA, $vB", IIC_VecGeneral,
                          [(set v2i64:$vD,
diff --git a/llvm/test/MC/Disassembler/PowerPC/ppc64-encoding-ISA31.txt b/llvm/test/MC/Disassembler/PowerPC/ppc64-encoding-ISA31.txt
index 038a7f1a84fe2..c8dae6a160a5b 100644
--- a/llvm/test/MC/Disassembler/PowerPC/ppc64-encoding-ISA31.txt
+++ b/llvm/test/MC/Disassembler/PowerPC/ppc64-encoding-ISA31.txt
@@ -330,6 +330,30 @@
 # CHECK: vinsdrx 1, 2, 3
 0x10 0x22 0x1b 0xcf
 
+# CHECK: vextdubvlx 1, 2, 3, 3
+0x10 0x22 0x18 0xd8
+
+# CHECK: vextdubvrx 1, 2, 3, 3
+0x10 0x22 0x18 0xd9
+
+# CHECK: vextduhvlx 1, 2, 3, 3
+0x10 0x22 0x18 0xda
+
+# CHECK: vextduhvrx 1, 2, 3, 3
+0x10 0x22 0x18 0xdb
+
+# CHECK: vextduwvlx 1, 2, 3, 3
+0x10 0x22 0x18 0xdc
+
+# CHECK: vextduwvrx 1, 2, 3, 3
+0x10 0x22 0x18 0xdd
+
+# CHECK: vextddvlx 1, 2, 3, 3
+0x10 0x22 0x18 0xde
+
+# CHECK: vextddvrx 1, 2, 3, 3
+0x10 0x22 0x18 0xdf
+
 # CHECK: lxvrbx 32, 1, 2
 0x7c 0x01 0x10 0x1b
 
diff --git a/llvm/test/MC/PowerPC/ppc64-encoding-ISA31.s b/llvm/test/MC/PowerPC/ppc64-encoding-ISA31.s
index c9f5547dd4c86..bd1187f18ed8f 100644
--- a/llvm/test/MC/PowerPC/ppc64-encoding-ISA31.s
+++ b/llvm/test/MC/PowerPC/ppc64-encoding-ISA31.s
@@ -456,6 +456,30 @@
 # CHECK-BE: vinsdrx 1, 2, 3                       # encoding: [0x10,0x22,0x1b,0xcf]
 # CHECK-LE: vinsdrx 1, 2, 3                       # encoding: [0xcf,0x1b,0x22,0x10]
             vinsdrx 1, 2, 3
+# CHECK-BE: vextdubvlx 1, 2, 3, 3                 # encoding: [0x10,0x22,0x18,0xd8]
+# CHECK-LE: vextdubvlx 1, 2, 3, 3                 # encoding: [0xd8,0x18,0x22,0x10]
+            vextdubvlx 1, 2, 3, 3
+# CHECK-BE: vextdubvrx 1, 2, 3, 3                 # encoding: [0x10,0x22,0x18,0xd9]
+# CHECK-LE: vextdubvrx 1, 2, 3, 3                 # encoding: [0xd9,0x18,0x22,0x10]
+            vextdubvrx 1, 2, 3, 3
+# CHECK-BE: vextduhvlx 1, 2, 3, 3                 # encoding: [0x10,0x22,0x18,0xda]
+# CHECK-LE: vextduhvlx 1, 2, 3, 3                 # encoding: [0xda,0x18,0x22,0x10]
+            vextduhvlx 1, 2, 3, 3
+# CHECK-BE: vextduhvrx 1, 2, 3, 3                 # encoding: [0x10,0x22,0x18,0xdb]
+# CHECK-LE: vextduhvrx 1, 2, 3, 3                 # encoding: [0xdb,0x18,0x22,0x10]
+            vextduhvrx 1, 2, 3, 3
+# CHECK-BE: vextduwvlx 1, 2, 3, 3                 # encoding: [0x10,0x22,0x18,0xdc]
+# CHECK-LE: vextduwvlx 1, 2, 3, 3                 # encoding: [0xdc,0x18,0x22,0x10]
+            vextduwvlx 1, 2, 3, 3
+# CHECK-BE: vextduwvrx 1, 2, 3, 3                 # encoding: [0x10,0x22,0x18,0xdd]
+# CHECK-LE: vextduwvrx 1, 2, 3, 3                 # encoding: [0xdd,0x18,0x22,0x10]
+            vextduwvrx 1, 2, 3, 3
+# CHECK-BE: vextddvlx 1, 2, 3, 3                  # encoding: [0x10,0x22,0x18,0xde]
+# CHECK-LE: vextddvlx 1, 2, 3, 3                  # encoding: [0xde,0x18,0x22,0x10]
+            vextddvlx 1, 2, 3, 3
+# CHECK-BE: vextddvrx 1, 2, 3, 3                  # encoding: [0x10,0x22,0x18,0xdf]
+# CHECK-LE: vextddvrx 1, 2, 3, 3                  # encoding: [0xdf,0x18,0x22,0x10]
+            vextddvrx 1, 2, 3, 3
 # CHECK-BE: lxvrbx 32, 1, 2                       # encoding: [0x7c,0x01,0x10,0x1b]
 # CHECK-LE: lxvrbx 32, 1, 2                       # encoding: [0x1b,0x10,0x01,0x7c]
             lxvrbx 32, 1, 2

From eed333149d178b69fdaf39b9419b7ca032520182 Mon Sep 17 00:00:00 2001
From: Yuanfang Chen <yuanfang.chen@sony.com>
Date: Sun, 26 Jul 2020 22:32:24 -0700
Subject: [PATCH 0160/1035] [NewPM] NFC. remove obsolete TODO comment

The deleted TODO was implemented in D82344.
---
 llvm/include/llvm/IR/PassInstrumentation.h | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/llvm/include/llvm/IR/PassInstrumentation.h b/llvm/include/llvm/IR/PassInstrumentation.h
index 37390e4e682ba..94833bb0d2c63 100644
--- a/llvm/include/llvm/IR/PassInstrumentation.h
+++ b/llvm/include/llvm/IR/PassInstrumentation.h
@@ -44,10 +44,6 @@
 ///      of a pass. For those callbacks returning false means pass will not be
 ///      executed.
 ///
-/// TODO: currently there is no way for a pass to opt-out of execution control
-/// (e.g. become unskippable). PassManager is the only entity that determines
-/// how pass instrumentation affects pass execution.
-///
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_IR_PASSINSTRUMENTATION_H

From f2ab2134c7c4949eea3f6ddd35bcea236fe12592 Mon Sep 17 00:00:00 2001
From: Roman Lebedev <lebedev.ri@gmail.com>
Date: Mon, 27 Jul 2020 10:15:44 +0300
Subject: [PATCH 0161/1035] [XRay] Account: recursion detection

Summary:
Recursion detection can be non-trivial. Currently, the state-of-the-art for LLVM,
as far as i'm concerned, is D72362 `[clang-tidy] misc-no-recursion: a new check`.
However, it is quite limited:
* It does very basic call-graph based analysis, in the sense it will report even dynamically-unreachable recursion.
* It is inherently limited to a single TU
* It is hard to gauge how problematic each recursion is in practice.

Some of that can be addressed by adding clang analyzer-based check,
then it would at least support multiple TU's.

However, we can approach this problem from another angle - dynamic run-time analysis.
We already have means to capture a run-time callgraph (XRay, duh),
and there are already means to reconstruct it within `llvm-xray` tool.

This proposes to add a `-recursive-calls-only` switch to the `account` tool.
When the switch is on, when re-constructing callgraph for latency reconstruction,
each time we enter/leave some function, we increment/decrement an entry for the function
in a "recursion depth" map. If, when we leave the function, said entry was at `1`,
then that means the function didn't call itself, however if it is at `2` or more,
then that means the function (possibly indirectly) called itself.

If the depth is 1, we don't account the time spent there,
unless within this call stack the function already recursed into itself.
Note that we don't pay for recursion depth tracking when `recursive-calls-only` is not on,
and the perf impact is insignificant (+0.3% regression)

The overhead of the option is actually negative, around -5.26% user time on a medium-sized (3.5G) XRay log.
As a practical example, that 3.5G log is a capture of the entire middle-end opt pipeline
at `-O3` for RawSpeed unity build. There are total of `5500` functions in the log,
however `-recursive-calls-only` says that `269`, or 5%, are recursive.

Having this functionality could be helpful for recursion eradication.

Reviewers: dberris, mboerger

Reviewed By: dberris

Subscribers: llvm-commits

Tags: #llvm

Differential Revision: https://reviews.llvm.org/D84582
---
 ...ursive-calls-only-tail-call-deduction.yaml | 46 +++++++++++
 .../X86/account-recursive-calls-only.yaml     | 34 +++++++++
 llvm/tools/llvm-xray/xray-account.cpp         | 76 +++++++++++++++----
 llvm/tools/llvm-xray/xray-account.h           | 21 ++++-
 4 files changed, 158 insertions(+), 19 deletions(-)
 create mode 100644 llvm/test/tools/llvm-xray/X86/account-recursive-calls-only-tail-call-deduction.yaml
 create mode 100644 llvm/test/tools/llvm-xray/X86/account-recursive-calls-only.yaml

diff --git a/llvm/test/tools/llvm-xray/X86/account-recursive-calls-only-tail-call-deduction.yaml b/llvm/test/tools/llvm-xray/X86/account-recursive-calls-only-tail-call-deduction.yaml
new file mode 100644
index 0000000000000..4f5d01b2cf9b1
--- /dev/null
+++ b/llvm/test/tools/llvm-xray/X86/account-recursive-calls-only-tail-call-deduction.yaml
@@ -0,0 +1,46 @@
+# RUN: llvm-xray account -d %s -o - -m %S/Inputs/simple-instrmap.yaml | FileCheck --check-prefixes=ALL %s
+# RUN: llvm-xray account -d -recursive-calls-only %s -o - -m %S/Inputs/simple-instrmap.yaml | FileCheck --check-prefixes=RECURSIVE %s
+
+---
+header:
+  version: 1
+  type: 0
+  constant-tsc: true
+  nonstop-tsc: true
+  cycle-frequency: 0
+records:
+# Here we reconstruct the following call trace:
+#
+#   f1()
+#     f2()
+#       f3()
+#   f2()
+#
+# But we find that we're missing an exit record for f2() because it's
+# tail-called f3(). We make sure that if we see a trace like this that we can
+# deduce tail calls, and account the time (potentially wrongly) to f2() when
+# f1() exits. That is because we don't go back to f3()'s entry record to
+# properly do the math on the timing of f2().
+#
+# As a result, we can deduce that f2() is not recursive here.
+#
+# Note that by default, tail/sibling call deduction is disabled, and is enabled
+# with a flag "-d" or "-deduce-sibling-calls".
+#
+  - { type: 0, func-id: 1, cpu: 1, thread: 111, kind: function-enter, tsc: 10000 }
+  - { type: 0, func-id: 2, cpu: 1, thread: 111, kind: function-enter, tsc: 10001 }
+  - { type: 0, func-id: 3, cpu: 1, thread: 111, kind: function-enter, tsc: 10002 }
+  - { type: 0, func-id: 3, cpu: 1, thread: 111, kind: function-exit,  tsc: 10003 }
+  - { type: 0, func-id: 1, cpu: 1, thread: 111, kind: function-exit,  tsc: 10004 }
+  - { type: 0, func-id: 2, cpu: 1, thread: 111, kind: function-enter, tsc: 10005 }
+  - { type: 0, func-id: 2, cpu: 1, thread: 111, kind: function-exit,  tsc: 10006 }
+...
+
+# ALL:      Functions with latencies: 3
+# ALL-NEXT:    funcid      count [      min,       med,       90p,       99p,       max]       sum  function
+# ALL-NEXT:         1          1 [ 4.000000,  4.000000,  4.000000,  4.000000,  4.000000]  4.000000  <invalid>:0:0: @(1)
+# ALL-NEXT:         2          2 [ 1.000000,  3.000000,  3.000000,  3.000000,  3.000000]  4.000000  <invalid>:0:0: @(2)
+# ALL-NEXT:         3          1 [ 1.000000,  1.000000,  1.000000,  1.000000,  1.000000]  1.000000  <invalid>:0:0: @(3)
+
+# RECURSIVE:      Functions with latencies: 0
+# RECURSIVE-NEXT:    funcid      count [      min,       med,       90p,       99p,       max]       sum  function
diff --git a/llvm/test/tools/llvm-xray/X86/account-recursive-calls-only.yaml b/llvm/test/tools/llvm-xray/X86/account-recursive-calls-only.yaml
new file mode 100644
index 0000000000000..d7b36200d10d3
--- /dev/null
+++ b/llvm/test/tools/llvm-xray/X86/account-recursive-calls-only.yaml
@@ -0,0 +1,34 @@
+# RUN: llvm-xray account %s -o - -m %S/Inputs/simple-instrmap.yaml | FileCheck --check-prefixes=ALL %s
+# RUN: llvm-xray account -recursive-calls-only %s -o - -m %S/Inputs/simple-instrmap.yaml | FileCheck --check-prefixes=RECURSIVE %s
+
+---
+header:
+  version: 1
+  type: 0
+  constant-tsc: true
+  nonstop-tsc: true
+  cycle-frequency: 2601000000
+records:
+  - { type: 0, func-id: 1, cpu: 1, thread: 111, kind: function-enter, tsc: 0 }
+  - { type: 0, func-id: 1, cpu: 1, thread: 111, kind: function-exit,  tsc: 100000000 }
+  - { type: 0, func-id: 2, cpu: 1, thread: 111, kind: function-enter, tsc: 200000000 }
+  - { type: 0, func-id: 2, cpu: 1, thread: 111, kind: function-exit,  tsc: 300000000 }
+  - { type: 0, func-id: 2, cpu: 1, thread: 111, kind: function-enter, tsc: 400000000 }
+  - { type: 0, func-id: 3, cpu: 1, thread: 111, kind: function-enter, tsc: 500000000 }
+  - { type: 0, func-id: 3, cpu: 1, thread: 111, kind: function-enter, tsc: 600000000 }
+  - { type: 0, func-id: 3, cpu: 1, thread: 111, kind: function-exit,  tsc: 700000000 }
+  - { type: 0, func-id: 3, cpu: 1, thread: 111, kind: function-exit,  tsc: 800000000 }
+  - { type: 0, func-id: 2, cpu: 1, thread: 111, kind: function-exit,  tsc: 900000000 }
+  - { type: 0, func-id: 3, cpu: 1, thread: 111, kind: function-enter, tsc: 1000000000 }
+  - { type: 0, func-id: 3, cpu: 1, thread: 111, kind: function-exit,  tsc: 1100000000 }
+...
+
+# ALL:      Functions with latencies: 3
+# ALL-NEXT:    funcid      count [      min,       med,       90p,       99p,       max]       sum  function
+# ALL-NEXT:         1          1 [ 0.038447,  0.038447,  0.038447,  0.038447,  0.038447]  0.038447  <invalid>:0:0: @(1)
+# ALL-NEXT:         2          2 [ 0.038447,  0.192234,  0.192234,  0.192234,  0.192234]  0.230681  <invalid>:0:0: @(2)
+# ALL-NEXT:         3          3 [ 0.038447,  0.038447,  0.115340,  0.115340,  0.115340]  0.192234  <invalid>:0:0: @(3)
+
+# RECURSIVE:      Functions with latencies: 1
+# RECURSIVE-NEXT:    funcid      count [      min,       med,       90p,       99p,       max]       sum  function
+# RECURSIVE-NEXT:         3          2 [ 0.038447,  0.115340,  0.115340,  0.115340,  0.115340]  0.153787  <invalid>:0:0: @(3)
diff --git a/llvm/tools/llvm-xray/xray-account.cpp b/llvm/tools/llvm-xray/xray-account.cpp
index 21da53f362b09..bde028a432fed 100644
--- a/llvm/tools/llvm-xray/xray-account.cpp
+++ b/llvm/tools/llvm-xray/xray-account.cpp
@@ -35,6 +35,9 @@ static cl::opt<bool>
                      cl::sub(Account), cl::init(false));
 static cl::alias AccountKeepGoing2("k", cl::aliasopt(AccountKeepGoing),
                                    cl::desc("Alias for -keep_going"));
+static cl::opt<bool> AccountRecursiveCallsOnly(
+    "recursive-calls-only", cl::desc("Only count the calls that are recursive"),
+    cl::sub(Account), cl::init(false));
 static cl::opt<bool> AccountDeduceSiblingCalls(
     "deduce-sibling-calls",
     cl::desc("Deduce sibling calls when unrolling function call stacks"),
@@ -126,6 +129,32 @@ template <class T> T diff(T L, T R) { return std::max(L, R) - std::min(L, R); }
 
 } // namespace
 
+using RecursionStatus = LatencyAccountant::FunctionStack::RecursionStatus;
+RecursionStatus &RecursionStatus::operator++() {
+  auto Depth = Bitfield::get<RecursionStatus::Depth>(Storage);
+  assert(Depth >= 0 && Depth < std::numeric_limits<decltype(Depth)>::max());
+  ++Depth;
+  Bitfield::set<RecursionStatus::Depth>(Storage, Depth); // ++Storage
+  // Did this function just (maybe indirectly) call itself the first time?
+  if (!isRecursive() && Depth == 2) // Storage == 2  /  Storage s> 1
+    Bitfield::set<RecursionStatus::IsRecursive>(Storage,
+                                                true); // Storage |= INT_MIN
+  return *this;
+}
+RecursionStatus &RecursionStatus::operator--() {
+  auto Depth = Bitfield::get<RecursionStatus::Depth>(Storage);
+  assert(Depth > 0);
+  --Depth;
+  Bitfield::set<RecursionStatus::Depth>(Storage, Depth); // --Storage
+  // Did we leave a function that previouly (maybe indirectly) called itself?
+  if (isRecursive() && Depth == 0) // Storage == INT_MIN
+    Bitfield::set<RecursionStatus::IsRecursive>(Storage, false); // Storage = 0
+  return *this;
+}
+bool RecursionStatus::isRecursive() const {
+  return Bitfield::get<RecursionStatus::IsRecursive>(Storage); // Storage s< 0
+}
+
 bool LatencyAccountant::accountRecord(const XRayRecord &Record) {
   setMinMax(PerThreadMinMaxTSC[Record.TId], Record.TSC);
   setMinMax(PerCPUMinMaxTSC[Record.CPU], Record.TSC);
@@ -137,6 +166,8 @@ bool LatencyAccountant::accountRecord(const XRayRecord &Record) {
     return false;
 
   auto &ThreadStack = PerThreadFunctionStack[Record.TId];
+  if (RecursiveCallsOnly && !ThreadStack.RecursionDepth)
+    ThreadStack.RecursionDepth.emplace();
   switch (Record.Type) {
   case RecordTypes::CUSTOM_EVENT:
   case RecordTypes::TYPED_EVENT:
@@ -144,18 +175,24 @@ bool LatencyAccountant::accountRecord(const XRayRecord &Record) {
     return true;
   case RecordTypes::ENTER:
   case RecordTypes::ENTER_ARG: {
-    ThreadStack.emplace_back(Record.FuncId, Record.TSC);
+    ThreadStack.Stack.emplace_back(Record.FuncId, Record.TSC);
+    if (ThreadStack.RecursionDepth)
+      ++(*ThreadStack.RecursionDepth)[Record.FuncId];
     break;
   }
   case RecordTypes::EXIT:
   case RecordTypes::TAIL_EXIT: {
-    if (ThreadStack.empty())
+    if (ThreadStack.Stack.empty())
       return false;
 
-    if (ThreadStack.back().first == Record.FuncId) {
-      const auto &Top = ThreadStack.back();
-      recordLatency(Top.first, diff(Top.second, Record.TSC));
-      ThreadStack.pop_back();
+    if (ThreadStack.Stack.back().first == Record.FuncId) {
+      const auto &Top = ThreadStack.Stack.back();
+      if (!ThreadStack.RecursionDepth ||
+          (*ThreadStack.RecursionDepth)[Top.first].isRecursive())
+        recordLatency(Top.first, diff(Top.second, Record.TSC));
+      if (ThreadStack.RecursionDepth)
+        --(*ThreadStack.RecursionDepth)[Top.first];
+      ThreadStack.Stack.pop_back();
       break;
     }
 
@@ -164,11 +201,11 @@ bool LatencyAccountant::accountRecord(const XRayRecord &Record) {
 
     // Look for the parent up the stack.
     auto Parent =
-        std::find_if(ThreadStack.rbegin(), ThreadStack.rend(),
+        std::find_if(ThreadStack.Stack.rbegin(), ThreadStack.Stack.rend(),
                      [&](const std::pair<const int32_t, uint64_t> &E) {
                        return E.first == Record.FuncId;
                      });
-    if (Parent == ThreadStack.rend())
+    if (Parent == ThreadStack.Stack.rend())
       return false;
 
     // Account time for this apparently sibling call exit up the stack.
@@ -199,11 +236,17 @@ bool LatencyAccountant::accountRecord(const XRayRecord &Record) {
     // complexity to do correctly (need to backtrack, etc.).
     //
     // FIXME: Potentially implement the more complex deduction algorithm?
-    auto I = std::next(Parent).base();
-    for (auto &E : make_range(I, ThreadStack.end())) {
-      recordLatency(E.first, diff(E.second, Record.TSC));
+    auto R = make_range(std::next(Parent).base(), ThreadStack.Stack.end());
+    for (auto &E : R) {
+      if (!ThreadStack.RecursionDepth ||
+          (*ThreadStack.RecursionDepth)[E.first].isRecursive())
+        recordLatency(E.first, diff(E.second, Record.TSC));
+    }
+    for (auto &Top : reverse(R)) {
+      if (ThreadStack.RecursionDepth)
+        --(*ThreadStack.RecursionDepth)[Top.first];
+      ThreadStack.Stack.pop_back();
     }
-    ThreadStack.erase(I, ThreadStack.end());
     break;
   }
   }
@@ -425,7 +468,8 @@ static CommandRegistration Unused(&Account, []() -> Error {
   symbolize::LLVMSymbolizer Symbolizer;
   llvm::xray::FuncIdConversionHelper FuncIdHelper(AccountInstrMap, Symbolizer,
                                                   FunctionAddresses);
-  xray::LatencyAccountant FCA(FuncIdHelper, AccountDeduceSiblingCalls);
+  xray::LatencyAccountant FCA(FuncIdHelper, AccountRecursiveCallsOnly,
+                              AccountDeduceSiblingCalls);
   auto TraceOrErr = loadTraceFile(AccountInput);
   if (!TraceOrErr)
     return joinErrors(
@@ -447,12 +491,12 @@ static CommandRegistration Unused(&Account, []() -> Error {
         << '\n';
     for (const auto &ThreadStack : FCA.getPerThreadFunctionStack()) {
       errs() << "Thread ID: " << ThreadStack.first << "\n";
-      if (ThreadStack.second.empty()) {
+      if (ThreadStack.second.Stack.empty()) {
         errs() << "  (empty stack)\n";
         continue;
       }
-      auto Level = ThreadStack.second.size();
-      for (const auto &Entry : llvm::reverse(ThreadStack.second))
+      auto Level = ThreadStack.second.Stack.size();
+      for (const auto &Entry : llvm::reverse(ThreadStack.second.Stack))
         errs() << "  #" << Level-- << "\t"
                << FuncIdHelper.SymbolOrNumber(Entry.first) << '\n';
     }
diff --git a/llvm/tools/llvm-xray/xray-account.h b/llvm/tools/llvm-xray/xray-account.h
index 575114d6096a2..371a9cc708e9a 100644
--- a/llvm/tools/llvm-xray/xray-account.h
+++ b/llvm/tools/llvm-xray/xray-account.h
@@ -18,6 +18,7 @@
 #include <vector>
 
 #include "func-id-helper.h"
+#include "llvm/ADT/Bitfields.h"
 #include "llvm/Support/Program.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/XRay/XRayRecord.h"
@@ -33,7 +34,19 @@ class LatencyAccountant {
       PerThreadMinMaxTSCMap;
   typedef llvm::DenseMap<uint8_t, std::pair<uint64_t, uint64_t>>
       PerCPUMinMaxTSCMap;
-  typedef llvm::SmallVector<std::pair<int32_t, uint64_t>, 32> FunctionStack;
+  struct FunctionStack {
+    llvm::SmallVector<std::pair<int32_t, uint64_t>, 32> Stack;
+    class RecursionStatus {
+      uint32_t Storage = 0;
+      using Depth = Bitfield::Element<int32_t, 0, 31>;    // Low 31 bits.
+      using IsRecursive = Bitfield::Element<bool, 31, 1>; // Sign bit.
+    public:
+      RecursionStatus &operator++();
+      RecursionStatus &operator--();
+      bool isRecursive() const;
+    };
+    Optional<llvm::DenseMap<int32_t, RecursionStatus>> RecursionDepth;
+  };
   typedef llvm::DenseMap<uint32_t, FunctionStack> PerThreadFunctionStackMap;
 
 private:
@@ -43,6 +56,7 @@ class LatencyAccountant {
   PerCPUMinMaxTSCMap PerCPUMinMaxTSC;
   FuncIdConversionHelper &FuncIdHelper;
 
+  bool RecursiveCallsOnly = false;
   bool DeduceSiblingCalls = false;
   uint64_t CurrentMaxTSC = 0;
 
@@ -52,8 +66,9 @@ class LatencyAccountant {
 
 public:
   explicit LatencyAccountant(FuncIdConversionHelper &FuncIdHelper,
-                             bool DeduceSiblingCalls)
-      : FuncIdHelper(FuncIdHelper), DeduceSiblingCalls(DeduceSiblingCalls) {}
+                             bool RecursiveCallsOnly, bool DeduceSiblingCalls)
+      : FuncIdHelper(FuncIdHelper), RecursiveCallsOnly(RecursiveCallsOnly),
+        DeduceSiblingCalls(DeduceSiblingCalls) {}
 
   const FunctionLatencyMap &getFunctionLatencies() const {
     return FunctionLatencies;

From 14bc85e0ebb6c00c1672158ab6a692bfbb11e1cc Mon Sep 17 00:00:00 2001
From: David Sherwood <david.sherwood@arm.com>
Date: Tue, 14 Jul 2020 16:20:00 +0100
Subject: [PATCH 0162/1035] [SVE] Don't use LocalStackAllocation for SVE
 objects

I have introduced a new TargetFrameLowering query function:

  isStackIdSafeForLocalArea

that queries whether or not it is safe for objects of a given stack
id to be bundled into the local area. The default behaviour is to
always bundle regardless of the stack id, however for AArch64 this is
overriden so that it's only safe for fixed-size stack objects.
There is future work here to extend this algorithm for multiple local
areas so that SVE stack objects can be bundled together and accessed
from their own virtual base-pointer.

Differential Revision: https://reviews.llvm.org/D83859
---
 .../llvm/CodeGen/TargetFrameLowering.h        |  6 ++
 llvm/lib/CodeGen/LocalStackSlotAllocation.cpp |  4 ++
 .../lib/Target/AArch64/AArch64FrameLowering.h |  6 ++
 .../CodeGen/AArch64/sve-localstackalloc.mir   | 61 +++++++++++++++++++
 4 files changed, 77 insertions(+)
 create mode 100644 llvm/test/CodeGen/AArch64/sve-localstackalloc.mir

diff --git a/llvm/include/llvm/CodeGen/TargetFrameLowering.h b/llvm/include/llvm/CodeGen/TargetFrameLowering.h
index c3a11b1996759..d6580430daf73 100644
--- a/llvm/include/llvm/CodeGen/TargetFrameLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetFrameLowering.h
@@ -134,6 +134,12 @@ class TargetFrameLowering {
   /// was called).
   virtual unsigned getStackAlignmentSkew(const MachineFunction &MF) const;
 
+  /// This method returns whether or not it is safe for an object with the
+  /// given stack id to be bundled into the local area.
+  virtual bool isStackIdSafeForLocalArea(unsigned StackId) const {
+    return true;
+  }
+
   /// getOffsetOfLocalArea - This method returns the offset of the local area
   /// from the stack pointer on entrance to a function.
   ///
diff --git a/llvm/lib/CodeGen/LocalStackSlotAllocation.cpp b/llvm/lib/CodeGen/LocalStackSlotAllocation.cpp
index 6c5ef0255a082..204fb556d8105 100644
--- a/llvm/lib/CodeGen/LocalStackSlotAllocation.cpp
+++ b/llvm/lib/CodeGen/LocalStackSlotAllocation.cpp
@@ -220,6 +220,8 @@ void LocalStackSlotPass::calculateFrameObjectOffsets(MachineFunction &Fn) {
         continue;
       if (StackProtectorFI == (int)i)
         continue;
+      if (!TFI.isStackIdSafeForLocalArea(MFI.getStackID(i)))
+        continue;
 
       switch (MFI.getObjectSSPLayout(i)) {
       case MachineFrameInfo::SSPLK_None:
@@ -254,6 +256,8 @@ void LocalStackSlotPass::calculateFrameObjectOffsets(MachineFunction &Fn) {
       continue;
     if (ProtectedObjs.count(i))
       continue;
+    if (!TFI.isStackIdSafeForLocalArea(MFI.getStackID(i)))
+      continue;
 
     AdjustStackOffset(MFI, i, Offset, StackGrowsDown, MaxAlign);
   }
diff --git a/llvm/lib/Target/AArch64/AArch64FrameLowering.h b/llvm/lib/Target/AArch64/AArch64FrameLowering.h
index 9d0a6d9eaf255..444740cb50ab9 100644
--- a/llvm/lib/Target/AArch64/AArch64FrameLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64FrameLowering.h
@@ -105,6 +105,12 @@ class AArch64FrameLowering : public TargetFrameLowering {
     }
   }
 
+  bool isStackIdSafeForLocalArea(unsigned StackId) const override {
+    // We don't support putting SVE objects into the pre-allocated local
+    // frame block at the moment.
+    return StackId != TargetStackID::SVEVector;
+  }
+
 private:
   bool shouldCombineCSRLocalStackBump(MachineFunction &MF,
                                       uint64_t StackBumpBytes) const;
diff --git a/llvm/test/CodeGen/AArch64/sve-localstackalloc.mir b/llvm/test/CodeGen/AArch64/sve-localstackalloc.mir
new file mode 100644
index 0000000000000..c20846c54b6a2
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-localstackalloc.mir
@@ -0,0 +1,61 @@
+# RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve -run-pass=localstackalloc -o - %s | FileCheck %s
+
+--- |
+  ; ModuleID = '<stdin>'
+  source_filename = "<stdin>"
+  target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+  target triple = "aarch64-unknown-linux-gnu"
+
+  define <vscale x 32 x i8> @insert_32i8_idx(<vscale x 32 x i8> %a, i8 %elt, i64 %idx) #0 {
+    %ins = insertelement <vscale x 32 x i8> %a, i8 %elt, i64 %idx
+    ret <vscale x 32 x i8> %ins
+  }
+
+  attributes #0 = { "target-features"="+sve" }
+
+...
+---
+name:            insert_32i8_idx
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: zpr, preferred-register: '' }
+  - { id: 1, class: zpr, preferred-register: '' }
+  - { id: 2, class: gpr32, preferred-register: '' }
+  - { id: 3, class: gpr64, preferred-register: '' }
+  - { id: 5, class: ppr_3b, preferred-register: '' }
+  - { id: 6, class: gpr64sp, preferred-register: '' }
+  - { id: 7, class: zpr, preferred-register: '' }
+  - { id: 8, class: zpr, preferred-register: '' }
+liveins:
+  - { reg: '$z0', virtual-reg: '%0' }
+  - { reg: '$z1', virtual-reg: '%1' }
+  - { reg: '$w0', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+  maxCallFrameSize: 0
+# CHECK-LABEL: name: insert_32i8_idx
+# CHECK: localFrameSize:  0
+stack:
+  - { id: 0, name: '', type: default, offset: 0, size: 32, alignment: 16,
+      stack-id: sve-vec, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+machineFunctionInfo: {}
+body:             |
+  bb.0 (%ir-block.0):
+    liveins: $z0, $z1, $w0
+
+    %2:gpr32 = COPY $w0
+    %1:zpr = COPY $z1
+    %0:zpr = COPY $z0
+    %5:ppr_3b = PTRUE_B 31
+    %6:gpr64sp = ADDXri %stack.0, 0, 0
+    ST1B_IMM %1, %5, %6, 1 :: (store unknown-size, align 16)
+    ST1B_IMM %0, %5, %stack.0, 0 :: (store unknown-size into %stack.0, align 16)
+    %7:zpr = LD1B_IMM %5, %6, 1 :: (load unknown-size from %stack.0 + 16, align 16)
+    %8:zpr = LD1B_IMM %5, %stack.0, 0 :: (load unknown-size from %stack.0, align 16)
+    $z0 = COPY %8
+    $z1 = COPY %7
+    RET_ReallyLR implicit $z0, implicit $z1
+
+...

From 590dd73c6ebdc9fe1314dfa5bda5c2367d866574 Mon Sep 17 00:00:00 2001
From: Piotr Sobczak <Piotr.Sobczak@amd.com>
Date: Thu, 23 Jul 2020 19:26:49 +0200
Subject: [PATCH 0163/1035] [AMDGPU] Make generating cache invalidating
 instructions optional

Summary:
D78800 skipped generating cache invalidating instrucions altogether
on AMDPAL. However, this is sometimes too restrictive - we want a
more flexible option to be able to toggle this behaviour on and off
while we work towards developing a correct implementation of the
alternative memory model.

Subscribers: arsenm, kzhuravl, jvesely, wdng, nhaehnle, yaxunl, dstuttard, tpr, t-tye, hiraditya, dexonsmith, kerbowa, llvm-commits

Tags: #llvm

Differential Revision: https://reviews.llvm.org/D84448
---
 llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp  |  8 +-
 .../CodeGen/AMDGPU/memory-legalizer-amdpal.ll | 86 +++++++++++++++----
 2 files changed, 75 insertions(+), 19 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp b/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
index 4e6c72ca20e28..21419aab1a43f 100644
--- a/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
+++ b/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
@@ -47,6 +47,10 @@ using namespace llvm::AMDGPU;
 #define DEBUG_TYPE "si-memory-legalizer"
 #define PASS_NAME "SI Memory Legalizer"
 
+static cl::opt<bool> AmdgcnSkipCacheInvalidations(
+    "amdgcn-skip-cache-invalidations", cl::init(false), cl::Hidden,
+    cl::desc("Use this to skip inserting cache invalidating instructions."));
+
 namespace {
 
 LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE();
@@ -254,7 +258,7 @@ class SICacheControl {
 
   IsaVersion IV;
 
-  /// Whether to insert cache invalidation instructions.
+  /// Whether to insert cache invalidating instructions.
   bool InsertCacheInv;
 
   SICacheControl(const GCNSubtarget &ST);
@@ -653,7 +657,7 @@ Optional<SIMemOpInfo> SIMemOpAccess::getAtomicCmpxchgOrRmwInfo(
 SICacheControl::SICacheControl(const GCNSubtarget &ST) {
   TII = ST.getInstrInfo();
   IV = getIsaVersion(ST.getCPU());
-  InsertCacheInv = !ST.isAmdPalOS();
+  InsertCacheInv = !AmdgcnSkipCacheInvalidations;
 }
 
 /* static */
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-amdpal.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-amdpal.ll
index 6fe24c1dfb948..b414c83374b89 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-amdpal.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-amdpal.ll
@@ -1,15 +1,23 @@
-; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx600 -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN,GCN9 %s
-; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN,GCN9 %s
-; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN,GCN9 %s
-; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN,GCN9 %s
-; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN,GCN10 %s
+; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx600 -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN,GCN9,CACHE_INV %s
+; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN,GCN9,CACHE_INV %s
+; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN,GCN9,CACHE_INV %s
+; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN,GCN9,CACHE_INV %s
+; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN,GCN10,CACHE_INV10 %s
+
+; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx900 -amdgcn-skip-cache-invalidations -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN,GCN9,SKIP_CACHE_INV %s
+; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx1010 -amdgcn-skip-cache-invalidations -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN,GCN10,SKIP_CACHE_INV %s
+
 
 ; FUNC-LABEL: {{^}}system_acquire:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
 ; GCN10:      s_waitcnt_vscnt null, 0x0
-; GCN-NOT:    buffer_wbinvl1{{$}}
+; CACHE_INV:  buffer_wbinvl1{{$}}
+; CACHE_INV10: buffer_gl0_inv
+; CACHE_INV10: buffer_gl1_inv
+; SKIP_CACHE_INV-NOT: buffer_wbinvl1{{$}}
+; SKIP_CACHE_INV-NOT: buffer_gl
 ; GCN:        s_endpgm
 define amdgpu_kernel void @system_acquire() {
 entry:
@@ -34,7 +42,11 @@ entry:
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
 ; GCN10:      s_waitcnt_vscnt null, 0x0
-; GCN-NOT:    buffer_wbinvl1{{$}}
+; CACHE_INV:  buffer_wbinvl1{{$}}
+; CACHE_INV10: buffer_gl0_inv
+; CACHE_INV10: buffer_gl1_inv
+; SKIP_CACHE_INV-NOT: buffer_wbinvl1{{$}}
+; SKIP_CACHE_INV-NOT: buffer_gl
 ; GCN:        s_endpgm
 define amdgpu_kernel void @system_acq_rel() {
 entry:
@@ -47,7 +59,11 @@ entry:
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
 ; GCN10:      s_waitcnt_vscnt null, 0x0
-; GCN-NOT:    buffer_wbinvl1{{$}}
+; CACHE_INV:  buffer_wbinvl1{{$}}
+; CACHE_INV10: buffer_gl0_inv
+; CACHE_INV10: buffer_gl1_inv
+; SKIP_CACHE_INV-NOT: buffer_wbinvl1{{$}}
+; SKIP_CACHE_INV-NOT: buffer_gl
 ; GCN:        s_endpgm
 define amdgpu_kernel void @system_seq_cst() {
 entry:
@@ -60,7 +76,11 @@ entry:
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_waitcnt vmcnt(0){{$}}
 ; GCN10:      s_waitcnt_vscnt null, 0x0
-; GCN-NOT:    buffer_wbinvl1{{$}}
+; CACHE_INV:  buffer_wbinvl1{{$}}
+; CACHE_INV10: buffer_gl0_inv
+; CACHE_INV10: buffer_gl1_inv
+; SKIP_CACHE_INV-NOT: buffer_wbinvl1{{$}}
+; SKIP_CACHE_INV-NOT: buffer_gl
 ; GCN:        s_endpgm
 define amdgpu_kernel void @system_one_as_acquire() {
 entry:
@@ -85,7 +105,11 @@ entry:
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_waitcnt vmcnt(0){{$}}
 ; GCN10:      s_waitcnt_vscnt null, 0x0
-; GCN-NOT:    buffer_wbinvl1{{$}}
+; CACHE_INV:  buffer_wbinvl1{{$}}
+; CACHE_INV10: buffer_gl0_inv
+; CACHE_INV10: buffer_gl1_inv
+; SKIP_CACHE_INV-NOT: buffer_wbinvl1{{$}}
+; SKIP_CACHE_INV-NOT: buffer_gl
 ; GCN:        s_endpgm
 define amdgpu_kernel void @system_one_as_acq_rel() {
 entry:
@@ -98,7 +122,11 @@ entry:
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_waitcnt vmcnt(0){{$}}
 ; GCN10:      s_waitcnt_vscnt null, 0x0
-; GCN-NOT:    buffer_wbinvl1{{$}}
+; CACHE_INV:    buffer_wbinvl1{{$}}
+; CACHE_INV10: buffer_gl0_inv
+; CACHE_INV10: buffer_gl1_inv
+; SKIP_CACHE_INV-NOT: buffer_wbinvl1{{$}}
+; SKIP_CACHE_INV-NOT: buffer_gl
 ; GCN:        s_endpgm
 define amdgpu_kernel void @system_one_as_seq_cst() {
 entry:
@@ -191,7 +219,11 @@ entry:
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
 ; GCN10:      s_waitcnt_vscnt null, 0x0
-; GCN-NOT:    buffer_wbinvl1{{$}}
+; CACHE_INV:  buffer_wbinvl1{{$}}
+; CACHE_INV10: buffer_gl0_inv
+; CACHE_INV10: buffer_gl1_inv
+; SKIP_CACHE_INV-NOT: buffer_wbinvl1{{$}}
+; SKIP_CACHE_INV-NOT: buffer_gl
 ; GCN:        s_endpgm
 define amdgpu_kernel void @agent_acquire() {
 entry:
@@ -216,7 +248,11 @@ entry:
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
 ; GCN10:      s_waitcnt_vscnt null, 0x0
-; GCN-NOT:    buffer_wbinvl1{{$}}
+; CACHE_INV:  buffer_wbinvl1{{$}}
+; CACHE_INV10: buffer_gl0_inv
+; CACHE_INV10: buffer_gl1_inv
+; SKIP_CACHE_INV-NOT: buffer_wbinvl1{{$}}
+; SKIP_CACHE_INV-NOT: buffer_gl
 ; GCN:        s_endpgm
 define amdgpu_kernel void @agent_acq_rel() {
 entry:
@@ -229,7 +265,11 @@ entry:
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
 ; GCN10:      s_waitcnt_vscnt null, 0x0
-; GCN-NOT:    buffer_wbinvl1{{$}}
+; CACHE_INV:  buffer_wbinvl1{{$}}
+; CACHE_INV10: buffer_gl0_inv
+; CACHE_INV10: buffer_gl1_inv
+; SKIP_CACHE_INV-NOT: buffer_wbinvl1{{$}}
+; SKIP_CACHE_INV-NOT: buffer_gl
 ; GCN:        s_endpgm
 define amdgpu_kernel void @agent_seq_cst() {
 entry:
@@ -242,7 +282,11 @@ entry:
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_waitcnt vmcnt(0){{$}}
 ; GCN10:      s_waitcnt_vscnt null, 0x0
-; GCN-NOT:    buffer_wbinvl1{{$}}
+; CACHE_INV:  buffer_wbinvl1{{$}}
+; CACHE_INV10: buffer_gl0_inv
+; CACHE_INV10: buffer_gl1_inv
+; SKIP_CACHE_INV-NOT: buffer_wbinvl1{{$}}
+; SKIP_CACHE_INV-NOT: buffer_gl
 ; GCN:        s_endpgm
 define amdgpu_kernel void @agent_one_as_acquire() {
 entry:
@@ -267,7 +311,11 @@ entry:
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_waitcnt vmcnt(0){{$}}
 ; GCN10:      s_waitcnt_vscnt null, 0x0
-; GCN-NOT:    buffer_wbinvl1{{$}}
+; CACHE_INV:  buffer_wbinvl1{{$}}
+; CACHE_INV10: buffer_gl0_inv
+; CACHE_INV10: buffer_gl1_inv
+; SKIP_CACHE_INV-NOT: buffer_wbinvl1{{$}}
+; SKIP_CACHE_INV-NOT: buffer_gl
 ; GCN:        s_endpgm
 define amdgpu_kernel void @agent_one_as_acq_rel() {
 entry:
@@ -280,7 +328,11 @@ entry:
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_waitcnt vmcnt(0){{$}}
 ; GCN10:      s_waitcnt_vscnt null, 0x0
-; GCN-NOT:    buffer_wbinvl1{{$}}
+; CACHE_INV:  buffer_wbinvl1{{$}}
+; CACHE_INV10: buffer_gl0_inv
+; CACHE_INV10: buffer_gl1_inv
+; SKIP_CACHE_INV-NOT: buffer_wbinvl1{{$}}
+; SKIP_CACHE_INV-NOT: buffer_gl
 ; GCN:        s_endpgm
 define amdgpu_kernel void @agent_one_as_seq_cst() {
 entry:

From 36618274f3e2cdea98cd8202204b8ad2913aae46 Mon Sep 17 00:00:00 2001
From: George Mitenkov <georgemitenk0v@gmail.com>
Date: Mon, 27 Jul 2020 10:19:48 +0300
Subject: [PATCH 0164/1035] [MLIR][LLVMDialect] Added volatile and nontemporal
 attributes to load/store

This patch introduces 2 new optional attributes to `llvm.load`
and `llvm.store` ops: `volatile` and `nontemporal`. These attributes
are translated into proper LLVM as a `volatile` marker and a metadata node
respectively. They are also helpful with SPIR-V to LLVM dialect conversion
since they are the mappings for `Volatile` and `NonTemporal` Memory Operands.

Reviewed By: ftynse

Differential Revision: https://reviews.llvm.org/D84396
---
 mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td | 62 ++++++++++++---------
 mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp  | 58 ++++++++++++++++---
 mlir/test/Target/llvmir.mlir                | 29 ++++++++++
 3 files changed, 115 insertions(+), 34 deletions(-)

diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
index 5322e243427a9..29d7fd930030a 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
@@ -235,6 +235,19 @@ class MemoryOpWithAlignmentBase {
   }];
 }
 
+// Code definition that is used for nontemporal metadata creation.
+class MemoryOpWithAlignmentAndAttributes : MemoryOpWithAlignmentBase {
+  code setNonTemporalMetadataCode = [{
+    if ($nontemporal) {
+      llvm::Module *module = builder.GetInsertBlock()->getModule();
+      llvm::MDNode *metadata = llvm::MDNode::get(
+          inst->getContext(), llvm::ConstantAsMetadata::get(
+              builder.getInt32(1)));
+      inst->setMetadata(module->getMDKindID("nontemporal"), metadata);
+    }
+  }];
+}
+
 // Memory-related operations.
 def LLVM_AllocaOp :
     MemoryOpWithAlignmentBase,
@@ -266,52 +279,49 @@ def LLVM_GEPOp : LLVM_OneResultOp<"getelementptr", [NoSideEffect]>,
   }];
 }
 def LLVM_LoadOp :
-    MemoryOpWithAlignmentBase,
+    MemoryOpWithAlignmentAndAttributes,
     LLVM_OneResultOp<"load">,
-    Arguments<(ins LLVM_Type:$addr, OptionalAttr<I64Attr>:$alignment)> {
+    Arguments<(ins LLVM_Type:$addr,
+                   OptionalAttr<I64Attr>:$alignment,
+                   UnitAttr:$volatile_,
+                   UnitAttr:$nontemporal)> {
   string llvmBuilder = [{
-    auto *inst = builder.CreateLoad($addr);
-  }] # setAlignmentCode # [{
+    auto *inst = builder.CreateLoad($addr, $volatile_);
+  }] # setAlignmentCode # setNonTemporalMetadataCode # [{
     $res = inst;
   }];
   let builders = [OpBuilder<
-    "OpBuilder &b, OperationState &result, Value addr, unsigned alignment = 0",
+    "OpBuilder &b, OperationState &result, Value addr, "
+    "unsigned alignment = 0, bool isVolatile = false, "
+    "bool isNonTemporal = false",
     [{
       auto type = addr.getType().cast<LLVM::LLVMType>().getPointerElementTy();
-      build(b, result, type, addr, alignment);
+      build(b, result, type, addr, alignment, isVolatile, isNonTemporal);
     }]>,
     OpBuilder<
     "OpBuilder &b, OperationState &result, Type t, Value addr, "
-    "unsigned alignment = 0",
-    [{
-      if (alignment == 0)
-        return build(b, result, t, addr, IntegerAttr());
-      build(b, result, t, addr, b.getI64IntegerAttr(alignment));
-    }]>];
+    "unsigned alignment = 0, bool isVolatile = false, "
+    "bool isNonTemporal = false">];
   let parser = [{ return parseLoadOp(parser, result); }];
   let printer = [{ printLoadOp(p, *this); }];
   let verifier = alignmentVerifierCode;
 }
 def LLVM_StoreOp :
-    MemoryOpWithAlignmentBase,
+    MemoryOpWithAlignmentAndAttributes,
     LLVM_ZeroResultOp<"store">,
     Arguments<(ins LLVM_Type:$value,
                    LLVM_Type:$addr,
-                   OptionalAttr<I64Attr>:$alignment)> {
+                   OptionalAttr<I64Attr>:$alignment,
+                   UnitAttr:$volatile_,
+                   UnitAttr:$nontemporal)> {
   string llvmBuilder = [{
-    auto *inst = builder.CreateStore($value, $addr);
-  }] # setAlignmentCode;
-  let builders = [
-    OpBuilder<
+    auto *inst = builder.CreateStore($value, $addr, $volatile_);
+  }] # setAlignmentCode # setNonTemporalMetadataCode;
+  let builders = [OpBuilder<
     "OpBuilder &b, OperationState &result, Value value, Value addr, "
-    "unsigned alignment = 0",
-    [{
-      if (alignment == 0)
-        return build(b, result, ArrayRef<Type>{}, value, addr, IntegerAttr());
-      build(b, result, ArrayRef<Type>{}, value, addr, 
-            b.getI64IntegerAttr(alignment));
-    }]
-  >];
+    "unsigned alignment = 0, bool isVolatile = false, "
+    "bool isNonTemporal = false">
+    ];
   let parser = [{ return parseStoreOp(parser, result); }];
   let printer = [{ printStoreOp(p, *this); }];
   let verifier = alignmentVerifierCode;
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
index f448427099a44..cf7a5d926528f 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
@@ -31,6 +31,9 @@
 using namespace mlir;
 using namespace mlir::LLVM;
 
+static constexpr const char kVolatileAttrName[] = "volatile_";
+static constexpr const char kNonTemporalAttrName[] = "nontemporal";
+
 #include "mlir/Dialect/LLVMIR/LLVMOpsEnums.cpp.inc"
 
 //===----------------------------------------------------------------------===//
@@ -178,12 +181,28 @@ CondBrOp::getMutableSuccessorOperands(unsigned index) {
 }
 
 //===----------------------------------------------------------------------===//
-// Printing/parsing for LLVM::LoadOp.
+// Builder, printer and parser for for LLVM::LoadOp.
 //===----------------------------------------------------------------------===//
 
+void LoadOp::build(OpBuilder &builder, OperationState &result, Type t,
+                   Value addr, unsigned alignment, bool isVolatile,
+                   bool isNonTemporal) {
+  result.addOperands(addr);
+  result.addTypes(t);
+  if (isVolatile)
+    result.addAttribute(kVolatileAttrName, builder.getUnitAttr());
+  if (isNonTemporal)
+    result.addAttribute(kNonTemporalAttrName, builder.getUnitAttr());
+  if (alignment != 0)
+    result.addAttribute("alignment", builder.getI64IntegerAttr(alignment));
+}
+
 static void printLoadOp(OpAsmPrinter &p, LoadOp &op) {
-  p << op.getOperationName() << ' ' << op.addr();
-  p.printOptionalAttrDict(op.getAttrs());
+  p << op.getOperationName() << ' ';
+  if (op.volatile_())
+    p << "volatile ";
+  p << op.addr();
+  p.printOptionalAttrDict(op.getAttrs(), {kVolatileAttrName});
   p << " : " << op.addr().getType();
 }
 
@@ -201,12 +220,15 @@ static Type getLoadStoreElementType(OpAsmParser &parser, Type type,
   return llvmTy.getPointerElementTy();
 }
 
-// <operation> ::= `llvm.load` ssa-use attribute-dict? `:` type
+// <operation> ::= `llvm.load` `volatile` ssa-use attribute-dict? `:` type
 static ParseResult parseLoadOp(OpAsmParser &parser, OperationState &result) {
   OpAsmParser::OperandType addr;
   Type type;
   llvm::SMLoc trailingTypeLoc;
 
+  if (succeeded(parser.parseOptionalKeyword("volatile")))
+    result.addAttribute(kVolatileAttrName, parser.getBuilder().getUnitAttr());
+
   if (parser.parseOperand(addr) ||
       parser.parseOptionalAttrDict(result.attributes) || parser.parseColon() ||
       parser.getCurrentLocation(&trailingTypeLoc) || parser.parseType(type) ||
@@ -220,21 +242,41 @@ static ParseResult parseLoadOp(OpAsmParser &parser, OperationState &result) {
 }
 
 //===----------------------------------------------------------------------===//
-// Printing/parsing for LLVM::StoreOp.
+// Builder, printer and parser for LLVM::StoreOp.
 //===----------------------------------------------------------------------===//
 
+void StoreOp::build(OpBuilder &builder, OperationState &result, Value value,
+                    Value addr, unsigned alignment, bool isVolatile,
+                    bool isNonTemporal) {
+  result.addOperands({value, addr});
+  result.addTypes(ArrayRef<Type>{});
+  if (isVolatile)
+    result.addAttribute(kVolatileAttrName, builder.getUnitAttr());
+  if (isNonTemporal)
+    result.addAttribute(kNonTemporalAttrName, builder.getUnitAttr());
+  if (alignment != 0)
+    result.addAttribute("alignment", builder.getI64IntegerAttr(alignment));
+}
+
 static void printStoreOp(OpAsmPrinter &p, StoreOp &op) {
-  p << op.getOperationName() << ' ' << op.value() << ", " << op.addr();
-  p.printOptionalAttrDict(op.getAttrs());
+  p << op.getOperationName() << ' ';
+  if (op.volatile_())
+    p << "volatile ";
+  p << op.value() << ", " << op.addr();
+  p.printOptionalAttrDict(op.getAttrs(), {kVolatileAttrName});
   p << " : " << op.addr().getType();
 }
 
-// <operation> ::= `llvm.store` ssa-use `,` ssa-use attribute-dict? `:` type
+// <operation> ::= `llvm.store` `volatile` ssa-use `,` ssa-use
+//                 attribute-dict? `:` type
 static ParseResult parseStoreOp(OpAsmParser &parser, OperationState &result) {
   OpAsmParser::OperandType addr, value;
   Type type;
   llvm::SMLoc trailingTypeLoc;
 
+  if (succeeded(parser.parseOptionalKeyword("volatile")))
+    result.addAttribute(kVolatileAttrName, parser.getBuilder().getUnitAttr());
+
   if (parser.parseOperand(value) || parser.parseComma() ||
       parser.parseOperand(addr) ||
       parser.parseOptionalAttrDict(result.attributes) || parser.parseColon() ||
diff --git a/mlir/test/Target/llvmir.mlir b/mlir/test/Target/llvmir.mlir
index 954b5b1345412..d6180cbf18494 100644
--- a/mlir/test/Target/llvmir.mlir
+++ b/mlir/test/Target/llvmir.mlir
@@ -1266,3 +1266,32 @@ llvm.func @cond_br_weights(%cond : !llvm.i1, %arg0 : !llvm.i32,  %arg1 : !llvm.i
 }
 
 // CHECK: ![[NODE]] = !{!"branch_weights", i32 5, i32 10}
+
+// -----
+
+llvm.func @volatile_store_and_load() {
+  %val = llvm.mlir.constant(5 : i32) : !llvm.i32
+  %size = llvm.mlir.constant(1 : i64) : !llvm.i64
+  %0 = llvm.alloca %size x !llvm.i32 : (!llvm.i64) -> (!llvm<"i32*">)
+  // CHECK: store volatile i32 5, i32* %{{.*}}
+  llvm.store volatile %val, %0 : !llvm<"i32*">
+  // CHECK: %{{.*}} = load volatile i32, i32* %{{.*}}
+  %1 = llvm.load volatile %0: !llvm<"i32*">
+  llvm.return
+}
+
+// -----
+
+// Check that nontemporal attribute is exported as metadata node.
+llvm.func @nontemoral_store_and_load() {
+  %val = llvm.mlir.constant(5 : i32) : !llvm.i32
+  %size = llvm.mlir.constant(1 : i64) : !llvm.i64
+  %0 = llvm.alloca %size x !llvm.i32 : (!llvm.i64) -> (!llvm<"i32*">)
+  // CHECK: !nontemporal ![[NODE:[0-9]+]]
+  llvm.store %val, %0 {nontemporal} : !llvm<"i32*">
+  // CHECK: !nontemporal ![[NODE]]
+  %1 = llvm.load %0 {nontemporal} : !llvm<"i32*">
+  llvm.return
+}
+
+// CHECK: ![[NODE]] = !{i32 1}

From 81d7ebaf5c369d42b77f9e3e47e2ac22c306ec04 Mon Sep 17 00:00:00 2001
From: Pavel Labath <pavel@labath.sk>
Date: Fri, 24 Jul 2020 14:49:17 +0200
Subject: [PATCH 0165/1035] [lldb/Utility] Fix a bug in
 RangeMap::CombineConsecutiveRanges

The function didn't combine a large entry which overlapped several other
entries, if those other entries were not overlapping among each other.

E.g., (0,20),(5,6),(10,11) produced (0,20),(10,11)

Now it just produced (0,20).
---
 lldb/include/lldb/Utility/RangeMap.h    | 54 ++++++++++---------------
 lldb/unittests/Utility/RangeMapTest.cpp | 26 ++++++++++++
 2 files changed, 47 insertions(+), 33 deletions(-)

diff --git a/lldb/include/lldb/Utility/RangeMap.h b/lldb/include/lldb/Utility/RangeMap.h
index fb24c5a434792..118fdfd85fa9f 100644
--- a/lldb/include/lldb/Utility/RangeMap.h
+++ b/lldb/include/lldb/Utility/RangeMap.h
@@ -194,41 +194,25 @@ template <typename B, typename S, unsigned N = 0> class RangeVector {
 #ifdef ASSERT_RANGEMAP_ARE_SORTED
     assert(IsSorted());
 #endif
-    // Can't combine if ranges if we have zero or one range
-    if (m_entries.size() > 1) {
-      // The list should be sorted prior to calling this function
-      typename Collection::iterator pos;
-      typename Collection::iterator end;
-      typename Collection::iterator prev;
-      bool can_combine = false;
-      // First we determine if we can combine any of the Entry objects so we
-      // don't end up allocating and making a new collection for no reason
-      for (pos = m_entries.begin(), end = m_entries.end(), prev = end;
-           pos != end; prev = pos++) {
-        if (prev != end && prev->DoesAdjoinOrIntersect(*pos)) {
-          can_combine = true;
-          break;
-        }
-      }
+    auto first_intersect = std::adjacent_find(
+        m_entries.begin(), m_entries.end(), [](const Entry &a, const Entry &b) {
+          return a.DoesAdjoinOrIntersect(b);
+        });
+    if (first_intersect == m_entries.end())
+      return;
 
-      // We we can combine at least one entry, then we make a new collection
-      // and populate it accordingly, and then swap it into place.
-      if (can_combine) {
-        Collection minimal_ranges;
-        for (pos = m_entries.begin(), end = m_entries.end(), prev = end;
-             pos != end; prev = pos++) {
-          if (prev != end && prev->DoesAdjoinOrIntersect(*pos))
-            minimal_ranges.back().SetRangeEnd(
-                std::max<BaseType>(prev->GetRangeEnd(), pos->GetRangeEnd()));
-          else
-            minimal_ranges.push_back(*pos);
-        }
-        // Use the swap technique in case our new vector is much smaller. We
-        // must swap when using the STL because std::vector objects never
-        // release or reduce the memory once it has been allocated/reserved.
-        m_entries.swap(minimal_ranges);
-      }
+    // We we can combine at least one entry, then we make a new collection and
+    // populate it accordingly, and then swap it into place.
+    auto pos = std::next(first_intersect);
+    Collection minimal_ranges(m_entries.begin(), pos);
+    for (; pos != m_entries.end(); ++pos) {
+      Entry &back = minimal_ranges.back();
+      if (back.DoesAdjoinOrIntersect(*pos))
+        back.SetRangeEnd(std::max(back.GetRangeEnd(), pos->GetRangeEnd()));
+      else
+        minimal_ranges.push_back(*pos);
     }
+    m_entries.swap(minimal_ranges);
   }
 
   BaseType GetMinRangeBase(BaseType fail_value) const {
@@ -353,6 +337,10 @@ template <typename B, typename S, unsigned N = 0> class RangeVector {
     return nullptr;
   }
 
+  using const_iterator = typename Collection::const_iterator;
+  const_iterator begin() const { return m_entries.begin(); }
+  const_iterator end() const { return m_entries.end(); }
+
 protected:
   void CombinePrevAndNext(typename Collection::iterator pos) {
     // Check if the prev or next entries in case they need to be unioned with
diff --git a/lldb/unittests/Utility/RangeMapTest.cpp b/lldb/unittests/Utility/RangeMapTest.cpp
index 8a243b6562181..97432dca983d3 100644
--- a/lldb/unittests/Utility/RangeMapTest.cpp
+++ b/lldb/unittests/Utility/RangeMapTest.cpp
@@ -12,6 +12,32 @@
 
 using namespace lldb_private;
 
+TEST(RangeVector, CombineConsecutiveRanges) {
+  using RangeVector = RangeVector<uint32_t, uint32_t>;
+  using Entry = RangeVector::Entry;
+
+  RangeVector V;
+  V.Append(0, 1);
+  V.Append(5, 1);
+  V.Append(6, 1);
+  V.Append(10, 9);
+  V.Append(15, 1);
+  V.Append(20, 9);
+  V.Append(21, 9);
+  V.Sort();
+  V.CombineConsecutiveRanges();
+  EXPECT_THAT(V, testing::ElementsAre(Entry(0, 1), Entry(5, 2), Entry(10, 9),
+                                      Entry(20, 10)));
+
+  V.Clear();
+  V.Append(0, 20);
+  V.Append(5, 1);
+  V.Append(10, 1);
+  V.Sort();
+  V.CombineConsecutiveRanges();
+  EXPECT_THAT(V, testing::ElementsAre(Entry(0, 20)));
+}
+
 using RangeDataVectorT = RangeDataVector<uint32_t, uint32_t, uint32_t>;
 using EntryT = RangeDataVectorT::Entry;
 

From e89414f4060d3ff2afcd1c89fc028d61270c4d22 Mon Sep 17 00:00:00 2001
From: Pavel Labath <pavel@labath.sk>
Date: Mon, 20 Jul 2020 16:42:01 +0200
Subject: [PATCH 0166/1035] [lldb/Utility] Clean up Scalar constructors

- move initialization to initializer lists
- make desctructor non-virtual (nothing else is)
- fix long double constructor so that it actually works
---
 lldb/include/lldb/Utility/Scalar.h    | 53 +++++++++++----------------
 lldb/source/Utility/Scalar.cpp        |  4 --
 lldb/unittests/Utility/ScalarTest.cpp |  1 +
 3 files changed, 22 insertions(+), 36 deletions(-)

diff --git a/lldb/include/lldb/Utility/Scalar.h b/lldb/include/lldb/Utility/Scalar.h
index 524b715230744..1dbcf80bfd89c 100644
--- a/lldb/include/lldb/Utility/Scalar.h
+++ b/lldb/include/lldb/Utility/Scalar.h
@@ -60,41 +60,30 @@ class Scalar {
   };
 
   // Constructors and Destructors
-  Scalar();
-  Scalar(int v) : m_type(e_sint), m_float(static_cast<float>(0)) {
-    m_integer = llvm::APInt(sizeof(int) * 8, v, true);
-  }
-  Scalar(unsigned int v) : m_type(e_uint), m_float(static_cast<float>(0)) {
-    m_integer = llvm::APInt(sizeof(int) * 8, v);
-  }
-  Scalar(long v) : m_type(e_slong), m_float(static_cast<float>(0)) {
-    m_integer = llvm::APInt(sizeof(long) * 8, v, true);
-  }
-  Scalar(unsigned long v) : m_type(e_ulong), m_float(static_cast<float>(0)) {
-    m_integer = llvm::APInt(sizeof(long) * 8, v);
-  }
-  Scalar(long long v) : m_type(e_slonglong), m_float(static_cast<float>(0)) {
-    m_integer = llvm::APInt(sizeof(long long) * 8, v, true);
-  }
+  Scalar() : m_type(e_void), m_float(0.0f) {}
+  Scalar(int v)
+      : m_type(e_sint), m_integer(sizeof(v) * 8, v, true), m_float(0.0f) {}
+  Scalar(unsigned int v)
+      : m_type(e_uint), m_integer(sizeof(v) * 8, v, false), m_float(0.0f) {}
+  Scalar(long v)
+      : m_type(e_slong), m_integer(sizeof(v) * 8, v, true), m_float(0.0f) {}
+  Scalar(unsigned long v)
+      : m_type(e_ulong), m_integer(sizeof(v) * 8, v, false), m_float(0.0f) {}
+  Scalar(long long v)
+      : m_type(e_slonglong), m_integer(sizeof(v) * 8, v, true), m_float(0.0f) {}
   Scalar(unsigned long long v)
-      : m_type(e_ulonglong), m_float(static_cast<float>(0)) {
-    m_integer = llvm::APInt(sizeof(long long) * 8, v);
-  }
-  Scalar(float v) : m_type(e_float), m_float(v) { m_float = llvm::APFloat(v); }
-  Scalar(double v) : m_type(e_double), m_float(v) {
-    m_float = llvm::APFloat(v);
+      : m_type(e_ulonglong), m_integer(sizeof(v) * 8, v, false), m_float(0.0f) {
   }
-  Scalar(long double v)
-      : m_type(e_long_double),
-        m_float(llvm::APFloat::x87DoubleExtended(),
-                llvm::APInt(BITWIDTH_INT128, NUM_OF_WORDS_INT128,
-                            (reinterpret_cast<type128 *>(&v))->x)) {}
-  Scalar(llvm::APInt v) : m_type(), m_float(static_cast<float>(0)) {
-    m_integer = llvm::APInt(std::move(v));
-    m_type = GetBestTypeForBitSize(m_integer.getBitWidth(), true);
+  Scalar(float v) : m_type(e_float), m_float(v) {}
+  Scalar(double v) : m_type(e_double), m_float(v) {}
+  Scalar(long double v) : m_type(e_long_double), m_float(double(v)) {
+    bool ignore;
+    m_float.convert(llvm::APFloat::x87DoubleExtended(),
+                    llvm::APFloat::rmNearestTiesToEven, &ignore);
   }
-  // Scalar(const RegisterValue& reg_value);
-  virtual ~Scalar();
+  Scalar(llvm::APInt v)
+      : m_type(GetBestTypeForBitSize(v.getBitWidth(), true)),
+        m_integer(std::move(v)), m_float(0.0f) {}
 
   /// Return the most efficient Scalar::Type for the requested bit size.
   static Type GetBestTypeForBitSize(size_t bit_size, bool sign);
diff --git a/lldb/source/Utility/Scalar.cpp b/lldb/source/Utility/Scalar.cpp
index 27d5b3b88d338..9309f8d662da8 100644
--- a/lldb/source/Utility/Scalar.cpp
+++ b/lldb/source/Utility/Scalar.cpp
@@ -126,8 +126,6 @@ static Scalar::Type PromoteToMaxType(
   return Scalar::e_void;
 }
 
-Scalar::Scalar() : m_type(e_void), m_float(static_cast<float>(0)) {}
-
 bool Scalar::GetData(DataExtractor &data, size_t limit_byte_size) const {
   size_t byte_size = GetByteSize();
   if (byte_size == 0) {
@@ -232,8 +230,6 @@ void Scalar::GetValue(Stream *s, bool show_type) const {
   }
 }
 
-Scalar::~Scalar() = default;
-
 Scalar::Type Scalar::GetBestTypeForBitSize(size_t bit_size, bool sign) {
   // Scalar types are always host types, hence the sizeof().
   if (sign) {
diff --git a/lldb/unittests/Utility/ScalarTest.cpp b/lldb/unittests/Utility/ScalarTest.cpp
index f6bc6a404c159..70ce0a81627d2 100644
--- a/lldb/unittests/Utility/ScalarTest.cpp
+++ b/lldb/unittests/Utility/ScalarTest.cpp
@@ -92,6 +92,7 @@ TEST(ScalarTest, Getters) {
   CheckConversion<unsigned long long>(0x8765432112345678ull);
   CheckConversion<float>(42.25f);
   CheckConversion<double>(42.25);
+  CheckConversion<long double>(42.25L);
 
   EXPECT_EQ(APInt(128, 1) << 70, Scalar(std::pow(2.0f, 70.0f)).SInt128(APInt()));
   EXPECT_EQ(APInt(128, -1, true) << 70,

From e1eacf27c6f4ba82b8da34e62f62b44b81ffa316 Mon Sep 17 00:00:00 2001
From: Juneyoung Lee <aqjune@gmail.com>
Date: Mon, 27 Jul 2020 17:07:27 +0900
Subject: [PATCH 0167/1035] [InstCombine] Fold freeze into phi if one operand
 is not undef

 This patch adds folding freeze into phi if it has only one operand to target.

Reviewed By: nikic

Differential Revision: https://reviews.llvm.org/D84601
---
 .../InstCombine/InstructionCombining.cpp      | 19 ++++++++++++++-
 .../test/Transforms/InstCombine/freeze-phi.ll | 24 +++++++++----------
 2 files changed, 30 insertions(+), 13 deletions(-)

diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
index fdf0aaf9b176d..711be57a0bafa 100644
--- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
@@ -1048,7 +1048,9 @@ Instruction *InstCombinerImpl::foldOpIntoPhi(Instruction &I, PHINode *PN) {
   BasicBlock *NonConstBB = nullptr;
   for (unsigned i = 0; i != NumPHIValues; ++i) {
     Value *InVal = PN->getIncomingValue(i);
-    if (isa<Constant>(InVal) && !isa<ConstantExpr>(InVal))
+    // If I is a freeze instruction, count undef as a non-constant.
+    if (isa<Constant>(InVal) && !isa<ConstantExpr>(InVal) &&
+        (!isa<FreezeInst>(I) || isGuaranteedNotToBeUndefOrPoison(InVal)))
       continue;
 
     if (isa<PHINode>(InVal)) return nullptr;  // Itself a phi.
@@ -1141,6 +1143,15 @@ Instruction *InstCombinerImpl::foldOpIntoPhi(Instruction &I, PHINode *PN) {
                                              Builder);
       NewPN->addIncoming(InV, PN->getIncomingBlock(i));
     }
+  } else if (auto *FI = dyn_cast<FreezeInst>(&I)) {
+    for (unsigned i = 0; i != NumPHIValues; ++i) {
+      Value *InV;
+      if (NonConstBB == PN->getIncomingBlock(i))
+        InV = Builder.CreateFreeze(PN->getIncomingValue(i), "phi.fr");
+      else
+        InV = PN->getIncomingValue(i);
+      NewPN->addIncoming(InV, PN->getIncomingBlock(i));
+    }
   } else {
     CastInst *CI = cast<CastInst>(&I);
     Type *RetTy = CI->getType();
@@ -3370,6 +3381,12 @@ Instruction *InstCombinerImpl::visitFreeze(FreezeInst &I) {
   if (Value *V = SimplifyFreezeInst(Op0, SQ.getWithInstruction(&I)))
     return replaceInstUsesWith(I, V);
 
+  // freeze (phi const, x) --> phi const, (freeze x)
+  if (auto *PN = dyn_cast<PHINode>(Op0)) {
+    if (Instruction *NV = foldOpIntoPhi(I, PN))
+      return NV;
+  }
+
   return nullptr;
 }
 
diff --git a/llvm/test/Transforms/InstCombine/freeze-phi.ll b/llvm/test/Transforms/InstCombine/freeze-phi.ll
index 430c2d2e8fe66..b8dde691a11c1 100644
--- a/llvm/test/Transforms/InstCombine/freeze-phi.ll
+++ b/llvm/test/Transforms/InstCombine/freeze-phi.ll
@@ -51,11 +51,11 @@ define <2 x i32> @vec_undef(i1 %cond) {
 ; CHECK:       A:
 ; CHECK-NEXT:    br label [[C:%.*]]
 ; CHECK:       B:
+; CHECK-NEXT:    [[PHI_FR:%.*]] = freeze <2 x i32> <i32 2, i32 undef>
 ; CHECK-NEXT:    br label [[C]]
 ; CHECK:       C:
-; CHECK-NEXT:    [[Y:%.*]] = phi <2 x i32> [ <i32 0, i32 1>, [[A]] ], [ <i32 2, i32 undef>, [[B]] ]
-; CHECK-NEXT:    [[Y_FR:%.*]] = freeze <2 x i32> [[Y]]
-; CHECK-NEXT:    ret <2 x i32> [[Y_FR]]
+; CHECK-NEXT:    [[Y:%.*]] = phi <2 x i32> [ <i32 0, i32 1>, [[A]] ], [ [[PHI_FR]], [[B]] ]
+; CHECK-NEXT:    ret <2 x i32> [[Y]]
 ;
   br i1 %cond, label %A, label %B
 A:
@@ -74,11 +74,11 @@ define i32 @one(i1 %cond, i32 %x) {
 ; CHECK:       A:
 ; CHECK-NEXT:    br label [[C:%.*]]
 ; CHECK:       B:
+; CHECK-NEXT:    [[PHI_FR:%.*]] = freeze i32 [[X:%.*]]
 ; CHECK-NEXT:    br label [[C]]
 ; CHECK:       C:
-; CHECK-NEXT:    [[Y:%.*]] = phi i32 [ 0, [[A]] ], [ [[X:%.*]], [[B]] ]
-; CHECK-NEXT:    [[Y_FR:%.*]] = freeze i32 [[Y]]
-; CHECK-NEXT:    ret i32 [[Y_FR]]
+; CHECK-NEXT:    [[Y:%.*]] = phi i32 [ 0, [[A]] ], [ [[PHI_FR]], [[B]] ]
+; CHECK-NEXT:    ret i32 [[Y]]
 ;
   br i1 %cond, label %A, label %B
 A:
@@ -154,15 +154,15 @@ define i32 @one_undef(i8 %cond) {
 ; CHECK-NEXT:    i8 1, label [[C:%.*]]
 ; CHECK-NEXT:    ]
 ; CHECK:       A:
+; CHECK-NEXT:    [[PHI_FR:%.*]] = freeze i32 undef
 ; CHECK-NEXT:    br label [[D:%.*]]
 ; CHECK:       B:
 ; CHECK-NEXT:    br label [[D]]
 ; CHECK:       C:
 ; CHECK-NEXT:    br label [[D]]
 ; CHECK:       D:
-; CHECK-NEXT:    [[Y:%.*]] = phi i32 [ undef, [[A]] ], [ 32, [[B]] ], [ 0, [[C]] ]
-; CHECK-NEXT:    [[Y_FR:%.*]] = freeze i32 [[Y]]
-; CHECK-NEXT:    ret i32 [[Y_FR]]
+; CHECK-NEXT:    [[Y:%.*]] = phi i32 [ [[PHI_FR]], [[A]] ], [ 32, [[B]] ], [ 0, [[C]] ]
+; CHECK-NEXT:    ret i32 [[Y]]
 ;
   switch i8 %cond, label %A [
   i8 0, label %B
@@ -189,15 +189,15 @@ define i32 @one_constexpr(i8 %cond, i32 %x) {
 ; CHECK-NEXT:    i8 1, label [[C:%.*]]
 ; CHECK-NEXT:    ]
 ; CHECK:       A:
+; CHECK-NEXT:    [[PHI_FR:%.*]] = freeze i32 ptrtoint (i8* getelementptr inbounds (i8, i8* @glb, i64 2) to i32)
 ; CHECK-NEXT:    br label [[D:%.*]]
 ; CHECK:       B:
 ; CHECK-NEXT:    br label [[D]]
 ; CHECK:       C:
 ; CHECK-NEXT:    br label [[D]]
 ; CHECK:       D:
-; CHECK-NEXT:    [[Y:%.*]] = phi i32 [ ptrtoint (i8* getelementptr inbounds (i8, i8* @glb, i64 2) to i32), [[A]] ], [ 32, [[B]] ], [ 0, [[C]] ]
-; CHECK-NEXT:    [[Y_FR:%.*]] = freeze i32 [[Y]]
-; CHECK-NEXT:    ret i32 [[Y_FR]]
+; CHECK-NEXT:    [[Y:%.*]] = phi i32 [ [[PHI_FR]], [[A]] ], [ 32, [[B]] ], [ 0, [[C]] ]
+; CHECK-NEXT:    ret i32 [[Y]]
 ;
   switch i8 %cond, label %A [
   i8 0, label %B

From d9bbe85943f6322e8fa1e85f72e53dd579c14a2f Mon Sep 17 00:00:00 2001
From: Guillaume Chatelet <gchatelet@google.com>
Date: Mon, 27 Jul 2020 08:16:28 +0000
Subject: [PATCH 0168/1035] [Alignment][NFC] Update Bitcodewriter to use Align

Differential Revision: https://reviews.llvm.org/D83533
---
 llvm/include/llvm/Bitstream/BitcodeCommon.h | 30 ++++++++++++++++
 llvm/lib/Bitcode/Reader/BitcodeReader.cpp   | 20 +++++------
 llvm/lib/Bitcode/Writer/BitcodeWriter.cpp   | 40 ++++++++++++---------
 3 files changed, 62 insertions(+), 28 deletions(-)
 create mode 100644 llvm/include/llvm/Bitstream/BitcodeCommon.h

diff --git a/llvm/include/llvm/Bitstream/BitcodeCommon.h b/llvm/include/llvm/Bitstream/BitcodeCommon.h
new file mode 100644
index 0000000000000..84b35987c4a90
--- /dev/null
+++ b/llvm/include/llvm/Bitstream/BitcodeCommon.h
@@ -0,0 +1,30 @@
+//===- BitcodeCommon.h - Common code for encode/decode   --------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This header defines common code to be used by BitcodeWriter and
+// BitcodeReader.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_BITSTREAM_BITCODECOMMON_H
+#define LLVM_BITSTREAM_BITCODECOMMON_H
+
+#include "llvm/ADT/Bitfields.h"
+
+namespace llvm {
+
+struct AllocaPackedValues {
+  using Align = Bitfield::Element<unsigned, 0, 5>;
+  using UsedWithInAlloca = Bitfield::Element<bool, Align::NextBit, 1>;
+  using ExplicitType = Bitfield::Element<bool, UsedWithInAlloca::NextBit, 1>;
+  using SwiftError = Bitfield::Element<bool, ExplicitType::NextBit, 1>;
+};
+
+} // namespace llvm
+
+#endif // LLVM_BITSTREAM_BITCODECOMMON_H
diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index 9632b5700e8af..908e70b2789da 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -20,8 +20,9 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/Triple.h"
 #include "llvm/ADT/Twine.h"
-#include "llvm/Bitstream/BitstreamReader.h"
 #include "llvm/Bitcode/LLVMBitCodes.h"
+#include "llvm/Bitstream/BitcodeCommon.h"
+#include "llvm/Bitstream/BitstreamReader.h"
 #include "llvm/Config/llvm-config.h"
 #include "llvm/IR/Argument.h"
 #include "llvm/IR/Attributes.h"
@@ -4813,17 +4814,13 @@ Error BitcodeReader::parseFunctionBody(Function *F) {
     case bitc::FUNC_CODE_INST_ALLOCA: { // ALLOCA: [instty, opty, op, align]
       if (Record.size() != 4)
         return error("Invalid record");
-      uint64_t AlignRecord = Record[3];
-      const uint64_t InAllocaMask = uint64_t(1) << 5;
-      const uint64_t ExplicitTypeMask = uint64_t(1) << 6;
-      const uint64_t SwiftErrorMask = uint64_t(1) << 7;
-      const uint64_t FlagMask = InAllocaMask | ExplicitTypeMask |
-                                SwiftErrorMask;
-      bool InAlloca = AlignRecord & InAllocaMask;
-      bool SwiftError = AlignRecord & SwiftErrorMask;
+      using APV = AllocaPackedValues;
+      const uint64_t Rec = Record[3];
+      const bool InAlloca = Bitfield::get<APV::UsedWithInAlloca>(Rec);
+      const bool SwiftError = Bitfield::get<APV::SwiftError>(Rec);
       FullTy = getFullyStructuredTypeByID(Record[0]);
       Type *Ty = flattenPointerTypes(FullTy);
-      if ((AlignRecord & ExplicitTypeMask) == 0) {
+      if (!Bitfield::get<APV::ExplicitType>(Rec)) {
         auto *PTy = dyn_cast_or_null<PointerType>(Ty);
         if (!PTy)
           return error("Old-style alloca with a non-pointer type");
@@ -4832,7 +4829,8 @@ Error BitcodeReader::parseFunctionBody(Function *F) {
       Type *OpTy = getTypeByID(Record[1]);
       Value *Size = getFnValueByID(Record[2], OpTy);
       MaybeAlign Align;
-      if (Error Err = parseAlignmentValue(AlignRecord & ~FlagMask, Align)) {
+      if (Error Err =
+              parseAlignmentValue(Bitfield::get<APV::Align>(Rec), Align)) {
         return Err;
       }
       if (!Ty || !Size)
diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
index 7d410b24573e7..f566ddf0864c6 100644
--- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -27,6 +27,7 @@
 #include "llvm/Bitcode/BitcodeReader.h"
 #include "llvm/Bitcode/LLVMBitCodes.h"
 #include "llvm/Bitstream/BitCodes.h"
+#include "llvm/Bitstream/BitcodeCommon.h"
 #include "llvm/Bitstream/BitstreamWriter.h"
 #include "llvm/Config/llvm-config.h"
 #include "llvm/IR/Attributes.h"
@@ -394,6 +395,8 @@ class ModuleBitcodeWriter : public ModuleBitcodeWriterBase {
   unsigned getEncodedSyncScopeID(SyncScope::ID SSID) {
     return unsigned(SSID);
   }
+
+  unsigned getEncodedAlign(MaybeAlign Alignment) { return encode(Alignment); }
 };
 
 /// Class to manage the bitcode writing for a combined index.
@@ -1181,10 +1184,14 @@ void ModuleBitcodeWriter::writeModuleInfo() {
   // compute the maximum alignment value.
   std::map<std::string, unsigned> SectionMap;
   std::map<std::string, unsigned> GCMap;
-  unsigned MaxAlignment = 0;
+  MaybeAlign MaxAlignment;
   unsigned MaxGlobalType = 0;
+  const auto UpdateMaxAlignment = [&MaxAlignment](const MaybeAlign A) {
+    if (A)
+      MaxAlignment = !MaxAlignment ? *A : std::max(*MaxAlignment, *A);
+  };
   for (const GlobalVariable &GV : M.globals()) {
-    MaxAlignment = std::max(MaxAlignment, GV.getAlignment());
+    UpdateMaxAlignment(GV.getAlign());
     MaxGlobalType = std::max(MaxGlobalType, VE.getTypeID(GV.getValueType()));
     if (GV.hasSection()) {
       // Give section names unique ID's.
@@ -1197,7 +1204,7 @@ void ModuleBitcodeWriter::writeModuleInfo() {
     }
   }
   for (const Function &F : M) {
-    MaxAlignment = std::max(MaxAlignment, F.getAlignment());
+    UpdateMaxAlignment(F.getAlign());
     if (F.hasSection()) {
       // Give section names unique ID's.
       unsigned &Entry = SectionMap[std::string(F.getSection())];
@@ -1233,10 +1240,10 @@ void ModuleBitcodeWriter::writeModuleInfo() {
                                                            //| constant
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));   // Initializer.
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 5)); // Linkage.
-    if (MaxAlignment == 0)                                 // Alignment.
+    if (!MaxAlignment)                                     // Alignment.
       Abbv->Add(BitCodeAbbrevOp(0));
     else {
-      unsigned MaxEncAlignment = Log2_32(MaxAlignment)+1;
+      unsigned MaxEncAlignment = getEncodedAlign(MaxAlignment);
       Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,
                                Log2_32_Ceil(MaxEncAlignment+1)));
     }
@@ -1289,7 +1296,7 @@ void ModuleBitcodeWriter::writeModuleInfo() {
     Vals.push_back(GV.isDeclaration() ? 0 :
                    (VE.getValueID(GV.getInitializer()) + 1));
     Vals.push_back(getEncodedLinkage(GV));
-    Vals.push_back(Log2_32(GV.getAlignment())+1);
+    Vals.push_back(getEncodedAlign(GV.getAlign()));
     Vals.push_back(GV.hasSection() ? SectionMap[std::string(GV.getSection())]
                                    : 0);
     if (GV.isThreadLocal() ||
@@ -1335,7 +1342,7 @@ void ModuleBitcodeWriter::writeModuleInfo() {
     Vals.push_back(F.isDeclaration());
     Vals.push_back(getEncodedLinkage(F));
     Vals.push_back(VE.getAttributeListID(F.getAttributes()));
-    Vals.push_back(Log2_32(F.getAlignment())+1);
+    Vals.push_back(getEncodedAlign(F.getAlign()));
     Vals.push_back(F.hasSection() ? SectionMap[std::string(F.getSection())]
                                   : 0);
     Vals.push_back(getEncodedVisibility(F));
@@ -2945,14 +2952,13 @@ void ModuleBitcodeWriter::writeInstruction(const Instruction &I,
     Vals.push_back(VE.getTypeID(AI.getAllocatedType()));
     Vals.push_back(VE.getTypeID(I.getOperand(0)->getType()));
     Vals.push_back(VE.getValueID(I.getOperand(0))); // size.
-    unsigned AlignRecord = Log2_32(AI.getAlignment()) + 1;
-    assert(Log2_32(Value::MaximumAlignment) + 1 < 1 << 5 &&
-           "not enough bits for maximum alignment");
-    assert(AlignRecord < 1 << 5 && "alignment greater than 1 << 64");
-    AlignRecord |= AI.isUsedWithInAlloca() << 5;
-    AlignRecord |= 1 << 6;
-    AlignRecord |= AI.isSwiftError() << 7;
-    Vals.push_back(AlignRecord);
+    using APV = AllocaPackedValues;
+    unsigned Record = 0;
+    Bitfield::set<APV::Align>(Record, getEncodedAlign(AI.getAlign()));
+    Bitfield::set<APV::UsedWithInAlloca>(Record, AI.isUsedWithInAlloca());
+    Bitfield::set<APV::ExplicitType>(Record, true);
+    Bitfield::set<APV::SwiftError>(Record, AI.isSwiftError());
+    Vals.push_back(Record);
     break;
   }
 
@@ -2966,7 +2972,7 @@ void ModuleBitcodeWriter::writeInstruction(const Instruction &I,
         AbbrevToUse = FUNCTION_INST_LOAD_ABBREV;
     }
     Vals.push_back(VE.getTypeID(I.getType()));
-    Vals.push_back(Log2_32(cast<LoadInst>(I).getAlignment())+1);
+    Vals.push_back(getEncodedAlign(cast<LoadInst>(I).getAlign()));
     Vals.push_back(cast<LoadInst>(I).isVolatile());
     if (cast<LoadInst>(I).isAtomic()) {
       Vals.push_back(getEncodedOrdering(cast<LoadInst>(I).getOrdering()));
@@ -2980,7 +2986,7 @@ void ModuleBitcodeWriter::writeInstruction(const Instruction &I,
       Code = bitc::FUNC_CODE_INST_STORE;
     pushValueAndType(I.getOperand(1), InstID, Vals); // ptrty + ptr
     pushValueAndType(I.getOperand(0), InstID, Vals); // valty + val
-    Vals.push_back(Log2_32(cast<StoreInst>(I).getAlignment())+1);
+    Vals.push_back(getEncodedAlign(cast<StoreInst>(I).getAlign()));
     Vals.push_back(cast<StoreInst>(I).isVolatile());
     if (cast<StoreInst>(I).isAtomic()) {
       Vals.push_back(getEncodedOrdering(cast<StoreInst>(I).getOrdering()));

From 974ffee9ccd70703c6edb880ac4934a5dc12e56d Mon Sep 17 00:00:00 2001
From: Kirill Bobyrev <kbobyrev@google.com>
Date: Mon, 27 Jul 2020 10:43:38 +0200
Subject: [PATCH 0169/1035] [clangd] Switch from EXPECT_TRUE to ASSERT_TRUE in
 remote marshalling tests

Summary:
When dereferencing Optional's it makes sense to use ASSERT_TRUE for better
test failures readability. Switch from EXPECT_TRUE to ASSERT_TRUE where
it is appropriate.

Reviewers: kadircet

Reviewed By: kadircet

Subscribers: ilya-biryukov, MaskRay, jkorous, arphaman, usaxena95, cfe-commits

Tags: #clang

Differential Revision: https://reviews.llvm.org/D84535

Signed-off-by: Kirill Bobyrev <kbobyrev@google.com>
---
 .../unittests/remote/MarshallingTests.cpp     | 55 ++++++++-----------
 1 file changed, 23 insertions(+), 32 deletions(-)

diff --git a/clang-tools-extra/clangd/unittests/remote/MarshallingTests.cpp b/clang-tools-extra/clangd/unittests/remote/MarshallingTests.cpp
index 7db7c03d61c9b..147601b665c4c 100644
--- a/clang-tools-extra/clangd/unittests/remote/MarshallingTests.cpp
+++ b/clang-tools-extra/clangd/unittests/remote/MarshallingTests.cpp
@@ -49,11 +49,11 @@ TEST(RemoteMarshallingTest, URITranslation) {
                   "clangd/unittests/remote/MarshallingTests.cpp",
                   Strings);
   auto Serialized = ProtobufMarshaller.toProtobuf(Original);
-  EXPECT_TRUE(Serialized);
+  ASSERT_TRUE(Serialized);
   EXPECT_EQ(Serialized->location().file_path(),
             "clang-tools-extra/clangd/unittests/remote/MarshallingTests.cpp");
   auto Deserialized = ProtobufMarshaller.fromProtobuf(*Serialized);
-  EXPECT_TRUE(Deserialized);
+  ASSERT_TRUE(Deserialized);
   EXPECT_STREQ(Deserialized->Location.FileURI,
                testPathURI("home/my-projects/llvm-project/clang-tools-extra/"
                            "clangd/unittests/remote/MarshallingTests.cpp",
@@ -61,38 +61,34 @@ TEST(RemoteMarshallingTest, URITranslation) {
 
   // Can't have empty paths.
   *Serialized->mutable_location()->mutable_file_path() = std::string();
-  Deserialized = ProtobufMarshaller.fromProtobuf(*Serialized);
-  EXPECT_FALSE(Deserialized);
+  EXPECT_FALSE(ProtobufMarshaller.fromProtobuf(*Serialized));
 
   clangd::Ref WithInvalidURI;
   // Invalid URI results in serialization failure.
   WithInvalidURI.Location.FileURI = "This is not a URI";
-  Serialized = ProtobufMarshaller.toProtobuf(WithInvalidURI);
-  EXPECT_FALSE(Serialized);
+  EXPECT_FALSE(ProtobufMarshaller.toProtobuf(WithInvalidURI));
 
   // Can not use URIs with scheme different from "file".
   auto UnittestURI =
       URI::create(testPath("project/lib/HelloWorld.cpp"), "unittest");
-  EXPECT_TRUE(bool(UnittestURI));
+  ASSERT_TRUE(bool(UnittestURI));
   WithInvalidURI.Location.FileURI =
       Strings.save(UnittestURI->toString()).begin();
-  Serialized = ProtobufMarshaller.toProtobuf(WithInvalidURI);
-  EXPECT_FALSE(Serialized);
+  EXPECT_FALSE(ProtobufMarshaller.toProtobuf(WithInvalidURI));
 
   // Paths transmitted over the wire can not be absolute, they have to be
   // relative.
   Ref WithAbsolutePath;
   *WithAbsolutePath.mutable_location()->mutable_file_path() =
       "/usr/local/user/home/HelloWorld.cpp";
-  Deserialized = ProtobufMarshaller.fromProtobuf(WithAbsolutePath);
-  EXPECT_FALSE(Deserialized);
+  EXPECT_FALSE(ProtobufMarshaller.fromProtobuf(WithAbsolutePath));
 }
 
 TEST(RemoteMarshallingTest, SymbolSerialization) {
   clangd::Symbol Sym;
 
   auto ID = SymbolID::fromStr("057557CEBF6E6B2D");
-  EXPECT_TRUE(bool(ID));
+  ASSERT_TRUE(bool(ID));
   Sym.ID = *ID;
 
   index::SymbolInfo Info;
@@ -140,9 +136,9 @@ TEST(RemoteMarshallingTest, SymbolSerialization) {
   // Check that symbols are exactly the same if the path to indexed project is
   // the same on indexing machine and the client.
   auto Serialized = ProtobufMarshaller.toProtobuf(Sym);
-  EXPECT_TRUE(Serialized);
+  ASSERT_TRUE(Serialized);
   auto Deserialized = ProtobufMarshaller.fromProtobuf(*Serialized);
-  EXPECT_TRUE(Deserialized);
+  ASSERT_TRUE(Deserialized);
   EXPECT_EQ(toYAML(Sym), toYAML(*Deserialized));
   // Serialized paths are relative and have UNIX slashes.
   EXPECT_EQ(convert_to_slash(Serialized->definition().file_path(),
@@ -154,44 +150,39 @@ TEST(RemoteMarshallingTest, SymbolSerialization) {
   // Missing definition is OK.
   Sym.Definition = clangd::SymbolLocation();
   Serialized = ProtobufMarshaller.toProtobuf(Sym);
-  EXPECT_TRUE(Serialized);
-  Deserialized = ProtobufMarshaller.fromProtobuf(*Serialized);
-  EXPECT_TRUE(Deserialized);
+  ASSERT_TRUE(Serialized);
+  EXPECT_TRUE(ProtobufMarshaller.fromProtobuf(*Serialized));
 
   // Relative path is absolute.
   *Serialized->mutable_canonical_declaration()->mutable_file_path() =
       convert_to_slash("/path/to/Declaration.h");
-  Deserialized = ProtobufMarshaller.fromProtobuf(*Serialized);
-  EXPECT_FALSE(Deserialized);
+  EXPECT_FALSE(ProtobufMarshaller.fromProtobuf(*Serialized));
 
   // Fail with an invalid URI.
   Location.FileURI = "Not A URI";
   Sym.Definition = Location;
-  Serialized = ProtobufMarshaller.toProtobuf(Sym);
-  EXPECT_FALSE(Serialized);
+  EXPECT_FALSE(ProtobufMarshaller.toProtobuf(Sym));
 
   // Schemes other than "file" can not be used.
   auto UnittestURI = URI::create(testPath("home/SomePath.h"), "unittest");
-  EXPECT_TRUE(bool(UnittestURI));
+  ASSERT_TRUE(bool(UnittestURI));
   Location.FileURI = Strings.save(UnittestURI->toString()).begin();
   Sym.Definition = Location;
-  Serialized = ProtobufMarshaller.toProtobuf(Sym);
-  EXPECT_FALSE(Serialized);
+  EXPECT_FALSE(ProtobufMarshaller.toProtobuf(Sym));
 
   // Passing root that is not prefix of the original file path.
   Location.FileURI = testPathURI("home/File.h", Strings);
   Sym.Definition = Location;
   // Check that the symbol is valid and passing the correct path works.
   Serialized = ProtobufMarshaller.toProtobuf(Sym);
-  EXPECT_TRUE(Serialized);
+  ASSERT_TRUE(Serialized);
   Deserialized = ProtobufMarshaller.fromProtobuf(*Serialized);
-  EXPECT_TRUE(Deserialized);
+  ASSERT_TRUE(Deserialized);
   EXPECT_STREQ(Deserialized->Definition.FileURI,
                testPathURI("home/File.h", Strings));
   // Fail with a wrong root.
   Marshaller WrongMarshaller(testPath("nothome/"), testPath("home/"));
-  Serialized = WrongMarshaller.toProtobuf(Sym);
-  EXPECT_FALSE(Serialized);
+  EXPECT_FALSE(WrongMarshaller.toProtobuf(Sym));
 }
 
 TEST(RemoteMarshallingTest, RefSerialization) {
@@ -214,9 +205,9 @@ TEST(RemoteMarshallingTest, RefSerialization) {
                                 testPath("llvm-project/"));
 
   auto Serialized = ProtobufMarshaller.toProtobuf(Ref);
-  EXPECT_TRUE(Serialized);
+  ASSERT_TRUE(Serialized);
   auto Deserialized = ProtobufMarshaller.fromProtobuf(*Serialized);
-  EXPECT_TRUE(Deserialized);
+  ASSERT_TRUE(Deserialized);
   EXPECT_EQ(toYAML(Ref), toYAML(*Deserialized));
 }
 
@@ -270,11 +261,11 @@ TEST(RemoteMarshallingTest, IncludeHeaderURIs) {
   Marshaller ProtobufMarshaller(convert_to_slash("/"), convert_to_slash("/"));
 
   auto Serialized = ProtobufMarshaller.toProtobuf(Sym);
+  ASSERT_TRUE(Serialized);
   EXPECT_EQ(static_cast<size_t>(Serialized->headers_size()),
             ValidHeaders.size());
-  EXPECT_TRUE(Serialized);
   auto Deserialized = ProtobufMarshaller.fromProtobuf(*Serialized);
-  EXPECT_TRUE(Deserialized);
+  ASSERT_TRUE(Deserialized);
 
   Sym.IncludeHeaders = ValidHeaders;
   EXPECT_EQ(toYAML(Sym), toYAML(*Deserialized));

From 19e472fd84ec75f5323a147bedfffabbb23d0978 Mon Sep 17 00:00:00 2001
From: Alex Richardson <Alexander.Richardson@cl.cam.ac.uk>
Date: Mon, 27 Jul 2020 10:15:17 +0100
Subject: [PATCH 0170/1035] [libcxx][lit] Fix running testsuite with python2.7
 after 9020d28688492c437abb648b6ab69baeba523219

Python 2.7 fails with TypeError: unsupported operand type(s) for +: 'NoneType' and 'str'
if you pass None as the prefix argument to NamedTemporaryFile.

Reviewed By: ldionne, bjope, #libc

Differential Revision: https://reviews.llvm.org/D84595
---
 libcxx/utils/libcxx/test/dsl.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libcxx/utils/libcxx/test/dsl.py b/libcxx/utils/libcxx/test/dsl.py
index cd500e132946e..95d23df7ece13 100644
--- a/libcxx/utils/libcxx/test/dsl.py
+++ b/libcxx/utils/libcxx/test/dsl.py
@@ -52,7 +52,7 @@ def _executeScriptInternal(test, commands):
     res = ('', '', 127, None)
   return res
 
-def _makeConfigTest(config, testPrefix=None):
+def _makeConfigTest(config, testPrefix=''):
   sourceRoot = os.path.join(config.test_exec_root, '__config_src__')
   execRoot = os.path.join(config.test_exec_root, '__config_exec__')
   suite = lit.Test.TestSuite('__config__', sourceRoot, execRoot, config)
@@ -83,7 +83,7 @@ def sourceBuilds(config, source):
     _executeScriptInternal(test, ['rm %t.exe'])
     return exitCode == 0
 
-def programOutput(config, program, args=[], testPrefix=None):
+def programOutput(config, program, args=[], testPrefix=''):
   """
   Compiles a program for the test target, run it on the test target and return
   the output.

From fa1145a8d2f1bd00a60d0ed4572901d2b1403157 Mon Sep 17 00:00:00 2001
From: Isaac Richter <isaac@irichter.net>
Date: Mon, 27 Jul 2020 11:49:24 +0300
Subject: [PATCH 0171/1035] [lld][ELF] Add LOG2CEIL builtin ldscript function

This patch adds support for the LOG2CEIL builtin function in linker scripts: https://sourceware.org/binutils/docs/ld/Builtin-Functions.html#index-LOG2CEIL_0028exp_0029

As documented for LD, and to keep compatibility, LOG2CEIL(0) returns 0 (not -inf).

The test vectors are somewhat arbitrary. We check minimum values (0-4); middle values (2^32, and 2^32+1); and the maximum value (2^64-1).

The checks for LOG2CEIL explicitly use full 64-bit values (16 hex digits). This is needed to properly verify that -inf and other interesting results aren't returned. (For some reason, all other tests in operators.test use only 14 digits.)

Differential revision: https://reviews.llvm.org/D84054
---
 lld/ELF/ScriptParser.cpp                 | 10 ++++++++++
 lld/test/ELF/linkerscript/operators.test | 16 ++++++++++++++++
 2 files changed, 26 insertions(+)

diff --git a/lld/ELF/ScriptParser.cpp b/lld/ELF/ScriptParser.cpp
index fea6b7a274e77..17ac7ff6d5f4a 100644
--- a/lld/ELF/ScriptParser.cpp
+++ b/lld/ELF/ScriptParser.cpp
@@ -29,6 +29,7 @@
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/FileSystem.h"
+#include "llvm/Support/MathExtras.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/ScopedPrinter.h"
 #include <cassert>
@@ -1329,6 +1330,15 @@ Expr ScriptParser::readPrimary() {
       return cmd->getLMA();
     };
   }
+  if (tok == "LOG2CEIL") {
+    expect("(");
+    Expr a = readExpr();
+    expect(")");
+    return [=] {
+      // LOG2CEIL(0) is defined to be 0.
+      return llvm::Log2_64_Ceil(std::max(a().getValue(), UINT64_C(1)));
+    };
+  }
   if (tok == "MAX" || tok == "MIN") {
     expect("(");
     Expr a = readExpr();
diff --git a/lld/test/ELF/linkerscript/operators.test b/lld/test/ELF/linkerscript/operators.test
index 1d40c81321d96..8ba8ee7ea41dc 100644
--- a/lld/test/ELF/linkerscript/operators.test
+++ b/lld/test/ELF/linkerscript/operators.test
@@ -38,6 +38,14 @@ SECTIONS {
   minus_abs = _end - _start;
   max = MAX(11, 22);
   min = MIN(11, 22);
+  log2ceil0 = LOG2CEIL(0);
+  log2ceil1 = LOG2CEIL(1);
+  log2ceil2 = LOG2CEIL(2);
+  log2ceil3 = LOG2CEIL(3);
+  log2ceil4 = LOG2CEIL(4);
+  log2ceil100000000 = LOG2CEIL(0x100000000);
+  log2ceil100000001 = LOG2CEIL(0x100000001);
+  log2ceilmax = LOG2CEIL(0xffffffffffffffff);
   logicaland1 = 0 && 0;
   logicaland2 = 0 && 1;
   logicaland3 = 1 && 0;
@@ -78,6 +86,14 @@ SECTIONS {
 # CHECK-NEXT: 0000000000fff0 A minus_abs
 # CHECK-NEXT: 00000000000016 A max
 # CHECK-NEXT: 0000000000000b A min
+# CHECK-NEXT: 0000000000000000 A log2ceil0
+# CHECK-NEXT: 0000000000000000 A log2ceil1
+# CHECK-NEXT: 0000000000000001 A log2ceil2
+# CHECK-NEXT: 0000000000000002 A log2ceil3
+# CHECK-NEXT: 0000000000000002 A log2ceil4
+# CHECK-NEXT: 0000000000000020 A log2ceil100000000
+# CHECK-NEXT: 0000000000000021 A log2ceil100000001
+# CHECK-NEXT: 0000000000000040 A log2ceilmax
 # CHECK-NEXT: 00000000000000 A logicaland1
 # CHECK-NEXT: 00000000000000 A logicaland2
 # CHECK-NEXT: 00000000000000 A logicaland3

From 8b74596b7e7c82396efdf95858fcec615a656caf Mon Sep 17 00:00:00 2001
From: Afanasyev Ivan <ivafanas@gmail.com>
Date: Mon, 27 Jul 2020 10:19:55 +0100
Subject: [PATCH 0172/1035] [Docs] remove unused arguments in documentation
 examples on vectorization passes

Reviewers: nadav, tyler.nowicki

Reviewed By: nadav

Differential Revision: https://reviews.llvm.org/D83851
---
 llvm/docs/Vectorizers.rst | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/llvm/docs/Vectorizers.rst b/llvm/docs/Vectorizers.rst
index c322797025fb6..702090447c692 100644
--- a/llvm/docs/Vectorizers.rst
+++ b/llvm/docs/Vectorizers.rst
@@ -193,7 +193,7 @@ reduction operations, such as addition, multiplication, XOR, AND and OR.
 
 .. code-block:: c++
 
-  int foo(int *A, int *B, int n) {
+  int foo(int *A, int n) {
     unsigned sum = 0;
     for (int i = 0; i < n; ++i)
       sum += A[i] + 5;
@@ -210,7 +210,7 @@ array. The Loop Vectorizer knows to vectorize induction variables.
 
 .. code-block:: c++
 
-  void bar(float *A, float* B, float K, int n) {
+  void bar(float *A, int n) {
     for (int i = 0; i < n; ++i)
       A[i] = i;
   }
@@ -254,7 +254,7 @@ The Loop Vectorizer can vectorize loops that count backwards.
 
 .. code-block:: c++
 
-  int foo(int *A, int *B, int n) {
+  int foo(int *A, int n) {
     for (int i = n; i > 0; --i)
       A[i] +=1;
   }
@@ -284,7 +284,7 @@ vectorization is profitable.
 
 .. code-block:: c++
 
-  int foo(int *A, char *B, int n, int k) {
+  int foo(int *A, char *B, int n) {
     for (int i = 0; i < n; ++i)
       A[i] += 4 * B[i];
   }
@@ -360,7 +360,7 @@ to be used simultaneously.
 
 .. code-block:: c++
 
-  int foo(int *A, int *B, int n) {
+  int foo(int *A, int n) {
     unsigned sum = 0;
     for (int i = 0; i < n; ++i)
         sum += A[i];

From 37ac559fccd46dcec246ceb3907c8d3910728c69 Mon Sep 17 00:00:00 2001
From: Kirill Bobyrev <kbobyrev@google.com>
Date: Mon, 27 Jul 2020 11:21:55 +0200
Subject: [PATCH 0173/1035] [clangd] Add option to use remote index as static
 index

Reviewers: hokein

Reviewed By: hokein

Subscribers: usaxena95, mgorny, ilya-biryukov, MaskRay, jkorous, arphaman, kadircet, cfe-commits

Tags: #clang

Differential Revision: https://reviews.llvm.org/D83817
---
 clang-tools-extra/clangd/Features.inc.in     |  1 +
 clang-tools-extra/clangd/tool/CMakeLists.txt |  1 +
 clang-tools-extra/clangd/tool/ClangdMain.cpp | 33 ++++++++++++++++++++
 3 files changed, 35 insertions(+)

diff --git a/clang-tools-extra/clangd/Features.inc.in b/clang-tools-extra/clangd/Features.inc.in
index da75aa67a65b8..8584b87c62051 100644
--- a/clang-tools-extra/clangd/Features.inc.in
+++ b/clang-tools-extra/clangd/Features.inc.in
@@ -1 +1,2 @@
 #define CLANGD_BUILD_XPC @CLANGD_BUILD_XPC@
+#define CLANGD_ENABLE_REMOTE @CLANGD_ENABLE_REMTE@
diff --git a/clang-tools-extra/clangd/tool/CMakeLists.txt b/clang-tools-extra/clangd/tool/CMakeLists.txt
index 3368013f50798..670e5a17013ab 100644
--- a/clang-tools-extra/clangd/tool/CMakeLists.txt
+++ b/clang-tools-extra/clangd/tool/CMakeLists.txt
@@ -27,6 +27,7 @@ clang_target_link_libraries(clangd
   clangToolingCore
   clangToolingRefactoring
   clangToolingSyntax
+  clangdRemoteIndex
   )
 target_link_libraries(clangd
   PRIVATE
diff --git a/clang-tools-extra/clangd/tool/ClangdMain.cpp b/clang-tools-extra/clangd/tool/ClangdMain.cpp
index 7bce1c062e817..8d1bf5c422605 100644
--- a/clang-tools-extra/clangd/tool/ClangdMain.cpp
+++ b/clang-tools-extra/clangd/tool/ClangdMain.cpp
@@ -14,6 +14,7 @@
 #include "Transport.h"
 #include "index/Background.h"
 #include "index/Serialization.h"
+#include "index/remote/Client.h"
 #include "refactor/Rename.h"
 #include "support/Path.h"
 #include "support/Shutdown.h"
@@ -449,6 +450,21 @@ opt<bool> EnableConfig{
     init(true),
 };
 
+#ifdef CLANGD_ENABLE_REMOTE
+opt<std::string> RemoteIndexAddress{
+    "remote-index-address",
+    cat(Features),
+    desc("Address of the remote index server"),
+};
+
+// FIXME(kirillbobyrev): Should this be the location of compile_commands.json?
+opt<std::string> ProjectRoot{
+    "project-root",
+    cat(Features),
+    desc("Path to the project root. Requires remote-index-address to be set."),
+};
+#endif
+
 /// Supports a test URI scheme with relaxed constraints for lit tests.
 /// The path in a test URI will be combined with a platform-specific fake
 /// directory to form an absolute path. For example, test:///a.cpp is resolved
@@ -680,6 +696,23 @@ clangd accepts flags on the commandline, and in the CLANGD_FLAGS environment var
     if (Sync)
       AsyncIndexLoad.wait();
   }
+#ifdef CLANGD_ENABLE_REMOTE
+  if (RemoteIndexAddress.empty() != ProjectRoot.empty()) {
+    llvm::errs() << "remote-index-address and project-path have to be "
+                    "specified at the same time.";
+    return 1;
+  }
+  if (!RemoteIndexAddress.empty()) {
+    if (IndexFile.empty()) {
+      log("Connecting to remote index at {0}", RemoteIndexAddress);
+      StaticIdx = remote::getClient(RemoteIndexAddress, ProjectRoot);
+      EnableBackgroundIndex = false;
+    } else {
+      elog("When enabling remote index, IndexFile should not be specified. "
+           "Only one can be used at time. Remote index will ignored.");
+    }
+  }
+#endif
   Opts.StaticIndex = StaticIdx.get();
   Opts.AsyncThreadsCount = WorkerThreadsCount;
   Opts.BuildRecoveryAST = RecoveryAST;

From 2e828e7579928e8cc1c5e53c84ab99ffb5afca03 Mon Sep 17 00:00:00 2001
From: Pavel Labath <pavel@labath.sk>
Date: Mon, 27 Jul 2020 11:46:51 +0200
Subject: [PATCH 0174/1035] [lldb] Fix e89414f406 for msvc

MSVC finds the APInt construction ambiguous. Use a case to help it
choose the right constructor.
---
 lldb/include/lldb/Utility/Scalar.h | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/lldb/include/lldb/Utility/Scalar.h b/lldb/include/lldb/Utility/Scalar.h
index 1dbcf80bfd89c..45ba7c012229b 100644
--- a/lldb/include/lldb/Utility/Scalar.h
+++ b/lldb/include/lldb/Utility/Scalar.h
@@ -62,18 +62,23 @@ class Scalar {
   // Constructors and Destructors
   Scalar() : m_type(e_void), m_float(0.0f) {}
   Scalar(int v)
-      : m_type(e_sint), m_integer(sizeof(v) * 8, v, true), m_float(0.0f) {}
+      : m_type(e_sint), m_integer(sizeof(v) * 8, uint64_t(v), true),
+        m_float(0.0f) {}
   Scalar(unsigned int v)
-      : m_type(e_uint), m_integer(sizeof(v) * 8, v, false), m_float(0.0f) {}
+      : m_type(e_uint), m_integer(sizeof(v) * 8, uint64_t(v), false),
+        m_float(0.0f) {}
   Scalar(long v)
-      : m_type(e_slong), m_integer(sizeof(v) * 8, v, true), m_float(0.0f) {}
+      : m_type(e_slong), m_integer(sizeof(v) * 8, uint64_t(v), true),
+        m_float(0.0f) {}
   Scalar(unsigned long v)
-      : m_type(e_ulong), m_integer(sizeof(v) * 8, v, false), m_float(0.0f) {}
+      : m_type(e_ulong), m_integer(sizeof(v) * 8, uint64_t(v), false),
+        m_float(0.0f) {}
   Scalar(long long v)
-      : m_type(e_slonglong), m_integer(sizeof(v) * 8, v, true), m_float(0.0f) {}
+      : m_type(e_slonglong), m_integer(sizeof(v) * 8, uint64_t(v), true),
+        m_float(0.0f) {}
   Scalar(unsigned long long v)
-      : m_type(e_ulonglong), m_integer(sizeof(v) * 8, v, false), m_float(0.0f) {
-  }
+      : m_type(e_ulonglong), m_integer(sizeof(v) * 8, uint64_t(v), false),
+        m_float(0.0f) {}
   Scalar(float v) : m_type(e_float), m_float(v) {}
   Scalar(double v) : m_type(e_double), m_float(v) {}
   Scalar(long double v) : m_type(e_long_double), m_float(double(v)) {

From c891f519e17e38ec466deab1730c48e9d7a2eca8 Mon Sep 17 00:00:00 2001
From: Juneyoung Lee <aqjune@gmail.com>
Date: Mon, 27 Jul 2020 19:04:50 +0900
Subject: [PATCH 0175/1035] [JumpThreading] Add a test that threads jumps with
 frozen branch conditions

---
 .../freeze-lvi-edgevaluelocal.ll              | 111 ++++++++++++++++++
 1 file changed, 111 insertions(+)
 create mode 100644 llvm/test/Transforms/JumpThreading/freeze-lvi-edgevaluelocal.ll

diff --git a/llvm/test/Transforms/JumpThreading/freeze-lvi-edgevaluelocal.ll b/llvm/test/Transforms/JumpThreading/freeze-lvi-edgevaluelocal.ll
new file mode 100644
index 0000000000000..07dd956777f68
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/freeze-lvi-edgevaluelocal.ll
@@ -0,0 +1,111 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -jump-threading -S < %s | FileCheck %s
+
+declare void @f()
+declare void @f2()
+declare void @f3()
+
+; br on cond.fr should be removed & ENTRY -> A -> B should be threaded
+define i32 @simple(i1 %cond) {
+; CHECK-LABEL: @simple(
+; CHECK-NEXT:  ENTRY:
+; CHECK-NEXT:    [[COND_FR:%.*]] = freeze i1 [[COND:%.*]]
+; CHECK-NEXT:    br i1 [[COND]], label [[A:%.*]], label [[EXIT:%.*]]
+; CHECK:       A:
+; CHECK-NEXT:    br i1 [[COND_FR]], label [[B:%.*]], label [[EXIT]]
+; CHECK:       B:
+; CHECK-NEXT:    call void @f()
+; CHECK-NEXT:    ret i32 1
+; CHECK:       EXIT:
+; CHECK-NEXT:    ret i32 0
+;
+ENTRY:
+  %cond.fr = freeze i1 %cond
+  br i1 %cond, label %A, label %EXIT
+A:
+  br i1 %cond.fr, label %B, label %EXIT
+B:
+  call void @f()
+  ret i32 1
+EXIT:
+  ret i32 0
+}
+
+define void @switch(i32 %cond) {
+; CHECK-LABEL: @switch(
+; CHECK-NEXT:  ENTRY:
+; CHECK-NEXT:    [[COND_FR:%.*]] = freeze i32 [[COND:%.*]]
+; CHECK-NEXT:    switch i32 [[COND]], label [[DEFAULT:%.*]] [
+; CHECK-NEXT:    i32 0, label [[A:%.*]]
+; CHECK-NEXT:    i32 1, label [[B:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       DEFAULT:
+; CHECK-NEXT:    switch i32 [[COND_FR]], label [[PRESERVED1:%.*]] [
+; CHECK-NEXT:    i32 0, label [[PRESERVED2:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       PRESERVED1:
+; CHECK-NEXT:    call void @f()
+; CHECK-NEXT:    ret void
+; CHECK:       PRESERVED2:
+; CHECK-NEXT:    call void @f2()
+; CHECK-NEXT:    ret void
+; CHECK:       A:
+; CHECK-NEXT:    switch i32 [[COND_FR]], label [[A_NOTTAKEN:%.*]] [
+; CHECK-NEXT:    i32 0, label [[A_TAKEN:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       A_TAKEN:
+; CHECK-NEXT:    call void @f()
+; CHECK-NEXT:    ret void
+; CHECK:       A_NOTTAKEN:
+; CHECK-NEXT:    call void @f2()
+; CHECK-NEXT:    ret void
+; CHECK:       B:
+; CHECK-NEXT:    switch i32 [[COND_FR]], label [[B_TAKEN:%.*]] [
+; CHECK-NEXT:    i32 0, label [[B_NOTTAKEN:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       B_TAKEN:
+; CHECK-NEXT:    call void @f()
+; CHECK-NEXT:    ret void
+; CHECK:       B_NOTTAKEN:
+; CHECK-NEXT:    call void @f2()
+; CHECK-NEXT:    ret void
+;
+ENTRY:
+  %cond.fr = freeze i32 %cond
+  switch i32 %cond, label %DEFAULT [
+  i32 0, label %A
+  i32 1, label %B
+  ]
+DEFAULT:
+  switch i32 %cond.fr, label %PRESERVED1 [
+  i32 0, label %PRESERVED2
+  ]
+PRESERVED1:
+  call void @f()
+  ret void
+PRESERVED2:
+  call void @f2()
+  ret void
+
+A:
+  switch i32 %cond.fr, label %A_NOTTAKEN [
+  i32 0, label %A_TAKEN
+  ]
+A_TAKEN:
+  call void @f()
+  ret void
+A_NOTTAKEN:
+  call void @f2()
+  ret void
+
+B:
+  switch i32 %cond.fr, label %B_TAKEN [
+  i32 0, label %B_NOTTAKEN
+  ]
+B_TAKEN:
+  call void @f()
+  ret void
+B_NOTTAKEN:
+  call void @f2()
+  ret void
+}

From 6701c0bf730157775736c1314758e89bfdccb14a Mon Sep 17 00:00:00 2001
From: Juneyoung Lee <aqjune@gmail.com>
Date: Mon, 27 Jul 2020 19:08:45 +0900
Subject: [PATCH 0176/1035] [JumpThreading] Add a test case that has a phi with
 undef; NFC

---
 llvm/test/Transforms/JumpThreading/freeze.ll | 44 ++++++++++++++++++++
 1 file changed, 44 insertions(+)

diff --git a/llvm/test/Transforms/JumpThreading/freeze.ll b/llvm/test/Transforms/JumpThreading/freeze.ll
index 37d2ad672f789..3c6aa98c32b84 100644
--- a/llvm/test/Transforms/JumpThreading/freeze.ll
+++ b/llvm/test/Transforms/JumpThreading/freeze.ll
@@ -49,6 +49,50 @@ F2:
   ret i32 %B
 }
 
+define i32 @test1_undef(i1 %cond) {
+; CHECK-LABEL: @test1_undef(
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[T1:%.*]], label [[F1:%.*]]
+; CHECK:       T1:
+; CHECK-NEXT:    [[V1:%.*]] = call i32 @f1()
+; CHECK-NEXT:    br label [[MERGE:%.*]]
+; CHECK:       F1:
+; CHECK-NEXT:    [[V2:%.*]] = call i32 @f2()
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       Merge:
+; CHECK-NEXT:    [[A:%.*]] = phi i1 [ true, [[T1]] ], [ undef, [[F1]] ]
+; CHECK-NEXT:    [[B:%.*]] = phi i32 [ [[V1]], [[T1]] ], [ [[V2]], [[F1]] ]
+; CHECK-NEXT:    [[A_FR:%.*]] = freeze i1 [[A]]
+; CHECK-NEXT:    br i1 [[A_FR]], label [[T2:%.*]], label [[F2:%.*]]
+; CHECK:       T2:
+; CHECK-NEXT:    call void @f3()
+; CHECK-NEXT:    ret i32 [[B]]
+; CHECK:       F2:
+; CHECK-NEXT:    ret i32 [[B]]
+;
+  br i1 %cond, label %T1, label %F1
+
+T1:
+  %v1 = call i32 @f1()
+  br label %Merge
+
+F1:
+  %v2 = call i32 @f2()
+  br label %Merge
+
+Merge:
+  %A = phi i1 [true, %T1], [undef, %F1]
+  %B = phi i32 [%v1, %T1], [%v2, %F1]
+  %A.fr = freeze i1 %A
+  br i1 %A.fr, label %T2, label %F2
+
+T2:
+  call void @f3()
+  ret i32 %B
+
+F2:
+  ret i32 %B
+}
+
 define i32 @test2(i1 %cond, i1 %cond2) {
 ; CHECK-LABEL: @test2(
 ; CHECK-NEXT:    br i1 [[COND:%.*]], label [[T1:%.*]], label [[F1:%.*]]

From 1956cf1042d3c406d9e9cefe47d3b43adf2bdbe1 Mon Sep 17 00:00:00 2001
From: Pavel Labath <pavel@labath.sk>
Date: Tue, 7 Jul 2020 15:28:24 +0200
Subject: [PATCH 0177/1035] [lldb/DWARF] Don't treat class declarations with
 children as definitions

Summary:
This effectively reverts r188124, which added code to handle
(DW_AT_)declarations of structures with some kinds of children as
definitions. The commit message claims this is a workaround for some
kind of debug info produced by gcc. However, it does not go into
specifics, so it's hard to reproduce or verify that this is indeed still a
problem.

Having this code is definitely a problem though, because it mistakenly
declares incomplete dwarf declarations to be complete. Both clang (with
-flimit-debug-info) and gcc (by default) generate DW_AT_declarations of
structs with children. This happens when full debug info for a class is
not emitted in a given compile unit (e.g. because of vtable homing), but
the class has inline methods which are used in the given compile unit.
In that case, the compilers emit a DW_AT_declaration of a class, but
add a DW_TAG_subprogram child to it to describe the inlined instance of
the method.

Even though the class tag has some children, it definitely does not
contain enough information to construct a full class definition (most
notably, it lacks any members). Keeping the class as incomplete allows
us to search for a real definition in other modules, helping the
-flimit-debug-info flow. And in case the definition is not found we can
display a error message saying that, instead of just showing an empty
struct.

Reviewers: clayborg, aprantl, JDevlieghere, shafik

Subscribers: lldb-commits

Tags: #lldb

Differential Revision: https://reviews.llvm.org/D83302
---
 .../SymbolFile/DWARF/DWARFASTParserClang.cpp  |  27 ---
 .../limit-debug-info/TestLimitDebugInfo.py    |  25 ++-
 .../functionalities/limit-debug-info/main.cpp |   5 +-
 .../functionalities/limit-debug-info/one.cpp  |   3 +
 .../functionalities/limit-debug-info/onetwo.h |  15 ++
 .../functionalities/limit-debug-info/two.cpp  |   4 +
 .../DWARF/DW_AT_declaration-with-children.s   | 160 ++++++++++++++++++
 7 files changed, 208 insertions(+), 31 deletions(-)
 create mode 100644 lldb/test/Shell/SymbolFile/DWARF/DW_AT_declaration-with-children.s

diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
index 35e7c34734e2c..7e3628504727a 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
@@ -1641,33 +1641,6 @@ DWARFASTParserClang::ParseStructureLikeDIE(const SymbolContext &sc,
   dwarf->GetUniqueDWARFASTTypeMap().Insert(unique_typename,
                                            *unique_ast_entry_up);
 
-  if (attrs.is_forward_declaration && die.HasChildren()) {
-    // Check to see if the DIE actually has a definition, some version of
-    // GCC will
-    // emit DIEs with DW_AT_declaration set to true, but yet still have
-    // subprogram, members, or inheritance, so we can't trust it
-    DWARFDIE child_die = die.GetFirstChild();
-    while (child_die) {
-      switch (child_die.Tag()) {
-      case DW_TAG_inheritance:
-      case DW_TAG_subprogram:
-      case DW_TAG_member:
-      case DW_TAG_APPLE_property:
-      case DW_TAG_class_type:
-      case DW_TAG_structure_type:
-      case DW_TAG_enumeration_type:
-      case DW_TAG_typedef:
-      case DW_TAG_union_type:
-        child_die.Clear();
-        attrs.is_forward_declaration = false;
-        break;
-      default:
-        child_die = child_die.GetSibling();
-        break;
-      }
-    }
-  }
-
   if (!attrs.is_forward_declaration) {
     // Always start the definition for a class type so that if the class
     // has child classes or types that require the class to be created
diff --git a/lldb/test/API/functionalities/limit-debug-info/TestLimitDebugInfo.py b/lldb/test/API/functionalities/limit-debug-info/TestLimitDebugInfo.py
index 9408ad6eee1d1..aa383d0005e41 100644
--- a/lldb/test/API/functionalities/limit-debug-info/TestLimitDebugInfo.py
+++ b/lldb/test/API/functionalities/limit-debug-info/TestLimitDebugInfo.py
@@ -38,7 +38,8 @@ def test_one_and_two_debug(self):
 
         self._check_debug_info_is_limited(target)
 
-        self.registerSharedLibrariesWithTarget(target, ["one", "two"])
+        lldbutil.run_to_name_breakpoint(self, "main",
+                extra_images=["one", "two"])
 
         # But when other shared libraries are loaded, we should be able to see
         # all members.
@@ -58,6 +59,10 @@ def test_one_and_two_debug(self):
         self.expect_expr("array_of_two[2].one[2].member", result_value="174")
         self.expect_expr("array_of_two[2].member", result_value="274")
 
+        self.expect_expr("get_one().member", result_value="124")
+        self.expect_expr("get_two().one().member", result_value="124")
+        self.expect_expr("get_two().member", result_value="224")
+
     @skipIf(bugnumber="pr46284", debug_info="gmodules")
     @skipIfWindows # Clang emits type info even with -flimit-debug-info
     def test_two_debug(self):
@@ -66,7 +71,8 @@ def test_two_debug(self):
 
         self._check_debug_info_is_limited(target)
 
-        self.registerSharedLibrariesWithTarget(target, ["one", "two"])
+        lldbutil.run_to_name_breakpoint(self, "main",
+                extra_images=["one", "two"])
 
         # This time, we should only see the members from the second library.
         self.expect_expr("inherits_from_one.member", result_value="47")
@@ -91,6 +97,12 @@ def test_two_debug(self):
                 substrs=["no member named 'member' in 'array::One'"])
         self.expect_expr("array_of_two[2].member", result_value="274")
 
+        self.expect("expr get_one().member", error=True,
+                substrs=["calling 'get_one' with incomplete return type 'result::One'"])
+        self.expect("expr get_two().one().member", error=True,
+                substrs=["calling 'one' with incomplete return type 'result::One'"])
+        self.expect_expr("get_two().member", result_value="224")
+
     @skipIf(bugnumber="pr46284", debug_info="gmodules")
     @skipIfWindows # Clang emits type info even with -flimit-debug-info
     def test_one_debug(self):
@@ -99,7 +111,8 @@ def test_one_debug(self):
 
         self._check_debug_info_is_limited(target)
 
-        self.registerSharedLibrariesWithTarget(target, ["one", "two"])
+        lldbutil.run_to_name_breakpoint(self, "main",
+                extra_images=["one", "two"])
 
         # In this case we should only see the members from the second library.
         # Note that we cannot see inherits_from_two.one because without debug
@@ -126,3 +139,9 @@ def test_one_debug(self):
                 substrs=["no member named 'one' in 'array::Two'"])
         self.expect("expr array_of_two[2].member", error=True,
                 substrs=["no member named 'member' in 'array::Two'"])
+
+        self.expect_expr("get_one().member", result_value="124")
+        self.expect("expr get_two().one().member", error=True,
+                substrs=["calling 'get_two' with incomplete return type 'result::Two'"])
+        self.expect("expr get_two().member", error=True,
+                substrs=["calling 'get_two' with incomplete return type 'result::Two'"])
diff --git a/lldb/test/API/functionalities/limit-debug-info/main.cpp b/lldb/test/API/functionalities/limit-debug-info/main.cpp
index 0a25de13d9fb2..1aad7e6f1e610 100644
--- a/lldb/test/API/functionalities/limit-debug-info/main.cpp
+++ b/lldb/test/API/functionalities/limit-debug-info/main.cpp
@@ -25,4 +25,7 @@ struct TwoAsMember {
 array::One array_of_one[3];
 array::Two array_of_two[3];
 
-int main() { return 0; }
+result::One get_one() { return result::One(124); }
+result::Two get_two() { return result::Two(224); }
+
+int main() { return get_one().member; }
diff --git a/lldb/test/API/functionalities/limit-debug-info/one.cpp b/lldb/test/API/functionalities/limit-debug-info/one.cpp
index c1eb6310dd679..70353a084edc5 100644
--- a/lldb/test/API/functionalities/limit-debug-info/one.cpp
+++ b/lldb/test/API/functionalities/limit-debug-info/one.cpp
@@ -3,3 +3,6 @@
 One::~One() = default;
 member::One::~One() = default;
 array::One::~One() = default;
+
+result::One::One(int member) : member(member) {}
+result::One::~One() = default;
diff --git a/lldb/test/API/functionalities/limit-debug-info/onetwo.h b/lldb/test/API/functionalities/limit-debug-info/onetwo.h
index 67609dd7ff61c..24a18f6a5dcc3 100644
--- a/lldb/test/API/functionalities/limit-debug-info/onetwo.h
+++ b/lldb/test/API/functionalities/limit-debug-info/onetwo.h
@@ -39,3 +39,18 @@ struct Two {
   virtual ~Two();
 };
 } // namespace array
+
+namespace result {
+struct One {
+  int member;
+  One(int member);
+  virtual ~One();
+};
+
+struct Two {
+  int member;
+  Two(int member);
+  One one() const;
+  virtual ~Two();
+};
+} // namespace result
diff --git a/lldb/test/API/functionalities/limit-debug-info/two.cpp b/lldb/test/API/functionalities/limit-debug-info/two.cpp
index 04683da6e9cce..468cb91f6a5c3 100644
--- a/lldb/test/API/functionalities/limit-debug-info/two.cpp
+++ b/lldb/test/API/functionalities/limit-debug-info/two.cpp
@@ -3,3 +3,7 @@
 Two::~Two() = default;
 member::Two::~Two() = default;
 array::Two::~Two() = default;
+
+result::Two::Two(int member) : member(member) {}
+result::Two::~Two() = default;
+result::One result::Two::one() const { return One(member - 100); }
diff --git a/lldb/test/Shell/SymbolFile/DWARF/DW_AT_declaration-with-children.s b/lldb/test/Shell/SymbolFile/DWARF/DW_AT_declaration-with-children.s
new file mode 100644
index 0000000000000..7ed33ce50297b
--- /dev/null
+++ b/lldb/test/Shell/SymbolFile/DWARF/DW_AT_declaration-with-children.s
@@ -0,0 +1,160 @@
+# Test that a forward-declared (DW_AT_declaration) structure is treated as a
+# forward-declaration even if it has children. These types can be produced due
+# to vtable-based type homing, or other -flimit-debug-info optimizations.
+
+# REQUIRES: x86
+
+# RUN: llvm-mc --triple x86_64-pc-linux %s --filetype=obj > %t
+# RUN: %lldb %t -o "expr a" -o exit 2>&1 | FileCheck %s --check-prefix=EXPR
+# RUN: %lldb %t -o "target var a" -o exit 2>&1 | FileCheck %s --check-prefix=VAR
+
+# EXPR: incomplete type 'A' where a complete type is required
+
+# FIXME: This should also produce some kind of an error.
+# VAR: (A) a = {}
+
+        .text
+_ZN1AC2Ev:
+        retq
+.LZN1AC2Ev_end:
+
+        .data
+a:
+        .quad   $_ZTV1A+16
+        .quad   $0xdeadbeef
+
+        .section        .debug_abbrev,"",@progbits
+        .byte   1                               # Abbreviation Code
+        .byte   17                              # DW_TAG_compile_unit
+        .byte   1                               # DW_CHILDREN_yes
+        .byte   37                              # DW_AT_producer
+        .byte   8                               # DW_FORM_string
+        .byte   17                              # DW_AT_low_pc
+        .byte   1                               # DW_FORM_addr
+        .byte   18                              # DW_AT_high_pc
+        .byte   6                               # DW_FORM_data4
+        .byte   0                               # EOM(1)
+        .byte   0                               # EOM(2)
+        .byte   2                               # Abbreviation Code
+        .byte   52                              # DW_TAG_variable
+        .byte   0                               # DW_CHILDREN_no
+        .byte   3                               # DW_AT_name
+        .byte   8                               # DW_FORM_string
+        .byte   73                              # DW_AT_type
+        .byte   19                              # DW_FORM_ref4
+        .byte   2                               # DW_AT_location
+        .byte   24                              # DW_FORM_exprloc
+        .byte   0                               # EOM(1)
+        .byte   0                               # EOM(2)
+        .byte   3                               # Abbreviation Code
+        .byte   19                              # DW_TAG_structure_type
+        .byte   1                               # DW_CHILDREN_yes
+        .byte   3                               # DW_AT_name
+        .byte   8                               # DW_FORM_string
+        .byte   60                              # DW_AT_declaration
+        .byte   25                              # DW_FORM_flag_present
+        .byte   0                               # EOM(1)
+        .byte   0                               # EOM(2)
+        .byte   4                               # Abbreviation Code
+        .byte   46                              # DW_TAG_subprogram
+        .byte   1                               # DW_CHILDREN_yes
+        .byte   3                               # DW_AT_name
+        .byte   8                               # DW_FORM_string
+        .byte   60                              # DW_AT_declaration
+        .byte   25                              # DW_FORM_flag_present
+        .byte   0                               # EOM(1)
+        .byte   0                               # EOM(2)
+        .byte   5                               # Abbreviation Code
+        .byte   5                               # DW_TAG_formal_parameter
+        .byte   0                               # DW_CHILDREN_no
+        .byte   73                              # DW_AT_type
+        .byte   19                              # DW_FORM_ref4
+        .byte   52                              # DW_AT_artificial
+        .byte   25                              # DW_FORM_flag_present
+        .byte   0                               # EOM(1)
+        .byte   0                               # EOM(2)
+        .byte   8                               # Abbreviation Code
+        .byte   15                              # DW_TAG_pointer_type
+        .byte   0                               # DW_CHILDREN_no
+        .byte   73                              # DW_AT_type
+        .byte   19                              # DW_FORM_ref4
+        .byte   0                               # EOM(1)
+        .byte   0                               # EOM(2)
+        .byte   10                              # Abbreviation Code
+        .byte   46                              # DW_TAG_subprogram
+        .byte   1                               # DW_CHILDREN_yes
+        .byte   17                              # DW_AT_low_pc
+        .byte   1                               # DW_FORM_addr
+        .byte   18                              # DW_AT_high_pc
+        .byte   6                               # DW_FORM_data4
+        .byte   64                              # DW_AT_frame_base
+        .byte   24                              # DW_FORM_exprloc
+        .byte   100                             # DW_AT_object_pointer
+        .byte   19                              # DW_FORM_ref4
+        .byte   71                              # DW_AT_specification
+        .byte   19                              # DW_FORM_ref4
+        .byte   0                               # EOM(1)
+        .byte   0                               # EOM(2)
+        .byte   11                              # Abbreviation Code
+        .byte   5                               # DW_TAG_formal_parameter
+        .byte   0                               # DW_CHILDREN_no
+        .byte   2                               # DW_AT_location
+        .byte   24                              # DW_FORM_exprloc
+        .byte   3                               # DW_AT_name
+        .byte   8                               # DW_FORM_string
+        .byte   73                              # DW_AT_type
+        .byte   19                              # DW_FORM_ref4
+        .byte   52                              # DW_AT_artificial
+        .byte   25                              # DW_FORM_flag_present
+        .byte   0                               # EOM(1)
+        .byte   0                               # EOM(2)
+        .byte   0                               # EOM(3)
+        .section        .debug_info,"",@progbits
+.Lcu_begin0:
+        .long   .Ldebug_info_end0-.Ldebug_info_start0 # Length of Unit
+.Ldebug_info_start0:
+        .short  4                               # DWARF version number
+        .long   .debug_abbrev                   # Offset Into Abbrev. Section
+        .byte   8                               # Address Size (in bytes)
+        .byte   1                               # Abbrev [1] DW_TAG_compile_unit
+        .asciz  "Hand-written DWARF"            # DW_AT_producer
+        .quad   _ZN1AC2Ev                       # DW_AT_low_pc
+        .long   .LZN1AC2Ev_end-_ZN1AC2Ev        # DW_AT_high_pc
+        .byte   2                               # Abbrev [2] DW_TAG_variable
+        .asciz  "a"                             # DW_AT_name
+        .long   .LA-.Lcu_begin0                 # DW_AT_type
+        .byte   9                               # DW_AT_location
+        .byte   3
+        .quad   a
+.LA:
+        .byte   3                               # Abbrev [3] DW_TAG_structure_type
+        .asciz  "A"                             # DW_AT_name
+                                                # DW_AT_declaration
+        .byte   4                               # Abbrev [4] DW_TAG_subprogram
+        .asciz  "A"                             # DW_AT_name
+                                                # DW_AT_declaration
+        .byte   5                               # Abbrev [5] DW_TAG_formal_parameter
+        .long   .LAptr-.Lcu_begin0              # DW_AT_type
+                                                # DW_AT_artificial
+        .byte   0                               # End Of Children Mark
+        .byte   0                               # End Of Children Mark
+.LAptr:
+        .byte   8                               # Abbrev [8] DW_TAG_pointer_type
+        .long   .LA-.Lcu_begin0                 # DW_AT_type
+        .byte   10                              # Abbrev [10] DW_TAG_subprogram
+        .quad   _ZN1AC2Ev                       # DW_AT_low_pc
+        .long   .LZN1AC2Ev_end-_ZN1AC2Ev        # DW_AT_high_pc
+        .byte   1                               # DW_AT_frame_base
+        .byte   86
+        .long   147                             # DW_AT_object_pointer
+        .long   68                              # DW_AT_specification
+        .byte   11                              # Abbrev [11] DW_TAG_formal_parameter
+        .byte   2                               # DW_AT_location
+        .byte   145
+        .byte   120
+        .asciz  "this"                          # DW_AT_name
+        .long   .LAptr-.Lcu_begin0              # DW_AT_type
+                                                # DW_AT_artificial
+        .byte   0                               # End Of Children Mark
+        .byte   0                               # End Of Children Mark
+.Ldebug_info_end0:

From a51829913dba28dae603fdcdddd242c7e20192a1 Mon Sep 17 00:00:00 2001
From: Alex Zinenko <zinenko@google.com>
Date: Wed, 22 Jul 2020 13:03:24 +0200
Subject: [PATCH 0178/1035] [mlir] Support for mutable types

Introduce support for mutable storage in the StorageUniquer infrastructure.
This makes MLIR have key-value storage instead of just uniqued key storage. A
storage instance now contains a unique immutable key and a mutable value, both
stored in the arena allocator that belongs to the context. This is a
preconditio for supporting recursive types that require delayed initialization,
in particular LLVM structure types.  The functionality is exercised in the test
pass with trivial self-recursive type. So far, recursive types can only be
printed in parsed in a closed type system. Removing this restriction is left
for future work.

Differential Revision: https://reviews.llvm.org/D84171
---
 .../Tutorials/DefiningAttributesAndTypes.md   | 131 +++++++++++++++++-
 mlir/include/mlir/IR/AttributeSupport.h       |   7 +
 mlir/include/mlir/IR/Attributes.h             |   8 +-
 mlir/include/mlir/IR/StorageUniquerSupport.h  |   8 ++
 mlir/include/mlir/IR/TypeSupport.h            |   9 ++
 mlir/include/mlir/IR/Types.h                  |  16 ++-
 mlir/include/mlir/Support/StorageUniquer.h    |  30 ++++
 mlir/lib/Support/StorageUniquer.cpp           |  16 +++
 mlir/test/IR/recursive-type.mlir              |  16 +++
 mlir/test/lib/Dialect/Test/TestDialect.cpp    |  67 ++++++++-
 mlir/test/lib/Dialect/Test/TestTypes.h        |  54 ++++++++
 mlir/test/lib/IR/CMakeLists.txt               |   1 +
 mlir/test/lib/IR/TestTypes.cpp                |  78 +++++++++++
 mlir/tools/mlir-opt/mlir-opt.cpp              |   2 +
 14 files changed, 425 insertions(+), 18 deletions(-)
 create mode 100644 mlir/test/IR/recursive-type.mlir
 create mode 100644 mlir/test/lib/IR/TestTypes.cpp

diff --git a/mlir/docs/Tutorials/DefiningAttributesAndTypes.md b/mlir/docs/Tutorials/DefiningAttributesAndTypes.md
index cab2441b33200..45756e1a31eab 100644
--- a/mlir/docs/Tutorials/DefiningAttributesAndTypes.md
+++ b/mlir/docs/Tutorials/DefiningAttributesAndTypes.md
@@ -47,7 +47,8 @@ namespace MyTypes {
 enum Kinds {
   // These kinds will be used in the examples below.
   Simple = Type::Kind::FIRST_PRIVATE_EXPERIMENTAL_0_TYPE,
-  Complex
+  Complex,
+  Recursive
 };
 }
 ```
@@ -58,13 +59,17 @@ As described above, `Type` objects in MLIR are value-typed and rely on having an
 implicitly internal storage object that holds the actual data for the type. When
 defining a new `Type` it isn't always necessary to define a new storage class.
 So before defining the derived `Type`, it's important to know which of the two
-classes of `Type` we are defining. Some types are `primitives` meaning they do
+classes of `Type` we are defining. Some types are _primitives_ meaning they do
 not have any parameters and are singletons uniqued by kind, like the
 [`index` type](LangRef.md#index-type). Parametric types on the other hand, have
 additional information that differentiates different instances of the same
 `Type` kind. For example the [`integer` type](LangRef.md#integer-type) has a
 bitwidth, making `i8` and `i16` be different instances of
-[`integer` type](LangRef.md#integer-type).
+[`integer` type](LangRef.md#integer-type). Types can also have a mutable
+component, which can be used, for example, to construct self-referring recursive
+types. The mutable component _cannot_ be used to differentiate types within the
+same kind, so usually such types are also parametric where the parameters serve
+to identify them.
 
 #### Simple non-parametric types
 
@@ -240,6 +245,126 @@ public:
 };
 ```
 
+#### Types with a mutable component
+
+Types with a mutable component require defining a type storage class regardless
+of being parametric. The storage contains both the parameters and the mutable
+component and is accessed in a thread-safe way by the type support
+infrastructure.
+
+##### Defining a type storage
+
+In addition to the requirements for the type storage class for parametric types,
+the storage class for types with a mutable component must additionally obey the
+following.
+
+*   The mutable component must not participate in the storage key.
+*   Provide a mutation method that is used to modify an existing instance of the
+    storage. This method modifies the mutable component based on arguments,
+    using `allocator` for any new dynamically-allocated storage, and indicates
+    whether the modification was successful.
+    -   `LogicalResult mutate(StorageAllocator &allocator, Args ...&& args)`
+
+Let's define a simple storage for recursive types, where a type is identified by
+its name and can contain another type including itself.
+
+```c++
+/// Here we define a storage class for a RecursiveType that is identified by its
+/// name and contains another type.
+struct RecursiveTypeStorage : public TypeStorage {
+  /// The type is uniquely identified by its name. Note that the contained type
+  /// is _not_ a part of the key.
+  using KeyTy = StringRef;
+
+  /// Construct the storage from the type name. Explicitly initialize the
+  /// containedType to nullptr, which is used as marker for the mutable
+  /// component being not yet initialized.
+  RecursiveTypeStorage(StringRef name) : name(name), containedType(nullptr) {}
+
+  /// Define the comparison function.
+  bool operator==(const KeyTy &key) const { return key == name; }
+
+  /// Define a construction method for creating a new instance of the storage.
+  static RecursiveTypeStorage *construct(StorageAllocator &allocator,
+                                         const KeyTy &key) {
+    // Note that the key string is copied into the allocator to ensure it
+    // remains live as long as the storage itself.
+    return new (allocator.allocate<RecursiveTypeStorage>())
+        RecursiveTypeStorage(allocator.copyInto(key));
+  }
+
+  /// Define a mutation method for changing the type after it is created. In
+  /// many cases, we only want to set the mutable component once and reject
+  /// any further modification, which can be achieved by returning failure from
+  /// this function.
+  LogicalResult mutate(StorageAllocator &, Type body) {
+    // If the contained type has been initialized already, and the call tries
+    // to change it, reject the change.
+    if (containedType && containedType != body)
+      return failure();
+
+    // Change the body successfully.
+    containedType = body;
+    return success();
+  }
+
+  StringRef name;
+  Type containedType;
+};
+```
+
+##### Type class definition
+
+Having defined the storage class, we can define the type class itself. This is
+similar to parametric types. `Type::TypeBase` provides a `mutate` method that
+forwards its arguments to the `mutate` method of the storage and ensures the
+modification happens under lock.
+
+```c++
+class RecursiveType : public Type::TypeBase<RecursiveType, Type,
+                                            RecursiveTypeStorage> {
+public:
+  /// Inherit parent constructors.
+  using Base::Base;
+
+  /// This static method is used to support type inquiry through isa, cast,
+  /// and dyn_cast.
+  static bool kindof(unsigned kind) { return kind == MyTypes::Recursive; }
+
+  /// Creates an instance of the Recursive type. This only takes the type name
+  /// and returns the type with uninitialized body.
+  static RecursiveType get(MLIRContext *ctx, StringRef name) {
+    // Call into the base to get a uniqued instance of this type. The parameter
+    // (name) is passed after the kind.
+    return Base::get(ctx, MyTypes::Recursive, name);
+  }
+
+  /// Now we can change the mutable component of the type. This is an instance
+  /// method callable on an already existing RecursiveType.
+  void setBody(Type body) {
+    // Call into the base to mutate the type.
+    LogicalResult result = Base::mutate(body);
+    // Most types expect mutation to always succeed, but types can implement
+    // custom logic for handling mutation failures.
+    assert(succeeded(result) &&
+           "attempting to change the body of an already-initialized type");
+    // Avoid unused-variable warning when building without assertions.
+    (void) result;
+  }
+
+  /// Returns the contained type, which may be null if it has not been
+  /// initialized yet.
+  Type getBody() {
+    return getImpl()->containedType;
+  }
+
+  /// Returns the name.
+  StringRef getName() {
+    return getImpl()->name;
+  }
+};
+```
+
 ### Registering types with a Dialect
 
 Once the dialect types have been defined, they must then be registered with a
diff --git a/mlir/include/mlir/IR/AttributeSupport.h b/mlir/include/mlir/IR/AttributeSupport.h
index cd722bb0f2c5b..72a89be438673 100644
--- a/mlir/include/mlir/IR/AttributeSupport.h
+++ b/mlir/include/mlir/IR/AttributeSupport.h
@@ -139,6 +139,13 @@ class AttributeUniquer {
         kind, std::forward<Args>(args)...);
   }
 
+  template <typename ImplType, typename... Args>
+  static LogicalResult mutate(MLIRContext *ctx, ImplType *impl,
+                              Args &&...args) {
+    assert(impl && "cannot mutate null attribute");
+    return ctx->getAttributeUniquer().mutate(impl, std::forward<Args>(args)...);
+  }
+
 private:
   /// Initialize the given attribute storage instance.
   static void initializeAttributeStorage(AttributeStorage *storage,
diff --git a/mlir/include/mlir/IR/Attributes.h b/mlir/include/mlir/IR/Attributes.h
index 89dad2ec40cf7..5ecf5763ecd47 100644
--- a/mlir/include/mlir/IR/Attributes.h
+++ b/mlir/include/mlir/IR/Attributes.h
@@ -48,10 +48,10 @@ struct SparseElementsAttributeStorage;
 
 /// Attributes are known-constant values of operations and functions.
 ///
-/// Instances of the Attribute class are references to immutable, uniqued,
-/// and immortal values owned by MLIRContext. As such, an Attribute is a thin
-/// wrapper around an underlying storage pointer. Attributes are usually passed
-/// by value.
+/// Instances of the Attribute class are references to immortal key-value pairs
+/// with immutable, uniqued key owned by MLIRContext. As such, an Attribute is a
+/// thin wrapper around an underlying storage pointer. Attributes are usually
+/// passed by value.
 class Attribute {
 public:
   /// Integer identifier for all the concrete attribute kinds.
diff --git a/mlir/include/mlir/IR/StorageUniquerSupport.h b/mlir/include/mlir/IR/StorageUniquerSupport.h
index c2250e8547163..4c7693c28d2fd 100644
--- a/mlir/include/mlir/IR/StorageUniquerSupport.h
+++ b/mlir/include/mlir/IR/StorageUniquerSupport.h
@@ -105,6 +105,14 @@ class StorageUserBase : public BaseT, public Traits<ConcreteT>... {
     return UniquerT::template get<ConcreteT>(loc.getContext(), kind, args...);
   }
 
+  /// Mutate the current storage instance. This will not change the unique key.
+  /// The arguments are forwarded to 'ConcreteT::mutate'.
+  template <typename... Args>
+  LogicalResult mutate(Args &&...args) {
+    return UniquerT::mutate(this->getContext(), getImpl(),
+                            std::forward<Args>(args)...);
+  }
+
   /// Default implementation that just returns success.
   template <typename... Args>
   static LogicalResult verifyConstructionInvariants(Args... args) {
diff --git a/mlir/include/mlir/IR/TypeSupport.h b/mlir/include/mlir/IR/TypeSupport.h
index 7961dd22d47dd..ddb91e09dc89a 100644
--- a/mlir/include/mlir/IR/TypeSupport.h
+++ b/mlir/include/mlir/IR/TypeSupport.h
@@ -132,6 +132,15 @@ struct TypeUniquer {
         },
         kind, std::forward<Args>(args)...);
   }
+
+  /// Change the mutable component of the given type instance in the provided
+  /// context.
+  template <typename ImplType, typename... Args>
+  static LogicalResult mutate(MLIRContext *ctx, ImplType *impl,
+                              Args &&...args) {
+    assert(impl && "cannot mutate null type");
+    return ctx->getTypeUniquer().mutate(impl, std::forward<Args>(args)...);
+  }
 };
 } // namespace detail
 
diff --git a/mlir/include/mlir/IR/Types.h b/mlir/include/mlir/IR/Types.h
index c14f8558d8503..83636585c499f 100644
--- a/mlir/include/mlir/IR/Types.h
+++ b/mlir/include/mlir/IR/Types.h
@@ -27,15 +27,17 @@ struct FunctionTypeStorage;
 struct OpaqueTypeStorage;
 } // namespace detail
 
-/// Instances of the Type class are immutable and uniqued.  They wrap a pointer
-/// to the storage object owned by MLIRContext.  Therefore, instances of Type
-/// are passed around by value.
+/// Instances of the Type class are uniqued, have an immutable identifier and an
+/// optional mutable component.  They wrap a pointer to the storage object owned
+/// by MLIRContext.  Therefore, instances of Type are passed around by value.
 ///
 /// Some types are "primitives" meaning they do not have any parameters, for
 /// example the Index type.  Parametric types have additional information that
 /// differentiates the types of the same kind between them, for example the
 /// Integer type has bitwidth, making i8 and i16 belong to the same kind by be
-/// different instances of the IntegerType.
+/// different instances of the IntegerType.  Type parameters are part of the
+/// unique immutable key.  The mutable component of the type can be modified
+/// after the type is created, but cannot affect the identity of the type.
 ///
 /// Types are constructed and uniqued via the 'detail::TypeUniquer' class.
 ///
@@ -62,6 +64,7 @@ struct OpaqueTypeStorage;
 ///    - The type kind (for LLVM-style RTTI).
 ///    - The dialect that defined the type.
 ///    - Any parameters of the type.
+///    - An optional mutable component.
 /// For non-parametric types, a convenience DefaultTypeStorage is provided.
 /// Parametric storage types must derive TypeStorage and respect the following:
 ///    - Define a type alias, KeyTy, to a type that uniquely identifies the
@@ -75,11 +78,14 @@ struct OpaqueTypeStorage;
 ///    - Provide a method, 'bool operator==(const KeyTy &) const', to
 ///      compare the storage instance against an instance of the key type.
 ///
-///    - Provide a construction method:
+///    - Provide a static construction method:
 ///        'DerivedStorage *construct(TypeStorageAllocator &, const KeyTy &key)'
 ///      that builds a unique instance of the derived storage. The arguments to
 ///      this function are an allocator to store any uniqued data within the
 ///      context and the key type for this storage.
+///
+///    - If they have a mutable component, this component must not be a part of
+//       the key.
 class Type {
 public:
   /// Integer identifier for all the concrete type kinds.
diff --git a/mlir/include/mlir/Support/StorageUniquer.h b/mlir/include/mlir/Support/StorageUniquer.h
index f13a2fef9d50e..3100b4454197b 100644
--- a/mlir/include/mlir/Support/StorageUniquer.h
+++ b/mlir/include/mlir/Support/StorageUniquer.h
@@ -10,6 +10,7 @@
 #define MLIR_SUPPORT_STORAGEUNIQUER_H
 
 #include "mlir/Support/LLVM.h"
+#include "mlir/Support/LogicalResult.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/Support/Allocator.h"
 
@@ -60,6 +61,20 @@ using has_impltype_hash_t = decltype(ImplTy::hashKey(std::declval<T>()));
 ///      that is called when erasing a storage instance. This should cleanup any
 ///      fields of the storage as necessary and not attempt to free the memory
 ///      of the storage itself.
+///
+/// Storage classes may have an optional mutable component, which must not take
+/// part in the unique immutable key. In this case, storage classes may be
+/// mutated with `mutate` and must additionally respect the following:
+///    - Provide a mutation method:
+///        'LogicalResult mutate(StorageAllocator &, <...>)'
+///      that is called when mutating a storage instance. The first argument is
+///      an allocator to store any mutable data, and the remaining arguments are
+///      forwarded from the call site. The storage can be mutated at any time
+///      after creation. Care must be taken to avoid excessive mutation since
+///      the allocated storage can keep containing previous states. The return
+///      value of the function is used to indicate whether the mutation was
+///      successful, e.g., to limit the number of mutations or enable deferred
+///      one-time assignment of the mutable component.
 class StorageUniquer {
 public:
   StorageUniquer();
@@ -166,6 +181,17 @@ class StorageUniquer {
     return static_cast<Storage *>(getImpl(kind, ctorFn));
   }
 
+  /// Changes the mutable component of 'storage' by forwarding the trailing
+  /// arguments to the 'mutate' function of the derived class.
+  template <typename Storage, typename... Args>
+  LogicalResult mutate(Storage *storage, Args &&...args) {
+    auto mutationFn = [&](StorageAllocator &allocator) -> LogicalResult {
+      return static_cast<Storage &>(*storage).mutate(
+          allocator, std::forward<Args>(args)...);
+    };
+    return mutateImpl(mutationFn);
+  }
+
   /// Erases a uniqued instance of 'Storage'. This function is used for derived
   /// types that have complex storage or uniquing constraints.
   template <typename Storage, typename Arg, typename... Args>
@@ -206,6 +232,10 @@ class StorageUniquer {
                  function_ref<bool(const BaseStorage *)> isEqual,
                  function_ref<void(BaseStorage *)> cleanupFn);
 
+  /// Implementation for mutating an instance of a derived storage.
+  LogicalResult
+  mutateImpl(function_ref<LogicalResult(StorageAllocator &)> mutationFn);
+
   /// The internal implementation class.
   std::unique_ptr<detail::StorageUniquerImpl> impl;
 
diff --git a/mlir/lib/Support/StorageUniquer.cpp b/mlir/lib/Support/StorageUniquer.cpp
index 40304a544c4fb..f7c953e98140f 100644
--- a/mlir/lib/Support/StorageUniquer.cpp
+++ b/mlir/lib/Support/StorageUniquer.cpp
@@ -124,6 +124,16 @@ struct StorageUniquerImpl {
     storageTypes.erase(existing);
   }
 
+  /// Mutates an instance of a derived storage in a thread-safe way.
+  LogicalResult
+  mutate(function_ref<LogicalResult(StorageAllocator &)> mutationFn) {
+    if (!threadingIsEnabled)
+      return mutationFn(allocator);
+
+    llvm::sys::SmartScopedWriter<true> lock(mutex);
+    return mutationFn(allocator);
+  }
+
   //===--------------------------------------------------------------------===//
   // Instance Storage
   //===--------------------------------------------------------------------===//
@@ -214,3 +224,9 @@ void StorageUniquer::eraseImpl(unsigned kind, unsigned hashValue,
                                function_ref<void(BaseStorage *)> cleanupFn) {
   impl->erase(kind, hashValue, isEqual, cleanupFn);
 }
+
+/// Implementation for mutating an instance of a derived storage.
+LogicalResult StorageUniquer::mutateImpl(
+    function_ref<LogicalResult(StorageAllocator &)> mutationFn) {
+  return impl->mutate(mutationFn);
+}
diff --git a/mlir/test/IR/recursive-type.mlir b/mlir/test/IR/recursive-type.mlir
new file mode 100644
index 0000000000000..6f90c3c780518
--- /dev/null
+++ b/mlir/test/IR/recursive-type.mlir
@@ -0,0 +1,16 @@
+// RUN: mlir-opt %s -test-recursive-types | FileCheck %s
+
+// CHECK-LABEL: @roundtrip
+func @roundtrip() {
+  // CHECK: !test.test_rec<a, test_rec<b, test_type>>
+  "test.dummy_op_for_roundtrip"() : () -> !test.test_rec<a, test_rec<b, test_type>>
+  // CHECK: !test.test_rec<c, test_rec<c>>
+  "test.dummy_op_for_roundtrip"() : () -> !test.test_rec<c, test_rec<c>>
+  return
+}
+
+// CHECK-LABEL: @create
+func @create() {
+  // CHECK: !test.test_rec<some_long_and_unique_name, test_rec<some_long_and_unique_name>>
+  return
+}
diff --git a/mlir/test/lib/Dialect/Test/TestDialect.cpp b/mlir/test/lib/Dialect/Test/TestDialect.cpp
index 47aa86c45cc64..cdbf974679bde 100644
--- a/mlir/test/lib/Dialect/Test/TestDialect.cpp
+++ b/mlir/test/lib/Dialect/Test/TestDialect.cpp
@@ -16,6 +16,7 @@
 #include "mlir/IR/TypeUtilities.h"
 #include "mlir/Transforms/FoldUtils.h"
 #include "mlir/Transforms/InliningUtils.h"
+#include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/StringSwitch.h"
 
 using namespace mlir;
@@ -137,19 +138,73 @@ TestDialect::TestDialect(MLIRContext *context)
       >();
   addInterfaces<TestOpAsmInterface, TestOpFolderDialectInterface,
                 TestInlinerInterface>();
-  addTypes<TestType>();
+  addTypes<TestType, TestRecursiveType>();
   allowUnknownOperations();
 }
 
-Type TestDialect::parseType(DialectAsmParser &parser) const {
-  if (failed(parser.parseKeyword("test_type")))
+static Type parseTestType(DialectAsmParser &parser,
+                          llvm::SetVector<Type> &stack) {
+  StringRef typeTag;
+  if (failed(parser.parseKeyword(&typeTag)))
+    return Type();
+
+  if (typeTag == "test_type")
+    return TestType::get(parser.getBuilder().getContext());
+
+  if (typeTag != "test_rec")
+    return Type();
+
+  StringRef name;
+  if (parser.parseLess() || parser.parseKeyword(&name))
+    return Type();
+  auto rec = TestRecursiveType::create(parser.getBuilder().getContext(), name);
+
+  // If this type already has been parsed above in the stack, expect just the
+  // name.
+  if (stack.contains(rec)) {
+    if (failed(parser.parseGreater()))
+      return Type();
+    return rec;
+  }
+
+  // Otherwise, parse the body and update the type.
+  if (failed(parser.parseComma()))
+    return Type();
+  stack.insert(rec);
+  Type subtype = parseTestType(parser, stack);
+  stack.pop_back();
+  if (!subtype || failed(parser.parseGreater()) || failed(rec.setBody(subtype)))
     return Type();
-  return TestType::get(getContext());
+
+  return rec;
+}
+
+Type TestDialect::parseType(DialectAsmParser &parser) const {
+  llvm::SetVector<Type> stack;
+  return parseTestType(parser, stack);
+}
+
+static void printTestType(Type type, DialectAsmPrinter &printer,
+                          llvm::SetVector<Type> &stack) {
+  if (type.isa<TestType>()) {
+    printer << "test_type";
+    return;
+  }
+
+  auto rec = type.cast<TestRecursiveType>();
+  printer << "test_rec<" << rec.getName();
+  if (!stack.contains(rec)) {
+    printer << ", ";
+    stack.insert(rec);
+    printTestType(rec.getBody(), printer, stack);
+    stack.pop_back();
+  }
+  printer << ">";
 }
 
 void TestDialect::printType(Type type, DialectAsmPrinter &printer) const {
-  assert(type.isa<TestType>() && "unexpected type");
-  printer << "test_type";
+  llvm::SetVector<Type> stack;
+  printTestType(type, printer, stack);
 }
 
 LogicalResult TestDialect::verifyOperationAttribute(Operation *op,
diff --git a/mlir/test/lib/Dialect/Test/TestTypes.h b/mlir/test/lib/Dialect/Test/TestTypes.h
index 0596f61c1fa1c..9e2c297c6a892 100644
--- a/mlir/test/lib/Dialect/Test/TestTypes.h
+++ b/mlir/test/lib/Dialect/Test/TestTypes.h
@@ -39,6 +39,60 @@ struct TestType : public Type::TypeBase<TestType, Type, TypeStorage,
     emitRemark(loc) << *this << " - TestC";
   }
 };
+
+/// Storage for simple named recursive types, where the type is identified by
+/// its name and can "contain" another type, including itself.
+struct TestRecursiveTypeStorage : public TypeStorage {
+  using KeyTy = StringRef;
+
+  explicit TestRecursiveTypeStorage(StringRef key) : name(key), body(Type()) {}
+
+  bool operator==(const KeyTy &other) const { return name == other; }
+
+  static TestRecursiveTypeStorage *construct(TypeStorageAllocator &allocator,
+                                             const KeyTy &key) {
+    return new (allocator.allocate<TestRecursiveTypeStorage>())
+        TestRecursiveTypeStorage(allocator.copyInto(key));
+  }
+
+  LogicalResult mutate(TypeStorageAllocator &allocator, Type newBody) {
+    // Cannot set a different body than before.
+    if (body && body != newBody)
+      return failure();
+
+    body = newBody;
+    return success();
+  }
+
+  StringRef name;
+  Type body;
+};
+
+/// Simple recursive type identified by its name and pointing to another named
+/// type, potentially itself. This requires the body to be mutated separately
+/// from type creation.
+class TestRecursiveType
+    : public Type::TypeBase<TestRecursiveType, Type, TestRecursiveTypeStorage> {
+public:
+  using Base::Base;
+
+  static bool kindof(unsigned kind) {
+    return kind == Type::Kind::FIRST_PRIVATE_EXPERIMENTAL_9_TYPE + 1;
+  }
+
+  static TestRecursiveType create(MLIRContext *ctx, StringRef name) {
+    return Base::get(ctx, Type::Kind::FIRST_PRIVATE_EXPERIMENTAL_9_TYPE + 1,
+                     name);
+  }
+
+  /// Body getter and setter.
+  LogicalResult setBody(Type body) { return Base::mutate(body); }
+  Type getBody() { return getImpl()->body; }
+
+  /// Name/key getter.
+  StringRef getName() { return getImpl()->name; }
+};
+
 } // end namespace mlir
 
 #endif // MLIR_TESTTYPES_H
diff --git a/mlir/test/lib/IR/CMakeLists.txt b/mlir/test/lib/IR/CMakeLists.txt
index 5456dc9e8816e..f77b26e5ca184 100644
--- a/mlir/test/lib/IR/CMakeLists.txt
+++ b/mlir/test/lib/IR/CMakeLists.txt
@@ -5,6 +5,7 @@ add_mlir_library(MLIRTestIR
   TestMatchers.cpp
   TestSideEffects.cpp
   TestSymbolUses.cpp
+  TestTypes.cpp
 
   EXCLUDE_FROM_LIBMLIR
 
diff --git a/mlir/test/lib/IR/TestTypes.cpp b/mlir/test/lib/IR/TestTypes.cpp
new file mode 100644
index 0000000000000..f62c06eededfb
--- /dev/null
+++ b/mlir/test/lib/IR/TestTypes.cpp
@@ -0,0 +1,78 @@
+//===- TestTypes.cpp - Test passes for MLIR types -------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "TestTypes.h"
+#include "TestDialect.h"
+#include "mlir/Pass/Pass.h"
+
+using namespace mlir;
+
+namespace {
+struct TestRecursiveTypesPass
+    : public PassWrapper<TestRecursiveTypesPass, FunctionPass> {
+  LogicalResult createIRWithTypes();
+
+  void runOnFunction() override {
+    FuncOp func = getFunction();
+
+    // Just make sure recurisve types are printed and parsed.
+    if (func.getName() == "roundtrip")
+      return;
+
+    // Create a recursive type and print it as a part of a dummy op.
+    if (func.getName() == "create") {
+      if (failed(createIRWithTypes()))
+        signalPassFailure();
+      return;
+    }
+
+    // Unknown key.
+    func.emitOpError() << "unexpected function name";
+    signalPassFailure();
+  }
+};
+} // end namespace
+
+LogicalResult TestRecursiveTypesPass::createIRWithTypes() {
+  MLIRContext *ctx = &getContext();
+  FuncOp func = getFunction();
+  auto type = TestRecursiveType::create(ctx, "some_long_and_unique_name");
+  if (failed(type.setBody(type)))
+    return func.emitError("expected to be able to set the type body");
+
+  // Setting the same body is fine.
+  if (failed(type.setBody(type)))
+    return func.emitError(
+        "expected to be able to set the type body to the same value");
+
+  // Setting a different body is not.
+  if (succeeded(type.setBody(IndexType::get(ctx))))
+    return func.emitError(
+        "not expected to be able to change function body more than once");
+
+  // Expecting to get the same type for the same name.
+  auto other = TestRecursiveType::create(ctx, "some_long_and_unique_name");
+  if (type != other)
+    return func.emitError("expected type name to be the uniquing key");
+
+  // Create the op to check how the type is printed.
+  OperationState state(func.getLoc(), "test.dummy_type_test_op");
+  state.addTypes(type);
+  func.getBody().front().push_front(Operation::create(state));
+
+  return success();
+}
+
+namespace mlir {
+
+void registerTestRecursiveTypesPass() {
+  PassRegistration<TestRecursiveTypesPass> reg(
+      "test-recursive-types", "Test support for recursive types");
+}
+
+} // end namespace mlir
diff --git a/mlir/tools/mlir-opt/mlir-opt.cpp b/mlir/tools/mlir-opt/mlir-opt.cpp
index f749c7ad98adf..f60864a6a371b 100644
--- a/mlir/tools/mlir-opt/mlir-opt.cpp
+++ b/mlir/tools/mlir-opt/mlir-opt.cpp
@@ -63,6 +63,7 @@ void registerTestMemRefDependenceCheck();
 void registerTestMemRefStrideCalculation();
 void registerTestOpaqueLoc();
 void registerTestPreparationPassWithAllowedMemrefResults();
+void registerTestRecursiveTypesPass();
 void registerTestReducer();
 void registerTestGpuParallelLoopMappingPass();
 void registerTestSCFUtilsPass();
@@ -138,6 +139,7 @@ void registerTestPasses() {
   registerTestMemRefStrideCalculation();
   registerTestOpaqueLoc();
   registerTestPreparationPassWithAllowedMemrefResults();
+  registerTestRecursiveTypesPass();
   registerTestReducer();
   registerTestGpuParallelLoopMappingPass();
   registerTestSCFUtilsPass();

From 102997cd0edb127050553a33a902bd6621c2c184 Mon Sep 17 00:00:00 2001
From: Hans Wennborg <hans@chromium.org>
Date: Mon, 27 Jul 2020 13:35:35 +0200
Subject: [PATCH 0179/1035] Speculative build fix for
 scudo/standalone/tests/combined_test.cpp

---
 compiler-rt/lib/scudo/standalone/tests/combined_test.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp b/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp
index b5035d05aabb0..c144ad0ae32a3 100644
--- a/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp
+++ b/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp
@@ -12,6 +12,7 @@
 #include "combined.h"
 
 #include <condition_variable>
+#include <memory>
 #include <mutex>
 #include <thread>
 #include <vector>

From 47a0254229ca425aa4e169c2db14e92b8db86784 Mon Sep 17 00:00:00 2001
From: Hans Wennborg <hans@chromium.org>
Date: Mon, 27 Jul 2020 13:39:54 +0200
Subject: [PATCH 0180/1035] Speculative build fix for clangd/Features.inc.in

---
 clang-tools-extra/clangd/Features.inc.in | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang-tools-extra/clangd/Features.inc.in b/clang-tools-extra/clangd/Features.inc.in
index 8584b87c62051..6797232ddac7c 100644
--- a/clang-tools-extra/clangd/Features.inc.in
+++ b/clang-tools-extra/clangd/Features.inc.in
@@ -1,2 +1,2 @@
 #define CLANGD_BUILD_XPC @CLANGD_BUILD_XPC@
-#define CLANGD_ENABLE_REMOTE @CLANGD_ENABLE_REMTE@
+#define CLANGD_ENABLE_REMOTE @CLANGD_ENABLE_REMOTE@

From 13c9bbc28ef9cf9976a0962e6c930a7dfc52c877 Mon Sep 17 00:00:00 2001
From: Nathan James <n.james93@hotmail.co.uk>
Date: Mon, 27 Jul 2020 12:48:53 +0100
Subject: [PATCH 0181/1035] [clang-tidy] Refactor IncludeInserter

Simplified how `IncludeInserter` is used in Checks by abstracting away the SourceManager and PPCallbacks inside the method `registerPreprocessor`.
Changed checks that use `IncludeInserter` to no longer use a `std::unique_ptr`, instead the IncludeInserter is just a member of the class thats initialized with an `IncludeStyle`.
Saving an unnecessary allocation.

This results in the removal of the field `IncludeSorter::IncludeStyle` from the checks, as its wrapped in the `IncludeInserter`.
No longer need to create an instance of the `IncludeInserter` in the registerPPCallbacks, now that method only needs to contain:
```
Inserter.registerPreprocessor(PP);
```
Also added a helper method to `IncludeInserter` called `createMainFileInclusionInsertion`, purely sugar but does better express intentions.

Reviewed By: aaron.ballman

Differential Revision: https://reviews.llvm.org/D83680
---
 .../abseil/StringFindStartswithCheck.cpp      | 12 ++--
 .../abseil/StringFindStartswithCheck.h        |  3 +-
 .../cppcoreguidelines/InitVariablesCheck.cpp  | 12 ++--
 .../cppcoreguidelines/InitVariablesCheck.h    |  3 +-
 .../ProBoundsConstantArrayIndexCheck.cpp      | 15 ++---
 .../ProBoundsConstantArrayIndexCheck.h        |  3 +-
 .../modernize/MakeSmartPtrCheck.cpp           | 12 ++--
 .../clang-tidy/modernize/MakeSmartPtrCheck.h  |  3 +-
 .../clang-tidy/modernize/PassByValueCheck.cpp | 12 ++--
 .../clang-tidy/modernize/PassByValueCheck.h   |  3 +-
 .../modernize/ReplaceAutoPtrCheck.cpp         | 21 +++----
 .../modernize/ReplaceAutoPtrCheck.h           |  3 +-
 .../modernize/ReplaceRandomShuffleCheck.cpp   | 13 ++--
 .../modernize/ReplaceRandomShuffleCheck.h     |  3 +-
 .../performance/MoveConstructorInitCheck.cpp  | 10 ++--
 .../performance/MoveConstructorInitCheck.h    |  3 +-
 .../TypePromotionInMathFnCheck.cpp            | 13 ++--
 .../performance/TypePromotionInMathFnCheck.h  |  3 +-
 .../UnnecessaryValueParamCheck.cpp            | 12 ++--
 .../performance/UnnecessaryValueParamCheck.h  |  3 +-
 .../clang-tidy/utils/IncludeInserter.cpp      | 46 ++++++++++-----
 .../clang-tidy/utils/IncludeInserter.h        | 59 +++++++++++--------
 .../utils/TransformerClangTidyCheck.cpp       | 23 ++++----
 .../utils/TransformerClangTidyCheck.h         |  3 +-
 .../clang-tidy/IncludeInserterTest.cpp        | 10 ++--
 25 files changed, 149 insertions(+), 154 deletions(-)

diff --git a/clang-tools-extra/clang-tidy/abseil/StringFindStartswithCheck.cpp b/clang-tools-extra/clang-tidy/abseil/StringFindStartswithCheck.cpp
index 11bbcbcb527f5..e775fc21d2d0f 100644
--- a/clang-tools-extra/clang-tidy/abseil/StringFindStartswithCheck.cpp
+++ b/clang-tools-extra/clang-tidy/abseil/StringFindStartswithCheck.cpp
@@ -26,8 +26,8 @@ StringFindStartswithCheck::StringFindStartswithCheck(StringRef Name,
     : ClangTidyCheck(Name, Context),
       StringLikeClasses(utils::options::parseStringList(
           Options.get("StringLikeClasses", "::std::basic_string"))),
-      IncludeStyle(Options.getLocalOrGlobal("IncludeStyle",
-                                            utils::IncludeSorter::IS_LLVM)),
+      IncludeInserter(Options.getLocalOrGlobal("IncludeStyle",
+                                               utils::IncludeSorter::IS_LLVM)),
       AbseilStringsMatchHeader(
           Options.get("AbseilStringsMatchHeader", "absl/strings/match.h")) {}
 
@@ -105,23 +105,21 @@ void StringFindStartswithCheck::check(const MatchFinder::MatchResult &Result) {
 
   // Create a preprocessor #include FixIt hint (CreateIncludeInsertion checks
   // whether this already exists).
-  Diagnostic << IncludeInserter->CreateIncludeInsertion(
+  Diagnostic << IncludeInserter.createIncludeInsertion(
       Source.getFileID(ComparisonExpr->getBeginLoc()), AbseilStringsMatchHeader,
       false);
 }
 
 void StringFindStartswithCheck::registerPPCallbacks(
     const SourceManager &SM, Preprocessor *PP, Preprocessor *ModuleExpanderPP) {
-  IncludeInserter = std::make_unique<utils::IncludeInserter>(SM, getLangOpts(),
-                                                              IncludeStyle);
-  PP->addPPCallbacks(IncludeInserter->CreatePPCallbacks());
+  IncludeInserter.registerPreprocessor(PP);
 }
 
 void StringFindStartswithCheck::storeOptions(
     ClangTidyOptions::OptionMap &Opts) {
   Options.store(Opts, "StringLikeClasses",
                 utils::options::serializeStringList(StringLikeClasses));
-  Options.store(Opts, "IncludeStyle", IncludeStyle);
+  Options.store(Opts, "IncludeStyle", IncludeInserter.getStyle());
   Options.store(Opts, "AbseilStringsMatchHeader", AbseilStringsMatchHeader);
 }
 
diff --git a/clang-tools-extra/clang-tidy/abseil/StringFindStartswithCheck.h b/clang-tools-extra/clang-tidy/abseil/StringFindStartswithCheck.h
index d232d3b3efb61..2bb20f78b68ce 100644
--- a/clang-tools-extra/clang-tidy/abseil/StringFindStartswithCheck.h
+++ b/clang-tools-extra/clang-tidy/abseil/StringFindStartswithCheck.h
@@ -35,9 +35,8 @@ class StringFindStartswithCheck : public ClangTidyCheck {
   void storeOptions(ClangTidyOptions::OptionMap &Opts) override;
 
 private:
-  std::unique_ptr<clang::tidy::utils::IncludeInserter> IncludeInserter;
   const std::vector<std::string> StringLikeClasses;
-  const utils::IncludeSorter::IncludeStyle IncludeStyle;
+  utils::IncludeInserter IncludeInserter;
   const std::string AbseilStringsMatchHeader;
 };
 
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/InitVariablesCheck.cpp b/clang-tools-extra/clang-tidy/cppcoreguidelines/InitVariablesCheck.cpp
index f1755d3f9b855..3f51ef595b463 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/InitVariablesCheck.cpp
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/InitVariablesCheck.cpp
@@ -26,12 +26,12 @@ AST_MATCHER(VarDecl, isLocalVarDecl) { return Node.isLocalVarDecl(); }
 InitVariablesCheck::InitVariablesCheck(StringRef Name,
                                        ClangTidyContext *Context)
     : ClangTidyCheck(Name, Context),
-      IncludeStyle(Options.getLocalOrGlobal("IncludeStyle",
-                                            utils::IncludeSorter::IS_LLVM)),
+      IncludeInserter(Options.getLocalOrGlobal("IncludeStyle",
+                                               utils::IncludeSorter::IS_LLVM)),
       MathHeader(Options.get("MathHeader", "math.h")) {}
 
 void InitVariablesCheck::storeOptions(ClangTidyOptions::OptionMap &Opts) {
-  Options.store(Opts, "IncludeStyle", IncludeStyle);
+  Options.store(Opts, "IncludeStyle", IncludeInserter.getStyle());
   Options.store(Opts, "MathHeader", MathHeader);
 }
 
@@ -51,9 +51,7 @@ void InitVariablesCheck::registerMatchers(MatchFinder *Finder) {
 void InitVariablesCheck::registerPPCallbacks(const SourceManager &SM,
                                              Preprocessor *PP,
                                              Preprocessor *ModuleExpanderPP) {
-  IncludeInserter =
-      std::make_unique<utils::IncludeInserter>(SM, getLangOpts(), IncludeStyle);
-  PP->addPPCallbacks(IncludeInserter->CreatePPCallbacks());
+  IncludeInserter.registerPreprocessor(PP);
 }
 
 void InitVariablesCheck::check(const MatchFinder::MatchResult &Result) {
@@ -104,7 +102,7 @@ void InitVariablesCheck::check(const MatchFinder::MatchResult &Result) {
                    MatchedDecl->getName().size()),
                InitializationString);
     if (AddMathInclude) {
-      Diagnostic << IncludeInserter->CreateIncludeInsertion(
+      Diagnostic << IncludeInserter.createIncludeInsertion(
           Source.getFileID(MatchedDecl->getBeginLoc()), MathHeader, false);
     }
   }
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/InitVariablesCheck.h b/clang-tools-extra/clang-tidy/cppcoreguidelines/InitVariablesCheck.h
index 61521b118a99e..0f778104ce398 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/InitVariablesCheck.h
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/InitVariablesCheck.h
@@ -31,8 +31,7 @@ class InitVariablesCheck : public ClangTidyCheck {
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
 
 private:
-  std::unique_ptr<clang::tidy::utils::IncludeInserter> IncludeInserter;
-  const utils::IncludeSorter::IncludeStyle IncludeStyle;
+  utils::IncludeInserter IncludeInserter;
   const std::string MathHeader;
 };
 
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsConstantArrayIndexCheck.cpp b/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsConstantArrayIndexCheck.cpp
index 96b0bb0f9b02d..f45801f1ea723 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsConstantArrayIndexCheck.cpp
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsConstantArrayIndexCheck.cpp
@@ -21,20 +21,18 @@ namespace cppcoreguidelines {
 ProBoundsConstantArrayIndexCheck::ProBoundsConstantArrayIndexCheck(
     StringRef Name, ClangTidyContext *Context)
     : ClangTidyCheck(Name, Context), GslHeader(Options.get("GslHeader", "")),
-      IncludeStyle(Options.getLocalOrGlobal("IncludeStyle",
-                                            utils::IncludeSorter::IS_LLVM)) {}
+      Inserter(Options.getLocalOrGlobal("IncludeStyle",
+                                        utils::IncludeSorter::IS_LLVM)) {}
 
 void ProBoundsConstantArrayIndexCheck::storeOptions(
     ClangTidyOptions::OptionMap &Opts) {
   Options.store(Opts, "GslHeader", GslHeader);
-  Options.store(Opts, "IncludeStyle", IncludeStyle);
+  Options.store(Opts, "IncludeStyle", Inserter.getStyle());
 }
 
 void ProBoundsConstantArrayIndexCheck::registerPPCallbacks(
     const SourceManager &SM, Preprocessor *PP, Preprocessor *ModuleExpanderPP) {
-  Inserter = std::make_unique<utils::IncludeInserter>(SM, getLangOpts(),
-                                                       IncludeStyle);
-  PP->addPPCallbacks(Inserter->CreatePPCallbacks());
+  Inserter.registerPreprocessor(PP);
 }
 
 void ProBoundsConstantArrayIndexCheck::registerMatchers(MatchFinder *Finder) {
@@ -87,9 +85,8 @@ void ProBoundsConstantArrayIndexCheck::check(
                               IndexRange.getBegin().getLocWithOffset(-1)),
                   ", ")
            << FixItHint::CreateReplacement(Matched->getEndLoc(), ")")
-           << Inserter->CreateIncludeInsertion(
-                  Result.SourceManager->getMainFileID(), GslHeader,
-                  /*IsAngled=*/false);
+           << Inserter.createMainFileIncludeInsertion(GslHeader,
+                                                      /*IsAngled=*/false);
     }
     return;
   }
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsConstantArrayIndexCheck.h b/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsConstantArrayIndexCheck.h
index ac7475b4372db..04a51b93a04c5 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsConstantArrayIndexCheck.h
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsConstantArrayIndexCheck.h
@@ -23,8 +23,7 @@ namespace cppcoreguidelines {
 /// http://clang.llvm.org/extra/clang-tidy/checks/cppcoreguidelines-pro-bounds-constant-array-index.html
 class ProBoundsConstantArrayIndexCheck : public ClangTidyCheck {
   const std::string GslHeader;
-  const utils::IncludeSorter::IncludeStyle IncludeStyle;
-  std::unique_ptr<utils::IncludeInserter> Inserter;
+  utils::IncludeInserter Inserter;
 
 public:
   ProBoundsConstantArrayIndexCheck(StringRef Name, ClangTidyContext *Context);
diff --git a/clang-tools-extra/clang-tidy/modernize/MakeSmartPtrCheck.cpp b/clang-tools-extra/clang-tidy/modernize/MakeSmartPtrCheck.cpp
index c677043946f7f..5818b8cd06b5c 100644
--- a/clang-tools-extra/clang-tidy/modernize/MakeSmartPtrCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/MakeSmartPtrCheck.cpp
@@ -44,8 +44,8 @@ const char MakeSmartPtrCheck::PointerType[] = "pointerType";
 MakeSmartPtrCheck::MakeSmartPtrCheck(StringRef Name, ClangTidyContext *Context,
                                      StringRef MakeSmartPtrFunctionName)
     : ClangTidyCheck(Name, Context),
-      IncludeStyle(Options.getLocalOrGlobal("IncludeStyle",
-                                            utils::IncludeSorter::IS_LLVM)),
+      Inserter(Options.getLocalOrGlobal("IncludeStyle",
+                                        utils::IncludeSorter::IS_LLVM)),
       MakeSmartPtrFunctionHeader(
           Options.get("MakeSmartPtrFunctionHeader", StdMemoryHeader)),
       MakeSmartPtrFunctionName(
@@ -53,7 +53,7 @@ MakeSmartPtrCheck::MakeSmartPtrCheck(StringRef Name, ClangTidyContext *Context,
       IgnoreMacros(Options.getLocalOrGlobal("IgnoreMacros", true)) {}
 
 void MakeSmartPtrCheck::storeOptions(ClangTidyOptions::OptionMap &Opts) {
-  Options.store(Opts, "IncludeStyle", IncludeStyle);
+  Options.store(Opts, "IncludeStyle", Inserter.getStyle());
   Options.store(Opts, "MakeSmartPtrFunctionHeader", MakeSmartPtrFunctionHeader);
   Options.store(Opts, "MakeSmartPtrFunction", MakeSmartPtrFunctionName);
   Options.store(Opts, "IgnoreMacros", IgnoreMacros);
@@ -67,9 +67,7 @@ bool MakeSmartPtrCheck::isLanguageVersionSupported(
 void MakeSmartPtrCheck::registerPPCallbacks(const SourceManager &SM,
                                             Preprocessor *PP,
                                             Preprocessor *ModuleExpanderPP) {
-    Inserter = std::make_unique<utils::IncludeInserter>(SM, getLangOpts(),
-                                                         IncludeStyle);
-    PP->addPPCallbacks(Inserter->CreatePPCallbacks());
+  Inserter.registerPreprocessor(PP);
 }
 
 void MakeSmartPtrCheck::registerMatchers(ast_matchers::MatchFinder *Finder) {
@@ -432,7 +430,7 @@ void MakeSmartPtrCheck::insertHeader(DiagnosticBuilder &Diag, FileID FD) {
   if (MakeSmartPtrFunctionHeader.empty()) {
     return;
   }
-  Diag << Inserter->CreateIncludeInsertion(
+  Diag << Inserter.createIncludeInsertion(
       FD, MakeSmartPtrFunctionHeader,
       /*IsAngled=*/MakeSmartPtrFunctionHeader == StdMemoryHeader);
 }
diff --git a/clang-tools-extra/clang-tidy/modernize/MakeSmartPtrCheck.h b/clang-tools-extra/clang-tidy/modernize/MakeSmartPtrCheck.h
index 1f738737ab264..7a1bba624c539 100644
--- a/clang-tools-extra/clang-tidy/modernize/MakeSmartPtrCheck.h
+++ b/clang-tools-extra/clang-tidy/modernize/MakeSmartPtrCheck.h
@@ -46,8 +46,7 @@ class MakeSmartPtrCheck : public ClangTidyCheck {
   static const char PointerType[];
 
 private:
-  std::unique_ptr<utils::IncludeInserter> Inserter;
-  const utils::IncludeSorter::IncludeStyle IncludeStyle;
+  utils::IncludeInserter Inserter;
   const std::string MakeSmartPtrFunctionHeader;
   const std::string MakeSmartPtrFunctionName;
   const bool IgnoreMacros;
diff --git a/clang-tools-extra/clang-tidy/modernize/PassByValueCheck.cpp b/clang-tools-extra/clang-tidy/modernize/PassByValueCheck.cpp
index b6dedfbc2b6eb..b955ea7f7572b 100644
--- a/clang-tools-extra/clang-tidy/modernize/PassByValueCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/PassByValueCheck.cpp
@@ -120,12 +120,12 @@ collectParamDecls(const CXXConstructorDecl *Ctor,
 
 PassByValueCheck::PassByValueCheck(StringRef Name, ClangTidyContext *Context)
     : ClangTidyCheck(Name, Context),
-      IncludeStyle(Options.getLocalOrGlobal("IncludeStyle",
-                                            utils::IncludeSorter::IS_LLVM)),
+      Inserter(Options.getLocalOrGlobal("IncludeStyle",
+                                        utils::IncludeSorter::IS_LLVM)),
       ValuesOnly(Options.get("ValuesOnly", false)) {}
 
 void PassByValueCheck::storeOptions(ClangTidyOptions::OptionMap &Opts) {
-  Options.store(Opts, "IncludeStyle", IncludeStyle);
+  Options.store(Opts, "IncludeStyle", Inserter.getStyle());
   Options.store(Opts, "ValuesOnly", ValuesOnly);
 }
 
@@ -167,9 +167,7 @@ void PassByValueCheck::registerMatchers(MatchFinder *Finder) {
 void PassByValueCheck::registerPPCallbacks(const SourceManager &SM,
                                            Preprocessor *PP,
                                            Preprocessor *ModuleExpanderPP) {
-    Inserter = std::make_unique<utils::IncludeInserter>(SM, getLangOpts(),
-                                                         IncludeStyle);
-    PP->addPPCallbacks(Inserter->CreatePPCallbacks());
+  Inserter.registerPreprocessor(PP);
 }
 
 void PassByValueCheck::check(const MatchFinder::MatchResult &Result) {
@@ -216,7 +214,7 @@ void PassByValueCheck::check(const MatchFinder::MatchResult &Result) {
   Diag << FixItHint::CreateInsertion(Initializer->getRParenLoc(), ")")
        << FixItHint::CreateInsertion(
               Initializer->getLParenLoc().getLocWithOffset(1), "std::move(")
-       << Inserter->CreateIncludeInsertion(
+       << Inserter.createIncludeInsertion(
               Result.SourceManager->getFileID(Initializer->getSourceLocation()),
               "utility",
               /*IsAngled=*/true);
diff --git a/clang-tools-extra/clang-tidy/modernize/PassByValueCheck.h b/clang-tools-extra/clang-tidy/modernize/PassByValueCheck.h
index 7abda91ac5632..82cd9d44c5e3a 100644
--- a/clang-tools-extra/clang-tidy/modernize/PassByValueCheck.h
+++ b/clang-tools-extra/clang-tidy/modernize/PassByValueCheck.h
@@ -31,8 +31,7 @@ class PassByValueCheck : public ClangTidyCheck {
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
 
 private:
-  std::unique_ptr<utils::IncludeInserter> Inserter;
-  const utils::IncludeSorter::IncludeStyle IncludeStyle;
+  utils::IncludeInserter Inserter;
   const bool ValuesOnly;
 };
 
diff --git a/clang-tools-extra/clang-tidy/modernize/ReplaceAutoPtrCheck.cpp b/clang-tools-extra/clang-tidy/modernize/ReplaceAutoPtrCheck.cpp
index f98254dbf7c83..25ffbe2b8738d 100644
--- a/clang-tools-extra/clang-tidy/modernize/ReplaceAutoPtrCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/ReplaceAutoPtrCheck.cpp
@@ -74,11 +74,11 @@ AST_MATCHER(Decl, isFromStdNamespace) {
 ReplaceAutoPtrCheck::ReplaceAutoPtrCheck(StringRef Name,
                                          ClangTidyContext *Context)
     : ClangTidyCheck(Name, Context),
-      IncludeStyle(Options.getLocalOrGlobal("IncludeStyle",
-                                            utils::IncludeSorter::IS_LLVM)) {}
+      Inserter(Options.getLocalOrGlobal("IncludeStyle",
+                                        utils::IncludeSorter::IS_LLVM)) {}
 
 void ReplaceAutoPtrCheck::storeOptions(ClangTidyOptions::OptionMap &Opts) {
-  Options.store(Opts, "IncludeStyle", IncludeStyle);
+  Options.store(Opts, "IncludeStyle", Inserter.getStyle());
 }
 
 void ReplaceAutoPtrCheck::registerMatchers(MatchFinder *Finder) {
@@ -131,9 +131,7 @@ void ReplaceAutoPtrCheck::registerMatchers(MatchFinder *Finder) {
 void ReplaceAutoPtrCheck::registerPPCallbacks(const SourceManager &SM,
                                               Preprocessor *PP,
                                               Preprocessor *ModuleExpanderPP) {
-  Inserter = std::make_unique<utils::IncludeInserter>(SM, getLangOpts(),
-                                                       IncludeStyle);
-  PP->addPPCallbacks(Inserter->CreatePPCallbacks());
+  Inserter.registerPreprocessor(PP);
 }
 
 void ReplaceAutoPtrCheck::check(const MatchFinder::MatchResult &Result) {
@@ -146,12 +144,11 @@ void ReplaceAutoPtrCheck::check(const MatchFinder::MatchResult &Result) {
     if (Range.isInvalid())
       return;
 
-    auto Diag =
-        diag(Range.getBegin(), "use std::move to transfer ownership")
-        << FixItHint::CreateInsertion(Range.getBegin(), "std::move(")
-        << FixItHint::CreateInsertion(Range.getEnd(), ")")
-        << Inserter->CreateIncludeInsertion(SM.getMainFileID(), "utility",
-                                            /*IsAngled=*/true);
+    auto Diag = diag(Range.getBegin(), "use std::move to transfer ownership")
+                << FixItHint::CreateInsertion(Range.getBegin(), "std::move(")
+                << FixItHint::CreateInsertion(Range.getEnd(), ")")
+                << Inserter.createMainFileIncludeInsertion("utility",
+                                                           /*IsAngled=*/true);
 
     return;
   }
diff --git a/clang-tools-extra/clang-tidy/modernize/ReplaceAutoPtrCheck.h b/clang-tools-extra/clang-tidy/modernize/ReplaceAutoPtrCheck.h
index e2b04073a65f4..8288c7e47d35e 100644
--- a/clang-tools-extra/clang-tidy/modernize/ReplaceAutoPtrCheck.h
+++ b/clang-tools-extra/clang-tidy/modernize/ReplaceAutoPtrCheck.h
@@ -53,8 +53,7 @@ class ReplaceAutoPtrCheck : public ClangTidyCheck {
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
 
 private:
-  std::unique_ptr<utils::IncludeInserter> Inserter;
-  const utils::IncludeSorter::IncludeStyle IncludeStyle;
+  utils::IncludeInserter Inserter;
 };
 
 } // namespace modernize
diff --git a/clang-tools-extra/clang-tidy/modernize/ReplaceRandomShuffleCheck.cpp b/clang-tools-extra/clang-tidy/modernize/ReplaceRandomShuffleCheck.cpp
index 66917df3e91d2..0191f5d5c5deb 100644
--- a/clang-tools-extra/clang-tidy/modernize/ReplaceRandomShuffleCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/ReplaceRandomShuffleCheck.cpp
@@ -23,8 +23,9 @@ namespace modernize {
 ReplaceRandomShuffleCheck::ReplaceRandomShuffleCheck(StringRef Name,
                                                      ClangTidyContext *Context)
     : ClangTidyCheck(Name, Context),
-      IncludeStyle(Options.getLocalOrGlobal("IncludeStyle",
-                                            utils::IncludeSorter::IS_LLVM)) {}
+      IncludeInserter(Options.getLocalOrGlobal("IncludeStyle",
+                                               utils::IncludeSorter::IS_LLVM)) {
+}
 
 void ReplaceRandomShuffleCheck::registerMatchers(MatchFinder *Finder) {
   const auto Begin = hasArgument(0, expr());
@@ -44,14 +45,12 @@ void ReplaceRandomShuffleCheck::registerMatchers(MatchFinder *Finder) {
 
 void ReplaceRandomShuffleCheck::registerPPCallbacks(
     const SourceManager &SM, Preprocessor *PP, Preprocessor *ModuleExpanderPP) {
-  IncludeInserter = std::make_unique<utils::IncludeInserter>(SM, getLangOpts(),
-                                                              IncludeStyle);
-  PP->addPPCallbacks(IncludeInserter->CreatePPCallbacks());
+  IncludeInserter.registerPreprocessor(PP);
 }
 
 void ReplaceRandomShuffleCheck::storeOptions(
     ClangTidyOptions::OptionMap &Opts) {
-  Options.store(Opts, "IncludeStyle", IncludeStyle);
+  Options.store(Opts, "IncludeStyle", IncludeInserter.getStyle());
 }
 
 void ReplaceRandomShuffleCheck::check(const MatchFinder::MatchResult &Result) {
@@ -92,7 +91,7 @@ void ReplaceRandomShuffleCheck::check(const MatchFinder::MatchResult &Result) {
 
   Diag << FixItHint::CreateRemoval(MatchedDecl->getSourceRange());
   Diag << FixItHint::CreateInsertion(MatchedDecl->getBeginLoc(), NewName);
-  Diag << IncludeInserter->CreateIncludeInsertion(
+  Diag << IncludeInserter.createIncludeInsertion(
       Result.Context->getSourceManager().getFileID(
           MatchedCallExpr->getBeginLoc()),
       "random", /*IsAngled=*/true);
diff --git a/clang-tools-extra/clang-tidy/modernize/ReplaceRandomShuffleCheck.h b/clang-tools-extra/clang-tidy/modernize/ReplaceRandomShuffleCheck.h
index c4ac74d6e662a..990dcffc79a6a 100644
--- a/clang-tools-extra/clang-tidy/modernize/ReplaceRandomShuffleCheck.h
+++ b/clang-tools-extra/clang-tidy/modernize/ReplaceRandomShuffleCheck.h
@@ -34,8 +34,7 @@ class ReplaceRandomShuffleCheck : public ClangTidyCheck {
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
 
 private:
-  std::unique_ptr<utils::IncludeInserter> IncludeInserter;
-  const utils::IncludeSorter::IncludeStyle IncludeStyle;
+  utils::IncludeInserter IncludeInserter;
 };
 
 } // namespace modernize
diff --git a/clang-tools-extra/clang-tidy/performance/MoveConstructorInitCheck.cpp b/clang-tools-extra/clang-tidy/performance/MoveConstructorInitCheck.cpp
index 4cbb014867c4d..6b42cd3180d73 100644
--- a/clang-tools-extra/clang-tidy/performance/MoveConstructorInitCheck.cpp
+++ b/clang-tools-extra/clang-tidy/performance/MoveConstructorInitCheck.cpp
@@ -23,8 +23,8 @@ namespace performance {
 MoveConstructorInitCheck::MoveConstructorInitCheck(StringRef Name,
                                                    ClangTidyContext *Context)
     : ClangTidyCheck(Name, Context),
-      IncludeStyle(Options.getLocalOrGlobal("IncludeStyle",
-                                            utils::IncludeSorter::IS_LLVM)) {}
+      Inserter(Options.getLocalOrGlobal("IncludeStyle",
+                                        utils::IncludeSorter::IS_LLVM)) {}
 
 void MoveConstructorInitCheck::registerMatchers(MatchFinder *Finder) {
   Finder->addMatcher(
@@ -90,13 +90,11 @@ void MoveConstructorInitCheck::check(const MatchFinder::MatchResult &Result) {
 
 void MoveConstructorInitCheck::registerPPCallbacks(
     const SourceManager &SM, Preprocessor *PP, Preprocessor *ModuleExpanderPP) {
-  Inserter = std::make_unique<utils::IncludeInserter>(SM, getLangOpts(),
-                                                       IncludeStyle);
-  PP->addPPCallbacks(Inserter->CreatePPCallbacks());
+  Inserter.registerPreprocessor(PP);
 }
 
 void MoveConstructorInitCheck::storeOptions(ClangTidyOptions::OptionMap &Opts) {
-  Options.store(Opts, "IncludeStyle", IncludeStyle);
+  Options.store(Opts, "IncludeStyle", Inserter.getStyle());
 }
 
 } // namespace performance
diff --git a/clang-tools-extra/clang-tidy/performance/MoveConstructorInitCheck.h b/clang-tools-extra/clang-tidy/performance/MoveConstructorInitCheck.h
index 0473978f29db4..0b637b617782e 100644
--- a/clang-tools-extra/clang-tidy/performance/MoveConstructorInitCheck.h
+++ b/clang-tools-extra/clang-tidy/performance/MoveConstructorInitCheck.h
@@ -36,8 +36,7 @@ class MoveConstructorInitCheck : public ClangTidyCheck {
   void storeOptions(ClangTidyOptions::OptionMap &Opts) override;
 
 private:
-  std::unique_ptr<utils::IncludeInserter> Inserter;
-  const utils::IncludeSorter::IncludeStyle IncludeStyle;
+  utils::IncludeInserter Inserter;
 };
 
 } // namespace performance
diff --git a/clang-tools-extra/clang-tidy/performance/TypePromotionInMathFnCheck.cpp b/clang-tools-extra/clang-tidy/performance/TypePromotionInMathFnCheck.cpp
index 597445d0fc266..2105aa9947bb0 100644
--- a/clang-tools-extra/clang-tidy/performance/TypePromotionInMathFnCheck.cpp
+++ b/clang-tools-extra/clang-tidy/performance/TypePromotionInMathFnCheck.cpp
@@ -31,19 +31,18 @@ AST_MATCHER_P(Type, isBuiltinType, BuiltinType::Kind, Kind) {
 TypePromotionInMathFnCheck::TypePromotionInMathFnCheck(
     StringRef Name, ClangTidyContext *Context)
     : ClangTidyCheck(Name, Context),
-      IncludeStyle(Options.getLocalOrGlobal("IncludeStyle",
-                                            utils::IncludeSorter::IS_LLVM)) {}
+      IncludeInserter(Options.getLocalOrGlobal("IncludeStyle",
+                                               utils::IncludeSorter::IS_LLVM)) {
+}
 
 void TypePromotionInMathFnCheck::registerPPCallbacks(
     const SourceManager &SM, Preprocessor *PP, Preprocessor *ModuleExpanderPP) {
-  IncludeInserter = std::make_unique<utils::IncludeInserter>(SM, getLangOpts(),
-                                                              IncludeStyle);
-  PP->addPPCallbacks(IncludeInserter->CreatePPCallbacks());
+  IncludeInserter.registerPreprocessor(PP);
 }
 
 void TypePromotionInMathFnCheck::storeOptions(
     ClangTidyOptions::OptionMap &Opts) {
-  Options.store(Opts, "IncludeStyle", IncludeStyle);
+  Options.store(Opts, "IncludeStyle", IncludeInserter.getStyle());
 }
 
 void TypePromotionInMathFnCheck::registerMatchers(MatchFinder *Finder) {
@@ -191,7 +190,7 @@ void TypePromotionInMathFnCheck::check(const MatchFinder::MatchResult &Result) {
   // <math.h>, because the functions we're suggesting moving away from are all
   // declared in <math.h>.
   if (FnInCmath)
-    Diag << IncludeInserter->CreateIncludeInsertion(
+    Diag << IncludeInserter.createIncludeInsertion(
         Result.Context->getSourceManager().getFileID(Call->getBeginLoc()),
         "cmath", /*IsAngled=*/true);
 }
diff --git a/clang-tools-extra/clang-tidy/performance/TypePromotionInMathFnCheck.h b/clang-tools-extra/clang-tidy/performance/TypePromotionInMathFnCheck.h
index d1cc042ca6a35..dd7c1c090bad3 100644
--- a/clang-tools-extra/clang-tidy/performance/TypePromotionInMathFnCheck.h
+++ b/clang-tools-extra/clang-tidy/performance/TypePromotionInMathFnCheck.h
@@ -36,8 +36,7 @@ class TypePromotionInMathFnCheck : public ClangTidyCheck {
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
 
 private:
-  std::unique_ptr<utils::IncludeInserter> IncludeInserter;
-  const utils::IncludeSorter::IncludeStyle IncludeStyle;
+  utils::IncludeInserter IncludeInserter;
 };
 
 } // namespace performance
diff --git a/clang-tools-extra/clang-tidy/performance/UnnecessaryValueParamCheck.cpp b/clang-tools-extra/clang-tidy/performance/UnnecessaryValueParamCheck.cpp
index 5de53b1840f12..9aef5a8681694 100644
--- a/clang-tools-extra/clang-tidy/performance/UnnecessaryValueParamCheck.cpp
+++ b/clang-tools-extra/clang-tidy/performance/UnnecessaryValueParamCheck.cpp
@@ -68,8 +68,8 @@ bool isExplicitTemplateSpecialization(const FunctionDecl &Function) {
 UnnecessaryValueParamCheck::UnnecessaryValueParamCheck(
     StringRef Name, ClangTidyContext *Context)
     : ClangTidyCheck(Name, Context),
-      IncludeStyle(Options.getLocalOrGlobal("IncludeStyle",
-                                            utils::IncludeSorter::IS_LLVM)),
+      Inserter(Options.getLocalOrGlobal("IncludeStyle",
+                                        utils::IncludeSorter::IS_LLVM)),
       AllowedTypes(
           utils::options::parseStringList(Options.get("AllowedTypes", ""))) {}
 
@@ -173,14 +173,12 @@ void UnnecessaryValueParamCheck::check(const MatchFinder::MatchResult &Result) {
 
 void UnnecessaryValueParamCheck::registerPPCallbacks(
     const SourceManager &SM, Preprocessor *PP, Preprocessor *ModuleExpanderPP) {
-  Inserter = std::make_unique<utils::IncludeInserter>(SM, getLangOpts(),
-                                                      IncludeStyle);
-  PP->addPPCallbacks(Inserter->CreatePPCallbacks());
+  Inserter.registerPreprocessor(PP);
 }
 
 void UnnecessaryValueParamCheck::storeOptions(
     ClangTidyOptions::OptionMap &Opts) {
-  Options.store(Opts, "IncludeStyle", IncludeStyle);
+  Options.store(Opts, "IncludeStyle", Inserter.getStyle());
   Options.store(Opts, "AllowedTypes",
                 utils::options::serializeStringList(AllowedTypes));
 }
@@ -204,7 +202,7 @@ void UnnecessaryValueParamCheck::handleMoveFix(const ParmVarDecl &Var,
                                            Context.getLangOpts());
   Diag << FixItHint::CreateInsertion(CopyArgument.getBeginLoc(), "std::move(")
        << FixItHint::CreateInsertion(EndLoc, ")")
-       << Inserter->CreateIncludeInsertion(
+       << Inserter.createIncludeInsertion(
               SM.getFileID(CopyArgument.getBeginLoc()), "utility",
               /*IsAngled=*/true);
 }
diff --git a/clang-tools-extra/clang-tidy/performance/UnnecessaryValueParamCheck.h b/clang-tools-extra/clang-tidy/performance/UnnecessaryValueParamCheck.h
index 1d2367148ef82..a84079e7da229 100644
--- a/clang-tools-extra/clang-tidy/performance/UnnecessaryValueParamCheck.h
+++ b/clang-tools-extra/clang-tidy/performance/UnnecessaryValueParamCheck.h
@@ -41,8 +41,7 @@ class UnnecessaryValueParamCheck : public ClangTidyCheck {
 
   llvm::DenseMap<const FunctionDecl *, FunctionParmMutationAnalyzer>
       MutationAnalyzers;
-  std::unique_ptr<utils::IncludeInserter> Inserter;
-  const utils::IncludeSorter::IncludeStyle IncludeStyle;
+  utils::IncludeInserter Inserter;
   const std::vector<std::string> AllowedTypes;
 };
 
diff --git a/clang-tools-extra/clang-tidy/utils/IncludeInserter.cpp b/clang-tools-extra/clang-tidy/utils/IncludeInserter.cpp
index df87dbe49cff8..268692c3ba42e 100644
--- a/clang-tools-extra/clang-tidy/utils/IncludeInserter.cpp
+++ b/clang-tools-extra/clang-tidy/utils/IncludeInserter.cpp
@@ -7,6 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "IncludeInserter.h"
+#include "clang/Lex/PPCallbacks.h"
+#include "clang/Lex/Preprocessor.h"
 #include "clang/Lex/Token.h"
 
 namespace clang {
@@ -26,7 +28,7 @@ class IncludeInserterCallback : public PPCallbacks {
                           StringRef /*SearchPath*/, StringRef /*RelativePath*/,
                           const Module * /*ImportedModule*/,
                           SrcMgr::CharacteristicKind /*FileType*/) override {
-    Inserter->AddInclude(FileNameRef, IsAngled, HashLocation,
+    Inserter->addInclude(FileNameRef, IsAngled, HashLocation,
                          IncludeToken.getEndLoc());
   }
 
@@ -34,45 +36,61 @@ class IncludeInserterCallback : public PPCallbacks {
   IncludeInserter *Inserter;
 };
 
-IncludeInserter::IncludeInserter(const SourceManager &SourceMgr,
-                                 const LangOptions &LangOpts,
-                                 IncludeSorter::IncludeStyle Style)
-    : SourceMgr(SourceMgr), Style(Style) {}
+IncludeInserter::IncludeInserter(IncludeSorter::IncludeStyle Style)
+    : Style(Style) {}
 
-IncludeInserter::~IncludeInserter() {}
+void IncludeInserter::registerPreprocessor(Preprocessor *PP) {
+  assert(PP && "PP shouldn't be null");
+  SourceMgr = &PP->getSourceManager();
 
-std::unique_ptr<PPCallbacks> IncludeInserter::CreatePPCallbacks() {
-  return std::make_unique<IncludeInserterCallback>(this);
+  // If this gets registered multiple times, clear the maps
+  if (!IncludeSorterByFile.empty())
+    IncludeSorterByFile.clear();
+  if (!InsertedHeaders.empty())
+    InsertedHeaders.clear();
+  PP->addPPCallbacks(std::make_unique<IncludeInserterCallback>(this));
 }
 
 IncludeSorter &IncludeInserter::getOrCreate(FileID FileID) {
+  assert(SourceMgr && "SourceMgr shouldn't be null; did you remember to call "
+                      "registerPreprocessor()?");
   // std::unique_ptr is cheap to construct, so force a construction now to save
   // the lookup needed if we were to insert into the map.
   std::unique_ptr<IncludeSorter> &Entry = IncludeSorterByFile[FileID];
   if (!Entry) {
     // If it wasn't found, Entry will be default constructed to nullptr.
     Entry = std::make_unique<IncludeSorter>(
-        &SourceMgr, FileID,
-        SourceMgr.getFilename(SourceMgr.getLocForStartOfFile(FileID)), Style);
+        SourceMgr, FileID,
+        SourceMgr->getFilename(SourceMgr->getLocForStartOfFile(FileID)), Style);
   }
   return *Entry;
 }
 
 llvm::Optional<FixItHint>
-IncludeInserter::CreateIncludeInsertion(FileID FileID, StringRef Header,
+IncludeInserter::createIncludeInsertion(FileID FileID, StringRef Header,
                                         bool IsAngled) {
   // We assume the same Header will never be included both angled and not
   // angled.
-  if (!InsertedHeaders[FileID].insert(std::string(Header)).second)
+  if (!InsertedHeaders[FileID].insert(Header).second)
     return llvm::None;
 
   return getOrCreate(FileID).CreateIncludeInsertion(Header, IsAngled);
 }
 
-void IncludeInserter::AddInclude(StringRef FileName, bool IsAngled,
+llvm::Optional<FixItHint>
+IncludeInserter::createMainFileIncludeInsertion(StringRef Header,
+                                                bool IsAngled) {
+  assert(SourceMgr && "SourceMgr shouldn't be null; did you remember to call "
+                      "registerPreprocessor()?");
+  return createIncludeInsertion(SourceMgr->getMainFileID(), Header, IsAngled);
+}
+
+void IncludeInserter::addInclude(StringRef FileName, bool IsAngled,
                                  SourceLocation HashLocation,
                                  SourceLocation EndLocation) {
-  FileID FileID = SourceMgr.getFileID(HashLocation);
+  assert(SourceMgr && "SourceMgr shouldn't be null; did you remember to call "
+                      "registerPreprocessor()?");
+  FileID FileID = SourceMgr->getFileID(HashLocation);
   getOrCreate(FileID).AddInclude(FileName, IsAngled, HashLocation, EndLocation);
 }
 
diff --git a/clang-tools-extra/clang-tidy/utils/IncludeInserter.h b/clang-tools-extra/clang-tidy/utils/IncludeInserter.h
index 0d4b951beb1ff..70c36ce8895c4 100644
--- a/clang-tools-extra/clang-tidy/utils/IncludeInserter.h
+++ b/clang-tools-extra/clang-tidy/utils/IncludeInserter.h
@@ -11,13 +11,11 @@
 
 #include "IncludeSorter.h"
 #include "clang/Basic/Diagnostic.h"
-#include "clang/Basic/LangOptions.h"
-#include "clang/Basic/SourceManager.h"
-#include "clang/Lex/PPCallbacks.h"
+#include "llvm/ADT/StringSet.h"
 #include <memory>
-#include <string>
 
 namespace clang {
+class Preprocessor;
 namespace tidy {
 namespace utils {
 
@@ -26,16 +24,17 @@ namespace utils {
 ///
 /// ``IncludeInserter`` can be used in clang-tidy checks in the following way:
 /// \code
+/// #include "../ClangTidyCheck.h"
 /// #include "../utils/IncludeInserter.h"
-/// #include "clang/Frontend/CompilerInstance.h"
+///
+/// namespace clang {
+/// namespace tidy {
 ///
 /// class MyCheck : public ClangTidyCheck {
 ///  public:
 ///   void registerPPCallbacks(const SourceManager &SM, Preprocessor *PP,
 ///                            Preprocessor *ModuleExpanderPP) override {
-///     Inserter = std::make_unique<IncludeInserter>(
-///         SM, getLangOpts(), utils::IncludeSorter::IS_Google);
-///     PP->addPPCallbacks(Inserter->CreatePPCallbacks());
+///     Inserter.registerPreprocessor();
 ///   }
 ///
 ///   void registerMatchers(ast_matchers::MatchFinder* Finder) override { ... }
@@ -43,39 +42,53 @@ namespace utils {
 ///   void check(
 ///       const ast_matchers::MatchFinder::MatchResult& Result) override {
 ///     ...
-///     Inserter->CreateIncludeInsertion(
-///         Result.SourceManager->getMainFileID(), "path/to/Header.h",
-///         /*IsAngled=*/false);
+///     Inserter.createMainFileIncludeInsertion("path/to/Header.h",
+///                                             /*IsAngled=*/false);
 ///     ...
 ///   }
 ///
 ///  private:
-///   std::unique_ptr<clang::tidy::utils::IncludeInserter> Inserter;
+///   utils::IncludeInserter Inserter{utils::IncludeSorter::IS_Google};
 /// };
+/// } // namespace tidy
+/// } // namespace clang
 /// \endcode
 class IncludeInserter {
 public:
-  IncludeInserter(const SourceManager &SourceMgr, const LangOptions &LangOpts,
-                  IncludeSorter::IncludeStyle Style);
-  ~IncludeInserter();
+  /// Initializes the IncludeInserter using the IncludeStyle \p Style.
+  /// In most cases the \p Style will be retrieved from the ClangTidyOptions
+  /// using \code
+  ///   Options.getLocalOrGlobal("IncludeStyle", <DefaultStyle>)
+  /// \endcode
+  explicit IncludeInserter(IncludeSorter::IncludeStyle Style);
+
+  /// Registers this with the Preprocessor \p PP, must be called before this
+  /// class is used.
+  void registerPreprocessor(Preprocessor *PP);
 
-  /// Create ``PPCallbacks`` for registration with the compiler's preprocessor.
-  std::unique_ptr<PPCallbacks> CreatePPCallbacks();
+  /// Creates a \p Header inclusion directive fixit in the File \p FileID.
+  /// Returns ``llvm::None`` on error or if the inclusion directive already
+  /// exists.
+  llvm::Optional<FixItHint>
+  createIncludeInsertion(FileID FileID, llvm::StringRef Header, bool IsAngled);
 
-  /// Creates a \p Header inclusion directive fixit. Returns ``llvm::None`` on
-  /// error or if inclusion directive already exists.
+  /// Creates a \p Header inclusion directive fixit in the main file.
+  /// Returns``llvm::None`` on error or if the inclusion directive already
+  /// exists.
   llvm::Optional<FixItHint>
-  CreateIncludeInsertion(FileID FileID, llvm::StringRef Header, bool IsAngled);
+  createMainFileIncludeInsertion(llvm::StringRef Header, bool IsAngled);
+
+  IncludeSorter::IncludeStyle getStyle() const { return Style; }
 
 private:
-  void AddInclude(StringRef FileName, bool IsAngled,
+  void addInclude(StringRef FileName, bool IsAngled,
                   SourceLocation HashLocation, SourceLocation EndLocation);
 
   IncludeSorter &getOrCreate(FileID FileID);
 
   llvm::DenseMap<FileID, std::unique_ptr<IncludeSorter>> IncludeSorterByFile;
-  llvm::DenseMap<FileID, std::set<std::string>> InsertedHeaders;
-  const SourceManager &SourceMgr;
+  llvm::DenseMap<FileID, llvm::StringSet<>> InsertedHeaders;
+  const SourceManager *SourceMgr{nullptr};
   const IncludeSorter::IncludeStyle Style;
   friend class IncludeInserterCallback;
 };
diff --git a/clang-tools-extra/clang-tidy/utils/TransformerClangTidyCheck.cpp b/clang-tools-extra/clang-tidy/utils/TransformerClangTidyCheck.cpp
index 03af5dd1565f8..2c116b210d050 100644
--- a/clang-tools-extra/clang-tidy/utils/TransformerClangTidyCheck.cpp
+++ b/clang-tools-extra/clang-tidy/utils/TransformerClangTidyCheck.cpp
@@ -32,8 +32,8 @@ TransformerClangTidyCheck::TransformerClangTidyCheck(
         MakeRule,
     StringRef Name, ClangTidyContext *Context)
     : ClangTidyCheck(Name, Context), Rule(MakeRule(getLangOpts(), Options)),
-      IncludeStyle(Options.getLocalOrGlobal("IncludeStyle",
-                                            IncludeSorter::IS_LLVM)) {
+      Inserter(
+          Options.getLocalOrGlobal("IncludeStyle", IncludeSorter::IS_LLVM)) {
   if (Rule)
     assert(llvm::all_of(Rule->Cases, hasExplanation) &&
            "clang-tidy checks must have an explanation by default;"
@@ -44,8 +44,8 @@ TransformerClangTidyCheck::TransformerClangTidyCheck(RewriteRule R,
                                                      StringRef Name,
                                                      ClangTidyContext *Context)
     : ClangTidyCheck(Name, Context), Rule(std::move(R)),
-      IncludeStyle(Options.getLocalOrGlobal("IncludeStyle",
-                                            IncludeSorter::IS_LLVM)) {
+      Inserter(
+          Options.getLocalOrGlobal("IncludeStyle", IncludeSorter::IS_LLVM)) {
   assert(llvm::all_of(Rule->Cases, hasExplanation) &&
          "clang-tidy checks must have an explanation by default;"
          " explicitly provide an empty explanation if none is desired");
@@ -53,15 +53,12 @@ TransformerClangTidyCheck::TransformerClangTidyCheck(RewriteRule R,
 
 void TransformerClangTidyCheck::registerPPCallbacks(
     const SourceManager &SM, Preprocessor *PP, Preprocessor *ModuleExpanderPP) {
-  // Only allocate and register the IncludeInsert when some `Case` will add
+  // Only register the IncludeInsert when some `Case` will add
   // includes.
   if (Rule && llvm::any_of(Rule->Cases, [](const RewriteRule::Case &C) {
         return !C.AddedIncludes.empty();
-      })) {
-    Inserter =
-        std::make_unique<IncludeInserter>(SM, getLangOpts(), IncludeStyle);
-    PP->addPPCallbacks(Inserter->CreatePPCallbacks());
-  }
+      }))
+    Inserter.registerPreprocessor(PP);
 }
 
 void TransformerClangTidyCheck::registerMatchers(
@@ -102,15 +99,15 @@ void TransformerClangTidyCheck::check(
     Diag << FixItHint::CreateReplacement(T.Range, T.Replacement);
 
   for (const auto &I : Case.AddedIncludes) {
-    Diag << Inserter->CreateIncludeInsertion(
-        Result.SourceManager->getMainFileID(), I.first,
+    Diag << Inserter.createMainFileIncludeInsertion(
+        I.first,
         /*IsAngled=*/I.second == transformer::IncludeFormat::Angled);
   }
 }
 
 void TransformerClangTidyCheck::storeOptions(
     ClangTidyOptions::OptionMap &Opts) {
-  Options.store(Opts, "IncludeStyle", IncludeStyle);
+  Options.store(Opts, "IncludeStyle", Inserter.getStyle());
 }
 
 } // namespace utils
diff --git a/clang-tools-extra/clang-tidy/utils/TransformerClangTidyCheck.h b/clang-tools-extra/clang-tidy/utils/TransformerClangTidyCheck.h
index 829a22fe8e2cc..404f474a24cae 100644
--- a/clang-tools-extra/clang-tidy/utils/TransformerClangTidyCheck.h
+++ b/clang-tools-extra/clang-tidy/utils/TransformerClangTidyCheck.h
@@ -70,8 +70,7 @@ class TransformerClangTidyCheck : public ClangTidyCheck {
 
 private:
   Optional<transformer::RewriteRule> Rule;
-  const IncludeSorter::IncludeStyle IncludeStyle;
-  std::unique_ptr<IncludeInserter> Inserter;
+  IncludeInserter Inserter;
 };
 
 } // namespace utils
diff --git a/clang-tools-extra/unittests/clang-tidy/IncludeInserterTest.cpp b/clang-tools-extra/unittests/clang-tidy/IncludeInserterTest.cpp
index ed5f02576f04a..e70d3fb91bf25 100644
--- a/clang-tools-extra/unittests/clang-tidy/IncludeInserterTest.cpp
+++ b/clang-tools-extra/unittests/clang-tidy/IncludeInserterTest.cpp
@@ -33,9 +33,7 @@ class IncludeInserterCheckBase : public ClangTidyCheck {
 
   void registerPPCallbacks(const SourceManager &SM, Preprocessor *PP,
                            Preprocessor *ModuleExpanderPP) override {
-    Inserter = std::make_unique<utils::IncludeInserter>(
-        SM, getLangOpts(), utils::IncludeSorter::IS_Google);
-    PP->addPPCallbacks(Inserter->CreatePPCallbacks());
+    Inserter.registerPreprocessor(PP);
   }
 
   void registerMatchers(ast_matchers::MatchFinder *Finder) override {
@@ -46,15 +44,15 @@ class IncludeInserterCheckBase : public ClangTidyCheck {
     auto Diag = diag(Result.Nodes.getNodeAs<DeclStmt>("stmt")->getBeginLoc(),
                      "foo, bar");
     for (StringRef Header : HeadersToInclude()) {
-      Diag << Inserter->CreateIncludeInsertion(
-          Result.SourceManager->getMainFileID(), Header, IsAngledInclude());
+      Diag << Inserter.createMainFileIncludeInsertion(Header,
+                                                      IsAngledInclude());
     }
   }
 
   virtual std::vector<StringRef> HeadersToInclude() const = 0;
   virtual bool IsAngledInclude() const = 0;
 
-  std::unique_ptr<utils::IncludeInserter> Inserter;
+  utils::IncludeInserter Inserter{utils::IncludeSorter::IS_Google};
 };
 
 class NonSystemHeaderInserterCheck : public IncludeInserterCheckBase {

From 216b67e2023315ff30c2802c911a8ae0c7640c30 Mon Sep 17 00:00:00 2001
From: Tim Northover <t.p.northover@gmail.com>
Date: Wed, 22 Jul 2020 14:53:26 +0100
Subject: [PATCH 0182/1035] AArch64: diagnose out of range relocation addends
 on MachO.

MachO only has 24-bit addends for most relocations, small enough that it can
overflow in semi-reasonable functions and cause insidious bugs if compiled
without assertions enabled. Switch it to an actual error instead.

The condition isn't quite identical because ld64 treats the addend as a signed
number.
---
 .../MCTargetDesc/AArch64MachObjectWriter.cpp       |  6 +++++-
 llvm/test/MC/AArch64/macho-addend-range.s          | 14 ++++++++++++++
 2 files changed, 19 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/MC/AArch64/macho-addend-range.s

diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MachObjectWriter.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MachObjectWriter.cpp
index b0f414bd27edd..012661edbbfda 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MachObjectWriter.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MachObjectWriter.cpp
@@ -373,7 +373,11 @@ void AArch64MachObjectWriter::recordRelocation(
        Type == MachO::ARM64_RELOC_PAGE21 ||
        Type == MachO::ARM64_RELOC_PAGEOFF12) &&
       Value) {
-    assert((Value & 0xff000000) == 0 && "Added relocation out of range!");
+    if (!isInt<24>(Value)) {
+      Asm.getContext().reportError(Fixup.getLoc(),
+                                   "addend too big for relocation");
+      return;
+    }
 
     MachO::any_relocation_info MRE;
     MRE.r_word0 = FixupOffset;
diff --git a/llvm/test/MC/AArch64/macho-addend-range.s b/llvm/test/MC/AArch64/macho-addend-range.s
new file mode 100644
index 0000000000000..fdcb9123c7684
--- /dev/null
+++ b/llvm/test/MC/AArch64/macho-addend-range.s
@@ -0,0 +1,14 @@
+// RUN: not llvm-mc -triple arm64-apple-ios -filetype=obj -o /dev/null %s 2>&1 | FileCheck %s
+        .global _foo
+        adrp x0, (_foo + 1)@PAGE
+        adrp x0, (_foo - 1)@PAGE
+        adrp x0, (_foo + 0x7fffff)@PAGE
+        adrp x0, (_foo - 0x800000)@PAGE
+
+        // CHECK-NOT: error:
+        // CHECK: error: addend too big for relocation
+        // CHECK:      adrp x0, (_foo + 0x800000)@PAGE
+        // CHECK: error: addend too big for relocation
+        // CHECK:      adrp x0, (_foo - 0x800001)@PAGE
+        adrp x0, (_foo + 0x800000)@PAGE
+        adrp x0, (_foo - 0x800001)@PAGE

From 40d11a878044711708fb6738e4b78a4c9ac3de7b Mon Sep 17 00:00:00 2001
From: Hans Wennborg <hans@chromium.org>
Date: Mon, 27 Jul 2020 14:03:28 +0200
Subject: [PATCH 0183/1035] ClangdMain.cpp: this #ifdef should be an #if

CLANGD_ENABLE_REMOTE is always defined; to 0 or 1.
---
 clang-tools-extra/clangd/tool/ClangdMain.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang-tools-extra/clangd/tool/ClangdMain.cpp b/clang-tools-extra/clangd/tool/ClangdMain.cpp
index 8d1bf5c422605..f04dad7186df9 100644
--- a/clang-tools-extra/clangd/tool/ClangdMain.cpp
+++ b/clang-tools-extra/clangd/tool/ClangdMain.cpp
@@ -696,7 +696,7 @@ clangd accepts flags on the commandline, and in the CLANGD_FLAGS environment var
     if (Sync)
       AsyncIndexLoad.wait();
   }
-#ifdef CLANGD_ENABLE_REMOTE
+#if CLANGD_ENABLE_REMOTE
   if (RemoteIndexAddress.empty() != ProjectRoot.empty()) {
     llvm::errs() << "remote-index-address and project-path have to be "
                     "specified at the same time.";

From 0de629d7ae53e821274dfbe0fb3676ea08893a35 Mon Sep 17 00:00:00 2001
From: Hans Wennborg <hans@chromium.org>
Date: Mon, 27 Jul 2020 14:03:55 +0200
Subject: [PATCH 0184/1035] [gn] Set CLANGD_ENABLE_REMOTE=0

To fix the build after 37ac559fccd4.
---
 llvm/utils/gn/secondary/clang-tools-extra/clangd/BUILD.gn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/utils/gn/secondary/clang-tools-extra/clangd/BUILD.gn b/llvm/utils/gn/secondary/clang-tools-extra/clangd/BUILD.gn
index dade4cc73bf9f..84d3f14bb2f27 100644
--- a/llvm/utils/gn/secondary/clang-tools-extra/clangd/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang-tools-extra/clangd/BUILD.gn
@@ -16,6 +16,7 @@ write_cmake_config("features") {
   } else {
     values += [ "CLANGD_BUILD_XPC=0" ]
   }
+  values += [ "CLANGD_ENABLE_REMOTE=0" ]
   public_configs = [ ":features_config" ]
 }
 

From 529441e88e81a2e7dae6108e3d95e043c670e1a6 Mon Sep 17 00:00:00 2001
From: Hans Wennborg <hans@chromium.org>
Date: Mon, 27 Jul 2020 14:11:29 +0200
Subject: [PATCH 0185/1035] Fix another #ifdef CLANGD_ENABLE_REMOTE

---
 clang-tools-extra/clangd/tool/ClangdMain.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang-tools-extra/clangd/tool/ClangdMain.cpp b/clang-tools-extra/clangd/tool/ClangdMain.cpp
index f04dad7186df9..0d4267774c92f 100644
--- a/clang-tools-extra/clangd/tool/ClangdMain.cpp
+++ b/clang-tools-extra/clangd/tool/ClangdMain.cpp
@@ -450,7 +450,7 @@ opt<bool> EnableConfig{
     init(true),
 };
 
-#ifdef CLANGD_ENABLE_REMOTE
+#if CLANGD_ENABLE_REMOTE
 opt<std::string> RemoteIndexAddress{
     "remote-index-address",
     cat(Features),

From f49a7ad8c0854a01b945c27de2fd313b9013ae0d Mon Sep 17 00:00:00 2001
From: Kirill Bobyrev <kbobyrev@google.com>
Date: Mon, 27 Jul 2020 14:15:59 +0200
Subject: [PATCH 0186/1035] [clangd] Add marshalling code for all request types

Summary:
Only FuzzyFindRequest is implemented via Marshaller even though other requests
also follow a similar pattern. Unify them under the marshalling umbrella and
make the server requests even more uniform to complement D84499.

Reviewers: kadircet

Reviewed By: kadircet

Subscribers: ilya-biryukov, MaskRay, jkorous, arphaman, usaxena95, cfe-commits, sammccall

Tags: #clang

Differential Revision: https://reviews.llvm.org/D84525
---
 .../index/remote/marshalling/Marshalling.cpp  | 63 +++++++++++++++----
 .../index/remote/marshalling/Marshalling.h    |  7 ++-
 .../clangd/index/remote/server/Server.cpp     | 34 +++++-----
 .../unittests/remote/MarshallingTests.cpp     | 59 ++++++++++++++++-
 4 files changed, 133 insertions(+), 30 deletions(-)

diff --git a/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp b/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp
index b6c83c9740727..b2085bc21f486 100644
--- a/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp
+++ b/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp
@@ -17,6 +17,7 @@
 #include "index/SymbolOrigin.h"
 #include "support/Logger.h"
 #include "clang/Index/IndexSymbol.h"
+#include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/None.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/SmallString.h"
@@ -30,6 +31,22 @@ namespace clang {
 namespace clangd {
 namespace remote {
 
+namespace {
+
+template <typename MessageT>
+llvm::Expected<llvm::DenseSet<SymbolID>> getIDs(MessageT *Message) {
+  llvm::DenseSet<SymbolID> Result;
+  for (const auto &ID : Message->ids()) {
+    auto SID = SymbolID::fromStr(StringRef(ID));
+    if (!SID)
+      return SID.takeError();
+    Result.insert(*SID);
+  }
+  return Result;
+}
+
+} // namespace
+
 Marshaller::Marshaller(llvm::StringRef RemoteIndexRoot,
                        llvm::StringRef LocalIndexRoot)
     : Strings(Arena) {
@@ -49,27 +66,50 @@ Marshaller::Marshaller(llvm::StringRef RemoteIndexRoot,
   assert(!RemoteIndexRoot.empty() || !LocalIndexRoot.empty());
 }
 
-clangd::FuzzyFindRequest
-Marshaller::fromProtobuf(const FuzzyFindRequest *Request) {
+llvm::Expected<clangd::LookupRequest>
+Marshaller::fromProtobuf(const LookupRequest *Message) {
+  clangd::LookupRequest Req;
+  auto IDs = getIDs(Message);
+  if (!IDs)
+    return IDs.takeError();
+  Req.IDs = std::move(*IDs);
+  return Req;
+}
+
+llvm::Expected<clangd::FuzzyFindRequest>
+Marshaller::fromProtobuf(const FuzzyFindRequest *Message) {
   assert(RemoteIndexRoot);
   clangd::FuzzyFindRequest Result;
-  Result.Query = Request->query();
-  for (const auto &Scope : Request->scopes())
+  Result.Query = Message->query();
+  for (const auto &Scope : Message->scopes())
     Result.Scopes.push_back(Scope);
-  Result.AnyScope = Request->any_scope();
-  if (Request->limit())
-    Result.Limit = Request->limit();
-  Result.RestrictForCodeCompletion = Request->restricted_for_code_completion();
-  for (const auto &Path : Request->proximity_paths()) {
+  Result.AnyScope = Message->any_scope();
+  if (Message->limit())
+    Result.Limit = Message->limit();
+  Result.RestrictForCodeCompletion = Message->restricted_for_code_completion();
+  for (const auto &Path : Message->proximity_paths()) {
     llvm::SmallString<256> LocalPath = llvm::StringRef(*RemoteIndexRoot);
     llvm::sys::path::append(LocalPath, Path);
     Result.ProximityPaths.push_back(std::string(LocalPath));
   }
-  for (const auto &Type : Request->preferred_types())
+  for (const auto &Type : Message->preferred_types())
     Result.ProximityPaths.push_back(Type);
   return Result;
 }
 
+llvm::Expected<clangd::RefsRequest>
+Marshaller::fromProtobuf(const RefsRequest *Message) {
+  clangd::RefsRequest Req;
+  auto IDs = getIDs(Message);
+  if (!IDs)
+    return IDs.takeError();
+  Req.IDs = std::move(*IDs);
+  Req.Filter = static_cast<RefKind>(Message->filter());
+  if (Message->limit())
+    Req.Limit = Message->limit();
+  return Req;
+}
+
 llvm::Optional<clangd::Symbol> Marshaller::fromProtobuf(const Symbol &Message) {
   if (!Message.has_info() || !Message.has_canonical_declaration()) {
     elog("Cannot convert Symbol from protobuf (missing info, definition or "
@@ -157,8 +197,7 @@ FuzzyFindRequest Marshaller::toProtobuf(const clangd::FuzzyFindRequest &From) {
   RPCRequest.set_restricted_for_code_completion(From.RestrictForCodeCompletion);
   for (const auto &Path : From.ProximityPaths) {
     llvm::SmallString<256> RelativePath = llvm::StringRef(Path);
-    if (llvm::sys::path::replace_path_prefix(RelativePath, *LocalIndexRoot,
-                                             ""))
+    if (llvm::sys::path::replace_path_prefix(RelativePath, *LocalIndexRoot, ""))
       RPCRequest.add_proximity_paths(llvm::sys::path::convert_to_slash(
           RelativePath, llvm::sys::path::Style::posix));
   }
diff --git a/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.h b/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.h
index 9129cff24db57..5d82cdb7e7650 100644
--- a/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.h
+++ b/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.h
@@ -38,10 +38,15 @@ class Marshaller {
   Marshaller() = delete;
   Marshaller(llvm::StringRef RemoteIndexRoot, llvm::StringRef LocalIndexRoot);
 
-  clangd::FuzzyFindRequest fromProtobuf(const FuzzyFindRequest *Request);
   llvm::Optional<clangd::Symbol> fromProtobuf(const Symbol &Message);
   llvm::Optional<clangd::Ref> fromProtobuf(const Ref &Message);
 
+  llvm::Expected<clangd::LookupRequest>
+  fromProtobuf(const LookupRequest *Message);
+  llvm::Expected<clangd::FuzzyFindRequest>
+  fromProtobuf(const FuzzyFindRequest *Message);
+  llvm::Expected<clangd::RefsRequest> fromProtobuf(const RefsRequest *Message);
+
   /// toProtobuf() functions serialize native clangd types and strip IndexRoot
   /// from the file paths specific to indexing machine. fromProtobuf() functions
   /// deserialize clangd types and translate relative paths into machine-native
diff --git a/clang-tools-extra/clangd/index/remote/server/Server.cpp b/clang-tools-extra/clangd/index/remote/server/Server.cpp
index 07b1c736b6725..7bf47a288e79c 100644
--- a/clang-tools-extra/clangd/index/remote/server/Server.cpp
+++ b/clang-tools-extra/clangd/index/remote/server/Server.cpp
@@ -9,6 +9,7 @@
 #include "index/Index.h"
 #include "index/Serialization.h"
 #include "index/remote/marshalling/Marshalling.h"
+#include "support/Logger.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Path.h"
@@ -59,14 +60,12 @@ class RemoteIndexServer final : public SymbolIndex::Service {
   grpc::Status Lookup(grpc::ServerContext *Context,
                       const LookupRequest *Request,
                       grpc::ServerWriter<LookupReply> *Reply) override {
-    clangd::LookupRequest Req;
-    for (const auto &ID : Request->ids()) {
-      auto SID = SymbolID::fromStr(StringRef(ID));
-      if (!SID)
-        return grpc::Status::CANCELLED;
-      Req.IDs.insert(*SID);
+    auto Req = ProtobufMarshaller->fromProtobuf(Request);
+    if (!Req) {
+      elog("Can not parse LookupRequest from protobuf: {0}", Req.takeError());
+      return grpc::Status::CANCELLED;
     }
-    Index->lookup(Req, [&](const clangd::Symbol &Sym) {
+    Index->lookup(*Req, [&](const clangd::Symbol &Sym) {
       auto SerializedSymbol = ProtobufMarshaller->toProtobuf(Sym);
       if (!SerializedSymbol)
         return;
@@ -83,8 +82,13 @@ class RemoteIndexServer final : public SymbolIndex::Service {
   grpc::Status FuzzyFind(grpc::ServerContext *Context,
                          const FuzzyFindRequest *Request,
                          grpc::ServerWriter<FuzzyFindReply> *Reply) override {
-    const auto Req = ProtobufMarshaller->fromProtobuf(Request);
-    bool HasMore = Index->fuzzyFind(Req, [&](const clangd::Symbol &Sym) {
+    auto Req = ProtobufMarshaller->fromProtobuf(Request);
+    if (!Req) {
+      elog("Can not parse FuzzyFindRequest from protobuf: {0}",
+           Req.takeError());
+      return grpc::Status::CANCELLED;
+    }
+    bool HasMore = Index->fuzzyFind(*Req, [&](const clangd::Symbol &Sym) {
       auto SerializedSymbol = ProtobufMarshaller->toProtobuf(Sym);
       if (!SerializedSymbol)
         return;
@@ -100,14 +104,12 @@ class RemoteIndexServer final : public SymbolIndex::Service {
 
   grpc::Status Refs(grpc::ServerContext *Context, const RefsRequest *Request,
                     grpc::ServerWriter<RefsReply> *Reply) override {
-    clangd::RefsRequest Req;
-    for (const auto &ID : Request->ids()) {
-      auto SID = SymbolID::fromStr(StringRef(ID));
-      if (!SID)
-        return grpc::Status::CANCELLED;
-      Req.IDs.insert(*SID);
+    auto Req = ProtobufMarshaller->fromProtobuf(Request);
+    if (!Req) {
+      elog("Can not parse RefsRequest from protobuf: {0}", Req.takeError());
+      return grpc::Status::CANCELLED;
     }
-    bool HasMore = Index->refs(Req, [&](const clangd::Ref &Reference) {
+    bool HasMore = Index->refs(*Req, [&](const clangd::Ref &Reference) {
       auto SerializedRef = ProtobufMarshaller->toProtobuf(Reference);
       if (!SerializedRef)
         return;
diff --git a/clang-tools-extra/clangd/unittests/remote/MarshallingTests.cpp b/clang-tools-extra/clangd/unittests/remote/MarshallingTests.cpp
index 147601b665c4c..f975d1c35e1ed 100644
--- a/clang-tools-extra/clangd/unittests/remote/MarshallingTests.cpp
+++ b/clang-tools-extra/clangd/unittests/remote/MarshallingTests.cpp
@@ -18,6 +18,7 @@
 #include "clang/Index/IndexSymbol.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Error.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/StringSaver.h"
 #include "gmock/gmock.h"
@@ -271,6 +272,30 @@ TEST(RemoteMarshallingTest, IncludeHeaderURIs) {
   EXPECT_EQ(toYAML(Sym), toYAML(*Deserialized));
 }
 
+TEST(RemoteMarshallingTest, LookupRequestSerialization) {
+  clangd::LookupRequest Request;
+  Request.IDs.insert(llvm::cantFail(SymbolID::fromStr("0000000000000001")));
+  Request.IDs.insert(llvm::cantFail(SymbolID::fromStr("0000000000000002")));
+
+  Marshaller ProtobufMarshaller(testPath("remote/"), testPath("local/"));
+
+  auto Serialized = ProtobufMarshaller.toProtobuf(Request);
+  EXPECT_EQ(static_cast<unsigned>(Serialized.ids_size()), Request.IDs.size());
+  auto Deserialized = ProtobufMarshaller.fromProtobuf(&Serialized);
+  ASSERT_TRUE(bool(Deserialized));
+  EXPECT_EQ(Deserialized->IDs, Request.IDs);
+}
+
+TEST(RemoteMarshallingTest, LookupRequestFailingSerialization) {
+  clangd::LookupRequest Request;
+  Marshaller ProtobufMarshaller(testPath("remote/"), testPath("local/"));
+  auto Serialized = ProtobufMarshaller.toProtobuf(Request);
+  Serialized.add_ids("Invalid Symbol ID");
+  auto Deserialized = ProtobufMarshaller.fromProtobuf(&Serialized);
+  EXPECT_FALSE(Deserialized);
+  llvm::consumeError(Deserialized.takeError());
+}
+
 TEST(RemoteMarshallingTest, FuzzyFindRequestSerialization) {
   clangd::FuzzyFindRequest Request;
   Request.ProximityPaths = {testPath("local/Header.h"),
@@ -280,11 +305,43 @@ TEST(RemoteMarshallingTest, FuzzyFindRequestSerialization) {
   auto Serialized = ProtobufMarshaller.toProtobuf(Request);
   EXPECT_EQ(Serialized.proximity_paths_size(), 2);
   auto Deserialized = ProtobufMarshaller.fromProtobuf(&Serialized);
-  EXPECT_THAT(Deserialized.ProximityPaths,
+  ASSERT_TRUE(bool(Deserialized));
+  EXPECT_THAT(Deserialized->ProximityPaths,
               testing::ElementsAre(testPath("remote/Header.h"),
                                    testPath("remote/subdir/OtherHeader.h")));
 }
 
+TEST(RemoteMarshallingTest, RefsRequestSerialization) {
+  clangd::RefsRequest Request;
+  Request.IDs.insert(llvm::cantFail(SymbolID::fromStr("0000000000000001")));
+  Request.IDs.insert(llvm::cantFail(SymbolID::fromStr("0000000000000002")));
+
+  Request.Limit = 9000;
+  Request.Filter = RefKind::Spelled | RefKind::Declaration;
+
+  Marshaller ProtobufMarshaller(testPath("remote/"), testPath("local/"));
+
+  auto Serialized = ProtobufMarshaller.toProtobuf(Request);
+  EXPECT_EQ(static_cast<unsigned>(Serialized.ids_size()), Request.IDs.size());
+  EXPECT_EQ(Serialized.limit(), Request.Limit);
+  auto Deserialized = ProtobufMarshaller.fromProtobuf(&Serialized);
+  ASSERT_TRUE(bool(Deserialized));
+  EXPECT_EQ(Deserialized->IDs, Request.IDs);
+  ASSERT_TRUE(Deserialized->Limit);
+  EXPECT_EQ(*Deserialized->Limit, Request.Limit);
+  EXPECT_EQ(Deserialized->Filter, Request.Filter);
+}
+
+TEST(RemoteMarshallingTest, RefsRequestFailingSerialization) {
+  clangd::RefsRequest Request;
+  Marshaller ProtobufMarshaller(testPath("remote/"), testPath("local/"));
+  auto Serialized = ProtobufMarshaller.toProtobuf(Request);
+  Serialized.add_ids("Invalid Symbol ID");
+  auto Deserialized = ProtobufMarshaller.fromProtobuf(&Serialized);
+  EXPECT_FALSE(Deserialized);
+  llvm::consumeError(Deserialized.takeError());
+}
+
 TEST(RemoteMarshallingTest, RelativePathToURITranslation) {
   Marshaller ProtobufMarshaller(/*RemoteIndexRoot=*/"",
                                 /*LocalIndexRoot=*/testPath("home/project/"));

From 1ebcf03551c3136b8153d44968204c022f121ae6 Mon Sep 17 00:00:00 2001
From: Sanjay Patel <spatel@rotateright.com>
Date: Sun, 26 Jul 2020 15:19:09 -0400
Subject: [PATCH 0187/1035] [InstSimplify] add tests for min/max intrinsics;
 NFC

---
 .../InstSimplify/maxmin_intrinsics.ll         | 218 ++++++++++++++++++
 1 file changed, 218 insertions(+)

diff --git a/llvm/test/Transforms/InstSimplify/maxmin_intrinsics.ll b/llvm/test/Transforms/InstSimplify/maxmin_intrinsics.ll
index 99a8656b34aa6..13c9f0ebd4abb 100644
--- a/llvm/test/Transforms/InstSimplify/maxmin_intrinsics.ll
+++ b/llvm/test/Transforms/InstSimplify/maxmin_intrinsics.ll
@@ -1,8 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -instsimplify -S | FileCheck %s
 
+declare i81 @llvm.smax.i81(i81, i81)
 declare i8 @llvm.smax.i8(i8, i8)
 declare <2 x i8> @llvm.smax.v2i8(<2 x i8>, <2 x i8>)
+declare i3 @llvm.smin.i3(i3, i3)
 declare i8 @llvm.smin.i8(i8, i8)
 declare <2 x i8> @llvm.smin.v2i8(<2 x i8>, <2 x i8>)
 declare i8 @llvm.umax.i8(i8, i8)
@@ -10,6 +12,78 @@ declare <2 x i8> @llvm.umax.v2i8(<2 x i8>, <2 x i8>)
 declare i8 @llvm.umin.i8(i8, i8)
 declare <2 x i8> @llvm.umin.v2i8(<2 x i8>, <2 x i8>)
 
+define i81 @smax_sameval(i81 %x) {
+; CHECK-LABEL: @smax_sameval(
+; CHECK-NEXT:    [[R:%.*]] = call i81 @llvm.smax.i81(i81 [[X:%.*]], i81 [[X]])
+; CHECK-NEXT:    ret i81 [[R]]
+;
+  %r = call i81 @llvm.smax.i81(i81 %x, i81 %x)
+  ret i81 %r
+}
+
+define i3 @smin_sameval(i3 %x) {
+; CHECK-LABEL: @smin_sameval(
+; CHECK-NEXT:    [[R:%.*]] = call i3 @llvm.smin.i3(i3 [[X:%.*]], i3 [[X]])
+; CHECK-NEXT:    ret i3 [[R]]
+;
+  %r = call i3 @llvm.smin.i3(i3 %x, i3 %x)
+  ret i3 %r
+}
+
+define <2 x i8> @umax_sameval(<2 x i8> %x) {
+; CHECK-LABEL: @umax_sameval(
+; CHECK-NEXT:    [[R:%.*]] = call <2 x i8> @llvm.umax.v2i8(<2 x i8> [[X:%.*]], <2 x i8> [[X]])
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %r = call <2 x i8> @llvm.umax.v2i8(<2 x i8> %x, <2 x i8> %x)
+  ret <2 x i8> %r
+}
+
+define <2 x i8> @umin_sameval(<2 x i8> %x) {
+; CHECK-LABEL: @umin_sameval(
+; CHECK-NEXT:    [[R:%.*]] = call <2 x i8> @llvm.umin.v2i8(<2 x i8> [[X:%.*]], <2 x i8> [[X]])
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %r = call <2 x i8> @llvm.umin.v2i8(<2 x i8> %x, <2 x i8> %x)
+  ret <2 x i8> %r
+}
+
+define i81 @smax_undef(i81 %x) {
+; CHECK-LABEL: @smax_undef(
+; CHECK-NEXT:    [[R:%.*]] = call i81 @llvm.smax.i81(i81 undef, i81 [[X:%.*]])
+; CHECK-NEXT:    ret i81 [[R]]
+;
+  %r = call i81 @llvm.smax.i81(i81 undef, i81 %x)
+  ret i81 %r
+}
+
+define i3 @smin_undef(i3 %x) {
+; CHECK-LABEL: @smin_undef(
+; CHECK-NEXT:    [[R:%.*]] = call i3 @llvm.smin.i3(i3 [[X:%.*]], i3 undef)
+; CHECK-NEXT:    ret i3 [[R]]
+;
+  %r = call i3 @llvm.smin.i3(i3 %x, i3 undef)
+  ret i3 %r
+}
+
+define <2 x i8> @umax_undef(<2 x i8> %x) {
+; CHECK-LABEL: @umax_undef(
+; CHECK-NEXT:    [[R:%.*]] = call <2 x i8> @llvm.umax.v2i8(<2 x i8> undef, <2 x i8> [[X:%.*]])
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %r = call <2 x i8> @llvm.umax.v2i8(<2 x i8> undef, <2 x i8> %x)
+  ret <2 x i8> %r
+}
+
+define <2 x i8> @umin_undef(<2 x i8> %x) {
+; CHECK-LABEL: @umin_undef(
+; CHECK-NEXT:    [[R:%.*]] = call <2 x i8> @llvm.umin.v2i8(<2 x i8> [[X:%.*]], <2 x i8> undef)
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %r = call <2 x i8> @llvm.umin.v2i8(<2 x i8> %x, <2 x i8> undef)
+  ret <2 x i8> %r
+}
+
 define i8 @smax_maxval(i8 %x) {
 ; CHECK-LABEL: @smax_maxval(
 ; CHECK-NEXT:    ret i8 127
@@ -73,3 +147,147 @@ define <2 x i8> @umin_minval_commute(<2 x i8> %x) {
   %r = call <2 x i8> @llvm.umin.v2i8(<2 x i8> %x, <2 x i8> zeroinitializer)
   ret <2 x i8> %r
 }
+
+define i8 @smax_minval(i8 %x) {
+; CHECK-LABEL: @smax_minval(
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.smax.i8(i8 [[X:%.*]], i8 -128)
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %r = call i8 @llvm.smax.i8(i8 %x, i8 -128)
+  ret i8 %r
+}
+
+define <2 x i8> @smax_minval_commute(<2 x i8> %x) {
+; CHECK-LABEL: @smax_minval_commute(
+; CHECK-NEXT:    [[R:%.*]] = call <2 x i8> @llvm.smax.v2i8(<2 x i8> <i8 -128, i8 -128>, <2 x i8> [[X:%.*]])
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %r = call <2 x i8> @llvm.smax.v2i8(<2 x i8> <i8 -128, i8 -128>, <2 x i8> %x)
+  ret <2 x i8> %r
+}
+
+define i8 @smin_maxval(i8 %x) {
+; CHECK-LABEL: @smin_maxval(
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.smin.i8(i8 127, i8 [[X:%.*]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %r = call i8 @llvm.smin.i8(i8 127, i8 %x)
+  ret i8 %r
+}
+
+define <2 x i8> @smin_maxval_commute(<2 x i8> %x) {
+; CHECK-LABEL: @smin_maxval_commute(
+; CHECK-NEXT:    [[R:%.*]] = call <2 x i8> @llvm.smin.v2i8(<2 x i8> [[X:%.*]], <2 x i8> <i8 127, i8 127>)
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %r = call <2 x i8> @llvm.smin.v2i8(<2 x i8> %x, <2 x i8> <i8 127, i8 127>)
+  ret <2 x i8> %r
+}
+
+define i8 @umax_minval(i8 %x) {
+; CHECK-LABEL: @umax_minval(
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.umax.i8(i8 [[X:%.*]], i8 0)
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %r = call i8 @llvm.umax.i8(i8 %x, i8 0)
+  ret i8 %r
+}
+
+define <2 x i8> @umax_minval_commute(<2 x i8> %x) {
+; CHECK-LABEL: @umax_minval_commute(
+; CHECK-NEXT:    [[R:%.*]] = call <2 x i8> @llvm.umax.v2i8(<2 x i8> zeroinitializer, <2 x i8> [[X:%.*]])
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %r = call <2 x i8> @llvm.umax.v2i8(<2 x i8> zeroinitializer, <2 x i8> %x)
+  ret <2 x i8> %r
+}
+
+define i8 @umin_maxval(i8 %x) {
+; CHECK-LABEL: @umin_maxval(
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.umin.i8(i8 -1, i8 [[X:%.*]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %r = call i8 @llvm.umin.i8(i8 255, i8 %x)
+  ret i8 %r
+}
+
+define <2 x i8> @umin_maxval_commute(<2 x i8> %x) {
+; CHECK-LABEL: @umin_maxval_commute(
+; CHECK-NEXT:    [[R:%.*]] = call <2 x i8> @llvm.umin.v2i8(<2 x i8> [[X:%.*]], <2 x i8> <i8 -1, i8 -1>)
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %r = call <2 x i8> @llvm.umin.v2i8(<2 x i8> %x, <2 x i8> <i8 255, i8 255>)
+  ret <2 x i8> %r
+}
+
+define <2 x i8> @smax_maxval_partial_undef(<2 x i8> %x) {
+; CHECK-LABEL: @smax_maxval_partial_undef(
+; CHECK-NEXT:    [[R:%.*]] = call <2 x i8> @llvm.smax.v2i8(<2 x i8> <i8 undef, i8 127>, <2 x i8> [[X:%.*]])
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %r = call <2 x i8> @llvm.smax.v2i8(<2 x i8> <i8 undef, i8 127>, <2 x i8> %x)
+  ret <2 x i8> %r
+}
+
+define <2 x i8> @smin_minval_partial_undef(<2 x i8> %x) {
+; CHECK-LABEL: @smin_minval_partial_undef(
+; CHECK-NEXT:    [[R:%.*]] = call <2 x i8> @llvm.smin.v2i8(<2 x i8> [[X:%.*]], <2 x i8> <i8 -128, i8 undef>)
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %r = call <2 x i8> @llvm.smin.v2i8(<2 x i8> %x, <2 x i8> <i8 -128, i8 undef>)
+  ret <2 x i8> %r
+}
+
+define <2 x i8> @umax_maxval_partial_undef(<2 x i8> %x) {
+; CHECK-LABEL: @umax_maxval_partial_undef(
+; CHECK-NEXT:    [[R:%.*]] = call <2 x i8> @llvm.umax.v2i8(<2 x i8> <i8 -1, i8 undef>, <2 x i8> [[X:%.*]])
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %r = call <2 x i8> @llvm.umax.v2i8(<2 x i8> <i8 255, i8 undef>, <2 x i8> %x)
+  ret <2 x i8> %r
+}
+
+define <2 x i8> @umin_minval_partial_undef(<2 x i8> %x) {
+; CHECK-LABEL: @umin_minval_partial_undef(
+; CHECK-NEXT:    [[R:%.*]] = call <2 x i8> @llvm.umin.v2i8(<2 x i8> [[X:%.*]], <2 x i8> <i8 undef, i8 0>)
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %r = call <2 x i8> @llvm.umin.v2i8(<2 x i8> %x, <2 x i8> <i8 undef, i8 0>)
+  ret <2 x i8> %r
+}
+
+define <2 x i8> @smax_minval_partial_undef(<2 x i8> %x) {
+; CHECK-LABEL: @smax_minval_partial_undef(
+; CHECK-NEXT:    [[R:%.*]] = call <2 x i8> @llvm.smax.v2i8(<2 x i8> <i8 undef, i8 -128>, <2 x i8> [[X:%.*]])
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %r = call <2 x i8> @llvm.smax.v2i8(<2 x i8> <i8 undef, i8 -128>, <2 x i8> %x)
+  ret <2 x i8> %r
+}
+
+define <2 x i8> @smin_maxval_partial_undef(<2 x i8> %x) {
+; CHECK-LABEL: @smin_maxval_partial_undef(
+; CHECK-NEXT:    [[R:%.*]] = call <2 x i8> @llvm.smin.v2i8(<2 x i8> [[X:%.*]], <2 x i8> <i8 undef, i8 127>)
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %r = call <2 x i8> @llvm.smin.v2i8(<2 x i8> %x, <2 x i8> <i8 undef, i8 127>)
+  ret <2 x i8> %r
+}
+
+define <2 x i8> @umax_minval_partial_undef(<2 x i8> %x) {
+; CHECK-LABEL: @umax_minval_partial_undef(
+; CHECK-NEXT:    [[R:%.*]] = call <2 x i8> @llvm.umax.v2i8(<2 x i8> <i8 0, i8 undef>, <2 x i8> [[X:%.*]])
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %r = call <2 x i8> @llvm.umax.v2i8(<2 x i8> <i8 0, i8 undef>, <2 x i8> %x)
+  ret <2 x i8> %r
+}
+
+define <2 x i8> @umin_maxval_partial_undef(<2 x i8> %x) {
+; CHECK-LABEL: @umin_maxval_partial_undef(
+; CHECK-NEXT:    [[R:%.*]] = call <2 x i8> @llvm.umin.v2i8(<2 x i8> [[X:%.*]], <2 x i8> <i8 -1, i8 undef>)
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %r = call <2 x i8> @llvm.umin.v2i8(<2 x i8> %x, <2 x i8> <i8 255, i8 undef>)
+  ret <2 x i8> %r
+}

From db203e0268479d16d36a318c726cce5a4a5f75a6 Mon Sep 17 00:00:00 2001
From: Raphael Isemann <teemperor@gmail.com>
Date: Mon, 27 Jul 2020 14:05:23 +0200
Subject: [PATCH 0188/1035] [lldb] Modernize away some snprintf calls

Reviewers: #lldb, JDevlieghere

Reviewed By: #lldb, JDevlieghere

Subscribers: JDevlieghere

Differential Revision: https://reviews.llvm.org/D84530
---
 lldb/source/Commands/CommandObjectProcess.cpp | 20 +++++++++----------
 lldb/source/Core/Communication.cpp            |  5 ++---
 lldb/source/Core/Debugger.cpp                 |  4 +---
 lldb/source/Core/SourceManager.cpp            |  9 +++++----
 lldb/source/Core/ValueObject.cpp              | 10 +++-------
 lldb/source/Core/ValueObjectChild.cpp         | 11 ++--------
 .../source/Interpreter/CommandInterpreter.cpp | 10 ++--------
 7 files changed, 25 insertions(+), 44 deletions(-)

diff --git a/lldb/source/Commands/CommandObjectProcess.cpp b/lldb/source/Commands/CommandObjectProcess.cpp
index f86779d85b5fa..fd8d38e856372 100644
--- a/lldb/source/Commands/CommandObjectProcess.cpp
+++ b/lldb/source/Commands/CommandObjectProcess.cpp
@@ -48,19 +48,19 @@ class CommandObjectProcessLaunchOrAttach : public CommandObjectParsed {
       state = process->GetState();
 
       if (process->IsAlive() && state != eStateConnected) {
-        char message[1024];
+        std::string message;
         if (process->GetState() == eStateAttaching)
-          ::snprintf(message, sizeof(message),
-                     "There is a pending attach, abort it and %s?",
-                     m_new_process_action.c_str());
+          message =
+              llvm::formatv("There is a pending attach, abort it and {0}?",
+                            m_new_process_action);
         else if (process->GetShouldDetach())
-          ::snprintf(message, sizeof(message),
-                     "There is a running process, detach from it and %s?",
-                     m_new_process_action.c_str());
+          message = llvm::formatv(
+              "There is a running process, detach from it and {0}?",
+              m_new_process_action);
         else
-          ::snprintf(message, sizeof(message),
-                     "There is a running process, kill it and %s?",
-                     m_new_process_action.c_str());
+          message =
+              llvm::formatv("There is a running process, kill it and {0}?",
+                            m_new_process_action);
 
         if (!m_interpreter.Confirm(message, true)) {
           result.SetStatus(eReturnStatusFailed);
diff --git a/lldb/source/Core/Communication.cpp b/lldb/source/Core/Communication.cpp
index b358e70b1a91e..859f5be74b439 100644
--- a/lldb/source/Core/Communication.cpp
+++ b/lldb/source/Core/Communication.cpp
@@ -199,9 +199,8 @@ bool Communication::StartReadThread(Status *error_ptr) {
   LLDB_LOG(lldb_private::GetLogIfAllCategoriesSet(LIBLLDB_LOG_COMMUNICATION),
            "{0} Communication::StartReadThread ()", this);
 
-  char thread_name[1024];
-  snprintf(thread_name, sizeof(thread_name), "<lldb.comm.%s>",
-           GetBroadcasterName().AsCString());
+  const std::string thread_name =
+      llvm::formatv("<lldb.comm.{0}>", GetBroadcasterName());
 
   m_read_thread_enabled = true;
   m_read_thread_did_exit = false;
diff --git a/lldb/source/Core/Debugger.cpp b/lldb/source/Core/Debugger.cpp
index 5f4f1e266d81c..05cfac19915e1 100644
--- a/lldb/source/Core/Debugger.cpp
+++ b/lldb/source/Core/Debugger.cpp
@@ -666,9 +666,7 @@ Debugger::Debugger(lldb::LogOutputCallback log_callback, void *baton)
       m_event_handler_thread(), m_io_handler_thread(),
       m_sync_broadcaster(nullptr, "lldb.debugger.sync"),
       m_forward_listener_sp(), m_clear_once() {
-  char instance_cstr[256];
-  snprintf(instance_cstr, sizeof(instance_cstr), "debugger_%d", (int)GetID());
-  m_instance_name.SetCString(instance_cstr);
+  m_instance_name.SetString(llvm::formatv("debugger_{0}", GetID()).str());
   if (log_callback)
     m_log_callback_stream_sp =
         std::make_shared<StreamCallback>(log_callback, baton);
diff --git a/lldb/source/Core/SourceManager.cpp b/lldb/source/Core/SourceManager.cpp
index 7414dd281d43a..e79fcb48742d8 100644
--- a/lldb/source/Core/SourceManager.cpp
+++ b/lldb/source/Core/SourceManager.cpp
@@ -183,14 +183,14 @@ size_t SourceManager::DisplaySourceLinesWithLineNumbersUsingLastFile(
         break;
       }
 
-      char prefix[32] = "";
+      std::string prefix;
       if (bp_locs) {
         uint32_t bp_count = bp_locs->NumLineEntriesWithLine(line);
 
         if (bp_count > 0)
-          ::snprintf(prefix, sizeof(prefix), "[%u] ", bp_count);
+          prefix = llvm::formatv("[{0}]", bp_count);
         else
-          ::snprintf(prefix, sizeof(prefix), "    ");
+          prefix = "    ";
       }
 
       char buffer[3];
@@ -206,7 +206,8 @@ size_t SourceManager::DisplaySourceLinesWithLineNumbersUsingLastFile(
                 .str());
       }
 
-      s->Printf("%s%s %-4u\t", prefix, current_line_highlight.c_str(), line);
+      s->Printf("%s%s %-4u\t", prefix.c_str(), current_line_highlight.c_str(),
+                line);
 
       // So far we treated column 0 as a special 'no column value', but
       // DisplaySourceLines starts counting columns from 0 (and no column is
diff --git a/lldb/source/Core/ValueObject.cpp b/lldb/source/Core/ValueObject.cpp
index 3a775b07e5e1f..d3a1971235ca0 100644
--- a/lldb/source/Core/ValueObject.cpp
+++ b/lldb/source/Core/ValueObject.cpp
@@ -1702,8 +1702,7 @@ ValueObjectSP ValueObject::GetSyntheticArrayMember(size_t index,
                                                    bool can_create) {
   ValueObjectSP synthetic_child_sp;
   if (IsPointerType() || IsArrayType()) {
-    char index_str[64];
-    snprintf(index_str, sizeof(index_str), "[%" PRIu64 "]", (uint64_t)index);
+    std::string index_str = llvm::formatv("[{0}]", index);
     ConstString index_const_str(index_str);
     // Check if we have already created a synthetic array member in this valid
     // object. If we have we will re-use it.
@@ -1730,8 +1729,7 @@ ValueObjectSP ValueObject::GetSyntheticBitFieldChild(uint32_t from, uint32_t to,
                                                      bool can_create) {
   ValueObjectSP synthetic_child_sp;
   if (IsScalarType()) {
-    char index_str[64];
-    snprintf(index_str, sizeof(index_str), "[%i-%i]", from, to);
+    std::string index_str = llvm::formatv("[{0}-{1}]", from, to);
     ConstString index_const_str(index_str);
     // Check if we have already created a synthetic array member in this valid
     // object. If we have we will re-use it.
@@ -1768,9 +1766,7 @@ ValueObjectSP ValueObject::GetSyntheticChildAtOffset(
   ValueObjectSP synthetic_child_sp;
 
   if (name_const_str.IsEmpty()) {
-    char name_str[64];
-    snprintf(name_str, sizeof(name_str), "@%i", offset);
-    name_const_str.SetCString(name_str);
+    name_const_str.SetString("@" + std::to_string(offset));
   }
 
   // Check if we have already created a synthetic array member in this valid
diff --git a/lldb/source/Core/ValueObjectChild.cpp b/lldb/source/Core/ValueObjectChild.cpp
index 6205ed32c615a..28cb49328f34e 100644
--- a/lldb/source/Core/ValueObjectChild.cpp
+++ b/lldb/source/Core/ValueObjectChild.cpp
@@ -57,15 +57,8 @@ size_t ValueObjectChild::CalculateNumChildren(uint32_t max) {
 
 static void AdjustForBitfieldness(ConstString &name,
                                   uint8_t bitfield_bit_size) {
-  if (name && bitfield_bit_size) {
-    const char *compiler_type_name = name.AsCString();
-    if (compiler_type_name) {
-      std::vector<char> bitfield_type_name(strlen(compiler_type_name) + 32, 0);
-      ::snprintf(&bitfield_type_name.front(), bitfield_type_name.size(),
-                 "%s:%u", compiler_type_name, bitfield_bit_size);
-      name.SetCString(&bitfield_type_name.front());
-    }
-  }
+  if (name && bitfield_bit_size)
+    name.SetString(llvm::formatv("{0}:{1}", name, bitfield_bit_size).str());
 }
 
 ConstString ValueObjectChild::GetTypeName() {
diff --git a/lldb/source/Interpreter/CommandInterpreter.cpp b/lldb/source/Interpreter/CommandInterpreter.cpp
index 50a69b29260ca..aca3654b03097 100644
--- a/lldb/source/Interpreter/CommandInterpreter.cpp
+++ b/lldb/source/Interpreter/CommandInterpreter.cpp
@@ -1987,10 +1987,7 @@ void CommandInterpreter::BuildAliasCommandArgs(CommandObject *alias_cmd_obj,
         if (value_type != OptionParser::eOptionalArgument)
           new_args.AppendArgument(value);
         else {
-          char buffer[255];
-          ::snprintf(buffer, sizeof(buffer), "%s%s", option.c_str(),
-                     value.c_str());
-          new_args.AppendArgument(llvm::StringRef(buffer));
+          new_args.AppendArgument(option + value);
         }
 
       } else if (static_cast<size_t>(index) >= cmd_args.GetArgumentCount()) {
@@ -2012,10 +2009,7 @@ void CommandInterpreter::BuildAliasCommandArgs(CommandObject *alias_cmd_obj,
         if (value_type != OptionParser::eOptionalArgument)
           new_args.AppendArgument(cmd_args.GetArgumentAtIndex(index));
         else {
-          char buffer[255];
-          ::snprintf(buffer, sizeof(buffer), "%s%s", option.c_str(),
-                     cmd_args.GetArgumentAtIndex(index));
-          new_args.AppendArgument(buffer);
+          new_args.AppendArgument(option + cmd_args.GetArgumentAtIndex(index));
         }
         used[index] = true;
       }

From 90684d1545167ee4e0c93d8eaf6ba4a3c7ab710e Mon Sep 17 00:00:00 2001
From: Aleksandr Platonov <platonov.aleksandr@huawei.com>
Date: Mon, 27 Jul 2020 14:39:31 +0300
Subject: [PATCH 0189/1035] [clangd] Collect references for externally visible
 main-file symbols

Summary:
Without this patch clangd does not collect references for main-file symbols if there is no public declaration in preamble.
Example:
`test1.c`
```
void f1() {}
```

`test2.c`
```
extern void f1();
void f2() {
  f^1();
}
```
`Find all references` does not show definition of f1() in the result, but GTD works OK.

Reviewers: sammccall, kadircet

Reviewed By: kadircet

Subscribers: ilya-golovenko, ilya-biryukov, MaskRay, jkorous, arphaman, kadircet, usaxena95, cfe-commits

Tags: #clang

Differential Revision: https://reviews.llvm.org/D84513
---
 .../clangd/index/SymbolCollector.cpp           |  3 ++-
 .../clangd/unittests/BackgroundIndexTests.cpp  |  2 +-
 .../clangd/unittests/SymbolCollectorTests.cpp  | 18 ++++++++++++------
 3 files changed, 15 insertions(+), 8 deletions(-)

diff --git a/clang-tools-extra/clangd/index/SymbolCollector.cpp b/clang-tools-extra/clangd/index/SymbolCollector.cpp
index 6c11399c87b68..c163951aff9ba 100644
--- a/clang-tools-extra/clangd/index/SymbolCollector.cpp
+++ b/clang-tools-extra/clangd/index/SymbolCollector.cpp
@@ -314,7 +314,8 @@ bool SymbolCollector::handleDeclOccurrence(
   // file locations for references (as it aligns the behavior of clangd's
   // AST-based xref).
   // FIXME: we should try to use the file locations for other fields.
-  if (CollectRef && !IsMainFileOnly && !isa<NamespaceDecl>(ND) &&
+  if (CollectRef && (!IsMainFileOnly || ND->isExternallyVisible()) &&
+      !isa<NamespaceDecl>(ND) &&
       (Opts.RefsInHeaders ||
        SM.getFileID(SM.getFileLoc(Loc)) == SM.getMainFileID()))
     DeclRefs[ND].emplace_back(SM.getFileLoc(Loc), Roles);
diff --git a/clang-tools-extra/clangd/unittests/BackgroundIndexTests.cpp b/clang-tools-extra/clangd/unittests/BackgroundIndexTests.cpp
index 70d5156b10723..f1c582ef1abe5 100644
--- a/clang-tools-extra/clangd/unittests/BackgroundIndexTests.cpp
+++ b/clang-tools-extra/clangd/unittests/BackgroundIndexTests.cpp
@@ -171,7 +171,7 @@ TEST_F(BackgroundIndexTest, IndexTwoFiles) {
       #endif
       )cpp";
   FS.Files[testPath("root/A.cc")] =
-      "#include \"A.h\"\nvoid g() { (void)common; }";
+      "#include \"A.h\"\nstatic void g() { (void)common; }";
   FS.Files[testPath("root/B.cc")] =
       R"cpp(
       #define A 0
diff --git a/clang-tools-extra/clangd/unittests/SymbolCollectorTests.cpp b/clang-tools-extra/clangd/unittests/SymbolCollectorTests.cpp
index 9e4f75b5cca3f..3614ab2c5cb9e 100644
--- a/clang-tools-extra/clangd/unittests/SymbolCollectorTests.cpp
+++ b/clang-tools-extra/clangd/unittests/SymbolCollectorTests.cpp
@@ -624,11 +624,13 @@ TEST_F(SymbolCollectorTest, Refs) {
   EXPECT_THAT(Refs, Not(Contains(Pair(findSymbol(Symbols, "NS").ID, _))));
   EXPECT_THAT(Refs, Contains(Pair(findSymbol(Symbols, "MACRO").ID,
                                   HaveRanges(Main.ranges("macro")))));
-  // Symbols *only* in the main file (a, b, c, FUNC) had no refs collected.
+  // Symbols *only* in the main file:
+  // - (a, b) externally visible and should have refs.
+  // - (c, FUNC) externally invisible and had no refs collected.
   auto MainSymbols =
       TestTU::withHeaderCode(SymbolsOnlyInMainCode.code()).headerSymbols();
-  EXPECT_THAT(Refs, Not(Contains(Pair(findSymbol(MainSymbols, "a").ID, _))));
-  EXPECT_THAT(Refs, Not(Contains(Pair(findSymbol(MainSymbols, "b").ID, _))));
+  EXPECT_THAT(Refs, Contains(Pair(findSymbol(MainSymbols, "a").ID, _)));
+  EXPECT_THAT(Refs, Contains(Pair(findSymbol(MainSymbols, "b").ID, _)));
   EXPECT_THAT(Refs, Not(Contains(Pair(findSymbol(MainSymbols, "c").ID, _))));
   EXPECT_THAT(Refs, Not(Contains(Pair(findSymbol(MainSymbols, "FUNC").ID, _))));
 }
@@ -816,11 +818,15 @@ TEST_F(SymbolCollectorTest, HeaderAsMainFile) {
     $Foo[[Foo]] fo;
   }
   )");
-  // The main file is normal .cpp file, we shouldn't collect any refs of symbols
-  // which are not declared in the preamble.
+  // The main file is normal .cpp file, we should collect the refs
+  // for externally visible symbols.
   TestFileName = testPath("foo.cpp");
   runSymbolCollector("", Header.code());
-  EXPECT_THAT(Refs, UnorderedElementsAre());
+  EXPECT_THAT(Refs,
+              UnorderedElementsAre(Pair(findSymbol(Symbols, "Foo").ID,
+                                        HaveRanges(Header.ranges("Foo"))),
+                                   Pair(findSymbol(Symbols, "Func").ID,
+                                        HaveRanges(Header.ranges("Func")))));
 
   // Run the .h file as main file, we should collect the refs.
   TestFileName = testPath("foo.h");

From 432241955e032fba3d8b584ee6388212909bee9b Mon Sep 17 00:00:00 2001
From: Raphael Isemann <teemperor@gmail.com>
Date: Mon, 27 Jul 2020 14:28:30 +0200
Subject: [PATCH 0190/1035] [lldb][NFC] Use a StringRef for
 AddRegexCommand::AddRegexCommand parameters

Summary: This way we can get rid of this 1024 char buffer workaround.

Reviewers: #lldb, labath

Reviewed By: labath

Subscribers: JDevlieghere

Differential Revision: https://reviews.llvm.org/D84528
---
 .../lldb/Interpreter/CommandObjectRegexCommand.h      |  2 +-
 lldb/source/Commands/CommandObjectCommands.cpp        |  2 +-
 lldb/source/Interpreter/CommandInterpreter.cpp        | 11 +++--------
 lldb/source/Interpreter/CommandObjectRegexCommand.cpp |  9 ++++-----
 4 files changed, 9 insertions(+), 15 deletions(-)

diff --git a/lldb/include/lldb/Interpreter/CommandObjectRegexCommand.h b/lldb/include/lldb/Interpreter/CommandObjectRegexCommand.h
index 01d7c6d118d46..cbd50511c483c 100644
--- a/lldb/include/lldb/Interpreter/CommandObjectRegexCommand.h
+++ b/lldb/include/lldb/Interpreter/CommandObjectRegexCommand.h
@@ -30,7 +30,7 @@ class CommandObjectRegexCommand : public CommandObjectRaw {
 
   bool IsRemovable() const override { return m_is_removable; }
 
-  bool AddRegexCommand(const char *re_cstr, const char *command_cstr);
+  bool AddRegexCommand(llvm::StringRef re_cstr, llvm::StringRef command_cstr);
 
   bool HasRegexEntries() const { return !m_entries.empty(); }
 
diff --git a/lldb/source/Commands/CommandObjectCommands.cpp b/lldb/source/Commands/CommandObjectCommands.cpp
index 255fbe53fb2ea..eaf22344fafa2 100644
--- a/lldb/source/Commands/CommandObjectCommands.cpp
+++ b/lldb/source/Commands/CommandObjectCommands.cpp
@@ -970,7 +970,7 @@ a number follows 'f':"
       std::string subst(std::string(regex_sed.substr(
           second_separator_char_pos + 1,
           third_separator_char_pos - second_separator_char_pos - 1)));
-      m_regex_cmd_up->AddRegexCommand(regex.c_str(), subst.c_str());
+      m_regex_cmd_up->AddRegexCommand(regex, subst);
     }
     return error;
   }
diff --git a/lldb/source/Interpreter/CommandInterpreter.cpp b/lldb/source/Interpreter/CommandInterpreter.cpp
index aca3654b03097..4786e4602e4b4 100644
--- a/lldb/source/Interpreter/CommandInterpreter.cpp
+++ b/lldb/source/Interpreter/CommandInterpreter.cpp
@@ -631,15 +631,10 @@ void CommandInterpreter::LoadCommandDictionary() {
   if (tbreak_regex_cmd_up) {
     bool success = true;
     for (size_t i = 0; i < num_regexes; i++) {
-      // If you add a resultant command string longer than 1024 characters be
-      // sure to increase the size of this buffer.
-      char buffer[1024];
-      int num_printed =
-          snprintf(buffer, 1024, "%s %s", break_regexes[i][1], "-o 1");
-      lldbassert(num_printed < 1024);
-      UNUSED_IF_ASSERT_DISABLED(num_printed);
+      std::string command = break_regexes[i][1];
+      command += " -o 1";
       success =
-          tbreak_regex_cmd_up->AddRegexCommand(break_regexes[i][0], buffer);
+          tbreak_regex_cmd_up->AddRegexCommand(break_regexes[i][0], command);
       if (!success)
         break;
     }
diff --git a/lldb/source/Interpreter/CommandObjectRegexCommand.cpp b/lldb/source/Interpreter/CommandObjectRegexCommand.cpp
index 5a0265e58c5c6..7485fd76cc25f 100644
--- a/lldb/source/Interpreter/CommandObjectRegexCommand.cpp
+++ b/lldb/source/Interpreter/CommandObjectRegexCommand.cpp
@@ -69,14 +69,13 @@ bool CommandObjectRegexCommand::DoExecute(llvm::StringRef command,
   return false;
 }
 
-bool CommandObjectRegexCommand::AddRegexCommand(const char *re_cstr,
-                                                const char *command_cstr) {
+bool CommandObjectRegexCommand::AddRegexCommand(llvm::StringRef re_cstr,
+                                                llvm::StringRef command_cstr) {
   m_entries.resize(m_entries.size() + 1);
   // Only add the regular expression if it compiles
-  m_entries.back().regex =
-      RegularExpression(llvm::StringRef::withNullAsEmpty(re_cstr));
+  m_entries.back().regex = RegularExpression(re_cstr);
   if (m_entries.back().regex.IsValid()) {
-    m_entries.back().command.assign(command_cstr);
+    m_entries.back().command = command_cstr.str();
     return true;
   }
   // The regex didn't compile...

From d1271127240b0920e8758519d95a948d03a832ad Mon Sep 17 00:00:00 2001
From: Nathan James <n.james93@hotmail.co.uk>
Date: Mon, 27 Jul 2020 13:37:21 +0100
Subject: [PATCH 0191/1035] [llvm][NFC] Silence unused variable warning by
 using isa over dyn_cast

---
 llvm/lib/Transforms/InstCombine/InstructionCombining.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
index 711be57a0bafa..965a72d3a6500 100644
--- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
@@ -1143,7 +1143,7 @@ Instruction *InstCombinerImpl::foldOpIntoPhi(Instruction &I, PHINode *PN) {
                                              Builder);
       NewPN->addIncoming(InV, PN->getIncomingBlock(i));
     }
-  } else if (auto *FI = dyn_cast<FreezeInst>(&I)) {
+  } else if (isa<FreezeInst>(&I)) {
     for (unsigned i = 0; i != NumPHIValues; ++i) {
       Value *InV;
       if (NonConstBB == PN->getIncomingBlock(i))

From 1bac5101cdaabfbc755a6d28936962d11240f932 Mon Sep 17 00:00:00 2001
From: Roman Lebedev <lebedev.ri@gmail.com>
Date: Mon, 27 Jul 2020 13:30:39 +0300
Subject: [PATCH 0192/1035] [Reduce] Function reduction: replace all users of
 function with undef

There may be other users of a function other than CallInsts,
but what's more important, we can't actually replace function pointer
with undef, because for constants, that would not preserve the type
and RAUW would assert.

In particular, that affects blockaddress, however it proves to be
prohibitively complex to come up with a good test involving blockaddress:
we'd need to both ensure that the function body survives until
this pass, and is not interesting in this pass.
---
 .../llvm-reduce/deltas/ReduceFunctions.cpp    | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/llvm/tools/llvm-reduce/deltas/ReduceFunctions.cpp b/llvm/tools/llvm-reduce/deltas/ReduceFunctions.cpp
index b29df88261d91..4dd7b98fca3b6 100644
--- a/llvm/tools/llvm-reduce/deltas/ReduceFunctions.cpp
+++ b/llvm/tools/llvm-reduce/deltas/ReduceFunctions.cpp
@@ -32,22 +32,21 @@ static void extractFunctionsFromModule(const std::vector<Chunk> &ChunksToKeep,
     if (O.shouldKeep())
       FuncsToKeep.insert(&F);
 
-  // Delete out-of-chunk functions, and replace their calls with undef
+  // Delete out-of-chunk functions, and replace their users with undef
   std::vector<Function *> FuncsToRemove;
-  SetVector<CallInst *> CallsToRemove;
+  SetVector<Instruction *> InstrsToRemove;
   for (auto &F : *Program)
     if (!FuncsToKeep.count(&F)) {
-      for (auto U : F.users())
-        if (auto *Call = dyn_cast<CallInst>(U)) {
-          Call->replaceAllUsesWith(UndefValue::get(Call->getType()));
-          CallsToRemove.insert(Call);
-        }
-      F.replaceAllUsesWith(UndefValue::get(F.getType()));
+      for (auto U : F.users()) {
+        U->replaceAllUsesWith(UndefValue::get(U->getType()));
+        if (auto *I = dyn_cast<Instruction>(U))
+          InstrsToRemove.insert(I);
+      }
       FuncsToRemove.push_back(&F);
     }
 
-  for (auto *C : CallsToRemove)
-    C->eraseFromParent();
+  for (auto *I : InstrsToRemove)
+    I->eraseFromParent();
 
   for (auto *F : FuncsToRemove)
     F->eraseFromParent();

From 61480db6019d01a7a97de6ec64991664bf9b4996 Mon Sep 17 00:00:00 2001
From: Roman Lebedev <lebedev.ri@gmail.com>
Date: Mon, 27 Jul 2020 15:36:07 +0300
Subject: [PATCH 0193/1035] [Reduce] Argument reduction: shoe-horn new function
 into remaining uses of old function

Much like with function reduction, there may be remaining unhandled uses
of function, in particular in blockaddress. And in constants we can't
RAUW it with undef, because undef is not a function.
Instead, let's try to pretent that in the remaining cases, the new
signature didn't change, by bitcasting it.

A new (previously crashing) test case added.
---
 ...arguments-of-funcs-used-in-blockaddress.ll | 30 +++++++++++++++++++
 llvm/test/Reduce/remove-invoked-functions.ll  |  2 +-
 .../llvm-reduce/deltas/ReduceArguments.cpp    |  2 +-
 3 files changed, 32 insertions(+), 2 deletions(-)
 create mode 100644 llvm/test/Reduce/remove-function-arguments-of-funcs-used-in-blockaddress.ll

diff --git a/llvm/test/Reduce/remove-function-arguments-of-funcs-used-in-blockaddress.ll b/llvm/test/Reduce/remove-function-arguments-of-funcs-used-in-blockaddress.ll
new file mode 100644
index 0000000000000..f1ad5db49bfb6
--- /dev/null
+++ b/llvm/test/Reduce/remove-function-arguments-of-funcs-used-in-blockaddress.ll
@@ -0,0 +1,30 @@
+; RUN: llvm-reduce --test FileCheck --test-arg --check-prefixes=CHECK-ALL,CHECK-INTERESTINGNESS --test-arg %s --test-arg --input-file %s -o %t
+; RUN: cat %t | FileCheck --check-prefixes=CHECK-ALL,CHECK-FINAL %s
+
+; CHECK-INTERESTINGNESS: define void @func(
+; CHECK-FINAL: define void @func()
+define void @func(i1 %arg) {
+; CHECK-ALL: bb:
+; CHECK-ALL: br label %bb4
+bb:
+  br label %bb4
+
+; CHECK-ALL: bb4
+bb4:
+; CHECK-INTERESTINGNESS; callbr void asm
+; CHECK-INTERESTINGNESS-SAME; blockaddress
+; CHECK-FINAL: callbr void asm sideeffect "", "X"(i8* blockaddress(@func, %bb11))
+; CHECK-ALL: to label %bb5 [label %bb11]
+  callbr void asm sideeffect "", "X"(i8* blockaddress(@func, %bb11))
+          to label %bb5 [label %bb11]
+
+; CHECK-ALL: bb5:
+; CHECK-ALL: br label %bb11
+bb5:
+  br label %bb11
+
+; CHECK-ALL: bb11:
+; CHECK-ALL: ret void
+bb11:
+  ret void
+}
diff --git a/llvm/test/Reduce/remove-invoked-functions.ll b/llvm/test/Reduce/remove-invoked-functions.ll
index e4458e662fee3..d291859478033 100644
--- a/llvm/test/Reduce/remove-invoked-functions.ll
+++ b/llvm/test/Reduce/remove-invoked-functions.ll
@@ -23,7 +23,7 @@ define void @caller(i32 %arg) personality i8* bitcast (i32 (...)* @__gxx_persona
 ; CHECK-ALL: bb:
 bb:
 ; CHECK-INTERESTINGNESS: %i0 = invoke i32
-; CHECK-FINAL: %i0 = invoke i32 undef(i32 %arg)
+; CHECK-FINAL: %i0 = invoke i32 bitcast (i32 ()* @maybe_throwing_callee to i32 (i32)*)(i32 %arg)
 ; CHECK-ALL: to label %bb3 unwind label %bb1
   %i0 = invoke i32 @maybe_throwing_callee(i32 %arg)
           to label %bb3 unwind label %bb1
diff --git a/llvm/tools/llvm-reduce/deltas/ReduceArguments.cpp b/llvm/tools/llvm-reduce/deltas/ReduceArguments.cpp
index 9488d71b71c35..c3c7dee83db10 100644
--- a/llvm/tools/llvm-reduce/deltas/ReduceArguments.cpp
+++ b/llvm/tools/llvm-reduce/deltas/ReduceArguments.cpp
@@ -94,7 +94,7 @@ static void extractArgumentsFromModule(std::vector<Chunk> ChunksToKeep,
     replaceFunctionCalls(*F, *ClonedFunc, ArgIndexesToKeep);
     // Rename Cloned Function to Old's name
     std::string FName = std::string(F->getName());
-    F->replaceAllUsesWith(UndefValue::get(F->getType()));
+    F->replaceAllUsesWith(ConstantExpr::getBitCast(ClonedFunc, F->getType()));
     F->eraseFromParent();
     ClonedFunc->setName(FName);
   }

From 1da9834557cd4302a5183b8228ce063e69f82602 Mon Sep 17 00:00:00 2001
From: Roman Lebedev <lebedev.ri@gmail.com>
Date: Mon, 27 Jul 2020 15:07:51 +0300
Subject: [PATCH 0194/1035] [JumpThreading] ProcessBranchOnXOR(): bailout if
 any pred ends in indirect branch (PR46857)

SplitBlockPredecessors() can not split blocks that have such terminators,
and in two other places we already ensure that we don't end up calling
SplitBlockPredecessors() on such blocks. Do so in one more place.

Fixes https://bugs.llvm.org/show_bug.cgi?id=46857
---
 llvm/lib/Transforms/Scalar/JumpThreading.cpp  |  8 +++
 .../JumpThreading/pr46857-callbr.ll           | 52 +++++++++++++++++++
 2 files changed, 60 insertions(+)
 create mode 100644 llvm/test/Transforms/JumpThreading/pr46857-callbr.ll

diff --git a/llvm/lib/Transforms/Scalar/JumpThreading.cpp b/llvm/lib/Transforms/Scalar/JumpThreading.cpp
index 9d0500419a7f5..2f379b7f61608 100644
--- a/llvm/lib/Transforms/Scalar/JumpThreading.cpp
+++ b/llvm/lib/Transforms/Scalar/JumpThreading.cpp
@@ -1859,6 +1859,14 @@ bool JumpThreadingPass::ProcessBranchOnXOR(BinaryOperator *BO) {
     return true;
   }
 
+  // If any of predecessors end with an indirect goto, we can't change its
+  // destination. Same for CallBr.
+  if (any_of(BlocksToFoldInto, [](BasicBlock *Pred) {
+        return isa<IndirectBrInst>(Pred->getTerminator()) ||
+               isa<CallBrInst>(Pred->getTerminator());
+      }))
+    return false;
+
   // Try to duplicate BB into PredBB.
   return DuplicateCondBranchOnPHIIntoPred(BB, BlocksToFoldInto);
 }
diff --git a/llvm/test/Transforms/JumpThreading/pr46857-callbr.ll b/llvm/test/Transforms/JumpThreading/pr46857-callbr.ll
new file mode 100644
index 0000000000000..3de7d6265136d
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/pr46857-callbr.ll
@@ -0,0 +1,52 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -jump-threading -S | FileCheck %s
+
+; CHECK-ALL-LABEL: @func(
+
+define i1 @func(i1 %arg, i32 %arg1, i1 %arg2) {
+; CHECK-LABEL: @func(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    br i1 [[ARG:%.*]], label [[BB3:%.*]], label [[BB4:%.*]]
+; CHECK:       bb3:
+; CHECK-NEXT:    [[I:%.*]] = icmp eq i32 [[ARG1:%.*]], 0
+; CHECK-NEXT:    br label [[BB7:%.*]]
+; CHECK:       bb4:
+; CHECK-NEXT:    callbr void asm sideeffect "", "X"(i8* blockaddress(@func, [[BB7]]))
+; CHECK-NEXT:    to label [[BB5:%.*]] [label %bb7]
+; CHECK:       bb5:
+; CHECK-NEXT:    br label [[BB7]]
+; CHECK:       bb7:
+; CHECK-NEXT:    [[I8:%.*]] = phi i1 [ [[I]], [[BB3]] ], [ [[ARG2:%.*]], [[BB5]] ], [ [[ARG2]], [[BB4]] ]
+; CHECK-NEXT:    [[I9:%.*]] = xor i1 [[I8]], [[ARG]]
+; CHECK-NEXT:    br i1 [[I9]], label [[BB11:%.*]], label [[BB11]]
+; CHECK:       bb11:
+; CHECK-NEXT:    ret i1 [[I9]]
+;
+bb:
+  br i1 %arg, label %bb3, label %bb4
+
+bb3:
+  %i = icmp eq i32 %arg1, 0
+  br label %bb7
+
+bb4:
+  callbr void asm sideeffect "", "X"(i8* blockaddress(@func, %bb6))
+  to label %bb5 [label %bb6]
+
+bb5:
+  br label %bb6
+
+bb6:
+  br label %bb7
+
+bb7:
+  %i8 = phi i1 [ %i, %bb3 ], [ %arg2, %bb6 ]
+  %i9 = xor i1 %i8, %arg
+  br i1 %i9, label %bb11, label %bb10
+
+bb10:
+  br label %bb11
+
+bb11:
+  ret i1 %i9
+}

From 08e9556d5d77fb424b8cb99fe16ffe2bc77f555e Mon Sep 17 00:00:00 2001
From: Hans Wennborg <hans@chromium.org>
Date: Mon, 27 Jul 2020 14:41:29 +0200
Subject: [PATCH 0195/1035] 
 llvm_canonicalize_cmake_booleans(CLANGD_ENABLE_REMOTE)

Otherwise it got defined as e.g. OFF in Features.inc.
---
 clang-tools-extra/clangd/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/clang-tools-extra/clangd/CMakeLists.txt b/clang-tools-extra/clangd/CMakeLists.txt
index 8db6656e5291a..639441e8130ab 100644
--- a/clang-tools-extra/clangd/CMakeLists.txt
+++ b/clang-tools-extra/clangd/CMakeLists.txt
@@ -15,6 +15,7 @@ if (NOT DEFINED CLANGD_BUILD_XPC)
 endif ()
 
 llvm_canonicalize_cmake_booleans(CLANGD_BUILD_XPC)
+llvm_canonicalize_cmake_booleans(CLANGD_ENABLE_REMOTE)
 
 configure_file(
   ${CMAKE_CURRENT_SOURCE_DIR}/Features.inc.in

From 0f1494be43f0916516533fea9d99e9211bb4c581 Mon Sep 17 00:00:00 2001
From: Tim Northover <t.p.northover@gmail.com>
Date: Mon, 27 Jul 2020 13:29:49 +0100
Subject: [PATCH 0196/1035] AArch64: avoid UB shift of negative value

Left shifting a negative value is undefined behaviour, so this just moves the
negation afterwards to avoid it.
---
 llvm/lib/Target/AArch64/AArch64InstrInfo.cpp | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index 08f80c9aa361b..323ac76e903fd 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -6847,10 +6847,9 @@ Optional<RegImmPair> AArch64InstrInfo::isAddImmediate(const MachineInstr &MI,
     if (!MI.getOperand(0).isReg() || !MI.getOperand(1).isReg() ||
         !MI.getOperand(2).isImm())
       return None;
-    Offset = MI.getOperand(2).getImm() * Sign;
     int Shift = MI.getOperand(3).getImm();
     assert((Shift == 0 || Shift == 12) && "Shift can be either 0 or 12");
-    Offset = Offset << Shift;
+    Offset = Sign * (MI.getOperand(2).getImm() << Shift);
   }
   }
   return RegImmPair{MI.getOperand(1).getReg(), Offset};

From 005eee8b3ef7f244daf12f574a191079bfc1918b Mon Sep 17 00:00:00 2001
From: Mitch Phillips <31459023+hctim@users.noreply.github.com>
Date: Mon, 27 Jul 2020 05:46:43 -0700
Subject: [PATCH 0197/1035] [GWP-ASan] Fix uninitialized memory use in
 sigaction.

Fix up a small bug where we used a partially-uninitialized sigaction
struct in the optional signal handler. Shouldn't be a user-visible
change.
---
 compiler-rt/lib/gwp_asan/optional/segv_handler_posix.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/compiler-rt/lib/gwp_asan/optional/segv_handler_posix.cpp b/compiler-rt/lib/gwp_asan/optional/segv_handler_posix.cpp
index 1bd7a606c2136..9a80436efb311 100644
--- a/compiler-rt/lib/gwp_asan/optional/segv_handler_posix.cpp
+++ b/compiler-rt/lib/gwp_asan/optional/segv_handler_posix.cpp
@@ -144,7 +144,7 @@ void installSignalHandlers(gwp_asan::GuardedPoolAllocator *GPA, Printf_t Printf,
   PrintBacktraceForSignalHandler = PrintBacktrace;
   BacktraceForSignalHandler = SegvBacktrace;
 
-  struct sigaction Action;
+  struct sigaction Action = {};
   Action.sa_sigaction = sigSegvHandler;
   Action.sa_flags = SA_SIGINFO;
   sigaction(SIGSEGV, &Action, &PreviousHandler);

From bec77ece14890d2aa40c76eedc6a7a406d84f1fc Mon Sep 17 00:00:00 2001
From: Sergey Dmitriev <serguei.n.dmitriev@intel.com>
Date: Mon, 27 Jul 2020 06:02:06 -0700
Subject: [PATCH 0198/1035] [CallGraph] Preserve call records vector when
 replacing call edge

Summary:
Try not to resize vector of call records in a call graph node when
replacing call edge. That would prevent invalidation of iterators
stored in the CG SCC pass manager's scc_iterator.

Reviewers: jdoerfert

Reviewed By: jdoerfert

Subscribers: hiraditya, llvm-commits

Tags: #llvm

Differential Revision: https://reviews.llvm.org/D84295
---
 llvm/lib/Analysis/CallGraph.cpp             | 34 ++++++--
 llvm/unittests/IR/LegacyPassManagerTest.cpp | 86 +++++++++++++++++++++
 2 files changed, 115 insertions(+), 5 deletions(-)

diff --git a/llvm/lib/Analysis/CallGraph.cpp b/llvm/lib/Analysis/CallGraph.cpp
index 55adb454b7338..19c128b6633c0 100644
--- a/llvm/lib/Analysis/CallGraph.cpp
+++ b/llvm/lib/Analysis/CallGraph.cpp
@@ -281,13 +281,37 @@ void CallGraphNode::replaceCallEdge(CallBase &Call, CallBase &NewCall,
       I->second = NewNode;
       NewNode->AddRef();
 
-      // Refresh callback references.
-      forEachCallbackFunction(Call, [=](Function *CB) {
-        removeOneAbstractEdgeTo(CG->getOrInsertFunction(CB));
+      // Refresh callback references. Do not resize CalledFunctions if the
+      // number of callbacks is the same for new and old call sites.
+      SmallVector<CallGraphNode *, 4u> OldCBs;
+      SmallVector<CallGraphNode *, 4u> NewCBs;
+      forEachCallbackFunction(Call, [this, &OldCBs](Function *CB) {
+        OldCBs.push_back(CG->getOrInsertFunction(CB));
       });
-      forEachCallbackFunction(NewCall, [=](Function *CB) {
-        addCalledFunction(nullptr, CG->getOrInsertFunction(CB));
+      forEachCallbackFunction(NewCall, [this, &NewCBs](Function *CB) {
+        NewCBs.push_back(CG->getOrInsertFunction(CB));
       });
+      if (OldCBs.size() == NewCBs.size()) {
+        for (unsigned N = 0; N < OldCBs.size(); ++N) {
+          CallGraphNode *OldNode = OldCBs[N];
+          CallGraphNode *NewNode = NewCBs[N];
+          for (auto J = CalledFunctions.begin();; ++J) {
+            assert(J != CalledFunctions.end() &&
+                   "Cannot find callsite to update!");
+            if (!J->first && J->second == OldNode) {
+              J->second = NewNode;
+              OldNode->DropRef();
+              NewNode->AddRef();
+              break;
+            }
+          }
+        }
+      } else {
+        for (auto *CGN : OldCBs)
+          removeOneAbstractEdgeTo(CGN);
+        for (auto *CGN : NewCBs)
+          addCalledFunction(nullptr, CGN);
+      }
       return;
     }
   }
diff --git a/llvm/unittests/IR/LegacyPassManagerTest.cpp b/llvm/unittests/IR/LegacyPassManagerTest.cpp
index 72ac4be229974..f461bcc8c7761 100644
--- a/llvm/unittests/IR/LegacyPassManagerTest.cpp
+++ b/llvm/unittests/IR/LegacyPassManagerTest.cpp
@@ -16,6 +16,8 @@
 #include "llvm/Analysis/CallGraphSCCPass.h"
 #include "llvm/Analysis/LoopInfo.h"
 #include "llvm/Analysis/LoopPass.h"
+#include "llvm/AsmParser/Parser.h"
+#include "llvm/IR/AbstractCallSite.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/CallingConv.h"
 #include "llvm/IR/DataLayout.h"
@@ -28,6 +30,7 @@
 #include "llvm/IR/OptBisect.h"
 #include "llvm/InitializePasses.h"
 #include "llvm/Support/MathExtras.h"
+#include "llvm/Support/SourceMgr.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Transforms/Utils/CallGraphUpdater.h"
 #include "gtest/gtest.h"
@@ -694,6 +697,89 @@ namespace llvm {
       ASSERT_EQ(P->NumExtCalledBefore, /* test1, 2a, 2b, 3, 4 */ 5U);
       ASSERT_EQ(P->NumExtCalledAfter, /* test1, 3repl, 4 */ 3U);
     }
+
+    // Test for call graph SCC pass that replaces all callback call instructions
+    // with clones and updates CallGraph by calling CallGraph::replaceCallEdge()
+    // method. Test is expected to complete successfully after running pass on
+    // all SCCs in the test module.
+    struct CallbackCallsModifierPass : public CGPass {
+      bool runOnSCC(CallGraphSCC &SCC) override {
+        CGPass::run();
+
+        CallGraph &CG = const_cast<CallGraph &>(SCC.getCallGraph());
+
+        bool Changed = false;
+        for (CallGraphNode *CGN : SCC) {
+          Function *F = CGN->getFunction();
+          if (!F || F->isDeclaration())
+            continue;
+
+          SmallVector<CallBase *, 4u> Calls;
+          for (Use &U : F->uses()) {
+            AbstractCallSite ACS(&U);
+            if (!ACS || !ACS.isCallbackCall() || !ACS.isCallee(&U))
+              continue;
+            Calls.push_back(cast<CallBase>(ACS.getInstruction()));
+          }
+          if (Calls.empty())
+            continue;
+
+          for (CallBase *OldCB : Calls) {
+            CallGraphNode *CallerCGN = CG[OldCB->getParent()->getParent()];
+            assert(any_of(*CallerCGN,
+                          [CGN](const CallGraphNode::CallRecord &CallRecord) {
+                            return CallRecord.second == CGN;
+                          }) &&
+                   "function is not a callee");
+
+            CallBase *NewCB = cast<CallBase>(OldCB->clone());
+
+            NewCB->insertBefore(OldCB);
+            NewCB->takeName(OldCB);
+
+            CallerCGN->replaceCallEdge(*OldCB, *NewCB, CG[F]);
+
+            OldCB->replaceAllUsesWith(NewCB);
+            OldCB->eraseFromParent();
+          }
+          Changed = true;
+        }
+        return Changed;
+      }
+    };
+
+    TEST(PassManager, CallbackCallsModifier0) {
+      LLVMContext Context;
+
+      const char *IR = "define void @foo() {\n"
+                       "  call void @broker(void (i8*)* @callback0, i8* null)\n"
+                       "  call void @broker(void (i8*)* @callback1, i8* null)\n"
+                       "  ret void\n"
+                       "}\n"
+                       "\n"
+                       "declare !callback !0 void @broker(void (i8*)*, i8*)\n"
+                       "\n"
+                       "define internal void @callback0(i8* %arg) {\n"
+                       "  ret void\n"
+                       "}\n"
+                       "\n"
+                       "define internal void @callback1(i8* %arg) {\n"
+                       "  ret void\n"
+                       "}\n"
+                       "\n"
+                       "!0 = !{!1}\n"
+                       "!1 = !{i64 0, i64 1, i1 false}";
+
+      SMDiagnostic Err;
+      std::unique_ptr<Module> M = parseAssemblyString(IR, Err, Context);
+      if (!M)
+        Err.print("LegacyPassManagerTest", errs());
+
+      CallbackCallsModifierPass *P = new CallbackCallsModifierPass();
+      legacy::PassManager Passes;
+      Passes.add(P);
+      Passes.run(*M);
+    }
   }
 }
 

From 2dd7a9cc2d0572c3d1e5b9ce554a0800079863c9 Mon Sep 17 00:00:00 2001
From: Christian Sigg <csigg@google.com>
Date: Sat, 25 Jul 2020 14:56:35 +0200
Subject: [PATCH 0199/1035] [MLIR] NFC: Rename mcuMemHostRegister* to
 mgpuMemHostRegister* to make it consistent with the other cuda-runner
 functions and ROCm.

Summary: Rename mcuMemHostRegister* to mgpuMemHostRegister*.

Reviewers: herhut

Reviewed By: herhut

Subscribers: yaxunl, mehdi_amini, rriddle, jpienaar, shauheen, antiagainst, nicolasvasilache, arpith-jacob, mgester, lucyrfox, aartbik, liufengdb, stephenneuendorffer, Joonsoo, grosul1, Kayjukh, jurahul, msifontes

Tags: #mlir

Differential Revision: https://reviews.llvm.org/D84583
---
 mlir/test/mlir-cuda-runner/all-reduce-and.mlir        |  6 +++---
 mlir/test/mlir-cuda-runner/all-reduce-max.mlir        |  6 +++---
 mlir/test/mlir-cuda-runner/all-reduce-min.mlir        |  6 +++---
 mlir/test/mlir-cuda-runner/all-reduce-op.mlir         |  4 ++--
 mlir/test/mlir-cuda-runner/all-reduce-or.mlir         |  6 +++---
 mlir/test/mlir-cuda-runner/all-reduce-region.mlir     |  4 ++--
 mlir/test/mlir-cuda-runner/all-reduce-xor.mlir        |  6 +++---
 mlir/test/mlir-cuda-runner/gpu-to-cubin.mlir          |  4 ++--
 mlir/test/mlir-cuda-runner/multiple-all-reduce.mlir   |  8 ++++----
 mlir/test/mlir-cuda-runner/shuffle.mlir               |  4 ++--
 mlir/test/mlir-cuda-runner/two-modules.mlir           |  4 ++--
 mlir/tools/mlir-cuda-runner/cuda-runtime-wrappers.cpp | 10 +++++-----
 12 files changed, 34 insertions(+), 34 deletions(-)

diff --git a/mlir/test/mlir-cuda-runner/all-reduce-and.mlir b/mlir/test/mlir-cuda-runner/all-reduce-and.mlir
index d3ad7a802537b..f89f914157248 100644
--- a/mlir/test/mlir-cuda-runner/all-reduce-and.mlir
+++ b/mlir/test/mlir-cuda-runner/all-reduce-and.mlir
@@ -25,9 +25,9 @@ func @main() {
   %c6 = constant 6 : index
 
   %cast_data = memref_cast %data : memref<2x6xi32> to memref<*xi32>
-  call @mcuMemHostRegisterInt32(%cast_data) : (memref<*xi32>) -> ()
+  call @mgpuMemHostRegisterInt32(%cast_data) : (memref<*xi32>) -> ()
   %cast_sum = memref_cast %sum : memref<2xi32> to memref<*xi32>
-  call @mcuMemHostRegisterInt32(%cast_sum) : (memref<*xi32>) -> ()
+  call @mgpuMemHostRegisterInt32(%cast_sum) : (memref<*xi32>) -> ()
 
   store %cst0, %data[%c0, %c0] : memref<2x6xi32>
   store %cst1, %data[%c0, %c1] : memref<2x6xi32>
@@ -58,6 +58,6 @@ func @main() {
   return
 }
 
-func @mcuMemHostRegisterInt32(%ptr : memref<*xi32>)
+func @mgpuMemHostRegisterInt32(%ptr : memref<*xi32>)
 func @print_memref_i32(memref<*xi32>)
 
diff --git a/mlir/test/mlir-cuda-runner/all-reduce-max.mlir b/mlir/test/mlir-cuda-runner/all-reduce-max.mlir
index ae2f6c3d6b3e5..4adf8a73d924c 100644
--- a/mlir/test/mlir-cuda-runner/all-reduce-max.mlir
+++ b/mlir/test/mlir-cuda-runner/all-reduce-max.mlir
@@ -25,9 +25,9 @@ func @main() {
   %c6 = constant 6 : index
 
   %cast_data = memref_cast %data : memref<2x6xi32> to memref<*xi32>
-  call @mcuMemHostRegisterInt32(%cast_data) : (memref<*xi32>) -> ()
+  call @mgpuMemHostRegisterInt32(%cast_data) : (memref<*xi32>) -> ()
   %cast_sum = memref_cast %sum : memref<2xi32> to memref<*xi32>
-  call @mcuMemHostRegisterInt32(%cast_sum) : (memref<*xi32>) -> ()
+  call @mgpuMemHostRegisterInt32(%cast_sum) : (memref<*xi32>) -> ()
 
   store %cst0, %data[%c0, %c0] : memref<2x6xi32>
   store %cst1, %data[%c0, %c1] : memref<2x6xi32>
@@ -58,6 +58,6 @@ func @main() {
   return
 }
 
-func @mcuMemHostRegisterInt32(%ptr : memref<*xi32>)
+func @mgpuMemHostRegisterInt32(%ptr : memref<*xi32>)
 func @print_memref_i32(memref<*xi32>)
 
diff --git a/mlir/test/mlir-cuda-runner/all-reduce-min.mlir b/mlir/test/mlir-cuda-runner/all-reduce-min.mlir
index 0cd4f11daf105..8cb3116e9d0dd 100644
--- a/mlir/test/mlir-cuda-runner/all-reduce-min.mlir
+++ b/mlir/test/mlir-cuda-runner/all-reduce-min.mlir
@@ -25,9 +25,9 @@ func @main() {
   %c6 = constant 6 : index
 
   %cast_data = memref_cast %data : memref<2x6xi32> to memref<*xi32>
-  call @mcuMemHostRegisterInt32(%cast_data) : (memref<*xi32>) -> ()
+  call @mgpuMemHostRegisterInt32(%cast_data) : (memref<*xi32>) -> ()
   %cast_sum = memref_cast %sum : memref<2xi32> to memref<*xi32>
-  call @mcuMemHostRegisterInt32(%cast_sum) : (memref<*xi32>) -> ()
+  call @mgpuMemHostRegisterInt32(%cast_sum) : (memref<*xi32>) -> ()
 
   store %cst0, %data[%c0, %c0] : memref<2x6xi32>
   store %cst1, %data[%c0, %c1] : memref<2x6xi32>
@@ -58,6 +58,6 @@ func @main() {
   return
 }
 
-func @mcuMemHostRegisterInt32(%ptr : memref<*xi32>)
+func @mgpuMemHostRegisterInt32(%ptr : memref<*xi32>)
 func @print_memref_i32(memref<*xi32>)
 
diff --git a/mlir/test/mlir-cuda-runner/all-reduce-op.mlir b/mlir/test/mlir-cuda-runner/all-reduce-op.mlir
index 67c4f96d36f47..72306674c3ff9 100644
--- a/mlir/test/mlir-cuda-runner/all-reduce-op.mlir
+++ b/mlir/test/mlir-cuda-runner/all-reduce-op.mlir
@@ -11,7 +11,7 @@ func @main() {
   %sy = dim %dst, %c1 : memref<?x?x?xf32>
   %sz = dim %dst, %c0 : memref<?x?x?xf32>
   %cast_dst = memref_cast %dst : memref<?x?x?xf32> to memref<*xf32>
-  call @mcuMemHostRegisterFloat(%cast_dst) : (memref<*xf32>) -> ()
+  call @mgpuMemHostRegisterFloat(%cast_dst) : (memref<*xf32>) -> ()
   gpu.launch blocks(%bx, %by, %bz) in (%grid_x = %c1, %grid_y = %c1, %grid_z = %c1)
              threads(%tx, %ty, %tz) in (%block_x = %sx, %block_y = %sy, %block_z = %sz) {
     %t0 = muli %tz, %block_y : index
@@ -28,5 +28,5 @@ func @main() {
   return
 }
 
-func @mcuMemHostRegisterFloat(%ptr : memref<*xf32>)
+func @mgpuMemHostRegisterFloat(%ptr : memref<*xf32>)
 func @print_memref_f32(%ptr : memref<*xf32>)
diff --git a/mlir/test/mlir-cuda-runner/all-reduce-or.mlir b/mlir/test/mlir-cuda-runner/all-reduce-or.mlir
index cc9eae9e8b660..7d0ed929322e8 100644
--- a/mlir/test/mlir-cuda-runner/all-reduce-or.mlir
+++ b/mlir/test/mlir-cuda-runner/all-reduce-or.mlir
@@ -25,9 +25,9 @@ func @main() {
   %c6 = constant 6 : index
 
   %cast_data = memref_cast %data : memref<2x6xi32> to memref<*xi32>
-  call @mcuMemHostRegisterInt32(%cast_data) : (memref<*xi32>) -> ()
+  call @mgpuMemHostRegisterInt32(%cast_data) : (memref<*xi32>) -> ()
   %cast_sum = memref_cast %sum : memref<2xi32> to memref<*xi32>
-  call @mcuMemHostRegisterInt32(%cast_sum) : (memref<*xi32>) -> ()
+  call @mgpuMemHostRegisterInt32(%cast_sum) : (memref<*xi32>) -> ()
 
   store %cst0, %data[%c0, %c0] : memref<2x6xi32>
   store %cst1, %data[%c0, %c1] : memref<2x6xi32>
@@ -58,6 +58,6 @@ func @main() {
   return
 }
 
-func @mcuMemHostRegisterInt32(%ptr : memref<*xi32>)
+func @mgpuMemHostRegisterInt32(%ptr : memref<*xi32>)
 func @print_memref_i32(memref<*xi32>)
 
diff --git a/mlir/test/mlir-cuda-runner/all-reduce-region.mlir b/mlir/test/mlir-cuda-runner/all-reduce-region.mlir
index afd3d7cb038a6..a9426c6589787 100644
--- a/mlir/test/mlir-cuda-runner/all-reduce-region.mlir
+++ b/mlir/test/mlir-cuda-runner/all-reduce-region.mlir
@@ -8,7 +8,7 @@ func @main() {
   %c0 = constant 0 : index
   %sx = dim %dst, %c0 : memref<?xf32>
   %cast_dst = memref_cast %dst : memref<?xf32> to memref<*xf32>
-  call @mcuMemHostRegisterFloat(%cast_dst) : (memref<*xf32>) -> ()
+  call @mgpuMemHostRegisterFloat(%cast_dst) : (memref<*xf32>) -> ()
   gpu.launch blocks(%bx, %by, %bz) in (%grid_x = %one, %grid_y = %one, %grid_z = %one)
              threads(%tx, %ty, %tz) in (%block_x = %sx, %block_y = %one, %block_z = %one) {
     %val = index_cast %tx : index to i32
@@ -25,5 +25,5 @@ func @main() {
   return
 }
 
-func @mcuMemHostRegisterFloat(%ptr : memref<*xf32>)
+func @mgpuMemHostRegisterFloat(%ptr : memref<*xf32>)
 func @print_memref_f32(memref<*xf32>)
diff --git a/mlir/test/mlir-cuda-runner/all-reduce-xor.mlir b/mlir/test/mlir-cuda-runner/all-reduce-xor.mlir
index a32c4d3eb93e5..67461783b2570 100644
--- a/mlir/test/mlir-cuda-runner/all-reduce-xor.mlir
+++ b/mlir/test/mlir-cuda-runner/all-reduce-xor.mlir
@@ -25,9 +25,9 @@ func @main() {
   %c6 = constant 6 : index
 
   %cast_data = memref_cast %data : memref<2x6xi32> to memref<*xi32>
-  call @mcuMemHostRegisterInt32(%cast_data) : (memref<*xi32>) -> ()
+  call @mgpuMemHostRegisterInt32(%cast_data) : (memref<*xi32>) -> ()
   %cast_sum = memref_cast %sum : memref<2xi32> to memref<*xi32>
-  call @mcuMemHostRegisterInt32(%cast_sum) : (memref<*xi32>) -> ()
+  call @mgpuMemHostRegisterInt32(%cast_sum) : (memref<*xi32>) -> ()
 
   store %cst0, %data[%c0, %c0] : memref<2x6xi32>
   store %cst1, %data[%c0, %c1] : memref<2x6xi32>
@@ -58,6 +58,6 @@ func @main() {
   return
 }
 
-func @mcuMemHostRegisterInt32(%ptr : memref<*xi32>)
+func @mgpuMemHostRegisterInt32(%ptr : memref<*xi32>)
 func @print_memref_i32(memref<*xi32>)
 
diff --git a/mlir/test/mlir-cuda-runner/gpu-to-cubin.mlir b/mlir/test/mlir-cuda-runner/gpu-to-cubin.mlir
index 0ef33ea6112a2..80339c36fb384 100644
--- a/mlir/test/mlir-cuda-runner/gpu-to-cubin.mlir
+++ b/mlir/test/mlir-cuda-runner/gpu-to-cubin.mlir
@@ -18,7 +18,7 @@ func @main() {
   %21 = constant 5 : i32
   %22 = memref_cast %arg0 : memref<5xf32> to memref<?xf32>
   %23 = memref_cast %22 : memref<?xf32> to memref<*xf32>
-  call @mcuMemHostRegisterFloat(%23) : (memref<*xf32>) -> ()
+  call @mgpuMemHostRegisterFloat(%23) : (memref<*xf32>) -> ()
   call @print_memref_f32(%23) : (memref<*xf32>) -> ()
   %24 = constant 1.0 : f32
   call @other_func(%24, %22) : (f32, memref<?xf32>) -> ()
@@ -26,5 +26,5 @@ func @main() {
   return
 }
 
-func @mcuMemHostRegisterFloat(%ptr : memref<*xf32>)
+func @mgpuMemHostRegisterFloat(%ptr : memref<*xf32>)
 func @print_memref_f32(%ptr : memref<*xf32>)
diff --git a/mlir/test/mlir-cuda-runner/multiple-all-reduce.mlir b/mlir/test/mlir-cuda-runner/multiple-all-reduce.mlir
index a7b143f760a75..b88d8e1b8ba1a 100644
--- a/mlir/test/mlir-cuda-runner/multiple-all-reduce.mlir
+++ b/mlir/test/mlir-cuda-runner/multiple-all-reduce.mlir
@@ -26,11 +26,11 @@ func @main() {
   %c6 = constant 6 : index
 
   %cast_data = memref_cast %data : memref<2x6xf32> to memref<*xf32>
-  call @mcuMemHostRegisterFloat(%cast_data) : (memref<*xf32>) -> ()
+  call @mgpuMemHostRegisterFloat(%cast_data) : (memref<*xf32>) -> ()
   %cast_sum = memref_cast %sum : memref<2xf32> to memref<*xf32>
-  call @mcuMemHostRegisterFloat(%cast_sum) : (memref<*xf32>) -> ()
+  call @mgpuMemHostRegisterFloat(%cast_sum) : (memref<*xf32>) -> ()
   %cast_mul = memref_cast %mul : memref<2xf32> to memref<*xf32>
-  call @mcuMemHostRegisterFloat(%cast_mul) : (memref<*xf32>) -> ()
+  call @mgpuMemHostRegisterFloat(%cast_mul) : (memref<*xf32>) -> ()
 
   store %cst0, %data[%c0, %c0] : memref<2x6xf32>
   store %cst1, %data[%c0, %c1] : memref<2x6xf32>
@@ -66,5 +66,5 @@ func @main() {
   return
 }
 
-func @mcuMemHostRegisterFloat(%ptr : memref<*xf32>)
+func @mgpuMemHostRegisterFloat(%ptr : memref<*xf32>)
 func @print_memref_f32(memref<*xf32>)
diff --git a/mlir/test/mlir-cuda-runner/shuffle.mlir b/mlir/test/mlir-cuda-runner/shuffle.mlir
index 0f8cdca3a8eb3..a4563cc0c381d 100644
--- a/mlir/test/mlir-cuda-runner/shuffle.mlir
+++ b/mlir/test/mlir-cuda-runner/shuffle.mlir
@@ -8,7 +8,7 @@ func @main() {
   %c0 = constant 0 : index
   %sx = dim %dst, %c0 : memref<?xf32>
   %cast_dest = memref_cast %dst : memref<?xf32> to memref<*xf32>
-  call @mcuMemHostRegisterFloat(%cast_dest) : (memref<*xf32>) -> ()
+  call @mgpuMemHostRegisterFloat(%cast_dest) : (memref<*xf32>) -> ()
   gpu.launch blocks(%bx, %by, %bz) in (%grid_x = %one, %grid_y = %one, %grid_z = %one)
              threads(%tx, %ty, %tz) in (%block_x = %sx, %block_y = %one, %block_z = %one) {
     %t0 = index_cast %tx : index to i32
@@ -28,5 +28,5 @@ func @main() {
   return
 }
 
-func @mcuMemHostRegisterFloat(%ptr : memref<*xf32>)
+func @mgpuMemHostRegisterFloat(%ptr : memref<*xf32>)
 func @print_memref_f32(%ptr : memref<*xf32>)
diff --git a/mlir/test/mlir-cuda-runner/two-modules.mlir b/mlir/test/mlir-cuda-runner/two-modules.mlir
index 3229879d2fb50..ef4dd0c48b8df 100644
--- a/mlir/test/mlir-cuda-runner/two-modules.mlir
+++ b/mlir/test/mlir-cuda-runner/two-modules.mlir
@@ -8,7 +8,7 @@ func @main() {
   %c0 = constant 0 : index
   %sx = dim %dst, %c0 : memref<?xi32>
   %cast_dst = memref_cast %dst : memref<?xi32> to memref<*xi32>
-  call @mcuMemHostRegisterInt32(%cast_dst) : (memref<*xi32>) -> ()
+  call @mgpuMemHostRegisterInt32(%cast_dst) : (memref<*xi32>) -> ()
   gpu.launch blocks(%bx, %by, %bz) in (%grid_x = %one, %grid_y = %one, %grid_z = %one)
              threads(%tx, %ty, %tz) in (%block_x = %sx, %block_y = %one, %block_z = %one) {
     %t0 = index_cast %tx : index to i32
@@ -25,5 +25,5 @@ func @main() {
   return
 }
 
-func @mcuMemHostRegisterInt32(%memref : memref<*xi32>)
+func @mgpuMemHostRegisterInt32(%memref : memref<*xi32>)
 func @print_memref_i32(%memref : memref<*xi32>)
diff --git a/mlir/tools/mlir-cuda-runner/cuda-runtime-wrappers.cpp b/mlir/tools/mlir-cuda-runner/cuda-runtime-wrappers.cpp
index 705fa9f00930a..2b71eb34703bd 100644
--- a/mlir/tools/mlir-cuda-runner/cuda-runtime-wrappers.cpp
+++ b/mlir/tools/mlir-cuda-runner/cuda-runtime-wrappers.cpp
@@ -83,7 +83,7 @@ extern "C" void mgpuMemHostRegister(void *ptr, uint64_t sizeBytes) {
 // Allows to register a MemRef with the CUDA runtime. Initializes array with
 // value. Helpful until we have transfer functions implemented.
 template <typename T>
-void mcuMemHostRegisterMemRef(const DynamicMemRefType<T> &mem_ref, T value) {
+void mgpuMemHostRegisterMemRef(const DynamicMemRefType<T> &mem_ref, T value) {
   llvm::SmallVector<int64_t, 4> denseStrides(mem_ref.rank);
   llvm::ArrayRef<int64_t> sizes(mem_ref.sizes, mem_ref.rank);
   llvm::ArrayRef<int64_t> strides(mem_ref.strides, mem_ref.rank);
@@ -103,12 +103,12 @@ void mcuMemHostRegisterMemRef(const DynamicMemRefType<T> &mem_ref, T value) {
   mgpuMemHostRegister(pointer, count * sizeof(T));
 }
 
-extern "C" void mcuMemHostRegisterFloat(int64_t rank, void *ptr) {
+extern "C" void mgpuMemHostRegisterFloat(int64_t rank, void *ptr) {
   UnrankedMemRefType<float> mem_ref = {rank, ptr};
-  mcuMemHostRegisterMemRef(DynamicMemRefType<float>(mem_ref), 1.23f);
+  mgpuMemHostRegisterMemRef(DynamicMemRefType<float>(mem_ref), 1.23f);
 }
 
-extern "C" void mcuMemHostRegisterInt32(int64_t rank, void *ptr) {
+extern "C" void mgpuMemHostRegisterInt32(int64_t rank, void *ptr) {
   UnrankedMemRefType<int32_t> mem_ref = {rank, ptr};
-  mcuMemHostRegisterMemRef(DynamicMemRefType<int32_t>(mem_ref), 123);
+  mgpuMemHostRegisterMemRef(DynamicMemRefType<int32_t>(mem_ref), 123);
 }

From 5b5b3ce0ad6d308471ed09c644eee4b5c337cd1b Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Sat, 25 Jul 2020 15:52:05 +0100
Subject: [PATCH 0200/1035] IRPrintingPasses.h - simplify unnecessary header
 with forward declarations. NFC.

Remove duplicate PassManager.h include in IRPrintingPasses.cpp
---
 llvm/include/llvm/IR/IRPrintingPasses.h | 3 ++-
 llvm/lib/IR/IRPrintingPasses.cpp        | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/llvm/include/llvm/IR/IRPrintingPasses.h b/llvm/include/llvm/IR/IRPrintingPasses.h
index 3a1c489ee09f2..ed7082a31a423 100644
--- a/llvm/include/llvm/IR/IRPrintingPasses.h
+++ b/llvm/include/llvm/IR/IRPrintingPasses.h
@@ -18,11 +18,12 @@
 #ifndef LLVM_IR_IRPRINTINGPASSES_H
 #define LLVM_IR_IRPRINTINGPASSES_H
 
-#include "llvm/ADT/StringRef.h"
 #include "llvm/IR/PassManager.h"
 #include <string>
 
 namespace llvm {
+class raw_ostream;
+class StringRef;
 
 /// Create and return a pass that writes the module to the specified
 /// \c raw_ostream.
diff --git a/llvm/lib/IR/IRPrintingPasses.cpp b/llvm/lib/IR/IRPrintingPasses.cpp
index 03657ff8d9d43..7c73d2ab98711 100644
--- a/llvm/lib/IR/IRPrintingPasses.cpp
+++ b/llvm/lib/IR/IRPrintingPasses.cpp
@@ -11,13 +11,14 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/IR/IRPrintingPasses.h"
+#include "llvm/ADT/StringRef.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/Module.h"
-#include "llvm/IR/PassManager.h"
 #include "llvm/InitializePasses.h"
 #include "llvm/Pass.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
+
 using namespace llvm;
 
 PrintModulePass::PrintModulePass() : OS(dbgs()) {}

From f720c9c68c70003ba56d17fff1549f30f509778f Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Mon, 27 Jul 2020 13:14:13 +0100
Subject: [PATCH 0201/1035] [X86] combineExtractSubvector - pull out repeated
 getSizeInBits() calls. NFCI.

---
 llvm/lib/Target/X86/X86ISelLowering.cpp | 24 +++++++++++++-----------
 1 file changed, 13 insertions(+), 11 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 7c134a8c7cb92..a807c5117631b 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -48278,12 +48278,14 @@ static SDValue combineExtractSubvector(SDNode *N, SelectionDAG &DAG,
   unsigned IdxVal = N->getConstantOperandVal(1);
   SDValue InVecBC = peekThroughBitcasts(InVec);
   EVT InVecVT = InVec.getValueType();
+  unsigned SizeInBits = VT.getSizeInBits();
+  unsigned InSizeInBits = InVecVT.getSizeInBits();
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
 
   if (Subtarget.hasAVX() && !Subtarget.hasAVX2() &&
       TLI.isTypeLegal(InVecVT) &&
-      InVecVT.getSizeInBits() == 256 && InVecBC.getOpcode() == ISD::AND) {
-    auto isConcatenatedNot = [] (SDValue V) {
+      InSizeInBits == 256 && InVecBC.getOpcode() == ISD::AND) {
+    auto isConcatenatedNot = [](SDValue V) {
       V = peekThroughBitcasts(V);
       if (!isBitwiseNot(V))
         return false;
@@ -48326,7 +48328,7 @@ static SDValue combineExtractSubvector(SDNode *N, SelectionDAG &DAG,
       InVec.getOpcode() == ISD::INSERT_SUBVECTOR && IdxVal == 0 &&
       InVec.hasOneUse() && isNullConstant(InVec.getOperand(2)) &&
       ISD::isBuildVectorAllZeros(InVec.getOperand(0).getNode()) &&
-      InVec.getOperand(1).getValueSizeInBits() <= VT.getSizeInBits()) {
+      InVec.getOperand(1).getValueSizeInBits() <= SizeInBits) {
     SDLoc DL(N);
     return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,
                        getZeroVector(VT, Subtarget, DAG, DL),
@@ -48337,14 +48339,14 @@ static SDValue combineExtractSubvector(SDNode *N, SelectionDAG &DAG,
   // broadcasting to the smaller type directly, assuming this is the only use.
   // As its a broadcast we don't care about the extraction index.
   if (InVec.getOpcode() == X86ISD::VBROADCAST && InVec.hasOneUse() &&
-      InVec.getOperand(0).getValueSizeInBits() <= VT.getSizeInBits())
+      InVec.getOperand(0).getValueSizeInBits() <= SizeInBits)
     return DAG.getNode(X86ISD::VBROADCAST, SDLoc(N), VT, InVec.getOperand(0));
 
   if (InVec.getOpcode() == X86ISD::VBROADCAST_LOAD && InVec.hasOneUse()) {
     auto *MemIntr = cast<MemIntrinsicSDNode>(InVec);
-    if (MemIntr->getMemoryVT().getSizeInBits() <= VT.getSizeInBits()) {
+    if (MemIntr->getMemoryVT().getSizeInBits() <= SizeInBits) {
       SDVTList Tys = DAG.getVTList(VT, MVT::Other);
-      SDValue Ops[] = { MemIntr->getChain(), MemIntr->getBasePtr() };
+      SDValue Ops[] = {MemIntr->getChain(), MemIntr->getBasePtr()};
       SDValue BcastLd =
           DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, SDLoc(N), Tys, Ops,
                                   MemIntr->getMemoryVT(),
@@ -48359,7 +48361,7 @@ static SDValue combineExtractSubvector(SDNode *N, SelectionDAG &DAG,
   // SimplifyDemandedVectorElts do more simplifications.
   if (IdxVal != 0 && (InVec.getOpcode() == X86ISD::VBROADCAST ||
                       InVec.getOpcode() == X86ISD::VBROADCAST_LOAD))
-    return extractSubVector(InVec, 0, DAG, SDLoc(N), VT.getSizeInBits());
+    return extractSubVector(InVec, 0, DAG, SDLoc(N), SizeInBits);
 
   // If we're extracting a broadcasted subvector, just use the source.
   if (InVec.getOpcode() == X86ISD::SUBV_BROADCAST &&
@@ -48367,12 +48369,12 @@ static SDValue combineExtractSubvector(SDNode *N, SelectionDAG &DAG,
     return InVec.getOperand(0);
 
   // Attempt to extract from the source of a shuffle vector.
-  if ((InVecVT.getSizeInBits() % VT.getSizeInBits()) == 0 &&
+  if ((InSizeInBits % SizeInBits) == 0 &&
       (IdxVal % VT.getVectorNumElements()) == 0) {
     SmallVector<int, 32> ShuffleMask;
     SmallVector<int, 32> ScaledMask;
     SmallVector<SDValue, 2> ShuffleInputs;
-    unsigned NumSubVecs = InVecVT.getSizeInBits() / VT.getSizeInBits();
+    unsigned NumSubVecs = InSizeInBits / SizeInBits;
     // Decode the shuffle mask and scale it so its shuffling subvectors.
     if (getTargetShuffleInputs(InVecBC, ShuffleInputs, ShuffleMask, DAG) &&
         scaleShuffleElements(ShuffleMask, NumSubVecs, ScaledMask)) {
@@ -48382,11 +48384,11 @@ static SDValue combineExtractSubvector(SDNode *N, SelectionDAG &DAG,
       if (ScaledMask[SubVecIdx] == SM_SentinelZero)
         return getZeroVector(VT, Subtarget, DAG, SDLoc(N));
       SDValue Src = ShuffleInputs[ScaledMask[SubVecIdx] / NumSubVecs];
-      if (Src.getValueSizeInBits() == InVecVT.getSizeInBits()) {
+      if (Src.getValueSizeInBits() == InSizeInBits) {
         unsigned SrcSubVecIdx = ScaledMask[SubVecIdx] % NumSubVecs;
         unsigned SrcEltIdx = SrcSubVecIdx * VT.getVectorNumElements();
         return extractSubVector(DAG.getBitcast(InVecVT, Src), SrcEltIdx, DAG,
-                                SDLoc(N), VT.getSizeInBits());
+                                SDLoc(N), SizeInBits);
       }
     }
   }

From ab4ffa52f0a62447c2e1be872adb0aa3e357f071 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Mon, 27 Jul 2020 14:32:32 +0100
Subject: [PATCH 0202/1035] [X86][AVX] Fold extract_subvector(truncate(x),0) ->
 truncate(extract_subvector(x),0)

This is currently only supported for VLX targets where the op should be legal.
---
 llvm/lib/Target/X86/X86ISelLowering.cpp    |  8 ++++++++
 llvm/test/CodeGen/X86/vector-reduce-mul.ll | 20 ++++++++++----------
 2 files changed, 18 insertions(+), 10 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index a807c5117631b..6abe6c6b83156 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -48435,6 +48435,14 @@ static SDValue combineExtractSubvector(SDNode *N, SelectionDAG &DAG,
       SDValue Ext2 = extractSubVector(InVec.getOperand(2), 0, DAG, DL, 128);
       return DAG.getNode(InOpcode, DL, VT, Ext0, Ext1, Ext2);
     }
+    if (InOpcode == ISD::TRUNCATE && Subtarget.hasVLX() &&
+        (VT.is128BitVector() || VT.is256BitVector())) {
+      SDLoc DL(N);
+      SDValue InVecSrc = InVec.getOperand(0);
+      unsigned Scale = InVecSrc.getValueSizeInBits() / InSizeInBits;
+      SDValue Ext = extractSubVector(InVecSrc, 0, DAG, DL, Scale * SizeInBits);
+      return DAG.getNode(InOpcode, DL, VT, Ext);
+    }
   }
 
   return SDValue();
diff --git a/llvm/test/CodeGen/X86/vector-reduce-mul.ll b/llvm/test/CodeGen/X86/vector-reduce-mul.ll
index f7e1a72f9a91f..6e9bfbe5a79fd 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-mul.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-mul.ll
@@ -2008,24 +2008,24 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX512BWVL-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
 ; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
-; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
-; AVX512BWVL-NEXT:    vpmovwb %zmm0, %ymm1
+; AVX512BWVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm1
 ; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
-; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
-; AVX512BWVL-NEXT:    vpmovwb %zmm0, %ymm1
+; AVX512BWVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm1
 ; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
 ; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
-; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
-; AVX512BWVL-NEXT:    vpmovwb %zmm0, %ymm1
+; AVX512BWVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm1
 ; AVX512BWVL-NEXT:    vpsrld $16, %xmm1, %xmm1
 ; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
-; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
-; AVX512BWVL-NEXT:    vpmovwb %zmm0, %ymm1
+; AVX512BWVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm1
 ; AVX512BWVL-NEXT:    vpsrlw $8, %xmm1, %xmm1
 ; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
-; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
-; AVX512BWVL-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512BWVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm0
 ; AVX512BWVL-NEXT:    vmovd %xmm0, %eax
 ; AVX512BWVL-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512BWVL-NEXT:    vzeroupper

From 1b4d24912a1f8730475d4b01e7da89dfae90ae9c Mon Sep 17 00:00:00 2001
From: Guillaume Chatelet <gchatelet@google.com>
Date: Mon, 27 Jul 2020 13:54:46 +0000
Subject: [PATCH 0203/1035] [NFC] Replace ".size() < 1" with ".empty()"

---
 llvm/lib/Bitcode/Reader/BitcodeReader.cpp | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index 908e70b2789da..f0377df8648a4 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -1716,7 +1716,7 @@ Error BitcodeReader::parseTypeTableBody() {
     case bitc::TYPE_CODE_NUMENTRY: // TYPE_CODE_NUMENTRY: [numentries]
       // TYPE_CODE_NUMENTRY contains a count of the number of types in the
       // type list.  This allows us to reserve space.
-      if (Record.size() < 1)
+      if (Record.empty())
         return error("Invalid record");
       TypeList.resize(Record[0]);
       continue;
@@ -1757,7 +1757,7 @@ Error BitcodeReader::parseTypeTableBody() {
       ResultTy = Type::getTokenTy(Context);
       break;
     case bitc::TYPE_CODE_INTEGER: { // INTEGER: [width]
-      if (Record.size() < 1)
+      if (Record.empty())
         return error("Invalid record");
 
       uint64_t NumBits = Record[0];
@@ -1769,7 +1769,7 @@ Error BitcodeReader::parseTypeTableBody() {
     }
     case bitc::TYPE_CODE_POINTER: { // POINTER: [pointee type] or
                                     //          [pointee type, address space]
-      if (Record.size() < 1)
+      if (Record.empty())
         return error("Invalid record");
       unsigned AddressSpace = 0;
       if (Record.size() == 2)
@@ -1824,7 +1824,7 @@ Error BitcodeReader::parseTypeTableBody() {
       break;
     }
     case bitc::TYPE_CODE_STRUCT_ANON: {  // STRUCT: [ispacked, eltty x N]
-      if (Record.size() < 1)
+      if (Record.empty())
         return error("Invalid record");
       SmallVector<Type*, 8> EltTys;
       for (unsigned i = 1, e = Record.size(); i != e; ++i) {
@@ -1844,7 +1844,7 @@ Error BitcodeReader::parseTypeTableBody() {
       continue;
 
     case bitc::TYPE_CODE_STRUCT_NAMED: { // STRUCT: [ispacked, eltty x N]
-      if (Record.size() < 1)
+      if (Record.empty())
         return error("Invalid record");
 
       if (NumRecords >= TypeList.size())
@@ -3716,7 +3716,7 @@ Error BitcodeReader::parseModule(uint64_t ResumeBit,
       break;
     /// MODULE_CODE_VSTOFFSET: [offset]
     case bitc::MODULE_CODE_VSTOFFSET:
-      if (Record.size() < 1)
+      if (Record.empty())
         return error("Invalid record");
       // Note that we subtract 1 here because the offset is relative to one word
       // before the start of the identification or module block, which was
@@ -3870,7 +3870,7 @@ Error BitcodeReader::parseFunctionBody(Function *F) {
     default: // Default behavior: reject
       return error("Invalid value");
     case bitc::FUNC_CODE_DECLAREBLOCKS: {   // DECLAREBLOCKS: [nblocks]
-      if (Record.size() < 1 || Record[0] == 0)
+      if (Record.empty() || Record[0] == 0)
         return error("Invalid record");
       // Create all the basic blocks for the function.
       FunctionBBs.resize(Record[0]);
@@ -4712,7 +4712,7 @@ Error BitcodeReader::parseFunctionBody(Function *F) {
       InstructionList.push_back(I);
       break;
     case bitc::FUNC_CODE_INST_PHI: { // PHI: [ty, val0,bb0, ...]
-      if (Record.size() < 1)
+      if (Record.empty())
         return error("Invalid record");
       // The first record specifies the type.
       FullTy = getFullyStructuredTypeByID(Record[0]);
@@ -5205,7 +5205,7 @@ Error BitcodeReader::parseFunctionBody(Function *F) {
       // number of operand bundle blocks.  These blocks are read into
       // OperandBundles and consumed at the next call or invoke instruction.
 
-      if (Record.size() < 1 || Record[0] >= BundleTags.size())
+      if (Record.empty() || Record[0] >= BundleTags.size())
         return error("Invalid record");
 
       std::vector<Value *> Inputs;
@@ -5738,7 +5738,7 @@ Error ModuleSummaryIndexBitcodeReader::parseModule() {
         }
         /// MODULE_CODE_VSTOFFSET: [offset]
         case bitc::MODULE_CODE_VSTOFFSET:
-          if (Record.size() < 1)
+          if (Record.empty())
             return error("Invalid record");
           // Note that we subtract 1 here because the offset is relative to one
           // word before the start of the identification or module block, which

From 92fa91bb402921a5705507c38f583e9b8e9d84e4 Mon Sep 17 00:00:00 2001
From: Anastasia Stulova <anastasia.stulova@arm.com>
Date: Mon, 27 Jul 2020 13:27:21 +0100
Subject: [PATCH 0204/1035] [OpenCL] Fixed missing address space for templated
 copy constructor.

Added missing address space for the parameter of copy ctor created
for templated constructor with an R-value reference.

Patch by Ole Strohm (olestrohm)!

Tags: #clang

Differential Revision: https://reviews.llvm.org/D83665
---
 clang/lib/Sema/SemaTemplateDeduction.cpp       |  5 ++++-
 .../SemaOpenCLCXX/address-space-templates.cl   | 18 ++++++++++++++++++
 2 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/clang/lib/Sema/SemaTemplateDeduction.cpp b/clang/lib/Sema/SemaTemplateDeduction.cpp
index 52062e9a5039c..8e7b4e1655ea9 100644
--- a/clang/lib/Sema/SemaTemplateDeduction.cpp
+++ b/clang/lib/Sema/SemaTemplateDeduction.cpp
@@ -3815,8 +3815,11 @@ static bool AdjustFunctionParmAndArgTypesForDeduction(
     //   If P is a forwarding reference and the argument is an lvalue, the type
     //   "lvalue reference to A" is used in place of A for type deduction.
     if (isForwardingReference(QualType(ParamRefType, 0), FirstInnerIndex) &&
-        Arg->isLValue())
+        Arg->isLValue()) {
+      if (S.getLangOpts().OpenCL)
+        ArgType = S.Context.getAddrSpaceQualType(ArgType, LangAS::opencl_generic);
       ArgType = S.Context.getLValueReferenceType(ArgType);
+    }
   } else {
     // C++ [temp.deduct.call]p2:
     //   If P is not a reference type:
diff --git a/clang/test/SemaOpenCLCXX/address-space-templates.cl b/clang/test/SemaOpenCLCXX/address-space-templates.cl
index 6b304d2fdda45..be187de5684b0 100644
--- a/clang/test/SemaOpenCLCXX/address-space-templates.cl
+++ b/clang/test/SemaOpenCLCXX/address-space-templates.cl
@@ -22,10 +22,28 @@ void foo3() {
   __private T ii; // expected-error{{conflicting address space qualifiers are provided between types '__private T' and '__global int'}}
 }
 
+template <class _Tp> struct remove_reference         { typedef _Tp type; };
+template <class _Tp> struct remove_reference<_Tp &>  { typedef _Tp type; };
+template <class _Tp> struct as_pointer {
+    typedef typename remove_reference<_Tp>::type* type;
+};
+
+struct rep {
+  // CHECK |-CXXConstructorDecl {{.*}} rep 'void (const __generic rep &__private) __generic'
+  template<class U, class = typename as_pointer<U>::type>
+  rep(U&& v) {}
+};
+
+struct rep_outer : private rep {
+  rep_outer()
+    : rep(0) {}
+};
+
 void bar() {
   S<const __global int> sintgl; // expected-note{{in instantiation of template class 'S<const __global int>' requested here}}
 
   foo1<__local int>(1); // expected-error{{no matching function for call to 'foo1'}}
   foo2<__global int>(0);
   foo3<__global int>(); // expected-note{{in instantiation of function template specialization 'foo3<__global int>' requested here}}
+  rep_outer r;
 }

From a7044edde71be0a4c70e79c503c39ce6bdd930e5 Mon Sep 17 00:00:00 2001
From: Luofan Chen <clfbbn@gmail.com>
Date: Mon, 27 Jul 2020 22:28:39 +0800
Subject: [PATCH 0205/1035] [Attributor] Fix qualifier warning of the unittest

Reviewed By: jdoerfert

Differential Revision: https://reviews.llvm.org/D84532
---
 llvm/unittests/Transforms/IPO/AttributorTest.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/llvm/unittests/Transforms/IPO/AttributorTest.cpp b/llvm/unittests/Transforms/IPO/AttributorTest.cpp
index 8c1682c01f307..7d496ed0d14fb 100644
--- a/llvm/unittests/Transforms/IPO/AttributorTest.cpp
+++ b/llvm/unittests/Transforms/IPO/AttributorTest.cpp
@@ -44,8 +44,8 @@ TEST_F(AttributorTestBase, TestCast) {
 
   Function *F = M.getFunction("foo");
 
-  AbstractAttribute *AA = (AbstractAttribute *)&(
-      A.getOrCreateAAFor<AAIsDead>(IRPosition::function(*F)));
+  const AbstractAttribute *AA =
+      &A.getOrCreateAAFor<AAIsDead>(IRPosition::function(*F));
 
   EXPECT_TRUE(AA);
 

From 343ffa70fc4c55f4dc0d717cf8c168865beaa9c4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Martin=20Storsj=C3=B6?= <martin@martin.st>
Date: Sat, 25 Jul 2020 12:25:19 +0300
Subject: [PATCH 0206/1035] [LLD] [COFF] Fix mingw comdat associativity for
 leader symbols with a different name

For a weak symbol func in a comdat, the actual leader symbol ends up
named like .weak.func.default*. Likewise, for stdcall on i386, the symbol
may be named _func@4, while the section suffix only is "func", which the
previous implementation didn't handle.

This fixes unwinding through weak functions when using
-ffunction-sections in mingw environments.

Differential Revision: https://reviews.llvm.org/D84607
---
 lld/COFF/InputFiles.cpp                       |  8 +--
 lld/test/COFF/associative-comdat-mingw-i386.s | 21 ++++++-
 lld/test/COFF/associative-comdat-mingw-weak.s | 63 +++++++++++++++++++
 3 files changed, 87 insertions(+), 5 deletions(-)
 create mode 100644 lld/test/COFF/associative-comdat-mingw-weak.s

diff --git a/lld/COFF/InputFiles.cpp b/lld/COFF/InputFiles.cpp
index 0adc2b91bd999..4346b3a2ffa7b 100644
--- a/lld/COFF/InputFiles.cpp
+++ b/lld/COFF/InputFiles.cpp
@@ -348,13 +348,13 @@ void ObjFile::recordPrevailingSymbolForMingw(
   // of the section chunk we actually include instead of discarding it,
   // add the symbol to a map to allow using it for implicitly
   // associating .[px]data$<func> sections to it.
+  // Use the suffix from the .text$<func> instead of the leader symbol
+  // name, for cases where the names differ (i386 mangling/decorations,
+  // cases where the leader is a weak symbol named .weak.func.default*).
   int32_t sectionNumber = sym.getSectionNumber();
   SectionChunk *sc = sparseChunks[sectionNumber];
   if (sc && sc->getOutputCharacteristics() & IMAGE_SCN_MEM_EXECUTE) {
-    StringRef name;
-    name = check(coffObj->getSymbolName(sym));
-    if (getMachineType() == I386)
-      name.consume_front("_");
+    StringRef name = sc->getSectionName().split('$').second;
     prevailingSectionMap[name] = sectionNumber;
   }
 }
diff --git a/lld/test/COFF/associative-comdat-mingw-i386.s b/lld/test/COFF/associative-comdat-mingw-i386.s
index 8d89478d4eb03..3ba8c1cd9a75b 100644
--- a/lld/test/COFF/associative-comdat-mingw-i386.s
+++ b/lld/test/COFF/associative-comdat-mingw-i386.s
@@ -1,10 +1,14 @@
 # REQUIRES: x86
 
-# RUN: llvm-mc -triple=i686-windows-gnu %s -filetype=obj -o %t.obj
+# RUN: llvm-mc -triple=i686-windows-gnu %s -defsym stdcall=0 -filetype=obj -o %t.obj
 
 # RUN: lld-link -lldmingw -entry:main %t.obj -out:%t.exe
 # RUN: llvm-objdump -s %t.exe | FileCheck %s
 
+# RUN: llvm-mc -triple=i686-windows-gnu %s -defsym stdcall=1 -filetype=obj -o %t.stdcall.obj
+# RUN: lld-link -lldmingw -entry:main %t.stdcall.obj -out:%t.stdcall.exe
+# RUN: llvm-objdump -s %t.stdcall.exe | FileCheck %s
+
 # Check that the .eh_frame comdat was included, even if it had no symbols,
 # due to associativity with the symbol _foo.
 
@@ -19,19 +23,34 @@
         .globl          _main
         .p2align        4, 0x90
 _main:
+.if stdcall==0
         call            _foo
+.else
+        call            _foo@0
+.endif
         ret
 
         .section        .eh_frame$foo,"dr"
         .linkonce       discard
         .byte           0x42
 
+.if stdcall==0
         .def            _foo;
+.else
+        .def            _foo@0;
+.endif
         .scl            2;
         .type           32;
         .endef
+.if stdcall==0
         .section        .text$foo,"xr",discard,_foo
         .globl          _foo
         .p2align        4
 _foo:
+.else
+        .section        .text$foo,"xr",discard,_foo@0
+        .globl          _foo@0
+        .p2align        4
+_foo@0:
+.endif
         ret
diff --git a/lld/test/COFF/associative-comdat-mingw-weak.s b/lld/test/COFF/associative-comdat-mingw-weak.s
new file mode 100644
index 0000000000000..80c738b436be4
--- /dev/null
+++ b/lld/test/COFF/associative-comdat-mingw-weak.s
@@ -0,0 +1,63 @@
+# REQUIRES: x86
+
+# RUN: llvm-mc -triple=x86_64-windows-gnu %s -filetype=obj -o %t.obj
+# RUN: llvm-readobj --symbols %t.obj | FileCheck %s --check-prefix=SYMBOL
+
+# RUN: lld-link -lldmingw -entry:main %t.obj -out:%t.exe -lldmap:%t.map -verbose
+# RUN: llvm-readobj --sections %t.exe | FileCheck %s
+
+# CHECK: Sections [
+# CHECK:   Section {
+# CHECK:     Number: 2
+# CHECK-LABEL:     Name: .rdata (2E 72 64 61 74 61 00 00)
+#             This is the critical check to show that .xdata$foo was
+#             retained, while .xdata$bar wasn't. This *must* be 0x24
+#             (0x4 for the .xdata section and 0x20 for the
+#             .ctors/.dtors headers/ends).
+# CHECK-NEXT:     VirtualSize: 0x24
+
+# Check that the weak symbols still are emitted as it was when the test was
+# written, to make sure the test still actually tests what was intended.
+
+# SYMBOL:       Symbol {
+# SYMBOL:         Name: foo
+# SYMBOL-NEXT:    Value: 0
+# SYMBOL-NEXT:    Section: IMAGE_SYM_UNDEFINED (0)
+# SYMBOL-NEXT:    BaseType: Null (0x0)
+# SYMBOL-NEXT:    ComplexType: Null (0x0)
+# SYMBOL-NEXT:    StorageClass: WeakExternal (0x69)
+# SYMBOL-NEXT:    AuxSymbolCount: 1
+# SYMBOL-NEXT:    AuxWeakExternal {
+# SYMBOL-NEXT:      Linked: .weak.foo.default.main (19)
+# SYMBOL-NEXT:      Search: Alias (0x3)
+# SYMBOL-NEXT:    }
+# SYMBOL-NEXT:  }
+
+        .text
+        .globl          main
+main:
+        call            foo
+        retq
+
+# See associative-comdat-mingw.s for the general setup. Here, the leader
+# symbols are weak, which causes the functions foo and bar to be undefined
+# weak externals, while the actual leader symbols are named like
+# .weak.foo.default.main.
+
+        .section        .xdata$foo,"dr"
+        .linkonce       discard
+        .long           42
+
+        .section        .xdata$bar,"dr"
+        .linkonce       discard
+        .long           43
+
+        .section        .text$foo,"xr",discard,foo
+        .weak           foo
+foo:
+        ret
+
+        .section        .text$bar,"xr",discard,bar
+        .weak           bar
+bar:
+        ret

From 7c182663a857fc87552fa2861c7f94046d55845e Mon Sep 17 00:00:00 2001
From: Amy Kwan <amy.kwan1@ibm.com>
Date: Mon, 27 Jul 2020 09:36:38 -0500
Subject: [PATCH 0207/1035] Revert "Re-apply:" Emit DW_OP_implicit_value for
 Floating point constants""

This patch reverts commit `59a76d957a26` as it has caused failure on the
big endian PowerPC buildbots (as well as the SystemZ buildbots).
---
 llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp    | 21 +-----
 .../CodeGen/AsmPrinter/DwarfExpression.cpp    | 32 ---------
 llvm/lib/CodeGen/AsmPrinter/DwarfExpression.h |  3 -
 .../test/DebugInfo/X86/float_const_loclist.ll |  4 +-
 .../DebugInfo/X86/implicit_value-double.ll    | 66 -----------------
 .../DebugInfo/X86/implicit_value-float.ll     | 65 -----------------
 llvm/test/DebugInfo/X86/implicit_value-ld.ll  | 71 -------------------
 7 files changed, 6 insertions(+), 256 deletions(-)
 delete mode 100644 llvm/test/DebugInfo/X86/implicit_value-double.ll
 delete mode 100644 llvm/test/DebugInfo/X86/implicit_value-float.ll
 delete mode 100644 llvm/test/DebugInfo/X86/implicit_value-ld.ll

diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
index 5a4c4dfb90a2a..1169adaaf470c 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
@@ -2488,7 +2488,6 @@ void DwarfDebug::emitDebugLocValue(const AsmPrinter &AP, const DIBasicType *BT,
       DwarfExpr.addSignedConstant(Value.getInt());
     else
       DwarfExpr.addUnsignedConstant(Value.getInt());
-    DwarfExpr.addExpression(std::move(ExprCursor));
   } else if (Value.isLocation()) {
     MachineLocation Location = Value.getLoc();
     DwarfExpr.setLocation(Location, DIExpr);
@@ -2509,24 +2508,10 @@ void DwarfDebug::emitDebugLocValue(const AsmPrinter &AP, const DIBasicType *BT,
       DwarfExpr.addExpression(std::move(ExprCursor));
       return;
   } else if (Value.isConstantFP()) {
-    if (AP.getDwarfVersion() >= 4 && AP.getDwarfDebug()->tuneForGDB()) {
-      DwarfExpr.addConstantFP(Value.getConstantFP()->getValueAPF());
-      return;
-    } else if (Value.getConstantFP()
-                   ->getValueAPF()
-                   .bitcastToAPInt()
-                   .getBitWidth() <= 64 /*bits*/) {
-      DwarfExpr.addUnsignedConstant(
-          Value.getConstantFP()->getValueAPF().bitcastToAPInt());
-      DwarfExpr.addExpression(std::move(ExprCursor));
-      return;
-    }
-    LLVM_DEBUG(
-        dbgs()
-        << "Skipped DwarfExpression creation for ConstantFP of size: "
-        << Value.getConstantFP()->getValueAPF().bitcastToAPInt().getBitWidth()
-        << " bits\n");
+    APInt RawBytes = Value.getConstantFP()->getValueAPF().bitcastToAPInt();
+    DwarfExpr.addUnsignedConstant(RawBytes);
   }
+  DwarfExpr.addExpression(std::move(ExprCursor));
 }
 
 void DebugLocEntry::finalize(const AsmPrinter &AP,
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.cpp
index 7e6f4a0425d87..d4762121d1050 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.cpp
@@ -25,8 +25,6 @@
 
 using namespace llvm;
 
-#define DEBUG_TYPE "dwarfdebug"
-
 void DwarfExpression::emitConstu(uint64_t Value) {
   if (Value < 32)
     emitOp(dwarf::DW_OP_lit0 + Value);
@@ -221,36 +219,6 @@ void DwarfExpression::addUnsignedConstant(const APInt &Value) {
   }
 }
 
-void DwarfExpression::addConstantFP(const APFloat &Value) {
-  assert(isImplicitLocation() || isUnknownLocation());
-  APInt RawBytes = Value.bitcastToAPInt();
-  int NumBytes = RawBytes.getBitWidth() / 8;
-  const char *Data = (const char *)RawBytes.getRawData();
-  emitOp(dwarf::DW_OP_implicit_value);
-  if (NumBytes == 4 /*float*/ || NumBytes == 8 /*double*/) {
-    emitUnsigned(NumBytes /*Size of the block in bytes*/);
-    for (int i = 0; i < NumBytes; ++i)
-      emitData1(Data[i]);
-    return;
-  }
-  if (NumBytes == 10 /*long double*/) {
-    // long double IEEE representation uses 80 bits(10 bytes).
-    // 6 bytes are padded to make it 128 bits(16 bytes) due to
-    // addressing restrictions.
-    emitUnsigned(16 /*Size of the block in bytes*/);
-    // Emit the block of bytes.
-    for (int i = 0; i < NumBytes; ++i)
-      emitData1(Data[i]);
-    // Emit the rest as padding bytes.
-    for (int i = 0; i < 16 - NumBytes; ++i)
-      emitData1(0);
-    return;
-  }
-  LLVM_DEBUG(
-      dbgs() << "Skipped DW_OP_implicit_value creation for ConstantFP of size: "
-             << RawBytes.getBitWidth() << " bits\n");
-}
-
 bool DwarfExpression::addMachineRegExpression(const TargetRegisterInfo &TRI,
                                               DIExpressionCursor &ExprCursor,
                                               unsigned MachineReg,
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.h b/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.h
index 097208f1cfade..757b175114535 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.h
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.h
@@ -299,9 +299,6 @@ class DwarfExpression {
   /// Emit an unsigned constant.
   void addUnsignedConstant(const APInt &Value);
 
-  /// Emit floating point constant.
-  void addConstantFP(const APFloat &Value);
-
   /// Lock this down to become a memory location description.
   void setMemoryLocationKind() {
     assert(isUnknownLocation());
diff --git a/llvm/test/DebugInfo/X86/float_const_loclist.ll b/llvm/test/DebugInfo/X86/float_const_loclist.ll
index 24ee16444836a..f9008209e5dfb 100644
--- a/llvm/test/DebugInfo/X86/float_const_loclist.ll
+++ b/llvm/test/DebugInfo/X86/float_const_loclist.ll
@@ -20,10 +20,12 @@
 ;
 ; CHECK: .debug_info contents:
 ; CHECK: DW_TAG_variable
+; CHECK-NEXT:  DW_AT_location {{.*}} (
+; CHECK-NEXT:    [0x[[START:.*]], 0x[[END:.*]]): DW_OP_constu 0xc8f5c28f5c28f800, DW_OP_piece 0x8, DW_OP_constu 0x4000, DW_OP_bit_piece 0x10 0x40)
 ; CHECK-NEXT:  DW_AT_name {{.*}}"ld"
 ; CHECK: DW_TAG_variable
 ; CHECK-NEXT:  DW_AT_location {{.*}} (
-; CHECK-NEXT:    [0x{{.*}}, 0x{{.*}}): DW_OP_constu 0x4048f5c3)
+; CHECK-NEXT:    [0x[[START]], 0x[[END]]): DW_OP_constu 0x4048f5c3)
 ; CHECK-NEXT:  DW_AT_name {{.*}}"f"
 
 source_filename = "test.c"
diff --git a/llvm/test/DebugInfo/X86/implicit_value-double.ll b/llvm/test/DebugInfo/X86/implicit_value-double.ll
deleted file mode 100644
index 3c14c7dfefce4..0000000000000
--- a/llvm/test/DebugInfo/X86/implicit_value-double.ll
+++ /dev/null
@@ -1,66 +0,0 @@
-;; This test checks for emission of DW_OP_implicit_value operation
-;; for double type.
-
-; RUN: llc -debugger-tune=gdb -filetype=obj %s -o - | llvm-dwarfdump - | FileCheck %s
-
-; CHECK: .debug_info contents:
-; CHECK: DW_TAG_variable
-; CHECK-NEXT:  DW_AT_location        ({{.*}}
-; CHECK-NEXT:                     [{{.*}}): DW_OP_implicit_value 0x8 0x1f 0x85 0xeb 0x51 0xb8 0x1e 0x09 0x40)
-; CHECK-NEXT:  DW_AT_name    ("d")
-
-;; Generated from: clang -ggdb -O1
-;;int main() {
-;;        double d = 3.14;
-;;        printf("dummy\n");
-;;        d *= d;
-;;        return 0;
-;;}
-
-; ModuleID = 'implicit_value-double.c'
-source_filename = "implicit_value-double.c"
-target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
-target triple = "x86_64-unknown-linux-gnu"
-
-@str = private unnamed_addr constant [6 x i8] c"dummy\00", align 1
-
-; Function Attrs: nofree nounwind uwtable
-define dso_local i32 @main() local_unnamed_addr #0 !dbg !7 {
-entry:
-  call void @llvm.dbg.value(metadata double 3.140000e+00, metadata !12, metadata !DIExpression()), !dbg !14
-  %puts = call i32 @puts(i8* nonnull dereferenceable(1) getelementptr inbounds ([6 x i8], [6 x i8]* @str, i64 0, i64 0)), !dbg !15
-  call void @llvm.dbg.value(metadata double undef, metadata !12, metadata !DIExpression()), !dbg !14
-  ret i32 0, !dbg !16
-}
-
-; Function Attrs: nounwind readnone speculatable willreturn
-declare void @llvm.dbg.value(metadata, metadata, metadata) #1
-
-; Function Attrs: nofree nounwind
-declare i32 @puts(i8* nocapture readonly) local_unnamed_addr #2
-
-attributes #0 = { nofree nounwind uwtable }
-attributes #1 = { nounwind readnone speculatable willreturn }
-attributes #2 = { nofree nounwind }
-
-!llvm.dbg.cu = !{!0}
-!llvm.module.flags = !{!3, !4, !5}
-!llvm.ident = !{!6}
-
-!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 11.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, splitDebugInlining: false, nameTableKind: None)
-!1 = !DIFile(filename: "implicit_value-double.c", directory: "/home/")
-!2 = !{}
-!3 = !{i32 7, !"Dwarf Version", i32 4}
-!4 = !{i32 2, !"Debug Info Version", i32 3}
-!5 = !{i32 1, !"wchar_size", i32 4}
-!6 = !{!"clang version 11.0.0"}
-!7 = distinct !DISubprogram(name: "main", scope: !1, file: !1, line: 1, type: !8, scopeLine: 1, flags: DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !11)
-!8 = !DISubroutineType(types: !9)
-!9 = !{!10}
-!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
-!11 = !{!12}
-!12 = !DILocalVariable(name: "d", scope: !7, file: !1, line: 2, type: !13)
-!13 = !DIBasicType(name: "double", size: 64, encoding: DW_ATE_float)
-!14 = !DILocation(line: 0, scope: !7)
-!15 = !DILocation(line: 3, column: 2, scope: !7)
-!16 = !DILocation(line: 5, column: 2, scope: !7)
diff --git a/llvm/test/DebugInfo/X86/implicit_value-float.ll b/llvm/test/DebugInfo/X86/implicit_value-float.ll
deleted file mode 100644
index 8c51b49481773..0000000000000
--- a/llvm/test/DebugInfo/X86/implicit_value-float.ll
+++ /dev/null
@@ -1,65 +0,0 @@
-;; This test checks for emission of DW_OP_implicit_value operation
-;; for float type.
-
-; RUN: llc -debugger-tune=gdb -filetype=obj %s -o - | llvm-dwarfdump - | FileCheck %s
-
-; CHECK: .debug_info contents:
-; CHECK: DW_TAG_variable
-; CHECK-NEXT:  DW_AT_location        ({{.*}}
-; CHECK-NEXT:                     [{{.*}}): DW_OP_implicit_value 0x4 0xc3 0xf5 0x48 0x40)
-; CHECK-NEXT:  DW_AT_name    ("f")
-
-;; Generated from: clang -ggdb -O1
-;;int main() {
-;;        float f = 3.14f;
-;;        printf("dummy\n");
-;;        f *= f;
-;;        return 0;
-;;}
-; ModuleID = 'implicit_value-float.c'
-source_filename = "implicit_value-float.c"
-target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
-target triple = "x86_64-unknown-linux-gnu"
-
-@str = private unnamed_addr constant [6 x i8] c"dummy\00", align 1
-
-; Function Attrs: nofree nounwind uwtable
-define dso_local i32 @main() local_unnamed_addr #0 !dbg !7 {
-entry:
-  call void @llvm.dbg.value(metadata float 0x40091EB860000000, metadata !12, metadata !DIExpression()), !dbg !14
-  %puts = call i32 @puts(i8* nonnull dereferenceable(1) getelementptr inbounds ([6 x i8], [6 x i8]* @str, i64 0, i64 0)), !dbg !15
-  call void @llvm.dbg.value(metadata float undef, metadata !12, metadata !DIExpression()), !dbg !14
-  ret i32 0, !dbg !16
-}
-
-; Function Attrs: nounwind readnone speculatable willreturn
-declare void @llvm.dbg.value(metadata, metadata, metadata) #1
-
-; Function Attrs: nofree nounwind
-declare i32 @puts(i8* nocapture readonly) local_unnamed_addr #2
-
-attributes #0 = { nofree nounwind uwtable }
-attributes #1 = { nounwind readnone speculatable willreturn }
-attributes #2 = { nofree nounwind }
-
-!llvm.dbg.cu = !{!0}
-!llvm.module.flags = !{!3, !4, !5}
-!llvm.ident = !{!6}
-
-!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 11.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, splitDebugInlining: false, nameTableKind: None)
-!1 = !DIFile(filename: "implicit_value-float.c", directory: "/home/")
-!2 = !{}
-!3 = !{i32 7, !"Dwarf Version", i32 4}
-!4 = !{i32 2, !"Debug Info Version", i32 3}
-!5 = !{i32 1, !"wchar_size", i32 4}
-!6 = !{!"clang version 11.0.0"}
-!7 = distinct !DISubprogram(name: "main", scope: !1, file: !1, line: 1, type: !8, scopeLine: 1, flags: DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !11)
-!8 = !DISubroutineType(types: !9)
-!9 = !{!10}
-!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
-!11 = !{!12}
-!12 = !DILocalVariable(name: "f", scope: !7, file: !1, line: 2, type: !13)
-!13 = !DIBasicType(name: "float", size: 32, encoding: DW_ATE_float)
-!14 = !DILocation(line: 0, scope: !7)
-!15 = !DILocation(line: 3, column: 2, scope: !7)
-!16 = !DILocation(line: 5, column: 2, scope: !7)
diff --git a/llvm/test/DebugInfo/X86/implicit_value-ld.ll b/llvm/test/DebugInfo/X86/implicit_value-ld.ll
deleted file mode 100644
index 5fdaf396c5347..0000000000000
--- a/llvm/test/DebugInfo/X86/implicit_value-ld.ll
+++ /dev/null
@@ -1,71 +0,0 @@
-;; This test checks for emission of DW_OP_implicit_value operation
-;; for long double type.
-
-; RUN: llc -debugger-tune=gdb -filetype=obj %s -o - | llvm-dwarfdump - | FileCheck %s
-
-; CHECK: .debug_info contents:
-; CHECK: DW_TAG_variable
-; CHECK-NEXT:  DW_AT_location        ({{.*}}
-; CHECK-NEXT:                     [{{.*}}): DW_OP_implicit_value 0x10 0x00 0xf8 0x28 0x5c 0x8f 0xc2 0xf5 0xc8 0x00 0x40 0x00 0x00 0x00 0x00 0x00 0x00)
-; CHECK-NEXT:  DW_AT_name    ("ld")
-
-;; Generated from: clang -ggdb -O1
-;;int main() {
-;;        long double ld = 3.14;
-;;        printf("dummy\n");
-;;        ld *= ld;
-;;        return 0;
-;;}
-
-; ModuleID = 'implicit_value-ld.c'
-source_filename = "implicit_value-ld.c"
-target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
-target triple = "x86_64-unknown-linux-gnu"
-
-@str = private unnamed_addr constant [6 x i8] c"dummy\00", align 1
-
-; Function Attrs: nofree nounwind uwtable
-define dso_local i32 @main() local_unnamed_addr #0 !dbg !7 {
-entry:
-  call void @llvm.dbg.declare(metadata [6 x i8]* undef, metadata !12, metadata !DIExpression(DW_OP_LLVM_fragment, 80, 48)), !dbg !14
-  call void @llvm.dbg.value(metadata x86_fp80 0xK4000C8F5C28F5C28F800, metadata !12, metadata !DIExpression()), !dbg !15
-  %puts = call i32 @puts(i8* nonnull dereferenceable(1) getelementptr inbounds ([6 x i8], [6 x i8]* @str, i64 0, i64 0)), !dbg !16
-  call void @llvm.dbg.value(metadata x86_fp80 undef, metadata !12, metadata !DIExpression()), !dbg !15
-  ret i32 0, !dbg !17
-}
-
-; Function Attrs: nounwind readnone speculatable willreturn
-declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
-
-; Function Attrs: nounwind readnone speculatable willreturn
-declare void @llvm.dbg.value(metadata, metadata, metadata) #1
-
-; Function Attrs: nofree nounwind
-declare i32 @puts(i8* nocapture readonly) local_unnamed_addr #2
-
-attributes #0 = { nofree nounwind uwtable }
-attributes #1 = { nounwind readnone speculatable willreturn }
-attributes #2 = { nofree nounwind }
-
-!llvm.dbg.cu = !{!0}
-!llvm.module.flags = !{!3, !4, !5}
-!llvm.ident = !{!6}
-
-!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 11.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, splitDebugInlining: false, nameTableKind: None)
-!1 = !DIFile(filename: "implicit_value-ld.c", directory: "/home/")
-!2 = !{}
-!3 = !{i32 7, !"Dwarf Version", i32 4}
-!4 = !{i32 2, !"Debug Info Version", i32 3}
-!5 = !{i32 1, !"wchar_size", i32 4}
-!6 = !{!"clang version 11.0.0"}
-!7 = distinct !DISubprogram(name: "main", scope: !1, file: !1, line: 1, type: !8, scopeLine: 1, flags: DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !11)
-!8 = !DISubroutineType(types: !9)
-!9 = !{!10}
-!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
-!11 = !{!12}
-!12 = !DILocalVariable(name: "ld", scope: !7, file: !1, line: 2, type: !13)
-!13 = !DIBasicType(name: "long double", size: 128, encoding: DW_ATE_float)
-!14 = !DILocation(line: 2, column: 14, scope: !7)
-!15 = !DILocation(line: 0, scope: !7)
-!16 = !DILocation(line: 3, column: 2, scope: !7)
-!17 = !DILocation(line: 5, column: 2, scope: !7)

From 4dd5c2bee366514cbc3fc4e6da46462bc11a0a3d Mon Sep 17 00:00:00 2001
From: Sergej Jaskiewicz <jaskiewiczs@icloud.com>
Date: Wed, 15 Jul 2020 18:55:37 +0300
Subject: [PATCH 0208/1035] [lit] Don't expand escapes until all substitutions
 have been applied

Otherwise, if a Lit script contains escaped substitutions (like %%p in this test https://github.com/llvm/llvm-project/blob/master/compiler-rt/test/asan/TestCases/Darwin/asan-symbolize-partial-report-with-module-map.cpp#L10), they are unescaped during recursive application of substitutions, and the results are unexpected.

We solve it using the fact that double percent signs are first replaced with #_MARKER_#, and only after all the other substitutions have been applied, #_MARKER_# is replaced with a single percent sign. The only change is that instead of replacing #_MARKER_# at each recursion step, we replace it once after the last recursion step.

Differential Revision: https://reviews.llvm.org/D83894
---
 llvm/utils/lit/lit/TestRunner.py              | 20 +++++++++++--------
 .../escaping/lit.cfg                          | 10 ++++++++++
 .../escaping/test.py                          |  1 +
 .../tests/shtest-recursive-substitution.py    |  4 ++++
 4 files changed, 27 insertions(+), 8 deletions(-)
 create mode 100644 llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/escaping/lit.cfg
 create mode 100644 llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/escaping/test.py

diff --git a/llvm/utils/lit/lit/TestRunner.py b/llvm/utils/lit/lit/TestRunner.py
index d80f0aeee8ce9..643b03fc279a1 100644
--- a/llvm/utils/lit/lit/TestRunner.py
+++ b/llvm/utils/lit/lit/TestRunner.py
@@ -1081,9 +1081,7 @@ def getDefaultSubstitutions(test, tmpDir, tmpBase, normalize_slashes=False):
         tmpDir = tmpDir.replace('\\', '/')
         tmpBase = tmpBase.replace('\\', '/')
 
-    # We use #_MARKER_# to hide %% while we do the other substitutions.
     substitutions = []
-    substitutions.extend([('%%', '#_MARKER_#')])
     substitutions.extend(test.config.substitutions)
     tmpName = tmpBase + '.tmp'
     baseName = os.path.basename(tmpBase)
@@ -1093,8 +1091,7 @@ def getDefaultSubstitutions(test, tmpDir, tmpBase, normalize_slashes=False):
                           ('%{pathsep}', os.pathsep),
                           ('%t', tmpName),
                           ('%basename_t', baseName),
-                          ('%T', tmpDir),
-                          ('#_MARKER_#', '%')])
+                          ('%T', tmpDir)])
 
     # "%/[STpst]" should be normalized.
     substitutions.extend([
@@ -1159,6 +1156,14 @@ def applySubstitutions(script, substitutions, recursion_limit=None):
     `recursion_limit` times, it is an error. If the `recursion_limit` is
     `None` (the default), no recursive substitution is performed at all.
     """
+
+    # We use #_MARKER_# to hide %% while we do the other substitutions.
+    def escape(ln):
+        return _caching_re_compile('%%').sub('#_MARKER_#', ln)
+
+    def unescape(ln):
+        return _caching_re_compile('#_MARKER_#').sub('%', ln)
+
     def processLine(ln):
         # Apply substitutions
         for a,b in substitutions:
@@ -1171,7 +1176,7 @@ def processLine(ln):
             # short-lived, since the set of substitutions is fairly small, and
             # since thrashing has such bad consequences, not bounding the cache
             # seems reasonable.
-            ln = _caching_re_compile(a).sub(str(b), ln)
+            ln = _caching_re_compile(a).sub(str(b), escape(ln))
 
         # Strip the trailing newline and any extra whitespace.
         return ln.strip()
@@ -1193,10 +1198,9 @@ def processLineToFixedPoint(ln):
 
         return processed
 
-    # Note Python 3 map() gives an iterator rather than a list so explicitly
-    # convert to list before returning.
     process = processLine if recursion_limit is None else processLineToFixedPoint
-    return list(map(process, script))
+    
+    return [unescape(process(ln)) for ln in script]
 
 
 class ParserKind(object):
diff --git a/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/escaping/lit.cfg b/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/escaping/lit.cfg
new file mode 100644
index 0000000000000..97a4faac23870
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/escaping/lit.cfg
@@ -0,0 +1,10 @@
+import lit.formats
+config.name = 'escaping'
+config.suffixes = ['.py']
+config.test_format = lit.formats.ShTest()
+config.test_source_root = None
+config.test_exec_root = None
+
+config.substitutions = [("%rec1", "%%s"), ("%rec2", "%rec1")]
+
+config.recursiveExpansionLimit = 5
diff --git a/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/escaping/test.py b/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/escaping/test.py
new file mode 100644
index 0000000000000..74056e71caf47
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/escaping/test.py
@@ -0,0 +1 @@
+# RUN: echo %rec2 %%s %%%%s
diff --git a/llvm/utils/lit/tests/shtest-recursive-substitution.py b/llvm/utils/lit/tests/shtest-recursive-substitution.py
index 1b3d5f4c44519..d446c422139bc 100644
--- a/llvm/utils/lit/tests/shtest-recursive-substitution.py
+++ b/llvm/utils/lit/tests/shtest-recursive-substitution.py
@@ -21,3 +21,7 @@
 
 # RUN: %{lit} -j 1 %{inputs}/shtest-recursive-substitution/set-to-none --show-all | FileCheck --check-prefix=CHECK-TEST6 %s
 # CHECK-TEST6: PASS: set-to-none :: test.py
+
+# RUN: %{lit} -j 1 %{inputs}/shtest-recursive-substitution/escaping --show-all | FileCheck --check-prefix=CHECK-TEST7 %s
+# CHECK-TEST7: PASS: escaping :: test.py
+# CHECK-TEST7: $ "echo" "%s" "%s" "%%s"

From f5e1ec8c5804ab7bd36f9acd43124b2029fbabc4 Mon Sep 17 00:00:00 2001
From: Jon Roelofs <jonathan_roelofs@apple.com>
Date: Tue, 14 Jul 2020 15:45:05 -0600
Subject: [PATCH 0209/1035] [AArch64] fjcvtzs,rmif,cfinv,setf* all clobber nzcv

Differential Revision: https://reviews.llvm.org/D83818
---
 llvm/lib/Target/AArch64/AArch64InstrFormats.td |  1 +
 llvm/lib/Target/AArch64/AArch64InstrInfo.td    |  4 ++--
 llvm/test/CodeGen/AArch64/cfinv-def-nzcv.mir   | 17 +++++++++++++++++
 llvm/test/CodeGen/AArch64/cfinv-use-nzcv.mir   | 17 +++++++++++++++++
 llvm/test/CodeGen/AArch64/fjcvtzs.mir          | 17 +++++++++++++++++
 llvm/test/CodeGen/AArch64/rmif-def-nzcv.mir    | 16 ++++++++++++++++
 llvm/test/CodeGen/AArch64/rmif-use-nzcv.mir    | 16 ++++++++++++++++
 llvm/test/CodeGen/AArch64/setf16-def-nzcv.mir  | 16 ++++++++++++++++
 llvm/test/CodeGen/AArch64/setf16-use-nzcv.mir  | 16 ++++++++++++++++
 llvm/test/CodeGen/AArch64/setf8-def-nzcv.mir   | 16 ++++++++++++++++
 llvm/test/CodeGen/AArch64/setf8-use-nzcv.mir   | 16 ++++++++++++++++
 11 files changed, 150 insertions(+), 2 deletions(-)
 create mode 100644 llvm/test/CodeGen/AArch64/cfinv-def-nzcv.mir
 create mode 100644 llvm/test/CodeGen/AArch64/cfinv-use-nzcv.mir
 create mode 100644 llvm/test/CodeGen/AArch64/fjcvtzs.mir
 create mode 100644 llvm/test/CodeGen/AArch64/rmif-def-nzcv.mir
 create mode 100644 llvm/test/CodeGen/AArch64/rmif-use-nzcv.mir
 create mode 100644 llvm/test/CodeGen/AArch64/setf16-def-nzcv.mir
 create mode 100644 llvm/test/CodeGen/AArch64/setf16-use-nzcv.mir
 create mode 100644 llvm/test/CodeGen/AArch64/setf8-def-nzcv.mir
 create mode 100644 llvm/test/CodeGen/AArch64/setf8-use-nzcv.mir

diff --git a/llvm/lib/Target/AArch64/AArch64InstrFormats.td b/llvm/lib/Target/AArch64/AArch64InstrFormats.td
index 4f4ba692c2db4..e2403c5f6347f 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrFormats.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrFormats.td
@@ -1942,6 +1942,7 @@ class BaseFlagManipulation<bit sf, bit sz, dag iops, string asm, string ops>
     : I<(outs), iops, asm, ops, "", []>,
       Sched<[WriteI, ReadI, ReadI]> {
   let Uses = [NZCV];
+  let Defs = [NZCV];
   bits<5> Rn;
   let Inst{31}    = sf;
   let Inst{30-15} = 0b0111010000000000;
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index d4e984754d0e9..9ea497838b634 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -1030,7 +1030,7 @@ let Predicates = [HasPA] in {
 }
 
 // v8.3a floating point conversion for javascript
-let Predicates = [HasJS, HasFPARMv8] in
+let Predicates = [HasJS, HasFPARMv8], Defs = [NZCV] in
 def FJCVTZS  : BaseFPToIntegerUnscaled<0b01, 0b11, 0b110, FPR64, GPR32,
                                       "fjcvtzs",
                                       [(set GPR32:$Rd,
@@ -1039,7 +1039,7 @@ def FJCVTZS  : BaseFPToIntegerUnscaled<0b01, 0b11, 0b110, FPR64, GPR32,
 } // HasJS, HasFPARMv8
 
 // v8.4 Flag manipulation instructions
-let Predicates = [HasFMI] in {
+let Predicates = [HasFMI], Defs = [NZCV], Uses = [NZCV] in {
 def CFINV : SimpleSystemI<0, (ins), "cfinv", "">, Sched<[WriteSys]> {
   let Inst{20-5} = 0b0000001000000000;
 }
diff --git a/llvm/test/CodeGen/AArch64/cfinv-def-nzcv.mir b/llvm/test/CodeGen/AArch64/cfinv-def-nzcv.mir
new file mode 100644
index 0000000000000..cfcda7de52c52
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/cfinv-def-nzcv.mir
@@ -0,0 +1,17 @@
+# RUN: not llc -o - %s -mtriple=arm64-eabi -run-pass=legalizer 2>&1 | FileCheck %s
+
+# CHECK: [[@LINE+11]]:29: missing implicit register operand 'implicit $nzcv'
+...
+---
+name:            test_flags
+liveins:
+  - { reg: '$w0' }
+  - { reg: '$x0' }
+body:             |
+  bb.0:
+    liveins: $w0, $x0
+
+    CFINV implicit-def $nzcv
+    RET undef $lr, implicit killed $w0
+
+
diff --git a/llvm/test/CodeGen/AArch64/cfinv-use-nzcv.mir b/llvm/test/CodeGen/AArch64/cfinv-use-nzcv.mir
new file mode 100644
index 0000000000000..4e3b3ead77155
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/cfinv-use-nzcv.mir
@@ -0,0 +1,17 @@
+# RUN: not llc -o - %s -mtriple=arm64-eabi -run-pass=legalizer 2>&1 | FileCheck %s
+
+# CHECK: [[@LINE+11]]:25: missing implicit register operand 'implicit-def $nzcv'
+...
+---
+name:            test_flags
+liveins:
+  - { reg: '$w0' }
+  - { reg: '$x0' }
+body:             |
+  bb.0:
+    liveins: $w0, $x0
+
+    CFINV implicit $nzcv
+    RET undef $lr, implicit killed $w0
+
+
diff --git a/llvm/test/CodeGen/AArch64/fjcvtzs.mir b/llvm/test/CodeGen/AArch64/fjcvtzs.mir
new file mode 100644
index 0000000000000..efdee63669e29
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/fjcvtzs.mir
@@ -0,0 +1,17 @@
+# RUN: not llc -o - %s -mtriple=arm64-eabi -mattr=+jsconv -run-pass=legalizer 2>&1 | FileCheck %s
+
+# CHECK: [[@LINE+11]]:49: missing implicit register operand 'implicit-def $nzcv'
+
+...
+---
+name:            test_jcvt
+liveins:
+  - { reg: '$d0' }
+body:             |
+  bb.0:
+    liveins: $d0
+
+    renamable $w0 = FJCVTZS killed renamable $d0
+    RET undef $lr, implicit killed $w0
+
+...
diff --git a/llvm/test/CodeGen/AArch64/rmif-def-nzcv.mir b/llvm/test/CodeGen/AArch64/rmif-def-nzcv.mir
new file mode 100644
index 0000000000000..1d0903d1af8a7
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/rmif-def-nzcv.mir
@@ -0,0 +1,16 @@
+# RUN: not llc -o - %s -mtriple=arm64-eabi -run-pass=legalizer 2>&1 | FileCheck %s
+
+# CHECK: [[@LINE+10]]:49: missing implicit register operand 'implicit $nzcv'
+...
+---
+name:            test_flags
+liveins:
+  - { reg: '$x0' }
+body:             |
+  bb.0:
+    liveins: $x0
+
+    RMIF renamable $x0, 0, 0, implicit-def $nzcv
+    RET undef $lr, implicit killed $w0
+
+
diff --git a/llvm/test/CodeGen/AArch64/rmif-use-nzcv.mir b/llvm/test/CodeGen/AArch64/rmif-use-nzcv.mir
new file mode 100644
index 0000000000000..74274a2acb2e6
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/rmif-use-nzcv.mir
@@ -0,0 +1,16 @@
+# RUN: not llc -o - %s -mtriple=arm64-eabi -run-pass=legalizer 2>&1 | FileCheck %s
+
+# CHECK: [[@LINE+10]]:45: missing implicit register operand 'implicit-def $nzcv'
+...
+---
+name:            test_flags
+liveins:
+  - { reg: '$x0' }
+body:             |
+  bb.0:
+    liveins: $x0
+
+    RMIF renamable $x0, 0, 0, implicit $nzcv
+    RET undef $lr, implicit killed $w0
+
+
diff --git a/llvm/test/CodeGen/AArch64/setf16-def-nzcv.mir b/llvm/test/CodeGen/AArch64/setf16-def-nzcv.mir
new file mode 100644
index 0000000000000..e828ddad68e19
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/setf16-def-nzcv.mir
@@ -0,0 +1,16 @@
+# RUN: not llc -o - %s -mtriple=arm64-eabi -run-pass=legalizer 2>&1 | FileCheck %s
+
+# CHECK: [[@LINE+10]]:45: missing implicit register operand 'implicit $nzcv'
+...
+---
+name:            test_flags
+liveins:
+  - { reg: '$w0' }
+body:             |
+  bb.0:
+    liveins: $w0
+
+    SETF16 renamable $w0, implicit-def $nzcv
+    RET undef $lr, implicit killed $w0
+
+
diff --git a/llvm/test/CodeGen/AArch64/setf16-use-nzcv.mir b/llvm/test/CodeGen/AArch64/setf16-use-nzcv.mir
new file mode 100644
index 0000000000000..7f2c5606f2e8f
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/setf16-use-nzcv.mir
@@ -0,0 +1,16 @@
+# RUN: not llc -o - %s -mtriple=arm64-eabi -run-pass=legalizer 2>&1 | FileCheck %s
+
+# CHECK: [[@LINE+10]]:41: missing implicit register operand 'implicit-def $nzcv'
+...
+---
+name:            test_flags
+liveins:
+  - { reg: '$w0' }
+body:             |
+  bb.0:
+    liveins: $w0
+
+    SETF16 renamable $w0, implicit $nzcv
+    RET undef $lr, implicit killed $w0
+
+
diff --git a/llvm/test/CodeGen/AArch64/setf8-def-nzcv.mir b/llvm/test/CodeGen/AArch64/setf8-def-nzcv.mir
new file mode 100644
index 0000000000000..d9a9ef00efd59
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/setf8-def-nzcv.mir
@@ -0,0 +1,16 @@
+# RUN: not llc -o - %s -mtriple=arm64-eabi -run-pass=legalizer 2>&1 | FileCheck %s
+
+# CHECK: [[@LINE+10]]:44: missing implicit register operand 'implicit $nzcv'
+...
+---
+name:            test_flags
+liveins:
+  - { reg: '$w0' }
+body:             |
+  bb.0:
+    liveins: $w0
+
+    SETF8 renamable $w0, implicit-def $nzcv
+    RET undef $lr, implicit killed $w0
+
+
diff --git a/llvm/test/CodeGen/AArch64/setf8-use-nzcv.mir b/llvm/test/CodeGen/AArch64/setf8-use-nzcv.mir
new file mode 100644
index 0000000000000..05d803f7b7b61
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/setf8-use-nzcv.mir
@@ -0,0 +1,16 @@
+# RUN: not llc -o - %s -mtriple=arm64-eabi -run-pass=legalizer 2>&1 | FileCheck %s
+
+# CHECK: [[@LINE+10]]:40: missing implicit register operand 'implicit-def $nzcv'
+...
+---
+name:            test_flags
+liveins:
+  - { reg: '$w0' }
+body:             |
+  bb.0:
+    liveins: $w0
+
+    SETF8 renamable $w0, implicit $nzcv
+    RET undef $lr, implicit killed $w0
+
+

From c25f61cf6a61bc323af118d351a27603fdd0158d Mon Sep 17 00:00:00 2001
From: jasonliu <jasonliu.development@gmail.com>
Date: Mon, 27 Jul 2020 14:04:59 +0000
Subject: [PATCH 0210/1035] [XCOFF][AIX] Handle llvm.used and
 llvm.compiler.used global array

For now, just return and do nothing when we see llvm.used and
llvm.compiler.used global array.
Hopefully, we could come up with a good solution later to prevent
linker from eliminating symbols in llvm.used array.

Reviewed By: DiggerLin, daltenty

Differential Revision: https://reviews.llvm.org/D84363
---
 llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp   | 36 ++++++++++++++-------
 llvm/test/CodeGen/PowerPC/aix-xcoff-used.ll | 26 +++++++++++++++
 2 files changed, 50 insertions(+), 12 deletions(-)
 create mode 100644 llvm/test/CodeGen/PowerPC/aix-xcoff-used.ll

diff --git a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
index bc869c39e3934..540e620a845bc 100644
--- a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
+++ b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
@@ -1678,22 +1678,31 @@ void PPCAIXAsmPrinter::ValidateGV(const GlobalVariable *GV) {
     report_fatal_error("COMDAT not yet supported by AIX.");
 }
 
-static bool isSpecialLLVMGlobalArrayForStaticInit(const GlobalVariable *GV) {
-  return StringSwitch<bool>(GV->getName())
-      .Cases("llvm.global_ctors", "llvm.global_dtors", true)
-      .Default(false);
+static bool isSpecialLLVMGlobalArrayToSkip(const GlobalVariable *GV) {
+  return GV->hasAppendingLinkage() &&
+         StringSwitch<bool>(GV->getName())
+             // TODO: Update the handling of global arrays for static init when
+             // we support the ".ref" directive.
+             // Otherwise, we can skip these arrays, because the AIX linker
+             // collects static init functions simply based on their name.
+             .Cases("llvm.global_ctors", "llvm.global_dtors", true)
+             // TODO: Linker could still eliminate the GV if we just skip
+             // handling llvm.used array. Skipping them for now until we or the
+             // AIX OS team come up with a good solution.
+             .Case("llvm.used", true)
+             // It's correct to just skip llvm.compiler.used array here.
+             .Case("llvm.compiler.used", true)
+             .Default(false);
 }
 
 void PPCAIXAsmPrinter::emitGlobalVariable(const GlobalVariable *GV) {
-  ValidateGV(GV);
-
-  // TODO: Update the handling of global arrays for static init when we support
-  // the ".ref" directive.
-  // Otherwise, we can skip these arrays, because the AIX linker collects
-  // static init functions simply based on their name.
-  if (isSpecialLLVMGlobalArrayForStaticInit(GV))
+  if (isSpecialLLVMGlobalArrayToSkip(GV))
     return;
 
+  assert(!GV->getName().startswith("llvm.") &&
+         "Unhandled intrinsic global variable.");
+  ValidateGV(GV);
+
   // Create the symbol, set its storage class.
   MCSymbolXCOFF *GVSym = cast<MCSymbolXCOFF>(getSymbol(GV));
   GVSym->setStorageClass(
@@ -1836,8 +1845,11 @@ bool PPCAIXAsmPrinter::doInitialization(Module &M) {
   // We need to know, up front, the alignment of csects for the assembly path,
   // because once a .csect directive gets emitted, we could not change the
   // alignment value on it.
-  for (const auto &G : M.globals())
+  for (const auto &G : M.globals()) {
+    if (isSpecialLLVMGlobalArrayToSkip(&G))
+      continue;
     setCsectAlignment(&G);
+  }
 
   for (const auto &F : M)
     setCsectAlignment(&F);
diff --git a/llvm/test/CodeGen/PowerPC/aix-xcoff-used.ll b/llvm/test/CodeGen/PowerPC/aix-xcoff-used.ll
new file mode 100644
index 0000000000000..dd0812f3d8c78
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/aix-xcoff-used.ll
@@ -0,0 +1,26 @@
+;; This test verifies llc on AIX would not crash when llvm.used and
+;; llvm.compiler.used is presented in the IR.
+
+; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mtriple powerpc-ibm-aix-xcoff < %s | \
+; RUN:   FileCheck %s
+
+; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mtriple powerpc64-ibm-aix-xcoff < %s | \
+; RUN:   FileCheck %s
+
+@keep_this = internal global i32 2, align 4
+@keep_this2 = internal global i32 3, align 4
+@llvm.used = appending global [1 x i8*] [i8* bitcast (i32* @keep_this to i8*)], section "llvm.metadata"
+@llvm.compiler.used = appending global [1 x i8*] [i8* bitcast (i32* @keep_this2 to i8*)], section "llvm.metadata"
+
+; CHECK-NOT: llvm.metadata
+; CHECK-NOT: llvm.used
+; CHECK-NOT: llvm.compiler.used
+
+; CHECK:    .lglobl keep_this
+; CHECK:  keep_this:
+; CHECK:    .lglobl keep_this2
+; CHECK:  keep_this2:
+
+; CHECK-NOT: llvm.metadata
+; CHECK-NOT: llvm.used
+; CHECK-NOT: llvm.compiler.used

From 48c948abeb7dd8a2d022749d1cc1561ddc45d8dc Mon Sep 17 00:00:00 2001
From: Vy Nguyen <vyng@google.com>
Date: Fri, 17 Jul 2020 19:08:48 -0400
Subject: [PATCH 0211/1035] [llvm-exegesis] Check perf_branch_entry for field
 cycles

Summary: Follow up to breakages reported in D77422

Reviewers: ondrasej, gchatelet

Tags: #llvm

Differential Revision: https://reviews.llvm.org/D84076
---
 llvm/cmake/modules/FindLibpfm.cmake     | 14 ++++++++++++++
 llvm/include/llvm/Config/config.h.cmake |  3 +++
 2 files changed, 17 insertions(+)

diff --git a/llvm/cmake/modules/FindLibpfm.cmake b/llvm/cmake/modules/FindLibpfm.cmake
index 202bb030e3803..38cf3af3953d9 100644
--- a/llvm/cmake/modules/FindLibpfm.cmake
+++ b/llvm/cmake/modules/FindLibpfm.cmake
@@ -7,6 +7,7 @@
 
 include(CheckIncludeFile)
 include(CheckLibraryExists)
+include(CheckCXXSourceCompiles)
 
 if (LLVM_ENABLE_LIBPFM)
   check_library_exists(pfm pfm_initialize "" HAVE_LIBPFM_INITIALIZE)
@@ -16,6 +17,19 @@ if (LLVM_ENABLE_LIBPFM)
     check_include_file(perfmon/pfmlib_perf_event.h HAVE_PERFMON_PFMLIB_PERF_EVENT_H)
     if(HAVE_PERFMON_PERF_EVENT_H AND HAVE_PERFMON_PFMLIB_H AND HAVE_PERFMON_PFMLIB_PERF_EVENT_H)
       set(HAVE_LIBPFM 1)
+      # Check to see if perf_branch_entry has the field 'cycles'.
+      # We couldn't use CheckStructHasMember here because 'cycles' is a bit field which is not
+      # supported by CheckStructHasMember.
+      CHECK_CXX_SOURCE_COMPILES("
+        #include <perfmon/perf_event.h>
+        int main() {
+          perf_branch_entry entry;
+          entry.cycles = 2;
+          return 0;
+        }" COMPILE_WITH_CYCLES)
+      if(COMPILE_WITH_CYCLES)
+        set(LIBPFM_HAS_FIELD_CYCLES 1)
+      endif()
     endif()
   endif()
 endif()
diff --git a/llvm/include/llvm/Config/config.h.cmake b/llvm/include/llvm/Config/config.h.cmake
index 298fab318c10d..4d76b27df6b61 100644
--- a/llvm/include/llvm/Config/config.h.cmake
+++ b/llvm/include/llvm/Config/config.h.cmake
@@ -97,6 +97,9 @@
 /* Define to 1 if you have the `pfm' library (-lpfm). */
 #cmakedefine HAVE_LIBPFM ${HAVE_LIBPFM}
 
+/* Define to 1 if the `perf_branch_entry' struct has field cycles. */
+#cmakedefine LIBPFM_HAS_FIELD_CYCLES ${LIBPFM_HAS_FIELD_CYCLES}
+
 /* Define to 1 if you have the `psapi' library (-lpsapi). */
 #cmakedefine HAVE_LIBPSAPI ${HAVE_LIBPSAPI}
 

From a52aea0ba624fcf46602bff8463b7b831e87ba55 Mon Sep 17 00:00:00 2001
From: Logan Smith <logan.r.smith0@gmail.com>
Date: Mon, 27 Jul 2020 08:37:01 -0700
Subject: [PATCH 0212/1035] Use INTERFACE_COMPILE_OPTIONS to disable
 -Wsuggest-override for any target that links to gtest

This cleans up several CMakeLists.txt's where -Wno-suggest-override was manually specified. These test targets now inherit this flag from the gtest target.

Some unittests CMakeLists.txt's, in particular Flang and LLDB, are not touched by this patch. Flang manually adds the gtest sources itself in some configurations, rather than linking to LLVM's gtest target, so this fix would be insufficient to cover those cases. Similarly, LLDB has subdirectories that manually add the gtest headers to their include path without linking to the gtest target, so those subdirectories still need -Wno-suggest-override to be manually specified to compile without warnings.

Differential Revision: https://reviews.llvm.org/D84554
---
 clang-tools-extra/clangd/unittests/CMakeLists.txt |  4 ----
 clang-tools-extra/unittests/CMakeLists.txt        |  4 ----
 clang/unittests/CMakeLists.txt                    |  4 ----
 lld/unittests/CMakeLists.txt                      |  4 ----
 llvm/lib/Testing/Support/CMakeLists.txt           |  4 ----
 llvm/unittests/CMakeLists.txt                     |  4 ----
 llvm/utils/unittest/CMakeLists.txt                | 11 ++++++++---
 mlir/unittests/CMakeLists.txt                     |  4 ----
 polly/unittests/CMakeLists.txt                    |  4 ----
 9 files changed, 8 insertions(+), 35 deletions(-)

diff --git a/clang-tools-extra/clangd/unittests/CMakeLists.txt b/clang-tools-extra/clangd/unittests/CMakeLists.txt
index 8ede92c16f7ad..c25e2b7f81037 100644
--- a/clang-tools-extra/clangd/unittests/CMakeLists.txt
+++ b/clang-tools-extra/clangd/unittests/CMakeLists.txt
@@ -13,10 +13,6 @@ include_directories(
   ${CLANGD_BINARY_DIR}
   )
 
-if (CXX_SUPPORTS_SUGGEST_OVERRIDE_FLAG)
-  add_compile_options("-Wno-suggest-override")
-endif()
-
 if(CLANG_BUILT_STANDALONE)
   # LLVMTestingSupport library is needed for clangd tests.
   if (EXISTS ${LLVM_MAIN_SRC_DIR}/lib/Testing/Support
diff --git a/clang-tools-extra/unittests/CMakeLists.txt b/clang-tools-extra/unittests/CMakeLists.txt
index 72abe0fa6d0c1..086a68e638307 100644
--- a/clang-tools-extra/unittests/CMakeLists.txt
+++ b/clang-tools-extra/unittests/CMakeLists.txt
@@ -5,10 +5,6 @@ function(add_extra_unittest test_dirname)
   add_unittest(ExtraToolsUnitTests ${test_dirname} ${ARGN})
 endfunction()
 
-if (CXX_SUPPORTS_SUGGEST_OVERRIDE_FLAG)
-  add_compile_options("-Wno-suggest-override")
-endif()
-
 add_subdirectory(clang-apply-replacements)
 add_subdirectory(clang-change-namespace)
 add_subdirectory(clang-doc)
diff --git a/clang/unittests/CMakeLists.txt b/clang/unittests/CMakeLists.txt
index 9a52b9fb02620..4c222e24599f0 100644
--- a/clang/unittests/CMakeLists.txt
+++ b/clang/unittests/CMakeLists.txt
@@ -1,10 +1,6 @@
 add_custom_target(ClangUnitTests)
 set_target_properties(ClangUnitTests PROPERTIES FOLDER "Clang tests")
 
-if (CXX_SUPPORTS_SUGGEST_OVERRIDE_FLAG)
-  add_compile_options("-Wno-suggest-override")
-endif()
-
 if(CLANG_BUILT_STANDALONE)
   # LLVMTestingSupport library is needed for some of the unittests.
   if (EXISTS ${LLVM_MAIN_SRC_DIR}/lib/Testing/Support
diff --git a/lld/unittests/CMakeLists.txt b/lld/unittests/CMakeLists.txt
index 88cb85a084017..84d35d43f4e87 100644
--- a/lld/unittests/CMakeLists.txt
+++ b/lld/unittests/CMakeLists.txt
@@ -12,9 +12,5 @@ function(add_lld_unittest test_dirname)
   target_link_libraries(${test_dirname} ${LLVM_COMMON_LIBS})
 endfunction()
 
-if (CXX_SUPPORTS_SUGGEST_OVERRIDE_FLAG)
-  add_compile_options("-Wno-suggest-override")
-endif()
-
 add_subdirectory(DriverTests)
 add_subdirectory(MachOTests)
diff --git a/llvm/lib/Testing/Support/CMakeLists.txt b/llvm/lib/Testing/Support/CMakeLists.txt
index 595221a105cd3..fe460aeefc91f 100644
--- a/llvm/lib/Testing/Support/CMakeLists.txt
+++ b/llvm/lib/Testing/Support/CMakeLists.txt
@@ -1,10 +1,6 @@
 add_definitions(-DGTEST_LANG_CXX11=1)
 add_definitions(-DGTEST_HAS_TR1_TUPLE=0)
 
-if (CXX_SUPPORTS_SUGGEST_OVERRIDE_FLAG)
-  add_compile_options("-Wno-suggest-override")
-endif()
-
 add_llvm_library(LLVMTestingSupport
   Annotations.cpp
   Error.cpp
diff --git a/llvm/unittests/CMakeLists.txt b/llvm/unittests/CMakeLists.txt
index e90f07448c100..d7dbaeaa32fe8 100644
--- a/llvm/unittests/CMakeLists.txt
+++ b/llvm/unittests/CMakeLists.txt
@@ -14,10 +14,6 @@ function(add_llvm_target_unittest test_dir_name)
   add_llvm_unittest(${test_dir_name} DISABLE_LLVM_LINK_LLVM_DYLIB ${ARGN})
 endfunction()
 
-if (CXX_SUPPORTS_SUGGEST_OVERRIDE_FLAG)
-  add_compile_options("-Wno-suggest-override")
-endif()
-
 add_subdirectory(ADT)
 add_subdirectory(Analysis)
 add_subdirectory(AsmParser)
diff --git a/llvm/utils/unittest/CMakeLists.txt b/llvm/utils/unittest/CMakeLists.txt
index 36761a60d9f72..bcae36fa150d1 100644
--- a/llvm/utils/unittest/CMakeLists.txt
+++ b/llvm/utils/unittest/CMakeLists.txt
@@ -43,9 +43,6 @@ endif()
 if(CXX_SUPPORTS_COVERED_SWITCH_DEFAULT_FLAG)
   add_definitions("-Wno-covered-switch-default")
 endif()
-if(CXX_SUPPORTS_SUGGEST_OVERRIDE_FLAG)
-  add_definitions("-Wno-suggest-override")
-endif()
 
 set(LLVM_REQUIRES_RTTI 1)
 add_definitions( -DGTEST_HAS_RTTI=0 )
@@ -73,6 +70,14 @@ add_llvm_library(gtest
   BUILDTREE_ONLY
 )
 
+# The googletest and googlemock sources don't presently use the 'override'
+# keyword, which leads to lots of warnings from -Wsuggest-override. Disable
+# that warning here for any targets that link to gtest.
+if(CXX_SUPPORTS_SUGGEST_OVERRIDE_FLAG)
+  add_definitions("-Wno-suggest-override")
+  set_target_properties(gtest PROPERTIES INTERFACE_COMPILE_OPTIONS "-Wno-suggest-override")
+endif()
+
 add_subdirectory(UnitTestMain)
 
 # When LLVM_LINK_LLVM_DYLIB is enabled, libLLVM.so is added to the interface
diff --git a/mlir/unittests/CMakeLists.txt b/mlir/unittests/CMakeLists.txt
index 1dc07413a8850..851092c5b56a4 100644
--- a/mlir/unittests/CMakeLists.txt
+++ b/mlir/unittests/CMakeLists.txt
@@ -5,10 +5,6 @@ function(add_mlir_unittest test_dirname)
   add_unittest(MLIRUnitTests ${test_dirname} ${ARGN})
 endfunction()
 
-if (CXX_SUPPORTS_SUGGEST_OVERRIDE_FLAG)
-  add_compile_options("-Wno-suggest-override")
-endif()
-
 add_subdirectory(Analysis)
 add_subdirectory(Dialect)
 add_subdirectory(IR)
diff --git a/polly/unittests/CMakeLists.txt b/polly/unittests/CMakeLists.txt
index 1a6881fde3fd0..fac70383de948 100644
--- a/polly/unittests/CMakeLists.txt
+++ b/polly/unittests/CMakeLists.txt
@@ -19,10 +19,6 @@ function(add_polly_unittest test_name)
   target_link_libraries(${test_name} PRIVATE Polly)
 endfunction()
 
-if (CXX_SUPPORTS_SUGGEST_OVERRIDE_FLAG)
-  add_compile_options("-Wno-suggest-override")
-endif()
-
 add_subdirectory(Isl)
 add_subdirectory(Flatten)
 add_subdirectory(DeLICM)

From 88ce9f9b441ecbe2798f20c33c67abbfc4863b08 Mon Sep 17 00:00:00 2001
From: Jon Roelofs <jonathan_roelofs@apple.com>
Date: Wed, 22 Jul 2020 16:13:20 -0600
Subject: [PATCH 0213/1035] [TableGen][CGS] Print better errors on overlapping
 InstRW

Differential Revision: https://reviews.llvm.org/D83588
---
 llvm/include/llvm/TableGen/Error.h            |  2 +
 llvm/lib/TableGen/Error.cpp                   |  7 ++++
 .../CodeGenSchedule-duplicate-instrw.td       | 21 ++++++++++
 llvm/utils/TableGen/CodeGenSchedule.cpp       | 38 +++++++++----------
 4 files changed, 48 insertions(+), 20 deletions(-)
 create mode 100644 llvm/test/TableGen/CodeGenSchedule-duplicate-instrw.td

diff --git a/llvm/include/llvm/TableGen/Error.h b/llvm/include/llvm/TableGen/Error.h
index cf990427f5770..1eed622ab393f 100644
--- a/llvm/include/llvm/TableGen/Error.h
+++ b/llvm/include/llvm/TableGen/Error.h
@@ -20,6 +20,8 @@ namespace llvm {
 
 void PrintNote(const Twine &Msg);
 void PrintNote(ArrayRef<SMLoc> NoteLoc, const Twine &Msg);
+LLVM_ATTRIBUTE_NORETURN void PrintFatalNote(ArrayRef<SMLoc> ErrorLoc,
+                                            const Twine &Msg);
 
 void PrintWarning(ArrayRef<SMLoc> WarningLoc, const Twine &Msg);
 void PrintWarning(const char *Loc, const Twine &Msg);
diff --git a/llvm/lib/TableGen/Error.cpp b/llvm/lib/TableGen/Error.cpp
index 54b063cb4f8d2..1dfba9fb6b5dc 100644
--- a/llvm/lib/TableGen/Error.cpp
+++ b/llvm/lib/TableGen/Error.cpp
@@ -45,6 +45,13 @@ void PrintNote(ArrayRef<SMLoc> NoteLoc, const Twine &Msg) {
   PrintMessage(NoteLoc, SourceMgr::DK_Note, Msg);
 }
 
+void PrintFatalNote(ArrayRef<SMLoc> NoteLoc, const Twine &Msg) {
+  PrintNote(NoteLoc, Msg);
+  // The following call runs the file cleanup handlers.
+  sys::RunInterruptHandlers();
+  std::exit(1);
+}
+
 void PrintWarning(ArrayRef<SMLoc> WarningLoc, const Twine &Msg) {
   PrintMessage(WarningLoc, SourceMgr::DK_Warning, Msg);
 }
diff --git a/llvm/test/TableGen/CodeGenSchedule-duplicate-instrw.td b/llvm/test/TableGen/CodeGenSchedule-duplicate-instrw.td
new file mode 100644
index 0000000000000..e8b13d473ec8c
--- /dev/null
+++ b/llvm/test/TableGen/CodeGenSchedule-duplicate-instrw.td
@@ -0,0 +1,21 @@
+// RUN: not llvm-tblgen --gen-subtarget -I %p/../../include -I %p/Common %s -o - 2>&1 | FileCheck %s
+
+include "llvm/Target/Target.td"
+
+def FakeTarget : Target { }
+
+def FakeModel : SchedMachineModel { }
+
+def WriteA : SchedWrite;
+def WriteB : SchedWrite;
+
+let SchedModel = NoSchedModel in {
+  def : InstRW<[WriteA], (instrs COPY)>;
+
+  def : InstRW<[WriteB], (instrs COPY)>;
+// CHECK: [[@LINE-1]]:3: error: Overlapping InstRW definition for "COPY" also matches previous "(instrs COPY)".
+// CHECK-NEXT: def : InstRW<[WriteB], (instrs COPY)>;
+
+// CHECK: [[@LINE-6]]:3: note: Previous match was here.
+// CHECK-NEXT: def : InstRW<[WriteA], (instrs COPY)>;
+}
\ No newline at end of file
diff --git a/llvm/utils/TableGen/CodeGenSchedule.cpp b/llvm/utils/TableGen/CodeGenSchedule.cpp
index 67583c736cd2f..31ef38c33b56e 100644
--- a/llvm/utils/TableGen/CodeGenSchedule.cpp
+++ b/llvm/utils/TableGen/CodeGenSchedule.cpp
@@ -248,8 +248,7 @@ void CodeGenSchedModels::checkSTIPredicates() const {
     }
 
     PrintError(R->getLoc(), "STIPredicate " + Name + " multiply declared.");
-    PrintNote(It->second->getLoc(), "Previous declaration was here.");
-    PrintFatalError(R->getLoc(), "Invalid STIPredicateDecl found.");
+    PrintFatalNote(It->second->getLoc(), "Previous declaration was here.");
   }
 
   // Disallow InstructionEquivalenceClasses with an empty instruction list.
@@ -454,10 +453,8 @@ void CodeGenSchedModels::checkMCInstPredicates() const {
 
     PrintError(TIIPred->getLoc(),
                "TIIPredicate " + Name + " is multiply defined.");
-    PrintNote(It->second->getLoc(),
-              " Previous definition of " + Name + " was here.");
-    PrintFatalError(TIIPred->getLoc(),
-                    "Found conflicting definitions of TIIPredicate.");
+    PrintFatalNote(It->second->getLoc(),
+                   " Previous definition of " + Name + " was here.");
   }
 }
 
@@ -1083,13 +1080,14 @@ void CodeGenSchedModels::createInstRWClass(Record *InstRWDef) {
             if (RWD->getValueAsDef("SchedModel") == RWModelDef &&
                 RWModelDef->getValueAsBit("FullInstRWOverlapCheck")) {
               assert(!InstDefs.empty()); // Checked at function start.
-              PrintFatalError
-                  (InstRWDef->getLoc(),
-                   "Overlapping InstRW definition for \"" +
-                   InstDefs.front()->getName() +
-                   "\" also matches previous \"" +
-                   RWD->getValue("Instrs")->getValue()->getAsString() +
-                   "\".");
+              PrintError(
+                  InstRWDef->getLoc(),
+                  "Overlapping InstRW definition for \"" +
+                      InstDefs.front()->getName() +
+                      "\" also matches previous \"" +
+                      RWD->getValue("Instrs")->getValue()->getAsString() +
+                      "\".");
+              PrintFatalNote(RWD->getLoc(), "Previous match was here.");
             }
           }
           LLVM_DEBUG(dbgs() << "InstRW: Reuse SC " << OldSCIdx << ":"
@@ -1118,13 +1116,13 @@ void CodeGenSchedModels::createInstRWClass(Record *InstRWDef) {
       for (Record *OldRWDef : SchedClasses[OldSCIdx].InstRWs) {
         if (OldRWDef->getValueAsDef("SchedModel") == RWModelDef) {
           assert(!InstDefs.empty()); // Checked at function start.
-          PrintFatalError
-              (InstRWDef->getLoc(),
-               "Overlapping InstRW definition for \"" +
-               InstDefs.front()->getName() +
-               "\" also matches previous \"" +
-               OldRWDef->getValue("Instrs")->getValue()->getAsString() +
-               "\".");
+          PrintError(
+              InstRWDef->getLoc(),
+              "Overlapping InstRW definition for \"" +
+                  InstDefs.front()->getName() + "\" also matches previous \"" +
+                  OldRWDef->getValue("Instrs")->getValue()->getAsString() +
+                  "\".");
+          PrintFatalNote(OldRWDef->getLoc(), "Previous match was here.");
         }
         assert(OldRWDef != InstRWDef &&
                "SchedClass has duplicate InstRW def");

From 52dd18ab1d3848f0156e4033bba2b65c3ae18a9e Mon Sep 17 00:00:00 2001
From: Nico Weber <thakis@chromium.org>
Date: Mon, 27 Jul 2020 12:30:09 -0400
Subject: [PATCH 0214/1035] [gn build] (manually) merge 48c948abeb7

---
 llvm/utils/gn/secondary/llvm/include/llvm/Config/BUILD.gn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/utils/gn/secondary/llvm/include/llvm/Config/BUILD.gn b/llvm/utils/gn/secondary/llvm/include/llvm/Config/BUILD.gn
index 72b5796cd9aaa..94f0a66ecb182 100644
--- a/llvm/utils/gn/secondary/llvm/include/llvm/Config/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/include/llvm/Config/BUILD.gn
@@ -113,6 +113,7 @@ write_cmake_config("config") {
     "HAVE____CHKSTK=",
     "HAVE____CHKSTK_MS=",
     "HOST_LINK_VERSION=",
+    "LIBPFM_HAS_FIELD_CYCLES=",
     "LLVM_TARGET_TRIPLE_ENV=",
     "LLVM_VERSION_INFO=",
     "LLVM_VERSION_PRINTER_SHOW_HOST_TARGET_INFO=1",

From ee7caa75939afb75547c00744c5df4d04d45e517 Mon Sep 17 00:00:00 2001
From: Vy Nguyen <vyng@google.com>
Date: Mon, 27 Jul 2020 12:38:05 -0400
Subject: [PATCH 0215/1035] Reland [llvm-exegesis] Add benchmark latency option
 on X86 that uses LBR for more precise measurements.

        Starting with Skylake, the LBR contains the precise number of cycles between the two
        consecutive branches.
        Making use of this will hopefully make the measurements more precise than the
        existing methods of using RDTSC.

                Differential Revision: https://reviews.llvm.org/D77422

New change: check for existence of field `cycles` in perf_branch_entry before enabling this mode.
This should prevent compilation errors when building for older kernel whose headers don't support it.
---
 llvm/docs/CommandGuide/llvm-exegesis.rst      |  20 +-
 .../llvm-exegesis/X86/lbr/Inputs/mov_add.att  |   4 +
 .../tools/llvm-exegesis/X86/lbr/lit.local.cfg |  31 +++
 .../tools/llvm-exegesis/X86/lbr/mov-add.s     |  18 ++
 .../llvm-exegesis/lib/BenchmarkRunner.cpp     |   5 +-
 llvm/tools/llvm-exegesis/lib/PerfHelper.cpp   |   6 +-
 llvm/tools/llvm-exegesis/lib/PerfHelper.h     |  15 +-
 .../llvm-exegesis/lib/X86/CMakeLists.txt      |   1 +
 llvm/tools/llvm-exegesis/lib/X86/Target.cpp   |  47 ++++
 .../llvm-exegesis/lib/X86/X86Counter.cpp      | 212 ++++++++++++++++++
 llvm/tools/llvm-exegesis/lib/X86/X86Counter.h |  55 +++++
 llvm/tools/llvm-exegesis/llvm-exegesis.cpp    |  13 ++
 12 files changed, 415 insertions(+), 12 deletions(-)
 create mode 100644 llvm/test/tools/llvm-exegesis/X86/lbr/Inputs/mov_add.att
 create mode 100644 llvm/test/tools/llvm-exegesis/X86/lbr/lit.local.cfg
 create mode 100644 llvm/test/tools/llvm-exegesis/X86/lbr/mov-add.s
 create mode 100644 llvm/tools/llvm-exegesis/lib/X86/X86Counter.cpp
 create mode 100644 llvm/tools/llvm-exegesis/lib/X86/X86Counter.h

diff --git a/llvm/docs/CommandGuide/llvm-exegesis.rst b/llvm/docs/CommandGuide/llvm-exegesis.rst
index 321cdf5a6dab1..8cc1a237e9969 100644
--- a/llvm/docs/CommandGuide/llvm-exegesis.rst
+++ b/llvm/docs/CommandGuide/llvm-exegesis.rst
@@ -192,10 +192,24 @@ OPTIONS
 
 .. option:: -mode=[latency|uops|inverse_throughput|analysis]
 
- Specify the run mode. Note that if you pick `analysis` mode, you also need
- to specify at least one of the `-analysis-clusters-output-file=` and
- `-analysis-inconsistencies-output-file=`.
+ Specify the run mode. Note that some modes have additional requirements and options.
 
+ `latency` mode can be  make use of either RDTSC or LBR.
+ `latency[LBR]` is only available on X86 (at least `Skylake`).
+  To run in this mode, a positive value  must be specified for `x86-lbr-sample-period` and `--repetition-mode=loop`
+
+ In `analysis` mode, you also need to specify at least one of the
+ `-analysis-clusters-output-file=` and `-analysis-inconsistencies-output-file=`.
+
+.. option:: -x86-lbr-sample-period=<nBranches/sample>
+
+  Specify the LBR sampling period - how many branches before we take a sample.
+  When a positive value is specified for this option and when the mode is `latency`,
+  we will use LBRs for measuring.
+  On choosing the "right" sampling period, a small value is preferred, but throttling
+  could occur if the sampling is too frequent. A prime number should be used to
+  avoid consistently skipping certain blocks.
+  
 .. option:: -repetition-mode=[duplicate|loop|min]
 
  Specify the repetition mode. `duplicate` will create a large, straight line
diff --git a/llvm/test/tools/llvm-exegesis/X86/lbr/Inputs/mov_add.att b/llvm/test/tools/llvm-exegesis/X86/lbr/Inputs/mov_add.att
new file mode 100644
index 0000000000000..8f85b395e7319
--- /dev/null
+++ b/llvm/test/tools/llvm-exegesis/X86/lbr/Inputs/mov_add.att
@@ -0,0 +1,4 @@
+# LLVM-EXEGESIS-LIVEIN RDI
+# LLVM-EXEGESIS-DEFREG XMM1 42
+movq $2, %rdi
+addq $0x10, %rdi
\ No newline at end of file
diff --git a/llvm/test/tools/llvm-exegesis/X86/lbr/lit.local.cfg b/llvm/test/tools/llvm-exegesis/X86/lbr/lit.local.cfg
new file mode 100644
index 0000000000000..431967c1ec9b0
--- /dev/null
+++ b/llvm/test/tools/llvm-exegesis/X86/lbr/lit.local.cfg
@@ -0,0 +1,31 @@
+import subprocess
+import lit.util
+
+if not ('X86' in config.root.targets):
+    # We need support for X86.
+    config.unsupported = True
+
+elif not ('x86_64' in config.root.host_triple):
+    # We need to be running on an X86 host.
+    config.unsupported = True
+    
+else:    
+    # We need libpfm to be installed and the host to be at least skylake.
+    llvm_exegesis_exe = lit.util.which('llvm-exegesis', config.llvm_tools_dir)
+    if not llvm_exegesis_exe:
+        print('llvm-exegesis not found')
+        config.unsupported = True
+    else:
+      try:
+          with open(os.devnull, 'w') as quiet:
+              check_llvm_exegesis_uops_result = subprocess.call(
+                [llvm_exegesis_exe, '-allowed-host-cpu', 'skylake', '-allowed-host-cpu', 'skylake-avx512', '-mode', 'uops', '-snippets-file', '/dev/null'], stdout=quiet, stderr=quiet)
+              check_llvm_exegesis_latency_result = subprocess.call(
+                [llvm_exegesis_exe, '-allowed-host-cpu', 'skylake', '-allowed-host-cpu', 'skylake-avx512', '-mode', 'latency', '-snippets-file', '/dev/null'], stdout=quiet, stderr=quiet)
+      except OSError:
+          print('could not exec llvm-exegesis')
+          config.unsupported = True
+      if not check_llvm_exegesis_uops_result == 0:
+        config.unsupported = True
+      if not check_llvm_exegesis_latency_result == 0:
+        config.unsupported = True
diff --git a/llvm/test/tools/llvm-exegesis/X86/lbr/mov-add.s b/llvm/test/tools/llvm-exegesis/X86/lbr/mov-add.s
new file mode 100644
index 0000000000000..5f72e8f99b30d
--- /dev/null
+++ b/llvm/test/tools/llvm-exegesis/X86/lbr/mov-add.s
@@ -0,0 +1,18 @@
+# RUN: llvm-exegesis -mode=latency --repetition-mode=loop --x86-lbr-sample-period=521 --snippets-file=%p/Inputs/mov_add.att
+
+
+CHECK:      ---
+CHECK-NEXT: mode: latency
+CHECK-NEXT: key:
+CHECK-NEXT:   instructions:
+CHECK-NEXT:     'MOV64ri32 RDI i_0x2'
+CHECK-NEXT:     'ADD64ri8 RDI RDI i_0x10'
+CHECK-NEXT: config: ''
+CHECK-NEXT: {{.*}}
+CHECK-NEXT: {{.*}}
+CHECK-NEXT: {{.*}}
+CHECK-NEXT: {{.*}}
+CHECK-NEXT: num_repetitions: 10000
+CHECK-NEXT: measurements:
+CHECK-NEXT: {{.*}} value: 0.0001, per_snippet_value: 0.0002 {{.*}}
+CHECK-LAST: ...
diff --git a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp
index bdef8f8a89189..f015147b0fc2f 100644
--- a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp
+++ b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp
@@ -55,7 +55,6 @@ class FunctionExecutorImpl : public BenchmarkRunner::FunctionExecutor {
   static void
   accumulateCounterValues(const llvm::SmallVector<int64_t, 4> &NewValues,
                           llvm::SmallVector<int64_t, 4> *Result) {
-
     const size_t NumValues = std::max(NewValues.size(), Result->size());
     if (NumValues > Result->size())
       Result->resize(NumValues, 0);
@@ -106,10 +105,10 @@ class FunctionExecutorImpl : public BenchmarkRunner::FunctionExecutor {
         if (Crashed)
           return make_error<SnippetCrash>("snippet crashed while running");
       }
-      auto ValueOrError = Counter->readOrError();
+
+      auto ValueOrError = Counter->readOrError(Function.getFunctionBytes());
       if (!ValueOrError)
         return ValueOrError.takeError();
-
       accumulateCounterValues(ValueOrError.get(), &CounterValues);
     }
     return CounterValues;
diff --git a/llvm/tools/llvm-exegesis/lib/PerfHelper.cpp b/llvm/tools/llvm-exegesis/lib/PerfHelper.cpp
index cba4846709e80..58e1f4dc2a2b2 100644
--- a/llvm/tools/llvm-exegesis/lib/PerfHelper.cpp
+++ b/llvm/tools/llvm-exegesis/lib/PerfHelper.cpp
@@ -128,7 +128,8 @@ int64_t Counter::read() const {
   return -1;
 }
 
-llvm::Expected<llvm::SmallVector<int64_t, 4>> Counter::readOrError() const {
+llvm::Expected<llvm::SmallVector<int64_t, 4>>
+Counter::readOrError(StringRef /*unused*/) const {
   int64_t Count = 0;
   ssize_t ReadSize = ::read(FileDescriptor, &Count, sizeof(Count));
   if (ReadSize != sizeof(Count))
@@ -152,7 +153,8 @@ void Counter::stop() {}
 
 int64_t Counter::read() const { return 42; }
 
-llvm::Expected<llvm::SmallVector<int64_t, 4>> Counter::readOrError() const {
+llvm::Expected<llvm::SmallVector<int64_t, 4>>
+Counter::readOrError(StringRef /*unused*/) const {
   return llvm::make_error<llvm::StringError>("Not implemented",
                                              llvm::errc::io_error);
 }
diff --git a/llvm/tools/llvm-exegesis/lib/PerfHelper.h b/llvm/tools/llvm-exegesis/lib/PerfHelper.h
index d41b090e85f17..19a35595c9af7 100644
--- a/llvm/tools/llvm-exegesis/lib/PerfHelper.h
+++ b/llvm/tools/llvm-exegesis/lib/PerfHelper.h
@@ -59,8 +59,9 @@ class PerfEvent {
   // e.g. "snb_ep::INSTRUCTION_RETIRED:e=0:i=0:c=0:t=0:u=1:k=0:mg=0:mh=1"
   StringRef getPfmEventString() const;
 
-private:
-  const std::string EventString;
+protected:
+  PerfEvent() = default;
+  std::string EventString;
   std::string FullQualifiedEventString;
   perf_event_attr *Attr;
 };
@@ -87,11 +88,17 @@ class Counter {
   int64_t read() const;
 
   /// Returns the current value of the counter or error if it cannot be read.
-  virtual llvm::Expected<llvm::SmallVector<int64_t, 4>> readOrError() const;
+  /// FunctionBytes: The benchmark function being executed.
+  /// This is used to filter out the measurements to ensure they are only
+  /// within the benchmarked code.
+  /// If empty (or not specified), then no filtering will be done.
+  /// Not all counters choose to use this.
+  virtual llvm::Expected<llvm::SmallVector<int64_t, 4>>
+  readOrError(StringRef FunctionBytes = StringRef()) const;
 
   virtual int numValues() const;
 
-private:
+protected:
   PerfEvent Event;
 #ifdef HAVE_LIBPFM
   int FileDescriptor = -1;
diff --git a/llvm/tools/llvm-exegesis/lib/X86/CMakeLists.txt b/llvm/tools/llvm-exegesis/lib/X86/CMakeLists.txt
index 912877dd6ed1f..ce3bbd5908a83 100644
--- a/llvm/tools/llvm-exegesis/lib/X86/CMakeLists.txt
+++ b/llvm/tools/llvm-exegesis/lib/X86/CMakeLists.txt
@@ -6,6 +6,7 @@ include_directories(
 add_library(LLVMExegesisX86
   STATIC
   Target.cpp
+  X86Counter.cpp
   )
 
 llvm_update_compile_flags(LLVMExegesisX86)
diff --git a/llvm/tools/llvm-exegesis/lib/X86/Target.cpp b/llvm/tools/llvm-exegesis/lib/X86/Target.cpp
index 7a84f936e0d0e..9f045fa11aa24 100644
--- a/llvm/tools/llvm-exegesis/lib/X86/Target.cpp
+++ b/llvm/tools/llvm-exegesis/lib/X86/Target.cpp
@@ -14,15 +14,40 @@
 #include "MCTargetDesc/X86BaseInfo.h"
 #include "MCTargetDesc/X86MCTargetDesc.h"
 #include "X86.h"
+#include "X86Counter.h"
 #include "X86RegisterInfo.h"
 #include "X86Subtarget.h"
 #include "llvm/ADT/Sequence.h"
 #include "llvm/MC/MCInstBuilder.h"
+#include "llvm/Support/Errc.h"
+#include "llvm/Support/Error.h"
 #include "llvm/Support/FormatVariadic.h"
 
+#include <memory>
+#include <string>
+#include <vector>
+
 namespace llvm {
 namespace exegesis {
 
+static cl::OptionCategory
+    BenchmarkOptions("llvm-exegesis benchmark x86-options");
+
+// If a positive value is specified, we are going to use the LBR in
+// latency-mode.
+//
+// Note:
+//  -  A small value is preferred, but too low a value could result in
+//     throttling.
+//  -  A prime number is preferred to avoid always skipping certain blocks.
+//
+static cl::opt<unsigned> LbrSamplingPeriod(
+    "x86-lbr-sample-period",
+    cl::desc("The sample period (nbranches/sample), used for LBR sampling"),
+    cl::cat(BenchmarkOptions), cl::init(0));
+
+// FIXME: Validates that repetition-mode is loop if LBR is requested.
+
 // Returns a non-null reason if we cannot handle the memory references in this
 // instruction.
 static const char *isInvalidMemoryInstr(const Instruction &Instr) {
@@ -568,10 +593,32 @@ void ConstantInliner::initStack(unsigned Bytes) {
 #include "X86GenExegesis.inc"
 
 namespace {
+
 class ExegesisX86Target : public ExegesisTarget {
 public:
   ExegesisX86Target() : ExegesisTarget(X86CpuPfmCounters) {}
 
+  Expected<std::unique_ptr<pfm::Counter>>
+  createCounter(StringRef CounterName, const LLVMState &State) const override {
+    // If LbrSamplingPeriod was provided, then ignore the
+    // CounterName because we only have one for LBR.
+    if (LbrSamplingPeriod > 0) {
+      // Can't use LBR without HAVE_LIBPFM, LIBPFM_HAS_FIELD_CYCLES, or without
+      // __linux__ (for now)
+#if defined(HAVE_LIBPFM) && defined(LIBPFM_HAS_FIELD_CYCLES) &&                \
+    defined(__linux__)
+      return std::make_unique<X86LbrCounter>(
+          X86LbrPerfEvent(LbrSamplingPeriod));
+#else
+      return llvm::make_error<llvm::StringError>(
+          "LBR counter requested without HAVE_LIBPFM, LIBPFM_HAS_FIELD_CYCLES, "
+          "or running on Linux.",
+          llvm::errc::invalid_argument);
+#endif
+    }
+    return ExegesisTarget::createCounter(CounterName, State);
+  }
+
 private:
   void addTargetSpecificPasses(PassManagerBase &PM) const override;
 
diff --git a/llvm/tools/llvm-exegesis/lib/X86/X86Counter.cpp b/llvm/tools/llvm-exegesis/lib/X86/X86Counter.cpp
new file mode 100644
index 0000000000000..57b493818aaad
--- /dev/null
+++ b/llvm/tools/llvm-exegesis/lib/X86/X86Counter.cpp
@@ -0,0 +1,212 @@
+//===-- X86Counter.cpp ------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "X86Counter.h"
+
+// FIXME: Use appropriate wrappers for poll.h and mman.h
+// to support Windows and remove this linux-only guard.
+#ifdef __linux__
+#include "llvm/Support/Endian.h"
+#include "llvm/Support/Errc.h"
+
+#ifdef HAVE_LIBPFM
+#include "perfmon/perf_event.h"
+#include "perfmon/pfmlib.h"
+#include "perfmon/pfmlib_perf_event.h"
+#endif // HAVE_LIBPFM
+
+#include <atomic>
+#include <cstddef>
+#include <cstdint>
+#include <limits>
+#include <memory>
+#include <vector>
+
+#include <poll.h>
+#include <sys/mman.h>
+#include <unistd.h>
+
+#if defined(HAVE_LIBPFM) && defined(LIBPFM_HAS_FIELD_CYCLES)
+namespace llvm {
+namespace exegesis {
+
+static constexpr size_t kBufferPages = 8;
+static const size_t kDataBufferSize = kBufferPages * getpagesize();
+
+// Waits for the LBR perf events.
+static int pollLbrPerfEvent(const int FileDescriptor) {
+  struct pollfd PollFd;
+  PollFd.fd = FileDescriptor;
+  PollFd.events = POLLIN;
+  PollFd.revents = 0;
+  return poll(&PollFd, 1 /* num of fds */, 10000 /* timeout in ms */);
+}
+
+// Copies the data-buffer into Buf, given the pointer to MMapped.
+static void copyDataBuffer(void *MMappedBuffer, char *Buf, uint64_t Tail,
+                           size_t DataSize) {
+  // First page is reserved for perf_event_mmap_page. Data buffer starts on
+  // the next page.
+  char *Start = reinterpret_cast<char *>(MMappedBuffer) + getpagesize();
+  // The LBR buffer is a cyclic buffer, we copy data to another buffer.
+  uint64_t Offset = Tail % kDataBufferSize;
+  size_t CopySize = kDataBufferSize - Offset;
+  memcpy(Buf, Start + Offset, CopySize);
+  if (CopySize >= DataSize)
+    return;
+
+  memcpy(Buf + CopySize, Start, Offset);
+  return;
+}
+
+// Parses the given data-buffer for stats and fill the CycleArray.
+// If data has been extracted successfully, also modifies the code to jump
+// out the benchmark loop.
+static llvm::Error parseDataBuffer(const char *DataBuf, size_t DataSize,
+                                   const void *From, const void *To,
+                                   llvm::SmallVector<int64_t, 4> *CycleArray) {
+  assert(From != nullptr && To != nullptr);
+  const char *DataPtr = DataBuf;
+  while (DataPtr < DataBuf + DataSize) {
+    struct perf_event_header Header;
+    memcpy(&Header, DataPtr, sizeof(struct perf_event_header));
+    if (Header.type != PERF_RECORD_SAMPLE) {
+      // Ignores non-sample records.
+      DataPtr += Header.size;
+      continue;
+    }
+    DataPtr += sizeof(Header);
+    uint64_t Count = llvm::support::endian::read64(DataPtr, support::native);
+    DataPtr += sizeof(Count);
+
+    struct perf_branch_entry Entry;
+    memcpy(&Entry, DataPtr, sizeof(struct perf_branch_entry));
+
+    // Read the perf_branch_entry array.
+    for (uint64_t i = 0; i < Count; ++i) {
+      const uint64_t BlockStart = From == nullptr
+                                      ? std::numeric_limits<uint64_t>::min()
+                                      : reinterpret_cast<uint64_t>(From);
+      const uint64_t BlockEnd = To == nullptr
+                                    ? std::numeric_limits<uint64_t>::max()
+                                    : reinterpret_cast<uint64_t>(To);
+
+      if (BlockStart <= Entry.from && BlockEnd >= Entry.to)
+        CycleArray->push_back(Entry.cycles);
+
+      if (i == Count - 1)
+        // We've reached the last entry.
+        return llvm::Error::success();
+
+      // Advance to next entry
+      DataPtr += sizeof(Entry);
+      memcpy(&Entry, DataPtr, sizeof(struct perf_branch_entry));
+    }
+  }
+  return llvm::make_error<llvm::StringError>("Unable to parse databuffer.",
+                                             llvm::errc::io_error);
+}
+
+X86LbrPerfEvent::X86LbrPerfEvent(unsigned SamplingPeriod) {
+  assert(SamplingPeriod > 0 && "SamplingPeriod must be positive");
+  EventString = "BR_INST_RETIRED.NEAR_TAKEN";
+  Attr = new perf_event_attr();
+  Attr->size = sizeof(*Attr);
+  Attr->type = PERF_TYPE_RAW;
+  // FIXME This is SKL's encoding. Not sure if it'll change.
+  Attr->config = 0x20c4; // BR_INST_RETIRED.NEAR_TAKEN
+  Attr->sample_type = PERF_SAMPLE_BRANCH_STACK;
+  // Don't need to specify "USER" because we've already excluded HV and Kernel.
+  Attr->branch_sample_type = PERF_SAMPLE_BRANCH_ANY;
+  Attr->sample_period = SamplingPeriod;
+  Attr->wakeup_events = 1; // We need this even when using ioctl REFRESH.
+  Attr->disabled = 1;
+  Attr->exclude_kernel = 1;
+  Attr->exclude_hv = 1;
+  Attr->read_format = PERF_FORMAT_GROUP;
+
+  FullQualifiedEventString = EventString;
+}
+
+X86LbrCounter::X86LbrCounter(pfm::PerfEvent &&NewEvent)
+    : Counter(std::move(NewEvent)) {
+  // First page is reserved for perf_event_mmap_page. Data buffer starts on
+  // the next page, so we allocate one more page.
+  MMappedBuffer = mmap(nullptr, (kBufferPages + 1) * getpagesize(),
+                       PROT_READ | PROT_WRITE, MAP_SHARED, FileDescriptor, 0);
+  if (MMappedBuffer == MAP_FAILED)
+    llvm::errs() << "Failed to mmap buffer.";
+}
+
+X86LbrCounter::~X86LbrCounter() { close(FileDescriptor); }
+
+void X86LbrCounter::start() {
+  ioctl(FileDescriptor, PERF_EVENT_IOC_REFRESH, 1024 /* kMaxPollsPerFd */);
+}
+
+llvm::Expected<llvm::SmallVector<int64_t, 4>>
+X86LbrCounter::readOrError(StringRef FunctionBytes) const {
+  // The max number of time-outs/retries before we give up.
+  static constexpr int kMaxTimeouts = 160;
+
+  // Disable the event before reading
+  ioctl(FileDescriptor, PERF_EVENT_IOC_DISABLE, 0);
+
+  // Parses the LBR buffer and fills CycleArray with the sequence of cycle
+  // counts from the buffer.
+  llvm::SmallVector<int64_t, 4> CycleArray;
+  std::unique_ptr<char[]> DataBuf(new char[kDataBufferSize]);
+  int NumTimeouts = 0;
+  int PollResult = 0;
+
+  // Find the boundary of the function so that we could filter the LBRs
+  // to keep only the relevant records.
+  if (FunctionBytes.empty())
+    return llvm::make_error<llvm::StringError>("Empty function bytes",
+                                               llvm::errc::invalid_argument);
+  const void *From = reinterpret_cast<const void *>(FunctionBytes.data());
+  const void *To = reinterpret_cast<const void *>(FunctionBytes.data() +
+                                                  FunctionBytes.size());
+  while (PollResult <= 0) {
+    PollResult = pollLbrPerfEvent(FileDescriptor);
+    if (PollResult > 0)
+      break;
+    if (PollResult == -1)
+      return llvm::make_error<llvm::StringError>("Cannot poll LBR perf event.",
+                                                 llvm::errc::io_error);
+    if (NumTimeouts++ >= kMaxTimeouts)
+      return llvm::make_error<llvm::StringError>(
+          "LBR polling still timed out after max number of attempts.",
+          llvm::errc::device_or_resource_busy);
+  }
+
+  struct perf_event_mmap_page Page;
+  memcpy(&Page, MMappedBuffer, sizeof(struct perf_event_mmap_page));
+
+  const uint64_t DataTail = Page.data_tail;
+  const uint64_t DataHead = Page.data_head;
+  // We're supposed to use a barrier after reading data_head.
+  std::atomic_thread_fence(std::memory_order_acq_rel);
+  const size_t DataSize = DataHead - DataTail;
+  if (DataSize > kDataBufferSize)
+    return llvm::make_error<llvm::StringError>(
+        "DataSize larger than buffer size.", llvm::errc::invalid_argument);
+
+  copyDataBuffer(MMappedBuffer, DataBuf.get(), DataTail, DataSize);
+  llvm::Error error =
+      parseDataBuffer(DataBuf.get(), DataSize, From, To, &CycleArray);
+  if (!error)
+    return CycleArray;
+  return std::move(error);
+}
+
+} // namespace exegesis
+} // namespace llvm
+
+#endif //  defined(HAVE_LIBPFM) && defined(LIBPFM_HAS_FIELD_CYCLES)
+#endif // __linux__
diff --git a/llvm/tools/llvm-exegesis/lib/X86/X86Counter.h b/llvm/tools/llvm-exegesis/lib/X86/X86Counter.h
new file mode 100644
index 0000000000000..94062012917df
--- /dev/null
+++ b/llvm/tools/llvm-exegesis/lib/X86/X86Counter.h
@@ -0,0 +1,55 @@
+//===-- X86Counter.h --------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Perf counter that reads the LBRs for measuring the benchmarked block's
+/// throughput.
+///
+/// More info at: https://lwn.net/Articles/680985
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_TOOLS_LLVM_EXEGESIS_LIB_X86_X86COUNTER_H
+#define LLVM_TOOLS_LLVM_EXEGESIS_LIB_X86_X86COUNTER_H
+
+#include "../PerfHelper.h"
+#include "llvm/Support/Error.h"
+
+// FIXME: Use appropriate wrappers for poll.h and mman.h
+// to support Windows and remove this linux-only guard.
+#if defined(__linux__) && defined(HAVE_LIBPFM) &&                              \
+    defined(LIBPFM_HAS_FIELD_CYCLES)
+
+namespace llvm {
+namespace exegesis {
+
+class X86LbrPerfEvent : public pfm::PerfEvent {
+public:
+  X86LbrPerfEvent(unsigned SamplingPeriod);
+};
+
+class X86LbrCounter : public pfm::Counter {
+public:
+  explicit X86LbrCounter(pfm::PerfEvent &&Event);
+
+  virtual ~X86LbrCounter();
+
+  void start() override;
+
+  llvm::Expected<llvm::SmallVector<int64_t, 4>>
+  readOrError(StringRef FunctionBytes) const override;
+
+private:
+  void *MMappedBuffer = nullptr;
+};
+
+} // namespace exegesis
+} // namespace llvm
+
+#endif // defined(__linux__) && defined(HAVE_LIBPFM) &&
+       // defined(LIBPFM_HAS_FIELD_CYCLES)
+
+#endif // LLVM_TOOLS_LLVM_EXEGESIS_LIB_X86_X86COUNTER_H
diff --git a/llvm/tools/llvm-exegesis/llvm-exegesis.cpp b/llvm/tools/llvm-exegesis/llvm-exegesis.cpp
index 507015b97472b..8eeda48823859 100644
--- a/llvm/tools/llvm-exegesis/llvm-exegesis.cpp
+++ b/llvm/tools/llvm-exegesis/llvm-exegesis.cpp
@@ -160,6 +160,12 @@ static cl::opt<std::string>
                                       cl::desc(""), cl::cat(AnalysisOptions),
                                       cl::init(""));
 
+static cl::list<std::string>
+    AllowedHostCpus("allowed-host-cpu",
+                    cl::desc("If specified, only run the benchmark if the host "
+                             "CPU matches the names"),
+                    cl::cat(Options), cl::ZeroOrMore);
+
 static cl::opt<bool> AnalysisDisplayUnstableOpcodes(
     "analysis-display-unstable-clusters",
     cl::desc("if there is more than one benchmark for an opcode, said "
@@ -296,6 +302,13 @@ void benchmarkMain() {
 
   const LLVMState State(CpuName);
 
+  llvm::StringRef ActualCpu = State.getTargetMachine().getTargetCPU();
+  for (auto Begin = AllowedHostCpus.begin(); Begin != AllowedHostCpus.end();
+       ++Begin) {
+    if (ActualCpu != *Begin)
+      ExitWithError(llvm::Twine("Unexpected host CPU ").concat(ActualCpu));
+  }
+
   const std::unique_ptr<BenchmarkRunner> Runner =
       ExitOnErr(State.getExegesisTarget().createBenchmarkRunner(
           BenchmarkMode, State, ResultAggMode));

From b6902d977ac07ad093655230dcee0a7d920e47b7 Mon Sep 17 00:00:00 2001
From: Nico Weber <thakis@chromium.org>
Date: Mon, 27 Jul 2020 12:44:47 -0400
Subject: [PATCH 0216/1035] [gn build] Make syncing to single-line source files
 work after 1afd889d0b43

---
 llvm/utils/gn/build/sync_source_lists_from_cmake.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/utils/gn/build/sync_source_lists_from_cmake.py b/llvm/utils/gn/build/sync_source_lists_from_cmake.py
index b88b3b8007712..e0c550ed7085b 100755
--- a/llvm/utils/gn/build/sync_source_lists_from_cmake.py
+++ b/llvm/utils/gn/build/sync_source_lists_from_cmake.py
@@ -27,7 +27,7 @@ def patch_gn_file(gn_file, add, remove):
     with open(gn_file) as f:
         gn_contents = f.read()
 
-    srcs_tok = 'sources = [\n'
+    srcs_tok = 'sources = ['
     tokloc = gn_contents.find(srcs_tok)
 
     if tokloc == -1: raise ValueError(gn_file + ': Failed to find source list')

From e77ff4abb38778904e72a38942cc8f160f661583 Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
Date: Mon, 27 Jul 2020 16:45:49 +0000
Subject: [PATCH 0217/1035] [gn build] Port ee7caa75939

---
 .../gn/secondary/llvm/tools/llvm-exegesis/lib/X86/BUILD.gn   | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/llvm/utils/gn/secondary/llvm/tools/llvm-exegesis/lib/X86/BUILD.gn b/llvm/utils/gn/secondary/llvm/tools/llvm-exegesis/lib/X86/BUILD.gn
index 42bce295c9a7e..a4e028e653186 100644
--- a/llvm/utils/gn/secondary/llvm/tools/llvm-exegesis/lib/X86/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/tools/llvm-exegesis/lib/X86/BUILD.gn
@@ -14,6 +14,9 @@ static_library("X86") {
     # depend on this Target/X86-internal build target.
     "//llvm/lib/Target/X86/MCTargetDesc",
   ]
-  sources = [ "Target.cpp" ]
+  sources = [
+    "Target.cpp",
+    "X86Counter.cpp",
+  ]
   include_dirs = [ "//llvm/lib/Target/X86" ]
 }

From 4d84d94969d6a72f8cf189c5fba1e2366ed88ee3 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Mon, 27 Jul 2020 17:32:53 +0100
Subject: [PATCH 0218/1035] [X86][SSE] Relax 128-bit restriction on
 extract_subvector(ext_vector_inreg(X),0) ->
 ext_vector_inreg(extract_subvector(X,0)) fold

We only need to ensure that the source is larger than the subvector result type
---
 llvm/lib/Target/X86/X86ISelLowering.cpp    | 10 +++++++---
 llvm/test/CodeGen/X86/vector-reduce-mul.ll | 12 ++++++------
 2 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 6abe6c6b83156..b986c42b9563f 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -48420,10 +48420,14 @@ static SDValue combineExtractSubvector(SDNode *N, SelectionDAG &DAG,
          InOpcode == ISD::ZERO_EXTEND_VECTOR_INREG ||
          InOpcode == ISD::SIGN_EXTEND ||
          InOpcode == ISD::SIGN_EXTEND_VECTOR_INREG) &&
-        VT.is128BitVector() &&
-        InVec.getOperand(0).getSimpleValueType().is128BitVector()) {
+        (SizeInBits == 128 || SizeInBits == 256) &&
+        InVec.getOperand(0).getValueSizeInBits() >= SizeInBits) {
+      SDLoc DL(N);
+      SDValue Ext = InVec.getOperand(0);
+      if (Ext.getValueSizeInBits() > SizeInBits)
+        Ext = extractSubVector(Ext, 0, DAG, DL, SizeInBits);
       unsigned ExtOp = getOpcode_EXTEND_VECTOR_INREG(InOpcode);
-      return DAG.getNode(ExtOp, SDLoc(N), VT, InVec.getOperand(0));
+      return DAG.getNode(ExtOp, DL, VT, Ext);
     }
     if (InOpcode == ISD::VSELECT &&
         InVec.getOperand(0).getValueType().is256BitVector() &&
diff --git a/llvm/test/CodeGen/X86/vector-reduce-mul.ll b/llvm/test/CodeGen/X86/vector-reduce-mul.ll
index 6e9bfbe5a79fd..23bf09c7ec9c7 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-mul.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-mul.ll
@@ -2006,24 +2006,24 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX512BWVL-LABEL: test_v32i8:
 ; AVX512BWVL:       # %bb.0:
 ; AVX512BWVL-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
-; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
+; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
 ; AVX512BWVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm1
 ; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
-; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
+; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
 ; AVX512BWVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm1
 ; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
-; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
+; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
 ; AVX512BWVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm1
 ; AVX512BWVL-NEXT:    vpsrld $16, %xmm1, %xmm1
-; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
+; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
 ; AVX512BWVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm1
 ; AVX512BWVL-NEXT:    vpsrlw $8, %xmm1, %xmm1
-; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
+; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
 ; AVX512BWVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm0
 ; AVX512BWVL-NEXT:    vmovd %xmm0, %eax

From 8f6e84ba7b590bb6fdcd6b2b3ba32782b7802bd3 Mon Sep 17 00:00:00 2001
From: MaheshRavishankar <ravishankarm@google.com>
Date: Mon, 27 Jul 2020 09:48:36 -0700
Subject: [PATCH 0219/1035] [mlir][Linalg] Enable fusion of std.constant
 (producer) with linalg.indexed_generic (consumer) with tensor arguments.

The implementation of fusing std.constant producer with a
linalg.indexed_generic consumer was already in place. It is exposed
with this change. Also cleaning up some of the patterns that implement
the fusion to not be templated, thereby avoiding lot of conditional
checks for calling the right instantiation.

Differential Revision: https://reviews.llvm.org/D84566
---
 mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp | 184 +++++++++---------
 mlir/test/Dialect/Linalg/fusion-tensor.mlir   |  64 ++++++
 2 files changed, 161 insertions(+), 87 deletions(-)

diff --git a/mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp b/mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp
index 82dfa75fc1f4b..9080a202a824d 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp
@@ -440,6 +440,10 @@ namespace {
 struct FuseGenericOpsOnTensors {
   static bool isFusible(LinalgOp producer, LinalgOp consumer,
                         unsigned consumerIdx) {
+    // Producer and consumer must have tensor semantics.
+    if (!producer.hasTensorSemantics() || !consumer.hasTensorSemantics())
+      return false;
+
     // Verify that
     // - the producer has all "parallel" iterator type.
     if (producer.getNumParallelLoops() != producer.getNumLoops())
@@ -457,9 +461,9 @@ struct FuseGenericOpsOnTensors {
     return producerResultIndexMap.isPermutation();
   }
 
-  static Operation *fuse(LinalgOp producer, LinalgOp consumer,
-                         unsigned consumerIdx, PatternRewriter &rewriter,
-                         OperationFolder *folder = nullptr) {
+  static LinalgOp fuse(LinalgOp producer, LinalgOp consumer,
+                       unsigned consumerIdx, PatternRewriter &rewriter,
+                       OperationFolder *folder = nullptr) {
     if (!isFusible(producer, consumer, consumerIdx))
       return nullptr;
 
@@ -736,24 +740,45 @@ static bool isTensorReshapeOpFusible(TensorReshapeOp reshapeOp,
   return useIndexMap.isIdentity();
 }
 
+/// Based on the type of `op` create a linalg op of the same type, i.e. if `op`
+/// is a linalg.generic operation, the create a `linalg.generic` operation with
+/// the given `args`. Expects `op` to be `linalg.generic` or
+/// `linalg.indexed_generic`.
+template <typename... Args>
+static LinalgOp createLinalgOpOfSameType(LinalgOp op, PatternRewriter &rewriter,
+                                         Args... args) {
+  if (isa<GenericOp>(op.getOperation()))
+    return cast<LinalgOp>(rewriter.create<GenericOp>(args...).getOperation());
+  if (isa<IndexedGenericOp>(op.getOperation()))
+    return cast<LinalgOp>(
+        rewriter.create<IndexedGenericOp>(args...).getOperation());
+  llvm_unreachable(
+      "expected only linalg.generic or linalg.indexed_generic ops");
+  return nullptr;
+}
+
 namespace {
+
 /// Implementation of fusion on tensor ops when producer is a TensorReshapeOp.
-template <typename LinalgOpTy> struct FuseTensorReshapeOpAsProducer {
-  static bool isFusible(TensorReshapeOp producer, LinalgOpTy consumer,
+struct FuseTensorReshapeOpAsProducer {
+  static bool isFusible(TensorReshapeOp producer, LinalgOp consumer,
                         unsigned consumerIdx) {
-    return isTensorReshapeOpFusible(
-        producer, consumer.getInputIndexingMap(consumerIdx), true);
+    return isa<GenericOp, IndexedGenericOp>(consumer.getOperation()) &&
+           consumer.hasTensorSemantics() &&
+           isTensorReshapeOpFusible(producer,
+                                    consumer.getInputIndexingMap(consumerIdx),
+                                    /*asProducer=*/true);
   }
 
-  static Operation *fuse(TensorReshapeOp producer, LinalgOpTy consumer,
-                         unsigned consumerIdx, PatternRewriter &rewriter,
-                         OperationFolder *folder = nullptr) {
+  static LinalgOp fuse(TensorReshapeOp producer, LinalgOp consumer,
+                       unsigned consumerIdx, PatternRewriter &rewriter,
+                       OperationFolder *folder = nullptr) {
     if (!isFusible(producer, consumer, consumerIdx))
       return nullptr;
 
     // Compute the fused operands list,
-    SmallVector<Value, 2> fusedOperands(consumer.operand_begin(),
-                                        consumer.operand_end());
+    Operation *consumerOp = consumer.getOperation();
+    SmallVector<Value, 2> fusedOperands(consumerOp->getOperands());
     fusedOperands[consumerIdx] = producer.src();
 
     // Compute indexing_maps for the fused operation. The indexing_maps for the
@@ -783,32 +808,35 @@ template <typename LinalgOpTy> struct FuseTensorReshapeOpAsProducer {
         llvm::map_range(fusedIndexMaps, [](AffineMap map) -> Attribute {
           return AffineMapAttr::get(map);
         }));
-    auto fusedOp = rewriter.create<LinalgOpTy>(
-        rewriter.getUnknownLoc(), consumer.getResultTypes(), fusedOperands,
+    LinalgOp fusedOp = createLinalgOpOfSameType(
+        consumer, rewriter, rewriter.getUnknownLoc(),
+        consumerOp->getResultTypes(), fusedOperands,
         rewriter.getI64IntegerAttr(fusedOperands.size()),
-        rewriter.getI64IntegerAttr(consumer.getNumResults()),
+        rewriter.getI64IntegerAttr(consumerOp->getNumResults()),
         rewriter.getArrayAttr(indexMapAttrs), consumer.iterator_types(),
         /*doc=*/nullptr,
         /*library_call=*/nullptr,
         /*symbol_source=*/nullptr);
-    auto &fusedRegion = fusedOp.region();
-    rewriter.cloneRegionBefore(consumer.region(), fusedRegion,
+    auto &fusedRegion = fusedOp.getOperation()->getRegion(0);
+    rewriter.cloneRegionBefore(consumerOp->getRegion(0), fusedRegion,
                                fusedRegion.begin());
     return fusedOp;
   }
 };
 
 /// Implementation of fusion on tensor ops when consumer is a TensorReshapeOp.
-template <typename LinalgOpTy> struct FuseTensorReshapeOpAsConsumer {
-  static bool isFusible(LinalgOpTy producer, TensorReshapeOp consumer,
+struct FuseTensorReshapeOpAsConsumer {
+  static bool isFusible(LinalgOp producer, TensorReshapeOp consumer,
                         unsigned consumerIdx) {
-    return isTensorReshapeOpFusible(consumer, producer.getOutputIndexingMap(0),
-                                    false);
+    return isa<GenericOp, IndexedGenericOp>(producer.getOperation()) &&
+           producer.hasTensorSemantics() &&
+           isTensorReshapeOpFusible(consumer, producer.getOutputIndexingMap(0),
+                                    /*asProducer=*/false);
   }
 
-  static Operation *fuse(LinalgOpTy producer, TensorReshapeOp consumer,
-                         unsigned consumerIdx, PatternRewriter &rewriter,
-                         OperationFolder *folder = nullptr) {
+  static LinalgOp fuse(LinalgOp producer, TensorReshapeOp consumer,
+                       unsigned consumerIdx, PatternRewriter &rewriter,
+                       OperationFolder *folder = nullptr) {
     if (!isFusible(producer, consumer, consumerIdx))
       return nullptr;
 
@@ -839,33 +867,36 @@ template <typename LinalgOpTy> struct FuseTensorReshapeOpAsConsumer {
           return AffineMapAttr::get(map);
         }));
 
-    auto fusedOp = rewriter.create<LinalgOpTy>(
-        rewriter.getUnknownLoc(), consumer.getResultType(),
-        producer.getOperands(),
-        rewriter.getI64IntegerAttr(producer.getNumOperands()),
+    Operation *producerOp = producer.getOperation();
+    LinalgOp fusedOp = createLinalgOpOfSameType(
+        producer, rewriter, rewriter.getUnknownLoc(), consumer.getResultType(),
+        producerOp->getOperands(),
+        rewriter.getI64IntegerAttr(producerOp->getNumOperands()),
         rewriter.getI64IntegerAttr(1), rewriter.getArrayAttr(indexMapAttrs),
         producer.iterator_types(),
         /*doc=*/nullptr,
         /*library_call=*/nullptr,
         /*symbol_source=*/nullptr);
-    auto &fusedRegion = fusedOp.region();
-    rewriter.cloneRegionBefore(producer.region(), fusedRegion,
+    auto &fusedRegion = fusedOp.getOperation()->getRegion(0);
+    rewriter.cloneRegionBefore(producerOp->getRegion(0), fusedRegion,
                                fusedRegion.begin());
     return fusedOp;
   }
 };
 
 /// Implementation of fusion on tensor ops when producer is a splat constant.
-template <typename LinalgOpTy> struct FuseConstantOpAsProducer {
-  static bool isFusible(ConstantOp producer, LinalgOpTy consumer,
+struct FuseConstantOpAsProducer {
+  static bool isFusible(ConstantOp producer, LinalgOp consumer,
                         unsigned consumerIdx) {
-    return producer.getResult().getType().isa<RankedTensorType>() &&
-           producer.value().template cast<DenseElementsAttr>().isSplat();
+    return isa<GenericOp, IndexedGenericOp>(consumer.getOperation()) &&
+           consumer.hasTensorSemantics() &&
+           producer.getResult().getType().isa<RankedTensorType>() &&
+           producer.value().cast<DenseElementsAttr>().isSplat();
   }
 
-  static Operation *fuse(ConstantOp producer, LinalgOpTy consumer,
-                         unsigned consumerIdx, PatternRewriter &rewriter,
-                         OperationFolder *folder = nullptr) {
+  static LinalgOp fuse(ConstantOp producer, LinalgOp consumer,
+                       unsigned consumerIdx, PatternRewriter &rewriter,
+                       OperationFolder *folder = nullptr) {
     if (!isFusible(producer, consumer, consumerIdx))
       return nullptr;
 
@@ -881,19 +912,20 @@ template <typename LinalgOpTy> struct FuseConstantOpAsProducer {
 
     // The operands list is same as the consumer with the argument for constant
     // index dropped.
-    SmallVector<Value, 4> fusedOperands(consumer.operand_begin(),
-                                        consumer.operand_end());
+    Operation *consumerOp = consumer.getOperation();
+    SmallVector<Value, 4> fusedOperands(consumerOp->getOperands());
     fusedOperands.erase(std::next(fusedOperands.begin(), consumerIdx));
 
     // Create a constant scalar value from the splat constant.
     Value scalarConstant = rewriter.create<ConstantOp>(
         producer.getLoc(),
-        producer.value().template cast<DenseElementsAttr>().getSplatValue());
+        producer.value().cast<DenseElementsAttr>().getSplatValue());
 
-    auto fusedOp = rewriter.create<LinalgOpTy>(
-        rewriter.getUnknownLoc(), consumer.getResultTypes(), fusedOperands,
-        rewriter.getI64IntegerAttr(consumer.getNumOperands() - 1),
-        rewriter.getI64IntegerAttr(consumer.getNumResults()),
+    LinalgOp fusedOp = createLinalgOpOfSameType(
+        consumer, rewriter, rewriter.getUnknownLoc(),
+        consumerOp->getResultTypes(), fusedOperands,
+        rewriter.getI64IntegerAttr(consumerOp->getNumOperands() - 1),
+        rewriter.getI64IntegerAttr(consumerOp->getNumResults()),
         rewriter.getAffineMapArrayAttr(fusedIndexMaps),
         consumer.iterator_types(),
         /*doc=*/nullptr,
@@ -902,19 +934,18 @@ template <typename LinalgOpTy> struct FuseConstantOpAsProducer {
 
     // Map the block argument corresponding to the replaced argument with the
     // scalar constant.
-    Region &consumerRegion = consumer.region();
+    Region &consumerRegion = consumerOp->getRegion(0);
     Block &entryBlock = *consumerRegion.begin();
-    unsigned argIndex =
-        entryBlock.getNumArguments() - consumer.getNumOperands() + consumerIdx;
+    unsigned argIndex = entryBlock.getNumArguments() -
+                        consumerOp->getNumOperands() + consumerIdx;
     BlockAndValueMapping mapping;
     mapping.map(entryBlock.getArgument(argIndex), scalarConstant);
-    Region &fusedRegion = fusedOp.region();
+    Region &fusedRegion = fusedOp.getOperation()->getRegion(0);
     rewriter.cloneRegionBefore(consumerRegion, fusedRegion, fusedRegion.begin(),
                                mapping);
     return fusedOp;
   }
 };
-
 } // namespace
 
 Operation *mlir::linalg::fuseTensorOps(PatternRewriter &rewriter,
@@ -929,48 +960,27 @@ Operation *mlir::linalg::fuseTensorOps(PatternRewriter &rewriter,
 
   // Fuse when consumer is GenericOp or IndexedGenericOp.
   if (isa<GenericOp, IndexedGenericOp>(consumer)) {
-    auto linalgOpConsumer = cast<LinalgOp>(consumer);
-    if (!linalgOpConsumer.hasTensorSemantics())
-      return nullptr;
-    if (isa<GenericOp, IndexedGenericOp>(producer)) {
-      auto linalgOpProducer = cast<LinalgOp>(producer);
-      if (linalgOpProducer.hasTensorSemantics())
-        return FuseGenericOpsOnTensors::fuse(linalgOpProducer, linalgOpConsumer,
-                                             consumerIdx, rewriter, folder);
-    } else if (auto reshapeOpProducer = dyn_cast<TensorReshapeOp>(producer)) {
-      if (auto genericOpConsumer = dyn_cast<GenericOp>(consumer)) {
-        return FuseTensorReshapeOpAsProducer<GenericOp>::fuse(
-            reshapeOpProducer, genericOpConsumer, consumerIdx, rewriter,
-            folder);
-      } else if (auto indexedGenericOpConsumer =
-                     dyn_cast<IndexedGenericOp>(consumer)) {
-        return FuseTensorReshapeOpAsProducer<IndexedGenericOp>::fuse(
-            reshapeOpProducer, indexedGenericOpConsumer, consumerIdx, rewriter,
-            folder);
-      }
-    } else if (auto constantOpProducer = dyn_cast<ConstantOp>(producer)) {
-      if (auto genericOpConsumer = dyn_cast<GenericOp>(consumer)) {
-        return FuseConstantOpAsProducer<GenericOp>::fuse(
-            constantOpProducer, genericOpConsumer, consumerIdx, rewriter,
-            folder);
-      }
-    }
+    if (isa<GenericOp, IndexedGenericOp>(producer))
+      return FuseGenericOpsOnTensors::fuse(cast<LinalgOp>(producer),
+                                           cast<LinalgOp>(consumer),
+                                           consumerIdx, rewriter, folder);
+    if (auto reshapeOpProducer = dyn_cast<TensorReshapeOp>(producer))
+      return FuseTensorReshapeOpAsProducer::fuse(reshapeOpProducer,
+                                                 cast<LinalgOp>(consumer),
+                                                 consumerIdx, rewriter, folder);
+    if (auto constantOpProducer = dyn_cast<ConstantOp>(producer))
+      return FuseConstantOpAsProducer::fuse(constantOpProducer,
+                                            cast<LinalgOp>(consumer),
+                                            consumerIdx, rewriter, folder);
     return nullptr;
   }
 
-  // Fuse when consumer is a TensorReshapeOp.
-  if (TensorReshapeOp reshapeOp = dyn_cast<TensorReshapeOp>(consumer)) {
-    if (auto genericOpProducer = dyn_cast<GenericOp>(producer)) {
-      if (genericOpProducer.hasTensorSemantics())
-        return FuseTensorReshapeOpAsConsumer<GenericOp>::fuse(
-            genericOpProducer, reshapeOp, consumerIdx, rewriter, folder);
-    } else if (auto indexedGenericOpProducer =
-                   dyn_cast<IndexedGenericOp>(producer)) {
-      if (indexedGenericOpProducer.hasTensorSemantics())
-        return FuseTensorReshapeOpAsConsumer<IndexedGenericOp>::fuse(
-            indexedGenericOpProducer, reshapeOp, consumerIdx, rewriter, folder);
+  if (isa<GenericOp, IndexedGenericOp>(producer)) {
+    // Fuse when consumer is a TensorReshapeOp.
+    if (TensorReshapeOp reshapeOp = dyn_cast<TensorReshapeOp>(consumer)) {
+      return FuseTensorReshapeOpAsConsumer::fuse(
+          cast<LinalgOp>(producer), reshapeOp, consumerIdx, rewriter, folder);
     }
-    return nullptr;
   }
 
   return nullptr;
diff --git a/mlir/test/Dialect/Linalg/fusion-tensor.mlir b/mlir/test/Dialect/Linalg/fusion-tensor.mlir
index 5c49a59227c14..4e7f1f6152b1a 100644
--- a/mlir/test/Dialect/Linalg/fusion-tensor.mlir
+++ b/mlir/test/Dialect/Linalg/fusion-tensor.mlir
@@ -249,6 +249,38 @@ func @generic_op_constant_fusion(%arg0 : tensor<5x?x?xf32>) -> tensor<5x?x?xf32>
 
 // -----
 
+#map0 = affine_map<(d0, d1, d2) -> (d0)>
+#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
+func @indexed_generic_op_constant_fusion(%arg0 : tensor<5x?x?xf32>)
+                                         -> tensor<5x?x?xf32>
+{
+  %0 = constant dense<42.0> : tensor<5xf32>
+  %1 = linalg.indexed_generic
+       {args_in = 2 : i64, args_out = 1 : i64,
+         indexing_maps = [#map0, #map1, #map1],
+         iterator_types = ["parallel", "parallel", "parallel"]}
+       %0, %arg0 {
+       ^bb0(%arg1: index, %arg2: index, %arg3: index, %arg4: f32, %arg5 : f32):
+         %2 = mulf %arg4, %arg5 : f32
+         linalg.yield %2 : f32
+       }: tensor<5xf32>, tensor<5x?x?xf32> -> tensor<5x?x?xf32>
+  return %1 : tensor<5x?x?xf32>
+}
+//   CHECK-DAG: #[[$MAP0:.*]] = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
+// CHECK-LABEL: func @indexed_generic_op_constant_fusion
+//       CHECK:   %[[CST:.*]] = constant {{.*}} : f32
+//       CHECK:   linalg.indexed_generic
+//  CHECK-SAME:     args_in = 1 : i64
+//  CHECK-SAME:     args_out = 1 : i64
+//       CHECK:   ^{{[a-zA-Z0-9_]*}}
+//  CHECK-SAME:     %[[ARG1:[a-zA-Z0-9]*]]: index
+//  CHECK-SAME:     %[[ARG2:[a-zA-Z0-9]*]]: index
+//  CHECK-SAME:     %[[ARG3:[a-zA-Z0-9]*]]: index
+//  CHECK-SAME:     %[[ARG4:.*]]: f32)
+//       CHECK:     mulf %[[CST]], %[[ARG4]]
+
+// -----
+
 #map0 = affine_map<(d0, d1, d2) -> ()>
 #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
 func @generic_op_zero_dim_constant_fusion(%arg0 : tensor<5x?x?xf32>)
@@ -277,6 +309,38 @@ func @generic_op_zero_dim_constant_fusion(%arg0 : tensor<5x?x?xf32>)
 
 // -----
 
+#map0 = affine_map<(d0, d1, d2) -> ()>
+#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
+func @indexed_generic_op_zero_dim_constant_fusion
+  (%arg0 : tensor<5x?x?xf32>) -> tensor<5x?x?xf32>
+{
+  %0 = constant dense<42.0> : tensor<f32>
+  %1 = linalg.indexed_generic
+       {args_in = 2 : i64, args_out = 1 : i64,
+         indexing_maps = [#map0, #map1, #map1],
+         iterator_types = ["parallel", "parallel", "parallel"]}
+       %0, %arg0 {
+       ^bb0(%arg1 : index, %arg2 : index, %arg3 : index, %arg4: f32, %arg5: f32):
+         %2 = mulf %arg4, %arg5 : f32
+         linalg.yield %2 : f32
+       }: tensor<f32>, tensor<5x?x?xf32> -> tensor<5x?x?xf32>
+  return %1 : tensor<5x?x?xf32>
+}
+//   CHECK-DAG: #[[$MAP0:.*]] = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
+// CHECK-LABEL: func @indexed_generic_op_zero_dim_constant_fusion
+//       CHECK:   %[[CST:.*]] = constant {{.*}} : f32
+//       CHECK:   linalg.indexed_generic
+//  CHECK-SAME:     args_in = 1 : i64
+//  CHECK-SAME:     args_out = 1 : i64
+//       CHECK:   ^{{[a-zA-Z0-9_]*}}
+//  CHECK-SAME:     %[[ARG1:[a-zA-Z0-9]*]]: index
+//  CHECK-SAME:     %[[ARG2:[a-zA-Z0-9]*]]: index
+//  CHECK-SAME:     %[[ARG3:[a-zA-Z0-9]*]]: index
+//  CHECK-SAME:     %[[ARG4:.*]]: f32)
+//       CHECK:     mulf %[[CST]], %[[ARG4]]
+
+// -----
+
 #map0 = affine_map<(d0, d1) -> (d0, d1)>
 func @generic_op_indexed_generic_op_fusion(%arg0: tensor<?x?xi32>,
                                            %arg1: tensor<?x?xi32>) {

From dbeb184b7f54db2d3ef20ac153b1c77f81cf0b99 Mon Sep 17 00:00:00 2001
From: Francesco Petrogalli <francesco.petrogalli@arm.com>
Date: Mon, 27 Jul 2020 16:24:18 +0000
Subject: [PATCH 0220/1035] [NFC][AArch64] Replace some template
 methods/invocations...

...with the non-template version, as the template version might
increase the size of the compiler build.

Methods affected:

1.`findAddrModeSVELoadStore`
2. `SelectPredicatedStore`

Also, remove the `const` qualifier from the `unsigned` parameters of
the methods to conform with other similar methods in the class.
---
 .../Target/AArch64/AArch64ISelDAGToDAG.cpp    | 67 +++++++------------
 1 file changed, 26 insertions(+), 41 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
index 10c4778533533..dbd7db7ee8e6f 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
@@ -262,14 +262,12 @@ class AArch64DAGToDAGISel : public SelectionDAGISel {
   void SelectPostStore(SDNode *N, unsigned NumVecs, unsigned Opc);
   void SelectStoreLane(SDNode *N, unsigned NumVecs, unsigned Opc);
   void SelectPostStoreLane(SDNode *N, unsigned NumVecs, unsigned Opc);
-  template <unsigned Scale>
-  void SelectPredicatedStore(SDNode *N, unsigned NumVecs, const unsigned Opc_rr,
-                             const unsigned Opc_ri);
-  template <unsigned Scale>
+  void SelectPredicatedStore(SDNode *N, unsigned NumVecs, unsigned Scale,
+                             unsigned Opc_rr, unsigned Opc_ri);
   std::tuple<unsigned, SDValue, SDValue>
-  findAddrModeSVELoadStore(SDNode *N, const unsigned Opc_rr,
-                           const unsigned Opc_ri, const SDValue &OldBase,
-                           const SDValue &OldOffset);
+  findAddrModeSVELoadStore(SDNode *N, unsigned Opc_rr, unsigned Opc_ri,
+                           const SDValue &OldBase, const SDValue &OldOffset,
+                           unsigned Scale);
 
   bool tryBitfieldExtractOp(SDNode *N);
   bool tryBitfieldExtractOpFromSExt(SDNode *N);
@@ -1414,12 +1412,12 @@ void AArch64DAGToDAGISel::SelectPostLoad(SDNode *N, unsigned NumVecs,
 /// Optimize \param OldBase and \param OldOffset selecting the best addressing
 /// mode. Returns a tuple consisting of an Opcode, an SDValue representing the
 /// new Base and an SDValue representing the new offset.
-template <unsigned Scale>
 std::tuple<unsigned, SDValue, SDValue>
-AArch64DAGToDAGISel::findAddrModeSVELoadStore(SDNode *N, const unsigned Opc_rr,
-                                              const unsigned Opc_ri,
+AArch64DAGToDAGISel::findAddrModeSVELoadStore(SDNode *N, unsigned Opc_rr,
+                                              unsigned Opc_ri,
                                               const SDValue &OldBase,
-                                              const SDValue &OldOffset) {
+                                              const SDValue &OldOffset,
+                                              unsigned Scale) {
   SDValue NewBase = OldBase;
   SDValue NewOffset = OldOffset;
   // Detect a possible Reg+Imm addressing mode.
@@ -1429,7 +1427,7 @@ AArch64DAGToDAGISel::findAddrModeSVELoadStore(SDNode *N, const unsigned Opc_rr,
   // Detect a possible reg+reg addressing mode, but only if we haven't already
   // detected a Reg+Imm one.
   const bool IsRegReg =
-      !IsRegImm && SelectSVERegRegAddrMode<Scale>(OldBase, NewBase, NewOffset);
+      !IsRegImm && SelectSVERegRegAddrMode(OldBase, Scale, NewBase, NewOffset);
 
   // Select the instruction.
   return std::make_tuple(IsRegReg ? Opc_rr : Opc_ri, NewBase, NewOffset);
@@ -1479,10 +1477,9 @@ void AArch64DAGToDAGISel::SelectStore(SDNode *N, unsigned NumVecs,
   ReplaceNode(N, St);
 }
 
-template <unsigned Scale>
 void AArch64DAGToDAGISel::SelectPredicatedStore(SDNode *N, unsigned NumVecs,
-                                                const unsigned Opc_rr,
-                                                const unsigned Opc_ri) {
+                                                unsigned Scale, unsigned Opc_rr,
+                                                unsigned Opc_ri) {
   SDLoc dl(N);
 
   // Form a REG_SEQUENCE to force register allocation.
@@ -1492,9 +1489,9 @@ void AArch64DAGToDAGISel::SelectPredicatedStore(SDNode *N, unsigned NumVecs,
   // Optimize addressing mode.
   unsigned Opc;
   SDValue Offset, Base;
-  std::tie(Opc, Base, Offset) = findAddrModeSVELoadStore<Scale>(
+  std::tie(Opc, Base, Offset) = findAddrModeSVELoadStore(
       N, Opc_rr, Opc_ri, N->getOperand(NumVecs + 3),
-      CurDAG->getTargetConstant(0, dl, MVT::i64));
+      CurDAG->getTargetConstant(0, dl, MVT::i64), Scale);
 
   SDValue Ops[] = {RegSeq, N->getOperand(NumVecs + 2), // predicate
                    Base,                               // address
@@ -4085,63 +4082,51 @@ void AArch64DAGToDAGISel::Select(SDNode *Node) {
     }
     case Intrinsic::aarch64_sve_st2: {
       if (VT == MVT::nxv16i8) {
-        SelectPredicatedStore</*Scale=*/0>(Node, 2, AArch64::ST2B,
-                                           AArch64::ST2B_IMM);
+        SelectPredicatedStore(Node, 2, 0, AArch64::ST2B, AArch64::ST2B_IMM);
         return;
       } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
                  (VT == MVT::nxv8bf16 && Subtarget->hasBF16())) {
-        SelectPredicatedStore</*Scale=*/1>(Node, 2, AArch64::ST2H,
-                                           AArch64::ST2H_IMM);
+        SelectPredicatedStore(Node, 2, 1, AArch64::ST2H, AArch64::ST2H_IMM);
         return;
       } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
-        SelectPredicatedStore</*Scale=*/2>(Node, 2, AArch64::ST2W,
-                                           AArch64::ST2W_IMM);
+        SelectPredicatedStore(Node, 2, 2, AArch64::ST2W, AArch64::ST2W_IMM);
         return;
       } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
-        SelectPredicatedStore</*Scale=*/3>(Node, 2, AArch64::ST2D,
-                                           AArch64::ST2D_IMM);
+        SelectPredicatedStore(Node, 2, 3, AArch64::ST2D, AArch64::ST2D_IMM);
         return;
       }
       break;
     }
     case Intrinsic::aarch64_sve_st3: {
       if (VT == MVT::nxv16i8) {
-        SelectPredicatedStore</*Scale=*/0>(Node, 3, AArch64::ST3B,
-                                           AArch64::ST3B_IMM);
+        SelectPredicatedStore(Node, 3, 0, AArch64::ST3B, AArch64::ST3B_IMM);
         return;
       } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
                  (VT == MVT::nxv8bf16 && Subtarget->hasBF16())) {
-        SelectPredicatedStore</*Scale=*/1>(Node, 3, AArch64::ST3H,
-                                           AArch64::ST3H_IMM);
+        SelectPredicatedStore(Node, 3, 1, AArch64::ST3H, AArch64::ST3H_IMM);
         return;
       } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
-        SelectPredicatedStore</*Scale=*/2>(Node, 3, AArch64::ST3W,
-                                           AArch64::ST3W_IMM);
+        SelectPredicatedStore(Node, 3, 2, AArch64::ST3W, AArch64::ST3W_IMM);
         return;
       } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
-        SelectPredicatedStore</*Scale=*/3>(Node, 3, AArch64::ST3D,
-                                           AArch64::ST3D_IMM);
+        SelectPredicatedStore(Node, 3, 3, AArch64::ST3D, AArch64::ST3D_IMM);
         return;
       }
       break;
     }
     case Intrinsic::aarch64_sve_st4: {
       if (VT == MVT::nxv16i8) {
-        SelectPredicatedStore</*Scale=*/0>(Node, 4, AArch64::ST4B,
-                                           AArch64::ST4B_IMM);
+        SelectPredicatedStore(Node, 4, 0, AArch64::ST4B, AArch64::ST4B_IMM);
         return;
       } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
                  (VT == MVT::nxv8bf16 && Subtarget->hasBF16())) {
-        SelectPredicatedStore</*Scale=*/1>(Node, 4, AArch64::ST4H,
-                                           AArch64::ST4H_IMM);
+        SelectPredicatedStore(Node, 4, 1, AArch64::ST4H, AArch64::ST4H_IMM);
         return;
       } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
-        SelectPredicatedStore</*Scale=*/2>(Node, 4, AArch64::ST4W,
-                                           AArch64::ST4W_IMM);
+        SelectPredicatedStore(Node, 4, 2, AArch64::ST4W, AArch64::ST4W_IMM);
         return;
       } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
-        SelectPredicatedStore</*Scale=*/3>(Node, 4, AArch64::ST4D,
-                                           AArch64::ST4D_IMM);
+        SelectPredicatedStore(Node, 4, 3, AArch64::ST4D, AArch64::ST4D_IMM);
         return;
       }
       break;

From 4a4cafabc9067fced5890a245b03ef5897ad988b Mon Sep 17 00:00:00 2001
From: AlexisPerry <aperry@lanl.gov>
Date: Mon, 27 Jul 2020 09:57:31 -0700
Subject: [PATCH 0221/1035] [flang] Temp Driver - pass the flag to change the
 default integer kind through to F18_FC

fixes BUG 46307

Differential Revision: https://reviews.llvm.org/D84266
---
 flang/tools/f18/f18.cpp | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/flang/tools/f18/f18.cpp b/flang/tools/f18/f18.cpp
index bcafb0d53cc7b..2b2eacc2e6f11 100644
--- a/flang/tools/f18/f18.cpp
+++ b/flang/tools/f18/f18.cpp
@@ -545,6 +545,11 @@ int main(int argc, char *const argv[]) {
       defaultKinds.set_defaultIntegerKind(8);
       defaultKinds.set_subscriptIntegerKind(8);
       defaultKinds.set_sizeIntegerKind(8);
+      if (isPGF90) {
+        driver.F18_FCArgs.push_back("-i8");
+      } else {
+        driver.F18_FCArgs.push_back("-fdefault-integer-8");
+      }
     } else if (arg == "-Mlargearray") {
     } else if (arg == "-Mnolargearray") {
     } else if (arg == "-flarge-sizes") {

From 51e1c028d4021eba64a56e684d34bb164c244d43 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@intel.com>
Date: Mon, 27 Jul 2020 09:40:26 -0700
Subject: [PATCH 0222/1035] [X86] Add back comment inadvertently lost in
 1a1448e6568d9b11f198e510fa9c4cb6b1f4216a.

---
 llvm/lib/Target/X86/X86MCInstLower.cpp | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/llvm/lib/Target/X86/X86MCInstLower.cpp b/llvm/lib/Target/X86/X86MCInstLower.cpp
index 8f3e32727371b..af86c1fc7206d 100644
--- a/llvm/lib/Target/X86/X86MCInstLower.cpp
+++ b/llvm/lib/Target/X86/X86MCInstLower.cpp
@@ -1083,6 +1083,9 @@ void X86AsmPrinter::LowerTlsAddr(X86MCInstLower &MCInstLowering,
 /// bytes.  Return the size of nop emitted.
 static unsigned emitNop(MCStreamer &OS, unsigned NumBytes,
                         const X86Subtarget *Subtarget) {
+  // Determine the longest nop which can be efficiently decoded for the given
+  // target cpu.  15-bytes is the longest single NOP instruction, but some
+  // platforms can't decode the longest forms efficiently.
   unsigned MaxNopLength = 1;
   if (Subtarget->is64Bit()) {
     // FIXME: We can use NOOPL on 32-bit targets with FeatureNOPL, but the

From df880b77302d2e12d988e620eba242defdd6d4a7 Mon Sep 17 00:00:00 2001
From: Nadav Rotem <nadav256@gmail.com>
Date: Mon, 27 Jul 2020 10:06:36 -0700
Subject: [PATCH 0223/1035] [StackProtector] Speed up RequiresStackProtector

Speed up the method RequiresStackProtector by checking the intrinsic
value of the call. The original code calls getName() that returns an
allocating std::string on each check. This change removes about 96072
std::string instances when compiling sqlite3.c; The function was
discovered with a Facebook-internal performance tool.

Differential Revision: https://reviews.llvm.org/D84620
---
 llvm/lib/CodeGen/StackProtector.cpp | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/CodeGen/StackProtector.cpp b/llvm/lib/CodeGen/StackProtector.cpp
index a343791807e64..e246c2e5f55cb 100644
--- a/llvm/lib/CodeGen/StackProtector.cpp
+++ b/llvm/lib/CodeGen/StackProtector.cpp
@@ -251,10 +251,9 @@ bool StackProtector::HasAddressTaken(const Instruction *AI,
 static const CallInst *findStackProtectorIntrinsic(Function &F) {
   for (const BasicBlock &BB : F)
     for (const Instruction &I : BB)
-      if (const CallInst *CI = dyn_cast<CallInst>(&I))
-        if (CI->getCalledFunction() ==
-            Intrinsic::getDeclaration(F.getParent(), Intrinsic::stackprotector))
-          return CI;
+      if (const auto *II = dyn_cast<IntrinsicInst>(&I))
+        if (II->getIntrinsicID() == Intrinsic::stackprotector)
+          return II;
   return nullptr;
 }
 

From 902cbcd59e22ccd853f6b0c22acc772fd955dc46 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Thu, 23 Jul 2020 23:13:44 -0700
Subject: [PATCH 0224/1035] Use llvm::is_contained where appropriate (NFC)

Summary:
This patch replaces std::find with llvm::is_contained where
appropriate.

Reviewers: efriedma, nhaehnle

Reviewed By: nhaehnle

Subscribers: arsenm, jvesely, nhaehnle, hiraditya, rogfer01, kerbowa, llvm-commits, vkmr

Tags: #llvm

Differential Revision: https://reviews.llvm.org/D84489
---
 llvm/lib/Analysis/AssumptionCache.cpp              | 2 +-
 llvm/lib/CodeGen/GlobalISel/LegalityPredicates.cpp | 4 ++--
 llvm/lib/Target/AMDGPU/AMDGPULibCalls.cpp          | 3 +--
 llvm/lib/Target/AMDGPU/AMDGPUUnifyMetadata.cpp     | 2 +-
 llvm/lib/Target/BPF/BPFMIChecking.cpp              | 6 ++----
 llvm/lib/Transforms/IPO/GlobalOpt.cpp              | 3 +--
 llvm/lib/Transforms/IPO/PartialInlining.cpp        | 3 +--
 llvm/lib/Transforms/Scalar/GVNSink.cpp             | 6 ++----
 llvm/lib/Transforms/Vectorize/VPlanSLP.cpp         | 2 +-
 llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp    | 8 ++------
 llvm/tools/bugpoint/ExtractFunction.cpp            | 2 +-
 llvm/tools/llvm-exegesis/lib/Analysis.cpp          | 2 +-
 12 files changed, 16 insertions(+), 27 deletions(-)

diff --git a/llvm/lib/Analysis/AssumptionCache.cpp b/llvm/lib/Analysis/AssumptionCache.cpp
index 16bfd5c75902f..bc0cdc4c4c77a 100644
--- a/llvm/lib/Analysis/AssumptionCache.cpp
+++ b/llvm/lib/Analysis/AssumptionCache.cpp
@@ -175,7 +175,7 @@ void AssumptionCache::transferAffectedValuesInCache(Value *OV, Value *NV) {
     return;
 
   for (auto &A : AVI->second)
-    if (std::find(NAVV.begin(), NAVV.end(), A) == NAVV.end())
+    if (!llvm::is_contained(NAVV, A))
       NAVV.push_back(A);
   AffectedValues.erase(OV);
 }
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalityPredicates.cpp b/llvm/lib/CodeGen/GlobalISel/LegalityPredicates.cpp
index a83742f2138fc..17bce517814de 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalityPredicates.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalityPredicates.cpp
@@ -24,7 +24,7 @@ LegalityPredicates::typeInSet(unsigned TypeIdx,
                               std::initializer_list<LLT> TypesInit) {
   SmallVector<LLT, 4> Types = TypesInit;
   return [=](const LegalityQuery &Query) {
-    return std::find(Types.begin(), Types.end(), Query.Types[TypeIdx]) != Types.end();
+    return llvm::is_contained(Types, Query.Types[TypeIdx]);
   };
 }
 
@@ -34,7 +34,7 @@ LegalityPredicate LegalityPredicates::typePairInSet(
   SmallVector<std::pair<LLT, LLT>, 4> Types = TypesInit;
   return [=](const LegalityQuery &Query) {
     std::pair<LLT, LLT> Match = {Query.Types[TypeIdx0], Query.Types[TypeIdx1]};
-    return std::find(Types.begin(), Types.end(), Match) != Types.end();
+    return llvm::is_contained(Types, Match);
   };
 }
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULibCalls.cpp b/llvm/lib/Target/AMDGPU/AMDGPULibCalls.cpp
index 4a14259f1bdb1..538a22df514fe 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULibCalls.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULibCalls.cpp
@@ -495,8 +495,7 @@ bool AMDGPULibCalls::isUnsafeMath(const CallInst *CI) const {
 }
 
 bool AMDGPULibCalls::useNativeFunc(const StringRef F) const {
-  return AllNative ||
-         std::find(UseNative.begin(), UseNative.end(), F) != UseNative.end();
+  return AllNative || llvm::is_contained(UseNative, F);
 }
 
 void AMDGPULibCalls::initNativeFuncs() {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUUnifyMetadata.cpp b/llvm/lib/Target/AMDGPU/AMDGPUUnifyMetadata.cpp
index 281ae6d646e9e..f898456203a10 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUUnifyMetadata.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUUnifyMetadata.cpp
@@ -96,7 +96,7 @@ namespace {
     SmallVector<Metadata *, 4> All;
     for (auto MD : NamedMD->operands())
       for (const auto &Op : MD->operands())
-        if (std::find(All.begin(), All.end(), Op.get()) == All.end())
+        if (!llvm::is_contained(All, Op.get()))
           All.push_back(Op.get());
 
     NamedMD->eraseFromParent();
diff --git a/llvm/lib/Target/BPF/BPFMIChecking.cpp b/llvm/lib/Target/BPF/BPFMIChecking.cpp
index f82f166eda4d0..ac40b24596a26 100644
--- a/llvm/lib/Target/BPF/BPFMIChecking.cpp
+++ b/llvm/lib/Target/BPF/BPFMIChecking.cpp
@@ -143,12 +143,10 @@ static bool hasLiveDefs(const MachineInstr &MI, const TargetRegisterInfo *TRI) {
     return true;
 
   // Otherwise, return true if any aliased SuperReg of GPR32 is not dead.
-  std::vector<unsigned>::iterator search_begin = GPR64DeadDefs.begin();
-  std::vector<unsigned>::iterator search_end = GPR64DeadDefs.end();
   for (auto I : GPR32LiveDefs)
     for (MCSuperRegIterator SR(I, TRI); SR.isValid(); ++SR)
-       if (std::find(search_begin, search_end, *SR) == search_end)
-         return true;
+      if (!llvm::is_contained(GPR64DeadDefs, *SR))
+        return true;
 
   return false;
 }
diff --git a/llvm/lib/Transforms/IPO/GlobalOpt.cpp b/llvm/lib/Transforms/IPO/GlobalOpt.cpp
index d9fb820f7cb53..0b650b3b58f43 100644
--- a/llvm/lib/Transforms/IPO/GlobalOpt.cpp
+++ b/llvm/lib/Transforms/IPO/GlobalOpt.cpp
@@ -2221,8 +2221,7 @@ isValidCandidateForColdCC(Function &F,
     BlockFrequencyInfo &CallerBFI = GetBFI(*CallerFunc);
     if (!isColdCallSite(CB, CallerBFI))
       return false;
-    auto It = std::find(AllCallsCold.begin(), AllCallsCold.end(), CallerFunc);
-    if (It == AllCallsCold.end())
+    if (!llvm::is_contained(AllCallsCold, CallerFunc))
       return false;
   }
   return true;
diff --git a/llvm/lib/Transforms/IPO/PartialInlining.cpp b/llvm/lib/Transforms/IPO/PartialInlining.cpp
index 5d863f1330a44..e1dc036ae413c 100644
--- a/llvm/lib/Transforms/IPO/PartialInlining.cpp
+++ b/llvm/lib/Transforms/IPO/PartialInlining.cpp
@@ -1181,8 +1181,7 @@ PartialInlinerImpl::FunctionCloner::doSingleRegionFunctionOutlining() {
   // (i.e. not to be extracted to the out of line function)
   auto ToBeInlined = [&, this](BasicBlock *BB) {
     return BB == ClonedOI->ReturnBlock ||
-           (std::find(ClonedOI->Entries.begin(), ClonedOI->Entries.end(), BB) !=
-            ClonedOI->Entries.end());
+           llvm::is_contained(ClonedOI->Entries, BB);
   };
 
   assert(ClonedOI && "Expecting OutlineInfo for single region outline");
diff --git a/llvm/lib/Transforms/Scalar/GVNSink.cpp b/llvm/lib/Transforms/Scalar/GVNSink.cpp
index dfb4b7e038ba5..5a34ad18158b4 100644
--- a/llvm/lib/Transforms/Scalar/GVNSink.cpp
+++ b/llvm/lib/Transforms/Scalar/GVNSink.cpp
@@ -158,8 +158,7 @@ class LockstepReverseIterator {
 
   void restrictToBlocks(SmallSetVector<BasicBlock *, 4> &Blocks) {
     for (auto II = Insts.begin(); II != Insts.end();) {
-      if (std::find(Blocks.begin(), Blocks.end(), (*II)->getParent()) ==
-          Blocks.end()) {
+      if (!llvm::is_contained(Blocks, (*II)->getParent())) {
         ActiveBlocks.remove((*II)->getParent());
         II = Insts.erase(II);
       } else {
@@ -277,8 +276,7 @@ class ModelledPHI {
     auto VI = Values.begin();
     while (BI != Blocks.end()) {
       assert(VI != Values.end());
-      if (std::find(NewBlocks.begin(), NewBlocks.end(), *BI) ==
-          NewBlocks.end()) {
+      if (!llvm::is_contained(NewBlocks, *BI)) {
         BI = Blocks.erase(BI);
         VI = Values.erase(VI);
       } else {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanSLP.cpp b/llvm/lib/Transforms/Vectorize/VPlanSLP.cpp
index 9019ed15ec5ff..b072ca9c39206 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanSLP.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanSLP.cpp
@@ -124,7 +124,7 @@ bool VPlanSlp::areVectorizable(ArrayRef<VPValue *> Operands) const {
     for (auto &I : *Parent) {
       auto *VPI = cast<VPInstruction>(&I);
       if (VPI->getOpcode() == Instruction::Load &&
-          std::find(Operands.begin(), Operands.end(), VPI) != Operands.end())
+          llvm::is_contained(Operands, VPI))
         LoadsSeen++;
 
       if (LoadsSeen == Operands.size())
diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
index b384c94121e9b..6eec8d14de4af 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
@@ -65,9 +65,7 @@ static void verifyBlocksInRegion(const VPRegionBlock *Region) {
     for (const VPBlockBase *Succ : Successors) {
       // There must be a bi-directional link between block and successor.
       const auto &SuccPreds = Succ->getPredecessors();
-      assert(std::find(SuccPreds.begin(), SuccPreds.end(), VPB) !=
-                 SuccPreds.end() &&
-             "Missing predecessor link.");
+      assert(llvm::is_contained(SuccPreds, VPB) && "Missing predecessor link.");
       (void)SuccPreds;
     }
 
@@ -86,9 +84,7 @@ static void verifyBlocksInRegion(const VPRegionBlock *Region) {
 
       // There must be a bi-directional link between block and predecessor.
       const auto &PredSuccs = Pred->getSuccessors();
-      assert(std::find(PredSuccs.begin(), PredSuccs.end(), VPB) !=
-                 PredSuccs.end() &&
-             "Missing successor link.");
+      assert(llvm::is_contained(PredSuccs, VPB) && "Missing successor link.");
       (void)PredSuccs;
     }
   }
diff --git a/llvm/tools/bugpoint/ExtractFunction.cpp b/llvm/tools/bugpoint/ExtractFunction.cpp
index d9047acd30e11..7a75cb90edc53 100644
--- a/llvm/tools/bugpoint/ExtractFunction.cpp
+++ b/llvm/tools/bugpoint/ExtractFunction.cpp
@@ -386,7 +386,7 @@ BugDriver::extractMappedBlocksFromModule(const std::vector<BasicBlock *> &BBs,
   for (Function &F : *M)
     for (BasicBlock &BB : F)
       // Check if this block is going to be extracted.
-      if (std::find(BBs.begin(), BBs.end(), &BB) == BBs.end())
+      if (!llvm::is_contained(BBs, &BB))
         BlocksToExtract.push_back(&BB);
 
   raw_fd_ostream OS(Temp->FD, /*shouldClose*/ false);
diff --git a/llvm/tools/llvm-exegesis/lib/Analysis.cpp b/llvm/tools/llvm-exegesis/lib/Analysis.cpp
index 5e9023b8127ac..077acf9aff6b9 100644
--- a/llvm/tools/llvm-exegesis/lib/Analysis.cpp
+++ b/llvm/tools/llvm-exegesis/lib/Analysis.cpp
@@ -28,7 +28,7 @@ enum EscapeTag { kEscapeCsv, kEscapeHtml, kEscapeHtmlString };
 template <EscapeTag Tag> void writeEscaped(raw_ostream &OS, const StringRef S);
 
 template <> void writeEscaped<kEscapeCsv>(raw_ostream &OS, const StringRef S) {
-  if (std::find(S.begin(), S.end(), kCsvSep) == S.end()) {
+  if (!llvm::is_contained(S, kCsvSep)) {
     OS << S;
   } else {
     // Needs escaping.

From d19af2f2476b5e13a65d5283cce9859e2c1ef763 Mon Sep 17 00:00:00 2001
From: Mitch Phillips <31459023+hctim@users.noreply.github.com>
Date: Mon, 27 Jul 2020 10:51:53 -0700
Subject: [PATCH 0225/1035] [GWP-ASan] Crash handler API returns
 sizeof(collected trace)

Summary:
Fix up a slight bug with the crash handler API, where we say that we
return the size of the collected trace (instead of the size of the trace
that's returned) when the return buffer is too small, and the result is
truncated.

Also, as a result, patch up a small uninitialized memory bug.

Reviewers: morehouse, eugenis

Reviewed By: eugenis

Subscribers: #sanitizers

Tags: #sanitizers

Differential Revision: https://reviews.llvm.org/D84652
---
 compiler-rt/lib/gwp_asan/common.cpp          |  3 ++
 compiler-rt/lib/gwp_asan/crash_handler.cpp   | 21 ++++++++--
 compiler-rt/lib/gwp_asan/tests/backtrace.cpp | 44 ++++++++++++++++++--
 3 files changed, 61 insertions(+), 7 deletions(-)

diff --git a/compiler-rt/lib/gwp_asan/common.cpp b/compiler-rt/lib/gwp_asan/common.cpp
index 3438c4b91893b..483694d57b7e6 100644
--- a/compiler-rt/lib/gwp_asan/common.cpp
+++ b/compiler-rt/lib/gwp_asan/common.cpp
@@ -34,6 +34,9 @@ const char *ErrorToString(const Error &E) {
   __builtin_trap();
 }
 
+constexpr size_t AllocationMetadata::kStackFrameStorageBytes;
+constexpr size_t AllocationMetadata::kMaxTraceLengthToCollect;
+
 void AllocationMetadata::RecordAllocation(uintptr_t AllocAddr,
                                           size_t AllocSize) {
   Addr = AllocAddr;
diff --git a/compiler-rt/lib/gwp_asan/crash_handler.cpp b/compiler-rt/lib/gwp_asan/crash_handler.cpp
index c3b9e1467bd9f..3c640256706f5 100644
--- a/compiler-rt/lib/gwp_asan/crash_handler.cpp
+++ b/compiler-rt/lib/gwp_asan/crash_handler.cpp
@@ -10,6 +10,7 @@
 #include "gwp_asan/stack_trace_compressor.h"
 
 #include <assert.h>
+#include <string.h>
 
 using AllocationMetadata = gwp_asan::AllocationMetadata;
 using Error = gwp_asan::Error;
@@ -112,9 +113,15 @@ uint64_t __gwp_asan_get_allocation_thread_id(
 size_t __gwp_asan_get_allocation_trace(
     const gwp_asan::AllocationMetadata *AllocationMeta, uintptr_t *Buffer,
     size_t BufferLen) {
-  return gwp_asan::compression::unpack(
+  uintptr_t UncompressedBuffer[AllocationMetadata::kMaxTraceLengthToCollect];
+  size_t UnpackedLength = gwp_asan::compression::unpack(
       AllocationMeta->AllocationTrace.CompressedTrace,
-      AllocationMeta->AllocationTrace.TraceSize, Buffer, BufferLen);
+      AllocationMeta->AllocationTrace.TraceSize, UncompressedBuffer,
+      AllocationMetadata::kMaxTraceLengthToCollect);
+  if (UnpackedLength < BufferLen)
+    BufferLen = UnpackedLength;
+  memcpy(Buffer, UncompressedBuffer, BufferLen * sizeof(*Buffer));
+  return UnpackedLength;
 }
 
 bool __gwp_asan_is_deallocated(
@@ -130,9 +137,15 @@ uint64_t __gwp_asan_get_deallocation_thread_id(
 size_t __gwp_asan_get_deallocation_trace(
     const gwp_asan::AllocationMetadata *AllocationMeta, uintptr_t *Buffer,
     size_t BufferLen) {
-  return gwp_asan::compression::unpack(
+  uintptr_t UncompressedBuffer[AllocationMetadata::kMaxTraceLengthToCollect];
+  size_t UnpackedLength = gwp_asan::compression::unpack(
       AllocationMeta->DeallocationTrace.CompressedTrace,
-      AllocationMeta->DeallocationTrace.TraceSize, Buffer, BufferLen);
+      AllocationMeta->DeallocationTrace.TraceSize, UncompressedBuffer,
+      AllocationMetadata::kMaxTraceLengthToCollect);
+  if (UnpackedLength < BufferLen)
+    BufferLen = UnpackedLength;
+  memcpy(Buffer, UncompressedBuffer, BufferLen * sizeof(*Buffer));
+  return UnpackedLength;
 }
 
 #ifdef __cplusplus
diff --git a/compiler-rt/lib/gwp_asan/tests/backtrace.cpp b/compiler-rt/lib/gwp_asan/tests/backtrace.cpp
index b3d44270bb2a1..95150653ff61a 100644
--- a/compiler-rt/lib/gwp_asan/tests/backtrace.cpp
+++ b/compiler-rt/lib/gwp_asan/tests/backtrace.cpp
@@ -8,6 +8,7 @@
 
 #include <string>
 
+#include "gwp_asan/common.h"
 #include "gwp_asan/crash_handler.h"
 #include "gwp_asan/tests/harness.h"
 
@@ -76,9 +77,46 @@ TEST(Backtrace, Short) {
 TEST(Backtrace, ExceedsStorableLength) {
   gwp_asan::AllocationMetadata Meta;
   Meta.AllocationTrace.RecordBacktrace(
-      [](uintptr_t * /* TraceBuffer */, size_t /* Size */) -> size_t {
-        return SIZE_MAX; // Wow, that's big!
+      [](uintptr_t *TraceBuffer, size_t Size) -> size_t {
+        // Need to inintialise the elements that will be packed.
+        memset(TraceBuffer, 0u, Size * sizeof(*TraceBuffer));
+
+        // Indicate that there were more frames, and we just didn't have enough
+        // room to store them.
+        return Size * 2;
+      });
+  // Retrieve a frame from the collected backtrace, make sure it works E2E.
+  uintptr_t TraceOutput;
+  EXPECT_EQ(gwp_asan::AllocationMetadata::kMaxTraceLengthToCollect,
+            __gwp_asan_get_allocation_trace(&Meta, &TraceOutput, 1));
+}
+
+TEST(Backtrace, ExceedsRetrievableAllocLength) {
+  gwp_asan::AllocationMetadata Meta;
+  constexpr size_t kNumFramesToStore = 3u;
+  Meta.AllocationTrace.RecordBacktrace(
+      [](uintptr_t *TraceBuffer, size_t /* Size */) -> size_t {
+        memset(TraceBuffer, kNumFramesToStore,
+               kNumFramesToStore * sizeof(*TraceBuffer));
+        return kNumFramesToStore;
+      });
+  uintptr_t TraceOutput;
+  // Ask for one element, get told that there's `kNumFramesToStore` available.
+  EXPECT_EQ(kNumFramesToStore,
+            __gwp_asan_get_allocation_trace(&Meta, &TraceOutput, 1));
+}
+
+TEST(Backtrace, ExceedsRetrievableDeallocLength) {
+  gwp_asan::AllocationMetadata Meta;
+  constexpr size_t kNumFramesToStore = 3u;
+  Meta.DeallocationTrace.RecordBacktrace(
+      [](uintptr_t *TraceBuffer, size_t /* Size */) -> size_t {
+        memset(TraceBuffer, kNumFramesToStore,
+               kNumFramesToStore * sizeof(*TraceBuffer));
+        return kNumFramesToStore;
       });
   uintptr_t TraceOutput;
-  EXPECT_EQ(1u, __gwp_asan_get_allocation_trace(&Meta, &TraceOutput, 1));
+  // Ask for one element, get told that there's `kNumFramesToStore` available.
+  EXPECT_EQ(kNumFramesToStore,
+            __gwp_asan_get_deallocation_trace(&Meta, &TraceOutput, 1));
 }

From 2a672767ccca4525baa323e9911f7a946cc1693d Mon Sep 17 00:00:00 2001
From: Arthur Eubanks <aeubanks@google.com>
Date: Mon, 27 Jul 2020 10:49:16 -0700
Subject: [PATCH 0226/1035] Prefix some AArch64/ARM passes with
 "aarch64-"/"arm-"

 For consistency with other target specific passes.

Reviewed By: arsenm

Differential Revision: https://reviews.llvm.org/D84560
---
 llvm/lib/Target/AArch64/AArch64FalkorHWPFFix.cpp          | 6 +++---
 llvm/lib/Target/AArch64/AArch64StackTagging.cpp           | 2 +-
 llvm/lib/Target/AArch64/AArch64TargetMachine.cpp          | 8 ++++----
 llvm/lib/Target/AArch64/SVEIntrinsicOpts.cpp              | 2 +-
 llvm/lib/Target/ARM/MVEGatherScatterLowering.cpp          | 2 +-
 llvm/test/CodeGen/AArch64/falkor-hwpf-fix.mir             | 2 +-
 llvm/test/CodeGen/AArch64/falkor-hwpf.ll                  | 4 ++--
 llvm/test/CodeGen/AArch64/stack-tagging-dbg.ll            | 2 +-
 llvm/test/CodeGen/AArch64/stack-tagging-ex-1.ll           | 2 +-
 llvm/test/CodeGen/AArch64/stack-tagging-ex-2.ll           | 2 +-
 .../CodeGen/AArch64/stack-tagging-initializer-merge.ll    | 2 +-
 .../test/CodeGen/AArch64/stack-tagging-untag-placement.ll | 2 +-
 llvm/test/CodeGen/AArch64/stack-tagging.ll                | 4 ++--
 llvm/test/CodeGen/AArch64/sve-intrinsic-opts-ptest.ll     | 2 +-
 .../CodeGen/AArch64/sve-intrinsic-opts-reinterpret.ll     | 2 +-
 llvm/test/CodeGen/Thumb2/mve-gather-optimisation-deep.ll  | 2 +-
 16 files changed, 23 insertions(+), 23 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64FalkorHWPFFix.cpp b/llvm/lib/Target/AArch64/AArch64FalkorHWPFFix.cpp
index 538863ebe95af..5d421cfbc2da8 100644
--- a/llvm/lib/Target/AArch64/AArch64FalkorHWPFFix.cpp
+++ b/llvm/lib/Target/AArch64/AArch64FalkorHWPFFix.cpp
@@ -54,7 +54,7 @@
 
 using namespace llvm;
 
-#define DEBUG_TYPE "falkor-hwpf-fix"
+#define DEBUG_TYPE "aarch64-falkor-hwpf-fix"
 
 STATISTIC(NumStridedLoadsMarked, "Number of strided loads marked");
 STATISTIC(NumCollisionsAvoided,
@@ -224,10 +224,10 @@ struct LoadInfo {
 
 char FalkorHWPFFix::ID = 0;
 
-INITIALIZE_PASS_BEGIN(FalkorHWPFFix, "falkor-hwpf-fix-late",
+INITIALIZE_PASS_BEGIN(FalkorHWPFFix, "aarch64-falkor-hwpf-fix-late",
                       "Falkor HW Prefetch Fix Late Phase", false, false)
 INITIALIZE_PASS_DEPENDENCY(MachineLoopInfo)
-INITIALIZE_PASS_END(FalkorHWPFFix, "falkor-hwpf-fix-late",
+INITIALIZE_PASS_END(FalkorHWPFFix, "aarch64-falkor-hwpf-fix-late",
                     "Falkor HW Prefetch Fix Late Phase", false, false)
 
 static unsigned makeTag(unsigned Dest, unsigned Base, unsigned Offset) {
diff --git a/llvm/lib/Target/AArch64/AArch64StackTagging.cpp b/llvm/lib/Target/AArch64/AArch64StackTagging.cpp
index 61f27cbc3b29d..0ed576fbc75f9 100644
--- a/llvm/lib/Target/AArch64/AArch64StackTagging.cpp
+++ b/llvm/lib/Target/AArch64/AArch64StackTagging.cpp
@@ -59,7 +59,7 @@
 
 using namespace llvm;
 
-#define DEBUG_TYPE "stack-tagging"
+#define DEBUG_TYPE "aarch64-stack-tagging"
 
 static cl::opt<bool> ClMergeInit(
     "stack-tagging-merge-init", cl::Hidden, cl::init(true), cl::ZeroOrMore,
diff --git a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
index b0cef9b66e017..40d71def6d09a 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
@@ -148,10 +148,10 @@ static cl::opt<int> EnableGlobalISelAtO(
     cl::desc("Enable GlobalISel at or below an opt level (-1 to disable)"),
     cl::init(0));
 
-static cl::opt<bool> EnableSVEIntrinsicOpts(
-    "aarch64-sve-intrinsic-opts", cl::Hidden,
-    cl::desc("Enable SVE intrinsic opts"),
-    cl::init(true));
+static cl::opt<bool>
+    EnableSVEIntrinsicOpts("aarch64-enable-sve-intrinsic-opts", cl::Hidden,
+                           cl::desc("Enable SVE intrinsic opts"),
+                           cl::init(true));
 
 static cl::opt<bool> EnableFalkorHWPFFix("aarch64-enable-falkor-hwpf-fix",
                                          cl::init(true), cl::Hidden);
diff --git a/llvm/lib/Target/AArch64/SVEIntrinsicOpts.cpp b/llvm/lib/Target/AArch64/SVEIntrinsicOpts.cpp
index 74fe0cdd1ea7f..542d2c9645150 100644
--- a/llvm/lib/Target/AArch64/SVEIntrinsicOpts.cpp
+++ b/llvm/lib/Target/AArch64/SVEIntrinsicOpts.cpp
@@ -37,7 +37,7 @@
 using namespace llvm;
 using namespace llvm::PatternMatch;
 
-#define DEBUG_TYPE "sve-intrinsic-opts"
+#define DEBUG_TYPE "aarch64-sve-intrinsic-opts"
 
 namespace llvm {
 void initializeSVEIntrinsicOptsPass(PassRegistry &);
diff --git a/llvm/lib/Target/ARM/MVEGatherScatterLowering.cpp b/llvm/lib/Target/ARM/MVEGatherScatterLowering.cpp
index 4d7ad6cd60cb9..e0ba7f88db951 100644
--- a/llvm/lib/Target/ARM/MVEGatherScatterLowering.cpp
+++ b/llvm/lib/Target/ARM/MVEGatherScatterLowering.cpp
@@ -44,7 +44,7 @@
 
 using namespace llvm;
 
-#define DEBUG_TYPE "mve-gather-scatter-lowering"
+#define DEBUG_TYPE "arm-mve-gather-scatter-lowering"
 
 cl::opt<bool> EnableMaskedGatherScatters(
     "enable-arm-maskedgatscat", cl::Hidden, cl::init(false),
diff --git a/llvm/test/CodeGen/AArch64/falkor-hwpf-fix.mir b/llvm/test/CodeGen/AArch64/falkor-hwpf-fix.mir
index e37d8be349480..4f144e1ef8bff 100644
--- a/llvm/test/CodeGen/AArch64/falkor-hwpf-fix.mir
+++ b/llvm/test/CodeGen/AArch64/falkor-hwpf-fix.mir
@@ -1,4 +1,4 @@
-# RUN: llc -mtriple=aarch64-linux-gnu -mcpu=falkor -run-pass falkor-hwpf-fix-late -o - %s | FileCheck %s
+# RUN: llc -mtriple=aarch64-linux-gnu -mcpu=falkor -run-pass aarch64-falkor-hwpf-fix-late -o - %s | FileCheck %s
 ---
 # Verify that the tag collision between the loads is resolved for various load opcodes.
 
diff --git a/llvm/test/CodeGen/AArch64/falkor-hwpf.ll b/llvm/test/CodeGen/AArch64/falkor-hwpf.ll
index b9c7291313fb9..aa4a43f2430ba 100644
--- a/llvm/test/CodeGen/AArch64/falkor-hwpf.ll
+++ b/llvm/test/CodeGen/AArch64/falkor-hwpf.ll
@@ -1,5 +1,5 @@
-; RUN: opt < %s -S -falkor-hwpf-fix -mtriple aarch64 -mcpu=falkor | FileCheck %s
-; RUN: opt < %s -S -falkor-hwpf-fix -mtriple aarch64 -mcpu=cortex-a57 | FileCheck %s --check-prefix=NOHWPF
+; RUN: opt < %s -S -aarch64-falkor-hwpf-fix -mtriple aarch64 -mcpu=falkor | FileCheck %s
+; RUN: opt < %s -S -aarch64-falkor-hwpf-fix -mtriple aarch64 -mcpu=cortex-a57 | FileCheck %s --check-prefix=NOHWPF
 
 ; Check that strided access metadata is added to loads in inner loops when compiling for Falkor.
 
diff --git a/llvm/test/CodeGen/AArch64/stack-tagging-dbg.ll b/llvm/test/CodeGen/AArch64/stack-tagging-dbg.ll
index 9ca4ff59b24f2..1525b2d3e3531 100644
--- a/llvm/test/CodeGen/AArch64/stack-tagging-dbg.ll
+++ b/llvm/test/CodeGen/AArch64/stack-tagging-dbg.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -stack-tagging -S -o - | FileCheck %s
+; RUN: opt < %s -aarch64-stack-tagging -S -o - | FileCheck %s
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64--linux-android"
diff --git a/llvm/test/CodeGen/AArch64/stack-tagging-ex-1.ll b/llvm/test/CodeGen/AArch64/stack-tagging-ex-1.ll
index 258586824530c..2099e722fe523 100644
--- a/llvm/test/CodeGen/AArch64/stack-tagging-ex-1.ll
+++ b/llvm/test/CodeGen/AArch64/stack-tagging-ex-1.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -stack-tagging %s -o - | FileCheck %s
+; RUN: opt -S -aarch64-stack-tagging %s -o - | FileCheck %s
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64-arm-unknown-eabi"
diff --git a/llvm/test/CodeGen/AArch64/stack-tagging-ex-2.ll b/llvm/test/CodeGen/AArch64/stack-tagging-ex-2.ll
index 820041e061a3c..11389d5d5ba7c 100644
--- a/llvm/test/CodeGen/AArch64/stack-tagging-ex-2.ll
+++ b/llvm/test/CodeGen/AArch64/stack-tagging-ex-2.ll
@@ -24,7 +24,7 @@
 ;   return 0;
 ; }
 
-; RUN: opt -S -stack-tagging %s -o - | FileCheck %s
+; RUN: opt -S -aarch64-stack-tagging %s -o - | FileCheck %s
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64-unknown-unknown-eabi"
diff --git a/llvm/test/CodeGen/AArch64/stack-tagging-initializer-merge.ll b/llvm/test/CodeGen/AArch64/stack-tagging-initializer-merge.ll
index 5c4be4354ea7b..9dc08c192a01b 100644
--- a/llvm/test/CodeGen/AArch64/stack-tagging-initializer-merge.ll
+++ b/llvm/test/CodeGen/AArch64/stack-tagging-initializer-merge.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -stack-tagging -S -o - | FileCheck %s
+; RUN: opt < %s -aarch64-stack-tagging -S -o - | FileCheck %s
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64--linux-android"
diff --git a/llvm/test/CodeGen/AArch64/stack-tagging-untag-placement.ll b/llvm/test/CodeGen/AArch64/stack-tagging-untag-placement.ll
index 3c0dc3df98dbd..a73c79d6cc985 100644
--- a/llvm/test/CodeGen/AArch64/stack-tagging-untag-placement.ll
+++ b/llvm/test/CodeGen/AArch64/stack-tagging-untag-placement.ll
@@ -1,4 +1,4 @@
-;; RUN:  opt -S -stack-tagging %s -o - | FileCheck %s
+;; RUN:  opt -S -aarch64-stack-tagging %s -o - | FileCheck %s
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64-arm-unknown-eabi"
 
diff --git a/llvm/test/CodeGen/AArch64/stack-tagging.ll b/llvm/test/CodeGen/AArch64/stack-tagging.ll
index 20d3c6bebdec4..275b8a7dbad7e 100644
--- a/llvm/test/CodeGen/AArch64/stack-tagging.ll
+++ b/llvm/test/CodeGen/AArch64/stack-tagging.ll
@@ -1,5 +1,5 @@
-; RUN: opt < %s -stack-tagging -S -o - | FileCheck %s --check-prefixes=CHECK,SSI
-; RUN: opt < %s -stack-tagging -stack-tagging-use-stack-safety=0 -S -o - | FileCheck %s --check-prefixes=CHECK,NOSSI
+; RUN: opt < %s -aarch64-stack-tagging -S -o - | FileCheck %s --check-prefixes=CHECK,SSI
+; RUN: opt < %s -aarch64-stack-tagging -stack-tagging-use-stack-safety=0 -S -o - | FileCheck %s --check-prefixes=CHECK,NOSSI
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64--linux-android"
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsic-opts-ptest.ll b/llvm/test/CodeGen/AArch64/sve-intrinsic-opts-ptest.ll
index 191fddacffd1d..96c754778d52e 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsic-opts-ptest.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsic-opts-ptest.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -sve-intrinsic-opts -mtriple=aarch64-linux-gnu -mattr=+sve < %s 2>%t | FileCheck --check-prefix OPT %s
+; RUN: opt -S -aarch64-sve-intrinsic-opts -mtriple=aarch64-linux-gnu -mattr=+sve < %s 2>%t | FileCheck --check-prefix OPT %s
 ; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t
 
 ; If this check fails please read test/CodeGen/AArch64/README for instructions on how to resolve it.
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsic-opts-reinterpret.ll b/llvm/test/CodeGen/AArch64/sve-intrinsic-opts-reinterpret.ll
index 723ffd8c17330..47e0ff8f19c7f 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsic-opts-reinterpret.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsic-opts-reinterpret.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -sve-intrinsic-opts -mtriple=aarch64-linux-gnu -mattr=+sve < %s 2>%t | FileCheck --check-prefix OPT %s
+; RUN: opt -S -aarch64-sve-intrinsic-opts -mtriple=aarch64-linux-gnu -mattr=+sve < %s 2>%t | FileCheck --check-prefix OPT %s
 ; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t
 
 ; If this check fails please read test/CodeGen/AArch64/README for instructions on how to resolve it.
diff --git a/llvm/test/CodeGen/Thumb2/mve-gather-optimisation-deep.ll b/llvm/test/CodeGen/Thumb2/mve-gather-optimisation-deep.ll
index 22f6cd6bc3d05..e594fee53f9dc 100644
--- a/llvm/test/CodeGen/Thumb2/mve-gather-optimisation-deep.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-gather-optimisation-deep.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 
 
-; RUN: opt --mve-gather-scatter-lowering -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve.fp -enable-arm-maskedgatscat %s -S -o 2>/dev/null - | FileCheck %s
+; RUN: opt --arm-mve-gather-scatter-lowering -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve.fp -enable-arm-maskedgatscat %s -S -o 2>/dev/null - | FileCheck %s
 
 define arm_aapcs_vfpcc void @push_out_add_sub_block(i32* noalias nocapture readonly %data, i32* noalias nocapture %dst, i32 %n.vec) {
 ; CHECK-LABEL: @push_out_add_sub_block(

From 7832d0f63d3210b4a4f0e8bfc968ebe44dba0da5 Mon Sep 17 00:00:00 2001
From: aartbik <ajcbik@google.com>
Date: Mon, 27 Jul 2020 11:22:26 -0700
Subject: [PATCH 0227/1035] [mlir] [VectorOps] [integration_test] Sparse matrix
 times vector (DOT version)

Integration test that illustrates the gather operation with a
real-world operation expressed in mostly the Vector dialect.
Uses jagged diagonal storage.

Reviewed By: bondhugula

Differential Revision: https://reviews.llvm.org/D84571
---
 .../Vector/CPU/test-sparse-dot-matvec.mlir    | 270 ++++++++++++++++++
 1 file changed, 270 insertions(+)
 create mode 100644 mlir/integration_test/Dialect/Vector/CPU/test-sparse-dot-matvec.mlir

diff --git a/mlir/integration_test/Dialect/Vector/CPU/test-sparse-dot-matvec.mlir b/mlir/integration_test/Dialect/Vector/CPU/test-sparse-dot-matvec.mlir
new file mode 100644
index 0000000000000..088e68e0e507c
--- /dev/null
+++ b/mlir/integration_test/Dialect/Vector/CPU/test-sparse-dot-matvec.mlir
@@ -0,0 +1,270 @@
+// RUN: mlir-opt %s -convert-scf-to-std -convert-vector-to-llvm -convert-std-to-llvm | \
+// RUN: mlir-cpu-runner -e entry -entry-point-result=void \
+// RUN:   -shared-libs=%mlir_integration_test_dir/libmlir_c_runner_utils%shlibext | \
+// RUN: FileCheck %s
+
+// Illustrates an 8x8 Sparse Matrix x Vector implemented with only operations
+// of the vector dialect (and some std/scf). Essentially, this example performs
+// the following multiplication:
+//
+//     0  1  2  3  4  5  6  7
+//   +------------------------+
+// 0 | 1  0  2  0  0  1  0  1 |   | 1 |   | 21 |
+// 1 | 1  8  0  0  3  0  1  0 |   | 2 |   | 39 |
+// 2 | 0  0  1  0  0  2  6  2 |   | 3 |   | 73 |
+// 3 | 0  3  0  1  0  1  0  1 | x | 4 | = | 24 |
+// 4 | 5  0  0  1  1  1  0  0 |   | 5 |   | 20 |
+// 5 | 0  3  0  0  2  1  2  0 |   | 6 |   | 36 |
+// 6 | 4  0  7  0  1  0  1  0 |   | 7 |   | 37 |
+// 7 | 0  3  0  2  0  0  1  1 |   | 8 |   | 29 |
+//   +------------------------+
+//
+// The sparse storage scheme used is an extended column scheme (also referred
+// to as jagged diagonal, which is essentially a vector friendly variant of
+// the general sparse row-wise scheme (also called compressed row storage),
+// using fixed length vectors and no explicit pointer indexing into the
+// value array to find the rows.
+//
+// The extended column storage for the matrix shown above is as follows.
+//
+//      VALUE           INDEX
+//   +---------+     +---------+
+// 0 | 1 2 1 1 |     | 0 2 5 7 |
+// 1 | 1 8 3 1 |     | 0 1 4 6 |
+// 2 | 1 2 6 2 |     | 2 5 6 7 |
+// 3 | 3 1 1 1 |     | 1 3 5 7 |
+// 4 | 5 1 1 1 |     | 0 3 4 5 |
+// 5 | 3 2 1 2 |     | 1 4 5 6 |
+// 6 | 4 7 1 1 |     | 0 2 4 6 |
+// 7 | 3 2 1 1 |     | 1 3 6 7 |
+//   +---------+     +---------+
+//
+// This example illustrates a DOT version for the operation. Another example
+// in this directory illustrates an effective SAXPY version that operates on the
+// transposed jagged diagonal storage to obtain higher vector lengths.
+
+#contraction_accesses = [
+  affine_map<(i) -> (i)>,
+  affine_map<(i) -> (i)>,
+  affine_map<(i) -> ()>
+]
+#dot_trait = {
+  indexing_maps = #contraction_accesses,
+  iterator_types = ["reduction"]
+}
+
+func @spmv8x8(%AVAL: memref<8xvector<4xf32>>,
+              %AIDX: memref<8xvector<4xi32>>, %X: memref<?xf32>, %B: memref<?xf32>) {
+  %c0 = constant 0 : index
+  %c1 = constant 1 : index
+  %cn = constant 8 : index
+  %f0 = constant 0.0 : f32
+  %mask = vector.constant_mask [4] : vector<4xi1>
+  scf.for %i = %c0 to %cn step %c1 {
+    %aval = load %AVAL[%i] : memref<8xvector<4xf32>>
+    %aidx = load %AIDX[%i] : memref<8xvector<4xi32>>
+    %0 = vector.gather %X, %aidx, %mask
+       : (memref<?xf32>, vector<4xi32>, vector<4xi1>) -> vector<4xf32>
+    %1 = vector.contract #dot_trait %aval, %0, %f0 : vector<4xf32>, vector<4xf32> into f32
+    store %1, %B[%i] : memref<?xf32>
+  }
+  return
+}
+
+func @entry() {
+  %c0 = constant 0 : index
+  %c1 = constant 1 : index
+  %c2 = constant 2 : index
+  %c3 = constant 3 : index
+  %c4 = constant 4 : index
+  %c5 = constant 5 : index
+  %c6 = constant 6 : index
+  %c7 = constant 7 : index
+  %c8 = constant 8 : index
+
+  %f0 = constant 0.0 : f32
+  %f1 = constant 1.0 : f32
+  %f2 = constant 2.0 : f32
+  %f3 = constant 3.0 : f32
+  %f4 = constant 4.0 : f32
+  %f5 = constant 5.0 : f32
+  %f6 = constant 6.0 : f32
+  %f7 = constant 7.0 : f32
+  %f8 = constant 8.0 : f32
+
+  %i0 = constant 0 : i32
+  %i1 = constant 1 : i32
+  %i2 = constant 2 : i32
+  %i3 = constant 3 : i32
+  %i4 = constant 4 : i32
+  %i5 = constant 5 : i32
+  %i6 = constant 6 : i32
+  %i7 = constant 7 : i32
+
+  //
+  // Allocate.
+  //
+
+  %AVAL = alloc()    {alignment = 64} : memref<8xvector<4xf32>>
+  %AIDX = alloc()    {alignment = 64} : memref<8xvector<4xi32>>
+  %X    = alloc(%c8) {alignment = 64} : memref<?xf32>
+  %B    = alloc(%c8) {alignment = 64} : memref<?xf32>
+
+  //
+  // Initialize.
+  //
+
+  %vf1 = vector.broadcast %f1 : f32 to vector<4xf32>
+
+  %0 = vector.insert %f2, %vf1[1] : f32 into vector<4xf32>
+  store %0, %AVAL[%c0] : memref<8xvector<4xf32>>
+
+  %1 = vector.insert %f8, %vf1[1] : f32 into vector<4xf32>
+  %2 = vector.insert %f3, %1[2]   : f32 into vector<4xf32>
+  store %2, %AVAL[%c1] : memref<8xvector<4xf32>>
+
+  %3 = vector.insert %f2, %vf1[1] : f32 into vector<4xf32>
+  %4 = vector.insert %f6, %3[2]   : f32 into vector<4xf32>
+  %5 = vector.insert %f2, %4[3]   : f32 into vector<4xf32>
+  store %5, %AVAL[%c2] : memref<8xvector<4xf32>>
+
+  %6 = vector.insert %f3, %vf1[0] : f32 into vector<4xf32>
+  store %6, %AVAL[%c3] : memref<8xvector<4xf32>>
+
+  %7 = vector.insert %f5, %vf1[0] : f32 into vector<4xf32>
+  store %7, %AVAL[%c4] : memref<8xvector<4xf32>>
+
+  %8 = vector.insert %f3, %vf1[0] : f32 into vector<4xf32>
+  %9 = vector.insert %f2, %8[1]   : f32 into vector<4xf32>
+  %10 = vector.insert %f2, %9[3]   : f32 into vector<4xf32>
+  store %10, %AVAL[%c5] : memref<8xvector<4xf32>>
+
+  %11 = vector.insert %f4, %vf1[0] : f32 into vector<4xf32>
+  %12 = vector.insert %f7, %11[1]   : f32 into vector<4xf32>
+  store %12, %AVAL[%c6] : memref<8xvector<4xf32>>
+
+  %13 = vector.insert %f3, %vf1[0] : f32 into vector<4xf32>
+  %14 = vector.insert %f2, %13[1]   : f32 into vector<4xf32>
+  store %14, %AVAL[%c7] : memref<8xvector<4xf32>>
+
+  %vi0 = vector.broadcast %i0 : i32 to vector<4xi32>
+
+  %20 = vector.insert %i2, %vi0[1] : i32 into vector<4xi32>
+  %21 = vector.insert %i5, %20[2] : i32 into vector<4xi32>
+  %22 = vector.insert %i7, %21[3] : i32 into vector<4xi32>
+  store %22, %AIDX[%c0] : memref<8xvector<4xi32>>
+
+  %23 = vector.insert %i1, %vi0[1] : i32 into vector<4xi32>
+  %24 = vector.insert %i4, %23[2] : i32 into vector<4xi32>
+  %25 = vector.insert %i6, %24[3] : i32 into vector<4xi32>
+  store %25, %AIDX[%c1] : memref<8xvector<4xi32>>
+
+  %26 = vector.insert %i2, %vi0[0] : i32 into vector<4xi32>
+  %27 = vector.insert %i5, %26[1] : i32 into vector<4xi32>
+  %28 = vector.insert %i6, %27[2] : i32 into vector<4xi32>
+  %29 = vector.insert %i7, %28[3] : i32 into vector<4xi32>
+  store %29, %AIDX[%c2] : memref<8xvector<4xi32>>
+
+  %30 = vector.insert %i1, %vi0[0] : i32 into vector<4xi32>
+  %31 = vector.insert %i3, %30[1] : i32 into vector<4xi32>
+  %32 = vector.insert %i5, %31[2] : i32 into vector<4xi32>
+  %33 = vector.insert %i7, %32[3] : i32 into vector<4xi32>
+  store %33, %AIDX[%c3] : memref<8xvector<4xi32>>
+
+  %34 = vector.insert %i3, %vi0[1] : i32 into vector<4xi32>
+  %35 = vector.insert %i4, %34[2] : i32 into vector<4xi32>
+  %36 = vector.insert %i5, %35[3] : i32 into vector<4xi32>
+  store %36, %AIDX[%c4] : memref<8xvector<4xi32>>
+
+  %37 = vector.insert %i1, %vi0[0] : i32 into vector<4xi32>
+  %38 = vector.insert %i4, %37[1] : i32 into vector<4xi32>
+  %39 = vector.insert %i5, %38[2] : i32 into vector<4xi32>
+  %40 = vector.insert %i6, %39[3] : i32 into vector<4xi32>
+  store %40, %AIDX[%c5] : memref<8xvector<4xi32>>
+
+  %41 = vector.insert %i2, %vi0[1] : i32 into vector<4xi32>
+  %42 = vector.insert %i4, %41[2] : i32 into vector<4xi32>
+  %43 = vector.insert %i6, %42[3] : i32 into vector<4xi32>
+  store %43, %AIDX[%c6] : memref<8xvector<4xi32>>
+
+  %44 = vector.insert %i1, %vi0[0] : i32 into vector<4xi32>
+  %45 = vector.insert %i3, %44[1] : i32 into vector<4xi32>
+  %46 = vector.insert %i6, %45[2] : i32 into vector<4xi32>
+  %47 = vector.insert %i7, %46[3] : i32 into vector<4xi32>
+  store %47, %AIDX[%c7] : memref<8xvector<4xi32>>
+
+  scf.for %i = %c0 to %c8 step %c1 {
+    %ix = addi %i, %c1 : index
+    %kx = index_cast %ix : index to i32
+    %fx = sitofp %kx : i32 to f32
+    store %fx, %X[%i] : memref<?xf32>
+    store %f0, %B[%i] : memref<?xf32>
+  }
+
+  //
+  // Multiply.
+  //
+
+  call @spmv8x8(%AVAL, %AIDX, %X, %B) : (memref<8xvector<4xf32>>,
+                                         memref<8xvector<4xi32>>,
+                                         memref<?xf32>, memref<?xf32>) -> ()
+
+  //
+  // Print and verify.
+  //
+
+  scf.for %i = %c0 to %c8 step %c1 {
+    %aval = load %AVAL[%i] : memref<8xvector<4xf32>>
+    vector.print %aval : vector<4xf32>
+  }
+
+  scf.for %i = %c0 to %c8 step %c1 {
+    %aidx = load %AIDX[%i] : memref<8xvector<4xi32>>
+    vector.print %aidx : vector<4xi32>
+  }
+
+  scf.for %i = %c0 to %c8 step %c1 {
+    %ldb = load %B[%i] : memref<?xf32>
+    vector.print %ldb : f32
+  }
+
+  //
+  // CHECK:      ( 1, 2, 1, 1 )
+  // CHECK-NEXT: ( 1, 8, 3, 1 )
+  // CHECK-NEXT: ( 1, 2, 6, 2 )
+  // CHECK-NEXT: ( 3, 1, 1, 1 )
+  // CHECK-NEXT: ( 5, 1, 1, 1 )
+  // CHECK-NEXT: ( 3, 2, 1, 2 )
+  // CHECK-NEXT: ( 4, 7, 1, 1 )
+  // CHECK-NEXT: ( 3, 2, 1, 1 )
+  //
+  // CHECK-NEXT: ( 0, 2, 5, 7 )
+  // CHECK-NEXT: ( 0, 1, 4, 6 )
+  // CHECK-NEXT: ( 2, 5, 6, 7 )
+  // CHECK-NEXT: ( 1, 3, 5, 7 )
+  // CHECK-NEXT: ( 0, 3, 4, 5 )
+  // CHECK-NEXT: ( 1, 4, 5, 6 )
+  // CHECK-NEXT: ( 0, 2, 4, 6 )
+  // CHECK-NEXT: ( 1, 3, 6, 7 )
+  //
+  // CHECK-NEXT: 21
+  // CHECK-NEXT: 39
+  // CHECK-NEXT: 73
+  // CHECK-NEXT: 24
+  // CHECK-NEXT: 20
+  // CHECK-NEXT: 36
+  // CHECK-NEXT: 37
+  // CHECK-NEXT: 29
+  //
+
+  //
+  // Free.
+  //
+
+  dealloc %AVAL : memref<8xvector<4xf32>>
+  dealloc %AIDX : memref<8xvector<4xi32>>
+  dealloc %X    : memref<?xf32>
+  dealloc %B    : memref<?xf32>
+
+  return
+}

From b52b2e1c188072e3cbc91500cfd503fb26d50ffc Mon Sep 17 00:00:00 2001
From: Dokyung Song <dokyungs@google.com>
Date: Thu, 16 Jul 2020 21:24:06 +0000
Subject: [PATCH 0228/1035] Recommit "[libFuzzer] Disable implicit builtin
 knowledge about memcmp-like functions when -fsanitize=fuzzer-no-link is
 given."

Summary: This patch disables implicit builtin knowledge about memcmp-like functions when compiling the program for fuzzing, i.e., when -fsanitize=fuzzer(-no-link) is given. This allows libFuzzer to always intercept memcmp-like functions as it effectively disables optimizing calls to such functions into different forms. This is done by adding a set of flags (-fno-builtin-memcmp and others) in the clang driver. Individual -fno-builtin-* flags previously used in several libFuzzer tests are now removed, as it is now done automatically in the clang driver.

The patch was once reverted in 8ef9e2bf355d05bc81d8b0fe1e5333eec59a0a91, as this patch was dependent on a reverted commit f78d9fceea736d431e9e3cbca291e3909e3aa46d. This reverted commit was recommitted in 831ae45e3dc609e43ba561af07670a8fe47461ef, so relanding this dependent patch too.

Reviewers: morehouse, hctim

Subscribers: cfe-commits, #sanitizers

Tags: #clang, #sanitizers

Differential Revision: https://reviews.llvm.org/D83987
---
 clang/lib/Driver/SanitizerArgs.cpp           | 17 +++++++++++++++++
 compiler-rt/test/fuzzer/noasan-bcmp.test     |  4 ++++
 compiler-rt/test/fuzzer/noasan-memcmp.test   |  4 ++--
 compiler-rt/test/fuzzer/noasan-memcmp64.test |  2 +-
 compiler-rt/test/fuzzer/noasan-strcmp.test   |  4 ++--
 compiler-rt/test/fuzzer/noasan-strncmp.test  |  4 ++--
 compiler-rt/test/fuzzer/noasan-strstr.test   |  4 ++--
 7 files changed, 30 insertions(+), 9 deletions(-)
 create mode 100644 compiler-rt/test/fuzzer/noasan-bcmp.test

diff --git a/clang/lib/Driver/SanitizerArgs.cpp b/clang/lib/Driver/SanitizerArgs.cpp
index e4fda752c041d..8c49e92b2c0f9 100644
--- a/clang/lib/Driver/SanitizerArgs.cpp
+++ b/clang/lib/Driver/SanitizerArgs.cpp
@@ -1088,6 +1088,23 @@ void SanitizerArgs::addArgs(const ToolChain &TC, const llvm::opt::ArgList &Args,
       Sanitizers.has(SanitizerKind::Address))
     CmdArgs.push_back("-fno-assume-sane-operator-new");
 
+  // libFuzzer wants to intercept calls to certain library functions, so the
+  // following -fno-builtin-* flags force the compiler to emit interposable
+  // libcalls to these functions. Other sanitizers effectively do the same thing
+  // by marking all library call sites with NoBuiltin attribute in their LLVM
+  // pass. (see llvm::maybeMarkSanitizerLibraryCallNoBuiltin)
+  if (Sanitizers.has(SanitizerKind::FuzzerNoLink)) {
+    CmdArgs.push_back("-fno-builtin-bcmp");
+    CmdArgs.push_back("-fno-builtin-memcmp");
+    CmdArgs.push_back("-fno-builtin-strncmp");
+    CmdArgs.push_back("-fno-builtin-strcmp");
+    CmdArgs.push_back("-fno-builtin-strncasecmp");
+    CmdArgs.push_back("-fno-builtin-strcasecmp");
+    CmdArgs.push_back("-fno-builtin-strstr");
+    CmdArgs.push_back("-fno-builtin-strcasestr");
+    CmdArgs.push_back("-fno-builtin-memmem");
+  }
+
   // Require -fvisibility= flag on non-Windows when compiling if vptr CFI is
   // enabled.
   if (Sanitizers.hasOneOf(CFIClasses) && !TC.getTriple().isOSWindows() &&
diff --git a/compiler-rt/test/fuzzer/noasan-bcmp.test b/compiler-rt/test/fuzzer/noasan-bcmp.test
new file mode 100644
index 0000000000000..a3dd17bf6e2b8
--- /dev/null
+++ b/compiler-rt/test/fuzzer/noasan-bcmp.test
@@ -0,0 +1,4 @@
+UNSUPPORTED: darwin, freebsd, windows
+RUN: %cpp_compiler -fno-sanitize=address -DMEMCMP=bcmp %S/MemcmpTest.cpp -o %t
+RUN: not %run %t -seed=1 -runs=10000000 2>&1 | FileCheck %s
+CHECK: BINGO
diff --git a/compiler-rt/test/fuzzer/noasan-memcmp.test b/compiler-rt/test/fuzzer/noasan-memcmp.test
index c90755c53a900..c5ce2fff8c9fa 100644
--- a/compiler-rt/test/fuzzer/noasan-memcmp.test
+++ b/compiler-rt/test/fuzzer/noasan-memcmp.test
@@ -1,9 +1,9 @@
 UNSUPPORTED: darwin, freebsd, windows
 
-RUN: %cpp_compiler -fno-sanitize=address -fno-builtin-memcmp %S/MemcmpTest.cpp -o %t-NoAsanMemcmpTest
+RUN: %cpp_compiler -fno-sanitize=address %S/MemcmpTest.cpp -o %t-NoAsanMemcmpTest
 RUN: not %run %t-NoAsanMemcmpTest -seed=1 -runs=10000000   2>&1 | FileCheck %s
 
-RUN: %cpp_compiler -fno-sanitize=address -fno-builtin-calloc -fno-builtin-memcmp %S/CustomAllocator.cpp %S/MemcmpTest.cpp -o %t-NoAsanCustomAllocatorMemcmpTest
+RUN: %cpp_compiler -fno-sanitize=address -fno-builtin-calloc %S/CustomAllocator.cpp %S/MemcmpTest.cpp -o %t-NoAsanCustomAllocatorMemcmpTest
 RUN: not %run %t-NoAsanCustomAllocatorMemcmpTest -seed=1 -runs=10000000   2>&1 | FileCheck %s
 
 CHECK: BINGO
diff --git a/compiler-rt/test/fuzzer/noasan-memcmp64.test b/compiler-rt/test/fuzzer/noasan-memcmp64.test
index a6b8f88594d03..496ee386193e9 100644
--- a/compiler-rt/test/fuzzer/noasan-memcmp64.test
+++ b/compiler-rt/test/fuzzer/noasan-memcmp64.test
@@ -1,6 +1,6 @@
 UNSUPPORTED: darwin, freebsd, windows
 
-RUN: %cpp_compiler -fno-sanitize=address -fno-builtin-memcmp %S/Memcmp64BytesTest.cpp -o %t-NoAsanMemcmp64BytesTest
+RUN: %cpp_compiler -fno-sanitize=address %S/Memcmp64BytesTest.cpp -o %t-NoAsanMemcmp64BytesTest
 RUN: not %run %t-NoAsanMemcmp64BytesTest -seed=1 -runs=1000000   2>&1 | FileCheck %s
 
 CHECK: BINGO
diff --git a/compiler-rt/test/fuzzer/noasan-strcmp.test b/compiler-rt/test/fuzzer/noasan-strcmp.test
index 76b7c5de7c7bb..c264dec6bea2f 100644
--- a/compiler-rt/test/fuzzer/noasan-strcmp.test
+++ b/compiler-rt/test/fuzzer/noasan-strcmp.test
@@ -1,9 +1,9 @@
 UNSUPPORTED: darwin, freebsd, windows
 
-RUN: %cpp_compiler -fno-sanitize=address -fno-builtin-strcmp %S/StrcmpTest.cpp -o %t-NoAsanStrcmpTest
+RUN: %cpp_compiler -fno-sanitize=address %S/StrcmpTest.cpp -o %t-NoAsanStrcmpTest
 RUN: not %run %t-NoAsanStrcmpTest -seed=1 -runs=2000000   2>&1 | FileCheck %s
 
-RUN: %cpp_compiler -fno-sanitize=address -fno-builtin-calloc -fno-builtin-strcmp %S/CustomAllocator.cpp %S/StrcmpTest.cpp -o %t-NoAsanCustomAllocatorStrcmpTest
+RUN: %cpp_compiler -fno-sanitize=address -fno-builtin-calloc %S/CustomAllocator.cpp %S/StrcmpTest.cpp -o %t-NoAsanCustomAllocatorStrcmpTest
 RUN: not %run %t-NoAsanCustomAllocatorStrcmpTest -seed=1 -runs=2000000   2>&1 | FileCheck %s
 
 CHECK: BINGO
diff --git a/compiler-rt/test/fuzzer/noasan-strncmp.test b/compiler-rt/test/fuzzer/noasan-strncmp.test
index 705781ec39586..dd0f254609221 100644
--- a/compiler-rt/test/fuzzer/noasan-strncmp.test
+++ b/compiler-rt/test/fuzzer/noasan-strncmp.test
@@ -1,9 +1,9 @@
 UNSUPPORTED: darwin, freebsd, windows
 
-RUN: %cpp_compiler -fno-sanitize=address -fno-builtin-strncmp %S/StrncmpTest.cpp -o %t-NoAsanStrncmpTest
+RUN: %cpp_compiler -fno-sanitize=address %S/StrncmpTest.cpp -o %t-NoAsanStrncmpTest
 RUN: not %run %t-NoAsanStrncmpTest -seed=2 -runs=10000000   2>&1 | FileCheck %s
 
-RUN: %cpp_compiler -fno-sanitize=address -fno-builtin-calloc -fno-builtin-strncmp %S/CustomAllocator.cpp %S/StrncmpTest.cpp -o %t-NoAsanCustomAllocatorStrncmpTest
+RUN: %cpp_compiler -fno-sanitize=address -fno-builtin-calloc %S/CustomAllocator.cpp %S/StrncmpTest.cpp -o %t-NoAsanCustomAllocatorStrncmpTest
 RUN: not %run %t-NoAsanCustomAllocatorStrncmpTest -seed=2 -runs=10000000   2>&1 | FileCheck %s
 
 CHECK: BINGO
diff --git a/compiler-rt/test/fuzzer/noasan-strstr.test b/compiler-rt/test/fuzzer/noasan-strstr.test
index f06e903149bd9..e969170bfac98 100644
--- a/compiler-rt/test/fuzzer/noasan-strstr.test
+++ b/compiler-rt/test/fuzzer/noasan-strstr.test
@@ -1,9 +1,9 @@
 UNSUPPORTED: darwin, freebsd, windows
 
-RUN: %cpp_compiler -fno-sanitize=address -fno-builtin-strstr %S/StrstrTest.cpp -o %t-NoAsanStrstrTest
+RUN: %cpp_compiler -fno-sanitize=address %S/StrstrTest.cpp -o %t-NoAsanStrstrTest
 RUN: not %run %t-NoAsanStrstrTest -seed=1 -runs=2000000   2>&1 | FileCheck %s
 
-RUN: %cpp_compiler -fno-sanitize=address -fno-builtin-calloc -fno-builtin-strstr %S/CustomAllocator.cpp %S/StrstrTest.cpp -o %t-NoAsanCustomAllocatorStrstrTest
+RUN: %cpp_compiler -fno-sanitize=address -fno-builtin-calloc %S/CustomAllocator.cpp %S/StrstrTest.cpp -o %t-NoAsanCustomAllocatorStrstrTest
 RUN: not %run %t-NoAsanCustomAllocatorStrstrTest -seed=1 -runs=2000000   2>&1 | FileCheck %s
 
 CHECK: BINGO

From 34ddf0b2b040918a6c946f589eeaf1d4fef95e7a Mon Sep 17 00:00:00 2001
From: Matt Morehouse <mascasa@google.com>
Date: Mon, 27 Jul 2020 18:15:35 +0000
Subject: [PATCH 0229/1035] Replace fuzzer::FuzzerDriver's INTERFACE marking
 with new LLVMRunFuzzerDriver.

This adds a new extern "C" function that serves the same purpose. This removes the need for external users to depend on internal headers in order to use this feature. It also standardizes the interface in a way that other fuzzing engines will be able to match.

Patch By: IanPudney

Reviewed By: kcc

Differential Revision: https://reviews.llvm.org/D84561
---
 compiler-rt/lib/fuzzer/FuzzerDriver.cpp |  6 +++++
 llvm/docs/LibFuzzer.rst                 | 29 +++++++++++++++++++++++++
 2 files changed, 35 insertions(+)

diff --git a/compiler-rt/lib/fuzzer/FuzzerDriver.cpp b/compiler-rt/lib/fuzzer/FuzzerDriver.cpp
index 00a33a413d2f3..8339697396c21 100644
--- a/compiler-rt/lib/fuzzer/FuzzerDriver.cpp
+++ b/compiler-rt/lib/fuzzer/FuzzerDriver.cpp
@@ -858,6 +858,12 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) {
   exit(0);  // Don't let F destroy itself.
 }
 
+extern "C" ATTRIBUTE_INTERFACE int
+LLVMFuzzerRunDriver(int *argc, char ***argv,
+                    int (*UserCb)(const uint8_t *Data, size_t Size)) {
+  return FuzzerDriver(argc, argv, UserCb);
+}
+
 // Storage for global ExternalFunctions object.
 ExternalFunctions *EF = nullptr;
 
diff --git a/llvm/docs/LibFuzzer.rst b/llvm/docs/LibFuzzer.rst
index 4e83955a05460..70a3f029c6f3e 100644
--- a/llvm/docs/LibFuzzer.rst
+++ b/llvm/docs/LibFuzzer.rst
@@ -617,6 +617,35 @@ really need to access ``argv``/``argc``.
     return 0;
    }
 
+Using libFuzzer as a library
+----------------------------
+If the code being fuzzed must provide its own `main`, it's possible to
+invoke libFuzzer as a library. Be sure to pass ``-fsanitize=fuzzer-no-link``
+during compilation, and link your binary against the no-main version of
+libFuzzer. On Linux installations, this is typically located at:
+
+.. code-block:: bash
+
+  /usr/lib/<llvm-version>/lib/clang/<clang-version>/lib/linux/libclang_rt.fuzzer_no_main-<architecture>.a
+
+If building libFuzzer from source, this is located at the following path
+in the build output directory:
+
+.. code-block:: bash
+
+  lib/linux/libclang_rt.fuzzer_no_main-<architecture>.a
+
+From here, the code can do whatever setup it requires, and when it's ready
+to start fuzzing, it can call `LLVMFuzzerRunDriver`, passing in the program
+arguments and a callback. This callback is invoked just like
+`LLVMFuzzerTestOneInput`, and has the same signature.
+
+.. code-block:: c++
+
+  extern "C" int LLVMFuzzerRunDriver(int *argc, char ***argv,
+                    int (*UserCb)(const uint8_t *Data, size_t Size));
+
+
 
 Leaks
 -----

From 731043c0c494672efe1eeea9ee0f0c7788813dea Mon Sep 17 00:00:00 2001
From: Kirill Bobyrev <kbobyrev@google.com>
Date: Mon, 27 Jul 2020 20:45:05 +0200
Subject: [PATCH 0230/1035] [clangd] Add more logs and attach tracers to remote
 index server routines

Reviewers: kadircet

Reviewed By: kadircet

Subscribers: sammccall, ilya-biryukov, MaskRay, jkorous, arphaman, usaxena95, cfe-commits

Tags: #clang

Differential Revision: https://reviews.llvm.org/D84499
---
 .../clangd/index/remote/Client.cpp            |  9 +-
 .../clangd/index/remote/server/Server.cpp     | 89 ++++++++++++++++---
 2 files changed, 83 insertions(+), 15 deletions(-)

diff --git a/clang-tools-extra/clangd/index/remote/Client.cpp b/clang-tools-extra/clangd/index/remote/Client.cpp
index 35ce84068f406..5a33fd2eaf143 100644
--- a/clang-tools-extra/clangd/index/remote/Client.cpp
+++ b/clang-tools-extra/clangd/index/remote/Client.cpp
@@ -37,12 +37,15 @@ class IndexClient : public clangd::SymbolIndex {
     bool FinalResult = false;
     trace::Span Tracer(RequestT::descriptor()->name());
     const auto RPCRequest = ProtobufMarshaller->toProtobuf(Request);
+    SPAN_ATTACH(Tracer, "Request", RPCRequest.DebugString());
     grpc::ClientContext Context;
     std::chrono::system_clock::time_point Deadline =
         std::chrono::system_clock::now() + DeadlineWaitingTime;
     Context.set_deadline(Deadline);
     auto Reader = (Stub.get()->*RPCCall)(&Context, RPCRequest);
     ReplyT Reply;
+    unsigned Successful = 0;
+    unsigned FailedToParse = 0;
     while (Reader->Read(&Reply)) {
       if (!Reply.has_stream_result()) {
         FinalResult = Reply.final_result();
@@ -51,11 +54,15 @@ class IndexClient : public clangd::SymbolIndex {
       auto Response = ProtobufMarshaller->fromProtobuf(Reply.stream_result());
       if (!Response) {
         elog("Received invalid {0}", ReplyT::descriptor()->name());
+        ++FailedToParse;
         continue;
       }
       Callback(*Response);
+      ++Successful;
     }
-    SPAN_ATTACH(Tracer, "status", Reader->Finish().ok());
+    SPAN_ATTACH(Tracer, "Status", Reader->Finish().ok());
+    SPAN_ATTACH(Tracer, "Successful", Successful);
+    SPAN_ATTACH(Tracer, "Failed to parse", FailedToParse);
     return FinalResult;
   }
 
diff --git a/clang-tools-extra/clangd/index/remote/server/Server.cpp b/clang-tools-extra/clangd/index/remote/server/Server.cpp
index 7bf47a288e79c..364e4db6503c2 100644
--- a/clang-tools-extra/clangd/index/remote/server/Server.cpp
+++ b/clang-tools-extra/clangd/index/remote/server/Server.cpp
@@ -6,10 +6,13 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "Index.pb.h"
 #include "index/Index.h"
 #include "index/Serialization.h"
+#include "index/Symbol.h"
 #include "index/remote/marshalling/Marshalling.h"
 #include "support/Logger.h"
+#include "support/Trace.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Path.h"
@@ -36,6 +39,16 @@ llvm::cl::opt<std::string> IndexPath(llvm::cl::desc("<INDEX FILE>"),
 llvm::cl::opt<std::string> IndexRoot(llvm::cl::desc("<PROJECT ROOT>"),
                                      llvm::cl::Positional, llvm::cl::Required);
 
+llvm::cl::opt<std::string> TraceFile(
+    "trace-file",
+    llvm::cl::desc("Path to the file where tracer logs will be stored"));
+
+llvm::cl::opt<bool> PrettyPrint{
+    "pretty",
+    llvm::cl::desc("Pretty-print JSON output in the trace"),
+    llvm::cl::init(false),
+};
+
 llvm::cl::opt<std::string> ServerAddress(
     "server-address", llvm::cl::init("0.0.0.0:50051"),
     llvm::cl::desc("Address of the invoked server. Defaults to 0.0.0.0:50051"));
@@ -60,66 +73,90 @@ class RemoteIndexServer final : public SymbolIndex::Service {
   grpc::Status Lookup(grpc::ServerContext *Context,
                       const LookupRequest *Request,
                       grpc::ServerWriter<LookupReply> *Reply) override {
+    trace::Span Tracer(LookupRequest::descriptor()->name());
     auto Req = ProtobufMarshaller->fromProtobuf(Request);
     if (!Req) {
       elog("Can not parse LookupRequest from protobuf: {0}", Req.takeError());
       return grpc::Status::CANCELLED;
     }
-    Index->lookup(*Req, [&](const clangd::Symbol &Sym) {
-      auto SerializedSymbol = ProtobufMarshaller->toProtobuf(Sym);
-      if (!SerializedSymbol)
+    unsigned Sent = 0;
+    unsigned FailedToSend = 0;
+    Index->lookup(*Req, [&](const auto &Item) {
+      auto SerializedItem = ProtobufMarshaller->toProtobuf(Item);
+      if (!SerializedItem) {
+        ++FailedToSend;
         return;
+      }
       LookupReply NextMessage;
-      *NextMessage.mutable_stream_result() = *SerializedSymbol;
+      *NextMessage.mutable_stream_result() = *SerializedItem;
       Reply->Write(NextMessage);
+      ++Sent;
     });
     LookupReply LastMessage;
     LastMessage.set_final_result(true);
     Reply->Write(LastMessage);
+    SPAN_ATTACH(Tracer, "Sent", Sent);
+    SPAN_ATTACH(Tracer, "Failed to send", FailedToSend);
     return grpc::Status::OK;
   }
 
   grpc::Status FuzzyFind(grpc::ServerContext *Context,
                          const FuzzyFindRequest *Request,
                          grpc::ServerWriter<FuzzyFindReply> *Reply) override {
+    trace::Span Tracer(FuzzyFindRequest::descriptor()->name());
     auto Req = ProtobufMarshaller->fromProtobuf(Request);
     if (!Req) {
       elog("Can not parse FuzzyFindRequest from protobuf: {0}",
            Req.takeError());
       return grpc::Status::CANCELLED;
     }
-    bool HasMore = Index->fuzzyFind(*Req, [&](const clangd::Symbol &Sym) {
-      auto SerializedSymbol = ProtobufMarshaller->toProtobuf(Sym);
-      if (!SerializedSymbol)
+    unsigned Sent = 0;
+    unsigned FailedToSend = 0;
+    bool HasMore = Index->fuzzyFind(*Req, [&](const auto &Item) {
+      auto SerializedItem = ProtobufMarshaller->toProtobuf(Item);
+      if (!SerializedItem) {
+        ++FailedToSend;
         return;
+      }
       FuzzyFindReply NextMessage;
-      *NextMessage.mutable_stream_result() = *SerializedSymbol;
+      *NextMessage.mutable_stream_result() = *SerializedItem;
       Reply->Write(NextMessage);
+      ++Sent;
     });
     FuzzyFindReply LastMessage;
     LastMessage.set_final_result(HasMore);
     Reply->Write(LastMessage);
+    SPAN_ATTACH(Tracer, "Sent", Sent);
+    SPAN_ATTACH(Tracer, "Failed to send", FailedToSend);
     return grpc::Status::OK;
   }
 
   grpc::Status Refs(grpc::ServerContext *Context, const RefsRequest *Request,
                     grpc::ServerWriter<RefsReply> *Reply) override {
+    trace::Span Tracer(RefsRequest::descriptor()->name());
     auto Req = ProtobufMarshaller->fromProtobuf(Request);
     if (!Req) {
       elog("Can not parse RefsRequest from protobuf: {0}", Req.takeError());
       return grpc::Status::CANCELLED;
     }
-    bool HasMore = Index->refs(*Req, [&](const clangd::Ref &Reference) {
-      auto SerializedRef = ProtobufMarshaller->toProtobuf(Reference);
-      if (!SerializedRef)
+    unsigned Sent = 0;
+    unsigned FailedToSend = 0;
+    bool HasMore = Index->refs(*Req, [&](const auto &Item) {
+      auto SerializedItem = ProtobufMarshaller->toProtobuf(Item);
+      if (!SerializedItem) {
+        ++FailedToSend;
         return;
+      }
       RefsReply NextMessage;
-      *NextMessage.mutable_stream_result() = *SerializedRef;
+      *NextMessage.mutable_stream_result() = *SerializedItem;
       Reply->Write(NextMessage);
+      ++Sent;
     });
     RefsReply LastMessage;
     LastMessage.set_final_result(HasMore);
     Reply->Write(LastMessage);
+    SPAN_ATTACH(Tracer, "Sent", Sent);
+    SPAN_ATTACH(Tracer, "Failed to send", FailedToSend);
     return grpc::Status::OK;
   }
 
@@ -146,20 +183,44 @@ void runServer(std::unique_ptr<clangd::SymbolIndex> Index,
 } // namespace clangd
 } // namespace clang
 
+using clang::clangd::elog;
+
 int main(int argc, char *argv[]) {
   using namespace clang::clangd::remote;
   llvm::cl::ParseCommandLineOptions(argc, argv, Overview);
   llvm::sys::PrintStackTraceOnErrorSignal(argv[0]);
 
   if (!llvm::sys::path::is_absolute(IndexRoot)) {
-    llvm::errs() << "Index root should be an absolute path.\n";
+    elog("Index root should be an absolute path.");
     return -1;
   }
 
+  llvm::Optional<llvm::raw_fd_ostream> TracerStream;
+  std::unique_ptr<clang::clangd::trace::EventTracer> Tracer;
+  if (!TraceFile.empty()) {
+    std::error_code EC;
+    TracerStream.emplace(TraceFile, EC,
+                         llvm::sys::fs::FA_Read | llvm::sys::fs::FA_Write);
+    if (EC) {
+      TracerStream.reset();
+      elog("Error while opening trace file {0}: {1}", TraceFile, EC.message());
+    } else {
+      // FIXME(kirillbobyrev): Also create metrics tracer to track latency and
+      // accumulate other request statistics.
+      Tracer = clang::clangd::trace::createJSONTracer(*TracerStream,
+                                                      /*PrettyPrint=*/false);
+      clang::clangd::vlog("Successfully created a tracer.");
+    }
+  }
+
+  llvm::Optional<clang::clangd::trace::Session> TracingSession;
+  if (Tracer)
+    TracingSession.emplace(*Tracer);
+
   std::unique_ptr<clang::clangd::SymbolIndex> Index = openIndex(IndexPath);
 
   if (!Index) {
-    llvm::errs() << "Failed to open the index.\n";
+    elog("Failed to open the index.");
     return -1;
   }
 

From bef19abcf7e2807f8b48b00445b221ff20dd3923 Mon Sep 17 00:00:00 2001
From: Shinji Okumura <okuraofvegetable@gmail.com>
Date: Tue, 28 Jul 2020 03:50:43 +0900
Subject: [PATCH 0231/1035] [Attributor][NFC] Add tests to noalias.ll

Summary: Add tests to `noalias.ll` to make changes in D84665 clear

Reviewers: jdoerfert, sstefan1, uenoku, homerdin, baziotis

Subscribers: uenoku, kuter, bbn, llvm-commits

Tags: #llvm

Differential Revision: https://reviews.llvm.org/D84688
---
 llvm/test/Transforms/Attributor/noalias.ll | 131 +++++++++++++++++++++
 1 file changed, 131 insertions(+)

diff --git a/llvm/test/Transforms/Attributor/noalias.ll b/llvm/test/Transforms/Attributor/noalias.ll
index 71cf2263867cb..d17a98685d72c 100644
--- a/llvm/test/Transforms/Attributor/noalias.ll
+++ b/llvm/test/Transforms/Attributor/noalias.ll
@@ -786,3 +786,134 @@ define void @test16_caller(i32* %p, i32 %c) {
   tail call void @test16_sub(i32* %p, i32 %c, i32 %c)
   ret void
 }
+
+; test 17
+;
+; only_store is not called after make_alias is called.
+;
+; void test17_caller(int* p, int c) {
+;   if(c) {
+;     make_alias(p);
+;     if(0 == 0) {
+;       goto l3;
+;     } else {
+;       goto l2;
+;     }
+;   }
+;   l2:
+;     only_store(p);
+;   l3:
+;   return;
+; }
+
+define void @test17_caller(i32* noalias %p, i32 %c) {
+; NOT_CGSCC_NPM: Function Attrs: nofree nosync nounwind willreturn writeonly
+; NOT_CGSCC_NPM-LABEL: define {{[^@]+}}@test17_caller
+; NOT_CGSCC_NPM-SAME: (i32* noalias nofree writeonly [[P:%.*]], i32 [[C:%.*]])
+; NOT_CGSCC_NPM-NEXT:  entry:
+; NOT_CGSCC_NPM-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[C]], 0
+; NOT_CGSCC_NPM-NEXT:    br i1 [[TOBOOL]], label [[L1:%.*]], label [[L2:%.*]]
+; NOT_CGSCC_NPM:       l1:
+; NOT_CGSCC_NPM-NEXT:    tail call void @make_alias(i32* nofree writeonly [[P]])
+; NOT_CGSCC_NPM-NEXT:    br label [[L3:%.*]]
+; NOT_CGSCC_NPM:       l2:
+; NOT_CGSCC_NPM-NEXT:    tail call void @only_store(i32* nocapture nofree writeonly align 4 [[P]])
+; NOT_CGSCC_NPM-NEXT:    br label [[L3]]
+; NOT_CGSCC_NPM:       l3:
+; NOT_CGSCC_NPM-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind willreturn writeonly
+; IS__CGSCC____-LABEL: define {{[^@]+}}@test17_caller
+; IS__CGSCC____-SAME: (i32* noalias nofree writeonly [[P:%.*]], i32 [[C:%.*]])
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[C]], 0
+; IS__CGSCC____-NEXT:    br i1 [[TOBOOL]], label [[L1:%.*]], label [[L2:%.*]]
+; IS__CGSCC____:       l1:
+; IS__CGSCC____-NEXT:    tail call void @make_alias(i32* nofree writeonly [[P]])
+; IS__CGSCC____-NEXT:    br label [[L3:%.*]]
+; IS__CGSCC____:       l2:
+; IS__CGSCC____-NEXT:    tail call void @only_store(i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[P]])
+; IS__CGSCC____-NEXT:    br label [[L3]]
+; IS__CGSCC____:       l3:
+; IS__CGSCC____-NEXT:    ret void
+;
+entry:
+  %tobool = icmp eq i32 %c, 0
+  br i1 %tobool, label %l1, label %l2
+
+l1:
+  tail call void @make_alias(i32* %p)
+  %tobool2 = icmp eq i32 0, 0
+  br i1 %tobool2, label %l3, label %l2
+
+l2:
+  tail call void @only_store(i32* %p)
+  br label %l3
+
+l3:
+  ret void
+}
+
+; test 18
+; void test18_caller(int* p, int c) {
+;   if(c) {
+;     make_alias(p);
+;     noreturn();
+;   }
+;   only_store(p);
+;   return;
+; }
+
+define void @noreturn() {
+; NOT_CGSCC_NPM: Function Attrs: nofree noreturn nosync nounwind readnone willreturn
+; NOT_CGSCC_NPM-LABEL: define {{[^@]+}}@noreturn()
+; NOT_CGSCC_NPM-NEXT:    unreachable
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse noreturn nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@noreturn()
+; IS__CGSCC____-NEXT:    unreachable
+;
+  call void @noreturn()
+  ret void
+}
+
+define void @test18_caller(i32* noalias %p, i32 %c) {
+; NOT_CGSCC_NPM: Function Attrs: nofree nosync nounwind willreturn writeonly
+; NOT_CGSCC_NPM-LABEL: define {{[^@]+}}@test18_caller
+; NOT_CGSCC_NPM-SAME: (i32* noalias nofree writeonly [[P:%.*]], i32 [[C:%.*]])
+; NOT_CGSCC_NPM-NEXT:  entry:
+; NOT_CGSCC_NPM-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[C]], 0
+; NOT_CGSCC_NPM-NEXT:    br i1 [[TOBOOL]], label [[L1:%.*]], label [[L2:%.*]]
+; NOT_CGSCC_NPM:       l1:
+; NOT_CGSCC_NPM-NEXT:    tail call void @make_alias(i32* nofree writeonly [[P]])
+; NOT_CGSCC_NPM-NEXT:    unreachable
+; NOT_CGSCC_NPM:       l2:
+; NOT_CGSCC_NPM-NEXT:    tail call void @only_store(i32* nocapture nofree writeonly align 4 [[P]])
+; NOT_CGSCC_NPM-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind willreturn writeonly
+; IS__CGSCC____-LABEL: define {{[^@]+}}@test18_caller
+; IS__CGSCC____-SAME: (i32* noalias nofree nonnull writeonly align 4 dereferenceable(4) [[P:%.*]], i32 [[C:%.*]])
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[C]], 0
+; IS__CGSCC____-NEXT:    br i1 [[TOBOOL]], label [[L1:%.*]], label [[L2:%.*]]
+; IS__CGSCC____:       l1:
+; IS__CGSCC____-NEXT:    tail call void @make_alias(i32* nofree nonnull writeonly align 4 dereferenceable(4) [[P]])
+; IS__CGSCC____-NEXT:    unreachable
+; IS__CGSCC____:       l2:
+; IS__CGSCC____-NEXT:    tail call void @only_store(i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[P]])
+; IS__CGSCC____-NEXT:    ret void
+;
+entry:
+  %tobool = icmp eq i32 %c, 0
+  br i1 %tobool, label %l1, label %l2
+
+l1:
+  tail call void @make_alias(i32* %p)
+  tail call void @noreturn()
+  br label %l2
+
+l2:
+  tail call void @only_store(i32* %p)
+  ret void
+}

From 05ad8e942996f36cc694478542ccd84aa5bbb80f Mon Sep 17 00:00:00 2001
From: Xiangling Liao <Xiangling.Liao@ibm.com>
Date: Tue, 2 Jun 2020 11:51:58 -0400
Subject: [PATCH 0232/1035] [AIX] Implement AIX special alignment rule about
 double/long double

Implement AIX default `power` alignment rule by adding `PreferredAlignment` and
`PreferredNVAlignment` in ASTRecordLayout class.

The patchh aims at returning correct value for `__alignof(x)` and `alignof(x)`
under `power` alignment rules.

Differential Revision: https://reviews.llvm.org/D79719
---
 clang/include/clang/AST/RecordLayout.h        |  61 ++-
 clang/include/clang/Basic/TargetInfo.h        |   3 +
 clang/lib/AST/ASTContext.cpp                  |  25 +-
 clang/lib/AST/RecordLayout.cpp                |  41 +-
 clang/lib/AST/RecordLayoutBuilder.cpp         | 308 +++++++++----
 clang/lib/Basic/Targets/OSTargets.h           |   2 +
 clang/lib/Basic/Targets/PPC.h                 |  11 +-
 .../aix-Wpacked-expecting-diagnostics.cpp     |  30 ++
 .../Layout/aix-Wpacked-no-diagnostics.cpp     |  31 ++
 .../test/Layout/aix-double-struct-member.cpp  | 428 ++++++++++++++++++
 .../aix-no-unique-address-with-double.cpp     | 158 +++++++
 clang/test/Layout/aix-pack-attr-on-base.cpp   |  20 +
 .../Layout/aix-power-alignment-typedef-2.cpp  |  15 +
 .../Layout/aix-power-alignment-typedef.cpp    |  39 ++
 ...-virtual-function-and-base-with-double.cpp | 112 +++++
 15 files changed, 1148 insertions(+), 136 deletions(-)
 create mode 100644 clang/test/Layout/aix-Wpacked-expecting-diagnostics.cpp
 create mode 100644 clang/test/Layout/aix-Wpacked-no-diagnostics.cpp
 create mode 100644 clang/test/Layout/aix-double-struct-member.cpp
 create mode 100644 clang/test/Layout/aix-no-unique-address-with-double.cpp
 create mode 100644 clang/test/Layout/aix-pack-attr-on-base.cpp
 create mode 100644 clang/test/Layout/aix-power-alignment-typedef-2.cpp
 create mode 100644 clang/test/Layout/aix-power-alignment-typedef.cpp
 create mode 100644 clang/test/Layout/aix-virtual-function-and-base-with-double.cpp

diff --git a/clang/include/clang/AST/RecordLayout.h b/clang/include/clang/AST/RecordLayout.h
index b259791af509d..946fbd8f4ce24 100644
--- a/clang/include/clang/AST/RecordLayout.h
+++ b/clang/include/clang/AST/RecordLayout.h
@@ -70,6 +70,11 @@ class ASTRecordLayout {
   // Alignment - Alignment of record in characters.
   CharUnits Alignment;
 
+  // PreferredAlignment - Preferred alignment of record in characters. This
+  // can be different than Alignment in cases where it is beneficial for
+  // performance or backwards compatibility preserving (e.g. AIX-ABI).
+  CharUnits PreferredAlignment;
+
   // UnadjustedAlignment - Maximum of the alignments of the record members in
   // characters.
   CharUnits UnadjustedAlignment;
@@ -91,6 +96,11 @@ class ASTRecordLayout {
     /// which is the alignment of the object without virtual bases.
     CharUnits NonVirtualAlignment;
 
+    /// PreferredNVAlignment - The preferred non-virtual alignment (in chars) of
+    /// an object, which is the preferred alignment of the object without
+    /// virtual bases.
+    CharUnits PreferredNVAlignment;
+
     /// SizeOfLargestEmptySubobject - The size of the largest empty subobject
     /// (either a base or a member). Will be zero if the class doesn't contain
     /// any empty subobjects.
@@ -139,30 +149,26 @@ class ASTRecordLayout {
   CXXRecordLayoutInfo *CXXInfo = nullptr;
 
   ASTRecordLayout(const ASTContext &Ctx, CharUnits size, CharUnits alignment,
-                  CharUnits unadjustedAlignment,
+                  CharUnits preferredAlignment, CharUnits unadjustedAlignment,
                   CharUnits requiredAlignment, CharUnits datasize,
                   ArrayRef<uint64_t> fieldoffsets);
 
   using BaseOffsetsMapTy = CXXRecordLayoutInfo::BaseOffsetsMapTy;
 
   // Constructor for C++ records.
-  ASTRecordLayout(const ASTContext &Ctx,
-                  CharUnits size, CharUnits alignment,
-                  CharUnits unadjustedAlignment,
-                  CharUnits requiredAlignment,
-                  bool hasOwnVFPtr, bool hasExtendableVFPtr,
-                  CharUnits vbptroffset,
-                  CharUnits datasize,
-                  ArrayRef<uint64_t> fieldoffsets,
+  ASTRecordLayout(const ASTContext &Ctx, CharUnits size, CharUnits alignment,
+                  CharUnits preferredAlignment, CharUnits unadjustedAlignment,
+                  CharUnits requiredAlignment, bool hasOwnVFPtr,
+                  bool hasExtendableVFPtr, CharUnits vbptroffset,
+                  CharUnits datasize, ArrayRef<uint64_t> fieldoffsets,
                   CharUnits nonvirtualsize, CharUnits nonvirtualalignment,
+                  CharUnits preferrednvalignment,
                   CharUnits SizeOfLargestEmptySubobject,
-                  const CXXRecordDecl *PrimaryBase,
-                  bool IsPrimaryBaseVirtual,
+                  const CXXRecordDecl *PrimaryBase, bool IsPrimaryBaseVirtual,
                   const CXXRecordDecl *BaseSharingVBPtr,
-                  bool EndsWithZeroSizedObject,
-                  bool LeadsWithZeroSizedBase,
-                  const BaseOffsetsMapTy& BaseOffsets,
-                  const VBaseOffsetsMapTy& VBaseOffsets);
+                  bool EndsWithZeroSizedObject, bool LeadsWithZeroSizedBase,
+                  const BaseOffsetsMapTy &BaseOffsets,
+                  const VBaseOffsetsMapTy &VBaseOffsets);
 
   ~ASTRecordLayout() = default;
 
@@ -175,6 +181,10 @@ class ASTRecordLayout {
   /// getAlignment - Get the record alignment in characters.
   CharUnits getAlignment() const { return Alignment; }
 
+  /// getPreferredFieldAlignment - Get the record preferred alignment in
+  /// characters.
+  CharUnits getPreferredAlignment() const { return PreferredAlignment; }
+
   /// getUnadjustedAlignment - Get the record alignment in characters, before
   /// alignment adjustement.
   CharUnits getUnadjustedAlignment() const { return UnadjustedAlignment; }
@@ -193,9 +203,7 @@ class ASTRecordLayout {
 
   /// getDataSize() - Get the record data size, which is the record size
   /// without tail padding, in characters.
-  CharUnits getDataSize() const {
-    return DataSize;
-  }
+  CharUnits getDataSize() const { return DataSize; }
 
   /// getNonVirtualSize - Get the non-virtual size (in chars) of an object,
   /// which is the size of the object without virtual bases.
@@ -205,14 +213,23 @@ class ASTRecordLayout {
     return CXXInfo->NonVirtualSize;
   }
 
-  /// getNonVirtualSize - Get the non-virtual alignment (in chars) of an object,
-  /// which is the alignment of the object without virtual bases.
+  /// getNonVirtualAlignment - Get the non-virtual alignment (in chars) of an
+  /// object, which is the alignment of the object without virtual bases.
   CharUnits getNonVirtualAlignment() const {
     assert(CXXInfo && "Record layout does not have C++ specific info!");
 
     return CXXInfo->NonVirtualAlignment;
   }
 
+  /// getPreferredNVAlignment - Get the preferred non-virtual alignment (in
+  /// chars) of an object, which is the preferred alignment of the object
+  /// without virtual bases.
+  CharUnits getPreferredNVAlignment() const {
+    assert(CXXInfo && "Record layout does not have C++ specific info!");
+
+    return CXXInfo->PreferredNVAlignment;
+  }
+
   /// getPrimaryBase - Get the primary base for this record.
   const CXXRecordDecl *getPrimaryBase() const {
     assert(CXXInfo && "Record layout does not have C++ specific info!");
@@ -287,9 +304,7 @@ class ASTRecordLayout {
     return !CXXInfo->VBPtrOffset.isNegative();
   }
 
-  CharUnits getRequiredAlignment() const {
-    return RequiredAlignment;
-  }
+  CharUnits getRequiredAlignment() const { return RequiredAlignment; }
 
   bool endsWithZeroSizedObject() const {
     return CXXInfo && CXXInfo->EndsWithZeroSizedObject;
diff --git a/clang/include/clang/Basic/TargetInfo.h b/clang/include/clang/Basic/TargetInfo.h
index 2ee3b16596302..004990ee31528 100644
--- a/clang/include/clang/Basic/TargetInfo.h
+++ b/clang/include/clang/Basic/TargetInfo.h
@@ -1401,6 +1401,9 @@ class TargetInfo : public virtual TransferrableTargetInfo,
   /// Whether target allows to overalign ABI-specified preferred alignment
   virtual bool allowsLargerPreferedTypeAlignment() const { return true; }
 
+  /// Whether target defaults to the `power` alignment rules of AIX.
+  virtual bool defaultsToAIXPowerAlignment() const { return false; }
+
   /// Set supported OpenCL extensions and optional core features.
   virtual void setSupportedOpenCLOpts() {}
 
diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index fc7631712c3cf..e7518a538fe67 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -2439,8 +2439,8 @@ CharUnits ASTContext::getTypeUnadjustedAlignInChars(const Type *T) const {
 
 /// getPreferredTypeAlign - Return the "preferred" alignment of the specified
 /// type for the current target in bits.  This can be different than the ABI
-/// alignment in cases where it is beneficial for performance to overalign
-/// a data type.
+/// alignment in cases where it is beneficial for performance or backwards
+/// compatibility preserving to overalign a data type.
 unsigned ASTContext::getPreferredTypeAlign(const Type *T) const {
   TypeInfo TI = getTypeInfo(T);
   unsigned ABIAlign = TI.Align;
@@ -2450,18 +2450,33 @@ unsigned ASTContext::getPreferredTypeAlign(const Type *T) const {
   // The preferred alignment of member pointers is that of a pointer.
   if (T->isMemberPointerType())
     return getPreferredTypeAlign(getPointerDiffType().getTypePtr());
-
+ 
   if (!Target->allowsLargerPreferedTypeAlignment())
     return ABIAlign;
 
-  // Double and long long should be naturally aligned if possible.
+  if (const auto *RT = T->getAs<RecordType>()) {
+    if (TI.AlignIsRequired)
+      return ABIAlign;
+
+    unsigned PreferredAlign = static_cast<unsigned>(
+        toBits(getASTRecordLayout(RT->getDecl()).PreferredAlignment));
+    assert(PreferredAlign >= ABIAlign &&
+           "PreferredAlign should be at least as large as ABIAlign.");
+    return PreferredAlign;
+  }
+
+  // Double (and, for targets supporting AIX `power` alignment, long double) and
+  // long long should be naturally aligned (despite requiring less alignment) if
+  // possible.
   if (const auto *CT = T->getAs<ComplexType>())
     T = CT->getElementType().getTypePtr();
   if (const auto *ET = T->getAs<EnumType>())
     T = ET->getDecl()->getIntegerType().getTypePtr();
   if (T->isSpecificBuiltinType(BuiltinType::Double) ||
       T->isSpecificBuiltinType(BuiltinType::LongLong) ||
-      T->isSpecificBuiltinType(BuiltinType::ULongLong))
+      T->isSpecificBuiltinType(BuiltinType::ULongLong) ||
+      (T->isSpecificBuiltinType(BuiltinType::LongDouble) &&
+       Target->defaultsToAIXPowerAlignment()))
     // Don't increase the alignment if an alignment attribute was specified on a
     // typedef declaration.
     if (!TI.AlignIsRequired)
diff --git a/clang/lib/AST/RecordLayout.cpp b/clang/lib/AST/RecordLayout.cpp
index e7b500e1902d7..8f70a20729262 100644
--- a/clang/lib/AST/RecordLayout.cpp
+++ b/clang/lib/AST/RecordLayout.cpp
@@ -29,45 +29,42 @@ void ASTRecordLayout::Destroy(ASTContext &Ctx) {
 
 ASTRecordLayout::ASTRecordLayout(const ASTContext &Ctx, CharUnits size,
                                  CharUnits alignment,
+                                 CharUnits preferredAlignment,
                                  CharUnits unadjustedAlignment,
                                  CharUnits requiredAlignment,
                                  CharUnits datasize,
                                  ArrayRef<uint64_t> fieldoffsets)
     : Size(size), DataSize(datasize), Alignment(alignment),
+      PreferredAlignment(preferredAlignment),
       UnadjustedAlignment(unadjustedAlignment),
       RequiredAlignment(requiredAlignment) {
   FieldOffsets.append(Ctx, fieldoffsets.begin(), fieldoffsets.end());
 }
 
 // Constructor for C++ records.
-ASTRecordLayout::ASTRecordLayout(const ASTContext &Ctx,
-                                 CharUnits size, CharUnits alignment,
-                                 CharUnits unadjustedAlignment,
-                                 CharUnits requiredAlignment,
-                                 bool hasOwnVFPtr, bool hasExtendableVFPtr,
-                                 CharUnits vbptroffset,
-                                 CharUnits datasize,
-                                 ArrayRef<uint64_t> fieldoffsets,
-                                 CharUnits nonvirtualsize,
-                                 CharUnits nonvirtualalignment,
-                                 CharUnits SizeOfLargestEmptySubobject,
-                                 const CXXRecordDecl *PrimaryBase,
-                                 bool IsPrimaryBaseVirtual,
-                                 const CXXRecordDecl *BaseSharingVBPtr,
-                                 bool EndsWithZeroSizedObject,
-                                 bool LeadsWithZeroSizedBase,
-                                 const BaseOffsetsMapTy& BaseOffsets,
-                                 const VBaseOffsetsMapTy& VBaseOffsets)
-  : Size(size), DataSize(datasize), Alignment(alignment),
-    UnadjustedAlignment(unadjustedAlignment),
-    RequiredAlignment(requiredAlignment), CXXInfo(new (Ctx) CXXRecordLayoutInfo)
-{
+ASTRecordLayout::ASTRecordLayout(
+    const ASTContext &Ctx, CharUnits size, CharUnits alignment,
+    CharUnits preferredAlignment, CharUnits unadjustedAlignment,
+    CharUnits requiredAlignment, bool hasOwnVFPtr, bool hasExtendableVFPtr,
+    CharUnits vbptroffset, CharUnits datasize, ArrayRef<uint64_t> fieldoffsets,
+    CharUnits nonvirtualsize, CharUnits nonvirtualalignment,
+    CharUnits preferrednvalignment, CharUnits SizeOfLargestEmptySubobject,
+    const CXXRecordDecl *PrimaryBase, bool IsPrimaryBaseVirtual,
+    const CXXRecordDecl *BaseSharingVBPtr, bool EndsWithZeroSizedObject,
+    bool LeadsWithZeroSizedBase, const BaseOffsetsMapTy &BaseOffsets,
+    const VBaseOffsetsMapTy &VBaseOffsets)
+    : Size(size), DataSize(datasize), Alignment(alignment),
+      PreferredAlignment(preferredAlignment),
+      UnadjustedAlignment(unadjustedAlignment),
+      RequiredAlignment(requiredAlignment),
+      CXXInfo(new (Ctx) CXXRecordLayoutInfo) {
   FieldOffsets.append(Ctx, fieldoffsets.begin(), fieldoffsets.end());
 
   CXXInfo->PrimaryBase.setPointer(PrimaryBase);
   CXXInfo->PrimaryBase.setInt(IsPrimaryBaseVirtual);
   CXXInfo->NonVirtualSize = nonvirtualsize;
   CXXInfo->NonVirtualAlignment = nonvirtualalignment;
+  CXXInfo->PreferredNVAlignment = preferrednvalignment;
   CXXInfo->SizeOfLargestEmptySubobject = SizeOfLargestEmptySubobject;
   CXXInfo->BaseOffsets = BaseOffsets;
   CXXInfo->VBaseOffsets = VBaseOffsets;
diff --git a/clang/lib/AST/RecordLayoutBuilder.cpp b/clang/lib/AST/RecordLayoutBuilder.cpp
index d56c7e2ab8c0e..0afe91b446ee4 100644
--- a/clang/lib/AST/RecordLayoutBuilder.cpp
+++ b/clang/lib/AST/RecordLayoutBuilder.cpp
@@ -6,7 +6,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/AST/RecordLayout.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/ASTDiagnostic.h"
 #include "clang/AST/Attr.h"
@@ -16,6 +15,7 @@
 #include "clang/AST/DeclObjC.h"
 #include "clang/AST/Expr.h"
 #include "clang/AST/VTableBuilder.h"
+#include "clang/AST/RecordLayout.h"
 #include "clang/Basic/TargetInfo.h"
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/Support/Format.h"
@@ -589,6 +589,9 @@ class ItaniumRecordLayoutBuilder {
   /// Alignment - The current alignment of the record layout.
   CharUnits Alignment;
 
+  /// PreferredAlignment - The preferred alignment of the record layout.
+  CharUnits PreferredAlignment;
+
   /// The alignment if attribute packed is not used.
   CharUnits UnpackedAlignment;
 
@@ -632,6 +635,7 @@ class ItaniumRecordLayoutBuilder {
 
   CharUnits NonVirtualSize;
   CharUnits NonVirtualAlignment;
+  CharUnits PreferredNVAlignment;
 
   /// If we've laid out a field but not included its tail padding in Size yet,
   /// this is the size up to the end of that field.
@@ -652,6 +656,12 @@ class ItaniumRecordLayoutBuilder {
   /// the flag of field offset changing due to packed attribute.
   bool HasPackedField;
 
+  /// HandledFirstNonOverlappingEmptyField - An auxiliary field used for AIX.
+  /// When there are OverlappingEmptyFields existing in the aggregate, the
+  /// flag shows if the following first non-empty or empty-but-non-overlapping
+  /// field has been handled, if any.
+  bool HandledFirstNonOverlappingEmptyField;
+
   typedef llvm::DenseMap<const CXXRecordDecl *, CharUnits> BaseOffsetsMapTy;
 
   /// Bases - base classes and their offsets in the record.
@@ -678,17 +688,19 @@ class ItaniumRecordLayoutBuilder {
   ItaniumRecordLayoutBuilder(const ASTContext &Context,
                              EmptySubobjectMap *EmptySubobjects)
       : Context(Context), EmptySubobjects(EmptySubobjects), Size(0),
-        Alignment(CharUnits::One()), UnpackedAlignment(CharUnits::One()),
-        UnadjustedAlignment(CharUnits::One()),
-        UseExternalLayout(false), InferAlignment(false), Packed(false),
-        IsUnion(false), IsMac68kAlign(false), IsMsStruct(false),
-        UnfilledBitsInLastUnit(0), LastBitfieldTypeSize(0),
-        MaxFieldAlignment(CharUnits::Zero()), DataSize(0),
-        NonVirtualSize(CharUnits::Zero()),
+        Alignment(CharUnits::One()), PreferredAlignment(CharUnits::One()),
+        UnpackedAlignment(CharUnits::One()),
+        UnadjustedAlignment(CharUnits::One()), UseExternalLayout(false),
+        InferAlignment(false), Packed(false), IsUnion(false),
+        IsMac68kAlign(false), IsMsStruct(false), UnfilledBitsInLastUnit(0),
+        LastBitfieldTypeSize(0), MaxFieldAlignment(CharUnits::Zero()),
+        DataSize(0), NonVirtualSize(CharUnits::Zero()),
         NonVirtualAlignment(CharUnits::One()),
+        PreferredNVAlignment(CharUnits::One()),
         PaddedFieldSize(CharUnits::Zero()), PrimaryBase(nullptr),
-        PrimaryBaseIsVirtual(false), HasOwnVFPtr(false),
-        HasPackedField(false), FirstNearlyEmptyVBase(nullptr) {}
+        PrimaryBaseIsVirtual(false), HasOwnVFPtr(false), HasPackedField(false),
+        HandledFirstNonOverlappingEmptyField(false),
+        FirstNearlyEmptyVBase(nullptr) {}
 
   void Layout(const RecordDecl *D);
   void Layout(const CXXRecordDecl *D);
@@ -763,9 +775,13 @@ class ItaniumRecordLayoutBuilder {
   /// alignment.
   void FinishLayout(const NamedDecl *D);
 
-  void UpdateAlignment(CharUnits NewAlignment, CharUnits UnpackedNewAlignment);
+  void UpdateAlignment(CharUnits NewAlignment, CharUnits UnpackedNewAlignment,
+                       CharUnits PreferredAlignment);
+  void UpdateAlignment(CharUnits NewAlignment, CharUnits UnpackedNewAlignment) {
+    UpdateAlignment(NewAlignment, UnpackedNewAlignment, NewAlignment);
+  }
   void UpdateAlignment(CharUnits NewAlignment) {
-    UpdateAlignment(NewAlignment, NewAlignment);
+    UpdateAlignment(NewAlignment, NewAlignment, NewAlignment);
   }
 
   /// Retrieve the externally-supplied field offset for the given
@@ -998,7 +1014,7 @@ void ItaniumRecordLayoutBuilder::EnsureVTablePointerAlignment(
   setSize(getSize().alignTo(BaseAlign));
 
   // Update the alignment.
-  UpdateAlignment(BaseAlign, UnpackedBaseAlign);
+  UpdateAlignment(BaseAlign, UnpackedBaseAlign, BaseAlign);
 }
 
 void ItaniumRecordLayoutBuilder::LayoutNonVirtualBases(
@@ -1044,6 +1060,10 @@ void ItaniumRecordLayoutBuilder::LayoutNonVirtualBases(
       Context.toCharUnitsFromBits(Context.getTargetInfo().getPointerAlign(0));
     EnsureVTablePointerAlignment(PtrAlign);
     HasOwnVFPtr = true;
+
+    assert(!IsUnion && "Unions cannot be dynamic classes.");
+    HandledFirstNonOverlappingEmptyField = true;
+
     setSize(getSize() + PtrWidth);
     setDataSize(getSize());
   }
@@ -1179,9 +1199,9 @@ void ItaniumRecordLayoutBuilder::LayoutVirtualBase(
 
 CharUnits
 ItaniumRecordLayoutBuilder::LayoutBase(const BaseSubobjectInfo *Base) {
-  const ASTRecordLayout &Layout = Context.getASTRecordLayout(Base->Class);
-
+  assert(!IsUnion && "Unions cannot have base classes.");
 
+  const ASTRecordLayout &Layout = Context.getASTRecordLayout(Base->Class);
   CharUnits Offset;
 
   // Query the external layout to see if it provides an offset.
@@ -1193,45 +1213,77 @@ ItaniumRecordLayoutBuilder::LayoutBase(const BaseSubobjectInfo *Base) {
       HasExternalLayout = External.getExternalNVBaseOffset(Base->Class, Offset);
   }
 
-  // Clang <= 6 incorrectly applied the 'packed' attribute to base classes.
-  // Per GCC's documentation, it only applies to non-static data members.
+  auto getBaseOrPreferredBaseAlignFromUnpacked = [&](CharUnits UnpackedAlign) {
+    // Clang <= 6 incorrectly applied the 'packed' attribute to base classes.
+    // Per GCC's documentation, it only applies to non-static data members.
+    return (Packed && ((Context.getLangOpts().getClangABICompat() <=
+                        LangOptions::ClangABI::Ver6) ||
+                       Context.getTargetInfo().getTriple().isPS4() ||
+                       Context.getTargetInfo().getTriple().isOSAIX()))
+               ? CharUnits::One()
+               : UnpackedAlign;
+  };
+
   CharUnits UnpackedBaseAlign = Layout.getNonVirtualAlignment();
+  CharUnits UnpackedPreferredBaseAlign = Layout.getPreferredNVAlignment();
   CharUnits BaseAlign =
-      (Packed && ((Context.getLangOpts().getClangABICompat() <=
-                   LangOptions::ClangABI::Ver6) ||
-                  Context.getTargetInfo().getTriple().isPS4()))
-          ? CharUnits::One()
-          : UnpackedBaseAlign;
+      getBaseOrPreferredBaseAlignFromUnpacked(UnpackedBaseAlign);
+  CharUnits PreferredBaseAlign =
+      getBaseOrPreferredBaseAlignFromUnpacked(UnpackedPreferredBaseAlign);
+
+  const bool DefaultsToAIXPowerAlignment =
+      Context.getTargetInfo().defaultsToAIXPowerAlignment();
+  if (DefaultsToAIXPowerAlignment) {
+    // AIX `power` alignment does not apply the preferred alignment for
+    // non-union classes if the source of the alignment (the current base in
+    // this context) follows introduction of the first subobject with
+    // exclusively allocated space or zero-extent array.
+    if (!Base->Class->isEmpty() && !HandledFirstNonOverlappingEmptyField) {
+      // By handling a base class that is not empty, we're handling the
+      // "first (inherited) member".
+      HandledFirstNonOverlappingEmptyField = true;
+    } else {
+      UnpackedPreferredBaseAlign = UnpackedBaseAlign;
+      PreferredBaseAlign = BaseAlign;
+    }
+  }
 
+  CharUnits UnpackedAlignTo = !DefaultsToAIXPowerAlignment
+                                  ? UnpackedBaseAlign
+                                  : UnpackedPreferredBaseAlign;
   // If we have an empty base class, try to place it at offset 0.
   if (Base->Class->isEmpty() &&
       (!HasExternalLayout || Offset == CharUnits::Zero()) &&
       EmptySubobjects->CanPlaceBaseAtOffset(Base, CharUnits::Zero())) {
     setSize(std::max(getSize(), Layout.getSize()));
-    UpdateAlignment(BaseAlign, UnpackedBaseAlign);
+    UpdateAlignment(BaseAlign, UnpackedAlignTo, PreferredBaseAlign);
 
     return CharUnits::Zero();
   }
 
-  // The maximum field alignment overrides base align.
+  // The maximum field alignment overrides the base align/(AIX-only) preferred
+  // base align.
   if (!MaxFieldAlignment.isZero()) {
     BaseAlign = std::min(BaseAlign, MaxFieldAlignment);
-    UnpackedBaseAlign = std::min(UnpackedBaseAlign, MaxFieldAlignment);
+    PreferredBaseAlign = std::min(PreferredBaseAlign, MaxFieldAlignment);
+    UnpackedAlignTo = std::min(UnpackedAlignTo, MaxFieldAlignment);
   }
 
+  CharUnits AlignTo =
+      !DefaultsToAIXPowerAlignment ? BaseAlign : PreferredBaseAlign;
   if (!HasExternalLayout) {
     // Round up the current record size to the base's alignment boundary.
-    Offset = getDataSize().alignTo(BaseAlign);
+    Offset = getDataSize().alignTo(AlignTo);
 
     // Try to place the base.
     while (!EmptySubobjects->CanPlaceBaseAtOffset(Base, Offset))
-      Offset += BaseAlign;
+      Offset += AlignTo;
   } else {
     bool Allowed = EmptySubobjects->CanPlaceBaseAtOffset(Base, Offset);
     (void)Allowed;
     assert(Allowed && "Base subobject externally placed at overlapping offset");
 
-    if (InferAlignment && Offset < getDataSize().alignTo(BaseAlign)) {
+    if (InferAlignment && Offset < getDataSize().alignTo(AlignTo)) {
       // The externally-supplied base offset is before the base offset we
       // computed. Assume that the structure is packed.
       Alignment = CharUnits::One();
@@ -1248,7 +1300,7 @@ ItaniumRecordLayoutBuilder::LayoutBase(const BaseSubobjectInfo *Base) {
     setSize(std::max(getSize(), Offset + Layout.getSize()));
 
   // Remember max struct/class alignment.
-  UpdateAlignment(BaseAlign, UnpackedBaseAlign);
+  UpdateAlignment(BaseAlign, UnpackedAlignTo, PreferredBaseAlign);
 
   return Offset;
 }
@@ -1260,6 +1312,8 @@ void ItaniumRecordLayoutBuilder::InitializeLayout(const Decl *D) {
   }
 
   Packed = D->hasAttr<PackedAttr>();
+  HandledFirstNonOverlappingEmptyField =
+      !Context.getTargetInfo().defaultsToAIXPowerAlignment();
 
   // Honor the default struct packing maximum alignment flag.
   if (unsigned DefaultMaxFieldAlignment = Context.getLangOpts().PackStruct) {
@@ -1274,6 +1328,7 @@ void ItaniumRecordLayoutBuilder::InitializeLayout(const Decl *D) {
     IsMac68kAlign = true;
     MaxFieldAlignment = CharUnits::fromQuantity(2);
     Alignment = CharUnits::fromQuantity(2);
+    PreferredAlignment = CharUnits::fromQuantity(2);
   } else {
     if (const MaxFieldAlignmentAttr *MFAA = D->getAttr<MaxFieldAlignmentAttr>())
       MaxFieldAlignment = Context.toCharUnitsFromBits(MFAA->getAlignment());
@@ -1293,6 +1348,7 @@ void ItaniumRecordLayoutBuilder::InitializeLayout(const Decl *D) {
       if (UseExternalLayout) {
         if (External.Align > 0) {
           Alignment = Context.toCharUnitsFromBits(External.Align);
+          PreferredAlignment = Context.toCharUnitsFromBits(External.Align);
         } else {
           // The external source didn't have alignment information; infer it.
           InferAlignment = true;
@@ -1321,6 +1377,7 @@ void ItaniumRecordLayoutBuilder::Layout(const CXXRecordDecl *RD) {
   NonVirtualSize = Context.toCharUnitsFromBits(
       llvm::alignTo(getSizeInBits(), Context.getTargetInfo().getCharAlign()));
   NonVirtualAlignment = Alignment;
+  PreferredNVAlignment = PreferredAlignment;
 
   // Lay out the virtual bases and add the primary virtual base offsets.
   LayoutVirtualBases(RD, RD);
@@ -1733,25 +1790,46 @@ void ItaniumRecordLayoutBuilder::LayoutBitField(const FieldDecl *D) {
 
 void ItaniumRecordLayoutBuilder::LayoutField(const FieldDecl *D,
                                              bool InsertExtraPadding) {
+  auto *FieldClass = D->getType()->getAsCXXRecordDecl();
+  bool PotentiallyOverlapping = D->hasAttr<NoUniqueAddressAttr>() && FieldClass;
+  bool IsOverlappingEmptyField =
+      PotentiallyOverlapping && FieldClass->isEmpty();
+
+  CharUnits FieldOffset =
+      (IsUnion || IsOverlappingEmptyField) ? CharUnits::Zero() : getDataSize();
+
+  const bool DefaultsToAIXPowerAlignment =
+      Context.getTargetInfo().defaultsToAIXPowerAlignment();
+  bool FoundFirstNonOverlappingEmptyFieldForAIX = false;
+  if (DefaultsToAIXPowerAlignment && !HandledFirstNonOverlappingEmptyField) {
+    assert(FieldOffset == CharUnits::Zero() &&
+           "The first non-overlapping empty field should have been handled.");
+
+    if (!IsOverlappingEmptyField) {
+      FoundFirstNonOverlappingEmptyFieldForAIX = true;
+
+      // We're going to handle the "first member" based on
+      // `FoundFirstNonOverlappingEmptyFieldForAIX` during the current
+      // invocation of this function; record it as handled for future
+      // invocations (except for unions, because the current field does not
+      // represent all "firsts").
+      HandledFirstNonOverlappingEmptyField = !IsUnion;
+    }
+  }
+
   if (D->isBitField()) {
     LayoutBitField(D);
     return;
   }
 
   uint64_t UnpaddedFieldOffset = getDataSizeInBits() - UnfilledBitsInLastUnit;
-
   // Reset the unfilled bits.
   UnfilledBitsInLastUnit = 0;
   LastBitfieldTypeSize = 0;
 
-  auto *FieldClass = D->getType()->getAsCXXRecordDecl();
-  bool PotentiallyOverlapping = D->hasAttr<NoUniqueAddressAttr>() && FieldClass;
-  bool IsOverlappingEmptyField = PotentiallyOverlapping && FieldClass->isEmpty();
   bool FieldPacked = Packed || D->hasAttr<PackedAttr>();
 
-  CharUnits FieldOffset = (IsUnion || IsOverlappingEmptyField)
-                              ? CharUnits::Zero()
-                              : getDataSize();
+  bool AlignIsRequired = false;
   CharUnits FieldSize;
   CharUnits FieldAlign;
   // The amount of this class's dsize occupied by the field.
@@ -1759,25 +1837,27 @@ void ItaniumRecordLayoutBuilder::LayoutField(const FieldDecl *D,
   // into the field's tail padding.
   CharUnits EffectiveFieldSize;
 
+  auto setDeclInfo = [&](bool IsIncompleteArrayType) {
+    TypeInfo TI = Context.getTypeInfo(D->getType());
+    FieldAlign = Context.toCharUnitsFromBits(TI.Align);
+    // Flexible array members don't have any size, but they have to be
+    // aligned appropriately for their element type.
+    EffectiveFieldSize = FieldSize =
+        IsIncompleteArrayType ? CharUnits::Zero()
+                              : Context.toCharUnitsFromBits(TI.Width);
+    AlignIsRequired = TI.AlignIsRequired;
+  };
+
   if (D->getType()->isIncompleteArrayType()) {
-    // This is a flexible array member; we can't directly
-    // query getTypeInfo about these, so we figure it out here.
-    // Flexible array members don't have any size, but they
-    // have to be aligned appropriately for their element type.
-    EffectiveFieldSize = FieldSize = CharUnits::Zero();
-    const ArrayType* ATy = Context.getAsArrayType(D->getType());
-    FieldAlign = Context.getTypeAlignInChars(ATy->getElementType());
+    setDeclInfo(true /* IsIncompleteArrayType */);
   } else if (const ReferenceType *RT = D->getType()->getAs<ReferenceType>()) {
     unsigned AS = Context.getTargetAddressSpace(RT->getPointeeType());
-    EffectiveFieldSize = FieldSize =
-      Context.toCharUnitsFromBits(Context.getTargetInfo().getPointerWidth(AS));
-    FieldAlign =
-      Context.toCharUnitsFromBits(Context.getTargetInfo().getPointerAlign(AS));
+    EffectiveFieldSize = FieldSize = Context.toCharUnitsFromBits(
+        Context.getTargetInfo().getPointerWidth(AS));
+    FieldAlign = Context.toCharUnitsFromBits(
+        Context.getTargetInfo().getPointerAlign(AS));
   } else {
-    std::pair<CharUnits, CharUnits> FieldInfo =
-      Context.getTypeInfoInChars(D->getType());
-    EffectiveFieldSize = FieldSize = FieldInfo.first;
-    FieldAlign = FieldInfo.second;
+    setDeclInfo(false /* IsIncompleteArrayType */);
 
     // A potentially-overlapping field occupies its dsize or nvsize, whichever
     // is larger.
@@ -1829,31 +1909,72 @@ void ItaniumRecordLayoutBuilder::LayoutField(const FieldDecl *D,
     }
   }
 
+  // The AIX `power` alignment rules apply the natural alignment of the
+  // "first member" if it is of a floating-point data type (or is an aggregate
+  // whose recursively "first" member or element is such a type). The alignment
+  // associated with these types for subsequent members use an alignment value
+  // where the floating-point data type is considered to have 4-byte alignment.
+  //
+  // For the purposes of the foregoing: vtable pointers, non-empty base classes,
+  // and zero-width bit-fields count as prior members; members of empty class
+  // types marked `no_unique_address` are not considered to be prior members.
+  CharUnits PreferredAlign = FieldAlign;
+  if (DefaultsToAIXPowerAlignment && !AlignIsRequired &&
+      FoundFirstNonOverlappingEmptyFieldForAIX) {
+    auto performBuiltinTypeAlignmentUpgrade = [&](const BuiltinType *BTy) {
+      if (BTy->getKind() == BuiltinType::Double ||
+          BTy->getKind() == BuiltinType::LongDouble) {
+        assert(PreferredAlign == CharUnits::fromQuantity(4) &&
+               "No need to upgrade the alignment value.");
+        PreferredAlign = CharUnits::fromQuantity(8);
+      }
+    };
+
+    const Type *Ty = D->getType()->getBaseElementTypeUnsafe();
+    if (const ComplexType *CTy = Ty->getAs<ComplexType>()) {
+      performBuiltinTypeAlignmentUpgrade(CTy->getElementType()->castAs<BuiltinType>());
+    } else if (const BuiltinType *BTy = Ty->getAs<BuiltinType>()) {
+      performBuiltinTypeAlignmentUpgrade(BTy);
+    } else if (const RecordType *RT = Ty->getAs<RecordType>()) {
+      const RecordDecl *RD = RT->getDecl();
+      assert(RD && "Expected non-null RecordDecl.");
+      const ASTRecordLayout &FieldRecord = Context.getASTRecordLayout(RD);
+      PreferredAlign = FieldRecord.getPreferredAlignment();
+    }
+  }
+
   // The align if the field is not packed. This is to check if the attribute
   // was unnecessary (-Wpacked).
-  CharUnits UnpackedFieldAlign = FieldAlign;
+  CharUnits UnpackedFieldAlign =
+      !DefaultsToAIXPowerAlignment ? FieldAlign : PreferredAlign;
   CharUnits UnpackedFieldOffset = FieldOffset;
 
-  if (FieldPacked)
+  if (FieldPacked) {
     FieldAlign = CharUnits::One();
+    PreferredAlign = CharUnits::One();
+  }
   CharUnits MaxAlignmentInChars =
-    Context.toCharUnitsFromBits(D->getMaxAlignment());
+      Context.toCharUnitsFromBits(D->getMaxAlignment());
   FieldAlign = std::max(FieldAlign, MaxAlignmentInChars);
+  PreferredAlign = std::max(PreferredAlign, MaxAlignmentInChars);
   UnpackedFieldAlign = std::max(UnpackedFieldAlign, MaxAlignmentInChars);
 
   // The maximum field alignment overrides the aligned attribute.
   if (!MaxFieldAlignment.isZero()) {
     FieldAlign = std::min(FieldAlign, MaxFieldAlignment);
+    PreferredAlign = std::min(PreferredAlign, MaxFieldAlignment);
     UnpackedFieldAlign = std::min(UnpackedFieldAlign, MaxFieldAlignment);
   }
 
+  CharUnits AlignTo =
+      !DefaultsToAIXPowerAlignment ? FieldAlign : PreferredAlign;
   // Round up the current record size to the field's alignment boundary.
-  FieldOffset = FieldOffset.alignTo(FieldAlign);
+  FieldOffset = FieldOffset.alignTo(AlignTo);
   UnpackedFieldOffset = UnpackedFieldOffset.alignTo(UnpackedFieldAlign);
 
   if (UseExternalLayout) {
     FieldOffset = Context.toCharUnitsFromBits(
-                    updateExternalFieldOffset(D, Context.toBits(FieldOffset)));
+        updateExternalFieldOffset(D, Context.toBits(FieldOffset)));
 
     if (!IsUnion && EmptySubobjects) {
       // Record the fact that we're placing a field at this offset.
@@ -1869,9 +1990,9 @@ void ItaniumRecordLayoutBuilder::LayoutField(const FieldDecl *D,
         // We try offset 0 (for an empty field) and then dsize(C) onwards.
         if (FieldOffset == CharUnits::Zero() &&
             getDataSize() != CharUnits::Zero())
-          FieldOffset = getDataSize().alignTo(FieldAlign);
+          FieldOffset = getDataSize().alignTo(AlignTo);
         else
-          FieldOffset += FieldAlign;
+          FieldOffset += AlignTo;
       }
     }
   }
@@ -1908,9 +2029,9 @@ void ItaniumRecordLayoutBuilder::LayoutField(const FieldDecl *D,
                      (uint64_t)Context.toBits(FieldOffset + FieldSize)));
   }
 
-  // Remember max struct/class alignment.
+  // Remember max struct/class ABI-specified alignment.
   UnadjustedAlignment = std::max(UnadjustedAlignment, FieldAlign);
-  UpdateAlignment(FieldAlign, UnpackedFieldAlign);
+  UpdateAlignment(FieldAlign, UnpackedFieldAlign, PreferredAlign);
 }
 
 void ItaniumRecordLayoutBuilder::FinishLayout(const NamedDecl *D) {
@@ -1936,8 +2057,12 @@ void ItaniumRecordLayoutBuilder::FinishLayout(const NamedDecl *D) {
   uint64_t UnpaddedSize = getSizeInBits() - UnfilledBitsInLastUnit;
   uint64_t UnpackedSizeInBits =
       llvm::alignTo(getSizeInBits(), Context.toBits(UnpackedAlignment));
-  uint64_t RoundedSize =
-      llvm::alignTo(getSizeInBits(), Context.toBits(Alignment));
+
+  uint64_t RoundedSize = llvm::alignTo(
+      getSizeInBits(),
+      Context.toBits(!Context.getTargetInfo().defaultsToAIXPowerAlignment()
+                         ? Alignment
+                         : PreferredAlignment));
 
   if (UseExternalLayout) {
     // If we're inferring alignment, and the external size is smaller than
@@ -1945,6 +2070,7 @@ void ItaniumRecordLayoutBuilder::FinishLayout(const NamedDecl *D) {
     // alignment to 1.
     if (InferAlignment && External.Size < RoundedSize) {
       Alignment = CharUnits::One();
+      PreferredAlignment = CharUnits::One();
       InferAlignment = false;
     }
     setSize(External.Size);
@@ -1981,7 +2107,8 @@ void ItaniumRecordLayoutBuilder::FinishLayout(const NamedDecl *D) {
 }
 
 void ItaniumRecordLayoutBuilder::UpdateAlignment(
-    CharUnits NewAlignment, CharUnits UnpackedNewAlignment) {
+    CharUnits NewAlignment, CharUnits UnpackedNewAlignment,
+    CharUnits PreferredNewAlignment) {
   // The alignment is not modified when using 'mac68k' alignment or when
   // we have an externally-supplied layout that also provides overall alignment.
   if (IsMac68kAlign || (UseExternalLayout && !InferAlignment))
@@ -1998,6 +2125,12 @@ void ItaniumRecordLayoutBuilder::UpdateAlignment(
            "Alignment not a power of 2");
     UnpackedAlignment = UnpackedNewAlignment;
   }
+
+  if (PreferredNewAlignment > PreferredAlignment) {
+    assert(llvm::isPowerOf2_64(PreferredNewAlignment.getQuantity()) &&
+           "Alignment not a power of 2");
+    PreferredAlignment = PreferredNewAlignment;
+  }
 }
 
 uint64_t
@@ -2009,6 +2142,7 @@ ItaniumRecordLayoutBuilder::updateExternalFieldOffset(const FieldDecl *Field,
     // The externally-supplied field offset is before the field offset we
     // computed. Assume that the structure is packed.
     Alignment = CharUnits::One();
+    PreferredAlignment = CharUnits::One();
     InferAlignment = false;
   }
 
@@ -3063,10 +3197,10 @@ ASTContext::getASTRecordLayout(const RecordDecl *D) const {
       Builder.cxxLayout(RD);
       NewEntry = new (*this) ASTRecordLayout(
           *this, Builder.Size, Builder.Alignment, Builder.Alignment,
-          Builder.RequiredAlignment,
-          Builder.HasOwnVFPtr, Builder.HasOwnVFPtr || Builder.PrimaryBase,
-          Builder.VBPtrOffset, Builder.DataSize, Builder.FieldOffsets,
-          Builder.NonVirtualSize, Builder.Alignment, CharUnits::Zero(),
+          Builder.Alignment, Builder.RequiredAlignment, Builder.HasOwnVFPtr,
+          Builder.HasOwnVFPtr || Builder.PrimaryBase, Builder.VBPtrOffset,
+          Builder.DataSize, Builder.FieldOffsets, Builder.NonVirtualSize,
+          Builder.Alignment, Builder.Alignment, CharUnits::Zero(),
           Builder.PrimaryBase, false, Builder.SharedVBPtrBase,
           Builder.EndsWithZeroSizedObject, Builder.LeadsWithZeroSizedBase,
           Builder.Bases, Builder.VBases);
@@ -3074,8 +3208,8 @@ ASTContext::getASTRecordLayout(const RecordDecl *D) const {
       Builder.layout(D);
       NewEntry = new (*this) ASTRecordLayout(
           *this, Builder.Size, Builder.Alignment, Builder.Alignment,
-          Builder.RequiredAlignment,
-          Builder.Size, Builder.FieldOffsets);
+          Builder.Alignment, Builder.RequiredAlignment, Builder.Size,
+          Builder.FieldOffsets);
     }
   } else {
     if (const auto *RD = dyn_cast<CXXRecordDecl>(D)) {
@@ -3095,11 +3229,13 @@ ASTContext::getASTRecordLayout(const RecordDecl *D) const {
       CharUnits NonVirtualSize =
           skipTailPadding ? DataSize : Builder.NonVirtualSize;
       NewEntry = new (*this) ASTRecordLayout(
-          *this, Builder.getSize(), Builder.Alignment, Builder.UnadjustedAlignment,
+          *this, Builder.getSize(), Builder.Alignment,
+          Builder.PreferredAlignment, Builder.UnadjustedAlignment,
           /*RequiredAlignment : used by MS-ABI)*/
           Builder.Alignment, Builder.HasOwnVFPtr, RD->isDynamicClass(),
           CharUnits::fromQuantity(-1), DataSize, Builder.FieldOffsets,
           NonVirtualSize, Builder.NonVirtualAlignment,
+          Builder.PreferredNVAlignment,
           EmptySubobjects.SizeOfLargestEmptySubobject, Builder.PrimaryBase,
           Builder.PrimaryBaseIsVirtual, nullptr, false, false, Builder.Bases,
           Builder.VBases);
@@ -3108,7 +3244,8 @@ ASTContext::getASTRecordLayout(const RecordDecl *D) const {
       Builder.Layout(D);
 
       NewEntry = new (*this) ASTRecordLayout(
-          *this, Builder.getSize(), Builder.Alignment, Builder.UnadjustedAlignment,
+          *this, Builder.getSize(), Builder.Alignment,
+          Builder.PreferredAlignment, Builder.UnadjustedAlignment,
           /*RequiredAlignment : used by MS-ABI)*/
           Builder.Alignment, Builder.getSize(), Builder.FieldOffsets);
     }
@@ -3260,14 +3397,11 @@ ASTContext::getObjCLayout(const ObjCInterfaceDecl *D,
   ItaniumRecordLayoutBuilder Builder(*this, /*EmptySubobjects=*/nullptr);
   Builder.Layout(D);
 
-  const ASTRecordLayout *NewEntry =
-    new (*this) ASTRecordLayout(*this, Builder.getSize(),
-                                Builder.Alignment,
-                                Builder.UnadjustedAlignment,
-                                /*RequiredAlignment : used by MS-ABI)*/
-                                Builder.Alignment,
-                                Builder.getDataSize(),
-                                Builder.FieldOffsets);
+  const ASTRecordLayout *NewEntry = new (*this) ASTRecordLayout(
+      *this, Builder.getSize(), Builder.Alignment, Builder.PreferredAlignment,
+      Builder.UnadjustedAlignment,
+      /*RequiredAlignment : used by MS-ABI)*/
+      Builder.Alignment, Builder.getDataSize(), Builder.FieldOffsets);
 
   ObjCLayouts[Key] = NewEntry;
 
@@ -3430,22 +3564,26 @@ static void DumpRecordLayout(raw_ostream &OS, const RecordDecl *RD,
   if (CXXRD && !isMsLayout(C))
     OS << ", dsize=" << Layout.getDataSize().getQuantity();
   OS << ", align=" << Layout.getAlignment().getQuantity();
+  if (C.getTargetInfo().defaultsToAIXPowerAlignment())
+    OS << ", preferredalign=" << Layout.getPreferredAlignment().getQuantity();
 
   if (CXXRD) {
     OS << ",\n";
     PrintIndentNoOffset(OS, IndentLevel - 1);
     OS << " nvsize=" << Layout.getNonVirtualSize().getQuantity();
     OS << ", nvalign=" << Layout.getNonVirtualAlignment().getQuantity();
+    if (C.getTargetInfo().defaultsToAIXPowerAlignment())
+      OS << ", preferrednvalign="
+         << Layout.getPreferredNVAlignment().getQuantity();
   }
   OS << "]\n";
 }
 
-void ASTContext::DumpRecordLayout(const RecordDecl *RD,
-                                  raw_ostream &OS,
+void ASTContext::DumpRecordLayout(const RecordDecl *RD, raw_ostream &OS,
                                   bool Simple) const {
   if (!Simple) {
     ::DumpRecordLayout(OS, RD, *this, CharUnits(), 0, nullptr,
-                       /*PrintSizeInfo*/true,
+                       /*PrintSizeInfo*/ true,
                        /*IncludeVirtualBases=*/true);
     return;
   }
@@ -3465,9 +3603,13 @@ void ASTContext::DumpRecordLayout(const RecordDecl *RD,
   if (!isMsLayout(*this))
     OS << "  DataSize:" << toBits(Info.getDataSize()) << "\n";
   OS << "  Alignment:" << toBits(Info.getAlignment()) << "\n";
+  if (Target->defaultsToAIXPowerAlignment())
+    OS << "  PreferredAlignment:" << toBits(Info.getPreferredAlignment())
+       << "\n";
   OS << "  FieldOffsets: [";
   for (unsigned i = 0, e = Info.getFieldCount(); i != e; ++i) {
-    if (i) OS << ", ";
+    if (i)
+      OS << ", ";
     OS << Info.getFieldOffset(i);
   }
   OS << "]>\n";
diff --git a/clang/lib/Basic/Targets/OSTargets.h b/clang/lib/Basic/Targets/OSTargets.h
index cfa362bef1b1c..7b3acc335a352 100644
--- a/clang/lib/Basic/Targets/OSTargets.h
+++ b/clang/lib/Basic/Targets/OSTargets.h
@@ -719,6 +719,8 @@ class AIXTargetInfo : public OSTargetInfo<Target> {
   // AIX sets FLT_EVAL_METHOD to be 1.
   unsigned getFloatEvalMethod() const override { return 1; }
   bool hasInt128Type() const override { return false; }
+
+  bool defaultsToAIXPowerAlignment() const override { return true; }
 };
 
 void addWindowsDefines(const llvm::Triple &Triple, const LangOptions &Opts,
diff --git a/clang/lib/Basic/Targets/PPC.h b/clang/lib/Basic/Targets/PPC.h
index 858059bacb86b..ff8579b6c3cf4 100644
--- a/clang/lib/Basic/Targets/PPC.h
+++ b/clang/lib/Basic/Targets/PPC.h
@@ -371,13 +371,16 @@ class LLVM_LIBRARY_VISIBILITY PPC32TargetInfo : public PPCTargetInfo {
       PtrDiffType = SignedLong;
       IntPtrType = SignedLong;
       SuitableAlign = 64;
+      LongDoubleWidth = 64;
+      LongDoubleAlign = DoubleAlign = 32;
+      LongDoubleFormat = &llvm::APFloat::IEEEdouble();
       break;
     default:
       break;
     }
 
     if (Triple.isOSFreeBSD() || Triple.isOSNetBSD() || Triple.isOSOpenBSD() ||
-        Triple.getOS() == llvm::Triple::AIX || Triple.isMusl()) {
+        Triple.isMusl()) {
       LongDoubleWidth = LongDoubleAlign = 64;
       LongDoubleFormat = &llvm::APFloat::IEEEdouble();
     }
@@ -406,6 +409,9 @@ class LLVM_LIBRARY_VISIBILITY PPC64TargetInfo : public PPCTargetInfo {
       // TODO: Set appropriate ABI for AIX platform.
       resetDataLayout("E-m:a-i64:64-n32:64");
       SuitableAlign = 64;
+      LongDoubleWidth = 64;
+      LongDoubleAlign = DoubleAlign = 32;
+      LongDoubleFormat = &llvm::APFloat::IEEEdouble();
     } else if ((Triple.getArch() == llvm::Triple::ppc64le)) {
       resetDataLayout("e-m:e-i64:64-n32:64");
       ABI = "elfv2";
@@ -414,8 +420,7 @@ class LLVM_LIBRARY_VISIBILITY PPC64TargetInfo : public PPCTargetInfo {
       ABI = "elfv1";
     }
 
-    if (Triple.isOSFreeBSD() || Triple.getOS() == llvm::Triple::AIX ||
-        Triple.isMusl()) {
+    if (Triple.isOSFreeBSD() || Triple.isMusl()) {
       LongDoubleWidth = LongDoubleAlign = 64;
       LongDoubleFormat = &llvm::APFloat::IEEEdouble();
     }
diff --git a/clang/test/Layout/aix-Wpacked-expecting-diagnostics.cpp b/clang/test/Layout/aix-Wpacked-expecting-diagnostics.cpp
new file mode 100644
index 0000000000000..1980c04877c51
--- /dev/null
+++ b/clang/test/Layout/aix-Wpacked-expecting-diagnostics.cpp
@@ -0,0 +1,30 @@
+// RUN: %clang_cc1 -triple powerpc-ibm-aix-xcoff -Wpacked \
+// RUN:     -fdump-record-layouts -fsyntax-only -verify -x c++ < %s | \
+// RUN:   FileCheck %s
+
+// RUN: %clang_cc1 -triple powerpc64-ibm-aix-xcoff -Wpacked \
+// RUN:     -fdump-record-layouts -fsyntax-only -verify -x c++ < %s | \
+// RUN:   FileCheck %s
+
+struct A {
+  double d;
+};
+
+struct B {
+  char x[8];
+};
+
+struct [[gnu::packed]] C : B, A { // expected-warning{{packed attribute is unnecessary for 'C'}}
+  char x alignas(4)[8];
+};
+
+int b = sizeof(C);
+
+// CHECK:               0 | struct C
+// CHECK-NEXT:          0 |   struct B (base)
+// CHECK-NEXT:          0 |     char [8] x
+// CHECK-NEXT:          8 |   struct A (base)
+// CHECK-NEXT:          8 |     double d
+// CHECK-NEXT:         16 |   char [8] x
+// CHECK-NEXT:            | [sizeof=24, dsize=24, align=4, preferredalign=4,
+// CHECK-NEXT:            |  nvsize=24, nvalign=4, preferrednvalign=4]
diff --git a/clang/test/Layout/aix-Wpacked-no-diagnostics.cpp b/clang/test/Layout/aix-Wpacked-no-diagnostics.cpp
new file mode 100644
index 0000000000000..ed5362b3e8cd9
--- /dev/null
+++ b/clang/test/Layout/aix-Wpacked-no-diagnostics.cpp
@@ -0,0 +1,31 @@
+// RUN: %clang_cc1 -triple powerpc-ibm-aix-xcoff -Wpacked \
+// RUN:     -fdump-record-layouts -fsyntax-only -verify -x c++ < %s | \
+// RUN:   FileCheck %s
+
+// RUN: %clang_cc1 -triple powerpc64-ibm-aix-xcoff -Wpacked \
+// RUN:     -fdump-record-layouts -fsyntax-only -verify -x c++ < %s | \
+// RUN:   FileCheck %s
+
+// expected-no-diagnostics
+
+struct [[gnu::packed]] Q {
+  double x [[gnu::aligned(4)]];
+};
+
+struct QQ : Q { char x; };
+
+int a = sizeof(QQ);
+
+// CHECK:      *** Dumping AST Record Layout
+// CHECK-NEXT:          0 | struct Q
+// CHECK-NEXT:          0 |   double x
+// CHECK-NEXT:            | [sizeof=8, dsize=8, align=4, preferredalign=4,
+// CHECK-NEXT:            |  nvsize=8, nvalign=4, preferrednvalign=4]
+
+// CHECK:      *** Dumping AST Record Layout
+// CHECK-NEXT:          0 | struct QQ
+// CHECK-NEXT:          0 |   struct Q (base)
+// CHECK-NEXT:          0 |     double x
+// CHECK-NEXT:          8 |   char x
+// CHECK-NEXT:            | [sizeof=12, dsize=9, align=4, preferredalign=4,
+// CHECK-NEXT:            |  nvsize=9, nvalign=4, preferrednvalign=4]
diff --git a/clang/test/Layout/aix-double-struct-member.cpp b/clang/test/Layout/aix-double-struct-member.cpp
new file mode 100644
index 0000000000000..b51d10467bceb
--- /dev/null
+++ b/clang/test/Layout/aix-double-struct-member.cpp
@@ -0,0 +1,428 @@
+// RUN: %clang_cc1 -triple powerpc-ibm-aix-xcoff -fdump-record-layouts \
+// RUN:     -fsyntax-only  %s | \
+// RUN:   FileCheck %s
+
+// RUN: %clang_cc1 -triple powerpc64-ibm-aix-xcoff -fdump-record-layouts \
+// RUN:     -fsyntax-only %s | \
+// RUN:   FileCheck %s
+
+namespace test1 {
+// Test the class layout when having a double which is/is not the first struct
+// member.
+struct D {
+  double d1;
+  int i1;
+};
+
+struct DoubleFirst {
+  struct D d2;
+  int i2;
+};
+
+struct IntFirst {
+  int i3;
+  struct D d3;
+};
+
+int a = sizeof(DoubleFirst);
+int b = sizeof(IntFirst);
+
+// CHECK:     *** Dumping AST Record Layout
+// CHECK-NEXT:         0 | struct test1::D
+// CHECK-NEXT:         0 |   double d1
+// CHECK-NEXT:         8 |   int i1
+// CHECK-NEXT:           | [sizeof=16, dsize=16, align=4, preferredalign=8,
+// CHECK-NEXT:           |  nvsize=16, nvalign=4, preferrednvalign=8]
+
+// CHECK:     *** Dumping AST Record Layout
+// CHECK-NEXT:         0 | struct test1::DoubleFirst
+// CHECK-NEXT:         0 |   struct test1::D d2
+// CHECK-NEXT:         0 |     double d1
+// CHECK-NEXT:         8 |     int i1
+// CHECK-NEXT:        16 |   int i2
+// CHECK-NEXT:           | [sizeof=24, dsize=24, align=4, preferredalign=8,
+// CHECK-NEXT:           |  nvsize=24, nvalign=4, preferrednvalign=8]
+
+// CHECK:     *** Dumping AST Record Layout
+// CHECK-NEXT:         0 | struct test1::IntFirst
+// CHECK-NEXT:         0 |   int i3
+// CHECK-NEXT:         4 |   struct test1::D d3
+// CHECK-NEXT:         4 |     double d1
+// CHECK-NEXT:        12 |     int i1
+// CHECK-NEXT:           | [sizeof=20, dsize=20, align=4, preferredalign=4,
+// CHECK-NEXT:           |  nvsize=20, nvalign=4, preferrednvalign=4]
+} // namespace test1
+
+namespace test2 {
+// Test the class layout when having a zero-sized bitfield followed by double.
+struct Double {
+  int : 0;
+  double d;
+};
+
+int a = sizeof(Double);
+
+// CHECK:     *** Dumping AST Record Layout
+// CHECK-NEXT:         0 | struct test2::Double
+// CHECK-NEXT:       0:- |   int
+// CHECK-NEXT:         0 |   double d
+// CHECK-NEXT:           | [sizeof=8, dsize=8, align=4, preferredalign=4,
+// CHECK-NEXT:           |  nvsize=8, nvalign=4, preferrednvalign=4]
+} // namespace test2
+
+namespace test3 {
+// Test the class layout when having a double member in union.
+union A {
+  int b;
+  double d;
+};
+
+struct UnionStruct {
+  union A a;
+  int i;
+};
+
+int a = sizeof(UnionStruct);
+
+// CHECK:      *** Dumping AST Record Layout
+// CHECK-NEXT:          0 | union test3::A
+// CHECK-NEXT:          0 |   int b
+// CHECK-NEXT:          0 |   double d
+// CHECK-NEXT:            | [sizeof=8, dsize=8, align=4, preferredalign=8,
+// CHECK-NEXT:            |  nvsize=8, nvalign=4, preferrednvalign=8]
+
+// CHECK:      *** Dumping AST Record Layout
+// CHECK-NEXT:          0 | struct test3::UnionStruct
+// CHECK-NEXT:          0 |   union test3::A a
+// CHECK-NEXT:          0 |     int b
+// CHECK-NEXT:          0 |     double d
+// CHECK-NEXT:          8 |   int i
+// CHECK-NEXT:            | [sizeof=16, dsize=16, align=4, preferredalign=8,
+// CHECK-NEXT:            |  nvsize=16, nvalign=4, preferrednvalign=8]
+
+} // namespace test3
+
+namespace test4 {
+// Test the class layout when having multiple base classes.
+struct A {
+  int a;
+};
+
+struct B {
+  double d;
+};
+
+class S : A, B {
+};
+
+int a = sizeof(S);
+
+// CHECK:      *** Dumping AST Record Layout
+// CHECK-NEXT:          0 | struct test4::A
+// CHECK-NEXT:          0 |   int a
+// CHECK-NEXT:            | [sizeof=4, dsize=4, align=4, preferredalign=4,
+// CHECK-NEXT:            |  nvsize=4, nvalign=4, preferrednvalign=4]
+
+// CHECK:      *** Dumping AST Record Layout
+// CHECK-NEXT:          0 | struct test4::B
+// CHECK-NEXT:          0 |   double d
+// CHECK-NEXT:            | [sizeof=8, dsize=8, align=4, preferredalign=8,
+// CHECK-NEXT:            |  nvsize=8, nvalign=4, preferrednvalign=8]
+
+// CHECK:      *** Dumping AST Record Layout
+// CHECK-NEXT:          0 | class test4::S
+// CHECK-NEXT:          0 |   struct test4::A (base)
+// CHECK-NEXT:          0 |     int a
+// CHECK-NEXT:          4 |   struct test4::B (base)
+// CHECK-NEXT:          4 |     double d
+// CHECK-NEXT:            | [sizeof=12, dsize=12, align=4, preferredalign=4,
+// CHECK-NEXT:            |  nvsize=12, nvalign=4, preferrednvalign=4]
+} // namespace test4
+
+namespace test5 {
+struct Empty {
+};
+
+struct EmptyDer : Empty {
+  double d;
+};
+
+struct NonEmpty {
+  int i;
+};
+
+struct NonEmptyDer : NonEmpty {
+  double d;
+};
+
+int a = sizeof(EmptyDer);
+int b = sizeof(NonEmptyDer);
+
+// CHECK:      *** Dumping AST Record Layout
+// CHECK-NEXT:          0 | struct test5::Empty (empty)
+// CHECK-NEXT:            | [sizeof=1, dsize=1, align=1, preferredalign=1,
+// CHECK-NEXT:            |  nvsize=1, nvalign=1, preferrednvalign=1]
+
+// CHECK:      *** Dumping AST Record Layout
+// CHECK-NEXT:          0 | struct test5::EmptyDer
+// CHECK-NEXT:          0 |   struct test5::Empty (base) (empty)
+// CHECK-NEXT:          0 |   double d
+// CHECK-NEXT:            | [sizeof=8, dsize=8, align=4, preferredalign=8,
+// CHECK-NEXT:            |  nvsize=8, nvalign=4, preferrednvalign=8]
+
+// CHECK:      *** Dumping AST Record Layout
+// CHECK-NEXT:          0 | struct test5::NonEmpty
+// CHECK-NEXT:          0 |   int i
+// CHECK-NEXT:            | [sizeof=4, dsize=4, align=4, preferredalign=4,
+// CHECK-NEXT:            |  nvsize=4, nvalign=4, preferrednvalign=4]
+
+// CHECK:      *** Dumping AST Record Layout
+// CHECK-NEXT:          0 | struct test5::NonEmptyDer
+// CHECK-NEXT:          0 |   struct test5::NonEmpty (base)
+// CHECK-NEXT:          0 |     int i
+// CHECK-NEXT:          4 |   double d
+// CHECK-NEXT:            | [sizeof=12, dsize=12, align=4, preferredalign=4,
+// CHECK-NEXT:            |  nvsize=12, nvalign=4, preferrednvalign=4]
+} // namespace test5
+
+namespace test6 {
+struct A {
+  struct B {
+    double d[3];
+  } b;
+};
+
+int a = sizeof(A);
+
+// CHECK:      *** Dumping AST Record Layout
+// CHECK-NEXT:          0 | struct test6::A::B
+// CHECK-NEXT:          0 |   double [3] d
+// CHECK-NEXT:            | [sizeof=24, dsize=24, align=4, preferredalign=8,
+// CHECK-NEXT:            |  nvsize=24, nvalign=4, preferrednvalign=8]
+
+// CHECK:      *** Dumping AST Record Layout
+// CHECK-NEXT:          0 | struct test6::A
+// CHECK-NEXT:          0 |   struct test6::A::B b
+// CHECK-NEXT:          0 |     double [3] d
+// CHECK-NEXT:            | [sizeof=24, dsize=24, align=4, preferredalign=8,
+// CHECK-NEXT:            |  nvsize=24, nvalign=4, preferrednvalign=8]
+
+} // namespace test6
+
+namespace test7 {
+struct A {
+  struct B {
+    long double _Complex d[3];
+  } b;
+};
+
+int a = sizeof(A);
+
+// CHECK:      *** Dumping AST Record Layout
+// CHECK-NEXT:          0 | struct test7::A::B
+// CHECK-NEXT:          0 |   _Complex long double [3] d
+// CHECK-NEXT:            | [sizeof=48, dsize=48, align=4, preferredalign=8,
+// CHECK-NEXT:            |  nvsize=48, nvalign=4, preferrednvalign=8]
+
+// CHECK:      *** Dumping AST Record Layout
+// CHECK-NEXT:          0 | struct test7::A
+// CHECK-NEXT:          0 |   struct test7::A::B b
+// CHECK-NEXT:          0 |     _Complex long double [3] d
+// CHECK-NEXT:            | [sizeof=48, dsize=48, align=4, preferredalign=8,
+// CHECK-NEXT:            |  nvsize=48, nvalign=4, preferrednvalign=8]
+
+} // namespace test7
+
+namespace test8 {
+struct Emp {};
+
+struct Y : Emp {
+  double d;
+};
+
+struct Z : Emp {
+  Y y;
+};
+
+int a = sizeof(Z);
+
+// CHECK:      *** Dumping AST Record Layout
+// CHECK-NEXT:          0 | struct test8::Emp (empty)
+// CHECK-NEXT:            | [sizeof=1, dsize=1, align=1, preferredalign=1,
+// CHECK-NEXT:            |  nvsize=1, nvalign=1, preferrednvalign=1]
+
+// CHECK:      *** Dumping AST Record Layout
+// CHECK-NEXT:          0 | struct test8::Y
+// CHECK-NEXT:          0 |   struct test8::Emp (base) (empty)
+// CHECK-NEXT:          0 |   double d
+// CHECK-NEXT:            | [sizeof=8, dsize=8, align=4, preferredalign=8,
+// CHECK-NEXT:            |  nvsize=8, nvalign=4, preferrednvalign=8]
+
+// CHECK:      *** Dumping AST Record Layout
+// CHECK-NEXT:          0 | struct test8::Z
+// CHECK-NEXT:          0 |   struct test8::Emp (base) (empty)
+// CHECK-NEXT:          8 |   struct test8::Y y
+// CHECK-NEXT:          8 |     struct test8::Emp (base) (empty)
+// CHECK-NEXT:          8 |     double d
+// CHECK-NEXT:            | [sizeof=16, dsize=16, align=4, preferredalign=8,
+// CHECK-NEXT:            |  nvsize=16, nvalign=4, preferrednvalign=8]
+
+} // namespace test8
+
+namespace test9 {
+// Test the class layout when having a zero-extent array in a base class, which
+// renders the base class not empty.
+struct A { char zea[0]; };
+
+struct B : A { double d; };
+
+struct C { double d; };
+struct D : A, C { char x; };
+
+int a = sizeof(B);
+int b = sizeof(D);
+
+// CHECK:               0 | struct test9::B
+// CHECK-NEXT:          0 |   struct test9::A (base)
+// CHECK-NEXT:          0 |     char [0] zea
+// CHECK-NEXT:          0 |   double d
+// CHECK-NEXT:            | [sizeof=8, dsize=8, align=4, preferredalign=4,
+// CHECK-NEXT:            |  nvsize=8, nvalign=4, preferrednvalign=4]
+
+// CHECK:               0 | struct test9::D
+// CHECK-NEXT:          0 |   struct test9::A (base)
+// CHECK-NEXT:          0 |     char [0] zea
+// CHECK-NEXT:          0 |   struct test9::C (base)
+// CHECK-NEXT:          0 |     double d
+// CHECK-NEXT:          8 |   char x
+// CHECK-NEXT:            | [sizeof=12, dsize=9, align=4, preferredalign=4,
+// CHECK-NEXT:            |  nvsize=9, nvalign=4, preferrednvalign=4]
+
+} // namespace test9
+
+namespace test10 {
+struct A { double x; };
+struct B : A {};
+
+int a = sizeof(B);
+
+// CHECK:               0 | struct test10::B
+// CHECK-NEXT:          0 |   struct test10::A (base)
+// CHECK-NEXT:          0 |     double x
+// CHECK-NEXT:            | [sizeof=8, dsize=8, align=4, preferredalign=8,
+// CHECK-NEXT:            |  nvsize=8, nvalign=4, preferrednvalign=8]
+
+} // namespace test10
+
+namespace test11 {
+// Test how #pragma pack and align attribute interacts with AIX `power`
+// alignment rules.
+struct A {
+  char a;
+  double __attribute__((aligned(16))) d;
+  int i;
+};
+
+struct B {
+  double __attribute__((aligned(4))) d1;
+  char a;
+  double d2;
+};
+
+#pragma pack(2)
+struct C {
+  int i;
+  short j;
+  double k;
+};
+
+#pragma pack(2)
+struct D {
+  double d;
+  short j;
+  int i;
+};
+
+#pragma pack(8)
+struct E {
+  double __attribute__((aligned(4))) d;
+  short s;
+};
+
+#pragma pack(4)
+struct F : public D {
+  double d;
+};
+
+#pragma pack(2)
+struct G : public E {
+  int i;
+};
+
+int a = sizeof(A);
+int b = sizeof(B);
+int c = sizeof(C);
+int d = sizeof(D);
+int e = sizeof(E);
+int f = sizeof(F);
+int g = sizeof(G);
+
+// CHECK:      *** Dumping AST Record Layout
+// CHECK-NEXT:          0 | struct test11::A
+// CHECK-NEXT:          0 |   char a
+// CHECK-NEXT:         16 |   double d
+// CHECK-NEXT:         24 |   int i
+// CHECK-NEXT:            | [sizeof=32, dsize=32, align=16, preferredalign=16,
+// CHECK-NEXT:            |  nvsize=32, nvalign=16, preferrednvalign=16]
+
+// CHECK:      *** Dumping AST Record Layout
+// CHECK-NEXT:          0 | struct test11::B
+// CHECK-NEXT:          0 |   double d1
+// CHECK-NEXT:          8 |   char a
+// CHECK-NEXT:         12 |   double d2
+// CHECK-NEXT:            | [sizeof=24, dsize=24, align=4, preferredalign=8,
+// CHECK-NEXT:            |  nvsize=24, nvalign=4, preferrednvalign=8]
+
+// CHECK:      *** Dumping AST Record Layout
+// CHECK-NEXT:          0 | struct test11::C
+// CHECK-NEXT:          0 |   int i
+// CHECK-NEXT:          4 |   short j
+// CHECK-NEXT:          6 |   double k
+// CHECK-NEXT:            | [sizeof=14, dsize=14, align=2, preferredalign=2,
+// CHECK-NEXT:            |  nvsize=14, nvalign=2, preferrednvalign=2]
+
+// CHECK:      *** Dumping AST Record Layout
+// CHECK-NEXT:          0 | struct test11::D
+// CHECK-NEXT:          0 |   double d
+// CHECK-NEXT:          8 |   short j
+// CHECK-NEXT:         10 |   int i
+// CHECK-NEXT:            | [sizeof=14, dsize=14, align=2, preferredalign=2,
+// CHECK-NEXT:            |  nvsize=14, nvalign=2, preferrednvalign=2]
+
+// CHECK:      *** Dumping AST Record Layout
+// CHECK-NEXT:          0 | struct test11::E
+// CHECK-NEXT:          0 |   double d
+// CHECK-NEXT:          8 |   short s
+// CHECK-NEXT:            | [sizeof=16, dsize=16, align=4, preferredalign=8,
+// CHECK-NEXT:            |  nvsize=16, nvalign=4, preferrednvalign=8]
+
+// CHECK:      *** Dumping AST Record Layout
+// CHECK-NEXT:          0 | struct test11::F
+// CHECK-NEXT:          0 |   struct test11::D (base)
+// CHECK-NEXT:          0 |     double d
+// CHECK-NEXT:          8 |     short j
+// CHECK-NEXT:         10 |     int i
+// CHECK-NEXT:         16 |   double d
+// CHECK-NEXT:            | [sizeof=24, dsize=24, align=4, preferredalign=4,
+// CHECK-NEXT:            |  nvsize=24, nvalign=4, preferrednvalign=4]
+
+// CHECK:      *** Dumping AST Record Layout
+// CHECK-NEXT:          0 | struct test11::G
+// CHECK-NEXT:          0 |   struct test11::E (base)
+// CHECK-NEXT:          0 |     double d
+// CHECK-NEXT:          8 |     short s
+// CHECK-NEXT:         16 |   int i
+// CHECK-NEXT:            | [sizeof=20, dsize=20, align=2, preferredalign=2,
+// CHECK-NEXT:            |  nvsize=20, nvalign=2, preferrednvalign=2]
+
+} // namespace test11
diff --git a/clang/test/Layout/aix-no-unique-address-with-double.cpp b/clang/test/Layout/aix-no-unique-address-with-double.cpp
new file mode 100644
index 0000000000000..5188bf128bc50
--- /dev/null
+++ b/clang/test/Layout/aix-no-unique-address-with-double.cpp
@@ -0,0 +1,158 @@
+// RUN: %clang_cc1 -triple powerpc-ibm-aix-xcoff -fdump-record-layouts \
+// RUN:     -fsyntax-only %s | \
+// RUN:   FileCheck %s
+
+// RUN: %clang_cc1 -triple powerpc64-ibm-aix-xcoff -fdump-record-layouts \
+// RUN:     -fsyntax-only %s | \
+// RUN:   FileCheck %s
+
+struct Empty {};
+
+struct A {
+  double d;
+};
+
+// CHECK:     *** Dumping AST Record Layout
+// CHECK-NEXT:          0 | struct Empty (empty)
+// CHECK-NEXT:            | [sizeof=1, dsize=1, align=1, preferredalign=1,
+// CHECK-NEXT:            |  nvsize=1, nvalign=1, preferrednvalign=1]
+
+// CHECK:     *** Dumping AST Record Layout
+// CHECK-NEXT:          0 | struct A
+// CHECK-NEXT:          0 |   double d
+// CHECK-NEXT:            | [sizeof=8, dsize=8, align=4, preferredalign=8,
+// CHECK-NEXT:            |  nvsize=8, nvalign=4, preferrednvalign=8]
+
+struct B {
+  ~B();
+
+  Empty emp;
+  A a;
+  char c;
+};
+
+struct B1 {
+  [[no_unique_address]] B b;
+  char ext[7];
+};
+
+// CHECK:     *** Dumping AST Record Layout
+// CHECK-NEXT:          0 | struct B
+// CHECK-NEXT:          0 |   struct Empty emp (empty)
+// CHECK-NEXT:          4 |   struct A a
+// CHECK-NEXT:          4 |     double d
+// CHECK-NEXT:         12 |   char c
+// CHECK-NEXT:            | [sizeof=16, dsize=13, align=4, preferredalign=4,
+// CHECK-NEXT:            |  nvsize=13, nvalign=4, preferrednvalign=4]
+
+// CHECK:     *** Dumping AST Record Layout
+// CHECK-NEXT:          0 | struct B1
+// CHECK-NEXT:          0 |   struct B b
+// CHECK-NEXT:          0 |     struct Empty emp (empty)
+// CHECK-NEXT:          4 |     struct A a
+// CHECK-NEXT:          4 |       double d
+// CHECK-NEXT:         12 |     char c
+// CHECK-NEXT:         13 |   char [7] ext
+// CHECK-NEXT:            | [sizeof=20, dsize=20, align=4, preferredalign=4,
+// CHECK-NEXT:            |  nvsize=20, nvalign=4, preferrednvalign=4]
+
+struct C {
+  ~C();
+
+  [[no_unique_address]] Empty emp;
+  A a;
+  char c;
+};
+
+struct C1 {
+  [[no_unique_address]] C c;
+  char ext[7];
+};
+
+// CHECK:     *** Dumping AST Record Layout
+// CHECK-NEXT:          0 | struct C
+// CHECK-NEXT:          0 |   struct Empty emp (empty)
+// CHECK-NEXT:          0 |   struct A a
+// CHECK-NEXT:          0 |     double d
+// CHECK-NEXT:          8 |   char c
+// CHECK-NEXT:            | [sizeof=16, dsize=9, align=4, preferredalign=8,
+// CHECK-NEXT:            |  nvsize=9, nvalign=4, preferrednvalign=8]
+
+// CHECK:     *** Dumping AST Record Layout
+// CHECK-NEXT:          0 | struct C1
+// CHECK-NEXT:          0 |   struct C c
+// CHECK-NEXT:          0 |     struct Empty emp (empty)
+// CHECK-NEXT:          0 |     struct A a
+// CHECK-NEXT:          0 |       double d
+// CHECK-NEXT:          8 |     char c
+// CHECK-NEXT:          9 |   char [7] ext
+// CHECK-NEXT:            | [sizeof=16, dsize=16, align=4, preferredalign=8,
+// CHECK-NEXT:            |  nvsize=16, nvalign=4, preferrednvalign=8]
+
+struct D {
+  ~D();
+
+  [[no_unique_address]] char notEmp;
+  A a;
+  char c;
+};
+
+struct D1 {
+  [[no_unique_address]] D d;
+  char ext[7];
+};
+
+// CHECK:     *** Dumping AST Record Layout
+// CHECK-NEXT:          0 | struct D
+// CHECK-NEXT:          0 |   char notEmp
+// CHECK-NEXT:          4 |   struct A a
+// CHECK-NEXT:          4 |     double d
+// CHECK-NEXT:         12 |   char c
+// CHECK-NEXT:            | [sizeof=16, dsize=13, align=4, preferredalign=4,
+// CHECK-NEXT:            |  nvsize=13, nvalign=4, preferrednvalign=4]
+
+// CHECK:     *** Dumping AST Record Layout
+// CHECK-NEXT:          0 | struct D1
+// CHECK-NEXT:          0 |   struct D d
+// CHECK-NEXT:          0 |     char notEmp
+// CHECK-NEXT:          4 |     struct A a
+// CHECK-NEXT:          4 |       double d
+// CHECK-NEXT:         12 |     char c
+// CHECK-NEXT:         13 |   char [7] ext
+// CHECK-NEXT:            | [sizeof=20, dsize=20, align=4, preferredalign=4,
+// CHECK-NEXT:            |  nvsize=20, nvalign=4, preferrednvalign=4]
+
+struct E {
+  [[no_unique_address]] Empty emp;
+  int : 0;
+  double d;
+};
+
+// CHECK:     *** Dumping AST Record Layout
+// CHECK-NEXT:          0 | struct E
+// CHECK-NEXT:          0 |   struct Empty emp (empty)
+// CHECK-NEXT:        0:- |   int
+// CHECK-NEXT:          0 |   double d
+// CHECK-NEXT:            | [sizeof=8, dsize=8, align=4, preferredalign=4,
+// CHECK-NEXT:            |  nvsize=8, nvalign=4, preferrednvalign=4]
+
+struct F {
+  [[no_unique_address]] Empty emp, emp2;
+  double d;
+};
+
+// CHECK:      *** Dumping AST Record Layout
+// CHECK-NEXT: 0 | struct F
+// CHECK-NEXT: 0 |   struct Empty emp (empty)
+// CHECK-NEXT: 1 |   struct Empty emp2 (empty)
+// CHECK-NEXT: 0 |   double d
+// CHECK-NEXT:   | [sizeof=8, dsize=8, align=4, preferredalign=8,
+// CHECK-NEXT:   |  nvsize=8, nvalign=4, preferrednvalign=8]
+
+int a = sizeof(Empty);
+int b = sizeof(A);
+int c = sizeof(B1);
+int d = sizeof(C1);
+int e = sizeof(D1);
+int f = sizeof(E);
+int g = sizeof(F);
diff --git a/clang/test/Layout/aix-pack-attr-on-base.cpp b/clang/test/Layout/aix-pack-attr-on-base.cpp
new file mode 100644
index 0000000000000..3d0ebabf79a82
--- /dev/null
+++ b/clang/test/Layout/aix-pack-attr-on-base.cpp
@@ -0,0 +1,20 @@
+// RUN: %clang_cc1 -emit-llvm -triple powerpc-ibm-aix-xcoff -x c++ < %s | \
+// RUN:   FileCheck %s
+
+// RUN: %clang_cc1 -emit-llvm -triple powerpc64-ibm-aix-xcoff -x c++ < %s | \
+// RUN:   FileCheck %s
+
+struct A {
+  char x;
+};
+
+struct B {
+  int x;
+};
+
+struct __attribute__((__packed__)) C : A, B {} c;
+
+int s = sizeof(c);
+
+// CHECK: @c = global %struct.C zeroinitializer, align 1
+// CHECK: @s = global i32 5
diff --git a/clang/test/Layout/aix-power-alignment-typedef-2.cpp b/clang/test/Layout/aix-power-alignment-typedef-2.cpp
new file mode 100644
index 0000000000000..8e7e3db47c602
--- /dev/null
+++ b/clang/test/Layout/aix-power-alignment-typedef-2.cpp
@@ -0,0 +1,15 @@
+// RUN: %clang_cc1 -triple powerpc-ibm-aix-xcoff -S -emit-llvm -x c++ < %s | \
+// RUN:   FileCheck %s
+
+// RUN: %clang_cc1 -triple powerpc64-ibm-aix-xcoff -S -emit-llvm -x c++ < %s | \
+// RUN:   FileCheck %s
+
+struct C {
+  double x;
+};
+
+typedef struct C __attribute__((__aligned__(2))) CC;
+
+CC cc;
+
+// CHECK: @cc = global %struct.C zeroinitializer, align 2
diff --git a/clang/test/Layout/aix-power-alignment-typedef.cpp b/clang/test/Layout/aix-power-alignment-typedef.cpp
new file mode 100644
index 0000000000000..fc973a1fdfd81
--- /dev/null
+++ b/clang/test/Layout/aix-power-alignment-typedef.cpp
@@ -0,0 +1,39 @@
+// RUN: %clang_cc1 -triple powerpc-ibm-aix-xcoff -fdump-record-layouts %s | \
+// RUN:   FileCheck %s
+
+// RUN: %clang_cc1 -triple powerpc64-ibm-aix-xcoff -fdump-record-layouts %s | \
+// RUN:   FileCheck %s
+
+namespace test1 {
+typedef double __attribute__((__aligned__(2))) Dbl;
+struct A {
+  Dbl x;
+};
+
+int b = sizeof(A);
+
+// CHECK:          0 | struct test1::A
+// CHECK-NEXT:     0 |   test1::Dbl x
+// CHECK-NEXT:       | [sizeof=8, dsize=8, align=2, preferredalign=2,
+// CHECK-NEXT:       |  nvsize=8, nvalign=2, preferrednvalign=2]
+
+} // namespace test1
+
+namespace test2 {
+typedef double Dbl __attribute__((__aligned__(2)));
+typedef Dbl DblArr[];
+
+union U {
+  DblArr da;
+  char x;
+};
+
+int x = sizeof(U);
+
+// CHECK:          0 | union test2::U
+// CHECK-NEXT:     0 |   test2::DblArr da
+// CHECK-NEXT:     0 |   char x
+// CHECK-NEXT:       | [sizeof=2, dsize=2, align=2, preferredalign=2,
+// CHECK-NEXT:       |  nvsize=2, nvalign=2, preferrednvalign=2]
+
+} // namespace test2
diff --git a/clang/test/Layout/aix-virtual-function-and-base-with-double.cpp b/clang/test/Layout/aix-virtual-function-and-base-with-double.cpp
new file mode 100644
index 0000000000000..d3bc4418db1fe
--- /dev/null
+++ b/clang/test/Layout/aix-virtual-function-and-base-with-double.cpp
@@ -0,0 +1,112 @@
+// RUN: %clang_cc1 -triple powerpc-ibm-aix-xcoff -fdump-record-layouts \
+// RUN:     -fsyntax-only %s | \
+// RUN:   FileCheck --check-prefixes=CHECK,CHECK32 %s
+
+// RUN: %clang_cc1 -triple powerpc64-ibm-aix-xcoff -fdump-record-layouts \
+// RUN:     -fsyntax-only %s | \
+// RUN:   FileCheck --check-prefixes=CHECK,CHECK64 %s
+
+namespace test1 {
+struct A {
+  double d1;
+  virtual void boo() {}
+};
+
+struct B {
+  double d2;
+  A a;
+};
+
+struct C : public A {
+  double d3;
+};
+
+int i = sizeof(B);
+int j = sizeof(C);
+
+// CHECK:      *** Dumping AST Record Layout
+// CHECK-NEXT:            0 | struct test1::A
+// CHECK-NEXT:            0 |   (A vtable pointer)
+// CHECK32-NEXT:          4 |   double d1
+// CHECK32-NEXT:            | [sizeof=12, dsize=12, align=4, preferredalign=4,
+// CHECK32-NEXT:            |  nvsize=12, nvalign=4, preferrednvalign=4]
+// CHECK64-NEXT:          8 |   double d1
+// CHECK64-NEXT:            | [sizeof=16, dsize=16, align=8, preferredalign=8,
+// CHECK64-NEXT:            |  nvsize=16, nvalign=8, preferrednvalign=8]
+
+// CHECK:      *** Dumping AST Record Layout
+// CHECK-NEXT:            0 | struct test1::B
+// CHECK-NEXT:            0 |   double d2
+// CHECK-NEXT:            8 |   struct test1::A a
+// CHECK-NEXT:            8 |     (A vtable pointer)
+// CHECK32-NEXT:         12 |     double d1
+// CHECK32-NEXT:            | [sizeof=24, dsize=20, align=4, preferredalign=8,
+// CHECK32-NEXT:            |  nvsize=20, nvalign=4, preferrednvalign=8]
+// CHECK64-NEXT:         16 |     double d1
+// CHECK64-NEXT:            | [sizeof=24, dsize=24, align=8, preferredalign=8,
+// CHECK64-NEXT:            |  nvsize=24, nvalign=8, preferrednvalign=8]
+
+// CHECK:      *** Dumping AST Record Layout
+// CHECK-NEXT:            0 | struct test1::C
+// CHECK-NEXT:            0 |   struct test1::A (primary base)
+// CHECK-NEXT:            0 |     (A vtable pointer)
+// CHECK32-NEXT:          4 |     double d1
+// CHECK32-NEXT:         12 |   double d3
+// CHECK32-NEXT:            | [sizeof=20, dsize=20, align=4, preferredalign=4,
+// CHECK32-NEXT:            |  nvsize=20, nvalign=4, preferrednvalign=4]
+// CHECK64-NEXT:          8 |     double d1
+// CHECK64-NEXT:         16 |   double d3
+// CHECK64-NEXT:            | [sizeof=24, dsize=24, align=8, preferredalign=8,
+// CHECK64-NEXT:            |  nvsize=24, nvalign=8, preferrednvalign=8]
+
+} // namespace test1
+
+namespace test2 {
+struct A {
+  long long l1;
+};
+
+struct B : public virtual A {
+  double d2;
+};
+
+#pragma pack(2)
+struct C : public virtual A {
+  double __attribute__((aligned(4))) d3;
+};
+
+int i = sizeof(B);
+int j = sizeof(C);
+
+// CHECK:      *** Dumping AST Record Layout
+// CHECK-NEXT:            0 | struct test2::A
+// CHECK-NEXT:            0 |   long long l1
+// CHECK-NEXT:              | [sizeof=8, dsize=8, align=8, preferredalign=8,
+// CHECK-NEXT:              |  nvsize=8, nvalign=8, preferrednvalign=8]
+
+// CHECK:      *** Dumping AST Record Layout
+// CHECK-NEXT:            0 | struct test2::B
+// CHECK-NEXT:            0 |   (B vtable pointer)
+// CHECK32-NEXT:          4 |   double d2
+// CHECK64-NEXT:          8 |   double d2
+// CHECK-NEXT:           16 |   struct test2::A (virtual base)
+// CHECK-NEXT:           16 |     long long l1
+// CHECK-NEXT:              | [sizeof=24, dsize=24, align=8, preferredalign=8,
+// CHECK32-NEXT:            |  nvsize=12, nvalign=4, preferrednvalign=4]
+// CHECK64-NEXT:            |  nvsize=16, nvalign=8, preferrednvalign=8]
+
+// CHECK:      *** Dumping AST Record Layout
+// CHECK-NEXT:            0 | struct test2::C
+// CHECK-NEXT:            0 |   (C vtable pointer)
+// CHECK32-NEXT:          4 |   double d3
+// CHECK32-NEXT:         12 |   struct test2::A (virtual base)
+// CHECK32-NEXT:         12 |     long long l1
+// CHECK32-NEXT:            | [sizeof=20, dsize=20, align=2, preferredalign=2,
+// CHECK32-NEXT:            |  nvsize=12, nvalign=2, preferrednvalign=2]
+// CHECK64-NEXT:          8 |   double d3
+// CHECK64-NEXT:         16 |   struct test2::A (virtual base)
+// CHECK64-NEXT:         16 |     long long l1
+// CHECK64-NEXT:            | [sizeof=24, dsize=24, align=2, preferredalign=2,
+// CHECK64-NEXT:            |  nvsize=16, nvalign=2, preferrednvalign=2]
+
+} // namespace test2

From fbe911ee750fe62061eb15c5c8f71270fdc2fe98 Mon Sep 17 00:00:00 2001
From: MaheshRavishankar <ravishankarm@google.com>
Date: Mon, 27 Jul 2020 12:13:08 -0700
Subject: [PATCH 0233/1035] [mlir][AffineToStandard] Make LowerAffine pass
 Op-agnostic.

The LowerAffine psas was a FunctionPass only for legacy
reasons. Making this Op-agnostic allows it to be used from command
line when affine expressions are within operations other than
`std.func`.

Differential Revision: https://reviews.llvm.org/D84590
---
 mlir/include/mlir/Conversion/Passes.td            |  4 ++--
 mlir/include/mlir/Transforms/Passes.h             |  2 +-
 .../AffineToStandard/AffineToStandard.cpp         |  6 +++---
 .../AffineToStandard/lower-affine-gpu.mlir        | 15 +++++++++++++++
 4 files changed, 21 insertions(+), 6 deletions(-)
 create mode 100644 mlir/test/Conversion/AffineToStandard/lower-affine-gpu.mlir

diff --git a/mlir/include/mlir/Conversion/Passes.td b/mlir/include/mlir/Conversion/Passes.td
index 1c3b776e97808..f4c790655b1f1 100644
--- a/mlir/include/mlir/Conversion/Passes.td
+++ b/mlir/include/mlir/Conversion/Passes.td
@@ -15,12 +15,12 @@ include "mlir/Pass/PassBase.td"
 // AffineToStandard
 //===----------------------------------------------------------------------===//
 
-def ConvertAffineToStandard : FunctionPass<"lower-affine"> {
+def ConvertAffineToStandard : Pass<"lower-affine"> {
   let summary = "Lower Affine operations to a combination of Standard and SCF "
                 "operations";
   let description = [{
 
-    Convert operations from the affine dialect into operations from the loop and
+    Convert operations from the affine dialect into operations from the SCF and
     standard dialects.
 
     `affine.for` operations are converted to `scf.for` operations that are free
diff --git a/mlir/include/mlir/Transforms/Passes.h b/mlir/include/mlir/Transforms/Passes.h
index d528723af8ec6..955b0e99a1d17 100644
--- a/mlir/include/mlir/Transforms/Passes.h
+++ b/mlir/include/mlir/Transforms/Passes.h
@@ -57,7 +57,7 @@ std::unique_ptr<OperationPass<FuncOp>> createPipelineDataTransferPass();
 /// Lowers affine control flow operations (ForStmt, IfStmt and AffineApplyOp)
 /// to equivalent lower-level constructs (flow of basic blocks and arithmetic
 /// primitives).
-std::unique_ptr<OperationPass<FuncOp>> createLowerAffinePass();
+std::unique_ptr<Pass> createLowerAffinePass();
 
 /// Creates a pass that transforms perfectly nested loops with independent
 /// bounds into a single loop.
diff --git a/mlir/lib/Conversion/AffineToStandard/AffineToStandard.cpp b/mlir/lib/Conversion/AffineToStandard/AffineToStandard.cpp
index bc48ef35fcd10..a0cfdb8e6fa7b 100644
--- a/mlir/lib/Conversion/AffineToStandard/AffineToStandard.cpp
+++ b/mlir/lib/Conversion/AffineToStandard/AffineToStandard.cpp
@@ -672,14 +672,14 @@ void mlir::populateAffineToVectorConversionPatterns(
 
 namespace {
 class LowerAffinePass : public ConvertAffineToStandardBase<LowerAffinePass> {
-  void runOnFunction() override {
+  void runOnOperation() override {
     OwningRewritePatternList patterns;
     populateAffineToStdConversionPatterns(patterns, &getContext());
     populateAffineToVectorConversionPatterns(patterns, &getContext());
     ConversionTarget target(getContext());
     target
         .addLegalDialect<scf::SCFDialect, StandardOpsDialect, VectorDialect>();
-    if (failed(applyPartialConversion(getFunction(), target, patterns)))
+    if (failed(applyPartialConversion(getOperation(), target, patterns)))
       signalPassFailure();
   }
 };
@@ -687,6 +687,6 @@ class LowerAffinePass : public ConvertAffineToStandardBase<LowerAffinePass> {
 
 /// Lowers If and For operations within a function into their lower level CFG
 /// equivalent blocks.
-std::unique_ptr<OperationPass<FuncOp>> mlir::createLowerAffinePass() {
+std::unique_ptr<Pass> mlir::createLowerAffinePass() {
   return std::make_unique<LowerAffinePass>();
 }
diff --git a/mlir/test/Conversion/AffineToStandard/lower-affine-gpu.mlir b/mlir/test/Conversion/AffineToStandard/lower-affine-gpu.mlir
new file mode 100644
index 0000000000000..8e0d4f4fe8a4e
--- /dev/null
+++ b/mlir/test/Conversion/AffineToStandard/lower-affine-gpu.mlir
@@ -0,0 +1,15 @@
+// RUN: mlir-opt -pass-pipeline="gpu.module(lower-affine)" %s | FileCheck %s
+
+#map0gpufunc = affine_map<(d0) -> (d0)>
+gpu.module @kernels {
+  gpu.func @foo(%arg0 : index, %arg1 : memref<?xf32>) -> f32 {
+    %0 = affine.apply #map0gpufunc(%arg0)
+    %1 = load %arg1[%0] : memref<?xf32>
+    gpu.return %1 : f32
+  }
+
+//      CHECK: gpu.func
+// CHECK-SAME: %[[ARG0:.*]]: index
+//  CHECK-NOT:   affine.apply
+//      CHECK:   load %{{.*}}[%[[ARG0]]]
+}

From adffce71538e219aab4eeb024819baa7687262ff Mon Sep 17 00:00:00 2001
From: Jinsong Ji <jji@us.ibm.com>
Date: Mon, 27 Jul 2020 18:01:40 +0000
Subject: [PATCH 0234/1035] [PowerPC] Remove QPX/A2Q BGQ/BGP CNK support

Per RFC http://lists.llvm.org/pipermail/llvm-dev/2020-April/141295.html
no one is making use of QPX/A2Q/BGQ/BGP CNK anymore.

This patch remove the support of QPX/A2Q in llvm, BGQ/BGP in clang,
CNK support in openmp/polly.

Reviewed By: hfinkel

Differential Revision: https://reviews.llvm.org/D83915
---
 clang/lib/Basic/Targets/PPC.cpp               |   39 +-
 clang/lib/Basic/Targets/PPC.h                 |    3 -
 clang/lib/Driver/ToolChains/Arch/PPC.cpp      |    1 -
 clang/lib/Driver/ToolChains/Clang.cpp         |   12 -
 clang/test/Driver/clang-translation.c         |    6 -
 clang/test/Driver/ppc-abi.c                   |   20 -
 clang/test/Misc/target-invalid-cpu-note.c     |    2 +-
 clang/test/Preprocessor/init-ppc64.c          |   16 -
 llvm/docs/LangRef.rst                         |   11 +-
 llvm/include/llvm/ADT/Triple.h                |    3 -
 llvm/include/llvm/IR/IntrinsicsPowerPC.td     |  176 ---
 llvm/lib/Support/Triple.cpp                   |    6 -
 .../Target/PowerPC/AsmParser/PPCAsmParser.cpp |   18 -
 llvm/lib/Target/PowerPC/CMakeLists.txt        |    1 -
 .../PowerPC/Disassembler/PPCDisassembler.cpp  |   15 +-
 .../PowerPC/MCTargetDesc/PPCInstPrinter.cpp   |   12 -
 .../PowerPC/MCTargetDesc/PPCMCTargetDesc.h    |    1 -
 llvm/lib/Target/PowerPC/PPC.h                 |    2 -
 llvm/lib/Target/PowerPC/PPC.td                |   14 +-
 llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp     |    3 -
 llvm/lib/Target/PowerPC/PPCCallingConv.td     |   16 -
 llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp   |   16 +-
 llvm/lib/Target/PowerPC/PPCISelLowering.cpp   | 1025 +-------------
 llvm/lib/Target/PowerPC/PPCISelLowering.h     |   20 -
 llvm/lib/Target/PowerPC/PPCInstrFormats.td    |   52 -
 llvm/lib/Target/PowerPC/PPCInstrInfo.cpp      |   27 +-
 llvm/lib/Target/PowerPC/PPCInstrInfo.h        |   23 +-
 llvm/lib/Target/PowerPC/PPCInstrInfo.td       |   11 -
 llvm/lib/Target/PowerPC/PPCInstrQPX.td        | 1212 -----------------
 llvm/lib/Target/PowerPC/PPCQPXLoadSplat.cpp   |  161 ---
 llvm/lib/Target/PowerPC/PPCRegisterInfo.cpp   |    3 -
 llvm/lib/Target/PowerPC/PPCRegisterInfo.h     |    1 -
 llvm/lib/Target/PowerPC/PPCRegisterInfo.td    |   23 -
 llvm/lib/Target/PowerPC/PPCScheduleP9.td      |    9 +-
 llvm/lib/Target/PowerPC/PPCSubtarget.cpp      |   12 +-
 llvm/lib/Target/PowerPC/PPCSubtarget.h        |   14 -
 llvm/lib/Target/PowerPC/PPCTargetMachine.cpp  |   23 +-
 .../Target/PowerPC/PPCTargetTransformInfo.cpp |   76 +-
 .../Instrumentation/MemorySanitizer.cpp       |    7 +-
 llvm/test/Analysis/BasicAA/phi-spec-order.ll  |    2 +-
 .../CostModel/PowerPC/unal-vec-ldst.ll        |   73 -
 .../CodeGen/PowerPC/2012-11-16-mischedcall.ll |    2 +-
 ...leHoistingDueToBlockHotnessProfileData.mir |    2 +-
 .../NoCRFieldRedefWhenSpillingCRBIT.mir       |    2 +-
 llvm/test/CodeGen/PowerPC/a2q-stackalign.ll   |   23 -
 llvm/test/CodeGen/PowerPC/a2q.ll              |   10 -
 .../PowerPC/aantidep-inline-asm-use.ll        |    2 +-
 llvm/test/CodeGen/PowerPC/asm-Zy.ll           |    3 +-
 llvm/test/CodeGen/PowerPC/asm-constraints.ll  |    2 +-
 ...rt-rr-to-ri-instrs-R0-special-handling.mir |    4 +-
 .../convert-rr-to-ri-instrs-out-of-range.mir  |    2 +-
 .../PowerPC/convert-rr-to-ri-instrs.mir       |    8 +-
 llvm/test/CodeGen/PowerPC/ctr-minmaxnum.ll    |   11 -
 .../CodeGen/PowerPC/ctrloop-shortLoops.ll     |    7 -
 llvm/test/CodeGen/PowerPC/ec-input.ll         |    2 +-
 .../CodeGen/PowerPC/extra-toc-reg-deps.ll     |    8 +-
 .../CodeGen/PowerPC/fast-isel-icmp-split.ll   |    2 +-
 .../PowerPC/fma-mutate-duplicate-vreg.ll      |    2 +-
 .../CodeGen/PowerPC/fp2int2fp-ppcfp128.ll     |    3 +-
 .../CodeGen/PowerPC/glob-comp-aa-crash.ll     |    4 +-
 .../PowerPC/ifcvt-forked-bug-2016-08-08.ll    |    2 +-
 .../test/CodeGen/PowerPC/inlineasm-i64-reg.ll |    4 +-
 llvm/test/CodeGen/PowerPC/load-two-flts.ll    |    3 +-
 .../PowerPC/loop-data-prefetch-inner.ll       |    4 +-
 .../CodeGen/PowerPC/loop-data-prefetch.ll     |    4 +-
 llvm/test/CodeGen/PowerPC/loop-prep-all.ll    |   10 +-
 .../PowerPC/lxv-aligned-stack-slots.ll        |    2 +-
 llvm/test/CodeGen/PowerPC/machine-combiner.ll |   24 -
 llvm/test/CodeGen/PowerPC/mc-instrlat.ll      |    4 +-
 llvm/test/CodeGen/PowerPC/mcount-insertion.ll |    3 +-
 llvm/test/CodeGen/PowerPC/memcpy-vec.ll       |   23 -
 llvm/test/CodeGen/PowerPC/memset-nc.ll        |   48 -
 .../PowerPC/misched-inorder-latency.ll        |    3 +-
 llvm/test/CodeGen/PowerPC/misched.ll          |    1 -
 .../CodeGen/PowerPC/optnone-crbits-i1-ret.ll  |    3 +-
 .../CodeGen/PowerPC/pcrel-local-caller-toc.ll |    6 +-
 llvm/test/CodeGen/PowerPC/popcnt.ll           |    2 -
 llvm/test/CodeGen/PowerPC/ppc-passname.ll     |   11 -
 llvm/test/CodeGen/PowerPC/ppc64-sibcall.ll    |   21 +-
 llvm/test/CodeGen/PowerPC/pr24546.ll          |    4 +-
 llvm/test/CodeGen/PowerPC/pr27350.ll          |    2 +-
 llvm/test/CodeGen/PowerPC/pr28130.ll          |    2 +-
 .../CodeGen/PowerPC/preinc-ld-sel-crash.ll    |    2 +-
 llvm/test/CodeGen/PowerPC/qpx-bv-sint.ll      |   33 -
 llvm/test/CodeGen/PowerPC/qpx-bv.ll           |   37 -
 llvm/test/CodeGen/PowerPC/qpx-func-clobber.ll |   22 -
 llvm/test/CodeGen/PowerPC/qpx-load-splat.ll   |   80 --
 llvm/test/CodeGen/PowerPC/qpx-load.ll         |   26 -
 llvm/test/CodeGen/PowerPC/qpx-qvfmadd.ll      |   79 --
 llvm/test/CodeGen/PowerPC/qpx-recipest.ll     |  473 -------
 llvm/test/CodeGen/PowerPC/qpx-rounding-ops.ll |  109 --
 llvm/test/CodeGen/PowerPC/qpx-s-load.ll       |   26 -
 llvm/test/CodeGen/PowerPC/qpx-s-sel.ll        |  143 --
 llvm/test/CodeGen/PowerPC/qpx-s-store.ll      |   25 -
 llvm/test/CodeGen/PowerPC/qpx-sel.ll          |  151 --
 llvm/test/CodeGen/PowerPC/qpx-split-vsetcc.ll |   31 -
 llvm/test/CodeGen/PowerPC/qpx-store.ll        |   25 -
 .../test/CodeGen/PowerPC/qpx-unal-cons-lds.ll |  217 ---
 llvm/test/CodeGen/PowerPC/qpx-unalperm.ll     |   64 -
 llvm/test/CodeGen/PowerPC/rlwimi-and.ll       |    4 +-
 .../CodeGen/PowerPC/rlwinm_rldicl_to_andi.mir |    2 +-
 .../CodeGen/PowerPC/s000-alias-misched.ll     |    5 +-
 llvm/test/CodeGen/PowerPC/select-i1-vs-i1.ll  |  571 --------
 .../selectiondag-extload-computeknownbits.ll  |    2 +-
 llvm/test/CodeGen/PowerPC/setcr_bc.mir        |    4 +-
 llvm/test/CodeGen/PowerPC/setcr_bc2.mir       |    4 +-
 llvm/test/CodeGen/PowerPC/stwu-sched.ll       |    2 +-
 llvm/test/CodeGen/PowerPC/unal-vec-ldst.ll    |  149 --
 llvm/test/CodeGen/PowerPC/uwtables.ll         |    2 +-
 .../MemorySanitizer/PowerPC/vararg-ppc64.ll   |   15 -
 llvm/test/MC/Disassembler/PowerPC/qpx.txt     |  371 -----
 llvm/test/MC/PowerPC/qpx.s                    |  252 ----
 .../IPConstantProp/fp-bc-icmp-const-fold.ll   |    2 +-
 .../MSSA/combined-partial-overwrites.ll       |    2 +-
 .../combined-partial-overwrites.ll            |    2 +-
 .../EntryExitInstrumenter/mcount.ll           |    2 +-
 .../InstCombine/PowerPC/aligned-qpx.ll        |  165 ---
 .../LoopDataPrefetch/PowerPC/basic.ll         |    5 +-
 .../test/Transforms/LoopSimplify/dup-preds.ll |    2 +-
 llvm/test/Transforms/LoopUnroll/pr14167.ll    |    2 +-
 .../PowerPC/agg-interleave-a2.ll              |   40 -
 .../PowerPC/vectorize-only-for-real.ll        |    2 +-
 llvm/test/Transforms/NewGVN/pr31483.ll        |    4 +-
 .../Transforms/SCCP/fp-bc-icmp-const-fold.ll  |    2 +-
 llvm/unittests/ADT/TripleTest.cpp             |   35 -
 .../llvm/lib/Target/PowerPC/BUILD.gn          |    1 -
 openmp/runtime/src/kmp.h                      |    3 -
 openmp/runtime/src/kmp_csupport.cpp           |   11 -
 openmp/runtime/src/kmp_lock.h                 |    2 +-
 openmp/runtime/src/kmp_os.h                   |    2 +-
 openmp/runtime/src/kmp_platform.h             |    6 -
 openmp/runtime/src/z_Linux_asm.S              |    7 +-
 openmp/runtime/src/z_Linux_util.cpp           |    2 +-
 polly/lib/External/isl/config.sub             |   11 +-
 polly/lib/External/ppcg/config.sub            |   11 +-
 135 files changed, 174 insertions(+), 6525 deletions(-)
 delete mode 100644 llvm/lib/Target/PowerPC/PPCInstrQPX.td
 delete mode 100644 llvm/lib/Target/PowerPC/PPCQPXLoadSplat.cpp
 delete mode 100644 llvm/test/CodeGen/PowerPC/a2q-stackalign.ll
 delete mode 100644 llvm/test/CodeGen/PowerPC/a2q.ll
 delete mode 100644 llvm/test/CodeGen/PowerPC/memset-nc.ll
 delete mode 100644 llvm/test/CodeGen/PowerPC/qpx-bv-sint.ll
 delete mode 100644 llvm/test/CodeGen/PowerPC/qpx-bv.ll
 delete mode 100644 llvm/test/CodeGen/PowerPC/qpx-func-clobber.ll
 delete mode 100644 llvm/test/CodeGen/PowerPC/qpx-load-splat.ll
 delete mode 100644 llvm/test/CodeGen/PowerPC/qpx-load.ll
 delete mode 100644 llvm/test/CodeGen/PowerPC/qpx-qvfmadd.ll
 delete mode 100644 llvm/test/CodeGen/PowerPC/qpx-recipest.ll
 delete mode 100644 llvm/test/CodeGen/PowerPC/qpx-rounding-ops.ll
 delete mode 100644 llvm/test/CodeGen/PowerPC/qpx-s-load.ll
 delete mode 100644 llvm/test/CodeGen/PowerPC/qpx-s-sel.ll
 delete mode 100644 llvm/test/CodeGen/PowerPC/qpx-s-store.ll
 delete mode 100644 llvm/test/CodeGen/PowerPC/qpx-sel.ll
 delete mode 100644 llvm/test/CodeGen/PowerPC/qpx-split-vsetcc.ll
 delete mode 100644 llvm/test/CodeGen/PowerPC/qpx-store.ll
 delete mode 100644 llvm/test/CodeGen/PowerPC/qpx-unal-cons-lds.ll
 delete mode 100644 llvm/test/CodeGen/PowerPC/qpx-unalperm.ll
 delete mode 100644 llvm/test/MC/Disassembler/PowerPC/qpx.txt
 delete mode 100644 llvm/test/MC/PowerPC/qpx.s
 delete mode 100644 llvm/test/Transforms/InstCombine/PowerPC/aligned-qpx.ll
 delete mode 100644 llvm/test/Transforms/LoopVectorize/PowerPC/agg-interleave-a2.ll

diff --git a/clang/lib/Basic/Targets/PPC.cpp b/clang/lib/Basic/Targets/PPC.cpp
index f0de2bf070ea4..5f716a541ae92 100644
--- a/clang/lib/Basic/Targets/PPC.cpp
+++ b/clang/lib/Basic/Targets/PPC.cpp
@@ -46,8 +46,6 @@ bool PPCTargetInfo::handleTargetFeatures(std::vector<std::string> &Features,
       HasP8Crypto = true;
     } else if (Feature == "+direct-move") {
       HasDirectMove = true;
-    } else if (Feature == "+qpx") {
-      HasQPX = true;
     } else if (Feature == "+htm") {
       HasHTM = true;
     } else if (Feature == "+float128") {
@@ -99,7 +97,7 @@ void PPCTargetInfo::getTargetDefines(const LangOptions &Opts,
   }
 
   // ABI options.
-  if (ABI == "elfv1" || ABI == "elfv1-qpx")
+  if (ABI == "elfv1")
     Builder.defineMacro("_CALL_ELF", "1");
   if (ABI == "elfv2")
     Builder.defineMacro("_CALL_ELF", "2");
@@ -159,22 +157,11 @@ void PPCTargetInfo::getTargetDefines(const LangOptions &Opts,
     Builder.defineMacro("_ARCH_PWR10");
   if (ArchDefs & ArchDefineA2)
     Builder.defineMacro("_ARCH_A2");
-  if (ArchDefs & ArchDefineA2q) {
-    Builder.defineMacro("_ARCH_A2Q");
-    Builder.defineMacro("_ARCH_QP");
-  }
   if (ArchDefs & ArchDefineE500)
     Builder.defineMacro("__NO_LWSYNC__");
   if (ArchDefs & ArchDefineFuture)
     Builder.defineMacro("_ARCH_PWR_FUTURE");
 
-  if (getTriple().getVendor() == llvm::Triple::BGQ) {
-    Builder.defineMacro("__bg__");
-    Builder.defineMacro("__THW_BLUEGENE__");
-    Builder.defineMacro("__bgq__");
-    Builder.defineMacro("__TOS_BGQ__");
-  }
-
   if (HasAltivec) {
     Builder.defineMacro("__VEC__", "10206");
     Builder.defineMacro("__ALTIVEC__");
@@ -277,7 +264,6 @@ bool PPCTargetInfo::initFeatureMap(
                             .Case("ppc64le", true)
                             .Default(false);
 
-  Features["qpx"] = (CPU == "a2q");
   Features["power9-vector"] = (CPU == "pwr9");
   Features["crypto"] = llvm::StringSwitch<bool>(CPU)
                            .Case("ppc64le", true)
@@ -373,7 +359,6 @@ bool PPCTargetInfo::hasFeature(StringRef Feature) const {
       .Case("power8-vector", HasP8Vector)
       .Case("crypto", HasP8Crypto)
       .Case("direct-move", HasDirectMove)
-      .Case("qpx", HasQPX)
       .Case("htm", HasHTM)
       .Case("bpermd", HasBPERMD)
       .Case("extdiv", HasExtDiv)
@@ -503,17 +488,17 @@ ArrayRef<TargetInfo::AddlRegName> PPCTargetInfo::getGCCAddlRegNames() const {
 }
 
 static constexpr llvm::StringLiteral ValidCPUNames[] = {
-    {"generic"},     {"440"},     {"450"},     {"601"},       {"602"},
-    {"603"},         {"603e"},    {"603ev"},   {"604"},       {"604e"},
-    {"620"},         {"630"},     {"g3"},      {"7400"},      {"g4"},
-    {"7450"},        {"g4+"},     {"750"},     {"8548"},      {"970"},
-    {"g5"},          {"a2"},      {"a2q"},     {"e500"},      {"e500mc"},
-    {"e5500"},       {"power3"},  {"pwr3"},    {"power4"},    {"pwr4"},
-    {"power5"},      {"pwr5"},    {"power5x"}, {"pwr5x"},     {"power6"},
-    {"pwr6"},        {"power6x"}, {"pwr6x"},   {"power7"},    {"pwr7"},
-    {"power8"},      {"pwr8"},    {"power9"},  {"pwr9"},      {"power10"},
-    {"pwr10"},       {"powerpc"}, {"ppc"},     {"powerpc64"}, {"ppc64"},
-    {"powerpc64le"}, {"ppc64le"}, {"future"}};
+    {"generic"}, {"440"},     {"450"},       {"601"},     {"602"},
+    {"603"},     {"603e"},    {"603ev"},     {"604"},     {"604e"},
+    {"620"},     {"630"},     {"g3"},        {"7400"},    {"g4"},
+    {"7450"},    {"g4+"},     {"750"},       {"8548"},    {"970"},
+    {"g5"},      {"a2"},      {"e500"},      {"e500mc"},  {"e5500"},
+    {"power3"},  {"pwr3"},    {"power4"},    {"pwr4"},    {"power5"},
+    {"pwr5"},    {"power5x"}, {"pwr5x"},     {"power6"},  {"pwr6"},
+    {"power6x"}, {"pwr6x"},   {"power7"},    {"pwr7"},    {"power8"},
+    {"pwr8"},    {"power9"},  {"pwr9"},      {"power10"}, {"pwr10"},
+    {"powerpc"}, {"ppc"},     {"powerpc64"}, {"ppc64"},   {"powerpc64le"},
+    {"ppc64le"}, {"future"}};
 
 bool PPCTargetInfo::isValidCPUName(StringRef Name) const {
   return llvm::find(ValidCPUNames, Name) != std::end(ValidCPUNames);
diff --git a/clang/lib/Basic/Targets/PPC.h b/clang/lib/Basic/Targets/PPC.h
index ff8579b6c3cf4..c2048b2145918 100644
--- a/clang/lib/Basic/Targets/PPC.h
+++ b/clang/lib/Basic/Targets/PPC.h
@@ -46,7 +46,6 @@ class LLVM_LIBRARY_VISIBILITY PPCTargetInfo : public TargetInfo {
     ArchDefinePwr10 = 1 << 14,
     ArchDefineFuture = 1 << 15,
     ArchDefineA2 = 1 << 16,
-    ArchDefineA2q = 1 << 17,
     ArchDefineE500 = 1 << 18
   } ArchDefineTypes;
 
@@ -63,7 +62,6 @@ class LLVM_LIBRARY_VISIBILITY PPCTargetInfo : public TargetInfo {
   bool HasP8Vector = false;
   bool HasP8Crypto = false;
   bool HasDirectMove = false;
-  bool HasQPX = false;
   bool HasHTM = false;
   bool HasBPERMD = false;
   bool HasExtDiv = false;
@@ -118,7 +116,6 @@ class LLVM_LIBRARY_VISIBILITY PPCTargetInfo : public TargetInfo {
               .Case("970", ArchDefineName | ArchDefinePwr4 | ArchDefinePpcgr |
                                ArchDefinePpcsq)
               .Case("a2", ArchDefineA2)
-              .Case("a2q", ArchDefineName | ArchDefineA2 | ArchDefineA2q)
               .Cases("power3", "pwr3", ArchDefinePpcgr)
               .Cases("power4", "pwr4",
                      ArchDefinePwr4 | ArchDefinePpcgr | ArchDefinePpcsq)
diff --git a/clang/lib/Driver/ToolChains/Arch/PPC.cpp b/clang/lib/Driver/ToolChains/Arch/PPC.cpp
index 144e276a6bd87..bcaecf4b2d980 100644
--- a/clang/lib/Driver/ToolChains/Arch/PPC.cpp
+++ b/clang/lib/Driver/ToolChains/Arch/PPC.cpp
@@ -57,7 +57,6 @@ std::string ppc::getPPCTargetCPU(const ArgList &Args) {
         .Case("970", "970")
         .Case("G5", "g5")
         .Case("a2", "a2")
-        .Case("a2q", "a2q")
         .Case("e500", "e500")
         .Case("e500mc", "e500mc")
         .Case("e5500", "e5500")
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index 7a73eea013bdf..b0de225f8abf5 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -1883,18 +1883,6 @@ void Clang::AddPPCTargetArgs(const ArgList &Args,
   if (T.isOSBinFormatELF()) {
     switch (getToolChain().getArch()) {
     case llvm::Triple::ppc64: {
-      // When targeting a processor that supports QPX, or if QPX is
-      // specifically enabled, default to using the ABI that supports QPX (so
-      // long as it is not specifically disabled).
-      bool HasQPX = false;
-      if (Arg *A = Args.getLastArg(options::OPT_mcpu_EQ))
-        HasQPX = A->getValue() == StringRef("a2q");
-      HasQPX = Args.hasFlag(options::OPT_mqpx, options::OPT_mno_qpx, HasQPX);
-      if (HasQPX) {
-        ABIName = "elfv1-qpx";
-        break;
-      }
-
       if (T.isMusl() || (T.isOSFreeBSD() && T.getOSMajorVersion() >= 13))
         ABIName = "elfv2";
       else
diff --git a/clang/test/Driver/clang-translation.c b/clang/test/Driver/clang-translation.c
index 2f02970a2a8ee..d1daeb80004b7 100644
--- a/clang/test/Driver/clang-translation.c
+++ b/clang/test/Driver/clang-translation.c
@@ -167,12 +167,6 @@
 // PPCPWR8: "-cc1"
 // PPCPWR8: "-target-cpu" "pwr8"
 
-// RUN: %clang -target powerpc64-unknown-linux-gnu \
-// RUN: -### -S %s -mcpu=a2q 2>&1 | FileCheck -check-prefix=PPCA2Q %s
-// PPCA2Q: clang
-// PPCA2Q: "-cc1"
-// PPCA2Q: "-target-cpu" "a2q"
-
 // RUN: %clang -target powerpc64-unknown-linux-gnu \
 // RUN: -### -S %s -mcpu=630 2>&1 | FileCheck -check-prefix=PPC630 %s
 // PPC630: clang
diff --git a/clang/test/Driver/ppc-abi.c b/clang/test/Driver/ppc-abi.c
index acc4981a2eee6..2b5cc463e7c3d 100644
--- a/clang/test/Driver/ppc-abi.c
+++ b/clang/test/Driver/ppc-abi.c
@@ -5,14 +5,6 @@
 // RUN: %clang -target powerpc64-unknown-linux-gnu %s -### -o %t.o 2>&1 \
 // RUN:   -mabi=elfv1 | FileCheck -check-prefix=CHECK-ELFv1 %s
 // RUN: %clang -target powerpc64-unknown-linux-gnu %s -### -o %t.o 2>&1 \
-// RUN:   -mabi=elfv1-qpx | FileCheck -check-prefix=CHECK-ELFv1-QPX %s
-// RUN: %clang -target powerpc64-unknown-linux-gnu %s -### -o %t.o 2>&1 \
-// RUN:   -mcpu=a2q | FileCheck -check-prefix=CHECK-ELFv1-QPX %s
-// RUN: %clang -target powerpc64-unknown-linux-gnu %s -### -o %t.o 2>&1 \
-// RUN:   -mcpu=a2 -mqpx | FileCheck -check-prefix=CHECK-ELFv1-QPX %s
-// RUN: %clang -target powerpc64-unknown-linux-gnu %s -### -o %t.o 2>&1 \
-// RUN:   -mcpu=a2q -mno-qpx | FileCheck -check-prefix=CHECK-ELFv1 %s
-// RUN: %clang -target powerpc64-unknown-linux-gnu %s -### -o %t.o 2>&1 \
 // RUN:   -mabi=elfv2 | FileCheck -check-prefix=CHECK-ELFv2-BE %s
 
 // RUN: %clang -target powerpc64le-unknown-linux-gnu %s -### -o %t.o 2>&1 \
@@ -34,8 +26,6 @@
 // CHECK-ELFv1: "-target-abi" "elfv1"
 // CHECK-ELFv1-LE: "-mrelocation-model" "static"
 // CHECK-ELFv1-LE: "-target-abi" "elfv1"
-// CHECK-ELFv1-QPX: "-mrelocation-model" "static"
-// CHECK-ELFv1-QPX: "-target-abi" "elfv1-qpx"
 // CHECK-ELFv2: "-mrelocation-model" "static"
 // CHECK-ELFv2: "-target-abi" "elfv2"
 // CHECK-ELFv2-BE: "-mrelocation-model" "static"
@@ -48,14 +38,6 @@
 // RUN: %clang -fPIC -target powerpc64-unknown-linux-gnu %s -### -o %t.o 2>&1 \
 // RUN:   -mabi=elfv1 | FileCheck -check-prefix=CHECK-ELFv1-PIC %s
 // RUN: %clang -fPIC -target powerpc64-unknown-linux-gnu %s -### -o %t.o 2>&1 \
-// RUN:   -mabi=elfv1-qpx | FileCheck -check-prefix=CHECK-ELFv1-QPX-PIC %s
-// RUN: %clang -fPIC -target powerpc64-unknown-linux-gnu %s -### -o %t.o 2>&1 \
-// RUN:   -mcpu=a2q | FileCheck -check-prefix=CHECK-ELFv1-QPX-PIC %s
-// RUN: %clang -fPIC -target powerpc64-unknown-linux-gnu %s -### -o %t.o 2>&1 \
-// RUN:   -mcpu=a2 -mqpx | FileCheck -check-prefix=CHECK-ELFv1-QPX-PIC %s
-// RUN: %clang -fPIC -target powerpc64-unknown-linux-gnu %s -### -o %t.o 2>&1 \
-// RUN:   -mcpu=a2q -mno-qpx | FileCheck -check-prefix=CHECK-ELFv1-PIC %s
-// RUN: %clang -fPIC -target powerpc64-unknown-linux-gnu %s -### -o %t.o 2>&1 \
 // RUN:   -mabi=elfv2 | FileCheck -check-prefix=CHECK-ELFv2-PIC %s
 
 // RUN: %clang -fPIC -target powerpc64le-unknown-linux-gnu %s -### -o %t.o 2>&1 \
@@ -69,8 +51,6 @@
 
 // CHECK-ELFv1-PIC: "-mrelocation-model" "pic" "-pic-level" "2"
 // CHECK-ELFv1-PIC: "-target-abi" "elfv1"
-// CHECK-ELFv1-QPX-PIC: "-mrelocation-model" "pic" "-pic-level" "2"
-// CHECK-ELFv1-QPX-PIC: "-target-abi" "elfv1-qpx"
 // CHECK-ELFv2-PIC: "-mrelocation-model" "pic" "-pic-level" "2"
 // CHECK-ELFv2-PIC: "-target-abi" "elfv2"
 
diff --git a/clang/test/Misc/target-invalid-cpu-note.c b/clang/test/Misc/target-invalid-cpu-note.c
index 3a376a7caab46..bf6eaefe0b3ca 100644
--- a/clang/test/Misc/target-invalid-cpu-note.c
+++ b/clang/test/Misc/target-invalid-cpu-note.c
@@ -79,7 +79,7 @@
 // PPC: error: unknown target CPU 'not-a-cpu'
 // PPC: note: valid target CPU values are: generic, 440, 450, 601, 602, 603,
 // PPC-SAME: 603e, 603ev, 604, 604e, 620, 630, g3, 7400, g4, 7450, g4+, 750,
-// PPC-SAME: 8548, 970, g5, a2, a2q, e500, e500mc, e5500, power3, pwr3, power4,
+// PPC-SAME: 8548, 970, g5, a2, e500, e500mc, e5500, power3, pwr3, power4,
 // PPC-SAME: pwr4, power5, pwr5, power5x, pwr5x, power6, pwr6, power6x, pwr6x,
 // PPC-SAME: power7, pwr7, power8, pwr8, power9, pwr9, power10, pwr10, powerpc, ppc, powerpc64,
 // PPC-SAME: ppc64, powerpc64le, ppc64le, future
diff --git a/clang/test/Preprocessor/init-ppc64.c b/clang/test/Preprocessor/init-ppc64.c
index ed8601636554e..48d35c95aa570 100644
--- a/clang/test/Preprocessor/init-ppc64.c
+++ b/clang/test/Preprocessor/init-ppc64.c
@@ -408,21 +408,6 @@
 // PPC64LE:#define __ppc64__ 1
 // PPC64LE:#define __ppc__ 1
 //
-// RUN: %clang_cc1 -E -dM -ffreestanding -triple=powerpc64-none-none -target-cpu a2q -fno-signed-char < /dev/null | FileCheck -match-full-lines -check-prefix PPCA2Q %s
-//
-// PPCA2Q:#define _ARCH_A2 1
-// PPCA2Q:#define _ARCH_A2Q 1
-// PPCA2Q:#define _ARCH_PPC 1
-// PPCA2Q:#define _ARCH_PPC64 1
-// PPCA2Q:#define _ARCH_QP 1
-//
-// RUN: %clang_cc1 -E -dM -ffreestanding -triple=powerpc64-bgq-linux -fno-signed-char < /dev/null | FileCheck -match-full-lines -check-prefix PPCBGQ %s
-//
-// PPCBGQ:#define __THW_BLUEGENE__ 1
-// PPCBGQ:#define __TOS_BGQ__ 1
-// PPCBGQ:#define __bg__ 1
-// PPCBGQ:#define __bgq__ 1
-//
 // RUN: %clang_cc1 -E -dM -ffreestanding -triple=powerpc64-none-none -target-cpu 630 -fno-signed-char < /dev/null | FileCheck -match-full-lines -check-prefix PPC630 %s
 //
 // PPC630:#define _ARCH_630 1
@@ -1069,7 +1054,6 @@
 
 // RUN: %clang_cc1 -E -dM -ffreestanding -fgnuc-version=4.2.1 -triple=powerpc64-unknown-linux-gnu < /dev/null | FileCheck -match-full-lines -check-prefix PPC64-ELFv1 %s
 // RUN: %clang_cc1 -E -dM -ffreestanding -fgnuc-version=4.2.1 -triple=powerpc64-unknown-linux-gnu -target-abi elfv1 < /dev/null | FileCheck -match-full-lines -check-prefix PPC64-ELFv1 %s
-// RUN: %clang_cc1 -E -dM -ffreestanding -fgnuc-version=4.2.1 -triple=powerpc64-unknown-linux-gnu -target-abi elfv1-qpx < /dev/null | FileCheck -match-full-lines -check-prefix PPC64-ELFv1 %s
 // RUN: %clang_cc1 -E -dM -ffreestanding -fgnuc-version=4.2.1 -triple=powerpc64-unknown-linux-gnu -target-abi elfv2 < /dev/null | FileCheck -match-full-lines -check-prefix PPC64-ELFv2 %s
 // RUN: %clang_cc1 -E -dM -ffreestanding -fgnuc-version=4.2.1 -triple=powerpc64le-unknown-linux-gnu < /dev/null | FileCheck -match-full-lines -check-prefix PPC64-ELFv2 %s
 // RUN: %clang_cc1 -E -dM -ffreestanding -fgnuc-version=4.2.1 -triple=powerpc64le-unknown-linux-gnu -target-abi elfv1 < /dev/null | FileCheck -match-full-lines -check-prefix PPC64-ELFv1 %s
diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index 6b9c5c6899819..af93a6ed5c56e 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -4310,14 +4310,9 @@ PowerPC:
 - ``r``: A 32 or 64-bit integer register.
 - ``b``: A 32 or 64-bit integer register, excluding ``R0`` (that is:
   ``R1-R31``).
-- ``f``: A 32 or 64-bit float register (``F0-F31``), or when QPX is enabled, a
-  128 or 256-bit QPX register (``Q0-Q31``; aliases the ``F`` registers).
-- ``v``: For ``4 x f32`` or ``4 x f64`` types, when QPX is enabled, a
-  128 or 256-bit QPX register (``Q0-Q31``), otherwise a 128-bit
-  altivec vector register (``V0-V31``).
-
-  .. FIXME: is this a bug that v accepts QPX registers? I think this
-     is supposed to only use the altivec vector registers?
+- ``f``: A 32 or 64-bit float register (``F0-F31``), 
+- ``v``: For ``4 x f32`` or ``4 x f64`` types, a 128-bit altivec vector
+   register (``V0-V31``).
 
 - ``y``: Condition register (``CR0-CR7``).
 - ``wc``: An individual CR bit in a CR register.
diff --git a/llvm/include/llvm/ADT/Triple.h b/llvm/include/llvm/ADT/Triple.h
index 6bad18f19244e..c578c097c6f64 100644
--- a/llvm/include/llvm/ADT/Triple.h
+++ b/llvm/include/llvm/ADT/Triple.h
@@ -142,8 +142,6 @@ class Triple {
     Apple,
     PC,
     SCEI,
-    BGP,
-    BGQ,
     Freescale,
     IBM,
     ImaginationTechnologies,
@@ -179,7 +177,6 @@ class Triple {
     Minix,
     RTEMS,
     NaCl,       // Native Client
-    CNK,        // BG/P Compute-Node Kernel
     AIX,
     CUDA,       // NVIDIA CUDA
     NVCL,       // NVIDIA OpenCL
diff --git a/llvm/include/llvm/IR/IntrinsicsPowerPC.td b/llvm/include/llvm/IR/IntrinsicsPowerPC.td
index 23bcf3ce1959c..853d26c67ee3d 100644
--- a/llvm/include/llvm/IR/IntrinsicsPowerPC.td
+++ b/llvm/include/llvm/IR/IntrinsicsPowerPC.td
@@ -1109,182 +1109,6 @@ def  int_ppc_vsx_xxblendvd: GCCBuiltin<"__builtin_vsx_xxblendvd">,
                  [IntrNoMem]>;
 }
 
-//===----------------------------------------------------------------------===//
-// PowerPC QPX Intrinsics.
-//
-
-let TargetPrefix = "ppc" in {  // All PPC intrinsics start with "llvm.ppc.".
-  /// PowerPC_QPX_Intrinsic - Base class for all QPX intrinsics.
-  class PowerPC_QPX_Intrinsic<string GCCIntSuffix, list<LLVMType> ret_types,
-                              list<LLVMType> param_types,
-                              list<IntrinsicProperty> properties>
-    : GCCBuiltin<!strconcat("__builtin_qpx_", GCCIntSuffix)>,
-      Intrinsic<ret_types, param_types, properties>;
-}
-
-//===----------------------------------------------------------------------===//
-// PowerPC QPX Intrinsic Class Definitions.
-//
-
-/// PowerPC_QPX_FF_Intrinsic - A PowerPC intrinsic that takes one v4f64
-/// vector and returns one.  These intrinsics have no side effects.
-class PowerPC_QPX_FF_Intrinsic<string GCCIntSuffix>
-  : PowerPC_QPX_Intrinsic<GCCIntSuffix,
-                          [llvm_v4f64_ty], [llvm_v4f64_ty], [IntrNoMem]>;
-
-/// PowerPC_QPX_FFF_Intrinsic - A PowerPC intrinsic that takes two v4f64
-/// vectors and returns one.  These intrinsics have no side effects.
-class PowerPC_QPX_FFF_Intrinsic<string GCCIntSuffix>
-  : PowerPC_QPX_Intrinsic<GCCIntSuffix,
-                          [llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty],
-                          [IntrNoMem]>;
-
-/// PowerPC_QPX_FFFF_Intrinsic - A PowerPC intrinsic that takes three v4f64
-/// vectors and returns one.  These intrinsics have no side effects.
-class PowerPC_QPX_FFFF_Intrinsic<string GCCIntSuffix>
-  : PowerPC_QPX_Intrinsic<GCCIntSuffix,
-                          [llvm_v4f64_ty],
-                          [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4f64_ty],
-                          [IntrNoMem]>;
-
-/// PowerPC_QPX_Load_Intrinsic - A PowerPC intrinsic that takes a pointer
-/// and returns a v4f64.
-class PowerPC_QPX_Load_Intrinsic<string GCCIntSuffix>
-  : PowerPC_QPX_Intrinsic<GCCIntSuffix,
-                          [llvm_v4f64_ty], [llvm_ptr_ty], [IntrReadMem, IntrArgMemOnly]>;
-
-/// PowerPC_QPX_LoadPerm_Intrinsic - A PowerPC intrinsic that takes a pointer
-/// and returns a v4f64 permutation.
-class PowerPC_QPX_LoadPerm_Intrinsic<string GCCIntSuffix>
-  : PowerPC_QPX_Intrinsic<GCCIntSuffix,
-                          [llvm_v4f64_ty], [llvm_ptr_ty], [IntrNoMem]>;
-
-/// PowerPC_QPX_Store_Intrinsic - A PowerPC intrinsic that takes a pointer
-/// and stores a v4f64.
-class PowerPC_QPX_Store_Intrinsic<string GCCIntSuffix>
-  : PowerPC_QPX_Intrinsic<GCCIntSuffix,
-                          [], [llvm_v4f64_ty, llvm_ptr_ty],
-                          [IntrWriteMem, IntrArgMemOnly]>;
-
-//===----------------------------------------------------------------------===//
-// PowerPC QPX Intrinsic Definitions.
-
-let TargetPrefix = "ppc" in {  // All intrinsics start with "llvm.ppc.".
-  // Add Instructions
-  def int_ppc_qpx_qvfadd : PowerPC_QPX_FFF_Intrinsic<"qvfadd">;
-  def int_ppc_qpx_qvfadds : PowerPC_QPX_FFF_Intrinsic<"qvfadds">;
-  def int_ppc_qpx_qvfsub : PowerPC_QPX_FFF_Intrinsic<"qvfsub">;
-  def int_ppc_qpx_qvfsubs : PowerPC_QPX_FFF_Intrinsic<"qvfsubs">;
-
-  // Estimate Instructions
-  def int_ppc_qpx_qvfre : PowerPC_QPX_FF_Intrinsic<"qvfre">;
-  def int_ppc_qpx_qvfres : PowerPC_QPX_FF_Intrinsic<"qvfres">;
-  def int_ppc_qpx_qvfrsqrte : PowerPC_QPX_FF_Intrinsic<"qvfrsqrte">;
-  def int_ppc_qpx_qvfrsqrtes : PowerPC_QPX_FF_Intrinsic<"qvfrsqrtes">;
-
-  // Multiply Instructions
-  def int_ppc_qpx_qvfmul : PowerPC_QPX_FFF_Intrinsic<"qvfmul">;
-  def int_ppc_qpx_qvfmuls : PowerPC_QPX_FFF_Intrinsic<"qvfmuls">;
-  def int_ppc_qpx_qvfxmul : PowerPC_QPX_FFF_Intrinsic<"qvfxmul">;
-  def int_ppc_qpx_qvfxmuls : PowerPC_QPX_FFF_Intrinsic<"qvfxmuls">;
-
-  // Multiply-add instructions
-  def int_ppc_qpx_qvfmadd : PowerPC_QPX_FFFF_Intrinsic<"qvfmadd">;
-  def int_ppc_qpx_qvfmadds : PowerPC_QPX_FFFF_Intrinsic<"qvfmadds">;
-  def int_ppc_qpx_qvfnmadd : PowerPC_QPX_FFFF_Intrinsic<"qvfnmadd">;
-  def int_ppc_qpx_qvfnmadds : PowerPC_QPX_FFFF_Intrinsic<"qvfnmadds">;
-  def int_ppc_qpx_qvfmsub : PowerPC_QPX_FFFF_Intrinsic<"qvfmsub">;
-  def int_ppc_qpx_qvfmsubs : PowerPC_QPX_FFFF_Intrinsic<"qvfmsubs">;
-  def int_ppc_qpx_qvfnmsub : PowerPC_QPX_FFFF_Intrinsic<"qvfnmsub">;
-  def int_ppc_qpx_qvfnmsubs : PowerPC_QPX_FFFF_Intrinsic<"qvfnmsubs">;
-  def int_ppc_qpx_qvfxmadd : PowerPC_QPX_FFFF_Intrinsic<"qvfxmadd">;
-  def int_ppc_qpx_qvfxmadds : PowerPC_QPX_FFFF_Intrinsic<"qvfxmadds">;
-  def int_ppc_qpx_qvfxxnpmadd : PowerPC_QPX_FFFF_Intrinsic<"qvfxxnpmadd">;
-  def int_ppc_qpx_qvfxxnpmadds : PowerPC_QPX_FFFF_Intrinsic<"qvfxxnpmadds">;
-  def int_ppc_qpx_qvfxxcpnmadd : PowerPC_QPX_FFFF_Intrinsic<"qvfxxcpnmadd">;
-  def int_ppc_qpx_qvfxxcpnmadds : PowerPC_QPX_FFFF_Intrinsic<"qvfxxcpnmadds">;
-  def int_ppc_qpx_qvfxxmadd : PowerPC_QPX_FFFF_Intrinsic<"qvfxxmadd">;
-  def int_ppc_qpx_qvfxxmadds : PowerPC_QPX_FFFF_Intrinsic<"qvfxxmadds">;
-
-  // Select Instruction
-  def int_ppc_qpx_qvfsel : PowerPC_QPX_FFFF_Intrinsic<"qvfsel">;
-
-  // Permute Instruction
-  def int_ppc_qpx_qvfperm : PowerPC_QPX_FFFF_Intrinsic<"qvfperm">;
-
-  // Convert and Round Instructions
-  def int_ppc_qpx_qvfctid : PowerPC_QPX_FF_Intrinsic<"qvfctid">;
-  def int_ppc_qpx_qvfctidu : PowerPC_QPX_FF_Intrinsic<"qvfctidu">;
-  def int_ppc_qpx_qvfctidz : PowerPC_QPX_FF_Intrinsic<"qvfctidz">;
-  def int_ppc_qpx_qvfctiduz : PowerPC_QPX_FF_Intrinsic<"qvfctiduz">;
-  def int_ppc_qpx_qvfctiw : PowerPC_QPX_FF_Intrinsic<"qvfctiw">;
-  def int_ppc_qpx_qvfctiwu : PowerPC_QPX_FF_Intrinsic<"qvfctiwu">;
-  def int_ppc_qpx_qvfctiwz : PowerPC_QPX_FF_Intrinsic<"qvfctiwz">;
-  def int_ppc_qpx_qvfctiwuz : PowerPC_QPX_FF_Intrinsic<"qvfctiwuz">;
-  def int_ppc_qpx_qvfcfid : PowerPC_QPX_FF_Intrinsic<"qvfcfid">;
-  def int_ppc_qpx_qvfcfidu : PowerPC_QPX_FF_Intrinsic<"qvfcfidu">;
-  def int_ppc_qpx_qvfcfids : PowerPC_QPX_FF_Intrinsic<"qvfcfids">;
-  def int_ppc_qpx_qvfcfidus : PowerPC_QPX_FF_Intrinsic<"qvfcfidus">;
-  def int_ppc_qpx_qvfrsp : PowerPC_QPX_FF_Intrinsic<"qvfrsp">;
-  def int_ppc_qpx_qvfriz : PowerPC_QPX_FF_Intrinsic<"qvfriz">;
-  def int_ppc_qpx_qvfrin : PowerPC_QPX_FF_Intrinsic<"qvfrin">;
-  def int_ppc_qpx_qvfrip : PowerPC_QPX_FF_Intrinsic<"qvfrip">;
-  def int_ppc_qpx_qvfrim : PowerPC_QPX_FF_Intrinsic<"qvfrim">;
-
-  // Move Instructions
-  def int_ppc_qpx_qvfneg : PowerPC_QPX_FF_Intrinsic<"qvfneg">;
-  def int_ppc_qpx_qvfabs : PowerPC_QPX_FF_Intrinsic<"qvfabs">;
-  def int_ppc_qpx_qvfnabs : PowerPC_QPX_FF_Intrinsic<"qvfnabs">;
-  def int_ppc_qpx_qvfcpsgn : PowerPC_QPX_FFF_Intrinsic<"qvfcpsgn">;
-
-  // Compare Instructions
-  def int_ppc_qpx_qvftstnan : PowerPC_QPX_FFF_Intrinsic<"qvftstnan">;
-  def int_ppc_qpx_qvfcmplt : PowerPC_QPX_FFF_Intrinsic<"qvfcmplt">;
-  def int_ppc_qpx_qvfcmpgt : PowerPC_QPX_FFF_Intrinsic<"qvfcmpgt">;
-  def int_ppc_qpx_qvfcmpeq : PowerPC_QPX_FFF_Intrinsic<"qvfcmpeq">;
-
-  // Load instructions
-  def int_ppc_qpx_qvlfd : PowerPC_QPX_Load_Intrinsic<"qvlfd">;
-  def int_ppc_qpx_qvlfda : PowerPC_QPX_Load_Intrinsic<"qvlfda">;
-  def int_ppc_qpx_qvlfs : PowerPC_QPX_Load_Intrinsic<"qvlfs">;
-  def int_ppc_qpx_qvlfsa : PowerPC_QPX_Load_Intrinsic<"qvlfsa">;
-
-  def int_ppc_qpx_qvlfcda : PowerPC_QPX_Load_Intrinsic<"qvlfcda">;
-  def int_ppc_qpx_qvlfcd : PowerPC_QPX_Load_Intrinsic<"qvlfcd">;
-  def int_ppc_qpx_qvlfcsa : PowerPC_QPX_Load_Intrinsic<"qvlfcsa">;
-  def int_ppc_qpx_qvlfcs : PowerPC_QPX_Load_Intrinsic<"qvlfcs">;
-  def int_ppc_qpx_qvlfiwaa : PowerPC_QPX_Load_Intrinsic<"qvlfiwaa">;
-  def int_ppc_qpx_qvlfiwa : PowerPC_QPX_Load_Intrinsic<"qvlfiwa">;
-  def int_ppc_qpx_qvlfiwza : PowerPC_QPX_Load_Intrinsic<"qvlfiwza">;
-  def int_ppc_qpx_qvlfiwz : PowerPC_QPX_Load_Intrinsic<"qvlfiwz">;
-
-  def int_ppc_qpx_qvlpcld : PowerPC_QPX_LoadPerm_Intrinsic<"qvlpcld">;
-  def int_ppc_qpx_qvlpcls : PowerPC_QPX_LoadPerm_Intrinsic<"qvlpcls">;
-  def int_ppc_qpx_qvlpcrd : PowerPC_QPX_LoadPerm_Intrinsic<"qvlpcrd">;
-  def int_ppc_qpx_qvlpcrs : PowerPC_QPX_LoadPerm_Intrinsic<"qvlpcrs">;
-
-  // Store instructions
-  def int_ppc_qpx_qvstfd : PowerPC_QPX_Store_Intrinsic<"qvstfd">;
-  def int_ppc_qpx_qvstfda : PowerPC_QPX_Store_Intrinsic<"qvstfda">;
-  def int_ppc_qpx_qvstfs : PowerPC_QPX_Store_Intrinsic<"qvstfs">;
-  def int_ppc_qpx_qvstfsa : PowerPC_QPX_Store_Intrinsic<"qvstfsa">;
-
-  def int_ppc_qpx_qvstfcda : PowerPC_QPX_Store_Intrinsic<"qvstfcda">;
-  def int_ppc_qpx_qvstfcd : PowerPC_QPX_Store_Intrinsic<"qvstfcd">;
-  def int_ppc_qpx_qvstfcsa : PowerPC_QPX_Store_Intrinsic<"qvstfcsa">;
-  def int_ppc_qpx_qvstfcs : PowerPC_QPX_Store_Intrinsic<"qvstfcs">;
-  def int_ppc_qpx_qvstfiwa : PowerPC_QPX_Store_Intrinsic<"qvstfiwa">;
-  def int_ppc_qpx_qvstfiw : PowerPC_QPX_Store_Intrinsic<"qvstfiw">;
-
-  // Logical and permutation formation
-  def int_ppc_qpx_qvflogical : PowerPC_QPX_Intrinsic<"qvflogical",
-                          [llvm_v4f64_ty],
-                          [llvm_v4f64_ty, llvm_v4f64_ty, llvm_i32_ty],
-                          [IntrNoMem]>;
-  def int_ppc_qpx_qvgpci : PowerPC_QPX_Intrinsic<"qvgpci",
-                          [llvm_v4f64_ty], [llvm_i32_ty], [IntrNoMem]>;
-}
-
 //===----------------------------------------------------------------------===//
 // PowerPC HTM Intrinsic Definitions.
 
diff --git a/llvm/lib/Support/Triple.cpp b/llvm/lib/Support/Triple.cpp
index fec1985ccacae..72648273b4cd5 100644
--- a/llvm/lib/Support/Triple.cpp
+++ b/llvm/lib/Support/Triple.cpp
@@ -160,8 +160,6 @@ StringRef Triple::getVendorTypeName(VendorType Kind) {
 
   case AMD: return "amd";
   case Apple: return "apple";
-  case BGP: return "bgp";
-  case BGQ: return "bgq";
   case CSR: return "csr";
   case Freescale: return "fsl";
   case IBM: return "ibm";
@@ -187,7 +185,6 @@ StringRef Triple::getOSTypeName(OSType Kind) {
   case AMDHSA: return "amdhsa";
   case AMDPAL: return "amdpal";
   case Ananas: return "ananas";
-  case CNK: return "cnk";
   case CUDA: return "cuda";
   case CloudABI: return "cloudabi";
   case Contiki: return "contiki";
@@ -470,8 +467,6 @@ static Triple::VendorType parseVendor(StringRef VendorName) {
     .Case("apple", Triple::Apple)
     .Case("pc", Triple::PC)
     .Case("scei", Triple::SCEI)
-    .Case("bgp", Triple::BGP)
-    .Case("bgq", Triple::BGQ)
     .Case("fsl", Triple::Freescale)
     .Case("ibm", Triple::IBM)
     .Case("img", Triple::ImaginationTechnologies)
@@ -508,7 +503,6 @@ static Triple::OSType parseOS(StringRef OSName) {
     .StartsWith("minix", Triple::Minix)
     .StartsWith("rtems", Triple::RTEMS)
     .StartsWith("nacl", Triple::NaCl)
-    .StartsWith("cnk", Triple::CNK)
     .StartsWith("aix", Triple::AIX)
     .StartsWith("cuda", Triple::CUDA)
     .StartsWith("nvcl", Triple::NVCL)
diff --git a/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp b/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp
index 13fd7d05ab9f4..81008d3ea5662 100644
--- a/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp
+++ b/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp
@@ -492,21 +492,6 @@ struct PPCOperand : public MCParsedAsmOperand {
     Inst.addOperand(MCOperand::createReg(VSSRegs[getVSReg()]));
   }
 
-  void addRegQFRCOperands(MCInst &Inst, unsigned N) const {
-    assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::createReg(QFRegs[getReg()]));
-  }
-
-  void addRegQSRCOperands(MCInst &Inst, unsigned N) const {
-    assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::createReg(QFRegs[getReg()]));
-  }
-
-  void addRegQBRCOperands(MCInst &Inst, unsigned N) const {
-    assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::createReg(QFRegs[getReg()]));
-  }
-
   void addRegSPE4RCOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
     Inst.addOperand(MCOperand::createReg(RRegs[getReg()]));
@@ -1207,9 +1192,6 @@ bool PPCAsmParser::MatchRegisterName(unsigned &RegNo, int64_t &IntVal) {
     } else if (Name.startswith_lower("v") &&
                !Name.substr(1).getAsInteger(10, IntVal) && IntVal < 32) {
       RegNo = VRegs[IntVal];
-    } else if (Name.startswith_lower("q") &&
-               !Name.substr(1).getAsInteger(10, IntVal) && IntVal < 32) {
-      RegNo = QFRegs[IntVal];
     } else if (Name.startswith_lower("cr") &&
                !Name.substr(2).getAsInteger(10, IntVal) && IntVal < 8) {
       RegNo = CRRegs[IntVal];
diff --git a/llvm/lib/Target/PowerPC/CMakeLists.txt b/llvm/lib/Target/PowerPC/CMakeLists.txt
index 91021d4e584e1..5a06faa16be19 100644
--- a/llvm/lib/Target/PowerPC/CMakeLists.txt
+++ b/llvm/lib/Target/PowerPC/CMakeLists.txt
@@ -36,7 +36,6 @@ add_llvm_target(PowerPCCodeGen
   PPCMacroFusion.cpp
   PPCMIPeephole.cpp
   PPCRegisterInfo.cpp
-  PPCQPXLoadSplat.cpp
   PPCSubtarget.cpp
   PPCTargetMachine.cpp
   PPCTargetObjectFile.cpp
diff --git a/llvm/lib/Target/PowerPC/Disassembler/PPCDisassembler.cpp b/llvm/lib/Target/PowerPC/Disassembler/PPCDisassembler.cpp
index 74c6fd3733f03..362ddf7204557 100644
--- a/llvm/lib/Target/PowerPC/Disassembler/PPCDisassembler.cpp
+++ b/llvm/lib/Target/PowerPC/Disassembler/PPCDisassembler.cpp
@@ -167,12 +167,6 @@ static DecodeStatus DecodeG8RC_NOX0RegisterClass(MCInst &Inst, uint64_t RegNo,
 #define DecodePointerLikeRegClass0 DecodeGPRCRegisterClass
 #define DecodePointerLikeRegClass1 DecodeGPRC_NOR0RegisterClass
 
-static DecodeStatus DecodeQFRCRegisterClass(MCInst &Inst, uint64_t RegNo,
-                                            uint64_t Address,
-                                            const void *Decoder) {
-  return decodeRegisterClass(Inst, RegNo, QFRegs);
-}
-
 static DecodeStatus DecodeSPERCRegisterClass(MCInst &Inst, uint64_t RegNo,
                                             uint64_t Address,
                                             const void *Decoder) {
@@ -401,14 +395,9 @@ DecodeStatus PPCDisassembler::getInstruction(MCInst &MI, uint64_t &Size,
   // Read the instruction in the proper endianness.
   uint64_t Inst = ReadFunc(Bytes.data());
 
-  if (STI.getFeatureBits()[PPC::FeatureQPX]) {
-    DecodeStatus result =
-      decodeInstruction(DecoderTableQPX32, MI, Inst, Address, this, STI);
-    if (result != MCDisassembler::Fail)
-      return result;
-  } else if (STI.getFeatureBits()[PPC::FeatureSPE]) {
+  if (STI.getFeatureBits()[PPC::FeatureSPE]) {
     DecodeStatus result =
-      decodeInstruction(DecoderTableSPE32, MI, Inst, Address, this, STI);
+        decodeInstruction(DecoderTableSPE32, MI, Inst, Address, this, STI);
     if (result != MCDisassembler::Fail)
       return result;
   }
diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCInstPrinter.cpp b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCInstPrinter.cpp
index 222bf2fa82836..ce1a43a0c25b2 100644
--- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCInstPrinter.cpp
+++ b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCInstPrinter.cpp
@@ -49,18 +49,6 @@ FullRegNamesWithPercent("ppc-reg-with-percent-prefix", cl::Hidden,
 
 void PPCInstPrinter::printRegName(raw_ostream &OS, unsigned RegNo) const {
   const char *RegName = getRegisterName(RegNo);
-  if (RegName[0] == 'q' /* QPX */) {
-    // The system toolchain on the BG/Q does not understand QPX register names
-    // in .cfi_* directives, so print the name of the floating-point
-    // subregister instead.
-    std::string RN(RegName);
-
-    RN[0] = 'f';
-    OS << RN;
-
-    return;
-  }
-
   OS << RegName;
 }
 
diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.h b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.h
index 719e005d98135..325ede0fc17ac 100644
--- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.h
+++ b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.h
@@ -159,7 +159,6 @@ using llvm::MCPhysReg;
   static const MCPhysReg SPERegs[32] = PPC_REGS0_31(PPC::S); \
   static const MCPhysReg VFRegs[32] = PPC_REGS0_31(PPC::VF); \
   static const MCPhysReg VRegs[32] = PPC_REGS0_31(PPC::V); \
-  static const MCPhysReg QFRegs[32] = PPC_REGS0_31(PPC::QF); \
   static const MCPhysReg RRegsNoR0[32] = \
     PPC_REGS_NO0_31(PPC::ZERO, PPC::R); \
   static const MCPhysReg XRegsNoX0[32] = \
diff --git a/llvm/lib/Target/PowerPC/PPC.h b/llvm/lib/Target/PowerPC/PPC.h
index 3106290442afa..24a9d419d3ea5 100644
--- a/llvm/lib/Target/PowerPC/PPC.h
+++ b/llvm/lib/Target/PowerPC/PPC.h
@@ -44,7 +44,6 @@ namespace llvm {
   FunctionPass *createPPCMIPeepholePass();
   FunctionPass *createPPCBranchSelectionPass();
   FunctionPass *createPPCBranchCoalescingPass();
-  FunctionPass *createPPCQPXLoadSplatPass();
   FunctionPass *createPPCISelDag(PPCTargetMachine &TM, CodeGenOpt::Level OL);
   FunctionPass *createPPCTLSDynamicCallPass();
   FunctionPass *createPPCBoolRetToIntPass();
@@ -68,7 +67,6 @@ namespace llvm {
   void initializePPCReduceCRLogicalsPass(PassRegistry&);
   void initializePPCBSelPass(PassRegistry&);
   void initializePPCBranchCoalescingPass(PassRegistry&);
-  void initializePPCQPXLoadSplatPass(PassRegistry&);
   void initializePPCBoolRetToIntPass(PassRegistry&);
   void initializePPCExpandISELPass(PassRegistry &);
   void initializePPCPreEmitPeepholePass(PassRegistry &);
diff --git a/llvm/lib/Target/PowerPC/PPC.td b/llvm/lib/Target/PowerPC/PPC.td
index 9ad78bf67fe6c..adb9366217d51 100644
--- a/llvm/lib/Target/PowerPC/PPC.td
+++ b/llvm/lib/Target/PowerPC/PPC.td
@@ -132,9 +132,6 @@ def FeaturePPC4xx    : SubtargetFeature<"ppc4xx", "IsPPC4xx", "true",
                                         "Enable PPC 4xx instructions">;
 def FeaturePPC6xx    : SubtargetFeature<"ppc6xx", "IsPPC6xx", "true",
                                         "Enable PPC 6xx instructions">;
-def FeatureQPX       : SubtargetFeature<"qpx","HasQPX", "true",
-                                        "Enable QPX instructions",
-                                        [FeatureFPU]>;
 def FeatureVSX       : SubtargetFeature<"vsx","HasVSX", "true",
                                         "Enable VSX instructions",
                                         [FeatureAltivec]>;
@@ -193,7 +190,7 @@ def FeatureFloat128 :
 def FeaturePOPCNTD   : SubtargetFeature<"popcntd","HasPOPCNTD",
                                         "POPCNTD_Fast",
                                         "Enable the popcnt[dw] instructions">;
-// Note that for the a2/a2q processor models we should not use popcnt[dw] by
+// Note that for the a2 processor models we should not use popcnt[dw] by
 // default. These processors do support the instructions, but they're
 // microcoded, and the software emulation is about twice as fast.
 def FeatureSlowPOPCNTD : SubtargetFeature<"slow-popcntd","HasPOPCNTD",
@@ -514,15 +511,6 @@ def : ProcessorModel<"a2", PPCA2Model,
                    FeatureFPRND, FeatureFPCVT, FeatureISEL,
                    FeatureSlowPOPCNTD, FeatureCMPB, FeatureLDBRX,
                    Feature64Bit /*, Feature64BitRegs */, FeatureMFTB]>;
-def : ProcessorModel<"a2q", PPCA2Model,
-                  [DirectiveA2, FeatureICBT, FeatureBookE, FeatureMFOCRF,
-                   FeatureFCPSGN, FeatureFSqrt, FeatureFRE, FeatureFRES,
-                   FeatureFRSQRTE, FeatureFRSQRTES, FeatureRecipPrec,
-                   FeatureSTFIWX, FeatureLFIWAX,
-                   FeatureFPRND, FeatureFPCVT, FeatureISEL,
-                   FeatureSlowPOPCNTD, FeatureCMPB, FeatureLDBRX,
-                   Feature64Bit /*, Feature64BitRegs */, FeatureQPX,
-                   FeatureMFTB]>;
 def : ProcessorModel<"pwr3", G5Model,
                   [DirectivePwr3, FeatureAltivec,
                    FeatureFRES, FeatureFRSQRTE, FeatureMFOCRF,
diff --git a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
index 540e620a845bc..5affddd8d147a 100644
--- a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
+++ b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
@@ -549,9 +549,6 @@ void PPCAsmPrinter::emitInstruction(const MachineInstr *MI) {
         if (Subtarget->hasSPE()) {
           if (PPC::F4RCRegClass.contains(Reg) ||
               PPC::F8RCRegClass.contains(Reg) ||
-              PPC::QBRCRegClass.contains(Reg) ||
-              PPC::QFRCRegClass.contains(Reg) ||
-              PPC::QSRCRegClass.contains(Reg) ||
               PPC::VFRCRegClass.contains(Reg) ||
               PPC::VRRCRegClass.contains(Reg) ||
               PPC::VSFRCRegClass.contains(Reg) ||
diff --git a/llvm/lib/Target/PowerPC/PPCCallingConv.td b/llvm/lib/Target/PowerPC/PPCCallingConv.td
index 1eaa7f7a44b39..9a15490f1fb0d 100644
--- a/llvm/lib/Target/PowerPC/PPCCallingConv.td
+++ b/llvm/lib/Target/PowerPC/PPCCallingConv.td
@@ -61,9 +61,6 @@ def RetCC_PPC_Cold : CallingConv<[
   CCIfType<[f64], CCAssignToReg<[F1]>>,
   CCIfType<[f128], CCIfSubtarget<"hasP9Vector()", CCAssignToReg<[V2]>>>,
 
-  CCIfType<[v4f64, v4f32, v4i1],
-           CCIfSubtarget<"hasQPX()", CCAssignToReg<[QF1]>>>,
-
   CCIfType<[v16i8, v8i16, v4i32, v2i64, v1i128, v4f32, v2f64],
            CCIfSubtarget<"hasAltivec()",
            CCAssignToReg<[V2]>>>
@@ -98,10 +95,6 @@ def RetCC_PPC : CallingConv<[
            CCIfSubtarget<"hasP9Vector()",
            CCAssignToReg<[V2, V3, V4, V5, V6, V7, V8, V9]>>>,
 
-  // QPX vectors are returned in QF1 and QF2. 
-  CCIfType<[v4f64, v4f32, v4i1],
-           CCIfSubtarget<"hasQPX()", CCAssignToReg<[QF1, QF2]>>>,
- 
   // Vector types returned as "direct" go into V2 .. V9; note that only the
   // ELFv2 ABI fully utilizes all these registers.
   CCIfType<[v16i8, v8i16, v4i32, v2i64, v1i128, v4f32, v2f64],
@@ -158,8 +151,6 @@ def RetCC_PPC64_ELF_FIS : CallingConv<[
   CCIfType<[f128],
            CCIfSubtarget<"hasP9Vector()",
            CCAssignToReg<[V2, V3, V4, V5, V6, V7, V8, V9]>>>,
-  CCIfType<[v4f64, v4f32, v4i1],
-           CCIfSubtarget<"hasQPX()", CCAssignToReg<[QF1, QF2]>>>,
   CCIfType<[v16i8, v8i16, v4i32, v2i64, v1i128, v4f32, v2f64],
            CCIfSubtarget<"hasAltivec()",
            CCAssignToReg<[V2, V3, V4, V5, V6, V7, V8, V9]>>>
@@ -223,9 +214,6 @@ def CC_PPC32_SVR4_Common : CallingConv<[
   CCIfType<[f32], CCIfSubtarget<"hasSPE()", CCAssignToStack<4, 4>>>,
   CCIfType<[f64], CCIfSubtarget<"hasSPE()", CCAssignToStack<8, 8>>>,
 
-  // QPX vectors that are stored in double precision need 32-byte alignment.
-  CCIfType<[v4f64, v4i1], CCAssignToStack<32, 32>>,
-
   // Vectors and float128 get 16-byte stack slots that are 16-byte aligned.
   CCIfType<[v16i8, v8i16, v4i32, v4f32, v2f64, v2i64], CCAssignToStack<16, 16>>,
   CCIfType<[f128], CCIfSubtarget<"hasP9Vector()", CCAssignToStack<16, 16>>>
@@ -243,10 +231,6 @@ def CC_PPC32_SVR4_VarArg : CallingConv<[
 // put vector arguments in vector registers before putting them on the stack.
 let Entry = 1 in
 def CC_PPC32_SVR4 : CallingConv<[
-  // QPX vectors mirror the scalar FP convention.
-  CCIfType<[v4f64, v4f32, v4i1], CCIfSubtarget<"hasQPX()",
-    CCAssignToReg<[QF1, QF2, QF3, QF4, QF5, QF6, QF7, QF8]>>>,
-
   // The first 12 Vector arguments are passed in AltiVec registers.
   CCIfType<[v16i8, v8i16, v4i32, v2i64, v1i128, v4f32, v2f64],
            CCIfSubtarget<"hasAltivec()", CCAssignToReg<[V2, V3, V4, V5, V6, V7,
diff --git a/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp b/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp
index 8ffd89ef5ccd2..3e218e14d8d44 100644
--- a/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp
@@ -4142,7 +4142,7 @@ bool PPCDAGToDAGISel::trySETCC(SDNode *N) {
   // Altivec Vector compare instructions do not set any CR register by default and
   // vector compare operations return the same type as the operands.
   if (LHS.getValueType().isVector()) {
-    if (Subtarget->hasQPX() || Subtarget->hasSPE())
+    if (Subtarget->hasSPE())
       return false;
 
     EVT VecVT = LHS.getValueType();
@@ -4813,8 +4813,6 @@ void PPCDAGToDAGISel::Select(SDNode *N) {
         assert((!isSExt || LoadedVT == MVT::i16) && "Invalid sext update load");
         switch (LoadedVT.getSimpleVT().SimpleTy) {
           default: llvm_unreachable("Invalid PPC load type!");
-          case MVT::v4f64: Opcode = PPC::QVLFDUX; break; // QPX
-          case MVT::v4f32: Opcode = PPC::QVLFSUX; break; // QPX
           case MVT::f64: Opcode = PPC::LFDUX; break;
           case MVT::f32: Opcode = PPC::LFSUX; break;
           case MVT::i32: Opcode = PPC::LWZUX; break;
@@ -5095,12 +5093,6 @@ void PPCDAGToDAGISel::Select(SDNode *N) {
       SelectCCOp = PPC::SELECT_CC_F16;
     else if (Subtarget->hasSPE())
       SelectCCOp = PPC::SELECT_CC_SPE;
-    else if (Subtarget->hasQPX() && N->getValueType(0) == MVT::v4f64)
-      SelectCCOp = PPC::SELECT_CC_QFRC;
-    else if (Subtarget->hasQPX() && N->getValueType(0) == MVT::v4f32)
-      SelectCCOp = PPC::SELECT_CC_QSRC;
-    else if (Subtarget->hasQPX() && N->getValueType(0) == MVT::v4i1)
-      SelectCCOp = PPC::SELECT_CC_QBRC;
     else if (N->getValueType(0) == MVT::v2f64 ||
              N->getValueType(0) == MVT::v2i64)
       SelectCCOp = PPC::SELECT_CC_VSRC;
@@ -5856,9 +5848,6 @@ void PPCDAGToDAGISel::PeepholeCROps() {
       case PPC::SELECT_I8:
       case PPC::SELECT_F4:
       case PPC::SELECT_F8:
-      case PPC::SELECT_QFRC:
-      case PPC::SELECT_QSRC:
-      case PPC::SELECT_QBRC:
       case PPC::SELECT_SPE:
       case PPC::SELECT_SPE4:
       case PPC::SELECT_VRRC:
@@ -6177,9 +6166,6 @@ void PPCDAGToDAGISel::PeepholeCROps() {
       case PPC::SELECT_I8:
       case PPC::SELECT_F4:
       case PPC::SELECT_F8:
-      case PPC::SELECT_QFRC:
-      case PPC::SELECT_QSRC:
-      case PPC::SELECT_QBRC:
       case PPC::SELECT_SPE:
       case PPC::SELECT_SPE4:
       case PPC::SELECT_VRRC:
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index ae840a9fa37de..db3833d595797 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -1095,161 +1095,6 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM,
     }
   }
 
-  if (Subtarget.hasQPX()) {
-    setOperationAction(ISD::FADD, MVT::v4f64, Legal);
-    setOperationAction(ISD::FSUB, MVT::v4f64, Legal);
-    setOperationAction(ISD::FMUL, MVT::v4f64, Legal);
-    setOperationAction(ISD::FREM, MVT::v4f64, Expand);
-
-    setOperationAction(ISD::FCOPYSIGN, MVT::v4f64, Legal);
-    setOperationAction(ISD::FGETSIGN, MVT::v4f64, Expand);
-
-    setOperationAction(ISD::LOAD  , MVT::v4f64, Custom);
-    setOperationAction(ISD::STORE , MVT::v4f64, Custom);
-
-    setTruncStoreAction(MVT::v4f64, MVT::v4f32, Custom);
-    setLoadExtAction(ISD::EXTLOAD, MVT::v4f64, MVT::v4f32, Custom);
-
-    if (!Subtarget.useCRBits())
-      setOperationAction(ISD::SELECT, MVT::v4f64, Expand);
-    setOperationAction(ISD::VSELECT, MVT::v4f64, Legal);
-
-    setOperationAction(ISD::EXTRACT_VECTOR_ELT , MVT::v4f64, Legal);
-    setOperationAction(ISD::INSERT_VECTOR_ELT , MVT::v4f64, Expand);
-    setOperationAction(ISD::CONCAT_VECTORS , MVT::v4f64, Expand);
-    setOperationAction(ISD::EXTRACT_SUBVECTOR , MVT::v4f64, Expand);
-    setOperationAction(ISD::VECTOR_SHUFFLE , MVT::v4f64, Custom);
-    setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4f64, Legal);
-    setOperationAction(ISD::BUILD_VECTOR, MVT::v4f64, Custom);
-
-    setOperationAction(ISD::FP_TO_SINT , MVT::v4f64, Legal);
-    setOperationAction(ISD::FP_TO_UINT , MVT::v4f64, Expand);
-
-    setOperationAction(ISD::FP_ROUND , MVT::v4f32, Legal);
-    setOperationAction(ISD::FP_EXTEND, MVT::v4f64, Legal);
-
-    setOperationAction(ISD::FNEG , MVT::v4f64, Legal);
-    setOperationAction(ISD::FABS , MVT::v4f64, Legal);
-    setOperationAction(ISD::FSIN , MVT::v4f64, Expand);
-    setOperationAction(ISD::FCOS , MVT::v4f64, Expand);
-    setOperationAction(ISD::FPOW , MVT::v4f64, Expand);
-    setOperationAction(ISD::FLOG , MVT::v4f64, Expand);
-    setOperationAction(ISD::FLOG2 , MVT::v4f64, Expand);
-    setOperationAction(ISD::FLOG10 , MVT::v4f64, Expand);
-    setOperationAction(ISD::FEXP , MVT::v4f64, Expand);
-    setOperationAction(ISD::FEXP2 , MVT::v4f64, Expand);
-
-    setOperationAction(ISD::FMINNUM, MVT::v4f64, Legal);
-    setOperationAction(ISD::FMAXNUM, MVT::v4f64, Legal);
-
-    setIndexedLoadAction(ISD::PRE_INC, MVT::v4f64, Legal);
-    setIndexedStoreAction(ISD::PRE_INC, MVT::v4f64, Legal);
-
-    addRegisterClass(MVT::v4f64, &PPC::QFRCRegClass);
-
-    setOperationAction(ISD::FADD, MVT::v4f32, Legal);
-    setOperationAction(ISD::FSUB, MVT::v4f32, Legal);
-    setOperationAction(ISD::FMUL, MVT::v4f32, Legal);
-    setOperationAction(ISD::FREM, MVT::v4f32, Expand);
-
-    setOperationAction(ISD::FCOPYSIGN, MVT::v4f32, Legal);
-    setOperationAction(ISD::FGETSIGN, MVT::v4f32, Expand);
-
-    setOperationAction(ISD::LOAD  , MVT::v4f32, Custom);
-    setOperationAction(ISD::STORE , MVT::v4f32, Custom);
-
-    if (!Subtarget.useCRBits())
-      setOperationAction(ISD::SELECT, MVT::v4f32, Expand);
-    setOperationAction(ISD::VSELECT, MVT::v4f32, Legal);
-
-    setOperationAction(ISD::EXTRACT_VECTOR_ELT , MVT::v4f32, Legal);
-    setOperationAction(ISD::INSERT_VECTOR_ELT , MVT::v4f32, Expand);
-    setOperationAction(ISD::CONCAT_VECTORS , MVT::v4f32, Expand);
-    setOperationAction(ISD::EXTRACT_SUBVECTOR , MVT::v4f32, Expand);
-    setOperationAction(ISD::VECTOR_SHUFFLE , MVT::v4f32, Custom);
-    setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4f32, Legal);
-    setOperationAction(ISD::BUILD_VECTOR, MVT::v4f32, Custom);
-
-    setOperationAction(ISD::FP_TO_SINT , MVT::v4f32, Legal);
-    setOperationAction(ISD::FP_TO_UINT , MVT::v4f32, Expand);
-
-    setOperationAction(ISD::FNEG , MVT::v4f32, Legal);
-    setOperationAction(ISD::FABS , MVT::v4f32, Legal);
-    setOperationAction(ISD::FSIN , MVT::v4f32, Expand);
-    setOperationAction(ISD::FCOS , MVT::v4f32, Expand);
-    setOperationAction(ISD::FPOW , MVT::v4f32, Expand);
-    setOperationAction(ISD::FLOG , MVT::v4f32, Expand);
-    setOperationAction(ISD::FLOG2 , MVT::v4f32, Expand);
-    setOperationAction(ISD::FLOG10 , MVT::v4f32, Expand);
-    setOperationAction(ISD::FEXP , MVT::v4f32, Expand);
-    setOperationAction(ISD::FEXP2 , MVT::v4f32, Expand);
-
-    setOperationAction(ISD::FMINNUM, MVT::v4f32, Legal);
-    setOperationAction(ISD::FMAXNUM, MVT::v4f32, Legal);
-
-    setIndexedLoadAction(ISD::PRE_INC, MVT::v4f32, Legal);
-    setIndexedStoreAction(ISD::PRE_INC, MVT::v4f32, Legal);
-
-    addRegisterClass(MVT::v4f32, &PPC::QSRCRegClass);
-
-    setOperationAction(ISD::AND , MVT::v4i1, Legal);
-    setOperationAction(ISD::OR , MVT::v4i1, Legal);
-    setOperationAction(ISD::XOR , MVT::v4i1, Legal);
-
-    if (!Subtarget.useCRBits())
-      setOperationAction(ISD::SELECT, MVT::v4i1, Expand);
-    setOperationAction(ISD::VSELECT, MVT::v4i1, Legal);
-
-    setOperationAction(ISD::LOAD  , MVT::v4i1, Custom);
-    setOperationAction(ISD::STORE , MVT::v4i1, Custom);
-
-    setOperationAction(ISD::EXTRACT_VECTOR_ELT , MVT::v4i1, Custom);
-    setOperationAction(ISD::INSERT_VECTOR_ELT , MVT::v4i1, Expand);
-    setOperationAction(ISD::CONCAT_VECTORS , MVT::v4i1, Expand);
-    setOperationAction(ISD::EXTRACT_SUBVECTOR , MVT::v4i1, Expand);
-    setOperationAction(ISD::VECTOR_SHUFFLE , MVT::v4i1, Custom);
-    setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4i1, Expand);
-    setOperationAction(ISD::BUILD_VECTOR, MVT::v4i1, Custom);
-
-    setOperationAction(ISD::SINT_TO_FP, MVT::v4i1, Custom);
-    setOperationAction(ISD::UINT_TO_FP, MVT::v4i1, Custom);
-
-    addRegisterClass(MVT::v4i1, &PPC::QBRCRegClass);
-
-    setOperationAction(ISD::FFLOOR, MVT::v4f64, Legal);
-    setOperationAction(ISD::FCEIL,  MVT::v4f64, Legal);
-    setOperationAction(ISD::FTRUNC, MVT::v4f64, Legal);
-    setOperationAction(ISD::FROUND, MVT::v4f64, Legal);
-
-    setOperationAction(ISD::FFLOOR, MVT::v4f32, Legal);
-    setOperationAction(ISD::FCEIL,  MVT::v4f32, Legal);
-    setOperationAction(ISD::FTRUNC, MVT::v4f32, Legal);
-    setOperationAction(ISD::FROUND, MVT::v4f32, Legal);
-
-    setOperationAction(ISD::FNEARBYINT, MVT::v4f64, Expand);
-    setOperationAction(ISD::FNEARBYINT, MVT::v4f32, Expand);
-
-    // These need to set FE_INEXACT, and so cannot be vectorized here.
-    setOperationAction(ISD::FRINT, MVT::v4f64, Expand);
-    setOperationAction(ISD::FRINT, MVT::v4f32, Expand);
-
-    if (TM.Options.UnsafeFPMath) {
-      setOperationAction(ISD::FDIV, MVT::v4f64, Legal);
-      setOperationAction(ISD::FSQRT, MVT::v4f64, Legal);
-
-      setOperationAction(ISD::FDIV, MVT::v4f32, Legal);
-      setOperationAction(ISD::FSQRT, MVT::v4f32, Legal);
-    } else {
-      setOperationAction(ISD::FDIV, MVT::v4f64, Expand);
-      setOperationAction(ISD::FSQRT, MVT::v4f64, Expand);
-
-      setOperationAction(ISD::FDIV, MVT::v4f32, Expand);
-      setOperationAction(ISD::FSQRT, MVT::v4f32, Expand);
-    }
-
-    // TODO: Handle constrained floating-point operations of v4f64
-  }
-
   if (Subtarget.has64BitSupport())
     setOperationAction(ISD::PREFETCH, MVT::Other, Legal);
 
@@ -1438,8 +1283,8 @@ unsigned PPCTargetLowering::getByValTypeAlignment(Type *Ty,
   // 16byte and wider vectors are passed on 16byte boundary.
   // The rest is 8 on PPC64 and 4 on PPC32 boundary.
   Align Alignment = Subtarget.isPPC64() ? Align(8) : Align(4);
-  if (Subtarget.hasAltivec() || Subtarget.hasQPX())
-    getMaxByValAlign(Ty, Alignment, Subtarget.hasQPX() ? Align(32) : Align(16));
+  if (Subtarget.hasAltivec())
+    getMaxByValAlign(Ty, Alignment, Align(16));
   return Alignment.value();
 }
 
@@ -1577,12 +1422,6 @@ const char *PPCTargetLowering::getTargetNodeName(unsigned Opcode) const {
   case PPCISD::XXSWAPD:         return "PPCISD::XXSWAPD";
   case PPCISD::SWAP_NO_CHAIN:   return "PPCISD::SWAP_NO_CHAIN";
   case PPCISD::VABSD:           return "PPCISD::VABSD";
-  case PPCISD::QVFPERM:         return "PPCISD::QVFPERM";
-  case PPCISD::QVGPCI:          return "PPCISD::QVGPCI";
-  case PPCISD::QVALIGNI:        return "PPCISD::QVALIGNI";
-  case PPCISD::QVESPLATI:       return "PPCISD::QVESPLATI";
-  case PPCISD::QBFLT:           return "PPCISD::QBFLT";
-  case PPCISD::QVLFSb:          return "PPCISD::QVLFSb";
   case PPCISD::BUILD_FP128:     return "PPCISD::BUILD_FP128";
   case PPCISD::BUILD_SPE64:     return "PPCISD::BUILD_SPE64";
   case PPCISD::EXTRACT_SPE:     return "PPCISD::EXTRACT_SPE";
@@ -1601,9 +1440,6 @@ EVT PPCTargetLowering::getSetCCResultType(const DataLayout &DL, LLVMContext &C,
   if (!VT.isVector())
     return Subtarget.useCRBits() ? MVT::i1 : MVT::i32;
 
-  if (Subtarget.hasQPX())
-    return EVT::getVectorVT(C, MVT::i1, VT.getVectorNumElements());
-
   return VT.changeVectorElementTypeToInteger();
 }
 
@@ -2777,16 +2613,9 @@ bool PPCTargetLowering::getPreIndexedAddressParts(SDNode *N, SDValue &Base,
     return false;
   }
 
-  // PowerPC doesn't have preinc load/store instructions for vectors (except
-  // for QPX, which does have preinc r+r forms).
-  if (VT.isVector()) {
-    if (!Subtarget.hasQPX() || (VT != MVT::v4f64 && VT != MVT::v4f32)) {
-      return false;
-    } else if (SelectAddressRegRegOnly(Ptr, Offset, Base, DAG)) {
-      AM = ISD::PRE_INC;
-      return true;
-    }
-  }
+  // PowerPC doesn't have preinc load/store instructions for vectors
+  if (VT.isVector())
+    return false;
 
   if (SelectAddressRegReg(Ptr, Base, Offset, DAG)) {
     // Common code will reject creating a pre-inc form if the base pointer
@@ -3508,11 +3337,6 @@ static const MCPhysReg FPR[] = {PPC::F1,  PPC::F2,  PPC::F3, PPC::F4, PPC::F5,
                                 PPC::F6,  PPC::F7,  PPC::F8, PPC::F9, PPC::F10,
                                 PPC::F11, PPC::F12, PPC::F13};
 
-/// QFPR - The set of QPX registers that should be allocated for arguments.
-static const MCPhysReg QFPR[] = {
-    PPC::QF1, PPC::QF2, PPC::QF3,  PPC::QF4,  PPC::QF5,  PPC::QF6, PPC::QF7,
-    PPC::QF8, PPC::QF9, PPC::QF10, PPC::QF11, PPC::QF12, PPC::QF13};
-
 /// CalculateStackSlotSize - Calculates the size reserved for this argument on
 /// the stack.
 static unsigned CalculateStackSlotSize(EVT ArgVT, ISD::ArgFlagsTy Flags,
@@ -3542,10 +3366,6 @@ static Align CalculateStackSlotAlignment(EVT ArgVT, EVT OrigVT,
       ArgVT == MVT::v2f64 || ArgVT == MVT::v2i64 ||
       ArgVT == MVT::v1i128 || ArgVT == MVT::f128)
     Alignment = Align(16);
-  // QPX vector types stored in double-precision are padded to a 32 byte
-  // boundary.
-  else if (ArgVT == MVT::v4f64 || ArgVT == MVT::v4i1)
-    Alignment = Align(32);
 
   // ByVal parameters are aligned as requested.
   if (Flags.isByVal()) {
@@ -3577,14 +3397,11 @@ static Align CalculateStackSlotAlignment(EVT ArgVT, EVT OrigVT,
 /// stack slot (instead of being passed in registers).  ArgOffset,
 /// AvailableFPRs, and AvailableVRs must hold the current argument
 /// position, and will be updated to account for this argument.
-static bool CalculateStackSlotUsed(EVT ArgVT, EVT OrigVT,
-                                   ISD::ArgFlagsTy Flags,
-                                   unsigned PtrByteSize,
-                                   unsigned LinkageSize,
-                                   unsigned ParamAreaSize,
-                                   unsigned &ArgOffset,
+static bool CalculateStackSlotUsed(EVT ArgVT, EVT OrigVT, ISD::ArgFlagsTy Flags,
+                                   unsigned PtrByteSize, unsigned LinkageSize,
+                                   unsigned ParamAreaSize, unsigned &ArgOffset,
                                    unsigned &AvailableFPRs,
-                                   unsigned &AvailableVRs, bool HasQPX) {
+                                   unsigned &AvailableVRs) {
   bool UseMemory = false;
 
   // Respect alignment of argument on the stack.
@@ -3608,11 +3425,7 @@ static bool CalculateStackSlotUsed(EVT ArgVT, EVT OrigVT,
   // However, if the argument is actually passed in an FPR or a VR,
   // we don't use memory after all.
   if (!Flags.isByVal()) {
-    if (ArgVT == MVT::f32 || ArgVT == MVT::f64 ||
-        // QPX registers overlap with the scalar FP registers.
-        (HasQPX && (ArgVT == MVT::v4f32 ||
-                    ArgVT == MVT::v4f64 ||
-                    ArgVT == MVT::v4i1)))
+    if (ArgVT == MVT::f32 || ArgVT == MVT::f64)
       if (AvailableFPRs > 0) {
         --AvailableFPRs;
         return false;
@@ -3751,18 +3564,12 @@ SDValue PPCTargetLowering::LowerFormalArguments_32SVR4(
           RC = &PPC::VRRCRegClass;
           break;
         case MVT::v4f32:
-          RC = Subtarget.hasQPX() ? &PPC::QSRCRegClass : &PPC::VRRCRegClass;
+          RC = &PPC::VRRCRegClass;
           break;
         case MVT::v2f64:
         case MVT::v2i64:
           RC = &PPC::VRRCRegClass;
           break;
-        case MVT::v4f64:
-          RC = &PPC::QFRCRegClass;
-          break;
-        case MVT::v4i1:
-          RC = &PPC::QBRCRegClass;
-          break;
       }
 
       SDValue ArgValue;
@@ -3961,7 +3768,6 @@ SDValue PPCTargetLowering::LowerFormalArguments_64SVR4(
   const unsigned Num_GPR_Regs = array_lengthof(GPR);
   const unsigned Num_FPR_Regs = useSoftFloat() ? 0 : 13;
   const unsigned Num_VR_Regs  = array_lengthof(VR);
-  const unsigned Num_QFPR_Regs = Num_FPR_Regs;
 
   // Do a first pass over the arguments to determine whether the ABI
   // guarantees that our caller has allocated the parameter save area
@@ -3980,8 +3786,7 @@ SDValue PPCTargetLowering::LowerFormalArguments_64SVR4(
 
     if (CalculateStackSlotUsed(Ins[i].VT, Ins[i].ArgVT, Ins[i].Flags,
                                PtrByteSize, LinkageSize, ParamAreaSize,
-                               NumBytes, AvailableFPRs, AvailableVRs,
-                               Subtarget.hasQPX()))
+                               NumBytes, AvailableFPRs, AvailableVRs))
       HasParameterArea = true;
   }
 
@@ -3991,7 +3796,6 @@ SDValue PPCTargetLowering::LowerFormalArguments_64SVR4(
 
   unsigned ArgOffset = LinkageSize;
   unsigned GPR_idx = 0, FPR_idx = 0, VR_idx = 0;
-  unsigned &QFPR_idx = FPR_idx;
   SmallVector<SDValue, 8> MemOps;
   Function::const_arg_iterator FuncArg = MF.getFunction().arg_begin();
   unsigned CurArgIdx = 0;
@@ -4234,51 +4038,20 @@ SDValue PPCTargetLowering::LowerFormalArguments_64SVR4(
     case MVT::v2i64:
     case MVT::v1i128:
     case MVT::f128:
-      if (!Subtarget.hasQPX()) {
-        // These can be scalar arguments or elements of a vector array type
-        // passed directly.  The latter are used to implement ELFv2 homogenous
-        // vector aggregates.
-        if (VR_idx != Num_VR_Regs) {
-          unsigned VReg = MF.addLiveIn(VR[VR_idx], &PPC::VRRCRegClass);
-          ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, ObjectVT);
-          ++VR_idx;
-        } else {
-          if (CallConv == CallingConv::Fast)
-            ComputeArgOffset();
-          needsLoad = true;
-        }
-        if (CallConv != CallingConv::Fast || needsLoad)
-          ArgOffset += 16;
-        break;
-      } // not QPX
-
-      assert(ObjectVT.getSimpleVT().SimpleTy == MVT::v4f32 &&
-             "Invalid QPX parameter type");
-      LLVM_FALLTHROUGH;
-
-    case MVT::v4f64:
-    case MVT::v4i1:
-      // QPX vectors are treated like their scalar floating-point subregisters
-      // (except that they're larger).
-      unsigned Sz = ObjectVT.getSimpleVT().SimpleTy == MVT::v4f32 ? 16 : 32;
-      if (QFPR_idx != Num_QFPR_Regs) {
-        const TargetRegisterClass *RC;
-        switch (ObjectVT.getSimpleVT().SimpleTy) {
-        case MVT::v4f64: RC = &PPC::QFRCRegClass; break;
-        case MVT::v4f32: RC = &PPC::QSRCRegClass; break;
-        default:         RC = &PPC::QBRCRegClass; break;
-        }
-
-        unsigned VReg = MF.addLiveIn(QFPR[QFPR_idx], RC);
+      // These can be scalar arguments or elements of a vector array type
+      // passed directly.  The latter are used to implement ELFv2 homogenous
+      // vector aggregates.
+      if (VR_idx != Num_VR_Regs) {
+        unsigned VReg = MF.addLiveIn(VR[VR_idx], &PPC::VRRCRegClass);
         ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, ObjectVT);
-        ++QFPR_idx;
+        ++VR_idx;
       } else {
         if (CallConv == CallingConv::Fast)
           ComputeArgOffset();
         needsLoad = true;
       }
       if (CallConv != CallingConv::Fast || needsLoad)
-        ArgOffset += Sz;
+        ArgOffset += 16;
       break;
     }
 
@@ -4831,10 +4604,9 @@ needStackSlotPassParameters(const PPCSubtarget &Subtarget,
   for (const ISD::OutputArg& Param : Outs) {
     if (Param.Flags.isNest()) continue;
 
-    if (CalculateStackSlotUsed(Param.VT, Param.ArgVT, Param.Flags,
-                               PtrByteSize, LinkageSize, ParamAreaSize,
-                               NumBytes, AvailableFPRs, AvailableVRs,
-                               Subtarget.hasQPX()))
+    if (CalculateStackSlotUsed(Param.VT, Param.ArgVT, Param.Flags, PtrByteSize,
+                               LinkageSize, ParamAreaSize, NumBytes,
+                               AvailableFPRs, AvailableVRs))
       return true;
   }
   return false;
@@ -6064,7 +5836,6 @@ SDValue PPCTargetLowering::LowerCall_64SVR4(
   unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize();
   unsigned NumBytes = LinkageSize;
   unsigned GPR_idx = 0, FPR_idx = 0, VR_idx = 0;
-  unsigned &QFPR_idx = FPR_idx;
 
   static const MCPhysReg GPR[] = {
     PPC::X3, PPC::X4, PPC::X5, PPC::X6,
@@ -6078,7 +5849,6 @@ SDValue PPCTargetLowering::LowerCall_64SVR4(
   const unsigned NumGPRs = array_lengthof(GPR);
   const unsigned NumFPRs = useSoftFloat() ? 0 : 13;
   const unsigned NumVRs  = array_lengthof(VR);
-  const unsigned NumQFPRs = NumFPRs;
 
   // On ELFv2, we can avoid allocating the parameter area if all the arguments
   // can be passed to the callee in registers.
@@ -6093,9 +5863,8 @@ SDValue PPCTargetLowering::LowerCall_64SVR4(
     for (unsigned i = 0; i != NumOps; ++i) {
       if (Outs[i].Flags.isNest()) continue;
       if (CalculateStackSlotUsed(Outs[i].VT, Outs[i].ArgVT, Outs[i].Flags,
-                                PtrByteSize, LinkageSize, ParamAreaSize,
-                                NumBytesTmp, AvailableFPRs, AvailableVRs,
-                                Subtarget.hasQPX()))
+                                 PtrByteSize, LinkageSize, ParamAreaSize,
+                                 NumBytesTmp, AvailableFPRs, AvailableVRs))
         HasParameterArea = true;
     }
   }
@@ -6143,20 +5912,11 @@ SDValue PPCTargetLowering::LowerCall_64SVR4(
             continue;
           break;
         case MVT::v4f32:
-          // When using QPX, this is handled like a FP register, otherwise, it
-          // is an Altivec register.
-          if (Subtarget.hasQPX()) {
-            if (++NumFPRsUsed <= NumFPRs)
-              continue;
-          } else {
-            if (++NumVRsUsed <= NumVRs)
-              continue;
-          }
+          if (++NumVRsUsed <= NumVRs)
+            continue;
           break;
         case MVT::f32:
         case MVT::f64:
-        case MVT::v4f64: // QPX
-        case MVT::v4i1:  // QPX
           if (++NumFPRsUsed <= NumFPRs)
             continue;
           break;
@@ -6518,7 +6278,6 @@ SDValue PPCTargetLowering::LowerCall_64SVR4(
     case MVT::v2i64:
     case MVT::v1i128:
     case MVT::f128:
-      if (!Subtarget.hasQPX()) {
       // These can be scalar arguments or elements of a vector array type
       // passed directly.  The latter are used to implement ELFv2 homogenous
       // vector aggregates.
@@ -6574,63 +6333,6 @@ SDValue PPCTargetLowering::LowerCall_64SVR4(
       if (!IsFastCall)
         ArgOffset += 16;
       break;
-      } // not QPX
-
-      assert(Arg.getValueType().getSimpleVT().SimpleTy == MVT::v4f32 &&
-             "Invalid QPX parameter type");
-
-      LLVM_FALLTHROUGH;
-    case MVT::v4f64:
-    case MVT::v4i1: {
-      bool IsF32 = Arg.getValueType().getSimpleVT().SimpleTy == MVT::v4f32;
-      if (CFlags.IsVarArg) {
-        assert(HasParameterArea &&
-               "Parameter area must exist if we have a varargs call.");
-        // We could elide this store in the case where the object fits
-        // entirely in R registers.  Maybe later.
-        SDValue Store =
-            DAG.getStore(Chain, dl, Arg, PtrOff, MachinePointerInfo());
-        MemOpChains.push_back(Store);
-        if (QFPR_idx != NumQFPRs) {
-          SDValue Load = DAG.getLoad(IsF32 ? MVT::v4f32 : MVT::v4f64, dl, Store,
-                                     PtrOff, MachinePointerInfo());
-          MemOpChains.push_back(Load.getValue(1));
-          RegsToPass.push_back(std::make_pair(QFPR[QFPR_idx++], Load));
-        }
-        ArgOffset += (IsF32 ? 16 : 32);
-        for (unsigned i = 0; i < (IsF32 ? 16U : 32U); i += PtrByteSize) {
-          if (GPR_idx == NumGPRs)
-            break;
-          SDValue Ix = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff,
-                                   DAG.getConstant(i, dl, PtrVT));
-          SDValue Load =
-              DAG.getLoad(PtrVT, dl, Store, Ix, MachinePointerInfo());
-          MemOpChains.push_back(Load.getValue(1));
-          RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load));
-        }
-        break;
-      }
-
-      // Non-varargs QPX params go into registers or on the stack.
-      if (QFPR_idx != NumQFPRs) {
-        RegsToPass.push_back(std::make_pair(QFPR[QFPR_idx++], Arg));
-      } else {
-        if (IsFastCall)
-          ComputePtrOff();
-
-        assert(HasParameterArea &&
-               "Parameter area must exist to pass an argument in memory.");
-        LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset,
-                         true, CFlags.IsTailCall, true, MemOpChains,
-                         TailCallArguments, dl);
-        if (IsFastCall)
-          ArgOffset += (IsF32 ? 16 : 32);
-      }
-
-      if (!IsFastCall)
-        ArgOffset += (IsF32 ? 16 : 32);
-      break;
-      }
     }
   }
 
@@ -7301,8 +7003,6 @@ SDValue PPCTargetLowering::LowerFormalArguments_AIX(
 
   const PPCSubtarget &Subtarget =
       static_cast<const PPCSubtarget &>(DAG.getSubtarget());
-  if (Subtarget.hasQPX())
-    report_fatal_error("QPX support is not supported on AIX.");
 
   const bool IsPPC64 = Subtarget.isPPC64();
   const unsigned PtrByteSize = IsPPC64 ? 8 : 4;
@@ -7522,8 +7222,6 @@ SDValue PPCTargetLowering::LowerCall_AIX(
 
   const PPCSubtarget& Subtarget =
       static_cast<const PPCSubtarget&>(DAG.getSubtarget());
-  if (Subtarget.hasQPX())
-    report_fatal_error("QPX is not supported on AIX.");
   if (Subtarget.hasAltivec())
     report_fatal_error("Altivec support is unimplemented on AIX.");
 
@@ -7991,8 +7689,6 @@ SDValue PPCTargetLowering::lowerEH_SJLJ_LONGJMP(SDValue Op,
 }
 
 SDValue PPCTargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
-  if (Op.getValueType().isVector())
-    return LowerVectorLoad(Op, DAG);
 
   assert(Op.getValueType() == MVT::i1 &&
          "Custom lowering only for i1 loads");
@@ -8016,9 +7712,6 @@ SDValue PPCTargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
 }
 
 SDValue PPCTargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
-  if (Op.getOperand(1).getValueType().isVector())
-    return LowerVectorStore(Op, DAG);
-
   assert(Op.getOperand(1).getValueType() == MVT::i1 &&
          "Custom lowering only for i1 stores");
 
@@ -8595,27 +8288,6 @@ SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op,
   if (Op.getValueType() == MVT::f128)
     return Op;
 
-  if (Subtarget.hasQPX() && Op.getOperand(0).getValueType() == MVT::v4i1) {
-    if (Op.getValueType() != MVT::v4f32 && Op.getValueType() != MVT::v4f64)
-      return SDValue();
-
-    SDValue Value = Op.getOperand(0);
-    // The values are now known to be -1 (false) or 1 (true). To convert this
-    // into 0 (false) and 1 (true), add 1 and then divide by 2 (multiply by 0.5).
-    // This can be done with an fma and the 0.5 constant: (V+1.0)*0.5 = 0.5*V+0.5
-    Value = DAG.getNode(PPCISD::QBFLT, dl, MVT::v4f64, Value);
-
-    SDValue FPHalfs = DAG.getConstantFP(0.5, dl, MVT::v4f64);
-
-    Value = DAG.getNode(ISD::FMA, dl, MVT::v4f64, Value, FPHalfs, FPHalfs);
-
-    if (Op.getValueType() != MVT::v4f64)
-      Value = DAG.getNode(ISD::FP_ROUND, dl,
-                          Op.getValueType(), Value,
-                          DAG.getIntPtrConstant(1, dl));
-    return Value;
-  }
-
   // Don't handle ppc_fp128 here; let it be lowered to a libcall.
   if (Op.getValueType() != MVT::f32 && Op.getValueType() != MVT::f64)
     return SDValue();
@@ -9184,110 +8856,6 @@ SDValue PPCTargetLowering::LowerBUILD_VECTOR(SDValue Op,
   BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(Op.getNode());
   assert(BVN && "Expected a BuildVectorSDNode in LowerBUILD_VECTOR");
 
-  if (Subtarget.hasQPX() && Op.getValueType() == MVT::v4i1) {
-    // We first build an i32 vector, load it into a QPX register,
-    // then convert it to a floating-point vector and compare it
-    // to a zero vector to get the boolean result.
-    MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
-    int FrameIdx = MFI.CreateStackObject(16, Align(16), false);
-    MachinePointerInfo PtrInfo =
-        MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx);
-    EVT PtrVT = getPointerTy(DAG.getDataLayout());
-    SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT);
-
-    assert(BVN->getNumOperands() == 4 &&
-      "BUILD_VECTOR for v4i1 does not have 4 operands");
-
-    bool IsConst = true;
-    for (unsigned i = 0; i < 4; ++i) {
-      if (BVN->getOperand(i).isUndef()) continue;
-      if (!isa<ConstantSDNode>(BVN->getOperand(i))) {
-        IsConst = false;
-        break;
-      }
-    }
-
-    if (IsConst) {
-      Constant *One =
-        ConstantFP::get(Type::getFloatTy(*DAG.getContext()), 1.0);
-      Constant *NegOne =
-        ConstantFP::get(Type::getFloatTy(*DAG.getContext()), -1.0);
-
-      Constant *CV[4];
-      for (unsigned i = 0; i < 4; ++i) {
-        if (BVN->getOperand(i).isUndef())
-          CV[i] = UndefValue::get(Type::getFloatTy(*DAG.getContext()));
-        else if (isNullConstant(BVN->getOperand(i)))
-          CV[i] = NegOne;
-        else
-          CV[i] = One;
-      }
-
-      Constant *CP = ConstantVector::get(CV);
-      SDValue CPIdx =
-          DAG.getConstantPool(CP, getPointerTy(DAG.getDataLayout()), Align(16));
-
-      SDValue Ops[] = {DAG.getEntryNode(), CPIdx};
-      SDVTList VTs = DAG.getVTList({MVT::v4i1, /*chain*/ MVT::Other});
-      return DAG.getMemIntrinsicNode(
-          PPCISD::QVLFSb, dl, VTs, Ops, MVT::v4f32,
-          MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
-    }
-
-    SmallVector<SDValue, 4> Stores;
-    for (unsigned i = 0; i < 4; ++i) {
-      if (BVN->getOperand(i).isUndef()) continue;
-
-      unsigned Offset = 4*i;
-      SDValue Idx = DAG.getConstant(Offset, dl, FIdx.getValueType());
-      Idx = DAG.getNode(ISD::ADD, dl, FIdx.getValueType(), FIdx, Idx);
-
-      unsigned StoreSize = BVN->getOperand(i).getValueType().getStoreSize();
-      if (StoreSize > 4) {
-        Stores.push_back(
-            DAG.getTruncStore(DAG.getEntryNode(), dl, BVN->getOperand(i), Idx,
-                              PtrInfo.getWithOffset(Offset), MVT::i32));
-      } else {
-        SDValue StoreValue = BVN->getOperand(i);
-        if (StoreSize < 4)
-          StoreValue = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, StoreValue);
-
-        Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl, StoreValue, Idx,
-                                      PtrInfo.getWithOffset(Offset)));
-      }
-    }
-
-    SDValue StoreChain;
-    if (!Stores.empty())
-      StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
-    else
-      StoreChain = DAG.getEntryNode();
-
-    // Now load from v4i32 into the QPX register; this will extend it to
-    // v4i64 but not yet convert it to a floating point. Nevertheless, this
-    // is typed as v4f64 because the QPX register integer states are not
-    // explicitly represented.
-
-    SDValue Ops[] = {StoreChain,
-                     DAG.getConstant(Intrinsic::ppc_qpx_qvlfiwz, dl, MVT::i32),
-                     FIdx};
-    SDVTList VTs = DAG.getVTList({MVT::v4f64, /*chain*/ MVT::Other});
-
-    SDValue LoadedVect = DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN,
-      dl, VTs, Ops, MVT::v4i32, PtrInfo);
-    LoadedVect = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f64,
-      DAG.getConstant(Intrinsic::ppc_qpx_qvfcfidu, dl, MVT::i32),
-      LoadedVect);
-
-    SDValue FPZeros = DAG.getConstantFP(0.0, dl, MVT::v4f64);
-
-    return DAG.getSetCC(dl, MVT::v4i1, LoadedVect, FPZeros, ISD::SETEQ);
-  }
-
-  // All other QPX vectors are handled by generic code.
-  if (Subtarget.hasQPX())
-    return SDValue();
-
   // Check if this is a splat of a constant value.
   APInt APSplatBits, APSplatUndef;
   unsigned SplatBitSize;
@@ -10080,42 +9648,6 @@ SDValue PPCTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
     }
   }
 
-  if (Subtarget.hasQPX()) {
-    if (VT.getVectorNumElements() != 4)
-      return SDValue();
-
-    if (V2.isUndef()) V2 = V1;
-
-    int AlignIdx = PPC::isQVALIGNIShuffleMask(SVOp);
-    if (AlignIdx != -1) {
-      return DAG.getNode(PPCISD::QVALIGNI, dl, VT, V1, V2,
-                         DAG.getConstant(AlignIdx, dl, MVT::i32));
-    } else if (SVOp->isSplat()) {
-      int SplatIdx = SVOp->getSplatIndex();
-      if (SplatIdx >= 4) {
-        std::swap(V1, V2);
-        SplatIdx -= 4;
-      }
-
-      return DAG.getNode(PPCISD::QVESPLATI, dl, VT, V1,
-                         DAG.getConstant(SplatIdx, dl, MVT::i32));
-    }
-
-    // Lower this into a qvgpci/qvfperm pair.
-
-    // Compute the qvgpci literal
-    unsigned idx = 0;
-    for (unsigned i = 0; i < 4; ++i) {
-      int m = SVOp->getMaskElt(i);
-      unsigned mm = m >= 0 ? (unsigned) m : i;
-      idx |= mm << (3-i)*3;
-    }
-
-    SDValue V3 = DAG.getNode(PPCISD::QVGPCI, dl, MVT::v4f64,
-                             DAG.getConstant(idx, dl, MVT::i32));
-    return DAG.getNode(PPCISD::QVFPERM, dl, VT, V1, V2, V3);
-  }
-
   // Cases that are handled by instructions that take permute immediates
   // (such as vsplt*) should be left as VECTOR_SHUFFLE nodes so they can be
   // selected by the instruction selector.
@@ -10703,279 +10235,6 @@ SDValue PPCTargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
   return Op;
 }
 
-SDValue PPCTargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
-                                                   SelectionDAG &DAG) const {
-  SDLoc dl(Op);
-  SDNode *N = Op.getNode();
-
-  assert(N->getOperand(0).getValueType() == MVT::v4i1 &&
-         "Unknown extract_vector_elt type");
-
-  SDValue Value = N->getOperand(0);
-
-  // The first part of this is like the store lowering except that we don't
-  // need to track the chain.
-
-  // The values are now known to be -1 (false) or 1 (true). To convert this
-  // into 0 (false) and 1 (true), add 1 and then divide by 2 (multiply by 0.5).
-  // This can be done with an fma and the 0.5 constant: (V+1.0)*0.5 = 0.5*V+0.5
-  Value = DAG.getNode(PPCISD::QBFLT, dl, MVT::v4f64, Value);
-
-  // FIXME: We can make this an f32 vector, but the BUILD_VECTOR code needs to
-  // understand how to form the extending load.
-  SDValue FPHalfs = DAG.getConstantFP(0.5, dl, MVT::v4f64);
-
-  Value = DAG.getNode(ISD::FMA, dl, MVT::v4f64, Value, FPHalfs, FPHalfs);
-
-  // Now convert to an integer and store.
-  Value = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f64,
-    DAG.getConstant(Intrinsic::ppc_qpx_qvfctiwu, dl, MVT::i32),
-    Value);
-
-  MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
-  int FrameIdx = MFI.CreateStackObject(16, Align(16), false);
-  MachinePointerInfo PtrInfo =
-      MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx);
-  EVT PtrVT = getPointerTy(DAG.getDataLayout());
-  SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT);
-
-  SDValue StoreChain = DAG.getEntryNode();
-  SDValue Ops[] = {StoreChain,
-                   DAG.getConstant(Intrinsic::ppc_qpx_qvstfiw, dl, MVT::i32),
-                   Value, FIdx};
-  SDVTList VTs = DAG.getVTList(/*chain*/ MVT::Other);
-
-  StoreChain = DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID,
-    dl, VTs, Ops, MVT::v4i32, PtrInfo);
-
-  // Extract the value requested.
-  unsigned Offset = 4*cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
-  SDValue Idx = DAG.getConstant(Offset, dl, FIdx.getValueType());
-  Idx = DAG.getNode(ISD::ADD, dl, FIdx.getValueType(), FIdx, Idx);
-
-  SDValue IntVal =
-      DAG.getLoad(MVT::i32, dl, StoreChain, Idx, PtrInfo.getWithOffset(Offset));
-
-  if (!Subtarget.useCRBits())
-    return IntVal;
-
-  return DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, IntVal);
-}
-
-/// Lowering for QPX v4i1 loads
-SDValue PPCTargetLowering::LowerVectorLoad(SDValue Op,
-                                           SelectionDAG &DAG) const {
-  SDLoc dl(Op);
-  LoadSDNode *LN = cast<LoadSDNode>(Op.getNode());
-  SDValue LoadChain = LN->getChain();
-  SDValue BasePtr = LN->getBasePtr();
-
-  if (Op.getValueType() == MVT::v4f64 ||
-      Op.getValueType() == MVT::v4f32) {
-    EVT MemVT = LN->getMemoryVT();
-    unsigned Alignment = LN->getAlignment();
-
-    // If this load is properly aligned, then it is legal.
-    if (Alignment >= MemVT.getStoreSize())
-      return Op;
-
-    EVT ScalarVT = Op.getValueType().getScalarType(),
-        ScalarMemVT = MemVT.getScalarType();
-    unsigned Stride = ScalarMemVT.getStoreSize();
-
-    SDValue Vals[4], LoadChains[4];
-    for (unsigned Idx = 0; Idx < 4; ++Idx) {
-      SDValue Load;
-      if (ScalarVT != ScalarMemVT)
-        Load = DAG.getExtLoad(LN->getExtensionType(), dl, ScalarVT, LoadChain,
-                              BasePtr,
-                              LN->getPointerInfo().getWithOffset(Idx * Stride),
-                              ScalarMemVT, MinAlign(Alignment, Idx * Stride),
-                              LN->getMemOperand()->getFlags(), LN->getAAInfo());
-      else
-        Load = DAG.getLoad(ScalarVT, dl, LoadChain, BasePtr,
-                           LN->getPointerInfo().getWithOffset(Idx * Stride),
-                           MinAlign(Alignment, Idx * Stride),
-                           LN->getMemOperand()->getFlags(), LN->getAAInfo());
-
-      if (Idx == 0 && LN->isIndexed()) {
-        assert(LN->getAddressingMode() == ISD::PRE_INC &&
-               "Unknown addressing mode on vector load");
-        Load = DAG.getIndexedLoad(Load, dl, BasePtr, LN->getOffset(),
-                                  LN->getAddressingMode());
-      }
-
-      Vals[Idx] = Load;
-      LoadChains[Idx] = Load.getValue(1);
-
-      BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr,
-                            DAG.getConstant(Stride, dl,
-                                            BasePtr.getValueType()));
-    }
-
-    SDValue TF =  DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains);
-    SDValue Value = DAG.getBuildVector(Op.getValueType(), dl, Vals);
-
-    if (LN->isIndexed()) {
-      SDValue RetOps[] = { Value, Vals[0].getValue(1), TF };
-      return DAG.getMergeValues(RetOps, dl);
-    }
-
-    SDValue RetOps[] = { Value, TF };
-    return DAG.getMergeValues(RetOps, dl);
-  }
-
-  assert(Op.getValueType() == MVT::v4i1 && "Unknown load to lower");
-  assert(LN->isUnindexed() && "Indexed v4i1 loads are not supported");
-
-  // To lower v4i1 from a byte array, we load the byte elements of the
-  // vector and then reuse the BUILD_VECTOR logic.
-
-  SDValue VectElmts[4], VectElmtChains[4];
-  for (unsigned i = 0; i < 4; ++i) {
-    SDValue Idx = DAG.getConstant(i, dl, BasePtr.getValueType());
-    Idx = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, Idx);
-
-    VectElmts[i] = DAG.getExtLoad(
-        ISD::EXTLOAD, dl, MVT::i32, LoadChain, Idx,
-        LN->getPointerInfo().getWithOffset(i), MVT::i8,
-        /* Alignment = */ 1, LN->getMemOperand()->getFlags(), LN->getAAInfo());
-    VectElmtChains[i] = VectElmts[i].getValue(1);
-  }
-
-  LoadChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, VectElmtChains);
-  SDValue Value = DAG.getBuildVector(MVT::v4i1, dl, VectElmts);
-
-  SDValue RVals[] = { Value, LoadChain };
-  return DAG.getMergeValues(RVals, dl);
-}
-
-/// Lowering for QPX v4i1 stores
-SDValue PPCTargetLowering::LowerVectorStore(SDValue Op,
-                                            SelectionDAG &DAG) const {
-  SDLoc dl(Op);
-  StoreSDNode *SN = cast<StoreSDNode>(Op.getNode());
-  SDValue StoreChain = SN->getChain();
-  SDValue BasePtr = SN->getBasePtr();
-  SDValue Value = SN->getValue();
-
-  if (Value.getValueType() == MVT::v4f64 ||
-      Value.getValueType() == MVT::v4f32) {
-    EVT MemVT = SN->getMemoryVT();
-    unsigned Alignment = SN->getAlignment();
-
-    // If this store is properly aligned, then it is legal.
-    if (Alignment >= MemVT.getStoreSize())
-      return Op;
-
-    EVT ScalarVT = Value.getValueType().getScalarType(),
-        ScalarMemVT = MemVT.getScalarType();
-    unsigned Stride = ScalarMemVT.getStoreSize();
-
-    SDValue Stores[4];
-    for (unsigned Idx = 0; Idx < 4; ++Idx) {
-      SDValue Ex = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ScalarVT, Value,
-                               DAG.getVectorIdxConstant(Idx, dl));
-      SDValue Store;
-      if (ScalarVT != ScalarMemVT)
-        Store =
-            DAG.getTruncStore(StoreChain, dl, Ex, BasePtr,
-                              SN->getPointerInfo().getWithOffset(Idx * Stride),
-                              ScalarMemVT, MinAlign(Alignment, Idx * Stride),
-                              SN->getMemOperand()->getFlags(), SN->getAAInfo());
-      else
-        Store = DAG.getStore(StoreChain, dl, Ex, BasePtr,
-                             SN->getPointerInfo().getWithOffset(Idx * Stride),
-                             MinAlign(Alignment, Idx * Stride),
-                             SN->getMemOperand()->getFlags(), SN->getAAInfo());
-
-      if (Idx == 0 && SN->isIndexed()) {
-        assert(SN->getAddressingMode() == ISD::PRE_INC &&
-               "Unknown addressing mode on vector store");
-        Store = DAG.getIndexedStore(Store, dl, BasePtr, SN->getOffset(),
-                                    SN->getAddressingMode());
-      }
-
-      BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr,
-                            DAG.getConstant(Stride, dl,
-                                            BasePtr.getValueType()));
-      Stores[Idx] = Store;
-    }
-
-    SDValue TF =  DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
-
-    if (SN->isIndexed()) {
-      SDValue RetOps[] = { TF, Stores[0].getValue(1) };
-      return DAG.getMergeValues(RetOps, dl);
-    }
-
-    return TF;
-  }
-
-  assert(SN->isUnindexed() && "Indexed v4i1 stores are not supported");
-  assert(Value.getValueType() == MVT::v4i1 && "Unknown store to lower");
-
-  // The values are now known to be -1 (false) or 1 (true). To convert this
-  // into 0 (false) and 1 (true), add 1 and then divide by 2 (multiply by 0.5).
-  // This can be done with an fma and the 0.5 constant: (V+1.0)*0.5 = 0.5*V+0.5
-  Value = DAG.getNode(PPCISD::QBFLT, dl, MVT::v4f64, Value);
-
-  // FIXME: We can make this an f32 vector, but the BUILD_VECTOR code needs to
-  // understand how to form the extending load.
-  SDValue FPHalfs = DAG.getConstantFP(0.5, dl, MVT::v4f64);
-
-  Value = DAG.getNode(ISD::FMA, dl, MVT::v4f64, Value, FPHalfs, FPHalfs);
-
-  // Now convert to an integer and store.
-  Value = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f64,
-    DAG.getConstant(Intrinsic::ppc_qpx_qvfctiwu, dl, MVT::i32),
-    Value);
-
-  MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
-  int FrameIdx = MFI.CreateStackObject(16, Align(16), false);
-  MachinePointerInfo PtrInfo =
-      MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx);
-  EVT PtrVT = getPointerTy(DAG.getDataLayout());
-  SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT);
-
-  SDValue Ops[] = {StoreChain,
-                   DAG.getConstant(Intrinsic::ppc_qpx_qvstfiw, dl, MVT::i32),
-                   Value, FIdx};
-  SDVTList VTs = DAG.getVTList(/*chain*/ MVT::Other);
-
-  StoreChain = DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID,
-    dl, VTs, Ops, MVT::v4i32, PtrInfo);
-
-  // Move data into the byte array.
-  SDValue Loads[4], LoadChains[4];
-  for (unsigned i = 0; i < 4; ++i) {
-    unsigned Offset = 4*i;
-    SDValue Idx = DAG.getConstant(Offset, dl, FIdx.getValueType());
-    Idx = DAG.getNode(ISD::ADD, dl, FIdx.getValueType(), FIdx, Idx);
-
-    Loads[i] = DAG.getLoad(MVT::i32, dl, StoreChain, Idx,
-                           PtrInfo.getWithOffset(Offset));
-    LoadChains[i] = Loads[i].getValue(1);
-  }
-
-  StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains);
-
-  SDValue Stores[4];
-  for (unsigned i = 0; i < 4; ++i) {
-    SDValue Idx = DAG.getConstant(i, dl, BasePtr.getValueType());
-    Idx = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, Idx);
-
-    Stores[i] = DAG.getTruncStore(
-        StoreChain, dl, Loads[i], Idx, SN->getPointerInfo().getWithOffset(i),
-        MVT::i8, /* Alignment = */ 1, SN->getMemOperand()->getFlags(),
-        SN->getAAInfo());
-  }
-
-  StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
-
-  return StoreChain;
-}
-
 SDValue PPCTargetLowering::LowerMUL(SDValue Op, SelectionDAG &DAG) const {
   SDLoc dl(Op);
   if (Op.getValueType() == MVT::v4i32) {
@@ -11204,7 +10463,6 @@ SDValue PPCTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
   case ISD::VECTOR_SHUFFLE:     return LowerVECTOR_SHUFFLE(Op, DAG);
   case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG);
   case ISD::SCALAR_TO_VECTOR:   return LowerSCALAR_TO_VECTOR(Op, DAG);
-  case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG);
   case ISD::INSERT_VECTOR_ELT:  return LowerINSERT_VECTOR_ELT(Op, DAG);
   case ISD::MUL:                return LowerMUL(Op, DAG);
   case ISD::ABS:                return LowerABS(Op, DAG);
@@ -12148,9 +11406,6 @@ PPCTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
   } else if (MI.getOpcode() == PPC::SELECT_CC_F4 ||
              MI.getOpcode() == PPC::SELECT_CC_F8 ||
              MI.getOpcode() == PPC::SELECT_CC_F16 ||
-             MI.getOpcode() == PPC::SELECT_CC_QFRC ||
-             MI.getOpcode() == PPC::SELECT_CC_QSRC ||
-             MI.getOpcode() == PPC::SELECT_CC_QBRC ||
              MI.getOpcode() == PPC::SELECT_CC_VRRC ||
              MI.getOpcode() == PPC::SELECT_CC_VSFRC ||
              MI.getOpcode() == PPC::SELECT_CC_VSSRC ||
@@ -12160,9 +11415,6 @@ PPCTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
              MI.getOpcode() == PPC::SELECT_F4 ||
              MI.getOpcode() == PPC::SELECT_F8 ||
              MI.getOpcode() == PPC::SELECT_F16 ||
-             MI.getOpcode() == PPC::SELECT_QFRC ||
-             MI.getOpcode() == PPC::SELECT_QSRC ||
-             MI.getOpcode() == PPC::SELECT_QBRC ||
              MI.getOpcode() == PPC::SELECT_SPE ||
              MI.getOpcode() == PPC::SELECT_SPE4 ||
              MI.getOpcode() == PPC::SELECT_VRRC ||
@@ -12200,9 +11452,6 @@ PPCTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
         MI.getOpcode() == PPC::SELECT_F16 ||
         MI.getOpcode() == PPC::SELECT_SPE4 ||
         MI.getOpcode() == PPC::SELECT_SPE ||
-        MI.getOpcode() == PPC::SELECT_QFRC ||
-        MI.getOpcode() == PPC::SELECT_QSRC ||
-        MI.getOpcode() == PPC::SELECT_QBRC ||
         MI.getOpcode() == PPC::SELECT_VRRC ||
         MI.getOpcode() == PPC::SELECT_VSFRC ||
         MI.getOpcode() == PPC::SELECT_VSSRC ||
@@ -12895,9 +12144,7 @@ SDValue PPCTargetLowering::getSqrtEstimate(SDValue Operand, SelectionDAG &DAG,
   if ((VT == MVT::f32 && Subtarget.hasFRSQRTES()) ||
       (VT == MVT::f64 && Subtarget.hasFRSQRTE()) ||
       (VT == MVT::v4f32 && Subtarget.hasAltivec()) ||
-      (VT == MVT::v2f64 && Subtarget.hasVSX()) ||
-      (VT == MVT::v4f32 && Subtarget.hasQPX()) ||
-      (VT == MVT::v4f64 && Subtarget.hasQPX())) {
+      (VT == MVT::v2f64 && Subtarget.hasVSX())) {
     if (RefinementSteps == ReciprocalEstimate::Unspecified)
       RefinementSteps = getEstimateRefinementSteps(VT, Subtarget);
 
@@ -12916,9 +12163,7 @@ SDValue PPCTargetLowering::getRecipEstimate(SDValue Operand, SelectionDAG &DAG,
   if ((VT == MVT::f32 && Subtarget.hasFRES()) ||
       (VT == MVT::f64 && Subtarget.hasFRE()) ||
       (VT == MVT::v4f32 && Subtarget.hasAltivec()) ||
-      (VT == MVT::v2f64 && Subtarget.hasVSX()) ||
-      (VT == MVT::v4f32 && Subtarget.hasQPX()) ||
-      (VT == MVT::v4f64 && Subtarget.hasQPX())) {
+      (VT == MVT::v2f64 && Subtarget.hasVSX())) {
     if (RefinementSteps == ReciprocalEstimate::Unspecified)
       RefinementSteps = getEstimateRefinementSteps(VT, Subtarget);
     return DAG.getNode(PPCISD::FRE, SDLoc(Operand), VT, Operand);
@@ -13016,24 +12261,6 @@ static bool isConsecutiveLS(SDNode *N, LSBaseSDNode *Base,
     EVT VT;
     switch (cast<ConstantSDNode>(N->getOperand(1))->getZExtValue()) {
     default: return false;
-    case Intrinsic::ppc_qpx_qvlfd:
-    case Intrinsic::ppc_qpx_qvlfda:
-      VT = MVT::v4f64;
-      break;
-    case Intrinsic::ppc_qpx_qvlfs:
-    case Intrinsic::ppc_qpx_qvlfsa:
-      VT = MVT::v4f32;
-      break;
-    case Intrinsic::ppc_qpx_qvlfcd:
-    case Intrinsic::ppc_qpx_qvlfcda:
-      VT = MVT::v2f64;
-      break;
-    case Intrinsic::ppc_qpx_qvlfcs:
-    case Intrinsic::ppc_qpx_qvlfcsa:
-      VT = MVT::v2f32;
-      break;
-    case Intrinsic::ppc_qpx_qvlfiwa:
-    case Intrinsic::ppc_qpx_qvlfiwz:
     case Intrinsic::ppc_altivec_lvx:
     case Intrinsic::ppc_altivec_lvxl:
     case Intrinsic::ppc_vsx_lxvw4x:
@@ -13062,24 +12289,6 @@ static bool isConsecutiveLS(SDNode *N, LSBaseSDNode *Base,
     EVT VT;
     switch (cast<ConstantSDNode>(N->getOperand(1))->getZExtValue()) {
     default: return false;
-    case Intrinsic::ppc_qpx_qvstfd:
-    case Intrinsic::ppc_qpx_qvstfda:
-      VT = MVT::v4f64;
-      break;
-    case Intrinsic::ppc_qpx_qvstfs:
-    case Intrinsic::ppc_qpx_qvstfsa:
-      VT = MVT::v4f32;
-      break;
-    case Intrinsic::ppc_qpx_qvstfcd:
-    case Intrinsic::ppc_qpx_qvstfcda:
-      VT = MVT::v2f64;
-      break;
-    case Intrinsic::ppc_qpx_qvstfcs:
-    case Intrinsic::ppc_qpx_qvstfcsa:
-      VT = MVT::v2f32;
-      break;
-    case Intrinsic::ppc_qpx_qvstfiw:
-    case Intrinsic::ppc_qpx_qvstfiwa:
     case Intrinsic::ppc_altivec_stvx:
     case Intrinsic::ppc_altivec_stvxl:
     case Intrinsic::ppc_vsx_stxvw4x:
@@ -15077,18 +14286,14 @@ SDValue PPCTargetLowering::PerformDAGCombine(SDNode *N,
     EVT MemVT = LD->getMemoryVT();
     Type *Ty = MemVT.getTypeForEVT(*DAG.getContext());
     Align ABIAlignment = DAG.getDataLayout().getABITypeAlign(Ty);
-    Type *STy = MemVT.getScalarType().getTypeForEVT(*DAG.getContext());
-    Align ScalarABIAlignment = DAG.getDataLayout().getABITypeAlign(STy);
     if (LD->isUnindexed() && VT.isVector() &&
         ((Subtarget.hasAltivec() && ISD::isNON_EXTLoad(N) &&
           // P8 and later hardware should just use LOAD.
           !Subtarget.hasP8Vector() &&
           (VT == MVT::v16i8 || VT == MVT::v8i16 || VT == MVT::v4i32 ||
-           VT == MVT::v4f32)) ||
-         (Subtarget.hasQPX() && (VT == MVT::v4f64 || VT == MVT::v4f32) &&
-          LD->getAlign() >= ScalarABIAlignment)) &&
+           VT == MVT::v4f32))) &&
         LD->getAlign() < ABIAlignment) {
-      // This is a type-legal unaligned Altivec or QPX load.
+      // This is a type-legal unaligned Altivec load.
       SDValue Chain = LD->getChain();
       SDValue Ptr = LD->getBasePtr();
       bool isLittleEndian = Subtarget.isLittleEndian();
@@ -15119,24 +14324,13 @@ SDValue PPCTargetLowering::PerformDAGCombine(SDNode *N,
       // optimization later.
       Intrinsic::ID Intr, IntrLD, IntrPerm;
       MVT PermCntlTy, PermTy, LDTy;
-      if (Subtarget.hasAltivec()) {
-        Intr = isLittleEndian ?  Intrinsic::ppc_altivec_lvsr :
-                                 Intrinsic::ppc_altivec_lvsl;
-        IntrLD = Intrinsic::ppc_altivec_lvx;
-        IntrPerm = Intrinsic::ppc_altivec_vperm;
-        PermCntlTy = MVT::v16i8;
-        PermTy = MVT::v4i32;
-        LDTy = MVT::v4i32;
-      } else {
-        Intr =   MemVT == MVT::v4f64 ? Intrinsic::ppc_qpx_qvlpcld :
-                                       Intrinsic::ppc_qpx_qvlpcls;
-        IntrLD = MemVT == MVT::v4f64 ? Intrinsic::ppc_qpx_qvlfd :
-                                       Intrinsic::ppc_qpx_qvlfs;
-        IntrPerm = Intrinsic::ppc_qpx_qvfperm;
-        PermCntlTy = MVT::v4f64;
-        PermTy = MVT::v4f64;
-        LDTy = MemVT.getSimpleVT();
-      }
+      Intr = isLittleEndian ? Intrinsic::ppc_altivec_lvsr
+                            : Intrinsic::ppc_altivec_lvsl;
+      IntrLD = Intrinsic::ppc_altivec_lvx;
+      IntrPerm = Intrinsic::ppc_altivec_vperm;
+      PermCntlTy = MVT::v16i8;
+      PermTy = MVT::v4i32;
+      LDTy = MVT::v4i32;
 
       SDValue PermCntl = BuildIntrinsicOp(Intr, Ptr, DAG, dl, PermCntlTy);
 
@@ -15207,10 +14401,10 @@ SDValue PPCTargetLowering::PerformDAGCombine(SDNode *N,
                                 BaseLoad, ExtraLoad, PermCntl, DAG, dl);
 
       if (VT != PermTy)
-        Perm = Subtarget.hasAltivec() ?
-                 DAG.getNode(ISD::BITCAST, dl, VT, Perm) :
-                 DAG.getNode(ISD::FP_ROUND, dl, VT, Perm, // QPX
-                               DAG.getTargetConstant(1, dl, MVT::i64));
+        Perm = Subtarget.hasAltivec()
+                   ? DAG.getNode(ISD::BITCAST, dl, VT, Perm)
+                   : DAG.getNode(ISD::FP_ROUND, dl, VT, Perm,
+                                 DAG.getTargetConstant(1, dl, MVT::i64));
                                // second argument is 1 because this rounding
                                // is always exact.
 
@@ -15226,14 +14420,10 @@ SDValue PPCTargetLowering::PerformDAGCombine(SDNode *N,
       unsigned IID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
       Intrinsic::ID Intr = (isLittleEndian ? Intrinsic::ppc_altivec_lvsr
                                            : Intrinsic::ppc_altivec_lvsl);
-      if ((IID == Intr ||
-           IID == Intrinsic::ppc_qpx_qvlpcld  ||
-           IID == Intrinsic::ppc_qpx_qvlpcls) &&
-        N->getOperand(1)->getOpcode() == ISD::ADD) {
+      if (IID == Intr && N->getOperand(1)->getOpcode() == ISD::ADD) {
         SDValue Add = N->getOperand(1);
 
-        int Bits = IID == Intrinsic::ppc_qpx_qvlpcld ?
-                   5 /* 32 byte alignment */ : 4 /* 16 byte alignment */;
+        int Bits = 4 /* 16 byte alignment */;
 
         if (DAG.MaskedValueIsZero(Add->getOperand(1),
                                   APInt::getAllOnesValue(Bits /* alignment */)
@@ -15243,7 +14433,8 @@ SDValue PPCTargetLowering::PerformDAGCombine(SDNode *N,
                                     UE = BasePtr->use_end();
                UI != UE; ++UI) {
             if (UI->getOpcode() == ISD::INTRINSIC_WO_CHAIN &&
-                cast<ConstantSDNode>(UI->getOperand(0))->getZExtValue() == IID) {
+                cast<ConstantSDNode>(UI->getOperand(0))->getZExtValue() ==
+                    IID) {
               // We've found another LVSL/LVSR, and this address is an aligned
               // multiple of that one. The results will be the same, so use the
               // one we've just found instead.
@@ -15792,17 +14983,9 @@ PPCTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
           return std::make_pair(0U, &PPC::F4RCRegClass);
         if (VT == MVT::f64 || VT == MVT::i64)
           return std::make_pair(0U, &PPC::F8RCRegClass);
-        if (VT == MVT::v4f64 && Subtarget.hasQPX())
-          return std::make_pair(0U, &PPC::QFRCRegClass);
-        if (VT == MVT::v4f32 && Subtarget.hasQPX())
-          return std::make_pair(0U, &PPC::QSRCRegClass);
       }
       break;
     case 'v':
-      if (VT == MVT::v4f64 && Subtarget.hasQPX())
-        return std::make_pair(0U, &PPC::QFRCRegClass);
-      if (VT == MVT::v4f32 && Subtarget.hasQPX())
-        return std::make_pair(0U, &PPC::QSRCRegClass);
       if (Subtarget.hasAltivec())
         return std::make_pair(0U, &PPC::VRRCRegClass);
       break;
@@ -16094,12 +15277,6 @@ bool PPCTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
                                            MachineFunction &MF,
                                            unsigned Intrinsic) const {
   switch (Intrinsic) {
-  case Intrinsic::ppc_qpx_qvlfd:
-  case Intrinsic::ppc_qpx_qvlfs:
-  case Intrinsic::ppc_qpx_qvlfcd:
-  case Intrinsic::ppc_qpx_qvlfcs:
-  case Intrinsic::ppc_qpx_qvlfiwa:
-  case Intrinsic::ppc_qpx_qvlfiwz:
   case Intrinsic::ppc_altivec_lvx:
   case Intrinsic::ppc_altivec_lvxl:
   case Intrinsic::ppc_altivec_lvebx:
@@ -16121,18 +15298,6 @@ bool PPCTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
     case Intrinsic::ppc_vsx_lxvd2x:
       VT = MVT::v2f64;
       break;
-    case Intrinsic::ppc_qpx_qvlfd:
-      VT = MVT::v4f64;
-      break;
-    case Intrinsic::ppc_qpx_qvlfs:
-      VT = MVT::v4f32;
-      break;
-    case Intrinsic::ppc_qpx_qvlfcd:
-      VT = MVT::v2f64;
-      break;
-    case Intrinsic::ppc_qpx_qvlfcs:
-      VT = MVT::v2f32;
-      break;
     default:
       VT = MVT::v4i32;
       break;
@@ -16147,45 +15312,6 @@ bool PPCTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
     Info.flags = MachineMemOperand::MOLoad;
     return true;
   }
-  case Intrinsic::ppc_qpx_qvlfda:
-  case Intrinsic::ppc_qpx_qvlfsa:
-  case Intrinsic::ppc_qpx_qvlfcda:
-  case Intrinsic::ppc_qpx_qvlfcsa:
-  case Intrinsic::ppc_qpx_qvlfiwaa:
-  case Intrinsic::ppc_qpx_qvlfiwza: {
-    EVT VT;
-    switch (Intrinsic) {
-    case Intrinsic::ppc_qpx_qvlfda:
-      VT = MVT::v4f64;
-      break;
-    case Intrinsic::ppc_qpx_qvlfsa:
-      VT = MVT::v4f32;
-      break;
-    case Intrinsic::ppc_qpx_qvlfcda:
-      VT = MVT::v2f64;
-      break;
-    case Intrinsic::ppc_qpx_qvlfcsa:
-      VT = MVT::v2f32;
-      break;
-    default:
-      VT = MVT::v4i32;
-      break;
-    }
-
-    Info.opc = ISD::INTRINSIC_W_CHAIN;
-    Info.memVT = VT;
-    Info.ptrVal = I.getArgOperand(0);
-    Info.offset = 0;
-    Info.size = VT.getStoreSize();
-    Info.align = Align(1);
-    Info.flags = MachineMemOperand::MOLoad;
-    return true;
-  }
-  case Intrinsic::ppc_qpx_qvstfd:
-  case Intrinsic::ppc_qpx_qvstfs:
-  case Intrinsic::ppc_qpx_qvstfcd:
-  case Intrinsic::ppc_qpx_qvstfcs:
-  case Intrinsic::ppc_qpx_qvstfiw:
   case Intrinsic::ppc_altivec_stvx:
   case Intrinsic::ppc_altivec_stvxl:
   case Intrinsic::ppc_altivec_stvebx:
@@ -16207,18 +15333,6 @@ bool PPCTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
     case Intrinsic::ppc_vsx_stxvd2x:
       VT = MVT::v2f64;
       break;
-    case Intrinsic::ppc_qpx_qvstfd:
-      VT = MVT::v4f64;
-      break;
-    case Intrinsic::ppc_qpx_qvstfs:
-      VT = MVT::v4f32;
-      break;
-    case Intrinsic::ppc_qpx_qvstfcd:
-      VT = MVT::v2f64;
-      break;
-    case Intrinsic::ppc_qpx_qvstfcs:
-      VT = MVT::v2f32;
-      break;
     default:
       VT = MVT::v4i32;
       break;
@@ -16233,39 +15347,6 @@ bool PPCTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
     Info.flags = MachineMemOperand::MOStore;
     return true;
   }
-  case Intrinsic::ppc_qpx_qvstfda:
-  case Intrinsic::ppc_qpx_qvstfsa:
-  case Intrinsic::ppc_qpx_qvstfcda:
-  case Intrinsic::ppc_qpx_qvstfcsa:
-  case Intrinsic::ppc_qpx_qvstfiwa: {
-    EVT VT;
-    switch (Intrinsic) {
-    case Intrinsic::ppc_qpx_qvstfda:
-      VT = MVT::v4f64;
-      break;
-    case Intrinsic::ppc_qpx_qvstfsa:
-      VT = MVT::v4f32;
-      break;
-    case Intrinsic::ppc_qpx_qvstfcda:
-      VT = MVT::v2f64;
-      break;
-    case Intrinsic::ppc_qpx_qvstfcsa:
-      VT = MVT::v2f32;
-      break;
-    default:
-      VT = MVT::v4i32;
-      break;
-    }
-
-    Info.opc = ISD::INTRINSIC_VOID;
-    Info.memVT = VT;
-    Info.ptrVal = I.getArgOperand(1);
-    Info.offset = 0;
-    Info.size = VT.getStoreSize();
-    Info.align = Align(1);
-    Info.flags = MachineMemOperand::MOStore;
-    return true;
-  }
   default:
     break;
   }
@@ -16278,14 +15359,6 @@ bool PPCTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
 EVT PPCTargetLowering::getOptimalMemOpType(
     const MemOp &Op, const AttributeList &FuncAttributes) const {
   if (getTargetMachine().getOptLevel() != CodeGenOpt::None) {
-    // When expanding a memset, require at least two QPX instructions to cover
-    // the cost of loading the value to be stored from the constant pool.
-    if (Subtarget.hasQPX() && Op.size() >= 32 &&
-        (Op.isMemcpy() || Op.size() >= 64) && Op.isAligned(Align(32)) &&
-        !FuncAttributes.hasFnAttribute(Attribute::NoImplicitFloat)) {
-      return MVT::v4f64;
-    }
-
     // We should use Altivec/VSX loads and stores when available. For unaligned
     // addresses, unaligned VSX loads are only fast starting with the P8.
     if (Subtarget.hasAltivec() && Op.size() >= 16 &&
@@ -16504,7 +15577,7 @@ PPCTargetLowering::shouldExpandBuildVectorWithShuffles(
   if (VT == MVT::v2i64)
     return Subtarget.hasDirectMove(); // Don't need stack ops with direct moves
 
-  if (Subtarget.hasVSX() || Subtarget.hasQPX())
+  if (Subtarget.hasVSX())
     return true;
 
   return TargetLowering::shouldExpandBuildVectorWithShuffles(VT, DefinedValues);
@@ -16550,8 +15623,7 @@ SDValue PPCTargetLowering::getNegatedExpression(SDValue Op, SelectionDAG &DAG,
 
   switch (Opc) {
   case PPCISD::FNMSUB:
-    // TODO: QPX subtarget is deprecated. No transformation here.
-    if (!Op.hasOneUse() || !isTypeLegal(VT) || Subtarget.hasQPX())
+    if (!Op.hasOneUse() || !isTypeLegal(VT))
       break;
 
     const TargetOptions &Options = getTargetMachine().Options;
@@ -17032,8 +16104,7 @@ SDValue PPCTargetLowering::combineFMALike(SDNode *N,
   bool LegalOps = !DCI.isBeforeLegalizeOps();
   SDLoc Loc(N);
 
-  // TODO: QPX subtarget is deprecated. No transformation here.
-  if (Subtarget.hasQPX() || !isOperationLegal(ISD::FMA, VT))
+  if (!isOperationLegal(ISD::FMA, VT))
     return SDValue();
 
   // Allowing transformation to FNMSUB may change sign of zeroes when ab-c=0
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.h b/llvm/lib/Target/PowerPC/PPCISelLowering.h
index 768eaa43e0135..8cc42226d7f0b 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.h
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.h
@@ -427,22 +427,6 @@ namespace llvm {
     ///               => VABSDUW((XVNEGSP a), (XVNEGSP b))
     VABSD,
 
-    /// QVFPERM = This corresponds to the QPX qvfperm instruction.
-    QVFPERM,
-
-    /// QVGPCI = This corresponds to the QPX qvgpci instruction.
-    QVGPCI,
-
-    /// QVALIGNI = This corresponds to the QPX qvaligni instruction.
-    QVALIGNI,
-
-    /// QVESPLATI = This corresponds to the QPX qvesplati instruction.
-    QVESPLATI,
-
-    /// QBFLT = Access the underlying QPX floating-point boolean
-    /// representation.
-    QBFLT,
-
     /// FP_EXTEND_HALF(VECTOR, IDX) - Custom extend upper (IDX=0) half or
     /// lower (IDX=1) half of v4f32 to v2f64.
     FP_EXTEND_HALF,
@@ -519,10 +503,6 @@ namespace llvm {
     /// Store scalar integers from VSR.
     ST_VSR_SCAL_INT,
 
-    /// QBRC, CHAIN = QVLFSb CHAIN, Ptr
-    /// The 4xf32 load used for v4i1 constants.
-    QVLFSb,
-
     /// ATOMIC_CMP_SWAP - the exact same as the target-independent nodes
     /// except they ensure that the compare input is zero-extended for
     /// sub-word versions because the atomic loads zero-extend.
diff --git a/llvm/lib/Target/PowerPC/PPCInstrFormats.td b/llvm/lib/Target/PowerPC/PPCInstrFormats.td
index 632d4d9deb8a2..5ff5fc78326ba 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrFormats.td
+++ b/llvm/lib/Target/PowerPC/PPCInstrFormats.td
@@ -642,7 +642,6 @@ class XForm_17a<bits<6> opcode, bits<10> xo, dag OOL, dag IOL, string asmstr,
   let FRA = 0;
 }
 
-// Used for QPX
 class XForm_18<bits<6> opcode, bits<10> xo, dag OOL, dag IOL, string asmstr,
                InstrItinClass itin, list<dag> pattern>
          : I<opcode, OOL, IOL, asmstr, itin> {
@@ -1781,14 +1780,6 @@ class AForm_4<bits<6> opcode, bits<5> xo, dag OOL, dag IOL, string asmstr,
   let Inst{31}    = 0;
 }
 
-// Used for QPX
-class AForm_4a<bits<6> opcode, bits<5> xo, dag OOL, dag IOL, string asmstr,
-              InstrItinClass itin, list<dag> pattern>
-  : AForm_1<opcode, xo, OOL, IOL, asmstr, itin, pattern> {
-  let FRA = 0;
-  let FRC = 0;
-}
-
 // 1.7.13 M-Form
 class MForm_1<bits<6> opcode, dag OOL, dag IOL, string asmstr,
               InstrItinClass itin, list<dag> pattern>
@@ -2099,49 +2090,6 @@ class VX_RD5_RSp5_PS1_XO9<bits<9> xo, dag OOL, dag IOL, string asmstr,
   let Inst{23-31} = xo;
 }
 
-// Z23-Form (used by QPX)
-class Z23Form_1<bits<6> opcode, bits<8> xo, dag OOL, dag IOL, string asmstr, 
-              InstrItinClass itin, list<dag> pattern>
-         : I<opcode, OOL, IOL, asmstr, itin> {
-  bits<5> FRT;
-  bits<5> FRA;
-  bits<5> FRB;
-  bits<2> idx;
-
-  let Pattern = pattern;
-
-  bit RC = 0;    // set by isRecordForm
-
-  let Inst{6-10}  = FRT;
-  let Inst{11-15} = FRA;
-  let Inst{16-20} = FRB;
-  let Inst{21-22} = idx;
-  let Inst{23-30} = xo;
-  let Inst{31}    = RC;
-}
-
-class Z23Form_2<bits<6> opcode, bits<8> xo, dag OOL, dag IOL, string asmstr,
-              InstrItinClass itin, list<dag> pattern>
-  : Z23Form_1<opcode, xo, OOL, IOL, asmstr, itin, pattern> {
-  let FRB = 0;
-}
-
-class Z23Form_3<bits<6> opcode, bits<8> xo, dag OOL, dag IOL, string asmstr, 
-              InstrItinClass itin, list<dag> pattern>
-         : I<opcode, OOL, IOL, asmstr, itin> {
-  bits<5> FRT;
-  bits<12> idx;
-
-  let Pattern = pattern;
-
-  bit RC = 0;    // set by isRecordForm
-
-  let Inst{6-10}  = FRT;
-  let Inst{11-22} = idx;
-  let Inst{23-30} = xo;
-  let Inst{31}    = RC;
-}
-
 class Z23Form_8<bits<6> opcode, bits<8> xo, dag OOL, dag IOL, string asmstr,
               InstrItinClass itin, list<dag> pattern>
          : I<opcode, OOL, IOL, asmstr, itin> {
diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp b/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
index 99e25bb130ce4..d4c3c5f5504c7 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
+++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
@@ -259,16 +259,6 @@ bool PPCInstrInfo::isAssociativeAndCommutative(const MachineInstr &Inst) const {
   case PPC::XVMULDP:
   case PPC::XVMULSP:
   case PPC::XSMULSP:
-  // QPX Add:
-  case PPC::QVFADD:
-  case PPC::QVFADDS:
-  case PPC::QVFADDSs:
-  // QPX Multiply:
-  case PPC::QVFMUL:
-  case PPC::QVFMULS:
-  case PPC::QVFMULSs:
-    return Inst.getFlag(MachineInstr::MIFlag::FmReassoc) &&
-           Inst.getFlag(MachineInstr::MIFlag::FmNsz);
   // Fixed point:
   // Multiply:
   case PPC::MULHD:
@@ -300,9 +290,7 @@ static const uint16_t FMAOpIdxInfo[][5] = {
     {PPC::XVMADDADP, PPC::XVADDDP, PPC::XVMULDP, 1, 2},
     {PPC::XVMADDASP, PPC::XVADDSP, PPC::XVMULSP, 1, 2},
     {PPC::FMADD, PPC::FADD, PPC::FMUL, 3, 1},
-    {PPC::FMADDS, PPC::FADDS, PPC::FMULS, 3, 1},
-    {PPC::QVFMADDSs, PPC::QVFADDSs, PPC::QVFMULSs, 3, 1},
-    {PPC::QVFMADD, PPC::QVFADD, PPC::QVFMUL, 3, 1}};
+    {PPC::FMADDS, PPC::FADDS, PPC::FMULS, 3, 1}};
 
 // Check if an opcode is a FMA instruction. If it is, return the index in array
 // FMAOpIdxInfo. Otherwise, return -1.
@@ -666,7 +654,6 @@ bool PPCInstrInfo::isReallyTriviallyReMaterializable(const MachineInstr &MI,
   case PPC::LI8:
   case PPC::LIS:
   case PPC::LIS8:
-  case PPC::QVGPCI:
   case PPC::ADDIStocHA:
   case PPC::ADDIStocHA8:
   case PPC::ADDItocL:
@@ -1343,12 +1330,6 @@ void PPCInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
   else if (PPC::VSFRCRegClass.contains(DestReg, SrcReg) ||
            PPC::VSSRCRegClass.contains(DestReg, SrcReg))
     Opc = (Subtarget.hasP9Vector()) ? PPC::XSCPSGNDP : PPC::XXLORf;
-  else if (PPC::QFRCRegClass.contains(DestReg, SrcReg))
-    Opc = PPC::QVFMR;
-  else if (PPC::QSRCRegClass.contains(DestReg, SrcReg))
-    Opc = PPC::QVFMRs;
-  else if (PPC::QBRCRegClass.contains(DestReg, SrcReg))
-    Opc = PPC::QVFMRb;
   else if (PPC::CRBITRCRegClass.contains(DestReg, SrcReg))
     Opc = PPC::CROR;
   else if (PPC::SPERCRegClass.contains(DestReg, SrcReg))
@@ -1393,12 +1374,6 @@ static unsigned getSpillIndex(const TargetRegisterClass *RC) {
     OpcodeIndex = SOK_VectorFloat4Spill;
   } else if (PPC::VRSAVERCRegClass.hasSubClassEq(RC)) {
     OpcodeIndex = SOK_VRSaveSpill;
-  } else if (PPC::QFRCRegClass.hasSubClassEq(RC)) {
-    OpcodeIndex = SOK_QuadFloat8Spill;
-  } else if (PPC::QSRCRegClass.hasSubClassEq(RC)) {
-    OpcodeIndex = SOK_QuadFloat4Spill;
-  } else if (PPC::QBRCRegClass.hasSubClassEq(RC)) {
-    OpcodeIndex = SOK_QuadBitSpill;
   } else if (PPC::SPILLTOVSRRCRegClass.hasSubClassEq(RC)) {
     OpcodeIndex = SOK_SpillToVSR;
   } else {
diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.h b/llvm/lib/Target/PowerPC/PPCInstrInfo.h
index 43973c627fcf1..bdcfa76505daf 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrInfo.h
+++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.h
@@ -123,9 +123,6 @@ enum SpillOpcodeKey {
   SOK_VectorFloat8Spill,
   SOK_VectorFloat4Spill,
   SOK_VRSaveSpill,
-  SOK_QuadFloat8Spill,
-  SOK_QuadFloat4Spill,
-  SOK_QuadBitSpill,
   SOK_SpillToVSR,
   SOK_SPESpill,
   SOK_LastOpcodeSpill // This must be last on the enum.
@@ -136,32 +133,28 @@ enum SpillOpcodeKey {
   {                                                                            \
     PPC::LWZ, PPC::LD, PPC::LFD, PPC::LFS, PPC::RESTORE_CR,                    \
         PPC::RESTORE_CRBIT, PPC::LVX, PPC::LXVD2X, PPC::LXSDX, PPC::LXSSPX,    \
-        PPC::RESTORE_VRSAVE, PPC::QVLFDX, PPC::QVLFSXs, PPC::QVLFDXb,          \
-        PPC::SPILLTOVSR_LD, PPC::EVLDD                                         \
+        PPC::RESTORE_VRSAVE, PPC::SPILLTOVSR_LD, PPC::EVLDD                    \
   }
 
 #define Pwr9LoadOpcodes                                                        \
   {                                                                            \
     PPC::LWZ, PPC::LD, PPC::LFD, PPC::LFS, PPC::RESTORE_CR,                    \
         PPC::RESTORE_CRBIT, PPC::LVX, PPC::LXV, PPC::DFLOADf64,                \
-        PPC::DFLOADf32, PPC::RESTORE_VRSAVE, PPC::QVLFDX, PPC::QVLFSXs,        \
-        PPC::QVLFDXb, PPC::SPILLTOVSR_LD                                       \
+        PPC::DFLOADf32, PPC::RESTORE_VRSAVE, PPC::SPILLTOVSR_LD                \
   }
 
 #define Pwr8StoreOpcodes                                                       \
   {                                                                            \
     PPC::STW, PPC::STD, PPC::STFD, PPC::STFS, PPC::SPILL_CR, PPC::SPILL_CRBIT, \
         PPC::STVX, PPC::STXVD2X, PPC::STXSDX, PPC::STXSSPX, PPC::SPILL_VRSAVE, \
-        PPC::QVSTFDX, PPC::QVSTFSXs, PPC::QVSTFDXb, PPC::SPILLTOVSR_ST,        \
-        PPC::EVSTDD                                                            \
+        PPC::SPILLTOVSR_ST, PPC::EVSTDD                                        \
   }
 
 #define Pwr9StoreOpcodes                                                       \
   {                                                                            \
     PPC::STW, PPC::STD, PPC::STFD, PPC::STFS, PPC::SPILL_CR, PPC::SPILL_CRBIT, \
         PPC::STVX, PPC::STXV, PPC::DFSTOREf64, PPC::DFSTOREf32,                \
-        PPC::SPILL_VRSAVE, PPC::QVSTFDX, PPC::QVSTFSXs, PPC::QVSTFDXb,         \
-        PPC::SPILLTOVSR_ST                                                     \
+        PPC::SPILL_VRSAVE, PPC::SPILLTOVSR_ST                                  \
   }
 
 // Initialize arrays for load and store spill opcodes on supported subtargets.
@@ -273,10 +266,10 @@ class PPCInstrInfo : public PPCGenInstrInfo {
   }
 
   static bool isSameClassPhysRegCopy(unsigned Opcode) {
-    unsigned CopyOpcodes[] =
-      { PPC::OR, PPC::OR8, PPC::FMR, PPC::VOR, PPC::XXLOR, PPC::XXLORf,
-        PPC::XSCPSGNDP, PPC::MCRF, PPC::QVFMR, PPC::QVFMRs, PPC::QVFMRb,
-        PPC::CROR, PPC::EVOR, -1U };
+    unsigned CopyOpcodes[] = {PPC::OR,        PPC::OR8,   PPC::FMR,
+                              PPC::VOR,       PPC::XXLOR, PPC::XXLORf,
+                              PPC::XSCPSGNDP, PPC::MCRF,  PPC::CROR,
+                              PPC::EVOR,      -1U};
     for (int i = 0; CopyOpcodes[i] != -1U; i++)
       if (Opcode == CopyOpcodes[i])
         return true;
diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.td b/llvm/lib/Target/PowerPC/PPCInstrInfo.td
index c565758973bf5..83a434f5e793a 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrInfo.td
+++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.td
@@ -203,16 +203,6 @@ def PPCvecinsert : SDNode<"PPCISD::VECINSERT", SDT_PPCVecInsert, []>;
 def PPCxxpermdi  : SDNode<"PPCISD::XXPERMDI", SDT_PPCxxpermdi, []>;
 def PPCvecshl    : SDNode<"PPCISD::VECSHL", SDT_PPCVecShift, []>;
 
-def PPCqvfperm   : SDNode<"PPCISD::QVFPERM", SDT_PPCqvfperm, []>;
-def PPCqvgpci    : SDNode<"PPCISD::QVGPCI", SDT_PPCqvgpci, []>;
-def PPCqvaligni  : SDNode<"PPCISD::QVALIGNI", SDT_PPCqvaligni, []>;
-def PPCqvesplati : SDNode<"PPCISD::QVESPLATI", SDT_PPCqvesplati, []>;
-
-def PPCqbflt     : SDNode<"PPCISD::QBFLT", SDT_PPCqbflt, []>;
-
-def PPCqvlfsb    : SDNode<"PPCISD::QVLFSb", SDT_PPCqvlfsb,
-                          [SDNPHasChain, SDNPMayLoad]>;
-
 def PPCcmpb     : SDNode<"PPCISD::CMPB", SDTIntBinOp, []>;
 
 // These nodes represent the 32-bit PPC shifts that operate on 6-bit shift
@@ -3461,7 +3451,6 @@ include "PPCInstrAltivec.td"
 include "PPCInstrSPE.td"
 include "PPCInstr64Bit.td"
 include "PPCInstrVSX.td"
-include "PPCInstrQPX.td"
 include "PPCInstrHTM.td"
 
 def crnot : OutPatFrag<(ops node:$in),
diff --git a/llvm/lib/Target/PowerPC/PPCInstrQPX.td b/llvm/lib/Target/PowerPC/PPCInstrQPX.td
deleted file mode 100644
index 2265af2815cb5..0000000000000
--- a/llvm/lib/Target/PowerPC/PPCInstrQPX.td
+++ /dev/null
@@ -1,1212 +0,0 @@
-//===- PPCInstrQPX.td - The PowerPC QPX Extension --*- tablegen -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file describes the QPX extension to the PowerPC instruction set.
-// Reference:
-// Book Q: QPX Architecture Definition. IBM (as updated in) 2011.
-//
-//===----------------------------------------------------------------------===//
-
-def PPCRegQFRCAsmOperand : AsmOperandClass {
-  let Name = "RegQFRC"; let PredicateMethod = "isRegNumber";
-}
-def qfrc : RegisterOperand<QFRC> {
-  let ParserMatchClass = PPCRegQFRCAsmOperand;
-}
-def PPCRegQSRCAsmOperand : AsmOperandClass {
-  let Name = "RegQSRC"; let PredicateMethod = "isRegNumber";
-}
-def qsrc : RegisterOperand<QSRC> {
-  let ParserMatchClass = PPCRegQSRCAsmOperand;
-}
-def PPCRegQBRCAsmOperand : AsmOperandClass {
-  let Name = "RegQBRC"; let PredicateMethod = "isRegNumber";
-}
-def qbrc : RegisterOperand<QBRC> {
-  let ParserMatchClass = PPCRegQBRCAsmOperand;
-}
-
-//===----------------------------------------------------------------------===//
-// Helpers for defining instructions that directly correspond to intrinsics.
-
-// QPXA1_Int - A AForm_1 intrinsic definition.
-class QPXA1_Int<bits<6> opcode, bits<5> xo, string opc, Intrinsic IntID>
-  : AForm_1<opcode, xo, (outs qfrc:$FRT), (ins qfrc:$FRA, qfrc:$FRB, qfrc:$FRC),
-              !strconcat(opc, " $FRT, $FRA, $FRC, $FRB"), IIC_FPFused,
-                       [(set v4f64:$FRT, (IntID v4f64:$FRA, v4f64:$FRB, v4f64:$FRC))]>;
-// QPXA1s_Int - A AForm_1 intrinsic definition (simple instructions).
-class QPXA1s_Int<bits<6> opcode, bits<5> xo, string opc, Intrinsic IntID>
-  : AForm_1<opcode, xo, (outs qfrc:$FRT), (ins qfrc:$FRA, qfrc:$FRB, qfrc:$FRC),
-              !strconcat(opc, " $FRT, $FRA, $FRC, $FRB"), IIC_VecPerm,
-                       [(set v4f64:$FRT, (IntID v4f64:$FRA, v4f64:$FRB, v4f64:$FRC))]>;
-// QPXA2_Int - A AForm_2 intrinsic definition.
-class QPXA2_Int<bits<6> opcode, bits<5> xo, string opc, Intrinsic IntID>
-  : AForm_2<opcode, xo, (outs qfrc:$FRT), (ins qfrc:$FRA, qfrc:$FRB),
-              !strconcat(opc, " $FRT, $FRA, $FRB"), IIC_FPGeneral,
-                       [(set v4f64:$FRT, (IntID v4f64:$FRA, v4f64:$FRB))]>;
-// QPXA3_Int - A AForm_3 intrinsic definition.
-class QPXA3_Int<bits<6> opcode, bits<5> xo, string opc, Intrinsic IntID>
-  : AForm_3<opcode, xo, (outs qfrc:$FRT), (ins qfrc:$FRA, qfrc:$FRC),
-              !strconcat(opc, " $FRT, $FRA, $FRC"), IIC_FPGeneral,
-                       [(set v4f64:$FRT, (IntID v4f64:$FRA, v4f64:$FRC))]>;
-// QPXA4_Int - A AForm_4a intrinsic definition.
-class QPXA4_Int<bits<6> opcode, bits<5> xo, string opc, Intrinsic IntID>
-  : AForm_4a<opcode, xo, (outs qfrc:$FRT), (ins qfrc:$FRB),
-              !strconcat(opc, " $FRT, $FRB"), IIC_FPGeneral,
-                       [(set v4f64:$FRT, (IntID v4f64:$FRB))]>;
-// QPXX18_Int - A XForm_18 intrinsic definition.
-class QPXX18_Int<bits<6> opcode, bits<10> xo, string opc, Intrinsic IntID>
-  : XForm_18<opcode, xo, (outs qfrc:$FRT), (ins qfrc:$FRA, qfrc:$FRB),
-              !strconcat(opc, " $FRT, $FRA, $FRB"), IIC_FPCompare,
-                       [(set v4f64:$FRT, (IntID v4f64:$FRA, v4f64:$FRB))]>;
-// QPXX19_Int - A XForm_19 intrinsic definition.
-class QPXX19_Int<bits<6> opcode, bits<10> xo, string opc, Intrinsic IntID>
-  : XForm_19<opcode, xo, (outs qfrc:$FRT), (ins qfrc:$FRB),
-              !strconcat(opc, " $FRT, $FRB"), IIC_FPGeneral,
-                       [(set v4f64:$FRT, (IntID v4f64:$FRB))]>;
-
-//===----------------------------------------------------------------------===//
-// Pattern Frags.
-
-def extloadv4f32 : PatFrag<(ops node:$ptr), (extload node:$ptr), [{
-  return cast<LoadSDNode>(N)->getMemoryVT() == MVT::v4f32;
-}]>;
-
-def truncstorev4f32 : PatFrag<(ops node:$val, node:$ptr),
-                            (truncstore node:$val, node:$ptr), [{
-  return cast<StoreSDNode>(N)->getMemoryVT() == MVT::v4f32;
-}]>;
-def pre_truncstv4f32 : PatFrag<(ops node:$val, node:$base, node:$offset),
-                               (pre_truncst node:$val,
-                                            node:$base, node:$offset), [{
-  return cast<StoreSDNode>(N)->getMemoryVT() == MVT::v4f32;
-}]>;
-
-def fround_inexact : PatFrag<(ops node:$val), (fpround node:$val), [{
-  return cast<ConstantSDNode>(N->getOperand(1))->getZExtValue() == 0;
-}]>;
-
-def fround_exact : PatFrag<(ops node:$val), (fpround node:$val), [{
-  return cast<ConstantSDNode>(N->getOperand(1))->getZExtValue() == 1;
-}]>;
-
-let FastIselShouldIgnore = 1 in // FastIsel should ignore all u12 instrs.
-  def u12 : ImmLeaf<i32, [{ return (Imm & 0xFFF) == Imm; }]>;
-
-//===----------------------------------------------------------------------===//
-// Instruction Definitions.
-
-def HasQPX : Predicate<"Subtarget->hasQPX()">;
-let Predicates = [HasQPX] in {
-let DecoderNamespace = "QPX" in {
-let hasSideEffects = 0 in { // QPX instructions don't have side effects.
-let Uses = [RM] in {
-  // Add Instructions
-  let isCommutable = 1 in {
-    def QVFADD : AForm_2<4, 21,
-                        (outs qfrc:$FRT), (ins qfrc:$FRA, qfrc:$FRB),
-                        "qvfadd $FRT, $FRA, $FRB", IIC_FPGeneral,
-                        [(set v4f64:$FRT, (fadd v4f64:$FRA, v4f64:$FRB))]>;
-    let isCodeGenOnly = 1 in
-      def QVFADDS : QPXA2_Int<0, 21, "qvfadds", int_ppc_qpx_qvfadds>;
-    def QVFADDSs : AForm_2<0, 21,
-                          (outs qsrc:$FRT), (ins qsrc:$FRA, qsrc:$FRB),
-                          "qvfadds $FRT, $FRA, $FRB", IIC_FPGeneral,
-                          [(set v4f32:$FRT, (fadd v4f32:$FRA, v4f32:$FRB))]>;
-  }
-  def QVFSUB : AForm_2<4, 20,
-                      (outs qfrc:$FRT), (ins qfrc:$FRA, qfrc:$FRB),
-                      "qvfsub $FRT, $FRA, $FRB", IIC_FPGeneral,
-                      [(set v4f64:$FRT, (fsub v4f64:$FRA, v4f64:$FRB))]>;
-  let isCodeGenOnly = 1 in
-    def QVFSUBS : QPXA2_Int<0, 20, "qvfsubs", int_ppc_qpx_qvfsubs>;
-  def QVFSUBSs : AForm_2<0, 20,
-                        (outs qsrc:$FRT), (ins qsrc:$FRA, qsrc:$FRB),
-                        "qvfsubs $FRT, $FRA, $FRB", IIC_FPGeneral,
-                        [(set v4f32:$FRT, (fsub v4f32:$FRA, v4f32:$FRB))]>;
-
-  // Estimate Instructions
-  def QVFRE : AForm_4a<4, 24, (outs qfrc:$FRT), (ins qfrc:$FRB),
-                       "qvfre $FRT, $FRB", IIC_FPGeneral,
-                       [(set v4f64:$FRT, (PPCfre v4f64:$FRB))]>;
-  def QVFRES : QPXA4_Int<0, 24, "qvfres", int_ppc_qpx_qvfres>;
-  let isCodeGenOnly = 1 in
-  def QVFRESs : AForm_4a<0, 24, (outs qsrc:$FRT), (ins qsrc:$FRB),
-                         "qvfres $FRT, $FRB", IIC_FPGeneral,
-                         [(set v4f32:$FRT, (PPCfre v4f32:$FRB))]>;
-
-  def QVFRSQRTE : AForm_4a<4, 26, (outs qfrc:$FRT), (ins qfrc:$FRB),
-                           "qvfrsqrte $FRT, $FRB", IIC_FPGeneral,
-                           [(set v4f64:$FRT, (PPCfrsqrte v4f64:$FRB))]>;
-  def QVFRSQRTES : QPXA4_Int<0, 26, "qvfrsqrtes", int_ppc_qpx_qvfrsqrtes>;
-  let isCodeGenOnly = 1 in
-  def QVFRSQRTESs : AForm_4a<0, 26, (outs qsrc:$FRT), (ins qsrc:$FRB),
-                             "qvfrsqrtes $FRT, $FRB", IIC_FPGeneral,
-                             [(set v4f32:$FRT, (PPCfrsqrte v4f32:$FRB))]>;
-
-  // Multiply Instructions
-  let isCommutable = 1 in {
-    def QVFMUL : AForm_3<4, 25,
-                        (outs qfrc:$FRT), (ins qfrc:$FRA, qfrc:$FRC),
-                        "qvfmul $FRT, $FRA, $FRC", IIC_FPGeneral,
-                        [(set v4f64:$FRT, (fmul v4f64:$FRA, v4f64:$FRC))]>;
-    let isCodeGenOnly = 1 in
-      def QVFMULS : QPXA3_Int<0, 25, "qvfmuls", int_ppc_qpx_qvfmuls>;
-    def QVFMULSs : AForm_3<0, 25,
-                          (outs qsrc:$FRT), (ins qsrc:$FRA, qsrc:$FRC),
-                          "qvfmuls $FRT, $FRA, $FRC", IIC_FPGeneral,
-                          [(set v4f32:$FRT, (fmul v4f32:$FRA, v4f32:$FRC))]>;
-  }
-  def QVFXMUL : QPXA3_Int<4, 17, "qvfxmul", int_ppc_qpx_qvfxmul>;
-  def QVFXMULS : QPXA3_Int<0, 17, "qvfxmuls", int_ppc_qpx_qvfxmuls>;
-
-  // Multiply-add instructions
-  def QVFMADD : AForm_1<4, 29,
-                      (outs qfrc:$FRT), (ins qfrc:$FRA, qfrc:$FRC, qfrc:$FRB),
-                      "qvfmadd $FRT, $FRA, $FRC, $FRB", IIC_FPFused,
-                      [(set v4f64:$FRT, (fma v4f64:$FRA, v4f64:$FRC, v4f64:$FRB))]>;
-  let isCodeGenOnly = 1 in
-    def QVFMADDS : QPXA1_Int<0, 29, "qvfmadds", int_ppc_qpx_qvfmadds>;
-  def QVFMADDSs : AForm_1<0, 29,
-                        (outs qsrc:$FRT), (ins qsrc:$FRA, qsrc:$FRC, qsrc:$FRB),
-                        "qvfmadds $FRT, $FRA, $FRC, $FRB", IIC_FPFused,
-                        [(set v4f32:$FRT, (fma v4f32:$FRA, v4f32:$FRC, v4f32:$FRB))]>;
-  def QVFNMADD : AForm_1<4, 31,
-                      (outs qfrc:$FRT), (ins qfrc:$FRA, qfrc:$FRC, qfrc:$FRB),
-                      "qvfnmadd $FRT, $FRA, $FRC, $FRB", IIC_FPFused,
-                      [(set v4f64:$FRT, (fneg (fma v4f64:$FRA, v4f64:$FRC,
-                                                   v4f64:$FRB)))]>;
-  let isCodeGenOnly = 1 in
-    def QVFNMADDS : QPXA1_Int<0, 31, "qvfnmadds", int_ppc_qpx_qvfnmadds>;
-  def QVFNMADDSs : AForm_1<0, 31,
-                        (outs qsrc:$FRT), (ins qsrc:$FRA, qsrc:$FRC, qsrc:$FRB),
-                        "qvfnmadds $FRT, $FRA, $FRC, $FRB", IIC_FPFused,
-                        [(set v4f32:$FRT, (fneg (fma v4f32:$FRA, v4f32:$FRC,
-                                                     v4f32:$FRB)))]>;
-  def QVFMSUB : AForm_1<4, 28,
-                      (outs qfrc:$FRT), (ins qfrc:$FRA, qfrc:$FRC, qfrc:$FRB),
-                      "qvfmsub $FRT, $FRA, $FRC, $FRB", IIC_FPFused,
-                      [(set v4f64:$FRT, (fma v4f64:$FRA, v4f64:$FRC,
-                                             (fneg v4f64:$FRB)))]>;
-  let isCodeGenOnly = 1 in
-    def QVFMSUBS : QPXA1_Int<0, 28, "qvfmsubs", int_ppc_qpx_qvfmsubs>;
-  def QVFMSUBSs : AForm_1<0, 28,
-                      (outs qsrc:$FRT), (ins qsrc:$FRA, qsrc:$FRC, qsrc:$FRB),
-                      "qvfmsubs $FRT, $FRA, $FRC, $FRB", IIC_FPFused,
-                      [(set v4f32:$FRT, (fma v4f32:$FRA, v4f32:$FRC,
-                                             (fneg v4f32:$FRB)))]>;
-  def QVFNMSUB : AForm_1<4, 30,
-                      (outs qfrc:$FRT), (ins qfrc:$FRA, qfrc:$FRC, qfrc:$FRB),
-                      "qvfnmsub $FRT, $FRA, $FRC, $FRB", IIC_FPFused,
-                      [(set v4f64:$FRT, (fneg (fma v4f64:$FRA, v4f64:$FRC,
-                                              (fneg v4f64:$FRB))))]>;
-  let isCodeGenOnly = 1 in
-    def QVFNMSUBS : QPXA1_Int<0, 30, "qvfnmsubs", int_ppc_qpx_qvfnmsubs>;
-  def QVFNMSUBSs : AForm_1<0, 30,
-                      (outs qsrc:$FRT), (ins qsrc:$FRA, qsrc:$FRC, qsrc:$FRB),
-                      "qvfnmsubs $FRT, $FRA, $FRC, $FRB", IIC_FPFused,
-                      [(set v4f32:$FRT, (fneg (fma v4f32:$FRA, v4f32:$FRC,
-                                              (fneg v4f32:$FRB))))]>;
-  def QVFXMADD : QPXA1_Int<4, 9, "qvfxmadd", int_ppc_qpx_qvfxmadd>;
-  def QVFXMADDS : QPXA1_Int<0, 9, "qvfxmadds", int_ppc_qpx_qvfxmadds>;
-  def QVFXXNPMADD : QPXA1_Int<4, 11, "qvfxxnpmadd", int_ppc_qpx_qvfxxnpmadd>;
-  def QVFXXNPMADDS : QPXA1_Int<0, 11, "qvfxxnpmadds", int_ppc_qpx_qvfxxnpmadds>;
-  def QVFXXCPNMADD : QPXA1_Int<4, 3, "qvfxxcpnmadd", int_ppc_qpx_qvfxxcpnmadd>;
-  def QVFXXCPNMADDS : QPXA1_Int<0, 3, "qvfxxcpnmadds", int_ppc_qpx_qvfxxcpnmadds>;
-  def QVFXXMADD : QPXA1_Int<4, 1, "qvfxxmadd", int_ppc_qpx_qvfxxmadd>;
-  def QVFXXMADDS : QPXA1_Int<0, 1, "qvfxxmadds", int_ppc_qpx_qvfxxmadds>;
-
-  // Select Instruction
-  let isCodeGenOnly = 1 in
-    def QVFSEL : QPXA1s_Int<4, 23, "qvfsel", int_ppc_qpx_qvfsel>;
-  def QVFSELb : AForm_1<4, 23, (outs qfrc:$FRT),
-                        (ins qbrc:$FRA, qfrc:$FRB, qfrc:$FRC),
-                        "qvfsel $FRT, $FRA, $FRC, $FRB", IIC_VecPerm,
-                        [(set v4f64:$FRT, (vselect v4i1:$FRA,
-                                                   v4f64:$FRC, v4f64:$FRB))]>;
-  let isCodeGenOnly = 1 in
-  def QVFSELbs : AForm_1<4, 23, (outs qsrc:$FRT),
-                        (ins qbrc:$FRA, qsrc:$FRB, qsrc:$FRC),
-                        "qvfsel $FRT, $FRA, $FRC, $FRB", IIC_VecPerm,
-                        [(set v4f32:$FRT, (vselect v4i1:$FRA,
-                                                   v4f32:$FRC, v4f32:$FRB))]>;
-  let isCodeGenOnly = 1 in
-  def QVFSELbb: AForm_1<4, 23, (outs qbrc:$FRT),
-                        (ins qbrc:$FRA, qbrc:$FRB, qbrc:$FRC),
-                        "qvfsel $FRT, $FRA, $FRC, $FRB", IIC_VecPerm,
-                        [(set v4i1:$FRT, (vselect v4i1:$FRA,
-                                                  v4i1:$FRC, v4i1:$FRB))]>;
-
-  // SELECT_CC_* - Used to implement the SELECT_CC DAG operation.  Expanded after
-  // instruction selection into a branch sequence.
-  def SELECT_CC_QFRC: PPCCustomInserterPseudo<(outs qfrc:$dst), (ins crrc:$cond, qfrc:$T, qfrc:$F,
-                              i32imm:$BROPC), "#SELECT_CC_QFRC",
-                              []>;
-  def SELECT_CC_QSRC: PPCCustomInserterPseudo<(outs qsrc:$dst), (ins crrc:$cond, qsrc:$T, qsrc:$F,
-                              i32imm:$BROPC), "#SELECT_CC_QSRC",
-                              []>;
-  def SELECT_CC_QBRC: PPCCustomInserterPseudo<(outs qbrc:$dst), (ins crrc:$cond, qbrc:$T, qbrc:$F,
-                              i32imm:$BROPC), "#SELECT_CC_QBRC",
-                              []>;
-
-  // SELECT_* pseudo instructions, like SELECT_CC_* but taking condition
-  // register bit directly.
-  def SELECT_QFRC: PPCCustomInserterPseudo<(outs qfrc:$dst), (ins crbitrc:$cond,
-                          qfrc:$T, qfrc:$F), "#SELECT_QFRC",
-                          [(set v4f64:$dst,
-                                (select i1:$cond, v4f64:$T, v4f64:$F))]>;
-  def SELECT_QSRC: PPCCustomInserterPseudo<(outs qsrc:$dst), (ins crbitrc:$cond,
-                          qsrc:$T, qsrc:$F), "#SELECT_QSRC",
-                          [(set v4f32:$dst,
-                                (select i1:$cond, v4f32:$T, v4f32:$F))]>;
-  def SELECT_QBRC: PPCCustomInserterPseudo<(outs qbrc:$dst), (ins crbitrc:$cond,
-                          qbrc:$T, qbrc:$F), "#SELECT_QBRC",
-                          [(set v4i1:$dst,
-                                (select i1:$cond, v4i1:$T, v4i1:$F))]>;
-
-  // Convert and Round Instructions
-  def QVFCTID : QPXX19_Int<4, 814, "qvfctid", int_ppc_qpx_qvfctid>;
-  let isCodeGenOnly = 1 in
-    def QVFCTIDb : XForm_19<4, 814, (outs qbrc:$FRT), (ins qbrc:$FRB),
-                            "qvfctid $FRT, $FRB", IIC_FPGeneral, []>;
-
-  def QVFCTIDU : QPXX19_Int<4, 942, "qvfctidu", int_ppc_qpx_qvfctidu>;
-  def QVFCTIDZ : QPXX19_Int<4, 815, "qvfctidz", int_ppc_qpx_qvfctidz>;
-  def QVFCTIDUZ : QPXX19_Int<4, 943, "qvfctiduz", int_ppc_qpx_qvfctiduz>;
-  def QVFCTIW : QPXX19_Int<4, 14, "qvfctiw", int_ppc_qpx_qvfctiw>;
-  def QVFCTIWU : QPXX19_Int<4, 142, "qvfctiwu", int_ppc_qpx_qvfctiwu>;
-  def QVFCTIWZ : QPXX19_Int<4, 15, "qvfctiwz", int_ppc_qpx_qvfctiwz>;
-  def QVFCTIWUZ : QPXX19_Int<4, 143, "qvfctiwuz", int_ppc_qpx_qvfctiwuz>;
-  def QVFCFID : QPXX19_Int<4, 846, "qvfcfid", int_ppc_qpx_qvfcfid>;
-  let isCodeGenOnly = 1 in
-    def QVFCFIDb : XForm_19<4, 846, (outs qbrc:$FRT), (ins qbrc:$FRB),
-                            "qvfcfid $FRT, $FRB", IIC_FPGeneral, []>;
-
-  def QVFCFIDU : QPXX19_Int<4, 974, "qvfcfidu", int_ppc_qpx_qvfcfidu>;
-  def QVFCFIDS : QPXX19_Int<0, 846, "qvfcfids", int_ppc_qpx_qvfcfids>;
-  def QVFCFIDUS : QPXX19_Int<0, 974, "qvfcfidus", int_ppc_qpx_qvfcfidus>;
-
-  let isCodeGenOnly = 1 in
-    def QVFRSP : QPXX19_Int<4, 12, "qvfrsp", int_ppc_qpx_qvfrsp>;
-  def QVFRSPs : XForm_19<4, 12,
-                      (outs qsrc:$FRT), (ins qfrc:$FRB),
-                      "qvfrsp $FRT, $FRB", IIC_FPGeneral,
-                      [(set v4f32:$FRT, (fround_inexact v4f64:$FRB))]>;
-
-  def QVFRIZ : XForm_19<4, 424, (outs qfrc:$FRT), (ins qfrc:$FRB),
-                        "qvfriz $FRT, $FRB", IIC_FPGeneral,
-                        [(set v4f64:$FRT, (ftrunc v4f64:$FRB))]>;
-  let isCodeGenOnly = 1 in
-    def QVFRIZs : XForm_19<4, 424, (outs qsrc:$FRT), (ins qsrc:$FRB),
-                           "qvfriz $FRT, $FRB", IIC_FPGeneral,
-                           [(set v4f32:$FRT, (ftrunc v4f32:$FRB))]>;
-
-  def QVFRIN : XForm_19<4, 392, (outs qfrc:$FRT), (ins qfrc:$FRB),
-                        "qvfrin $FRT, $FRB", IIC_FPGeneral,
-                        [(set v4f64:$FRT, (fround v4f64:$FRB))]>;
-  let isCodeGenOnly = 1 in
-    def QVFRINs : XForm_19<4, 392, (outs qsrc:$FRT), (ins qsrc:$FRB),
-                           "qvfrin $FRT, $FRB", IIC_FPGeneral,
-                           [(set v4f32:$FRT, (fround v4f32:$FRB))]>;
-
-  def QVFRIP : XForm_19<4, 456, (outs qfrc:$FRT), (ins qfrc:$FRB),
-                        "qvfrip $FRT, $FRB", IIC_FPGeneral,
-                        [(set v4f64:$FRT, (fceil v4f64:$FRB))]>;
-  let isCodeGenOnly = 1 in
-    def QVFRIPs : XForm_19<4, 456, (outs qsrc:$FRT), (ins qsrc:$FRB),
-                           "qvfrip $FRT, $FRB", IIC_FPGeneral,
-                           [(set v4f32:$FRT, (fceil v4f32:$FRB))]>;
-
-  def QVFRIM : XForm_19<4, 488, (outs qfrc:$FRT), (ins qfrc:$FRB),
-                        "qvfrim $FRT, $FRB", IIC_FPGeneral,
-                        [(set v4f64:$FRT, (ffloor v4f64:$FRB))]>;
-  let isCodeGenOnly = 1 in
-    def QVFRIMs : XForm_19<4, 488, (outs qsrc:$FRT), (ins qsrc:$FRB),
-                           "qvfrim $FRT, $FRB", IIC_FPGeneral,
-                           [(set v4f32:$FRT, (ffloor v4f32:$FRB))]>;
-
-  // Move Instructions
-  def QVFMR : XForm_19<4, 72,
-                      (outs qfrc:$FRT), (ins qfrc:$FRB),
-                      "qvfmr $FRT, $FRB", IIC_VecPerm,
-                      [/* (set v4f64:$FRT, v4f64:$FRB) */]>;
-  let isCodeGenOnly = 1 in {
-    def QVFMRs : XForm_19<4, 72,
-                         (outs qsrc:$FRT), (ins qsrc:$FRB),
-                         "qvfmr $FRT, $FRB", IIC_VecPerm,
-                         [/* (set v4f32:$FRT, v4f32:$FRB) */]>;
-    def QVFMRb : XForm_19<4, 72,
-                         (outs qbrc:$FRT), (ins qbrc:$FRB),
-                         "qvfmr $FRT, $FRB", IIC_VecPerm,
-                         [/* (set v4i1:$FRT, v4i1:$FRB) */]>;
-  }
-  def QVFNEG : XForm_19<4, 40,
-                      (outs qfrc:$FRT), (ins qfrc:$FRB),
-                      "qvfneg $FRT, $FRB", IIC_VecPerm,
-                      [(set v4f64:$FRT, (fneg v4f64:$FRB))]>;
-  let isCodeGenOnly = 1 in
-    def QVFNEGs : XForm_19<4, 40,
-                         (outs qsrc:$FRT), (ins qsrc:$FRB),
-                         "qvfneg $FRT, $FRB", IIC_VecPerm,
-                         [(set v4f32:$FRT, (fneg v4f32:$FRB))]>;
-  def QVFABS : XForm_19<4, 264,
-                      (outs qfrc:$FRT), (ins qfrc:$FRB),
-                      "qvfabs $FRT, $FRB", IIC_VecPerm,
-                      [(set v4f64:$FRT, (fabs v4f64:$FRB))]>;
-  let isCodeGenOnly = 1 in
-    def QVFABSs : XForm_19<4, 264,
-                         (outs qsrc:$FRT), (ins qsrc:$FRB),
-                         "qvfabs $FRT, $FRB", IIC_VecPerm,
-                         [(set v4f32:$FRT, (fabs v4f32:$FRB))]>;
-  def QVFNABS : XForm_19<4, 136,
-                      (outs qfrc:$FRT), (ins qfrc:$FRB),
-                      "qvfnabs $FRT, $FRB", IIC_VecPerm,
-                      [(set v4f64:$FRT, (fneg (fabs v4f64:$FRB)))]>;
-  let isCodeGenOnly = 1 in
-    def QVFNABSs : XForm_19<4, 136,
-                         (outs qsrc:$FRT), (ins qsrc:$FRB),
-                         "qvfnabs $FRT, $FRB", IIC_VecPerm,
-                         [(set v4f32:$FRT, (fneg (fabs v4f32:$FRB)))]>;
-  def QVFCPSGN : XForm_18<4, 8,
-                      (outs qfrc:$FRT), (ins qfrc:$FRA, qfrc:$FRB),
-                      "qvfcpsgn $FRT, $FRA, $FRB", IIC_VecPerm,
-                      [(set v4f64:$FRT, (fcopysign v4f64:$FRB, v4f64:$FRA))]>;
-  let isCodeGenOnly = 1 in
-    def QVFCPSGNs : XForm_18<4, 8,
-                         (outs qsrc:$FRT), (ins qsrc:$FRA, qsrc:$FRB),
-                         "qvfcpsgn $FRT, $FRA, $FRB", IIC_VecPerm,
-                         [(set v4f32:$FRT, (fcopysign v4f32:$FRB, v4f32:$FRA))]>;
-
-  def QVALIGNI : Z23Form_1<4, 5,
-                      (outs qfrc:$FRT), (ins qfrc:$FRA, qfrc:$FRB, u2imm:$idx),
-                      "qvaligni $FRT, $FRA, $FRB, $idx", IIC_VecPerm,
-                      [(set v4f64:$FRT,
-                            (PPCqvaligni v4f64:$FRA, v4f64:$FRB,
-                                         (i32 imm:$idx)))]>;
-  let isCodeGenOnly = 1 in
-     def QVALIGNIs : Z23Form_1<4, 5,
-                         (outs qsrc:$FRT), (ins qsrc:$FRA, qsrc:$FRB, u2imm:$idx),
-                         "qvaligni $FRT, $FRA, $FRB, $idx", IIC_VecPerm,
-                         [(set v4f32:$FRT,
-                               (PPCqvaligni v4f32:$FRA, v4f32:$FRB,
-                                            (i32 imm:$idx)))]>;
-  let isCodeGenOnly = 1 in
-     def QVALIGNIb : Z23Form_1<4, 5,
-                         (outs qbrc:$FRT), (ins qbrc:$FRA, qbrc:$FRB, u2imm:$idx),
-                         "qvaligni $FRT, $FRA, $FRB, $idx", IIC_VecPerm,
-                         [(set v4i1:$FRT,
-                               (PPCqvaligni v4i1:$FRA, v4i1:$FRB,
-                                            (i32 imm:$idx)))]>;
-
-  def QVESPLATI : Z23Form_2<4, 37,
-                      (outs qfrc:$FRT), (ins qfrc:$FRA, u2imm:$idx),
-                      "qvesplati $FRT, $FRA, $idx", IIC_VecPerm,
-                      [(set v4f64:$FRT,
-                            (PPCqvesplati v4f64:$FRA, (i32 imm:$idx)))]>;
-  let isCodeGenOnly = 1 in
-     def QVESPLATIs : Z23Form_2<4, 37,
-                         (outs qsrc:$FRT), (ins qsrc:$FRA, u2imm:$idx),
-                         "qvesplati $FRT, $FRA, $idx", IIC_VecPerm,
-                         [(set v4f32:$FRT,
-                               (PPCqvesplati v4f32:$FRA, (i32 imm:$idx)))]>;
-  let isCodeGenOnly = 1 in
-     def QVESPLATIb : Z23Form_2<4, 37,
-                         (outs qbrc:$FRT), (ins qbrc:$FRA, u2imm:$idx),
-                         "qvesplati $FRT, $FRA, $idx", IIC_VecPerm,
-                         [(set v4i1:$FRT,
-                               (PPCqvesplati v4i1:$FRA, (i32 imm:$idx)))]>;
-
-  def QVFPERM : AForm_1<4, 6,
-                      (outs qfrc:$FRT), (ins qfrc:$FRA, qfrc:$FRB, qfrc:$FRC),
-                      "qvfperm $FRT, $FRA, $FRB, $FRC", IIC_VecPerm,
-                      [(set v4f64:$FRT,
-                            (PPCqvfperm v4f64:$FRA, v4f64:$FRB, v4f64:$FRC))]>;
-  let isCodeGenOnly = 1 in
-     def QVFPERMs : AForm_1<4, 6,
-                         (outs qsrc:$FRT), (ins qsrc:$FRA, qsrc:$FRB, qfrc:$FRC),
-                         "qvfperm $FRT, $FRA, $FRB, $FRC", IIC_VecPerm,
-                         [(set v4f32:$FRT,
-                               (PPCqvfperm v4f32:$FRA, v4f32:$FRB, v4f64:$FRC))]>;
-
-  let isReMaterializable = 1, isAsCheapAsAMove = 1 in
-  def QVGPCI : Z23Form_3<4, 133,
-                      (outs qfrc:$FRT), (ins u12imm:$idx),
-                      "qvgpci $FRT, $idx", IIC_VecPerm,
-                      [(set v4f64:$FRT, (PPCqvgpci (u12:$idx)))]>;
-
-  // Compare Instruction
-  let isCodeGenOnly = 1 in
-    def QVFTSTNAN : QPXX18_Int<4, 64, "qvftstnan", int_ppc_qpx_qvftstnan>;
-  def QVFTSTNANb : XForm_18<4, 64, (outs qbrc:$FRT), (ins qfrc:$FRA, qfrc:$FRB),
-                           "qvftstnan $FRT, $FRA, $FRB", IIC_FPCompare,
-                           [(set v4i1:$FRT,
-                                 (setcc v4f64:$FRA, v4f64:$FRB, SETUO))]>;
-  let isCodeGenOnly = 1 in
-  def QVFTSTNANbs : XForm_18<4, 64, (outs qbrc:$FRT), (ins qsrc:$FRA, qsrc:$FRB),
-                            "qvftstnan $FRT, $FRA, $FRB", IIC_FPCompare,
-                            [(set v4i1:$FRT,
-                                  (setcc v4f32:$FRA, v4f32:$FRB, SETUO))]>;
-  let isCodeGenOnly = 1 in
-    def QVFCMPLT : QPXX18_Int<4, 96, "qvfcmplt", int_ppc_qpx_qvfcmplt>;
-  def QVFCMPLTb : XForm_18<4, 96, (outs qbrc:$FRT), (ins qfrc:$FRA, qfrc:$FRB),
-                           "qvfcmplt $FRT, $FRA, $FRB", IIC_FPCompare,
-                           [(set v4i1:$FRT,
-                                 (setcc v4f64:$FRA, v4f64:$FRB, SETOLT))]>;
-  let isCodeGenOnly = 1 in
-  def QVFCMPLTbs : XForm_18<4, 96, (outs qbrc:$FRT), (ins qsrc:$FRA, qsrc:$FRB),
-                            "qvfcmplt $FRT, $FRA, $FRB", IIC_FPCompare,
-                            [(set v4i1:$FRT,
-                                  (setcc v4f32:$FRA, v4f32:$FRB, SETOLT))]>;
-  let isCodeGenOnly = 1 in
-    def QVFCMPGT : QPXX18_Int<4, 32, "qvfcmpgt", int_ppc_qpx_qvfcmpgt>;
-  def QVFCMPGTb : XForm_18<4, 32, (outs qbrc:$FRT), (ins qfrc:$FRA, qfrc:$FRB),
-                           "qvfcmpgt $FRT, $FRA, $FRB", IIC_FPCompare,
-                           [(set v4i1:$FRT,
-                                 (setcc v4f64:$FRA, v4f64:$FRB, SETOGT))]>;
-  let isCodeGenOnly = 1 in
-  def QVFCMPGTbs : XForm_18<4, 32, (outs qbrc:$FRT), (ins qsrc:$FRA, qsrc:$FRB),
-                            "qvfcmpgt $FRT, $FRA, $FRB", IIC_FPCompare,
-                            [(set v4i1:$FRT,
-                                  (setcc v4f32:$FRA, v4f32:$FRB, SETOGT))]>;
-  let isCodeGenOnly = 1 in
-    def QVFCMPEQ : QPXX18_Int<4, 0, "qvfcmpeq", int_ppc_qpx_qvfcmpeq>;
-  def QVFCMPEQb : XForm_18<4, 0, (outs qbrc:$FRT), (ins qfrc:$FRA, qfrc:$FRB),
-                           "qvfcmpeq $FRT, $FRA, $FRB", IIC_FPCompare,
-                           [(set v4i1:$FRT,
-                                 (setcc v4f64:$FRA, v4f64:$FRB, SETOEQ))]>;
-  let isCodeGenOnly = 1 in
-  def QVFCMPEQbs : XForm_18<4, 0, (outs qbrc:$FRT), (ins qsrc:$FRA, qsrc:$FRB),
-                            "qvfcmpeq $FRT, $FRA, $FRB", IIC_FPCompare,
-                            [(set v4i1:$FRT,
-                                  (setcc v4f32:$FRA, v4f32:$FRB, SETOEQ))]>;
-
-  let isCodeGenOnly = 1 in
-  def QVFLOGICAL : XForm_20<4, 4,
-                      (outs qfrc:$FRT), (ins qfrc:$FRA, qfrc:$FRB, u12imm:$tttt),
-                      "qvflogical $FRT, $FRA, $FRB, $tttt", IIC_VecPerm, []>;
-  def QVFLOGICALb : XForm_20<4, 4,
-                      (outs qbrc:$FRT), (ins qbrc:$FRA, qbrc:$FRB, u12imm:$tttt),
-                      "qvflogical $FRT, $FRA, $FRB, $tttt", IIC_VecPerm, []>;
-  let isCodeGenOnly = 1 in
-  def QVFLOGICALs : XForm_20<4, 4,
-                      (outs qbrc:$FRT), (ins qbrc:$FRA, qbrc:$FRB, u12imm:$tttt),
-                      "qvflogical $FRT, $FRA, $FRB, $tttt", IIC_VecPerm, []>;
-
-  // Load indexed instructions
-  let mayLoad = 1 in {
-    def QVLFDX : XForm_1_memOp<31, 583,
-                              (outs qfrc:$FRT), (ins memrr:$src),
-                              "qvlfdx $FRT, $src", IIC_LdStLFD,
-                              [(set v4f64:$FRT, (load xoaddr:$src))]>;
-    let isCodeGenOnly = 1 in
-    def QVLFDXb : XForm_1_memOp<31, 583,
-                                (outs qbrc:$FRT), (ins memrr:$src),
-                                "qvlfdx $FRT, $src", IIC_LdStLFD, []>;
-
-    let RC = 1 in
-    def QVLFDXA : XForm_1<31, 583,
-                        (outs qfrc:$FRT), (ins memrr:$src),
-                        "qvlfdxa $FRT, $src", IIC_LdStLFD, []>;
-
-    def QVLFDUX : XForm_1<31, 615,
-                        (outs qfrc:$FRT, ptr_rc_nor0:$ea_result),
-                        (ins memrr:$src),
-                        "qvlfdux $FRT, $src", IIC_LdStLFDU, []>,
-                        RegConstraint<"$src.ptrreg = $ea_result">,
-                        NoEncode<"$ea_result">;
-    let RC = 1 in
-    def QVLFDUXA : XForm_1<31, 615,
-                        (outs qfrc:$FRT), (ins memrr:$src),
-                        "qvlfduxa $FRT, $src", IIC_LdStLFD, []>;
-
-    def QVLFSX : XForm_1_memOp<31, 519,
-                              (outs qfrc:$FRT), (ins memrr:$src),
-                              "qvlfsx $FRT, $src", IIC_LdStLFD,
-                              [(set v4f64:$FRT, (extloadv4f32 xoaddr:$src))]>;
-
-    let isCodeGenOnly = 1 in
-    def QVLFSXb : XForm_1<31, 519,
-                        (outs qbrc:$FRT), (ins memrr:$src),
-                        "qvlfsx $FRT, $src", IIC_LdStLFD,
-                        [(set v4i1:$FRT, (PPCqvlfsb xoaddr:$src))]>;
-    let isCodeGenOnly = 1 in
-    def QVLFSXs : XForm_1_memOp<31, 519,
-                                (outs qsrc:$FRT), (ins memrr:$src),
-                                "qvlfsx $FRT, $src", IIC_LdStLFD,
-                                [(set v4f32:$FRT, (load xoaddr:$src))]>;
-
-    let RC = 1 in
-    def QVLFSXA : XForm_1<31, 519,
-                        (outs qfrc:$FRT), (ins memrr:$src),
-                        "qvlfsxa $FRT, $src", IIC_LdStLFD, []>;
-
-    def QVLFSUX : XForm_1<31, 551,
-                        (outs qsrc:$FRT, ptr_rc_nor0:$ea_result),
-                        (ins memrr:$src),
-                        "qvlfsux $FRT, $src", IIC_LdStLFDU, []>,
-                        RegConstraint<"$src.ptrreg = $ea_result">,
-                        NoEncode<"$ea_result">;
-
-    let RC = 1 in
-    def QVLFSUXA : XForm_1<31, 551,
-                        (outs qfrc:$FRT), (ins memrr:$src),
-                        "qvlfsuxa $FRT, $src", IIC_LdStLFD, []>;
-
-    def QVLFCDX : XForm_1<31, 71,
-                        (outs qfrc:$FRT), (ins memrr:$src),
-                        "qvlfcdx $FRT, $src", IIC_LdStLFD, []>;
-    let RC = 1 in
-    def QVLFCDXA : XForm_1<31, 71,
-                        (outs qfrc:$FRT), (ins memrr:$src),
-                        "qvlfcdxa $FRT, $src", IIC_LdStLFD, []>;
-
-    def QVLFCDUX : XForm_1<31, 103,
-                        (outs qfrc:$FRT), (ins memrr:$src),
-                        "qvlfcdux $FRT, $src", IIC_LdStLFD, []>;
-    let RC = 1 in
-    def QVLFCDUXA : XForm_1<31, 103,
-                        (outs qfrc:$FRT), (ins memrr:$src),
-                        "qvlfcduxa $FRT, $src", IIC_LdStLFD, []>;
-
-    def QVLFCSX : XForm_1<31, 7,
-                        (outs qfrc:$FRT), (ins memrr:$src),
-                        "qvlfcsx $FRT, $src", IIC_LdStLFD, []>;
-    let isCodeGenOnly = 1 in
-    def QVLFCSXs : XForm_1<31, 7,
-                         (outs qsrc:$FRT), (ins memrr:$src),
-                         "qvlfcsx $FRT, $src", IIC_LdStLFD, []>;
-
-    let RC = 1 in
-    def QVLFCSXA : XForm_1<31, 7,
-                        (outs qfrc:$FRT), (ins memrr:$src),
-                        "qvlfcsxa $FRT, $src", IIC_LdStLFD, []>;
-
-    def QVLFCSUX : XForm_1<31, 39,
-                        (outs qfrc:$FRT), (ins memrr:$src),
-                        "qvlfcsux $FRT, $src", IIC_LdStLFD, []>;
-    let RC = 1 in
-    def QVLFCSUXA : XForm_1<31, 39,
-                        (outs qfrc:$FRT), (ins memrr:$src),
-                        "qvlfcsuxa $FRT, $src", IIC_LdStLFD, []>;
-
-    def QVLFIWAX : XForm_1<31, 871,
-                        (outs qfrc:$FRT), (ins memrr:$src),
-                        "qvlfiwax $FRT, $src", IIC_LdStLFD, []>;
-    let RC = 1 in
-    def QVLFIWAXA : XForm_1<31, 871,
-                        (outs qfrc:$FRT), (ins memrr:$src),
-                        "qvlfiwaxa $FRT, $src", IIC_LdStLFD, []>;
-
-    def QVLFIWZX : XForm_1<31, 839,
-                        (outs qfrc:$FRT), (ins memrr:$src),
-                        "qvlfiwzx $FRT, $src", IIC_LdStLFD, []>;
-    let RC = 1 in
-    def QVLFIWZXA : XForm_1<31, 839,
-                        (outs qfrc:$FRT), (ins memrr:$src),
-                        "qvlfiwzxa $FRT, $src", IIC_LdStLFD, []>;
-  }
-
-
-  def QVLPCLDX : XForm_1<31, 582,
-                      (outs qfrc:$FRT), (ins memrr:$src),
-                      "qvlpcldx $FRT, $src", IIC_LdStLFD, []>;
-  def QVLPCLSX : XForm_1<31, 518,
-                      (outs qfrc:$FRT), (ins memrr:$src),
-                      "qvlpclsx $FRT, $src", IIC_LdStLFD, []>;
-  let isCodeGenOnly = 1 in
-    def QVLPCLSXint : XForm_11<31, 518,
-                              (outs qfrc:$FRT), (ins G8RC:$src),
-                              "qvlpclsx $FRT, 0, $src", IIC_LdStLFD, []>;
-  def QVLPCRDX : XForm_1<31, 70,
-                      (outs qfrc:$FRT), (ins memrr:$src),
-                      "qvlpcrdx $FRT, $src", IIC_LdStLFD, []>;
-  def QVLPCRSX : XForm_1<31, 6,
-                      (outs qfrc:$FRT), (ins memrr:$src),
-                      "qvlpcrsx $FRT, $src", IIC_LdStLFD, []>;
-
-  // Store indexed instructions
-  let mayStore = 1 in {
-    def QVSTFDX : XForm_8_memOp<31, 711,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfdx $FRT, $dst", IIC_LdStSTFD,
-                        [(store qfrc:$FRT, xoaddr:$dst)]>;
-    let isCodeGenOnly = 1 in
-    def QVSTFDXb : XForm_8_memOp<31, 711,
-                        (outs), (ins qbrc:$FRT, memrr:$dst),
-                        "qvstfdx $FRT, $dst", IIC_LdStSTFD, []>;
-
-    let RC = 1 in
-    def QVSTFDXA : XForm_8<31, 711,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfdxa $FRT, $dst", IIC_LdStSTFD, []>;
-
-    def QVSTFDUX : XForm_8<31, 743, (outs ptr_rc_nor0:$ea_res),
-                           (ins qfrc:$FRT, memrr:$dst),
-                           "qvstfdux $FRT, $dst", IIC_LdStSTFDU, []>,
-                           RegConstraint<"$dst.ptrreg = $ea_res">,
-                           NoEncode<"$ea_res">;
-
-    let RC = 1 in
-    def QVSTFDUXA : XForm_8<31, 743,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfduxa $FRT, $dst", IIC_LdStSTFD, []>;
-
-    def QVSTFDXI : XForm_8<31, 709,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfdxi $FRT, $dst", IIC_LdStSTFD, []>;
-    let RC = 1 in
-    def QVSTFDXIA : XForm_8<31, 709,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfdxia $FRT, $dst", IIC_LdStSTFD, []>;
-
-    def QVSTFDUXI : XForm_8<31, 741,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfduxi $FRT, $dst", IIC_LdStSTFD, []>;
-    let RC = 1 in
-    def QVSTFDUXIA : XForm_8<31, 741,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfduxia $FRT, $dst", IIC_LdStSTFD, []>;
-
-    def QVSTFSX : XForm_8_memOp<31, 647,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfsx $FRT, $dst", IIC_LdStSTFD,
-                        [(truncstorev4f32 qfrc:$FRT, xoaddr:$dst)]>;
-    let isCodeGenOnly = 1 in
-    def QVSTFSXs : XForm_8_memOp<31, 647,
-                         (outs), (ins qsrc:$FRT, memrr:$dst),
-                         "qvstfsx $FRT, $dst", IIC_LdStSTFD,
-                         [(store qsrc:$FRT, xoaddr:$dst)]>;
-
-    let RC = 1 in
-    def QVSTFSXA : XForm_8<31, 647,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfsxa $FRT, $dst", IIC_LdStSTFD, []>;
-
-    def QVSTFSUX : XForm_8<31, 679, (outs ptr_rc_nor0:$ea_res),
-                           (ins qsrc:$FRT, memrr:$dst),
-                           "qvstfsux $FRT, $dst", IIC_LdStSTFDU, []>,
-                           RegConstraint<"$dst.ptrreg = $ea_res">,
-                           NoEncode<"$ea_res">;
-    let isCodeGenOnly = 1 in
-    def QVSTFSUXs: XForm_8<31, 679, (outs ptr_rc_nor0:$ea_res),
-                           (ins qfrc:$FRT, memrr:$dst),
-                           "qvstfsux $FRT, $dst", IIC_LdStSTFDU, []>,
-                           RegConstraint<"$dst.ptrreg = $ea_res">,
-                           NoEncode<"$ea_res">;
-
-    let RC = 1 in
-    def QVSTFSUXA : XForm_8<31, 679,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfsuxa $FRT, $dst", IIC_LdStSTFD, []>;
-
-    def QVSTFSXI : XForm_8<31, 645,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfsxi $FRT, $dst", IIC_LdStSTFD, []>;
-    let RC = 1 in
-    def QVSTFSXIA : XForm_8<31, 645,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfsxia $FRT, $dst", IIC_LdStSTFD, []>;
-
-    def QVSTFSUXI : XForm_8<31, 677,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfsuxi $FRT, $dst", IIC_LdStSTFD, []>;
-    let RC = 1 in
-    def QVSTFSUXIA : XForm_8<31, 677,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfsuxia $FRT, $dst", IIC_LdStSTFD, []>;
-
-    def QVSTFCDX : XForm_8<31, 199,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfcdx $FRT, $dst", IIC_LdStSTFD, []>;
-    let RC = 1 in
-    def QVSTFCDXA : XForm_8<31, 199,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfcdxa $FRT, $dst", IIC_LdStSTFD, []>;
-
-    def QVSTFCSX : XForm_8<31, 135,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfcsx $FRT, $dst", IIC_LdStSTFD, []>;
-    let isCodeGenOnly = 1 in
-    def QVSTFCSXs : XForm_8<31, 135,
-                         (outs), (ins qsrc:$FRT, memrr:$dst),
-                         "qvstfcsx $FRT, $dst", IIC_LdStSTFD, []>;
-
-    let RC = 1 in
-    def QVSTFCSXA : XForm_8<31, 135,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfcsxa $FRT, $dst", IIC_LdStSTFD, []>;
-
-    def QVSTFCDUX : XForm_8<31, 231,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfcdux $FRT, $dst", IIC_LdStSTFD, []>;
-    let RC = 1 in
-    def QVSTFCDUXA : XForm_8<31, 231,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfcduxa $FRT, $dst", IIC_LdStSTFD, []>;
-
-    def QVSTFCSUX : XForm_8<31, 167,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfcsux $FRT, $dst", IIC_LdStSTFD, []>;
-    let RC = 1 in
-    def QVSTFCSUXA : XForm_8<31, 167,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfcsuxa $FRT, $dst", IIC_LdStSTFD, []>;
-
-    def QVSTFCDXI : XForm_8<31, 197,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfcdxi $FRT, $dst", IIC_LdStSTFD, []>;
-    let RC = 1 in
-    def QVSTFCDXIA : XForm_8<31, 197,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfcdxia $FRT, $dst", IIC_LdStSTFD, []>;
-
-    def QVSTFCSXI : XForm_8<31, 133,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfcsxi $FRT, $dst", IIC_LdStSTFD, []>;
-    let RC = 1 in
-    def QVSTFCSXIA : XForm_8<31, 133,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfcsxia $FRT, $dst", IIC_LdStSTFD, []>;
-
-    def QVSTFCDUXI : XForm_8<31, 229,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfcduxi $FRT, $dst", IIC_LdStSTFD, []>;
-    let RC = 1 in
-    def QVSTFCDUXIA : XForm_8<31, 229,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfcduxia $FRT, $dst", IIC_LdStSTFD, []>;
-
-    def QVSTFCSUXI : XForm_8<31, 165,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfcsuxi $FRT, $dst", IIC_LdStSTFD, []>;
-    let RC = 1 in
-    def QVSTFCSUXIA : XForm_8<31, 165,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfcsuxia $FRT, $dst", IIC_LdStSTFD, []>;
-
-    def QVSTFIWX : XForm_8<31, 967,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfiwx $FRT, $dst", IIC_LdStSTFD, []>;
-    let RC = 1 in
-    def QVSTFIWXA : XForm_8<31, 967,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfiwxa $FRT, $dst", IIC_LdStSTFD, []>;
-  }
-}
-
-} // neverHasSideEffects
-}
-
-def : InstAlias<"qvfclr $FRT",
-                (QVFLOGICALb qbrc:$FRT, qbrc:$FRT, qbrc:$FRT, 0)>;
-def : InstAlias<"qvfand $FRT, $FRA, $FRB",
-                (QVFLOGICALb qbrc:$FRT, qbrc:$FRA, qbrc:$FRB, 1)>;
-def : InstAlias<"qvfandc $FRT, $FRA, $FRB",
-                (QVFLOGICALb qbrc:$FRT, qbrc:$FRA, qbrc:$FRB, 4)>;
-def : InstAlias<"qvfctfb $FRT, $FRA",
-                (QVFLOGICALb qbrc:$FRT, qbrc:$FRA, qbrc:$FRA, 5)>;
-def : InstAlias<"qvfxor $FRT, $FRA, $FRB",
-                (QVFLOGICALb qbrc:$FRT, qbrc:$FRA, qbrc:$FRB, 6)>;
-def : InstAlias<"qvfor $FRT, $FRA, $FRB",
-                (QVFLOGICALb qbrc:$FRT, qbrc:$FRA, qbrc:$FRB, 7)>;
-def : InstAlias<"qvfnor $FRT, $FRA, $FRB",
-                (QVFLOGICALb qbrc:$FRT, qbrc:$FRA, qbrc:$FRB, 8)>;
-def : InstAlias<"qvfequ $FRT, $FRA, $FRB",
-                (QVFLOGICALb qbrc:$FRT, qbrc:$FRA, qbrc:$FRB, 9)>;
-def : InstAlias<"qvfnot $FRT, $FRA",
-                (QVFLOGICALb qbrc:$FRT, qbrc:$FRA, qbrc:$FRA, 10)>;
-def : InstAlias<"qvforc $FRT, $FRA, $FRB",
-                (QVFLOGICALb qbrc:$FRT, qbrc:$FRA, qbrc:$FRB, 13)>;
-def : InstAlias<"qvfnand $FRT, $FRA, $FRB",
-                (QVFLOGICALb qbrc:$FRT, qbrc:$FRA, qbrc:$FRB, 14)>;
-def : InstAlias<"qvfset $FRT",
-                (QVFLOGICALb qbrc:$FRT, qbrc:$FRT, qbrc:$FRT, 15)>;
-
-//===----------------------------------------------------------------------===//
-// Additional QPX Patterns
-//
-
-def : Pat<(v4f64 (scalar_to_vector f64:$A)),
-          (INSERT_SUBREG (v4f64 (IMPLICIT_DEF)), $A, sub_64)>;
-def : Pat<(v4f32 (scalar_to_vector f32:$A)),
-          (INSERT_SUBREG (v4f32 (IMPLICIT_DEF)), $A, sub_64)>;
-
-def : Pat<(f64 (extractelt v4f64:$S, 0)),
-          (EXTRACT_SUBREG $S, sub_64)>;
-def : Pat<(f32 (extractelt v4f32:$S, 0)),
-          (EXTRACT_SUBREG $S, sub_64)>;
-
-def : Pat<(f64 (extractelt v4f64:$S, 1)),
-          (EXTRACT_SUBREG (QVESPLATI $S, 1), sub_64)>;
-def : Pat<(f64 (extractelt v4f64:$S, 2)),
-          (EXTRACT_SUBREG (QVESPLATI $S, 2), sub_64)>;
-def : Pat<(f64 (extractelt v4f64:$S, 3)),
-          (EXTRACT_SUBREG (QVESPLATI $S, 3), sub_64)>;
-
-def : Pat<(f32 (extractelt v4f32:$S, 1)),
-          (EXTRACT_SUBREG (QVESPLATIs $S, 1), sub_64)>;
-def : Pat<(f32 (extractelt v4f32:$S, 2)),
-          (EXTRACT_SUBREG (QVESPLATIs $S, 2), sub_64)>;
-def : Pat<(f32 (extractelt v4f32:$S, 3)),
-          (EXTRACT_SUBREG (QVESPLATIs $S, 3), sub_64)>;
-
-def : Pat<(f64 (extractelt v4f64:$S, i64:$F)),
-          (EXTRACT_SUBREG (QVFPERM $S, $S,
-                                   (QVLPCLSXint (RLDICR $F, 2,
-                                                        /* 63-2 = */ 61))),
-                          sub_64)>;
-def : Pat<(f32 (extractelt v4f32:$S, i64:$F)),
-          (EXTRACT_SUBREG (QVFPERMs $S, $S,
-                                    (QVLPCLSXint (RLDICR $F, 2,
-                                                         /* 63-2 = */ 61))),
-                          sub_64)>;
-
-def : Pat<(int_ppc_qpx_qvfperm v4f64:$A, v4f64:$B, v4f64:$C),
-          (QVFPERM $A, $B, $C)>;
-
-def : Pat<(int_ppc_qpx_qvfcpsgn v4f64:$A, v4f64:$B),
-          (QVFCPSGN $A, $B)>;
-
-// FCOPYSIGN's operand types need not agree.
-def : Pat<(fcopysign v4f64:$frB, v4f32:$frA),
-          (QVFCPSGN (COPY_TO_REGCLASS $frA, QFRC), $frB)>;
-def : Pat<(fcopysign QSRC:$frB, QFRC:$frA),
-          (QVFCPSGNs (COPY_TO_REGCLASS $frA, QSRC), $frB)>;
-
-def : Pat<(int_ppc_qpx_qvfneg v4f64:$A), (QVFNEG $A)>;
-def : Pat<(int_ppc_qpx_qvfabs v4f64:$A), (QVFABS $A)>;
-def : Pat<(int_ppc_qpx_qvfnabs v4f64:$A), (QVFNABS $A)>;
-
-def : Pat<(int_ppc_qpx_qvfriz v4f64:$A), (QVFRIZ $A)>;
-def : Pat<(int_ppc_qpx_qvfrin v4f64:$A), (QVFRIN $A)>;
-def : Pat<(int_ppc_qpx_qvfrip v4f64:$A), (QVFRIP $A)>;
-def : Pat<(int_ppc_qpx_qvfrim v4f64:$A), (QVFRIM $A)>;
-
-def : Pat<(int_ppc_qpx_qvfre v4f64:$A), (QVFRE $A)>;
-def : Pat<(int_ppc_qpx_qvfrsqrte v4f64:$A), (QVFRSQRTE $A)>;
-
-def : Pat<(int_ppc_qpx_qvfadd v4f64:$A, v4f64:$B),
-          (QVFADD $A, $B)>;
-def : Pat<(int_ppc_qpx_qvfsub v4f64:$A, v4f64:$B),
-          (QVFSUB $A, $B)>;
-def : Pat<(int_ppc_qpx_qvfmul v4f64:$A, v4f64:$B),
-          (QVFMUL $A, $B)>;
-
-// Additional QVFNMSUB patterns: -a*c + b == -(a*c - b)
-def : Pat<(fma (fneg v4f64:$A), v4f64:$C, v4f64:$B),
-          (QVFNMSUB $A, $C, $B)>;
-def : Pat<(fma v4f64:$A, (fneg v4f64:$C), v4f64:$B),
-          (QVFNMSUB $A, $C, $B)>;
-def : Pat<(fma (fneg v4f32:$A), v4f32:$C, v4f32:$B),
-          (QVFNMSUBSs $A, $C, $B)>;
-def : Pat<(fma v4f32:$A, (fneg v4f32:$C), v4f32:$B),
-          (QVFNMSUBSs $A, $C, $B)>;
-
-def : Pat<(int_ppc_qpx_qvfmadd v4f64:$A, v4f64:$B, v4f64:$C),
-          (QVFMADD $A, $B, $C)>;
-def : Pat<(int_ppc_qpx_qvfnmadd v4f64:$A, v4f64:$B, v4f64:$C),
-          (QVFNMADD $A, $B, $C)>;
-def : Pat<(int_ppc_qpx_qvfmsub v4f64:$A, v4f64:$B, v4f64:$C),
-          (QVFMSUB $A, $B, $C)>;
-def : Pat<(int_ppc_qpx_qvfnmsub v4f64:$A, v4f64:$B, v4f64:$C),
-          (QVFNMSUB $A, $B, $C)>;
-
-def : Pat<(int_ppc_qpx_qvlfd xoaddr:$src),
-          (QVLFDX xoaddr:$src)>;
-def : Pat<(int_ppc_qpx_qvlfda xoaddr:$src),
-          (QVLFDXA xoaddr:$src)>;
-def : Pat<(int_ppc_qpx_qvlfs xoaddr:$src),
-          (QVLFSX xoaddr:$src)>;
-def : Pat<(int_ppc_qpx_qvlfsa xoaddr:$src),
-          (QVLFSXA xoaddr:$src)>;
-def : Pat<(int_ppc_qpx_qvlfcda xoaddr:$src),
-          (QVLFCDXA xoaddr:$src)>;
-def : Pat<(int_ppc_qpx_qvlfcd xoaddr:$src),
-          (QVLFCDX xoaddr:$src)>;
-def : Pat<(int_ppc_qpx_qvlfcsa xoaddr:$src),
-          (QVLFCSXA xoaddr:$src)>;
-def : Pat<(int_ppc_qpx_qvlfcs xoaddr:$src),
-          (QVLFCSX xoaddr:$src)>;
-def : Pat<(int_ppc_qpx_qvlfda xoaddr:$src),
-          (QVLFDXA xoaddr:$src)>;
-def : Pat<(int_ppc_qpx_qvlfiwaa xoaddr:$src),
-          (QVLFIWAXA xoaddr:$src)>;
-def : Pat<(int_ppc_qpx_qvlfiwa xoaddr:$src),
-          (QVLFIWAX xoaddr:$src)>;
-def : Pat<(int_ppc_qpx_qvlfiwza xoaddr:$src),
-          (QVLFIWZXA xoaddr:$src)>;
-def : Pat<(int_ppc_qpx_qvlfiwz xoaddr:$src),
-          (QVLFIWZX xoaddr:$src)>;
-def : Pat<(int_ppc_qpx_qvlfsa xoaddr:$src),
-          (QVLFSXA xoaddr:$src)>;
-def : Pat<(int_ppc_qpx_qvlpcld xoaddr:$src),
-          (QVLPCLDX xoaddr:$src)>;
-def : Pat<(int_ppc_qpx_qvlpcls xoaddr:$src),
-          (QVLPCLSX xoaddr:$src)>;
-def : Pat<(int_ppc_qpx_qvlpcrd xoaddr:$src),
-          (QVLPCRDX xoaddr:$src)>;
-def : Pat<(int_ppc_qpx_qvlpcrs xoaddr:$src),
-          (QVLPCRSX xoaddr:$src)>;
-
-def : Pat<(int_ppc_qpx_qvstfd v4f64:$T, xoaddr:$dst),
-          (QVSTFDX $T, xoaddr:$dst)>;
-def : Pat<(int_ppc_qpx_qvstfs v4f64:$T, xoaddr:$dst),
-          (QVSTFSX $T, xoaddr:$dst)>;
-def : Pat<(int_ppc_qpx_qvstfcda v4f64:$T, xoaddr:$dst),
-          (QVSTFCDXA $T, xoaddr:$dst)>;
-def : Pat<(int_ppc_qpx_qvstfcd v4f64:$T, xoaddr:$dst),
-          (QVSTFCDX $T, xoaddr:$dst)>;
-def : Pat<(int_ppc_qpx_qvstfcsa v4f64:$T, xoaddr:$dst),
-          (QVSTFCSXA $T, xoaddr:$dst)>;
-def : Pat<(int_ppc_qpx_qvstfcs v4f64:$T, xoaddr:$dst),
-          (QVSTFCSX $T, xoaddr:$dst)>;
-def : Pat<(int_ppc_qpx_qvstfda v4f64:$T, xoaddr:$dst),
-          (QVSTFDXA $T, xoaddr:$dst)>;
-def : Pat<(int_ppc_qpx_qvstfiwa v4f64:$T, xoaddr:$dst),
-          (QVSTFIWXA $T, xoaddr:$dst)>;
-def : Pat<(int_ppc_qpx_qvstfiw v4f64:$T, xoaddr:$dst),
-          (QVSTFIWX $T, xoaddr:$dst)>;
-def : Pat<(int_ppc_qpx_qvstfsa v4f64:$T, xoaddr:$dst),
-          (QVSTFSXA $T, xoaddr:$dst)>;
-
-def : Pat<(pre_store v4f64:$rS, iPTR:$ptrreg, iPTR:$ptroff),
-          (QVSTFDUX $rS, $ptrreg, $ptroff)>;
-def : Pat<(pre_store v4f32:$rS, iPTR:$ptrreg, iPTR:$ptroff),
-          (QVSTFSUX $rS, $ptrreg, $ptroff)>;
-def : Pat<(pre_truncstv4f32 v4f64:$rS, iPTR:$ptrreg, iPTR:$ptroff),
-          (QVSTFSUXs $rS, $ptrreg, $ptroff)>;
-
-def : Pat<(int_ppc_qpx_qvflogical  v4f64:$A, v4f64:$B, (i32 imm:$idx)),
-          (QVFLOGICAL $A, $B, imm:$idx)>;
-def : Pat<(int_ppc_qpx_qvgpci (u12:$idx)),
-          (QVGPCI imm:$idx)>;
-
-def : Pat<(setcc v4f64:$FRA, v4f64:$FRB, SETOGE),
-          (QVFLOGICALb (QVFCMPLTb $FRA, $FRB),
-                       (QVFTSTNANb $FRA, $FRB), (i32 8))>;
-def : Pat<(setcc v4f64:$FRA, v4f64:$FRB, SETOLE),
-          (QVFLOGICALb (QVFCMPGTb $FRA, $FRB),
-                       (QVFTSTNANb $FRA, $FRB), (i32 8))>;
-def : Pat<(setcc v4f64:$FRA, v4f64:$FRB, SETONE),
-          (QVFLOGICALb (QVFCMPEQb $FRA, $FRB),
-                       (QVFTSTNANb $FRA, $FRB), (i32 8))>;
-def : Pat<(setcc v4f64:$FRA, v4f64:$FRB, SETO),
-          (QVFLOGICALb (QVFTSTNANb $FRA, $FRB),
-                       (QVFTSTNANb $FRA, $FRB), (i32 10))>;
-def : Pat<(setcc v4f64:$FRA, v4f64:$FRB, SETUEQ),
-          (QVFLOGICALb (QVFCMPEQb $FRA, $FRB),
-                       (QVFTSTNANb $FRA, $FRB), (i32 7))>;
-def : Pat<(setcc v4f64:$FRA, v4f64:$FRB, SETUGT),
-          (QVFLOGICALb (QVFCMPGTb $FRA, $FRB),
-                       (QVFTSTNANb $FRA, $FRB), (i32 7))>;
-def : Pat<(setcc v4f64:$FRA, v4f64:$FRB, SETUGE),
-          (QVFLOGICALb (QVFTSTNANb $FRA, $FRB),
-                       (QVFCMPLTb $FRA, $FRB), (i32 13))>;
-def : Pat<(setcc v4f64:$FRA, v4f64:$FRB, SETULT),
-          (QVFLOGICALb (QVFCMPLTb $FRA, $FRB),
-                       (QVFTSTNANb $FRA, $FRB), (i32 7))>;
-def : Pat<(setcc v4f64:$FRA, v4f64:$FRB, SETULE),
-          (QVFLOGICALb (QVFTSTNANb $FRA, $FRB),
-                       (QVFCMPGTb $FRA, $FRB), (i32 13))>;
-def : Pat<(setcc v4f64:$FRA, v4f64:$FRB, SETUNE),
-          (QVFLOGICALb (QVFTSTNANb $FRA, $FRB),
-                       (QVFCMPEQb $FRA, $FRB), (i32 13))>;
-
-def : Pat<(setcc v4f64:$FRA, v4f64:$FRB, SETEQ),
-          (QVFCMPEQb $FRA, $FRB)>;
-def : Pat<(setcc v4f64:$FRA, v4f64:$FRB, SETGT),
-          (QVFCMPGTb $FRA, $FRB)>;
-def : Pat<(setcc v4f64:$FRA, v4f64:$FRB, SETGE),
-          (QVFLOGICALb (QVFCMPLTb $FRA, $FRB),
-                       (QVFCMPLTb $FRA, $FRB), (i32 10))>;
-def : Pat<(setcc v4f64:$FRA, v4f64:$FRB, SETLT),
-          (QVFCMPLTb $FRA, $FRB)>;
-def : Pat<(setcc v4f64:$FRA, v4f64:$FRB, SETLE),
-          (QVFLOGICALb (QVFCMPGTb $FRA, $FRB),
-                       (QVFCMPGTb $FRA, $FRB), (i32 10))>;
-def : Pat<(setcc v4f64:$FRA, v4f64:$FRB, SETNE),
-          (QVFLOGICALb (QVFCMPEQb $FRA, $FRB),
-                       (QVFCMPEQb $FRA, $FRB), (i32 10))>;
-
-def : Pat<(setcc v4f32:$FRA, v4f32:$FRB, SETOGE),
-          (QVFLOGICALb (QVFCMPLTbs $FRA, $FRB),
-                       (QVFTSTNANbs $FRA, $FRB), (i32 8))>;
-def : Pat<(setcc v4f32:$FRA, v4f32:$FRB, SETOLE),
-          (QVFLOGICALb (QVFCMPGTbs $FRA, $FRB),
-                       (QVFTSTNANbs $FRA, $FRB), (i32 8))>;
-def : Pat<(setcc v4f32:$FRA, v4f32:$FRB, SETONE),
-          (QVFLOGICALb (QVFCMPEQbs $FRA, $FRB),
-                       (QVFTSTNANbs $FRA, $FRB), (i32 8))>;
-def : Pat<(setcc v4f32:$FRA, v4f32:$FRB, SETO),
-          (QVFLOGICALb (QVFTSTNANbs $FRA, $FRB),
-                       (QVFTSTNANbs $FRA, $FRB), (i32 10))>;
-def : Pat<(setcc v4f32:$FRA, v4f32:$FRB, SETUEQ),
-          (QVFLOGICALb (QVFCMPEQbs $FRA, $FRB),
-                       (QVFTSTNANbs $FRA, $FRB), (i32 7))>;
-def : Pat<(setcc v4f32:$FRA, v4f32:$FRB, SETUGT),
-          (QVFLOGICALb (QVFCMPGTbs $FRA, $FRB),
-                       (QVFTSTNANbs $FRA, $FRB), (i32 7))>;
-def : Pat<(setcc v4f32:$FRA, v4f32:$FRB, SETUGE),
-          (QVFLOGICALb (QVFTSTNANbs $FRA, $FRB),
-                       (QVFCMPLTbs $FRA, $FRB), (i32 13))>;
-def : Pat<(setcc v4f32:$FRA, v4f32:$FRB, SETULT),
-          (QVFLOGICALb (QVFCMPLTbs $FRA, $FRB),
-                       (QVFTSTNANbs $FRA, $FRB), (i32 7))>;
-def : Pat<(setcc v4f32:$FRA, v4f32:$FRB, SETULE),
-          (QVFLOGICALb (QVFTSTNANbs $FRA, $FRB),
-                       (QVFCMPGTbs $FRA, $FRB), (i32 13))>;
-def : Pat<(setcc v4f32:$FRA, v4f32:$FRB, SETUNE),
-          (QVFLOGICALb (QVFTSTNANbs $FRA, $FRB),
-                       (QVFCMPEQbs $FRA, $FRB), (i32 13))>;
-
-def : Pat<(setcc v4f32:$FRA, v4f32:$FRB, SETEQ),
-          (QVFCMPEQbs $FRA, $FRB)>;
-def : Pat<(setcc v4f32:$FRA, v4f32:$FRB, SETGT),
-          (QVFCMPGTbs $FRA, $FRB)>;
-def : Pat<(setcc v4f32:$FRA, v4f32:$FRB, SETGE),
-          (QVFLOGICALb (QVFCMPLTbs $FRA, $FRB),
-                       (QVFCMPLTbs $FRA, $FRB), (i32 10))>;
-def : Pat<(setcc v4f32:$FRA, v4f32:$FRB, SETLT),
-          (QVFCMPLTbs $FRA, $FRB)>;
-def : Pat<(setcc v4f32:$FRA, v4f32:$FRB, SETLE),
-          (QVFLOGICALb (QVFCMPGTbs $FRA, $FRB),
-                       (QVFCMPGTbs $FRA, $FRB), (i32 10))>;
-def : Pat<(setcc v4f32:$FRA, v4f32:$FRB, SETNE),
-          (QVFLOGICALb (QVFCMPEQbs $FRA, $FRB),
-                       (QVFCMPEQbs $FRA, $FRB), (i32 10))>;
-
-def : Pat<(and v4i1:$FRA, (not v4i1:$FRB)),
-          (QVFLOGICALb $FRA, $FRB, (i32 4))>;
-def : Pat<(not (or v4i1:$FRA, v4i1:$FRB)),
-          (QVFLOGICALb $FRA, $FRB, (i32 8))>;
-def : Pat<(not (xor v4i1:$FRA, v4i1:$FRB)),
-          (QVFLOGICALb $FRA, $FRB, (i32 9))>;
-def : Pat<(or v4i1:$FRA, (not v4i1:$FRB)),
-          (QVFLOGICALb $FRA, $FRB, (i32 13))>;
-def : Pat<(not (and v4i1:$FRA, v4i1:$FRB)),
-          (QVFLOGICALb $FRA, $FRB, (i32 14))>;
-
-def : Pat<(and v4i1:$FRA, v4i1:$FRB),
-          (QVFLOGICALb $FRA, $FRB, (i32 1))>;
-def : Pat<(or v4i1:$FRA, v4i1:$FRB),
-          (QVFLOGICALb $FRA, $FRB, (i32 7))>;
-def : Pat<(xor v4i1:$FRA, v4i1:$FRB),
-          (QVFLOGICALb $FRA, $FRB, (i32 6))>;
-def : Pat<(not v4i1:$FRA),
-          (QVFLOGICALb $FRA, $FRA, (i32 10))>;
-
-def : Pat<(v4f64 (fpextend v4f32:$src)),
-          (COPY_TO_REGCLASS $src, QFRC)>;
-
-def : Pat<(v4f32 (fround_exact v4f64:$src)),
-          (COPY_TO_REGCLASS $src, QSRC)>;
-
-// Extract the underlying floating-point values from the
-// QPX (-1.0, 1.0) boolean representation.
-def : Pat<(v4f64 (PPCqbflt v4i1:$src)),
-          (COPY_TO_REGCLASS $src, QFRC)>;
-
-def : Pat<(v4f64 (selectcc i1:$lhs, i1:$rhs, v4f64:$tval, v4f64:$fval, SETLT)),
-          (SELECT_QFRC (CRANDC $lhs, $rhs), $tval, $fval)>;
-def : Pat<(v4f64 (selectcc i1:$lhs, i1:$rhs, v4f64:$tval, v4f64:$fval, SETULT)),
-          (SELECT_QFRC (CRANDC $rhs, $lhs), $tval, $fval)>;
-def : Pat<(v4f64 (selectcc i1:$lhs, i1:$rhs, v4f64:$tval, v4f64:$fval, SETLE)),
-          (SELECT_QFRC (CRORC  $lhs, $rhs), $tval, $fval)>;
-def : Pat<(v4f64 (selectcc i1:$lhs, i1:$rhs, v4f64:$tval, v4f64:$fval, SETULE)),
-          (SELECT_QFRC (CRORC  $rhs, $lhs), $tval, $fval)>;
-def : Pat<(v4f64 (selectcc i1:$lhs, i1:$rhs, v4f64:$tval, v4f64:$fval, SETEQ)),
-          (SELECT_QFRC (CREQV $lhs, $rhs), $tval, $fval)>;
-def : Pat<(v4f64 (selectcc i1:$lhs, i1:$rhs, v4f64:$tval, v4f64:$fval, SETGE)),
-          (SELECT_QFRC (CRORC  $rhs, $lhs), $tval, $fval)>;
-def : Pat<(v4f64 (selectcc i1:$lhs, i1:$rhs, v4f64:$tval, v4f64:$fval, SETUGE)),
-          (SELECT_QFRC (CRORC  $lhs, $rhs), $tval, $fval)>;
-def : Pat<(v4f64 (selectcc i1:$lhs, i1:$rhs, v4f64:$tval, v4f64:$fval, SETGT)),
-          (SELECT_QFRC (CRANDC $rhs, $lhs), $tval, $fval)>;
-def : Pat<(v4f64 (selectcc i1:$lhs, i1:$rhs, v4f64:$tval, v4f64:$fval, SETUGT)),
-          (SELECT_QFRC (CRANDC $lhs, $rhs), $tval, $fval)>;
-def : Pat<(v4f64 (selectcc i1:$lhs, i1:$rhs, v4f64:$tval, v4f64:$fval, SETNE)),
-          (SELECT_QFRC (CRXOR $lhs, $rhs), $tval, $fval)>;
-
-def : Pat<(v4f32 (selectcc i1:$lhs, i1:$rhs, v4f32:$tval, v4f32:$fval, SETLT)),
-          (SELECT_QSRC (CRANDC $lhs, $rhs), $tval, $fval)>;
-def : Pat<(v4f32 (selectcc i1:$lhs, i1:$rhs, v4f32:$tval, v4f32:$fval, SETULT)),
-          (SELECT_QSRC (CRANDC $rhs, $lhs), $tval, $fval)>;
-def : Pat<(v4f32 (selectcc i1:$lhs, i1:$rhs, v4f32:$tval, v4f32:$fval, SETLE)),
-          (SELECT_QSRC (CRORC  $lhs, $rhs), $tval, $fval)>;
-def : Pat<(v4f32 (selectcc i1:$lhs, i1:$rhs, v4f32:$tval, v4f32:$fval, SETULE)),
-          (SELECT_QSRC (CRORC  $rhs, $lhs), $tval, $fval)>;
-def : Pat<(v4f32 (selectcc i1:$lhs, i1:$rhs, v4f32:$tval, v4f32:$fval, SETEQ)),
-          (SELECT_QSRC (CREQV $lhs, $rhs), $tval, $fval)>;
-def : Pat<(v4f32 (selectcc i1:$lhs, i1:$rhs, v4f32:$tval, v4f32:$fval, SETGE)),
-          (SELECT_QSRC (CRORC  $rhs, $lhs), $tval, $fval)>;
-def : Pat<(v4f32 (selectcc i1:$lhs, i1:$rhs, v4f32:$tval, v4f32:$fval, SETUGE)),
-          (SELECT_QSRC (CRORC  $lhs, $rhs), $tval, $fval)>;
-def : Pat<(v4f32 (selectcc i1:$lhs, i1:$rhs, v4f32:$tval, v4f32:$fval, SETGT)),
-          (SELECT_QSRC (CRANDC $rhs, $lhs), $tval, $fval)>;
-def : Pat<(v4f32 (selectcc i1:$lhs, i1:$rhs, v4f32:$tval, v4f32:$fval, SETUGT)),
-          (SELECT_QSRC (CRANDC $lhs, $rhs), $tval, $fval)>;
-def : Pat<(v4f32 (selectcc i1:$lhs, i1:$rhs, v4f32:$tval, v4f32:$fval, SETNE)),
-          (SELECT_QSRC (CRXOR $lhs, $rhs), $tval, $fval)>;
-
-def : Pat<(v4i1 (selectcc i1:$lhs, i1:$rhs, v4i1:$tval, v4i1:$fval, SETLT)),
-          (SELECT_QBRC (CRANDC $lhs, $rhs), $tval, $fval)>;
-def : Pat<(v4i1 (selectcc i1:$lhs, i1:$rhs, v4i1:$tval, v4i1:$fval, SETULT)),
-          (SELECT_QBRC (CRANDC $rhs, $lhs), $tval, $fval)>;
-def : Pat<(v4i1 (selectcc i1:$lhs, i1:$rhs, v4i1:$tval, v4i1:$fval, SETLE)),
-          (SELECT_QBRC (CRORC  $lhs, $rhs), $tval, $fval)>;
-def : Pat<(v4i1 (selectcc i1:$lhs, i1:$rhs, v4i1:$tval, v4i1:$fval, SETULE)),
-          (SELECT_QBRC (CRORC  $rhs, $lhs), $tval, $fval)>;
-def : Pat<(v4i1 (selectcc i1:$lhs, i1:$rhs, v4i1:$tval, v4i1:$fval, SETEQ)),
-          (SELECT_QBRC (CREQV $lhs, $rhs), $tval, $fval)>;
-def : Pat<(v4i1 (selectcc i1:$lhs, i1:$rhs, v4i1:$tval, v4i1:$fval, SETGE)),
-          (SELECT_QBRC (CRORC  $rhs, $lhs), $tval, $fval)>;
-def : Pat<(v4i1 (selectcc i1:$lhs, i1:$rhs, v4i1:$tval, v4i1:$fval, SETUGE)),
-          (SELECT_QBRC (CRORC  $lhs, $rhs), $tval, $fval)>;
-def : Pat<(v4i1 (selectcc i1:$lhs, i1:$rhs, v4i1:$tval, v4i1:$fval, SETGT)),
-          (SELECT_QBRC (CRANDC $rhs, $lhs), $tval, $fval)>;
-def : Pat<(v4i1 (selectcc i1:$lhs, i1:$rhs, v4i1:$tval, v4i1:$fval, SETUGT)),
-          (SELECT_QBRC (CRANDC $lhs, $rhs), $tval, $fval)>;
-def : Pat<(v4i1 (selectcc i1:$lhs, i1:$rhs, v4i1:$tval, v4i1:$fval, SETNE)),
-          (SELECT_QBRC (CRXOR $lhs, $rhs), $tval, $fval)>;
-
-} // end HasQPX
-
-let Predicates = [HasQPX, NoNaNsFPMath] in {
-def : Pat<(fminnum v4f64:$FRA, v4f64:$FRB),
-          (QVFSELb (QVFCMPLTb $FRA, $FRB), $FRB, $FRA)>;
-def : Pat<(fmaxnum v4f64:$FRA, v4f64:$FRB),
-          (QVFSELb (QVFCMPGTb $FRA, $FRB), $FRB, $FRA)>;
-
-def : Pat<(fminnum v4f32:$FRA, v4f32:$FRB),
-          (QVFSELbs (QVFCMPLTbs $FRA, $FRB), $FRB, $FRA)>;
-def : Pat<(fmaxnum v4f32:$FRA, v4f32:$FRB),
-          (QVFSELbs (QVFCMPGTbs $FRA, $FRB), $FRB, $FRA)>;
-}
-
-let Predicates = [HasQPX, NaNsFPMath] in {
-// When either of these operands is NaN, we should return the other operand.
-// QVFCMPLT/QVFCMPGT return false is either operand is NaN, which means we need
-// to explicitly or with a NaN test on the second operand.
-def : Pat<(fminnum v4f64:$FRA, v4f64:$FRB),
-          (QVFSELb (QVFLOGICALb (QVFCMPLTb $FRA, $FRB),
-                                (QVFTSTNANb $FRB, $FRB), (i32 7)),
-                   $FRB, $FRA)>;
-def : Pat<(fmaxnum v4f64:$FRA, v4f64:$FRB),
-          (QVFSELb (QVFLOGICALb (QVFCMPGTb $FRA, $FRB),
-                                (QVFTSTNANb $FRB, $FRB), (i32 7)),
-                   $FRB, $FRA)>;
-
-def : Pat<(fminnum v4f32:$FRA, v4f32:$FRB),
-          (QVFSELbs (QVFLOGICALb (QVFCMPLTbs $FRA, $FRB),
-                                 (QVFTSTNANbs $FRB, $FRB), (i32 7)),
-                   $FRB, $FRA)>;
-def : Pat<(fmaxnum v4f32:$FRA, v4f32:$FRB),
-          (QVFSELbs (QVFLOGICALb (QVFCMPGTbs $FRA, $FRB),
-                                 (QVFTSTNANbs $FRB, $FRB), (i32 7)),
-                   $FRB, $FRA)>;
-}
diff --git a/llvm/lib/Target/PowerPC/PPCQPXLoadSplat.cpp b/llvm/lib/Target/PowerPC/PPCQPXLoadSplat.cpp
deleted file mode 100644
index 6e90426438208..0000000000000
--- a/llvm/lib/Target/PowerPC/PPCQPXLoadSplat.cpp
+++ /dev/null
@@ -1,161 +0,0 @@
-//===----- PPCQPXLoadSplat.cpp - QPX Load Splat Simplification ------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// The QPX vector registers overlay the scalar floating-point registers, and
-// any scalar floating-point loads splat their value across all vector lanes.
-// Thus, if we have a scalar load followed by a splat, we can remove the splat
-// (i.e. replace the load with a load-and-splat pseudo instruction).
-//
-// This pass must run after anything that might do store-to-load forwarding.
-//
-//===----------------------------------------------------------------------===//
-
-#include "PPC.h"
-#include "PPCInstrBuilder.h"
-#include "PPCInstrInfo.h"
-#include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/Statistic.h"
-#include "llvm/CodeGen/MachineFunctionPass.h"
-#include "llvm/CodeGen/TargetSubtargetInfo.h"
-#include "llvm/Support/MathExtras.h"
-#include "llvm/Target/TargetMachine.h"
-using namespace llvm;
-
-#define DEBUG_TYPE "ppc-qpx-load-splat"
-
-STATISTIC(NumSimplified, "Number of QPX load splats simplified");
-
-namespace {
-  struct PPCQPXLoadSplat : public MachineFunctionPass {
-    static char ID;
-    PPCQPXLoadSplat() : MachineFunctionPass(ID) {
-      initializePPCQPXLoadSplatPass(*PassRegistry::getPassRegistry());
-    }
-
-    bool runOnMachineFunction(MachineFunction &Fn) override;
-
-    StringRef getPassName() const override {
-      return "PowerPC QPX Load Splat Simplification";
-    }
-  };
-  char PPCQPXLoadSplat::ID = 0;
-}
-
-INITIALIZE_PASS(PPCQPXLoadSplat, "ppc-qpx-load-splat",
-                "PowerPC QPX Load Splat Simplification",
-                false, false)
-
-FunctionPass *llvm::createPPCQPXLoadSplatPass() {
-  return new PPCQPXLoadSplat();
-}
-
-bool PPCQPXLoadSplat::runOnMachineFunction(MachineFunction &MF) {
-  if (skipFunction(MF.getFunction()))
-    return false;
-
-  bool MadeChange = false;
-  const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
-
-  for (auto MFI = MF.begin(), MFIE = MF.end(); MFI != MFIE; ++MFI) {
-    MachineBasicBlock *MBB = &*MFI;
-    SmallVector<MachineInstr *, 4> Splats;
-
-    for (auto MBBI = MBB->rbegin(); MBBI != MBB->rend(); ++MBBI) {
-      MachineInstr *MI = &*MBBI;
-
-      if (MI->hasUnmodeledSideEffects() || MI->isCall()) {
-        Splats.clear();
-        continue;
-      }
-
-      // We're looking for a sequence like this:
-      // %f0 = LFD 0, killed %x3, implicit-def %qf0; mem:LD8[%a](tbaa=!2)
-      // %qf1 = QVESPLATI killed %qf0, 0, implicit %rm
-
-      for (auto SI = Splats.begin(); SI != Splats.end();) {
-        MachineInstr *SMI = *SI;
-        Register SplatReg = SMI->getOperand(0).getReg();
-        Register SrcReg = SMI->getOperand(1).getReg();
-
-        if (MI->modifiesRegister(SrcReg, TRI)) {
-          switch (MI->getOpcode()) {
-          default:
-            SI = Splats.erase(SI);
-            continue;
-          case PPC::LFS:
-          case PPC::LFD:
-          case PPC::LFSU:
-          case PPC::LFDU:
-          case PPC::LFSUX:
-          case PPC::LFDUX:
-          case PPC::LFSX:
-          case PPC::LFDX:
-          case PPC::LFIWAX:
-          case PPC::LFIWZX:
-            if (SplatReg != SrcReg) {
-              // We need to change the load to define the scalar subregister of
-              // the QPX splat source register.
-              unsigned SubRegIndex =
-                TRI->getSubRegIndex(SrcReg, MI->getOperand(0).getReg());
-              Register SplatSubReg = TRI->getSubReg(SplatReg, SubRegIndex);
-
-              // Substitute both the explicit defined register, and also the
-              // implicit def of the containing QPX register.
-              MI->getOperand(0).setReg(SplatSubReg);
-              MI->substituteRegister(SrcReg, SplatReg, 0, *TRI);
-            }
-
-            SI = Splats.erase(SI);
-
-            // If SMI is directly after MI, then MBBI's base iterator is
-            // pointing at SMI.  Adjust MBBI around the call to erase SMI to
-            // avoid invalidating MBBI.
-            ++MBBI;
-            SMI->eraseFromParent();
-            --MBBI;
-
-            ++NumSimplified;
-            MadeChange = true;
-            continue;
-          }
-        }
-
-        // If this instruction defines the splat register, then we cannot move
-        // the previous definition above it. If it reads from the splat
-        // register, then it must already be alive from some previous
-        // definition, and if the splat register is different from the source
-        // register, then this definition must not be the load for which we're
-        // searching.
-        if (MI->modifiesRegister(SplatReg, TRI) ||
-            (SrcReg != SplatReg &&
-             MI->readsRegister(SplatReg, TRI))) {
-          SI = Splats.erase(SI);
-          continue;
-        }
-
-        ++SI;
-      }
-
-      if (MI->getOpcode() != PPC::QVESPLATI &&
-          MI->getOpcode() != PPC::QVESPLATIs &&
-          MI->getOpcode() != PPC::QVESPLATIb)
-        continue;
-      if (MI->getOperand(2).getImm() != 0)
-        continue;
-
-      // If there are other uses of the scalar value after this, replacing
-      // those uses might be non-trivial.
-      if (!MI->getOperand(1).isKill())
-        continue;
-
-      Splats.push_back(MI);
-    }
-  }
-
-  return MadeChange;
-}
diff --git a/llvm/lib/Target/PowerPC/PPCRegisterInfo.cpp b/llvm/lib/Target/PowerPC/PPCRegisterInfo.cpp
index ed8948a639728..96666ad58dfe5 100644
--- a/llvm/lib/Target/PowerPC/PPCRegisterInfo.cpp
+++ b/llvm/lib/Target/PowerPC/PPCRegisterInfo.cpp
@@ -404,9 +404,6 @@ unsigned PPCRegisterInfo::getRegPressureLimit(const TargetRegisterClass *RC,
   }
   case PPC::F8RCRegClassID:
   case PPC::F4RCRegClassID:
-  case PPC::QFRCRegClassID:
-  case PPC::QSRCRegClassID:
-  case PPC::QBRCRegClassID:
   case PPC::VRRCRegClassID:
   case PPC::VFRCRegClassID:
   case PPC::VSLRCRegClassID:
diff --git a/llvm/lib/Target/PowerPC/PPCRegisterInfo.h b/llvm/lib/Target/PowerPC/PPCRegisterInfo.h
index 61acd955e1cba..a931967862c7b 100644
--- a/llvm/lib/Target/PowerPC/PPCRegisterInfo.h
+++ b/llvm/lib/Target/PowerPC/PPCRegisterInfo.h
@@ -153,7 +153,6 @@ class PPCRegisterInfo : public PPCGenRegisterInfo {
     switch (RegName[0]) {
       case 'r':
       case 'f':
-      case 'q': // for QPX
       case 'v':
         if (RegName[1] == 's')
           return RegName + 2;
diff --git a/llvm/lib/Target/PowerPC/PPCRegisterInfo.td b/llvm/lib/Target/PowerPC/PPCRegisterInfo.td
index b45757c1acc5e..e07b960ae305b 100644
--- a/llvm/lib/Target/PowerPC/PPCRegisterInfo.td
+++ b/llvm/lib/Target/PowerPC/PPCRegisterInfo.td
@@ -54,13 +54,6 @@ class FPR<bits<5> num, string n> : PPCReg<n> {
   let HWEncoding{4-0} = num;
 }
 
-// QFPR - One of the 32 256-bit floating-point vector registers (used for QPX)
-class QFPR<FPR SubReg, string n> : PPCReg<n> {
-  let HWEncoding = SubReg.HWEncoding;
-  let SubRegs = [SubReg];
-  let SubRegIndices = [sub_64];
-}
-
 // VF - One of the 32 64-bit floating-point subregisters of the vector
 // registers (used by VSX).
 class VF<bits<5> num, string n> : PPCReg<n> {
@@ -132,12 +125,6 @@ foreach Index = 0-31 in {
                  DwarfRegNum<[!add(Index, 77), !add(Index, 77)]>;
 }
 
-// QPX Floating-point registers
-foreach Index = 0-31 in {
-  def QF#Index : QFPR<!cast<FPR>("F"#Index), "q"#Index>,
-                 DwarfRegNum<[!add(Index, 32), !add(Index, 32)]>;
-}
-
 // Vector registers
 foreach Index = 0-31 in {
   def V#Index : VR<!cast<VF>("VF"#Index), "v"#Index>,
@@ -343,16 +330,6 @@ def SPILLTOVSRRC : RegisterClass<"PPC", [i64, f64], 64, (add G8RC, (sub VSFRC,
 // Register class for single precision scalars in VSX registers
 def VSSRC : RegisterClass<"PPC", [f32], 32, (add VSFRC)>;
 
-// For QPX
-def QFRC : RegisterClass<"PPC", [v4f64], 256, (add (sequence "QF%u", 0, 13),
-                                                (sequence "QF%u", 31, 14))>;
-def QSRC : RegisterClass<"PPC", [v4f32], 128, (add QFRC)>;
-def QBRC : RegisterClass<"PPC", [v4i1], 256, (add QFRC)> {
-  // These are actually stored as floating-point values where a positive
-  // number is true and anything else (including NaN) is false.
-  let Size = 256;
-}
-
 def CRBITRC : RegisterClass<"PPC", [i1], 32,
   (add CR2LT, CR2GT, CR2EQ, CR2UN,
        CR3LT, CR3GT, CR3EQ, CR3UN,
diff --git a/llvm/lib/Target/PowerPC/PPCScheduleP9.td b/llvm/lib/Target/PowerPC/PPCScheduleP9.td
index 0a1ae7e55b3c2..311d5cf165f63 100644
--- a/llvm/lib/Target/PowerPC/PPCScheduleP9.td
+++ b/llvm/lib/Target/PowerPC/PPCScheduleP9.td
@@ -40,12 +40,9 @@ def P9Model : SchedMachineModel {
 
   let CompleteModel = 1;
 
-  // Do not support QPX (Quad Processing eXtension), SPE (Signal Processing
-  // Engine), prefixed instructions on Power 9, PC relative mem ops, or
-  // instructions introduced in ISA 3.1.
-  let UnsupportedFeatures = [HasQPX, HasSPE, PrefixInstrs, PCRelativeMemops,
-                             IsISA3_1];
-
+  // Do not support SPE (Signal Processing Engine), prefixed instructions on
+  // Power 9, PC relative mem ops, or instructions introduced in ISA 3.1.
+  let UnsupportedFeatures = [HasSPE, PrefixInstrs, PCRelativeMemops, IsISA3_1];
 }
 
 let SchedModel = P9Model in {
diff --git a/llvm/lib/Target/PowerPC/PPCSubtarget.cpp b/llvm/lib/Target/PowerPC/PPCSubtarget.cpp
index 3836cc960394f..85d2966654970 100644
--- a/llvm/lib/Target/PowerPC/PPCSubtarget.cpp
+++ b/llvm/lib/Target/PowerPC/PPCSubtarget.cpp
@@ -35,10 +35,6 @@ using namespace llvm;
 static cl::opt<bool> UseSubRegLiveness("ppc-track-subreg-liveness",
 cl::desc("Enable subregister liveness tracking for PPC"), cl::Hidden);
 
-static cl::opt<bool> QPXStackUnaligned("qpx-stack-unaligned",
-  cl::desc("Even when QPX is enabled the stack is not 32-byte aligned"),
-  cl::Hidden);
-
 static cl::opt<bool>
     EnableMachinePipeliner("ppc-enable-pipeliner",
                            cl::desc("Enable Machine Pipeliner for PPC"),
@@ -70,7 +66,6 @@ void PPCSubtarget::initializeEnvironment() {
   HasAltivec = false;
   HasSPE = false;
   HasFPU = false;
-  HasQPX = false;
   HasVSX = false;
   NeedsTwoConstNR = false;
   HasP8Vector = false;
@@ -109,7 +104,6 @@ void PPCSubtarget::initializeEnvironment() {
   HasInvariantFunctionDescriptors = false;
   HasPartwordAtomics = false;
   HasDirectMove = false;
-  IsQPXStackUnaligned = false;
   HasHTM = false;
   HasFloat128 = false;
   HasFusion = false;
@@ -158,7 +152,7 @@ void PPCSubtarget::initSubtargetFeatures(StringRef CPU, StringRef FS) {
 
   if (HasSPE && IsPPC64)
     report_fatal_error( "SPE is only supported for 32-bit targets.\n", false);
-  if (HasSPE && (HasAltivec || HasQPX || HasVSX || HasFPU))
+  if (HasSPE && (HasAltivec || HasVSX || HasFPU))
     report_fatal_error(
         "SPE and traditional floating point cannot both be enabled.\n", false);
 
@@ -166,10 +160,6 @@ void PPCSubtarget::initSubtargetFeatures(StringRef CPU, StringRef FS) {
   if (!HasSPE)
     HasFPU = true;
 
-  // QPX requires a 32-byte aligned stack. Note that we need to do this if
-  // we're compiling for a BG/Q system regardless of whether or not QPX
-  // is enabled because external functions will assume this alignment.
-  IsQPXStackUnaligned = QPXStackUnaligned;
   StackAlignment = getPlatformStackAlignment();
 
   // Determine endianness.
diff --git a/llvm/lib/Target/PowerPC/PPCSubtarget.h b/llvm/lib/Target/PowerPC/PPCSubtarget.h
index ec329022c4572..8a4041518e3c2 100644
--- a/llvm/lib/Target/PowerPC/PPCSubtarget.h
+++ b/llvm/lib/Target/PowerPC/PPCSubtarget.h
@@ -97,7 +97,6 @@ class PPCSubtarget : public PPCGenSubtargetInfo {
   bool HasAltivec;
   bool HasFPU;
   bool HasSPE;
-  bool HasQPX;
   bool HasVSX;
   bool NeedsTwoConstNR;
   bool HasP8Vector;
@@ -150,11 +149,6 @@ class PPCSubtarget : public PPCGenSubtargetInfo {
 
   POPCNTDKind HasPOPCNTD;
 
-  /// When targeting QPX running a stock PPC64 Linux kernel where the stack
-  /// alignment has not been changed, we need to keep the 16-byte alignment
-  /// of the stack.
-  bool IsQPXStackUnaligned;
-
   const PPCTargetMachine &TM;
   PPCFrameLowering FrameLowering;
   PPCInstrInfo InstrInfo;
@@ -255,7 +249,6 @@ class PPCSubtarget : public PPCGenSubtargetInfo {
   bool hasAltivec() const { return HasAltivec; }
   bool hasSPE() const { return HasSPE; }
   bool hasFPU() const { return HasFPU; }
-  bool hasQPX() const { return HasQPX; }
   bool hasVSX() const { return HasVSX; }
   bool needsTwoConstNR() const { return NeedsTwoConstNR; }
   bool hasP8Vector() const { return HasP8Vector; }
@@ -291,11 +284,7 @@ class PPCSubtarget : public PPCGenSubtargetInfo {
   bool hasPartwordAtomics() const { return HasPartwordAtomics; }
   bool hasDirectMove() const { return HasDirectMove; }
 
-  bool isQPXStackUnaligned() const { return IsQPXStackUnaligned; }
   Align getPlatformStackAlignment() const {
-    if ((hasQPX() || isBGQ()) && !isQPXStackUnaligned())
-      return Align(32);
-
     return Align(16);
   }
 
@@ -325,9 +314,6 @@ class PPCSubtarget : public PPCGenSubtargetInfo {
 
   const Triple &getTargetTriple() const { return TargetTriple; }
 
-  /// isBGQ - True if this is a BG/Q platform.
-  bool isBGQ() const { return TargetTriple.getVendor() == Triple::BGQ; }
-
   bool isTargetELF() const { return TargetTriple.isOSBinFormatELF(); }
   bool isTargetMachO() const { return TargetTriple.isOSBinFormatMachO(); }
   bool isTargetLinux() const { return TargetTriple.isOSLinux(); }
diff --git a/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp b/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp
index f15f9c7f49429..27de5b29cd341 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp
+++ b/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp
@@ -63,10 +63,6 @@ static cl::
 opt<bool> DisableVSXSwapRemoval("disable-ppc-vsx-swap-removal", cl::Hidden,
                                 cl::desc("Disable VSX Swap Removal for PPC"));
 
-static cl::
-opt<bool> DisableQPXLoadSplat("disable-ppc-qpx-load-splat", cl::Hidden,
-                              cl::desc("Disable QPX load splat simplification"));
-
 static cl::
 opt<bool> DisableMIPeephole("disable-ppc-peephole", cl::Hidden,
                             cl::desc("Disable machine peepholes for PPC"));
@@ -114,7 +110,6 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializePowerPCTarget() {
   initializePPCReduceCRLogicalsPass(PR);
   initializePPCBSelPass(PR);
   initializePPCBranchCoalescingPass(PR);
-  initializePPCQPXLoadSplatPass(PR);
   initializePPCBoolRetToIntPass(PR);
   initializePPCExpandISELPass(PR);
   initializePPCPreEmitPeepholePass(PR);
@@ -411,14 +406,9 @@ void PPCPassConfig::addIRPasses() {
 
   // Lower generic MASSV routines to PowerPC subtarget-specific entries.
   addPass(createPPCLowerMASSVEntriesPass());
-  
-  // For the BG/Q (or if explicitly requested), add explicit data prefetch
-  // intrinsics.
-  bool UsePrefetching = TM->getTargetTriple().getVendor() == Triple::BGQ &&
-                        getOptLevel() != CodeGenOpt::None;
+
+  // If explicitly requested, add explicit data prefetch intrinsics.
   if (EnablePrefetch.getNumOccurrences() > 0)
-    UsePrefetching = EnablePrefetch;
-  if (UsePrefetching)
     addPass(createLoopDataPrefetchPass());
 
   if (TM->getOptLevel() >= CodeGenOpt::Default && EnableGEPOpt) {
@@ -515,15 +505,8 @@ void PPCPassConfig::addPreRegAlloc() {
 }
 
 void PPCPassConfig::addPreSched2() {
-  if (getOptLevel() != CodeGenOpt::None) {
+  if (getOptLevel() != CodeGenOpt::None)
     addPass(&IfConverterID);
-
-    // This optimization must happen after anything that might do store-to-load
-    // forwarding. Here we're after RA (and, thus, when spills are inserted)
-    // but before post-RA scheduling.
-    if (!DisableQPXLoadSplat)
-      addPass(createPPCQPXLoadSplatPass());
-  }
 }
 
 void PPCPassConfig::addPreEmitPass() {
diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
index bbb4239d36da5..ee8842f4d8663 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
+++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
@@ -25,8 +25,7 @@ using namespace llvm;
 static cl::opt<bool> DisablePPCConstHoist("disable-ppc-constant-hoisting",
 cl::desc("disable constant hoisting on PPC"), cl::init(false), cl::Hidden);
 
-// This is currently only used for the data prefetch pass which is only enabled
-// for BG/Q by default.
+// This is currently only used for the data prefetch pass
 static cl::opt<unsigned>
 CacheLineSize("ppc-loop-prefetch-cache-line", cl::Hidden, cl::init(64),
               cl::desc("The loop prefetch cache line size"));
@@ -104,55 +103,6 @@ PPCTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
     Value *Ptr = IC.Builder.CreateBitCast(II.getArgOperand(1), OpPtrTy);
     return new StoreInst(II.getArgOperand(0), Ptr, false, Align(1));
   }
-  case Intrinsic::ppc_qpx_qvlfs:
-    // Turn PPC QPX qvlfs -> load if the pointer is known aligned.
-    if (getOrEnforceKnownAlignment(
-            II.getArgOperand(0), Align(16), IC.getDataLayout(), &II,
-            &IC.getAssumptionCache(), &IC.getDominatorTree()) >= 16) {
-      Type *VTy =
-          VectorType::get(IC.Builder.getFloatTy(),
-                          cast<VectorType>(II.getType())->getElementCount());
-      Value *Ptr = IC.Builder.CreateBitCast(II.getArgOperand(0),
-                                            PointerType::getUnqual(VTy));
-      Value *Load = IC.Builder.CreateLoad(VTy, Ptr);
-      return new FPExtInst(Load, II.getType());
-    }
-    break;
-  case Intrinsic::ppc_qpx_qvlfd:
-    // Turn PPC QPX qvlfd -> load if the pointer is known aligned.
-    if (getOrEnforceKnownAlignment(
-            II.getArgOperand(0), Align(32), IC.getDataLayout(), &II,
-            &IC.getAssumptionCache(), &IC.getDominatorTree()) >= 32) {
-      Value *Ptr = IC.Builder.CreateBitCast(
-          II.getArgOperand(0), PointerType::getUnqual(II.getType()));
-      return new LoadInst(II.getType(), Ptr, "", false, Align(32));
-    }
-    break;
-  case Intrinsic::ppc_qpx_qvstfs:
-    // Turn PPC QPX qvstfs -> store if the pointer is known aligned.
-    if (getOrEnforceKnownAlignment(
-            II.getArgOperand(1), Align(16), IC.getDataLayout(), &II,
-            &IC.getAssumptionCache(), &IC.getDominatorTree()) >= 16) {
-      Type *VTy = VectorType::get(
-          IC.Builder.getFloatTy(),
-          cast<VectorType>(II.getArgOperand(0)->getType())->getElementCount());
-      Value *TOp = IC.Builder.CreateFPTrunc(II.getArgOperand(0), VTy);
-      Type *OpPtrTy = PointerType::getUnqual(VTy);
-      Value *Ptr = IC.Builder.CreateBitCast(II.getArgOperand(1), OpPtrTy);
-      return new StoreInst(TOp, Ptr, false, Align(16));
-    }
-    break;
-  case Intrinsic::ppc_qpx_qvstfd:
-    // Turn PPC QPX qvstfd -> store if the pointer is known aligned.
-    if (getOrEnforceKnownAlignment(
-            II.getArgOperand(1), Align(32), IC.getDataLayout(), &II,
-            &IC.getAssumptionCache(), &IC.getDominatorTree()) >= 32) {
-      Type *OpPtrTy = PointerType::getUnqual(II.getArgOperand(0)->getType());
-      Value *Ptr = IC.Builder.CreateBitCast(II.getArgOperand(1), OpPtrTy);
-      return new StoreInst(II.getArgOperand(0), Ptr, false, Align(32));
-    }
-    break;
-
   case Intrinsic::ppc_altivec_vperm:
     // Turn vperm(V1,V2,mask) -> shuffle(V1,V2,mask) if mask is a constant.
     // Note that ppc_altivec_vperm has a big-endian bias, so when creating
@@ -736,10 +686,7 @@ bool PPCTTIImpl::useColdCCForColdCall(Function &F) {
 }
 
 bool PPCTTIImpl::enableAggressiveInterleaving(bool LoopHasReductions) {
-  // On the A2, always unroll aggressively. For QPX unaligned loads, we depend
-  // on combining the loads generated for consecutive accesses, and failure to
-  // do so is particularly expensive. This makes it much more likely (compared
-  // to only using concatenation unrolling).
+  // On the A2, always unroll aggressively.
   if (ST->getCPUDirective() == PPC::DIR_A2)
     return true;
 
@@ -799,7 +746,6 @@ const char* PPCTTIImpl::getRegisterClassName(unsigned ClassID) const {
 
 unsigned PPCTTIImpl::getRegisterBitWidth(bool Vector) const {
   if (Vector) {
-    if (ST->hasQPX()) return 256;
     if (ST->hasAltivec()) return 128;
     return 0;
   }
@@ -828,8 +774,6 @@ unsigned PPCTTIImpl::getCacheLineSize() const {
 }
 
 unsigned PPCTTIImpl::getPrefetchDistance() const {
-  // This seems like a reasonable default for the BG/Q (this pass is enabled, by
-  // default, only on the BG/Q).
   return 300;
 }
 
@@ -918,7 +862,7 @@ int PPCTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, Type *Tp, int Index,
   // Legalize the type.
   std::pair<int, MVT> LT = TLI->getTypeLegalizationCost(DL, Tp);
 
-  // PPC, for both Altivec/VSX and QPX, support cheap arbitrary permutations
+  // PPC, for both Altivec/VSX, support cheap arbitrary permutations
   // (at least in the sense that there need only be one non-loop-invariant
   // instruction). We need one such shuffle instruction for each actual
   // register (this is not true for arbitrary shuffles, but is true for the
@@ -974,13 +918,6 @@ int PPCTTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val, unsigned Index) {
 
     return Cost;
 
-  } else if (ST->hasQPX() && Val->getScalarType()->isFloatingPointTy()) {
-    // Floating point scalars are already located in index #0.
-    if (Index == 0)
-      return 0;
-
-    return Cost;
-
   } else if (Val->getScalarType()->isIntegerTy() && Index != -1U) {
     if (ST->hasP9Altivec()) {
       if (ISD == ISD::INSERT_VECTOR_ELT)
@@ -1055,8 +992,6 @@ int PPCTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
                         LT.second == MVT::v4i32 || LT.second == MVT::v4f32);
   bool IsVSXType = ST->hasVSX() &&
                    (LT.second == MVT::v2f64 || LT.second == MVT::v2i64);
-  bool IsQPXType = ST->hasQPX() &&
-                   (LT.second == MVT::v4f64 || LT.second == MVT::v4f32);
 
   // VSX has 32b/64b load instructions. Legalization can handle loading of
   // 32b/64b to VSR correctly and cheaply. But BaseT::getMemoryOpCost and
@@ -1079,8 +1014,7 @@ int PPCTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
   // for Altivec types using the VSX instructions, but that's more expensive
   // than using the permutation-based load sequence. On the P8, that's no
   // longer true.
-  if (Opcode == Instruction::Load &&
-      ((!ST->hasP8Vector() && IsAltivecType) || IsQPXType) &&
+  if (Opcode == Instruction::Load && (!ST->hasP8Vector() && IsAltivecType) &&
       *Alignment >= LT.second.getScalarType().getStoreSize())
     return Cost + LT.first; // Add the cost of the permutations.
 
@@ -1133,7 +1067,7 @@ int PPCTTIImpl::getInterleavedMemoryOpCost(
       getMemoryOpCost(Opcode, VecTy, MaybeAlign(Alignment), AddressSpace,
                       CostKind);
 
-  // PPC, for both Altivec/VSX and QPX, support cheap arbitrary permutations
+  // PPC, for both Altivec/VSX, support cheap arbitrary permutations
   // (at least in the sense that there need only be one non-loop-invariant
   // instruction). For each result vector, we need one shuffle per incoming
   // vector (except that the first shuffle can take two incoming vectors
diff --git a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
index 427abde4277d4..aa06e8144f634 100644
--- a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
@@ -4751,15 +4751,14 @@ struct VarArgPowerPC64Helper : public VarArgHelper {
     // For PowerPC, we need to deal with alignment of stack arguments -
     // they are mostly aligned to 8 bytes, but vectors and i128 arrays
     // are aligned to 16 bytes, byvals can be aligned to 8 or 16 bytes,
-    // and QPX vectors are aligned to 32 bytes.  For that reason, we
-    // compute current offset from stack pointer (which is always properly
-    // aligned), and offset for the first vararg, then subtract them.
+    // For that reason, we compute current offset from stack pointer (which is
+    // always properly aligned), and offset for the first vararg, then subtract
+    // them.
     unsigned VAArgBase;
     Triple TargetTriple(F.getParent()->getTargetTriple());
     // Parameter save area starts at 48 bytes from frame pointer for ABIv1,
     // and 32 bytes for ABIv2.  This is usually determined by target
     // endianness, but in theory could be overridden by function attribute.
-    // For simplicity, we ignore it here (it'd only matter for QPX vectors).
     if (TargetTriple.getArch() == Triple::ppc64)
       VAArgBase = 48;
     else
diff --git a/llvm/test/Analysis/BasicAA/phi-spec-order.ll b/llvm/test/Analysis/BasicAA/phi-spec-order.ll
index f8586f094c2ce..e5d435c09ccc7 100644
--- a/llvm/test/Analysis/BasicAA/phi-spec-order.ll
+++ b/llvm/test/Analysis/BasicAA/phi-spec-order.ll
@@ -1,5 +1,5 @@
 target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
-target triple = "powerpc64-bgq-linux"
+target triple = "powerpc64le-unknown-linux"
 ; RUN: opt < %s -basic-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 
 @X = external global [16000 x double], align 32
diff --git a/llvm/test/Analysis/CostModel/PowerPC/unal-vec-ldst.ll b/llvm/test/Analysis/CostModel/PowerPC/unal-vec-ldst.ll
index 3b1bc3b3fdbc0..e5fbf070cf32a 100644
--- a/llvm/test/Analysis/CostModel/PowerPC/unal-vec-ldst.ll
+++ b/llvm/test/Analysis/CostModel/PowerPC/unal-vec-ldst.ll
@@ -218,42 +218,6 @@ entry:
 ; CHECK: cost of 2 for instruction:   %r = load <4 x double>, <4 x double>* %p, align 8
 }
 
-define <4 x float> @test_l_qv4float(<4 x float>* %p) #1 {
-entry:
-  %r = load <4 x float>, <4 x float>* %p, align 4
-  ret <4 x float> %r
-
-; CHECK-LABEL: test_l_qv4float
-; CHECK: cost of 2 for instruction:   %r = load <4 x float>, <4 x float>* %p, align 4
-}
-
-define <8 x float> @test_l_qv8float(<8 x float>* %p) #1 {
-entry:
-  %r = load <8 x float>, <8 x float>* %p, align 4
-  ret <8 x float> %r
-
-; CHECK-LABEL: test_l_qv8float
-; CHECK: cost of 4 for instruction:   %r = load <8 x float>, <8 x float>* %p, align 4
-}
-
-define <4 x double> @test_l_qv4double(<4 x double>* %p) #1 {
-entry:
-  %r = load <4 x double>, <4 x double>* %p, align 8
-  ret <4 x double> %r
-
-; CHECK-LABEL: test_l_qv4double
-; CHECK: cost of 2 for instruction:   %r = load <4 x double>, <4 x double>* %p, align 8
-}
-
-define <8 x double> @test_l_qv8double(<8 x double>* %p) #1 {
-entry:
-  %r = load <8 x double>, <8 x double>* %p, align 8
-  ret <8 x double> %r
-
-; CHECK-LABEL: test_l_qv8double
-; CHECK: cost of 4 for instruction:   %r = load <8 x double>, <8 x double>* %p, align 8
-}
-
 define void @test_s_v16i8(<16 x i8>* %p, <16 x i8> %v) #0 {
 entry:
   store <16 x i8> %v, <16 x i8>* %p, align 1
@@ -362,43 +326,6 @@ entry:
 ; CHECK: cost of 2 for instruction:   store <4 x double> %v, <4 x double>* %p, align 8
 }
 
-define void @test_s_qv4float(<4 x float>* %p, <4 x float> %v) #1 {
-entry:
-  store <4 x float> %v, <4 x float>* %p, align 4
-  ret void
-
-; CHECK-LABEL: test_s_qv4float
-; CHECK: cost of 7 for instruction:   store <4 x float> %v, <4 x float>* %p, align 4
-}
-
-define void @test_s_qv8float(<8 x float>* %p, <8 x float> %v) #1 {
-entry:
-  store <8 x float> %v, <8 x float>* %p, align 4
-  ret void
-
-; CHECK-LABEL: test_s_qv8float
-; CHECK: cost of 15 for instruction:   store <8 x float> %v, <8 x float>* %p, align 4
-}
-
-define void @test_s_qv4double(<4 x double>* %p, <4 x double> %v) #1 {
-entry:
-  store <4 x double> %v, <4 x double>* %p, align 8
-  ret void
-
-; CHECK-LABEL: test_s_qv4double
-; CHECK: cost of 7 for instruction:   store <4 x double> %v, <4 x double>* %p, align 8
-}
-
-define void @test_s_qv8double(<8 x double>* %p, <8 x double> %v) #1 {
-entry:
-  store <8 x double> %v, <8 x double>* %p, align 8
-  ret void
-
-; CHECK-LABEL: test_s_qv8double
-; CHECK: cost of 15 for instruction:   store <8 x double> %v, <8 x double>* %p, align 8
-}
-
 attributes #0 = { nounwind "target-cpu"="pwr7" }
-attributes #1 = { nounwind "target-cpu"="a2q" }
 attributes #2 = { nounwind "target-cpu"="pwr8" }
 
diff --git a/llvm/test/CodeGen/PowerPC/2012-11-16-mischedcall.ll b/llvm/test/CodeGen/PowerPC/2012-11-16-mischedcall.ll
index 69f9cff5c525f..d93f192b1274d 100644
--- a/llvm/test/CodeGen/PowerPC/2012-11-16-mischedcall.ll
+++ b/llvm/test/CodeGen/PowerPC/2012-11-16-mischedcall.ll
@@ -1,4 +1,4 @@
-; RUN: llc -verify-machineinstrs -mtriple=powerpc64-bgq-linux -enable-misched < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux -enable-misched < %s | FileCheck %s
 ;
 ; PR14315: misched should not move the physreg copy of %t below the calls.
 
diff --git a/llvm/test/CodeGen/PowerPC/DisableHoistingDueToBlockHotnessProfileData.mir b/llvm/test/CodeGen/PowerPC/DisableHoistingDueToBlockHotnessProfileData.mir
index 738aa1df5dd9d..a0139879f8c91 100644
--- a/llvm/test/CodeGen/PowerPC/DisableHoistingDueToBlockHotnessProfileData.mir
+++ b/llvm/test/CodeGen/PowerPC/DisableHoistingDueToBlockHotnessProfileData.mir
@@ -55,7 +55,7 @@
   ; Function Attrs: nounwind
   declare void @llvm.stackprotector(i8*, i8**) #1
 
-  attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+  attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector" "unsafe-fp-math"="false" "use-soft-float"="false" }
   attributes #1 = { nounwind }
 
   !llvm.module.flags = !{!0, !1}
diff --git a/llvm/test/CodeGen/PowerPC/NoCRFieldRedefWhenSpillingCRBIT.mir b/llvm/test/CodeGen/PowerPC/NoCRFieldRedefWhenSpillingCRBIT.mir
index bcd51d31c6cfd..01ce79995512a 100644
--- a/llvm/test/CodeGen/PowerPC/NoCRFieldRedefWhenSpillingCRBIT.mir
+++ b/llvm/test/CodeGen/PowerPC/NoCRFieldRedefWhenSpillingCRBIT.mir
@@ -30,7 +30,7 @@
   ; Function Attrs: nounwind
   declare void @llvm.stackprotector(i8*, i8**) #1
   
-  attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+  attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector" "unsafe-fp-math"="false" "use-soft-float"="false" }
   attributes #1 = { nounwind }
   
   !llvm.ident = !{!0}
diff --git a/llvm/test/CodeGen/PowerPC/a2q-stackalign.ll b/llvm/test/CodeGen/PowerPC/a2q-stackalign.ll
deleted file mode 100644
index 17e3df6d58ccc..0000000000000
--- a/llvm/test/CodeGen/PowerPC/a2q-stackalign.ll
+++ /dev/null
@@ -1,23 +0,0 @@
-; RUN: llc -verify-machineinstrs < %s -mtriple=ppc64-- -mcpu=a2 | FileCheck -check-prefix=CHECK-A2 %s
-; RUN: llc -verify-machineinstrs < %s -mtriple=ppc64-- -mcpu=a2q | FileCheck -check-prefix=CHECK-A2Q %s
-; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc64-bgq-linux -mcpu=a2 | FileCheck -check-prefix=CHECK-BGQ %s
-target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
-target triple = "powerpc64-unknown-linux-gnu"
-
-declare i32 @bar(i8* %a) nounwind;
-define i32 @foo() nounwind {
-  %p = alloca i8, i8 115
-  store i8 0, i8* %p
-  %r = call i32 @bar(i8* %p)
-  ret i32 %r
-}
-
-; Without QPX, the allocated stack frame is 240 bytes, but with QPX
-; (because we require 32-byte alignment), it is 256 bytes.
-; CHECK-A2: @foo
-; CHECK-A2: stdu 1, -240(1)
-; CHECK-A2Q: @foo
-; CHECK-A2Q: stdu 1, -256(1)
-; CHECK-BGQ: @foo
-; CHECK-BGQ: stdu 1, -256(1)
-
diff --git a/llvm/test/CodeGen/PowerPC/a2q.ll b/llvm/test/CodeGen/PowerPC/a2q.ll
deleted file mode 100644
index 84e2dfa991d78..0000000000000
--- a/llvm/test/CodeGen/PowerPC/a2q.ll
+++ /dev/null
@@ -1,10 +0,0 @@
-; RUN: llc -verify-machineinstrs < %s -mtriple=ppc64-- -mcpu=a2q | FileCheck %s
-; RUN: llc -verify-machineinstrs < %s -mtriple=ppc64-- -mcpu=a2 -mattr=+qpx | FileCheck %s
-
-define void @foo() {
-entry:
-  ret void
-}
-
-; CHECK: @foo
-
diff --git a/llvm/test/CodeGen/PowerPC/aantidep-inline-asm-use.ll b/llvm/test/CodeGen/PowerPC/aantidep-inline-asm-use.ll
index 1b0ea26f1fdea..d629148535aa7 100644
--- a/llvm/test/CodeGen/PowerPC/aantidep-inline-asm-use.ll
+++ b/llvm/test/CodeGen/PowerPC/aantidep-inline-asm-use.ll
@@ -298,7 +298,7 @@ _ZN10SubProcess12SafeSyscalls5fcntlEiil.exit:     ; preds = %_ZN10SubProcess12Sa
 ; Function Attrs: nounwind argmemonly
 declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i1) #1
 
-attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pwr8" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pwr8" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind argmemonly }
 attributes #2 = { nounwind }
 
diff --git a/llvm/test/CodeGen/PowerPC/asm-Zy.ll b/llvm/test/CodeGen/PowerPC/asm-Zy.ll
index 78bb0f4c73eca..c8b5e9f1aa1d1 100644
--- a/llvm/test/CodeGen/PowerPC/asm-Zy.ll
+++ b/llvm/test/CodeGen/PowerPC/asm-Zy.ll
@@ -1,6 +1,5 @@
-; RUN: llc -verify-machineinstrs < %s -mcpu=a2 | FileCheck %s
+; RUN: llc -verify-machineinstrs < %s -mcpu=a2 -mtriple=powerpc64-unknown-linux-gnu | FileCheck %s
 target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
-target triple = "powerpc64-bgq-linux"
 
 define i32 @zytest(i32 %a) nounwind {
 entry:
diff --git a/llvm/test/CodeGen/PowerPC/asm-constraints.ll b/llvm/test/CodeGen/PowerPC/asm-constraints.ll
index a3e573d8935e9..da77d1a169792 100644
--- a/llvm/test/CodeGen/PowerPC/asm-constraints.ll
+++ b/llvm/test/CodeGen/PowerPC/asm-constraints.ll
@@ -65,7 +65,7 @@ entry:
 }
 
 
-attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 attributes #1 = { nounwind }
 
diff --git a/llvm/test/CodeGen/PowerPC/convert-rr-to-ri-instrs-R0-special-handling.mir b/llvm/test/CodeGen/PowerPC/convert-rr-to-ri-instrs-R0-special-handling.mir
index 2081e6fd02f51..904210ee13477 100644
--- a/llvm/test/CodeGen/PowerPC/convert-rr-to-ri-instrs-R0-special-handling.mir
+++ b/llvm/test/CodeGen/PowerPC/convert-rr-to-ri-instrs-R0-special-handling.mir
@@ -63,8 +63,8 @@
     ret i64 %2
   }
   
-  attributes #0 = { norecurse nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
-  attributes #1 = { norecurse nounwind readonly "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+  attributes #0 = { norecurse nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector" "unsafe-fp-math"="false" "use-soft-float"="false" }
+  attributes #1 = { norecurse nounwind readonly "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector" "unsafe-fp-math"="false" "use-soft-float"="false" }
   
   !llvm.module.flags = !{!0, !1}
   !llvm.ident = !{!2}
diff --git a/llvm/test/CodeGen/PowerPC/convert-rr-to-ri-instrs-out-of-range.mir b/llvm/test/CodeGen/PowerPC/convert-rr-to-ri-instrs-out-of-range.mir
index b52e0a4103add..f46d4fc0a42a4 100644
--- a/llvm/test/CodeGen/PowerPC/convert-rr-to-ri-instrs-out-of-range.mir
+++ b/llvm/test/CodeGen/PowerPC/convert-rr-to-ri-instrs-out-of-range.mir
@@ -187,7 +187,7 @@
     ret i64 %cond
   }
   
-  attributes #0 = { norecurse nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pwr9" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+power9-vector,+vsx,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+  attributes #0 = { norecurse nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pwr9" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+power9-vector,+vsx" "unsafe-fp-math"="false" "use-soft-float"="false" }
   
   !llvm.module.flags = !{!0, !1}
   !llvm.ident = !{!2}
diff --git a/llvm/test/CodeGen/PowerPC/convert-rr-to-ri-instrs.mir b/llvm/test/CodeGen/PowerPC/convert-rr-to-ri-instrs.mir
index 4d2595e1abdcb..ba950dc3d3ae9 100644
--- a/llvm/test/CodeGen/PowerPC/convert-rr-to-ri-instrs.mir
+++ b/llvm/test/CodeGen/PowerPC/convert-rr-to-ri-instrs.mir
@@ -983,10 +983,10 @@
     ret i64 %xor
   }
   
-  attributes #0 = { norecurse nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pwr9" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+power9-vector,+vsx,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
-  attributes #1 = { norecurse nounwind readonly "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pwr9" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+power9-vector,+vsx,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
-  attributes #2 = { norecurse nounwind readonly "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pwr9" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+power9-vector,-vsx,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
-  attributes #3 = { norecurse nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pwr9" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+power9-vector,+vsx,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+  attributes #0 = { norecurse nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pwr9" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+power9-vector,+vsx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+  attributes #1 = { norecurse nounwind readonly "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pwr9" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+power9-vector,+vsx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+  attributes #2 = { norecurse nounwind readonly "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pwr9" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+power9-vector,-vsx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+  attributes #3 = { norecurse nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pwr9" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+power9-vector,+vsx" "unsafe-fp-math"="false" "use-soft-float"="false" }
   
   !llvm.module.flags = !{!0, !1}
   !llvm.ident = !{!2}
diff --git a/llvm/test/CodeGen/PowerPC/ctr-minmaxnum.ll b/llvm/test/CodeGen/PowerPC/ctr-minmaxnum.ll
index ed3c9f07c1a85..75640d1d26072 100644
--- a/llvm/test/CodeGen/PowerPC/ctr-minmaxnum.ll
+++ b/llvm/test/CodeGen/PowerPC/ctr-minmaxnum.ll
@@ -1,5 +1,4 @@
 ; RUN: llc -mtriple=powerpc64-unknown-linux-gnu -verify-machineinstrs -mcpu=pwr7 < %s | FileCheck %s
-; RUN: llc -mtriple=powerpc64-unknown-linux-gnu -verify-machineinstrs -mcpu=a2q < %s | FileCheck %s --check-prefix=QPX
 
 declare float @fabsf(float)
 
@@ -64,11 +63,6 @@ loop_exit:
 ; CHECK-NOT: xsmindp
 ; CHECK: blr
 
-; QPX-LABEL: test1v:
-; QPX: mtctr
-; QPX-NOT: bl fminf
-; QPX: blr
-
 define void @test1a(float %f, float* %fp) {
 entry:
   br label %loop_body
@@ -139,11 +133,6 @@ loop_exit:
 ; CHECK-NOT: xsmaxdp
 ; CHECK: blr
 
-; QPX-LABEL: test2v:
-; QPX: mtctr
-; QPX-NOT: bl fmax
-; QPX: blr
-
 define void @test2a(float %f, float* %fp) {
 entry:
   br label %loop_body
diff --git a/llvm/test/CodeGen/PowerPC/ctrloop-shortLoops.ll b/llvm/test/CodeGen/PowerPC/ctrloop-shortLoops.ll
index 44acfcdd6e66a..636c86b815c8c 100644
--- a/llvm/test/CodeGen/PowerPC/ctrloop-shortLoops.ll
+++ b/llvm/test/CodeGen/PowerPC/ctrloop-shortLoops.ll
@@ -1,5 +1,4 @@
 ; RUN: llc < %s -mtriple=powerpc64le-unknown-linux-gnu -verify-machineinstrs -mcpu=pwr8 | FileCheck %s --check-prefixes=CHECK,CHECK-PWR8
-; RUN: llc < %s -mtriple=powerpc64le-unknown-linux-gnu -verify-machineinstrs -mcpu=a2q | FileCheck %s --check-prefixes=CHECK,CHECK-A2Q
 
 ; Verify that we do NOT generate the mtctr instruction for loop trip counts < 4
 ; The latency of the mtctr is only justified if there are more than 4 comparisons that are removed as a result.
@@ -86,11 +85,8 @@ for.body:                                         ; preds = %entry, %for.body
 }
 
 ; Function Attrs: norecurse nounwind
-; On core a2q, IssueWidth is 1. On core pwr8, IssueWidth is 8.
-; a2q should use mtctr, but pwr8 should not use mtctr.
 define signext i32 @testTripCount2NonSmallLoop() {
 ; CHECK-LABEL: testTripCount2NonSmallLoop:
-; CHECK-A2Q: mtctr
 ; CHECK-PWR8-NOT: mtctr
 ; CHECK: blr
 
@@ -121,12 +117,9 @@ for.end:                                          ; preds = %if.end
   ret i32 %conv
 }
 
-; On core a2q, IssueWidth is 1. On core pwr8, IssueWidth is 8.
-; a2q should use mtctr, but pwr8 should not use mtctr.
 define signext i32 @testTripCount5() {
 ; CHECK-LABEL: testTripCount5:
 ; CHECK-PWR8-NOT: mtctr
-; CHECK-A2Q: mtctr
  
 entry:
   %.prea = load i32, i32* @a, align 4
diff --git a/llvm/test/CodeGen/PowerPC/ec-input.ll b/llvm/test/CodeGen/PowerPC/ec-input.ll
index 9a1c121699a69..425bc1985d419 100644
--- a/llvm/test/CodeGen/PowerPC/ec-input.ll
+++ b/llvm/test/CodeGen/PowerPC/ec-input.ll
@@ -5,7 +5,7 @@
 ; that were both inputs to the inline asm and also early-clobber outputs).
 
 target datalayout = "E-m:e-i64:64-n32:64"
-target triple = "powerpc64-bgq-linux"
+target triple = "powerpc64le-unknown-linux"
 
 %struct._IO_FILE.119.8249.32639.195239.200117.211499.218003.221255.222881.224507.226133.240767.244019.245645.248897.260279.271661.281417.283043.302555.304181.325319.326945.344713 = type { i32, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, %struct._IO_marker.118.8248.32638.195238.200116.211498.218002.221254.222880.224506.226132.240766.244018.245644.248896.260278.271660.281416.283042.302554.304180.325318.326944.344712*, %struct._IO_FILE.119.8249.32639.195239.200117.211499.218003.221255.222881.224507.226133.240767.244019.245645.248897.260279.271661.281417.283043.302555.304181.325319.326945.344713*, i32, i32, i64, i16, i8, [1 x i8], i8*, i64, i8*, i8*, i8*, i8*, i64, i32, [20 x i8] }
 %struct._IO_marker.118.8248.32638.195238.200116.211498.218002.221254.222880.224506.226132.240766.244018.245644.248896.260278.271660.281416.283042.302554.304180.325318.326944.344712 = type { %struct._IO_marker.118.8248.32638.195238.200116.211498.218002.221254.222880.224506.226132.240766.244018.245644.248896.260278.271660.281416.283042.302554.304180.325318.326944.344712*, %struct._IO_FILE.119.8249.32639.195239.200117.211499.218003.221255.222881.224507.226133.240767.244019.245645.248897.260279.271661.281417.283043.302555.304181.325319.326945.344713*, i32 }
diff --git a/llvm/test/CodeGen/PowerPC/extra-toc-reg-deps.ll b/llvm/test/CodeGen/PowerPC/extra-toc-reg-deps.ll
index e066b45d3ca4b..023928bcb5896 100644
--- a/llvm/test/CodeGen/PowerPC/extra-toc-reg-deps.ll
+++ b/llvm/test/CodeGen/PowerPC/extra-toc-reg-deps.ll
@@ -1,6 +1,6 @@
-; RUN: llc -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mcpu=pwr8 < %s | FileCheck %s
 target datalayout = "E-m:e-i64:64-n32:64"
-target triple = "powerpc64-bgq-linux"
+target triple = "powerpc64-unknown-linux"
 
 %"class.Foam::messageStream.6" = type <{ %"class.Foam::string.5", i32, i32, i32, [4 x i8] }>
 %"class.Foam::string.5" = type { %"class.std::basic_string.4" }
@@ -419,8 +419,8 @@ declare void @_ZN4Foam11regIOobjectD2Ev() #0
 
 declare void @_ZN4Foam6reduceIiNS_5sumOpIiEEEEvRKNS_4ListINS_8UPstream11commsStructEEERT_RKT0_ii() #0
 
-attributes #0 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="a2q" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { inlinehint "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="a2q" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { inlinehint "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.module.flags = !{!0}
 
diff --git a/llvm/test/CodeGen/PowerPC/fast-isel-icmp-split.ll b/llvm/test/CodeGen/PowerPC/fast-isel-icmp-split.ll
index fdd0fc2767803..b08b050f2c2fd 100644
--- a/llvm/test/CodeGen/PowerPC/fast-isel-icmp-split.ll
+++ b/llvm/test/CodeGen/PowerPC/fast-isel-icmp-split.ll
@@ -1,6 +1,6 @@
 ; RUN: llc -verify-machineinstrs -O0 -relocation-model=pic < %s | FileCheck %s
 target datalayout = "E-m:e-i64:64-n32:64"
-target triple = "powerpc64-bgq-linux"
+target triple = "powerpc64le-unknown-linux"
 
 %"class.std::__1::__tree_node.130.151" = type { %"class.std::__1::__tree_node_base.base.128.149", %"class.boost::serialization::extended_type_info.129.150"* }
 %"class.std::__1::__tree_node_base.base.128.149" = type <{ %"class.std::__1::__tree_end_node.127.148", %"class.std::__1::__tree_node_base.126.147"*, %"class.std::__1::__tree_node_base.126.147"*, i8 }>
diff --git a/llvm/test/CodeGen/PowerPC/fma-mutate-duplicate-vreg.ll b/llvm/test/CodeGen/PowerPC/fma-mutate-duplicate-vreg.ll
index eef6e0ccac02b..a336fc796ca52 100644
--- a/llvm/test/CodeGen/PowerPC/fma-mutate-duplicate-vreg.ll
+++ b/llvm/test/CodeGen/PowerPC/fma-mutate-duplicate-vreg.ll
@@ -33,4 +33,4 @@ define float @f(float %xf) #0 {
   ret float %25
 }
 
-attributes #0 = { norecurse nounwind readnone "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pwr8" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse nounwind readnone "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pwr8" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/PowerPC/fp2int2fp-ppcfp128.ll b/llvm/test/CodeGen/PowerPC/fp2int2fp-ppcfp128.ll
index 2feb4556dfab7..3b555cf898f57 100644
--- a/llvm/test/CodeGen/PowerPC/fp2int2fp-ppcfp128.ll
+++ b/llvm/test/CodeGen/PowerPC/fp2int2fp-ppcfp128.ll
@@ -1,6 +1,5 @@
-; RUN: llc -verify-machineinstrs -mcpu=a2 < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mcpu=a2 -mtriple=powerpc64-unknown-linux-gnu < %s | FileCheck %s
 target datalayout = "E-m:e-i64:64-n32:64"
-target triple = "powerpc64-bgq-linux"
 
 define linkonce_odr double @test1(ppc_fp128 %input) {
 entry:
diff --git a/llvm/test/CodeGen/PowerPC/glob-comp-aa-crash.ll b/llvm/test/CodeGen/PowerPC/glob-comp-aa-crash.ll
index 54c3e11528b7b..2aa5239f25eb8 100644
--- a/llvm/test/CodeGen/PowerPC/glob-comp-aa-crash.ll
+++ b/llvm/test/CodeGen/PowerPC/glob-comp-aa-crash.ll
@@ -1,6 +1,6 @@
-; RUN: llc -verify-machineinstrs -mtriple=powerpc64-bgq-linux -mcpu=a2 < %s
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux -mcpu=a2 < %s
 target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
-target triple = "powerpc64-bgq-linux"
+target triple = "powerpc64le-unknown-linux"
 
 %"class.std::__1::__assoc_sub_state" = type { %"class.std::__1::__shared_count", %"class.std::__exception_ptr::exception_ptr", %"class.std::__1::mutex", %"class.std::__1::condition_variable", i32 }
 %"class.std::__1::__shared_count" = type { i32 (...)**, i64 }
diff --git a/llvm/test/CodeGen/PowerPC/ifcvt-forked-bug-2016-08-08.ll b/llvm/test/CodeGen/PowerPC/ifcvt-forked-bug-2016-08-08.ll
index 74bfa75e5e313..a2d0eb599f91d 100644
--- a/llvm/test/CodeGen/PowerPC/ifcvt-forked-bug-2016-08-08.ll
+++ b/llvm/test/CodeGen/PowerPC/ifcvt-forked-bug-2016-08-08.ll
@@ -33,5 +33,5 @@ declare i8* @_ZN11__sanitizer21internal_start_threadEPFvPvES0_(void (i8*)*, i8*)
 
 declare hidden void @_ZN11__sanitizer16BackgroundThreadEPv(i8* nocapture readnone) #5
 
-attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #7 = { nobuiltin nounwind }
diff --git a/llvm/test/CodeGen/PowerPC/inlineasm-i64-reg.ll b/llvm/test/CodeGen/PowerPC/inlineasm-i64-reg.ll
index e4dfd6c58f0e8..6f1bc76d816ae 100644
--- a/llvm/test/CodeGen/PowerPC/inlineasm-i64-reg.ll
+++ b/llvm/test/CodeGen/PowerPC/inlineasm-i64-reg.ll
@@ -1,6 +1,6 @@
-; RUN: llc -verify-machineinstrs -mtriple=powerpc64-bgq-linux -mcpu=a2 < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux -mcpu=a2 < %s | FileCheck %s
 target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
-target triple = "powerpc64-bgq-linux"
+target triple = "powerpc64le-unknown-linux"
 
 %struct.BG_CoordinateMapping_t = type { [4 x i8] }
 
diff --git a/llvm/test/CodeGen/PowerPC/load-two-flts.ll b/llvm/test/CodeGen/PowerPC/load-two-flts.ll
index 1cfcff5e01601..19e21faf47232 100644
--- a/llvm/test/CodeGen/PowerPC/load-two-flts.ll
+++ b/llvm/test/CodeGen/PowerPC/load-two-flts.ll
@@ -1,6 +1,5 @@
-; RUN: llc -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu < %s | FileCheck %s
 target datalayout = "E-m:e-i64:64-n32:64"
-target triple = "powerpc64-bgq-linux"
 
 define void @_Z4testSt7complexIfE(float %v0, float %v1, i64* %ref.tmp, float* %_M_value.realp.i.i, float* %_M_value.imagp.i.i) {
 entry:
diff --git a/llvm/test/CodeGen/PowerPC/loop-data-prefetch-inner.ll b/llvm/test/CodeGen/PowerPC/loop-data-prefetch-inner.ll
index f4664788930d4..2cbb70bb14cb5 100644
--- a/llvm/test/CodeGen/PowerPC/loop-data-prefetch-inner.ll
+++ b/llvm/test/CodeGen/PowerPC/loop-data-prefetch-inner.ll
@@ -1,6 +1,6 @@
-; RUN: llc -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -enable-ppc-prefetching=true -verify-machineinstrs < %s | FileCheck %s
 target datalayout = "E-m:e-i64:64-n32:64"
-target triple = "powerpc64-bgq-linux"
+target triple = "powerpc64le-unknown-linux"
 
 ; Function Attrs: nounwind
 define void @foo(double* %x, double* nocapture readonly %y) #0 {
diff --git a/llvm/test/CodeGen/PowerPC/loop-data-prefetch.ll b/llvm/test/CodeGen/PowerPC/loop-data-prefetch.ll
index f4821564c202b..defc52eec8e0d 100644
--- a/llvm/test/CodeGen/PowerPC/loop-data-prefetch.ll
+++ b/llvm/test/CodeGen/PowerPC/loop-data-prefetch.ll
@@ -1,6 +1,6 @@
-; RUN: llc -verify-machineinstrs -mcpu=a2 < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -enable-ppc-prefetching=true -mcpu=a2 < %s | FileCheck %s
 target datalayout = "E-m:e-i64:64-n32:64"
-target triple = "powerpc64-bgq-linux"
+target triple = "powerpc64le-unknown-linux"
 
 ; Function Attrs: nounwind
 define void @foo(double* nocapture %a, double* nocapture readonly %b) #0 {
diff --git a/llvm/test/CodeGen/PowerPC/loop-prep-all.ll b/llvm/test/CodeGen/PowerPC/loop-prep-all.ll
index a13192d3e6586..7fdabcd4be210 100644
--- a/llvm/test/CodeGen/PowerPC/loop-prep-all.ll
+++ b/llvm/test/CodeGen/PowerPC/loop-prep-all.ll
@@ -1,5 +1,4 @@
 ; RUN: llc -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -verify-machineinstrs -mtriple=powerpc64-bgq-linux < %s | FileCheck %s -check-prefix=CHECK -check-prefix=CHECK-BGQ
 target datalayout = "E-m:e-i64:64-n32:64"
 target triple = "powerpc64-unknown-linux-gnu"
 
@@ -21,7 +20,6 @@ for.body:                                         ; preds = %for.body, %entry
 
 ; CHECK-LABEL: @foo
 
-; CHECK-BGQ-DAG: dcbt 4, 5
 ; CHECK-DAG: lfdu [[REG1:[0-9]+]], 8({{[0-9]+}})
 ; CHECK-DAG: fadd [[REG2:[0-9]+]], [[REG1]], 0
 ; CHECK-DAG: stfdu [[REG2]], 8({{[0-9]+}})
@@ -34,15 +32,13 @@ for.cond.cleanup6:                                ; preds = %for.body7
 
 for.body7:                                        ; preds = %for.body, %for.body7
   %i3.017 = phi i32 [ %inc9, %for.body7 ], [ 0, %for.body ]
-  tail call void bitcast (void (...)* @bar to void ()*)() #2
+  tail call void bitcast (void (...)* @bar to void ()*)() #0
   %inc9 = add nuw nsw i32 %i3.017, 1
   %exitcond = icmp eq i32 %inc9, 1024
   br i1 %exitcond, label %for.cond.cleanup6, label %for.body7
 }
 
-declare void @bar(...) #1
+declare void @bar(...) 
 
-attributes #0 = { nounwind "target-cpu"="a2q" }
-attributes #1 = { "target-cpu"="a2q" }
-attributes #2 = { nounwind }
+attributes #0 = { nounwind }
 
diff --git a/llvm/test/CodeGen/PowerPC/lxv-aligned-stack-slots.ll b/llvm/test/CodeGen/PowerPC/lxv-aligned-stack-slots.ll
index 93868007d0d36..aa618d2b732c7 100644
--- a/llvm/test/CodeGen/PowerPC/lxv-aligned-stack-slots.ll
+++ b/llvm/test/CodeGen/PowerPC/lxv-aligned-stack-slots.ll
@@ -41,6 +41,6 @@ define void @aligned_slot() #0 {
 ; Function Attrs: argmemonly nounwind
 declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i1) #1
 
-attributes #0 = { nounwind "target-cpu"="pwr9" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+power9-vector,+vsx,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "target-cpu"="pwr9" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+power9-vector,+vsx" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { argmemonly nounwind }
 attributes #2 = { nounwind }
diff --git a/llvm/test/CodeGen/PowerPC/machine-combiner.ll b/llvm/test/CodeGen/PowerPC/machine-combiner.ll
index 2e834b1fe788c..16fc3ee3e5202 100644
--- a/llvm/test/CodeGen/PowerPC/machine-combiner.ll
+++ b/llvm/test/CodeGen/PowerPC/machine-combiner.ll
@@ -1,5 +1,4 @@
 ; RUN: llc -verify-machineinstrs -O3 -mcpu=pwr7 < %s | FileCheck  %s -check-prefix=CHECK -check-prefix=CHECK-PWR
-; RUN: llc -verify-machineinstrs -O3 -mcpu=a2q < %s | FileCheck  %s -check-prefix=CHECK -check-prefix=CHECK-QPX
 ; RUN: llc -verify-machineinstrs -O3 -mcpu=pwr9 < %s | FileCheck  %s -check-prefix=FIXPOINT
 target datalayout = "E-m:e-i64:64-n32:64"
 target triple = "powerpc64-unknown-linux-gnu"
@@ -93,9 +92,6 @@ define float @reassociate_adds5(float %x0, float %x1, float %x2, float %x3, floa
 define <4 x float> @vector_reassociate_adds1(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, <4 x float> %x3) {
 ; CHECK-LABEL: vector_reassociate_adds1:
 ; CHECK:       # %bb.0:
-; CHECK-QPX:       qvfadds [[REG0:[0-9]+]], 1, 2
-; CHECK-QPX:       qvfadds [[REG1:[0-9]+]], 3, 4
-; CHECK-QPX:       qvfadds 1, [[REG0]], [[REG1]]
 ; CHECK-PWR:       xvaddsp [[REG0:[0-9]+]], 34, 35
 ; CHECK-PWR:       xvaddsp [[REG1:[0-9]+]], 36, 37
 ; CHECK-PWR:       xvaddsp 34, [[REG0]], [[REG1]]
@@ -110,9 +106,6 @@ define <4 x float> @vector_reassociate_adds1(<4 x float> %x0, <4 x float> %x1, <
 define <4 x float> @vector_reassociate_adds2(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, <4 x float> %x3) {
 ; CHECK-LABEL: vector_reassociate_adds2:
 ; CHECK:       # %bb.0:
-; CHECK-QPX:       qvfadds [[REG0:[0-9]+]], 1, 2
-; CHECK-QPX:       qvfadds [[REG1:[0-9]+]], 3, 4
-; CHECK-QPX:       qvfadds 1, [[REG0]], [[REG1]]
 ; CHECK-PWR:       xvaddsp [[REG0:[0-9]+]], 34, 35
 ; CHECK-PWR:       xvaddsp [[REG1:[0-9]+]], 36, 37
 ; CHECK-PWR:       xvaddsp 34, [[REG0]], [[REG1]]
@@ -127,9 +120,6 @@ define <4 x float> @vector_reassociate_adds2(<4 x float> %x0, <4 x float> %x1, <
 define <4 x float> @vector_reassociate_adds3(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, <4 x float> %x3) {
 ; CHECK-LABEL: vector_reassociate_adds3:
 ; CHECK:       # %bb.0:
-; CHECK-QPX:       qvfadds [[REG0:[0-9]+]], 1, 2
-; CHECK-QPX:       qvfadds [[REG1:[0-9]+]], 3, 4
-; CHECK-QPX:       qvfadds 1, [[REG0]], [[REG1]]
 ; CHECK-PWR:       xvaddsp [[REG0:[0-9]+]], 34, 35
 ; CHECK-PWR:       xvaddsp [[REG1:[0-9]+]], 36, 37
 ; CHECK-PWR:       xvaddsp 34, [[REG0]], [[REG1]]
@@ -144,9 +134,6 @@ define <4 x float> @vector_reassociate_adds3(<4 x float> %x0, <4 x float> %x1, <
 define <4 x float> @vector_reassociate_adds4(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, <4 x float> %x3) {
 ; CHECK-LABEL: vector_reassociate_adds4:
 ; CHECK:       # %bb.0:
-; CHECK-QPX:       qvfadds [[REG0:[0-9]+]], 1, 2
-; CHECK-QPX:       qvfadds [[REG1:[0-9]+]], 3, 4
-; CHECK-QPX:       qvfadds 1, [[REG0]], [[REG1]]
 ; CHECK-PWR:       xvaddsp [[REG0:[0-9]+]], 34, 35
 ; CHECK-PWR:       xvaddsp [[REG1:[0-9]+]], 36, 37
 ; CHECK-PWR:       xvaddsp 34, [[REG0]], [[REG1]]
@@ -217,9 +204,6 @@ define i64 @reassociate_mulld(i64 %x0, i64 %x1, i64 %x2, i64 %x3) {
 define double @reassociate_mamaa_double(double %0, double %1, double %2, double %3, double %4, double %5) {
 ; CHECK-LABEL: reassociate_mamaa_double:
 ; CHECK:       # %bb.0:
-; CHECK-QPX-DAG:   fmadd [[REG0:[0-9]+]], 4, 3, 2
-; CHECK-QPX-DAG:   fmadd [[REG1:[0-9]+]], 6, 5, 1
-; CHECK-QPX:       fadd 1, [[REG0]], [[REG1]]
 ; CHECK-PWR-DAG:   xsmaddadp 1, 6, 5
 ; CHECK-PWR-DAG:   xsmaddadp 2, 4, 3
 ; CHECK-PWR:       xsadddp 1, 2, 1
@@ -250,9 +234,6 @@ define float @reassociate_mamaa_float(float %0, float %1, float %2, float %3, fl
 define <4 x float> @reassociate_mamaa_vec(<4 x float> %0, <4 x float> %1, <4 x float> %2, <4 x float> %3, <4 x float> %4, <4 x float> %5) {
 ; CHECK-LABEL: reassociate_mamaa_vec:
 ; CHECK:       # %bb.0:
-; CHECK-QPX-DAG:   qvfmadds [[REG0:[0-9]+]], 4, 3, 2
-; CHECK-QPX-DAG:   qvfmadds [[REG1:[0-9]+]], 6, 5, 1
-; CHECK-QPX:       qvfadds 1, [[REG0]], [[REG1]]
 ; CHECK-PWR-DAG:   xvmaddasp [[REG0:[0-9]+]], 39, 38
 ; CHECK-PWR-DAG:   xvmaddasp [[REG1:[0-9]+]], 37, 36
 ; CHECK-PWR:       xvaddsp 34, [[REG1]], [[REG0]]
@@ -268,11 +249,6 @@ define <4 x float> @reassociate_mamaa_vec(<4 x float> %0, <4 x float> %1, <4 x f
 define double @reassociate_mamama_double(double %0, double %1, double %2, double %3, double %4, double %5, double %6, double %7, double %8) {
 ; CHECK-LABEL: reassociate_mamama_double:
 ; CHECK:       # %bb.0:
-; CHECK-QPX:       fmadd [[REG0:[0-9]+]], 2, 1, 7
-; CHECK-QPX-DAG:   fmul [[REG1:[0-9]+]], 4, 3
-; CHECK-QPX-DAG:   fmadd [[REG2:[0-9]+]], 6, 5, [[REG0]]
-; CHECK-QPX-DAG:   fmadd [[REG3:[0-9]+]], 9, 8, [[REG1]]
-; CHECK-QPX:       fadd 1, [[REG2]], [[REG3]]
 ; CHECK-PWR:       xsmaddadp 7, 2, 1
 ; CHECK-PWR-DAG:   xsmuldp [[REG0:[0-9]+]], 4, 3
 ; CHECK-PWR-DAG:   xsmaddadp 7, 6, 5
diff --git a/llvm/test/CodeGen/PowerPC/mc-instrlat.ll b/llvm/test/CodeGen/PowerPC/mc-instrlat.ll
index e135986a2894c..f807f4fa20d25 100644
--- a/llvm/test/CodeGen/PowerPC/mc-instrlat.ll
+++ b/llvm/test/CodeGen/PowerPC/mc-instrlat.ll
@@ -19,7 +19,7 @@ entry:
 
 declare void @bar(double) #1
 
-attributes #0 = { nounwind "no-infs-fp-math"="true" "no-nans-fp-math"="true" "target-cpu"="ppc64" "target-features"="+altivec,-bpermd,-crypto,-direct-move,-extdiv,-power8-vector,-qpx,-vsx" "unsafe-fp-math"="true" "use-soft-float"="false" }
-attributes #1 = { "no-infs-fp-math"="true" "no-nans-fp-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="ppc64" "target-features"="+altivec,-bpermd,-crypto,-direct-move,-extdiv,-power8-vector,-qpx,-vsx" "unsafe-fp-math"="true" "use-soft-float"="false" }
+attributes #0 = { nounwind "no-infs-fp-math"="true" "no-nans-fp-math"="true" "target-cpu"="ppc64" "target-features"="+altivec,-bpermd,-crypto,-direct-move,-extdiv,-power8-vector,-vsx" "unsafe-fp-math"="true" "use-soft-float"="false" }
+attributes #1 = { "no-infs-fp-math"="true" "no-nans-fp-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="ppc64" "target-features"="+altivec,-bpermd,-crypto,-direct-move,-extdiv,-power8-vector,-vsx" "unsafe-fp-math"="true" "use-soft-float"="false" }
 attributes #2 = { nounwind }
 
diff --git a/llvm/test/CodeGen/PowerPC/mcount-insertion.ll b/llvm/test/CodeGen/PowerPC/mcount-insertion.ll
index cbb7947be2198..502347a3af198 100644
--- a/llvm/test/CodeGen/PowerPC/mcount-insertion.ll
+++ b/llvm/test/CodeGen/PowerPC/mcount-insertion.ll
@@ -1,9 +1,8 @@
-; RUN: opt -ee-instrument < %s | opt -inline | llc | FileCheck %s
+; RUN: opt -ee-instrument < %s | opt -inline | llc -mtriple=powerpc64-unknown-linux-gnu | FileCheck %s
 
 ; The run-line mimics how Clang might run the instrumentation passes.
 
 target datalayout = "E-m:e-i64:64-n32:64"
-target triple = "powerpc64-bgq-linux"
 
 
 define void @leaf_function() #0 {
diff --git a/llvm/test/CodeGen/PowerPC/memcpy-vec.ll b/llvm/test/CodeGen/PowerPC/memcpy-vec.ll
index cd0abd6149bde..c4e60f8c4b1f5 100644
--- a/llvm/test/CodeGen/PowerPC/memcpy-vec.ll
+++ b/llvm/test/CodeGen/PowerPC/memcpy-vec.ll
@@ -1,6 +1,5 @@
 ; RUN: llc -verify-machineinstrs -mcpu=pwr7 < %s | FileCheck  %s -check-prefix=PWR7
 ; RUN: llc -verify-machineinstrs -mcpu=pwr8 < %s | FileCheck  %s -check-prefix=PWR8
-; RUN: llc -verify-machineinstrs -mcpu=a2q < %s | FileCheck  %s -check-prefix=A2Q
 target datalayout = "E-m:e-i64:64-n32:64"
 target triple = "powerpc64-unknown-linux-gnu"
 
@@ -25,12 +24,6 @@ entry:
 ; PWR8: lxvw4x
 ; PWR8: stxvw4x
 ; PWR8: blr
-
-; A2Q-LABEL: @foo1
-; A2Q-NOT: bl memcpy
-; A2Q: ld {{[0-9]+}}, {{[0-9]+}}(4)
-; A2Q: std {{[0-9]+}}, {{[0-9]+}}(3)
-; A2Q: blr
 }
 
 ; Function Attrs: nounwind
@@ -52,12 +45,6 @@ entry:
 ; PWR8: lxvw4x
 ; PWR8: stxvw4x
 ; PWR8: blr
-
-; A2Q-LABEL: @foo2
-; A2Q-NOT: bl memcpy
-; A2Q: ld {{[0-9]+}}, {{[0-9]+}}(4)
-; A2Q: std {{[0-9]+}}, {{[0-9]+}}(3)
-; A2Q: blr
 }
 
 ; Function Attrs: nounwind
@@ -76,11 +63,6 @@ entry:
 ; PWR8-NOT: bl memset
 ; PWR8: stxvw4x
 ; PWR8: blr
-
-; A2Q-LABEL: @bar1
-; A2Q-NOT: bl memset
-; A2Q: std {{[0-9]+}}, {{[0-9]+}}(3)
-; A2Q: blr
 }
 
 ; Function Attrs: nounwind
@@ -99,11 +81,6 @@ entry:
 ; PWR8-NOT: bl memset
 ; PWR8: stxvw4x
 ; PWR8: blr
-
-; A2Q-LABEL: @bar2
-; A2Q-NOT: bl memset
-; A2Q: qvstfdx
-; A2Q: blr
 }
 
 ; Function Attrs: nounwind
diff --git a/llvm/test/CodeGen/PowerPC/memset-nc.ll b/llvm/test/CodeGen/PowerPC/memset-nc.ll
deleted file mode 100644
index 663d0cb1d6785..0000000000000
--- a/llvm/test/CodeGen/PowerPC/memset-nc.ll
+++ /dev/null
@@ -1,48 +0,0 @@
-; RUN: llc -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -verify-machineinstrs -O0 < %s | FileCheck %s -check-prefix=CHECK-O0
-target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
-target triple = "powerpc64-bgq-linux"
-
-; Function Attrs: nounwind
-define void @test_qpx() unnamed_addr #0 align 2 {
-entry:
-  %0 = load i32, i32* undef, align 4
-  %1 = trunc i32 %0 to i8
-  call void @llvm.memset.p0i8.i64(i8* align 32 null, i8 %1, i64 64, i1 false)
-  ret void
-
-; CHECK-LABEL: @test_qpx
-; CHECK: qvstfdx
-; CHECK: qvstfdx
-; CHECK: blr
-
-; CHECK-O0-LABEL: @test_qpx
-; CHECK-O0-NOT: qvstfdx
-; CHECK-O0: blr
-}
-
-; Function Attrs: nounwind
-declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i1) #1
-
-; Function Attrs: nounwind
-define void @test_vsx() unnamed_addr #2 align 2 {
-entry:
-  %0 = load i32, i32* undef, align 4
-  %1 = trunc i32 %0 to i8
-  call void @llvm.memset.p0i8.i64(i8* null, i8 %1, i64 32, i1 false)
-  ret void
-
-; CHECK-LABEL: @test_vsx
-; CHECK: stxvw4x
-; CHECK: stxvw4x
-; CHECK: blr
-
-; CHECK-O0-LABEL: @test_vsx
-; CHECK-O0-NOT: stxvw4x
-; CHECK-O0: blr
-}
-
-attributes #0 = { nounwind "target-cpu"="a2q" }
-attributes #1 = { nounwind }
-attributes #2 = { nounwind "target-cpu"="pwr7" }
-
diff --git a/llvm/test/CodeGen/PowerPC/misched-inorder-latency.ll b/llvm/test/CodeGen/PowerPC/misched-inorder-latency.ll
index 26663d81f3575..089c947713b9d 100644
--- a/llvm/test/CodeGen/PowerPC/misched-inorder-latency.ll
+++ b/llvm/test/CodeGen/PowerPC/misched-inorder-latency.ll
@@ -1,8 +1,7 @@
 ; RUN: llc -verify-machineinstrs < %s -enable-misched -pre-RA-sched=source -scheditins=false \
-; RUN:          -disable-ifcvt-triangle-false -disable-post-ra | FileCheck %s
+; RUN:          -disable-ifcvt-triangle-false -disable-post-ra -mtriple=powerpc64-unknown-linux-gnu | FileCheck %s
 ;
 target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
-target triple = "powerpc64-bgq-linux"
 
 ; %val1 is a load live out of %entry. It should be hoisted
 ; above the add.
diff --git a/llvm/test/CodeGen/PowerPC/misched.ll b/llvm/test/CodeGen/PowerPC/misched.ll
index 1c868b3f171c9..9a75fe44b7176 100644
--- a/llvm/test/CodeGen/PowerPC/misched.ll
+++ b/llvm/test/CodeGen/PowerPC/misched.ll
@@ -1,7 +1,6 @@
 ; RUN: llc < %s -enable-misched -verify-machineinstrs
 ; PR14302
 target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
-target triple = "powerpc64-bgq-linux"
 
 @b = external global [16000 x double], align 32
 
diff --git a/llvm/test/CodeGen/PowerPC/optnone-crbits-i1-ret.ll b/llvm/test/CodeGen/PowerPC/optnone-crbits-i1-ret.ll
index f59df4291c48f..ad5976318fe3a 100644
--- a/llvm/test/CodeGen/PowerPC/optnone-crbits-i1-ret.ll
+++ b/llvm/test/CodeGen/PowerPC/optnone-crbits-i1-ret.ll
@@ -1,6 +1,5 @@
-; RUN: llc -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu < %s | FileCheck %s
 target datalayout = "E-m:e-i64:64-n32:64"
-target triple = "powerpc64-bgq-linux"
 
 declare zeroext i1 @ri1()
 declare void @se1()
diff --git a/llvm/test/CodeGen/PowerPC/pcrel-local-caller-toc.ll b/llvm/test/CodeGen/PowerPC/pcrel-local-caller-toc.ll
index 2e248506c7b7b..2871e077df565 100644
--- a/llvm/test/CodeGen/PowerPC/pcrel-local-caller-toc.ll
+++ b/llvm/test/CodeGen/PowerPC/pcrel-local-caller-toc.ll
@@ -92,7 +92,7 @@ entry:
 
 ; Left the target features in this test because it is important that caller has
 ; -pcrelative-memops while callee has +pcrelative-memops
-attributes #0 = { nounwind "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+pcrelative-memops,+power8-vector,+power9-vector,+vsx,-htm,-qpx,-spe" }
-attributes #1 = { "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+pcrelative-memops,+power8-vector,+power9-vector,+vsx,-htm,-qpx,-spe" }
-attributes #2 = { nounwind "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+power9-vector,+vsx,-htm,-pcrelative-memops,-qpx,-spe" }
+attributes #0 = { nounwind "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+pcrelative-memops,+power8-vector,+power9-vector,+vsx,-htm,-spe" }
+attributes #1 = { "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+pcrelative-memops,+power8-vector,+power9-vector,+vsx,-htm,-spe" }
+attributes #2 = { nounwind "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+power9-vector,+vsx,-htm,-pcrelative-memops,-spe" }
 attributes #3 = { nounwind }
diff --git a/llvm/test/CodeGen/PowerPC/popcnt.ll b/llvm/test/CodeGen/PowerPC/popcnt.ll
index a06c59d4b945a..695863d87f16e 100644
--- a/llvm/test/CodeGen/PowerPC/popcnt.ll
+++ b/llvm/test/CodeGen/PowerPC/popcnt.ll
@@ -1,8 +1,6 @@
 ; RUN: llc -verify-machineinstrs -mtriple=ppc64-- -mattr=+popcntd < %s | FileCheck %s
 ; RUN: llc -verify-machineinstrs -mtriple=ppc64-- -mattr=+slow-popcntd < %s | FileCheck %s --check-prefix=SLOWPC
 ; RUN: llc -verify-machineinstrs -mtriple=ppc64-- -mcpu=pwr7 < %s | FileCheck %s
-; RUN: llc -verify-machineinstrs -mtriple=ppc64-- -mcpu=a2q < %s | FileCheck %s --check-prefix=SLOWPC
-; RUN: llc -verify-machineinstrs -mtriple=ppc64-- -mcpu=a2q -mattr=+popcntd < %s | FileCheck %s
 
 define i64 @_cntb64(i64 %x) nounwind readnone {
   %cnt = tail call i64 @llvm.ppc.popcntb(i64 %x)
diff --git a/llvm/test/CodeGen/PowerPC/ppc-passname.ll b/llvm/test/CodeGen/PowerPC/ppc-passname.ll
index 98343bdb535c2..06f13278d84cd 100644
--- a/llvm/test/CodeGen/PowerPC/ppc-passname.ll
+++ b/llvm/test/CodeGen/PowerPC/ppc-passname.ll
@@ -105,14 +105,3 @@
 ; STOP-AFTER-BRANCH-COALESCING-NOT: "ppc-branch-coalescing" pass is not registered.
 ; STOP-AFTER-BRANCH-COALESCING: Branch Coalescing 
 
-
-; Test pass name: ppc-qpx-load-splat.
-; RUN: llc -mtriple=powerpc64le-unknown-unknown < %s -debug-pass=Structure -stop-before=ppc-qpx-load-splat -o /dev/null 2>&1 | FileCheck %s -check-prefix=STOP-BEFORE-QPX-LOAD-SPLAT
-; STOP-BEFORE-QPX-LOAD-SPLAT-NOT: -ppc-qpx-load-splat
-; STOP-BEFORE-QPX-LOAD-SPLAT-NOT: "ppc-qpx-load-splat" pass is not registered.
-; STOP-BEFORE-QPX-LOAD-SPLAT-NOT: PowerPC QPX Load Splat Simplification
-
-; RUN: llc -mtriple=powerpc64le-unknown-unknown < %s -debug-pass=Structure -stop-after=ppc-qpx-load-splat -o /dev/null 2>&1 | FileCheck %s -check-prefix=STOP-AFTER-QPX-LOAD-SPLAT
-; STOP-AFTER-QPX-LOAD-SPLAT: -ppc-qpx-load-splat
-; STOP-AFTER-QPX-LOAD-SPLAT-NOT: "ppc-qpx-load-splat" pass is not registered.
-; STOP-AFTER-QPX-LOAD-SPLAT: PowerPC QPX Load Splat Simplification
diff --git a/llvm/test/CodeGen/PowerPC/ppc64-sibcall.ll b/llvm/test/CodeGen/PowerPC/ppc64-sibcall.ll
index fc0e71f878cab..357f28e88b184 100644
--- a/llvm/test/CodeGen/PowerPC/ppc64-sibcall.ll
+++ b/llvm/test/CodeGen/PowerPC/ppc64-sibcall.ll
@@ -1,6 +1,6 @@
 ; RUN: llc < %s -relocation-model=static -O1 -disable-ppc-sco=false -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu | FileCheck %s -check-prefix=CHECK-SCO
-; RUN: llc < %s -relocation-model=static -O1 -disable-ppc-sco=false -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr8 | FileCheck %s -check-prefix=CHECK-SCO-HASQPX
-; RUN: llc < %s -relocation-model=static -O1 -disable-ppc-sco=false -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -mcpu=pwr8 | FileCheck %s -check-prefix=CHECK-SCO-HASQPX
+; RUN: llc < %s -relocation-model=static -O1 -disable-ppc-sco=false -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr8 | FileCheck %s -check-prefix=CHECK-SCO
+; RUN: llc < %s -relocation-model=static -O1 -disable-ppc-sco=false -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -mcpu=pwr8 | FileCheck %s -check-prefix=CHECK-SCO
 ; RUN: llc < %s -relocation-model=static -O1 -disable-ppc-sco=false -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -mcpu=pwr8 -code-model=small | FileCheck %s -check-prefix=SCM
 
 ; No combination of "powerpc64le-unknown-linux-gnu" + "CHECK-SCO", because
@@ -117,23 +117,6 @@ define void @caller_local_sret_32(%S_32* %a) #1 {
 attributes #0 = { noinline nounwind  }
 attributes #1 = { nounwind }
 
-; vector <4 x i1> test
-
-define void @callee_v4i1(i8 %a, <4 x i1> %b, <4 x i1> %c) { ret void }
-define void @caller_v4i1_reorder(i8 %a, <4 x i1> %b, <4 x i1> %c) {
-  tail call void @callee_v4i1(i8 %a, <4 x i1> %c, <4 x i1> %b)
-  ret void
-
-; <4 x i1> is 32 bytes aligned, if subtarget doesn't support qpx, then we can't
-; place b, c to qpx register, so we can't do sco on caller_v4i1_reorder
-
-; CHECK-SCO-LABEL: caller_v4i1_reorder:
-; CHECK-SCO: bl callee_v4i1
-
-; CHECK-SCO-HASQPX-LABEL: caller_v4i1_reorder:
-; CHECK-SCO-HASQPX: b callee_v4i1
-}
-
 define void @f128_callee(i32* %ptr, ppc_fp128 %a, ppc_fp128 %b) { ret void }
 define void @f128_caller(i32* %ptr, ppc_fp128 %a, ppc_fp128 %b) {
   tail call void @f128_callee(i32* %ptr, ppc_fp128 %a, ppc_fp128 %b)
diff --git a/llvm/test/CodeGen/PowerPC/pr24546.ll b/llvm/test/CodeGen/PowerPC/pr24546.ll
index 28c03293680e5..028fd2d8f0064 100644
--- a/llvm/test/CodeGen/PowerPC/pr24546.ll
+++ b/llvm/test/CodeGen/PowerPC/pr24546.ll
@@ -47,8 +47,8 @@ declare double @pow(double, double) #0
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.value(metadata, i64, metadata, metadata) #2
 
-attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind readnone }
 attributes #3 = { nounwind }
 
diff --git a/llvm/test/CodeGen/PowerPC/pr27350.ll b/llvm/test/CodeGen/PowerPC/pr27350.ll
index 982023a1fcdc8..93dbd10fecdeb 100644
--- a/llvm/test/CodeGen/PowerPC/pr27350.ll
+++ b/llvm/test/CodeGen/PowerPC/pr27350.ll
@@ -18,7 +18,7 @@ entry:
 declare fastcc void @bar([2 x i64], [2 x i64]) unnamed_addr #1 align 2
 
 attributes #0 = { argmemonly nounwind }
-attributes #1 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind }
 
 !llvm.ident = !{!0}
diff --git a/llvm/test/CodeGen/PowerPC/pr28130.ll b/llvm/test/CodeGen/PowerPC/pr28130.ll
index cb703dfda8a59..4da415bd29269 100644
--- a/llvm/test/CodeGen/PowerPC/pr28130.ll
+++ b/llvm/test/CodeGen/PowerPC/pr28130.ll
@@ -67,4 +67,4 @@ bb:
   ret void
 }
 
-attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pwr8" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pwr8" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/PowerPC/preinc-ld-sel-crash.ll b/llvm/test/CodeGen/PowerPC/preinc-ld-sel-crash.ll
index 04dee1ee182bb..35aec57ec2640 100644
--- a/llvm/test/CodeGen/PowerPC/preinc-ld-sel-crash.ll
+++ b/llvm/test/CodeGen/PowerPC/preinc-ld-sel-crash.ll
@@ -1,6 +1,6 @@
 ; RUN: llc -verify-machineinstrs < %s | FileCheck %s
 target datalayout = "E-m:e-i64:64-n32:64"
-target triple = "powerpc64-bgq-linux"
+target triple = "powerpc64le-unknown-linux"
 
 %t1 = type { %t2*, %t3* }
 %t2 = type <{ %t3*, i32, [4 x i8] }>
diff --git a/llvm/test/CodeGen/PowerPC/qpx-bv-sint.ll b/llvm/test/CodeGen/PowerPC/qpx-bv-sint.ll
deleted file mode 100644
index 4e0aef4c3df71..0000000000000
--- a/llvm/test/CodeGen/PowerPC/qpx-bv-sint.ll
+++ /dev/null
@@ -1,33 +0,0 @@
-; RUN: llc -verify-machineinstrs < %s -mcpu=a2q | FileCheck %s
-target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
-target triple = "powerpc64-bgq-linux"
-
-define void @s452(i32 %inp1) nounwind {
-entry:
-  br label %for.body4
-
-for.body4:                                        ; preds = %for.body4, %entry
-  %conv.4 = sitofp i32 %inp1 to double
-  %conv.5 = sitofp i32 %inp1 to double
-  %mul.4.v.i0.1 = insertelement <2 x double> undef, double %conv.4, i32 0
-  %v = insertelement <2 x double> %mul.4.v.i0.1, double %conv.5, i32 1
-  %vv = fmul <2 x double> %v, %v
-  %add7.4 = fadd <2 x double> %vv, %vv
-  store <2 x double> %add7.4, <2 x double>* undef, align 16
-  br i1 undef, label %for.end, label %for.body4
-
-for.end:                                          ; preds = %for.body4
-  unreachable
-; CHECK-LABEL: @s452
-; CHECK: lfiwax [[REG1:[0-9]+]],
-; CHECK: fcfid [[REG2:[0-9]+]], [[REG1]]
-; FIXME: We could 'promote' this to a vector earlier and remove this splat.
-; CHECK: qvesplati {{[0-9]+}}, [[REG2]], 0
-; CHECK: qvfmul
-; CHECK: qvfadd
-; CHECK: qvesplati {{[0-9]+}},
-; FIXME: We can use qvstfcdx here instead of two stores.
-; CHECK: stfd
-; CHECK: stfd
-}
-
diff --git a/llvm/test/CodeGen/PowerPC/qpx-bv.ll b/llvm/test/CodeGen/PowerPC/qpx-bv.ll
deleted file mode 100644
index 93a739b864c1d..0000000000000
--- a/llvm/test/CodeGen/PowerPC/qpx-bv.ll
+++ /dev/null
@@ -1,37 +0,0 @@
-; RUN: llc -verify-machineinstrs < %s -mcpu=a2q | FileCheck %s
-
-target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
-target triple = "powerpc64-bgq-linux"
-
-define <4 x double> @foo(double %f1, double %f2, double %f3, double %f4) {
-  %v1 = insertelement <4 x double> undef, double %f1, i32 0
-  %v2 = insertelement <4 x double> %v1,   double %f2, i32 1
-  %v3 = insertelement <4 x double> %v2,   double %f3, i32 2
-  %v4 = insertelement <4 x double> %v3,   double %f4, i32 3
-  ret <4 x double> %v4
-
-; CHECK-LABEL: @foo
-; CHECK: qvgpci [[REG1:[0-9]+]], 275
-; CHECK-DAG: qvgpci [[REG2:[0-9]+]], 101
-; CHECK-DAG: qvfperm [[REG3:[0-9]+]], 3, 4, [[REG1]]
-; CHECK-DAG: qvfperm [[REG4:[0-9]+]], 1, 2, [[REG1]]
-; CHECK-DAG: qvfperm 1, [[REG4]], [[REG3]], [[REG2]]
-; CHECK: blr
-}
-
-define <4 x float> @goo(float %f1, float %f2, float %f3, float %f4) {
-  %v1 = insertelement <4 x float> undef, float %f1, i32 0
-  %v2 = insertelement <4 x float> %v1,   float %f2, i32 1
-  %v3 = insertelement <4 x float> %v2,   float %f3, i32 2
-  %v4 = insertelement <4 x float> %v3,   float %f4, i32 3
-  ret <4 x float> %v4
-
-; CHECK-LABEL: @goo
-; CHECK: qvgpci [[REG1:[0-9]+]], 275
-; CHECK-DAG: qvgpci [[REG2:[0-9]+]], 101
-; CHECK-DAG: qvfperm [[REG3:[0-9]+]], 3, 4, [[REG1]]
-; CHECK-DAG: qvfperm [[REG4:[0-9]+]], 1, 2, [[REG1]]
-; CHECK-DAG: qvfperm 1, [[REG4]], [[REG3]], [[REG2]]
-; CHECK: blr
-}
-
diff --git a/llvm/test/CodeGen/PowerPC/qpx-func-clobber.ll b/llvm/test/CodeGen/PowerPC/qpx-func-clobber.ll
deleted file mode 100644
index ccbbd162a0cdb..0000000000000
--- a/llvm/test/CodeGen/PowerPC/qpx-func-clobber.ll
+++ /dev/null
@@ -1,22 +0,0 @@
-; RUN: llc -verify-machineinstrs < %s -mcpu=a2q | FileCheck %s
-target triple = "powerpc64-bgq-linux"
-
-declare <4 x double> @foo(<4 x double> %p)
-
-define <4 x double> @bar(<4 x double> %p, <4 x double> %q) {
-entry:
-  %v = call <4 x double> @foo(<4 x double> %p)
-  %w = call <4 x double> @foo(<4 x double> %q)
-  %x = fadd <4 x double> %v, %w
-  ret <4 x double> %x
-
-; CHECK-LABEL: @bar
-; CHECK: qvstfdx 2,
-; CHECK: bl foo
-; CHECK: qvstfdx 1,
-; CHECK: qvlfdx 1,
-; CHECK: bl foo
-; CHECK: qvlfdx [[REG:[0-9]+]],
-; CHECK: qvfadd 1, [[REG]], 1
-}
-
diff --git a/llvm/test/CodeGen/PowerPC/qpx-load-splat.ll b/llvm/test/CodeGen/PowerPC/qpx-load-splat.ll
deleted file mode 100644
index 50b864980d985..0000000000000
--- a/llvm/test/CodeGen/PowerPC/qpx-load-splat.ll
+++ /dev/null
@@ -1,80 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=powerpc64le-unknown-linux-gnu -ppc-vsr-nums-as-vr \
-; RUN:   -ppc-asm-full-reg-names -verify-machineinstrs < %s | FileCheck %s
-
-; Function Attrs: norecurse nounwind readonly
-define <4 x double> @foo(double* nocapture readonly %a) #0 {
-; CHECK-LABEL: foo:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    lxvdsx v2, 0, r3
-; CHECK-NEXT:    vmr v3, v2
-; CHECK-NEXT:    blr
-entry:
-  %0 = load double, double* %a, align 8
-  %vecinit.i = insertelement <4 x double> undef, double %0, i32 0
-  %shuffle.i = shufflevector <4 x double> %vecinit.i, <4 x double> undef, <4 x i32> zeroinitializer
-  ret <4 x double> %shuffle.i
-}
-
-define <4 x double> @foox(double* nocapture readonly %a, i64 %idx) #0 {
-; CHECK-LABEL: foox:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    sldi r4, r4, 3
-; CHECK-NEXT:    lxvdsx v2, r3, r4
-; CHECK-NEXT:    vmr v3, v2
-; CHECK-NEXT:    blr
-entry:
-  %p = getelementptr double, double* %a, i64 %idx
-  %0 = load double, double* %p, align 8
-  %vecinit.i = insertelement <4 x double> undef, double %0, i32 0
-  %shuffle.i = shufflevector <4 x double> %vecinit.i, <4 x double> undef, <4 x i32> zeroinitializer
-  ret <4 x double> %shuffle.i
-}
-
-define <4 x double> @fooxu(double* nocapture readonly %a, i64 %idx, double** %pptr) #0 {
-; CHECK-LABEL: fooxu:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    sldi r4, r4, 3
-; CHECK-NEXT:    add r6, r3, r4
-; CHECK-NEXT:    std r6, 0(r5)
-; CHECK-NEXT:    lxvdsx v2, r3, r4
-; CHECK-NEXT:    vmr v3, v2
-; CHECK-NEXT:    blr
-entry:
-  %p = getelementptr double, double* %a, i64 %idx
-  %0 = load double, double* %p, align 8
-  %vecinit.i = insertelement <4 x double> undef, double %0, i32 0
-  %shuffle.i = shufflevector <4 x double> %vecinit.i, <4 x double> undef, <4 x i32> zeroinitializer
-  store double* %p, double** %pptr, align 8
-  ret <4 x double> %shuffle.i
-}
-
-define <4 x float> @foof(float* nocapture readonly %a) #0 {
-; CHECK-LABEL: foof:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    lfiwzx f0, 0, r3
-; CHECK-NEXT:    xxspltw v2, vs0, 1
-; CHECK-NEXT:    blr
-entry:
-  %0 = load float, float* %a, align 4
-  %vecinit.i = insertelement <4 x float> undef, float %0, i32 0
-  %shuffle.i = shufflevector <4 x float> %vecinit.i, <4 x float> undef, <4 x i32> zeroinitializer
-  ret <4 x float> %shuffle.i
-}
-
-define <4 x float> @foofx(float* nocapture readonly %a, i64 %idx) #0 {
-; CHECK-LABEL: foofx:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    sldi r4, r4, 2
-; CHECK-NEXT:    lfiwzx f0, r3, r4
-; CHECK-NEXT:    xxspltw v2, vs0, 1
-; CHECK-NEXT:    blr
-entry:
-  %p = getelementptr float, float* %a, i64 %idx
-  %0 = load float, float* %p, align 4
-  %vecinit.i = insertelement <4 x float> undef, float %0, i32 0
-  %shuffle.i = shufflevector <4 x float> %vecinit.i, <4 x float> undef, <4 x i32> zeroinitializer
-  ret <4 x float> %shuffle.i
-}
-
-
diff --git a/llvm/test/CodeGen/PowerPC/qpx-load.ll b/llvm/test/CodeGen/PowerPC/qpx-load.ll
deleted file mode 100644
index 514f0934b6cfc..0000000000000
--- a/llvm/test/CodeGen/PowerPC/qpx-load.ll
+++ /dev/null
@@ -1,26 +0,0 @@
-; RUN: llc -verify-machineinstrs < %s -mcpu=a2q | FileCheck %s
-target triple = "powerpc64-bgq-linux"
-
-define <4 x double> @foo(<4 x double>* %p) {
-entry:
-  %v = load <4 x double>, <4 x double>* %p, align 8
-  ret <4 x double> %v
-}
-
-; CHECK: @foo
-; CHECK-DAG: li [[REG1:[0-9]+]], 31
-; CHECK-DAG: qvlfdx [[REG4:[0-9]+]], 0, 3
-; CHECK-DAG: qvlfdx [[REG2:[0-9]+]], 3, [[REG1]]
-; CHECK-DAG: qvlpcldx [[REG3:[0-9]+]], 0, 3
-; CHECK-DAG: qvfperm 1, [[REG4]], [[REG2]], [[REG3]]
-; CHECK: blr
-
-define <4 x double> @bar(<4 x double>* %p) {
-entry:
-  %v = load <4 x double>, <4 x double>* %p, align 32
-  ret <4 x double> %v
-}
-
-; CHECK: @bar
-; CHECK: qvlfdx
-
diff --git a/llvm/test/CodeGen/PowerPC/qpx-qvfmadd.ll b/llvm/test/CodeGen/PowerPC/qpx-qvfmadd.ll
deleted file mode 100644
index eab4d6af7e9fc..0000000000000
--- a/llvm/test/CodeGen/PowerPC/qpx-qvfmadd.ll
+++ /dev/null
@@ -1,79 +0,0 @@
-; RUN: llc -verify-machineinstrs -stop-after=finalize-isel < %s -mcpu=a2q | FileCheck %s
-target triple = "powerpc64-bgq-linux"
-
-define <2 x double> @test_qvfmadd(<2 x double> %0, <2 x double> %1, <2 x double> %2) {
-; CHECK: test_qvfmadd
-; CHECK: QVFMADD %2, %1, %0, implicit $rm
-;
-  %4 = fmul reassoc nsz <2 x double> %2, %1
-  %5 = fadd reassoc nsz <2 x double> %4, %0
-  ret <2 x double> %5
-}
-
-define <4 x float> @test_qvfmadds(<4 x float> %0, <4 x float> %1, <4 x float> %2) {
-; CHECK: test_qvfmadds
-; CHECK: QVFMADDSs %2, %1, %0, implicit $rm
-;
-  %4 = fmul reassoc nsz <4 x float> %2, %1
-  %5 = fadd reassoc nsz <4 x float> %4, %0
-  ret <4 x float> %5
-}
-
-define <2 x double> @test_qvfnmadd(<2 x double> %0, <2 x double> %1, <2 x double> %2) {
-; CHECK: test_qvfnmadd
-; CHECK: QVFNMADD %2, %1, %0, implicit $rm
-;
-  %4 = fmul reassoc nsz <2 x double> %2, %1
-  %5 = fadd reassoc nsz <2 x double> %4, %0
-  %6 = fneg reassoc nsz <2 x double> %5
-  ret <2 x double> %6
-}
-
-define <4 x float> @test_qvfnmadds(<4 x float> %0, <4 x float> %1, <4 x float> %2) {
-; CHECK: test_qvfnmadds
-; CHECK: QVFNMADDSs %2, %1, %0, implicit $rm
-;
-  %4 = fmul reassoc nsz <4 x float> %2, %1
-  %5 = fadd reassoc nsz <4 x float> %4, %0
-  %6 = fneg reassoc nsz <4 x float> %5
-  ret <4 x float> %6
-}
-
-define <2 x double> @test_qvfmsub(<2 x double> %0, <2 x double> %1, <2 x double> %2) {
-; CHECK: test_qvfmsub
-; CHECK: QVFMSUB %2, %1, %0, implicit $rm
-;
-  %4 = fmul reassoc nsz <2 x double> %2, %1
-  %5 = fsub reassoc nsz <2 x double> %4, %0
-  ret <2 x double> %5
-}
-
-define <4 x float> @test_qvfmsubs(<4 x float> %0, <4 x float> %1, <4 x float> %2) {
-; CHECK: test_qvfmsubs
-; CHECK: QVFMSUBSs %2, %1, %0, implicit $rm
-;
-  %4 = fmul reassoc nsz <4 x float> %2, %1
-  %5 = fsub reassoc nsz <4 x float> %4, %0
-  ret <4 x float> %5
-}
-
-define <2 x double> @test_qvfnmsub(<2 x double> %0, <2 x double> %1, <2 x double> %2) {
-; CHECK: test_qvfnmsub
-; CHECK: QVFNMSUB %2, %1, %0, implicit $rm
-;
-  %4 = fmul reassoc nsz <2 x double> %2, %1
-  %5 = fsub reassoc nsz <2 x double> %4, %0
-  %6 = fneg reassoc nsz <2 x double> %5
-  ret <2 x double> %6
-}
-
-define <4 x float> @test_qvfnmsubs(<4 x float> %0, <4 x float> %1, <4 x float> %2) {
-; CHECK: test_qvfnmsubs
-; CHECK: QVFNMSUBSs %2, %1, %0, implicit $rm
-;
-  %4 = fmul reassoc nsz <4 x float> %2, %1
-  %5 = fsub reassoc nsz <4 x float> %4, %0
-  %6 = fneg reassoc nsz <4 x float> %5
-  ret <4 x float> %6
-}
-
diff --git a/llvm/test/CodeGen/PowerPC/qpx-recipest.ll b/llvm/test/CodeGen/PowerPC/qpx-recipest.ll
deleted file mode 100644
index 498ab62819ced..0000000000000
--- a/llvm/test/CodeGen/PowerPC/qpx-recipest.ll
+++ /dev/null
@@ -1,473 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc64-unknown-linux-gnu -mcpu=a2q | FileCheck %s
-target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
-target triple = "powerpc64-unknown-linux-gnu"
-
-declare <4 x double> @llvm.sqrt.v4f64(<4 x double>)
-declare <4 x float> @llvm.sqrt.v4f32(<4 x float>)
-
-define <4 x double> @foo_fmf(<4 x double> %a, <4 x double> %b) nounwind {
-; CHECK-LABEL: foo_fmf:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    addis 3, 2, .LCPI0_0@toc@ha
-; CHECK-NEXT:    qvfrsqrte 3, 2
-; CHECK-NEXT:    addi 3, 3, .LCPI0_0@toc@l
-; CHECK-NEXT:    qvlfdx 0, 0, 3
-; CHECK-NEXT:    qvfmul 4, 3, 3
-; CHECK-NEXT:    qvfmsub 2, 2, 0, 2
-; CHECK-NEXT:    qvfnmsub 4, 2, 4, 0
-; CHECK-NEXT:    qvfmul 3, 3, 4
-; CHECK-NEXT:    qvfmul 4, 3, 3
-; CHECK-NEXT:    qvfnmsub 0, 2, 4, 0
-; CHECK-NEXT:    qvfmul 0, 3, 0
-; CHECK-NEXT:    qvfmul 1, 1, 0
-; CHECK-NEXT:    blr
-entry:
-  %x = call ninf afn reassoc <4 x double> @llvm.sqrt.v4f64(<4 x double> %b)
-  %r = fdiv arcp reassoc <4 x double> %a, %x
-  ret <4 x double> %r
-}
-
-define <4 x double> @foo_safe(<4 x double> %a, <4 x double> %b) nounwind {
-; CHECK-LABEL: foo_safe:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    qvesplati 5, 2, 3
-; CHECK-NEXT:    qvesplati 3, 2, 1
-; CHECK-NEXT:    qvesplati 4, 2, 2
-; CHECK-NEXT:    fsqrt 2, 2
-; CHECK-NEXT:    fsqrt 5, 5
-; CHECK-NEXT:    fsqrt 4, 4
-; CHECK-NEXT:    fsqrt 3, 3
-; CHECK-NEXT:    qvesplati 6, 1, 3
-; CHECK-NEXT:    qvgpci 0, 275
-; CHECK-NEXT:    fdiv 2, 1, 2
-; CHECK-NEXT:    fdiv 5, 6, 5
-; CHECK-NEXT:    qvesplati 6, 1, 2
-; CHECK-NEXT:    qvesplati 1, 1, 1
-; CHECK-NEXT:    fdiv 4, 6, 4
-; CHECK-NEXT:    fdiv 1, 1, 3
-; CHECK-NEXT:    qvfperm 3, 4, 5, 0
-; CHECK-NEXT:    qvfperm 0, 2, 1, 0
-; CHECK-NEXT:    qvgpci 1, 101
-; CHECK-NEXT:    qvfperm 1, 0, 3, 1
-; CHECK-NEXT:    blr
-entry:
-  %x = call <4 x double> @llvm.sqrt.v4f64(<4 x double> %b)
-  %r = fdiv <4 x double> %a, %x
-  ret <4 x double> %r
-}
-
-define <4 x double> @foof_fmf(<4 x double> %a, <4 x float> %b) nounwind {
-; CHECK-LABEL: foof_fmf:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    addis 3, 2, .LCPI2_0@toc@ha
-; CHECK-NEXT:    qvfrsqrtes 3, 2
-; CHECK-NEXT:    addi 3, 3, .LCPI2_0@toc@l
-; CHECK-NEXT:    qvlfsx 0, 0, 3
-; CHECK-NEXT:    qvfmuls 4, 3, 3
-; CHECK-NEXT:    qvfmsubs 2, 2, 0, 2
-; CHECK-NEXT:    qvfnmsubs 0, 2, 4, 0
-; CHECK-NEXT:    qvfmuls 0, 3, 0
-; CHECK-NEXT:    qvfmul 1, 1, 0
-; CHECK-NEXT:    blr
-entry:
-  %x = call afn ninf reassoc <4 x float> @llvm.sqrt.v4f32(<4 x float> %b)
-  %y = fpext <4 x float> %x to <4 x double>
-  %r = fdiv arcp reassoc nsz <4 x double> %a, %y
-  ret <4 x double> %r
-}
-
-define <4 x double> @foof_safe(<4 x double> %a, <4 x float> %b) nounwind {
-; CHECK-LABEL: foof_safe:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    qvesplati 0, 2, 3
-; CHECK-NEXT:    qvesplati 3, 2, 2
-; CHECK-NEXT:    fsqrts 4, 2
-; CHECK-NEXT:    qvesplati 2, 2, 1
-; CHECK-NEXT:    fsqrts 0, 0
-; CHECK-NEXT:    fsqrts 3, 3
-; CHECK-NEXT:    fsqrts 2, 2
-; CHECK-NEXT:    qvgpci 5, 275
-; CHECK-NEXT:    qvgpci 6, 101
-; CHECK-NEXT:    qvfperm 0, 3, 0, 5
-; CHECK-NEXT:    qvesplati 3, 1, 2
-; CHECK-NEXT:    qvfperm 2, 4, 2, 5
-; CHECK-NEXT:    qvfperm 0, 2, 0, 6
-; CHECK-NEXT:    qvesplati 2, 1, 3
-; CHECK-NEXT:    qvesplati 4, 0, 3
-; CHECK-NEXT:    fdiv 2, 2, 4
-; CHECK-NEXT:    qvesplati 4, 0, 2
-; CHECK-NEXT:    fdiv 3, 3, 4
-; CHECK-NEXT:    qvesplati 4, 1, 1
-; CHECK-NEXT:    fdiv 1, 1, 0
-; CHECK-NEXT:    qvesplati 0, 0, 1
-; CHECK-NEXT:    fdiv 0, 4, 0
-; CHECK-NEXT:    qvfperm 2, 3, 2, 5
-; CHECK-NEXT:    qvfperm 0, 1, 0, 5
-; CHECK-NEXT:    qvfperm 1, 0, 2, 6
-; CHECK-NEXT:    blr
-entry:
-  %x = call <4 x float> @llvm.sqrt.v4f32(<4 x float> %b)
-  %y = fpext <4 x float> %x to <4 x double>
-  %r = fdiv <4 x double> %a, %y
-  ret <4 x double> %r
-}
-
-define <4 x float> @food_fmf(<4 x float> %a, <4 x double> %b) nounwind {
-; CHECK-LABEL: food_fmf:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    addis 3, 2, .LCPI4_0@toc@ha
-; CHECK-NEXT:    qvfrsqrte 3, 2
-; CHECK-NEXT:    addi 3, 3, .LCPI4_0@toc@l
-; CHECK-NEXT:    qvlfdx 0, 0, 3
-; CHECK-NEXT:    qvfmul 4, 3, 3
-; CHECK-NEXT:    qvfmsub 2, 2, 0, 2
-; CHECK-NEXT:    qvfnmsub 4, 2, 4, 0
-; CHECK-NEXT:    qvfmul 3, 3, 4
-; CHECK-NEXT:    qvfmul 4, 3, 3
-; CHECK-NEXT:    qvfnmsub 0, 2, 4, 0
-; CHECK-NEXT:    qvfmul 0, 3, 0
-; CHECK-NEXT:    qvfrsp 0, 0
-; CHECK-NEXT:    qvfmuls 1, 1, 0
-; CHECK-NEXT:    blr
-entry:
-  %x = call afn ninf reassoc <4 x double> @llvm.sqrt.v4f64(<4 x double> %b)
-  %y = fptrunc <4 x double> %x to <4 x float>
-  %r = fdiv arcp reassoc <4 x float> %a, %y
-  ret <4 x float> %r
-}
-
-define <4 x float> @food_safe(<4 x float> %a, <4 x double> %b) nounwind {
-; CHECK-LABEL: food_safe:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    qvesplati 0, 2, 3
-; CHECK-NEXT:    qvesplati 3, 2, 2
-; CHECK-NEXT:    fsqrt 4, 2
-; CHECK-NEXT:    qvesplati 2, 2, 1
-; CHECK-NEXT:    fsqrt 0, 0
-; CHECK-NEXT:    fsqrt 3, 3
-; CHECK-NEXT:    fsqrt 2, 2
-; CHECK-NEXT:    qvgpci 5, 275
-; CHECK-NEXT:    qvgpci 6, 101
-; CHECK-NEXT:    qvfperm 0, 3, 0, 5
-; CHECK-NEXT:    qvesplati 3, 1, 2
-; CHECK-NEXT:    qvfperm 2, 4, 2, 5
-; CHECK-NEXT:    qvfperm 0, 2, 0, 6
-; CHECK-NEXT:    qvesplati 2, 1, 3
-; CHECK-NEXT:    qvfrsp 0, 0
-; CHECK-NEXT:    qvesplati 4, 0, 3
-; CHECK-NEXT:    fdivs 2, 2, 4
-; CHECK-NEXT:    qvesplati 4, 0, 2
-; CHECK-NEXT:    fdivs 3, 3, 4
-; CHECK-NEXT:    qvesplati 4, 1, 1
-; CHECK-NEXT:    fdivs 1, 1, 0
-; CHECK-NEXT:    qvesplati 0, 0, 1
-; CHECK-NEXT:    fdivs 0, 4, 0
-; CHECK-NEXT:    qvfperm 2, 3, 2, 5
-; CHECK-NEXT:    qvfperm 0, 1, 0, 5
-; CHECK-NEXT:    qvfperm 1, 0, 2, 6
-; CHECK-NEXT:    blr
-entry:
-  %x = call <4 x double> @llvm.sqrt.v4f64(<4 x double> %b)
-  %y = fptrunc <4 x double> %x to <4 x float>
-  %r = fdiv <4 x float> %a, %y
-  ret <4 x float> %r
-}
-
-define <4 x float> @goo_fmf(<4 x float> %a, <4 x float> %b) nounwind {
-; CHECK-LABEL: goo_fmf:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    addis 3, 2, .LCPI6_0@toc@ha
-; CHECK-NEXT:    qvfrsqrtes 3, 2
-; CHECK-NEXT:    addi 3, 3, .LCPI6_0@toc@l
-; CHECK-NEXT:    qvlfsx 0, 0, 3
-; CHECK-NEXT:    qvfmuls 4, 3, 3
-; CHECK-NEXT:    qvfmsubs 2, 2, 0, 2
-; CHECK-NEXT:    qvfnmsubs 0, 2, 4, 0
-; CHECK-NEXT:    qvfmuls 0, 3, 0
-; CHECK-NEXT:    qvfmuls 1, 1, 0
-; CHECK-NEXT:    blr
-entry:
-  %x = call afn ninf reassoc <4 x float> @llvm.sqrt.v4f32(<4 x float> %b)
-  %r = fdiv arcp reassoc nsz <4 x float> %a, %x
-  ret <4 x float> %r
-}
-
-define <4 x float> @goo_safe(<4 x float> %a, <4 x float> %b) nounwind {
-; CHECK-LABEL: goo_safe:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    qvesplati 5, 2, 3
-; CHECK-NEXT:    qvesplati 3, 2, 1
-; CHECK-NEXT:    qvesplati 4, 2, 2
-; CHECK-NEXT:    fsqrts 2, 2
-; CHECK-NEXT:    fsqrts 5, 5
-; CHECK-NEXT:    fsqrts 4, 4
-; CHECK-NEXT:    fsqrts 3, 3
-; CHECK-NEXT:    qvesplati 6, 1, 3
-; CHECK-NEXT:    qvgpci 0, 275
-; CHECK-NEXT:    fdivs 2, 1, 2
-; CHECK-NEXT:    fdivs 5, 6, 5
-; CHECK-NEXT:    qvesplati 6, 1, 2
-; CHECK-NEXT:    qvesplati 1, 1, 1
-; CHECK-NEXT:    fdivs 4, 6, 4
-; CHECK-NEXT:    fdivs 1, 1, 3
-; CHECK-NEXT:    qvfperm 3, 4, 5, 0
-; CHECK-NEXT:    qvfperm 0, 2, 1, 0
-; CHECK-NEXT:    qvgpci 1, 101
-; CHECK-NEXT:    qvfperm 1, 0, 3, 1
-; CHECK-NEXT:    blr
-entry:
-  %x = call <4 x float> @llvm.sqrt.v4f32(<4 x float> %b)
-  %r = fdiv <4 x float> %a, %x
-  ret <4 x float> %r
-}
-
-define <4 x double> @foo2_fmf(<4 x double> %a, <4 x double> %b) nounwind {
-; CHECK-LABEL: foo2_fmf:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    addis 3, 2, .LCPI8_0@toc@ha
-; CHECK-NEXT:    qvfre 3, 2
-; CHECK-NEXT:    addi 3, 3, .LCPI8_0@toc@l
-; CHECK-NEXT:    qvlfdx 0, 0, 3
-; CHECK-NEXT:    qvfmadd 0, 2, 3, 0
-; CHECK-NEXT:    qvfnmsub 0, 3, 0, 3
-; CHECK-NEXT:    qvfmul 3, 1, 0
-; CHECK-NEXT:    qvfnmsub 1, 2, 3, 1
-; CHECK-NEXT:    qvfmadd 1, 0, 1, 3
-; CHECK-NEXT:    blr
-entry:
-  %r = fdiv arcp reassoc nsz ninf <4 x double> %a, %b
-  ret <4 x double> %r
-}
-
-define <4 x double> @foo2_safe(<4 x double> %a, <4 x double> %b) nounwind {
-; CHECK-LABEL: foo2_safe:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    qvesplati 3, 2, 3
-; CHECK-NEXT:    qvesplati 4, 1, 3
-; CHECK-NEXT:    qvesplati 5, 2, 2
-; CHECK-NEXT:    qvgpci 0, 275
-; CHECK-NEXT:    fdiv 3, 4, 3
-; CHECK-NEXT:    qvesplati 4, 1, 2
-; CHECK-NEXT:    fdiv 4, 4, 5
-; CHECK-NEXT:    fdiv 5, 1, 2
-; CHECK-NEXT:    qvesplati 2, 2, 1
-; CHECK-NEXT:    qvesplati 1, 1, 1
-; CHECK-NEXT:    fdiv 1, 1, 2
-; CHECK-NEXT:    qvfperm 2, 4, 3, 0
-; CHECK-NEXT:    qvfperm 0, 5, 1, 0
-; CHECK-NEXT:    qvgpci 1, 101
-; CHECK-NEXT:    qvfperm 1, 0, 2, 1
-; CHECK-NEXT:    blr
-  %r = fdiv <4 x double> %a, %b
-  ret <4 x double> %r
-}
-
-define <4 x float> @goo2_fmf(<4 x float> %a, <4 x float> %b) nounwind {
-; CHECK-LABEL: goo2_fmf:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    qvfres 0, 2
-; CHECK-NEXT:    qvfmuls 3, 1, 0
-; CHECK-NEXT:    qvfnmsubs 1, 2, 3, 1
-; CHECK-NEXT:    qvfmadds 1, 0, 1, 3
-; CHECK-NEXT:    blr
-entry:
-  %r = fdiv arcp reassoc ninf <4 x float> %a, %b
-  ret <4 x float> %r
-}
-
-define <4 x float> @goo2_safe(<4 x float> %a, <4 x float> %b) nounwind {
-; CHECK-LABEL: goo2_safe:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    qvesplati 3, 2, 3
-; CHECK-NEXT:    qvesplati 4, 1, 3
-; CHECK-NEXT:    qvesplati 5, 2, 2
-; CHECK-NEXT:    qvgpci 0, 275
-; CHECK-NEXT:    fdivs 3, 4, 3
-; CHECK-NEXT:    qvesplati 4, 1, 2
-; CHECK-NEXT:    fdivs 4, 4, 5
-; CHECK-NEXT:    fdivs 5, 1, 2
-; CHECK-NEXT:    qvesplati 2, 2, 1
-; CHECK-NEXT:    qvesplati 1, 1, 1
-; CHECK-NEXT:    fdivs 1, 1, 2
-; CHECK-NEXT:    qvfperm 2, 4, 3, 0
-; CHECK-NEXT:    qvfperm 0, 5, 1, 0
-; CHECK-NEXT:    qvgpci 1, 101
-; CHECK-NEXT:    qvfperm 1, 0, 2, 1
-; CHECK-NEXT:    blr
-entry:
-  %r = fdiv <4 x float> %a, %b
-  ret <4 x float> %r
-}
-
-define <4 x double> @foo3_fmf_denorm_on(<4 x double> %a) #0 {
-; CHECK-LABEL: foo3_fmf_denorm_on:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    addis 3, 2, .LCPI12_0@toc@ha
-; CHECK-NEXT:    qvfrsqrte 0, 1
-; CHECK-NEXT:    addi 3, 3, .LCPI12_0@toc@l
-; CHECK-NEXT:    qvlfdx 2, 0, 3
-; CHECK-NEXT:    addis 3, 2, .LCPI12_1@toc@ha
-; CHECK-NEXT:    addi 3, 3, .LCPI12_1@toc@l
-; CHECK-NEXT:    qvfmul 3, 0, 0
-; CHECK-NEXT:    qvfmsub 4, 1, 2, 1
-; CHECK-NEXT:    qvfnmsub 3, 4, 3, 2
-; CHECK-NEXT:    qvfmul 0, 0, 3
-; CHECK-NEXT:    qvfmul 3, 0, 0
-; CHECK-NEXT:    qvfnmsub 2, 4, 3, 2
-; CHECK-NEXT:    qvfmul 0, 0, 2
-; CHECK-NEXT:    qvlfdx 2, 0, 3
-; CHECK-NEXT:    addis 3, 2, .LCPI12_2@toc@ha
-; CHECK-NEXT:    addi 3, 3, .LCPI12_2@toc@l
-; CHECK-NEXT:    qvlfdx 3, 0, 3
-; CHECK-NEXT:    qvfmul 0, 0, 1
-; CHECK-NEXT:    qvfabs 1, 1
-; CHECK-NEXT:    qvfcmplt 1, 1, 2
-; CHECK-NEXT:    qvfsel 1, 1, 3, 0
-; CHECK-NEXT:    blr
-entry:
-  %r = call reassoc ninf afn <4 x double> @llvm.sqrt.v4f64(<4 x double> %a)
-  ret <4 x double> %r
-}
-
-define <4 x double> @foo3_fmf_denorm_off(<4 x double> %a) #1 {
-; CHECK-LABEL: foo3_fmf_denorm_off:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    addis 3, 2, .LCPI13_0@toc@ha
-; CHECK-NEXT:    qvfrsqrte 0, 1
-; CHECK-NEXT:    addi 3, 3, .LCPI13_0@toc@l
-; CHECK-NEXT:    qvlfdx 2, 0, 3
-; CHECK-NEXT:    addis 3, 2, .LCPI13_1@toc@ha
-; CHECK-NEXT:    addi 3, 3, .LCPI13_1@toc@l
-; CHECK-NEXT:    qvfmul 3, 0, 0
-; CHECK-NEXT:    qvfmsub 4, 1, 2, 1
-; CHECK-NEXT:    qvfnmsub 3, 4, 3, 2
-; CHECK-NEXT:    qvfmul 0, 0, 3
-; CHECK-NEXT:    qvfmul 3, 0, 0
-; CHECK-NEXT:    qvfnmsub 2, 4, 3, 2
-; CHECK-NEXT:    qvfmul 0, 0, 2
-; CHECK-NEXT:    qvlfdx 2, 0, 3
-; CHECK-NEXT:    qvfmul 0, 0, 1
-; CHECK-NEXT:    qvfcmpeq 1, 1, 2
-; CHECK-NEXT:    qvfsel 1, 1, 2, 0
-; CHECK-NEXT:    blr
-entry:
-  %r = call afn reassoc ninf <4 x double> @llvm.sqrt.v4f64(<4 x double> %a)
-  ret <4 x double> %r
-}
-
-define <4 x double> @foo3_safe_denorm_on(<4 x double> %a) #0 {
-; CHECK-LABEL: foo3_safe_denorm_on:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    qvesplati 2, 1, 3
-; CHECK-NEXT:    qvesplati 3, 1, 2
-; CHECK-NEXT:    fsqrt 4, 1
-; CHECK-NEXT:    qvesplati 1, 1, 1
-; CHECK-NEXT:    fsqrt 2, 2
-; CHECK-NEXT:    fsqrt 3, 3
-; CHECK-NEXT:    fsqrt 1, 1
-; CHECK-NEXT:    qvgpci 0, 275
-; CHECK-NEXT:    qvfperm 2, 3, 2, 0
-; CHECK-NEXT:    qvfperm 0, 4, 1, 0
-; CHECK-NEXT:    qvgpci 1, 101
-; CHECK-NEXT:    qvfperm 1, 0, 2, 1
-; CHECK-NEXT:    blr
-entry:
-  %r = call <4 x double> @llvm.sqrt.v4f64(<4 x double> %a)
-  ret <4 x double> %r
-}
-
-define <4 x double> @foo3_safe_denorm_off(<4 x double> %a) #1 {
-; CHECK-LABEL: foo3_safe_denorm_off:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    qvesplati 2, 1, 3
-; CHECK-NEXT:    qvesplati 3, 1, 2
-; CHECK-NEXT:    fsqrt 4, 1
-; CHECK-NEXT:    qvesplati 1, 1, 1
-; CHECK-NEXT:    fsqrt 2, 2
-; CHECK-NEXT:    fsqrt 3, 3
-; CHECK-NEXT:    fsqrt 1, 1
-; CHECK-NEXT:    qvgpci 0, 275
-; CHECK-NEXT:    qvfperm 2, 3, 2, 0
-; CHECK-NEXT:    qvfperm 0, 4, 1, 0
-; CHECK-NEXT:    qvgpci 1, 101
-; CHECK-NEXT:    qvfperm 1, 0, 2, 1
-; CHECK-NEXT:    blr
-entry:
-  %r = call <4 x double> @llvm.sqrt.v4f64(<4 x double> %a)
-  ret <4 x double> %r
-}
-
-define <4 x float> @goo3_fmf_denorm_on(<4 x float> %a) #0 {
-; CHECK-LABEL: goo3_fmf_denorm_on:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    addis 3, 2, .LCPI16_1@toc@ha
-; CHECK-NEXT:    qvfrsqrtes 2, 1
-; CHECK-NEXT:    addi 3, 3, .LCPI16_1@toc@l
-; CHECK-NEXT:    qvlfsx 0, 0, 3
-; CHECK-NEXT:    addis 3, 2, .LCPI16_0@toc@ha
-; CHECK-NEXT:    addi 3, 3, .LCPI16_0@toc@l
-; CHECK-NEXT:    qvfmuls 4, 2, 2
-; CHECK-NEXT:    qvfmsubs 3, 1, 0, 1
-; CHECK-NEXT:    qvfnmsubs 0, 3, 4, 0
-; CHECK-NEXT:    qvlfsx 3, 0, 3
-; CHECK-NEXT:    addis 3, 2, .LCPI16_2@toc@ha
-; CHECK-NEXT:    addi 3, 3, .LCPI16_2@toc@l
-; CHECK-NEXT:    qvlfsx 4, 0, 3
-; CHECK-NEXT:    qvfmuls 0, 2, 0
-; CHECK-NEXT:    qvfabs 2, 1
-; CHECK-NEXT:    qvfmuls 0, 0, 1
-; CHECK-NEXT:    qvfcmplt 1, 2, 3
-; CHECK-NEXT:    qvfsel 1, 1, 4, 0
-; CHECK-NEXT:    blr
-entry:
-  %r = call reassoc afn ninf nsz <4 x float> @llvm.sqrt.v4f32(<4 x float> %a)
-  ret <4 x float> %r
-}
-
-define <4 x float> @goo3_fmf_denorm_off(<4 x float> %a) #1 {
-; CHECK-LABEL: goo3_fmf_denorm_off:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    addis 3, 2, .LCPI17_1@toc@ha
-; CHECK-NEXT:    qvfrsqrtes 2, 1
-; CHECK-NEXT:    addi 3, 3, .LCPI17_1@toc@l
-; CHECK-NEXT:    qvlfsx 0, 0, 3
-; CHECK-NEXT:    addis 3, 2, .LCPI17_0@toc@ha
-; CHECK-NEXT:    addi 3, 3, .LCPI17_0@toc@l
-; CHECK-NEXT:    qvfmuls 4, 2, 2
-; CHECK-NEXT:    qvfmsubs 3, 1, 0, 1
-; CHECK-NEXT:    qvfnmsubs 0, 3, 4, 0
-; CHECK-NEXT:    qvlfsx 3, 0, 3
-; CHECK-NEXT:    qvfmuls 0, 2, 0
-; CHECK-NEXT:    qvfmuls 0, 0, 1
-; CHECK-NEXT:    qvfcmpeq 1, 1, 3
-; CHECK-NEXT:    qvfsel 1, 1, 3, 0
-; CHECK-NEXT:    blr
-entry:
-  %r = call reassoc ninf afn nsz <4 x float> @llvm.sqrt.v4f32(<4 x float> %a)
-  ret <4 x float> %r
-}
-
-define <4 x float> @goo3_safe(<4 x float> %a) nounwind {
-; CHECK-LABEL: goo3_safe:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    qvesplati 2, 1, 3
-; CHECK-NEXT:    qvesplati 3, 1, 2
-; CHECK-NEXT:    fsqrts 4, 1
-; CHECK-NEXT:    qvesplati 1, 1, 1
-; CHECK-NEXT:    fsqrts 2, 2
-; CHECK-NEXT:    fsqrts 3, 3
-; CHECK-NEXT:    fsqrts 1, 1
-; CHECK-NEXT:    qvgpci 0, 275
-; CHECK-NEXT:    qvfperm 2, 3, 2, 0
-; CHECK-NEXT:    qvfperm 0, 4, 1, 0
-; CHECK-NEXT:    qvgpci 1, 101
-; CHECK-NEXT:    qvfperm 1, 0, 2, 1
-; CHECK-NEXT:    blr
-entry:
-  %r = call <4 x float> @llvm.sqrt.v4f32(<4 x float> %a)
-  ret <4 x float> %r
-}
-
-attributes #0 = { nounwind "denormal-fp-math"="ieee,ieee" }
-attributes #1 = { nounwind "denormal-fp-math"="preserve-sign,preserve-sign" }
diff --git a/llvm/test/CodeGen/PowerPC/qpx-rounding-ops.ll b/llvm/test/CodeGen/PowerPC/qpx-rounding-ops.ll
deleted file mode 100644
index ee3357156a6c0..0000000000000
--- a/llvm/test/CodeGen/PowerPC/qpx-rounding-ops.ll
+++ /dev/null
@@ -1,109 +0,0 @@
-; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc64-unknown-linux-gnu -mcpu=a2q | FileCheck %s
-; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc64-unknown-linux-gnu -mcpu=a2q -enable-unsafe-fp-math | FileCheck -check-prefix=CHECK-FM %s
-target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
-target triple = "powerpc64-unknown-linux-gnu"
-
-define <4 x float> @test1(<4 x float> %x) nounwind  {
-  %call = tail call <4 x float> @llvm.floor.v4f32(<4 x float> %x) nounwind readnone
-  ret <4 x float> %call
-
-; CHECK: test1:
-; CHECK: qvfrim 1, 1
-
-; CHECK-FM: test1:
-; CHECK-FM: qvfrim 1, 1
-}
-
-declare <4 x float> @llvm.floor.v4f32(<4 x float>) nounwind readnone
-
-define <4 x double> @test2(<4 x double> %x) nounwind  {
-  %call = tail call <4 x double> @llvm.floor.v4f64(<4 x double> %x) nounwind readnone
-  ret <4 x double> %call
-
-; CHECK: test2:
-; CHECK: qvfrim 1, 1
-
-; CHECK-FM: test2:
-; CHECK-FM: qvfrim 1, 1
-}
-
-declare <4 x double> @llvm.floor.v4f64(<4 x double>) nounwind readnone
-
-define <4 x float> @test3(<4 x float> %x) nounwind  {
-  %call = tail call <4 x float> @llvm.nearbyint.v4f32(<4 x float> %x) nounwind readnone
-  ret <4 x float> %call
-
-; CHECK: test3:
-; CHECK-NOT: qvfrin
-
-; CHECK-FM: test3:
-; CHECK-FM-NOT: qvfrin
-}
-
-declare <4 x float> @llvm.nearbyint.v4f32(<4 x float>) nounwind readnone
-
-define <4 x double> @test4(<4 x double> %x) nounwind  {
-  %call = tail call <4 x double> @llvm.nearbyint.v4f64(<4 x double> %x) nounwind readnone
-  ret <4 x double> %call
-
-; CHECK: test4:
-; CHECK-NOT: qvfrin
-
-; CHECK-FM: test4:
-; CHECK-FM-NOT: qvfrin
-}
-
-declare <4 x double> @llvm.nearbyint.v4f64(<4 x double>) nounwind readnone
-
-define <4 x float> @test5(<4 x float> %x) nounwind  {
-  %call = tail call <4 x float> @llvm.ceil.v4f32(<4 x float> %x) nounwind readnone
-  ret <4 x float> %call
-
-; CHECK: test5:
-; CHECK: qvfrip 1, 1
-
-; CHECK-FM: test5:
-; CHECK-FM: qvfrip 1, 1
-}
-
-declare <4 x float> @llvm.ceil.v4f32(<4 x float>) nounwind readnone
-
-define <4 x double> @test6(<4 x double> %x) nounwind  {
-  %call = tail call <4 x double> @llvm.ceil.v4f64(<4 x double> %x) nounwind readnone
-  ret <4 x double> %call
-
-; CHECK: test6:
-; CHECK: qvfrip 1, 1
-
-; CHECK-FM: test6:
-; CHECK-FM: qvfrip 1, 1
-}
-
-declare <4 x double> @llvm.ceil.v4f64(<4 x double>) nounwind readnone
-
-define <4 x float> @test9(<4 x float> %x) nounwind  {
-  %call = tail call <4 x float> @llvm.trunc.v4f32(<4 x float> %x) nounwind readnone
-  ret <4 x float> %call
-
-; CHECK: test9:
-; CHECK: qvfriz 1, 1
-
-; CHECK-FM: test9:
-; CHECK-FM: qvfriz 1, 1
-}
-
-declare <4 x float> @llvm.trunc.v4f32(<4 x float>) nounwind readnone
-
-define <4 x double> @test10(<4 x double> %x) nounwind  {
-  %call = tail call <4 x double> @llvm.trunc.v4f64(<4 x double> %x) nounwind readnone
-  ret <4 x double> %call
-
-; CHECK: test10:
-; CHECK: qvfriz 1, 1
-
-; CHECK-FM: test10:
-; CHECK-FM: qvfriz 1, 1
-}
-
-declare <4 x double> @llvm.trunc.v4f64(<4 x double>) nounwind readnone
-
diff --git a/llvm/test/CodeGen/PowerPC/qpx-s-load.ll b/llvm/test/CodeGen/PowerPC/qpx-s-load.ll
deleted file mode 100644
index 57d7e3b0ded3c..0000000000000
--- a/llvm/test/CodeGen/PowerPC/qpx-s-load.ll
+++ /dev/null
@@ -1,26 +0,0 @@
-; RUN: llc -verify-machineinstrs < %s -mcpu=a2q | FileCheck %s
-target triple = "powerpc64-bgq-linux"
-
-define <4 x float> @foo(<4 x float>* %p) {
-entry:
-  %v = load <4 x float>, <4 x float>* %p, align 4
-  ret <4 x float> %v
-}
-
-; CHECK: @foo
-; CHECK-DAG: li [[REG1:[0-9]+]], 15
-; CHECK-DAG: qvlfsx [[REG4:[0-9]+]], 0, 3
-; CHECK-DAG: qvlfsx [[REG2:[0-9]+]], 3, [[REG1]]
-; CHECK-DAG: qvlpclsx [[REG3:[0-9]+]], 0, 3
-; CHECK-DAG: qvfperm 1, [[REG4]], [[REG2]], [[REG3]]
-; CHECK: blr
-
-define <4 x float> @bar(<4 x float>* %p) {
-entry:
-  %v = load <4 x float>, <4 x float>* %p, align 16
-  ret <4 x float> %v
-}
-
-; CHECK: @bar
-; CHECK: qvlfsx
-
diff --git a/llvm/test/CodeGen/PowerPC/qpx-s-sel.ll b/llvm/test/CodeGen/PowerPC/qpx-s-sel.ll
deleted file mode 100644
index 5d42b9a529953..0000000000000
--- a/llvm/test/CodeGen/PowerPC/qpx-s-sel.ll
+++ /dev/null
@@ -1,143 +0,0 @@
-; RUN: llc -verify-machineinstrs < %s -mcpu=a2q | FileCheck %s
-target triple = "powerpc64-bgq-linux"
-
-@R = global <4 x i1> <i1 0, i1 0, i1 0, i1 0>, align 16
-
-define <4 x float> @test1(<4 x float> %a, <4 x float> %b, <4 x i1> %c) nounwind readnone {
-entry:
-  %r = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
-  ret <4 x float> %r
-
-; CHECK-LABEL: @test1
-; CHECK: qvfsel 1, 3, 1, 2
-; CHECK: blr
-}
-
-define <4 x float> @test2(<4 x float> %a, <4 x float> %b, i1 %c1, i1 %c2, i1 %c3, i1 %c4) nounwind readnone {
-entry:
-  %v = insertelement <4 x i1> undef, i1 %c1, i32 0
-  %v2 = insertelement <4 x i1> %v, i1 %c2, i32 1
-  %v3 = insertelement <4 x i1> %v2, i1 %c3, i32 2
-  %v4 = insertelement <4 x i1> %v3, i1 %c4, i32 3
-  %r = select <4 x i1> %v4, <4 x float> %a, <4 x float> %b
-  ret <4 x float> %r
-
-; CHECK-LABEL: @test2
-; CHECK: stw
-; CHECK-DAG: qvlfiwzx [[REG1:[0-9]+]],
-; CHECK-DAG: qvlfdx [[REG2:[0-9]+]],
-; CHECK-DAG: qvfcfidu [[REG3:[0-9]+]], [[REG1]]
-; CHECK: qvfcmpeq [[REG4:[0-9]+]], [[REG3]], [[REG2]]
-; CHECK: qvfsel 1, [[REG4]], 1, 2
-; CHECK: blr
-}
-
-define <4 x i1> @test3(<4 x i1> %a) nounwind readnone {
-entry:
-  %v = and <4 x i1> %a, <i1 0, i1 undef, i1 1, i1 1>
-  ret <4 x i1> %v
-
-; CHECK-LABEL: @test3
-; CHECK: qvlfsx [[REG:[0-9]+]],
-; qvflogical 1, 1, [[REG]], 1
-; blr
-}
-
-define <4 x i1> @test4(<4 x i1> %a, <4 x i1>* %t) nounwind {
-entry:
-  %q = load <4 x i1>, <4 x i1>* %t, align 16
-  %v = and <4 x i1> %a, %q
-  ret <4 x i1> %v
-
-; CHECK-LABEL: @test4
-; CHECK-DAG: lbz
-; CHECK-DAG: qvlfdx [[REG1:[0-9]+]],
-; CHECK-DAG: stw
-; CHECK-DAG: qvlfiwzx [[REG2:[0-9]+]],
-; CHECK-DAG: qvfcfidu [[REG3:[0-9]+]], [[REG2]]
-; CHECK: qvfcmpeq [[REG4:[0-9]+]], [[REG3]], [[REG1]]
-; CHECK: qvfand 1, 1, [[REG4]]
-; CHECK: blr
-}
-
-define void @test5(<4 x i1> %a) nounwind {
-entry:
-  store <4 x i1> %a, <4 x i1>* @R
-  ret void
-
-; CHECK-LABEL: @test5
-; CHECK: qvlfdx [[REG1:[0-9]+]],
-; CHECK: qvfmadd [[REG2:[0-9]+]], 1, [[REG1]], [[REG1]]
-; CHECK: qvfctiwu [[REG3:[0-9]+]], [[REG2]]
-; CHECK: qvstfiwx [[REG3]],
-; CHECK: lwz
-; CHECK: stb
-; CHECK: blr
-}
-
-define i1 @test6(<4 x i1> %a) nounwind {
-entry:
-  %r = extractelement <4 x i1> %a, i32 2
-  ret i1 %r
-
-; CHECK-LABEL: @test6
-; CHECK: qvlfdx [[REG1:[0-9]+]],
-; CHECK: qvfmadd [[REG2:[0-9]+]], 1, [[REG1]], [[REG1]]
-; CHECK: qvfctiwu [[REG3:[0-9]+]], [[REG2]]
-; CHECK: qvstfiwx [[REG3]],
-; CHECK: lwz
-; CHECK: blr
-}
-
-define i1 @test7(<4 x i1> %a) nounwind {
-entry:
-  %r = extractelement <4 x i1> %a, i32 2
-  %s = extractelement <4 x i1> %a, i32 3
-  %q = and i1 %r, %s
-  ret i1 %q
-
-; CHECK-LABEL: @test7
-; CHECK: qvlfdx [[REG1:[0-9]+]],
-; CHECK: qvfmadd [[REG2:[0-9]+]], 1, [[REG1]], [[REG1]]
-; CHECK: qvfctiwu [[REG3:[0-9]+]], [[REG2]]
-; CHECK: qvstfiwx [[REG3]],
-; CHECK-DAG: lwz [[REG4:[0-9]+]],
-; FIXME: We're storing the vector twice, and that's silly.
-; CHECK-DAG: qvstfiwx [[REG3]],
-; CHECK: lwz [[REG5:[0-9]+]],
-; CHECK: and 3,
-; CHECK: blr
-}
-
-define i1 @test8(<3 x i1> %a) nounwind {
-entry:
-  %r = extractelement <3 x i1> %a, i32 2
-  ret i1 %r
-
-; CHECK-LABEL: @test8
-; CHECK: qvlfdx [[REG1:[0-9]+]],
-; CHECK: qvfmadd [[REG2:[0-9]+]], 1, [[REG1]], [[REG1]]
-; CHECK: qvfctiwu [[REG3:[0-9]+]], [[REG2]]
-; CHECK: qvstfiwx [[REG3]],
-; CHECK: lwz
-; CHECK: blr
-}
-
-define <3 x float> @test9(<3 x float> %a, <3 x float> %b, i1 %c1, i1 %c2, i1 %c3) nounwind readnone {
-entry:
-  %v = insertelement <3 x i1> undef, i1 %c1, i32 0
-  %v2 = insertelement <3 x i1> %v, i1 %c2, i32 1
-  %v3 = insertelement <3 x i1> %v2, i1 %c3, i32 2
-  %r = select <3 x i1> %v3, <3 x float> %a, <3 x float> %b
-  ret <3 x float> %r
-
-; CHECK-LABEL: @test9
-; CHECK: stw
-; CHECK-DAG: qvlfiwzx [[REG1:[0-9]+]],
-; CHECK-DAG: qvlfdx [[REG2:[0-9]+]],
-; CHECK-DAG: qvfcfidu [[REG3:[0-9]+]], [[REG1]]
-; CHECK: qvfcmpeq [[REG4:[0-9]+]], [[REG3]], [[REG2]]
-; CHECK: qvfsel 1, [[REG4]], 1, 2
-; CHECK: blr
-}
-
diff --git a/llvm/test/CodeGen/PowerPC/qpx-s-store.ll b/llvm/test/CodeGen/PowerPC/qpx-s-store.ll
deleted file mode 100644
index 81cff7b6457f1..0000000000000
--- a/llvm/test/CodeGen/PowerPC/qpx-s-store.ll
+++ /dev/null
@@ -1,25 +0,0 @@
-; RUN: llc -verify-machineinstrs < %s -mcpu=a2q | FileCheck %s
-target triple = "powerpc64-bgq-linux"
-
-define void @foo(<4 x float> %v, <4 x float>* %p) {
-entry:
-  store <4 x float> %v, <4 x float>* %p, align 4
-  ret void
-}
-
-; CHECK: @foo
-; CHECK: stfs
-; CHECK: stfs
-; CHECK: stfs
-; CHECK: stfs
-; CHECK: blr
-
-define void @bar(<4 x float> %v, <4 x float>* %p) {
-entry:
-  store <4 x float> %v, <4 x float>* %p, align 16
-  ret void
-}
-
-; CHECK: @bar
-; CHECK: qvstfsx
-
diff --git a/llvm/test/CodeGen/PowerPC/qpx-sel.ll b/llvm/test/CodeGen/PowerPC/qpx-sel.ll
deleted file mode 100644
index abc92c9e98b13..0000000000000
--- a/llvm/test/CodeGen/PowerPC/qpx-sel.ll
+++ /dev/null
@@ -1,151 +0,0 @@
-; RUN: llc -verify-machineinstrs < %s -mcpu=a2q | FileCheck %s
-target triple = "powerpc64-bgq-linux"
-
-@R = global <4 x i1> <i1 0, i1 0, i1 0, i1 0>, align 16
-
-define <4 x double> @test1(<4 x double> %a, <4 x double> %b, <4 x i1> %c) nounwind readnone {
-entry:
-  %r = select <4 x i1> %c, <4 x double> %a, <4 x double> %b
-  ret <4 x double> %r
-
-; CHECK-LABEL: @test1
-; CHECK: qvfsel 1, 3, 1, 2
-; CHECK: blr
-}
-
-define <4 x double> @test2(<4 x double> %a, <4 x double> %b, i1 %c1, i1 %c2, i1 %c3, i1 %c4) nounwind readnone {
-entry:
-  %v = insertelement <4 x i1> undef, i1 %c1, i32 0
-  %v2 = insertelement <4 x i1> %v, i1 %c2, i32 1
-  %v3 = insertelement <4 x i1> %v2, i1 %c3, i32 2
-  %v4 = insertelement <4 x i1> %v3, i1 %c4, i32 3
-  %r = select <4 x i1> %v4, <4 x double> %a, <4 x double> %b
-  ret <4 x double> %r
-
-; CHECK-LABEL: @test2
-
-; FIXME: This load/store sequence is unnecessary.
-; CHECK-DAG: lbz
-; CHECK-DAG: stw
-
-; CHECK-DAG: qvlfiwzx [[REG1:[0-9]+]],
-; CHECK-DAG: qvlfdx [[REG2:[0-9]+]],
-; CHECK-DAG: qvfcfidu [[REG3:[0-9]+]], [[REG1]]
-; CHECK: qvfcmpeq [[REG4:[0-9]+]], [[REG3]], [[REG2]]
-; CHECK: qvfsel 1, [[REG4]], 1, 2
-; CHECK: blr
-}
-
-define <4 x i1> @test3(<4 x i1> %a) nounwind readnone {
-entry:
-  %v = and <4 x i1> %a, <i1 0, i1 undef, i1 1, i1 1>
-  ret <4 x i1> %v
-
-; CHECK-LABEL: @test3
-; CHECK: qvlfsx [[REG:[0-9]+]],
-; qvflogical 1, 1, [[REG]], 1
-; blr
-}
-
-define <4 x i1> @test4(<4 x i1> %a, <4 x i1>* %t) nounwind {
-entry:
-  %q = load <4 x i1>, <4 x i1>* %t, align 16
-  %v = and <4 x i1> %a, %q
-  ret <4 x i1> %v
-
-; CHECK-LABEL: @test4
-; CHECK-DAG: lbz
-; CHECK-DAG: qvlfdx [[REG1:[0-9]+]],
-; CHECK-DAG: stw
-; CHECK-DAG: qvlfiwzx [[REG2:[0-9]+]],
-; CHECK-DAG: qvfcfidu [[REG3:[0-9]+]], [[REG2]]
-; CHECK: qvfcmpeq [[REG4:[0-9]+]], [[REG3]], [[REG1]]
-; CHECK: qvfand 1, 1, [[REG4]]
-; CHECK: blr
-}
-
-define void @test5(<4 x i1> %a) nounwind {
-entry:
-  store <4 x i1> %a, <4 x i1>* @R
-  ret void
-
-; CHECK-LABEL: @test5
-; CHECK: qvlfdx [[REG1:[0-9]+]],
-; CHECK: qvfmadd [[REG2:[0-9]+]], 1, [[REG1]], [[REG1]]
-; CHECK: qvfctiwu [[REG3:[0-9]+]], [[REG2]]
-; CHECK: qvstfiwx [[REG3]],
-; CHECK: lwz
-; CHECK: stb
-; CHECK: blr
-}
-
-define i1 @test6(<4 x i1> %a) nounwind {
-entry:
-  %r = extractelement <4 x i1> %a, i32 2
-  ret i1 %r
-
-; CHECK-LABEL: @test6
-; CHECK: qvlfdx [[REG1:[0-9]+]],
-; CHECK: qvfmadd [[REG2:[0-9]+]], 1, [[REG1]], [[REG1]]
-; CHECK: qvfctiwu [[REG3:[0-9]+]], [[REG2]]
-; CHECK: qvstfiwx [[REG3]],
-; CHECK: lwz
-; CHECK: blr
-}
-
-define i1 @test7(<4 x i1> %a) nounwind {
-entry:
-  %r = extractelement <4 x i1> %a, i32 2
-  %s = extractelement <4 x i1> %a, i32 3
-  %q = and i1 %r, %s
-  ret i1 %q
-
-; CHECK-LABEL: @test7
-; CHECK: qvlfdx [[REG1:[0-9]+]],
-; CHECK: qvfmadd [[REG2:[0-9]+]], 1, [[REG1]], [[REG1]]
-; CHECK: qvfctiwu [[REG3:[0-9]+]], [[REG2]]
-; CHECK: qvstfiwx [[REG3]],
-; CHECK-DAG: lwz [[REG4:[0-9]+]],
-; FIXME: We're storing the vector twice, and that's silly.
-; CHECK-DAG: qvstfiwx [[REG3]],
-; CHECK-DAG: lwz [[REG5:[0-9]+]],
-; CHECK: and 3,
-; CHECK: blr
-}
-
-define i1 @test8(<3 x i1> %a) nounwind {
-entry:
-  %r = extractelement <3 x i1> %a, i32 2
-  ret i1 %r
-
-; CHECK-LABEL: @test8
-; CHECK: qvlfdx [[REG1:[0-9]+]],
-; CHECK: qvfmadd [[REG2:[0-9]+]], 1, [[REG1]], [[REG1]]
-; CHECK: qvfctiwu [[REG3:[0-9]+]], [[REG2]]
-; CHECK: qvstfiwx [[REG3]],
-; CHECK: lwz
-; CHECK: blr
-}
-
-define <3 x double> @test9(<3 x double> %a, <3 x double> %b, i1 %c1, i1 %c2, i1 %c3) nounwind readnone {
-entry:
-  %v = insertelement <3 x i1> undef, i1 %c1, i32 0
-  %v2 = insertelement <3 x i1> %v, i1 %c2, i32 1
-  %v3 = insertelement <3 x i1> %v2, i1 %c3, i32 2
-  %r = select <3 x i1> %v3, <3 x double> %a, <3 x double> %b
-  ret <3 x double> %r
-
-; CHECK-LABEL: @test9
-
-; FIXME: This load/store sequence is unnecessary.
-; CHECK-DAG: lbz
-; CHECK-DAG: stw
-
-; CHECK-DAG: qvlfiwzx [[REG1:[0-9]+]],
-; CHECK-DAG: qvlfdx [[REG2:[0-9]+]],
-; CHECK-DAG: qvfcfidu [[REG3:[0-9]+]], [[REG1]]
-; CHECK: qvfcmpeq [[REG4:[0-9]+]], [[REG3]], [[REG2]]
-; CHECK: qvfsel 1, [[REG4]], 1, 2
-; CHECK: blr
-}
-
diff --git a/llvm/test/CodeGen/PowerPC/qpx-split-vsetcc.ll b/llvm/test/CodeGen/PowerPC/qpx-split-vsetcc.ll
deleted file mode 100644
index df3e0befaef8a..0000000000000
--- a/llvm/test/CodeGen/PowerPC/qpx-split-vsetcc.ll
+++ /dev/null
@@ -1,31 +0,0 @@
-; RUN: llc -verify-machineinstrs -mcpu=a2q < %s | FileCheck %s
-target datalayout = "E-m:e-i64:64-n32:64"
-target triple = "powerpc64-bgq-linux"
-
-; Function Attrs: nounwind
-define void @gsl_sf_legendre_Pl_deriv_array(<4 x i32> %inp1, <4 x double> %inp2) #0 {
-entry:
-  br label %vector.body198
-
-vector.body198:                                   ; preds = %vector.body198, %for.body46.lr.ph
-  %0 = icmp ne <4 x i32> %inp1, zeroinitializer
-  %1 = select <4 x i1> %0, <4 x double> <double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01>, <4 x double> <double -5.000000e-01, double -5.000000e-01, double -5.000000e-01, double -5.000000e-01>
-  %2 = fmul <4 x double> %inp2, %1
-  %3 = fmul <4 x double> %inp2, %2
-  %4 = fmul <4 x double> %3, %inp2
-  store <4 x double> %4, <4 x double>* undef, align 8
-  br label %return
-
-; CHECK-LABEL: @gsl_sf_legendre_Pl_deriv_array
-; CHECK: qvlfiwzx
-; CHECK: qvfcfidu
-; CHECK: qvfcmpeq
-; CHECK: qvfsel
-; CHECK: qvfmul
-
-return:                                           ; preds = %if.else.i
-  ret void
-}
-
-attributes #0 = { nounwind }
-
diff --git a/llvm/test/CodeGen/PowerPC/qpx-store.ll b/llvm/test/CodeGen/PowerPC/qpx-store.ll
deleted file mode 100644
index 2b96576ce4493..0000000000000
--- a/llvm/test/CodeGen/PowerPC/qpx-store.ll
+++ /dev/null
@@ -1,25 +0,0 @@
-; RUN: llc -verify-machineinstrs < %s -mcpu=a2q | FileCheck %s
-target triple = "powerpc64-bgq-linux"
-
-define void @foo(<4 x double> %v, <4 x double>* %p) {
-entry:
-  store <4 x double> %v, <4 x double>* %p, align 8
-  ret void
-}
-
-; CHECK: @foo
-; CHECK: stfd
-; CHECK: stfd
-; CHECK: stfd
-; CHECK: stfd
-; CHECK: blr
-
-define void @bar(<4 x double> %v, <4 x double>* %p) {
-entry:
-  store <4 x double> %v, <4 x double>* %p, align 32
-  ret void
-}
-
-; CHECK: @bar
-; CHECK: qvstfdx
-
diff --git a/llvm/test/CodeGen/PowerPC/qpx-unal-cons-lds.ll b/llvm/test/CodeGen/PowerPC/qpx-unal-cons-lds.ll
deleted file mode 100644
index e7ab92db6efc9..0000000000000
--- a/llvm/test/CodeGen/PowerPC/qpx-unal-cons-lds.ll
+++ /dev/null
@@ -1,217 +0,0 @@
-; RUN: llc -verify-machineinstrs < %s | FileCheck %s
-target datalayout = "E-m:e-i64:64-n32:64"
-target triple = "powerpc64-unknown-linux-gnu"
-
-; Function Attrs: nounwind
-define void @foo(double* noalias nocapture %a, double* noalias nocapture readonly %b) #0 {
-entry:
-  br label %vector.body
-
-; CHECK-LABEL: @foo
-; Make sure that the offset constants we use are all even (only the last should be odd).
-; CHECK-DAG: li {{[0-9]+}}, 1056
-; CHECK-DAG: li {{[0-9]+}}, 1088
-; CHECK-DAG: li {{[0-9]+}}, 1152
-; CHECK-DAG: li {{[0-9]+}}, 1216
-; CHECK-DAG: li {{[0-9]+}}, 1280
-; CHECK-DAG: li {{[0-9]+}}, 1344
-; CHECK-DAG: li {{[0-9]+}}, 1408
-; CHECK-DAG: li {{[0-9]+}}, 1472
-; CHECK-DAG: li {{[0-9]+}}, 1536
-; CHECK-DAG: li {{[0-9]+}}, 1600
-; CHECK-DAG: li {{[0-9]+}}, 1568
-; CHECK-DAG: li {{[0-9]+}}, 1664
-; CHECK-DAG: li {{[0-9]+}}, 1632
-; CHECK-DAG: li {{[0-9]+}}, 1728
-; CHECK-DAG: li {{[0-9]+}}, 1696
-; CHECK-DAG: li {{[0-9]+}}, 1792
-; CHECK-DAG: li {{[0-9]+}}, 1760
-; CHECK-DAG: li {{[0-9]+}}, 1856
-; CHECK-DAG: li {{[0-9]+}}, 1824
-; CHECK-DAG: li {{[0-9]+}}, 1920
-; CHECK-DAG: li {{[0-9]+}}, 1888
-; CHECK-DAG: li {{[0-9]+}}, 1984
-; CHECK-DAG: li {{[0-9]+}}, 1952
-; CHECK-DAG: li {{[0-9]+}}, 2016
-; CHECK-DAG: li {{[0-9]+}}, 1024
-; CHECK-DAG: li {{[0-9]+}}, 1120
-; CHECK-DAG: li {{[0-9]+}}, 1184
-; CHECK-DAG: li {{[0-9]+}}, 1248
-; CHECK-DAG: li {{[0-9]+}}, 1312
-; CHECK-DAG: li {{[0-9]+}}, 1376
-; CHECK-DAG: li {{[0-9]+}}, 1440
-; CHECK-DAG: li {{[0-9]+}}, 1504
-; CHECK-DAG: li {{[0-9]+}}, 2047
-; CHECK: blr
-
-vector.body:                                      ; preds = %vector.body, %entry
-  %index = phi i64 [ 0, %entry ], [ %index.next.15, %vector.body ]
-  %0 = shl i64 %index, 1
-  %1 = getelementptr inbounds double, double* %b, i64 %0
-  %2 = bitcast double* %1 to <8 x double>*
-  %wide.vec = load <8 x double>, <8 x double>* %2, align 8
-  %strided.vec = shufflevector <8 x double> %wide.vec, <8 x double> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-  %3 = fadd <4 x double> %strided.vec, <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
-  %4 = getelementptr inbounds double, double* %a, i64 %index
-  %5 = bitcast double* %4 to <4 x double>*
-  store <4 x double> %3, <4 x double>* %5, align 8
-  %index.next = or i64 %index, 4
-  %6 = shl i64 %index.next, 1
-  %7 = getelementptr inbounds double, double* %b, i64 %6
-  %8 = bitcast double* %7 to <8 x double>*
-  %wide.vec.1 = load <8 x double>, <8 x double>* %8, align 8
-  %strided.vec.1 = shufflevector <8 x double> %wide.vec.1, <8 x double> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-  %9 = fadd <4 x double> %strided.vec.1, <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
-  %10 = getelementptr inbounds double, double* %a, i64 %index.next
-  %11 = bitcast double* %10 to <4 x double>*
-  store <4 x double> %9, <4 x double>* %11, align 8
-  %index.next.1 = or i64 %index, 8
-  %12 = shl i64 %index.next.1, 1
-  %13 = getelementptr inbounds double, double* %b, i64 %12
-  %14 = bitcast double* %13 to <8 x double>*
-  %wide.vec.2 = load <8 x double>, <8 x double>* %14, align 8
-  %strided.vec.2 = shufflevector <8 x double> %wide.vec.2, <8 x double> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-  %15 = fadd <4 x double> %strided.vec.2, <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
-  %16 = getelementptr inbounds double, double* %a, i64 %index.next.1
-  %17 = bitcast double* %16 to <4 x double>*
-  store <4 x double> %15, <4 x double>* %17, align 8
-  %index.next.2 = or i64 %index, 12
-  %18 = shl i64 %index.next.2, 1
-  %19 = getelementptr inbounds double, double* %b, i64 %18
-  %20 = bitcast double* %19 to <8 x double>*
-  %wide.vec.3 = load <8 x double>, <8 x double>* %20, align 8
-  %strided.vec.3 = shufflevector <8 x double> %wide.vec.3, <8 x double> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-  %21 = fadd <4 x double> %strided.vec.3, <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
-  %22 = getelementptr inbounds double, double* %a, i64 %index.next.2
-  %23 = bitcast double* %22 to <4 x double>*
-  store <4 x double> %21, <4 x double>* %23, align 8
-  %index.next.3 = or i64 %index, 16
-  %24 = shl i64 %index.next.3, 1
-  %25 = getelementptr inbounds double, double* %b, i64 %24
-  %26 = bitcast double* %25 to <8 x double>*
-  %wide.vec.4 = load <8 x double>, <8 x double>* %26, align 8
-  %strided.vec.4 = shufflevector <8 x double> %wide.vec.4, <8 x double> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-  %27 = fadd <4 x double> %strided.vec.4, <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
-  %28 = getelementptr inbounds double, double* %a, i64 %index.next.3
-  %29 = bitcast double* %28 to <4 x double>*
-  store <4 x double> %27, <4 x double>* %29, align 8
-  %index.next.4 = or i64 %index, 20
-  %30 = shl i64 %index.next.4, 1
-  %31 = getelementptr inbounds double, double* %b, i64 %30
-  %32 = bitcast double* %31 to <8 x double>*
-  %wide.vec.5 = load <8 x double>, <8 x double>* %32, align 8
-  %strided.vec.5 = shufflevector <8 x double> %wide.vec.5, <8 x double> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-  %33 = fadd <4 x double> %strided.vec.5, <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
-  %34 = getelementptr inbounds double, double* %a, i64 %index.next.4
-  %35 = bitcast double* %34 to <4 x double>*
-  store <4 x double> %33, <4 x double>* %35, align 8
-  %index.next.5 = or i64 %index, 24
-  %36 = shl i64 %index.next.5, 1
-  %37 = getelementptr inbounds double, double* %b, i64 %36
-  %38 = bitcast double* %37 to <8 x double>*
-  %wide.vec.6 = load <8 x double>, <8 x double>* %38, align 8
-  %strided.vec.6 = shufflevector <8 x double> %wide.vec.6, <8 x double> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-  %39 = fadd <4 x double> %strided.vec.6, <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
-  %40 = getelementptr inbounds double, double* %a, i64 %index.next.5
-  %41 = bitcast double* %40 to <4 x double>*
-  store <4 x double> %39, <4 x double>* %41, align 8
-  %index.next.6 = or i64 %index, 28
-  %42 = shl i64 %index.next.6, 1
-  %43 = getelementptr inbounds double, double* %b, i64 %42
-  %44 = bitcast double* %43 to <8 x double>*
-  %wide.vec.7 = load <8 x double>, <8 x double>* %44, align 8
-  %strided.vec.7 = shufflevector <8 x double> %wide.vec.7, <8 x double> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-  %45 = fadd <4 x double> %strided.vec.7, <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
-  %46 = getelementptr inbounds double, double* %a, i64 %index.next.6
-  %47 = bitcast double* %46 to <4 x double>*
-  store <4 x double> %45, <4 x double>* %47, align 8
-  %index.next.7 = or i64 %index, 32
-  %48 = shl i64 %index.next.7, 1
-  %49 = getelementptr inbounds double, double* %b, i64 %48
-  %50 = bitcast double* %49 to <8 x double>*
-  %wide.vec.8 = load <8 x double>, <8 x double>* %50, align 8
-  %strided.vec.8 = shufflevector <8 x double> %wide.vec.8, <8 x double> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-  %51 = fadd <4 x double> %strided.vec.8, <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
-  %52 = getelementptr inbounds double, double* %a, i64 %index.next.7
-  %53 = bitcast double* %52 to <4 x double>*
-  store <4 x double> %51, <4 x double>* %53, align 8
-  %index.next.8 = or i64 %index, 36
-  %54 = shl i64 %index.next.8, 1
-  %55 = getelementptr inbounds double, double* %b, i64 %54
-  %56 = bitcast double* %55 to <8 x double>*
-  %wide.vec.9 = load <8 x double>, <8 x double>* %56, align 8
-  %strided.vec.9 = shufflevector <8 x double> %wide.vec.9, <8 x double> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-  %57 = fadd <4 x double> %strided.vec.9, <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
-  %58 = getelementptr inbounds double, double* %a, i64 %index.next.8
-  %59 = bitcast double* %58 to <4 x double>*
-  store <4 x double> %57, <4 x double>* %59, align 8
-  %index.next.9 = or i64 %index, 40
-  %60 = shl i64 %index.next.9, 1
-  %61 = getelementptr inbounds double, double* %b, i64 %60
-  %62 = bitcast double* %61 to <8 x double>*
-  %wide.vec.10 = load <8 x double>, <8 x double>* %62, align 8
-  %strided.vec.10 = shufflevector <8 x double> %wide.vec.10, <8 x double> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-  %63 = fadd <4 x double> %strided.vec.10, <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
-  %64 = getelementptr inbounds double, double* %a, i64 %index.next.9
-  %65 = bitcast double* %64 to <4 x double>*
-  store <4 x double> %63, <4 x double>* %65, align 8
-  %index.next.10 = or i64 %index, 44
-  %66 = shl i64 %index.next.10, 1
-  %67 = getelementptr inbounds double, double* %b, i64 %66
-  %68 = bitcast double* %67 to <8 x double>*
-  %wide.vec.11 = load <8 x double>, <8 x double>* %68, align 8
-  %strided.vec.11 = shufflevector <8 x double> %wide.vec.11, <8 x double> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-  %69 = fadd <4 x double> %strided.vec.11, <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
-  %70 = getelementptr inbounds double, double* %a, i64 %index.next.10
-  %71 = bitcast double* %70 to <4 x double>*
-  store <4 x double> %69, <4 x double>* %71, align 8
-  %index.next.11 = or i64 %index, 48
-  %72 = shl i64 %index.next.11, 1
-  %73 = getelementptr inbounds double, double* %b, i64 %72
-  %74 = bitcast double* %73 to <8 x double>*
-  %wide.vec.12 = load <8 x double>, <8 x double>* %74, align 8
-  %strided.vec.12 = shufflevector <8 x double> %wide.vec.12, <8 x double> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-  %75 = fadd <4 x double> %strided.vec.12, <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
-  %76 = getelementptr inbounds double, double* %a, i64 %index.next.11
-  %77 = bitcast double* %76 to <4 x double>*
-  store <4 x double> %75, <4 x double>* %77, align 8
-  %index.next.12 = or i64 %index, 52
-  %78 = shl i64 %index.next.12, 1
-  %79 = getelementptr inbounds double, double* %b, i64 %78
-  %80 = bitcast double* %79 to <8 x double>*
-  %wide.vec.13 = load <8 x double>, <8 x double>* %80, align 8
-  %strided.vec.13 = shufflevector <8 x double> %wide.vec.13, <8 x double> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-  %81 = fadd <4 x double> %strided.vec.13, <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
-  %82 = getelementptr inbounds double, double* %a, i64 %index.next.12
-  %83 = bitcast double* %82 to <4 x double>*
-  store <4 x double> %81, <4 x double>* %83, align 8
-  %index.next.13 = or i64 %index, 56
-  %84 = shl i64 %index.next.13, 1
-  %85 = getelementptr inbounds double, double* %b, i64 %84
-  %86 = bitcast double* %85 to <8 x double>*
-  %wide.vec.14 = load <8 x double>, <8 x double>* %86, align 8
-  %strided.vec.14 = shufflevector <8 x double> %wide.vec.14, <8 x double> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-  %87 = fadd <4 x double> %strided.vec.14, <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
-  %88 = getelementptr inbounds double, double* %a, i64 %index.next.13
-  %89 = bitcast double* %88 to <4 x double>*
-  store <4 x double> %87, <4 x double>* %89, align 8
-  %index.next.14 = or i64 %index, 60
-  %90 = shl i64 %index.next.14, 1
-  %91 = getelementptr inbounds double, double* %b, i64 %90
-  %92 = bitcast double* %91 to <8 x double>*
-  %wide.vec.15 = load <8 x double>, <8 x double>* %92, align 8
-  %strided.vec.15 = shufflevector <8 x double> %wide.vec.15, <8 x double> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-  %93 = fadd <4 x double> %strided.vec.15, <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
-  %94 = getelementptr inbounds double, double* %a, i64 %index.next.14
-  %95 = bitcast double* %94 to <4 x double>*
-  store <4 x double> %93, <4 x double>* %95, align 8
-  %index.next.15 = add nsw i64 %index, 64
-  %96 = icmp eq i64 %index.next.15, 1600
-  br i1 %96, label %for.cond.cleanup, label %vector.body
-
-for.cond.cleanup:                                 ; preds = %vector.body
-  ret void
-}
-
-attributes #0 = { nounwind "target-cpu"="a2q" }
-
diff --git a/llvm/test/CodeGen/PowerPC/qpx-unalperm.ll b/llvm/test/CodeGen/PowerPC/qpx-unalperm.ll
deleted file mode 100644
index fdee919fdfc32..0000000000000
--- a/llvm/test/CodeGen/PowerPC/qpx-unalperm.ll
+++ /dev/null
@@ -1,64 +0,0 @@
-; RUN: llc -verify-machineinstrs < %s -mcpu=a2q | FileCheck %s
-target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
-target triple = "powerpc64-bgq-linux"
-
-define <4 x double> @foo(<4 x double>* %a) {
-entry:
-  %r = load <4 x double>, <4 x double>* %a, align 32
-  ret <4 x double> %r
-; CHECK: qvlfdx
-; CHECK: blr
-}
-
-define <4 x double> @bar(<4 x double>* %a) {
-entry:
-  %r = load <4 x double>, <4 x double>* %a, align 8
-  %b = getelementptr <4 x double>, <4 x double>* %a, i32 16
-  %s = load <4 x double>, <4 x double>* %b, align 32
-  %t = fadd <4 x double> %r, %s
-  ret <4 x double> %t
-; CHECK: qvlpcldx
-; CHECK: qvlfdx
-; CHECK: qvfperm
-; CHECK: blr
-}
-
-define <4 x double> @bar1(<4 x double>* %a) {
-entry:
-  %r = load <4 x double>, <4 x double>* %a, align 8
-  %b = getelementptr <4 x double>, <4 x double>* %a, i32 16
-  %s = load <4 x double>, <4 x double>* %b, align 8
-  %t = fadd <4 x double> %r, %s
-  ret <4 x double> %t
-}
-
-define <4 x double> @bar2(<4 x double>* %a) {
-entry:
-  %r = load <4 x double>, <4 x double>* %a, align 8
-  %b = getelementptr <4 x double>, <4 x double>* %a, i32 1
-  %s = load <4 x double>, <4 x double>* %b, align 32
-  %t = fadd <4 x double> %r, %s
-  ret <4 x double> %t
-}
-
-define <4 x double> @bar3(<4 x double>* %a) {
-entry:
-  %r = load <4 x double>, <4 x double>* %a, align 8
-  %b = getelementptr <4 x double>, <4 x double>* %a, i32 1
-  %s = load <4 x double>, <4 x double>* %b, align 8
-  %t = fadd <4 x double> %r, %s
-  ret <4 x double> %t
-}
-
-define <4 x double> @bar4(<4 x double>* %a) {
-entry:
-  %r = load <4 x double>, <4 x double>* %a, align 8
-  %b = getelementptr <4 x double>, <4 x double>* %a, i32 1
-  %s = load <4 x double>, <4 x double>* %b, align 8
-  %c = getelementptr <4 x double>, <4 x double>* %b, i32 1
-  %t = load <4 x double>, <4 x double>* %c, align 8
-  %u = fadd <4 x double> %r, %s
-  %v = fadd <4 x double> %u, %t
-  ret <4 x double> %v
-}
-
diff --git a/llvm/test/CodeGen/PowerPC/rlwimi-and.ll b/llvm/test/CodeGen/PowerPC/rlwimi-and.ll
index e8fc409527588..d512f51a76e7a 100644
--- a/llvm/test/CodeGen/PowerPC/rlwimi-and.ll
+++ b/llvm/test/CodeGen/PowerPC/rlwimi-and.ll
@@ -1,6 +1,4 @@
-; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mattr=-crbits < %s | FileCheck %s
-target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
-target triple = "powerpc64-bgq-linux"
+; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mattr=-crbits -mtriple=powerpc64-unknown-linux-gnu < %s | FileCheck %s
 
 define void @test() align 2 {
 entry:
diff --git a/llvm/test/CodeGen/PowerPC/rlwinm_rldicl_to_andi.mir b/llvm/test/CodeGen/PowerPC/rlwinm_rldicl_to_andi.mir
index e3aeb5605b42c..dbe314b5251fe 100644
--- a/llvm/test/CodeGen/PowerPC/rlwinm_rldicl_to_andi.mir
+++ b/llvm/test/CodeGen/PowerPC/rlwinm_rldicl_to_andi.mir
@@ -60,7 +60,7 @@
     ret i64 %cond
   }
   
-  attributes #0 = { norecurse nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+  attributes #0 = { norecurse nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector" "unsafe-fp-math"="false" "use-soft-float"="false" }
   
   !llvm.module.flags = !{!0, !1}
   !llvm.ident = !{!2}
diff --git a/llvm/test/CodeGen/PowerPC/s000-alias-misched.ll b/llvm/test/CodeGen/PowerPC/s000-alias-misched.ll
index 5c15145af2378..20071ea1710c5 100644
--- a/llvm/test/CodeGen/PowerPC/s000-alias-misched.ll
+++ b/llvm/test/CodeGen/PowerPC/s000-alias-misched.ll
@@ -1,7 +1,6 @@
-; RUN: llc -verify-machineinstrs < %s -enable-misched -mcpu=a2 | FileCheck %s
-; RUN: llc -verify-machineinstrs < %s -enable-misched -enable-aa-sched-mi -mcpu=a2 | FileCheck %s
+; RUN: llc -verify-machineinstrs < %s -enable-misched -mcpu=a2 -mtriple=powerpc64-unknown-linux-gnu | FileCheck %s
+; RUN: llc -verify-machineinstrs < %s -enable-misched -enable-aa-sched-mi -mcpu=a2 -mtriple=powerpc64-unknown-linux-gnu | FileCheck %s
 target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
-target triple = "powerpc64-bgq-linux"
 
 @aa = external global [256 x [256 x double]], align 32
 @bb = external global [256 x [256 x double]], align 32
diff --git a/llvm/test/CodeGen/PowerPC/select-i1-vs-i1.ll b/llvm/test/CodeGen/PowerPC/select-i1-vs-i1.ll
index 80ac733156197..9f458ebcf0a6e 100644
--- a/llvm/test/CodeGen/PowerPC/select-i1-vs-i1.ll
+++ b/llvm/test/CodeGen/PowerPC/select-i1-vs-i1.ll
@@ -1225,576 +1225,5 @@ entry:
 ; CHECK: blr
 }
 
-define <4 x double> @testqv4doubleslt(float %c1, float %c2, float %c3, float %c4, <4 x double> %a1, <4 x double> %a2) #1 {
-entry:
-  %cmp1 = fcmp oeq float %c3, %c4
-  %cmp3tmp = fcmp oeq float %c1, %c2
-  %cmp3 = icmp slt i1 %cmp3tmp, %cmp1
-  %cond = select i1 %cmp3, <4 x double> %a1, <4 x double> %a2
-  ret <4 x double> %cond
-
-; CHECK-LABEL: @testqv4doubleslt
-; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
-; CHECK: bc 12, 2, .LBB[[BB1:[0-9_]+]]
-; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: bc 12, 2, .LBB[[BB2:[0-9_]+]]
-; CHECK: .LBB[[BB1]]:
-; CHECK: qvfmr 5, 6
-; CHECK: .LBB[[BB2]]:
-; CHECK: qvfmr 1, 5
-; CHECK: blr
-}
-
-define <4 x double> @testqv4doubleult(float %c1, float %c2, float %c3, float %c4, <4 x double> %a1, <4 x double> %a2) #1 {
-entry:
-  %cmp1 = fcmp oeq float %c3, %c4
-  %cmp3tmp = fcmp oeq float %c1, %c2
-  %cmp3 = icmp ult i1 %cmp3tmp, %cmp1
-  %cond = select i1 %cmp3, <4 x double> %a1, <4 x double> %a2
-  ret <4 x double> %cond
-
-; CHECK-LABEL: @testqv4doubleult
-; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
-; CHECK: bc 4, 2, .LBB[[BB1:[0-9_]+]]
-; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: bc 4, 2, .LBB[[BB2:[0-9_]+]]
-; CHECK: .LBB[[BB1]]:
-; CHECK: qvfmr 5, 6
-; CHECK: .LBB[[BB2]]:
-; CHECK: qvfmr 1, 5
-; CHECK: blr
-}
-
-define <4 x double> @testqv4doublesle(float %c1, float %c2, float %c3, float %c4, <4 x double> %a1, <4 x double> %a2) #1 {
-entry:
-  %cmp1 = fcmp oeq float %c3, %c4
-  %cmp3tmp = fcmp oeq float %c1, %c2
-  %cmp3 = icmp sle i1 %cmp3tmp, %cmp1
-  %cond = select i1 %cmp3, <4 x double> %a1, <4 x double> %a2
-  ret <4 x double> %cond
-
-; CHECK-LABEL: @testqv4doublesle
-; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
-; CHECK: bc 4, 2, .LBB[[BB:[0-9_]+]]
-; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: bc 12, 2, .LBB[[BB]]
-; CHECK: qvfmr 5, 6
-; CHECK: .LBB[[BB]]:
-; CHECK: qvfmr 1, 5
-; CHECK: blr
-}
-
-define <4 x double> @testqv4doubleule(float %c1, float %c2, float %c3, float %c4, <4 x double> %a1, <4 x double> %a2) #1 {
-entry:
-  %cmp1 = fcmp oeq float %c3, %c4
-  %cmp3tmp = fcmp oeq float %c1, %c2
-  %cmp3 = icmp ule i1 %cmp3tmp, %cmp1
-  %cond = select i1 %cmp3, <4 x double> %a1, <4 x double> %a2
-  ret <4 x double> %cond
-
-; CHECK-LABEL: @testqv4doubleule
-; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
-; CHECK: bc 12, 2, .LBB[[BB:[0-9_]+]]
-; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: bc 4, 2, .LBB[[BB]]
-; CHECK: qvfmr 5, 6
-; CHECK: .LBB[[BB]]:
-; CHECK: qvfmr 1, 5
-; CHECK: blr
-}
-
-define <4 x double> @testqv4doubleeq(float %c1, float %c2, float %c3, float %c4, <4 x double> %a1, <4 x double> %a2) #1 {
-entry:
-  %cmp1 = fcmp oeq float %c3, %c4
-  %cmp3tmp = fcmp oeq float %c1, %c2
-  %cmp3 = icmp eq i1 %cmp3tmp, %cmp1
-  %cond = select i1 %cmp3, <4 x double> %a1, <4 x double> %a2
-  ret <4 x double> %cond
-
-; CHECK-LABEL: @testqv4doubleeq
-; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
-; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: creqv [[REG1:[0-9]+]], {{[0-9]+}}, {{[0-9]+}}
-; CHECK: bclr 12, [[REG1]], 0
-; CHECK: qvfmr 1, 6
-; CHECK: blr
-}
-
-define <4 x double> @testqv4doublesge(float %c1, float %c2, float %c3, float %c4, <4 x double> %a1, <4 x double> %a2) #1 {
-entry:
-  %cmp1 = fcmp oeq float %c3, %c4
-  %cmp3tmp = fcmp oeq float %c1, %c2
-  %cmp3 = icmp sge i1 %cmp3tmp, %cmp1
-  %cond = select i1 %cmp3, <4 x double> %a1, <4 x double> %a2
-  ret <4 x double> %cond
-
-; CHECK-LABEL: @testqv4doublesge
-; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
-; CHECK: bc 12, 2, .LBB[[BB:[0-9_]+]]
-; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: bc 4, 2, .LBB[[BB]]
-; CHECK: qvfmr 5, 6
-; CHECK: .LBB[[BB]]:
-; CHECK: qvfmr 1, 5
-; CHECK: blr
-}
-
-define <4 x double> @testqv4doubleuge(float %c1, float %c2, float %c3, float %c4, <4 x double> %a1, <4 x double> %a2) #1 {
-entry:
-  %cmp1 = fcmp oeq float %c3, %c4
-  %cmp3tmp = fcmp oeq float %c1, %c2
-  %cmp3 = icmp uge i1 %cmp3tmp, %cmp1
-  %cond = select i1 %cmp3, <4 x double> %a1, <4 x double> %a2
-  ret <4 x double> %cond
-
-; CHECK-LABEL: @testqv4doubleuge
-; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
-; CHECK: bc 4, 2, .LBB[[BB:[0-9_]+]]
-; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: bc 12, 2, .LBB[[BB]]
-; CHECK: qvfmr 5, 6
-; CHECK: .LBB[[BB]]:
-; CHECK: qvfmr 1, 5
-; CHECK: blr
-}
-
-define <4 x double> @testqv4doublesgt(float %c1, float %c2, float %c3, float %c4, <4 x double> %a1, <4 x double> %a2) #1 {
-entry:
-  %cmp1 = fcmp oeq float %c3, %c4
-  %cmp3tmp = fcmp oeq float %c1, %c2
-  %cmp3 = icmp sgt i1 %cmp3tmp, %cmp1
-  %cond = select i1 %cmp3, <4 x double> %a1, <4 x double> %a2
-  ret <4 x double> %cond
-
-; CHECK-LABEL: @testqv4doublesgt
-; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
-; CHECK: bc 4, 2, .LBB[[BB1:[0-9_]+]]
-; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: bc 4, 2, .LBB[[BB2:[0-9_]+]]
-; CHECK: .LBB[[BB1]]:
-; CHECK: qvfmr 5, 6
-; CHECK: .LBB[[BB2]]:
-; CHECK: qvfmr 1, 5
-; CHECK: blr
-}
-
-define <4 x double> @testqv4doubleugt(float %c1, float %c2, float %c3, float %c4, <4 x double> %a1, <4 x double> %a2) #1 {
-entry:
-  %cmp1 = fcmp oeq float %c3, %c4
-  %cmp3tmp = fcmp oeq float %c1, %c2
-  %cmp3 = icmp ugt i1 %cmp3tmp, %cmp1
-  %cond = select i1 %cmp3, <4 x double> %a1, <4 x double> %a2
-  ret <4 x double> %cond
-
-; CHECK-LABEL: @testqv4doubleugt
-; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
-; CHECK: bc 12, 2, .LBB[[BB1:[0-9_]+]]
-; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: bc 12, 2, .LBB[[BB2:[0-9_]+]]
-; CHECK: .LBB[[BB1]]:
-; CHECK: qvfmr 5, 6
-; CHECK: .LBB[[BB2]]:
-; CHECK: qvfmr 1, 5
-; CHECK: blr
-}
-
-define <4 x double> @testqv4doublene(float %c1, float %c2, float %c3, float %c4, <4 x double> %a1, <4 x double> %a2) #1 {
-entry:
-  %cmp1 = fcmp oeq float %c3, %c4
-  %cmp3tmp = fcmp oeq float %c1, %c2
-  %cmp3 = icmp ne i1 %cmp3tmp, %cmp1
-  %cond = select i1 %cmp3, <4 x double> %a1, <4 x double> %a2
-  ret <4 x double> %cond
-
-; CHECK-LABEL: @testqv4doublene
-; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
-; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: crxor [[REG1:[0-9]+]], {{[0-9]+}}, {{[0-9]+}}
-; CHECK: bclr 12, [[REG1]], 0
-; CHECK: qvfmr 1, 6
-; CHECK: blr
-}
-
-define <4 x float> @testqv4floatslt(float %c1, float %c2, float %c3, float %c4, <4 x float> %a1, <4 x float> %a2) #1 {
-entry:
-  %cmp1 = fcmp oeq float %c3, %c4
-  %cmp3tmp = fcmp oeq float %c1, %c2
-  %cmp3 = icmp slt i1 %cmp3tmp, %cmp1
-  %cond = select i1 %cmp3, <4 x float> %a1, <4 x float> %a2
-  ret <4 x float> %cond
-
-; CHECK-LABEL: @testqv4floatslt
-; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
-; CHECK: bc 12, 2, .LBB[[BB1:[0-9_]+]]
-; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: bc 12, 2, .LBB[[BB2:[0-9_]+]]
-; CHECK: .LBB[[BB1]]:
-; CHECK: qvfmr 5, 6
-; CHECK: .LBB[[BB2]]:
-; CHECK: qvfmr 1, 5
-; CHECK: blr
-}
-
-define <4 x float> @testqv4floatult(float %c1, float %c2, float %c3, float %c4, <4 x float> %a1, <4 x float> %a2) #1 {
-entry:
-  %cmp1 = fcmp oeq float %c3, %c4
-  %cmp3tmp = fcmp oeq float %c1, %c2
-  %cmp3 = icmp ult i1 %cmp3tmp, %cmp1
-  %cond = select i1 %cmp3, <4 x float> %a1, <4 x float> %a2
-  ret <4 x float> %cond
-
-; CHECK-LABEL: @testqv4floatult
-; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
-; CHECK: bc 4, 2, .LBB[[BB1:[0-9_]+]]
-; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: bc 4, 2, .LBB[[BB2:[0-9_]+]]
-; CHECK: .LBB[[BB1]]:
-; CHECK: qvfmr 5, 6
-; CHECK: .LBB[[BB2]]:
-; CHECK: qvfmr 1, 5
-; CHECK: blr
-}
-
-define <4 x float> @testqv4floatsle(float %c1, float %c2, float %c3, float %c4, <4 x float> %a1, <4 x float> %a2) #1 {
-entry:
-  %cmp1 = fcmp oeq float %c3, %c4
-  %cmp3tmp = fcmp oeq float %c1, %c2
-  %cmp3 = icmp sle i1 %cmp3tmp, %cmp1
-  %cond = select i1 %cmp3, <4 x float> %a1, <4 x float> %a2
-  ret <4 x float> %cond
-
-; CHECK-LABEL: @testqv4floatsle
-; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
-; CHECK: bc 4, 2, .LBB[[BB:[0-9_]+]]
-; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: bc 12, 2, .LBB[[BB]]
-; CHECK: qvfmr 5, 6
-; CHECK: .LBB[[BB]]:
-; CHECK: qvfmr 1, 5
-; CHECK: blr
-}
-
-define <4 x float> @testqv4floatule(float %c1, float %c2, float %c3, float %c4, <4 x float> %a1, <4 x float> %a2) #1 {
-entry:
-  %cmp1 = fcmp oeq float %c3, %c4
-  %cmp3tmp = fcmp oeq float %c1, %c2
-  %cmp3 = icmp ule i1 %cmp3tmp, %cmp1
-  %cond = select i1 %cmp3, <4 x float> %a1, <4 x float> %a2
-  ret <4 x float> %cond
-
-; CHECK-LABEL: @testqv4floatule
-; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
-; CHECK: bc 12, 2, .LBB[[BB:[0-9_]+]]
-; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: bc 4, 2, .LBB[[BB]]
-; CHECK: qvfmr 5, 6
-; CHECK: .LBB[[BB]]:
-; CHECK: qvfmr 1, 5
-; CHECK: blr
-}
-
-define <4 x float> @testqv4floateq(float %c1, float %c2, float %c3, float %c4, <4 x float> %a1, <4 x float> %a2) #1 {
-entry:
-  %cmp1 = fcmp oeq float %c3, %c4
-  %cmp3tmp = fcmp oeq float %c1, %c2
-  %cmp3 = icmp eq i1 %cmp3tmp, %cmp1
-  %cond = select i1 %cmp3, <4 x float> %a1, <4 x float> %a2
-  ret <4 x float> %cond
-
-; CHECK-LABEL: @testqv4floateq
-; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
-; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: creqv [[REG1:[0-9]+]], {{[0-9]+}}, {{[0-9]+}}
-; CHECK: bclr 12, [[REG1]], 0
-; CHECK: qvfmr 1, 6
-; CHECK: blr
-}
-
-define <4 x float> @testqv4floatsge(float %c1, float %c2, float %c3, float %c4, <4 x float> %a1, <4 x float> %a2) #1 {
-entry:
-  %cmp1 = fcmp oeq float %c3, %c4
-  %cmp3tmp = fcmp oeq float %c1, %c2
-  %cmp3 = icmp sge i1 %cmp3tmp, %cmp1
-  %cond = select i1 %cmp3, <4 x float> %a1, <4 x float> %a2
-  ret <4 x float> %cond
-
-; CHECK-LABEL: @testqv4floatsge
-; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
-; CHECK: bc 12, 2, .LBB[[BB:[0-9_]+]]
-; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: bc 4, 2, .LBB[[BB]]
-; CHECK: qvfmr 5, 6
-; CHECK: .LBB[[BB]]:
-; CHECK: qvfmr 1, 5
-; CHECK: blr
-}
-
-define <4 x float> @testqv4floatuge(float %c1, float %c2, float %c3, float %c4, <4 x float> %a1, <4 x float> %a2) #1 {
-entry:
-  %cmp1 = fcmp oeq float %c3, %c4
-  %cmp3tmp = fcmp oeq float %c1, %c2
-  %cmp3 = icmp uge i1 %cmp3tmp, %cmp1
-  %cond = select i1 %cmp3, <4 x float> %a1, <4 x float> %a2
-  ret <4 x float> %cond
-
-; CHECK-LABEL: @testqv4floatuge
-; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
-; CHECK: bc 4, 2, .LBB[[BB:[0-9_]+]]
-; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: bc 12, 2, .LBB[[BB]]
-; CHECK: qvfmr 5, 6
-; CHECK: .LBB[[BB]]:
-; CHECK: qvfmr 1, 5
-; CHECK: blr
-}
-
-define <4 x float> @testqv4floatsgt(float %c1, float %c2, float %c3, float %c4, <4 x float> %a1, <4 x float> %a2) #1 {
-entry:
-  %cmp1 = fcmp oeq float %c3, %c4
-  %cmp3tmp = fcmp oeq float %c1, %c2
-  %cmp3 = icmp sgt i1 %cmp3tmp, %cmp1
-  %cond = select i1 %cmp3, <4 x float> %a1, <4 x float> %a2
-  ret <4 x float> %cond
-
-; CHECK-LABEL: @testqv4floatsgt
-; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
-; CHECK: bc 4, 2, .LBB[[BB1:[0-9_]+]]
-; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: bc 4, 2, .LBB[[BB2:[0-9_]+]]
-; CHECK: .LBB[[BB1]]:
-; CHECK: qvfmr 5, 6
-; CHECK: .LBB[[BB2]]:
-; CHECK: qvfmr 1, 5
-; CHECK: blr
-}
-
-define <4 x float> @testqv4floatugt(float %c1, float %c2, float %c3, float %c4, <4 x float> %a1, <4 x float> %a2) #1 {
-entry:
-  %cmp1 = fcmp oeq float %c3, %c4
-  %cmp3tmp = fcmp oeq float %c1, %c2
-  %cmp3 = icmp ugt i1 %cmp3tmp, %cmp1
-  %cond = select i1 %cmp3, <4 x float> %a1, <4 x float> %a2
-  ret <4 x float> %cond
-
-; CHECK-LABEL: @testqv4floatugt
-; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
-; CHECK: bc 12, 2, .LBB[[BB1:[0-9_]+]]
-; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: bc 12, 2, .LBB[[BB2:[0-9_]+]]
-; CHECK: .LBB[[BB1]]:
-; CHECK: qvfmr 5, 6
-; CHECK: .LBB[[BB2]]:
-; CHECK: qvfmr 1, 5
-; CHECK: blr
-}
-
-define <4 x float> @testqv4floatne(float %c1, float %c2, float %c3, float %c4, <4 x float> %a1, <4 x float> %a2) #1 {
-entry:
-  %cmp1 = fcmp oeq float %c3, %c4
-  %cmp3tmp = fcmp oeq float %c1, %c2
-  %cmp3 = icmp ne i1 %cmp3tmp, %cmp1
-  %cond = select i1 %cmp3, <4 x float> %a1, <4 x float> %a2
-  ret <4 x float> %cond
-
-; CHECK-LABEL: @testqv4floatne
-; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
-; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: crxor [[REG1:[0-9]+]], {{[0-9]+}}, {{[0-9]+}}
-; CHECK: bclr 12, [[REG1]], 0
-; CHECK: qvfmr 1, 6
-; CHECK: blr
-}
-
-define <4 x i1> @testqv4i1slt(float %c1, float %c2, float %c3, float %c4, <4 x i1> %a1, <4 x i1> %a2) #1 {
-entry:
-  %cmp1 = fcmp oeq float %c3, %c4
-  %cmp3tmp = fcmp oeq float %c1, %c2
-  %cmp3 = icmp slt i1 %cmp3tmp, %cmp1
-  %cond = select i1 %cmp3, <4 x i1> %a1, <4 x i1> %a2
-  ret <4 x i1> %cond
-
-; CHECK-LABEL: @testqv4i1slt
-; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
-; CHECK: bc 12, 2, .LBB[[BB1:[0-9_]+]]
-; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: bc 12, 2, .LBB[[BB2:[0-9_]+]]
-; CHECK: .LBB[[BB1]]:
-; CHECK: qvfmr 5, 6
-; CHECK: .LBB[[BB2]]:
-; CHECK: qvfmr 1, 5
-; CHECK: blr
-}
-
-define <4 x i1> @testqv4i1ult(float %c1, float %c2, float %c3, float %c4, <4 x i1> %a1, <4 x i1> %a2) #1 {
-entry:
-  %cmp1 = fcmp oeq float %c3, %c4
-  %cmp3tmp = fcmp oeq float %c1, %c2
-  %cmp3 = icmp ult i1 %cmp3tmp, %cmp1
-  %cond = select i1 %cmp3, <4 x i1> %a1, <4 x i1> %a2
-  ret <4 x i1> %cond
-
-; CHECK-LABEL: @testqv4i1ult
-; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
-; CHECK: bc 4, 2, .LBB[[BB1:[0-9_]+]]
-; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: bc 4, 2, .LBB[[BB2:[0-9_]+]]
-; CHECK: .LBB[[BB1]]:
-; CHECK: qvfmr 5, 6
-; CHECK: .LBB[[BB2]]:
-; CHECK: qvfmr 1, 5
-; CHECK: blr
-}
-
-define <4 x i1> @testqv4i1sle(float %c1, float %c2, float %c3, float %c4, <4 x i1> %a1, <4 x i1> %a2) #1 {
-entry:
-  %cmp1 = fcmp oeq float %c3, %c4
-  %cmp3tmp = fcmp oeq float %c1, %c2
-  %cmp3 = icmp sle i1 %cmp3tmp, %cmp1
-  %cond = select i1 %cmp3, <4 x i1> %a1, <4 x i1> %a2
-  ret <4 x i1> %cond
-
-; CHECK-LABEL: @testqv4i1sle
-; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
-; CHECK: bc 4, 2, .LBB[[BB:[0-9_]+]]
-; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: bc 12, 2, .LBB[[BB]]
-; CHECK: qvfmr 5, 6
-; CHECK: .LBB[[BB]]:
-; CHECK: qvfmr 1, 5
-; CHECK: blr
-}
-
-define <4 x i1> @testqv4i1ule(float %c1, float %c2, float %c3, float %c4, <4 x i1> %a1, <4 x i1> %a2) #1 {
-entry:
-  %cmp1 = fcmp oeq float %c3, %c4
-  %cmp3tmp = fcmp oeq float %c1, %c2
-  %cmp3 = icmp ule i1 %cmp3tmp, %cmp1
-  %cond = select i1 %cmp3, <4 x i1> %a1, <4 x i1> %a2
-  ret <4 x i1> %cond
-
-; CHECK-LABEL: @testqv4i1ule
-; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
-; CHECK: bc 12, 2, .LBB[[BB:[0-9_]+]]
-; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: bc 4, 2, .LBB[[BB]]
-; CHECK: qvfmr 5, 6
-; CHECK: .LBB[[BB]]:
-; CHECK: qvfmr 1, 5
-; CHECK: blr
-}
-
-define <4 x i1> @testqv4i1eq(float %c1, float %c2, float %c3, float %c4, <4 x i1> %a1, <4 x i1> %a2) #1 {
-entry:
-  %cmp1 = fcmp oeq float %c3, %c4
-  %cmp3tmp = fcmp oeq float %c1, %c2
-  %cmp3 = icmp eq i1 %cmp3tmp, %cmp1
-  %cond = select i1 %cmp3, <4 x i1> %a1, <4 x i1> %a2
-  ret <4 x i1> %cond
-
-; CHECK-LABEL: @testqv4i1eq
-; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
-; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: creqv [[REG1:[0-9]+]], {{[0-9]+}}, {{[0-9]+}}
-; CHECK: bclr 12, [[REG1]], 0
-; CHECK: qvfmr 1, 6
-; CHECK: blr
-}
-
-define <4 x i1> @testqv4i1sge(float %c1, float %c2, float %c3, float %c4, <4 x i1> %a1, <4 x i1> %a2) #1 {
-entry:
-  %cmp1 = fcmp oeq float %c3, %c4
-  %cmp3tmp = fcmp oeq float %c1, %c2
-  %cmp3 = icmp sge i1 %cmp3tmp, %cmp1
-  %cond = select i1 %cmp3, <4 x i1> %a1, <4 x i1> %a2
-  ret <4 x i1> %cond
-
-; CHECK-LABEL: @testqv4i1sge
-; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
-; CHECK: bc 12, 2, .LBB[[BB:[0-9_]+]]
-; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: bc 4, 2, .LBB[[BB]]
-; CHECK: qvfmr 5, 6
-; CHECK: .LBB[[BB]]:
-; CHECK: qvfmr 1, 5
-; CHECK: blr
-}
-
-define <4 x i1> @testqv4i1uge(float %c1, float %c2, float %c3, float %c4, <4 x i1> %a1, <4 x i1> %a2) #1 {
-entry:
-  %cmp1 = fcmp oeq float %c3, %c4
-  %cmp3tmp = fcmp oeq float %c1, %c2
-  %cmp3 = icmp uge i1 %cmp3tmp, %cmp1
-  %cond = select i1 %cmp3, <4 x i1> %a1, <4 x i1> %a2
-  ret <4 x i1> %cond
-
-; CHECK-LABEL: @testqv4i1uge
-; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
-; CHECK: bc 4, 2, .LBB[[BB:[0-9_]+]]
-; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: bc 12, 2, .LBB[[BB]]
-; CHECK: qvfmr 5, 6
-; CHECK: .LBB[[BB]]:
-; CHECK: qvfmr 1, 5
-; CHECK: blr
-}
-
-define <4 x i1> @testqv4i1sgt(float %c1, float %c2, float %c3, float %c4, <4 x i1> %a1, <4 x i1> %a2) #1 {
-entry:
-  %cmp1 = fcmp oeq float %c3, %c4
-  %cmp3tmp = fcmp oeq float %c1, %c2
-  %cmp3 = icmp sgt i1 %cmp3tmp, %cmp1
-  %cond = select i1 %cmp3, <4 x i1> %a1, <4 x i1> %a2
-  ret <4 x i1> %cond
-
-; CHECK-LABEL: @testqv4i1sgt
-; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
-; CHECK: bc 4, 2, .LBB[[BB1:[0-9_]+]]
-; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: bc 4, 2, .LBB[[BB2:[0-9_]+]]
-; CHECK: .LBB[[BB1]]:
-; CHECK: qvfmr 5, 6
-; CHECK: .LBB[[BB2]]:
-; CHECK: qvfmr 1, 5
-; CHECK: blr
-}
-
-define <4 x i1> @testqv4i1ugt(float %c1, float %c2, float %c3, float %c4, <4 x i1> %a1, <4 x i1> %a2) #1 {
-entry:
-  %cmp1 = fcmp oeq float %c3, %c4
-  %cmp3tmp = fcmp oeq float %c1, %c2
-  %cmp3 = icmp ugt i1 %cmp3tmp, %cmp1
-  %cond = select i1 %cmp3, <4 x i1> %a1, <4 x i1> %a2
-  ret <4 x i1> %cond
-
-; CHECK-LABEL: @testqv4i1ugt
-; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
-; CHECK: bc 12, 2, .LBB[[BB1:[0-9_]+]]
-; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: bc 12, 2, .LBB[[BB2:[0-9_]+]]
-; CHECK: .LBB[[BB1]]:
-; CHECK: qvfmr 5, 6
-; CHECK: .LBB[[BB2]]:
-; CHECK: qvfmr 1, 5
-; CHECK: blr
-}
-
-define <4 x i1> @testqv4i1ne(float %c1, float %c2, float %c3, float %c4, <4 x i1> %a1, <4 x i1> %a2) #1 {
-entry:
-  %cmp1 = fcmp oeq float %c3, %c4
-  %cmp3tmp = fcmp oeq float %c1, %c2
-  %cmp3 = icmp ne i1 %cmp3tmp, %cmp1
-  %cond = select i1 %cmp3, <4 x i1> %a1, <4 x i1> %a2
-  ret <4 x i1> %cond
-
-; CHECK-LABEL: @testqv4i1ne
-; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
-; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: crxor [[REG1:[0-9]+]], {{[0-9]+}}, {{[0-9]+}}
-; CHECK: bclr 12, [[REG1]], 0
-; CHECK: qvfmr 1, 6
-; CHECK: blr
-}
-
 attributes #0 = { nounwind readnone "target-cpu"="pwr7" }
-attributes #1 = { nounwind readnone "target-cpu"="a2q" }
 
diff --git a/llvm/test/CodeGen/PowerPC/selectiondag-extload-computeknownbits.ll b/llvm/test/CodeGen/PowerPC/selectiondag-extload-computeknownbits.ll
index 53d17d8668270..73fce78c33aa7 100644
--- a/llvm/test/CodeGen/PowerPC/selectiondag-extload-computeknownbits.ll
+++ b/llvm/test/CodeGen/PowerPC/selectiondag-extload-computeknownbits.ll
@@ -1,4 +1,4 @@
-; RUN: llc -verify-machineinstrs -mtriple=powerpc64-bgq-linux < %s
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux < %s
 
 ; Check that llc does not crash due to an illegal APInt operation
 
diff --git a/llvm/test/CodeGen/PowerPC/setcr_bc.mir b/llvm/test/CodeGen/PowerPC/setcr_bc.mir
index e9d81da681fcc..564ee7d45957b 100644
--- a/llvm/test/CodeGen/PowerPC/setcr_bc.mir
+++ b/llvm/test/CodeGen/PowerPC/setcr_bc.mir
@@ -32,8 +32,8 @@
     ret i32 %call2.i.sink
   }
   
-  attributes #0 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
-  attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+  attributes #0 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector" "unsafe-fp-math"="false" "use-soft-float"="false" }
+  attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 ...
 ---
diff --git a/llvm/test/CodeGen/PowerPC/setcr_bc2.mir b/llvm/test/CodeGen/PowerPC/setcr_bc2.mir
index 582284d6d0a59..513cb85e1580a 100644
--- a/llvm/test/CodeGen/PowerPC/setcr_bc2.mir
+++ b/llvm/test/CodeGen/PowerPC/setcr_bc2.mir
@@ -32,8 +32,8 @@
     ret i32 %call2.i.sink
   }
   
-  attributes #0 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
-  attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+  attributes #0 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector" "unsafe-fp-math"="false" "use-soft-float"="false" }
+  attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 ...
 ---
diff --git a/llvm/test/CodeGen/PowerPC/stwu-sched.ll b/llvm/test/CodeGen/PowerPC/stwu-sched.ll
index 0afd2ee406894..36afaf84a296b 100644
--- a/llvm/test/CodeGen/PowerPC/stwu-sched.ll
+++ b/llvm/test/CodeGen/PowerPC/stwu-sched.ll
@@ -58,7 +58,7 @@ define void @initCombList(%0* nocapture, i32 signext) local_unnamed_addr #0 {
   ret void
 }
 
-attributes #0 = { norecurse nounwind writeonly "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse nounwind writeonly "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.ident = !{!0}
 
diff --git a/llvm/test/CodeGen/PowerPC/unal-vec-ldst.ll b/llvm/test/CodeGen/PowerPC/unal-vec-ldst.ll
index 497add38e0444..79a368dd095ac 100644
--- a/llvm/test/CodeGen/PowerPC/unal-vec-ldst.ll
+++ b/llvm/test/CodeGen/PowerPC/unal-vec-ldst.ll
@@ -327,72 +327,6 @@ entry:
 
 }
 
-define <4 x float> @test_l_qv4float(<4 x float>* %p) #1 {
-; CHECK-LABEL: test_l_qv4float:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    li 4, 15
-; CHECK-NEXT:    qvlpclsx 0, 0, 3
-; CHECK-NEXT:    qvlfsx 1, 3, 4
-; CHECK-NEXT:    qvlfsx 2, 0, 3
-; CHECK-NEXT:    qvfperm 1, 2, 1, 0
-; CHECK-NEXT:    blr
-entry:
-  %r = load <4 x float>, <4 x float>* %p, align 4
-  ret <4 x float> %r
-
-}
-
-define <8 x float> @test_l_qv8float(<8 x float>* %p) #1 {
-; CHECK-LABEL: test_l_qv8float:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    li 4, 31
-; CHECK-NEXT:    qvlpclsx 1, 0, 3
-; CHECK-NEXT:    qvlfsx 0, 3, 4
-; CHECK-NEXT:    li 4, 16
-; CHECK-NEXT:    qvlfsx 3, 3, 4
-; CHECK-NEXT:    qvlfsx 4, 0, 3
-; CHECK-NEXT:    qvfperm 2, 3, 0, 1
-; CHECK-NEXT:    qvfperm 1, 4, 3, 1
-; CHECK-NEXT:    blr
-entry:
-  %r = load <8 x float>, <8 x float>* %p, align 4
-  ret <8 x float> %r
-
-}
-
-define <4 x double> @test_l_qv4double(<4 x double>* %p) #1 {
-; CHECK-LABEL: test_l_qv4double:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    li 4, 31
-; CHECK-NEXT:    qvlpcldx 0, 0, 3
-; CHECK-NEXT:    qvlfdx 1, 3, 4
-; CHECK-NEXT:    qvlfdx 2, 0, 3
-; CHECK-NEXT:    qvfperm 1, 2, 1, 0
-; CHECK-NEXT:    blr
-entry:
-  %r = load <4 x double>, <4 x double>* %p, align 8
-  ret <4 x double> %r
-
-}
-
-define <8 x double> @test_l_qv8double(<8 x double>* %p) #1 {
-; CHECK-LABEL: test_l_qv8double:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    li 4, 63
-; CHECK-NEXT:    qvlpcldx 1, 0, 3
-; CHECK-NEXT:    qvlfdx 0, 3, 4
-; CHECK-NEXT:    li 4, 32
-; CHECK-NEXT:    qvlfdx 3, 3, 4
-; CHECK-NEXT:    qvlfdx 4, 0, 3
-; CHECK-NEXT:    qvfperm 2, 3, 0, 1
-; CHECK-NEXT:    qvfperm 1, 4, 3, 1
-; CHECK-NEXT:    blr
-entry:
-  %r = load <8 x double>, <8 x double>* %p, align 8
-  ret <8 x double> %r
-
-}
-
 define void @test_s_v16i8(<16 x i8>* %p, <16 x i8> %v) #0 {
 ; CHECK-LABEL: test_s_v16i8:
 ; CHECK:       # %bb.0: # %entry
@@ -537,89 +471,6 @@ entry:
 
 }
 
-define void @test_s_qv4float(<4 x float>* %p, <4 x float> %v) #1 {
-; CHECK-LABEL: test_s_qv4float:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    qvesplati 0, 1, 3
-; CHECK-NEXT:    stfs 1, 0(3)
-; CHECK-NEXT:    stfs 0, 12(3)
-; CHECK-NEXT:    qvesplati 0, 1, 2
-; CHECK-NEXT:    qvesplati 1, 1, 1
-; CHECK-NEXT:    stfs 0, 8(3)
-; CHECK-NEXT:    stfs 1, 4(3)
-; CHECK-NEXT:    blr
-entry:
-  store <4 x float> %v, <4 x float>* %p, align 4
-  ret void
-
-}
-
-define void @test_s_qv8float(<8 x float>* %p, <8 x float> %v) #1 {
-; CHECK-LABEL: test_s_qv8float:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    qvesplati 0, 2, 3
-; CHECK-NEXT:    stfs 2, 16(3)
-; CHECK-NEXT:    stfs 0, 28(3)
-; CHECK-NEXT:    qvesplati 0, 2, 2
-; CHECK-NEXT:    qvesplati 2, 2, 1
-; CHECK-NEXT:    stfs 1, 0(3)
-; CHECK-NEXT:    stfs 0, 24(3)
-; CHECK-NEXT:    qvesplati 0, 1, 3
-; CHECK-NEXT:    stfs 2, 20(3)
-; CHECK-NEXT:    qvesplati 2, 1, 2
-; CHECK-NEXT:    qvesplati 1, 1, 1
-; CHECK-NEXT:    stfs 0, 12(3)
-; CHECK-NEXT:    stfs 2, 8(3)
-; CHECK-NEXT:    stfs 1, 4(3)
-; CHECK-NEXT:    blr
-entry:
-  store <8 x float> %v, <8 x float>* %p, align 4
-  ret void
-
-}
-
-define void @test_s_qv4double(<4 x double>* %p, <4 x double> %v) #1 {
-; CHECK-LABEL: test_s_qv4double:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    qvesplati 0, 1, 3
-; CHECK-NEXT:    stfd 1, 0(3)
-; CHECK-NEXT:    stfd 0, 24(3)
-; CHECK-NEXT:    qvesplati 0, 1, 2
-; CHECK-NEXT:    qvesplati 1, 1, 1
-; CHECK-NEXT:    stfd 0, 16(3)
-; CHECK-NEXT:    stfd 1, 8(3)
-; CHECK-NEXT:    blr
-entry:
-  store <4 x double> %v, <4 x double>* %p, align 8
-  ret void
-
-}
-
-define void @test_s_qv8double(<8 x double>* %p, <8 x double> %v) #1 {
-; CHECK-LABEL: test_s_qv8double:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    qvesplati 0, 2, 3
-; CHECK-NEXT:    stfd 2, 32(3)
-; CHECK-NEXT:    stfd 0, 56(3)
-; CHECK-NEXT:    qvesplati 0, 2, 2
-; CHECK-NEXT:    qvesplati 2, 2, 1
-; CHECK-NEXT:    stfd 1, 0(3)
-; CHECK-NEXT:    stfd 0, 48(3)
-; CHECK-NEXT:    qvesplati 0, 1, 3
-; CHECK-NEXT:    stfd 2, 40(3)
-; CHECK-NEXT:    qvesplati 2, 1, 2
-; CHECK-NEXT:    qvesplati 1, 1, 1
-; CHECK-NEXT:    stfd 0, 24(3)
-; CHECK-NEXT:    stfd 2, 16(3)
-; CHECK-NEXT:    stfd 1, 8(3)
-; CHECK-NEXT:    blr
-entry:
-  store <8 x double> %v, <8 x double>* %p, align 8
-  ret void
-
-}
-
 attributes #0 = { nounwind "target-cpu"="pwr7" }
-attributes #1 = { nounwind "target-cpu"="a2q" }
 attributes #2 = { nounwind "target-cpu"="pwr8" }
 
diff --git a/llvm/test/CodeGen/PowerPC/uwtables.ll b/llvm/test/CodeGen/PowerPC/uwtables.ll
index 7523d04d73d38..e302934ab8d6b 100644
--- a/llvm/test/CodeGen/PowerPC/uwtables.ll
+++ b/llvm/test/CodeGen/PowerPC/uwtables.ll
@@ -47,5 +47,5 @@ declare i32 @__gxx_personality_v0(...)
 declare void @__cxa_call_unexpected(i8*) local_unnamed_addr
 
 
-attributes #0 = { noreturn nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noreturn nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
diff --git a/llvm/test/Instrumentation/MemorySanitizer/PowerPC/vararg-ppc64.ll b/llvm/test/Instrumentation/MemorySanitizer/PowerPC/vararg-ppc64.ll
index 36da7add88015..33f3d82c3683d 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/PowerPC/vararg-ppc64.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/PowerPC/vararg-ppc64.ll
@@ -55,21 +55,6 @@ define i32 @bar2() {
 ; CHECK: store <2 x i64> zeroinitializer, <2 x i64>* inttoptr (i64 add (i64 ptrtoint ([100 x i64]* @__msan_va_arg_tls to i64), i64 8) to <2 x i64>*), align 8
 ; CHECK: store {{.*}} 24, {{.*}} @__msan_va_arg_overflow_size_tls
 
-; Check QPX vector argument.
-define i32 @bar3() "target-features"="+qpx" {
-  %1 = call i32 (i32, ...) @foo(i32 0, i32 1, i32 2, <4 x double> <double 1.0, double 2.0, double 3.0, double 4.0>)
-  ret i32 %1
-}
-
-; That one is even stranger: the parameter save area starts at offset 48 from
-; (32-byte aligned) stack pointer, the vector parameter is at 96 bytes from
-; the stack pointer, so its offset from parameter save area is misaligned.
-; CHECK-LABEL: @bar3
-; CHECK: store i32 0, i32* inttoptr (i64 add (i64 ptrtoint ([100 x i64]* @__msan_va_arg_tls to i64), i64 4) to i32*), align 8
-; CHECK: store i32 0, i32* inttoptr (i64 add (i64 ptrtoint ([100 x i64]* @__msan_va_arg_tls to i64), i64 12) to i32*), align 8
-; CHECK: store <4 x i64> zeroinitializer, <4 x i64>* inttoptr (i64 add (i64 ptrtoint ([100 x i64]* @__msan_va_arg_tls to i64), i64 40) to <4 x i64>*), align 8
-; CHECK: store {{.*}} 72, {{.*}} @__msan_va_arg_overflow_size_tls
-
 ; Check i64 array.
 define i32 @bar4() {
   %1 = call i32 (i32, ...) @foo(i32 0, [2 x i64] [i64 1, i64 2])
diff --git a/llvm/test/MC/Disassembler/PowerPC/qpx.txt b/llvm/test/MC/Disassembler/PowerPC/qpx.txt
deleted file mode 100644
index 00e598bd4356e..0000000000000
--- a/llvm/test/MC/Disassembler/PowerPC/qpx.txt
+++ /dev/null
@@ -1,371 +0,0 @@
-# RUN: llvm-mc --disassemble %s -triple powerpc64-bgq-linux -mcpu=a2q | FileCheck %s
-
-# CHECK: qvfabs 3, 5
-0x10 0x60 0x2a 0x10
-
-# CHECK: qvfadd 3, 4, 5
-0x10 0x64 0x28 0x2a
-
-# CHECK: qvfadds 3, 4, 5
-0x00 0x64 0x28 0x2a
-
-# CHECK: qvfandc 3, 4, 5
-0x10 0x64 0x2a 0x08
-
-# CHECK: qvfand 3, 4, 5
-0x10 0x64 0x28 0x88
-
-# CHECK: qvfcfid 3, 5
-0x10 0x60 0x2e 0x9c
-
-# CHECK: qvfcfids 3, 5
-0x00 0x60 0x2e 0x9c
-
-# CHECK: qvfcfidu 3, 5
-0x10 0x60 0x2f 0x9c
-
-# CHECK: qvfcfidus 3, 5
-0x00 0x60 0x2f 0x9c
-
-# CHECK: qvfclr 3
-0x10 0x63 0x18 0x08
-
-# CHECK: qvfcpsgn 3, 4, 5
-0x10 0x64 0x28 0x10
-
-# CHECK: qvfctfb 3, 4
-0x10 0x64 0x22 0x88
-
-# CHECK: qvfctid 3, 5
-0x10 0x60 0x2e 0x5c
-
-# CHECK: qvfctidu 3, 5
-0x10 0x60 0x2f 0x5c
-
-# CHECK: qvfctiduz 3, 5
-0x10 0x60 0x2f 0x5e
-
-# CHECK: qvfctidz 3, 5
-0x10 0x60 0x2e 0x5e
-
-# CHECK: qvfctiw 3, 5
-0x10 0x60 0x28 0x1c
-
-# CHECK: qvfctiwu 3, 5
-0x10 0x60 0x29 0x1c
-
-# CHECK: qvfctiwuz 3, 5
-0x10 0x60 0x29 0x1e
-
-# CHECK: qvfctiwz 3, 5
-0x10 0x60 0x28 0x1e
-
-# CHECK: qvfequ 3, 4, 5
-0x10 0x64 0x2c 0x88
-
-# CHECK: qvflogical 3, 4, 5, 12
-0x10 0x64 0x2e 0x08
-
-# CHECK: qvfmadd 3, 4, 6, 5
-0x10 0x64 0x29 0xba
-
-# CHECK: qvfmadds 3, 4, 6, 5
-0x00 0x64 0x29 0xba
-
-# CHECK: qvfmr 3, 5
-0x10 0x60 0x28 0x90
-
-# CHECK: qvfmsub 3, 4, 6, 5
-0x10 0x64 0x29 0xb8
-
-# CHECK: qvfmsubs 3, 4, 6, 5
-0x00 0x64 0x29 0xb8
-
-# CHECK: qvfmul 3, 4, 6
-0x10 0x64 0x01 0xb2
-
-# CHECK: qvfmuls 3, 4, 6
-0x00 0x64 0x01 0xb2
-
-# CHECK: qvfnabs 3, 5
-0x10 0x60 0x29 0x10
-
-# CHECK: qvfnand 3, 4, 5
-0x10 0x64 0x2f 0x08
-
-# CHECK: qvfneg 3, 5
-0x10 0x60 0x28 0x50
-
-# CHECK: qvfnmadd 3, 4, 6, 5
-0x10 0x64 0x29 0xbe
-
-# CHECK: qvfnmadds 3, 4, 6, 5
-0x00 0x64 0x29 0xbe
-
-# CHECK: qvfnmsub 3, 4, 6, 5
-0x10 0x64 0x29 0xbc
-
-# CHECK: qvfnmsubs 3, 4, 6, 5
-0x00 0x64 0x29 0xbc
-
-# CHECK: qvfnor 3, 4, 5
-0x10 0x64 0x2c 0x08
-
-# CHECK: qvfnot 3, 4
-0x10 0x64 0x25 0x08
-
-# CHECK: qvforc 3, 4, 5
-0x10 0x64 0x2e 0x88
-
-# CHECK: qvfor 3, 4, 5
-0x10 0x64 0x2b 0x88
-
-# CHECK: qvfperm 3, 4, 5, 6
-0x10 0x64 0x29 0x8c
-
-# CHECK: qvfre 3, 5
-0x10 0x60 0x28 0x30
-
-# CHECK: qvfres 3, 5
-0x00 0x60 0x28 0x30
-
-# CHECK: qvfrim 3, 5
-0x10 0x60 0x2b 0xd0
-
-# CHECK: qvfrin 3, 5
-0x10 0x60 0x2b 0x10
-
-# CHECK: qvfrip 3, 5
-0x10 0x60 0x2b 0x90
-
-# CHECK: qvfriz 3, 5
-0x10 0x60 0x2b 0x50
-
-# CHECK: qvfrsp 3, 5
-0x10 0x60 0x28 0x18
-
-# CHECK: qvfrsqrte 3, 5
-0x10 0x60 0x28 0x34
-
-# CHECK: qvfrsqrtes 3, 5
-0x00 0x60 0x28 0x34
-
-# CHECK: qvfsel 3, 4, 6, 5
-0x10 0x64 0x29 0xae
-
-# CHECK: qvfset 3
-0x10 0x63 0x1f 0x88
-
-# CHECK: qvfsub 3, 4, 5
-0x10 0x64 0x28 0x28
-
-# CHECK: qvfsubs 3, 4, 5
-0x00 0x64 0x28 0x28
-
-# CHECK: qvfxmadd 3, 4, 6, 5
-0x10 0x64 0x29 0x92
-
-# CHECK: qvfxmadds 3, 4, 6, 5
-0x00 0x64 0x29 0x92
-
-# CHECK: qvfxmul 3, 4, 6
-0x10 0x64 0x01 0xa2
-
-# CHECK: qvfxmuls 3, 4, 6
-0x00 0x64 0x01 0xa2
-
-# CHECK: qvfxor 3, 4, 5
-0x10 0x64 0x2b 0x08
-
-# CHECK: qvfxxcpnmadd 3, 4, 6, 5
-0x10 0x64 0x29 0x86
-
-# CHECK: qvfxxcpnmadds 3, 4, 6, 5
-0x00 0x64 0x29 0x86
-
-# CHECK: qvfxxmadd 3, 4, 6, 5
-0x10 0x64 0x29 0x82
-
-# CHECK: qvfxxmadds 3, 4, 6, 5
-0x00 0x64 0x29 0x82
-
-# CHECK: qvfxxnpmadd 3, 4, 6, 5
-0x10 0x64 0x29 0x96
-
-# CHECK: qvfxxnpmadds 3, 4, 6, 5
-0x00 0x64 0x29 0x96
-
-# CHECK: qvlfcduxa 3, 9, 11
-0x7c 0x69 0x58 0xcf
-
-# CHECK: qvlfcdux 3, 9, 11
-0x7c 0x69 0x58 0xce
-
-# CHECK: qvlfcdxa 3, 10, 11
-0x7c 0x6a 0x58 0x8f
-
-# CHECK: qvlfcdx 3, 10, 11
-0x7c 0x6a 0x58 0x8e
-
-# CHECK: qvlfcsuxa 3, 9, 11
-0x7c 0x69 0x58 0x4f
-
-# CHECK: qvlfcsux 3, 9, 11
-0x7c 0x69 0x58 0x4e
-
-# CHECK: qvlfcsxa 3, 10, 11
-0x7c 0x6a 0x58 0x0f
-
-# CHECK: qvlfcsx 3, 10, 11
-0x7c 0x6a 0x58 0x0e
-
-# CHECK: qvlfduxa 3, 9, 11
-0x7c 0x69 0x5c 0xcf
-
-# CHECK: qvlfdux 3, 9, 11
-0x7c 0x69 0x5c 0xce
-
-# CHECK: qvlfdxa 3, 10, 11
-0x7c 0x6a 0x5c 0x8f
-
-# CHECK: qvlfdx 3, 10, 11
-0x7c 0x6a 0x5c 0x8e
-
-# CHECK: qvlfiwaxa 3, 10, 11
-0x7c 0x6a 0x5e 0xcf
-
-# CHECK: qvlfiwax 3, 10, 11
-0x7c 0x6a 0x5e 0xce
-
-# CHECK: qvlfiwzxa 3, 10, 11
-0x7c 0x6a 0x5e 0x8f
-
-# CHECK: qvlfiwzx 3, 10, 11
-0x7c 0x6a 0x5e 0x8e
-
-# CHECK: qvlfsuxa 3, 9, 11
-0x7c 0x69 0x5c 0x4f
-
-# CHECK: qvlfsux 3, 9, 11
-0x7c 0x69 0x5c 0x4e
-
-# CHECK: qvlfsxa 3, 10, 11
-0x7c 0x6a 0x5c 0x0f
-
-# CHECK: qvlfsx 3, 10, 11
-0x7c 0x6a 0x5c 0x0e
-
-# CHECK: qvlpcldx 3, 10, 11
-0x7c 0x6a 0x5c 0x8c
-
-# CHECK: qvlpclsx 3, 10, 11
-0x7c 0x6a 0x5c 0x0c
-
-# CHECK: qvlpcrdx 3, 10, 11
-0x7c 0x6a 0x58 0x8c
-
-# CHECK: qvlpcrsx 3, 10, 11
-0x7c 0x6a 0x58 0x0c
-
-# CHECK: qvstfcduxa 2, 9, 11
-0x7c 0x49 0x59 0xcf
-
-# CHECK: qvstfcduxia 2, 9, 11
-0x7c 0x49 0x59 0xcb
-
-# CHECK: qvstfcduxi 2, 9, 11
-0x7c 0x49 0x59 0xca
-
-# CHECK: qvstfcdux 2, 9, 11
-0x7c 0x49 0x59 0xce
-
-# CHECK: qvstfcdxa 2, 10, 11
-0x7c 0x4a 0x59 0x8f
-
-# CHECK: qvstfcdxia 2, 10, 11
-0x7c 0x4a 0x59 0x8b
-
-# CHECK: qvstfcdxi 2, 10, 11
-0x7c 0x4a 0x59 0x8a
-
-# CHECK: qvstfcdx 2, 10, 11
-0x7c 0x4a 0x59 0x8e
-
-# CHECK: qvstfcsuxa 2, 9, 11
-0x7c 0x49 0x59 0x4f
-
-# CHECK: qvstfcsuxia 2, 9, 11
-0x7c 0x49 0x59 0x4b
-
-# CHECK: qvstfcsuxi 2, 9, 11
-0x7c 0x49 0x59 0x4a
-
-# CHECK: qvstfcsux 2, 9, 11
-0x7c 0x49 0x59 0x4e
-
-# CHECK: qvstfcsxa 2, 10, 11
-0x7c 0x4a 0x59 0x0f
-
-# CHECK: qvstfcsxia 2, 10, 11
-0x7c 0x4a 0x59 0x0b
-
-# CHECK: qvstfcsxi 2, 10, 11
-0x7c 0x4a 0x59 0x0a
-
-# CHECK: qvstfcsx 2, 10, 11
-0x7c 0x4a 0x59 0x0e
-
-# CHECK: qvstfduxa 2, 9, 11
-0x7c 0x49 0x5d 0xcf
-
-# CHECK: qvstfduxia 2, 9, 11
-0x7c 0x49 0x5d 0xcb
-
-# CHECK: qvstfduxi 2, 9, 11
-0x7c 0x49 0x5d 0xca
-
-# CHECK: qvstfdux 2, 9, 11
-0x7c 0x49 0x5d 0xce
-
-# CHECK: qvstfdxa 2, 10, 11
-0x7c 0x4a 0x5d 0x8f
-
-# CHECK: qvstfdxia 2, 10, 11
-0x7c 0x4a 0x5d 0x8b
-
-# CHECK: qvstfdxi 2, 10, 11
-0x7c 0x4a 0x5d 0x8a
-
-# CHECK: qvstfdx 2, 10, 11
-0x7c 0x4a 0x5d 0x8e
-
-# CHECK: qvstfiwxa 2, 10, 11
-0x7c 0x4a 0x5f 0x8f
-
-# CHECK: qvstfiwx 2, 10, 11
-0x7c 0x4a 0x5f 0x8e
-
-# CHECK: qvstfsuxa 2, 9, 11
-0x7c 0x49 0x5d 0x4f
-
-# CHECK: qvstfsuxia 2, 9, 11
-0x7c 0x49 0x5d 0x4b
-
-# CHECK: qvstfsuxi 2, 9, 11
-0x7c 0x49 0x5d 0x4a
-
-# CHECK: qvstfsux 2, 9, 11
-0x7c 0x49 0x5d 0x4e
-
-# CHECK: qvstfsxa 2, 10, 11
-0x7c 0x4a 0x5d 0x0f
-
-# CHECK: qvstfsxia 2, 10, 11
-0x7c 0x4a 0x5d 0x0b
-
-# CHECK: qvstfsxi 2, 10, 11
-0x7c 0x4a 0x5d 0x0a
-
-# CHECK: qvstfsx 2, 10, 11
-0x7c 0x4a 0x5d 0x0e
-
diff --git a/llvm/test/MC/PowerPC/qpx.s b/llvm/test/MC/PowerPC/qpx.s
deleted file mode 100644
index a1fb2090f8fff..0000000000000
--- a/llvm/test/MC/PowerPC/qpx.s
+++ /dev/null
@@ -1,252 +0,0 @@
-# RUN: llvm-mc -triple powerpc64-bgq-linux --show-encoding %s | FileCheck %s
-
-# CHECK: qvfabs 3, 5                     # encoding: [0x10,0x60,0x2a,0x10]
-         qvfabs %q3, %q5
-
-# CHECK: qvfabs 3, 5                     # encoding: [0x10,0x60,0x2a,0x10]
-         qvfabs 3, 5
-# CHECK: qvfadd 3, 4, 5                  # encoding: [0x10,0x64,0x28,0x2a]
-         qvfadd 3, 4, 5
-# CHECK: qvfadds 3, 4, 5                 # encoding: [0x00,0x64,0x28,0x2a]
-         qvfadds 3, 4, 5
-# CHECK: qvfandc 3, 4, 5                 # encoding: [0x10,0x64,0x2a,0x08]
-         qvfandc 3, 4, 5
-# CHECK: qvfand 3, 4, 5                  # encoding: [0x10,0x64,0x28,0x88]
-         qvfand 3, 4, 5
-# CHECK: qvfcfid 3, 5                    # encoding: [0x10,0x60,0x2e,0x9c]
-         qvfcfid 3, 5
-# CHECK: qvfcfids 3, 5                   # encoding: [0x00,0x60,0x2e,0x9c]
-         qvfcfids 3, 5
-# CHECK: qvfcfidu 3, 5                   # encoding: [0x10,0x60,0x2f,0x9c]
-         qvfcfidu 3, 5
-# CHECK: qvfcfidus 3, 5                  # encoding: [0x00,0x60,0x2f,0x9c]
-         qvfcfidus 3, 5
-# CHECK: qvfclr 3                        # encoding: [0x10,0x63,0x18,0x08]
-         qvfclr 3
-# CHECK: qvfcpsgn 3, 4, 5                # encoding: [0x10,0x64,0x28,0x10]
-         qvfcpsgn 3, 4, 5
-# CHECK: qvfctfb 3, 4                    # encoding: [0x10,0x64,0x22,0x88]
-         qvfctfb 3, 4
-# CHECK: qvfctid 3, 5                    # encoding: [0x10,0x60,0x2e,0x5c]
-         qvfctid 3, 5
-# CHECK: qvfctidu 3, 5                   # encoding: [0x10,0x60,0x2f,0x5c]
-         qvfctidu 3, 5
-# CHECK: qvfctiduz 3, 5                  # encoding: [0x10,0x60,0x2f,0x5e]
-         qvfctiduz 3, 5
-# CHECK: qvfctidz 3, 5                   # encoding: [0x10,0x60,0x2e,0x5e]
-         qvfctidz 3, 5
-# CHECK: qvfctiw 3, 5                    # encoding: [0x10,0x60,0x28,0x1c]
-         qvfctiw 3, 5
-# CHECK: qvfctiwu 3, 5                   # encoding: [0x10,0x60,0x29,0x1c]
-         qvfctiwu 3, 5
-# CHECK: qvfctiwuz 3, 5                  # encoding: [0x10,0x60,0x29,0x1e]
-         qvfctiwuz 3, 5
-# CHECK: qvfctiwz 3, 5                   # encoding: [0x10,0x60,0x28,0x1e]
-         qvfctiwz 3, 5
-# CHECK: qvfequ 3, 4, 5                  # encoding: [0x10,0x64,0x2c,0x88]
-         qvfequ 3, 4, 5
-# CHECK: qvflogical 3, 4, 5, 12          # encoding: [0x10,0x64,0x2e,0x08]
-         qvflogical 3, 4, 5, 12
-# CHECK: qvfmadd 3, 4, 6, 5              # encoding: [0x10,0x64,0x29,0xba]
-         qvfmadd 3, 4, 6, 5
-# CHECK: qvfmadds 3, 4, 6, 5             # encoding: [0x00,0x64,0x29,0xba]
-         qvfmadds 3, 4, 6, 5
-# CHECK: qvfmr 3, 5                      # encoding: [0x10,0x60,0x28,0x90]
-         qvfmr 3, 5
-# CHECK: qvfmsub 3, 4, 6, 5              # encoding: [0x10,0x64,0x29,0xb8]
-         qvfmsub 3, 4, 6, 5
-# CHECK: qvfmsubs 3, 4, 6, 5             # encoding: [0x00,0x64,0x29,0xb8]
-         qvfmsubs 3, 4, 6, 5
-# CHECK: qvfmul 3, 4, 6                  # encoding: [0x10,0x64,0x01,0xb2]
-         qvfmul 3, 4, 6
-# CHECK: qvfmuls 3, 4, 6                 # encoding: [0x00,0x64,0x01,0xb2]
-         qvfmuls 3, 4, 6
-# CHECK: qvfnabs 3, 5                    # encoding: [0x10,0x60,0x29,0x10]
-         qvfnabs 3, 5
-# CHECK: qvfnand 3, 4, 5                 # encoding: [0x10,0x64,0x2f,0x08]
-         qvfnand 3, 4, 5
-# CHECK: qvfneg 3, 5                     # encoding: [0x10,0x60,0x28,0x50]
-         qvfneg 3, 5
-# CHECK: qvfnmadd 3, 4, 6, 5             # encoding: [0x10,0x64,0x29,0xbe]
-         qvfnmadd 3, 4, 6, 5
-# CHECK: qvfnmadds 3, 4, 6, 5            # encoding: [0x00,0x64,0x29,0xbe]
-         qvfnmadds 3, 4, 6, 5
-# CHECK: qvfnmsub 3, 4, 6, 5             # encoding: [0x10,0x64,0x29,0xbc]
-         qvfnmsub 3, 4, 6, 5
-# CHECK: qvfnmsubs 3, 4, 6, 5            # encoding: [0x00,0x64,0x29,0xbc]
-         qvfnmsubs 3, 4, 6, 5
-# CHECK: qvfnor 3, 4, 5                  # encoding: [0x10,0x64,0x2c,0x08]
-         qvfnor 3, 4, 5
-# CHECK: qvfnot 3, 4                     # encoding: [0x10,0x64,0x25,0x08]
-         qvfnot 3, 4
-# CHECK: qvforc 3, 4, 5                  # encoding: [0x10,0x64,0x2e,0x88]
-         qvforc 3, 4, 5
-# CHECK: qvfor 3, 4, 5                   # encoding: [0x10,0x64,0x2b,0x88]
-         qvfor 3, 4, 5
-# CHECK: qvfperm 3, 4, 5, 6              # encoding: [0x10,0x64,0x29,0x8c]
-         qvfperm 3, 4, 5, 6
-# CHECK: qvfre 3, 5                      # encoding: [0x10,0x60,0x28,0x30]
-         qvfre 3, 5
-# CHECK: qvfres 3, 5                     # encoding: [0x00,0x60,0x28,0x30]
-         qvfres 3, 5
-# CHECK: qvfrim 3, 5                     # encoding: [0x10,0x60,0x2b,0xd0]
-         qvfrim 3, 5
-# CHECK: qvfrin 3, 5                     # encoding: [0x10,0x60,0x2b,0x10]
-         qvfrin 3, 5
-# CHECK: qvfrip 3, 5                     # encoding: [0x10,0x60,0x2b,0x90]
-         qvfrip 3, 5
-# CHECK: qvfriz 3, 5                     # encoding: [0x10,0x60,0x2b,0x50]
-         qvfriz 3, 5
-# CHECK: qvfrsp 3, 5                     # encoding: [0x10,0x60,0x28,0x18]
-         qvfrsp 3, 5
-# CHECK: qvfrsqrte 3, 5                  # encoding: [0x10,0x60,0x28,0x34]
-         qvfrsqrte 3, 5
-# CHECK: qvfrsqrtes 3, 5                 # encoding: [0x00,0x60,0x28,0x34]
-         qvfrsqrtes 3, 5
-# CHECK: qvfsel 3, 4, 6, 5               # encoding: [0x10,0x64,0x29,0xae]
-         qvfsel 3, 4, 6, 5
-# CHECK: qvfset 3                        # encoding: [0x10,0x63,0x1f,0x88]
-         qvfset 3
-# CHECK: qvfsub 3, 4, 5                  # encoding: [0x10,0x64,0x28,0x28]
-         qvfsub 3, 4, 5
-# CHECK: qvfsubs 3, 4, 5                 # encoding: [0x00,0x64,0x28,0x28]
-         qvfsubs 3, 4, 5
-# CHECK: qvfxmadd 3, 4, 6, 5             # encoding: [0x10,0x64,0x29,0x92]
-         qvfxmadd 3, 4, 6, 5
-# CHECK: qvfxmadds 3, 4, 6, 5            # encoding: [0x00,0x64,0x29,0x92]
-         qvfxmadds 3, 4, 6, 5
-# CHECK: qvfxmul 3, 4, 6                 # encoding: [0x10,0x64,0x01,0xa2]
-         qvfxmul 3, 4, 6
-# CHECK: qvfxmuls 3, 4, 6                # encoding: [0x00,0x64,0x01,0xa2]
-         qvfxmuls 3, 4, 6
-# CHECK: qvfxor 3, 4, 5                  # encoding: [0x10,0x64,0x2b,0x08]
-         qvfxor 3, 4, 5
-# CHECK: qvfxxcpnmadd 3, 4, 6, 5         # encoding: [0x10,0x64,0x29,0x86]
-         qvfxxcpnmadd 3, 4, 6, 5
-# CHECK: qvfxxcpnmadds 3, 4, 6, 5        # encoding: [0x00,0x64,0x29,0x86]
-         qvfxxcpnmadds 3, 4, 6, 5
-# CHECK: qvfxxmadd 3, 4, 6, 5            # encoding: [0x10,0x64,0x29,0x82]
-         qvfxxmadd 3, 4, 6, 5
-# CHECK: qvfxxmadds 3, 4, 6, 5           # encoding: [0x00,0x64,0x29,0x82]
-         qvfxxmadds 3, 4, 6, 5
-# CHECK: qvfxxnpmadd 3, 4, 6, 5          # encoding: [0x10,0x64,0x29,0x96]
-         qvfxxnpmadd 3, 4, 6, 5
-# CHECK: qvfxxnpmadds 3, 4, 6, 5         # encoding: [0x00,0x64,0x29,0x96]
-         qvfxxnpmadds 3, 4, 6, 5
-# CHECK: qvlfcduxa 3, 9, 11              # encoding: [0x7c,0x69,0x58,0xcf]
-         qvlfcduxa 3, 9, 11
-# CHECK: qvlfcdux 3, 9, 11               # encoding: [0x7c,0x69,0x58,0xce]
-         qvlfcdux 3, 9, 11
-# CHECK: qvlfcdxa 3, 10, 11              # encoding: [0x7c,0x6a,0x58,0x8f]
-         qvlfcdxa 3, 10, 11
-# CHECK: qvlfcdx 3, 10, 11               # encoding: [0x7c,0x6a,0x58,0x8e]
-         qvlfcdx 3, 10, 11
-# CHECK: qvlfcsuxa 3, 9, 11              # encoding: [0x7c,0x69,0x58,0x4f]
-         qvlfcsuxa 3, 9, 11
-# CHECK: qvlfcsux 3, 9, 11               # encoding: [0x7c,0x69,0x58,0x4e]
-         qvlfcsux 3, 9, 11
-# CHECK: qvlfcsxa 3, 10, 11              # encoding: [0x7c,0x6a,0x58,0x0f]
-         qvlfcsxa 3, 10, 11
-# CHECK: qvlfcsx 3, 10, 11               # encoding: [0x7c,0x6a,0x58,0x0e]
-         qvlfcsx 3, 10, 11
-# CHECK: qvlfduxa 3, 9, 11               # encoding: [0x7c,0x69,0x5c,0xcf]
-         qvlfduxa 3, 9, 11
-# CHECK: qvlfdux 3, 9, 11                # encoding: [0x7c,0x69,0x5c,0xce]
-         qvlfdux 3, 9, 11
-# CHECK: qvlfdxa 3, 10, 11               # encoding: [0x7c,0x6a,0x5c,0x8f]
-         qvlfdxa 3, 10, 11
-# CHECK: qvlfdx 3, 10, 11                # encoding: [0x7c,0x6a,0x5c,0x8e]
-         qvlfdx 3, 10, 11
-# CHECK: qvlfiwaxa 3, 10, 11             # encoding: [0x7c,0x6a,0x5e,0xcf]
-         qvlfiwaxa 3, 10, 11
-# CHECK: qvlfiwax 3, 10, 11              # encoding: [0x7c,0x6a,0x5e,0xce]
-         qvlfiwax 3, 10, 11
-# CHECK: qvlfiwzxa 3, 10, 11             # encoding: [0x7c,0x6a,0x5e,0x8f]
-         qvlfiwzxa 3, 10, 11
-# CHECK: qvlfiwzx 3, 10, 11              # encoding: [0x7c,0x6a,0x5e,0x8e]
-         qvlfiwzx 3, 10, 11
-# CHECK: qvlfsuxa 3, 9, 11               # encoding: [0x7c,0x69,0x5c,0x4f]
-         qvlfsuxa 3, 9, 11
-# CHECK: qvlfsux 3, 9, 11                # encoding: [0x7c,0x69,0x5c,0x4e]
-         qvlfsux 3, 9, 11
-# CHECK: qvlfsxa 3, 10, 11               # encoding: [0x7c,0x6a,0x5c,0x0f]
-         qvlfsxa 3, 10, 11
-# CHECK: qvlfsx 3, 10, 11                # encoding: [0x7c,0x6a,0x5c,0x0e]
-         qvlfsx 3, 10, 11
-# CHECK: qvlpcldx 3, 10, 11              # encoding: [0x7c,0x6a,0x5c,0x8c]
-         qvlpcldx 3, 10, 11
-# CHECK: qvlpclsx 3, 10, 11              # encoding: [0x7c,0x6a,0x5c,0x0c]
-         qvlpclsx 3, 10, 11
-# CHECK: qvlpcrdx 3, 10, 11              # encoding: [0x7c,0x6a,0x58,0x8c]
-         qvlpcrdx 3, 10, 11
-# CHECK: qvlpcrsx 3, 10, 11              # encoding: [0x7c,0x6a,0x58,0x0c]
-         qvlpcrsx 3, 10, 11
-# CHECK: qvstfcduxa 2, 9, 11             # encoding: [0x7c,0x49,0x59,0xcf]
-         qvstfcduxa 2, 9, 11
-# CHECK: qvstfcduxia 2, 9, 11            # encoding: [0x7c,0x49,0x59,0xcb]
-         qvstfcduxia 2, 9, 11
-# CHECK: qvstfcduxi 2, 9, 11             # encoding: [0x7c,0x49,0x59,0xca]
-         qvstfcduxi 2, 9, 11
-# CHECK: qvstfcdux 2, 9, 11              # encoding: [0x7c,0x49,0x59,0xce]
-         qvstfcdux 2, 9, 11
-# CHECK: qvstfcdxa 2, 10, 11             # encoding: [0x7c,0x4a,0x59,0x8f]
-         qvstfcdxa 2, 10, 11
-# CHECK: qvstfcdxia 2, 10, 11            # encoding: [0x7c,0x4a,0x59,0x8b]
-         qvstfcdxia 2, 10, 11
-# CHECK: qvstfcdxi 2, 10, 11             # encoding: [0x7c,0x4a,0x59,0x8a]
-         qvstfcdxi 2, 10, 11
-# CHECK: qvstfcdx 2, 10, 11              # encoding: [0x7c,0x4a,0x59,0x8e]
-         qvstfcdx 2, 10, 11
-# CHECK: qvstfcsuxa 2, 9, 11             # encoding: [0x7c,0x49,0x59,0x4f]
-         qvstfcsuxa 2, 9, 11
-# CHECK: qvstfcsuxia 2, 9, 11            # encoding: [0x7c,0x49,0x59,0x4b]
-         qvstfcsuxia 2, 9, 11
-# CHECK: qvstfcsuxi 2, 9, 11             # encoding: [0x7c,0x49,0x59,0x4a]
-         qvstfcsuxi 2, 9, 11
-# CHECK: qvstfcsux 2, 9, 11              # encoding: [0x7c,0x49,0x59,0x4e]
-         qvstfcsux 2, 9, 11
-# CHECK: qvstfcsxa 2, 10, 11             # encoding: [0x7c,0x4a,0x59,0x0f]
-         qvstfcsxa 2, 10, 11
-# CHECK: qvstfcsxia 2, 10, 11            # encoding: [0x7c,0x4a,0x59,0x0b]
-         qvstfcsxia 2, 10, 11
-# CHECK: qvstfcsxi 2, 10, 11             # encoding: [0x7c,0x4a,0x59,0x0a]
-         qvstfcsxi 2, 10, 11
-# CHECK: qvstfcsx 2, 10, 11              # encoding: [0x7c,0x4a,0x59,0x0e]
-         qvstfcsx 2, 10, 11
-# CHECK: qvstfduxa 2, 9, 11              # encoding: [0x7c,0x49,0x5d,0xcf]
-         qvstfduxa 2, 9, 11
-# CHECK: qvstfduxia 2, 9, 11             # encoding: [0x7c,0x49,0x5d,0xcb]
-         qvstfduxia 2, 9, 11
-# CHECK: qvstfduxi 2, 9, 11              # encoding: [0x7c,0x49,0x5d,0xca]
-         qvstfduxi 2, 9, 11
-# CHECK: qvstfdux 2, 9, 11               # encoding: [0x7c,0x49,0x5d,0xce]
-         qvstfdux 2, 9, 11
-# CHECK: qvstfdxa 2, 10, 11              # encoding: [0x7c,0x4a,0x5d,0x8f]
-         qvstfdxa 2, 10, 11
-# CHECK: qvstfdxia 2, 10, 11             # encoding: [0x7c,0x4a,0x5d,0x8b]
-         qvstfdxia 2, 10, 11
-# CHECK: qvstfdxi 2, 10, 11              # encoding: [0x7c,0x4a,0x5d,0x8a]
-         qvstfdxi 2, 10, 11
-# CHECK: qvstfdx 2, 10, 11               # encoding: [0x7c,0x4a,0x5d,0x8e]
-         qvstfdx 2, 10, 11
-# CHECK: qvstfiwxa 2, 10, 11             # encoding: [0x7c,0x4a,0x5f,0x8f]
-         qvstfiwxa 2, 10, 11
-# CHECK: qvstfiwx 2, 10, 11              # encoding: [0x7c,0x4a,0x5f,0x8e]
-         qvstfiwx 2, 10, 11
-# CHECK: qvstfsuxa 2, 9, 11              # encoding: [0x7c,0x49,0x5d,0x4f]
-         qvstfsuxa 2, 9, 11
-# CHECK: qvstfsuxia 2, 9, 11             # encoding: [0x7c,0x49,0x5d,0x4b]
-         qvstfsuxia 2, 9, 11
-# CHECK: qvstfsuxi 2, 9, 11              # encoding: [0x7c,0x49,0x5d,0x4a]
-         qvstfsuxi 2, 9, 11
-# CHECK: qvstfsux 2, 9, 11               # encoding: [0x7c,0x49,0x5d,0x4e]
-         qvstfsux 2, 9, 11
-# CHECK: qvstfsxa 2, 10, 11              # encoding: [0x7c,0x4a,0x5d,0x0f]
-         qvstfsxa 2, 10, 11
-# CHECK: qvstfsxia 2, 10, 11             # encoding: [0x7c,0x4a,0x5d,0x0b]
-         qvstfsxia 2, 10, 11
-# CHECK: qvstfsxi 2, 10, 11              # encoding: [0x7c,0x4a,0x5d,0x0a]
-         qvstfsxi 2, 10, 11
-# CHECK: qvstfsx 2, 10, 11               # encoding: [0x7c,0x4a,0x5d,0x0e]
-         qvstfsx 2, 10, 11
-
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/fp-bc-icmp-const-fold.ll b/llvm/test/Transforms/Attributor/IPConstantProp/fp-bc-icmp-const-fold.ll
index c45c48d502343..67c22f9470779 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/fp-bc-icmp-const-fold.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/fp-bc-icmp-const-fold.ll
@@ -4,7 +4,7 @@
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 target datalayout = "E-m:e-i64:64-n32:64"
-target triple = "powerpc64-bgq-linux"
+target triple = "powerpc64le-unknown-linux"
 
 define void @test(i32 signext %n) {
 ; IS__TUNIT____: Function Attrs: nofree noreturn nosync nounwind readnone
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/combined-partial-overwrites.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/combined-partial-overwrites.ll
index 6cd77a59df6b1..faf7041bfc387 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/combined-partial-overwrites.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/combined-partial-overwrites.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S -dse -enable-dse-memoryssa -enable-dse-partial-store-merging=false < %s | FileCheck %s
 target datalayout = "E-m:e-i64:64-n32:64"
-target triple = "powerpc64-bgq-linux"
+target triple = "powerpc64le-unknown-linux"
 
 %"struct.std::complex" = type { { float, float } }
 
diff --git a/llvm/test/Transforms/DeadStoreElimination/combined-partial-overwrites.ll b/llvm/test/Transforms/DeadStoreElimination/combined-partial-overwrites.ll
index 2a61fff15ade0..a57693a1da38e 100644
--- a/llvm/test/Transforms/DeadStoreElimination/combined-partial-overwrites.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/combined-partial-overwrites.ll
@@ -1,6 +1,6 @@
 ; RUN: opt -S -dse -enable-dse-partial-store-merging=false < %s | FileCheck %s
 target datalayout = "E-m:e-i64:64-n32:64"
-target triple = "powerpc64-bgq-linux"
+target triple = "powerpc64le-unknown-linux"
 
 %"struct.std::complex" = type { { float, float } }
 
diff --git a/llvm/test/Transforms/EntryExitInstrumenter/mcount.ll b/llvm/test/Transforms/EntryExitInstrumenter/mcount.ll
index 4c5f18a26657c..5a6daa2c9a008 100644
--- a/llvm/test/Transforms/EntryExitInstrumenter/mcount.ll
+++ b/llvm/test/Transforms/EntryExitInstrumenter/mcount.ll
@@ -4,7 +4,7 @@
 ; RUN: opt -passes="function(ee-instrument),function(ee-instrument),cgscc(inline),function(post-inline-ee-instrument),function(post-inline-ee-instrument)" -S < %s | FileCheck %s
 
 target datalayout = "E-m:e-i64:64-n32:64"
-target triple = "powerpc64-bgq-linux"
+target triple = "powerpc64le-unknown-linux"
 
 define void @leaf_function() #0 {
 entry:
diff --git a/llvm/test/Transforms/InstCombine/PowerPC/aligned-qpx.ll b/llvm/test/Transforms/InstCombine/PowerPC/aligned-qpx.ll
deleted file mode 100644
index e9710df5670cd..0000000000000
--- a/llvm/test/Transforms/InstCombine/PowerPC/aligned-qpx.ll
+++ /dev/null
@@ -1,165 +0,0 @@
-; RUN: opt -S -instcombine < %s | FileCheck %s
-target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
-target triple = "powerpc64-unknown-linux-gnu"
-
-declare <4 x double> @llvm.ppc.qpx.qvlfs(i8*) #1
-
-define <4 x double> @test1(<4 x float>* %h) #0 {
-entry:
-  %h1 = getelementptr <4 x float>, <4 x float>* %h, i64 1
-  %hv = bitcast <4 x float>* %h1 to i8*
-  %vl = call <4 x double> @llvm.ppc.qpx.qvlfs(i8* %hv)
-
-; CHECK-LABEL: @test1
-; CHECK: @llvm.ppc.qpx.qvlfs
-; CHECK: ret <4 x double>
-
-  %v0 = load <4 x float>, <4 x float>* %h, align 8
-  %v0e = fpext <4 x float> %v0 to <4 x double>
-  %a = fadd <4 x double> %v0e, %vl
-  ret <4 x double> %a
-}
-
-define <4 x double> @test1a(<4 x float>* align 16 %h) #0 {
-entry:
-  %h1 = getelementptr <4 x float>, <4 x float>* %h, i64 1
-  %hv = bitcast <4 x float>* %h1 to i8*
-  %vl = call <4 x double> @llvm.ppc.qpx.qvlfs(i8* %hv)
-
-; CHECK-LABEL: @test1a
-; CHECK-NOT: @llvm.ppc.qpx.qvlfs
-; CHECK-NOT: load <4 x double>
-; CHECK: ret <4 x double>
-
-  %v0 = load <4 x float>, <4 x float>* %h, align 8
-  %v0e = fpext <4 x float> %v0 to <4 x double>
-  %a = fadd <4 x double> %v0e, %vl
-  ret <4 x double> %a
-}
-
-declare void @llvm.ppc.qpx.qvstfs(<4 x double>, i8*) #0
-
-define <4 x float> @test2(<4 x float>* %h, <4 x double> %d) #0 {
-entry:
-  %h1 = getelementptr <4 x float>, <4 x float>* %h, i64 1
-  %hv = bitcast <4 x float>* %h1 to i8*
-  call void @llvm.ppc.qpx.qvstfs(<4 x double> %d, i8* %hv)
-
-  %v0 = load <4 x float>, <4 x float>* %h, align 8
-  ret <4 x float> %v0
-
-; CHECK-LABEL: @test2
-; CHECK: @llvm.ppc.qpx.qvstfs
-; CHECK: ret <4 x float>
-}
-
-define <4 x float> @test2a(<4 x float>* align 16 %h, <4 x double> %d) #0 {
-entry:
-  %h1 = getelementptr <4 x float>, <4 x float>* %h, i64 1
-  %hv = bitcast <4 x float>* %h1 to i8*
-  call void @llvm.ppc.qpx.qvstfs(<4 x double> %d, i8* %hv)
-
-  %v0 = load <4 x float>, <4 x float>* %h, align 8
-  ret <4 x float> %v0
-
-; CHECK-LABEL: @test2
-; CHECK: fptrunc <4 x double> %d to <4 x float>
-; CHECK-NOT: @llvm.ppc.qpx.qvstfs
-; CHECK-NOT: store <4 x double>
-; CHECK: ret <4 x float>
-}
-
-declare <4 x double> @llvm.ppc.qpx.qvlfd(i8*) #1
-
-define <4 x double> @test1l(<4 x double>* %h) #0 {
-entry:
-  %h1 = getelementptr <4 x double>, <4 x double>* %h, i64 1
-  %hv = bitcast <4 x double>* %h1 to i8*
-  %vl = call <4 x double> @llvm.ppc.qpx.qvlfd(i8* %hv)
-
-; CHECK-LABEL: @test1l
-; CHECK: @llvm.ppc.qpx.qvlfd
-; CHECK: ret <4 x double>
-
-  %v0 = load <4 x double>, <4 x double>* %h, align 8
-  %a = fadd <4 x double> %v0, %vl
-  ret <4 x double> %a
-}
-
-define <4 x double> @test1ln(<4 x double>* align 16 %h) #0 {
-entry:
-  %h1 = getelementptr <4 x double>, <4 x double>* %h, i64 1
-  %hv = bitcast <4 x double>* %h1 to i8*
-  %vl = call <4 x double> @llvm.ppc.qpx.qvlfd(i8* %hv)
-
-; CHECK-LABEL: @test1ln
-; CHECK: @llvm.ppc.qpx.qvlfd
-; CHECK: ret <4 x double>
-
-  %v0 = load <4 x double>, <4 x double>* %h, align 8
-  %a = fadd <4 x double> %v0, %vl
-  ret <4 x double> %a
-}
-
-define <4 x double> @test1la(<4 x double>* align 32 %h) #0 {
-entry:
-  %h1 = getelementptr <4 x double>, <4 x double>* %h, i64 1
-  %hv = bitcast <4 x double>* %h1 to i8*
-  %vl = call <4 x double> @llvm.ppc.qpx.qvlfd(i8* %hv)
-
-; CHECK-LABEL: @test1la
-; CHECK-NOT: @llvm.ppc.qpx.qvlfd
-; CHECK: ret <4 x double>
-
-  %v0 = load <4 x double>, <4 x double>* %h, align 8
-  %a = fadd <4 x double> %v0, %vl
-  ret <4 x double> %a
-}
-
-declare void @llvm.ppc.qpx.qvstfd(<4 x double>, i8*) #0
-
-define <4 x double> @test2l(<4 x double>* %h, <4 x double> %d) #0 {
-entry:
-  %h1 = getelementptr <4 x double>, <4 x double>* %h, i64 1
-  %hv = bitcast <4 x double>* %h1 to i8*
-  call void @llvm.ppc.qpx.qvstfd(<4 x double> %d, i8* %hv)
-
-  %v0 = load <4 x double>, <4 x double>* %h, align 8
-  ret <4 x double> %v0
-
-; CHECK-LABEL: @test2l
-; CHECK: @llvm.ppc.qpx.qvstfd
-; CHECK: ret <4 x double>
-}
-
-define <4 x double> @test2ln(<4 x double>* align 16 %h, <4 x double> %d) #0 {
-entry:
-  %h1 = getelementptr <4 x double>, <4 x double>* %h, i64 1
-  %hv = bitcast <4 x double>* %h1 to i8*
-  call void @llvm.ppc.qpx.qvstfd(<4 x double> %d, i8* %hv)
-
-  %v0 = load <4 x double>, <4 x double>* %h, align 8
-  ret <4 x double> %v0
-
-; CHECK-LABEL: @test2ln
-; CHECK: @llvm.ppc.qpx.qvstfd
-; CHECK: ret <4 x double>
-}
-
-define <4 x double> @test2la(<4 x double>* align 32 %h, <4 x double> %d) #0 {
-entry:
-  %h1 = getelementptr <4 x double>, <4 x double>* %h, i64 1
-  %hv = bitcast <4 x double>* %h1 to i8*
-  call void @llvm.ppc.qpx.qvstfd(<4 x double> %d, i8* %hv)
-
-  %v0 = load <4 x double>, <4 x double>* %h, align 8
-  ret <4 x double> %v0
-
-; CHECK-LABEL: @test2l
-; CHECK-NOT: @llvm.ppc.qpx.qvstfd
-; CHECK: ret <4 x double>
-}
-
-attributes #0 = { nounwind }
-attributes #1 = { nounwind readonly }
-
diff --git a/llvm/test/Transforms/LoopDataPrefetch/PowerPC/basic.ll b/llvm/test/Transforms/LoopDataPrefetch/PowerPC/basic.ll
index ea46fd0d5a8f8..68c75af14f3e9 100644
--- a/llvm/test/Transforms/LoopDataPrefetch/PowerPC/basic.ll
+++ b/llvm/test/Transforms/LoopDataPrefetch/PowerPC/basic.ll
@@ -1,7 +1,6 @@
-; RUN: opt -mcpu=a2 -loop-data-prefetch -S < %s | FileCheck %s
-; RUN: opt -mcpu=a2 -passes=loop-data-prefetch -S < %s | FileCheck %s
+; RUN: opt -mcpu=a2 -loop-data-prefetch -mtriple=powerpc64le-unknown-linux -enable-ppc-prefetching -S < %s | FileCheck %s
+; RUN: opt -mcpu=a2 -passes=loop-data-prefetch -mtriple=powerpc64le-unknown-linux -enable-ppc-prefetching -S < %s | FileCheck %s
 target datalayout = "E-m:e-i64:64-n32:64"
-target triple = "powerpc64-bgq-linux"
 
 define void @foo(double* nocapture %a, double* nocapture readonly %b) {
 entry:
diff --git a/llvm/test/Transforms/LoopSimplify/dup-preds.ll b/llvm/test/Transforms/LoopSimplify/dup-preds.ll
index c9253fa51a65f..362d834686d41 100644
--- a/llvm/test/Transforms/LoopSimplify/dup-preds.ll
+++ b/llvm/test/Transforms/LoopSimplify/dup-preds.ll
@@ -1,6 +1,6 @@
 ; RUN: opt -loop-simplify -S %s | FileCheck %s
 target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
-target triple = "powerpc64-bgq-linux"
+target triple = "powerpc64le-unknown-linux"
 
 define fastcc void @do_update_md([3 x float]* nocapture readonly %x) #0 {
 entry:
diff --git a/llvm/test/Transforms/LoopUnroll/pr14167.ll b/llvm/test/Transforms/LoopUnroll/pr14167.ll
index 9aac70115d9ae..3097c234fb933 100644
--- a/llvm/test/Transforms/LoopUnroll/pr14167.ll
+++ b/llvm/test/Transforms/LoopUnroll/pr14167.ll
@@ -1,6 +1,6 @@
 ; RUN: opt < %s -S -loop-unroll -unroll-runtime | FileCheck %s
 target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v128:128:128-n32:64"
-target triple = "powerpc64-bgq-linux"
+target triple = "powerpc64le-unknown-linux"
 
 define void @test1() nounwind {
 ; Ensure that we don't crash when the trip count == -1.
diff --git a/llvm/test/Transforms/LoopVectorize/PowerPC/agg-interleave-a2.ll b/llvm/test/Transforms/LoopVectorize/PowerPC/agg-interleave-a2.ll
deleted file mode 100644
index 9fdfb6f90e7bf..0000000000000
--- a/llvm/test/Transforms/LoopVectorize/PowerPC/agg-interleave-a2.ll
+++ /dev/null
@@ -1,40 +0,0 @@
-; RUN: opt -S -basic-aa -loop-vectorize < %s | FileCheck %s
-target datalayout = "E-m:e-i64:64-n32:64"
-target triple = "powerpc64-unknown-linux-gnu"
-
-; Function Attrs: nounwind
-define void @foo(double* noalias nocapture %a, double* noalias nocapture readonly %b, double* noalias nocapture readonly %c) #0 {
-entry:
-  br label %for.body
-
-; CHECK-LABEL: @foo
-; CHECK: fmul <4 x double> %{{[^,]+}}, <double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00>
-; CHECK-NEXT: fmul <4 x double> %{{[^,]+}}, <double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00>
-
-for.cond.cleanup:                                 ; preds = %for.body
-  ret void
-
-for.body:                                         ; preds = %for.body, %entry
-  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
-  %arrayidx = getelementptr inbounds double, double* %b, i64 %indvars.iv
-  %0 = load double, double* %arrayidx, align 8
-  %mul = fmul double %0, 2.000000e+00
-  %mul3 = fmul double %0, %mul
-  %arrayidx5 = getelementptr inbounds double, double* %c, i64 %indvars.iv
-  %1 = load double, double* %arrayidx5, align 8
-  %mul6 = fmul double %1, 3.000000e+00
-  %mul9 = fmul double %1, %mul6
-  %add = fadd double %mul3, %mul9
-  %mul12 = fmul double %0, 4.000000e+00
-  %mul15 = fmul double %mul12, %1
-  %add16 = fadd double %mul15, %add
-  %add17 = fadd double %add16, 1.000000e+00
-  %arrayidx19 = getelementptr inbounds double, double* %a, i64 %indvars.iv
-  store double %add17, double* %arrayidx19, align 8
-  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
-  %exitcond = icmp eq i64 %indvars.iv.next, 1600
-  br i1 %exitcond, label %for.cond.cleanup, label %for.body
-}
-
-attributes #0 = { nounwind "target-cpu"="a2q" }
-
diff --git a/llvm/test/Transforms/LoopVectorize/PowerPC/vectorize-only-for-real.ll b/llvm/test/Transforms/LoopVectorize/PowerPC/vectorize-only-for-real.ll
index 8abc25ece35c6..cddddba579473 100644
--- a/llvm/test/Transforms/LoopVectorize/PowerPC/vectorize-only-for-real.ll
+++ b/llvm/test/Transforms/LoopVectorize/PowerPC/vectorize-only-for-real.ll
@@ -1,6 +1,6 @@
 ; RUN: opt -S -loop-vectorize < %s | FileCheck %s
 target datalayout = "E-m:e-i64:64-n32:64"
-target triple = "powerpc64-bgq-linux"
+target triple = "powerpc64le-unknown-linux"
 
 ; Function Attrs: nounwind
 define zeroext i32 @test() #0 {
diff --git a/llvm/test/Transforms/NewGVN/pr31483.ll b/llvm/test/Transforms/NewGVN/pr31483.ll
index 999ff74ad5881..5bf7e1a695011 100644
--- a/llvm/test/Transforms/NewGVN/pr31483.ll
+++ b/llvm/test/Transforms/NewGVN/pr31483.ll
@@ -100,7 +100,7 @@ declare signext i32 @zot(i8*, ...) #1
 ; Function Attrs: nounwind
 declare void @llvm.va_end(i8*) #2
 
-attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64" "target-features"="+altivec,-bpermd,-crypto,-direct-move,-extdiv,-power8-vector,-qpx,-vsx" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64" "target-features"="+altivec,-bpermd,-crypto,-direct-move,-extdiv,-power8-vector,-qpx,-vsx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64" "target-features"="+altivec,-bpermd,-crypto,-direct-move,-extdiv,-power8-vector,-vsx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64" "target-features"="+altivec,-bpermd,-crypto,-direct-move,-extdiv,-power8-vector,-vsx" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind }
 
diff --git a/llvm/test/Transforms/SCCP/fp-bc-icmp-const-fold.ll b/llvm/test/Transforms/SCCP/fp-bc-icmp-const-fold.ll
index 8f97225ca446b..20c44384504e2 100644
--- a/llvm/test/Transforms/SCCP/fp-bc-icmp-const-fold.ll
+++ b/llvm/test/Transforms/SCCP/fp-bc-icmp-const-fold.ll
@@ -1,6 +1,6 @@
 ; RUN: opt -S -ipsccp < %s | FileCheck %s
 target datalayout = "E-m:e-i64:64-n32:64"
-target triple = "powerpc64-bgq-linux"
+target triple = "powerpc64le-unknown-linux"
 
 define void @test(i32 signext %n) {
 
diff --git a/llvm/unittests/ADT/TripleTest.cpp b/llvm/unittests/ADT/TripleTest.cpp
index dc7a28c72f208..1852d7b6a1b0d 100644
--- a/llvm/unittests/ADT/TripleTest.cpp
+++ b/llvm/unittests/ADT/TripleTest.cpp
@@ -111,41 +111,6 @@ TEST(TripleTest, ParsedIDs) {
   EXPECT_EQ(Triple::Linux, T.getOS());
   EXPECT_EQ(Triple::Musl, T.getEnvironment());
 
-  T = Triple("powerpc-bgp-linux");
-  EXPECT_EQ(Triple::ppc, T.getArch());
-  EXPECT_EQ(Triple::BGP, T.getVendor());
-  EXPECT_EQ(Triple::Linux, T.getOS());
-  EXPECT_EQ(Triple::UnknownEnvironment, T.getEnvironment());
-
-  T = Triple("powerpc-bgp-cnk");
-  EXPECT_EQ(Triple::ppc, T.getArch());
-  EXPECT_EQ(Triple::BGP, T.getVendor());
-  EXPECT_EQ(Triple::CNK, T.getOS());
-  EXPECT_EQ(Triple::UnknownEnvironment, T.getEnvironment());
-
-  T = Triple("ppc-bgp-linux");
-  EXPECT_EQ(Triple::ppc, T.getArch());
-  EXPECT_EQ(Triple::BGP, T.getVendor());
-  EXPECT_EQ(Triple::Linux, T.getOS());
-  EXPECT_EQ(Triple::UnknownEnvironment, T.getEnvironment());
-
-  T = Triple("ppc32-bgp-linux");
-  EXPECT_EQ(Triple::ppc, T.getArch());
-  EXPECT_EQ(Triple::BGP, T.getVendor());
-  EXPECT_EQ(Triple::Linux, T.getOS());
-  EXPECT_EQ(Triple::UnknownEnvironment, T.getEnvironment());
-
-  T = Triple("powerpc64-bgq-linux");
-  EXPECT_EQ(Triple::ppc64, T.getArch());
-  EXPECT_EQ(Triple::BGQ, T.getVendor());
-  EXPECT_EQ(Triple::Linux, T.getOS());
-  EXPECT_EQ(Triple::UnknownEnvironment, T.getEnvironment());
-
-  T = Triple("ppc64-bgq-linux");
-  EXPECT_EQ(Triple::ppc64, T.getArch());
-  EXPECT_EQ(Triple::BGQ, T.getVendor());
-  EXPECT_EQ(Triple::Linux, T.getOS());
-
   T = Triple("powerpc-ibm-aix");
   EXPECT_EQ(Triple::ppc, T.getArch());
   EXPECT_EQ(Triple::IBM, T.getVendor());
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/PowerPC/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/PowerPC/BUILD.gn
index 043a672a76e1e..3a452fc6e0601 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/PowerPC/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/PowerPC/BUILD.gn
@@ -61,7 +61,6 @@ static_library("LLVMPowerPCCodeGen") {
     "PPCMachineScheduler.cpp",
     "PPCMacroFusion.cpp",
     "PPCPreEmitPeephole.cpp",
-    "PPCQPXLoadSplat.cpp",
     "PPCReduceCRLogicals.cpp",
     "PPCRegisterInfo.cpp",
     "PPCSubtarget.cpp",
diff --git a/openmp/runtime/src/kmp.h b/openmp/runtime/src/kmp.h
index 933573bc810cb..bb6cee740ace7 100644
--- a/openmp/runtime/src/kmp.h
+++ b/openmp/runtime/src/kmp.h
@@ -1116,9 +1116,6 @@ extern kmp_uint64 __kmp_now_nsec();
 #if KMP_OS_WINDOWS
 #define KMP_INIT_WAIT 64U /* initial number of spin-tests   */
 #define KMP_NEXT_WAIT 32U /* susequent number of spin-tests */
-#elif KMP_OS_CNK
-#define KMP_INIT_WAIT 16U /* initial number of spin-tests   */
-#define KMP_NEXT_WAIT 8U /* susequent number of spin-tests */
 #elif KMP_OS_LINUX
 #define KMP_INIT_WAIT 1024U /* initial number of spin-tests   */
 #define KMP_NEXT_WAIT 512U /* susequent number of spin-tests */
diff --git a/openmp/runtime/src/kmp_csupport.cpp b/openmp/runtime/src/kmp_csupport.cpp
index b5c641cc7273c..f6fb1e602c297 100644
--- a/openmp/runtime/src/kmp_csupport.cpp
+++ b/openmp/runtime/src/kmp_csupport.cpp
@@ -680,17 +680,6 @@ void __kmpc_flush(ident_t *loc) {
 // Nothing to see here move along
 #elif KMP_ARCH_PPC64
 // Nothing needed here (we have a real MB above).
-#if KMP_OS_CNK
-  // The flushing thread needs to yield here; this prevents a
-  // busy-waiting thread from saturating the pipeline. flush is
-  // often used in loops like this:
-  // while (!flag) {
-  //   #pragma omp flush(flag)
-  // }
-  // and adding the yield here is good for at least a 10x speedup
-  // when running >2 threads per core (on the NAS LU benchmark).
-  __kmp_yield();
-#endif
 #else
 #error Unknown or unsupported architecture
 #endif
diff --git a/openmp/runtime/src/kmp_lock.h b/openmp/runtime/src/kmp_lock.h
index e54f6812b8b34..b80e54777e8c2 100644
--- a/openmp/runtime/src/kmp_lock.h
+++ b/openmp/runtime/src/kmp_lock.h
@@ -159,7 +159,7 @@ extern void __kmp_destroy_nested_tas_lock(kmp_tas_lock_t *lck);
 #define KMP_LOCK_ACQUIRED_NEXT 0
 #ifndef KMP_USE_FUTEX
 #define KMP_USE_FUTEX                                                          \
-  (KMP_OS_LINUX && !KMP_OS_CNK &&                                              \
+  (KMP_OS_LINUX &&                                                             \
    (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64))
 #endif
 #if KMP_USE_FUTEX
diff --git a/openmp/runtime/src/kmp_os.h b/openmp/runtime/src/kmp_os.h
index bfe7765b2a967..33735cf455c7e 100644
--- a/openmp/runtime/src/kmp_os.h
+++ b/openmp/runtime/src/kmp_os.h
@@ -69,7 +69,7 @@
 #error Unknown compiler
 #endif
 
-#if (KMP_OS_LINUX || KMP_OS_WINDOWS || KMP_OS_FREEBSD) && !KMP_OS_CNK
+#if (KMP_OS_LINUX || KMP_OS_WINDOWS || KMP_OS_FREEBSD)
 #define KMP_AFFINITY_SUPPORTED 1
 #if KMP_OS_WINDOWS && KMP_ARCH_X86_64
 #define KMP_GROUP_AFFINITY 1
diff --git a/openmp/runtime/src/kmp_platform.h b/openmp/runtime/src/kmp_platform.h
index 779c08e9771d5..4296ca31d67d9 100644
--- a/openmp/runtime/src/kmp_platform.h
+++ b/openmp/runtime/src/kmp_platform.h
@@ -22,7 +22,6 @@
 #define KMP_OS_OPENBSD 0
 #define KMP_OS_DARWIN 0
 #define KMP_OS_WINDOWS 0
-#define KMP_OS_CNK 0
 #define KMP_OS_HURD 0
 #define KMP_OS_UNIX 0 /* disjunction of KMP_OS_LINUX, KMP_OS_DARWIN etc. */
 
@@ -66,11 +65,6 @@
 #define KMP_OS_OPENBSD 1
 #endif
 
-#if (defined __bgq__)
-#undef KMP_OS_CNK
-#define KMP_OS_CNK 1
-#endif
-
 #if (defined __GNU__)
 #undef KMP_OS_HURD
 #define KMP_OS_HURD 1
diff --git a/openmp/runtime/src/z_Linux_asm.S b/openmp/runtime/src/z_Linux_asm.S
index 8090ff759fe1b..16059a3762bf4 100644
--- a/openmp/runtime/src/z_Linux_asm.S
+++ b/openmp/runtime/src/z_Linux_asm.S
@@ -1433,13 +1433,8 @@ __kmp_invoke_microtask:
 	add 12, 0, 12
 	neg 12, 12
 
-// We need to make sure that the stack frame stays aligned (to 16 bytes, except
-// under the BG/Q CNK, where it must be to 32 bytes).
-# if KMP_OS_CNK
-	li 0, -32
-# else
+// We need to make sure that the stack frame stays aligned (to 16 bytes).
 	li 0, -16
-# endif
 	and 12, 0, 12
 
 // Establish the local stack frame.
diff --git a/openmp/runtime/src/z_Linux_util.cpp b/openmp/runtime/src/z_Linux_util.cpp
index 3b5910fc95e89..58cc4d25f6080 100644
--- a/openmp/runtime/src/z_Linux_util.cpp
+++ b/openmp/runtime/src/z_Linux_util.cpp
@@ -31,7 +31,7 @@
 #include <sys/times.h>
 #include <unistd.h>
 
-#if KMP_OS_LINUX && !KMP_OS_CNK
+#if KMP_OS_LINUX
 #include <sys/sysinfo.h>
 #if KMP_USE_FUTEX
 // We should really include <futex.h>, but that causes compatibility problems on
diff --git a/polly/lib/External/isl/config.sub b/polly/lib/External/isl/config.sub
index 1d8e98bcee23a..bc4db70f82abf 100755
--- a/polly/lib/External/isl/config.sub
+++ b/polly/lib/External/isl/config.sub
@@ -152,9 +152,6 @@ case $os in
 		os=
 		basic_machine=$1
 		;;
-	-bluegene*)
-		os=-cnk
-		;;
 	-sim | -cisco | -oki | -wec | -winbond)
 		os=
 		basic_machine=$1
@@ -539,10 +536,6 @@ case $basic_machine in
 		basic_machine=bfin-`echo "$basic_machine" | sed 's/^[^-]*-//'`
 		os=-linux
 		;;
-	bluegene*)
-		basic_machine=powerpc-ibm
-		os=-cnk
-		;;
 	c54x-*)
 		basic_machine=tic54x-`echo "$basic_machine" | sed 's/^[^-]*-//'`
 		;;
@@ -1364,7 +1357,7 @@ case $os in
 	# Each alternative MUST end in a * to match a version number.
 	# -sysv* is not here because it comes later, after sysvr4.
 	-gnu* | -bsd* | -mach* | -minix* | -genix* | -ultrix* | -irix* \
-	      | -*vms* | -sco* | -esix* | -isc* | -aix* | -cnk* | -sunos | -sunos[34]*\
+	      | -*vms* | -sco* | -esix* | -isc* | -aix* | -sunos | -sunos[34]*\
 	      | -hpux* | -unos* | -osf* | -luna* | -dgux* | -auroraux* | -solaris* \
 	      | -sym* | -kopensolaris* | -plan9* \
 	      | -amigaos* | -amigados* | -msdos* | -newsos* | -unicos* | -aof* \
@@ -1728,7 +1721,7 @@ case $basic_machine in
 			-sunos*)
 				vendor=sun
 				;;
-			-cnk*|-aix*)
+			-aix*)
 				vendor=ibm
 				;;
 			-beos*)
diff --git a/polly/lib/External/ppcg/config.sub b/polly/lib/External/ppcg/config.sub
index 6205f8423d6aa..d97f3009f9f09 100644
--- a/polly/lib/External/ppcg/config.sub
+++ b/polly/lib/External/ppcg/config.sub
@@ -160,9 +160,6 @@ case $os in
 		os=
 		basic_machine=$1
 		;;
-	-bluegene*)
-		os=-cnk
-		;;
 	-sim | -cisco | -oki | -wec | -winbond)
 		os=
 		basic_machine=$1
@@ -524,10 +521,6 @@ case $basic_machine in
 		basic_machine=bfin-`echo $basic_machine | sed 's/^[^-]*-//'`
 		os=-linux
 		;;
-	bluegene*)
-		basic_machine=powerpc-ibm
-		os=-cnk
-		;;
 	c54x-*)
 		basic_machine=tic54x-`echo $basic_machine | sed 's/^[^-]*-//'`
 		;;
@@ -1344,7 +1337,7 @@ case $os in
 	# Each alternative MUST END IN A *, to match a version number.
 	# -sysv* is not here because it comes later, after sysvr4.
 	-gnu* | -bsd* | -mach* | -minix* | -genix* | -ultrix* | -irix* \
-	      | -*vms* | -sco* | -esix* | -isc* | -aix* | -cnk* | -sunos | -sunos[34]*\
+	      | -*vms* | -sco* | -esix* | -isc* | -aix* | -sunos | -sunos[34]*\
 	      | -hpux* | -unos* | -osf* | -luna* | -dgux* | -auroraux* | -solaris* \
 	      | -sym* | -kopensolaris* \
 	      | -amigaos* | -amigados* | -msdos* | -newsos* | -unicos* | -aof* \
@@ -1709,7 +1702,7 @@ case $basic_machine in
 			-sunos*)
 				vendor=sun
 				;;
-			-cnk*|-aix*)
+			-aix*)
 				vendor=ibm
 				;;
 			-beos*)

From 536baa11cfe12362ea646ad731a2274a07208cc0 Mon Sep 17 00:00:00 2001
From: Jonas Devlieghere <jonas@devlieghere.com>
Date: Mon, 27 Jul 2020 12:30:09 -0700
Subject: [PATCH 0235/1035] [lldb] Remove CMAKE_VERSION checks now that the
 minimum version is 3.13.4

---
 .../modules/FindPythonInterpAndLibs.cmake     | 38 ++++---------------
 lldb/cmake/modules/LLDBConfig.cmake           |  5 ---
 2 files changed, 7 insertions(+), 36 deletions(-)

diff --git a/lldb/cmake/modules/FindPythonInterpAndLibs.cmake b/lldb/cmake/modules/FindPythonInterpAndLibs.cmake
index 243e0463f48b6..3a64ebbcf9721 100644
--- a/lldb/cmake/modules/FindPythonInterpAndLibs.cmake
+++ b/lldb/cmake/modules/FindPythonInterpAndLibs.cmake
@@ -61,46 +61,22 @@ if(PYTHON_LIBRARIES AND PYTHON_INCLUDE_DIRS AND PYTHON_EXECUTABLE AND SWIG_EXECU
 else()
   find_package(SWIG 2.0)
   if (SWIG_FOUND)
-    if(NOT CMAKE_VERSION VERSION_LESS 3.12)
-      if (LLDB_PYTHON_VERSION)
-        if (LLDB_PYTHON_VERSION VERSION_EQUAL "2")
-          FindPython2()
-        elseif(LLDB_PYTHON_VERSION VERSION_EQUAL "3")
-          FindPython3()
-        endif()
-      else()
+    if (LLDB_PYTHON_VERSION)
+      if (LLDB_PYTHON_VERSION VERSION_EQUAL "2")
+        FindPython2()
+      elseif(LLDB_PYTHON_VERSION VERSION_EQUAL "3")
         FindPython3()
-        if (NOT PYTHON3_FOUND AND NOT CMAKE_SYSTEM_NAME STREQUAL Windows)
-          FindPython2()
-        endif()
       endif()
     else()
-      find_package(PythonInterp)
-      find_package(PythonLibs)
-      if(PYTHONINTERP_FOUND AND PYTHONLIBS_FOUND AND SWIG_FOUND)
-        if (NOT CMAKE_CROSSCOMPILING)
-          string(REPLACE "." ";" pythonlibs_version_list ${PYTHONLIBS_VERSION_STRING})
-          list(GET pythonlibs_version_list 0 pythonlibs_major)
-          list(GET pythonlibs_version_list 1 pythonlibs_minor)
-
-          # Ignore the patch version. Some versions of macOS report a different
-          # patch version for the system provided interpreter and libraries.
-          if (CMAKE_CROSSCOMPILING OR (PYTHON_VERSION_MAJOR VERSION_EQUAL pythonlibs_major AND
-              PYTHON_VERSION_MINOR VERSION_EQUAL pythonlibs_minor))
-            mark_as_advanced(
-              PYTHON_LIBRARIES
-              PYTHON_INCLUDE_DIRS
-              PYTHON_EXECUTABLE
-              SWIG_EXECUTABLE)
-          endif()
-        endif()
+      FindPython3()
+      if (NOT PYTHON3_FOUND AND NOT CMAKE_SYSTEM_NAME STREQUAL Windows)
+        FindPython2()
       endif()
     endif()
   else()
     message(STATUS "SWIG 2 or later is required for Python support in LLDB but could not be found")
   endif()
 
-
   include(FindPackageHandleStandardArgs)
   find_package_handle_standard_args(PythonInterpAndLibs
                                     FOUND_VAR
diff --git a/lldb/cmake/modules/LLDBConfig.cmake b/lldb/cmake/modules/LLDBConfig.cmake
index 8465cfe3b7b72..7e5848c800f87 100644
--- a/lldb/cmake/modules/LLDBConfig.cmake
+++ b/lldb/cmake/modules/LLDBConfig.cmake
@@ -79,11 +79,6 @@ if(LLDB_BUILD_FRAMEWORK)
   if(NOT APPLE)
     message(FATAL_ERROR "LLDB.framework can only be generated when targeting Apple platforms")
   endif()
-  # CMake 3.6 did not correctly emit POST_BUILD commands for Apple Framework targets
-  # CMake < 3.8 did not have the BUILD_RPATH target property
-  if(CMAKE_VERSION VERSION_LESS 3.8)
-    message(FATAL_ERROR "LLDB_BUILD_FRAMEWORK is not supported on CMake < 3.8")
-  endif()
 
   set(LLDB_FRAMEWORK_VERSION A CACHE STRING "LLDB.framework version (default is A)")
   set(LLDB_FRAMEWORK_BUILD_DIR bin CACHE STRING "Output directory for LLDB.framework")

From 6dadf7cb654ce131c1d05b3add8e30344b6d22be Mon Sep 17 00:00:00 2001
From: Jon Roelofs <jonathan_roelofs@apple.com>
Date: Mon, 27 Jul 2020 13:37:35 -0600
Subject: [PATCH 0236/1035] [llvm][examples][SimplifyCFG] Fix pass's IR changed
 reporting

... under the EXPENSIVE_CHECKS build, this fails the assert in the LegacyPM
that verifies whether a pass really did leave the IR alone when it reports no
changes back from its return status.
---
 llvm/examples/IRTransforms/SimplifyCFG.cpp | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/llvm/examples/IRTransforms/SimplifyCFG.cpp b/llvm/examples/IRTransforms/SimplifyCFG.cpp
index 10658c9f09590..82368d7494212 100644
--- a/llvm/examples/IRTransforms/SimplifyCFG.cpp
+++ b/llvm/examples/IRTransforms/SimplifyCFG.cpp
@@ -354,18 +354,18 @@ static bool mergeIntoSinglePredecessor_v2(Function &F, DominatorTree &DT) {
 }
 
 static bool doSimplify_v1(Function &F) {
-  return eliminateCondBranches_v1(F) & mergeIntoSinglePredecessor_v1(F) &
+  return eliminateCondBranches_v1(F) | mergeIntoSinglePredecessor_v1(F) |
          removeDeadBlocks_v1(F);
 }
 
 static bool doSimplify_v2(Function &F, DominatorTree &DT) {
-  return eliminateCondBranches_v2(F, DT) &
-         mergeIntoSinglePredecessor_v2(F, DT) & removeDeadBlocks_v2(F, DT);
+  return eliminateCondBranches_v2(F, DT) |
+         mergeIntoSinglePredecessor_v2(F, DT) | removeDeadBlocks_v2(F, DT);
 }
 
 static bool doSimplify_v3(Function &F, DominatorTree &DT) {
-  return eliminateCondBranches_v3(F, DT) &
-         mergeIntoSinglePredecessor_v2(F, DT) & removeDeadBlocks_v2(F, DT);
+  return eliminateCondBranches_v3(F, DT) |
+         mergeIntoSinglePredecessor_v2(F, DT) | removeDeadBlocks_v2(F, DT);
 }
 
 namespace {

From 4c6eebf86a0734779cd20473cfcaa9d7c8899298 Mon Sep 17 00:00:00 2001
From: Fred Riss <friss@apple.com>
Date: Fri, 24 Jul 2020 09:24:41 -0700
Subject: [PATCH 0237/1035] [lldb/AppleSimulator] Always provide a -simulator
 environment

Summary:
This commit is somewhat NFC-ish today as the environment of triples
is not considered when comparing s if one of them is
not set (I plan to change that).

We have made simulator triples unambiguous these days, but the
simulator platforms still advertise triples without the
environment. This wasn't an issue when the sims ran only on
a very different architecure than the real device, but this
has changed with Apple Silicon.

This patch simplifies the way GetSupportedArchitectureAtIndex
is implemented for the sim platforms and adds the environment.
It also trivially adds support for Apple Silicon to those
platforms.

Reviewers: aprantl

Subscribers: lldb-commits
---
 .../MacOSX/PlatformAppleSimulator.cpp         |  8 ++
 .../Platform/MacOSX/PlatformAppleSimulator.h  |  6 ++
 .../MacOSX/PlatformAppleTVSimulator.cpp       | 36 ++++-----
 .../MacOSX/PlatformAppleTVSimulator.h         |  3 -
 .../MacOSX/PlatformAppleWatchSimulator.cpp    | 40 +++++-----
 .../MacOSX/PlatformAppleWatchSimulator.h      |  3 -
 .../Platform/MacOSX/PlatformiOSSimulator.cpp  | 61 +++++----------
 .../Platform/MacOSX/PlatformiOSSimulator.h    |  3 -
 lldb/unittests/Platform/CMakeLists.txt        |  1 +
 .../Platform/PlatformAppleSimulatorTest.cpp   | 74 +++++++++++++++++++
 10 files changed, 146 insertions(+), 89 deletions(-)
 create mode 100644 lldb/unittests/Platform/PlatformAppleSimulatorTest.cpp

diff --git a/lldb/source/Plugins/Platform/MacOSX/PlatformAppleSimulator.cpp b/lldb/source/Plugins/Platform/MacOSX/PlatformAppleSimulator.cpp
index bd0a231303bd1..0160fb95c58a9 100644
--- a/lldb/source/Plugins/Platform/MacOSX/PlatformAppleSimulator.cpp
+++ b/lldb/source/Plugins/Platform/MacOSX/PlatformAppleSimulator.cpp
@@ -253,3 +253,11 @@ CoreSimulatorSupport::Device PlatformAppleSimulator::GetSimulatorDevice() {
     return CoreSimulatorSupport::Device();
 }
 #endif
+
+bool PlatformAppleSimulator::GetSupportedArchitectureAtIndex(uint32_t idx,
+                                                             ArchSpec &arch) {
+  if (idx >= m_supported_triples.size())
+    return false;
+  arch = ArchSpec(m_supported_triples[idx]);
+  return true;
+}
diff --git a/lldb/source/Plugins/Platform/MacOSX/PlatformAppleSimulator.h b/lldb/source/Plugins/Platform/MacOSX/PlatformAppleSimulator.h
index 8c0174f2946ed..6182acaf229ac 100644
--- a/lldb/source/Plugins/Platform/MacOSX/PlatformAppleSimulator.h
+++ b/lldb/source/Plugins/Platform/MacOSX/PlatformAppleSimulator.h
@@ -44,6 +44,9 @@ class PlatformAppleSimulator : public PlatformDarwin {
                                lldb_private::Target *target,
                                lldb_private::Status &error) override;
 
+  bool GetSupportedArchitectureAtIndex(uint32_t idx,
+                                       lldb_private::ArchSpec &arch) override;
+
 protected:
   std::mutex m_core_sim_path_mutex;
   llvm::Optional<lldb_private::FileSpec> m_core_simulator_framework_path;
@@ -52,6 +55,9 @@ class PlatformAppleSimulator : public PlatformDarwin {
 
   lldb_private::FileSpec GetCoreSimulatorPath();
 
+  llvm::Triple::OSType m_os_type = llvm::Triple::UnknownOS;
+  llvm::ArrayRef<llvm::StringRef> m_supported_triples = {};
+
   void LoadCoreSimulator();
 
 #if defined(__APPLE__)
diff --git a/lldb/source/Plugins/Platform/MacOSX/PlatformAppleTVSimulator.cpp b/lldb/source/Plugins/Platform/MacOSX/PlatformAppleTVSimulator.cpp
index 461624a2adaa8..27f798b00ebf4 100644
--- a/lldb/source/Plugins/Platform/MacOSX/PlatformAppleTVSimulator.cpp
+++ b/lldb/source/Plugins/Platform/MacOSX/PlatformAppleTVSimulator.cpp
@@ -77,6 +77,7 @@ PlatformSP PlatformAppleTVSimulator::CreateInstance(bool force,
   bool create = force;
   if (!create && arch && arch->IsValid()) {
     switch (arch->GetMachine()) {
+    case llvm::Triple::aarch64:
     case llvm::Triple::x86_64: {
       const llvm::Triple &triple = arch->GetTriple();
       switch (triple.getVendor()) {
@@ -144,7 +145,24 @@ const char *PlatformAppleTVSimulator::GetDescriptionStatic() {
 /// Default Constructor
 PlatformAppleTVSimulator::PlatformAppleTVSimulator()
     : PlatformAppleSimulator(
-          CoreSimulatorSupport::DeviceType::ProductFamilyID::appleTV) {}
+          CoreSimulatorSupport::DeviceType::ProductFamilyID::appleTV) {
+#ifdef __APPLE__
+#if __arm64__
+  static const llvm::StringRef supported_triples[] = {
+      "arm64e-apple-tvos-simulator",
+      "arm64-apple-tvos-simulator",
+      "x86_64h-apple-tvos-simulator",
+      "x86_64-apple-tvos-simulator",
+  };
+#else
+  static const llvm::StringRef supported_triples[] = {
+      "x86_64h-apple-tvos-simulator",
+      "x86_64-apple-tvos-simulator",
+  };
+#endif
+  m_supported_triples = supported_triples;
+#endif
+}
 
 /// Destructor.
 ///
@@ -322,19 +340,3 @@ uint32_t PlatformAppleTVSimulator::FindProcesses(
   }
   return process_infos.size();
 }
-
-bool PlatformAppleTVSimulator::GetSupportedArchitectureAtIndex(uint32_t idx,
-                                                               ArchSpec &arch) {
-  static const ArchSpec platform_arch(
-      HostInfo::GetArchitecture(HostInfo::eArchKind64));
-
-  if (idx == 0) {
-    arch = platform_arch;
-    if (arch.IsValid()) {
-      arch.GetTriple().setOS(llvm::Triple::TvOS);
-      arch.GetTriple().setEnvironment(llvm::Triple::Simulator);
-      return true;
-    }
-  }
-  return false;
-}
diff --git a/lldb/source/Plugins/Platform/MacOSX/PlatformAppleTVSimulator.h b/lldb/source/Plugins/Platform/MacOSX/PlatformAppleTVSimulator.h
index 5a7b0ee0d7dc9..a94f94f9f57f7 100644
--- a/lldb/source/Plugins/Platform/MacOSX/PlatformAppleTVSimulator.h
+++ b/lldb/source/Plugins/Platform/MacOSX/PlatformAppleTVSimulator.h
@@ -62,9 +62,6 @@ class PlatformAppleTVSimulator : public PlatformAppleSimulator {
   FindProcesses(const lldb_private::ProcessInstanceInfoMatch &match_info,
                 lldb_private::ProcessInstanceInfoList &process_infos) override;
 
-  bool GetSupportedArchitectureAtIndex(uint32_t idx,
-                                       lldb_private::ArchSpec &arch) override;
-
   void
   AddClangModuleCompilationOptions(lldb_private::Target *target,
                                    std::vector<std::string> &options) override {
diff --git a/lldb/source/Plugins/Platform/MacOSX/PlatformAppleWatchSimulator.cpp b/lldb/source/Plugins/Platform/MacOSX/PlatformAppleWatchSimulator.cpp
index 03a8fcd313602..79f254c43a6ae 100644
--- a/lldb/source/Plugins/Platform/MacOSX/PlatformAppleWatchSimulator.cpp
+++ b/lldb/source/Plugins/Platform/MacOSX/PlatformAppleWatchSimulator.cpp
@@ -76,6 +76,7 @@ PlatformSP PlatformAppleWatchSimulator::CreateInstance(bool force,
   bool create = force;
   if (!create && arch && arch->IsValid()) {
     switch (arch->GetMachine()) {
+    case llvm::Triple::aarch64:
     case llvm::Triple::x86_64:
     case llvm::Triple::x86: {
       const llvm::Triple &triple = arch->GetTriple();
@@ -145,7 +146,23 @@ const char *PlatformAppleWatchSimulator::GetDescriptionStatic() {
 /// Default Constructor
 PlatformAppleWatchSimulator::PlatformAppleWatchSimulator()
     : PlatformAppleSimulator(
-          CoreSimulatorSupport::DeviceType::ProductFamilyID::appleWatch) {}
+          CoreSimulatorSupport::DeviceType::ProductFamilyID::appleWatch) {
+#ifdef __APPLE__
+#if __arm64__
+  static const llvm::StringRef supported_triples[] = {
+      "arm64e-apple-watchos-simulator",
+      "arm64-apple-watchos-simulator",
+  };
+#else
+  static const llvm::StringRef supported_triples[] = {
+      "x86_64-apple-watchos-simulator",
+      "x86_64h-apple-watchos-simulator",
+      "i386-apple-watchos-simulator",
+  };
+#endif
+  m_supported_triples = supported_triples;
+#endif
+}
 
 /// Destructor.
 ///
@@ -325,24 +342,3 @@ uint32_t PlatformAppleWatchSimulator::FindProcesses(
   return process_infos.size();
 }
 
-bool PlatformAppleWatchSimulator::GetSupportedArchitectureAtIndex(
-    uint32_t idx, ArchSpec &arch) {
-  if (idx == 0) {
-    arch = HostInfo::GetArchitecture(HostInfo::eArchKind32);
-    if (arch.IsValid()) {
-      arch.GetTriple().setOS(llvm::Triple::WatchOS);
-      arch.GetTriple().setEnvironment(llvm::Triple::Simulator);
-      return true;
-    }
-  }
-
-  if (idx == 1) {
-    arch = HostInfo::GetArchitecture(HostInfo::eArchKind64);
-    if (arch.IsValid()) {
-      arch.GetTriple().setOS(llvm::Triple::WatchOS);
-      arch.GetTriple().setEnvironment(llvm::Triple::Simulator);
-      return true;
-    }
-  }
-  return false;
-}
diff --git a/lldb/source/Plugins/Platform/MacOSX/PlatformAppleWatchSimulator.h b/lldb/source/Plugins/Platform/MacOSX/PlatformAppleWatchSimulator.h
index 96dcd16ffa993..78b936691b0c7 100644
--- a/lldb/source/Plugins/Platform/MacOSX/PlatformAppleWatchSimulator.h
+++ b/lldb/source/Plugins/Platform/MacOSX/PlatformAppleWatchSimulator.h
@@ -62,9 +62,6 @@ class PlatformAppleWatchSimulator : public PlatformAppleSimulator {
   FindProcesses(const lldb_private::ProcessInstanceInfoMatch &match_info,
                 lldb_private::ProcessInstanceInfoList &process_infos) override;
 
-  bool GetSupportedArchitectureAtIndex(uint32_t idx,
-                                       lldb_private::ArchSpec &arch) override;
-
   void
   AddClangModuleCompilationOptions(lldb_private::Target *target,
                                    std::vector<std::string> &options) override {
diff --git a/lldb/source/Plugins/Platform/MacOSX/PlatformiOSSimulator.cpp b/lldb/source/Plugins/Platform/MacOSX/PlatformiOSSimulator.cpp
index a890d0afdf1e3..b73c06fcdc8b4 100644
--- a/lldb/source/Plugins/Platform/MacOSX/PlatformiOSSimulator.cpp
+++ b/lldb/source/Plugins/Platform/MacOSX/PlatformiOSSimulator.cpp
@@ -76,6 +76,7 @@ PlatformSP PlatformiOSSimulator::CreateInstance(bool force,
   bool create = force;
   if (!create && arch && arch->IsValid()) {
     switch (arch->GetMachine()) {
+    case llvm::Triple::aarch64:
     case llvm::Triple::x86_64:
     case llvm::Triple::x86: {
       const llvm::Triple &triple = arch->GetTriple();
@@ -148,7 +149,25 @@ const char *PlatformiOSSimulator::GetDescriptionStatic() {
 /// Default Constructor
 PlatformiOSSimulator::PlatformiOSSimulator()
     : PlatformAppleSimulator(
-          CoreSimulatorSupport::DeviceType::ProductFamilyID::iPhone) {}
+          CoreSimulatorSupport::DeviceType::ProductFamilyID::iPhone) {
+#ifdef __APPLE__
+#if __arm64__
+  static const llvm::StringRef supported_triples[] = {
+      "arm64e-apple-ios-simulator",
+      "arm64-apple-ios-simulator",
+      "x86_64-apple-ios-simulator",
+      "x86_64h-apple-ios-simulator",
+  };
+#else
+  static const llvm::StringRef supported_triples[] = {
+      "x86_64h-apple-ios-simulator",
+      "x86_64-apple-ios-simulator",
+      "i386-apple-ios-simulator",
+  };
+#endif
+  m_supported_triples = supported_triples;
+#endif
+}
 
 /// Destructor.
 ///
@@ -328,43 +347,3 @@ PlatformiOSSimulator::FindProcesses(const ProcessInstanceInfoMatch &match_info,
   return process_infos.size();
 }
 
-bool PlatformiOSSimulator::GetSupportedArchitectureAtIndex(uint32_t idx,
-                                                           ArchSpec &arch) {
-  static const ArchSpec platform_arch(
-      HostInfo::GetArchitecture(HostInfo::eArchKindDefault));
-  static const ArchSpec platform_arch64(
-      HostInfo::GetArchitecture(HostInfo::eArchKind64));
-
-  if (idx == 0) {
-    arch = platform_arch;
-    if (arch.IsValid()) {
-      arch.GetTriple().setOS(llvm::Triple::IOS);
-      arch.GetTriple().setEnvironment(llvm::Triple::Simulator);
-      return true;
-    }
-  } else {
-    if (platform_arch.IsExactMatch(platform_arch64)) {
-      // This macosx platform supports both 32 and 64 bit.
-      if (idx == 1) {
-        // 32/64: return "x86_64-apple-macosx" for architecture 1
-        arch = platform_arch64;
-        return true;
-      } else if (idx == 2 || idx == 3) {
-        arch = HostInfo::GetArchitecture(HostInfo::eArchKind32);
-        if (arch.IsValid()) {
-          if (idx == 2)
-            arch.GetTriple().setOS(llvm::Triple::IOS);
-          // 32/64: return "i386-apple-ios" for architecture 2 32/64: return
-          // "i386-apple-macosx" for architecture 3
-          return true;
-        }
-      }
-    } else if (idx == 1) {
-      // This macosx platform supports only 32 bit, so return the *-apple-
-      // macosx version
-      arch = platform_arch;
-      return true;
-    }
-  }
-  return false;
-}
diff --git a/lldb/source/Plugins/Platform/MacOSX/PlatformiOSSimulator.h b/lldb/source/Plugins/Platform/MacOSX/PlatformiOSSimulator.h
index 4d416d759bd2d..982f8e2de5e7a 100644
--- a/lldb/source/Plugins/Platform/MacOSX/PlatformiOSSimulator.h
+++ b/lldb/source/Plugins/Platform/MacOSX/PlatformiOSSimulator.h
@@ -64,9 +64,6 @@ class PlatformiOSSimulator : public PlatformAppleSimulator {
   FindProcesses(const lldb_private::ProcessInstanceInfoMatch &match_info,
                 lldb_private::ProcessInstanceInfoList &process_infos) override;
 
-  bool GetSupportedArchitectureAtIndex(uint32_t idx,
-                                       lldb_private::ArchSpec &arch) override;
-
   void
   AddClangModuleCompilationOptions(lldb_private::Target *target,
                                    std::vector<std::string> &options) override {
diff --git a/lldb/unittests/Platform/CMakeLists.txt b/lldb/unittests/Platform/CMakeLists.txt
index eb7f0a6ca3c41..ca5031b9b43e0 100644
--- a/lldb/unittests/Platform/CMakeLists.txt
+++ b/lldb/unittests/Platform/CMakeLists.txt
@@ -1,4 +1,5 @@
 add_lldb_unittest(LLDBPlatformTests
+  PlatformAppleSimulatorTest.cpp
   PlatformDarwinTest.cpp
 
   LINK_LIBS
diff --git a/lldb/unittests/Platform/PlatformAppleSimulatorTest.cpp b/lldb/unittests/Platform/PlatformAppleSimulatorTest.cpp
new file mode 100644
index 0000000000000..0b90380b797c5
--- /dev/null
+++ b/lldb/unittests/Platform/PlatformAppleSimulatorTest.cpp
@@ -0,0 +1,74 @@
+//===-- PlatformAppleSimulatorTest.cpp ------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "gtest/gtest.h"
+
+#include "Plugins/Platform/MacOSX/PlatformAppleTVSimulator.h"
+#include "Plugins/Platform/MacOSX/PlatformAppleWatchSimulator.h"
+#include "Plugins/Platform/MacOSX/PlatformiOSSimulator.h"
+#include "TestingSupport/SubsystemRAII.h"
+#include "lldb/Host/FileSystem.h"
+#include "lldb/Host/HostInfo.h"
+#include "lldb/Target/Platform.h"
+
+using namespace lldb;
+using namespace lldb_private;
+
+class PlatformAppleSimulatorTest : public ::testing::Test {
+  SubsystemRAII<FileSystem, HostInfo, PlatformAppleTVSimulator,
+                PlatformiOSSimulator, PlatformAppleWatchSimulator>
+      subsystems;
+};
+
+#ifdef __APPLE__
+
+static void testSimPlatformArchHasSimEnvironment(llvm::StringRef name) {
+  Status error;
+  auto platform_sp = Platform::Create(ConstString(name), error);
+  ASSERT_TRUE(platform_sp);
+  int num_arches = 0;
+
+  while (true) {
+    ArchSpec arch;
+    if (!platform_sp->GetSupportedArchitectureAtIndex(num_arches, arch))
+      break;
+    EXPECT_EQ(arch.GetTriple().getEnvironment(), llvm::Triple::Simulator);
+    num_arches++;
+  }
+
+  EXPECT_GT(num_arches, 0);
+}
+
+TEST_F(PlatformAppleSimulatorTest, TestSimHasSimEnvionament) {
+  testSimPlatformArchHasSimEnvironment("ios-simulator");
+  testSimPlatformArchHasSimEnvironment("tvos-simulator");
+  testSimPlatformArchHasSimEnvironment("watchos-simulator");
+}
+
+TEST_F(PlatformAppleSimulatorTest, TestHostPlatformToSim) {
+  static const ArchSpec platform_arch(
+      HostInfo::GetArchitecture(HostInfo::eArchKindDefault));
+
+  const llvm::Triple::OSType sim_platforms[] = {
+      llvm::Triple::IOS,
+      llvm::Triple::TvOS,
+      llvm::Triple::WatchOS,
+  };
+
+  for (auto sim : sim_platforms) {
+    ArchSpec arch = platform_arch;
+    arch.GetTriple().setOS(sim);
+    arch.GetTriple().setEnvironment(llvm::Triple::Simulator);
+
+    Status error;
+    auto platform_sp = Platform::Create(arch, nullptr, error);
+    EXPECT_TRUE(platform_sp);
+  }
+}
+
+#endif

From 932316660179c1273e365d9dbbe648478bc5c4f1 Mon Sep 17 00:00:00 2001
From: Ye Luo <yeluo@anl.gov>
Date: Mon, 27 Jul 2020 16:08:19 -0400
Subject: [PATCH 0238/1035] [OpenMP] Add more pass-through functions in
 DeviceTy

Summary:
1. Add DeviceTy::data_alloc, DeviceTy::data_delete, DeviceTy::data_alloc, DeviceTy::synchronize pass-through functions. Avoid directly accessing Device.RTL
2. Fix the type of the first argument of synchronize_ty in rth.h, device id is int32_t which is consistent with other functions.

Reviewers: tianshilei1992, jdoerfert

Reviewed By: tianshilei1992

Subscribers: yaxunl, guansong, sstefan1, openmp-commits

Tags: #openmp

Differential Revision: https://reviews.llvm.org/D84487
---
 openmp/libomptarget/src/api.cpp       |  6 ++----
 openmp/libomptarget/src/device.cpp    | 18 ++++++++++++++++--
 openmp/libomptarget/src/device.h      | 16 ++++++++++++++++
 openmp/libomptarget/src/omptarget.cpp | 10 +++-------
 openmp/libomptarget/src/rtl.h         |  2 +-
 5 files changed, 38 insertions(+), 14 deletions(-)

diff --git a/openmp/libomptarget/src/api.cpp b/openmp/libomptarget/src/api.cpp
index 7baeebeb0a2a9..6a5ad86bd18a4 100644
--- a/openmp/libomptarget/src/api.cpp
+++ b/openmp/libomptarget/src/api.cpp
@@ -57,8 +57,7 @@ EXTERN void *omp_target_alloc(size_t size, int device_num) {
     return NULL;
   }
 
-  DeviceTy &Device = Devices[device_num];
-  rc = Device.RTL->data_alloc(Device.RTLDeviceID, size, NULL);
+  rc = Devices[device_num].data_alloc(size);
   DP("omp_target_alloc returns device ptr " DPxMOD "\n", DPxPTR(rc));
   return rc;
 }
@@ -83,8 +82,7 @@ EXTERN void omp_target_free(void *device_ptr, int device_num) {
     return;
   }
 
-  DeviceTy &Device = Devices[device_num];
-  Device.RTL->data_delete(Device.RTLDeviceID, (void *)device_ptr);
+  Devices[device_num].data_delete(device_ptr);
   DP("omp_target_free deallocated device ptr\n");
 }
 
diff --git a/openmp/libomptarget/src/device.cpp b/openmp/libomptarget/src/device.cpp
index 867083fde4892..6da8320462eef 100644
--- a/openmp/libomptarget/src/device.cpp
+++ b/openmp/libomptarget/src/device.cpp
@@ -217,7 +217,7 @@ void *DeviceTy::getOrAllocTgtPtr(void *HstPtrBegin, void *HstPtrBase,
   } else if (Size) {
     // If it is not contained and Size > 0, we should create a new entry for it.
     IsNew = true;
-    uintptr_t tp = (uintptr_t)RTL->data_alloc(RTLDeviceID, Size, HstPtrBegin);
+    uintptr_t tp = (uintptr_t)data_alloc(Size, HstPtrBegin);
     DP("Creating new map entry: HstBase=" DPxMOD ", HstBegin=" DPxMOD ", "
        "HstEnd=" DPxMOD ", TgtBegin=" DPxMOD "\n",
        DPxPTR(HstPtrBase), DPxPTR(HstPtrBegin),
@@ -299,7 +299,7 @@ int DeviceTy::deallocTgtPtr(void *HstPtrBegin, int64_t Size, bool ForceDelete,
     if (HT.decRefCount() == 0) {
       DP("Deleting tgt data " DPxMOD " of size %ld\n",
           DPxPTR(HT.TgtPtrBegin), Size);
-      RTL->data_delete(RTLDeviceID, (void *)HT.TgtPtrBegin);
+      data_delete((void *)HT.TgtPtrBegin);
       DP("Removing%s mapping with HstPtrBegin=" DPxMOD ", TgtPtrBegin=" DPxMOD
           ", Size=%ld\n", (ForceDelete ? " (forced)" : ""),
           DPxPTR(HT.HstPtrBegin), DPxPTR(HT.TgtPtrBegin), Size);
@@ -351,6 +351,14 @@ __tgt_target_table *DeviceTy::load_binary(void *Img) {
   return rc;
 }
 
+void *DeviceTy::data_alloc(int64_t Size, void *HstPtr) {
+  return RTL->data_alloc(RTLDeviceID, Size, HstPtr);
+}
+
+int32_t DeviceTy::data_delete(void *TgtPtrBegin) {
+  return RTL->data_delete(RTLDeviceID, TgtPtrBegin);
+}
+
 // Submit data to device
 int32_t DeviceTy::data_submit(void *TgtPtrBegin, void *HstPtrBegin,
                               int64_t Size, __tgt_async_info *AsyncInfoPtr) {
@@ -423,6 +431,12 @@ bool DeviceTy::isDataExchangable(const DeviceTy &DstDevice) {
   return false;
 }
 
+int32_t DeviceTy::synchronize(__tgt_async_info *AsyncInfoPtr) {
+  if (RTL->synchronize)
+    return RTL->synchronize(RTLDeviceID, AsyncInfoPtr);
+  return OFFLOAD_SUCCESS;
+}
+
 /// Check whether a device has an associated RTL and initialize it if it's not
 /// already initialized.
 bool device_is_ready(int device_num) {
diff --git a/openmp/libomptarget/src/device.h b/openmp/libomptarget/src/device.h
index ebec76c3cf62a..72b919c04df01 100644
--- a/openmp/libomptarget/src/device.h
+++ b/openmp/libomptarget/src/device.h
@@ -192,6 +192,18 @@ struct DeviceTy {
   int32_t initOnce();
   __tgt_target_table *load_binary(void *Img);
 
+  // device memory allocation/deallocation routines
+  /// Allocates \p Size bytes on the device and returns the address/nullptr when
+  /// succeeds/fails. \p HstPtr is an address of the host data which the
+  /// allocated target data will be associated with. If it is unknown, the
+  /// default value of \p HstPtr is nullptr. Note: this function doesn't do
+  /// pointer association. Actually, all the __tgt_rtl_data_alloc
+  /// implementations ignore \p HstPtr.
+  void *data_alloc(int64_t Size, void *HstPtr = nullptr);
+  /// Deallocates memory which \p TgtPtrBegin points at and returns
+  /// OFFLOAD_SUCCESS/OFFLOAD_FAIL when succeeds/fails.
+  int32_t data_delete(void *TgtPtrBegin);
+
   // Data transfer. When AsyncInfoPtr is nullptr, the transfer will be
   // synchronous.
   // Copy data from host to device
@@ -213,6 +225,10 @@ struct DeviceTy {
                           uint64_t LoopTripCount,
                           __tgt_async_info *AsyncInfoPtr);
 
+  /// Synchronize device/queue/event based on \p AsyncInfoPtr and return
+  /// OFFLOAD_SUCCESS/OFFLOAD_FAIL when succeeds/fails.
+  int32_t synchronize(__tgt_async_info *AsyncInfoPtr);
+
 private:
   // Call to RTL
   void init(); // To be called only via DeviceTy::initOnce()
diff --git a/openmp/libomptarget/src/omptarget.cpp b/openmp/libomptarget/src/omptarget.cpp
index 47971b9c0a00a..25722d75f6ea4 100644
--- a/openmp/libomptarget/src/omptarget.cpp
+++ b/openmp/libomptarget/src/omptarget.cpp
@@ -845,8 +845,7 @@ int target(int64_t device_id, void *host_ptr, int32_t arg_num,
       TgtBaseOffset = 0;
     } else if (arg_types[i] & OMP_TGT_MAPTYPE_PRIVATE) {
       // Allocate memory for (first-)private array
-      TgtPtrBegin = Device.RTL->data_alloc(Device.RTLDeviceID,
-          arg_sizes[i], HstPtrBegin);
+      TgtPtrBegin = Device.data_alloc(arg_sizes[i], HstPtrBegin);
       if (!TgtPtrBegin) {
         DP ("Data allocation for %sprivate array " DPxMOD " failed, "
             "abort target.\n",
@@ -929,7 +928,7 @@ int target(int64_t device_id, void *host_ptr, int32_t arg_num,
 
   // Deallocate (first-)private arrays
   for (auto it : fpArrays) {
-    int rt = Device.RTL->data_delete(Device.RTLDeviceID, it);
+    int rt = Device.data_delete(it);
     if (rt != OFFLOAD_SUCCESS) {
       DP("Deallocation of (first-)private arrays failed.\n");
       return OFFLOAD_FAIL;
@@ -944,8 +943,5 @@ int target(int64_t device_id, void *host_ptr, int32_t arg_num,
     return OFFLOAD_FAIL;
   }
 
-  if (Device.RTL->synchronize)
-    return Device.RTL->synchronize(device_id, &AsyncInfo);
-
-  return OFFLOAD_SUCCESS;
+  return Device.synchronize(&AsyncInfo);
 }
diff --git a/openmp/libomptarget/src/rtl.h b/openmp/libomptarget/src/rtl.h
index 941461f22b5c6..a9695a1022c21 100644
--- a/openmp/libomptarget/src/rtl.h
+++ b/openmp/libomptarget/src/rtl.h
@@ -52,7 +52,7 @@ struct RTLInfoTy {
                                             int32_t, uint64_t,
                                             __tgt_async_info *);
   typedef int64_t(init_requires_ty)(int64_t);
-  typedef int64_t(synchronize_ty)(int64_t, __tgt_async_info *);
+  typedef int64_t(synchronize_ty)(int32_t, __tgt_async_info *);
 
   int32_t Idx = -1;             // RTL index, index is the number of devices
                                 // of other RTLs that were registered before,

From 145acacaea1d7fb4ffc055a3e92ee8fee7c58634 Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne@apple.com>
Date: Fri, 24 Jul 2020 10:13:53 -0400
Subject: [PATCH 0239/1035] [libunwind] Remove old keymgr related logic

keymgr used to be used on MacOSX <= 10.6, however we don't build libunwind
from scratch for such old systems anymore. Hence, this code isn't useful
anymore.

Differential Revision: https://reviews.llvm.org/D84677
---
 libunwind/src/AddressSpace.hpp       | 11 -----
 libunwind/src/Unwind_AppleExtras.cpp | 68 ----------------------------
 2 files changed, 79 deletions(-)

diff --git a/libunwind/src/AddressSpace.hpp b/libunwind/src/AddressSpace.hpp
index 764aaa3489f26..3d1e810f43c08 100644
--- a/libunwind/src/AddressSpace.hpp
+++ b/libunwind/src/AddressSpace.hpp
@@ -39,13 +39,6 @@ struct EHABIIndexEntry {
 };
 #endif
 
-#ifdef __APPLE__
-#include <mach-o/getsect.h>
-namespace libunwind {
-   bool checkKeyMgrRegisteredFDEs(uintptr_t targetAddr, void *&fde);
-}
-#endif
-
 #include "libunwind.h"
 #include "config.h"
 #include "dwarf2.h"
@@ -651,14 +644,10 @@ inline bool LocalAddressSpace::findUnwindSections(pint_t targetAddr,
 
 
 inline bool LocalAddressSpace::findOtherFDE(pint_t targetAddr, pint_t &fde) {
-#ifdef __APPLE__
-  return checkKeyMgrRegisteredFDEs(targetAddr, *((void**)&fde));
-#else
   // TO DO: if OS has way to dynamically register FDEs, check that.
   (void)targetAddr;
   (void)fde;
   return false;
-#endif
 }
 
 inline bool LocalAddressSpace::findFunctionName(pint_t addr, char *buf,
diff --git a/libunwind/src/Unwind_AppleExtras.cpp b/libunwind/src/Unwind_AppleExtras.cpp
index 536303993effb..1d9948aced355 100644
--- a/libunwind/src/Unwind_AppleExtras.cpp
+++ b/libunwind/src/Unwind_AppleExtras.cpp
@@ -12,33 +12,6 @@
 #include "DwarfParser.hpp"
 
 
-// private keymgr stuff
-#define KEYMGR_GCC3_DW2_OBJ_LIST 302
-extern "C" {
- extern void _keymgr_set_and_unlock_processwide_ptr(int key, void *ptr);
- extern void *_keymgr_get_and_lock_processwide_ptr(int key);
-}
-
-// undocumented libgcc "struct object"
-struct libgcc_object {
-  void          *start;
-  void          *unused1;
-  void          *unused2;
-  void          *fde;
-  unsigned long  encoding;
-  void          *fde_end;
-  libgcc_object *next;
-};
-
-// undocumented libgcc "struct km_object_info" referenced by
-// KEYMGR_GCC3_DW2_OBJ_LIST
-struct libgcc_object_info {
-  libgcc_object   *seen_objects;
-  libgcc_object   *unseen_objects;
-  unsigned         spare[2];
-};
-
-
 // static linker symbols to prevent wrong two level namespace for _Unwind symbols
 #if defined(__arm__)
    #define NOT_HERE_BEFORE_5_0(sym)     \
@@ -140,44 +113,3 @@ NOT_HERE_BEFORE_5_0(_Unwind_SjLj_Resume_or_Rethrow)
 NOT_HERE_BEFORE_5_0(_Unwind_SjLj_Unregister)
 
 #endif // defined(_LIBUNWIND_BUILD_SJLJ_APIS)
-
-
-namespace libunwind {
-
-_LIBUNWIND_HIDDEN
-bool checkKeyMgrRegisteredFDEs(uintptr_t pc, void *&fde) {
-#if __MAC_OS_X_VERSION_MIN_REQUIRED
-  // lastly check for old style keymgr registration of dynamically generated
-  // FDEs acquire exclusive access to libgcc_object_info
-  libgcc_object_info *head = (libgcc_object_info *)
-                _keymgr_get_and_lock_processwide_ptr(KEYMGR_GCC3_DW2_OBJ_LIST);
-  if (head != NULL) {
-    // look at each FDE in keymgr
-    for (libgcc_object *ob = head->unseen_objects; ob != NULL; ob = ob->next) {
-      CFI_Parser<LocalAddressSpace>::FDE_Info fdeInfo;
-      CFI_Parser<LocalAddressSpace>::CIE_Info cieInfo;
-      const char *msg = CFI_Parser<LocalAddressSpace>::decodeFDE(
-                                      LocalAddressSpace::sThisAddressSpace,
-                                      (uintptr_t)ob->fde, &fdeInfo, &cieInfo);
-      if (msg == NULL) {
-        // Check if this FDE is for a function that includes the pc
-        if ((fdeInfo.pcStart <= pc) && (pc < fdeInfo.pcEnd)) {
-          fde = (void*)fdeInfo.pcStart;
-          _keymgr_set_and_unlock_processwide_ptr(KEYMGR_GCC3_DW2_OBJ_LIST,
-                                                 head);
-          return true;
-        }
-      }
-    }
-  }
-  // release libgcc_object_info
-  _keymgr_set_and_unlock_processwide_ptr(KEYMGR_GCC3_DW2_OBJ_LIST, head);
-#else
-  (void)pc;
-  (void)fde;
-#endif
-  return false;
-}
-
-}
-

From 113f56fbb80e8d6f705be19f8ae169a3fee2e4f8 Mon Sep 17 00:00:00 2001
From: Adrian Prantl <aprantl@apple.com>
Date: Sat, 25 Jul 2020 08:27:21 -0700
Subject: [PATCH 0240/1035] Unify the return value of GetByteSize to an
 llvm::Optional<uint64_t> (NFC-ish)

This cleanup patch unifies all methods called GetByteSize() in the
ValueObject hierarchy to return an optional, like the methods in
CompilerType do. This means fewer magic 0 values, which could fix bugs
down the road in languages where types can have a size of zero, such
as Swift and C (but not C++).

Differential Revision: https://reviews.llvm.org/D84285

This re-lands the patch with bogus :m_byte_size(0) initalizations removed.
---
 lldb/include/lldb/Core/ValueObject.h          |  2 +-
 lldb/include/lldb/Core/ValueObjectCast.h      |  2 +-
 lldb/include/lldb/Core/ValueObjectChild.h     |  2 +-
 .../lldb/Core/ValueObjectConstResult.h        |  4 ++--
 .../lldb/Core/ValueObjectDynamicValue.h       |  2 +-
 lldb/include/lldb/Core/ValueObjectMemory.h    |  2 +-
 lldb/include/lldb/Core/ValueObjectRegister.h  |  4 ++--
 .../lldb/Core/ValueObjectSyntheticFilter.h    |  2 +-
 lldb/include/lldb/Core/ValueObjectVariable.h  |  2 +-
 .../lldb/Expression/ExpressionVariable.h      |  2 +-
 .../lldb/Target/StackFrameRecognizer.h        |  4 +++-
 lldb/source/API/SBValue.cpp                   |  2 +-
 .../Commands/CommandObjectWatchpoint.cpp      |  2 +-
 lldb/source/Core/ValueObject.cpp              | 12 +++++-----
 lldb/source/Core/ValueObjectCast.cpp          |  2 +-
 lldb/source/Core/ValueObjectConstResult.cpp   | 21 +++++++----------
 lldb/source/Core/ValueObjectDynamicValue.cpp  |  2 +-
 lldb/source/Core/ValueObjectMemory.cpp        |  8 +++----
 lldb/source/Core/ValueObjectRegister.cpp      |  6 +++--
 .../Core/ValueObjectSyntheticFilter.cpp       |  4 +++-
 lldb/source/Core/ValueObjectVariable.cpp      |  6 ++---
 lldb/source/Expression/ExpressionVariable.cpp |  8 +++----
 lldb/source/Expression/Materializer.cpp       | 23 +++++++++++--------
 lldb/source/Target/StackFrame.cpp             |  6 ++---
 24 files changed, 66 insertions(+), 64 deletions(-)

diff --git a/lldb/include/lldb/Core/ValueObject.h b/lldb/include/lldb/Core/ValueObject.h
index 0080368fd9965..a557d69f3ae30 100644
--- a/lldb/include/lldb/Core/ValueObject.h
+++ b/lldb/include/lldb/Core/ValueObject.h
@@ -358,7 +358,7 @@ class ValueObject : public UserID {
   virtual bool CanProvideValue();
 
   // Subclasses must implement the functions below.
-  virtual uint64_t GetByteSize() = 0;
+  virtual llvm::Optional<uint64_t> GetByteSize() = 0;
 
   virtual lldb::ValueType GetValueType() const = 0;
 
diff --git a/lldb/include/lldb/Core/ValueObjectCast.h b/lldb/include/lldb/Core/ValueObjectCast.h
index d91ca6a92be8d..342803f8ca63a 100644
--- a/lldb/include/lldb/Core/ValueObjectCast.h
+++ b/lldb/include/lldb/Core/ValueObjectCast.h
@@ -30,7 +30,7 @@ class ValueObjectCast : public ValueObject {
                                     ConstString name,
                                     const CompilerType &cast_type);
 
-  uint64_t GetByteSize() override;
+  llvm::Optional<uint64_t> GetByteSize() override;
 
   size_t CalculateNumChildren(uint32_t max) override;
 
diff --git a/lldb/include/lldb/Core/ValueObjectChild.h b/lldb/include/lldb/Core/ValueObjectChild.h
index c6f44a29b0591..9a9fd9294261a 100644
--- a/lldb/include/lldb/Core/ValueObjectChild.h
+++ b/lldb/include/lldb/Core/ValueObjectChild.h
@@ -30,7 +30,7 @@ class ValueObjectChild : public ValueObject {
 public:
   ~ValueObjectChild() override;
 
-  uint64_t GetByteSize() override { return m_byte_size; }
+  llvm::Optional<uint64_t> GetByteSize() override { return m_byte_size; }
 
   lldb::offset_t GetByteOffset() override { return m_byte_offset; }
 
diff --git a/lldb/include/lldb/Core/ValueObjectConstResult.h b/lldb/include/lldb/Core/ValueObjectConstResult.h
index 0e868c687e931..8d823baa0b7b4 100644
--- a/lldb/include/lldb/Core/ValueObjectConstResult.h
+++ b/lldb/include/lldb/Core/ValueObjectConstResult.h
@@ -62,7 +62,7 @@ class ValueObjectConstResult : public ValueObject {
   static lldb::ValueObjectSP Create(ExecutionContextScope *exe_scope,
                                     const Status &error);
 
-  uint64_t GetByteSize() override;
+  llvm::Optional<uint64_t> GetByteSize() override;
 
   lldb::ValueType GetValueType() const override;
 
@@ -113,7 +113,7 @@ class ValueObjectConstResult : public ValueObject {
   CompilerType GetCompilerTypeImpl() override;
 
   ConstString m_type_name;
-  uint64_t m_byte_size;
+  llvm::Optional<uint64_t> m_byte_size;
 
   ValueObjectConstResultImpl m_impl;
 
diff --git a/lldb/include/lldb/Core/ValueObjectDynamicValue.h b/lldb/include/lldb/Core/ValueObjectDynamicValue.h
index 9f5304b55e934..2806857339efb 100644
--- a/lldb/include/lldb/Core/ValueObjectDynamicValue.h
+++ b/lldb/include/lldb/Core/ValueObjectDynamicValue.h
@@ -34,7 +34,7 @@ class ValueObjectDynamicValue : public ValueObject {
 public:
   ~ValueObjectDynamicValue() override;
 
-  uint64_t GetByteSize() override;
+  llvm::Optional<uint64_t> GetByteSize() override;
 
   ConstString GetTypeName() override;
 
diff --git a/lldb/include/lldb/Core/ValueObjectMemory.h b/lldb/include/lldb/Core/ValueObjectMemory.h
index d1cd6ae41445d..b5d5e6ecf4c0e 100644
--- a/lldb/include/lldb/Core/ValueObjectMemory.h
+++ b/lldb/include/lldb/Core/ValueObjectMemory.h
@@ -40,7 +40,7 @@ class ValueObjectMemory : public ValueObject {
                                     const Address &address,
                                     const CompilerType &ast_type);
 
-  uint64_t GetByteSize() override;
+  llvm::Optional<uint64_t> GetByteSize() override;
 
   ConstString GetTypeName() override;
 
diff --git a/lldb/include/lldb/Core/ValueObjectRegister.h b/lldb/include/lldb/Core/ValueObjectRegister.h
index 41051d93b707e..3968584ad5185 100644
--- a/lldb/include/lldb/Core/ValueObjectRegister.h
+++ b/lldb/include/lldb/Core/ValueObjectRegister.h
@@ -36,7 +36,7 @@ class ValueObjectRegisterSet : public ValueObject {
                                     lldb::RegisterContextSP &reg_ctx_sp,
                                     uint32_t set_idx);
 
-  uint64_t GetByteSize() override;
+  llvm::Optional<uint64_t> GetByteSize() override;
 
   lldb::ValueType GetValueType() const override {
     return lldb::eValueTypeRegisterSet;
@@ -86,7 +86,7 @@ class ValueObjectRegister : public ValueObject {
                                     lldb::RegisterContextSP &reg_ctx_sp,
                                     uint32_t reg_num);
 
-  uint64_t GetByteSize() override;
+  llvm::Optional<uint64_t> GetByteSize() override;
 
   lldb::ValueType GetValueType() const override {
     return lldb::eValueTypeRegister;
diff --git a/lldb/include/lldb/Core/ValueObjectSyntheticFilter.h b/lldb/include/lldb/Core/ValueObjectSyntheticFilter.h
index cb471657aec9b..41c461ce13f0d 100644
--- a/lldb/include/lldb/Core/ValueObjectSyntheticFilter.h
+++ b/lldb/include/lldb/Core/ValueObjectSyntheticFilter.h
@@ -36,7 +36,7 @@ class ValueObjectSynthetic : public ValueObject {
 public:
   ~ValueObjectSynthetic() override;
 
-  uint64_t GetByteSize() override;
+  llvm::Optional<uint64_t> GetByteSize() override;
 
   ConstString GetTypeName() override;
 
diff --git a/lldb/include/lldb/Core/ValueObjectVariable.h b/lldb/include/lldb/Core/ValueObjectVariable.h
index b7e262574a14d..23fdedbf5a4a6 100644
--- a/lldb/include/lldb/Core/ValueObjectVariable.h
+++ b/lldb/include/lldb/Core/ValueObjectVariable.h
@@ -37,7 +37,7 @@ class ValueObjectVariable : public ValueObject {
   static lldb::ValueObjectSP Create(ExecutionContextScope *exe_scope,
                                     const lldb::VariableSP &var_sp);
 
-  uint64_t GetByteSize() override;
+  llvm::Optional<uint64_t> GetByteSize() override;
 
   ConstString GetTypeName() override;
 
diff --git a/lldb/include/lldb/Expression/ExpressionVariable.h b/lldb/include/lldb/Expression/ExpressionVariable.h
index 60062d212badf..4259e6395da47 100644
--- a/lldb/include/lldb/Expression/ExpressionVariable.h
+++ b/lldb/include/lldb/Expression/ExpressionVariable.h
@@ -32,7 +32,7 @@ class ExpressionVariable
 
   virtual ~ExpressionVariable();
 
-  size_t GetByteSize() { return m_frozen_sp->GetByteSize(); }
+  llvm::Optional<uint64_t> GetByteSize() { return m_frozen_sp->GetByteSize(); }
 
   ConstString GetName() { return m_frozen_sp->GetName(); }
 
diff --git a/lldb/include/lldb/Target/StackFrameRecognizer.h b/lldb/include/lldb/Target/StackFrameRecognizer.h
index 302b56bec907b..baffc890bb065 100644
--- a/lldb/include/lldb/Target/StackFrameRecognizer.h
+++ b/lldb/include/lldb/Target/StackFrameRecognizer.h
@@ -154,7 +154,9 @@ class ValueObjectRecognizerSynthesizedValue : public ValueObject {
     SetName(parent.GetName());
   }
 
-  uint64_t GetByteSize() override { return m_parent->GetByteSize(); }
+  llvm::Optional<uint64_t> GetByteSize() override {
+    return m_parent->GetByteSize();
+  }
   lldb::ValueType GetValueType() const override { return m_type; }
   bool UpdateValue() override {
     if (!m_parent->UpdateValueIfNeeded()) return false;
diff --git a/lldb/source/API/SBValue.cpp b/lldb/source/API/SBValue.cpp
index 7485b0ee1838e..686d1f23a75a8 100644
--- a/lldb/source/API/SBValue.cpp
+++ b/lldb/source/API/SBValue.cpp
@@ -333,7 +333,7 @@ size_t SBValue::GetByteSize() {
   ValueLocker locker;
   lldb::ValueObjectSP value_sp(GetSP(locker));
   if (value_sp) {
-    result = value_sp->GetByteSize();
+    result = value_sp->GetByteSize().getValueOr(0);
   }
 
   return result;
diff --git a/lldb/source/Commands/CommandObjectWatchpoint.cpp b/lldb/source/Commands/CommandObjectWatchpoint.cpp
index ce4662930a7c2..c2a008af79d6f 100644
--- a/lldb/source/Commands/CommandObjectWatchpoint.cpp
+++ b/lldb/source/Commands/CommandObjectWatchpoint.cpp
@@ -905,7 +905,7 @@ corresponding to the byte size of the data type.");
         // We're in business.
         // Find out the size of this variable.
         size = m_option_watchpoint.watch_size == 0
-                   ? valobj_sp->GetByteSize()
+                   ? valobj_sp->GetByteSize().getValueOr(0)
                    : m_option_watchpoint.watch_size;
       }
       compiler_type = valobj_sp->GetCompilerType();
diff --git a/lldb/source/Core/ValueObject.cpp b/lldb/source/Core/ValueObject.cpp
index d3a1971235ca0..78711c4e42ce2 100644
--- a/lldb/source/Core/ValueObject.cpp
+++ b/lldb/source/Core/ValueObject.cpp
@@ -849,7 +849,7 @@ bool ValueObject::SetData(DataExtractor &data, Status &error) {
   uint64_t count = 0;
   const Encoding encoding = GetCompilerType().GetEncoding(count);
 
-  const size_t byte_size = GetByteSize();
+  const size_t byte_size = GetByteSize().getValueOr(0);
 
   Value::ValueType value_type = m_value.GetValueType();
 
@@ -1524,7 +1524,7 @@ bool ValueObject::SetValueFromCString(const char *value_str, Status &error) {
   uint64_t count = 0;
   const Encoding encoding = GetCompilerType().GetEncoding(count);
 
-  const size_t byte_size = GetByteSize();
+  const size_t byte_size = GetByteSize().getValueOr(0);
 
   Value::ValueType value_type = m_value.GetValueType();
 
@@ -1739,13 +1739,13 @@ ValueObjectSP ValueObject::GetSyntheticBitFieldChild(uint32_t from, uint32_t to,
       uint32_t bit_field_offset = from;
       if (GetDataExtractor().GetByteOrder() == eByteOrderBig)
         bit_field_offset =
-            GetByteSize() * 8 - bit_field_size - bit_field_offset;
+            GetByteSize().getValueOr(0) * 8 - bit_field_size - bit_field_offset;
       // We haven't made a synthetic array member for INDEX yet, so lets make
       // one and cache it for any future reference.
       ValueObjectChild *synthetic_child = new ValueObjectChild(
-          *this, GetCompilerType(), index_const_str, GetByteSize(), 0,
-          bit_field_size, bit_field_offset, false, false, eAddressTypeInvalid,
-          0);
+          *this, GetCompilerType(), index_const_str,
+          GetByteSize().getValueOr(0), 0, bit_field_size, bit_field_offset,
+          false, false, eAddressTypeInvalid, 0);
 
       // Cache the value if we got one back...
       if (synthetic_child) {
diff --git a/lldb/source/Core/ValueObjectCast.cpp b/lldb/source/Core/ValueObjectCast.cpp
index 22e856be539b5..7b6d3591faf44 100644
--- a/lldb/source/Core/ValueObjectCast.cpp
+++ b/lldb/source/Core/ValueObjectCast.cpp
@@ -47,7 +47,7 @@ size_t ValueObjectCast::CalculateNumChildren(uint32_t max) {
   return children_count <= max ? children_count : max;
 }
 
-uint64_t ValueObjectCast::GetByteSize() {
+llvm::Optional<uint64_t> ValueObjectCast::GetByteSize() {
   ExecutionContext exe_ctx(GetExecutionContextRef());
   return m_value.GetValueByteSize(nullptr, &exe_ctx);
 }
diff --git a/lldb/source/Core/ValueObjectConstResult.cpp b/lldb/source/Core/ValueObjectConstResult.cpp
index 8d84f8e62ccc5..ceb4491f86663 100644
--- a/lldb/source/Core/ValueObjectConstResult.cpp
+++ b/lldb/source/Core/ValueObjectConstResult.cpp
@@ -40,8 +40,7 @@ ValueObjectConstResult::ValueObjectConstResult(ExecutionContextScope *exe_scope,
                                                ByteOrder byte_order,
                                                uint32_t addr_byte_size,
                                                lldb::addr_t address)
-    : ValueObject(exe_scope, manager), m_type_name(), m_byte_size(0),
-      m_impl(this, address) {
+    : ValueObject(exe_scope, manager), m_impl(this, address) {
   SetIsConstant();
   SetValueIsValid(true);
   m_data.SetByteOrder(byte_order);
@@ -64,8 +63,7 @@ ValueObjectConstResult::ValueObjectConstResult(
     ExecutionContextScope *exe_scope, ValueObjectManager &manager,
     const CompilerType &compiler_type, ConstString name,
     const DataExtractor &data, lldb::addr_t address)
-    : ValueObject(exe_scope, manager), m_type_name(), m_byte_size(0),
-      m_impl(this, address) {
+    : ValueObject(exe_scope, manager), m_impl(this, address) {
   m_data = data;
 
   if (!m_data.GetSharedDataBuffer()) {
@@ -112,8 +110,7 @@ ValueObjectConstResult::ValueObjectConstResult(
     const CompilerType &compiler_type, ConstString name,
     const lldb::DataBufferSP &data_sp, lldb::ByteOrder data_byte_order,
     uint32_t data_addr_size, lldb::addr_t address)
-    : ValueObject(exe_scope, manager), m_type_name(), m_byte_size(0),
-      m_impl(this, address) {
+    : ValueObject(exe_scope, manager), m_impl(this, address) {
   m_data.SetByteOrder(data_byte_order);
   m_data.SetAddressByteSize(data_addr_size);
   m_data.SetData(data_sp);
@@ -143,7 +140,7 @@ ValueObjectConstResult::ValueObjectConstResult(
     ExecutionContextScope *exe_scope, ValueObjectManager &manager,
     const CompilerType &compiler_type, ConstString name, lldb::addr_t address,
     AddressType address_type, uint32_t addr_byte_size)
-    : ValueObject(exe_scope, manager), m_type_name(), m_byte_size(0),
+    : ValueObject(exe_scope, manager), m_type_name(),
       m_impl(this, address) {
   m_value.GetScalar() = address;
   m_data.SetAddressByteSize(addr_byte_size);
@@ -179,8 +176,7 @@ ValueObjectSP ValueObjectConstResult::Create(ExecutionContextScope *exe_scope,
 ValueObjectConstResult::ValueObjectConstResult(ExecutionContextScope *exe_scope,
                                                ValueObjectManager &manager,
                                                const Status &error)
-    : ValueObject(exe_scope, manager), m_type_name(), m_byte_size(0),
-      m_impl(this) {
+    : ValueObject(exe_scope, manager), m_impl(this) {
   m_error = error;
   SetIsConstant();
 }
@@ -189,8 +185,7 @@ ValueObjectConstResult::ValueObjectConstResult(ExecutionContextScope *exe_scope,
                                                ValueObjectManager &manager,
                                                const Value &value,
                                                ConstString name, Module *module)
-    : ValueObject(exe_scope, manager), m_type_name(), m_byte_size(0),
-      m_impl(this) {
+    : ValueObject(exe_scope, manager), m_impl(this) {
   m_value = value;
   m_name = name;
   ExecutionContext exe_ctx;
@@ -208,9 +203,9 @@ lldb::ValueType ValueObjectConstResult::GetValueType() const {
   return eValueTypeConstResult;
 }
 
-uint64_t ValueObjectConstResult::GetByteSize() {
+llvm::Optional<uint64_t> ValueObjectConstResult::GetByteSize() {
   ExecutionContext exe_ctx(GetExecutionContextRef());
-  if (m_byte_size == 0) {
+  if (!m_byte_size) {
     if (auto size =
         GetCompilerType().GetByteSize(exe_ctx.GetBestExecutionContextScope()))
       SetByteSize(*size);
diff --git a/lldb/source/Core/ValueObjectDynamicValue.cpp b/lldb/source/Core/ValueObjectDynamicValue.cpp
index ca66740cb55d4..1c25b8c85a059 100644
--- a/lldb/source/Core/ValueObjectDynamicValue.cpp
+++ b/lldb/source/Core/ValueObjectDynamicValue.cpp
@@ -98,7 +98,7 @@ size_t ValueObjectDynamicValue::CalculateNumChildren(uint32_t max) {
     return m_parent->GetNumChildren(max);
 }
 
-uint64_t ValueObjectDynamicValue::GetByteSize() {
+llvm::Optional<uint64_t> ValueObjectDynamicValue::GetByteSize() {
   const bool success = UpdateValueIfNeeded(false);
   if (success && m_dynamic_type_info.HasType()) {
     ExecutionContext exe_ctx(GetExecutionContextRef());
diff --git a/lldb/source/Core/ValueObjectMemory.cpp b/lldb/source/Core/ValueObjectMemory.cpp
index 8e7d3ebc93f69..17fade9e5fdc3 100644
--- a/lldb/source/Core/ValueObjectMemory.cpp
+++ b/lldb/source/Core/ValueObjectMemory.cpp
@@ -139,13 +139,11 @@ size_t ValueObjectMemory::CalculateNumChildren(uint32_t max) {
   return child_count <= max ? child_count : max;
 }
 
-uint64_t ValueObjectMemory::GetByteSize() {
+llvm::Optional<uint64_t> ValueObjectMemory::GetByteSize() {
   ExecutionContext exe_ctx(GetExecutionContextRef());
   if (m_type_sp)
-    return m_type_sp->GetByteSize(exe_ctx.GetBestExecutionContextScope())
-        .getValueOr(0);
-  return m_compiler_type.GetByteSize(exe_ctx.GetBestExecutionContextScope())
-      .getValueOr(0);
+    return m_type_sp->GetByteSize(exe_ctx.GetBestExecutionContextScope());
+  return m_compiler_type.GetByteSize(exe_ctx.GetBestExecutionContextScope());
 }
 
 lldb::ValueType ValueObjectMemory::GetValueType() const {
diff --git a/lldb/source/Core/ValueObjectRegister.cpp b/lldb/source/Core/ValueObjectRegister.cpp
index ec87c38fb3679..27461e9cebc41 100644
--- a/lldb/source/Core/ValueObjectRegister.cpp
+++ b/lldb/source/Core/ValueObjectRegister.cpp
@@ -81,7 +81,7 @@ size_t ValueObjectRegisterSet::CalculateNumChildren(uint32_t max) {
   return 0;
 }
 
-uint64_t ValueObjectRegisterSet::GetByteSize() { return 0; }
+llvm::Optional<uint64_t> ValueObjectRegisterSet::GetByteSize() { return 0; }
 
 bool ValueObjectRegisterSet::UpdateValue() {
   m_error.Clear();
@@ -229,7 +229,9 @@ size_t ValueObjectRegister::CalculateNumChildren(uint32_t max) {
   return children_count <= max ? children_count : max;
 }
 
-uint64_t ValueObjectRegister::GetByteSize() { return m_reg_info.byte_size; }
+llvm::Optional<uint64_t> ValueObjectRegister::GetByteSize() {
+  return m_reg_info.byte_size;
+}
 
 bool ValueObjectRegister::UpdateValue() {
   m_error.Clear();
diff --git a/lldb/source/Core/ValueObjectSyntheticFilter.cpp b/lldb/source/Core/ValueObjectSyntheticFilter.cpp
index 32d1e6ab8368c..fb2d32e602cea 100644
--- a/lldb/source/Core/ValueObjectSyntheticFilter.cpp
+++ b/lldb/source/Core/ValueObjectSyntheticFilter.cpp
@@ -121,7 +121,9 @@ bool ValueObjectSynthetic::MightHaveChildren() {
   return (m_might_have_children != eLazyBoolNo);
 }
 
-uint64_t ValueObjectSynthetic::GetByteSize() { return m_parent->GetByteSize(); }
+llvm::Optional<uint64_t> ValueObjectSynthetic::GetByteSize() {
+  return m_parent->GetByteSize();
+}
 
 lldb::ValueType ValueObjectSynthetic::GetValueType() const {
   return m_parent->GetValueType();
diff --git a/lldb/source/Core/ValueObjectVariable.cpp b/lldb/source/Core/ValueObjectVariable.cpp
index 0d1e7b047a0ac..ab67e3038cf0a 100644
--- a/lldb/source/Core/ValueObjectVariable.cpp
+++ b/lldb/source/Core/ValueObjectVariable.cpp
@@ -105,15 +105,15 @@ size_t ValueObjectVariable::CalculateNumChildren(uint32_t max) {
   return child_count <= max ? child_count : max;
 }
 
-uint64_t ValueObjectVariable::GetByteSize() {
+llvm::Optional<uint64_t> ValueObjectVariable::GetByteSize() {
   ExecutionContext exe_ctx(GetExecutionContextRef());
 
   CompilerType type(GetCompilerType());
 
   if (!type.IsValid())
-    return 0;
+    return {};
 
-  return type.GetByteSize(exe_ctx.GetBestExecutionContextScope()).getValueOr(0);
+  return type.GetByteSize(exe_ctx.GetBestExecutionContextScope());
 }
 
 lldb::ValueType ValueObjectVariable::GetValueType() const {
diff --git a/lldb/source/Expression/ExpressionVariable.cpp b/lldb/source/Expression/ExpressionVariable.cpp
index d95f0745cf4ba..8b3dda7b2fe10 100644
--- a/lldb/source/Expression/ExpressionVariable.cpp
+++ b/lldb/source/Expression/ExpressionVariable.cpp
@@ -16,10 +16,10 @@ using namespace lldb_private;
 ExpressionVariable::~ExpressionVariable() {}
 
 uint8_t *ExpressionVariable::GetValueBytes() {
-  const size_t byte_size = m_frozen_sp->GetByteSize();
-  if (byte_size > 0) {
-    if (m_frozen_sp->GetDataExtractor().GetByteSize() < byte_size) {
-      m_frozen_sp->GetValue().ResizeData(byte_size);
+  llvm::Optional<uint64_t> byte_size = m_frozen_sp->GetByteSize();
+  if (byte_size && *byte_size) {
+    if (m_frozen_sp->GetDataExtractor().GetByteSize() < *byte_size) {
+      m_frozen_sp->GetValue().ResizeData(*byte_size);
       m_frozen_sp->GetValue().GetData(m_frozen_sp->GetDataExtractor());
     }
     return const_cast<uint8_t *>(
diff --git a/lldb/source/Expression/Materializer.cpp b/lldb/source/Expression/Materializer.cpp
index 6f8d9b154570a..327e15a26266f 100644
--- a/lldb/source/Expression/Materializer.cpp
+++ b/lldb/source/Expression/Materializer.cpp
@@ -67,7 +67,7 @@ class EntityPersistentVariable : public Materializer::Entity {
     const bool zero_memory = false;
 
     lldb::addr_t mem = map.Malloc(
-        m_persistent_variable_sp->GetByteSize(), 8,
+        m_persistent_variable_sp->GetByteSize().getValueOr(0), 8,
         lldb::ePermissionsReadable | lldb::ePermissionsWritable,
         IRMemoryMap::eAllocationPolicyMirror, zero_memory, allocate_error);
 
@@ -106,7 +106,8 @@ class EntityPersistentVariable : public Materializer::Entity {
     Status write_error;
 
     map.WriteMemory(mem, m_persistent_variable_sp->GetValueBytes(),
-                    m_persistent_variable_sp->GetByteSize(), write_error);
+                    m_persistent_variable_sp->GetByteSize().getValueOr(0),
+                    write_error);
 
     if (!write_error.Success()) {
       err.SetErrorStringWithFormat(
@@ -234,7 +235,7 @@ class EntityPersistentVariable : public Materializer::Entity {
             map.GetBestExecutionContextScope(),
             m_persistent_variable_sp.get()->GetCompilerType(),
             m_persistent_variable_sp->GetName(), location, eAddressTypeLoad,
-            m_persistent_variable_sp->GetByteSize());
+            m_persistent_variable_sp->GetByteSize().getValueOr(0));
 
         if (frame_top != LLDB_INVALID_ADDRESS &&
             frame_bottom != LLDB_INVALID_ADDRESS && location >= frame_bottom &&
@@ -279,7 +280,8 @@ class EntityPersistentVariable : public Materializer::Entity {
         LLDB_LOGF(log, "Dematerializing %s from 0x%" PRIx64 " (size = %llu)",
                   m_persistent_variable_sp->GetName().GetCString(),
                   (uint64_t)mem,
-                  (unsigned long long)m_persistent_variable_sp->GetByteSize());
+                  (unsigned long long)m_persistent_variable_sp->GetByteSize()
+                      .getValueOr(0));
 
         // Read the contents of the spare memory area
 
@@ -288,7 +290,7 @@ class EntityPersistentVariable : public Materializer::Entity {
         Status read_error;
 
         map.ReadMemory(m_persistent_variable_sp->GetValueBytes(), mem,
-                       m_persistent_variable_sp->GetByteSize(), read_error);
+                       m_persistent_variable_sp->GetByteSize().getValueOr(0), read_error);
 
         if (!read_error.Success()) {
           err.SetErrorStringWithFormat(
@@ -369,10 +371,11 @@ class EntityPersistentVariable : public Materializer::Entity {
       if (!err.Success()) {
         dump_stream.Printf("  <could not be read>\n");
       } else {
-        DataBufferHeap data(m_persistent_variable_sp->GetByteSize(), 0);
+        DataBufferHeap data(
+            m_persistent_variable_sp->GetByteSize().getValueOr(0), 0);
 
         map.ReadMemory(data.GetBytes(), target_address,
-                       m_persistent_variable_sp->GetByteSize(), err);
+                       m_persistent_variable_sp->GetByteSize().getValueOr(0), err);
 
         if (!err.Success()) {
           dump_stream.Printf("  <could not be read>\n");
@@ -621,8 +624,8 @@ class EntityVariable : public Materializer::Entity {
 
       Status extract_error;
 
-      map.GetMemoryData(data, m_temporary_allocation, valobj_sp->GetByteSize(),
-                        extract_error);
+      map.GetMemoryData(data, m_temporary_allocation,
+                        valobj_sp->GetByteSize().getValueOr(0), extract_error);
 
       if (!extract_error.Success()) {
         err.SetErrorStringWithFormat("couldn't get the data for variable %s",
@@ -919,7 +922,7 @@ class EntityResultVariable : public Materializer::Entity {
 
     ret->ValueUpdated();
 
-    const size_t pvar_byte_size = ret->GetByteSize();
+    const size_t pvar_byte_size = ret->GetByteSize().getValueOr(0);
     uint8_t *pvar_data = ret->GetValueBytes();
 
     map.ReadMemory(pvar_data, address, pvar_byte_size, read_error);
diff --git a/lldb/source/Target/StackFrame.cpp b/lldb/source/Target/StackFrame.cpp
index 098aed9cd8125..22bca52d7f98a 100644
--- a/lldb/source/Target/StackFrame.cpp
+++ b/lldb/source/Target/StackFrame.cpp
@@ -1408,7 +1408,7 @@ ValueObjectSP GetValueForOffset(StackFrame &frame, ValueObjectSP &parent,
     }
 
     int64_t child_offset = child_sp->GetByteOffset();
-    int64_t child_size = child_sp->GetByteSize();
+    int64_t child_size = child_sp->GetByteSize().getValueOr(0);
 
     if (offset >= child_offset && offset < (child_offset + child_size)) {
       return GetValueForOffset(frame, child_sp, offset - child_offset);
@@ -1441,8 +1441,8 @@ ValueObjectSP GetValueForDereferincingOffset(StackFrame &frame,
   }
 
   if (offset >= 0 && uint64_t(offset) >= pointee->GetByteSize()) {
-    int64_t index = offset / pointee->GetByteSize();
-    offset = offset % pointee->GetByteSize();
+    int64_t index = offset / pointee->GetByteSize().getValueOr(1);
+    offset = offset % pointee->GetByteSize().getValueOr(1);
     const bool can_create = true;
     pointee = base->GetSyntheticArrayMember(index, can_create);
   }

From 351d234d866ea441d8f8ad1e9b483a98fd51da19 Mon Sep 17 00:00:00 2001
From: Roman Lebedev <lebedev.ri@gmail.com>
Date: Mon, 27 Jul 2020 23:35:51 +0300
Subject: [PATCH 0241/1035] [OpenMPOpt] Most SCC's are uninteresting, don't
 waste time on them (up to 16x faster)

Summary:
This seems obvious in hindsight, but the result is surprising.
I've measured compile-time of `-openmpopt` pass standalone
on RawSpeed unity build, and while there is some OpenMP stuff,
most is not OpenMP. But nonetheless the pass does a lot of costly
preparations before ever trying to look for OpenMP stuff in SCC.

Numbers (n=25): 0.094624s  ->  0.005976s, an -93.68% improvement, or ~16x

Reviewers: jdoerfert

Reviewed By: jdoerfert

Subscribers: yaxunl, hiraditya, guansong, llvm-commits, sstefan1

Tags: #llvm

Differential Revision: https://reviews.llvm.org/D84689
---
 llvm/include/llvm/Transforms/IPO/OpenMPOpt.h | 10 ++++
 llvm/lib/Transforms/IPO/OpenMPOpt.cpp        | 48 ++++++++++++++++----
 2 files changed, 48 insertions(+), 10 deletions(-)

diff --git a/llvm/include/llvm/Transforms/IPO/OpenMPOpt.h b/llvm/include/llvm/Transforms/IPO/OpenMPOpt.h
index d96187b73f9bb..9b72ee0afd284 100644
--- a/llvm/include/llvm/Transforms/IPO/OpenMPOpt.h
+++ b/llvm/include/llvm/Transforms/IPO/OpenMPOpt.h
@@ -33,6 +33,11 @@ struct OpenMPInModule {
   bool isKnown() { return Value != OpenMP::UNKNOWN; }
   operator bool() { return Value != OpenMP::NOT_FOUND; }
 
+  /// Does this function \p F contain any OpenMP runtime calls?
+  bool containsOMPRuntimeCalls(Function *F) const {
+    return FuncsWithOMPRuntimeCalls.contains(F);
+  }
+
   /// Return the known kernels (=GPU entry points) in the module.
   SmallPtrSetImpl<Kernel> &getKernels() { return Kernels; }
 
@@ -42,6 +47,11 @@ struct OpenMPInModule {
 private:
   enum class OpenMP { FOUND, NOT_FOUND, UNKNOWN } Value = OpenMP::UNKNOWN;
 
+  friend bool containsOpenMP(Module &M, OpenMPInModule &OMPInModule);
+
+  /// In which functions are OpenMP runtime calls present?
+  SmallPtrSet<Function *, 32> FuncsWithOMPRuntimeCalls;
+
   /// Collection of known kernels (=GPU entry points) in the module.
   SmallPtrSet<Kernel, 8> Kernels;
 };
diff --git a/llvm/lib/Transforms/IPO/OpenMPOpt.cpp b/llvm/lib/Transforms/IPO/OpenMPOpt.cpp
index f664a24173747..93f1e5392eb2c 100644
--- a/llvm/lib/Transforms/IPO/OpenMPOpt.cpp
+++ b/llvm/lib/Transforms/IPO/OpenMPOpt.cpp
@@ -1339,10 +1339,21 @@ PreservedAnalyses OpenMPOptPass::run(LazyCallGraph::SCC &C,
     return PreservedAnalyses::all();
 
   SmallVector<Function *, 16> SCC;
-  for (LazyCallGraph::Node &N : C)
-    SCC.push_back(&N.getFunction());
+  // If there are kernels in the module, we have to run on all SCC's.
+  bool SCCIsInteresting = !OMPInModule.getKernels().empty();
+  for (LazyCallGraph::Node &N : C) {
+    Function *Fn = &N.getFunction();
+    SCC.push_back(Fn);
+
+    // Do we already know that the SCC contains kernels,
+    // or that OpenMP functions are called from this SCC?
+    if (SCCIsInteresting)
+      continue;
+    // If not, let's check that.
+    SCCIsInteresting |= OMPInModule.containsOMPRuntimeCalls(Fn);
+  }
 
-  if (SCC.empty())
+  if (!SCCIsInteresting || SCC.empty())
     return PreservedAnalyses::all();
 
   FunctionAnalysisManager &FAM =
@@ -1401,12 +1412,23 @@ struct OpenMPOptLegacyPass : public CallGraphSCCPass {
       return false;
 
     SmallVector<Function *, 16> SCC;
-    for (CallGraphNode *CGN : CGSCC)
-      if (Function *Fn = CGN->getFunction())
-        if (!Fn->isDeclaration())
-          SCC.push_back(Fn);
+    // If there are kernels in the module, we have to run on all SCC's.
+    bool SCCIsInteresting = !OMPInModule.getKernels().empty();
+    for (CallGraphNode *CGN : CGSCC) {
+      Function *Fn = CGN->getFunction();
+      if (!Fn || Fn->isDeclaration())
+        continue;
+      SCC.push_back(Fn);
 
-    if (SCC.empty())
+      // Do we already know that the SCC contains kernels,
+      // or that OpenMP functions are called from this SCC?
+      if (SCCIsInteresting)
+        continue;
+      // If not, let's check that.
+      SCCIsInteresting |= OMPInModule.containsOMPRuntimeCalls(Fn);
+    }
+
+    if (!SCCIsInteresting || SCC.empty())
       return false;
 
     CallGraph &CG = getAnalysis<CallGraphWrapperPass>().getCallGraph();
@@ -1468,13 +1490,19 @@ bool llvm::omp::containsOpenMP(Module &M, OpenMPInModule &OMPInModule) {
   if (OMPInModule.isKnown())
     return OMPInModule;
 
+  auto RecordFunctionsContainingUsesOf = [&](Function *F) {
+    for (User *U : F->users())
+      if (auto *I = dyn_cast<Instruction>(U))
+        OMPInModule.FuncsWithOMPRuntimeCalls.insert(I->getFunction());
+  };
+
   // MSVC doesn't like long if-else chains for some reason and instead just
   // issues an error. Work around it..
   do {
 #define OMP_RTL(_Enum, _Name, ...)                                             \
-  if (M.getFunction(_Name)) {                                                  \
+  if (Function *F = M.getFunction(_Name)) {                                    \
+    RecordFunctionsContainingUsesOf(F);                                        \
     OMPInModule = true;                                                        \
-    break;                                                                     \
   }
 #include "llvm/Frontend/OpenMP/OMPKinds.def"
   } while (false);

From beb7e3bb702f69be4c81cf344528577806ab7c5b Mon Sep 17 00:00:00 2001
From: Arthur Eubanks <aeubanks@google.com>
Date: Mon, 27 Jul 2020 13:42:13 -0700
Subject: [PATCH 0242/1035] Rename t2-reduce-size -> thumb2-reduce-size

For readability and consistency with other thumb2 passes like
"thumb2-it".

Reviewed By: arsenm

Differential Revision: https://reviews.llvm.org/D84696
---
 llvm/lib/Target/ARM/Thumb2SizeReduction.cpp  | 2 +-
 llvm/test/CodeGen/Thumb2/t2-teq-reduce.mir   | 2 +-
 llvm/test/CodeGen/Thumb2/t2sizereduction.mir | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/llvm/lib/Target/ARM/Thumb2SizeReduction.cpp b/llvm/lib/Target/ARM/Thumb2SizeReduction.cpp
index ae661594bdc93..0f7e190386731 100644
--- a/llvm/lib/Target/ARM/Thumb2SizeReduction.cpp
+++ b/llvm/lib/Target/ARM/Thumb2SizeReduction.cpp
@@ -43,7 +43,7 @@
 
 using namespace llvm;
 
-#define DEBUG_TYPE "t2-reduce-size"
+#define DEBUG_TYPE "thumb2-reduce-size"
 #define THUMB2_SIZE_REDUCE_NAME "Thumb2 instruction size reduce pass"
 
 STATISTIC(NumNarrows,  "Number of 32-bit instrs reduced to 16-bit ones");
diff --git a/llvm/test/CodeGen/Thumb2/t2-teq-reduce.mir b/llvm/test/CodeGen/Thumb2/t2-teq-reduce.mir
index d0bcc666dfacb..1729a59819217 100644
--- a/llvm/test/CodeGen/Thumb2/t2-teq-reduce.mir
+++ b/llvm/test/CodeGen/Thumb2/t2-teq-reduce.mir
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -run-pass=t2-reduce-size %s -o - | FileCheck %s
+# RUN: llc -run-pass=thumb2-reduce-size %s -o - | FileCheck %s
 
 --- |
   target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
diff --git a/llvm/test/CodeGen/Thumb2/t2sizereduction.mir b/llvm/test/CodeGen/Thumb2/t2sizereduction.mir
index 166914d5dedde..48b75ed5e3465 100644
--- a/llvm/test/CodeGen/Thumb2/t2sizereduction.mir
+++ b/llvm/test/CodeGen/Thumb2/t2sizereduction.mir
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -run-pass=t2-reduce-size %s -o - | FileCheck %s
+# RUN: llc -run-pass=thumb2-reduce-size %s -o - | FileCheck %s
 
 --- |
   target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"

From a566e616202b2db8f3aa66c97e2c38f5e734adfb Mon Sep 17 00:00:00 2001
From: Sergej Jaskiewicz <jaskiewiczs@icloud.com>
Date: Tue, 14 Jul 2020 11:37:27 +0300
Subject: [PATCH 0243/1035] Reland "[compiler-rt] [test] Allow expanding lit
 substitutions recursively"

The commit 8372d505082aceb38417e0b561cd32f2e227597b has been reverted
(eafeb8af34946306a7382fa3801cf6e39a1c7226) because it broke asan
tests on green dragon buildbots.

The underlying issue has been fixed in 4dd5c2bee366514cbc3fc4e6da46462bc11a0a3d.
---
 compiler-rt/test/lit.common.cfg.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/compiler-rt/test/lit.common.cfg.py b/compiler-rt/test/lit.common.cfg.py
index fdc28a4637ff7..d773f3766fd33 100644
--- a/compiler-rt/test/lit.common.cfg.py
+++ b/compiler-rt/test/lit.common.cfg.py
@@ -23,6 +23,9 @@
     # bash on Windows is usually very slow.
     execute_external = (not sys.platform in ['win32'])
 
+# Allow expanding substitutions that are based on other substitutions
+config.recursiveExpansionLimit = 10
+
 # Setup test format.
 config.test_format = lit.formats.ShTest(execute_external)
 if execute_external:

From 754deffd11c733d709c3ed66d3b9a6b54d081474 Mon Sep 17 00:00:00 2001
From: Guillaume Chatelet <gchatelet@google.com>
Date: Mon, 27 Jul 2020 20:49:01 +0000
Subject: [PATCH 0244/1035] [NFC] Move BitcodeCommon.h from Bitstream to
 Bitcode

---
 llvm/include/llvm/{Bitstream => Bitcode}/BitcodeCommon.h | 6 +++---
 llvm/lib/Bitcode/Reader/BitcodeReader.cpp                | 2 +-
 llvm/lib/Bitcode/Writer/BitcodeWriter.cpp                | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)
 rename llvm/include/llvm/{Bitstream => Bitcode}/BitcodeCommon.h (88%)

diff --git a/llvm/include/llvm/Bitstream/BitcodeCommon.h b/llvm/include/llvm/Bitcode/BitcodeCommon.h
similarity index 88%
rename from llvm/include/llvm/Bitstream/BitcodeCommon.h
rename to llvm/include/llvm/Bitcode/BitcodeCommon.h
index 84b35987c4a90..6a3e74550bc48 100644
--- a/llvm/include/llvm/Bitstream/BitcodeCommon.h
+++ b/llvm/include/llvm/Bitcode/BitcodeCommon.h
@@ -11,8 +11,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_BITSTREAM_BITCODECOMMON_H
-#define LLVM_BITSTREAM_BITCODECOMMON_H
+#ifndef LLVM_BITCODE_BITCODECOMMON_H
+#define LLVM_BITCODE_BITCODECOMMON_H
 
 #include "llvm/ADT/Bitfields.h"
 
@@ -27,4 +27,4 @@ struct AllocaPackedValues {
 
 } // namespace llvm
 
-#endif // LLVM_BITSTREAM_BITCODECOMMON_H
+#endif // LLVM_BITCODE_BITCODECOMMON_H
diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index f0377df8648a4..82b6f2078695f 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -20,8 +20,8 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/Triple.h"
 #include "llvm/ADT/Twine.h"
+#include "llvm/Bitcode/BitcodeCommon.h"
 #include "llvm/Bitcode/LLVMBitCodes.h"
-#include "llvm/Bitstream/BitcodeCommon.h"
 #include "llvm/Bitstream/BitstreamReader.h"
 #include "llvm/Config/llvm-config.h"
 #include "llvm/IR/Argument.h"
diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
index f566ddf0864c6..7afef397e05ed 100644
--- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -24,10 +24,10 @@
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/Triple.h"
+#include "llvm/Bitcode/BitcodeCommon.h"
 #include "llvm/Bitcode/BitcodeReader.h"
 #include "llvm/Bitcode/LLVMBitCodes.h"
 #include "llvm/Bitstream/BitCodes.h"
-#include "llvm/Bitstream/BitcodeCommon.h"
 #include "llvm/Bitstream/BitstreamWriter.h"
 #include "llvm/Config/llvm-config.h"
 #include "llvm/IR/Attributes.h"

From bf544fa1c3cb80f24d85e84559fb11193846259f Mon Sep 17 00:00:00 2001
From: Jinsong Ji <jji@us.ibm.com>
Date: Mon, 27 Jul 2020 20:48:44 +0000
Subject: [PATCH 0245/1035] Revert "[PowerPC] Remove QPX/A2Q BGQ/BGP CNK
 support"

This reverts commit adffce71538e219aab4eeb024819baa7687262ff.

This is breaking test-suite, revert while investigation.
---
 clang/lib/Basic/Targets/PPC.cpp               |   39 +-
 clang/lib/Basic/Targets/PPC.h                 |    3 +
 clang/lib/Driver/ToolChains/Arch/PPC.cpp      |    1 +
 clang/lib/Driver/ToolChains/Clang.cpp         |   12 +
 clang/test/Driver/clang-translation.c         |    6 +
 clang/test/Driver/ppc-abi.c                   |   20 +
 clang/test/Misc/target-invalid-cpu-note.c     |    2 +-
 clang/test/Preprocessor/init-ppc64.c          |   16 +
 llvm/docs/LangRef.rst                         |   11 +-
 llvm/include/llvm/ADT/Triple.h                |    3 +
 llvm/include/llvm/IR/IntrinsicsPowerPC.td     |  176 +++
 llvm/lib/Support/Triple.cpp                   |    6 +
 .../Target/PowerPC/AsmParser/PPCAsmParser.cpp |   18 +
 llvm/lib/Target/PowerPC/CMakeLists.txt        |    1 +
 .../PowerPC/Disassembler/PPCDisassembler.cpp  |   15 +-
 .../PowerPC/MCTargetDesc/PPCInstPrinter.cpp   |   12 +
 .../PowerPC/MCTargetDesc/PPCMCTargetDesc.h    |    1 +
 llvm/lib/Target/PowerPC/PPC.h                 |    2 +
 llvm/lib/Target/PowerPC/PPC.td                |   14 +-
 llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp     |    3 +
 llvm/lib/Target/PowerPC/PPCCallingConv.td     |   16 +
 llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp   |   16 +-
 llvm/lib/Target/PowerPC/PPCISelLowering.cpp   | 1025 +++++++++++++-
 llvm/lib/Target/PowerPC/PPCISelLowering.h     |   20 +
 llvm/lib/Target/PowerPC/PPCInstrFormats.td    |   52 +
 llvm/lib/Target/PowerPC/PPCInstrInfo.cpp      |   27 +-
 llvm/lib/Target/PowerPC/PPCInstrInfo.h        |   23 +-
 llvm/lib/Target/PowerPC/PPCInstrInfo.td       |   11 +
 llvm/lib/Target/PowerPC/PPCInstrQPX.td        | 1212 +++++++++++++++++
 llvm/lib/Target/PowerPC/PPCQPXLoadSplat.cpp   |  161 +++
 llvm/lib/Target/PowerPC/PPCRegisterInfo.cpp   |    3 +
 llvm/lib/Target/PowerPC/PPCRegisterInfo.h     |    1 +
 llvm/lib/Target/PowerPC/PPCRegisterInfo.td    |   23 +
 llvm/lib/Target/PowerPC/PPCScheduleP9.td      |    9 +-
 llvm/lib/Target/PowerPC/PPCSubtarget.cpp      |   12 +-
 llvm/lib/Target/PowerPC/PPCSubtarget.h        |   14 +
 llvm/lib/Target/PowerPC/PPCTargetMachine.cpp  |   23 +-
 .../Target/PowerPC/PPCTargetTransformInfo.cpp |   76 +-
 .../Instrumentation/MemorySanitizer.cpp       |    7 +-
 llvm/test/Analysis/BasicAA/phi-spec-order.ll  |    2 +-
 .../CostModel/PowerPC/unal-vec-ldst.ll        |   73 +
 .../CodeGen/PowerPC/2012-11-16-mischedcall.ll |    2 +-
 ...leHoistingDueToBlockHotnessProfileData.mir |    2 +-
 .../NoCRFieldRedefWhenSpillingCRBIT.mir       |    2 +-
 llvm/test/CodeGen/PowerPC/a2q-stackalign.ll   |   23 +
 llvm/test/CodeGen/PowerPC/a2q.ll              |   10 +
 .../PowerPC/aantidep-inline-asm-use.ll        |    2 +-
 llvm/test/CodeGen/PowerPC/asm-Zy.ll           |    3 +-
 llvm/test/CodeGen/PowerPC/asm-constraints.ll  |    2 +-
 ...rt-rr-to-ri-instrs-R0-special-handling.mir |    4 +-
 .../convert-rr-to-ri-instrs-out-of-range.mir  |    2 +-
 .../PowerPC/convert-rr-to-ri-instrs.mir       |    8 +-
 llvm/test/CodeGen/PowerPC/ctr-minmaxnum.ll    |   11 +
 .../CodeGen/PowerPC/ctrloop-shortLoops.ll     |    7 +
 llvm/test/CodeGen/PowerPC/ec-input.ll         |    2 +-
 .../CodeGen/PowerPC/extra-toc-reg-deps.ll     |    8 +-
 .../CodeGen/PowerPC/fast-isel-icmp-split.ll   |    2 +-
 .../PowerPC/fma-mutate-duplicate-vreg.ll      |    2 +-
 .../CodeGen/PowerPC/fp2int2fp-ppcfp128.ll     |    3 +-
 .../CodeGen/PowerPC/glob-comp-aa-crash.ll     |    4 +-
 .../PowerPC/ifcvt-forked-bug-2016-08-08.ll    |    2 +-
 .../test/CodeGen/PowerPC/inlineasm-i64-reg.ll |    4 +-
 llvm/test/CodeGen/PowerPC/load-two-flts.ll    |    3 +-
 .../PowerPC/loop-data-prefetch-inner.ll       |    4 +-
 .../CodeGen/PowerPC/loop-data-prefetch.ll     |    4 +-
 llvm/test/CodeGen/PowerPC/loop-prep-all.ll    |   10 +-
 .../PowerPC/lxv-aligned-stack-slots.ll        |    2 +-
 llvm/test/CodeGen/PowerPC/machine-combiner.ll |   24 +
 llvm/test/CodeGen/PowerPC/mc-instrlat.ll      |    4 +-
 llvm/test/CodeGen/PowerPC/mcount-insertion.ll |    3 +-
 llvm/test/CodeGen/PowerPC/memcpy-vec.ll       |   23 +
 llvm/test/CodeGen/PowerPC/memset-nc.ll        |   48 +
 .../PowerPC/misched-inorder-latency.ll        |    3 +-
 llvm/test/CodeGen/PowerPC/misched.ll          |    1 +
 .../CodeGen/PowerPC/optnone-crbits-i1-ret.ll  |    3 +-
 .../CodeGen/PowerPC/pcrel-local-caller-toc.ll |    6 +-
 llvm/test/CodeGen/PowerPC/popcnt.ll           |    2 +
 llvm/test/CodeGen/PowerPC/ppc-passname.ll     |   11 +
 llvm/test/CodeGen/PowerPC/ppc64-sibcall.ll    |   21 +-
 llvm/test/CodeGen/PowerPC/pr24546.ll          |    4 +-
 llvm/test/CodeGen/PowerPC/pr27350.ll          |    2 +-
 llvm/test/CodeGen/PowerPC/pr28130.ll          |    2 +-
 .../CodeGen/PowerPC/preinc-ld-sel-crash.ll    |    2 +-
 llvm/test/CodeGen/PowerPC/qpx-bv-sint.ll      |   33 +
 llvm/test/CodeGen/PowerPC/qpx-bv.ll           |   37 +
 llvm/test/CodeGen/PowerPC/qpx-func-clobber.ll |   22 +
 llvm/test/CodeGen/PowerPC/qpx-load-splat.ll   |   80 ++
 llvm/test/CodeGen/PowerPC/qpx-load.ll         |   26 +
 llvm/test/CodeGen/PowerPC/qpx-qvfmadd.ll      |   79 ++
 llvm/test/CodeGen/PowerPC/qpx-recipest.ll     |  473 +++++++
 llvm/test/CodeGen/PowerPC/qpx-rounding-ops.ll |  109 ++
 llvm/test/CodeGen/PowerPC/qpx-s-load.ll       |   26 +
 llvm/test/CodeGen/PowerPC/qpx-s-sel.ll        |  143 ++
 llvm/test/CodeGen/PowerPC/qpx-s-store.ll      |   25 +
 llvm/test/CodeGen/PowerPC/qpx-sel.ll          |  151 ++
 llvm/test/CodeGen/PowerPC/qpx-split-vsetcc.ll |   31 +
 llvm/test/CodeGen/PowerPC/qpx-store.ll        |   25 +
 .../test/CodeGen/PowerPC/qpx-unal-cons-lds.ll |  217 +++
 llvm/test/CodeGen/PowerPC/qpx-unalperm.ll     |   64 +
 llvm/test/CodeGen/PowerPC/rlwimi-and.ll       |    4 +-
 .../CodeGen/PowerPC/rlwinm_rldicl_to_andi.mir |    2 +-
 .../CodeGen/PowerPC/s000-alias-misched.ll     |    5 +-
 llvm/test/CodeGen/PowerPC/select-i1-vs-i1.ll  |  571 ++++++++
 .../selectiondag-extload-computeknownbits.ll  |    2 +-
 llvm/test/CodeGen/PowerPC/setcr_bc.mir        |    4 +-
 llvm/test/CodeGen/PowerPC/setcr_bc2.mir       |    4 +-
 llvm/test/CodeGen/PowerPC/stwu-sched.ll       |    2 +-
 llvm/test/CodeGen/PowerPC/unal-vec-ldst.ll    |  149 ++
 llvm/test/CodeGen/PowerPC/uwtables.ll         |    2 +-
 .../MemorySanitizer/PowerPC/vararg-ppc64.ll   |   15 +
 llvm/test/MC/Disassembler/PowerPC/qpx.txt     |  371 +++++
 llvm/test/MC/PowerPC/qpx.s                    |  252 ++++
 .../IPConstantProp/fp-bc-icmp-const-fold.ll   |    2 +-
 .../MSSA/combined-partial-overwrites.ll       |    2 +-
 .../combined-partial-overwrites.ll            |    2 +-
 .../EntryExitInstrumenter/mcount.ll           |    2 +-
 .../InstCombine/PowerPC/aligned-qpx.ll        |  165 +++
 .../LoopDataPrefetch/PowerPC/basic.ll         |    5 +-
 .../test/Transforms/LoopSimplify/dup-preds.ll |    2 +-
 llvm/test/Transforms/LoopUnroll/pr14167.ll    |    2 +-
 .../PowerPC/agg-interleave-a2.ll              |   40 +
 .../PowerPC/vectorize-only-for-real.ll        |    2 +-
 llvm/test/Transforms/NewGVN/pr31483.ll        |    4 +-
 .../Transforms/SCCP/fp-bc-icmp-const-fold.ll  |    2 +-
 llvm/unittests/ADT/TripleTest.cpp             |   35 +
 .../llvm/lib/Target/PowerPC/BUILD.gn          |    1 +
 openmp/runtime/src/kmp.h                      |    3 +
 openmp/runtime/src/kmp_csupport.cpp           |   11 +
 openmp/runtime/src/kmp_lock.h                 |    2 +-
 openmp/runtime/src/kmp_os.h                   |    2 +-
 openmp/runtime/src/kmp_platform.h             |    6 +
 openmp/runtime/src/z_Linux_asm.S              |    7 +-
 openmp/runtime/src/z_Linux_util.cpp           |    2 +-
 polly/lib/External/isl/config.sub             |   11 +-
 polly/lib/External/ppcg/config.sub            |   11 +-
 135 files changed, 6525 insertions(+), 174 deletions(-)
 create mode 100644 llvm/lib/Target/PowerPC/PPCInstrQPX.td
 create mode 100644 llvm/lib/Target/PowerPC/PPCQPXLoadSplat.cpp
 create mode 100644 llvm/test/CodeGen/PowerPC/a2q-stackalign.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/a2q.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/memset-nc.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/qpx-bv-sint.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/qpx-bv.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/qpx-func-clobber.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/qpx-load-splat.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/qpx-load.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/qpx-qvfmadd.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/qpx-recipest.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/qpx-rounding-ops.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/qpx-s-load.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/qpx-s-sel.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/qpx-s-store.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/qpx-sel.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/qpx-split-vsetcc.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/qpx-store.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/qpx-unal-cons-lds.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/qpx-unalperm.ll
 create mode 100644 llvm/test/MC/Disassembler/PowerPC/qpx.txt
 create mode 100644 llvm/test/MC/PowerPC/qpx.s
 create mode 100644 llvm/test/Transforms/InstCombine/PowerPC/aligned-qpx.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/PowerPC/agg-interleave-a2.ll

diff --git a/clang/lib/Basic/Targets/PPC.cpp b/clang/lib/Basic/Targets/PPC.cpp
index 5f716a541ae92..f0de2bf070ea4 100644
--- a/clang/lib/Basic/Targets/PPC.cpp
+++ b/clang/lib/Basic/Targets/PPC.cpp
@@ -46,6 +46,8 @@ bool PPCTargetInfo::handleTargetFeatures(std::vector<std::string> &Features,
       HasP8Crypto = true;
     } else if (Feature == "+direct-move") {
       HasDirectMove = true;
+    } else if (Feature == "+qpx") {
+      HasQPX = true;
     } else if (Feature == "+htm") {
       HasHTM = true;
     } else if (Feature == "+float128") {
@@ -97,7 +99,7 @@ void PPCTargetInfo::getTargetDefines(const LangOptions &Opts,
   }
 
   // ABI options.
-  if (ABI == "elfv1")
+  if (ABI == "elfv1" || ABI == "elfv1-qpx")
     Builder.defineMacro("_CALL_ELF", "1");
   if (ABI == "elfv2")
     Builder.defineMacro("_CALL_ELF", "2");
@@ -157,11 +159,22 @@ void PPCTargetInfo::getTargetDefines(const LangOptions &Opts,
     Builder.defineMacro("_ARCH_PWR10");
   if (ArchDefs & ArchDefineA2)
     Builder.defineMacro("_ARCH_A2");
+  if (ArchDefs & ArchDefineA2q) {
+    Builder.defineMacro("_ARCH_A2Q");
+    Builder.defineMacro("_ARCH_QP");
+  }
   if (ArchDefs & ArchDefineE500)
     Builder.defineMacro("__NO_LWSYNC__");
   if (ArchDefs & ArchDefineFuture)
     Builder.defineMacro("_ARCH_PWR_FUTURE");
 
+  if (getTriple().getVendor() == llvm::Triple::BGQ) {
+    Builder.defineMacro("__bg__");
+    Builder.defineMacro("__THW_BLUEGENE__");
+    Builder.defineMacro("__bgq__");
+    Builder.defineMacro("__TOS_BGQ__");
+  }
+
   if (HasAltivec) {
     Builder.defineMacro("__VEC__", "10206");
     Builder.defineMacro("__ALTIVEC__");
@@ -264,6 +277,7 @@ bool PPCTargetInfo::initFeatureMap(
                             .Case("ppc64le", true)
                             .Default(false);
 
+  Features["qpx"] = (CPU == "a2q");
   Features["power9-vector"] = (CPU == "pwr9");
   Features["crypto"] = llvm::StringSwitch<bool>(CPU)
                            .Case("ppc64le", true)
@@ -359,6 +373,7 @@ bool PPCTargetInfo::hasFeature(StringRef Feature) const {
       .Case("power8-vector", HasP8Vector)
       .Case("crypto", HasP8Crypto)
       .Case("direct-move", HasDirectMove)
+      .Case("qpx", HasQPX)
       .Case("htm", HasHTM)
       .Case("bpermd", HasBPERMD)
       .Case("extdiv", HasExtDiv)
@@ -488,17 +503,17 @@ ArrayRef<TargetInfo::AddlRegName> PPCTargetInfo::getGCCAddlRegNames() const {
 }
 
 static constexpr llvm::StringLiteral ValidCPUNames[] = {
-    {"generic"}, {"440"},     {"450"},       {"601"},     {"602"},
-    {"603"},     {"603e"},    {"603ev"},     {"604"},     {"604e"},
-    {"620"},     {"630"},     {"g3"},        {"7400"},    {"g4"},
-    {"7450"},    {"g4+"},     {"750"},       {"8548"},    {"970"},
-    {"g5"},      {"a2"},      {"e500"},      {"e500mc"},  {"e5500"},
-    {"power3"},  {"pwr3"},    {"power4"},    {"pwr4"},    {"power5"},
-    {"pwr5"},    {"power5x"}, {"pwr5x"},     {"power6"},  {"pwr6"},
-    {"power6x"}, {"pwr6x"},   {"power7"},    {"pwr7"},    {"power8"},
-    {"pwr8"},    {"power9"},  {"pwr9"},      {"power10"}, {"pwr10"},
-    {"powerpc"}, {"ppc"},     {"powerpc64"}, {"ppc64"},   {"powerpc64le"},
-    {"ppc64le"}, {"future"}};
+    {"generic"},     {"440"},     {"450"},     {"601"},       {"602"},
+    {"603"},         {"603e"},    {"603ev"},   {"604"},       {"604e"},
+    {"620"},         {"630"},     {"g3"},      {"7400"},      {"g4"},
+    {"7450"},        {"g4+"},     {"750"},     {"8548"},      {"970"},
+    {"g5"},          {"a2"},      {"a2q"},     {"e500"},      {"e500mc"},
+    {"e5500"},       {"power3"},  {"pwr3"},    {"power4"},    {"pwr4"},
+    {"power5"},      {"pwr5"},    {"power5x"}, {"pwr5x"},     {"power6"},
+    {"pwr6"},        {"power6x"}, {"pwr6x"},   {"power7"},    {"pwr7"},
+    {"power8"},      {"pwr8"},    {"power9"},  {"pwr9"},      {"power10"},
+    {"pwr10"},       {"powerpc"}, {"ppc"},     {"powerpc64"}, {"ppc64"},
+    {"powerpc64le"}, {"ppc64le"}, {"future"}};
 
 bool PPCTargetInfo::isValidCPUName(StringRef Name) const {
   return llvm::find(ValidCPUNames, Name) != std::end(ValidCPUNames);
diff --git a/clang/lib/Basic/Targets/PPC.h b/clang/lib/Basic/Targets/PPC.h
index c2048b2145918..ff8579b6c3cf4 100644
--- a/clang/lib/Basic/Targets/PPC.h
+++ b/clang/lib/Basic/Targets/PPC.h
@@ -46,6 +46,7 @@ class LLVM_LIBRARY_VISIBILITY PPCTargetInfo : public TargetInfo {
     ArchDefinePwr10 = 1 << 14,
     ArchDefineFuture = 1 << 15,
     ArchDefineA2 = 1 << 16,
+    ArchDefineA2q = 1 << 17,
     ArchDefineE500 = 1 << 18
   } ArchDefineTypes;
 
@@ -62,6 +63,7 @@ class LLVM_LIBRARY_VISIBILITY PPCTargetInfo : public TargetInfo {
   bool HasP8Vector = false;
   bool HasP8Crypto = false;
   bool HasDirectMove = false;
+  bool HasQPX = false;
   bool HasHTM = false;
   bool HasBPERMD = false;
   bool HasExtDiv = false;
@@ -116,6 +118,7 @@ class LLVM_LIBRARY_VISIBILITY PPCTargetInfo : public TargetInfo {
               .Case("970", ArchDefineName | ArchDefinePwr4 | ArchDefinePpcgr |
                                ArchDefinePpcsq)
               .Case("a2", ArchDefineA2)
+              .Case("a2q", ArchDefineName | ArchDefineA2 | ArchDefineA2q)
               .Cases("power3", "pwr3", ArchDefinePpcgr)
               .Cases("power4", "pwr4",
                      ArchDefinePwr4 | ArchDefinePpcgr | ArchDefinePpcsq)
diff --git a/clang/lib/Driver/ToolChains/Arch/PPC.cpp b/clang/lib/Driver/ToolChains/Arch/PPC.cpp
index bcaecf4b2d980..144e276a6bd87 100644
--- a/clang/lib/Driver/ToolChains/Arch/PPC.cpp
+++ b/clang/lib/Driver/ToolChains/Arch/PPC.cpp
@@ -57,6 +57,7 @@ std::string ppc::getPPCTargetCPU(const ArgList &Args) {
         .Case("970", "970")
         .Case("G5", "g5")
         .Case("a2", "a2")
+        .Case("a2q", "a2q")
         .Case("e500", "e500")
         .Case("e500mc", "e500mc")
         .Case("e5500", "e5500")
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index b0de225f8abf5..7a73eea013bdf 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -1883,6 +1883,18 @@ void Clang::AddPPCTargetArgs(const ArgList &Args,
   if (T.isOSBinFormatELF()) {
     switch (getToolChain().getArch()) {
     case llvm::Triple::ppc64: {
+      // When targeting a processor that supports QPX, or if QPX is
+      // specifically enabled, default to using the ABI that supports QPX (so
+      // long as it is not specifically disabled).
+      bool HasQPX = false;
+      if (Arg *A = Args.getLastArg(options::OPT_mcpu_EQ))
+        HasQPX = A->getValue() == StringRef("a2q");
+      HasQPX = Args.hasFlag(options::OPT_mqpx, options::OPT_mno_qpx, HasQPX);
+      if (HasQPX) {
+        ABIName = "elfv1-qpx";
+        break;
+      }
+
       if (T.isMusl() || (T.isOSFreeBSD() && T.getOSMajorVersion() >= 13))
         ABIName = "elfv2";
       else
diff --git a/clang/test/Driver/clang-translation.c b/clang/test/Driver/clang-translation.c
index d1daeb80004b7..2f02970a2a8ee 100644
--- a/clang/test/Driver/clang-translation.c
+++ b/clang/test/Driver/clang-translation.c
@@ -167,6 +167,12 @@
 // PPCPWR8: "-cc1"
 // PPCPWR8: "-target-cpu" "pwr8"
 
+// RUN: %clang -target powerpc64-unknown-linux-gnu \
+// RUN: -### -S %s -mcpu=a2q 2>&1 | FileCheck -check-prefix=PPCA2Q %s
+// PPCA2Q: clang
+// PPCA2Q: "-cc1"
+// PPCA2Q: "-target-cpu" "a2q"
+
 // RUN: %clang -target powerpc64-unknown-linux-gnu \
 // RUN: -### -S %s -mcpu=630 2>&1 | FileCheck -check-prefix=PPC630 %s
 // PPC630: clang
diff --git a/clang/test/Driver/ppc-abi.c b/clang/test/Driver/ppc-abi.c
index 2b5cc463e7c3d..acc4981a2eee6 100644
--- a/clang/test/Driver/ppc-abi.c
+++ b/clang/test/Driver/ppc-abi.c
@@ -5,6 +5,14 @@
 // RUN: %clang -target powerpc64-unknown-linux-gnu %s -### -o %t.o 2>&1 \
 // RUN:   -mabi=elfv1 | FileCheck -check-prefix=CHECK-ELFv1 %s
 // RUN: %clang -target powerpc64-unknown-linux-gnu %s -### -o %t.o 2>&1 \
+// RUN:   -mabi=elfv1-qpx | FileCheck -check-prefix=CHECK-ELFv1-QPX %s
+// RUN: %clang -target powerpc64-unknown-linux-gnu %s -### -o %t.o 2>&1 \
+// RUN:   -mcpu=a2q | FileCheck -check-prefix=CHECK-ELFv1-QPX %s
+// RUN: %clang -target powerpc64-unknown-linux-gnu %s -### -o %t.o 2>&1 \
+// RUN:   -mcpu=a2 -mqpx | FileCheck -check-prefix=CHECK-ELFv1-QPX %s
+// RUN: %clang -target powerpc64-unknown-linux-gnu %s -### -o %t.o 2>&1 \
+// RUN:   -mcpu=a2q -mno-qpx | FileCheck -check-prefix=CHECK-ELFv1 %s
+// RUN: %clang -target powerpc64-unknown-linux-gnu %s -### -o %t.o 2>&1 \
 // RUN:   -mabi=elfv2 | FileCheck -check-prefix=CHECK-ELFv2-BE %s
 
 // RUN: %clang -target powerpc64le-unknown-linux-gnu %s -### -o %t.o 2>&1 \
@@ -26,6 +34,8 @@
 // CHECK-ELFv1: "-target-abi" "elfv1"
 // CHECK-ELFv1-LE: "-mrelocation-model" "static"
 // CHECK-ELFv1-LE: "-target-abi" "elfv1"
+// CHECK-ELFv1-QPX: "-mrelocation-model" "static"
+// CHECK-ELFv1-QPX: "-target-abi" "elfv1-qpx"
 // CHECK-ELFv2: "-mrelocation-model" "static"
 // CHECK-ELFv2: "-target-abi" "elfv2"
 // CHECK-ELFv2-BE: "-mrelocation-model" "static"
@@ -38,6 +48,14 @@
 // RUN: %clang -fPIC -target powerpc64-unknown-linux-gnu %s -### -o %t.o 2>&1 \
 // RUN:   -mabi=elfv1 | FileCheck -check-prefix=CHECK-ELFv1-PIC %s
 // RUN: %clang -fPIC -target powerpc64-unknown-linux-gnu %s -### -o %t.o 2>&1 \
+// RUN:   -mabi=elfv1-qpx | FileCheck -check-prefix=CHECK-ELFv1-QPX-PIC %s
+// RUN: %clang -fPIC -target powerpc64-unknown-linux-gnu %s -### -o %t.o 2>&1 \
+// RUN:   -mcpu=a2q | FileCheck -check-prefix=CHECK-ELFv1-QPX-PIC %s
+// RUN: %clang -fPIC -target powerpc64-unknown-linux-gnu %s -### -o %t.o 2>&1 \
+// RUN:   -mcpu=a2 -mqpx | FileCheck -check-prefix=CHECK-ELFv1-QPX-PIC %s
+// RUN: %clang -fPIC -target powerpc64-unknown-linux-gnu %s -### -o %t.o 2>&1 \
+// RUN:   -mcpu=a2q -mno-qpx | FileCheck -check-prefix=CHECK-ELFv1-PIC %s
+// RUN: %clang -fPIC -target powerpc64-unknown-linux-gnu %s -### -o %t.o 2>&1 \
 // RUN:   -mabi=elfv2 | FileCheck -check-prefix=CHECK-ELFv2-PIC %s
 
 // RUN: %clang -fPIC -target powerpc64le-unknown-linux-gnu %s -### -o %t.o 2>&1 \
@@ -51,6 +69,8 @@
 
 // CHECK-ELFv1-PIC: "-mrelocation-model" "pic" "-pic-level" "2"
 // CHECK-ELFv1-PIC: "-target-abi" "elfv1"
+// CHECK-ELFv1-QPX-PIC: "-mrelocation-model" "pic" "-pic-level" "2"
+// CHECK-ELFv1-QPX-PIC: "-target-abi" "elfv1-qpx"
 // CHECK-ELFv2-PIC: "-mrelocation-model" "pic" "-pic-level" "2"
 // CHECK-ELFv2-PIC: "-target-abi" "elfv2"
 
diff --git a/clang/test/Misc/target-invalid-cpu-note.c b/clang/test/Misc/target-invalid-cpu-note.c
index bf6eaefe0b3ca..3a376a7caab46 100644
--- a/clang/test/Misc/target-invalid-cpu-note.c
+++ b/clang/test/Misc/target-invalid-cpu-note.c
@@ -79,7 +79,7 @@
 // PPC: error: unknown target CPU 'not-a-cpu'
 // PPC: note: valid target CPU values are: generic, 440, 450, 601, 602, 603,
 // PPC-SAME: 603e, 603ev, 604, 604e, 620, 630, g3, 7400, g4, 7450, g4+, 750,
-// PPC-SAME: 8548, 970, g5, a2, e500, e500mc, e5500, power3, pwr3, power4,
+// PPC-SAME: 8548, 970, g5, a2, a2q, e500, e500mc, e5500, power3, pwr3, power4,
 // PPC-SAME: pwr4, power5, pwr5, power5x, pwr5x, power6, pwr6, power6x, pwr6x,
 // PPC-SAME: power7, pwr7, power8, pwr8, power9, pwr9, power10, pwr10, powerpc, ppc, powerpc64,
 // PPC-SAME: ppc64, powerpc64le, ppc64le, future
diff --git a/clang/test/Preprocessor/init-ppc64.c b/clang/test/Preprocessor/init-ppc64.c
index 48d35c95aa570..ed8601636554e 100644
--- a/clang/test/Preprocessor/init-ppc64.c
+++ b/clang/test/Preprocessor/init-ppc64.c
@@ -408,6 +408,21 @@
 // PPC64LE:#define __ppc64__ 1
 // PPC64LE:#define __ppc__ 1
 //
+// RUN: %clang_cc1 -E -dM -ffreestanding -triple=powerpc64-none-none -target-cpu a2q -fno-signed-char < /dev/null | FileCheck -match-full-lines -check-prefix PPCA2Q %s
+//
+// PPCA2Q:#define _ARCH_A2 1
+// PPCA2Q:#define _ARCH_A2Q 1
+// PPCA2Q:#define _ARCH_PPC 1
+// PPCA2Q:#define _ARCH_PPC64 1
+// PPCA2Q:#define _ARCH_QP 1
+//
+// RUN: %clang_cc1 -E -dM -ffreestanding -triple=powerpc64-bgq-linux -fno-signed-char < /dev/null | FileCheck -match-full-lines -check-prefix PPCBGQ %s
+//
+// PPCBGQ:#define __THW_BLUEGENE__ 1
+// PPCBGQ:#define __TOS_BGQ__ 1
+// PPCBGQ:#define __bg__ 1
+// PPCBGQ:#define __bgq__ 1
+//
 // RUN: %clang_cc1 -E -dM -ffreestanding -triple=powerpc64-none-none -target-cpu 630 -fno-signed-char < /dev/null | FileCheck -match-full-lines -check-prefix PPC630 %s
 //
 // PPC630:#define _ARCH_630 1
@@ -1054,6 +1069,7 @@
 
 // RUN: %clang_cc1 -E -dM -ffreestanding -fgnuc-version=4.2.1 -triple=powerpc64-unknown-linux-gnu < /dev/null | FileCheck -match-full-lines -check-prefix PPC64-ELFv1 %s
 // RUN: %clang_cc1 -E -dM -ffreestanding -fgnuc-version=4.2.1 -triple=powerpc64-unknown-linux-gnu -target-abi elfv1 < /dev/null | FileCheck -match-full-lines -check-prefix PPC64-ELFv1 %s
+// RUN: %clang_cc1 -E -dM -ffreestanding -fgnuc-version=4.2.1 -triple=powerpc64-unknown-linux-gnu -target-abi elfv1-qpx < /dev/null | FileCheck -match-full-lines -check-prefix PPC64-ELFv1 %s
 // RUN: %clang_cc1 -E -dM -ffreestanding -fgnuc-version=4.2.1 -triple=powerpc64-unknown-linux-gnu -target-abi elfv2 < /dev/null | FileCheck -match-full-lines -check-prefix PPC64-ELFv2 %s
 // RUN: %clang_cc1 -E -dM -ffreestanding -fgnuc-version=4.2.1 -triple=powerpc64le-unknown-linux-gnu < /dev/null | FileCheck -match-full-lines -check-prefix PPC64-ELFv2 %s
 // RUN: %clang_cc1 -E -dM -ffreestanding -fgnuc-version=4.2.1 -triple=powerpc64le-unknown-linux-gnu -target-abi elfv1 < /dev/null | FileCheck -match-full-lines -check-prefix PPC64-ELFv1 %s
diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index af93a6ed5c56e..6b9c5c6899819 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -4310,9 +4310,14 @@ PowerPC:
 - ``r``: A 32 or 64-bit integer register.
 - ``b``: A 32 or 64-bit integer register, excluding ``R0`` (that is:
   ``R1-R31``).
-- ``f``: A 32 or 64-bit float register (``F0-F31``), 
-- ``v``: For ``4 x f32`` or ``4 x f64`` types, a 128-bit altivec vector
-   register (``V0-V31``).
+- ``f``: A 32 or 64-bit float register (``F0-F31``), or when QPX is enabled, a
+  128 or 256-bit QPX register (``Q0-Q31``; aliases the ``F`` registers).
+- ``v``: For ``4 x f32`` or ``4 x f64`` types, when QPX is enabled, a
+  128 or 256-bit QPX register (``Q0-Q31``), otherwise a 128-bit
+  altivec vector register (``V0-V31``).
+
+  .. FIXME: is this a bug that v accepts QPX registers? I think this
+     is supposed to only use the altivec vector registers?
 
 - ``y``: Condition register (``CR0-CR7``).
 - ``wc``: An individual CR bit in a CR register.
diff --git a/llvm/include/llvm/ADT/Triple.h b/llvm/include/llvm/ADT/Triple.h
index c578c097c6f64..6bad18f19244e 100644
--- a/llvm/include/llvm/ADT/Triple.h
+++ b/llvm/include/llvm/ADT/Triple.h
@@ -142,6 +142,8 @@ class Triple {
     Apple,
     PC,
     SCEI,
+    BGP,
+    BGQ,
     Freescale,
     IBM,
     ImaginationTechnologies,
@@ -177,6 +179,7 @@ class Triple {
     Minix,
     RTEMS,
     NaCl,       // Native Client
+    CNK,        // BG/P Compute-Node Kernel
     AIX,
     CUDA,       // NVIDIA CUDA
     NVCL,       // NVIDIA OpenCL
diff --git a/llvm/include/llvm/IR/IntrinsicsPowerPC.td b/llvm/include/llvm/IR/IntrinsicsPowerPC.td
index 853d26c67ee3d..23bcf3ce1959c 100644
--- a/llvm/include/llvm/IR/IntrinsicsPowerPC.td
+++ b/llvm/include/llvm/IR/IntrinsicsPowerPC.td
@@ -1109,6 +1109,182 @@ def  int_ppc_vsx_xxblendvd: GCCBuiltin<"__builtin_vsx_xxblendvd">,
                  [IntrNoMem]>;
 }
 
+//===----------------------------------------------------------------------===//
+// PowerPC QPX Intrinsics.
+//
+
+let TargetPrefix = "ppc" in {  // All PPC intrinsics start with "llvm.ppc.".
+  /// PowerPC_QPX_Intrinsic - Base class for all QPX intrinsics.
+  class PowerPC_QPX_Intrinsic<string GCCIntSuffix, list<LLVMType> ret_types,
+                              list<LLVMType> param_types,
+                              list<IntrinsicProperty> properties>
+    : GCCBuiltin<!strconcat("__builtin_qpx_", GCCIntSuffix)>,
+      Intrinsic<ret_types, param_types, properties>;
+}
+
+//===----------------------------------------------------------------------===//
+// PowerPC QPX Intrinsic Class Definitions.
+//
+
+/// PowerPC_QPX_FF_Intrinsic - A PowerPC intrinsic that takes one v4f64
+/// vector and returns one.  These intrinsics have no side effects.
+class PowerPC_QPX_FF_Intrinsic<string GCCIntSuffix>
+  : PowerPC_QPX_Intrinsic<GCCIntSuffix,
+                          [llvm_v4f64_ty], [llvm_v4f64_ty], [IntrNoMem]>;
+
+/// PowerPC_QPX_FFF_Intrinsic - A PowerPC intrinsic that takes two v4f64
+/// vectors and returns one.  These intrinsics have no side effects.
+class PowerPC_QPX_FFF_Intrinsic<string GCCIntSuffix>
+  : PowerPC_QPX_Intrinsic<GCCIntSuffix,
+                          [llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty],
+                          [IntrNoMem]>;
+
+/// PowerPC_QPX_FFFF_Intrinsic - A PowerPC intrinsic that takes three v4f64
+/// vectors and returns one.  These intrinsics have no side effects.
+class PowerPC_QPX_FFFF_Intrinsic<string GCCIntSuffix>
+  : PowerPC_QPX_Intrinsic<GCCIntSuffix,
+                          [llvm_v4f64_ty],
+                          [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4f64_ty],
+                          [IntrNoMem]>;
+
+/// PowerPC_QPX_Load_Intrinsic - A PowerPC intrinsic that takes a pointer
+/// and returns a v4f64.
+class PowerPC_QPX_Load_Intrinsic<string GCCIntSuffix>
+  : PowerPC_QPX_Intrinsic<GCCIntSuffix,
+                          [llvm_v4f64_ty], [llvm_ptr_ty], [IntrReadMem, IntrArgMemOnly]>;
+
+/// PowerPC_QPX_LoadPerm_Intrinsic - A PowerPC intrinsic that takes a pointer
+/// and returns a v4f64 permutation.
+class PowerPC_QPX_LoadPerm_Intrinsic<string GCCIntSuffix>
+  : PowerPC_QPX_Intrinsic<GCCIntSuffix,
+                          [llvm_v4f64_ty], [llvm_ptr_ty], [IntrNoMem]>;
+
+/// PowerPC_QPX_Store_Intrinsic - A PowerPC intrinsic that takes a pointer
+/// and stores a v4f64.
+class PowerPC_QPX_Store_Intrinsic<string GCCIntSuffix>
+  : PowerPC_QPX_Intrinsic<GCCIntSuffix,
+                          [], [llvm_v4f64_ty, llvm_ptr_ty],
+                          [IntrWriteMem, IntrArgMemOnly]>;
+
+//===----------------------------------------------------------------------===//
+// PowerPC QPX Intrinsic Definitions.
+
+let TargetPrefix = "ppc" in {  // All intrinsics start with "llvm.ppc.".
+  // Add Instructions
+  def int_ppc_qpx_qvfadd : PowerPC_QPX_FFF_Intrinsic<"qvfadd">;
+  def int_ppc_qpx_qvfadds : PowerPC_QPX_FFF_Intrinsic<"qvfadds">;
+  def int_ppc_qpx_qvfsub : PowerPC_QPX_FFF_Intrinsic<"qvfsub">;
+  def int_ppc_qpx_qvfsubs : PowerPC_QPX_FFF_Intrinsic<"qvfsubs">;
+
+  // Estimate Instructions
+  def int_ppc_qpx_qvfre : PowerPC_QPX_FF_Intrinsic<"qvfre">;
+  def int_ppc_qpx_qvfres : PowerPC_QPX_FF_Intrinsic<"qvfres">;
+  def int_ppc_qpx_qvfrsqrte : PowerPC_QPX_FF_Intrinsic<"qvfrsqrte">;
+  def int_ppc_qpx_qvfrsqrtes : PowerPC_QPX_FF_Intrinsic<"qvfrsqrtes">;
+
+  // Multiply Instructions
+  def int_ppc_qpx_qvfmul : PowerPC_QPX_FFF_Intrinsic<"qvfmul">;
+  def int_ppc_qpx_qvfmuls : PowerPC_QPX_FFF_Intrinsic<"qvfmuls">;
+  def int_ppc_qpx_qvfxmul : PowerPC_QPX_FFF_Intrinsic<"qvfxmul">;
+  def int_ppc_qpx_qvfxmuls : PowerPC_QPX_FFF_Intrinsic<"qvfxmuls">;
+
+  // Multiply-add instructions
+  def int_ppc_qpx_qvfmadd : PowerPC_QPX_FFFF_Intrinsic<"qvfmadd">;
+  def int_ppc_qpx_qvfmadds : PowerPC_QPX_FFFF_Intrinsic<"qvfmadds">;
+  def int_ppc_qpx_qvfnmadd : PowerPC_QPX_FFFF_Intrinsic<"qvfnmadd">;
+  def int_ppc_qpx_qvfnmadds : PowerPC_QPX_FFFF_Intrinsic<"qvfnmadds">;
+  def int_ppc_qpx_qvfmsub : PowerPC_QPX_FFFF_Intrinsic<"qvfmsub">;
+  def int_ppc_qpx_qvfmsubs : PowerPC_QPX_FFFF_Intrinsic<"qvfmsubs">;
+  def int_ppc_qpx_qvfnmsub : PowerPC_QPX_FFFF_Intrinsic<"qvfnmsub">;
+  def int_ppc_qpx_qvfnmsubs : PowerPC_QPX_FFFF_Intrinsic<"qvfnmsubs">;
+  def int_ppc_qpx_qvfxmadd : PowerPC_QPX_FFFF_Intrinsic<"qvfxmadd">;
+  def int_ppc_qpx_qvfxmadds : PowerPC_QPX_FFFF_Intrinsic<"qvfxmadds">;
+  def int_ppc_qpx_qvfxxnpmadd : PowerPC_QPX_FFFF_Intrinsic<"qvfxxnpmadd">;
+  def int_ppc_qpx_qvfxxnpmadds : PowerPC_QPX_FFFF_Intrinsic<"qvfxxnpmadds">;
+  def int_ppc_qpx_qvfxxcpnmadd : PowerPC_QPX_FFFF_Intrinsic<"qvfxxcpnmadd">;
+  def int_ppc_qpx_qvfxxcpnmadds : PowerPC_QPX_FFFF_Intrinsic<"qvfxxcpnmadds">;
+  def int_ppc_qpx_qvfxxmadd : PowerPC_QPX_FFFF_Intrinsic<"qvfxxmadd">;
+  def int_ppc_qpx_qvfxxmadds : PowerPC_QPX_FFFF_Intrinsic<"qvfxxmadds">;
+
+  // Select Instruction
+  def int_ppc_qpx_qvfsel : PowerPC_QPX_FFFF_Intrinsic<"qvfsel">;
+
+  // Permute Instruction
+  def int_ppc_qpx_qvfperm : PowerPC_QPX_FFFF_Intrinsic<"qvfperm">;
+
+  // Convert and Round Instructions
+  def int_ppc_qpx_qvfctid : PowerPC_QPX_FF_Intrinsic<"qvfctid">;
+  def int_ppc_qpx_qvfctidu : PowerPC_QPX_FF_Intrinsic<"qvfctidu">;
+  def int_ppc_qpx_qvfctidz : PowerPC_QPX_FF_Intrinsic<"qvfctidz">;
+  def int_ppc_qpx_qvfctiduz : PowerPC_QPX_FF_Intrinsic<"qvfctiduz">;
+  def int_ppc_qpx_qvfctiw : PowerPC_QPX_FF_Intrinsic<"qvfctiw">;
+  def int_ppc_qpx_qvfctiwu : PowerPC_QPX_FF_Intrinsic<"qvfctiwu">;
+  def int_ppc_qpx_qvfctiwz : PowerPC_QPX_FF_Intrinsic<"qvfctiwz">;
+  def int_ppc_qpx_qvfctiwuz : PowerPC_QPX_FF_Intrinsic<"qvfctiwuz">;
+  def int_ppc_qpx_qvfcfid : PowerPC_QPX_FF_Intrinsic<"qvfcfid">;
+  def int_ppc_qpx_qvfcfidu : PowerPC_QPX_FF_Intrinsic<"qvfcfidu">;
+  def int_ppc_qpx_qvfcfids : PowerPC_QPX_FF_Intrinsic<"qvfcfids">;
+  def int_ppc_qpx_qvfcfidus : PowerPC_QPX_FF_Intrinsic<"qvfcfidus">;
+  def int_ppc_qpx_qvfrsp : PowerPC_QPX_FF_Intrinsic<"qvfrsp">;
+  def int_ppc_qpx_qvfriz : PowerPC_QPX_FF_Intrinsic<"qvfriz">;
+  def int_ppc_qpx_qvfrin : PowerPC_QPX_FF_Intrinsic<"qvfrin">;
+  def int_ppc_qpx_qvfrip : PowerPC_QPX_FF_Intrinsic<"qvfrip">;
+  def int_ppc_qpx_qvfrim : PowerPC_QPX_FF_Intrinsic<"qvfrim">;
+
+  // Move Instructions
+  def int_ppc_qpx_qvfneg : PowerPC_QPX_FF_Intrinsic<"qvfneg">;
+  def int_ppc_qpx_qvfabs : PowerPC_QPX_FF_Intrinsic<"qvfabs">;
+  def int_ppc_qpx_qvfnabs : PowerPC_QPX_FF_Intrinsic<"qvfnabs">;
+  def int_ppc_qpx_qvfcpsgn : PowerPC_QPX_FFF_Intrinsic<"qvfcpsgn">;
+
+  // Compare Instructions
+  def int_ppc_qpx_qvftstnan : PowerPC_QPX_FFF_Intrinsic<"qvftstnan">;
+  def int_ppc_qpx_qvfcmplt : PowerPC_QPX_FFF_Intrinsic<"qvfcmplt">;
+  def int_ppc_qpx_qvfcmpgt : PowerPC_QPX_FFF_Intrinsic<"qvfcmpgt">;
+  def int_ppc_qpx_qvfcmpeq : PowerPC_QPX_FFF_Intrinsic<"qvfcmpeq">;
+
+  // Load instructions
+  def int_ppc_qpx_qvlfd : PowerPC_QPX_Load_Intrinsic<"qvlfd">;
+  def int_ppc_qpx_qvlfda : PowerPC_QPX_Load_Intrinsic<"qvlfda">;
+  def int_ppc_qpx_qvlfs : PowerPC_QPX_Load_Intrinsic<"qvlfs">;
+  def int_ppc_qpx_qvlfsa : PowerPC_QPX_Load_Intrinsic<"qvlfsa">;
+
+  def int_ppc_qpx_qvlfcda : PowerPC_QPX_Load_Intrinsic<"qvlfcda">;
+  def int_ppc_qpx_qvlfcd : PowerPC_QPX_Load_Intrinsic<"qvlfcd">;
+  def int_ppc_qpx_qvlfcsa : PowerPC_QPX_Load_Intrinsic<"qvlfcsa">;
+  def int_ppc_qpx_qvlfcs : PowerPC_QPX_Load_Intrinsic<"qvlfcs">;
+  def int_ppc_qpx_qvlfiwaa : PowerPC_QPX_Load_Intrinsic<"qvlfiwaa">;
+  def int_ppc_qpx_qvlfiwa : PowerPC_QPX_Load_Intrinsic<"qvlfiwa">;
+  def int_ppc_qpx_qvlfiwza : PowerPC_QPX_Load_Intrinsic<"qvlfiwza">;
+  def int_ppc_qpx_qvlfiwz : PowerPC_QPX_Load_Intrinsic<"qvlfiwz">;
+
+  def int_ppc_qpx_qvlpcld : PowerPC_QPX_LoadPerm_Intrinsic<"qvlpcld">;
+  def int_ppc_qpx_qvlpcls : PowerPC_QPX_LoadPerm_Intrinsic<"qvlpcls">;
+  def int_ppc_qpx_qvlpcrd : PowerPC_QPX_LoadPerm_Intrinsic<"qvlpcrd">;
+  def int_ppc_qpx_qvlpcrs : PowerPC_QPX_LoadPerm_Intrinsic<"qvlpcrs">;
+
+  // Store instructions
+  def int_ppc_qpx_qvstfd : PowerPC_QPX_Store_Intrinsic<"qvstfd">;
+  def int_ppc_qpx_qvstfda : PowerPC_QPX_Store_Intrinsic<"qvstfda">;
+  def int_ppc_qpx_qvstfs : PowerPC_QPX_Store_Intrinsic<"qvstfs">;
+  def int_ppc_qpx_qvstfsa : PowerPC_QPX_Store_Intrinsic<"qvstfsa">;
+
+  def int_ppc_qpx_qvstfcda : PowerPC_QPX_Store_Intrinsic<"qvstfcda">;
+  def int_ppc_qpx_qvstfcd : PowerPC_QPX_Store_Intrinsic<"qvstfcd">;
+  def int_ppc_qpx_qvstfcsa : PowerPC_QPX_Store_Intrinsic<"qvstfcsa">;
+  def int_ppc_qpx_qvstfcs : PowerPC_QPX_Store_Intrinsic<"qvstfcs">;
+  def int_ppc_qpx_qvstfiwa : PowerPC_QPX_Store_Intrinsic<"qvstfiwa">;
+  def int_ppc_qpx_qvstfiw : PowerPC_QPX_Store_Intrinsic<"qvstfiw">;
+
+  // Logical and permutation formation
+  def int_ppc_qpx_qvflogical : PowerPC_QPX_Intrinsic<"qvflogical",
+                          [llvm_v4f64_ty],
+                          [llvm_v4f64_ty, llvm_v4f64_ty, llvm_i32_ty],
+                          [IntrNoMem]>;
+  def int_ppc_qpx_qvgpci : PowerPC_QPX_Intrinsic<"qvgpci",
+                          [llvm_v4f64_ty], [llvm_i32_ty], [IntrNoMem]>;
+}
+
 //===----------------------------------------------------------------------===//
 // PowerPC HTM Intrinsic Definitions.
 
diff --git a/llvm/lib/Support/Triple.cpp b/llvm/lib/Support/Triple.cpp
index 72648273b4cd5..fec1985ccacae 100644
--- a/llvm/lib/Support/Triple.cpp
+++ b/llvm/lib/Support/Triple.cpp
@@ -160,6 +160,8 @@ StringRef Triple::getVendorTypeName(VendorType Kind) {
 
   case AMD: return "amd";
   case Apple: return "apple";
+  case BGP: return "bgp";
+  case BGQ: return "bgq";
   case CSR: return "csr";
   case Freescale: return "fsl";
   case IBM: return "ibm";
@@ -185,6 +187,7 @@ StringRef Triple::getOSTypeName(OSType Kind) {
   case AMDHSA: return "amdhsa";
   case AMDPAL: return "amdpal";
   case Ananas: return "ananas";
+  case CNK: return "cnk";
   case CUDA: return "cuda";
   case CloudABI: return "cloudabi";
   case Contiki: return "contiki";
@@ -467,6 +470,8 @@ static Triple::VendorType parseVendor(StringRef VendorName) {
     .Case("apple", Triple::Apple)
     .Case("pc", Triple::PC)
     .Case("scei", Triple::SCEI)
+    .Case("bgp", Triple::BGP)
+    .Case("bgq", Triple::BGQ)
     .Case("fsl", Triple::Freescale)
     .Case("ibm", Triple::IBM)
     .Case("img", Triple::ImaginationTechnologies)
@@ -503,6 +508,7 @@ static Triple::OSType parseOS(StringRef OSName) {
     .StartsWith("minix", Triple::Minix)
     .StartsWith("rtems", Triple::RTEMS)
     .StartsWith("nacl", Triple::NaCl)
+    .StartsWith("cnk", Triple::CNK)
     .StartsWith("aix", Triple::AIX)
     .StartsWith("cuda", Triple::CUDA)
     .StartsWith("nvcl", Triple::NVCL)
diff --git a/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp b/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp
index 81008d3ea5662..13fd7d05ab9f4 100644
--- a/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp
+++ b/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp
@@ -492,6 +492,21 @@ struct PPCOperand : public MCParsedAsmOperand {
     Inst.addOperand(MCOperand::createReg(VSSRegs[getVSReg()]));
   }
 
+  void addRegQFRCOperands(MCInst &Inst, unsigned N) const {
+    assert(N == 1 && "Invalid number of operands!");
+    Inst.addOperand(MCOperand::createReg(QFRegs[getReg()]));
+  }
+
+  void addRegQSRCOperands(MCInst &Inst, unsigned N) const {
+    assert(N == 1 && "Invalid number of operands!");
+    Inst.addOperand(MCOperand::createReg(QFRegs[getReg()]));
+  }
+
+  void addRegQBRCOperands(MCInst &Inst, unsigned N) const {
+    assert(N == 1 && "Invalid number of operands!");
+    Inst.addOperand(MCOperand::createReg(QFRegs[getReg()]));
+  }
+
   void addRegSPE4RCOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
     Inst.addOperand(MCOperand::createReg(RRegs[getReg()]));
@@ -1192,6 +1207,9 @@ bool PPCAsmParser::MatchRegisterName(unsigned &RegNo, int64_t &IntVal) {
     } else if (Name.startswith_lower("v") &&
                !Name.substr(1).getAsInteger(10, IntVal) && IntVal < 32) {
       RegNo = VRegs[IntVal];
+    } else if (Name.startswith_lower("q") &&
+               !Name.substr(1).getAsInteger(10, IntVal) && IntVal < 32) {
+      RegNo = QFRegs[IntVal];
     } else if (Name.startswith_lower("cr") &&
                !Name.substr(2).getAsInteger(10, IntVal) && IntVal < 8) {
       RegNo = CRRegs[IntVal];
diff --git a/llvm/lib/Target/PowerPC/CMakeLists.txt b/llvm/lib/Target/PowerPC/CMakeLists.txt
index 5a06faa16be19..91021d4e584e1 100644
--- a/llvm/lib/Target/PowerPC/CMakeLists.txt
+++ b/llvm/lib/Target/PowerPC/CMakeLists.txt
@@ -36,6 +36,7 @@ add_llvm_target(PowerPCCodeGen
   PPCMacroFusion.cpp
   PPCMIPeephole.cpp
   PPCRegisterInfo.cpp
+  PPCQPXLoadSplat.cpp
   PPCSubtarget.cpp
   PPCTargetMachine.cpp
   PPCTargetObjectFile.cpp
diff --git a/llvm/lib/Target/PowerPC/Disassembler/PPCDisassembler.cpp b/llvm/lib/Target/PowerPC/Disassembler/PPCDisassembler.cpp
index 362ddf7204557..74c6fd3733f03 100644
--- a/llvm/lib/Target/PowerPC/Disassembler/PPCDisassembler.cpp
+++ b/llvm/lib/Target/PowerPC/Disassembler/PPCDisassembler.cpp
@@ -167,6 +167,12 @@ static DecodeStatus DecodeG8RC_NOX0RegisterClass(MCInst &Inst, uint64_t RegNo,
 #define DecodePointerLikeRegClass0 DecodeGPRCRegisterClass
 #define DecodePointerLikeRegClass1 DecodeGPRC_NOR0RegisterClass
 
+static DecodeStatus DecodeQFRCRegisterClass(MCInst &Inst, uint64_t RegNo,
+                                            uint64_t Address,
+                                            const void *Decoder) {
+  return decodeRegisterClass(Inst, RegNo, QFRegs);
+}
+
 static DecodeStatus DecodeSPERCRegisterClass(MCInst &Inst, uint64_t RegNo,
                                             uint64_t Address,
                                             const void *Decoder) {
@@ -395,9 +401,14 @@ DecodeStatus PPCDisassembler::getInstruction(MCInst &MI, uint64_t &Size,
   // Read the instruction in the proper endianness.
   uint64_t Inst = ReadFunc(Bytes.data());
 
-  if (STI.getFeatureBits()[PPC::FeatureSPE]) {
+  if (STI.getFeatureBits()[PPC::FeatureQPX]) {
+    DecodeStatus result =
+      decodeInstruction(DecoderTableQPX32, MI, Inst, Address, this, STI);
+    if (result != MCDisassembler::Fail)
+      return result;
+  } else if (STI.getFeatureBits()[PPC::FeatureSPE]) {
     DecodeStatus result =
-        decodeInstruction(DecoderTableSPE32, MI, Inst, Address, this, STI);
+      decodeInstruction(DecoderTableSPE32, MI, Inst, Address, this, STI);
     if (result != MCDisassembler::Fail)
       return result;
   }
diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCInstPrinter.cpp b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCInstPrinter.cpp
index ce1a43a0c25b2..222bf2fa82836 100644
--- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCInstPrinter.cpp
+++ b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCInstPrinter.cpp
@@ -49,6 +49,18 @@ FullRegNamesWithPercent("ppc-reg-with-percent-prefix", cl::Hidden,
 
 void PPCInstPrinter::printRegName(raw_ostream &OS, unsigned RegNo) const {
   const char *RegName = getRegisterName(RegNo);
+  if (RegName[0] == 'q' /* QPX */) {
+    // The system toolchain on the BG/Q does not understand QPX register names
+    // in .cfi_* directives, so print the name of the floating-point
+    // subregister instead.
+    std::string RN(RegName);
+
+    RN[0] = 'f';
+    OS << RN;
+
+    return;
+  }
+
   OS << RegName;
 }
 
diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.h b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.h
index 325ede0fc17ac..719e005d98135 100644
--- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.h
+++ b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.h
@@ -159,6 +159,7 @@ using llvm::MCPhysReg;
   static const MCPhysReg SPERegs[32] = PPC_REGS0_31(PPC::S); \
   static const MCPhysReg VFRegs[32] = PPC_REGS0_31(PPC::VF); \
   static const MCPhysReg VRegs[32] = PPC_REGS0_31(PPC::V); \
+  static const MCPhysReg QFRegs[32] = PPC_REGS0_31(PPC::QF); \
   static const MCPhysReg RRegsNoR0[32] = \
     PPC_REGS_NO0_31(PPC::ZERO, PPC::R); \
   static const MCPhysReg XRegsNoX0[32] = \
diff --git a/llvm/lib/Target/PowerPC/PPC.h b/llvm/lib/Target/PowerPC/PPC.h
index 24a9d419d3ea5..3106290442afa 100644
--- a/llvm/lib/Target/PowerPC/PPC.h
+++ b/llvm/lib/Target/PowerPC/PPC.h
@@ -44,6 +44,7 @@ namespace llvm {
   FunctionPass *createPPCMIPeepholePass();
   FunctionPass *createPPCBranchSelectionPass();
   FunctionPass *createPPCBranchCoalescingPass();
+  FunctionPass *createPPCQPXLoadSplatPass();
   FunctionPass *createPPCISelDag(PPCTargetMachine &TM, CodeGenOpt::Level OL);
   FunctionPass *createPPCTLSDynamicCallPass();
   FunctionPass *createPPCBoolRetToIntPass();
@@ -67,6 +68,7 @@ namespace llvm {
   void initializePPCReduceCRLogicalsPass(PassRegistry&);
   void initializePPCBSelPass(PassRegistry&);
   void initializePPCBranchCoalescingPass(PassRegistry&);
+  void initializePPCQPXLoadSplatPass(PassRegistry&);
   void initializePPCBoolRetToIntPass(PassRegistry&);
   void initializePPCExpandISELPass(PassRegistry &);
   void initializePPCPreEmitPeepholePass(PassRegistry &);
diff --git a/llvm/lib/Target/PowerPC/PPC.td b/llvm/lib/Target/PowerPC/PPC.td
index adb9366217d51..9ad78bf67fe6c 100644
--- a/llvm/lib/Target/PowerPC/PPC.td
+++ b/llvm/lib/Target/PowerPC/PPC.td
@@ -132,6 +132,9 @@ def FeaturePPC4xx    : SubtargetFeature<"ppc4xx", "IsPPC4xx", "true",
                                         "Enable PPC 4xx instructions">;
 def FeaturePPC6xx    : SubtargetFeature<"ppc6xx", "IsPPC6xx", "true",
                                         "Enable PPC 6xx instructions">;
+def FeatureQPX       : SubtargetFeature<"qpx","HasQPX", "true",
+                                        "Enable QPX instructions",
+                                        [FeatureFPU]>;
 def FeatureVSX       : SubtargetFeature<"vsx","HasVSX", "true",
                                         "Enable VSX instructions",
                                         [FeatureAltivec]>;
@@ -190,7 +193,7 @@ def FeatureFloat128 :
 def FeaturePOPCNTD   : SubtargetFeature<"popcntd","HasPOPCNTD",
                                         "POPCNTD_Fast",
                                         "Enable the popcnt[dw] instructions">;
-// Note that for the a2 processor models we should not use popcnt[dw] by
+// Note that for the a2/a2q processor models we should not use popcnt[dw] by
 // default. These processors do support the instructions, but they're
 // microcoded, and the software emulation is about twice as fast.
 def FeatureSlowPOPCNTD : SubtargetFeature<"slow-popcntd","HasPOPCNTD",
@@ -511,6 +514,15 @@ def : ProcessorModel<"a2", PPCA2Model,
                    FeatureFPRND, FeatureFPCVT, FeatureISEL,
                    FeatureSlowPOPCNTD, FeatureCMPB, FeatureLDBRX,
                    Feature64Bit /*, Feature64BitRegs */, FeatureMFTB]>;
+def : ProcessorModel<"a2q", PPCA2Model,
+                  [DirectiveA2, FeatureICBT, FeatureBookE, FeatureMFOCRF,
+                   FeatureFCPSGN, FeatureFSqrt, FeatureFRE, FeatureFRES,
+                   FeatureFRSQRTE, FeatureFRSQRTES, FeatureRecipPrec,
+                   FeatureSTFIWX, FeatureLFIWAX,
+                   FeatureFPRND, FeatureFPCVT, FeatureISEL,
+                   FeatureSlowPOPCNTD, FeatureCMPB, FeatureLDBRX,
+                   Feature64Bit /*, Feature64BitRegs */, FeatureQPX,
+                   FeatureMFTB]>;
 def : ProcessorModel<"pwr3", G5Model,
                   [DirectivePwr3, FeatureAltivec,
                    FeatureFRES, FeatureFRSQRTE, FeatureMFOCRF,
diff --git a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
index 5affddd8d147a..540e620a845bc 100644
--- a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
+++ b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
@@ -549,6 +549,9 @@ void PPCAsmPrinter::emitInstruction(const MachineInstr *MI) {
         if (Subtarget->hasSPE()) {
           if (PPC::F4RCRegClass.contains(Reg) ||
               PPC::F8RCRegClass.contains(Reg) ||
+              PPC::QBRCRegClass.contains(Reg) ||
+              PPC::QFRCRegClass.contains(Reg) ||
+              PPC::QSRCRegClass.contains(Reg) ||
               PPC::VFRCRegClass.contains(Reg) ||
               PPC::VRRCRegClass.contains(Reg) ||
               PPC::VSFRCRegClass.contains(Reg) ||
diff --git a/llvm/lib/Target/PowerPC/PPCCallingConv.td b/llvm/lib/Target/PowerPC/PPCCallingConv.td
index 9a15490f1fb0d..1eaa7f7a44b39 100644
--- a/llvm/lib/Target/PowerPC/PPCCallingConv.td
+++ b/llvm/lib/Target/PowerPC/PPCCallingConv.td
@@ -61,6 +61,9 @@ def RetCC_PPC_Cold : CallingConv<[
   CCIfType<[f64], CCAssignToReg<[F1]>>,
   CCIfType<[f128], CCIfSubtarget<"hasP9Vector()", CCAssignToReg<[V2]>>>,
 
+  CCIfType<[v4f64, v4f32, v4i1],
+           CCIfSubtarget<"hasQPX()", CCAssignToReg<[QF1]>>>,
+
   CCIfType<[v16i8, v8i16, v4i32, v2i64, v1i128, v4f32, v2f64],
            CCIfSubtarget<"hasAltivec()",
            CCAssignToReg<[V2]>>>
@@ -95,6 +98,10 @@ def RetCC_PPC : CallingConv<[
            CCIfSubtarget<"hasP9Vector()",
            CCAssignToReg<[V2, V3, V4, V5, V6, V7, V8, V9]>>>,
 
+  // QPX vectors are returned in QF1 and QF2. 
+  CCIfType<[v4f64, v4f32, v4i1],
+           CCIfSubtarget<"hasQPX()", CCAssignToReg<[QF1, QF2]>>>,
+ 
   // Vector types returned as "direct" go into V2 .. V9; note that only the
   // ELFv2 ABI fully utilizes all these registers.
   CCIfType<[v16i8, v8i16, v4i32, v2i64, v1i128, v4f32, v2f64],
@@ -151,6 +158,8 @@ def RetCC_PPC64_ELF_FIS : CallingConv<[
   CCIfType<[f128],
            CCIfSubtarget<"hasP9Vector()",
            CCAssignToReg<[V2, V3, V4, V5, V6, V7, V8, V9]>>>,
+  CCIfType<[v4f64, v4f32, v4i1],
+           CCIfSubtarget<"hasQPX()", CCAssignToReg<[QF1, QF2]>>>,
   CCIfType<[v16i8, v8i16, v4i32, v2i64, v1i128, v4f32, v2f64],
            CCIfSubtarget<"hasAltivec()",
            CCAssignToReg<[V2, V3, V4, V5, V6, V7, V8, V9]>>>
@@ -214,6 +223,9 @@ def CC_PPC32_SVR4_Common : CallingConv<[
   CCIfType<[f32], CCIfSubtarget<"hasSPE()", CCAssignToStack<4, 4>>>,
   CCIfType<[f64], CCIfSubtarget<"hasSPE()", CCAssignToStack<8, 8>>>,
 
+  // QPX vectors that are stored in double precision need 32-byte alignment.
+  CCIfType<[v4f64, v4i1], CCAssignToStack<32, 32>>,
+
   // Vectors and float128 get 16-byte stack slots that are 16-byte aligned.
   CCIfType<[v16i8, v8i16, v4i32, v4f32, v2f64, v2i64], CCAssignToStack<16, 16>>,
   CCIfType<[f128], CCIfSubtarget<"hasP9Vector()", CCAssignToStack<16, 16>>>
@@ -231,6 +243,10 @@ def CC_PPC32_SVR4_VarArg : CallingConv<[
 // put vector arguments in vector registers before putting them on the stack.
 let Entry = 1 in
 def CC_PPC32_SVR4 : CallingConv<[
+  // QPX vectors mirror the scalar FP convention.
+  CCIfType<[v4f64, v4f32, v4i1], CCIfSubtarget<"hasQPX()",
+    CCAssignToReg<[QF1, QF2, QF3, QF4, QF5, QF6, QF7, QF8]>>>,
+
   // The first 12 Vector arguments are passed in AltiVec registers.
   CCIfType<[v16i8, v8i16, v4i32, v2i64, v1i128, v4f32, v2f64],
            CCIfSubtarget<"hasAltivec()", CCAssignToReg<[V2, V3, V4, V5, V6, V7,
diff --git a/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp b/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp
index 3e218e14d8d44..8ffd89ef5ccd2 100644
--- a/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp
@@ -4142,7 +4142,7 @@ bool PPCDAGToDAGISel::trySETCC(SDNode *N) {
   // Altivec Vector compare instructions do not set any CR register by default and
   // vector compare operations return the same type as the operands.
   if (LHS.getValueType().isVector()) {
-    if (Subtarget->hasSPE())
+    if (Subtarget->hasQPX() || Subtarget->hasSPE())
       return false;
 
     EVT VecVT = LHS.getValueType();
@@ -4813,6 +4813,8 @@ void PPCDAGToDAGISel::Select(SDNode *N) {
         assert((!isSExt || LoadedVT == MVT::i16) && "Invalid sext update load");
         switch (LoadedVT.getSimpleVT().SimpleTy) {
           default: llvm_unreachable("Invalid PPC load type!");
+          case MVT::v4f64: Opcode = PPC::QVLFDUX; break; // QPX
+          case MVT::v4f32: Opcode = PPC::QVLFSUX; break; // QPX
           case MVT::f64: Opcode = PPC::LFDUX; break;
           case MVT::f32: Opcode = PPC::LFSUX; break;
           case MVT::i32: Opcode = PPC::LWZUX; break;
@@ -5093,6 +5095,12 @@ void PPCDAGToDAGISel::Select(SDNode *N) {
       SelectCCOp = PPC::SELECT_CC_F16;
     else if (Subtarget->hasSPE())
       SelectCCOp = PPC::SELECT_CC_SPE;
+    else if (Subtarget->hasQPX() && N->getValueType(0) == MVT::v4f64)
+      SelectCCOp = PPC::SELECT_CC_QFRC;
+    else if (Subtarget->hasQPX() && N->getValueType(0) == MVT::v4f32)
+      SelectCCOp = PPC::SELECT_CC_QSRC;
+    else if (Subtarget->hasQPX() && N->getValueType(0) == MVT::v4i1)
+      SelectCCOp = PPC::SELECT_CC_QBRC;
     else if (N->getValueType(0) == MVT::v2f64 ||
              N->getValueType(0) == MVT::v2i64)
       SelectCCOp = PPC::SELECT_CC_VSRC;
@@ -5848,6 +5856,9 @@ void PPCDAGToDAGISel::PeepholeCROps() {
       case PPC::SELECT_I8:
       case PPC::SELECT_F4:
       case PPC::SELECT_F8:
+      case PPC::SELECT_QFRC:
+      case PPC::SELECT_QSRC:
+      case PPC::SELECT_QBRC:
       case PPC::SELECT_SPE:
       case PPC::SELECT_SPE4:
       case PPC::SELECT_VRRC:
@@ -6166,6 +6177,9 @@ void PPCDAGToDAGISel::PeepholeCROps() {
       case PPC::SELECT_I8:
       case PPC::SELECT_F4:
       case PPC::SELECT_F8:
+      case PPC::SELECT_QFRC:
+      case PPC::SELECT_QSRC:
+      case PPC::SELECT_QBRC:
       case PPC::SELECT_SPE:
       case PPC::SELECT_SPE4:
       case PPC::SELECT_VRRC:
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index db3833d595797..ae840a9fa37de 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -1095,6 +1095,161 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM,
     }
   }
 
+  if (Subtarget.hasQPX()) {
+    setOperationAction(ISD::FADD, MVT::v4f64, Legal);
+    setOperationAction(ISD::FSUB, MVT::v4f64, Legal);
+    setOperationAction(ISD::FMUL, MVT::v4f64, Legal);
+    setOperationAction(ISD::FREM, MVT::v4f64, Expand);
+
+    setOperationAction(ISD::FCOPYSIGN, MVT::v4f64, Legal);
+    setOperationAction(ISD::FGETSIGN, MVT::v4f64, Expand);
+
+    setOperationAction(ISD::LOAD  , MVT::v4f64, Custom);
+    setOperationAction(ISD::STORE , MVT::v4f64, Custom);
+
+    setTruncStoreAction(MVT::v4f64, MVT::v4f32, Custom);
+    setLoadExtAction(ISD::EXTLOAD, MVT::v4f64, MVT::v4f32, Custom);
+
+    if (!Subtarget.useCRBits())
+      setOperationAction(ISD::SELECT, MVT::v4f64, Expand);
+    setOperationAction(ISD::VSELECT, MVT::v4f64, Legal);
+
+    setOperationAction(ISD::EXTRACT_VECTOR_ELT , MVT::v4f64, Legal);
+    setOperationAction(ISD::INSERT_VECTOR_ELT , MVT::v4f64, Expand);
+    setOperationAction(ISD::CONCAT_VECTORS , MVT::v4f64, Expand);
+    setOperationAction(ISD::EXTRACT_SUBVECTOR , MVT::v4f64, Expand);
+    setOperationAction(ISD::VECTOR_SHUFFLE , MVT::v4f64, Custom);
+    setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4f64, Legal);
+    setOperationAction(ISD::BUILD_VECTOR, MVT::v4f64, Custom);
+
+    setOperationAction(ISD::FP_TO_SINT , MVT::v4f64, Legal);
+    setOperationAction(ISD::FP_TO_UINT , MVT::v4f64, Expand);
+
+    setOperationAction(ISD::FP_ROUND , MVT::v4f32, Legal);
+    setOperationAction(ISD::FP_EXTEND, MVT::v4f64, Legal);
+
+    setOperationAction(ISD::FNEG , MVT::v4f64, Legal);
+    setOperationAction(ISD::FABS , MVT::v4f64, Legal);
+    setOperationAction(ISD::FSIN , MVT::v4f64, Expand);
+    setOperationAction(ISD::FCOS , MVT::v4f64, Expand);
+    setOperationAction(ISD::FPOW , MVT::v4f64, Expand);
+    setOperationAction(ISD::FLOG , MVT::v4f64, Expand);
+    setOperationAction(ISD::FLOG2 , MVT::v4f64, Expand);
+    setOperationAction(ISD::FLOG10 , MVT::v4f64, Expand);
+    setOperationAction(ISD::FEXP , MVT::v4f64, Expand);
+    setOperationAction(ISD::FEXP2 , MVT::v4f64, Expand);
+
+    setOperationAction(ISD::FMINNUM, MVT::v4f64, Legal);
+    setOperationAction(ISD::FMAXNUM, MVT::v4f64, Legal);
+
+    setIndexedLoadAction(ISD::PRE_INC, MVT::v4f64, Legal);
+    setIndexedStoreAction(ISD::PRE_INC, MVT::v4f64, Legal);
+
+    addRegisterClass(MVT::v4f64, &PPC::QFRCRegClass);
+
+    setOperationAction(ISD::FADD, MVT::v4f32, Legal);
+    setOperationAction(ISD::FSUB, MVT::v4f32, Legal);
+    setOperationAction(ISD::FMUL, MVT::v4f32, Legal);
+    setOperationAction(ISD::FREM, MVT::v4f32, Expand);
+
+    setOperationAction(ISD::FCOPYSIGN, MVT::v4f32, Legal);
+    setOperationAction(ISD::FGETSIGN, MVT::v4f32, Expand);
+
+    setOperationAction(ISD::LOAD  , MVT::v4f32, Custom);
+    setOperationAction(ISD::STORE , MVT::v4f32, Custom);
+
+    if (!Subtarget.useCRBits())
+      setOperationAction(ISD::SELECT, MVT::v4f32, Expand);
+    setOperationAction(ISD::VSELECT, MVT::v4f32, Legal);
+
+    setOperationAction(ISD::EXTRACT_VECTOR_ELT , MVT::v4f32, Legal);
+    setOperationAction(ISD::INSERT_VECTOR_ELT , MVT::v4f32, Expand);
+    setOperationAction(ISD::CONCAT_VECTORS , MVT::v4f32, Expand);
+    setOperationAction(ISD::EXTRACT_SUBVECTOR , MVT::v4f32, Expand);
+    setOperationAction(ISD::VECTOR_SHUFFLE , MVT::v4f32, Custom);
+    setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4f32, Legal);
+    setOperationAction(ISD::BUILD_VECTOR, MVT::v4f32, Custom);
+
+    setOperationAction(ISD::FP_TO_SINT , MVT::v4f32, Legal);
+    setOperationAction(ISD::FP_TO_UINT , MVT::v4f32, Expand);
+
+    setOperationAction(ISD::FNEG , MVT::v4f32, Legal);
+    setOperationAction(ISD::FABS , MVT::v4f32, Legal);
+    setOperationAction(ISD::FSIN , MVT::v4f32, Expand);
+    setOperationAction(ISD::FCOS , MVT::v4f32, Expand);
+    setOperationAction(ISD::FPOW , MVT::v4f32, Expand);
+    setOperationAction(ISD::FLOG , MVT::v4f32, Expand);
+    setOperationAction(ISD::FLOG2 , MVT::v4f32, Expand);
+    setOperationAction(ISD::FLOG10 , MVT::v4f32, Expand);
+    setOperationAction(ISD::FEXP , MVT::v4f32, Expand);
+    setOperationAction(ISD::FEXP2 , MVT::v4f32, Expand);
+
+    setOperationAction(ISD::FMINNUM, MVT::v4f32, Legal);
+    setOperationAction(ISD::FMAXNUM, MVT::v4f32, Legal);
+
+    setIndexedLoadAction(ISD::PRE_INC, MVT::v4f32, Legal);
+    setIndexedStoreAction(ISD::PRE_INC, MVT::v4f32, Legal);
+
+    addRegisterClass(MVT::v4f32, &PPC::QSRCRegClass);
+
+    setOperationAction(ISD::AND , MVT::v4i1, Legal);
+    setOperationAction(ISD::OR , MVT::v4i1, Legal);
+    setOperationAction(ISD::XOR , MVT::v4i1, Legal);
+
+    if (!Subtarget.useCRBits())
+      setOperationAction(ISD::SELECT, MVT::v4i1, Expand);
+    setOperationAction(ISD::VSELECT, MVT::v4i1, Legal);
+
+    setOperationAction(ISD::LOAD  , MVT::v4i1, Custom);
+    setOperationAction(ISD::STORE , MVT::v4i1, Custom);
+
+    setOperationAction(ISD::EXTRACT_VECTOR_ELT , MVT::v4i1, Custom);
+    setOperationAction(ISD::INSERT_VECTOR_ELT , MVT::v4i1, Expand);
+    setOperationAction(ISD::CONCAT_VECTORS , MVT::v4i1, Expand);
+    setOperationAction(ISD::EXTRACT_SUBVECTOR , MVT::v4i1, Expand);
+    setOperationAction(ISD::VECTOR_SHUFFLE , MVT::v4i1, Custom);
+    setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4i1, Expand);
+    setOperationAction(ISD::BUILD_VECTOR, MVT::v4i1, Custom);
+
+    setOperationAction(ISD::SINT_TO_FP, MVT::v4i1, Custom);
+    setOperationAction(ISD::UINT_TO_FP, MVT::v4i1, Custom);
+
+    addRegisterClass(MVT::v4i1, &PPC::QBRCRegClass);
+
+    setOperationAction(ISD::FFLOOR, MVT::v4f64, Legal);
+    setOperationAction(ISD::FCEIL,  MVT::v4f64, Legal);
+    setOperationAction(ISD::FTRUNC, MVT::v4f64, Legal);
+    setOperationAction(ISD::FROUND, MVT::v4f64, Legal);
+
+    setOperationAction(ISD::FFLOOR, MVT::v4f32, Legal);
+    setOperationAction(ISD::FCEIL,  MVT::v4f32, Legal);
+    setOperationAction(ISD::FTRUNC, MVT::v4f32, Legal);
+    setOperationAction(ISD::FROUND, MVT::v4f32, Legal);
+
+    setOperationAction(ISD::FNEARBYINT, MVT::v4f64, Expand);
+    setOperationAction(ISD::FNEARBYINT, MVT::v4f32, Expand);
+
+    // These need to set FE_INEXACT, and so cannot be vectorized here.
+    setOperationAction(ISD::FRINT, MVT::v4f64, Expand);
+    setOperationAction(ISD::FRINT, MVT::v4f32, Expand);
+
+    if (TM.Options.UnsafeFPMath) {
+      setOperationAction(ISD::FDIV, MVT::v4f64, Legal);
+      setOperationAction(ISD::FSQRT, MVT::v4f64, Legal);
+
+      setOperationAction(ISD::FDIV, MVT::v4f32, Legal);
+      setOperationAction(ISD::FSQRT, MVT::v4f32, Legal);
+    } else {
+      setOperationAction(ISD::FDIV, MVT::v4f64, Expand);
+      setOperationAction(ISD::FSQRT, MVT::v4f64, Expand);
+
+      setOperationAction(ISD::FDIV, MVT::v4f32, Expand);
+      setOperationAction(ISD::FSQRT, MVT::v4f32, Expand);
+    }
+
+    // TODO: Handle constrained floating-point operations of v4f64
+  }
+
   if (Subtarget.has64BitSupport())
     setOperationAction(ISD::PREFETCH, MVT::Other, Legal);
 
@@ -1283,8 +1438,8 @@ unsigned PPCTargetLowering::getByValTypeAlignment(Type *Ty,
   // 16byte and wider vectors are passed on 16byte boundary.
   // The rest is 8 on PPC64 and 4 on PPC32 boundary.
   Align Alignment = Subtarget.isPPC64() ? Align(8) : Align(4);
-  if (Subtarget.hasAltivec())
-    getMaxByValAlign(Ty, Alignment, Align(16));
+  if (Subtarget.hasAltivec() || Subtarget.hasQPX())
+    getMaxByValAlign(Ty, Alignment, Subtarget.hasQPX() ? Align(32) : Align(16));
   return Alignment.value();
 }
 
@@ -1422,6 +1577,12 @@ const char *PPCTargetLowering::getTargetNodeName(unsigned Opcode) const {
   case PPCISD::XXSWAPD:         return "PPCISD::XXSWAPD";
   case PPCISD::SWAP_NO_CHAIN:   return "PPCISD::SWAP_NO_CHAIN";
   case PPCISD::VABSD:           return "PPCISD::VABSD";
+  case PPCISD::QVFPERM:         return "PPCISD::QVFPERM";
+  case PPCISD::QVGPCI:          return "PPCISD::QVGPCI";
+  case PPCISD::QVALIGNI:        return "PPCISD::QVALIGNI";
+  case PPCISD::QVESPLATI:       return "PPCISD::QVESPLATI";
+  case PPCISD::QBFLT:           return "PPCISD::QBFLT";
+  case PPCISD::QVLFSb:          return "PPCISD::QVLFSb";
   case PPCISD::BUILD_FP128:     return "PPCISD::BUILD_FP128";
   case PPCISD::BUILD_SPE64:     return "PPCISD::BUILD_SPE64";
   case PPCISD::EXTRACT_SPE:     return "PPCISD::EXTRACT_SPE";
@@ -1440,6 +1601,9 @@ EVT PPCTargetLowering::getSetCCResultType(const DataLayout &DL, LLVMContext &C,
   if (!VT.isVector())
     return Subtarget.useCRBits() ? MVT::i1 : MVT::i32;
 
+  if (Subtarget.hasQPX())
+    return EVT::getVectorVT(C, MVT::i1, VT.getVectorNumElements());
+
   return VT.changeVectorElementTypeToInteger();
 }
 
@@ -2613,9 +2777,16 @@ bool PPCTargetLowering::getPreIndexedAddressParts(SDNode *N, SDValue &Base,
     return false;
   }
 
-  // PowerPC doesn't have preinc load/store instructions for vectors
-  if (VT.isVector())
-    return false;
+  // PowerPC doesn't have preinc load/store instructions for vectors (except
+  // for QPX, which does have preinc r+r forms).
+  if (VT.isVector()) {
+    if (!Subtarget.hasQPX() || (VT != MVT::v4f64 && VT != MVT::v4f32)) {
+      return false;
+    } else if (SelectAddressRegRegOnly(Ptr, Offset, Base, DAG)) {
+      AM = ISD::PRE_INC;
+      return true;
+    }
+  }
 
   if (SelectAddressRegReg(Ptr, Base, Offset, DAG)) {
     // Common code will reject creating a pre-inc form if the base pointer
@@ -3337,6 +3508,11 @@ static const MCPhysReg FPR[] = {PPC::F1,  PPC::F2,  PPC::F3, PPC::F4, PPC::F5,
                                 PPC::F6,  PPC::F7,  PPC::F8, PPC::F9, PPC::F10,
                                 PPC::F11, PPC::F12, PPC::F13};
 
+/// QFPR - The set of QPX registers that should be allocated for arguments.
+static const MCPhysReg QFPR[] = {
+    PPC::QF1, PPC::QF2, PPC::QF3,  PPC::QF4,  PPC::QF5,  PPC::QF6, PPC::QF7,
+    PPC::QF8, PPC::QF9, PPC::QF10, PPC::QF11, PPC::QF12, PPC::QF13};
+
 /// CalculateStackSlotSize - Calculates the size reserved for this argument on
 /// the stack.
 static unsigned CalculateStackSlotSize(EVT ArgVT, ISD::ArgFlagsTy Flags,
@@ -3366,6 +3542,10 @@ static Align CalculateStackSlotAlignment(EVT ArgVT, EVT OrigVT,
       ArgVT == MVT::v2f64 || ArgVT == MVT::v2i64 ||
       ArgVT == MVT::v1i128 || ArgVT == MVT::f128)
     Alignment = Align(16);
+  // QPX vector types stored in double-precision are padded to a 32 byte
+  // boundary.
+  else if (ArgVT == MVT::v4f64 || ArgVT == MVT::v4i1)
+    Alignment = Align(32);
 
   // ByVal parameters are aligned as requested.
   if (Flags.isByVal()) {
@@ -3397,11 +3577,14 @@ static Align CalculateStackSlotAlignment(EVT ArgVT, EVT OrigVT,
 /// stack slot (instead of being passed in registers).  ArgOffset,
 /// AvailableFPRs, and AvailableVRs must hold the current argument
 /// position, and will be updated to account for this argument.
-static bool CalculateStackSlotUsed(EVT ArgVT, EVT OrigVT, ISD::ArgFlagsTy Flags,
-                                   unsigned PtrByteSize, unsigned LinkageSize,
-                                   unsigned ParamAreaSize, unsigned &ArgOffset,
+static bool CalculateStackSlotUsed(EVT ArgVT, EVT OrigVT,
+                                   ISD::ArgFlagsTy Flags,
+                                   unsigned PtrByteSize,
+                                   unsigned LinkageSize,
+                                   unsigned ParamAreaSize,
+                                   unsigned &ArgOffset,
                                    unsigned &AvailableFPRs,
-                                   unsigned &AvailableVRs) {
+                                   unsigned &AvailableVRs, bool HasQPX) {
   bool UseMemory = false;
 
   // Respect alignment of argument on the stack.
@@ -3425,7 +3608,11 @@ static bool CalculateStackSlotUsed(EVT ArgVT, EVT OrigVT, ISD::ArgFlagsTy Flags,
   // However, if the argument is actually passed in an FPR or a VR,
   // we don't use memory after all.
   if (!Flags.isByVal()) {
-    if (ArgVT == MVT::f32 || ArgVT == MVT::f64)
+    if (ArgVT == MVT::f32 || ArgVT == MVT::f64 ||
+        // QPX registers overlap with the scalar FP registers.
+        (HasQPX && (ArgVT == MVT::v4f32 ||
+                    ArgVT == MVT::v4f64 ||
+                    ArgVT == MVT::v4i1)))
       if (AvailableFPRs > 0) {
         --AvailableFPRs;
         return false;
@@ -3564,12 +3751,18 @@ SDValue PPCTargetLowering::LowerFormalArguments_32SVR4(
           RC = &PPC::VRRCRegClass;
           break;
         case MVT::v4f32:
-          RC = &PPC::VRRCRegClass;
+          RC = Subtarget.hasQPX() ? &PPC::QSRCRegClass : &PPC::VRRCRegClass;
           break;
         case MVT::v2f64:
         case MVT::v2i64:
           RC = &PPC::VRRCRegClass;
           break;
+        case MVT::v4f64:
+          RC = &PPC::QFRCRegClass;
+          break;
+        case MVT::v4i1:
+          RC = &PPC::QBRCRegClass;
+          break;
       }
 
       SDValue ArgValue;
@@ -3768,6 +3961,7 @@ SDValue PPCTargetLowering::LowerFormalArguments_64SVR4(
   const unsigned Num_GPR_Regs = array_lengthof(GPR);
   const unsigned Num_FPR_Regs = useSoftFloat() ? 0 : 13;
   const unsigned Num_VR_Regs  = array_lengthof(VR);
+  const unsigned Num_QFPR_Regs = Num_FPR_Regs;
 
   // Do a first pass over the arguments to determine whether the ABI
   // guarantees that our caller has allocated the parameter save area
@@ -3786,7 +3980,8 @@ SDValue PPCTargetLowering::LowerFormalArguments_64SVR4(
 
     if (CalculateStackSlotUsed(Ins[i].VT, Ins[i].ArgVT, Ins[i].Flags,
                                PtrByteSize, LinkageSize, ParamAreaSize,
-                               NumBytes, AvailableFPRs, AvailableVRs))
+                               NumBytes, AvailableFPRs, AvailableVRs,
+                               Subtarget.hasQPX()))
       HasParameterArea = true;
   }
 
@@ -3796,6 +3991,7 @@ SDValue PPCTargetLowering::LowerFormalArguments_64SVR4(
 
   unsigned ArgOffset = LinkageSize;
   unsigned GPR_idx = 0, FPR_idx = 0, VR_idx = 0;
+  unsigned &QFPR_idx = FPR_idx;
   SmallVector<SDValue, 8> MemOps;
   Function::const_arg_iterator FuncArg = MF.getFunction().arg_begin();
   unsigned CurArgIdx = 0;
@@ -4038,20 +4234,51 @@ SDValue PPCTargetLowering::LowerFormalArguments_64SVR4(
     case MVT::v2i64:
     case MVT::v1i128:
     case MVT::f128:
-      // These can be scalar arguments or elements of a vector array type
-      // passed directly.  The latter are used to implement ELFv2 homogenous
-      // vector aggregates.
-      if (VR_idx != Num_VR_Regs) {
-        unsigned VReg = MF.addLiveIn(VR[VR_idx], &PPC::VRRCRegClass);
+      if (!Subtarget.hasQPX()) {
+        // These can be scalar arguments or elements of a vector array type
+        // passed directly.  The latter are used to implement ELFv2 homogenous
+        // vector aggregates.
+        if (VR_idx != Num_VR_Regs) {
+          unsigned VReg = MF.addLiveIn(VR[VR_idx], &PPC::VRRCRegClass);
+          ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, ObjectVT);
+          ++VR_idx;
+        } else {
+          if (CallConv == CallingConv::Fast)
+            ComputeArgOffset();
+          needsLoad = true;
+        }
+        if (CallConv != CallingConv::Fast || needsLoad)
+          ArgOffset += 16;
+        break;
+      } // not QPX
+
+      assert(ObjectVT.getSimpleVT().SimpleTy == MVT::v4f32 &&
+             "Invalid QPX parameter type");
+      LLVM_FALLTHROUGH;
+
+    case MVT::v4f64:
+    case MVT::v4i1:
+      // QPX vectors are treated like their scalar floating-point subregisters
+      // (except that they're larger).
+      unsigned Sz = ObjectVT.getSimpleVT().SimpleTy == MVT::v4f32 ? 16 : 32;
+      if (QFPR_idx != Num_QFPR_Regs) {
+        const TargetRegisterClass *RC;
+        switch (ObjectVT.getSimpleVT().SimpleTy) {
+        case MVT::v4f64: RC = &PPC::QFRCRegClass; break;
+        case MVT::v4f32: RC = &PPC::QSRCRegClass; break;
+        default:         RC = &PPC::QBRCRegClass; break;
+        }
+
+        unsigned VReg = MF.addLiveIn(QFPR[QFPR_idx], RC);
         ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, ObjectVT);
-        ++VR_idx;
+        ++QFPR_idx;
       } else {
         if (CallConv == CallingConv::Fast)
           ComputeArgOffset();
         needsLoad = true;
       }
       if (CallConv != CallingConv::Fast || needsLoad)
-        ArgOffset += 16;
+        ArgOffset += Sz;
       break;
     }
 
@@ -4604,9 +4831,10 @@ needStackSlotPassParameters(const PPCSubtarget &Subtarget,
   for (const ISD::OutputArg& Param : Outs) {
     if (Param.Flags.isNest()) continue;
 
-    if (CalculateStackSlotUsed(Param.VT, Param.ArgVT, Param.Flags, PtrByteSize,
-                               LinkageSize, ParamAreaSize, NumBytes,
-                               AvailableFPRs, AvailableVRs))
+    if (CalculateStackSlotUsed(Param.VT, Param.ArgVT, Param.Flags,
+                               PtrByteSize, LinkageSize, ParamAreaSize,
+                               NumBytes, AvailableFPRs, AvailableVRs,
+                               Subtarget.hasQPX()))
       return true;
   }
   return false;
@@ -5836,6 +6064,7 @@ SDValue PPCTargetLowering::LowerCall_64SVR4(
   unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize();
   unsigned NumBytes = LinkageSize;
   unsigned GPR_idx = 0, FPR_idx = 0, VR_idx = 0;
+  unsigned &QFPR_idx = FPR_idx;
 
   static const MCPhysReg GPR[] = {
     PPC::X3, PPC::X4, PPC::X5, PPC::X6,
@@ -5849,6 +6078,7 @@ SDValue PPCTargetLowering::LowerCall_64SVR4(
   const unsigned NumGPRs = array_lengthof(GPR);
   const unsigned NumFPRs = useSoftFloat() ? 0 : 13;
   const unsigned NumVRs  = array_lengthof(VR);
+  const unsigned NumQFPRs = NumFPRs;
 
   // On ELFv2, we can avoid allocating the parameter area if all the arguments
   // can be passed to the callee in registers.
@@ -5863,8 +6093,9 @@ SDValue PPCTargetLowering::LowerCall_64SVR4(
     for (unsigned i = 0; i != NumOps; ++i) {
       if (Outs[i].Flags.isNest()) continue;
       if (CalculateStackSlotUsed(Outs[i].VT, Outs[i].ArgVT, Outs[i].Flags,
-                                 PtrByteSize, LinkageSize, ParamAreaSize,
-                                 NumBytesTmp, AvailableFPRs, AvailableVRs))
+                                PtrByteSize, LinkageSize, ParamAreaSize,
+                                NumBytesTmp, AvailableFPRs, AvailableVRs,
+                                Subtarget.hasQPX()))
         HasParameterArea = true;
     }
   }
@@ -5912,11 +6143,20 @@ SDValue PPCTargetLowering::LowerCall_64SVR4(
             continue;
           break;
         case MVT::v4f32:
-          if (++NumVRsUsed <= NumVRs)
-            continue;
+          // When using QPX, this is handled like a FP register, otherwise, it
+          // is an Altivec register.
+          if (Subtarget.hasQPX()) {
+            if (++NumFPRsUsed <= NumFPRs)
+              continue;
+          } else {
+            if (++NumVRsUsed <= NumVRs)
+              continue;
+          }
           break;
         case MVT::f32:
         case MVT::f64:
+        case MVT::v4f64: // QPX
+        case MVT::v4i1:  // QPX
           if (++NumFPRsUsed <= NumFPRs)
             continue;
           break;
@@ -6278,6 +6518,7 @@ SDValue PPCTargetLowering::LowerCall_64SVR4(
     case MVT::v2i64:
     case MVT::v1i128:
     case MVT::f128:
+      if (!Subtarget.hasQPX()) {
       // These can be scalar arguments or elements of a vector array type
       // passed directly.  The latter are used to implement ELFv2 homogenous
       // vector aggregates.
@@ -6333,6 +6574,63 @@ SDValue PPCTargetLowering::LowerCall_64SVR4(
       if (!IsFastCall)
         ArgOffset += 16;
       break;
+      } // not QPX
+
+      assert(Arg.getValueType().getSimpleVT().SimpleTy == MVT::v4f32 &&
+             "Invalid QPX parameter type");
+
+      LLVM_FALLTHROUGH;
+    case MVT::v4f64:
+    case MVT::v4i1: {
+      bool IsF32 = Arg.getValueType().getSimpleVT().SimpleTy == MVT::v4f32;
+      if (CFlags.IsVarArg) {
+        assert(HasParameterArea &&
+               "Parameter area must exist if we have a varargs call.");
+        // We could elide this store in the case where the object fits
+        // entirely in R registers.  Maybe later.
+        SDValue Store =
+            DAG.getStore(Chain, dl, Arg, PtrOff, MachinePointerInfo());
+        MemOpChains.push_back(Store);
+        if (QFPR_idx != NumQFPRs) {
+          SDValue Load = DAG.getLoad(IsF32 ? MVT::v4f32 : MVT::v4f64, dl, Store,
+                                     PtrOff, MachinePointerInfo());
+          MemOpChains.push_back(Load.getValue(1));
+          RegsToPass.push_back(std::make_pair(QFPR[QFPR_idx++], Load));
+        }
+        ArgOffset += (IsF32 ? 16 : 32);
+        for (unsigned i = 0; i < (IsF32 ? 16U : 32U); i += PtrByteSize) {
+          if (GPR_idx == NumGPRs)
+            break;
+          SDValue Ix = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff,
+                                   DAG.getConstant(i, dl, PtrVT));
+          SDValue Load =
+              DAG.getLoad(PtrVT, dl, Store, Ix, MachinePointerInfo());
+          MemOpChains.push_back(Load.getValue(1));
+          RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load));
+        }
+        break;
+      }
+
+      // Non-varargs QPX params go into registers or on the stack.
+      if (QFPR_idx != NumQFPRs) {
+        RegsToPass.push_back(std::make_pair(QFPR[QFPR_idx++], Arg));
+      } else {
+        if (IsFastCall)
+          ComputePtrOff();
+
+        assert(HasParameterArea &&
+               "Parameter area must exist to pass an argument in memory.");
+        LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset,
+                         true, CFlags.IsTailCall, true, MemOpChains,
+                         TailCallArguments, dl);
+        if (IsFastCall)
+          ArgOffset += (IsF32 ? 16 : 32);
+      }
+
+      if (!IsFastCall)
+        ArgOffset += (IsF32 ? 16 : 32);
+      break;
+      }
     }
   }
 
@@ -7003,6 +7301,8 @@ SDValue PPCTargetLowering::LowerFormalArguments_AIX(
 
   const PPCSubtarget &Subtarget =
       static_cast<const PPCSubtarget &>(DAG.getSubtarget());
+  if (Subtarget.hasQPX())
+    report_fatal_error("QPX support is not supported on AIX.");
 
   const bool IsPPC64 = Subtarget.isPPC64();
   const unsigned PtrByteSize = IsPPC64 ? 8 : 4;
@@ -7222,6 +7522,8 @@ SDValue PPCTargetLowering::LowerCall_AIX(
 
   const PPCSubtarget& Subtarget =
       static_cast<const PPCSubtarget&>(DAG.getSubtarget());
+  if (Subtarget.hasQPX())
+    report_fatal_error("QPX is not supported on AIX.");
   if (Subtarget.hasAltivec())
     report_fatal_error("Altivec support is unimplemented on AIX.");
 
@@ -7689,6 +7991,8 @@ SDValue PPCTargetLowering::lowerEH_SJLJ_LONGJMP(SDValue Op,
 }
 
 SDValue PPCTargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
+  if (Op.getValueType().isVector())
+    return LowerVectorLoad(Op, DAG);
 
   assert(Op.getValueType() == MVT::i1 &&
          "Custom lowering only for i1 loads");
@@ -7712,6 +8016,9 @@ SDValue PPCTargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
 }
 
 SDValue PPCTargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
+  if (Op.getOperand(1).getValueType().isVector())
+    return LowerVectorStore(Op, DAG);
+
   assert(Op.getOperand(1).getValueType() == MVT::i1 &&
          "Custom lowering only for i1 stores");
 
@@ -8288,6 +8595,27 @@ SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op,
   if (Op.getValueType() == MVT::f128)
     return Op;
 
+  if (Subtarget.hasQPX() && Op.getOperand(0).getValueType() == MVT::v4i1) {
+    if (Op.getValueType() != MVT::v4f32 && Op.getValueType() != MVT::v4f64)
+      return SDValue();
+
+    SDValue Value = Op.getOperand(0);
+    // The values are now known to be -1 (false) or 1 (true). To convert this
+    // into 0 (false) and 1 (true), add 1 and then divide by 2 (multiply by 0.5).
+    // This can be done with an fma and the 0.5 constant: (V+1.0)*0.5 = 0.5*V+0.5
+    Value = DAG.getNode(PPCISD::QBFLT, dl, MVT::v4f64, Value);
+
+    SDValue FPHalfs = DAG.getConstantFP(0.5, dl, MVT::v4f64);
+
+    Value = DAG.getNode(ISD::FMA, dl, MVT::v4f64, Value, FPHalfs, FPHalfs);
+
+    if (Op.getValueType() != MVT::v4f64)
+      Value = DAG.getNode(ISD::FP_ROUND, dl,
+                          Op.getValueType(), Value,
+                          DAG.getIntPtrConstant(1, dl));
+    return Value;
+  }
+
   // Don't handle ppc_fp128 here; let it be lowered to a libcall.
   if (Op.getValueType() != MVT::f32 && Op.getValueType() != MVT::f64)
     return SDValue();
@@ -8856,6 +9184,110 @@ SDValue PPCTargetLowering::LowerBUILD_VECTOR(SDValue Op,
   BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(Op.getNode());
   assert(BVN && "Expected a BuildVectorSDNode in LowerBUILD_VECTOR");
 
+  if (Subtarget.hasQPX() && Op.getValueType() == MVT::v4i1) {
+    // We first build an i32 vector, load it into a QPX register,
+    // then convert it to a floating-point vector and compare it
+    // to a zero vector to get the boolean result.
+    MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
+    int FrameIdx = MFI.CreateStackObject(16, Align(16), false);
+    MachinePointerInfo PtrInfo =
+        MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx);
+    EVT PtrVT = getPointerTy(DAG.getDataLayout());
+    SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT);
+
+    assert(BVN->getNumOperands() == 4 &&
+      "BUILD_VECTOR for v4i1 does not have 4 operands");
+
+    bool IsConst = true;
+    for (unsigned i = 0; i < 4; ++i) {
+      if (BVN->getOperand(i).isUndef()) continue;
+      if (!isa<ConstantSDNode>(BVN->getOperand(i))) {
+        IsConst = false;
+        break;
+      }
+    }
+
+    if (IsConst) {
+      Constant *One =
+        ConstantFP::get(Type::getFloatTy(*DAG.getContext()), 1.0);
+      Constant *NegOne =
+        ConstantFP::get(Type::getFloatTy(*DAG.getContext()), -1.0);
+
+      Constant *CV[4];
+      for (unsigned i = 0; i < 4; ++i) {
+        if (BVN->getOperand(i).isUndef())
+          CV[i] = UndefValue::get(Type::getFloatTy(*DAG.getContext()));
+        else if (isNullConstant(BVN->getOperand(i)))
+          CV[i] = NegOne;
+        else
+          CV[i] = One;
+      }
+
+      Constant *CP = ConstantVector::get(CV);
+      SDValue CPIdx =
+          DAG.getConstantPool(CP, getPointerTy(DAG.getDataLayout()), Align(16));
+
+      SDValue Ops[] = {DAG.getEntryNode(), CPIdx};
+      SDVTList VTs = DAG.getVTList({MVT::v4i1, /*chain*/ MVT::Other});
+      return DAG.getMemIntrinsicNode(
+          PPCISD::QVLFSb, dl, VTs, Ops, MVT::v4f32,
+          MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
+    }
+
+    SmallVector<SDValue, 4> Stores;
+    for (unsigned i = 0; i < 4; ++i) {
+      if (BVN->getOperand(i).isUndef()) continue;
+
+      unsigned Offset = 4*i;
+      SDValue Idx = DAG.getConstant(Offset, dl, FIdx.getValueType());
+      Idx = DAG.getNode(ISD::ADD, dl, FIdx.getValueType(), FIdx, Idx);
+
+      unsigned StoreSize = BVN->getOperand(i).getValueType().getStoreSize();
+      if (StoreSize > 4) {
+        Stores.push_back(
+            DAG.getTruncStore(DAG.getEntryNode(), dl, BVN->getOperand(i), Idx,
+                              PtrInfo.getWithOffset(Offset), MVT::i32));
+      } else {
+        SDValue StoreValue = BVN->getOperand(i);
+        if (StoreSize < 4)
+          StoreValue = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, StoreValue);
+
+        Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl, StoreValue, Idx,
+                                      PtrInfo.getWithOffset(Offset)));
+      }
+    }
+
+    SDValue StoreChain;
+    if (!Stores.empty())
+      StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
+    else
+      StoreChain = DAG.getEntryNode();
+
+    // Now load from v4i32 into the QPX register; this will extend it to
+    // v4i64 but not yet convert it to a floating point. Nevertheless, this
+    // is typed as v4f64 because the QPX register integer states are not
+    // explicitly represented.
+
+    SDValue Ops[] = {StoreChain,
+                     DAG.getConstant(Intrinsic::ppc_qpx_qvlfiwz, dl, MVT::i32),
+                     FIdx};
+    SDVTList VTs = DAG.getVTList({MVT::v4f64, /*chain*/ MVT::Other});
+
+    SDValue LoadedVect = DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN,
+      dl, VTs, Ops, MVT::v4i32, PtrInfo);
+    LoadedVect = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f64,
+      DAG.getConstant(Intrinsic::ppc_qpx_qvfcfidu, dl, MVT::i32),
+      LoadedVect);
+
+    SDValue FPZeros = DAG.getConstantFP(0.0, dl, MVT::v4f64);
+
+    return DAG.getSetCC(dl, MVT::v4i1, LoadedVect, FPZeros, ISD::SETEQ);
+  }
+
+  // All other QPX vectors are handled by generic code.
+  if (Subtarget.hasQPX())
+    return SDValue();
+
   // Check if this is a splat of a constant value.
   APInt APSplatBits, APSplatUndef;
   unsigned SplatBitSize;
@@ -9648,6 +10080,42 @@ SDValue PPCTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
     }
   }
 
+  if (Subtarget.hasQPX()) {
+    if (VT.getVectorNumElements() != 4)
+      return SDValue();
+
+    if (V2.isUndef()) V2 = V1;
+
+    int AlignIdx = PPC::isQVALIGNIShuffleMask(SVOp);
+    if (AlignIdx != -1) {
+      return DAG.getNode(PPCISD::QVALIGNI, dl, VT, V1, V2,
+                         DAG.getConstant(AlignIdx, dl, MVT::i32));
+    } else if (SVOp->isSplat()) {
+      int SplatIdx = SVOp->getSplatIndex();
+      if (SplatIdx >= 4) {
+        std::swap(V1, V2);
+        SplatIdx -= 4;
+      }
+
+      return DAG.getNode(PPCISD::QVESPLATI, dl, VT, V1,
+                         DAG.getConstant(SplatIdx, dl, MVT::i32));
+    }
+
+    // Lower this into a qvgpci/qvfperm pair.
+
+    // Compute the qvgpci literal
+    unsigned idx = 0;
+    for (unsigned i = 0; i < 4; ++i) {
+      int m = SVOp->getMaskElt(i);
+      unsigned mm = m >= 0 ? (unsigned) m : i;
+      idx |= mm << (3-i)*3;
+    }
+
+    SDValue V3 = DAG.getNode(PPCISD::QVGPCI, dl, MVT::v4f64,
+                             DAG.getConstant(idx, dl, MVT::i32));
+    return DAG.getNode(PPCISD::QVFPERM, dl, VT, V1, V2, V3);
+  }
+
   // Cases that are handled by instructions that take permute immediates
   // (such as vsplt*) should be left as VECTOR_SHUFFLE nodes so they can be
   // selected by the instruction selector.
@@ -10235,6 +10703,279 @@ SDValue PPCTargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
   return Op;
 }
 
+SDValue PPCTargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
+                                                   SelectionDAG &DAG) const {
+  SDLoc dl(Op);
+  SDNode *N = Op.getNode();
+
+  assert(N->getOperand(0).getValueType() == MVT::v4i1 &&
+         "Unknown extract_vector_elt type");
+
+  SDValue Value = N->getOperand(0);
+
+  // The first part of this is like the store lowering except that we don't
+  // need to track the chain.
+
+  // The values are now known to be -1 (false) or 1 (true). To convert this
+  // into 0 (false) and 1 (true), add 1 and then divide by 2 (multiply by 0.5).
+  // This can be done with an fma and the 0.5 constant: (V+1.0)*0.5 = 0.5*V+0.5
+  Value = DAG.getNode(PPCISD::QBFLT, dl, MVT::v4f64, Value);
+
+  // FIXME: We can make this an f32 vector, but the BUILD_VECTOR code needs to
+  // understand how to form the extending load.
+  SDValue FPHalfs = DAG.getConstantFP(0.5, dl, MVT::v4f64);
+
+  Value = DAG.getNode(ISD::FMA, dl, MVT::v4f64, Value, FPHalfs, FPHalfs);
+
+  // Now convert to an integer and store.
+  Value = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f64,
+    DAG.getConstant(Intrinsic::ppc_qpx_qvfctiwu, dl, MVT::i32),
+    Value);
+
+  MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
+  int FrameIdx = MFI.CreateStackObject(16, Align(16), false);
+  MachinePointerInfo PtrInfo =
+      MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx);
+  EVT PtrVT = getPointerTy(DAG.getDataLayout());
+  SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT);
+
+  SDValue StoreChain = DAG.getEntryNode();
+  SDValue Ops[] = {StoreChain,
+                   DAG.getConstant(Intrinsic::ppc_qpx_qvstfiw, dl, MVT::i32),
+                   Value, FIdx};
+  SDVTList VTs = DAG.getVTList(/*chain*/ MVT::Other);
+
+  StoreChain = DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID,
+    dl, VTs, Ops, MVT::v4i32, PtrInfo);
+
+  // Extract the value requested.
+  unsigned Offset = 4*cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
+  SDValue Idx = DAG.getConstant(Offset, dl, FIdx.getValueType());
+  Idx = DAG.getNode(ISD::ADD, dl, FIdx.getValueType(), FIdx, Idx);
+
+  SDValue IntVal =
+      DAG.getLoad(MVT::i32, dl, StoreChain, Idx, PtrInfo.getWithOffset(Offset));
+
+  if (!Subtarget.useCRBits())
+    return IntVal;
+
+  return DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, IntVal);
+}
+
+/// Lowering for QPX v4i1 loads
+SDValue PPCTargetLowering::LowerVectorLoad(SDValue Op,
+                                           SelectionDAG &DAG) const {
+  SDLoc dl(Op);
+  LoadSDNode *LN = cast<LoadSDNode>(Op.getNode());
+  SDValue LoadChain = LN->getChain();
+  SDValue BasePtr = LN->getBasePtr();
+
+  if (Op.getValueType() == MVT::v4f64 ||
+      Op.getValueType() == MVT::v4f32) {
+    EVT MemVT = LN->getMemoryVT();
+    unsigned Alignment = LN->getAlignment();
+
+    // If this load is properly aligned, then it is legal.
+    if (Alignment >= MemVT.getStoreSize())
+      return Op;
+
+    EVT ScalarVT = Op.getValueType().getScalarType(),
+        ScalarMemVT = MemVT.getScalarType();
+    unsigned Stride = ScalarMemVT.getStoreSize();
+
+    SDValue Vals[4], LoadChains[4];
+    for (unsigned Idx = 0; Idx < 4; ++Idx) {
+      SDValue Load;
+      if (ScalarVT != ScalarMemVT)
+        Load = DAG.getExtLoad(LN->getExtensionType(), dl, ScalarVT, LoadChain,
+                              BasePtr,
+                              LN->getPointerInfo().getWithOffset(Idx * Stride),
+                              ScalarMemVT, MinAlign(Alignment, Idx * Stride),
+                              LN->getMemOperand()->getFlags(), LN->getAAInfo());
+      else
+        Load = DAG.getLoad(ScalarVT, dl, LoadChain, BasePtr,
+                           LN->getPointerInfo().getWithOffset(Idx * Stride),
+                           MinAlign(Alignment, Idx * Stride),
+                           LN->getMemOperand()->getFlags(), LN->getAAInfo());
+
+      if (Idx == 0 && LN->isIndexed()) {
+        assert(LN->getAddressingMode() == ISD::PRE_INC &&
+               "Unknown addressing mode on vector load");
+        Load = DAG.getIndexedLoad(Load, dl, BasePtr, LN->getOffset(),
+                                  LN->getAddressingMode());
+      }
+
+      Vals[Idx] = Load;
+      LoadChains[Idx] = Load.getValue(1);
+
+      BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr,
+                            DAG.getConstant(Stride, dl,
+                                            BasePtr.getValueType()));
+    }
+
+    SDValue TF =  DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains);
+    SDValue Value = DAG.getBuildVector(Op.getValueType(), dl, Vals);
+
+    if (LN->isIndexed()) {
+      SDValue RetOps[] = { Value, Vals[0].getValue(1), TF };
+      return DAG.getMergeValues(RetOps, dl);
+    }
+
+    SDValue RetOps[] = { Value, TF };
+    return DAG.getMergeValues(RetOps, dl);
+  }
+
+  assert(Op.getValueType() == MVT::v4i1 && "Unknown load to lower");
+  assert(LN->isUnindexed() && "Indexed v4i1 loads are not supported");
+
+  // To lower v4i1 from a byte array, we load the byte elements of the
+  // vector and then reuse the BUILD_VECTOR logic.
+
+  SDValue VectElmts[4], VectElmtChains[4];
+  for (unsigned i = 0; i < 4; ++i) {
+    SDValue Idx = DAG.getConstant(i, dl, BasePtr.getValueType());
+    Idx = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, Idx);
+
+    VectElmts[i] = DAG.getExtLoad(
+        ISD::EXTLOAD, dl, MVT::i32, LoadChain, Idx,
+        LN->getPointerInfo().getWithOffset(i), MVT::i8,
+        /* Alignment = */ 1, LN->getMemOperand()->getFlags(), LN->getAAInfo());
+    VectElmtChains[i] = VectElmts[i].getValue(1);
+  }
+
+  LoadChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, VectElmtChains);
+  SDValue Value = DAG.getBuildVector(MVT::v4i1, dl, VectElmts);
+
+  SDValue RVals[] = { Value, LoadChain };
+  return DAG.getMergeValues(RVals, dl);
+}
+
+/// Lowering for QPX v4i1 stores
+SDValue PPCTargetLowering::LowerVectorStore(SDValue Op,
+                                            SelectionDAG &DAG) const {
+  SDLoc dl(Op);
+  StoreSDNode *SN = cast<StoreSDNode>(Op.getNode());
+  SDValue StoreChain = SN->getChain();
+  SDValue BasePtr = SN->getBasePtr();
+  SDValue Value = SN->getValue();
+
+  if (Value.getValueType() == MVT::v4f64 ||
+      Value.getValueType() == MVT::v4f32) {
+    EVT MemVT = SN->getMemoryVT();
+    unsigned Alignment = SN->getAlignment();
+
+    // If this store is properly aligned, then it is legal.
+    if (Alignment >= MemVT.getStoreSize())
+      return Op;
+
+    EVT ScalarVT = Value.getValueType().getScalarType(),
+        ScalarMemVT = MemVT.getScalarType();
+    unsigned Stride = ScalarMemVT.getStoreSize();
+
+    SDValue Stores[4];
+    for (unsigned Idx = 0; Idx < 4; ++Idx) {
+      SDValue Ex = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ScalarVT, Value,
+                               DAG.getVectorIdxConstant(Idx, dl));
+      SDValue Store;
+      if (ScalarVT != ScalarMemVT)
+        Store =
+            DAG.getTruncStore(StoreChain, dl, Ex, BasePtr,
+                              SN->getPointerInfo().getWithOffset(Idx * Stride),
+                              ScalarMemVT, MinAlign(Alignment, Idx * Stride),
+                              SN->getMemOperand()->getFlags(), SN->getAAInfo());
+      else
+        Store = DAG.getStore(StoreChain, dl, Ex, BasePtr,
+                             SN->getPointerInfo().getWithOffset(Idx * Stride),
+                             MinAlign(Alignment, Idx * Stride),
+                             SN->getMemOperand()->getFlags(), SN->getAAInfo());
+
+      if (Idx == 0 && SN->isIndexed()) {
+        assert(SN->getAddressingMode() == ISD::PRE_INC &&
+               "Unknown addressing mode on vector store");
+        Store = DAG.getIndexedStore(Store, dl, BasePtr, SN->getOffset(),
+                                    SN->getAddressingMode());
+      }
+
+      BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr,
+                            DAG.getConstant(Stride, dl,
+                                            BasePtr.getValueType()));
+      Stores[Idx] = Store;
+    }
+
+    SDValue TF =  DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
+
+    if (SN->isIndexed()) {
+      SDValue RetOps[] = { TF, Stores[0].getValue(1) };
+      return DAG.getMergeValues(RetOps, dl);
+    }
+
+    return TF;
+  }
+
+  assert(SN->isUnindexed() && "Indexed v4i1 stores are not supported");
+  assert(Value.getValueType() == MVT::v4i1 && "Unknown store to lower");
+
+  // The values are now known to be -1 (false) or 1 (true). To convert this
+  // into 0 (false) and 1 (true), add 1 and then divide by 2 (multiply by 0.5).
+  // This can be done with an fma and the 0.5 constant: (V+1.0)*0.5 = 0.5*V+0.5
+  Value = DAG.getNode(PPCISD::QBFLT, dl, MVT::v4f64, Value);
+
+  // FIXME: We can make this an f32 vector, but the BUILD_VECTOR code needs to
+  // understand how to form the extending load.
+  SDValue FPHalfs = DAG.getConstantFP(0.5, dl, MVT::v4f64);
+
+  Value = DAG.getNode(ISD::FMA, dl, MVT::v4f64, Value, FPHalfs, FPHalfs);
+
+  // Now convert to an integer and store.
+  Value = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f64,
+    DAG.getConstant(Intrinsic::ppc_qpx_qvfctiwu, dl, MVT::i32),
+    Value);
+
+  MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
+  int FrameIdx = MFI.CreateStackObject(16, Align(16), false);
+  MachinePointerInfo PtrInfo =
+      MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx);
+  EVT PtrVT = getPointerTy(DAG.getDataLayout());
+  SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT);
+
+  SDValue Ops[] = {StoreChain,
+                   DAG.getConstant(Intrinsic::ppc_qpx_qvstfiw, dl, MVT::i32),
+                   Value, FIdx};
+  SDVTList VTs = DAG.getVTList(/*chain*/ MVT::Other);
+
+  StoreChain = DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID,
+    dl, VTs, Ops, MVT::v4i32, PtrInfo);
+
+  // Move data into the byte array.
+  SDValue Loads[4], LoadChains[4];
+  for (unsigned i = 0; i < 4; ++i) {
+    unsigned Offset = 4*i;
+    SDValue Idx = DAG.getConstant(Offset, dl, FIdx.getValueType());
+    Idx = DAG.getNode(ISD::ADD, dl, FIdx.getValueType(), FIdx, Idx);
+
+    Loads[i] = DAG.getLoad(MVT::i32, dl, StoreChain, Idx,
+                           PtrInfo.getWithOffset(Offset));
+    LoadChains[i] = Loads[i].getValue(1);
+  }
+
+  StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains);
+
+  SDValue Stores[4];
+  for (unsigned i = 0; i < 4; ++i) {
+    SDValue Idx = DAG.getConstant(i, dl, BasePtr.getValueType());
+    Idx = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, Idx);
+
+    Stores[i] = DAG.getTruncStore(
+        StoreChain, dl, Loads[i], Idx, SN->getPointerInfo().getWithOffset(i),
+        MVT::i8, /* Alignment = */ 1, SN->getMemOperand()->getFlags(),
+        SN->getAAInfo());
+  }
+
+  StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
+
+  return StoreChain;
+}
+
 SDValue PPCTargetLowering::LowerMUL(SDValue Op, SelectionDAG &DAG) const {
   SDLoc dl(Op);
   if (Op.getValueType() == MVT::v4i32) {
@@ -10463,6 +11204,7 @@ SDValue PPCTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
   case ISD::VECTOR_SHUFFLE:     return LowerVECTOR_SHUFFLE(Op, DAG);
   case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG);
   case ISD::SCALAR_TO_VECTOR:   return LowerSCALAR_TO_VECTOR(Op, DAG);
+  case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG);
   case ISD::INSERT_VECTOR_ELT:  return LowerINSERT_VECTOR_ELT(Op, DAG);
   case ISD::MUL:                return LowerMUL(Op, DAG);
   case ISD::ABS:                return LowerABS(Op, DAG);
@@ -11406,6 +12148,9 @@ PPCTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
   } else if (MI.getOpcode() == PPC::SELECT_CC_F4 ||
              MI.getOpcode() == PPC::SELECT_CC_F8 ||
              MI.getOpcode() == PPC::SELECT_CC_F16 ||
+             MI.getOpcode() == PPC::SELECT_CC_QFRC ||
+             MI.getOpcode() == PPC::SELECT_CC_QSRC ||
+             MI.getOpcode() == PPC::SELECT_CC_QBRC ||
              MI.getOpcode() == PPC::SELECT_CC_VRRC ||
              MI.getOpcode() == PPC::SELECT_CC_VSFRC ||
              MI.getOpcode() == PPC::SELECT_CC_VSSRC ||
@@ -11415,6 +12160,9 @@ PPCTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
              MI.getOpcode() == PPC::SELECT_F4 ||
              MI.getOpcode() == PPC::SELECT_F8 ||
              MI.getOpcode() == PPC::SELECT_F16 ||
+             MI.getOpcode() == PPC::SELECT_QFRC ||
+             MI.getOpcode() == PPC::SELECT_QSRC ||
+             MI.getOpcode() == PPC::SELECT_QBRC ||
              MI.getOpcode() == PPC::SELECT_SPE ||
              MI.getOpcode() == PPC::SELECT_SPE4 ||
              MI.getOpcode() == PPC::SELECT_VRRC ||
@@ -11452,6 +12200,9 @@ PPCTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
         MI.getOpcode() == PPC::SELECT_F16 ||
         MI.getOpcode() == PPC::SELECT_SPE4 ||
         MI.getOpcode() == PPC::SELECT_SPE ||
+        MI.getOpcode() == PPC::SELECT_QFRC ||
+        MI.getOpcode() == PPC::SELECT_QSRC ||
+        MI.getOpcode() == PPC::SELECT_QBRC ||
         MI.getOpcode() == PPC::SELECT_VRRC ||
         MI.getOpcode() == PPC::SELECT_VSFRC ||
         MI.getOpcode() == PPC::SELECT_VSSRC ||
@@ -12144,7 +12895,9 @@ SDValue PPCTargetLowering::getSqrtEstimate(SDValue Operand, SelectionDAG &DAG,
   if ((VT == MVT::f32 && Subtarget.hasFRSQRTES()) ||
       (VT == MVT::f64 && Subtarget.hasFRSQRTE()) ||
       (VT == MVT::v4f32 && Subtarget.hasAltivec()) ||
-      (VT == MVT::v2f64 && Subtarget.hasVSX())) {
+      (VT == MVT::v2f64 && Subtarget.hasVSX()) ||
+      (VT == MVT::v4f32 && Subtarget.hasQPX()) ||
+      (VT == MVT::v4f64 && Subtarget.hasQPX())) {
     if (RefinementSteps == ReciprocalEstimate::Unspecified)
       RefinementSteps = getEstimateRefinementSteps(VT, Subtarget);
 
@@ -12163,7 +12916,9 @@ SDValue PPCTargetLowering::getRecipEstimate(SDValue Operand, SelectionDAG &DAG,
   if ((VT == MVT::f32 && Subtarget.hasFRES()) ||
       (VT == MVT::f64 && Subtarget.hasFRE()) ||
       (VT == MVT::v4f32 && Subtarget.hasAltivec()) ||
-      (VT == MVT::v2f64 && Subtarget.hasVSX())) {
+      (VT == MVT::v2f64 && Subtarget.hasVSX()) ||
+      (VT == MVT::v4f32 && Subtarget.hasQPX()) ||
+      (VT == MVT::v4f64 && Subtarget.hasQPX())) {
     if (RefinementSteps == ReciprocalEstimate::Unspecified)
       RefinementSteps = getEstimateRefinementSteps(VT, Subtarget);
     return DAG.getNode(PPCISD::FRE, SDLoc(Operand), VT, Operand);
@@ -12261,6 +13016,24 @@ static bool isConsecutiveLS(SDNode *N, LSBaseSDNode *Base,
     EVT VT;
     switch (cast<ConstantSDNode>(N->getOperand(1))->getZExtValue()) {
     default: return false;
+    case Intrinsic::ppc_qpx_qvlfd:
+    case Intrinsic::ppc_qpx_qvlfda:
+      VT = MVT::v4f64;
+      break;
+    case Intrinsic::ppc_qpx_qvlfs:
+    case Intrinsic::ppc_qpx_qvlfsa:
+      VT = MVT::v4f32;
+      break;
+    case Intrinsic::ppc_qpx_qvlfcd:
+    case Intrinsic::ppc_qpx_qvlfcda:
+      VT = MVT::v2f64;
+      break;
+    case Intrinsic::ppc_qpx_qvlfcs:
+    case Intrinsic::ppc_qpx_qvlfcsa:
+      VT = MVT::v2f32;
+      break;
+    case Intrinsic::ppc_qpx_qvlfiwa:
+    case Intrinsic::ppc_qpx_qvlfiwz:
     case Intrinsic::ppc_altivec_lvx:
     case Intrinsic::ppc_altivec_lvxl:
     case Intrinsic::ppc_vsx_lxvw4x:
@@ -12289,6 +13062,24 @@ static bool isConsecutiveLS(SDNode *N, LSBaseSDNode *Base,
     EVT VT;
     switch (cast<ConstantSDNode>(N->getOperand(1))->getZExtValue()) {
     default: return false;
+    case Intrinsic::ppc_qpx_qvstfd:
+    case Intrinsic::ppc_qpx_qvstfda:
+      VT = MVT::v4f64;
+      break;
+    case Intrinsic::ppc_qpx_qvstfs:
+    case Intrinsic::ppc_qpx_qvstfsa:
+      VT = MVT::v4f32;
+      break;
+    case Intrinsic::ppc_qpx_qvstfcd:
+    case Intrinsic::ppc_qpx_qvstfcda:
+      VT = MVT::v2f64;
+      break;
+    case Intrinsic::ppc_qpx_qvstfcs:
+    case Intrinsic::ppc_qpx_qvstfcsa:
+      VT = MVT::v2f32;
+      break;
+    case Intrinsic::ppc_qpx_qvstfiw:
+    case Intrinsic::ppc_qpx_qvstfiwa:
     case Intrinsic::ppc_altivec_stvx:
     case Intrinsic::ppc_altivec_stvxl:
     case Intrinsic::ppc_vsx_stxvw4x:
@@ -14286,14 +15077,18 @@ SDValue PPCTargetLowering::PerformDAGCombine(SDNode *N,
     EVT MemVT = LD->getMemoryVT();
     Type *Ty = MemVT.getTypeForEVT(*DAG.getContext());
     Align ABIAlignment = DAG.getDataLayout().getABITypeAlign(Ty);
+    Type *STy = MemVT.getScalarType().getTypeForEVT(*DAG.getContext());
+    Align ScalarABIAlignment = DAG.getDataLayout().getABITypeAlign(STy);
     if (LD->isUnindexed() && VT.isVector() &&
         ((Subtarget.hasAltivec() && ISD::isNON_EXTLoad(N) &&
           // P8 and later hardware should just use LOAD.
           !Subtarget.hasP8Vector() &&
           (VT == MVT::v16i8 || VT == MVT::v8i16 || VT == MVT::v4i32 ||
-           VT == MVT::v4f32))) &&
+           VT == MVT::v4f32)) ||
+         (Subtarget.hasQPX() && (VT == MVT::v4f64 || VT == MVT::v4f32) &&
+          LD->getAlign() >= ScalarABIAlignment)) &&
         LD->getAlign() < ABIAlignment) {
-      // This is a type-legal unaligned Altivec load.
+      // This is a type-legal unaligned Altivec or QPX load.
       SDValue Chain = LD->getChain();
       SDValue Ptr = LD->getBasePtr();
       bool isLittleEndian = Subtarget.isLittleEndian();
@@ -14324,13 +15119,24 @@ SDValue PPCTargetLowering::PerformDAGCombine(SDNode *N,
       // optimization later.
       Intrinsic::ID Intr, IntrLD, IntrPerm;
       MVT PermCntlTy, PermTy, LDTy;
-      Intr = isLittleEndian ? Intrinsic::ppc_altivec_lvsr
-                            : Intrinsic::ppc_altivec_lvsl;
-      IntrLD = Intrinsic::ppc_altivec_lvx;
-      IntrPerm = Intrinsic::ppc_altivec_vperm;
-      PermCntlTy = MVT::v16i8;
-      PermTy = MVT::v4i32;
-      LDTy = MVT::v4i32;
+      if (Subtarget.hasAltivec()) {
+        Intr = isLittleEndian ?  Intrinsic::ppc_altivec_lvsr :
+                                 Intrinsic::ppc_altivec_lvsl;
+        IntrLD = Intrinsic::ppc_altivec_lvx;
+        IntrPerm = Intrinsic::ppc_altivec_vperm;
+        PermCntlTy = MVT::v16i8;
+        PermTy = MVT::v4i32;
+        LDTy = MVT::v4i32;
+      } else {
+        Intr =   MemVT == MVT::v4f64 ? Intrinsic::ppc_qpx_qvlpcld :
+                                       Intrinsic::ppc_qpx_qvlpcls;
+        IntrLD = MemVT == MVT::v4f64 ? Intrinsic::ppc_qpx_qvlfd :
+                                       Intrinsic::ppc_qpx_qvlfs;
+        IntrPerm = Intrinsic::ppc_qpx_qvfperm;
+        PermCntlTy = MVT::v4f64;
+        PermTy = MVT::v4f64;
+        LDTy = MemVT.getSimpleVT();
+      }
 
       SDValue PermCntl = BuildIntrinsicOp(Intr, Ptr, DAG, dl, PermCntlTy);
 
@@ -14401,10 +15207,10 @@ SDValue PPCTargetLowering::PerformDAGCombine(SDNode *N,
                                 BaseLoad, ExtraLoad, PermCntl, DAG, dl);
 
       if (VT != PermTy)
-        Perm = Subtarget.hasAltivec()
-                   ? DAG.getNode(ISD::BITCAST, dl, VT, Perm)
-                   : DAG.getNode(ISD::FP_ROUND, dl, VT, Perm,
-                                 DAG.getTargetConstant(1, dl, MVT::i64));
+        Perm = Subtarget.hasAltivec() ?
+                 DAG.getNode(ISD::BITCAST, dl, VT, Perm) :
+                 DAG.getNode(ISD::FP_ROUND, dl, VT, Perm, // QPX
+                               DAG.getTargetConstant(1, dl, MVT::i64));
                                // second argument is 1 because this rounding
                                // is always exact.
 
@@ -14420,10 +15226,14 @@ SDValue PPCTargetLowering::PerformDAGCombine(SDNode *N,
       unsigned IID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
       Intrinsic::ID Intr = (isLittleEndian ? Intrinsic::ppc_altivec_lvsr
                                            : Intrinsic::ppc_altivec_lvsl);
-      if (IID == Intr && N->getOperand(1)->getOpcode() == ISD::ADD) {
+      if ((IID == Intr ||
+           IID == Intrinsic::ppc_qpx_qvlpcld  ||
+           IID == Intrinsic::ppc_qpx_qvlpcls) &&
+        N->getOperand(1)->getOpcode() == ISD::ADD) {
         SDValue Add = N->getOperand(1);
 
-        int Bits = 4 /* 16 byte alignment */;
+        int Bits = IID == Intrinsic::ppc_qpx_qvlpcld ?
+                   5 /* 32 byte alignment */ : 4 /* 16 byte alignment */;
 
         if (DAG.MaskedValueIsZero(Add->getOperand(1),
                                   APInt::getAllOnesValue(Bits /* alignment */)
@@ -14433,8 +15243,7 @@ SDValue PPCTargetLowering::PerformDAGCombine(SDNode *N,
                                     UE = BasePtr->use_end();
                UI != UE; ++UI) {
             if (UI->getOpcode() == ISD::INTRINSIC_WO_CHAIN &&
-                cast<ConstantSDNode>(UI->getOperand(0))->getZExtValue() ==
-                    IID) {
+                cast<ConstantSDNode>(UI->getOperand(0))->getZExtValue() == IID) {
               // We've found another LVSL/LVSR, and this address is an aligned
               // multiple of that one. The results will be the same, so use the
               // one we've just found instead.
@@ -14983,9 +15792,17 @@ PPCTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
           return std::make_pair(0U, &PPC::F4RCRegClass);
         if (VT == MVT::f64 || VT == MVT::i64)
           return std::make_pair(0U, &PPC::F8RCRegClass);
+        if (VT == MVT::v4f64 && Subtarget.hasQPX())
+          return std::make_pair(0U, &PPC::QFRCRegClass);
+        if (VT == MVT::v4f32 && Subtarget.hasQPX())
+          return std::make_pair(0U, &PPC::QSRCRegClass);
       }
       break;
     case 'v':
+      if (VT == MVT::v4f64 && Subtarget.hasQPX())
+        return std::make_pair(0U, &PPC::QFRCRegClass);
+      if (VT == MVT::v4f32 && Subtarget.hasQPX())
+        return std::make_pair(0U, &PPC::QSRCRegClass);
       if (Subtarget.hasAltivec())
         return std::make_pair(0U, &PPC::VRRCRegClass);
       break;
@@ -15277,6 +16094,12 @@ bool PPCTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
                                            MachineFunction &MF,
                                            unsigned Intrinsic) const {
   switch (Intrinsic) {
+  case Intrinsic::ppc_qpx_qvlfd:
+  case Intrinsic::ppc_qpx_qvlfs:
+  case Intrinsic::ppc_qpx_qvlfcd:
+  case Intrinsic::ppc_qpx_qvlfcs:
+  case Intrinsic::ppc_qpx_qvlfiwa:
+  case Intrinsic::ppc_qpx_qvlfiwz:
   case Intrinsic::ppc_altivec_lvx:
   case Intrinsic::ppc_altivec_lvxl:
   case Intrinsic::ppc_altivec_lvebx:
@@ -15298,6 +16121,18 @@ bool PPCTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
     case Intrinsic::ppc_vsx_lxvd2x:
       VT = MVT::v2f64;
       break;
+    case Intrinsic::ppc_qpx_qvlfd:
+      VT = MVT::v4f64;
+      break;
+    case Intrinsic::ppc_qpx_qvlfs:
+      VT = MVT::v4f32;
+      break;
+    case Intrinsic::ppc_qpx_qvlfcd:
+      VT = MVT::v2f64;
+      break;
+    case Intrinsic::ppc_qpx_qvlfcs:
+      VT = MVT::v2f32;
+      break;
     default:
       VT = MVT::v4i32;
       break;
@@ -15312,6 +16147,45 @@ bool PPCTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
     Info.flags = MachineMemOperand::MOLoad;
     return true;
   }
+  case Intrinsic::ppc_qpx_qvlfda:
+  case Intrinsic::ppc_qpx_qvlfsa:
+  case Intrinsic::ppc_qpx_qvlfcda:
+  case Intrinsic::ppc_qpx_qvlfcsa:
+  case Intrinsic::ppc_qpx_qvlfiwaa:
+  case Intrinsic::ppc_qpx_qvlfiwza: {
+    EVT VT;
+    switch (Intrinsic) {
+    case Intrinsic::ppc_qpx_qvlfda:
+      VT = MVT::v4f64;
+      break;
+    case Intrinsic::ppc_qpx_qvlfsa:
+      VT = MVT::v4f32;
+      break;
+    case Intrinsic::ppc_qpx_qvlfcda:
+      VT = MVT::v2f64;
+      break;
+    case Intrinsic::ppc_qpx_qvlfcsa:
+      VT = MVT::v2f32;
+      break;
+    default:
+      VT = MVT::v4i32;
+      break;
+    }
+
+    Info.opc = ISD::INTRINSIC_W_CHAIN;
+    Info.memVT = VT;
+    Info.ptrVal = I.getArgOperand(0);
+    Info.offset = 0;
+    Info.size = VT.getStoreSize();
+    Info.align = Align(1);
+    Info.flags = MachineMemOperand::MOLoad;
+    return true;
+  }
+  case Intrinsic::ppc_qpx_qvstfd:
+  case Intrinsic::ppc_qpx_qvstfs:
+  case Intrinsic::ppc_qpx_qvstfcd:
+  case Intrinsic::ppc_qpx_qvstfcs:
+  case Intrinsic::ppc_qpx_qvstfiw:
   case Intrinsic::ppc_altivec_stvx:
   case Intrinsic::ppc_altivec_stvxl:
   case Intrinsic::ppc_altivec_stvebx:
@@ -15333,6 +16207,18 @@ bool PPCTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
     case Intrinsic::ppc_vsx_stxvd2x:
       VT = MVT::v2f64;
       break;
+    case Intrinsic::ppc_qpx_qvstfd:
+      VT = MVT::v4f64;
+      break;
+    case Intrinsic::ppc_qpx_qvstfs:
+      VT = MVT::v4f32;
+      break;
+    case Intrinsic::ppc_qpx_qvstfcd:
+      VT = MVT::v2f64;
+      break;
+    case Intrinsic::ppc_qpx_qvstfcs:
+      VT = MVT::v2f32;
+      break;
     default:
       VT = MVT::v4i32;
       break;
@@ -15347,6 +16233,39 @@ bool PPCTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
     Info.flags = MachineMemOperand::MOStore;
     return true;
   }
+  case Intrinsic::ppc_qpx_qvstfda:
+  case Intrinsic::ppc_qpx_qvstfsa:
+  case Intrinsic::ppc_qpx_qvstfcda:
+  case Intrinsic::ppc_qpx_qvstfcsa:
+  case Intrinsic::ppc_qpx_qvstfiwa: {
+    EVT VT;
+    switch (Intrinsic) {
+    case Intrinsic::ppc_qpx_qvstfda:
+      VT = MVT::v4f64;
+      break;
+    case Intrinsic::ppc_qpx_qvstfsa:
+      VT = MVT::v4f32;
+      break;
+    case Intrinsic::ppc_qpx_qvstfcda:
+      VT = MVT::v2f64;
+      break;
+    case Intrinsic::ppc_qpx_qvstfcsa:
+      VT = MVT::v2f32;
+      break;
+    default:
+      VT = MVT::v4i32;
+      break;
+    }
+
+    Info.opc = ISD::INTRINSIC_VOID;
+    Info.memVT = VT;
+    Info.ptrVal = I.getArgOperand(1);
+    Info.offset = 0;
+    Info.size = VT.getStoreSize();
+    Info.align = Align(1);
+    Info.flags = MachineMemOperand::MOStore;
+    return true;
+  }
   default:
     break;
   }
@@ -15359,6 +16278,14 @@ bool PPCTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
 EVT PPCTargetLowering::getOptimalMemOpType(
     const MemOp &Op, const AttributeList &FuncAttributes) const {
   if (getTargetMachine().getOptLevel() != CodeGenOpt::None) {
+    // When expanding a memset, require at least two QPX instructions to cover
+    // the cost of loading the value to be stored from the constant pool.
+    if (Subtarget.hasQPX() && Op.size() >= 32 &&
+        (Op.isMemcpy() || Op.size() >= 64) && Op.isAligned(Align(32)) &&
+        !FuncAttributes.hasFnAttribute(Attribute::NoImplicitFloat)) {
+      return MVT::v4f64;
+    }
+
     // We should use Altivec/VSX loads and stores when available. For unaligned
     // addresses, unaligned VSX loads are only fast starting with the P8.
     if (Subtarget.hasAltivec() && Op.size() >= 16 &&
@@ -15577,7 +16504,7 @@ PPCTargetLowering::shouldExpandBuildVectorWithShuffles(
   if (VT == MVT::v2i64)
     return Subtarget.hasDirectMove(); // Don't need stack ops with direct moves
 
-  if (Subtarget.hasVSX())
+  if (Subtarget.hasVSX() || Subtarget.hasQPX())
     return true;
 
   return TargetLowering::shouldExpandBuildVectorWithShuffles(VT, DefinedValues);
@@ -15623,7 +16550,8 @@ SDValue PPCTargetLowering::getNegatedExpression(SDValue Op, SelectionDAG &DAG,
 
   switch (Opc) {
   case PPCISD::FNMSUB:
-    if (!Op.hasOneUse() || !isTypeLegal(VT))
+    // TODO: QPX subtarget is deprecated. No transformation here.
+    if (!Op.hasOneUse() || !isTypeLegal(VT) || Subtarget.hasQPX())
       break;
 
     const TargetOptions &Options = getTargetMachine().Options;
@@ -16104,7 +17032,8 @@ SDValue PPCTargetLowering::combineFMALike(SDNode *N,
   bool LegalOps = !DCI.isBeforeLegalizeOps();
   SDLoc Loc(N);
 
-  if (!isOperationLegal(ISD::FMA, VT))
+  // TODO: QPX subtarget is deprecated. No transformation here.
+  if (Subtarget.hasQPX() || !isOperationLegal(ISD::FMA, VT))
     return SDValue();
 
   // Allowing transformation to FNMSUB may change sign of zeroes when ab-c=0
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.h b/llvm/lib/Target/PowerPC/PPCISelLowering.h
index 8cc42226d7f0b..768eaa43e0135 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.h
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.h
@@ -427,6 +427,22 @@ namespace llvm {
     ///               => VABSDUW((XVNEGSP a), (XVNEGSP b))
     VABSD,
 
+    /// QVFPERM = This corresponds to the QPX qvfperm instruction.
+    QVFPERM,
+
+    /// QVGPCI = This corresponds to the QPX qvgpci instruction.
+    QVGPCI,
+
+    /// QVALIGNI = This corresponds to the QPX qvaligni instruction.
+    QVALIGNI,
+
+    /// QVESPLATI = This corresponds to the QPX qvesplati instruction.
+    QVESPLATI,
+
+    /// QBFLT = Access the underlying QPX floating-point boolean
+    /// representation.
+    QBFLT,
+
     /// FP_EXTEND_HALF(VECTOR, IDX) - Custom extend upper (IDX=0) half or
     /// lower (IDX=1) half of v4f32 to v2f64.
     FP_EXTEND_HALF,
@@ -503,6 +519,10 @@ namespace llvm {
     /// Store scalar integers from VSR.
     ST_VSR_SCAL_INT,
 
+    /// QBRC, CHAIN = QVLFSb CHAIN, Ptr
+    /// The 4xf32 load used for v4i1 constants.
+    QVLFSb,
+
     /// ATOMIC_CMP_SWAP - the exact same as the target-independent nodes
     /// except they ensure that the compare input is zero-extended for
     /// sub-word versions because the atomic loads zero-extend.
diff --git a/llvm/lib/Target/PowerPC/PPCInstrFormats.td b/llvm/lib/Target/PowerPC/PPCInstrFormats.td
index 5ff5fc78326ba..632d4d9deb8a2 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrFormats.td
+++ b/llvm/lib/Target/PowerPC/PPCInstrFormats.td
@@ -642,6 +642,7 @@ class XForm_17a<bits<6> opcode, bits<10> xo, dag OOL, dag IOL, string asmstr,
   let FRA = 0;
 }
 
+// Used for QPX
 class XForm_18<bits<6> opcode, bits<10> xo, dag OOL, dag IOL, string asmstr,
                InstrItinClass itin, list<dag> pattern>
          : I<opcode, OOL, IOL, asmstr, itin> {
@@ -1780,6 +1781,14 @@ class AForm_4<bits<6> opcode, bits<5> xo, dag OOL, dag IOL, string asmstr,
   let Inst{31}    = 0;
 }
 
+// Used for QPX
+class AForm_4a<bits<6> opcode, bits<5> xo, dag OOL, dag IOL, string asmstr,
+              InstrItinClass itin, list<dag> pattern>
+  : AForm_1<opcode, xo, OOL, IOL, asmstr, itin, pattern> {
+  let FRA = 0;
+  let FRC = 0;
+}
+
 // 1.7.13 M-Form
 class MForm_1<bits<6> opcode, dag OOL, dag IOL, string asmstr,
               InstrItinClass itin, list<dag> pattern>
@@ -2090,6 +2099,49 @@ class VX_RD5_RSp5_PS1_XO9<bits<9> xo, dag OOL, dag IOL, string asmstr,
   let Inst{23-31} = xo;
 }
 
+// Z23-Form (used by QPX)
+class Z23Form_1<bits<6> opcode, bits<8> xo, dag OOL, dag IOL, string asmstr, 
+              InstrItinClass itin, list<dag> pattern>
+         : I<opcode, OOL, IOL, asmstr, itin> {
+  bits<5> FRT;
+  bits<5> FRA;
+  bits<5> FRB;
+  bits<2> idx;
+
+  let Pattern = pattern;
+
+  bit RC = 0;    // set by isRecordForm
+
+  let Inst{6-10}  = FRT;
+  let Inst{11-15} = FRA;
+  let Inst{16-20} = FRB;
+  let Inst{21-22} = idx;
+  let Inst{23-30} = xo;
+  let Inst{31}    = RC;
+}
+
+class Z23Form_2<bits<6> opcode, bits<8> xo, dag OOL, dag IOL, string asmstr,
+              InstrItinClass itin, list<dag> pattern>
+  : Z23Form_1<opcode, xo, OOL, IOL, asmstr, itin, pattern> {
+  let FRB = 0;
+}
+
+class Z23Form_3<bits<6> opcode, bits<8> xo, dag OOL, dag IOL, string asmstr, 
+              InstrItinClass itin, list<dag> pattern>
+         : I<opcode, OOL, IOL, asmstr, itin> {
+  bits<5> FRT;
+  bits<12> idx;
+
+  let Pattern = pattern;
+
+  bit RC = 0;    // set by isRecordForm
+
+  let Inst{6-10}  = FRT;
+  let Inst{11-22} = idx;
+  let Inst{23-30} = xo;
+  let Inst{31}    = RC;
+}
+
 class Z23Form_8<bits<6> opcode, bits<8> xo, dag OOL, dag IOL, string asmstr,
               InstrItinClass itin, list<dag> pattern>
          : I<opcode, OOL, IOL, asmstr, itin> {
diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp b/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
index d4c3c5f5504c7..99e25bb130ce4 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
+++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
@@ -259,6 +259,16 @@ bool PPCInstrInfo::isAssociativeAndCommutative(const MachineInstr &Inst) const {
   case PPC::XVMULDP:
   case PPC::XVMULSP:
   case PPC::XSMULSP:
+  // QPX Add:
+  case PPC::QVFADD:
+  case PPC::QVFADDS:
+  case PPC::QVFADDSs:
+  // QPX Multiply:
+  case PPC::QVFMUL:
+  case PPC::QVFMULS:
+  case PPC::QVFMULSs:
+    return Inst.getFlag(MachineInstr::MIFlag::FmReassoc) &&
+           Inst.getFlag(MachineInstr::MIFlag::FmNsz);
   // Fixed point:
   // Multiply:
   case PPC::MULHD:
@@ -290,7 +300,9 @@ static const uint16_t FMAOpIdxInfo[][5] = {
     {PPC::XVMADDADP, PPC::XVADDDP, PPC::XVMULDP, 1, 2},
     {PPC::XVMADDASP, PPC::XVADDSP, PPC::XVMULSP, 1, 2},
     {PPC::FMADD, PPC::FADD, PPC::FMUL, 3, 1},
-    {PPC::FMADDS, PPC::FADDS, PPC::FMULS, 3, 1}};
+    {PPC::FMADDS, PPC::FADDS, PPC::FMULS, 3, 1},
+    {PPC::QVFMADDSs, PPC::QVFADDSs, PPC::QVFMULSs, 3, 1},
+    {PPC::QVFMADD, PPC::QVFADD, PPC::QVFMUL, 3, 1}};
 
 // Check if an opcode is a FMA instruction. If it is, return the index in array
 // FMAOpIdxInfo. Otherwise, return -1.
@@ -654,6 +666,7 @@ bool PPCInstrInfo::isReallyTriviallyReMaterializable(const MachineInstr &MI,
   case PPC::LI8:
   case PPC::LIS:
   case PPC::LIS8:
+  case PPC::QVGPCI:
   case PPC::ADDIStocHA:
   case PPC::ADDIStocHA8:
   case PPC::ADDItocL:
@@ -1330,6 +1343,12 @@ void PPCInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
   else if (PPC::VSFRCRegClass.contains(DestReg, SrcReg) ||
            PPC::VSSRCRegClass.contains(DestReg, SrcReg))
     Opc = (Subtarget.hasP9Vector()) ? PPC::XSCPSGNDP : PPC::XXLORf;
+  else if (PPC::QFRCRegClass.contains(DestReg, SrcReg))
+    Opc = PPC::QVFMR;
+  else if (PPC::QSRCRegClass.contains(DestReg, SrcReg))
+    Opc = PPC::QVFMRs;
+  else if (PPC::QBRCRegClass.contains(DestReg, SrcReg))
+    Opc = PPC::QVFMRb;
   else if (PPC::CRBITRCRegClass.contains(DestReg, SrcReg))
     Opc = PPC::CROR;
   else if (PPC::SPERCRegClass.contains(DestReg, SrcReg))
@@ -1374,6 +1393,12 @@ static unsigned getSpillIndex(const TargetRegisterClass *RC) {
     OpcodeIndex = SOK_VectorFloat4Spill;
   } else if (PPC::VRSAVERCRegClass.hasSubClassEq(RC)) {
     OpcodeIndex = SOK_VRSaveSpill;
+  } else if (PPC::QFRCRegClass.hasSubClassEq(RC)) {
+    OpcodeIndex = SOK_QuadFloat8Spill;
+  } else if (PPC::QSRCRegClass.hasSubClassEq(RC)) {
+    OpcodeIndex = SOK_QuadFloat4Spill;
+  } else if (PPC::QBRCRegClass.hasSubClassEq(RC)) {
+    OpcodeIndex = SOK_QuadBitSpill;
   } else if (PPC::SPILLTOVSRRCRegClass.hasSubClassEq(RC)) {
     OpcodeIndex = SOK_SpillToVSR;
   } else {
diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.h b/llvm/lib/Target/PowerPC/PPCInstrInfo.h
index bdcfa76505daf..43973c627fcf1 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrInfo.h
+++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.h
@@ -123,6 +123,9 @@ enum SpillOpcodeKey {
   SOK_VectorFloat8Spill,
   SOK_VectorFloat4Spill,
   SOK_VRSaveSpill,
+  SOK_QuadFloat8Spill,
+  SOK_QuadFloat4Spill,
+  SOK_QuadBitSpill,
   SOK_SpillToVSR,
   SOK_SPESpill,
   SOK_LastOpcodeSpill // This must be last on the enum.
@@ -133,28 +136,32 @@ enum SpillOpcodeKey {
   {                                                                            \
     PPC::LWZ, PPC::LD, PPC::LFD, PPC::LFS, PPC::RESTORE_CR,                    \
         PPC::RESTORE_CRBIT, PPC::LVX, PPC::LXVD2X, PPC::LXSDX, PPC::LXSSPX,    \
-        PPC::RESTORE_VRSAVE, PPC::SPILLTOVSR_LD, PPC::EVLDD                    \
+        PPC::RESTORE_VRSAVE, PPC::QVLFDX, PPC::QVLFSXs, PPC::QVLFDXb,          \
+        PPC::SPILLTOVSR_LD, PPC::EVLDD                                         \
   }
 
 #define Pwr9LoadOpcodes                                                        \
   {                                                                            \
     PPC::LWZ, PPC::LD, PPC::LFD, PPC::LFS, PPC::RESTORE_CR,                    \
         PPC::RESTORE_CRBIT, PPC::LVX, PPC::LXV, PPC::DFLOADf64,                \
-        PPC::DFLOADf32, PPC::RESTORE_VRSAVE, PPC::SPILLTOVSR_LD                \
+        PPC::DFLOADf32, PPC::RESTORE_VRSAVE, PPC::QVLFDX, PPC::QVLFSXs,        \
+        PPC::QVLFDXb, PPC::SPILLTOVSR_LD                                       \
   }
 
 #define Pwr8StoreOpcodes                                                       \
   {                                                                            \
     PPC::STW, PPC::STD, PPC::STFD, PPC::STFS, PPC::SPILL_CR, PPC::SPILL_CRBIT, \
         PPC::STVX, PPC::STXVD2X, PPC::STXSDX, PPC::STXSSPX, PPC::SPILL_VRSAVE, \
-        PPC::SPILLTOVSR_ST, PPC::EVSTDD                                        \
+        PPC::QVSTFDX, PPC::QVSTFSXs, PPC::QVSTFDXb, PPC::SPILLTOVSR_ST,        \
+        PPC::EVSTDD                                                            \
   }
 
 #define Pwr9StoreOpcodes                                                       \
   {                                                                            \
     PPC::STW, PPC::STD, PPC::STFD, PPC::STFS, PPC::SPILL_CR, PPC::SPILL_CRBIT, \
         PPC::STVX, PPC::STXV, PPC::DFSTOREf64, PPC::DFSTOREf32,                \
-        PPC::SPILL_VRSAVE, PPC::SPILLTOVSR_ST                                  \
+        PPC::SPILL_VRSAVE, PPC::QVSTFDX, PPC::QVSTFSXs, PPC::QVSTFDXb,         \
+        PPC::SPILLTOVSR_ST                                                     \
   }
 
 // Initialize arrays for load and store spill opcodes on supported subtargets.
@@ -266,10 +273,10 @@ class PPCInstrInfo : public PPCGenInstrInfo {
   }
 
   static bool isSameClassPhysRegCopy(unsigned Opcode) {
-    unsigned CopyOpcodes[] = {PPC::OR,        PPC::OR8,   PPC::FMR,
-                              PPC::VOR,       PPC::XXLOR, PPC::XXLORf,
-                              PPC::XSCPSGNDP, PPC::MCRF,  PPC::CROR,
-                              PPC::EVOR,      -1U};
+    unsigned CopyOpcodes[] =
+      { PPC::OR, PPC::OR8, PPC::FMR, PPC::VOR, PPC::XXLOR, PPC::XXLORf,
+        PPC::XSCPSGNDP, PPC::MCRF, PPC::QVFMR, PPC::QVFMRs, PPC::QVFMRb,
+        PPC::CROR, PPC::EVOR, -1U };
     for (int i = 0; CopyOpcodes[i] != -1U; i++)
       if (Opcode == CopyOpcodes[i])
         return true;
diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.td b/llvm/lib/Target/PowerPC/PPCInstrInfo.td
index 83a434f5e793a..c565758973bf5 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrInfo.td
+++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.td
@@ -203,6 +203,16 @@ def PPCvecinsert : SDNode<"PPCISD::VECINSERT", SDT_PPCVecInsert, []>;
 def PPCxxpermdi  : SDNode<"PPCISD::XXPERMDI", SDT_PPCxxpermdi, []>;
 def PPCvecshl    : SDNode<"PPCISD::VECSHL", SDT_PPCVecShift, []>;
 
+def PPCqvfperm   : SDNode<"PPCISD::QVFPERM", SDT_PPCqvfperm, []>;
+def PPCqvgpci    : SDNode<"PPCISD::QVGPCI", SDT_PPCqvgpci, []>;
+def PPCqvaligni  : SDNode<"PPCISD::QVALIGNI", SDT_PPCqvaligni, []>;
+def PPCqvesplati : SDNode<"PPCISD::QVESPLATI", SDT_PPCqvesplati, []>;
+
+def PPCqbflt     : SDNode<"PPCISD::QBFLT", SDT_PPCqbflt, []>;
+
+def PPCqvlfsb    : SDNode<"PPCISD::QVLFSb", SDT_PPCqvlfsb,
+                          [SDNPHasChain, SDNPMayLoad]>;
+
 def PPCcmpb     : SDNode<"PPCISD::CMPB", SDTIntBinOp, []>;
 
 // These nodes represent the 32-bit PPC shifts that operate on 6-bit shift
@@ -3451,6 +3461,7 @@ include "PPCInstrAltivec.td"
 include "PPCInstrSPE.td"
 include "PPCInstr64Bit.td"
 include "PPCInstrVSX.td"
+include "PPCInstrQPX.td"
 include "PPCInstrHTM.td"
 
 def crnot : OutPatFrag<(ops node:$in),
diff --git a/llvm/lib/Target/PowerPC/PPCInstrQPX.td b/llvm/lib/Target/PowerPC/PPCInstrQPX.td
new file mode 100644
index 0000000000000..2265af2815cb5
--- /dev/null
+++ b/llvm/lib/Target/PowerPC/PPCInstrQPX.td
@@ -0,0 +1,1212 @@
+//===- PPCInstrQPX.td - The PowerPC QPX Extension --*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file describes the QPX extension to the PowerPC instruction set.
+// Reference:
+// Book Q: QPX Architecture Definition. IBM (as updated in) 2011.
+//
+//===----------------------------------------------------------------------===//
+
+def PPCRegQFRCAsmOperand : AsmOperandClass {
+  let Name = "RegQFRC"; let PredicateMethod = "isRegNumber";
+}
+def qfrc : RegisterOperand<QFRC> {
+  let ParserMatchClass = PPCRegQFRCAsmOperand;
+}
+def PPCRegQSRCAsmOperand : AsmOperandClass {
+  let Name = "RegQSRC"; let PredicateMethod = "isRegNumber";
+}
+def qsrc : RegisterOperand<QSRC> {
+  let ParserMatchClass = PPCRegQSRCAsmOperand;
+}
+def PPCRegQBRCAsmOperand : AsmOperandClass {
+  let Name = "RegQBRC"; let PredicateMethod = "isRegNumber";
+}
+def qbrc : RegisterOperand<QBRC> {
+  let ParserMatchClass = PPCRegQBRCAsmOperand;
+}
+
+//===----------------------------------------------------------------------===//
+// Helpers for defining instructions that directly correspond to intrinsics.
+
+// QPXA1_Int - A AForm_1 intrinsic definition.
+class QPXA1_Int<bits<6> opcode, bits<5> xo, string opc, Intrinsic IntID>
+  : AForm_1<opcode, xo, (outs qfrc:$FRT), (ins qfrc:$FRA, qfrc:$FRB, qfrc:$FRC),
+              !strconcat(opc, " $FRT, $FRA, $FRC, $FRB"), IIC_FPFused,
+                       [(set v4f64:$FRT, (IntID v4f64:$FRA, v4f64:$FRB, v4f64:$FRC))]>;
+// QPXA1s_Int - A AForm_1 intrinsic definition (simple instructions).
+class QPXA1s_Int<bits<6> opcode, bits<5> xo, string opc, Intrinsic IntID>
+  : AForm_1<opcode, xo, (outs qfrc:$FRT), (ins qfrc:$FRA, qfrc:$FRB, qfrc:$FRC),
+              !strconcat(opc, " $FRT, $FRA, $FRC, $FRB"), IIC_VecPerm,
+                       [(set v4f64:$FRT, (IntID v4f64:$FRA, v4f64:$FRB, v4f64:$FRC))]>;
+// QPXA2_Int - A AForm_2 intrinsic definition.
+class QPXA2_Int<bits<6> opcode, bits<5> xo, string opc, Intrinsic IntID>
+  : AForm_2<opcode, xo, (outs qfrc:$FRT), (ins qfrc:$FRA, qfrc:$FRB),
+              !strconcat(opc, " $FRT, $FRA, $FRB"), IIC_FPGeneral,
+                       [(set v4f64:$FRT, (IntID v4f64:$FRA, v4f64:$FRB))]>;
+// QPXA3_Int - A AForm_3 intrinsic definition.
+class QPXA3_Int<bits<6> opcode, bits<5> xo, string opc, Intrinsic IntID>
+  : AForm_3<opcode, xo, (outs qfrc:$FRT), (ins qfrc:$FRA, qfrc:$FRC),
+              !strconcat(opc, " $FRT, $FRA, $FRC"), IIC_FPGeneral,
+                       [(set v4f64:$FRT, (IntID v4f64:$FRA, v4f64:$FRC))]>;
+// QPXA4_Int - A AForm_4a intrinsic definition.
+class QPXA4_Int<bits<6> opcode, bits<5> xo, string opc, Intrinsic IntID>
+  : AForm_4a<opcode, xo, (outs qfrc:$FRT), (ins qfrc:$FRB),
+              !strconcat(opc, " $FRT, $FRB"), IIC_FPGeneral,
+                       [(set v4f64:$FRT, (IntID v4f64:$FRB))]>;
+// QPXX18_Int - A XForm_18 intrinsic definition.
+class QPXX18_Int<bits<6> opcode, bits<10> xo, string opc, Intrinsic IntID>
+  : XForm_18<opcode, xo, (outs qfrc:$FRT), (ins qfrc:$FRA, qfrc:$FRB),
+              !strconcat(opc, " $FRT, $FRA, $FRB"), IIC_FPCompare,
+                       [(set v4f64:$FRT, (IntID v4f64:$FRA, v4f64:$FRB))]>;
+// QPXX19_Int - A XForm_19 intrinsic definition.
+class QPXX19_Int<bits<6> opcode, bits<10> xo, string opc, Intrinsic IntID>
+  : XForm_19<opcode, xo, (outs qfrc:$FRT), (ins qfrc:$FRB),
+              !strconcat(opc, " $FRT, $FRB"), IIC_FPGeneral,
+                       [(set v4f64:$FRT, (IntID v4f64:$FRB))]>;
+
+//===----------------------------------------------------------------------===//
+// Pattern Frags.
+
+def extloadv4f32 : PatFrag<(ops node:$ptr), (extload node:$ptr), [{
+  return cast<LoadSDNode>(N)->getMemoryVT() == MVT::v4f32;
+}]>;
+
+def truncstorev4f32 : PatFrag<(ops node:$val, node:$ptr),
+                            (truncstore node:$val, node:$ptr), [{
+  return cast<StoreSDNode>(N)->getMemoryVT() == MVT::v4f32;
+}]>;
+def pre_truncstv4f32 : PatFrag<(ops node:$val, node:$base, node:$offset),
+                               (pre_truncst node:$val,
+                                            node:$base, node:$offset), [{
+  return cast<StoreSDNode>(N)->getMemoryVT() == MVT::v4f32;
+}]>;
+
+def fround_inexact : PatFrag<(ops node:$val), (fpround node:$val), [{
+  return cast<ConstantSDNode>(N->getOperand(1))->getZExtValue() == 0;
+}]>;
+
+def fround_exact : PatFrag<(ops node:$val), (fpround node:$val), [{
+  return cast<ConstantSDNode>(N->getOperand(1))->getZExtValue() == 1;
+}]>;
+
+let FastIselShouldIgnore = 1 in // FastIsel should ignore all u12 instrs.
+  def u12 : ImmLeaf<i32, [{ return (Imm & 0xFFF) == Imm; }]>;
+
+//===----------------------------------------------------------------------===//
+// Instruction Definitions.
+
+def HasQPX : Predicate<"Subtarget->hasQPX()">;
+let Predicates = [HasQPX] in {
+let DecoderNamespace = "QPX" in {
+let hasSideEffects = 0 in { // QPX instructions don't have side effects.
+let Uses = [RM] in {
+  // Add Instructions
+  let isCommutable = 1 in {
+    def QVFADD : AForm_2<4, 21,
+                        (outs qfrc:$FRT), (ins qfrc:$FRA, qfrc:$FRB),
+                        "qvfadd $FRT, $FRA, $FRB", IIC_FPGeneral,
+                        [(set v4f64:$FRT, (fadd v4f64:$FRA, v4f64:$FRB))]>;
+    let isCodeGenOnly = 1 in
+      def QVFADDS : QPXA2_Int<0, 21, "qvfadds", int_ppc_qpx_qvfadds>;
+    def QVFADDSs : AForm_2<0, 21,
+                          (outs qsrc:$FRT), (ins qsrc:$FRA, qsrc:$FRB),
+                          "qvfadds $FRT, $FRA, $FRB", IIC_FPGeneral,
+                          [(set v4f32:$FRT, (fadd v4f32:$FRA, v4f32:$FRB))]>;
+  }
+  def QVFSUB : AForm_2<4, 20,
+                      (outs qfrc:$FRT), (ins qfrc:$FRA, qfrc:$FRB),
+                      "qvfsub $FRT, $FRA, $FRB", IIC_FPGeneral,
+                      [(set v4f64:$FRT, (fsub v4f64:$FRA, v4f64:$FRB))]>;
+  let isCodeGenOnly = 1 in
+    def QVFSUBS : QPXA2_Int<0, 20, "qvfsubs", int_ppc_qpx_qvfsubs>;
+  def QVFSUBSs : AForm_2<0, 20,
+                        (outs qsrc:$FRT), (ins qsrc:$FRA, qsrc:$FRB),
+                        "qvfsubs $FRT, $FRA, $FRB", IIC_FPGeneral,
+                        [(set v4f32:$FRT, (fsub v4f32:$FRA, v4f32:$FRB))]>;
+
+  // Estimate Instructions
+  def QVFRE : AForm_4a<4, 24, (outs qfrc:$FRT), (ins qfrc:$FRB),
+                       "qvfre $FRT, $FRB", IIC_FPGeneral,
+                       [(set v4f64:$FRT, (PPCfre v4f64:$FRB))]>;
+  def QVFRES : QPXA4_Int<0, 24, "qvfres", int_ppc_qpx_qvfres>;
+  let isCodeGenOnly = 1 in
+  def QVFRESs : AForm_4a<0, 24, (outs qsrc:$FRT), (ins qsrc:$FRB),
+                         "qvfres $FRT, $FRB", IIC_FPGeneral,
+                         [(set v4f32:$FRT, (PPCfre v4f32:$FRB))]>;
+
+  def QVFRSQRTE : AForm_4a<4, 26, (outs qfrc:$FRT), (ins qfrc:$FRB),
+                           "qvfrsqrte $FRT, $FRB", IIC_FPGeneral,
+                           [(set v4f64:$FRT, (PPCfrsqrte v4f64:$FRB))]>;
+  def QVFRSQRTES : QPXA4_Int<0, 26, "qvfrsqrtes", int_ppc_qpx_qvfrsqrtes>;
+  let isCodeGenOnly = 1 in
+  def QVFRSQRTESs : AForm_4a<0, 26, (outs qsrc:$FRT), (ins qsrc:$FRB),
+                             "qvfrsqrtes $FRT, $FRB", IIC_FPGeneral,
+                             [(set v4f32:$FRT, (PPCfrsqrte v4f32:$FRB))]>;
+
+  // Multiply Instructions
+  let isCommutable = 1 in {
+    def QVFMUL : AForm_3<4, 25,
+                        (outs qfrc:$FRT), (ins qfrc:$FRA, qfrc:$FRC),
+                        "qvfmul $FRT, $FRA, $FRC", IIC_FPGeneral,
+                        [(set v4f64:$FRT, (fmul v4f64:$FRA, v4f64:$FRC))]>;
+    let isCodeGenOnly = 1 in
+      def QVFMULS : QPXA3_Int<0, 25, "qvfmuls", int_ppc_qpx_qvfmuls>;
+    def QVFMULSs : AForm_3<0, 25,
+                          (outs qsrc:$FRT), (ins qsrc:$FRA, qsrc:$FRC),
+                          "qvfmuls $FRT, $FRA, $FRC", IIC_FPGeneral,
+                          [(set v4f32:$FRT, (fmul v4f32:$FRA, v4f32:$FRC))]>;
+  }
+  def QVFXMUL : QPXA3_Int<4, 17, "qvfxmul", int_ppc_qpx_qvfxmul>;
+  def QVFXMULS : QPXA3_Int<0, 17, "qvfxmuls", int_ppc_qpx_qvfxmuls>;
+
+  // Multiply-add instructions
+  def QVFMADD : AForm_1<4, 29,
+                      (outs qfrc:$FRT), (ins qfrc:$FRA, qfrc:$FRC, qfrc:$FRB),
+                      "qvfmadd $FRT, $FRA, $FRC, $FRB", IIC_FPFused,
+                      [(set v4f64:$FRT, (fma v4f64:$FRA, v4f64:$FRC, v4f64:$FRB))]>;
+  let isCodeGenOnly = 1 in
+    def QVFMADDS : QPXA1_Int<0, 29, "qvfmadds", int_ppc_qpx_qvfmadds>;
+  def QVFMADDSs : AForm_1<0, 29,
+                        (outs qsrc:$FRT), (ins qsrc:$FRA, qsrc:$FRC, qsrc:$FRB),
+                        "qvfmadds $FRT, $FRA, $FRC, $FRB", IIC_FPFused,
+                        [(set v4f32:$FRT, (fma v4f32:$FRA, v4f32:$FRC, v4f32:$FRB))]>;
+  def QVFNMADD : AForm_1<4, 31,
+                      (outs qfrc:$FRT), (ins qfrc:$FRA, qfrc:$FRC, qfrc:$FRB),
+                      "qvfnmadd $FRT, $FRA, $FRC, $FRB", IIC_FPFused,
+                      [(set v4f64:$FRT, (fneg (fma v4f64:$FRA, v4f64:$FRC,
+                                                   v4f64:$FRB)))]>;
+  let isCodeGenOnly = 1 in
+    def QVFNMADDS : QPXA1_Int<0, 31, "qvfnmadds", int_ppc_qpx_qvfnmadds>;
+  def QVFNMADDSs : AForm_1<0, 31,
+                        (outs qsrc:$FRT), (ins qsrc:$FRA, qsrc:$FRC, qsrc:$FRB),
+                        "qvfnmadds $FRT, $FRA, $FRC, $FRB", IIC_FPFused,
+                        [(set v4f32:$FRT, (fneg (fma v4f32:$FRA, v4f32:$FRC,
+                                                     v4f32:$FRB)))]>;
+  def QVFMSUB : AForm_1<4, 28,
+                      (outs qfrc:$FRT), (ins qfrc:$FRA, qfrc:$FRC, qfrc:$FRB),
+                      "qvfmsub $FRT, $FRA, $FRC, $FRB", IIC_FPFused,
+                      [(set v4f64:$FRT, (fma v4f64:$FRA, v4f64:$FRC,
+                                             (fneg v4f64:$FRB)))]>;
+  let isCodeGenOnly = 1 in
+    def QVFMSUBS : QPXA1_Int<0, 28, "qvfmsubs", int_ppc_qpx_qvfmsubs>;
+  def QVFMSUBSs : AForm_1<0, 28,
+                      (outs qsrc:$FRT), (ins qsrc:$FRA, qsrc:$FRC, qsrc:$FRB),
+                      "qvfmsubs $FRT, $FRA, $FRC, $FRB", IIC_FPFused,
+                      [(set v4f32:$FRT, (fma v4f32:$FRA, v4f32:$FRC,
+                                             (fneg v4f32:$FRB)))]>;
+  def QVFNMSUB : AForm_1<4, 30,
+                      (outs qfrc:$FRT), (ins qfrc:$FRA, qfrc:$FRC, qfrc:$FRB),
+                      "qvfnmsub $FRT, $FRA, $FRC, $FRB", IIC_FPFused,
+                      [(set v4f64:$FRT, (fneg (fma v4f64:$FRA, v4f64:$FRC,
+                                              (fneg v4f64:$FRB))))]>;
+  let isCodeGenOnly = 1 in
+    def QVFNMSUBS : QPXA1_Int<0, 30, "qvfnmsubs", int_ppc_qpx_qvfnmsubs>;
+  def QVFNMSUBSs : AForm_1<0, 30,
+                      (outs qsrc:$FRT), (ins qsrc:$FRA, qsrc:$FRC, qsrc:$FRB),
+                      "qvfnmsubs $FRT, $FRA, $FRC, $FRB", IIC_FPFused,
+                      [(set v4f32:$FRT, (fneg (fma v4f32:$FRA, v4f32:$FRC,
+                                              (fneg v4f32:$FRB))))]>;
+  def QVFXMADD : QPXA1_Int<4, 9, "qvfxmadd", int_ppc_qpx_qvfxmadd>;
+  def QVFXMADDS : QPXA1_Int<0, 9, "qvfxmadds", int_ppc_qpx_qvfxmadds>;
+  def QVFXXNPMADD : QPXA1_Int<4, 11, "qvfxxnpmadd", int_ppc_qpx_qvfxxnpmadd>;
+  def QVFXXNPMADDS : QPXA1_Int<0, 11, "qvfxxnpmadds", int_ppc_qpx_qvfxxnpmadds>;
+  def QVFXXCPNMADD : QPXA1_Int<4, 3, "qvfxxcpnmadd", int_ppc_qpx_qvfxxcpnmadd>;
+  def QVFXXCPNMADDS : QPXA1_Int<0, 3, "qvfxxcpnmadds", int_ppc_qpx_qvfxxcpnmadds>;
+  def QVFXXMADD : QPXA1_Int<4, 1, "qvfxxmadd", int_ppc_qpx_qvfxxmadd>;
+  def QVFXXMADDS : QPXA1_Int<0, 1, "qvfxxmadds", int_ppc_qpx_qvfxxmadds>;
+
+  // Select Instruction
+  let isCodeGenOnly = 1 in
+    def QVFSEL : QPXA1s_Int<4, 23, "qvfsel", int_ppc_qpx_qvfsel>;
+  def QVFSELb : AForm_1<4, 23, (outs qfrc:$FRT),
+                        (ins qbrc:$FRA, qfrc:$FRB, qfrc:$FRC),
+                        "qvfsel $FRT, $FRA, $FRC, $FRB", IIC_VecPerm,
+                        [(set v4f64:$FRT, (vselect v4i1:$FRA,
+                                                   v4f64:$FRC, v4f64:$FRB))]>;
+  let isCodeGenOnly = 1 in
+  def QVFSELbs : AForm_1<4, 23, (outs qsrc:$FRT),
+                        (ins qbrc:$FRA, qsrc:$FRB, qsrc:$FRC),
+                        "qvfsel $FRT, $FRA, $FRC, $FRB", IIC_VecPerm,
+                        [(set v4f32:$FRT, (vselect v4i1:$FRA,
+                                                   v4f32:$FRC, v4f32:$FRB))]>;
+  let isCodeGenOnly = 1 in
+  def QVFSELbb: AForm_1<4, 23, (outs qbrc:$FRT),
+                        (ins qbrc:$FRA, qbrc:$FRB, qbrc:$FRC),
+                        "qvfsel $FRT, $FRA, $FRC, $FRB", IIC_VecPerm,
+                        [(set v4i1:$FRT, (vselect v4i1:$FRA,
+                                                  v4i1:$FRC, v4i1:$FRB))]>;
+
+  // SELECT_CC_* - Used to implement the SELECT_CC DAG operation.  Expanded after
+  // instruction selection into a branch sequence.
+  def SELECT_CC_QFRC: PPCCustomInserterPseudo<(outs qfrc:$dst), (ins crrc:$cond, qfrc:$T, qfrc:$F,
+                              i32imm:$BROPC), "#SELECT_CC_QFRC",
+                              []>;
+  def SELECT_CC_QSRC: PPCCustomInserterPseudo<(outs qsrc:$dst), (ins crrc:$cond, qsrc:$T, qsrc:$F,
+                              i32imm:$BROPC), "#SELECT_CC_QSRC",
+                              []>;
+  def SELECT_CC_QBRC: PPCCustomInserterPseudo<(outs qbrc:$dst), (ins crrc:$cond, qbrc:$T, qbrc:$F,
+                              i32imm:$BROPC), "#SELECT_CC_QBRC",
+                              []>;
+
+  // SELECT_* pseudo instructions, like SELECT_CC_* but taking condition
+  // register bit directly.
+  def SELECT_QFRC: PPCCustomInserterPseudo<(outs qfrc:$dst), (ins crbitrc:$cond,
+                          qfrc:$T, qfrc:$F), "#SELECT_QFRC",
+                          [(set v4f64:$dst,
+                                (select i1:$cond, v4f64:$T, v4f64:$F))]>;
+  def SELECT_QSRC: PPCCustomInserterPseudo<(outs qsrc:$dst), (ins crbitrc:$cond,
+                          qsrc:$T, qsrc:$F), "#SELECT_QSRC",
+                          [(set v4f32:$dst,
+                                (select i1:$cond, v4f32:$T, v4f32:$F))]>;
+  def SELECT_QBRC: PPCCustomInserterPseudo<(outs qbrc:$dst), (ins crbitrc:$cond,
+                          qbrc:$T, qbrc:$F), "#SELECT_QBRC",
+                          [(set v4i1:$dst,
+                                (select i1:$cond, v4i1:$T, v4i1:$F))]>;
+
+  // Convert and Round Instructions
+  def QVFCTID : QPXX19_Int<4, 814, "qvfctid", int_ppc_qpx_qvfctid>;
+  let isCodeGenOnly = 1 in
+    def QVFCTIDb : XForm_19<4, 814, (outs qbrc:$FRT), (ins qbrc:$FRB),
+                            "qvfctid $FRT, $FRB", IIC_FPGeneral, []>;
+
+  def QVFCTIDU : QPXX19_Int<4, 942, "qvfctidu", int_ppc_qpx_qvfctidu>;
+  def QVFCTIDZ : QPXX19_Int<4, 815, "qvfctidz", int_ppc_qpx_qvfctidz>;
+  def QVFCTIDUZ : QPXX19_Int<4, 943, "qvfctiduz", int_ppc_qpx_qvfctiduz>;
+  def QVFCTIW : QPXX19_Int<4, 14, "qvfctiw", int_ppc_qpx_qvfctiw>;
+  def QVFCTIWU : QPXX19_Int<4, 142, "qvfctiwu", int_ppc_qpx_qvfctiwu>;
+  def QVFCTIWZ : QPXX19_Int<4, 15, "qvfctiwz", int_ppc_qpx_qvfctiwz>;
+  def QVFCTIWUZ : QPXX19_Int<4, 143, "qvfctiwuz", int_ppc_qpx_qvfctiwuz>;
+  def QVFCFID : QPXX19_Int<4, 846, "qvfcfid", int_ppc_qpx_qvfcfid>;
+  let isCodeGenOnly = 1 in
+    def QVFCFIDb : XForm_19<4, 846, (outs qbrc:$FRT), (ins qbrc:$FRB),
+                            "qvfcfid $FRT, $FRB", IIC_FPGeneral, []>;
+
+  def QVFCFIDU : QPXX19_Int<4, 974, "qvfcfidu", int_ppc_qpx_qvfcfidu>;
+  def QVFCFIDS : QPXX19_Int<0, 846, "qvfcfids", int_ppc_qpx_qvfcfids>;
+  def QVFCFIDUS : QPXX19_Int<0, 974, "qvfcfidus", int_ppc_qpx_qvfcfidus>;
+
+  let isCodeGenOnly = 1 in
+    def QVFRSP : QPXX19_Int<4, 12, "qvfrsp", int_ppc_qpx_qvfrsp>;
+  def QVFRSPs : XForm_19<4, 12,
+                      (outs qsrc:$FRT), (ins qfrc:$FRB),
+                      "qvfrsp $FRT, $FRB", IIC_FPGeneral,
+                      [(set v4f32:$FRT, (fround_inexact v4f64:$FRB))]>;
+
+  def QVFRIZ : XForm_19<4, 424, (outs qfrc:$FRT), (ins qfrc:$FRB),
+                        "qvfriz $FRT, $FRB", IIC_FPGeneral,
+                        [(set v4f64:$FRT, (ftrunc v4f64:$FRB))]>;
+  let isCodeGenOnly = 1 in
+    def QVFRIZs : XForm_19<4, 424, (outs qsrc:$FRT), (ins qsrc:$FRB),
+                           "qvfriz $FRT, $FRB", IIC_FPGeneral,
+                           [(set v4f32:$FRT, (ftrunc v4f32:$FRB))]>;
+
+  def QVFRIN : XForm_19<4, 392, (outs qfrc:$FRT), (ins qfrc:$FRB),
+                        "qvfrin $FRT, $FRB", IIC_FPGeneral,
+                        [(set v4f64:$FRT, (fround v4f64:$FRB))]>;
+  let isCodeGenOnly = 1 in
+    def QVFRINs : XForm_19<4, 392, (outs qsrc:$FRT), (ins qsrc:$FRB),
+                           "qvfrin $FRT, $FRB", IIC_FPGeneral,
+                           [(set v4f32:$FRT, (fround v4f32:$FRB))]>;
+
+  def QVFRIP : XForm_19<4, 456, (outs qfrc:$FRT), (ins qfrc:$FRB),
+                        "qvfrip $FRT, $FRB", IIC_FPGeneral,
+                        [(set v4f64:$FRT, (fceil v4f64:$FRB))]>;
+  let isCodeGenOnly = 1 in
+    def QVFRIPs : XForm_19<4, 456, (outs qsrc:$FRT), (ins qsrc:$FRB),
+                           "qvfrip $FRT, $FRB", IIC_FPGeneral,
+                           [(set v4f32:$FRT, (fceil v4f32:$FRB))]>;
+
+  def QVFRIM : XForm_19<4, 488, (outs qfrc:$FRT), (ins qfrc:$FRB),
+                        "qvfrim $FRT, $FRB", IIC_FPGeneral,
+                        [(set v4f64:$FRT, (ffloor v4f64:$FRB))]>;
+  let isCodeGenOnly = 1 in
+    def QVFRIMs : XForm_19<4, 488, (outs qsrc:$FRT), (ins qsrc:$FRB),
+                           "qvfrim $FRT, $FRB", IIC_FPGeneral,
+                           [(set v4f32:$FRT, (ffloor v4f32:$FRB))]>;
+
+  // Move Instructions
+  def QVFMR : XForm_19<4, 72,
+                      (outs qfrc:$FRT), (ins qfrc:$FRB),
+                      "qvfmr $FRT, $FRB", IIC_VecPerm,
+                      [/* (set v4f64:$FRT, v4f64:$FRB) */]>;
+  let isCodeGenOnly = 1 in {
+    def QVFMRs : XForm_19<4, 72,
+                         (outs qsrc:$FRT), (ins qsrc:$FRB),
+                         "qvfmr $FRT, $FRB", IIC_VecPerm,
+                         [/* (set v4f32:$FRT, v4f32:$FRB) */]>;
+    def QVFMRb : XForm_19<4, 72,
+                         (outs qbrc:$FRT), (ins qbrc:$FRB),
+                         "qvfmr $FRT, $FRB", IIC_VecPerm,
+                         [/* (set v4i1:$FRT, v4i1:$FRB) */]>;
+  }
+  def QVFNEG : XForm_19<4, 40,
+                      (outs qfrc:$FRT), (ins qfrc:$FRB),
+                      "qvfneg $FRT, $FRB", IIC_VecPerm,
+                      [(set v4f64:$FRT, (fneg v4f64:$FRB))]>;
+  let isCodeGenOnly = 1 in
+    def QVFNEGs : XForm_19<4, 40,
+                         (outs qsrc:$FRT), (ins qsrc:$FRB),
+                         "qvfneg $FRT, $FRB", IIC_VecPerm,
+                         [(set v4f32:$FRT, (fneg v4f32:$FRB))]>;
+  def QVFABS : XForm_19<4, 264,
+                      (outs qfrc:$FRT), (ins qfrc:$FRB),
+                      "qvfabs $FRT, $FRB", IIC_VecPerm,
+                      [(set v4f64:$FRT, (fabs v4f64:$FRB))]>;
+  let isCodeGenOnly = 1 in
+    def QVFABSs : XForm_19<4, 264,
+                         (outs qsrc:$FRT), (ins qsrc:$FRB),
+                         "qvfabs $FRT, $FRB", IIC_VecPerm,
+                         [(set v4f32:$FRT, (fabs v4f32:$FRB))]>;
+  def QVFNABS : XForm_19<4, 136,
+                      (outs qfrc:$FRT), (ins qfrc:$FRB),
+                      "qvfnabs $FRT, $FRB", IIC_VecPerm,
+                      [(set v4f64:$FRT, (fneg (fabs v4f64:$FRB)))]>;
+  let isCodeGenOnly = 1 in
+    def QVFNABSs : XForm_19<4, 136,
+                         (outs qsrc:$FRT), (ins qsrc:$FRB),
+                         "qvfnabs $FRT, $FRB", IIC_VecPerm,
+                         [(set v4f32:$FRT, (fneg (fabs v4f32:$FRB)))]>;
+  def QVFCPSGN : XForm_18<4, 8,
+                      (outs qfrc:$FRT), (ins qfrc:$FRA, qfrc:$FRB),
+                      "qvfcpsgn $FRT, $FRA, $FRB", IIC_VecPerm,
+                      [(set v4f64:$FRT, (fcopysign v4f64:$FRB, v4f64:$FRA))]>;
+  let isCodeGenOnly = 1 in
+    def QVFCPSGNs : XForm_18<4, 8,
+                         (outs qsrc:$FRT), (ins qsrc:$FRA, qsrc:$FRB),
+                         "qvfcpsgn $FRT, $FRA, $FRB", IIC_VecPerm,
+                         [(set v4f32:$FRT, (fcopysign v4f32:$FRB, v4f32:$FRA))]>;
+
+  def QVALIGNI : Z23Form_1<4, 5,
+                      (outs qfrc:$FRT), (ins qfrc:$FRA, qfrc:$FRB, u2imm:$idx),
+                      "qvaligni $FRT, $FRA, $FRB, $idx", IIC_VecPerm,
+                      [(set v4f64:$FRT,
+                            (PPCqvaligni v4f64:$FRA, v4f64:$FRB,
+                                         (i32 imm:$idx)))]>;
+  let isCodeGenOnly = 1 in
+     def QVALIGNIs : Z23Form_1<4, 5,
+                         (outs qsrc:$FRT), (ins qsrc:$FRA, qsrc:$FRB, u2imm:$idx),
+                         "qvaligni $FRT, $FRA, $FRB, $idx", IIC_VecPerm,
+                         [(set v4f32:$FRT,
+                               (PPCqvaligni v4f32:$FRA, v4f32:$FRB,
+                                            (i32 imm:$idx)))]>;
+  let isCodeGenOnly = 1 in
+     def QVALIGNIb : Z23Form_1<4, 5,
+                         (outs qbrc:$FRT), (ins qbrc:$FRA, qbrc:$FRB, u2imm:$idx),
+                         "qvaligni $FRT, $FRA, $FRB, $idx", IIC_VecPerm,
+                         [(set v4i1:$FRT,
+                               (PPCqvaligni v4i1:$FRA, v4i1:$FRB,
+                                            (i32 imm:$idx)))]>;
+
+  def QVESPLATI : Z23Form_2<4, 37,
+                      (outs qfrc:$FRT), (ins qfrc:$FRA, u2imm:$idx),
+                      "qvesplati $FRT, $FRA, $idx", IIC_VecPerm,
+                      [(set v4f64:$FRT,
+                            (PPCqvesplati v4f64:$FRA, (i32 imm:$idx)))]>;
+  let isCodeGenOnly = 1 in
+     def QVESPLATIs : Z23Form_2<4, 37,
+                         (outs qsrc:$FRT), (ins qsrc:$FRA, u2imm:$idx),
+                         "qvesplati $FRT, $FRA, $idx", IIC_VecPerm,
+                         [(set v4f32:$FRT,
+                               (PPCqvesplati v4f32:$FRA, (i32 imm:$idx)))]>;
+  let isCodeGenOnly = 1 in
+     def QVESPLATIb : Z23Form_2<4, 37,
+                         (outs qbrc:$FRT), (ins qbrc:$FRA, u2imm:$idx),
+                         "qvesplati $FRT, $FRA, $idx", IIC_VecPerm,
+                         [(set v4i1:$FRT,
+                               (PPCqvesplati v4i1:$FRA, (i32 imm:$idx)))]>;
+
+  def QVFPERM : AForm_1<4, 6,
+                      (outs qfrc:$FRT), (ins qfrc:$FRA, qfrc:$FRB, qfrc:$FRC),
+                      "qvfperm $FRT, $FRA, $FRB, $FRC", IIC_VecPerm,
+                      [(set v4f64:$FRT,
+                            (PPCqvfperm v4f64:$FRA, v4f64:$FRB, v4f64:$FRC))]>;
+  let isCodeGenOnly = 1 in
+     def QVFPERMs : AForm_1<4, 6,
+                         (outs qsrc:$FRT), (ins qsrc:$FRA, qsrc:$FRB, qfrc:$FRC),
+                         "qvfperm $FRT, $FRA, $FRB, $FRC", IIC_VecPerm,
+                         [(set v4f32:$FRT,
+                               (PPCqvfperm v4f32:$FRA, v4f32:$FRB, v4f64:$FRC))]>;
+
+  let isReMaterializable = 1, isAsCheapAsAMove = 1 in
+  def QVGPCI : Z23Form_3<4, 133,
+                      (outs qfrc:$FRT), (ins u12imm:$idx),
+                      "qvgpci $FRT, $idx", IIC_VecPerm,
+                      [(set v4f64:$FRT, (PPCqvgpci (u12:$idx)))]>;
+
+  // Compare Instruction
+  let isCodeGenOnly = 1 in
+    def QVFTSTNAN : QPXX18_Int<4, 64, "qvftstnan", int_ppc_qpx_qvftstnan>;
+  def QVFTSTNANb : XForm_18<4, 64, (outs qbrc:$FRT), (ins qfrc:$FRA, qfrc:$FRB),
+                           "qvftstnan $FRT, $FRA, $FRB", IIC_FPCompare,
+                           [(set v4i1:$FRT,
+                                 (setcc v4f64:$FRA, v4f64:$FRB, SETUO))]>;
+  let isCodeGenOnly = 1 in
+  def QVFTSTNANbs : XForm_18<4, 64, (outs qbrc:$FRT), (ins qsrc:$FRA, qsrc:$FRB),
+                            "qvftstnan $FRT, $FRA, $FRB", IIC_FPCompare,
+                            [(set v4i1:$FRT,
+                                  (setcc v4f32:$FRA, v4f32:$FRB, SETUO))]>;
+  let isCodeGenOnly = 1 in
+    def QVFCMPLT : QPXX18_Int<4, 96, "qvfcmplt", int_ppc_qpx_qvfcmplt>;
+  def QVFCMPLTb : XForm_18<4, 96, (outs qbrc:$FRT), (ins qfrc:$FRA, qfrc:$FRB),
+                           "qvfcmplt $FRT, $FRA, $FRB", IIC_FPCompare,
+                           [(set v4i1:$FRT,
+                                 (setcc v4f64:$FRA, v4f64:$FRB, SETOLT))]>;
+  let isCodeGenOnly = 1 in
+  def QVFCMPLTbs : XForm_18<4, 96, (outs qbrc:$FRT), (ins qsrc:$FRA, qsrc:$FRB),
+                            "qvfcmplt $FRT, $FRA, $FRB", IIC_FPCompare,
+                            [(set v4i1:$FRT,
+                                  (setcc v4f32:$FRA, v4f32:$FRB, SETOLT))]>;
+  let isCodeGenOnly = 1 in
+    def QVFCMPGT : QPXX18_Int<4, 32, "qvfcmpgt", int_ppc_qpx_qvfcmpgt>;
+  def QVFCMPGTb : XForm_18<4, 32, (outs qbrc:$FRT), (ins qfrc:$FRA, qfrc:$FRB),
+                           "qvfcmpgt $FRT, $FRA, $FRB", IIC_FPCompare,
+                           [(set v4i1:$FRT,
+                                 (setcc v4f64:$FRA, v4f64:$FRB, SETOGT))]>;
+  let isCodeGenOnly = 1 in
+  def QVFCMPGTbs : XForm_18<4, 32, (outs qbrc:$FRT), (ins qsrc:$FRA, qsrc:$FRB),
+                            "qvfcmpgt $FRT, $FRA, $FRB", IIC_FPCompare,
+                            [(set v4i1:$FRT,
+                                  (setcc v4f32:$FRA, v4f32:$FRB, SETOGT))]>;
+  let isCodeGenOnly = 1 in
+    def QVFCMPEQ : QPXX18_Int<4, 0, "qvfcmpeq", int_ppc_qpx_qvfcmpeq>;
+  def QVFCMPEQb : XForm_18<4, 0, (outs qbrc:$FRT), (ins qfrc:$FRA, qfrc:$FRB),
+                           "qvfcmpeq $FRT, $FRA, $FRB", IIC_FPCompare,
+                           [(set v4i1:$FRT,
+                                 (setcc v4f64:$FRA, v4f64:$FRB, SETOEQ))]>;
+  let isCodeGenOnly = 1 in
+  def QVFCMPEQbs : XForm_18<4, 0, (outs qbrc:$FRT), (ins qsrc:$FRA, qsrc:$FRB),
+                            "qvfcmpeq $FRT, $FRA, $FRB", IIC_FPCompare,
+                            [(set v4i1:$FRT,
+                                  (setcc v4f32:$FRA, v4f32:$FRB, SETOEQ))]>;
+
+  let isCodeGenOnly = 1 in
+  def QVFLOGICAL : XForm_20<4, 4,
+                      (outs qfrc:$FRT), (ins qfrc:$FRA, qfrc:$FRB, u12imm:$tttt),
+                      "qvflogical $FRT, $FRA, $FRB, $tttt", IIC_VecPerm, []>;
+  def QVFLOGICALb : XForm_20<4, 4,
+                      (outs qbrc:$FRT), (ins qbrc:$FRA, qbrc:$FRB, u12imm:$tttt),
+                      "qvflogical $FRT, $FRA, $FRB, $tttt", IIC_VecPerm, []>;
+  let isCodeGenOnly = 1 in
+  def QVFLOGICALs : XForm_20<4, 4,
+                      (outs qbrc:$FRT), (ins qbrc:$FRA, qbrc:$FRB, u12imm:$tttt),
+                      "qvflogical $FRT, $FRA, $FRB, $tttt", IIC_VecPerm, []>;
+
+  // Load indexed instructions
+  let mayLoad = 1 in {
+    def QVLFDX : XForm_1_memOp<31, 583,
+                              (outs qfrc:$FRT), (ins memrr:$src),
+                              "qvlfdx $FRT, $src", IIC_LdStLFD,
+                              [(set v4f64:$FRT, (load xoaddr:$src))]>;
+    let isCodeGenOnly = 1 in
+    def QVLFDXb : XForm_1_memOp<31, 583,
+                                (outs qbrc:$FRT), (ins memrr:$src),
+                                "qvlfdx $FRT, $src", IIC_LdStLFD, []>;
+
+    let RC = 1 in
+    def QVLFDXA : XForm_1<31, 583,
+                        (outs qfrc:$FRT), (ins memrr:$src),
+                        "qvlfdxa $FRT, $src", IIC_LdStLFD, []>;
+
+    def QVLFDUX : XForm_1<31, 615,
+                        (outs qfrc:$FRT, ptr_rc_nor0:$ea_result),
+                        (ins memrr:$src),
+                        "qvlfdux $FRT, $src", IIC_LdStLFDU, []>,
+                        RegConstraint<"$src.ptrreg = $ea_result">,
+                        NoEncode<"$ea_result">;
+    let RC = 1 in
+    def QVLFDUXA : XForm_1<31, 615,
+                        (outs qfrc:$FRT), (ins memrr:$src),
+                        "qvlfduxa $FRT, $src", IIC_LdStLFD, []>;
+
+    def QVLFSX : XForm_1_memOp<31, 519,
+                              (outs qfrc:$FRT), (ins memrr:$src),
+                              "qvlfsx $FRT, $src", IIC_LdStLFD,
+                              [(set v4f64:$FRT, (extloadv4f32 xoaddr:$src))]>;
+
+    let isCodeGenOnly = 1 in
+    def QVLFSXb : XForm_1<31, 519,
+                        (outs qbrc:$FRT), (ins memrr:$src),
+                        "qvlfsx $FRT, $src", IIC_LdStLFD,
+                        [(set v4i1:$FRT, (PPCqvlfsb xoaddr:$src))]>;
+    let isCodeGenOnly = 1 in
+    def QVLFSXs : XForm_1_memOp<31, 519,
+                                (outs qsrc:$FRT), (ins memrr:$src),
+                                "qvlfsx $FRT, $src", IIC_LdStLFD,
+                                [(set v4f32:$FRT, (load xoaddr:$src))]>;
+
+    let RC = 1 in
+    def QVLFSXA : XForm_1<31, 519,
+                        (outs qfrc:$FRT), (ins memrr:$src),
+                        "qvlfsxa $FRT, $src", IIC_LdStLFD, []>;
+
+    def QVLFSUX : XForm_1<31, 551,
+                        (outs qsrc:$FRT, ptr_rc_nor0:$ea_result),
+                        (ins memrr:$src),
+                        "qvlfsux $FRT, $src", IIC_LdStLFDU, []>,
+                        RegConstraint<"$src.ptrreg = $ea_result">,
+                        NoEncode<"$ea_result">;
+
+    let RC = 1 in
+    def QVLFSUXA : XForm_1<31, 551,
+                        (outs qfrc:$FRT), (ins memrr:$src),
+                        "qvlfsuxa $FRT, $src", IIC_LdStLFD, []>;
+
+    def QVLFCDX : XForm_1<31, 71,
+                        (outs qfrc:$FRT), (ins memrr:$src),
+                        "qvlfcdx $FRT, $src", IIC_LdStLFD, []>;
+    let RC = 1 in
+    def QVLFCDXA : XForm_1<31, 71,
+                        (outs qfrc:$FRT), (ins memrr:$src),
+                        "qvlfcdxa $FRT, $src", IIC_LdStLFD, []>;
+
+    def QVLFCDUX : XForm_1<31, 103,
+                        (outs qfrc:$FRT), (ins memrr:$src),
+                        "qvlfcdux $FRT, $src", IIC_LdStLFD, []>;
+    let RC = 1 in
+    def QVLFCDUXA : XForm_1<31, 103,
+                        (outs qfrc:$FRT), (ins memrr:$src),
+                        "qvlfcduxa $FRT, $src", IIC_LdStLFD, []>;
+
+    def QVLFCSX : XForm_1<31, 7,
+                        (outs qfrc:$FRT), (ins memrr:$src),
+                        "qvlfcsx $FRT, $src", IIC_LdStLFD, []>;
+    let isCodeGenOnly = 1 in
+    def QVLFCSXs : XForm_1<31, 7,
+                         (outs qsrc:$FRT), (ins memrr:$src),
+                         "qvlfcsx $FRT, $src", IIC_LdStLFD, []>;
+
+    let RC = 1 in
+    def QVLFCSXA : XForm_1<31, 7,
+                        (outs qfrc:$FRT), (ins memrr:$src),
+                        "qvlfcsxa $FRT, $src", IIC_LdStLFD, []>;
+
+    def QVLFCSUX : XForm_1<31, 39,
+                        (outs qfrc:$FRT), (ins memrr:$src),
+                        "qvlfcsux $FRT, $src", IIC_LdStLFD, []>;
+    let RC = 1 in
+    def QVLFCSUXA : XForm_1<31, 39,
+                        (outs qfrc:$FRT), (ins memrr:$src),
+                        "qvlfcsuxa $FRT, $src", IIC_LdStLFD, []>;
+
+    def QVLFIWAX : XForm_1<31, 871,
+                        (outs qfrc:$FRT), (ins memrr:$src),
+                        "qvlfiwax $FRT, $src", IIC_LdStLFD, []>;
+    let RC = 1 in
+    def QVLFIWAXA : XForm_1<31, 871,
+                        (outs qfrc:$FRT), (ins memrr:$src),
+                        "qvlfiwaxa $FRT, $src", IIC_LdStLFD, []>;
+
+    def QVLFIWZX : XForm_1<31, 839,
+                        (outs qfrc:$FRT), (ins memrr:$src),
+                        "qvlfiwzx $FRT, $src", IIC_LdStLFD, []>;
+    let RC = 1 in
+    def QVLFIWZXA : XForm_1<31, 839,
+                        (outs qfrc:$FRT), (ins memrr:$src),
+                        "qvlfiwzxa $FRT, $src", IIC_LdStLFD, []>;
+  }
+
+
+  def QVLPCLDX : XForm_1<31, 582,
+                      (outs qfrc:$FRT), (ins memrr:$src),
+                      "qvlpcldx $FRT, $src", IIC_LdStLFD, []>;
+  def QVLPCLSX : XForm_1<31, 518,
+                      (outs qfrc:$FRT), (ins memrr:$src),
+                      "qvlpclsx $FRT, $src", IIC_LdStLFD, []>;
+  let isCodeGenOnly = 1 in
+    def QVLPCLSXint : XForm_11<31, 518,
+                              (outs qfrc:$FRT), (ins G8RC:$src),
+                              "qvlpclsx $FRT, 0, $src", IIC_LdStLFD, []>;
+  def QVLPCRDX : XForm_1<31, 70,
+                      (outs qfrc:$FRT), (ins memrr:$src),
+                      "qvlpcrdx $FRT, $src", IIC_LdStLFD, []>;
+  def QVLPCRSX : XForm_1<31, 6,
+                      (outs qfrc:$FRT), (ins memrr:$src),
+                      "qvlpcrsx $FRT, $src", IIC_LdStLFD, []>;
+
+  // Store indexed instructions
+  let mayStore = 1 in {
+    def QVSTFDX : XForm_8_memOp<31, 711,
+                        (outs), (ins qfrc:$FRT, memrr:$dst),
+                        "qvstfdx $FRT, $dst", IIC_LdStSTFD,
+                        [(store qfrc:$FRT, xoaddr:$dst)]>;
+    let isCodeGenOnly = 1 in
+    def QVSTFDXb : XForm_8_memOp<31, 711,
+                        (outs), (ins qbrc:$FRT, memrr:$dst),
+                        "qvstfdx $FRT, $dst", IIC_LdStSTFD, []>;
+
+    let RC = 1 in
+    def QVSTFDXA : XForm_8<31, 711,
+                        (outs), (ins qfrc:$FRT, memrr:$dst),
+                        "qvstfdxa $FRT, $dst", IIC_LdStSTFD, []>;
+
+    def QVSTFDUX : XForm_8<31, 743, (outs ptr_rc_nor0:$ea_res),
+                           (ins qfrc:$FRT, memrr:$dst),
+                           "qvstfdux $FRT, $dst", IIC_LdStSTFDU, []>,
+                           RegConstraint<"$dst.ptrreg = $ea_res">,
+                           NoEncode<"$ea_res">;
+
+    let RC = 1 in
+    def QVSTFDUXA : XForm_8<31, 743,
+                        (outs), (ins qfrc:$FRT, memrr:$dst),
+                        "qvstfduxa $FRT, $dst", IIC_LdStSTFD, []>;
+
+    def QVSTFDXI : XForm_8<31, 709,
+                        (outs), (ins qfrc:$FRT, memrr:$dst),
+                        "qvstfdxi $FRT, $dst", IIC_LdStSTFD, []>;
+    let RC = 1 in
+    def QVSTFDXIA : XForm_8<31, 709,
+                        (outs), (ins qfrc:$FRT, memrr:$dst),
+                        "qvstfdxia $FRT, $dst", IIC_LdStSTFD, []>;
+
+    def QVSTFDUXI : XForm_8<31, 741,
+                        (outs), (ins qfrc:$FRT, memrr:$dst),
+                        "qvstfduxi $FRT, $dst", IIC_LdStSTFD, []>;
+    let RC = 1 in
+    def QVSTFDUXIA : XForm_8<31, 741,
+                        (outs), (ins qfrc:$FRT, memrr:$dst),
+                        "qvstfduxia $FRT, $dst", IIC_LdStSTFD, []>;
+
+    def QVSTFSX : XForm_8_memOp<31, 647,
+                        (outs), (ins qfrc:$FRT, memrr:$dst),
+                        "qvstfsx $FRT, $dst", IIC_LdStSTFD,
+                        [(truncstorev4f32 qfrc:$FRT, xoaddr:$dst)]>;
+    let isCodeGenOnly = 1 in
+    def QVSTFSXs : XForm_8_memOp<31, 647,
+                         (outs), (ins qsrc:$FRT, memrr:$dst),
+                         "qvstfsx $FRT, $dst", IIC_LdStSTFD,
+                         [(store qsrc:$FRT, xoaddr:$dst)]>;
+
+    let RC = 1 in
+    def QVSTFSXA : XForm_8<31, 647,
+                        (outs), (ins qfrc:$FRT, memrr:$dst),
+                        "qvstfsxa $FRT, $dst", IIC_LdStSTFD, []>;
+
+    def QVSTFSUX : XForm_8<31, 679, (outs ptr_rc_nor0:$ea_res),
+                           (ins qsrc:$FRT, memrr:$dst),
+                           "qvstfsux $FRT, $dst", IIC_LdStSTFDU, []>,
+                           RegConstraint<"$dst.ptrreg = $ea_res">,
+                           NoEncode<"$ea_res">;
+    let isCodeGenOnly = 1 in
+    def QVSTFSUXs: XForm_8<31, 679, (outs ptr_rc_nor0:$ea_res),
+                           (ins qfrc:$FRT, memrr:$dst),
+                           "qvstfsux $FRT, $dst", IIC_LdStSTFDU, []>,
+                           RegConstraint<"$dst.ptrreg = $ea_res">,
+                           NoEncode<"$ea_res">;
+
+    let RC = 1 in
+    def QVSTFSUXA : XForm_8<31, 679,
+                        (outs), (ins qfrc:$FRT, memrr:$dst),
+                        "qvstfsuxa $FRT, $dst", IIC_LdStSTFD, []>;
+
+    def QVSTFSXI : XForm_8<31, 645,
+                        (outs), (ins qfrc:$FRT, memrr:$dst),
+                        "qvstfsxi $FRT, $dst", IIC_LdStSTFD, []>;
+    let RC = 1 in
+    def QVSTFSXIA : XForm_8<31, 645,
+                        (outs), (ins qfrc:$FRT, memrr:$dst),
+                        "qvstfsxia $FRT, $dst", IIC_LdStSTFD, []>;
+
+    def QVSTFSUXI : XForm_8<31, 677,
+                        (outs), (ins qfrc:$FRT, memrr:$dst),
+                        "qvstfsuxi $FRT, $dst", IIC_LdStSTFD, []>;
+    let RC = 1 in
+    def QVSTFSUXIA : XForm_8<31, 677,
+                        (outs), (ins qfrc:$FRT, memrr:$dst),
+                        "qvstfsuxia $FRT, $dst", IIC_LdStSTFD, []>;
+
+    def QVSTFCDX : XForm_8<31, 199,
+                        (outs), (ins qfrc:$FRT, memrr:$dst),
+                        "qvstfcdx $FRT, $dst", IIC_LdStSTFD, []>;
+    let RC = 1 in
+    def QVSTFCDXA : XForm_8<31, 199,
+                        (outs), (ins qfrc:$FRT, memrr:$dst),
+                        "qvstfcdxa $FRT, $dst", IIC_LdStSTFD, []>;
+
+    def QVSTFCSX : XForm_8<31, 135,
+                        (outs), (ins qfrc:$FRT, memrr:$dst),
+                        "qvstfcsx $FRT, $dst", IIC_LdStSTFD, []>;
+    let isCodeGenOnly = 1 in
+    def QVSTFCSXs : XForm_8<31, 135,
+                         (outs), (ins qsrc:$FRT, memrr:$dst),
+                         "qvstfcsx $FRT, $dst", IIC_LdStSTFD, []>;
+
+    let RC = 1 in
+    def QVSTFCSXA : XForm_8<31, 135,
+                        (outs), (ins qfrc:$FRT, memrr:$dst),
+                        "qvstfcsxa $FRT, $dst", IIC_LdStSTFD, []>;
+
+    def QVSTFCDUX : XForm_8<31, 231,
+                        (outs), (ins qfrc:$FRT, memrr:$dst),
+                        "qvstfcdux $FRT, $dst", IIC_LdStSTFD, []>;
+    let RC = 1 in
+    def QVSTFCDUXA : XForm_8<31, 231,
+                        (outs), (ins qfrc:$FRT, memrr:$dst),
+                        "qvstfcduxa $FRT, $dst", IIC_LdStSTFD, []>;
+
+    def QVSTFCSUX : XForm_8<31, 167,
+                        (outs), (ins qfrc:$FRT, memrr:$dst),
+                        "qvstfcsux $FRT, $dst", IIC_LdStSTFD, []>;
+    let RC = 1 in
+    def QVSTFCSUXA : XForm_8<31, 167,
+                        (outs), (ins qfrc:$FRT, memrr:$dst),
+                        "qvstfcsuxa $FRT, $dst", IIC_LdStSTFD, []>;
+
+    def QVSTFCDXI : XForm_8<31, 197,
+                        (outs), (ins qfrc:$FRT, memrr:$dst),
+                        "qvstfcdxi $FRT, $dst", IIC_LdStSTFD, []>;
+    let RC = 1 in
+    def QVSTFCDXIA : XForm_8<31, 197,
+                        (outs), (ins qfrc:$FRT, memrr:$dst),
+                        "qvstfcdxia $FRT, $dst", IIC_LdStSTFD, []>;
+
+    def QVSTFCSXI : XForm_8<31, 133,
+                        (outs), (ins qfrc:$FRT, memrr:$dst),
+                        "qvstfcsxi $FRT, $dst", IIC_LdStSTFD, []>;
+    let RC = 1 in
+    def QVSTFCSXIA : XForm_8<31, 133,
+                        (outs), (ins qfrc:$FRT, memrr:$dst),
+                        "qvstfcsxia $FRT, $dst", IIC_LdStSTFD, []>;
+
+    def QVSTFCDUXI : XForm_8<31, 229,
+                        (outs), (ins qfrc:$FRT, memrr:$dst),
+                        "qvstfcduxi $FRT, $dst", IIC_LdStSTFD, []>;
+    let RC = 1 in
+    def QVSTFCDUXIA : XForm_8<31, 229,
+                        (outs), (ins qfrc:$FRT, memrr:$dst),
+                        "qvstfcduxia $FRT, $dst", IIC_LdStSTFD, []>;
+
+    def QVSTFCSUXI : XForm_8<31, 165,
+                        (outs), (ins qfrc:$FRT, memrr:$dst),
+                        "qvstfcsuxi $FRT, $dst", IIC_LdStSTFD, []>;
+    let RC = 1 in
+    def QVSTFCSUXIA : XForm_8<31, 165,
+                        (outs), (ins qfrc:$FRT, memrr:$dst),
+                        "qvstfcsuxia $FRT, $dst", IIC_LdStSTFD, []>;
+
+    def QVSTFIWX : XForm_8<31, 967,
+                        (outs), (ins qfrc:$FRT, memrr:$dst),
+                        "qvstfiwx $FRT, $dst", IIC_LdStSTFD, []>;
+    let RC = 1 in
+    def QVSTFIWXA : XForm_8<31, 967,
+                        (outs), (ins qfrc:$FRT, memrr:$dst),
+                        "qvstfiwxa $FRT, $dst", IIC_LdStSTFD, []>;
+  }
+}
+
+} // neverHasSideEffects
+}
+
+def : InstAlias<"qvfclr $FRT",
+                (QVFLOGICALb qbrc:$FRT, qbrc:$FRT, qbrc:$FRT, 0)>;
+def : InstAlias<"qvfand $FRT, $FRA, $FRB",
+                (QVFLOGICALb qbrc:$FRT, qbrc:$FRA, qbrc:$FRB, 1)>;
+def : InstAlias<"qvfandc $FRT, $FRA, $FRB",
+                (QVFLOGICALb qbrc:$FRT, qbrc:$FRA, qbrc:$FRB, 4)>;
+def : InstAlias<"qvfctfb $FRT, $FRA",
+                (QVFLOGICALb qbrc:$FRT, qbrc:$FRA, qbrc:$FRA, 5)>;
+def : InstAlias<"qvfxor $FRT, $FRA, $FRB",
+                (QVFLOGICALb qbrc:$FRT, qbrc:$FRA, qbrc:$FRB, 6)>;
+def : InstAlias<"qvfor $FRT, $FRA, $FRB",
+                (QVFLOGICALb qbrc:$FRT, qbrc:$FRA, qbrc:$FRB, 7)>;
+def : InstAlias<"qvfnor $FRT, $FRA, $FRB",
+                (QVFLOGICALb qbrc:$FRT, qbrc:$FRA, qbrc:$FRB, 8)>;
+def : InstAlias<"qvfequ $FRT, $FRA, $FRB",
+                (QVFLOGICALb qbrc:$FRT, qbrc:$FRA, qbrc:$FRB, 9)>;
+def : InstAlias<"qvfnot $FRT, $FRA",
+                (QVFLOGICALb qbrc:$FRT, qbrc:$FRA, qbrc:$FRA, 10)>;
+def : InstAlias<"qvforc $FRT, $FRA, $FRB",
+                (QVFLOGICALb qbrc:$FRT, qbrc:$FRA, qbrc:$FRB, 13)>;
+def : InstAlias<"qvfnand $FRT, $FRA, $FRB",
+                (QVFLOGICALb qbrc:$FRT, qbrc:$FRA, qbrc:$FRB, 14)>;
+def : InstAlias<"qvfset $FRT",
+                (QVFLOGICALb qbrc:$FRT, qbrc:$FRT, qbrc:$FRT, 15)>;
+
+//===----------------------------------------------------------------------===//
+// Additional QPX Patterns
+//
+
+def : Pat<(v4f64 (scalar_to_vector f64:$A)),
+          (INSERT_SUBREG (v4f64 (IMPLICIT_DEF)), $A, sub_64)>;
+def : Pat<(v4f32 (scalar_to_vector f32:$A)),
+          (INSERT_SUBREG (v4f32 (IMPLICIT_DEF)), $A, sub_64)>;
+
+def : Pat<(f64 (extractelt v4f64:$S, 0)),
+          (EXTRACT_SUBREG $S, sub_64)>;
+def : Pat<(f32 (extractelt v4f32:$S, 0)),
+          (EXTRACT_SUBREG $S, sub_64)>;
+
+def : Pat<(f64 (extractelt v4f64:$S, 1)),
+          (EXTRACT_SUBREG (QVESPLATI $S, 1), sub_64)>;
+def : Pat<(f64 (extractelt v4f64:$S, 2)),
+          (EXTRACT_SUBREG (QVESPLATI $S, 2), sub_64)>;
+def : Pat<(f64 (extractelt v4f64:$S, 3)),
+          (EXTRACT_SUBREG (QVESPLATI $S, 3), sub_64)>;
+
+def : Pat<(f32 (extractelt v4f32:$S, 1)),
+          (EXTRACT_SUBREG (QVESPLATIs $S, 1), sub_64)>;
+def : Pat<(f32 (extractelt v4f32:$S, 2)),
+          (EXTRACT_SUBREG (QVESPLATIs $S, 2), sub_64)>;
+def : Pat<(f32 (extractelt v4f32:$S, 3)),
+          (EXTRACT_SUBREG (QVESPLATIs $S, 3), sub_64)>;
+
+def : Pat<(f64 (extractelt v4f64:$S, i64:$F)),
+          (EXTRACT_SUBREG (QVFPERM $S, $S,
+                                   (QVLPCLSXint (RLDICR $F, 2,
+                                                        /* 63-2 = */ 61))),
+                          sub_64)>;
+def : Pat<(f32 (extractelt v4f32:$S, i64:$F)),
+          (EXTRACT_SUBREG (QVFPERMs $S, $S,
+                                    (QVLPCLSXint (RLDICR $F, 2,
+                                                         /* 63-2 = */ 61))),
+                          sub_64)>;
+
+def : Pat<(int_ppc_qpx_qvfperm v4f64:$A, v4f64:$B, v4f64:$C),
+          (QVFPERM $A, $B, $C)>;
+
+def : Pat<(int_ppc_qpx_qvfcpsgn v4f64:$A, v4f64:$B),
+          (QVFCPSGN $A, $B)>;
+
+// FCOPYSIGN's operand types need not agree.
+def : Pat<(fcopysign v4f64:$frB, v4f32:$frA),
+          (QVFCPSGN (COPY_TO_REGCLASS $frA, QFRC), $frB)>;
+def : Pat<(fcopysign QSRC:$frB, QFRC:$frA),
+          (QVFCPSGNs (COPY_TO_REGCLASS $frA, QSRC), $frB)>;
+
+def : Pat<(int_ppc_qpx_qvfneg v4f64:$A), (QVFNEG $A)>;
+def : Pat<(int_ppc_qpx_qvfabs v4f64:$A), (QVFABS $A)>;
+def : Pat<(int_ppc_qpx_qvfnabs v4f64:$A), (QVFNABS $A)>;
+
+def : Pat<(int_ppc_qpx_qvfriz v4f64:$A), (QVFRIZ $A)>;
+def : Pat<(int_ppc_qpx_qvfrin v4f64:$A), (QVFRIN $A)>;
+def : Pat<(int_ppc_qpx_qvfrip v4f64:$A), (QVFRIP $A)>;
+def : Pat<(int_ppc_qpx_qvfrim v4f64:$A), (QVFRIM $A)>;
+
+def : Pat<(int_ppc_qpx_qvfre v4f64:$A), (QVFRE $A)>;
+def : Pat<(int_ppc_qpx_qvfrsqrte v4f64:$A), (QVFRSQRTE $A)>;
+
+def : Pat<(int_ppc_qpx_qvfadd v4f64:$A, v4f64:$B),
+          (QVFADD $A, $B)>;
+def : Pat<(int_ppc_qpx_qvfsub v4f64:$A, v4f64:$B),
+          (QVFSUB $A, $B)>;
+def : Pat<(int_ppc_qpx_qvfmul v4f64:$A, v4f64:$B),
+          (QVFMUL $A, $B)>;
+
+// Additional QVFNMSUB patterns: -a*c + b == -(a*c - b)
+def : Pat<(fma (fneg v4f64:$A), v4f64:$C, v4f64:$B),
+          (QVFNMSUB $A, $C, $B)>;
+def : Pat<(fma v4f64:$A, (fneg v4f64:$C), v4f64:$B),
+          (QVFNMSUB $A, $C, $B)>;
+def : Pat<(fma (fneg v4f32:$A), v4f32:$C, v4f32:$B),
+          (QVFNMSUBSs $A, $C, $B)>;
+def : Pat<(fma v4f32:$A, (fneg v4f32:$C), v4f32:$B),
+          (QVFNMSUBSs $A, $C, $B)>;
+
+def : Pat<(int_ppc_qpx_qvfmadd v4f64:$A, v4f64:$B, v4f64:$C),
+          (QVFMADD $A, $B, $C)>;
+def : Pat<(int_ppc_qpx_qvfnmadd v4f64:$A, v4f64:$B, v4f64:$C),
+          (QVFNMADD $A, $B, $C)>;
+def : Pat<(int_ppc_qpx_qvfmsub v4f64:$A, v4f64:$B, v4f64:$C),
+          (QVFMSUB $A, $B, $C)>;
+def : Pat<(int_ppc_qpx_qvfnmsub v4f64:$A, v4f64:$B, v4f64:$C),
+          (QVFNMSUB $A, $B, $C)>;
+
+def : Pat<(int_ppc_qpx_qvlfd xoaddr:$src),
+          (QVLFDX xoaddr:$src)>;
+def : Pat<(int_ppc_qpx_qvlfda xoaddr:$src),
+          (QVLFDXA xoaddr:$src)>;
+def : Pat<(int_ppc_qpx_qvlfs xoaddr:$src),
+          (QVLFSX xoaddr:$src)>;
+def : Pat<(int_ppc_qpx_qvlfsa xoaddr:$src),
+          (QVLFSXA xoaddr:$src)>;
+def : Pat<(int_ppc_qpx_qvlfcda xoaddr:$src),
+          (QVLFCDXA xoaddr:$src)>;
+def : Pat<(int_ppc_qpx_qvlfcd xoaddr:$src),
+          (QVLFCDX xoaddr:$src)>;
+def : Pat<(int_ppc_qpx_qvlfcsa xoaddr:$src),
+          (QVLFCSXA xoaddr:$src)>;
+def : Pat<(int_ppc_qpx_qvlfcs xoaddr:$src),
+          (QVLFCSX xoaddr:$src)>;
+def : Pat<(int_ppc_qpx_qvlfda xoaddr:$src),
+          (QVLFDXA xoaddr:$src)>;
+def : Pat<(int_ppc_qpx_qvlfiwaa xoaddr:$src),
+          (QVLFIWAXA xoaddr:$src)>;
+def : Pat<(int_ppc_qpx_qvlfiwa xoaddr:$src),
+          (QVLFIWAX xoaddr:$src)>;
+def : Pat<(int_ppc_qpx_qvlfiwza xoaddr:$src),
+          (QVLFIWZXA xoaddr:$src)>;
+def : Pat<(int_ppc_qpx_qvlfiwz xoaddr:$src),
+          (QVLFIWZX xoaddr:$src)>;
+def : Pat<(int_ppc_qpx_qvlfsa xoaddr:$src),
+          (QVLFSXA xoaddr:$src)>;
+def : Pat<(int_ppc_qpx_qvlpcld xoaddr:$src),
+          (QVLPCLDX xoaddr:$src)>;
+def : Pat<(int_ppc_qpx_qvlpcls xoaddr:$src),
+          (QVLPCLSX xoaddr:$src)>;
+def : Pat<(int_ppc_qpx_qvlpcrd xoaddr:$src),
+          (QVLPCRDX xoaddr:$src)>;
+def : Pat<(int_ppc_qpx_qvlpcrs xoaddr:$src),
+          (QVLPCRSX xoaddr:$src)>;
+
+def : Pat<(int_ppc_qpx_qvstfd v4f64:$T, xoaddr:$dst),
+          (QVSTFDX $T, xoaddr:$dst)>;
+def : Pat<(int_ppc_qpx_qvstfs v4f64:$T, xoaddr:$dst),
+          (QVSTFSX $T, xoaddr:$dst)>;
+def : Pat<(int_ppc_qpx_qvstfcda v4f64:$T, xoaddr:$dst),
+          (QVSTFCDXA $T, xoaddr:$dst)>;
+def : Pat<(int_ppc_qpx_qvstfcd v4f64:$T, xoaddr:$dst),
+          (QVSTFCDX $T, xoaddr:$dst)>;
+def : Pat<(int_ppc_qpx_qvstfcsa v4f64:$T, xoaddr:$dst),
+          (QVSTFCSXA $T, xoaddr:$dst)>;
+def : Pat<(int_ppc_qpx_qvstfcs v4f64:$T, xoaddr:$dst),
+          (QVSTFCSX $T, xoaddr:$dst)>;
+def : Pat<(int_ppc_qpx_qvstfda v4f64:$T, xoaddr:$dst),
+          (QVSTFDXA $T, xoaddr:$dst)>;
+def : Pat<(int_ppc_qpx_qvstfiwa v4f64:$T, xoaddr:$dst),
+          (QVSTFIWXA $T, xoaddr:$dst)>;
+def : Pat<(int_ppc_qpx_qvstfiw v4f64:$T, xoaddr:$dst),
+          (QVSTFIWX $T, xoaddr:$dst)>;
+def : Pat<(int_ppc_qpx_qvstfsa v4f64:$T, xoaddr:$dst),
+          (QVSTFSXA $T, xoaddr:$dst)>;
+
+def : Pat<(pre_store v4f64:$rS, iPTR:$ptrreg, iPTR:$ptroff),
+          (QVSTFDUX $rS, $ptrreg, $ptroff)>;
+def : Pat<(pre_store v4f32:$rS, iPTR:$ptrreg, iPTR:$ptroff),
+          (QVSTFSUX $rS, $ptrreg, $ptroff)>;
+def : Pat<(pre_truncstv4f32 v4f64:$rS, iPTR:$ptrreg, iPTR:$ptroff),
+          (QVSTFSUXs $rS, $ptrreg, $ptroff)>;
+
+def : Pat<(int_ppc_qpx_qvflogical  v4f64:$A, v4f64:$B, (i32 imm:$idx)),
+          (QVFLOGICAL $A, $B, imm:$idx)>;
+def : Pat<(int_ppc_qpx_qvgpci (u12:$idx)),
+          (QVGPCI imm:$idx)>;
+
+def : Pat<(setcc v4f64:$FRA, v4f64:$FRB, SETOGE),
+          (QVFLOGICALb (QVFCMPLTb $FRA, $FRB),
+                       (QVFTSTNANb $FRA, $FRB), (i32 8))>;
+def : Pat<(setcc v4f64:$FRA, v4f64:$FRB, SETOLE),
+          (QVFLOGICALb (QVFCMPGTb $FRA, $FRB),
+                       (QVFTSTNANb $FRA, $FRB), (i32 8))>;
+def : Pat<(setcc v4f64:$FRA, v4f64:$FRB, SETONE),
+          (QVFLOGICALb (QVFCMPEQb $FRA, $FRB),
+                       (QVFTSTNANb $FRA, $FRB), (i32 8))>;
+def : Pat<(setcc v4f64:$FRA, v4f64:$FRB, SETO),
+          (QVFLOGICALb (QVFTSTNANb $FRA, $FRB),
+                       (QVFTSTNANb $FRA, $FRB), (i32 10))>;
+def : Pat<(setcc v4f64:$FRA, v4f64:$FRB, SETUEQ),
+          (QVFLOGICALb (QVFCMPEQb $FRA, $FRB),
+                       (QVFTSTNANb $FRA, $FRB), (i32 7))>;
+def : Pat<(setcc v4f64:$FRA, v4f64:$FRB, SETUGT),
+          (QVFLOGICALb (QVFCMPGTb $FRA, $FRB),
+                       (QVFTSTNANb $FRA, $FRB), (i32 7))>;
+def : Pat<(setcc v4f64:$FRA, v4f64:$FRB, SETUGE),
+          (QVFLOGICALb (QVFTSTNANb $FRA, $FRB),
+                       (QVFCMPLTb $FRA, $FRB), (i32 13))>;
+def : Pat<(setcc v4f64:$FRA, v4f64:$FRB, SETULT),
+          (QVFLOGICALb (QVFCMPLTb $FRA, $FRB),
+                       (QVFTSTNANb $FRA, $FRB), (i32 7))>;
+def : Pat<(setcc v4f64:$FRA, v4f64:$FRB, SETULE),
+          (QVFLOGICALb (QVFTSTNANb $FRA, $FRB),
+                       (QVFCMPGTb $FRA, $FRB), (i32 13))>;
+def : Pat<(setcc v4f64:$FRA, v4f64:$FRB, SETUNE),
+          (QVFLOGICALb (QVFTSTNANb $FRA, $FRB),
+                       (QVFCMPEQb $FRA, $FRB), (i32 13))>;
+
+def : Pat<(setcc v4f64:$FRA, v4f64:$FRB, SETEQ),
+          (QVFCMPEQb $FRA, $FRB)>;
+def : Pat<(setcc v4f64:$FRA, v4f64:$FRB, SETGT),
+          (QVFCMPGTb $FRA, $FRB)>;
+def : Pat<(setcc v4f64:$FRA, v4f64:$FRB, SETGE),
+          (QVFLOGICALb (QVFCMPLTb $FRA, $FRB),
+                       (QVFCMPLTb $FRA, $FRB), (i32 10))>;
+def : Pat<(setcc v4f64:$FRA, v4f64:$FRB, SETLT),
+          (QVFCMPLTb $FRA, $FRB)>;
+def : Pat<(setcc v4f64:$FRA, v4f64:$FRB, SETLE),
+          (QVFLOGICALb (QVFCMPGTb $FRA, $FRB),
+                       (QVFCMPGTb $FRA, $FRB), (i32 10))>;
+def : Pat<(setcc v4f64:$FRA, v4f64:$FRB, SETNE),
+          (QVFLOGICALb (QVFCMPEQb $FRA, $FRB),
+                       (QVFCMPEQb $FRA, $FRB), (i32 10))>;
+
+def : Pat<(setcc v4f32:$FRA, v4f32:$FRB, SETOGE),
+          (QVFLOGICALb (QVFCMPLTbs $FRA, $FRB),
+                       (QVFTSTNANbs $FRA, $FRB), (i32 8))>;
+def : Pat<(setcc v4f32:$FRA, v4f32:$FRB, SETOLE),
+          (QVFLOGICALb (QVFCMPGTbs $FRA, $FRB),
+                       (QVFTSTNANbs $FRA, $FRB), (i32 8))>;
+def : Pat<(setcc v4f32:$FRA, v4f32:$FRB, SETONE),
+          (QVFLOGICALb (QVFCMPEQbs $FRA, $FRB),
+                       (QVFTSTNANbs $FRA, $FRB), (i32 8))>;
+def : Pat<(setcc v4f32:$FRA, v4f32:$FRB, SETO),
+          (QVFLOGICALb (QVFTSTNANbs $FRA, $FRB),
+                       (QVFTSTNANbs $FRA, $FRB), (i32 10))>;
+def : Pat<(setcc v4f32:$FRA, v4f32:$FRB, SETUEQ),
+          (QVFLOGICALb (QVFCMPEQbs $FRA, $FRB),
+                       (QVFTSTNANbs $FRA, $FRB), (i32 7))>;
+def : Pat<(setcc v4f32:$FRA, v4f32:$FRB, SETUGT),
+          (QVFLOGICALb (QVFCMPGTbs $FRA, $FRB),
+                       (QVFTSTNANbs $FRA, $FRB), (i32 7))>;
+def : Pat<(setcc v4f32:$FRA, v4f32:$FRB, SETUGE),
+          (QVFLOGICALb (QVFTSTNANbs $FRA, $FRB),
+                       (QVFCMPLTbs $FRA, $FRB), (i32 13))>;
+def : Pat<(setcc v4f32:$FRA, v4f32:$FRB, SETULT),
+          (QVFLOGICALb (QVFCMPLTbs $FRA, $FRB),
+                       (QVFTSTNANbs $FRA, $FRB), (i32 7))>;
+def : Pat<(setcc v4f32:$FRA, v4f32:$FRB, SETULE),
+          (QVFLOGICALb (QVFTSTNANbs $FRA, $FRB),
+                       (QVFCMPGTbs $FRA, $FRB), (i32 13))>;
+def : Pat<(setcc v4f32:$FRA, v4f32:$FRB, SETUNE),
+          (QVFLOGICALb (QVFTSTNANbs $FRA, $FRB),
+                       (QVFCMPEQbs $FRA, $FRB), (i32 13))>;
+
+def : Pat<(setcc v4f32:$FRA, v4f32:$FRB, SETEQ),
+          (QVFCMPEQbs $FRA, $FRB)>;
+def : Pat<(setcc v4f32:$FRA, v4f32:$FRB, SETGT),
+          (QVFCMPGTbs $FRA, $FRB)>;
+def : Pat<(setcc v4f32:$FRA, v4f32:$FRB, SETGE),
+          (QVFLOGICALb (QVFCMPLTbs $FRA, $FRB),
+                       (QVFCMPLTbs $FRA, $FRB), (i32 10))>;
+def : Pat<(setcc v4f32:$FRA, v4f32:$FRB, SETLT),
+          (QVFCMPLTbs $FRA, $FRB)>;
+def : Pat<(setcc v4f32:$FRA, v4f32:$FRB, SETLE),
+          (QVFLOGICALb (QVFCMPGTbs $FRA, $FRB),
+                       (QVFCMPGTbs $FRA, $FRB), (i32 10))>;
+def : Pat<(setcc v4f32:$FRA, v4f32:$FRB, SETNE),
+          (QVFLOGICALb (QVFCMPEQbs $FRA, $FRB),
+                       (QVFCMPEQbs $FRA, $FRB), (i32 10))>;
+
+def : Pat<(and v4i1:$FRA, (not v4i1:$FRB)),
+          (QVFLOGICALb $FRA, $FRB, (i32 4))>;
+def : Pat<(not (or v4i1:$FRA, v4i1:$FRB)),
+          (QVFLOGICALb $FRA, $FRB, (i32 8))>;
+def : Pat<(not (xor v4i1:$FRA, v4i1:$FRB)),
+          (QVFLOGICALb $FRA, $FRB, (i32 9))>;
+def : Pat<(or v4i1:$FRA, (not v4i1:$FRB)),
+          (QVFLOGICALb $FRA, $FRB, (i32 13))>;
+def : Pat<(not (and v4i1:$FRA, v4i1:$FRB)),
+          (QVFLOGICALb $FRA, $FRB, (i32 14))>;
+
+def : Pat<(and v4i1:$FRA, v4i1:$FRB),
+          (QVFLOGICALb $FRA, $FRB, (i32 1))>;
+def : Pat<(or v4i1:$FRA, v4i1:$FRB),
+          (QVFLOGICALb $FRA, $FRB, (i32 7))>;
+def : Pat<(xor v4i1:$FRA, v4i1:$FRB),
+          (QVFLOGICALb $FRA, $FRB, (i32 6))>;
+def : Pat<(not v4i1:$FRA),
+          (QVFLOGICALb $FRA, $FRA, (i32 10))>;
+
+def : Pat<(v4f64 (fpextend v4f32:$src)),
+          (COPY_TO_REGCLASS $src, QFRC)>;
+
+def : Pat<(v4f32 (fround_exact v4f64:$src)),
+          (COPY_TO_REGCLASS $src, QSRC)>;
+
+// Extract the underlying floating-point values from the
+// QPX (-1.0, 1.0) boolean representation.
+def : Pat<(v4f64 (PPCqbflt v4i1:$src)),
+          (COPY_TO_REGCLASS $src, QFRC)>;
+
+def : Pat<(v4f64 (selectcc i1:$lhs, i1:$rhs, v4f64:$tval, v4f64:$fval, SETLT)),
+          (SELECT_QFRC (CRANDC $lhs, $rhs), $tval, $fval)>;
+def : Pat<(v4f64 (selectcc i1:$lhs, i1:$rhs, v4f64:$tval, v4f64:$fval, SETULT)),
+          (SELECT_QFRC (CRANDC $rhs, $lhs), $tval, $fval)>;
+def : Pat<(v4f64 (selectcc i1:$lhs, i1:$rhs, v4f64:$tval, v4f64:$fval, SETLE)),
+          (SELECT_QFRC (CRORC  $lhs, $rhs), $tval, $fval)>;
+def : Pat<(v4f64 (selectcc i1:$lhs, i1:$rhs, v4f64:$tval, v4f64:$fval, SETULE)),
+          (SELECT_QFRC (CRORC  $rhs, $lhs), $tval, $fval)>;
+def : Pat<(v4f64 (selectcc i1:$lhs, i1:$rhs, v4f64:$tval, v4f64:$fval, SETEQ)),
+          (SELECT_QFRC (CREQV $lhs, $rhs), $tval, $fval)>;
+def : Pat<(v4f64 (selectcc i1:$lhs, i1:$rhs, v4f64:$tval, v4f64:$fval, SETGE)),
+          (SELECT_QFRC (CRORC  $rhs, $lhs), $tval, $fval)>;
+def : Pat<(v4f64 (selectcc i1:$lhs, i1:$rhs, v4f64:$tval, v4f64:$fval, SETUGE)),
+          (SELECT_QFRC (CRORC  $lhs, $rhs), $tval, $fval)>;
+def : Pat<(v4f64 (selectcc i1:$lhs, i1:$rhs, v4f64:$tval, v4f64:$fval, SETGT)),
+          (SELECT_QFRC (CRANDC $rhs, $lhs), $tval, $fval)>;
+def : Pat<(v4f64 (selectcc i1:$lhs, i1:$rhs, v4f64:$tval, v4f64:$fval, SETUGT)),
+          (SELECT_QFRC (CRANDC $lhs, $rhs), $tval, $fval)>;
+def : Pat<(v4f64 (selectcc i1:$lhs, i1:$rhs, v4f64:$tval, v4f64:$fval, SETNE)),
+          (SELECT_QFRC (CRXOR $lhs, $rhs), $tval, $fval)>;
+
+def : Pat<(v4f32 (selectcc i1:$lhs, i1:$rhs, v4f32:$tval, v4f32:$fval, SETLT)),
+          (SELECT_QSRC (CRANDC $lhs, $rhs), $tval, $fval)>;
+def : Pat<(v4f32 (selectcc i1:$lhs, i1:$rhs, v4f32:$tval, v4f32:$fval, SETULT)),
+          (SELECT_QSRC (CRANDC $rhs, $lhs), $tval, $fval)>;
+def : Pat<(v4f32 (selectcc i1:$lhs, i1:$rhs, v4f32:$tval, v4f32:$fval, SETLE)),
+          (SELECT_QSRC (CRORC  $lhs, $rhs), $tval, $fval)>;
+def : Pat<(v4f32 (selectcc i1:$lhs, i1:$rhs, v4f32:$tval, v4f32:$fval, SETULE)),
+          (SELECT_QSRC (CRORC  $rhs, $lhs), $tval, $fval)>;
+def : Pat<(v4f32 (selectcc i1:$lhs, i1:$rhs, v4f32:$tval, v4f32:$fval, SETEQ)),
+          (SELECT_QSRC (CREQV $lhs, $rhs), $tval, $fval)>;
+def : Pat<(v4f32 (selectcc i1:$lhs, i1:$rhs, v4f32:$tval, v4f32:$fval, SETGE)),
+          (SELECT_QSRC (CRORC  $rhs, $lhs), $tval, $fval)>;
+def : Pat<(v4f32 (selectcc i1:$lhs, i1:$rhs, v4f32:$tval, v4f32:$fval, SETUGE)),
+          (SELECT_QSRC (CRORC  $lhs, $rhs), $tval, $fval)>;
+def : Pat<(v4f32 (selectcc i1:$lhs, i1:$rhs, v4f32:$tval, v4f32:$fval, SETGT)),
+          (SELECT_QSRC (CRANDC $rhs, $lhs), $tval, $fval)>;
+def : Pat<(v4f32 (selectcc i1:$lhs, i1:$rhs, v4f32:$tval, v4f32:$fval, SETUGT)),
+          (SELECT_QSRC (CRANDC $lhs, $rhs), $tval, $fval)>;
+def : Pat<(v4f32 (selectcc i1:$lhs, i1:$rhs, v4f32:$tval, v4f32:$fval, SETNE)),
+          (SELECT_QSRC (CRXOR $lhs, $rhs), $tval, $fval)>;
+
+def : Pat<(v4i1 (selectcc i1:$lhs, i1:$rhs, v4i1:$tval, v4i1:$fval, SETLT)),
+          (SELECT_QBRC (CRANDC $lhs, $rhs), $tval, $fval)>;
+def : Pat<(v4i1 (selectcc i1:$lhs, i1:$rhs, v4i1:$tval, v4i1:$fval, SETULT)),
+          (SELECT_QBRC (CRANDC $rhs, $lhs), $tval, $fval)>;
+def : Pat<(v4i1 (selectcc i1:$lhs, i1:$rhs, v4i1:$tval, v4i1:$fval, SETLE)),
+          (SELECT_QBRC (CRORC  $lhs, $rhs), $tval, $fval)>;
+def : Pat<(v4i1 (selectcc i1:$lhs, i1:$rhs, v4i1:$tval, v4i1:$fval, SETULE)),
+          (SELECT_QBRC (CRORC  $rhs, $lhs), $tval, $fval)>;
+def : Pat<(v4i1 (selectcc i1:$lhs, i1:$rhs, v4i1:$tval, v4i1:$fval, SETEQ)),
+          (SELECT_QBRC (CREQV $lhs, $rhs), $tval, $fval)>;
+def : Pat<(v4i1 (selectcc i1:$lhs, i1:$rhs, v4i1:$tval, v4i1:$fval, SETGE)),
+          (SELECT_QBRC (CRORC  $rhs, $lhs), $tval, $fval)>;
+def : Pat<(v4i1 (selectcc i1:$lhs, i1:$rhs, v4i1:$tval, v4i1:$fval, SETUGE)),
+          (SELECT_QBRC (CRORC  $lhs, $rhs), $tval, $fval)>;
+def : Pat<(v4i1 (selectcc i1:$lhs, i1:$rhs, v4i1:$tval, v4i1:$fval, SETGT)),
+          (SELECT_QBRC (CRANDC $rhs, $lhs), $tval, $fval)>;
+def : Pat<(v4i1 (selectcc i1:$lhs, i1:$rhs, v4i1:$tval, v4i1:$fval, SETUGT)),
+          (SELECT_QBRC (CRANDC $lhs, $rhs), $tval, $fval)>;
+def : Pat<(v4i1 (selectcc i1:$lhs, i1:$rhs, v4i1:$tval, v4i1:$fval, SETNE)),
+          (SELECT_QBRC (CRXOR $lhs, $rhs), $tval, $fval)>;
+
+} // end HasQPX
+
+let Predicates = [HasQPX, NoNaNsFPMath] in {
+def : Pat<(fminnum v4f64:$FRA, v4f64:$FRB),
+          (QVFSELb (QVFCMPLTb $FRA, $FRB), $FRB, $FRA)>;
+def : Pat<(fmaxnum v4f64:$FRA, v4f64:$FRB),
+          (QVFSELb (QVFCMPGTb $FRA, $FRB), $FRB, $FRA)>;
+
+def : Pat<(fminnum v4f32:$FRA, v4f32:$FRB),
+          (QVFSELbs (QVFCMPLTbs $FRA, $FRB), $FRB, $FRA)>;
+def : Pat<(fmaxnum v4f32:$FRA, v4f32:$FRB),
+          (QVFSELbs (QVFCMPGTbs $FRA, $FRB), $FRB, $FRA)>;
+}
+
+let Predicates = [HasQPX, NaNsFPMath] in {
+// When either of these operands is NaN, we should return the other operand.
+// QVFCMPLT/QVFCMPGT return false is either operand is NaN, which means we need
+// to explicitly or with a NaN test on the second operand.
+def : Pat<(fminnum v4f64:$FRA, v4f64:$FRB),
+          (QVFSELb (QVFLOGICALb (QVFCMPLTb $FRA, $FRB),
+                                (QVFTSTNANb $FRB, $FRB), (i32 7)),
+                   $FRB, $FRA)>;
+def : Pat<(fmaxnum v4f64:$FRA, v4f64:$FRB),
+          (QVFSELb (QVFLOGICALb (QVFCMPGTb $FRA, $FRB),
+                                (QVFTSTNANb $FRB, $FRB), (i32 7)),
+                   $FRB, $FRA)>;
+
+def : Pat<(fminnum v4f32:$FRA, v4f32:$FRB),
+          (QVFSELbs (QVFLOGICALb (QVFCMPLTbs $FRA, $FRB),
+                                 (QVFTSTNANbs $FRB, $FRB), (i32 7)),
+                   $FRB, $FRA)>;
+def : Pat<(fmaxnum v4f32:$FRA, v4f32:$FRB),
+          (QVFSELbs (QVFLOGICALb (QVFCMPGTbs $FRA, $FRB),
+                                 (QVFTSTNANbs $FRB, $FRB), (i32 7)),
+                   $FRB, $FRA)>;
+}
diff --git a/llvm/lib/Target/PowerPC/PPCQPXLoadSplat.cpp b/llvm/lib/Target/PowerPC/PPCQPXLoadSplat.cpp
new file mode 100644
index 0000000000000..6e90426438208
--- /dev/null
+++ b/llvm/lib/Target/PowerPC/PPCQPXLoadSplat.cpp
@@ -0,0 +1,161 @@
+//===----- PPCQPXLoadSplat.cpp - QPX Load Splat Simplification ------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// The QPX vector registers overlay the scalar floating-point registers, and
+// any scalar floating-point loads splat their value across all vector lanes.
+// Thus, if we have a scalar load followed by a splat, we can remove the splat
+// (i.e. replace the load with a load-and-splat pseudo instruction).
+//
+// This pass must run after anything that might do store-to-load forwarding.
+//
+//===----------------------------------------------------------------------===//
+
+#include "PPC.h"
+#include "PPCInstrBuilder.h"
+#include "PPCInstrInfo.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/TargetSubtargetInfo.h"
+#include "llvm/Support/MathExtras.h"
+#include "llvm/Target/TargetMachine.h"
+using namespace llvm;
+
+#define DEBUG_TYPE "ppc-qpx-load-splat"
+
+STATISTIC(NumSimplified, "Number of QPX load splats simplified");
+
+namespace {
+  struct PPCQPXLoadSplat : public MachineFunctionPass {
+    static char ID;
+    PPCQPXLoadSplat() : MachineFunctionPass(ID) {
+      initializePPCQPXLoadSplatPass(*PassRegistry::getPassRegistry());
+    }
+
+    bool runOnMachineFunction(MachineFunction &Fn) override;
+
+    StringRef getPassName() const override {
+      return "PowerPC QPX Load Splat Simplification";
+    }
+  };
+  char PPCQPXLoadSplat::ID = 0;
+}
+
+INITIALIZE_PASS(PPCQPXLoadSplat, "ppc-qpx-load-splat",
+                "PowerPC QPX Load Splat Simplification",
+                false, false)
+
+FunctionPass *llvm::createPPCQPXLoadSplatPass() {
+  return new PPCQPXLoadSplat();
+}
+
+bool PPCQPXLoadSplat::runOnMachineFunction(MachineFunction &MF) {
+  if (skipFunction(MF.getFunction()))
+    return false;
+
+  bool MadeChange = false;
+  const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
+
+  for (auto MFI = MF.begin(), MFIE = MF.end(); MFI != MFIE; ++MFI) {
+    MachineBasicBlock *MBB = &*MFI;
+    SmallVector<MachineInstr *, 4> Splats;
+
+    for (auto MBBI = MBB->rbegin(); MBBI != MBB->rend(); ++MBBI) {
+      MachineInstr *MI = &*MBBI;
+
+      if (MI->hasUnmodeledSideEffects() || MI->isCall()) {
+        Splats.clear();
+        continue;
+      }
+
+      // We're looking for a sequence like this:
+      // %f0 = LFD 0, killed %x3, implicit-def %qf0; mem:LD8[%a](tbaa=!2)
+      // %qf1 = QVESPLATI killed %qf0, 0, implicit %rm
+
+      for (auto SI = Splats.begin(); SI != Splats.end();) {
+        MachineInstr *SMI = *SI;
+        Register SplatReg = SMI->getOperand(0).getReg();
+        Register SrcReg = SMI->getOperand(1).getReg();
+
+        if (MI->modifiesRegister(SrcReg, TRI)) {
+          switch (MI->getOpcode()) {
+          default:
+            SI = Splats.erase(SI);
+            continue;
+          case PPC::LFS:
+          case PPC::LFD:
+          case PPC::LFSU:
+          case PPC::LFDU:
+          case PPC::LFSUX:
+          case PPC::LFDUX:
+          case PPC::LFSX:
+          case PPC::LFDX:
+          case PPC::LFIWAX:
+          case PPC::LFIWZX:
+            if (SplatReg != SrcReg) {
+              // We need to change the load to define the scalar subregister of
+              // the QPX splat source register.
+              unsigned SubRegIndex =
+                TRI->getSubRegIndex(SrcReg, MI->getOperand(0).getReg());
+              Register SplatSubReg = TRI->getSubReg(SplatReg, SubRegIndex);
+
+              // Substitute both the explicit defined register, and also the
+              // implicit def of the containing QPX register.
+              MI->getOperand(0).setReg(SplatSubReg);
+              MI->substituteRegister(SrcReg, SplatReg, 0, *TRI);
+            }
+
+            SI = Splats.erase(SI);
+
+            // If SMI is directly after MI, then MBBI's base iterator is
+            // pointing at SMI.  Adjust MBBI around the call to erase SMI to
+            // avoid invalidating MBBI.
+            ++MBBI;
+            SMI->eraseFromParent();
+            --MBBI;
+
+            ++NumSimplified;
+            MadeChange = true;
+            continue;
+          }
+        }
+
+        // If this instruction defines the splat register, then we cannot move
+        // the previous definition above it. If it reads from the splat
+        // register, then it must already be alive from some previous
+        // definition, and if the splat register is different from the source
+        // register, then this definition must not be the load for which we're
+        // searching.
+        if (MI->modifiesRegister(SplatReg, TRI) ||
+            (SrcReg != SplatReg &&
+             MI->readsRegister(SplatReg, TRI))) {
+          SI = Splats.erase(SI);
+          continue;
+        }
+
+        ++SI;
+      }
+
+      if (MI->getOpcode() != PPC::QVESPLATI &&
+          MI->getOpcode() != PPC::QVESPLATIs &&
+          MI->getOpcode() != PPC::QVESPLATIb)
+        continue;
+      if (MI->getOperand(2).getImm() != 0)
+        continue;
+
+      // If there are other uses of the scalar value after this, replacing
+      // those uses might be non-trivial.
+      if (!MI->getOperand(1).isKill())
+        continue;
+
+      Splats.push_back(MI);
+    }
+  }
+
+  return MadeChange;
+}
diff --git a/llvm/lib/Target/PowerPC/PPCRegisterInfo.cpp b/llvm/lib/Target/PowerPC/PPCRegisterInfo.cpp
index 96666ad58dfe5..ed8948a639728 100644
--- a/llvm/lib/Target/PowerPC/PPCRegisterInfo.cpp
+++ b/llvm/lib/Target/PowerPC/PPCRegisterInfo.cpp
@@ -404,6 +404,9 @@ unsigned PPCRegisterInfo::getRegPressureLimit(const TargetRegisterClass *RC,
   }
   case PPC::F8RCRegClassID:
   case PPC::F4RCRegClassID:
+  case PPC::QFRCRegClassID:
+  case PPC::QSRCRegClassID:
+  case PPC::QBRCRegClassID:
   case PPC::VRRCRegClassID:
   case PPC::VFRCRegClassID:
   case PPC::VSLRCRegClassID:
diff --git a/llvm/lib/Target/PowerPC/PPCRegisterInfo.h b/llvm/lib/Target/PowerPC/PPCRegisterInfo.h
index a931967862c7b..61acd955e1cba 100644
--- a/llvm/lib/Target/PowerPC/PPCRegisterInfo.h
+++ b/llvm/lib/Target/PowerPC/PPCRegisterInfo.h
@@ -153,6 +153,7 @@ class PPCRegisterInfo : public PPCGenRegisterInfo {
     switch (RegName[0]) {
       case 'r':
       case 'f':
+      case 'q': // for QPX
       case 'v':
         if (RegName[1] == 's')
           return RegName + 2;
diff --git a/llvm/lib/Target/PowerPC/PPCRegisterInfo.td b/llvm/lib/Target/PowerPC/PPCRegisterInfo.td
index e07b960ae305b..b45757c1acc5e 100644
--- a/llvm/lib/Target/PowerPC/PPCRegisterInfo.td
+++ b/llvm/lib/Target/PowerPC/PPCRegisterInfo.td
@@ -54,6 +54,13 @@ class FPR<bits<5> num, string n> : PPCReg<n> {
   let HWEncoding{4-0} = num;
 }
 
+// QFPR - One of the 32 256-bit floating-point vector registers (used for QPX)
+class QFPR<FPR SubReg, string n> : PPCReg<n> {
+  let HWEncoding = SubReg.HWEncoding;
+  let SubRegs = [SubReg];
+  let SubRegIndices = [sub_64];
+}
+
 // VF - One of the 32 64-bit floating-point subregisters of the vector
 // registers (used by VSX).
 class VF<bits<5> num, string n> : PPCReg<n> {
@@ -125,6 +132,12 @@ foreach Index = 0-31 in {
                  DwarfRegNum<[!add(Index, 77), !add(Index, 77)]>;
 }
 
+// QPX Floating-point registers
+foreach Index = 0-31 in {
+  def QF#Index : QFPR<!cast<FPR>("F"#Index), "q"#Index>,
+                 DwarfRegNum<[!add(Index, 32), !add(Index, 32)]>;
+}
+
 // Vector registers
 foreach Index = 0-31 in {
   def V#Index : VR<!cast<VF>("VF"#Index), "v"#Index>,
@@ -330,6 +343,16 @@ def SPILLTOVSRRC : RegisterClass<"PPC", [i64, f64], 64, (add G8RC, (sub VSFRC,
 // Register class for single precision scalars in VSX registers
 def VSSRC : RegisterClass<"PPC", [f32], 32, (add VSFRC)>;
 
+// For QPX
+def QFRC : RegisterClass<"PPC", [v4f64], 256, (add (sequence "QF%u", 0, 13),
+                                                (sequence "QF%u", 31, 14))>;
+def QSRC : RegisterClass<"PPC", [v4f32], 128, (add QFRC)>;
+def QBRC : RegisterClass<"PPC", [v4i1], 256, (add QFRC)> {
+  // These are actually stored as floating-point values where a positive
+  // number is true and anything else (including NaN) is false.
+  let Size = 256;
+}
+
 def CRBITRC : RegisterClass<"PPC", [i1], 32,
   (add CR2LT, CR2GT, CR2EQ, CR2UN,
        CR3LT, CR3GT, CR3EQ, CR3UN,
diff --git a/llvm/lib/Target/PowerPC/PPCScheduleP9.td b/llvm/lib/Target/PowerPC/PPCScheduleP9.td
index 311d5cf165f63..0a1ae7e55b3c2 100644
--- a/llvm/lib/Target/PowerPC/PPCScheduleP9.td
+++ b/llvm/lib/Target/PowerPC/PPCScheduleP9.td
@@ -40,9 +40,12 @@ def P9Model : SchedMachineModel {
 
   let CompleteModel = 1;
 
-  // Do not support SPE (Signal Processing Engine), prefixed instructions on
-  // Power 9, PC relative mem ops, or instructions introduced in ISA 3.1.
-  let UnsupportedFeatures = [HasSPE, PrefixInstrs, PCRelativeMemops, IsISA3_1];
+  // Do not support QPX (Quad Processing eXtension), SPE (Signal Processing
+  // Engine), prefixed instructions on Power 9, PC relative mem ops, or
+  // instructions introduced in ISA 3.1.
+  let UnsupportedFeatures = [HasQPX, HasSPE, PrefixInstrs, PCRelativeMemops,
+                             IsISA3_1];
+
 }
 
 let SchedModel = P9Model in {
diff --git a/llvm/lib/Target/PowerPC/PPCSubtarget.cpp b/llvm/lib/Target/PowerPC/PPCSubtarget.cpp
index 85d2966654970..3836cc960394f 100644
--- a/llvm/lib/Target/PowerPC/PPCSubtarget.cpp
+++ b/llvm/lib/Target/PowerPC/PPCSubtarget.cpp
@@ -35,6 +35,10 @@ using namespace llvm;
 static cl::opt<bool> UseSubRegLiveness("ppc-track-subreg-liveness",
 cl::desc("Enable subregister liveness tracking for PPC"), cl::Hidden);
 
+static cl::opt<bool> QPXStackUnaligned("qpx-stack-unaligned",
+  cl::desc("Even when QPX is enabled the stack is not 32-byte aligned"),
+  cl::Hidden);
+
 static cl::opt<bool>
     EnableMachinePipeliner("ppc-enable-pipeliner",
                            cl::desc("Enable Machine Pipeliner for PPC"),
@@ -66,6 +70,7 @@ void PPCSubtarget::initializeEnvironment() {
   HasAltivec = false;
   HasSPE = false;
   HasFPU = false;
+  HasQPX = false;
   HasVSX = false;
   NeedsTwoConstNR = false;
   HasP8Vector = false;
@@ -104,6 +109,7 @@ void PPCSubtarget::initializeEnvironment() {
   HasInvariantFunctionDescriptors = false;
   HasPartwordAtomics = false;
   HasDirectMove = false;
+  IsQPXStackUnaligned = false;
   HasHTM = false;
   HasFloat128 = false;
   HasFusion = false;
@@ -152,7 +158,7 @@ void PPCSubtarget::initSubtargetFeatures(StringRef CPU, StringRef FS) {
 
   if (HasSPE && IsPPC64)
     report_fatal_error( "SPE is only supported for 32-bit targets.\n", false);
-  if (HasSPE && (HasAltivec || HasVSX || HasFPU))
+  if (HasSPE && (HasAltivec || HasQPX || HasVSX || HasFPU))
     report_fatal_error(
         "SPE and traditional floating point cannot both be enabled.\n", false);
 
@@ -160,6 +166,10 @@ void PPCSubtarget::initSubtargetFeatures(StringRef CPU, StringRef FS) {
   if (!HasSPE)
     HasFPU = true;
 
+  // QPX requires a 32-byte aligned stack. Note that we need to do this if
+  // we're compiling for a BG/Q system regardless of whether or not QPX
+  // is enabled because external functions will assume this alignment.
+  IsQPXStackUnaligned = QPXStackUnaligned;
   StackAlignment = getPlatformStackAlignment();
 
   // Determine endianness.
diff --git a/llvm/lib/Target/PowerPC/PPCSubtarget.h b/llvm/lib/Target/PowerPC/PPCSubtarget.h
index 8a4041518e3c2..ec329022c4572 100644
--- a/llvm/lib/Target/PowerPC/PPCSubtarget.h
+++ b/llvm/lib/Target/PowerPC/PPCSubtarget.h
@@ -97,6 +97,7 @@ class PPCSubtarget : public PPCGenSubtargetInfo {
   bool HasAltivec;
   bool HasFPU;
   bool HasSPE;
+  bool HasQPX;
   bool HasVSX;
   bool NeedsTwoConstNR;
   bool HasP8Vector;
@@ -149,6 +150,11 @@ class PPCSubtarget : public PPCGenSubtargetInfo {
 
   POPCNTDKind HasPOPCNTD;
 
+  /// When targeting QPX running a stock PPC64 Linux kernel where the stack
+  /// alignment has not been changed, we need to keep the 16-byte alignment
+  /// of the stack.
+  bool IsQPXStackUnaligned;
+
   const PPCTargetMachine &TM;
   PPCFrameLowering FrameLowering;
   PPCInstrInfo InstrInfo;
@@ -249,6 +255,7 @@ class PPCSubtarget : public PPCGenSubtargetInfo {
   bool hasAltivec() const { return HasAltivec; }
   bool hasSPE() const { return HasSPE; }
   bool hasFPU() const { return HasFPU; }
+  bool hasQPX() const { return HasQPX; }
   bool hasVSX() const { return HasVSX; }
   bool needsTwoConstNR() const { return NeedsTwoConstNR; }
   bool hasP8Vector() const { return HasP8Vector; }
@@ -284,7 +291,11 @@ class PPCSubtarget : public PPCGenSubtargetInfo {
   bool hasPartwordAtomics() const { return HasPartwordAtomics; }
   bool hasDirectMove() const { return HasDirectMove; }
 
+  bool isQPXStackUnaligned() const { return IsQPXStackUnaligned; }
   Align getPlatformStackAlignment() const {
+    if ((hasQPX() || isBGQ()) && !isQPXStackUnaligned())
+      return Align(32);
+
     return Align(16);
   }
 
@@ -314,6 +325,9 @@ class PPCSubtarget : public PPCGenSubtargetInfo {
 
   const Triple &getTargetTriple() const { return TargetTriple; }
 
+  /// isBGQ - True if this is a BG/Q platform.
+  bool isBGQ() const { return TargetTriple.getVendor() == Triple::BGQ; }
+
   bool isTargetELF() const { return TargetTriple.isOSBinFormatELF(); }
   bool isTargetMachO() const { return TargetTriple.isOSBinFormatMachO(); }
   bool isTargetLinux() const { return TargetTriple.isOSLinux(); }
diff --git a/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp b/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp
index 27de5b29cd341..f15f9c7f49429 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp
+++ b/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp
@@ -63,6 +63,10 @@ static cl::
 opt<bool> DisableVSXSwapRemoval("disable-ppc-vsx-swap-removal", cl::Hidden,
                                 cl::desc("Disable VSX Swap Removal for PPC"));
 
+static cl::
+opt<bool> DisableQPXLoadSplat("disable-ppc-qpx-load-splat", cl::Hidden,
+                              cl::desc("Disable QPX load splat simplification"));
+
 static cl::
 opt<bool> DisableMIPeephole("disable-ppc-peephole", cl::Hidden,
                             cl::desc("Disable machine peepholes for PPC"));
@@ -110,6 +114,7 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializePowerPCTarget() {
   initializePPCReduceCRLogicalsPass(PR);
   initializePPCBSelPass(PR);
   initializePPCBranchCoalescingPass(PR);
+  initializePPCQPXLoadSplatPass(PR);
   initializePPCBoolRetToIntPass(PR);
   initializePPCExpandISELPass(PR);
   initializePPCPreEmitPeepholePass(PR);
@@ -406,9 +411,14 @@ void PPCPassConfig::addIRPasses() {
 
   // Lower generic MASSV routines to PowerPC subtarget-specific entries.
   addPass(createPPCLowerMASSVEntriesPass());
-
-  // If explicitly requested, add explicit data prefetch intrinsics.
+  
+  // For the BG/Q (or if explicitly requested), add explicit data prefetch
+  // intrinsics.
+  bool UsePrefetching = TM->getTargetTriple().getVendor() == Triple::BGQ &&
+                        getOptLevel() != CodeGenOpt::None;
   if (EnablePrefetch.getNumOccurrences() > 0)
+    UsePrefetching = EnablePrefetch;
+  if (UsePrefetching)
     addPass(createLoopDataPrefetchPass());
 
   if (TM->getOptLevel() >= CodeGenOpt::Default && EnableGEPOpt) {
@@ -505,8 +515,15 @@ void PPCPassConfig::addPreRegAlloc() {
 }
 
 void PPCPassConfig::addPreSched2() {
-  if (getOptLevel() != CodeGenOpt::None)
+  if (getOptLevel() != CodeGenOpt::None) {
     addPass(&IfConverterID);
+
+    // This optimization must happen after anything that might do store-to-load
+    // forwarding. Here we're after RA (and, thus, when spills are inserted)
+    // but before post-RA scheduling.
+    if (!DisableQPXLoadSplat)
+      addPass(createPPCQPXLoadSplatPass());
+  }
 }
 
 void PPCPassConfig::addPreEmitPass() {
diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
index ee8842f4d8663..bbb4239d36da5 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
+++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
@@ -25,7 +25,8 @@ using namespace llvm;
 static cl::opt<bool> DisablePPCConstHoist("disable-ppc-constant-hoisting",
 cl::desc("disable constant hoisting on PPC"), cl::init(false), cl::Hidden);
 
-// This is currently only used for the data prefetch pass
+// This is currently only used for the data prefetch pass which is only enabled
+// for BG/Q by default.
 static cl::opt<unsigned>
 CacheLineSize("ppc-loop-prefetch-cache-line", cl::Hidden, cl::init(64),
               cl::desc("The loop prefetch cache line size"));
@@ -103,6 +104,55 @@ PPCTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
     Value *Ptr = IC.Builder.CreateBitCast(II.getArgOperand(1), OpPtrTy);
     return new StoreInst(II.getArgOperand(0), Ptr, false, Align(1));
   }
+  case Intrinsic::ppc_qpx_qvlfs:
+    // Turn PPC QPX qvlfs -> load if the pointer is known aligned.
+    if (getOrEnforceKnownAlignment(
+            II.getArgOperand(0), Align(16), IC.getDataLayout(), &II,
+            &IC.getAssumptionCache(), &IC.getDominatorTree()) >= 16) {
+      Type *VTy =
+          VectorType::get(IC.Builder.getFloatTy(),
+                          cast<VectorType>(II.getType())->getElementCount());
+      Value *Ptr = IC.Builder.CreateBitCast(II.getArgOperand(0),
+                                            PointerType::getUnqual(VTy));
+      Value *Load = IC.Builder.CreateLoad(VTy, Ptr);
+      return new FPExtInst(Load, II.getType());
+    }
+    break;
+  case Intrinsic::ppc_qpx_qvlfd:
+    // Turn PPC QPX qvlfd -> load if the pointer is known aligned.
+    if (getOrEnforceKnownAlignment(
+            II.getArgOperand(0), Align(32), IC.getDataLayout(), &II,
+            &IC.getAssumptionCache(), &IC.getDominatorTree()) >= 32) {
+      Value *Ptr = IC.Builder.CreateBitCast(
+          II.getArgOperand(0), PointerType::getUnqual(II.getType()));
+      return new LoadInst(II.getType(), Ptr, "", false, Align(32));
+    }
+    break;
+  case Intrinsic::ppc_qpx_qvstfs:
+    // Turn PPC QPX qvstfs -> store if the pointer is known aligned.
+    if (getOrEnforceKnownAlignment(
+            II.getArgOperand(1), Align(16), IC.getDataLayout(), &II,
+            &IC.getAssumptionCache(), &IC.getDominatorTree()) >= 16) {
+      Type *VTy = VectorType::get(
+          IC.Builder.getFloatTy(),
+          cast<VectorType>(II.getArgOperand(0)->getType())->getElementCount());
+      Value *TOp = IC.Builder.CreateFPTrunc(II.getArgOperand(0), VTy);
+      Type *OpPtrTy = PointerType::getUnqual(VTy);
+      Value *Ptr = IC.Builder.CreateBitCast(II.getArgOperand(1), OpPtrTy);
+      return new StoreInst(TOp, Ptr, false, Align(16));
+    }
+    break;
+  case Intrinsic::ppc_qpx_qvstfd:
+    // Turn PPC QPX qvstfd -> store if the pointer is known aligned.
+    if (getOrEnforceKnownAlignment(
+            II.getArgOperand(1), Align(32), IC.getDataLayout(), &II,
+            &IC.getAssumptionCache(), &IC.getDominatorTree()) >= 32) {
+      Type *OpPtrTy = PointerType::getUnqual(II.getArgOperand(0)->getType());
+      Value *Ptr = IC.Builder.CreateBitCast(II.getArgOperand(1), OpPtrTy);
+      return new StoreInst(II.getArgOperand(0), Ptr, false, Align(32));
+    }
+    break;
+
   case Intrinsic::ppc_altivec_vperm:
     // Turn vperm(V1,V2,mask) -> shuffle(V1,V2,mask) if mask is a constant.
     // Note that ppc_altivec_vperm has a big-endian bias, so when creating
@@ -686,7 +736,10 @@ bool PPCTTIImpl::useColdCCForColdCall(Function &F) {
 }
 
 bool PPCTTIImpl::enableAggressiveInterleaving(bool LoopHasReductions) {
-  // On the A2, always unroll aggressively.
+  // On the A2, always unroll aggressively. For QPX unaligned loads, we depend
+  // on combining the loads generated for consecutive accesses, and failure to
+  // do so is particularly expensive. This makes it much more likely (compared
+  // to only using concatenation unrolling).
   if (ST->getCPUDirective() == PPC::DIR_A2)
     return true;
 
@@ -746,6 +799,7 @@ const char* PPCTTIImpl::getRegisterClassName(unsigned ClassID) const {
 
 unsigned PPCTTIImpl::getRegisterBitWidth(bool Vector) const {
   if (Vector) {
+    if (ST->hasQPX()) return 256;
     if (ST->hasAltivec()) return 128;
     return 0;
   }
@@ -774,6 +828,8 @@ unsigned PPCTTIImpl::getCacheLineSize() const {
 }
 
 unsigned PPCTTIImpl::getPrefetchDistance() const {
+  // This seems like a reasonable default for the BG/Q (this pass is enabled, by
+  // default, only on the BG/Q).
   return 300;
 }
 
@@ -862,7 +918,7 @@ int PPCTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, Type *Tp, int Index,
   // Legalize the type.
   std::pair<int, MVT> LT = TLI->getTypeLegalizationCost(DL, Tp);
 
-  // PPC, for both Altivec/VSX, support cheap arbitrary permutations
+  // PPC, for both Altivec/VSX and QPX, support cheap arbitrary permutations
   // (at least in the sense that there need only be one non-loop-invariant
   // instruction). We need one such shuffle instruction for each actual
   // register (this is not true for arbitrary shuffles, but is true for the
@@ -918,6 +974,13 @@ int PPCTTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val, unsigned Index) {
 
     return Cost;
 
+  } else if (ST->hasQPX() && Val->getScalarType()->isFloatingPointTy()) {
+    // Floating point scalars are already located in index #0.
+    if (Index == 0)
+      return 0;
+
+    return Cost;
+
   } else if (Val->getScalarType()->isIntegerTy() && Index != -1U) {
     if (ST->hasP9Altivec()) {
       if (ISD == ISD::INSERT_VECTOR_ELT)
@@ -992,6 +1055,8 @@ int PPCTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
                         LT.second == MVT::v4i32 || LT.second == MVT::v4f32);
   bool IsVSXType = ST->hasVSX() &&
                    (LT.second == MVT::v2f64 || LT.second == MVT::v2i64);
+  bool IsQPXType = ST->hasQPX() &&
+                   (LT.second == MVT::v4f64 || LT.second == MVT::v4f32);
 
   // VSX has 32b/64b load instructions. Legalization can handle loading of
   // 32b/64b to VSR correctly and cheaply. But BaseT::getMemoryOpCost and
@@ -1014,7 +1079,8 @@ int PPCTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
   // for Altivec types using the VSX instructions, but that's more expensive
   // than using the permutation-based load sequence. On the P8, that's no
   // longer true.
-  if (Opcode == Instruction::Load && (!ST->hasP8Vector() && IsAltivecType) &&
+  if (Opcode == Instruction::Load &&
+      ((!ST->hasP8Vector() && IsAltivecType) || IsQPXType) &&
       *Alignment >= LT.second.getScalarType().getStoreSize())
     return Cost + LT.first; // Add the cost of the permutations.
 
@@ -1067,7 +1133,7 @@ int PPCTTIImpl::getInterleavedMemoryOpCost(
       getMemoryOpCost(Opcode, VecTy, MaybeAlign(Alignment), AddressSpace,
                       CostKind);
 
-  // PPC, for both Altivec/VSX, support cheap arbitrary permutations
+  // PPC, for both Altivec/VSX and QPX, support cheap arbitrary permutations
   // (at least in the sense that there need only be one non-loop-invariant
   // instruction). For each result vector, we need one shuffle per incoming
   // vector (except that the first shuffle can take two incoming vectors
diff --git a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
index aa06e8144f634..427abde4277d4 100644
--- a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
@@ -4751,14 +4751,15 @@ struct VarArgPowerPC64Helper : public VarArgHelper {
     // For PowerPC, we need to deal with alignment of stack arguments -
     // they are mostly aligned to 8 bytes, but vectors and i128 arrays
     // are aligned to 16 bytes, byvals can be aligned to 8 or 16 bytes,
-    // For that reason, we compute current offset from stack pointer (which is
-    // always properly aligned), and offset for the first vararg, then subtract
-    // them.
+    // and QPX vectors are aligned to 32 bytes.  For that reason, we
+    // compute current offset from stack pointer (which is always properly
+    // aligned), and offset for the first vararg, then subtract them.
     unsigned VAArgBase;
     Triple TargetTriple(F.getParent()->getTargetTriple());
     // Parameter save area starts at 48 bytes from frame pointer for ABIv1,
     // and 32 bytes for ABIv2.  This is usually determined by target
     // endianness, but in theory could be overridden by function attribute.
+    // For simplicity, we ignore it here (it'd only matter for QPX vectors).
     if (TargetTriple.getArch() == Triple::ppc64)
       VAArgBase = 48;
     else
diff --git a/llvm/test/Analysis/BasicAA/phi-spec-order.ll b/llvm/test/Analysis/BasicAA/phi-spec-order.ll
index e5d435c09ccc7..f8586f094c2ce 100644
--- a/llvm/test/Analysis/BasicAA/phi-spec-order.ll
+++ b/llvm/test/Analysis/BasicAA/phi-spec-order.ll
@@ -1,5 +1,5 @@
 target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
-target triple = "powerpc64le-unknown-linux"
+target triple = "powerpc64-bgq-linux"
 ; RUN: opt < %s -basic-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 
 @X = external global [16000 x double], align 32
diff --git a/llvm/test/Analysis/CostModel/PowerPC/unal-vec-ldst.ll b/llvm/test/Analysis/CostModel/PowerPC/unal-vec-ldst.ll
index e5fbf070cf32a..3b1bc3b3fdbc0 100644
--- a/llvm/test/Analysis/CostModel/PowerPC/unal-vec-ldst.ll
+++ b/llvm/test/Analysis/CostModel/PowerPC/unal-vec-ldst.ll
@@ -218,6 +218,42 @@ entry:
 ; CHECK: cost of 2 for instruction:   %r = load <4 x double>, <4 x double>* %p, align 8
 }
 
+define <4 x float> @test_l_qv4float(<4 x float>* %p) #1 {
+entry:
+  %r = load <4 x float>, <4 x float>* %p, align 4
+  ret <4 x float> %r
+
+; CHECK-LABEL: test_l_qv4float
+; CHECK: cost of 2 for instruction:   %r = load <4 x float>, <4 x float>* %p, align 4
+}
+
+define <8 x float> @test_l_qv8float(<8 x float>* %p) #1 {
+entry:
+  %r = load <8 x float>, <8 x float>* %p, align 4
+  ret <8 x float> %r
+
+; CHECK-LABEL: test_l_qv8float
+; CHECK: cost of 4 for instruction:   %r = load <8 x float>, <8 x float>* %p, align 4
+}
+
+define <4 x double> @test_l_qv4double(<4 x double>* %p) #1 {
+entry:
+  %r = load <4 x double>, <4 x double>* %p, align 8
+  ret <4 x double> %r
+
+; CHECK-LABEL: test_l_qv4double
+; CHECK: cost of 2 for instruction:   %r = load <4 x double>, <4 x double>* %p, align 8
+}
+
+define <8 x double> @test_l_qv8double(<8 x double>* %p) #1 {
+entry:
+  %r = load <8 x double>, <8 x double>* %p, align 8
+  ret <8 x double> %r
+
+; CHECK-LABEL: test_l_qv8double
+; CHECK: cost of 4 for instruction:   %r = load <8 x double>, <8 x double>* %p, align 8
+}
+
 define void @test_s_v16i8(<16 x i8>* %p, <16 x i8> %v) #0 {
 entry:
   store <16 x i8> %v, <16 x i8>* %p, align 1
@@ -326,6 +362,43 @@ entry:
 ; CHECK: cost of 2 for instruction:   store <4 x double> %v, <4 x double>* %p, align 8
 }
 
+define void @test_s_qv4float(<4 x float>* %p, <4 x float> %v) #1 {
+entry:
+  store <4 x float> %v, <4 x float>* %p, align 4
+  ret void
+
+; CHECK-LABEL: test_s_qv4float
+; CHECK: cost of 7 for instruction:   store <4 x float> %v, <4 x float>* %p, align 4
+}
+
+define void @test_s_qv8float(<8 x float>* %p, <8 x float> %v) #1 {
+entry:
+  store <8 x float> %v, <8 x float>* %p, align 4
+  ret void
+
+; CHECK-LABEL: test_s_qv8float
+; CHECK: cost of 15 for instruction:   store <8 x float> %v, <8 x float>* %p, align 4
+}
+
+define void @test_s_qv4double(<4 x double>* %p, <4 x double> %v) #1 {
+entry:
+  store <4 x double> %v, <4 x double>* %p, align 8
+  ret void
+
+; CHECK-LABEL: test_s_qv4double
+; CHECK: cost of 7 for instruction:   store <4 x double> %v, <4 x double>* %p, align 8
+}
+
+define void @test_s_qv8double(<8 x double>* %p, <8 x double> %v) #1 {
+entry:
+  store <8 x double> %v, <8 x double>* %p, align 8
+  ret void
+
+; CHECK-LABEL: test_s_qv8double
+; CHECK: cost of 15 for instruction:   store <8 x double> %v, <8 x double>* %p, align 8
+}
+
 attributes #0 = { nounwind "target-cpu"="pwr7" }
+attributes #1 = { nounwind "target-cpu"="a2q" }
 attributes #2 = { nounwind "target-cpu"="pwr8" }
 
diff --git a/llvm/test/CodeGen/PowerPC/2012-11-16-mischedcall.ll b/llvm/test/CodeGen/PowerPC/2012-11-16-mischedcall.ll
index d93f192b1274d..69f9cff5c525f 100644
--- a/llvm/test/CodeGen/PowerPC/2012-11-16-mischedcall.ll
+++ b/llvm/test/CodeGen/PowerPC/2012-11-16-mischedcall.ll
@@ -1,4 +1,4 @@
-; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux -enable-misched < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64-bgq-linux -enable-misched < %s | FileCheck %s
 ;
 ; PR14315: misched should not move the physreg copy of %t below the calls.
 
diff --git a/llvm/test/CodeGen/PowerPC/DisableHoistingDueToBlockHotnessProfileData.mir b/llvm/test/CodeGen/PowerPC/DisableHoistingDueToBlockHotnessProfileData.mir
index a0139879f8c91..738aa1df5dd9d 100644
--- a/llvm/test/CodeGen/PowerPC/DisableHoistingDueToBlockHotnessProfileData.mir
+++ b/llvm/test/CodeGen/PowerPC/DisableHoistingDueToBlockHotnessProfileData.mir
@@ -55,7 +55,7 @@
   ; Function Attrs: nounwind
   declare void @llvm.stackprotector(i8*, i8**) #1
 
-  attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector" "unsafe-fp-math"="false" "use-soft-float"="false" }
+  attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
   attributes #1 = { nounwind }
 
   !llvm.module.flags = !{!0, !1}
diff --git a/llvm/test/CodeGen/PowerPC/NoCRFieldRedefWhenSpillingCRBIT.mir b/llvm/test/CodeGen/PowerPC/NoCRFieldRedefWhenSpillingCRBIT.mir
index 01ce79995512a..bcd51d31c6cfd 100644
--- a/llvm/test/CodeGen/PowerPC/NoCRFieldRedefWhenSpillingCRBIT.mir
+++ b/llvm/test/CodeGen/PowerPC/NoCRFieldRedefWhenSpillingCRBIT.mir
@@ -30,7 +30,7 @@
   ; Function Attrs: nounwind
   declare void @llvm.stackprotector(i8*, i8**) #1
   
-  attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector" "unsafe-fp-math"="false" "use-soft-float"="false" }
+  attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
   attributes #1 = { nounwind }
   
   !llvm.ident = !{!0}
diff --git a/llvm/test/CodeGen/PowerPC/a2q-stackalign.ll b/llvm/test/CodeGen/PowerPC/a2q-stackalign.ll
new file mode 100644
index 0000000000000..17e3df6d58ccc
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/a2q-stackalign.ll
@@ -0,0 +1,23 @@
+; RUN: llc -verify-machineinstrs < %s -mtriple=ppc64-- -mcpu=a2 | FileCheck -check-prefix=CHECK-A2 %s
+; RUN: llc -verify-machineinstrs < %s -mtriple=ppc64-- -mcpu=a2q | FileCheck -check-prefix=CHECK-A2Q %s
+; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc64-bgq-linux -mcpu=a2 | FileCheck -check-prefix=CHECK-BGQ %s
+target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
+target triple = "powerpc64-unknown-linux-gnu"
+
+declare i32 @bar(i8* %a) nounwind;
+define i32 @foo() nounwind {
+  %p = alloca i8, i8 115
+  store i8 0, i8* %p
+  %r = call i32 @bar(i8* %p)
+  ret i32 %r
+}
+
+; Without QPX, the allocated stack frame is 240 bytes, but with QPX
+; (because we require 32-byte alignment), it is 256 bytes.
+; CHECK-A2: @foo
+; CHECK-A2: stdu 1, -240(1)
+; CHECK-A2Q: @foo
+; CHECK-A2Q: stdu 1, -256(1)
+; CHECK-BGQ: @foo
+; CHECK-BGQ: stdu 1, -256(1)
+
diff --git a/llvm/test/CodeGen/PowerPC/a2q.ll b/llvm/test/CodeGen/PowerPC/a2q.ll
new file mode 100644
index 0000000000000..84e2dfa991d78
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/a2q.ll
@@ -0,0 +1,10 @@
+; RUN: llc -verify-machineinstrs < %s -mtriple=ppc64-- -mcpu=a2q | FileCheck %s
+; RUN: llc -verify-machineinstrs < %s -mtriple=ppc64-- -mcpu=a2 -mattr=+qpx | FileCheck %s
+
+define void @foo() {
+entry:
+  ret void
+}
+
+; CHECK: @foo
+
diff --git a/llvm/test/CodeGen/PowerPC/aantidep-inline-asm-use.ll b/llvm/test/CodeGen/PowerPC/aantidep-inline-asm-use.ll
index d629148535aa7..1b0ea26f1fdea 100644
--- a/llvm/test/CodeGen/PowerPC/aantidep-inline-asm-use.ll
+++ b/llvm/test/CodeGen/PowerPC/aantidep-inline-asm-use.ll
@@ -298,7 +298,7 @@ _ZN10SubProcess12SafeSyscalls5fcntlEiil.exit:     ; preds = %_ZN10SubProcess12Sa
 ; Function Attrs: nounwind argmemonly
 declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i1) #1
 
-attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pwr8" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pwr8" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind argmemonly }
 attributes #2 = { nounwind }
 
diff --git a/llvm/test/CodeGen/PowerPC/asm-Zy.ll b/llvm/test/CodeGen/PowerPC/asm-Zy.ll
index c8b5e9f1aa1d1..78bb0f4c73eca 100644
--- a/llvm/test/CodeGen/PowerPC/asm-Zy.ll
+++ b/llvm/test/CodeGen/PowerPC/asm-Zy.ll
@@ -1,5 +1,6 @@
-; RUN: llc -verify-machineinstrs < %s -mcpu=a2 -mtriple=powerpc64-unknown-linux-gnu | FileCheck %s
+; RUN: llc -verify-machineinstrs < %s -mcpu=a2 | FileCheck %s
 target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
+target triple = "powerpc64-bgq-linux"
 
 define i32 @zytest(i32 %a) nounwind {
 entry:
diff --git a/llvm/test/CodeGen/PowerPC/asm-constraints.ll b/llvm/test/CodeGen/PowerPC/asm-constraints.ll
index da77d1a169792..a3e573d8935e9 100644
--- a/llvm/test/CodeGen/PowerPC/asm-constraints.ll
+++ b/llvm/test/CodeGen/PowerPC/asm-constraints.ll
@@ -65,7 +65,7 @@ entry:
 }
 
 
-attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 attributes #1 = { nounwind }
 
diff --git a/llvm/test/CodeGen/PowerPC/convert-rr-to-ri-instrs-R0-special-handling.mir b/llvm/test/CodeGen/PowerPC/convert-rr-to-ri-instrs-R0-special-handling.mir
index 904210ee13477..2081e6fd02f51 100644
--- a/llvm/test/CodeGen/PowerPC/convert-rr-to-ri-instrs-R0-special-handling.mir
+++ b/llvm/test/CodeGen/PowerPC/convert-rr-to-ri-instrs-R0-special-handling.mir
@@ -63,8 +63,8 @@
     ret i64 %2
   }
   
-  attributes #0 = { norecurse nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector" "unsafe-fp-math"="false" "use-soft-float"="false" }
-  attributes #1 = { norecurse nounwind readonly "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector" "unsafe-fp-math"="false" "use-soft-float"="false" }
+  attributes #0 = { norecurse nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+  attributes #1 = { norecurse nounwind readonly "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
   
   !llvm.module.flags = !{!0, !1}
   !llvm.ident = !{!2}
diff --git a/llvm/test/CodeGen/PowerPC/convert-rr-to-ri-instrs-out-of-range.mir b/llvm/test/CodeGen/PowerPC/convert-rr-to-ri-instrs-out-of-range.mir
index f46d4fc0a42a4..b52e0a4103add 100644
--- a/llvm/test/CodeGen/PowerPC/convert-rr-to-ri-instrs-out-of-range.mir
+++ b/llvm/test/CodeGen/PowerPC/convert-rr-to-ri-instrs-out-of-range.mir
@@ -187,7 +187,7 @@
     ret i64 %cond
   }
   
-  attributes #0 = { norecurse nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pwr9" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+power9-vector,+vsx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+  attributes #0 = { norecurse nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pwr9" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+power9-vector,+vsx,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
   
   !llvm.module.flags = !{!0, !1}
   !llvm.ident = !{!2}
diff --git a/llvm/test/CodeGen/PowerPC/convert-rr-to-ri-instrs.mir b/llvm/test/CodeGen/PowerPC/convert-rr-to-ri-instrs.mir
index ba950dc3d3ae9..4d2595e1abdcb 100644
--- a/llvm/test/CodeGen/PowerPC/convert-rr-to-ri-instrs.mir
+++ b/llvm/test/CodeGen/PowerPC/convert-rr-to-ri-instrs.mir
@@ -983,10 +983,10 @@
     ret i64 %xor
   }
   
-  attributes #0 = { norecurse nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pwr9" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+power9-vector,+vsx" "unsafe-fp-math"="false" "use-soft-float"="false" }
-  attributes #1 = { norecurse nounwind readonly "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pwr9" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+power9-vector,+vsx" "unsafe-fp-math"="false" "use-soft-float"="false" }
-  attributes #2 = { norecurse nounwind readonly "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pwr9" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+power9-vector,-vsx" "unsafe-fp-math"="false" "use-soft-float"="false" }
-  attributes #3 = { norecurse nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pwr9" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+power9-vector,+vsx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+  attributes #0 = { norecurse nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pwr9" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+power9-vector,+vsx,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+  attributes #1 = { norecurse nounwind readonly "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pwr9" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+power9-vector,+vsx,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+  attributes #2 = { norecurse nounwind readonly "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pwr9" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+power9-vector,-vsx,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+  attributes #3 = { norecurse nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pwr9" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+power9-vector,+vsx,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
   
   !llvm.module.flags = !{!0, !1}
   !llvm.ident = !{!2}
diff --git a/llvm/test/CodeGen/PowerPC/ctr-minmaxnum.ll b/llvm/test/CodeGen/PowerPC/ctr-minmaxnum.ll
index 75640d1d26072..ed3c9f07c1a85 100644
--- a/llvm/test/CodeGen/PowerPC/ctr-minmaxnum.ll
+++ b/llvm/test/CodeGen/PowerPC/ctr-minmaxnum.ll
@@ -1,4 +1,5 @@
 ; RUN: llc -mtriple=powerpc64-unknown-linux-gnu -verify-machineinstrs -mcpu=pwr7 < %s | FileCheck %s
+; RUN: llc -mtriple=powerpc64-unknown-linux-gnu -verify-machineinstrs -mcpu=a2q < %s | FileCheck %s --check-prefix=QPX
 
 declare float @fabsf(float)
 
@@ -63,6 +64,11 @@ loop_exit:
 ; CHECK-NOT: xsmindp
 ; CHECK: blr
 
+; QPX-LABEL: test1v:
+; QPX: mtctr
+; QPX-NOT: bl fminf
+; QPX: blr
+
 define void @test1a(float %f, float* %fp) {
 entry:
   br label %loop_body
@@ -133,6 +139,11 @@ loop_exit:
 ; CHECK-NOT: xsmaxdp
 ; CHECK: blr
 
+; QPX-LABEL: test2v:
+; QPX: mtctr
+; QPX-NOT: bl fmax
+; QPX: blr
+
 define void @test2a(float %f, float* %fp) {
 entry:
   br label %loop_body
diff --git a/llvm/test/CodeGen/PowerPC/ctrloop-shortLoops.ll b/llvm/test/CodeGen/PowerPC/ctrloop-shortLoops.ll
index 636c86b815c8c..44acfcdd6e66a 100644
--- a/llvm/test/CodeGen/PowerPC/ctrloop-shortLoops.ll
+++ b/llvm/test/CodeGen/PowerPC/ctrloop-shortLoops.ll
@@ -1,4 +1,5 @@
 ; RUN: llc < %s -mtriple=powerpc64le-unknown-linux-gnu -verify-machineinstrs -mcpu=pwr8 | FileCheck %s --check-prefixes=CHECK,CHECK-PWR8
+; RUN: llc < %s -mtriple=powerpc64le-unknown-linux-gnu -verify-machineinstrs -mcpu=a2q | FileCheck %s --check-prefixes=CHECK,CHECK-A2Q
 
 ; Verify that we do NOT generate the mtctr instruction for loop trip counts < 4
 ; The latency of the mtctr is only justified if there are more than 4 comparisons that are removed as a result.
@@ -85,8 +86,11 @@ for.body:                                         ; preds = %entry, %for.body
 }
 
 ; Function Attrs: norecurse nounwind
+; On core a2q, IssueWidth is 1. On core pwr8, IssueWidth is 8.
+; a2q should use mtctr, but pwr8 should not use mtctr.
 define signext i32 @testTripCount2NonSmallLoop() {
 ; CHECK-LABEL: testTripCount2NonSmallLoop:
+; CHECK-A2Q: mtctr
 ; CHECK-PWR8-NOT: mtctr
 ; CHECK: blr
 
@@ -117,9 +121,12 @@ for.end:                                          ; preds = %if.end
   ret i32 %conv
 }
 
+; On core a2q, IssueWidth is 1. On core pwr8, IssueWidth is 8.
+; a2q should use mtctr, but pwr8 should not use mtctr.
 define signext i32 @testTripCount5() {
 ; CHECK-LABEL: testTripCount5:
 ; CHECK-PWR8-NOT: mtctr
+; CHECK-A2Q: mtctr
  
 entry:
   %.prea = load i32, i32* @a, align 4
diff --git a/llvm/test/CodeGen/PowerPC/ec-input.ll b/llvm/test/CodeGen/PowerPC/ec-input.ll
index 425bc1985d419..9a1c121699a69 100644
--- a/llvm/test/CodeGen/PowerPC/ec-input.ll
+++ b/llvm/test/CodeGen/PowerPC/ec-input.ll
@@ -5,7 +5,7 @@
 ; that were both inputs to the inline asm and also early-clobber outputs).
 
 target datalayout = "E-m:e-i64:64-n32:64"
-target triple = "powerpc64le-unknown-linux"
+target triple = "powerpc64-bgq-linux"
 
 %struct._IO_FILE.119.8249.32639.195239.200117.211499.218003.221255.222881.224507.226133.240767.244019.245645.248897.260279.271661.281417.283043.302555.304181.325319.326945.344713 = type { i32, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, %struct._IO_marker.118.8248.32638.195238.200116.211498.218002.221254.222880.224506.226132.240766.244018.245644.248896.260278.271660.281416.283042.302554.304180.325318.326944.344712*, %struct._IO_FILE.119.8249.32639.195239.200117.211499.218003.221255.222881.224507.226133.240767.244019.245645.248897.260279.271661.281417.283043.302555.304181.325319.326945.344713*, i32, i32, i64, i16, i8, [1 x i8], i8*, i64, i8*, i8*, i8*, i8*, i64, i32, [20 x i8] }
 %struct._IO_marker.118.8248.32638.195238.200116.211498.218002.221254.222880.224506.226132.240766.244018.245644.248896.260278.271660.281416.283042.302554.304180.325318.326944.344712 = type { %struct._IO_marker.118.8248.32638.195238.200116.211498.218002.221254.222880.224506.226132.240766.244018.245644.248896.260278.271660.281416.283042.302554.304180.325318.326944.344712*, %struct._IO_FILE.119.8249.32639.195239.200117.211499.218003.221255.222881.224507.226133.240767.244019.245645.248897.260279.271661.281417.283043.302555.304181.325319.326945.344713*, i32 }
diff --git a/llvm/test/CodeGen/PowerPC/extra-toc-reg-deps.ll b/llvm/test/CodeGen/PowerPC/extra-toc-reg-deps.ll
index 023928bcb5896..e066b45d3ca4b 100644
--- a/llvm/test/CodeGen/PowerPC/extra-toc-reg-deps.ll
+++ b/llvm/test/CodeGen/PowerPC/extra-toc-reg-deps.ll
@@ -1,6 +1,6 @@
-; RUN: llc -verify-machineinstrs -mcpu=pwr8 < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs < %s | FileCheck %s
 target datalayout = "E-m:e-i64:64-n32:64"
-target triple = "powerpc64-unknown-linux"
+target triple = "powerpc64-bgq-linux"
 
 %"class.Foam::messageStream.6" = type <{ %"class.Foam::string.5", i32, i32, i32, [4 x i8] }>
 %"class.Foam::string.5" = type { %"class.std::basic_string.4" }
@@ -419,8 +419,8 @@ declare void @_ZN4Foam11regIOobjectD2Ev() #0
 
 declare void @_ZN4Foam6reduceIiNS_5sumOpIiEEEEvRKNS_4ListINS_8UPstream11commsStructEEERT_RKT0_ii() #0
 
-attributes #0 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { inlinehint "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="a2q" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { inlinehint "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="a2q" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.module.flags = !{!0}
 
diff --git a/llvm/test/CodeGen/PowerPC/fast-isel-icmp-split.ll b/llvm/test/CodeGen/PowerPC/fast-isel-icmp-split.ll
index b08b050f2c2fd..fdd0fc2767803 100644
--- a/llvm/test/CodeGen/PowerPC/fast-isel-icmp-split.ll
+++ b/llvm/test/CodeGen/PowerPC/fast-isel-icmp-split.ll
@@ -1,6 +1,6 @@
 ; RUN: llc -verify-machineinstrs -O0 -relocation-model=pic < %s | FileCheck %s
 target datalayout = "E-m:e-i64:64-n32:64"
-target triple = "powerpc64le-unknown-linux"
+target triple = "powerpc64-bgq-linux"
 
 %"class.std::__1::__tree_node.130.151" = type { %"class.std::__1::__tree_node_base.base.128.149", %"class.boost::serialization::extended_type_info.129.150"* }
 %"class.std::__1::__tree_node_base.base.128.149" = type <{ %"class.std::__1::__tree_end_node.127.148", %"class.std::__1::__tree_node_base.126.147"*, %"class.std::__1::__tree_node_base.126.147"*, i8 }>
diff --git a/llvm/test/CodeGen/PowerPC/fma-mutate-duplicate-vreg.ll b/llvm/test/CodeGen/PowerPC/fma-mutate-duplicate-vreg.ll
index a336fc796ca52..eef6e0ccac02b 100644
--- a/llvm/test/CodeGen/PowerPC/fma-mutate-duplicate-vreg.ll
+++ b/llvm/test/CodeGen/PowerPC/fma-mutate-duplicate-vreg.ll
@@ -33,4 +33,4 @@ define float @f(float %xf) #0 {
   ret float %25
 }
 
-attributes #0 = { norecurse nounwind readnone "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pwr8" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse nounwind readnone "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pwr8" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/PowerPC/fp2int2fp-ppcfp128.ll b/llvm/test/CodeGen/PowerPC/fp2int2fp-ppcfp128.ll
index 3b555cf898f57..2feb4556dfab7 100644
--- a/llvm/test/CodeGen/PowerPC/fp2int2fp-ppcfp128.ll
+++ b/llvm/test/CodeGen/PowerPC/fp2int2fp-ppcfp128.ll
@@ -1,5 +1,6 @@
-; RUN: llc -verify-machineinstrs -mcpu=a2 -mtriple=powerpc64-unknown-linux-gnu < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mcpu=a2 < %s | FileCheck %s
 target datalayout = "E-m:e-i64:64-n32:64"
+target triple = "powerpc64-bgq-linux"
 
 define linkonce_odr double @test1(ppc_fp128 %input) {
 entry:
diff --git a/llvm/test/CodeGen/PowerPC/glob-comp-aa-crash.ll b/llvm/test/CodeGen/PowerPC/glob-comp-aa-crash.ll
index 2aa5239f25eb8..54c3e11528b7b 100644
--- a/llvm/test/CodeGen/PowerPC/glob-comp-aa-crash.ll
+++ b/llvm/test/CodeGen/PowerPC/glob-comp-aa-crash.ll
@@ -1,6 +1,6 @@
-; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux -mcpu=a2 < %s
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64-bgq-linux -mcpu=a2 < %s
 target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
-target triple = "powerpc64le-unknown-linux"
+target triple = "powerpc64-bgq-linux"
 
 %"class.std::__1::__assoc_sub_state" = type { %"class.std::__1::__shared_count", %"class.std::__exception_ptr::exception_ptr", %"class.std::__1::mutex", %"class.std::__1::condition_variable", i32 }
 %"class.std::__1::__shared_count" = type { i32 (...)**, i64 }
diff --git a/llvm/test/CodeGen/PowerPC/ifcvt-forked-bug-2016-08-08.ll b/llvm/test/CodeGen/PowerPC/ifcvt-forked-bug-2016-08-08.ll
index a2d0eb599f91d..74bfa75e5e313 100644
--- a/llvm/test/CodeGen/PowerPC/ifcvt-forked-bug-2016-08-08.ll
+++ b/llvm/test/CodeGen/PowerPC/ifcvt-forked-bug-2016-08-08.ll
@@ -33,5 +33,5 @@ declare i8* @_ZN11__sanitizer21internal_start_threadEPFvPvES0_(void (i8*)*, i8*)
 
 declare hidden void @_ZN11__sanitizer16BackgroundThreadEPv(i8* nocapture readnone) #5
 
-attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #7 = { nobuiltin nounwind }
diff --git a/llvm/test/CodeGen/PowerPC/inlineasm-i64-reg.ll b/llvm/test/CodeGen/PowerPC/inlineasm-i64-reg.ll
index 6f1bc76d816ae..e4dfd6c58f0e8 100644
--- a/llvm/test/CodeGen/PowerPC/inlineasm-i64-reg.ll
+++ b/llvm/test/CodeGen/PowerPC/inlineasm-i64-reg.ll
@@ -1,6 +1,6 @@
-; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux -mcpu=a2 < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64-bgq-linux -mcpu=a2 < %s | FileCheck %s
 target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
-target triple = "powerpc64le-unknown-linux"
+target triple = "powerpc64-bgq-linux"
 
 %struct.BG_CoordinateMapping_t = type { [4 x i8] }
 
diff --git a/llvm/test/CodeGen/PowerPC/load-two-flts.ll b/llvm/test/CodeGen/PowerPC/load-two-flts.ll
index 19e21faf47232..1cfcff5e01601 100644
--- a/llvm/test/CodeGen/PowerPC/load-two-flts.ll
+++ b/llvm/test/CodeGen/PowerPC/load-two-flts.ll
@@ -1,5 +1,6 @@
-; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs < %s | FileCheck %s
 target datalayout = "E-m:e-i64:64-n32:64"
+target triple = "powerpc64-bgq-linux"
 
 define void @_Z4testSt7complexIfE(float %v0, float %v1, i64* %ref.tmp, float* %_M_value.realp.i.i, float* %_M_value.imagp.i.i) {
 entry:
diff --git a/llvm/test/CodeGen/PowerPC/loop-data-prefetch-inner.ll b/llvm/test/CodeGen/PowerPC/loop-data-prefetch-inner.ll
index 2cbb70bb14cb5..f4664788930d4 100644
--- a/llvm/test/CodeGen/PowerPC/loop-data-prefetch-inner.ll
+++ b/llvm/test/CodeGen/PowerPC/loop-data-prefetch-inner.ll
@@ -1,6 +1,6 @@
-; RUN: llc -enable-ppc-prefetching=true -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs < %s | FileCheck %s
 target datalayout = "E-m:e-i64:64-n32:64"
-target triple = "powerpc64le-unknown-linux"
+target triple = "powerpc64-bgq-linux"
 
 ; Function Attrs: nounwind
 define void @foo(double* %x, double* nocapture readonly %y) #0 {
diff --git a/llvm/test/CodeGen/PowerPC/loop-data-prefetch.ll b/llvm/test/CodeGen/PowerPC/loop-data-prefetch.ll
index defc52eec8e0d..f4821564c202b 100644
--- a/llvm/test/CodeGen/PowerPC/loop-data-prefetch.ll
+++ b/llvm/test/CodeGen/PowerPC/loop-data-prefetch.ll
@@ -1,6 +1,6 @@
-; RUN: llc -verify-machineinstrs -enable-ppc-prefetching=true -mcpu=a2 < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mcpu=a2 < %s | FileCheck %s
 target datalayout = "E-m:e-i64:64-n32:64"
-target triple = "powerpc64le-unknown-linux"
+target triple = "powerpc64-bgq-linux"
 
 ; Function Attrs: nounwind
 define void @foo(double* nocapture %a, double* nocapture readonly %b) #0 {
diff --git a/llvm/test/CodeGen/PowerPC/loop-prep-all.ll b/llvm/test/CodeGen/PowerPC/loop-prep-all.ll
index 7fdabcd4be210..a13192d3e6586 100644
--- a/llvm/test/CodeGen/PowerPC/loop-prep-all.ll
+++ b/llvm/test/CodeGen/PowerPC/loop-prep-all.ll
@@ -1,4 +1,5 @@
 ; RUN: llc -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64-bgq-linux < %s | FileCheck %s -check-prefix=CHECK -check-prefix=CHECK-BGQ
 target datalayout = "E-m:e-i64:64-n32:64"
 target triple = "powerpc64-unknown-linux-gnu"
 
@@ -20,6 +21,7 @@ for.body:                                         ; preds = %for.body, %entry
 
 ; CHECK-LABEL: @foo
 
+; CHECK-BGQ-DAG: dcbt 4, 5
 ; CHECK-DAG: lfdu [[REG1:[0-9]+]], 8({{[0-9]+}})
 ; CHECK-DAG: fadd [[REG2:[0-9]+]], [[REG1]], 0
 ; CHECK-DAG: stfdu [[REG2]], 8({{[0-9]+}})
@@ -32,13 +34,15 @@ for.cond.cleanup6:                                ; preds = %for.body7
 
 for.body7:                                        ; preds = %for.body, %for.body7
   %i3.017 = phi i32 [ %inc9, %for.body7 ], [ 0, %for.body ]
-  tail call void bitcast (void (...)* @bar to void ()*)() #0
+  tail call void bitcast (void (...)* @bar to void ()*)() #2
   %inc9 = add nuw nsw i32 %i3.017, 1
   %exitcond = icmp eq i32 %inc9, 1024
   br i1 %exitcond, label %for.cond.cleanup6, label %for.body7
 }
 
-declare void @bar(...) 
+declare void @bar(...) #1
 
-attributes #0 = { nounwind }
+attributes #0 = { nounwind "target-cpu"="a2q" }
+attributes #1 = { "target-cpu"="a2q" }
+attributes #2 = { nounwind }
 
diff --git a/llvm/test/CodeGen/PowerPC/lxv-aligned-stack-slots.ll b/llvm/test/CodeGen/PowerPC/lxv-aligned-stack-slots.ll
index aa618d2b732c7..93868007d0d36 100644
--- a/llvm/test/CodeGen/PowerPC/lxv-aligned-stack-slots.ll
+++ b/llvm/test/CodeGen/PowerPC/lxv-aligned-stack-slots.ll
@@ -41,6 +41,6 @@ define void @aligned_slot() #0 {
 ; Function Attrs: argmemonly nounwind
 declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i1) #1
 
-attributes #0 = { nounwind "target-cpu"="pwr9" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+power9-vector,+vsx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "target-cpu"="pwr9" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+power9-vector,+vsx,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { argmemonly nounwind }
 attributes #2 = { nounwind }
diff --git a/llvm/test/CodeGen/PowerPC/machine-combiner.ll b/llvm/test/CodeGen/PowerPC/machine-combiner.ll
index 16fc3ee3e5202..2e834b1fe788c 100644
--- a/llvm/test/CodeGen/PowerPC/machine-combiner.ll
+++ b/llvm/test/CodeGen/PowerPC/machine-combiner.ll
@@ -1,4 +1,5 @@
 ; RUN: llc -verify-machineinstrs -O3 -mcpu=pwr7 < %s | FileCheck  %s -check-prefix=CHECK -check-prefix=CHECK-PWR
+; RUN: llc -verify-machineinstrs -O3 -mcpu=a2q < %s | FileCheck  %s -check-prefix=CHECK -check-prefix=CHECK-QPX
 ; RUN: llc -verify-machineinstrs -O3 -mcpu=pwr9 < %s | FileCheck  %s -check-prefix=FIXPOINT
 target datalayout = "E-m:e-i64:64-n32:64"
 target triple = "powerpc64-unknown-linux-gnu"
@@ -92,6 +93,9 @@ define float @reassociate_adds5(float %x0, float %x1, float %x2, float %x3, floa
 define <4 x float> @vector_reassociate_adds1(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, <4 x float> %x3) {
 ; CHECK-LABEL: vector_reassociate_adds1:
 ; CHECK:       # %bb.0:
+; CHECK-QPX:       qvfadds [[REG0:[0-9]+]], 1, 2
+; CHECK-QPX:       qvfadds [[REG1:[0-9]+]], 3, 4
+; CHECK-QPX:       qvfadds 1, [[REG0]], [[REG1]]
 ; CHECK-PWR:       xvaddsp [[REG0:[0-9]+]], 34, 35
 ; CHECK-PWR:       xvaddsp [[REG1:[0-9]+]], 36, 37
 ; CHECK-PWR:       xvaddsp 34, [[REG0]], [[REG1]]
@@ -106,6 +110,9 @@ define <4 x float> @vector_reassociate_adds1(<4 x float> %x0, <4 x float> %x1, <
 define <4 x float> @vector_reassociate_adds2(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, <4 x float> %x3) {
 ; CHECK-LABEL: vector_reassociate_adds2:
 ; CHECK:       # %bb.0:
+; CHECK-QPX:       qvfadds [[REG0:[0-9]+]], 1, 2
+; CHECK-QPX:       qvfadds [[REG1:[0-9]+]], 3, 4
+; CHECK-QPX:       qvfadds 1, [[REG0]], [[REG1]]
 ; CHECK-PWR:       xvaddsp [[REG0:[0-9]+]], 34, 35
 ; CHECK-PWR:       xvaddsp [[REG1:[0-9]+]], 36, 37
 ; CHECK-PWR:       xvaddsp 34, [[REG0]], [[REG1]]
@@ -120,6 +127,9 @@ define <4 x float> @vector_reassociate_adds2(<4 x float> %x0, <4 x float> %x1, <
 define <4 x float> @vector_reassociate_adds3(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, <4 x float> %x3) {
 ; CHECK-LABEL: vector_reassociate_adds3:
 ; CHECK:       # %bb.0:
+; CHECK-QPX:       qvfadds [[REG0:[0-9]+]], 1, 2
+; CHECK-QPX:       qvfadds [[REG1:[0-9]+]], 3, 4
+; CHECK-QPX:       qvfadds 1, [[REG0]], [[REG1]]
 ; CHECK-PWR:       xvaddsp [[REG0:[0-9]+]], 34, 35
 ; CHECK-PWR:       xvaddsp [[REG1:[0-9]+]], 36, 37
 ; CHECK-PWR:       xvaddsp 34, [[REG0]], [[REG1]]
@@ -134,6 +144,9 @@ define <4 x float> @vector_reassociate_adds3(<4 x float> %x0, <4 x float> %x1, <
 define <4 x float> @vector_reassociate_adds4(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, <4 x float> %x3) {
 ; CHECK-LABEL: vector_reassociate_adds4:
 ; CHECK:       # %bb.0:
+; CHECK-QPX:       qvfadds [[REG0:[0-9]+]], 1, 2
+; CHECK-QPX:       qvfadds [[REG1:[0-9]+]], 3, 4
+; CHECK-QPX:       qvfadds 1, [[REG0]], [[REG1]]
 ; CHECK-PWR:       xvaddsp [[REG0:[0-9]+]], 34, 35
 ; CHECK-PWR:       xvaddsp [[REG1:[0-9]+]], 36, 37
 ; CHECK-PWR:       xvaddsp 34, [[REG0]], [[REG1]]
@@ -204,6 +217,9 @@ define i64 @reassociate_mulld(i64 %x0, i64 %x1, i64 %x2, i64 %x3) {
 define double @reassociate_mamaa_double(double %0, double %1, double %2, double %3, double %4, double %5) {
 ; CHECK-LABEL: reassociate_mamaa_double:
 ; CHECK:       # %bb.0:
+; CHECK-QPX-DAG:   fmadd [[REG0:[0-9]+]], 4, 3, 2
+; CHECK-QPX-DAG:   fmadd [[REG1:[0-9]+]], 6, 5, 1
+; CHECK-QPX:       fadd 1, [[REG0]], [[REG1]]
 ; CHECK-PWR-DAG:   xsmaddadp 1, 6, 5
 ; CHECK-PWR-DAG:   xsmaddadp 2, 4, 3
 ; CHECK-PWR:       xsadddp 1, 2, 1
@@ -234,6 +250,9 @@ define float @reassociate_mamaa_float(float %0, float %1, float %2, float %3, fl
 define <4 x float> @reassociate_mamaa_vec(<4 x float> %0, <4 x float> %1, <4 x float> %2, <4 x float> %3, <4 x float> %4, <4 x float> %5) {
 ; CHECK-LABEL: reassociate_mamaa_vec:
 ; CHECK:       # %bb.0:
+; CHECK-QPX-DAG:   qvfmadds [[REG0:[0-9]+]], 4, 3, 2
+; CHECK-QPX-DAG:   qvfmadds [[REG1:[0-9]+]], 6, 5, 1
+; CHECK-QPX:       qvfadds 1, [[REG0]], [[REG1]]
 ; CHECK-PWR-DAG:   xvmaddasp [[REG0:[0-9]+]], 39, 38
 ; CHECK-PWR-DAG:   xvmaddasp [[REG1:[0-9]+]], 37, 36
 ; CHECK-PWR:       xvaddsp 34, [[REG1]], [[REG0]]
@@ -249,6 +268,11 @@ define <4 x float> @reassociate_mamaa_vec(<4 x float> %0, <4 x float> %1, <4 x f
 define double @reassociate_mamama_double(double %0, double %1, double %2, double %3, double %4, double %5, double %6, double %7, double %8) {
 ; CHECK-LABEL: reassociate_mamama_double:
 ; CHECK:       # %bb.0:
+; CHECK-QPX:       fmadd [[REG0:[0-9]+]], 2, 1, 7
+; CHECK-QPX-DAG:   fmul [[REG1:[0-9]+]], 4, 3
+; CHECK-QPX-DAG:   fmadd [[REG2:[0-9]+]], 6, 5, [[REG0]]
+; CHECK-QPX-DAG:   fmadd [[REG3:[0-9]+]], 9, 8, [[REG1]]
+; CHECK-QPX:       fadd 1, [[REG2]], [[REG3]]
 ; CHECK-PWR:       xsmaddadp 7, 2, 1
 ; CHECK-PWR-DAG:   xsmuldp [[REG0:[0-9]+]], 4, 3
 ; CHECK-PWR-DAG:   xsmaddadp 7, 6, 5
diff --git a/llvm/test/CodeGen/PowerPC/mc-instrlat.ll b/llvm/test/CodeGen/PowerPC/mc-instrlat.ll
index f807f4fa20d25..e135986a2894c 100644
--- a/llvm/test/CodeGen/PowerPC/mc-instrlat.ll
+++ b/llvm/test/CodeGen/PowerPC/mc-instrlat.ll
@@ -19,7 +19,7 @@ entry:
 
 declare void @bar(double) #1
 
-attributes #0 = { nounwind "no-infs-fp-math"="true" "no-nans-fp-math"="true" "target-cpu"="ppc64" "target-features"="+altivec,-bpermd,-crypto,-direct-move,-extdiv,-power8-vector,-vsx" "unsafe-fp-math"="true" "use-soft-float"="false" }
-attributes #1 = { "no-infs-fp-math"="true" "no-nans-fp-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="ppc64" "target-features"="+altivec,-bpermd,-crypto,-direct-move,-extdiv,-power8-vector,-vsx" "unsafe-fp-math"="true" "use-soft-float"="false" }
+attributes #0 = { nounwind "no-infs-fp-math"="true" "no-nans-fp-math"="true" "target-cpu"="ppc64" "target-features"="+altivec,-bpermd,-crypto,-direct-move,-extdiv,-power8-vector,-qpx,-vsx" "unsafe-fp-math"="true" "use-soft-float"="false" }
+attributes #1 = { "no-infs-fp-math"="true" "no-nans-fp-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="ppc64" "target-features"="+altivec,-bpermd,-crypto,-direct-move,-extdiv,-power8-vector,-qpx,-vsx" "unsafe-fp-math"="true" "use-soft-float"="false" }
 attributes #2 = { nounwind }
 
diff --git a/llvm/test/CodeGen/PowerPC/mcount-insertion.ll b/llvm/test/CodeGen/PowerPC/mcount-insertion.ll
index 502347a3af198..cbb7947be2198 100644
--- a/llvm/test/CodeGen/PowerPC/mcount-insertion.ll
+++ b/llvm/test/CodeGen/PowerPC/mcount-insertion.ll
@@ -1,8 +1,9 @@
-; RUN: opt -ee-instrument < %s | opt -inline | llc -mtriple=powerpc64-unknown-linux-gnu | FileCheck %s
+; RUN: opt -ee-instrument < %s | opt -inline | llc | FileCheck %s
 
 ; The run-line mimics how Clang might run the instrumentation passes.
 
 target datalayout = "E-m:e-i64:64-n32:64"
+target triple = "powerpc64-bgq-linux"
 
 
 define void @leaf_function() #0 {
diff --git a/llvm/test/CodeGen/PowerPC/memcpy-vec.ll b/llvm/test/CodeGen/PowerPC/memcpy-vec.ll
index c4e60f8c4b1f5..cd0abd6149bde 100644
--- a/llvm/test/CodeGen/PowerPC/memcpy-vec.ll
+++ b/llvm/test/CodeGen/PowerPC/memcpy-vec.ll
@@ -1,5 +1,6 @@
 ; RUN: llc -verify-machineinstrs -mcpu=pwr7 < %s | FileCheck  %s -check-prefix=PWR7
 ; RUN: llc -verify-machineinstrs -mcpu=pwr8 < %s | FileCheck  %s -check-prefix=PWR8
+; RUN: llc -verify-machineinstrs -mcpu=a2q < %s | FileCheck  %s -check-prefix=A2Q
 target datalayout = "E-m:e-i64:64-n32:64"
 target triple = "powerpc64-unknown-linux-gnu"
 
@@ -24,6 +25,12 @@ entry:
 ; PWR8: lxvw4x
 ; PWR8: stxvw4x
 ; PWR8: blr
+
+; A2Q-LABEL: @foo1
+; A2Q-NOT: bl memcpy
+; A2Q: ld {{[0-9]+}}, {{[0-9]+}}(4)
+; A2Q: std {{[0-9]+}}, {{[0-9]+}}(3)
+; A2Q: blr
 }
 
 ; Function Attrs: nounwind
@@ -45,6 +52,12 @@ entry:
 ; PWR8: lxvw4x
 ; PWR8: stxvw4x
 ; PWR8: blr
+
+; A2Q-LABEL: @foo2
+; A2Q-NOT: bl memcpy
+; A2Q: ld {{[0-9]+}}, {{[0-9]+}}(4)
+; A2Q: std {{[0-9]+}}, {{[0-9]+}}(3)
+; A2Q: blr
 }
 
 ; Function Attrs: nounwind
@@ -63,6 +76,11 @@ entry:
 ; PWR8-NOT: bl memset
 ; PWR8: stxvw4x
 ; PWR8: blr
+
+; A2Q-LABEL: @bar1
+; A2Q-NOT: bl memset
+; A2Q: std {{[0-9]+}}, {{[0-9]+}}(3)
+; A2Q: blr
 }
 
 ; Function Attrs: nounwind
@@ -81,6 +99,11 @@ entry:
 ; PWR8-NOT: bl memset
 ; PWR8: stxvw4x
 ; PWR8: blr
+
+; A2Q-LABEL: @bar2
+; A2Q-NOT: bl memset
+; A2Q: qvstfdx
+; A2Q: blr
 }
 
 ; Function Attrs: nounwind
diff --git a/llvm/test/CodeGen/PowerPC/memset-nc.ll b/llvm/test/CodeGen/PowerPC/memset-nc.ll
new file mode 100644
index 0000000000000..663d0cb1d6785
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/memset-nc.ll
@@ -0,0 +1,48 @@
+; RUN: llc -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -O0 < %s | FileCheck %s -check-prefix=CHECK-O0
+target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
+target triple = "powerpc64-bgq-linux"
+
+; Function Attrs: nounwind
+define void @test_qpx() unnamed_addr #0 align 2 {
+entry:
+  %0 = load i32, i32* undef, align 4
+  %1 = trunc i32 %0 to i8
+  call void @llvm.memset.p0i8.i64(i8* align 32 null, i8 %1, i64 64, i1 false)
+  ret void
+
+; CHECK-LABEL: @test_qpx
+; CHECK: qvstfdx
+; CHECK: qvstfdx
+; CHECK: blr
+
+; CHECK-O0-LABEL: @test_qpx
+; CHECK-O0-NOT: qvstfdx
+; CHECK-O0: blr
+}
+
+; Function Attrs: nounwind
+declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i1) #1
+
+; Function Attrs: nounwind
+define void @test_vsx() unnamed_addr #2 align 2 {
+entry:
+  %0 = load i32, i32* undef, align 4
+  %1 = trunc i32 %0 to i8
+  call void @llvm.memset.p0i8.i64(i8* null, i8 %1, i64 32, i1 false)
+  ret void
+
+; CHECK-LABEL: @test_vsx
+; CHECK: stxvw4x
+; CHECK: stxvw4x
+; CHECK: blr
+
+; CHECK-O0-LABEL: @test_vsx
+; CHECK-O0-NOT: stxvw4x
+; CHECK-O0: blr
+}
+
+attributes #0 = { nounwind "target-cpu"="a2q" }
+attributes #1 = { nounwind }
+attributes #2 = { nounwind "target-cpu"="pwr7" }
+
diff --git a/llvm/test/CodeGen/PowerPC/misched-inorder-latency.ll b/llvm/test/CodeGen/PowerPC/misched-inorder-latency.ll
index 089c947713b9d..26663d81f3575 100644
--- a/llvm/test/CodeGen/PowerPC/misched-inorder-latency.ll
+++ b/llvm/test/CodeGen/PowerPC/misched-inorder-latency.ll
@@ -1,7 +1,8 @@
 ; RUN: llc -verify-machineinstrs < %s -enable-misched -pre-RA-sched=source -scheditins=false \
-; RUN:          -disable-ifcvt-triangle-false -disable-post-ra -mtriple=powerpc64-unknown-linux-gnu | FileCheck %s
+; RUN:          -disable-ifcvt-triangle-false -disable-post-ra | FileCheck %s
 ;
 target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
+target triple = "powerpc64-bgq-linux"
 
 ; %val1 is a load live out of %entry. It should be hoisted
 ; above the add.
diff --git a/llvm/test/CodeGen/PowerPC/misched.ll b/llvm/test/CodeGen/PowerPC/misched.ll
index 9a75fe44b7176..1c868b3f171c9 100644
--- a/llvm/test/CodeGen/PowerPC/misched.ll
+++ b/llvm/test/CodeGen/PowerPC/misched.ll
@@ -1,6 +1,7 @@
 ; RUN: llc < %s -enable-misched -verify-machineinstrs
 ; PR14302
 target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
+target triple = "powerpc64-bgq-linux"
 
 @b = external global [16000 x double], align 32
 
diff --git a/llvm/test/CodeGen/PowerPC/optnone-crbits-i1-ret.ll b/llvm/test/CodeGen/PowerPC/optnone-crbits-i1-ret.ll
index ad5976318fe3a..f59df4291c48f 100644
--- a/llvm/test/CodeGen/PowerPC/optnone-crbits-i1-ret.ll
+++ b/llvm/test/CodeGen/PowerPC/optnone-crbits-i1-ret.ll
@@ -1,5 +1,6 @@
-; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs < %s | FileCheck %s
 target datalayout = "E-m:e-i64:64-n32:64"
+target triple = "powerpc64-bgq-linux"
 
 declare zeroext i1 @ri1()
 declare void @se1()
diff --git a/llvm/test/CodeGen/PowerPC/pcrel-local-caller-toc.ll b/llvm/test/CodeGen/PowerPC/pcrel-local-caller-toc.ll
index 2871e077df565..2e248506c7b7b 100644
--- a/llvm/test/CodeGen/PowerPC/pcrel-local-caller-toc.ll
+++ b/llvm/test/CodeGen/PowerPC/pcrel-local-caller-toc.ll
@@ -92,7 +92,7 @@ entry:
 
 ; Left the target features in this test because it is important that caller has
 ; -pcrelative-memops while callee has +pcrelative-memops
-attributes #0 = { nounwind "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+pcrelative-memops,+power8-vector,+power9-vector,+vsx,-htm,-spe" }
-attributes #1 = { "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+pcrelative-memops,+power8-vector,+power9-vector,+vsx,-htm,-spe" }
-attributes #2 = { nounwind "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+power9-vector,+vsx,-htm,-pcrelative-memops,-spe" }
+attributes #0 = { nounwind "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+pcrelative-memops,+power8-vector,+power9-vector,+vsx,-htm,-qpx,-spe" }
+attributes #1 = { "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+pcrelative-memops,+power8-vector,+power9-vector,+vsx,-htm,-qpx,-spe" }
+attributes #2 = { nounwind "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+power9-vector,+vsx,-htm,-pcrelative-memops,-qpx,-spe" }
 attributes #3 = { nounwind }
diff --git a/llvm/test/CodeGen/PowerPC/popcnt.ll b/llvm/test/CodeGen/PowerPC/popcnt.ll
index 695863d87f16e..a06c59d4b945a 100644
--- a/llvm/test/CodeGen/PowerPC/popcnt.ll
+++ b/llvm/test/CodeGen/PowerPC/popcnt.ll
@@ -1,6 +1,8 @@
 ; RUN: llc -verify-machineinstrs -mtriple=ppc64-- -mattr=+popcntd < %s | FileCheck %s
 ; RUN: llc -verify-machineinstrs -mtriple=ppc64-- -mattr=+slow-popcntd < %s | FileCheck %s --check-prefix=SLOWPC
 ; RUN: llc -verify-machineinstrs -mtriple=ppc64-- -mcpu=pwr7 < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mtriple=ppc64-- -mcpu=a2q < %s | FileCheck %s --check-prefix=SLOWPC
+; RUN: llc -verify-machineinstrs -mtriple=ppc64-- -mcpu=a2q -mattr=+popcntd < %s | FileCheck %s
 
 define i64 @_cntb64(i64 %x) nounwind readnone {
   %cnt = tail call i64 @llvm.ppc.popcntb(i64 %x)
diff --git a/llvm/test/CodeGen/PowerPC/ppc-passname.ll b/llvm/test/CodeGen/PowerPC/ppc-passname.ll
index 06f13278d84cd..98343bdb535c2 100644
--- a/llvm/test/CodeGen/PowerPC/ppc-passname.ll
+++ b/llvm/test/CodeGen/PowerPC/ppc-passname.ll
@@ -105,3 +105,14 @@
 ; STOP-AFTER-BRANCH-COALESCING-NOT: "ppc-branch-coalescing" pass is not registered.
 ; STOP-AFTER-BRANCH-COALESCING: Branch Coalescing 
 
+
+; Test pass name: ppc-qpx-load-splat.
+; RUN: llc -mtriple=powerpc64le-unknown-unknown < %s -debug-pass=Structure -stop-before=ppc-qpx-load-splat -o /dev/null 2>&1 | FileCheck %s -check-prefix=STOP-BEFORE-QPX-LOAD-SPLAT
+; STOP-BEFORE-QPX-LOAD-SPLAT-NOT: -ppc-qpx-load-splat
+; STOP-BEFORE-QPX-LOAD-SPLAT-NOT: "ppc-qpx-load-splat" pass is not registered.
+; STOP-BEFORE-QPX-LOAD-SPLAT-NOT: PowerPC QPX Load Splat Simplification
+
+; RUN: llc -mtriple=powerpc64le-unknown-unknown < %s -debug-pass=Structure -stop-after=ppc-qpx-load-splat -o /dev/null 2>&1 | FileCheck %s -check-prefix=STOP-AFTER-QPX-LOAD-SPLAT
+; STOP-AFTER-QPX-LOAD-SPLAT: -ppc-qpx-load-splat
+; STOP-AFTER-QPX-LOAD-SPLAT-NOT: "ppc-qpx-load-splat" pass is not registered.
+; STOP-AFTER-QPX-LOAD-SPLAT: PowerPC QPX Load Splat Simplification
diff --git a/llvm/test/CodeGen/PowerPC/ppc64-sibcall.ll b/llvm/test/CodeGen/PowerPC/ppc64-sibcall.ll
index 357f28e88b184..fc0e71f878cab 100644
--- a/llvm/test/CodeGen/PowerPC/ppc64-sibcall.ll
+++ b/llvm/test/CodeGen/PowerPC/ppc64-sibcall.ll
@@ -1,6 +1,6 @@
 ; RUN: llc < %s -relocation-model=static -O1 -disable-ppc-sco=false -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu | FileCheck %s -check-prefix=CHECK-SCO
-; RUN: llc < %s -relocation-model=static -O1 -disable-ppc-sco=false -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr8 | FileCheck %s -check-prefix=CHECK-SCO
-; RUN: llc < %s -relocation-model=static -O1 -disable-ppc-sco=false -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -mcpu=pwr8 | FileCheck %s -check-prefix=CHECK-SCO
+; RUN: llc < %s -relocation-model=static -O1 -disable-ppc-sco=false -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr8 | FileCheck %s -check-prefix=CHECK-SCO-HASQPX
+; RUN: llc < %s -relocation-model=static -O1 -disable-ppc-sco=false -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -mcpu=pwr8 | FileCheck %s -check-prefix=CHECK-SCO-HASQPX
 ; RUN: llc < %s -relocation-model=static -O1 -disable-ppc-sco=false -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -mcpu=pwr8 -code-model=small | FileCheck %s -check-prefix=SCM
 
 ; No combination of "powerpc64le-unknown-linux-gnu" + "CHECK-SCO", because
@@ -117,6 +117,23 @@ define void @caller_local_sret_32(%S_32* %a) #1 {
 attributes #0 = { noinline nounwind  }
 attributes #1 = { nounwind }
 
+; vector <4 x i1> test
+
+define void @callee_v4i1(i8 %a, <4 x i1> %b, <4 x i1> %c) { ret void }
+define void @caller_v4i1_reorder(i8 %a, <4 x i1> %b, <4 x i1> %c) {
+  tail call void @callee_v4i1(i8 %a, <4 x i1> %c, <4 x i1> %b)
+  ret void
+
+; <4 x i1> is 32 bytes aligned, if subtarget doesn't support qpx, then we can't
+; place b, c to qpx register, so we can't do sco on caller_v4i1_reorder
+
+; CHECK-SCO-LABEL: caller_v4i1_reorder:
+; CHECK-SCO: bl callee_v4i1
+
+; CHECK-SCO-HASQPX-LABEL: caller_v4i1_reorder:
+; CHECK-SCO-HASQPX: b callee_v4i1
+}
+
 define void @f128_callee(i32* %ptr, ppc_fp128 %a, ppc_fp128 %b) { ret void }
 define void @f128_caller(i32* %ptr, ppc_fp128 %a, ppc_fp128 %b) {
   tail call void @f128_callee(i32* %ptr, ppc_fp128 %a, ppc_fp128 %b)
diff --git a/llvm/test/CodeGen/PowerPC/pr24546.ll b/llvm/test/CodeGen/PowerPC/pr24546.ll
index 028fd2d8f0064..28c03293680e5 100644
--- a/llvm/test/CodeGen/PowerPC/pr24546.ll
+++ b/llvm/test/CodeGen/PowerPC/pr24546.ll
@@ -47,8 +47,8 @@ declare double @pow(double, double) #0
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.value(metadata, i64, metadata, metadata) #2
 
-attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind readnone }
 attributes #3 = { nounwind }
 
diff --git a/llvm/test/CodeGen/PowerPC/pr27350.ll b/llvm/test/CodeGen/PowerPC/pr27350.ll
index 93dbd10fecdeb..982023a1fcdc8 100644
--- a/llvm/test/CodeGen/PowerPC/pr27350.ll
+++ b/llvm/test/CodeGen/PowerPC/pr27350.ll
@@ -18,7 +18,7 @@ entry:
 declare fastcc void @bar([2 x i64], [2 x i64]) unnamed_addr #1 align 2
 
 attributes #0 = { argmemonly nounwind }
-attributes #1 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind }
 
 !llvm.ident = !{!0}
diff --git a/llvm/test/CodeGen/PowerPC/pr28130.ll b/llvm/test/CodeGen/PowerPC/pr28130.ll
index 4da415bd29269..cb703dfda8a59 100644
--- a/llvm/test/CodeGen/PowerPC/pr28130.ll
+++ b/llvm/test/CodeGen/PowerPC/pr28130.ll
@@ -67,4 +67,4 @@ bb:
   ret void
 }
 
-attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pwr8" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pwr8" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/PowerPC/preinc-ld-sel-crash.ll b/llvm/test/CodeGen/PowerPC/preinc-ld-sel-crash.ll
index 35aec57ec2640..04dee1ee182bb 100644
--- a/llvm/test/CodeGen/PowerPC/preinc-ld-sel-crash.ll
+++ b/llvm/test/CodeGen/PowerPC/preinc-ld-sel-crash.ll
@@ -1,6 +1,6 @@
 ; RUN: llc -verify-machineinstrs < %s | FileCheck %s
 target datalayout = "E-m:e-i64:64-n32:64"
-target triple = "powerpc64le-unknown-linux"
+target triple = "powerpc64-bgq-linux"
 
 %t1 = type { %t2*, %t3* }
 %t2 = type <{ %t3*, i32, [4 x i8] }>
diff --git a/llvm/test/CodeGen/PowerPC/qpx-bv-sint.ll b/llvm/test/CodeGen/PowerPC/qpx-bv-sint.ll
new file mode 100644
index 0000000000000..4e0aef4c3df71
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/qpx-bv-sint.ll
@@ -0,0 +1,33 @@
+; RUN: llc -verify-machineinstrs < %s -mcpu=a2q | FileCheck %s
+target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
+target triple = "powerpc64-bgq-linux"
+
+define void @s452(i32 %inp1) nounwind {
+entry:
+  br label %for.body4
+
+for.body4:                                        ; preds = %for.body4, %entry
+  %conv.4 = sitofp i32 %inp1 to double
+  %conv.5 = sitofp i32 %inp1 to double
+  %mul.4.v.i0.1 = insertelement <2 x double> undef, double %conv.4, i32 0
+  %v = insertelement <2 x double> %mul.4.v.i0.1, double %conv.5, i32 1
+  %vv = fmul <2 x double> %v, %v
+  %add7.4 = fadd <2 x double> %vv, %vv
+  store <2 x double> %add7.4, <2 x double>* undef, align 16
+  br i1 undef, label %for.end, label %for.body4
+
+for.end:                                          ; preds = %for.body4
+  unreachable
+; CHECK-LABEL: @s452
+; CHECK: lfiwax [[REG1:[0-9]+]],
+; CHECK: fcfid [[REG2:[0-9]+]], [[REG1]]
+; FIXME: We could 'promote' this to a vector earlier and remove this splat.
+; CHECK: qvesplati {{[0-9]+}}, [[REG2]], 0
+; CHECK: qvfmul
+; CHECK: qvfadd
+; CHECK: qvesplati {{[0-9]+}},
+; FIXME: We can use qvstfcdx here instead of two stores.
+; CHECK: stfd
+; CHECK: stfd
+}
+
diff --git a/llvm/test/CodeGen/PowerPC/qpx-bv.ll b/llvm/test/CodeGen/PowerPC/qpx-bv.ll
new file mode 100644
index 0000000000000..93a739b864c1d
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/qpx-bv.ll
@@ -0,0 +1,37 @@
+; RUN: llc -verify-machineinstrs < %s -mcpu=a2q | FileCheck %s
+
+target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
+target triple = "powerpc64-bgq-linux"
+
+define <4 x double> @foo(double %f1, double %f2, double %f3, double %f4) {
+  %v1 = insertelement <4 x double> undef, double %f1, i32 0
+  %v2 = insertelement <4 x double> %v1,   double %f2, i32 1
+  %v3 = insertelement <4 x double> %v2,   double %f3, i32 2
+  %v4 = insertelement <4 x double> %v3,   double %f4, i32 3
+  ret <4 x double> %v4
+
+; CHECK-LABEL: @foo
+; CHECK: qvgpci [[REG1:[0-9]+]], 275
+; CHECK-DAG: qvgpci [[REG2:[0-9]+]], 101
+; CHECK-DAG: qvfperm [[REG3:[0-9]+]], 3, 4, [[REG1]]
+; CHECK-DAG: qvfperm [[REG4:[0-9]+]], 1, 2, [[REG1]]
+; CHECK-DAG: qvfperm 1, [[REG4]], [[REG3]], [[REG2]]
+; CHECK: blr
+}
+
+define <4 x float> @goo(float %f1, float %f2, float %f3, float %f4) {
+  %v1 = insertelement <4 x float> undef, float %f1, i32 0
+  %v2 = insertelement <4 x float> %v1,   float %f2, i32 1
+  %v3 = insertelement <4 x float> %v2,   float %f3, i32 2
+  %v4 = insertelement <4 x float> %v3,   float %f4, i32 3
+  ret <4 x float> %v4
+
+; CHECK-LABEL: @goo
+; CHECK: qvgpci [[REG1:[0-9]+]], 275
+; CHECK-DAG: qvgpci [[REG2:[0-9]+]], 101
+; CHECK-DAG: qvfperm [[REG3:[0-9]+]], 3, 4, [[REG1]]
+; CHECK-DAG: qvfperm [[REG4:[0-9]+]], 1, 2, [[REG1]]
+; CHECK-DAG: qvfperm 1, [[REG4]], [[REG3]], [[REG2]]
+; CHECK: blr
+}
+
diff --git a/llvm/test/CodeGen/PowerPC/qpx-func-clobber.ll b/llvm/test/CodeGen/PowerPC/qpx-func-clobber.ll
new file mode 100644
index 0000000000000..ccbbd162a0cdb
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/qpx-func-clobber.ll
@@ -0,0 +1,22 @@
+; RUN: llc -verify-machineinstrs < %s -mcpu=a2q | FileCheck %s
+target triple = "powerpc64-bgq-linux"
+
+declare <4 x double> @foo(<4 x double> %p)
+
+define <4 x double> @bar(<4 x double> %p, <4 x double> %q) {
+entry:
+  %v = call <4 x double> @foo(<4 x double> %p)
+  %w = call <4 x double> @foo(<4 x double> %q)
+  %x = fadd <4 x double> %v, %w
+  ret <4 x double> %x
+
+; CHECK-LABEL: @bar
+; CHECK: qvstfdx 2,
+; CHECK: bl foo
+; CHECK: qvstfdx 1,
+; CHECK: qvlfdx 1,
+; CHECK: bl foo
+; CHECK: qvlfdx [[REG:[0-9]+]],
+; CHECK: qvfadd 1, [[REG]], 1
+}
+
diff --git a/llvm/test/CodeGen/PowerPC/qpx-load-splat.ll b/llvm/test/CodeGen/PowerPC/qpx-load-splat.ll
new file mode 100644
index 0000000000000..50b864980d985
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/qpx-load-splat.ll
@@ -0,0 +1,80 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=powerpc64le-unknown-linux-gnu -ppc-vsr-nums-as-vr \
+; RUN:   -ppc-asm-full-reg-names -verify-machineinstrs < %s | FileCheck %s
+
+; Function Attrs: norecurse nounwind readonly
+define <4 x double> @foo(double* nocapture readonly %a) #0 {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lxvdsx v2, 0, r3
+; CHECK-NEXT:    vmr v3, v2
+; CHECK-NEXT:    blr
+entry:
+  %0 = load double, double* %a, align 8
+  %vecinit.i = insertelement <4 x double> undef, double %0, i32 0
+  %shuffle.i = shufflevector <4 x double> %vecinit.i, <4 x double> undef, <4 x i32> zeroinitializer
+  ret <4 x double> %shuffle.i
+}
+
+define <4 x double> @foox(double* nocapture readonly %a, i64 %idx) #0 {
+; CHECK-LABEL: foox:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sldi r4, r4, 3
+; CHECK-NEXT:    lxvdsx v2, r3, r4
+; CHECK-NEXT:    vmr v3, v2
+; CHECK-NEXT:    blr
+entry:
+  %p = getelementptr double, double* %a, i64 %idx
+  %0 = load double, double* %p, align 8
+  %vecinit.i = insertelement <4 x double> undef, double %0, i32 0
+  %shuffle.i = shufflevector <4 x double> %vecinit.i, <4 x double> undef, <4 x i32> zeroinitializer
+  ret <4 x double> %shuffle.i
+}
+
+define <4 x double> @fooxu(double* nocapture readonly %a, i64 %idx, double** %pptr) #0 {
+; CHECK-LABEL: fooxu:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sldi r4, r4, 3
+; CHECK-NEXT:    add r6, r3, r4
+; CHECK-NEXT:    std r6, 0(r5)
+; CHECK-NEXT:    lxvdsx v2, r3, r4
+; CHECK-NEXT:    vmr v3, v2
+; CHECK-NEXT:    blr
+entry:
+  %p = getelementptr double, double* %a, i64 %idx
+  %0 = load double, double* %p, align 8
+  %vecinit.i = insertelement <4 x double> undef, double %0, i32 0
+  %shuffle.i = shufflevector <4 x double> %vecinit.i, <4 x double> undef, <4 x i32> zeroinitializer
+  store double* %p, double** %pptr, align 8
+  ret <4 x double> %shuffle.i
+}
+
+define <4 x float> @foof(float* nocapture readonly %a) #0 {
+; CHECK-LABEL: foof:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lfiwzx f0, 0, r3
+; CHECK-NEXT:    xxspltw v2, vs0, 1
+; CHECK-NEXT:    blr
+entry:
+  %0 = load float, float* %a, align 4
+  %vecinit.i = insertelement <4 x float> undef, float %0, i32 0
+  %shuffle.i = shufflevector <4 x float> %vecinit.i, <4 x float> undef, <4 x i32> zeroinitializer
+  ret <4 x float> %shuffle.i
+}
+
+define <4 x float> @foofx(float* nocapture readonly %a, i64 %idx) #0 {
+; CHECK-LABEL: foofx:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sldi r4, r4, 2
+; CHECK-NEXT:    lfiwzx f0, r3, r4
+; CHECK-NEXT:    xxspltw v2, vs0, 1
+; CHECK-NEXT:    blr
+entry:
+  %p = getelementptr float, float* %a, i64 %idx
+  %0 = load float, float* %p, align 4
+  %vecinit.i = insertelement <4 x float> undef, float %0, i32 0
+  %shuffle.i = shufflevector <4 x float> %vecinit.i, <4 x float> undef, <4 x i32> zeroinitializer
+  ret <4 x float> %shuffle.i
+}
+
+
diff --git a/llvm/test/CodeGen/PowerPC/qpx-load.ll b/llvm/test/CodeGen/PowerPC/qpx-load.ll
new file mode 100644
index 0000000000000..514f0934b6cfc
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/qpx-load.ll
@@ -0,0 +1,26 @@
+; RUN: llc -verify-machineinstrs < %s -mcpu=a2q | FileCheck %s
+target triple = "powerpc64-bgq-linux"
+
+define <4 x double> @foo(<4 x double>* %p) {
+entry:
+  %v = load <4 x double>, <4 x double>* %p, align 8
+  ret <4 x double> %v
+}
+
+; CHECK: @foo
+; CHECK-DAG: li [[REG1:[0-9]+]], 31
+; CHECK-DAG: qvlfdx [[REG4:[0-9]+]], 0, 3
+; CHECK-DAG: qvlfdx [[REG2:[0-9]+]], 3, [[REG1]]
+; CHECK-DAG: qvlpcldx [[REG3:[0-9]+]], 0, 3
+; CHECK-DAG: qvfperm 1, [[REG4]], [[REG2]], [[REG3]]
+; CHECK: blr
+
+define <4 x double> @bar(<4 x double>* %p) {
+entry:
+  %v = load <4 x double>, <4 x double>* %p, align 32
+  ret <4 x double> %v
+}
+
+; CHECK: @bar
+; CHECK: qvlfdx
+
diff --git a/llvm/test/CodeGen/PowerPC/qpx-qvfmadd.ll b/llvm/test/CodeGen/PowerPC/qpx-qvfmadd.ll
new file mode 100644
index 0000000000000..eab4d6af7e9fc
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/qpx-qvfmadd.ll
@@ -0,0 +1,79 @@
+; RUN: llc -verify-machineinstrs -stop-after=finalize-isel < %s -mcpu=a2q | FileCheck %s
+target triple = "powerpc64-bgq-linux"
+
+define <2 x double> @test_qvfmadd(<2 x double> %0, <2 x double> %1, <2 x double> %2) {
+; CHECK: test_qvfmadd
+; CHECK: QVFMADD %2, %1, %0, implicit $rm
+;
+  %4 = fmul reassoc nsz <2 x double> %2, %1
+  %5 = fadd reassoc nsz <2 x double> %4, %0
+  ret <2 x double> %5
+}
+
+define <4 x float> @test_qvfmadds(<4 x float> %0, <4 x float> %1, <4 x float> %2) {
+; CHECK: test_qvfmadds
+; CHECK: QVFMADDSs %2, %1, %0, implicit $rm
+;
+  %4 = fmul reassoc nsz <4 x float> %2, %1
+  %5 = fadd reassoc nsz <4 x float> %4, %0
+  ret <4 x float> %5
+}
+
+define <2 x double> @test_qvfnmadd(<2 x double> %0, <2 x double> %1, <2 x double> %2) {
+; CHECK: test_qvfnmadd
+; CHECK: QVFNMADD %2, %1, %0, implicit $rm
+;
+  %4 = fmul reassoc nsz <2 x double> %2, %1
+  %5 = fadd reassoc nsz <2 x double> %4, %0
+  %6 = fneg reassoc nsz <2 x double> %5
+  ret <2 x double> %6
+}
+
+define <4 x float> @test_qvfnmadds(<4 x float> %0, <4 x float> %1, <4 x float> %2) {
+; CHECK: test_qvfnmadds
+; CHECK: QVFNMADDSs %2, %1, %0, implicit $rm
+;
+  %4 = fmul reassoc nsz <4 x float> %2, %1
+  %5 = fadd reassoc nsz <4 x float> %4, %0
+  %6 = fneg reassoc nsz <4 x float> %5
+  ret <4 x float> %6
+}
+
+define <2 x double> @test_qvfmsub(<2 x double> %0, <2 x double> %1, <2 x double> %2) {
+; CHECK: test_qvfmsub
+; CHECK: QVFMSUB %2, %1, %0, implicit $rm
+;
+  %4 = fmul reassoc nsz <2 x double> %2, %1
+  %5 = fsub reassoc nsz <2 x double> %4, %0
+  ret <2 x double> %5
+}
+
+define <4 x float> @test_qvfmsubs(<4 x float> %0, <4 x float> %1, <4 x float> %2) {
+; CHECK: test_qvfmsubs
+; CHECK: QVFMSUBSs %2, %1, %0, implicit $rm
+;
+  %4 = fmul reassoc nsz <4 x float> %2, %1
+  %5 = fsub reassoc nsz <4 x float> %4, %0
+  ret <4 x float> %5
+}
+
+define <2 x double> @test_qvfnmsub(<2 x double> %0, <2 x double> %1, <2 x double> %2) {
+; CHECK: test_qvfnmsub
+; CHECK: QVFNMSUB %2, %1, %0, implicit $rm
+;
+  %4 = fmul reassoc nsz <2 x double> %2, %1
+  %5 = fsub reassoc nsz <2 x double> %4, %0
+  %6 = fneg reassoc nsz <2 x double> %5
+  ret <2 x double> %6
+}
+
+define <4 x float> @test_qvfnmsubs(<4 x float> %0, <4 x float> %1, <4 x float> %2) {
+; CHECK: test_qvfnmsubs
+; CHECK: QVFNMSUBSs %2, %1, %0, implicit $rm
+;
+  %4 = fmul reassoc nsz <4 x float> %2, %1
+  %5 = fsub reassoc nsz <4 x float> %4, %0
+  %6 = fneg reassoc nsz <4 x float> %5
+  ret <4 x float> %6
+}
+
diff --git a/llvm/test/CodeGen/PowerPC/qpx-recipest.ll b/llvm/test/CodeGen/PowerPC/qpx-recipest.ll
new file mode 100644
index 0000000000000..498ab62819ced
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/qpx-recipest.ll
@@ -0,0 +1,473 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc64-unknown-linux-gnu -mcpu=a2q | FileCheck %s
+target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
+target triple = "powerpc64-unknown-linux-gnu"
+
+declare <4 x double> @llvm.sqrt.v4f64(<4 x double>)
+declare <4 x float> @llvm.sqrt.v4f32(<4 x float>)
+
+define <4 x double> @foo_fmf(<4 x double> %a, <4 x double> %b) nounwind {
+; CHECK-LABEL: foo_fmf:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addis 3, 2, .LCPI0_0@toc@ha
+; CHECK-NEXT:    qvfrsqrte 3, 2
+; CHECK-NEXT:    addi 3, 3, .LCPI0_0@toc@l
+; CHECK-NEXT:    qvlfdx 0, 0, 3
+; CHECK-NEXT:    qvfmul 4, 3, 3
+; CHECK-NEXT:    qvfmsub 2, 2, 0, 2
+; CHECK-NEXT:    qvfnmsub 4, 2, 4, 0
+; CHECK-NEXT:    qvfmul 3, 3, 4
+; CHECK-NEXT:    qvfmul 4, 3, 3
+; CHECK-NEXT:    qvfnmsub 0, 2, 4, 0
+; CHECK-NEXT:    qvfmul 0, 3, 0
+; CHECK-NEXT:    qvfmul 1, 1, 0
+; CHECK-NEXT:    blr
+entry:
+  %x = call ninf afn reassoc <4 x double> @llvm.sqrt.v4f64(<4 x double> %b)
+  %r = fdiv arcp reassoc <4 x double> %a, %x
+  ret <4 x double> %r
+}
+
+define <4 x double> @foo_safe(<4 x double> %a, <4 x double> %b) nounwind {
+; CHECK-LABEL: foo_safe:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    qvesplati 5, 2, 3
+; CHECK-NEXT:    qvesplati 3, 2, 1
+; CHECK-NEXT:    qvesplati 4, 2, 2
+; CHECK-NEXT:    fsqrt 2, 2
+; CHECK-NEXT:    fsqrt 5, 5
+; CHECK-NEXT:    fsqrt 4, 4
+; CHECK-NEXT:    fsqrt 3, 3
+; CHECK-NEXT:    qvesplati 6, 1, 3
+; CHECK-NEXT:    qvgpci 0, 275
+; CHECK-NEXT:    fdiv 2, 1, 2
+; CHECK-NEXT:    fdiv 5, 6, 5
+; CHECK-NEXT:    qvesplati 6, 1, 2
+; CHECK-NEXT:    qvesplati 1, 1, 1
+; CHECK-NEXT:    fdiv 4, 6, 4
+; CHECK-NEXT:    fdiv 1, 1, 3
+; CHECK-NEXT:    qvfperm 3, 4, 5, 0
+; CHECK-NEXT:    qvfperm 0, 2, 1, 0
+; CHECK-NEXT:    qvgpci 1, 101
+; CHECK-NEXT:    qvfperm 1, 0, 3, 1
+; CHECK-NEXT:    blr
+entry:
+  %x = call <4 x double> @llvm.sqrt.v4f64(<4 x double> %b)
+  %r = fdiv <4 x double> %a, %x
+  ret <4 x double> %r
+}
+
+define <4 x double> @foof_fmf(<4 x double> %a, <4 x float> %b) nounwind {
+; CHECK-LABEL: foof_fmf:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addis 3, 2, .LCPI2_0@toc@ha
+; CHECK-NEXT:    qvfrsqrtes 3, 2
+; CHECK-NEXT:    addi 3, 3, .LCPI2_0@toc@l
+; CHECK-NEXT:    qvlfsx 0, 0, 3
+; CHECK-NEXT:    qvfmuls 4, 3, 3
+; CHECK-NEXT:    qvfmsubs 2, 2, 0, 2
+; CHECK-NEXT:    qvfnmsubs 0, 2, 4, 0
+; CHECK-NEXT:    qvfmuls 0, 3, 0
+; CHECK-NEXT:    qvfmul 1, 1, 0
+; CHECK-NEXT:    blr
+entry:
+  %x = call afn ninf reassoc <4 x float> @llvm.sqrt.v4f32(<4 x float> %b)
+  %y = fpext <4 x float> %x to <4 x double>
+  %r = fdiv arcp reassoc nsz <4 x double> %a, %y
+  ret <4 x double> %r
+}
+
+define <4 x double> @foof_safe(<4 x double> %a, <4 x float> %b) nounwind {
+; CHECK-LABEL: foof_safe:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    qvesplati 0, 2, 3
+; CHECK-NEXT:    qvesplati 3, 2, 2
+; CHECK-NEXT:    fsqrts 4, 2
+; CHECK-NEXT:    qvesplati 2, 2, 1
+; CHECK-NEXT:    fsqrts 0, 0
+; CHECK-NEXT:    fsqrts 3, 3
+; CHECK-NEXT:    fsqrts 2, 2
+; CHECK-NEXT:    qvgpci 5, 275
+; CHECK-NEXT:    qvgpci 6, 101
+; CHECK-NEXT:    qvfperm 0, 3, 0, 5
+; CHECK-NEXT:    qvesplati 3, 1, 2
+; CHECK-NEXT:    qvfperm 2, 4, 2, 5
+; CHECK-NEXT:    qvfperm 0, 2, 0, 6
+; CHECK-NEXT:    qvesplati 2, 1, 3
+; CHECK-NEXT:    qvesplati 4, 0, 3
+; CHECK-NEXT:    fdiv 2, 2, 4
+; CHECK-NEXT:    qvesplati 4, 0, 2
+; CHECK-NEXT:    fdiv 3, 3, 4
+; CHECK-NEXT:    qvesplati 4, 1, 1
+; CHECK-NEXT:    fdiv 1, 1, 0
+; CHECK-NEXT:    qvesplati 0, 0, 1
+; CHECK-NEXT:    fdiv 0, 4, 0
+; CHECK-NEXT:    qvfperm 2, 3, 2, 5
+; CHECK-NEXT:    qvfperm 0, 1, 0, 5
+; CHECK-NEXT:    qvfperm 1, 0, 2, 6
+; CHECK-NEXT:    blr
+entry:
+  %x = call <4 x float> @llvm.sqrt.v4f32(<4 x float> %b)
+  %y = fpext <4 x float> %x to <4 x double>
+  %r = fdiv <4 x double> %a, %y
+  ret <4 x double> %r
+}
+
+define <4 x float> @food_fmf(<4 x float> %a, <4 x double> %b) nounwind {
+; CHECK-LABEL: food_fmf:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addis 3, 2, .LCPI4_0@toc@ha
+; CHECK-NEXT:    qvfrsqrte 3, 2
+; CHECK-NEXT:    addi 3, 3, .LCPI4_0@toc@l
+; CHECK-NEXT:    qvlfdx 0, 0, 3
+; CHECK-NEXT:    qvfmul 4, 3, 3
+; CHECK-NEXT:    qvfmsub 2, 2, 0, 2
+; CHECK-NEXT:    qvfnmsub 4, 2, 4, 0
+; CHECK-NEXT:    qvfmul 3, 3, 4
+; CHECK-NEXT:    qvfmul 4, 3, 3
+; CHECK-NEXT:    qvfnmsub 0, 2, 4, 0
+; CHECK-NEXT:    qvfmul 0, 3, 0
+; CHECK-NEXT:    qvfrsp 0, 0
+; CHECK-NEXT:    qvfmuls 1, 1, 0
+; CHECK-NEXT:    blr
+entry:
+  %x = call afn ninf reassoc <4 x double> @llvm.sqrt.v4f64(<4 x double> %b)
+  %y = fptrunc <4 x double> %x to <4 x float>
+  %r = fdiv arcp reassoc <4 x float> %a, %y
+  ret <4 x float> %r
+}
+
+define <4 x float> @food_safe(<4 x float> %a, <4 x double> %b) nounwind {
+; CHECK-LABEL: food_safe:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    qvesplati 0, 2, 3
+; CHECK-NEXT:    qvesplati 3, 2, 2
+; CHECK-NEXT:    fsqrt 4, 2
+; CHECK-NEXT:    qvesplati 2, 2, 1
+; CHECK-NEXT:    fsqrt 0, 0
+; CHECK-NEXT:    fsqrt 3, 3
+; CHECK-NEXT:    fsqrt 2, 2
+; CHECK-NEXT:    qvgpci 5, 275
+; CHECK-NEXT:    qvgpci 6, 101
+; CHECK-NEXT:    qvfperm 0, 3, 0, 5
+; CHECK-NEXT:    qvesplati 3, 1, 2
+; CHECK-NEXT:    qvfperm 2, 4, 2, 5
+; CHECK-NEXT:    qvfperm 0, 2, 0, 6
+; CHECK-NEXT:    qvesplati 2, 1, 3
+; CHECK-NEXT:    qvfrsp 0, 0
+; CHECK-NEXT:    qvesplati 4, 0, 3
+; CHECK-NEXT:    fdivs 2, 2, 4
+; CHECK-NEXT:    qvesplati 4, 0, 2
+; CHECK-NEXT:    fdivs 3, 3, 4
+; CHECK-NEXT:    qvesplati 4, 1, 1
+; CHECK-NEXT:    fdivs 1, 1, 0
+; CHECK-NEXT:    qvesplati 0, 0, 1
+; CHECK-NEXT:    fdivs 0, 4, 0
+; CHECK-NEXT:    qvfperm 2, 3, 2, 5
+; CHECK-NEXT:    qvfperm 0, 1, 0, 5
+; CHECK-NEXT:    qvfperm 1, 0, 2, 6
+; CHECK-NEXT:    blr
+entry:
+  %x = call <4 x double> @llvm.sqrt.v4f64(<4 x double> %b)
+  %y = fptrunc <4 x double> %x to <4 x float>
+  %r = fdiv <4 x float> %a, %y
+  ret <4 x float> %r
+}
+
+define <4 x float> @goo_fmf(<4 x float> %a, <4 x float> %b) nounwind {
+; CHECK-LABEL: goo_fmf:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addis 3, 2, .LCPI6_0@toc@ha
+; CHECK-NEXT:    qvfrsqrtes 3, 2
+; CHECK-NEXT:    addi 3, 3, .LCPI6_0@toc@l
+; CHECK-NEXT:    qvlfsx 0, 0, 3
+; CHECK-NEXT:    qvfmuls 4, 3, 3
+; CHECK-NEXT:    qvfmsubs 2, 2, 0, 2
+; CHECK-NEXT:    qvfnmsubs 0, 2, 4, 0
+; CHECK-NEXT:    qvfmuls 0, 3, 0
+; CHECK-NEXT:    qvfmuls 1, 1, 0
+; CHECK-NEXT:    blr
+entry:
+  %x = call afn ninf reassoc <4 x float> @llvm.sqrt.v4f32(<4 x float> %b)
+  %r = fdiv arcp reassoc nsz <4 x float> %a, %x
+  ret <4 x float> %r
+}
+
+define <4 x float> @goo_safe(<4 x float> %a, <4 x float> %b) nounwind {
+; CHECK-LABEL: goo_safe:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    qvesplati 5, 2, 3
+; CHECK-NEXT:    qvesplati 3, 2, 1
+; CHECK-NEXT:    qvesplati 4, 2, 2
+; CHECK-NEXT:    fsqrts 2, 2
+; CHECK-NEXT:    fsqrts 5, 5
+; CHECK-NEXT:    fsqrts 4, 4
+; CHECK-NEXT:    fsqrts 3, 3
+; CHECK-NEXT:    qvesplati 6, 1, 3
+; CHECK-NEXT:    qvgpci 0, 275
+; CHECK-NEXT:    fdivs 2, 1, 2
+; CHECK-NEXT:    fdivs 5, 6, 5
+; CHECK-NEXT:    qvesplati 6, 1, 2
+; CHECK-NEXT:    qvesplati 1, 1, 1
+; CHECK-NEXT:    fdivs 4, 6, 4
+; CHECK-NEXT:    fdivs 1, 1, 3
+; CHECK-NEXT:    qvfperm 3, 4, 5, 0
+; CHECK-NEXT:    qvfperm 0, 2, 1, 0
+; CHECK-NEXT:    qvgpci 1, 101
+; CHECK-NEXT:    qvfperm 1, 0, 3, 1
+; CHECK-NEXT:    blr
+entry:
+  %x = call <4 x float> @llvm.sqrt.v4f32(<4 x float> %b)
+  %r = fdiv <4 x float> %a, %x
+  ret <4 x float> %r
+}
+
+define <4 x double> @foo2_fmf(<4 x double> %a, <4 x double> %b) nounwind {
+; CHECK-LABEL: foo2_fmf:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addis 3, 2, .LCPI8_0@toc@ha
+; CHECK-NEXT:    qvfre 3, 2
+; CHECK-NEXT:    addi 3, 3, .LCPI8_0@toc@l
+; CHECK-NEXT:    qvlfdx 0, 0, 3
+; CHECK-NEXT:    qvfmadd 0, 2, 3, 0
+; CHECK-NEXT:    qvfnmsub 0, 3, 0, 3
+; CHECK-NEXT:    qvfmul 3, 1, 0
+; CHECK-NEXT:    qvfnmsub 1, 2, 3, 1
+; CHECK-NEXT:    qvfmadd 1, 0, 1, 3
+; CHECK-NEXT:    blr
+entry:
+  %r = fdiv arcp reassoc nsz ninf <4 x double> %a, %b
+  ret <4 x double> %r
+}
+
+define <4 x double> @foo2_safe(<4 x double> %a, <4 x double> %b) nounwind {
+; CHECK-LABEL: foo2_safe:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    qvesplati 3, 2, 3
+; CHECK-NEXT:    qvesplati 4, 1, 3
+; CHECK-NEXT:    qvesplati 5, 2, 2
+; CHECK-NEXT:    qvgpci 0, 275
+; CHECK-NEXT:    fdiv 3, 4, 3
+; CHECK-NEXT:    qvesplati 4, 1, 2
+; CHECK-NEXT:    fdiv 4, 4, 5
+; CHECK-NEXT:    fdiv 5, 1, 2
+; CHECK-NEXT:    qvesplati 2, 2, 1
+; CHECK-NEXT:    qvesplati 1, 1, 1
+; CHECK-NEXT:    fdiv 1, 1, 2
+; CHECK-NEXT:    qvfperm 2, 4, 3, 0
+; CHECK-NEXT:    qvfperm 0, 5, 1, 0
+; CHECK-NEXT:    qvgpci 1, 101
+; CHECK-NEXT:    qvfperm 1, 0, 2, 1
+; CHECK-NEXT:    blr
+  %r = fdiv <4 x double> %a, %b
+  ret <4 x double> %r
+}
+
+define <4 x float> @goo2_fmf(<4 x float> %a, <4 x float> %b) nounwind {
+; CHECK-LABEL: goo2_fmf:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    qvfres 0, 2
+; CHECK-NEXT:    qvfmuls 3, 1, 0
+; CHECK-NEXT:    qvfnmsubs 1, 2, 3, 1
+; CHECK-NEXT:    qvfmadds 1, 0, 1, 3
+; CHECK-NEXT:    blr
+entry:
+  %r = fdiv arcp reassoc ninf <4 x float> %a, %b
+  ret <4 x float> %r
+}
+
+define <4 x float> @goo2_safe(<4 x float> %a, <4 x float> %b) nounwind {
+; CHECK-LABEL: goo2_safe:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    qvesplati 3, 2, 3
+; CHECK-NEXT:    qvesplati 4, 1, 3
+; CHECK-NEXT:    qvesplati 5, 2, 2
+; CHECK-NEXT:    qvgpci 0, 275
+; CHECK-NEXT:    fdivs 3, 4, 3
+; CHECK-NEXT:    qvesplati 4, 1, 2
+; CHECK-NEXT:    fdivs 4, 4, 5
+; CHECK-NEXT:    fdivs 5, 1, 2
+; CHECK-NEXT:    qvesplati 2, 2, 1
+; CHECK-NEXT:    qvesplati 1, 1, 1
+; CHECK-NEXT:    fdivs 1, 1, 2
+; CHECK-NEXT:    qvfperm 2, 4, 3, 0
+; CHECK-NEXT:    qvfperm 0, 5, 1, 0
+; CHECK-NEXT:    qvgpci 1, 101
+; CHECK-NEXT:    qvfperm 1, 0, 2, 1
+; CHECK-NEXT:    blr
+entry:
+  %r = fdiv <4 x float> %a, %b
+  ret <4 x float> %r
+}
+
+define <4 x double> @foo3_fmf_denorm_on(<4 x double> %a) #0 {
+; CHECK-LABEL: foo3_fmf_denorm_on:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addis 3, 2, .LCPI12_0@toc@ha
+; CHECK-NEXT:    qvfrsqrte 0, 1
+; CHECK-NEXT:    addi 3, 3, .LCPI12_0@toc@l
+; CHECK-NEXT:    qvlfdx 2, 0, 3
+; CHECK-NEXT:    addis 3, 2, .LCPI12_1@toc@ha
+; CHECK-NEXT:    addi 3, 3, .LCPI12_1@toc@l
+; CHECK-NEXT:    qvfmul 3, 0, 0
+; CHECK-NEXT:    qvfmsub 4, 1, 2, 1
+; CHECK-NEXT:    qvfnmsub 3, 4, 3, 2
+; CHECK-NEXT:    qvfmul 0, 0, 3
+; CHECK-NEXT:    qvfmul 3, 0, 0
+; CHECK-NEXT:    qvfnmsub 2, 4, 3, 2
+; CHECK-NEXT:    qvfmul 0, 0, 2
+; CHECK-NEXT:    qvlfdx 2, 0, 3
+; CHECK-NEXT:    addis 3, 2, .LCPI12_2@toc@ha
+; CHECK-NEXT:    addi 3, 3, .LCPI12_2@toc@l
+; CHECK-NEXT:    qvlfdx 3, 0, 3
+; CHECK-NEXT:    qvfmul 0, 0, 1
+; CHECK-NEXT:    qvfabs 1, 1
+; CHECK-NEXT:    qvfcmplt 1, 1, 2
+; CHECK-NEXT:    qvfsel 1, 1, 3, 0
+; CHECK-NEXT:    blr
+entry:
+  %r = call reassoc ninf afn <4 x double> @llvm.sqrt.v4f64(<4 x double> %a)
+  ret <4 x double> %r
+}
+
+define <4 x double> @foo3_fmf_denorm_off(<4 x double> %a) #1 {
+; CHECK-LABEL: foo3_fmf_denorm_off:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addis 3, 2, .LCPI13_0@toc@ha
+; CHECK-NEXT:    qvfrsqrte 0, 1
+; CHECK-NEXT:    addi 3, 3, .LCPI13_0@toc@l
+; CHECK-NEXT:    qvlfdx 2, 0, 3
+; CHECK-NEXT:    addis 3, 2, .LCPI13_1@toc@ha
+; CHECK-NEXT:    addi 3, 3, .LCPI13_1@toc@l
+; CHECK-NEXT:    qvfmul 3, 0, 0
+; CHECK-NEXT:    qvfmsub 4, 1, 2, 1
+; CHECK-NEXT:    qvfnmsub 3, 4, 3, 2
+; CHECK-NEXT:    qvfmul 0, 0, 3
+; CHECK-NEXT:    qvfmul 3, 0, 0
+; CHECK-NEXT:    qvfnmsub 2, 4, 3, 2
+; CHECK-NEXT:    qvfmul 0, 0, 2
+; CHECK-NEXT:    qvlfdx 2, 0, 3
+; CHECK-NEXT:    qvfmul 0, 0, 1
+; CHECK-NEXT:    qvfcmpeq 1, 1, 2
+; CHECK-NEXT:    qvfsel 1, 1, 2, 0
+; CHECK-NEXT:    blr
+entry:
+  %r = call afn reassoc ninf <4 x double> @llvm.sqrt.v4f64(<4 x double> %a)
+  ret <4 x double> %r
+}
+
+define <4 x double> @foo3_safe_denorm_on(<4 x double> %a) #0 {
+; CHECK-LABEL: foo3_safe_denorm_on:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    qvesplati 2, 1, 3
+; CHECK-NEXT:    qvesplati 3, 1, 2
+; CHECK-NEXT:    fsqrt 4, 1
+; CHECK-NEXT:    qvesplati 1, 1, 1
+; CHECK-NEXT:    fsqrt 2, 2
+; CHECK-NEXT:    fsqrt 3, 3
+; CHECK-NEXT:    fsqrt 1, 1
+; CHECK-NEXT:    qvgpci 0, 275
+; CHECK-NEXT:    qvfperm 2, 3, 2, 0
+; CHECK-NEXT:    qvfperm 0, 4, 1, 0
+; CHECK-NEXT:    qvgpci 1, 101
+; CHECK-NEXT:    qvfperm 1, 0, 2, 1
+; CHECK-NEXT:    blr
+entry:
+  %r = call <4 x double> @llvm.sqrt.v4f64(<4 x double> %a)
+  ret <4 x double> %r
+}
+
+define <4 x double> @foo3_safe_denorm_off(<4 x double> %a) #1 {
+; CHECK-LABEL: foo3_safe_denorm_off:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    qvesplati 2, 1, 3
+; CHECK-NEXT:    qvesplati 3, 1, 2
+; CHECK-NEXT:    fsqrt 4, 1
+; CHECK-NEXT:    qvesplati 1, 1, 1
+; CHECK-NEXT:    fsqrt 2, 2
+; CHECK-NEXT:    fsqrt 3, 3
+; CHECK-NEXT:    fsqrt 1, 1
+; CHECK-NEXT:    qvgpci 0, 275
+; CHECK-NEXT:    qvfperm 2, 3, 2, 0
+; CHECK-NEXT:    qvfperm 0, 4, 1, 0
+; CHECK-NEXT:    qvgpci 1, 101
+; CHECK-NEXT:    qvfperm 1, 0, 2, 1
+; CHECK-NEXT:    blr
+entry:
+  %r = call <4 x double> @llvm.sqrt.v4f64(<4 x double> %a)
+  ret <4 x double> %r
+}
+
+define <4 x float> @goo3_fmf_denorm_on(<4 x float> %a) #0 {
+; CHECK-LABEL: goo3_fmf_denorm_on:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addis 3, 2, .LCPI16_1@toc@ha
+; CHECK-NEXT:    qvfrsqrtes 2, 1
+; CHECK-NEXT:    addi 3, 3, .LCPI16_1@toc@l
+; CHECK-NEXT:    qvlfsx 0, 0, 3
+; CHECK-NEXT:    addis 3, 2, .LCPI16_0@toc@ha
+; CHECK-NEXT:    addi 3, 3, .LCPI16_0@toc@l
+; CHECK-NEXT:    qvfmuls 4, 2, 2
+; CHECK-NEXT:    qvfmsubs 3, 1, 0, 1
+; CHECK-NEXT:    qvfnmsubs 0, 3, 4, 0
+; CHECK-NEXT:    qvlfsx 3, 0, 3
+; CHECK-NEXT:    addis 3, 2, .LCPI16_2@toc@ha
+; CHECK-NEXT:    addi 3, 3, .LCPI16_2@toc@l
+; CHECK-NEXT:    qvlfsx 4, 0, 3
+; CHECK-NEXT:    qvfmuls 0, 2, 0
+; CHECK-NEXT:    qvfabs 2, 1
+; CHECK-NEXT:    qvfmuls 0, 0, 1
+; CHECK-NEXT:    qvfcmplt 1, 2, 3
+; CHECK-NEXT:    qvfsel 1, 1, 4, 0
+; CHECK-NEXT:    blr
+entry:
+  %r = call reassoc afn ninf nsz <4 x float> @llvm.sqrt.v4f32(<4 x float> %a)
+  ret <4 x float> %r
+}
+
+define <4 x float> @goo3_fmf_denorm_off(<4 x float> %a) #1 {
+; CHECK-LABEL: goo3_fmf_denorm_off:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addis 3, 2, .LCPI17_1@toc@ha
+; CHECK-NEXT:    qvfrsqrtes 2, 1
+; CHECK-NEXT:    addi 3, 3, .LCPI17_1@toc@l
+; CHECK-NEXT:    qvlfsx 0, 0, 3
+; CHECK-NEXT:    addis 3, 2, .LCPI17_0@toc@ha
+; CHECK-NEXT:    addi 3, 3, .LCPI17_0@toc@l
+; CHECK-NEXT:    qvfmuls 4, 2, 2
+; CHECK-NEXT:    qvfmsubs 3, 1, 0, 1
+; CHECK-NEXT:    qvfnmsubs 0, 3, 4, 0
+; CHECK-NEXT:    qvlfsx 3, 0, 3
+; CHECK-NEXT:    qvfmuls 0, 2, 0
+; CHECK-NEXT:    qvfmuls 0, 0, 1
+; CHECK-NEXT:    qvfcmpeq 1, 1, 3
+; CHECK-NEXT:    qvfsel 1, 1, 3, 0
+; CHECK-NEXT:    blr
+entry:
+  %r = call reassoc ninf afn nsz <4 x float> @llvm.sqrt.v4f32(<4 x float> %a)
+  ret <4 x float> %r
+}
+
+define <4 x float> @goo3_safe(<4 x float> %a) nounwind {
+; CHECK-LABEL: goo3_safe:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    qvesplati 2, 1, 3
+; CHECK-NEXT:    qvesplati 3, 1, 2
+; CHECK-NEXT:    fsqrts 4, 1
+; CHECK-NEXT:    qvesplati 1, 1, 1
+; CHECK-NEXT:    fsqrts 2, 2
+; CHECK-NEXT:    fsqrts 3, 3
+; CHECK-NEXT:    fsqrts 1, 1
+; CHECK-NEXT:    qvgpci 0, 275
+; CHECK-NEXT:    qvfperm 2, 3, 2, 0
+; CHECK-NEXT:    qvfperm 0, 4, 1, 0
+; CHECK-NEXT:    qvgpci 1, 101
+; CHECK-NEXT:    qvfperm 1, 0, 2, 1
+; CHECK-NEXT:    blr
+entry:
+  %r = call <4 x float> @llvm.sqrt.v4f32(<4 x float> %a)
+  ret <4 x float> %r
+}
+
+attributes #0 = { nounwind "denormal-fp-math"="ieee,ieee" }
+attributes #1 = { nounwind "denormal-fp-math"="preserve-sign,preserve-sign" }
diff --git a/llvm/test/CodeGen/PowerPC/qpx-rounding-ops.ll b/llvm/test/CodeGen/PowerPC/qpx-rounding-ops.ll
new file mode 100644
index 0000000000000..ee3357156a6c0
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/qpx-rounding-ops.ll
@@ -0,0 +1,109 @@
+; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc64-unknown-linux-gnu -mcpu=a2q | FileCheck %s
+; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc64-unknown-linux-gnu -mcpu=a2q -enable-unsafe-fp-math | FileCheck -check-prefix=CHECK-FM %s
+target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
+target triple = "powerpc64-unknown-linux-gnu"
+
+define <4 x float> @test1(<4 x float> %x) nounwind  {
+  %call = tail call <4 x float> @llvm.floor.v4f32(<4 x float> %x) nounwind readnone
+  ret <4 x float> %call
+
+; CHECK: test1:
+; CHECK: qvfrim 1, 1
+
+; CHECK-FM: test1:
+; CHECK-FM: qvfrim 1, 1
+}
+
+declare <4 x float> @llvm.floor.v4f32(<4 x float>) nounwind readnone
+
+define <4 x double> @test2(<4 x double> %x) nounwind  {
+  %call = tail call <4 x double> @llvm.floor.v4f64(<4 x double> %x) nounwind readnone
+  ret <4 x double> %call
+
+; CHECK: test2:
+; CHECK: qvfrim 1, 1
+
+; CHECK-FM: test2:
+; CHECK-FM: qvfrim 1, 1
+}
+
+declare <4 x double> @llvm.floor.v4f64(<4 x double>) nounwind readnone
+
+define <4 x float> @test3(<4 x float> %x) nounwind  {
+  %call = tail call <4 x float> @llvm.nearbyint.v4f32(<4 x float> %x) nounwind readnone
+  ret <4 x float> %call
+
+; CHECK: test3:
+; CHECK-NOT: qvfrin
+
+; CHECK-FM: test3:
+; CHECK-FM-NOT: qvfrin
+}
+
+declare <4 x float> @llvm.nearbyint.v4f32(<4 x float>) nounwind readnone
+
+define <4 x double> @test4(<4 x double> %x) nounwind  {
+  %call = tail call <4 x double> @llvm.nearbyint.v4f64(<4 x double> %x) nounwind readnone
+  ret <4 x double> %call
+
+; CHECK: test4:
+; CHECK-NOT: qvfrin
+
+; CHECK-FM: test4:
+; CHECK-FM-NOT: qvfrin
+}
+
+declare <4 x double> @llvm.nearbyint.v4f64(<4 x double>) nounwind readnone
+
+define <4 x float> @test5(<4 x float> %x) nounwind  {
+  %call = tail call <4 x float> @llvm.ceil.v4f32(<4 x float> %x) nounwind readnone
+  ret <4 x float> %call
+
+; CHECK: test5:
+; CHECK: qvfrip 1, 1
+
+; CHECK-FM: test5:
+; CHECK-FM: qvfrip 1, 1
+}
+
+declare <4 x float> @llvm.ceil.v4f32(<4 x float>) nounwind readnone
+
+define <4 x double> @test6(<4 x double> %x) nounwind  {
+  %call = tail call <4 x double> @llvm.ceil.v4f64(<4 x double> %x) nounwind readnone
+  ret <4 x double> %call
+
+; CHECK: test6:
+; CHECK: qvfrip 1, 1
+
+; CHECK-FM: test6:
+; CHECK-FM: qvfrip 1, 1
+}
+
+declare <4 x double> @llvm.ceil.v4f64(<4 x double>) nounwind readnone
+
+define <4 x float> @test9(<4 x float> %x) nounwind  {
+  %call = tail call <4 x float> @llvm.trunc.v4f32(<4 x float> %x) nounwind readnone
+  ret <4 x float> %call
+
+; CHECK: test9:
+; CHECK: qvfriz 1, 1
+
+; CHECK-FM: test9:
+; CHECK-FM: qvfriz 1, 1
+}
+
+declare <4 x float> @llvm.trunc.v4f32(<4 x float>) nounwind readnone
+
+define <4 x double> @test10(<4 x double> %x) nounwind  {
+  %call = tail call <4 x double> @llvm.trunc.v4f64(<4 x double> %x) nounwind readnone
+  ret <4 x double> %call
+
+; CHECK: test10:
+; CHECK: qvfriz 1, 1
+
+; CHECK-FM: test10:
+; CHECK-FM: qvfriz 1, 1
+}
+
+declare <4 x double> @llvm.trunc.v4f64(<4 x double>) nounwind readnone
+
diff --git a/llvm/test/CodeGen/PowerPC/qpx-s-load.ll b/llvm/test/CodeGen/PowerPC/qpx-s-load.ll
new file mode 100644
index 0000000000000..57d7e3b0ded3c
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/qpx-s-load.ll
@@ -0,0 +1,26 @@
+; RUN: llc -verify-machineinstrs < %s -mcpu=a2q | FileCheck %s
+target triple = "powerpc64-bgq-linux"
+
+define <4 x float> @foo(<4 x float>* %p) {
+entry:
+  %v = load <4 x float>, <4 x float>* %p, align 4
+  ret <4 x float> %v
+}
+
+; CHECK: @foo
+; CHECK-DAG: li [[REG1:[0-9]+]], 15
+; CHECK-DAG: qvlfsx [[REG4:[0-9]+]], 0, 3
+; CHECK-DAG: qvlfsx [[REG2:[0-9]+]], 3, [[REG1]]
+; CHECK-DAG: qvlpclsx [[REG3:[0-9]+]], 0, 3
+; CHECK-DAG: qvfperm 1, [[REG4]], [[REG2]], [[REG3]]
+; CHECK: blr
+
+define <4 x float> @bar(<4 x float>* %p) {
+entry:
+  %v = load <4 x float>, <4 x float>* %p, align 16
+  ret <4 x float> %v
+}
+
+; CHECK: @bar
+; CHECK: qvlfsx
+
diff --git a/llvm/test/CodeGen/PowerPC/qpx-s-sel.ll b/llvm/test/CodeGen/PowerPC/qpx-s-sel.ll
new file mode 100644
index 0000000000000..5d42b9a529953
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/qpx-s-sel.ll
@@ -0,0 +1,143 @@
+; RUN: llc -verify-machineinstrs < %s -mcpu=a2q | FileCheck %s
+target triple = "powerpc64-bgq-linux"
+
+@R = global <4 x i1> <i1 0, i1 0, i1 0, i1 0>, align 16
+
+define <4 x float> @test1(<4 x float> %a, <4 x float> %b, <4 x i1> %c) nounwind readnone {
+entry:
+  %r = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %r
+
+; CHECK-LABEL: @test1
+; CHECK: qvfsel 1, 3, 1, 2
+; CHECK: blr
+}
+
+define <4 x float> @test2(<4 x float> %a, <4 x float> %b, i1 %c1, i1 %c2, i1 %c3, i1 %c4) nounwind readnone {
+entry:
+  %v = insertelement <4 x i1> undef, i1 %c1, i32 0
+  %v2 = insertelement <4 x i1> %v, i1 %c2, i32 1
+  %v3 = insertelement <4 x i1> %v2, i1 %c3, i32 2
+  %v4 = insertelement <4 x i1> %v3, i1 %c4, i32 3
+  %r = select <4 x i1> %v4, <4 x float> %a, <4 x float> %b
+  ret <4 x float> %r
+
+; CHECK-LABEL: @test2
+; CHECK: stw
+; CHECK-DAG: qvlfiwzx [[REG1:[0-9]+]],
+; CHECK-DAG: qvlfdx [[REG2:[0-9]+]],
+; CHECK-DAG: qvfcfidu [[REG3:[0-9]+]], [[REG1]]
+; CHECK: qvfcmpeq [[REG4:[0-9]+]], [[REG3]], [[REG2]]
+; CHECK: qvfsel 1, [[REG4]], 1, 2
+; CHECK: blr
+}
+
+define <4 x i1> @test3(<4 x i1> %a) nounwind readnone {
+entry:
+  %v = and <4 x i1> %a, <i1 0, i1 undef, i1 1, i1 1>
+  ret <4 x i1> %v
+
+; CHECK-LABEL: @test3
+; CHECK: qvlfsx [[REG:[0-9]+]],
+; qvflogical 1, 1, [[REG]], 1
+; blr
+}
+
+define <4 x i1> @test4(<4 x i1> %a, <4 x i1>* %t) nounwind {
+entry:
+  %q = load <4 x i1>, <4 x i1>* %t, align 16
+  %v = and <4 x i1> %a, %q
+  ret <4 x i1> %v
+
+; CHECK-LABEL: @test4
+; CHECK-DAG: lbz
+; CHECK-DAG: qvlfdx [[REG1:[0-9]+]],
+; CHECK-DAG: stw
+; CHECK-DAG: qvlfiwzx [[REG2:[0-9]+]],
+; CHECK-DAG: qvfcfidu [[REG3:[0-9]+]], [[REG2]]
+; CHECK: qvfcmpeq [[REG4:[0-9]+]], [[REG3]], [[REG1]]
+; CHECK: qvfand 1, 1, [[REG4]]
+; CHECK: blr
+}
+
+define void @test5(<4 x i1> %a) nounwind {
+entry:
+  store <4 x i1> %a, <4 x i1>* @R
+  ret void
+
+; CHECK-LABEL: @test5
+; CHECK: qvlfdx [[REG1:[0-9]+]],
+; CHECK: qvfmadd [[REG2:[0-9]+]], 1, [[REG1]], [[REG1]]
+; CHECK: qvfctiwu [[REG3:[0-9]+]], [[REG2]]
+; CHECK: qvstfiwx [[REG3]],
+; CHECK: lwz
+; CHECK: stb
+; CHECK: blr
+}
+
+define i1 @test6(<4 x i1> %a) nounwind {
+entry:
+  %r = extractelement <4 x i1> %a, i32 2
+  ret i1 %r
+
+; CHECK-LABEL: @test6
+; CHECK: qvlfdx [[REG1:[0-9]+]],
+; CHECK: qvfmadd [[REG2:[0-9]+]], 1, [[REG1]], [[REG1]]
+; CHECK: qvfctiwu [[REG3:[0-9]+]], [[REG2]]
+; CHECK: qvstfiwx [[REG3]],
+; CHECK: lwz
+; CHECK: blr
+}
+
+define i1 @test7(<4 x i1> %a) nounwind {
+entry:
+  %r = extractelement <4 x i1> %a, i32 2
+  %s = extractelement <4 x i1> %a, i32 3
+  %q = and i1 %r, %s
+  ret i1 %q
+
+; CHECK-LABEL: @test7
+; CHECK: qvlfdx [[REG1:[0-9]+]],
+; CHECK: qvfmadd [[REG2:[0-9]+]], 1, [[REG1]], [[REG1]]
+; CHECK: qvfctiwu [[REG3:[0-9]+]], [[REG2]]
+; CHECK: qvstfiwx [[REG3]],
+; CHECK-DAG: lwz [[REG4:[0-9]+]],
+; FIXME: We're storing the vector twice, and that's silly.
+; CHECK-DAG: qvstfiwx [[REG3]],
+; CHECK: lwz [[REG5:[0-9]+]],
+; CHECK: and 3,
+; CHECK: blr
+}
+
+define i1 @test8(<3 x i1> %a) nounwind {
+entry:
+  %r = extractelement <3 x i1> %a, i32 2
+  ret i1 %r
+
+; CHECK-LABEL: @test8
+; CHECK: qvlfdx [[REG1:[0-9]+]],
+; CHECK: qvfmadd [[REG2:[0-9]+]], 1, [[REG1]], [[REG1]]
+; CHECK: qvfctiwu [[REG3:[0-9]+]], [[REG2]]
+; CHECK: qvstfiwx [[REG3]],
+; CHECK: lwz
+; CHECK: blr
+}
+
+define <3 x float> @test9(<3 x float> %a, <3 x float> %b, i1 %c1, i1 %c2, i1 %c3) nounwind readnone {
+entry:
+  %v = insertelement <3 x i1> undef, i1 %c1, i32 0
+  %v2 = insertelement <3 x i1> %v, i1 %c2, i32 1
+  %v3 = insertelement <3 x i1> %v2, i1 %c3, i32 2
+  %r = select <3 x i1> %v3, <3 x float> %a, <3 x float> %b
+  ret <3 x float> %r
+
+; CHECK-LABEL: @test9
+; CHECK: stw
+; CHECK-DAG: qvlfiwzx [[REG1:[0-9]+]],
+; CHECK-DAG: qvlfdx [[REG2:[0-9]+]],
+; CHECK-DAG: qvfcfidu [[REG3:[0-9]+]], [[REG1]]
+; CHECK: qvfcmpeq [[REG4:[0-9]+]], [[REG3]], [[REG2]]
+; CHECK: qvfsel 1, [[REG4]], 1, 2
+; CHECK: blr
+}
+
diff --git a/llvm/test/CodeGen/PowerPC/qpx-s-store.ll b/llvm/test/CodeGen/PowerPC/qpx-s-store.ll
new file mode 100644
index 0000000000000..81cff7b6457f1
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/qpx-s-store.ll
@@ -0,0 +1,25 @@
+; RUN: llc -verify-machineinstrs < %s -mcpu=a2q | FileCheck %s
+target triple = "powerpc64-bgq-linux"
+
+define void @foo(<4 x float> %v, <4 x float>* %p) {
+entry:
+  store <4 x float> %v, <4 x float>* %p, align 4
+  ret void
+}
+
+; CHECK: @foo
+; CHECK: stfs
+; CHECK: stfs
+; CHECK: stfs
+; CHECK: stfs
+; CHECK: blr
+
+define void @bar(<4 x float> %v, <4 x float>* %p) {
+entry:
+  store <4 x float> %v, <4 x float>* %p, align 16
+  ret void
+}
+
+; CHECK: @bar
+; CHECK: qvstfsx
+
diff --git a/llvm/test/CodeGen/PowerPC/qpx-sel.ll b/llvm/test/CodeGen/PowerPC/qpx-sel.ll
new file mode 100644
index 0000000000000..abc92c9e98b13
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/qpx-sel.ll
@@ -0,0 +1,151 @@
+; RUN: llc -verify-machineinstrs < %s -mcpu=a2q | FileCheck %s
+target triple = "powerpc64-bgq-linux"
+
+@R = global <4 x i1> <i1 0, i1 0, i1 0, i1 0>, align 16
+
+define <4 x double> @test1(<4 x double> %a, <4 x double> %b, <4 x i1> %c) nounwind readnone {
+entry:
+  %r = select <4 x i1> %c, <4 x double> %a, <4 x double> %b
+  ret <4 x double> %r
+
+; CHECK-LABEL: @test1
+; CHECK: qvfsel 1, 3, 1, 2
+; CHECK: blr
+}
+
+define <4 x double> @test2(<4 x double> %a, <4 x double> %b, i1 %c1, i1 %c2, i1 %c3, i1 %c4) nounwind readnone {
+entry:
+  %v = insertelement <4 x i1> undef, i1 %c1, i32 0
+  %v2 = insertelement <4 x i1> %v, i1 %c2, i32 1
+  %v3 = insertelement <4 x i1> %v2, i1 %c3, i32 2
+  %v4 = insertelement <4 x i1> %v3, i1 %c4, i32 3
+  %r = select <4 x i1> %v4, <4 x double> %a, <4 x double> %b
+  ret <4 x double> %r
+
+; CHECK-LABEL: @test2
+
+; FIXME: This load/store sequence is unnecessary.
+; CHECK-DAG: lbz
+; CHECK-DAG: stw
+
+; CHECK-DAG: qvlfiwzx [[REG1:[0-9]+]],
+; CHECK-DAG: qvlfdx [[REG2:[0-9]+]],
+; CHECK-DAG: qvfcfidu [[REG3:[0-9]+]], [[REG1]]
+; CHECK: qvfcmpeq [[REG4:[0-9]+]], [[REG3]], [[REG2]]
+; CHECK: qvfsel 1, [[REG4]], 1, 2
+; CHECK: blr
+}
+
+define <4 x i1> @test3(<4 x i1> %a) nounwind readnone {
+entry:
+  %v = and <4 x i1> %a, <i1 0, i1 undef, i1 1, i1 1>
+  ret <4 x i1> %v
+
+; CHECK-LABEL: @test3
+; CHECK: qvlfsx [[REG:[0-9]+]],
+; qvflogical 1, 1, [[REG]], 1
+; blr
+}
+
+define <4 x i1> @test4(<4 x i1> %a, <4 x i1>* %t) nounwind {
+entry:
+  %q = load <4 x i1>, <4 x i1>* %t, align 16
+  %v = and <4 x i1> %a, %q
+  ret <4 x i1> %v
+
+; CHECK-LABEL: @test4
+; CHECK-DAG: lbz
+; CHECK-DAG: qvlfdx [[REG1:[0-9]+]],
+; CHECK-DAG: stw
+; CHECK-DAG: qvlfiwzx [[REG2:[0-9]+]],
+; CHECK-DAG: qvfcfidu [[REG3:[0-9]+]], [[REG2]]
+; CHECK: qvfcmpeq [[REG4:[0-9]+]], [[REG3]], [[REG1]]
+; CHECK: qvfand 1, 1, [[REG4]]
+; CHECK: blr
+}
+
+define void @test5(<4 x i1> %a) nounwind {
+entry:
+  store <4 x i1> %a, <4 x i1>* @R
+  ret void
+
+; CHECK-LABEL: @test5
+; CHECK: qvlfdx [[REG1:[0-9]+]],
+; CHECK: qvfmadd [[REG2:[0-9]+]], 1, [[REG1]], [[REG1]]
+; CHECK: qvfctiwu [[REG3:[0-9]+]], [[REG2]]
+; CHECK: qvstfiwx [[REG3]],
+; CHECK: lwz
+; CHECK: stb
+; CHECK: blr
+}
+
+define i1 @test6(<4 x i1> %a) nounwind {
+entry:
+  %r = extractelement <4 x i1> %a, i32 2
+  ret i1 %r
+
+; CHECK-LABEL: @test6
+; CHECK: qvlfdx [[REG1:[0-9]+]],
+; CHECK: qvfmadd [[REG2:[0-9]+]], 1, [[REG1]], [[REG1]]
+; CHECK: qvfctiwu [[REG3:[0-9]+]], [[REG2]]
+; CHECK: qvstfiwx [[REG3]],
+; CHECK: lwz
+; CHECK: blr
+}
+
+define i1 @test7(<4 x i1> %a) nounwind {
+entry:
+  %r = extractelement <4 x i1> %a, i32 2
+  %s = extractelement <4 x i1> %a, i32 3
+  %q = and i1 %r, %s
+  ret i1 %q
+
+; CHECK-LABEL: @test7
+; CHECK: qvlfdx [[REG1:[0-9]+]],
+; CHECK: qvfmadd [[REG2:[0-9]+]], 1, [[REG1]], [[REG1]]
+; CHECK: qvfctiwu [[REG3:[0-9]+]], [[REG2]]
+; CHECK: qvstfiwx [[REG3]],
+; CHECK-DAG: lwz [[REG4:[0-9]+]],
+; FIXME: We're storing the vector twice, and that's silly.
+; CHECK-DAG: qvstfiwx [[REG3]],
+; CHECK-DAG: lwz [[REG5:[0-9]+]],
+; CHECK: and 3,
+; CHECK: blr
+}
+
+define i1 @test8(<3 x i1> %a) nounwind {
+entry:
+  %r = extractelement <3 x i1> %a, i32 2
+  ret i1 %r
+
+; CHECK-LABEL: @test8
+; CHECK: qvlfdx [[REG1:[0-9]+]],
+; CHECK: qvfmadd [[REG2:[0-9]+]], 1, [[REG1]], [[REG1]]
+; CHECK: qvfctiwu [[REG3:[0-9]+]], [[REG2]]
+; CHECK: qvstfiwx [[REG3]],
+; CHECK: lwz
+; CHECK: blr
+}
+
+define <3 x double> @test9(<3 x double> %a, <3 x double> %b, i1 %c1, i1 %c2, i1 %c3) nounwind readnone {
+entry:
+  %v = insertelement <3 x i1> undef, i1 %c1, i32 0
+  %v2 = insertelement <3 x i1> %v, i1 %c2, i32 1
+  %v3 = insertelement <3 x i1> %v2, i1 %c3, i32 2
+  %r = select <3 x i1> %v3, <3 x double> %a, <3 x double> %b
+  ret <3 x double> %r
+
+; CHECK-LABEL: @test9
+
+; FIXME: This load/store sequence is unnecessary.
+; CHECK-DAG: lbz
+; CHECK-DAG: stw
+
+; CHECK-DAG: qvlfiwzx [[REG1:[0-9]+]],
+; CHECK-DAG: qvlfdx [[REG2:[0-9]+]],
+; CHECK-DAG: qvfcfidu [[REG3:[0-9]+]], [[REG1]]
+; CHECK: qvfcmpeq [[REG4:[0-9]+]], [[REG3]], [[REG2]]
+; CHECK: qvfsel 1, [[REG4]], 1, 2
+; CHECK: blr
+}
+
diff --git a/llvm/test/CodeGen/PowerPC/qpx-split-vsetcc.ll b/llvm/test/CodeGen/PowerPC/qpx-split-vsetcc.ll
new file mode 100644
index 0000000000000..df3e0befaef8a
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/qpx-split-vsetcc.ll
@@ -0,0 +1,31 @@
+; RUN: llc -verify-machineinstrs -mcpu=a2q < %s | FileCheck %s
+target datalayout = "E-m:e-i64:64-n32:64"
+target triple = "powerpc64-bgq-linux"
+
+; Function Attrs: nounwind
+define void @gsl_sf_legendre_Pl_deriv_array(<4 x i32> %inp1, <4 x double> %inp2) #0 {
+entry:
+  br label %vector.body198
+
+vector.body198:                                   ; preds = %vector.body198, %for.body46.lr.ph
+  %0 = icmp ne <4 x i32> %inp1, zeroinitializer
+  %1 = select <4 x i1> %0, <4 x double> <double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01>, <4 x double> <double -5.000000e-01, double -5.000000e-01, double -5.000000e-01, double -5.000000e-01>
+  %2 = fmul <4 x double> %inp2, %1
+  %3 = fmul <4 x double> %inp2, %2
+  %4 = fmul <4 x double> %3, %inp2
+  store <4 x double> %4, <4 x double>* undef, align 8
+  br label %return
+
+; CHECK-LABEL: @gsl_sf_legendre_Pl_deriv_array
+; CHECK: qvlfiwzx
+; CHECK: qvfcfidu
+; CHECK: qvfcmpeq
+; CHECK: qvfsel
+; CHECK: qvfmul
+
+return:                                           ; preds = %if.else.i
+  ret void
+}
+
+attributes #0 = { nounwind }
+
diff --git a/llvm/test/CodeGen/PowerPC/qpx-store.ll b/llvm/test/CodeGen/PowerPC/qpx-store.ll
new file mode 100644
index 0000000000000..2b96576ce4493
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/qpx-store.ll
@@ -0,0 +1,25 @@
+; RUN: llc -verify-machineinstrs < %s -mcpu=a2q | FileCheck %s
+target triple = "powerpc64-bgq-linux"
+
+define void @foo(<4 x double> %v, <4 x double>* %p) {
+entry:
+  store <4 x double> %v, <4 x double>* %p, align 8
+  ret void
+}
+
+; CHECK: @foo
+; CHECK: stfd
+; CHECK: stfd
+; CHECK: stfd
+; CHECK: stfd
+; CHECK: blr
+
+define void @bar(<4 x double> %v, <4 x double>* %p) {
+entry:
+  store <4 x double> %v, <4 x double>* %p, align 32
+  ret void
+}
+
+; CHECK: @bar
+; CHECK: qvstfdx
+
diff --git a/llvm/test/CodeGen/PowerPC/qpx-unal-cons-lds.ll b/llvm/test/CodeGen/PowerPC/qpx-unal-cons-lds.ll
new file mode 100644
index 0000000000000..e7ab92db6efc9
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/qpx-unal-cons-lds.ll
@@ -0,0 +1,217 @@
+; RUN: llc -verify-machineinstrs < %s | FileCheck %s
+target datalayout = "E-m:e-i64:64-n32:64"
+target triple = "powerpc64-unknown-linux-gnu"
+
+; Function Attrs: nounwind
+define void @foo(double* noalias nocapture %a, double* noalias nocapture readonly %b) #0 {
+entry:
+  br label %vector.body
+
+; CHECK-LABEL: @foo
+; Make sure that the offset constants we use are all even (only the last should be odd).
+; CHECK-DAG: li {{[0-9]+}}, 1056
+; CHECK-DAG: li {{[0-9]+}}, 1088
+; CHECK-DAG: li {{[0-9]+}}, 1152
+; CHECK-DAG: li {{[0-9]+}}, 1216
+; CHECK-DAG: li {{[0-9]+}}, 1280
+; CHECK-DAG: li {{[0-9]+}}, 1344
+; CHECK-DAG: li {{[0-9]+}}, 1408
+; CHECK-DAG: li {{[0-9]+}}, 1472
+; CHECK-DAG: li {{[0-9]+}}, 1536
+; CHECK-DAG: li {{[0-9]+}}, 1600
+; CHECK-DAG: li {{[0-9]+}}, 1568
+; CHECK-DAG: li {{[0-9]+}}, 1664
+; CHECK-DAG: li {{[0-9]+}}, 1632
+; CHECK-DAG: li {{[0-9]+}}, 1728
+; CHECK-DAG: li {{[0-9]+}}, 1696
+; CHECK-DAG: li {{[0-9]+}}, 1792
+; CHECK-DAG: li {{[0-9]+}}, 1760
+; CHECK-DAG: li {{[0-9]+}}, 1856
+; CHECK-DAG: li {{[0-9]+}}, 1824
+; CHECK-DAG: li {{[0-9]+}}, 1920
+; CHECK-DAG: li {{[0-9]+}}, 1888
+; CHECK-DAG: li {{[0-9]+}}, 1984
+; CHECK-DAG: li {{[0-9]+}}, 1952
+; CHECK-DAG: li {{[0-9]+}}, 2016
+; CHECK-DAG: li {{[0-9]+}}, 1024
+; CHECK-DAG: li {{[0-9]+}}, 1120
+; CHECK-DAG: li {{[0-9]+}}, 1184
+; CHECK-DAG: li {{[0-9]+}}, 1248
+; CHECK-DAG: li {{[0-9]+}}, 1312
+; CHECK-DAG: li {{[0-9]+}}, 1376
+; CHECK-DAG: li {{[0-9]+}}, 1440
+; CHECK-DAG: li {{[0-9]+}}, 1504
+; CHECK-DAG: li {{[0-9]+}}, 2047
+; CHECK: blr
+
+vector.body:                                      ; preds = %vector.body, %entry
+  %index = phi i64 [ 0, %entry ], [ %index.next.15, %vector.body ]
+  %0 = shl i64 %index, 1
+  %1 = getelementptr inbounds double, double* %b, i64 %0
+  %2 = bitcast double* %1 to <8 x double>*
+  %wide.vec = load <8 x double>, <8 x double>* %2, align 8
+  %strided.vec = shufflevector <8 x double> %wide.vec, <8 x double> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  %3 = fadd <4 x double> %strided.vec, <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
+  %4 = getelementptr inbounds double, double* %a, i64 %index
+  %5 = bitcast double* %4 to <4 x double>*
+  store <4 x double> %3, <4 x double>* %5, align 8
+  %index.next = or i64 %index, 4
+  %6 = shl i64 %index.next, 1
+  %7 = getelementptr inbounds double, double* %b, i64 %6
+  %8 = bitcast double* %7 to <8 x double>*
+  %wide.vec.1 = load <8 x double>, <8 x double>* %8, align 8
+  %strided.vec.1 = shufflevector <8 x double> %wide.vec.1, <8 x double> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  %9 = fadd <4 x double> %strided.vec.1, <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
+  %10 = getelementptr inbounds double, double* %a, i64 %index.next
+  %11 = bitcast double* %10 to <4 x double>*
+  store <4 x double> %9, <4 x double>* %11, align 8
+  %index.next.1 = or i64 %index, 8
+  %12 = shl i64 %index.next.1, 1
+  %13 = getelementptr inbounds double, double* %b, i64 %12
+  %14 = bitcast double* %13 to <8 x double>*
+  %wide.vec.2 = load <8 x double>, <8 x double>* %14, align 8
+  %strided.vec.2 = shufflevector <8 x double> %wide.vec.2, <8 x double> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  %15 = fadd <4 x double> %strided.vec.2, <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
+  %16 = getelementptr inbounds double, double* %a, i64 %index.next.1
+  %17 = bitcast double* %16 to <4 x double>*
+  store <4 x double> %15, <4 x double>* %17, align 8
+  %index.next.2 = or i64 %index, 12
+  %18 = shl i64 %index.next.2, 1
+  %19 = getelementptr inbounds double, double* %b, i64 %18
+  %20 = bitcast double* %19 to <8 x double>*
+  %wide.vec.3 = load <8 x double>, <8 x double>* %20, align 8
+  %strided.vec.3 = shufflevector <8 x double> %wide.vec.3, <8 x double> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  %21 = fadd <4 x double> %strided.vec.3, <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
+  %22 = getelementptr inbounds double, double* %a, i64 %index.next.2
+  %23 = bitcast double* %22 to <4 x double>*
+  store <4 x double> %21, <4 x double>* %23, align 8
+  %index.next.3 = or i64 %index, 16
+  %24 = shl i64 %index.next.3, 1
+  %25 = getelementptr inbounds double, double* %b, i64 %24
+  %26 = bitcast double* %25 to <8 x double>*
+  %wide.vec.4 = load <8 x double>, <8 x double>* %26, align 8
+  %strided.vec.4 = shufflevector <8 x double> %wide.vec.4, <8 x double> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  %27 = fadd <4 x double> %strided.vec.4, <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
+  %28 = getelementptr inbounds double, double* %a, i64 %index.next.3
+  %29 = bitcast double* %28 to <4 x double>*
+  store <4 x double> %27, <4 x double>* %29, align 8
+  %index.next.4 = or i64 %index, 20
+  %30 = shl i64 %index.next.4, 1
+  %31 = getelementptr inbounds double, double* %b, i64 %30
+  %32 = bitcast double* %31 to <8 x double>*
+  %wide.vec.5 = load <8 x double>, <8 x double>* %32, align 8
+  %strided.vec.5 = shufflevector <8 x double> %wide.vec.5, <8 x double> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  %33 = fadd <4 x double> %strided.vec.5, <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
+  %34 = getelementptr inbounds double, double* %a, i64 %index.next.4
+  %35 = bitcast double* %34 to <4 x double>*
+  store <4 x double> %33, <4 x double>* %35, align 8
+  %index.next.5 = or i64 %index, 24
+  %36 = shl i64 %index.next.5, 1
+  %37 = getelementptr inbounds double, double* %b, i64 %36
+  %38 = bitcast double* %37 to <8 x double>*
+  %wide.vec.6 = load <8 x double>, <8 x double>* %38, align 8
+  %strided.vec.6 = shufflevector <8 x double> %wide.vec.6, <8 x double> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  %39 = fadd <4 x double> %strided.vec.6, <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
+  %40 = getelementptr inbounds double, double* %a, i64 %index.next.5
+  %41 = bitcast double* %40 to <4 x double>*
+  store <4 x double> %39, <4 x double>* %41, align 8
+  %index.next.6 = or i64 %index, 28
+  %42 = shl i64 %index.next.6, 1
+  %43 = getelementptr inbounds double, double* %b, i64 %42
+  %44 = bitcast double* %43 to <8 x double>*
+  %wide.vec.7 = load <8 x double>, <8 x double>* %44, align 8
+  %strided.vec.7 = shufflevector <8 x double> %wide.vec.7, <8 x double> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  %45 = fadd <4 x double> %strided.vec.7, <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
+  %46 = getelementptr inbounds double, double* %a, i64 %index.next.6
+  %47 = bitcast double* %46 to <4 x double>*
+  store <4 x double> %45, <4 x double>* %47, align 8
+  %index.next.7 = or i64 %index, 32
+  %48 = shl i64 %index.next.7, 1
+  %49 = getelementptr inbounds double, double* %b, i64 %48
+  %50 = bitcast double* %49 to <8 x double>*
+  %wide.vec.8 = load <8 x double>, <8 x double>* %50, align 8
+  %strided.vec.8 = shufflevector <8 x double> %wide.vec.8, <8 x double> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  %51 = fadd <4 x double> %strided.vec.8, <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
+  %52 = getelementptr inbounds double, double* %a, i64 %index.next.7
+  %53 = bitcast double* %52 to <4 x double>*
+  store <4 x double> %51, <4 x double>* %53, align 8
+  %index.next.8 = or i64 %index, 36
+  %54 = shl i64 %index.next.8, 1
+  %55 = getelementptr inbounds double, double* %b, i64 %54
+  %56 = bitcast double* %55 to <8 x double>*
+  %wide.vec.9 = load <8 x double>, <8 x double>* %56, align 8
+  %strided.vec.9 = shufflevector <8 x double> %wide.vec.9, <8 x double> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  %57 = fadd <4 x double> %strided.vec.9, <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
+  %58 = getelementptr inbounds double, double* %a, i64 %index.next.8
+  %59 = bitcast double* %58 to <4 x double>*
+  store <4 x double> %57, <4 x double>* %59, align 8
+  %index.next.9 = or i64 %index, 40
+  %60 = shl i64 %index.next.9, 1
+  %61 = getelementptr inbounds double, double* %b, i64 %60
+  %62 = bitcast double* %61 to <8 x double>*
+  %wide.vec.10 = load <8 x double>, <8 x double>* %62, align 8
+  %strided.vec.10 = shufflevector <8 x double> %wide.vec.10, <8 x double> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  %63 = fadd <4 x double> %strided.vec.10, <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
+  %64 = getelementptr inbounds double, double* %a, i64 %index.next.9
+  %65 = bitcast double* %64 to <4 x double>*
+  store <4 x double> %63, <4 x double>* %65, align 8
+  %index.next.10 = or i64 %index, 44
+  %66 = shl i64 %index.next.10, 1
+  %67 = getelementptr inbounds double, double* %b, i64 %66
+  %68 = bitcast double* %67 to <8 x double>*
+  %wide.vec.11 = load <8 x double>, <8 x double>* %68, align 8
+  %strided.vec.11 = shufflevector <8 x double> %wide.vec.11, <8 x double> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  %69 = fadd <4 x double> %strided.vec.11, <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
+  %70 = getelementptr inbounds double, double* %a, i64 %index.next.10
+  %71 = bitcast double* %70 to <4 x double>*
+  store <4 x double> %69, <4 x double>* %71, align 8
+  %index.next.11 = or i64 %index, 48
+  %72 = shl i64 %index.next.11, 1
+  %73 = getelementptr inbounds double, double* %b, i64 %72
+  %74 = bitcast double* %73 to <8 x double>*
+  %wide.vec.12 = load <8 x double>, <8 x double>* %74, align 8
+  %strided.vec.12 = shufflevector <8 x double> %wide.vec.12, <8 x double> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  %75 = fadd <4 x double> %strided.vec.12, <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
+  %76 = getelementptr inbounds double, double* %a, i64 %index.next.11
+  %77 = bitcast double* %76 to <4 x double>*
+  store <4 x double> %75, <4 x double>* %77, align 8
+  %index.next.12 = or i64 %index, 52
+  %78 = shl i64 %index.next.12, 1
+  %79 = getelementptr inbounds double, double* %b, i64 %78
+  %80 = bitcast double* %79 to <8 x double>*
+  %wide.vec.13 = load <8 x double>, <8 x double>* %80, align 8
+  %strided.vec.13 = shufflevector <8 x double> %wide.vec.13, <8 x double> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  %81 = fadd <4 x double> %strided.vec.13, <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
+  %82 = getelementptr inbounds double, double* %a, i64 %index.next.12
+  %83 = bitcast double* %82 to <4 x double>*
+  store <4 x double> %81, <4 x double>* %83, align 8
+  %index.next.13 = or i64 %index, 56
+  %84 = shl i64 %index.next.13, 1
+  %85 = getelementptr inbounds double, double* %b, i64 %84
+  %86 = bitcast double* %85 to <8 x double>*
+  %wide.vec.14 = load <8 x double>, <8 x double>* %86, align 8
+  %strided.vec.14 = shufflevector <8 x double> %wide.vec.14, <8 x double> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  %87 = fadd <4 x double> %strided.vec.14, <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
+  %88 = getelementptr inbounds double, double* %a, i64 %index.next.13
+  %89 = bitcast double* %88 to <4 x double>*
+  store <4 x double> %87, <4 x double>* %89, align 8
+  %index.next.14 = or i64 %index, 60
+  %90 = shl i64 %index.next.14, 1
+  %91 = getelementptr inbounds double, double* %b, i64 %90
+  %92 = bitcast double* %91 to <8 x double>*
+  %wide.vec.15 = load <8 x double>, <8 x double>* %92, align 8
+  %strided.vec.15 = shufflevector <8 x double> %wide.vec.15, <8 x double> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  %93 = fadd <4 x double> %strided.vec.15, <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
+  %94 = getelementptr inbounds double, double* %a, i64 %index.next.14
+  %95 = bitcast double* %94 to <4 x double>*
+  store <4 x double> %93, <4 x double>* %95, align 8
+  %index.next.15 = add nsw i64 %index, 64
+  %96 = icmp eq i64 %index.next.15, 1600
+  br i1 %96, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup:                                 ; preds = %vector.body
+  ret void
+}
+
+attributes #0 = { nounwind "target-cpu"="a2q" }
+
diff --git a/llvm/test/CodeGen/PowerPC/qpx-unalperm.ll b/llvm/test/CodeGen/PowerPC/qpx-unalperm.ll
new file mode 100644
index 0000000000000..fdee919fdfc32
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/qpx-unalperm.ll
@@ -0,0 +1,64 @@
+; RUN: llc -verify-machineinstrs < %s -mcpu=a2q | FileCheck %s
+target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
+target triple = "powerpc64-bgq-linux"
+
+define <4 x double> @foo(<4 x double>* %a) {
+entry:
+  %r = load <4 x double>, <4 x double>* %a, align 32
+  ret <4 x double> %r
+; CHECK: qvlfdx
+; CHECK: blr
+}
+
+define <4 x double> @bar(<4 x double>* %a) {
+entry:
+  %r = load <4 x double>, <4 x double>* %a, align 8
+  %b = getelementptr <4 x double>, <4 x double>* %a, i32 16
+  %s = load <4 x double>, <4 x double>* %b, align 32
+  %t = fadd <4 x double> %r, %s
+  ret <4 x double> %t
+; CHECK: qvlpcldx
+; CHECK: qvlfdx
+; CHECK: qvfperm
+; CHECK: blr
+}
+
+define <4 x double> @bar1(<4 x double>* %a) {
+entry:
+  %r = load <4 x double>, <4 x double>* %a, align 8
+  %b = getelementptr <4 x double>, <4 x double>* %a, i32 16
+  %s = load <4 x double>, <4 x double>* %b, align 8
+  %t = fadd <4 x double> %r, %s
+  ret <4 x double> %t
+}
+
+define <4 x double> @bar2(<4 x double>* %a) {
+entry:
+  %r = load <4 x double>, <4 x double>* %a, align 8
+  %b = getelementptr <4 x double>, <4 x double>* %a, i32 1
+  %s = load <4 x double>, <4 x double>* %b, align 32
+  %t = fadd <4 x double> %r, %s
+  ret <4 x double> %t
+}
+
+define <4 x double> @bar3(<4 x double>* %a) {
+entry:
+  %r = load <4 x double>, <4 x double>* %a, align 8
+  %b = getelementptr <4 x double>, <4 x double>* %a, i32 1
+  %s = load <4 x double>, <4 x double>* %b, align 8
+  %t = fadd <4 x double> %r, %s
+  ret <4 x double> %t
+}
+
+define <4 x double> @bar4(<4 x double>* %a) {
+entry:
+  %r = load <4 x double>, <4 x double>* %a, align 8
+  %b = getelementptr <4 x double>, <4 x double>* %a, i32 1
+  %s = load <4 x double>, <4 x double>* %b, align 8
+  %c = getelementptr <4 x double>, <4 x double>* %b, i32 1
+  %t = load <4 x double>, <4 x double>* %c, align 8
+  %u = fadd <4 x double> %r, %s
+  %v = fadd <4 x double> %u, %t
+  ret <4 x double> %v
+}
+
diff --git a/llvm/test/CodeGen/PowerPC/rlwimi-and.ll b/llvm/test/CodeGen/PowerPC/rlwimi-and.ll
index d512f51a76e7a..e8fc409527588 100644
--- a/llvm/test/CodeGen/PowerPC/rlwimi-and.ll
+++ b/llvm/test/CodeGen/PowerPC/rlwimi-and.ll
@@ -1,4 +1,6 @@
-; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mattr=-crbits -mtriple=powerpc64-unknown-linux-gnu < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mattr=-crbits < %s | FileCheck %s
+target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
+target triple = "powerpc64-bgq-linux"
 
 define void @test() align 2 {
 entry:
diff --git a/llvm/test/CodeGen/PowerPC/rlwinm_rldicl_to_andi.mir b/llvm/test/CodeGen/PowerPC/rlwinm_rldicl_to_andi.mir
index dbe314b5251fe..e3aeb5605b42c 100644
--- a/llvm/test/CodeGen/PowerPC/rlwinm_rldicl_to_andi.mir
+++ b/llvm/test/CodeGen/PowerPC/rlwinm_rldicl_to_andi.mir
@@ -60,7 +60,7 @@
     ret i64 %cond
   }
   
-  attributes #0 = { norecurse nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector" "unsafe-fp-math"="false" "use-soft-float"="false" }
+  attributes #0 = { norecurse nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
   
   !llvm.module.flags = !{!0, !1}
   !llvm.ident = !{!2}
diff --git a/llvm/test/CodeGen/PowerPC/s000-alias-misched.ll b/llvm/test/CodeGen/PowerPC/s000-alias-misched.ll
index 20071ea1710c5..5c15145af2378 100644
--- a/llvm/test/CodeGen/PowerPC/s000-alias-misched.ll
+++ b/llvm/test/CodeGen/PowerPC/s000-alias-misched.ll
@@ -1,6 +1,7 @@
-; RUN: llc -verify-machineinstrs < %s -enable-misched -mcpu=a2 -mtriple=powerpc64-unknown-linux-gnu | FileCheck %s
-; RUN: llc -verify-machineinstrs < %s -enable-misched -enable-aa-sched-mi -mcpu=a2 -mtriple=powerpc64-unknown-linux-gnu | FileCheck %s
+; RUN: llc -verify-machineinstrs < %s -enable-misched -mcpu=a2 | FileCheck %s
+; RUN: llc -verify-machineinstrs < %s -enable-misched -enable-aa-sched-mi -mcpu=a2 | FileCheck %s
 target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
+target triple = "powerpc64-bgq-linux"
 
 @aa = external global [256 x [256 x double]], align 32
 @bb = external global [256 x [256 x double]], align 32
diff --git a/llvm/test/CodeGen/PowerPC/select-i1-vs-i1.ll b/llvm/test/CodeGen/PowerPC/select-i1-vs-i1.ll
index 9f458ebcf0a6e..80ac733156197 100644
--- a/llvm/test/CodeGen/PowerPC/select-i1-vs-i1.ll
+++ b/llvm/test/CodeGen/PowerPC/select-i1-vs-i1.ll
@@ -1225,5 +1225,576 @@ entry:
 ; CHECK: blr
 }
 
+define <4 x double> @testqv4doubleslt(float %c1, float %c2, float %c3, float %c4, <4 x double> %a1, <4 x double> %a2) #1 {
+entry:
+  %cmp1 = fcmp oeq float %c3, %c4
+  %cmp3tmp = fcmp oeq float %c1, %c2
+  %cmp3 = icmp slt i1 %cmp3tmp, %cmp1
+  %cond = select i1 %cmp3, <4 x double> %a1, <4 x double> %a2
+  ret <4 x double> %cond
+
+; CHECK-LABEL: @testqv4doubleslt
+; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
+; CHECK: bc 12, 2, .LBB[[BB1:[0-9_]+]]
+; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
+; CHECK: bc 12, 2, .LBB[[BB2:[0-9_]+]]
+; CHECK: .LBB[[BB1]]:
+; CHECK: qvfmr 5, 6
+; CHECK: .LBB[[BB2]]:
+; CHECK: qvfmr 1, 5
+; CHECK: blr
+}
+
+define <4 x double> @testqv4doubleult(float %c1, float %c2, float %c3, float %c4, <4 x double> %a1, <4 x double> %a2) #1 {
+entry:
+  %cmp1 = fcmp oeq float %c3, %c4
+  %cmp3tmp = fcmp oeq float %c1, %c2
+  %cmp3 = icmp ult i1 %cmp3tmp, %cmp1
+  %cond = select i1 %cmp3, <4 x double> %a1, <4 x double> %a2
+  ret <4 x double> %cond
+
+; CHECK-LABEL: @testqv4doubleult
+; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
+; CHECK: bc 4, 2, .LBB[[BB1:[0-9_]+]]
+; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
+; CHECK: bc 4, 2, .LBB[[BB2:[0-9_]+]]
+; CHECK: .LBB[[BB1]]:
+; CHECK: qvfmr 5, 6
+; CHECK: .LBB[[BB2]]:
+; CHECK: qvfmr 1, 5
+; CHECK: blr
+}
+
+define <4 x double> @testqv4doublesle(float %c1, float %c2, float %c3, float %c4, <4 x double> %a1, <4 x double> %a2) #1 {
+entry:
+  %cmp1 = fcmp oeq float %c3, %c4
+  %cmp3tmp = fcmp oeq float %c1, %c2
+  %cmp3 = icmp sle i1 %cmp3tmp, %cmp1
+  %cond = select i1 %cmp3, <4 x double> %a1, <4 x double> %a2
+  ret <4 x double> %cond
+
+; CHECK-LABEL: @testqv4doublesle
+; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
+; CHECK: bc 4, 2, .LBB[[BB:[0-9_]+]]
+; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
+; CHECK: bc 12, 2, .LBB[[BB]]
+; CHECK: qvfmr 5, 6
+; CHECK: .LBB[[BB]]:
+; CHECK: qvfmr 1, 5
+; CHECK: blr
+}
+
+define <4 x double> @testqv4doubleule(float %c1, float %c2, float %c3, float %c4, <4 x double> %a1, <4 x double> %a2) #1 {
+entry:
+  %cmp1 = fcmp oeq float %c3, %c4
+  %cmp3tmp = fcmp oeq float %c1, %c2
+  %cmp3 = icmp ule i1 %cmp3tmp, %cmp1
+  %cond = select i1 %cmp3, <4 x double> %a1, <4 x double> %a2
+  ret <4 x double> %cond
+
+; CHECK-LABEL: @testqv4doubleule
+; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
+; CHECK: bc 12, 2, .LBB[[BB:[0-9_]+]]
+; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
+; CHECK: bc 4, 2, .LBB[[BB]]
+; CHECK: qvfmr 5, 6
+; CHECK: .LBB[[BB]]:
+; CHECK: qvfmr 1, 5
+; CHECK: blr
+}
+
+define <4 x double> @testqv4doubleeq(float %c1, float %c2, float %c3, float %c4, <4 x double> %a1, <4 x double> %a2) #1 {
+entry:
+  %cmp1 = fcmp oeq float %c3, %c4
+  %cmp3tmp = fcmp oeq float %c1, %c2
+  %cmp3 = icmp eq i1 %cmp3tmp, %cmp1
+  %cond = select i1 %cmp3, <4 x double> %a1, <4 x double> %a2
+  ret <4 x double> %cond
+
+; CHECK-LABEL: @testqv4doubleeq
+; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
+; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
+; CHECK: creqv [[REG1:[0-9]+]], {{[0-9]+}}, {{[0-9]+}}
+; CHECK: bclr 12, [[REG1]], 0
+; CHECK: qvfmr 1, 6
+; CHECK: blr
+}
+
+define <4 x double> @testqv4doublesge(float %c1, float %c2, float %c3, float %c4, <4 x double> %a1, <4 x double> %a2) #1 {
+entry:
+  %cmp1 = fcmp oeq float %c3, %c4
+  %cmp3tmp = fcmp oeq float %c1, %c2
+  %cmp3 = icmp sge i1 %cmp3tmp, %cmp1
+  %cond = select i1 %cmp3, <4 x double> %a1, <4 x double> %a2
+  ret <4 x double> %cond
+
+; CHECK-LABEL: @testqv4doublesge
+; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
+; CHECK: bc 12, 2, .LBB[[BB:[0-9_]+]]
+; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
+; CHECK: bc 4, 2, .LBB[[BB]]
+; CHECK: qvfmr 5, 6
+; CHECK: .LBB[[BB]]:
+; CHECK: qvfmr 1, 5
+; CHECK: blr
+}
+
+define <4 x double> @testqv4doubleuge(float %c1, float %c2, float %c3, float %c4, <4 x double> %a1, <4 x double> %a2) #1 {
+entry:
+  %cmp1 = fcmp oeq float %c3, %c4
+  %cmp3tmp = fcmp oeq float %c1, %c2
+  %cmp3 = icmp uge i1 %cmp3tmp, %cmp1
+  %cond = select i1 %cmp3, <4 x double> %a1, <4 x double> %a2
+  ret <4 x double> %cond
+
+; CHECK-LABEL: @testqv4doubleuge
+; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
+; CHECK: bc 4, 2, .LBB[[BB:[0-9_]+]]
+; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
+; CHECK: bc 12, 2, .LBB[[BB]]
+; CHECK: qvfmr 5, 6
+; CHECK: .LBB[[BB]]:
+; CHECK: qvfmr 1, 5
+; CHECK: blr
+}
+
+define <4 x double> @testqv4doublesgt(float %c1, float %c2, float %c3, float %c4, <4 x double> %a1, <4 x double> %a2) #1 {
+entry:
+  %cmp1 = fcmp oeq float %c3, %c4
+  %cmp3tmp = fcmp oeq float %c1, %c2
+  %cmp3 = icmp sgt i1 %cmp3tmp, %cmp1
+  %cond = select i1 %cmp3, <4 x double> %a1, <4 x double> %a2
+  ret <4 x double> %cond
+
+; CHECK-LABEL: @testqv4doublesgt
+; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
+; CHECK: bc 4, 2, .LBB[[BB1:[0-9_]+]]
+; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
+; CHECK: bc 4, 2, .LBB[[BB2:[0-9_]+]]
+; CHECK: .LBB[[BB1]]:
+; CHECK: qvfmr 5, 6
+; CHECK: .LBB[[BB2]]:
+; CHECK: qvfmr 1, 5
+; CHECK: blr
+}
+
+define <4 x double> @testqv4doubleugt(float %c1, float %c2, float %c3, float %c4, <4 x double> %a1, <4 x double> %a2) #1 {
+entry:
+  %cmp1 = fcmp oeq float %c3, %c4
+  %cmp3tmp = fcmp oeq float %c1, %c2
+  %cmp3 = icmp ugt i1 %cmp3tmp, %cmp1
+  %cond = select i1 %cmp3, <4 x double> %a1, <4 x double> %a2
+  ret <4 x double> %cond
+
+; CHECK-LABEL: @testqv4doubleugt
+; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
+; CHECK: bc 12, 2, .LBB[[BB1:[0-9_]+]]
+; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
+; CHECK: bc 12, 2, .LBB[[BB2:[0-9_]+]]
+; CHECK: .LBB[[BB1]]:
+; CHECK: qvfmr 5, 6
+; CHECK: .LBB[[BB2]]:
+; CHECK: qvfmr 1, 5
+; CHECK: blr
+}
+
+define <4 x double> @testqv4doublene(float %c1, float %c2, float %c3, float %c4, <4 x double> %a1, <4 x double> %a2) #1 {
+entry:
+  %cmp1 = fcmp oeq float %c3, %c4
+  %cmp3tmp = fcmp oeq float %c1, %c2
+  %cmp3 = icmp ne i1 %cmp3tmp, %cmp1
+  %cond = select i1 %cmp3, <4 x double> %a1, <4 x double> %a2
+  ret <4 x double> %cond
+
+; CHECK-LABEL: @testqv4doublene
+; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
+; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
+; CHECK: crxor [[REG1:[0-9]+]], {{[0-9]+}}, {{[0-9]+}}
+; CHECK: bclr 12, [[REG1]], 0
+; CHECK: qvfmr 1, 6
+; CHECK: blr
+}
+
+define <4 x float> @testqv4floatslt(float %c1, float %c2, float %c3, float %c4, <4 x float> %a1, <4 x float> %a2) #1 {
+entry:
+  %cmp1 = fcmp oeq float %c3, %c4
+  %cmp3tmp = fcmp oeq float %c1, %c2
+  %cmp3 = icmp slt i1 %cmp3tmp, %cmp1
+  %cond = select i1 %cmp3, <4 x float> %a1, <4 x float> %a2
+  ret <4 x float> %cond
+
+; CHECK-LABEL: @testqv4floatslt
+; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
+; CHECK: bc 12, 2, .LBB[[BB1:[0-9_]+]]
+; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
+; CHECK: bc 12, 2, .LBB[[BB2:[0-9_]+]]
+; CHECK: .LBB[[BB1]]:
+; CHECK: qvfmr 5, 6
+; CHECK: .LBB[[BB2]]:
+; CHECK: qvfmr 1, 5
+; CHECK: blr
+}
+
+define <4 x float> @testqv4floatult(float %c1, float %c2, float %c3, float %c4, <4 x float> %a1, <4 x float> %a2) #1 {
+entry:
+  %cmp1 = fcmp oeq float %c3, %c4
+  %cmp3tmp = fcmp oeq float %c1, %c2
+  %cmp3 = icmp ult i1 %cmp3tmp, %cmp1
+  %cond = select i1 %cmp3, <4 x float> %a1, <4 x float> %a2
+  ret <4 x float> %cond
+
+; CHECK-LABEL: @testqv4floatult
+; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
+; CHECK: bc 4, 2, .LBB[[BB1:[0-9_]+]]
+; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
+; CHECK: bc 4, 2, .LBB[[BB2:[0-9_]+]]
+; CHECK: .LBB[[BB1]]:
+; CHECK: qvfmr 5, 6
+; CHECK: .LBB[[BB2]]:
+; CHECK: qvfmr 1, 5
+; CHECK: blr
+}
+
+define <4 x float> @testqv4floatsle(float %c1, float %c2, float %c3, float %c4, <4 x float> %a1, <4 x float> %a2) #1 {
+entry:
+  %cmp1 = fcmp oeq float %c3, %c4
+  %cmp3tmp = fcmp oeq float %c1, %c2
+  %cmp3 = icmp sle i1 %cmp3tmp, %cmp1
+  %cond = select i1 %cmp3, <4 x float> %a1, <4 x float> %a2
+  ret <4 x float> %cond
+
+; CHECK-LABEL: @testqv4floatsle
+; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
+; CHECK: bc 4, 2, .LBB[[BB:[0-9_]+]]
+; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
+; CHECK: bc 12, 2, .LBB[[BB]]
+; CHECK: qvfmr 5, 6
+; CHECK: .LBB[[BB]]:
+; CHECK: qvfmr 1, 5
+; CHECK: blr
+}
+
+define <4 x float> @testqv4floatule(float %c1, float %c2, float %c3, float %c4, <4 x float> %a1, <4 x float> %a2) #1 {
+entry:
+  %cmp1 = fcmp oeq float %c3, %c4
+  %cmp3tmp = fcmp oeq float %c1, %c2
+  %cmp3 = icmp ule i1 %cmp3tmp, %cmp1
+  %cond = select i1 %cmp3, <4 x float> %a1, <4 x float> %a2
+  ret <4 x float> %cond
+
+; CHECK-LABEL: @testqv4floatule
+; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
+; CHECK: bc 12, 2, .LBB[[BB:[0-9_]+]]
+; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
+; CHECK: bc 4, 2, .LBB[[BB]]
+; CHECK: qvfmr 5, 6
+; CHECK: .LBB[[BB]]:
+; CHECK: qvfmr 1, 5
+; CHECK: blr
+}
+
+define <4 x float> @testqv4floateq(float %c1, float %c2, float %c3, float %c4, <4 x float> %a1, <4 x float> %a2) #1 {
+entry:
+  %cmp1 = fcmp oeq float %c3, %c4
+  %cmp3tmp = fcmp oeq float %c1, %c2
+  %cmp3 = icmp eq i1 %cmp3tmp, %cmp1
+  %cond = select i1 %cmp3, <4 x float> %a1, <4 x float> %a2
+  ret <4 x float> %cond
+
+; CHECK-LABEL: @testqv4floateq
+; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
+; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
+; CHECK: creqv [[REG1:[0-9]+]], {{[0-9]+}}, {{[0-9]+}}
+; CHECK: bclr 12, [[REG1]], 0
+; CHECK: qvfmr 1, 6
+; CHECK: blr
+}
+
+define <4 x float> @testqv4floatsge(float %c1, float %c2, float %c3, float %c4, <4 x float> %a1, <4 x float> %a2) #1 {
+entry:
+  %cmp1 = fcmp oeq float %c3, %c4
+  %cmp3tmp = fcmp oeq float %c1, %c2
+  %cmp3 = icmp sge i1 %cmp3tmp, %cmp1
+  %cond = select i1 %cmp3, <4 x float> %a1, <4 x float> %a2
+  ret <4 x float> %cond
+
+; CHECK-LABEL: @testqv4floatsge
+; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
+; CHECK: bc 12, 2, .LBB[[BB:[0-9_]+]]
+; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
+; CHECK: bc 4, 2, .LBB[[BB]]
+; CHECK: qvfmr 5, 6
+; CHECK: .LBB[[BB]]:
+; CHECK: qvfmr 1, 5
+; CHECK: blr
+}
+
+define <4 x float> @testqv4floatuge(float %c1, float %c2, float %c3, float %c4, <4 x float> %a1, <4 x float> %a2) #1 {
+entry:
+  %cmp1 = fcmp oeq float %c3, %c4
+  %cmp3tmp = fcmp oeq float %c1, %c2
+  %cmp3 = icmp uge i1 %cmp3tmp, %cmp1
+  %cond = select i1 %cmp3, <4 x float> %a1, <4 x float> %a2
+  ret <4 x float> %cond
+
+; CHECK-LABEL: @testqv4floatuge
+; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
+; CHECK: bc 4, 2, .LBB[[BB:[0-9_]+]]
+; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
+; CHECK: bc 12, 2, .LBB[[BB]]
+; CHECK: qvfmr 5, 6
+; CHECK: .LBB[[BB]]:
+; CHECK: qvfmr 1, 5
+; CHECK: blr
+}
+
+define <4 x float> @testqv4floatsgt(float %c1, float %c2, float %c3, float %c4, <4 x float> %a1, <4 x float> %a2) #1 {
+entry:
+  %cmp1 = fcmp oeq float %c3, %c4
+  %cmp3tmp = fcmp oeq float %c1, %c2
+  %cmp3 = icmp sgt i1 %cmp3tmp, %cmp1
+  %cond = select i1 %cmp3, <4 x float> %a1, <4 x float> %a2
+  ret <4 x float> %cond
+
+; CHECK-LABEL: @testqv4floatsgt
+; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
+; CHECK: bc 4, 2, .LBB[[BB1:[0-9_]+]]
+; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
+; CHECK: bc 4, 2, .LBB[[BB2:[0-9_]+]]
+; CHECK: .LBB[[BB1]]:
+; CHECK: qvfmr 5, 6
+; CHECK: .LBB[[BB2]]:
+; CHECK: qvfmr 1, 5
+; CHECK: blr
+}
+
+define <4 x float> @testqv4floatugt(float %c1, float %c2, float %c3, float %c4, <4 x float> %a1, <4 x float> %a2) #1 {
+entry:
+  %cmp1 = fcmp oeq float %c3, %c4
+  %cmp3tmp = fcmp oeq float %c1, %c2
+  %cmp3 = icmp ugt i1 %cmp3tmp, %cmp1
+  %cond = select i1 %cmp3, <4 x float> %a1, <4 x float> %a2
+  ret <4 x float> %cond
+
+; CHECK-LABEL: @testqv4floatugt
+; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
+; CHECK: bc 12, 2, .LBB[[BB1:[0-9_]+]]
+; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
+; CHECK: bc 12, 2, .LBB[[BB2:[0-9_]+]]
+; CHECK: .LBB[[BB1]]:
+; CHECK: qvfmr 5, 6
+; CHECK: .LBB[[BB2]]:
+; CHECK: qvfmr 1, 5
+; CHECK: blr
+}
+
+define <4 x float> @testqv4floatne(float %c1, float %c2, float %c3, float %c4, <4 x float> %a1, <4 x float> %a2) #1 {
+entry:
+  %cmp1 = fcmp oeq float %c3, %c4
+  %cmp3tmp = fcmp oeq float %c1, %c2
+  %cmp3 = icmp ne i1 %cmp3tmp, %cmp1
+  %cond = select i1 %cmp3, <4 x float> %a1, <4 x float> %a2
+  ret <4 x float> %cond
+
+; CHECK-LABEL: @testqv4floatne
+; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
+; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
+; CHECK: crxor [[REG1:[0-9]+]], {{[0-9]+}}, {{[0-9]+}}
+; CHECK: bclr 12, [[REG1]], 0
+; CHECK: qvfmr 1, 6
+; CHECK: blr
+}
+
+define <4 x i1> @testqv4i1slt(float %c1, float %c2, float %c3, float %c4, <4 x i1> %a1, <4 x i1> %a2) #1 {
+entry:
+  %cmp1 = fcmp oeq float %c3, %c4
+  %cmp3tmp = fcmp oeq float %c1, %c2
+  %cmp3 = icmp slt i1 %cmp3tmp, %cmp1
+  %cond = select i1 %cmp3, <4 x i1> %a1, <4 x i1> %a2
+  ret <4 x i1> %cond
+
+; CHECK-LABEL: @testqv4i1slt
+; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
+; CHECK: bc 12, 2, .LBB[[BB1:[0-9_]+]]
+; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
+; CHECK: bc 12, 2, .LBB[[BB2:[0-9_]+]]
+; CHECK: .LBB[[BB1]]:
+; CHECK: qvfmr 5, 6
+; CHECK: .LBB[[BB2]]:
+; CHECK: qvfmr 1, 5
+; CHECK: blr
+}
+
+define <4 x i1> @testqv4i1ult(float %c1, float %c2, float %c3, float %c4, <4 x i1> %a1, <4 x i1> %a2) #1 {
+entry:
+  %cmp1 = fcmp oeq float %c3, %c4
+  %cmp3tmp = fcmp oeq float %c1, %c2
+  %cmp3 = icmp ult i1 %cmp3tmp, %cmp1
+  %cond = select i1 %cmp3, <4 x i1> %a1, <4 x i1> %a2
+  ret <4 x i1> %cond
+
+; CHECK-LABEL: @testqv4i1ult
+; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
+; CHECK: bc 4, 2, .LBB[[BB1:[0-9_]+]]
+; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
+; CHECK: bc 4, 2, .LBB[[BB2:[0-9_]+]]
+; CHECK: .LBB[[BB1]]:
+; CHECK: qvfmr 5, 6
+; CHECK: .LBB[[BB2]]:
+; CHECK: qvfmr 1, 5
+; CHECK: blr
+}
+
+define <4 x i1> @testqv4i1sle(float %c1, float %c2, float %c3, float %c4, <4 x i1> %a1, <4 x i1> %a2) #1 {
+entry:
+  %cmp1 = fcmp oeq float %c3, %c4
+  %cmp3tmp = fcmp oeq float %c1, %c2
+  %cmp3 = icmp sle i1 %cmp3tmp, %cmp1
+  %cond = select i1 %cmp3, <4 x i1> %a1, <4 x i1> %a2
+  ret <4 x i1> %cond
+
+; CHECK-LABEL: @testqv4i1sle
+; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
+; CHECK: bc 4, 2, .LBB[[BB:[0-9_]+]]
+; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
+; CHECK: bc 12, 2, .LBB[[BB]]
+; CHECK: qvfmr 5, 6
+; CHECK: .LBB[[BB]]:
+; CHECK: qvfmr 1, 5
+; CHECK: blr
+}
+
+define <4 x i1> @testqv4i1ule(float %c1, float %c2, float %c3, float %c4, <4 x i1> %a1, <4 x i1> %a2) #1 {
+entry:
+  %cmp1 = fcmp oeq float %c3, %c4
+  %cmp3tmp = fcmp oeq float %c1, %c2
+  %cmp3 = icmp ule i1 %cmp3tmp, %cmp1
+  %cond = select i1 %cmp3, <4 x i1> %a1, <4 x i1> %a2
+  ret <4 x i1> %cond
+
+; CHECK-LABEL: @testqv4i1ule
+; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
+; CHECK: bc 12, 2, .LBB[[BB:[0-9_]+]]
+; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
+; CHECK: bc 4, 2, .LBB[[BB]]
+; CHECK: qvfmr 5, 6
+; CHECK: .LBB[[BB]]:
+; CHECK: qvfmr 1, 5
+; CHECK: blr
+}
+
+define <4 x i1> @testqv4i1eq(float %c1, float %c2, float %c3, float %c4, <4 x i1> %a1, <4 x i1> %a2) #1 {
+entry:
+  %cmp1 = fcmp oeq float %c3, %c4
+  %cmp3tmp = fcmp oeq float %c1, %c2
+  %cmp3 = icmp eq i1 %cmp3tmp, %cmp1
+  %cond = select i1 %cmp3, <4 x i1> %a1, <4 x i1> %a2
+  ret <4 x i1> %cond
+
+; CHECK-LABEL: @testqv4i1eq
+; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
+; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
+; CHECK: creqv [[REG1:[0-9]+]], {{[0-9]+}}, {{[0-9]+}}
+; CHECK: bclr 12, [[REG1]], 0
+; CHECK: qvfmr 1, 6
+; CHECK: blr
+}
+
+define <4 x i1> @testqv4i1sge(float %c1, float %c2, float %c3, float %c4, <4 x i1> %a1, <4 x i1> %a2) #1 {
+entry:
+  %cmp1 = fcmp oeq float %c3, %c4
+  %cmp3tmp = fcmp oeq float %c1, %c2
+  %cmp3 = icmp sge i1 %cmp3tmp, %cmp1
+  %cond = select i1 %cmp3, <4 x i1> %a1, <4 x i1> %a2
+  ret <4 x i1> %cond
+
+; CHECK-LABEL: @testqv4i1sge
+; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
+; CHECK: bc 12, 2, .LBB[[BB:[0-9_]+]]
+; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
+; CHECK: bc 4, 2, .LBB[[BB]]
+; CHECK: qvfmr 5, 6
+; CHECK: .LBB[[BB]]:
+; CHECK: qvfmr 1, 5
+; CHECK: blr
+}
+
+define <4 x i1> @testqv4i1uge(float %c1, float %c2, float %c3, float %c4, <4 x i1> %a1, <4 x i1> %a2) #1 {
+entry:
+  %cmp1 = fcmp oeq float %c3, %c4
+  %cmp3tmp = fcmp oeq float %c1, %c2
+  %cmp3 = icmp uge i1 %cmp3tmp, %cmp1
+  %cond = select i1 %cmp3, <4 x i1> %a1, <4 x i1> %a2
+  ret <4 x i1> %cond
+
+; CHECK-LABEL: @testqv4i1uge
+; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
+; CHECK: bc 4, 2, .LBB[[BB:[0-9_]+]]
+; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
+; CHECK: bc 12, 2, .LBB[[BB]]
+; CHECK: qvfmr 5, 6
+; CHECK: .LBB[[BB]]:
+; CHECK: qvfmr 1, 5
+; CHECK: blr
+}
+
+define <4 x i1> @testqv4i1sgt(float %c1, float %c2, float %c3, float %c4, <4 x i1> %a1, <4 x i1> %a2) #1 {
+entry:
+  %cmp1 = fcmp oeq float %c3, %c4
+  %cmp3tmp = fcmp oeq float %c1, %c2
+  %cmp3 = icmp sgt i1 %cmp3tmp, %cmp1
+  %cond = select i1 %cmp3, <4 x i1> %a1, <4 x i1> %a2
+  ret <4 x i1> %cond
+
+; CHECK-LABEL: @testqv4i1sgt
+; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
+; CHECK: bc 4, 2, .LBB[[BB1:[0-9_]+]]
+; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
+; CHECK: bc 4, 2, .LBB[[BB2:[0-9_]+]]
+; CHECK: .LBB[[BB1]]:
+; CHECK: qvfmr 5, 6
+; CHECK: .LBB[[BB2]]:
+; CHECK: qvfmr 1, 5
+; CHECK: blr
+}
+
+define <4 x i1> @testqv4i1ugt(float %c1, float %c2, float %c3, float %c4, <4 x i1> %a1, <4 x i1> %a2) #1 {
+entry:
+  %cmp1 = fcmp oeq float %c3, %c4
+  %cmp3tmp = fcmp oeq float %c1, %c2
+  %cmp3 = icmp ugt i1 %cmp3tmp, %cmp1
+  %cond = select i1 %cmp3, <4 x i1> %a1, <4 x i1> %a2
+  ret <4 x i1> %cond
+
+; CHECK-LABEL: @testqv4i1ugt
+; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
+; CHECK: bc 12, 2, .LBB[[BB1:[0-9_]+]]
+; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
+; CHECK: bc 12, 2, .LBB[[BB2:[0-9_]+]]
+; CHECK: .LBB[[BB1]]:
+; CHECK: qvfmr 5, 6
+; CHECK: .LBB[[BB2]]:
+; CHECK: qvfmr 1, 5
+; CHECK: blr
+}
+
+define <4 x i1> @testqv4i1ne(float %c1, float %c2, float %c3, float %c4, <4 x i1> %a1, <4 x i1> %a2) #1 {
+entry:
+  %cmp1 = fcmp oeq float %c3, %c4
+  %cmp3tmp = fcmp oeq float %c1, %c2
+  %cmp3 = icmp ne i1 %cmp3tmp, %cmp1
+  %cond = select i1 %cmp3, <4 x i1> %a1, <4 x i1> %a2
+  ret <4 x i1> %cond
+
+; CHECK-LABEL: @testqv4i1ne
+; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
+; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
+; CHECK: crxor [[REG1:[0-9]+]], {{[0-9]+}}, {{[0-9]+}}
+; CHECK: bclr 12, [[REG1]], 0
+; CHECK: qvfmr 1, 6
+; CHECK: blr
+}
+
 attributes #0 = { nounwind readnone "target-cpu"="pwr7" }
+attributes #1 = { nounwind readnone "target-cpu"="a2q" }
 
diff --git a/llvm/test/CodeGen/PowerPC/selectiondag-extload-computeknownbits.ll b/llvm/test/CodeGen/PowerPC/selectiondag-extload-computeknownbits.ll
index 73fce78c33aa7..53d17d8668270 100644
--- a/llvm/test/CodeGen/PowerPC/selectiondag-extload-computeknownbits.ll
+++ b/llvm/test/CodeGen/PowerPC/selectiondag-extload-computeknownbits.ll
@@ -1,4 +1,4 @@
-; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux < %s
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64-bgq-linux < %s
 
 ; Check that llc does not crash due to an illegal APInt operation
 
diff --git a/llvm/test/CodeGen/PowerPC/setcr_bc.mir b/llvm/test/CodeGen/PowerPC/setcr_bc.mir
index 564ee7d45957b..e9d81da681fcc 100644
--- a/llvm/test/CodeGen/PowerPC/setcr_bc.mir
+++ b/llvm/test/CodeGen/PowerPC/setcr_bc.mir
@@ -32,8 +32,8 @@
     ret i32 %call2.i.sink
   }
   
-  attributes #0 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector" "unsafe-fp-math"="false" "use-soft-float"="false" }
-  attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector" "unsafe-fp-math"="false" "use-soft-float"="false" }
+  attributes #0 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+  attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 ...
 ---
diff --git a/llvm/test/CodeGen/PowerPC/setcr_bc2.mir b/llvm/test/CodeGen/PowerPC/setcr_bc2.mir
index 513cb85e1580a..582284d6d0a59 100644
--- a/llvm/test/CodeGen/PowerPC/setcr_bc2.mir
+++ b/llvm/test/CodeGen/PowerPC/setcr_bc2.mir
@@ -32,8 +32,8 @@
     ret i32 %call2.i.sink
   }
   
-  attributes #0 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector" "unsafe-fp-math"="false" "use-soft-float"="false" }
-  attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector" "unsafe-fp-math"="false" "use-soft-float"="false" }
+  attributes #0 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+  attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 ...
 ---
diff --git a/llvm/test/CodeGen/PowerPC/stwu-sched.ll b/llvm/test/CodeGen/PowerPC/stwu-sched.ll
index 36afaf84a296b..0afd2ee406894 100644
--- a/llvm/test/CodeGen/PowerPC/stwu-sched.ll
+++ b/llvm/test/CodeGen/PowerPC/stwu-sched.ll
@@ -58,7 +58,7 @@ define void @initCombList(%0* nocapture, i32 signext) local_unnamed_addr #0 {
   ret void
 }
 
-attributes #0 = { norecurse nounwind writeonly "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse nounwind writeonly "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.ident = !{!0}
 
diff --git a/llvm/test/CodeGen/PowerPC/unal-vec-ldst.ll b/llvm/test/CodeGen/PowerPC/unal-vec-ldst.ll
index 79a368dd095ac..497add38e0444 100644
--- a/llvm/test/CodeGen/PowerPC/unal-vec-ldst.ll
+++ b/llvm/test/CodeGen/PowerPC/unal-vec-ldst.ll
@@ -327,6 +327,72 @@ entry:
 
 }
 
+define <4 x float> @test_l_qv4float(<4 x float>* %p) #1 {
+; CHECK-LABEL: test_l_qv4float:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li 4, 15
+; CHECK-NEXT:    qvlpclsx 0, 0, 3
+; CHECK-NEXT:    qvlfsx 1, 3, 4
+; CHECK-NEXT:    qvlfsx 2, 0, 3
+; CHECK-NEXT:    qvfperm 1, 2, 1, 0
+; CHECK-NEXT:    blr
+entry:
+  %r = load <4 x float>, <4 x float>* %p, align 4
+  ret <4 x float> %r
+
+}
+
+define <8 x float> @test_l_qv8float(<8 x float>* %p) #1 {
+; CHECK-LABEL: test_l_qv8float:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li 4, 31
+; CHECK-NEXT:    qvlpclsx 1, 0, 3
+; CHECK-NEXT:    qvlfsx 0, 3, 4
+; CHECK-NEXT:    li 4, 16
+; CHECK-NEXT:    qvlfsx 3, 3, 4
+; CHECK-NEXT:    qvlfsx 4, 0, 3
+; CHECK-NEXT:    qvfperm 2, 3, 0, 1
+; CHECK-NEXT:    qvfperm 1, 4, 3, 1
+; CHECK-NEXT:    blr
+entry:
+  %r = load <8 x float>, <8 x float>* %p, align 4
+  ret <8 x float> %r
+
+}
+
+define <4 x double> @test_l_qv4double(<4 x double>* %p) #1 {
+; CHECK-LABEL: test_l_qv4double:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li 4, 31
+; CHECK-NEXT:    qvlpcldx 0, 0, 3
+; CHECK-NEXT:    qvlfdx 1, 3, 4
+; CHECK-NEXT:    qvlfdx 2, 0, 3
+; CHECK-NEXT:    qvfperm 1, 2, 1, 0
+; CHECK-NEXT:    blr
+entry:
+  %r = load <4 x double>, <4 x double>* %p, align 8
+  ret <4 x double> %r
+
+}
+
+define <8 x double> @test_l_qv8double(<8 x double>* %p) #1 {
+; CHECK-LABEL: test_l_qv8double:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li 4, 63
+; CHECK-NEXT:    qvlpcldx 1, 0, 3
+; CHECK-NEXT:    qvlfdx 0, 3, 4
+; CHECK-NEXT:    li 4, 32
+; CHECK-NEXT:    qvlfdx 3, 3, 4
+; CHECK-NEXT:    qvlfdx 4, 0, 3
+; CHECK-NEXT:    qvfperm 2, 3, 0, 1
+; CHECK-NEXT:    qvfperm 1, 4, 3, 1
+; CHECK-NEXT:    blr
+entry:
+  %r = load <8 x double>, <8 x double>* %p, align 8
+  ret <8 x double> %r
+
+}
+
 define void @test_s_v16i8(<16 x i8>* %p, <16 x i8> %v) #0 {
 ; CHECK-LABEL: test_s_v16i8:
 ; CHECK:       # %bb.0: # %entry
@@ -471,6 +537,89 @@ entry:
 
 }
 
+define void @test_s_qv4float(<4 x float>* %p, <4 x float> %v) #1 {
+; CHECK-LABEL: test_s_qv4float:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    qvesplati 0, 1, 3
+; CHECK-NEXT:    stfs 1, 0(3)
+; CHECK-NEXT:    stfs 0, 12(3)
+; CHECK-NEXT:    qvesplati 0, 1, 2
+; CHECK-NEXT:    qvesplati 1, 1, 1
+; CHECK-NEXT:    stfs 0, 8(3)
+; CHECK-NEXT:    stfs 1, 4(3)
+; CHECK-NEXT:    blr
+entry:
+  store <4 x float> %v, <4 x float>* %p, align 4
+  ret void
+
+}
+
+define void @test_s_qv8float(<8 x float>* %p, <8 x float> %v) #1 {
+; CHECK-LABEL: test_s_qv8float:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    qvesplati 0, 2, 3
+; CHECK-NEXT:    stfs 2, 16(3)
+; CHECK-NEXT:    stfs 0, 28(3)
+; CHECK-NEXT:    qvesplati 0, 2, 2
+; CHECK-NEXT:    qvesplati 2, 2, 1
+; CHECK-NEXT:    stfs 1, 0(3)
+; CHECK-NEXT:    stfs 0, 24(3)
+; CHECK-NEXT:    qvesplati 0, 1, 3
+; CHECK-NEXT:    stfs 2, 20(3)
+; CHECK-NEXT:    qvesplati 2, 1, 2
+; CHECK-NEXT:    qvesplati 1, 1, 1
+; CHECK-NEXT:    stfs 0, 12(3)
+; CHECK-NEXT:    stfs 2, 8(3)
+; CHECK-NEXT:    stfs 1, 4(3)
+; CHECK-NEXT:    blr
+entry:
+  store <8 x float> %v, <8 x float>* %p, align 4
+  ret void
+
+}
+
+define void @test_s_qv4double(<4 x double>* %p, <4 x double> %v) #1 {
+; CHECK-LABEL: test_s_qv4double:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    qvesplati 0, 1, 3
+; CHECK-NEXT:    stfd 1, 0(3)
+; CHECK-NEXT:    stfd 0, 24(3)
+; CHECK-NEXT:    qvesplati 0, 1, 2
+; CHECK-NEXT:    qvesplati 1, 1, 1
+; CHECK-NEXT:    stfd 0, 16(3)
+; CHECK-NEXT:    stfd 1, 8(3)
+; CHECK-NEXT:    blr
+entry:
+  store <4 x double> %v, <4 x double>* %p, align 8
+  ret void
+
+}
+
+define void @test_s_qv8double(<8 x double>* %p, <8 x double> %v) #1 {
+; CHECK-LABEL: test_s_qv8double:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    qvesplati 0, 2, 3
+; CHECK-NEXT:    stfd 2, 32(3)
+; CHECK-NEXT:    stfd 0, 56(3)
+; CHECK-NEXT:    qvesplati 0, 2, 2
+; CHECK-NEXT:    qvesplati 2, 2, 1
+; CHECK-NEXT:    stfd 1, 0(3)
+; CHECK-NEXT:    stfd 0, 48(3)
+; CHECK-NEXT:    qvesplati 0, 1, 3
+; CHECK-NEXT:    stfd 2, 40(3)
+; CHECK-NEXT:    qvesplati 2, 1, 2
+; CHECK-NEXT:    qvesplati 1, 1, 1
+; CHECK-NEXT:    stfd 0, 24(3)
+; CHECK-NEXT:    stfd 2, 16(3)
+; CHECK-NEXT:    stfd 1, 8(3)
+; CHECK-NEXT:    blr
+entry:
+  store <8 x double> %v, <8 x double>* %p, align 8
+  ret void
+
+}
+
 attributes #0 = { nounwind "target-cpu"="pwr7" }
+attributes #1 = { nounwind "target-cpu"="a2q" }
 attributes #2 = { nounwind "target-cpu"="pwr8" }
 
diff --git a/llvm/test/CodeGen/PowerPC/uwtables.ll b/llvm/test/CodeGen/PowerPC/uwtables.ll
index e302934ab8d6b..7523d04d73d38 100644
--- a/llvm/test/CodeGen/PowerPC/uwtables.ll
+++ b/llvm/test/CodeGen/PowerPC/uwtables.ll
@@ -47,5 +47,5 @@ declare i32 @__gxx_personality_v0(...)
 declare void @__cxa_call_unexpected(i8*) local_unnamed_addr
 
 
-attributes #0 = { noreturn nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noreturn nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
diff --git a/llvm/test/Instrumentation/MemorySanitizer/PowerPC/vararg-ppc64.ll b/llvm/test/Instrumentation/MemorySanitizer/PowerPC/vararg-ppc64.ll
index 33f3d82c3683d..36da7add88015 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/PowerPC/vararg-ppc64.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/PowerPC/vararg-ppc64.ll
@@ -55,6 +55,21 @@ define i32 @bar2() {
 ; CHECK: store <2 x i64> zeroinitializer, <2 x i64>* inttoptr (i64 add (i64 ptrtoint ([100 x i64]* @__msan_va_arg_tls to i64), i64 8) to <2 x i64>*), align 8
 ; CHECK: store {{.*}} 24, {{.*}} @__msan_va_arg_overflow_size_tls
 
+; Check QPX vector argument.
+define i32 @bar3() "target-features"="+qpx" {
+  %1 = call i32 (i32, ...) @foo(i32 0, i32 1, i32 2, <4 x double> <double 1.0, double 2.0, double 3.0, double 4.0>)
+  ret i32 %1
+}
+
+; That one is even stranger: the parameter save area starts at offset 48 from
+; (32-byte aligned) stack pointer, the vector parameter is at 96 bytes from
+; the stack pointer, so its offset from parameter save area is misaligned.
+; CHECK-LABEL: @bar3
+; CHECK: store i32 0, i32* inttoptr (i64 add (i64 ptrtoint ([100 x i64]* @__msan_va_arg_tls to i64), i64 4) to i32*), align 8
+; CHECK: store i32 0, i32* inttoptr (i64 add (i64 ptrtoint ([100 x i64]* @__msan_va_arg_tls to i64), i64 12) to i32*), align 8
+; CHECK: store <4 x i64> zeroinitializer, <4 x i64>* inttoptr (i64 add (i64 ptrtoint ([100 x i64]* @__msan_va_arg_tls to i64), i64 40) to <4 x i64>*), align 8
+; CHECK: store {{.*}} 72, {{.*}} @__msan_va_arg_overflow_size_tls
+
 ; Check i64 array.
 define i32 @bar4() {
   %1 = call i32 (i32, ...) @foo(i32 0, [2 x i64] [i64 1, i64 2])
diff --git a/llvm/test/MC/Disassembler/PowerPC/qpx.txt b/llvm/test/MC/Disassembler/PowerPC/qpx.txt
new file mode 100644
index 0000000000000..00e598bd4356e
--- /dev/null
+++ b/llvm/test/MC/Disassembler/PowerPC/qpx.txt
@@ -0,0 +1,371 @@
+# RUN: llvm-mc --disassemble %s -triple powerpc64-bgq-linux -mcpu=a2q | FileCheck %s
+
+# CHECK: qvfabs 3, 5
+0x10 0x60 0x2a 0x10
+
+# CHECK: qvfadd 3, 4, 5
+0x10 0x64 0x28 0x2a
+
+# CHECK: qvfadds 3, 4, 5
+0x00 0x64 0x28 0x2a
+
+# CHECK: qvfandc 3, 4, 5
+0x10 0x64 0x2a 0x08
+
+# CHECK: qvfand 3, 4, 5
+0x10 0x64 0x28 0x88
+
+# CHECK: qvfcfid 3, 5
+0x10 0x60 0x2e 0x9c
+
+# CHECK: qvfcfids 3, 5
+0x00 0x60 0x2e 0x9c
+
+# CHECK: qvfcfidu 3, 5
+0x10 0x60 0x2f 0x9c
+
+# CHECK: qvfcfidus 3, 5
+0x00 0x60 0x2f 0x9c
+
+# CHECK: qvfclr 3
+0x10 0x63 0x18 0x08
+
+# CHECK: qvfcpsgn 3, 4, 5
+0x10 0x64 0x28 0x10
+
+# CHECK: qvfctfb 3, 4
+0x10 0x64 0x22 0x88
+
+# CHECK: qvfctid 3, 5
+0x10 0x60 0x2e 0x5c
+
+# CHECK: qvfctidu 3, 5
+0x10 0x60 0x2f 0x5c
+
+# CHECK: qvfctiduz 3, 5
+0x10 0x60 0x2f 0x5e
+
+# CHECK: qvfctidz 3, 5
+0x10 0x60 0x2e 0x5e
+
+# CHECK: qvfctiw 3, 5
+0x10 0x60 0x28 0x1c
+
+# CHECK: qvfctiwu 3, 5
+0x10 0x60 0x29 0x1c
+
+# CHECK: qvfctiwuz 3, 5
+0x10 0x60 0x29 0x1e
+
+# CHECK: qvfctiwz 3, 5
+0x10 0x60 0x28 0x1e
+
+# CHECK: qvfequ 3, 4, 5
+0x10 0x64 0x2c 0x88
+
+# CHECK: qvflogical 3, 4, 5, 12
+0x10 0x64 0x2e 0x08
+
+# CHECK: qvfmadd 3, 4, 6, 5
+0x10 0x64 0x29 0xba
+
+# CHECK: qvfmadds 3, 4, 6, 5
+0x00 0x64 0x29 0xba
+
+# CHECK: qvfmr 3, 5
+0x10 0x60 0x28 0x90
+
+# CHECK: qvfmsub 3, 4, 6, 5
+0x10 0x64 0x29 0xb8
+
+# CHECK: qvfmsubs 3, 4, 6, 5
+0x00 0x64 0x29 0xb8
+
+# CHECK: qvfmul 3, 4, 6
+0x10 0x64 0x01 0xb2
+
+# CHECK: qvfmuls 3, 4, 6
+0x00 0x64 0x01 0xb2
+
+# CHECK: qvfnabs 3, 5
+0x10 0x60 0x29 0x10
+
+# CHECK: qvfnand 3, 4, 5
+0x10 0x64 0x2f 0x08
+
+# CHECK: qvfneg 3, 5
+0x10 0x60 0x28 0x50
+
+# CHECK: qvfnmadd 3, 4, 6, 5
+0x10 0x64 0x29 0xbe
+
+# CHECK: qvfnmadds 3, 4, 6, 5
+0x00 0x64 0x29 0xbe
+
+# CHECK: qvfnmsub 3, 4, 6, 5
+0x10 0x64 0x29 0xbc
+
+# CHECK: qvfnmsubs 3, 4, 6, 5
+0x00 0x64 0x29 0xbc
+
+# CHECK: qvfnor 3, 4, 5
+0x10 0x64 0x2c 0x08
+
+# CHECK: qvfnot 3, 4
+0x10 0x64 0x25 0x08
+
+# CHECK: qvforc 3, 4, 5
+0x10 0x64 0x2e 0x88
+
+# CHECK: qvfor 3, 4, 5
+0x10 0x64 0x2b 0x88
+
+# CHECK: qvfperm 3, 4, 5, 6
+0x10 0x64 0x29 0x8c
+
+# CHECK: qvfre 3, 5
+0x10 0x60 0x28 0x30
+
+# CHECK: qvfres 3, 5
+0x00 0x60 0x28 0x30
+
+# CHECK: qvfrim 3, 5
+0x10 0x60 0x2b 0xd0
+
+# CHECK: qvfrin 3, 5
+0x10 0x60 0x2b 0x10
+
+# CHECK: qvfrip 3, 5
+0x10 0x60 0x2b 0x90
+
+# CHECK: qvfriz 3, 5
+0x10 0x60 0x2b 0x50
+
+# CHECK: qvfrsp 3, 5
+0x10 0x60 0x28 0x18
+
+# CHECK: qvfrsqrte 3, 5
+0x10 0x60 0x28 0x34
+
+# CHECK: qvfrsqrtes 3, 5
+0x00 0x60 0x28 0x34
+
+# CHECK: qvfsel 3, 4, 6, 5
+0x10 0x64 0x29 0xae
+
+# CHECK: qvfset 3
+0x10 0x63 0x1f 0x88
+
+# CHECK: qvfsub 3, 4, 5
+0x10 0x64 0x28 0x28
+
+# CHECK: qvfsubs 3, 4, 5
+0x00 0x64 0x28 0x28
+
+# CHECK: qvfxmadd 3, 4, 6, 5
+0x10 0x64 0x29 0x92
+
+# CHECK: qvfxmadds 3, 4, 6, 5
+0x00 0x64 0x29 0x92
+
+# CHECK: qvfxmul 3, 4, 6
+0x10 0x64 0x01 0xa2
+
+# CHECK: qvfxmuls 3, 4, 6
+0x00 0x64 0x01 0xa2
+
+# CHECK: qvfxor 3, 4, 5
+0x10 0x64 0x2b 0x08
+
+# CHECK: qvfxxcpnmadd 3, 4, 6, 5
+0x10 0x64 0x29 0x86
+
+# CHECK: qvfxxcpnmadds 3, 4, 6, 5
+0x00 0x64 0x29 0x86
+
+# CHECK: qvfxxmadd 3, 4, 6, 5
+0x10 0x64 0x29 0x82
+
+# CHECK: qvfxxmadds 3, 4, 6, 5
+0x00 0x64 0x29 0x82
+
+# CHECK: qvfxxnpmadd 3, 4, 6, 5
+0x10 0x64 0x29 0x96
+
+# CHECK: qvfxxnpmadds 3, 4, 6, 5
+0x00 0x64 0x29 0x96
+
+# CHECK: qvlfcduxa 3, 9, 11
+0x7c 0x69 0x58 0xcf
+
+# CHECK: qvlfcdux 3, 9, 11
+0x7c 0x69 0x58 0xce
+
+# CHECK: qvlfcdxa 3, 10, 11
+0x7c 0x6a 0x58 0x8f
+
+# CHECK: qvlfcdx 3, 10, 11
+0x7c 0x6a 0x58 0x8e
+
+# CHECK: qvlfcsuxa 3, 9, 11
+0x7c 0x69 0x58 0x4f
+
+# CHECK: qvlfcsux 3, 9, 11
+0x7c 0x69 0x58 0x4e
+
+# CHECK: qvlfcsxa 3, 10, 11
+0x7c 0x6a 0x58 0x0f
+
+# CHECK: qvlfcsx 3, 10, 11
+0x7c 0x6a 0x58 0x0e
+
+# CHECK: qvlfduxa 3, 9, 11
+0x7c 0x69 0x5c 0xcf
+
+# CHECK: qvlfdux 3, 9, 11
+0x7c 0x69 0x5c 0xce
+
+# CHECK: qvlfdxa 3, 10, 11
+0x7c 0x6a 0x5c 0x8f
+
+# CHECK: qvlfdx 3, 10, 11
+0x7c 0x6a 0x5c 0x8e
+
+# CHECK: qvlfiwaxa 3, 10, 11
+0x7c 0x6a 0x5e 0xcf
+
+# CHECK: qvlfiwax 3, 10, 11
+0x7c 0x6a 0x5e 0xce
+
+# CHECK: qvlfiwzxa 3, 10, 11
+0x7c 0x6a 0x5e 0x8f
+
+# CHECK: qvlfiwzx 3, 10, 11
+0x7c 0x6a 0x5e 0x8e
+
+# CHECK: qvlfsuxa 3, 9, 11
+0x7c 0x69 0x5c 0x4f
+
+# CHECK: qvlfsux 3, 9, 11
+0x7c 0x69 0x5c 0x4e
+
+# CHECK: qvlfsxa 3, 10, 11
+0x7c 0x6a 0x5c 0x0f
+
+# CHECK: qvlfsx 3, 10, 11
+0x7c 0x6a 0x5c 0x0e
+
+# CHECK: qvlpcldx 3, 10, 11
+0x7c 0x6a 0x5c 0x8c
+
+# CHECK: qvlpclsx 3, 10, 11
+0x7c 0x6a 0x5c 0x0c
+
+# CHECK: qvlpcrdx 3, 10, 11
+0x7c 0x6a 0x58 0x8c
+
+# CHECK: qvlpcrsx 3, 10, 11
+0x7c 0x6a 0x58 0x0c
+
+# CHECK: qvstfcduxa 2, 9, 11
+0x7c 0x49 0x59 0xcf
+
+# CHECK: qvstfcduxia 2, 9, 11
+0x7c 0x49 0x59 0xcb
+
+# CHECK: qvstfcduxi 2, 9, 11
+0x7c 0x49 0x59 0xca
+
+# CHECK: qvstfcdux 2, 9, 11
+0x7c 0x49 0x59 0xce
+
+# CHECK: qvstfcdxa 2, 10, 11
+0x7c 0x4a 0x59 0x8f
+
+# CHECK: qvstfcdxia 2, 10, 11
+0x7c 0x4a 0x59 0x8b
+
+# CHECK: qvstfcdxi 2, 10, 11
+0x7c 0x4a 0x59 0x8a
+
+# CHECK: qvstfcdx 2, 10, 11
+0x7c 0x4a 0x59 0x8e
+
+# CHECK: qvstfcsuxa 2, 9, 11
+0x7c 0x49 0x59 0x4f
+
+# CHECK: qvstfcsuxia 2, 9, 11
+0x7c 0x49 0x59 0x4b
+
+# CHECK: qvstfcsuxi 2, 9, 11
+0x7c 0x49 0x59 0x4a
+
+# CHECK: qvstfcsux 2, 9, 11
+0x7c 0x49 0x59 0x4e
+
+# CHECK: qvstfcsxa 2, 10, 11
+0x7c 0x4a 0x59 0x0f
+
+# CHECK: qvstfcsxia 2, 10, 11
+0x7c 0x4a 0x59 0x0b
+
+# CHECK: qvstfcsxi 2, 10, 11
+0x7c 0x4a 0x59 0x0a
+
+# CHECK: qvstfcsx 2, 10, 11
+0x7c 0x4a 0x59 0x0e
+
+# CHECK: qvstfduxa 2, 9, 11
+0x7c 0x49 0x5d 0xcf
+
+# CHECK: qvstfduxia 2, 9, 11
+0x7c 0x49 0x5d 0xcb
+
+# CHECK: qvstfduxi 2, 9, 11
+0x7c 0x49 0x5d 0xca
+
+# CHECK: qvstfdux 2, 9, 11
+0x7c 0x49 0x5d 0xce
+
+# CHECK: qvstfdxa 2, 10, 11
+0x7c 0x4a 0x5d 0x8f
+
+# CHECK: qvstfdxia 2, 10, 11
+0x7c 0x4a 0x5d 0x8b
+
+# CHECK: qvstfdxi 2, 10, 11
+0x7c 0x4a 0x5d 0x8a
+
+# CHECK: qvstfdx 2, 10, 11
+0x7c 0x4a 0x5d 0x8e
+
+# CHECK: qvstfiwxa 2, 10, 11
+0x7c 0x4a 0x5f 0x8f
+
+# CHECK: qvstfiwx 2, 10, 11
+0x7c 0x4a 0x5f 0x8e
+
+# CHECK: qvstfsuxa 2, 9, 11
+0x7c 0x49 0x5d 0x4f
+
+# CHECK: qvstfsuxia 2, 9, 11
+0x7c 0x49 0x5d 0x4b
+
+# CHECK: qvstfsuxi 2, 9, 11
+0x7c 0x49 0x5d 0x4a
+
+# CHECK: qvstfsux 2, 9, 11
+0x7c 0x49 0x5d 0x4e
+
+# CHECK: qvstfsxa 2, 10, 11
+0x7c 0x4a 0x5d 0x0f
+
+# CHECK: qvstfsxia 2, 10, 11
+0x7c 0x4a 0x5d 0x0b
+
+# CHECK: qvstfsxi 2, 10, 11
+0x7c 0x4a 0x5d 0x0a
+
+# CHECK: qvstfsx 2, 10, 11
+0x7c 0x4a 0x5d 0x0e
+
diff --git a/llvm/test/MC/PowerPC/qpx.s b/llvm/test/MC/PowerPC/qpx.s
new file mode 100644
index 0000000000000..a1fb2090f8fff
--- /dev/null
+++ b/llvm/test/MC/PowerPC/qpx.s
@@ -0,0 +1,252 @@
+# RUN: llvm-mc -triple powerpc64-bgq-linux --show-encoding %s | FileCheck %s
+
+# CHECK: qvfabs 3, 5                     # encoding: [0x10,0x60,0x2a,0x10]
+         qvfabs %q3, %q5
+
+# CHECK: qvfabs 3, 5                     # encoding: [0x10,0x60,0x2a,0x10]
+         qvfabs 3, 5
+# CHECK: qvfadd 3, 4, 5                  # encoding: [0x10,0x64,0x28,0x2a]
+         qvfadd 3, 4, 5
+# CHECK: qvfadds 3, 4, 5                 # encoding: [0x00,0x64,0x28,0x2a]
+         qvfadds 3, 4, 5
+# CHECK: qvfandc 3, 4, 5                 # encoding: [0x10,0x64,0x2a,0x08]
+         qvfandc 3, 4, 5
+# CHECK: qvfand 3, 4, 5                  # encoding: [0x10,0x64,0x28,0x88]
+         qvfand 3, 4, 5
+# CHECK: qvfcfid 3, 5                    # encoding: [0x10,0x60,0x2e,0x9c]
+         qvfcfid 3, 5
+# CHECK: qvfcfids 3, 5                   # encoding: [0x00,0x60,0x2e,0x9c]
+         qvfcfids 3, 5
+# CHECK: qvfcfidu 3, 5                   # encoding: [0x10,0x60,0x2f,0x9c]
+         qvfcfidu 3, 5
+# CHECK: qvfcfidus 3, 5                  # encoding: [0x00,0x60,0x2f,0x9c]
+         qvfcfidus 3, 5
+# CHECK: qvfclr 3                        # encoding: [0x10,0x63,0x18,0x08]
+         qvfclr 3
+# CHECK: qvfcpsgn 3, 4, 5                # encoding: [0x10,0x64,0x28,0x10]
+         qvfcpsgn 3, 4, 5
+# CHECK: qvfctfb 3, 4                    # encoding: [0x10,0x64,0x22,0x88]
+         qvfctfb 3, 4
+# CHECK: qvfctid 3, 5                    # encoding: [0x10,0x60,0x2e,0x5c]
+         qvfctid 3, 5
+# CHECK: qvfctidu 3, 5                   # encoding: [0x10,0x60,0x2f,0x5c]
+         qvfctidu 3, 5
+# CHECK: qvfctiduz 3, 5                  # encoding: [0x10,0x60,0x2f,0x5e]
+         qvfctiduz 3, 5
+# CHECK: qvfctidz 3, 5                   # encoding: [0x10,0x60,0x2e,0x5e]
+         qvfctidz 3, 5
+# CHECK: qvfctiw 3, 5                    # encoding: [0x10,0x60,0x28,0x1c]
+         qvfctiw 3, 5
+# CHECK: qvfctiwu 3, 5                   # encoding: [0x10,0x60,0x29,0x1c]
+         qvfctiwu 3, 5
+# CHECK: qvfctiwuz 3, 5                  # encoding: [0x10,0x60,0x29,0x1e]
+         qvfctiwuz 3, 5
+# CHECK: qvfctiwz 3, 5                   # encoding: [0x10,0x60,0x28,0x1e]
+         qvfctiwz 3, 5
+# CHECK: qvfequ 3, 4, 5                  # encoding: [0x10,0x64,0x2c,0x88]
+         qvfequ 3, 4, 5
+# CHECK: qvflogical 3, 4, 5, 12          # encoding: [0x10,0x64,0x2e,0x08]
+         qvflogical 3, 4, 5, 12
+# CHECK: qvfmadd 3, 4, 6, 5              # encoding: [0x10,0x64,0x29,0xba]
+         qvfmadd 3, 4, 6, 5
+# CHECK: qvfmadds 3, 4, 6, 5             # encoding: [0x00,0x64,0x29,0xba]
+         qvfmadds 3, 4, 6, 5
+# CHECK: qvfmr 3, 5                      # encoding: [0x10,0x60,0x28,0x90]
+         qvfmr 3, 5
+# CHECK: qvfmsub 3, 4, 6, 5              # encoding: [0x10,0x64,0x29,0xb8]
+         qvfmsub 3, 4, 6, 5
+# CHECK: qvfmsubs 3, 4, 6, 5             # encoding: [0x00,0x64,0x29,0xb8]
+         qvfmsubs 3, 4, 6, 5
+# CHECK: qvfmul 3, 4, 6                  # encoding: [0x10,0x64,0x01,0xb2]
+         qvfmul 3, 4, 6
+# CHECK: qvfmuls 3, 4, 6                 # encoding: [0x00,0x64,0x01,0xb2]
+         qvfmuls 3, 4, 6
+# CHECK: qvfnabs 3, 5                    # encoding: [0x10,0x60,0x29,0x10]
+         qvfnabs 3, 5
+# CHECK: qvfnand 3, 4, 5                 # encoding: [0x10,0x64,0x2f,0x08]
+         qvfnand 3, 4, 5
+# CHECK: qvfneg 3, 5                     # encoding: [0x10,0x60,0x28,0x50]
+         qvfneg 3, 5
+# CHECK: qvfnmadd 3, 4, 6, 5             # encoding: [0x10,0x64,0x29,0xbe]
+         qvfnmadd 3, 4, 6, 5
+# CHECK: qvfnmadds 3, 4, 6, 5            # encoding: [0x00,0x64,0x29,0xbe]
+         qvfnmadds 3, 4, 6, 5
+# CHECK: qvfnmsub 3, 4, 6, 5             # encoding: [0x10,0x64,0x29,0xbc]
+         qvfnmsub 3, 4, 6, 5
+# CHECK: qvfnmsubs 3, 4, 6, 5            # encoding: [0x00,0x64,0x29,0xbc]
+         qvfnmsubs 3, 4, 6, 5
+# CHECK: qvfnor 3, 4, 5                  # encoding: [0x10,0x64,0x2c,0x08]
+         qvfnor 3, 4, 5
+# CHECK: qvfnot 3, 4                     # encoding: [0x10,0x64,0x25,0x08]
+         qvfnot 3, 4
+# CHECK: qvforc 3, 4, 5                  # encoding: [0x10,0x64,0x2e,0x88]
+         qvforc 3, 4, 5
+# CHECK: qvfor 3, 4, 5                   # encoding: [0x10,0x64,0x2b,0x88]
+         qvfor 3, 4, 5
+# CHECK: qvfperm 3, 4, 5, 6              # encoding: [0x10,0x64,0x29,0x8c]
+         qvfperm 3, 4, 5, 6
+# CHECK: qvfre 3, 5                      # encoding: [0x10,0x60,0x28,0x30]
+         qvfre 3, 5
+# CHECK: qvfres 3, 5                     # encoding: [0x00,0x60,0x28,0x30]
+         qvfres 3, 5
+# CHECK: qvfrim 3, 5                     # encoding: [0x10,0x60,0x2b,0xd0]
+         qvfrim 3, 5
+# CHECK: qvfrin 3, 5                     # encoding: [0x10,0x60,0x2b,0x10]
+         qvfrin 3, 5
+# CHECK: qvfrip 3, 5                     # encoding: [0x10,0x60,0x2b,0x90]
+         qvfrip 3, 5
+# CHECK: qvfriz 3, 5                     # encoding: [0x10,0x60,0x2b,0x50]
+         qvfriz 3, 5
+# CHECK: qvfrsp 3, 5                     # encoding: [0x10,0x60,0x28,0x18]
+         qvfrsp 3, 5
+# CHECK: qvfrsqrte 3, 5                  # encoding: [0x10,0x60,0x28,0x34]
+         qvfrsqrte 3, 5
+# CHECK: qvfrsqrtes 3, 5                 # encoding: [0x00,0x60,0x28,0x34]
+         qvfrsqrtes 3, 5
+# CHECK: qvfsel 3, 4, 6, 5               # encoding: [0x10,0x64,0x29,0xae]
+         qvfsel 3, 4, 6, 5
+# CHECK: qvfset 3                        # encoding: [0x10,0x63,0x1f,0x88]
+         qvfset 3
+# CHECK: qvfsub 3, 4, 5                  # encoding: [0x10,0x64,0x28,0x28]
+         qvfsub 3, 4, 5
+# CHECK: qvfsubs 3, 4, 5                 # encoding: [0x00,0x64,0x28,0x28]
+         qvfsubs 3, 4, 5
+# CHECK: qvfxmadd 3, 4, 6, 5             # encoding: [0x10,0x64,0x29,0x92]
+         qvfxmadd 3, 4, 6, 5
+# CHECK: qvfxmadds 3, 4, 6, 5            # encoding: [0x00,0x64,0x29,0x92]
+         qvfxmadds 3, 4, 6, 5
+# CHECK: qvfxmul 3, 4, 6                 # encoding: [0x10,0x64,0x01,0xa2]
+         qvfxmul 3, 4, 6
+# CHECK: qvfxmuls 3, 4, 6                # encoding: [0x00,0x64,0x01,0xa2]
+         qvfxmuls 3, 4, 6
+# CHECK: qvfxor 3, 4, 5                  # encoding: [0x10,0x64,0x2b,0x08]
+         qvfxor 3, 4, 5
+# CHECK: qvfxxcpnmadd 3, 4, 6, 5         # encoding: [0x10,0x64,0x29,0x86]
+         qvfxxcpnmadd 3, 4, 6, 5
+# CHECK: qvfxxcpnmadds 3, 4, 6, 5        # encoding: [0x00,0x64,0x29,0x86]
+         qvfxxcpnmadds 3, 4, 6, 5
+# CHECK: qvfxxmadd 3, 4, 6, 5            # encoding: [0x10,0x64,0x29,0x82]
+         qvfxxmadd 3, 4, 6, 5
+# CHECK: qvfxxmadds 3, 4, 6, 5           # encoding: [0x00,0x64,0x29,0x82]
+         qvfxxmadds 3, 4, 6, 5
+# CHECK: qvfxxnpmadd 3, 4, 6, 5          # encoding: [0x10,0x64,0x29,0x96]
+         qvfxxnpmadd 3, 4, 6, 5
+# CHECK: qvfxxnpmadds 3, 4, 6, 5         # encoding: [0x00,0x64,0x29,0x96]
+         qvfxxnpmadds 3, 4, 6, 5
+# CHECK: qvlfcduxa 3, 9, 11              # encoding: [0x7c,0x69,0x58,0xcf]
+         qvlfcduxa 3, 9, 11
+# CHECK: qvlfcdux 3, 9, 11               # encoding: [0x7c,0x69,0x58,0xce]
+         qvlfcdux 3, 9, 11
+# CHECK: qvlfcdxa 3, 10, 11              # encoding: [0x7c,0x6a,0x58,0x8f]
+         qvlfcdxa 3, 10, 11
+# CHECK: qvlfcdx 3, 10, 11               # encoding: [0x7c,0x6a,0x58,0x8e]
+         qvlfcdx 3, 10, 11
+# CHECK: qvlfcsuxa 3, 9, 11              # encoding: [0x7c,0x69,0x58,0x4f]
+         qvlfcsuxa 3, 9, 11
+# CHECK: qvlfcsux 3, 9, 11               # encoding: [0x7c,0x69,0x58,0x4e]
+         qvlfcsux 3, 9, 11
+# CHECK: qvlfcsxa 3, 10, 11              # encoding: [0x7c,0x6a,0x58,0x0f]
+         qvlfcsxa 3, 10, 11
+# CHECK: qvlfcsx 3, 10, 11               # encoding: [0x7c,0x6a,0x58,0x0e]
+         qvlfcsx 3, 10, 11
+# CHECK: qvlfduxa 3, 9, 11               # encoding: [0x7c,0x69,0x5c,0xcf]
+         qvlfduxa 3, 9, 11
+# CHECK: qvlfdux 3, 9, 11                # encoding: [0x7c,0x69,0x5c,0xce]
+         qvlfdux 3, 9, 11
+# CHECK: qvlfdxa 3, 10, 11               # encoding: [0x7c,0x6a,0x5c,0x8f]
+         qvlfdxa 3, 10, 11
+# CHECK: qvlfdx 3, 10, 11                # encoding: [0x7c,0x6a,0x5c,0x8e]
+         qvlfdx 3, 10, 11
+# CHECK: qvlfiwaxa 3, 10, 11             # encoding: [0x7c,0x6a,0x5e,0xcf]
+         qvlfiwaxa 3, 10, 11
+# CHECK: qvlfiwax 3, 10, 11              # encoding: [0x7c,0x6a,0x5e,0xce]
+         qvlfiwax 3, 10, 11
+# CHECK: qvlfiwzxa 3, 10, 11             # encoding: [0x7c,0x6a,0x5e,0x8f]
+         qvlfiwzxa 3, 10, 11
+# CHECK: qvlfiwzx 3, 10, 11              # encoding: [0x7c,0x6a,0x5e,0x8e]
+         qvlfiwzx 3, 10, 11
+# CHECK: qvlfsuxa 3, 9, 11               # encoding: [0x7c,0x69,0x5c,0x4f]
+         qvlfsuxa 3, 9, 11
+# CHECK: qvlfsux 3, 9, 11                # encoding: [0x7c,0x69,0x5c,0x4e]
+         qvlfsux 3, 9, 11
+# CHECK: qvlfsxa 3, 10, 11               # encoding: [0x7c,0x6a,0x5c,0x0f]
+         qvlfsxa 3, 10, 11
+# CHECK: qvlfsx 3, 10, 11                # encoding: [0x7c,0x6a,0x5c,0x0e]
+         qvlfsx 3, 10, 11
+# CHECK: qvlpcldx 3, 10, 11              # encoding: [0x7c,0x6a,0x5c,0x8c]
+         qvlpcldx 3, 10, 11
+# CHECK: qvlpclsx 3, 10, 11              # encoding: [0x7c,0x6a,0x5c,0x0c]
+         qvlpclsx 3, 10, 11
+# CHECK: qvlpcrdx 3, 10, 11              # encoding: [0x7c,0x6a,0x58,0x8c]
+         qvlpcrdx 3, 10, 11
+# CHECK: qvlpcrsx 3, 10, 11              # encoding: [0x7c,0x6a,0x58,0x0c]
+         qvlpcrsx 3, 10, 11
+# CHECK: qvstfcduxa 2, 9, 11             # encoding: [0x7c,0x49,0x59,0xcf]
+         qvstfcduxa 2, 9, 11
+# CHECK: qvstfcduxia 2, 9, 11            # encoding: [0x7c,0x49,0x59,0xcb]
+         qvstfcduxia 2, 9, 11
+# CHECK: qvstfcduxi 2, 9, 11             # encoding: [0x7c,0x49,0x59,0xca]
+         qvstfcduxi 2, 9, 11
+# CHECK: qvstfcdux 2, 9, 11              # encoding: [0x7c,0x49,0x59,0xce]
+         qvstfcdux 2, 9, 11
+# CHECK: qvstfcdxa 2, 10, 11             # encoding: [0x7c,0x4a,0x59,0x8f]
+         qvstfcdxa 2, 10, 11
+# CHECK: qvstfcdxia 2, 10, 11            # encoding: [0x7c,0x4a,0x59,0x8b]
+         qvstfcdxia 2, 10, 11
+# CHECK: qvstfcdxi 2, 10, 11             # encoding: [0x7c,0x4a,0x59,0x8a]
+         qvstfcdxi 2, 10, 11
+# CHECK: qvstfcdx 2, 10, 11              # encoding: [0x7c,0x4a,0x59,0x8e]
+         qvstfcdx 2, 10, 11
+# CHECK: qvstfcsuxa 2, 9, 11             # encoding: [0x7c,0x49,0x59,0x4f]
+         qvstfcsuxa 2, 9, 11
+# CHECK: qvstfcsuxia 2, 9, 11            # encoding: [0x7c,0x49,0x59,0x4b]
+         qvstfcsuxia 2, 9, 11
+# CHECK: qvstfcsuxi 2, 9, 11             # encoding: [0x7c,0x49,0x59,0x4a]
+         qvstfcsuxi 2, 9, 11
+# CHECK: qvstfcsux 2, 9, 11              # encoding: [0x7c,0x49,0x59,0x4e]
+         qvstfcsux 2, 9, 11
+# CHECK: qvstfcsxa 2, 10, 11             # encoding: [0x7c,0x4a,0x59,0x0f]
+         qvstfcsxa 2, 10, 11
+# CHECK: qvstfcsxia 2, 10, 11            # encoding: [0x7c,0x4a,0x59,0x0b]
+         qvstfcsxia 2, 10, 11
+# CHECK: qvstfcsxi 2, 10, 11             # encoding: [0x7c,0x4a,0x59,0x0a]
+         qvstfcsxi 2, 10, 11
+# CHECK: qvstfcsx 2, 10, 11              # encoding: [0x7c,0x4a,0x59,0x0e]
+         qvstfcsx 2, 10, 11
+# CHECK: qvstfduxa 2, 9, 11              # encoding: [0x7c,0x49,0x5d,0xcf]
+         qvstfduxa 2, 9, 11
+# CHECK: qvstfduxia 2, 9, 11             # encoding: [0x7c,0x49,0x5d,0xcb]
+         qvstfduxia 2, 9, 11
+# CHECK: qvstfduxi 2, 9, 11              # encoding: [0x7c,0x49,0x5d,0xca]
+         qvstfduxi 2, 9, 11
+# CHECK: qvstfdux 2, 9, 11               # encoding: [0x7c,0x49,0x5d,0xce]
+         qvstfdux 2, 9, 11
+# CHECK: qvstfdxa 2, 10, 11              # encoding: [0x7c,0x4a,0x5d,0x8f]
+         qvstfdxa 2, 10, 11
+# CHECK: qvstfdxia 2, 10, 11             # encoding: [0x7c,0x4a,0x5d,0x8b]
+         qvstfdxia 2, 10, 11
+# CHECK: qvstfdxi 2, 10, 11              # encoding: [0x7c,0x4a,0x5d,0x8a]
+         qvstfdxi 2, 10, 11
+# CHECK: qvstfdx 2, 10, 11               # encoding: [0x7c,0x4a,0x5d,0x8e]
+         qvstfdx 2, 10, 11
+# CHECK: qvstfiwxa 2, 10, 11             # encoding: [0x7c,0x4a,0x5f,0x8f]
+         qvstfiwxa 2, 10, 11
+# CHECK: qvstfiwx 2, 10, 11              # encoding: [0x7c,0x4a,0x5f,0x8e]
+         qvstfiwx 2, 10, 11
+# CHECK: qvstfsuxa 2, 9, 11              # encoding: [0x7c,0x49,0x5d,0x4f]
+         qvstfsuxa 2, 9, 11
+# CHECK: qvstfsuxia 2, 9, 11             # encoding: [0x7c,0x49,0x5d,0x4b]
+         qvstfsuxia 2, 9, 11
+# CHECK: qvstfsuxi 2, 9, 11              # encoding: [0x7c,0x49,0x5d,0x4a]
+         qvstfsuxi 2, 9, 11
+# CHECK: qvstfsux 2, 9, 11               # encoding: [0x7c,0x49,0x5d,0x4e]
+         qvstfsux 2, 9, 11
+# CHECK: qvstfsxa 2, 10, 11              # encoding: [0x7c,0x4a,0x5d,0x0f]
+         qvstfsxa 2, 10, 11
+# CHECK: qvstfsxia 2, 10, 11             # encoding: [0x7c,0x4a,0x5d,0x0b]
+         qvstfsxia 2, 10, 11
+# CHECK: qvstfsxi 2, 10, 11              # encoding: [0x7c,0x4a,0x5d,0x0a]
+         qvstfsxi 2, 10, 11
+# CHECK: qvstfsx 2, 10, 11               # encoding: [0x7c,0x4a,0x5d,0x0e]
+         qvstfsx 2, 10, 11
+
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/fp-bc-icmp-const-fold.ll b/llvm/test/Transforms/Attributor/IPConstantProp/fp-bc-icmp-const-fold.ll
index 67c22f9470779..c45c48d502343 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/fp-bc-icmp-const-fold.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/fp-bc-icmp-const-fold.ll
@@ -4,7 +4,7 @@
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 target datalayout = "E-m:e-i64:64-n32:64"
-target triple = "powerpc64le-unknown-linux"
+target triple = "powerpc64-bgq-linux"
 
 define void @test(i32 signext %n) {
 ; IS__TUNIT____: Function Attrs: nofree noreturn nosync nounwind readnone
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/combined-partial-overwrites.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/combined-partial-overwrites.ll
index faf7041bfc387..6cd77a59df6b1 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/combined-partial-overwrites.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/combined-partial-overwrites.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S -dse -enable-dse-memoryssa -enable-dse-partial-store-merging=false < %s | FileCheck %s
 target datalayout = "E-m:e-i64:64-n32:64"
-target triple = "powerpc64le-unknown-linux"
+target triple = "powerpc64-bgq-linux"
 
 %"struct.std::complex" = type { { float, float } }
 
diff --git a/llvm/test/Transforms/DeadStoreElimination/combined-partial-overwrites.ll b/llvm/test/Transforms/DeadStoreElimination/combined-partial-overwrites.ll
index a57693a1da38e..2a61fff15ade0 100644
--- a/llvm/test/Transforms/DeadStoreElimination/combined-partial-overwrites.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/combined-partial-overwrites.ll
@@ -1,6 +1,6 @@
 ; RUN: opt -S -dse -enable-dse-partial-store-merging=false < %s | FileCheck %s
 target datalayout = "E-m:e-i64:64-n32:64"
-target triple = "powerpc64le-unknown-linux"
+target triple = "powerpc64-bgq-linux"
 
 %"struct.std::complex" = type { { float, float } }
 
diff --git a/llvm/test/Transforms/EntryExitInstrumenter/mcount.ll b/llvm/test/Transforms/EntryExitInstrumenter/mcount.ll
index 5a6daa2c9a008..4c5f18a26657c 100644
--- a/llvm/test/Transforms/EntryExitInstrumenter/mcount.ll
+++ b/llvm/test/Transforms/EntryExitInstrumenter/mcount.ll
@@ -4,7 +4,7 @@
 ; RUN: opt -passes="function(ee-instrument),function(ee-instrument),cgscc(inline),function(post-inline-ee-instrument),function(post-inline-ee-instrument)" -S < %s | FileCheck %s
 
 target datalayout = "E-m:e-i64:64-n32:64"
-target triple = "powerpc64le-unknown-linux"
+target triple = "powerpc64-bgq-linux"
 
 define void @leaf_function() #0 {
 entry:
diff --git a/llvm/test/Transforms/InstCombine/PowerPC/aligned-qpx.ll b/llvm/test/Transforms/InstCombine/PowerPC/aligned-qpx.ll
new file mode 100644
index 0000000000000..e9710df5670cd
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/PowerPC/aligned-qpx.ll
@@ -0,0 +1,165 @@
+; RUN: opt -S -instcombine < %s | FileCheck %s
+target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
+target triple = "powerpc64-unknown-linux-gnu"
+
+declare <4 x double> @llvm.ppc.qpx.qvlfs(i8*) #1
+
+define <4 x double> @test1(<4 x float>* %h) #0 {
+entry:
+  %h1 = getelementptr <4 x float>, <4 x float>* %h, i64 1
+  %hv = bitcast <4 x float>* %h1 to i8*
+  %vl = call <4 x double> @llvm.ppc.qpx.qvlfs(i8* %hv)
+
+; CHECK-LABEL: @test1
+; CHECK: @llvm.ppc.qpx.qvlfs
+; CHECK: ret <4 x double>
+
+  %v0 = load <4 x float>, <4 x float>* %h, align 8
+  %v0e = fpext <4 x float> %v0 to <4 x double>
+  %a = fadd <4 x double> %v0e, %vl
+  ret <4 x double> %a
+}
+
+define <4 x double> @test1a(<4 x float>* align 16 %h) #0 {
+entry:
+  %h1 = getelementptr <4 x float>, <4 x float>* %h, i64 1
+  %hv = bitcast <4 x float>* %h1 to i8*
+  %vl = call <4 x double> @llvm.ppc.qpx.qvlfs(i8* %hv)
+
+; CHECK-LABEL: @test1a
+; CHECK-NOT: @llvm.ppc.qpx.qvlfs
+; CHECK-NOT: load <4 x double>
+; CHECK: ret <4 x double>
+
+  %v0 = load <4 x float>, <4 x float>* %h, align 8
+  %v0e = fpext <4 x float> %v0 to <4 x double>
+  %a = fadd <4 x double> %v0e, %vl
+  ret <4 x double> %a
+}
+
+declare void @llvm.ppc.qpx.qvstfs(<4 x double>, i8*) #0
+
+define <4 x float> @test2(<4 x float>* %h, <4 x double> %d) #0 {
+entry:
+  %h1 = getelementptr <4 x float>, <4 x float>* %h, i64 1
+  %hv = bitcast <4 x float>* %h1 to i8*
+  call void @llvm.ppc.qpx.qvstfs(<4 x double> %d, i8* %hv)
+
+  %v0 = load <4 x float>, <4 x float>* %h, align 8
+  ret <4 x float> %v0
+
+; CHECK-LABEL: @test2
+; CHECK: @llvm.ppc.qpx.qvstfs
+; CHECK: ret <4 x float>
+}
+
+define <4 x float> @test2a(<4 x float>* align 16 %h, <4 x double> %d) #0 {
+entry:
+  %h1 = getelementptr <4 x float>, <4 x float>* %h, i64 1
+  %hv = bitcast <4 x float>* %h1 to i8*
+  call void @llvm.ppc.qpx.qvstfs(<4 x double> %d, i8* %hv)
+
+  %v0 = load <4 x float>, <4 x float>* %h, align 8
+  ret <4 x float> %v0
+
+; CHECK-LABEL: @test2
+; CHECK: fptrunc <4 x double> %d to <4 x float>
+; CHECK-NOT: @llvm.ppc.qpx.qvstfs
+; CHECK-NOT: store <4 x double>
+; CHECK: ret <4 x float>
+}
+
+declare <4 x double> @llvm.ppc.qpx.qvlfd(i8*) #1
+
+define <4 x double> @test1l(<4 x double>* %h) #0 {
+entry:
+  %h1 = getelementptr <4 x double>, <4 x double>* %h, i64 1
+  %hv = bitcast <4 x double>* %h1 to i8*
+  %vl = call <4 x double> @llvm.ppc.qpx.qvlfd(i8* %hv)
+
+; CHECK-LABEL: @test1l
+; CHECK: @llvm.ppc.qpx.qvlfd
+; CHECK: ret <4 x double>
+
+  %v0 = load <4 x double>, <4 x double>* %h, align 8
+  %a = fadd <4 x double> %v0, %vl
+  ret <4 x double> %a
+}
+
+define <4 x double> @test1ln(<4 x double>* align 16 %h) #0 {
+entry:
+  %h1 = getelementptr <4 x double>, <4 x double>* %h, i64 1
+  %hv = bitcast <4 x double>* %h1 to i8*
+  %vl = call <4 x double> @llvm.ppc.qpx.qvlfd(i8* %hv)
+
+; CHECK-LABEL: @test1ln
+; CHECK: @llvm.ppc.qpx.qvlfd
+; CHECK: ret <4 x double>
+
+  %v0 = load <4 x double>, <4 x double>* %h, align 8
+  %a = fadd <4 x double> %v0, %vl
+  ret <4 x double> %a
+}
+
+define <4 x double> @test1la(<4 x double>* align 32 %h) #0 {
+entry:
+  %h1 = getelementptr <4 x double>, <4 x double>* %h, i64 1
+  %hv = bitcast <4 x double>* %h1 to i8*
+  %vl = call <4 x double> @llvm.ppc.qpx.qvlfd(i8* %hv)
+
+; CHECK-LABEL: @test1la
+; CHECK-NOT: @llvm.ppc.qpx.qvlfd
+; CHECK: ret <4 x double>
+
+  %v0 = load <4 x double>, <4 x double>* %h, align 8
+  %a = fadd <4 x double> %v0, %vl
+  ret <4 x double> %a
+}
+
+declare void @llvm.ppc.qpx.qvstfd(<4 x double>, i8*) #0
+
+define <4 x double> @test2l(<4 x double>* %h, <4 x double> %d) #0 {
+entry:
+  %h1 = getelementptr <4 x double>, <4 x double>* %h, i64 1
+  %hv = bitcast <4 x double>* %h1 to i8*
+  call void @llvm.ppc.qpx.qvstfd(<4 x double> %d, i8* %hv)
+
+  %v0 = load <4 x double>, <4 x double>* %h, align 8
+  ret <4 x double> %v0
+
+; CHECK-LABEL: @test2l
+; CHECK: @llvm.ppc.qpx.qvstfd
+; CHECK: ret <4 x double>
+}
+
+define <4 x double> @test2ln(<4 x double>* align 16 %h, <4 x double> %d) #0 {
+entry:
+  %h1 = getelementptr <4 x double>, <4 x double>* %h, i64 1
+  %hv = bitcast <4 x double>* %h1 to i8*
+  call void @llvm.ppc.qpx.qvstfd(<4 x double> %d, i8* %hv)
+
+  %v0 = load <4 x double>, <4 x double>* %h, align 8
+  ret <4 x double> %v0
+
+; CHECK-LABEL: @test2ln
+; CHECK: @llvm.ppc.qpx.qvstfd
+; CHECK: ret <4 x double>
+}
+
+define <4 x double> @test2la(<4 x double>* align 32 %h, <4 x double> %d) #0 {
+entry:
+  %h1 = getelementptr <4 x double>, <4 x double>* %h, i64 1
+  %hv = bitcast <4 x double>* %h1 to i8*
+  call void @llvm.ppc.qpx.qvstfd(<4 x double> %d, i8* %hv)
+
+  %v0 = load <4 x double>, <4 x double>* %h, align 8
+  ret <4 x double> %v0
+
+; CHECK-LABEL: @test2l
+; CHECK-NOT: @llvm.ppc.qpx.qvstfd
+; CHECK: ret <4 x double>
+}
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind readonly }
+
diff --git a/llvm/test/Transforms/LoopDataPrefetch/PowerPC/basic.ll b/llvm/test/Transforms/LoopDataPrefetch/PowerPC/basic.ll
index 68c75af14f3e9..ea46fd0d5a8f8 100644
--- a/llvm/test/Transforms/LoopDataPrefetch/PowerPC/basic.ll
+++ b/llvm/test/Transforms/LoopDataPrefetch/PowerPC/basic.ll
@@ -1,6 +1,7 @@
-; RUN: opt -mcpu=a2 -loop-data-prefetch -mtriple=powerpc64le-unknown-linux -enable-ppc-prefetching -S < %s | FileCheck %s
-; RUN: opt -mcpu=a2 -passes=loop-data-prefetch -mtriple=powerpc64le-unknown-linux -enable-ppc-prefetching -S < %s | FileCheck %s
+; RUN: opt -mcpu=a2 -loop-data-prefetch -S < %s | FileCheck %s
+; RUN: opt -mcpu=a2 -passes=loop-data-prefetch -S < %s | FileCheck %s
 target datalayout = "E-m:e-i64:64-n32:64"
+target triple = "powerpc64-bgq-linux"
 
 define void @foo(double* nocapture %a, double* nocapture readonly %b) {
 entry:
diff --git a/llvm/test/Transforms/LoopSimplify/dup-preds.ll b/llvm/test/Transforms/LoopSimplify/dup-preds.ll
index 362d834686d41..c9253fa51a65f 100644
--- a/llvm/test/Transforms/LoopSimplify/dup-preds.ll
+++ b/llvm/test/Transforms/LoopSimplify/dup-preds.ll
@@ -1,6 +1,6 @@
 ; RUN: opt -loop-simplify -S %s | FileCheck %s
 target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
-target triple = "powerpc64le-unknown-linux"
+target triple = "powerpc64-bgq-linux"
 
 define fastcc void @do_update_md([3 x float]* nocapture readonly %x) #0 {
 entry:
diff --git a/llvm/test/Transforms/LoopUnroll/pr14167.ll b/llvm/test/Transforms/LoopUnroll/pr14167.ll
index 3097c234fb933..9aac70115d9ae 100644
--- a/llvm/test/Transforms/LoopUnroll/pr14167.ll
+++ b/llvm/test/Transforms/LoopUnroll/pr14167.ll
@@ -1,6 +1,6 @@
 ; RUN: opt < %s -S -loop-unroll -unroll-runtime | FileCheck %s
 target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v128:128:128-n32:64"
-target triple = "powerpc64le-unknown-linux"
+target triple = "powerpc64-bgq-linux"
 
 define void @test1() nounwind {
 ; Ensure that we don't crash when the trip count == -1.
diff --git a/llvm/test/Transforms/LoopVectorize/PowerPC/agg-interleave-a2.ll b/llvm/test/Transforms/LoopVectorize/PowerPC/agg-interleave-a2.ll
new file mode 100644
index 0000000000000..9fdfb6f90e7bf
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/PowerPC/agg-interleave-a2.ll
@@ -0,0 +1,40 @@
+; RUN: opt -S -basic-aa -loop-vectorize < %s | FileCheck %s
+target datalayout = "E-m:e-i64:64-n32:64"
+target triple = "powerpc64-unknown-linux-gnu"
+
+; Function Attrs: nounwind
+define void @foo(double* noalias nocapture %a, double* noalias nocapture readonly %b, double* noalias nocapture readonly %c) #0 {
+entry:
+  br label %for.body
+
+; CHECK-LABEL: @foo
+; CHECK: fmul <4 x double> %{{[^,]+}}, <double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00>
+; CHECK-NEXT: fmul <4 x double> %{{[^,]+}}, <double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00>
+
+for.cond.cleanup:                                 ; preds = %for.body
+  ret void
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds double, double* %b, i64 %indvars.iv
+  %0 = load double, double* %arrayidx, align 8
+  %mul = fmul double %0, 2.000000e+00
+  %mul3 = fmul double %0, %mul
+  %arrayidx5 = getelementptr inbounds double, double* %c, i64 %indvars.iv
+  %1 = load double, double* %arrayidx5, align 8
+  %mul6 = fmul double %1, 3.000000e+00
+  %mul9 = fmul double %1, %mul6
+  %add = fadd double %mul3, %mul9
+  %mul12 = fmul double %0, 4.000000e+00
+  %mul15 = fmul double %mul12, %1
+  %add16 = fadd double %mul15, %add
+  %add17 = fadd double %add16, 1.000000e+00
+  %arrayidx19 = getelementptr inbounds double, double* %a, i64 %indvars.iv
+  store double %add17, double* %arrayidx19, align 8
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1600
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+attributes #0 = { nounwind "target-cpu"="a2q" }
+
diff --git a/llvm/test/Transforms/LoopVectorize/PowerPC/vectorize-only-for-real.ll b/llvm/test/Transforms/LoopVectorize/PowerPC/vectorize-only-for-real.ll
index cddddba579473..8abc25ece35c6 100644
--- a/llvm/test/Transforms/LoopVectorize/PowerPC/vectorize-only-for-real.ll
+++ b/llvm/test/Transforms/LoopVectorize/PowerPC/vectorize-only-for-real.ll
@@ -1,6 +1,6 @@
 ; RUN: opt -S -loop-vectorize < %s | FileCheck %s
 target datalayout = "E-m:e-i64:64-n32:64"
-target triple = "powerpc64le-unknown-linux"
+target triple = "powerpc64-bgq-linux"
 
 ; Function Attrs: nounwind
 define zeroext i32 @test() #0 {
diff --git a/llvm/test/Transforms/NewGVN/pr31483.ll b/llvm/test/Transforms/NewGVN/pr31483.ll
index 5bf7e1a695011..999ff74ad5881 100644
--- a/llvm/test/Transforms/NewGVN/pr31483.ll
+++ b/llvm/test/Transforms/NewGVN/pr31483.ll
@@ -100,7 +100,7 @@ declare signext i32 @zot(i8*, ...) #1
 ; Function Attrs: nounwind
 declare void @llvm.va_end(i8*) #2
 
-attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64" "target-features"="+altivec,-bpermd,-crypto,-direct-move,-extdiv,-power8-vector,-vsx" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64" "target-features"="+altivec,-bpermd,-crypto,-direct-move,-extdiv,-power8-vector,-vsx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64" "target-features"="+altivec,-bpermd,-crypto,-direct-move,-extdiv,-power8-vector,-qpx,-vsx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64" "target-features"="+altivec,-bpermd,-crypto,-direct-move,-extdiv,-power8-vector,-qpx,-vsx" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind }
 
diff --git a/llvm/test/Transforms/SCCP/fp-bc-icmp-const-fold.ll b/llvm/test/Transforms/SCCP/fp-bc-icmp-const-fold.ll
index 20c44384504e2..8f97225ca446b 100644
--- a/llvm/test/Transforms/SCCP/fp-bc-icmp-const-fold.ll
+++ b/llvm/test/Transforms/SCCP/fp-bc-icmp-const-fold.ll
@@ -1,6 +1,6 @@
 ; RUN: opt -S -ipsccp < %s | FileCheck %s
 target datalayout = "E-m:e-i64:64-n32:64"
-target triple = "powerpc64le-unknown-linux"
+target triple = "powerpc64-bgq-linux"
 
 define void @test(i32 signext %n) {
 
diff --git a/llvm/unittests/ADT/TripleTest.cpp b/llvm/unittests/ADT/TripleTest.cpp
index 1852d7b6a1b0d..dc7a28c72f208 100644
--- a/llvm/unittests/ADT/TripleTest.cpp
+++ b/llvm/unittests/ADT/TripleTest.cpp
@@ -111,6 +111,41 @@ TEST(TripleTest, ParsedIDs) {
   EXPECT_EQ(Triple::Linux, T.getOS());
   EXPECT_EQ(Triple::Musl, T.getEnvironment());
 
+  T = Triple("powerpc-bgp-linux");
+  EXPECT_EQ(Triple::ppc, T.getArch());
+  EXPECT_EQ(Triple::BGP, T.getVendor());
+  EXPECT_EQ(Triple::Linux, T.getOS());
+  EXPECT_EQ(Triple::UnknownEnvironment, T.getEnvironment());
+
+  T = Triple("powerpc-bgp-cnk");
+  EXPECT_EQ(Triple::ppc, T.getArch());
+  EXPECT_EQ(Triple::BGP, T.getVendor());
+  EXPECT_EQ(Triple::CNK, T.getOS());
+  EXPECT_EQ(Triple::UnknownEnvironment, T.getEnvironment());
+
+  T = Triple("ppc-bgp-linux");
+  EXPECT_EQ(Triple::ppc, T.getArch());
+  EXPECT_EQ(Triple::BGP, T.getVendor());
+  EXPECT_EQ(Triple::Linux, T.getOS());
+  EXPECT_EQ(Triple::UnknownEnvironment, T.getEnvironment());
+
+  T = Triple("ppc32-bgp-linux");
+  EXPECT_EQ(Triple::ppc, T.getArch());
+  EXPECT_EQ(Triple::BGP, T.getVendor());
+  EXPECT_EQ(Triple::Linux, T.getOS());
+  EXPECT_EQ(Triple::UnknownEnvironment, T.getEnvironment());
+
+  T = Triple("powerpc64-bgq-linux");
+  EXPECT_EQ(Triple::ppc64, T.getArch());
+  EXPECT_EQ(Triple::BGQ, T.getVendor());
+  EXPECT_EQ(Triple::Linux, T.getOS());
+  EXPECT_EQ(Triple::UnknownEnvironment, T.getEnvironment());
+
+  T = Triple("ppc64-bgq-linux");
+  EXPECT_EQ(Triple::ppc64, T.getArch());
+  EXPECT_EQ(Triple::BGQ, T.getVendor());
+  EXPECT_EQ(Triple::Linux, T.getOS());
+
   T = Triple("powerpc-ibm-aix");
   EXPECT_EQ(Triple::ppc, T.getArch());
   EXPECT_EQ(Triple::IBM, T.getVendor());
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/PowerPC/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/PowerPC/BUILD.gn
index 3a452fc6e0601..043a672a76e1e 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/PowerPC/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/PowerPC/BUILD.gn
@@ -61,6 +61,7 @@ static_library("LLVMPowerPCCodeGen") {
     "PPCMachineScheduler.cpp",
     "PPCMacroFusion.cpp",
     "PPCPreEmitPeephole.cpp",
+    "PPCQPXLoadSplat.cpp",
     "PPCReduceCRLogicals.cpp",
     "PPCRegisterInfo.cpp",
     "PPCSubtarget.cpp",
diff --git a/openmp/runtime/src/kmp.h b/openmp/runtime/src/kmp.h
index bb6cee740ace7..933573bc810cb 100644
--- a/openmp/runtime/src/kmp.h
+++ b/openmp/runtime/src/kmp.h
@@ -1116,6 +1116,9 @@ extern kmp_uint64 __kmp_now_nsec();
 #if KMP_OS_WINDOWS
 #define KMP_INIT_WAIT 64U /* initial number of spin-tests   */
 #define KMP_NEXT_WAIT 32U /* susequent number of spin-tests */
+#elif KMP_OS_CNK
+#define KMP_INIT_WAIT 16U /* initial number of spin-tests   */
+#define KMP_NEXT_WAIT 8U /* susequent number of spin-tests */
 #elif KMP_OS_LINUX
 #define KMP_INIT_WAIT 1024U /* initial number of spin-tests   */
 #define KMP_NEXT_WAIT 512U /* susequent number of spin-tests */
diff --git a/openmp/runtime/src/kmp_csupport.cpp b/openmp/runtime/src/kmp_csupport.cpp
index f6fb1e602c297..b5c641cc7273c 100644
--- a/openmp/runtime/src/kmp_csupport.cpp
+++ b/openmp/runtime/src/kmp_csupport.cpp
@@ -680,6 +680,17 @@ void __kmpc_flush(ident_t *loc) {
 // Nothing to see here move along
 #elif KMP_ARCH_PPC64
 // Nothing needed here (we have a real MB above).
+#if KMP_OS_CNK
+  // The flushing thread needs to yield here; this prevents a
+  // busy-waiting thread from saturating the pipeline. flush is
+  // often used in loops like this:
+  // while (!flag) {
+  //   #pragma omp flush(flag)
+  // }
+  // and adding the yield here is good for at least a 10x speedup
+  // when running >2 threads per core (on the NAS LU benchmark).
+  __kmp_yield();
+#endif
 #else
 #error Unknown or unsupported architecture
 #endif
diff --git a/openmp/runtime/src/kmp_lock.h b/openmp/runtime/src/kmp_lock.h
index b80e54777e8c2..e54f6812b8b34 100644
--- a/openmp/runtime/src/kmp_lock.h
+++ b/openmp/runtime/src/kmp_lock.h
@@ -159,7 +159,7 @@ extern void __kmp_destroy_nested_tas_lock(kmp_tas_lock_t *lck);
 #define KMP_LOCK_ACQUIRED_NEXT 0
 #ifndef KMP_USE_FUTEX
 #define KMP_USE_FUTEX                                                          \
-  (KMP_OS_LINUX &&                                                             \
+  (KMP_OS_LINUX && !KMP_OS_CNK &&                                              \
    (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64))
 #endif
 #if KMP_USE_FUTEX
diff --git a/openmp/runtime/src/kmp_os.h b/openmp/runtime/src/kmp_os.h
index 33735cf455c7e..bfe7765b2a967 100644
--- a/openmp/runtime/src/kmp_os.h
+++ b/openmp/runtime/src/kmp_os.h
@@ -69,7 +69,7 @@
 #error Unknown compiler
 #endif
 
-#if (KMP_OS_LINUX || KMP_OS_WINDOWS || KMP_OS_FREEBSD)
+#if (KMP_OS_LINUX || KMP_OS_WINDOWS || KMP_OS_FREEBSD) && !KMP_OS_CNK
 #define KMP_AFFINITY_SUPPORTED 1
 #if KMP_OS_WINDOWS && KMP_ARCH_X86_64
 #define KMP_GROUP_AFFINITY 1
diff --git a/openmp/runtime/src/kmp_platform.h b/openmp/runtime/src/kmp_platform.h
index 4296ca31d67d9..779c08e9771d5 100644
--- a/openmp/runtime/src/kmp_platform.h
+++ b/openmp/runtime/src/kmp_platform.h
@@ -22,6 +22,7 @@
 #define KMP_OS_OPENBSD 0
 #define KMP_OS_DARWIN 0
 #define KMP_OS_WINDOWS 0
+#define KMP_OS_CNK 0
 #define KMP_OS_HURD 0
 #define KMP_OS_UNIX 0 /* disjunction of KMP_OS_LINUX, KMP_OS_DARWIN etc. */
 
@@ -65,6 +66,11 @@
 #define KMP_OS_OPENBSD 1
 #endif
 
+#if (defined __bgq__)
+#undef KMP_OS_CNK
+#define KMP_OS_CNK 1
+#endif
+
 #if (defined __GNU__)
 #undef KMP_OS_HURD
 #define KMP_OS_HURD 1
diff --git a/openmp/runtime/src/z_Linux_asm.S b/openmp/runtime/src/z_Linux_asm.S
index 16059a3762bf4..8090ff759fe1b 100644
--- a/openmp/runtime/src/z_Linux_asm.S
+++ b/openmp/runtime/src/z_Linux_asm.S
@@ -1433,8 +1433,13 @@ __kmp_invoke_microtask:
 	add 12, 0, 12
 	neg 12, 12
 
-// We need to make sure that the stack frame stays aligned (to 16 bytes).
+// We need to make sure that the stack frame stays aligned (to 16 bytes, except
+// under the BG/Q CNK, where it must be to 32 bytes).
+# if KMP_OS_CNK
+	li 0, -32
+# else
 	li 0, -16
+# endif
 	and 12, 0, 12
 
 // Establish the local stack frame.
diff --git a/openmp/runtime/src/z_Linux_util.cpp b/openmp/runtime/src/z_Linux_util.cpp
index 58cc4d25f6080..3b5910fc95e89 100644
--- a/openmp/runtime/src/z_Linux_util.cpp
+++ b/openmp/runtime/src/z_Linux_util.cpp
@@ -31,7 +31,7 @@
 #include <sys/times.h>
 #include <unistd.h>
 
-#if KMP_OS_LINUX
+#if KMP_OS_LINUX && !KMP_OS_CNK
 #include <sys/sysinfo.h>
 #if KMP_USE_FUTEX
 // We should really include <futex.h>, but that causes compatibility problems on
diff --git a/polly/lib/External/isl/config.sub b/polly/lib/External/isl/config.sub
index bc4db70f82abf..1d8e98bcee23a 100755
--- a/polly/lib/External/isl/config.sub
+++ b/polly/lib/External/isl/config.sub
@@ -152,6 +152,9 @@ case $os in
 		os=
 		basic_machine=$1
 		;;
+	-bluegene*)
+		os=-cnk
+		;;
 	-sim | -cisco | -oki | -wec | -winbond)
 		os=
 		basic_machine=$1
@@ -536,6 +539,10 @@ case $basic_machine in
 		basic_machine=bfin-`echo "$basic_machine" | sed 's/^[^-]*-//'`
 		os=-linux
 		;;
+	bluegene*)
+		basic_machine=powerpc-ibm
+		os=-cnk
+		;;
 	c54x-*)
 		basic_machine=tic54x-`echo "$basic_machine" | sed 's/^[^-]*-//'`
 		;;
@@ -1357,7 +1364,7 @@ case $os in
 	# Each alternative MUST end in a * to match a version number.
 	# -sysv* is not here because it comes later, after sysvr4.
 	-gnu* | -bsd* | -mach* | -minix* | -genix* | -ultrix* | -irix* \
-	      | -*vms* | -sco* | -esix* | -isc* | -aix* | -sunos | -sunos[34]*\
+	      | -*vms* | -sco* | -esix* | -isc* | -aix* | -cnk* | -sunos | -sunos[34]*\
 	      | -hpux* | -unos* | -osf* | -luna* | -dgux* | -auroraux* | -solaris* \
 	      | -sym* | -kopensolaris* | -plan9* \
 	      | -amigaos* | -amigados* | -msdos* | -newsos* | -unicos* | -aof* \
@@ -1721,7 +1728,7 @@ case $basic_machine in
 			-sunos*)
 				vendor=sun
 				;;
-			-aix*)
+			-cnk*|-aix*)
 				vendor=ibm
 				;;
 			-beos*)
diff --git a/polly/lib/External/ppcg/config.sub b/polly/lib/External/ppcg/config.sub
index d97f3009f9f09..6205f8423d6aa 100644
--- a/polly/lib/External/ppcg/config.sub
+++ b/polly/lib/External/ppcg/config.sub
@@ -160,6 +160,9 @@ case $os in
 		os=
 		basic_machine=$1
 		;;
+	-bluegene*)
+		os=-cnk
+		;;
 	-sim | -cisco | -oki | -wec | -winbond)
 		os=
 		basic_machine=$1
@@ -521,6 +524,10 @@ case $basic_machine in
 		basic_machine=bfin-`echo $basic_machine | sed 's/^[^-]*-//'`
 		os=-linux
 		;;
+	bluegene*)
+		basic_machine=powerpc-ibm
+		os=-cnk
+		;;
 	c54x-*)
 		basic_machine=tic54x-`echo $basic_machine | sed 's/^[^-]*-//'`
 		;;
@@ -1337,7 +1344,7 @@ case $os in
 	# Each alternative MUST END IN A *, to match a version number.
 	# -sysv* is not here because it comes later, after sysvr4.
 	-gnu* | -bsd* | -mach* | -minix* | -genix* | -ultrix* | -irix* \
-	      | -*vms* | -sco* | -esix* | -isc* | -aix* | -sunos | -sunos[34]*\
+	      | -*vms* | -sco* | -esix* | -isc* | -aix* | -cnk* | -sunos | -sunos[34]*\
 	      | -hpux* | -unos* | -osf* | -luna* | -dgux* | -auroraux* | -solaris* \
 	      | -sym* | -kopensolaris* \
 	      | -amigaos* | -amigados* | -msdos* | -newsos* | -unicos* | -aof* \
@@ -1702,7 +1709,7 @@ case $basic_machine in
 			-sunos*)
 				vendor=sun
 				;;
-			-aix*)
+			-cnk*|-aix*)
 				vendor=ibm
 				;;
 			-beos*)

From 4b5412b5dbc87096e420de5172837b4bd5ab9485 Mon Sep 17 00:00:00 2001
From: Sridhar Gopinath <sridhar.g@utexas.edu>
Date: Fri, 24 Jul 2020 10:44:48 -0700
Subject: [PATCH 0246/1035] Fix the move constructor of MMI to move
 MachineFunctions map

The move constructor of MachineModuleInfo currently does not copy the
MachineFunctions map. This commit fixes this issue.

Patch by Sridhar Gopinath. Thanks!

Differential Revision: https://reviews.llvm.org/D84274
---
 llvm/lib/CodeGen/MachineModuleInfo.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/CodeGen/MachineModuleInfo.cpp b/llvm/lib/CodeGen/MachineModuleInfo.cpp
index f866c7ca53c68..be08f0ae31171 100644
--- a/llvm/lib/CodeGen/MachineModuleInfo.cpp
+++ b/llvm/lib/CodeGen/MachineModuleInfo.cpp
@@ -178,7 +178,8 @@ void MachineModuleInfo::finalize() {
 MachineModuleInfo::MachineModuleInfo(MachineModuleInfo &&MMI)
     : TM(std::move(MMI.TM)),
       Context(MMI.TM.getMCAsmInfo(), MMI.TM.getMCRegisterInfo(),
-              MMI.TM.getObjFileLowering(), nullptr, nullptr, false) {
+              MMI.TM.getObjFileLowering(), nullptr, nullptr, false),
+      MachineFunctions(std::move(MMI.MachineFunctions)) {
   ObjFileMMI = MMI.ObjFileMMI;
   CurCallSite = MMI.CurCallSite;
   UsesMSVCFloatingPoint = MMI.UsesMSVCFloatingPoint;

From ef748b58d3b3edfaf0278d454cb30f7816c04aee Mon Sep 17 00:00:00 2001
From: Fred Riss <friss@apple.com>
Date: Mon, 27 Jul 2020 13:51:07 -0700
Subject: [PATCH 0247/1035] [lldb] NFC: Use early exit in ArchSpec::IsEqualTo

---
 lldb/source/Utility/ArchSpec.cpp | 113 +++++++++++++++----------------
 1 file changed, 53 insertions(+), 60 deletions(-)

diff --git a/lldb/source/Utility/ArchSpec.cpp b/lldb/source/Utility/ArchSpec.cpp
index a77ae8633070e..cd382a322da70 100644
--- a/lldb/source/Utility/ArchSpec.cpp
+++ b/lldb/source/Utility/ArchSpec.cpp
@@ -1010,77 +1010,70 @@ static bool IsCompatibleEnvironment(llvm::Triple::EnvironmentType lhs,
 bool ArchSpec::IsEqualTo(const ArchSpec &rhs, bool exact_match) const {
   // explicitly ignoring m_distribution_id in this method.
 
-  if (GetByteOrder() != rhs.GetByteOrder())
+  if (GetByteOrder() != rhs.GetByteOrder() ||
+      !cores_match(GetCore(), rhs.GetCore(), true, exact_match))
     return false;
 
-  const ArchSpec::Core lhs_core = GetCore();
-  const ArchSpec::Core rhs_core = rhs.GetCore();
+  const llvm::Triple &lhs_triple = GetTriple();
+  const llvm::Triple &rhs_triple = rhs.GetTriple();
+
+  const llvm::Triple::VendorType lhs_triple_vendor = lhs_triple.getVendor();
+  const llvm::Triple::VendorType rhs_triple_vendor = rhs_triple.getVendor();
+  if (lhs_triple_vendor != rhs_triple_vendor) {
+    const bool rhs_vendor_specified = rhs.TripleVendorWasSpecified();
+    const bool lhs_vendor_specified = TripleVendorWasSpecified();
+    // Both architectures had the vendor specified, so if they aren't equal
+    // then we return false
+    if (rhs_vendor_specified && lhs_vendor_specified)
+      return false;
 
-  const bool core_match = cores_match(lhs_core, rhs_core, true, exact_match);
-
-  if (core_match) {
-    const llvm::Triple &lhs_triple = GetTriple();
-    const llvm::Triple &rhs_triple = rhs.GetTriple();
-
-    const llvm::Triple::VendorType lhs_triple_vendor = lhs_triple.getVendor();
-    const llvm::Triple::VendorType rhs_triple_vendor = rhs_triple.getVendor();
-    if (lhs_triple_vendor != rhs_triple_vendor) {
-      const bool rhs_vendor_specified = rhs.TripleVendorWasSpecified();
-      const bool lhs_vendor_specified = TripleVendorWasSpecified();
-      // Both architectures had the vendor specified, so if they aren't equal
-      // then we return false
-      if (rhs_vendor_specified && lhs_vendor_specified)
-        return false;
-
-      // Only fail if both vendor types are not unknown
-      if (lhs_triple_vendor != llvm::Triple::UnknownVendor &&
-          rhs_triple_vendor != llvm::Triple::UnknownVendor)
-        return false;
-    }
+    // Only fail if both vendor types are not unknown
+    if (lhs_triple_vendor != llvm::Triple::UnknownVendor &&
+        rhs_triple_vendor != llvm::Triple::UnknownVendor)
+      return false;
+  }
 
-    const llvm::Triple::OSType lhs_triple_os = lhs_triple.getOS();
-    const llvm::Triple::OSType rhs_triple_os = rhs_triple.getOS();
-    const llvm::Triple::EnvironmentType lhs_triple_env =
+  const llvm::Triple::OSType lhs_triple_os = lhs_triple.getOS();
+  const llvm::Triple::OSType rhs_triple_os = rhs_triple.getOS();
+  const llvm::Triple::EnvironmentType lhs_triple_env =
       lhs_triple.getEnvironment();
-    const llvm::Triple::EnvironmentType rhs_triple_env =
+  const llvm::Triple::EnvironmentType rhs_triple_env =
       rhs_triple.getEnvironment();
 
-    if (!exact_match) {
-      // x86_64-apple-ios-macabi, x86_64-apple-macosx are compatible, no match.
-      if ((lhs_triple_os == llvm::Triple::IOS &&
-           lhs_triple_env == llvm::Triple::MacABI &&
-           rhs_triple_os == llvm::Triple::MacOSX) ||
-          (lhs_triple_os == llvm::Triple::MacOSX &&
-           rhs_triple_os == llvm::Triple::IOS &&
-           rhs_triple_env == llvm::Triple::MacABI))
-        return true;
-    }
-
-    if (lhs_triple_os != rhs_triple_os) {
-      const bool rhs_os_specified = rhs.TripleOSWasSpecified();
-      const bool lhs_os_specified = TripleOSWasSpecified();
-      // Both architectures had the OS specified, so if they aren't equal then
-      // we return false
-      if (rhs_os_specified && lhs_os_specified)
-        return false;
-
-      // Only fail if both os types are not unknown
-      if (lhs_triple_os != llvm::Triple::UnknownOS &&
-          rhs_triple_os != llvm::Triple::UnknownOS)
-        return false;
-    }
+  if (!exact_match) {
+    // x86_64-apple-ios-macabi, x86_64-apple-macosx are compatible, no match.
+    if ((lhs_triple_os == llvm::Triple::IOS &&
+         lhs_triple_env == llvm::Triple::MacABI &&
+         rhs_triple_os == llvm::Triple::MacOSX) ||
+        (lhs_triple_os == llvm::Triple::MacOSX &&
+         rhs_triple_os == llvm::Triple::IOS &&
+         rhs_triple_env == llvm::Triple::MacABI))
+      return true;
+  }
 
-    // x86_64-apple-ios-macabi and x86_64-apple-ios are not compatible.
-    if (lhs_triple_os == llvm::Triple::IOS &&
-        rhs_triple_os == llvm::Triple::IOS &&
-        (lhs_triple_env == llvm::Triple::MacABI ||
-         rhs_triple_env == llvm::Triple::MacABI) &&
-        lhs_triple_env != rhs_triple_env)
+  if (lhs_triple_os != rhs_triple_os) {
+    const bool rhs_os_specified = rhs.TripleOSWasSpecified();
+    const bool lhs_os_specified = TripleOSWasSpecified();
+    // Both architectures had the OS specified, so if they aren't equal then
+    // we return false
+    if (rhs_os_specified && lhs_os_specified)
       return false;
 
-    return IsCompatibleEnvironment(lhs_triple_env, rhs_triple_env);
+    // Only fail if both os types are not unknown
+    if (lhs_triple_os != llvm::Triple::UnknownOS &&
+        rhs_triple_os != llvm::Triple::UnknownOS)
+      return false;
   }
-  return false;
+
+  // x86_64-apple-ios-macabi and x86_64-apple-ios are not compatible.
+  if (lhs_triple_os == llvm::Triple::IOS &&
+      rhs_triple_os == llvm::Triple::IOS &&
+      (lhs_triple_env == llvm::Triple::MacABI ||
+       rhs_triple_env == llvm::Triple::MacABI) &&
+      lhs_triple_env != rhs_triple_env)
+    return false;
+
+  return IsCompatibleEnvironment(lhs_triple_env, rhs_triple_env);
 }
 
 void ArchSpec::UpdateCore() {

From c37bb5e2a541df867d371278f6ddfaf85c299771 Mon Sep 17 00:00:00 2001
From: Arthur Eubanks <aeubanks@google.com>
Date: Mon, 27 Jul 2020 14:58:42 -0700
Subject: [PATCH 0248/1035] [DFSan] Remove unused DataFlowSanitizer vars

Reviewed By: morehouse

Differential Revision: https://reviews.llvm.org/D84704
---
 .../include/llvm/Transforms/Instrumentation.h |  3 +-
 .../Instrumentation/DataFlowSanitizer.cpp     | 58 +++++--------------
 2 files changed, 17 insertions(+), 44 deletions(-)

diff --git a/llvm/include/llvm/Transforms/Instrumentation.h b/llvm/include/llvm/Transforms/Instrumentation.h
index d4373d7b39eaa..f2e084041dd67 100644
--- a/llvm/include/llvm/Transforms/Instrumentation.h
+++ b/llvm/include/llvm/Transforms/Instrumentation.h
@@ -144,8 +144,7 @@ ModulePass *createInstrOrderFilePass();
 
 // Insert DataFlowSanitizer (dynamic data flow analysis) instrumentation
 ModulePass *createDataFlowSanitizerPass(
-    const std::vector<std::string> &ABIListFiles = std::vector<std::string>(),
-    void *(*getArgTLS)() = nullptr, void *(*getRetValTLS)() = nullptr);
+    const std::vector<std::string> &ABIListFiles = std::vector<std::string>());
 
 // Options for sanitizer coverage instrumentation.
 struct SanitizerCoverageOptions {
diff --git a/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
index 2846319007318..a5a785cb55f78 100644
--- a/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
@@ -344,8 +344,6 @@ class DataFlowSanitizer : public ModulePass {
   ConstantInt *ShadowPtrMul;
   Constant *ArgTLS;
   Constant *RetvalTLS;
-  void *(*GetArgTLSPtr)();
-  void *(*GetRetvalTLSPtr)();
   FunctionType *GetArgTLSTy;
   FunctionType *GetRetvalTLSTy;
   Constant *GetArgTLS;
@@ -395,9 +393,8 @@ class DataFlowSanitizer : public ModulePass {
 public:
   static char ID;
 
-  DataFlowSanitizer(
-      const std::vector<std::string> &ABIListFiles = std::vector<std::string>(),
-      void *(*getArgTLS)() = nullptr, void *(*getRetValTLS)() = nullptr);
+  DataFlowSanitizer(const std::vector<std::string> &ABIListFiles =
+                        std::vector<std::string>());
 
   bool doInitialization(Module &M) override;
   bool runOnModule(Module &M) override;
@@ -490,17 +487,14 @@ char DataFlowSanitizer::ID;
 INITIALIZE_PASS(DataFlowSanitizer, "dfsan",
                 "DataFlowSanitizer: dynamic data flow analysis.", false, false)
 
-ModulePass *
-llvm::createDataFlowSanitizerPass(const std::vector<std::string> &ABIListFiles,
-                                  void *(*getArgTLS)(),
-                                  void *(*getRetValTLS)()) {
-  return new DataFlowSanitizer(ABIListFiles, getArgTLS, getRetValTLS);
+ModulePass *llvm::createDataFlowSanitizerPass(
+    const std::vector<std::string> &ABIListFiles) {
+  return new DataFlowSanitizer(ABIListFiles);
 }
 
 DataFlowSanitizer::DataFlowSanitizer(
-    const std::vector<std::string> &ABIListFiles, void *(*getArgTLS)(),
-    void *(*getRetValTLS)())
-    : ModulePass(ID), GetArgTLSPtr(getArgTLS), GetRetvalTLSPtr(getRetValTLS) {
+    const std::vector<std::string> &ABIListFiles)
+    : ModulePass(ID) {
   std::vector<std::string> AllABIListFiles(std::move(ABIListFiles));
   AllABIListFiles.insert(AllABIListFiles.end(), ClABIListFiles.begin(),
                          ClABIListFiles.end());
@@ -613,22 +607,6 @@ bool DataFlowSanitizer::doInitialization(Module &M) {
       FunctionType::get(Type::getVoidTy(*Ctx), DFSanMemTransferCallbackArgs,
                         /*isVarArg=*/false);
 
-  if (GetArgTLSPtr) {
-    Type *ArgTLSTy = ArrayType::get(ShadowTy, 64);
-    ArgTLS = nullptr;
-    GetArgTLSTy = FunctionType::get(PointerType::getUnqual(ArgTLSTy), false);
-    GetArgTLS = ConstantExpr::getIntToPtr(
-        ConstantInt::get(IntptrTy, uintptr_t(GetArgTLSPtr)),
-        PointerType::getUnqual(GetArgTLSTy));
-  }
-  if (GetRetvalTLSPtr) {
-    RetvalTLS = nullptr;
-    GetRetvalTLSTy = FunctionType::get(PointerType::getUnqual(ShadowTy), false);
-    GetRetvalTLS = ConstantExpr::getIntToPtr(
-        ConstantInt::get(IntptrTy, uintptr_t(GetRetvalTLSPtr)),
-        PointerType::getUnqual(GetRetvalTLSTy));
-  }
-
   ColdCallWeights = MDBuilder(*Ctx).createBranchWeights(1, 1000);
   return true;
 }
@@ -816,20 +794,16 @@ bool DataFlowSanitizer::runOnModule(Module &M) {
 
   bool Changed = false;
 
-  if (!GetArgTLSPtr) {
-    Type *ArgTLSTy = ArrayType::get(ShadowTy, 64);
-    ArgTLS = Mod->getOrInsertGlobal("__dfsan_arg_tls", ArgTLSTy);
-    if (GlobalVariable *G = dyn_cast<GlobalVariable>(ArgTLS)) {
-      Changed |= G->getThreadLocalMode() != GlobalVariable::InitialExecTLSModel;
-      G->setThreadLocalMode(GlobalVariable::InitialExecTLSModel);
-    }
+  Type *ArgTLSTy = ArrayType::get(ShadowTy, 64);
+  ArgTLS = Mod->getOrInsertGlobal("__dfsan_arg_tls", ArgTLSTy);
+  if (GlobalVariable *G = dyn_cast<GlobalVariable>(ArgTLS)) {
+    Changed |= G->getThreadLocalMode() != GlobalVariable::InitialExecTLSModel;
+    G->setThreadLocalMode(GlobalVariable::InitialExecTLSModel);
   }
-  if (!GetRetvalTLSPtr) {
-    RetvalTLS = Mod->getOrInsertGlobal("__dfsan_retval_tls", ShadowTy);
-    if (GlobalVariable *G = dyn_cast<GlobalVariable>(RetvalTLS)) {
-      Changed |= G->getThreadLocalMode() != GlobalVariable::InitialExecTLSModel;
-      G->setThreadLocalMode(GlobalVariable::InitialExecTLSModel);
-    }
+  RetvalTLS = Mod->getOrInsertGlobal("__dfsan_retval_tls", ShadowTy);
+  if (GlobalVariable *G = dyn_cast<GlobalVariable>(RetvalTLS)) {
+    Changed |= G->getThreadLocalMode() != GlobalVariable::InitialExecTLSModel;
+    G->setThreadLocalMode(GlobalVariable::InitialExecTLSModel);
   }
 
   ExternalShadowMask =

From e57464151d4c4912a7ec4d6fd0920056b2f75c7c Mon Sep 17 00:00:00 2001
From: peter klausler <pklausler@nvidia.com>
Date: Wed, 22 Jul 2020 16:56:37 -0700
Subject: [PATCH 0249/1035] [flang] Allow omission of comma in FORMAT(1PE5.2)
 in runtime

A comma is not required between a scale factor and a following
data edit descriptor (C1302).

Reviewed By: PeteSteinfeld

Differential Revision: https://reviews.llvm.org/D84369
---
 flang/runtime/format-implementation.h | 12 +++++++-----
 flang/unittests/Runtime/hello.cpp     |  2 ++
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/flang/runtime/format-implementation.h b/flang/runtime/format-implementation.h
index a4453cd172ea5..ad8bbcbcdcbdf 100644
--- a/flang/runtime/format-implementation.h
+++ b/flang/runtime/format-implementation.h
@@ -330,11 +330,13 @@ int FormatControl<CONTEXT>::CueUpNextDataEdit(Context &context, bool stop) {
       offset_ += *repeat;
     } else if (ch >= 'A' && ch <= 'Z') {
       int start{offset_ - 1};
-      CharType next{Capitalize(PeekNext())};
-      if (next >= 'A' && next <= 'Z') {
-        ++offset_;
-      } else {
-        next = '\0';
+      CharType next{'\0'};
+      if (ch != 'P') { // 1PE5.2 - comma not required (C1302)
+        CharType peek{Capitalize(PeekNext())};
+        if (peek >= 'A' && peek <= 'Z') {
+          next = peek;
+          ++offset_;
+        }
       }
       if (ch == 'E' ||
           (!next &&
diff --git a/flang/unittests/Runtime/hello.cpp b/flang/unittests/Runtime/hello.cpp
index f6db4a8e47dc8..c38aedf4f6549 100644
--- a/flang/unittests/Runtime/hello.cpp
+++ b/flang/unittests/Runtime/hello.cpp
@@ -175,6 +175,7 @@ int main() {
       {"(E32.17E0,';')", "          0.00000000000000000E+0;"},
       {"(G32.17E0,';')", "          0.0000000000000000    ;"},
       {"(1P,E32.17,';')", "         0.00000000000000000E+00;"},
+      {"(1PE32.17,';')", "         0.00000000000000000E+00;"}, // no comma
       {"(1P,F32.17,';')", "             0.00000000000000000;"},
       {"(1P,G32.17,';')", "          0.0000000000000000    ;"},
       {"(2P,E32.17,';')", "         00.0000000000000000E+00;"},
@@ -195,6 +196,7 @@ int main() {
       {"(E32.17E4,';')", "       0.10000000000000000E+0001;"},
       {"(G32.17E4,';')", "        1.0000000000000000      ;"},
       {"(1P,E32.17,';')", "         1.00000000000000000E+00;"},
+      {"(1PE32.17,';')", "         1.00000000000000000E+00;"}, // no comma
       {"(1P,F32.17,';')", "             0.10000000000000000;"},
       {"(1P,G32.17,';')", "          1.0000000000000000    ;"},
       {"(ES32.17,';')", "         1.00000000000000000E+00;"},

From adb28e0fb2b0e97ea9dce422c09b36979cf7cd2f Mon Sep 17 00:00:00 2001
From: Francesco Petrogalli <francesco.petrogalli@arm.com>
Date: Mon, 6 Jul 2020 17:46:59 +0000
Subject: [PATCH 0250/1035] [llvm][CodeGen] Addressing modes for SVE ldN.

Reviewers: c-rhodes, efriedma, sdesmalen

Subscribers: huihuiz, tschuett, hiraditya, llvm-commits

Tags: #llvm

Differential Revision: https://reviews.llvm.org/D77251
---
 .../Target/AArch64/AArch64ISelDAGToDAG.cpp    |  64 ++-
 .../sve-intrinsics-ldN-reg+imm-addr-mode.ll   | 495 ++++++++++++++++++
 .../sve-intrinsics-ldN-reg+reg-addr-mode.ll   | 259 +++++++++
 3 files changed, 798 insertions(+), 20 deletions(-)
 create mode 100644 llvm/test/CodeGen/AArch64/sve-intrinsics-ldN-reg+imm-addr-mode.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-intrinsics-ldN-reg+reg-addr-mode.ll

diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
index dbd7db7ee8e6f..7799ebfbd68e6 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
@@ -245,7 +245,8 @@ class AArch64DAGToDAGISel : public SelectionDAGISel {
                          unsigned SubRegIdx);
   void SelectLoadLane(SDNode *N, unsigned NumVecs, unsigned Opc);
   void SelectPostLoadLane(SDNode *N, unsigned NumVecs, unsigned Opc);
-  void SelectPredicatedLoad(SDNode *N, unsigned NumVecs, const unsigned Opc);
+  void SelectPredicatedLoad(SDNode *N, unsigned NumVecs, unsigned Scale,
+                            unsigned Opc_rr, unsigned Opc_ri);
 
   bool SelectAddrModeFrameIndexSVE(SDValue N, SDValue &Base, SDValue &OffImm);
   /// SVE Reg+Imm addressing mode.
@@ -1434,14 +1435,23 @@ AArch64DAGToDAGISel::findAddrModeSVELoadStore(SDNode *N, unsigned Opc_rr,
 }
 
 void AArch64DAGToDAGISel::SelectPredicatedLoad(SDNode *N, unsigned NumVecs,
-                                               const unsigned Opc) {
+                                               unsigned Scale, unsigned Opc_ri,
+                                               unsigned Opc_rr) {
+  assert(Scale < 4 && "Invalid scaling value.");
   SDLoc DL(N);
   EVT VT = N->getValueType(0);
   SDValue Chain = N->getOperand(0);
 
+  // Optimize addressing mode.
+  SDValue Base, Offset;
+  unsigned Opc;
+  std::tie(Opc, Base, Offset) = findAddrModeSVELoadStore(
+      N, Opc_rr, Opc_ri, N->getOperand(2),
+      CurDAG->getTargetConstant(0, DL, MVT::i64), Scale);
+
   SDValue Ops[] = {N->getOperand(1), // Predicate
-                   N->getOperand(2), // Memory operand
-                   CurDAG->getTargetConstant(0, DL, MVT::i64), Chain};
+                   Base,             // Memory operand
+                   Offset, Chain};
 
   const EVT ResTys[] = {MVT::Untyped, MVT::Other};
 
@@ -4726,51 +4736,51 @@ void AArch64DAGToDAGISel::Select(SDNode *Node) {
   }
   case AArch64ISD::SVE_LD2_MERGE_ZERO: {
     if (VT == MVT::nxv16i8) {
-      SelectPredicatedLoad(Node, 2, AArch64::LD2B_IMM);
+      SelectPredicatedLoad(Node, 2, 0, AArch64::LD2B_IMM, AArch64::LD2B);
       return;
     } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
                (VT == MVT::nxv8bf16 && Subtarget->hasBF16())) {
-      SelectPredicatedLoad(Node, 2, AArch64::LD2H_IMM);
+      SelectPredicatedLoad(Node, 2, 1, AArch64::LD2H_IMM, AArch64::LD2H);
       return;
     } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
-      SelectPredicatedLoad(Node, 2, AArch64::LD2W_IMM);
+      SelectPredicatedLoad(Node, 2, 2, AArch64::LD2W_IMM, AArch64::LD2W);
       return;
     } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
-      SelectPredicatedLoad(Node, 2, AArch64::LD2D_IMM);
+      SelectPredicatedLoad(Node, 2, 3, AArch64::LD2D_IMM, AArch64::LD2D);
       return;
     }
     break;
   }
   case AArch64ISD::SVE_LD3_MERGE_ZERO: {
     if (VT == MVT::nxv16i8) {
-      SelectPredicatedLoad(Node, 3, AArch64::LD3B_IMM);
+      SelectPredicatedLoad(Node, 3, 0, AArch64::LD3B_IMM, AArch64::LD3B);
       return;
     } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
                (VT == MVT::nxv8bf16 && Subtarget->hasBF16())) {
-      SelectPredicatedLoad(Node, 3, AArch64::LD3H_IMM);
+      SelectPredicatedLoad(Node, 3, 1, AArch64::LD3H_IMM, AArch64::LD3H);
       return;
     } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
-      SelectPredicatedLoad(Node, 3, AArch64::LD3W_IMM);
+      SelectPredicatedLoad(Node, 3, 2, AArch64::LD3W_IMM, AArch64::LD3W);
       return;
     } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
-      SelectPredicatedLoad(Node, 3, AArch64::LD3D_IMM);
+      SelectPredicatedLoad(Node, 3, 3, AArch64::LD3D_IMM, AArch64::LD3D);
       return;
     }
     break;
   }
   case AArch64ISD::SVE_LD4_MERGE_ZERO: {
     if (VT == MVT::nxv16i8) {
-      SelectPredicatedLoad(Node, 4, AArch64::LD4B_IMM);
+      SelectPredicatedLoad(Node, 4, 0, AArch64::LD4B_IMM, AArch64::LD4B);
       return;
     } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
                (VT == MVT::nxv8bf16 && Subtarget->hasBF16())) {
-      SelectPredicatedLoad(Node, 4, AArch64::LD4H_IMM);
+      SelectPredicatedLoad(Node, 4, 1, AArch64::LD4H_IMM, AArch64::LD4H);
       return;
     } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
-      SelectPredicatedLoad(Node, 4, AArch64::LD4W_IMM);
+      SelectPredicatedLoad(Node, 4, 2, AArch64::LD4W_IMM, AArch64::LD4W);
       return;
     } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
-      SelectPredicatedLoad(Node, 4, AArch64::LD4D_IMM);
+      SelectPredicatedLoad(Node, 4, 3, AArch64::LD4D_IMM, AArch64::LD4D);
       return;
     }
     break;
@@ -4790,10 +4800,14 @@ FunctionPass *llvm::createAArch64ISelDag(AArch64TargetMachine &TM,
 
 /// When \p PredVT is a scalable vector predicate in the form
 /// MVT::nx<M>xi1, it builds the correspondent scalable vector of
-/// integers MVT::nx<M>xi<bits> s.t. M x bits = 128. If the input
+/// integers MVT::nx<M>xi<bits> s.t. M x bits = 128. When targeting
+/// structured vectors (NumVec >1), the output data type is
+/// MVT::nx<M*NumVec>xi<bits> s.t. M x bits = 128. If the input
 /// PredVT is not in the form MVT::nx<M>xi1, it returns an invalid
 /// EVT.
-static EVT getPackedVectorTypeFromPredicateType(LLVMContext &Ctx, EVT PredVT) {
+static EVT getPackedVectorTypeFromPredicateType(LLVMContext &Ctx, EVT PredVT,
+                                                unsigned NumVec) {
+  assert(NumVec > 0 && NumVec < 5 && "Invalid number of vectors.");
   if (!PredVT.isScalableVector() || PredVT.getVectorElementType() != MVT::i1)
     return EVT();
 
@@ -4803,7 +4817,8 @@ static EVT getPackedVectorTypeFromPredicateType(LLVMContext &Ctx, EVT PredVT) {
 
   ElementCount EC = PredVT.getVectorElementCount();
   EVT ScalarVT = EVT::getIntegerVT(Ctx, AArch64::SVEBitsPerBlock / EC.Min);
-  EVT MemVT = EVT::getVectorVT(Ctx, ScalarVT, EC);
+  EVT MemVT = EVT::getVectorVT(Ctx, ScalarVT, EC * NumVec);
+
   return MemVT;
 }
 
@@ -4827,6 +4842,15 @@ static EVT getMemVTFromNode(LLVMContext &Ctx, SDNode *Root) {
     return cast<VTSDNode>(Root->getOperand(3))->getVT();
   case AArch64ISD::ST1_PRED:
     return cast<VTSDNode>(Root->getOperand(4))->getVT();
+  case AArch64ISD::SVE_LD2_MERGE_ZERO:
+    return getPackedVectorTypeFromPredicateType(
+        Ctx, Root->getOperand(1)->getValueType(0), /*NumVec=*/2);
+  case AArch64ISD::SVE_LD3_MERGE_ZERO:
+    return getPackedVectorTypeFromPredicateType(
+        Ctx, Root->getOperand(1)->getValueType(0), /*NumVec=*/3);
+  case AArch64ISD::SVE_LD4_MERGE_ZERO:
+    return getPackedVectorTypeFromPredicateType(
+        Ctx, Root->getOperand(1)->getValueType(0), /*NumVec=*/4);
   default:
     break;
   }
@@ -4842,7 +4866,7 @@ static EVT getMemVTFromNode(LLVMContext &Ctx, SDNode *Root) {
   // We are using an SVE prefetch intrinsic. Type must be inferred
   // from the width of the predicate.
   return getPackedVectorTypeFromPredicateType(
-      Ctx, Root->getOperand(2)->getValueType(0));
+      Ctx, Root->getOperand(2)->getValueType(0), /*NumVec=*/1);
 }
 
 /// SelectAddrModeIndexedSVE - Attempt selection of the addressing mode:
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-ldN-reg+imm-addr-mode.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-ldN-reg+imm-addr-mode.ll
new file mode 100644
index 0000000000000..1ffa78ec27352
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-ldN-reg+imm-addr-mode.ll
@@ -0,0 +1,495 @@
+; RUN: llc -mtriple=aarch64--linux-gnu -mattr=sve < %s | FileCheck %s
+
+; NOTE: invalid, upper and lower bound immediate values of the regimm
+; addressing mode are checked only for the byte version of each
+; instruction (`ld<N>b`), as the code for detecting the immediate is
+; common to all instructions, and varies only for the number of
+; elements of the structure store, which is <N> = 2, 3, 4.
+
+; ld2b
+define <vscale x 32 x i8> @ld2.nxv32i8(<vscale x 16 x i1> %Pg, <vscale x 16 x i8> *%addr) {
+; CHECK-LABEL: ld2.nxv32i8:
+; CHECK: ld2b { z0.b, z1.b }, p0/z, [x0, #2, mul vl]
+; CHECK-NEXT: ret
+%base = getelementptr <vscale x 16 x i8>, <vscale x 16 x i8>* %addr, i64 2
+%base_ptr = bitcast <vscale x 16 x i8>* %base to i8*
+%res = call <vscale x 32 x i8> @llvm.aarch64.sve.ld2.nxv32i8.nxv16i1.p0i8(<vscale x 16 x i1> %Pg, i8 *%base_ptr)
+ret <vscale x 32 x i8> %res
+}
+
+define <vscale x 32 x i8> @ld2.nxv32i8_lower_bound(<vscale x 16 x i1> %Pg, <vscale x 16 x i8> *%addr) {
+; CHECK-LABEL: ld2.nxv32i8_lower_bound:
+; CHECK: ld2b { z0.b, z1.b }, p0/z, [x0, #-16, mul vl]
+; CHECK-NEXT: ret
+%base = getelementptr <vscale x 16 x i8>, <vscale x 16 x i8>* %addr, i64 -16
+%base_ptr = bitcast <vscale x 16 x i8>* %base to i8 *
+%res = call <vscale x 32 x i8> @llvm.aarch64.sve.ld2.nxv32i8.nxv16i1.p0i8(<vscale x 16 x i1> %Pg, i8 *%base_ptr)
+ret <vscale x 32 x i8> %res
+}
+
+define <vscale x 32 x i8> @ld2.nxv32i8_upper_bound(<vscale x 16 x i1> %Pg, <vscale x 16 x i8> *%addr) {
+; CHECK-LABEL: ld2.nxv32i8_upper_bound:
+; CHECK: ld2b { z0.b, z1.b }, p0/z, [x0, #14, mul vl]
+; CHECK-NEXT: ret
+%base = getelementptr <vscale x 16 x i8>, <vscale x 16 x i8>* %addr, i64 14
+%base_ptr = bitcast <vscale x 16 x i8>* %base to i8 *
+%res = call <vscale x 32 x i8> @llvm.aarch64.sve.ld2.nxv32i8.nxv16i1.p0i8(<vscale x 16 x i1> %Pg, i8 *%base_ptr)
+ret <vscale x 32 x i8> %res
+}
+
+define <vscale x 32 x i8> @ld2.nxv32i8_not_multiple_of_2(<vscale x 16 x i1> %Pg, <vscale x 16 x i8> *%addr) {
+; CHECK-LABEL: ld2.nxv32i8_not_multiple_of_2:
+; CHECK:      rdvl x[[OFFSET:[0-9]]], #3
+; CHECK-NEXT: ld2b { z0.b, z1.b }, p0/z, [x0, x[[OFFSET]]]
+; CHECK-NEXT: ret
+%base = getelementptr <vscale x 16 x i8>, <vscale x 16 x i8>* %addr, i64 3
+%base_ptr = bitcast <vscale x 16 x i8>* %base to i8 *
+%res = call <vscale x 32 x i8> @llvm.aarch64.sve.ld2.nxv32i8.nxv16i1.p0i8(<vscale x 16 x i1> %Pg, i8 *%base_ptr)
+ret <vscale x 32 x i8> %res
+}
+
+define <vscale x 32 x i8> @ld2.nxv32i8_outside_lower_bound(<vscale x 16 x i1> %Pg, <vscale x 16 x i8> *%addr) {
+; CHECK-LABEL: ld2.nxv32i8_outside_lower_bound:
+; CHECK:      rdvl x[[OFFSET:[0-9]]], #-18
+; CHECK-NEXT: ld2b { z0.b, z1.b }, p0/z, [x0, x[[OFFSET]]]
+; CHECK-NEXT: ret
+%base = getelementptr <vscale x 16 x i8>, <vscale x 16 x i8>* %addr, i64 -18
+%base_ptr = bitcast <vscale x 16 x i8>* %base to i8 *
+%res = call <vscale x 32 x i8> @llvm.aarch64.sve.ld2.nxv32i8.nxv16i1.p0i8(<vscale x 16 x i1> %Pg, i8 *%base_ptr)
+ret <vscale x 32 x i8> %res
+}
+
+define <vscale x 32 x i8> @ld2.nxv32i8_outside_upper_bound(<vscale x 16 x i1> %Pg, <vscale x 16 x i8> *%addr) {
+; CHECK-LABEL: ld2.nxv32i8_outside_upper_bound:
+; CHECK:      rdvl x[[OFFSET:[0-9]]], #16
+; CHECK-NEXT: ld2b { z0.b, z1.b }, p0/z, [x0, x[[OFFSET]]]
+; CHECK-NEXT: ret
+%base = getelementptr <vscale x 16 x i8>, <vscale x 16 x i8>* %addr, i64 16
+%base_ptr = bitcast <vscale x 16 x i8>* %base to i8 *
+%res = call <vscale x 32 x i8> @llvm.aarch64.sve.ld2.nxv32i8.nxv16i1.p0i8(<vscale x 16 x i1> %Pg, i8 *%base_ptr)
+ret <vscale x 32 x i8> %res
+}
+
+; ld2h
+define <vscale x 16 x i16> @ld2.nxv16i16(<vscale x 8 x i1> %Pg, <vscale x 8 x i16>* %addr) {
+; CHECK-LABEL: ld2.nxv16i16:
+; CHECK: ld2h { z0.h, z1.h }, p0/z, [x0, #14, mul vl]
+; CHECK-NEXT: ret
+%base = getelementptr <vscale x 8 x i16>, <vscale x 8 x i16>* %addr, i64 14
+%base_ptr = bitcast <vscale x 8 x i16>* %base to i16 *
+%res = call <vscale x 16 x i16> @llvm.aarch64.sve.ld2.nxv16i16.nxv8i1.p0i16(<vscale x 8 x i1> %Pg, i16 *%base_ptr)
+ret <vscale x 16 x i16> %res
+}
+
+define <vscale x 16 x half> @ld2.nxv16f16(<vscale x 8 x i1> %Pg, <vscale x 8 x half>* %addr) {
+; CHECK-LABEL: ld2.nxv16f16:
+; CHECK: ld2h { z0.h, z1.h }, p0/z, [x0, #-16, mul vl]
+; CHECK-NEXT: ret
+%base = getelementptr <vscale x 8 x half>, <vscale x 8 x half>* %addr, i64 -16
+%base_ptr = bitcast <vscale x 8 x half>* %base to half *
+%res = call <vscale x 16 x half> @llvm.aarch64.sve.ld2.nxv16f16.nxv8i1.p0f16(<vscale x 8 x i1> %Pg, half *%base_ptr)
+ret <vscale x 16 x half> %res
+}
+
+define <vscale x 16 x bfloat> @ld2.nxv16bf16(<vscale x 8 x i1> %Pg, <vscale x 8 x bfloat>* %addr) #0 {
+; CHECK-LABEL: ld2.nxv16bf16:
+; CHECK: ld2h { z0.h, z1.h }, p0/z, [x0, #12, mul vl]
+; CHECK-NEXT: ret
+%base = getelementptr <vscale x 8 x bfloat>, <vscale x 8 x bfloat>* %addr, i64 12
+%base_ptr = bitcast <vscale x 8 x bfloat>* %base to bfloat *
+%res = call <vscale x 16 x bfloat> @llvm.aarch64.sve.ld2.nxv16bf16.nxv8i1.p0bf16(<vscale x 8 x i1> %Pg, bfloat *%base_ptr)
+ret <vscale x 16 x bfloat> %res
+}
+
+; ld2w
+define <vscale x 8 x i32> @ld2.nxv8i32(<vscale x 4 x i1> %Pg, <vscale x 4 x i32>* %addr) {
+; CHECK-LABEL: ld2.nxv8i32:
+; CHECK: ld2w { z0.s, z1.s }, p0/z, [x0, #14, mul vl]
+; CHECK-NEXT: ret
+%base = getelementptr <vscale x 4 x i32>, <vscale x 4 x i32>* %addr, i64 14
+%base_ptr = bitcast <vscale x 4 x i32>* %base to i32 *
+%res = call <vscale x 8 x i32> @llvm.aarch64.sve.ld2.nxv8i32.nxv4i1.p0i32(<vscale x 4 x i1> %Pg, i32 *%base_ptr)
+ret <vscale x 8 x i32> %res
+}
+
+define <vscale x 8 x float> @ld2.nxv8f32(<vscale x 4 x i1> %Pg, <vscale x 4 x float>* %addr) {
+; CHECK-LABEL: ld2.nxv8f32:
+; CHECK: ld2w { z0.s, z1.s }, p0/z, [x0, #-16, mul vl]
+; CHECK-NEXT: ret
+%base = getelementptr <vscale x 4 x float>, <vscale x 4 x float>* %addr, i64 -16
+%base_ptr = bitcast <vscale x 4 x float>* %base to float *
+%res = call <vscale x 8 x float> @llvm.aarch64.sve.ld2.nxv8f32.nxv4i1.p0f32(<vscale x 4 x i1> %Pg, float *%base_ptr)
+ret <vscale x 8 x float> %res
+}
+
+; ld2d
+define <vscale x 4 x i64> @ld2.nxv4i64(<vscale x 2 x i1> %Pg, <vscale x 2 x i64>* %addr) {
+; CHECK-LABEL: ld2.nxv4i64:
+; CHECK: ld2d { z0.d, z1.d }, p0/z, [x0, #14, mul vl]
+; CHECK-NEXT: ret
+%base = getelementptr <vscale x 2 x i64>, <vscale x 2 x i64>* %addr, i64 14
+%base_ptr = bitcast <vscale x 2 x i64>* %base to i64 *
+%res = call <vscale x 4 x i64> @llvm.aarch64.sve.ld2.nxv4i64.nxv2i1.p0i64(<vscale x 2 x i1> %Pg, i64 *%base_ptr)
+ret <vscale x 4 x i64> %res
+}
+
+define <vscale x 4 x double> @ld2.nxv4f64(<vscale x 2 x i1> %Pg, <vscale x 2 x double>* %addr) {
+; CHECK-LABEL: ld2.nxv4f64:
+; CHECK: ld2d { z0.d, z1.d }, p0/z, [x0, #-16, mul vl]
+; CHECK-NEXT: ret
+%base = getelementptr <vscale x 2 x double>, <vscale x 2 x double>* %addr, i64 -16
+%base_ptr = bitcast <vscale x 2 x double>* %base to double *
+%res = call <vscale x 4 x double> @llvm.aarch64.sve.ld2.nxv4f64.nxv2i1.p0f64(<vscale x 2 x i1> %Pg, double *%base_ptr)
+ret <vscale x 4 x double> %res
+}
+
+; ld3b
+define <vscale x 48 x i8> @ld3.nxv48i8(<vscale x 16 x i1> %Pg, <vscale x 16 x i8> *%addr) {
+; CHECK-LABEL: ld3.nxv48i8:
+; CHECK: ld3b { z0.b, z1.b, z2.b }, p0/z, [x0, #3, mul vl]
+; CHECK-NEXT: ret
+%base = getelementptr <vscale x 16 x i8>, <vscale x 16 x i8>* %addr, i64 3
+%base_ptr = bitcast <vscale x 16 x i8>* %base to i8 *
+%res = call <vscale x 48 x i8> @llvm.aarch64.sve.ld3.nxv48i8.nxv16i1.p0i8(<vscale x 16 x i1> %Pg, i8 *%base_ptr)
+ret <vscale x 48 x i8> %res
+}
+
+define <vscale x 48 x i8> @ld3.nxv48i8_lower_bound(<vscale x 16 x i1> %Pg, <vscale x 16 x i8> *%addr) {
+; CHECK-LABEL: ld3.nxv48i8_lower_bound:
+; CHECK: ld3b { z0.b, z1.b, z2.b }, p0/z, [x0, #-24, mul vl]
+; CHECK-NEXT: ret
+%base = getelementptr <vscale x 16 x i8>, <vscale x 16 x i8>* %addr, i64 -24
+%base_ptr = bitcast <vscale x 16 x i8>* %base to i8 *
+%res = call <vscale x 48 x i8> @llvm.aarch64.sve.ld3.nxv48i8.nxv16i1.p0i8(<vscale x 16 x i1> %Pg, i8 *%base_ptr)
+ret <vscale x 48 x i8> %res
+}
+
+define <vscale x 48 x i8> @ld3.nxv48i8_upper_bound(<vscale x 16 x i1> %Pg, <vscale x 16 x i8> *%addr) {
+; CHECK-LABEL: ld3.nxv48i8_upper_bound:
+; CHECK: ld3b { z0.b, z1.b, z2.b }, p0/z, [x0, #21, mul vl]
+; CHECK-NEXT: ret
+%base = getelementptr <vscale x 16 x i8>, <vscale x 16 x i8>* %addr, i64 21
+%base_ptr = bitcast <vscale x 16 x i8>* %base to i8 *
+%res = call <vscale x 48 x i8> @llvm.aarch64.sve.ld3.nxv48i8.nxv16i1.p0i8(<vscale x 16 x i1> %Pg, i8 *%base_ptr)
+ret <vscale x 48 x i8> %res
+}
+
+define <vscale x 48 x i8> @ld3.nxv48i8_not_multiple_of_3_01(<vscale x 16 x i1> %Pg, <vscale x 16 x i8> *%addr) {
+; CHECK-LABEL: ld3.nxv48i8_not_multiple_of_3_01:
+; CHECK:      rdvl x[[OFFSET:[0-9]]], #4
+; CHECK-NEXT: ld3b { z0.b, z1.b, z2.b }, p0/z, [x0, x[[OFFSET]]]
+; CHECK-NEXT: ret
+%base = getelementptr <vscale x 16 x i8>, <vscale x 16 x i8>* %addr, i64 4
+%base_ptr = bitcast <vscale x 16 x i8>* %base to i8 *
+%res = call <vscale x 48 x i8> @llvm.aarch64.sve.ld3.nxv48i8.nxv16i1.p0i8(<vscale x 16 x i1> %Pg, i8 *%base_ptr)
+ret <vscale x 48 x i8> %res
+}
+
+define <vscale x 48 x i8> @ld3.nxv48i8_not_multiple_of_3_02(<vscale x 16 x i1> %Pg, <vscale x 16 x i8> *%addr) {
+; CHECK-LABEL: ld3.nxv48i8_not_multiple_of_3_02:
+; CHECK:      rdvl x[[OFFSET:[0-9]]], #5
+; CHECK-NEXT: ld3b { z0.b, z1.b, z2.b }, p0/z, [x0, x[[OFFSET]]]
+; CHECK-NEXT: ret
+%base = getelementptr <vscale x 16 x i8>, <vscale x 16 x i8>* %addr, i64 5
+%base_ptr = bitcast <vscale x 16 x i8>* %base to i8 *
+%res = call <vscale x 48 x i8> @llvm.aarch64.sve.ld3.nxv48i8.nxv16i1.p0i8(<vscale x 16 x i1> %Pg, i8 *%base_ptr)
+ret <vscale x 48 x i8> %res
+}
+
+define <vscale x 48 x i8> @ld3.nxv48i8_outside_lower_bound(<vscale x 16 x i1> %Pg, <vscale x 16 x i8> *%addr) {
+; CHECK-LABEL: ld3.nxv48i8_outside_lower_bound:
+; CHECK:      rdvl x[[OFFSET:[0-9]]], #-27
+; CHECK-NEXT: ld3b { z0.b, z1.b, z2.b }, p0/z, [x0, x[[OFFSET]]]
+; CHECK-NEXT: ret
+%base = getelementptr <vscale x 16 x i8>, <vscale x 16 x i8>* %addr, i64 -27
+%base_ptr = bitcast <vscale x 16 x i8>* %base to i8 *
+%res = call <vscale x 48 x i8> @llvm.aarch64.sve.ld3.nxv48i8.nxv16i1.p0i8(<vscale x 16 x i1> %Pg, i8 *%base_ptr)
+ret <vscale x 48 x i8> %res
+}
+
+define <vscale x 48 x i8> @ld3.nxv48i8_outside_upper_bound(<vscale x 16 x i1> %Pg, <vscale x 16 x i8> *%addr) {
+; CHECK-LABEL: ld3.nxv48i8_outside_upper_bound:
+; CHECK:      rdvl x[[OFFSET:[0-9]]], #24
+; CHECK-NEXT: ld3b { z0.b, z1.b, z2.b }, p0/z, [x0, x[[OFFSET]]]
+; CHECK-NEXT: ret
+%base = getelementptr <vscale x 16 x i8>, <vscale x 16 x i8>* %addr, i64 24
+%base_ptr = bitcast <vscale x 16 x i8>* %base to i8 *
+%res = call <vscale x 48 x i8> @llvm.aarch64.sve.ld3.nxv48i8.nxv16i1.p0i8(<vscale x 16 x i1> %Pg, i8 *%base_ptr)
+ret <vscale x 48 x i8> %res
+}
+
+; ld3h
+define <vscale x 24 x i16> @ld3.nxv24i16(<vscale x 8 x i1> %Pg, <vscale x 8 x i16> *%addr) {
+; CHECK-LABEL: ld3.nxv24i16:
+; CHECK: ld3h { z0.h, z1.h, z2.h }, p0/z, [x0, #21, mul vl]
+; CHECK-NEXT: ret
+%base = getelementptr <vscale x 8 x i16>, <vscale x 8 x i16>* %addr, i64 21
+%base_ptr = bitcast <vscale x 8 x i16>* %base to i16 *
+%res = call <vscale x 24 x i16> @llvm.aarch64.sve.ld3.nxv24i16.nxv8i1.p0i16(<vscale x 8 x i1> %Pg, i16 *%base_ptr)
+ret <vscale x 24 x i16> %res
+}
+
+define <vscale x 24 x half> @ld3.nxv24f16(<vscale x 8 x i1> %Pg, <vscale x 8 x half> *%addr) {
+; CHECK-LABEL: ld3.nxv24f16:
+; CHECK: ld3h { z0.h, z1.h, z2.h }, p0/z, [x0, #21, mul vl]
+; CHECK-NEXT: ret
+%base = getelementptr <vscale x 8 x half>, <vscale x 8 x half>* %addr, i64 21
+%base_ptr = bitcast <vscale x 8 x half>* %base to half *
+%res = call <vscale x 24 x half> @llvm.aarch64.sve.ld3.nxv24f16.nxv8i1.p0f16(<vscale x 8 x i1> %Pg, half *%base_ptr)
+ret <vscale x 24 x half> %res
+}
+
+define <vscale x 24 x bfloat> @ld3.nxv24bf16(<vscale x 8 x i1> %Pg, <vscale x 8 x bfloat> *%addr) #0 {
+; CHECK-LABEL: ld3.nxv24bf16:
+; CHECK: ld3h { z0.h, z1.h, z2.h }, p0/z, [x0, #-24, mul vl]
+; CHECK-NEXT: ret
+%base = getelementptr <vscale x 8 x bfloat>, <vscale x 8 x bfloat>* %addr, i64 -24
+%base_ptr = bitcast <vscale x 8 x bfloat>* %base to bfloat *
+%res = call <vscale x 24 x bfloat> @llvm.aarch64.sve.ld3.nxv24bf16.nxv8i1.p0bf16(<vscale x 8 x i1> %Pg, bfloat *%base_ptr)
+ret <vscale x 24 x bfloat> %res
+}
+
+; ld3w
+define <vscale x 12 x i32> @ld3.nxv12i32(<vscale x 4 x i1> %Pg, <vscale x 4 x i32> *%addr) {
+; CHECK-LABEL: ld3.nxv12i32:
+; CHECK: ld3w { z0.s, z1.s, z2.s }, p0/z, [x0, #21, mul vl]
+; CHECK-NEXT: ret
+%base = getelementptr <vscale x 4 x i32>, <vscale x 4 x i32>* %addr, i64 21
+%base_ptr = bitcast <vscale x 4 x i32>* %base to i32 *
+%res = call <vscale x 12 x i32> @llvm.aarch64.sve.ld3.nxv12i32.nxv4i1.p0i32(<vscale x 4 x i1> %Pg, i32 *%base_ptr)
+ret <vscale x 12 x i32> %res
+}
+
+define <vscale x 12 x float> @ld3.nxv12f32(<vscale x 4 x i1> %Pg, <vscale x 4 x float> *%addr) {
+; CHECK-LABEL: ld3.nxv12f32:
+; CHECK: ld3w { z0.s, z1.s, z2.s }, p0/z, [x0, #-24, mul vl]
+; CHECK-NEXT: ret
+%base = getelementptr <vscale x 4 x float>, <vscale x 4 x float>* %addr, i64 -24
+%base_ptr = bitcast <vscale x 4 x float>* %base to float *
+%res = call <vscale x 12 x float> @llvm.aarch64.sve.ld3.nxv12f32.nxv4i1.p0f32(<vscale x 4 x i1> %Pg, float *%base_ptr)
+ret <vscale x 12 x float> %res
+}
+
+; ld3d
+define <vscale x 6 x i64> @ld3.nxv6i64(<vscale x 2 x i1> %Pg, <vscale x 2 x i64> *%addr) {
+; CHECK-LABEL: ld3.nxv6i64:
+; CHECK: ld3d { z0.d, z1.d, z2.d }, p0/z, [x0, #21, mul vl]
+; CHECK-NEXT: ret
+%base = getelementptr <vscale x 2 x i64>, <vscale x 2 x i64>* %addr, i64 21
+%base_ptr = bitcast <vscale x 2 x i64>* %base to i64 *
+%res = call <vscale x 6 x i64> @llvm.aarch64.sve.ld3.nxv6i64.nxv2i1.p0i64(<vscale x 2 x i1> %Pg, i64 *%base_ptr)
+ret <vscale x 6 x i64> %res
+}
+
+define <vscale x 6 x double> @ld3.nxv6f64(<vscale x 2 x i1> %Pg, <vscale x 2 x double> *%addr) {
+; CHECK-LABEL: ld3.nxv6f64:
+; CHECK: ld3d { z0.d, z1.d, z2.d }, p0/z, [x0, #-24, mul vl]
+; CHECK-NEXT: ret
+%base = getelementptr <vscale x 2 x double>, <vscale x 2 x double>* %addr, i64 -24
+%base_ptr = bitcast <vscale x 2 x double>* %base to double *
+%res = call <vscale x 6 x double> @llvm.aarch64.sve.ld3.nxv6f64.nxv2i1.p0f64(<vscale x 2 x i1> %Pg, double *%base_ptr)
+ret <vscale x 6 x double> %res
+}
+
+; ; ld4b
+define <vscale x 64 x i8> @ld4.nxv64i8(<vscale x 16 x i1> %Pg, <vscale x 16 x i8> *%addr) {
+; CHECK-LABEL: ld4.nxv64i8:
+; CHECK: ld4b { z0.b, z1.b, z2.b, z3.b }, p0/z, [x0, #4, mul vl]
+; CHECK-NEXT: ret
+%base = getelementptr <vscale x 16 x i8>, <vscale x 16 x i8>* %addr, i64 4
+%base_ptr = bitcast <vscale x 16 x i8>* %base to i8 *
+%res = call <vscale x 64 x i8> @llvm.aarch64.sve.ld4.nxv64i8.nxv16i1.p0i8(<vscale x 16 x i1> %Pg, i8 *%base_ptr)
+ret <vscale x 64 x i8> %res
+}
+
+define <vscale x 64 x i8> @ld4.nxv64i8_lower_bound(<vscale x 16 x i1> %Pg, <vscale x 16 x i8> *%addr) {
+; CHECK-LABEL: ld4.nxv64i8_lower_bound:
+; CHECK: ld4b { z0.b, z1.b, z2.b, z3.b }, p0/z, [x0, #-32, mul vl]
+; CHECK-NEXT: ret
+%base = getelementptr <vscale x 16 x i8>, <vscale x 16 x i8>* %addr, i64 -32
+%base_ptr = bitcast <vscale x 16 x i8>* %base to i8 *
+%res = call <vscale x 64 x i8> @llvm.aarch64.sve.ld4.nxv64i8.nxv16i1.p0i8(<vscale x 16 x i1> %Pg, i8 *%base_ptr)
+ret <vscale x 64 x i8> %res
+}
+
+define <vscale x 64 x i8> @ld4.nxv64i8_upper_bound(<vscale x 16 x i1> %Pg, <vscale x 16 x i8> *%addr) {
+; CHECK-LABEL: ld4.nxv64i8_upper_bound:
+; CHECK: ld4b { z0.b, z1.b, z2.b, z3.b }, p0/z, [x0, #28, mul vl]
+; CHECK-NEXT: ret
+%base = getelementptr <vscale x 16 x i8>, <vscale x 16 x i8>* %addr, i64 28
+%base_ptr = bitcast <vscale x 16 x i8>* %base to i8 *
+%res = call <vscale x 64 x i8> @llvm.aarch64.sve.ld4.nxv64i8.nxv16i1.p0i8(<vscale x 16 x i1> %Pg, i8 *%base_ptr)
+ret <vscale x 64 x i8> %res
+}
+
+define <vscale x 64 x i8> @ld4.nxv64i8_not_multiple_of_4_01(<vscale x 16 x i1> %Pg, <vscale x 16 x i8> *%addr) {
+; CHECK-LABEL: ld4.nxv64i8_not_multiple_of_4_01:
+; CHECK:      rdvl x[[OFFSET:[0-9]]], #5
+; CHECK-NEXT: ld4b { z0.b, z1.b, z2.b, z3.b }, p0/z, [x0, x[[OFFSET]]]
+; CHECK-NEXT: ret
+%base = getelementptr <vscale x 16 x i8>, <vscale x 16 x i8>* %addr, i64 5
+%base_ptr = bitcast <vscale x 16 x i8>* %base to i8 *
+%res = call <vscale x 64 x i8> @llvm.aarch64.sve.ld4.nxv64i8.nxv16i1.p0i8(<vscale x 16 x i1> %Pg, i8 *%base_ptr)
+ret <vscale x 64 x i8> %res
+}
+
+define <vscale x 64 x i8> @ld4.nxv64i8_not_multiple_of_4_02(<vscale x 16 x i1> %Pg, <vscale x 16 x i8> *%addr) {
+; CHECK-LABEL: ld4.nxv64i8_not_multiple_of_4_02:
+; CHECK:      rdvl x[[OFFSET:[0-9]]], #6
+; CHECK-NEXT: ld4b { z0.b, z1.b, z2.b, z3.b }, p0/z, [x0, x[[OFFSET]]]
+; CHECK-NEXT: ret
+%base = getelementptr <vscale x 16 x i8>, <vscale x 16 x i8>* %addr, i64 6
+%base_ptr = bitcast <vscale x 16 x i8>* %base to i8 *
+%res = call <vscale x 64 x i8> @llvm.aarch64.sve.ld4.nxv64i8.nxv16i1.p0i8(<vscale x 16 x i1> %Pg, i8 *%base_ptr)
+ret <vscale x 64 x i8> %res
+}
+
+define <vscale x 64 x i8> @ld4.nxv64i8_not_multiple_of_4_03(<vscale x 16 x i1> %Pg, <vscale x 16 x i8> *%addr) {
+; CHECK-LABEL: ld4.nxv64i8_not_multiple_of_4_03:
+; CHECK:      rdvl x[[OFFSET:[0-9]]], #7
+; CHECK-NEXT: ld4b { z0.b, z1.b, z2.b, z3.b }, p0/z, [x0, x[[OFFSET]]]
+; CHECK-NEXT: ret
+%base = getelementptr <vscale x 16 x i8>, <vscale x 16 x i8>* %addr, i64 7
+%base_ptr = bitcast <vscale x 16 x i8>* %base to i8 *
+%res = call <vscale x 64 x i8> @llvm.aarch64.sve.ld4.nxv64i8.nxv16i1.p0i8(<vscale x 16 x i1> %Pg, i8 *%base_ptr)
+ret <vscale x 64 x i8> %res
+}
+
+define <vscale x 64 x i8> @ld4.nxv64i8_outside_lower_bound(<vscale x 16 x i1> %Pg, <vscale x 16 x i8> *%addr) {
+; CHECK-LABEL: ld4.nxv64i8_outside_lower_bound:
+; FIXME: optimize OFFSET computation so that xOFFSET = (mul (RDVL #4) #9)
+; xM = -9 * 2^6
+; xP = RDVL * 2^-4 
+; xOFFSET = RDVL * 2^-4 * -9 * 2^6 = RDVL * -36
+; CHECK:      rdvl x[[N:[0-9]]], #1
+; CHECK-DAG:  mov  x[[M:[0-9]]], #-576
+; CHECK-DAG:  lsr  x[[P:[0-9]]], x[[N]], #4
+; CHECK-DAG:  mul  x[[OFFSET:[0-9]]], x[[P]], x[[M]]
+; CHECK-NEXT: ld4b { z0.b, z1.b, z2.b, z3.b }, p0/z, [x0, x[[OFFSET]]]
+; CHECK-NEXT: ret
+%base = getelementptr <vscale x 16 x i8>, <vscale x 16 x i8>* %addr, i64 -36
+%base_ptr = bitcast <vscale x 16 x i8>* %base to i8 *
+%res = call <vscale x 64 x i8> @llvm.aarch64.sve.ld4.nxv64i8.nxv16i1.p0i8(<vscale x 16 x i1> %Pg, i8 *%base_ptr)
+ret <vscale x 64 x i8> %res
+}
+
+define <vscale x 64 x i8> @ld4.nxv64i8_outside_upper_bound(<vscale x 16 x i1> %Pg, <vscale x 16 x i8> *%addr) {
+; CHECK-LABEL: ld4.nxv64i8_outside_upper_bound:
+; FIXME: optimize OFFSET computation so that xOFFSET = (mul (RDVL #16) #2)
+; xM = 2^9
+; xP = RDVL * 2^-4
+; xOFFSET = RDVL * 2^-4 * 2^9 = RDVL * 32
+; CHECK:      rdvl x[[N:[0-9]]], #1
+; CHECK-DAG:  mov  w[[M:[0-9]]], #512
+; CHECK-DAG:  lsr  x[[P:[0-9]]], x[[N]], #4
+; CHECK-DAG:  mul  x[[OFFSET:[0-9]]], x[[P]], x[[M]]
+; CHECK-NEXT: ld4b { z0.b, z1.b, z2.b, z3.b }, p0/z, [x0, x[[OFFSET]]]
+; CHECK-NEXT: ret
+%base = getelementptr <vscale x 16 x i8>, <vscale x 16 x i8>* %addr, i64 32
+%base_ptr = bitcast <vscale x 16 x i8>* %base to i8 *
+%res = call <vscale x 64 x i8> @llvm.aarch64.sve.ld4.nxv64i8.nxv16i1.p0i8(<vscale x 16 x i1> %Pg, i8 *%base_ptr)
+ret <vscale x 64 x i8> %res
+}
+
+; ld4h
+define <vscale x 32 x i16> @ld4.nxv32i16(<vscale x 8 x i1> %Pg, <vscale x 8 x i16> *%addr) {
+; CHECK-LABEL: ld4.nxv32i16:
+; CHECK: ld4h { z0.h, z1.h, z2.h, z3.h }, p0/z, [x0, #8, mul vl]
+; CHECK-NEXT: ret
+%base = getelementptr <vscale x 8 x i16>, <vscale x 8 x i16>* %addr, i64 8
+%base_ptr = bitcast <vscale x 8 x i16>* %base to i16 *
+%res = call <vscale x 32 x i16> @llvm.aarch64.sve.ld4.nxv32i16.nxv8i1.p0i16(<vscale x 8 x i1> %Pg, i16 *%base_ptr)
+ret <vscale x 32 x i16> %res
+}
+
+define <vscale x 32 x half> @ld4.nxv32f16(<vscale x 8 x i1> %Pg, <vscale x 8 x half> *%addr) {
+; CHECK-LABEL: ld4.nxv32f16:
+; CHECK: ld4h { z0.h, z1.h, z2.h, z3.h }, p0/z, [x0, #28, mul vl]
+; CHECK-NEXT: ret
+%base = getelementptr <vscale x 8 x half>, <vscale x 8 x half>* %addr, i64 28
+%base_ptr = bitcast <vscale x 8 x half>* %base to half *
+%res = call <vscale x 32 x half> @llvm.aarch64.sve.ld4.nxv32f16.nxv8i1.p0f16(<vscale x 8 x i1> %Pg, half *%base_ptr)
+ret <vscale x 32 x half> %res
+}
+
+define <vscale x 32 x bfloat> @ld4.nxv32bf16(<vscale x 8 x i1> %Pg, <vscale x 8 x bfloat> *%addr) #0 {
+; CHECK-LABEL: ld4.nxv32bf16:
+; CHECK: ld4h { z0.h, z1.h, z2.h, z3.h }, p0/z, [x0, #-32, mul vl]
+; CHECK-NEXT: ret
+%base = getelementptr <vscale x 8 x bfloat>, <vscale x 8 x bfloat>* %addr, i64 -32
+%base_ptr = bitcast <vscale x 8 x bfloat>* %base to bfloat *
+%res = call <vscale x 32 x bfloat> @llvm.aarch64.sve.ld4.nxv32bf16.nxv8i1.p0bf16(<vscale x 8 x i1> %Pg, bfloat *%base_ptr)
+ret <vscale x 32 x bfloat> %res
+}
+
+; ld4w
+define <vscale x 16 x i32> @ld4.nxv16i32(<vscale x 4 x i1> %Pg, <vscale x 4 x i32> *%addr) {
+; CHECK-LABEL: ld4.nxv16i32:
+; CHECK: ld4w { z0.s, z1.s, z2.s, z3.s }, p0/z, [x0, #28, mul vl]
+; CHECK-NEXT: ret
+%base = getelementptr <vscale x 4 x i32>, <vscale x 4 x i32>* %addr, i64 28
+%base_ptr = bitcast <vscale x 4 x i32>* %base to i32 *
+%res = call <vscale x 16 x i32> @llvm.aarch64.sve.ld4.nxv16i32.nxv4i1.p0i32(<vscale x 4 x i1> %Pg, i32 *%base_ptr)
+ret <vscale x 16 x i32> %res
+}
+
+define <vscale x 16 x float> @ld4.nxv16f32(<vscale x 4 x i1> %Pg, <vscale x 4 x float>* %addr) {
+; CHECK-LABEL: ld4.nxv16f32:
+; CHECK: ld4w { z0.s, z1.s, z2.s, z3.s }, p0/z, [x0, #-32, mul vl]
+; CHECK-NEXT: ret
+%base = getelementptr <vscale x 4 x float>, <vscale x 4 x float>* %addr, i64 -32
+%base_ptr = bitcast <vscale x 4 x float>* %base to float *
+%res = call <vscale x 16 x float> @llvm.aarch64.sve.ld4.nxv16f32.nxv4i1.p0f32(<vscale x 4 x i1> %Pg, float *%base_ptr)
+ret <vscale x 16 x float> %res
+}
+
+; ld4d
+define <vscale x 8 x i64> @ld4.nxv8i64(<vscale x 2 x i1> %Pg, <vscale x 2 x i64> *%addr) {
+; CHECK-LABEL: ld4.nxv8i64:
+; CHECK: ld4d { z0.d, z1.d, z2.d, z3.d }, p0/z, [x0, #28, mul vl]
+; CHECK-NEXT: ret
+%base = getelementptr <vscale x 2 x i64>, <vscale x 2 x i64>* %addr, i64 28
+%base_ptr = bitcast <vscale x 2 x i64>* %base to i64 *
+%res = call <vscale x 8 x i64> @llvm.aarch64.sve.ld4.nxv8i64.nxv2i1.p0i64(<vscale x 2 x i1> %Pg, i64 *%base_ptr)
+ret <vscale x 8 x i64> %res
+}
+
+define <vscale x 8 x double> @ld4.nxv8f64(<vscale x 2 x i1> %Pg, <vscale x 2 x double> *%addr) {
+; CHECK-LABEL: ld4.nxv8f64:
+; CHECK: ld4d { z0.d, z1.d, z2.d, z3.d }, p0/z, [x0, #-32, mul vl]
+; CHECK-NEXT: ret
+%base = getelementptr <vscale x 2 x double>, <vscale x 2 x double>* %addr, i64 -32
+%base_ptr = bitcast <vscale x 2 x double>* %base to double *
+%res = call <vscale x 8 x double> @llvm.aarch64.sve.ld4.nxv8f64.nxv2i1.p0f64(<vscale x 2 x i1> %Pg, double * %base_ptr)
+ret <vscale x 8 x double> %res
+}
+
+declare <vscale x 32 x i8> @llvm.aarch64.sve.ld2.nxv32i8.nxv16i1.p0i8(<vscale x 16 x i1>, i8*)
+declare <vscale x 16 x i16> @llvm.aarch64.sve.ld2.nxv16i16.nxv8i1.p0i16(<vscale x 8 x i1>, i16*)
+declare <vscale x 8 x i32> @llvm.aarch64.sve.ld2.nxv8i32.nxv4i1.p0i32(<vscale x 4 x i1>, i32*)
+declare <vscale x 4 x i64> @llvm.aarch64.sve.ld2.nxv4i64.nxv2i1.p0i64(<vscale x 2 x i1>, i64*)
+declare <vscale x 16 x half> @llvm.aarch64.sve.ld2.nxv16f16.nxv8i1.p0f16(<vscale x 8 x i1>, half*)
+declare <vscale x 16 x bfloat> @llvm.aarch64.sve.ld2.nxv16bf16.nxv8i1.p0bf16(<vscale x 8 x i1>, bfloat*)
+declare <vscale x 8 x float> @llvm.aarch64.sve.ld2.nxv8f32.nxv4i1.p0f32(<vscale x 4 x i1>, float*)
+declare <vscale x 4 x double> @llvm.aarch64.sve.ld2.nxv4f64.nxv2i1.p0f64(<vscale x 2 x i1>, double*)
+
+declare <vscale x 48 x i8> @llvm.aarch64.sve.ld3.nxv48i8.nxv16i1.p0i8(<vscale x 16 x i1>, i8*)
+declare <vscale x 24 x i16> @llvm.aarch64.sve.ld3.nxv24i16.nxv8i1.p0i16(<vscale x 8 x i1>, i16*)
+declare <vscale x 12 x i32> @llvm.aarch64.sve.ld3.nxv12i32.nxv4i1.p0i32(<vscale x 4 x i1>, i32*)
+declare <vscale x 6 x i64> @llvm.aarch64.sve.ld3.nxv6i64.nxv2i1.p0i64(<vscale x 2 x i1>, i64*)
+declare <vscale x 24 x half> @llvm.aarch64.sve.ld3.nxv24f16.nxv8i1.p0f16(<vscale x 8 x i1>, half*)
+declare <vscale x 24 x bfloat> @llvm.aarch64.sve.ld3.nxv24bf16.nxv8i1.p0bf16(<vscale x 8 x i1>, bfloat*)
+declare <vscale x 12 x float> @llvm.aarch64.sve.ld3.nxv12f32.nxv4i1.p0f32(<vscale x 4 x i1>, float*)
+declare <vscale x 6 x double> @llvm.aarch64.sve.ld3.nxv6f64.nxv2i1.p0f64(<vscale x 2 x i1>, double*)
+
+declare <vscale x 64 x i8> @llvm.aarch64.sve.ld4.nxv64i8.nxv16i1.p0i8(<vscale x 16 x i1>, i8*)
+declare <vscale x 32 x i16> @llvm.aarch64.sve.ld4.nxv32i16.nxv8i1.p0i16(<vscale x 8 x i1>, i16*)
+declare <vscale x 16 x i32> @llvm.aarch64.sve.ld4.nxv16i32.nxv4i1.p0i32(<vscale x 4 x i1>, i32*)
+declare <vscale x 8 x i64> @llvm.aarch64.sve.ld4.nxv8i64.nxv2i1.p0i64(<vscale x 2 x i1>, i64*)
+declare <vscale x 32 x half> @llvm.aarch64.sve.ld4.nxv32f16.nxv8i1.p0f16(<vscale x 8 x i1>, half*)
+declare <vscale x 32 x bfloat> @llvm.aarch64.sve.ld4.nxv32bf16.nxv8i1.p0bf16(<vscale x 8 x i1>, bfloat*)
+declare <vscale x 16 x float> @llvm.aarch64.sve.ld4.nxv16f32.nxv4i1.p0f32(<vscale x 4 x i1>, float*)
+declare <vscale x 8 x double> @llvm.aarch64.sve.ld4.nxv8f64.nxv2i1.p0f64(<vscale x 2 x i1>, double*)
+
+; +bf16 is required for the bfloat version.
+attributes #0 = { "target-features"="+sve,+bf16" }
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-ldN-reg+reg-addr-mode.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-ldN-reg+reg-addr-mode.ll
new file mode 100644
index 0000000000000..ab59c84137958
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-ldN-reg+reg-addr-mode.ll
@@ -0,0 +1,259 @@
+; RUN: llc -mtriple=aarch64--linux-gnu -mattr=sve < %s | FileCheck %s
+
+; ld2b
+define <vscale x 32 x i8> @ld2.nxv32i8(<vscale x 16 x i1> %Pg, i8 *%addr, i64 %a) {
+; CHECK-LABEL: ld2.nxv32i8:
+; CHECK: ld2b { z0.b, z1.b }, p0/z, [x0, x1]
+; CHECK-NEXT: ret
+%addr2 = getelementptr i8, i8 *  %addr, i64 %a
+%res = call <vscale x 32 x i8> @llvm.aarch64.sve.ld2.nxv32i8.nxv16i1.p0i8(<vscale x 16 x i1> %Pg, i8 *%addr2)
+ret <vscale x 32 x i8> %res
+}
+
+; ld2h
+define <vscale x 16 x i16> @ld2.nxv16i16(<vscale x 8 x i1> %Pg, i16 *%addr, i64 %a) {
+; CHECK-LABEL: ld2.nxv16i16:
+; CHECK: ld2h { z0.h, z1.h }, p0/z, [x0, x1, lsl #1]
+; CHECK-NEXT: ret
+%addr2 = getelementptr i16, i16 *  %addr, i64 %a
+%res = call <vscale x 16 x i16> @llvm.aarch64.sve.ld2.nxv16i16.nxv8i1.p0i16(<vscale x 8 x i1> %Pg, i16 *%addr2)
+ret <vscale x 16 x i16> %res
+}
+
+define <vscale x 16 x half> @ld2.nxv16f16(<vscale x 8 x i1> %Pg, half *%addr, i64 %a) {
+; CHECK-LABEL: ld2.nxv16f16:
+; CHECK: ld2h { z0.h, z1.h }, p0/z, [x0, x1, lsl #1]
+; CHECK-NEXT: ret
+%addr2 = getelementptr half, half *  %addr, i64 %a
+%res = call <vscale x 16 x half> @llvm.aarch64.sve.ld2.nxv16f16.nxv8i1.p0f16(<vscale x 8 x i1> %Pg, half *%addr2)
+ret <vscale x 16 x half> %res
+}
+
+define <vscale x 16 x bfloat> @ld2.nxv16bf16(<vscale x 8 x i1> %Pg, bfloat *%addr, i64 %a) #0 {
+; CHECK-LABEL: ld2.nxv16bf16:
+; CHECK: ld2h { z0.h, z1.h }, p0/z, [x0, x1, lsl #1]
+; CHECK-NEXT: ret
+%addr2 = getelementptr bfloat, bfloat *  %addr, i64 %a
+%res = call <vscale x 16 x bfloat> @llvm.aarch64.sve.ld2.nxv16bf16.nxv8i1.p0bf16(<vscale x 8 x i1> %Pg, bfloat *%addr2)
+ret <vscale x 16 x bfloat> %res
+}
+
+; ld2w
+define <vscale x 8 x i32> @ld2.nxv8i32(<vscale x 4 x i1> %Pg, i32 *%addr, i64 %a) {
+; CHECK-LABEL: ld2.nxv8i32:
+; CHECK: ld2w { z0.s, z1.s }, p0/z, [x0, x1, lsl #2]
+; CHECK-NEXT: ret
+%addr2 = getelementptr i32, i32 *  %addr, i64 %a
+%res = call <vscale x 8 x i32> @llvm.aarch64.sve.ld2.nxv8i32.nxv4i1.p0i32(<vscale x 4 x i1> %Pg, i32 *%addr2)
+ret <vscale x 8 x i32> %res
+}
+
+define <vscale x 8 x float> @ld2.nxv8f32(<vscale x 4 x i1> %Pg, float *%addr, i64 %a) {
+; CHECK-LABEL: ld2.nxv8f32:
+; CHECK: ld2w { z0.s, z1.s }, p0/z, [x0, x1, lsl #2]
+; CHECK-NEXT: ret
+%addr2 = getelementptr float, float *  %addr, i64 %a
+%res = call <vscale x 8 x float> @llvm.aarch64.sve.ld2.nxv8f32.nxv4i1.p0f32(<vscale x 4 x i1> %Pg, float *%addr2)
+ret <vscale x 8 x float> %res
+}
+
+; ld2d
+define <vscale x 4 x i64> @ld2.nxv4i64(<vscale x 2 x i1> %Pg, i64 *%addr, i64 %a) {
+; CHECK-LABEL: ld2.nxv4i64:
+; CHECK: ld2d { z0.d, z1.d }, p0/z, [x0, x1, lsl #3]
+; CHECK-NEXT: ret
+%addr2 = getelementptr i64, i64 *  %addr, i64 %a
+%res = call <vscale x 4 x i64> @llvm.aarch64.sve.ld2.nxv4i64.nxv2i1.p0i64(<vscale x 2 x i1> %Pg, i64 *%addr2)
+ret <vscale x 4 x i64> %res
+}
+
+define <vscale x 4 x double> @ld2.nxv4f64(<vscale x 2 x i1> %Pg, double *%addr, i64 %a) {
+; CHECK-LABEL: ld2.nxv4f64:
+; CHECK: ld2d { z0.d, z1.d }, p0/z, [x0, x1, lsl #3]
+; CHECK-NEXT: ret
+%addr2 = getelementptr double, double *  %addr, i64 %a
+%res = call <vscale x 4 x double> @llvm.aarch64.sve.ld2.nxv4f64.nxv2i1.p0f64(<vscale x 2 x i1> %Pg, double *%addr2)
+ret <vscale x 4 x double> %res
+}
+
+; ld3b
+define <vscale x 48 x i8> @ld3.nxv48i8(<vscale x 16 x i1> %Pg, i8 *%addr, i64 %a) {
+; CHECK-LABEL: ld3.nxv48i8:
+; CHECK: ld3b { z0.b, z1.b, z2.b }, p0/z, [x0, x1]
+; CHECK-NEXT: ret
+%addr2 = getelementptr i8, i8 *  %addr, i64 %a
+%res = call <vscale x 48 x i8> @llvm.aarch64.sve.ld3.nxv48i8.nxv16i1.p0i8(<vscale x 16 x i1> %Pg, i8 *%addr2)
+ret <vscale x 48 x i8> %res
+}
+
+; ld3h
+define <vscale x 24 x i16> @ld3.nxv24i16(<vscale x 8 x i1> %Pg, i16 *%addr, i64 %a) {
+; CHECK-LABEL: ld3.nxv24i16:
+; CHECK: ld3h { z0.h, z1.h, z2.h }, p0/z, [x0, x1, lsl #1]
+; CHECK-NEXT: ret
+%addr2 = getelementptr i16, i16 *  %addr, i64 %a
+%res = call <vscale x 24 x i16> @llvm.aarch64.sve.ld3.nxv24i16.nxv8i1.p0i16(<vscale x 8 x i1> %Pg, i16 *%addr2)
+ret <vscale x 24 x i16> %res
+}
+
+define <vscale x 24 x half> @ld3.nxv24f16(<vscale x 8 x i1> %Pg, half *%addr, i64 %a) {
+; CHECK-LABEL: ld3.nxv24f16:
+; CHECK: ld3h { z0.h, z1.h, z2.h }, p0/z, [x0, x1, lsl #1]
+; CHECK-NEXT: ret
+%addr2 = getelementptr half, half *  %addr, i64 %a
+%res = call <vscale x 24 x half> @llvm.aarch64.sve.ld3.nxv24f16.nxv8i1.p0f16(<vscale x 8 x i1> %Pg, half *%addr2)
+ret <vscale x 24 x half> %res
+}
+
+define <vscale x 24 x bfloat> @ld3.nxv24bf16(<vscale x 8 x i1> %Pg, bfloat *%addr, i64 %a) #0 {
+; CHECK-LABEL: ld3.nxv24bf16:
+; CHECK: ld3h { z0.h, z1.h, z2.h }, p0/z, [x0, x1, lsl #1]
+; CHECK-NEXT: ret
+%addr2 = getelementptr bfloat, bfloat *  %addr, i64 %a
+%res = call <vscale x 24 x bfloat> @llvm.aarch64.sve.ld3.nxv24bf16.nxv8i1.p0bf16(<vscale x 8 x i1> %Pg, bfloat *%addr2)
+ret <vscale x 24 x bfloat> %res
+}
+
+; ld3w
+define <vscale x 12 x i32> @ld3.nxv12i32(<vscale x 4 x i1> %Pg, i32 *%addr, i64 %a) {
+; CHECK-LABEL: ld3.nxv12i32:
+; CHECK: ld3w { z0.s, z1.s, z2.s }, p0/z, [x0, x1, lsl #2]
+; CHECK-NEXT: ret
+%addr2 = getelementptr i32, i32 *  %addr, i64 %a
+%res = call <vscale x 12 x i32> @llvm.aarch64.sve.ld3.nxv12i32.nxv4i1.p0i32(<vscale x 4 x i1> %Pg, i32 *%addr2)
+ret <vscale x 12 x i32> %res
+}
+
+define <vscale x 12 x float> @ld3.nxv12f32(<vscale x 4 x i1> %Pg, float *%addr, i64 %a) {
+; CHECK-LABEL: ld3.nxv12f32:
+; CHECK: ld3w { z0.s, z1.s, z2.s }, p0/z, [x0, x1, lsl #2]
+; CHECK-NEXT: ret
+%addr2 = getelementptr float, float *  %addr, i64 %a
+%res = call <vscale x 12 x float> @llvm.aarch64.sve.ld3.nxv12f32.nxv4i1.p0f32(<vscale x 4 x i1> %Pg, float *%addr2)
+ret <vscale x 12 x float> %res
+}
+
+; ld3d
+define <vscale x 6 x i64> @ld3.nxv6i64(<vscale x 2 x i1> %Pg, i64 *%addr, i64 %a) {
+; CHECK-LABEL: ld3.nxv6i64:
+; CHECK: ld3d { z0.d, z1.d, z2.d }, p0/z, [x0, x1, lsl #3]
+; CHECK-NEXT: ret
+%addr2 = getelementptr i64, i64 *  %addr, i64 %a
+%res = call <vscale x 6 x i64> @llvm.aarch64.sve.ld3.nxv6i64.nxv2i1.p0i64(<vscale x 2 x i1> %Pg, i64 *%addr2)
+ret <vscale x 6 x i64> %res
+}
+
+define <vscale x 6 x double> @ld3.nxv6f64(<vscale x 2 x i1> %Pg, double *%addr, i64 %a) {
+; CHECK-LABEL: ld3.nxv6f64:
+; CHECK: ld3d { z0.d, z1.d, z2.d }, p0/z, [x0, x1, lsl #3]
+; CHECK-NEXT: ret
+%addr2 = getelementptr double, double *  %addr, i64 %a
+%res = call <vscale x 6 x double> @llvm.aarch64.sve.ld3.nxv6f64.nxv2i1.p0f64(<vscale x 2 x i1> %Pg, double *%addr2)
+ret <vscale x 6 x double> %res
+}
+
+; ld4b
+define <vscale x 64 x i8> @ld4.nxv64i8(<vscale x 16 x i1> %Pg, i8 *%addr, i64 %a) {
+; CHECK-LABEL: ld4.nxv64i8:
+; CHECK: ld4b { z0.b, z1.b, z2.b, z3.b }, p0/z, [x0, x1]
+; CHECK-NEXT: ret
+%addr2 = getelementptr i8, i8 *  %addr, i64 %a
+%res = call <vscale x 64 x i8> @llvm.aarch64.sve.ld4.nxv64i8.nxv16i1.p0i8(<vscale x 16 x i1> %Pg, i8 *%addr2)
+ret <vscale x 64 x i8> %res
+}
+
+; ld4h
+define <vscale x 32 x i16> @ld4.nxv32i16(<vscale x 8 x i1> %Pg, i16 *%addr, i64 %a) {
+; CHECK-LABEL: ld4.nxv32i16:
+; CHECK: ld4h { z0.h, z1.h, z2.h, z3.h }, p0/z, [x0, x1, lsl #1]
+; CHECK-NEXT: ret
+%addr2 = getelementptr i16, i16 *  %addr, i64 %a
+%res = call <vscale x 32 x i16> @llvm.aarch64.sve.ld4.nxv32i16.nxv8i1.p0i16(<vscale x 8 x i1> %Pg, i16 *%addr2)
+ret <vscale x 32 x i16> %res
+}
+
+define <vscale x 32 x half> @ld4.nxv32f16(<vscale x 8 x i1> %Pg, half *%addr, i64 %a) {
+; CHECK-LABEL: ld4.nxv32f16:
+; CHECK: ld4h { z0.h, z1.h, z2.h, z3.h }, p0/z, [x0, x1, lsl #1]
+; CHECK-NEXT: ret
+%addr2 = getelementptr half, half *  %addr, i64 %a
+%res = call <vscale x 32 x half> @llvm.aarch64.sve.ld4.nxv32f16.nxv8i1.p0f16(<vscale x 8 x i1> %Pg, half *%addr2)
+ret <vscale x 32 x half> %res
+}
+
+define <vscale x 32 x bfloat> @ld4.nxv32bf16(<vscale x 8 x i1> %Pg, bfloat *%addr, i64 %a) #0 {
+; CHECK-LABEL: ld4.nxv32bf16:
+; CHECK: ld4h { z0.h, z1.h, z2.h, z3.h }, p0/z, [x0, x1, lsl #1]
+; CHECK-NEXT: ret
+%addr2 = getelementptr bfloat, bfloat *  %addr, i64 %a
+%res = call <vscale x 32 x bfloat> @llvm.aarch64.sve.ld4.nxv32bf16.nxv8i1.p0bf16(<vscale x 8 x i1> %Pg, bfloat *%addr2)
+ret <vscale x 32 x bfloat> %res
+}
+
+; ld4w
+define <vscale x 16 x i32> @ld4.nxv16i32(<vscale x 4 x i1> %Pg, i32 *%addr, i64 %a) {
+; CHECK-LABEL: ld4.nxv16i32:
+; CHECK: ld4w { z0.s, z1.s, z2.s, z3.s }, p0/z, [x0, x1, lsl #2]
+; CHECK-NEXT: ret
+%addr2 = getelementptr i32, i32 *  %addr, i64 %a
+%res = call <vscale x 16 x i32> @llvm.aarch64.sve.ld4.nxv16i32.nxv4i1.p0i32(<vscale x 4 x i1> %Pg, i32 *%addr2)
+ret <vscale x 16 x i32> %res
+}
+
+define <vscale x 16 x float> @ld4.nxv16f32(<vscale x 4 x i1> %Pg, float *%addr, i64 %a) {
+; CHECK-LABEL: ld4.nxv16f32:
+; CHECK: ld4w { z0.s, z1.s, z2.s, z3.s }, p0/z, [x0, x1, lsl #2]
+; CHECK-NEXT: ret
+%addr2 = getelementptr float, float *  %addr, i64 %a
+%res = call <vscale x 16 x float> @llvm.aarch64.sve.ld4.nxv16f32.nxv4i1.p0f32(<vscale x 4 x i1> %Pg, float *%addr2)
+ret <vscale x 16 x float> %res
+}
+
+; ld4d
+define <vscale x 8 x i64> @ld4.nxv8i64(<vscale x 2 x i1> %Pg, i64 *%addr, i64 %a) {
+; CHECK-LABEL: ld4.nxv8i64:
+; CHECK: ld4d { z0.d, z1.d, z2.d, z3.d }, p0/z, [x0, x1, lsl #3]
+; CHECK-NEXT: ret
+%addr2 = getelementptr i64, i64 *  %addr, i64 %a
+%res = call <vscale x 8 x i64> @llvm.aarch64.sve.ld4.nxv8i64.nxv2i1.p0i64(<vscale x 2 x i1> %Pg, i64 *%addr2)
+ret <vscale x 8 x i64> %res
+}
+
+define <vscale x 8 x double> @ld4.nxv8f64(<vscale x 2 x i1> %Pg, double *%addr, i64 %a) {
+; CHECK-LABEL: ld4.nxv8f64:
+; CHECK: ld4d { z0.d, z1.d, z2.d, z3.d }, p0/z, [x0, x1, lsl #3]
+; CHECK-NEXT: ret
+%addr2 = getelementptr double, double *  %addr, i64 %a
+%res = call <vscale x 8 x double> @llvm.aarch64.sve.ld4.nxv8f64.nxv2i1.p0f64(<vscale x 2 x i1> %Pg, double *%addr2)
+ret <vscale x 8 x double> %res
+}
+
+declare <vscale x 32 x i8> @llvm.aarch64.sve.ld2.nxv32i8.nxv16i1.p0i8(<vscale x 16 x i1>, i8*)
+declare <vscale x 16 x i16> @llvm.aarch64.sve.ld2.nxv16i16.nxv8i1.p0i16(<vscale x 8 x i1>, i16*)
+declare <vscale x 8 x i32> @llvm.aarch64.sve.ld2.nxv8i32.nxv4i1.p0i32(<vscale x 4 x i1>, i32*)
+declare <vscale x 4 x i64> @llvm.aarch64.sve.ld2.nxv4i64.nxv2i1.p0i64(<vscale x 2 x i1>, i64*)
+declare <vscale x 16 x half> @llvm.aarch64.sve.ld2.nxv16f16.nxv8i1.p0f16(<vscale x 8 x i1>, half*)
+declare <vscale x 16 x bfloat> @llvm.aarch64.sve.ld2.nxv16bf16.nxv8i1.p0bf16(<vscale x 8 x i1>, bfloat*)
+declare <vscale x 8 x float> @llvm.aarch64.sve.ld2.nxv8f32.nxv4i1.p0f32(<vscale x 4 x i1>, float*)
+declare <vscale x 4 x double> @llvm.aarch64.sve.ld2.nxv4f64.nxv2i1.p0f64(<vscale x 2 x i1>, double*)
+
+declare <vscale x 48 x i8> @llvm.aarch64.sve.ld3.nxv48i8.nxv16i1.p0i8(<vscale x 16 x i1>, i8*)
+declare <vscale x 24 x i16> @llvm.aarch64.sve.ld3.nxv24i16.nxv8i1.p0i16(<vscale x 8 x i1>, i16*)
+declare <vscale x 12 x i32> @llvm.aarch64.sve.ld3.nxv12i32.nxv4i1.p0i32(<vscale x 4 x i1>, i32*)
+declare <vscale x 6 x i64> @llvm.aarch64.sve.ld3.nxv6i64.nxv2i1.p0i64(<vscale x 2 x i1>, i64*)
+declare <vscale x 24 x half> @llvm.aarch64.sve.ld3.nxv24f16.nxv8i1.p0f16(<vscale x 8 x i1>, half*)
+declare <vscale x 24 x bfloat> @llvm.aarch64.sve.ld3.nxv24bf16.nxv8i1.p0bf16(<vscale x 8 x i1>, bfloat*)
+declare <vscale x 12 x float> @llvm.aarch64.sve.ld3.nxv12f32.nxv4i1.p0f32(<vscale x 4 x i1>, float*)
+declare <vscale x 6 x double> @llvm.aarch64.sve.ld3.nxv6f64.nxv2i1.p0f64(<vscale x 2 x i1>, double*)
+
+declare <vscale x 64 x i8> @llvm.aarch64.sve.ld4.nxv64i8.nxv16i1.p0i8(<vscale x 16 x i1>, i8*)
+declare <vscale x 32 x i16> @llvm.aarch64.sve.ld4.nxv32i16.nxv8i1.p0i16(<vscale x 8 x i1>, i16*)
+declare <vscale x 16 x i32> @llvm.aarch64.sve.ld4.nxv16i32.nxv4i1.p0i32(<vscale x 4 x i1>, i32*)
+declare <vscale x 8 x i64> @llvm.aarch64.sve.ld4.nxv8i64.nxv2i1.p0i64(<vscale x 2 x i1>, i64*)
+declare <vscale x 32 x half> @llvm.aarch64.sve.ld4.nxv32f16.nxv8i1.p0f16(<vscale x 8 x i1>, half*)
+declare <vscale x 32 x bfloat> @llvm.aarch64.sve.ld4.nxv32bf16.nxv8i1.p0bf16(<vscale x 8 x i1>, bfloat*)
+declare <vscale x 16 x float> @llvm.aarch64.sve.ld4.nxv16f32.nxv4i1.p0f32(<vscale x 4 x i1>, float*)
+declare <vscale x 8 x double> @llvm.aarch64.sve.ld4.nxv8f64.nxv2i1.p0f64(<vscale x 2 x i1>, double*)
+
+; +bf16 is required for the bfloat version.
+attributes #0 = { "target-features"="+sve,+bf16" }

From ee3feef5aaaa3c385fbe08bdb2d48829ad440b56 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Mon, 13 Jul 2020 08:59:38 -0400
Subject: [PATCH 0251/1035] TableGen/GlobalISel: Allow output instructions with
 multiple defs

The DAG behavior allows matchching input patterns with a single result
to the first result of an output instruction that defines multiple
results. The remaining defs are implicitly dead.

This starts to fix using manual selection for AMDGPU add/sub (although
it's still needed, mostly because it's also still needed for
G_PTR_ADD).
---
 .../AMDGPU/GlobalISel/inst-select-add.mir     |  3 +-
 .../Common/GlobalISelEmitterCommon.td         |  3 +
 .../GlobalISelEmitter-output-discard.td       | 27 ++++++++
 llvm/utils/TableGen/GlobalISelEmitter.cpp     | 61 ++++++++++++++-----
 4 files changed, 78 insertions(+), 16 deletions(-)
 create mode 100644 llvm/test/TableGen/GlobalISelEmitter-output-discard.td

diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-add.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-add.mir
index baed490c07581..47b4a5c400d37 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-add.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-add.mir
@@ -94,8 +94,7 @@ body: |
     ; GFX6-LABEL: name: add_neg_inline_const_64_to_sub_s32_v
     ; GFX6: liveins: $vgpr0
     ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; GFX6: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -64, implicit $exec
-    ; GFX6: %2:vgpr_32, dead %3:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
+    ; GFX6: %2:vgpr_32, dead %3:sreg_64 = V_SUB_CO_U32_e64 [[COPY]], 64, 0, implicit $exec
     ; GFX6: S_ENDPGM 0, implicit %2
     ; GFX9-LABEL: name: add_neg_inline_const_64_to_sub_s32_v
     ; GFX9: liveins: $vgpr0
diff --git a/llvm/test/TableGen/Common/GlobalISelEmitterCommon.td b/llvm/test/TableGen/Common/GlobalISelEmitterCommon.td
index 2fe84fb95296e..9ae6c8be7dc2c 100644
--- a/llvm/test/TableGen/Common/GlobalISelEmitterCommon.td
+++ b/llvm/test/TableGen/Common/GlobalISelEmitterCommon.td
@@ -12,6 +12,9 @@ def GPR32Op : RegisterOperand<GPR32>;
 def F0 : Register<"f0"> { let Namespace = "MyTarget"; }
 def FPR32 : RegisterClass<"MyTarget", [f32], 32, (add F0)>;
 def FPR32Op : RegisterOperand<FPR32>;
+def B0 : Register<"b0"> { let Namespace = "MyTarget"; }
+def GPR8 : RegisterClass<"MyTarget", [i8], 8, (add B0)>;
+
 def p0 : PtrValueType <i32, 0>;
 
 class I<dag OOps, dag IOps, list<dag> Pat>
diff --git a/llvm/test/TableGen/GlobalISelEmitter-output-discard.td b/llvm/test/TableGen/GlobalISelEmitter-output-discard.td
new file mode 100644
index 0000000000000..c755d8377e61d
--- /dev/null
+++ b/llvm/test/TableGen/GlobalISelEmitter-output-discard.td
@@ -0,0 +1,27 @@
+// RUN: llvm-tblgen -gen-global-isel -warn-on-skipped-patterns -I %p/../../include -I %p/Common %s -o - < %s | FileCheck -check-prefix=GISEL %s
+
+include "llvm/Target/Target.td"
+include "GlobalISelEmitterCommon.td"
+
+// Test that extra explicit results are treated as dead defs.
+def ADD_CO : I<(outs GPR32:$dst, GPR8:$flag),
+               (ins GPR32Op:$src0, GPR32Op:$src1), []>;
+
+// GISEL: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_ADD,
+// GISEL-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
+// GISEL-NEXT: GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32,
+// GISEL-NEXT: GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32,
+// GISEL-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID,
+// GISEL-NEXT: // (add:{ *:[i32] } i32:{ *:[i32] }:$src0, i32:{ *:[i32] }:$src1)  =>  (ADD_CO:{ *:[i32] }:{ *:[i8] } GPR32:{ *:[i32] }:$src0, GPR32:{ *:[i32] }:$src1)
+// GISEL-NEXT: GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s8,
+// GISEL-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::ADD_CO,
+// GISEL-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst
+// GISEL-NEXT: GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/RegState::Define|RegState::Dead,
+// GISEL-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // src0
+// GISEL-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/2, // src1
+// GISEL-NEXT: GIR_EraseFromParent, /*InsnID*/0,
+// GISEL-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0,
+def : Pat <
+  (add i32:$src0, i32:$src1),
+  (ADD_CO GPR32:$src0, GPR32:$src1)
+>;
diff --git a/llvm/utils/TableGen/GlobalISelEmitter.cpp b/llvm/utils/TableGen/GlobalISelEmitter.cpp
index 4e8dcc52fc202..a9ebf8f1beaf2 100644
--- a/llvm/utils/TableGen/GlobalISelEmitter.cpp
+++ b/llvm/utils/TableGen/GlobalISelEmitter.cpp
@@ -2628,12 +2628,14 @@ class TempRegRenderer : public OperandRenderer {
   unsigned TempRegID;
   const CodeGenSubRegIndex *SubRegIdx;
   bool IsDef;
+  bool IsDead;
 
 public:
   TempRegRenderer(unsigned InsnID, unsigned TempRegID, bool IsDef = false,
-                  const CodeGenSubRegIndex *SubReg = nullptr)
+                  const CodeGenSubRegIndex *SubReg = nullptr,
+                  bool IsDead = false)
       : OperandRenderer(OR_Register), InsnID(InsnID), TempRegID(TempRegID),
-        SubRegIdx(SubReg), IsDef(IsDef) {}
+        SubRegIdx(SubReg), IsDef(IsDef), IsDead(IsDead) {}
 
   static bool classof(const OperandRenderer *R) {
     return R->getKind() == OR_TempRegister;
@@ -2650,9 +2652,13 @@ class TempRegRenderer : public OperandRenderer {
           << MatchTable::Comment("TempRegID") << MatchTable::IntValue(TempRegID)
           << MatchTable::Comment("TempRegFlags");
 
-    if (IsDef)
-      Table << MatchTable::NamedValue("RegState::Define");
-    else
+    if (IsDef) {
+      SmallString<32> RegFlags;
+      RegFlags += "RegState::Define";
+      if (IsDead)
+        RegFlags += "|RegState::Dead";
+      Table << MatchTable::NamedValue(RegFlags);
+    } else
       Table << MatchTable::IntValue(0);
 
     if (SubRegIdx)
@@ -3394,7 +3400,11 @@ class GlobalISelEmitter {
   Expected<action_iterator>
   createInstructionRenderer(action_iterator InsertPt, RuleMatcher &M,
                             const TreePatternNode *Dst);
-  void importExplicitDefRenderers(BuildMIAction &DstMIBuilder);
+
+  Expected<action_iterator>
+  importExplicitDefRenderers(action_iterator InsertPt, RuleMatcher &M,
+                             BuildMIAction &DstMIBuilder,
+                             const TreePatternNode *Dst);
 
   Expected<action_iterator>
   importExplicitUseRenderers(action_iterator InsertPt, RuleMatcher &M,
@@ -4220,7 +4230,9 @@ Expected<BuildMIAction &> GlobalISelEmitter::createAndImportInstructionRenderer(
     CopyToPhysRegMIBuilder.addRenderer<CopyPhysRegRenderer>(PhysInput.first);
   }
 
-  importExplicitDefRenderers(DstMIBuilder);
+  if (auto Error = importExplicitDefRenderers(InsertPt, M, DstMIBuilder, Dst)
+                       .takeError())
+    return std::move(Error);
 
   if (auto Error = importExplicitUseRenderers(InsertPt, M, DstMIBuilder, Dst)
                        .takeError())
@@ -4372,13 +4384,34 @@ Expected<action_iterator> GlobalISelEmitter::createInstructionRenderer(
                                        DstI);
 }
 
-void GlobalISelEmitter::importExplicitDefRenderers(
-    BuildMIAction &DstMIBuilder) {
+Expected<action_iterator> GlobalISelEmitter::importExplicitDefRenderers(
+    action_iterator InsertPt, RuleMatcher &M, BuildMIAction &DstMIBuilder,
+    const TreePatternNode *Dst) {
   const CodeGenInstruction *DstI = DstMIBuilder.getCGI();
-  for (unsigned I = 0; I < DstI->Operands.NumDefs; ++I) {
-    const CGIOperandList::OperandInfo &DstIOperand = DstI->Operands[I];
-    DstMIBuilder.addRenderer<CopyRenderer>(DstIOperand.Name);
+  const unsigned NumDefs = DstI->Operands.NumDefs;
+  if (NumDefs == 0)
+    return InsertPt;
+
+  DstMIBuilder.addRenderer<CopyRenderer>(DstI->Operands[0].Name);
+
+  // Patterns only handle a single result, so any result after the first is an
+  // implicitly dead def.
+  for (unsigned I = 1; I < NumDefs; ++I) {
+    const TypeSetByHwMode &ExtTy = Dst->getExtType(I);
+    if (!ExtTy.isMachineValueType())
+      return failedImport("unsupported typeset");
+
+    auto OpTy = MVTToLLT(ExtTy.getMachineValueType().SimpleTy);
+    if (!OpTy)
+      return failedImport("unsupported type");
+
+    unsigned TempRegID = M.allocateTempRegID();
+    InsertPt =
+      M.insertAction<MakeTempRegisterAction>(InsertPt, *OpTy, TempRegID);
+    DstMIBuilder.addRenderer<TempRegRenderer>(TempRegID, true, nullptr, true);
   }
+
+  return InsertPt;
 }
 
 Expected<action_iterator> GlobalISelEmitter::importExplicitUseRenderers(
@@ -4814,8 +4847,8 @@ Expected<RuleMatcher> GlobalISelEmitter::runOnPattern(const PatternToMatch &P) {
   auto &DstI = Target.getInstruction(DstOp);
   StringRef DstIName = DstI.TheDef->getName();
 
-  if (DstI.Operands.NumDefs != Src->getExtTypes().size())
-    return failedImport("Src pattern results and dst MI defs are different (" +
+  if (DstI.Operands.NumDefs < Src->getExtTypes().size())
+    return failedImport("Src pattern result has more defs than dst MI (" +
                         to_string(Src->getExtTypes().size()) + " def(s) vs " +
                         to_string(DstI.Operands.NumDefs) + " def(s))");
 

From f9fec0447e12da9e8cf4b628f6d45f4941e7d182 Mon Sep 17 00:00:00 2001
From: Jonas Devlieghere <jonas@devlieghere.com>
Date: Mon, 27 Jul 2020 15:36:16 -0700
Subject: [PATCH 0252/1035] [llvm] Make ZLIB handling compatible with
 multi-configuration generators

The CMAKE_BUILD_TYPE is only meaningful to single-configuration
generators (such as make and Ninja). For multi-configuration generators
like Xcode and MSVC this variable won't be set, resulting in a CMake
error.
---
 llvm/lib/Support/CMakeLists.txt | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Support/CMakeLists.txt b/llvm/lib/Support/CMakeLists.txt
index 82b7ceb856f25..7b45dc628160e 100644
--- a/llvm/lib/Support/CMakeLists.txt
+++ b/llvm/lib/Support/CMakeLists.txt
@@ -200,8 +200,11 @@ add_llvm_component_library(LLVMSupport
 set(llvm_system_libs ${system_libs})
 
 if(LLVM_ENABLE_ZLIB)
-  string(TOUPPER ${CMAKE_BUILD_TYPE} build_type)
-  get_property(zlib_library TARGET ZLIB::ZLIB PROPERTY LOCATION_${build_type})
+  # CMAKE_BUILD_TYPE is only meaningful to single-configuration generators.
+  if(CMAKE_BUILD_TYPE)
+    string(TOUPPER ${CMAKE_BUILD_TYPE} build_type)
+    get_property(zlib_library TARGET ZLIB::ZLIB PROPERTY LOCATION_${build_type})
+  endif()
   if(NOT zlib_library)
     get_property(zlib_library TARGET ZLIB::ZLIB PROPERTY LOCATION)
   endif()

From 8b81d0633f1ebaf1392e6509b34694de05c94de7 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Tue, 21 Jul 2020 20:53:57 -0400
Subject: [PATCH 0253/1035] AMDGPU: global_atomic_csub is not always
 dereferenceable

---
 llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index fa2a96d33952b..c92065f7495a2 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -1137,7 +1137,6 @@ bool SITargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
     Info.align.reset();
     Info.flags = MachineMemOperand::MOLoad |
                  MachineMemOperand::MOStore |
-                 MachineMemOperand::MODereferenceable |
                  MachineMemOperand::MOVolatile;
     return true;
   }

From ce944af33c1e011b69665b6892eb3ea142afcdf0 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Sat, 25 Jul 2020 10:17:49 -0400
Subject: [PATCH 0254/1035] AMDGPU/GlobalISel: Mark G_ATOMICRMW_{NAND|FSUB} as
 lower

These aren't implemented and we're still relying on the AtomicExpand
pass, but mark these as lower to eliminate a few of the few remaining
no rules defined cases.
---
 llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index f1962db35bc01..bf0ebd322aa9e 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -1493,6 +1493,8 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
       G_FCOPYSIGN,
 
       G_ATOMIC_CMPXCHG_WITH_SUCCESS,
+      G_ATOMICRMW_NAND,
+      G_ATOMICRMW_FSUB,
       G_READ_REGISTER,
       G_WRITE_REGISTER,
 

From 5f802be4e554f4eeb2c2c4137bed91b5b4ae740e Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Tue, 21 Jul 2020 19:29:37 -0400
Subject: [PATCH 0255/1035] GlobalISel: Don't fail translate on intrinsics with
 metadata

---
 llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp  |  9 ++++----
 .../GlobalISel/irtranslator-metadata.ll       | 22 +++++++++++++++++++
 2 files changed, 27 insertions(+), 4 deletions(-)
 create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-metadata.ll

diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index e6b3d51bf1ec4..6433c13e990bf 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -1758,10 +1758,6 @@ bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) {
     MIB->copyIRFlags(CI);
 
   for (auto &Arg : enumerate(CI.arg_operands())) {
-    // Some intrinsics take metadata parameters. Reject them.
-    if (isa<MetadataAsValue>(Arg.value()))
-      return false;
-
     // If this is required to be an immediate, don't materialize it in a
     // register.
     if (CI.paramHasAttr(Arg.index(), Attribute::ImmArg)) {
@@ -1774,6 +1770,11 @@ bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) {
       } else {
         MIB.addFPImm(cast<ConstantFP>(Arg.value()));
       }
+    } else if (auto MD = dyn_cast<MetadataAsValue>(Arg.value())) {
+      auto *MDN = dyn_cast<MDNode>(MD->getMetadata());
+      if (!MDN) // This was probably an MDString.
+        return false;
+      MIB.addMetadata(MDN);
     } else {
       ArrayRef<Register> VRegs = getOrCreateVRegs(*Arg.value());
       if (VRegs.size() > 1)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-metadata.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-metadata.ll
new file mode 100644
index 0000000000000..6a3b3b8128661
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-metadata.ll
@@ -0,0 +1,22 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -global-isel -march=amdgcn -mcpu=gfx900 -stop-after=irtranslator -o - %s | FileCheck %s
+; Make sure intrinsics with metadata arguments are translated
+
+define i32 @reloc_constant() {
+  ; CHECK-LABEL: name: reloc_constant
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[INT:%[0-9]+]]:_(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.reloc.constant), !0
+  ; CHECK:   $vgpr0 = COPY [[INT]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; CHECK:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0
+  %val = call i32 @llvm.amdgcn.reloc.constant(metadata !0)
+  ret i32 %val
+}
+
+declare i32 @llvm.amdgcn.reloc.constant(metadata) #0
+
+attributes #0 = { nounwind readnone speculatable willreturn }
+
+!0 = !{!"arst"}

From f1d4db4f0cdcbfeaee0840bf8a4fb5dc1b9b56fd Mon Sep 17 00:00:00 2001
From: Alina Sbirlea <asbirlea@google.com>
Date: Thu, 16 Jul 2020 15:46:54 -0700
Subject: [PATCH 0256/1035] [GraphDiff] Use class method getChildren instead of
 GraphTraits.

Summary:
Use getChildren() method in GraphDiff instead of GraphTraits.

This simplifies the code and allows for refactorigns inside GraphDiff.
All usecase need not have a light-weight/copyable range.
Clean GraphTraits implementation.

Reviewers: dblaikie

Subscribers: hiraditya, llvm-commits, george.burgess.iv

Tags: #llvm

Differential Revision: https://reviews.llvm.org/D84562
---
 .../llvm/Analysis/IteratedDominanceFrontier.h |  8 +-
 llvm/include/llvm/Analysis/MemorySSAUpdater.h |  2 -
 llvm/include/llvm/Support/CFGDiff.h           | 98 +------------------
 llvm/lib/Analysis/MemorySSAUpdater.cpp        | 11 +--
 4 files changed, 8 insertions(+), 111 deletions(-)

diff --git a/llvm/include/llvm/Analysis/IteratedDominanceFrontier.h b/llvm/include/llvm/Analysis/IteratedDominanceFrontier.h
index fb66052851563..8166b52aa226c 100644
--- a/llvm/include/llvm/Analysis/IteratedDominanceFrontier.h
+++ b/llvm/include/llvm/Analysis/IteratedDominanceFrontier.h
@@ -73,13 +73,7 @@ ChildrenGetterTy<BasicBlock, IsPostDom>::get(const NodeRef &N) {
     return {Children.begin(), Children.end()};
   }
 
-  using SnapShotBBPairTy =
-      std::pair<const GraphDiff<BasicBlock *, IsPostDom> *, OrderedNodeTy>;
-
-  ChildrenTy Ret;
-  for (const auto &SnapShotBBPair : children<SnapShotBBPairTy>({GD, N}))
-    Ret.emplace_back(SnapShotBBPair.second);
-  return Ret;
+  return GD->template getChildren<IsPostDom>(N);
 }
 
 } // end of namespace IDFCalculatorDetail
diff --git a/llvm/include/llvm/Analysis/MemorySSAUpdater.h b/llvm/include/llvm/Analysis/MemorySSAUpdater.h
index 20588ef083c59..d41b932099794 100644
--- a/llvm/include/llvm/Analysis/MemorySSAUpdater.h
+++ b/llvm/include/llvm/Analysis/MemorySSAUpdater.h
@@ -52,8 +52,6 @@ class LoopBlocksRPO;
 using ValueToValueMapTy = ValueMap<const Value *, WeakTrackingVH>;
 using PhiToDefMap = SmallDenseMap<MemoryPhi *, MemoryAccess *>;
 using CFGUpdate = cfg::Update<BasicBlock *>;
-using GraphDiffInvBBPair =
-    std::pair<const GraphDiff<BasicBlock *> *, Inverse<BasicBlock *>>;
 
 class MemorySSAUpdater {
 private:
diff --git a/llvm/include/llvm/Support/CFGDiff.h b/llvm/include/llvm/Support/CFGDiff.h
index 269984872bfac..9cbf311f68014 100644
--- a/llvm/include/llvm/Support/CFGDiff.h
+++ b/llvm/include/llvm/Support/CFGDiff.h
@@ -30,44 +30,6 @@
 // a non-inversed graph, the children are naturally the successors when
 // InverseEdge is false and the predecessors when InverseEdge is true.
 
-// We define two base clases that call into GraphDiff, one for successors
-// (CFGSuccessors), where InverseEdge is false, and one for predecessors
-// (CFGPredecessors), where InverseEdge is true.
-// FIXME: Further refactoring may merge the two base classes into a single one
-// templated / parametrized on using succ_iterator/pred_iterator and false/true
-// for the InverseEdge.
-
-// CFGViewChildren and CFGViewPredecessors, both can be parametrized to
-// consider the graph inverted or not (i.e. InverseGraph). Successors
-// implicitly has InverseEdge = false and Predecessors implicitly has
-// InverseEdge = true (see calls to GraphDiff methods in there). The GraphTraits
-// instantiations that follow define the value of InverseGraph.
-
-// GraphTraits instantiations:
-// - GraphDiff<BasicBlock *> is equivalent to InverseGraph = false
-// - GraphDiff<Inverse<BasicBlock *>> is equivalent to InverseGraph = true
-// - second pair item is BasicBlock *, then InverseEdge = false (so it inherits
-// from CFGViewChildren).
-// - second pair item is Inverse<BasicBlock *>, then InverseEdge = true (so it
-// inherits from CFGViewPredecessors).
-
-// The 4 GraphTraits are as follows:
-// 1. std::pair<const GraphDiff<BasicBlock *> *, BasicBlock *>> :
-//        CFGViewChildren<false>
-// Regular CFG, children means successors, InverseGraph = false,
-// InverseEdge = false.
-// 2. std::pair<const GraphDiff<Inverse<BasicBlock *>> *, BasicBlock *>> :
-//        CFGViewChildren<true>
-// Reverse the graph, get successors but reverse-apply updates,
-// InverseGraph = true, InverseEdge = false.
-// 3. std::pair<const GraphDiff<BasicBlock *> *, Inverse<BasicBlock *>>> :
-//        CFGViewPredecessors<false>
-// Regular CFG, reverse edges, so children mean predecessors,
-// InverseGraph = false, InverseEdge = true.
-// 4. std::pair<const GraphDiff<Inverse<BasicBlock *>> *, Inverse<BasicBlock *>>
-//        : CFGViewPredecessors<true>
-// Reverse the graph and the edges, InverseGraph = true, InverseEdge = true.
-
 namespace llvm {
 
 namespace detail {
@@ -87,9 +49,9 @@ template <bool B, typename Range> auto reverse_if(Range &&R) {
 }
 } // namespace detail
 
-// GraphDiff defines a CFG snapshot: given a set of Update<NodePtr>, provide
-// utilities to skip edges marked as deleted and return a set of edges marked as
-// newly inserted. The current diff treats the CFG as a graph rather than a
+// GraphDiff defines a CFG snapshot: given a set of Update<NodePtr>, provides
+// a getChildren method to get a Node's children based on the additional updates
+// in the snapshot. The current diff treats the CFG as a graph rather than a
 // multigraph. Added edges are pruned to be unique, and deleted edges will
 // remove all existing edges between two blocks.
 template <typename NodePtr, bool InverseGraph = false> class GraphDiff {
@@ -191,7 +153,6 @@ template <typename NodePtr, bool InverseGraph = false> class GraphDiff {
   }
 
   using VectRet = SmallVector<NodePtr, 8>;
-
   template <bool InverseEdge> VectRet getChildren(NodePtr N) const {
     using DirectedNodeT =
         std::conditional_t<InverseEdge, Inverse<NodePtr>, NodePtr>;
@@ -228,59 +189,6 @@ template <typename NodePtr, bool InverseGraph = false> class GraphDiff {
   LLVM_DUMP_METHOD void dump() const { print(dbgs()); }
 #endif
 };
-
-template <typename GraphT, bool InverseGraph = false, bool InverseEdge = false,
-          typename GT = GraphTraits<GraphT>>
-struct CFGViewChildren {
-  using DataRef = const GraphDiff<typename GT::NodeRef, InverseGraph> *;
-  using NodeRef = std::pair<DataRef, typename GT::NodeRef>;
-
-  template<typename Range>
-  static auto makeChildRange(Range &&R, DataRef DR) {
-    using Iter = WrappedPairNodeDataIterator<decltype(std::forward<Range>(R).begin()), NodeRef, DataRef>;
-    return make_range(Iter(R.begin(), DR), Iter(R.end(), DR));
-  }
-
-  static auto children(NodeRef N) {
-
-    // filter iterator init:
-    auto R = make_range(GT::child_begin(N.second), GT::child_end(N.second));
-    auto RR = detail::reverse_if<!InverseEdge>(R);
-    // This lambda is copied into the iterators and persists to callers, ensure
-    // captures are by value or otherwise have sufficient lifetime.
-    auto First = make_filter_range(makeChildRange(RR, N.first), [N](NodeRef C) {
-      return !C.first->ignoreChild(N.second, C.second, InverseEdge);
-    });
-
-    // new inserts iterator init:
-    auto InsertVec = N.first->getAddedChildren(N.second, InverseEdge);
-    auto Second = makeChildRange(InsertVec, N.first);
-
-    auto CR = concat<NodeRef>(First, Second);
-
-    // concat_range contains references to other ranges, returning it would
-    // leave those references dangling - the iterators contain
-    // other iterators by value so they're safe to return.
-    return make_range(CR.begin(), CR.end());
-  }
-
-  static auto child_begin(NodeRef N) {
-    return children(N).begin();
-  }
-
-  static auto child_end(NodeRef N) {
-    return children(N).end();
-  }
-
-  using ChildIteratorType = decltype(child_end(std::declval<NodeRef>()));
-};
-
-template <typename T, bool B>
-struct GraphTraits<std::pair<const GraphDiff<T, B> *, T>>
-    : CFGViewChildren<T, B> {};
-template <typename T, bool B>
-struct GraphTraits<std::pair<const GraphDiff<T, B> *, Inverse<T>>>
-    : CFGViewChildren<Inverse<T>, B, true> {};
 } // end namespace llvm
 
 #endif // LLVM_SUPPORT_CFGDIFF_H
diff --git a/llvm/lib/Analysis/MemorySSAUpdater.cpp b/llvm/lib/Analysis/MemorySSAUpdater.cpp
index 85af091772e7e..21cbdcd67147a 100644
--- a/llvm/lib/Analysis/MemorySSAUpdater.cpp
+++ b/llvm/lib/Analysis/MemorySSAUpdater.cpp
@@ -832,8 +832,8 @@ void MemorySSAUpdater::applyInsertUpdates(ArrayRef<CFGUpdate> Updates,
       // Check number of predecessors, we only care if there's more than one.
       unsigned Count = 0;
       BasicBlock *Pred = nullptr;
-      for (auto &Pair : children<GraphDiffInvBBPair>({GD, BB})) {
-        Pred = Pair.second;
+      for (auto *Pi : GD->template getChildren</*InverseEdge=*/true>(BB)) {
+        Pred = Pi;
         Count++;
         if (Count == 2)
           break;
@@ -926,8 +926,7 @@ void MemorySSAUpdater::applyInsertUpdates(ArrayRef<CFGUpdate> Updates,
     auto *BB = BBPredPair.first;
     const auto &AddedBlockSet = BBPredPair.second.Added;
     auto &PrevBlockSet = BBPredPair.second.Prev;
-    for (auto &Pair : children<GraphDiffInvBBPair>({GD, BB})) {
-      BasicBlock *Pi = Pair.second;
+    for (auto *Pi : GD->template getChildren</*InverseEdge=*/true>(BB)) {
       if (!AddedBlockSet.count(Pi))
         PrevBlockSet.insert(Pi);
       EdgeCountMap[{Pi, BB}]++;
@@ -1078,10 +1077,8 @@ void MemorySSAUpdater::applyInsertUpdates(ArrayRef<CFGUpdate> Updates,
         for (unsigned I = 0, E = IDFPhi->getNumIncomingValues(); I < E; ++I)
           IDFPhi->setIncomingValue(I, GetLastDef(IDFPhi->getIncomingBlock(I)));
       } else {
-        for (auto &Pair : children<GraphDiffInvBBPair>({GD, BBIDF})) {
-          BasicBlock *Pi = Pair.second;
+        for (auto *Pi : GD->template getChildren</*InverseEdge=*/true>(BBIDF))
           IDFPhi->addIncoming(GetLastDef(Pi), Pi);
-        }
       }
     }
   }

From f250eb37cd4fabcc9f222ca2da80b62d110d9fff Mon Sep 17 00:00:00 2001
From: "Joel E. Denny" <jdenny.ornl@gmail.com>
Date: Mon, 27 Jul 2020 19:22:05 -0400
Subject: [PATCH 0257/1035] [OpenMP][Docs] Update `present` modifier status

---
 clang/docs/OpenMPSupport.rst | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/clang/docs/OpenMPSupport.rst b/clang/docs/OpenMPSupport.rst
index 26fbfab96bc8c..a1d1b120bcecc 100644
--- a/clang/docs/OpenMPSupport.rst
+++ b/clang/docs/OpenMPSupport.rst
@@ -268,5 +268,7 @@ want to help with the implementation.
 +------------------------------+--------------------------------------------------------------+--------------------------+-----------------------------------------------------------------------+
 | loop extension               | Loop tiling transformation                                   | :part:`claimed`          |                                                                       |
 +------------------------------+--------------------------------------------------------------+--------------------------+-----------------------------------------------------------------------+
-| device extension             | 'present' map type modifier                                  | :part:`worked on`        | D83061, D83062                                                        |
+| device extension             | 'present' map type modifier                                  | :part:`mostly done`      | D83061, D83062, D84422                                                |
++------------------------------+--------------------------------------------------------------+--------------------------+-----------------------------------------------------------------------+
+| device extension             | 'present' motion modifier                                    | :part:`worked on`        | D84711, D84712                                                        |
 +------------------------------+--------------------------------------------------------------+--------------------------+-----------------------------------------------------------------------+

From 64d99cc6abed78c00a2a7863b02ce54911a5264f Mon Sep 17 00:00:00 2001
From: Petr Hosek <phosek@google.com>
Date: Mon, 27 Jul 2020 11:55:52 -0700
Subject: [PATCH 0258/1035] [CMake] Move find_package(ZLIB) to LLVMConfig

This way, downstream projects don't have to invoke find_package(ZLIB)
reducing the amount of boilerplate.

Differential Revision: https://reviews.llvm.org/D84691
---
 clang/CMakeLists.txt                    | 4 ----
 lld/CMakeLists.txt                      | 4 ----
 lldb/cmake/modules/LLDBStandalone.cmake | 4 ----
 llvm/cmake/modules/LLVMConfig.cmake.in  | 3 +++
 mlir/examples/standalone/CMakeLists.txt | 4 ----
 5 files changed, 3 insertions(+), 16 deletions(-)

diff --git a/clang/CMakeLists.txt b/clang/CMakeLists.txt
index 1c4c22b1aaad0..1a6a20a271f36 100644
--- a/clang/CMakeLists.txt
+++ b/clang/CMakeLists.txt
@@ -114,10 +114,6 @@ if( CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR )
   option(CLANG_ENABLE_BOOTSTRAP "Generate the clang bootstrap target" OFF)
   option(LLVM_ENABLE_LIBXML2 "Use libxml2 if available." ON)
 
-  if(LLVM_ENABLE_ZLIB)
-    find_package(ZLIB)
-  endif()
-
   include(AddLLVM)
   include(TableGen)
   include(HandleLLVMOptions)
diff --git a/lld/CMakeLists.txt b/lld/CMakeLists.txt
index bcfc2c6270b3f..e9bd1bd29c5cf 100644
--- a/lld/CMakeLists.txt
+++ b/lld/CMakeLists.txt
@@ -51,10 +51,6 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
   set(LLVM_RUNTIME_OUTPUT_INTDIR ${CMAKE_BINARY_DIR}/${CMAKE_CFG_INTDIR}/bin)
   find_program(LLVM_TABLEGEN_EXE "llvm-tblgen" ${LLVM_TOOLS_BINARY_DIR} NO_DEFAULT_PATH)
 
-  if(LLVM_ENABLE_ZLIB)
-    find_package(ZLIB)
-  endif()
-
   include(AddLLVM)
   include(TableGen)
   include(HandleLLVMOptions)
diff --git a/lldb/cmake/modules/LLDBStandalone.cmake b/lldb/cmake/modules/LLDBStandalone.cmake
index edd2b34ec8655..94781c3583744 100644
--- a/lldb/cmake/modules/LLDBStandalone.cmake
+++ b/lldb/cmake/modules/LLDBStandalone.cmake
@@ -74,10 +74,6 @@ endif()
 # CMake modules to be in that directory as well.
 list(APPEND CMAKE_MODULE_PATH "${LLVM_DIR}")
 
-if(LLVM_ENABLE_ZLIB)
-  find_package(ZLIB)
-endif()
-
 include(AddLLVM)
 include(TableGen)
 include(HandleLLVMOptions)
diff --git a/llvm/cmake/modules/LLVMConfig.cmake.in b/llvm/cmake/modules/LLVMConfig.cmake.in
index e729a839f614d..17cc5eacc57b7 100644
--- a/llvm/cmake/modules/LLVMConfig.cmake.in
+++ b/llvm/cmake/modules/LLVMConfig.cmake.in
@@ -50,6 +50,9 @@ set(LLVM_ENABLE_THREADS @LLVM_ENABLE_THREADS@)
 set(LLVM_ENABLE_UNWIND_TABLES @LLVM_ENABLE_UNWIND_TABLES@)
 
 set(LLVM_ENABLE_ZLIB @LLVM_ENABLE_ZLIB@)
+if(LLVM_ENABLE_ZLIB)
+  find_package(ZLIB)
+endif()
 
 set(LLVM_LIBXML2_ENABLED @LLVM_LIBXML2_ENABLED@)
 
diff --git a/mlir/examples/standalone/CMakeLists.txt b/mlir/examples/standalone/CMakeLists.txt
index 59d3c693546f4..3f46dda4e4f64 100644
--- a/mlir/examples/standalone/CMakeLists.txt
+++ b/mlir/examples/standalone/CMakeLists.txt
@@ -29,10 +29,6 @@ set(MLIR_BINARY_DIR ${CMAKE_BINARY_DIR})
 list(APPEND CMAKE_MODULE_PATH "${MLIR_CMAKE_DIR}")
 list(APPEND CMAKE_MODULE_PATH "${LLVM_CMAKE_DIR}")
 
-if(LLVM_ENABLE_ZLIB)
-  find_package(ZLIB)
-endif()
-
 include(TableGen)
 include(AddLLVM)
 include(AddMLIR)

From 8120eba5fce378083ef22651f2b7b6dcaa54a098 Mon Sep 17 00:00:00 2001
From: Fred Riss <friss@apple.com>
Date: Mon, 27 Jul 2020 17:05:27 -0700
Subject: [PATCH 0259/1035] [lldb/ArchSpec] Always match simulator environment
 in IsEqualTo

Summary:
Initially, Apple simulator binarie triples didn't use a `-simulator`
environment and were just differentiated based on the architecture.
For example, `x86_64-apple-ios` would obviously be a simualtor as iOS
doesn't run on x86_64. With Catalyst, we made the disctinction
explicit and today, all simulator triples (even the legacy ones) are
constructed with an environment. This is especially important on Apple
Silicon were the architecture is not different from the one of the
simulated device.

This change makes the simulator part of the environment always part of
the criteria to detect whether 2 `ArchSpec`s are equal or compatible.

Reviewers: aprantl

Subscribers: inglorion, dexonsmith, lldb-commits

Tags: #lldb

Differential Revision: https://reviews.llvm.org/D84716
---
 lldb/source/Utility/ArchSpec.cpp        | 6 ++++++
 lldb/unittests/Utility/ArchSpecTest.cpp | 8 ++++++++
 2 files changed, 14 insertions(+)

diff --git a/lldb/source/Utility/ArchSpec.cpp b/lldb/source/Utility/ArchSpec.cpp
index cd382a322da70..6e4f1b5326dd9 100644
--- a/lldb/source/Utility/ArchSpec.cpp
+++ b/lldb/source/Utility/ArchSpec.cpp
@@ -987,6 +987,12 @@ static bool IsCompatibleEnvironment(llvm::Triple::EnvironmentType lhs,
   if (lhs == rhs)
     return true;
 
+  // Apple simulators are a different platform than what they simulate.
+  // As the environments are different at this point, if one of them is a
+  // simulator, then they are different.
+  if (lhs == llvm::Triple::Simulator || rhs == llvm::Triple::Simulator)
+    return false;
+
   // If any of the environment is unknown then they are compatible
   if (lhs == llvm::Triple::UnknownEnvironment ||
       rhs == llvm::Triple::UnknownEnvironment)
diff --git a/lldb/unittests/Utility/ArchSpecTest.cpp b/lldb/unittests/Utility/ArchSpecTest.cpp
index a8f43ed7dc7c9..ad0a8ac18cd19 100644
--- a/lldb/unittests/Utility/ArchSpecTest.cpp
+++ b/lldb/unittests/Utility/ArchSpecTest.cpp
@@ -306,6 +306,14 @@ TEST(ArchSpecTest, Compatibility) {
     ASSERT_FALSE(A.IsExactMatch(B));
     ASSERT_FALSE(A.IsCompatibleMatch(B));
   }
+  {
+    ArchSpec A("arm64-apple-ios");
+    ArchSpec B("arm64-apple-ios-simulator");
+    ASSERT_FALSE(A.IsExactMatch(B));
+    ASSERT_FALSE(A.IsCompatibleMatch(B));
+    ASSERT_FALSE(B.IsCompatibleMatch(A));
+    ASSERT_FALSE(B.IsCompatibleMatch(A));
+  }
   {
     ArchSpec A("arm64-*-*");
     ArchSpec B("arm64-apple-ios");

From 18c725e735b1bfb756c2320d812bb095f2c98574 Mon Sep 17 00:00:00 2001
From: Alina Sbirlea <asbirlea@google.com>
Date: Mon, 27 Jul 2020 16:41:55 -0700
Subject: [PATCH 0260/1035] [DomTree] Remove dead code.[NFC]

---
 llvm/include/llvm/Support/GenericDomTreeConstruction.h | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/llvm/include/llvm/Support/GenericDomTreeConstruction.h b/llvm/include/llvm/Support/GenericDomTreeConstruction.h
index 709276ab7a29f..5a1f03c879db4 100644
--- a/llvm/include/llvm/Support/GenericDomTreeConstruction.h
+++ b/llvm/include/llvm/Support/GenericDomTreeConstruction.h
@@ -92,13 +92,9 @@ struct SemiNCAInfo {
 
   BatchUpdateInfo *BatchUpdates;
   using BatchUpdatePtr = BatchUpdateInfo *;
-  std::unique_ptr<GraphDiffT> EmptyGD;
 
   // If BUI is a nullptr, then there's no batch update in progress.
-  SemiNCAInfo(BatchUpdatePtr BUI) : BatchUpdates(BUI) {
-    if (!BatchUpdates)
-      EmptyGD = std::make_unique<GraphDiffT>();
-  }
+  SemiNCAInfo(BatchUpdatePtr BUI) : BatchUpdates(BUI) {}
 
   void clear() {
     NumToNode = {nullptr}; // Restore to initial state with a dummy start node.

From fbca317694bf635bff4309768df11259387eb371 Mon Sep 17 00:00:00 2001
From: Alina Sbirlea <asbirlea@google.com>
Date: Fri, 24 Jul 2020 16:29:58 -0700
Subject: [PATCH 0261/1035] [CFGDiff] Refactor Succ/Pred maps.

Summary:
Refactor Succ/Pred maps to have a single map lookup when constructing
children. The preivous desing made sense when used by GraphTraits.
This more closely matches the previous approach in DomTree.

Reviewers: dblaikie

Subscribers: llvm-commits

Tags: #llvm

Differential Revision: https://reviews.llvm.org/D84567
---
 llvm/include/llvm/ADT/STLExtras.h   |   8 ++
 llvm/include/llvm/Support/CFGDiff.h | 115 ++++++++++++----------------
 2 files changed, 57 insertions(+), 66 deletions(-)

diff --git a/llvm/include/llvm/ADT/STLExtras.h b/llvm/include/llvm/ADT/STLExtras.h
index 92eea4e83f693..b9b5e175f50ea 100644
--- a/llvm/include/llvm/ADT/STLExtras.h
+++ b/llvm/include/llvm/ADT/STLExtras.h
@@ -1653,6 +1653,14 @@ void erase_if(Container &C, UnaryPredicate P) {
   C.erase(remove_if(C, P), C.end());
 }
 
+/// Wrapper function to remove a value from a container:
+///
+/// C.erase(remove(C.begin(), C.end(), V), C.end());
+template <typename Container, typename ValueType>
+void erase_value(Container &C, ValueType V) {
+  C.erase(std::remove(C.begin(), C.end(), V), C.end());
+}
+
 /// Given a sequence container Cont, replace the range [ContIt, ContEnd) with
 /// the range [ValIt, ValEnd) (which is not from the same container).
 template<typename Container, typename RandomAccessIterator>
diff --git a/llvm/include/llvm/Support/CFGDiff.h b/llvm/include/llvm/Support/CFGDiff.h
index 9cbf311f68014..a4a4b2ca44b1f 100644
--- a/llvm/include/llvm/Support/CFGDiff.h
+++ b/llvm/include/llvm/Support/CFGDiff.h
@@ -55,21 +55,18 @@ template <bool B, typename Range> auto reverse_if(Range &&R) {
 // multigraph. Added edges are pruned to be unique, and deleted edges will
 // remove all existing edges between two blocks.
 template <typename NodePtr, bool InverseGraph = false> class GraphDiff {
-  using UpdateMapType = SmallDenseMap<NodePtr, SmallVector<NodePtr, 2>>;
-  struct EdgesInsertedDeleted {
-    UpdateMapType Succ;
-    UpdateMapType Pred;
+  struct DeletesInserts {
+    SmallVector<NodePtr, 2> DI[2];
   };
-  // Store Deleted edges on position 0, and Inserted edges on position 1.
-  EdgesInsertedDeleted Edges[2];
+  using UpdateMapType = SmallDenseMap<NodePtr, DeletesInserts>;
+  UpdateMapType Succ;
+  UpdateMapType Pred;
+
   // By default, it is assumed that, given a CFG and a set of updates, we wish
   // to apply these updates as given. If UpdatedAreReverseApplied is set, the
   // updates will be applied in reverse: deleted edges are considered re-added
   // and inserted edges are considered deleted when returning children.
   bool UpdatedAreReverseApplied;
-  // Using a singleton empty vector for all node requests with no
-  // children.
-  SmallVector<NodePtr, 0> Empty;
 
   // Keep the list of legalized updates for a deterministic order of updates
   // when using a GraphDiff for incremental updates in the DominatorTree.
@@ -77,14 +74,19 @@ template <typename NodePtr, bool InverseGraph = false> class GraphDiff {
   SmallVector<cfg::Update<NodePtr>, 4> LegalizedUpdates;
 
   void printMap(raw_ostream &OS, const UpdateMapType &M) const {
-    for (auto Pair : M)
-      for (auto Child : Pair.second) {
-        OS << "(";
-        Pair.first->printAsOperand(OS, false);
-        OS << ", ";
-        Child->printAsOperand(OS, false);
-        OS << ") ";
+    StringRef DIText[2] = {"Delete", "Insert"};
+    for (auto Pair : M) {
+      for (unsigned IsInsert = 0; IsInsert <= 1; ++IsInsert) {
+        OS << DIText[IsInsert] << " edges: \n";
+        for (auto Child : Pair.second.DI[IsInsert]) {
+          OS << "(";
+          Pair.first->printAsOperand(OS, false);
+          OS << ", ";
+          Child->printAsOperand(OS, false);
+          OS << ") ";
+        }
       }
+    }
     OS << "\n";
   }
 
@@ -93,13 +95,11 @@ template <typename NodePtr, bool InverseGraph = false> class GraphDiff {
   GraphDiff(ArrayRef<cfg::Update<NodePtr>> Updates,
             bool ReverseApplyUpdates = false) {
     cfg::LegalizeUpdates<NodePtr>(Updates, LegalizedUpdates, InverseGraph);
-    // The legalized updates are stored in reverse so we can pop_back when doing
-    // incremental updates.
     for (auto U : LegalizedUpdates) {
       unsigned IsInsert =
           (U.getKind() == cfg::UpdateKind::Insert) == !ReverseApplyUpdates;
-      Edges[IsInsert].Succ[U.getFrom()].push_back(U.getTo());
-      Edges[IsInsert].Pred[U.getTo()].push_back(U.getFrom());
+      Succ[U.getFrom()].DI[IsInsert].push_back(U.getTo());
+      Pred[U.getTo()].DI[IsInsert].push_back(U.getFrom());
     }
     UpdatedAreReverseApplied = ReverseApplyUpdates;
   }
@@ -115,73 +115,56 @@ template <typename NodePtr, bool InverseGraph = false> class GraphDiff {
     auto U = LegalizedUpdates.pop_back_val();
     unsigned IsInsert =
         (U.getKind() == cfg::UpdateKind::Insert) == !UpdatedAreReverseApplied;
-    auto &SuccList = Edges[IsInsert].Succ[U.getFrom()];
+    auto &SuccDIList = Succ[U.getFrom()];
+    auto &SuccList = SuccDIList.DI[IsInsert];
     assert(SuccList.back() == U.getTo());
     SuccList.pop_back();
-    if (SuccList.empty())
-      Edges[IsInsert].Succ.erase(U.getFrom());
+    if (SuccList.empty() && SuccDIList.DI[!IsInsert].empty())
+      Succ.erase(U.getFrom());
 
-    auto &PredList = Edges[IsInsert].Pred[U.getTo()];
+    auto &PredDIList = Pred[U.getTo()];
+    auto &PredList = PredDIList.DI[IsInsert];
     assert(PredList.back() == U.getFrom());
     PredList.pop_back();
-    if (PredList.empty())
-      Edges[IsInsert].Pred.erase(U.getTo());
+    if (PredList.empty() && PredDIList.DI[!IsInsert].empty())
+      Pred.erase(U.getTo());
     return U;
   }
 
-  bool ignoreChild(const NodePtr BB, NodePtr EdgeEnd, bool InverseEdge) const {
-    // Used to filter nullptr in clang.
-    if (EdgeEnd == nullptr)
-      return true;
-    auto &DeleteChildren =
-        (InverseEdge != InverseGraph) ? Edges[0].Pred : Edges[0].Succ;
-    auto It = DeleteChildren.find(BB);
-    if (It == DeleteChildren.end())
-      return false;
-    auto &EdgesForBB = It->second;
-    return llvm::find(EdgesForBB, EdgeEnd) != EdgesForBB.end();
-  }
-
-  iterator_range<typename SmallVectorImpl<NodePtr>::const_iterator>
-  getAddedChildren(const NodePtr BB, bool InverseEdge) const {
-    auto &InsertChildren =
-        (InverseEdge != InverseGraph) ? Edges[1].Pred : Edges[1].Succ;
-    auto It = InsertChildren.find(BB);
-    if (It == InsertChildren.end())
-      return make_range(Empty.begin(), Empty.end());
-    return make_range(It->second.begin(), It->second.end());
-  }
-
   using VectRet = SmallVector<NodePtr, 8>;
   template <bool InverseEdge> VectRet getChildren(NodePtr N) const {
     using DirectedNodeT =
         std::conditional_t<InverseEdge, Inverse<NodePtr>, NodePtr>;
     auto R = children<DirectedNodeT>(N);
-    auto CurrentCFGChildren = detail::reverse_if<!InverseEdge>(R);
+    VectRet Res = VectRet(detail::reverse_if<!InverseEdge>(R));
+
+    // Remove nullptr children for clang.
+    llvm::erase_value(Res, nullptr);
+
+    auto &Children = (InverseEdge != InverseGraph) ? Pred : Succ;
+    auto It = Children.find(N);
+    if (It == Children.end())
+      return Res;
+
+    // Remove children present in the CFG but not in the snapshot.
+    for (auto *Child : It->second.DI[0])
+      llvm::erase_value(Res, Child);
 
-    VectRet UpdatedCFGChildren;
-    for (auto Child : CurrentCFGChildren)
-      if (Child && !ignoreChild(N, Child, InverseEdge))
-        UpdatedCFGChildren.push_back(Child);
+    // Add children present in the snapshot for not in the real CFG.
+    auto &AddedChildren = It->second.DI[1];
+    Res.insert(Res.end(), AddedChildren.begin(), AddedChildren.end());
 
-    auto AddedCFGChildren = getAddedChildren(N, InverseEdge);
-    UpdatedCFGChildren.insert(UpdatedCFGChildren.end(),
-                              AddedCFGChildren.begin(), AddedCFGChildren.end());
-    return UpdatedCFGChildren;
+    return Res;
   }
 
   void print(raw_ostream &OS) const {
     OS << "===== GraphDiff: CFG edge changes to create a CFG snapshot. \n"
           "===== (Note: notion of children/inverse_children depends on "
           "the direction of edges and the graph.)\n";
-    OS << "Children to insert:\n\t";
-    printMap(OS, Edges[1].Succ);
-    OS << "Children to delete:\n\t";
-    printMap(OS, Edges[0].Succ);
-    OS << "Inverse_children to insert:\n\t";
-    printMap(OS, Edges[1].Pred);
-    OS << "Inverse_children to delete:\n\t";
-    printMap(OS, Edges[0].Pred);
+    OS << "Children to delete/insert:\n\t";
+    printMap(OS, Succ);
+    OS << "Inverse_children to delete/insert:\n\t";
+    printMap(OS, Pred);
     OS << "\n";
   }
 

From 930fc0b300b01890d3cafabfa85a8a50b2ca890e Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Mon, 27 Jul 2020 20:29:53 -0400
Subject: [PATCH 0262/1035] TableGen: Check if pattern outputs matches
 instruction defs

Attempt to fix address sanitizer bots when building ARM.
---
 llvm/utils/TableGen/GlobalISelEmitter.cpp | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/llvm/utils/TableGen/GlobalISelEmitter.cpp b/llvm/utils/TableGen/GlobalISelEmitter.cpp
index a9ebf8f1beaf2..52fe9b2af2f08 100644
--- a/llvm/utils/TableGen/GlobalISelEmitter.cpp
+++ b/llvm/utils/TableGen/GlobalISelEmitter.cpp
@@ -4394,6 +4394,11 @@ Expected<action_iterator> GlobalISelEmitter::importExplicitDefRenderers(
 
   DstMIBuilder.addRenderer<CopyRenderer>(DstI->Operands[0].Name);
 
+  // Some instructions have multiple defs, but are missing a type entry
+  // (e.g. s_cc_out operands).
+  if (Dst->getExtTypes().size() < NumDefs)
+    return failedImport("unhandled discarded def");
+
   // Patterns only handle a single result, so any result after the first is an
   // implicitly dead def.
   for (unsigned I = 1; I < NumDefs; ++I) {

From 6bf989b9474ace6a35021e6123d13b7fd59bf9f4 Mon Sep 17 00:00:00 2001
From: Xing GUO <higuoxing@gmail.com>
Date: Tue, 28 Jul 2020 09:53:59 +0800
Subject: [PATCH 0263/1035] [llvm-readelf] Fix emitting incorrect number of
 spaces in '--hex-dump'.

This patch helps teach llvm-readelf to emit a correct number spaces when
dumping in hex format.

Before this patch, when the hex data doesn't fill the 4th column, some
spaces are missing.

```
Hex dump of section '.sec':
0x00000000 00000000 00000000 00000000 00000000 ................
0x00000010 00000000 00000000 00000000 0000 ..............
```

After this patch:

```
Hex dump of section '.sec':
0x00000000 00000000 00000000 00000000 00000000 ................
0x00000010 00000000 00000000 00000000 0000     ..............
```

Reviewed By: grimar

Differential Revision: https://reviews.llvm.org/D84640
---
 .../test/tools/llvm-readobj/ELF/hex-dump.test | 43 ++++++++++++++++++-
 llvm/tools/llvm-readobj/ObjDumper.cpp         |  5 ++-
 2 files changed, 45 insertions(+), 3 deletions(-)

diff --git a/llvm/test/tools/llvm-readobj/ELF/hex-dump.test b/llvm/test/tools/llvm-readobj/ELF/hex-dump.test
index b1f54009110e2..5954153c63451 100644
--- a/llvm/test/tools/llvm-readobj/ELF/hex-dump.test
+++ b/llvm/test/tools/llvm-readobj/ELF/hex-dump.test
@@ -1,6 +1,6 @@
 ## Test that the -x alias can be used flexibly. Create a baseline and ensure
 ## all other combinations are identical.
-# RUN: yaml2obj %s -o %t
+# RUN: yaml2obj --docnum=1 %s -o %t
 # RUN: llvm-readelf --file-header --hex-dump=.shstrtab %t > %t.hexdump.out
 # RUN: llvm-readelf -h --hex-dump .shstrtab %t > %t.hexdump.1
 # RUN: llvm-readelf -h -x .shstrtab %t > %t.hexdump.2
@@ -48,3 +48,44 @@ FileHeader:
   Data:    ELFDATA2LSB
   Type:    ET_DYN
   Machine: EM_386
+
+## Test that llvm-readelf emits a correct amount of spaces between the hex data
+## and its ascii representation.
+
+## a) When the hex data doesn't fill the column whose index isn't 4.
+# RUN: yaml2obj --docnum=2 -DSIZE=18 %s -o %t2.out1
+# RUN: llvm-readelf --hex-dump=.sec %t2.out1 | \
+# RUN:   FileCheck %s --match-full-lines --strict-whitespace --check-prefix=SPACES1
+
+#      SPACES1:Hex dump of section '.sec':
+# SPACES1-NEXT:0x00000000 00000000 00000000 00000000 00000000 ................
+# SPACES1-NEXT:0x00000010 0000                                ..
+
+## b) When the hex data doesn't fill the column whose index is 4.
+# RUN: yaml2obj --docnum=2 -DSIZE=30 %s -o %t2.out2
+# RUN: llvm-readelf --hex-dump=.sec %t2.out2 | \
+# RUN:   FileCheck %s --match-full-lines --strict-whitespace --check-prefix=SPACES2
+
+#      SPACES2:Hex dump of section '.sec':
+# SPACES2-NEXT:0x00000000 00000000 00000000 00000000 00000000 ................
+# SPACES2-NEXT:0x00000010 00000000 00000000 00000000 0000     ..............
+
+## c) When the hex data fills the column.
+# RUN: yaml2obj --docnum=2 -DSIZE=28 %s -o %t2.out3
+# RUN: llvm-readelf --hex-dump=.sec %t2.out3 | \
+# RUN:   FileCheck %s --match-full-lines --strict-whitespace --check-prefix=SPACES3
+
+#      SPACES3:Hex dump of section '.sec':
+# SPACES3-NEXT:0x00000000 00000000 00000000 00000000 00000000 ................
+# SPACES3-NEXT:0x00000010 00000000 00000000 00000000          ............
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS32
+  Data:    ELFDATA2LSB
+  Type:    ET_DYN
+  Machine: EM_386
+Sections:
+  - Name: .sec
+    Type: SHT_PROGBITS
+    Size: [[SIZE]]
diff --git a/llvm/tools/llvm-readobj/ObjDumper.cpp b/llvm/tools/llvm-readobj/ObjDumper.cpp
index ce61f1c53a4dd..7b7c9553827f5 100644
--- a/llvm/tools/llvm-readobj/ObjDumper.cpp
+++ b/llvm/tools/llvm-readobj/ObjDumper.cpp
@@ -155,8 +155,9 @@ void ObjDumper::printSectionsAsHex(const object::ObjectFile *Obj,
       // Least, if we cut in a middle of a row, we add the remaining characters,
       // which is (8 - (k * 2)).
       if (i < 4)
-        W.startLine() << format("%*c", (4 - i) * 8 + (4 - i) + (8 - (k * 2)),
-                                ' ');
+        W.startLine() << format("%*c", (4 - i) * 8 + (4 - i), ' ');
+      if (k < 4)
+        W.startLine() << format("%*c", 8 - k * 2, ' ');
 
       TmpSecPtr = SecPtr;
       for (i = 0; TmpSecPtr + i < SecEnd && i < 16; ++i)

From 23d6525cbdc9de7cbfe7640d1e9e4f25a0c5dd85 Mon Sep 17 00:00:00 2001
From: Richard Smith <richard@metafoo.co.uk>
Date: Mon, 27 Jul 2020 17:28:06 -0700
Subject: [PATCH 0264/1035] Don't form a 'context-independent expr' reference
 to a member during name annotation.

Instead, defer forming the member access expression or DeclRefExpr until
we build the use of ClassifyName's result. Just build an
UnresolvedLookupExpr to track the LookupResult until we're ready to
consume it.

This also reverts commit 2f7269b6773de2750f9cd1417ef5f21cd6cf7a91 (other
than its testcase). That change was an attempted workaround for the same
problem.
---
 clang/include/clang/Basic/TokenKinds.def      |  8 ++--
 clang/include/clang/Sema/Sema.h               | 25 +++++-----
 clang/lib/Parse/ParseDecl.cpp                 |  2 +-
 clang/lib/Parse/ParseExpr.cpp                 | 18 ++-----
 clang/lib/Parse/ParseTentative.cpp            |  9 ----
 clang/lib/Parse/Parser.cpp                    |  5 +-
 clang/lib/Sema/SemaDecl.cpp                   | 47 ++++++++++++++-----
 clang/lib/Sema/SemaExprMember.cpp             | 12 ++---
 .../test/SemaTemplate/member-access-expr.cpp  | 11 +++++
 9 files changed, 74 insertions(+), 63 deletions(-)

diff --git a/clang/include/clang/Basic/TokenKinds.def b/clang/include/clang/Basic/TokenKinds.def
index 2b353269ed52d..daaa54c3db7c2 100644
--- a/clang/include/clang/Basic/TokenKinds.def
+++ b/clang/include/clang/Basic/TokenKinds.def
@@ -757,10 +757,10 @@ ANNOTATION(non_type_undeclared) // annotation for an undeclared identifier that
                                 // was assumed to be an ADL-only function name
 ANNOTATION(non_type_dependent)  // annotation for an assumed non-type member of
                                 // a dependent base class
-ANNOTATION(primary_expr) // annotation for a primary expression
-ANNOTATION(
-    uneval_primary_expr) // annotation for a primary expression which should be
-                         // transformed to potentially evaluated
+ANNOTATION(overload_set) // annotation for an unresolved overload set
+ANNOTATION(primary_expr) // annotation for a primary expression, used when
+                         // tentatively parsing a lambda init-capture or ObjC
+                         // message send
 ANNOTATION(decltype)     // annotation for a decltype expression,
                          // e.g., "decltype(foo.bar())"
 
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 0721720f79085..63e2d0d17fca2 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -2117,9 +2117,11 @@ class Sema final {
     /// resolved. ActOnNameClassifiedAsDependentNonType should be called to
     /// convert the result to an expression.
     NC_DependentNonType,
-    /// The name was classified as a non-type, and an expression representing
-    /// that name has been formed.
-    NC_ContextIndependentExpr,
+    /// The name was classified as an overload set, and an expression
+    /// representing that overload set has been formed.
+    /// ActOnNameClassifiedAsOverloadSet should be called to form a suitable
+    /// expression referencing the overload set.
+    NC_OverloadSet,
     /// The name was classified as a template whose specializations are types.
     NC_TypeTemplate,
     /// The name was classified as a variable template name.
@@ -2156,8 +2158,8 @@ class Sema final {
       return NameClassification(NC_Unknown);
     }
 
-    static NameClassification ContextIndependentExpr(ExprResult E) {
-      NameClassification Result(NC_ContextIndependentExpr);
+    static NameClassification OverloadSet(ExprResult E) {
+      NameClassification Result(NC_OverloadSet);
       Result.Expr = E;
       return Result;
     }
@@ -2209,7 +2211,7 @@ class Sema final {
     NameClassificationKind getKind() const { return Kind; }
 
     ExprResult getExpression() const {
-      assert(Kind == NC_ContextIndependentExpr);
+      assert(Kind == NC_OverloadSet);
       return Expr;
     }
 
@@ -2289,6 +2291,8 @@ class Sema final {
                                           NamedDecl *Found,
                                           SourceLocation NameLoc,
                                           const Token &NextToken);
+  /// Act on the result of classifying a name as an overload set.
+  ExprResult ActOnNameClassifiedAsOverloadSet(Scope *S, Expr *OverloadSet);
 
   /// Describes the detailed kind of a template name. Used in diagnostics.
   enum class TemplateNameKindForDiagnostics {
@@ -4846,11 +4850,10 @@ class Sema final {
       Expr *baseObjectExpr = nullptr,
       SourceLocation opLoc = SourceLocation());
 
-  ExprResult BuildPossibleImplicitMemberExpr(const CXXScopeSpec &SS,
-                                             SourceLocation TemplateKWLoc,
-                                             LookupResult &R,
-                                const TemplateArgumentListInfo *TemplateArgs,
-                                             const Scope *S);
+  ExprResult BuildPossibleImplicitMemberExpr(
+      const CXXScopeSpec &SS, SourceLocation TemplateKWLoc, LookupResult &R,
+      const TemplateArgumentListInfo *TemplateArgs, const Scope *S,
+      UnresolvedLookupExpr *AsULE = nullptr);
   ExprResult BuildImplicitMemberExpr(const CXXScopeSpec &SS,
                                      SourceLocation TemplateKWLoc,
                                      LookupResult &R,
diff --git a/clang/lib/Parse/ParseDecl.cpp b/clang/lib/Parse/ParseDecl.cpp
index c87d240a8206a..7b3a98edb3726 100644
--- a/clang/lib/Parse/ParseDecl.cpp
+++ b/clang/lib/Parse/ParseDecl.cpp
@@ -2837,7 +2837,7 @@ Parser::DiagnoseMissingSemiAfterTagDefinition(DeclSpec &DS, AccessSpecifier AS,
       case Sema::NC_Unknown:
       case Sema::NC_NonType:
       case Sema::NC_DependentNonType:
-      case Sema::NC_ContextIndependentExpr:
+      case Sema::NC_OverloadSet:
       case Sema::NC_VarTemplate:
       case Sema::NC_FunctionTemplate:
       case Sema::NC_Concept:
diff --git a/clang/lib/Parse/ParseExpr.cpp b/clang/lib/Parse/ParseExpr.cpp
index 81e87582c6ade..4f662f00e1df3 100644
--- a/clang/lib/Parse/ParseExpr.cpp
+++ b/clang/lib/Parse/ParseExpr.cpp
@@ -1007,23 +1007,11 @@ ExprResult Parser::ParseCastExpression(CastParseKind ParseKind,
     Res = Actions.ActOnCXXNullPtrLiteral(ConsumeToken());
     break;
 
-  case tok::annot_uneval_primary_expr:
   case tok::annot_primary_expr:
+  case tok::annot_overload_set:
     Res = getExprAnnotation(Tok);
-    if (SavedKind == tok::annot_uneval_primary_expr) {
-      if (Expr *E = Res.get()) {
-        if (!E->isTypeDependent() && !E->containsErrors()) {
-          // TransformToPotentiallyEvaluated expects that it will still be in a
-          // (temporary) unevaluated context and then looks through that context
-          // to build it in the surrounding context. So we need to push an
-          // unevaluated context to balance things out.
-          EnterExpressionEvaluationContext Unevaluated(
-              Actions, Sema::ExpressionEvaluationContext::Unevaluated,
-              Sema::ReuseLambdaContextDecl);
-          Res = Actions.TransformToPotentiallyEvaluated(Res.get());
-        }
-      }
-    }
+    if (!Res.isInvalid() && Tok.getKind() == tok::annot_overload_set)
+      Res = Actions.ActOnNameClassifiedAsOverloadSet(getCurScope(), Res.get());
     ConsumeAnnotationToken();
     if (!Res.isInvalid() && Tok.is(tok::less))
       checkPotentialAngleBracket(Res);
diff --git a/clang/lib/Parse/ParseTentative.cpp b/clang/lib/Parse/ParseTentative.cpp
index f026f3a1bfb29..d0f1d2e09a872 100644
--- a/clang/lib/Parse/ParseTentative.cpp
+++ b/clang/lib/Parse/ParseTentative.cpp
@@ -1276,15 +1276,6 @@ Parser::isCXXDeclarationSpecifier(Parser::TPResult BracedCastResult,
       // this is ambiguous. Typo-correct to type and expression keywords and
       // to types and identifiers, in order to try to recover from errors.
       TentativeParseCCC CCC(Next);
-      // Tentative parsing may not be done in the right evaluation context
-      // for the ultimate expression.  Enter an unevaluated context to prevent
-      // Sema from immediately e.g. treating this lookup as a potential ODR-use.
-      // If we generate an expression annotation token and the parser actually
-      // claims it as an expression, we'll transform the expression to a
-      // potentially-evaluated one then.
-      EnterExpressionEvaluationContext Unevaluated(
-          Actions, Sema::ExpressionEvaluationContext::Unevaluated,
-          Sema::ReuseLambdaContextDecl);
       switch (TryAnnotateName(&CCC)) {
       case ANK_Error:
         return TPResult::Error;
diff --git a/clang/lib/Parse/Parser.cpp b/clang/lib/Parse/Parser.cpp
index 764d4e8e9d522..45cf855cf8c9b 100644
--- a/clang/lib/Parse/Parser.cpp
+++ b/clang/lib/Parse/Parser.cpp
@@ -1691,9 +1691,8 @@ Parser::TryAnnotateName(CorrectionCandidateCallback *CCC) {
     return ANK_Success;
   }
 
-  case Sema::NC_ContextIndependentExpr:
-    Tok.setKind(Actions.isUnevaluatedContext() ? tok::annot_uneval_primary_expr
-                                               : tok::annot_primary_expr);
+  case Sema::NC_OverloadSet:
+    Tok.setKind(tok::annot_overload_set);
     setExprAnnotation(Tok, Classification.getExpression());
     Tok.setAnnotationEndLoc(NameLoc);
     if (SS.isNotEmpty())
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index 53f5132a46b65..869e4de02cc41 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -1184,23 +1184,20 @@ Sema::NameClassification Sema::ClassifyName(Scope *S, CXXScopeSpec &SS,
     return ParsedType::make(T);
   }
 
-  // FIXME: This is context-dependent. We need to defer building the member
-  // expression until the classification is consumed.
-  if (FirstDecl->isCXXClassMember())
-    return NameClassification::ContextIndependentExpr(
-        BuildPossibleImplicitMemberExpr(SS, SourceLocation(), Result, nullptr,
-                                        S));
-
   // If we already know which single declaration is referenced, just annotate
-  // that declaration directly.
+  // that declaration directly. Defer resolving even non-overloaded class
+  // member accesses, as we need to defer certain access checks until we know
+  // the context.
   bool ADL = UseArgumentDependentLookup(SS, Result, NextToken.is(tok::l_paren));
-  if (Result.isSingleResult() && !ADL)
+  if (Result.isSingleResult() && !ADL && !FirstDecl->isCXXClassMember())
     return NameClassification::NonType(Result.getRepresentativeDecl());
 
-  // Build an UnresolvedLookupExpr. Note that this doesn't depend on the
-  // context in which we performed classification, so it's safe to do now.
-  return NameClassification::ContextIndependentExpr(
-      BuildDeclarationNameExpr(SS, Result, ADL));
+  // Otherwise, this is an overload set that we will need to resolve later.
+  Result.suppressDiagnostics();
+  return NameClassification::OverloadSet(UnresolvedLookupExpr::Create(
+      Context, Result.getNamingClass(), SS.getWithLocInContext(Context),
+      Result.getLookupNameInfo(), ADL, Result.isOverloadedResult(),
+      Result.begin(), Result.end()));
 }
 
 ExprResult
@@ -1240,6 +1237,30 @@ ExprResult Sema::ActOnNameClassifiedAsNonType(Scope *S, const CXXScopeSpec &SS,
   return BuildDeclarationNameExpr(SS, Result, ADL);
 }
 
+ExprResult Sema::ActOnNameClassifiedAsOverloadSet(Scope *S, Expr *E) {
+  // For an implicit class member access, transform the result into a member
+  // access expression if necessary.
+  auto *ULE = cast<UnresolvedLookupExpr>(E);
+  if ((*ULE->decls_begin())->isCXXClassMember()) {
+    CXXScopeSpec SS;
+    SS.Adopt(ULE->getQualifierLoc());
+
+    // Reconstruct the lookup result.
+    LookupResult Result(*this, ULE->getName(), ULE->getNameLoc(),
+                        LookupOrdinaryName);
+    Result.setNamingClass(ULE->getNamingClass());
+    for (auto I = ULE->decls_begin(), E = ULE->decls_end(); I != E; ++I)
+      Result.addDecl(*I, I.getAccess());
+    Result.resolveKind();
+    return BuildPossibleImplicitMemberExpr(SS, SourceLocation(), Result,
+                                           nullptr, S);
+  }
+
+  // Otherwise, this is already in the form we needed, and no further checks
+  // are necessary.
+  return ULE;
+}
+
 Sema::TemplateNameKindForDiagnostics
 Sema::getTemplateNameKindForDiagnostics(TemplateName Name) {
   auto *TD = Name.getAsTemplateDecl();
diff --git a/clang/lib/Sema/SemaExprMember.cpp b/clang/lib/Sema/SemaExprMember.cpp
index ebfc1ec4b9749..466d1fe59c715 100644
--- a/clang/lib/Sema/SemaExprMember.cpp
+++ b/clang/lib/Sema/SemaExprMember.cpp
@@ -231,12 +231,10 @@ static void diagnoseInstanceReference(Sema &SemaRef,
 }
 
 /// Builds an expression which might be an implicit member expression.
-ExprResult
-Sema::BuildPossibleImplicitMemberExpr(const CXXScopeSpec &SS,
-                                      SourceLocation TemplateKWLoc,
-                                      LookupResult &R,
-                                const TemplateArgumentListInfo *TemplateArgs,
-                                      const Scope *S) {
+ExprResult Sema::BuildPossibleImplicitMemberExpr(
+    const CXXScopeSpec &SS, SourceLocation TemplateKWLoc, LookupResult &R,
+    const TemplateArgumentListInfo *TemplateArgs, const Scope *S,
+    UnresolvedLookupExpr *AsULE) {
   switch (ClassifyImplicitMemberAccess(*this, R)) {
   case IMA_Instance:
     return BuildImplicitMemberExpr(SS, TemplateKWLoc, R, TemplateArgs, true, S);
@@ -257,7 +255,7 @@ Sema::BuildPossibleImplicitMemberExpr(const CXXScopeSpec &SS,
   case IMA_Unresolved_StaticContext:
     if (TemplateArgs || TemplateKWLoc.isValid())
       return BuildTemplateIdExpr(SS, TemplateKWLoc, R, false, TemplateArgs);
-    return BuildDeclarationNameExpr(SS, R, false);
+    return AsULE ? AsULE : BuildDeclarationNameExpr(SS, R, false);
 
   case IMA_Error_StaticContext:
   case IMA_Error_Unrelated:
diff --git a/clang/test/SemaTemplate/member-access-expr.cpp b/clang/test/SemaTemplate/member-access-expr.cpp
index 36d6022577289..d6627b954a287 100644
--- a/clang/test/SemaTemplate/member-access-expr.cpp
+++ b/clang/test/SemaTemplate/member-access-expr.cpp
@@ -160,3 +160,14 @@ namespace test6 {
     }
   };
 }
+
+namespace test7 {
+  struct C { void g(); };
+  template<typename T> struct A {
+    T x;
+    static void f() {
+      (x.g()); // expected-error {{invalid use of member 'x' in static member function}}
+    }
+  };
+  void h() { A<C>::f(); }
+}

From a23f62343cb79a3306fa64545db1d61c2d76b9ca Mon Sep 17 00:00:00 2001
From: Wei Mi <wmi@google.com>
Date: Wed, 8 Jul 2020 15:19:44 -0700
Subject: [PATCH 0265/1035] Supplement instr profile with sample profile.

PGO profile is usually more precise than sample profile. However, PGO profile
needs to be collected from loadtest and loadtest may not be representative
enough to the production workload. Sample profile collected from production
can be used as a supplement -- for functions cold in loadtest but warm/hot
in production, we can scale up the related function in PGO profile if the
function is warm or hot in sample profile.

The implementation contains changes in compiler side and llvm-profdata side.
Given an instr profile and a sample profile, for a function cold in PGO
profile but warm/hot in sample profile, llvm-profdata will either mark
all the counters in the profile to be -1 or scale up the max count in the
function to be above hot threshold, depending on the zero counter ratio in
the profile. The assumption is if there are too many counters being zero
in the function profile, the profile is more likely to cause harm than good,
then llvm-profdata will mark all the counters to be -1 indicating the
function is hot but the profile is unaccountable. In compiler side, if a
function profile with all -1 counters is seen, the function entry count will
be set to be above hot threshold but its internal profile will be dropped.

In the long run, it may be useful to let compiler support using PGO profile
and sample profile at the same time, but that requires more careful design
and more substantial changes to make two profiles work seamlessly. The patch
here serves as a simple intermediate solution.

Differential Revision: https://reviews.llvm.org/D81981
---
 llvm/docs/CommandGuide/llvm-profdata.rst      |  24 +++
 llvm/include/llvm/ProfileData/InstrProf.h     |  12 +-
 .../llvm/ProfileData/InstrProfWriter.h        |   2 +
 llvm/lib/ProfileData/InstrProf.cpp            |  15 +-
 llvm/lib/ProfileData/InstrProfWriter.cpp      |   2 +-
 .../lib/ProfileData/ProfileSummaryBuilder.cpp |  11 +-
 .../Instrumentation/PGOInstrumentation.cpp    |  25 ++-
 .../PGOProfile/Inputs/sample-profile.proftext |  12 ++
 .../PGOProfile/Inputs/suppl-profile.proftext  |  15 ++
 .../Transforms/PGOProfile/suppl-profile.ll    |  37 ++++
 .../llvm-profdata/Inputs/mix_instr.proftext   |  25 +++
 .../llvm-profdata/Inputs/mix_sample.proftext  |  17 ++
 .../tools/llvm-profdata/overflow-instr.test   |  14 +-
 .../suppl-instr-with-sample.test              | 102 +++++++++
 llvm/tools/llvm-profdata/llvm-profdata.cpp    | 201 ++++++++++++++++++
 15 files changed, 488 insertions(+), 26 deletions(-)
 create mode 100644 llvm/test/Transforms/PGOProfile/Inputs/sample-profile.proftext
 create mode 100644 llvm/test/Transforms/PGOProfile/Inputs/suppl-profile.proftext
 create mode 100644 llvm/test/Transforms/PGOProfile/suppl-profile.ll
 create mode 100644 llvm/test/tools/llvm-profdata/Inputs/mix_instr.proftext
 create mode 100644 llvm/test/tools/llvm-profdata/Inputs/mix_sample.proftext
 create mode 100644 llvm/test/tools/llvm-profdata/suppl-instr-with-sample.test

diff --git a/llvm/docs/CommandGuide/llvm-profdata.rst b/llvm/docs/CommandGuide/llvm-profdata.rst
index 13a66dc48cef4..647232020e4b1 100644
--- a/llvm/docs/CommandGuide/llvm-profdata.rst
+++ b/llvm/docs/CommandGuide/llvm-profdata.rst
@@ -161,6 +161,30 @@ OPTIONS
  coverage for the optimized target. This option can only be used with
  sample-based profile in extbinary format.
 
+.. option:: -supplement-instr-with-sample=path_to_sample_profile
+
+ Supplement an instrumentation profile with sample profile. The sample profile
+ is the input of the flag. Output will be in instrumentation format (only works
+ with -instr).
+
+.. option:: -zero-counter-threshold=threshold_float_number
+
+ For the function which is cold in instr profile but hot in sample profile, if
+ the ratio of the number of zero counters divided by the the total number of
+ counters is above the threshold, the profile of the function will be regarded
+ as being harmful for performance and will be dropped.
+
+.. option:: -instr-prof-cold-threshold=threshold_int_number
+
+ User specified cold threshold for instr profile which will override the cold
+ threshold got from profile summary.
+
+.. option:: -suppl-min-size-threshold=threshold_int_number
+
+ If the size of a function is smaller than the threshold, assume it can be
+ inlined by PGO early inliner and it will not be adjusted based on sample
+ profile.
+
 EXAMPLES
 ^^^^^^^^
 Basic Usage
diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h
index a3359ca901339..50c4857537812 100644
--- a/llvm/include/llvm/ProfileData/InstrProf.h
+++ b/llvm/include/llvm/ProfileData/InstrProf.h
@@ -678,8 +678,8 @@ struct InstrProfValueSiteRecord {
   /// Optionally scale merged counts by \p Weight.
   void merge(InstrProfValueSiteRecord &Input, uint64_t Weight,
              function_ref<void(instrprof_error)> Warn);
-  /// Scale up value profile data counts.
-  void scale(uint64_t Weight, function_ref<void(instrprof_error)> Warn);
+  /// Scale up value profile data counts by N (Numerator) / D (Denominator).
+  void scale(uint64_t N, uint64_t D, function_ref<void(instrprof_error)> Warn);
 
   /// Compute the overlap b/w this record and Input record.
   void overlap(InstrProfValueSiteRecord &Input, uint32_t ValueKind,
@@ -753,8 +753,8 @@ struct InstrProfRecord {
              function_ref<void(instrprof_error)> Warn);
 
   /// Scale up profile counts (including value profile data) by
-  /// \p Weight.
-  void scale(uint64_t Weight, function_ref<void(instrprof_error)> Warn);
+  /// a factor of (N / D).
+  void scale(uint64_t N, uint64_t D, function_ref<void(instrprof_error)> Warn);
 
   /// Sort value profile data (per site) by count.
   void sortValueData() {
@@ -839,8 +839,8 @@ struct InstrProfRecord {
                           uint64_t Weight,
                           function_ref<void(instrprof_error)> Warn);
 
-  // Scale up value profile data count.
-  void scaleValueProfData(uint32_t ValueKind, uint64_t Weight,
+  // Scale up value profile data count by N (Numerator) / D (Denominator).
+  void scaleValueProfData(uint32_t ValueKind, uint64_t N, uint64_t D,
                           function_ref<void(instrprof_error)> Warn);
 };
 
diff --git a/llvm/include/llvm/ProfileData/InstrProfWriter.h b/llvm/include/llvm/ProfileData/InstrProfWriter.h
index 2d69bba26a29c..35c2669d55a69 100644
--- a/llvm/include/llvm/ProfileData/InstrProfWriter.h
+++ b/llvm/include/llvm/ProfileData/InstrProfWriter.h
@@ -48,6 +48,8 @@ class InstrProfWriter {
   InstrProfWriter(bool Sparse = false, bool InstrEntryBBEnabled = false);
   ~InstrProfWriter();
 
+  StringMap<ProfilingData> &getProfileData() { return FunctionData; }
+
   /// Add function counts for the given function. If there are already counts
   /// for this function and the hash and number of counts match, each counter is
   /// summed. Optionally scale counts by \p Weight.
diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp
index 9b429bf37d742..fb788ef4c7655 100644
--- a/llvm/lib/ProfileData/InstrProf.cpp
+++ b/llvm/lib/ProfileData/InstrProf.cpp
@@ -625,11 +625,11 @@ void InstrProfValueSiteRecord::merge(InstrProfValueSiteRecord &Input,
   }
 }
 
-void InstrProfValueSiteRecord::scale(uint64_t Weight,
+void InstrProfValueSiteRecord::scale(uint64_t N, uint64_t D,
                                      function_ref<void(instrprof_error)> Warn) {
   for (auto I = ValueData.begin(), IE = ValueData.end(); I != IE; ++I) {
     bool Overflowed;
-    I->Count = SaturatingMultiply(I->Count, Weight, &Overflowed);
+    I->Count = SaturatingMultiply(I->Count, N, &Overflowed) / D;
     if (Overflowed)
       Warn(instrprof_error::counter_overflow);
   }
@@ -678,22 +678,23 @@ void InstrProfRecord::merge(InstrProfRecord &Other, uint64_t Weight,
 }
 
 void InstrProfRecord::scaleValueProfData(
-    uint32_t ValueKind, uint64_t Weight,
+    uint32_t ValueKind, uint64_t N, uint64_t D,
     function_ref<void(instrprof_error)> Warn) {
   for (auto &R : getValueSitesForKind(ValueKind))
-    R.scale(Weight, Warn);
+    R.scale(N, D, Warn);
 }
 
-void InstrProfRecord::scale(uint64_t Weight,
+void InstrProfRecord::scale(uint64_t N, uint64_t D,
                             function_ref<void(instrprof_error)> Warn) {
+  assert(D != 0 && "D cannot be 0");
   for (auto &Count : this->Counts) {
     bool Overflowed;
-    Count = SaturatingMultiply(Count, Weight, &Overflowed);
+    Count = SaturatingMultiply(Count, N, &Overflowed) / D;
     if (Overflowed)
       Warn(instrprof_error::counter_overflow);
   }
   for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
-    scaleValueProfData(Kind, Weight, Warn);
+    scaleValueProfData(Kind, N, D, Warn);
 }
 
 // Map indirect call target name hash to name string.
diff --git a/llvm/lib/ProfileData/InstrProfWriter.cpp b/llvm/lib/ProfileData/InstrProfWriter.cpp
index 88445f186e835..d07668322354e 100644
--- a/llvm/lib/ProfileData/InstrProfWriter.cpp
+++ b/llvm/lib/ProfileData/InstrProfWriter.cpp
@@ -241,7 +241,7 @@ void InstrProfWriter::addRecord(StringRef Name, uint64_t Hash,
     // We've never seen a function with this name and hash, add it.
     Dest = std::move(I);
     if (Weight > 1)
-      Dest.scale(Weight, MapWarn);
+      Dest.scale(Weight, 1, MapWarn);
   } else {
     // We're updating a function we've seen before.
     Dest.merge(I, Weight, MapWarn);
diff --git a/llvm/lib/ProfileData/ProfileSummaryBuilder.cpp b/llvm/lib/ProfileData/ProfileSummaryBuilder.cpp
index 5d3a076409427..d2603097c550b 100644
--- a/llvm/lib/ProfileData/ProfileSummaryBuilder.cpp
+++ b/llvm/lib/ProfileData/ProfileSummaryBuilder.cpp
@@ -119,13 +119,22 @@ std::unique_ptr<ProfileSummary> InstrProfSummaryBuilder::getSummary() {
 }
 
 void InstrProfSummaryBuilder::addEntryCount(uint64_t Count) {
-  addCount(Count);
   NumFunctions++;
+
+  // Skip invalid count.
+  if (Count == (uint64_t)-1)
+    return;
+
+  addCount(Count);
   if (Count > MaxFunctionCount)
     MaxFunctionCount = Count;
 }
 
 void InstrProfSummaryBuilder::addInternalCount(uint64_t Count) {
+  // Skip invalid count.
+  if (Count == (uint64_t)-1)
+    return;
+
   addCount(Count);
   if (Count > MaxInternalBlockCount)
     MaxInternalBlockCount = Count;
diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
index c4a43abaa53cc..7a14f777b565a 100644
--- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
+++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
@@ -1020,7 +1020,8 @@ class PGOUseFunc {
         FreqAttr(FFA_Normal), IsCS(IsCS) {}
 
   // Read counts for the instrumented BB from profile.
-  bool readCounters(IndexedInstrProfReader *PGOReader, bool &AllZeros);
+  bool readCounters(IndexedInstrProfReader *PGOReader, bool &AllZeros,
+                    bool &AllMinusOnes);
 
   // Populate the counts for all BBs.
   void populateCounters();
@@ -1203,7 +1204,8 @@ void PGOUseFunc::setEdgeCount(DirectEdges &Edges, uint64_t Value) {
 // Read the profile from ProfileFileName and assign the value to the
 // instrumented BB and the edges. This function also updates ProgramMaxCount.
 // Return true if the profile are successfully read, and false on errors.
-bool PGOUseFunc::readCounters(IndexedInstrProfReader *PGOReader, bool &AllZeros) {
+bool PGOUseFunc::readCounters(IndexedInstrProfReader *PGOReader, bool &AllZeros,
+                              bool &AllMinusOnes) {
   auto &Ctx = M->getContext();
   Expected<InstrProfRecord> Result =
       PGOReader->getInstrProfRecord(FuncInfo.FuncName, FuncInfo.FunctionHash);
@@ -1246,10 +1248,13 @@ bool PGOUseFunc::readCounters(IndexedInstrProfReader *PGOReader, bool &AllZeros)
 
   IsCS ? NumOfCSPGOFunc++ : NumOfPGOFunc++;
   LLVM_DEBUG(dbgs() << CountFromProfile.size() << " counts\n");
+  AllMinusOnes = (CountFromProfile.size() > 0);
   uint64_t ValueSum = 0;
   for (unsigned I = 0, S = CountFromProfile.size(); I < S; I++) {
     LLVM_DEBUG(dbgs() << "  " << I << ": " << CountFromProfile[I] << "\n");
     ValueSum += CountFromProfile[I];
+    if (CountFromProfile[I] != (uint64_t)-1)
+      AllMinusOnes = false;
   }
   AllZeros = (ValueSum == 0);
 
@@ -1657,8 +1662,13 @@ static bool annotateAllFunctions(
     SplitIndirectBrCriticalEdges(F, BPI, BFI);
     PGOUseFunc Func(F, &M, TLI, ComdatMembers, BPI, BFI, PSI, IsCS,
                     InstrumentFuncEntry);
+    // When AllMinusOnes is true, it means the profile for the function
+    // is unrepresentative and this function is actually hot. Set the
+    // entry count of the function to be multiple times of hot threshold
+    // and drop all its internal counters.
+    bool AllMinusOnes = false;
     bool AllZeros = false;
-    if (!Func.readCounters(PGOReader.get(), AllZeros))
+    if (!Func.readCounters(PGOReader.get(), AllZeros, AllMinusOnes))
       continue;
     if (AllZeros) {
       F.setEntryCount(ProfileCount(0, Function::PCT_Real));
@@ -1666,6 +1676,15 @@ static bool annotateAllFunctions(
         ColdFunctions.push_back(&F);
       continue;
     }
+    const unsigned MultiplyFactor = 3;
+    if (AllMinusOnes) {
+      uint64_t HotThreshold = PSI->getHotCountThreshold();
+      if (HotThreshold)
+        F.setEntryCount(
+            ProfileCount(HotThreshold * MultiplyFactor, Function::PCT_Real));
+      HotFunctions.push_back(&F);
+      continue;
+    }
     Func.populateCounters();
     Func.setBranchWeights();
     Func.annotateValueSites();
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/sample-profile.proftext b/llvm/test/Transforms/PGOProfile/Inputs/sample-profile.proftext
new file mode 100644
index 0000000000000..0ab7207783eba
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/Inputs/sample-profile.proftext
@@ -0,0 +1,12 @@
+test_simple_for:4000:4000
+ 1: 1000
+ 2: 1000
+ 3: 1000
+ 4: 1000
+
+moo:10:10
+ 1: 2
+ 2: 2
+ 3: 2
+ 4: 2
+ 5: 2
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/suppl-profile.proftext b/llvm/test/Transforms/PGOProfile/Inputs/suppl-profile.proftext
new file mode 100644
index 0000000000000..c82311b3e0c06
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/Inputs/suppl-profile.proftext
@@ -0,0 +1,15 @@
+# :ir is the flag to indicate this is IR level profile.
+:ir
+test_simple_for
+34137660316
+2
+0
+0
+
+foo
+2582734
+4
+1000
+270
+180
+760
diff --git a/llvm/test/Transforms/PGOProfile/suppl-profile.ll b/llvm/test/Transforms/PGOProfile/suppl-profile.ll
new file mode 100644
index 0000000000000..e47838883dc6d
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/suppl-profile.ll
@@ -0,0 +1,37 @@
+; Supplement instr profile suppl-profile.proftext with sample profile
+; sample-profile.proftext.
+; RUN: llvm-profdata merge -instr -suppl-min-size-threshold=0 \
+; RUN:   -supplement-instr-with-sample=%p/Inputs/sample-profile.proftext \
+; RUN:   %S/Inputs/suppl-profile.proftext -o %t.profdata
+; RUN: opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s
+; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Check test_simple_for has a non-zero entry count and doesn't have any other
+; prof metadata.
+; CHECK: @test_simple_for(i32 %n) {{.*}} !prof ![[ENTRY_COUNT:[0-9]+]]
+; CHECK-NOT: !prof !
+; CHECK: ![[ENTRY_COUNT]] = !{!"function_entry_count", i64 540}
+define i32 @test_simple_for(i32 %n) {
+entry:
+  br label %for.cond
+
+for.cond:
+  %i = phi i32 [ 0, %entry ], [ %inc1, %for.inc ]
+  %sum = phi i32 [ 1, %entry ], [ %inc, %for.inc ]
+  %cmp = icmp slt i32 %i, %n
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:
+  %inc = add nsw i32 %sum, 1
+  br label %for.inc
+
+for.inc:
+  %inc1 = add nsw i32 %i, 1
+  br label %for.cond
+
+for.end:
+  ret i32 %sum
+}
diff --git a/llvm/test/tools/llvm-profdata/Inputs/mix_instr.proftext b/llvm/test/tools/llvm-profdata/Inputs/mix_instr.proftext
new file mode 100644
index 0000000000000..d7059e8c4cd7e
--- /dev/null
+++ b/llvm/test/tools/llvm-profdata/Inputs/mix_instr.proftext
@@ -0,0 +1,25 @@
+:ir
+foo
+7
+5
+12
+13
+0
+0
+0
+
+goo
+5
+3
+0
+0
+0
+
+moo
+9
+4
+3000
+1000
+2000
+500
+
diff --git a/llvm/test/tools/llvm-profdata/Inputs/mix_sample.proftext b/llvm/test/tools/llvm-profdata/Inputs/mix_sample.proftext
new file mode 100644
index 0000000000000..f61ec7fd5ede7
--- /dev/null
+++ b/llvm/test/tools/llvm-profdata/Inputs/mix_sample.proftext
@@ -0,0 +1,17 @@
+foo:2000:2000
+ 1: 2000
+goo:3000:1500
+ 1: 1200
+ 2: 800
+ 3: 1000
+moo:1000:1000
+ 1: 1000
+hoo:50:1
+ 1: 1
+ 2: 2
+ 3: 3
+ 4: 4
+ 5: 5
+ 6: 6
+ 7: 7
+ 8: 8
diff --git a/llvm/test/tools/llvm-profdata/overflow-instr.test b/llvm/test/tools/llvm-profdata/overflow-instr.test
index 5b9a94af9b294..73acbd937dd3b 100644
--- a/llvm/test/tools/llvm-profdata/overflow-instr.test
+++ b/llvm/test/tools/llvm-profdata/overflow-instr.test
@@ -2,16 +2,14 @@ Tests for overflow when merging instrumented profiles.
 
 1- Merge profile having maximum counts with itself and verify overflow detected and saturation occurred
 RUN: llvm-profdata merge -instr %p/Inputs/overflow-instr.proftext %p/Inputs/overflow-instr.proftext -o %t.out 2>&1 | FileCheck %s -check-prefix=MERGE_OVERFLOW
-RUN: llvm-profdata show -instr %t.out | FileCheck %s --check-prefix=SHOW_OVERFLOW
+RUN: llvm-profdata show -instr -all-functions -counts %t.out | FileCheck %s --check-prefix=SHOW_OVERFLOW
 MERGE_OVERFLOW: {{.*}}: overflow: Counter overflow
-SHOW_OVERFLOW: Total functions: 1
-SHOW_OVERFLOW-NEXT: Maximum function count: 18446744073709551615
-SHOW_OVERFLOW-NEXT: Maximum internal block count: 18446744073709551615
+SHOW_OVERFLOW: Function count: 18446744073709551615
+SHOW_OVERFLOW-NEXT: Block counts: [18446744073709551615, 18446744073709551615]
 
 2- Merge profile having maximum counts by itself and verify no overflow
 RUN: llvm-profdata merge -instr %p/Inputs/overflow-instr.proftext -o %t.out 2>&1 | FileCheck %s -check-prefix=MERGE_NO_OVERFLOW -allow-empty
-RUN: llvm-profdata show -instr %t.out | FileCheck %s --check-prefix=SHOW_NO_OVERFLOW
+RUN: llvm-profdata show -instr -all-functions -counts %t.out | FileCheck %s --check-prefix=SHOW_NO_OVERFLOW
 MERGE_NO_OVERFLOW-NOT: {{.*}}: overflow: Counter overflow
-SHOW_NO_OVERFLOW: Total functions: 1
-SHOW_NO_OVERFLOW-NEXT: Maximum function count: 18446744073709551615
-SHOW_NO_OVERFLOW-NEXT: Maximum internal block count: 18446744073709551615
+SHOW_NO_OVERFLOW: Function count: 18446744073709551615
+SHOW_NO_OVERFLOW-NEXT: Block counts: [9223372036854775808, 18446744073709551615]
diff --git a/llvm/test/tools/llvm-profdata/suppl-instr-with-sample.test b/llvm/test/tools/llvm-profdata/suppl-instr-with-sample.test
new file mode 100644
index 0000000000000..29d3c7c66b0f3
--- /dev/null
+++ b/llvm/test/tools/llvm-profdata/suppl-instr-with-sample.test
@@ -0,0 +1,102 @@
+Some basic tests for supplementing instrumentation profile with sample profile.
+
+Test all of goo's counters will be set to -1.
+RUN: llvm-profdata merge \
+RUN:     -supplement-instr-with-sample=%p/Inputs/mix_sample.proftext \
+RUN:     -suppl-min-size-threshold=0 %p/Inputs/mix_instr.proftext -o %t
+RUN: llvm-profdata show %t -all-functions -counts | FileCheck %s --check-prefix=MIX1
+
+MIX1: foo:
+MIX1-NEXT: Hash: 0x0000000000000007
+MIX1-NEXT: Counters: 5
+MIX1-NEXT: Block counts: [12, 13, 0, 0, 0]
+MIX1: goo:
+MIX1-NEXT: Hash: 0x0000000000000005
+MIX1-NEXT: Counters: 3
+MIX1-NEXT: Block counts: [18446744073709551615, 18446744073709551615, 18446744073709551615]
+MIX1: moo:
+MIX1-NEXT: Hash: 0x0000000000000009
+MIX1-NEXT: Counters: 4
+MIX1-NEXT: Block counts: [3000, 1000, 2000, 500]
+
+Test when the zero counter ratio of foo is higher than zero-counter-threshold.
+RUN: llvm-profdata merge \
+RUN:     -supplement-instr-with-sample=%p/Inputs/mix_sample.proftext \
+RUN:     -suppl-min-size-threshold=0 -zero-counter-threshold=0.5 \
+RUN:     -instr-prof-cold-threshold=30 %p/Inputs/mix_instr.proftext -o %t
+RUN: llvm-profdata show %t -all-functions -counts | FileCheck %s --check-prefix=MIX2
+
+MIX2: foo:
+MIX2-NEXT: Hash: 0x0000000000000007
+MIX2-NEXT: Counters: 5
+MIX2-NEXT: Block counts: [18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615]
+MIX2: goo:
+MIX2-NEXT: Hash: 0x0000000000000005
+MIX2-NEXT: Counters: 3
+MIX2-NEXT: Block counts: [18446744073709551615, 18446744073709551615, 18446744073709551615]
+MIX2: moo:
+MIX2-NEXT: Hash: 0x0000000000000009
+MIX2-NEXT: Counters: 4
+MIX2-NEXT: Block counts: [3000, 1000, 2000, 500]
+
+Test when the zero counter ratio of foo is lower than zero-counter-threshold.
+RUN: llvm-profdata merge \
+RUN:     -supplement-instr-with-sample=%p/Inputs/mix_sample.proftext \
+RUN:     -suppl-min-size-threshold=0 -zero-counter-threshold=0.7 \
+RUN:     -instr-prof-cold-threshold=30 %p/Inputs/mix_instr.proftext -o %t
+RUN: llvm-profdata show %t -all-functions -counts | FileCheck %s --check-prefix=MIX3
+
+MIX3: foo:
+MIX3-NEXT: Hash: 0x0000000000000007
+MIX3-NEXT: Counters: 5
+MIX3-NEXT: Block counts: [1384, 1500, 0, 0, 0]
+MIX3: goo:
+MIX3-NEXT: Hash: 0x0000000000000005
+MIX3-NEXT: Counters: 3
+MIX3-NEXT: Block counts: [18446744073709551615, 18446744073709551615, 18446744073709551615]
+MIX3: moo:
+MIX3-NEXT: Hash: 0x0000000000000009
+MIX3-NEXT: Counters: 4
+MIX3-NEXT: Block counts: [3000, 1000, 2000, 500]
+
+Test foo's profile won't be adjusted because its size is smaller
+than suppl-min-size-threshold.
+RUN: llvm-profdata merge \
+RUN:     -supplement-instr-with-sample=%p/Inputs/mix_sample.proftext \
+RUN:     -suppl-min-size-threshold=2 -zero-counter-threshold=0.7 \
+RUN:     -instr-prof-cold-threshold=30 %p/Inputs/mix_instr.proftext -o %t
+RUN: llvm-profdata show %t -all-functions -counts | FileCheck %s --check-prefix=MIX4
+
+MIX4: foo:
+MIX4-NEXT: Hash: 0x0000000000000007
+MIX4-NEXT: Counters: 5
+MIX4-NEXT: Block counts: [12, 13, 0, 0, 0]
+MIX4: goo:
+MIX4-NEXT: Hash: 0x0000000000000005
+MIX4-NEXT: Counters: 3
+MIX4-NEXT: Block counts: [18446744073709551615, 18446744073709551615, 18446744073709551615]
+MIX4: moo:
+MIX4-NEXT: Hash: 0x0000000000000009
+MIX4-NEXT: Counters: 4
+MIX4-NEXT: Block counts: [3000, 1000, 2000, 500]
+
+Test profile summary won't be affected by -1 counter.
+RUN: llvm-profdata merge \
+RUN:     -supplement-instr-with-sample=%p/Inputs/mix_sample.proftext \
+RUN:     -suppl-min-size-threshold=0 %p/Inputs/mix_instr.proftext -o %t
+RUN: llvm-profdata show %t -detailed-summary | FileCheck %s --check-prefix=MIX5
+
+MIX5: Instrumentation level: IR
+MIX5-NEXT: Total functions: 3
+MIX5-NEXT: Maximum function count: 3000
+MIX5-NEXT: Maximum internal block count: 2000
+MIX5-NEXT: Total number of blocks: 9
+MIX5-NEXT: Total count: 6525
+MIX5-NEXT: Detailed summary:
+MIX5-NEXT: 3 blocks with count >= 1000 account for 80 percentage of the total counts.
+MIX5-NEXT: 3 blocks with count >= 1000 account for 90 percentage of the total counts.
+MIX5-NEXT: 4 blocks with count >= 500 account for 95 percentage of the total counts.
+MIX5-NEXT: 4 blocks with count >= 500 account for 99 percentage of the total counts.
+MIX5-NEXT: 6 blocks with count >= 12 account for 99.9 percentage of the total counts.
+MIX5-NEXT: 6 blocks with count >= 12 account for 99.99 percentage of the total counts.
+MIX5-NEXT: 6 blocks with count >= 12 account for 99.999 percentage of the total counts.
diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp
index 41f6a4d723eec..771aec89720eb 100644
--- a/llvm/tools/llvm-profdata/llvm-profdata.cpp
+++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp
@@ -386,6 +386,172 @@ static void mergeInstrProfile(const WeightedFileVector &Inputs,
   writeInstrProfile(OutputFilename, OutputFormat, Contexts[0]->Writer);
 }
 
+/// The profile entry for a function in instrumentation profile.
+struct InstrProfileEntry {
+  uint64_t MaxCount = 0;
+  float ZeroCounterRatio = 0.0;
+  InstrProfRecord *ProfRecord;
+  InstrProfileEntry(InstrProfRecord *Record);
+  InstrProfileEntry() = default;
+};
+
+InstrProfileEntry::InstrProfileEntry(InstrProfRecord *Record) {
+  ProfRecord = Record;
+  uint64_t CntNum = Record->Counts.size();
+  uint64_t ZeroCntNum = 0;
+  for (size_t I = 0; I < CntNum; ++I) {
+    MaxCount = std::max(MaxCount, Record->Counts[I]);
+    ZeroCntNum += !Record->Counts[I];
+  }
+  ZeroCounterRatio = (float)ZeroCntNum / CntNum;
+}
+
+/// Either set all the counters in the instr profile entry \p IFE to -1
+/// in order to drop the profile or scale up the counters in \p IFP to
+/// be above hot threshold. We use the ratio of zero counters in the
+/// profile of a function to decide the profile is helpful or harmful
+/// for performance, and to choose whether to scale up or drop it.
+static void updateInstrProfileEntry(InstrProfileEntry &IFE,
+                                    uint64_t HotInstrThreshold,
+                                    float ZeroCounterThreshold) {
+  InstrProfRecord *ProfRecord = IFE.ProfRecord;
+  if (!IFE.MaxCount || IFE.ZeroCounterRatio > ZeroCounterThreshold) {
+    // If all or most of the counters of the function are zero, the
+    // profile is unaccountable and shuld be dropped. Reset all the
+    // counters to be -1 and PGO profile-use will drop the profile.
+    // All counters being -1 also implies that the function is hot so
+    // PGO profile-use will also set the entry count metadata to be
+    // above hot threshold.
+    for (size_t I = 0; I < ProfRecord->Counts.size(); ++I)
+      ProfRecord->Counts[I] = -1;
+    return;
+  }
+
+  // Scale up the MaxCount to be multiple times above hot threshold.
+  const unsigned MultiplyFactor = 3;
+  uint64_t Numerator = HotInstrThreshold * MultiplyFactor;
+  uint64_t Denominator = IFE.MaxCount;
+  ProfRecord->scale(Numerator, Denominator, [&](instrprof_error E) {
+    warn(toString(make_error<InstrProfError>(E)));
+  });
+}
+
+const uint64_t ColdPercentileIdx = 15;
+const uint64_t HotPercentileIdx = 11;
+
+/// Adjust the instr profile in \p WC based on the sample profile in
+/// \p Reader.
+static void
+adjustInstrProfile(std::unique_ptr<WriterContext> &WC,
+                   std::unique_ptr<sampleprof::SampleProfileReader> &Reader,
+                   unsigned SupplMinSizeThreshold, float ZeroCounterThreshold,
+                   unsigned InstrProfColdThreshold) {
+  // Function to its entry in instr profile.
+  StringMap<InstrProfileEntry> InstrProfileMap;
+  InstrProfSummaryBuilder IPBuilder(ProfileSummaryBuilder::DefaultCutoffs);
+  for (auto &PD : WC->Writer.getProfileData()) {
+    // Populate IPBuilder.
+    for (const auto &PDV : PD.getValue()) {
+      InstrProfRecord Record = PDV.second;
+      IPBuilder.addRecord(Record);
+    }
+
+    // If a function has multiple entries in instr profile, skip it.
+    if (PD.getValue().size() != 1)
+      continue;
+
+    // Initialize InstrProfileMap.
+    InstrProfRecord *R = &PD.getValue().begin()->second;
+    InstrProfileMap[PD.getKey()] = InstrProfileEntry(R);
+  }
+
+  ProfileSummary InstrPS = *IPBuilder.getSummary();
+  ProfileSummary SamplePS = Reader->getSummary();
+
+  // Compute cold thresholds for instr profile and sample profile.
+  uint64_t ColdSampleThreshold =
+      ProfileSummaryBuilder::getEntryForPercentile(
+          SamplePS.getDetailedSummary(),
+          ProfileSummaryBuilder::DefaultCutoffs[ColdPercentileIdx])
+          .MinCount;
+  uint64_t HotInstrThreshold =
+      ProfileSummaryBuilder::getEntryForPercentile(
+          InstrPS.getDetailedSummary(),
+          ProfileSummaryBuilder::DefaultCutoffs[HotPercentileIdx])
+          .MinCount;
+  uint64_t ColdInstrThreshold =
+      InstrProfColdThreshold
+          ? InstrProfColdThreshold
+          : ProfileSummaryBuilder::getEntryForPercentile(
+                InstrPS.getDetailedSummary(),
+                ProfileSummaryBuilder::DefaultCutoffs[ColdPercentileIdx])
+                .MinCount;
+
+  // Find hot/warm functions in sample profile which is cold in instr profile
+  // and adjust the profiles of those functions in the instr profile.
+  for (const auto &PD : Reader->getProfiles()) {
+    StringRef FName = PD.getKey();
+    const sampleprof::FunctionSamples &FS = PD.getValue();
+    auto It = InstrProfileMap.find(FName);
+    if (FS.getHeadSamples() > ColdSampleThreshold &&
+        It != InstrProfileMap.end() &&
+        It->second.MaxCount <= ColdInstrThreshold &&
+        FS.getBodySamples().size() >= SupplMinSizeThreshold) {
+      updateInstrProfileEntry(It->second, HotInstrThreshold,
+                              ZeroCounterThreshold);
+    }
+  }
+}
+
+/// The main function to supplement instr profile with sample profile.
+/// \Inputs contains the instr profile. \p SampleFilename specifies the
+/// sample profile. \p OutputFilename specifies the output profile name.
+/// \p OutputFormat specifies the output profile format. \p OutputSparse
+/// specifies whether to generate sparse profile. \p SupplMinSizeThreshold
+/// specifies the minimal size for the functions whose profile will be
+/// adjusted. \p ZeroCounterThreshold is the threshold to check whether
+/// a function contains too many zero counters and whether its profile
+/// should be dropped. \p InstrProfColdThreshold is the user specified
+/// cold threshold which will override the cold threshold got from the
+/// instr profile summary.
+static void supplementInstrProfile(
+    const WeightedFileVector &Inputs, StringRef SampleFilename,
+    StringRef OutputFilename, ProfileFormat OutputFormat, bool OutputSparse,
+    unsigned SupplMinSizeThreshold, float ZeroCounterThreshold,
+    unsigned InstrProfColdThreshold) {
+  if (OutputFilename.compare("-") == 0)
+    exitWithError("Cannot write indexed profdata format to stdout.");
+  if (Inputs.size() != 1)
+    exitWithError("Expect one input to be an instr profile.");
+  if (Inputs[0].Weight != 1)
+    exitWithError("Expect instr profile doesn't have weight.");
+
+  StringRef InstrFilename = Inputs[0].Filename;
+
+  // Read sample profile.
+  LLVMContext Context;
+  auto ReaderOrErr =
+      sampleprof::SampleProfileReader::create(SampleFilename.str(), Context);
+  if (std::error_code EC = ReaderOrErr.getError())
+    exitWithErrorCode(EC, SampleFilename);
+  auto Reader = std::move(ReaderOrErr.get());
+  if (std::error_code EC = Reader->read())
+    exitWithErrorCode(EC, SampleFilename);
+
+  // Read instr profile.
+  std::mutex ErrorLock;
+  SmallSet<instrprof_error, 4> WriterErrorCodes;
+  auto WC = std::make_unique<WriterContext>(OutputSparse, ErrorLock,
+                                            WriterErrorCodes);
+  loadInput(Inputs[0], nullptr, WC.get());
+  if (WC->Errors.size() > 0)
+    exitWithError(std::move(WC->Errors[0].first), InstrFilename);
+
+  adjustInstrProfile(WC, Reader, SupplMinSizeThreshold, ZeroCounterThreshold,
+                     InstrProfColdThreshold);
+  writeInstrProfile(OutputFilename, OutputFormat, WC->Writer);
+}
+
 /// Make a copy of the given function samples with all symbol names remapped
 /// by the provided symbol remapper.
 static sampleprof::FunctionSamples
@@ -680,6 +846,28 @@ static int merge_main(int argc, const char *argv[]) {
   cl::opt<bool> GenPartialProfile(
       "gen-partial-profile", cl::init(false), cl::Hidden,
       cl::desc("Generate a partial profile (only meaningful for -extbinary)"));
+  cl::opt<std::string> SupplInstrWithSample(
+      "supplement-instr-with-sample", cl::init(""), cl::Hidden,
+      cl::desc("Supplement an instr profile with sample profile, to correct "
+               "the profile unrepresentativeness issue. The sample "
+               "profile is the input of the flag. Output will be in instr "
+               "format (The flag only works with -instr)"));
+  cl::opt<float> ZeroCounterThreshold(
+      "zero-counter-threshold", cl::init(0.7), cl::Hidden,
+      cl::desc("For the function which is cold in instr profile but hot in "
+               "sample profile, if the ratio of the number of zero counters "
+               "divided by the the total number of counters is above the "
+               "threshold, the profile of the function will be regarded as "
+               "being harmful for performance and will be dropped. "));
+  cl::opt<unsigned> SupplMinSizeThreshold(
+      "suppl-min-size-threshold", cl::init(10), cl::Hidden,
+      cl::desc("If the size of a function is smaller than the threshold, "
+               "assume it can be inlined by PGO early inliner and it won't "
+               "be adjusted based on sample profile. "));
+  cl::opt<unsigned> InstrProfColdThreshold(
+      "instr-prof-cold-threshold", cl::init(0), cl::Hidden,
+      cl::desc("User specified cold threshold for instr profile which will "
+               "override the cold threshold got from profile summary. "));
 
   cl::ParseCommandLineOptions(argc, argv, "LLVM profile data merger\n");
 
@@ -708,6 +896,17 @@ static int merge_main(int argc, const char *argv[]) {
   if (!RemappingFile.empty())
     Remapper = SymbolRemapper::create(RemappingFile);
 
+  if (!SupplInstrWithSample.empty()) {
+    if (ProfileKind != instr)
+      exitWithError(
+          "-supplement-instr-with-sample can only work with -instr. ");
+
+    supplementInstrProfile(WeightedInputs, SupplInstrWithSample, OutputFilename,
+                           OutputFormat, OutputSparse, SupplMinSizeThreshold,
+                           ZeroCounterThreshold, InstrProfColdThreshold);
+    return 0;
+  }
+
   if (ProfileKind == instr)
     mergeInstrProfile(WeightedInputs, Remapper.get(), OutputFilename,
                       OutputFormat, OutputSparse, NumThreads, FailureMode);
@@ -904,6 +1103,8 @@ static int showInstrProfile(const std::string &Filename, bool ShowCounts,
     uint64_t FuncMax = 0;
     uint64_t FuncSum = 0;
     for (size_t I = 0, E = Func.Counts.size(); I < E; ++I) {
+      if (Func.Counts[I] == (uint64_t)-1)
+        continue;
       FuncMax = std::max(FuncMax, Func.Counts[I]);
       FuncSum += Func.Counts[I];
     }

From a0ebac52df6d890fcba52e7db9ac66d0fc7c2582 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@intel.com>
Date: Mon, 27 Jul 2020 21:11:42 -0700
Subject: [PATCH 0266/1035] [X86] Properly encode a 32-bit address with an
 index register and no base register in 16-bit mode.

In 16-bit mode we can encode a 32-bit address using 0x67 prefix.
We were failing to do this when the index register was a 32-bit
register, the base register was not present, and the displacement
fit in 16-bits.

Fixes PR46866.
---
 .../X86/MCTargetDesc/X86MCCodeEmitter.cpp     | 20 +++++++++++--------
 llvm/test/MC/X86/code16gcc.s                  |  2 ++
 2 files changed, 14 insertions(+), 8 deletions(-)

diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp
index 7dea0760a8310..5f1b5b5e2b96e 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp
@@ -164,17 +164,20 @@ static MCFixupKind getImmFixupKind(uint64_t TSFlags) {
 /// \returns true if the specified instruction has a 16-bit memory operand.
 static bool is16BitMemOperand(const MCInst &MI, unsigned Op,
                               const MCSubtargetInfo &STI) {
-  const MCOperand &BaseReg = MI.getOperand(Op + X86::AddrBaseReg);
-  const MCOperand &IndexReg = MI.getOperand(Op + X86::AddrIndexReg);
+  const MCOperand &Base = MI.getOperand(Op + X86::AddrBaseReg);
+  const MCOperand &Index = MI.getOperand(Op + X86::AddrIndexReg);
   const MCOperand &Disp = MI.getOperand(Op + X86::AddrDisp);
 
-  if (STI.hasFeature(X86::Mode16Bit) && BaseReg.getReg() == 0 && Disp.isImm() &&
-      Disp.getImm() < 0x10000)
+  unsigned BaseReg = Base.getReg();
+  unsigned IndexReg = Index.getReg();
+
+  if (STI.hasFeature(X86::Mode16Bit) && BaseReg == 0 && IndexReg == 0 &&
+      Disp.isImm() && Disp.getImm() < 0x10000)
     return true;
-  if ((BaseReg.getReg() != 0 &&
-       X86MCRegisterClasses[X86::GR16RegClassID].contains(BaseReg.getReg())) ||
-      (IndexReg.getReg() != 0 &&
-       X86MCRegisterClasses[X86::GR16RegClassID].contains(IndexReg.getReg())))
+  if ((BaseReg != 0 &&
+       X86MCRegisterClasses[X86::GR16RegClassID].contains(BaseReg)) ||
+      (IndexReg != 0 &&
+       X86MCRegisterClasses[X86::GR16RegClassID].contains(IndexReg)))
     return true;
   return false;
 }
@@ -498,6 +501,7 @@ void X86MCCodeEmitter::emitMemModRMByte(const MCInst &MI, unsigned Op,
       // This is the [REG]+disp16 case.
       emitByte(modRMByte(2, RegOpcodeField, RMfield), OS);
     } else {
+      assert(IndexReg.getReg() == 0 && "Unexpected index register!");
       // There is no BaseReg; this is the plain [disp16] case.
       emitByte(modRMByte(0, RegOpcodeField, 6), OS);
     }
diff --git a/llvm/test/MC/X86/code16gcc.s b/llvm/test/MC/X86/code16gcc.s
index 2391aeffe99a0..8d8589a1117da 100644
--- a/llvm/test/MC/X86/code16gcc.s
+++ b/llvm/test/MC/X86/code16gcc.s
@@ -62,6 +62,8 @@
 	//CHECK:	popfl                           # encoding: [0x66,0x9d]
 	pushw 4
 	//CHECK:	pushw	4                       # encoding: [0xff,0x36,0x04,0x00]
+	addw $1, (,%eax,4)
+	//CHECK:	addw $1, (,%eax,4)              # encoding: [0x67,0x83,0x04,0x85,0x00,0x00,0x00,0x00,0x01]
 
 	
 

From 25f193fb46dbdcc178946765aa929535199e2a4b Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@intel.com>
Date: Mon, 27 Jul 2020 21:11:48 -0700
Subject: [PATCH 0267/1035] [X86] Add support for {disp32} to control size of
 jmp and jcc instructions in the assembler

By default we pick a 1 byte displacement and let relaxation enlarge it if necessary. The GNU assembler supports a pseudo prefix to basically pre-relax the instruction the larger size.

I plan to add {disp8} and {disp32} support for memory operands in another patch which is why I've included the parsing code and enum for {disp8} pseudo prefix as well.

Reviewed By: echristo

Differential Revision: https://reviews.llvm.org/D84709
---
 .../lib/Target/X86/AsmParser/X86AsmParser.cpp | 33 +++++++++++++++++++
 llvm/test/MC/X86/x86-16.s                     | 11 +++++++
 llvm/test/MC/X86/x86-32.s                     | 11 +++++++
 3 files changed, 55 insertions(+)

diff --git a/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp b/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
index a3014b2aba92c..bb9919a25847b 100644
--- a/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
+++ b/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
@@ -87,6 +87,14 @@ class X86AsmParser : public MCTargetAsmParser {
 
   VEXEncoding ForcedVEXEncoding = VEXEncoding_Default;
 
+  enum DispEncoding {
+    DispEncoding_Default,
+    DispEncoding_Disp8,
+    DispEncoding_Disp32,
+  };
+
+  DispEncoding ForcedDispEncoding = DispEncoding_Default;
+
 private:
   SMLoc consumeToken() {
     MCAsmParser &Parser = getParser();
@@ -2592,6 +2600,7 @@ bool X86AsmParser::ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
 
   // Reset the forced VEX encoding.
   ForcedVEXEncoding = VEXEncoding_Default;
+  ForcedDispEncoding = DispEncoding_Default;
 
   // Parse pseudo prefixes.
   while (1) {
@@ -2610,6 +2619,10 @@ bool X86AsmParser::ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
         ForcedVEXEncoding = VEXEncoding_VEX3;
       else if (Prefix == "evex")
         ForcedVEXEncoding = VEXEncoding_EVEX;
+      else if (Prefix == "disp8")
+        ForcedDispEncoding = DispEncoding_Disp8;
+      else if (Prefix == "disp32")
+        ForcedDispEncoding = DispEncoding_Disp32;
       else
         return Error(NameLoc, "unknown prefix");
 
@@ -3118,6 +3131,26 @@ bool X86AsmParser::processInstruction(MCInst &Inst, const OperandVector &Ops) {
 
   switch (Inst.getOpcode()) {
   default: return false;
+  case X86::JMP_1:
+    // {disp32} forces a larger displacement as if the instruction was relaxed.
+    // NOTE: 16-bit mode uses 16-bit displacement even though it says {disp32}.
+    // This matches GNU assembler.
+    if (ForcedDispEncoding == DispEncoding_Disp32) {
+      Inst.setOpcode(is16BitMode() ? X86::JMP_2 : X86::JMP_4);
+      return true;
+    }
+
+    return false;
+  case X86::JCC_1:
+    // {disp32} forces a larger displacement as if the instruction was relaxed.
+    // NOTE: 16-bit mode uses 16-bit displacement even though it says {disp32}.
+    // This matches GNU assembler.
+    if (ForcedDispEncoding == DispEncoding_Disp32) {
+      Inst.setOpcode(is16BitMode() ? X86::JCC_2 : X86::JCC_4);
+      return true;
+    }
+
+    return false;
   case X86::VMOVZPQILo2PQIrr:
   case X86::VMOVAPDrr:
   case X86::VMOVAPDYrr:
diff --git a/llvm/test/MC/X86/x86-16.s b/llvm/test/MC/X86/x86-16.s
index ed3540902894e..f92164e57314a 100644
--- a/llvm/test/MC/X86/x86-16.s
+++ b/llvm/test/MC/X86/x86-16.s
@@ -1045,3 +1045,14 @@ xsusldtrk
 // CHECK: xresldtrk
 // CHECK: encoding: [0xf2,0x0f,0x01,0xe9]
 xresldtrk
+
+// CHECK: jmp foo
+// CHECK:  encoding: [0xe9,A,A]
+// CHECK:  fixup A - offset: 1, value: foo-2, kind: FK_PCRel_2
+{disp32} jmp foo
+foo:
+
+// CHECK: je foo
+// CHECK:  encoding: [0x0f,0x84,A,A]
+// CHECK:  fixup A - offset: 2, value: foo-2, kind: FK_PCRel_2
+{disp32} je foo
diff --git a/llvm/test/MC/X86/x86-32.s b/llvm/test/MC/X86/x86-32.s
index fdd3c53ed88f5..256d8351e74d8 100644
--- a/llvm/test/MC/X86/x86-32.s
+++ b/llvm/test/MC/X86/x86-32.s
@@ -1109,3 +1109,14 @@ ptwritel 0xdeadbeef(%ebx,%ecx,8)
 // CHECK: ptwritel %eax
 // CHECK:  encoding: [0xf3,0x0f,0xae,0xe0]
 ptwritel %eax
+
+// CHECK: jmp foo
+// CHECK:  encoding: [0xe9,A,A,A,A]
+// CHECK:  fixup A - offset: 1, value: foo-4, kind: FK_PCRel_4
+{disp32} jmp foo
+foo:
+
+// CHECK: je foo
+// CHECK:  encoding: [0x0f,0x84,A,A,A,A]
+// CHECK:  fixup A - offset: 2, value: foo-4, kind: FK_PCRel_4
+{disp32} je foo

From 647e861e080382593648b234668ad2f5a376ac5e Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@intel.com>
Date: Mon, 27 Jul 2020 21:11:54 -0700
Subject: [PATCH 0268/1035] [X86] Detect if EFLAGs is live across XBEGIN pseudo
 instruction. Add it as livein to the basic blocks created when expanding the
 pseudo

XBEGIN causes several based blocks to be inserted. If flags are live across it we need to make eflags live in the new basic blocks to avoid machine verifier errors.

Fixes PR46827

Reviewed By: ivanbaev

Differential Revision: https://reviews.llvm.org/D84479
---
 llvm/lib/Target/X86/X86ISelLowering.cpp | 57 ++++++++++++++++---------
 llvm/test/CodeGen/X86/pr46827.ll        | 39 +++++++++++++++++
 2 files changed, 75 insertions(+), 21 deletions(-)
 create mode 100644 llvm/test/CodeGen/X86/pr46827.ll

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index b986c42b9563f..390a20c3e71f5 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -30954,6 +30954,34 @@ bool X86TargetLowering::areJTsAllowed(const Function *Fn) const {
 //                           X86 Scheduler Hooks
 //===----------------------------------------------------------------------===//
 
+// Returns true if EFLAG is consumed after this iterator in the rest of the
+// basic block or any successors of the basic block.
+static bool isEFLAGSLiveAfter(MachineBasicBlock::iterator Itr,
+                              MachineBasicBlock *BB) {
+  // Scan forward through BB for a use/def of EFLAGS.
+  for (MachineBasicBlock::iterator miI = std::next(Itr), miE = BB->end();
+         miI != miE; ++miI) {
+    const MachineInstr& mi = *miI;
+    if (mi.readsRegister(X86::EFLAGS))
+      return true;
+    // If we found a def, we can stop searching.
+    if (mi.definesRegister(X86::EFLAGS))
+      return false;
+  }
+
+  // If we hit the end of the block, check whether EFLAGS is live into a
+  // successor.
+  for (MachineBasicBlock::succ_iterator sItr = BB->succ_begin(),
+                                        sEnd = BB->succ_end();
+       sItr != sEnd; ++sItr) {
+    MachineBasicBlock* succ = *sItr;
+    if (succ->isLiveIn(X86::EFLAGS))
+      return true;
+  }
+
+  return false;
+}
+
 /// Utility function to emit xbegin specifying the start of an RTM region.
 static MachineBasicBlock *emitXBegin(MachineInstr &MI, MachineBasicBlock *MBB,
                                      const TargetInstrInfo *TII) {
@@ -30986,6 +31014,12 @@ static MachineBasicBlock *emitXBegin(MachineInstr &MI, MachineBasicBlock *MBB,
   MF->insert(I, fallMBB);
   MF->insert(I, sinkMBB);
 
+  if (isEFLAGSLiveAfter(MI, MBB)) {
+    mainMBB->addLiveIn(X86::EFLAGS);
+    fallMBB->addLiveIn(X86::EFLAGS);
+    sinkMBB->addLiveIn(X86::EFLAGS);
+  }
+
   // Transfer the remainder of BB and its successor edges to sinkMBB.
   sinkMBB->splice(sinkMBB->begin(), MBB,
                   std::next(MachineBasicBlock::iterator(MI)), MBB->end());
@@ -31374,27 +31408,8 @@ MachineBasicBlock *X86TargetLowering::EmitVAStartSaveXMMRegsWithCustomInserter(
 static bool checkAndUpdateEFLAGSKill(MachineBasicBlock::iterator SelectItr,
                                      MachineBasicBlock* BB,
                                      const TargetRegisterInfo* TRI) {
-  // Scan forward through BB for a use/def of EFLAGS.
-  MachineBasicBlock::iterator miI(std::next(SelectItr));
-  for (MachineBasicBlock::iterator miE = BB->end(); miI != miE; ++miI) {
-    const MachineInstr& mi = *miI;
-    if (mi.readsRegister(X86::EFLAGS))
-      return false;
-    if (mi.definesRegister(X86::EFLAGS))
-      break; // Should have kill-flag - update below.
-  }
-
-  // If we hit the end of the block, check whether EFLAGS is live into a
-  // successor.
-  if (miI == BB->end()) {
-    for (MachineBasicBlock::succ_iterator sItr = BB->succ_begin(),
-                                          sEnd = BB->succ_end();
-         sItr != sEnd; ++sItr) {
-      MachineBasicBlock* succ = *sItr;
-      if (succ->isLiveIn(X86::EFLAGS))
-        return false;
-    }
-  }
+  if (isEFLAGSLiveAfter(SelectItr, BB))
+    return false;
 
   // We found a def, or hit the end of the basic block and EFLAGS wasn't live
   // out. SelectMI should have a kill flag on EFLAGS.
diff --git a/llvm/test/CodeGen/X86/pr46827.ll b/llvm/test/CodeGen/X86/pr46827.ll
new file mode 100644
index 0000000000000..438b13c3400fe
--- /dev/null
+++ b/llvm/test/CodeGen/X86/pr46827.ll
@@ -0,0 +1,39 @@
+; RUN: llc < %s -mtriple=i686-pc-linux -mattr=+rtm -verify-machineinstrs -stop-after=finalize-isel | FileCheck %s
+
+; CHECK: body:             |
+; CHECK:   bb.0.bb107:
+; CHECK:     successors: %bb.3(0x40000000), %bb.4(0x40000000)
+; CHECK:     %0:gr32 = MOV32rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (load 4 from %fixed-stack.0, align 16)
+; CHECK:     %1:gr32 = SUB32ri8 %0, 1, implicit-def $eflags
+; CHECK:     XBEGIN_4 %bb.4, implicit-def $eax
+; CHECK:   bb.3.bb107:
+; CHECK:     successors: %bb.5(0x80000000)
+; CHECK:     liveins: $eflags
+; CHECK:     %3:gr32 = MOV32ri -1
+; CHECK:     JMP_1 %bb.5
+; CHECK:   bb.4.bb107:
+; CHECK:     successors: %bb.5(0x80000000)
+; CHECK:     liveins: $eflags
+; CHECK:     XABORT_DEF implicit-def $eax
+; CHECK:     %4:gr32 = COPY $eax
+; CHECK:   bb.5.bb107:
+; CHECK:     successors: %bb.1(0x40000000), %bb.2(0x40000000)
+; CHECK:     liveins: $eflags
+; CHECK:     %2:gr32 = PHI %3, %bb.3, %4, %bb.4
+; CHECK:     JCC_1 %bb.2, 5, implicit $eflags
+; CHECK:     JMP_1 %bb.1
+
+declare i32 @llvm.x86.xbegin() #0
+
+define void @wobble.12(i32 %tmp116) {
+bb107:                                            ; preds = %bb42
+  %tmp117 = icmp eq i32 %tmp116, 1
+  %tmp127 = tail call i32 @llvm.x86.xbegin() #0
+  br i1 %tmp117, label %bb129, label %bb250
+
+bb129:                                            ; preds = %bb107
+  unreachable
+
+bb250:                                            ; preds = %bb107
+  unreachable
+}

From 8c9241a051fd677cfbfd9c79c6af9d714be7c792 Mon Sep 17 00:00:00 2001
From: Artem Dergachev <artem.dergachev@gmail.com>
Date: Thu, 23 Jul 2020 11:37:45 -0700
Subject: [PATCH 0269/1035] [clang-tidy] Suppress one unittest on macOS.

Possibly a linker bug but I'm in a hurry to fix a buildbot.

Differential Revision: https://reviews.llvm.org/D84453
---
 .../clang-tidy/ClangTidyDiagnosticConsumerTest.cpp    | 11 +++++++++++
 .../unittests/clang-tidy/ClangTidyOptionsTest.cpp     |  1 +
 2 files changed, 12 insertions(+)

diff --git a/clang-tools-extra/unittests/clang-tidy/ClangTidyDiagnosticConsumerTest.cpp b/clang-tools-extra/unittests/clang-tidy/ClangTidyDiagnosticConsumerTest.cpp
index 2a4ed64b88508..a8729660bdcec 100644
--- a/clang-tools-extra/unittests/clang-tidy/ClangTidyDiagnosticConsumerTest.cpp
+++ b/clang-tools-extra/unittests/clang-tidy/ClangTidyDiagnosticConsumerTest.cpp
@@ -21,6 +21,16 @@ class TestCheck : public ClangTidyCheck {
   }
 };
 
+// FIXME: This test seems to cause a strange linking interference
+// with the ValidConfiguration.ValidEnumOptions test on macOS.
+// If both tests are enabled, this test will fail as if
+// runCheckOnCode() is not invoked at all. Looks like a linker bug.
+// For now both tests are disabled on macOS. It is not sufficient
+// to only disable the other test because this test keeps failing
+// under Address Sanitizer, which may be an indication of more
+// such linking interference with other tests and this test
+// seems to be in the center of it.
+#ifndef __APPLE__
 TEST(ClangTidyDiagnosticConsumer, SortsErrors) {
   std::vector<ClangTidyError> Errors;
   runCheckOnCode<TestCheck>("int a;", &Errors);
@@ -28,6 +38,7 @@ TEST(ClangTidyDiagnosticConsumer, SortsErrors) {
   EXPECT_EQ("type specifier", Errors[0].Message.Message);
   EXPECT_EQ("variable", Errors[1].Message.Message);
 }
+#endif
 
 } // namespace test
 } // namespace tidy
diff --git a/clang-tools-extra/unittests/clang-tidy/ClangTidyOptionsTest.cpp b/clang-tools-extra/unittests/clang-tidy/ClangTidyOptionsTest.cpp
index 63f9a06e91bec..c4239af0e7673 100644
--- a/clang-tools-extra/unittests/clang-tidy/ClangTidyOptionsTest.cpp
+++ b/clang-tools-extra/unittests/clang-tidy/ClangTidyOptionsTest.cpp
@@ -223,6 +223,7 @@ TEST(CheckOptionsValidation, ValidIntOptions) {
 }
 
 // FIXME: Figure out why this test causes crashes on mac os.
+// See also comments around the ClangTidyDiagnosticConsumer.SortsErrors test.
 #ifndef __APPLE__
 TEST(ValidConfiguration, ValidEnumOptions) {
 

From 486d2750c7151d3d93b785a4669e2d7d5c9286ac Mon Sep 17 00:00:00 2001
From: Ehsan Toosi <ehsan.nadjaran_toosi@dfki.de>
Date: Fri, 24 Jul 2020 15:43:59 +0200
Subject: [PATCH 0270/1035] [mlir][NFC] Polish copy removal transform

Address a few remaining comments in copy removal transform.

Differential Revision: https://reviews.llvm.org/D84529
---
 mlir/lib/Transforms/CopyRemoval.cpp | 41 ++++++++++++++++-------------
 1 file changed, 23 insertions(+), 18 deletions(-)

diff --git a/mlir/lib/Transforms/CopyRemoval.cpp b/mlir/lib/Transforms/CopyRemoval.cpp
index 28648e0b4294a..ccfd02630ac28 100644
--- a/mlir/lib/Transforms/CopyRemoval.cpp
+++ b/mlir/lib/Transforms/CopyRemoval.cpp
@@ -19,16 +19,28 @@ namespace {
 //===----------------------------------------------------------------------===//
 // CopyRemovalPass
 //===----------------------------------------------------------------------===//
+
 /// This pass removes the redundant Copy operations. Additionally, it
 /// removes the leftover definition and deallocation operations by erasing the
 /// copy operation.
 class CopyRemovalPass : public PassWrapper<CopyRemovalPass, OperationPass<>> {
+public:
+  void runOnOperation() override {
+    getOperation()->walk([&](CopyOpInterface copyOp) {
+      reuseCopySourceAsTarget(copyOp);
+      reuseCopyTargetAsSource(copyOp);
+    });
+    for (Operation *op : eraseList)
+      op->erase();
+  }
+
 private:
   /// List of operations that need to be removed.
   DenseSet<Operation *> eraseList;
 
   /// Returns the deallocation operation for `value` in `block` if it exists.
   Operation *getDeallocationInBlock(Value value, Block *block) {
+    assert(block && "Block cannot be null");
     auto valueUsers = value.getUsers();
     auto it = llvm::find_if(valueUsers, [&](Operation *op) {
       auto effects = dyn_cast<MemoryEffectOpInterface>(op);
@@ -40,12 +52,12 @@ class CopyRemovalPass : public PassWrapper<CopyRemovalPass, OperationPass<>> {
   /// Returns true if an operation between start and end operations has memory
   /// effect.
   bool hasMemoryEffectOpBetween(Operation *start, Operation *end) {
+    assert((start || end) && "Start and end operations cannot be null");
     assert(start->getBlock() == end->getBlock() &&
            "Start and end operations should be in the same block.");
     Operation *op = start->getNextNode();
     while (op->isBeforeInBlock(end)) {
-      auto effects = dyn_cast<MemoryEffectOpInterface>(op);
-      if (effects)
+      if (isa<MemoryEffectOpInterface>(op))
         return true;
       op = op->getNextNode();
     }
@@ -55,6 +67,7 @@ class CopyRemovalPass : public PassWrapper<CopyRemovalPass, OperationPass<>> {
   /// Returns true if `val` value has at least a user between `start` and
   /// `end` operations.
   bool hasUsersBetween(Value val, Operation *start, Operation *end) {
+    assert((start || end) && "Start and end operations cannot be null");
     Block *block = start->getBlock();
     assert(block == end->getBlock() &&
            "Start and end operations should be in the same block.");
@@ -65,10 +78,11 @@ class CopyRemovalPass : public PassWrapper<CopyRemovalPass, OperationPass<>> {
   };
 
   bool areOpsInTheSameBlock(ArrayRef<Operation *> operations) {
-    llvm::SmallPtrSet<Block *, 4> blocks;
-    for (Operation *op : operations)
-      blocks.insert(op->getBlock());
-    return blocks.size() == 1;
+    assert(!operations.empty() &&
+           "The operations list should contain at least a single operation");
+    Block *block = operations.front()->getBlock();
+    return llvm::none_of(
+        operations, [&](Operation *op) { return block != op->getBlock(); });
   }
 
   /// Input:
@@ -97,7 +111,7 @@ class CopyRemovalPass : public PassWrapper<CopyRemovalPass, OperationPass<>> {
   /// TODO: Alias analysis is not available at the moment. Currently, we check
   /// if there are any operations with memory effects between copy and
   /// deallocation operations.
-  void ReuseCopySourceAsTarget(CopyOpInterface copyOp) {
+  void reuseCopySourceAsTarget(CopyOpInterface copyOp) {
     if (eraseList.count(copyOp))
       return;
 
@@ -147,7 +161,7 @@ class CopyRemovalPass : public PassWrapper<CopyRemovalPass, OperationPass<>> {
   /// TODO: Alias analysis is not available at the moment. Currently, we check
   /// if there are any operations with memory effects between copy and
   /// deallocation operations.
-  void ReuseCopyTargetAsSource(CopyOpInterface copyOp) {
+  void reuseCopyTargetAsSource(CopyOpInterface copyOp) {
     if (eraseList.count(copyOp))
       return;
 
@@ -169,16 +183,6 @@ class CopyRemovalPass : public PassWrapper<CopyRemovalPass, OperationPass<>> {
     eraseList.insert(fromDefiningOp);
     eraseList.insert(fromFreeingOp);
   }
-
-public:
-  void runOnOperation() override {
-    getOperation()->walk([&](CopyOpInterface copyOp) {
-      ReuseCopySourceAsTarget(copyOp);
-      ReuseCopyTargetAsSource(copyOp);
-    });
-    for (Operation *op : eraseList)
-      op->erase();
-  }
 };
 
 } // end anonymous namespace
@@ -186,6 +190,7 @@ class CopyRemovalPass : public PassWrapper<CopyRemovalPass, OperationPass<>> {
 //===----------------------------------------------------------------------===//
 // CopyRemovalPass construction
 //===----------------------------------------------------------------------===//
+
 std::unique_ptr<Pass> mlir::createCopyRemovalPass() {
   return std::make_unique<CopyRemovalPass>();
 }

From e40315d2b4ed1e38962a8f33ff151693ed4ada63 Mon Sep 17 00:00:00 2001
From: Roman Lebedev <lebedev.ri@gmail.com>
Date: Tue, 28 Jul 2020 10:16:52 +0300
Subject: [PATCH 0271/1035] [GVN] Rewrite IsValueFullyAvailableInBlock(): no
 recursion, less false-negatives

While this doesn't appear to help with the perf issue being exposed by
D84108, the function as-is is very weird, convoluted, and what's worse,
recursive.

There was no need for `SpeculativelyAvaliableAndUsedForSpeculation`,
tri-state choice is enough. We don't even ever check for that state.

The basic idea here is that we need to perform a depth-first traversal
of the predecessors of the basic block in question, either finding a
preexisting state for the block in a map, or inserting a "placeholder"
`SpeculativelyAvaliable`,

If we encounter an `Unavaliable` block, then we need to give up search,
and back-propagate the `Unavaliable` state to the each successor of
said block, more specifically to the each `SpeculativelyAvaliable`
we've just created.

However, if we have traversed entirety of the predecessors and have not
encountered an `Unavaliable` block, then it must mean the value is fully
available. We could update each inserted `SpeculativelyAvaliable` into
a `Avaliable`, but we don't need to, as assertion excersizes,
because we can assume that if we see an `SpeculativelyAvaliable` entry,
it is actually `Avaliable`, because during the time we've produced it,
if we would have found that it has an `Unavaliable` predecessor,
we would have updated it's successors, including this block,
into `Unavaliable`

Reviewed By: fhahn

Differential Revision: https://reviews.llvm.org/D84181
---
 llvm/lib/Transforms/Scalar/GVN.cpp            | 198 +++++++++++-------
 .../GVN/loadpre-missed-opportunity.ll         |  34 ++-
 2 files changed, 155 insertions(+), 77 deletions(-)

diff --git a/llvm/lib/Transforms/Scalar/GVN.cpp b/llvm/lib/Transforms/Scalar/GVN.cpp
index 0b416cc4afb86..1d82664ed4076 100644
--- a/llvm/lib/Transforms/Scalar/GVN.cpp
+++ b/llvm/lib/Transforms/Scalar/GVN.cpp
@@ -98,21 +98,30 @@ STATISTIC(NumGVNSimpl,  "Number of instructions simplified");
 STATISTIC(NumGVNEqProp, "Number of equalities propagated");
 STATISTIC(NumPRELoad,   "Number of loads PRE'd");
 
+STATISTIC(IsValueFullyAvailableInBlockNumSpeculationsMax,
+          "Number of blocks speculated as available in "
+          "IsValueFullyAvailableInBlock(), max");
+STATISTIC(MaxBBSpeculationCutoffReachedTimes,
+          "Number of times we we reached gvn-max-block-speculations cut-off "
+          "preventing further exploration");
+
 static cl::opt<bool> GVNEnablePRE("enable-pre", cl::init(true), cl::Hidden);
 static cl::opt<bool> GVNEnableLoadPRE("enable-load-pre", cl::init(true));
 static cl::opt<bool> GVNEnableLoadInLoopPRE("enable-load-in-loop-pre",
                                             cl::init(true));
 static cl::opt<bool> GVNEnableMemDep("enable-gvn-memdep", cl::init(true));
 
-// Maximum allowed recursion depth.
-static cl::opt<uint32_t>
-MaxRecurseDepth("gvn-max-recurse-depth", cl::Hidden, cl::init(1000), cl::ZeroOrMore,
-                cl::desc("Max recurse depth in GVN (default = 1000)"));
-
 static cl::opt<uint32_t> MaxNumDeps(
     "gvn-max-num-deps", cl::Hidden, cl::init(100), cl::ZeroOrMore,
     cl::desc("Max number of dependences to attempt Load PRE (default = 100)"));
 
+// This is based on IsValueFullyAvailableInBlockNumSpeculationsMax stat.
+static cl::opt<uint32_t> MaxBBSpeculations(
+    "gvn-max-block-speculations", cl::Hidden, cl::init(600), cl::ZeroOrMore,
+    cl::desc("Max number of blocks we're willing to speculate on (and recurse "
+             "into) when deducing if a value is fully avaliable or not in GVN "
+             "(default = 600)"));
+
 struct llvm::GVN::Expression {
   uint32_t opcode;
   bool commutative = false;
@@ -669,15 +678,14 @@ LLVM_DUMP_METHOD void GVN::dump(DenseMap<uint32_t, Value*>& d) const {
 
 enum class AvaliabilityState : char {
   /// We know the block *is not* fully available. This is a fixpoint.
-  Unavaliable = 0,
+  Unavailable = 0,
   /// We know the block *is* fully available. This is a fixpoint.
-  Avaliable = 1,
+  Available = 1,
   /// We do not know whether the block is fully available or not,
   /// but we are currently speculating that it will be.
-  SpeculativelyAvaliable = 2,
-  /// We are speculating for this block and have used that
-  /// to speculate for other blocks.
-  SpeculativelyAvaliableAndUsedForSpeculation = 3,
+  /// If it would have turned out that the block was, in fact, not fully
+  /// available, this would have been cleaned up into an Unavailable.
+  SpeculativelyAvailable = 2,
 };
 
 /// Return true if we can prove that the value
@@ -688,80 +696,118 @@ enum class AvaliabilityState : char {
 ///   1) we know the block *is* fully available.
 ///   2) we do not know whether the block is fully available or not, but we are
 ///      currently speculating that it will be.
-///   3) we are speculating for this block and have used that to speculate for
-///      other blocks.
 static bool IsValueFullyAvailableInBlock(
     BasicBlock *BB,
-    DenseMap<BasicBlock *, AvaliabilityState> &FullyAvailableBlocks,
-    uint32_t RecurseDepth) {
-  if (RecurseDepth > MaxRecurseDepth)
-    return false;
-
-  // Optimistically assume that the block is speculatively available and check
-  // to see if we already know about this block in one lookup.
-  std::pair<DenseMap<BasicBlock *, AvaliabilityState>::iterator, bool> IV =
-      FullyAvailableBlocks.insert(
-          std::make_pair(BB, AvaliabilityState::SpeculativelyAvaliable));
-
-  // If the entry already existed for this block, return the precomputed value.
-  if (!IV.second) {
-    // If this is a speculative "available" value, mark it as being used for
-    // speculation of other blocks.
-    if (IV.first->second == AvaliabilityState::SpeculativelyAvaliable)
-      IV.first->second =
-          AvaliabilityState::SpeculativelyAvaliableAndUsedForSpeculation;
-    return IV.first->second != AvaliabilityState::Unavaliable;
-  }
+    DenseMap<BasicBlock *, AvaliabilityState> &FullyAvailableBlocks) {
+  SmallVector<BasicBlock *, 32> Worklist;
+  Optional<BasicBlock *> UnavailableBB;
 
-  // Otherwise, see if it is fully available in all predecessors.
-  pred_iterator PI = pred_begin(BB), PE = pred_end(BB);
+  // The number of times we didn't find an entry for a block in a map and
+  // optimistically inserted an entry marking block as speculatively avaliable.
+  unsigned NumNewNewSpeculativelyAvailableBBs = 0;
 
-  // If this block has no predecessors, it isn't live-in here.
-  if (PI == PE)
-    goto SpeculationFailure;
+#ifndef NDEBUG
+  SmallSet<BasicBlock *, 32> NewSpeculativelyAvailableBBs;
+  SmallVector<BasicBlock *, 32> AvailableBBs;
+#endif
 
-  for (; PI != PE; ++PI)
-    // If the value isn't fully available in one of our predecessors, then it
-    // isn't fully available in this block either.  Undo our previous
-    // optimistic assumption and bail out.
-    if (!IsValueFullyAvailableInBlock(*PI, FullyAvailableBlocks,RecurseDepth+1))
-      goto SpeculationFailure;
+  Worklist.emplace_back(BB);
+  while (!Worklist.empty()) {
+    BasicBlock *CurrBB = Worklist.pop_back_val(); // LIFO - depth-first!
+    // Optimistically assume that the block is Speculatively Available and check
+    // to see if we already know about this block in one lookup.
+    std::pair<DenseMap<BasicBlock *, AvaliabilityState>::iterator, bool> IV =
+        FullyAvailableBlocks.try_emplace(
+            CurrBB, AvaliabilityState::SpeculativelyAvailable);
+    AvaliabilityState &State = IV.first->second;
+
+    // Did the entry already exist for this block?
+    if (!IV.second) {
+      if (State == AvaliabilityState::Unavailable) {
+        UnavailableBB = CurrBB;
+        break; // Backpropagate unavaliability info.
+      }
 
-  return true;
+#ifndef NDEBUG
+      AvailableBBs.emplace_back(CurrBB);
+#endif
+      continue; // Don't recurse further, but continue processing worklist.
+    }
 
-// If we get here, we found out that this is not, after
-// all, a fully-available block.  We have a problem if we speculated on this and
-// used the speculation to mark other blocks as available.
-SpeculationFailure:
-  AvaliabilityState &BBVal = FullyAvailableBlocks[BB];
+    // No entry found for block.
+    ++NumNewNewSpeculativelyAvailableBBs;
+    bool OutOfBudget = NumNewNewSpeculativelyAvailableBBs > MaxBBSpeculations;
+
+    // If we have exhausted our budget, mark this block as unavailable.
+    // Also, if this block has no predecessors, the value isn't live-in here.
+    if (OutOfBudget || pred_empty(CurrBB)) {
+      MaxBBSpeculationCutoffReachedTimes += (int)OutOfBudget;
+      State = AvaliabilityState::Unavailable;
+      UnavailableBB = CurrBB;
+      break; // Backpropagate unavaliability info.
+    }
 
-  // If we didn't speculate on this, just return with it set to unavaliable.
-  if (BBVal == AvaliabilityState::SpeculativelyAvaliable) {
-    BBVal = AvaliabilityState::Unavaliable;
-    return false;
+    // Tentatively consider this block as speculatively available.
+#ifndef NDEBUG
+    NewSpeculativelyAvailableBBs.insert(CurrBB);
+#endif
+    // And further recurse into block's predecessors, in depth-first order!
+    Worklist.append(pred_begin(CurrBB), pred_end(CurrBB));
   }
 
-  // If we did speculate on this value, we could have blocks set to
-  // speculatively avaliable that are incorrect. Walk the (transitive)
-  // successors of this block and mark them as unavaliable instead.
-  SmallVector<BasicBlock*, 32> BBWorklist;
-  BBWorklist.push_back(BB);
-
-  do {
-    BasicBlock *Entry = BBWorklist.pop_back_val();
-    // Note that this sets blocks to unavailable if they happen to not
-    // already be in FullyAvailableBlocks.  This is safe.
-    AvaliabilityState &EntryVal = FullyAvailableBlocks[Entry];
-    if (EntryVal == AvaliabilityState::Unavaliable)
-      continue; // Already unavailable.
-
-    // Mark as unavailable.
-    EntryVal = AvaliabilityState::Unavaliable;
+#if LLVM_ENABLE_STATS
+  IsValueFullyAvailableInBlockNumSpeculationsMax.updateMax(
+      NumNewNewSpeculativelyAvailableBBs);
+#endif
 
-    BBWorklist.append(succ_begin(Entry), succ_end(Entry));
-  } while (!BBWorklist.empty());
+  // If the block isn't marked as fixpoint yet
+  // (the Unavailable and Available states are fixpoints)
+  auto MarkAsFixpointAndEnqueueSuccessors =
+      [&](BasicBlock *BB, AvaliabilityState FixpointState) {
+        auto It = FullyAvailableBlocks.find(BB);
+        if (It == FullyAvailableBlocks.end())
+          return; // Never queried this block, leave as-is.
+        switch (AvaliabilityState &State = It->second) {
+        case AvaliabilityState::Unavailable:
+        case AvaliabilityState::Available:
+          return; // Don't backpropagate further, continue processing worklist.
+        case AvaliabilityState::SpeculativelyAvailable: // Fix it!
+          State = FixpointState;
+#ifndef NDEBUG
+          assert(NewSpeculativelyAvailableBBs.erase(BB) &&
+                 "Found a speculatively available successor leftover?");
+#endif
+          // Queue successors for further processing.
+          Worklist.append(succ_begin(BB), succ_end(BB));
+          return;
+        }
+      };
+
+  if (UnavailableBB) {
+    // Okay, we have encountered an unavailable block.
+    // Mark speculatively available blocks reachable from UnavailableBB as
+    // unavailable as well. Paths are terminated when they reach blocks not in
+    // FullyAvailableBlocks or they are not marked as speculatively available.
+    Worklist.clear();
+    Worklist.append(succ_begin(*UnavailableBB), succ_end(*UnavailableBB));
+    while (!Worklist.empty())
+      MarkAsFixpointAndEnqueueSuccessors(Worklist.pop_back_val(),
+                                         AvaliabilityState::Unavailable);
+  }
+
+#ifndef NDEBUG
+  Worklist.clear();
+  for (BasicBlock *AvailableBB : AvailableBBs)
+    Worklist.append(succ_begin(AvailableBB), succ_end(AvailableBB));
+  while (!Worklist.empty())
+    MarkAsFixpointAndEnqueueSuccessors(Worklist.pop_back_val(),
+                                       AvaliabilityState::Available);
+
+  assert(NewSpeculativelyAvailableBBs.empty() &&
+         "Must have fixed all the new speculatively available blocks.");
+#endif
 
-  return false;
+  return !UnavailableBB;
 }
 
 /// Given a set of loads specified by ValuesPerBlock,
@@ -1126,9 +1172,9 @@ bool GVN::PerformLoadPRE(LoadInst *LI, AvailValInBlkVect &ValuesPerBlock,
   MapVector<BasicBlock *, Value *> PredLoads;
   DenseMap<BasicBlock *, AvaliabilityState> FullyAvailableBlocks;
   for (const AvailableValueInBlock &AV : ValuesPerBlock)
-    FullyAvailableBlocks[AV.BB] = AvaliabilityState::Avaliable;
+    FullyAvailableBlocks[AV.BB] = AvaliabilityState::Available;
   for (BasicBlock *UnavailableBB : UnavailableBlocks)
-    FullyAvailableBlocks[UnavailableBB] = AvaliabilityState::Unavaliable;
+    FullyAvailableBlocks[UnavailableBB] = AvaliabilityState::Unavailable;
 
   SmallVector<BasicBlock *, 4> CriticalEdgePred;
   for (BasicBlock *Pred : predecessors(LoadBB)) {
@@ -1141,7 +1187,7 @@ bool GVN::PerformLoadPRE(LoadInst *LI, AvailValInBlkVect &ValuesPerBlock,
       return false;
     }
 
-    if (IsValueFullyAvailableInBlock(Pred, FullyAvailableBlocks, 0)) {
+    if (IsValueFullyAvailableInBlock(Pred, FullyAvailableBlocks)) {
       continue;
     }
 
diff --git a/llvm/test/Transforms/GVN/loadpre-missed-opportunity.ll b/llvm/test/Transforms/GVN/loadpre-missed-opportunity.ll
index 1c967f48d546c..013672fee11e9 100644
--- a/llvm/test/Transforms/GVN/loadpre-missed-opportunity.ll
+++ b/llvm/test/Transforms/GVN/loadpre-missed-opportunity.ll
@@ -1,7 +1,39 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -gvn -S | FileCheck %s
+; RUN: opt < %s -gvn -gvn-max-block-speculations=1 -S | FileCheck -check-prefixes=ALL,PRE %s
+; RUN: opt < %s -gvn -gvn-max-block-speculations=0 -S | FileCheck -check-prefixes=ALL,CHECK %s
 
 define i32 @loadpre_opportunity(i32** %arg, i1 %arg1, i1 %arg2, i1 %arg3) {
+; PRE-LABEL: @loadpre_opportunity(
+; PRE-NEXT:  bb:
+; PRE-NEXT:    [[I:%.*]] = load i32*, i32** [[ARG:%.*]], align 8
+; PRE-NEXT:    [[I6:%.*]] = call i32 @use(i32* [[I]])
+; PRE-NEXT:    br label [[BB11:%.*]]
+; PRE:       bb7:
+; PRE-NEXT:    [[I8:%.*]] = phi i32* [ [[I8_PRE:%.*]], [[BB17_BB7_CRIT_EDGE:%.*]] ], [ [[I81:%.*]], [[BB11]] ]
+; PRE-NEXT:    [[I10:%.*]] = call i32 @use(i32* [[I8]])
+; PRE-NEXT:    br label [[BB11]]
+; PRE:       bb11:
+; PRE-NEXT:    [[I81]] = phi i32* [ [[I]], [[BB:%.*]] ], [ [[I8]], [[BB7:%.*]] ]
+; PRE-NEXT:    [[I12:%.*]] = phi i32 [ [[I6]], [[BB]] ], [ [[I10]], [[BB7]] ]
+; PRE-NEXT:    br i1 [[ARG1:%.*]], label [[BB7]], label [[BB13:%.*]]
+; PRE:       bb13:
+; PRE-NEXT:    call void @somecall()
+; PRE-NEXT:    br i1 [[ARG2:%.*]], label [[BB14:%.*]], label [[BB17:%.*]]
+; PRE:       bb14:
+; PRE-NEXT:    br label [[BB15:%.*]]
+; PRE:       bb15:
+; PRE-NEXT:    br i1 [[ARG3:%.*]], label [[BB16:%.*]], label [[BB15]]
+; PRE:       bb16:
+; PRE-NEXT:    br label [[BB17]]
+; PRE:       bb17:
+; PRE-NEXT:    [[I18:%.*]] = call i1 @cond()
+; PRE-NEXT:    br i1 [[I18]], label [[BB17_BB7_CRIT_EDGE]], label [[BB19:%.*]]
+; PRE:       bb17.bb7_crit_edge:
+; PRE-NEXT:    [[I8_PRE]] = load i32*, i32** [[ARG]], align 8
+; PRE-NEXT:    br label [[BB7]]
+; PRE:       bb19:
+; PRE-NEXT:    ret i32 [[I12]]
+;
 ; CHECK-LABEL: @loadpre_opportunity(
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[I:%.*]] = load i32*, i32** [[ARG:%.*]], align 8

From c2abdec722f119ebda0cee330fe8dd7bf9c6d506 Mon Sep 17 00:00:00 2001
From: Chen Zheng <czhengsz@cn.ibm.com>
Date: Tue, 28 Jul 2020 02:56:35 -0400
Subject: [PATCH 0272/1035] [PowerPC] test case for adding dq form to
 isLegalAddressingMode, nfc

---
 llvm/test/CodeGen/PowerPC/prefer-dqform.ll | 96 ++++++++++++++++++++++
 1 file changed, 96 insertions(+)
 create mode 100644 llvm/test/CodeGen/PowerPC/prefer-dqform.ll

diff --git a/llvm/test/CodeGen/PowerPC/prefer-dqform.ll b/llvm/test/CodeGen/PowerPC/prefer-dqform.ll
new file mode 100644
index 0000000000000..0d1992763d0ff
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/prefer-dqform.ll
@@ -0,0 +1,96 @@
+; RUN: llc -verify-machineinstrs -disable-ppc-instr-form-prep=true -mcpu=pwr9 < %s \
+; RUN:   -ppc-vsr-nums-as-vr -ppc-asm-full-reg-names | FileCheck %s -check-prefix=CHECK-P9
+; RUN: llc -verify-machineinstrs -disable-ppc-instr-form-prep=true -mcpu=pwr10 < %s \
+; RUN:   -ppc-vsr-nums-as-vr -ppc-asm-full-reg-names | FileCheck %s -check-prefix=CHECK-P10
+
+target triple = "powerpc64le-unknown-linux-gnu"
+
+%_elem_type_of_a = type <{ double }>
+%_elem_type_of_x = type <{ double }>
+%_elem_type_of_y = type <{ double }>
+
+define void @test(i32* dereferenceable(4) %.ial, i32* noalias dereferenceable(4) %.m, i32* noalias dereferenceable(4) %.n, [0 x %_elem_type_of_a]*  %.a, i32* noalias dereferenceable(4) %.lda, [0 x %_elem_type_of_x]* noalias %.x, [0 x %_elem_type_of_y]* noalias %.y) {
+; CHECK-P9-LABEL: test:
+; CHECK-P9:       .LBB0_2: # %_loop_2_do_
+; CHECK-P9:         lxvx
+; CHECK-P9:         lxvx
+; CHECK-P9-DAG:     lxvx
+; CHECK-P9-DAG:     lxvx
+; CHECK-P9-DAG:     xvmaddadp
+; CHECK-P9-DAG:     xvmaddadp
+; CHECK-P9-DAG:     stxvx
+; CHECK-P9:         stxvx
+; CHECK-P9:         bdnz .LBB0_2
+;
+; CHECK-P10-LABEL: test:
+; CHECK-P10:       .LBB0_2: # %_loop_2_do_
+; CHECK-P10:         lxvx
+; CHECK-P10:         lxvx
+; CHECK-P10-DAG:     lxvx
+; CHECK-P10-DAG:     lxvx
+; CHECK-P10-DAG:     xvmaddadp
+; CHECK-P10-DAG:     xvmaddadp
+; CHECK-P10-DAG:     stxvx
+; CHECK-P10:         stxvx
+; CHECK-P10:         bdnz .LBB0_2
+test_entry:
+  %_conv5 = ptrtoint [0 x %_elem_type_of_a]* %.a to i64
+  %_andi_tmp = and i64 %_conv5, 15
+  %_equ_tmp = icmp eq i64 %_andi_tmp, 0
+  %. = select i1 %_equ_tmp, i32 1, i32 2
+  %_val_m_ = load i32, i32* %.m, align 4
+  %_sub_tmp9 = sub nsw i32 1, %.
+  %_add_tmp10 = add i32 %_sub_tmp9, %_val_m_
+  %_mod_tmp = srem i32 %_add_tmp10, 16
+  %_sub_tmp11 = sub i32 %_val_m_, %_mod_tmp
+  %_val_n_ = load i32, i32* %.n, align 4
+  %x_rvo_based_addr_17 = getelementptr inbounds [0 x %_elem_type_of_x], [0 x %_elem_type_of_x]* %.x, i64 0, i64 -1
+  %_div_tmp = sdiv i32 %_val_n_, 2
+  %_conv16 = sext i32 %_div_tmp to i64
+  %_ind_cast = getelementptr inbounds %_elem_type_of_x, %_elem_type_of_x* %x_rvo_based_addr_17, i64 %_conv16, i32 0
+  %_val_x_ = load double, double* %_ind_cast, align 8
+  %.splatinsert = insertelement <2 x double> undef, double %_val_x_, i32 0
+  %.splat = shufflevector <2 x double> %.splatinsert, <2 x double> undef, <2 x i32> zeroinitializer
+  %_grt_tmp21 = icmp sgt i32 %., %_sub_tmp11
+  br i1 %_grt_tmp21, label %_return_bb, label %_loop_2_do_.lr.ph
+
+_loop_2_do_.lr.ph:                                ; preds = %test_entry
+  %_val_lda_ = load i32, i32* %.lda, align 4
+  %_conv = sext i32 %_val_lda_ to i64
+  %_mult_tmp = shl nsw i64 %_conv, 3
+  %_sub_tmp4 = sub nuw nsw i64 -8, %_mult_tmp
+  %y_rvo_based_addr_19 = getelementptr inbounds [0 x %_elem_type_of_y], [0 x %_elem_type_of_y]* %.y, i64 0, i64 -1
+  %a_byte_ptr_ = bitcast [0 x %_elem_type_of_a]* %.a to i8*
+  %a_rvo_based_addr_ = getelementptr inbounds i8, i8* %a_byte_ptr_, i64 %_sub_tmp4
+  %0 = zext i32 %. to i64
+  %1 = sext i32 %_sub_tmp11 to i64
+  br label %_loop_2_do_
+
+_loop_2_do_:                                      ; preds = %_loop_2_do_.lr.ph, %_loop_2_do_
+  %indvars.iv = phi i64 [ %0, %_loop_2_do_.lr.ph ], [ %indvars.iv.next, %_loop_2_do_ ]
+  %_ix_x_len19 = shl nuw nsw i64 %indvars.iv, 3
+  %y_ix_dim_0_20 = getelementptr inbounds %_elem_type_of_y, %_elem_type_of_y* %y_rvo_based_addr_19, i64 %indvars.iv
+  %2 = bitcast %_elem_type_of_y* %y_ix_dim_0_20 to <2 x double>*
+  %3 = load <2 x double>, <2 x double>* %2, align 1
+  %4 = getelementptr %_elem_type_of_y, %_elem_type_of_y* %y_ix_dim_0_20, i64 2
+  %5 = bitcast %_elem_type_of_y* %4 to <2 x double>*
+  %6 = load <2 x double>, <2 x double>* %5, align 1
+  %a_ix_dim_1_ = getelementptr inbounds i8, i8* %a_rvo_based_addr_, i64 %_ix_x_len19
+  %7 = bitcast i8* %a_ix_dim_1_ to <2 x double>*
+  %8 = load <2 x double>, <2 x double>* %7, align 1
+  %9 = getelementptr i8, i8* %a_ix_dim_1_, i64 16
+  %10 = bitcast i8* %9 to <2 x double>*
+  %11 = load <2 x double>, <2 x double>* %10, align 1
+  %12 = tail call nsz contract <2 x double> @llvm.fma.v2f64(<2 x double> %8, <2 x double> %3, <2 x double> %3)
+  %13 = tail call nsz contract <2 x double> @llvm.fma.v2f64(<2 x double> %11, <2 x double> %.splat, <2 x double> %6)
+  store <2 x double> %12, <2 x double>* %2, align 1
+  store <2 x double> %13, <2 x double>* %5, align 1
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 16
+  %_grt_tmp = icmp sgt i64 %indvars.iv.next, %1
+  br i1 %_grt_tmp, label %_return_bb, label %_loop_2_do_
+
+_return_bb:                                       ; preds = %_loop_2_do_, %test_entry
+  ret void
+}
+
+declare <2 x double> @llvm.fma.v2f64(<2 x double>, <2 x double>, <2 x double>)

From ad4ab81dccaa72d9b5137433a0923d325ff76135 Mon Sep 17 00:00:00 2001
From: Kristina Bessonova <kbessonova@accesssoftek.com>
Date: Fri, 29 May 2020 13:14:51 +0200
Subject: [PATCH 0273/1035] [clang][cmake] Force CMAKE_LINKER for multistage
 build in case of BOOTSTRAP_LLVM_ENABLE_LLD and MSVC

The issue with LLVM_ENABLE_LLD is that it just passes -fuse-ld=lld
to compiler/linker options which makes sense only for those platforms
where cmake invokes a compiler driver for linking. On Windows (MSVC) cmake
invokes the linker directly and requires CMAKE_LINKER to be specified
otherwise it defaults CMAKE_LINKER to be link.exe.

This patch allows BOOTSTRAP_LLVM_ENABLE_LLD to set CMAKE_LINKER in two cases:
* if building for host Windows,
* if crosscompiling for target Windows.

It also skips adding '-fuse-ld=lld' to make lld-link not warning
about 'unknown argument'.

This fixes build with `clang/cmake/caches/DistributionExample.cmake`
on Windows.

Reviewed By: phosek

Differential Revision: https://reviews.llvm.org/D80873
---
 clang/CMakeLists.txt                       | 8 ++++++++
 llvm/cmake/modules/HandleLLVMOptions.cmake | 7 ++++++-
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/clang/CMakeLists.txt b/clang/CMakeLists.txt
index 1a6a20a271f36..0f08538495fca 100644
--- a/clang/CMakeLists.txt
+++ b/clang/CMakeLists.txt
@@ -753,6 +753,14 @@ if (CLANG_ENABLE_BOOTSTRAP)
     -DCMAKE_ASM_COMPILER=${LLVM_RUNTIME_OUTPUT_INTDIR}/${C_COMPILER}
     -DCMAKE_ASM_COMPILER_ID=Clang)
 
+  # cmake requires CMAKE_LINKER to be specified if the compiler is MSVC-like,
+  # otherwise it defaults the linker to be link.exe.
+  if(BOOTSTRAP_LLVM_ENABLE_LLD)
+    if((WIN32 AND NOT BOOTSTRAP_CMAKE_SYSTEM_NAME) OR BOOTSTRAP_CMAKE_SYSTEM_NAME STREQUAL "Windows")
+      set(${CLANG_STAGE}_LINKER -DCMAKE_LINKER=${LLVM_RUNTIME_OUTPUT_INTDIR}/lld-link${CMAKE_EXECUTABLE_SUFFIX})
+    endif()
+  endif()
+
   if(BOOTSTRAP_CMAKE_SYSTEM_NAME)
     set(${CLANG_STAGE}_CONFIG -DLLVM_CONFIG_PATH=${LLVM_RUNTIME_OUTPUT_INTDIR}/llvm-config)
     set(${CLANG_STAGE}_TABLEGEN
diff --git a/llvm/cmake/modules/HandleLLVMOptions.cmake b/llvm/cmake/modules/HandleLLVMOptions.cmake
index 62dd0ef79cf48..89f7016a7db44 100644
--- a/llvm/cmake/modules/HandleLLVMOptions.cmake
+++ b/llvm/cmake/modules/HandleLLVMOptions.cmake
@@ -261,7 +261,12 @@ if( LLVM_ENABLE_LLD )
   if ( LLVM_USE_LINKER )
     message(FATAL_ERROR "LLVM_ENABLE_LLD and LLVM_USE_LINKER can't be set at the same time")
   endif()
-  set(LLVM_USE_LINKER "lld")
+  # In case of MSVC cmake always invokes the linker directly, so the linker
+  # should be specified by CMAKE_LINKER cmake variable instead of by -fuse-ld
+  # compiler option.
+  if ( NOT MSVC )
+    set(LLVM_USE_LINKER "lld")
+  endif()
 endif()
 
 if( LLVM_USE_LINKER )

From ee068aafbc5c6722158d5113290a211503e1cfe4 Mon Sep 17 00:00:00 2001
From: Georgii Rymar <grimar@accesssoftek.com>
Date: Thu, 23 Jul 2020 13:32:05 +0300
Subject: [PATCH 0274/1035] [llvm-readelf] - Do not treat SHT_ANDROID_RELR
 sections the same as SHT_RELR.

Currently, when dumping section headers, llvm-readelf
prints "RELR" for SHT_ANDROID_RELR/SHT_RELR sections.
The behavior was introduced in D47919 and revealed in D84330.

But "SHT_ANDROID_RELR" has a different value from "SHT_RELR".
Also, "SHT_ANDROID_REL/SHT_ANDROID_RELA" are printed as "ANDROID_REL/ANDROID_RELA",
what makes the handling of the "SHT_ANDROID_RELR" inconsistent.

This patch makes llvm-readelf to print "ANDROID_RELR" instead of "RELR".

Differential revision: https://reviews.llvm.org/D84393
---
 llvm/test/tools/llvm-readobj/ELF/section-types.test | 2 +-
 llvm/tools/llvm-readobj/ELFDumper.cpp               | 5 -----
 2 files changed, 1 insertion(+), 6 deletions(-)

diff --git a/llvm/test/tools/llvm-readobj/ELF/section-types.test b/llvm/test/tools/llvm-readobj/ELF/section-types.test
index 8718f5894c68a..edbc59772f88b 100644
--- a/llvm/test/tools/llvm-readobj/ELF/section-types.test
+++ b/llvm/test/tools/llvm-readobj/ELF/section-types.test
@@ -110,7 +110,7 @@
 # GNU-NEXT: relr                    RELR
 # GNU-NEXT: android_rel             ANDROID_REL
 # GNU-NEXT: android_rela            ANDROID_RELA
-# GNU-NEXT: android_relr            RELR
+# GNU-NEXT: android_relr            ANDROID_RELR
 # GNU-NEXT: llvm_odrtab             LLVM_ODRTAB
 # GNU-NEXT: linker_options          LLVM_LINKER_OPTIONS
 # GNU-NEXT: llvm_call_graph_profile LLVM_CALL_GRAPH_PROFILE
diff --git a/llvm/tools/llvm-readobj/ELFDumper.cpp b/llvm/tools/llvm-readobj/ELFDumper.cpp
index 16270cb0f58c2..72dcb9c5a15aa 100644
--- a/llvm/tools/llvm-readobj/ELFDumper.cpp
+++ b/llvm/tools/llvm-readobj/ELFDumper.cpp
@@ -3757,11 +3757,6 @@ static std::string getSectionTypeString(unsigned Machine, unsigned Type) {
   if (Name == "SHT_SYMTAB_SHNDX")
     return "SYMTAB SECTION INDICES";
 
-  // The SHT_ANDROID_RELR is special, all other SHT_ANDROID_* types are handled
-  // in the common block below.
-  if (Name == "SHT_ANDROID_RELR")
-    return "RELR";
-
   if (Name.startswith("SHT_"))
     return Name.drop_front(4).str();
   return getSectionTypeOffsetString(Type);

From 3218c064d6d8c2cc910570c9a13cb859a2e91d08 Mon Sep 17 00:00:00 2001
From: serge-sans-paille <sguelton@redhat.com>
Date: Tue, 28 Jul 2020 10:55:09 +0200
Subject: [PATCH 0275/1035] [legacyPM] Do not compute preserved analysis if
 there's no local change

All analysis are preserved if there's no local change, and thanks to
3667d87a33d3c8d4072a41fd84bb880c59347dc0 this property is enforced for all
passes.

Skipping the dependency computation improves the performance when there's a lot
of small functions, where only a few change happen.

Thanks to Nikita Popov who provided this numbers (extract below)

https://llvm-compile-time-tracker.com/compare.php?from=183342c0a9850e60dd7a004b651c83dfb3a7d25e&to=f2f91e6a2743070471cc9471e4e8c646e50c653c&stat=instructions

O3: (number of instructions)
Benchmark               Old             New
kimwitu++               60783M          59968M          (-1.34%)
sqlite3                 73200M          73083M          (-0.16%)
consumer-typeset        52776M          52712M          (-0.12%)
Bullet                  133709M         132940M         (-0.58%)
tramp3d-v4              123864M         123186M         (-0.55%)
mafft                   55534M          55477M          (-0.10%)
ClamAV                  76292M          76164M          (-0.17%)
lencod                  103190M         103061M         (-0.13%)
SPASS                   64068M          63713M          (-0.55%)
7zip                    197332M         196308M         (-0.52%)
geomean                 85750M          85389M          (-0.42%)

Differential Revision: https://reviews.llvm.org/D80707
---
 llvm/lib/IR/LegacyPassManager.cpp | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/IR/LegacyPassManager.cpp b/llvm/lib/IR/LegacyPassManager.cpp
index 74869fa62c66f..c01696e4e575e 100644
--- a/llvm/lib/IR/LegacyPassManager.cpp
+++ b/llvm/lib/IR/LegacyPassManager.cpp
@@ -1614,7 +1614,8 @@ bool FPPassManager::runOnFunction(Function &F) {
     dumpUsedSet(FP);
 
     verifyPreservedAnalysis(FP);
-    removeNotPreservedAnalysis(FP);
+    if (LocalChanged)
+      removeNotPreservedAnalysis(FP);
     recordAvailableAnalysis(FP);
     removeDeadPasses(FP, F.getName(), ON_FUNCTION_MSG);
   }
@@ -1723,7 +1724,8 @@ MPPassManager::runOnModule(Module &M) {
     dumpUsedSet(MP);
 
     verifyPreservedAnalysis(MP);
-    removeNotPreservedAnalysis(MP);
+    if (LocalChanged)
+      removeNotPreservedAnalysis(MP);
     recordAvailableAnalysis(MP);
     removeDeadPasses(MP, M.getModuleIdentifier(), ON_MODULE_MSG);
   }

From 182111777b4ec215eeebe8ab5cc2a324e2f055ff Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Tue, 28 Jul 2020 09:52:38 +0100
Subject: [PATCH 0276/1035] [X86][SSE] Attempt to match
 OP(SHUFFLE(X,Y),SHUFFLE(X,Y)) -> SHUFFLE(HOP(X,Y))

An initial backend patch towards fixing the various poor HADD combines (PR34724, PR41813, PR45747 etc.).

This extends isHorizontalBinOp to check if we have per-element horizontal ops (odd+even element pairs), but not in the expected serial order - in which case we build a "post shuffle mask" that we can apply to the HOP result, assuming we have fast-hops/optsize etc.

The next step will be to extend the SHUFFLE(HOP(X,Y)) combines as suggested on PR41813 - accepting more post-shuffle masks even on slow-hop targets if we can fold it into another shuffle.

Differential Revision: https://reviews.llvm.org/D83789
---
 llvm/lib/Target/X86/X86ISelLowering.cpp       |  74 ++++++--
 llvm/test/CodeGen/X86/haddsub-3.ll            |  54 ++++--
 llvm/test/CodeGen/X86/haddsub-4.ll            | 169 +++++-------------
 llvm/test/CodeGen/X86/haddsub-shuf.ll         | 125 ++++---------
 llvm/test/CodeGen/X86/haddsub-undef.ll        |  98 ++++++----
 .../test/CodeGen/X86/vector-shuffle-256-v4.ll |  48 +++--
 .../test/CodeGen/X86/vector-shuffle-256-v8.ll | 162 ++++-------------
 7 files changed, 321 insertions(+), 409 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 390a20c3e71f5..5eadd9c287c79 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -44371,8 +44371,8 @@ static SDValue combineVEXTRACT_STORE(SDNode *N, SelectionDAG &DAG,
 /// A horizontal-op B, for some already available A and B, and if so then LHS is
 /// set to A, RHS to B, and the routine returns 'true'.
 static bool isHorizontalBinOp(SDValue &LHS, SDValue &RHS, SelectionDAG &DAG,
-                              const X86Subtarget &Subtarget,
-                              bool IsCommutative) {
+                              const X86Subtarget &Subtarget, bool IsCommutative,
+                              SmallVectorImpl<int> &PostShuffleMask) {
   // If either operand is undef, bail out. The binop should be simplified.
   if (LHS.isUndef() || RHS.isUndef())
     return false;
@@ -44465,6 +44465,12 @@ static bool isHorizontalBinOp(SDValue &LHS, SDValue &RHS, SelectionDAG &DAG,
       RMask.push_back(i);
   }
 
+  // Avoid 128-bit lane crossing if pre-AVX2 and FP (integer will split).
+  if (!Subtarget.hasAVX2() && VT.isFloatingPoint() &&
+      (isLaneCrossingShuffleMask(128, VT.getScalarSizeInBits(), LMask) ||
+       isLaneCrossingShuffleMask(128, VT.getScalarSizeInBits(), RMask)))
+    return false;
+
   // If A and B occur in reverse order in RHS, then canonicalize by commuting
   // RHS operands and shuffle mask.
   if (A != C) {
@@ -44475,6 +44481,9 @@ static bool isHorizontalBinOp(SDValue &LHS, SDValue &RHS, SelectionDAG &DAG,
   if (!(A == C && B == D))
     return false;
 
+  PostShuffleMask.clear();
+  PostShuffleMask.append(NumElts, SM_SentinelUndef);
+
   // LHS and RHS are now:
   //   LHS = shuffle A, B, LMask
   //   RHS = shuffle A, B, RMask
@@ -44483,6 +44492,7 @@ static bool isHorizontalBinOp(SDValue &LHS, SDValue &RHS, SelectionDAG &DAG,
   // so we just repeat the inner loop if this is a 256-bit op.
   unsigned Num128BitChunks = VT.getSizeInBits() / 128;
   unsigned NumEltsPer128BitChunk = NumElts / Num128BitChunks;
+  unsigned NumEltsPer64BitChunk = NumEltsPer128BitChunk / 2;
   assert((NumEltsPer128BitChunk % 2 == 0) &&
          "Vector type should have an even number of elements in each lane");
   for (unsigned j = 0; j != NumElts; j += NumEltsPer128BitChunk) {
@@ -44494,25 +44504,40 @@ static bool isHorizontalBinOp(SDValue &LHS, SDValue &RHS, SelectionDAG &DAG,
           (!B.getNode() && (LIdx >= (int)NumElts || RIdx >= (int)NumElts)))
         continue;
 
+      // Check that successive odd/even elements are being operated on. If not,
+      // this is not a horizontal operation.
+      if (!((RIdx & 1) == 1 && (LIdx + 1) == RIdx) &&
+          !((LIdx & 1) == 1 && (RIdx + 1) == LIdx && IsCommutative))
+        return false;
+
+      // Compute the post-shuffle mask index based on where the element
+      // is stored in the HOP result, and where it needs to be moved to.
+      int Base = LIdx & ~1u;
+      int Index = ((Base % NumEltsPer128BitChunk) / 2) +
+                  ((Base % NumElts) & ~(NumEltsPer128BitChunk - 1));
+
       // The  low half of the 128-bit result must choose from A.
       // The high half of the 128-bit result must choose from B,
       // unless B is undef. In that case, we are always choosing from A.
-      unsigned NumEltsPer64BitChunk = NumEltsPer128BitChunk / 2;
-      unsigned Src = B.getNode() ? i >= NumEltsPer64BitChunk : 0;
-
-      // Check that successive elements are being operated on. If not, this is
-      // not a horizontal operation.
-      int Index = 2 * (i % NumEltsPer64BitChunk) + NumElts * Src + j;
-      if (!(LIdx == Index && RIdx == Index + 1) &&
-          !(IsCommutative && LIdx == Index + 1 && RIdx == Index))
-        return false;
+      if ((B && Base >= (int)NumElts) || (!B && i >= NumEltsPer64BitChunk))
+        Index += NumEltsPer64BitChunk;
+      PostShuffleMask[i + j] = Index;
     }
   }
 
   LHS = A.getNode() ? A : B; // If A is 'UNDEF', use B for it.
   RHS = B.getNode() ? B : A; // If B is 'UNDEF', use A for it.
 
-  if (!shouldUseHorizontalOp(LHS == RHS && NumShuffles < 2, DAG, Subtarget))
+  bool IsIdentityPostShuffle =
+      isSequentialOrUndefInRange(PostShuffleMask, 0, NumElts, 0);
+  if (IsIdentityPostShuffle)
+    PostShuffleMask.clear();
+
+  // Assume a SingleSource HOP if we only shuffle one input and don't need to
+  // shuffle the result.
+  if (!shouldUseHorizontalOp(LHS == RHS &&
+                                 (NumShuffles < 2 || !IsIdentityPostShuffle),
+                             DAG, Subtarget))
     return false;
 
   LHS = DAG.getBitcast(VT, LHS);
@@ -44531,10 +44556,16 @@ static SDValue combineFaddFsub(SDNode *N, SelectionDAG &DAG,
   assert((IsFadd || N->getOpcode() == ISD::FSUB) && "Wrong opcode");
 
   // Try to synthesize horizontal add/sub from adds/subs of shuffles.
+  SmallVector<int, 8> PostShuffleMask;
   if (((Subtarget.hasSSE3() && (VT == MVT::v4f32 || VT == MVT::v2f64)) ||
        (Subtarget.hasAVX() && (VT == MVT::v8f32 || VT == MVT::v4f64))) &&
-      isHorizontalBinOp(LHS, RHS, DAG, Subtarget, IsFadd))
-    return DAG.getNode(HorizOpcode, SDLoc(N), VT, LHS, RHS);
+      isHorizontalBinOp(LHS, RHS, DAG, Subtarget, IsFadd, PostShuffleMask)) {
+    SDValue HorizBinOp = DAG.getNode(HorizOpcode, SDLoc(N), VT, LHS, RHS);
+    if (!PostShuffleMask.empty())
+      HorizBinOp = DAG.getVectorShuffle(VT, SDLoc(HorizBinOp), HorizBinOp,
+                                        DAG.getUNDEF(VT), PostShuffleMask);
+    return HorizBinOp;
+  }
 
   // NOTE: isHorizontalBinOp may have changed LHS/RHS variables.
 
@@ -47636,17 +47667,22 @@ static SDValue combineAddOrSubToHADDorHSUB(SDNode *N, SelectionDAG &DAG,
   bool IsAdd = N->getOpcode() == ISD::ADD;
   assert((IsAdd || N->getOpcode() == ISD::SUB) && "Wrong opcode");
 
+  SmallVector<int, 8> PostShuffleMask;
   if ((VT == MVT::v8i16 || VT == MVT::v4i32 || VT == MVT::v16i16 ||
        VT == MVT::v8i32) &&
       Subtarget.hasSSSE3() &&
-      isHorizontalBinOp(Op0, Op1, DAG, Subtarget, IsAdd)) {
+      isHorizontalBinOp(Op0, Op1, DAG, Subtarget, IsAdd, PostShuffleMask)) {
     auto HOpBuilder = [IsAdd](SelectionDAG &DAG, const SDLoc &DL,
                               ArrayRef<SDValue> Ops) {
-      return DAG.getNode(IsAdd ? X86ISD::HADD : X86ISD::HSUB,
-                         DL, Ops[0].getValueType(), Ops);
+      return DAG.getNode(IsAdd ? X86ISD::HADD : X86ISD::HSUB, DL,
+                         Ops[0].getValueType(), Ops);
     };
-    return SplitOpsAndApply(DAG, Subtarget, SDLoc(N), VT, {Op0, Op1},
-                            HOpBuilder);
+    SDValue HorizBinOp =
+        SplitOpsAndApply(DAG, Subtarget, SDLoc(N), VT, {Op0, Op1}, HOpBuilder);
+    if (!PostShuffleMask.empty())
+      HorizBinOp = DAG.getVectorShuffle(VT, SDLoc(HorizBinOp), HorizBinOp,
+                                        DAG.getUNDEF(VT), PostShuffleMask);
+    return HorizBinOp;
   }
 
   return SDValue();
diff --git a/llvm/test/CodeGen/X86/haddsub-3.ll b/llvm/test/CodeGen/X86/haddsub-3.ll
index f603ace202a1e..6abba1bbfe9f3 100644
--- a/llvm/test/CodeGen/X86/haddsub-3.ll
+++ b/llvm/test/CodeGen/X86/haddsub-3.ll
@@ -17,22 +17,46 @@ define float @pr26491(<4 x float> %a0) {
 ; SSE2-NEXT:    addss %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
-; SSSE3-LABEL: pr26491:
-; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    movshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    addps %xmm0, %xmm1
-; SSSE3-NEXT:    movaps %xmm1, %xmm0
-; SSSE3-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1]
-; SSSE3-NEXT:    addss %xmm1, %xmm0
-; SSSE3-NEXT:    retq
+; SSSE3-SLOW-LABEL: pr26491:
+; SSSE3-SLOW:       # %bb.0:
+; SSSE3-SLOW-NEXT:    movshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; SSSE3-SLOW-NEXT:    addps %xmm0, %xmm1
+; SSSE3-SLOW-NEXT:    movaps %xmm1, %xmm0
+; SSSE3-SLOW-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1]
+; SSSE3-SLOW-NEXT:    addss %xmm1, %xmm0
+; SSSE3-SLOW-NEXT:    retq
 ;
-; AVX-LABEL: pr26491:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
-; AVX-NEXT:    vaddps %xmm0, %xmm1, %xmm0
-; AVX-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
-; AVX-NEXT:    vaddss %xmm0, %xmm1, %xmm0
-; AVX-NEXT:    retq
+; SSSE3-FAST-LABEL: pr26491:
+; SSSE3-FAST:       # %bb.0:
+; SSSE3-FAST-NEXT:    haddps %xmm0, %xmm0
+; SSSE3-FAST-NEXT:    movaps %xmm0, %xmm1
+; SSSE3-FAST-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1],xmm0[2,3]
+; SSSE3-FAST-NEXT:    addss %xmm0, %xmm1
+; SSSE3-FAST-NEXT:    movaps %xmm1, %xmm0
+; SSSE3-FAST-NEXT:    retq
+;
+; AVX1-SLOW-LABEL: pr26491:
+; AVX1-SLOW:       # %bb.0:
+; AVX1-SLOW-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX1-SLOW-NEXT:    vaddps %xmm0, %xmm1, %xmm0
+; AVX1-SLOW-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
+; AVX1-SLOW-NEXT:    vaddss %xmm0, %xmm1, %xmm0
+; AVX1-SLOW-NEXT:    retq
+;
+; AVX1-FAST-LABEL: pr26491:
+; AVX1-FAST:       # %bb.0:
+; AVX1-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; AVX1-FAST-NEXT:    vaddss %xmm0, %xmm1, %xmm0
+; AVX1-FAST-NEXT:    retq
+;
+; AVX2-LABEL: pr26491:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX2-NEXT:    vaddps %xmm0, %xmm1, %xmm0
+; AVX2-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
+; AVX2-NEXT:    vaddss %xmm0, %xmm1, %xmm0
+; AVX2-NEXT:    retq
   %1 = shufflevector <4 x float> %a0, <4 x float> undef, <4 x i32> <i32 1, i32 1, i32 3, i32 3>
   %2 = fadd <4 x float> %1, %a0
   %3 = extractelement <4 x float> %2, i32 2
diff --git a/llvm/test/CodeGen/X86/haddsub-4.ll b/llvm/test/CodeGen/X86/haddsub-4.ll
index 5c8e9a7c72f2a..4c1dc71982aa4 100644
--- a/llvm/test/CodeGen/X86/haddsub-4.ll
+++ b/llvm/test/CodeGen/X86/haddsub-4.ll
@@ -9,30 +9,16 @@
 define <8 x i16> @hadd_reverse_v8i16(<8 x i16> %a0, <8 x i16> %a1) nounwind {
 ; SSE-LABEL: hadd_reverse_v8i16:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [14,15,10,11,6,7,2,3,14,15,10,11,12,13,14,15]
-; SSE-NEXT:    movdqa %xmm1, %xmm3
-; SSE-NEXT:    pshufb %xmm2, %xmm3
-; SSE-NEXT:    movdqa %xmm0, %xmm4
-; SSE-NEXT:    pshufb %xmm2, %xmm4
-; SSE-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm3[0]
-; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [12,13,8,9,4,5,0,1,8,9,12,13,12,13,14,15]
-; SSE-NEXT:    pshufb %xmm2, %xmm1
-; SSE-NEXT:    pshufb %xmm2, %xmm0
-; SSE-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSE-NEXT:    paddw %xmm4, %xmm0
+; SSE-NEXT:    phaddw %xmm1, %xmm0
+; SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[3,2,1,0,4,5,6,7]
+; SSE-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,6,5,4]
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: hadd_reverse_v8i16:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vmovdqa {{.*#+}} xmm2 = [14,15,10,11,6,7,2,3,14,15,10,11,12,13,14,15]
-; AVX-NEXT:    vpshufb %xmm2, %xmm1, %xmm3
-; AVX-NEXT:    vpshufb %xmm2, %xmm0, %xmm2
-; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; AVX-NEXT:    vmovdqa {{.*#+}} xmm3 = [12,13,8,9,4,5,0,1,8,9,12,13,12,13,14,15]
-; AVX-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
-; AVX-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
-; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX-NEXT:    vpaddw %xmm0, %xmm2, %xmm0
+; AVX-NEXT:    vphaddw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[3,2,1,0,4,5,6,7]
+; AVX-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,6,5,4]
 ; AVX-NEXT:    retq
   %lhs = shufflevector <8 x i16> %a0, <8 x i16> %a1, <8 x i32> <i32 7, i32 5, i32 3, i32 1, i32 15, i32 13, i32 11, i32 9>
   %rhs = shufflevector <8 x i16> %a0, <8 x i16> %a1, <8 x i32> <i32 6, i32 4, i32 2, i32 0, i32 14, i32 12, i32 10, i32 8>
@@ -67,67 +53,34 @@ define <8 x i16> @hadd_reverse2_v8i16(<8 x i16> %a0, <8 x i16> %a1) nounwind {
 define <16 x i16> @hadd_reverse_v16i16(<16 x i16> %a0, <16 x i16> %a1) nounwind {
 ; SSE-LABEL: hadd_reverse_v16i16:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    movdqa %xmm0, %xmm4
-; SSE-NEXT:    movdqa {{.*#+}} xmm0 = [14,15,10,11,6,7,2,3,14,15,10,11,12,13,14,15]
-; SSE-NEXT:    movdqa %xmm2, %xmm5
-; SSE-NEXT:    pshufb %xmm0, %xmm5
-; SSE-NEXT:    movdqa %xmm4, %xmm6
-; SSE-NEXT:    pshufb %xmm0, %xmm6
-; SSE-NEXT:    punpcklqdq {{.*#+}} xmm6 = xmm6[0],xmm5[0]
-; SSE-NEXT:    movdqa %xmm3, %xmm5
-; SSE-NEXT:    pshufb %xmm0, %xmm5
-; SSE-NEXT:    movdqa %xmm1, %xmm7
-; SSE-NEXT:    pshufb %xmm0, %xmm7
-; SSE-NEXT:    punpcklqdq {{.*#+}} xmm7 = xmm7[0],xmm5[0]
-; SSE-NEXT:    movdqa {{.*#+}} xmm0 = [12,13,8,9,4,5,0,1,8,9,12,13,12,13,14,15]
-; SSE-NEXT:    pshufb %xmm0, %xmm2
-; SSE-NEXT:    pshufb %xmm0, %xmm4
-; SSE-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm2[0]
-; SSE-NEXT:    paddw %xmm6, %xmm4
-; SSE-NEXT:    pshufb %xmm0, %xmm3
-; SSE-NEXT:    pshufb %xmm0, %xmm1
-; SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
-; SSE-NEXT:    paddw %xmm7, %xmm1
-; SSE-NEXT:    movdqa %xmm1, %xmm0
-; SSE-NEXT:    movdqa %xmm4, %xmm1
+; SSE-NEXT:    phaddw %xmm3, %xmm1
+; SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[3,2,1,0,4,5,6,7]
+; SSE-NEXT:    pshufhw {{.*#+}} xmm3 = xmm1[0,1,2,3,7,6,5,4]
+; SSE-NEXT:    phaddw %xmm2, %xmm0
+; SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[3,2,1,0,4,5,6,7]
+; SSE-NEXT:    pshufhw {{.*#+}} xmm1 = xmm0[0,1,2,3,7,6,5,4]
+; SSE-NEXT:    movdqa %xmm3, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: hadd_reverse_v16i16:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [14,15,10,11,6,7,2,3,14,15,10,11,12,13,14,15]
-; AVX1-NEXT:    vpshufb %xmm3, %xmm2, %xmm4
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm5
-; AVX1-NEXT:    vpshufb %xmm3, %xmm5, %xmm6
-; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm4 = xmm6[0],xmm4[0]
-; AVX1-NEXT:    vpshufb %xmm3, %xmm1, %xmm6
-; AVX1-NEXT:    vpshufb %xmm3, %xmm0, %xmm3
-; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm6[0]
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [12,13,8,9,4,5,0,1,8,9,12,13,12,13,14,15]
-; AVX1-NEXT:    vpshufb %xmm6, %xmm2, %xmm2
-; AVX1-NEXT:    vpshufb %xmm6, %xmm5, %xmm5
-; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm5[0],xmm2[0]
-; AVX1-NEXT:    vpaddw %xmm2, %xmm4, %xmm2
-; AVX1-NEXT:    vpshufb %xmm6, %xmm1, %xmm1
-; AVX1-NEXT:    vpshufb %xmm6, %xmm0, %xmm0
-; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX1-NEXT:    vpaddw %xmm0, %xmm3, %xmm0
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX1-NEXT:    vphaddw %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm2[3,2,1,0,4,5,6,7]
+; AVX1-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,6,5,4]
+; AVX1-NEXT:    vphaddw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[3,2,1,0,4,5,6,7]
+; AVX1-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,6,5,4]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: hadd_reverse_v16i16:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpshuflw {{.*#+}} ymm2 = ymm1[3,1,2,3,4,5,6,7,11,9,10,11,12,13,14,15]
-; AVX2-NEXT:    vpshufhw {{.*#+}} ymm2 = ymm2[0,1,2,3,7,5,6,7,8,9,10,11,15,13,14,15]
-; AVX2-NEXT:    vpshuflw {{.*#+}} ymm3 = ymm0[3,1,2,3,4,5,6,7,11,9,10,11,12,13,14,15]
-; AVX2-NEXT:    vpshufhw {{.*#+}} ymm3 = ymm3[0,1,2,3,7,5,6,7,8,9,10,11,15,13,14,15]
-; AVX2-NEXT:    vshufps {{.*#+}} ymm2 = ymm3[2,0],ymm2[2,0],ymm3[6,4],ymm2[6,4]
-; AVX2-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[2,3,0,1]
-; AVX2-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[u,u,u,u,u,u,u,u,12,13,8,9,4,5,0,1,u,u,u,u,u,u,u,u,28,29,24,25,20,21,16,17]
-; AVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[12,13,8,9,4,5,0,1,u,u,u,u,u,u,u,u,28,29,24,25,20,21,16,17,u,u,u,u,u,u,u,u]
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
+; AVX2-NEXT:    vphaddw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm0[3,2,1,0,4,5,6,7,11,10,9,8,12,13,14,15]
+; AVX2-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,7,6,5,4,8,9,10,11,15,14,13,12]
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,3,0,1]
-; AVX2-NEXT:    vpaddw %ymm0, %ymm2, %ymm0
 ; AVX2-NEXT:    retq
   %lhs = shufflevector <16 x i16> %a0, <16 x i16> %a1, <16 x i32> <i32 15, i32 13, i32 11, i32 9, i32 31, i32 29, i32 27, i32 25, i32 7, i32 5, i32 3, i32 1, i32 23, i32 21, i32 19, i32 17>
   %rhs = shufflevector <16 x i16> %a0, <16 x i16> %a1, <16 x i32> <i32 14, i32 12, i32 10, i32 8, i32 30, i32 28, i32 26, i32 24, i32 6, i32 4, i32 2, i32 0, i32 22, i32 20, i32 18, i32 16>
@@ -209,21 +162,11 @@ define <8 x double> @hadd_reverse_v8f64(<8 x double> %a0, <8 x double> %a1) noun
 ;
 ; AVX2-LABEL: hadd_reverse_v8f64:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpermpd {{.*#+}} ymm4 = ymm2[0,3,2,1]
-; AVX2-NEXT:    vpermpd {{.*#+}} ymm5 = ymm0[0,3,2,1]
-; AVX2-NEXT:    vunpckhpd {{.*#+}} ymm4 = ymm5[1],ymm4[1],ymm5[3],ymm4[3]
-; AVX2-NEXT:    vpermpd {{.*#+}} ymm5 = ymm3[0,3,2,1]
-; AVX2-NEXT:    vpermpd {{.*#+}} ymm6 = ymm1[0,3,2,1]
-; AVX2-NEXT:    vunpckhpd {{.*#+}} ymm5 = ymm6[1],ymm5[1],ymm6[3],ymm5[3]
-; AVX2-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[2,1,0,3]
-; AVX2-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[2,1,0,3]
-; AVX2-NEXT:    vunpcklpd {{.*#+}} ymm0 = ymm0[0],ymm2[0],ymm0[2],ymm2[2]
-; AVX2-NEXT:    vaddpd %ymm0, %ymm4, %ymm2
-; AVX2-NEXT:    vpermpd {{.*#+}} ymm0 = ymm3[2,1,0,3]
-; AVX2-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[2,1,0,3]
-; AVX2-NEXT:    vunpcklpd {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX2-NEXT:    vaddpd %ymm0, %ymm5, %ymm0
-; AVX2-NEXT:    vmovapd %ymm2, %ymm1
+; AVX2-NEXT:    vhaddpd %ymm3, %ymm1, %ymm1
+; AVX2-NEXT:    vpermpd {{.*#+}} ymm3 = ymm1[2,3,0,1]
+; AVX2-NEXT:    vhaddpd %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpermpd {{.*#+}} ymm1 = ymm0[2,3,0,1]
+; AVX2-NEXT:    vmovapd %ymm3, %ymm0
 ; AVX2-NEXT:    retq
   %lhs = shufflevector <8 x double> %a0, <8 x double> %a1, <8 x i32> <i32 7, i32 15, i32 5, i32 13, i32 3, i32 11, i32 1, i32 9>
   %rhs = shufflevector <8 x double> %a0, <8 x double> %a1, <8 x i32> <i32 6, i32 14, i32 4, i32 12, i32 2, i32 10, i32 0, i32 8>
@@ -290,22 +233,14 @@ define <16 x float> @hadd_reverse_v16f32(<16 x float> %a0, <16 x float> %a1) nou
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    movaps %xmm5, %xmm8
 ; SSE-NEXT:    movaps %xmm1, %xmm5
-; SSE-NEXT:    movaps %xmm8, %xmm1
-; SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1],xmm4[3,1]
-; SSE-NEXT:    movaps %xmm5, %xmm9
-; SSE-NEXT:    shufps {{.*#+}} xmm9 = xmm9[3,1],xmm0[3,1]
-; SSE-NEXT:    movaps %xmm7, %xmm10
-; SSE-NEXT:    shufps {{.*#+}} xmm10 = xmm10[3,1],xmm6[3,1]
-; SSE-NEXT:    movaps %xmm3, %xmm11
-; SSE-NEXT:    shufps {{.*#+}} xmm11 = xmm11[3,1],xmm2[3,1]
-; SSE-NEXT:    shufps {{.*#+}} xmm8 = xmm8[2,0],xmm4[2,0]
-; SSE-NEXT:    addps %xmm1, %xmm8
-; SSE-NEXT:    shufps {{.*#+}} xmm5 = xmm5[2,0],xmm0[2,0]
-; SSE-NEXT:    addps %xmm9, %xmm5
-; SSE-NEXT:    shufps {{.*#+}} xmm7 = xmm7[2,0],xmm6[2,0]
-; SSE-NEXT:    addps %xmm10, %xmm7
-; SSE-NEXT:    shufps {{.*#+}} xmm3 = xmm3[2,0],xmm2[2,0]
-; SSE-NEXT:    addps %xmm11, %xmm3
+; SSE-NEXT:    haddps %xmm2, %xmm3
+; SSE-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,0,3,2]
+; SSE-NEXT:    haddps %xmm6, %xmm7
+; SSE-NEXT:    shufps {{.*#+}} xmm7 = xmm7[1,0,3,2]
+; SSE-NEXT:    haddps %xmm0, %xmm5
+; SSE-NEXT:    shufps {{.*#+}} xmm5 = xmm5[1,0,3,2]
+; SSE-NEXT:    haddps %xmm4, %xmm8
+; SSE-NEXT:    shufps {{.*#+}} xmm8 = xmm8[1,0,3,2]
 ; SSE-NEXT:    movaps %xmm3, %xmm0
 ; SSE-NEXT:    movaps %xmm7, %xmm1
 ; SSE-NEXT:    movaps %xmm5, %xmm2
@@ -316,29 +251,23 @@ define <16 x float> @hadd_reverse_v16f32(<16 x float> %a0, <16 x float> %a1) nou
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm4 = ymm0[2,3],ymm2[2,3]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX1-NEXT:    vshufps {{.*#+}} ymm2 = ymm4[3,1],ymm0[3,1],ymm4[7,5],ymm0[7,5]
-; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm5 = ymm1[2,3],ymm3[2,3]
-; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm3
-; AVX1-NEXT:    vshufps {{.*#+}} ymm6 = ymm5[3,1],ymm3[3,1],ymm5[7,5],ymm3[7,5]
-; AVX1-NEXT:    vshufps {{.*#+}} ymm0 = ymm4[2,0],ymm0[2,0],ymm4[6,4],ymm0[6,4]
-; AVX1-NEXT:    vaddps %ymm0, %ymm2, %ymm1
-; AVX1-NEXT:    vshufps {{.*#+}} ymm0 = ymm5[2,0],ymm3[2,0],ymm5[6,4],ymm3[6,4]
-; AVX1-NEXT:    vaddps %ymm0, %ymm6, %ymm0
+; AVX1-NEXT:    vhaddps %ymm0, %ymm4, %ymm2
+; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm1[2,3],ymm3[2,3]
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
+; AVX1-NEXT:    vhaddps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vpermilps {{.*#+}} ymm0 = ymm0[1,0,3,2,5,4,7,6]
+; AVX1-NEXT:    vpermilps {{.*#+}} ymm1 = ymm2[1,0,3,2,5,4,7,6]
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: hadd_reverse_v16f32:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vshufps {{.*#+}} ymm4 = ymm0[3,1],ymm2[3,1],ymm0[7,5],ymm2[7,5]
-; AVX2-NEXT:    vpermpd {{.*#+}} ymm4 = ymm4[2,0,3,1]
-; AVX2-NEXT:    vshufps {{.*#+}} ymm5 = ymm1[3,1],ymm3[3,1],ymm1[7,5],ymm3[7,5]
-; AVX2-NEXT:    vpermpd {{.*#+}} ymm5 = ymm5[2,0,3,1]
-; AVX2-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[2,0],ymm2[2,0],ymm0[6,4],ymm2[6,4]
-; AVX2-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[2,0,3,1]
-; AVX2-NEXT:    vaddps %ymm0, %ymm4, %ymm2
-; AVX2-NEXT:    vshufps {{.*#+}} ymm0 = ymm1[2,0],ymm3[2,0],ymm1[6,4],ymm3[6,4]
-; AVX2-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[2,0,3,1]
-; AVX2-NEXT:    vaddps %ymm0, %ymm5, %ymm0
-; AVX2-NEXT:    vmovaps %ymm2, %ymm1
+; AVX2-NEXT:    vhaddps %ymm3, %ymm1, %ymm1
+; AVX2-NEXT:    vpermilps {{.*#+}} ymm1 = ymm1[1,0,3,2,5,4,7,6]
+; AVX2-NEXT:    vpermpd {{.*#+}} ymm3 = ymm1[2,0,3,1]
+; AVX2-NEXT:    vhaddps %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpermilps {{.*#+}} ymm0 = ymm0[1,0,3,2,5,4,7,6]
+; AVX2-NEXT:    vpermpd {{.*#+}} ymm1 = ymm0[2,0,3,1]
+; AVX2-NEXT:    vmovaps %ymm3, %ymm0
 ; AVX2-NEXT:    retq
   %lhs = shufflevector <16 x float> %a0, <16 x float> %a1, <16 x i32> <i32 15, i32 13, i32 11, i32 9, i32 31, i32 29, i32 27, i32 25, i32 7, i32 5, i32 3, i32 1, i32 23, i32 21, i32 19, i32 17>
   %rhs = shufflevector <16 x float> %a0, <16 x float> %a1, <16 x i32> <i32 14, i32 12, i32 10, i32 8, i32 30, i32 28, i32 26, i32 24, i32 6, i32 4, i32 2, i32 0, i32 22, i32 20, i32 18, i32 16>
diff --git a/llvm/test/CodeGen/X86/haddsub-shuf.ll b/llvm/test/CodeGen/X86/haddsub-shuf.ll
index 7bedbeb581099..76ef7afbebf33 100644
--- a/llvm/test/CodeGen/X86/haddsub-shuf.ll
+++ b/llvm/test/CodeGen/X86/haddsub-shuf.ll
@@ -879,77 +879,59 @@ declare <4 x float> @llvm.x86.sse3.hadd.ps(<4 x float>, <4 x float>)
 define <4 x float> @PR34724_1(<4 x float> %a, <4 x float> %b) {
 ; SSSE3_SLOW-LABEL: PR34724_1:
 ; SSSE3_SLOW:       # %bb.0:
-; SSSE3_SLOW-NEXT:    movaps %xmm1, %xmm2
-; SSSE3_SLOW-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,0],xmm0[3,2]
-; SSSE3_SLOW-NEXT:    shufps {{.*#+}} xmm2 = xmm2[2,0],xmm0[2,3]
 ; SSSE3_SLOW-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
-; SSSE3_SLOW-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; SSSE3_SLOW-NEXT:    addps %xmm0, %xmm2
-; SSSE3_SLOW-NEXT:    movsldup {{.*#+}} xmm0 = xmm1[0,0,2,2]
-; SSSE3_SLOW-NEXT:    addps %xmm1, %xmm0
-; SSSE3_SLOW-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,0],xmm2[1,0]
-; SSSE3_SLOW-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,0],xmm0[2,0]
-; SSSE3_SLOW-NEXT:    movaps %xmm2, %xmm0
+; SSSE3_SLOW-NEXT:    haddps %xmm1, %xmm0
+; SSSE3_SLOW-NEXT:    movsldup {{.*#+}} xmm2 = xmm1[0,0,2,2]
+; SSSE3_SLOW-NEXT:    addps %xmm1, %xmm2
+; SSSE3_SLOW-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,0],xmm0[2,0]
+; SSSE3_SLOW-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,0],xmm2[2,0]
 ; SSSE3_SLOW-NEXT:    retq
 ;
 ; SSSE3_FAST-LABEL: PR34724_1:
 ; SSSE3_FAST:       # %bb.0:
-; SSSE3_FAST-NEXT:    movaps %xmm1, %xmm2
-; SSSE3_FAST-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,0],xmm0[3,2]
-; SSSE3_FAST-NEXT:    shufps {{.*#+}} xmm2 = xmm2[2,0],xmm0[2,3]
 ; SSSE3_FAST-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
-; SSSE3_FAST-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; SSSE3_FAST-NEXT:    addps %xmm0, %xmm2
+; SSSE3_FAST-NEXT:    haddps %xmm1, %xmm0
 ; SSSE3_FAST-NEXT:    haddps %xmm1, %xmm1
-; SSSE3_FAST-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,0],xmm2[1,0]
-; SSSE3_FAST-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,0],xmm1[2,0]
-; SSSE3_FAST-NEXT:    movaps %xmm2, %xmm0
+; SSSE3_FAST-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,0],xmm0[2,0]
+; SSSE3_FAST-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,0],xmm1[2,0]
 ; SSSE3_FAST-NEXT:    retq
 ;
 ; AVX1_SLOW-LABEL: PR34724_1:
 ; AVX1_SLOW:       # %bb.0:
-; AVX1_SLOW-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
-; AVX1_SLOW-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0],xmm1[0],zero,zero
-; AVX1_SLOW-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[3],xmm1[1],zero,zero
-; AVX1_SLOW-NEXT:    vaddps %xmm0, %xmm2, %xmm0
+; AVX1_SLOW-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
+; AVX1_SLOW-NEXT:    vhaddps %xmm1, %xmm0, %xmm0
 ; AVX1_SLOW-NEXT:    vmovsldup {{.*#+}} xmm2 = xmm1[0,0,2,2]
 ; AVX1_SLOW-NEXT:    vaddps %xmm1, %xmm2, %xmm1
-; AVX1_SLOW-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
-; AVX1_SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,0,1,3]
+; AVX1_SLOW-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3]
+; AVX1_SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,0,2,3]
 ; AVX1_SLOW-NEXT:    retq
 ;
 ; AVX1_FAST-LABEL: PR34724_1:
 ; AVX1_FAST:       # %bb.0:
-; AVX1_FAST-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
-; AVX1_FAST-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0],xmm1[0],zero,zero
-; AVX1_FAST-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[3],xmm1[1],zero,zero
-; AVX1_FAST-NEXT:    vaddps %xmm0, %xmm2, %xmm0
+; AVX1_FAST-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
+; AVX1_FAST-NEXT:    vhaddps %xmm1, %xmm0, %xmm0
 ; AVX1_FAST-NEXT:    vhaddps %xmm1, %xmm1, %xmm1
-; AVX1_FAST-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
-; AVX1_FAST-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,0,1,3]
+; AVX1_FAST-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3]
+; AVX1_FAST-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,0,2,3]
 ; AVX1_FAST-NEXT:    retq
 ;
 ; AVX2_SLOW-LABEL: PR34724_1:
 ; AVX2_SLOW:       # %bb.0:
-; AVX2_SLOW-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
-; AVX2_SLOW-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0],xmm1[0],zero,zero
-; AVX2_SLOW-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[3],xmm1[1],zero,zero
-; AVX2_SLOW-NEXT:    vaddps %xmm0, %xmm2, %xmm0
+; AVX2_SLOW-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
+; AVX2_SLOW-NEXT:    vhaddps %xmm1, %xmm0, %xmm0
 ; AVX2_SLOW-NEXT:    vmovsldup {{.*#+}} xmm2 = xmm1[0,0,2,2]
 ; AVX2_SLOW-NEXT:    vaddps %xmm1, %xmm2, %xmm1
-; AVX2_SLOW-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
-; AVX2_SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,0,1,3]
+; AVX2_SLOW-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3]
+; AVX2_SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,0,2,3]
 ; AVX2_SLOW-NEXT:    retq
 ;
 ; AVX2_FAST-LABEL: PR34724_1:
 ; AVX2_FAST:       # %bb.0:
-; AVX2_FAST-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
-; AVX2_FAST-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0],xmm1[0],zero,zero
-; AVX2_FAST-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[3],xmm1[1],zero,zero
-; AVX2_FAST-NEXT:    vaddps %xmm0, %xmm2, %xmm0
+; AVX2_FAST-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
+; AVX2_FAST-NEXT:    vhaddps %xmm1, %xmm0, %xmm0
 ; AVX2_FAST-NEXT:    vhaddps %xmm1, %xmm1, %xmm1
-; AVX2_FAST-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
-; AVX2_FAST-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,0,1,3]
+; AVX2_FAST-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3]
+; AVX2_FAST-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,0,2,3]
 ; AVX2_FAST-NEXT:    retq
   %t0 = shufflevector <4 x float> %a, <4 x float> %b, <2 x i32> <i32 2, i32 4>
   %t1 = shufflevector <4 x float> %a, <4 x float> %b, <2 x i32> <i32 3, i32 5>
@@ -964,78 +946,49 @@ define <4 x float> @PR34724_1(<4 x float> %a, <4 x float> %b) {
 define <4 x float> @PR34724_2(<4 x float> %a, <4 x float> %b) {
 ; SSSE3_SLOW-LABEL: PR34724_2:
 ; SSSE3_SLOW:       # %bb.0:
-; SSSE3_SLOW-NEXT:    movaps %xmm1, %xmm2
-; SSSE3_SLOW-NEXT:    movsldup {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSSE3_SLOW-NEXT:    addps %xmm1, %xmm3
-; SSSE3_SLOW-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,0],xmm0[2,0]
-; SSSE3_SLOW-NEXT:    shufps {{.*#+}} xmm1 = xmm1[2,0],xmm0[2,3]
-; SSSE3_SLOW-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,0],xmm0[3,0]
-; SSSE3_SLOW-NEXT:    shufps {{.*#+}} xmm2 = xmm2[2,0],xmm0[2,3]
+; SSSE3_SLOW-NEXT:    haddps %xmm1, %xmm0
+; SSSE3_SLOW-NEXT:    movsldup {{.*#+}} xmm2 = xmm1[0,0,2,2]
 ; SSSE3_SLOW-NEXT:    addps %xmm1, %xmm2
-; SSSE3_SLOW-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,0],xmm2[1,0]
-; SSSE3_SLOW-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,0],xmm3[2,0]
-; SSSE3_SLOW-NEXT:    movaps %xmm2, %xmm0
+; SSSE3_SLOW-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,0],xmm0[2,0]
+; SSSE3_SLOW-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,0]
 ; SSSE3_SLOW-NEXT:    retq
 ;
 ; SSSE3_FAST-LABEL: PR34724_2:
 ; SSSE3_FAST:       # %bb.0:
-; SSSE3_FAST-NEXT:    movaps %xmm1, %xmm3
-; SSSE3_FAST-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,0],xmm0[2,0]
-; SSSE3_FAST-NEXT:    shufps {{.*#+}} xmm3 = xmm3[2,0],xmm0[2,3]
-; SSSE3_FAST-NEXT:    movaps %xmm1, %xmm2
-; SSSE3_FAST-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,0],xmm0[3,0]
-; SSSE3_FAST-NEXT:    shufps {{.*#+}} xmm2 = xmm2[2,0],xmm0[2,3]
-; SSSE3_FAST-NEXT:    addps %xmm3, %xmm2
+; SSSE3_FAST-NEXT:    haddps %xmm1, %xmm0
 ; SSSE3_FAST-NEXT:    haddps %xmm1, %xmm1
-; SSSE3_FAST-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,0],xmm2[1,0]
-; SSSE3_FAST-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,0],xmm1[2,0]
-; SSSE3_FAST-NEXT:    movaps %xmm2, %xmm0
+; SSSE3_FAST-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,0],xmm0[2,0]
+; SSSE3_FAST-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,0]
 ; SSSE3_FAST-NEXT:    retq
 ;
 ; AVX1_SLOW-LABEL: PR34724_2:
 ; AVX1_SLOW:       # %bb.0:
-; AVX1_SLOW-NEXT:    vblendps {{.*#+}} xmm2 = xmm1[0,1],xmm0[2,3]
-; AVX1_SLOW-NEXT:    vpermilps {{.*#+}} xmm2 = xmm2[2,0,2,3]
-; AVX1_SLOW-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[3],xmm1[1],zero,zero
-; AVX1_SLOW-NEXT:    vaddps %xmm0, %xmm2, %xmm0
+; AVX1_SLOW-NEXT:    vhaddps %xmm1, %xmm0, %xmm0
 ; AVX1_SLOW-NEXT:    vmovsldup {{.*#+}} xmm2 = xmm1[0,0,2,2]
 ; AVX1_SLOW-NEXT:    vaddps %xmm1, %xmm2, %xmm1
-; AVX1_SLOW-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
-; AVX1_SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,0,1,3]
+; AVX1_SLOW-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3]
 ; AVX1_SLOW-NEXT:    retq
 ;
 ; AVX1_FAST-LABEL: PR34724_2:
 ; AVX1_FAST:       # %bb.0:
-; AVX1_FAST-NEXT:    vblendps {{.*#+}} xmm2 = xmm1[0,1],xmm0[2,3]
-; AVX1_FAST-NEXT:    vpermilps {{.*#+}} xmm2 = xmm2[2,0,2,3]
-; AVX1_FAST-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[3],xmm1[1],zero,zero
-; AVX1_FAST-NEXT:    vaddps %xmm0, %xmm2, %xmm0
+; AVX1_FAST-NEXT:    vhaddps %xmm1, %xmm0, %xmm0
 ; AVX1_FAST-NEXT:    vhaddps %xmm1, %xmm1, %xmm1
-; AVX1_FAST-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
-; AVX1_FAST-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,0,1,3]
+; AVX1_FAST-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3]
 ; AVX1_FAST-NEXT:    retq
 ;
 ; AVX2_SLOW-LABEL: PR34724_2:
 ; AVX2_SLOW:       # %bb.0:
-; AVX2_SLOW-NEXT:    vblendps {{.*#+}} xmm2 = xmm1[0,1],xmm0[2,3]
-; AVX2_SLOW-NEXT:    vpermilps {{.*#+}} xmm2 = xmm2[2,0,2,3]
-; AVX2_SLOW-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[3],xmm1[1],zero,zero
-; AVX2_SLOW-NEXT:    vaddps %xmm0, %xmm2, %xmm0
+; AVX2_SLOW-NEXT:    vhaddps %xmm1, %xmm0, %xmm0
 ; AVX2_SLOW-NEXT:    vmovsldup {{.*#+}} xmm2 = xmm1[0,0,2,2]
 ; AVX2_SLOW-NEXT:    vaddps %xmm1, %xmm2, %xmm1
-; AVX2_SLOW-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
-; AVX2_SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,0,1,3]
+; AVX2_SLOW-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3]
 ; AVX2_SLOW-NEXT:    retq
 ;
 ; AVX2_FAST-LABEL: PR34724_2:
 ; AVX2_FAST:       # %bb.0:
-; AVX2_FAST-NEXT:    vblendps {{.*#+}} xmm2 = xmm1[0,1],xmm0[2,3]
-; AVX2_FAST-NEXT:    vpermilps {{.*#+}} xmm2 = xmm2[2,0,2,3]
-; AVX2_FAST-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[3],xmm1[1],zero,zero
-; AVX2_FAST-NEXT:    vaddps %xmm0, %xmm2, %xmm0
+; AVX2_FAST-NEXT:    vhaddps %xmm1, %xmm0, %xmm0
 ; AVX2_FAST-NEXT:    vhaddps %xmm1, %xmm1, %xmm1
-; AVX2_FAST-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
-; AVX2_FAST-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,0,1,3]
+; AVX2_FAST-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3]
 ; AVX2_FAST-NEXT:    retq
   %t0 = shufflevector <4 x float> %a, <4 x float> %b, <4 x i32> <i32 2, i32 4, i32 undef, i32 undef>
   %t1 = shufflevector <4 x float> %a, <4 x float> %b, <4 x i32> <i32 3, i32 5, i32 undef, i32 undef>
diff --git a/llvm/test/CodeGen/X86/haddsub-undef.ll b/llvm/test/CodeGen/X86/haddsub-undef.ll
index f8648f5b7018f..ae53f2d8905fe 100644
--- a/llvm/test/CodeGen/X86/haddsub-undef.ll
+++ b/llvm/test/CodeGen/X86/haddsub-undef.ll
@@ -818,12 +818,25 @@ define <4 x double> @PR44694(<4 x double> %0, <4 x double> %1) {
 ; SSE-FAST-NEXT:    movapd %xmm2, %xmm1
 ; SSE-FAST-NEXT:    retq
 ;
-; AVX-LABEL: PR44694:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],ymm1[2,3]
-; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1
-; AVX-NEXT:    vhaddpd %ymm0, %ymm1, %ymm0
-; AVX-NEXT:    retq
+; AVX1-SLOW-LABEL: PR44694:
+; AVX1-SLOW:       # %bb.0:
+; AVX1-SLOW-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],ymm1[2,3]
+; AVX1-SLOW-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1
+; AVX1-SLOW-NEXT:    vhaddpd %ymm0, %ymm1, %ymm0
+; AVX1-SLOW-NEXT:    retq
+;
+; AVX1-FAST-LABEL: PR44694:
+; AVX1-FAST:       # %bb.0:
+; AVX1-FAST-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],ymm1[2,3]
+; AVX1-FAST-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1
+; AVX1-FAST-NEXT:    vhaddpd %ymm0, %ymm1, %ymm0
+; AVX1-FAST-NEXT:    retq
+;
+; AVX512-LABEL: PR44694:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vhaddpd %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX512-NEXT:    retq
   %3 = shufflevector <4 x double> %0, <4 x double> %1, <4 x i32> <i32 undef, i32 2, i32 4, i32 6>
   %4 = shufflevector <4 x double> %0, <4 x double> %1, <4 x i32> <i32 undef, i32 3, i32 5, i32 7>
   %5 = fadd <4 x double> %3, %4
@@ -831,20 +844,30 @@ define <4 x double> @PR44694(<4 x double> %0, <4 x double> %1) {
 }
 
 define <4 x float> @PR45747_1(<4 x float> %a, <4 x float> %b) nounwind {
-; SSE-LABEL: PR45747_1:
-; SSE:       # %bb.0:
-; SSE-NEXT:    movshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
-; SSE-NEXT:    addps %xmm0, %xmm1
-; SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; SSE-NEXT:    movaps %xmm1, %xmm0
-; SSE-NEXT:    retq
+; SSE-SLOW-LABEL: PR45747_1:
+; SSE-SLOW:       # %bb.0:
+; SSE-SLOW-NEXT:    movshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; SSE-SLOW-NEXT:    addps %xmm0, %xmm1
+; SSE-SLOW-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; SSE-SLOW-NEXT:    movaps %xmm1, %xmm0
+; SSE-SLOW-NEXT:    retq
 ;
-; AVX-LABEL: PR45747_1:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
-; AVX-NEXT:    vaddps %xmm0, %xmm1, %xmm0
-; AVX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; AVX-NEXT:    retq
+; SSE-FAST-LABEL: PR45747_1:
+; SSE-FAST:       # %bb.0:
+; SSE-FAST-NEXT:    haddps %xmm0, %xmm0
+; SSE-FAST-NEXT:    retq
+;
+; AVX-SLOW-LABEL: PR45747_1:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX-SLOW-NEXT:    vaddps %xmm0, %xmm1, %xmm0
+; AVX-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX-SLOW-NEXT:    retq
+;
+; AVX-FAST-LABEL: PR45747_1:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    retq
   %t0 = shufflevector <4 x float> %a, <4 x float> undef, <4 x i32> <i32 undef, i32 undef, i32 3, i32 undef>
   %t1 = fadd <4 x float> %t0, %a
   %shuffle = shufflevector <4 x float> %t1, <4 x float> undef, <4 x i32> <i32 undef, i32 2, i32 undef, i32 undef>
@@ -852,19 +875,32 @@ define <4 x float> @PR45747_1(<4 x float> %a, <4 x float> %b) nounwind {
 }
 
 define <4 x float> @PR45747_2(<4 x float> %a, <4 x float> %b) nounwind {
-; SSE-LABEL: PR45747_2:
-; SSE:       # %bb.0:
-; SSE-NEXT:    movshdup {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE-NEXT:    addps %xmm1, %xmm0
-; SSE-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
-; SSE-NEXT:    retq
+; SSE-SLOW-LABEL: PR45747_2:
+; SSE-SLOW:       # %bb.0:
+; SSE-SLOW-NEXT:    movshdup {{.*#+}} xmm0 = xmm1[1,1,3,3]
+; SSE-SLOW-NEXT:    addps %xmm1, %xmm0
+; SSE-SLOW-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; SSE-SLOW-NEXT:    retq
 ;
-; AVX-LABEL: PR45747_2:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; AVX-NEXT:    vaddps %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
-; AVX-NEXT:    retq
+; SSE-FAST-LABEL: PR45747_2:
+; SSE-FAST:       # %bb.0:
+; SSE-FAST-NEXT:    movaps %xmm1, %xmm0
+; SSE-FAST-NEXT:    haddps %xmm1, %xmm0
+; SSE-FAST-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE-FAST-NEXT:    retq
+;
+; AVX-SLOW-LABEL: PR45747_2:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm1[1,1,3,3]
+; AVX-SLOW-NEXT:    vaddps %xmm1, %xmm0, %xmm0
+; AVX-SLOW-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
+; AVX-SLOW-NEXT:    retq
+;
+; AVX-FAST-LABEL: PR45747_2:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vhaddps %xmm1, %xmm1, %xmm0
+; AVX-FAST-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX-FAST-NEXT:    retq
   %t0 = shufflevector <4 x float> %b, <4 x float> undef, <4 x i32> <i32 undef, i32 undef, i32 3, i32 undef>
   %t1 = fadd <4 x float> %t0, %b
   %shuffle = shufflevector <4 x float> %t1, <4 x float> undef, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef>
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-256-v4.ll b/llvm/test/CodeGen/X86/vector-shuffle-256-v4.ll
index 5195f5f0e0c75..1fd61912ed4af 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-256-v4.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-256-v4.ll
@@ -1766,12 +1766,24 @@ define <4 x i64> @shuffle_v4i64_1z2z(<4 x i64> %a, <4 x i64> %b) {
 }
 
 define <4 x double> @add_v4f64_0246_1357(<4 x double> %a, <4 x double> %b) {
-; ALL-LABEL: add_v4f64_0246_1357:
-; ALL:       # %bb.0: # %entry
-; ALL-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm0[2,3],ymm1[2,3]
-; ALL-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; ALL-NEXT:    vhaddpd %ymm2, %ymm0, %ymm0
-; ALL-NEXT:    retq
+; AVX1-LABEL: add_v4f64_0246_1357:
+; AVX1:       # %bb.0: # %entry
+; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm0[2,3],ymm1[2,3]
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vhaddpd %ymm2, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: add_v4f64_0246_1357:
+; AVX2:       # %bb.0: # %entry
+; AVX2-NEXT:    vhaddpd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-NEXT:    retq
+;
+; AVX512VL-LABEL: add_v4f64_0246_1357:
+; AVX512VL:       # %bb.0: # %entry
+; AVX512VL-NEXT:    vhaddpd %ymm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX512VL-NEXT:    retq
 entry:
   %shuffle = shufflevector <4 x double> %a, <4 x double> %b, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
   %shuffle1 = shufflevector <4 x double> %a, <4 x double> %b, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
@@ -1780,12 +1792,24 @@ entry:
 }
 
 define <4 x double> @add_v4f64_4602_5713(<4 x double> %a, <4 x double> %b) {
-; ALL-LABEL: add_v4f64_4602_5713:
-; ALL:       # %bb.0: # %entry
-; ALL-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm1[2,3],ymm0[2,3]
-; ALL-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; ALL-NEXT:    vhaddpd %ymm2, %ymm0, %ymm0
-; ALL-NEXT:    retq
+; AVX1-LABEL: add_v4f64_4602_5713:
+; AVX1:       # %bb.0: # %entry
+; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm1[2,3],ymm0[2,3]
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX1-NEXT:    vhaddpd %ymm2, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: add_v4f64_4602_5713:
+; AVX2:       # %bb.0: # %entry
+; AVX2-NEXT:    vhaddpd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[1,3,0,2]
+; AVX2-NEXT:    retq
+;
+; AVX512VL-LABEL: add_v4f64_4602_5713:
+; AVX512VL:       # %bb.0: # %entry
+; AVX512VL-NEXT:    vhaddpd %ymm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[1,3,0,2]
+; AVX512VL-NEXT:    retq
 entry:
   %shuffle = shufflevector <4 x double> %a, <4 x double> %b, <4 x i32> <i32 4, i32 6, i32 0, i32 2>
   %shuffle1 = shufflevector <4 x double> %a, <4 x double> %b, <4 x i32> <i32 5, i32 7, i32 1, i32 3>
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-256-v8.ll b/llvm/test/CodeGen/X86/vector-shuffle-256-v8.ll
index 4798b4b1d38a2..3077428386a26 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-256-v8.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-256-v8.ll
@@ -3039,32 +3039,11 @@ define <8 x float> @add_v8f32_02468ACE_13579BDF(<8 x float> %a, <8 x float> %b)
 ; AVX1-NEXT:    vhaddps %ymm2, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
-; AVX2-LABEL: add_v8f32_02468ACE_13579BDF:
-; AVX2:       # %bb.0: # %entry
-; AVX2-NEXT:    vshufps {{.*#+}} ymm2 = ymm0[0,2],ymm1[0,2],ymm0[4,6],ymm1[4,6]
-; AVX2-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,2,1,3]
-; AVX2-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[1,3],ymm1[1,3],ymm0[5,7],ymm1[5,7]
-; AVX2-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX2-NEXT:    vaddps %ymm0, %ymm2, %ymm0
-; AVX2-NEXT:    retq
-;
-; AVX512VL-SLOW-LABEL: add_v8f32_02468ACE_13579BDF:
-; AVX512VL-SLOW:       # %bb.0: # %entry
-; AVX512VL-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm0[0,2],ymm1[0,2],ymm0[4,6],ymm1[4,6]
-; AVX512VL-SLOW-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,2,1,3]
-; AVX512VL-SLOW-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[1,3],ymm1[1,3],ymm0[5,7],ymm1[5,7]
-; AVX512VL-SLOW-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX512VL-SLOW-NEXT:    vaddps %ymm0, %ymm2, %ymm0
-; AVX512VL-SLOW-NEXT:    retq
-;
-; AVX512VL-FAST-LABEL: add_v8f32_02468ACE_13579BDF:
-; AVX512VL-FAST:       # %bb.0: # %entry
-; AVX512VL-FAST-NEXT:    vmovaps {{.*#+}} ymm2 = [0,2,4,6,8,10,12,14]
-; AVX512VL-FAST-NEXT:    vpermi2ps %ymm1, %ymm0, %ymm2
-; AVX512VL-FAST-NEXT:    vmovaps {{.*#+}} ymm3 = [1,3,5,7,9,11,13,15]
-; AVX512VL-FAST-NEXT:    vpermi2ps %ymm1, %ymm0, %ymm3
-; AVX512VL-FAST-NEXT:    vaddps %ymm3, %ymm2, %ymm0
-; AVX512VL-FAST-NEXT:    retq
+; AVX2OR512VL-LABEL: add_v8f32_02468ACE_13579BDF:
+; AVX2OR512VL:       # %bb.0: # %entry
+; AVX2OR512VL-NEXT:    vhaddps %ymm1, %ymm0, %ymm0
+; AVX2OR512VL-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2OR512VL-NEXT:    retq
 entry:
   %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
   %shuffle1 = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
@@ -3080,32 +3059,11 @@ define <8 x float> @add_v8f32_8ACE0246_9BDF1357(<8 x float> %a, <8 x float> %b)
 ; AVX1-NEXT:    vhaddps %ymm2, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
-; AVX2-LABEL: add_v8f32_8ACE0246_9BDF1357:
-; AVX2:       # %bb.0: # %entry
-; AVX2-NEXT:    vshufps {{.*#+}} ymm2 = ymm1[0,2],ymm0[0,2],ymm1[4,6],ymm0[4,6]
-; AVX2-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,2,1,3]
-; AVX2-NEXT:    vshufps {{.*#+}} ymm0 = ymm1[1,3],ymm0[1,3],ymm1[5,7],ymm0[5,7]
-; AVX2-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX2-NEXT:    vaddps %ymm0, %ymm2, %ymm0
-; AVX2-NEXT:    retq
-;
-; AVX512VL-SLOW-LABEL: add_v8f32_8ACE0246_9BDF1357:
-; AVX512VL-SLOW:       # %bb.0: # %entry
-; AVX512VL-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm1[0,2],ymm0[0,2],ymm1[4,6],ymm0[4,6]
-; AVX512VL-SLOW-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,2,1,3]
-; AVX512VL-SLOW-NEXT:    vshufps {{.*#+}} ymm0 = ymm1[1,3],ymm0[1,3],ymm1[5,7],ymm0[5,7]
-; AVX512VL-SLOW-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX512VL-SLOW-NEXT:    vaddps %ymm0, %ymm2, %ymm0
-; AVX512VL-SLOW-NEXT:    retq
-;
-; AVX512VL-FAST-LABEL: add_v8f32_8ACE0246_9BDF1357:
-; AVX512VL-FAST:       # %bb.0: # %entry
-; AVX512VL-FAST-NEXT:    vmovaps {{.*#+}} ymm2 = [0,2,4,6,8,10,12,14]
-; AVX512VL-FAST-NEXT:    vpermi2ps %ymm0, %ymm1, %ymm2
-; AVX512VL-FAST-NEXT:    vmovaps {{.*#+}} ymm3 = [1,3,5,7,9,11,13,15]
-; AVX512VL-FAST-NEXT:    vpermi2ps %ymm0, %ymm1, %ymm3
-; AVX512VL-FAST-NEXT:    vaddps %ymm3, %ymm2, %ymm0
-; AVX512VL-FAST-NEXT:    retq
+; AVX2OR512VL-LABEL: add_v8f32_8ACE0246_9BDF1357:
+; AVX2OR512VL:       # %bb.0: # %entry
+; AVX2OR512VL-NEXT:    vhaddps %ymm1, %ymm0, %ymm0
+; AVX2OR512VL-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[1,3,0,2]
+; AVX2OR512VL-NEXT:    retq
 entry:
   %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> <i32 8, i32 10, i32 12, i32 14, i32 0, i32 2, i32 4, i32 6>
   %shuffle1 = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> <i32 9, i32 11, i32 13, i32 15, i32 1, i32 3, i32 5, i32 7>
@@ -3116,45 +3074,21 @@ entry:
 define <8 x i32> @add_v8i32_02468ACE_13579BDF(<8 x i32> %a, <8 x i32> %b) {
 ; AVX1-LABEL: add_v8i32_02468ACE_13579BDF:
 ; AVX1:       # %bb.0: # %entry
-; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm0[2,3],ymm1[2,3]
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX1-NEXT:    vshufps {{.*#+}} ymm1 = ymm0[0,2],ymm2[0,2],ymm0[4,6],ymm2[4,6]
-; AVX1-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[1,3],ymm2[1,3],ymm0[5,7],ymm2[5,7]
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; AVX1-NEXT:    vpaddd %xmm2, %xmm3, %xmm2
-; AVX1-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    vphaddd %xmm1, %xmm0, %xmm2
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm2, %ymm2
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vphaddd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
+; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,2,3]
+; AVX1-NEXT:    vshufpd {{.*#+}} ymm0 = ymm2[0],ymm0[0],ymm2[3],ymm0[3]
 ; AVX1-NEXT:    retq
 ;
-; AVX2-LABEL: add_v8i32_02468ACE_13579BDF:
-; AVX2:       # %bb.0: # %entry
-; AVX2-NEXT:    vshufps {{.*#+}} ymm2 = ymm0[0,2],ymm1[0,2],ymm0[4,6],ymm1[4,6]
-; AVX2-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,2,1,3]
-; AVX2-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[1,3],ymm1[1,3],ymm0[5,7],ymm1[5,7]
-; AVX2-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX2-NEXT:    vpaddd %ymm0, %ymm2, %ymm0
-; AVX2-NEXT:    retq
-;
-; AVX512VL-SLOW-LABEL: add_v8i32_02468ACE_13579BDF:
-; AVX512VL-SLOW:       # %bb.0: # %entry
-; AVX512VL-SLOW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; AVX512VL-SLOW-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm2
-; AVX512VL-SLOW-NEXT:    vpmovqd %zmm2, %ymm2
-; AVX512VL-SLOW-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[1,3],ymm1[1,3],ymm0[5,7],ymm1[5,7]
-; AVX512VL-SLOW-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX512VL-SLOW-NEXT:    vpaddd %ymm0, %ymm2, %ymm0
-; AVX512VL-SLOW-NEXT:    retq
-;
-; AVX512VL-FAST-LABEL: add_v8i32_02468ACE_13579BDF:
-; AVX512VL-FAST:       # %bb.0: # %entry
-; AVX512VL-FAST-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; AVX512VL-FAST-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm2
-; AVX512VL-FAST-NEXT:    vpmovqd %zmm2, %ymm2
-; AVX512VL-FAST-NEXT:    vmovdqa {{.*#+}} ymm3 = [1,3,5,7,9,11,13,15]
-; AVX512VL-FAST-NEXT:    vpermi2d %ymm1, %ymm0, %ymm3
-; AVX512VL-FAST-NEXT:    vpaddd %ymm3, %ymm2, %ymm0
-; AVX512VL-FAST-NEXT:    retq
+; AVX2OR512VL-LABEL: add_v8i32_02468ACE_13579BDF:
+; AVX2OR512VL:       # %bb.0: # %entry
+; AVX2OR512VL-NEXT:    vphaddd %ymm1, %ymm0, %ymm0
+; AVX2OR512VL-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2OR512VL-NEXT:    retq
 entry:
   %shuffle = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
   %shuffle1 = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
@@ -3165,45 +3099,21 @@ entry:
 define <8 x i32> @add_v8i32_8ACE0246_9BDF1357(<8 x i32> %a, <8 x i32> %b) {
 ; AVX1-LABEL: add_v8i32_8ACE0246_9BDF1357:
 ; AVX1:       # %bb.0: # %entry
-; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm1[2,3],ymm0[2,3]
-; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; AVX1-NEXT:    vshufps {{.*#+}} ymm1 = ymm0[0,2],ymm2[0,2],ymm0[4,6],ymm2[4,6]
-; AVX1-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[1,3],ymm2[1,3],ymm0[5,7],ymm2[5,7]
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; AVX1-NEXT:    vpaddd %xmm2, %xmm3, %xmm2
-; AVX1-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    vphaddd %xmm1, %xmm0, %xmm2
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm2, %ymm2
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vphaddd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
+; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,2,3]
+; AVX1-NEXT:    vshufpd {{.*#+}} ymm0 = ymm2[1],ymm0[1],ymm2[2],ymm0[2]
 ; AVX1-NEXT:    retq
 ;
-; AVX2-LABEL: add_v8i32_8ACE0246_9BDF1357:
-; AVX2:       # %bb.0: # %entry
-; AVX2-NEXT:    vshufps {{.*#+}} ymm2 = ymm1[0,2],ymm0[0,2],ymm1[4,6],ymm0[4,6]
-; AVX2-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,2,1,3]
-; AVX2-NEXT:    vshufps {{.*#+}} ymm0 = ymm1[1,3],ymm0[1,3],ymm1[5,7],ymm0[5,7]
-; AVX2-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX2-NEXT:    vpaddd %ymm0, %ymm2, %ymm0
-; AVX2-NEXT:    retq
-;
-; AVX512VL-SLOW-LABEL: add_v8i32_8ACE0246_9BDF1357:
-; AVX512VL-SLOW:       # %bb.0: # %entry
-; AVX512VL-SLOW-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
-; AVX512VL-SLOW-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm2
-; AVX512VL-SLOW-NEXT:    vpmovqd %zmm2, %ymm2
-; AVX512VL-SLOW-NEXT:    vshufps {{.*#+}} ymm0 = ymm1[1,3],ymm0[1,3],ymm1[5,7],ymm0[5,7]
-; AVX512VL-SLOW-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX512VL-SLOW-NEXT:    vpaddd %ymm0, %ymm2, %ymm0
-; AVX512VL-SLOW-NEXT:    retq
-;
-; AVX512VL-FAST-LABEL: add_v8i32_8ACE0246_9BDF1357:
-; AVX512VL-FAST:       # %bb.0: # %entry
-; AVX512VL-FAST-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
-; AVX512VL-FAST-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm2
-; AVX512VL-FAST-NEXT:    vpmovqd %zmm2, %ymm2
-; AVX512VL-FAST-NEXT:    vmovdqa {{.*#+}} ymm3 = [1,3,5,7,9,11,13,15]
-; AVX512VL-FAST-NEXT:    vpermi2d %ymm0, %ymm1, %ymm3
-; AVX512VL-FAST-NEXT:    vpaddd %ymm3, %ymm2, %ymm0
-; AVX512VL-FAST-NEXT:    retq
+; AVX2OR512VL-LABEL: add_v8i32_8ACE0246_9BDF1357:
+; AVX2OR512VL:       # %bb.0: # %entry
+; AVX2OR512VL-NEXT:    vphaddd %ymm1, %ymm0, %ymm0
+; AVX2OR512VL-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[1,3,0,2]
+; AVX2OR512VL-NEXT:    retq
 entry:
   %shuffle = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32> <i32 8, i32 10, i32 12, i32 14, i32 0, i32 2, i32 4, i32 6>
   %shuffle1 = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32> <i32 9, i32 11, i32 13, i32 15, i32 1, i32 3, i32 5, i32 7>

From e2f5444c9ce2e0558a69fb32bd001017244eeebc Mon Sep 17 00:00:00 2001
From: Joachim Protze <protze@itc.rwth-aachen.de>
Date: Tue, 28 Jul 2020 11:08:24 +0200
Subject: [PATCH 0277/1035] [OpenMP][Tests] Enable nvptx64 testing for most
 libomptarget tests

Also add $BUILD/lib to the LIBRARY_PATH to fix
https://bugs.llvm.org/show_bug.cgi?id=46836.

Reviewed By: JonChesterfield

Differential Revision: https://reviews.llvm.org/D84557
---
 openmp/libomptarget/test/env/omp_target_debug.c              | 2 ++
 openmp/libomptarget/test/lit.cfg                             | 3 +++
 openmp/libomptarget/test/mapping/alloc_fail.c                | 4 ++++
 openmp/libomptarget/test/mapping/declare_mapper_api.cpp      | 1 +
 openmp/libomptarget/test/mapping/declare_mapper_target.cpp   | 2 +-
 .../libomptarget/test/mapping/declare_mapper_target_data.cpp | 2 +-
 .../test/mapping/declare_mapper_target_data_enter_exit.cpp   | 2 +-
 .../test/mapping/declare_mapper_target_update.cpp            | 2 +-
 openmp/libomptarget/test/mapping/delete_inf_refcount.c       | 1 +
 openmp/libomptarget/test/mapping/pr38704.c                   | 1 +
 openmp/libomptarget/test/offloading/d2d_memcpy.c             | 5 ++---
 openmp/libomptarget/test/offloading/dynamic_module.c         | 1 +
 openmp/libomptarget/test/offloading/dynamic_module_load.c    | 1 +
 openmp/libomptarget/test/offloading/looptripcnt.c            | 1 +
 openmp/libomptarget/test/offloading/offloading_success.c     | 1 +
 openmp/libomptarget/test/offloading/offloading_success.cpp   | 1 +
 .../libomptarget/test/offloading/parallel_offloading_map.c   | 1 +
 openmp/libomptarget/test/offloading/requires.c               | 1 +
 openmp/libomptarget/test/offloading/target_depend_nowait.cpp | 1 +
 19 files changed, 26 insertions(+), 7 deletions(-)

diff --git a/openmp/libomptarget/test/env/omp_target_debug.c b/openmp/libomptarget/test/env/omp_target_debug.c
index ce84c9842f64f..34a71793d9271 100644
--- a/openmp/libomptarget/test/env/omp_target_debug.c
+++ b/openmp/libomptarget/test/env/omp_target_debug.c
@@ -6,6 +6,8 @@
 // RUN: %libomptarget-compile-powerpc64le-ibm-linux-gnu && env LIBOMPTARGET_DEBUG=0 %libomptarget-run-powerpc64le-ibm-linux-gnu 2>&1 | %fcheck-powerpc64le-ibm-linux-gnu -allow-empty -check-prefix=NDEBUG
 // RUN: %libomptarget-compile-x86_64-pc-linux-gnu && env LIBOMPTARGET_DEBUG=1 %libomptarget-run-x86_64-pc-linux-gnu 2>&1 | %fcheck-x86_64-pc-linux-gnu -allow-empty -check-prefix=DEBUG
 // RUN: %libomptarget-compile-x86_64-pc-linux-gnu && env LIBOMPTARGET_DEBUG=0 %libomptarget-run-x86_64-pc-linux-gnu 2>&1 | %fcheck-x86_64-pc-linux-gnu -allow-empty -check-prefix=NDEBUG
+// RUN: %libomptarget-compile-nvptx64-nvidia-cuda && env LIBOMPTARGET_DEBUG=1 %libomptarget-run-nvptx64-nvidia-cuda 2>&1 | %fcheck-nvptx64-nvidia-cuda -allow-empty -check-prefix=DEBUG
+// RUN: %libomptarget-compile-nvptx64-nvidia-cuda && env LIBOMPTARGET_DEBUG=0 %libomptarget-run-nvptx64-nvidia-cuda 2>&1 | %fcheck-nvptx64-nvidia-cuda -allow-empty -check-prefix=NDEBUG
 // REQUIRES: libomptarget-debug
 
 int main(void) {
diff --git a/openmp/libomptarget/test/lit.cfg b/openmp/libomptarget/test/lit.cfg
index 3dbe8789d5162..77476c6fec793 100644
--- a/openmp/libomptarget/test/lit.cfg
+++ b/openmp/libomptarget/test/lit.cfg
@@ -71,6 +71,9 @@ else: # Unices
     append_dynamic_library_path('LD_LIBRARY_PATH', config.library_dir, ":")
     append_dynamic_library_path('LD_LIBRARY_PATH', \
         config.omp_host_rtl_directory, ":")
+    append_dynamic_library_path('LIBRARY_PATH', config.library_dir, ":")
+    append_dynamic_library_path('LIBRARY_PATH', \
+        config.omp_host_rtl_directory, ":")
 
 # substitutions
 # - for targets that exist in the system create the actual command.
diff --git a/openmp/libomptarget/test/mapping/alloc_fail.c b/openmp/libomptarget/test/mapping/alloc_fail.c
index ca15a429f130c..6d1f708dcb2ec 100644
--- a/openmp/libomptarget/test/mapping/alloc_fail.c
+++ b/openmp/libomptarget/test/mapping/alloc_fail.c
@@ -14,6 +14,10 @@
 // RUN: %libomptarget-run-fail-x86_64-pc-linux-gnu 2>&1 \
 // RUN: | %fcheck-x86_64-pc-linux-gnu
 
+// RUN: %libomptarget-compile-nvptx64-nvidia-cuda
+// RUN: %libomptarget-run-fail-nvptx64-nvidia-cuda 2>&1 \
+// RUN: | %fcheck-nvptx64-nvidia-cuda
+
 // CHECK: Libomptarget fatal error 1: failure of target construct while offloading is mandatory
 
 int main() {
diff --git a/openmp/libomptarget/test/mapping/declare_mapper_api.cpp b/openmp/libomptarget/test/mapping/declare_mapper_api.cpp
index 45bc076d4110a..54a5ad61538b9 100644
--- a/openmp/libomptarget/test/mapping/declare_mapper_api.cpp
+++ b/openmp/libomptarget/test/mapping/declare_mapper_api.cpp
@@ -2,6 +2,7 @@
 // RUN: %libomptarget-compilexx-run-and-check-powerpc64-ibm-linux-gnu
 // RUN: %libomptarget-compilexx-run-and-check-powerpc64le-ibm-linux-gnu
 // RUN: %libomptarget-compilexx-run-and-check-x86_64-pc-linux-gnu
+// RUN: %libomptarget-compilexx-run-and-check-nvptx64-nvidia-cuda
 
 #include <cstdio>
 #include <cstdlib>
diff --git a/openmp/libomptarget/test/mapping/declare_mapper_target.cpp b/openmp/libomptarget/test/mapping/declare_mapper_target.cpp
index 6246285283393..096431f6edc0b 100644
--- a/openmp/libomptarget/test/mapping/declare_mapper_target.cpp
+++ b/openmp/libomptarget/test/mapping/declare_mapper_target.cpp
@@ -1,8 +1,8 @@
 // RUN: %libomptarget-compile-run-and-check-aarch64-unknown-linux-gnu
-// RUN: %libomptarget-compile-run-and-check-nvptx64-nvidia-cuda
 // RUN: %libomptarget-compile-run-and-check-powerpc64-ibm-linux-gnu
 // RUN: %libomptarget-compile-run-and-check-powerpc64le-ibm-linux-gnu
 // RUN: %libomptarget-compile-run-and-check-x86_64-pc-linux-gnu
+// RUN: %libomptarget-compile-run-and-check-nvptx64-nvidia-cuda
 
 #include <cstdio>
 #include <cstdlib>
diff --git a/openmp/libomptarget/test/mapping/declare_mapper_target_data.cpp b/openmp/libomptarget/test/mapping/declare_mapper_target_data.cpp
index b457048a2e580..e2636f157d659 100644
--- a/openmp/libomptarget/test/mapping/declare_mapper_target_data.cpp
+++ b/openmp/libomptarget/test/mapping/declare_mapper_target_data.cpp
@@ -1,8 +1,8 @@
 // RUN: %libomptarget-compile-run-and-check-aarch64-unknown-linux-gnu
-// RUN: %libomptarget-compile-run-and-check-nvptx64-nvidia-cuda
 // RUN: %libomptarget-compile-run-and-check-powerpc64-ibm-linux-gnu
 // RUN: %libomptarget-compile-run-and-check-powerpc64le-ibm-linux-gnu
 // RUN: %libomptarget-compile-run-and-check-x86_64-pc-linux-gnu
+// RUN: %libomptarget-compile-run-and-check-nvptx64-nvidia-cuda
 
 #include <cstdio>
 #include <cstdlib>
diff --git a/openmp/libomptarget/test/mapping/declare_mapper_target_data_enter_exit.cpp b/openmp/libomptarget/test/mapping/declare_mapper_target_data_enter_exit.cpp
index ac915a08ea8e2..44fcc20e8ee13 100644
--- a/openmp/libomptarget/test/mapping/declare_mapper_target_data_enter_exit.cpp
+++ b/openmp/libomptarget/test/mapping/declare_mapper_target_data_enter_exit.cpp
@@ -1,8 +1,8 @@
 // RUN: %libomptarget-compile-run-and-check-aarch64-unknown-linux-gnu
-// RUN: %libomptarget-compile-run-and-check-nvptx64-nvidia-cuda
 // RUN: %libomptarget-compile-run-and-check-powerpc64-ibm-linux-gnu
 // RUN: %libomptarget-compile-run-and-check-powerpc64le-ibm-linux-gnu
 // RUN: %libomptarget-compile-run-and-check-x86_64-pc-linux-gnu
+// RUN: %libomptarget-compile-run-and-check-nvptx64-nvidia-cuda
 
 #include <cstdio>
 #include <cstdlib>
diff --git a/openmp/libomptarget/test/mapping/declare_mapper_target_update.cpp b/openmp/libomptarget/test/mapping/declare_mapper_target_update.cpp
index 689275962f2c5..c894c92dc09a6 100644
--- a/openmp/libomptarget/test/mapping/declare_mapper_target_update.cpp
+++ b/openmp/libomptarget/test/mapping/declare_mapper_target_update.cpp
@@ -1,8 +1,8 @@
 // RUN: %libomptarget-compile-run-and-check-aarch64-unknown-linux-gnu
-// RUN: %libomptarget-compile-run-and-check-nvptx64-nvidia-cuda
 // RUN: %libomptarget-compile-run-and-check-powerpc64-ibm-linux-gnu
 // RUN: %libomptarget-compile-run-and-check-powerpc64le-ibm-linux-gnu
 // RUN: %libomptarget-compile-run-and-check-x86_64-pc-linux-gnu
+// RUN: %libomptarget-compile-run-and-check-nvptx64-nvidia-cuda
 
 #include <cstdio>
 #include <cstdlib>
diff --git a/openmp/libomptarget/test/mapping/delete_inf_refcount.c b/openmp/libomptarget/test/mapping/delete_inf_refcount.c
index b4106be04ab73..e6b6e94326968 100644
--- a/openmp/libomptarget/test/mapping/delete_inf_refcount.c
+++ b/openmp/libomptarget/test/mapping/delete_inf_refcount.c
@@ -2,6 +2,7 @@
 // RUN: %libomptarget-compile-run-and-check-powerpc64-ibm-linux-gnu
 // RUN: %libomptarget-compile-run-and-check-powerpc64le-ibm-linux-gnu
 // RUN: %libomptarget-compile-run-and-check-x86_64-pc-linux-gnu
+// RUN: %libomptarget-compile-run-and-check-nvptx64-nvidia-cuda
 
 #include <stdio.h>
 #include <omp.h>
diff --git a/openmp/libomptarget/test/mapping/pr38704.c b/openmp/libomptarget/test/mapping/pr38704.c
index 3e7135e284114..c4e80ca44f6b9 100644
--- a/openmp/libomptarget/test/mapping/pr38704.c
+++ b/openmp/libomptarget/test/mapping/pr38704.c
@@ -2,6 +2,7 @@
 // RUN: %libomptarget-compile-run-and-check-powerpc64-ibm-linux-gnu
 // RUN: %libomptarget-compile-run-and-check-powerpc64le-ibm-linux-gnu
 // RUN: %libomptarget-compile-run-and-check-x86_64-pc-linux-gnu
+// RUN: %libomptarget-compile-run-and-check-nvptx64-nvidia-cuda
 
 // Clang 6.0 doesn't use the new map interface, undefined behavior when
 // the compiler emits "old" interface code for structures.
diff --git a/openmp/libomptarget/test/offloading/d2d_memcpy.c b/openmp/libomptarget/test/offloading/d2d_memcpy.c
index 4c5f2c2ef5ffa..968f7112cd8b0 100644
--- a/openmp/libomptarget/test/offloading/d2d_memcpy.c
+++ b/openmp/libomptarget/test/offloading/d2d_memcpy.c
@@ -2,6 +2,7 @@
 // RUN: %libomptarget-compile-powerpc64-ibm-linux-gnu && env OMP_MAX_ACTIVE_LEVELS=2 %libomptarget-run-powerpc64-ibm-linux-gnu | %fcheck-powerpc64-ibm-linux-gnu
 // RUN: %libomptarget-compile-powerpc64le-ibm-linux-gnu && env OMP_MAX_ACTIVE_LEVELS=2 %libomptarget-run-powerpc64le-ibm-linux-gnu | %fcheck-powerpc64le-ibm-linux-gnu
 // RUN: %libomptarget-compile-x86_64-pc-linux-gnu && env OMP_MAX_ACTIVE_LEVELS=2 %libomptarget-run-x86_64-pc-linux-gnu | %fcheck-x86_64-pc-linux-gnu -allow-empty
+// RUN: %libomptarget-compile-nvptx64-nvidia-cuda && env OMP_MAX_ACTIVE_LEVELS=2 %libomptarget-run-nvptx64-nvidia-cuda | %fcheck-nvptx64-nvidia-cuda -allow-empty
 
 #include <assert.h>
 #include <omp.h>
@@ -21,9 +22,7 @@ int main(int argc, char *argv[]) {
   }
 
   const int src_device = 0;
-  int dst_device = 1;
-  if (dst_device >= num_devices)
-    dst_device = num_devices - 1;
+  int dst_device = num_devices - 1;
 
   int length = N * sizeof(int);
   int *src_ptr = omp_target_alloc(length, src_device);
diff --git a/openmp/libomptarget/test/offloading/dynamic_module.c b/openmp/libomptarget/test/offloading/dynamic_module.c
index 4cb6f35ba8992..ba3c025ba30d0 100644
--- a/openmp/libomptarget/test/offloading/dynamic_module.c
+++ b/openmp/libomptarget/test/offloading/dynamic_module.c
@@ -2,6 +2,7 @@
 // RUN: %libomptarget-compile-powerpc64-ibm-linux-gnu -DSHARED -fPIC -shared -o %t.so && %libomptarget-compile-powerpc64-ibm-linux-gnu %t.so && %libomptarget-run-powerpc64-ibm-linux-gnu 2>&1 | %fcheck-powerpc64-ibm-linux-gnu
 // RUN: %libomptarget-compile-powerpc64le-ibm-linux-gnu -DSHARED -fPIC -shared -o %t.so && %libomptarget-compile-powerpc64le-ibm-linux-gnu %t.so && %libomptarget-run-powerpc64le-ibm-linux-gnu 2>&1 | %fcheck-powerpc64le-ibm-linux-gnu
 // RUN: %libomptarget-compile-x86_64-pc-linux-gnu -DSHARED -fPIC -shared -o %t.so && %libomptarget-compile-x86_64-pc-linux-gnu %t.so && %libomptarget-run-x86_64-pc-linux-gnu 2>&1 | %fcheck-x86_64-pc-linux-gnu
+// RUN: %libomptarget-compile-nvptx64-nvidia-cuda -DSHARED -fPIC -shared -o %t.so && %libomptarget-compile-nvptx64-nvidia-cuda %t.so && %libomptarget-run-nvptx64-nvidia-cuda 2>&1 | %fcheck-nvptx64-nvidia-cuda
 
 #ifdef SHARED
 void foo() {}
diff --git a/openmp/libomptarget/test/offloading/dynamic_module_load.c b/openmp/libomptarget/test/offloading/dynamic_module_load.c
index 958fe2dec351c..f70d76523dc82 100644
--- a/openmp/libomptarget/test/offloading/dynamic_module_load.c
+++ b/openmp/libomptarget/test/offloading/dynamic_module_load.c
@@ -2,6 +2,7 @@
 // RUN: %libomptarget-compile-powerpc64-ibm-linux-gnu -DSHARED -fPIC -shared -o %t.so && %clang %flags %s -o %t-powerpc64-ibm-linux-gnu -ldl && %libomptarget-run-powerpc64-ibm-linux-gnu %t.so 2>&1 | %fcheck-powerpc64-ibm-linux-gnu
 // RUN: %libomptarget-compile-powerpc64le-ibm-linux-gnu -DSHARED -fPIC -shared -o %t.so && %clang %flags %s -o %t-powerpc64le-ibm-linux-gnu -ldl && %libomptarget-run-powerpc64le-ibm-linux-gnu %t.so 2>&1 | %fcheck-powerpc64le-ibm-linux-gnu
 // RUN: %libomptarget-compile-x86_64-pc-linux-gnu -DSHARED -fPIC -shared -o %t.so && %clang %flags %s -o %t-x86_64-pc-linux-gnu -ldl && %libomptarget-run-x86_64-pc-linux-gnu %t.so 2>&1 | %fcheck-x86_64-pc-linux-gnu
+// RUN: %libomptarget-compile-nvptx64-nvidia-cuda -DSHARED -fPIC -shared -o %t.so && %clang %flags %s -o %t-nvptx64-nvidia-cuda -ldl && %libomptarget-run-nvptx64-nvidia-cuda %t.so 2>&1 | %fcheck-nvptx64-nvidia-cuda
 
 #ifdef SHARED
 #include <stdio.h>
diff --git a/openmp/libomptarget/test/offloading/looptripcnt.c b/openmp/libomptarget/test/offloading/looptripcnt.c
index 025231b0c6d32..d4c3d6013328b 100644
--- a/openmp/libomptarget/test/offloading/looptripcnt.c
+++ b/openmp/libomptarget/test/offloading/looptripcnt.c
@@ -2,6 +2,7 @@
 // RUN: %libomptarget-compile-powerpc64-ibm-linux-gnu && env LIBOMPTARGET_DEBUG=1 %libomptarget-run-powerpc64-ibm-linux-gnu 2>&1 | %fcheck-powerpc64-ibm-linux-gnu -allow-empty -check-prefix=DEBUG
 // RUN: %libomptarget-compile-powerpc64le-ibm-linux-gnu && env LIBOMPTARGET_DEBUG=1 %libomptarget-run-powerpc64le-ibm-linux-gnu 2>&1 | %fcheck-powerpc64le-ibm-linux-gnu -allow-empty -check-prefix=DEBUG
 // RUN: %libomptarget-compile-x86_64-pc-linux-gnu && env LIBOMPTARGET_DEBUG=1 %libomptarget-run-x86_64-pc-linux-gnu 2>&1 | %fcheck-x86_64-pc-linux-gnu -allow-empty -check-prefix=DEBUG
+// RUN: %libomptarget-compile-nvptx64-nvidia-cuda && env LIBOMPTARGET_DEBUG=1 %libomptarget-run-nvptx64-nvidia-cuda 2>&1 | %fcheck-nvptx64-nvidia-cuda -allow-empty -check-prefix=DEBUG
 // REQUIRES: libomptarget-debug
 
 /*
diff --git a/openmp/libomptarget/test/offloading/offloading_success.c b/openmp/libomptarget/test/offloading/offloading_success.c
index 12e78fac1f5a3..000a38cb5478c 100644
--- a/openmp/libomptarget/test/offloading/offloading_success.c
+++ b/openmp/libomptarget/test/offloading/offloading_success.c
@@ -2,6 +2,7 @@
 // RUN: %libomptarget-compile-run-and-check-powerpc64-ibm-linux-gnu
 // RUN: %libomptarget-compile-run-and-check-powerpc64le-ibm-linux-gnu
 // RUN: %libomptarget-compile-run-and-check-x86_64-pc-linux-gnu
+// RUN: %libomptarget-compile-run-and-check-nvptx64-nvidia-cuda
 
 #include <stdio.h>
 #include <omp.h>
diff --git a/openmp/libomptarget/test/offloading/offloading_success.cpp b/openmp/libomptarget/test/offloading/offloading_success.cpp
index eecd97a3f317d..910cb1790e9d4 100644
--- a/openmp/libomptarget/test/offloading/offloading_success.cpp
+++ b/openmp/libomptarget/test/offloading/offloading_success.cpp
@@ -2,6 +2,7 @@
 // RUN: %libomptarget-compilexx-run-and-check-powerpc64-ibm-linux-gnu
 // RUN: %libomptarget-compilexx-run-and-check-powerpc64le-ibm-linux-gnu
 // RUN: %libomptarget-compilexx-run-and-check-x86_64-pc-linux-gnu
+// RUN: %libomptarget-compilexx-run-and-check-nvptx64-nvidia-cuda
 
 #include <stdio.h>
 #include <omp.h>
diff --git a/openmp/libomptarget/test/offloading/parallel_offloading_map.c b/openmp/libomptarget/test/offloading/parallel_offloading_map.c
index 3bd59574747d5..afac7225f37a4 100644
--- a/openmp/libomptarget/test/offloading/parallel_offloading_map.c
+++ b/openmp/libomptarget/test/offloading/parallel_offloading_map.c
@@ -2,6 +2,7 @@
 // RUN: %libomptarget-compile-powerpc64-ibm-linux-gnu && env OMP_MAX_ACTIVE_LEVELS=2 %libomptarget-run-powerpc64-ibm-linux-gnu | %fcheck-powerpc64-ibm-linux-gnu
 // RUN: %libomptarget-compile-powerpc64le-ibm-linux-gnu && env OMP_MAX_ACTIVE_LEVELS=2 %libomptarget-run-powerpc64le-ibm-linux-gnu | %fcheck-powerpc64le-ibm-linux-gnu
 // RUN: %libomptarget-compile-x86_64-pc-linux-gnu && env OMP_MAX_ACTIVE_LEVELS=2 %libomptarget-run-x86_64-pc-linux-gnu | %fcheck-x86_64-pc-linux-gnu -allow-empty
+// RUN: %libomptarget-compile-nvptx64-nvidia-cuda && env OMP_MAX_ACTIVE_LEVELS=2 %libomptarget-run-nvptx64-nvidia-cuda | %fcheck-nvptx64-nvidia-cuda -allow-empty
 #include <assert.h>
 #include <stdio.h>
 
diff --git a/openmp/libomptarget/test/offloading/requires.c b/openmp/libomptarget/test/offloading/requires.c
index 079ce5cb9348c..60346b1039ac9 100644
--- a/openmp/libomptarget/test/offloading/requires.c
+++ b/openmp/libomptarget/test/offloading/requires.c
@@ -2,6 +2,7 @@
 // RUN: %libomptarget-compile-powerpc64-ibm-linux-gnu && env LIBOMPTARGET_DEBUG=1 %libomptarget-run-powerpc64-ibm-linux-gnu 2>&1 | %fcheck-powerpc64-ibm-linux-gnu -allow-empty -check-prefix=DEBUG
 // RUN: %libomptarget-compile-powerpc64le-ibm-linux-gnu && env LIBOMPTARGET_DEBUG=1 %libomptarget-run-powerpc64le-ibm-linux-gnu 2>&1 | %fcheck-powerpc64le-ibm-linux-gnu -allow-empty -check-prefix=DEBUG
 // RUN: %libomptarget-compile-x86_64-pc-linux-gnu && env LIBOMPTARGET_DEBUG=1 %libomptarget-run-x86_64-pc-linux-gnu 2>&1 | %fcheck-x86_64-pc-linux-gnu -allow-empty -check-prefix=DEBUG
+// RUN: %libomptarget-compile-nvptx64-nvidia-cuda && env LIBOMPTARGET_DEBUG=1 %libomptarget-run-nvptx64-nvidia-cuda 2>&1 | %fcheck-nvptx64-nvidia-cuda -allow-empty -check-prefix=DEBUG
 // REQUIRES: libomptarget-debug
 
 /*
diff --git a/openmp/libomptarget/test/offloading/target_depend_nowait.cpp b/openmp/libomptarget/test/offloading/target_depend_nowait.cpp
index 2c1c7e7191882..32a85d9725392 100644
--- a/openmp/libomptarget/test/offloading/target_depend_nowait.cpp
+++ b/openmp/libomptarget/test/offloading/target_depend_nowait.cpp
@@ -2,6 +2,7 @@
 // RUN: %libomptarget-compilexx-run-and-check-powerpc64-ibm-linux-gnu
 // RUN: %libomptarget-compilexx-run-and-check-powerpc64le-ibm-linux-gnu
 // RUN: %libomptarget-compilexx-run-and-check-x86_64-pc-linux-gnu
+// RUN: %libomptarget-compilexx-run-and-check-nvptx64-nvidia-cuda
 
 #include <omp.h>
 #include <stdio.h>

From f5acd11d2c0ea228452aa5ed3abbc2c502009d56 Mon Sep 17 00:00:00 2001
From: Bruno Ricci <riccibrun@gmail.com>
Date: Mon, 27 Jul 2020 23:19:02 +0100
Subject: [PATCH 0278/1035] [clang-format][NFC] Be more careful about the
 layout of FormatToken.

The underlying ABI forces FormatToken to have a lot of padding.

Currently (on x86-64 linux) `sizeof(FormatToken) == 288`. After this patch
`sizeof(FormatToken) == 232`.

No functional changes.

Reviewed By: MyDeveloperDay

Differential Revision: https://reviews.llvm.org/D84306
---
 clang/lib/Format/ContinuationIndenter.cpp   |  20 +-
 clang/lib/Format/Format.cpp                 |   4 +-
 clang/lib/Format/FormatToken.cpp            |   4 +-
 clang/lib/Format/FormatToken.h              | 198 ++++++++++++--------
 clang/lib/Format/TokenAnnotator.cpp         |  43 +++--
 clang/lib/Format/UnwrappedLineFormatter.cpp |   8 +-
 clang/lib/Format/UnwrappedLineParser.cpp    |  44 ++---
 clang/lib/Format/WhitespaceManager.cpp      |   2 +-
 8 files changed, 183 insertions(+), 140 deletions(-)

diff --git a/clang/lib/Format/ContinuationIndenter.cpp b/clang/lib/Format/ContinuationIndenter.cpp
index b1497651a8fef..f3202bcb5bc14 100644
--- a/clang/lib/Format/ContinuationIndenter.cpp
+++ b/clang/lib/Format/ContinuationIndenter.cpp
@@ -284,7 +284,7 @@ bool ContinuationIndenter::canBreak(const LineState &State) {
   // The opening "{" of a braced list has to be on the same line as the first
   // element if it is nested in another braced init list or function call.
   if (!Current.MustBreakBefore && Previous.is(tok::l_brace) &&
-      Previous.isNot(TT_DictLiteral) && Previous.BlockKind == BK_BracedInit &&
+      Previous.isNot(TT_DictLiteral) && Previous.is(BK_BracedInit) &&
       Previous.Previous &&
       Previous.Previous->isOneOf(tok::l_brace, tok::l_paren, tok::comma))
     return false;
@@ -501,7 +501,7 @@ bool ContinuationIndenter::mustBreak(const LineState &State) {
   // The following could be precomputed as they do not depend on the state.
   // However, as they should take effect only if the UnwrappedLine does not fit
   // into the ColumnLimit, they are checked here in the ContinuationIndenter.
-  if (Style.ColumnLimit != 0 && Previous.BlockKind == BK_Block &&
+  if (Style.ColumnLimit != 0 && Previous.is(BK_Block) &&
       Previous.is(tok::l_brace) && !Current.isOneOf(tok::r_brace, tok::comment))
     return true;
 
@@ -627,7 +627,7 @@ void ContinuationIndenter::addTokenOnCurrentLine(LineState &State, bool DryRun,
   // opening parenthesis. Don't break if it doesn't conserve columns.
   if (Style.AlignAfterOpenBracket == FormatStyle::BAS_AlwaysBreak &&
       (Previous.isOneOf(tok::l_paren, TT_TemplateOpener, tok::l_square) ||
-       (Previous.is(tok::l_brace) && Previous.BlockKind != BK_Block &&
+       (Previous.is(tok::l_brace) && Previous.isNot(BK_Block) &&
         Style.Cpp11BracedListStyle)) &&
       State.Column > getNewLineColumn(State) &&
       (!Previous.Previous || !Previous.Previous->isOneOf(
@@ -648,7 +648,7 @@ void ContinuationIndenter::addTokenOnCurrentLine(LineState &State, bool DryRun,
   if (Style.AlignAfterOpenBracket != FormatStyle::BAS_DontAlign &&
       !State.Stack.back().IsCSharpGenericTypeConstraint &&
       Previous.opensScope() && Previous.isNot(TT_ObjCMethodExpr) &&
-      (Current.isNot(TT_LineComment) || Previous.BlockKind == BK_BracedInit)) {
+      (Current.isNot(TT_LineComment) || Previous.is(BK_BracedInit))) {
     State.Stack.back().Indent = State.Column + Spaces;
     State.Stack.back().IsAligned = true;
   }
@@ -972,7 +972,7 @@ unsigned ContinuationIndenter::getNewLineColumn(const LineState &State) {
     return (Style.IndentWidth * State.Line->First->IndentLevel) +
            Style.IndentWidth;
 
-  if (NextNonComment->is(tok::l_brace) && NextNonComment->BlockKind == BK_Block)
+  if (NextNonComment->is(tok::l_brace) && NextNonComment->is(BK_Block))
     return Current.NestingLevel == 0 ? State.FirstIndent
                                      : State.Stack.back().Indent;
   if ((Current.isOneOf(tok::r_brace, tok::r_square) ||
@@ -982,8 +982,7 @@ unsigned ContinuationIndenter::getNewLineColumn(const LineState &State) {
       State.Stack.size() > 1) {
     if (Current.closesBlockOrBlockTypeList(Style))
       return State.Stack[State.Stack.size() - 2].NestedBlockIndent;
-    if (Current.MatchingParen &&
-        Current.MatchingParen->BlockKind == BK_BracedInit)
+    if (Current.MatchingParen && Current.MatchingParen->is(BK_BracedInit))
       return State.Stack[State.Stack.size() - 2].LastSpace;
     return State.FirstIndent;
   }
@@ -1417,7 +1416,7 @@ void ContinuationIndenter::moveStatePastScopeOpener(LineState &State,
       State.Stack.back().IsCSharpGenericTypeConstraint)
     return;
 
-  if (Current.MatchingParen && Current.BlockKind == BK_Block) {
+  if (Current.MatchingParen && Current.is(BK_Block)) {
     moveStateToNewBlock(State);
     return;
   }
@@ -1486,9 +1485,8 @@ void ContinuationIndenter::moveStatePastScopeOpener(LineState &State,
         (State.Line->MustBeDeclaration && !BinPackDeclaration) ||
         (!State.Line->MustBeDeclaration && !Style.BinPackArguments) ||
         (Style.ExperimentalAutoDetectBinPacking &&
-         (Current.PackingKind == PPK_OnePerLine ||
-          (!BinPackInconclusiveFunctions &&
-           Current.PackingKind == PPK_Inconclusive)));
+         (Current.is(PPK_OnePerLine) ||
+          (!BinPackInconclusiveFunctions && Current.is(PPK_Inconclusive))));
 
     if (Current.is(TT_ObjCMethodExpr) && Current.MatchingParen &&
         Style.ObjCBreakBeforeNestedBlockParam) {
diff --git a/clang/lib/Format/Format.cpp b/clang/lib/Format/Format.cpp
index 3966f0a38639b..8c1d7c90e02a0 100644
--- a/clang/lib/Format/Format.cpp
+++ b/clang/lib/Format/Format.cpp
@@ -1575,9 +1575,9 @@ class Formatter : public TokenAnalyzer {
         continue;
       FormatToken *Tok = AnnotatedLines[i]->First->Next;
       while (Tok->Next) {
-        if (Tok->PackingKind == PPK_BinPacked)
+        if (Tok->is(PPK_BinPacked))
           HasBinPackedFunction = true;
-        if (Tok->PackingKind == PPK_OnePerLine)
+        if (Tok->is(PPK_OnePerLine))
           HasOnePerLineFunction = true;
 
         Tok = Tok->Next;
diff --git a/clang/lib/Format/FormatToken.cpp b/clang/lib/Format/FormatToken.cpp
index 7d792974cd577..4bc865b043fd2 100644
--- a/clang/lib/Format/FormatToken.cpp
+++ b/clang/lib/Format/FormatToken.cpp
@@ -85,8 +85,8 @@ unsigned CommaSeparatedList::formatAfterToken(LineState &State,
   const FormatToken *LBrace =
       State.NextToken->Previous->getPreviousNonComment();
   if (!LBrace || !LBrace->isOneOf(tok::l_brace, TT_ArrayInitializerLSquare) ||
-      LBrace->BlockKind == BK_Block || LBrace->getType() == TT_DictLiteral ||
-      LBrace->Next->getType() == TT_DesignatedInitializerPeriod)
+      LBrace->is(BK_Block) || LBrace->is(TT_DictLiteral) ||
+      LBrace->Next->is(TT_DesignatedInitializerPeriod))
     return 0;
 
   // Calculate the number of code points we have to format this list. As the
diff --git a/clang/lib/Format/FormatToken.h b/clang/lib/Format/FormatToken.h
index d4287f53fde37..ece1bf4b97f7b 100644
--- a/clang/lib/Format/FormatToken.h
+++ b/clang/lib/Format/FormatToken.h
@@ -140,80 +140,156 @@ class AnnotatedLine;
 /// A wrapper around a \c Token storing information about the
 /// whitespace characters preceding it.
 struct FormatToken {
-  FormatToken() {}
+  FormatToken()
+      : HasUnescapedNewline(false), IsMultiline(false), IsFirst(false),
+        MustBreakBefore(false), MustBreakAlignBefore(false),
+        IsUnterminatedLiteral(false), CanBreakBefore(false),
+        ClosesTemplateDeclaration(false), StartsBinaryExpression(false),
+        EndsBinaryExpression(false), PartOfMultiVariableDeclStmt(false),
+        ContinuesLineCommentSection(false), Finalized(false),
+        BlockKind(BK_Unknown), Type(TT_Unknown), Decision(FD_Unformatted),
+        PackingKind(PPK_Inconclusive) {}
 
   /// The \c Token.
   Token Tok;
 
-  /// The number of newlines immediately before the \c Token.
+  /// The raw text of the token.
   ///
-  /// This can be used to determine what the user wrote in the original code
-  /// and thereby e.g. leave an empty line between two function definitions.
-  unsigned NewlinesBefore = 0;
+  /// Contains the raw token text without leading whitespace and without leading
+  /// escaped newlines.
+  StringRef TokenText;
 
-  /// Whether there is at least one unescaped newline before the \c
-  /// Token.
-  bool HasUnescapedNewline = false;
+  /// A token can have a special role that can carry extra information
+  /// about the token's formatting.
+  std::unique_ptr<TokenRole> Role;
 
   /// The range of the whitespace immediately preceding the \c Token.
   SourceRange WhitespaceRange;
 
-  /// The offset just past the last '\n' in this token's leading
-  /// whitespace (relative to \c WhiteSpaceStart). 0 if there is no '\n'.
-  unsigned LastNewlineOffset = 0;
-
-  /// The width of the non-whitespace parts of the token (or its first
-  /// line for multi-line tokens) in columns.
-  /// We need this to correctly measure number of columns a token spans.
-  unsigned ColumnWidth = 0;
-
-  /// Contains the width in columns of the last line of a multi-line
-  /// token.
-  unsigned LastLineColumnWidth = 0;
+  /// Whether there is at least one unescaped newline before the \c
+  /// Token.
+  unsigned HasUnescapedNewline : 1;
 
   /// Whether the token text contains newlines (escaped or not).
-  bool IsMultiline = false;
+  unsigned IsMultiline : 1;
 
   /// Indicates that this is the first token of the file.
-  bool IsFirst = false;
+  unsigned IsFirst : 1;
 
   /// Whether there must be a line break before this token.
   ///
   /// This happens for example when a preprocessor directive ended directly
   /// before the token.
-  bool MustBreakBefore = false;
+  unsigned MustBreakBefore : 1;
 
   /// Whether to not align across this token
   ///
   /// This happens for example when a preprocessor directive ended directly
   /// before the token, but very rarely otherwise.
-  bool MustBreakAlignBefore = false;
+  unsigned MustBreakAlignBefore : 1;
 
-  /// The raw text of the token.
+  /// Set to \c true if this token is an unterminated literal.
+  unsigned IsUnterminatedLiteral : 1;
+
+  /// \c true if it is allowed to break before this token.
+  unsigned CanBreakBefore : 1;
+
+  /// \c true if this is the ">" of "template<..>".
+  unsigned ClosesTemplateDeclaration : 1;
+
+  /// \c true if this token starts a binary expression, i.e. has at least
+  /// one fake l_paren with a precedence greater than prec::Unknown.
+  unsigned StartsBinaryExpression : 1;
+  /// \c true if this token ends a binary expression.
+  unsigned EndsBinaryExpression : 1;
+
+  /// Is this token part of a \c DeclStmt defining multiple variables?
   ///
-  /// Contains the raw token text without leading whitespace and without leading
-  /// escaped newlines.
-  StringRef TokenText;
+  /// Only set if \c Type == \c TT_StartOfName.
+  unsigned PartOfMultiVariableDeclStmt : 1;
 
-  /// Set to \c true if this token is an unterminated literal.
-  bool IsUnterminatedLiteral = 0;
+  /// Does this line comment continue a line comment section?
+  ///
+  /// Only set to true if \c Type == \c TT_LineComment.
+  unsigned ContinuesLineCommentSection : 1;
 
+  /// If \c true, this token has been fully formatted (indented and
+  /// potentially re-formatted inside), and we do not allow further formatting
+  /// changes.
+  unsigned Finalized : 1;
+
+private:
   /// Contains the kind of block if this token is a brace.
-  BraceBlockKind BlockKind = BK_Unknown;
+  unsigned BlockKind : 2;
 
+public:
+  BraceBlockKind getBlockKind() const {
+    return static_cast<BraceBlockKind>(BlockKind);
+  }
+  void setBlockKind(BraceBlockKind BBK) {
+    BlockKind = BBK;
+    assert(getBlockKind() == BBK && "BraceBlockKind overflow!");
+  }
+
+private:
+  unsigned Type : 8;
+
+public:
   /// Returns the token's type, e.g. whether "<" is a template opener or
   /// binary operator.
-  TokenType getType() const { return Type; }
-  void setType(TokenType T) { Type = T; }
+  TokenType getType() const { return static_cast<TokenType>(Type); }
+  void setType(TokenType T) {
+    Type = T;
+    assert(getType() == T && "TokenType overflow!");
+  }
 
-  /// The number of spaces that should be inserted before this token.
-  unsigned SpacesRequiredBefore = 0;
+private:
+  /// Stores the formatting decision for the token once it was made.
+  unsigned Decision : 2;
 
-  /// \c true if it is allowed to break before this token.
-  bool CanBreakBefore = false;
+public:
+  FormatDecision getDecision() const {
+    return static_cast<FormatDecision>(Decision);
+  }
+  void setDecision(FormatDecision D) {
+    Decision = D;
+    assert(getDecision() == D && "FormatDecision overflow!");
+  }
 
-  /// \c true if this is the ">" of "template<..>".
-  bool ClosesTemplateDeclaration = false;
+private:
+  /// If this is an opening parenthesis, how are the parameters packed?
+  unsigned PackingKind : 2;
+
+public:
+  ParameterPackingKind getPackingKind() const {
+    return static_cast<ParameterPackingKind>(PackingKind);
+  }
+  void setPackingKind(ParameterPackingKind K) {
+    PackingKind = K;
+    assert(getPackingKind() == K && "ParameterPackingKind overflow!");
+  }
+
+  /// The number of newlines immediately before the \c Token.
+  ///
+  /// This can be used to determine what the user wrote in the original code
+  /// and thereby e.g. leave an empty line between two function definitions.
+  unsigned NewlinesBefore = 0;
+
+  /// The offset just past the last '\n' in this token's leading
+  /// whitespace (relative to \c WhiteSpaceStart). 0 if there is no '\n'.
+  unsigned LastNewlineOffset = 0;
+
+  /// The width of the non-whitespace parts of the token (or its first
+  /// line for multi-line tokens) in columns.
+  /// We need this to correctly measure number of columns a token spans.
+  unsigned ColumnWidth = 0;
+
+  /// Contains the width in columns of the last line of a multi-line
+  /// token.
+  unsigned LastLineColumnWidth = 0;
+
+  /// The number of spaces that should be inserted before this token.
+  unsigned SpacesRequiredBefore = 0;
 
   /// Number of parameters, if this is "(", "[" or "<".
   unsigned ParameterCount = 0;
@@ -226,13 +302,6 @@ struct FormatToken {
   /// the surrounding bracket.
   tok::TokenKind ParentBracket = tok::unknown;
 
-  /// A token can have a special role that can carry extra information
-  /// about the token's formatting.
-  std::unique_ptr<TokenRole> Role;
-
-  /// If this is an opening parenthesis, how are the parameters packed?
-  ParameterPackingKind PackingKind = PPK_Inconclusive;
-
   /// The total length of the unwrapped line up to and including this
   /// token.
   unsigned TotalLength = 0;
@@ -286,12 +355,6 @@ struct FormatToken {
   /// Insert this many fake ) after this token for correct indentation.
   unsigned FakeRParens = 0;
 
-  /// \c true if this token starts a binary expression, i.e. has at least
-  /// one fake l_paren with a precedence greater than prec::Unknown.
-  bool StartsBinaryExpression = false;
-  /// \c true if this token ends a binary expression.
-  bool EndsBinaryExpression = false;
-
   /// If this is an operator (or "."/"->") in a sequence of operators
   /// with the same precedence, contains the 0-based operator index.
   unsigned OperatorIndex = 0;
@@ -300,16 +363,6 @@ struct FormatToken {
   /// with the same precedence, points to the next operator.
   FormatToken *NextOperator = nullptr;
 
-  /// Is this token part of a \c DeclStmt defining multiple variables?
-  ///
-  /// Only set if \c Type == \c TT_StartOfName.
-  bool PartOfMultiVariableDeclStmt = false;
-
-  /// Does this line comment continue a line comment section?
-  ///
-  /// Only set to true if \c Type == \c TT_LineComment.
-  bool ContinuesLineCommentSection = false;
-
   /// If this is a bracket, this points to the matching one.
   FormatToken *MatchingParen = nullptr;
 
@@ -323,16 +376,8 @@ struct FormatToken {
   /// in it.
   SmallVector<AnnotatedLine *, 1> Children;
 
-  /// Stores the formatting decision for the token once it was made.
-  FormatDecision Decision = FD_Unformatted;
-
-  /// If \c true, this token has been fully formatted (indented and
-  /// potentially re-formatted inside), and we do not allow further formatting
-  /// changes.
-  bool Finalized = false;
-
   bool is(tok::TokenKind Kind) const { return Tok.is(Kind); }
-  bool is(TokenType TT) const { return Type == TT; }
+  bool is(TokenType TT) const { return getType() == TT; }
   bool is(const IdentifierInfo *II) const {
     return II && II == Tok.getIdentifierInfo();
   }
@@ -340,6 +385,9 @@ struct FormatToken {
     return Tok.getIdentifierInfo() &&
            Tok.getIdentifierInfo()->getPPKeywordID() == Kind;
   }
+  bool is(BraceBlockKind BBK) const { return getBlockKind() == BBK; }
+  bool is(ParameterPackingKind PPK) const { return getPackingKind() == PPK; }
+
   template <typename A, typename B> bool isOneOf(A K1, B K2) const {
     return is(K1) || is(K2);
   }
@@ -355,7 +403,7 @@ struct FormatToken {
   }
 
   bool closesScopeAfterBlock() const {
-    if (BlockKind == BK_Block)
+    if (getBlockKind() == BK_Block)
       return true;
     if (closesScope())
       return Previous->closesScopeAfterBlock();
@@ -525,13 +573,13 @@ struct FormatToken {
   /// list that should be indented with a block indent.
   bool opensBlockOrBlockTypeList(const FormatStyle &Style) const {
     // C# Does not indent object initialisers as continuations.
-    if (is(tok::l_brace) && BlockKind == BK_BracedInit && Style.isCSharp())
+    if (is(tok::l_brace) && getBlockKind() == BK_BracedInit && Style.isCSharp())
       return true;
     if (is(TT_TemplateString) && opensScope())
       return true;
     return is(TT_ArrayInitializerLSquare) || is(TT_ProtoExtensionLSquare) ||
            (is(tok::l_brace) &&
-            (BlockKind == BK_Block || is(TT_DictLiteral) ||
+            (getBlockKind() == BK_Block || is(TT_DictLiteral) ||
              (!Style.Cpp11BracedListStyle && NestingLevel == 0))) ||
            (is(tok::less) && (Style.Language == FormatStyle::LK_Proto ||
                               Style.Language == FormatStyle::LK_TextProto));
@@ -602,8 +650,6 @@ struct FormatToken {
       return Previous->endsSequenceInternal(K1, Tokens...);
     return is(K1) && Previous && Previous->endsSequenceInternal(Tokens...);
   }
-
-  TokenType Type = TT_Unknown;
 };
 
 class ContinuationIndenter;
diff --git a/clang/lib/Format/TokenAnnotator.cpp b/clang/lib/Format/TokenAnnotator.cpp
index 0ab09b4a1218a..b19fc34bcc802 100644
--- a/clang/lib/Format/TokenAnnotator.cpp
+++ b/clang/lib/Format/TokenAnnotator.cpp
@@ -343,11 +343,11 @@ class AnnotatingParser {
           CurrentToken->setType(TT_AttributeSquare);
 
         if (!HasMultipleLines)
-          Left->PackingKind = PPK_Inconclusive;
+          Left->setPackingKind(PPK_Inconclusive);
         else if (HasMultipleParametersOnALine)
-          Left->PackingKind = PPK_BinPacked;
+          Left->setPackingKind(PPK_BinPacked);
         else
-          Left->PackingKind = PPK_OnePerLine;
+          Left->setPackingKind(PPK_OnePerLine);
 
         next();
         return true;
@@ -704,7 +704,7 @@ class AnnotatingParser {
 
       ScopedContextCreator ContextCreator(*this, tok::l_brace, 1);
       Contexts.back().ColonIsDictLiteral = true;
-      if (Left->BlockKind == BK_BracedInit)
+      if (Left->is(BK_BracedInit))
         Contexts.back().IsExpression = true;
       if (Style.Language == FormatStyle::LK_JavaScript && Left->Previous &&
           Left->Previous->is(TT_JsTypeColon))
@@ -751,7 +751,7 @@ class AnnotatingParser {
     // For ObjC methods, the number of parameters is calculated differently as
     // method declarations have a different structure (the parameters are not
     // inside a bracket scope).
-    if (Current->is(tok::l_brace) && Current->BlockKind == BK_Block)
+    if (Current->is(tok::l_brace) && Current->is(BK_Block))
       ++Left->BlockParameterCount;
     if (Current->is(tok::comma)) {
       ++Left->ParameterCount;
@@ -2420,7 +2420,7 @@ void TokenAnnotator::calculateFormattingInformation(AnnotatedLine &Line) {
     if (isFunctionDeclarationName(*Current, Line))
       Current->setType(TT_FunctionDeclarationName);
     if (Current->is(TT_LineComment)) {
-      if (Current->Previous->BlockKind == BK_BracedInit &&
+      if (Current->Previous->is(BK_BracedInit) &&
           Current->Previous->opensScope())
         Current->SpacesRequiredBefore =
             (Style.Cpp11BracedListStyle && !Style.SpacesInParentheses) ? 0 : 1;
@@ -2755,8 +2755,8 @@ bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line,
   if (Left.isOneOf(tok::hashhash, tok::hash))
     return Right.is(tok::hash);
   if ((Left.is(tok::l_paren) && Right.is(tok::r_paren)) ||
-      (Left.is(tok::l_brace) && Left.BlockKind != BK_Block &&
-       Right.is(tok::r_brace) && Right.BlockKind != BK_Block))
+      (Left.is(tok::l_brace) && Left.isNot(BK_Block) &&
+       Right.is(tok::r_brace) && Right.isNot(BK_Block)))
     return Style.SpaceInEmptyParentheses;
   if (Style.SpacesInConditionalStatement) {
     if (Left.is(tok::l_paren) && Left.Previous &&
@@ -2836,7 +2836,7 @@ bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line,
     return Right.Tok.isLiteral() || Right.is(TT_BlockComment) ||
            (Right.isOneOf(Keywords.kw_override, Keywords.kw_final) &&
             !Right.is(TT_StartOfName)) ||
-           (Right.is(tok::l_brace) && Right.BlockKind == BK_Block) ||
+           (Right.is(tok::l_brace) && Right.is(BK_Block)) ||
            (!Right.isOneOf(TT_PointerOrReference, TT_ArraySubscriptLSquare,
                            tok::l_paren) &&
             (Style.PointerAlignment != FormatStyle::PAS_Right &&
@@ -2921,9 +2921,9 @@ bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line,
     return false;
   if (Left.is(tok::l_brace) && Right.is(tok::r_brace))
     return !Left.Children.empty(); // No spaces in "{}".
-  if ((Left.is(tok::l_brace) && Left.BlockKind != BK_Block) ||
+  if ((Left.is(tok::l_brace) && Left.isNot(BK_Block)) ||
       (Right.is(tok::r_brace) && Right.MatchingParen &&
-       Right.MatchingParen->BlockKind != BK_Block))
+       Right.MatchingParen->isNot(BK_Block)))
     return Style.Cpp11BracedListStyle ? Style.SpacesInParentheses : true;
   if (Left.is(TT_BlockComment))
     // No whitespace in x(/*foo=*/1), except for JavaScript.
@@ -2967,7 +2967,7 @@ bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line,
                     tok::r_paren) ||
        Left.isSimpleTypeSpecifier()) &&
       Right.is(tok::l_brace) && Right.getNextNonComment() &&
-      Right.BlockKind != BK_Block)
+      Right.isNot(BK_Block))
     return false;
   if (Left.is(tok::period) || Right.is(tok::period))
     return false;
@@ -3009,7 +3009,7 @@ bool TokenAnnotator::spaceRequiredBefore(const AnnotatedLine &Line,
   if (Style.isCpp()) {
     if (Left.is(tok::kw_operator))
       return Right.is(tok::coloncolon);
-    if (Right.is(tok::l_brace) && Right.BlockKind == BK_BracedInit &&
+    if (Right.is(tok::l_brace) && Right.is(BK_BracedInit) &&
         !Left.opensScope() && Style.SpaceBeforeCpp11BracedList)
       return true;
   } else if (Style.Language == FormatStyle::LK_Proto ||
@@ -3362,7 +3362,7 @@ bool TokenAnnotator::spaceRequiredBefore(const AnnotatedLine &Line,
 
 // Returns 'true' if 'Tok' is a brace we'd want to break before in Allman style.
 static bool isAllmanBrace(const FormatToken &Tok) {
-  return Tok.is(tok::l_brace) && Tok.BlockKind == BK_Block &&
+  return Tok.is(tok::l_brace) && Tok.is(BK_Block) &&
          !Tok.isOneOf(TT_ObjCBlockLBrace, TT_LambdaLBrace, TT_DictLiteral);
 }
 
@@ -3398,7 +3398,7 @@ static bool isOneChildWithoutMustBreakBefore(const FormatToken &Tok) {
   return true;
 }
 static bool isAllmanLambdaBrace(const FormatToken &Tok) {
-  return (Tok.is(tok::l_brace) && Tok.BlockKind == BK_Block &&
+  return (Tok.is(tok::l_brace) && Tok.is(BK_Block) &&
           !Tok.isOneOf(TT_ObjCBlockLBrace, TT_DictLiteral));
 }
 
@@ -3498,7 +3498,7 @@ bool TokenAnnotator::mustBreakBefore(const AnnotatedLine &Line,
     if ((Left.isOneOf(tok::l_brace, TT_ArrayInitializerLSquare) ||
          (Style.Language == FormatStyle::LK_JavaScript &&
           Left.is(tok::l_paren))) &&
-        Left.BlockKind != BK_Block && Left.MatchingParen)
+        Left.isNot(BK_Block) && Left.MatchingParen)
       BeforeClosingBrace = Left.MatchingParen->Previous;
     else if (Right.MatchingParen &&
              (Right.MatchingParen->isOneOf(tok::l_brace,
@@ -3512,8 +3512,7 @@ bool TokenAnnotator::mustBreakBefore(const AnnotatedLine &Line,
   }
 
   if (Right.is(tok::comment))
-    return Left.BlockKind != BK_BracedInit &&
-           Left.isNot(TT_CtorInitializerColon) &&
+    return Left.isNot(BK_BracedInit) && Left.isNot(TT_CtorInitializerColon) &&
            (Right.NewlinesBefore > 0 && Right.HasUnescapedNewline);
   if (Left.isTrailingComment())
     return true;
@@ -3822,7 +3821,7 @@ bool TokenAnnotator::canBreakBefore(const AnnotatedLine &Line,
     // The first comment in a braced lists is always interpreted as belonging to
     // the first list element. Otherwise, it should be placed outside of the
     // list.
-    return Left.BlockKind == BK_BracedInit ||
+    return Left.is(BK_BracedInit) ||
            (Left.is(TT_CtorInitializerColon) &&
             Style.BreakConstructorInitializers == FormatStyle::BCIS_AfterColon);
   if (Left.is(tok::question) && Right.is(tok::colon))
@@ -3923,7 +3922,7 @@ bool TokenAnnotator::canBreakBefore(const AnnotatedLine &Line,
   // We only break before r_brace if there was a corresponding break before
   // the l_brace, which is tracked by BreakBeforeClosingBrace.
   if (Right.is(tok::r_brace))
-    return Right.MatchingParen && Right.MatchingParen->BlockKind == BK_Block;
+    return Right.MatchingParen && Right.MatchingParen->is(BK_Block);
 
   // Allow breaking after a trailing annotation, e.g. after a method
   // declaration.
@@ -4008,9 +4007,9 @@ void TokenAnnotator::printDebugInfo(const AnnotatedLine &Line) {
                  << " T=" << getTokenTypeName(Tok->getType())
                  << " S=" << Tok->SpacesRequiredBefore
                  << " F=" << Tok->Finalized << " B=" << Tok->BlockParameterCount
-                 << " BK=" << Tok->BlockKind << " P=" << Tok->SplitPenalty
+                 << " BK=" << Tok->getBlockKind() << " P=" << Tok->SplitPenalty
                  << " Name=" << Tok->Tok.getName() << " L=" << Tok->TotalLength
-                 << " PPK=" << Tok->PackingKind << " FakeLParens=";
+                 << " PPK=" << Tok->getPackingKind() << " FakeLParens=";
     for (unsigned i = 0, e = Tok->FakeLParens.size(); i != e; ++i)
       llvm::errs() << Tok->FakeLParens[i] << "/";
     llvm::errs() << " FakeRParens=" << Tok->FakeRParens;
diff --git a/clang/lib/Format/UnwrappedLineFormatter.cpp b/clang/lib/Format/UnwrappedLineFormatter.cpp
index 22f27a668dccd..ec0c628fe750d 100644
--- a/clang/lib/Format/UnwrappedLineFormatter.cpp
+++ b/clang/lib/Format/UnwrappedLineFormatter.cpp
@@ -606,7 +606,7 @@ class LineJoiner {
         if (I[1]->Last->is(TT_LineComment))
           return 0;
         do {
-          if (Tok->is(tok::l_brace) && Tok->BlockKind != BK_BracedInit)
+          if (Tok->is(tok::l_brace) && Tok->isNot(BK_BracedInit))
             return 0;
           Tok = Tok->Next;
         } while (Tok);
@@ -767,8 +767,8 @@ class LineFormatter {
                       unsigned &Penalty) {
     const FormatToken *LBrace = State.NextToken->getPreviousNonComment();
     FormatToken &Previous = *State.NextToken->Previous;
-    if (!LBrace || LBrace->isNot(tok::l_brace) ||
-        LBrace->BlockKind != BK_Block || Previous.Children.size() == 0)
+    if (!LBrace || LBrace->isNot(tok::l_brace) || LBrace->isNot(BK_Block) ||
+        Previous.Children.size() == 0)
       // The previous token does not open a block. Nothing to do. We don't
       // assert so that we can simply call this function for all tokens.
       return true;
@@ -979,7 +979,7 @@ class OptimizingLineFormatter : public LineFormatter {
         // State already examined with lower penalty.
         continue;
 
-      FormatDecision LastFormat = Node->State.NextToken->Decision;
+      FormatDecision LastFormat = Node->State.NextToken->getDecision();
       if (LastFormat == FD_Unformatted || LastFormat == FD_Continue)
         addNextStateToQueue(Penalty, Node, /*NewLine=*/false, &Count, &Queue);
       if (LastFormat == FD_Unformatted || LastFormat == FD_Break)
diff --git a/clang/lib/Format/UnwrappedLineParser.cpp b/clang/lib/Format/UnwrappedLineParser.cpp
index a37386425aaed..b599168b48e17 100644
--- a/clang/lib/Format/UnwrappedLineParser.cpp
+++ b/clang/lib/Format/UnwrappedLineParser.cpp
@@ -472,19 +472,19 @@ void UnwrappedLineParser::calculateBraceTypes(bool ExpectClassBody) {
           // individual members in a type member list, which would normally
           // trigger BK_Block. In both cases, this must be parsed as an inline
           // braced init.
-          Tok->BlockKind = BK_BracedInit;
+          Tok->setBlockKind(BK_BracedInit);
         else if (PrevTok->is(tok::r_paren))
           // `) { }` can only occur in function or method declarations in JS.
-          Tok->BlockKind = BK_Block;
+          Tok->setBlockKind(BK_Block);
       } else {
-        Tok->BlockKind = BK_Unknown;
+        Tok->setBlockKind(BK_Unknown);
       }
       LBraceStack.push_back(Tok);
       break;
     case tok::r_brace:
       if (LBraceStack.empty())
         break;
-      if (LBraceStack.back()->BlockKind == BK_Unknown) {
+      if (LBraceStack.back()->is(BK_Unknown)) {
         bool ProbablyBracedList = false;
         if (Style.Language == FormatStyle::LK_Proto) {
           ProbablyBracedList = NextTok->isOneOf(tok::comma, tok::r_square);
@@ -524,11 +524,11 @@ void UnwrappedLineParser::calculateBraceTypes(bool ExpectClassBody) {
           }
         }
         if (ProbablyBracedList) {
-          Tok->BlockKind = BK_BracedInit;
-          LBraceStack.back()->BlockKind = BK_BracedInit;
+          Tok->setBlockKind(BK_BracedInit);
+          LBraceStack.back()->setBlockKind(BK_BracedInit);
         } else {
-          Tok->BlockKind = BK_Block;
-          LBraceStack.back()->BlockKind = BK_Block;
+          Tok->setBlockKind(BK_Block);
+          LBraceStack.back()->setBlockKind(BK_Block);
         }
       }
       LBraceStack.pop_back();
@@ -545,8 +545,8 @@ void UnwrappedLineParser::calculateBraceTypes(bool ExpectClassBody) {
     case tok::kw_switch:
     case tok::kw_try:
     case tok::kw___try:
-      if (!LBraceStack.empty() && LBraceStack.back()->BlockKind == BK_Unknown)
-        LBraceStack.back()->BlockKind = BK_Block;
+      if (!LBraceStack.empty() && LBraceStack.back()->is(BK_Unknown))
+        LBraceStack.back()->setBlockKind(BK_Block);
       break;
     default:
       break;
@@ -557,8 +557,8 @@ void UnwrappedLineParser::calculateBraceTypes(bool ExpectClassBody) {
 
   // Assume other blocks for all unclosed opening braces.
   for (unsigned i = 0, e = LBraceStack.size(); i != e; ++i) {
-    if (LBraceStack[i]->BlockKind == BK_Unknown)
-      LBraceStack[i]->BlockKind = BK_Block;
+    if (LBraceStack[i]->is(BK_Unknown))
+      LBraceStack[i]->setBlockKind(BK_Block);
   }
 
   FormatTok = Tokens->setPosition(StoredPosition);
@@ -584,7 +584,7 @@ void UnwrappedLineParser::parseBlock(bool MustBeDeclaration, bool AddLevel,
   assert(FormatTok->isOneOf(tok::l_brace, TT_MacroBlockBegin) &&
          "'{' or macro block token expected");
   const bool MacroBlock = FormatTok->is(TT_MacroBlockBegin);
-  FormatTok->BlockKind = BK_Block;
+  FormatTok->setBlockKind(BK_Block);
 
   size_t PPStartHash = computePPHash();
 
@@ -614,7 +614,7 @@ void UnwrappedLineParser::parseBlock(bool MustBeDeclaration, bool AddLevel,
   if (MacroBlock ? !FormatTok->is(TT_MacroBlockEnd)
                  : !FormatTok->is(tok::r_brace)) {
     Line->Level = InitialLevel;
-    FormatTok->BlockKind = BK_Block;
+    FormatTok->setBlockKind(BK_Block);
     return;
   }
 
@@ -690,7 +690,7 @@ static bool ShouldBreakBeforeBrace(const FormatStyle &Style,
 }
 
 void UnwrappedLineParser::parseChildBlock() {
-  FormatTok->BlockKind = BK_Block;
+  FormatTok->setBlockKind(BK_Block);
   nextToken();
   {
     bool SkipIndent = (Style.Language == FormatStyle::LK_JavaScript &&
@@ -1476,7 +1476,7 @@ void UnwrappedLineParser::parseStructuralElement() {
         // C# needs this change to ensure that array initialisers and object
         // initialisers are indented the same way.
         if (Style.isCSharp())
-          FormatTok->BlockKind = BK_BracedInit;
+          FormatTok->setBlockKind(BK_BracedInit);
         nextToken();
         parseBracedList();
       } else if (Style.Language == FormatStyle::LK_Proto &&
@@ -1747,10 +1747,10 @@ void UnwrappedLineParser::tryToParseJSFunction() {
 }
 
 bool UnwrappedLineParser::tryToParseBracedList() {
-  if (FormatTok->BlockKind == BK_Unknown)
+  if (FormatTok->is(BK_Unknown))
     calculateBraceTypes();
-  assert(FormatTok->BlockKind != BK_Unknown);
-  if (FormatTok->BlockKind == BK_Block)
+  assert(FormatTok->isNot(BK_Unknown));
+  if (FormatTok->is(BK_Block))
     return false;
   nextToken();
   parseBracedList();
@@ -1830,7 +1830,7 @@ bool UnwrappedLineParser::parseBracedList(bool ContinueOnSemicolons,
     case tok::l_brace:
       // Assume there are no blocks inside a braced init list apart
       // from the ones we explicitly parse out (like lambdas).
-      FormatTok->BlockKind = BK_BracedInit;
+      FormatTok->setBlockKind(BK_BracedInit);
       nextToken();
       parseBracedList();
       break;
@@ -2318,7 +2318,7 @@ bool UnwrappedLineParser::parseEnum() {
   // Just a declaration or something is wrong.
   if (FormatTok->isNot(tok::l_brace))
     return true;
-  FormatTok->BlockKind = BK_Block;
+  FormatTok->setBlockKind(BK_Block);
 
   if (Style.Language == FormatStyle::LK_Java) {
     // Java enums are different.
@@ -2726,7 +2726,7 @@ void UnwrappedLineParser::parseJavaScriptEs6ImportExport() {
       return;
     }
     if (FormatTok->is(tok::l_brace)) {
-      FormatTok->BlockKind = BK_Block;
+      FormatTok->setBlockKind(BK_Block);
       nextToken();
       parseBracedList();
     } else {
diff --git a/clang/lib/Format/WhitespaceManager.cpp b/clang/lib/Format/WhitespaceManager.cpp
index 32e0b685ea0f4..2d479817118db 100644
--- a/clang/lib/Format/WhitespaceManager.cpp
+++ b/clang/lib/Format/WhitespaceManager.cpp
@@ -49,7 +49,7 @@ void WhitespaceManager::replaceWhitespace(FormatToken &Tok, unsigned Newlines,
                                           bool IsAligned, bool InPPDirective) {
   if (Tok.Finalized)
     return;
-  Tok.Decision = (Newlines > 0) ? FD_Break : FD_Continue;
+  Tok.setDecision((Newlines > 0) ? FD_Break : FD_Continue);
   Changes.push_back(Change(Tok, /*CreateReplacement=*/true, Tok.WhitespaceRange,
                            Spaces, StartOfTokenColumn, Newlines, "", "",
                            IsAligned, InPPDirective && !Tok.IsFirst,

From eb10b065f2a870b425dcc2040b9955e0eee464b4 Mon Sep 17 00:00:00 2001
From: Bruno Ricci <riccibrun@gmail.com>
Date: Mon, 27 Jul 2020 23:22:21 +0100
Subject: [PATCH 0279/1035] [clang] Pass the NamedDecl* instead of the
 DeclarationName into many diagnostics.

Background:
-----------
There are two related argument types which can be sent into a diagnostic to
display the name of an entity: DeclarationName (ak_declarationname) or
NamedDecl* (ak_nameddecl) (there is also ak_identifierinfo for
IdentifierInfo*, but we are not concerned with it here).

A DeclarationName in a diagnostic will just be streamed to the output,
which will directly result in a call to DeclarationName::print.

A NamedDecl* in a diagnostic will also ultimately result in a call to
DeclarationName::print, but with two customisation points along the way:

The first customisation point is NamedDecl::getNameForDiagnostic which is
overloaded by FunctionDecl, ClassTemplateSpecializationDecl and
VarTemplateSpecializationDecl to print the template arguments, if any.

The second customisation point is NamedDecl::printName. By default it just
streams the stored DeclarationName into the output but it can be customised
to provide a user-friendly name for an entity. It is currently overloaded by
DecompositionDecl and MSGuidDecl.

What this patch does:
---------------------
For many diagnostics a DeclarationName is used instead of the NamedDecl*.
This bypasses the two customisation points mentioned above. This patches fix
this for diagnostics in Sema.cpp, SemaCast.cpp, SemaChecking.cpp, SemaDecl.cpp,
SemaDeclAttr.cpp, SemaDecl.cpp, SemaOverload.cpp and SemaStmt.cpp.

I have only modified diagnostics where I could construct a test-case which
demonstrates that the change is appropriate (either with this patch or the next
one).

Reviewed By: erichkeane, aaron.ballman

Differential Revision: https://reviews.llvm.org/D84656
---
 clang/lib/Sema/Sema.cpp                       | 23 +++++------
 clang/lib/Sema/SemaCast.cpp                   |  6 +--
 clang/lib/Sema/SemaChecking.cpp               |  3 +-
 clang/lib/Sema/SemaDecl.cpp                   | 37 +++++++++--------
 clang/lib/Sema/SemaDeclAttr.cpp               |  4 +-
 clang/lib/Sema/SemaExpr.cpp                   | 40 +++++++------------
 clang/lib/Sema/SemaOverload.cpp               |  4 +-
 clang/lib/Sema/SemaStmt.cpp                   | 16 +++-----
 .../dcl.dcl/dcl.attr/dcl.attr.noreturn/p1.cpp |  5 ++-
 .../CXX/dcl.decl/dcl.meaning/dcl.fct/p3.cpp   |  4 ++
 .../stmt.stmt/stmt.iter/stmt.ranged/p1.cpp    | 15 +++++++
 clang/test/CXX/temp/temp.param/p15-cxx0x.cpp  |  6 +--
 clang/test/Modules/module-private.cpp         | 10 +++--
 clang/test/SemaCXX/array-bounds.cpp           | 13 +++++-
 clang/test/SemaCXX/attr-unused.cpp            |  2 +-
 .../cxx1y-variable-templates_in_class.cpp     |  2 +-
 clang/test/SemaCXX/default2.cpp               |  2 +-
 clang/test/SemaCXX/incomplete-call.cpp        |  2 +-
 clang/test/SemaCXX/references.cpp             | 11 ++++-
 clang/test/SemaCXX/return-void.cpp            |  2 +-
 clang/test/SemaCXX/warn-func-not-needed.cpp   |  2 +-
 .../test/SemaCXX/warn-large-by-value-copy.cpp |  8 ++++
 clang/test/SemaCXX/warn-member-not-needed.cpp |  4 +-
 .../warn-pure-virtual-call-from-ctor-dtor.cpp |  8 ++--
 clang/test/SemaCXX/warn-pure-virtual-kext.cpp |  2 +-
 clang/test/SemaCXX/warn-unused-filescoped.cpp |  8 ++--
 .../test/SemaCXX/warn-variable-not-needed.cpp |  2 +-
 27 files changed, 137 insertions(+), 104 deletions(-)

diff --git a/clang/lib/Sema/Sema.cpp b/clang/lib/Sema/Sema.cpp
index 9c8f3fdcda4a6..7415d0d0766ba 100644
--- a/clang/lib/Sema/Sema.cpp
+++ b/clang/lib/Sema/Sema.cpp
@@ -1197,7 +1197,7 @@ void Sema::ActOnEndOfTranslationUnit() {
         if (DiagD->isReferenced()) {
           if (isa<CXXMethodDecl>(DiagD))
             Diag(DiagD->getLocation(), diag::warn_unneeded_member_function)
-                  << DiagD->getDeclName();
+                << DiagD;
           else {
             if (FD->getStorageClass() == SC_Static &&
                 !FD->isInlineSpecified() &&
@@ -1205,20 +1205,20 @@ void Sema::ActOnEndOfTranslationUnit() {
                    SourceMgr.getExpansionLoc(FD->getLocation())))
               Diag(DiagD->getLocation(),
                    diag::warn_unneeded_static_internal_decl)
-                  << DiagD->getDeclName();
+                  << DiagD;
             else
               Diag(DiagD->getLocation(), diag::warn_unneeded_internal_decl)
-                   << /*function*/0 << DiagD->getDeclName();
+                  << /*function*/ 0 << DiagD;
           }
         } else {
           if (FD->getDescribedFunctionTemplate())
             Diag(DiagD->getLocation(), diag::warn_unused_template)
-              << /*function*/0 << DiagD->getDeclName();
+                << /*function*/ 0 << DiagD;
           else
-            Diag(DiagD->getLocation(),
-                 isa<CXXMethodDecl>(DiagD) ? diag::warn_unused_member_function
+            Diag(DiagD->getLocation(), isa<CXXMethodDecl>(DiagD)
+                                           ? diag::warn_unused_member_function
                                            : diag::warn_unused_function)
-              << DiagD->getDeclName();
+                << DiagD;
         }
       } else {
         const VarDecl *DiagD = cast<VarDecl>(*I)->getDefinition();
@@ -1226,20 +1226,19 @@ void Sema::ActOnEndOfTranslationUnit() {
           DiagD = cast<VarDecl>(*I);
         if (DiagD->isReferenced()) {
           Diag(DiagD->getLocation(), diag::warn_unneeded_internal_decl)
-                << /*variable*/1 << DiagD->getDeclName();
+              << /*variable*/ 1 << DiagD;
         } else if (DiagD->getType().isConstQualified()) {
           const SourceManager &SM = SourceMgr;
           if (SM.getMainFileID() != SM.getFileID(DiagD->getLocation()) ||
               !PP.getLangOpts().IsHeaderFile)
             Diag(DiagD->getLocation(), diag::warn_unused_const_variable)
-                << DiagD->getDeclName();
+                << DiagD;
         } else {
           if (DiagD->getDescribedVarTemplate())
             Diag(DiagD->getLocation(), diag::warn_unused_template)
-              << /*variable*/1 << DiagD->getDeclName();
+                << /*variable*/ 1 << DiagD;
           else
-            Diag(DiagD->getLocation(), diag::warn_unused_variable)
-              << DiagD->getDeclName();
+            Diag(DiagD->getLocation(), diag::warn_unused_variable) << DiagD;
         }
       }
     }
diff --git a/clang/lib/Sema/SemaCast.cpp b/clang/lib/Sema/SemaCast.cpp
index 2efe26052c78c..58cf3a1be7301 100644
--- a/clang/lib/Sema/SemaCast.cpp
+++ b/clang/lib/Sema/SemaCast.cpp
@@ -510,12 +510,10 @@ static void diagnoseBadCast(Sema &S, unsigned msg, CastType castType,
     if (RecFrom && RecTo) {
       auto DeclFrom = RecFrom->getAsCXXRecordDecl();
       if (!DeclFrom->isCompleteDefinition())
-        S.Diag(DeclFrom->getLocation(), diag::note_type_incomplete)
-          << DeclFrom->getDeclName();
+        S.Diag(DeclFrom->getLocation(), diag::note_type_incomplete) << DeclFrom;
       auto DeclTo = RecTo->getAsCXXRecordDecl();
       if (!DeclTo->isCompleteDefinition())
-        S.Diag(DeclTo->getLocation(), diag::note_type_incomplete)
-          << DeclTo->getDeclName();
+        S.Diag(DeclTo->getLocation(), diag::note_type_incomplete) << DeclTo;
     }
   }
 }
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index f445272b020bf..77d5f3ff816ed 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -13962,8 +13962,7 @@ void Sema::CheckArrayAccess(const Expr *BaseExpr, const Expr *IndexExpr,
 
   if (ND)
     DiagRuntimeBehavior(ND->getBeginLoc(), BaseExpr,
-                        PDiag(diag::note_array_declared_here)
-                            << ND->getDeclName());
+                        PDiag(diag::note_array_declared_here) << ND);
 }
 
 void Sema::CheckArrayAccess(const Expr *expr) {
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index 869e4de02cc41..ddbf086ea6385 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -1926,7 +1926,7 @@ static void CheckPoppedLabel(LabelDecl *L, Sema &S) {
   else
     Diagnose = L->getStmt() == nullptr;
   if (Diagnose)
-    S.Diag(L->getLocation(), diag::err_undeclared_label_use) <<L->getDeclName();
+    S.Diag(L->getLocation(), diag::err_undeclared_label_use) << L;
 }
 
 void Sema::ActOnPopScope(SourceLocation Loc, Scope *S) {
@@ -7194,9 +7194,10 @@ NamedDecl *Sema::ActOnVariableDeclarator(
         << FixItHint::CreateRemoval(D.getDeclSpec().getModulePrivateSpecLoc());
     else if (NewVD->hasLocalStorage())
       Diag(NewVD->getLocation(), diag::err_module_private_local)
-        << 0 << NewVD->getDeclName()
-        << SourceRange(D.getDeclSpec().getModulePrivateSpecLoc())
-        << FixItHint::CreateRemoval(D.getDeclSpec().getModulePrivateSpecLoc());
+          << 0 << NewVD
+          << SourceRange(D.getDeclSpec().getModulePrivateSpecLoc())
+          << FixItHint::CreateRemoval(
+                 D.getDeclSpec().getModulePrivateSpecLoc());
     else {
       NewVD->setModulePrivate();
       if (NewTemplate)
@@ -12429,7 +12430,7 @@ void Sema::ActOnUninitializedDecl(Decl *RealDecl) {
             !Context.getTargetInfo().getCXXABI().isMicrosoft()) {
           Diag(Var->getLocation(),
                diag::err_constexpr_static_mem_var_requires_init)
-            << Var->getDeclName();
+              << Var;
           Var->setInvalidDecl();
           return;
         }
@@ -12562,8 +12563,7 @@ void Sema::ActOnUninitializedDecl(Decl *RealDecl) {
     // definitions with reference type.
     if (Type->isReferenceType()) {
       Diag(Var->getLocation(), diag::err_reference_var_requires_init)
-        << Var->getDeclName()
-        << SourceRange(Var->getLocation(), Var->getLocation());
+          << Var << SourceRange(Var->getLocation(), Var->getLocation());
       Var->setInvalidDecl();
       return;
     }
@@ -12696,7 +12696,7 @@ void Sema::ActOnCXXForRangeDecl(Decl *D) {
   }
   if (Error != -1) {
     Diag(VD->getOuterLocStart(), diag::err_for_range_storage_class)
-      << VD->getDeclName() << Error;
+        << VD << Error;
     D->setInvalidDecl();
   }
 }
@@ -13477,9 +13477,8 @@ Decl *Sema::ActOnParamDeclarator(Scope *S, Declarator &D) {
 
   if (D.getDeclSpec().isModulePrivateSpecified())
     Diag(New->getLocation(), diag::err_module_private_local)
-      << 1 << New->getDeclName()
-      << SourceRange(D.getDeclSpec().getModulePrivateSpecLoc())
-      << FixItHint::CreateRemoval(D.getDeclSpec().getModulePrivateSpecLoc());
+        << 1 << New << SourceRange(D.getDeclSpec().getModulePrivateSpecLoc())
+        << FixItHint::CreateRemoval(D.getDeclSpec().getModulePrivateSpecLoc());
 
   if (New->hasAttr<BlocksAttr>()) {
     Diag(New->getLocation(), diag::err_block_on_nonlocal);
@@ -13531,8 +13530,7 @@ void Sema::DiagnoseSizeOfParametersAndReturnValue(
   if (!ReturnTy->isDependentType() && ReturnTy.isPODType(Context)) {
     unsigned Size = Context.getTypeSizeInChars(ReturnTy).getQuantity();
     if (Size > LangOpts.NumLargeByValueCopy)
-      Diag(D->getLocation(), diag::warn_return_value_size)
-          << D->getDeclName() << Size;
+      Diag(D->getLocation(), diag::warn_return_value_size) << D << Size;
   }
 
   // Warn if any parameter is pass-by-value and larger than the specified
@@ -13544,7 +13542,7 @@ void Sema::DiagnoseSizeOfParametersAndReturnValue(
     unsigned Size = Context.getTypeSizeInChars(T).getQuantity();
     if (Size > LangOpts.NumLargeByValueCopy)
       Diag(Parameter->getLocation(), diag::warn_parameter_size)
-          << Parameter->getDeclName() << Size;
+          << Parameter << Size;
   }
 }
 
@@ -13852,9 +13850,9 @@ Sema::CheckForFunctionRedefinition(FunctionDecl *FD,
   if (getLangOpts().GNUMode && Definition->isInlineSpecified() &&
       Definition->getStorageClass() == SC_Extern)
     Diag(FD->getLocation(), diag::err_redefinition_extern_inline)
-        << FD->getDeclName() << getLangOpts().CPlusPlus;
+        << FD << getLangOpts().CPlusPlus;
   else
-    Diag(FD->getLocation(), diag::err_redefinition) << FD->getDeclName();
+    Diag(FD->getLocation(), diag::err_redefinition) << FD;
 
   Diag(Definition->getLocation(), diag::note_previous_definition);
   FD->setInvalidDecl();
@@ -14909,9 +14907,10 @@ TypedefDecl *Sema::ParseTypedefDecl(Scope *S, Declarator &D, QualType T,
   if (D.getDeclSpec().isModulePrivateSpecified()) {
     if (CurContext->isFunctionOrMethod())
       Diag(NewTD->getLocation(), diag::err_module_private_local)
-        << 2 << NewTD->getDeclName()
-        << SourceRange(D.getDeclSpec().getModulePrivateSpecLoc())
-        << FixItHint::CreateRemoval(D.getDeclSpec().getModulePrivateSpecLoc());
+          << 2 << NewTD
+          << SourceRange(D.getDeclSpec().getModulePrivateSpecLoc())
+          << FixItHint::CreateRemoval(
+                 D.getDeclSpec().getModulePrivateSpecLoc());
     else
       NewTD->setModulePrivate();
   }
diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp
index ece93cbd6a9bd..58602a4c58d4e 100644
--- a/clang/lib/Sema/SemaDeclAttr.cpp
+++ b/clang/lib/Sema/SemaDeclAttr.cpp
@@ -3643,8 +3643,8 @@ static void handleTransparentUnionAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
       unsigned FieldBits = isSize? S.Context.getTypeSize(FieldType)
                                  : S.Context.getTypeAlign(FieldType);
       S.Diag(Field->getLocation(),
-          diag::warn_transparent_union_attribute_field_size_align)
-        << isSize << Field->getDeclName() << FieldBits;
+             diag::warn_transparent_union_attribute_field_size_align)
+          << isSize << *Field << FieldBits;
       unsigned FirstBits = isSize? FirstSize : FirstAlign;
       S.Diag(FirstField->getLocation(),
              diag::note_transparent_union_first_field_size_align)
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index bb5a07e8079dd..21d3bbf419a9a 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -94,7 +94,7 @@ static void DiagnoseUnusedOfDecl(Sema &S, NamedDecl *D, SourceLocation Loc) {
         A->getSemanticSpelling() != UnusedAttr::C2x_maybe_unused) {
       const Decl *DC = cast_or_null<Decl>(S.getCurObjCLexicalContext());
       if (DC && !DC->hasAttr<UnusedAttr>())
-        S.Diag(Loc, diag::warn_used_but_marked_unused) << D->getDeclName();
+        S.Diag(Loc, diag::warn_used_but_marked_unused) << D;
     }
   }
 }
@@ -5567,9 +5567,8 @@ bool Sema::CheckCXXDefaultArgExpr(SourceLocation CallLoc, FunctionDecl *FD,
       return true;
     }
 
-    Diag(CallLoc,
-         diag::err_use_of_default_argument_to_function_declared_later) <<
-      FD << cast<CXXRecordDecl>(FD->getDeclContext())->getDeclName();
+    Diag(CallLoc, diag::err_use_of_default_argument_to_function_declared_later)
+        << FD << cast<CXXRecordDecl>(FD->getDeclContext());
     Diag(UnparsedDefaultArgLocs[Param],
          diag::note_default_argument_declared_here);
     return true;
@@ -16932,8 +16931,7 @@ static bool isVariableCapturable(CapturingScopeInfo *CSI, VarDecl *Var,
   if (Var->getType()->isVariablyModifiedType() && IsBlock) {
     if (Diagnose) {
       S.Diag(Loc, diag::err_ref_vm_type);
-      S.Diag(Var->getLocation(), diag::note_previous_decl)
-        << Var->getDeclName();
+      S.Diag(Var->getLocation(), diag::note_previous_decl) << Var;
     }
     return false;
   }
@@ -16945,10 +16943,8 @@ static bool isVariableCapturable(CapturingScopeInfo *CSI, VarDecl *Var,
         if (IsBlock)
           S.Diag(Loc, diag::err_ref_flexarray_type);
         else
-          S.Diag(Loc, diag::err_lambda_capture_flexarray_type)
-            << Var->getDeclName();
-        S.Diag(Var->getLocation(), diag::note_previous_decl)
-          << Var->getDeclName();
+          S.Diag(Loc, diag::err_lambda_capture_flexarray_type) << Var;
+        S.Diag(Var->getLocation(), diag::note_previous_decl) << Var;
       }
       return false;
     }
@@ -16958,10 +16954,8 @@ static bool isVariableCapturable(CapturingScopeInfo *CSI, VarDecl *Var,
   // variables; they don't support the expected semantics.
   if (HasBlocksAttr && (IsLambda || isa<CapturedRegionScopeInfo>(CSI))) {
     if (Diagnose) {
-      S.Diag(Loc, diag::err_capture_block_variable)
-        << Var->getDeclName() << !IsLambda;
-      S.Diag(Var->getLocation(), diag::note_previous_decl)
-        << Var->getDeclName();
+      S.Diag(Loc, diag::err_capture_block_variable) << Var << !IsLambda;
+      S.Diag(Var->getLocation(), diag::note_previous_decl) << Var;
     }
     return false;
   }
@@ -16992,8 +16986,7 @@ static bool captureInBlock(BlockScopeInfo *BSI, VarDecl *Var,
   if (!Invalid && !S.getLangOpts().OpenCL && CaptureType->isArrayType()) {
     if (BuildAndDiagnose) {
       S.Diag(Loc, diag::err_ref_array_type);
-      S.Diag(Var->getLocation(), diag::note_previous_decl)
-      << Var->getDeclName();
+      S.Diag(Var->getLocation(), diag::note_previous_decl) << Var;
       Invalid = true;
     } else {
       return false;
@@ -17006,8 +16999,7 @@ static bool captureInBlock(BlockScopeInfo *BSI, VarDecl *Var,
     if (BuildAndDiagnose) {
       S.Diag(Loc, diag::err_arc_autoreleasing_capture)
         << /*block*/ 0;
-      S.Diag(Var->getLocation(), diag::note_previous_decl)
-        << Var->getDeclName();
+      S.Diag(Var->getLocation(), diag::note_previous_decl) << Var;
       Invalid = true;
     } else {
       return false;
@@ -17277,9 +17269,8 @@ bool Sema::tryCaptureVariable(
       if (BuildAndDiagnose) {
         LambdaScopeInfo *LSI = cast<LambdaScopeInfo>(CSI);
         if (LSI->ImpCaptureStyle == CapturingScopeInfo::ImpCap_None) {
-          Diag(ExprLoc, diag::err_lambda_impcap) << Var->getDeclName();
-          Diag(Var->getLocation(), diag::note_previous_decl)
-             << Var->getDeclName();
+          Diag(ExprLoc, diag::err_lambda_impcap) << Var;
+          Diag(Var->getLocation(), diag::note_previous_decl) << Var;
           Diag(LSI->Lambda->getBeginLoc(), diag::note_lambda_decl);
         } else
           diagnoseUncapturableValueReference(*this, ExprLoc, Var, DC);
@@ -17353,9 +17344,8 @@ bool Sema::tryCaptureVariable(
       // No capture-default, and this is not an explicit capture
       // so cannot capture this variable.
       if (BuildAndDiagnose) {
-        Diag(ExprLoc, diag::err_lambda_impcap) << Var->getDeclName();
-        Diag(Var->getLocation(), diag::note_previous_decl)
-          << Var->getDeclName();
+        Diag(ExprLoc, diag::err_lambda_impcap) << Var;
+        Diag(Var->getLocation(), diag::note_previous_decl) << Var;
         if (cast<LambdaScopeInfo>(CSI)->Lambda)
           Diag(cast<LambdaScopeInfo>(CSI)->Lambda->getBeginLoc(),
                diag::note_lambda_decl);
@@ -18319,7 +18309,7 @@ bool Sema::CheckCallReturnType(QualType ReturnType, SourceLocation Loc,
       }
 
       S.Diag(Loc, diag::err_call_function_incomplete_return)
-        << CE->getSourceRange() << FD->getDeclName() << T;
+          << CE->getSourceRange() << FD << T;
       S.Diag(FD->getLocation(), diag::note_entity_declared_at)
           << FD->getDeclName();
     }
diff --git a/clang/lib/Sema/SemaOverload.cpp b/clang/lib/Sema/SemaOverload.cpp
index 5b4e7a2fdafa5..00563cff62cf6 100644
--- a/clang/lib/Sema/SemaOverload.cpp
+++ b/clang/lib/Sema/SemaOverload.cpp
@@ -14193,12 +14193,12 @@ Sema::BuildCallToMemberFunction(Scope *S, Expr *MemExprE,
       Diag(MemExpr->getBeginLoc(),
            diag::warn_call_to_pure_virtual_member_function_from_ctor_dtor)
           << MD->getDeclName() << isa<CXXDestructorDecl>(CurContext)
-          << MD->getParent()->getDeclName();
+          << MD->getParent();
 
       Diag(MD->getBeginLoc(), diag::note_previous_decl) << MD->getDeclName();
       if (getLangOpts().AppleKext)
         Diag(MemExpr->getBeginLoc(), diag::note_pure_qualified_call_kext)
-            << MD->getParent()->getDeclName() << MD->getDeclName();
+            << MD->getParent() << MD->getDeclName();
     }
   }
 
diff --git a/clang/lib/Sema/SemaStmt.cpp b/clang/lib/Sema/SemaStmt.cpp
index 948c187804dcc..9ca2411b33e78 100644
--- a/clang/lib/Sema/SemaStmt.cpp
+++ b/clang/lib/Sema/SemaStmt.cpp
@@ -3625,12 +3625,11 @@ StmtResult Sema::BuildReturnStmt(SourceLocation ReturnLoc, Expr *RetValExp) {
     if (FD->hasAttrs())
       Attrs = &FD->getAttrs();
     if (FD->isNoReturn())
-      Diag(ReturnLoc, diag::warn_noreturn_function_has_return_expr)
-        << FD->getDeclName();
+      Diag(ReturnLoc, diag::warn_noreturn_function_has_return_expr) << FD;
     if (FD->isMain() && RetValExp)
       if (isa<CXXBoolLiteralExpr>(RetValExp))
         Diag(ReturnLoc, diag::warn_main_returns_bool_literal)
-          << RetValExp->getSourceRange();
+            << RetValExp->getSourceRange();
     if (FD->hasAttr<CmseNSEntryAttr>() && RetValExp) {
       if (const auto *RT = dyn_cast<RecordType>(FnRetType.getCanonicalType())) {
         if (RT->getDecl()->isOrContainsUnion())
@@ -3701,8 +3700,7 @@ StmtResult Sema::BuildReturnStmt(SourceLocation ReturnLoc, Expr *RetValExp) {
           FunctionKind = 3;
 
         Diag(ReturnLoc, diag::err_return_init_list)
-          << CurDecl->getDeclName() << FunctionKind
-          << RetValExp->getSourceRange();
+            << CurDecl << FunctionKind << RetValExp->getSourceRange();
 
         // Drop the expression.
         RetValExp = nullptr;
@@ -3729,9 +3727,8 @@ StmtResult Sema::BuildReturnStmt(SourceLocation ReturnLoc, Expr *RetValExp) {
         // return of void in constructor/destructor is illegal in C++.
         if (D == diag::err_ctor_dtor_returns_void) {
           NamedDecl *CurDecl = getCurFunctionOrMethodDecl();
-          Diag(ReturnLoc, D)
-            << CurDecl->getDeclName() << isa<CXXDestructorDecl>(CurDecl)
-            << RetValExp->getSourceRange();
+          Diag(ReturnLoc, D) << CurDecl << isa<CXXDestructorDecl>(CurDecl)
+                             << RetValExp->getSourceRange();
         }
         // return (some void expression); is legal in C++.
         else if (D != diag::ext_return_has_void_expr ||
@@ -3747,8 +3744,7 @@ StmtResult Sema::BuildReturnStmt(SourceLocation ReturnLoc, Expr *RetValExp) {
             FunctionKind = 3;
 
           Diag(ReturnLoc, D)
-            << CurDecl->getDeclName() << FunctionKind
-            << RetValExp->getSourceRange();
+              << CurDecl << FunctionKind << RetValExp->getSourceRange();
         }
       }
 
diff --git a/clang/test/CXX/dcl.dcl/dcl.attr/dcl.attr.noreturn/p1.cpp b/clang/test/CXX/dcl.dcl/dcl.attr/dcl.attr.noreturn/p1.cpp
index 0d4d34ac0e147..d92356c1ec0b2 100644
--- a/clang/test/CXX/dcl.dcl/dcl.attr/dcl.attr.noreturn/p1.cpp
+++ b/clang/test/CXX/dcl.dcl/dcl.attr/dcl.attr.noreturn/p1.cpp
@@ -8,9 +8,10 @@ void a2 [[noreturn]] () {
 }
 
 template <typename T> void a3 [[noreturn]] () {}
-template <> void a3<int> () { return; } // expected-warning {{function 'a3' declared 'noreturn' should not return}}
+template <> void a3<int>() { return; } // expected-warning {{function 'a3<int>' declared 'noreturn' should not return}}
 
-template <typename T> void a4 [[noreturn]] () { return; } // expected-warning 2{{function 'a4' declared 'noreturn' should not return}}
+template <typename T> void a4 [[noreturn]] () { return; } // expected-warning {{function 'a4' declared 'noreturn' should not return}}
+                                                          // expected-warning@-1 {{function 'a4<long>' declared 'noreturn' should not return}}
 void a4_test() { a4<long>(); } // expected-note {{in instantiation of function template specialization 'a4<long>' requested here}}
 
 [[noreturn, noreturn]] void b() { throw 0; } // expected-error {{attribute 'noreturn' cannot appear multiple times in an attribute specifier}}
diff --git a/clang/test/CXX/dcl.decl/dcl.meaning/dcl.fct/p3.cpp b/clang/test/CXX/dcl.decl/dcl.meaning/dcl.fct/p3.cpp
index ad827fb7b3142..415d634d5f98c 100644
--- a/clang/test/CXX/dcl.decl/dcl.meaning/dcl.fct/p3.cpp
+++ b/clang/test/CXX/dcl.decl/dcl.meaning/dcl.fct/p3.cpp
@@ -1,3 +1,7 @@
 // RUN: %clang_cc1 -fsyntax-only -verify %s 
 void f(int) { } // expected-note {{previous definition is here}}
 void f(const int) { } // expected-error {{redefinition of 'f'}}
+
+template <typename T> void ft(T) {}
+template <> void ft(int) {} // expected-note {{previous definition is here}}
+template <> void ft(int) {} // expected-error {{redefinition of 'ft<int>'}}
diff --git a/clang/test/CXX/stmt.stmt/stmt.iter/stmt.ranged/p1.cpp b/clang/test/CXX/stmt.stmt/stmt.iter/stmt.ranged/p1.cpp
index 01c758bef19a5..7c95a3ca88eb3 100644
--- a/clang/test/CXX/stmt.stmt/stmt.iter/stmt.ranged/p1.cpp
+++ b/clang/test/CXX/stmt.stmt/stmt.iter/stmt.ranged/p1.cpp
@@ -64,6 +64,21 @@ namespace X {
   };
 
   constexpr int operator*(const C::It &) { return 0; }
+
+  struct D {
+    D();
+    using Ty = int[2];
+    Ty *begin();
+    Ty *end();
+  };
+
+  void test_D() {
+#if __cplusplus >= 201703L
+    for (extern auto [x, y] : D()) {
+    } // expected-error@-1 {{decomposition declaration cannot be declared 'extern'}}
+      // expected-error@-2 {{loop variable '[x, y]' may not be declared 'extern'}}
+#endif
+  }
 }
 
 using X::A;
diff --git a/clang/test/CXX/temp/temp.param/p15-cxx0x.cpp b/clang/test/CXX/temp/temp.param/p15-cxx0x.cpp
index 667152da1cbcd..63f56640b1ce9 100644
--- a/clang/test/CXX/temp/temp.param/p15-cxx0x.cpp
+++ b/clang/test/CXX/temp/temp.param/p15-cxx0x.cpp
@@ -1,5 +1,5 @@
 // RUN: %clang_cc1 -fsyntax-only -std=c++11 -verify %s
-template<typename T> struct X; // expected-note {{'X' is incomplete}}
+template <typename T> struct X; // expected-note {{'X<X<X<int>>>' is incomplete}}
 template<int I> struct Y;
 
 X<X<int>> *x1;
@@ -14,8 +14,8 @@ typedef X<int> X_int;
 struct Z : X_int { };
 
 void f(const X<int> x) {
-  (void)reinterpret_cast<X<int>>(x); // expected-error{{reinterpret_cast from}}
-  (void)reinterpret_cast<X<X<X<int>>>>(x); // expected-error{{reinterpret_cast from}}
+  (void)reinterpret_cast<X<int>>(x);       // expected-error{{reinterpret_cast from 'const X<int>' to 'X<int>' is not allowed}}
+  (void)reinterpret_cast<X<X<X<int>>>>(x); // expected-error{{reinterpret_cast from 'const X<int>' to 'X<X<X<int>>>' is not allowed}}
 
   X<X<int>> *x1;
 }
diff --git a/clang/test/Modules/module-private.cpp b/clang/test/Modules/module-private.cpp
index 30957865d1cd0..a4b3b0fd21d39 100644
--- a/clang/test/Modules/module-private.cpp
+++ b/clang/test/Modules/module-private.cpp
@@ -1,7 +1,7 @@
 // RUN: rm -rf %t
-// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -x objective-c++ -fmodules-cache-path=%t -fmodule-name=module_private_left -emit-module %S/Inputs/module.map
-// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -x objective-c++ -fmodules-cache-path=%t -fmodule-name=module_private_right -emit-module %S/Inputs/module.map
-// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -x objective-c++ -fmodules-cache-path=%t -I %S/Inputs %s -verify
+// RUN: %clang_cc1 -std=c++17 -fmodules -fimplicit-module-maps -x objective-c++ -fmodules-cache-path=%t -fmodule-name=module_private_left -emit-module %S/Inputs/module.map
+// RUN: %clang_cc1 -std=c++17 -fmodules -fimplicit-module-maps -x objective-c++ -fmodules-cache-path=%t -fmodule-name=module_private_right -emit-module %S/Inputs/module.map
+// RUN: %clang_cc1 -std=c++17 -fmodules -fimplicit-module-maps -x objective-c++ -fmodules-cache-path=%t -I %S/Inputs %s -verify
 // FIXME: When we have a syntax for modules in C++, use that.
 
 @import module_private_left;
@@ -79,11 +79,15 @@ __module_private__ struct public_class<T *> { }; // expected-error{{partial spec
 void local_var_private(__module_private__ int param) { // expected-error{{parameter 'param' cannot be declared __module_private__}}
   __module_private__ struct Local { int x, y; } local; //expected-error{{local variable 'local' cannot be declared __module_private__}}
 
+  __module_private__ auto [x, y] = local; // expected-error {{local variable '[x, y]' cannot be declared __module_private__}}
+
   __module_private__ struct OtherLocal { int x; }; // expected-error{{local struct cannot be declared __module_private__}}
 
   typedef __module_private__ int local_typedef; // expected-error{{typedef 'local_typedef' cannot be declared __module_private__}}
 }
 
+void param_private(__module_private__ int) {} // expected-error {{parameter '' cannot be declared __module_private}}
+
 // Check struct size
 struct LikeVisibleStruct {
   int field;
diff --git a/clang/test/SemaCXX/array-bounds.cpp b/clang/test/SemaCXX/array-bounds.cpp
index 495ccaf71bd6a..47be6c2423dc1 100644
--- a/clang/test/SemaCXX/array-bounds.cpp
+++ b/clang/test/SemaCXX/array-bounds.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -verify -std=c++11 %s
+// RUN: %clang_cc1 -verify -std=c++14 %s
 
 int foo() {
   int x[2]; // expected-note 4 {{array 'x' declared here}}
@@ -309,3 +309,14 @@ namespace PR41087 {
     foo<int>(); // expected-note 1{{in instantiation of function template specialization}}
   };
 }
+
+namespace var_template_array {
+template <typename T> int arr[2]; // expected-note {{array 'arr<int>' declared here}}
+template <> int arr<float>[1];    // expected-note {{array 'arr<float>' declared here}}
+
+void test() {
+  arr<int>[1] = 0;   // ok
+  arr<int>[2] = 0;   // expected-warning {{array index 2 is past the end of the array (which contains 2 elements)}}
+  arr<float>[1] = 0; // expected-warning {{array index 1 is past the end of the array (which contains 1 element)}}
+}
+} // namespace var_template_array
diff --git a/clang/test/SemaCXX/attr-unused.cpp b/clang/test/SemaCXX/attr-unused.cpp
index e3878152eca97..5bca693864e32 100644
--- a/clang/test/SemaCXX/attr-unused.cpp
+++ b/clang/test/SemaCXX/attr-unused.cpp
@@ -15,5 +15,5 @@ void f() {
   };
   (void) i;
 
-  C<int>(); // expected-warning {{'C' was marked unused but was used}}
+  C<int>(); // expected-warning {{'C<int>' was marked unused but was used}}
 }
diff --git a/clang/test/SemaCXX/cxx1y-variable-templates_in_class.cpp b/clang/test/SemaCXX/cxx1y-variable-templates_in_class.cpp
index fc49ec88d5537..1a24c66805690 100644
--- a/clang/test/SemaCXX/cxx1y-variable-templates_in_class.cpp
+++ b/clang/test/SemaCXX/cxx1y-variable-templates_in_class.cpp
@@ -164,7 +164,7 @@ namespace constexpred {
                                                       // expected-error {{non-static data member cannot be constexpr; did you intend to make it const?}}
     template<typename T> constexpr float right<float,T> = 5;  // expected-error {{non-static data member cannot be constexpr; did you intend to make it static?}}
     template<> static constexpr int right<int,int> = 7;
-    template<> static constexpr float right<float,int>; // expected-error {{requires an initializer}}
+    template <> static constexpr float right<float, int>; // expected-error {{declaration of constexpr static data member 'right<float, int>' requires an initializer}}
     template static constexpr int right<int,int>;     // expected-error {{expected '<' after 'template'}}
   };
 }
diff --git a/clang/test/SemaCXX/default2.cpp b/clang/test/SemaCXX/default2.cpp
index 7651233f8636f..c51d272853e72 100644
--- a/clang/test/SemaCXX/default2.cpp
+++ b/clang/test/SemaCXX/default2.cpp
@@ -119,7 +119,7 @@ class C2 {
 
 template <typename T> class C3;
 template <> class C3<int> {
-  static void g(int = f()); // expected-error {{use of default argument to function 'f' that is declared later in class 'C3'}}
+  static void g(int = f()); // expected-error {{use of default argument to function 'f' that is declared later in class 'C3<int>'}}
   static int f(int = 10); // expected-note {{default argument declared here}}
 };
 
diff --git a/clang/test/SemaCXX/incomplete-call.cpp b/clang/test/SemaCXX/incomplete-call.cpp
index 46f470e4a8810..208daa2988a66 100644
--- a/clang/test/SemaCXX/incomplete-call.cpp
+++ b/clang/test/SemaCXX/incomplete-call.cpp
@@ -40,7 +40,7 @@ void g() {
   A (B::*mfp)() = 0;
   (b.*mfp)(); // expected-error {{calling function with incomplete return type 'A'}}
 
-  ft(42); // expected-error {{calling 'ft' with incomplete return type 'A'}}
+  ft(42); // expected-error {{calling 'ft<int>' with incomplete return type 'A'}}
 }
 
 
diff --git a/clang/test/SemaCXX/references.cpp b/clang/test/SemaCXX/references.cpp
index eaab1ae833e4e..f059eb6e64602 100644
--- a/clang/test/SemaCXX/references.cpp
+++ b/clang/test/SemaCXX/references.cpp
@@ -1,5 +1,6 @@
 // RUN: %clang_cc1 -fsyntax-only -verify %s
-// RUN: %clang_cc1 -std=c++11 -fsyntax-only -verify %s 
+// RUN: %clang_cc1 -std=c++11 -fsyntax-only -verify %s
+// RUN: %clang_cc1 -std=c++14 -fsyntax-only -verify %s
 int g(int);
 
 void f() {
@@ -114,6 +115,14 @@ void test8(int& const,// expected-error{{'const' qualifier may not be applied to
   void restrict_ref(int &__restrict); // ok
 }
 
+namespace var_template {
+#if __cplusplus >= 201402L
+int i;
+template <typename> int &ref = i; // ok
+template <> int &ref<float>;      // expected-error {{declaration of reference variable 'ref<float>' requires an initializer}}
+#endif
+} // namespace var_template
+
 template<typename T> int const_param(const T) {}
 int const_ref_param = const_param<int&>(const_ref_param); // no-warning
 
diff --git a/clang/test/SemaCXX/return-void.cpp b/clang/test/SemaCXX/return-void.cpp
index b3aa203133dc3..c72fbdfae9fa9 100644
--- a/clang/test/SemaCXX/return-void.cpp
+++ b/clang/test/SemaCXX/return-void.cpp
@@ -4,7 +4,7 @@ void f1() { return {1,2}; } // expected-error {{void function 'f1' must not retu
 
 template <typename T> void f2() { return {1,2}; } // expected-error {{void function 'f2' must not return a value}}
 
-template <> void f2<float>() { return {1,2}; } // expected-error {{void function 'f2' must not return a value}}
+template <> void f2<float>() { return {1, 2}; } // expected-error {{void function 'f2<float>' must not return a value}}
 
 void test_f2() {
   f2<int>();
diff --git a/clang/test/SemaCXX/warn-func-not-needed.cpp b/clang/test/SemaCXX/warn-func-not-needed.cpp
index 5040aaad94601..cb3cae4cd6c76 100644
--- a/clang/test/SemaCXX/warn-func-not-needed.cpp
+++ b/clang/test/SemaCXX/warn-func-not-needed.cpp
@@ -11,7 +11,7 @@ void foo() {
 
 namespace test1_template {
 template <typename T> static void f() {}
-template <> void f<int>() {} // expected-warning {{function 'f' is not needed and will not be emitted}}
+template <> void f<int>() {} // expected-warning {{function 'f<int>' is not needed and will not be emitted}}
 template <typename T>
 void foo() {
   f<int>();
diff --git a/clang/test/SemaCXX/warn-large-by-value-copy.cpp b/clang/test/SemaCXX/warn-large-by-value-copy.cpp
index 3e419ec08f075..309fdc75dbd79 100644
--- a/clang/test/SemaCXX/warn-large-by-value-copy.cpp
+++ b/clang/test/SemaCXX/warn-large-by-value-copy.cpp
@@ -16,6 +16,14 @@ S100 f100(S100 s) { return s; }
 S101 f101(S101 s) { return s; } // expected-warning {{return value of 'f101' is a large (101 bytes) pass-by-value object}} \
                                 // expected-warning {{'s' is a large (101 bytes) pass-by-value argument}}
 
+void f101_no_param_name(S101) {} // expected-warning {{'' is a large (101 bytes) pass-by-value argument}}
+
+// FIXME: Don't warn when when the return value is subject to (N)RVO.
+
+template <typename T> T foo_template(T);
+template <> S101 foo_template(S101) { return S101(); } // expected-warning {{return value of 'foo_template<rdar8548050::S101>' is a large}}
+                                                       // expected-warning@-1 {{'' is a large (101 bytes) pass-by-value argument}}
+
 typedef int Arr[200];
 void farr(Arr a) { }
 
diff --git a/clang/test/SemaCXX/warn-member-not-needed.cpp b/clang/test/SemaCXX/warn-member-not-needed.cpp
index 95241f4f7fee0..c48447719ba99 100644
--- a/clang/test/SemaCXX/warn-member-not-needed.cpp
+++ b/clang/test/SemaCXX/warn-member-not-needed.cpp
@@ -4,8 +4,8 @@ namespace {
   class A {
     void g() {} // expected-warning {{member function 'g' is not needed and will not be emitted}}
     template <typename T> void gt(T) {}
-    template <> void gt<int>(int) {} // expected-warning {{member function 'gt' is not needed and will not be emitted}}
-    template <> void gt(float) {}    // expected-warning {{member function 'gt' is not needed and will not be emitted}}
+    template <> void gt<int>(int) {} // expected-warning {{member function 'gt<int>' is not needed and will not be emitted}}
+    template <> void gt(float) {}    // expected-warning {{member function 'gt<float>' is not needed and will not be emitted}}
 
     template <typename T>
     void foo() {
diff --git a/clang/test/SemaCXX/warn-pure-virtual-call-from-ctor-dtor.cpp b/clang/test/SemaCXX/warn-pure-virtual-call-from-ctor-dtor.cpp
index 789935e3470ac..9acf84c6ce8c5 100644
--- a/clang/test/SemaCXX/warn-pure-virtual-call-from-ctor-dtor.cpp
+++ b/clang/test/SemaCXX/warn-pure-virtual-call-from-ctor-dtor.cpp
@@ -22,8 +22,8 @@ struct C {
 };
 
 template <typename T> struct TA {
-  TA() { f(); } // expected-warning {{call to pure virtual member function 'f' has undefined behavior; overrides of 'f' in subclasses are not available in the constructor of 'TA'}}
-  ~TA() { f(); } // expected-warning {{call to pure virtual member function 'f' has undefined behavior; overrides of 'f' in subclasses are not available in the destructor of 'TA'}}
+  TA() { f(); }  // expected-warning {{call to pure virtual member function 'f' has undefined behavior; overrides of 'f' in subclasses are not available in the constructor of 'TA<float>'}}
+  ~TA() { f(); } // expected-warning {{call to pure virtual member function 'f' has undefined behavior; overrides of 'f' in subclasses are not available in the destructor of 'TA<float>'}}
 
   virtual void f() = 0; // expected-note 2{{'f' declared here}}
 };
@@ -35,8 +35,8 @@ template <> struct TA<int> {
 };
 
 template <> struct TA<long> {
-  TA() { f(); }  // expected-warning {{call to pure virtual member function 'f' has undefined behavior; overrides of 'f' in subclasses are not available in the constructor of 'TA'}}
-  ~TA() { f(); } // expected-warning {{call to pure virtual member function 'f' has undefined behavior; overrides of 'f' in subclasses are not available in the destructor of 'TA'}}
+  TA() { f(); }         // expected-warning {{call to pure virtual member function 'f' has undefined behavior; overrides of 'f' in subclasses are not available in the constructor of 'TA<long>'}}
+  ~TA() { f(); }        // expected-warning {{call to pure virtual member function 'f' has undefined behavior; overrides of 'f' in subclasses are not available in the destructor of 'TA<long>'}}
   virtual void f() = 0; // expected-note 2{{'f' declared here}}
 };
 
diff --git a/clang/test/SemaCXX/warn-pure-virtual-kext.cpp b/clang/test/SemaCXX/warn-pure-virtual-kext.cpp
index 8431e202ad714..d23456fa4fd5c 100644
--- a/clang/test/SemaCXX/warn-pure-virtual-kext.cpp
+++ b/clang/test/SemaCXX/warn-pure-virtual-kext.cpp
@@ -10,7 +10,7 @@ struct A {
 template <typename T> struct TA {
   virtual void f() = 0; // expected-note {{'f' declared here}}
 
-  TA() { TA::f(); } // expected-warning {{call to pure virtual member function 'f' has undefined behavior; overrides of 'f' in subclasses are not available in the constructor of 'TA'}} // expected-note {{qualified call to 'TA'::'f' is treated as a virtual call to 'f' due to -fapple-kext}}
+  TA() { TA::f(); } // expected-warning {{call to pure virtual member function 'f' has undefined behavior; overrides of 'f' in subclasses are not available in the constructor of 'TA<int>'}} // expected-note {{qualified call to 'TA<int>'::'f' is treated as a virtual call to 'f' due to -fapple-kext}}
 };
 
 struct B : TA<int> { // expected-note {{in instantiation of member function 'TA<int>::TA' requested here}}
diff --git a/clang/test/SemaCXX/warn-unused-filescoped.cpp b/clang/test/SemaCXX/warn-unused-filescoped.cpp
index 056543d5eeb08..d53608003b16d 100644
--- a/clang/test/SemaCXX/warn-unused-filescoped.cpp
+++ b/clang/test/SemaCXX/warn-unused-filescoped.cpp
@@ -67,7 +67,7 @@ struct S {
 
   template <typename T>
   void tf() {}                  // expected-warning{{unused function template 'tf'}}
-  template <> void tf<int>() {} // expected-warning{{unused function 'tf'}}
+  template <> void tf<int>() {} // expected-warning{{unused function 'tf<int>'}}
 
   struct VS {
     virtual void vm() { }
@@ -102,7 +102,7 @@ struct S2 {
 
   template <typename T, typename U> int vt = 0; // expected-warning {{unused variable template 'vt'}}
   template <typename T> int vt<T, void> = 0;
-  template <> int vt<void, void> = 0; // expected-warning {{unused variable 'vt'}}
+  template <> int vt<void, void> = 0; // expected-warning {{unused variable 'vt<void, void>'}}
 }
 
 namespace PR8841 {
@@ -132,7 +132,7 @@ namespace test4 {
 namespace rdar8733476 {
 static void foo() {}                         // expected-warning {{function 'foo' is not needed and will not be emitted}}
 template <typename T> static void foo_t() {} // expected-warning {{unused function template 'foo_t'}}
-template <> void foo_t<int>() {}             // expected-warning {{function 'foo_t' is not needed and will not be emitted}}
+template <> void foo_t<int>() {}             // expected-warning {{function 'foo_t<int>' is not needed and will not be emitted}}
 
 template <int>
 void bar() {
@@ -157,7 +157,7 @@ namespace test5 {
   namespace {
   // FIXME: Should be "unused variable template 'var_t'" instead.
   template <typename T> const double var_t = 0; // expected-warning {{unused variable 'var_t'}}
-  template <> const double var_t<int> = 0;      // expected-warning {{variable 'var_t' is not needed and will not be emitted}}
+  template <> const double var_t<int> = 0;      // expected-warning {{variable 'var_t<int>' is not needed and will not be emitted}}
   int z = sizeof(var_t<int>);                   // expected-warning {{unused variable 'z'}}
   }                                             // namespace
 }
diff --git a/clang/test/SemaCXX/warn-variable-not-needed.cpp b/clang/test/SemaCXX/warn-variable-not-needed.cpp
index 139c2923f4bae..103be189068f8 100644
--- a/clang/test/SemaCXX/warn-variable-not-needed.cpp
+++ b/clang/test/SemaCXX/warn-variable-not-needed.cpp
@@ -5,7 +5,7 @@ namespace test1 {
 
   namespace {
   template <typename T> int abc_template = 0;
-  template <> int abc_template<int> = 0; // expected-warning {{variable 'abc_template' is not needed and will not be emitted}}
+  template <> int abc_template<int> = 0; // expected-warning {{variable 'abc_template<int>' is not needed and will not be emitted}}
   }                                      // namespace
   template <typename T>
   int foo(void) {

From 536736995bf5d073853c7e884968c9847b4ae64d Mon Sep 17 00:00:00 2001
From: Serge Pavlov <sepavloff@gmail.com>
Date: Wed, 22 Apr 2020 23:00:12 +0700
Subject: [PATCH 0280/1035] [Support] Add file lock/unlock functions

This is recommit of f51bc4fb60fb, reverted in 8577595e03fa, because
the function `flock` is not available on Solaris. In this variant
`flock` was replaced with `fcntl`, which is a POSIX function.

New functions `lockFile`, `tryLockFile` and `unlockFile` implement
simple file locking. They lock or unlock entire file. This must be
enough to support simulataneous writes to log files in parallel builds.

Differential Revision: https://reviews.llvm.org/D78896
---
 llvm/include/llvm/Support/FileSystem.h | 37 ++++++++++++++++++
 llvm/lib/Support/Unix/Path.inc         | 45 +++++++++++++++++++++
 llvm/lib/Support/Windows/Path.inc      | 37 ++++++++++++++++++
 llvm/unittests/Support/ProgramTest.cpp | 54 ++++++++++++++++++++++++++
 4 files changed, 173 insertions(+)

diff --git a/llvm/include/llvm/Support/FileSystem.h b/llvm/include/llvm/Support/FileSystem.h
index a29a9d787947f..b6d2a9f3aad53 100644
--- a/llvm/include/llvm/Support/FileSystem.h
+++ b/llvm/include/llvm/Support/FileSystem.h
@@ -1131,6 +1131,43 @@ Expected<file_t>
 openNativeFileForRead(const Twine &Name, OpenFlags Flags = OF_None,
                       SmallVectorImpl<char> *RealPath = nullptr);
 
+/// Try to locks the file during the specified time.
+///
+/// This function implements advisory locking on entire file. If it returns
+/// <em>errc::success</em>, the file is locked by the calling process. Until the
+/// process unlocks the file by calling \a unlockFile, all attempts to lock the
+/// same file will fail/block. The process that locked the file may assume that
+/// none of other processes read or write this file, provided that all processes
+/// lock the file prior to accessing its content.
+///
+/// @param File    The descriptor representing the file to lock.
+/// @param Timeout Time in milliseconds that the process should wait before
+///                reporting lock failure. Zero value means try to get lock only
+///                once.
+/// @returns errc::success if lock is successfully obtained,
+/// errc::no_lock_available if the file cannot be locked, or platform-specific
+/// error_code otherwise.
+///
+/// @note Care should be taken when using this function in a multithreaded
+/// context, as it may not prevent other threads in the same process from
+/// obtaining a lock on the same file, even if they are using a different file
+/// descriptor.
+std::error_code
+tryLockFile(int FD,
+            std::chrono::milliseconds Timeout = std::chrono::milliseconds(0));
+
+/// Lock the file.
+///
+/// This function acts as @ref tryLockFile but it waits infinitely.
+std::error_code lockFile(int FD);
+
+/// Unlock the file.
+///
+/// @param File The descriptor representing the file to unlock.
+/// @returns errc::success if lock is successfully released or platform-specific
+/// error_code otherwise.
+std::error_code unlockFile(int FD);
+
 /// @brief Close the file object.  This should be used instead of ::close for
 /// portability. On error, the caller should assume the file is closed, as is
 /// the case for Process::SafelyCloseFileDescriptor
diff --git a/llvm/lib/Support/Unix/Path.inc b/llvm/lib/Support/Unix/Path.inc
index d91b269cc6d33..fa4682dd33d2f 100644
--- a/llvm/lib/Support/Unix/Path.inc
+++ b/llvm/lib/Support/Unix/Path.inc
@@ -33,6 +33,7 @@
 
 #include <dirent.h>
 #include <pwd.h>
+#include <sys/file.h>
 
 #ifdef __APPLE__
 #include <mach-o/dyld.h>
@@ -1078,6 +1079,50 @@ Expected<size_t> readNativeFileSlice(file_t FD, MutableArrayRef<char> Buf,
   return NumRead;
 }
 
+std::error_code tryLockFile(int FD, std::chrono::milliseconds Timeout) {
+  auto Start = std::chrono::steady_clock::now();
+  auto End = Start + Timeout;
+  do {
+    struct flock Lock;
+    memset(&Lock, 0, sizeof(Lock));
+    Lock.l_type = F_WRLCK;
+    Lock.l_whence = SEEK_SET;
+    Lock.l_start = 0;
+    Lock.l_len = 0;
+    if (::fcntl(FD, F_SETLK, &Lock) != -1)
+      return std::error_code();
+    int Error = errno;
+    if (Error != EACCES && Error != EAGAIN)
+      return std::error_code(Error, std::generic_category());
+    usleep(1000);
+  } while (std::chrono::steady_clock::now() < End);
+  return make_error_code(errc::no_lock_available);
+}
+
+std::error_code lockFile(int FD) {
+  struct flock Lock;
+  memset(&Lock, 0, sizeof(Lock));
+  Lock.l_type = F_WRLCK;
+  Lock.l_whence = SEEK_SET;
+  Lock.l_start = 0;
+  Lock.l_len = 0;
+  if (::fcntl(FD, F_SETLKW, &Lock) != -1)
+    return std::error_code();
+  int Error = errno;
+  return std::error_code(Error, std::generic_category());
+}
+
+std::error_code unlockFile(int FD) {
+  struct flock Lock;
+  Lock.l_type = F_UNLCK;
+  Lock.l_whence = SEEK_SET;
+  Lock.l_start = 0;
+  Lock.l_len = 0;
+  if (::fcntl(FD, F_SETLK, &Lock) != -1)
+    return std::error_code();
+  return std::error_code(errno, std::generic_category());
+}
+
 std::error_code closeFile(file_t &F) {
   file_t TmpF = F;
   F = kInvalidFile;
diff --git a/llvm/lib/Support/Windows/Path.inc b/llvm/lib/Support/Windows/Path.inc
index e352beb77616b..3570d1d6e0563 100644
--- a/llvm/lib/Support/Windows/Path.inc
+++ b/llvm/lib/Support/Windows/Path.inc
@@ -1260,6 +1260,43 @@ Expected<size_t> readNativeFileSlice(file_t FileHandle,
   return readNativeFileImpl(FileHandle, Buf, &Overlapped);
 }
 
+std::error_code tryLockFile(int FD, std::chrono::milliseconds Timeout) {
+  DWORD Flags = LOCKFILE_EXCLUSIVE_LOCK | LOCKFILE_FAIL_IMMEDIATELY;
+  OVERLAPPED OV = {0};
+  file_t File = convertFDToNativeFile(FD);
+  auto Start = std::chrono::steady_clock::now();
+  auto End = Start + Timeout;
+  do {
+    if (::LockFileEx(File, Flags, 0, MAXDWORD, MAXDWORD, &OV))
+      return std::error_code();
+    DWORD Error = ::GetLastError();
+    if (Error == ERROR_LOCK_VIOLATION) {
+      ::Sleep(1);
+      continue;
+    }
+    return mapWindowsError(Error);
+  } while (std::chrono::steady_clock::now() < End);
+  return mapWindowsError(ERROR_LOCK_VIOLATION);
+}
+
+std::error_code lockFile(int FD) {
+  DWORD Flags = LOCKFILE_EXCLUSIVE_LOCK;
+  OVERLAPPED OV = {0};
+  file_t File = convertFDToNativeFile(FD);
+  if (::LockFileEx(File, Flags, 0, MAXDWORD, MAXDWORD, &OV))
+    return std::error_code();
+  DWORD Error = ::GetLastError();
+  return mapWindowsError(Error);
+}
+
+std::error_code unlockFile(int FD) {
+  OVERLAPPED OV = {0};
+  file_t File = convertFDToNativeFile(FD);
+  if (::UnlockFileEx(File, 0, MAXDWORD, MAXDWORD, &OV))
+    return std::error_code();
+  return mapWindowsError(::GetLastError());
+}
+
 std::error_code closeFile(file_t &F) {
   file_t TmpF = F;
   F = kInvalidFile;
diff --git a/llvm/unittests/Support/ProgramTest.cpp b/llvm/unittests/Support/ProgramTest.cpp
index 9052b66b5fb9e..84a5d3f64cfe6 100644
--- a/llvm/unittests/Support/ProgramTest.cpp
+++ b/llvm/unittests/Support/ProgramTest.cpp
@@ -14,6 +14,7 @@
 #include "llvm/Support/Path.h"
 #include "gtest/gtest.h"
 #include <stdlib.h>
+#include <thread>
 #if defined(__APPLE__)
 # include <crt_externs.h>
 #elif !defined(_MSC_VER)
@@ -361,4 +362,57 @@ TEST_F(ProgramEnvTest, TestExecuteAndWaitStatistics) {
   ASSERT_GE(ProcStat->TotalTime, ProcStat->UserTime);
 }
 
+TEST_F(ProgramEnvTest, TestLockFile) {
+  using namespace llvm::sys;
+
+  if (const char *LockedFile = getenv("LLVM_PROGRAM_TEST_LOCKED_FILE")) {
+    // Child process.
+    int FD2;
+    ASSERT_NO_ERROR(fs::openFileForReadWrite(LockedFile, FD2,
+                                             fs::CD_OpenExisting, fs::OF_None));
+
+    std::error_code ErrC = fs::tryLockFile(FD2, std::chrono::seconds(5));
+    ASSERT_NO_ERROR(ErrC);
+    ASSERT_NO_ERROR(fs::unlockFile(FD2));
+    close(FD2);
+    exit(0);
+  }
+
+  // Create file that will be locked.
+  SmallString<64> LockedFile;
+  int FD1;
+  ASSERT_NO_ERROR(
+      fs::createTemporaryFile("TestLockFile", "temp", FD1, LockedFile));
+
+  std::string Executable =
+      sys::fs::getMainExecutable(TestMainArgv0, &ProgramTestStringArg1);
+  StringRef argv[] = {Executable, "--gtest_filter=ProgramEnvTest.TestLockFile"};
+
+  // Add LLVM_PROGRAM_TEST_LOCKED_FILE to the environment of the child.
+  std::string EnvVar = "LLVM_PROGRAM_TEST_LOCKED_FILE=";
+  EnvVar += LockedFile.str();
+  addEnvVar(EnvVar);
+
+  // Lock the file.
+  ASSERT_NO_ERROR(fs::tryLockFile(FD1));
+
+  std::string Error;
+  bool ExecutionFailed;
+  ProcessInfo PI2 = ExecuteNoWait(Executable, argv, getEnviron(), {}, 0, &Error,
+                                  &ExecutionFailed);
+  ASSERT_FALSE(ExecutionFailed) << Error;
+  ASSERT_TRUE(Error.empty());
+  ASSERT_NE(PI2.Pid, ProcessInfo::InvalidPid) << "Invalid process id";
+
+  // Wait some time to give the child process a chance to start.
+  std::this_thread::sleep_for(std::chrono::milliseconds(100));
+
+  ASSERT_NO_ERROR(fs::unlockFile(FD1));
+  ProcessInfo WaitResult = llvm::sys::Wait(PI2, 5 /* seconds */, true, &Error);
+  ASSERT_TRUE(Error.empty());
+  ASSERT_EQ(0, WaitResult.ReturnCode);
+  ASSERT_EQ(WaitResult.Pid, PI2.Pid);
+  sys::fs::remove(LockedFile);
+}
+
 } // end anonymous namespace

From b81fd5aeecd8047ef62348b67cab2cf9a1577d8e Mon Sep 17 00:00:00 2001
From: Bruno Ricci <riccibrun@gmail.com>
Date: Tue, 28 Jul 2020 10:58:34 +0100
Subject: [PATCH 0281/1035] [clang-format][NFC] Fix a Wdocumentation warning in
 TokenAnnotator.cpp

---
 clang/lib/Format/TokenAnnotator.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/lib/Format/TokenAnnotator.cpp b/clang/lib/Format/TokenAnnotator.cpp
index b19fc34bcc802..6cbaf8a30812b 100644
--- a/clang/lib/Format/TokenAnnotator.cpp
+++ b/clang/lib/Format/TokenAnnotator.cpp
@@ -27,7 +27,7 @@ namespace format {
 namespace {
 
 /// Returns \c true if the token can be used as an identifier in
-/// an Objective-C \c @selector, \c false otherwise.
+/// an Objective-C \c \@selector, \c false otherwise.
 ///
 /// Because getFormattingLangOpts() always lexes source code as
 /// Objective-C++, C++ keywords like \c new and \c delete are

From 5ee07dc53fcaaad35d33478e8702e443ac1cb058 Mon Sep 17 00:00:00 2001
From: Luofan Chen <clfbbn@gmail.com>
Date: Tue, 28 Jul 2020 18:02:49 +0800
Subject: [PATCH 0282/1035] [Attributor] Track AA dependency using dependency
 graph

This patch added dependency graph to the attributor so that we can dump the dependencies between AAs more easily. We can also apply general graph algorithms to the graph, making it easier for us to create deep wrappers.

Reviewed By: jdoerfert

Differential Revision: https://reviews.llvm.org/D78861
---
 llvm/include/llvm/Transforms/IPO/Attributor.h | 103 ++++++++--
 llvm/lib/Transforms/IPO/Attributor.cpp        | 180 ++++++++++++++++--
 .../Transforms/IPO/AttributorAttributes.cpp   |   8 +-
 llvm/test/Transforms/Attributor/depgraph.ll   | 174 +++++++++++++++++
 4 files changed, 428 insertions(+), 37 deletions(-)
 create mode 100644 llvm/test/Transforms/Attributor/depgraph.ll

diff --git a/llvm/include/llvm/Transforms/IPO/Attributor.h b/llvm/include/llvm/Transforms/IPO/Attributor.h
index 1eead8ee788de..48c65c37eec78 100644
--- a/llvm/include/llvm/Transforms/IPO/Attributor.h
+++ b/llvm/include/llvm/Transforms/IPO/Attributor.h
@@ -97,8 +97,10 @@
 #ifndef LLVM_TRANSFORMS_IPO_ATTRIBUTOR_H
 #define LLVM_TRANSFORMS_IPO_ATTRIBUTOR_H
 
+#include "llvm/ADT/GraphTraits.h"
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/SCCIterator.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Analysis/AssumeBundleQueries.h"
@@ -116,10 +118,15 @@
 #include "llvm/IR/ConstantRange.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/Support/Allocator.h"
+#include "llvm/Support/Casting.h"
+#include "llvm/Support/DOTGraphTraits.h"
+#include "llvm/Support/GraphWriter.h"
 #include "llvm/Transforms/Utils/CallGraphUpdater.h"
 
 namespace llvm {
 
+struct AADepGraphNode;
+struct AADepGraph;
 struct Attributor;
 struct AbstractAttribute;
 struct InformationCache;
@@ -144,6 +151,70 @@ enum class DepClassTy {
 };
 ///}
 
+/// The data structure for the nodes of a dependency graph
+struct AADepGraphNode {
+public:
+  virtual ~AADepGraphNode(){};
+  using DepTy = PointerIntPair<AADepGraphNode *, 1>;
+
+protected:
+  /// Set of dependency graph nodes which this one depends on.
+  /// The bit encodes if it is optional.
+  TinyPtrVector<DepTy> Deps;
+
+  static AADepGraphNode *DepGetVal(DepTy &DT) { return DT.getPointer(); }
+  static AbstractAttribute *DepGetValAA(DepTy &DT) {
+    return cast<AbstractAttribute>(DT.getPointer());
+  }
+
+  operator AbstractAttribute *() { return cast<AbstractAttribute>(this); }
+
+public:
+  using iterator =
+      mapped_iterator<TinyPtrVector<DepTy>::iterator, decltype(&DepGetVal)>;
+  using aaiterator =
+      mapped_iterator<TinyPtrVector<DepTy>::iterator, decltype(&DepGetValAA)>;
+
+  aaiterator begin() { return aaiterator(Deps.begin(), &DepGetValAA); }
+  aaiterator end() { return aaiterator(Deps.end(), &DepGetValAA); }
+  iterator child_begin() { return iterator(Deps.begin(), &DepGetVal); }
+  iterator child_end() { return iterator(Deps.end(), &DepGetVal); }
+
+  virtual void print(raw_ostream &OS) const { OS << "AADepNode Impl\n"; }
+  TinyPtrVector<DepTy> &getDeps() { return Deps; }
+
+  friend struct Attributor;
+  friend struct AADepGraph;
+};
+
+struct AADepGraph {
+  AADepGraph() {}
+  ~AADepGraph() {}
+
+  using DepTy = AADepGraphNode::DepTy;
+  static AADepGraphNode *DepGetVal(DepTy &DT) { return DT.getPointer(); }
+  using iterator =
+      mapped_iterator<TinyPtrVector<DepTy>::iterator, decltype(&DepGetVal)>;
+
+  /// There is no root node for the dependency graph. But the SCCIterator
+  /// requires a single entry point, so we maintain a fake("synthetic") root
+  /// node that depends on every node.
+  AADepGraphNode SyntheticRoot;
+
+  AADepGraphNode *GetEntryNode() { return &SyntheticRoot; }
+
+  iterator begin() { return SyntheticRoot.child_begin(); }
+  iterator end() { return SyntheticRoot.child_end(); }
+
+  void viewGraph();
+
+  /// Dump graph to file
+  void dumpGraph();
+
+  /// Print dependency graph
+  void print();
+};
+
 /// Helper to describe and deal with positions in the LLVM-IR.
 ///
 /// A position in the IR is described by an anchor value and an "offset" that
@@ -1020,7 +1091,9 @@ struct Attributor {
     assert(!AAPtr && "Attribute already in map!");
     AAPtr = &AA;
 
-    AllAbstractAttributes.push_back(&AA);
+    DG.SyntheticRoot.Deps.push_back(
+        AADepGraphNode::DepTy(&AA, unsigned(DepClassTy::REQUIRED)));
+
     return AA;
   }
 
@@ -1382,12 +1455,6 @@ struct Attributor {
   /// See getOrCreateAAFor.
   bool shouldSeedAttribute(AbstractAttribute &AA);
 
-  /// The set of all abstract attributes.
-  ///{
-  using AAVector = SmallVector<AbstractAttribute *, 64>;
-  AAVector AllAbstractAttributes;
-  ///}
-
   /// A nested map to lookup abstract attributes based on the argument position
   /// on the outer level, and the addresses of the static member (AAType::ID) on
   /// the inner level.
@@ -1409,6 +1476,9 @@ struct Attributor {
   /// Helper to update an underlying call graph.
   CallGraphUpdater &CGUpdater;
 
+  /// Abstract Attribute dependency graph
+  AADepGraph DG;
+
   /// Set of functions for which we modified the content such that it might
   /// impact the call graph.
   SmallPtrSet<Function *, 8> CGModifiedFunctions;
@@ -1458,6 +1528,8 @@ struct Attributor {
   SmallPtrSet<BasicBlock *, 8> ToBeDeletedBlocks;
   SmallDenseSet<WeakVH, 8> ToBeDeletedInsts;
   ///}
+
+  friend AADepGraph;
 };
 
 /// An interface to query the internal state of an abstract attribute.
@@ -2030,7 +2102,7 @@ struct IRAttribute : public BaseType {
 ///       both directions will be added in the future.
 /// NOTE: The mechanics of adding a new "concrete" abstract attribute are
 ///       described in the file comment.
-struct AbstractAttribute : public IRPosition {
+struct AbstractAttribute : public IRPosition, public AADepGraphNode {
   using StateType = AbstractState;
 
   AbstractAttribute(const IRPosition &IRP) : IRPosition(IRP) {}
@@ -2038,6 +2110,14 @@ struct AbstractAttribute : public IRPosition {
   /// Virtual destructor.
   virtual ~AbstractAttribute() {}
 
+  /// This function is used to identify if an \p DGN is of type
+  /// AbstractAttribute so that the dyn_cast and cast can use such information
+  /// to cast an AADepGraphNode to an AbstractAttribute.
+  ///
+  /// We eagerly return true here because all AADepGraphNodes except for the
+  /// Synthethis Node are of type AbstractAttribute
+  static bool classof(const AADepGraphNode *DGN) { return true; }
+
   /// Initialize the state with the information in the Attributor \p A.
   ///
   /// This function is called by the Attributor once all abstract attributes
@@ -2059,6 +2139,7 @@ struct AbstractAttribute : public IRPosition {
   /// Helper functions, for debug purposes only.
   ///{
   virtual void print(raw_ostream &OS) const;
+  virtual void printWithDeps(raw_ostream &OS) const;
   void dump() const { print(dbgs()); }
 
   /// This function should return the "summarized" assumed state as string.
@@ -2106,12 +2187,6 @@ struct AbstractAttribute : public IRPosition {
   ///
   /// \Return CHANGED if the internal state changed, otherwise UNCHANGED.
   virtual ChangeStatus updateImpl(Attributor &A) = 0;
-
-private:
-  /// Set of abstract attributes which were queried by this one. The bit encodes
-  /// if there is an optional of required dependence.
-  using DepTy = PointerIntPair<AbstractAttribute *, 1>;
-  TinyPtrVector<DepTy> Deps;
 };
 
 /// Forward declarations of output streams for debug purposes.
diff --git a/llvm/lib/Transforms/IPO/Attributor.cpp b/llvm/lib/Transforms/IPO/Attributor.cpp
index f96dac5f3515c..003baf39e8bdf 100644
--- a/llvm/lib/Transforms/IPO/Attributor.cpp
+++ b/llvm/lib/Transforms/IPO/Attributor.cpp
@@ -15,7 +15,10 @@
 
 #include "llvm/Transforms/IPO/Attributor.h"
 
+#include "llvm/ADT/GraphTraits.h"
+#include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/Statistic.h"
+#include "llvm/ADT/TinyPtrVector.h"
 #include "llvm/Analysis/LazyValueInfo.h"
 #include "llvm/Analysis/MustExecute.h"
 #include "llvm/Analysis/ValueTracking.h"
@@ -25,10 +28,15 @@
 #include "llvm/InitializePasses.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/GraphWriter.h"
+#include "llvm/Support/raw_ostream.h"
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
 #include "llvm/Transforms/Utils/Local.h"
 
 #include <cassert>
+#include <string>
 
 using namespace llvm;
 
@@ -85,6 +93,23 @@ static cl::list<std::string>
                            "allowed to be seeded."),
                   cl::ZeroOrMore, cl::CommaSeparated);
 
+static cl::opt<bool>
+    DumpDepGraph("attributor-dump-dep-graph", cl::Hidden,
+                 cl::desc("Dump the dependency graph to dot files."),
+                 cl::init(false));
+
+static cl::opt<std::string> DepGraphDotFileNamePrefix(
+    "attributor-depgraph-dot-filename-prefix", cl::Hidden,
+    cl::desc("The prefix used for the CallGraph dot file names."));
+
+static cl::opt<bool> ViewDepGraph("attributor-view-dep-graph", cl::Hidden,
+                                  cl::desc("View the dependency graph."),
+                                  cl::init(false));
+
+static cl::opt<bool> PrintDependencies("attributor-print-dep", cl::Hidden,
+                                       cl::desc("Print attribute dependencies"),
+                                       cl::init(false));
+
 /// Logic operators for the change status enum class.
 ///
 ///{
@@ -498,8 +523,10 @@ Attributor::getAssumedConstant(const Value &V, const AbstractAttribute &AA,
 Attributor::~Attributor() {
   // The abstract attributes are allocated via the BumpPtrAllocator Allocator,
   // thus we cannot delete them. We can, and want to, destruct them though.
-  for (AbstractAttribute *AA : AllAbstractAttributes)
+  for (auto &DepAA : DG.SyntheticRoot.Deps) {
+    AbstractAttribute *AA = cast<AbstractAttribute>(DepAA.getPointer());
     AA->~AbstractAttribute();
+  }
 }
 
 bool Attributor::isAssumedDead(const AbstractAttribute &AA,
@@ -904,7 +931,7 @@ bool Attributor::checkForAllReadWriteInstructions(
 
 void Attributor::runTillFixpoint() {
   LLVM_DEBUG(dbgs() << "[Attributor] Identified and initialized "
-                    << AllAbstractAttributes.size()
+                    << DG.SyntheticRoot.Deps.size()
                     << " abstract attributes.\n");
 
   // Now that all abstract attributes are collected and initialized we start
@@ -914,11 +941,11 @@ void Attributor::runTillFixpoint() {
 
   SmallVector<AbstractAttribute *, 32> ChangedAAs;
   SetVector<AbstractAttribute *> Worklist, InvalidAAs;
-  Worklist.insert(AllAbstractAttributes.begin(), AllAbstractAttributes.end());
+  Worklist.insert(DG.SyntheticRoot.begin(), DG.SyntheticRoot.end());
 
   do {
     // Remember the size to determine new attributes.
-    size_t NumAAs = AllAbstractAttributes.size();
+    size_t NumAAs = DG.SyntheticRoot.Deps.size();
     LLVM_DEBUG(dbgs() << "\n\n[Attributor] #Iteration: " << IterationCounter
                       << ", Worklist size: " << Worklist.size() << "\n");
 
@@ -935,7 +962,7 @@ void Attributor::runTillFixpoint() {
       while (!InvalidAA->Deps.empty()) {
         const auto &Dep = InvalidAA->Deps.back();
         InvalidAA->Deps.pop_back();
-        AbstractAttribute *DepAA = Dep.getPointer();
+        AbstractAttribute *DepAA = cast<AbstractAttribute>(Dep.getPointer());
         if (Dep.getInt() == unsigned(DepClassTy::OPTIONAL)) {
           Worklist.insert(DepAA);
           continue;
@@ -953,7 +980,8 @@ void Attributor::runTillFixpoint() {
     // changed to the work list.
     for (AbstractAttribute *ChangedAA : ChangedAAs)
       while (!ChangedAA->Deps.empty()) {
-        Worklist.insert(ChangedAA->Deps.back().getPointer());
+        Worklist.insert(
+            cast<AbstractAttribute>(ChangedAA->Deps.back().getPointer()));
         ChangedAA->Deps.pop_back();
       }
 
@@ -981,8 +1009,8 @@ void Attributor::runTillFixpoint() {
 
     // Add attributes to the changed set if they have been created in the last
     // iteration.
-    ChangedAAs.append(AllAbstractAttributes.begin() + NumAAs,
-                      AllAbstractAttributes.end());
+    ChangedAAs.append(DG.SyntheticRoot.begin() + NumAAs,
+                      DG.SyntheticRoot.end());
 
     // Reset the work list and repopulate with the changed abstract attributes.
     // Note that dependent ones are added above.
@@ -1015,7 +1043,8 @@ void Attributor::runTillFixpoint() {
     }
 
     while (!ChangedAA->Deps.empty()) {
-      ChangedAAs.push_back(ChangedAA->Deps.back().getPointer());
+      ChangedAAs.push_back(
+          cast<AbstractAttribute>(ChangedAA->Deps.back().getPointer()));
       ChangedAA->Deps.pop_back();
     }
   }
@@ -1037,12 +1066,13 @@ void Attributor::runTillFixpoint() {
 }
 
 ChangeStatus Attributor::manifestAttributes() {
-  size_t NumFinalAAs = AllAbstractAttributes.size();
+  size_t NumFinalAAs = DG.SyntheticRoot.Deps.size();
 
   unsigned NumManifested = 0;
   unsigned NumAtFixpoint = 0;
   ChangeStatus ManifestChange = ChangeStatus::UNCHANGED;
-  for (AbstractAttribute *AA : AllAbstractAttributes) {
+  for (auto &DepAA : DG.SyntheticRoot.Deps) {
+    AbstractAttribute *AA = cast<AbstractAttribute>(DepAA.getPointer());
     AbstractState &State = AA->getState();
 
     // If there is not already a fixpoint reached, we can now take the
@@ -1082,11 +1112,14 @@ ChangeStatus Attributor::manifestAttributes() {
   NumAttributesValidFixpoint += NumAtFixpoint;
 
   (void)NumFinalAAs;
-  if (NumFinalAAs != AllAbstractAttributes.size()) {
-    for (unsigned u = NumFinalAAs; u < AllAbstractAttributes.size(); ++u)
-      errs() << "Unexpected abstract attribute: " << *AllAbstractAttributes[u]
+  if (NumFinalAAs != DG.SyntheticRoot.Deps.size()) {
+    for (unsigned u = NumFinalAAs; u < DG.SyntheticRoot.Deps.size(); ++u)
+      errs() << "Unexpected abstract attribute: "
+             << cast<AbstractAttribute>(DG.SyntheticRoot.Deps[u].getPointer())
              << " :: "
-             << AllAbstractAttributes[u]->getIRPosition().getAssociatedValue()
+             << cast<AbstractAttribute>(DG.SyntheticRoot.Deps[u].getPointer())
+                    ->getIRPosition()
+                    .getAssociatedValue()
              << "\n";
     llvm_unreachable("Expected the final number of abstract attributes to "
                      "remain unchanged!");
@@ -1265,6 +1298,17 @@ ChangeStatus Attributor::cleanupIR() {
 ChangeStatus Attributor::run() {
   SeedingPeriod = false;
   runTillFixpoint();
+
+  // dump graphs on demand
+  if (DumpDepGraph)
+    DG.dumpGraph();
+
+  if (ViewDepGraph)
+    DG.viewGraph();
+
+  if (PrintDependencies)
+    DG.print();
+
   ChangeStatus ManifestChange = manifestAttributes();
   ChangeStatus CleanupChange = cleanupIR();
   return ManifestChange | CleanupChange;
@@ -2028,8 +2072,31 @@ raw_ostream &llvm::operator<<(raw_ostream &OS, const AbstractAttribute &AA) {
 }
 
 void AbstractAttribute::print(raw_ostream &OS) const {
-  OS << "[P: " << getIRPosition() << "][" << getAsStr() << "][S: " << getState()
-     << "]";
+  OS << "[";
+  OS << getName();
+  OS << "] for CtxI ";
+
+  if (auto *I = getCtxI()) {
+    OS << "'";
+    I->print(OS);
+    OS << "'";
+  } else
+    OS << "<<null inst>>";
+
+  OS << " at position " << getIRPosition() << " with state " << getAsStr()
+     << '\n';
+}
+
+void AbstractAttribute::printWithDeps(raw_ostream &OS) const {
+  print(OS);
+
+  for (const auto &DepAA : Deps) {
+    auto *AA = DepAA.getPointer();
+    OS << "  updates ";
+    AA->print(OS);
+  }
+
+  OS << '\n';
 }
 ///}
 
@@ -2064,8 +2131,8 @@ static bool runAttributorOnFunctions(InformationCache &InfoCache,
       NumFnWithoutExactDefinition++;
 
     // We look at internal functions only on-demand but if any use is not a
-    // direct call or outside the current set of analyzed functions, we have to
-    // do it eagerly.
+    // direct call or outside the current set of analyzed functions, we have
+    // to do it eagerly.
     if (F->hasLocalLinkage()) {
       if (llvm::all_of(F->uses(), [&Functions](const Use &U) {
             const auto *CB = dyn_cast<CallBase>(U.getUser());
@@ -2081,11 +2148,41 @@ static bool runAttributorOnFunctions(InformationCache &InfoCache,
   }
 
   ChangeStatus Changed = A.run();
+
   LLVM_DEBUG(dbgs() << "[Attributor] Done with " << Functions.size()
                     << " functions, result: " << Changed << ".\n");
   return Changed == ChangeStatus::CHANGED;
 }
 
+void AADepGraph::viewGraph() { llvm::ViewGraph(this, "Dependency Graph"); }
+
+void AADepGraph::dumpGraph() {
+  static std::atomic<int> CallTimes;
+  std::string Prefix;
+
+  if (!DepGraphDotFileNamePrefix.empty())
+    Prefix = DepGraphDotFileNamePrefix;
+  else
+    Prefix = "dep_graph";
+  std::string Filename =
+      Prefix + "_" + std::to_string(CallTimes.load()) + ".dot";
+
+  outs() << "Dependency graph dump to " << Filename << ".\n";
+
+  std::error_code EC;
+
+  raw_fd_ostream File(Filename, EC, sys::fs::OF_Text);
+  if (!EC)
+    llvm::WriteGraph(File, this);
+
+  CallTimes++;
+}
+
+void AADepGraph::print() {
+  for (auto DepAA : SyntheticRoot.Deps)
+    cast<AbstractAttribute>(DepAA.getPointer())->printWithDeps(outs());
+}
+
 PreservedAnalyses AttributorPass::run(Module &M, ModuleAnalysisManager &AM) {
   FunctionAnalysisManager &FAM =
       AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
@@ -2132,6 +2229,51 @@ PreservedAnalyses AttributorCGSCCPass::run(LazyCallGraph::SCC &C,
   return PreservedAnalyses::all();
 }
 
+namespace llvm {
+
+template <> struct GraphTraits<AADepGraphNode *> {
+  using NodeRef = AADepGraphNode *;
+  using DepTy = PointerIntPair<AADepGraphNode *, 1>;
+  using EdgeRef = PointerIntPair<AADepGraphNode *, 1>;
+
+  static NodeRef getEntryNode(AADepGraphNode *DGN) { return DGN; }
+  static NodeRef DepGetVal(DepTy &DT) { return DT.getPointer(); }
+
+  using ChildIteratorType =
+      mapped_iterator<TinyPtrVector<DepTy>::iterator, decltype(&DepGetVal)>;
+  using ChildEdgeIteratorType = TinyPtrVector<DepTy>::iterator;
+
+  static ChildIteratorType child_begin(NodeRef N) { return N->child_begin(); }
+
+  static ChildIteratorType child_end(NodeRef N) { return N->child_end(); }
+};
+
+template <>
+struct GraphTraits<AADepGraph *> : public GraphTraits<AADepGraphNode *> {
+  static NodeRef getEntryNode(AADepGraph *DG) { return DG->GetEntryNode(); }
+
+  using nodes_iterator =
+      mapped_iterator<TinyPtrVector<DepTy>::iterator, decltype(&DepGetVal)>;
+
+  static nodes_iterator nodes_begin(AADepGraph *DG) { return DG->begin(); }
+
+  static nodes_iterator nodes_end(AADepGraph *DG) { return DG->end(); }
+};
+
+template <> struct DOTGraphTraits<AADepGraph *> : public DefaultDOTGraphTraits {
+  DOTGraphTraits(bool isSimple = false) : DefaultDOTGraphTraits(isSimple) {}
+
+  static std::string getNodeLabel(const AADepGraphNode *Node,
+                                  const AADepGraph *DG) {
+    std::string AAString = "";
+    raw_string_ostream O(AAString);
+    Node->print(O);
+    return AAString;
+  }
+};
+
+} // end namespace llvm
+
 namespace {
 
 struct AttributorLegacyPass : public ModulePass {
diff --git a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
index dc916089d394e..cb25030ce71c4 100644
--- a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
+++ b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
@@ -1052,9 +1052,10 @@ ChangeStatus AAReturnedValuesImpl::updateImpl(Attributor &A) {
   // map, NewRVsMap.
   decltype(ReturnedValues) NewRVsMap;
 
-  auto HandleReturnValue = [&](Value *RV, SmallSetVector<ReturnInst *, 4> &RIs) {
-    LLVM_DEBUG(dbgs() << "[AAReturnedValues] Returned value: " << *RV
-                      << " by #" << RIs.size() << " RIs\n");
+  auto HandleReturnValue = [&](Value *RV,
+                               SmallSetVector<ReturnInst *, 4> &RIs) {
+    LLVM_DEBUG(dbgs() << "[AAReturnedValues] Returned value: " << *RV << " by #"
+                      << RIs.size() << " RIs\n");
     CallBase *CB = dyn_cast<CallBase>(RV);
     if (!CB || UnresolvedCalls.count(CB))
       return;
@@ -3425,7 +3426,6 @@ struct AADereferenceableFloating : AADereferenceableImpl {
         T.GlobalState &= DS.GlobalState;
       }
 
-
       // For now we do not try to "increase" dereferenceability due to negative
       // indices as we first have to come up with code to deal with loops and
       // for overflows of the dereferenceable bytes.
diff --git a/llvm/test/Transforms/Attributor/depgraph.ll b/llvm/test/Transforms/Attributor/depgraph.ll
new file mode 100644
index 0000000000000..059587789035e
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/depgraph.ll
@@ -0,0 +1,174 @@
+; RUN: opt -passes=attributor-cgscc -disable-output -attributor-print-dep < %s 2>&1 | FileCheck %s --check-prefixes=GRAPH
+; RUN: opt -passes=attributor-cgscc -disable-output -attributor-dump-dep-graph -attributor-depgraph-dot-filename-prefix=%t < %s 2>/dev/null
+; RUN: FileCheck %s -input-file=%t_0.dot --check-prefix=DOT
+
+; Test 0
+;
+; test copied from the attributor introduction video: checkAndAdvance(), and the C code is:
+; int *checkAndAdvance(int * __attribute__((aligned(16))) p) {
+;   if (*p == 0)
+;     return checkAndAdvance(p + 4);
+;   return p;
+; }
+;
+define i32* @checkAndAdvance(i32* align 16 %0) {
+  %2 = load i32, i32* %0, align 4
+  %3 = icmp eq i32 %2, 0
+  br i1 %3, label %4, label %7
+
+4:                                                ; preds = %1
+  %5 = getelementptr inbounds i32, i32* %0, i64 4
+  %6 = call i32* @checkAndAdvance(i32* %5)
+  br label %8
+
+7:                                                ; preds = %1
+  br label %8
+
+8:                                                ; preds = %7, %4
+  %.0 = phi i32* [ %6, %4 ], [ %0, %7 ]
+  ret i32* %.0
+}
+
+;
+; Check for graph
+;
+
+;GRAPH: [AANoUnwind] for CtxI '  %2 = load i32, i32* %0, align 4' at position {fn:checkAndAdvance [checkAndAdvance@-1]} with state nounwind
+;GRAPH:   updates [AANoCapture] for CtxI '  %2 = load i32, i32* %0, align 4' at position {arg: [@0]} with state assumed not-captured-maybe-returned
+;GRAPH:   updates [AANoCapture] for CtxI '  %2 = load i32, i32* %0, align 4' at position {arg: [@0]} with state assumed not-captured-maybe-returned
+;GRAPH:   updates [AANoUnwind] for CtxI '  %6 = call i32* @checkAndAdvance(i32* %5)' at position {cs: [@-1]} with state nounwind
+;GRAPH:   updates [AANoUnwind] for CtxI '  %6 = call i32* @checkAndAdvance(i32* %5)' at position {cs: [@-1]} with state nounwind
+;GRAPH:   updates [AANoUnwind] for CtxI '  %6 = call i32* @checkAndAdvance(i32* %5)' at position {cs: [@-1]} with state nounwind
+;GRAPH:   updates [AANoCapture] for CtxI '  %2 = load i32, i32* %0, align 4' at position {arg: [@0]} with state assumed not-captured-maybe-returned
+;GRAPH:   updates [AANoCapture] for CtxI '  %2 = load i32, i32* %0, align 4' at position {arg: [@0]} with state assumed not-captured-maybe-returned
+;GRAPH:   updates [AANoCapture] for CtxI '  %2 = load i32, i32* %0, align 4' at position {arg: [@0]} with state assumed not-captured-maybe-returned
+;GRAPH:   updates [AANoCapture] for CtxI '  %2 = load i32, i32* %0, align 4' at position {arg: [@0]} with state assumed not-captured-maybe-returned
+;GRAPH:   updates [AANoCapture] for CtxI '  %2 = load i32, i32* %0, align 4' at position {arg: [@0]} with state assumed not-captured-maybe-returned
+;GRAPH: [AANoSync] for CtxI '  %2 = load i32, i32* %0, align 4' at position {fn:checkAndAdvance [checkAndAdvance@-1]} with state nosync
+;GRAPH:   updates [AANoSync] for CtxI '  %6 = call i32* @checkAndAdvance(i32* %5)' at position {cs: [@-1]} with state nosync
+;GRAPH:   updates [AANoSync] for CtxI '  %6 = call i32* @checkAndAdvance(i32* %5)' at position {cs: [@-1]} with state nosync
+;GRAPH: [AANoFree] for CtxI '  %2 = load i32, i32* %0, align 4' at position {fn:checkAndAdvance [checkAndAdvance@-1]} with state nofree
+;GRAPH:   updates [AANoFree] for CtxI '  %2 = load i32, i32* %0, align 4' at position {arg: [@0]} with state nofree
+;GRAPH:   updates [AANoFree] for CtxI '  %2 = load i32, i32* %0, align 4' at position {arg: [@0]} with state nofree
+;GRAPH:   updates [AANoFree] for CtxI '  %6 = call i32* @checkAndAdvance(i32* %5)' at position {cs: [@-1]} with state nofree
+;GRAPH:   updates [AANoFree] for CtxI '  %6 = call i32* @checkAndAdvance(i32* %5)' at position {cs: [@-1]} with state nofree
+;GRAPH: [AAMemoryBehavior] for CtxI '  %2 = load i32, i32* %0, align 4' at position {fn:checkAndAdvance [checkAndAdvance@-1]} with state readonly
+;GRAPH:   updates [AANoCapture] for CtxI '  %2 = load i32, i32* %0, align 4' at position {arg: [@0]} with state assumed not-captured-maybe-returned
+;GRAPH:   updates [AAMemoryBehavior] for CtxI '  %2 = load i32, i32* %0, align 4' at position {arg: [@0]} with state readonly
+;GRAPH:   updates [AANoCapture] for CtxI '  %2 = load i32, i32* %0, align 4' at position {arg: [@0]} with state assumed not-captured-maybe-returned
+;GRAPH:   updates [AAMemoryBehavior] for CtxI '  %2 = load i32, i32* %0, align 4' at position {arg: [@0]} with state readonly
+;GRAPH:   updates [AAMemoryBehavior] for CtxI '  %6 = call i32* @checkAndAdvance(i32* %5)' at position {cs: [@-1]} with state readonly
+;GRAPH:   updates [AAMemoryBehavior] for CtxI '  %6 = call i32* @checkAndAdvance(i32* %5)' at position {cs: [@-1]} with state readonly
+;GRAPH:   updates [AAMemoryBehavior] for CtxI '  %2 = load i32, i32* %0, align 4' at position {arg: [@0]} with state readonly
+;GRAPH:   updates [AAMemoryBehavior] for CtxI '  %6 = call i32* @checkAndAdvance(i32* %5)' at position {cs: [@-1]} with state readonly
+;GRAPH:   updates [AAMemoryBehavior] for CtxI '  %6 = call i32* @checkAndAdvance(i32* %5)' at position {cs: [@-1]} with state readonly
+;GRAPH:   updates [AANoCapture] for CtxI '  %2 = load i32, i32* %0, align 4' at position {arg: [@0]} with state assumed not-captured-maybe-returned
+;GRAPH:   updates [AANoCapture] for CtxI '  %2 = load i32, i32* %0, align 4' at position {arg: [@0]} with state assumed not-captured-maybe-returned
+;GRAPH:   updates [AAMemoryBehavior] for CtxI '  %2 = load i32, i32* %0, align 4' at position {arg: [@0]} with state readonly
+;GRAPH:   updates [AANoCapture] for CtxI '  %2 = load i32, i32* %0, align 4' at position {arg: [@0]} with state assumed not-captured-maybe-returned
+;GRAPH:   updates [AANoCapture] for CtxI '  %2 = load i32, i32* %0, align 4' at position {arg: [@0]} with state assumed not-captured-maybe-returned
+;GRAPH:   updates [AANoCapture] for CtxI '  %2 = load i32, i32* %0, align 4' at position {arg: [@0]} with state assumed not-captured-maybe-returned
+;GRAPH: [AAMemoryLocation] for CtxI '  %2 = load i32, i32* %0, align 4' at position {fn:checkAndAdvance [checkAndAdvance@-1]} with state memory:argument
+;GRAPH:   updates [AAMemoryLocation] for CtxI '  %6 = call i32* @checkAndAdvance(i32* %5)' at position {cs: [@-1]} with state memory:argument
+;GRAPH:   updates [AAMemoryLocation] for CtxI '  %6 = call i32* @checkAndAdvance(i32* %5)' at position {cs: [@-1]} with state memory:argument
+;GRAPH: [AAAlign] for CtxI '  %2 = load i32, i32* %0, align 4' at position {fn_ret:checkAndAdvance [checkAndAdvance@-1]} with state align<0-16>
+;GRAPH:   updates [AAAlign] for CtxI '  %6 = call i32* @checkAndAdvance(i32* %5)' at position {cs_ret: [@-1]} with state align<1-16>
+;GRAPH:   updates [AAAlign] for CtxI '  %6 = call i32* @checkAndAdvance(i32* %5)' at position {cs_ret: [@-1]} with state align<1-16>
+;GRAPH: [AANonNull] for CtxI '  %2 = load i32, i32* %0, align 4' at position {fn_ret:checkAndAdvance [checkAndAdvance@-1]} with state nonnull
+;GRAPH:   updates [AANonNull] for CtxI '  %6 = call i32* @checkAndAdvance(i32* %5)' at position {cs_ret: [@-1]} with state nonnull
+;GRAPH:   updates [AANonNull] for CtxI '  %6 = call i32* @checkAndAdvance(i32* %5)' at position {cs_ret: [@-1]} with state nonnull
+;GRAPH: [AANoCapture] for CtxI '  %2 = load i32, i32* %0, align 4' at position {arg: [@0]} with state assumed not-captured-maybe-returned
+;GRAPH:   updates [AANoCapture] for CtxI '  %6 = call i32* @checkAndAdvance(i32* %5)' at position {cs_arg: [@0]} with state assumed not-captured-maybe-returned
+;GRAPH:   updates [AANoCapture] for CtxI '  %6 = call i32* @checkAndAdvance(i32* %5)' at position {cs_arg: [@0]} with state assumed not-captured-maybe-returned
+;GRAPH: [AAMemoryBehavior] for CtxI '  %2 = load i32, i32* %0, align 4' at position {arg: [@0]} with state readonly
+;GRAPH:   updates [AAMemoryBehavior] for CtxI '  %6 = call i32* @checkAndAdvance(i32* %5)' at position {cs_arg: [@0]} with state readonly
+;GRAPH:   updates [AAMemoryBehavior] for CtxI '  %6 = call i32* @checkAndAdvance(i32* %5)' at position {cs_arg: [@0]} with state readonly
+;GRAPH: [AANoFree] for CtxI '  %2 = load i32, i32* %0, align 4' at position {arg: [@0]} with state nofree
+;GRAPH:   updates [AANoFree] for CtxI '  %6 = call i32* @checkAndAdvance(i32* %5)' at position {cs_arg: [@0]} with state nofree
+;GRAPH: [AANoUnwind] for CtxI '  %6 = call i32* @checkAndAdvance(i32* %5)' at position {cs: [@-1]} with state nounwind
+;GRAPH:   updates [AAIsDead] for CtxI '  %6 = call i32* @checkAndAdvance(i32* %5)' at position {cs_ret: [@-1]} with state assumed-live
+;GRAPH:   updates [AAIsDead] for CtxI '  %6 = call i32* @checkAndAdvance(i32* %5)' at position {cs_ret: [@-1]} with state assumed-live
+;GRAPH:   updates [AANoUnwind] for CtxI '  %2 = load i32, i32* %0, align 4' at position {fn:checkAndAdvance [checkAndAdvance@-1]} with state nounwind
+;GRAPH: [AAMemoryBehavior] for CtxI '  %6 = call i32* @checkAndAdvance(i32* %5)' at position {cs: [@-1]} with state readonly
+;GRAPH:   updates [AAIsDead] for CtxI '  %6 = call i32* @checkAndAdvance(i32* %5)' at position {cs_ret: [@-1]} with state assumed-live
+;GRAPH:   updates [AAMemoryBehavior] for CtxI '  %2 = load i32, i32* %0, align 4' at position {fn:checkAndAdvance [checkAndAdvance@-1]} with state readonly
+;GRAPH: [AANoCapture] for CtxI '  %6 = call i32* @checkAndAdvance(i32* %5)' at position {cs_arg: [@0]} with state assumed not-captured-maybe-returned
+;GRAPH:   updates [AANoCapture] for CtxI '  %2 = load i32, i32* %0, align 4' at position {arg: [@0]} with state assumed not-captured-maybe-returned
+;GRAPH:   updates [AANoCapture] for CtxI '  %2 = load i32, i32* %0, align 4' at position {arg: [@0]} with state assumed not-captured-maybe-returned
+;GRAPH:   updates [AANoCapture] for CtxI '  %2 = load i32, i32* %0, align 4' at position {arg: [@0]} with state assumed not-captured-maybe-returned
+;GRAPH: [AAMemoryBehavior] for CtxI '  %6 = call i32* @checkAndAdvance(i32* %5)' at position {cs_arg: [@0]} with state readonly
+;GRAPH:   updates [AAMemoryLocation] for CtxI '  %2 = load i32, i32* %0, align 4' at position {fn:checkAndAdvance [checkAndAdvance@-1]} with state memory:argument
+;GRAPH:   updates [AAMemoryLocation] for CtxI '  %2 = load i32, i32* %0, align 4' at position {fn:checkAndAdvance [checkAndAdvance@-1]} with state memory:argument
+;GRAPH:   updates [AAMemoryLocation] for CtxI '  %2 = load i32, i32* %0, align 4' at position {fn:checkAndAdvance [checkAndAdvance@-1]} with state memory:argument
+;GRAPH: [AANonNull] for CtxI '  %5 = getelementptr inbounds i32, i32* %0, i64 4' at position {flt: [@-1]} with state nonnull
+;GRAPH:   updates [AANonNull] for CtxI '  %5 = getelementptr inbounds i32, i32* %0, i64 4' at position {flt: [@-1]} with state nonnull
+;GRAPH:   updates [AANonNull] for CtxI '  %6 = call i32* @checkAndAdvance(i32* %5)' at position {cs_arg: [@0]} with state nonnull
+;GRAPH:   updates [AANonNull] for CtxI '  %2 = load i32, i32* %0, align 4' at position {fn_ret:checkAndAdvance [checkAndAdvance@-1]} with state nonnull
+;GRAPH:   updates [AANonNull] for CtxI '  %2 = load i32, i32* %0, align 4' at position {fn_ret:checkAndAdvance [checkAndAdvance@-1]} with state nonnull
+;GRAPH: [AANoSync] for CtxI '  %6 = call i32* @checkAndAdvance(i32* %5)' at position {cs: [@-1]} with state nosync
+;GRAPH:   updates [AANoSync] for CtxI '  %2 = load i32, i32* %0, align 4' at position {fn:checkAndAdvance [checkAndAdvance@-1]} with state nosync
+;GRAPH: [AANoFree] for CtxI '  %6 = call i32* @checkAndAdvance(i32* %5)' at position {cs: [@-1]} with state nofree
+;GRAPH:   updates [AANoFree] for CtxI '  %2 = load i32, i32* %0, align 4' at position {fn:checkAndAdvance [checkAndAdvance@-1]} with state nofree
+;GRAPH: [AAMemoryLocation] for CtxI '  %6 = call i32* @checkAndAdvance(i32* %5)' at position {cs: [@-1]} with state memory:argument
+;GRAPH:   updates [AAMemoryLocation] for CtxI '  %2 = load i32, i32* %0, align 4' at position {fn:checkAndAdvance [checkAndAdvance@-1]} with state memory:argument
+;GRAPH: [AAAlign] for CtxI '  %6 = call i32* @checkAndAdvance(i32* %5)' at position {cs_ret: [@-1]} with state align<1-16>
+;GRAPH:   updates [AAAlign] for CtxI '  %2 = load i32, i32* %0, align 4' at position {fn_ret:checkAndAdvance [checkAndAdvance@-1]} with state align<0-16>
+;GRAPH: [AANonNull] for CtxI '  %6 = call i32* @checkAndAdvance(i32* %5)' at position {cs_ret: [@-1]} with state nonnull
+;GRAPH:   updates [AANonNull] for CtxI '  %2 = load i32, i32* %0, align 4' at position {fn_ret:checkAndAdvance [checkAndAdvance@-1]} with state nonnull
+
+;
+; Check for .dot file
+;
+
+; DOT-DAG: Node[[Node6:0x[a-z0-9]+]] [shape=record,label="{[AANoUnwind] for CtxI '  %2 = load i32, i32* %0, align 4' at position \{fn:checkAndAdvance [checkAndAdvance@-1]\}
+; DOT-DAG: Node[[Node34:0x[a-z0-9]+]] [shape=record,label="{[AANoCapture] for CtxI '  %2 = load i32, i32* %0, align 4' at position \{arg: [@0]\}
+; DOT-DAG: Node[[Node39:0x[a-z0-9]+]] [shape=record,label="{[AANoUnwind] for CtxI '  %6 = call i32* @checkAndAdvance(i32* %5)' at position \{cs: [@-1]\}
+; DOT-DAG: Node[[Node7:0x[a-z0-9]+]] [shape=record,label="{[AANoSync] for CtxI '  %2 = load i32, i32* %0, align 4' at position \{fn:checkAndAdvance [checkAndAdvance@-1]\}
+; DOT-DAG: Node[[Node61:0x[a-z0-9]+]] [shape=record,label="{[AANoSync] for CtxI '  %6 = call i32* @checkAndAdvance(i32* %5)' at position \{cs: [@-1]\}
+; DOT-DAG: Node[[Node13:0x[a-z0-9]+]] [shape=record,label="{[AANoFree] for CtxI '  %2 = load i32, i32* %0, align 4' at position \{fn:checkAndAdvance [checkAndAdvance@-1]\}
+; DOT-DAG: Node[[Node36:0x[a-z0-9]+]] [shape=record,label="{[AANoFree] for CtxI '  %2 = load i32, i32* %0, align 4' at position \{arg: [@0]\}
+; DOT-DAG: Node[[Node62:0x[a-z0-9]+]] [shape=record,label="{[AANoFree] for CtxI '  %6 = call i32* @checkAndAdvance(i32* %5)' at position \{cs: [@-1]\}
+; DOT-DAG: Node[[Node16:0x[a-z0-9]+]] [shape=record,label="{[AAMemoryBehavior] for CtxI '  %2 = load i32, i32* %0, align 4' at position \{fn:checkAndAdvance [checkAndAdvance@-1]\}
+; DOT-DAG: Node[[Node35:0x[a-z0-9]+]] [shape=record,label="{[AAMemoryBehavior] for CtxI '  %2 = load i32, i32* %0, align 4' at position \{arg: [@0]\}
+; DOT-DAG: Node[[Node40:0x[a-z0-9]+]] [shape=record,label="{[AAMemoryBehavior] for CtxI '  %6 = call i32* @checkAndAdvance(i32* %5)' at position \{cs: [@-1]\}
+; DOT-DAG: Node[[Node17:0x[a-z0-9]+]] [shape=record,label="{[AAMemoryLocation] for CtxI '  %2 = load i32, i32* %0, align 4' at position \{fn:checkAndAdvance [checkAndAdvance@-1]\}
+; DOT-DAG: Node[[Node63:0x[a-z0-9]+]] [shape=record,label="{[AAMemoryLocation] for CtxI '  %6 = call i32* @checkAndAdvance(i32* %5)' at position \{cs: [@-1]\}
+; DOT-DAG: Node[[Node22:0x[a-z0-9]+]] [shape=record,label="{[AAAlign] for CtxI '  %2 = load i32, i32* %0, align 4' at position \{fn_ret:checkAndAdvance [checkAndAdvance@-1]\}
+; DOT-DAG: Node[[Node65:0x[a-z0-9]+]] [shape=record,label="{[AAAlign] for CtxI '  %6 = call i32* @checkAndAdvance(i32* %5)' at position \{cs_ret: [@-1]\}
+; DOT-DAG: Node[[Node23:0x[a-z0-9]+]] [shape=record,label="{[AANonNull] for CtxI '  %2 = load i32, i32* %0, align 4' at position \{fn_ret:checkAndAdvance [checkAndAdvance@-1]\}
+; DOT-DAG: Node[[Node67:0x[a-z0-9]+]] [shape=record,label="{[AANonNull] for CtxI '  %6 = call i32* @checkAndAdvance(i32* %5)' at position \{cs_ret: [@-1]\}
+; DOT-DAG: Node[[Node43:0x[a-z0-9]+]] [shape=record,label="{[AANoCapture] for CtxI '  %6 = call i32* @checkAndAdvance(i32* %5)' at position \{cs_arg: [@0]\}
+; DOT-DAG: Node[[Node45:0x[a-z0-9]+]] [shape=record,label="{[AAMemoryBehavior] for CtxI '  %6 = call i32* @checkAndAdvance(i32* %5)' at position \{cs_arg: [@0]\}
+; DOT-DAG: Node[[Node46:0x[a-z0-9]+]] [shape=record,label="{[AANoFree] for CtxI '  %6 = call i32* @checkAndAdvance(i32* %5)' at position \{cs_arg: [@0]\}
+; DOT-DAG: Node[[Node38:0x[a-z0-9]+]] [shape=record,label="{[AAIsDead] for CtxI '  %6 = call i32* @checkAndAdvance(i32* %5)' at position \{cs_ret: [@-1]\}
+; DOT-DAG: Node[[Node55:0x[a-z0-9]+]] [shape=record,label="{[AANonNull] for CtxI '  %5 = getelementptr inbounds i32, i32* %0, i64 4' at position \{flt: [@-1]\}
+; DOT-DAG: Node[[Node31:0x[a-x0-9]+]] [shape=record,label="{[AANonNull] for CtxI '  %6 = call i32* @checkAndAdvance(i32* %5)' at position \{cs_arg: [@0]\}
+
+; DOT-DAG: Node[[Node6]] -> Node[[Node34]]
+; DOT-DAG: Node[[Node6]] -> Node[[Node39]]
+; DOT-DAG: Node[[Node7]] -> Node[[Node61]]
+; DOT-DAG: Node[[Node13]] -> Node[[Node36]]
+; DOT-DAG: Node[[Node13]] -> Node[[Node62]]
+; DOT-DAG: Node[[Node16]] -> Node[[Node34]]
+; DOT-DAG: Node[[Node16]] -> Node[[Node35]]
+; DOT-DAG: Node[[Node16]] -> Node[[Node40]]
+; DOT-DAG: Node[[Node17]] -> Node[[Node63]]
+; DOT-DAG: Node[[Node22]] -> Node[[Node65]]
+; DOT-DAG: Node[[Node23]] -> Node[[Node67]]
+; DOT-DAG: Node[[Node34]] -> Node[[Node43]]
+; DOT-DAG: Node[[Node35]] -> Node[[Node45]]
+; DOT-DAG: Node[[Node36]] -> Node[[Node46]]
+; DOT-DAG: Node[[Node39]] -> Node[[Node38]]
+; DOT-DAG: Node[[Node39]] -> Node[[Node6]]
+; DOT-DAG: Node[[Node40]] -> Node[[Node38]]
+; DOT-DAG: Node[[Node40]] -> Node[[Node16]]
+; DOT-DAG: Node[[Node43]] -> Node[[Node34]]
+; DOT-DAG: Node[[Node45]] -> Node[[Node17]]
+; DOT-DAG: Node[[Node55]] -> Node[[Node55]]
+; DOT-DAG: Node[[Node55]] -> Node[[Node31]]
+; DOT-DAG: Node[[Node55]] -> Node[[Node23]]
+; DOT-DAG: Node[[Node61]] -> Node[[Node7]]
+; DOT-DAG: Node[[Node62]] -> Node[[Node13]]
+; DOT-DAG: Node[[Node63]] -> Node[[Node17]]
+; DOT-DAG: Node[[Node65]] -> Node[[Node22]]
+; DOT-DAG: Node[[Node67]] -> Node[[Node23]]

From c0bd9fa137c28a3ef833b46b7f9770b060275281 Mon Sep 17 00:00:00 2001
From: Haojian Wu <hokein.wu@gmail.com>
Date: Tue, 28 Jul 2020 11:59:49 +0200
Subject: [PATCH 0283/1035] [Concepts] Fix ast dump for immediately declared
 constraint.

Reviewed By: nridge

Differential Revision: https://reviews.llvm.org/D84461
---
 clang/lib/AST/TextNodeDumper.cpp     |  2 +-
 clang/test/AST/ast-dump-concepts.cpp | 19 +++++++++++++++++++
 2 files changed, 20 insertions(+), 1 deletion(-)
 create mode 100644 clang/test/AST/ast-dump-concepts.cpp

diff --git a/clang/lib/AST/TextNodeDumper.cpp b/clang/lib/AST/TextNodeDumper.cpp
index 91b984820cd22..5b6c6085e02cf 100644
--- a/clang/lib/AST/TextNodeDumper.cpp
+++ b/clang/lib/AST/TextNodeDumper.cpp
@@ -1994,7 +1994,7 @@ void TextNodeDumper::VisitTemplateTypeParmDecl(const TemplateTypeParmDecl *D) {
       dumpBareDeclRef(TC->getFoundDecl());
       OS << ")";
     }
-    Visit(TC->getImmediatelyDeclaredConstraint());
+    AddChild([=] { Visit(TC->getImmediatelyDeclaredConstraint()); });
   } else if (D->wasDeclaredWithTypename())
     OS << " typename";
   else
diff --git a/clang/test/AST/ast-dump-concepts.cpp b/clang/test/AST/ast-dump-concepts.cpp
new file mode 100644
index 0000000000000..530c1baeffa77
--- /dev/null
+++ b/clang/test/AST/ast-dump-concepts.cpp
@@ -0,0 +1,19 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -std=c++2a -ast-dump -ast-dump-filter Foo %s | FileCheck -strict-whitespace %s
+
+// Test with serialization:
+// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -x c++ -std=c++20 -triple x86_64-unknown-unknown -include-pch %t \
+// RUN: -ast-dump-all -ast-dump-filter Foo /dev/null \
+// RUN: | FileCheck --strict-whitespace %s
+
+template <typename T, typename U>
+concept binary_concept = true;
+
+template <typename T>
+struct Foo {
+  // CHECK:      TemplateTypeParmDecl {{.*}} referenced Concept {{.*}} 'binary_concept'
+  // CHECK-NEXT: |-ConceptSpecializationExpr {{.*}} 'bool'
+  // CHECK-NEXT: `-TemplateArgument {{.*}} type 'int'
+  template <binary_concept<int> R>
+  Foo(R);
+};

From 67070d98fae5c49e183ff5d68ae8038e6fd8f5a9 Mon Sep 17 00:00:00 2001
From: Mikhail Kalashnikov <mikkqu@gmail.com>
Date: Tue, 28 Jul 2020 13:06:51 +0300
Subject: [PATCH 0284/1035] [llvm-readelf] Symbol index in symbol table
 printing is not reset

Stop using static variables for keeping track of symbol indices.

Bugfix for: https://bugs.llvm.org/show_bug.cgi?id=46777

Differential revision: https://reviews.llvm.org/D84606
---
 llvm/test/tools/llvm-readobj/ELF/symbols.test | 10 ++++++++++
 llvm/tools/llvm-readobj/ELFDumper.cpp         | 13 +------------
 2 files changed, 11 insertions(+), 12 deletions(-)

diff --git a/llvm/test/tools/llvm-readobj/ELF/symbols.test b/llvm/test/tools/llvm-readobj/ELF/symbols.test
index 69d59311f662b..0878827ab6765 100644
--- a/llvm/test/tools/llvm-readobj/ELF/symbols.test
+++ b/llvm/test/tools/llvm-readobj/ELF/symbols.test
@@ -103,6 +103,16 @@
 # RUN: llvm-readobj --dyn-symbols %t64 | FileCheck %s --implicit-check-not="{{^}}Symbols ["
 # RUN: llvm-readelf --dyn-symbols %t64 | FileCheck %s --implicit-check-not="Symbol table '.symtab'"
 
+## Case 6: Test that the Num index starts from zero at every new symbol table.
+# RUN: llvm-readelf --symbols %t64 %t64 | FileCheck %s --check-prefix=NUM-INDEX
+
+# NUM-INDEX:      Symbol table '.symtab' contains 3 entries:
+# NUM-INDEX-NEXT:    Num: {{.*}}
+# NUM-INDEX-NEXT:      0: {{.*}}
+# NUM-INDEX:      Symbol table '.symtab' contains 3 entries:
+# NUM-INDEX-NEXT:    Num: {{.*}}
+# NUM-INDEX-NEXT:      0: {{.*}}
+
 --- !ELF
 FileHeader:
   Class:   ELFCLASS[[BITS]]
diff --git a/llvm/tools/llvm-readobj/ELFDumper.cpp b/llvm/tools/llvm-readobj/ELFDumper.cpp
index 72dcb9c5a15aa..590108a86fa7e 100644
--- a/llvm/tools/llvm-readobj/ELFDumper.cpp
+++ b/llvm/tools/llvm-readobj/ELFDumper.cpp
@@ -3924,21 +3924,10 @@ void GNUStyle<ELFT>::printSymbol(const ELFO *Obj, const Elf_Sym *Symbol,
                                  const Elf_Sym *FirstSym,
                                  Optional<StringRef> StrTable, bool IsDynamic,
                                  bool NonVisibilityBitsUsed) {
-  static int Idx = 0;
-  static bool Dynamic = true;
-
-  // If this function was called with a different value from IsDynamic
-  // from last call, happens when we move from dynamic to static symbol
-  // table, "Num" field should be reset.
-  if (!Dynamic != !IsDynamic) {
-    Idx = 0;
-    Dynamic = false;
-  }
-
   unsigned Bias = ELFT::Is64Bits ? 8 : 0;
   Field Fields[8] = {0,         8,         17 + Bias, 23 + Bias,
                      31 + Bias, 38 + Bias, 48 + Bias, 51 + Bias};
-  Fields[0].Str = to_string(format_decimal(Idx++, 6)) + ":";
+  Fields[0].Str = to_string(format_decimal(Symbol - FirstSym, 6)) + ":";
   Fields[1].Str = to_string(
       format_hex_no_prefix(Symbol->st_value, ELFT::Is64Bits ? 16 : 8));
   Fields[2].Str = to_string(format_decimal(Symbol->st_size, 5));

From 93b7915504b708f39a75d72e08448443a899345e Mon Sep 17 00:00:00 2001
From: Georgii Rymar <grimar@accesssoftek.com>
Date: Tue, 28 Jul 2020 13:31:13 +0300
Subject: [PATCH 0285/1035] [llvm-readobj] - Add comments and formatting to
 mips-options-sec.test and mips-reginfo.test. NFCI.

This will allow to extend them (needed for D84651).
---
 .../llvm-readobj/ELF/mips-options-sec.test    | 24 ++++++++++---------
 .../tools/llvm-readobj/ELF/mips-reginfo.test  | 20 +++++++++-------
 2 files changed, 24 insertions(+), 20 deletions(-)

diff --git a/llvm/test/tools/llvm-readobj/ELF/mips-options-sec.test b/llvm/test/tools/llvm-readobj/ELF/mips-options-sec.test
index 3636d56cfe6e2..f53f04c55e9bf 100644
--- a/llvm/test/tools/llvm-readobj/ELF/mips-options-sec.test
+++ b/llvm/test/tools/llvm-readobj/ELF/mips-options-sec.test
@@ -1,12 +1,14 @@
-RUN: llvm-readobj -A %p/Inputs/options.obj.elf-mipsel | FileCheck %s
+## Check that we are able to dump the SHT_MIPS_OPTIONS section using -A properly.
 
-CHECK:      MIPS Options {
-CHECK-NEXT:   ODK_REGINFO {
-CHECK-NEXT:     GP: 0x0
-CHECK-NEXT:     General Mask: 0xF2000017
-CHECK-NEXT:     Co-Proc Mask0: 0x0
-CHECK-NEXT:     Co-Proc Mask1: 0x0
-CHECK-NEXT:     Co-Proc Mask2: 0x0
-CHECK-NEXT:     Co-Proc Mask3: 0x0
-CHECK-NEXT:   }
-CHECK-NEXT: }
+# RUN: llvm-readobj -A %p/Inputs/options.obj.elf-mipsel | FileCheck %s
+
+# CHECK:      MIPS Options {
+# CHECK-NEXT:   ODK_REGINFO {
+# CHECK-NEXT:     GP: 0x0
+# CHECK-NEXT:     General Mask: 0xF2000017
+# CHECK-NEXT:     Co-Proc Mask0: 0x0
+# CHECK-NEXT:     Co-Proc Mask1: 0x0
+# CHECK-NEXT:     Co-Proc Mask2: 0x0
+# CHECK-NEXT:     Co-Proc Mask3: 0x0
+# CHECK-NEXT:   }
+# CHECK-NEXT: }
diff --git a/llvm/test/tools/llvm-readobj/ELF/mips-reginfo.test b/llvm/test/tools/llvm-readobj/ELF/mips-reginfo.test
index 20177a99d8cb7..0074631843cd0 100644
--- a/llvm/test/tools/llvm-readobj/ELF/mips-reginfo.test
+++ b/llvm/test/tools/llvm-readobj/ELF/mips-reginfo.test
@@ -1,10 +1,12 @@
-RUN: llvm-readobj -A %p/Inputs/reginfo.obj.elf-mipsel | FileCheck %s
+## Check that we are able to dump the SHT_MIPS_REGINFO section using -A properly.
 
-CHECK:      MIPS RegInfo {
-CHECK-NEXT:   GP: 0x7FEF
-CHECK-NEXT:   General Mask: 0xB00001F6
-CHECK-NEXT:   Co-Proc Mask0: 0x0
-CHECK-NEXT:   Co-Proc Mask1: 0x0
-CHECK-NEXT:   Co-Proc Mask2: 0x0
-CHECK-NEXT:   Co-Proc Mask3: 0x0
-CHECK-NEXT: }
+# RUN: llvm-readobj -A %p/Inputs/reginfo.obj.elf-mipsel | FileCheck %s
+
+# CHECK:      MIPS RegInfo {
+# CHECK-NEXT:   GP: 0x7FEF
+# CHECK-NEXT:   General Mask: 0xB00001F6
+# CHECK-NEXT:   Co-Proc Mask0: 0x0
+# CHECK-NEXT:   Co-Proc Mask1: 0x0
+# CHECK-NEXT:   Co-Proc Mask2: 0x0
+# CHECK-NEXT:   Co-Proc Mask3: 0x0
+# CHECK-NEXT: }

From 7e8d5a90f2c101388d3b0bbce8555e871c670232 Mon Sep 17 00:00:00 2001
From: Hans Wennborg <hans@chromium.org>
Date: Tue, 28 Jul 2020 12:26:37 +0200
Subject: [PATCH 0286/1035] Avoid use of std::make_unique in
 compiler-rt/lib/scudo/standalone/tests/combined_test.cpp

make_unique is a C++14 feature, and this prevents us from building on
Ubuntu Trusty. While we do use a C++14 compatible toolchain for building
in general, we fall back to the system toolchain for building the
compiler-rt tests.

The reason is that those tests get cross-compiled for e.g. 32-bit and
64-bit x86, and while the toolchain provides libstdc++ in those
flavours, the resulting compiler-rt test binaries don't get RPATH set
and so won't start if they're linked with that toolchain.

We've tried linking the test binaries against libstdc++ statically, by
passing COMPILER_RT_TEST_COMPILER_CFLAGS=-static-libstdc++. That mostly
works, but some test targets append -lstdc++ to the compiler invocation.

So, after spending way too much time on this, let's just avoid C++14
here for now.
---
 .../lib/scudo/standalone/tests/combined_test.cpp     | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp b/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp
index c144ad0ae32a3..6cefe18b8f15c 100644
--- a/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp
+++ b/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp
@@ -78,7 +78,7 @@ template <typename Config> struct TestAllocator : scudo::Allocator<Config> {
 
 template <class Config> static void testAllocator() {
   using AllocatorT = TestAllocator<Config>;
-  auto Allocator = std::make_unique<AllocatorT>();
+  auto Allocator = std::unique_ptr<AllocatorT>(new AllocatorT());
 
   EXPECT_FALSE(Allocator->isOwned(&Mutex));
   EXPECT_FALSE(Allocator->isOwned(&Allocator));
@@ -352,7 +352,7 @@ template <typename AllocatorT> static void stressAllocator(AllocatorT *A) {
 
 template <class Config> static void testAllocatorThreaded() {
   using AllocatorT = TestAllocator<Config>;
-  auto Allocator = std::make_unique<AllocatorT>();
+  auto Allocator = std::unique_ptr<AllocatorT>(new AllocatorT());
   std::thread Threads[32];
   for (scudo::uptr I = 0; I < ARRAY_SIZE(Threads); I++)
     Threads[I] = std::thread(stressAllocator<AllocatorT>, Allocator.get());
@@ -399,7 +399,7 @@ struct DeathConfig {
 
 TEST(ScudoCombinedTest, DeathCombined) {
   using AllocatorT = TestAllocator<DeathConfig>;
-  auto Allocator = std::make_unique<AllocatorT>();
+  auto Allocator = std::unique_ptr<AllocatorT>(new AllocatorT());
 
   const scudo::uptr Size = 1000U;
   void *P = Allocator->allocate(Size, Origin);
@@ -434,7 +434,7 @@ TEST(ScudoCombinedTest, DeathCombined) {
 // operation without issue.
 TEST(ScudoCombinedTest, ReleaseToOS) {
   using AllocatorT = TestAllocator<DeathConfig>;
-  auto Allocator = std::make_unique<AllocatorT>();
+  auto Allocator = std::unique_ptr<AllocatorT>(new AllocatorT());
 
   Allocator->releaseToOS();
 }
@@ -443,7 +443,7 @@ TEST(ScudoCombinedTest, ReleaseToOS) {
 // fulfill the allocation through a larger size class.
 TEST(ScudoCombinedTest, FullRegion) {
   using AllocatorT = TestAllocator<DeathConfig>;
-  auto Allocator = std::make_unique<AllocatorT>();
+  auto Allocator = std::unique_ptr<AllocatorT>(new AllocatorT());
 
   std::vector<void *> V;
   scudo::uptr FailedAllocationsCount = 0;
@@ -474,7 +474,7 @@ TEST(ScudoCombinedTest, FullRegion) {
 TEST(ScudoCombinedTest, OddEven) {
   using AllocatorT = TestAllocator<scudo::AndroidConfig>;
   using SizeClassMap = AllocatorT::PrimaryT::SizeClassMap;
-  auto Allocator = std::make_unique<AllocatorT>();
+  auto Allocator = std::unique_ptr<AllocatorT>(new AllocatorT());
 
   if (!Allocator->useMemoryTagging())
     return;

From 946be75b9ec131519837e85487fc3e8bf475d001 Mon Sep 17 00:00:00 2001
From: lorenzo chelini <l.chelini@tue.nl>
Date: Tue, 28 Jul 2020 12:29:54 +0200
Subject: [PATCH 0287/1035] [MLIR][Linalg] Retire C++ DotOp in favor of a
 linalg-ods-gen'd op

- replace DotOp, now that DRR rules have been dropped.

- Capture arguments mismatch in the parser. The number of parsed arguments must
  equal the number of expected arguments.

Reviewed By: ftynse, nicolasvasilache

Differential Revision: https://reviews.llvm.org/D82952
---
 .../Linalg/IR/LinalgNamedStructuredOpsSpec.tc |  5 ++++
 .../mlir/Dialect/Linalg/IR/LinalgOps.h        |  6 ++---
 .../Dialect/Linalg/IR/LinalgStructuredOps.td  | 25 -------------------
 .../LinalgToStandard/LinalgToStandard.cpp     |  4 +--
 mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp      | 19 +++++++++++---
 mlir/lib/Dialect/Linalg/Transforms/Loops.cpp  | 16 ++----------
 mlir/test/Dialect/Linalg/invalid.mlir         |  4 +--
 mlir/test/Dialect/Linalg/loops.mlir           |  6 +++--
 mlir/test/Dialect/Linalg/roundtrip.mlir       | 16 ++++++------
 mlir/test/Dialect/Linalg/standard.mlir        |  6 ++---
 mlir/test/Dialect/Linalg/tile.mlir            |  8 +++---
 .../transform-patterns-matmul-to-vector.mlir  |  2 +-
 .../Dialect/Linalg/transform-patterns.mlir    | 10 ++++----
 .../linalg_integration_test.mlir              |  2 +-
 14 files changed, 56 insertions(+), 73 deletions(-)

diff --git a/mlir/include/mlir/Dialect/Linalg/IR/LinalgNamedStructuredOpsSpec.tc b/mlir/include/mlir/Dialect/Linalg/IR/LinalgNamedStructuredOpsSpec.tc
index bbd398585e5ff..056f0723e92dd 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/LinalgNamedStructuredOpsSpec.tc
+++ b/mlir/include/mlir/Dialect/Linalg/IR/LinalgNamedStructuredOpsSpec.tc
@@ -8,6 +8,11 @@ def matvec(A: f32(M, N), y: f32(N)) -> (x: f32(M)) {
   x(m) = std_addf<n>(std_mulf(A(m, n), y(n)));
 }
 
+ods_def<DotOp>:
+def dot(A: f32(M), B: f32(M)) -> (C: f32()) {
+  C() = std_addf<m>(std_mulf(A(m), B(m)));
+}
+
 ods_def<BatchMatmulOp>:
 def batch_matmul(A: f32(Batch, M, K), B: f32(Batch, K, N)) -> (C: f32(Batch, M, N)) {
   C(b, m, n) = std_addf<k>(std_mulf(A(b, m, k), B(b, k, n)));
diff --git a/mlir/include/mlir/Dialect/Linalg/IR/LinalgOps.h b/mlir/include/mlir/Dialect/Linalg/IR/LinalgOps.h
index f89965a96857f..21bff4185abf8 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/LinalgOps.h
+++ b/mlir/include/mlir/Dialect/Linalg/IR/LinalgOps.h
@@ -51,9 +51,9 @@ using ReassociationExprs = SmallVector<AffineExpr, 2>;
 /// 1. linalg.fill(%A, %f) : memref<f32>, f32
 ///   name mangles into `linalg_fill_viewf32_f32_impl`
 ///
-/// 2. linalg.dot(%A, %B, %C) :
-///      memref<?xf32, stride_specification>,
-///      memref<?xf32, stride_specification>, memref<f32>
+/// 2. linalg.dot %A, %B, %C :
+///      (memref<?xf32, stride_specification>,
+///       memref<?xf32, stride_specification>, memref<f32>)
 ///   name mangles into `linalg_dot_viewxf32_viewxf32_viewf32_impl`
 ///
 /// 3. linalg.matmul(...) :
diff --git a/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOps.td b/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOps.td
index 81f911a37cea2..1e3321af981e6 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOps.td
+++ b/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOps.td
@@ -180,31 +180,6 @@ def FillOp : LinalgStructured_Op<"fill", [NInputs<0>, NOutputs<1>]> {
   let hasFolder = 1;
 }
 
-def DotOp : LinalgStructured_Op<"dot", [NInputs<2>, NOutputs<1>]> {
-
-  let arguments = (ins AnyStridedMemRefOfRank<1>,
-                       AnyStridedMemRefOfRank<1>,
-                       AnyStridedMemRefOfRank<0>);
-
-  let extraClassDeclaration = libraryCallName # [{
-    llvm::Optional<SmallVector<StringRef, 8>> referenceIterators() {
-      return SmallVector<StringRef, 8>{getReductionIteratorTypeName()};
-    }
-
-    // A(r_i) * B(r_i) -> C()
-    llvm::Optional<SmallVector<AffineMap, 8>> referenceIndexingMaps() {
-      MLIRContext *context = getContext();
-      auto r_i = getAffineDimExpr(0, context);
-      return SmallVector<AffineMap, 8>{
-        AffineMap::get(1, 0, {r_i}, context),
-        AffineMap::get(1, 0, {r_i}, context),
-        AffineMap::get(1, 0, {}, context)};
-    }
-  }];
-
-  let hasFolder = 1;
-}
-
 /// A base class for pooling operation such as conv. The arguments must contain
 /// optional arguments `strides`, `dilations` and `padding` with following type:
 ///   OptionalAttr<I64ArrayAttr>:$strides
diff --git a/mlir/lib/Conversion/LinalgToStandard/LinalgToStandard.cpp b/mlir/lib/Conversion/LinalgToStandard/LinalgToStandard.cpp
index 43f9d88253275..8a54c93d7685e 100644
--- a/mlir/lib/Conversion/LinalgToStandard/LinalgToStandard.cpp
+++ b/mlir/lib/Conversion/LinalgToStandard/LinalgToStandard.cpp
@@ -235,13 +235,13 @@ void mlir::populateLinalgToStandardConversionPatterns(
       LinalgOpConversion<PoolingMaxOp>,
       LinalgOpConversion<PoolingMinOp>,
       LinalgOpConversion<PoolingSumOp>,
-      LinalgOpConversion<CopyOp>,
-      LinalgOpConversion<DotOp>,
+      LinalgOpConversion<CopyOp>, 
       LinalgOpConversion<FillOp>,
       LinalgOpConversion<GenericOp>,
       LinalgOpConversion<IndexedGenericOp>>(ctx);
   // TODO: collect all auto-generated named ops with a tblgen directive.
   patterns.insert<
+      LinalgOpConversion<DotOp>,
       LinalgOpConversion<BatchMatmulOp>,
       LinalgOpConversion<MatvecOp>,
       LinalgOpConversion<MatmulOp>>(ctx);
diff --git a/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp b/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
index 4c68f0265677c..192179b3ff506 100644
--- a/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
+++ b/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
@@ -1173,10 +1173,6 @@ LogicalResult CopyOp::fold(ArrayRef<Attribute>,
                            SmallVectorImpl<OpFoldResult> &) {
   return foldMemRefCast(*this);
 }
-LogicalResult DotOp::fold(ArrayRef<Attribute>,
-                          SmallVectorImpl<OpFoldResult> &) {
-  return foldMemRefCast(*this);
-}
 LogicalResult FillOp::fold(ArrayRef<Attribute>,
                            SmallVectorImpl<OpFoldResult> &) {
   return foldMemRefCast(*this);
@@ -1280,6 +1276,17 @@ static ParseResult parseNamedStructuredOp(OpAsmParser &parser,
   if (!tensorResultTypes.empty())
     result.addTypes(tensorResultTypes);
 
+  // The number of parsed arguments must equal
+  // the number of expected arguments for the current operation.
+  auto parsedArgs = operandsInfo.size();
+  auto expectedArgs = NamedStructuredOpType::getNumInputs() +
+                      NamedStructuredOpType::getNumOutputs();
+  if (parsedArgs != expectedArgs)
+    return parser.emitError(parser.getNameLoc(),
+                            "expects " + std::to_string(expectedArgs) +
+                                " operands, but found " +
+                                std::to_string(parsedArgs));
+
   buildNamedStructuredOpRegionAndAttributes<NamedStructuredOpType>(
       parser.getBuilder(), result, operandTypes, tensorResultTypes);
 
@@ -1299,6 +1306,10 @@ LogicalResult BatchMatmulOp::fold(ArrayRef<Attribute>,
                                   SmallVectorImpl<OpFoldResult> &) {
   return foldMemRefCast(*this);
 }
+LogicalResult DotOp::fold(ArrayRef<Attribute>,
+                          SmallVectorImpl<OpFoldResult> &) {
+  return foldMemRefCast(*this);
+}
 LogicalResult MatmulOp::fold(ArrayRef<Attribute>,
                              SmallVectorImpl<OpFoldResult> &) {
   return foldMemRefCast(*this);
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp b/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp
index 7fb1018fc5881..32e50cb597d7a 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp
@@ -295,18 +295,6 @@ void emitScalarImplementation(ArrayRef<Value> allIvs, FillOp fillOp) {
   nPar > 0 ? O(ivs) = fillOp.value() : O() = fillOp.value();
 }
 
-template <typename IndexedValueType>
-void emitScalarImplementation(ArrayRef<Value> allIvs, DotOp dotOp) {
-  assert(dotOp.hasBufferSemantics() &&
-         "expected linalg op with buffer semantics");
-  assert(allIvs.size() == 1);
-  Value r_i(allIvs[0]);
-  IndexedValueType A(dotOp.getInput(0)), B(dotOp.getInput(1)),
-      C(dotOp.getOutputBuffer(0));
-  // Emit scalar form.
-  C() = C() + A(r_i) * B(r_i);
-}
-
 template <typename IndexedValueType>
 Value getConvOpInput(ConvOp convOp, StdIndexedValue im,
                      MutableArrayRef<Value> imIdx) {
@@ -673,8 +661,6 @@ static Optional<LinalgLoops> linalgOpToLoopsImplSwitch(Operation *op,
     return linalgOpToLoopsImpl<LoopTy, CopyOp>(op, builder);
   if (isa<FillOp>(op))
     return linalgOpToLoopsImpl<LoopTy, FillOp>(op, builder);
-  if (isa<DotOp>(op))
-    return linalgOpToLoopsImpl<LoopTy, DotOp>(op, builder);
   if (isa<ConvOp>(op))
     return linalgOpToLoopsImpl<LoopTy, ConvOp>(op, builder);
   if (isa<PoolingMaxOp>(op))
@@ -693,6 +679,8 @@ static Optional<LinalgLoops> linalgOpToLoopsImplSwitch(Operation *op,
     return linalgOpToLoopsImpl<LoopTy, MatmulOp>(op, builder);
   if (isa<MatvecOp>(op))
     return linalgOpToLoopsImpl<LoopTy, MatvecOp>(op, builder);
+  if (isa<DotOp>(op))
+    return linalgOpToLoopsImpl<LoopTy, DotOp>(op, builder);
   if (isa<BatchMatmulOp>(op))
     return linalgOpToLoopsImpl<LoopTy, BatchMatmulOp>(op, builder);
   llvm_unreachable("Unexpected op in linalgOpToLoopsImpl");
diff --git a/mlir/test/Dialect/Linalg/invalid.mlir b/mlir/test/Dialect/Linalg/invalid.mlir
index 79d61d8d78921..ca59ecd387ec3 100644
--- a/mlir/test/Dialect/Linalg/invalid.mlir
+++ b/mlir/test/Dialect/Linalg/invalid.mlir
@@ -422,8 +422,8 @@ func @generic(%arg0: memref<?x?xi4>) {
 // -----
 
 func @generic_result_0_element_type(%arg0: memref<?xf32>) {
-  // expected-error @+1 {{'linalg.dot' op expected 3 operands, but found 2}}
-  linalg.dot(%arg0, %arg0): memref<?xf32>, memref<?xf32>
+  // expected-error @+1 {{'linalg.dot' expects 3 operands, but found 2}}
+  linalg.dot %arg0, %arg0 : (memref<?xf32>, memref<?xf32>)
 }
 
 // -----
diff --git a/mlir/test/Dialect/Linalg/loops.mlir b/mlir/test/Dialect/Linalg/loops.mlir
index 30bb90bdd43a1..b01beb7e8f175 100644
--- a/mlir/test/Dialect/Linalg/loops.mlir
+++ b/mlir/test/Dialect/Linalg/loops.mlir
@@ -123,7 +123,7 @@ func @dot(%arg0: memref<?xi8>, %M: index) {
   %1 = view %arg0[%c0][%M] : memref<?xi8> to memref<?xf32>
   %2 = view %arg0[%c0][%M] : memref<?xi8> to memref<?xf32>
   %3 = view %arg0[%c0][] : memref<?xi8> to memref<f32>
-  linalg.dot(%1, %2, %3) : memref<?xf32>, memref<?xf32>, memref<f32>
+  linalg.dot %1, %2, %3 : (memref<?xf32>, memref<?xf32>, memref<f32>)
   return
 }
 // CHECKLOOP-LABEL: func @dot(%{{.*}}: memref<?xi8>,
@@ -154,7 +154,9 @@ func @dot(%arg0: memref<?xi8>, %M: index) {
 
 
 func @dot_view(%arg0: memref<?xf32, offset: ?, strides: [1]>, %arg1: memref<?xf32, offset: ?, strides: [1]>, %arg2: memref<f32>) {
-  linalg.dot(%arg0, %arg1, %arg2) : memref<?xf32, offset: ?, strides: [1]>, memref<?xf32, offset: ?, strides: [1]>, memref<f32>
+  linalg.dot %arg0, %arg1, %arg2  : (memref<?xf32, offset: ?, strides: [1]>, 
+                                     memref<?xf32, offset: ?, strides: [1]>, 
+                                     memref<f32>)
   return
 }
 // CHECKLOOP-LABEL: func @dot_view(
diff --git a/mlir/test/Dialect/Linalg/roundtrip.mlir b/mlir/test/Dialect/Linalg/roundtrip.mlir
index 9e6c275479302..2696643246972 100644
--- a/mlir/test/Dialect/Linalg/roundtrip.mlir
+++ b/mlir/test/Dialect/Linalg/roundtrip.mlir
@@ -88,10 +88,10 @@ func @ops(%arg0: memref<?x?xf32, offset: ?, strides: [?, 1]>,
                                        memref<?x?xf32, offset: ?, strides: [?, 1]>)
   linalg.matvec %arg0, %arg1, %arg2 : (memref<?x?xf32, offset: ?, strides: [?, 1]>,
                                        memref<?xf32, offset: ?, strides: [1]>,
-                                       memref<?xf32, offset: ?, strides: [1]>)
-  linalg.dot(%arg1, %arg2, %arg3) : memref<?xf32, offset: ?, strides: [1]>,
-                                    memref<?xf32, offset: ?, strides: [1]>,
-                                    memref<f32>
+                                       memref<?xf32, offset: ?, strides: [1]>) 
+  linalg.dot %arg1, %arg2, %arg3  : (memref<?xf32, offset: ?, strides: [1]>,
+                                     memref<?xf32, offset: ?, strides: [1]>,
+                                     memref<f32>)
   return
 }
 // CHECK-LABEL: func @ops(%
@@ -103,10 +103,10 @@ func @ops(%arg0: memref<?x?xf32, offset: ?, strides: [?, 1]>,
 //  CHECK-SAME:    (memref<?x?xf32, #[[$strided2D]]>,
 //  CHECK-SAME:     memref<?xf32, #[[$strided1D]]>,
 //  CHECK-SAME:     memref<?xf32, #[[$strided1D]]>)
-//  CHECK-NEXT:  linalg.dot(%{{.*}}, %{{.*}}, %{{.*}}) :
-//  CHECK-SAME:     memref<?xf32, #[[$strided1D]]>,
-//  CHECK-SAME:     memref<?xf32, #[[$strided1D]]>,
-//  CHECK-SAME:     memref<f32>
+//  CHECK-NEXT:  linalg.dot %{{.*}}, %{{.*}}, %{{.*}} :
+//  CHECK-SAME:     (memref<?xf32, #[[$strided1D]]>,
+//  CHECK-SAME:      memref<?xf32, #[[$strided1D]]>,
+//  CHECK-SAME:      memref<f32>)
 
 // -----
 
diff --git a/mlir/test/Dialect/Linalg/standard.mlir b/mlir/test/Dialect/Linalg/standard.mlir
index 0ba3465443fa1..60b348110c4f6 100644
--- a/mlir/test/Dialect/Linalg/standard.mlir
+++ b/mlir/test/Dialect/Linalg/standard.mlir
@@ -13,9 +13,9 @@
 func @dot(%arg0: memref<?xf32, offset: ?, strides: [1]>,
           %arg1: memref<?xf32, offset: ?, strides: [1]>,
           %arg2: memref<f32>) {
-  linalg.dot(%arg0, %arg1, %arg2) : memref<?xf32, offset: ?, strides: [1]>,
-                                    memref<?xf32, offset: ?, strides: [1]>,
-                                    memref<f32>
+  linalg.dot %arg0, %arg1, %arg2  : (memref<?xf32, offset: ?, strides: [1]>,
+                                     memref<?xf32, offset: ?, strides: [1]>,
+                                     memref<f32>)
   return
 }
 // CHECK-LABEL: func @dot(
diff --git a/mlir/test/Dialect/Linalg/tile.mlir b/mlir/test/Dialect/Linalg/tile.mlir
index 049fb571bd51d..9a1bbfc1dc184 100644
--- a/mlir/test/Dialect/Linalg/tile.mlir
+++ b/mlir/test/Dialect/Linalg/tile.mlir
@@ -271,7 +271,9 @@ func @matvec(%arg0: memref<?x?xf32, offset: ?, strides: [?, 1]>, %arg1: memref<?
 //       TILE-234:      linalg.matvec %[[sAij]], %[[sBj]], %[[sCi]] : (memref<?x?xf32, #[[$strided2D]]>, memref<?xf32, #[[$strided1D]]>, memref<?xf32, #[[$strided1D]]>)
 
 func @dot(%arg0: memref<?xf32, offset: ?, strides: [1]>, %arg1: memref<?xf32, offset: ?, strides: [1]>, %arg2: memref<f32>) {
-  linalg.dot(%arg0, %arg1, %arg2) : memref<?xf32, offset: ?, strides: [1]>, memref<?xf32, offset: ?, strides: [1]>, memref<f32>
+  linalg.dot %arg0, %arg1, %arg2  : (memref<?xf32, offset: ?, strides: [1]>, 
+                                     memref<?xf32, offset: ?, strides: [1]>, 
+                                     memref<f32>)
   return
 }
 // TILE-2-LABEL: func @dot(
@@ -285,7 +287,7 @@ func @dot(%arg0: memref<?xf32, offset: ?, strides: [1]>, %arg1: memref<?xf32, of
 //       TILE-2:   %[[localM:.*]] = dim %{{.*}}, %c0
 //       TILE-2:   %[[szM:.*]] = affine.min #[[$bound_map]](%[[I]])[%[[localM]]]
 //       TILE-2:   %[[sBi:.*]] = subview %{{.*}}[%[[I]]] [%[[szM]]] [1] : memref<?xf32, #[[$strided1D]]> to memref<?xf32, #[[$strided1D]]>
-//       TILE-2:   linalg.dot(%[[sAi]], %[[sBi]], {{.*}}) : memref<?xf32, #[[$strided1D]]>, memref<?xf32, #[[$strided1D]]>, memref<f32>
+//       TILE-2:   linalg.dot %[[sAi]], %[[sBi]], {{.*}} : (memref<?xf32, #[[$strided1D]]>, memref<?xf32, #[[$strided1D]]>, memref<f32>)
 
 // TILE-02-LABEL: func @dot(
 //   TILE-02-NOT: scf.for
@@ -304,7 +306,7 @@ func @dot(%arg0: memref<?xf32, offset: ?, strides: [1]>, %arg1: memref<?xf32, of
 //       TILE-234:    %[[localM:.*]] = dim %{{.*}}, %c0
 //       TILE-234:    %[[szM:.*]] = affine.min #[[$bound_map_2]](%[[I]])[%[[localM]]]
 //       TILE-234:    %[[sBi:.*]] = subview %{{.*}}[%[[I]]] [%[[szM]]] [1] : memref<?xf32, #[[$strided1D]]> to memref<?xf32, #[[$strided1D]]>
-//       TILE-234:    linalg.dot(%[[sAi]], %[[sBi]], %{{.*}}) : memref<?xf32, #[[$strided1D]]>, memref<?xf32, #[[$strided1D]]>, memref<f32>
+//       TILE-234:    linalg.dot %[[sAi]], %[[sBi]], %{{.*}} : (memref<?xf32, #[[$strided1D]]>, memref<?xf32, #[[$strided1D]]>, memref<f32>)
 
 func @fill_static(%arg0: memref<127x99xf32>, %arg1: f32) {
   linalg.fill(%arg0, %arg1) : memref<127x99xf32>, f32
diff --git a/mlir/test/Dialect/Linalg/transform-patterns-matmul-to-vector.mlir b/mlir/test/Dialect/Linalg/transform-patterns-matmul-to-vector.mlir
index b0702f9fdcfd8..83e9461d66cc9 100644
--- a/mlir/test/Dialect/Linalg/transform-patterns-matmul-to-vector.mlir
+++ b/mlir/test/Dialect/Linalg/transform-patterns-matmul-to-vector.mlir
@@ -36,7 +36,7 @@ func @matmul(%A: memref<1584x1584xf32, offset: 0, strides: [1584, 1]>,
 func @contraction_dot(%A: memref<1584xf32>, %B: memref<1584xf32>, %C: memref<f32>) {
   // VECTOR-CONTRACTION: vector.contract
   // VECTOR-CONTRACTION-SAME: vector<1584xf32>, vector<1584xf32> into f32
-  linalg.dot(%A, %B, %C) : memref<1584xf32>, memref<1584xf32>, memref<f32>
+  linalg.dot %A, %B, %C : (memref<1584xf32>, memref<1584xf32>, memref<f32>)
   return
 }
 
diff --git a/mlir/test/Dialect/Linalg/transform-patterns.mlir b/mlir/test/Dialect/Linalg/transform-patterns.mlir
index 3f7d16497253d..1a4100403b007 100644
--- a/mlir/test/Dialect/Linalg/transform-patterns.mlir
+++ b/mlir/test/Dialect/Linalg/transform-patterns.mlir
@@ -14,10 +14,10 @@
 func @dot(%x: memref<?xf32, offset: ?, strides: [1]>,
           %y: memref<?xf32, offset: ?, strides: [1]>,
           %v: memref<f32>) {
-  linalg.dot(%x, %y, %v) { __internal_linalg_transform__ = "MEM" } :
-             memref<?xf32, offset: ?, strides: [1]>,
-             memref<?xf32, offset: ?, strides: [1]>,
-             memref<f32>
+  linalg.dot %x, %y, %v { __internal_linalg_transform__ = "MEM" } :
+             (memref<?xf32, offset: ?, strides: [1]>,
+              memref<?xf32, offset: ?, strides: [1]>,
+              memref<f32>)
   return
 }
 // CHECK-LABEL: func @dot
@@ -28,8 +28,8 @@ func @dot(%x: memref<?xf32, offset: ?, strides: [1]>,
 // CHECK:             scf.for {{.*}} = %[[c0]] to {{.*}} step %[[c1]] {
 // CHECK:               load
 // CHECK:               load
-// CHECK:               mulf
 // CHECK:               load
+// CHECK:               mulf
 // CHECK:               addf
 // CHECK:               store
 
diff --git a/mlir/test/mlir-cpu-runner/linalg_integration_test.mlir b/mlir/test/mlir-cpu-runner/linalg_integration_test.mlir
index dd6feb96240ee..ba2ea59cb22d7 100644
--- a/mlir/test/mlir-cpu-runner/linalg_integration_test.mlir
+++ b/mlir/test/mlir-cpu-runner/linalg_integration_test.mlir
@@ -51,7 +51,7 @@ func @dot() -> f32 {
   %B = view %bB[%c0][%c16] : memref<?xi8> to memref<?xf32>
   %C = view %bC[%c0][] : memref<?xi8> to memref<f32>
 
-  linalg.dot(%A, %B, %C) : memref<?xf32>, memref<?xf32>, memref<f32>
+  linalg.dot %A, %B, %C : (memref<?xf32>, memref<?xf32>, memref<f32>)
   %res = load %C[] : memref<f32>
 
   dealloc %bC : memref<?xi8>

From 7294ca3f6ecacd05a197bbf0637e10afcb99b6d6 Mon Sep 17 00:00:00 2001
From: Kai Nacke <kai.nacke@de.ibm.com>
Date: Thu, 2 Jul 2020 14:43:42 +0200
Subject: [PATCH 0288/1035] [SystemZ/ZOS] Implement
 setLastAccessAndModificationTime()

The function setLastAccessAndModificationTime() uses function
futimens() or futimes() by default. Both functions are not
available in z/OS, therefore functionality is implemented using
__fchattr() on z/OS.

Reviews by: abhina.sreeskantharajan

Differential Revision: https://reviews.llvm.org/D83945
---
 llvm/lib/Support/Unix/Path.inc | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/llvm/lib/Support/Unix/Path.inc b/llvm/lib/Support/Unix/Path.inc
index fa4682dd33d2f..01903ea10e814 100644
--- a/llvm/lib/Support/Unix/Path.inc
+++ b/llvm/lib/Support/Unix/Path.inc
@@ -792,6 +792,16 @@ std::error_code setLastAccessAndModificationTime(int FD, TimePoint<> AccessTime,
   if (::futimes(FD, Times))
     return std::error_code(errno, std::generic_category());
   return std::error_code();
+#elif defined(__MVS__)
+  attrib_t Attr;
+  memset(&Attr, 0, sizeof(Attr));
+  Attr.att_atimechg = 1;
+  Attr.att_atime = sys::toTimeT(AccessTime);
+  Attr.att_mtimechg = 1;
+  Attr.att_mtime = sys::toTimeT(ModificationTime);
+  if (::__fchattr(FD, &Attr, sizeof(Attr)) != 0)
+    return std::error_code(errno, std::generic_category());
+  return std::error_code();
 #else
 #warning Missing futimes() and futimens()
   return make_error_code(errc::function_not_supported);

From 3a2b05f9fe74fcf9560632cf2695058d47d8683b Mon Sep 17 00:00:00 2001
From: Evgeniy Brevnov <ybrevnov@azul.com>
Date: Fri, 24 Jul 2020 18:57:10 +0700
Subject: [PATCH 0289/1035] [BPI][NFC] Consolidate code to deal with SCCs under
 a dedicated data structure.

In order to facilitate review of D79485 here is a small NFC change which restructures code around handling of SCCs in BPI.

Reviewed By: davidxl

Differential Revision: https://reviews.llvm.org/D84514
---
 .../llvm/Analysis/BranchProbabilityInfo.h     |  71 +++++++-
 llvm/lib/Analysis/BranchProbabilityInfo.cpp   | 164 +++++++++++-------
 2 files changed, 169 insertions(+), 66 deletions(-)

diff --git a/llvm/include/llvm/Analysis/BranchProbabilityInfo.h b/llvm/include/llvm/Analysis/BranchProbabilityInfo.h
index 3e72afba36c30..7feb5b6259380 100644
--- a/llvm/include/llvm/Analysis/BranchProbabilityInfo.h
+++ b/llvm/include/llvm/Analysis/BranchProbabilityInfo.h
@@ -151,13 +151,66 @@ class BranchProbabilityInfo {
   /// Forget analysis results for the given basic block.
   void eraseBlock(const BasicBlock *BB);
 
-  // Use to track SCCs for handling irreducible loops.
-  using SccMap = DenseMap<const BasicBlock *, int>;
-  using SccHeaderMap = DenseMap<const BasicBlock *, bool>;
-  using SccHeaderMaps = std::vector<SccHeaderMap>;
-  struct SccInfo {
+  class SccInfo {
+    // Enum of types to classify basic blocks in SCC. Basic block belonging to
+    // SCC is 'Inner' until it is either 'Header' or 'Exiting'. Note that a
+    // basic block can be 'Header' and 'Exiting' at the same time.
+    enum SccBlockType {
+      Inner = 0x0,
+      Header = 0x1,
+      Exiting = 0x2,
+    };
+    // Map of basic blocks to SCC IDs they belong to. If basic block doesn't
+    // belong to any SCC it is not in the map.
+    using SccMap = DenseMap<const BasicBlock *, int>;
+    // Each basic block in SCC is attributed with one or several types from
+    // SccBlockType. Map value has uint32_t type (instead of SccBlockType)
+    // since basic block may be for example "Header" and "Exiting" at the same
+    // time and we need to be able to keep more than one value from
+    // SccBlockType.
+    using SccBlockTypeMap = DenseMap<const BasicBlock *, uint32_t>;
+    // Vector containing classification of basic blocks for all  SCCs where i'th
+    // vector element corresponds to SCC with ID equal to i.
+    using SccBlockTypeMaps = std::vector<SccBlockTypeMap>;
+
     SccMap SccNums;
-    SccHeaderMaps SccHeaders;
+    SccBlockTypeMaps SccBlocks;
+
+  public:
+    explicit SccInfo(const Function &F);
+
+    /// If \p BB belongs to some SCC then ID of that SCC is returned, otherwise
+    /// -1 is returned. If \p BB belongs to more than one SCC at the same time
+    /// result is undefined.
+    int getSCCNum(const BasicBlock *BB) const;
+    /// Returns true if \p BB is a 'header' block in SCC with \p SccNum ID,
+    /// false otherwise.
+    bool isSCCHeader(const BasicBlock *BB, int SccNum) const {
+      return getSccBlockType(BB, SccNum) & Header;
+    }
+    /// Returns true if \p BB is an 'exiting' block in SCC with \p SccNum ID,
+    /// false otherwise.
+    bool isSCCExitingBlock(const BasicBlock *BB, int SccNum) const {
+      return getSccBlockType(BB, SccNum) & Exiting;
+    }
+    /// Fills in \p Enters vector with all such blocks that don't belong to
+    /// SCC with \p SccNum ID but there is an edge to a block belonging to the
+    /// SCC.
+    void getSccEnterBlocks(int SccNum,
+                           SmallVectorImpl<BasicBlock *> &Enters) const;
+    /// Fills in \p Exits vector with all such blocks that don't belong to
+    /// SCC with \p SccNum ID but there is an edge from a block belonging to the
+    /// SCC.
+    void getSccExitBlocks(int SccNum,
+                          SmallVectorImpl<BasicBlock *> &Exits) const;
+
+  private:
+    /// Returns \p BB's type according to classification given by SccBlockType
+    /// enum. Please note that \p BB must belong to SSC with \p SccNum ID.
+    uint32_t getSccBlockType(const BasicBlock *BB, int SccNum) const;
+    /// Calculates \p BB's type and stores it in internal data structures for
+    /// future use. Please note that \p BB must belong to SSC with \p SccNum ID.
+    void calculateSccBlockType(const BasicBlock *BB, int SccNum);
   };
 
 private:
@@ -196,6 +249,9 @@ class BranchProbabilityInfo {
   /// Track the last function we run over for printing.
   const Function *LastF = nullptr;
 
+  /// Keeps information about all SCCs in a function.
+  std::unique_ptr<const SccInfo> SccI;
+
   /// Track the set of blocks directly succeeded by a returning block.
   SmallPtrSet<const BasicBlock *, 16> PostDominatedByUnreachable;
 
@@ -210,8 +266,7 @@ class BranchProbabilityInfo {
   bool calcMetadataWeights(const BasicBlock *BB);
   bool calcColdCallHeuristics(const BasicBlock *BB);
   bool calcPointerHeuristics(const BasicBlock *BB);
-  bool calcLoopBranchHeuristics(const BasicBlock *BB, const LoopInfo &LI,
-                                SccInfo &SccI);
+  bool calcLoopBranchHeuristics(const BasicBlock *BB, const LoopInfo &LI);
   bool calcZeroHeuristics(const BasicBlock *BB, const TargetLibraryInfo *TLI);
   bool calcFloatingPointHeuristics(const BasicBlock *BB);
   bool calcInvokeHeuristics(const BasicBlock *BB);
diff --git a/llvm/lib/Analysis/BranchProbabilityInfo.cpp b/llvm/lib/Analysis/BranchProbabilityInfo.cpp
index a396b5ad21c6a..195fc69d9601d 100644
--- a/llvm/lib/Analysis/BranchProbabilityInfo.cpp
+++ b/llvm/lib/Analysis/BranchProbabilityInfo.cpp
@@ -148,6 +148,105 @@ static const uint32_t IH_TAKEN_WEIGHT = 1024 * 1024 - 1;
 /// instruction. This is essentially never taken.
 static const uint32_t IH_NONTAKEN_WEIGHT = 1;
 
+BranchProbabilityInfo::SccInfo::SccInfo(const Function &F) {
+  // Record SCC numbers of blocks in the CFG to identify irreducible loops.
+  // FIXME: We could only calculate this if the CFG is known to be irreducible
+  // (perhaps cache this info in LoopInfo if we can easily calculate it there?).
+  int SccNum = 0;
+  for (scc_iterator<const Function *> It = scc_begin(&F); !It.isAtEnd();
+       ++It, ++SccNum) {
+    // Ignore single-block SCCs since they either aren't loops or LoopInfo will
+    // catch them.
+    const std::vector<const BasicBlock *> &Scc = *It;
+    if (Scc.size() == 1)
+      continue;
+
+    LLVM_DEBUG(dbgs() << "BPI: SCC " << SccNum << ":");
+    for (const auto *BB : Scc) {
+      LLVM_DEBUG(dbgs() << " " << BB->getName());
+      SccNums[BB] = SccNum;
+      calculateSccBlockType(BB, SccNum);
+    }
+    LLVM_DEBUG(dbgs() << "\n");
+  }
+}
+
+int BranchProbabilityInfo::SccInfo::getSCCNum(const BasicBlock *BB) const {
+  auto SccIt = SccNums.find(BB);
+  if (SccIt == SccNums.end())
+    return -1;
+  return SccIt->second;
+}
+
+void BranchProbabilityInfo::SccInfo::getSccEnterBlocks(
+    int SccNum, SmallVectorImpl<BasicBlock *> &Enters) const {
+
+  for (auto MapIt : SccBlocks[SccNum]) {
+    const auto *BB = MapIt.first;
+    if (isSCCHeader(BB, SccNum))
+      for (const auto *Pred : predecessors(BB))
+        if (getSCCNum(Pred) != SccNum)
+          Enters.push_back(const_cast<BasicBlock *>(BB));
+  }
+}
+
+void BranchProbabilityInfo::SccInfo::getSccExitBlocks(
+    int SccNum, SmallVectorImpl<BasicBlock *> &Exits) const {
+  for (auto MapIt : SccBlocks[SccNum]) {
+    const auto *BB = MapIt.first;
+    if (isSCCExitingBlock(BB, SccNum))
+      for (const auto *Succ : successors(BB))
+        if (getSCCNum(Succ) != SccNum)
+          Exits.push_back(const_cast<BasicBlock *>(BB));
+  }
+}
+
+uint32_t BranchProbabilityInfo::SccInfo::getSccBlockType(const BasicBlock *BB,
+                                                         int SccNum) const {
+  assert(getSCCNum(BB) == SccNum);
+
+  assert(SccBlocks.size() > static_cast<unsigned>(SccNum) && "Unknown SCC");
+  const auto &SccBlockTypes = SccBlocks[SccNum];
+
+  auto It = SccBlockTypes.find(BB);
+  if (It != SccBlockTypes.end()) {
+    return It->second;
+  }
+  return Inner;
+}
+
+void BranchProbabilityInfo::SccInfo::calculateSccBlockType(const BasicBlock *BB,
+                                                           int SccNum) {
+  assert(getSCCNum(BB) == SccNum);
+  uint32_t BlockType = Inner;
+
+  if (llvm::any_of(make_range(pred_begin(BB), pred_end(BB)),
+                   [&](const BasicBlock *Pred) {
+        // Consider any block that is an entry point to the SCC as
+        // a header.
+        return getSCCNum(Pred) != SccNum;
+      }))
+    BlockType |= Header;
+
+  if (llvm::any_of(
+          make_range(succ_begin(BB), succ_end(BB)),
+          [&](const BasicBlock *Succ) { return getSCCNum(Succ) != SccNum; }))
+    BlockType |= Exiting;
+
+  // Lazily compute the set of headers for a given SCC and cache the results
+  // in the SccHeaderMap.
+  if (SccBlocks.size() <= static_cast<unsigned>(SccNum))
+    SccBlocks.resize(SccNum + 1);
+  auto &SccBlockTypes = SccBlocks[SccNum];
+
+  if (BlockType != Inner) {
+    bool IsInserted;
+    std::tie(std::ignore, IsInserted) =
+        SccBlockTypes.insert(std::make_pair(BB, BlockType));
+    assert(IsInserted && "Duplicated block in SCC");
+  }
+}
+
 static void UpdatePDTWorklist(const BasicBlock *BB, PostDominatorTree *PDT,
                               SmallVectorImpl<const BasicBlock *> &WorkList,
                               SmallPtrSetImpl<const BasicBlock *> &TargetSet) {
@@ -511,38 +610,6 @@ bool BranchProbabilityInfo::calcPointerHeuristics(const BasicBlock *BB) {
   return true;
 }
 
-static int getSCCNum(const BasicBlock *BB,
-                     const BranchProbabilityInfo::SccInfo &SccI) {
-  auto SccIt = SccI.SccNums.find(BB);
-  if (SccIt == SccI.SccNums.end())
-    return -1;
-  return SccIt->second;
-}
-
-// Consider any block that is an entry point to the SCC as a header.
-static bool isSCCHeader(const BasicBlock *BB, int SccNum,
-                        BranchProbabilityInfo::SccInfo &SccI) {
-  assert(getSCCNum(BB, SccI) == SccNum);
-
-  // Lazily compute the set of headers for a given SCC and cache the results
-  // in the SccHeaderMap.
-  if (SccI.SccHeaders.size() <= static_cast<unsigned>(SccNum))
-    SccI.SccHeaders.resize(SccNum + 1);
-  auto &HeaderMap = SccI.SccHeaders[SccNum];
-  bool Inserted;
-  BranchProbabilityInfo::SccHeaderMap::iterator HeaderMapIt;
-  std::tie(HeaderMapIt, Inserted) = HeaderMap.insert(std::make_pair(BB, false));
-  if (Inserted) {
-    bool IsHeader = llvm::any_of(make_range(pred_begin(BB), pred_end(BB)),
-                                 [&](const BasicBlock *Pred) {
-                                   return getSCCNum(Pred, SccI) != SccNum;
-                                 });
-    HeaderMapIt->second = IsHeader;
-    return IsHeader;
-  } else
-    return HeaderMapIt->second;
-}
-
 // Compute the unlikely successors to the block BB in the loop L, specifically
 // those that are unlikely because this is a loop, and add them to the
 // UnlikelyBlocks set.
@@ -653,12 +720,11 @@ computeUnlikelySuccessors(const BasicBlock *BB, Loop *L,
 // Calculate Edge Weights using "Loop Branch Heuristics". Predict backedges
 // as taken, exiting edges as not-taken.
 bool BranchProbabilityInfo::calcLoopBranchHeuristics(const BasicBlock *BB,
-                                                     const LoopInfo &LI,
-                                                     SccInfo &SccI) {
+                                                     const LoopInfo &LI) {
   int SccNum;
   Loop *L = LI.getLoopFor(BB);
   if (!L) {
-    SccNum = getSCCNum(BB, SccI);
+    SccNum = SccI->getSCCNum(BB);
     if (SccNum < 0)
       return false;
   }
@@ -685,9 +751,9 @@ bool BranchProbabilityInfo::calcLoopBranchHeuristics(const BasicBlock *BB,
       else
         InEdges.push_back(I.getSuccessorIndex());
     } else {
-      if (getSCCNum(*I, SccI) != SccNum)
+      if (SccI->getSCCNum(*I) != SccNum)
         ExitingEdges.push_back(I.getSuccessorIndex());
-      else if (isSCCHeader(*I, SccNum, SccI))
+      else if (SccI->isSCCHeader(*I, SccNum))
         BackEdges.push_back(I.getSuccessorIndex());
       else
         InEdges.push_back(I.getSuccessorIndex());
@@ -1072,26 +1138,7 @@ void BranchProbabilityInfo::calculate(const Function &F, const LoopInfo &LI,
   assert(PostDominatedByUnreachable.empty());
   assert(PostDominatedByColdCall.empty());
 
-  // Record SCC numbers of blocks in the CFG to identify irreducible loops.
-  // FIXME: We could only calculate this if the CFG is known to be irreducible
-  // (perhaps cache this info in LoopInfo if we can easily calculate it there?).
-  int SccNum = 0;
-  SccInfo SccI;
-  for (scc_iterator<const Function *> It = scc_begin(&F); !It.isAtEnd();
-       ++It, ++SccNum) {
-    // Ignore single-block SCCs since they either aren't loops or LoopInfo will
-    // catch them.
-    const std::vector<const BasicBlock *> &Scc = *It;
-    if (Scc.size() == 1)
-      continue;
-
-    LLVM_DEBUG(dbgs() << "BPI: SCC " << SccNum << ":");
-    for (auto *BB : Scc) {
-      LLVM_DEBUG(dbgs() << " " << BB->getName());
-      SccI.SccNums[BB] = SccNum;
-    }
-    LLVM_DEBUG(dbgs() << "\n");
-  }
+  SccI = std::make_unique<SccInfo>(F);
 
   std::unique_ptr<PostDominatorTree> PDTPtr;
 
@@ -1119,7 +1166,7 @@ void BranchProbabilityInfo::calculate(const Function &F, const LoopInfo &LI,
       continue;
     if (calcColdCallHeuristics(BB))
       continue;
-    if (calcLoopBranchHeuristics(BB, LI, SccI))
+    if (calcLoopBranchHeuristics(BB, LI))
       continue;
     if (calcPointerHeuristics(BB))
       continue;
@@ -1131,6 +1178,7 @@ void BranchProbabilityInfo::calculate(const Function &F, const LoopInfo &LI,
 
   PostDominatedByUnreachable.clear();
   PostDominatedByColdCall.clear();
+  SccI.release();
 
   if (PrintBranchProb &&
       (PrintBranchProbFuncName.empty() ||

From 97470897c436a6a5d682fb8ab296d0bcdc6e32a4 Mon Sep 17 00:00:00 2001
From: Stefan Pintilie <stefanp@ca.ibm.com>
Date: Mon, 27 Jul 2020 12:27:30 -0500
Subject: [PATCH 0290/1035] [PowerPC] Split s34imm into two types

Currently the instruction paddi always takes s34imm as the type for the
34 bit immediate. However, the PC Relative form of the instruction should
not produce the same fixup as the non PC Relative form.
This patch splits the s34imm type into s34imm and s34imm_pcrel so that two
different fixups can be emitted.

Reviewed By: nemanjai, #powerpc, kamaub

Differential Revision: https://reviews.llvm.org/D83255
---
 .../PowerPC/MCTargetDesc/PPCAsmBackend.cpp    |  4 +++
 .../MCTargetDesc/PPCELFObjectWriter.cpp       |  3 ++
 .../PowerPC/MCTargetDesc/PPCFixupKinds.h      |  3 ++
 .../PowerPC/MCTargetDesc/PPCMCCodeEmitter.cpp | 30 ++++++++++++++-----
 .../PowerPC/MCTargetDesc/PPCMCCodeEmitter.h   |  9 +++++-
 llvm/lib/Target/PowerPC/PPCInstrInfo.td       |  8 ++++-
 llvm/lib/Target/PowerPC/PPCInstrPrefix.td     |  4 +--
 llvm/test/MC/PowerPC/ppc64-errors-emit-obj.s  |  7 +++++
 8 files changed, 57 insertions(+), 11 deletions(-)
 create mode 100644 llvm/test/MC/PowerPC/ppc64-errors-emit-obj.s

diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp
index dbaf221db9fc9..59cb2b994a4b3 100644
--- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp
+++ b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp
@@ -46,6 +46,7 @@ static uint64_t adjustFixupValue(unsigned Kind, uint64_t Value) {
   case PPC::fixup_ppc_half16ds:
     return Value & 0xfffc;
   case PPC::fixup_ppc_pcrel34:
+  case PPC::fixup_ppc_imm34:
     return Value & 0x3ffffffff;
   }
 }
@@ -68,6 +69,7 @@ static unsigned getFixupKindNumBytes(unsigned Kind) {
   case PPC::fixup_ppc_br24_notoc:
     return 4;
   case PPC::fixup_ppc_pcrel34:
+  case PPC::fixup_ppc_imm34:
   case FK_Data_8:
     return 8;
   case PPC::fixup_ppc_nofixup:
@@ -100,6 +102,7 @@ class PPCAsmBackend : public MCAsmBackend {
       { "fixup_ppc_half16",       0,     16,   0 },
       { "fixup_ppc_half16ds",     0,     14,   0 },
       { "fixup_ppc_pcrel34",     0,      34,   MCFixupKindInfo::FKF_IsPCRel },
+      { "fixup_ppc_imm34",       0,      34,   0 },
       { "fixup_ppc_nofixup",      0,      0,   0 }
     };
     const static MCFixupKindInfo InfosLE[PPC::NumTargetFixupKinds] = {
@@ -112,6 +115,7 @@ class PPCAsmBackend : public MCAsmBackend {
       { "fixup_ppc_half16",      0,      16,   0 },
       { "fixup_ppc_half16ds",    2,      14,   0 },
       { "fixup_ppc_pcrel34",     0,      34,   MCFixupKindInfo::FKF_IsPCRel },
+      { "fixup_ppc_imm34",       0,      34,   0 },
       { "fixup_ppc_nofixup",     0,       0,   0 }
     };
 
diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCELFObjectWriter.cpp b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCELFObjectWriter.cpp
index d8b3301e97f12..1cd190c6b04ea 100644
--- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCELFObjectWriter.cpp
+++ b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCELFObjectWriter.cpp
@@ -409,6 +409,9 @@ unsigned PPCELFObjectWriter::getRelocType(MCContext &Ctx, const MCValue &Target,
         break;
       }
       break;
+    case PPC::fixup_ppc_imm34:
+      report_fatal_error("Unsupported Modifier for fixup_ppc_imm34.");
+      break;
     case FK_Data_8:
       switch (Modifier) {
       default: llvm_unreachable("Unsupported Modifier");
diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCFixupKinds.h b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCFixupKinds.h
index 2fb8947fd4e0f..73292f7b7938f 100644
--- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCFixupKinds.h
+++ b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCFixupKinds.h
@@ -43,6 +43,9 @@ enum Fixups {
   // A 34-bit fixup corresponding to PC-relative paddi.
   fixup_ppc_pcrel34,
 
+  // A 34-bit fixup corresponding to Non-PC-relative paddi.
+  fixup_ppc_imm34,
+
   /// Not a true fixup, but ties a symbol to a call to __tls_get_addr for the
   /// TLS general and local dynamic models, or inserts the thread-pointer
   /// register number.
diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCCodeEmitter.cpp b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCCodeEmitter.cpp
index fb65e7320f2b0..8c0e0a80b1e2c 100644
--- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCCodeEmitter.cpp
+++ b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCCodeEmitter.cpp
@@ -104,20 +104,36 @@ unsigned PPCMCCodeEmitter::getImm16Encoding(const MCInst &MI, unsigned OpNo,
   return 0;
 }
 
-uint64_t
-PPCMCCodeEmitter::getImm34Encoding(const MCInst &MI, unsigned OpNo,
-                                   SmallVectorImpl<MCFixup> &Fixups,
-                                   const MCSubtargetInfo &STI) const {
+uint64_t PPCMCCodeEmitter::getImm34Encoding(const MCInst &MI, unsigned OpNo,
+                                            SmallVectorImpl<MCFixup> &Fixups,
+                                            const MCSubtargetInfo &STI,
+                                            MCFixupKind Fixup) const {
   const MCOperand &MO = MI.getOperand(OpNo);
-  if (MO.isReg() || MO.isImm())
+  assert(!MO.isReg() && "Not expecting a register for this operand.");
+  if (MO.isImm())
     return getMachineOpValue(MI, MO, Fixups, STI);
 
   // Add a fixup for the immediate field.
-  Fixups.push_back(MCFixup::create(0, MO.getExpr(),
-                                   (MCFixupKind)PPC::fixup_ppc_pcrel34));
+  Fixups.push_back(MCFixup::create(0, MO.getExpr(), Fixup));
   return 0;
 }
 
+uint64_t
+PPCMCCodeEmitter::getImm34EncodingNoPCRel(const MCInst &MI, unsigned OpNo,
+                                          SmallVectorImpl<MCFixup> &Fixups,
+                                          const MCSubtargetInfo &STI) const {
+  return getImm34Encoding(MI, OpNo, Fixups, STI,
+                          (MCFixupKind)PPC::fixup_ppc_imm34);
+}
+
+uint64_t
+PPCMCCodeEmitter::getImm34EncodingPCRel(const MCInst &MI, unsigned OpNo,
+                                        SmallVectorImpl<MCFixup> &Fixups,
+                                        const MCSubtargetInfo &STI) const {
+  return getImm34Encoding(MI, OpNo, Fixups, STI,
+                          (MCFixupKind)PPC::fixup_ppc_pcrel34);
+}
+
 unsigned PPCMCCodeEmitter::getMemRIEncoding(const MCInst &MI, unsigned OpNo,
                                             SmallVectorImpl<MCFixup> &Fixups,
                                             const MCSubtargetInfo &STI) const {
diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCCodeEmitter.h b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCCodeEmitter.h
index 588aa76bd8064..4504cc6a7405e 100644
--- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCCodeEmitter.h
+++ b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCCodeEmitter.h
@@ -52,7 +52,14 @@ class PPCMCCodeEmitter : public MCCodeEmitter {
                             const MCSubtargetInfo &STI) const;
   uint64_t getImm34Encoding(const MCInst &MI, unsigned OpNo,
                             SmallVectorImpl<MCFixup> &Fixups,
-                            const MCSubtargetInfo &STI) const;
+                            const MCSubtargetInfo &STI,
+                            MCFixupKind Fixup) const;
+  uint64_t getImm34EncodingNoPCRel(const MCInst &MI, unsigned OpNo,
+                                   SmallVectorImpl<MCFixup> &Fixups,
+                                   const MCSubtargetInfo &STI) const;
+  uint64_t getImm34EncodingPCRel(const MCInst &MI, unsigned OpNo,
+                                 SmallVectorImpl<MCFixup> &Fixups,
+                                 const MCSubtargetInfo &STI) const;
   unsigned getMemRIEncoding(const MCInst &MI, unsigned OpNo,
                             SmallVectorImpl<MCFixup> &Fixups,
                             const MCSubtargetInfo &STI) const;
diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.td b/llvm/lib/Target/PowerPC/PPCInstrInfo.td
index c565758973bf5..f807d61c75d23 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrInfo.td
+++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.td
@@ -757,7 +757,13 @@ def PPCS34ImmAsmOperand : AsmOperandClass {
 }
 def s34imm : Operand<i64> {
   let PrintMethod = "printS34ImmOperand";
-  let EncoderMethod = "getImm34Encoding";
+  let EncoderMethod = "getImm34EncodingNoPCRel";
+  let ParserMatchClass = PPCS34ImmAsmOperand;
+  let DecoderMethod = "decodeSImmOperand<34>";
+}
+def s34imm_pcrel : Operand<i64> {
+  let PrintMethod = "printS34ImmOperand";
+  let EncoderMethod = "getImm34EncodingPCRel";
   let ParserMatchClass = PPCS34ImmAsmOperand;
   let DecoderMethod = "decodeSImmOperand<34>";
 }
diff --git a/llvm/lib/Target/PowerPC/PPCInstrPrefix.td b/llvm/lib/Target/PowerPC/PPCInstrPrefix.td
index 418ef3b377282..fa21c54efc28b 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrPrefix.td
+++ b/llvm/lib/Target/PowerPC/PPCInstrPrefix.td
@@ -459,7 +459,7 @@ let Predicates = [PrefixInstrs] in {
   let Interpretation64Bit = 1, isCodeGenOnly = 1 in {
     defm PADDI8 :
       MLS_DForm_R_SI34_RTA5_p<14, (outs g8rc:$RT), (ins g8rc:$RA, s34imm:$SI),
-                              (ins immZero:$RA, s34imm:$SI),
+                              (ins immZero:$RA, s34imm_pcrel:$SI),
                               "paddi $RT, $RA, $SI", IIC_LdStLFD>;
     let isReMaterializable = 1, isAsCheapAsAMove = 1, isMoveImm = 1 in {
       def PLI8 : MLS_DForm_SI34_RT5<14, (outs g8rc:$RT),
@@ -469,7 +469,7 @@ let Predicates = [PrefixInstrs] in {
   }
   defm PADDI :
     MLS_DForm_R_SI34_RTA5_p<14, (outs gprc:$RT), (ins gprc:$RA, s34imm:$SI),
-                            (ins immZero:$RA, s34imm:$SI),
+                            (ins immZero:$RA, s34imm_pcrel:$SI),
                             "paddi $RT, $RA, $SI", IIC_LdStLFD>;
   let isReMaterializable = 1, isAsCheapAsAMove = 1, isMoveImm = 1 in {
     def PLI : MLS_DForm_SI34_RT5<14, (outs gprc:$RT),
diff --git a/llvm/test/MC/PowerPC/ppc64-errors-emit-obj.s b/llvm/test/MC/PowerPC/ppc64-errors-emit-obj.s
new file mode 100644
index 0000000000000..0d2c879380e0a
--- /dev/null
+++ b/llvm/test/MC/PowerPC/ppc64-errors-emit-obj.s
@@ -0,0 +1,7 @@
+# RUN: not --crash llvm-mc -triple powerpc64-- --filetype=obj < %s 2> %t
+# RUN: FileCheck < %t %s
+# RUN: not --crash llvm-mc -triple powerpc64le-- --filetype=obj < %s 2> %t
+# RUN: FileCheck < %t %s
+
+# CHECK: Unsupported Modifier for fixup_ppc_imm34.
+paddi 3, 13, symbol@toc, 0

From 443e734fb98df422c90cbc8177520a8182597912 Mon Sep 17 00:00:00 2001
From: Rainer Orth <ro@gcc.gnu.org>
Date: Tue, 28 Jul 2020 13:21:36 +0200
Subject: [PATCH 0291/1035] [compiler-rt][cmake] Don't pass --version-script to
 Illumos ld

Neither the Illumos `ld` nor the Solaris 11.3 one support the `--version-script` and
`z gnu-linker-script-compat` options, which breaks the `compiler-rt` build.

This patch checks for both options instead of hardcoding their use.

Tested on `amd-pc-solaris2.11` (all of Solaris 11.4, 11.3, and Illumos).

Differential Revision: https://reviews.llvm.org/D84559
---
 compiler-rt/cmake/config-ix.cmake    | 13 +++++++++++++
 compiler-rt/lib/asan/CMakeLists.txt  |  4 ++--
 compiler-rt/lib/ubsan/CMakeLists.txt |  4 ++--
 3 files changed, 17 insertions(+), 4 deletions(-)

diff --git a/compiler-rt/cmake/config-ix.cmake b/compiler-rt/cmake/config-ix.cmake
index 0a27910ed4943..74fef8933ef90 100644
--- a/compiler-rt/cmake/config-ix.cmake
+++ b/compiler-rt/cmake/config-ix.cmake
@@ -157,6 +157,19 @@ check_library_exists(stdc++ __cxa_throw "" COMPILER_RT_HAS_LIBSTDCXX)
 check_linker_flag("-Wl,-z,text" COMPILER_RT_HAS_Z_TEXT)
 check_linker_flag("-fuse-ld=lld" COMPILER_RT_HAS_FUSE_LD_LLD_FLAG)
 
+set(VERS_COMPAT_OPTION "-Wl,-z,gnu-version-script-compat")
+check_linker_flag("${VERS_COMPAT_OPTION}" COMPILER_RT_HAS_GNU_VERSION_SCRIPT_COMPAT)
+
+set(DUMMY_VERS ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/dummy.vers)
+file(WRITE ${DUMMY_VERS} "{};")
+set(VERS_OPTION "-Wl,--version-script,${DUMMY_VERS}")
+if(COMPILER_RT_HAS_GNU_VERSION_SCRIPT_COMPAT)
+  # Solaris 11.4 ld only supports --version-script with
+  # -z gnu-version-script-compat. 
+  string(APPEND VERS_OPTION " ${VERS_COMPAT_OPTION}")
+endif()
+check_linker_flag("${VERS_OPTION}" COMPILER_RT_HAS_VERSION_SCRIPT)
+
 if(ANDROID)
   check_linker_flag("-Wl,-z,global" COMPILER_RT_HAS_Z_GLOBAL)
   check_library_exists(log __android_log_write "" COMPILER_RT_HAS_LIBLOG)
diff --git a/compiler-rt/lib/asan/CMakeLists.txt b/compiler-rt/lib/asan/CMakeLists.txt
index 2a1bbb58cce41..0c29893ebfe23 100644
--- a/compiler-rt/lib/asan/CMakeLists.txt
+++ b/compiler-rt/lib/asan/CMakeLists.txt
@@ -224,7 +224,7 @@ else()
     PARENT_TARGET asan)
 
   foreach(arch ${ASAN_SUPPORTED_ARCH})
-    if (UNIX)
+    if (UNIX AND COMPILER_RT_HAS_VERSION_SCRIPT)
       add_sanitizer_rt_version_list(clang_rt.asan-dynamic-${arch}
                                     LIBS clang_rt.asan-${arch} clang_rt.asan_cxx-${arch}
                                     EXTRA asan.syms.extra)
@@ -232,7 +232,7 @@ else()
            -Wl,--version-script,${CMAKE_CURRENT_BINARY_DIR}/clang_rt.asan-dynamic-${arch}.vers)
       # The Solaris 11.4 linker supports a subset of GNU ld version scripts,
       # but requires a special option to enable it.
-      if (OS_NAME MATCHES "SunOS")
+      if (COMPILER_RT_HAS_GNU_VERSION_SCRIPT_COMPAT)
           list(APPEND VERSION_SCRIPT_FLAG -Wl,-z,gnu-version-script-compat)
       endif()
       set_property(SOURCE
diff --git a/compiler-rt/lib/ubsan/CMakeLists.txt b/compiler-rt/lib/ubsan/CMakeLists.txt
index dca02a65e9713..00b74a63cbe53 100644
--- a/compiler-rt/lib/ubsan/CMakeLists.txt
+++ b/compiler-rt/lib/ubsan/CMakeLists.txt
@@ -200,7 +200,7 @@ else()
       CFLAGS ${UBSAN_CXXFLAGS}
       PARENT_TARGET ubsan)
 
-    if (FUCHSIA OR UNIX)
+    if ((FUCHSIA OR UNIX) AND COMPILER_RT_HAS_VERSION_SCRIPT)
       file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/dummy.cpp "")
       add_compiler_rt_object_libraries(RTUbsan_dynamic_version_script_dummy
         ARCHS ${UBSAN_SUPPORTED_ARCH}
@@ -216,7 +216,7 @@ else()
             -Wl,--version-script,${CMAKE_CURRENT_BINARY_DIR}/clang_rt.ubsan_standalone-dynamic-${arch}.vers)
         # The Solaris 11.4 linker supports a subset of GNU ld version scripts,
         # but requires a special option to enable it.
-        if (OS_NAME MATCHES "SunOS")
+        if (COMPILER_RT_HAS_GNU_VERSION_SCRIPT_COMPAT)
             list(APPEND VERSION_SCRIPT_FLAG -Wl,-z,gnu-version-script-compat)
         endif()
         set_property(SOURCE

From 0f62a53db64a943972e51d3d58610595d22779fd Mon Sep 17 00:00:00 2001
From: Luofan Chen <clfbbn@gmail.com>
Date: Tue, 28 Jul 2020 19:19:23 +0800
Subject: [PATCH 0292/1035] [Attributor] Add override keyword to the print
 function of AA

The print() function in the `AbstractAttribute` structure overrides
the function in the `AADepGraphNode`, so we need to mark it as
override.

This should fix a buildbot failure introduced by 5ee07dc.
---
 llvm/include/llvm/Transforms/IPO/Attributor.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/include/llvm/Transforms/IPO/Attributor.h b/llvm/include/llvm/Transforms/IPO/Attributor.h
index 48c65c37eec78..a8076d33018cd 100644
--- a/llvm/include/llvm/Transforms/IPO/Attributor.h
+++ b/llvm/include/llvm/Transforms/IPO/Attributor.h
@@ -2138,7 +2138,7 @@ struct AbstractAttribute : public IRPosition, public AADepGraphNode {
 
   /// Helper functions, for debug purposes only.
   ///{
-  virtual void print(raw_ostream &OS) const;
+  void print(raw_ostream &OS) const override;
   virtual void printWithDeps(raw_ostream &OS) const;
   void dump() const { print(dbgs()); }
 

From d3557ecede8e72b60df6a6d933d6c6ec16c48154 Mon Sep 17 00:00:00 2001
From: Anirudh Prasad <anirudh.prasad@ibm.com>
Date: Tue, 28 Jul 2020 06:40:47 -0400
Subject: [PATCH 0293/1035] [Support] Use InitLLVM in llvm-stress, sancov and
 TableGen

This patch refactors the llvm tools namely, llvm-stress and sancov,
as well as the llvm TableGen utility, to use the new InitLLVM
interface which encapsulates PrettyStackTrace.

This is from https://reviews.llvm.org/D70702, but only for LLVM.

Reviewed-by: Kai

Differential Revision: https://reviews.llvm.org/D83484
---
 llvm/tools/llvm-stress/llvm-stress.cpp | 7 ++-----
 llvm/tools/sancov/sancov.cpp           | 9 ++-------
 llvm/utils/TableGen/TableGen.cpp       | 9 ++-------
 3 files changed, 6 insertions(+), 19 deletions(-)

diff --git a/llvm/tools/llvm-stress/llvm-stress.cpp b/llvm/tools/llvm-stress/llvm-stress.cpp
index 22f530dde1670..77cc59839129a 100644
--- a/llvm/tools/llvm-stress/llvm-stress.cpp
+++ b/llvm/tools/llvm-stress/llvm-stress.cpp
@@ -38,8 +38,7 @@
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/FileSystem.h"
-#include "llvm/Support/ManagedStatic.h"
-#include "llvm/Support/PrettyStackTrace.h"
+#include "llvm/Support/InitLLVM.h"
 #include "llvm/Support/ToolOutputFile.h"
 #include "llvm/Support/raw_ostream.h"
 #include <algorithm>
@@ -733,10 +732,8 @@ static void IntroduceControlFlow(Function *F, Random &R) {
 int main(int argc, char **argv) {
   using namespace llvm;
 
-  // Init LLVM, call llvm_shutdown() on exit, parse args, etc.
-  PrettyStackTraceProgram X(argc, argv);
+  InitLLVM X(argc, argv);
   cl::ParseCommandLineOptions(argc, argv, "llvm codegen stress-tester\n");
-  llvm_shutdown_obj Y;
 
   auto M = std::make_unique<Module>("/tmp/autogen.bc", Context);
   Function *F = GenEmptyFunction(M.get());
diff --git a/llvm/tools/sancov/sancov.cpp b/llvm/tools/sancov/sancov.cpp
index 37d06e385475e..f1d756f216d1a 100644
--- a/llvm/tools/sancov/sancov.cpp
+++ b/llvm/tools/sancov/sancov.cpp
@@ -32,15 +32,13 @@
 #include "llvm/Support/Errc.h"
 #include "llvm/Support/ErrorOr.h"
 #include "llvm/Support/FileSystem.h"
+#include "llvm/Support/InitLLVM.h"
 #include "llvm/Support/JSON.h"
 #include "llvm/Support/MD5.h"
-#include "llvm/Support/ManagedStatic.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Path.h"
-#include "llvm/Support/PrettyStackTrace.h"
 #include "llvm/Support/Regex.h"
 #include "llvm/Support/SHA1.h"
-#include "llvm/Support/Signals.h"
 #include "llvm/Support/SourceMgr.h"
 #include "llvm/Support/SpecialCaseList.h"
 #include "llvm/Support/TargetRegistry.h"
@@ -1134,10 +1132,7 @@ readSymbolizeAndMergeCmdArguments(std::vector<std::string> FileNames) {
 } // namespace
 
 int main(int Argc, char **Argv) {
-  // Print stack trace if we signal out.
-  sys::PrintStackTraceOnErrorSignal(Argv[0]);
-  PrettyStackTraceProgram X(Argc, Argv);
-  llvm_shutdown_obj Y; // Call llvm_shutdown() on exit.
+  llvm::InitLLVM X(Argc, Argv);
 
   llvm::InitializeAllTargetInfos();
   llvm::InitializeAllTargetMCs();
diff --git a/llvm/utils/TableGen/TableGen.cpp b/llvm/utils/TableGen/TableGen.cpp
index 8015a58471cad..5215c30b707f4 100644
--- a/llvm/utils/TableGen/TableGen.cpp
+++ b/llvm/utils/TableGen/TableGen.cpp
@@ -12,9 +12,7 @@
 
 #include "TableGenBackends.h" // Declares all backends.
 #include "llvm/Support/CommandLine.h"
-#include "llvm/Support/ManagedStatic.h"
-#include "llvm/Support/PrettyStackTrace.h"
-#include "llvm/Support/Signals.h"
+#include "llvm/Support/InitLLVM.h"
 #include "llvm/TableGen/Main.h"
 #include "llvm/TableGen/Record.h"
 #include "llvm/TableGen/SetTheory.h"
@@ -278,12 +276,9 @@ bool LLVMTableGenMain(raw_ostream &OS, RecordKeeper &Records) {
 }
 
 int main(int argc, char **argv) {
-  sys::PrintStackTraceOnErrorSignal(argv[0]);
-  PrettyStackTraceProgram X(argc, argv);
+  InitLLVM X(argc, argv);
   cl::ParseCommandLineOptions(argc, argv);
 
-  llvm_shutdown_obj Y;
-
   return TableGenMain(argv[0], &LLVMTableGenMain);
 }
 

From 6d10d317d8b0f1975dbb17850efd7c069f6ee8fd Mon Sep 17 00:00:00 2001
From: Stephan Herhut <herhut@google.com>
Date: Tue, 28 Jul 2020 13:09:45 +0200
Subject: [PATCH 0294/1035] [MLIR][Shape] Support transforming
 shape.num_elements on tensors

The current transformation to shape.reduce does not support tensor values.
This adds the required changes to make that work, including fixing the builder
for shape.reduce.

Differential Revision: https://reviews.llvm.org/D84744
---
 mlir/lib/Dialect/Shape/IR/Shape.cpp               |  8 +++++++-
 .../Shape/Transforms/ShapeToShapeLowering.cpp     | 13 +++++++++----
 mlir/test/Dialect/Shape/shape-to-shape.mlir       | 15 +++++++++++++++
 3 files changed, 31 insertions(+), 5 deletions(-)

diff --git a/mlir/lib/Dialect/Shape/IR/Shape.cpp b/mlir/lib/Dialect/Shape/IR/Shape.cpp
index 4887c87c1e5f1..3c71e3409923e 100644
--- a/mlir/lib/Dialect/Shape/IR/Shape.cpp
+++ b/mlir/lib/Dialect/Shape/IR/Shape.cpp
@@ -834,7 +834,13 @@ void ReduceOp::build(OpBuilder &builder, OperationState &result, Value shape,
   bodyRegion->push_back(new Block);
   Block &bodyBlock = bodyRegion->front();
   bodyBlock.addArgument(builder.getIndexType());
-  bodyBlock.addArgument(SizeType::get(builder.getContext()));
+
+  Type elementType;
+  if (auto tensorType = shape.getType().dyn_cast<TensorType>())
+    elementType = tensorType.getElementType();
+  else
+    elementType = SizeType::get(builder.getContext());
+  bodyBlock.addArgument(elementType);
 
   for (Type initValType : initVals.getTypes()) {
     bodyBlock.addArgument(initValType);
diff --git a/mlir/lib/Dialect/Shape/Transforms/ShapeToShapeLowering.cpp b/mlir/lib/Dialect/Shape/Transforms/ShapeToShapeLowering.cpp
index bb2b03b8ec081..a84fad1f94602 100644
--- a/mlir/lib/Dialect/Shape/Transforms/ShapeToShapeLowering.cpp
+++ b/mlir/lib/Dialect/Shape/Transforms/ShapeToShapeLowering.cpp
@@ -9,6 +9,7 @@
 #include "PassDetail.h"
 #include "mlir/Dialect/Shape/IR/Shape.h"
 #include "mlir/Dialect/Shape/Transforms/Passes.h"
+#include "mlir/Dialect/StandardOps/IR/Ops.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Pass/Pass.h"
@@ -32,14 +33,18 @@ LogicalResult
 NumElementsOpConverter::matchAndRewrite(NumElementsOp op,
                                         PatternRewriter &rewriter) const {
   auto loc = op.getLoc();
-  Value init = rewriter.create<ConstSizeOp>(loc, rewriter.getIndexAttr(1));
+  Type valueType = op.getResult().getType();
+  Value init = op.getDialect()
+                   ->materializeConstant(rewriter, rewriter.getIndexAttr(1),
+                                         valueType, loc)
+                   ->getResult(0);
   ReduceOp reduce = rewriter.create<ReduceOp>(loc, op.shape(), init);
 
   // Generate reduce operator.
   Block *body = reduce.getBody();
   OpBuilder b = OpBuilder::atBlockEnd(body);
-  Value product = b.create<MulOp>(loc, b.getType<SizeType>(),
-                                  body->getArgument(1), body->getArgument(2));
+  Value product = b.create<MulOp>(loc, valueType, body->getArgument(1),
+                                  body->getArgument(2));
   b.create<YieldOp>(loc, product);
 
   rewriter.replaceOp(op, reduce.result());
@@ -60,7 +65,7 @@ void ShapeToShapeLowering::runOnFunction() {
   populateShapeRewritePatterns(&ctx, patterns);
 
   ConversionTarget target(getContext());
-  target.addLegalDialect<ShapeDialect>();
+  target.addLegalDialect<ShapeDialect, StandardOpsDialect>();
   target.addIllegalOp<NumElementsOp>();
   if (failed(mlir::applyPartialConversion(getFunction(), target, patterns)))
     signalPassFailure();
diff --git a/mlir/test/Dialect/Shape/shape-to-shape.mlir b/mlir/test/Dialect/Shape/shape-to-shape.mlir
index d1b00bc12a22c..481d682942bb0 100644
--- a/mlir/test/Dialect/Shape/shape-to-shape.mlir
+++ b/mlir/test/Dialect/Shape/shape-to-shape.mlir
@@ -14,3 +14,18 @@ func @num_elements_to_reduce(%shape : !shape.shape) -> !shape.size {
 // CHECK: }
 // CHECK: return [[NUM_ELEMENTS]] : !shape.size
 
+// -----
+
+// CHECK-LABEL: func @num_elements_to_reduce_on_index
+// CHECK-SAME:  ([[ARG:%.*]]: tensor<?xindex>) -> index
+func @num_elements_to_reduce_on_index(%shape : tensor<?xindex>) -> index {
+  %num_elements = shape.num_elements %shape : tensor<?xindex> -> index
+  return %num_elements : index
+}
+// CHECK: [[C1:%.*]] = constant 1 : index
+// CHECK: [[NUM_ELEMENTS:%.*]] = shape.reduce([[ARG]], [[C1]]) : tensor<?xindex> -> index
+// CHECK: ^bb0({{.*}}: index, [[DIM:%.*]]: index, [[ACC:%.*]]: index
+// CHECK:   [[NEW_ACC:%.*]] = shape.mul [[DIM]], [[ACC]]
+// CHECK:   shape.yield [[NEW_ACC]] : index
+// CHECK: }
+// CHECK: return [[NUM_ELEMENTS]] : index

From 39108f4c7a2c52be88f73bd6abaa613f4b28d327 Mon Sep 17 00:00:00 2001
From: Tim Northover <t.p.northover@gmail.com>
Date: Mon, 27 Jul 2020 14:37:14 +0100
Subject: [PATCH 0295/1035] ARM: make Thumb1 instructions non-flag-setting in
 IT block.

Many Thumb1 instructions are defined to set CPSR if executed outside an IT
block, but leave it alone from inside one. In MachineIR this is represented by
whether an optional register is CPSR or NoReg (0), and affects how the
instructions are printed.

This sets the instruction to the appropriate form during if-conversion.
---
 llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp          | 12 ++++++++++++
 llvm/lib/Target/ARM/ARMInstrFormats.td            |  5 +++--
 llvm/test/CodeGen/ARM/thumb2-it-block.ll          |  4 +---
 llvm/test/CodeGen/Thumb2/ifcvt-rescan-diamonds.ll |  3 ++-
 4 files changed, 18 insertions(+), 6 deletions(-)

diff --git a/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp b/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
index 4cc2b6bf7e7e0..d340931f31d8d 100644
--- a/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
@@ -537,6 +537,18 @@ bool ARMBaseInstrInfo::PredicateInstruction(
     MachineOperand &PMO = MI.getOperand(PIdx);
     PMO.setImm(Pred[0].getImm());
     MI.getOperand(PIdx+1).setReg(Pred[1].getReg());
+
+    // Thumb 1 arithmetic instructions do not set CPSR when executed inside an
+    // IT block. This affects how they are printed.
+    const MCInstrDesc &MCID = MI.getDesc();
+    if (MCID.TSFlags & ARMII::ThumbArithFlagSetting) {
+      assert(MCID.OpInfo[1].isOptionalDef() && "CPSR def isn't expected operand");
+      assert((MI.getOperand(1).isDead() ||
+              MI.getOperand(1).getReg() != ARM::CPSR) &&
+             "if conversion tried to stop defining used CPSR");
+      MI.getOperand(1).setReg(ARM::NoRegister);
+    }
+
     return true;
   }
   return false;
diff --git a/llvm/lib/Target/ARM/ARMInstrFormats.td b/llvm/lib/Target/ARM/ARMInstrFormats.td
index e13f3437cc7bd..85da7c5a535e9 100644
--- a/llvm/lib/Target/ARM/ARMInstrFormats.td
+++ b/llvm/lib/Target/ARM/ARMInstrFormats.td
@@ -403,8 +403,9 @@ class InstTemplate<AddrMode am, int sz, IndexMode im,
   bit isUnaryDataProc = 0;
   bit canXformTo16Bit = 0;
   // The instruction is a 16-bit flag setting Thumb instruction. Used
-  // by the parser to determine whether to require the 'S' suffix on the
-  // mnemonic (when not in an IT block) or preclude it (when in an IT block).
+  // by the parser and if-converter to determine whether to require the 'S'
+  // suffix on the mnemonic (when not in an IT block) or preclude it (when
+  // in an IT block).
   bit thumbArithFlagSetting = 0;
 
   bit validForTailPredication = 0;
diff --git a/llvm/test/CodeGen/ARM/thumb2-it-block.ll b/llvm/test/CodeGen/ARM/thumb2-it-block.ll
index 6d93869ec10fd..c1c1c74dafd17 100644
--- a/llvm/test/CodeGen/ARM/thumb2-it-block.ll
+++ b/llvm/test/CodeGen/ARM/thumb2-it-block.ll
@@ -16,9 +16,7 @@ entry:
 
 ; CHECK:        cmp
 ; CHECK-NEXT:   it    mi
-; We shouldn't need to check for the extra 's' here; tRSB should be printed as
-; "rsb" inside an IT block, not "rsbs".
-; CHECK-NEXT:   rsb{{s?}}mi
+; CHECK-NEXT:   rsbmi
 ; CHECK-NEXT:   cmp
 ; CHECK-NEXT:   it    mi
 ; CHECK-NEXT:   rsb{{s?}}mi
diff --git a/llvm/test/CodeGen/Thumb2/ifcvt-rescan-diamonds.ll b/llvm/test/CodeGen/Thumb2/ifcvt-rescan-diamonds.ll
index b7ba741b24983..116084b276fc8 100644
--- a/llvm/test/CodeGen/Thumb2/ifcvt-rescan-diamonds.ll
+++ b/llvm/test/CodeGen/Thumb2/ifcvt-rescan-diamonds.ll
@@ -22,7 +22,8 @@ while.cond38:                                     ; preds = %if.end111, %entry
 ; CHECK-NEXT: it eq
 ; CHECK-NEXT: ldreq
 ; CHECK-NEXT: it ne
-; CHECK-NEXT: movsne
+  ; N.b. 16-bit mov instruction in IT block does not set flags.
+; CHECK-NEXT: movne
 ; CHECK-NEXT: mvns
 ; CHECK-NEXT: b
 cond.true77:                                      ; preds = %while.cond38

From 412b3932c6612f4a60bfa8e0811cf6ba1bd3e0e8 Mon Sep 17 00:00:00 2001
From: Evgeniy Brevnov <ybrevnov@azul.com>
Date: Tue, 28 Jul 2020 19:50:40 +0700
Subject: [PATCH 0296/1035] [BPI] Fix memory leak reported by sanitizer bots

There is a silly mistake where release() is used instead of reset() for free resources of unique pointer.

Reviewed By: ebrevnov

Differential Revision: https://reviews.llvm.org/D84747
---
 llvm/lib/Analysis/BranchProbabilityInfo.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Analysis/BranchProbabilityInfo.cpp b/llvm/lib/Analysis/BranchProbabilityInfo.cpp
index 195fc69d9601d..7e34f3b6c869b 100644
--- a/llvm/lib/Analysis/BranchProbabilityInfo.cpp
+++ b/llvm/lib/Analysis/BranchProbabilityInfo.cpp
@@ -1178,7 +1178,7 @@ void BranchProbabilityInfo::calculate(const Function &F, const LoopInfo &LI,
 
   PostDominatedByUnreachable.clear();
   PostDominatedByColdCall.clear();
-  SccI.release();
+  SccI.reset();
 
   if (PrintBranchProb &&
       (PrintBranchProbFuncName.empty() ||

From 9b4826d18b5fba5fe638e1045c9c47d3fe8da8c8 Mon Sep 17 00:00:00 2001
From: "Joel E. Denny" <jdenny.ornl@gmail.com>
Date: Tue, 28 Jul 2020 08:57:03 -0400
Subject: [PATCH 0297/1035] [OpenMP] Fix libomptarget negative tests to expect
 abort

On runtime failures, D83963 causes the runtime to abort instead of
merely exiting with a non-zero value, but many tests in the
libomptarget test suite still expect the former behavior.  This patch
updates the test suite and was discussed in post-commit comments on
D83963 and D84557.
---
 openmp/libomptarget/test/lit.cfg | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/openmp/libomptarget/test/lit.cfg b/openmp/libomptarget/test/lit.cfg
index 77476c6fec793..6936bfa0c3195 100644
--- a/openmp/libomptarget/test/lit.cfg
+++ b/openmp/libomptarget/test/lit.cfg
@@ -114,7 +114,7 @@ for libomptarget_target in config.libomptarget_all_targets:
             "%t-" + libomptarget_target))
         config.substitutions.append(("%libomptarget-run-fail-" + \
             libomptarget_target, \
-            "%not %t-" + libomptarget_target))
+            "%not --crash %t-" + libomptarget_target))
         config.substitutions.append(("%clangxx-" + libomptarget_target, \
             "%clangxx %openmp_flags %flags -fopenmp-targets=" + libomptarget_target))
         config.substitutions.append(("%clang-" + libomptarget_target, \

From bd93f5ce07ef2fb9c1897bddc576fe4afb464788 Mon Sep 17 00:00:00 2001
From: Georgii Rymar <grimar@accesssoftek.com>
Date: Tue, 28 Jul 2020 12:53:06 +0300
Subject: [PATCH 0298/1035] [yaml2obj] - Add a way to override sh_type section
 field.

This adds the `ShType` key similar to others `Sh*` keys we have.

My use case is the following. Imagine we have a `SHT_SYMTAB_SHNDX`
section and want to hide it from a dumper. The natural way would be to
do something like:

```
  - Name:    .symtab_shndx
    Type:    [[TYPE=SHT_SYMTAB_SHNDX]]
    Entries: [ 0, 1 ]

```

and then change the TYPE from `SHT_SYMTAB_SHNDX` to something else,
for example to `SHT_PROGBITS`.

But we have a problem: regular sections does not have `Entries` key,
so yaml2obj will be unable to produce a section.

The solution is to introduce a `ShType` key to override the final type.

This is not the first time I am facing the need to change the type. I
was able to invent workarounds or solved issues differently in the past,
but finally came to conclusion that we just should support the `ShType`.

Differential revision: https://reviews.llvm.org/D84738
---
 llvm/include/llvm/ObjectYAML/ELFYAML.h        |  6 +++
 llvm/lib/ObjectYAML/ELFEmitter.cpp            |  2 +
 llvm/lib/ObjectYAML/ELFYAML.cpp               |  4 +-
 .../tools/yaml2obj/ELF/override-shtype.yaml   | 44 +++++++++++++++++++
 4 files changed, 55 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/tools/yaml2obj/ELF/override-shtype.yaml

diff --git a/llvm/include/llvm/ObjectYAML/ELFYAML.h b/llvm/include/llvm/ObjectYAML/ELFYAML.h
index b1ffb20681ea8..9a5c06fdfcd5c 100644
--- a/llvm/include/llvm/ObjectYAML/ELFYAML.h
+++ b/llvm/include/llvm/ObjectYAML/ELFYAML.h
@@ -206,6 +206,12 @@ struct Section : public Chunk {
 
   // This can be used to override the sh_flags field.
   Optional<llvm::yaml::Hex64> ShFlags;
+
+  // This can be used to override the sh_type field. It is useful when we
+  // want to use specific YAML keys for a section of a particular type to
+  // describe the content, but still want to have a different final type
+  // for the section.
+  Optional<ELF_SHT> ShType;
 };
 
 // Fill is a block of data which is placed outside of sections. It is
diff --git a/llvm/lib/ObjectYAML/ELFEmitter.cpp b/llvm/lib/ObjectYAML/ELFEmitter.cpp
index 65b03050c7cdb..bc27c03cb6877 100644
--- a/llvm/lib/ObjectYAML/ELFEmitter.cpp
+++ b/llvm/lib/ObjectYAML/ELFEmitter.cpp
@@ -551,6 +551,8 @@ static void overrideFields(ELFYAML::Section *From, typename ELFT::Shdr &To) {
     To.sh_offset = *From->ShOffset;
   if (From->ShSize)
     To.sh_size = *From->ShSize;
+  if (From->ShType)
+    To.sh_type = *From->ShType;
 }
 
 template <class ELFT>
diff --git a/llvm/lib/ObjectYAML/ELFYAML.cpp b/llvm/lib/ObjectYAML/ELFYAML.cpp
index f85d6a5d30efa..f460a387540d2 100644
--- a/llvm/lib/ObjectYAML/ELFYAML.cpp
+++ b/llvm/lib/ObjectYAML/ELFYAML.cpp
@@ -1094,11 +1094,13 @@ static void commonSectionMapping(IO &IO, ELFYAML::Section &Section) {
   // automatically when they are not explicitly defined.
   assert(!IO.outputting() ||
          (!Section.ShOffset.hasValue() && !Section.ShSize.hasValue() &&
-          !Section.ShName.hasValue() && !Section.ShFlags.hasValue()));
+          !Section.ShName.hasValue() && !Section.ShFlags.hasValue() &&
+          !Section.ShType.hasValue()));
   IO.mapOptional("ShName", Section.ShName);
   IO.mapOptional("ShOffset", Section.ShOffset);
   IO.mapOptional("ShSize", Section.ShSize);
   IO.mapOptional("ShFlags", Section.ShFlags);
+  IO.mapOptional("ShType", Section.ShType);
 }
 
 static void sectionMapping(IO &IO, ELFYAML::DynamicSection &Section) {
diff --git a/llvm/test/tools/yaml2obj/ELF/override-shtype.yaml b/llvm/test/tools/yaml2obj/ELF/override-shtype.yaml
new file mode 100644
index 0000000000000..ac29b3b0e8bc5
--- /dev/null
+++ b/llvm/test/tools/yaml2obj/ELF/override-shtype.yaml
@@ -0,0 +1,44 @@
+## Check we are able to override the sh_type field for different sections.
+## When doing this we are still able to use YAML keys that can be normally used
+## to describe a section with the original type specified with the Type key.
+
+# RUN: yaml2obj %s -o %t1
+# RUN: llvm-readobj --sections --section-data %t1 | FileCheck %s --check-prefixes=COMMON,ORIGINAL
+
+## Check we can use a hex value for the ShType. SHT_PROGBITS == 0x1.
+# RUN: yaml2obj -DTYPE=0x1 %s -o %t2
+# RUN: llvm-readobj --sections --section-data %t2 | FileCheck %s --check-prefixes=COMMON,OVERRIDE
+
+# COMMON:        Name: .gnu.version_r
+# ORIGINAL-NEXT: Type: SHT_GNU_verneed
+# OVERRIDE-NEXT: Type: SHT_PROGBITS
+# COMMON:        SectionData (
+# COMMON-NEXT:     0000: 01000100 04000000 10000000 00000000 |
+# COMMON-NEXT:     0010: 91070000 00000300 01000000 00000000 |
+# COMMON-NEXT:   )
+
+## Check we can use a string type name for the ShType.
+# RUN: yaml2obj -DTYPE=SHT_PROGBITS %s -o %t3
+# RUN: cmp %t2 %t3
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name:   .gnu.version_r
+    Type:   SHT_GNU_verneed
+    Flags:  [ SHF_ALLOC ]
+    ShType: [[TYPE=SHT_GNU_verneed]]
+    Info:   0
+    Dependencies:
+      - Version: 1
+        File:    dso.so.0
+        Entries:
+          - Name:  v1
+            Hash:  1937
+            Flags: 0
+            Other: 3
+DynamicSymbols: []

From 7bae3188e08746566733148a4ceccdb3cf24e93b Mon Sep 17 00:00:00 2001
From: Nathan James <n.james93@hotmail.co.uk>
Date: Tue, 28 Jul 2020 14:52:32 +0100
Subject: [PATCH 0299/1035] [clang-tidy][NFC] Make OptionsView methods as const
 where missing

---
 clang-tools-extra/clang-tidy/ClangTidyCheck.cpp |  7 +++----
 clang-tools-extra/clang-tidy/ClangTidyCheck.h   | 17 +++++++++--------
 2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/clang-tools-extra/clang-tidy/ClangTidyCheck.cpp b/clang-tools-extra/clang-tidy/ClangTidyCheck.cpp
index c24b8553999cc..ffd5bf974ba2f 100644
--- a/clang-tools-extra/clang-tidy/ClangTidyCheck.cpp
+++ b/clang-tools-extra/clang-tidy/ClangTidyCheck.cpp
@@ -168,10 +168,9 @@ void ClangTidyCheck::OptionsView::store<bool>(
   store(Options, LocalName, Value ? StringRef("true") : StringRef("false"));
 }
 
-llvm::Expected<int64_t>
-ClangTidyCheck::OptionsView::getEnumInt(StringRef LocalName,
-                                        ArrayRef<NameAndValue> Mapping,
-                                        bool CheckGlobal, bool IgnoreCase) {
+llvm::Expected<int64_t> ClangTidyCheck::OptionsView::getEnumInt(
+    StringRef LocalName, ArrayRef<NameAndValue> Mapping, bool CheckGlobal,
+    bool IgnoreCase) const {
   auto Iter = CheckGlobal
                   ? findPriorityOption(CheckOptions, NamePrefix, LocalName)
                   : CheckOptions.find((NamePrefix + LocalName).str());
diff --git a/clang-tools-extra/clang-tidy/ClangTidyCheck.h b/clang-tools-extra/clang-tidy/ClangTidyCheck.h
index 54b7251267524..4df8071c841e0 100644
--- a/clang-tools-extra/clang-tidy/ClangTidyCheck.h
+++ b/clang-tools-extra/clang-tidy/ClangTidyCheck.h
@@ -330,7 +330,7 @@ class ClangTidyCheck : public ast_matchers::MatchFinder::MatchCallback {
     /// supply the mapping required to convert between ``T`` and a string.
     template <typename T>
     std::enable_if_t<std::is_enum<T>::value, llvm::Expected<T>>
-    get(StringRef LocalName, bool IgnoreCase = false) {
+    get(StringRef LocalName, bool IgnoreCase = false) const {
       if (llvm::Expected<int64_t> ValueOr =
               getEnumInt(LocalName, typeEraseMapping<T>(), false, IgnoreCase))
         return static_cast<T>(*ValueOr);
@@ -349,7 +349,7 @@ class ClangTidyCheck : public ast_matchers::MatchFinder::MatchCallback {
     /// supply the mapping required to convert between ``T`` and a string.
     template <typename T>
     std::enable_if_t<std::is_enum<T>::value, T>
-    get(StringRef LocalName, T Default, bool IgnoreCase = false) {
+    get(StringRef LocalName, T Default, bool IgnoreCase = false) const {
       if (auto ValueOr = get<T>(LocalName, IgnoreCase))
         return *ValueOr;
       else
@@ -370,8 +370,7 @@ class ClangTidyCheck : public ast_matchers::MatchFinder::MatchCallback {
     /// supply the mapping required to convert between ``T`` and a string.
     template <typename T>
     std::enable_if_t<std::is_enum<T>::value, llvm::Expected<T>>
-    getLocalOrGlobal(StringRef LocalName,
-                     bool IgnoreCase = false) {
+    getLocalOrGlobal(StringRef LocalName, bool IgnoreCase = false) const {
       if (llvm::Expected<int64_t> ValueOr =
               getEnumInt(LocalName, typeEraseMapping<T>(), true, IgnoreCase))
         return static_cast<T>(*ValueOr);
@@ -391,7 +390,8 @@ class ClangTidyCheck : public ast_matchers::MatchFinder::MatchCallback {
     /// supply the mapping required to convert between ``T`` and a string.
     template <typename T>
     std::enable_if_t<std::is_enum<T>::value, T>
-    getLocalOrGlobal(StringRef LocalName, T Default, bool IgnoreCase = false) {
+    getLocalOrGlobal(StringRef LocalName, T Default,
+                     bool IgnoreCase = false) const {
       if (auto ValueOr = getLocalOrGlobal<T>(LocalName, IgnoreCase))
         return *ValueOr;
       else
@@ -420,7 +420,8 @@ class ClangTidyCheck : public ast_matchers::MatchFinder::MatchCallback {
     /// supply the mapping required to convert between ``T`` and a string.
     template <typename T>
     std::enable_if_t<std::is_enum<T>::value>
-    store(ClangTidyOptions::OptionMap &Options, StringRef LocalName, T Value) {
+    store(ClangTidyOptions::OptionMap &Options, StringRef LocalName,
+          T Value) const {
       ArrayRef<std::pair<T, StringRef>> Mapping =
           OptionEnumMapping<T>::getEnumMapping();
       auto Iter = llvm::find_if(
@@ -436,11 +437,11 @@ class ClangTidyCheck : public ast_matchers::MatchFinder::MatchCallback {
 
     llvm::Expected<int64_t> getEnumInt(StringRef LocalName,
                                        ArrayRef<NameAndValue> Mapping,
-                                       bool CheckGlobal, bool IgnoreCase);
+                                       bool CheckGlobal, bool IgnoreCase) const;
 
     template <typename T>
     std::enable_if_t<std::is_enum<T>::value, std::vector<NameAndValue>>
-    typeEraseMapping() {
+    typeEraseMapping() const {
       ArrayRef<std::pair<T, StringRef>> Mapping =
           OptionEnumMapping<T>::getEnumMapping();
       std::vector<NameAndValue> Result;

From d28f86723f37b2329428dfbcf847d3261f38dcc8 Mon Sep 17 00:00:00 2001
From: Jinsong Ji <jji@us.ibm.com>
Date: Tue, 28 Jul 2020 03:02:20 +0000
Subject: [PATCH 0300/1035] Re-land "[PowerPC] Remove QPX/A2Q BGQ/BGP CNK
 support"

This reverts commit bf544fa1c3cb80f24d85e84559fb11193846259f.

Fixed the typo in PPCInstrInfo.cpp.
---
 clang/lib/Basic/Targets/PPC.cpp               |   39 +-
 clang/lib/Basic/Targets/PPC.h                 |    3 -
 clang/lib/Driver/ToolChains/Arch/PPC.cpp      |    1 -
 clang/lib/Driver/ToolChains/Clang.cpp         |   12 -
 clang/test/Driver/clang-translation.c         |    6 -
 clang/test/Driver/ppc-abi.c                   |   20 -
 clang/test/Misc/target-invalid-cpu-note.c     |    2 +-
 clang/test/Preprocessor/init-ppc64.c          |   16 -
 llvm/docs/LangRef.rst                         |   11 +-
 llvm/include/llvm/ADT/Triple.h                |    3 -
 llvm/include/llvm/IR/IntrinsicsPowerPC.td     |  176 ---
 llvm/lib/Support/Triple.cpp                   |    6 -
 .../Target/PowerPC/AsmParser/PPCAsmParser.cpp |   18 -
 llvm/lib/Target/PowerPC/CMakeLists.txt        |    1 -
 .../PowerPC/Disassembler/PPCDisassembler.cpp  |   15 +-
 .../PowerPC/MCTargetDesc/PPCInstPrinter.cpp   |   12 -
 .../PowerPC/MCTargetDesc/PPCMCTargetDesc.h    |    1 -
 llvm/lib/Target/PowerPC/PPC.h                 |    2 -
 llvm/lib/Target/PowerPC/PPC.td                |   14 +-
 llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp     |    3 -
 llvm/lib/Target/PowerPC/PPCCallingConv.td     |   16 -
 llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp   |   16 +-
 llvm/lib/Target/PowerPC/PPCISelLowering.cpp   | 1025 +-------------
 llvm/lib/Target/PowerPC/PPCISelLowering.h     |   20 -
 llvm/lib/Target/PowerPC/PPCInstrFormats.td    |   52 -
 llvm/lib/Target/PowerPC/PPCInstrInfo.cpp      |   25 +-
 llvm/lib/Target/PowerPC/PPCInstrInfo.h        |   23 +-
 llvm/lib/Target/PowerPC/PPCInstrInfo.td       |   11 -
 llvm/lib/Target/PowerPC/PPCInstrQPX.td        | 1212 -----------------
 llvm/lib/Target/PowerPC/PPCQPXLoadSplat.cpp   |  161 ---
 llvm/lib/Target/PowerPC/PPCRegisterInfo.cpp   |    3 -
 llvm/lib/Target/PowerPC/PPCRegisterInfo.h     |    1 -
 llvm/lib/Target/PowerPC/PPCRegisterInfo.td    |   23 -
 llvm/lib/Target/PowerPC/PPCScheduleP9.td      |    9 +-
 llvm/lib/Target/PowerPC/PPCSubtarget.cpp      |   12 +-
 llvm/lib/Target/PowerPC/PPCSubtarget.h        |   14 -
 llvm/lib/Target/PowerPC/PPCTargetMachine.cpp  |   23 +-
 .../Target/PowerPC/PPCTargetTransformInfo.cpp |   76 +-
 .../Instrumentation/MemorySanitizer.cpp       |    7 +-
 llvm/test/Analysis/BasicAA/phi-spec-order.ll  |    2 +-
 .../CostModel/PowerPC/unal-vec-ldst.ll        |   73 -
 .../CodeGen/PowerPC/2012-11-16-mischedcall.ll |    2 +-
 ...leHoistingDueToBlockHotnessProfileData.mir |    2 +-
 .../NoCRFieldRedefWhenSpillingCRBIT.mir       |    2 +-
 llvm/test/CodeGen/PowerPC/a2q-stackalign.ll   |   23 -
 llvm/test/CodeGen/PowerPC/a2q.ll              |   10 -
 .../PowerPC/aantidep-inline-asm-use.ll        |    2 +-
 llvm/test/CodeGen/PowerPC/asm-Zy.ll           |    3 +-
 llvm/test/CodeGen/PowerPC/asm-constraints.ll  |    2 +-
 ...rt-rr-to-ri-instrs-R0-special-handling.mir |    4 +-
 .../convert-rr-to-ri-instrs-out-of-range.mir  |    2 +-
 .../PowerPC/convert-rr-to-ri-instrs.mir       |    8 +-
 llvm/test/CodeGen/PowerPC/ctr-minmaxnum.ll    |   11 -
 .../CodeGen/PowerPC/ctrloop-shortLoops.ll     |    7 -
 llvm/test/CodeGen/PowerPC/ec-input.ll         |    2 +-
 .../CodeGen/PowerPC/extra-toc-reg-deps.ll     |    8 +-
 .../CodeGen/PowerPC/fast-isel-icmp-split.ll   |    2 +-
 .../PowerPC/fma-mutate-duplicate-vreg.ll      |    2 +-
 .../CodeGen/PowerPC/fp2int2fp-ppcfp128.ll     |    3 +-
 .../CodeGen/PowerPC/glob-comp-aa-crash.ll     |    4 +-
 .../PowerPC/ifcvt-forked-bug-2016-08-08.ll    |    2 +-
 .../test/CodeGen/PowerPC/inlineasm-i64-reg.ll |    4 +-
 llvm/test/CodeGen/PowerPC/load-two-flts.ll    |    3 +-
 .../PowerPC/loop-data-prefetch-inner.ll       |    4 +-
 .../CodeGen/PowerPC/loop-data-prefetch.ll     |    4 +-
 llvm/test/CodeGen/PowerPC/loop-prep-all.ll    |   10 +-
 .../PowerPC/lxv-aligned-stack-slots.ll        |    2 +-
 llvm/test/CodeGen/PowerPC/machine-combiner.ll |   24 -
 llvm/test/CodeGen/PowerPC/mc-instrlat.ll      |    4 +-
 llvm/test/CodeGen/PowerPC/mcount-insertion.ll |    3 +-
 llvm/test/CodeGen/PowerPC/memcpy-vec.ll       |   23 -
 llvm/test/CodeGen/PowerPC/memset-nc.ll        |   48 -
 .../PowerPC/misched-inorder-latency.ll        |    3 +-
 llvm/test/CodeGen/PowerPC/misched.ll          |    1 -
 .../CodeGen/PowerPC/optnone-crbits-i1-ret.ll  |    3 +-
 .../CodeGen/PowerPC/pcrel-local-caller-toc.ll |    6 +-
 llvm/test/CodeGen/PowerPC/popcnt.ll           |    2 -
 llvm/test/CodeGen/PowerPC/ppc-passname.ll     |   11 -
 llvm/test/CodeGen/PowerPC/ppc64-sibcall.ll    |   21 +-
 llvm/test/CodeGen/PowerPC/pr24546.ll          |    4 +-
 llvm/test/CodeGen/PowerPC/pr27350.ll          |    2 +-
 llvm/test/CodeGen/PowerPC/pr28130.ll          |    2 +-
 .../CodeGen/PowerPC/preinc-ld-sel-crash.ll    |    2 +-
 llvm/test/CodeGen/PowerPC/qpx-bv-sint.ll      |   33 -
 llvm/test/CodeGen/PowerPC/qpx-bv.ll           |   37 -
 llvm/test/CodeGen/PowerPC/qpx-func-clobber.ll |   22 -
 llvm/test/CodeGen/PowerPC/qpx-load-splat.ll   |   80 --
 llvm/test/CodeGen/PowerPC/qpx-load.ll         |   26 -
 llvm/test/CodeGen/PowerPC/qpx-qvfmadd.ll      |   79 --
 llvm/test/CodeGen/PowerPC/qpx-recipest.ll     |  473 -------
 llvm/test/CodeGen/PowerPC/qpx-rounding-ops.ll |  109 --
 llvm/test/CodeGen/PowerPC/qpx-s-load.ll       |   26 -
 llvm/test/CodeGen/PowerPC/qpx-s-sel.ll        |  143 --
 llvm/test/CodeGen/PowerPC/qpx-s-store.ll      |   25 -
 llvm/test/CodeGen/PowerPC/qpx-sel.ll          |  151 --
 llvm/test/CodeGen/PowerPC/qpx-split-vsetcc.ll |   31 -
 llvm/test/CodeGen/PowerPC/qpx-store.ll        |   25 -
 .../test/CodeGen/PowerPC/qpx-unal-cons-lds.ll |  217 ---
 llvm/test/CodeGen/PowerPC/qpx-unalperm.ll     |   64 -
 llvm/test/CodeGen/PowerPC/rlwimi-and.ll       |    4 +-
 .../CodeGen/PowerPC/rlwinm_rldicl_to_andi.mir |    2 +-
 .../CodeGen/PowerPC/s000-alias-misched.ll     |    5 +-
 llvm/test/CodeGen/PowerPC/select-i1-vs-i1.ll  |  571 --------
 .../selectiondag-extload-computeknownbits.ll  |    2 +-
 llvm/test/CodeGen/PowerPC/setcr_bc.mir        |    4 +-
 llvm/test/CodeGen/PowerPC/setcr_bc2.mir       |    4 +-
 llvm/test/CodeGen/PowerPC/stwu-sched.ll       |    2 +-
 llvm/test/CodeGen/PowerPC/unal-vec-ldst.ll    |  149 --
 llvm/test/CodeGen/PowerPC/uwtables.ll         |    2 +-
 .../MemorySanitizer/PowerPC/vararg-ppc64.ll   |   15 -
 llvm/test/MC/Disassembler/PowerPC/qpx.txt     |  371 -----
 llvm/test/MC/PowerPC/qpx.s                    |  252 ----
 .../IPConstantProp/fp-bc-icmp-const-fold.ll   |    2 +-
 .../MSSA/combined-partial-overwrites.ll       |    2 +-
 .../combined-partial-overwrites.ll            |    2 +-
 .../EntryExitInstrumenter/mcount.ll           |    2 +-
 .../InstCombine/PowerPC/aligned-qpx.ll        |  165 ---
 .../LoopDataPrefetch/PowerPC/basic.ll         |    5 +-
 .../test/Transforms/LoopSimplify/dup-preds.ll |    2 +-
 llvm/test/Transforms/LoopUnroll/pr14167.ll    |    2 +-
 .../PowerPC/agg-interleave-a2.ll              |   40 -
 .../PowerPC/vectorize-only-for-real.ll        |    2 +-
 llvm/test/Transforms/NewGVN/pr31483.ll        |    4 +-
 .../Transforms/SCCP/fp-bc-icmp-const-fold.ll  |    2 +-
 llvm/unittests/ADT/TripleTest.cpp             |   35 -
 .../llvm/lib/Target/PowerPC/BUILD.gn          |    1 -
 openmp/runtime/src/kmp.h                      |    3 -
 openmp/runtime/src/kmp_csupport.cpp           |   11 -
 openmp/runtime/src/kmp_lock.h                 |    2 +-
 openmp/runtime/src/kmp_os.h                   |    2 +-
 openmp/runtime/src/kmp_platform.h             |    6 -
 openmp/runtime/src/z_Linux_asm.S              |    7 +-
 openmp/runtime/src/z_Linux_util.cpp           |    2 +-
 polly/lib/External/isl/config.sub             |   11 +-
 polly/lib/External/ppcg/config.sub            |   11 +-
 135 files changed, 174 insertions(+), 6523 deletions(-)
 delete mode 100644 llvm/lib/Target/PowerPC/PPCInstrQPX.td
 delete mode 100644 llvm/lib/Target/PowerPC/PPCQPXLoadSplat.cpp
 delete mode 100644 llvm/test/CodeGen/PowerPC/a2q-stackalign.ll
 delete mode 100644 llvm/test/CodeGen/PowerPC/a2q.ll
 delete mode 100644 llvm/test/CodeGen/PowerPC/memset-nc.ll
 delete mode 100644 llvm/test/CodeGen/PowerPC/qpx-bv-sint.ll
 delete mode 100644 llvm/test/CodeGen/PowerPC/qpx-bv.ll
 delete mode 100644 llvm/test/CodeGen/PowerPC/qpx-func-clobber.ll
 delete mode 100644 llvm/test/CodeGen/PowerPC/qpx-load-splat.ll
 delete mode 100644 llvm/test/CodeGen/PowerPC/qpx-load.ll
 delete mode 100644 llvm/test/CodeGen/PowerPC/qpx-qvfmadd.ll
 delete mode 100644 llvm/test/CodeGen/PowerPC/qpx-recipest.ll
 delete mode 100644 llvm/test/CodeGen/PowerPC/qpx-rounding-ops.ll
 delete mode 100644 llvm/test/CodeGen/PowerPC/qpx-s-load.ll
 delete mode 100644 llvm/test/CodeGen/PowerPC/qpx-s-sel.ll
 delete mode 100644 llvm/test/CodeGen/PowerPC/qpx-s-store.ll
 delete mode 100644 llvm/test/CodeGen/PowerPC/qpx-sel.ll
 delete mode 100644 llvm/test/CodeGen/PowerPC/qpx-split-vsetcc.ll
 delete mode 100644 llvm/test/CodeGen/PowerPC/qpx-store.ll
 delete mode 100644 llvm/test/CodeGen/PowerPC/qpx-unal-cons-lds.ll
 delete mode 100644 llvm/test/CodeGen/PowerPC/qpx-unalperm.ll
 delete mode 100644 llvm/test/MC/Disassembler/PowerPC/qpx.txt
 delete mode 100644 llvm/test/MC/PowerPC/qpx.s
 delete mode 100644 llvm/test/Transforms/InstCombine/PowerPC/aligned-qpx.ll
 delete mode 100644 llvm/test/Transforms/LoopVectorize/PowerPC/agg-interleave-a2.ll

diff --git a/clang/lib/Basic/Targets/PPC.cpp b/clang/lib/Basic/Targets/PPC.cpp
index f0de2bf070ea4..5f716a541ae92 100644
--- a/clang/lib/Basic/Targets/PPC.cpp
+++ b/clang/lib/Basic/Targets/PPC.cpp
@@ -46,8 +46,6 @@ bool PPCTargetInfo::handleTargetFeatures(std::vector<std::string> &Features,
       HasP8Crypto = true;
     } else if (Feature == "+direct-move") {
       HasDirectMove = true;
-    } else if (Feature == "+qpx") {
-      HasQPX = true;
     } else if (Feature == "+htm") {
       HasHTM = true;
     } else if (Feature == "+float128") {
@@ -99,7 +97,7 @@ void PPCTargetInfo::getTargetDefines(const LangOptions &Opts,
   }
 
   // ABI options.
-  if (ABI == "elfv1" || ABI == "elfv1-qpx")
+  if (ABI == "elfv1")
     Builder.defineMacro("_CALL_ELF", "1");
   if (ABI == "elfv2")
     Builder.defineMacro("_CALL_ELF", "2");
@@ -159,22 +157,11 @@ void PPCTargetInfo::getTargetDefines(const LangOptions &Opts,
     Builder.defineMacro("_ARCH_PWR10");
   if (ArchDefs & ArchDefineA2)
     Builder.defineMacro("_ARCH_A2");
-  if (ArchDefs & ArchDefineA2q) {
-    Builder.defineMacro("_ARCH_A2Q");
-    Builder.defineMacro("_ARCH_QP");
-  }
   if (ArchDefs & ArchDefineE500)
     Builder.defineMacro("__NO_LWSYNC__");
   if (ArchDefs & ArchDefineFuture)
     Builder.defineMacro("_ARCH_PWR_FUTURE");
 
-  if (getTriple().getVendor() == llvm::Triple::BGQ) {
-    Builder.defineMacro("__bg__");
-    Builder.defineMacro("__THW_BLUEGENE__");
-    Builder.defineMacro("__bgq__");
-    Builder.defineMacro("__TOS_BGQ__");
-  }
-
   if (HasAltivec) {
     Builder.defineMacro("__VEC__", "10206");
     Builder.defineMacro("__ALTIVEC__");
@@ -277,7 +264,6 @@ bool PPCTargetInfo::initFeatureMap(
                             .Case("ppc64le", true)
                             .Default(false);
 
-  Features["qpx"] = (CPU == "a2q");
   Features["power9-vector"] = (CPU == "pwr9");
   Features["crypto"] = llvm::StringSwitch<bool>(CPU)
                            .Case("ppc64le", true)
@@ -373,7 +359,6 @@ bool PPCTargetInfo::hasFeature(StringRef Feature) const {
       .Case("power8-vector", HasP8Vector)
       .Case("crypto", HasP8Crypto)
       .Case("direct-move", HasDirectMove)
-      .Case("qpx", HasQPX)
       .Case("htm", HasHTM)
       .Case("bpermd", HasBPERMD)
       .Case("extdiv", HasExtDiv)
@@ -503,17 +488,17 @@ ArrayRef<TargetInfo::AddlRegName> PPCTargetInfo::getGCCAddlRegNames() const {
 }
 
 static constexpr llvm::StringLiteral ValidCPUNames[] = {
-    {"generic"},     {"440"},     {"450"},     {"601"},       {"602"},
-    {"603"},         {"603e"},    {"603ev"},   {"604"},       {"604e"},
-    {"620"},         {"630"},     {"g3"},      {"7400"},      {"g4"},
-    {"7450"},        {"g4+"},     {"750"},     {"8548"},      {"970"},
-    {"g5"},          {"a2"},      {"a2q"},     {"e500"},      {"e500mc"},
-    {"e5500"},       {"power3"},  {"pwr3"},    {"power4"},    {"pwr4"},
-    {"power5"},      {"pwr5"},    {"power5x"}, {"pwr5x"},     {"power6"},
-    {"pwr6"},        {"power6x"}, {"pwr6x"},   {"power7"},    {"pwr7"},
-    {"power8"},      {"pwr8"},    {"power9"},  {"pwr9"},      {"power10"},
-    {"pwr10"},       {"powerpc"}, {"ppc"},     {"powerpc64"}, {"ppc64"},
-    {"powerpc64le"}, {"ppc64le"}, {"future"}};
+    {"generic"}, {"440"},     {"450"},       {"601"},     {"602"},
+    {"603"},     {"603e"},    {"603ev"},     {"604"},     {"604e"},
+    {"620"},     {"630"},     {"g3"},        {"7400"},    {"g4"},
+    {"7450"},    {"g4+"},     {"750"},       {"8548"},    {"970"},
+    {"g5"},      {"a2"},      {"e500"},      {"e500mc"},  {"e5500"},
+    {"power3"},  {"pwr3"},    {"power4"},    {"pwr4"},    {"power5"},
+    {"pwr5"},    {"power5x"}, {"pwr5x"},     {"power6"},  {"pwr6"},
+    {"power6x"}, {"pwr6x"},   {"power7"},    {"pwr7"},    {"power8"},
+    {"pwr8"},    {"power9"},  {"pwr9"},      {"power10"}, {"pwr10"},
+    {"powerpc"}, {"ppc"},     {"powerpc64"}, {"ppc64"},   {"powerpc64le"},
+    {"ppc64le"}, {"future"}};
 
 bool PPCTargetInfo::isValidCPUName(StringRef Name) const {
   return llvm::find(ValidCPUNames, Name) != std::end(ValidCPUNames);
diff --git a/clang/lib/Basic/Targets/PPC.h b/clang/lib/Basic/Targets/PPC.h
index ff8579b6c3cf4..c2048b2145918 100644
--- a/clang/lib/Basic/Targets/PPC.h
+++ b/clang/lib/Basic/Targets/PPC.h
@@ -46,7 +46,6 @@ class LLVM_LIBRARY_VISIBILITY PPCTargetInfo : public TargetInfo {
     ArchDefinePwr10 = 1 << 14,
     ArchDefineFuture = 1 << 15,
     ArchDefineA2 = 1 << 16,
-    ArchDefineA2q = 1 << 17,
     ArchDefineE500 = 1 << 18
   } ArchDefineTypes;
 
@@ -63,7 +62,6 @@ class LLVM_LIBRARY_VISIBILITY PPCTargetInfo : public TargetInfo {
   bool HasP8Vector = false;
   bool HasP8Crypto = false;
   bool HasDirectMove = false;
-  bool HasQPX = false;
   bool HasHTM = false;
   bool HasBPERMD = false;
   bool HasExtDiv = false;
@@ -118,7 +116,6 @@ class LLVM_LIBRARY_VISIBILITY PPCTargetInfo : public TargetInfo {
               .Case("970", ArchDefineName | ArchDefinePwr4 | ArchDefinePpcgr |
                                ArchDefinePpcsq)
               .Case("a2", ArchDefineA2)
-              .Case("a2q", ArchDefineName | ArchDefineA2 | ArchDefineA2q)
               .Cases("power3", "pwr3", ArchDefinePpcgr)
               .Cases("power4", "pwr4",
                      ArchDefinePwr4 | ArchDefinePpcgr | ArchDefinePpcsq)
diff --git a/clang/lib/Driver/ToolChains/Arch/PPC.cpp b/clang/lib/Driver/ToolChains/Arch/PPC.cpp
index 144e276a6bd87..bcaecf4b2d980 100644
--- a/clang/lib/Driver/ToolChains/Arch/PPC.cpp
+++ b/clang/lib/Driver/ToolChains/Arch/PPC.cpp
@@ -57,7 +57,6 @@ std::string ppc::getPPCTargetCPU(const ArgList &Args) {
         .Case("970", "970")
         .Case("G5", "g5")
         .Case("a2", "a2")
-        .Case("a2q", "a2q")
         .Case("e500", "e500")
         .Case("e500mc", "e500mc")
         .Case("e5500", "e5500")
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index 7a73eea013bdf..b0de225f8abf5 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -1883,18 +1883,6 @@ void Clang::AddPPCTargetArgs(const ArgList &Args,
   if (T.isOSBinFormatELF()) {
     switch (getToolChain().getArch()) {
     case llvm::Triple::ppc64: {
-      // When targeting a processor that supports QPX, or if QPX is
-      // specifically enabled, default to using the ABI that supports QPX (so
-      // long as it is not specifically disabled).
-      bool HasQPX = false;
-      if (Arg *A = Args.getLastArg(options::OPT_mcpu_EQ))
-        HasQPX = A->getValue() == StringRef("a2q");
-      HasQPX = Args.hasFlag(options::OPT_mqpx, options::OPT_mno_qpx, HasQPX);
-      if (HasQPX) {
-        ABIName = "elfv1-qpx";
-        break;
-      }
-
       if (T.isMusl() || (T.isOSFreeBSD() && T.getOSMajorVersion() >= 13))
         ABIName = "elfv2";
       else
diff --git a/clang/test/Driver/clang-translation.c b/clang/test/Driver/clang-translation.c
index 2f02970a2a8ee..d1daeb80004b7 100644
--- a/clang/test/Driver/clang-translation.c
+++ b/clang/test/Driver/clang-translation.c
@@ -167,12 +167,6 @@
 // PPCPWR8: "-cc1"
 // PPCPWR8: "-target-cpu" "pwr8"
 
-// RUN: %clang -target powerpc64-unknown-linux-gnu \
-// RUN: -### -S %s -mcpu=a2q 2>&1 | FileCheck -check-prefix=PPCA2Q %s
-// PPCA2Q: clang
-// PPCA2Q: "-cc1"
-// PPCA2Q: "-target-cpu" "a2q"
-
 // RUN: %clang -target powerpc64-unknown-linux-gnu \
 // RUN: -### -S %s -mcpu=630 2>&1 | FileCheck -check-prefix=PPC630 %s
 // PPC630: clang
diff --git a/clang/test/Driver/ppc-abi.c b/clang/test/Driver/ppc-abi.c
index acc4981a2eee6..2b5cc463e7c3d 100644
--- a/clang/test/Driver/ppc-abi.c
+++ b/clang/test/Driver/ppc-abi.c
@@ -5,14 +5,6 @@
 // RUN: %clang -target powerpc64-unknown-linux-gnu %s -### -o %t.o 2>&1 \
 // RUN:   -mabi=elfv1 | FileCheck -check-prefix=CHECK-ELFv1 %s
 // RUN: %clang -target powerpc64-unknown-linux-gnu %s -### -o %t.o 2>&1 \
-// RUN:   -mabi=elfv1-qpx | FileCheck -check-prefix=CHECK-ELFv1-QPX %s
-// RUN: %clang -target powerpc64-unknown-linux-gnu %s -### -o %t.o 2>&1 \
-// RUN:   -mcpu=a2q | FileCheck -check-prefix=CHECK-ELFv1-QPX %s
-// RUN: %clang -target powerpc64-unknown-linux-gnu %s -### -o %t.o 2>&1 \
-// RUN:   -mcpu=a2 -mqpx | FileCheck -check-prefix=CHECK-ELFv1-QPX %s
-// RUN: %clang -target powerpc64-unknown-linux-gnu %s -### -o %t.o 2>&1 \
-// RUN:   -mcpu=a2q -mno-qpx | FileCheck -check-prefix=CHECK-ELFv1 %s
-// RUN: %clang -target powerpc64-unknown-linux-gnu %s -### -o %t.o 2>&1 \
 // RUN:   -mabi=elfv2 | FileCheck -check-prefix=CHECK-ELFv2-BE %s
 
 // RUN: %clang -target powerpc64le-unknown-linux-gnu %s -### -o %t.o 2>&1 \
@@ -34,8 +26,6 @@
 // CHECK-ELFv1: "-target-abi" "elfv1"
 // CHECK-ELFv1-LE: "-mrelocation-model" "static"
 // CHECK-ELFv1-LE: "-target-abi" "elfv1"
-// CHECK-ELFv1-QPX: "-mrelocation-model" "static"
-// CHECK-ELFv1-QPX: "-target-abi" "elfv1-qpx"
 // CHECK-ELFv2: "-mrelocation-model" "static"
 // CHECK-ELFv2: "-target-abi" "elfv2"
 // CHECK-ELFv2-BE: "-mrelocation-model" "static"
@@ -48,14 +38,6 @@
 // RUN: %clang -fPIC -target powerpc64-unknown-linux-gnu %s -### -o %t.o 2>&1 \
 // RUN:   -mabi=elfv1 | FileCheck -check-prefix=CHECK-ELFv1-PIC %s
 // RUN: %clang -fPIC -target powerpc64-unknown-linux-gnu %s -### -o %t.o 2>&1 \
-// RUN:   -mabi=elfv1-qpx | FileCheck -check-prefix=CHECK-ELFv1-QPX-PIC %s
-// RUN: %clang -fPIC -target powerpc64-unknown-linux-gnu %s -### -o %t.o 2>&1 \
-// RUN:   -mcpu=a2q | FileCheck -check-prefix=CHECK-ELFv1-QPX-PIC %s
-// RUN: %clang -fPIC -target powerpc64-unknown-linux-gnu %s -### -o %t.o 2>&1 \
-// RUN:   -mcpu=a2 -mqpx | FileCheck -check-prefix=CHECK-ELFv1-QPX-PIC %s
-// RUN: %clang -fPIC -target powerpc64-unknown-linux-gnu %s -### -o %t.o 2>&1 \
-// RUN:   -mcpu=a2q -mno-qpx | FileCheck -check-prefix=CHECK-ELFv1-PIC %s
-// RUN: %clang -fPIC -target powerpc64-unknown-linux-gnu %s -### -o %t.o 2>&1 \
 // RUN:   -mabi=elfv2 | FileCheck -check-prefix=CHECK-ELFv2-PIC %s
 
 // RUN: %clang -fPIC -target powerpc64le-unknown-linux-gnu %s -### -o %t.o 2>&1 \
@@ -69,8 +51,6 @@
 
 // CHECK-ELFv1-PIC: "-mrelocation-model" "pic" "-pic-level" "2"
 // CHECK-ELFv1-PIC: "-target-abi" "elfv1"
-// CHECK-ELFv1-QPX-PIC: "-mrelocation-model" "pic" "-pic-level" "2"
-// CHECK-ELFv1-QPX-PIC: "-target-abi" "elfv1-qpx"
 // CHECK-ELFv2-PIC: "-mrelocation-model" "pic" "-pic-level" "2"
 // CHECK-ELFv2-PIC: "-target-abi" "elfv2"
 
diff --git a/clang/test/Misc/target-invalid-cpu-note.c b/clang/test/Misc/target-invalid-cpu-note.c
index 3a376a7caab46..bf6eaefe0b3ca 100644
--- a/clang/test/Misc/target-invalid-cpu-note.c
+++ b/clang/test/Misc/target-invalid-cpu-note.c
@@ -79,7 +79,7 @@
 // PPC: error: unknown target CPU 'not-a-cpu'
 // PPC: note: valid target CPU values are: generic, 440, 450, 601, 602, 603,
 // PPC-SAME: 603e, 603ev, 604, 604e, 620, 630, g3, 7400, g4, 7450, g4+, 750,
-// PPC-SAME: 8548, 970, g5, a2, a2q, e500, e500mc, e5500, power3, pwr3, power4,
+// PPC-SAME: 8548, 970, g5, a2, e500, e500mc, e5500, power3, pwr3, power4,
 // PPC-SAME: pwr4, power5, pwr5, power5x, pwr5x, power6, pwr6, power6x, pwr6x,
 // PPC-SAME: power7, pwr7, power8, pwr8, power9, pwr9, power10, pwr10, powerpc, ppc, powerpc64,
 // PPC-SAME: ppc64, powerpc64le, ppc64le, future
diff --git a/clang/test/Preprocessor/init-ppc64.c b/clang/test/Preprocessor/init-ppc64.c
index ed8601636554e..48d35c95aa570 100644
--- a/clang/test/Preprocessor/init-ppc64.c
+++ b/clang/test/Preprocessor/init-ppc64.c
@@ -408,21 +408,6 @@
 // PPC64LE:#define __ppc64__ 1
 // PPC64LE:#define __ppc__ 1
 //
-// RUN: %clang_cc1 -E -dM -ffreestanding -triple=powerpc64-none-none -target-cpu a2q -fno-signed-char < /dev/null | FileCheck -match-full-lines -check-prefix PPCA2Q %s
-//
-// PPCA2Q:#define _ARCH_A2 1
-// PPCA2Q:#define _ARCH_A2Q 1
-// PPCA2Q:#define _ARCH_PPC 1
-// PPCA2Q:#define _ARCH_PPC64 1
-// PPCA2Q:#define _ARCH_QP 1
-//
-// RUN: %clang_cc1 -E -dM -ffreestanding -triple=powerpc64-bgq-linux -fno-signed-char < /dev/null | FileCheck -match-full-lines -check-prefix PPCBGQ %s
-//
-// PPCBGQ:#define __THW_BLUEGENE__ 1
-// PPCBGQ:#define __TOS_BGQ__ 1
-// PPCBGQ:#define __bg__ 1
-// PPCBGQ:#define __bgq__ 1
-//
 // RUN: %clang_cc1 -E -dM -ffreestanding -triple=powerpc64-none-none -target-cpu 630 -fno-signed-char < /dev/null | FileCheck -match-full-lines -check-prefix PPC630 %s
 //
 // PPC630:#define _ARCH_630 1
@@ -1069,7 +1054,6 @@
 
 // RUN: %clang_cc1 -E -dM -ffreestanding -fgnuc-version=4.2.1 -triple=powerpc64-unknown-linux-gnu < /dev/null | FileCheck -match-full-lines -check-prefix PPC64-ELFv1 %s
 // RUN: %clang_cc1 -E -dM -ffreestanding -fgnuc-version=4.2.1 -triple=powerpc64-unknown-linux-gnu -target-abi elfv1 < /dev/null | FileCheck -match-full-lines -check-prefix PPC64-ELFv1 %s
-// RUN: %clang_cc1 -E -dM -ffreestanding -fgnuc-version=4.2.1 -triple=powerpc64-unknown-linux-gnu -target-abi elfv1-qpx < /dev/null | FileCheck -match-full-lines -check-prefix PPC64-ELFv1 %s
 // RUN: %clang_cc1 -E -dM -ffreestanding -fgnuc-version=4.2.1 -triple=powerpc64-unknown-linux-gnu -target-abi elfv2 < /dev/null | FileCheck -match-full-lines -check-prefix PPC64-ELFv2 %s
 // RUN: %clang_cc1 -E -dM -ffreestanding -fgnuc-version=4.2.1 -triple=powerpc64le-unknown-linux-gnu < /dev/null | FileCheck -match-full-lines -check-prefix PPC64-ELFv2 %s
 // RUN: %clang_cc1 -E -dM -ffreestanding -fgnuc-version=4.2.1 -triple=powerpc64le-unknown-linux-gnu -target-abi elfv1 < /dev/null | FileCheck -match-full-lines -check-prefix PPC64-ELFv1 %s
diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index 6b9c5c6899819..af93a6ed5c56e 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -4310,14 +4310,9 @@ PowerPC:
 - ``r``: A 32 or 64-bit integer register.
 - ``b``: A 32 or 64-bit integer register, excluding ``R0`` (that is:
   ``R1-R31``).
-- ``f``: A 32 or 64-bit float register (``F0-F31``), or when QPX is enabled, a
-  128 or 256-bit QPX register (``Q0-Q31``; aliases the ``F`` registers).
-- ``v``: For ``4 x f32`` or ``4 x f64`` types, when QPX is enabled, a
-  128 or 256-bit QPX register (``Q0-Q31``), otherwise a 128-bit
-  altivec vector register (``V0-V31``).
-
-  .. FIXME: is this a bug that v accepts QPX registers? I think this
-     is supposed to only use the altivec vector registers?
+- ``f``: A 32 or 64-bit float register (``F0-F31``), 
+- ``v``: For ``4 x f32`` or ``4 x f64`` types, a 128-bit altivec vector
+   register (``V0-V31``).
 
 - ``y``: Condition register (``CR0-CR7``).
 - ``wc``: An individual CR bit in a CR register.
diff --git a/llvm/include/llvm/ADT/Triple.h b/llvm/include/llvm/ADT/Triple.h
index 6bad18f19244e..c578c097c6f64 100644
--- a/llvm/include/llvm/ADT/Triple.h
+++ b/llvm/include/llvm/ADT/Triple.h
@@ -142,8 +142,6 @@ class Triple {
     Apple,
     PC,
     SCEI,
-    BGP,
-    BGQ,
     Freescale,
     IBM,
     ImaginationTechnologies,
@@ -179,7 +177,6 @@ class Triple {
     Minix,
     RTEMS,
     NaCl,       // Native Client
-    CNK,        // BG/P Compute-Node Kernel
     AIX,
     CUDA,       // NVIDIA CUDA
     NVCL,       // NVIDIA OpenCL
diff --git a/llvm/include/llvm/IR/IntrinsicsPowerPC.td b/llvm/include/llvm/IR/IntrinsicsPowerPC.td
index 23bcf3ce1959c..853d26c67ee3d 100644
--- a/llvm/include/llvm/IR/IntrinsicsPowerPC.td
+++ b/llvm/include/llvm/IR/IntrinsicsPowerPC.td
@@ -1109,182 +1109,6 @@ def  int_ppc_vsx_xxblendvd: GCCBuiltin<"__builtin_vsx_xxblendvd">,
                  [IntrNoMem]>;
 }
 
-//===----------------------------------------------------------------------===//
-// PowerPC QPX Intrinsics.
-//
-
-let TargetPrefix = "ppc" in {  // All PPC intrinsics start with "llvm.ppc.".
-  /// PowerPC_QPX_Intrinsic - Base class for all QPX intrinsics.
-  class PowerPC_QPX_Intrinsic<string GCCIntSuffix, list<LLVMType> ret_types,
-                              list<LLVMType> param_types,
-                              list<IntrinsicProperty> properties>
-    : GCCBuiltin<!strconcat("__builtin_qpx_", GCCIntSuffix)>,
-      Intrinsic<ret_types, param_types, properties>;
-}
-
-//===----------------------------------------------------------------------===//
-// PowerPC QPX Intrinsic Class Definitions.
-//
-
-/// PowerPC_QPX_FF_Intrinsic - A PowerPC intrinsic that takes one v4f64
-/// vector and returns one.  These intrinsics have no side effects.
-class PowerPC_QPX_FF_Intrinsic<string GCCIntSuffix>
-  : PowerPC_QPX_Intrinsic<GCCIntSuffix,
-                          [llvm_v4f64_ty], [llvm_v4f64_ty], [IntrNoMem]>;
-
-/// PowerPC_QPX_FFF_Intrinsic - A PowerPC intrinsic that takes two v4f64
-/// vectors and returns one.  These intrinsics have no side effects.
-class PowerPC_QPX_FFF_Intrinsic<string GCCIntSuffix>
-  : PowerPC_QPX_Intrinsic<GCCIntSuffix,
-                          [llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty],
-                          [IntrNoMem]>;
-
-/// PowerPC_QPX_FFFF_Intrinsic - A PowerPC intrinsic that takes three v4f64
-/// vectors and returns one.  These intrinsics have no side effects.
-class PowerPC_QPX_FFFF_Intrinsic<string GCCIntSuffix>
-  : PowerPC_QPX_Intrinsic<GCCIntSuffix,
-                          [llvm_v4f64_ty],
-                          [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4f64_ty],
-                          [IntrNoMem]>;
-
-/// PowerPC_QPX_Load_Intrinsic - A PowerPC intrinsic that takes a pointer
-/// and returns a v4f64.
-class PowerPC_QPX_Load_Intrinsic<string GCCIntSuffix>
-  : PowerPC_QPX_Intrinsic<GCCIntSuffix,
-                          [llvm_v4f64_ty], [llvm_ptr_ty], [IntrReadMem, IntrArgMemOnly]>;
-
-/// PowerPC_QPX_LoadPerm_Intrinsic - A PowerPC intrinsic that takes a pointer
-/// and returns a v4f64 permutation.
-class PowerPC_QPX_LoadPerm_Intrinsic<string GCCIntSuffix>
-  : PowerPC_QPX_Intrinsic<GCCIntSuffix,
-                          [llvm_v4f64_ty], [llvm_ptr_ty], [IntrNoMem]>;
-
-/// PowerPC_QPX_Store_Intrinsic - A PowerPC intrinsic that takes a pointer
-/// and stores a v4f64.
-class PowerPC_QPX_Store_Intrinsic<string GCCIntSuffix>
-  : PowerPC_QPX_Intrinsic<GCCIntSuffix,
-                          [], [llvm_v4f64_ty, llvm_ptr_ty],
-                          [IntrWriteMem, IntrArgMemOnly]>;
-
-//===----------------------------------------------------------------------===//
-// PowerPC QPX Intrinsic Definitions.
-
-let TargetPrefix = "ppc" in {  // All intrinsics start with "llvm.ppc.".
-  // Add Instructions
-  def int_ppc_qpx_qvfadd : PowerPC_QPX_FFF_Intrinsic<"qvfadd">;
-  def int_ppc_qpx_qvfadds : PowerPC_QPX_FFF_Intrinsic<"qvfadds">;
-  def int_ppc_qpx_qvfsub : PowerPC_QPX_FFF_Intrinsic<"qvfsub">;
-  def int_ppc_qpx_qvfsubs : PowerPC_QPX_FFF_Intrinsic<"qvfsubs">;
-
-  // Estimate Instructions
-  def int_ppc_qpx_qvfre : PowerPC_QPX_FF_Intrinsic<"qvfre">;
-  def int_ppc_qpx_qvfres : PowerPC_QPX_FF_Intrinsic<"qvfres">;
-  def int_ppc_qpx_qvfrsqrte : PowerPC_QPX_FF_Intrinsic<"qvfrsqrte">;
-  def int_ppc_qpx_qvfrsqrtes : PowerPC_QPX_FF_Intrinsic<"qvfrsqrtes">;
-
-  // Multiply Instructions
-  def int_ppc_qpx_qvfmul : PowerPC_QPX_FFF_Intrinsic<"qvfmul">;
-  def int_ppc_qpx_qvfmuls : PowerPC_QPX_FFF_Intrinsic<"qvfmuls">;
-  def int_ppc_qpx_qvfxmul : PowerPC_QPX_FFF_Intrinsic<"qvfxmul">;
-  def int_ppc_qpx_qvfxmuls : PowerPC_QPX_FFF_Intrinsic<"qvfxmuls">;
-
-  // Multiply-add instructions
-  def int_ppc_qpx_qvfmadd : PowerPC_QPX_FFFF_Intrinsic<"qvfmadd">;
-  def int_ppc_qpx_qvfmadds : PowerPC_QPX_FFFF_Intrinsic<"qvfmadds">;
-  def int_ppc_qpx_qvfnmadd : PowerPC_QPX_FFFF_Intrinsic<"qvfnmadd">;
-  def int_ppc_qpx_qvfnmadds : PowerPC_QPX_FFFF_Intrinsic<"qvfnmadds">;
-  def int_ppc_qpx_qvfmsub : PowerPC_QPX_FFFF_Intrinsic<"qvfmsub">;
-  def int_ppc_qpx_qvfmsubs : PowerPC_QPX_FFFF_Intrinsic<"qvfmsubs">;
-  def int_ppc_qpx_qvfnmsub : PowerPC_QPX_FFFF_Intrinsic<"qvfnmsub">;
-  def int_ppc_qpx_qvfnmsubs : PowerPC_QPX_FFFF_Intrinsic<"qvfnmsubs">;
-  def int_ppc_qpx_qvfxmadd : PowerPC_QPX_FFFF_Intrinsic<"qvfxmadd">;
-  def int_ppc_qpx_qvfxmadds : PowerPC_QPX_FFFF_Intrinsic<"qvfxmadds">;
-  def int_ppc_qpx_qvfxxnpmadd : PowerPC_QPX_FFFF_Intrinsic<"qvfxxnpmadd">;
-  def int_ppc_qpx_qvfxxnpmadds : PowerPC_QPX_FFFF_Intrinsic<"qvfxxnpmadds">;
-  def int_ppc_qpx_qvfxxcpnmadd : PowerPC_QPX_FFFF_Intrinsic<"qvfxxcpnmadd">;
-  def int_ppc_qpx_qvfxxcpnmadds : PowerPC_QPX_FFFF_Intrinsic<"qvfxxcpnmadds">;
-  def int_ppc_qpx_qvfxxmadd : PowerPC_QPX_FFFF_Intrinsic<"qvfxxmadd">;
-  def int_ppc_qpx_qvfxxmadds : PowerPC_QPX_FFFF_Intrinsic<"qvfxxmadds">;
-
-  // Select Instruction
-  def int_ppc_qpx_qvfsel : PowerPC_QPX_FFFF_Intrinsic<"qvfsel">;
-
-  // Permute Instruction
-  def int_ppc_qpx_qvfperm : PowerPC_QPX_FFFF_Intrinsic<"qvfperm">;
-
-  // Convert and Round Instructions
-  def int_ppc_qpx_qvfctid : PowerPC_QPX_FF_Intrinsic<"qvfctid">;
-  def int_ppc_qpx_qvfctidu : PowerPC_QPX_FF_Intrinsic<"qvfctidu">;
-  def int_ppc_qpx_qvfctidz : PowerPC_QPX_FF_Intrinsic<"qvfctidz">;
-  def int_ppc_qpx_qvfctiduz : PowerPC_QPX_FF_Intrinsic<"qvfctiduz">;
-  def int_ppc_qpx_qvfctiw : PowerPC_QPX_FF_Intrinsic<"qvfctiw">;
-  def int_ppc_qpx_qvfctiwu : PowerPC_QPX_FF_Intrinsic<"qvfctiwu">;
-  def int_ppc_qpx_qvfctiwz : PowerPC_QPX_FF_Intrinsic<"qvfctiwz">;
-  def int_ppc_qpx_qvfctiwuz : PowerPC_QPX_FF_Intrinsic<"qvfctiwuz">;
-  def int_ppc_qpx_qvfcfid : PowerPC_QPX_FF_Intrinsic<"qvfcfid">;
-  def int_ppc_qpx_qvfcfidu : PowerPC_QPX_FF_Intrinsic<"qvfcfidu">;
-  def int_ppc_qpx_qvfcfids : PowerPC_QPX_FF_Intrinsic<"qvfcfids">;
-  def int_ppc_qpx_qvfcfidus : PowerPC_QPX_FF_Intrinsic<"qvfcfidus">;
-  def int_ppc_qpx_qvfrsp : PowerPC_QPX_FF_Intrinsic<"qvfrsp">;
-  def int_ppc_qpx_qvfriz : PowerPC_QPX_FF_Intrinsic<"qvfriz">;
-  def int_ppc_qpx_qvfrin : PowerPC_QPX_FF_Intrinsic<"qvfrin">;
-  def int_ppc_qpx_qvfrip : PowerPC_QPX_FF_Intrinsic<"qvfrip">;
-  def int_ppc_qpx_qvfrim : PowerPC_QPX_FF_Intrinsic<"qvfrim">;
-
-  // Move Instructions
-  def int_ppc_qpx_qvfneg : PowerPC_QPX_FF_Intrinsic<"qvfneg">;
-  def int_ppc_qpx_qvfabs : PowerPC_QPX_FF_Intrinsic<"qvfabs">;
-  def int_ppc_qpx_qvfnabs : PowerPC_QPX_FF_Intrinsic<"qvfnabs">;
-  def int_ppc_qpx_qvfcpsgn : PowerPC_QPX_FFF_Intrinsic<"qvfcpsgn">;
-
-  // Compare Instructions
-  def int_ppc_qpx_qvftstnan : PowerPC_QPX_FFF_Intrinsic<"qvftstnan">;
-  def int_ppc_qpx_qvfcmplt : PowerPC_QPX_FFF_Intrinsic<"qvfcmplt">;
-  def int_ppc_qpx_qvfcmpgt : PowerPC_QPX_FFF_Intrinsic<"qvfcmpgt">;
-  def int_ppc_qpx_qvfcmpeq : PowerPC_QPX_FFF_Intrinsic<"qvfcmpeq">;
-
-  // Load instructions
-  def int_ppc_qpx_qvlfd : PowerPC_QPX_Load_Intrinsic<"qvlfd">;
-  def int_ppc_qpx_qvlfda : PowerPC_QPX_Load_Intrinsic<"qvlfda">;
-  def int_ppc_qpx_qvlfs : PowerPC_QPX_Load_Intrinsic<"qvlfs">;
-  def int_ppc_qpx_qvlfsa : PowerPC_QPX_Load_Intrinsic<"qvlfsa">;
-
-  def int_ppc_qpx_qvlfcda : PowerPC_QPX_Load_Intrinsic<"qvlfcda">;
-  def int_ppc_qpx_qvlfcd : PowerPC_QPX_Load_Intrinsic<"qvlfcd">;
-  def int_ppc_qpx_qvlfcsa : PowerPC_QPX_Load_Intrinsic<"qvlfcsa">;
-  def int_ppc_qpx_qvlfcs : PowerPC_QPX_Load_Intrinsic<"qvlfcs">;
-  def int_ppc_qpx_qvlfiwaa : PowerPC_QPX_Load_Intrinsic<"qvlfiwaa">;
-  def int_ppc_qpx_qvlfiwa : PowerPC_QPX_Load_Intrinsic<"qvlfiwa">;
-  def int_ppc_qpx_qvlfiwza : PowerPC_QPX_Load_Intrinsic<"qvlfiwza">;
-  def int_ppc_qpx_qvlfiwz : PowerPC_QPX_Load_Intrinsic<"qvlfiwz">;
-
-  def int_ppc_qpx_qvlpcld : PowerPC_QPX_LoadPerm_Intrinsic<"qvlpcld">;
-  def int_ppc_qpx_qvlpcls : PowerPC_QPX_LoadPerm_Intrinsic<"qvlpcls">;
-  def int_ppc_qpx_qvlpcrd : PowerPC_QPX_LoadPerm_Intrinsic<"qvlpcrd">;
-  def int_ppc_qpx_qvlpcrs : PowerPC_QPX_LoadPerm_Intrinsic<"qvlpcrs">;
-
-  // Store instructions
-  def int_ppc_qpx_qvstfd : PowerPC_QPX_Store_Intrinsic<"qvstfd">;
-  def int_ppc_qpx_qvstfda : PowerPC_QPX_Store_Intrinsic<"qvstfda">;
-  def int_ppc_qpx_qvstfs : PowerPC_QPX_Store_Intrinsic<"qvstfs">;
-  def int_ppc_qpx_qvstfsa : PowerPC_QPX_Store_Intrinsic<"qvstfsa">;
-
-  def int_ppc_qpx_qvstfcda : PowerPC_QPX_Store_Intrinsic<"qvstfcda">;
-  def int_ppc_qpx_qvstfcd : PowerPC_QPX_Store_Intrinsic<"qvstfcd">;
-  def int_ppc_qpx_qvstfcsa : PowerPC_QPX_Store_Intrinsic<"qvstfcsa">;
-  def int_ppc_qpx_qvstfcs : PowerPC_QPX_Store_Intrinsic<"qvstfcs">;
-  def int_ppc_qpx_qvstfiwa : PowerPC_QPX_Store_Intrinsic<"qvstfiwa">;
-  def int_ppc_qpx_qvstfiw : PowerPC_QPX_Store_Intrinsic<"qvstfiw">;
-
-  // Logical and permutation formation
-  def int_ppc_qpx_qvflogical : PowerPC_QPX_Intrinsic<"qvflogical",
-                          [llvm_v4f64_ty],
-                          [llvm_v4f64_ty, llvm_v4f64_ty, llvm_i32_ty],
-                          [IntrNoMem]>;
-  def int_ppc_qpx_qvgpci : PowerPC_QPX_Intrinsic<"qvgpci",
-                          [llvm_v4f64_ty], [llvm_i32_ty], [IntrNoMem]>;
-}
-
 //===----------------------------------------------------------------------===//
 // PowerPC HTM Intrinsic Definitions.
 
diff --git a/llvm/lib/Support/Triple.cpp b/llvm/lib/Support/Triple.cpp
index fec1985ccacae..72648273b4cd5 100644
--- a/llvm/lib/Support/Triple.cpp
+++ b/llvm/lib/Support/Triple.cpp
@@ -160,8 +160,6 @@ StringRef Triple::getVendorTypeName(VendorType Kind) {
 
   case AMD: return "amd";
   case Apple: return "apple";
-  case BGP: return "bgp";
-  case BGQ: return "bgq";
   case CSR: return "csr";
   case Freescale: return "fsl";
   case IBM: return "ibm";
@@ -187,7 +185,6 @@ StringRef Triple::getOSTypeName(OSType Kind) {
   case AMDHSA: return "amdhsa";
   case AMDPAL: return "amdpal";
   case Ananas: return "ananas";
-  case CNK: return "cnk";
   case CUDA: return "cuda";
   case CloudABI: return "cloudabi";
   case Contiki: return "contiki";
@@ -470,8 +467,6 @@ static Triple::VendorType parseVendor(StringRef VendorName) {
     .Case("apple", Triple::Apple)
     .Case("pc", Triple::PC)
     .Case("scei", Triple::SCEI)
-    .Case("bgp", Triple::BGP)
-    .Case("bgq", Triple::BGQ)
     .Case("fsl", Triple::Freescale)
     .Case("ibm", Triple::IBM)
     .Case("img", Triple::ImaginationTechnologies)
@@ -508,7 +503,6 @@ static Triple::OSType parseOS(StringRef OSName) {
     .StartsWith("minix", Triple::Minix)
     .StartsWith("rtems", Triple::RTEMS)
     .StartsWith("nacl", Triple::NaCl)
-    .StartsWith("cnk", Triple::CNK)
     .StartsWith("aix", Triple::AIX)
     .StartsWith("cuda", Triple::CUDA)
     .StartsWith("nvcl", Triple::NVCL)
diff --git a/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp b/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp
index 13fd7d05ab9f4..81008d3ea5662 100644
--- a/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp
+++ b/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp
@@ -492,21 +492,6 @@ struct PPCOperand : public MCParsedAsmOperand {
     Inst.addOperand(MCOperand::createReg(VSSRegs[getVSReg()]));
   }
 
-  void addRegQFRCOperands(MCInst &Inst, unsigned N) const {
-    assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::createReg(QFRegs[getReg()]));
-  }
-
-  void addRegQSRCOperands(MCInst &Inst, unsigned N) const {
-    assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::createReg(QFRegs[getReg()]));
-  }
-
-  void addRegQBRCOperands(MCInst &Inst, unsigned N) const {
-    assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::createReg(QFRegs[getReg()]));
-  }
-
   void addRegSPE4RCOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
     Inst.addOperand(MCOperand::createReg(RRegs[getReg()]));
@@ -1207,9 +1192,6 @@ bool PPCAsmParser::MatchRegisterName(unsigned &RegNo, int64_t &IntVal) {
     } else if (Name.startswith_lower("v") &&
                !Name.substr(1).getAsInteger(10, IntVal) && IntVal < 32) {
       RegNo = VRegs[IntVal];
-    } else if (Name.startswith_lower("q") &&
-               !Name.substr(1).getAsInteger(10, IntVal) && IntVal < 32) {
-      RegNo = QFRegs[IntVal];
     } else if (Name.startswith_lower("cr") &&
                !Name.substr(2).getAsInteger(10, IntVal) && IntVal < 8) {
       RegNo = CRRegs[IntVal];
diff --git a/llvm/lib/Target/PowerPC/CMakeLists.txt b/llvm/lib/Target/PowerPC/CMakeLists.txt
index 91021d4e584e1..5a06faa16be19 100644
--- a/llvm/lib/Target/PowerPC/CMakeLists.txt
+++ b/llvm/lib/Target/PowerPC/CMakeLists.txt
@@ -36,7 +36,6 @@ add_llvm_target(PowerPCCodeGen
   PPCMacroFusion.cpp
   PPCMIPeephole.cpp
   PPCRegisterInfo.cpp
-  PPCQPXLoadSplat.cpp
   PPCSubtarget.cpp
   PPCTargetMachine.cpp
   PPCTargetObjectFile.cpp
diff --git a/llvm/lib/Target/PowerPC/Disassembler/PPCDisassembler.cpp b/llvm/lib/Target/PowerPC/Disassembler/PPCDisassembler.cpp
index 74c6fd3733f03..362ddf7204557 100644
--- a/llvm/lib/Target/PowerPC/Disassembler/PPCDisassembler.cpp
+++ b/llvm/lib/Target/PowerPC/Disassembler/PPCDisassembler.cpp
@@ -167,12 +167,6 @@ static DecodeStatus DecodeG8RC_NOX0RegisterClass(MCInst &Inst, uint64_t RegNo,
 #define DecodePointerLikeRegClass0 DecodeGPRCRegisterClass
 #define DecodePointerLikeRegClass1 DecodeGPRC_NOR0RegisterClass
 
-static DecodeStatus DecodeQFRCRegisterClass(MCInst &Inst, uint64_t RegNo,
-                                            uint64_t Address,
-                                            const void *Decoder) {
-  return decodeRegisterClass(Inst, RegNo, QFRegs);
-}
-
 static DecodeStatus DecodeSPERCRegisterClass(MCInst &Inst, uint64_t RegNo,
                                             uint64_t Address,
                                             const void *Decoder) {
@@ -401,14 +395,9 @@ DecodeStatus PPCDisassembler::getInstruction(MCInst &MI, uint64_t &Size,
   // Read the instruction in the proper endianness.
   uint64_t Inst = ReadFunc(Bytes.data());
 
-  if (STI.getFeatureBits()[PPC::FeatureQPX]) {
-    DecodeStatus result =
-      decodeInstruction(DecoderTableQPX32, MI, Inst, Address, this, STI);
-    if (result != MCDisassembler::Fail)
-      return result;
-  } else if (STI.getFeatureBits()[PPC::FeatureSPE]) {
+  if (STI.getFeatureBits()[PPC::FeatureSPE]) {
     DecodeStatus result =
-      decodeInstruction(DecoderTableSPE32, MI, Inst, Address, this, STI);
+        decodeInstruction(DecoderTableSPE32, MI, Inst, Address, this, STI);
     if (result != MCDisassembler::Fail)
       return result;
   }
diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCInstPrinter.cpp b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCInstPrinter.cpp
index 222bf2fa82836..ce1a43a0c25b2 100644
--- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCInstPrinter.cpp
+++ b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCInstPrinter.cpp
@@ -49,18 +49,6 @@ FullRegNamesWithPercent("ppc-reg-with-percent-prefix", cl::Hidden,
 
 void PPCInstPrinter::printRegName(raw_ostream &OS, unsigned RegNo) const {
   const char *RegName = getRegisterName(RegNo);
-  if (RegName[0] == 'q' /* QPX */) {
-    // The system toolchain on the BG/Q does not understand QPX register names
-    // in .cfi_* directives, so print the name of the floating-point
-    // subregister instead.
-    std::string RN(RegName);
-
-    RN[0] = 'f';
-    OS << RN;
-
-    return;
-  }
-
   OS << RegName;
 }
 
diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.h b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.h
index 719e005d98135..325ede0fc17ac 100644
--- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.h
+++ b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.h
@@ -159,7 +159,6 @@ using llvm::MCPhysReg;
   static const MCPhysReg SPERegs[32] = PPC_REGS0_31(PPC::S); \
   static const MCPhysReg VFRegs[32] = PPC_REGS0_31(PPC::VF); \
   static const MCPhysReg VRegs[32] = PPC_REGS0_31(PPC::V); \
-  static const MCPhysReg QFRegs[32] = PPC_REGS0_31(PPC::QF); \
   static const MCPhysReg RRegsNoR0[32] = \
     PPC_REGS_NO0_31(PPC::ZERO, PPC::R); \
   static const MCPhysReg XRegsNoX0[32] = \
diff --git a/llvm/lib/Target/PowerPC/PPC.h b/llvm/lib/Target/PowerPC/PPC.h
index 3106290442afa..24a9d419d3ea5 100644
--- a/llvm/lib/Target/PowerPC/PPC.h
+++ b/llvm/lib/Target/PowerPC/PPC.h
@@ -44,7 +44,6 @@ namespace llvm {
   FunctionPass *createPPCMIPeepholePass();
   FunctionPass *createPPCBranchSelectionPass();
   FunctionPass *createPPCBranchCoalescingPass();
-  FunctionPass *createPPCQPXLoadSplatPass();
   FunctionPass *createPPCISelDag(PPCTargetMachine &TM, CodeGenOpt::Level OL);
   FunctionPass *createPPCTLSDynamicCallPass();
   FunctionPass *createPPCBoolRetToIntPass();
@@ -68,7 +67,6 @@ namespace llvm {
   void initializePPCReduceCRLogicalsPass(PassRegistry&);
   void initializePPCBSelPass(PassRegistry&);
   void initializePPCBranchCoalescingPass(PassRegistry&);
-  void initializePPCQPXLoadSplatPass(PassRegistry&);
   void initializePPCBoolRetToIntPass(PassRegistry&);
   void initializePPCExpandISELPass(PassRegistry &);
   void initializePPCPreEmitPeepholePass(PassRegistry &);
diff --git a/llvm/lib/Target/PowerPC/PPC.td b/llvm/lib/Target/PowerPC/PPC.td
index 9ad78bf67fe6c..adb9366217d51 100644
--- a/llvm/lib/Target/PowerPC/PPC.td
+++ b/llvm/lib/Target/PowerPC/PPC.td
@@ -132,9 +132,6 @@ def FeaturePPC4xx    : SubtargetFeature<"ppc4xx", "IsPPC4xx", "true",
                                         "Enable PPC 4xx instructions">;
 def FeaturePPC6xx    : SubtargetFeature<"ppc6xx", "IsPPC6xx", "true",
                                         "Enable PPC 6xx instructions">;
-def FeatureQPX       : SubtargetFeature<"qpx","HasQPX", "true",
-                                        "Enable QPX instructions",
-                                        [FeatureFPU]>;
 def FeatureVSX       : SubtargetFeature<"vsx","HasVSX", "true",
                                         "Enable VSX instructions",
                                         [FeatureAltivec]>;
@@ -193,7 +190,7 @@ def FeatureFloat128 :
 def FeaturePOPCNTD   : SubtargetFeature<"popcntd","HasPOPCNTD",
                                         "POPCNTD_Fast",
                                         "Enable the popcnt[dw] instructions">;
-// Note that for the a2/a2q processor models we should not use popcnt[dw] by
+// Note that for the a2 processor models we should not use popcnt[dw] by
 // default. These processors do support the instructions, but they're
 // microcoded, and the software emulation is about twice as fast.
 def FeatureSlowPOPCNTD : SubtargetFeature<"slow-popcntd","HasPOPCNTD",
@@ -514,15 +511,6 @@ def : ProcessorModel<"a2", PPCA2Model,
                    FeatureFPRND, FeatureFPCVT, FeatureISEL,
                    FeatureSlowPOPCNTD, FeatureCMPB, FeatureLDBRX,
                    Feature64Bit /*, Feature64BitRegs */, FeatureMFTB]>;
-def : ProcessorModel<"a2q", PPCA2Model,
-                  [DirectiveA2, FeatureICBT, FeatureBookE, FeatureMFOCRF,
-                   FeatureFCPSGN, FeatureFSqrt, FeatureFRE, FeatureFRES,
-                   FeatureFRSQRTE, FeatureFRSQRTES, FeatureRecipPrec,
-                   FeatureSTFIWX, FeatureLFIWAX,
-                   FeatureFPRND, FeatureFPCVT, FeatureISEL,
-                   FeatureSlowPOPCNTD, FeatureCMPB, FeatureLDBRX,
-                   Feature64Bit /*, Feature64BitRegs */, FeatureQPX,
-                   FeatureMFTB]>;
 def : ProcessorModel<"pwr3", G5Model,
                   [DirectivePwr3, FeatureAltivec,
                    FeatureFRES, FeatureFRSQRTE, FeatureMFOCRF,
diff --git a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
index 540e620a845bc..5affddd8d147a 100644
--- a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
+++ b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
@@ -549,9 +549,6 @@ void PPCAsmPrinter::emitInstruction(const MachineInstr *MI) {
         if (Subtarget->hasSPE()) {
           if (PPC::F4RCRegClass.contains(Reg) ||
               PPC::F8RCRegClass.contains(Reg) ||
-              PPC::QBRCRegClass.contains(Reg) ||
-              PPC::QFRCRegClass.contains(Reg) ||
-              PPC::QSRCRegClass.contains(Reg) ||
               PPC::VFRCRegClass.contains(Reg) ||
               PPC::VRRCRegClass.contains(Reg) ||
               PPC::VSFRCRegClass.contains(Reg) ||
diff --git a/llvm/lib/Target/PowerPC/PPCCallingConv.td b/llvm/lib/Target/PowerPC/PPCCallingConv.td
index 1eaa7f7a44b39..9a15490f1fb0d 100644
--- a/llvm/lib/Target/PowerPC/PPCCallingConv.td
+++ b/llvm/lib/Target/PowerPC/PPCCallingConv.td
@@ -61,9 +61,6 @@ def RetCC_PPC_Cold : CallingConv<[
   CCIfType<[f64], CCAssignToReg<[F1]>>,
   CCIfType<[f128], CCIfSubtarget<"hasP9Vector()", CCAssignToReg<[V2]>>>,
 
-  CCIfType<[v4f64, v4f32, v4i1],
-           CCIfSubtarget<"hasQPX()", CCAssignToReg<[QF1]>>>,
-
   CCIfType<[v16i8, v8i16, v4i32, v2i64, v1i128, v4f32, v2f64],
            CCIfSubtarget<"hasAltivec()",
            CCAssignToReg<[V2]>>>
@@ -98,10 +95,6 @@ def RetCC_PPC : CallingConv<[
            CCIfSubtarget<"hasP9Vector()",
            CCAssignToReg<[V2, V3, V4, V5, V6, V7, V8, V9]>>>,
 
-  // QPX vectors are returned in QF1 and QF2. 
-  CCIfType<[v4f64, v4f32, v4i1],
-           CCIfSubtarget<"hasQPX()", CCAssignToReg<[QF1, QF2]>>>,
- 
   // Vector types returned as "direct" go into V2 .. V9; note that only the
   // ELFv2 ABI fully utilizes all these registers.
   CCIfType<[v16i8, v8i16, v4i32, v2i64, v1i128, v4f32, v2f64],
@@ -158,8 +151,6 @@ def RetCC_PPC64_ELF_FIS : CallingConv<[
   CCIfType<[f128],
            CCIfSubtarget<"hasP9Vector()",
            CCAssignToReg<[V2, V3, V4, V5, V6, V7, V8, V9]>>>,
-  CCIfType<[v4f64, v4f32, v4i1],
-           CCIfSubtarget<"hasQPX()", CCAssignToReg<[QF1, QF2]>>>,
   CCIfType<[v16i8, v8i16, v4i32, v2i64, v1i128, v4f32, v2f64],
            CCIfSubtarget<"hasAltivec()",
            CCAssignToReg<[V2, V3, V4, V5, V6, V7, V8, V9]>>>
@@ -223,9 +214,6 @@ def CC_PPC32_SVR4_Common : CallingConv<[
   CCIfType<[f32], CCIfSubtarget<"hasSPE()", CCAssignToStack<4, 4>>>,
   CCIfType<[f64], CCIfSubtarget<"hasSPE()", CCAssignToStack<8, 8>>>,
 
-  // QPX vectors that are stored in double precision need 32-byte alignment.
-  CCIfType<[v4f64, v4i1], CCAssignToStack<32, 32>>,
-
   // Vectors and float128 get 16-byte stack slots that are 16-byte aligned.
   CCIfType<[v16i8, v8i16, v4i32, v4f32, v2f64, v2i64], CCAssignToStack<16, 16>>,
   CCIfType<[f128], CCIfSubtarget<"hasP9Vector()", CCAssignToStack<16, 16>>>
@@ -243,10 +231,6 @@ def CC_PPC32_SVR4_VarArg : CallingConv<[
 // put vector arguments in vector registers before putting them on the stack.
 let Entry = 1 in
 def CC_PPC32_SVR4 : CallingConv<[
-  // QPX vectors mirror the scalar FP convention.
-  CCIfType<[v4f64, v4f32, v4i1], CCIfSubtarget<"hasQPX()",
-    CCAssignToReg<[QF1, QF2, QF3, QF4, QF5, QF6, QF7, QF8]>>>,
-
   // The first 12 Vector arguments are passed in AltiVec registers.
   CCIfType<[v16i8, v8i16, v4i32, v2i64, v1i128, v4f32, v2f64],
            CCIfSubtarget<"hasAltivec()", CCAssignToReg<[V2, V3, V4, V5, V6, V7,
diff --git a/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp b/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp
index 8ffd89ef5ccd2..3e218e14d8d44 100644
--- a/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp
@@ -4142,7 +4142,7 @@ bool PPCDAGToDAGISel::trySETCC(SDNode *N) {
   // Altivec Vector compare instructions do not set any CR register by default and
   // vector compare operations return the same type as the operands.
   if (LHS.getValueType().isVector()) {
-    if (Subtarget->hasQPX() || Subtarget->hasSPE())
+    if (Subtarget->hasSPE())
       return false;
 
     EVT VecVT = LHS.getValueType();
@@ -4813,8 +4813,6 @@ void PPCDAGToDAGISel::Select(SDNode *N) {
         assert((!isSExt || LoadedVT == MVT::i16) && "Invalid sext update load");
         switch (LoadedVT.getSimpleVT().SimpleTy) {
           default: llvm_unreachable("Invalid PPC load type!");
-          case MVT::v4f64: Opcode = PPC::QVLFDUX; break; // QPX
-          case MVT::v4f32: Opcode = PPC::QVLFSUX; break; // QPX
           case MVT::f64: Opcode = PPC::LFDUX; break;
           case MVT::f32: Opcode = PPC::LFSUX; break;
           case MVT::i32: Opcode = PPC::LWZUX; break;
@@ -5095,12 +5093,6 @@ void PPCDAGToDAGISel::Select(SDNode *N) {
       SelectCCOp = PPC::SELECT_CC_F16;
     else if (Subtarget->hasSPE())
       SelectCCOp = PPC::SELECT_CC_SPE;
-    else if (Subtarget->hasQPX() && N->getValueType(0) == MVT::v4f64)
-      SelectCCOp = PPC::SELECT_CC_QFRC;
-    else if (Subtarget->hasQPX() && N->getValueType(0) == MVT::v4f32)
-      SelectCCOp = PPC::SELECT_CC_QSRC;
-    else if (Subtarget->hasQPX() && N->getValueType(0) == MVT::v4i1)
-      SelectCCOp = PPC::SELECT_CC_QBRC;
     else if (N->getValueType(0) == MVT::v2f64 ||
              N->getValueType(0) == MVT::v2i64)
       SelectCCOp = PPC::SELECT_CC_VSRC;
@@ -5856,9 +5848,6 @@ void PPCDAGToDAGISel::PeepholeCROps() {
       case PPC::SELECT_I8:
       case PPC::SELECT_F4:
       case PPC::SELECT_F8:
-      case PPC::SELECT_QFRC:
-      case PPC::SELECT_QSRC:
-      case PPC::SELECT_QBRC:
       case PPC::SELECT_SPE:
       case PPC::SELECT_SPE4:
       case PPC::SELECT_VRRC:
@@ -6177,9 +6166,6 @@ void PPCDAGToDAGISel::PeepholeCROps() {
       case PPC::SELECT_I8:
       case PPC::SELECT_F4:
       case PPC::SELECT_F8:
-      case PPC::SELECT_QFRC:
-      case PPC::SELECT_QSRC:
-      case PPC::SELECT_QBRC:
       case PPC::SELECT_SPE:
       case PPC::SELECT_SPE4:
       case PPC::SELECT_VRRC:
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index ae840a9fa37de..db3833d595797 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -1095,161 +1095,6 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM,
     }
   }
 
-  if (Subtarget.hasQPX()) {
-    setOperationAction(ISD::FADD, MVT::v4f64, Legal);
-    setOperationAction(ISD::FSUB, MVT::v4f64, Legal);
-    setOperationAction(ISD::FMUL, MVT::v4f64, Legal);
-    setOperationAction(ISD::FREM, MVT::v4f64, Expand);
-
-    setOperationAction(ISD::FCOPYSIGN, MVT::v4f64, Legal);
-    setOperationAction(ISD::FGETSIGN, MVT::v4f64, Expand);
-
-    setOperationAction(ISD::LOAD  , MVT::v4f64, Custom);
-    setOperationAction(ISD::STORE , MVT::v4f64, Custom);
-
-    setTruncStoreAction(MVT::v4f64, MVT::v4f32, Custom);
-    setLoadExtAction(ISD::EXTLOAD, MVT::v4f64, MVT::v4f32, Custom);
-
-    if (!Subtarget.useCRBits())
-      setOperationAction(ISD::SELECT, MVT::v4f64, Expand);
-    setOperationAction(ISD::VSELECT, MVT::v4f64, Legal);
-
-    setOperationAction(ISD::EXTRACT_VECTOR_ELT , MVT::v4f64, Legal);
-    setOperationAction(ISD::INSERT_VECTOR_ELT , MVT::v4f64, Expand);
-    setOperationAction(ISD::CONCAT_VECTORS , MVT::v4f64, Expand);
-    setOperationAction(ISD::EXTRACT_SUBVECTOR , MVT::v4f64, Expand);
-    setOperationAction(ISD::VECTOR_SHUFFLE , MVT::v4f64, Custom);
-    setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4f64, Legal);
-    setOperationAction(ISD::BUILD_VECTOR, MVT::v4f64, Custom);
-
-    setOperationAction(ISD::FP_TO_SINT , MVT::v4f64, Legal);
-    setOperationAction(ISD::FP_TO_UINT , MVT::v4f64, Expand);
-
-    setOperationAction(ISD::FP_ROUND , MVT::v4f32, Legal);
-    setOperationAction(ISD::FP_EXTEND, MVT::v4f64, Legal);
-
-    setOperationAction(ISD::FNEG , MVT::v4f64, Legal);
-    setOperationAction(ISD::FABS , MVT::v4f64, Legal);
-    setOperationAction(ISD::FSIN , MVT::v4f64, Expand);
-    setOperationAction(ISD::FCOS , MVT::v4f64, Expand);
-    setOperationAction(ISD::FPOW , MVT::v4f64, Expand);
-    setOperationAction(ISD::FLOG , MVT::v4f64, Expand);
-    setOperationAction(ISD::FLOG2 , MVT::v4f64, Expand);
-    setOperationAction(ISD::FLOG10 , MVT::v4f64, Expand);
-    setOperationAction(ISD::FEXP , MVT::v4f64, Expand);
-    setOperationAction(ISD::FEXP2 , MVT::v4f64, Expand);
-
-    setOperationAction(ISD::FMINNUM, MVT::v4f64, Legal);
-    setOperationAction(ISD::FMAXNUM, MVT::v4f64, Legal);
-
-    setIndexedLoadAction(ISD::PRE_INC, MVT::v4f64, Legal);
-    setIndexedStoreAction(ISD::PRE_INC, MVT::v4f64, Legal);
-
-    addRegisterClass(MVT::v4f64, &PPC::QFRCRegClass);
-
-    setOperationAction(ISD::FADD, MVT::v4f32, Legal);
-    setOperationAction(ISD::FSUB, MVT::v4f32, Legal);
-    setOperationAction(ISD::FMUL, MVT::v4f32, Legal);
-    setOperationAction(ISD::FREM, MVT::v4f32, Expand);
-
-    setOperationAction(ISD::FCOPYSIGN, MVT::v4f32, Legal);
-    setOperationAction(ISD::FGETSIGN, MVT::v4f32, Expand);
-
-    setOperationAction(ISD::LOAD  , MVT::v4f32, Custom);
-    setOperationAction(ISD::STORE , MVT::v4f32, Custom);
-
-    if (!Subtarget.useCRBits())
-      setOperationAction(ISD::SELECT, MVT::v4f32, Expand);
-    setOperationAction(ISD::VSELECT, MVT::v4f32, Legal);
-
-    setOperationAction(ISD::EXTRACT_VECTOR_ELT , MVT::v4f32, Legal);
-    setOperationAction(ISD::INSERT_VECTOR_ELT , MVT::v4f32, Expand);
-    setOperationAction(ISD::CONCAT_VECTORS , MVT::v4f32, Expand);
-    setOperationAction(ISD::EXTRACT_SUBVECTOR , MVT::v4f32, Expand);
-    setOperationAction(ISD::VECTOR_SHUFFLE , MVT::v4f32, Custom);
-    setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4f32, Legal);
-    setOperationAction(ISD::BUILD_VECTOR, MVT::v4f32, Custom);
-
-    setOperationAction(ISD::FP_TO_SINT , MVT::v4f32, Legal);
-    setOperationAction(ISD::FP_TO_UINT , MVT::v4f32, Expand);
-
-    setOperationAction(ISD::FNEG , MVT::v4f32, Legal);
-    setOperationAction(ISD::FABS , MVT::v4f32, Legal);
-    setOperationAction(ISD::FSIN , MVT::v4f32, Expand);
-    setOperationAction(ISD::FCOS , MVT::v4f32, Expand);
-    setOperationAction(ISD::FPOW , MVT::v4f32, Expand);
-    setOperationAction(ISD::FLOG , MVT::v4f32, Expand);
-    setOperationAction(ISD::FLOG2 , MVT::v4f32, Expand);
-    setOperationAction(ISD::FLOG10 , MVT::v4f32, Expand);
-    setOperationAction(ISD::FEXP , MVT::v4f32, Expand);
-    setOperationAction(ISD::FEXP2 , MVT::v4f32, Expand);
-
-    setOperationAction(ISD::FMINNUM, MVT::v4f32, Legal);
-    setOperationAction(ISD::FMAXNUM, MVT::v4f32, Legal);
-
-    setIndexedLoadAction(ISD::PRE_INC, MVT::v4f32, Legal);
-    setIndexedStoreAction(ISD::PRE_INC, MVT::v4f32, Legal);
-
-    addRegisterClass(MVT::v4f32, &PPC::QSRCRegClass);
-
-    setOperationAction(ISD::AND , MVT::v4i1, Legal);
-    setOperationAction(ISD::OR , MVT::v4i1, Legal);
-    setOperationAction(ISD::XOR , MVT::v4i1, Legal);
-
-    if (!Subtarget.useCRBits())
-      setOperationAction(ISD::SELECT, MVT::v4i1, Expand);
-    setOperationAction(ISD::VSELECT, MVT::v4i1, Legal);
-
-    setOperationAction(ISD::LOAD  , MVT::v4i1, Custom);
-    setOperationAction(ISD::STORE , MVT::v4i1, Custom);
-
-    setOperationAction(ISD::EXTRACT_VECTOR_ELT , MVT::v4i1, Custom);
-    setOperationAction(ISD::INSERT_VECTOR_ELT , MVT::v4i1, Expand);
-    setOperationAction(ISD::CONCAT_VECTORS , MVT::v4i1, Expand);
-    setOperationAction(ISD::EXTRACT_SUBVECTOR , MVT::v4i1, Expand);
-    setOperationAction(ISD::VECTOR_SHUFFLE , MVT::v4i1, Custom);
-    setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4i1, Expand);
-    setOperationAction(ISD::BUILD_VECTOR, MVT::v4i1, Custom);
-
-    setOperationAction(ISD::SINT_TO_FP, MVT::v4i1, Custom);
-    setOperationAction(ISD::UINT_TO_FP, MVT::v4i1, Custom);
-
-    addRegisterClass(MVT::v4i1, &PPC::QBRCRegClass);
-
-    setOperationAction(ISD::FFLOOR, MVT::v4f64, Legal);
-    setOperationAction(ISD::FCEIL,  MVT::v4f64, Legal);
-    setOperationAction(ISD::FTRUNC, MVT::v4f64, Legal);
-    setOperationAction(ISD::FROUND, MVT::v4f64, Legal);
-
-    setOperationAction(ISD::FFLOOR, MVT::v4f32, Legal);
-    setOperationAction(ISD::FCEIL,  MVT::v4f32, Legal);
-    setOperationAction(ISD::FTRUNC, MVT::v4f32, Legal);
-    setOperationAction(ISD::FROUND, MVT::v4f32, Legal);
-
-    setOperationAction(ISD::FNEARBYINT, MVT::v4f64, Expand);
-    setOperationAction(ISD::FNEARBYINT, MVT::v4f32, Expand);
-
-    // These need to set FE_INEXACT, and so cannot be vectorized here.
-    setOperationAction(ISD::FRINT, MVT::v4f64, Expand);
-    setOperationAction(ISD::FRINT, MVT::v4f32, Expand);
-
-    if (TM.Options.UnsafeFPMath) {
-      setOperationAction(ISD::FDIV, MVT::v4f64, Legal);
-      setOperationAction(ISD::FSQRT, MVT::v4f64, Legal);
-
-      setOperationAction(ISD::FDIV, MVT::v4f32, Legal);
-      setOperationAction(ISD::FSQRT, MVT::v4f32, Legal);
-    } else {
-      setOperationAction(ISD::FDIV, MVT::v4f64, Expand);
-      setOperationAction(ISD::FSQRT, MVT::v4f64, Expand);
-
-      setOperationAction(ISD::FDIV, MVT::v4f32, Expand);
-      setOperationAction(ISD::FSQRT, MVT::v4f32, Expand);
-    }
-
-    // TODO: Handle constrained floating-point operations of v4f64
-  }
-
   if (Subtarget.has64BitSupport())
     setOperationAction(ISD::PREFETCH, MVT::Other, Legal);
 
@@ -1438,8 +1283,8 @@ unsigned PPCTargetLowering::getByValTypeAlignment(Type *Ty,
   // 16byte and wider vectors are passed on 16byte boundary.
   // The rest is 8 on PPC64 and 4 on PPC32 boundary.
   Align Alignment = Subtarget.isPPC64() ? Align(8) : Align(4);
-  if (Subtarget.hasAltivec() || Subtarget.hasQPX())
-    getMaxByValAlign(Ty, Alignment, Subtarget.hasQPX() ? Align(32) : Align(16));
+  if (Subtarget.hasAltivec())
+    getMaxByValAlign(Ty, Alignment, Align(16));
   return Alignment.value();
 }
 
@@ -1577,12 +1422,6 @@ const char *PPCTargetLowering::getTargetNodeName(unsigned Opcode) const {
   case PPCISD::XXSWAPD:         return "PPCISD::XXSWAPD";
   case PPCISD::SWAP_NO_CHAIN:   return "PPCISD::SWAP_NO_CHAIN";
   case PPCISD::VABSD:           return "PPCISD::VABSD";
-  case PPCISD::QVFPERM:         return "PPCISD::QVFPERM";
-  case PPCISD::QVGPCI:          return "PPCISD::QVGPCI";
-  case PPCISD::QVALIGNI:        return "PPCISD::QVALIGNI";
-  case PPCISD::QVESPLATI:       return "PPCISD::QVESPLATI";
-  case PPCISD::QBFLT:           return "PPCISD::QBFLT";
-  case PPCISD::QVLFSb:          return "PPCISD::QVLFSb";
   case PPCISD::BUILD_FP128:     return "PPCISD::BUILD_FP128";
   case PPCISD::BUILD_SPE64:     return "PPCISD::BUILD_SPE64";
   case PPCISD::EXTRACT_SPE:     return "PPCISD::EXTRACT_SPE";
@@ -1601,9 +1440,6 @@ EVT PPCTargetLowering::getSetCCResultType(const DataLayout &DL, LLVMContext &C,
   if (!VT.isVector())
     return Subtarget.useCRBits() ? MVT::i1 : MVT::i32;
 
-  if (Subtarget.hasQPX())
-    return EVT::getVectorVT(C, MVT::i1, VT.getVectorNumElements());
-
   return VT.changeVectorElementTypeToInteger();
 }
 
@@ -2777,16 +2613,9 @@ bool PPCTargetLowering::getPreIndexedAddressParts(SDNode *N, SDValue &Base,
     return false;
   }
 
-  // PowerPC doesn't have preinc load/store instructions for vectors (except
-  // for QPX, which does have preinc r+r forms).
-  if (VT.isVector()) {
-    if (!Subtarget.hasQPX() || (VT != MVT::v4f64 && VT != MVT::v4f32)) {
-      return false;
-    } else if (SelectAddressRegRegOnly(Ptr, Offset, Base, DAG)) {
-      AM = ISD::PRE_INC;
-      return true;
-    }
-  }
+  // PowerPC doesn't have preinc load/store instructions for vectors
+  if (VT.isVector())
+    return false;
 
   if (SelectAddressRegReg(Ptr, Base, Offset, DAG)) {
     // Common code will reject creating a pre-inc form if the base pointer
@@ -3508,11 +3337,6 @@ static const MCPhysReg FPR[] = {PPC::F1,  PPC::F2,  PPC::F3, PPC::F4, PPC::F5,
                                 PPC::F6,  PPC::F7,  PPC::F8, PPC::F9, PPC::F10,
                                 PPC::F11, PPC::F12, PPC::F13};
 
-/// QFPR - The set of QPX registers that should be allocated for arguments.
-static const MCPhysReg QFPR[] = {
-    PPC::QF1, PPC::QF2, PPC::QF3,  PPC::QF4,  PPC::QF5,  PPC::QF6, PPC::QF7,
-    PPC::QF8, PPC::QF9, PPC::QF10, PPC::QF11, PPC::QF12, PPC::QF13};
-
 /// CalculateStackSlotSize - Calculates the size reserved for this argument on
 /// the stack.
 static unsigned CalculateStackSlotSize(EVT ArgVT, ISD::ArgFlagsTy Flags,
@@ -3542,10 +3366,6 @@ static Align CalculateStackSlotAlignment(EVT ArgVT, EVT OrigVT,
       ArgVT == MVT::v2f64 || ArgVT == MVT::v2i64 ||
       ArgVT == MVT::v1i128 || ArgVT == MVT::f128)
     Alignment = Align(16);
-  // QPX vector types stored in double-precision are padded to a 32 byte
-  // boundary.
-  else if (ArgVT == MVT::v4f64 || ArgVT == MVT::v4i1)
-    Alignment = Align(32);
 
   // ByVal parameters are aligned as requested.
   if (Flags.isByVal()) {
@@ -3577,14 +3397,11 @@ static Align CalculateStackSlotAlignment(EVT ArgVT, EVT OrigVT,
 /// stack slot (instead of being passed in registers).  ArgOffset,
 /// AvailableFPRs, and AvailableVRs must hold the current argument
 /// position, and will be updated to account for this argument.
-static bool CalculateStackSlotUsed(EVT ArgVT, EVT OrigVT,
-                                   ISD::ArgFlagsTy Flags,
-                                   unsigned PtrByteSize,
-                                   unsigned LinkageSize,
-                                   unsigned ParamAreaSize,
-                                   unsigned &ArgOffset,
+static bool CalculateStackSlotUsed(EVT ArgVT, EVT OrigVT, ISD::ArgFlagsTy Flags,
+                                   unsigned PtrByteSize, unsigned LinkageSize,
+                                   unsigned ParamAreaSize, unsigned &ArgOffset,
                                    unsigned &AvailableFPRs,
-                                   unsigned &AvailableVRs, bool HasQPX) {
+                                   unsigned &AvailableVRs) {
   bool UseMemory = false;
 
   // Respect alignment of argument on the stack.
@@ -3608,11 +3425,7 @@ static bool CalculateStackSlotUsed(EVT ArgVT, EVT OrigVT,
   // However, if the argument is actually passed in an FPR or a VR,
   // we don't use memory after all.
   if (!Flags.isByVal()) {
-    if (ArgVT == MVT::f32 || ArgVT == MVT::f64 ||
-        // QPX registers overlap with the scalar FP registers.
-        (HasQPX && (ArgVT == MVT::v4f32 ||
-                    ArgVT == MVT::v4f64 ||
-                    ArgVT == MVT::v4i1)))
+    if (ArgVT == MVT::f32 || ArgVT == MVT::f64)
       if (AvailableFPRs > 0) {
         --AvailableFPRs;
         return false;
@@ -3751,18 +3564,12 @@ SDValue PPCTargetLowering::LowerFormalArguments_32SVR4(
           RC = &PPC::VRRCRegClass;
           break;
         case MVT::v4f32:
-          RC = Subtarget.hasQPX() ? &PPC::QSRCRegClass : &PPC::VRRCRegClass;
+          RC = &PPC::VRRCRegClass;
           break;
         case MVT::v2f64:
         case MVT::v2i64:
           RC = &PPC::VRRCRegClass;
           break;
-        case MVT::v4f64:
-          RC = &PPC::QFRCRegClass;
-          break;
-        case MVT::v4i1:
-          RC = &PPC::QBRCRegClass;
-          break;
       }
 
       SDValue ArgValue;
@@ -3961,7 +3768,6 @@ SDValue PPCTargetLowering::LowerFormalArguments_64SVR4(
   const unsigned Num_GPR_Regs = array_lengthof(GPR);
   const unsigned Num_FPR_Regs = useSoftFloat() ? 0 : 13;
   const unsigned Num_VR_Regs  = array_lengthof(VR);
-  const unsigned Num_QFPR_Regs = Num_FPR_Regs;
 
   // Do a first pass over the arguments to determine whether the ABI
   // guarantees that our caller has allocated the parameter save area
@@ -3980,8 +3786,7 @@ SDValue PPCTargetLowering::LowerFormalArguments_64SVR4(
 
     if (CalculateStackSlotUsed(Ins[i].VT, Ins[i].ArgVT, Ins[i].Flags,
                                PtrByteSize, LinkageSize, ParamAreaSize,
-                               NumBytes, AvailableFPRs, AvailableVRs,
-                               Subtarget.hasQPX()))
+                               NumBytes, AvailableFPRs, AvailableVRs))
       HasParameterArea = true;
   }
 
@@ -3991,7 +3796,6 @@ SDValue PPCTargetLowering::LowerFormalArguments_64SVR4(
 
   unsigned ArgOffset = LinkageSize;
   unsigned GPR_idx = 0, FPR_idx = 0, VR_idx = 0;
-  unsigned &QFPR_idx = FPR_idx;
   SmallVector<SDValue, 8> MemOps;
   Function::const_arg_iterator FuncArg = MF.getFunction().arg_begin();
   unsigned CurArgIdx = 0;
@@ -4234,51 +4038,20 @@ SDValue PPCTargetLowering::LowerFormalArguments_64SVR4(
     case MVT::v2i64:
     case MVT::v1i128:
     case MVT::f128:
-      if (!Subtarget.hasQPX()) {
-        // These can be scalar arguments or elements of a vector array type
-        // passed directly.  The latter are used to implement ELFv2 homogenous
-        // vector aggregates.
-        if (VR_idx != Num_VR_Regs) {
-          unsigned VReg = MF.addLiveIn(VR[VR_idx], &PPC::VRRCRegClass);
-          ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, ObjectVT);
-          ++VR_idx;
-        } else {
-          if (CallConv == CallingConv::Fast)
-            ComputeArgOffset();
-          needsLoad = true;
-        }
-        if (CallConv != CallingConv::Fast || needsLoad)
-          ArgOffset += 16;
-        break;
-      } // not QPX
-
-      assert(ObjectVT.getSimpleVT().SimpleTy == MVT::v4f32 &&
-             "Invalid QPX parameter type");
-      LLVM_FALLTHROUGH;
-
-    case MVT::v4f64:
-    case MVT::v4i1:
-      // QPX vectors are treated like their scalar floating-point subregisters
-      // (except that they're larger).
-      unsigned Sz = ObjectVT.getSimpleVT().SimpleTy == MVT::v4f32 ? 16 : 32;
-      if (QFPR_idx != Num_QFPR_Regs) {
-        const TargetRegisterClass *RC;
-        switch (ObjectVT.getSimpleVT().SimpleTy) {
-        case MVT::v4f64: RC = &PPC::QFRCRegClass; break;
-        case MVT::v4f32: RC = &PPC::QSRCRegClass; break;
-        default:         RC = &PPC::QBRCRegClass; break;
-        }
-
-        unsigned VReg = MF.addLiveIn(QFPR[QFPR_idx], RC);
+      // These can be scalar arguments or elements of a vector array type
+      // passed directly.  The latter are used to implement ELFv2 homogenous
+      // vector aggregates.
+      if (VR_idx != Num_VR_Regs) {
+        unsigned VReg = MF.addLiveIn(VR[VR_idx], &PPC::VRRCRegClass);
         ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, ObjectVT);
-        ++QFPR_idx;
+        ++VR_idx;
       } else {
         if (CallConv == CallingConv::Fast)
           ComputeArgOffset();
         needsLoad = true;
       }
       if (CallConv != CallingConv::Fast || needsLoad)
-        ArgOffset += Sz;
+        ArgOffset += 16;
       break;
     }
 
@@ -4831,10 +4604,9 @@ needStackSlotPassParameters(const PPCSubtarget &Subtarget,
   for (const ISD::OutputArg& Param : Outs) {
     if (Param.Flags.isNest()) continue;
 
-    if (CalculateStackSlotUsed(Param.VT, Param.ArgVT, Param.Flags,
-                               PtrByteSize, LinkageSize, ParamAreaSize,
-                               NumBytes, AvailableFPRs, AvailableVRs,
-                               Subtarget.hasQPX()))
+    if (CalculateStackSlotUsed(Param.VT, Param.ArgVT, Param.Flags, PtrByteSize,
+                               LinkageSize, ParamAreaSize, NumBytes,
+                               AvailableFPRs, AvailableVRs))
       return true;
   }
   return false;
@@ -6064,7 +5836,6 @@ SDValue PPCTargetLowering::LowerCall_64SVR4(
   unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize();
   unsigned NumBytes = LinkageSize;
   unsigned GPR_idx = 0, FPR_idx = 0, VR_idx = 0;
-  unsigned &QFPR_idx = FPR_idx;
 
   static const MCPhysReg GPR[] = {
     PPC::X3, PPC::X4, PPC::X5, PPC::X6,
@@ -6078,7 +5849,6 @@ SDValue PPCTargetLowering::LowerCall_64SVR4(
   const unsigned NumGPRs = array_lengthof(GPR);
   const unsigned NumFPRs = useSoftFloat() ? 0 : 13;
   const unsigned NumVRs  = array_lengthof(VR);
-  const unsigned NumQFPRs = NumFPRs;
 
   // On ELFv2, we can avoid allocating the parameter area if all the arguments
   // can be passed to the callee in registers.
@@ -6093,9 +5863,8 @@ SDValue PPCTargetLowering::LowerCall_64SVR4(
     for (unsigned i = 0; i != NumOps; ++i) {
       if (Outs[i].Flags.isNest()) continue;
       if (CalculateStackSlotUsed(Outs[i].VT, Outs[i].ArgVT, Outs[i].Flags,
-                                PtrByteSize, LinkageSize, ParamAreaSize,
-                                NumBytesTmp, AvailableFPRs, AvailableVRs,
-                                Subtarget.hasQPX()))
+                                 PtrByteSize, LinkageSize, ParamAreaSize,
+                                 NumBytesTmp, AvailableFPRs, AvailableVRs))
         HasParameterArea = true;
     }
   }
@@ -6143,20 +5912,11 @@ SDValue PPCTargetLowering::LowerCall_64SVR4(
             continue;
           break;
         case MVT::v4f32:
-          // When using QPX, this is handled like a FP register, otherwise, it
-          // is an Altivec register.
-          if (Subtarget.hasQPX()) {
-            if (++NumFPRsUsed <= NumFPRs)
-              continue;
-          } else {
-            if (++NumVRsUsed <= NumVRs)
-              continue;
-          }
+          if (++NumVRsUsed <= NumVRs)
+            continue;
           break;
         case MVT::f32:
         case MVT::f64:
-        case MVT::v4f64: // QPX
-        case MVT::v4i1:  // QPX
           if (++NumFPRsUsed <= NumFPRs)
             continue;
           break;
@@ -6518,7 +6278,6 @@ SDValue PPCTargetLowering::LowerCall_64SVR4(
     case MVT::v2i64:
     case MVT::v1i128:
     case MVT::f128:
-      if (!Subtarget.hasQPX()) {
       // These can be scalar arguments or elements of a vector array type
       // passed directly.  The latter are used to implement ELFv2 homogenous
       // vector aggregates.
@@ -6574,63 +6333,6 @@ SDValue PPCTargetLowering::LowerCall_64SVR4(
       if (!IsFastCall)
         ArgOffset += 16;
       break;
-      } // not QPX
-
-      assert(Arg.getValueType().getSimpleVT().SimpleTy == MVT::v4f32 &&
-             "Invalid QPX parameter type");
-
-      LLVM_FALLTHROUGH;
-    case MVT::v4f64:
-    case MVT::v4i1: {
-      bool IsF32 = Arg.getValueType().getSimpleVT().SimpleTy == MVT::v4f32;
-      if (CFlags.IsVarArg) {
-        assert(HasParameterArea &&
-               "Parameter area must exist if we have a varargs call.");
-        // We could elide this store in the case where the object fits
-        // entirely in R registers.  Maybe later.
-        SDValue Store =
-            DAG.getStore(Chain, dl, Arg, PtrOff, MachinePointerInfo());
-        MemOpChains.push_back(Store);
-        if (QFPR_idx != NumQFPRs) {
-          SDValue Load = DAG.getLoad(IsF32 ? MVT::v4f32 : MVT::v4f64, dl, Store,
-                                     PtrOff, MachinePointerInfo());
-          MemOpChains.push_back(Load.getValue(1));
-          RegsToPass.push_back(std::make_pair(QFPR[QFPR_idx++], Load));
-        }
-        ArgOffset += (IsF32 ? 16 : 32);
-        for (unsigned i = 0; i < (IsF32 ? 16U : 32U); i += PtrByteSize) {
-          if (GPR_idx == NumGPRs)
-            break;
-          SDValue Ix = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff,
-                                   DAG.getConstant(i, dl, PtrVT));
-          SDValue Load =
-              DAG.getLoad(PtrVT, dl, Store, Ix, MachinePointerInfo());
-          MemOpChains.push_back(Load.getValue(1));
-          RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load));
-        }
-        break;
-      }
-
-      // Non-varargs QPX params go into registers or on the stack.
-      if (QFPR_idx != NumQFPRs) {
-        RegsToPass.push_back(std::make_pair(QFPR[QFPR_idx++], Arg));
-      } else {
-        if (IsFastCall)
-          ComputePtrOff();
-
-        assert(HasParameterArea &&
-               "Parameter area must exist to pass an argument in memory.");
-        LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset,
-                         true, CFlags.IsTailCall, true, MemOpChains,
-                         TailCallArguments, dl);
-        if (IsFastCall)
-          ArgOffset += (IsF32 ? 16 : 32);
-      }
-
-      if (!IsFastCall)
-        ArgOffset += (IsF32 ? 16 : 32);
-      break;
-      }
     }
   }
 
@@ -7301,8 +7003,6 @@ SDValue PPCTargetLowering::LowerFormalArguments_AIX(
 
   const PPCSubtarget &Subtarget =
       static_cast<const PPCSubtarget &>(DAG.getSubtarget());
-  if (Subtarget.hasQPX())
-    report_fatal_error("QPX support is not supported on AIX.");
 
   const bool IsPPC64 = Subtarget.isPPC64();
   const unsigned PtrByteSize = IsPPC64 ? 8 : 4;
@@ -7522,8 +7222,6 @@ SDValue PPCTargetLowering::LowerCall_AIX(
 
   const PPCSubtarget& Subtarget =
       static_cast<const PPCSubtarget&>(DAG.getSubtarget());
-  if (Subtarget.hasQPX())
-    report_fatal_error("QPX is not supported on AIX.");
   if (Subtarget.hasAltivec())
     report_fatal_error("Altivec support is unimplemented on AIX.");
 
@@ -7991,8 +7689,6 @@ SDValue PPCTargetLowering::lowerEH_SJLJ_LONGJMP(SDValue Op,
 }
 
 SDValue PPCTargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
-  if (Op.getValueType().isVector())
-    return LowerVectorLoad(Op, DAG);
 
   assert(Op.getValueType() == MVT::i1 &&
          "Custom lowering only for i1 loads");
@@ -8016,9 +7712,6 @@ SDValue PPCTargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
 }
 
 SDValue PPCTargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
-  if (Op.getOperand(1).getValueType().isVector())
-    return LowerVectorStore(Op, DAG);
-
   assert(Op.getOperand(1).getValueType() == MVT::i1 &&
          "Custom lowering only for i1 stores");
 
@@ -8595,27 +8288,6 @@ SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op,
   if (Op.getValueType() == MVT::f128)
     return Op;
 
-  if (Subtarget.hasQPX() && Op.getOperand(0).getValueType() == MVT::v4i1) {
-    if (Op.getValueType() != MVT::v4f32 && Op.getValueType() != MVT::v4f64)
-      return SDValue();
-
-    SDValue Value = Op.getOperand(0);
-    // The values are now known to be -1 (false) or 1 (true). To convert this
-    // into 0 (false) and 1 (true), add 1 and then divide by 2 (multiply by 0.5).
-    // This can be done with an fma and the 0.5 constant: (V+1.0)*0.5 = 0.5*V+0.5
-    Value = DAG.getNode(PPCISD::QBFLT, dl, MVT::v4f64, Value);
-
-    SDValue FPHalfs = DAG.getConstantFP(0.5, dl, MVT::v4f64);
-
-    Value = DAG.getNode(ISD::FMA, dl, MVT::v4f64, Value, FPHalfs, FPHalfs);
-
-    if (Op.getValueType() != MVT::v4f64)
-      Value = DAG.getNode(ISD::FP_ROUND, dl,
-                          Op.getValueType(), Value,
-                          DAG.getIntPtrConstant(1, dl));
-    return Value;
-  }
-
   // Don't handle ppc_fp128 here; let it be lowered to a libcall.
   if (Op.getValueType() != MVT::f32 && Op.getValueType() != MVT::f64)
     return SDValue();
@@ -9184,110 +8856,6 @@ SDValue PPCTargetLowering::LowerBUILD_VECTOR(SDValue Op,
   BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(Op.getNode());
   assert(BVN && "Expected a BuildVectorSDNode in LowerBUILD_VECTOR");
 
-  if (Subtarget.hasQPX() && Op.getValueType() == MVT::v4i1) {
-    // We first build an i32 vector, load it into a QPX register,
-    // then convert it to a floating-point vector and compare it
-    // to a zero vector to get the boolean result.
-    MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
-    int FrameIdx = MFI.CreateStackObject(16, Align(16), false);
-    MachinePointerInfo PtrInfo =
-        MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx);
-    EVT PtrVT = getPointerTy(DAG.getDataLayout());
-    SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT);
-
-    assert(BVN->getNumOperands() == 4 &&
-      "BUILD_VECTOR for v4i1 does not have 4 operands");
-
-    bool IsConst = true;
-    for (unsigned i = 0; i < 4; ++i) {
-      if (BVN->getOperand(i).isUndef()) continue;
-      if (!isa<ConstantSDNode>(BVN->getOperand(i))) {
-        IsConst = false;
-        break;
-      }
-    }
-
-    if (IsConst) {
-      Constant *One =
-        ConstantFP::get(Type::getFloatTy(*DAG.getContext()), 1.0);
-      Constant *NegOne =
-        ConstantFP::get(Type::getFloatTy(*DAG.getContext()), -1.0);
-
-      Constant *CV[4];
-      for (unsigned i = 0; i < 4; ++i) {
-        if (BVN->getOperand(i).isUndef())
-          CV[i] = UndefValue::get(Type::getFloatTy(*DAG.getContext()));
-        else if (isNullConstant(BVN->getOperand(i)))
-          CV[i] = NegOne;
-        else
-          CV[i] = One;
-      }
-
-      Constant *CP = ConstantVector::get(CV);
-      SDValue CPIdx =
-          DAG.getConstantPool(CP, getPointerTy(DAG.getDataLayout()), Align(16));
-
-      SDValue Ops[] = {DAG.getEntryNode(), CPIdx};
-      SDVTList VTs = DAG.getVTList({MVT::v4i1, /*chain*/ MVT::Other});
-      return DAG.getMemIntrinsicNode(
-          PPCISD::QVLFSb, dl, VTs, Ops, MVT::v4f32,
-          MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
-    }
-
-    SmallVector<SDValue, 4> Stores;
-    for (unsigned i = 0; i < 4; ++i) {
-      if (BVN->getOperand(i).isUndef()) continue;
-
-      unsigned Offset = 4*i;
-      SDValue Idx = DAG.getConstant(Offset, dl, FIdx.getValueType());
-      Idx = DAG.getNode(ISD::ADD, dl, FIdx.getValueType(), FIdx, Idx);
-
-      unsigned StoreSize = BVN->getOperand(i).getValueType().getStoreSize();
-      if (StoreSize > 4) {
-        Stores.push_back(
-            DAG.getTruncStore(DAG.getEntryNode(), dl, BVN->getOperand(i), Idx,
-                              PtrInfo.getWithOffset(Offset), MVT::i32));
-      } else {
-        SDValue StoreValue = BVN->getOperand(i);
-        if (StoreSize < 4)
-          StoreValue = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, StoreValue);
-
-        Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl, StoreValue, Idx,
-                                      PtrInfo.getWithOffset(Offset)));
-      }
-    }
-
-    SDValue StoreChain;
-    if (!Stores.empty())
-      StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
-    else
-      StoreChain = DAG.getEntryNode();
-
-    // Now load from v4i32 into the QPX register; this will extend it to
-    // v4i64 but not yet convert it to a floating point. Nevertheless, this
-    // is typed as v4f64 because the QPX register integer states are not
-    // explicitly represented.
-
-    SDValue Ops[] = {StoreChain,
-                     DAG.getConstant(Intrinsic::ppc_qpx_qvlfiwz, dl, MVT::i32),
-                     FIdx};
-    SDVTList VTs = DAG.getVTList({MVT::v4f64, /*chain*/ MVT::Other});
-
-    SDValue LoadedVect = DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN,
-      dl, VTs, Ops, MVT::v4i32, PtrInfo);
-    LoadedVect = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f64,
-      DAG.getConstant(Intrinsic::ppc_qpx_qvfcfidu, dl, MVT::i32),
-      LoadedVect);
-
-    SDValue FPZeros = DAG.getConstantFP(0.0, dl, MVT::v4f64);
-
-    return DAG.getSetCC(dl, MVT::v4i1, LoadedVect, FPZeros, ISD::SETEQ);
-  }
-
-  // All other QPX vectors are handled by generic code.
-  if (Subtarget.hasQPX())
-    return SDValue();
-
   // Check if this is a splat of a constant value.
   APInt APSplatBits, APSplatUndef;
   unsigned SplatBitSize;
@@ -10080,42 +9648,6 @@ SDValue PPCTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
     }
   }
 
-  if (Subtarget.hasQPX()) {
-    if (VT.getVectorNumElements() != 4)
-      return SDValue();
-
-    if (V2.isUndef()) V2 = V1;
-
-    int AlignIdx = PPC::isQVALIGNIShuffleMask(SVOp);
-    if (AlignIdx != -1) {
-      return DAG.getNode(PPCISD::QVALIGNI, dl, VT, V1, V2,
-                         DAG.getConstant(AlignIdx, dl, MVT::i32));
-    } else if (SVOp->isSplat()) {
-      int SplatIdx = SVOp->getSplatIndex();
-      if (SplatIdx >= 4) {
-        std::swap(V1, V2);
-        SplatIdx -= 4;
-      }
-
-      return DAG.getNode(PPCISD::QVESPLATI, dl, VT, V1,
-                         DAG.getConstant(SplatIdx, dl, MVT::i32));
-    }
-
-    // Lower this into a qvgpci/qvfperm pair.
-
-    // Compute the qvgpci literal
-    unsigned idx = 0;
-    for (unsigned i = 0; i < 4; ++i) {
-      int m = SVOp->getMaskElt(i);
-      unsigned mm = m >= 0 ? (unsigned) m : i;
-      idx |= mm << (3-i)*3;
-    }
-
-    SDValue V3 = DAG.getNode(PPCISD::QVGPCI, dl, MVT::v4f64,
-                             DAG.getConstant(idx, dl, MVT::i32));
-    return DAG.getNode(PPCISD::QVFPERM, dl, VT, V1, V2, V3);
-  }
-
   // Cases that are handled by instructions that take permute immediates
   // (such as vsplt*) should be left as VECTOR_SHUFFLE nodes so they can be
   // selected by the instruction selector.
@@ -10703,279 +10235,6 @@ SDValue PPCTargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
   return Op;
 }
 
-SDValue PPCTargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
-                                                   SelectionDAG &DAG) const {
-  SDLoc dl(Op);
-  SDNode *N = Op.getNode();
-
-  assert(N->getOperand(0).getValueType() == MVT::v4i1 &&
-         "Unknown extract_vector_elt type");
-
-  SDValue Value = N->getOperand(0);
-
-  // The first part of this is like the store lowering except that we don't
-  // need to track the chain.
-
-  // The values are now known to be -1 (false) or 1 (true). To convert this
-  // into 0 (false) and 1 (true), add 1 and then divide by 2 (multiply by 0.5).
-  // This can be done with an fma and the 0.5 constant: (V+1.0)*0.5 = 0.5*V+0.5
-  Value = DAG.getNode(PPCISD::QBFLT, dl, MVT::v4f64, Value);
-
-  // FIXME: We can make this an f32 vector, but the BUILD_VECTOR code needs to
-  // understand how to form the extending load.
-  SDValue FPHalfs = DAG.getConstantFP(0.5, dl, MVT::v4f64);
-
-  Value = DAG.getNode(ISD::FMA, dl, MVT::v4f64, Value, FPHalfs, FPHalfs);
-
-  // Now convert to an integer and store.
-  Value = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f64,
-    DAG.getConstant(Intrinsic::ppc_qpx_qvfctiwu, dl, MVT::i32),
-    Value);
-
-  MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
-  int FrameIdx = MFI.CreateStackObject(16, Align(16), false);
-  MachinePointerInfo PtrInfo =
-      MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx);
-  EVT PtrVT = getPointerTy(DAG.getDataLayout());
-  SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT);
-
-  SDValue StoreChain = DAG.getEntryNode();
-  SDValue Ops[] = {StoreChain,
-                   DAG.getConstant(Intrinsic::ppc_qpx_qvstfiw, dl, MVT::i32),
-                   Value, FIdx};
-  SDVTList VTs = DAG.getVTList(/*chain*/ MVT::Other);
-
-  StoreChain = DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID,
-    dl, VTs, Ops, MVT::v4i32, PtrInfo);
-
-  // Extract the value requested.
-  unsigned Offset = 4*cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
-  SDValue Idx = DAG.getConstant(Offset, dl, FIdx.getValueType());
-  Idx = DAG.getNode(ISD::ADD, dl, FIdx.getValueType(), FIdx, Idx);
-
-  SDValue IntVal =
-      DAG.getLoad(MVT::i32, dl, StoreChain, Idx, PtrInfo.getWithOffset(Offset));
-
-  if (!Subtarget.useCRBits())
-    return IntVal;
-
-  return DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, IntVal);
-}
-
-/// Lowering for QPX v4i1 loads
-SDValue PPCTargetLowering::LowerVectorLoad(SDValue Op,
-                                           SelectionDAG &DAG) const {
-  SDLoc dl(Op);
-  LoadSDNode *LN = cast<LoadSDNode>(Op.getNode());
-  SDValue LoadChain = LN->getChain();
-  SDValue BasePtr = LN->getBasePtr();
-
-  if (Op.getValueType() == MVT::v4f64 ||
-      Op.getValueType() == MVT::v4f32) {
-    EVT MemVT = LN->getMemoryVT();
-    unsigned Alignment = LN->getAlignment();
-
-    // If this load is properly aligned, then it is legal.
-    if (Alignment >= MemVT.getStoreSize())
-      return Op;
-
-    EVT ScalarVT = Op.getValueType().getScalarType(),
-        ScalarMemVT = MemVT.getScalarType();
-    unsigned Stride = ScalarMemVT.getStoreSize();
-
-    SDValue Vals[4], LoadChains[4];
-    for (unsigned Idx = 0; Idx < 4; ++Idx) {
-      SDValue Load;
-      if (ScalarVT != ScalarMemVT)
-        Load = DAG.getExtLoad(LN->getExtensionType(), dl, ScalarVT, LoadChain,
-                              BasePtr,
-                              LN->getPointerInfo().getWithOffset(Idx * Stride),
-                              ScalarMemVT, MinAlign(Alignment, Idx * Stride),
-                              LN->getMemOperand()->getFlags(), LN->getAAInfo());
-      else
-        Load = DAG.getLoad(ScalarVT, dl, LoadChain, BasePtr,
-                           LN->getPointerInfo().getWithOffset(Idx * Stride),
-                           MinAlign(Alignment, Idx * Stride),
-                           LN->getMemOperand()->getFlags(), LN->getAAInfo());
-
-      if (Idx == 0 && LN->isIndexed()) {
-        assert(LN->getAddressingMode() == ISD::PRE_INC &&
-               "Unknown addressing mode on vector load");
-        Load = DAG.getIndexedLoad(Load, dl, BasePtr, LN->getOffset(),
-                                  LN->getAddressingMode());
-      }
-
-      Vals[Idx] = Load;
-      LoadChains[Idx] = Load.getValue(1);
-
-      BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr,
-                            DAG.getConstant(Stride, dl,
-                                            BasePtr.getValueType()));
-    }
-
-    SDValue TF =  DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains);
-    SDValue Value = DAG.getBuildVector(Op.getValueType(), dl, Vals);
-
-    if (LN->isIndexed()) {
-      SDValue RetOps[] = { Value, Vals[0].getValue(1), TF };
-      return DAG.getMergeValues(RetOps, dl);
-    }
-
-    SDValue RetOps[] = { Value, TF };
-    return DAG.getMergeValues(RetOps, dl);
-  }
-
-  assert(Op.getValueType() == MVT::v4i1 && "Unknown load to lower");
-  assert(LN->isUnindexed() && "Indexed v4i1 loads are not supported");
-
-  // To lower v4i1 from a byte array, we load the byte elements of the
-  // vector and then reuse the BUILD_VECTOR logic.
-
-  SDValue VectElmts[4], VectElmtChains[4];
-  for (unsigned i = 0; i < 4; ++i) {
-    SDValue Idx = DAG.getConstant(i, dl, BasePtr.getValueType());
-    Idx = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, Idx);
-
-    VectElmts[i] = DAG.getExtLoad(
-        ISD::EXTLOAD, dl, MVT::i32, LoadChain, Idx,
-        LN->getPointerInfo().getWithOffset(i), MVT::i8,
-        /* Alignment = */ 1, LN->getMemOperand()->getFlags(), LN->getAAInfo());
-    VectElmtChains[i] = VectElmts[i].getValue(1);
-  }
-
-  LoadChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, VectElmtChains);
-  SDValue Value = DAG.getBuildVector(MVT::v4i1, dl, VectElmts);
-
-  SDValue RVals[] = { Value, LoadChain };
-  return DAG.getMergeValues(RVals, dl);
-}
-
-/// Lowering for QPX v4i1 stores
-SDValue PPCTargetLowering::LowerVectorStore(SDValue Op,
-                                            SelectionDAG &DAG) const {
-  SDLoc dl(Op);
-  StoreSDNode *SN = cast<StoreSDNode>(Op.getNode());
-  SDValue StoreChain = SN->getChain();
-  SDValue BasePtr = SN->getBasePtr();
-  SDValue Value = SN->getValue();
-
-  if (Value.getValueType() == MVT::v4f64 ||
-      Value.getValueType() == MVT::v4f32) {
-    EVT MemVT = SN->getMemoryVT();
-    unsigned Alignment = SN->getAlignment();
-
-    // If this store is properly aligned, then it is legal.
-    if (Alignment >= MemVT.getStoreSize())
-      return Op;
-
-    EVT ScalarVT = Value.getValueType().getScalarType(),
-        ScalarMemVT = MemVT.getScalarType();
-    unsigned Stride = ScalarMemVT.getStoreSize();
-
-    SDValue Stores[4];
-    for (unsigned Idx = 0; Idx < 4; ++Idx) {
-      SDValue Ex = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ScalarVT, Value,
-                               DAG.getVectorIdxConstant(Idx, dl));
-      SDValue Store;
-      if (ScalarVT != ScalarMemVT)
-        Store =
-            DAG.getTruncStore(StoreChain, dl, Ex, BasePtr,
-                              SN->getPointerInfo().getWithOffset(Idx * Stride),
-                              ScalarMemVT, MinAlign(Alignment, Idx * Stride),
-                              SN->getMemOperand()->getFlags(), SN->getAAInfo());
-      else
-        Store = DAG.getStore(StoreChain, dl, Ex, BasePtr,
-                             SN->getPointerInfo().getWithOffset(Idx * Stride),
-                             MinAlign(Alignment, Idx * Stride),
-                             SN->getMemOperand()->getFlags(), SN->getAAInfo());
-
-      if (Idx == 0 && SN->isIndexed()) {
-        assert(SN->getAddressingMode() == ISD::PRE_INC &&
-               "Unknown addressing mode on vector store");
-        Store = DAG.getIndexedStore(Store, dl, BasePtr, SN->getOffset(),
-                                    SN->getAddressingMode());
-      }
-
-      BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr,
-                            DAG.getConstant(Stride, dl,
-                                            BasePtr.getValueType()));
-      Stores[Idx] = Store;
-    }
-
-    SDValue TF =  DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
-
-    if (SN->isIndexed()) {
-      SDValue RetOps[] = { TF, Stores[0].getValue(1) };
-      return DAG.getMergeValues(RetOps, dl);
-    }
-
-    return TF;
-  }
-
-  assert(SN->isUnindexed() && "Indexed v4i1 stores are not supported");
-  assert(Value.getValueType() == MVT::v4i1 && "Unknown store to lower");
-
-  // The values are now known to be -1 (false) or 1 (true). To convert this
-  // into 0 (false) and 1 (true), add 1 and then divide by 2 (multiply by 0.5).
-  // This can be done with an fma and the 0.5 constant: (V+1.0)*0.5 = 0.5*V+0.5
-  Value = DAG.getNode(PPCISD::QBFLT, dl, MVT::v4f64, Value);
-
-  // FIXME: We can make this an f32 vector, but the BUILD_VECTOR code needs to
-  // understand how to form the extending load.
-  SDValue FPHalfs = DAG.getConstantFP(0.5, dl, MVT::v4f64);
-
-  Value = DAG.getNode(ISD::FMA, dl, MVT::v4f64, Value, FPHalfs, FPHalfs);
-
-  // Now convert to an integer and store.
-  Value = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f64,
-    DAG.getConstant(Intrinsic::ppc_qpx_qvfctiwu, dl, MVT::i32),
-    Value);
-
-  MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
-  int FrameIdx = MFI.CreateStackObject(16, Align(16), false);
-  MachinePointerInfo PtrInfo =
-      MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx);
-  EVT PtrVT = getPointerTy(DAG.getDataLayout());
-  SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT);
-
-  SDValue Ops[] = {StoreChain,
-                   DAG.getConstant(Intrinsic::ppc_qpx_qvstfiw, dl, MVT::i32),
-                   Value, FIdx};
-  SDVTList VTs = DAG.getVTList(/*chain*/ MVT::Other);
-
-  StoreChain = DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID,
-    dl, VTs, Ops, MVT::v4i32, PtrInfo);
-
-  // Move data into the byte array.
-  SDValue Loads[4], LoadChains[4];
-  for (unsigned i = 0; i < 4; ++i) {
-    unsigned Offset = 4*i;
-    SDValue Idx = DAG.getConstant(Offset, dl, FIdx.getValueType());
-    Idx = DAG.getNode(ISD::ADD, dl, FIdx.getValueType(), FIdx, Idx);
-
-    Loads[i] = DAG.getLoad(MVT::i32, dl, StoreChain, Idx,
-                           PtrInfo.getWithOffset(Offset));
-    LoadChains[i] = Loads[i].getValue(1);
-  }
-
-  StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains);
-
-  SDValue Stores[4];
-  for (unsigned i = 0; i < 4; ++i) {
-    SDValue Idx = DAG.getConstant(i, dl, BasePtr.getValueType());
-    Idx = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, Idx);
-
-    Stores[i] = DAG.getTruncStore(
-        StoreChain, dl, Loads[i], Idx, SN->getPointerInfo().getWithOffset(i),
-        MVT::i8, /* Alignment = */ 1, SN->getMemOperand()->getFlags(),
-        SN->getAAInfo());
-  }
-
-  StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
-
-  return StoreChain;
-}
-
 SDValue PPCTargetLowering::LowerMUL(SDValue Op, SelectionDAG &DAG) const {
   SDLoc dl(Op);
   if (Op.getValueType() == MVT::v4i32) {
@@ -11204,7 +10463,6 @@ SDValue PPCTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
   case ISD::VECTOR_SHUFFLE:     return LowerVECTOR_SHUFFLE(Op, DAG);
   case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG);
   case ISD::SCALAR_TO_VECTOR:   return LowerSCALAR_TO_VECTOR(Op, DAG);
-  case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG);
   case ISD::INSERT_VECTOR_ELT:  return LowerINSERT_VECTOR_ELT(Op, DAG);
   case ISD::MUL:                return LowerMUL(Op, DAG);
   case ISD::ABS:                return LowerABS(Op, DAG);
@@ -12148,9 +11406,6 @@ PPCTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
   } else if (MI.getOpcode() == PPC::SELECT_CC_F4 ||
              MI.getOpcode() == PPC::SELECT_CC_F8 ||
              MI.getOpcode() == PPC::SELECT_CC_F16 ||
-             MI.getOpcode() == PPC::SELECT_CC_QFRC ||
-             MI.getOpcode() == PPC::SELECT_CC_QSRC ||
-             MI.getOpcode() == PPC::SELECT_CC_QBRC ||
              MI.getOpcode() == PPC::SELECT_CC_VRRC ||
              MI.getOpcode() == PPC::SELECT_CC_VSFRC ||
              MI.getOpcode() == PPC::SELECT_CC_VSSRC ||
@@ -12160,9 +11415,6 @@ PPCTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
              MI.getOpcode() == PPC::SELECT_F4 ||
              MI.getOpcode() == PPC::SELECT_F8 ||
              MI.getOpcode() == PPC::SELECT_F16 ||
-             MI.getOpcode() == PPC::SELECT_QFRC ||
-             MI.getOpcode() == PPC::SELECT_QSRC ||
-             MI.getOpcode() == PPC::SELECT_QBRC ||
              MI.getOpcode() == PPC::SELECT_SPE ||
              MI.getOpcode() == PPC::SELECT_SPE4 ||
              MI.getOpcode() == PPC::SELECT_VRRC ||
@@ -12200,9 +11452,6 @@ PPCTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
         MI.getOpcode() == PPC::SELECT_F16 ||
         MI.getOpcode() == PPC::SELECT_SPE4 ||
         MI.getOpcode() == PPC::SELECT_SPE ||
-        MI.getOpcode() == PPC::SELECT_QFRC ||
-        MI.getOpcode() == PPC::SELECT_QSRC ||
-        MI.getOpcode() == PPC::SELECT_QBRC ||
         MI.getOpcode() == PPC::SELECT_VRRC ||
         MI.getOpcode() == PPC::SELECT_VSFRC ||
         MI.getOpcode() == PPC::SELECT_VSSRC ||
@@ -12895,9 +12144,7 @@ SDValue PPCTargetLowering::getSqrtEstimate(SDValue Operand, SelectionDAG &DAG,
   if ((VT == MVT::f32 && Subtarget.hasFRSQRTES()) ||
       (VT == MVT::f64 && Subtarget.hasFRSQRTE()) ||
       (VT == MVT::v4f32 && Subtarget.hasAltivec()) ||
-      (VT == MVT::v2f64 && Subtarget.hasVSX()) ||
-      (VT == MVT::v4f32 && Subtarget.hasQPX()) ||
-      (VT == MVT::v4f64 && Subtarget.hasQPX())) {
+      (VT == MVT::v2f64 && Subtarget.hasVSX())) {
     if (RefinementSteps == ReciprocalEstimate::Unspecified)
       RefinementSteps = getEstimateRefinementSteps(VT, Subtarget);
 
@@ -12916,9 +12163,7 @@ SDValue PPCTargetLowering::getRecipEstimate(SDValue Operand, SelectionDAG &DAG,
   if ((VT == MVT::f32 && Subtarget.hasFRES()) ||
       (VT == MVT::f64 && Subtarget.hasFRE()) ||
       (VT == MVT::v4f32 && Subtarget.hasAltivec()) ||
-      (VT == MVT::v2f64 && Subtarget.hasVSX()) ||
-      (VT == MVT::v4f32 && Subtarget.hasQPX()) ||
-      (VT == MVT::v4f64 && Subtarget.hasQPX())) {
+      (VT == MVT::v2f64 && Subtarget.hasVSX())) {
     if (RefinementSteps == ReciprocalEstimate::Unspecified)
       RefinementSteps = getEstimateRefinementSteps(VT, Subtarget);
     return DAG.getNode(PPCISD::FRE, SDLoc(Operand), VT, Operand);
@@ -13016,24 +12261,6 @@ static bool isConsecutiveLS(SDNode *N, LSBaseSDNode *Base,
     EVT VT;
     switch (cast<ConstantSDNode>(N->getOperand(1))->getZExtValue()) {
     default: return false;
-    case Intrinsic::ppc_qpx_qvlfd:
-    case Intrinsic::ppc_qpx_qvlfda:
-      VT = MVT::v4f64;
-      break;
-    case Intrinsic::ppc_qpx_qvlfs:
-    case Intrinsic::ppc_qpx_qvlfsa:
-      VT = MVT::v4f32;
-      break;
-    case Intrinsic::ppc_qpx_qvlfcd:
-    case Intrinsic::ppc_qpx_qvlfcda:
-      VT = MVT::v2f64;
-      break;
-    case Intrinsic::ppc_qpx_qvlfcs:
-    case Intrinsic::ppc_qpx_qvlfcsa:
-      VT = MVT::v2f32;
-      break;
-    case Intrinsic::ppc_qpx_qvlfiwa:
-    case Intrinsic::ppc_qpx_qvlfiwz:
     case Intrinsic::ppc_altivec_lvx:
     case Intrinsic::ppc_altivec_lvxl:
     case Intrinsic::ppc_vsx_lxvw4x:
@@ -13062,24 +12289,6 @@ static bool isConsecutiveLS(SDNode *N, LSBaseSDNode *Base,
     EVT VT;
     switch (cast<ConstantSDNode>(N->getOperand(1))->getZExtValue()) {
     default: return false;
-    case Intrinsic::ppc_qpx_qvstfd:
-    case Intrinsic::ppc_qpx_qvstfda:
-      VT = MVT::v4f64;
-      break;
-    case Intrinsic::ppc_qpx_qvstfs:
-    case Intrinsic::ppc_qpx_qvstfsa:
-      VT = MVT::v4f32;
-      break;
-    case Intrinsic::ppc_qpx_qvstfcd:
-    case Intrinsic::ppc_qpx_qvstfcda:
-      VT = MVT::v2f64;
-      break;
-    case Intrinsic::ppc_qpx_qvstfcs:
-    case Intrinsic::ppc_qpx_qvstfcsa:
-      VT = MVT::v2f32;
-      break;
-    case Intrinsic::ppc_qpx_qvstfiw:
-    case Intrinsic::ppc_qpx_qvstfiwa:
     case Intrinsic::ppc_altivec_stvx:
     case Intrinsic::ppc_altivec_stvxl:
     case Intrinsic::ppc_vsx_stxvw4x:
@@ -15077,18 +14286,14 @@ SDValue PPCTargetLowering::PerformDAGCombine(SDNode *N,
     EVT MemVT = LD->getMemoryVT();
     Type *Ty = MemVT.getTypeForEVT(*DAG.getContext());
     Align ABIAlignment = DAG.getDataLayout().getABITypeAlign(Ty);
-    Type *STy = MemVT.getScalarType().getTypeForEVT(*DAG.getContext());
-    Align ScalarABIAlignment = DAG.getDataLayout().getABITypeAlign(STy);
     if (LD->isUnindexed() && VT.isVector() &&
         ((Subtarget.hasAltivec() && ISD::isNON_EXTLoad(N) &&
           // P8 and later hardware should just use LOAD.
           !Subtarget.hasP8Vector() &&
           (VT == MVT::v16i8 || VT == MVT::v8i16 || VT == MVT::v4i32 ||
-           VT == MVT::v4f32)) ||
-         (Subtarget.hasQPX() && (VT == MVT::v4f64 || VT == MVT::v4f32) &&
-          LD->getAlign() >= ScalarABIAlignment)) &&
+           VT == MVT::v4f32))) &&
         LD->getAlign() < ABIAlignment) {
-      // This is a type-legal unaligned Altivec or QPX load.
+      // This is a type-legal unaligned Altivec load.
       SDValue Chain = LD->getChain();
       SDValue Ptr = LD->getBasePtr();
       bool isLittleEndian = Subtarget.isLittleEndian();
@@ -15119,24 +14324,13 @@ SDValue PPCTargetLowering::PerformDAGCombine(SDNode *N,
       // optimization later.
       Intrinsic::ID Intr, IntrLD, IntrPerm;
       MVT PermCntlTy, PermTy, LDTy;
-      if (Subtarget.hasAltivec()) {
-        Intr = isLittleEndian ?  Intrinsic::ppc_altivec_lvsr :
-                                 Intrinsic::ppc_altivec_lvsl;
-        IntrLD = Intrinsic::ppc_altivec_lvx;
-        IntrPerm = Intrinsic::ppc_altivec_vperm;
-        PermCntlTy = MVT::v16i8;
-        PermTy = MVT::v4i32;
-        LDTy = MVT::v4i32;
-      } else {
-        Intr =   MemVT == MVT::v4f64 ? Intrinsic::ppc_qpx_qvlpcld :
-                                       Intrinsic::ppc_qpx_qvlpcls;
-        IntrLD = MemVT == MVT::v4f64 ? Intrinsic::ppc_qpx_qvlfd :
-                                       Intrinsic::ppc_qpx_qvlfs;
-        IntrPerm = Intrinsic::ppc_qpx_qvfperm;
-        PermCntlTy = MVT::v4f64;
-        PermTy = MVT::v4f64;
-        LDTy = MemVT.getSimpleVT();
-      }
+      Intr = isLittleEndian ? Intrinsic::ppc_altivec_lvsr
+                            : Intrinsic::ppc_altivec_lvsl;
+      IntrLD = Intrinsic::ppc_altivec_lvx;
+      IntrPerm = Intrinsic::ppc_altivec_vperm;
+      PermCntlTy = MVT::v16i8;
+      PermTy = MVT::v4i32;
+      LDTy = MVT::v4i32;
 
       SDValue PermCntl = BuildIntrinsicOp(Intr, Ptr, DAG, dl, PermCntlTy);
 
@@ -15207,10 +14401,10 @@ SDValue PPCTargetLowering::PerformDAGCombine(SDNode *N,
                                 BaseLoad, ExtraLoad, PermCntl, DAG, dl);
 
       if (VT != PermTy)
-        Perm = Subtarget.hasAltivec() ?
-                 DAG.getNode(ISD::BITCAST, dl, VT, Perm) :
-                 DAG.getNode(ISD::FP_ROUND, dl, VT, Perm, // QPX
-                               DAG.getTargetConstant(1, dl, MVT::i64));
+        Perm = Subtarget.hasAltivec()
+                   ? DAG.getNode(ISD::BITCAST, dl, VT, Perm)
+                   : DAG.getNode(ISD::FP_ROUND, dl, VT, Perm,
+                                 DAG.getTargetConstant(1, dl, MVT::i64));
                                // second argument is 1 because this rounding
                                // is always exact.
 
@@ -15226,14 +14420,10 @@ SDValue PPCTargetLowering::PerformDAGCombine(SDNode *N,
       unsigned IID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
       Intrinsic::ID Intr = (isLittleEndian ? Intrinsic::ppc_altivec_lvsr
                                            : Intrinsic::ppc_altivec_lvsl);
-      if ((IID == Intr ||
-           IID == Intrinsic::ppc_qpx_qvlpcld  ||
-           IID == Intrinsic::ppc_qpx_qvlpcls) &&
-        N->getOperand(1)->getOpcode() == ISD::ADD) {
+      if (IID == Intr && N->getOperand(1)->getOpcode() == ISD::ADD) {
         SDValue Add = N->getOperand(1);
 
-        int Bits = IID == Intrinsic::ppc_qpx_qvlpcld ?
-                   5 /* 32 byte alignment */ : 4 /* 16 byte alignment */;
+        int Bits = 4 /* 16 byte alignment */;
 
         if (DAG.MaskedValueIsZero(Add->getOperand(1),
                                   APInt::getAllOnesValue(Bits /* alignment */)
@@ -15243,7 +14433,8 @@ SDValue PPCTargetLowering::PerformDAGCombine(SDNode *N,
                                     UE = BasePtr->use_end();
                UI != UE; ++UI) {
             if (UI->getOpcode() == ISD::INTRINSIC_WO_CHAIN &&
-                cast<ConstantSDNode>(UI->getOperand(0))->getZExtValue() == IID) {
+                cast<ConstantSDNode>(UI->getOperand(0))->getZExtValue() ==
+                    IID) {
               // We've found another LVSL/LVSR, and this address is an aligned
               // multiple of that one. The results will be the same, so use the
               // one we've just found instead.
@@ -15792,17 +14983,9 @@ PPCTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
           return std::make_pair(0U, &PPC::F4RCRegClass);
         if (VT == MVT::f64 || VT == MVT::i64)
           return std::make_pair(0U, &PPC::F8RCRegClass);
-        if (VT == MVT::v4f64 && Subtarget.hasQPX())
-          return std::make_pair(0U, &PPC::QFRCRegClass);
-        if (VT == MVT::v4f32 && Subtarget.hasQPX())
-          return std::make_pair(0U, &PPC::QSRCRegClass);
       }
       break;
     case 'v':
-      if (VT == MVT::v4f64 && Subtarget.hasQPX())
-        return std::make_pair(0U, &PPC::QFRCRegClass);
-      if (VT == MVT::v4f32 && Subtarget.hasQPX())
-        return std::make_pair(0U, &PPC::QSRCRegClass);
       if (Subtarget.hasAltivec())
         return std::make_pair(0U, &PPC::VRRCRegClass);
       break;
@@ -16094,12 +15277,6 @@ bool PPCTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
                                            MachineFunction &MF,
                                            unsigned Intrinsic) const {
   switch (Intrinsic) {
-  case Intrinsic::ppc_qpx_qvlfd:
-  case Intrinsic::ppc_qpx_qvlfs:
-  case Intrinsic::ppc_qpx_qvlfcd:
-  case Intrinsic::ppc_qpx_qvlfcs:
-  case Intrinsic::ppc_qpx_qvlfiwa:
-  case Intrinsic::ppc_qpx_qvlfiwz:
   case Intrinsic::ppc_altivec_lvx:
   case Intrinsic::ppc_altivec_lvxl:
   case Intrinsic::ppc_altivec_lvebx:
@@ -16121,18 +15298,6 @@ bool PPCTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
     case Intrinsic::ppc_vsx_lxvd2x:
       VT = MVT::v2f64;
       break;
-    case Intrinsic::ppc_qpx_qvlfd:
-      VT = MVT::v4f64;
-      break;
-    case Intrinsic::ppc_qpx_qvlfs:
-      VT = MVT::v4f32;
-      break;
-    case Intrinsic::ppc_qpx_qvlfcd:
-      VT = MVT::v2f64;
-      break;
-    case Intrinsic::ppc_qpx_qvlfcs:
-      VT = MVT::v2f32;
-      break;
     default:
       VT = MVT::v4i32;
       break;
@@ -16147,45 +15312,6 @@ bool PPCTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
     Info.flags = MachineMemOperand::MOLoad;
     return true;
   }
-  case Intrinsic::ppc_qpx_qvlfda:
-  case Intrinsic::ppc_qpx_qvlfsa:
-  case Intrinsic::ppc_qpx_qvlfcda:
-  case Intrinsic::ppc_qpx_qvlfcsa:
-  case Intrinsic::ppc_qpx_qvlfiwaa:
-  case Intrinsic::ppc_qpx_qvlfiwza: {
-    EVT VT;
-    switch (Intrinsic) {
-    case Intrinsic::ppc_qpx_qvlfda:
-      VT = MVT::v4f64;
-      break;
-    case Intrinsic::ppc_qpx_qvlfsa:
-      VT = MVT::v4f32;
-      break;
-    case Intrinsic::ppc_qpx_qvlfcda:
-      VT = MVT::v2f64;
-      break;
-    case Intrinsic::ppc_qpx_qvlfcsa:
-      VT = MVT::v2f32;
-      break;
-    default:
-      VT = MVT::v4i32;
-      break;
-    }
-
-    Info.opc = ISD::INTRINSIC_W_CHAIN;
-    Info.memVT = VT;
-    Info.ptrVal = I.getArgOperand(0);
-    Info.offset = 0;
-    Info.size = VT.getStoreSize();
-    Info.align = Align(1);
-    Info.flags = MachineMemOperand::MOLoad;
-    return true;
-  }
-  case Intrinsic::ppc_qpx_qvstfd:
-  case Intrinsic::ppc_qpx_qvstfs:
-  case Intrinsic::ppc_qpx_qvstfcd:
-  case Intrinsic::ppc_qpx_qvstfcs:
-  case Intrinsic::ppc_qpx_qvstfiw:
   case Intrinsic::ppc_altivec_stvx:
   case Intrinsic::ppc_altivec_stvxl:
   case Intrinsic::ppc_altivec_stvebx:
@@ -16207,18 +15333,6 @@ bool PPCTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
     case Intrinsic::ppc_vsx_stxvd2x:
       VT = MVT::v2f64;
       break;
-    case Intrinsic::ppc_qpx_qvstfd:
-      VT = MVT::v4f64;
-      break;
-    case Intrinsic::ppc_qpx_qvstfs:
-      VT = MVT::v4f32;
-      break;
-    case Intrinsic::ppc_qpx_qvstfcd:
-      VT = MVT::v2f64;
-      break;
-    case Intrinsic::ppc_qpx_qvstfcs:
-      VT = MVT::v2f32;
-      break;
     default:
       VT = MVT::v4i32;
       break;
@@ -16233,39 +15347,6 @@ bool PPCTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
     Info.flags = MachineMemOperand::MOStore;
     return true;
   }
-  case Intrinsic::ppc_qpx_qvstfda:
-  case Intrinsic::ppc_qpx_qvstfsa:
-  case Intrinsic::ppc_qpx_qvstfcda:
-  case Intrinsic::ppc_qpx_qvstfcsa:
-  case Intrinsic::ppc_qpx_qvstfiwa: {
-    EVT VT;
-    switch (Intrinsic) {
-    case Intrinsic::ppc_qpx_qvstfda:
-      VT = MVT::v4f64;
-      break;
-    case Intrinsic::ppc_qpx_qvstfsa:
-      VT = MVT::v4f32;
-      break;
-    case Intrinsic::ppc_qpx_qvstfcda:
-      VT = MVT::v2f64;
-      break;
-    case Intrinsic::ppc_qpx_qvstfcsa:
-      VT = MVT::v2f32;
-      break;
-    default:
-      VT = MVT::v4i32;
-      break;
-    }
-
-    Info.opc = ISD::INTRINSIC_VOID;
-    Info.memVT = VT;
-    Info.ptrVal = I.getArgOperand(1);
-    Info.offset = 0;
-    Info.size = VT.getStoreSize();
-    Info.align = Align(1);
-    Info.flags = MachineMemOperand::MOStore;
-    return true;
-  }
   default:
     break;
   }
@@ -16278,14 +15359,6 @@ bool PPCTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
 EVT PPCTargetLowering::getOptimalMemOpType(
     const MemOp &Op, const AttributeList &FuncAttributes) const {
   if (getTargetMachine().getOptLevel() != CodeGenOpt::None) {
-    // When expanding a memset, require at least two QPX instructions to cover
-    // the cost of loading the value to be stored from the constant pool.
-    if (Subtarget.hasQPX() && Op.size() >= 32 &&
-        (Op.isMemcpy() || Op.size() >= 64) && Op.isAligned(Align(32)) &&
-        !FuncAttributes.hasFnAttribute(Attribute::NoImplicitFloat)) {
-      return MVT::v4f64;
-    }
-
     // We should use Altivec/VSX loads and stores when available. For unaligned
     // addresses, unaligned VSX loads are only fast starting with the P8.
     if (Subtarget.hasAltivec() && Op.size() >= 16 &&
@@ -16504,7 +15577,7 @@ PPCTargetLowering::shouldExpandBuildVectorWithShuffles(
   if (VT == MVT::v2i64)
     return Subtarget.hasDirectMove(); // Don't need stack ops with direct moves
 
-  if (Subtarget.hasVSX() || Subtarget.hasQPX())
+  if (Subtarget.hasVSX())
     return true;
 
   return TargetLowering::shouldExpandBuildVectorWithShuffles(VT, DefinedValues);
@@ -16550,8 +15623,7 @@ SDValue PPCTargetLowering::getNegatedExpression(SDValue Op, SelectionDAG &DAG,
 
   switch (Opc) {
   case PPCISD::FNMSUB:
-    // TODO: QPX subtarget is deprecated. No transformation here.
-    if (!Op.hasOneUse() || !isTypeLegal(VT) || Subtarget.hasQPX())
+    if (!Op.hasOneUse() || !isTypeLegal(VT))
       break;
 
     const TargetOptions &Options = getTargetMachine().Options;
@@ -17032,8 +16104,7 @@ SDValue PPCTargetLowering::combineFMALike(SDNode *N,
   bool LegalOps = !DCI.isBeforeLegalizeOps();
   SDLoc Loc(N);
 
-  // TODO: QPX subtarget is deprecated. No transformation here.
-  if (Subtarget.hasQPX() || !isOperationLegal(ISD::FMA, VT))
+  if (!isOperationLegal(ISD::FMA, VT))
     return SDValue();
 
   // Allowing transformation to FNMSUB may change sign of zeroes when ab-c=0
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.h b/llvm/lib/Target/PowerPC/PPCISelLowering.h
index 768eaa43e0135..8cc42226d7f0b 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.h
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.h
@@ -427,22 +427,6 @@ namespace llvm {
     ///               => VABSDUW((XVNEGSP a), (XVNEGSP b))
     VABSD,
 
-    /// QVFPERM = This corresponds to the QPX qvfperm instruction.
-    QVFPERM,
-
-    /// QVGPCI = This corresponds to the QPX qvgpci instruction.
-    QVGPCI,
-
-    /// QVALIGNI = This corresponds to the QPX qvaligni instruction.
-    QVALIGNI,
-
-    /// QVESPLATI = This corresponds to the QPX qvesplati instruction.
-    QVESPLATI,
-
-    /// QBFLT = Access the underlying QPX floating-point boolean
-    /// representation.
-    QBFLT,
-
     /// FP_EXTEND_HALF(VECTOR, IDX) - Custom extend upper (IDX=0) half or
     /// lower (IDX=1) half of v4f32 to v2f64.
     FP_EXTEND_HALF,
@@ -519,10 +503,6 @@ namespace llvm {
     /// Store scalar integers from VSR.
     ST_VSR_SCAL_INT,
 
-    /// QBRC, CHAIN = QVLFSb CHAIN, Ptr
-    /// The 4xf32 load used for v4i1 constants.
-    QVLFSb,
-
     /// ATOMIC_CMP_SWAP - the exact same as the target-independent nodes
     /// except they ensure that the compare input is zero-extended for
     /// sub-word versions because the atomic loads zero-extend.
diff --git a/llvm/lib/Target/PowerPC/PPCInstrFormats.td b/llvm/lib/Target/PowerPC/PPCInstrFormats.td
index 632d4d9deb8a2..5ff5fc78326ba 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrFormats.td
+++ b/llvm/lib/Target/PowerPC/PPCInstrFormats.td
@@ -642,7 +642,6 @@ class XForm_17a<bits<6> opcode, bits<10> xo, dag OOL, dag IOL, string asmstr,
   let FRA = 0;
 }
 
-// Used for QPX
 class XForm_18<bits<6> opcode, bits<10> xo, dag OOL, dag IOL, string asmstr,
                InstrItinClass itin, list<dag> pattern>
          : I<opcode, OOL, IOL, asmstr, itin> {
@@ -1781,14 +1780,6 @@ class AForm_4<bits<6> opcode, bits<5> xo, dag OOL, dag IOL, string asmstr,
   let Inst{31}    = 0;
 }
 
-// Used for QPX
-class AForm_4a<bits<6> opcode, bits<5> xo, dag OOL, dag IOL, string asmstr,
-              InstrItinClass itin, list<dag> pattern>
-  : AForm_1<opcode, xo, OOL, IOL, asmstr, itin, pattern> {
-  let FRA = 0;
-  let FRC = 0;
-}
-
 // 1.7.13 M-Form
 class MForm_1<bits<6> opcode, dag OOL, dag IOL, string asmstr,
               InstrItinClass itin, list<dag> pattern>
@@ -2099,49 +2090,6 @@ class VX_RD5_RSp5_PS1_XO9<bits<9> xo, dag OOL, dag IOL, string asmstr,
   let Inst{23-31} = xo;
 }
 
-// Z23-Form (used by QPX)
-class Z23Form_1<bits<6> opcode, bits<8> xo, dag OOL, dag IOL, string asmstr, 
-              InstrItinClass itin, list<dag> pattern>
-         : I<opcode, OOL, IOL, asmstr, itin> {
-  bits<5> FRT;
-  bits<5> FRA;
-  bits<5> FRB;
-  bits<2> idx;
-
-  let Pattern = pattern;
-
-  bit RC = 0;    // set by isRecordForm
-
-  let Inst{6-10}  = FRT;
-  let Inst{11-15} = FRA;
-  let Inst{16-20} = FRB;
-  let Inst{21-22} = idx;
-  let Inst{23-30} = xo;
-  let Inst{31}    = RC;
-}
-
-class Z23Form_2<bits<6> opcode, bits<8> xo, dag OOL, dag IOL, string asmstr,
-              InstrItinClass itin, list<dag> pattern>
-  : Z23Form_1<opcode, xo, OOL, IOL, asmstr, itin, pattern> {
-  let FRB = 0;
-}
-
-class Z23Form_3<bits<6> opcode, bits<8> xo, dag OOL, dag IOL, string asmstr, 
-              InstrItinClass itin, list<dag> pattern>
-         : I<opcode, OOL, IOL, asmstr, itin> {
-  bits<5> FRT;
-  bits<12> idx;
-
-  let Pattern = pattern;
-
-  bit RC = 0;    // set by isRecordForm
-
-  let Inst{6-10}  = FRT;
-  let Inst{11-22} = idx;
-  let Inst{23-30} = xo;
-  let Inst{31}    = RC;
-}
-
 class Z23Form_8<bits<6> opcode, bits<8> xo, dag OOL, dag IOL, string asmstr,
               InstrItinClass itin, list<dag> pattern>
          : I<opcode, OOL, IOL, asmstr, itin> {
diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp b/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
index 99e25bb130ce4..34618ed058050 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
+++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
@@ -259,14 +259,6 @@ bool PPCInstrInfo::isAssociativeAndCommutative(const MachineInstr &Inst) const {
   case PPC::XVMULDP:
   case PPC::XVMULSP:
   case PPC::XSMULSP:
-  // QPX Add:
-  case PPC::QVFADD:
-  case PPC::QVFADDS:
-  case PPC::QVFADDSs:
-  // QPX Multiply:
-  case PPC::QVFMUL:
-  case PPC::QVFMULS:
-  case PPC::QVFMULSs:
     return Inst.getFlag(MachineInstr::MIFlag::FmReassoc) &&
            Inst.getFlag(MachineInstr::MIFlag::FmNsz);
   // Fixed point:
@@ -300,9 +292,7 @@ static const uint16_t FMAOpIdxInfo[][5] = {
     {PPC::XVMADDADP, PPC::XVADDDP, PPC::XVMULDP, 1, 2},
     {PPC::XVMADDASP, PPC::XVADDSP, PPC::XVMULSP, 1, 2},
     {PPC::FMADD, PPC::FADD, PPC::FMUL, 3, 1},
-    {PPC::FMADDS, PPC::FADDS, PPC::FMULS, 3, 1},
-    {PPC::QVFMADDSs, PPC::QVFADDSs, PPC::QVFMULSs, 3, 1},
-    {PPC::QVFMADD, PPC::QVFADD, PPC::QVFMUL, 3, 1}};
+    {PPC::FMADDS, PPC::FADDS, PPC::FMULS, 3, 1}};
 
 // Check if an opcode is a FMA instruction. If it is, return the index in array
 // FMAOpIdxInfo. Otherwise, return -1.
@@ -666,7 +656,6 @@ bool PPCInstrInfo::isReallyTriviallyReMaterializable(const MachineInstr &MI,
   case PPC::LI8:
   case PPC::LIS:
   case PPC::LIS8:
-  case PPC::QVGPCI:
   case PPC::ADDIStocHA:
   case PPC::ADDIStocHA8:
   case PPC::ADDItocL:
@@ -1343,12 +1332,6 @@ void PPCInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
   else if (PPC::VSFRCRegClass.contains(DestReg, SrcReg) ||
            PPC::VSSRCRegClass.contains(DestReg, SrcReg))
     Opc = (Subtarget.hasP9Vector()) ? PPC::XSCPSGNDP : PPC::XXLORf;
-  else if (PPC::QFRCRegClass.contains(DestReg, SrcReg))
-    Opc = PPC::QVFMR;
-  else if (PPC::QSRCRegClass.contains(DestReg, SrcReg))
-    Opc = PPC::QVFMRs;
-  else if (PPC::QBRCRegClass.contains(DestReg, SrcReg))
-    Opc = PPC::QVFMRb;
   else if (PPC::CRBITRCRegClass.contains(DestReg, SrcReg))
     Opc = PPC::CROR;
   else if (PPC::SPERCRegClass.contains(DestReg, SrcReg))
@@ -1393,12 +1376,6 @@ static unsigned getSpillIndex(const TargetRegisterClass *RC) {
     OpcodeIndex = SOK_VectorFloat4Spill;
   } else if (PPC::VRSAVERCRegClass.hasSubClassEq(RC)) {
     OpcodeIndex = SOK_VRSaveSpill;
-  } else if (PPC::QFRCRegClass.hasSubClassEq(RC)) {
-    OpcodeIndex = SOK_QuadFloat8Spill;
-  } else if (PPC::QSRCRegClass.hasSubClassEq(RC)) {
-    OpcodeIndex = SOK_QuadFloat4Spill;
-  } else if (PPC::QBRCRegClass.hasSubClassEq(RC)) {
-    OpcodeIndex = SOK_QuadBitSpill;
   } else if (PPC::SPILLTOVSRRCRegClass.hasSubClassEq(RC)) {
     OpcodeIndex = SOK_SpillToVSR;
   } else {
diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.h b/llvm/lib/Target/PowerPC/PPCInstrInfo.h
index 43973c627fcf1..bdcfa76505daf 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrInfo.h
+++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.h
@@ -123,9 +123,6 @@ enum SpillOpcodeKey {
   SOK_VectorFloat8Spill,
   SOK_VectorFloat4Spill,
   SOK_VRSaveSpill,
-  SOK_QuadFloat8Spill,
-  SOK_QuadFloat4Spill,
-  SOK_QuadBitSpill,
   SOK_SpillToVSR,
   SOK_SPESpill,
   SOK_LastOpcodeSpill // This must be last on the enum.
@@ -136,32 +133,28 @@ enum SpillOpcodeKey {
   {                                                                            \
     PPC::LWZ, PPC::LD, PPC::LFD, PPC::LFS, PPC::RESTORE_CR,                    \
         PPC::RESTORE_CRBIT, PPC::LVX, PPC::LXVD2X, PPC::LXSDX, PPC::LXSSPX,    \
-        PPC::RESTORE_VRSAVE, PPC::QVLFDX, PPC::QVLFSXs, PPC::QVLFDXb,          \
-        PPC::SPILLTOVSR_LD, PPC::EVLDD                                         \
+        PPC::RESTORE_VRSAVE, PPC::SPILLTOVSR_LD, PPC::EVLDD                    \
   }
 
 #define Pwr9LoadOpcodes                                                        \
   {                                                                            \
     PPC::LWZ, PPC::LD, PPC::LFD, PPC::LFS, PPC::RESTORE_CR,                    \
         PPC::RESTORE_CRBIT, PPC::LVX, PPC::LXV, PPC::DFLOADf64,                \
-        PPC::DFLOADf32, PPC::RESTORE_VRSAVE, PPC::QVLFDX, PPC::QVLFSXs,        \
-        PPC::QVLFDXb, PPC::SPILLTOVSR_LD                                       \
+        PPC::DFLOADf32, PPC::RESTORE_VRSAVE, PPC::SPILLTOVSR_LD                \
   }
 
 #define Pwr8StoreOpcodes                                                       \
   {                                                                            \
     PPC::STW, PPC::STD, PPC::STFD, PPC::STFS, PPC::SPILL_CR, PPC::SPILL_CRBIT, \
         PPC::STVX, PPC::STXVD2X, PPC::STXSDX, PPC::STXSSPX, PPC::SPILL_VRSAVE, \
-        PPC::QVSTFDX, PPC::QVSTFSXs, PPC::QVSTFDXb, PPC::SPILLTOVSR_ST,        \
-        PPC::EVSTDD                                                            \
+        PPC::SPILLTOVSR_ST, PPC::EVSTDD                                        \
   }
 
 #define Pwr9StoreOpcodes                                                       \
   {                                                                            \
     PPC::STW, PPC::STD, PPC::STFD, PPC::STFS, PPC::SPILL_CR, PPC::SPILL_CRBIT, \
         PPC::STVX, PPC::STXV, PPC::DFSTOREf64, PPC::DFSTOREf32,                \
-        PPC::SPILL_VRSAVE, PPC::QVSTFDX, PPC::QVSTFSXs, PPC::QVSTFDXb,         \
-        PPC::SPILLTOVSR_ST                                                     \
+        PPC::SPILL_VRSAVE, PPC::SPILLTOVSR_ST                                  \
   }
 
 // Initialize arrays for load and store spill opcodes on supported subtargets.
@@ -273,10 +266,10 @@ class PPCInstrInfo : public PPCGenInstrInfo {
   }
 
   static bool isSameClassPhysRegCopy(unsigned Opcode) {
-    unsigned CopyOpcodes[] =
-      { PPC::OR, PPC::OR8, PPC::FMR, PPC::VOR, PPC::XXLOR, PPC::XXLORf,
-        PPC::XSCPSGNDP, PPC::MCRF, PPC::QVFMR, PPC::QVFMRs, PPC::QVFMRb,
-        PPC::CROR, PPC::EVOR, -1U };
+    unsigned CopyOpcodes[] = {PPC::OR,        PPC::OR8,   PPC::FMR,
+                              PPC::VOR,       PPC::XXLOR, PPC::XXLORf,
+                              PPC::XSCPSGNDP, PPC::MCRF,  PPC::CROR,
+                              PPC::EVOR,      -1U};
     for (int i = 0; CopyOpcodes[i] != -1U; i++)
       if (Opcode == CopyOpcodes[i])
         return true;
diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.td b/llvm/lib/Target/PowerPC/PPCInstrInfo.td
index f807d61c75d23..264002c229b9e 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrInfo.td
+++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.td
@@ -203,16 +203,6 @@ def PPCvecinsert : SDNode<"PPCISD::VECINSERT", SDT_PPCVecInsert, []>;
 def PPCxxpermdi  : SDNode<"PPCISD::XXPERMDI", SDT_PPCxxpermdi, []>;
 def PPCvecshl    : SDNode<"PPCISD::VECSHL", SDT_PPCVecShift, []>;
 
-def PPCqvfperm   : SDNode<"PPCISD::QVFPERM", SDT_PPCqvfperm, []>;
-def PPCqvgpci    : SDNode<"PPCISD::QVGPCI", SDT_PPCqvgpci, []>;
-def PPCqvaligni  : SDNode<"PPCISD::QVALIGNI", SDT_PPCqvaligni, []>;
-def PPCqvesplati : SDNode<"PPCISD::QVESPLATI", SDT_PPCqvesplati, []>;
-
-def PPCqbflt     : SDNode<"PPCISD::QBFLT", SDT_PPCqbflt, []>;
-
-def PPCqvlfsb    : SDNode<"PPCISD::QVLFSb", SDT_PPCqvlfsb,
-                          [SDNPHasChain, SDNPMayLoad]>;
-
 def PPCcmpb     : SDNode<"PPCISD::CMPB", SDTIntBinOp, []>;
 
 // These nodes represent the 32-bit PPC shifts that operate on 6-bit shift
@@ -3467,7 +3457,6 @@ include "PPCInstrAltivec.td"
 include "PPCInstrSPE.td"
 include "PPCInstr64Bit.td"
 include "PPCInstrVSX.td"
-include "PPCInstrQPX.td"
 include "PPCInstrHTM.td"
 
 def crnot : OutPatFrag<(ops node:$in),
diff --git a/llvm/lib/Target/PowerPC/PPCInstrQPX.td b/llvm/lib/Target/PowerPC/PPCInstrQPX.td
deleted file mode 100644
index 2265af2815cb5..0000000000000
--- a/llvm/lib/Target/PowerPC/PPCInstrQPX.td
+++ /dev/null
@@ -1,1212 +0,0 @@
-//===- PPCInstrQPX.td - The PowerPC QPX Extension --*- tablegen -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file describes the QPX extension to the PowerPC instruction set.
-// Reference:
-// Book Q: QPX Architecture Definition. IBM (as updated in) 2011.
-//
-//===----------------------------------------------------------------------===//
-
-def PPCRegQFRCAsmOperand : AsmOperandClass {
-  let Name = "RegQFRC"; let PredicateMethod = "isRegNumber";
-}
-def qfrc : RegisterOperand<QFRC> {
-  let ParserMatchClass = PPCRegQFRCAsmOperand;
-}
-def PPCRegQSRCAsmOperand : AsmOperandClass {
-  let Name = "RegQSRC"; let PredicateMethod = "isRegNumber";
-}
-def qsrc : RegisterOperand<QSRC> {
-  let ParserMatchClass = PPCRegQSRCAsmOperand;
-}
-def PPCRegQBRCAsmOperand : AsmOperandClass {
-  let Name = "RegQBRC"; let PredicateMethod = "isRegNumber";
-}
-def qbrc : RegisterOperand<QBRC> {
-  let ParserMatchClass = PPCRegQBRCAsmOperand;
-}
-
-//===----------------------------------------------------------------------===//
-// Helpers for defining instructions that directly correspond to intrinsics.
-
-// QPXA1_Int - A AForm_1 intrinsic definition.
-class QPXA1_Int<bits<6> opcode, bits<5> xo, string opc, Intrinsic IntID>
-  : AForm_1<opcode, xo, (outs qfrc:$FRT), (ins qfrc:$FRA, qfrc:$FRB, qfrc:$FRC),
-              !strconcat(opc, " $FRT, $FRA, $FRC, $FRB"), IIC_FPFused,
-                       [(set v4f64:$FRT, (IntID v4f64:$FRA, v4f64:$FRB, v4f64:$FRC))]>;
-// QPXA1s_Int - A AForm_1 intrinsic definition (simple instructions).
-class QPXA1s_Int<bits<6> opcode, bits<5> xo, string opc, Intrinsic IntID>
-  : AForm_1<opcode, xo, (outs qfrc:$FRT), (ins qfrc:$FRA, qfrc:$FRB, qfrc:$FRC),
-              !strconcat(opc, " $FRT, $FRA, $FRC, $FRB"), IIC_VecPerm,
-                       [(set v4f64:$FRT, (IntID v4f64:$FRA, v4f64:$FRB, v4f64:$FRC))]>;
-// QPXA2_Int - A AForm_2 intrinsic definition.
-class QPXA2_Int<bits<6> opcode, bits<5> xo, string opc, Intrinsic IntID>
-  : AForm_2<opcode, xo, (outs qfrc:$FRT), (ins qfrc:$FRA, qfrc:$FRB),
-              !strconcat(opc, " $FRT, $FRA, $FRB"), IIC_FPGeneral,
-                       [(set v4f64:$FRT, (IntID v4f64:$FRA, v4f64:$FRB))]>;
-// QPXA3_Int - A AForm_3 intrinsic definition.
-class QPXA3_Int<bits<6> opcode, bits<5> xo, string opc, Intrinsic IntID>
-  : AForm_3<opcode, xo, (outs qfrc:$FRT), (ins qfrc:$FRA, qfrc:$FRC),
-              !strconcat(opc, " $FRT, $FRA, $FRC"), IIC_FPGeneral,
-                       [(set v4f64:$FRT, (IntID v4f64:$FRA, v4f64:$FRC))]>;
-// QPXA4_Int - A AForm_4a intrinsic definition.
-class QPXA4_Int<bits<6> opcode, bits<5> xo, string opc, Intrinsic IntID>
-  : AForm_4a<opcode, xo, (outs qfrc:$FRT), (ins qfrc:$FRB),
-              !strconcat(opc, " $FRT, $FRB"), IIC_FPGeneral,
-                       [(set v4f64:$FRT, (IntID v4f64:$FRB))]>;
-// QPXX18_Int - A XForm_18 intrinsic definition.
-class QPXX18_Int<bits<6> opcode, bits<10> xo, string opc, Intrinsic IntID>
-  : XForm_18<opcode, xo, (outs qfrc:$FRT), (ins qfrc:$FRA, qfrc:$FRB),
-              !strconcat(opc, " $FRT, $FRA, $FRB"), IIC_FPCompare,
-                       [(set v4f64:$FRT, (IntID v4f64:$FRA, v4f64:$FRB))]>;
-// QPXX19_Int - A XForm_19 intrinsic definition.
-class QPXX19_Int<bits<6> opcode, bits<10> xo, string opc, Intrinsic IntID>
-  : XForm_19<opcode, xo, (outs qfrc:$FRT), (ins qfrc:$FRB),
-              !strconcat(opc, " $FRT, $FRB"), IIC_FPGeneral,
-                       [(set v4f64:$FRT, (IntID v4f64:$FRB))]>;
-
-//===----------------------------------------------------------------------===//
-// Pattern Frags.
-
-def extloadv4f32 : PatFrag<(ops node:$ptr), (extload node:$ptr), [{
-  return cast<LoadSDNode>(N)->getMemoryVT() == MVT::v4f32;
-}]>;
-
-def truncstorev4f32 : PatFrag<(ops node:$val, node:$ptr),
-                            (truncstore node:$val, node:$ptr), [{
-  return cast<StoreSDNode>(N)->getMemoryVT() == MVT::v4f32;
-}]>;
-def pre_truncstv4f32 : PatFrag<(ops node:$val, node:$base, node:$offset),
-                               (pre_truncst node:$val,
-                                            node:$base, node:$offset), [{
-  return cast<StoreSDNode>(N)->getMemoryVT() == MVT::v4f32;
-}]>;
-
-def fround_inexact : PatFrag<(ops node:$val), (fpround node:$val), [{
-  return cast<ConstantSDNode>(N->getOperand(1))->getZExtValue() == 0;
-}]>;
-
-def fround_exact : PatFrag<(ops node:$val), (fpround node:$val), [{
-  return cast<ConstantSDNode>(N->getOperand(1))->getZExtValue() == 1;
-}]>;
-
-let FastIselShouldIgnore = 1 in // FastIsel should ignore all u12 instrs.
-  def u12 : ImmLeaf<i32, [{ return (Imm & 0xFFF) == Imm; }]>;
-
-//===----------------------------------------------------------------------===//
-// Instruction Definitions.
-
-def HasQPX : Predicate<"Subtarget->hasQPX()">;
-let Predicates = [HasQPX] in {
-let DecoderNamespace = "QPX" in {
-let hasSideEffects = 0 in { // QPX instructions don't have side effects.
-let Uses = [RM] in {
-  // Add Instructions
-  let isCommutable = 1 in {
-    def QVFADD : AForm_2<4, 21,
-                        (outs qfrc:$FRT), (ins qfrc:$FRA, qfrc:$FRB),
-                        "qvfadd $FRT, $FRA, $FRB", IIC_FPGeneral,
-                        [(set v4f64:$FRT, (fadd v4f64:$FRA, v4f64:$FRB))]>;
-    let isCodeGenOnly = 1 in
-      def QVFADDS : QPXA2_Int<0, 21, "qvfadds", int_ppc_qpx_qvfadds>;
-    def QVFADDSs : AForm_2<0, 21,
-                          (outs qsrc:$FRT), (ins qsrc:$FRA, qsrc:$FRB),
-                          "qvfadds $FRT, $FRA, $FRB", IIC_FPGeneral,
-                          [(set v4f32:$FRT, (fadd v4f32:$FRA, v4f32:$FRB))]>;
-  }
-  def QVFSUB : AForm_2<4, 20,
-                      (outs qfrc:$FRT), (ins qfrc:$FRA, qfrc:$FRB),
-                      "qvfsub $FRT, $FRA, $FRB", IIC_FPGeneral,
-                      [(set v4f64:$FRT, (fsub v4f64:$FRA, v4f64:$FRB))]>;
-  let isCodeGenOnly = 1 in
-    def QVFSUBS : QPXA2_Int<0, 20, "qvfsubs", int_ppc_qpx_qvfsubs>;
-  def QVFSUBSs : AForm_2<0, 20,
-                        (outs qsrc:$FRT), (ins qsrc:$FRA, qsrc:$FRB),
-                        "qvfsubs $FRT, $FRA, $FRB", IIC_FPGeneral,
-                        [(set v4f32:$FRT, (fsub v4f32:$FRA, v4f32:$FRB))]>;
-
-  // Estimate Instructions
-  def QVFRE : AForm_4a<4, 24, (outs qfrc:$FRT), (ins qfrc:$FRB),
-                       "qvfre $FRT, $FRB", IIC_FPGeneral,
-                       [(set v4f64:$FRT, (PPCfre v4f64:$FRB))]>;
-  def QVFRES : QPXA4_Int<0, 24, "qvfres", int_ppc_qpx_qvfres>;
-  let isCodeGenOnly = 1 in
-  def QVFRESs : AForm_4a<0, 24, (outs qsrc:$FRT), (ins qsrc:$FRB),
-                         "qvfres $FRT, $FRB", IIC_FPGeneral,
-                         [(set v4f32:$FRT, (PPCfre v4f32:$FRB))]>;
-
-  def QVFRSQRTE : AForm_4a<4, 26, (outs qfrc:$FRT), (ins qfrc:$FRB),
-                           "qvfrsqrte $FRT, $FRB", IIC_FPGeneral,
-                           [(set v4f64:$FRT, (PPCfrsqrte v4f64:$FRB))]>;
-  def QVFRSQRTES : QPXA4_Int<0, 26, "qvfrsqrtes", int_ppc_qpx_qvfrsqrtes>;
-  let isCodeGenOnly = 1 in
-  def QVFRSQRTESs : AForm_4a<0, 26, (outs qsrc:$FRT), (ins qsrc:$FRB),
-                             "qvfrsqrtes $FRT, $FRB", IIC_FPGeneral,
-                             [(set v4f32:$FRT, (PPCfrsqrte v4f32:$FRB))]>;
-
-  // Multiply Instructions
-  let isCommutable = 1 in {
-    def QVFMUL : AForm_3<4, 25,
-                        (outs qfrc:$FRT), (ins qfrc:$FRA, qfrc:$FRC),
-                        "qvfmul $FRT, $FRA, $FRC", IIC_FPGeneral,
-                        [(set v4f64:$FRT, (fmul v4f64:$FRA, v4f64:$FRC))]>;
-    let isCodeGenOnly = 1 in
-      def QVFMULS : QPXA3_Int<0, 25, "qvfmuls", int_ppc_qpx_qvfmuls>;
-    def QVFMULSs : AForm_3<0, 25,
-                          (outs qsrc:$FRT), (ins qsrc:$FRA, qsrc:$FRC),
-                          "qvfmuls $FRT, $FRA, $FRC", IIC_FPGeneral,
-                          [(set v4f32:$FRT, (fmul v4f32:$FRA, v4f32:$FRC))]>;
-  }
-  def QVFXMUL : QPXA3_Int<4, 17, "qvfxmul", int_ppc_qpx_qvfxmul>;
-  def QVFXMULS : QPXA3_Int<0, 17, "qvfxmuls", int_ppc_qpx_qvfxmuls>;
-
-  // Multiply-add instructions
-  def QVFMADD : AForm_1<4, 29,
-                      (outs qfrc:$FRT), (ins qfrc:$FRA, qfrc:$FRC, qfrc:$FRB),
-                      "qvfmadd $FRT, $FRA, $FRC, $FRB", IIC_FPFused,
-                      [(set v4f64:$FRT, (fma v4f64:$FRA, v4f64:$FRC, v4f64:$FRB))]>;
-  let isCodeGenOnly = 1 in
-    def QVFMADDS : QPXA1_Int<0, 29, "qvfmadds", int_ppc_qpx_qvfmadds>;
-  def QVFMADDSs : AForm_1<0, 29,
-                        (outs qsrc:$FRT), (ins qsrc:$FRA, qsrc:$FRC, qsrc:$FRB),
-                        "qvfmadds $FRT, $FRA, $FRC, $FRB", IIC_FPFused,
-                        [(set v4f32:$FRT, (fma v4f32:$FRA, v4f32:$FRC, v4f32:$FRB))]>;
-  def QVFNMADD : AForm_1<4, 31,
-                      (outs qfrc:$FRT), (ins qfrc:$FRA, qfrc:$FRC, qfrc:$FRB),
-                      "qvfnmadd $FRT, $FRA, $FRC, $FRB", IIC_FPFused,
-                      [(set v4f64:$FRT, (fneg (fma v4f64:$FRA, v4f64:$FRC,
-                                                   v4f64:$FRB)))]>;
-  let isCodeGenOnly = 1 in
-    def QVFNMADDS : QPXA1_Int<0, 31, "qvfnmadds", int_ppc_qpx_qvfnmadds>;
-  def QVFNMADDSs : AForm_1<0, 31,
-                        (outs qsrc:$FRT), (ins qsrc:$FRA, qsrc:$FRC, qsrc:$FRB),
-                        "qvfnmadds $FRT, $FRA, $FRC, $FRB", IIC_FPFused,
-                        [(set v4f32:$FRT, (fneg (fma v4f32:$FRA, v4f32:$FRC,
-                                                     v4f32:$FRB)))]>;
-  def QVFMSUB : AForm_1<4, 28,
-                      (outs qfrc:$FRT), (ins qfrc:$FRA, qfrc:$FRC, qfrc:$FRB),
-                      "qvfmsub $FRT, $FRA, $FRC, $FRB", IIC_FPFused,
-                      [(set v4f64:$FRT, (fma v4f64:$FRA, v4f64:$FRC,
-                                             (fneg v4f64:$FRB)))]>;
-  let isCodeGenOnly = 1 in
-    def QVFMSUBS : QPXA1_Int<0, 28, "qvfmsubs", int_ppc_qpx_qvfmsubs>;
-  def QVFMSUBSs : AForm_1<0, 28,
-                      (outs qsrc:$FRT), (ins qsrc:$FRA, qsrc:$FRC, qsrc:$FRB),
-                      "qvfmsubs $FRT, $FRA, $FRC, $FRB", IIC_FPFused,
-                      [(set v4f32:$FRT, (fma v4f32:$FRA, v4f32:$FRC,
-                                             (fneg v4f32:$FRB)))]>;
-  def QVFNMSUB : AForm_1<4, 30,
-                      (outs qfrc:$FRT), (ins qfrc:$FRA, qfrc:$FRC, qfrc:$FRB),
-                      "qvfnmsub $FRT, $FRA, $FRC, $FRB", IIC_FPFused,
-                      [(set v4f64:$FRT, (fneg (fma v4f64:$FRA, v4f64:$FRC,
-                                              (fneg v4f64:$FRB))))]>;
-  let isCodeGenOnly = 1 in
-    def QVFNMSUBS : QPXA1_Int<0, 30, "qvfnmsubs", int_ppc_qpx_qvfnmsubs>;
-  def QVFNMSUBSs : AForm_1<0, 30,
-                      (outs qsrc:$FRT), (ins qsrc:$FRA, qsrc:$FRC, qsrc:$FRB),
-                      "qvfnmsubs $FRT, $FRA, $FRC, $FRB", IIC_FPFused,
-                      [(set v4f32:$FRT, (fneg (fma v4f32:$FRA, v4f32:$FRC,
-                                              (fneg v4f32:$FRB))))]>;
-  def QVFXMADD : QPXA1_Int<4, 9, "qvfxmadd", int_ppc_qpx_qvfxmadd>;
-  def QVFXMADDS : QPXA1_Int<0, 9, "qvfxmadds", int_ppc_qpx_qvfxmadds>;
-  def QVFXXNPMADD : QPXA1_Int<4, 11, "qvfxxnpmadd", int_ppc_qpx_qvfxxnpmadd>;
-  def QVFXXNPMADDS : QPXA1_Int<0, 11, "qvfxxnpmadds", int_ppc_qpx_qvfxxnpmadds>;
-  def QVFXXCPNMADD : QPXA1_Int<4, 3, "qvfxxcpnmadd", int_ppc_qpx_qvfxxcpnmadd>;
-  def QVFXXCPNMADDS : QPXA1_Int<0, 3, "qvfxxcpnmadds", int_ppc_qpx_qvfxxcpnmadds>;
-  def QVFXXMADD : QPXA1_Int<4, 1, "qvfxxmadd", int_ppc_qpx_qvfxxmadd>;
-  def QVFXXMADDS : QPXA1_Int<0, 1, "qvfxxmadds", int_ppc_qpx_qvfxxmadds>;
-
-  // Select Instruction
-  let isCodeGenOnly = 1 in
-    def QVFSEL : QPXA1s_Int<4, 23, "qvfsel", int_ppc_qpx_qvfsel>;
-  def QVFSELb : AForm_1<4, 23, (outs qfrc:$FRT),
-                        (ins qbrc:$FRA, qfrc:$FRB, qfrc:$FRC),
-                        "qvfsel $FRT, $FRA, $FRC, $FRB", IIC_VecPerm,
-                        [(set v4f64:$FRT, (vselect v4i1:$FRA,
-                                                   v4f64:$FRC, v4f64:$FRB))]>;
-  let isCodeGenOnly = 1 in
-  def QVFSELbs : AForm_1<4, 23, (outs qsrc:$FRT),
-                        (ins qbrc:$FRA, qsrc:$FRB, qsrc:$FRC),
-                        "qvfsel $FRT, $FRA, $FRC, $FRB", IIC_VecPerm,
-                        [(set v4f32:$FRT, (vselect v4i1:$FRA,
-                                                   v4f32:$FRC, v4f32:$FRB))]>;
-  let isCodeGenOnly = 1 in
-  def QVFSELbb: AForm_1<4, 23, (outs qbrc:$FRT),
-                        (ins qbrc:$FRA, qbrc:$FRB, qbrc:$FRC),
-                        "qvfsel $FRT, $FRA, $FRC, $FRB", IIC_VecPerm,
-                        [(set v4i1:$FRT, (vselect v4i1:$FRA,
-                                                  v4i1:$FRC, v4i1:$FRB))]>;
-
-  // SELECT_CC_* - Used to implement the SELECT_CC DAG operation.  Expanded after
-  // instruction selection into a branch sequence.
-  def SELECT_CC_QFRC: PPCCustomInserterPseudo<(outs qfrc:$dst), (ins crrc:$cond, qfrc:$T, qfrc:$F,
-                              i32imm:$BROPC), "#SELECT_CC_QFRC",
-                              []>;
-  def SELECT_CC_QSRC: PPCCustomInserterPseudo<(outs qsrc:$dst), (ins crrc:$cond, qsrc:$T, qsrc:$F,
-                              i32imm:$BROPC), "#SELECT_CC_QSRC",
-                              []>;
-  def SELECT_CC_QBRC: PPCCustomInserterPseudo<(outs qbrc:$dst), (ins crrc:$cond, qbrc:$T, qbrc:$F,
-                              i32imm:$BROPC), "#SELECT_CC_QBRC",
-                              []>;
-
-  // SELECT_* pseudo instructions, like SELECT_CC_* but taking condition
-  // register bit directly.
-  def SELECT_QFRC: PPCCustomInserterPseudo<(outs qfrc:$dst), (ins crbitrc:$cond,
-                          qfrc:$T, qfrc:$F), "#SELECT_QFRC",
-                          [(set v4f64:$dst,
-                                (select i1:$cond, v4f64:$T, v4f64:$F))]>;
-  def SELECT_QSRC: PPCCustomInserterPseudo<(outs qsrc:$dst), (ins crbitrc:$cond,
-                          qsrc:$T, qsrc:$F), "#SELECT_QSRC",
-                          [(set v4f32:$dst,
-                                (select i1:$cond, v4f32:$T, v4f32:$F))]>;
-  def SELECT_QBRC: PPCCustomInserterPseudo<(outs qbrc:$dst), (ins crbitrc:$cond,
-                          qbrc:$T, qbrc:$F), "#SELECT_QBRC",
-                          [(set v4i1:$dst,
-                                (select i1:$cond, v4i1:$T, v4i1:$F))]>;
-
-  // Convert and Round Instructions
-  def QVFCTID : QPXX19_Int<4, 814, "qvfctid", int_ppc_qpx_qvfctid>;
-  let isCodeGenOnly = 1 in
-    def QVFCTIDb : XForm_19<4, 814, (outs qbrc:$FRT), (ins qbrc:$FRB),
-                            "qvfctid $FRT, $FRB", IIC_FPGeneral, []>;
-
-  def QVFCTIDU : QPXX19_Int<4, 942, "qvfctidu", int_ppc_qpx_qvfctidu>;
-  def QVFCTIDZ : QPXX19_Int<4, 815, "qvfctidz", int_ppc_qpx_qvfctidz>;
-  def QVFCTIDUZ : QPXX19_Int<4, 943, "qvfctiduz", int_ppc_qpx_qvfctiduz>;
-  def QVFCTIW : QPXX19_Int<4, 14, "qvfctiw", int_ppc_qpx_qvfctiw>;
-  def QVFCTIWU : QPXX19_Int<4, 142, "qvfctiwu", int_ppc_qpx_qvfctiwu>;
-  def QVFCTIWZ : QPXX19_Int<4, 15, "qvfctiwz", int_ppc_qpx_qvfctiwz>;
-  def QVFCTIWUZ : QPXX19_Int<4, 143, "qvfctiwuz", int_ppc_qpx_qvfctiwuz>;
-  def QVFCFID : QPXX19_Int<4, 846, "qvfcfid", int_ppc_qpx_qvfcfid>;
-  let isCodeGenOnly = 1 in
-    def QVFCFIDb : XForm_19<4, 846, (outs qbrc:$FRT), (ins qbrc:$FRB),
-                            "qvfcfid $FRT, $FRB", IIC_FPGeneral, []>;
-
-  def QVFCFIDU : QPXX19_Int<4, 974, "qvfcfidu", int_ppc_qpx_qvfcfidu>;
-  def QVFCFIDS : QPXX19_Int<0, 846, "qvfcfids", int_ppc_qpx_qvfcfids>;
-  def QVFCFIDUS : QPXX19_Int<0, 974, "qvfcfidus", int_ppc_qpx_qvfcfidus>;
-
-  let isCodeGenOnly = 1 in
-    def QVFRSP : QPXX19_Int<4, 12, "qvfrsp", int_ppc_qpx_qvfrsp>;
-  def QVFRSPs : XForm_19<4, 12,
-                      (outs qsrc:$FRT), (ins qfrc:$FRB),
-                      "qvfrsp $FRT, $FRB", IIC_FPGeneral,
-                      [(set v4f32:$FRT, (fround_inexact v4f64:$FRB))]>;
-
-  def QVFRIZ : XForm_19<4, 424, (outs qfrc:$FRT), (ins qfrc:$FRB),
-                        "qvfriz $FRT, $FRB", IIC_FPGeneral,
-                        [(set v4f64:$FRT, (ftrunc v4f64:$FRB))]>;
-  let isCodeGenOnly = 1 in
-    def QVFRIZs : XForm_19<4, 424, (outs qsrc:$FRT), (ins qsrc:$FRB),
-                           "qvfriz $FRT, $FRB", IIC_FPGeneral,
-                           [(set v4f32:$FRT, (ftrunc v4f32:$FRB))]>;
-
-  def QVFRIN : XForm_19<4, 392, (outs qfrc:$FRT), (ins qfrc:$FRB),
-                        "qvfrin $FRT, $FRB", IIC_FPGeneral,
-                        [(set v4f64:$FRT, (fround v4f64:$FRB))]>;
-  let isCodeGenOnly = 1 in
-    def QVFRINs : XForm_19<4, 392, (outs qsrc:$FRT), (ins qsrc:$FRB),
-                           "qvfrin $FRT, $FRB", IIC_FPGeneral,
-                           [(set v4f32:$FRT, (fround v4f32:$FRB))]>;
-
-  def QVFRIP : XForm_19<4, 456, (outs qfrc:$FRT), (ins qfrc:$FRB),
-                        "qvfrip $FRT, $FRB", IIC_FPGeneral,
-                        [(set v4f64:$FRT, (fceil v4f64:$FRB))]>;
-  let isCodeGenOnly = 1 in
-    def QVFRIPs : XForm_19<4, 456, (outs qsrc:$FRT), (ins qsrc:$FRB),
-                           "qvfrip $FRT, $FRB", IIC_FPGeneral,
-                           [(set v4f32:$FRT, (fceil v4f32:$FRB))]>;
-
-  def QVFRIM : XForm_19<4, 488, (outs qfrc:$FRT), (ins qfrc:$FRB),
-                        "qvfrim $FRT, $FRB", IIC_FPGeneral,
-                        [(set v4f64:$FRT, (ffloor v4f64:$FRB))]>;
-  let isCodeGenOnly = 1 in
-    def QVFRIMs : XForm_19<4, 488, (outs qsrc:$FRT), (ins qsrc:$FRB),
-                           "qvfrim $FRT, $FRB", IIC_FPGeneral,
-                           [(set v4f32:$FRT, (ffloor v4f32:$FRB))]>;
-
-  // Move Instructions
-  def QVFMR : XForm_19<4, 72,
-                      (outs qfrc:$FRT), (ins qfrc:$FRB),
-                      "qvfmr $FRT, $FRB", IIC_VecPerm,
-                      [/* (set v4f64:$FRT, v4f64:$FRB) */]>;
-  let isCodeGenOnly = 1 in {
-    def QVFMRs : XForm_19<4, 72,
-                         (outs qsrc:$FRT), (ins qsrc:$FRB),
-                         "qvfmr $FRT, $FRB", IIC_VecPerm,
-                         [/* (set v4f32:$FRT, v4f32:$FRB) */]>;
-    def QVFMRb : XForm_19<4, 72,
-                         (outs qbrc:$FRT), (ins qbrc:$FRB),
-                         "qvfmr $FRT, $FRB", IIC_VecPerm,
-                         [/* (set v4i1:$FRT, v4i1:$FRB) */]>;
-  }
-  def QVFNEG : XForm_19<4, 40,
-                      (outs qfrc:$FRT), (ins qfrc:$FRB),
-                      "qvfneg $FRT, $FRB", IIC_VecPerm,
-                      [(set v4f64:$FRT, (fneg v4f64:$FRB))]>;
-  let isCodeGenOnly = 1 in
-    def QVFNEGs : XForm_19<4, 40,
-                         (outs qsrc:$FRT), (ins qsrc:$FRB),
-                         "qvfneg $FRT, $FRB", IIC_VecPerm,
-                         [(set v4f32:$FRT, (fneg v4f32:$FRB))]>;
-  def QVFABS : XForm_19<4, 264,
-                      (outs qfrc:$FRT), (ins qfrc:$FRB),
-                      "qvfabs $FRT, $FRB", IIC_VecPerm,
-                      [(set v4f64:$FRT, (fabs v4f64:$FRB))]>;
-  let isCodeGenOnly = 1 in
-    def QVFABSs : XForm_19<4, 264,
-                         (outs qsrc:$FRT), (ins qsrc:$FRB),
-                         "qvfabs $FRT, $FRB", IIC_VecPerm,
-                         [(set v4f32:$FRT, (fabs v4f32:$FRB))]>;
-  def QVFNABS : XForm_19<4, 136,
-                      (outs qfrc:$FRT), (ins qfrc:$FRB),
-                      "qvfnabs $FRT, $FRB", IIC_VecPerm,
-                      [(set v4f64:$FRT, (fneg (fabs v4f64:$FRB)))]>;
-  let isCodeGenOnly = 1 in
-    def QVFNABSs : XForm_19<4, 136,
-                         (outs qsrc:$FRT), (ins qsrc:$FRB),
-                         "qvfnabs $FRT, $FRB", IIC_VecPerm,
-                         [(set v4f32:$FRT, (fneg (fabs v4f32:$FRB)))]>;
-  def QVFCPSGN : XForm_18<4, 8,
-                      (outs qfrc:$FRT), (ins qfrc:$FRA, qfrc:$FRB),
-                      "qvfcpsgn $FRT, $FRA, $FRB", IIC_VecPerm,
-                      [(set v4f64:$FRT, (fcopysign v4f64:$FRB, v4f64:$FRA))]>;
-  let isCodeGenOnly = 1 in
-    def QVFCPSGNs : XForm_18<4, 8,
-                         (outs qsrc:$FRT), (ins qsrc:$FRA, qsrc:$FRB),
-                         "qvfcpsgn $FRT, $FRA, $FRB", IIC_VecPerm,
-                         [(set v4f32:$FRT, (fcopysign v4f32:$FRB, v4f32:$FRA))]>;
-
-  def QVALIGNI : Z23Form_1<4, 5,
-                      (outs qfrc:$FRT), (ins qfrc:$FRA, qfrc:$FRB, u2imm:$idx),
-                      "qvaligni $FRT, $FRA, $FRB, $idx", IIC_VecPerm,
-                      [(set v4f64:$FRT,
-                            (PPCqvaligni v4f64:$FRA, v4f64:$FRB,
-                                         (i32 imm:$idx)))]>;
-  let isCodeGenOnly = 1 in
-     def QVALIGNIs : Z23Form_1<4, 5,
-                         (outs qsrc:$FRT), (ins qsrc:$FRA, qsrc:$FRB, u2imm:$idx),
-                         "qvaligni $FRT, $FRA, $FRB, $idx", IIC_VecPerm,
-                         [(set v4f32:$FRT,
-                               (PPCqvaligni v4f32:$FRA, v4f32:$FRB,
-                                            (i32 imm:$idx)))]>;
-  let isCodeGenOnly = 1 in
-     def QVALIGNIb : Z23Form_1<4, 5,
-                         (outs qbrc:$FRT), (ins qbrc:$FRA, qbrc:$FRB, u2imm:$idx),
-                         "qvaligni $FRT, $FRA, $FRB, $idx", IIC_VecPerm,
-                         [(set v4i1:$FRT,
-                               (PPCqvaligni v4i1:$FRA, v4i1:$FRB,
-                                            (i32 imm:$idx)))]>;
-
-  def QVESPLATI : Z23Form_2<4, 37,
-                      (outs qfrc:$FRT), (ins qfrc:$FRA, u2imm:$idx),
-                      "qvesplati $FRT, $FRA, $idx", IIC_VecPerm,
-                      [(set v4f64:$FRT,
-                            (PPCqvesplati v4f64:$FRA, (i32 imm:$idx)))]>;
-  let isCodeGenOnly = 1 in
-     def QVESPLATIs : Z23Form_2<4, 37,
-                         (outs qsrc:$FRT), (ins qsrc:$FRA, u2imm:$idx),
-                         "qvesplati $FRT, $FRA, $idx", IIC_VecPerm,
-                         [(set v4f32:$FRT,
-                               (PPCqvesplati v4f32:$FRA, (i32 imm:$idx)))]>;
-  let isCodeGenOnly = 1 in
-     def QVESPLATIb : Z23Form_2<4, 37,
-                         (outs qbrc:$FRT), (ins qbrc:$FRA, u2imm:$idx),
-                         "qvesplati $FRT, $FRA, $idx", IIC_VecPerm,
-                         [(set v4i1:$FRT,
-                               (PPCqvesplati v4i1:$FRA, (i32 imm:$idx)))]>;
-
-  def QVFPERM : AForm_1<4, 6,
-                      (outs qfrc:$FRT), (ins qfrc:$FRA, qfrc:$FRB, qfrc:$FRC),
-                      "qvfperm $FRT, $FRA, $FRB, $FRC", IIC_VecPerm,
-                      [(set v4f64:$FRT,
-                            (PPCqvfperm v4f64:$FRA, v4f64:$FRB, v4f64:$FRC))]>;
-  let isCodeGenOnly = 1 in
-     def QVFPERMs : AForm_1<4, 6,
-                         (outs qsrc:$FRT), (ins qsrc:$FRA, qsrc:$FRB, qfrc:$FRC),
-                         "qvfperm $FRT, $FRA, $FRB, $FRC", IIC_VecPerm,
-                         [(set v4f32:$FRT,
-                               (PPCqvfperm v4f32:$FRA, v4f32:$FRB, v4f64:$FRC))]>;
-
-  let isReMaterializable = 1, isAsCheapAsAMove = 1 in
-  def QVGPCI : Z23Form_3<4, 133,
-                      (outs qfrc:$FRT), (ins u12imm:$idx),
-                      "qvgpci $FRT, $idx", IIC_VecPerm,
-                      [(set v4f64:$FRT, (PPCqvgpci (u12:$idx)))]>;
-
-  // Compare Instruction
-  let isCodeGenOnly = 1 in
-    def QVFTSTNAN : QPXX18_Int<4, 64, "qvftstnan", int_ppc_qpx_qvftstnan>;
-  def QVFTSTNANb : XForm_18<4, 64, (outs qbrc:$FRT), (ins qfrc:$FRA, qfrc:$FRB),
-                           "qvftstnan $FRT, $FRA, $FRB", IIC_FPCompare,
-                           [(set v4i1:$FRT,
-                                 (setcc v4f64:$FRA, v4f64:$FRB, SETUO))]>;
-  let isCodeGenOnly = 1 in
-  def QVFTSTNANbs : XForm_18<4, 64, (outs qbrc:$FRT), (ins qsrc:$FRA, qsrc:$FRB),
-                            "qvftstnan $FRT, $FRA, $FRB", IIC_FPCompare,
-                            [(set v4i1:$FRT,
-                                  (setcc v4f32:$FRA, v4f32:$FRB, SETUO))]>;
-  let isCodeGenOnly = 1 in
-    def QVFCMPLT : QPXX18_Int<4, 96, "qvfcmplt", int_ppc_qpx_qvfcmplt>;
-  def QVFCMPLTb : XForm_18<4, 96, (outs qbrc:$FRT), (ins qfrc:$FRA, qfrc:$FRB),
-                           "qvfcmplt $FRT, $FRA, $FRB", IIC_FPCompare,
-                           [(set v4i1:$FRT,
-                                 (setcc v4f64:$FRA, v4f64:$FRB, SETOLT))]>;
-  let isCodeGenOnly = 1 in
-  def QVFCMPLTbs : XForm_18<4, 96, (outs qbrc:$FRT), (ins qsrc:$FRA, qsrc:$FRB),
-                            "qvfcmplt $FRT, $FRA, $FRB", IIC_FPCompare,
-                            [(set v4i1:$FRT,
-                                  (setcc v4f32:$FRA, v4f32:$FRB, SETOLT))]>;
-  let isCodeGenOnly = 1 in
-    def QVFCMPGT : QPXX18_Int<4, 32, "qvfcmpgt", int_ppc_qpx_qvfcmpgt>;
-  def QVFCMPGTb : XForm_18<4, 32, (outs qbrc:$FRT), (ins qfrc:$FRA, qfrc:$FRB),
-                           "qvfcmpgt $FRT, $FRA, $FRB", IIC_FPCompare,
-                           [(set v4i1:$FRT,
-                                 (setcc v4f64:$FRA, v4f64:$FRB, SETOGT))]>;
-  let isCodeGenOnly = 1 in
-  def QVFCMPGTbs : XForm_18<4, 32, (outs qbrc:$FRT), (ins qsrc:$FRA, qsrc:$FRB),
-                            "qvfcmpgt $FRT, $FRA, $FRB", IIC_FPCompare,
-                            [(set v4i1:$FRT,
-                                  (setcc v4f32:$FRA, v4f32:$FRB, SETOGT))]>;
-  let isCodeGenOnly = 1 in
-    def QVFCMPEQ : QPXX18_Int<4, 0, "qvfcmpeq", int_ppc_qpx_qvfcmpeq>;
-  def QVFCMPEQb : XForm_18<4, 0, (outs qbrc:$FRT), (ins qfrc:$FRA, qfrc:$FRB),
-                           "qvfcmpeq $FRT, $FRA, $FRB", IIC_FPCompare,
-                           [(set v4i1:$FRT,
-                                 (setcc v4f64:$FRA, v4f64:$FRB, SETOEQ))]>;
-  let isCodeGenOnly = 1 in
-  def QVFCMPEQbs : XForm_18<4, 0, (outs qbrc:$FRT), (ins qsrc:$FRA, qsrc:$FRB),
-                            "qvfcmpeq $FRT, $FRA, $FRB", IIC_FPCompare,
-                            [(set v4i1:$FRT,
-                                  (setcc v4f32:$FRA, v4f32:$FRB, SETOEQ))]>;
-
-  let isCodeGenOnly = 1 in
-  def QVFLOGICAL : XForm_20<4, 4,
-                      (outs qfrc:$FRT), (ins qfrc:$FRA, qfrc:$FRB, u12imm:$tttt),
-                      "qvflogical $FRT, $FRA, $FRB, $tttt", IIC_VecPerm, []>;
-  def QVFLOGICALb : XForm_20<4, 4,
-                      (outs qbrc:$FRT), (ins qbrc:$FRA, qbrc:$FRB, u12imm:$tttt),
-                      "qvflogical $FRT, $FRA, $FRB, $tttt", IIC_VecPerm, []>;
-  let isCodeGenOnly = 1 in
-  def QVFLOGICALs : XForm_20<4, 4,
-                      (outs qbrc:$FRT), (ins qbrc:$FRA, qbrc:$FRB, u12imm:$tttt),
-                      "qvflogical $FRT, $FRA, $FRB, $tttt", IIC_VecPerm, []>;
-
-  // Load indexed instructions
-  let mayLoad = 1 in {
-    def QVLFDX : XForm_1_memOp<31, 583,
-                              (outs qfrc:$FRT), (ins memrr:$src),
-                              "qvlfdx $FRT, $src", IIC_LdStLFD,
-                              [(set v4f64:$FRT, (load xoaddr:$src))]>;
-    let isCodeGenOnly = 1 in
-    def QVLFDXb : XForm_1_memOp<31, 583,
-                                (outs qbrc:$FRT), (ins memrr:$src),
-                                "qvlfdx $FRT, $src", IIC_LdStLFD, []>;
-
-    let RC = 1 in
-    def QVLFDXA : XForm_1<31, 583,
-                        (outs qfrc:$FRT), (ins memrr:$src),
-                        "qvlfdxa $FRT, $src", IIC_LdStLFD, []>;
-
-    def QVLFDUX : XForm_1<31, 615,
-                        (outs qfrc:$FRT, ptr_rc_nor0:$ea_result),
-                        (ins memrr:$src),
-                        "qvlfdux $FRT, $src", IIC_LdStLFDU, []>,
-                        RegConstraint<"$src.ptrreg = $ea_result">,
-                        NoEncode<"$ea_result">;
-    let RC = 1 in
-    def QVLFDUXA : XForm_1<31, 615,
-                        (outs qfrc:$FRT), (ins memrr:$src),
-                        "qvlfduxa $FRT, $src", IIC_LdStLFD, []>;
-
-    def QVLFSX : XForm_1_memOp<31, 519,
-                              (outs qfrc:$FRT), (ins memrr:$src),
-                              "qvlfsx $FRT, $src", IIC_LdStLFD,
-                              [(set v4f64:$FRT, (extloadv4f32 xoaddr:$src))]>;
-
-    let isCodeGenOnly = 1 in
-    def QVLFSXb : XForm_1<31, 519,
-                        (outs qbrc:$FRT), (ins memrr:$src),
-                        "qvlfsx $FRT, $src", IIC_LdStLFD,
-                        [(set v4i1:$FRT, (PPCqvlfsb xoaddr:$src))]>;
-    let isCodeGenOnly = 1 in
-    def QVLFSXs : XForm_1_memOp<31, 519,
-                                (outs qsrc:$FRT), (ins memrr:$src),
-                                "qvlfsx $FRT, $src", IIC_LdStLFD,
-                                [(set v4f32:$FRT, (load xoaddr:$src))]>;
-
-    let RC = 1 in
-    def QVLFSXA : XForm_1<31, 519,
-                        (outs qfrc:$FRT), (ins memrr:$src),
-                        "qvlfsxa $FRT, $src", IIC_LdStLFD, []>;
-
-    def QVLFSUX : XForm_1<31, 551,
-                        (outs qsrc:$FRT, ptr_rc_nor0:$ea_result),
-                        (ins memrr:$src),
-                        "qvlfsux $FRT, $src", IIC_LdStLFDU, []>,
-                        RegConstraint<"$src.ptrreg = $ea_result">,
-                        NoEncode<"$ea_result">;
-
-    let RC = 1 in
-    def QVLFSUXA : XForm_1<31, 551,
-                        (outs qfrc:$FRT), (ins memrr:$src),
-                        "qvlfsuxa $FRT, $src", IIC_LdStLFD, []>;
-
-    def QVLFCDX : XForm_1<31, 71,
-                        (outs qfrc:$FRT), (ins memrr:$src),
-                        "qvlfcdx $FRT, $src", IIC_LdStLFD, []>;
-    let RC = 1 in
-    def QVLFCDXA : XForm_1<31, 71,
-                        (outs qfrc:$FRT), (ins memrr:$src),
-                        "qvlfcdxa $FRT, $src", IIC_LdStLFD, []>;
-
-    def QVLFCDUX : XForm_1<31, 103,
-                        (outs qfrc:$FRT), (ins memrr:$src),
-                        "qvlfcdux $FRT, $src", IIC_LdStLFD, []>;
-    let RC = 1 in
-    def QVLFCDUXA : XForm_1<31, 103,
-                        (outs qfrc:$FRT), (ins memrr:$src),
-                        "qvlfcduxa $FRT, $src", IIC_LdStLFD, []>;
-
-    def QVLFCSX : XForm_1<31, 7,
-                        (outs qfrc:$FRT), (ins memrr:$src),
-                        "qvlfcsx $FRT, $src", IIC_LdStLFD, []>;
-    let isCodeGenOnly = 1 in
-    def QVLFCSXs : XForm_1<31, 7,
-                         (outs qsrc:$FRT), (ins memrr:$src),
-                         "qvlfcsx $FRT, $src", IIC_LdStLFD, []>;
-
-    let RC = 1 in
-    def QVLFCSXA : XForm_1<31, 7,
-                        (outs qfrc:$FRT), (ins memrr:$src),
-                        "qvlfcsxa $FRT, $src", IIC_LdStLFD, []>;
-
-    def QVLFCSUX : XForm_1<31, 39,
-                        (outs qfrc:$FRT), (ins memrr:$src),
-                        "qvlfcsux $FRT, $src", IIC_LdStLFD, []>;
-    let RC = 1 in
-    def QVLFCSUXA : XForm_1<31, 39,
-                        (outs qfrc:$FRT), (ins memrr:$src),
-                        "qvlfcsuxa $FRT, $src", IIC_LdStLFD, []>;
-
-    def QVLFIWAX : XForm_1<31, 871,
-                        (outs qfrc:$FRT), (ins memrr:$src),
-                        "qvlfiwax $FRT, $src", IIC_LdStLFD, []>;
-    let RC = 1 in
-    def QVLFIWAXA : XForm_1<31, 871,
-                        (outs qfrc:$FRT), (ins memrr:$src),
-                        "qvlfiwaxa $FRT, $src", IIC_LdStLFD, []>;
-
-    def QVLFIWZX : XForm_1<31, 839,
-                        (outs qfrc:$FRT), (ins memrr:$src),
-                        "qvlfiwzx $FRT, $src", IIC_LdStLFD, []>;
-    let RC = 1 in
-    def QVLFIWZXA : XForm_1<31, 839,
-                        (outs qfrc:$FRT), (ins memrr:$src),
-                        "qvlfiwzxa $FRT, $src", IIC_LdStLFD, []>;
-  }
-
-
-  def QVLPCLDX : XForm_1<31, 582,
-                      (outs qfrc:$FRT), (ins memrr:$src),
-                      "qvlpcldx $FRT, $src", IIC_LdStLFD, []>;
-  def QVLPCLSX : XForm_1<31, 518,
-                      (outs qfrc:$FRT), (ins memrr:$src),
-                      "qvlpclsx $FRT, $src", IIC_LdStLFD, []>;
-  let isCodeGenOnly = 1 in
-    def QVLPCLSXint : XForm_11<31, 518,
-                              (outs qfrc:$FRT), (ins G8RC:$src),
-                              "qvlpclsx $FRT, 0, $src", IIC_LdStLFD, []>;
-  def QVLPCRDX : XForm_1<31, 70,
-                      (outs qfrc:$FRT), (ins memrr:$src),
-                      "qvlpcrdx $FRT, $src", IIC_LdStLFD, []>;
-  def QVLPCRSX : XForm_1<31, 6,
-                      (outs qfrc:$FRT), (ins memrr:$src),
-                      "qvlpcrsx $FRT, $src", IIC_LdStLFD, []>;
-
-  // Store indexed instructions
-  let mayStore = 1 in {
-    def QVSTFDX : XForm_8_memOp<31, 711,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfdx $FRT, $dst", IIC_LdStSTFD,
-                        [(store qfrc:$FRT, xoaddr:$dst)]>;
-    let isCodeGenOnly = 1 in
-    def QVSTFDXb : XForm_8_memOp<31, 711,
-                        (outs), (ins qbrc:$FRT, memrr:$dst),
-                        "qvstfdx $FRT, $dst", IIC_LdStSTFD, []>;
-
-    let RC = 1 in
-    def QVSTFDXA : XForm_8<31, 711,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfdxa $FRT, $dst", IIC_LdStSTFD, []>;
-
-    def QVSTFDUX : XForm_8<31, 743, (outs ptr_rc_nor0:$ea_res),
-                           (ins qfrc:$FRT, memrr:$dst),
-                           "qvstfdux $FRT, $dst", IIC_LdStSTFDU, []>,
-                           RegConstraint<"$dst.ptrreg = $ea_res">,
-                           NoEncode<"$ea_res">;
-
-    let RC = 1 in
-    def QVSTFDUXA : XForm_8<31, 743,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfduxa $FRT, $dst", IIC_LdStSTFD, []>;
-
-    def QVSTFDXI : XForm_8<31, 709,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfdxi $FRT, $dst", IIC_LdStSTFD, []>;
-    let RC = 1 in
-    def QVSTFDXIA : XForm_8<31, 709,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfdxia $FRT, $dst", IIC_LdStSTFD, []>;
-
-    def QVSTFDUXI : XForm_8<31, 741,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfduxi $FRT, $dst", IIC_LdStSTFD, []>;
-    let RC = 1 in
-    def QVSTFDUXIA : XForm_8<31, 741,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfduxia $FRT, $dst", IIC_LdStSTFD, []>;
-
-    def QVSTFSX : XForm_8_memOp<31, 647,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfsx $FRT, $dst", IIC_LdStSTFD,
-                        [(truncstorev4f32 qfrc:$FRT, xoaddr:$dst)]>;
-    let isCodeGenOnly = 1 in
-    def QVSTFSXs : XForm_8_memOp<31, 647,
-                         (outs), (ins qsrc:$FRT, memrr:$dst),
-                         "qvstfsx $FRT, $dst", IIC_LdStSTFD,
-                         [(store qsrc:$FRT, xoaddr:$dst)]>;
-
-    let RC = 1 in
-    def QVSTFSXA : XForm_8<31, 647,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfsxa $FRT, $dst", IIC_LdStSTFD, []>;
-
-    def QVSTFSUX : XForm_8<31, 679, (outs ptr_rc_nor0:$ea_res),
-                           (ins qsrc:$FRT, memrr:$dst),
-                           "qvstfsux $FRT, $dst", IIC_LdStSTFDU, []>,
-                           RegConstraint<"$dst.ptrreg = $ea_res">,
-                           NoEncode<"$ea_res">;
-    let isCodeGenOnly = 1 in
-    def QVSTFSUXs: XForm_8<31, 679, (outs ptr_rc_nor0:$ea_res),
-                           (ins qfrc:$FRT, memrr:$dst),
-                           "qvstfsux $FRT, $dst", IIC_LdStSTFDU, []>,
-                           RegConstraint<"$dst.ptrreg = $ea_res">,
-                           NoEncode<"$ea_res">;
-
-    let RC = 1 in
-    def QVSTFSUXA : XForm_8<31, 679,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfsuxa $FRT, $dst", IIC_LdStSTFD, []>;
-
-    def QVSTFSXI : XForm_8<31, 645,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfsxi $FRT, $dst", IIC_LdStSTFD, []>;
-    let RC = 1 in
-    def QVSTFSXIA : XForm_8<31, 645,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfsxia $FRT, $dst", IIC_LdStSTFD, []>;
-
-    def QVSTFSUXI : XForm_8<31, 677,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfsuxi $FRT, $dst", IIC_LdStSTFD, []>;
-    let RC = 1 in
-    def QVSTFSUXIA : XForm_8<31, 677,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfsuxia $FRT, $dst", IIC_LdStSTFD, []>;
-
-    def QVSTFCDX : XForm_8<31, 199,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfcdx $FRT, $dst", IIC_LdStSTFD, []>;
-    let RC = 1 in
-    def QVSTFCDXA : XForm_8<31, 199,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfcdxa $FRT, $dst", IIC_LdStSTFD, []>;
-
-    def QVSTFCSX : XForm_8<31, 135,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfcsx $FRT, $dst", IIC_LdStSTFD, []>;
-    let isCodeGenOnly = 1 in
-    def QVSTFCSXs : XForm_8<31, 135,
-                         (outs), (ins qsrc:$FRT, memrr:$dst),
-                         "qvstfcsx $FRT, $dst", IIC_LdStSTFD, []>;
-
-    let RC = 1 in
-    def QVSTFCSXA : XForm_8<31, 135,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfcsxa $FRT, $dst", IIC_LdStSTFD, []>;
-
-    def QVSTFCDUX : XForm_8<31, 231,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfcdux $FRT, $dst", IIC_LdStSTFD, []>;
-    let RC = 1 in
-    def QVSTFCDUXA : XForm_8<31, 231,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfcduxa $FRT, $dst", IIC_LdStSTFD, []>;
-
-    def QVSTFCSUX : XForm_8<31, 167,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfcsux $FRT, $dst", IIC_LdStSTFD, []>;
-    let RC = 1 in
-    def QVSTFCSUXA : XForm_8<31, 167,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfcsuxa $FRT, $dst", IIC_LdStSTFD, []>;
-
-    def QVSTFCDXI : XForm_8<31, 197,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfcdxi $FRT, $dst", IIC_LdStSTFD, []>;
-    let RC = 1 in
-    def QVSTFCDXIA : XForm_8<31, 197,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfcdxia $FRT, $dst", IIC_LdStSTFD, []>;
-
-    def QVSTFCSXI : XForm_8<31, 133,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfcsxi $FRT, $dst", IIC_LdStSTFD, []>;
-    let RC = 1 in
-    def QVSTFCSXIA : XForm_8<31, 133,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfcsxia $FRT, $dst", IIC_LdStSTFD, []>;
-
-    def QVSTFCDUXI : XForm_8<31, 229,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfcduxi $FRT, $dst", IIC_LdStSTFD, []>;
-    let RC = 1 in
-    def QVSTFCDUXIA : XForm_8<31, 229,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfcduxia $FRT, $dst", IIC_LdStSTFD, []>;
-
-    def QVSTFCSUXI : XForm_8<31, 165,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfcsuxi $FRT, $dst", IIC_LdStSTFD, []>;
-    let RC = 1 in
-    def QVSTFCSUXIA : XForm_8<31, 165,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfcsuxia $FRT, $dst", IIC_LdStSTFD, []>;
-
-    def QVSTFIWX : XForm_8<31, 967,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfiwx $FRT, $dst", IIC_LdStSTFD, []>;
-    let RC = 1 in
-    def QVSTFIWXA : XForm_8<31, 967,
-                        (outs), (ins qfrc:$FRT, memrr:$dst),
-                        "qvstfiwxa $FRT, $dst", IIC_LdStSTFD, []>;
-  }
-}
-
-} // neverHasSideEffects
-}
-
-def : InstAlias<"qvfclr $FRT",
-                (QVFLOGICALb qbrc:$FRT, qbrc:$FRT, qbrc:$FRT, 0)>;
-def : InstAlias<"qvfand $FRT, $FRA, $FRB",
-                (QVFLOGICALb qbrc:$FRT, qbrc:$FRA, qbrc:$FRB, 1)>;
-def : InstAlias<"qvfandc $FRT, $FRA, $FRB",
-                (QVFLOGICALb qbrc:$FRT, qbrc:$FRA, qbrc:$FRB, 4)>;
-def : InstAlias<"qvfctfb $FRT, $FRA",
-                (QVFLOGICALb qbrc:$FRT, qbrc:$FRA, qbrc:$FRA, 5)>;
-def : InstAlias<"qvfxor $FRT, $FRA, $FRB",
-                (QVFLOGICALb qbrc:$FRT, qbrc:$FRA, qbrc:$FRB, 6)>;
-def : InstAlias<"qvfor $FRT, $FRA, $FRB",
-                (QVFLOGICALb qbrc:$FRT, qbrc:$FRA, qbrc:$FRB, 7)>;
-def : InstAlias<"qvfnor $FRT, $FRA, $FRB",
-                (QVFLOGICALb qbrc:$FRT, qbrc:$FRA, qbrc:$FRB, 8)>;
-def : InstAlias<"qvfequ $FRT, $FRA, $FRB",
-                (QVFLOGICALb qbrc:$FRT, qbrc:$FRA, qbrc:$FRB, 9)>;
-def : InstAlias<"qvfnot $FRT, $FRA",
-                (QVFLOGICALb qbrc:$FRT, qbrc:$FRA, qbrc:$FRA, 10)>;
-def : InstAlias<"qvforc $FRT, $FRA, $FRB",
-                (QVFLOGICALb qbrc:$FRT, qbrc:$FRA, qbrc:$FRB, 13)>;
-def : InstAlias<"qvfnand $FRT, $FRA, $FRB",
-                (QVFLOGICALb qbrc:$FRT, qbrc:$FRA, qbrc:$FRB, 14)>;
-def : InstAlias<"qvfset $FRT",
-                (QVFLOGICALb qbrc:$FRT, qbrc:$FRT, qbrc:$FRT, 15)>;
-
-//===----------------------------------------------------------------------===//
-// Additional QPX Patterns
-//
-
-def : Pat<(v4f64 (scalar_to_vector f64:$A)),
-          (INSERT_SUBREG (v4f64 (IMPLICIT_DEF)), $A, sub_64)>;
-def : Pat<(v4f32 (scalar_to_vector f32:$A)),
-          (INSERT_SUBREG (v4f32 (IMPLICIT_DEF)), $A, sub_64)>;
-
-def : Pat<(f64 (extractelt v4f64:$S, 0)),
-          (EXTRACT_SUBREG $S, sub_64)>;
-def : Pat<(f32 (extractelt v4f32:$S, 0)),
-          (EXTRACT_SUBREG $S, sub_64)>;
-
-def : Pat<(f64 (extractelt v4f64:$S, 1)),
-          (EXTRACT_SUBREG (QVESPLATI $S, 1), sub_64)>;
-def : Pat<(f64 (extractelt v4f64:$S, 2)),
-          (EXTRACT_SUBREG (QVESPLATI $S, 2), sub_64)>;
-def : Pat<(f64 (extractelt v4f64:$S, 3)),
-          (EXTRACT_SUBREG (QVESPLATI $S, 3), sub_64)>;
-
-def : Pat<(f32 (extractelt v4f32:$S, 1)),
-          (EXTRACT_SUBREG (QVESPLATIs $S, 1), sub_64)>;
-def : Pat<(f32 (extractelt v4f32:$S, 2)),
-          (EXTRACT_SUBREG (QVESPLATIs $S, 2), sub_64)>;
-def : Pat<(f32 (extractelt v4f32:$S, 3)),
-          (EXTRACT_SUBREG (QVESPLATIs $S, 3), sub_64)>;
-
-def : Pat<(f64 (extractelt v4f64:$S, i64:$F)),
-          (EXTRACT_SUBREG (QVFPERM $S, $S,
-                                   (QVLPCLSXint (RLDICR $F, 2,
-                                                        /* 63-2 = */ 61))),
-                          sub_64)>;
-def : Pat<(f32 (extractelt v4f32:$S, i64:$F)),
-          (EXTRACT_SUBREG (QVFPERMs $S, $S,
-                                    (QVLPCLSXint (RLDICR $F, 2,
-                                                         /* 63-2 = */ 61))),
-                          sub_64)>;
-
-def : Pat<(int_ppc_qpx_qvfperm v4f64:$A, v4f64:$B, v4f64:$C),
-          (QVFPERM $A, $B, $C)>;
-
-def : Pat<(int_ppc_qpx_qvfcpsgn v4f64:$A, v4f64:$B),
-          (QVFCPSGN $A, $B)>;
-
-// FCOPYSIGN's operand types need not agree.
-def : Pat<(fcopysign v4f64:$frB, v4f32:$frA),
-          (QVFCPSGN (COPY_TO_REGCLASS $frA, QFRC), $frB)>;
-def : Pat<(fcopysign QSRC:$frB, QFRC:$frA),
-          (QVFCPSGNs (COPY_TO_REGCLASS $frA, QSRC), $frB)>;
-
-def : Pat<(int_ppc_qpx_qvfneg v4f64:$A), (QVFNEG $A)>;
-def : Pat<(int_ppc_qpx_qvfabs v4f64:$A), (QVFABS $A)>;
-def : Pat<(int_ppc_qpx_qvfnabs v4f64:$A), (QVFNABS $A)>;
-
-def : Pat<(int_ppc_qpx_qvfriz v4f64:$A), (QVFRIZ $A)>;
-def : Pat<(int_ppc_qpx_qvfrin v4f64:$A), (QVFRIN $A)>;
-def : Pat<(int_ppc_qpx_qvfrip v4f64:$A), (QVFRIP $A)>;
-def : Pat<(int_ppc_qpx_qvfrim v4f64:$A), (QVFRIM $A)>;
-
-def : Pat<(int_ppc_qpx_qvfre v4f64:$A), (QVFRE $A)>;
-def : Pat<(int_ppc_qpx_qvfrsqrte v4f64:$A), (QVFRSQRTE $A)>;
-
-def : Pat<(int_ppc_qpx_qvfadd v4f64:$A, v4f64:$B),
-          (QVFADD $A, $B)>;
-def : Pat<(int_ppc_qpx_qvfsub v4f64:$A, v4f64:$B),
-          (QVFSUB $A, $B)>;
-def : Pat<(int_ppc_qpx_qvfmul v4f64:$A, v4f64:$B),
-          (QVFMUL $A, $B)>;
-
-// Additional QVFNMSUB patterns: -a*c + b == -(a*c - b)
-def : Pat<(fma (fneg v4f64:$A), v4f64:$C, v4f64:$B),
-          (QVFNMSUB $A, $C, $B)>;
-def : Pat<(fma v4f64:$A, (fneg v4f64:$C), v4f64:$B),
-          (QVFNMSUB $A, $C, $B)>;
-def : Pat<(fma (fneg v4f32:$A), v4f32:$C, v4f32:$B),
-          (QVFNMSUBSs $A, $C, $B)>;
-def : Pat<(fma v4f32:$A, (fneg v4f32:$C), v4f32:$B),
-          (QVFNMSUBSs $A, $C, $B)>;
-
-def : Pat<(int_ppc_qpx_qvfmadd v4f64:$A, v4f64:$B, v4f64:$C),
-          (QVFMADD $A, $B, $C)>;
-def : Pat<(int_ppc_qpx_qvfnmadd v4f64:$A, v4f64:$B, v4f64:$C),
-          (QVFNMADD $A, $B, $C)>;
-def : Pat<(int_ppc_qpx_qvfmsub v4f64:$A, v4f64:$B, v4f64:$C),
-          (QVFMSUB $A, $B, $C)>;
-def : Pat<(int_ppc_qpx_qvfnmsub v4f64:$A, v4f64:$B, v4f64:$C),
-          (QVFNMSUB $A, $B, $C)>;
-
-def : Pat<(int_ppc_qpx_qvlfd xoaddr:$src),
-          (QVLFDX xoaddr:$src)>;
-def : Pat<(int_ppc_qpx_qvlfda xoaddr:$src),
-          (QVLFDXA xoaddr:$src)>;
-def : Pat<(int_ppc_qpx_qvlfs xoaddr:$src),
-          (QVLFSX xoaddr:$src)>;
-def : Pat<(int_ppc_qpx_qvlfsa xoaddr:$src),
-          (QVLFSXA xoaddr:$src)>;
-def : Pat<(int_ppc_qpx_qvlfcda xoaddr:$src),
-          (QVLFCDXA xoaddr:$src)>;
-def : Pat<(int_ppc_qpx_qvlfcd xoaddr:$src),
-          (QVLFCDX xoaddr:$src)>;
-def : Pat<(int_ppc_qpx_qvlfcsa xoaddr:$src),
-          (QVLFCSXA xoaddr:$src)>;
-def : Pat<(int_ppc_qpx_qvlfcs xoaddr:$src),
-          (QVLFCSX xoaddr:$src)>;
-def : Pat<(int_ppc_qpx_qvlfda xoaddr:$src),
-          (QVLFDXA xoaddr:$src)>;
-def : Pat<(int_ppc_qpx_qvlfiwaa xoaddr:$src),
-          (QVLFIWAXA xoaddr:$src)>;
-def : Pat<(int_ppc_qpx_qvlfiwa xoaddr:$src),
-          (QVLFIWAX xoaddr:$src)>;
-def : Pat<(int_ppc_qpx_qvlfiwza xoaddr:$src),
-          (QVLFIWZXA xoaddr:$src)>;
-def : Pat<(int_ppc_qpx_qvlfiwz xoaddr:$src),
-          (QVLFIWZX xoaddr:$src)>;
-def : Pat<(int_ppc_qpx_qvlfsa xoaddr:$src),
-          (QVLFSXA xoaddr:$src)>;
-def : Pat<(int_ppc_qpx_qvlpcld xoaddr:$src),
-          (QVLPCLDX xoaddr:$src)>;
-def : Pat<(int_ppc_qpx_qvlpcls xoaddr:$src),
-          (QVLPCLSX xoaddr:$src)>;
-def : Pat<(int_ppc_qpx_qvlpcrd xoaddr:$src),
-          (QVLPCRDX xoaddr:$src)>;
-def : Pat<(int_ppc_qpx_qvlpcrs xoaddr:$src),
-          (QVLPCRSX xoaddr:$src)>;
-
-def : Pat<(int_ppc_qpx_qvstfd v4f64:$T, xoaddr:$dst),
-          (QVSTFDX $T, xoaddr:$dst)>;
-def : Pat<(int_ppc_qpx_qvstfs v4f64:$T, xoaddr:$dst),
-          (QVSTFSX $T, xoaddr:$dst)>;
-def : Pat<(int_ppc_qpx_qvstfcda v4f64:$T, xoaddr:$dst),
-          (QVSTFCDXA $T, xoaddr:$dst)>;
-def : Pat<(int_ppc_qpx_qvstfcd v4f64:$T, xoaddr:$dst),
-          (QVSTFCDX $T, xoaddr:$dst)>;
-def : Pat<(int_ppc_qpx_qvstfcsa v4f64:$T, xoaddr:$dst),
-          (QVSTFCSXA $T, xoaddr:$dst)>;
-def : Pat<(int_ppc_qpx_qvstfcs v4f64:$T, xoaddr:$dst),
-          (QVSTFCSX $T, xoaddr:$dst)>;
-def : Pat<(int_ppc_qpx_qvstfda v4f64:$T, xoaddr:$dst),
-          (QVSTFDXA $T, xoaddr:$dst)>;
-def : Pat<(int_ppc_qpx_qvstfiwa v4f64:$T, xoaddr:$dst),
-          (QVSTFIWXA $T, xoaddr:$dst)>;
-def : Pat<(int_ppc_qpx_qvstfiw v4f64:$T, xoaddr:$dst),
-          (QVSTFIWX $T, xoaddr:$dst)>;
-def : Pat<(int_ppc_qpx_qvstfsa v4f64:$T, xoaddr:$dst),
-          (QVSTFSXA $T, xoaddr:$dst)>;
-
-def : Pat<(pre_store v4f64:$rS, iPTR:$ptrreg, iPTR:$ptroff),
-          (QVSTFDUX $rS, $ptrreg, $ptroff)>;
-def : Pat<(pre_store v4f32:$rS, iPTR:$ptrreg, iPTR:$ptroff),
-          (QVSTFSUX $rS, $ptrreg, $ptroff)>;
-def : Pat<(pre_truncstv4f32 v4f64:$rS, iPTR:$ptrreg, iPTR:$ptroff),
-          (QVSTFSUXs $rS, $ptrreg, $ptroff)>;
-
-def : Pat<(int_ppc_qpx_qvflogical  v4f64:$A, v4f64:$B, (i32 imm:$idx)),
-          (QVFLOGICAL $A, $B, imm:$idx)>;
-def : Pat<(int_ppc_qpx_qvgpci (u12:$idx)),
-          (QVGPCI imm:$idx)>;
-
-def : Pat<(setcc v4f64:$FRA, v4f64:$FRB, SETOGE),
-          (QVFLOGICALb (QVFCMPLTb $FRA, $FRB),
-                       (QVFTSTNANb $FRA, $FRB), (i32 8))>;
-def : Pat<(setcc v4f64:$FRA, v4f64:$FRB, SETOLE),
-          (QVFLOGICALb (QVFCMPGTb $FRA, $FRB),
-                       (QVFTSTNANb $FRA, $FRB), (i32 8))>;
-def : Pat<(setcc v4f64:$FRA, v4f64:$FRB, SETONE),
-          (QVFLOGICALb (QVFCMPEQb $FRA, $FRB),
-                       (QVFTSTNANb $FRA, $FRB), (i32 8))>;
-def : Pat<(setcc v4f64:$FRA, v4f64:$FRB, SETO),
-          (QVFLOGICALb (QVFTSTNANb $FRA, $FRB),
-                       (QVFTSTNANb $FRA, $FRB), (i32 10))>;
-def : Pat<(setcc v4f64:$FRA, v4f64:$FRB, SETUEQ),
-          (QVFLOGICALb (QVFCMPEQb $FRA, $FRB),
-                       (QVFTSTNANb $FRA, $FRB), (i32 7))>;
-def : Pat<(setcc v4f64:$FRA, v4f64:$FRB, SETUGT),
-          (QVFLOGICALb (QVFCMPGTb $FRA, $FRB),
-                       (QVFTSTNANb $FRA, $FRB), (i32 7))>;
-def : Pat<(setcc v4f64:$FRA, v4f64:$FRB, SETUGE),
-          (QVFLOGICALb (QVFTSTNANb $FRA, $FRB),
-                       (QVFCMPLTb $FRA, $FRB), (i32 13))>;
-def : Pat<(setcc v4f64:$FRA, v4f64:$FRB, SETULT),
-          (QVFLOGICALb (QVFCMPLTb $FRA, $FRB),
-                       (QVFTSTNANb $FRA, $FRB), (i32 7))>;
-def : Pat<(setcc v4f64:$FRA, v4f64:$FRB, SETULE),
-          (QVFLOGICALb (QVFTSTNANb $FRA, $FRB),
-                       (QVFCMPGTb $FRA, $FRB), (i32 13))>;
-def : Pat<(setcc v4f64:$FRA, v4f64:$FRB, SETUNE),
-          (QVFLOGICALb (QVFTSTNANb $FRA, $FRB),
-                       (QVFCMPEQb $FRA, $FRB), (i32 13))>;
-
-def : Pat<(setcc v4f64:$FRA, v4f64:$FRB, SETEQ),
-          (QVFCMPEQb $FRA, $FRB)>;
-def : Pat<(setcc v4f64:$FRA, v4f64:$FRB, SETGT),
-          (QVFCMPGTb $FRA, $FRB)>;
-def : Pat<(setcc v4f64:$FRA, v4f64:$FRB, SETGE),
-          (QVFLOGICALb (QVFCMPLTb $FRA, $FRB),
-                       (QVFCMPLTb $FRA, $FRB), (i32 10))>;
-def : Pat<(setcc v4f64:$FRA, v4f64:$FRB, SETLT),
-          (QVFCMPLTb $FRA, $FRB)>;
-def : Pat<(setcc v4f64:$FRA, v4f64:$FRB, SETLE),
-          (QVFLOGICALb (QVFCMPGTb $FRA, $FRB),
-                       (QVFCMPGTb $FRA, $FRB), (i32 10))>;
-def : Pat<(setcc v4f64:$FRA, v4f64:$FRB, SETNE),
-          (QVFLOGICALb (QVFCMPEQb $FRA, $FRB),
-                       (QVFCMPEQb $FRA, $FRB), (i32 10))>;
-
-def : Pat<(setcc v4f32:$FRA, v4f32:$FRB, SETOGE),
-          (QVFLOGICALb (QVFCMPLTbs $FRA, $FRB),
-                       (QVFTSTNANbs $FRA, $FRB), (i32 8))>;
-def : Pat<(setcc v4f32:$FRA, v4f32:$FRB, SETOLE),
-          (QVFLOGICALb (QVFCMPGTbs $FRA, $FRB),
-                       (QVFTSTNANbs $FRA, $FRB), (i32 8))>;
-def : Pat<(setcc v4f32:$FRA, v4f32:$FRB, SETONE),
-          (QVFLOGICALb (QVFCMPEQbs $FRA, $FRB),
-                       (QVFTSTNANbs $FRA, $FRB), (i32 8))>;
-def : Pat<(setcc v4f32:$FRA, v4f32:$FRB, SETO),
-          (QVFLOGICALb (QVFTSTNANbs $FRA, $FRB),
-                       (QVFTSTNANbs $FRA, $FRB), (i32 10))>;
-def : Pat<(setcc v4f32:$FRA, v4f32:$FRB, SETUEQ),
-          (QVFLOGICALb (QVFCMPEQbs $FRA, $FRB),
-                       (QVFTSTNANbs $FRA, $FRB), (i32 7))>;
-def : Pat<(setcc v4f32:$FRA, v4f32:$FRB, SETUGT),
-          (QVFLOGICALb (QVFCMPGTbs $FRA, $FRB),
-                       (QVFTSTNANbs $FRA, $FRB), (i32 7))>;
-def : Pat<(setcc v4f32:$FRA, v4f32:$FRB, SETUGE),
-          (QVFLOGICALb (QVFTSTNANbs $FRA, $FRB),
-                       (QVFCMPLTbs $FRA, $FRB), (i32 13))>;
-def : Pat<(setcc v4f32:$FRA, v4f32:$FRB, SETULT),
-          (QVFLOGICALb (QVFCMPLTbs $FRA, $FRB),
-                       (QVFTSTNANbs $FRA, $FRB), (i32 7))>;
-def : Pat<(setcc v4f32:$FRA, v4f32:$FRB, SETULE),
-          (QVFLOGICALb (QVFTSTNANbs $FRA, $FRB),
-                       (QVFCMPGTbs $FRA, $FRB), (i32 13))>;
-def : Pat<(setcc v4f32:$FRA, v4f32:$FRB, SETUNE),
-          (QVFLOGICALb (QVFTSTNANbs $FRA, $FRB),
-                       (QVFCMPEQbs $FRA, $FRB), (i32 13))>;
-
-def : Pat<(setcc v4f32:$FRA, v4f32:$FRB, SETEQ),
-          (QVFCMPEQbs $FRA, $FRB)>;
-def : Pat<(setcc v4f32:$FRA, v4f32:$FRB, SETGT),
-          (QVFCMPGTbs $FRA, $FRB)>;
-def : Pat<(setcc v4f32:$FRA, v4f32:$FRB, SETGE),
-          (QVFLOGICALb (QVFCMPLTbs $FRA, $FRB),
-                       (QVFCMPLTbs $FRA, $FRB), (i32 10))>;
-def : Pat<(setcc v4f32:$FRA, v4f32:$FRB, SETLT),
-          (QVFCMPLTbs $FRA, $FRB)>;
-def : Pat<(setcc v4f32:$FRA, v4f32:$FRB, SETLE),
-          (QVFLOGICALb (QVFCMPGTbs $FRA, $FRB),
-                       (QVFCMPGTbs $FRA, $FRB), (i32 10))>;
-def : Pat<(setcc v4f32:$FRA, v4f32:$FRB, SETNE),
-          (QVFLOGICALb (QVFCMPEQbs $FRA, $FRB),
-                       (QVFCMPEQbs $FRA, $FRB), (i32 10))>;
-
-def : Pat<(and v4i1:$FRA, (not v4i1:$FRB)),
-          (QVFLOGICALb $FRA, $FRB, (i32 4))>;
-def : Pat<(not (or v4i1:$FRA, v4i1:$FRB)),
-          (QVFLOGICALb $FRA, $FRB, (i32 8))>;
-def : Pat<(not (xor v4i1:$FRA, v4i1:$FRB)),
-          (QVFLOGICALb $FRA, $FRB, (i32 9))>;
-def : Pat<(or v4i1:$FRA, (not v4i1:$FRB)),
-          (QVFLOGICALb $FRA, $FRB, (i32 13))>;
-def : Pat<(not (and v4i1:$FRA, v4i1:$FRB)),
-          (QVFLOGICALb $FRA, $FRB, (i32 14))>;
-
-def : Pat<(and v4i1:$FRA, v4i1:$FRB),
-          (QVFLOGICALb $FRA, $FRB, (i32 1))>;
-def : Pat<(or v4i1:$FRA, v4i1:$FRB),
-          (QVFLOGICALb $FRA, $FRB, (i32 7))>;
-def : Pat<(xor v4i1:$FRA, v4i1:$FRB),
-          (QVFLOGICALb $FRA, $FRB, (i32 6))>;
-def : Pat<(not v4i1:$FRA),
-          (QVFLOGICALb $FRA, $FRA, (i32 10))>;
-
-def : Pat<(v4f64 (fpextend v4f32:$src)),
-          (COPY_TO_REGCLASS $src, QFRC)>;
-
-def : Pat<(v4f32 (fround_exact v4f64:$src)),
-          (COPY_TO_REGCLASS $src, QSRC)>;
-
-// Extract the underlying floating-point values from the
-// QPX (-1.0, 1.0) boolean representation.
-def : Pat<(v4f64 (PPCqbflt v4i1:$src)),
-          (COPY_TO_REGCLASS $src, QFRC)>;
-
-def : Pat<(v4f64 (selectcc i1:$lhs, i1:$rhs, v4f64:$tval, v4f64:$fval, SETLT)),
-          (SELECT_QFRC (CRANDC $lhs, $rhs), $tval, $fval)>;
-def : Pat<(v4f64 (selectcc i1:$lhs, i1:$rhs, v4f64:$tval, v4f64:$fval, SETULT)),
-          (SELECT_QFRC (CRANDC $rhs, $lhs), $tval, $fval)>;
-def : Pat<(v4f64 (selectcc i1:$lhs, i1:$rhs, v4f64:$tval, v4f64:$fval, SETLE)),
-          (SELECT_QFRC (CRORC  $lhs, $rhs), $tval, $fval)>;
-def : Pat<(v4f64 (selectcc i1:$lhs, i1:$rhs, v4f64:$tval, v4f64:$fval, SETULE)),
-          (SELECT_QFRC (CRORC  $rhs, $lhs), $tval, $fval)>;
-def : Pat<(v4f64 (selectcc i1:$lhs, i1:$rhs, v4f64:$tval, v4f64:$fval, SETEQ)),
-          (SELECT_QFRC (CREQV $lhs, $rhs), $tval, $fval)>;
-def : Pat<(v4f64 (selectcc i1:$lhs, i1:$rhs, v4f64:$tval, v4f64:$fval, SETGE)),
-          (SELECT_QFRC (CRORC  $rhs, $lhs), $tval, $fval)>;
-def : Pat<(v4f64 (selectcc i1:$lhs, i1:$rhs, v4f64:$tval, v4f64:$fval, SETUGE)),
-          (SELECT_QFRC (CRORC  $lhs, $rhs), $tval, $fval)>;
-def : Pat<(v4f64 (selectcc i1:$lhs, i1:$rhs, v4f64:$tval, v4f64:$fval, SETGT)),
-          (SELECT_QFRC (CRANDC $rhs, $lhs), $tval, $fval)>;
-def : Pat<(v4f64 (selectcc i1:$lhs, i1:$rhs, v4f64:$tval, v4f64:$fval, SETUGT)),
-          (SELECT_QFRC (CRANDC $lhs, $rhs), $tval, $fval)>;
-def : Pat<(v4f64 (selectcc i1:$lhs, i1:$rhs, v4f64:$tval, v4f64:$fval, SETNE)),
-          (SELECT_QFRC (CRXOR $lhs, $rhs), $tval, $fval)>;
-
-def : Pat<(v4f32 (selectcc i1:$lhs, i1:$rhs, v4f32:$tval, v4f32:$fval, SETLT)),
-          (SELECT_QSRC (CRANDC $lhs, $rhs), $tval, $fval)>;
-def : Pat<(v4f32 (selectcc i1:$lhs, i1:$rhs, v4f32:$tval, v4f32:$fval, SETULT)),
-          (SELECT_QSRC (CRANDC $rhs, $lhs), $tval, $fval)>;
-def : Pat<(v4f32 (selectcc i1:$lhs, i1:$rhs, v4f32:$tval, v4f32:$fval, SETLE)),
-          (SELECT_QSRC (CRORC  $lhs, $rhs), $tval, $fval)>;
-def : Pat<(v4f32 (selectcc i1:$lhs, i1:$rhs, v4f32:$tval, v4f32:$fval, SETULE)),
-          (SELECT_QSRC (CRORC  $rhs, $lhs), $tval, $fval)>;
-def : Pat<(v4f32 (selectcc i1:$lhs, i1:$rhs, v4f32:$tval, v4f32:$fval, SETEQ)),
-          (SELECT_QSRC (CREQV $lhs, $rhs), $tval, $fval)>;
-def : Pat<(v4f32 (selectcc i1:$lhs, i1:$rhs, v4f32:$tval, v4f32:$fval, SETGE)),
-          (SELECT_QSRC (CRORC  $rhs, $lhs), $tval, $fval)>;
-def : Pat<(v4f32 (selectcc i1:$lhs, i1:$rhs, v4f32:$tval, v4f32:$fval, SETUGE)),
-          (SELECT_QSRC (CRORC  $lhs, $rhs), $tval, $fval)>;
-def : Pat<(v4f32 (selectcc i1:$lhs, i1:$rhs, v4f32:$tval, v4f32:$fval, SETGT)),
-          (SELECT_QSRC (CRANDC $rhs, $lhs), $tval, $fval)>;
-def : Pat<(v4f32 (selectcc i1:$lhs, i1:$rhs, v4f32:$tval, v4f32:$fval, SETUGT)),
-          (SELECT_QSRC (CRANDC $lhs, $rhs), $tval, $fval)>;
-def : Pat<(v4f32 (selectcc i1:$lhs, i1:$rhs, v4f32:$tval, v4f32:$fval, SETNE)),
-          (SELECT_QSRC (CRXOR $lhs, $rhs), $tval, $fval)>;
-
-def : Pat<(v4i1 (selectcc i1:$lhs, i1:$rhs, v4i1:$tval, v4i1:$fval, SETLT)),
-          (SELECT_QBRC (CRANDC $lhs, $rhs), $tval, $fval)>;
-def : Pat<(v4i1 (selectcc i1:$lhs, i1:$rhs, v4i1:$tval, v4i1:$fval, SETULT)),
-          (SELECT_QBRC (CRANDC $rhs, $lhs), $tval, $fval)>;
-def : Pat<(v4i1 (selectcc i1:$lhs, i1:$rhs, v4i1:$tval, v4i1:$fval, SETLE)),
-          (SELECT_QBRC (CRORC  $lhs, $rhs), $tval, $fval)>;
-def : Pat<(v4i1 (selectcc i1:$lhs, i1:$rhs, v4i1:$tval, v4i1:$fval, SETULE)),
-          (SELECT_QBRC (CRORC  $rhs, $lhs), $tval, $fval)>;
-def : Pat<(v4i1 (selectcc i1:$lhs, i1:$rhs, v4i1:$tval, v4i1:$fval, SETEQ)),
-          (SELECT_QBRC (CREQV $lhs, $rhs), $tval, $fval)>;
-def : Pat<(v4i1 (selectcc i1:$lhs, i1:$rhs, v4i1:$tval, v4i1:$fval, SETGE)),
-          (SELECT_QBRC (CRORC  $rhs, $lhs), $tval, $fval)>;
-def : Pat<(v4i1 (selectcc i1:$lhs, i1:$rhs, v4i1:$tval, v4i1:$fval, SETUGE)),
-          (SELECT_QBRC (CRORC  $lhs, $rhs), $tval, $fval)>;
-def : Pat<(v4i1 (selectcc i1:$lhs, i1:$rhs, v4i1:$tval, v4i1:$fval, SETGT)),
-          (SELECT_QBRC (CRANDC $rhs, $lhs), $tval, $fval)>;
-def : Pat<(v4i1 (selectcc i1:$lhs, i1:$rhs, v4i1:$tval, v4i1:$fval, SETUGT)),
-          (SELECT_QBRC (CRANDC $lhs, $rhs), $tval, $fval)>;
-def : Pat<(v4i1 (selectcc i1:$lhs, i1:$rhs, v4i1:$tval, v4i1:$fval, SETNE)),
-          (SELECT_QBRC (CRXOR $lhs, $rhs), $tval, $fval)>;
-
-} // end HasQPX
-
-let Predicates = [HasQPX, NoNaNsFPMath] in {
-def : Pat<(fminnum v4f64:$FRA, v4f64:$FRB),
-          (QVFSELb (QVFCMPLTb $FRA, $FRB), $FRB, $FRA)>;
-def : Pat<(fmaxnum v4f64:$FRA, v4f64:$FRB),
-          (QVFSELb (QVFCMPGTb $FRA, $FRB), $FRB, $FRA)>;
-
-def : Pat<(fminnum v4f32:$FRA, v4f32:$FRB),
-          (QVFSELbs (QVFCMPLTbs $FRA, $FRB), $FRB, $FRA)>;
-def : Pat<(fmaxnum v4f32:$FRA, v4f32:$FRB),
-          (QVFSELbs (QVFCMPGTbs $FRA, $FRB), $FRB, $FRA)>;
-}
-
-let Predicates = [HasQPX, NaNsFPMath] in {
-// When either of these operands is NaN, we should return the other operand.
-// QVFCMPLT/QVFCMPGT return false is either operand is NaN, which means we need
-// to explicitly or with a NaN test on the second operand.
-def : Pat<(fminnum v4f64:$FRA, v4f64:$FRB),
-          (QVFSELb (QVFLOGICALb (QVFCMPLTb $FRA, $FRB),
-                                (QVFTSTNANb $FRB, $FRB), (i32 7)),
-                   $FRB, $FRA)>;
-def : Pat<(fmaxnum v4f64:$FRA, v4f64:$FRB),
-          (QVFSELb (QVFLOGICALb (QVFCMPGTb $FRA, $FRB),
-                                (QVFTSTNANb $FRB, $FRB), (i32 7)),
-                   $FRB, $FRA)>;
-
-def : Pat<(fminnum v4f32:$FRA, v4f32:$FRB),
-          (QVFSELbs (QVFLOGICALb (QVFCMPLTbs $FRA, $FRB),
-                                 (QVFTSTNANbs $FRB, $FRB), (i32 7)),
-                   $FRB, $FRA)>;
-def : Pat<(fmaxnum v4f32:$FRA, v4f32:$FRB),
-          (QVFSELbs (QVFLOGICALb (QVFCMPGTbs $FRA, $FRB),
-                                 (QVFTSTNANbs $FRB, $FRB), (i32 7)),
-                   $FRB, $FRA)>;
-}
diff --git a/llvm/lib/Target/PowerPC/PPCQPXLoadSplat.cpp b/llvm/lib/Target/PowerPC/PPCQPXLoadSplat.cpp
deleted file mode 100644
index 6e90426438208..0000000000000
--- a/llvm/lib/Target/PowerPC/PPCQPXLoadSplat.cpp
+++ /dev/null
@@ -1,161 +0,0 @@
-//===----- PPCQPXLoadSplat.cpp - QPX Load Splat Simplification ------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// The QPX vector registers overlay the scalar floating-point registers, and
-// any scalar floating-point loads splat their value across all vector lanes.
-// Thus, if we have a scalar load followed by a splat, we can remove the splat
-// (i.e. replace the load with a load-and-splat pseudo instruction).
-//
-// This pass must run after anything that might do store-to-load forwarding.
-//
-//===----------------------------------------------------------------------===//
-
-#include "PPC.h"
-#include "PPCInstrBuilder.h"
-#include "PPCInstrInfo.h"
-#include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/Statistic.h"
-#include "llvm/CodeGen/MachineFunctionPass.h"
-#include "llvm/CodeGen/TargetSubtargetInfo.h"
-#include "llvm/Support/MathExtras.h"
-#include "llvm/Target/TargetMachine.h"
-using namespace llvm;
-
-#define DEBUG_TYPE "ppc-qpx-load-splat"
-
-STATISTIC(NumSimplified, "Number of QPX load splats simplified");
-
-namespace {
-  struct PPCQPXLoadSplat : public MachineFunctionPass {
-    static char ID;
-    PPCQPXLoadSplat() : MachineFunctionPass(ID) {
-      initializePPCQPXLoadSplatPass(*PassRegistry::getPassRegistry());
-    }
-
-    bool runOnMachineFunction(MachineFunction &Fn) override;
-
-    StringRef getPassName() const override {
-      return "PowerPC QPX Load Splat Simplification";
-    }
-  };
-  char PPCQPXLoadSplat::ID = 0;
-}
-
-INITIALIZE_PASS(PPCQPXLoadSplat, "ppc-qpx-load-splat",
-                "PowerPC QPX Load Splat Simplification",
-                false, false)
-
-FunctionPass *llvm::createPPCQPXLoadSplatPass() {
-  return new PPCQPXLoadSplat();
-}
-
-bool PPCQPXLoadSplat::runOnMachineFunction(MachineFunction &MF) {
-  if (skipFunction(MF.getFunction()))
-    return false;
-
-  bool MadeChange = false;
-  const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
-
-  for (auto MFI = MF.begin(), MFIE = MF.end(); MFI != MFIE; ++MFI) {
-    MachineBasicBlock *MBB = &*MFI;
-    SmallVector<MachineInstr *, 4> Splats;
-
-    for (auto MBBI = MBB->rbegin(); MBBI != MBB->rend(); ++MBBI) {
-      MachineInstr *MI = &*MBBI;
-
-      if (MI->hasUnmodeledSideEffects() || MI->isCall()) {
-        Splats.clear();
-        continue;
-      }
-
-      // We're looking for a sequence like this:
-      // %f0 = LFD 0, killed %x3, implicit-def %qf0; mem:LD8[%a](tbaa=!2)
-      // %qf1 = QVESPLATI killed %qf0, 0, implicit %rm
-
-      for (auto SI = Splats.begin(); SI != Splats.end();) {
-        MachineInstr *SMI = *SI;
-        Register SplatReg = SMI->getOperand(0).getReg();
-        Register SrcReg = SMI->getOperand(1).getReg();
-
-        if (MI->modifiesRegister(SrcReg, TRI)) {
-          switch (MI->getOpcode()) {
-          default:
-            SI = Splats.erase(SI);
-            continue;
-          case PPC::LFS:
-          case PPC::LFD:
-          case PPC::LFSU:
-          case PPC::LFDU:
-          case PPC::LFSUX:
-          case PPC::LFDUX:
-          case PPC::LFSX:
-          case PPC::LFDX:
-          case PPC::LFIWAX:
-          case PPC::LFIWZX:
-            if (SplatReg != SrcReg) {
-              // We need to change the load to define the scalar subregister of
-              // the QPX splat source register.
-              unsigned SubRegIndex =
-                TRI->getSubRegIndex(SrcReg, MI->getOperand(0).getReg());
-              Register SplatSubReg = TRI->getSubReg(SplatReg, SubRegIndex);
-
-              // Substitute both the explicit defined register, and also the
-              // implicit def of the containing QPX register.
-              MI->getOperand(0).setReg(SplatSubReg);
-              MI->substituteRegister(SrcReg, SplatReg, 0, *TRI);
-            }
-
-            SI = Splats.erase(SI);
-
-            // If SMI is directly after MI, then MBBI's base iterator is
-            // pointing at SMI.  Adjust MBBI around the call to erase SMI to
-            // avoid invalidating MBBI.
-            ++MBBI;
-            SMI->eraseFromParent();
-            --MBBI;
-
-            ++NumSimplified;
-            MadeChange = true;
-            continue;
-          }
-        }
-
-        // If this instruction defines the splat register, then we cannot move
-        // the previous definition above it. If it reads from the splat
-        // register, then it must already be alive from some previous
-        // definition, and if the splat register is different from the source
-        // register, then this definition must not be the load for which we're
-        // searching.
-        if (MI->modifiesRegister(SplatReg, TRI) ||
-            (SrcReg != SplatReg &&
-             MI->readsRegister(SplatReg, TRI))) {
-          SI = Splats.erase(SI);
-          continue;
-        }
-
-        ++SI;
-      }
-
-      if (MI->getOpcode() != PPC::QVESPLATI &&
-          MI->getOpcode() != PPC::QVESPLATIs &&
-          MI->getOpcode() != PPC::QVESPLATIb)
-        continue;
-      if (MI->getOperand(2).getImm() != 0)
-        continue;
-
-      // If there are other uses of the scalar value after this, replacing
-      // those uses might be non-trivial.
-      if (!MI->getOperand(1).isKill())
-        continue;
-
-      Splats.push_back(MI);
-    }
-  }
-
-  return MadeChange;
-}
diff --git a/llvm/lib/Target/PowerPC/PPCRegisterInfo.cpp b/llvm/lib/Target/PowerPC/PPCRegisterInfo.cpp
index ed8948a639728..96666ad58dfe5 100644
--- a/llvm/lib/Target/PowerPC/PPCRegisterInfo.cpp
+++ b/llvm/lib/Target/PowerPC/PPCRegisterInfo.cpp
@@ -404,9 +404,6 @@ unsigned PPCRegisterInfo::getRegPressureLimit(const TargetRegisterClass *RC,
   }
   case PPC::F8RCRegClassID:
   case PPC::F4RCRegClassID:
-  case PPC::QFRCRegClassID:
-  case PPC::QSRCRegClassID:
-  case PPC::QBRCRegClassID:
   case PPC::VRRCRegClassID:
   case PPC::VFRCRegClassID:
   case PPC::VSLRCRegClassID:
diff --git a/llvm/lib/Target/PowerPC/PPCRegisterInfo.h b/llvm/lib/Target/PowerPC/PPCRegisterInfo.h
index 61acd955e1cba..a931967862c7b 100644
--- a/llvm/lib/Target/PowerPC/PPCRegisterInfo.h
+++ b/llvm/lib/Target/PowerPC/PPCRegisterInfo.h
@@ -153,7 +153,6 @@ class PPCRegisterInfo : public PPCGenRegisterInfo {
     switch (RegName[0]) {
       case 'r':
       case 'f':
-      case 'q': // for QPX
       case 'v':
         if (RegName[1] == 's')
           return RegName + 2;
diff --git a/llvm/lib/Target/PowerPC/PPCRegisterInfo.td b/llvm/lib/Target/PowerPC/PPCRegisterInfo.td
index b45757c1acc5e..e07b960ae305b 100644
--- a/llvm/lib/Target/PowerPC/PPCRegisterInfo.td
+++ b/llvm/lib/Target/PowerPC/PPCRegisterInfo.td
@@ -54,13 +54,6 @@ class FPR<bits<5> num, string n> : PPCReg<n> {
   let HWEncoding{4-0} = num;
 }
 
-// QFPR - One of the 32 256-bit floating-point vector registers (used for QPX)
-class QFPR<FPR SubReg, string n> : PPCReg<n> {
-  let HWEncoding = SubReg.HWEncoding;
-  let SubRegs = [SubReg];
-  let SubRegIndices = [sub_64];
-}
-
 // VF - One of the 32 64-bit floating-point subregisters of the vector
 // registers (used by VSX).
 class VF<bits<5> num, string n> : PPCReg<n> {
@@ -132,12 +125,6 @@ foreach Index = 0-31 in {
                  DwarfRegNum<[!add(Index, 77), !add(Index, 77)]>;
 }
 
-// QPX Floating-point registers
-foreach Index = 0-31 in {
-  def QF#Index : QFPR<!cast<FPR>("F"#Index), "q"#Index>,
-                 DwarfRegNum<[!add(Index, 32), !add(Index, 32)]>;
-}
-
 // Vector registers
 foreach Index = 0-31 in {
   def V#Index : VR<!cast<VF>("VF"#Index), "v"#Index>,
@@ -343,16 +330,6 @@ def SPILLTOVSRRC : RegisterClass<"PPC", [i64, f64], 64, (add G8RC, (sub VSFRC,
 // Register class for single precision scalars in VSX registers
 def VSSRC : RegisterClass<"PPC", [f32], 32, (add VSFRC)>;
 
-// For QPX
-def QFRC : RegisterClass<"PPC", [v4f64], 256, (add (sequence "QF%u", 0, 13),
-                                                (sequence "QF%u", 31, 14))>;
-def QSRC : RegisterClass<"PPC", [v4f32], 128, (add QFRC)>;
-def QBRC : RegisterClass<"PPC", [v4i1], 256, (add QFRC)> {
-  // These are actually stored as floating-point values where a positive
-  // number is true and anything else (including NaN) is false.
-  let Size = 256;
-}
-
 def CRBITRC : RegisterClass<"PPC", [i1], 32,
   (add CR2LT, CR2GT, CR2EQ, CR2UN,
        CR3LT, CR3GT, CR3EQ, CR3UN,
diff --git a/llvm/lib/Target/PowerPC/PPCScheduleP9.td b/llvm/lib/Target/PowerPC/PPCScheduleP9.td
index 0a1ae7e55b3c2..311d5cf165f63 100644
--- a/llvm/lib/Target/PowerPC/PPCScheduleP9.td
+++ b/llvm/lib/Target/PowerPC/PPCScheduleP9.td
@@ -40,12 +40,9 @@ def P9Model : SchedMachineModel {
 
   let CompleteModel = 1;
 
-  // Do not support QPX (Quad Processing eXtension), SPE (Signal Processing
-  // Engine), prefixed instructions on Power 9, PC relative mem ops, or
-  // instructions introduced in ISA 3.1.
-  let UnsupportedFeatures = [HasQPX, HasSPE, PrefixInstrs, PCRelativeMemops,
-                             IsISA3_1];
-
+  // Do not support SPE (Signal Processing Engine), prefixed instructions on
+  // Power 9, PC relative mem ops, or instructions introduced in ISA 3.1.
+  let UnsupportedFeatures = [HasSPE, PrefixInstrs, PCRelativeMemops, IsISA3_1];
 }
 
 let SchedModel = P9Model in {
diff --git a/llvm/lib/Target/PowerPC/PPCSubtarget.cpp b/llvm/lib/Target/PowerPC/PPCSubtarget.cpp
index 3836cc960394f..85d2966654970 100644
--- a/llvm/lib/Target/PowerPC/PPCSubtarget.cpp
+++ b/llvm/lib/Target/PowerPC/PPCSubtarget.cpp
@@ -35,10 +35,6 @@ using namespace llvm;
 static cl::opt<bool> UseSubRegLiveness("ppc-track-subreg-liveness",
 cl::desc("Enable subregister liveness tracking for PPC"), cl::Hidden);
 
-static cl::opt<bool> QPXStackUnaligned("qpx-stack-unaligned",
-  cl::desc("Even when QPX is enabled the stack is not 32-byte aligned"),
-  cl::Hidden);
-
 static cl::opt<bool>
     EnableMachinePipeliner("ppc-enable-pipeliner",
                            cl::desc("Enable Machine Pipeliner for PPC"),
@@ -70,7 +66,6 @@ void PPCSubtarget::initializeEnvironment() {
   HasAltivec = false;
   HasSPE = false;
   HasFPU = false;
-  HasQPX = false;
   HasVSX = false;
   NeedsTwoConstNR = false;
   HasP8Vector = false;
@@ -109,7 +104,6 @@ void PPCSubtarget::initializeEnvironment() {
   HasInvariantFunctionDescriptors = false;
   HasPartwordAtomics = false;
   HasDirectMove = false;
-  IsQPXStackUnaligned = false;
   HasHTM = false;
   HasFloat128 = false;
   HasFusion = false;
@@ -158,7 +152,7 @@ void PPCSubtarget::initSubtargetFeatures(StringRef CPU, StringRef FS) {
 
   if (HasSPE && IsPPC64)
     report_fatal_error( "SPE is only supported for 32-bit targets.\n", false);
-  if (HasSPE && (HasAltivec || HasQPX || HasVSX || HasFPU))
+  if (HasSPE && (HasAltivec || HasVSX || HasFPU))
     report_fatal_error(
         "SPE and traditional floating point cannot both be enabled.\n", false);
 
@@ -166,10 +160,6 @@ void PPCSubtarget::initSubtargetFeatures(StringRef CPU, StringRef FS) {
   if (!HasSPE)
     HasFPU = true;
 
-  // QPX requires a 32-byte aligned stack. Note that we need to do this if
-  // we're compiling for a BG/Q system regardless of whether or not QPX
-  // is enabled because external functions will assume this alignment.
-  IsQPXStackUnaligned = QPXStackUnaligned;
   StackAlignment = getPlatformStackAlignment();
 
   // Determine endianness.
diff --git a/llvm/lib/Target/PowerPC/PPCSubtarget.h b/llvm/lib/Target/PowerPC/PPCSubtarget.h
index ec329022c4572..8a4041518e3c2 100644
--- a/llvm/lib/Target/PowerPC/PPCSubtarget.h
+++ b/llvm/lib/Target/PowerPC/PPCSubtarget.h
@@ -97,7 +97,6 @@ class PPCSubtarget : public PPCGenSubtargetInfo {
   bool HasAltivec;
   bool HasFPU;
   bool HasSPE;
-  bool HasQPX;
   bool HasVSX;
   bool NeedsTwoConstNR;
   bool HasP8Vector;
@@ -150,11 +149,6 @@ class PPCSubtarget : public PPCGenSubtargetInfo {
 
   POPCNTDKind HasPOPCNTD;
 
-  /// When targeting QPX running a stock PPC64 Linux kernel where the stack
-  /// alignment has not been changed, we need to keep the 16-byte alignment
-  /// of the stack.
-  bool IsQPXStackUnaligned;
-
   const PPCTargetMachine &TM;
   PPCFrameLowering FrameLowering;
   PPCInstrInfo InstrInfo;
@@ -255,7 +249,6 @@ class PPCSubtarget : public PPCGenSubtargetInfo {
   bool hasAltivec() const { return HasAltivec; }
   bool hasSPE() const { return HasSPE; }
   bool hasFPU() const { return HasFPU; }
-  bool hasQPX() const { return HasQPX; }
   bool hasVSX() const { return HasVSX; }
   bool needsTwoConstNR() const { return NeedsTwoConstNR; }
   bool hasP8Vector() const { return HasP8Vector; }
@@ -291,11 +284,7 @@ class PPCSubtarget : public PPCGenSubtargetInfo {
   bool hasPartwordAtomics() const { return HasPartwordAtomics; }
   bool hasDirectMove() const { return HasDirectMove; }
 
-  bool isQPXStackUnaligned() const { return IsQPXStackUnaligned; }
   Align getPlatformStackAlignment() const {
-    if ((hasQPX() || isBGQ()) && !isQPXStackUnaligned())
-      return Align(32);
-
     return Align(16);
   }
 
@@ -325,9 +314,6 @@ class PPCSubtarget : public PPCGenSubtargetInfo {
 
   const Triple &getTargetTriple() const { return TargetTriple; }
 
-  /// isBGQ - True if this is a BG/Q platform.
-  bool isBGQ() const { return TargetTriple.getVendor() == Triple::BGQ; }
-
   bool isTargetELF() const { return TargetTriple.isOSBinFormatELF(); }
   bool isTargetMachO() const { return TargetTriple.isOSBinFormatMachO(); }
   bool isTargetLinux() const { return TargetTriple.isOSLinux(); }
diff --git a/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp b/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp
index f15f9c7f49429..27de5b29cd341 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp
+++ b/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp
@@ -63,10 +63,6 @@ static cl::
 opt<bool> DisableVSXSwapRemoval("disable-ppc-vsx-swap-removal", cl::Hidden,
                                 cl::desc("Disable VSX Swap Removal for PPC"));
 
-static cl::
-opt<bool> DisableQPXLoadSplat("disable-ppc-qpx-load-splat", cl::Hidden,
-                              cl::desc("Disable QPX load splat simplification"));
-
 static cl::
 opt<bool> DisableMIPeephole("disable-ppc-peephole", cl::Hidden,
                             cl::desc("Disable machine peepholes for PPC"));
@@ -114,7 +110,6 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializePowerPCTarget() {
   initializePPCReduceCRLogicalsPass(PR);
   initializePPCBSelPass(PR);
   initializePPCBranchCoalescingPass(PR);
-  initializePPCQPXLoadSplatPass(PR);
   initializePPCBoolRetToIntPass(PR);
   initializePPCExpandISELPass(PR);
   initializePPCPreEmitPeepholePass(PR);
@@ -411,14 +406,9 @@ void PPCPassConfig::addIRPasses() {
 
   // Lower generic MASSV routines to PowerPC subtarget-specific entries.
   addPass(createPPCLowerMASSVEntriesPass());
-  
-  // For the BG/Q (or if explicitly requested), add explicit data prefetch
-  // intrinsics.
-  bool UsePrefetching = TM->getTargetTriple().getVendor() == Triple::BGQ &&
-                        getOptLevel() != CodeGenOpt::None;
+
+  // If explicitly requested, add explicit data prefetch intrinsics.
   if (EnablePrefetch.getNumOccurrences() > 0)
-    UsePrefetching = EnablePrefetch;
-  if (UsePrefetching)
     addPass(createLoopDataPrefetchPass());
 
   if (TM->getOptLevel() >= CodeGenOpt::Default && EnableGEPOpt) {
@@ -515,15 +505,8 @@ void PPCPassConfig::addPreRegAlloc() {
 }
 
 void PPCPassConfig::addPreSched2() {
-  if (getOptLevel() != CodeGenOpt::None) {
+  if (getOptLevel() != CodeGenOpt::None)
     addPass(&IfConverterID);
-
-    // This optimization must happen after anything that might do store-to-load
-    // forwarding. Here we're after RA (and, thus, when spills are inserted)
-    // but before post-RA scheduling.
-    if (!DisableQPXLoadSplat)
-      addPass(createPPCQPXLoadSplatPass());
-  }
 }
 
 void PPCPassConfig::addPreEmitPass() {
diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
index bbb4239d36da5..ee8842f4d8663 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
+++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
@@ -25,8 +25,7 @@ using namespace llvm;
 static cl::opt<bool> DisablePPCConstHoist("disable-ppc-constant-hoisting",
 cl::desc("disable constant hoisting on PPC"), cl::init(false), cl::Hidden);
 
-// This is currently only used for the data prefetch pass which is only enabled
-// for BG/Q by default.
+// This is currently only used for the data prefetch pass
 static cl::opt<unsigned>
 CacheLineSize("ppc-loop-prefetch-cache-line", cl::Hidden, cl::init(64),
               cl::desc("The loop prefetch cache line size"));
@@ -104,55 +103,6 @@ PPCTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
     Value *Ptr = IC.Builder.CreateBitCast(II.getArgOperand(1), OpPtrTy);
     return new StoreInst(II.getArgOperand(0), Ptr, false, Align(1));
   }
-  case Intrinsic::ppc_qpx_qvlfs:
-    // Turn PPC QPX qvlfs -> load if the pointer is known aligned.
-    if (getOrEnforceKnownAlignment(
-            II.getArgOperand(0), Align(16), IC.getDataLayout(), &II,
-            &IC.getAssumptionCache(), &IC.getDominatorTree()) >= 16) {
-      Type *VTy =
-          VectorType::get(IC.Builder.getFloatTy(),
-                          cast<VectorType>(II.getType())->getElementCount());
-      Value *Ptr = IC.Builder.CreateBitCast(II.getArgOperand(0),
-                                            PointerType::getUnqual(VTy));
-      Value *Load = IC.Builder.CreateLoad(VTy, Ptr);
-      return new FPExtInst(Load, II.getType());
-    }
-    break;
-  case Intrinsic::ppc_qpx_qvlfd:
-    // Turn PPC QPX qvlfd -> load if the pointer is known aligned.
-    if (getOrEnforceKnownAlignment(
-            II.getArgOperand(0), Align(32), IC.getDataLayout(), &II,
-            &IC.getAssumptionCache(), &IC.getDominatorTree()) >= 32) {
-      Value *Ptr = IC.Builder.CreateBitCast(
-          II.getArgOperand(0), PointerType::getUnqual(II.getType()));
-      return new LoadInst(II.getType(), Ptr, "", false, Align(32));
-    }
-    break;
-  case Intrinsic::ppc_qpx_qvstfs:
-    // Turn PPC QPX qvstfs -> store if the pointer is known aligned.
-    if (getOrEnforceKnownAlignment(
-            II.getArgOperand(1), Align(16), IC.getDataLayout(), &II,
-            &IC.getAssumptionCache(), &IC.getDominatorTree()) >= 16) {
-      Type *VTy = VectorType::get(
-          IC.Builder.getFloatTy(),
-          cast<VectorType>(II.getArgOperand(0)->getType())->getElementCount());
-      Value *TOp = IC.Builder.CreateFPTrunc(II.getArgOperand(0), VTy);
-      Type *OpPtrTy = PointerType::getUnqual(VTy);
-      Value *Ptr = IC.Builder.CreateBitCast(II.getArgOperand(1), OpPtrTy);
-      return new StoreInst(TOp, Ptr, false, Align(16));
-    }
-    break;
-  case Intrinsic::ppc_qpx_qvstfd:
-    // Turn PPC QPX qvstfd -> store if the pointer is known aligned.
-    if (getOrEnforceKnownAlignment(
-            II.getArgOperand(1), Align(32), IC.getDataLayout(), &II,
-            &IC.getAssumptionCache(), &IC.getDominatorTree()) >= 32) {
-      Type *OpPtrTy = PointerType::getUnqual(II.getArgOperand(0)->getType());
-      Value *Ptr = IC.Builder.CreateBitCast(II.getArgOperand(1), OpPtrTy);
-      return new StoreInst(II.getArgOperand(0), Ptr, false, Align(32));
-    }
-    break;
-
   case Intrinsic::ppc_altivec_vperm:
     // Turn vperm(V1,V2,mask) -> shuffle(V1,V2,mask) if mask is a constant.
     // Note that ppc_altivec_vperm has a big-endian bias, so when creating
@@ -736,10 +686,7 @@ bool PPCTTIImpl::useColdCCForColdCall(Function &F) {
 }
 
 bool PPCTTIImpl::enableAggressiveInterleaving(bool LoopHasReductions) {
-  // On the A2, always unroll aggressively. For QPX unaligned loads, we depend
-  // on combining the loads generated for consecutive accesses, and failure to
-  // do so is particularly expensive. This makes it much more likely (compared
-  // to only using concatenation unrolling).
+  // On the A2, always unroll aggressively.
   if (ST->getCPUDirective() == PPC::DIR_A2)
     return true;
 
@@ -799,7 +746,6 @@ const char* PPCTTIImpl::getRegisterClassName(unsigned ClassID) const {
 
 unsigned PPCTTIImpl::getRegisterBitWidth(bool Vector) const {
   if (Vector) {
-    if (ST->hasQPX()) return 256;
     if (ST->hasAltivec()) return 128;
     return 0;
   }
@@ -828,8 +774,6 @@ unsigned PPCTTIImpl::getCacheLineSize() const {
 }
 
 unsigned PPCTTIImpl::getPrefetchDistance() const {
-  // This seems like a reasonable default for the BG/Q (this pass is enabled, by
-  // default, only on the BG/Q).
   return 300;
 }
 
@@ -918,7 +862,7 @@ int PPCTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, Type *Tp, int Index,
   // Legalize the type.
   std::pair<int, MVT> LT = TLI->getTypeLegalizationCost(DL, Tp);
 
-  // PPC, for both Altivec/VSX and QPX, support cheap arbitrary permutations
+  // PPC, for both Altivec/VSX, support cheap arbitrary permutations
   // (at least in the sense that there need only be one non-loop-invariant
   // instruction). We need one such shuffle instruction for each actual
   // register (this is not true for arbitrary shuffles, but is true for the
@@ -974,13 +918,6 @@ int PPCTTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val, unsigned Index) {
 
     return Cost;
 
-  } else if (ST->hasQPX() && Val->getScalarType()->isFloatingPointTy()) {
-    // Floating point scalars are already located in index #0.
-    if (Index == 0)
-      return 0;
-
-    return Cost;
-
   } else if (Val->getScalarType()->isIntegerTy() && Index != -1U) {
     if (ST->hasP9Altivec()) {
       if (ISD == ISD::INSERT_VECTOR_ELT)
@@ -1055,8 +992,6 @@ int PPCTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
                         LT.second == MVT::v4i32 || LT.second == MVT::v4f32);
   bool IsVSXType = ST->hasVSX() &&
                    (LT.second == MVT::v2f64 || LT.second == MVT::v2i64);
-  bool IsQPXType = ST->hasQPX() &&
-                   (LT.second == MVT::v4f64 || LT.second == MVT::v4f32);
 
   // VSX has 32b/64b load instructions. Legalization can handle loading of
   // 32b/64b to VSR correctly and cheaply. But BaseT::getMemoryOpCost and
@@ -1079,8 +1014,7 @@ int PPCTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
   // for Altivec types using the VSX instructions, but that's more expensive
   // than using the permutation-based load sequence. On the P8, that's no
   // longer true.
-  if (Opcode == Instruction::Load &&
-      ((!ST->hasP8Vector() && IsAltivecType) || IsQPXType) &&
+  if (Opcode == Instruction::Load && (!ST->hasP8Vector() && IsAltivecType) &&
       *Alignment >= LT.second.getScalarType().getStoreSize())
     return Cost + LT.first; // Add the cost of the permutations.
 
@@ -1133,7 +1067,7 @@ int PPCTTIImpl::getInterleavedMemoryOpCost(
       getMemoryOpCost(Opcode, VecTy, MaybeAlign(Alignment), AddressSpace,
                       CostKind);
 
-  // PPC, for both Altivec/VSX and QPX, support cheap arbitrary permutations
+  // PPC, for both Altivec/VSX, support cheap arbitrary permutations
   // (at least in the sense that there need only be one non-loop-invariant
   // instruction). For each result vector, we need one shuffle per incoming
   // vector (except that the first shuffle can take two incoming vectors
diff --git a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
index 427abde4277d4..aa06e8144f634 100644
--- a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
@@ -4751,15 +4751,14 @@ struct VarArgPowerPC64Helper : public VarArgHelper {
     // For PowerPC, we need to deal with alignment of stack arguments -
     // they are mostly aligned to 8 bytes, but vectors and i128 arrays
     // are aligned to 16 bytes, byvals can be aligned to 8 or 16 bytes,
-    // and QPX vectors are aligned to 32 bytes.  For that reason, we
-    // compute current offset from stack pointer (which is always properly
-    // aligned), and offset for the first vararg, then subtract them.
+    // For that reason, we compute current offset from stack pointer (which is
+    // always properly aligned), and offset for the first vararg, then subtract
+    // them.
     unsigned VAArgBase;
     Triple TargetTriple(F.getParent()->getTargetTriple());
     // Parameter save area starts at 48 bytes from frame pointer for ABIv1,
     // and 32 bytes for ABIv2.  This is usually determined by target
     // endianness, but in theory could be overridden by function attribute.
-    // For simplicity, we ignore it here (it'd only matter for QPX vectors).
     if (TargetTriple.getArch() == Triple::ppc64)
       VAArgBase = 48;
     else
diff --git a/llvm/test/Analysis/BasicAA/phi-spec-order.ll b/llvm/test/Analysis/BasicAA/phi-spec-order.ll
index f8586f094c2ce..e5d435c09ccc7 100644
--- a/llvm/test/Analysis/BasicAA/phi-spec-order.ll
+++ b/llvm/test/Analysis/BasicAA/phi-spec-order.ll
@@ -1,5 +1,5 @@
 target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
-target triple = "powerpc64-bgq-linux"
+target triple = "powerpc64le-unknown-linux"
 ; RUN: opt < %s -basic-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 
 @X = external global [16000 x double], align 32
diff --git a/llvm/test/Analysis/CostModel/PowerPC/unal-vec-ldst.ll b/llvm/test/Analysis/CostModel/PowerPC/unal-vec-ldst.ll
index 3b1bc3b3fdbc0..e5fbf070cf32a 100644
--- a/llvm/test/Analysis/CostModel/PowerPC/unal-vec-ldst.ll
+++ b/llvm/test/Analysis/CostModel/PowerPC/unal-vec-ldst.ll
@@ -218,42 +218,6 @@ entry:
 ; CHECK: cost of 2 for instruction:   %r = load <4 x double>, <4 x double>* %p, align 8
 }
 
-define <4 x float> @test_l_qv4float(<4 x float>* %p) #1 {
-entry:
-  %r = load <4 x float>, <4 x float>* %p, align 4
-  ret <4 x float> %r
-
-; CHECK-LABEL: test_l_qv4float
-; CHECK: cost of 2 for instruction:   %r = load <4 x float>, <4 x float>* %p, align 4
-}
-
-define <8 x float> @test_l_qv8float(<8 x float>* %p) #1 {
-entry:
-  %r = load <8 x float>, <8 x float>* %p, align 4
-  ret <8 x float> %r
-
-; CHECK-LABEL: test_l_qv8float
-; CHECK: cost of 4 for instruction:   %r = load <8 x float>, <8 x float>* %p, align 4
-}
-
-define <4 x double> @test_l_qv4double(<4 x double>* %p) #1 {
-entry:
-  %r = load <4 x double>, <4 x double>* %p, align 8
-  ret <4 x double> %r
-
-; CHECK-LABEL: test_l_qv4double
-; CHECK: cost of 2 for instruction:   %r = load <4 x double>, <4 x double>* %p, align 8
-}
-
-define <8 x double> @test_l_qv8double(<8 x double>* %p) #1 {
-entry:
-  %r = load <8 x double>, <8 x double>* %p, align 8
-  ret <8 x double> %r
-
-; CHECK-LABEL: test_l_qv8double
-; CHECK: cost of 4 for instruction:   %r = load <8 x double>, <8 x double>* %p, align 8
-}
-
 define void @test_s_v16i8(<16 x i8>* %p, <16 x i8> %v) #0 {
 entry:
   store <16 x i8> %v, <16 x i8>* %p, align 1
@@ -362,43 +326,6 @@ entry:
 ; CHECK: cost of 2 for instruction:   store <4 x double> %v, <4 x double>* %p, align 8
 }
 
-define void @test_s_qv4float(<4 x float>* %p, <4 x float> %v) #1 {
-entry:
-  store <4 x float> %v, <4 x float>* %p, align 4
-  ret void
-
-; CHECK-LABEL: test_s_qv4float
-; CHECK: cost of 7 for instruction:   store <4 x float> %v, <4 x float>* %p, align 4
-}
-
-define void @test_s_qv8float(<8 x float>* %p, <8 x float> %v) #1 {
-entry:
-  store <8 x float> %v, <8 x float>* %p, align 4
-  ret void
-
-; CHECK-LABEL: test_s_qv8float
-; CHECK: cost of 15 for instruction:   store <8 x float> %v, <8 x float>* %p, align 4
-}
-
-define void @test_s_qv4double(<4 x double>* %p, <4 x double> %v) #1 {
-entry:
-  store <4 x double> %v, <4 x double>* %p, align 8
-  ret void
-
-; CHECK-LABEL: test_s_qv4double
-; CHECK: cost of 7 for instruction:   store <4 x double> %v, <4 x double>* %p, align 8
-}
-
-define void @test_s_qv8double(<8 x double>* %p, <8 x double> %v) #1 {
-entry:
-  store <8 x double> %v, <8 x double>* %p, align 8
-  ret void
-
-; CHECK-LABEL: test_s_qv8double
-; CHECK: cost of 15 for instruction:   store <8 x double> %v, <8 x double>* %p, align 8
-}
-
 attributes #0 = { nounwind "target-cpu"="pwr7" }
-attributes #1 = { nounwind "target-cpu"="a2q" }
 attributes #2 = { nounwind "target-cpu"="pwr8" }
 
diff --git a/llvm/test/CodeGen/PowerPC/2012-11-16-mischedcall.ll b/llvm/test/CodeGen/PowerPC/2012-11-16-mischedcall.ll
index 69f9cff5c525f..d93f192b1274d 100644
--- a/llvm/test/CodeGen/PowerPC/2012-11-16-mischedcall.ll
+++ b/llvm/test/CodeGen/PowerPC/2012-11-16-mischedcall.ll
@@ -1,4 +1,4 @@
-; RUN: llc -verify-machineinstrs -mtriple=powerpc64-bgq-linux -enable-misched < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux -enable-misched < %s | FileCheck %s
 ;
 ; PR14315: misched should not move the physreg copy of %t below the calls.
 
diff --git a/llvm/test/CodeGen/PowerPC/DisableHoistingDueToBlockHotnessProfileData.mir b/llvm/test/CodeGen/PowerPC/DisableHoistingDueToBlockHotnessProfileData.mir
index 738aa1df5dd9d..a0139879f8c91 100644
--- a/llvm/test/CodeGen/PowerPC/DisableHoistingDueToBlockHotnessProfileData.mir
+++ b/llvm/test/CodeGen/PowerPC/DisableHoistingDueToBlockHotnessProfileData.mir
@@ -55,7 +55,7 @@
   ; Function Attrs: nounwind
   declare void @llvm.stackprotector(i8*, i8**) #1
 
-  attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+  attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector" "unsafe-fp-math"="false" "use-soft-float"="false" }
   attributes #1 = { nounwind }
 
   !llvm.module.flags = !{!0, !1}
diff --git a/llvm/test/CodeGen/PowerPC/NoCRFieldRedefWhenSpillingCRBIT.mir b/llvm/test/CodeGen/PowerPC/NoCRFieldRedefWhenSpillingCRBIT.mir
index bcd51d31c6cfd..01ce79995512a 100644
--- a/llvm/test/CodeGen/PowerPC/NoCRFieldRedefWhenSpillingCRBIT.mir
+++ b/llvm/test/CodeGen/PowerPC/NoCRFieldRedefWhenSpillingCRBIT.mir
@@ -30,7 +30,7 @@
   ; Function Attrs: nounwind
   declare void @llvm.stackprotector(i8*, i8**) #1
   
-  attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+  attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector" "unsafe-fp-math"="false" "use-soft-float"="false" }
   attributes #1 = { nounwind }
   
   !llvm.ident = !{!0}
diff --git a/llvm/test/CodeGen/PowerPC/a2q-stackalign.ll b/llvm/test/CodeGen/PowerPC/a2q-stackalign.ll
deleted file mode 100644
index 17e3df6d58ccc..0000000000000
--- a/llvm/test/CodeGen/PowerPC/a2q-stackalign.ll
+++ /dev/null
@@ -1,23 +0,0 @@
-; RUN: llc -verify-machineinstrs < %s -mtriple=ppc64-- -mcpu=a2 | FileCheck -check-prefix=CHECK-A2 %s
-; RUN: llc -verify-machineinstrs < %s -mtriple=ppc64-- -mcpu=a2q | FileCheck -check-prefix=CHECK-A2Q %s
-; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc64-bgq-linux -mcpu=a2 | FileCheck -check-prefix=CHECK-BGQ %s
-target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
-target triple = "powerpc64-unknown-linux-gnu"
-
-declare i32 @bar(i8* %a) nounwind;
-define i32 @foo() nounwind {
-  %p = alloca i8, i8 115
-  store i8 0, i8* %p
-  %r = call i32 @bar(i8* %p)
-  ret i32 %r
-}
-
-; Without QPX, the allocated stack frame is 240 bytes, but with QPX
-; (because we require 32-byte alignment), it is 256 bytes.
-; CHECK-A2: @foo
-; CHECK-A2: stdu 1, -240(1)
-; CHECK-A2Q: @foo
-; CHECK-A2Q: stdu 1, -256(1)
-; CHECK-BGQ: @foo
-; CHECK-BGQ: stdu 1, -256(1)
-
diff --git a/llvm/test/CodeGen/PowerPC/a2q.ll b/llvm/test/CodeGen/PowerPC/a2q.ll
deleted file mode 100644
index 84e2dfa991d78..0000000000000
--- a/llvm/test/CodeGen/PowerPC/a2q.ll
+++ /dev/null
@@ -1,10 +0,0 @@
-; RUN: llc -verify-machineinstrs < %s -mtriple=ppc64-- -mcpu=a2q | FileCheck %s
-; RUN: llc -verify-machineinstrs < %s -mtriple=ppc64-- -mcpu=a2 -mattr=+qpx | FileCheck %s
-
-define void @foo() {
-entry:
-  ret void
-}
-
-; CHECK: @foo
-
diff --git a/llvm/test/CodeGen/PowerPC/aantidep-inline-asm-use.ll b/llvm/test/CodeGen/PowerPC/aantidep-inline-asm-use.ll
index 1b0ea26f1fdea..d629148535aa7 100644
--- a/llvm/test/CodeGen/PowerPC/aantidep-inline-asm-use.ll
+++ b/llvm/test/CodeGen/PowerPC/aantidep-inline-asm-use.ll
@@ -298,7 +298,7 @@ _ZN10SubProcess12SafeSyscalls5fcntlEiil.exit:     ; preds = %_ZN10SubProcess12Sa
 ; Function Attrs: nounwind argmemonly
 declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i1) #1
 
-attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pwr8" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pwr8" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind argmemonly }
 attributes #2 = { nounwind }
 
diff --git a/llvm/test/CodeGen/PowerPC/asm-Zy.ll b/llvm/test/CodeGen/PowerPC/asm-Zy.ll
index 78bb0f4c73eca..c8b5e9f1aa1d1 100644
--- a/llvm/test/CodeGen/PowerPC/asm-Zy.ll
+++ b/llvm/test/CodeGen/PowerPC/asm-Zy.ll
@@ -1,6 +1,5 @@
-; RUN: llc -verify-machineinstrs < %s -mcpu=a2 | FileCheck %s
+; RUN: llc -verify-machineinstrs < %s -mcpu=a2 -mtriple=powerpc64-unknown-linux-gnu | FileCheck %s
 target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
-target triple = "powerpc64-bgq-linux"
 
 define i32 @zytest(i32 %a) nounwind {
 entry:
diff --git a/llvm/test/CodeGen/PowerPC/asm-constraints.ll b/llvm/test/CodeGen/PowerPC/asm-constraints.ll
index a3e573d8935e9..da77d1a169792 100644
--- a/llvm/test/CodeGen/PowerPC/asm-constraints.ll
+++ b/llvm/test/CodeGen/PowerPC/asm-constraints.ll
@@ -65,7 +65,7 @@ entry:
 }
 
 
-attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 attributes #1 = { nounwind }
 
diff --git a/llvm/test/CodeGen/PowerPC/convert-rr-to-ri-instrs-R0-special-handling.mir b/llvm/test/CodeGen/PowerPC/convert-rr-to-ri-instrs-R0-special-handling.mir
index 2081e6fd02f51..904210ee13477 100644
--- a/llvm/test/CodeGen/PowerPC/convert-rr-to-ri-instrs-R0-special-handling.mir
+++ b/llvm/test/CodeGen/PowerPC/convert-rr-to-ri-instrs-R0-special-handling.mir
@@ -63,8 +63,8 @@
     ret i64 %2
   }
   
-  attributes #0 = { norecurse nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
-  attributes #1 = { norecurse nounwind readonly "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+  attributes #0 = { norecurse nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector" "unsafe-fp-math"="false" "use-soft-float"="false" }
+  attributes #1 = { norecurse nounwind readonly "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector" "unsafe-fp-math"="false" "use-soft-float"="false" }
   
   !llvm.module.flags = !{!0, !1}
   !llvm.ident = !{!2}
diff --git a/llvm/test/CodeGen/PowerPC/convert-rr-to-ri-instrs-out-of-range.mir b/llvm/test/CodeGen/PowerPC/convert-rr-to-ri-instrs-out-of-range.mir
index b52e0a4103add..f46d4fc0a42a4 100644
--- a/llvm/test/CodeGen/PowerPC/convert-rr-to-ri-instrs-out-of-range.mir
+++ b/llvm/test/CodeGen/PowerPC/convert-rr-to-ri-instrs-out-of-range.mir
@@ -187,7 +187,7 @@
     ret i64 %cond
   }
   
-  attributes #0 = { norecurse nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pwr9" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+power9-vector,+vsx,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+  attributes #0 = { norecurse nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pwr9" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+power9-vector,+vsx" "unsafe-fp-math"="false" "use-soft-float"="false" }
   
   !llvm.module.flags = !{!0, !1}
   !llvm.ident = !{!2}
diff --git a/llvm/test/CodeGen/PowerPC/convert-rr-to-ri-instrs.mir b/llvm/test/CodeGen/PowerPC/convert-rr-to-ri-instrs.mir
index 4d2595e1abdcb..ba950dc3d3ae9 100644
--- a/llvm/test/CodeGen/PowerPC/convert-rr-to-ri-instrs.mir
+++ b/llvm/test/CodeGen/PowerPC/convert-rr-to-ri-instrs.mir
@@ -983,10 +983,10 @@
     ret i64 %xor
   }
   
-  attributes #0 = { norecurse nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pwr9" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+power9-vector,+vsx,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
-  attributes #1 = { norecurse nounwind readonly "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pwr9" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+power9-vector,+vsx,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
-  attributes #2 = { norecurse nounwind readonly "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pwr9" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+power9-vector,-vsx,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
-  attributes #3 = { norecurse nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pwr9" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+power9-vector,+vsx,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+  attributes #0 = { norecurse nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pwr9" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+power9-vector,+vsx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+  attributes #1 = { norecurse nounwind readonly "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pwr9" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+power9-vector,+vsx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+  attributes #2 = { norecurse nounwind readonly "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pwr9" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+power9-vector,-vsx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+  attributes #3 = { norecurse nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pwr9" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+power9-vector,+vsx" "unsafe-fp-math"="false" "use-soft-float"="false" }
   
   !llvm.module.flags = !{!0, !1}
   !llvm.ident = !{!2}
diff --git a/llvm/test/CodeGen/PowerPC/ctr-minmaxnum.ll b/llvm/test/CodeGen/PowerPC/ctr-minmaxnum.ll
index ed3c9f07c1a85..75640d1d26072 100644
--- a/llvm/test/CodeGen/PowerPC/ctr-minmaxnum.ll
+++ b/llvm/test/CodeGen/PowerPC/ctr-minmaxnum.ll
@@ -1,5 +1,4 @@
 ; RUN: llc -mtriple=powerpc64-unknown-linux-gnu -verify-machineinstrs -mcpu=pwr7 < %s | FileCheck %s
-; RUN: llc -mtriple=powerpc64-unknown-linux-gnu -verify-machineinstrs -mcpu=a2q < %s | FileCheck %s --check-prefix=QPX
 
 declare float @fabsf(float)
 
@@ -64,11 +63,6 @@ loop_exit:
 ; CHECK-NOT: xsmindp
 ; CHECK: blr
 
-; QPX-LABEL: test1v:
-; QPX: mtctr
-; QPX-NOT: bl fminf
-; QPX: blr
-
 define void @test1a(float %f, float* %fp) {
 entry:
   br label %loop_body
@@ -139,11 +133,6 @@ loop_exit:
 ; CHECK-NOT: xsmaxdp
 ; CHECK: blr
 
-; QPX-LABEL: test2v:
-; QPX: mtctr
-; QPX-NOT: bl fmax
-; QPX: blr
-
 define void @test2a(float %f, float* %fp) {
 entry:
   br label %loop_body
diff --git a/llvm/test/CodeGen/PowerPC/ctrloop-shortLoops.ll b/llvm/test/CodeGen/PowerPC/ctrloop-shortLoops.ll
index 44acfcdd6e66a..636c86b815c8c 100644
--- a/llvm/test/CodeGen/PowerPC/ctrloop-shortLoops.ll
+++ b/llvm/test/CodeGen/PowerPC/ctrloop-shortLoops.ll
@@ -1,5 +1,4 @@
 ; RUN: llc < %s -mtriple=powerpc64le-unknown-linux-gnu -verify-machineinstrs -mcpu=pwr8 | FileCheck %s --check-prefixes=CHECK,CHECK-PWR8
-; RUN: llc < %s -mtriple=powerpc64le-unknown-linux-gnu -verify-machineinstrs -mcpu=a2q | FileCheck %s --check-prefixes=CHECK,CHECK-A2Q
 
 ; Verify that we do NOT generate the mtctr instruction for loop trip counts < 4
 ; The latency of the mtctr is only justified if there are more than 4 comparisons that are removed as a result.
@@ -86,11 +85,8 @@ for.body:                                         ; preds = %entry, %for.body
 }
 
 ; Function Attrs: norecurse nounwind
-; On core a2q, IssueWidth is 1. On core pwr8, IssueWidth is 8.
-; a2q should use mtctr, but pwr8 should not use mtctr.
 define signext i32 @testTripCount2NonSmallLoop() {
 ; CHECK-LABEL: testTripCount2NonSmallLoop:
-; CHECK-A2Q: mtctr
 ; CHECK-PWR8-NOT: mtctr
 ; CHECK: blr
 
@@ -121,12 +117,9 @@ for.end:                                          ; preds = %if.end
   ret i32 %conv
 }
 
-; On core a2q, IssueWidth is 1. On core pwr8, IssueWidth is 8.
-; a2q should use mtctr, but pwr8 should not use mtctr.
 define signext i32 @testTripCount5() {
 ; CHECK-LABEL: testTripCount5:
 ; CHECK-PWR8-NOT: mtctr
-; CHECK-A2Q: mtctr
  
 entry:
   %.prea = load i32, i32* @a, align 4
diff --git a/llvm/test/CodeGen/PowerPC/ec-input.ll b/llvm/test/CodeGen/PowerPC/ec-input.ll
index 9a1c121699a69..425bc1985d419 100644
--- a/llvm/test/CodeGen/PowerPC/ec-input.ll
+++ b/llvm/test/CodeGen/PowerPC/ec-input.ll
@@ -5,7 +5,7 @@
 ; that were both inputs to the inline asm and also early-clobber outputs).
 
 target datalayout = "E-m:e-i64:64-n32:64"
-target triple = "powerpc64-bgq-linux"
+target triple = "powerpc64le-unknown-linux"
 
 %struct._IO_FILE.119.8249.32639.195239.200117.211499.218003.221255.222881.224507.226133.240767.244019.245645.248897.260279.271661.281417.283043.302555.304181.325319.326945.344713 = type { i32, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, %struct._IO_marker.118.8248.32638.195238.200116.211498.218002.221254.222880.224506.226132.240766.244018.245644.248896.260278.271660.281416.283042.302554.304180.325318.326944.344712*, %struct._IO_FILE.119.8249.32639.195239.200117.211499.218003.221255.222881.224507.226133.240767.244019.245645.248897.260279.271661.281417.283043.302555.304181.325319.326945.344713*, i32, i32, i64, i16, i8, [1 x i8], i8*, i64, i8*, i8*, i8*, i8*, i64, i32, [20 x i8] }
 %struct._IO_marker.118.8248.32638.195238.200116.211498.218002.221254.222880.224506.226132.240766.244018.245644.248896.260278.271660.281416.283042.302554.304180.325318.326944.344712 = type { %struct._IO_marker.118.8248.32638.195238.200116.211498.218002.221254.222880.224506.226132.240766.244018.245644.248896.260278.271660.281416.283042.302554.304180.325318.326944.344712*, %struct._IO_FILE.119.8249.32639.195239.200117.211499.218003.221255.222881.224507.226133.240767.244019.245645.248897.260279.271661.281417.283043.302555.304181.325319.326945.344713*, i32 }
diff --git a/llvm/test/CodeGen/PowerPC/extra-toc-reg-deps.ll b/llvm/test/CodeGen/PowerPC/extra-toc-reg-deps.ll
index e066b45d3ca4b..023928bcb5896 100644
--- a/llvm/test/CodeGen/PowerPC/extra-toc-reg-deps.ll
+++ b/llvm/test/CodeGen/PowerPC/extra-toc-reg-deps.ll
@@ -1,6 +1,6 @@
-; RUN: llc -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mcpu=pwr8 < %s | FileCheck %s
 target datalayout = "E-m:e-i64:64-n32:64"
-target triple = "powerpc64-bgq-linux"
+target triple = "powerpc64-unknown-linux"
 
 %"class.Foam::messageStream.6" = type <{ %"class.Foam::string.5", i32, i32, i32, [4 x i8] }>
 %"class.Foam::string.5" = type { %"class.std::basic_string.4" }
@@ -419,8 +419,8 @@ declare void @_ZN4Foam11regIOobjectD2Ev() #0
 
 declare void @_ZN4Foam6reduceIiNS_5sumOpIiEEEEvRKNS_4ListINS_8UPstream11commsStructEEERT_RKT0_ii() #0
 
-attributes #0 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="a2q" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { inlinehint "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="a2q" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { inlinehint "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.module.flags = !{!0}
 
diff --git a/llvm/test/CodeGen/PowerPC/fast-isel-icmp-split.ll b/llvm/test/CodeGen/PowerPC/fast-isel-icmp-split.ll
index fdd0fc2767803..b08b050f2c2fd 100644
--- a/llvm/test/CodeGen/PowerPC/fast-isel-icmp-split.ll
+++ b/llvm/test/CodeGen/PowerPC/fast-isel-icmp-split.ll
@@ -1,6 +1,6 @@
 ; RUN: llc -verify-machineinstrs -O0 -relocation-model=pic < %s | FileCheck %s
 target datalayout = "E-m:e-i64:64-n32:64"
-target triple = "powerpc64-bgq-linux"
+target triple = "powerpc64le-unknown-linux"
 
 %"class.std::__1::__tree_node.130.151" = type { %"class.std::__1::__tree_node_base.base.128.149", %"class.boost::serialization::extended_type_info.129.150"* }
 %"class.std::__1::__tree_node_base.base.128.149" = type <{ %"class.std::__1::__tree_end_node.127.148", %"class.std::__1::__tree_node_base.126.147"*, %"class.std::__1::__tree_node_base.126.147"*, i8 }>
diff --git a/llvm/test/CodeGen/PowerPC/fma-mutate-duplicate-vreg.ll b/llvm/test/CodeGen/PowerPC/fma-mutate-duplicate-vreg.ll
index eef6e0ccac02b..a336fc796ca52 100644
--- a/llvm/test/CodeGen/PowerPC/fma-mutate-duplicate-vreg.ll
+++ b/llvm/test/CodeGen/PowerPC/fma-mutate-duplicate-vreg.ll
@@ -33,4 +33,4 @@ define float @f(float %xf) #0 {
   ret float %25
 }
 
-attributes #0 = { norecurse nounwind readnone "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pwr8" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse nounwind readnone "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pwr8" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/PowerPC/fp2int2fp-ppcfp128.ll b/llvm/test/CodeGen/PowerPC/fp2int2fp-ppcfp128.ll
index 2feb4556dfab7..3b555cf898f57 100644
--- a/llvm/test/CodeGen/PowerPC/fp2int2fp-ppcfp128.ll
+++ b/llvm/test/CodeGen/PowerPC/fp2int2fp-ppcfp128.ll
@@ -1,6 +1,5 @@
-; RUN: llc -verify-machineinstrs -mcpu=a2 < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mcpu=a2 -mtriple=powerpc64-unknown-linux-gnu < %s | FileCheck %s
 target datalayout = "E-m:e-i64:64-n32:64"
-target triple = "powerpc64-bgq-linux"
 
 define linkonce_odr double @test1(ppc_fp128 %input) {
 entry:
diff --git a/llvm/test/CodeGen/PowerPC/glob-comp-aa-crash.ll b/llvm/test/CodeGen/PowerPC/glob-comp-aa-crash.ll
index 54c3e11528b7b..2aa5239f25eb8 100644
--- a/llvm/test/CodeGen/PowerPC/glob-comp-aa-crash.ll
+++ b/llvm/test/CodeGen/PowerPC/glob-comp-aa-crash.ll
@@ -1,6 +1,6 @@
-; RUN: llc -verify-machineinstrs -mtriple=powerpc64-bgq-linux -mcpu=a2 < %s
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux -mcpu=a2 < %s
 target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
-target triple = "powerpc64-bgq-linux"
+target triple = "powerpc64le-unknown-linux"
 
 %"class.std::__1::__assoc_sub_state" = type { %"class.std::__1::__shared_count", %"class.std::__exception_ptr::exception_ptr", %"class.std::__1::mutex", %"class.std::__1::condition_variable", i32 }
 %"class.std::__1::__shared_count" = type { i32 (...)**, i64 }
diff --git a/llvm/test/CodeGen/PowerPC/ifcvt-forked-bug-2016-08-08.ll b/llvm/test/CodeGen/PowerPC/ifcvt-forked-bug-2016-08-08.ll
index 74bfa75e5e313..a2d0eb599f91d 100644
--- a/llvm/test/CodeGen/PowerPC/ifcvt-forked-bug-2016-08-08.ll
+++ b/llvm/test/CodeGen/PowerPC/ifcvt-forked-bug-2016-08-08.ll
@@ -33,5 +33,5 @@ declare i8* @_ZN11__sanitizer21internal_start_threadEPFvPvES0_(void (i8*)*, i8*)
 
 declare hidden void @_ZN11__sanitizer16BackgroundThreadEPv(i8* nocapture readnone) #5
 
-attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #7 = { nobuiltin nounwind }
diff --git a/llvm/test/CodeGen/PowerPC/inlineasm-i64-reg.ll b/llvm/test/CodeGen/PowerPC/inlineasm-i64-reg.ll
index e4dfd6c58f0e8..6f1bc76d816ae 100644
--- a/llvm/test/CodeGen/PowerPC/inlineasm-i64-reg.ll
+++ b/llvm/test/CodeGen/PowerPC/inlineasm-i64-reg.ll
@@ -1,6 +1,6 @@
-; RUN: llc -verify-machineinstrs -mtriple=powerpc64-bgq-linux -mcpu=a2 < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux -mcpu=a2 < %s | FileCheck %s
 target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
-target triple = "powerpc64-bgq-linux"
+target triple = "powerpc64le-unknown-linux"
 
 %struct.BG_CoordinateMapping_t = type { [4 x i8] }
 
diff --git a/llvm/test/CodeGen/PowerPC/load-two-flts.ll b/llvm/test/CodeGen/PowerPC/load-two-flts.ll
index 1cfcff5e01601..19e21faf47232 100644
--- a/llvm/test/CodeGen/PowerPC/load-two-flts.ll
+++ b/llvm/test/CodeGen/PowerPC/load-two-flts.ll
@@ -1,6 +1,5 @@
-; RUN: llc -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu < %s | FileCheck %s
 target datalayout = "E-m:e-i64:64-n32:64"
-target triple = "powerpc64-bgq-linux"
 
 define void @_Z4testSt7complexIfE(float %v0, float %v1, i64* %ref.tmp, float* %_M_value.realp.i.i, float* %_M_value.imagp.i.i) {
 entry:
diff --git a/llvm/test/CodeGen/PowerPC/loop-data-prefetch-inner.ll b/llvm/test/CodeGen/PowerPC/loop-data-prefetch-inner.ll
index f4664788930d4..2cbb70bb14cb5 100644
--- a/llvm/test/CodeGen/PowerPC/loop-data-prefetch-inner.ll
+++ b/llvm/test/CodeGen/PowerPC/loop-data-prefetch-inner.ll
@@ -1,6 +1,6 @@
-; RUN: llc -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -enable-ppc-prefetching=true -verify-machineinstrs < %s | FileCheck %s
 target datalayout = "E-m:e-i64:64-n32:64"
-target triple = "powerpc64-bgq-linux"
+target triple = "powerpc64le-unknown-linux"
 
 ; Function Attrs: nounwind
 define void @foo(double* %x, double* nocapture readonly %y) #0 {
diff --git a/llvm/test/CodeGen/PowerPC/loop-data-prefetch.ll b/llvm/test/CodeGen/PowerPC/loop-data-prefetch.ll
index f4821564c202b..defc52eec8e0d 100644
--- a/llvm/test/CodeGen/PowerPC/loop-data-prefetch.ll
+++ b/llvm/test/CodeGen/PowerPC/loop-data-prefetch.ll
@@ -1,6 +1,6 @@
-; RUN: llc -verify-machineinstrs -mcpu=a2 < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -enable-ppc-prefetching=true -mcpu=a2 < %s | FileCheck %s
 target datalayout = "E-m:e-i64:64-n32:64"
-target triple = "powerpc64-bgq-linux"
+target triple = "powerpc64le-unknown-linux"
 
 ; Function Attrs: nounwind
 define void @foo(double* nocapture %a, double* nocapture readonly %b) #0 {
diff --git a/llvm/test/CodeGen/PowerPC/loop-prep-all.ll b/llvm/test/CodeGen/PowerPC/loop-prep-all.ll
index a13192d3e6586..7fdabcd4be210 100644
--- a/llvm/test/CodeGen/PowerPC/loop-prep-all.ll
+++ b/llvm/test/CodeGen/PowerPC/loop-prep-all.ll
@@ -1,5 +1,4 @@
 ; RUN: llc -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -verify-machineinstrs -mtriple=powerpc64-bgq-linux < %s | FileCheck %s -check-prefix=CHECK -check-prefix=CHECK-BGQ
 target datalayout = "E-m:e-i64:64-n32:64"
 target triple = "powerpc64-unknown-linux-gnu"
 
@@ -21,7 +20,6 @@ for.body:                                         ; preds = %for.body, %entry
 
 ; CHECK-LABEL: @foo
 
-; CHECK-BGQ-DAG: dcbt 4, 5
 ; CHECK-DAG: lfdu [[REG1:[0-9]+]], 8({{[0-9]+}})
 ; CHECK-DAG: fadd [[REG2:[0-9]+]], [[REG1]], 0
 ; CHECK-DAG: stfdu [[REG2]], 8({{[0-9]+}})
@@ -34,15 +32,13 @@ for.cond.cleanup6:                                ; preds = %for.body7
 
 for.body7:                                        ; preds = %for.body, %for.body7
   %i3.017 = phi i32 [ %inc9, %for.body7 ], [ 0, %for.body ]
-  tail call void bitcast (void (...)* @bar to void ()*)() #2
+  tail call void bitcast (void (...)* @bar to void ()*)() #0
   %inc9 = add nuw nsw i32 %i3.017, 1
   %exitcond = icmp eq i32 %inc9, 1024
   br i1 %exitcond, label %for.cond.cleanup6, label %for.body7
 }
 
-declare void @bar(...) #1
+declare void @bar(...) 
 
-attributes #0 = { nounwind "target-cpu"="a2q" }
-attributes #1 = { "target-cpu"="a2q" }
-attributes #2 = { nounwind }
+attributes #0 = { nounwind }
 
diff --git a/llvm/test/CodeGen/PowerPC/lxv-aligned-stack-slots.ll b/llvm/test/CodeGen/PowerPC/lxv-aligned-stack-slots.ll
index 93868007d0d36..aa618d2b732c7 100644
--- a/llvm/test/CodeGen/PowerPC/lxv-aligned-stack-slots.ll
+++ b/llvm/test/CodeGen/PowerPC/lxv-aligned-stack-slots.ll
@@ -41,6 +41,6 @@ define void @aligned_slot() #0 {
 ; Function Attrs: argmemonly nounwind
 declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i1) #1
 
-attributes #0 = { nounwind "target-cpu"="pwr9" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+power9-vector,+vsx,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "target-cpu"="pwr9" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+power9-vector,+vsx" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { argmemonly nounwind }
 attributes #2 = { nounwind }
diff --git a/llvm/test/CodeGen/PowerPC/machine-combiner.ll b/llvm/test/CodeGen/PowerPC/machine-combiner.ll
index 2e834b1fe788c..16fc3ee3e5202 100644
--- a/llvm/test/CodeGen/PowerPC/machine-combiner.ll
+++ b/llvm/test/CodeGen/PowerPC/machine-combiner.ll
@@ -1,5 +1,4 @@
 ; RUN: llc -verify-machineinstrs -O3 -mcpu=pwr7 < %s | FileCheck  %s -check-prefix=CHECK -check-prefix=CHECK-PWR
-; RUN: llc -verify-machineinstrs -O3 -mcpu=a2q < %s | FileCheck  %s -check-prefix=CHECK -check-prefix=CHECK-QPX
 ; RUN: llc -verify-machineinstrs -O3 -mcpu=pwr9 < %s | FileCheck  %s -check-prefix=FIXPOINT
 target datalayout = "E-m:e-i64:64-n32:64"
 target triple = "powerpc64-unknown-linux-gnu"
@@ -93,9 +92,6 @@ define float @reassociate_adds5(float %x0, float %x1, float %x2, float %x3, floa
 define <4 x float> @vector_reassociate_adds1(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, <4 x float> %x3) {
 ; CHECK-LABEL: vector_reassociate_adds1:
 ; CHECK:       # %bb.0:
-; CHECK-QPX:       qvfadds [[REG0:[0-9]+]], 1, 2
-; CHECK-QPX:       qvfadds [[REG1:[0-9]+]], 3, 4
-; CHECK-QPX:       qvfadds 1, [[REG0]], [[REG1]]
 ; CHECK-PWR:       xvaddsp [[REG0:[0-9]+]], 34, 35
 ; CHECK-PWR:       xvaddsp [[REG1:[0-9]+]], 36, 37
 ; CHECK-PWR:       xvaddsp 34, [[REG0]], [[REG1]]
@@ -110,9 +106,6 @@ define <4 x float> @vector_reassociate_adds1(<4 x float> %x0, <4 x float> %x1, <
 define <4 x float> @vector_reassociate_adds2(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, <4 x float> %x3) {
 ; CHECK-LABEL: vector_reassociate_adds2:
 ; CHECK:       # %bb.0:
-; CHECK-QPX:       qvfadds [[REG0:[0-9]+]], 1, 2
-; CHECK-QPX:       qvfadds [[REG1:[0-9]+]], 3, 4
-; CHECK-QPX:       qvfadds 1, [[REG0]], [[REG1]]
 ; CHECK-PWR:       xvaddsp [[REG0:[0-9]+]], 34, 35
 ; CHECK-PWR:       xvaddsp [[REG1:[0-9]+]], 36, 37
 ; CHECK-PWR:       xvaddsp 34, [[REG0]], [[REG1]]
@@ -127,9 +120,6 @@ define <4 x float> @vector_reassociate_adds2(<4 x float> %x0, <4 x float> %x1, <
 define <4 x float> @vector_reassociate_adds3(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, <4 x float> %x3) {
 ; CHECK-LABEL: vector_reassociate_adds3:
 ; CHECK:       # %bb.0:
-; CHECK-QPX:       qvfadds [[REG0:[0-9]+]], 1, 2
-; CHECK-QPX:       qvfadds [[REG1:[0-9]+]], 3, 4
-; CHECK-QPX:       qvfadds 1, [[REG0]], [[REG1]]
 ; CHECK-PWR:       xvaddsp [[REG0:[0-9]+]], 34, 35
 ; CHECK-PWR:       xvaddsp [[REG1:[0-9]+]], 36, 37
 ; CHECK-PWR:       xvaddsp 34, [[REG0]], [[REG1]]
@@ -144,9 +134,6 @@ define <4 x float> @vector_reassociate_adds3(<4 x float> %x0, <4 x float> %x1, <
 define <4 x float> @vector_reassociate_adds4(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, <4 x float> %x3) {
 ; CHECK-LABEL: vector_reassociate_adds4:
 ; CHECK:       # %bb.0:
-; CHECK-QPX:       qvfadds [[REG0:[0-9]+]], 1, 2
-; CHECK-QPX:       qvfadds [[REG1:[0-9]+]], 3, 4
-; CHECK-QPX:       qvfadds 1, [[REG0]], [[REG1]]
 ; CHECK-PWR:       xvaddsp [[REG0:[0-9]+]], 34, 35
 ; CHECK-PWR:       xvaddsp [[REG1:[0-9]+]], 36, 37
 ; CHECK-PWR:       xvaddsp 34, [[REG0]], [[REG1]]
@@ -217,9 +204,6 @@ define i64 @reassociate_mulld(i64 %x0, i64 %x1, i64 %x2, i64 %x3) {
 define double @reassociate_mamaa_double(double %0, double %1, double %2, double %3, double %4, double %5) {
 ; CHECK-LABEL: reassociate_mamaa_double:
 ; CHECK:       # %bb.0:
-; CHECK-QPX-DAG:   fmadd [[REG0:[0-9]+]], 4, 3, 2
-; CHECK-QPX-DAG:   fmadd [[REG1:[0-9]+]], 6, 5, 1
-; CHECK-QPX:       fadd 1, [[REG0]], [[REG1]]
 ; CHECK-PWR-DAG:   xsmaddadp 1, 6, 5
 ; CHECK-PWR-DAG:   xsmaddadp 2, 4, 3
 ; CHECK-PWR:       xsadddp 1, 2, 1
@@ -250,9 +234,6 @@ define float @reassociate_mamaa_float(float %0, float %1, float %2, float %3, fl
 define <4 x float> @reassociate_mamaa_vec(<4 x float> %0, <4 x float> %1, <4 x float> %2, <4 x float> %3, <4 x float> %4, <4 x float> %5) {
 ; CHECK-LABEL: reassociate_mamaa_vec:
 ; CHECK:       # %bb.0:
-; CHECK-QPX-DAG:   qvfmadds [[REG0:[0-9]+]], 4, 3, 2
-; CHECK-QPX-DAG:   qvfmadds [[REG1:[0-9]+]], 6, 5, 1
-; CHECK-QPX:       qvfadds 1, [[REG0]], [[REG1]]
 ; CHECK-PWR-DAG:   xvmaddasp [[REG0:[0-9]+]], 39, 38
 ; CHECK-PWR-DAG:   xvmaddasp [[REG1:[0-9]+]], 37, 36
 ; CHECK-PWR:       xvaddsp 34, [[REG1]], [[REG0]]
@@ -268,11 +249,6 @@ define <4 x float> @reassociate_mamaa_vec(<4 x float> %0, <4 x float> %1, <4 x f
 define double @reassociate_mamama_double(double %0, double %1, double %2, double %3, double %4, double %5, double %6, double %7, double %8) {
 ; CHECK-LABEL: reassociate_mamama_double:
 ; CHECK:       # %bb.0:
-; CHECK-QPX:       fmadd [[REG0:[0-9]+]], 2, 1, 7
-; CHECK-QPX-DAG:   fmul [[REG1:[0-9]+]], 4, 3
-; CHECK-QPX-DAG:   fmadd [[REG2:[0-9]+]], 6, 5, [[REG0]]
-; CHECK-QPX-DAG:   fmadd [[REG3:[0-9]+]], 9, 8, [[REG1]]
-; CHECK-QPX:       fadd 1, [[REG2]], [[REG3]]
 ; CHECK-PWR:       xsmaddadp 7, 2, 1
 ; CHECK-PWR-DAG:   xsmuldp [[REG0:[0-9]+]], 4, 3
 ; CHECK-PWR-DAG:   xsmaddadp 7, 6, 5
diff --git a/llvm/test/CodeGen/PowerPC/mc-instrlat.ll b/llvm/test/CodeGen/PowerPC/mc-instrlat.ll
index e135986a2894c..f807f4fa20d25 100644
--- a/llvm/test/CodeGen/PowerPC/mc-instrlat.ll
+++ b/llvm/test/CodeGen/PowerPC/mc-instrlat.ll
@@ -19,7 +19,7 @@ entry:
 
 declare void @bar(double) #1
 
-attributes #0 = { nounwind "no-infs-fp-math"="true" "no-nans-fp-math"="true" "target-cpu"="ppc64" "target-features"="+altivec,-bpermd,-crypto,-direct-move,-extdiv,-power8-vector,-qpx,-vsx" "unsafe-fp-math"="true" "use-soft-float"="false" }
-attributes #1 = { "no-infs-fp-math"="true" "no-nans-fp-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="ppc64" "target-features"="+altivec,-bpermd,-crypto,-direct-move,-extdiv,-power8-vector,-qpx,-vsx" "unsafe-fp-math"="true" "use-soft-float"="false" }
+attributes #0 = { nounwind "no-infs-fp-math"="true" "no-nans-fp-math"="true" "target-cpu"="ppc64" "target-features"="+altivec,-bpermd,-crypto,-direct-move,-extdiv,-power8-vector,-vsx" "unsafe-fp-math"="true" "use-soft-float"="false" }
+attributes #1 = { "no-infs-fp-math"="true" "no-nans-fp-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="ppc64" "target-features"="+altivec,-bpermd,-crypto,-direct-move,-extdiv,-power8-vector,-vsx" "unsafe-fp-math"="true" "use-soft-float"="false" }
 attributes #2 = { nounwind }
 
diff --git a/llvm/test/CodeGen/PowerPC/mcount-insertion.ll b/llvm/test/CodeGen/PowerPC/mcount-insertion.ll
index cbb7947be2198..502347a3af198 100644
--- a/llvm/test/CodeGen/PowerPC/mcount-insertion.ll
+++ b/llvm/test/CodeGen/PowerPC/mcount-insertion.ll
@@ -1,9 +1,8 @@
-; RUN: opt -ee-instrument < %s | opt -inline | llc | FileCheck %s
+; RUN: opt -ee-instrument < %s | opt -inline | llc -mtriple=powerpc64-unknown-linux-gnu | FileCheck %s
 
 ; The run-line mimics how Clang might run the instrumentation passes.
 
 target datalayout = "E-m:e-i64:64-n32:64"
-target triple = "powerpc64-bgq-linux"
 
 
 define void @leaf_function() #0 {
diff --git a/llvm/test/CodeGen/PowerPC/memcpy-vec.ll b/llvm/test/CodeGen/PowerPC/memcpy-vec.ll
index cd0abd6149bde..c4e60f8c4b1f5 100644
--- a/llvm/test/CodeGen/PowerPC/memcpy-vec.ll
+++ b/llvm/test/CodeGen/PowerPC/memcpy-vec.ll
@@ -1,6 +1,5 @@
 ; RUN: llc -verify-machineinstrs -mcpu=pwr7 < %s | FileCheck  %s -check-prefix=PWR7
 ; RUN: llc -verify-machineinstrs -mcpu=pwr8 < %s | FileCheck  %s -check-prefix=PWR8
-; RUN: llc -verify-machineinstrs -mcpu=a2q < %s | FileCheck  %s -check-prefix=A2Q
 target datalayout = "E-m:e-i64:64-n32:64"
 target triple = "powerpc64-unknown-linux-gnu"
 
@@ -25,12 +24,6 @@ entry:
 ; PWR8: lxvw4x
 ; PWR8: stxvw4x
 ; PWR8: blr
-
-; A2Q-LABEL: @foo1
-; A2Q-NOT: bl memcpy
-; A2Q: ld {{[0-9]+}}, {{[0-9]+}}(4)
-; A2Q: std {{[0-9]+}}, {{[0-9]+}}(3)
-; A2Q: blr
 }
 
 ; Function Attrs: nounwind
@@ -52,12 +45,6 @@ entry:
 ; PWR8: lxvw4x
 ; PWR8: stxvw4x
 ; PWR8: blr
-
-; A2Q-LABEL: @foo2
-; A2Q-NOT: bl memcpy
-; A2Q: ld {{[0-9]+}}, {{[0-9]+}}(4)
-; A2Q: std {{[0-9]+}}, {{[0-9]+}}(3)
-; A2Q: blr
 }
 
 ; Function Attrs: nounwind
@@ -76,11 +63,6 @@ entry:
 ; PWR8-NOT: bl memset
 ; PWR8: stxvw4x
 ; PWR8: blr
-
-; A2Q-LABEL: @bar1
-; A2Q-NOT: bl memset
-; A2Q: std {{[0-9]+}}, {{[0-9]+}}(3)
-; A2Q: blr
 }
 
 ; Function Attrs: nounwind
@@ -99,11 +81,6 @@ entry:
 ; PWR8-NOT: bl memset
 ; PWR8: stxvw4x
 ; PWR8: blr
-
-; A2Q-LABEL: @bar2
-; A2Q-NOT: bl memset
-; A2Q: qvstfdx
-; A2Q: blr
 }
 
 ; Function Attrs: nounwind
diff --git a/llvm/test/CodeGen/PowerPC/memset-nc.ll b/llvm/test/CodeGen/PowerPC/memset-nc.ll
deleted file mode 100644
index 663d0cb1d6785..0000000000000
--- a/llvm/test/CodeGen/PowerPC/memset-nc.ll
+++ /dev/null
@@ -1,48 +0,0 @@
-; RUN: llc -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -verify-machineinstrs -O0 < %s | FileCheck %s -check-prefix=CHECK-O0
-target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
-target triple = "powerpc64-bgq-linux"
-
-; Function Attrs: nounwind
-define void @test_qpx() unnamed_addr #0 align 2 {
-entry:
-  %0 = load i32, i32* undef, align 4
-  %1 = trunc i32 %0 to i8
-  call void @llvm.memset.p0i8.i64(i8* align 32 null, i8 %1, i64 64, i1 false)
-  ret void
-
-; CHECK-LABEL: @test_qpx
-; CHECK: qvstfdx
-; CHECK: qvstfdx
-; CHECK: blr
-
-; CHECK-O0-LABEL: @test_qpx
-; CHECK-O0-NOT: qvstfdx
-; CHECK-O0: blr
-}
-
-; Function Attrs: nounwind
-declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i1) #1
-
-; Function Attrs: nounwind
-define void @test_vsx() unnamed_addr #2 align 2 {
-entry:
-  %0 = load i32, i32* undef, align 4
-  %1 = trunc i32 %0 to i8
-  call void @llvm.memset.p0i8.i64(i8* null, i8 %1, i64 32, i1 false)
-  ret void
-
-; CHECK-LABEL: @test_vsx
-; CHECK: stxvw4x
-; CHECK: stxvw4x
-; CHECK: blr
-
-; CHECK-O0-LABEL: @test_vsx
-; CHECK-O0-NOT: stxvw4x
-; CHECK-O0: blr
-}
-
-attributes #0 = { nounwind "target-cpu"="a2q" }
-attributes #1 = { nounwind }
-attributes #2 = { nounwind "target-cpu"="pwr7" }
-
diff --git a/llvm/test/CodeGen/PowerPC/misched-inorder-latency.ll b/llvm/test/CodeGen/PowerPC/misched-inorder-latency.ll
index 26663d81f3575..089c947713b9d 100644
--- a/llvm/test/CodeGen/PowerPC/misched-inorder-latency.ll
+++ b/llvm/test/CodeGen/PowerPC/misched-inorder-latency.ll
@@ -1,8 +1,7 @@
 ; RUN: llc -verify-machineinstrs < %s -enable-misched -pre-RA-sched=source -scheditins=false \
-; RUN:          -disable-ifcvt-triangle-false -disable-post-ra | FileCheck %s
+; RUN:          -disable-ifcvt-triangle-false -disable-post-ra -mtriple=powerpc64-unknown-linux-gnu | FileCheck %s
 ;
 target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
-target triple = "powerpc64-bgq-linux"
 
 ; %val1 is a load live out of %entry. It should be hoisted
 ; above the add.
diff --git a/llvm/test/CodeGen/PowerPC/misched.ll b/llvm/test/CodeGen/PowerPC/misched.ll
index 1c868b3f171c9..9a75fe44b7176 100644
--- a/llvm/test/CodeGen/PowerPC/misched.ll
+++ b/llvm/test/CodeGen/PowerPC/misched.ll
@@ -1,7 +1,6 @@
 ; RUN: llc < %s -enable-misched -verify-machineinstrs
 ; PR14302
 target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
-target triple = "powerpc64-bgq-linux"
 
 @b = external global [16000 x double], align 32
 
diff --git a/llvm/test/CodeGen/PowerPC/optnone-crbits-i1-ret.ll b/llvm/test/CodeGen/PowerPC/optnone-crbits-i1-ret.ll
index f59df4291c48f..ad5976318fe3a 100644
--- a/llvm/test/CodeGen/PowerPC/optnone-crbits-i1-ret.ll
+++ b/llvm/test/CodeGen/PowerPC/optnone-crbits-i1-ret.ll
@@ -1,6 +1,5 @@
-; RUN: llc -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu < %s | FileCheck %s
 target datalayout = "E-m:e-i64:64-n32:64"
-target triple = "powerpc64-bgq-linux"
 
 declare zeroext i1 @ri1()
 declare void @se1()
diff --git a/llvm/test/CodeGen/PowerPC/pcrel-local-caller-toc.ll b/llvm/test/CodeGen/PowerPC/pcrel-local-caller-toc.ll
index 2e248506c7b7b..2871e077df565 100644
--- a/llvm/test/CodeGen/PowerPC/pcrel-local-caller-toc.ll
+++ b/llvm/test/CodeGen/PowerPC/pcrel-local-caller-toc.ll
@@ -92,7 +92,7 @@ entry:
 
 ; Left the target features in this test because it is important that caller has
 ; -pcrelative-memops while callee has +pcrelative-memops
-attributes #0 = { nounwind "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+pcrelative-memops,+power8-vector,+power9-vector,+vsx,-htm,-qpx,-spe" }
-attributes #1 = { "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+pcrelative-memops,+power8-vector,+power9-vector,+vsx,-htm,-qpx,-spe" }
-attributes #2 = { nounwind "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+power9-vector,+vsx,-htm,-pcrelative-memops,-qpx,-spe" }
+attributes #0 = { nounwind "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+pcrelative-memops,+power8-vector,+power9-vector,+vsx,-htm,-spe" }
+attributes #1 = { "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+pcrelative-memops,+power8-vector,+power9-vector,+vsx,-htm,-spe" }
+attributes #2 = { nounwind "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+power9-vector,+vsx,-htm,-pcrelative-memops,-spe" }
 attributes #3 = { nounwind }
diff --git a/llvm/test/CodeGen/PowerPC/popcnt.ll b/llvm/test/CodeGen/PowerPC/popcnt.ll
index a06c59d4b945a..695863d87f16e 100644
--- a/llvm/test/CodeGen/PowerPC/popcnt.ll
+++ b/llvm/test/CodeGen/PowerPC/popcnt.ll
@@ -1,8 +1,6 @@
 ; RUN: llc -verify-machineinstrs -mtriple=ppc64-- -mattr=+popcntd < %s | FileCheck %s
 ; RUN: llc -verify-machineinstrs -mtriple=ppc64-- -mattr=+slow-popcntd < %s | FileCheck %s --check-prefix=SLOWPC
 ; RUN: llc -verify-machineinstrs -mtriple=ppc64-- -mcpu=pwr7 < %s | FileCheck %s
-; RUN: llc -verify-machineinstrs -mtriple=ppc64-- -mcpu=a2q < %s | FileCheck %s --check-prefix=SLOWPC
-; RUN: llc -verify-machineinstrs -mtriple=ppc64-- -mcpu=a2q -mattr=+popcntd < %s | FileCheck %s
 
 define i64 @_cntb64(i64 %x) nounwind readnone {
   %cnt = tail call i64 @llvm.ppc.popcntb(i64 %x)
diff --git a/llvm/test/CodeGen/PowerPC/ppc-passname.ll b/llvm/test/CodeGen/PowerPC/ppc-passname.ll
index 98343bdb535c2..06f13278d84cd 100644
--- a/llvm/test/CodeGen/PowerPC/ppc-passname.ll
+++ b/llvm/test/CodeGen/PowerPC/ppc-passname.ll
@@ -105,14 +105,3 @@
 ; STOP-AFTER-BRANCH-COALESCING-NOT: "ppc-branch-coalescing" pass is not registered.
 ; STOP-AFTER-BRANCH-COALESCING: Branch Coalescing 
 
-
-; Test pass name: ppc-qpx-load-splat.
-; RUN: llc -mtriple=powerpc64le-unknown-unknown < %s -debug-pass=Structure -stop-before=ppc-qpx-load-splat -o /dev/null 2>&1 | FileCheck %s -check-prefix=STOP-BEFORE-QPX-LOAD-SPLAT
-; STOP-BEFORE-QPX-LOAD-SPLAT-NOT: -ppc-qpx-load-splat
-; STOP-BEFORE-QPX-LOAD-SPLAT-NOT: "ppc-qpx-load-splat" pass is not registered.
-; STOP-BEFORE-QPX-LOAD-SPLAT-NOT: PowerPC QPX Load Splat Simplification
-
-; RUN: llc -mtriple=powerpc64le-unknown-unknown < %s -debug-pass=Structure -stop-after=ppc-qpx-load-splat -o /dev/null 2>&1 | FileCheck %s -check-prefix=STOP-AFTER-QPX-LOAD-SPLAT
-; STOP-AFTER-QPX-LOAD-SPLAT: -ppc-qpx-load-splat
-; STOP-AFTER-QPX-LOAD-SPLAT-NOT: "ppc-qpx-load-splat" pass is not registered.
-; STOP-AFTER-QPX-LOAD-SPLAT: PowerPC QPX Load Splat Simplification
diff --git a/llvm/test/CodeGen/PowerPC/ppc64-sibcall.ll b/llvm/test/CodeGen/PowerPC/ppc64-sibcall.ll
index fc0e71f878cab..357f28e88b184 100644
--- a/llvm/test/CodeGen/PowerPC/ppc64-sibcall.ll
+++ b/llvm/test/CodeGen/PowerPC/ppc64-sibcall.ll
@@ -1,6 +1,6 @@
 ; RUN: llc < %s -relocation-model=static -O1 -disable-ppc-sco=false -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu | FileCheck %s -check-prefix=CHECK-SCO
-; RUN: llc < %s -relocation-model=static -O1 -disable-ppc-sco=false -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr8 | FileCheck %s -check-prefix=CHECK-SCO-HASQPX
-; RUN: llc < %s -relocation-model=static -O1 -disable-ppc-sco=false -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -mcpu=pwr8 | FileCheck %s -check-prefix=CHECK-SCO-HASQPX
+; RUN: llc < %s -relocation-model=static -O1 -disable-ppc-sco=false -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr8 | FileCheck %s -check-prefix=CHECK-SCO
+; RUN: llc < %s -relocation-model=static -O1 -disable-ppc-sco=false -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -mcpu=pwr8 | FileCheck %s -check-prefix=CHECK-SCO
 ; RUN: llc < %s -relocation-model=static -O1 -disable-ppc-sco=false -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -mcpu=pwr8 -code-model=small | FileCheck %s -check-prefix=SCM
 
 ; No combination of "powerpc64le-unknown-linux-gnu" + "CHECK-SCO", because
@@ -117,23 +117,6 @@ define void @caller_local_sret_32(%S_32* %a) #1 {
 attributes #0 = { noinline nounwind  }
 attributes #1 = { nounwind }
 
-; vector <4 x i1> test
-
-define void @callee_v4i1(i8 %a, <4 x i1> %b, <4 x i1> %c) { ret void }
-define void @caller_v4i1_reorder(i8 %a, <4 x i1> %b, <4 x i1> %c) {
-  tail call void @callee_v4i1(i8 %a, <4 x i1> %c, <4 x i1> %b)
-  ret void
-
-; <4 x i1> is 32 bytes aligned, if subtarget doesn't support qpx, then we can't
-; place b, c to qpx register, so we can't do sco on caller_v4i1_reorder
-
-; CHECK-SCO-LABEL: caller_v4i1_reorder:
-; CHECK-SCO: bl callee_v4i1
-
-; CHECK-SCO-HASQPX-LABEL: caller_v4i1_reorder:
-; CHECK-SCO-HASQPX: b callee_v4i1
-}
-
 define void @f128_callee(i32* %ptr, ppc_fp128 %a, ppc_fp128 %b) { ret void }
 define void @f128_caller(i32* %ptr, ppc_fp128 %a, ppc_fp128 %b) {
   tail call void @f128_callee(i32* %ptr, ppc_fp128 %a, ppc_fp128 %b)
diff --git a/llvm/test/CodeGen/PowerPC/pr24546.ll b/llvm/test/CodeGen/PowerPC/pr24546.ll
index 28c03293680e5..028fd2d8f0064 100644
--- a/llvm/test/CodeGen/PowerPC/pr24546.ll
+++ b/llvm/test/CodeGen/PowerPC/pr24546.ll
@@ -47,8 +47,8 @@ declare double @pow(double, double) #0
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.value(metadata, i64, metadata, metadata) #2
 
-attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind readnone }
 attributes #3 = { nounwind }
 
diff --git a/llvm/test/CodeGen/PowerPC/pr27350.ll b/llvm/test/CodeGen/PowerPC/pr27350.ll
index 982023a1fcdc8..93dbd10fecdeb 100644
--- a/llvm/test/CodeGen/PowerPC/pr27350.ll
+++ b/llvm/test/CodeGen/PowerPC/pr27350.ll
@@ -18,7 +18,7 @@ entry:
 declare fastcc void @bar([2 x i64], [2 x i64]) unnamed_addr #1 align 2
 
 attributes #0 = { argmemonly nounwind }
-attributes #1 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind }
 
 !llvm.ident = !{!0}
diff --git a/llvm/test/CodeGen/PowerPC/pr28130.ll b/llvm/test/CodeGen/PowerPC/pr28130.ll
index cb703dfda8a59..4da415bd29269 100644
--- a/llvm/test/CodeGen/PowerPC/pr28130.ll
+++ b/llvm/test/CodeGen/PowerPC/pr28130.ll
@@ -67,4 +67,4 @@ bb:
   ret void
 }
 
-attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pwr8" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pwr8" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+power8-vector,+vsx" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/PowerPC/preinc-ld-sel-crash.ll b/llvm/test/CodeGen/PowerPC/preinc-ld-sel-crash.ll
index 04dee1ee182bb..35aec57ec2640 100644
--- a/llvm/test/CodeGen/PowerPC/preinc-ld-sel-crash.ll
+++ b/llvm/test/CodeGen/PowerPC/preinc-ld-sel-crash.ll
@@ -1,6 +1,6 @@
 ; RUN: llc -verify-machineinstrs < %s | FileCheck %s
 target datalayout = "E-m:e-i64:64-n32:64"
-target triple = "powerpc64-bgq-linux"
+target triple = "powerpc64le-unknown-linux"
 
 %t1 = type { %t2*, %t3* }
 %t2 = type <{ %t3*, i32, [4 x i8] }>
diff --git a/llvm/test/CodeGen/PowerPC/qpx-bv-sint.ll b/llvm/test/CodeGen/PowerPC/qpx-bv-sint.ll
deleted file mode 100644
index 4e0aef4c3df71..0000000000000
--- a/llvm/test/CodeGen/PowerPC/qpx-bv-sint.ll
+++ /dev/null
@@ -1,33 +0,0 @@
-; RUN: llc -verify-machineinstrs < %s -mcpu=a2q | FileCheck %s
-target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
-target triple = "powerpc64-bgq-linux"
-
-define void @s452(i32 %inp1) nounwind {
-entry:
-  br label %for.body4
-
-for.body4:                                        ; preds = %for.body4, %entry
-  %conv.4 = sitofp i32 %inp1 to double
-  %conv.5 = sitofp i32 %inp1 to double
-  %mul.4.v.i0.1 = insertelement <2 x double> undef, double %conv.4, i32 0
-  %v = insertelement <2 x double> %mul.4.v.i0.1, double %conv.5, i32 1
-  %vv = fmul <2 x double> %v, %v
-  %add7.4 = fadd <2 x double> %vv, %vv
-  store <2 x double> %add7.4, <2 x double>* undef, align 16
-  br i1 undef, label %for.end, label %for.body4
-
-for.end:                                          ; preds = %for.body4
-  unreachable
-; CHECK-LABEL: @s452
-; CHECK: lfiwax [[REG1:[0-9]+]],
-; CHECK: fcfid [[REG2:[0-9]+]], [[REG1]]
-; FIXME: We could 'promote' this to a vector earlier and remove this splat.
-; CHECK: qvesplati {{[0-9]+}}, [[REG2]], 0
-; CHECK: qvfmul
-; CHECK: qvfadd
-; CHECK: qvesplati {{[0-9]+}},
-; FIXME: We can use qvstfcdx here instead of two stores.
-; CHECK: stfd
-; CHECK: stfd
-}
-
diff --git a/llvm/test/CodeGen/PowerPC/qpx-bv.ll b/llvm/test/CodeGen/PowerPC/qpx-bv.ll
deleted file mode 100644
index 93a739b864c1d..0000000000000
--- a/llvm/test/CodeGen/PowerPC/qpx-bv.ll
+++ /dev/null
@@ -1,37 +0,0 @@
-; RUN: llc -verify-machineinstrs < %s -mcpu=a2q | FileCheck %s
-
-target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
-target triple = "powerpc64-bgq-linux"
-
-define <4 x double> @foo(double %f1, double %f2, double %f3, double %f4) {
-  %v1 = insertelement <4 x double> undef, double %f1, i32 0
-  %v2 = insertelement <4 x double> %v1,   double %f2, i32 1
-  %v3 = insertelement <4 x double> %v2,   double %f3, i32 2
-  %v4 = insertelement <4 x double> %v3,   double %f4, i32 3
-  ret <4 x double> %v4
-
-; CHECK-LABEL: @foo
-; CHECK: qvgpci [[REG1:[0-9]+]], 275
-; CHECK-DAG: qvgpci [[REG2:[0-9]+]], 101
-; CHECK-DAG: qvfperm [[REG3:[0-9]+]], 3, 4, [[REG1]]
-; CHECK-DAG: qvfperm [[REG4:[0-9]+]], 1, 2, [[REG1]]
-; CHECK-DAG: qvfperm 1, [[REG4]], [[REG3]], [[REG2]]
-; CHECK: blr
-}
-
-define <4 x float> @goo(float %f1, float %f2, float %f3, float %f4) {
-  %v1 = insertelement <4 x float> undef, float %f1, i32 0
-  %v2 = insertelement <4 x float> %v1,   float %f2, i32 1
-  %v3 = insertelement <4 x float> %v2,   float %f3, i32 2
-  %v4 = insertelement <4 x float> %v3,   float %f4, i32 3
-  ret <4 x float> %v4
-
-; CHECK-LABEL: @goo
-; CHECK: qvgpci [[REG1:[0-9]+]], 275
-; CHECK-DAG: qvgpci [[REG2:[0-9]+]], 101
-; CHECK-DAG: qvfperm [[REG3:[0-9]+]], 3, 4, [[REG1]]
-; CHECK-DAG: qvfperm [[REG4:[0-9]+]], 1, 2, [[REG1]]
-; CHECK-DAG: qvfperm 1, [[REG4]], [[REG3]], [[REG2]]
-; CHECK: blr
-}
-
diff --git a/llvm/test/CodeGen/PowerPC/qpx-func-clobber.ll b/llvm/test/CodeGen/PowerPC/qpx-func-clobber.ll
deleted file mode 100644
index ccbbd162a0cdb..0000000000000
--- a/llvm/test/CodeGen/PowerPC/qpx-func-clobber.ll
+++ /dev/null
@@ -1,22 +0,0 @@
-; RUN: llc -verify-machineinstrs < %s -mcpu=a2q | FileCheck %s
-target triple = "powerpc64-bgq-linux"
-
-declare <4 x double> @foo(<4 x double> %p)
-
-define <4 x double> @bar(<4 x double> %p, <4 x double> %q) {
-entry:
-  %v = call <4 x double> @foo(<4 x double> %p)
-  %w = call <4 x double> @foo(<4 x double> %q)
-  %x = fadd <4 x double> %v, %w
-  ret <4 x double> %x
-
-; CHECK-LABEL: @bar
-; CHECK: qvstfdx 2,
-; CHECK: bl foo
-; CHECK: qvstfdx 1,
-; CHECK: qvlfdx 1,
-; CHECK: bl foo
-; CHECK: qvlfdx [[REG:[0-9]+]],
-; CHECK: qvfadd 1, [[REG]], 1
-}
-
diff --git a/llvm/test/CodeGen/PowerPC/qpx-load-splat.ll b/llvm/test/CodeGen/PowerPC/qpx-load-splat.ll
deleted file mode 100644
index 50b864980d985..0000000000000
--- a/llvm/test/CodeGen/PowerPC/qpx-load-splat.ll
+++ /dev/null
@@ -1,80 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=powerpc64le-unknown-linux-gnu -ppc-vsr-nums-as-vr \
-; RUN:   -ppc-asm-full-reg-names -verify-machineinstrs < %s | FileCheck %s
-
-; Function Attrs: norecurse nounwind readonly
-define <4 x double> @foo(double* nocapture readonly %a) #0 {
-; CHECK-LABEL: foo:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    lxvdsx v2, 0, r3
-; CHECK-NEXT:    vmr v3, v2
-; CHECK-NEXT:    blr
-entry:
-  %0 = load double, double* %a, align 8
-  %vecinit.i = insertelement <4 x double> undef, double %0, i32 0
-  %shuffle.i = shufflevector <4 x double> %vecinit.i, <4 x double> undef, <4 x i32> zeroinitializer
-  ret <4 x double> %shuffle.i
-}
-
-define <4 x double> @foox(double* nocapture readonly %a, i64 %idx) #0 {
-; CHECK-LABEL: foox:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    sldi r4, r4, 3
-; CHECK-NEXT:    lxvdsx v2, r3, r4
-; CHECK-NEXT:    vmr v3, v2
-; CHECK-NEXT:    blr
-entry:
-  %p = getelementptr double, double* %a, i64 %idx
-  %0 = load double, double* %p, align 8
-  %vecinit.i = insertelement <4 x double> undef, double %0, i32 0
-  %shuffle.i = shufflevector <4 x double> %vecinit.i, <4 x double> undef, <4 x i32> zeroinitializer
-  ret <4 x double> %shuffle.i
-}
-
-define <4 x double> @fooxu(double* nocapture readonly %a, i64 %idx, double** %pptr) #0 {
-; CHECK-LABEL: fooxu:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    sldi r4, r4, 3
-; CHECK-NEXT:    add r6, r3, r4
-; CHECK-NEXT:    std r6, 0(r5)
-; CHECK-NEXT:    lxvdsx v2, r3, r4
-; CHECK-NEXT:    vmr v3, v2
-; CHECK-NEXT:    blr
-entry:
-  %p = getelementptr double, double* %a, i64 %idx
-  %0 = load double, double* %p, align 8
-  %vecinit.i = insertelement <4 x double> undef, double %0, i32 0
-  %shuffle.i = shufflevector <4 x double> %vecinit.i, <4 x double> undef, <4 x i32> zeroinitializer
-  store double* %p, double** %pptr, align 8
-  ret <4 x double> %shuffle.i
-}
-
-define <4 x float> @foof(float* nocapture readonly %a) #0 {
-; CHECK-LABEL: foof:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    lfiwzx f0, 0, r3
-; CHECK-NEXT:    xxspltw v2, vs0, 1
-; CHECK-NEXT:    blr
-entry:
-  %0 = load float, float* %a, align 4
-  %vecinit.i = insertelement <4 x float> undef, float %0, i32 0
-  %shuffle.i = shufflevector <4 x float> %vecinit.i, <4 x float> undef, <4 x i32> zeroinitializer
-  ret <4 x float> %shuffle.i
-}
-
-define <4 x float> @foofx(float* nocapture readonly %a, i64 %idx) #0 {
-; CHECK-LABEL: foofx:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    sldi r4, r4, 2
-; CHECK-NEXT:    lfiwzx f0, r3, r4
-; CHECK-NEXT:    xxspltw v2, vs0, 1
-; CHECK-NEXT:    blr
-entry:
-  %p = getelementptr float, float* %a, i64 %idx
-  %0 = load float, float* %p, align 4
-  %vecinit.i = insertelement <4 x float> undef, float %0, i32 0
-  %shuffle.i = shufflevector <4 x float> %vecinit.i, <4 x float> undef, <4 x i32> zeroinitializer
-  ret <4 x float> %shuffle.i
-}
-
-
diff --git a/llvm/test/CodeGen/PowerPC/qpx-load.ll b/llvm/test/CodeGen/PowerPC/qpx-load.ll
deleted file mode 100644
index 514f0934b6cfc..0000000000000
--- a/llvm/test/CodeGen/PowerPC/qpx-load.ll
+++ /dev/null
@@ -1,26 +0,0 @@
-; RUN: llc -verify-machineinstrs < %s -mcpu=a2q | FileCheck %s
-target triple = "powerpc64-bgq-linux"
-
-define <4 x double> @foo(<4 x double>* %p) {
-entry:
-  %v = load <4 x double>, <4 x double>* %p, align 8
-  ret <4 x double> %v
-}
-
-; CHECK: @foo
-; CHECK-DAG: li [[REG1:[0-9]+]], 31
-; CHECK-DAG: qvlfdx [[REG4:[0-9]+]], 0, 3
-; CHECK-DAG: qvlfdx [[REG2:[0-9]+]], 3, [[REG1]]
-; CHECK-DAG: qvlpcldx [[REG3:[0-9]+]], 0, 3
-; CHECK-DAG: qvfperm 1, [[REG4]], [[REG2]], [[REG3]]
-; CHECK: blr
-
-define <4 x double> @bar(<4 x double>* %p) {
-entry:
-  %v = load <4 x double>, <4 x double>* %p, align 32
-  ret <4 x double> %v
-}
-
-; CHECK: @bar
-; CHECK: qvlfdx
-
diff --git a/llvm/test/CodeGen/PowerPC/qpx-qvfmadd.ll b/llvm/test/CodeGen/PowerPC/qpx-qvfmadd.ll
deleted file mode 100644
index eab4d6af7e9fc..0000000000000
--- a/llvm/test/CodeGen/PowerPC/qpx-qvfmadd.ll
+++ /dev/null
@@ -1,79 +0,0 @@
-; RUN: llc -verify-machineinstrs -stop-after=finalize-isel < %s -mcpu=a2q | FileCheck %s
-target triple = "powerpc64-bgq-linux"
-
-define <2 x double> @test_qvfmadd(<2 x double> %0, <2 x double> %1, <2 x double> %2) {
-; CHECK: test_qvfmadd
-; CHECK: QVFMADD %2, %1, %0, implicit $rm
-;
-  %4 = fmul reassoc nsz <2 x double> %2, %1
-  %5 = fadd reassoc nsz <2 x double> %4, %0
-  ret <2 x double> %5
-}
-
-define <4 x float> @test_qvfmadds(<4 x float> %0, <4 x float> %1, <4 x float> %2) {
-; CHECK: test_qvfmadds
-; CHECK: QVFMADDSs %2, %1, %0, implicit $rm
-;
-  %4 = fmul reassoc nsz <4 x float> %2, %1
-  %5 = fadd reassoc nsz <4 x float> %4, %0
-  ret <4 x float> %5
-}
-
-define <2 x double> @test_qvfnmadd(<2 x double> %0, <2 x double> %1, <2 x double> %2) {
-; CHECK: test_qvfnmadd
-; CHECK: QVFNMADD %2, %1, %0, implicit $rm
-;
-  %4 = fmul reassoc nsz <2 x double> %2, %1
-  %5 = fadd reassoc nsz <2 x double> %4, %0
-  %6 = fneg reassoc nsz <2 x double> %5
-  ret <2 x double> %6
-}
-
-define <4 x float> @test_qvfnmadds(<4 x float> %0, <4 x float> %1, <4 x float> %2) {
-; CHECK: test_qvfnmadds
-; CHECK: QVFNMADDSs %2, %1, %0, implicit $rm
-;
-  %4 = fmul reassoc nsz <4 x float> %2, %1
-  %5 = fadd reassoc nsz <4 x float> %4, %0
-  %6 = fneg reassoc nsz <4 x float> %5
-  ret <4 x float> %6
-}
-
-define <2 x double> @test_qvfmsub(<2 x double> %0, <2 x double> %1, <2 x double> %2) {
-; CHECK: test_qvfmsub
-; CHECK: QVFMSUB %2, %1, %0, implicit $rm
-;
-  %4 = fmul reassoc nsz <2 x double> %2, %1
-  %5 = fsub reassoc nsz <2 x double> %4, %0
-  ret <2 x double> %5
-}
-
-define <4 x float> @test_qvfmsubs(<4 x float> %0, <4 x float> %1, <4 x float> %2) {
-; CHECK: test_qvfmsubs
-; CHECK: QVFMSUBSs %2, %1, %0, implicit $rm
-;
-  %4 = fmul reassoc nsz <4 x float> %2, %1
-  %5 = fsub reassoc nsz <4 x float> %4, %0
-  ret <4 x float> %5
-}
-
-define <2 x double> @test_qvfnmsub(<2 x double> %0, <2 x double> %1, <2 x double> %2) {
-; CHECK: test_qvfnmsub
-; CHECK: QVFNMSUB %2, %1, %0, implicit $rm
-;
-  %4 = fmul reassoc nsz <2 x double> %2, %1
-  %5 = fsub reassoc nsz <2 x double> %4, %0
-  %6 = fneg reassoc nsz <2 x double> %5
-  ret <2 x double> %6
-}
-
-define <4 x float> @test_qvfnmsubs(<4 x float> %0, <4 x float> %1, <4 x float> %2) {
-; CHECK: test_qvfnmsubs
-; CHECK: QVFNMSUBSs %2, %1, %0, implicit $rm
-;
-  %4 = fmul reassoc nsz <4 x float> %2, %1
-  %5 = fsub reassoc nsz <4 x float> %4, %0
-  %6 = fneg reassoc nsz <4 x float> %5
-  ret <4 x float> %6
-}
-
diff --git a/llvm/test/CodeGen/PowerPC/qpx-recipest.ll b/llvm/test/CodeGen/PowerPC/qpx-recipest.ll
deleted file mode 100644
index 498ab62819ced..0000000000000
--- a/llvm/test/CodeGen/PowerPC/qpx-recipest.ll
+++ /dev/null
@@ -1,473 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc64-unknown-linux-gnu -mcpu=a2q | FileCheck %s
-target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
-target triple = "powerpc64-unknown-linux-gnu"
-
-declare <4 x double> @llvm.sqrt.v4f64(<4 x double>)
-declare <4 x float> @llvm.sqrt.v4f32(<4 x float>)
-
-define <4 x double> @foo_fmf(<4 x double> %a, <4 x double> %b) nounwind {
-; CHECK-LABEL: foo_fmf:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    addis 3, 2, .LCPI0_0@toc@ha
-; CHECK-NEXT:    qvfrsqrte 3, 2
-; CHECK-NEXT:    addi 3, 3, .LCPI0_0@toc@l
-; CHECK-NEXT:    qvlfdx 0, 0, 3
-; CHECK-NEXT:    qvfmul 4, 3, 3
-; CHECK-NEXT:    qvfmsub 2, 2, 0, 2
-; CHECK-NEXT:    qvfnmsub 4, 2, 4, 0
-; CHECK-NEXT:    qvfmul 3, 3, 4
-; CHECK-NEXT:    qvfmul 4, 3, 3
-; CHECK-NEXT:    qvfnmsub 0, 2, 4, 0
-; CHECK-NEXT:    qvfmul 0, 3, 0
-; CHECK-NEXT:    qvfmul 1, 1, 0
-; CHECK-NEXT:    blr
-entry:
-  %x = call ninf afn reassoc <4 x double> @llvm.sqrt.v4f64(<4 x double> %b)
-  %r = fdiv arcp reassoc <4 x double> %a, %x
-  ret <4 x double> %r
-}
-
-define <4 x double> @foo_safe(<4 x double> %a, <4 x double> %b) nounwind {
-; CHECK-LABEL: foo_safe:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    qvesplati 5, 2, 3
-; CHECK-NEXT:    qvesplati 3, 2, 1
-; CHECK-NEXT:    qvesplati 4, 2, 2
-; CHECK-NEXT:    fsqrt 2, 2
-; CHECK-NEXT:    fsqrt 5, 5
-; CHECK-NEXT:    fsqrt 4, 4
-; CHECK-NEXT:    fsqrt 3, 3
-; CHECK-NEXT:    qvesplati 6, 1, 3
-; CHECK-NEXT:    qvgpci 0, 275
-; CHECK-NEXT:    fdiv 2, 1, 2
-; CHECK-NEXT:    fdiv 5, 6, 5
-; CHECK-NEXT:    qvesplati 6, 1, 2
-; CHECK-NEXT:    qvesplati 1, 1, 1
-; CHECK-NEXT:    fdiv 4, 6, 4
-; CHECK-NEXT:    fdiv 1, 1, 3
-; CHECK-NEXT:    qvfperm 3, 4, 5, 0
-; CHECK-NEXT:    qvfperm 0, 2, 1, 0
-; CHECK-NEXT:    qvgpci 1, 101
-; CHECK-NEXT:    qvfperm 1, 0, 3, 1
-; CHECK-NEXT:    blr
-entry:
-  %x = call <4 x double> @llvm.sqrt.v4f64(<4 x double> %b)
-  %r = fdiv <4 x double> %a, %x
-  ret <4 x double> %r
-}
-
-define <4 x double> @foof_fmf(<4 x double> %a, <4 x float> %b) nounwind {
-; CHECK-LABEL: foof_fmf:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    addis 3, 2, .LCPI2_0@toc@ha
-; CHECK-NEXT:    qvfrsqrtes 3, 2
-; CHECK-NEXT:    addi 3, 3, .LCPI2_0@toc@l
-; CHECK-NEXT:    qvlfsx 0, 0, 3
-; CHECK-NEXT:    qvfmuls 4, 3, 3
-; CHECK-NEXT:    qvfmsubs 2, 2, 0, 2
-; CHECK-NEXT:    qvfnmsubs 0, 2, 4, 0
-; CHECK-NEXT:    qvfmuls 0, 3, 0
-; CHECK-NEXT:    qvfmul 1, 1, 0
-; CHECK-NEXT:    blr
-entry:
-  %x = call afn ninf reassoc <4 x float> @llvm.sqrt.v4f32(<4 x float> %b)
-  %y = fpext <4 x float> %x to <4 x double>
-  %r = fdiv arcp reassoc nsz <4 x double> %a, %y
-  ret <4 x double> %r
-}
-
-define <4 x double> @foof_safe(<4 x double> %a, <4 x float> %b) nounwind {
-; CHECK-LABEL: foof_safe:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    qvesplati 0, 2, 3
-; CHECK-NEXT:    qvesplati 3, 2, 2
-; CHECK-NEXT:    fsqrts 4, 2
-; CHECK-NEXT:    qvesplati 2, 2, 1
-; CHECK-NEXT:    fsqrts 0, 0
-; CHECK-NEXT:    fsqrts 3, 3
-; CHECK-NEXT:    fsqrts 2, 2
-; CHECK-NEXT:    qvgpci 5, 275
-; CHECK-NEXT:    qvgpci 6, 101
-; CHECK-NEXT:    qvfperm 0, 3, 0, 5
-; CHECK-NEXT:    qvesplati 3, 1, 2
-; CHECK-NEXT:    qvfperm 2, 4, 2, 5
-; CHECK-NEXT:    qvfperm 0, 2, 0, 6
-; CHECK-NEXT:    qvesplati 2, 1, 3
-; CHECK-NEXT:    qvesplati 4, 0, 3
-; CHECK-NEXT:    fdiv 2, 2, 4
-; CHECK-NEXT:    qvesplati 4, 0, 2
-; CHECK-NEXT:    fdiv 3, 3, 4
-; CHECK-NEXT:    qvesplati 4, 1, 1
-; CHECK-NEXT:    fdiv 1, 1, 0
-; CHECK-NEXT:    qvesplati 0, 0, 1
-; CHECK-NEXT:    fdiv 0, 4, 0
-; CHECK-NEXT:    qvfperm 2, 3, 2, 5
-; CHECK-NEXT:    qvfperm 0, 1, 0, 5
-; CHECK-NEXT:    qvfperm 1, 0, 2, 6
-; CHECK-NEXT:    blr
-entry:
-  %x = call <4 x float> @llvm.sqrt.v4f32(<4 x float> %b)
-  %y = fpext <4 x float> %x to <4 x double>
-  %r = fdiv <4 x double> %a, %y
-  ret <4 x double> %r
-}
-
-define <4 x float> @food_fmf(<4 x float> %a, <4 x double> %b) nounwind {
-; CHECK-LABEL: food_fmf:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    addis 3, 2, .LCPI4_0@toc@ha
-; CHECK-NEXT:    qvfrsqrte 3, 2
-; CHECK-NEXT:    addi 3, 3, .LCPI4_0@toc@l
-; CHECK-NEXT:    qvlfdx 0, 0, 3
-; CHECK-NEXT:    qvfmul 4, 3, 3
-; CHECK-NEXT:    qvfmsub 2, 2, 0, 2
-; CHECK-NEXT:    qvfnmsub 4, 2, 4, 0
-; CHECK-NEXT:    qvfmul 3, 3, 4
-; CHECK-NEXT:    qvfmul 4, 3, 3
-; CHECK-NEXT:    qvfnmsub 0, 2, 4, 0
-; CHECK-NEXT:    qvfmul 0, 3, 0
-; CHECK-NEXT:    qvfrsp 0, 0
-; CHECK-NEXT:    qvfmuls 1, 1, 0
-; CHECK-NEXT:    blr
-entry:
-  %x = call afn ninf reassoc <4 x double> @llvm.sqrt.v4f64(<4 x double> %b)
-  %y = fptrunc <4 x double> %x to <4 x float>
-  %r = fdiv arcp reassoc <4 x float> %a, %y
-  ret <4 x float> %r
-}
-
-define <4 x float> @food_safe(<4 x float> %a, <4 x double> %b) nounwind {
-; CHECK-LABEL: food_safe:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    qvesplati 0, 2, 3
-; CHECK-NEXT:    qvesplati 3, 2, 2
-; CHECK-NEXT:    fsqrt 4, 2
-; CHECK-NEXT:    qvesplati 2, 2, 1
-; CHECK-NEXT:    fsqrt 0, 0
-; CHECK-NEXT:    fsqrt 3, 3
-; CHECK-NEXT:    fsqrt 2, 2
-; CHECK-NEXT:    qvgpci 5, 275
-; CHECK-NEXT:    qvgpci 6, 101
-; CHECK-NEXT:    qvfperm 0, 3, 0, 5
-; CHECK-NEXT:    qvesplati 3, 1, 2
-; CHECK-NEXT:    qvfperm 2, 4, 2, 5
-; CHECK-NEXT:    qvfperm 0, 2, 0, 6
-; CHECK-NEXT:    qvesplati 2, 1, 3
-; CHECK-NEXT:    qvfrsp 0, 0
-; CHECK-NEXT:    qvesplati 4, 0, 3
-; CHECK-NEXT:    fdivs 2, 2, 4
-; CHECK-NEXT:    qvesplati 4, 0, 2
-; CHECK-NEXT:    fdivs 3, 3, 4
-; CHECK-NEXT:    qvesplati 4, 1, 1
-; CHECK-NEXT:    fdivs 1, 1, 0
-; CHECK-NEXT:    qvesplati 0, 0, 1
-; CHECK-NEXT:    fdivs 0, 4, 0
-; CHECK-NEXT:    qvfperm 2, 3, 2, 5
-; CHECK-NEXT:    qvfperm 0, 1, 0, 5
-; CHECK-NEXT:    qvfperm 1, 0, 2, 6
-; CHECK-NEXT:    blr
-entry:
-  %x = call <4 x double> @llvm.sqrt.v4f64(<4 x double> %b)
-  %y = fptrunc <4 x double> %x to <4 x float>
-  %r = fdiv <4 x float> %a, %y
-  ret <4 x float> %r
-}
-
-define <4 x float> @goo_fmf(<4 x float> %a, <4 x float> %b) nounwind {
-; CHECK-LABEL: goo_fmf:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    addis 3, 2, .LCPI6_0@toc@ha
-; CHECK-NEXT:    qvfrsqrtes 3, 2
-; CHECK-NEXT:    addi 3, 3, .LCPI6_0@toc@l
-; CHECK-NEXT:    qvlfsx 0, 0, 3
-; CHECK-NEXT:    qvfmuls 4, 3, 3
-; CHECK-NEXT:    qvfmsubs 2, 2, 0, 2
-; CHECK-NEXT:    qvfnmsubs 0, 2, 4, 0
-; CHECK-NEXT:    qvfmuls 0, 3, 0
-; CHECK-NEXT:    qvfmuls 1, 1, 0
-; CHECK-NEXT:    blr
-entry:
-  %x = call afn ninf reassoc <4 x float> @llvm.sqrt.v4f32(<4 x float> %b)
-  %r = fdiv arcp reassoc nsz <4 x float> %a, %x
-  ret <4 x float> %r
-}
-
-define <4 x float> @goo_safe(<4 x float> %a, <4 x float> %b) nounwind {
-; CHECK-LABEL: goo_safe:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    qvesplati 5, 2, 3
-; CHECK-NEXT:    qvesplati 3, 2, 1
-; CHECK-NEXT:    qvesplati 4, 2, 2
-; CHECK-NEXT:    fsqrts 2, 2
-; CHECK-NEXT:    fsqrts 5, 5
-; CHECK-NEXT:    fsqrts 4, 4
-; CHECK-NEXT:    fsqrts 3, 3
-; CHECK-NEXT:    qvesplati 6, 1, 3
-; CHECK-NEXT:    qvgpci 0, 275
-; CHECK-NEXT:    fdivs 2, 1, 2
-; CHECK-NEXT:    fdivs 5, 6, 5
-; CHECK-NEXT:    qvesplati 6, 1, 2
-; CHECK-NEXT:    qvesplati 1, 1, 1
-; CHECK-NEXT:    fdivs 4, 6, 4
-; CHECK-NEXT:    fdivs 1, 1, 3
-; CHECK-NEXT:    qvfperm 3, 4, 5, 0
-; CHECK-NEXT:    qvfperm 0, 2, 1, 0
-; CHECK-NEXT:    qvgpci 1, 101
-; CHECK-NEXT:    qvfperm 1, 0, 3, 1
-; CHECK-NEXT:    blr
-entry:
-  %x = call <4 x float> @llvm.sqrt.v4f32(<4 x float> %b)
-  %r = fdiv <4 x float> %a, %x
-  ret <4 x float> %r
-}
-
-define <4 x double> @foo2_fmf(<4 x double> %a, <4 x double> %b) nounwind {
-; CHECK-LABEL: foo2_fmf:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    addis 3, 2, .LCPI8_0@toc@ha
-; CHECK-NEXT:    qvfre 3, 2
-; CHECK-NEXT:    addi 3, 3, .LCPI8_0@toc@l
-; CHECK-NEXT:    qvlfdx 0, 0, 3
-; CHECK-NEXT:    qvfmadd 0, 2, 3, 0
-; CHECK-NEXT:    qvfnmsub 0, 3, 0, 3
-; CHECK-NEXT:    qvfmul 3, 1, 0
-; CHECK-NEXT:    qvfnmsub 1, 2, 3, 1
-; CHECK-NEXT:    qvfmadd 1, 0, 1, 3
-; CHECK-NEXT:    blr
-entry:
-  %r = fdiv arcp reassoc nsz ninf <4 x double> %a, %b
-  ret <4 x double> %r
-}
-
-define <4 x double> @foo2_safe(<4 x double> %a, <4 x double> %b) nounwind {
-; CHECK-LABEL: foo2_safe:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    qvesplati 3, 2, 3
-; CHECK-NEXT:    qvesplati 4, 1, 3
-; CHECK-NEXT:    qvesplati 5, 2, 2
-; CHECK-NEXT:    qvgpci 0, 275
-; CHECK-NEXT:    fdiv 3, 4, 3
-; CHECK-NEXT:    qvesplati 4, 1, 2
-; CHECK-NEXT:    fdiv 4, 4, 5
-; CHECK-NEXT:    fdiv 5, 1, 2
-; CHECK-NEXT:    qvesplati 2, 2, 1
-; CHECK-NEXT:    qvesplati 1, 1, 1
-; CHECK-NEXT:    fdiv 1, 1, 2
-; CHECK-NEXT:    qvfperm 2, 4, 3, 0
-; CHECK-NEXT:    qvfperm 0, 5, 1, 0
-; CHECK-NEXT:    qvgpci 1, 101
-; CHECK-NEXT:    qvfperm 1, 0, 2, 1
-; CHECK-NEXT:    blr
-  %r = fdiv <4 x double> %a, %b
-  ret <4 x double> %r
-}
-
-define <4 x float> @goo2_fmf(<4 x float> %a, <4 x float> %b) nounwind {
-; CHECK-LABEL: goo2_fmf:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    qvfres 0, 2
-; CHECK-NEXT:    qvfmuls 3, 1, 0
-; CHECK-NEXT:    qvfnmsubs 1, 2, 3, 1
-; CHECK-NEXT:    qvfmadds 1, 0, 1, 3
-; CHECK-NEXT:    blr
-entry:
-  %r = fdiv arcp reassoc ninf <4 x float> %a, %b
-  ret <4 x float> %r
-}
-
-define <4 x float> @goo2_safe(<4 x float> %a, <4 x float> %b) nounwind {
-; CHECK-LABEL: goo2_safe:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    qvesplati 3, 2, 3
-; CHECK-NEXT:    qvesplati 4, 1, 3
-; CHECK-NEXT:    qvesplati 5, 2, 2
-; CHECK-NEXT:    qvgpci 0, 275
-; CHECK-NEXT:    fdivs 3, 4, 3
-; CHECK-NEXT:    qvesplati 4, 1, 2
-; CHECK-NEXT:    fdivs 4, 4, 5
-; CHECK-NEXT:    fdivs 5, 1, 2
-; CHECK-NEXT:    qvesplati 2, 2, 1
-; CHECK-NEXT:    qvesplati 1, 1, 1
-; CHECK-NEXT:    fdivs 1, 1, 2
-; CHECK-NEXT:    qvfperm 2, 4, 3, 0
-; CHECK-NEXT:    qvfperm 0, 5, 1, 0
-; CHECK-NEXT:    qvgpci 1, 101
-; CHECK-NEXT:    qvfperm 1, 0, 2, 1
-; CHECK-NEXT:    blr
-entry:
-  %r = fdiv <4 x float> %a, %b
-  ret <4 x float> %r
-}
-
-define <4 x double> @foo3_fmf_denorm_on(<4 x double> %a) #0 {
-; CHECK-LABEL: foo3_fmf_denorm_on:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    addis 3, 2, .LCPI12_0@toc@ha
-; CHECK-NEXT:    qvfrsqrte 0, 1
-; CHECK-NEXT:    addi 3, 3, .LCPI12_0@toc@l
-; CHECK-NEXT:    qvlfdx 2, 0, 3
-; CHECK-NEXT:    addis 3, 2, .LCPI12_1@toc@ha
-; CHECK-NEXT:    addi 3, 3, .LCPI12_1@toc@l
-; CHECK-NEXT:    qvfmul 3, 0, 0
-; CHECK-NEXT:    qvfmsub 4, 1, 2, 1
-; CHECK-NEXT:    qvfnmsub 3, 4, 3, 2
-; CHECK-NEXT:    qvfmul 0, 0, 3
-; CHECK-NEXT:    qvfmul 3, 0, 0
-; CHECK-NEXT:    qvfnmsub 2, 4, 3, 2
-; CHECK-NEXT:    qvfmul 0, 0, 2
-; CHECK-NEXT:    qvlfdx 2, 0, 3
-; CHECK-NEXT:    addis 3, 2, .LCPI12_2@toc@ha
-; CHECK-NEXT:    addi 3, 3, .LCPI12_2@toc@l
-; CHECK-NEXT:    qvlfdx 3, 0, 3
-; CHECK-NEXT:    qvfmul 0, 0, 1
-; CHECK-NEXT:    qvfabs 1, 1
-; CHECK-NEXT:    qvfcmplt 1, 1, 2
-; CHECK-NEXT:    qvfsel 1, 1, 3, 0
-; CHECK-NEXT:    blr
-entry:
-  %r = call reassoc ninf afn <4 x double> @llvm.sqrt.v4f64(<4 x double> %a)
-  ret <4 x double> %r
-}
-
-define <4 x double> @foo3_fmf_denorm_off(<4 x double> %a) #1 {
-; CHECK-LABEL: foo3_fmf_denorm_off:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    addis 3, 2, .LCPI13_0@toc@ha
-; CHECK-NEXT:    qvfrsqrte 0, 1
-; CHECK-NEXT:    addi 3, 3, .LCPI13_0@toc@l
-; CHECK-NEXT:    qvlfdx 2, 0, 3
-; CHECK-NEXT:    addis 3, 2, .LCPI13_1@toc@ha
-; CHECK-NEXT:    addi 3, 3, .LCPI13_1@toc@l
-; CHECK-NEXT:    qvfmul 3, 0, 0
-; CHECK-NEXT:    qvfmsub 4, 1, 2, 1
-; CHECK-NEXT:    qvfnmsub 3, 4, 3, 2
-; CHECK-NEXT:    qvfmul 0, 0, 3
-; CHECK-NEXT:    qvfmul 3, 0, 0
-; CHECK-NEXT:    qvfnmsub 2, 4, 3, 2
-; CHECK-NEXT:    qvfmul 0, 0, 2
-; CHECK-NEXT:    qvlfdx 2, 0, 3
-; CHECK-NEXT:    qvfmul 0, 0, 1
-; CHECK-NEXT:    qvfcmpeq 1, 1, 2
-; CHECK-NEXT:    qvfsel 1, 1, 2, 0
-; CHECK-NEXT:    blr
-entry:
-  %r = call afn reassoc ninf <4 x double> @llvm.sqrt.v4f64(<4 x double> %a)
-  ret <4 x double> %r
-}
-
-define <4 x double> @foo3_safe_denorm_on(<4 x double> %a) #0 {
-; CHECK-LABEL: foo3_safe_denorm_on:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    qvesplati 2, 1, 3
-; CHECK-NEXT:    qvesplati 3, 1, 2
-; CHECK-NEXT:    fsqrt 4, 1
-; CHECK-NEXT:    qvesplati 1, 1, 1
-; CHECK-NEXT:    fsqrt 2, 2
-; CHECK-NEXT:    fsqrt 3, 3
-; CHECK-NEXT:    fsqrt 1, 1
-; CHECK-NEXT:    qvgpci 0, 275
-; CHECK-NEXT:    qvfperm 2, 3, 2, 0
-; CHECK-NEXT:    qvfperm 0, 4, 1, 0
-; CHECK-NEXT:    qvgpci 1, 101
-; CHECK-NEXT:    qvfperm 1, 0, 2, 1
-; CHECK-NEXT:    blr
-entry:
-  %r = call <4 x double> @llvm.sqrt.v4f64(<4 x double> %a)
-  ret <4 x double> %r
-}
-
-define <4 x double> @foo3_safe_denorm_off(<4 x double> %a) #1 {
-; CHECK-LABEL: foo3_safe_denorm_off:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    qvesplati 2, 1, 3
-; CHECK-NEXT:    qvesplati 3, 1, 2
-; CHECK-NEXT:    fsqrt 4, 1
-; CHECK-NEXT:    qvesplati 1, 1, 1
-; CHECK-NEXT:    fsqrt 2, 2
-; CHECK-NEXT:    fsqrt 3, 3
-; CHECK-NEXT:    fsqrt 1, 1
-; CHECK-NEXT:    qvgpci 0, 275
-; CHECK-NEXT:    qvfperm 2, 3, 2, 0
-; CHECK-NEXT:    qvfperm 0, 4, 1, 0
-; CHECK-NEXT:    qvgpci 1, 101
-; CHECK-NEXT:    qvfperm 1, 0, 2, 1
-; CHECK-NEXT:    blr
-entry:
-  %r = call <4 x double> @llvm.sqrt.v4f64(<4 x double> %a)
-  ret <4 x double> %r
-}
-
-define <4 x float> @goo3_fmf_denorm_on(<4 x float> %a) #0 {
-; CHECK-LABEL: goo3_fmf_denorm_on:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    addis 3, 2, .LCPI16_1@toc@ha
-; CHECK-NEXT:    qvfrsqrtes 2, 1
-; CHECK-NEXT:    addi 3, 3, .LCPI16_1@toc@l
-; CHECK-NEXT:    qvlfsx 0, 0, 3
-; CHECK-NEXT:    addis 3, 2, .LCPI16_0@toc@ha
-; CHECK-NEXT:    addi 3, 3, .LCPI16_0@toc@l
-; CHECK-NEXT:    qvfmuls 4, 2, 2
-; CHECK-NEXT:    qvfmsubs 3, 1, 0, 1
-; CHECK-NEXT:    qvfnmsubs 0, 3, 4, 0
-; CHECK-NEXT:    qvlfsx 3, 0, 3
-; CHECK-NEXT:    addis 3, 2, .LCPI16_2@toc@ha
-; CHECK-NEXT:    addi 3, 3, .LCPI16_2@toc@l
-; CHECK-NEXT:    qvlfsx 4, 0, 3
-; CHECK-NEXT:    qvfmuls 0, 2, 0
-; CHECK-NEXT:    qvfabs 2, 1
-; CHECK-NEXT:    qvfmuls 0, 0, 1
-; CHECK-NEXT:    qvfcmplt 1, 2, 3
-; CHECK-NEXT:    qvfsel 1, 1, 4, 0
-; CHECK-NEXT:    blr
-entry:
-  %r = call reassoc afn ninf nsz <4 x float> @llvm.sqrt.v4f32(<4 x float> %a)
-  ret <4 x float> %r
-}
-
-define <4 x float> @goo3_fmf_denorm_off(<4 x float> %a) #1 {
-; CHECK-LABEL: goo3_fmf_denorm_off:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    addis 3, 2, .LCPI17_1@toc@ha
-; CHECK-NEXT:    qvfrsqrtes 2, 1
-; CHECK-NEXT:    addi 3, 3, .LCPI17_1@toc@l
-; CHECK-NEXT:    qvlfsx 0, 0, 3
-; CHECK-NEXT:    addis 3, 2, .LCPI17_0@toc@ha
-; CHECK-NEXT:    addi 3, 3, .LCPI17_0@toc@l
-; CHECK-NEXT:    qvfmuls 4, 2, 2
-; CHECK-NEXT:    qvfmsubs 3, 1, 0, 1
-; CHECK-NEXT:    qvfnmsubs 0, 3, 4, 0
-; CHECK-NEXT:    qvlfsx 3, 0, 3
-; CHECK-NEXT:    qvfmuls 0, 2, 0
-; CHECK-NEXT:    qvfmuls 0, 0, 1
-; CHECK-NEXT:    qvfcmpeq 1, 1, 3
-; CHECK-NEXT:    qvfsel 1, 1, 3, 0
-; CHECK-NEXT:    blr
-entry:
-  %r = call reassoc ninf afn nsz <4 x float> @llvm.sqrt.v4f32(<4 x float> %a)
-  ret <4 x float> %r
-}
-
-define <4 x float> @goo3_safe(<4 x float> %a) nounwind {
-; CHECK-LABEL: goo3_safe:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    qvesplati 2, 1, 3
-; CHECK-NEXT:    qvesplati 3, 1, 2
-; CHECK-NEXT:    fsqrts 4, 1
-; CHECK-NEXT:    qvesplati 1, 1, 1
-; CHECK-NEXT:    fsqrts 2, 2
-; CHECK-NEXT:    fsqrts 3, 3
-; CHECK-NEXT:    fsqrts 1, 1
-; CHECK-NEXT:    qvgpci 0, 275
-; CHECK-NEXT:    qvfperm 2, 3, 2, 0
-; CHECK-NEXT:    qvfperm 0, 4, 1, 0
-; CHECK-NEXT:    qvgpci 1, 101
-; CHECK-NEXT:    qvfperm 1, 0, 2, 1
-; CHECK-NEXT:    blr
-entry:
-  %r = call <4 x float> @llvm.sqrt.v4f32(<4 x float> %a)
-  ret <4 x float> %r
-}
-
-attributes #0 = { nounwind "denormal-fp-math"="ieee,ieee" }
-attributes #1 = { nounwind "denormal-fp-math"="preserve-sign,preserve-sign" }
diff --git a/llvm/test/CodeGen/PowerPC/qpx-rounding-ops.ll b/llvm/test/CodeGen/PowerPC/qpx-rounding-ops.ll
deleted file mode 100644
index ee3357156a6c0..0000000000000
--- a/llvm/test/CodeGen/PowerPC/qpx-rounding-ops.ll
+++ /dev/null
@@ -1,109 +0,0 @@
-; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc64-unknown-linux-gnu -mcpu=a2q | FileCheck %s
-; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc64-unknown-linux-gnu -mcpu=a2q -enable-unsafe-fp-math | FileCheck -check-prefix=CHECK-FM %s
-target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
-target triple = "powerpc64-unknown-linux-gnu"
-
-define <4 x float> @test1(<4 x float> %x) nounwind  {
-  %call = tail call <4 x float> @llvm.floor.v4f32(<4 x float> %x) nounwind readnone
-  ret <4 x float> %call
-
-; CHECK: test1:
-; CHECK: qvfrim 1, 1
-
-; CHECK-FM: test1:
-; CHECK-FM: qvfrim 1, 1
-}
-
-declare <4 x float> @llvm.floor.v4f32(<4 x float>) nounwind readnone
-
-define <4 x double> @test2(<4 x double> %x) nounwind  {
-  %call = tail call <4 x double> @llvm.floor.v4f64(<4 x double> %x) nounwind readnone
-  ret <4 x double> %call
-
-; CHECK: test2:
-; CHECK: qvfrim 1, 1
-
-; CHECK-FM: test2:
-; CHECK-FM: qvfrim 1, 1
-}
-
-declare <4 x double> @llvm.floor.v4f64(<4 x double>) nounwind readnone
-
-define <4 x float> @test3(<4 x float> %x) nounwind  {
-  %call = tail call <4 x float> @llvm.nearbyint.v4f32(<4 x float> %x) nounwind readnone
-  ret <4 x float> %call
-
-; CHECK: test3:
-; CHECK-NOT: qvfrin
-
-; CHECK-FM: test3:
-; CHECK-FM-NOT: qvfrin
-}
-
-declare <4 x float> @llvm.nearbyint.v4f32(<4 x float>) nounwind readnone
-
-define <4 x double> @test4(<4 x double> %x) nounwind  {
-  %call = tail call <4 x double> @llvm.nearbyint.v4f64(<4 x double> %x) nounwind readnone
-  ret <4 x double> %call
-
-; CHECK: test4:
-; CHECK-NOT: qvfrin
-
-; CHECK-FM: test4:
-; CHECK-FM-NOT: qvfrin
-}
-
-declare <4 x double> @llvm.nearbyint.v4f64(<4 x double>) nounwind readnone
-
-define <4 x float> @test5(<4 x float> %x) nounwind  {
-  %call = tail call <4 x float> @llvm.ceil.v4f32(<4 x float> %x) nounwind readnone
-  ret <4 x float> %call
-
-; CHECK: test5:
-; CHECK: qvfrip 1, 1
-
-; CHECK-FM: test5:
-; CHECK-FM: qvfrip 1, 1
-}
-
-declare <4 x float> @llvm.ceil.v4f32(<4 x float>) nounwind readnone
-
-define <4 x double> @test6(<4 x double> %x) nounwind  {
-  %call = tail call <4 x double> @llvm.ceil.v4f64(<4 x double> %x) nounwind readnone
-  ret <4 x double> %call
-
-; CHECK: test6:
-; CHECK: qvfrip 1, 1
-
-; CHECK-FM: test6:
-; CHECK-FM: qvfrip 1, 1
-}
-
-declare <4 x double> @llvm.ceil.v4f64(<4 x double>) nounwind readnone
-
-define <4 x float> @test9(<4 x float> %x) nounwind  {
-  %call = tail call <4 x float> @llvm.trunc.v4f32(<4 x float> %x) nounwind readnone
-  ret <4 x float> %call
-
-; CHECK: test9:
-; CHECK: qvfriz 1, 1
-
-; CHECK-FM: test9:
-; CHECK-FM: qvfriz 1, 1
-}
-
-declare <4 x float> @llvm.trunc.v4f32(<4 x float>) nounwind readnone
-
-define <4 x double> @test10(<4 x double> %x) nounwind  {
-  %call = tail call <4 x double> @llvm.trunc.v4f64(<4 x double> %x) nounwind readnone
-  ret <4 x double> %call
-
-; CHECK: test10:
-; CHECK: qvfriz 1, 1
-
-; CHECK-FM: test10:
-; CHECK-FM: qvfriz 1, 1
-}
-
-declare <4 x double> @llvm.trunc.v4f64(<4 x double>) nounwind readnone
-
diff --git a/llvm/test/CodeGen/PowerPC/qpx-s-load.ll b/llvm/test/CodeGen/PowerPC/qpx-s-load.ll
deleted file mode 100644
index 57d7e3b0ded3c..0000000000000
--- a/llvm/test/CodeGen/PowerPC/qpx-s-load.ll
+++ /dev/null
@@ -1,26 +0,0 @@
-; RUN: llc -verify-machineinstrs < %s -mcpu=a2q | FileCheck %s
-target triple = "powerpc64-bgq-linux"
-
-define <4 x float> @foo(<4 x float>* %p) {
-entry:
-  %v = load <4 x float>, <4 x float>* %p, align 4
-  ret <4 x float> %v
-}
-
-; CHECK: @foo
-; CHECK-DAG: li [[REG1:[0-9]+]], 15
-; CHECK-DAG: qvlfsx [[REG4:[0-9]+]], 0, 3
-; CHECK-DAG: qvlfsx [[REG2:[0-9]+]], 3, [[REG1]]
-; CHECK-DAG: qvlpclsx [[REG3:[0-9]+]], 0, 3
-; CHECK-DAG: qvfperm 1, [[REG4]], [[REG2]], [[REG3]]
-; CHECK: blr
-
-define <4 x float> @bar(<4 x float>* %p) {
-entry:
-  %v = load <4 x float>, <4 x float>* %p, align 16
-  ret <4 x float> %v
-}
-
-; CHECK: @bar
-; CHECK: qvlfsx
-
diff --git a/llvm/test/CodeGen/PowerPC/qpx-s-sel.ll b/llvm/test/CodeGen/PowerPC/qpx-s-sel.ll
deleted file mode 100644
index 5d42b9a529953..0000000000000
--- a/llvm/test/CodeGen/PowerPC/qpx-s-sel.ll
+++ /dev/null
@@ -1,143 +0,0 @@
-; RUN: llc -verify-machineinstrs < %s -mcpu=a2q | FileCheck %s
-target triple = "powerpc64-bgq-linux"
-
-@R = global <4 x i1> <i1 0, i1 0, i1 0, i1 0>, align 16
-
-define <4 x float> @test1(<4 x float> %a, <4 x float> %b, <4 x i1> %c) nounwind readnone {
-entry:
-  %r = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
-  ret <4 x float> %r
-
-; CHECK-LABEL: @test1
-; CHECK: qvfsel 1, 3, 1, 2
-; CHECK: blr
-}
-
-define <4 x float> @test2(<4 x float> %a, <4 x float> %b, i1 %c1, i1 %c2, i1 %c3, i1 %c4) nounwind readnone {
-entry:
-  %v = insertelement <4 x i1> undef, i1 %c1, i32 0
-  %v2 = insertelement <4 x i1> %v, i1 %c2, i32 1
-  %v3 = insertelement <4 x i1> %v2, i1 %c3, i32 2
-  %v4 = insertelement <4 x i1> %v3, i1 %c4, i32 3
-  %r = select <4 x i1> %v4, <4 x float> %a, <4 x float> %b
-  ret <4 x float> %r
-
-; CHECK-LABEL: @test2
-; CHECK: stw
-; CHECK-DAG: qvlfiwzx [[REG1:[0-9]+]],
-; CHECK-DAG: qvlfdx [[REG2:[0-9]+]],
-; CHECK-DAG: qvfcfidu [[REG3:[0-9]+]], [[REG1]]
-; CHECK: qvfcmpeq [[REG4:[0-9]+]], [[REG3]], [[REG2]]
-; CHECK: qvfsel 1, [[REG4]], 1, 2
-; CHECK: blr
-}
-
-define <4 x i1> @test3(<4 x i1> %a) nounwind readnone {
-entry:
-  %v = and <4 x i1> %a, <i1 0, i1 undef, i1 1, i1 1>
-  ret <4 x i1> %v
-
-; CHECK-LABEL: @test3
-; CHECK: qvlfsx [[REG:[0-9]+]],
-; qvflogical 1, 1, [[REG]], 1
-; blr
-}
-
-define <4 x i1> @test4(<4 x i1> %a, <4 x i1>* %t) nounwind {
-entry:
-  %q = load <4 x i1>, <4 x i1>* %t, align 16
-  %v = and <4 x i1> %a, %q
-  ret <4 x i1> %v
-
-; CHECK-LABEL: @test4
-; CHECK-DAG: lbz
-; CHECK-DAG: qvlfdx [[REG1:[0-9]+]],
-; CHECK-DAG: stw
-; CHECK-DAG: qvlfiwzx [[REG2:[0-9]+]],
-; CHECK-DAG: qvfcfidu [[REG3:[0-9]+]], [[REG2]]
-; CHECK: qvfcmpeq [[REG4:[0-9]+]], [[REG3]], [[REG1]]
-; CHECK: qvfand 1, 1, [[REG4]]
-; CHECK: blr
-}
-
-define void @test5(<4 x i1> %a) nounwind {
-entry:
-  store <4 x i1> %a, <4 x i1>* @R
-  ret void
-
-; CHECK-LABEL: @test5
-; CHECK: qvlfdx [[REG1:[0-9]+]],
-; CHECK: qvfmadd [[REG2:[0-9]+]], 1, [[REG1]], [[REG1]]
-; CHECK: qvfctiwu [[REG3:[0-9]+]], [[REG2]]
-; CHECK: qvstfiwx [[REG3]],
-; CHECK: lwz
-; CHECK: stb
-; CHECK: blr
-}
-
-define i1 @test6(<4 x i1> %a) nounwind {
-entry:
-  %r = extractelement <4 x i1> %a, i32 2
-  ret i1 %r
-
-; CHECK-LABEL: @test6
-; CHECK: qvlfdx [[REG1:[0-9]+]],
-; CHECK: qvfmadd [[REG2:[0-9]+]], 1, [[REG1]], [[REG1]]
-; CHECK: qvfctiwu [[REG3:[0-9]+]], [[REG2]]
-; CHECK: qvstfiwx [[REG3]],
-; CHECK: lwz
-; CHECK: blr
-}
-
-define i1 @test7(<4 x i1> %a) nounwind {
-entry:
-  %r = extractelement <4 x i1> %a, i32 2
-  %s = extractelement <4 x i1> %a, i32 3
-  %q = and i1 %r, %s
-  ret i1 %q
-
-; CHECK-LABEL: @test7
-; CHECK: qvlfdx [[REG1:[0-9]+]],
-; CHECK: qvfmadd [[REG2:[0-9]+]], 1, [[REG1]], [[REG1]]
-; CHECK: qvfctiwu [[REG3:[0-9]+]], [[REG2]]
-; CHECK: qvstfiwx [[REG3]],
-; CHECK-DAG: lwz [[REG4:[0-9]+]],
-; FIXME: We're storing the vector twice, and that's silly.
-; CHECK-DAG: qvstfiwx [[REG3]],
-; CHECK: lwz [[REG5:[0-9]+]],
-; CHECK: and 3,
-; CHECK: blr
-}
-
-define i1 @test8(<3 x i1> %a) nounwind {
-entry:
-  %r = extractelement <3 x i1> %a, i32 2
-  ret i1 %r
-
-; CHECK-LABEL: @test8
-; CHECK: qvlfdx [[REG1:[0-9]+]],
-; CHECK: qvfmadd [[REG2:[0-9]+]], 1, [[REG1]], [[REG1]]
-; CHECK: qvfctiwu [[REG3:[0-9]+]], [[REG2]]
-; CHECK: qvstfiwx [[REG3]],
-; CHECK: lwz
-; CHECK: blr
-}
-
-define <3 x float> @test9(<3 x float> %a, <3 x float> %b, i1 %c1, i1 %c2, i1 %c3) nounwind readnone {
-entry:
-  %v = insertelement <3 x i1> undef, i1 %c1, i32 0
-  %v2 = insertelement <3 x i1> %v, i1 %c2, i32 1
-  %v3 = insertelement <3 x i1> %v2, i1 %c3, i32 2
-  %r = select <3 x i1> %v3, <3 x float> %a, <3 x float> %b
-  ret <3 x float> %r
-
-; CHECK-LABEL: @test9
-; CHECK: stw
-; CHECK-DAG: qvlfiwzx [[REG1:[0-9]+]],
-; CHECK-DAG: qvlfdx [[REG2:[0-9]+]],
-; CHECK-DAG: qvfcfidu [[REG3:[0-9]+]], [[REG1]]
-; CHECK: qvfcmpeq [[REG4:[0-9]+]], [[REG3]], [[REG2]]
-; CHECK: qvfsel 1, [[REG4]], 1, 2
-; CHECK: blr
-}
-
diff --git a/llvm/test/CodeGen/PowerPC/qpx-s-store.ll b/llvm/test/CodeGen/PowerPC/qpx-s-store.ll
deleted file mode 100644
index 81cff7b6457f1..0000000000000
--- a/llvm/test/CodeGen/PowerPC/qpx-s-store.ll
+++ /dev/null
@@ -1,25 +0,0 @@
-; RUN: llc -verify-machineinstrs < %s -mcpu=a2q | FileCheck %s
-target triple = "powerpc64-bgq-linux"
-
-define void @foo(<4 x float> %v, <4 x float>* %p) {
-entry:
-  store <4 x float> %v, <4 x float>* %p, align 4
-  ret void
-}
-
-; CHECK: @foo
-; CHECK: stfs
-; CHECK: stfs
-; CHECK: stfs
-; CHECK: stfs
-; CHECK: blr
-
-define void @bar(<4 x float> %v, <4 x float>* %p) {
-entry:
-  store <4 x float> %v, <4 x float>* %p, align 16
-  ret void
-}
-
-; CHECK: @bar
-; CHECK: qvstfsx
-
diff --git a/llvm/test/CodeGen/PowerPC/qpx-sel.ll b/llvm/test/CodeGen/PowerPC/qpx-sel.ll
deleted file mode 100644
index abc92c9e98b13..0000000000000
--- a/llvm/test/CodeGen/PowerPC/qpx-sel.ll
+++ /dev/null
@@ -1,151 +0,0 @@
-; RUN: llc -verify-machineinstrs < %s -mcpu=a2q | FileCheck %s
-target triple = "powerpc64-bgq-linux"
-
-@R = global <4 x i1> <i1 0, i1 0, i1 0, i1 0>, align 16
-
-define <4 x double> @test1(<4 x double> %a, <4 x double> %b, <4 x i1> %c) nounwind readnone {
-entry:
-  %r = select <4 x i1> %c, <4 x double> %a, <4 x double> %b
-  ret <4 x double> %r
-
-; CHECK-LABEL: @test1
-; CHECK: qvfsel 1, 3, 1, 2
-; CHECK: blr
-}
-
-define <4 x double> @test2(<4 x double> %a, <4 x double> %b, i1 %c1, i1 %c2, i1 %c3, i1 %c4) nounwind readnone {
-entry:
-  %v = insertelement <4 x i1> undef, i1 %c1, i32 0
-  %v2 = insertelement <4 x i1> %v, i1 %c2, i32 1
-  %v3 = insertelement <4 x i1> %v2, i1 %c3, i32 2
-  %v4 = insertelement <4 x i1> %v3, i1 %c4, i32 3
-  %r = select <4 x i1> %v4, <4 x double> %a, <4 x double> %b
-  ret <4 x double> %r
-
-; CHECK-LABEL: @test2
-
-; FIXME: This load/store sequence is unnecessary.
-; CHECK-DAG: lbz
-; CHECK-DAG: stw
-
-; CHECK-DAG: qvlfiwzx [[REG1:[0-9]+]],
-; CHECK-DAG: qvlfdx [[REG2:[0-9]+]],
-; CHECK-DAG: qvfcfidu [[REG3:[0-9]+]], [[REG1]]
-; CHECK: qvfcmpeq [[REG4:[0-9]+]], [[REG3]], [[REG2]]
-; CHECK: qvfsel 1, [[REG4]], 1, 2
-; CHECK: blr
-}
-
-define <4 x i1> @test3(<4 x i1> %a) nounwind readnone {
-entry:
-  %v = and <4 x i1> %a, <i1 0, i1 undef, i1 1, i1 1>
-  ret <4 x i1> %v
-
-; CHECK-LABEL: @test3
-; CHECK: qvlfsx [[REG:[0-9]+]],
-; qvflogical 1, 1, [[REG]], 1
-; blr
-}
-
-define <4 x i1> @test4(<4 x i1> %a, <4 x i1>* %t) nounwind {
-entry:
-  %q = load <4 x i1>, <4 x i1>* %t, align 16
-  %v = and <4 x i1> %a, %q
-  ret <4 x i1> %v
-
-; CHECK-LABEL: @test4
-; CHECK-DAG: lbz
-; CHECK-DAG: qvlfdx [[REG1:[0-9]+]],
-; CHECK-DAG: stw
-; CHECK-DAG: qvlfiwzx [[REG2:[0-9]+]],
-; CHECK-DAG: qvfcfidu [[REG3:[0-9]+]], [[REG2]]
-; CHECK: qvfcmpeq [[REG4:[0-9]+]], [[REG3]], [[REG1]]
-; CHECK: qvfand 1, 1, [[REG4]]
-; CHECK: blr
-}
-
-define void @test5(<4 x i1> %a) nounwind {
-entry:
-  store <4 x i1> %a, <4 x i1>* @R
-  ret void
-
-; CHECK-LABEL: @test5
-; CHECK: qvlfdx [[REG1:[0-9]+]],
-; CHECK: qvfmadd [[REG2:[0-9]+]], 1, [[REG1]], [[REG1]]
-; CHECK: qvfctiwu [[REG3:[0-9]+]], [[REG2]]
-; CHECK: qvstfiwx [[REG3]],
-; CHECK: lwz
-; CHECK: stb
-; CHECK: blr
-}
-
-define i1 @test6(<4 x i1> %a) nounwind {
-entry:
-  %r = extractelement <4 x i1> %a, i32 2
-  ret i1 %r
-
-; CHECK-LABEL: @test6
-; CHECK: qvlfdx [[REG1:[0-9]+]],
-; CHECK: qvfmadd [[REG2:[0-9]+]], 1, [[REG1]], [[REG1]]
-; CHECK: qvfctiwu [[REG3:[0-9]+]], [[REG2]]
-; CHECK: qvstfiwx [[REG3]],
-; CHECK: lwz
-; CHECK: blr
-}
-
-define i1 @test7(<4 x i1> %a) nounwind {
-entry:
-  %r = extractelement <4 x i1> %a, i32 2
-  %s = extractelement <4 x i1> %a, i32 3
-  %q = and i1 %r, %s
-  ret i1 %q
-
-; CHECK-LABEL: @test7
-; CHECK: qvlfdx [[REG1:[0-9]+]],
-; CHECK: qvfmadd [[REG2:[0-9]+]], 1, [[REG1]], [[REG1]]
-; CHECK: qvfctiwu [[REG3:[0-9]+]], [[REG2]]
-; CHECK: qvstfiwx [[REG3]],
-; CHECK-DAG: lwz [[REG4:[0-9]+]],
-; FIXME: We're storing the vector twice, and that's silly.
-; CHECK-DAG: qvstfiwx [[REG3]],
-; CHECK-DAG: lwz [[REG5:[0-9]+]],
-; CHECK: and 3,
-; CHECK: blr
-}
-
-define i1 @test8(<3 x i1> %a) nounwind {
-entry:
-  %r = extractelement <3 x i1> %a, i32 2
-  ret i1 %r
-
-; CHECK-LABEL: @test8
-; CHECK: qvlfdx [[REG1:[0-9]+]],
-; CHECK: qvfmadd [[REG2:[0-9]+]], 1, [[REG1]], [[REG1]]
-; CHECK: qvfctiwu [[REG3:[0-9]+]], [[REG2]]
-; CHECK: qvstfiwx [[REG3]],
-; CHECK: lwz
-; CHECK: blr
-}
-
-define <3 x double> @test9(<3 x double> %a, <3 x double> %b, i1 %c1, i1 %c2, i1 %c3) nounwind readnone {
-entry:
-  %v = insertelement <3 x i1> undef, i1 %c1, i32 0
-  %v2 = insertelement <3 x i1> %v, i1 %c2, i32 1
-  %v3 = insertelement <3 x i1> %v2, i1 %c3, i32 2
-  %r = select <3 x i1> %v3, <3 x double> %a, <3 x double> %b
-  ret <3 x double> %r
-
-; CHECK-LABEL: @test9
-
-; FIXME: This load/store sequence is unnecessary.
-; CHECK-DAG: lbz
-; CHECK-DAG: stw
-
-; CHECK-DAG: qvlfiwzx [[REG1:[0-9]+]],
-; CHECK-DAG: qvlfdx [[REG2:[0-9]+]],
-; CHECK-DAG: qvfcfidu [[REG3:[0-9]+]], [[REG1]]
-; CHECK: qvfcmpeq [[REG4:[0-9]+]], [[REG3]], [[REG2]]
-; CHECK: qvfsel 1, [[REG4]], 1, 2
-; CHECK: blr
-}
-
diff --git a/llvm/test/CodeGen/PowerPC/qpx-split-vsetcc.ll b/llvm/test/CodeGen/PowerPC/qpx-split-vsetcc.ll
deleted file mode 100644
index df3e0befaef8a..0000000000000
--- a/llvm/test/CodeGen/PowerPC/qpx-split-vsetcc.ll
+++ /dev/null
@@ -1,31 +0,0 @@
-; RUN: llc -verify-machineinstrs -mcpu=a2q < %s | FileCheck %s
-target datalayout = "E-m:e-i64:64-n32:64"
-target triple = "powerpc64-bgq-linux"
-
-; Function Attrs: nounwind
-define void @gsl_sf_legendre_Pl_deriv_array(<4 x i32> %inp1, <4 x double> %inp2) #0 {
-entry:
-  br label %vector.body198
-
-vector.body198:                                   ; preds = %vector.body198, %for.body46.lr.ph
-  %0 = icmp ne <4 x i32> %inp1, zeroinitializer
-  %1 = select <4 x i1> %0, <4 x double> <double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01>, <4 x double> <double -5.000000e-01, double -5.000000e-01, double -5.000000e-01, double -5.000000e-01>
-  %2 = fmul <4 x double> %inp2, %1
-  %3 = fmul <4 x double> %inp2, %2
-  %4 = fmul <4 x double> %3, %inp2
-  store <4 x double> %4, <4 x double>* undef, align 8
-  br label %return
-
-; CHECK-LABEL: @gsl_sf_legendre_Pl_deriv_array
-; CHECK: qvlfiwzx
-; CHECK: qvfcfidu
-; CHECK: qvfcmpeq
-; CHECK: qvfsel
-; CHECK: qvfmul
-
-return:                                           ; preds = %if.else.i
-  ret void
-}
-
-attributes #0 = { nounwind }
-
diff --git a/llvm/test/CodeGen/PowerPC/qpx-store.ll b/llvm/test/CodeGen/PowerPC/qpx-store.ll
deleted file mode 100644
index 2b96576ce4493..0000000000000
--- a/llvm/test/CodeGen/PowerPC/qpx-store.ll
+++ /dev/null
@@ -1,25 +0,0 @@
-; RUN: llc -verify-machineinstrs < %s -mcpu=a2q | FileCheck %s
-target triple = "powerpc64-bgq-linux"
-
-define void @foo(<4 x double> %v, <4 x double>* %p) {
-entry:
-  store <4 x double> %v, <4 x double>* %p, align 8
-  ret void
-}
-
-; CHECK: @foo
-; CHECK: stfd
-; CHECK: stfd
-; CHECK: stfd
-; CHECK: stfd
-; CHECK: blr
-
-define void @bar(<4 x double> %v, <4 x double>* %p) {
-entry:
-  store <4 x double> %v, <4 x double>* %p, align 32
-  ret void
-}
-
-; CHECK: @bar
-; CHECK: qvstfdx
-
diff --git a/llvm/test/CodeGen/PowerPC/qpx-unal-cons-lds.ll b/llvm/test/CodeGen/PowerPC/qpx-unal-cons-lds.ll
deleted file mode 100644
index e7ab92db6efc9..0000000000000
--- a/llvm/test/CodeGen/PowerPC/qpx-unal-cons-lds.ll
+++ /dev/null
@@ -1,217 +0,0 @@
-; RUN: llc -verify-machineinstrs < %s | FileCheck %s
-target datalayout = "E-m:e-i64:64-n32:64"
-target triple = "powerpc64-unknown-linux-gnu"
-
-; Function Attrs: nounwind
-define void @foo(double* noalias nocapture %a, double* noalias nocapture readonly %b) #0 {
-entry:
-  br label %vector.body
-
-; CHECK-LABEL: @foo
-; Make sure that the offset constants we use are all even (only the last should be odd).
-; CHECK-DAG: li {{[0-9]+}}, 1056
-; CHECK-DAG: li {{[0-9]+}}, 1088
-; CHECK-DAG: li {{[0-9]+}}, 1152
-; CHECK-DAG: li {{[0-9]+}}, 1216
-; CHECK-DAG: li {{[0-9]+}}, 1280
-; CHECK-DAG: li {{[0-9]+}}, 1344
-; CHECK-DAG: li {{[0-9]+}}, 1408
-; CHECK-DAG: li {{[0-9]+}}, 1472
-; CHECK-DAG: li {{[0-9]+}}, 1536
-; CHECK-DAG: li {{[0-9]+}}, 1600
-; CHECK-DAG: li {{[0-9]+}}, 1568
-; CHECK-DAG: li {{[0-9]+}}, 1664
-; CHECK-DAG: li {{[0-9]+}}, 1632
-; CHECK-DAG: li {{[0-9]+}}, 1728
-; CHECK-DAG: li {{[0-9]+}}, 1696
-; CHECK-DAG: li {{[0-9]+}}, 1792
-; CHECK-DAG: li {{[0-9]+}}, 1760
-; CHECK-DAG: li {{[0-9]+}}, 1856
-; CHECK-DAG: li {{[0-9]+}}, 1824
-; CHECK-DAG: li {{[0-9]+}}, 1920
-; CHECK-DAG: li {{[0-9]+}}, 1888
-; CHECK-DAG: li {{[0-9]+}}, 1984
-; CHECK-DAG: li {{[0-9]+}}, 1952
-; CHECK-DAG: li {{[0-9]+}}, 2016
-; CHECK-DAG: li {{[0-9]+}}, 1024
-; CHECK-DAG: li {{[0-9]+}}, 1120
-; CHECK-DAG: li {{[0-9]+}}, 1184
-; CHECK-DAG: li {{[0-9]+}}, 1248
-; CHECK-DAG: li {{[0-9]+}}, 1312
-; CHECK-DAG: li {{[0-9]+}}, 1376
-; CHECK-DAG: li {{[0-9]+}}, 1440
-; CHECK-DAG: li {{[0-9]+}}, 1504
-; CHECK-DAG: li {{[0-9]+}}, 2047
-; CHECK: blr
-
-vector.body:                                      ; preds = %vector.body, %entry
-  %index = phi i64 [ 0, %entry ], [ %index.next.15, %vector.body ]
-  %0 = shl i64 %index, 1
-  %1 = getelementptr inbounds double, double* %b, i64 %0
-  %2 = bitcast double* %1 to <8 x double>*
-  %wide.vec = load <8 x double>, <8 x double>* %2, align 8
-  %strided.vec = shufflevector <8 x double> %wide.vec, <8 x double> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-  %3 = fadd <4 x double> %strided.vec, <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
-  %4 = getelementptr inbounds double, double* %a, i64 %index
-  %5 = bitcast double* %4 to <4 x double>*
-  store <4 x double> %3, <4 x double>* %5, align 8
-  %index.next = or i64 %index, 4
-  %6 = shl i64 %index.next, 1
-  %7 = getelementptr inbounds double, double* %b, i64 %6
-  %8 = bitcast double* %7 to <8 x double>*
-  %wide.vec.1 = load <8 x double>, <8 x double>* %8, align 8
-  %strided.vec.1 = shufflevector <8 x double> %wide.vec.1, <8 x double> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-  %9 = fadd <4 x double> %strided.vec.1, <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
-  %10 = getelementptr inbounds double, double* %a, i64 %index.next
-  %11 = bitcast double* %10 to <4 x double>*
-  store <4 x double> %9, <4 x double>* %11, align 8
-  %index.next.1 = or i64 %index, 8
-  %12 = shl i64 %index.next.1, 1
-  %13 = getelementptr inbounds double, double* %b, i64 %12
-  %14 = bitcast double* %13 to <8 x double>*
-  %wide.vec.2 = load <8 x double>, <8 x double>* %14, align 8
-  %strided.vec.2 = shufflevector <8 x double> %wide.vec.2, <8 x double> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-  %15 = fadd <4 x double> %strided.vec.2, <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
-  %16 = getelementptr inbounds double, double* %a, i64 %index.next.1
-  %17 = bitcast double* %16 to <4 x double>*
-  store <4 x double> %15, <4 x double>* %17, align 8
-  %index.next.2 = or i64 %index, 12
-  %18 = shl i64 %index.next.2, 1
-  %19 = getelementptr inbounds double, double* %b, i64 %18
-  %20 = bitcast double* %19 to <8 x double>*
-  %wide.vec.3 = load <8 x double>, <8 x double>* %20, align 8
-  %strided.vec.3 = shufflevector <8 x double> %wide.vec.3, <8 x double> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-  %21 = fadd <4 x double> %strided.vec.3, <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
-  %22 = getelementptr inbounds double, double* %a, i64 %index.next.2
-  %23 = bitcast double* %22 to <4 x double>*
-  store <4 x double> %21, <4 x double>* %23, align 8
-  %index.next.3 = or i64 %index, 16
-  %24 = shl i64 %index.next.3, 1
-  %25 = getelementptr inbounds double, double* %b, i64 %24
-  %26 = bitcast double* %25 to <8 x double>*
-  %wide.vec.4 = load <8 x double>, <8 x double>* %26, align 8
-  %strided.vec.4 = shufflevector <8 x double> %wide.vec.4, <8 x double> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-  %27 = fadd <4 x double> %strided.vec.4, <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
-  %28 = getelementptr inbounds double, double* %a, i64 %index.next.3
-  %29 = bitcast double* %28 to <4 x double>*
-  store <4 x double> %27, <4 x double>* %29, align 8
-  %index.next.4 = or i64 %index, 20
-  %30 = shl i64 %index.next.4, 1
-  %31 = getelementptr inbounds double, double* %b, i64 %30
-  %32 = bitcast double* %31 to <8 x double>*
-  %wide.vec.5 = load <8 x double>, <8 x double>* %32, align 8
-  %strided.vec.5 = shufflevector <8 x double> %wide.vec.5, <8 x double> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-  %33 = fadd <4 x double> %strided.vec.5, <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
-  %34 = getelementptr inbounds double, double* %a, i64 %index.next.4
-  %35 = bitcast double* %34 to <4 x double>*
-  store <4 x double> %33, <4 x double>* %35, align 8
-  %index.next.5 = or i64 %index, 24
-  %36 = shl i64 %index.next.5, 1
-  %37 = getelementptr inbounds double, double* %b, i64 %36
-  %38 = bitcast double* %37 to <8 x double>*
-  %wide.vec.6 = load <8 x double>, <8 x double>* %38, align 8
-  %strided.vec.6 = shufflevector <8 x double> %wide.vec.6, <8 x double> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-  %39 = fadd <4 x double> %strided.vec.6, <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
-  %40 = getelementptr inbounds double, double* %a, i64 %index.next.5
-  %41 = bitcast double* %40 to <4 x double>*
-  store <4 x double> %39, <4 x double>* %41, align 8
-  %index.next.6 = or i64 %index, 28
-  %42 = shl i64 %index.next.6, 1
-  %43 = getelementptr inbounds double, double* %b, i64 %42
-  %44 = bitcast double* %43 to <8 x double>*
-  %wide.vec.7 = load <8 x double>, <8 x double>* %44, align 8
-  %strided.vec.7 = shufflevector <8 x double> %wide.vec.7, <8 x double> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-  %45 = fadd <4 x double> %strided.vec.7, <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
-  %46 = getelementptr inbounds double, double* %a, i64 %index.next.6
-  %47 = bitcast double* %46 to <4 x double>*
-  store <4 x double> %45, <4 x double>* %47, align 8
-  %index.next.7 = or i64 %index, 32
-  %48 = shl i64 %index.next.7, 1
-  %49 = getelementptr inbounds double, double* %b, i64 %48
-  %50 = bitcast double* %49 to <8 x double>*
-  %wide.vec.8 = load <8 x double>, <8 x double>* %50, align 8
-  %strided.vec.8 = shufflevector <8 x double> %wide.vec.8, <8 x double> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-  %51 = fadd <4 x double> %strided.vec.8, <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
-  %52 = getelementptr inbounds double, double* %a, i64 %index.next.7
-  %53 = bitcast double* %52 to <4 x double>*
-  store <4 x double> %51, <4 x double>* %53, align 8
-  %index.next.8 = or i64 %index, 36
-  %54 = shl i64 %index.next.8, 1
-  %55 = getelementptr inbounds double, double* %b, i64 %54
-  %56 = bitcast double* %55 to <8 x double>*
-  %wide.vec.9 = load <8 x double>, <8 x double>* %56, align 8
-  %strided.vec.9 = shufflevector <8 x double> %wide.vec.9, <8 x double> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-  %57 = fadd <4 x double> %strided.vec.9, <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
-  %58 = getelementptr inbounds double, double* %a, i64 %index.next.8
-  %59 = bitcast double* %58 to <4 x double>*
-  store <4 x double> %57, <4 x double>* %59, align 8
-  %index.next.9 = or i64 %index, 40
-  %60 = shl i64 %index.next.9, 1
-  %61 = getelementptr inbounds double, double* %b, i64 %60
-  %62 = bitcast double* %61 to <8 x double>*
-  %wide.vec.10 = load <8 x double>, <8 x double>* %62, align 8
-  %strided.vec.10 = shufflevector <8 x double> %wide.vec.10, <8 x double> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-  %63 = fadd <4 x double> %strided.vec.10, <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
-  %64 = getelementptr inbounds double, double* %a, i64 %index.next.9
-  %65 = bitcast double* %64 to <4 x double>*
-  store <4 x double> %63, <4 x double>* %65, align 8
-  %index.next.10 = or i64 %index, 44
-  %66 = shl i64 %index.next.10, 1
-  %67 = getelementptr inbounds double, double* %b, i64 %66
-  %68 = bitcast double* %67 to <8 x double>*
-  %wide.vec.11 = load <8 x double>, <8 x double>* %68, align 8
-  %strided.vec.11 = shufflevector <8 x double> %wide.vec.11, <8 x double> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-  %69 = fadd <4 x double> %strided.vec.11, <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
-  %70 = getelementptr inbounds double, double* %a, i64 %index.next.10
-  %71 = bitcast double* %70 to <4 x double>*
-  store <4 x double> %69, <4 x double>* %71, align 8
-  %index.next.11 = or i64 %index, 48
-  %72 = shl i64 %index.next.11, 1
-  %73 = getelementptr inbounds double, double* %b, i64 %72
-  %74 = bitcast double* %73 to <8 x double>*
-  %wide.vec.12 = load <8 x double>, <8 x double>* %74, align 8
-  %strided.vec.12 = shufflevector <8 x double> %wide.vec.12, <8 x double> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-  %75 = fadd <4 x double> %strided.vec.12, <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
-  %76 = getelementptr inbounds double, double* %a, i64 %index.next.11
-  %77 = bitcast double* %76 to <4 x double>*
-  store <4 x double> %75, <4 x double>* %77, align 8
-  %index.next.12 = or i64 %index, 52
-  %78 = shl i64 %index.next.12, 1
-  %79 = getelementptr inbounds double, double* %b, i64 %78
-  %80 = bitcast double* %79 to <8 x double>*
-  %wide.vec.13 = load <8 x double>, <8 x double>* %80, align 8
-  %strided.vec.13 = shufflevector <8 x double> %wide.vec.13, <8 x double> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-  %81 = fadd <4 x double> %strided.vec.13, <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
-  %82 = getelementptr inbounds double, double* %a, i64 %index.next.12
-  %83 = bitcast double* %82 to <4 x double>*
-  store <4 x double> %81, <4 x double>* %83, align 8
-  %index.next.13 = or i64 %index, 56
-  %84 = shl i64 %index.next.13, 1
-  %85 = getelementptr inbounds double, double* %b, i64 %84
-  %86 = bitcast double* %85 to <8 x double>*
-  %wide.vec.14 = load <8 x double>, <8 x double>* %86, align 8
-  %strided.vec.14 = shufflevector <8 x double> %wide.vec.14, <8 x double> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-  %87 = fadd <4 x double> %strided.vec.14, <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
-  %88 = getelementptr inbounds double, double* %a, i64 %index.next.13
-  %89 = bitcast double* %88 to <4 x double>*
-  store <4 x double> %87, <4 x double>* %89, align 8
-  %index.next.14 = or i64 %index, 60
-  %90 = shl i64 %index.next.14, 1
-  %91 = getelementptr inbounds double, double* %b, i64 %90
-  %92 = bitcast double* %91 to <8 x double>*
-  %wide.vec.15 = load <8 x double>, <8 x double>* %92, align 8
-  %strided.vec.15 = shufflevector <8 x double> %wide.vec.15, <8 x double> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-  %93 = fadd <4 x double> %strided.vec.15, <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
-  %94 = getelementptr inbounds double, double* %a, i64 %index.next.14
-  %95 = bitcast double* %94 to <4 x double>*
-  store <4 x double> %93, <4 x double>* %95, align 8
-  %index.next.15 = add nsw i64 %index, 64
-  %96 = icmp eq i64 %index.next.15, 1600
-  br i1 %96, label %for.cond.cleanup, label %vector.body
-
-for.cond.cleanup:                                 ; preds = %vector.body
-  ret void
-}
-
-attributes #0 = { nounwind "target-cpu"="a2q" }
-
diff --git a/llvm/test/CodeGen/PowerPC/qpx-unalperm.ll b/llvm/test/CodeGen/PowerPC/qpx-unalperm.ll
deleted file mode 100644
index fdee919fdfc32..0000000000000
--- a/llvm/test/CodeGen/PowerPC/qpx-unalperm.ll
+++ /dev/null
@@ -1,64 +0,0 @@
-; RUN: llc -verify-machineinstrs < %s -mcpu=a2q | FileCheck %s
-target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
-target triple = "powerpc64-bgq-linux"
-
-define <4 x double> @foo(<4 x double>* %a) {
-entry:
-  %r = load <4 x double>, <4 x double>* %a, align 32
-  ret <4 x double> %r
-; CHECK: qvlfdx
-; CHECK: blr
-}
-
-define <4 x double> @bar(<4 x double>* %a) {
-entry:
-  %r = load <4 x double>, <4 x double>* %a, align 8
-  %b = getelementptr <4 x double>, <4 x double>* %a, i32 16
-  %s = load <4 x double>, <4 x double>* %b, align 32
-  %t = fadd <4 x double> %r, %s
-  ret <4 x double> %t
-; CHECK: qvlpcldx
-; CHECK: qvlfdx
-; CHECK: qvfperm
-; CHECK: blr
-}
-
-define <4 x double> @bar1(<4 x double>* %a) {
-entry:
-  %r = load <4 x double>, <4 x double>* %a, align 8
-  %b = getelementptr <4 x double>, <4 x double>* %a, i32 16
-  %s = load <4 x double>, <4 x double>* %b, align 8
-  %t = fadd <4 x double> %r, %s
-  ret <4 x double> %t
-}
-
-define <4 x double> @bar2(<4 x double>* %a) {
-entry:
-  %r = load <4 x double>, <4 x double>* %a, align 8
-  %b = getelementptr <4 x double>, <4 x double>* %a, i32 1
-  %s = load <4 x double>, <4 x double>* %b, align 32
-  %t = fadd <4 x double> %r, %s
-  ret <4 x double> %t
-}
-
-define <4 x double> @bar3(<4 x double>* %a) {
-entry:
-  %r = load <4 x double>, <4 x double>* %a, align 8
-  %b = getelementptr <4 x double>, <4 x double>* %a, i32 1
-  %s = load <4 x double>, <4 x double>* %b, align 8
-  %t = fadd <4 x double> %r, %s
-  ret <4 x double> %t
-}
-
-define <4 x double> @bar4(<4 x double>* %a) {
-entry:
-  %r = load <4 x double>, <4 x double>* %a, align 8
-  %b = getelementptr <4 x double>, <4 x double>* %a, i32 1
-  %s = load <4 x double>, <4 x double>* %b, align 8
-  %c = getelementptr <4 x double>, <4 x double>* %b, i32 1
-  %t = load <4 x double>, <4 x double>* %c, align 8
-  %u = fadd <4 x double> %r, %s
-  %v = fadd <4 x double> %u, %t
-  ret <4 x double> %v
-}
-
diff --git a/llvm/test/CodeGen/PowerPC/rlwimi-and.ll b/llvm/test/CodeGen/PowerPC/rlwimi-and.ll
index e8fc409527588..d512f51a76e7a 100644
--- a/llvm/test/CodeGen/PowerPC/rlwimi-and.ll
+++ b/llvm/test/CodeGen/PowerPC/rlwimi-and.ll
@@ -1,6 +1,4 @@
-; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mattr=-crbits < %s | FileCheck %s
-target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
-target triple = "powerpc64-bgq-linux"
+; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mattr=-crbits -mtriple=powerpc64-unknown-linux-gnu < %s | FileCheck %s
 
 define void @test() align 2 {
 entry:
diff --git a/llvm/test/CodeGen/PowerPC/rlwinm_rldicl_to_andi.mir b/llvm/test/CodeGen/PowerPC/rlwinm_rldicl_to_andi.mir
index e3aeb5605b42c..dbe314b5251fe 100644
--- a/llvm/test/CodeGen/PowerPC/rlwinm_rldicl_to_andi.mir
+++ b/llvm/test/CodeGen/PowerPC/rlwinm_rldicl_to_andi.mir
@@ -60,7 +60,7 @@
     ret i64 %cond
   }
   
-  attributes #0 = { norecurse nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+  attributes #0 = { norecurse nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector" "unsafe-fp-math"="false" "use-soft-float"="false" }
   
   !llvm.module.flags = !{!0, !1}
   !llvm.ident = !{!2}
diff --git a/llvm/test/CodeGen/PowerPC/s000-alias-misched.ll b/llvm/test/CodeGen/PowerPC/s000-alias-misched.ll
index 5c15145af2378..20071ea1710c5 100644
--- a/llvm/test/CodeGen/PowerPC/s000-alias-misched.ll
+++ b/llvm/test/CodeGen/PowerPC/s000-alias-misched.ll
@@ -1,7 +1,6 @@
-; RUN: llc -verify-machineinstrs < %s -enable-misched -mcpu=a2 | FileCheck %s
-; RUN: llc -verify-machineinstrs < %s -enable-misched -enable-aa-sched-mi -mcpu=a2 | FileCheck %s
+; RUN: llc -verify-machineinstrs < %s -enable-misched -mcpu=a2 -mtriple=powerpc64-unknown-linux-gnu | FileCheck %s
+; RUN: llc -verify-machineinstrs < %s -enable-misched -enable-aa-sched-mi -mcpu=a2 -mtriple=powerpc64-unknown-linux-gnu | FileCheck %s
 target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
-target triple = "powerpc64-bgq-linux"
 
 @aa = external global [256 x [256 x double]], align 32
 @bb = external global [256 x [256 x double]], align 32
diff --git a/llvm/test/CodeGen/PowerPC/select-i1-vs-i1.ll b/llvm/test/CodeGen/PowerPC/select-i1-vs-i1.ll
index 80ac733156197..9f458ebcf0a6e 100644
--- a/llvm/test/CodeGen/PowerPC/select-i1-vs-i1.ll
+++ b/llvm/test/CodeGen/PowerPC/select-i1-vs-i1.ll
@@ -1225,576 +1225,5 @@ entry:
 ; CHECK: blr
 }
 
-define <4 x double> @testqv4doubleslt(float %c1, float %c2, float %c3, float %c4, <4 x double> %a1, <4 x double> %a2) #1 {
-entry:
-  %cmp1 = fcmp oeq float %c3, %c4
-  %cmp3tmp = fcmp oeq float %c1, %c2
-  %cmp3 = icmp slt i1 %cmp3tmp, %cmp1
-  %cond = select i1 %cmp3, <4 x double> %a1, <4 x double> %a2
-  ret <4 x double> %cond
-
-; CHECK-LABEL: @testqv4doubleslt
-; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
-; CHECK: bc 12, 2, .LBB[[BB1:[0-9_]+]]
-; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: bc 12, 2, .LBB[[BB2:[0-9_]+]]
-; CHECK: .LBB[[BB1]]:
-; CHECK: qvfmr 5, 6
-; CHECK: .LBB[[BB2]]:
-; CHECK: qvfmr 1, 5
-; CHECK: blr
-}
-
-define <4 x double> @testqv4doubleult(float %c1, float %c2, float %c3, float %c4, <4 x double> %a1, <4 x double> %a2) #1 {
-entry:
-  %cmp1 = fcmp oeq float %c3, %c4
-  %cmp3tmp = fcmp oeq float %c1, %c2
-  %cmp3 = icmp ult i1 %cmp3tmp, %cmp1
-  %cond = select i1 %cmp3, <4 x double> %a1, <4 x double> %a2
-  ret <4 x double> %cond
-
-; CHECK-LABEL: @testqv4doubleult
-; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
-; CHECK: bc 4, 2, .LBB[[BB1:[0-9_]+]]
-; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: bc 4, 2, .LBB[[BB2:[0-9_]+]]
-; CHECK: .LBB[[BB1]]:
-; CHECK: qvfmr 5, 6
-; CHECK: .LBB[[BB2]]:
-; CHECK: qvfmr 1, 5
-; CHECK: blr
-}
-
-define <4 x double> @testqv4doublesle(float %c1, float %c2, float %c3, float %c4, <4 x double> %a1, <4 x double> %a2) #1 {
-entry:
-  %cmp1 = fcmp oeq float %c3, %c4
-  %cmp3tmp = fcmp oeq float %c1, %c2
-  %cmp3 = icmp sle i1 %cmp3tmp, %cmp1
-  %cond = select i1 %cmp3, <4 x double> %a1, <4 x double> %a2
-  ret <4 x double> %cond
-
-; CHECK-LABEL: @testqv4doublesle
-; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
-; CHECK: bc 4, 2, .LBB[[BB:[0-9_]+]]
-; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: bc 12, 2, .LBB[[BB]]
-; CHECK: qvfmr 5, 6
-; CHECK: .LBB[[BB]]:
-; CHECK: qvfmr 1, 5
-; CHECK: blr
-}
-
-define <4 x double> @testqv4doubleule(float %c1, float %c2, float %c3, float %c4, <4 x double> %a1, <4 x double> %a2) #1 {
-entry:
-  %cmp1 = fcmp oeq float %c3, %c4
-  %cmp3tmp = fcmp oeq float %c1, %c2
-  %cmp3 = icmp ule i1 %cmp3tmp, %cmp1
-  %cond = select i1 %cmp3, <4 x double> %a1, <4 x double> %a2
-  ret <4 x double> %cond
-
-; CHECK-LABEL: @testqv4doubleule
-; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
-; CHECK: bc 12, 2, .LBB[[BB:[0-9_]+]]
-; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: bc 4, 2, .LBB[[BB]]
-; CHECK: qvfmr 5, 6
-; CHECK: .LBB[[BB]]:
-; CHECK: qvfmr 1, 5
-; CHECK: blr
-}
-
-define <4 x double> @testqv4doubleeq(float %c1, float %c2, float %c3, float %c4, <4 x double> %a1, <4 x double> %a2) #1 {
-entry:
-  %cmp1 = fcmp oeq float %c3, %c4
-  %cmp3tmp = fcmp oeq float %c1, %c2
-  %cmp3 = icmp eq i1 %cmp3tmp, %cmp1
-  %cond = select i1 %cmp3, <4 x double> %a1, <4 x double> %a2
-  ret <4 x double> %cond
-
-; CHECK-LABEL: @testqv4doubleeq
-; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
-; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: creqv [[REG1:[0-9]+]], {{[0-9]+}}, {{[0-9]+}}
-; CHECK: bclr 12, [[REG1]], 0
-; CHECK: qvfmr 1, 6
-; CHECK: blr
-}
-
-define <4 x double> @testqv4doublesge(float %c1, float %c2, float %c3, float %c4, <4 x double> %a1, <4 x double> %a2) #1 {
-entry:
-  %cmp1 = fcmp oeq float %c3, %c4
-  %cmp3tmp = fcmp oeq float %c1, %c2
-  %cmp3 = icmp sge i1 %cmp3tmp, %cmp1
-  %cond = select i1 %cmp3, <4 x double> %a1, <4 x double> %a2
-  ret <4 x double> %cond
-
-; CHECK-LABEL: @testqv4doublesge
-; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
-; CHECK: bc 12, 2, .LBB[[BB:[0-9_]+]]
-; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: bc 4, 2, .LBB[[BB]]
-; CHECK: qvfmr 5, 6
-; CHECK: .LBB[[BB]]:
-; CHECK: qvfmr 1, 5
-; CHECK: blr
-}
-
-define <4 x double> @testqv4doubleuge(float %c1, float %c2, float %c3, float %c4, <4 x double> %a1, <4 x double> %a2) #1 {
-entry:
-  %cmp1 = fcmp oeq float %c3, %c4
-  %cmp3tmp = fcmp oeq float %c1, %c2
-  %cmp3 = icmp uge i1 %cmp3tmp, %cmp1
-  %cond = select i1 %cmp3, <4 x double> %a1, <4 x double> %a2
-  ret <4 x double> %cond
-
-; CHECK-LABEL: @testqv4doubleuge
-; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
-; CHECK: bc 4, 2, .LBB[[BB:[0-9_]+]]
-; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: bc 12, 2, .LBB[[BB]]
-; CHECK: qvfmr 5, 6
-; CHECK: .LBB[[BB]]:
-; CHECK: qvfmr 1, 5
-; CHECK: blr
-}
-
-define <4 x double> @testqv4doublesgt(float %c1, float %c2, float %c3, float %c4, <4 x double> %a1, <4 x double> %a2) #1 {
-entry:
-  %cmp1 = fcmp oeq float %c3, %c4
-  %cmp3tmp = fcmp oeq float %c1, %c2
-  %cmp3 = icmp sgt i1 %cmp3tmp, %cmp1
-  %cond = select i1 %cmp3, <4 x double> %a1, <4 x double> %a2
-  ret <4 x double> %cond
-
-; CHECK-LABEL: @testqv4doublesgt
-; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
-; CHECK: bc 4, 2, .LBB[[BB1:[0-9_]+]]
-; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: bc 4, 2, .LBB[[BB2:[0-9_]+]]
-; CHECK: .LBB[[BB1]]:
-; CHECK: qvfmr 5, 6
-; CHECK: .LBB[[BB2]]:
-; CHECK: qvfmr 1, 5
-; CHECK: blr
-}
-
-define <4 x double> @testqv4doubleugt(float %c1, float %c2, float %c3, float %c4, <4 x double> %a1, <4 x double> %a2) #1 {
-entry:
-  %cmp1 = fcmp oeq float %c3, %c4
-  %cmp3tmp = fcmp oeq float %c1, %c2
-  %cmp3 = icmp ugt i1 %cmp3tmp, %cmp1
-  %cond = select i1 %cmp3, <4 x double> %a1, <4 x double> %a2
-  ret <4 x double> %cond
-
-; CHECK-LABEL: @testqv4doubleugt
-; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
-; CHECK: bc 12, 2, .LBB[[BB1:[0-9_]+]]
-; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: bc 12, 2, .LBB[[BB2:[0-9_]+]]
-; CHECK: .LBB[[BB1]]:
-; CHECK: qvfmr 5, 6
-; CHECK: .LBB[[BB2]]:
-; CHECK: qvfmr 1, 5
-; CHECK: blr
-}
-
-define <4 x double> @testqv4doublene(float %c1, float %c2, float %c3, float %c4, <4 x double> %a1, <4 x double> %a2) #1 {
-entry:
-  %cmp1 = fcmp oeq float %c3, %c4
-  %cmp3tmp = fcmp oeq float %c1, %c2
-  %cmp3 = icmp ne i1 %cmp3tmp, %cmp1
-  %cond = select i1 %cmp3, <4 x double> %a1, <4 x double> %a2
-  ret <4 x double> %cond
-
-; CHECK-LABEL: @testqv4doublene
-; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
-; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: crxor [[REG1:[0-9]+]], {{[0-9]+}}, {{[0-9]+}}
-; CHECK: bclr 12, [[REG1]], 0
-; CHECK: qvfmr 1, 6
-; CHECK: blr
-}
-
-define <4 x float> @testqv4floatslt(float %c1, float %c2, float %c3, float %c4, <4 x float> %a1, <4 x float> %a2) #1 {
-entry:
-  %cmp1 = fcmp oeq float %c3, %c4
-  %cmp3tmp = fcmp oeq float %c1, %c2
-  %cmp3 = icmp slt i1 %cmp3tmp, %cmp1
-  %cond = select i1 %cmp3, <4 x float> %a1, <4 x float> %a2
-  ret <4 x float> %cond
-
-; CHECK-LABEL: @testqv4floatslt
-; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
-; CHECK: bc 12, 2, .LBB[[BB1:[0-9_]+]]
-; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: bc 12, 2, .LBB[[BB2:[0-9_]+]]
-; CHECK: .LBB[[BB1]]:
-; CHECK: qvfmr 5, 6
-; CHECK: .LBB[[BB2]]:
-; CHECK: qvfmr 1, 5
-; CHECK: blr
-}
-
-define <4 x float> @testqv4floatult(float %c1, float %c2, float %c3, float %c4, <4 x float> %a1, <4 x float> %a2) #1 {
-entry:
-  %cmp1 = fcmp oeq float %c3, %c4
-  %cmp3tmp = fcmp oeq float %c1, %c2
-  %cmp3 = icmp ult i1 %cmp3tmp, %cmp1
-  %cond = select i1 %cmp3, <4 x float> %a1, <4 x float> %a2
-  ret <4 x float> %cond
-
-; CHECK-LABEL: @testqv4floatult
-; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
-; CHECK: bc 4, 2, .LBB[[BB1:[0-9_]+]]
-; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: bc 4, 2, .LBB[[BB2:[0-9_]+]]
-; CHECK: .LBB[[BB1]]:
-; CHECK: qvfmr 5, 6
-; CHECK: .LBB[[BB2]]:
-; CHECK: qvfmr 1, 5
-; CHECK: blr
-}
-
-define <4 x float> @testqv4floatsle(float %c1, float %c2, float %c3, float %c4, <4 x float> %a1, <4 x float> %a2) #1 {
-entry:
-  %cmp1 = fcmp oeq float %c3, %c4
-  %cmp3tmp = fcmp oeq float %c1, %c2
-  %cmp3 = icmp sle i1 %cmp3tmp, %cmp1
-  %cond = select i1 %cmp3, <4 x float> %a1, <4 x float> %a2
-  ret <4 x float> %cond
-
-; CHECK-LABEL: @testqv4floatsle
-; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
-; CHECK: bc 4, 2, .LBB[[BB:[0-9_]+]]
-; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: bc 12, 2, .LBB[[BB]]
-; CHECK: qvfmr 5, 6
-; CHECK: .LBB[[BB]]:
-; CHECK: qvfmr 1, 5
-; CHECK: blr
-}
-
-define <4 x float> @testqv4floatule(float %c1, float %c2, float %c3, float %c4, <4 x float> %a1, <4 x float> %a2) #1 {
-entry:
-  %cmp1 = fcmp oeq float %c3, %c4
-  %cmp3tmp = fcmp oeq float %c1, %c2
-  %cmp3 = icmp ule i1 %cmp3tmp, %cmp1
-  %cond = select i1 %cmp3, <4 x float> %a1, <4 x float> %a2
-  ret <4 x float> %cond
-
-; CHECK-LABEL: @testqv4floatule
-; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
-; CHECK: bc 12, 2, .LBB[[BB:[0-9_]+]]
-; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: bc 4, 2, .LBB[[BB]]
-; CHECK: qvfmr 5, 6
-; CHECK: .LBB[[BB]]:
-; CHECK: qvfmr 1, 5
-; CHECK: blr
-}
-
-define <4 x float> @testqv4floateq(float %c1, float %c2, float %c3, float %c4, <4 x float> %a1, <4 x float> %a2) #1 {
-entry:
-  %cmp1 = fcmp oeq float %c3, %c4
-  %cmp3tmp = fcmp oeq float %c1, %c2
-  %cmp3 = icmp eq i1 %cmp3tmp, %cmp1
-  %cond = select i1 %cmp3, <4 x float> %a1, <4 x float> %a2
-  ret <4 x float> %cond
-
-; CHECK-LABEL: @testqv4floateq
-; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
-; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: creqv [[REG1:[0-9]+]], {{[0-9]+}}, {{[0-9]+}}
-; CHECK: bclr 12, [[REG1]], 0
-; CHECK: qvfmr 1, 6
-; CHECK: blr
-}
-
-define <4 x float> @testqv4floatsge(float %c1, float %c2, float %c3, float %c4, <4 x float> %a1, <4 x float> %a2) #1 {
-entry:
-  %cmp1 = fcmp oeq float %c3, %c4
-  %cmp3tmp = fcmp oeq float %c1, %c2
-  %cmp3 = icmp sge i1 %cmp3tmp, %cmp1
-  %cond = select i1 %cmp3, <4 x float> %a1, <4 x float> %a2
-  ret <4 x float> %cond
-
-; CHECK-LABEL: @testqv4floatsge
-; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
-; CHECK: bc 12, 2, .LBB[[BB:[0-9_]+]]
-; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: bc 4, 2, .LBB[[BB]]
-; CHECK: qvfmr 5, 6
-; CHECK: .LBB[[BB]]:
-; CHECK: qvfmr 1, 5
-; CHECK: blr
-}
-
-define <4 x float> @testqv4floatuge(float %c1, float %c2, float %c3, float %c4, <4 x float> %a1, <4 x float> %a2) #1 {
-entry:
-  %cmp1 = fcmp oeq float %c3, %c4
-  %cmp3tmp = fcmp oeq float %c1, %c2
-  %cmp3 = icmp uge i1 %cmp3tmp, %cmp1
-  %cond = select i1 %cmp3, <4 x float> %a1, <4 x float> %a2
-  ret <4 x float> %cond
-
-; CHECK-LABEL: @testqv4floatuge
-; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
-; CHECK: bc 4, 2, .LBB[[BB:[0-9_]+]]
-; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: bc 12, 2, .LBB[[BB]]
-; CHECK: qvfmr 5, 6
-; CHECK: .LBB[[BB]]:
-; CHECK: qvfmr 1, 5
-; CHECK: blr
-}
-
-define <4 x float> @testqv4floatsgt(float %c1, float %c2, float %c3, float %c4, <4 x float> %a1, <4 x float> %a2) #1 {
-entry:
-  %cmp1 = fcmp oeq float %c3, %c4
-  %cmp3tmp = fcmp oeq float %c1, %c2
-  %cmp3 = icmp sgt i1 %cmp3tmp, %cmp1
-  %cond = select i1 %cmp3, <4 x float> %a1, <4 x float> %a2
-  ret <4 x float> %cond
-
-; CHECK-LABEL: @testqv4floatsgt
-; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
-; CHECK: bc 4, 2, .LBB[[BB1:[0-9_]+]]
-; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: bc 4, 2, .LBB[[BB2:[0-9_]+]]
-; CHECK: .LBB[[BB1]]:
-; CHECK: qvfmr 5, 6
-; CHECK: .LBB[[BB2]]:
-; CHECK: qvfmr 1, 5
-; CHECK: blr
-}
-
-define <4 x float> @testqv4floatugt(float %c1, float %c2, float %c3, float %c4, <4 x float> %a1, <4 x float> %a2) #1 {
-entry:
-  %cmp1 = fcmp oeq float %c3, %c4
-  %cmp3tmp = fcmp oeq float %c1, %c2
-  %cmp3 = icmp ugt i1 %cmp3tmp, %cmp1
-  %cond = select i1 %cmp3, <4 x float> %a1, <4 x float> %a2
-  ret <4 x float> %cond
-
-; CHECK-LABEL: @testqv4floatugt
-; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
-; CHECK: bc 12, 2, .LBB[[BB1:[0-9_]+]]
-; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: bc 12, 2, .LBB[[BB2:[0-9_]+]]
-; CHECK: .LBB[[BB1]]:
-; CHECK: qvfmr 5, 6
-; CHECK: .LBB[[BB2]]:
-; CHECK: qvfmr 1, 5
-; CHECK: blr
-}
-
-define <4 x float> @testqv4floatne(float %c1, float %c2, float %c3, float %c4, <4 x float> %a1, <4 x float> %a2) #1 {
-entry:
-  %cmp1 = fcmp oeq float %c3, %c4
-  %cmp3tmp = fcmp oeq float %c1, %c2
-  %cmp3 = icmp ne i1 %cmp3tmp, %cmp1
-  %cond = select i1 %cmp3, <4 x float> %a1, <4 x float> %a2
-  ret <4 x float> %cond
-
-; CHECK-LABEL: @testqv4floatne
-; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
-; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: crxor [[REG1:[0-9]+]], {{[0-9]+}}, {{[0-9]+}}
-; CHECK: bclr 12, [[REG1]], 0
-; CHECK: qvfmr 1, 6
-; CHECK: blr
-}
-
-define <4 x i1> @testqv4i1slt(float %c1, float %c2, float %c3, float %c4, <4 x i1> %a1, <4 x i1> %a2) #1 {
-entry:
-  %cmp1 = fcmp oeq float %c3, %c4
-  %cmp3tmp = fcmp oeq float %c1, %c2
-  %cmp3 = icmp slt i1 %cmp3tmp, %cmp1
-  %cond = select i1 %cmp3, <4 x i1> %a1, <4 x i1> %a2
-  ret <4 x i1> %cond
-
-; CHECK-LABEL: @testqv4i1slt
-; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
-; CHECK: bc 12, 2, .LBB[[BB1:[0-9_]+]]
-; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: bc 12, 2, .LBB[[BB2:[0-9_]+]]
-; CHECK: .LBB[[BB1]]:
-; CHECK: qvfmr 5, 6
-; CHECK: .LBB[[BB2]]:
-; CHECK: qvfmr 1, 5
-; CHECK: blr
-}
-
-define <4 x i1> @testqv4i1ult(float %c1, float %c2, float %c3, float %c4, <4 x i1> %a1, <4 x i1> %a2) #1 {
-entry:
-  %cmp1 = fcmp oeq float %c3, %c4
-  %cmp3tmp = fcmp oeq float %c1, %c2
-  %cmp3 = icmp ult i1 %cmp3tmp, %cmp1
-  %cond = select i1 %cmp3, <4 x i1> %a1, <4 x i1> %a2
-  ret <4 x i1> %cond
-
-; CHECK-LABEL: @testqv4i1ult
-; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
-; CHECK: bc 4, 2, .LBB[[BB1:[0-9_]+]]
-; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: bc 4, 2, .LBB[[BB2:[0-9_]+]]
-; CHECK: .LBB[[BB1]]:
-; CHECK: qvfmr 5, 6
-; CHECK: .LBB[[BB2]]:
-; CHECK: qvfmr 1, 5
-; CHECK: blr
-}
-
-define <4 x i1> @testqv4i1sle(float %c1, float %c2, float %c3, float %c4, <4 x i1> %a1, <4 x i1> %a2) #1 {
-entry:
-  %cmp1 = fcmp oeq float %c3, %c4
-  %cmp3tmp = fcmp oeq float %c1, %c2
-  %cmp3 = icmp sle i1 %cmp3tmp, %cmp1
-  %cond = select i1 %cmp3, <4 x i1> %a1, <4 x i1> %a2
-  ret <4 x i1> %cond
-
-; CHECK-LABEL: @testqv4i1sle
-; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
-; CHECK: bc 4, 2, .LBB[[BB:[0-9_]+]]
-; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: bc 12, 2, .LBB[[BB]]
-; CHECK: qvfmr 5, 6
-; CHECK: .LBB[[BB]]:
-; CHECK: qvfmr 1, 5
-; CHECK: blr
-}
-
-define <4 x i1> @testqv4i1ule(float %c1, float %c2, float %c3, float %c4, <4 x i1> %a1, <4 x i1> %a2) #1 {
-entry:
-  %cmp1 = fcmp oeq float %c3, %c4
-  %cmp3tmp = fcmp oeq float %c1, %c2
-  %cmp3 = icmp ule i1 %cmp3tmp, %cmp1
-  %cond = select i1 %cmp3, <4 x i1> %a1, <4 x i1> %a2
-  ret <4 x i1> %cond
-
-; CHECK-LABEL: @testqv4i1ule
-; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
-; CHECK: bc 12, 2, .LBB[[BB:[0-9_]+]]
-; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: bc 4, 2, .LBB[[BB]]
-; CHECK: qvfmr 5, 6
-; CHECK: .LBB[[BB]]:
-; CHECK: qvfmr 1, 5
-; CHECK: blr
-}
-
-define <4 x i1> @testqv4i1eq(float %c1, float %c2, float %c3, float %c4, <4 x i1> %a1, <4 x i1> %a2) #1 {
-entry:
-  %cmp1 = fcmp oeq float %c3, %c4
-  %cmp3tmp = fcmp oeq float %c1, %c2
-  %cmp3 = icmp eq i1 %cmp3tmp, %cmp1
-  %cond = select i1 %cmp3, <4 x i1> %a1, <4 x i1> %a2
-  ret <4 x i1> %cond
-
-; CHECK-LABEL: @testqv4i1eq
-; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
-; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: creqv [[REG1:[0-9]+]], {{[0-9]+}}, {{[0-9]+}}
-; CHECK: bclr 12, [[REG1]], 0
-; CHECK: qvfmr 1, 6
-; CHECK: blr
-}
-
-define <4 x i1> @testqv4i1sge(float %c1, float %c2, float %c3, float %c4, <4 x i1> %a1, <4 x i1> %a2) #1 {
-entry:
-  %cmp1 = fcmp oeq float %c3, %c4
-  %cmp3tmp = fcmp oeq float %c1, %c2
-  %cmp3 = icmp sge i1 %cmp3tmp, %cmp1
-  %cond = select i1 %cmp3, <4 x i1> %a1, <4 x i1> %a2
-  ret <4 x i1> %cond
-
-; CHECK-LABEL: @testqv4i1sge
-; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
-; CHECK: bc 12, 2, .LBB[[BB:[0-9_]+]]
-; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: bc 4, 2, .LBB[[BB]]
-; CHECK: qvfmr 5, 6
-; CHECK: .LBB[[BB]]:
-; CHECK: qvfmr 1, 5
-; CHECK: blr
-}
-
-define <4 x i1> @testqv4i1uge(float %c1, float %c2, float %c3, float %c4, <4 x i1> %a1, <4 x i1> %a2) #1 {
-entry:
-  %cmp1 = fcmp oeq float %c3, %c4
-  %cmp3tmp = fcmp oeq float %c1, %c2
-  %cmp3 = icmp uge i1 %cmp3tmp, %cmp1
-  %cond = select i1 %cmp3, <4 x i1> %a1, <4 x i1> %a2
-  ret <4 x i1> %cond
-
-; CHECK-LABEL: @testqv4i1uge
-; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
-; CHECK: bc 4, 2, .LBB[[BB:[0-9_]+]]
-; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: bc 12, 2, .LBB[[BB]]
-; CHECK: qvfmr 5, 6
-; CHECK: .LBB[[BB]]:
-; CHECK: qvfmr 1, 5
-; CHECK: blr
-}
-
-define <4 x i1> @testqv4i1sgt(float %c1, float %c2, float %c3, float %c4, <4 x i1> %a1, <4 x i1> %a2) #1 {
-entry:
-  %cmp1 = fcmp oeq float %c3, %c4
-  %cmp3tmp = fcmp oeq float %c1, %c2
-  %cmp3 = icmp sgt i1 %cmp3tmp, %cmp1
-  %cond = select i1 %cmp3, <4 x i1> %a1, <4 x i1> %a2
-  ret <4 x i1> %cond
-
-; CHECK-LABEL: @testqv4i1sgt
-; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
-; CHECK: bc 4, 2, .LBB[[BB1:[0-9_]+]]
-; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: bc 4, 2, .LBB[[BB2:[0-9_]+]]
-; CHECK: .LBB[[BB1]]:
-; CHECK: qvfmr 5, 6
-; CHECK: .LBB[[BB2]]:
-; CHECK: qvfmr 1, 5
-; CHECK: blr
-}
-
-define <4 x i1> @testqv4i1ugt(float %c1, float %c2, float %c3, float %c4, <4 x i1> %a1, <4 x i1> %a2) #1 {
-entry:
-  %cmp1 = fcmp oeq float %c3, %c4
-  %cmp3tmp = fcmp oeq float %c1, %c2
-  %cmp3 = icmp ugt i1 %cmp3tmp, %cmp1
-  %cond = select i1 %cmp3, <4 x i1> %a1, <4 x i1> %a2
-  ret <4 x i1> %cond
-
-; CHECK-LABEL: @testqv4i1ugt
-; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
-; CHECK: bc 12, 2, .LBB[[BB1:[0-9_]+]]
-; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: bc 12, 2, .LBB[[BB2:[0-9_]+]]
-; CHECK: .LBB[[BB1]]:
-; CHECK: qvfmr 5, 6
-; CHECK: .LBB[[BB2]]:
-; CHECK: qvfmr 1, 5
-; CHECK: blr
-}
-
-define <4 x i1> @testqv4i1ne(float %c1, float %c2, float %c3, float %c4, <4 x i1> %a1, <4 x i1> %a2) #1 {
-entry:
-  %cmp1 = fcmp oeq float %c3, %c4
-  %cmp3tmp = fcmp oeq float %c1, %c2
-  %cmp3 = icmp ne i1 %cmp3tmp, %cmp1
-  %cond = select i1 %cmp3, <4 x i1> %a1, <4 x i1> %a2
-  ret <4 x i1> %cond
-
-; CHECK-LABEL: @testqv4i1ne
-; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
-; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: crxor [[REG1:[0-9]+]], {{[0-9]+}}, {{[0-9]+}}
-; CHECK: bclr 12, [[REG1]], 0
-; CHECK: qvfmr 1, 6
-; CHECK: blr
-}
-
 attributes #0 = { nounwind readnone "target-cpu"="pwr7" }
-attributes #1 = { nounwind readnone "target-cpu"="a2q" }
 
diff --git a/llvm/test/CodeGen/PowerPC/selectiondag-extload-computeknownbits.ll b/llvm/test/CodeGen/PowerPC/selectiondag-extload-computeknownbits.ll
index 53d17d8668270..73fce78c33aa7 100644
--- a/llvm/test/CodeGen/PowerPC/selectiondag-extload-computeknownbits.ll
+++ b/llvm/test/CodeGen/PowerPC/selectiondag-extload-computeknownbits.ll
@@ -1,4 +1,4 @@
-; RUN: llc -verify-machineinstrs -mtriple=powerpc64-bgq-linux < %s
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux < %s
 
 ; Check that llc does not crash due to an illegal APInt operation
 
diff --git a/llvm/test/CodeGen/PowerPC/setcr_bc.mir b/llvm/test/CodeGen/PowerPC/setcr_bc.mir
index e9d81da681fcc..564ee7d45957b 100644
--- a/llvm/test/CodeGen/PowerPC/setcr_bc.mir
+++ b/llvm/test/CodeGen/PowerPC/setcr_bc.mir
@@ -32,8 +32,8 @@
     ret i32 %call2.i.sink
   }
   
-  attributes #0 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
-  attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+  attributes #0 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector" "unsafe-fp-math"="false" "use-soft-float"="false" }
+  attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 ...
 ---
diff --git a/llvm/test/CodeGen/PowerPC/setcr_bc2.mir b/llvm/test/CodeGen/PowerPC/setcr_bc2.mir
index 582284d6d0a59..513cb85e1580a 100644
--- a/llvm/test/CodeGen/PowerPC/setcr_bc2.mir
+++ b/llvm/test/CodeGen/PowerPC/setcr_bc2.mir
@@ -32,8 +32,8 @@
     ret i32 %call2.i.sink
   }
   
-  attributes #0 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
-  attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+  attributes #0 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector" "unsafe-fp-math"="false" "use-soft-float"="false" }
+  attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 ...
 ---
diff --git a/llvm/test/CodeGen/PowerPC/stwu-sched.ll b/llvm/test/CodeGen/PowerPC/stwu-sched.ll
index 0afd2ee406894..36afaf84a296b 100644
--- a/llvm/test/CodeGen/PowerPC/stwu-sched.ll
+++ b/llvm/test/CodeGen/PowerPC/stwu-sched.ll
@@ -58,7 +58,7 @@ define void @initCombList(%0* nocapture, i32 signext) local_unnamed_addr #0 {
   ret void
 }
 
-attributes #0 = { norecurse nounwind writeonly "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { norecurse nounwind writeonly "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.ident = !{!0}
 
diff --git a/llvm/test/CodeGen/PowerPC/unal-vec-ldst.ll b/llvm/test/CodeGen/PowerPC/unal-vec-ldst.ll
index 497add38e0444..79a368dd095ac 100644
--- a/llvm/test/CodeGen/PowerPC/unal-vec-ldst.ll
+++ b/llvm/test/CodeGen/PowerPC/unal-vec-ldst.ll
@@ -327,72 +327,6 @@ entry:
 
 }
 
-define <4 x float> @test_l_qv4float(<4 x float>* %p) #1 {
-; CHECK-LABEL: test_l_qv4float:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    li 4, 15
-; CHECK-NEXT:    qvlpclsx 0, 0, 3
-; CHECK-NEXT:    qvlfsx 1, 3, 4
-; CHECK-NEXT:    qvlfsx 2, 0, 3
-; CHECK-NEXT:    qvfperm 1, 2, 1, 0
-; CHECK-NEXT:    blr
-entry:
-  %r = load <4 x float>, <4 x float>* %p, align 4
-  ret <4 x float> %r
-
-}
-
-define <8 x float> @test_l_qv8float(<8 x float>* %p) #1 {
-; CHECK-LABEL: test_l_qv8float:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    li 4, 31
-; CHECK-NEXT:    qvlpclsx 1, 0, 3
-; CHECK-NEXT:    qvlfsx 0, 3, 4
-; CHECK-NEXT:    li 4, 16
-; CHECK-NEXT:    qvlfsx 3, 3, 4
-; CHECK-NEXT:    qvlfsx 4, 0, 3
-; CHECK-NEXT:    qvfperm 2, 3, 0, 1
-; CHECK-NEXT:    qvfperm 1, 4, 3, 1
-; CHECK-NEXT:    blr
-entry:
-  %r = load <8 x float>, <8 x float>* %p, align 4
-  ret <8 x float> %r
-
-}
-
-define <4 x double> @test_l_qv4double(<4 x double>* %p) #1 {
-; CHECK-LABEL: test_l_qv4double:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    li 4, 31
-; CHECK-NEXT:    qvlpcldx 0, 0, 3
-; CHECK-NEXT:    qvlfdx 1, 3, 4
-; CHECK-NEXT:    qvlfdx 2, 0, 3
-; CHECK-NEXT:    qvfperm 1, 2, 1, 0
-; CHECK-NEXT:    blr
-entry:
-  %r = load <4 x double>, <4 x double>* %p, align 8
-  ret <4 x double> %r
-
-}
-
-define <8 x double> @test_l_qv8double(<8 x double>* %p) #1 {
-; CHECK-LABEL: test_l_qv8double:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    li 4, 63
-; CHECK-NEXT:    qvlpcldx 1, 0, 3
-; CHECK-NEXT:    qvlfdx 0, 3, 4
-; CHECK-NEXT:    li 4, 32
-; CHECK-NEXT:    qvlfdx 3, 3, 4
-; CHECK-NEXT:    qvlfdx 4, 0, 3
-; CHECK-NEXT:    qvfperm 2, 3, 0, 1
-; CHECK-NEXT:    qvfperm 1, 4, 3, 1
-; CHECK-NEXT:    blr
-entry:
-  %r = load <8 x double>, <8 x double>* %p, align 8
-  ret <8 x double> %r
-
-}
-
 define void @test_s_v16i8(<16 x i8>* %p, <16 x i8> %v) #0 {
 ; CHECK-LABEL: test_s_v16i8:
 ; CHECK:       # %bb.0: # %entry
@@ -537,89 +471,6 @@ entry:
 
 }
 
-define void @test_s_qv4float(<4 x float>* %p, <4 x float> %v) #1 {
-; CHECK-LABEL: test_s_qv4float:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    qvesplati 0, 1, 3
-; CHECK-NEXT:    stfs 1, 0(3)
-; CHECK-NEXT:    stfs 0, 12(3)
-; CHECK-NEXT:    qvesplati 0, 1, 2
-; CHECK-NEXT:    qvesplati 1, 1, 1
-; CHECK-NEXT:    stfs 0, 8(3)
-; CHECK-NEXT:    stfs 1, 4(3)
-; CHECK-NEXT:    blr
-entry:
-  store <4 x float> %v, <4 x float>* %p, align 4
-  ret void
-
-}
-
-define void @test_s_qv8float(<8 x float>* %p, <8 x float> %v) #1 {
-; CHECK-LABEL: test_s_qv8float:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    qvesplati 0, 2, 3
-; CHECK-NEXT:    stfs 2, 16(3)
-; CHECK-NEXT:    stfs 0, 28(3)
-; CHECK-NEXT:    qvesplati 0, 2, 2
-; CHECK-NEXT:    qvesplati 2, 2, 1
-; CHECK-NEXT:    stfs 1, 0(3)
-; CHECK-NEXT:    stfs 0, 24(3)
-; CHECK-NEXT:    qvesplati 0, 1, 3
-; CHECK-NEXT:    stfs 2, 20(3)
-; CHECK-NEXT:    qvesplati 2, 1, 2
-; CHECK-NEXT:    qvesplati 1, 1, 1
-; CHECK-NEXT:    stfs 0, 12(3)
-; CHECK-NEXT:    stfs 2, 8(3)
-; CHECK-NEXT:    stfs 1, 4(3)
-; CHECK-NEXT:    blr
-entry:
-  store <8 x float> %v, <8 x float>* %p, align 4
-  ret void
-
-}
-
-define void @test_s_qv4double(<4 x double>* %p, <4 x double> %v) #1 {
-; CHECK-LABEL: test_s_qv4double:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    qvesplati 0, 1, 3
-; CHECK-NEXT:    stfd 1, 0(3)
-; CHECK-NEXT:    stfd 0, 24(3)
-; CHECK-NEXT:    qvesplati 0, 1, 2
-; CHECK-NEXT:    qvesplati 1, 1, 1
-; CHECK-NEXT:    stfd 0, 16(3)
-; CHECK-NEXT:    stfd 1, 8(3)
-; CHECK-NEXT:    blr
-entry:
-  store <4 x double> %v, <4 x double>* %p, align 8
-  ret void
-
-}
-
-define void @test_s_qv8double(<8 x double>* %p, <8 x double> %v) #1 {
-; CHECK-LABEL: test_s_qv8double:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    qvesplati 0, 2, 3
-; CHECK-NEXT:    stfd 2, 32(3)
-; CHECK-NEXT:    stfd 0, 56(3)
-; CHECK-NEXT:    qvesplati 0, 2, 2
-; CHECK-NEXT:    qvesplati 2, 2, 1
-; CHECK-NEXT:    stfd 1, 0(3)
-; CHECK-NEXT:    stfd 0, 48(3)
-; CHECK-NEXT:    qvesplati 0, 1, 3
-; CHECK-NEXT:    stfd 2, 40(3)
-; CHECK-NEXT:    qvesplati 2, 1, 2
-; CHECK-NEXT:    qvesplati 1, 1, 1
-; CHECK-NEXT:    stfd 0, 24(3)
-; CHECK-NEXT:    stfd 2, 16(3)
-; CHECK-NEXT:    stfd 1, 8(3)
-; CHECK-NEXT:    blr
-entry:
-  store <8 x double> %v, <8 x double>* %p, align 8
-  ret void
-
-}
-
 attributes #0 = { nounwind "target-cpu"="pwr7" }
-attributes #1 = { nounwind "target-cpu"="a2q" }
 attributes #2 = { nounwind "target-cpu"="pwr8" }
 
diff --git a/llvm/test/CodeGen/PowerPC/uwtables.ll b/llvm/test/CodeGen/PowerPC/uwtables.ll
index 7523d04d73d38..e302934ab8d6b 100644
--- a/llvm/test/CodeGen/PowerPC/uwtables.ll
+++ b/llvm/test/CodeGen/PowerPC/uwtables.ll
@@ -47,5 +47,5 @@ declare i32 @__gxx_personality_v0(...)
 declare void @__cxa_call_unexpected(i8*) local_unnamed_addr
 
 
-attributes #0 = { noreturn nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noreturn nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
diff --git a/llvm/test/Instrumentation/MemorySanitizer/PowerPC/vararg-ppc64.ll b/llvm/test/Instrumentation/MemorySanitizer/PowerPC/vararg-ppc64.ll
index 36da7add88015..33f3d82c3683d 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/PowerPC/vararg-ppc64.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/PowerPC/vararg-ppc64.ll
@@ -55,21 +55,6 @@ define i32 @bar2() {
 ; CHECK: store <2 x i64> zeroinitializer, <2 x i64>* inttoptr (i64 add (i64 ptrtoint ([100 x i64]* @__msan_va_arg_tls to i64), i64 8) to <2 x i64>*), align 8
 ; CHECK: store {{.*}} 24, {{.*}} @__msan_va_arg_overflow_size_tls
 
-; Check QPX vector argument.
-define i32 @bar3() "target-features"="+qpx" {
-  %1 = call i32 (i32, ...) @foo(i32 0, i32 1, i32 2, <4 x double> <double 1.0, double 2.0, double 3.0, double 4.0>)
-  ret i32 %1
-}
-
-; That one is even stranger: the parameter save area starts at offset 48 from
-; (32-byte aligned) stack pointer, the vector parameter is at 96 bytes from
-; the stack pointer, so its offset from parameter save area is misaligned.
-; CHECK-LABEL: @bar3
-; CHECK: store i32 0, i32* inttoptr (i64 add (i64 ptrtoint ([100 x i64]* @__msan_va_arg_tls to i64), i64 4) to i32*), align 8
-; CHECK: store i32 0, i32* inttoptr (i64 add (i64 ptrtoint ([100 x i64]* @__msan_va_arg_tls to i64), i64 12) to i32*), align 8
-; CHECK: store <4 x i64> zeroinitializer, <4 x i64>* inttoptr (i64 add (i64 ptrtoint ([100 x i64]* @__msan_va_arg_tls to i64), i64 40) to <4 x i64>*), align 8
-; CHECK: store {{.*}} 72, {{.*}} @__msan_va_arg_overflow_size_tls
-
 ; Check i64 array.
 define i32 @bar4() {
   %1 = call i32 (i32, ...) @foo(i32 0, [2 x i64] [i64 1, i64 2])
diff --git a/llvm/test/MC/Disassembler/PowerPC/qpx.txt b/llvm/test/MC/Disassembler/PowerPC/qpx.txt
deleted file mode 100644
index 00e598bd4356e..0000000000000
--- a/llvm/test/MC/Disassembler/PowerPC/qpx.txt
+++ /dev/null
@@ -1,371 +0,0 @@
-# RUN: llvm-mc --disassemble %s -triple powerpc64-bgq-linux -mcpu=a2q | FileCheck %s
-
-# CHECK: qvfabs 3, 5
-0x10 0x60 0x2a 0x10
-
-# CHECK: qvfadd 3, 4, 5
-0x10 0x64 0x28 0x2a
-
-# CHECK: qvfadds 3, 4, 5
-0x00 0x64 0x28 0x2a
-
-# CHECK: qvfandc 3, 4, 5
-0x10 0x64 0x2a 0x08
-
-# CHECK: qvfand 3, 4, 5
-0x10 0x64 0x28 0x88
-
-# CHECK: qvfcfid 3, 5
-0x10 0x60 0x2e 0x9c
-
-# CHECK: qvfcfids 3, 5
-0x00 0x60 0x2e 0x9c
-
-# CHECK: qvfcfidu 3, 5
-0x10 0x60 0x2f 0x9c
-
-# CHECK: qvfcfidus 3, 5
-0x00 0x60 0x2f 0x9c
-
-# CHECK: qvfclr 3
-0x10 0x63 0x18 0x08
-
-# CHECK: qvfcpsgn 3, 4, 5
-0x10 0x64 0x28 0x10
-
-# CHECK: qvfctfb 3, 4
-0x10 0x64 0x22 0x88
-
-# CHECK: qvfctid 3, 5
-0x10 0x60 0x2e 0x5c
-
-# CHECK: qvfctidu 3, 5
-0x10 0x60 0x2f 0x5c
-
-# CHECK: qvfctiduz 3, 5
-0x10 0x60 0x2f 0x5e
-
-# CHECK: qvfctidz 3, 5
-0x10 0x60 0x2e 0x5e
-
-# CHECK: qvfctiw 3, 5
-0x10 0x60 0x28 0x1c
-
-# CHECK: qvfctiwu 3, 5
-0x10 0x60 0x29 0x1c
-
-# CHECK: qvfctiwuz 3, 5
-0x10 0x60 0x29 0x1e
-
-# CHECK: qvfctiwz 3, 5
-0x10 0x60 0x28 0x1e
-
-# CHECK: qvfequ 3, 4, 5
-0x10 0x64 0x2c 0x88
-
-# CHECK: qvflogical 3, 4, 5, 12
-0x10 0x64 0x2e 0x08
-
-# CHECK: qvfmadd 3, 4, 6, 5
-0x10 0x64 0x29 0xba
-
-# CHECK: qvfmadds 3, 4, 6, 5
-0x00 0x64 0x29 0xba
-
-# CHECK: qvfmr 3, 5
-0x10 0x60 0x28 0x90
-
-# CHECK: qvfmsub 3, 4, 6, 5
-0x10 0x64 0x29 0xb8
-
-# CHECK: qvfmsubs 3, 4, 6, 5
-0x00 0x64 0x29 0xb8
-
-# CHECK: qvfmul 3, 4, 6
-0x10 0x64 0x01 0xb2
-
-# CHECK: qvfmuls 3, 4, 6
-0x00 0x64 0x01 0xb2
-
-# CHECK: qvfnabs 3, 5
-0x10 0x60 0x29 0x10
-
-# CHECK: qvfnand 3, 4, 5
-0x10 0x64 0x2f 0x08
-
-# CHECK: qvfneg 3, 5
-0x10 0x60 0x28 0x50
-
-# CHECK: qvfnmadd 3, 4, 6, 5
-0x10 0x64 0x29 0xbe
-
-# CHECK: qvfnmadds 3, 4, 6, 5
-0x00 0x64 0x29 0xbe
-
-# CHECK: qvfnmsub 3, 4, 6, 5
-0x10 0x64 0x29 0xbc
-
-# CHECK: qvfnmsubs 3, 4, 6, 5
-0x00 0x64 0x29 0xbc
-
-# CHECK: qvfnor 3, 4, 5
-0x10 0x64 0x2c 0x08
-
-# CHECK: qvfnot 3, 4
-0x10 0x64 0x25 0x08
-
-# CHECK: qvforc 3, 4, 5
-0x10 0x64 0x2e 0x88
-
-# CHECK: qvfor 3, 4, 5
-0x10 0x64 0x2b 0x88
-
-# CHECK: qvfperm 3, 4, 5, 6
-0x10 0x64 0x29 0x8c
-
-# CHECK: qvfre 3, 5
-0x10 0x60 0x28 0x30
-
-# CHECK: qvfres 3, 5
-0x00 0x60 0x28 0x30
-
-# CHECK: qvfrim 3, 5
-0x10 0x60 0x2b 0xd0
-
-# CHECK: qvfrin 3, 5
-0x10 0x60 0x2b 0x10
-
-# CHECK: qvfrip 3, 5
-0x10 0x60 0x2b 0x90
-
-# CHECK: qvfriz 3, 5
-0x10 0x60 0x2b 0x50
-
-# CHECK: qvfrsp 3, 5
-0x10 0x60 0x28 0x18
-
-# CHECK: qvfrsqrte 3, 5
-0x10 0x60 0x28 0x34
-
-# CHECK: qvfrsqrtes 3, 5
-0x00 0x60 0x28 0x34
-
-# CHECK: qvfsel 3, 4, 6, 5
-0x10 0x64 0x29 0xae
-
-# CHECK: qvfset 3
-0x10 0x63 0x1f 0x88
-
-# CHECK: qvfsub 3, 4, 5
-0x10 0x64 0x28 0x28
-
-# CHECK: qvfsubs 3, 4, 5
-0x00 0x64 0x28 0x28
-
-# CHECK: qvfxmadd 3, 4, 6, 5
-0x10 0x64 0x29 0x92
-
-# CHECK: qvfxmadds 3, 4, 6, 5
-0x00 0x64 0x29 0x92
-
-# CHECK: qvfxmul 3, 4, 6
-0x10 0x64 0x01 0xa2
-
-# CHECK: qvfxmuls 3, 4, 6
-0x00 0x64 0x01 0xa2
-
-# CHECK: qvfxor 3, 4, 5
-0x10 0x64 0x2b 0x08
-
-# CHECK: qvfxxcpnmadd 3, 4, 6, 5
-0x10 0x64 0x29 0x86
-
-# CHECK: qvfxxcpnmadds 3, 4, 6, 5
-0x00 0x64 0x29 0x86
-
-# CHECK: qvfxxmadd 3, 4, 6, 5
-0x10 0x64 0x29 0x82
-
-# CHECK: qvfxxmadds 3, 4, 6, 5
-0x00 0x64 0x29 0x82
-
-# CHECK: qvfxxnpmadd 3, 4, 6, 5
-0x10 0x64 0x29 0x96
-
-# CHECK: qvfxxnpmadds 3, 4, 6, 5
-0x00 0x64 0x29 0x96
-
-# CHECK: qvlfcduxa 3, 9, 11
-0x7c 0x69 0x58 0xcf
-
-# CHECK: qvlfcdux 3, 9, 11
-0x7c 0x69 0x58 0xce
-
-# CHECK: qvlfcdxa 3, 10, 11
-0x7c 0x6a 0x58 0x8f
-
-# CHECK: qvlfcdx 3, 10, 11
-0x7c 0x6a 0x58 0x8e
-
-# CHECK: qvlfcsuxa 3, 9, 11
-0x7c 0x69 0x58 0x4f
-
-# CHECK: qvlfcsux 3, 9, 11
-0x7c 0x69 0x58 0x4e
-
-# CHECK: qvlfcsxa 3, 10, 11
-0x7c 0x6a 0x58 0x0f
-
-# CHECK: qvlfcsx 3, 10, 11
-0x7c 0x6a 0x58 0x0e
-
-# CHECK: qvlfduxa 3, 9, 11
-0x7c 0x69 0x5c 0xcf
-
-# CHECK: qvlfdux 3, 9, 11
-0x7c 0x69 0x5c 0xce
-
-# CHECK: qvlfdxa 3, 10, 11
-0x7c 0x6a 0x5c 0x8f
-
-# CHECK: qvlfdx 3, 10, 11
-0x7c 0x6a 0x5c 0x8e
-
-# CHECK: qvlfiwaxa 3, 10, 11
-0x7c 0x6a 0x5e 0xcf
-
-# CHECK: qvlfiwax 3, 10, 11
-0x7c 0x6a 0x5e 0xce
-
-# CHECK: qvlfiwzxa 3, 10, 11
-0x7c 0x6a 0x5e 0x8f
-
-# CHECK: qvlfiwzx 3, 10, 11
-0x7c 0x6a 0x5e 0x8e
-
-# CHECK: qvlfsuxa 3, 9, 11
-0x7c 0x69 0x5c 0x4f
-
-# CHECK: qvlfsux 3, 9, 11
-0x7c 0x69 0x5c 0x4e
-
-# CHECK: qvlfsxa 3, 10, 11
-0x7c 0x6a 0x5c 0x0f
-
-# CHECK: qvlfsx 3, 10, 11
-0x7c 0x6a 0x5c 0x0e
-
-# CHECK: qvlpcldx 3, 10, 11
-0x7c 0x6a 0x5c 0x8c
-
-# CHECK: qvlpclsx 3, 10, 11
-0x7c 0x6a 0x5c 0x0c
-
-# CHECK: qvlpcrdx 3, 10, 11
-0x7c 0x6a 0x58 0x8c
-
-# CHECK: qvlpcrsx 3, 10, 11
-0x7c 0x6a 0x58 0x0c
-
-# CHECK: qvstfcduxa 2, 9, 11
-0x7c 0x49 0x59 0xcf
-
-# CHECK: qvstfcduxia 2, 9, 11
-0x7c 0x49 0x59 0xcb
-
-# CHECK: qvstfcduxi 2, 9, 11
-0x7c 0x49 0x59 0xca
-
-# CHECK: qvstfcdux 2, 9, 11
-0x7c 0x49 0x59 0xce
-
-# CHECK: qvstfcdxa 2, 10, 11
-0x7c 0x4a 0x59 0x8f
-
-# CHECK: qvstfcdxia 2, 10, 11
-0x7c 0x4a 0x59 0x8b
-
-# CHECK: qvstfcdxi 2, 10, 11
-0x7c 0x4a 0x59 0x8a
-
-# CHECK: qvstfcdx 2, 10, 11
-0x7c 0x4a 0x59 0x8e
-
-# CHECK: qvstfcsuxa 2, 9, 11
-0x7c 0x49 0x59 0x4f
-
-# CHECK: qvstfcsuxia 2, 9, 11
-0x7c 0x49 0x59 0x4b
-
-# CHECK: qvstfcsuxi 2, 9, 11
-0x7c 0x49 0x59 0x4a
-
-# CHECK: qvstfcsux 2, 9, 11
-0x7c 0x49 0x59 0x4e
-
-# CHECK: qvstfcsxa 2, 10, 11
-0x7c 0x4a 0x59 0x0f
-
-# CHECK: qvstfcsxia 2, 10, 11
-0x7c 0x4a 0x59 0x0b
-
-# CHECK: qvstfcsxi 2, 10, 11
-0x7c 0x4a 0x59 0x0a
-
-# CHECK: qvstfcsx 2, 10, 11
-0x7c 0x4a 0x59 0x0e
-
-# CHECK: qvstfduxa 2, 9, 11
-0x7c 0x49 0x5d 0xcf
-
-# CHECK: qvstfduxia 2, 9, 11
-0x7c 0x49 0x5d 0xcb
-
-# CHECK: qvstfduxi 2, 9, 11
-0x7c 0x49 0x5d 0xca
-
-# CHECK: qvstfdux 2, 9, 11
-0x7c 0x49 0x5d 0xce
-
-# CHECK: qvstfdxa 2, 10, 11
-0x7c 0x4a 0x5d 0x8f
-
-# CHECK: qvstfdxia 2, 10, 11
-0x7c 0x4a 0x5d 0x8b
-
-# CHECK: qvstfdxi 2, 10, 11
-0x7c 0x4a 0x5d 0x8a
-
-# CHECK: qvstfdx 2, 10, 11
-0x7c 0x4a 0x5d 0x8e
-
-# CHECK: qvstfiwxa 2, 10, 11
-0x7c 0x4a 0x5f 0x8f
-
-# CHECK: qvstfiwx 2, 10, 11
-0x7c 0x4a 0x5f 0x8e
-
-# CHECK: qvstfsuxa 2, 9, 11
-0x7c 0x49 0x5d 0x4f
-
-# CHECK: qvstfsuxia 2, 9, 11
-0x7c 0x49 0x5d 0x4b
-
-# CHECK: qvstfsuxi 2, 9, 11
-0x7c 0x49 0x5d 0x4a
-
-# CHECK: qvstfsux 2, 9, 11
-0x7c 0x49 0x5d 0x4e
-
-# CHECK: qvstfsxa 2, 10, 11
-0x7c 0x4a 0x5d 0x0f
-
-# CHECK: qvstfsxia 2, 10, 11
-0x7c 0x4a 0x5d 0x0b
-
-# CHECK: qvstfsxi 2, 10, 11
-0x7c 0x4a 0x5d 0x0a
-
-# CHECK: qvstfsx 2, 10, 11
-0x7c 0x4a 0x5d 0x0e
-
diff --git a/llvm/test/MC/PowerPC/qpx.s b/llvm/test/MC/PowerPC/qpx.s
deleted file mode 100644
index a1fb2090f8fff..0000000000000
--- a/llvm/test/MC/PowerPC/qpx.s
+++ /dev/null
@@ -1,252 +0,0 @@
-# RUN: llvm-mc -triple powerpc64-bgq-linux --show-encoding %s | FileCheck %s
-
-# CHECK: qvfabs 3, 5                     # encoding: [0x10,0x60,0x2a,0x10]
-         qvfabs %q3, %q5
-
-# CHECK: qvfabs 3, 5                     # encoding: [0x10,0x60,0x2a,0x10]
-         qvfabs 3, 5
-# CHECK: qvfadd 3, 4, 5                  # encoding: [0x10,0x64,0x28,0x2a]
-         qvfadd 3, 4, 5
-# CHECK: qvfadds 3, 4, 5                 # encoding: [0x00,0x64,0x28,0x2a]
-         qvfadds 3, 4, 5
-# CHECK: qvfandc 3, 4, 5                 # encoding: [0x10,0x64,0x2a,0x08]
-         qvfandc 3, 4, 5
-# CHECK: qvfand 3, 4, 5                  # encoding: [0x10,0x64,0x28,0x88]
-         qvfand 3, 4, 5
-# CHECK: qvfcfid 3, 5                    # encoding: [0x10,0x60,0x2e,0x9c]
-         qvfcfid 3, 5
-# CHECK: qvfcfids 3, 5                   # encoding: [0x00,0x60,0x2e,0x9c]
-         qvfcfids 3, 5
-# CHECK: qvfcfidu 3, 5                   # encoding: [0x10,0x60,0x2f,0x9c]
-         qvfcfidu 3, 5
-# CHECK: qvfcfidus 3, 5                  # encoding: [0x00,0x60,0x2f,0x9c]
-         qvfcfidus 3, 5
-# CHECK: qvfclr 3                        # encoding: [0x10,0x63,0x18,0x08]
-         qvfclr 3
-# CHECK: qvfcpsgn 3, 4, 5                # encoding: [0x10,0x64,0x28,0x10]
-         qvfcpsgn 3, 4, 5
-# CHECK: qvfctfb 3, 4                    # encoding: [0x10,0x64,0x22,0x88]
-         qvfctfb 3, 4
-# CHECK: qvfctid 3, 5                    # encoding: [0x10,0x60,0x2e,0x5c]
-         qvfctid 3, 5
-# CHECK: qvfctidu 3, 5                   # encoding: [0x10,0x60,0x2f,0x5c]
-         qvfctidu 3, 5
-# CHECK: qvfctiduz 3, 5                  # encoding: [0x10,0x60,0x2f,0x5e]
-         qvfctiduz 3, 5
-# CHECK: qvfctidz 3, 5                   # encoding: [0x10,0x60,0x2e,0x5e]
-         qvfctidz 3, 5
-# CHECK: qvfctiw 3, 5                    # encoding: [0x10,0x60,0x28,0x1c]
-         qvfctiw 3, 5
-# CHECK: qvfctiwu 3, 5                   # encoding: [0x10,0x60,0x29,0x1c]
-         qvfctiwu 3, 5
-# CHECK: qvfctiwuz 3, 5                  # encoding: [0x10,0x60,0x29,0x1e]
-         qvfctiwuz 3, 5
-# CHECK: qvfctiwz 3, 5                   # encoding: [0x10,0x60,0x28,0x1e]
-         qvfctiwz 3, 5
-# CHECK: qvfequ 3, 4, 5                  # encoding: [0x10,0x64,0x2c,0x88]
-         qvfequ 3, 4, 5
-# CHECK: qvflogical 3, 4, 5, 12          # encoding: [0x10,0x64,0x2e,0x08]
-         qvflogical 3, 4, 5, 12
-# CHECK: qvfmadd 3, 4, 6, 5              # encoding: [0x10,0x64,0x29,0xba]
-         qvfmadd 3, 4, 6, 5
-# CHECK: qvfmadds 3, 4, 6, 5             # encoding: [0x00,0x64,0x29,0xba]
-         qvfmadds 3, 4, 6, 5
-# CHECK: qvfmr 3, 5                      # encoding: [0x10,0x60,0x28,0x90]
-         qvfmr 3, 5
-# CHECK: qvfmsub 3, 4, 6, 5              # encoding: [0x10,0x64,0x29,0xb8]
-         qvfmsub 3, 4, 6, 5
-# CHECK: qvfmsubs 3, 4, 6, 5             # encoding: [0x00,0x64,0x29,0xb8]
-         qvfmsubs 3, 4, 6, 5
-# CHECK: qvfmul 3, 4, 6                  # encoding: [0x10,0x64,0x01,0xb2]
-         qvfmul 3, 4, 6
-# CHECK: qvfmuls 3, 4, 6                 # encoding: [0x00,0x64,0x01,0xb2]
-         qvfmuls 3, 4, 6
-# CHECK: qvfnabs 3, 5                    # encoding: [0x10,0x60,0x29,0x10]
-         qvfnabs 3, 5
-# CHECK: qvfnand 3, 4, 5                 # encoding: [0x10,0x64,0x2f,0x08]
-         qvfnand 3, 4, 5
-# CHECK: qvfneg 3, 5                     # encoding: [0x10,0x60,0x28,0x50]
-         qvfneg 3, 5
-# CHECK: qvfnmadd 3, 4, 6, 5             # encoding: [0x10,0x64,0x29,0xbe]
-         qvfnmadd 3, 4, 6, 5
-# CHECK: qvfnmadds 3, 4, 6, 5            # encoding: [0x00,0x64,0x29,0xbe]
-         qvfnmadds 3, 4, 6, 5
-# CHECK: qvfnmsub 3, 4, 6, 5             # encoding: [0x10,0x64,0x29,0xbc]
-         qvfnmsub 3, 4, 6, 5
-# CHECK: qvfnmsubs 3, 4, 6, 5            # encoding: [0x00,0x64,0x29,0xbc]
-         qvfnmsubs 3, 4, 6, 5
-# CHECK: qvfnor 3, 4, 5                  # encoding: [0x10,0x64,0x2c,0x08]
-         qvfnor 3, 4, 5
-# CHECK: qvfnot 3, 4                     # encoding: [0x10,0x64,0x25,0x08]
-         qvfnot 3, 4
-# CHECK: qvforc 3, 4, 5                  # encoding: [0x10,0x64,0x2e,0x88]
-         qvforc 3, 4, 5
-# CHECK: qvfor 3, 4, 5                   # encoding: [0x10,0x64,0x2b,0x88]
-         qvfor 3, 4, 5
-# CHECK: qvfperm 3, 4, 5, 6              # encoding: [0x10,0x64,0x29,0x8c]
-         qvfperm 3, 4, 5, 6
-# CHECK: qvfre 3, 5                      # encoding: [0x10,0x60,0x28,0x30]
-         qvfre 3, 5
-# CHECK: qvfres 3, 5                     # encoding: [0x00,0x60,0x28,0x30]
-         qvfres 3, 5
-# CHECK: qvfrim 3, 5                     # encoding: [0x10,0x60,0x2b,0xd0]
-         qvfrim 3, 5
-# CHECK: qvfrin 3, 5                     # encoding: [0x10,0x60,0x2b,0x10]
-         qvfrin 3, 5
-# CHECK: qvfrip 3, 5                     # encoding: [0x10,0x60,0x2b,0x90]
-         qvfrip 3, 5
-# CHECK: qvfriz 3, 5                     # encoding: [0x10,0x60,0x2b,0x50]
-         qvfriz 3, 5
-# CHECK: qvfrsp 3, 5                     # encoding: [0x10,0x60,0x28,0x18]
-         qvfrsp 3, 5
-# CHECK: qvfrsqrte 3, 5                  # encoding: [0x10,0x60,0x28,0x34]
-         qvfrsqrte 3, 5
-# CHECK: qvfrsqrtes 3, 5                 # encoding: [0x00,0x60,0x28,0x34]
-         qvfrsqrtes 3, 5
-# CHECK: qvfsel 3, 4, 6, 5               # encoding: [0x10,0x64,0x29,0xae]
-         qvfsel 3, 4, 6, 5
-# CHECK: qvfset 3                        # encoding: [0x10,0x63,0x1f,0x88]
-         qvfset 3
-# CHECK: qvfsub 3, 4, 5                  # encoding: [0x10,0x64,0x28,0x28]
-         qvfsub 3, 4, 5
-# CHECK: qvfsubs 3, 4, 5                 # encoding: [0x00,0x64,0x28,0x28]
-         qvfsubs 3, 4, 5
-# CHECK: qvfxmadd 3, 4, 6, 5             # encoding: [0x10,0x64,0x29,0x92]
-         qvfxmadd 3, 4, 6, 5
-# CHECK: qvfxmadds 3, 4, 6, 5            # encoding: [0x00,0x64,0x29,0x92]
-         qvfxmadds 3, 4, 6, 5
-# CHECK: qvfxmul 3, 4, 6                 # encoding: [0x10,0x64,0x01,0xa2]
-         qvfxmul 3, 4, 6
-# CHECK: qvfxmuls 3, 4, 6                # encoding: [0x00,0x64,0x01,0xa2]
-         qvfxmuls 3, 4, 6
-# CHECK: qvfxor 3, 4, 5                  # encoding: [0x10,0x64,0x2b,0x08]
-         qvfxor 3, 4, 5
-# CHECK: qvfxxcpnmadd 3, 4, 6, 5         # encoding: [0x10,0x64,0x29,0x86]
-         qvfxxcpnmadd 3, 4, 6, 5
-# CHECK: qvfxxcpnmadds 3, 4, 6, 5        # encoding: [0x00,0x64,0x29,0x86]
-         qvfxxcpnmadds 3, 4, 6, 5
-# CHECK: qvfxxmadd 3, 4, 6, 5            # encoding: [0x10,0x64,0x29,0x82]
-         qvfxxmadd 3, 4, 6, 5
-# CHECK: qvfxxmadds 3, 4, 6, 5           # encoding: [0x00,0x64,0x29,0x82]
-         qvfxxmadds 3, 4, 6, 5
-# CHECK: qvfxxnpmadd 3, 4, 6, 5          # encoding: [0x10,0x64,0x29,0x96]
-         qvfxxnpmadd 3, 4, 6, 5
-# CHECK: qvfxxnpmadds 3, 4, 6, 5         # encoding: [0x00,0x64,0x29,0x96]
-         qvfxxnpmadds 3, 4, 6, 5
-# CHECK: qvlfcduxa 3, 9, 11              # encoding: [0x7c,0x69,0x58,0xcf]
-         qvlfcduxa 3, 9, 11
-# CHECK: qvlfcdux 3, 9, 11               # encoding: [0x7c,0x69,0x58,0xce]
-         qvlfcdux 3, 9, 11
-# CHECK: qvlfcdxa 3, 10, 11              # encoding: [0x7c,0x6a,0x58,0x8f]
-         qvlfcdxa 3, 10, 11
-# CHECK: qvlfcdx 3, 10, 11               # encoding: [0x7c,0x6a,0x58,0x8e]
-         qvlfcdx 3, 10, 11
-# CHECK: qvlfcsuxa 3, 9, 11              # encoding: [0x7c,0x69,0x58,0x4f]
-         qvlfcsuxa 3, 9, 11
-# CHECK: qvlfcsux 3, 9, 11               # encoding: [0x7c,0x69,0x58,0x4e]
-         qvlfcsux 3, 9, 11
-# CHECK: qvlfcsxa 3, 10, 11              # encoding: [0x7c,0x6a,0x58,0x0f]
-         qvlfcsxa 3, 10, 11
-# CHECK: qvlfcsx 3, 10, 11               # encoding: [0x7c,0x6a,0x58,0x0e]
-         qvlfcsx 3, 10, 11
-# CHECK: qvlfduxa 3, 9, 11               # encoding: [0x7c,0x69,0x5c,0xcf]
-         qvlfduxa 3, 9, 11
-# CHECK: qvlfdux 3, 9, 11                # encoding: [0x7c,0x69,0x5c,0xce]
-         qvlfdux 3, 9, 11
-# CHECK: qvlfdxa 3, 10, 11               # encoding: [0x7c,0x6a,0x5c,0x8f]
-         qvlfdxa 3, 10, 11
-# CHECK: qvlfdx 3, 10, 11                # encoding: [0x7c,0x6a,0x5c,0x8e]
-         qvlfdx 3, 10, 11
-# CHECK: qvlfiwaxa 3, 10, 11             # encoding: [0x7c,0x6a,0x5e,0xcf]
-         qvlfiwaxa 3, 10, 11
-# CHECK: qvlfiwax 3, 10, 11              # encoding: [0x7c,0x6a,0x5e,0xce]
-         qvlfiwax 3, 10, 11
-# CHECK: qvlfiwzxa 3, 10, 11             # encoding: [0x7c,0x6a,0x5e,0x8f]
-         qvlfiwzxa 3, 10, 11
-# CHECK: qvlfiwzx 3, 10, 11              # encoding: [0x7c,0x6a,0x5e,0x8e]
-         qvlfiwzx 3, 10, 11
-# CHECK: qvlfsuxa 3, 9, 11               # encoding: [0x7c,0x69,0x5c,0x4f]
-         qvlfsuxa 3, 9, 11
-# CHECK: qvlfsux 3, 9, 11                # encoding: [0x7c,0x69,0x5c,0x4e]
-         qvlfsux 3, 9, 11
-# CHECK: qvlfsxa 3, 10, 11               # encoding: [0x7c,0x6a,0x5c,0x0f]
-         qvlfsxa 3, 10, 11
-# CHECK: qvlfsx 3, 10, 11                # encoding: [0x7c,0x6a,0x5c,0x0e]
-         qvlfsx 3, 10, 11
-# CHECK: qvlpcldx 3, 10, 11              # encoding: [0x7c,0x6a,0x5c,0x8c]
-         qvlpcldx 3, 10, 11
-# CHECK: qvlpclsx 3, 10, 11              # encoding: [0x7c,0x6a,0x5c,0x0c]
-         qvlpclsx 3, 10, 11
-# CHECK: qvlpcrdx 3, 10, 11              # encoding: [0x7c,0x6a,0x58,0x8c]
-         qvlpcrdx 3, 10, 11
-# CHECK: qvlpcrsx 3, 10, 11              # encoding: [0x7c,0x6a,0x58,0x0c]
-         qvlpcrsx 3, 10, 11
-# CHECK: qvstfcduxa 2, 9, 11             # encoding: [0x7c,0x49,0x59,0xcf]
-         qvstfcduxa 2, 9, 11
-# CHECK: qvstfcduxia 2, 9, 11            # encoding: [0x7c,0x49,0x59,0xcb]
-         qvstfcduxia 2, 9, 11
-# CHECK: qvstfcduxi 2, 9, 11             # encoding: [0x7c,0x49,0x59,0xca]
-         qvstfcduxi 2, 9, 11
-# CHECK: qvstfcdux 2, 9, 11              # encoding: [0x7c,0x49,0x59,0xce]
-         qvstfcdux 2, 9, 11
-# CHECK: qvstfcdxa 2, 10, 11             # encoding: [0x7c,0x4a,0x59,0x8f]
-         qvstfcdxa 2, 10, 11
-# CHECK: qvstfcdxia 2, 10, 11            # encoding: [0x7c,0x4a,0x59,0x8b]
-         qvstfcdxia 2, 10, 11
-# CHECK: qvstfcdxi 2, 10, 11             # encoding: [0x7c,0x4a,0x59,0x8a]
-         qvstfcdxi 2, 10, 11
-# CHECK: qvstfcdx 2, 10, 11              # encoding: [0x7c,0x4a,0x59,0x8e]
-         qvstfcdx 2, 10, 11
-# CHECK: qvstfcsuxa 2, 9, 11             # encoding: [0x7c,0x49,0x59,0x4f]
-         qvstfcsuxa 2, 9, 11
-# CHECK: qvstfcsuxia 2, 9, 11            # encoding: [0x7c,0x49,0x59,0x4b]
-         qvstfcsuxia 2, 9, 11
-# CHECK: qvstfcsuxi 2, 9, 11             # encoding: [0x7c,0x49,0x59,0x4a]
-         qvstfcsuxi 2, 9, 11
-# CHECK: qvstfcsux 2, 9, 11              # encoding: [0x7c,0x49,0x59,0x4e]
-         qvstfcsux 2, 9, 11
-# CHECK: qvstfcsxa 2, 10, 11             # encoding: [0x7c,0x4a,0x59,0x0f]
-         qvstfcsxa 2, 10, 11
-# CHECK: qvstfcsxia 2, 10, 11            # encoding: [0x7c,0x4a,0x59,0x0b]
-         qvstfcsxia 2, 10, 11
-# CHECK: qvstfcsxi 2, 10, 11             # encoding: [0x7c,0x4a,0x59,0x0a]
-         qvstfcsxi 2, 10, 11
-# CHECK: qvstfcsx 2, 10, 11              # encoding: [0x7c,0x4a,0x59,0x0e]
-         qvstfcsx 2, 10, 11
-# CHECK: qvstfduxa 2, 9, 11              # encoding: [0x7c,0x49,0x5d,0xcf]
-         qvstfduxa 2, 9, 11
-# CHECK: qvstfduxia 2, 9, 11             # encoding: [0x7c,0x49,0x5d,0xcb]
-         qvstfduxia 2, 9, 11
-# CHECK: qvstfduxi 2, 9, 11              # encoding: [0x7c,0x49,0x5d,0xca]
-         qvstfduxi 2, 9, 11
-# CHECK: qvstfdux 2, 9, 11               # encoding: [0x7c,0x49,0x5d,0xce]
-         qvstfdux 2, 9, 11
-# CHECK: qvstfdxa 2, 10, 11              # encoding: [0x7c,0x4a,0x5d,0x8f]
-         qvstfdxa 2, 10, 11
-# CHECK: qvstfdxia 2, 10, 11             # encoding: [0x7c,0x4a,0x5d,0x8b]
-         qvstfdxia 2, 10, 11
-# CHECK: qvstfdxi 2, 10, 11              # encoding: [0x7c,0x4a,0x5d,0x8a]
-         qvstfdxi 2, 10, 11
-# CHECK: qvstfdx 2, 10, 11               # encoding: [0x7c,0x4a,0x5d,0x8e]
-         qvstfdx 2, 10, 11
-# CHECK: qvstfiwxa 2, 10, 11             # encoding: [0x7c,0x4a,0x5f,0x8f]
-         qvstfiwxa 2, 10, 11
-# CHECK: qvstfiwx 2, 10, 11              # encoding: [0x7c,0x4a,0x5f,0x8e]
-         qvstfiwx 2, 10, 11
-# CHECK: qvstfsuxa 2, 9, 11              # encoding: [0x7c,0x49,0x5d,0x4f]
-         qvstfsuxa 2, 9, 11
-# CHECK: qvstfsuxia 2, 9, 11             # encoding: [0x7c,0x49,0x5d,0x4b]
-         qvstfsuxia 2, 9, 11
-# CHECK: qvstfsuxi 2, 9, 11              # encoding: [0x7c,0x49,0x5d,0x4a]
-         qvstfsuxi 2, 9, 11
-# CHECK: qvstfsux 2, 9, 11               # encoding: [0x7c,0x49,0x5d,0x4e]
-         qvstfsux 2, 9, 11
-# CHECK: qvstfsxa 2, 10, 11              # encoding: [0x7c,0x4a,0x5d,0x0f]
-         qvstfsxa 2, 10, 11
-# CHECK: qvstfsxia 2, 10, 11             # encoding: [0x7c,0x4a,0x5d,0x0b]
-         qvstfsxia 2, 10, 11
-# CHECK: qvstfsxi 2, 10, 11              # encoding: [0x7c,0x4a,0x5d,0x0a]
-         qvstfsxi 2, 10, 11
-# CHECK: qvstfsx 2, 10, 11               # encoding: [0x7c,0x4a,0x5d,0x0e]
-         qvstfsx 2, 10, 11
-
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/fp-bc-icmp-const-fold.ll b/llvm/test/Transforms/Attributor/IPConstantProp/fp-bc-icmp-const-fold.ll
index c45c48d502343..67c22f9470779 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/fp-bc-icmp-const-fold.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/fp-bc-icmp-const-fold.ll
@@ -4,7 +4,7 @@
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 target datalayout = "E-m:e-i64:64-n32:64"
-target triple = "powerpc64-bgq-linux"
+target triple = "powerpc64le-unknown-linux"
 
 define void @test(i32 signext %n) {
 ; IS__TUNIT____: Function Attrs: nofree noreturn nosync nounwind readnone
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/combined-partial-overwrites.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/combined-partial-overwrites.ll
index 6cd77a59df6b1..faf7041bfc387 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/combined-partial-overwrites.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/combined-partial-overwrites.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S -dse -enable-dse-memoryssa -enable-dse-partial-store-merging=false < %s | FileCheck %s
 target datalayout = "E-m:e-i64:64-n32:64"
-target triple = "powerpc64-bgq-linux"
+target triple = "powerpc64le-unknown-linux"
 
 %"struct.std::complex" = type { { float, float } }
 
diff --git a/llvm/test/Transforms/DeadStoreElimination/combined-partial-overwrites.ll b/llvm/test/Transforms/DeadStoreElimination/combined-partial-overwrites.ll
index 2a61fff15ade0..a57693a1da38e 100644
--- a/llvm/test/Transforms/DeadStoreElimination/combined-partial-overwrites.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/combined-partial-overwrites.ll
@@ -1,6 +1,6 @@
 ; RUN: opt -S -dse -enable-dse-partial-store-merging=false < %s | FileCheck %s
 target datalayout = "E-m:e-i64:64-n32:64"
-target triple = "powerpc64-bgq-linux"
+target triple = "powerpc64le-unknown-linux"
 
 %"struct.std::complex" = type { { float, float } }
 
diff --git a/llvm/test/Transforms/EntryExitInstrumenter/mcount.ll b/llvm/test/Transforms/EntryExitInstrumenter/mcount.ll
index 4c5f18a26657c..5a6daa2c9a008 100644
--- a/llvm/test/Transforms/EntryExitInstrumenter/mcount.ll
+++ b/llvm/test/Transforms/EntryExitInstrumenter/mcount.ll
@@ -4,7 +4,7 @@
 ; RUN: opt -passes="function(ee-instrument),function(ee-instrument),cgscc(inline),function(post-inline-ee-instrument),function(post-inline-ee-instrument)" -S < %s | FileCheck %s
 
 target datalayout = "E-m:e-i64:64-n32:64"
-target triple = "powerpc64-bgq-linux"
+target triple = "powerpc64le-unknown-linux"
 
 define void @leaf_function() #0 {
 entry:
diff --git a/llvm/test/Transforms/InstCombine/PowerPC/aligned-qpx.ll b/llvm/test/Transforms/InstCombine/PowerPC/aligned-qpx.ll
deleted file mode 100644
index e9710df5670cd..0000000000000
--- a/llvm/test/Transforms/InstCombine/PowerPC/aligned-qpx.ll
+++ /dev/null
@@ -1,165 +0,0 @@
-; RUN: opt -S -instcombine < %s | FileCheck %s
-target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
-target triple = "powerpc64-unknown-linux-gnu"
-
-declare <4 x double> @llvm.ppc.qpx.qvlfs(i8*) #1
-
-define <4 x double> @test1(<4 x float>* %h) #0 {
-entry:
-  %h1 = getelementptr <4 x float>, <4 x float>* %h, i64 1
-  %hv = bitcast <4 x float>* %h1 to i8*
-  %vl = call <4 x double> @llvm.ppc.qpx.qvlfs(i8* %hv)
-
-; CHECK-LABEL: @test1
-; CHECK: @llvm.ppc.qpx.qvlfs
-; CHECK: ret <4 x double>
-
-  %v0 = load <4 x float>, <4 x float>* %h, align 8
-  %v0e = fpext <4 x float> %v0 to <4 x double>
-  %a = fadd <4 x double> %v0e, %vl
-  ret <4 x double> %a
-}
-
-define <4 x double> @test1a(<4 x float>* align 16 %h) #0 {
-entry:
-  %h1 = getelementptr <4 x float>, <4 x float>* %h, i64 1
-  %hv = bitcast <4 x float>* %h1 to i8*
-  %vl = call <4 x double> @llvm.ppc.qpx.qvlfs(i8* %hv)
-
-; CHECK-LABEL: @test1a
-; CHECK-NOT: @llvm.ppc.qpx.qvlfs
-; CHECK-NOT: load <4 x double>
-; CHECK: ret <4 x double>
-
-  %v0 = load <4 x float>, <4 x float>* %h, align 8
-  %v0e = fpext <4 x float> %v0 to <4 x double>
-  %a = fadd <4 x double> %v0e, %vl
-  ret <4 x double> %a
-}
-
-declare void @llvm.ppc.qpx.qvstfs(<4 x double>, i8*) #0
-
-define <4 x float> @test2(<4 x float>* %h, <4 x double> %d) #0 {
-entry:
-  %h1 = getelementptr <4 x float>, <4 x float>* %h, i64 1
-  %hv = bitcast <4 x float>* %h1 to i8*
-  call void @llvm.ppc.qpx.qvstfs(<4 x double> %d, i8* %hv)
-
-  %v0 = load <4 x float>, <4 x float>* %h, align 8
-  ret <4 x float> %v0
-
-; CHECK-LABEL: @test2
-; CHECK: @llvm.ppc.qpx.qvstfs
-; CHECK: ret <4 x float>
-}
-
-define <4 x float> @test2a(<4 x float>* align 16 %h, <4 x double> %d) #0 {
-entry:
-  %h1 = getelementptr <4 x float>, <4 x float>* %h, i64 1
-  %hv = bitcast <4 x float>* %h1 to i8*
-  call void @llvm.ppc.qpx.qvstfs(<4 x double> %d, i8* %hv)
-
-  %v0 = load <4 x float>, <4 x float>* %h, align 8
-  ret <4 x float> %v0
-
-; CHECK-LABEL: @test2
-; CHECK: fptrunc <4 x double> %d to <4 x float>
-; CHECK-NOT: @llvm.ppc.qpx.qvstfs
-; CHECK-NOT: store <4 x double>
-; CHECK: ret <4 x float>
-}
-
-declare <4 x double> @llvm.ppc.qpx.qvlfd(i8*) #1
-
-define <4 x double> @test1l(<4 x double>* %h) #0 {
-entry:
-  %h1 = getelementptr <4 x double>, <4 x double>* %h, i64 1
-  %hv = bitcast <4 x double>* %h1 to i8*
-  %vl = call <4 x double> @llvm.ppc.qpx.qvlfd(i8* %hv)
-
-; CHECK-LABEL: @test1l
-; CHECK: @llvm.ppc.qpx.qvlfd
-; CHECK: ret <4 x double>
-
-  %v0 = load <4 x double>, <4 x double>* %h, align 8
-  %a = fadd <4 x double> %v0, %vl
-  ret <4 x double> %a
-}
-
-define <4 x double> @test1ln(<4 x double>* align 16 %h) #0 {
-entry:
-  %h1 = getelementptr <4 x double>, <4 x double>* %h, i64 1
-  %hv = bitcast <4 x double>* %h1 to i8*
-  %vl = call <4 x double> @llvm.ppc.qpx.qvlfd(i8* %hv)
-
-; CHECK-LABEL: @test1ln
-; CHECK: @llvm.ppc.qpx.qvlfd
-; CHECK: ret <4 x double>
-
-  %v0 = load <4 x double>, <4 x double>* %h, align 8
-  %a = fadd <4 x double> %v0, %vl
-  ret <4 x double> %a
-}
-
-define <4 x double> @test1la(<4 x double>* align 32 %h) #0 {
-entry:
-  %h1 = getelementptr <4 x double>, <4 x double>* %h, i64 1
-  %hv = bitcast <4 x double>* %h1 to i8*
-  %vl = call <4 x double> @llvm.ppc.qpx.qvlfd(i8* %hv)
-
-; CHECK-LABEL: @test1la
-; CHECK-NOT: @llvm.ppc.qpx.qvlfd
-; CHECK: ret <4 x double>
-
-  %v0 = load <4 x double>, <4 x double>* %h, align 8
-  %a = fadd <4 x double> %v0, %vl
-  ret <4 x double> %a
-}
-
-declare void @llvm.ppc.qpx.qvstfd(<4 x double>, i8*) #0
-
-define <4 x double> @test2l(<4 x double>* %h, <4 x double> %d) #0 {
-entry:
-  %h1 = getelementptr <4 x double>, <4 x double>* %h, i64 1
-  %hv = bitcast <4 x double>* %h1 to i8*
-  call void @llvm.ppc.qpx.qvstfd(<4 x double> %d, i8* %hv)
-
-  %v0 = load <4 x double>, <4 x double>* %h, align 8
-  ret <4 x double> %v0
-
-; CHECK-LABEL: @test2l
-; CHECK: @llvm.ppc.qpx.qvstfd
-; CHECK: ret <4 x double>
-}
-
-define <4 x double> @test2ln(<4 x double>* align 16 %h, <4 x double> %d) #0 {
-entry:
-  %h1 = getelementptr <4 x double>, <4 x double>* %h, i64 1
-  %hv = bitcast <4 x double>* %h1 to i8*
-  call void @llvm.ppc.qpx.qvstfd(<4 x double> %d, i8* %hv)
-
-  %v0 = load <4 x double>, <4 x double>* %h, align 8
-  ret <4 x double> %v0
-
-; CHECK-LABEL: @test2ln
-; CHECK: @llvm.ppc.qpx.qvstfd
-; CHECK: ret <4 x double>
-}
-
-define <4 x double> @test2la(<4 x double>* align 32 %h, <4 x double> %d) #0 {
-entry:
-  %h1 = getelementptr <4 x double>, <4 x double>* %h, i64 1
-  %hv = bitcast <4 x double>* %h1 to i8*
-  call void @llvm.ppc.qpx.qvstfd(<4 x double> %d, i8* %hv)
-
-  %v0 = load <4 x double>, <4 x double>* %h, align 8
-  ret <4 x double> %v0
-
-; CHECK-LABEL: @test2l
-; CHECK-NOT: @llvm.ppc.qpx.qvstfd
-; CHECK: ret <4 x double>
-}
-
-attributes #0 = { nounwind }
-attributes #1 = { nounwind readonly }
-
diff --git a/llvm/test/Transforms/LoopDataPrefetch/PowerPC/basic.ll b/llvm/test/Transforms/LoopDataPrefetch/PowerPC/basic.ll
index ea46fd0d5a8f8..68c75af14f3e9 100644
--- a/llvm/test/Transforms/LoopDataPrefetch/PowerPC/basic.ll
+++ b/llvm/test/Transforms/LoopDataPrefetch/PowerPC/basic.ll
@@ -1,7 +1,6 @@
-; RUN: opt -mcpu=a2 -loop-data-prefetch -S < %s | FileCheck %s
-; RUN: opt -mcpu=a2 -passes=loop-data-prefetch -S < %s | FileCheck %s
+; RUN: opt -mcpu=a2 -loop-data-prefetch -mtriple=powerpc64le-unknown-linux -enable-ppc-prefetching -S < %s | FileCheck %s
+; RUN: opt -mcpu=a2 -passes=loop-data-prefetch -mtriple=powerpc64le-unknown-linux -enable-ppc-prefetching -S < %s | FileCheck %s
 target datalayout = "E-m:e-i64:64-n32:64"
-target triple = "powerpc64-bgq-linux"
 
 define void @foo(double* nocapture %a, double* nocapture readonly %b) {
 entry:
diff --git a/llvm/test/Transforms/LoopSimplify/dup-preds.ll b/llvm/test/Transforms/LoopSimplify/dup-preds.ll
index c9253fa51a65f..362d834686d41 100644
--- a/llvm/test/Transforms/LoopSimplify/dup-preds.ll
+++ b/llvm/test/Transforms/LoopSimplify/dup-preds.ll
@@ -1,6 +1,6 @@
 ; RUN: opt -loop-simplify -S %s | FileCheck %s
 target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
-target triple = "powerpc64-bgq-linux"
+target triple = "powerpc64le-unknown-linux"
 
 define fastcc void @do_update_md([3 x float]* nocapture readonly %x) #0 {
 entry:
diff --git a/llvm/test/Transforms/LoopUnroll/pr14167.ll b/llvm/test/Transforms/LoopUnroll/pr14167.ll
index 9aac70115d9ae..3097c234fb933 100644
--- a/llvm/test/Transforms/LoopUnroll/pr14167.ll
+++ b/llvm/test/Transforms/LoopUnroll/pr14167.ll
@@ -1,6 +1,6 @@
 ; RUN: opt < %s -S -loop-unroll -unroll-runtime | FileCheck %s
 target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v128:128:128-n32:64"
-target triple = "powerpc64-bgq-linux"
+target triple = "powerpc64le-unknown-linux"
 
 define void @test1() nounwind {
 ; Ensure that we don't crash when the trip count == -1.
diff --git a/llvm/test/Transforms/LoopVectorize/PowerPC/agg-interleave-a2.ll b/llvm/test/Transforms/LoopVectorize/PowerPC/agg-interleave-a2.ll
deleted file mode 100644
index 9fdfb6f90e7bf..0000000000000
--- a/llvm/test/Transforms/LoopVectorize/PowerPC/agg-interleave-a2.ll
+++ /dev/null
@@ -1,40 +0,0 @@
-; RUN: opt -S -basic-aa -loop-vectorize < %s | FileCheck %s
-target datalayout = "E-m:e-i64:64-n32:64"
-target triple = "powerpc64-unknown-linux-gnu"
-
-; Function Attrs: nounwind
-define void @foo(double* noalias nocapture %a, double* noalias nocapture readonly %b, double* noalias nocapture readonly %c) #0 {
-entry:
-  br label %for.body
-
-; CHECK-LABEL: @foo
-; CHECK: fmul <4 x double> %{{[^,]+}}, <double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00>
-; CHECK-NEXT: fmul <4 x double> %{{[^,]+}}, <double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00>
-
-for.cond.cleanup:                                 ; preds = %for.body
-  ret void
-
-for.body:                                         ; preds = %for.body, %entry
-  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
-  %arrayidx = getelementptr inbounds double, double* %b, i64 %indvars.iv
-  %0 = load double, double* %arrayidx, align 8
-  %mul = fmul double %0, 2.000000e+00
-  %mul3 = fmul double %0, %mul
-  %arrayidx5 = getelementptr inbounds double, double* %c, i64 %indvars.iv
-  %1 = load double, double* %arrayidx5, align 8
-  %mul6 = fmul double %1, 3.000000e+00
-  %mul9 = fmul double %1, %mul6
-  %add = fadd double %mul3, %mul9
-  %mul12 = fmul double %0, 4.000000e+00
-  %mul15 = fmul double %mul12, %1
-  %add16 = fadd double %mul15, %add
-  %add17 = fadd double %add16, 1.000000e+00
-  %arrayidx19 = getelementptr inbounds double, double* %a, i64 %indvars.iv
-  store double %add17, double* %arrayidx19, align 8
-  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
-  %exitcond = icmp eq i64 %indvars.iv.next, 1600
-  br i1 %exitcond, label %for.cond.cleanup, label %for.body
-}
-
-attributes #0 = { nounwind "target-cpu"="a2q" }
-
diff --git a/llvm/test/Transforms/LoopVectorize/PowerPC/vectorize-only-for-real.ll b/llvm/test/Transforms/LoopVectorize/PowerPC/vectorize-only-for-real.ll
index 8abc25ece35c6..cddddba579473 100644
--- a/llvm/test/Transforms/LoopVectorize/PowerPC/vectorize-only-for-real.ll
+++ b/llvm/test/Transforms/LoopVectorize/PowerPC/vectorize-only-for-real.ll
@@ -1,6 +1,6 @@
 ; RUN: opt -S -loop-vectorize < %s | FileCheck %s
 target datalayout = "E-m:e-i64:64-n32:64"
-target triple = "powerpc64-bgq-linux"
+target triple = "powerpc64le-unknown-linux"
 
 ; Function Attrs: nounwind
 define zeroext i32 @test() #0 {
diff --git a/llvm/test/Transforms/NewGVN/pr31483.ll b/llvm/test/Transforms/NewGVN/pr31483.ll
index 999ff74ad5881..5bf7e1a695011 100644
--- a/llvm/test/Transforms/NewGVN/pr31483.ll
+++ b/llvm/test/Transforms/NewGVN/pr31483.ll
@@ -100,7 +100,7 @@ declare signext i32 @zot(i8*, ...) #1
 ; Function Attrs: nounwind
 declare void @llvm.va_end(i8*) #2
 
-attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64" "target-features"="+altivec,-bpermd,-crypto,-direct-move,-extdiv,-power8-vector,-qpx,-vsx" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64" "target-features"="+altivec,-bpermd,-crypto,-direct-move,-extdiv,-power8-vector,-qpx,-vsx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64" "target-features"="+altivec,-bpermd,-crypto,-direct-move,-extdiv,-power8-vector,-vsx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64" "target-features"="+altivec,-bpermd,-crypto,-direct-move,-extdiv,-power8-vector,-vsx" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #2 = { nounwind }
 
diff --git a/llvm/test/Transforms/SCCP/fp-bc-icmp-const-fold.ll b/llvm/test/Transforms/SCCP/fp-bc-icmp-const-fold.ll
index 8f97225ca446b..20c44384504e2 100644
--- a/llvm/test/Transforms/SCCP/fp-bc-icmp-const-fold.ll
+++ b/llvm/test/Transforms/SCCP/fp-bc-icmp-const-fold.ll
@@ -1,6 +1,6 @@
 ; RUN: opt -S -ipsccp < %s | FileCheck %s
 target datalayout = "E-m:e-i64:64-n32:64"
-target triple = "powerpc64-bgq-linux"
+target triple = "powerpc64le-unknown-linux"
 
 define void @test(i32 signext %n) {
 
diff --git a/llvm/unittests/ADT/TripleTest.cpp b/llvm/unittests/ADT/TripleTest.cpp
index dc7a28c72f208..1852d7b6a1b0d 100644
--- a/llvm/unittests/ADT/TripleTest.cpp
+++ b/llvm/unittests/ADT/TripleTest.cpp
@@ -111,41 +111,6 @@ TEST(TripleTest, ParsedIDs) {
   EXPECT_EQ(Triple::Linux, T.getOS());
   EXPECT_EQ(Triple::Musl, T.getEnvironment());
 
-  T = Triple("powerpc-bgp-linux");
-  EXPECT_EQ(Triple::ppc, T.getArch());
-  EXPECT_EQ(Triple::BGP, T.getVendor());
-  EXPECT_EQ(Triple::Linux, T.getOS());
-  EXPECT_EQ(Triple::UnknownEnvironment, T.getEnvironment());
-
-  T = Triple("powerpc-bgp-cnk");
-  EXPECT_EQ(Triple::ppc, T.getArch());
-  EXPECT_EQ(Triple::BGP, T.getVendor());
-  EXPECT_EQ(Triple::CNK, T.getOS());
-  EXPECT_EQ(Triple::UnknownEnvironment, T.getEnvironment());
-
-  T = Triple("ppc-bgp-linux");
-  EXPECT_EQ(Triple::ppc, T.getArch());
-  EXPECT_EQ(Triple::BGP, T.getVendor());
-  EXPECT_EQ(Triple::Linux, T.getOS());
-  EXPECT_EQ(Triple::UnknownEnvironment, T.getEnvironment());
-
-  T = Triple("ppc32-bgp-linux");
-  EXPECT_EQ(Triple::ppc, T.getArch());
-  EXPECT_EQ(Triple::BGP, T.getVendor());
-  EXPECT_EQ(Triple::Linux, T.getOS());
-  EXPECT_EQ(Triple::UnknownEnvironment, T.getEnvironment());
-
-  T = Triple("powerpc64-bgq-linux");
-  EXPECT_EQ(Triple::ppc64, T.getArch());
-  EXPECT_EQ(Triple::BGQ, T.getVendor());
-  EXPECT_EQ(Triple::Linux, T.getOS());
-  EXPECT_EQ(Triple::UnknownEnvironment, T.getEnvironment());
-
-  T = Triple("ppc64-bgq-linux");
-  EXPECT_EQ(Triple::ppc64, T.getArch());
-  EXPECT_EQ(Triple::BGQ, T.getVendor());
-  EXPECT_EQ(Triple::Linux, T.getOS());
-
   T = Triple("powerpc-ibm-aix");
   EXPECT_EQ(Triple::ppc, T.getArch());
   EXPECT_EQ(Triple::IBM, T.getVendor());
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/PowerPC/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/PowerPC/BUILD.gn
index 043a672a76e1e..3a452fc6e0601 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/PowerPC/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/PowerPC/BUILD.gn
@@ -61,7 +61,6 @@ static_library("LLVMPowerPCCodeGen") {
     "PPCMachineScheduler.cpp",
     "PPCMacroFusion.cpp",
     "PPCPreEmitPeephole.cpp",
-    "PPCQPXLoadSplat.cpp",
     "PPCReduceCRLogicals.cpp",
     "PPCRegisterInfo.cpp",
     "PPCSubtarget.cpp",
diff --git a/openmp/runtime/src/kmp.h b/openmp/runtime/src/kmp.h
index 933573bc810cb..bb6cee740ace7 100644
--- a/openmp/runtime/src/kmp.h
+++ b/openmp/runtime/src/kmp.h
@@ -1116,9 +1116,6 @@ extern kmp_uint64 __kmp_now_nsec();
 #if KMP_OS_WINDOWS
 #define KMP_INIT_WAIT 64U /* initial number of spin-tests   */
 #define KMP_NEXT_WAIT 32U /* susequent number of spin-tests */
-#elif KMP_OS_CNK
-#define KMP_INIT_WAIT 16U /* initial number of spin-tests   */
-#define KMP_NEXT_WAIT 8U /* susequent number of spin-tests */
 #elif KMP_OS_LINUX
 #define KMP_INIT_WAIT 1024U /* initial number of spin-tests   */
 #define KMP_NEXT_WAIT 512U /* susequent number of spin-tests */
diff --git a/openmp/runtime/src/kmp_csupport.cpp b/openmp/runtime/src/kmp_csupport.cpp
index b5c641cc7273c..f6fb1e602c297 100644
--- a/openmp/runtime/src/kmp_csupport.cpp
+++ b/openmp/runtime/src/kmp_csupport.cpp
@@ -680,17 +680,6 @@ void __kmpc_flush(ident_t *loc) {
 // Nothing to see here move along
 #elif KMP_ARCH_PPC64
 // Nothing needed here (we have a real MB above).
-#if KMP_OS_CNK
-  // The flushing thread needs to yield here; this prevents a
-  // busy-waiting thread from saturating the pipeline. flush is
-  // often used in loops like this:
-  // while (!flag) {
-  //   #pragma omp flush(flag)
-  // }
-  // and adding the yield here is good for at least a 10x speedup
-  // when running >2 threads per core (on the NAS LU benchmark).
-  __kmp_yield();
-#endif
 #else
 #error Unknown or unsupported architecture
 #endif
diff --git a/openmp/runtime/src/kmp_lock.h b/openmp/runtime/src/kmp_lock.h
index e54f6812b8b34..b80e54777e8c2 100644
--- a/openmp/runtime/src/kmp_lock.h
+++ b/openmp/runtime/src/kmp_lock.h
@@ -159,7 +159,7 @@ extern void __kmp_destroy_nested_tas_lock(kmp_tas_lock_t *lck);
 #define KMP_LOCK_ACQUIRED_NEXT 0
 #ifndef KMP_USE_FUTEX
 #define KMP_USE_FUTEX                                                          \
-  (KMP_OS_LINUX && !KMP_OS_CNK &&                                              \
+  (KMP_OS_LINUX &&                                                             \
    (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64))
 #endif
 #if KMP_USE_FUTEX
diff --git a/openmp/runtime/src/kmp_os.h b/openmp/runtime/src/kmp_os.h
index bfe7765b2a967..33735cf455c7e 100644
--- a/openmp/runtime/src/kmp_os.h
+++ b/openmp/runtime/src/kmp_os.h
@@ -69,7 +69,7 @@
 #error Unknown compiler
 #endif
 
-#if (KMP_OS_LINUX || KMP_OS_WINDOWS || KMP_OS_FREEBSD) && !KMP_OS_CNK
+#if (KMP_OS_LINUX || KMP_OS_WINDOWS || KMP_OS_FREEBSD)
 #define KMP_AFFINITY_SUPPORTED 1
 #if KMP_OS_WINDOWS && KMP_ARCH_X86_64
 #define KMP_GROUP_AFFINITY 1
diff --git a/openmp/runtime/src/kmp_platform.h b/openmp/runtime/src/kmp_platform.h
index 779c08e9771d5..4296ca31d67d9 100644
--- a/openmp/runtime/src/kmp_platform.h
+++ b/openmp/runtime/src/kmp_platform.h
@@ -22,7 +22,6 @@
 #define KMP_OS_OPENBSD 0
 #define KMP_OS_DARWIN 0
 #define KMP_OS_WINDOWS 0
-#define KMP_OS_CNK 0
 #define KMP_OS_HURD 0
 #define KMP_OS_UNIX 0 /* disjunction of KMP_OS_LINUX, KMP_OS_DARWIN etc. */
 
@@ -66,11 +65,6 @@
 #define KMP_OS_OPENBSD 1
 #endif
 
-#if (defined __bgq__)
-#undef KMP_OS_CNK
-#define KMP_OS_CNK 1
-#endif
-
 #if (defined __GNU__)
 #undef KMP_OS_HURD
 #define KMP_OS_HURD 1
diff --git a/openmp/runtime/src/z_Linux_asm.S b/openmp/runtime/src/z_Linux_asm.S
index 8090ff759fe1b..16059a3762bf4 100644
--- a/openmp/runtime/src/z_Linux_asm.S
+++ b/openmp/runtime/src/z_Linux_asm.S
@@ -1433,13 +1433,8 @@ __kmp_invoke_microtask:
 	add 12, 0, 12
 	neg 12, 12
 
-// We need to make sure that the stack frame stays aligned (to 16 bytes, except
-// under the BG/Q CNK, where it must be to 32 bytes).
-# if KMP_OS_CNK
-	li 0, -32
-# else
+// We need to make sure that the stack frame stays aligned (to 16 bytes).
 	li 0, -16
-# endif
 	and 12, 0, 12
 
 // Establish the local stack frame.
diff --git a/openmp/runtime/src/z_Linux_util.cpp b/openmp/runtime/src/z_Linux_util.cpp
index 3b5910fc95e89..58cc4d25f6080 100644
--- a/openmp/runtime/src/z_Linux_util.cpp
+++ b/openmp/runtime/src/z_Linux_util.cpp
@@ -31,7 +31,7 @@
 #include <sys/times.h>
 #include <unistd.h>
 
-#if KMP_OS_LINUX && !KMP_OS_CNK
+#if KMP_OS_LINUX
 #include <sys/sysinfo.h>
 #if KMP_USE_FUTEX
 // We should really include <futex.h>, but that causes compatibility problems on
diff --git a/polly/lib/External/isl/config.sub b/polly/lib/External/isl/config.sub
index 1d8e98bcee23a..bc4db70f82abf 100755
--- a/polly/lib/External/isl/config.sub
+++ b/polly/lib/External/isl/config.sub
@@ -152,9 +152,6 @@ case $os in
 		os=
 		basic_machine=$1
 		;;
-	-bluegene*)
-		os=-cnk
-		;;
 	-sim | -cisco | -oki | -wec | -winbond)
 		os=
 		basic_machine=$1
@@ -539,10 +536,6 @@ case $basic_machine in
 		basic_machine=bfin-`echo "$basic_machine" | sed 's/^[^-]*-//'`
 		os=-linux
 		;;
-	bluegene*)
-		basic_machine=powerpc-ibm
-		os=-cnk
-		;;
 	c54x-*)
 		basic_machine=tic54x-`echo "$basic_machine" | sed 's/^[^-]*-//'`
 		;;
@@ -1364,7 +1357,7 @@ case $os in
 	# Each alternative MUST end in a * to match a version number.
 	# -sysv* is not here because it comes later, after sysvr4.
 	-gnu* | -bsd* | -mach* | -minix* | -genix* | -ultrix* | -irix* \
-	      | -*vms* | -sco* | -esix* | -isc* | -aix* | -cnk* | -sunos | -sunos[34]*\
+	      | -*vms* | -sco* | -esix* | -isc* | -aix* | -sunos | -sunos[34]*\
 	      | -hpux* | -unos* | -osf* | -luna* | -dgux* | -auroraux* | -solaris* \
 	      | -sym* | -kopensolaris* | -plan9* \
 	      | -amigaos* | -amigados* | -msdos* | -newsos* | -unicos* | -aof* \
@@ -1728,7 +1721,7 @@ case $basic_machine in
 			-sunos*)
 				vendor=sun
 				;;
-			-cnk*|-aix*)
+			-aix*)
 				vendor=ibm
 				;;
 			-beos*)
diff --git a/polly/lib/External/ppcg/config.sub b/polly/lib/External/ppcg/config.sub
index 6205f8423d6aa..d97f3009f9f09 100644
--- a/polly/lib/External/ppcg/config.sub
+++ b/polly/lib/External/ppcg/config.sub
@@ -160,9 +160,6 @@ case $os in
 		os=
 		basic_machine=$1
 		;;
-	-bluegene*)
-		os=-cnk
-		;;
 	-sim | -cisco | -oki | -wec | -winbond)
 		os=
 		basic_machine=$1
@@ -524,10 +521,6 @@ case $basic_machine in
 		basic_machine=bfin-`echo $basic_machine | sed 's/^[^-]*-//'`
 		os=-linux
 		;;
-	bluegene*)
-		basic_machine=powerpc-ibm
-		os=-cnk
-		;;
 	c54x-*)
 		basic_machine=tic54x-`echo $basic_machine | sed 's/^[^-]*-//'`
 		;;
@@ -1344,7 +1337,7 @@ case $os in
 	# Each alternative MUST END IN A *, to match a version number.
 	# -sysv* is not here because it comes later, after sysvr4.
 	-gnu* | -bsd* | -mach* | -minix* | -genix* | -ultrix* | -irix* \
-	      | -*vms* | -sco* | -esix* | -isc* | -aix* | -cnk* | -sunos | -sunos[34]*\
+	      | -*vms* | -sco* | -esix* | -isc* | -aix* | -sunos | -sunos[34]*\
 	      | -hpux* | -unos* | -osf* | -luna* | -dgux* | -auroraux* | -solaris* \
 	      | -sym* | -kopensolaris* \
 	      | -amigaos* | -amigados* | -msdos* | -newsos* | -unicos* | -aof* \
@@ -1709,7 +1702,7 @@ case $basic_machine in
 			-sunos*)
 				vendor=sun
 				;;
-			-cnk*|-aix*)
+			-aix*)
 				vendor=ibm
 				;;
 			-beos*)

From 22ec861d28c0aa0cdf76b9618151b9ee87ba2221 Mon Sep 17 00:00:00 2001
From: Xing GUO <higuoxing@gmail.com>
Date: Tue, 28 Jul 2020 22:10:44 +0800
Subject: [PATCH 0301/1035] [DWARFYAML] Add support for emitting custom range
 list content.

This patch adds support for emitting custom range list content.

We are able to handcraft a custom range list via the following syntax.

```
debug_rnglists:
  - Lists:
      - Entries:
          - Operator: DW_RLE_startx_endx
            Values:   [ 0x1234, 0x1234 ]
      - Content: '1234567890abcdef'
      - Content: 'abcdef1234567890'
```

Reviewed By: jhenderson

Differential Revision: https://reviews.llvm.org/D84618
---
 llvm/include/llvm/ObjectYAML/DWARFYAML.h      |  6 +-
 llvm/lib/ObjectYAML/DWARFEmitter.cpp          | 17 ++++--
 llvm/lib/ObjectYAML/DWARFYAML.cpp             |  9 +++
 .../yaml2obj/ELF/DWARF/debug-rnglists.yaml    | 61 +++++++++++++++++++
 4 files changed, 86 insertions(+), 7 deletions(-)

diff --git a/llvm/include/llvm/ObjectYAML/DWARFYAML.h b/llvm/include/llvm/ObjectYAML/DWARFYAML.h
index 88ac404b21b13..f4ace738ef8cf 100644
--- a/llvm/include/llvm/ObjectYAML/DWARFYAML.h
+++ b/llvm/include/llvm/ObjectYAML/DWARFYAML.h
@@ -18,6 +18,7 @@
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/BinaryFormat/Dwarf.h"
+#include "llvm/ObjectYAML/YAML.h"
 #include "llvm/Support/YAMLTraits.h"
 #include <cstdint>
 #include <vector>
@@ -189,7 +190,8 @@ struct RnglistEntry {
 };
 
 template <typename EntryType> struct ListEntries {
-  std::vector<EntryType> Entries;
+  Optional<std::vector<EntryType>> Entries;
+  Optional<yaml::BinaryRef> Content;
 };
 
 template <typename EntryType> struct ListTable {
@@ -328,6 +330,8 @@ struct MappingTraits<DWARFYAML::ListTable<EntryType>> {
 template <typename EntryType>
 struct MappingTraits<DWARFYAML::ListEntries<EntryType>> {
   static void mapping(IO &IO, DWARFYAML::ListEntries<EntryType> &ListEntries);
+  static StringRef validate(IO &IO,
+                            DWARFYAML::ListEntries<EntryType> &ListEntries);
 };
 
 template <> struct MappingTraits<DWARFYAML::RnglistEntry> {
diff --git a/llvm/lib/ObjectYAML/DWARFEmitter.cpp b/llvm/lib/ObjectYAML/DWARFEmitter.cpp
index f61191022fb9b..b39bb003db9a5 100644
--- a/llvm/lib/ObjectYAML/DWARFEmitter.cpp
+++ b/llvm/lib/ObjectYAML/DWARFEmitter.cpp
@@ -658,12 +658,17 @@ Error writeDWARFLists(raw_ostream &OS,
 
     for (const DWARFYAML::ListEntries<EntryType> &List : Table.Lists) {
       Offsets.push_back(ListBufferOS.tell());
-      for (const EntryType &Entry : List.Entries) {
-        Expected<uint64_t> EntrySize =
-            writeListEntry(ListBufferOS, Entry, AddrSize, IsLittleEndian);
-        if (!EntrySize)
-          return EntrySize.takeError();
-        Length += *EntrySize;
+      if (List.Content) {
+        List.Content->writeAsBinary(ListBufferOS, UINT64_MAX);
+        Length += List.Content->binary_size();
+      } else if (List.Entries) {
+        for (const EntryType &Entry : *List.Entries) {
+          Expected<uint64_t> EntrySize =
+              writeListEntry(ListBufferOS, Entry, AddrSize, IsLittleEndian);
+          if (!EntrySize)
+            return EntrySize.takeError();
+          Length += *EntrySize;
+        }
       }
     }
 
diff --git a/llvm/lib/ObjectYAML/DWARFYAML.cpp b/llvm/lib/ObjectYAML/DWARFYAML.cpp
index e5c77bc3721fb..adc167249226b 100644
--- a/llvm/lib/ObjectYAML/DWARFYAML.cpp
+++ b/llvm/lib/ObjectYAML/DWARFYAML.cpp
@@ -246,6 +246,15 @@ template <typename EntryType>
 void MappingTraits<DWARFYAML::ListEntries<EntryType>>::mapping(
     IO &IO, DWARFYAML::ListEntries<EntryType> &ListEntries) {
   IO.mapOptional("Entries", ListEntries.Entries);
+  IO.mapOptional("Content", ListEntries.Content);
+}
+
+template <typename EntryType>
+StringRef MappingTraits<DWARFYAML::ListEntries<EntryType>>::validate(
+    IO &IO, DWARFYAML::ListEntries<EntryType> &ListEntries) {
+  if (ListEntries.Entries && ListEntries.Content)
+    return "Entries and Content can't be used together";
+  return StringRef();
 }
 
 template <typename EntryType>
diff --git a/llvm/test/tools/yaml2obj/ELF/DWARF/debug-rnglists.yaml b/llvm/test/tools/yaml2obj/ELF/DWARF/debug-rnglists.yaml
index 80dc9b9fdeb35..248cb190235b2 100644
--- a/llvm/test/tools/yaml2obj/ELF/DWARF/debug-rnglists.yaml
+++ b/llvm/test/tools/yaml2obj/ELF/DWARF/debug-rnglists.yaml
@@ -608,3 +608,64 @@ FileHeader:
   Machine: EM_X86_64
 DWARF:
   debug_rnglists: []
+
+## s) Test that we are able to generate a range list via raw binary data.
+
+# RUN: yaml2obj --docnum=17 %s -o %t17.o
+# RUN: llvm-readelf --hex-dump=.debug_rnglists %t17.o | \
+# RUN:   FileCheck %s --check-prefix=CUSTOM-LIST
+
+#      CUSTOM-LIST: Hex dump of section '.debug_rnglists':
+# CUSTOM-LIST-NEXT: 0x00000000 29000000 05000800 03000000 0c000000 )...............
+##                             ^-------                            unit_length (4-byte)
+##                                      ^---                       version (2-byte)
+##                                          ^-                     address_size (1-byte)
+##                                            ^-                   segment_selector_size (1-byte)
+##                                               ^-------          offset_entry_count (4-byte)
+##                                                        ^------- offsets[0] (4-byte)
+# CUSTOM-LIST-NEXT: 0x00000010 11000000 19000000 02b424b4 24123456 ..........$.$.4V
+##                             ^-------                            offsets[1] (4-byte)
+##                                      ^-------                   offsets[2] (4-byte)
+##                                               ^-                DW_RLE_startx_endx
+##                                                 ^---            operands[0] (ULEB128) 0x1234
+##                                                     ^----       operands[1] (ULEB128) 0x1234
+##                                                          ^----- custom list content
+# CUSTOM-LIST-NEXT: 0x00000020 7890abcd efabcdef 12345678 90       x........4Vx.
+##                             -----------
+##                                        ^-----------------       custom list content
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+DWARF:
+  debug_rnglists:
+    - Lists:
+        - Entries:
+            - Operator: DW_RLE_startx_endx
+              Values:   [ 0x1234, 0x1234 ]
+        - Content: '1234567890abcdef'
+        - Content: 'abcdef1234567890'
+
+## t) Test that yaml2obj emits an error message when 'Content' and 'Entries' are specified
+## at the same time.
+
+# RUN: not yaml2obj --docnum=18 %s 2>&1 | FileCheck %s --check-prefix=ERR
+
+#      ERR: YAML:{{.*}}: error: Entries and Content can't be used together
+# ERR-NEXT:         - Entries: []
+# ERR-NEXT:           ^
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+DWARF:
+  debug_rnglists:
+    - Lists:
+        - Entries: []
+          Content: ''

From c64c04bbaadbc35e265f12644b45787d6d077587 Mon Sep 17 00:00:00 2001
From: Christian Sigg <csigg@google.com>
Date: Tue, 28 Jul 2020 16:29:29 +0200
Subject: [PATCH 0302/1035] Clean up cuda-runtime-wrappers API.

Do not return error code, instead return created resource handles or void. Error reporting is done by the library function.

Reviewed By: herhut

Differential Revision: https://reviews.llvm.org/D84660
---
 .../ConvertLaunchFuncToRuntimeCalls.cpp       | 96 +++++++------------
 ...ower-launch-func-to-gpu-runtime-calls.mlir | 10 +-
 .../cuda-runtime-wrappers.cpp                 | 89 ++++++++---------
 .../rocm-runtime-wrappers.cpp                 | 80 ++++++++--------
 4 files changed, 121 insertions(+), 154 deletions(-)

diff --git a/mlir/lib/Conversion/GPUCommon/ConvertLaunchFuncToRuntimeCalls.cpp b/mlir/lib/Conversion/GPUCommon/ConvertLaunchFuncToRuntimeCalls.cpp
index 37b056263ab46..14011e08de027 100644
--- a/mlir/lib/Conversion/GPUCommon/ConvertLaunchFuncToRuntimeCalls.cpp
+++ b/mlir/lib/Conversion/GPUCommon/ConvertLaunchFuncToRuntimeCalls.cpp
@@ -39,7 +39,7 @@ static constexpr const char *kGpuModuleLoadName = "mgpuModuleLoad";
 static constexpr const char *kGpuModuleGetFunctionName =
     "mgpuModuleGetFunction";
 static constexpr const char *kGpuLaunchKernelName = "mgpuLaunchKernel";
-static constexpr const char *kGpuGetStreamHelperName = "mgpuGetStreamHelper";
+static constexpr const char *kGpuStreamCreateName = "mgpuStreamCreate";
 static constexpr const char *kGpuStreamSynchronizeName =
     "mgpuStreamSynchronize";
 static constexpr const char *kGpuMemHostRegisterName = "mgpuMemHostRegister";
@@ -100,12 +100,6 @@ class GpuLaunchFuncToGpuRuntimeCallsPass
         getLLVMDialect(), module.getDataLayout().getPointerSizeInBits());
   }
 
-  LLVM::LLVMType getGpuRuntimeResultType() {
-    // This is declared as an enum in both CUDA and ROCm (HIP), but helpers
-    // use i32.
-    return getInt32Type();
-  }
-
   // Allocate a void pointer on the stack.
   Value allocatePointer(OpBuilder &builder, Location loc) {
     auto one = builder.create<LLVM::ConstantOp>(loc, getInt32Type(),
@@ -168,27 +162,21 @@ void GpuLaunchFuncToGpuRuntimeCallsPass::declareGpuRuntimeFunctions(
   if (!module.lookupSymbol(kGpuModuleLoadName)) {
     builder.create<LLVM::LLVMFuncOp>(
         loc, kGpuModuleLoadName,
-        LLVM::LLVMType::getFunctionTy(
-            getGpuRuntimeResultType(),
-            {
-                getPointerPointerType(), /* CUmodule *module */
-                getPointerType()         /* void *cubin */
-            },
-            /*isVarArg=*/false));
+        LLVM::LLVMType::getFunctionTy(getPointerType(),
+                                      {getPointerType()}, /* void *cubin */
+                                      /*isVarArg=*/false));
   }
   if (!module.lookupSymbol(kGpuModuleGetFunctionName)) {
     // The helper uses void* instead of CUDA's opaque CUmodule and
     // CUfunction, or ROCm (HIP)'s opaque hipModule_t and hipFunction_t.
     builder.create<LLVM::LLVMFuncOp>(
         loc, kGpuModuleGetFunctionName,
-        LLVM::LLVMType::getFunctionTy(
-            getGpuRuntimeResultType(),
-            {
-                getPointerPointerType(), /* void **function */
-                getPointerType(),        /* void *module */
-                getPointerType()         /* char *name */
-            },
-            /*isVarArg=*/false));
+        LLVM::LLVMType::getFunctionTy(getPointerType(),
+                                      {
+                                          getPointerType(), /* void *module */
+                                          getPointerType()  /* char *name   */
+                                      },
+                                      /*isVarArg=*/false));
   }
   if (!module.lookupSymbol(kGpuLaunchKernelName)) {
     // Other than the CUDA or ROCm (HIP) api, the wrappers use uintptr_t to
@@ -198,7 +186,7 @@ void GpuLaunchFuncToGpuRuntimeCallsPass::declareGpuRuntimeFunctions(
     builder.create<LLVM::LLVMFuncOp>(
         loc, kGpuLaunchKernelName,
         LLVM::LLVMType::getFunctionTy(
-            getGpuRuntimeResultType(),
+            getVoidType(),
             {
                 getPointerType(),        /* void* f */
                 getIntPtrType(),         /* intptr_t gridXDim */
@@ -214,18 +202,18 @@ void GpuLaunchFuncToGpuRuntimeCallsPass::declareGpuRuntimeFunctions(
             },
             /*isVarArg=*/false));
   }
-  if (!module.lookupSymbol(kGpuGetStreamHelperName)) {
+  if (!module.lookupSymbol(kGpuStreamCreateName)) {
     // Helper function to get the current GPU compute stream. Uses void*
     // instead of CUDA's opaque CUstream, or ROCm (HIP)'s opaque hipStream_t.
     builder.create<LLVM::LLVMFuncOp>(
-        loc, kGpuGetStreamHelperName,
+        loc, kGpuStreamCreateName,
         LLVM::LLVMType::getFunctionTy(getPointerType(), /*isVarArg=*/false));
   }
   if (!module.lookupSymbol(kGpuStreamSynchronizeName)) {
     builder.create<LLVM::LLVMFuncOp>(
         loc, kGpuStreamSynchronizeName,
-        LLVM::LLVMType::getFunctionTy(getGpuRuntimeResultType(),
-                                      getPointerType() /* CUstream stream */,
+        LLVM::LLVMType::getFunctionTy(getVoidType(),
+                                      {getPointerType()}, /* void *stream */
                                       /*isVarArg=*/false));
   }
   if (!module.lookupSymbol(kGpuMemHostRegisterName)) {
@@ -365,17 +353,13 @@ Value GpuLaunchFuncToGpuRuntimeCallsPass::generateKernelNameConstant(
 // hsaco in the 'rocdl.hsaco' attribute of the kernel function in the IR.
 //
 // %0 = call %binarygetter
-// %1 = alloca sizeof(void*)
-// call %moduleLoad(%2, %1)
-// %2 = alloca sizeof(void*)
-// %3 = load %1
-// %4 = <see generateKernelNameConstant>
-// call %moduleGetFunction(%2, %3, %4)
-// %5 = call %getStreamHelper()
-// %6 = load %2
-// %7 = <see setupParamsArray>
-// call %launchKernel(%6, <launchOp operands 0..5>, 0, %5, %7, nullptr)
-// call %streamSynchronize(%5)
+// %1 = call %moduleLoad(%0)
+// %2 = <see generateKernelNameConstant>
+// %3 = call %moduleGetFunction(%1, %2)
+// %4 = call %streamCreate()
+// %5 = <see setupParamsArray>
+// call %launchKernel(%3, <launchOp operands 0..5>, 0, %4, %5, nullptr)
+// call %streamSynchronize(%4)
 void GpuLaunchFuncToGpuRuntimeCallsPass::translateGpuLaunchCalls(
     mlir::gpu::LaunchFuncOp launchOp) {
   OpBuilder builder(launchOp);
@@ -405,36 +389,30 @@ void GpuLaunchFuncToGpuRuntimeCallsPass::translateGpuLaunchCalls(
 
   // Emit the load module call to load the module data. Error checking is done
   // in the called helper function.
-  auto gpuModule = allocatePointer(builder, loc);
   auto gpuModuleLoad =
       getOperation().lookupSymbol<LLVM::LLVMFuncOp>(kGpuModuleLoadName);
-  builder.create<LLVM::CallOp>(loc, ArrayRef<Type>{getGpuRuntimeResultType()},
-                               builder.getSymbolRefAttr(gpuModuleLoad),
-                               ArrayRef<Value>{gpuModule, data});
+  auto module = builder.create<LLVM::CallOp>(
+      loc, ArrayRef<Type>{getPointerType()},
+      builder.getSymbolRefAttr(gpuModuleLoad), ArrayRef<Value>{data});
   // Get the function from the module. The name corresponds to the name of
   // the kernel function.
-  auto gpuOwningModuleRef =
-      builder.create<LLVM::LoadOp>(loc, getPointerType(), gpuModule);
   auto kernelName = generateKernelNameConstant(
       launchOp.getKernelModuleName(), launchOp.getKernelName(), loc, builder);
-  auto gpuFunction = allocatePointer(builder, loc);
   auto gpuModuleGetFunction =
       getOperation().lookupSymbol<LLVM::LLVMFuncOp>(kGpuModuleGetFunctionName);
-  builder.create<LLVM::CallOp>(
-      loc, ArrayRef<Type>{getGpuRuntimeResultType()},
+  auto function = builder.create<LLVM::CallOp>(
+      loc, ArrayRef<Type>{getPointerType()},
       builder.getSymbolRefAttr(gpuModuleGetFunction),
-      ArrayRef<Value>{gpuFunction, gpuOwningModuleRef, kernelName});
+      ArrayRef<Value>{module.getResult(0), kernelName});
   // Grab the global stream needed for execution.
-  auto gpuGetStreamHelper =
-      getOperation().lookupSymbol<LLVM::LLVMFuncOp>(kGpuGetStreamHelperName);
-  auto gpuStream = builder.create<LLVM::CallOp>(
+  auto gpuStreamCreate =
+      getOperation().lookupSymbol<LLVM::LLVMFuncOp>(kGpuStreamCreateName);
+  auto stream = builder.create<LLVM::CallOp>(
       loc, ArrayRef<Type>{getPointerType()},
-      builder.getSymbolRefAttr(gpuGetStreamHelper), ArrayRef<Value>{});
+      builder.getSymbolRefAttr(gpuStreamCreate), ArrayRef<Value>{});
   // Invoke the function with required arguments.
   auto gpuLaunchKernel =
       getOperation().lookupSymbol<LLVM::LLVMFuncOp>(kGpuLaunchKernelName);
-  auto gpuFunctionRef =
-      builder.create<LLVM::LoadOp>(loc, getPointerType(), gpuFunction);
   auto paramsArray = setupParamsArray(launchOp, builder);
   if (!paramsArray) {
     launchOp.emitOpError() << "cannot pass given parameters to the kernel";
@@ -443,21 +421,21 @@ void GpuLaunchFuncToGpuRuntimeCallsPass::translateGpuLaunchCalls(
   auto nullpointer =
       builder.create<LLVM::IntToPtrOp>(loc, getPointerPointerType(), zero);
   builder.create<LLVM::CallOp>(
-      loc, ArrayRef<Type>{getGpuRuntimeResultType()},
+      loc, ArrayRef<Type>{getVoidType()},
       builder.getSymbolRefAttr(gpuLaunchKernel),
-      ArrayRef<Value>{gpuFunctionRef, launchOp.getOperand(0),
+      ArrayRef<Value>{function.getResult(0), launchOp.getOperand(0),
                       launchOp.getOperand(1), launchOp.getOperand(2),
                       launchOp.getOperand(3), launchOp.getOperand(4),
                       launchOp.getOperand(5), zero, /* sharedMemBytes */
-                      gpuStream.getResult(0),       /* stream */
+                      stream.getResult(0),          /* stream */
                       paramsArray,                  /* kernel params */
                       nullpointer /* extra */});
   // Sync on the stream to make it synchronous.
   auto gpuStreamSync =
       getOperation().lookupSymbol<LLVM::LLVMFuncOp>(kGpuStreamSynchronizeName);
-  builder.create<LLVM::CallOp>(loc, ArrayRef<Type>{getGpuRuntimeResultType()},
+  builder.create<LLVM::CallOp>(loc, ArrayRef<Type>{getVoidType()},
                                builder.getSymbolRefAttr(gpuStreamSync),
-                               ArrayRef<Value>(gpuStream.getResult(0)));
+                               ArrayRef<Value>(stream.getResult(0)));
   launchOp.erase();
 }
 
diff --git a/mlir/test/Conversion/GPUCommon/lower-launch-func-to-gpu-runtime-calls.mlir b/mlir/test/Conversion/GPUCommon/lower-launch-func-to-gpu-runtime-calls.mlir
index a3381465ebf2c..bdcde0be60c20 100644
--- a/mlir/test/Conversion/GPUCommon/lower-launch-func-to-gpu-runtime-calls.mlir
+++ b/mlir/test/Conversion/GPUCommon/lower-launch-func-to-gpu-runtime-calls.mlir
@@ -20,13 +20,11 @@ module attributes {gpu.container_module} {
 
     // CHECK: %[[addressof:.*]] = llvm.mlir.addressof @[[global]]
     // CHECK: %[[c0:.*]] = llvm.mlir.constant(0 : index)
-    // CHECK: %[[binary_ptr:.*]] = llvm.getelementptr %[[addressof]][%[[c0]], %[[c0]]]
+    // CHECK: %[[binary:.*]] = llvm.getelementptr %[[addressof]][%[[c0]], %[[c0]]]
     // CHECK-SAME: -> !llvm<"i8*">
-    // CHECK: %[[module_ptr:.*]] = llvm.alloca {{.*}} x !llvm<"i8*"> : (!llvm.i32) -> !llvm<"i8**">
-    // CHECK: llvm.call @mgpuModuleLoad(%[[module_ptr]], %[[binary_ptr]]) : (!llvm<"i8**">, !llvm<"i8*">) -> !llvm.i32
-    // CHECK: %[[func_ptr:.*]] = llvm.alloca {{.*}} x !llvm<"i8*"> : (!llvm.i32) -> !llvm<"i8**">
-    // CHECK: llvm.call @mgpuModuleGetFunction(%[[func_ptr]], {{.*}}, {{.*}}) : (!llvm<"i8**">, !llvm<"i8*">, !llvm<"i8*">) -> !llvm.i32
-    // CHECK: llvm.call @mgpuGetStreamHelper
+    // CHECK: %[[module:.*]] = llvm.call @mgpuModuleLoad(%[[binary]]) : (!llvm<"i8*">) -> !llvm<"i8*">
+    // CHECK: %[[func:.*]] = llvm.call @mgpuModuleGetFunction(%[[module]], {{.*}}) : (!llvm<"i8*">, !llvm<"i8*">) -> !llvm<"i8*">
+    // CHECK: llvm.call @mgpuStreamCreate
     // CHECK: llvm.call @mgpuLaunchKernel
     // CHECK: llvm.call @mgpuStreamSynchronize
     "gpu.launch_func"(%cst, %cst, %cst, %cst, %cst, %cst, %0, %1) { kernel = @kernel_module::@kernel }
diff --git a/mlir/tools/mlir-cuda-runner/cuda-runtime-wrappers.cpp b/mlir/tools/mlir-cuda-runner/cuda-runtime-wrappers.cpp
index 2b71eb34703bd..8e2dc029fa9f0 100644
--- a/mlir/tools/mlir-cuda-runner/cuda-runtime-wrappers.cpp
+++ b/mlir/tools/mlir-cuda-runner/cuda-runtime-wrappers.cpp
@@ -21,54 +21,50 @@
 
 #include "cuda.h"
 
-namespace {
-int32_t reportErrorIfAny(CUresult result, const char *where) {
-  if (result != CUDA_SUCCESS) {
-    llvm::errs() << "CUDA failed with " << result << " in " << where << "\n";
-  }
-  return result;
+#define CUDA_REPORT_IF_ERROR(expr)                                             \
+  [](CUresult result) {                                                        \
+    if (!result)                                                               \
+      return;                                                                  \
+    const char *name = nullptr;                                                \
+    cuGetErrorName(result, &name);                                             \
+    if (!name)                                                                 \
+      name = "<unknown>";                                                      \
+    llvm::errs() << "'" << #expr << "' failed with '" << name << "'\n";        \
+  }(expr)
+
+extern "C" CUmodule mgpuModuleLoad(void *data) {
+  CUmodule module = nullptr;
+  CUDA_REPORT_IF_ERROR(cuModuleLoadData(&module, data));
+  return module;
 }
-} // anonymous namespace
 
-extern "C" int32_t mgpuModuleLoad(void **module, void *data) {
-  int32_t err = reportErrorIfAny(
-      cuModuleLoadData(reinterpret_cast<CUmodule *>(module), data),
-      "ModuleLoad");
-  return err;
-}
-
-extern "C" int32_t mgpuModuleGetFunction(void **function, void *module,
-                                         const char *name) {
-  return reportErrorIfAny(
-      cuModuleGetFunction(reinterpret_cast<CUfunction *>(function),
-                          reinterpret_cast<CUmodule>(module), name),
-      "GetFunction");
+extern "C" CUfunction mgpuModuleGetFunction(CUmodule module, const char *name) {
+  CUfunction function = nullptr;
+  CUDA_REPORT_IF_ERROR(cuModuleGetFunction(&function, module, name));
+  return function;
 }
 
 // The wrapper uses intptr_t instead of CUDA's unsigned int to match
 // the type of MLIR's index type. This avoids the need for casts in the
 // generated MLIR code.
-extern "C" int32_t mgpuLaunchKernel(void *function, intptr_t gridX,
-                                    intptr_t gridY, intptr_t gridZ,
-                                    intptr_t blockX, intptr_t blockY,
-                                    intptr_t blockZ, int32_t smem, void *stream,
-                                    void **params, void **extra) {
-  return reportErrorIfAny(
-      cuLaunchKernel(reinterpret_cast<CUfunction>(function), gridX, gridY,
-                     gridZ, blockX, blockY, blockZ, smem,
-                     reinterpret_cast<CUstream>(stream), params, extra),
-      "LaunchKernel");
+extern "C" void mgpuLaunchKernel(CUfunction function, intptr_t gridX,
+                                 intptr_t gridY, intptr_t gridZ,
+                                 intptr_t blockX, intptr_t blockY,
+                                 intptr_t blockZ, int32_t smem, CUstream stream,
+                                 void **params, void **extra) {
+  CUDA_REPORT_IF_ERROR(cuLaunchKernel(function, gridX, gridY, gridZ, blockX,
+                                      blockY, blockZ, smem, stream, params,
+                                      extra));
 }
 
-extern "C" void *mgpuGetStreamHelper() {
-  CUstream stream;
-  reportErrorIfAny(cuStreamCreate(&stream, CU_STREAM_DEFAULT), "StreamCreate");
+extern "C" CUstream mgpuStreamCreate() {
+  CUstream stream = nullptr;
+  CUDA_REPORT_IF_ERROR(cuStreamCreate(&stream, CU_STREAM_NON_BLOCKING));
   return stream;
 }
 
-extern "C" int32_t mgpuStreamSynchronize(void *stream) {
-  return reportErrorIfAny(
-      cuStreamSynchronize(reinterpret_cast<CUstream>(stream)), "StreamSync");
+extern "C" void mgpuStreamSynchronize(CUstream stream) {
+  CUDA_REPORT_IF_ERROR(cuStreamSynchronize(stream));
 }
 
 /// Helper functions for writing mlir example code
@@ -76,17 +72,16 @@ extern "C" int32_t mgpuStreamSynchronize(void *stream) {
 // Allows to register byte array with the CUDA runtime. Helpful until we have
 // transfer functions implemented.
 extern "C" void mgpuMemHostRegister(void *ptr, uint64_t sizeBytes) {
-  reportErrorIfAny(cuMemHostRegister(ptr, sizeBytes, /*flags=*/0),
-                   "MemHostRegister");
+  CUDA_REPORT_IF_ERROR(cuMemHostRegister(ptr, sizeBytes, /*flags=*/0));
 }
 
 // Allows to register a MemRef with the CUDA runtime. Initializes array with
 // value. Helpful until we have transfer functions implemented.
 template <typename T>
-void mgpuMemHostRegisterMemRef(const DynamicMemRefType<T> &mem_ref, T value) {
-  llvm::SmallVector<int64_t, 4> denseStrides(mem_ref.rank);
-  llvm::ArrayRef<int64_t> sizes(mem_ref.sizes, mem_ref.rank);
-  llvm::ArrayRef<int64_t> strides(mem_ref.strides, mem_ref.rank);
+void mgpuMemHostRegisterMemRef(const DynamicMemRefType<T> &memRef, T value) {
+  llvm::SmallVector<int64_t, 4> denseStrides(memRef.rank);
+  llvm::ArrayRef<int64_t> sizes(memRef.sizes, memRef.rank);
+  llvm::ArrayRef<int64_t> strides(memRef.strides, memRef.rank);
 
   std::partial_sum(sizes.rbegin(), sizes.rend(), denseStrides.rbegin(),
                    std::multiplies<int64_t>());
@@ -98,17 +93,17 @@ void mgpuMemHostRegisterMemRef(const DynamicMemRefType<T> &mem_ref, T value) {
   denseStrides.back() = 1;
   assert(strides == llvm::makeArrayRef(denseStrides));
 
-  auto *pointer = mem_ref.data + mem_ref.offset;
+  auto *pointer = memRef.data + memRef.offset;
   std::fill_n(pointer, count, value);
   mgpuMemHostRegister(pointer, count * sizeof(T));
 }
 
 extern "C" void mgpuMemHostRegisterFloat(int64_t rank, void *ptr) {
-  UnrankedMemRefType<float> mem_ref = {rank, ptr};
-  mgpuMemHostRegisterMemRef(DynamicMemRefType<float>(mem_ref), 1.23f);
+  UnrankedMemRefType<float> memRef = {rank, ptr};
+  mgpuMemHostRegisterMemRef(DynamicMemRefType<float>(memRef), 1.23f);
 }
 
 extern "C" void mgpuMemHostRegisterInt32(int64_t rank, void *ptr) {
-  UnrankedMemRefType<int32_t> mem_ref = {rank, ptr};
-  mgpuMemHostRegisterMemRef(DynamicMemRefType<int32_t>(mem_ref), 123);
+  UnrankedMemRefType<int32_t> memRef = {rank, ptr};
+  mgpuMemHostRegisterMemRef(DynamicMemRefType<int32_t>(memRef), 123);
 }
diff --git a/mlir/tools/mlir-rocm-runner/rocm-runtime-wrappers.cpp b/mlir/tools/mlir-rocm-runner/rocm-runtime-wrappers.cpp
index f49e6c91ea65e..b97ce695ac42c 100644
--- a/mlir/tools/mlir-rocm-runner/rocm-runtime-wrappers.cpp
+++ b/mlir/tools/mlir-rocm-runner/rocm-runtime-wrappers.cpp
@@ -21,56 +21,52 @@
 
 #include "hip/hip_runtime.h"
 
-namespace {
-int32_t reportErrorIfAny(hipError_t result, const char *where) {
-  if (result != hipSuccess) {
-    llvm::errs() << "HIP failed with " << result << " in " << where << "\n";
-  }
-  return result;
+#define HIP_REPORT_IF_ERROR(expr)                                              \
+  [](hipError_t result) {                                                      \
+    if (!result)                                                               \
+      return;                                                                  \
+    const char *name = nullptr;                                                \
+    hipGetErrorName(result, &name);                                            \
+    if (!name)                                                                 \
+      name = "<unknown>";                                                      \
+    llvm::errs() << "'" << #expr << "' failed with '" << name << "'\n";        \
+  }(expr)
+
+extern "C" hipModule_t mgpuModuleLoad(void *data) {
+  hipModule_t module = nullptr;
+  HIP_REPORT_IF_ERROR(hipModuleLoadData(&module, data));
+  return module;
 }
-} // anonymous namespace
 
-extern "C" int32_t mgpuModuleLoad(void **module, void *data) {
-  int32_t err = reportErrorIfAny(
-      hipModuleLoadData(reinterpret_cast<hipModule_t *>(module), data),
-      "ModuleLoad");
-  return err;
-}
-
-extern "C" int32_t mgpuModuleGetFunction(void **function, void *module,
-                                         const char *name) {
-  return reportErrorIfAny(
-      hipModuleGetFunction(reinterpret_cast<hipFunction_t *>(function),
-                           reinterpret_cast<hipModule_t>(module), name),
-      "GetFunction");
+extern "C" hipFunction_t mgpuModuleGetFunction(hipModule_t module,
+                                               const char *name) {
+  hipFunction_t function = nullptr;
+  HIP_REPORT_IF_ERROR(hipModuleGetFunction(&function, module, name));
+  return function;
 }
 
 // The wrapper uses intptr_t instead of ROCM's unsigned int to match
 // the type of MLIR's index type. This avoids the need for casts in the
 // generated MLIR code.
-extern "C" int32_t mgpuLaunchKernel(void *function, intptr_t gridX,
-                                    intptr_t gridY, intptr_t gridZ,
-                                    intptr_t blockX, intptr_t blockY,
-                                    intptr_t blockZ, int32_t smem, void *stream,
-                                    void **params, void **extra) {
-  return reportErrorIfAny(
-      hipModuleLaunchKernel(reinterpret_cast<hipFunction_t>(function), gridX,
-                            gridY, gridZ, blockX, blockY, blockZ, smem,
-                            reinterpret_cast<hipStream_t>(stream), params,
-                            extra),
-      "LaunchKernel");
+extern "C" void mgpuLaunchKernel(hipFunction_t function, intptr_t gridX,
+                                 intptr_t gridY, intptr_t gridZ,
+                                 intptr_t blockX, intptr_t blockY,
+                                 intptr_t blockZ, int32_t smem,
+                                 hipStream_t stream, void **params,
+                                 void **extra) {
+  HIP_REPORT_IF_ERROR(hipModuleLaunchKernel(function, gridX, gridY, gridZ,
+                                            blockX, blockY, blockZ, smem,
+                                            stream, params, extra));
 }
 
-extern "C" void *mgpuGetStreamHelper() {
-  hipStream_t stream;
-  reportErrorIfAny(hipStreamCreate(&stream), "StreamCreate");
+extern "C" void *mgpuStreamCreate() {
+  hipStream_t stream = nullptr;
+  HIP_REPORT_IF_ERROR(hipStreamCreate(&stream));
   return stream;
 }
 
-extern "C" int32_t mgpuStreamSynchronize(void *stream) {
-  return reportErrorIfAny(
-      hipStreamSynchronize(reinterpret_cast<hipStream_t>(stream)),
-      "StreamSync");
+extern "C" void mgpuStreamSynchronize(hipStream_t stream) {
+  return HIP_REPORT_IF_ERROR(hipStreamSynchronize(stream));
 }
 
 /// Helper functions for writing mlir example code
@@ -78,8 +74,8 @@ extern "C" int32_t mgpuStreamSynchronize(void *stream) {
 // Allows to register byte array with the ROCM runtime. Helpful until we have
 // transfer functions implemented.
 extern "C" void mgpuMemHostRegister(void *ptr, uint64_t sizeBytes) {
-  reportErrorIfAny(hipHostRegister(ptr, sizeBytes, /*flags=*/0),
-                   "MemHostRegister");
+  HIP_REPORT_IF_ERROR(hipHostRegister(ptr, sizeBytes, /*flags=*/0),
+                      "MemHostRegister");
 }
 
 // Allows to register a MemRef with the ROCM runtime. Initializes array with
@@ -120,8 +116,8 @@ extern "C" void mgpuMemHostRegisterInt32(int64_t rank, void *ptr) {
 
 template <typename T>
 void mgpuMemGetDevicePointer(T *hostPtr, T **devicePtr) {
-  reportErrorIfAny(hipSetDevice(0), "hipSetDevice");
-  reportErrorIfAny(
+  HIP_REPORT_IF_ERROR(hipSetDevice(0), "hipSetDevice");
+  HIP_REPORT_IF_ERROR(
       hipHostGetDevicePointer((void **)devicePtr, hostPtr, /*flags=*/0),
       "hipHostGetDevicePointer");
 }

From 01302ec08609ef3b113e13aba2a08443e4fa3046 Mon Sep 17 00:00:00 2001
From: Bruno Ricci <riccibrun@gmail.com>
Date: Tue, 28 Jul 2020 15:35:31 +0100
Subject: [PATCH 0303/1035] [Support][NFC] Fix a Wdocumentation warning in
 FileSystem.h

---
 llvm/include/llvm/Support/FileSystem.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/llvm/include/llvm/Support/FileSystem.h b/llvm/include/llvm/Support/FileSystem.h
index b6d2a9f3aad53..18d787a75f611 100644
--- a/llvm/include/llvm/Support/FileSystem.h
+++ b/llvm/include/llvm/Support/FileSystem.h
@@ -1140,7 +1140,7 @@ openNativeFileForRead(const Twine &Name, OpenFlags Flags = OF_None,
 /// none of other processes read or write this file, provided that all processes
 /// lock the file prior to accessing its content.
 ///
-/// @param File    The descriptor representing the file to lock.
+/// @param FD      The descriptor representing the file to lock.
 /// @param Timeout Time in milliseconds that the process should wait before
 ///                reporting lock failure. Zero value means try to get lock only
 ///                once.
@@ -1163,7 +1163,7 @@ std::error_code lockFile(int FD);
 
 /// Unlock the file.
 ///
-/// @param File The descriptor representing the file to unlock.
+/// @param FD The descriptor representing the file to unlock.
 /// @returns errc::success if lock is successfully released or platform-specific
 /// error_code otherwise.
 std::error_code unlockFile(int FD);

From 984cf99055a292b3afe4535c013d38914a3da880 Mon Sep 17 00:00:00 2001
From: Bruno Ricci <riccibrun@gmail.com>
Date: Tue, 28 Jul 2020 15:39:17 +0100
Subject: [PATCH 0304/1035] [clang][NFC] Add some documentation about the use
 of NamedDecl::getDeclName in diagnostics.

As explained in eb10b065f2a870b425dcc2040b9955e0eee464b4, sending a NamedDecl*
in a diagnostic should generally be preferred over sending the DeclarationName
from getDeclName(). Let's document that.
---
 clang/include/clang/AST/Decl.h | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/clang/include/clang/AST/Decl.h b/clang/include/clang/AST/Decl.h
index 28faa2c1fc780..4dd5e14d36e18 100644
--- a/clang/include/clang/AST/Decl.h
+++ b/clang/include/clang/AST/Decl.h
@@ -269,6 +269,19 @@ class NamedDecl : public Decl {
 
   /// Get the actual, stored name of the declaration, which may be a special
   /// name.
+  ///
+  /// Note that generally in diagnostics, the non-null \p NamedDecl* itself
+  /// should be sent into the diagnostic instead of using the result of
+  /// \p getDeclName().
+  ///
+  /// A \p DeclarationName in a diagnostic will just be streamed to the output,
+  /// which will directly result in a call to \p DeclarationName::print.
+  ///
+  /// A \p NamedDecl* in a diagnostic will also ultimately result in a call to
+  /// \p DeclarationName::print, but with two customisation points along the
+  /// way (\p getNameForDiagnostic and \p printName). These are used to print
+  /// the template arguments if any, and to provide a user-friendly name for
+  /// some entities (such as unnamed variables and anonymous records).
   DeclarationName getDeclName() const { return Name; }
 
   /// Set the name of this declaration.

From 54492a5843a34684ce21ae201dd8ca3e509288fd Mon Sep 17 00:00:00 2001
From: Sander de Smalen <sander.desmalen@arm.com>
Date: Mon, 27 Jul 2020 12:57:41 +0100
Subject: [PATCH 0305/1035] [AArch64][SVE] Don't support fixedStack for SVE
 objects.

Fixed stack objects are preallocated and defined to be allocated before
any of the regular stack objects. These are normally used to model stack
arguments.

The AAPCS does not support passing SVE registers on the stack by value
(only by reference). The current layout also doesn't place them before
all stack objects, but rather before all SVE objects. Removing this
simplifies the code that emits the allocation/deallocation
around callee-saved registers (D84042).

This patch also removes all uses of fixedStack from from
framelayout-sve.mir, where this was used purely for testing purposes.

Reviewers: paulwalker-arm, efriedma, rengolin

Reviewed By: paulwalker-arm

Differential Revision: https://reviews.llvm.org/D84538
---
 .../Target/AArch64/AArch64FrameLowering.cpp   |  13 +-
 llvm/test/CodeGen/AArch64/framelayout-sve.mir | 125 +++++++++---------
 2 files changed, 66 insertions(+), 72 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
index efa3fd5ca9cef..cc563dd706326 100644
--- a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
@@ -2595,20 +2595,21 @@ static int64_t determineSVEStackObjectOffsets(MachineFrameInfo &MFI,
                                               int &MinCSFrameIndex,
                                               int &MaxCSFrameIndex,
                                               bool AssignOffsets) {
+#ifndef NDEBUG
   // First process all fixed stack objects.
-  int64_t Offset = 0;
   for (int I = MFI.getObjectIndexBegin(); I != 0; ++I)
-    if (MFI.getStackID(I) == TargetStackID::SVEVector) {
-      int64_t FixedOffset = -MFI.getObjectOffset(I);
-      if (FixedOffset > Offset)
-        Offset = FixedOffset;
-    }
+    assert(MFI.getStackID(I) != TargetStackID::SVEVector &&
+           "SVE vectors should never be passed on the stack by value, only by "
+           "reference.");
+#endif
 
   auto Assign = [&MFI](int FI, int64_t Offset) {
     LLVM_DEBUG(dbgs() << "alloc FI(" << FI << ") at SP[" << Offset << "]\n");
     MFI.setObjectOffset(FI, Offset);
   };
 
+  int64_t Offset = 0;
+
   // Then process all callee saved slots.
   if (getSVECalleeSaveSlotRange(MFI, MinCSFrameIndex, MaxCSFrameIndex)) {
     // Make sure to align the last callee save slot.
diff --git a/llvm/test/CodeGen/AArch64/framelayout-sve.mir b/llvm/test/CodeGen/AArch64/framelayout-sve.mir
index 7903df64863bf..575c839fbd151 100644
--- a/llvm/test/CodeGen/AArch64/framelayout-sve.mir
+++ b/llvm/test/CodeGen/AArch64/framelayout-sve.mir
@@ -41,10 +41,10 @@
 # +----------+
 # |scratchreg|  // x29 is used as scratch reg.
 # +----------+
-# | %fixed-  |  // scalable SVE object of n * 18 bytes, aligned to 16 bytes,
-# |  stack.0 |  // to be materialized with 2*ADDVL (<=> 2 * n * 16bytes)
+# | %stack.0 |  // scalable SVE object of n * 18 bytes, aligned to 16 bytes,
+# |          |  // to be materialized with 2*ADDVL (<=> 2 * n * 16bytes)
 # +----------+
-# | %stack.0 |  // not scalable
+# | %stack.1 |  // not scalable
 # +----------+ <- SP
 
 # CHECK-LABEL: name: test_allocate_sve
@@ -60,10 +60,9 @@
 # CHECK-NEXT: $sp, $[[SCRATCH]] = frame-destroy LDRXpost $sp, 16
 # CHECK-NEXT: RET_ReallyLR
 name:            test_allocate_sve
-fixedStack:
-  - { id: 0, stack-id: sve-vec, size: 18, alignment: 2, offset: -18 }
 stack:
-  - { id: 0, stack-id: default, size: 16, alignment: 8 }
+  - { id: 0, stack-id: sve-vec, size: 18, alignment: 2 }
+  - { id: 1, stack-id: default, size: 16, alignment: 8 }
 body:             |
   bb.0.entry:
     RET_ReallyLR
@@ -73,10 +72,9 @@ body:             |
 # | x20, x21 |  // callee saves
 # |scratchreg|  // x29 is used as scratch reg.
 # +----------+
-# | %fixed-  |  // scalable objects
-# |  stack.0 |
+# | %stack.0 |  // scalable objects
 # +----------+
-# | %stack.0 |  // not scalable
+# | %stack.1 |  // not scalable
 # +----------+ <- SP
 
 # CHECK-LABEL: name: test_allocate_sve_gpr_callee_saves
@@ -95,10 +93,9 @@ body:             |
 # CHECK-NEXT: $sp, $[[SCRATCH]] = frame-destroy LDRXpost $sp, 32
 # CHECK-NEXT: RET_ReallyLR
 name:            test_allocate_sve_gpr_callee_saves
-fixedStack:
-  - { id: 0, stack-id: sve-vec, size: 18, alignment: 2, offset: -18 }
 stack:
-  - { id: 0, stack-id: default, size: 16, alignment: 8 }
+  - { id: 0, stack-id: sve-vec, size: 18, alignment: 2 }
+  - { id: 1, stack-id: default, size: 16, alignment: 8 }
 body:             |
   bb.0.entry:
     $x20 = IMPLICIT_DEF
@@ -109,11 +106,10 @@ body:             |
 # +----------+
 # |  lr, fp  |  // frame record
 # +----------+ <- FP
-# | %fixed-  |  // scalable objects
-# |  stack.0 |
+# | %stack.0 |  // scalable objects
 # +----------+
 # |//////////|  // alignment gap
-# | %stack.0 |  // not scalable
+# | %stack.1 |  // not scalable
 # +----------+ <- SP
 # CHECK-LABEL: name: test_allocate_sve_gpr_realigned
 # CHECK:       stackSize: 32
@@ -128,10 +124,9 @@ body:             |
 # CHECK-NEXT: $sp, $fp, $lr = frame-destroy LDPXpost $sp, 2
 # CHECK-NEXT: RET_ReallyLR
 name:            test_allocate_sve_gpr_realigned
-fixedStack:
-  - { id: 0, stack-id: sve-vec, size: 18, alignment: 2, offset: -18 }
 stack:
-  - { id: 0, stack-id: default, size: 16, alignment: 32 }
+  - { id: 0, stack-id: sve-vec, size: 18, alignment: 2  }
+  - { id: 1, stack-id: default, size: 16, alignment: 32 }
 body:             |
   bb.0.entry:
     RET_ReallyLR
@@ -144,7 +139,7 @@ body:             |
 # | %stack.1 |  // scalable @ SP + 16b + 16 scalable bytes
 # | %stack.2 |  // scalable @ SP + 16b + 14 scalable bytes
 # +----------+
-# | %stack.0 |  // not scalable
+# | %stack.3 |  // not scalable
 # +----------+ <- SP
 
 # CHECK-LABEL: name: test_address_sve
@@ -169,19 +164,18 @@ body:             |
 name:            test_address_sve
 frameInfo:
   maxAlignment:  16
-fixedStack:
-  - { id: 0, stack-id: sve-vec, size: 16, alignment: 8, offset: -16 }
-  - { id: 1, stack-id: sve-vec, size: 16, alignment: 8, offset: -32 }
-  - { id: 2, stack-id: sve-vec, size:  2, alignment: 2, offset: -34 }
 stack:
-  - { id: 0, stack-id: default, size: 16, alignment: 8 }
+  - { id: 0, stack-id: sve-vec, size: 16, alignment: 8 }
+  - { id: 1, stack-id: sve-vec, size: 16, alignment: 8 }
+  - { id: 2, stack-id: sve-vec, size:  2, alignment: 2 }
+  - { id: 3, stack-id: default, size: 16, alignment: 8 }
 body:             |
   bb.0.entry:
     liveins: $z0, $z1, $p0
 
-    STR_ZXI $z0, %fixed-stack.0, 0
-    STR_ZXI $z1, %fixed-stack.1, 0
-    STR_PXI $p0, %fixed-stack.2, 0
+    STR_ZXI $z0, %stack.0, 0
+    STR_ZXI $z1, %stack.1, 0
+    STR_PXI $p0, %stack.2, 0
 
     RET_ReallyLR
 ---
@@ -190,11 +184,11 @@ body:             |
 # | x20, x21  |  // callee saves
 # |  lr, fp   |  // frame record
 # +-----------+ <- FP
-# | %fstack.0 |  // scalable @ FP - 16 scalable bytes
-# | %fstack.1 |  // scalable @ FP - 32 scalable bytes
-# | %fstack.2 |  // scalable @ FP - 34 scalable bytes
+# | %stack.0  |  // scalable @ FP - 16 scalable bytes
+# | %stack.1  |  // scalable @ FP - 32 scalable bytes
+# | %stack.2  |  // scalable @ FP - 34 scalable bytes
 # +-----------+
-# | %stack.0  |  // not scalable
+# | %stack.3  |  // not scalable
 # +-----------+ <- SP
 
 # CHECK-LABEL: name: test_address_sve_fp
@@ -218,19 +212,18 @@ name:            test_address_sve_fp
 frameInfo:
   maxAlignment:  16
   isFrameAddressTaken: true
-fixedStack:
-  - { id: 0, stack-id: sve-vec, size: 16, alignment: 8, offset: -16 }
-  - { id: 1, stack-id: sve-vec, size: 16, alignment: 8, offset: -32 }
-  - { id: 2, stack-id: sve-vec, size:  2, alignment: 2, offset: -34 }
 stack:
-  - { id: 0, stack-id: default, size: 16, alignment: 8 }
+  - { id: 0, stack-id: sve-vec, size: 16, alignment: 8 }
+  - { id: 1, stack-id: sve-vec, size: 16, alignment: 8 }
+  - { id: 2, stack-id: sve-vec, size:  2, alignment: 2 }
+  - { id: 3, stack-id: default, size: 16, alignment: 8 }
 body:             |
   bb.0.entry:
     liveins: $z0, $z1, $p0
 
-    STR_ZXI $z0, %fixed-stack.0, 0
-    STR_ZXI $z1, %fixed-stack.1, 0
-    STR_PXI $p0, %fixed-stack.2, 0
+    STR_ZXI $z0, %stack.0, 0
+    STR_ZXI $z1, %stack.1, 0
+    STR_PXI $p0, %stack.2, 0
 
     RET_ReallyLR
 ---
@@ -240,9 +233,9 @@ body:             |
 # +-----------+
 # |callee save|  // register saved as scratch reg.
 # +-----------+
-# | %fstack.1 |  // vector of 16 scalable bytes
+# | %stack.0  |  // vector of 16 scalable bytes
 # +---------- +
-# | %stack.0  |  // not scalable, 16 bytes
+# | %stack.1  |  // not scalable, 16 bytes
 # +-----------+ <- SP
 # CHECK-LABEL: name: test_stack_arg_sve
 # CHECK:       stackSize: 32
@@ -262,9 +255,9 @@ body:             |
 name:             test_stack_arg_sve
 fixedStack:
   - { id: 0, stack-id: default, size: 16, alignment: 16, offset: 0 }
-  - { id: 1, stack-id: sve-vec, size: 16, alignment: 16, offset: -16 }
 stack:
-  - { id: 0, stack-id: default, size: 16, alignment: 16 }
+  - { id: 0, stack-id: sve-vec, size: 16, alignment: 16 }
+  - { id: 1, stack-id: default, size: 16, alignment: 16 }
 body:             |
   bb.0.entry:
     liveins: $x0
@@ -320,17 +313,17 @@ body:             |
 name:            test_address_sve_out_of_range
 frameInfo:
   maxAlignment:  16
-fixedStack:
-  - { id: 0, stack-id: sve-vec, size:   16, alignment: 16, offset: -16 }
-  - { id: 1, stack-id: sve-vec, size: 3584, alignment: 16, offset: -3600 }
-  - { id: 2, stack-id: sve-vec, size:  512, alignment: 16, offset: -4112 }
+stack:
+  - { id: 0, stack-id: sve-vec, size:   16, alignment: 16 }
+  - { id: 1, stack-id: sve-vec, size: 3584, alignment: 16 }
+  - { id: 2, stack-id: sve-vec, size:  512, alignment: 16 }
 
 body:             |
   bb.0.entry:
     liveins: $z0, $p0
 
-    STR_ZXI $z0, %fixed-stack.0, 0
-    STR_PXI $p0, %fixed-stack.1, 0
+    STR_ZXI $z0, %stack.0, 0
+    STR_PXI $p0, %stack.1, 0
 
     RET_ReallyLR
 ---
@@ -340,11 +333,11 @@ body:             |
 # access from the FP when there are also SVE objects on the stack.
 #
 # +----------+ <- FP
-# | %fstack.0|  // 16 scalable bytes
+# | %stack.0 |  // 16 scalable bytes
 # +----------+ <- @FP - 16 scalable bytes
-# | %stack.0 |  // 16 bytes
+# | %stack.1 |  // 16 bytes
 # +----------+ <- @BP
-# : %stack.1 :  // variable length
+# : %stack.2 :  // variable length
 # +----------+ <- SP
 
 # CHECK-LABEL: name: test_address_gpr_vla
@@ -354,16 +347,15 @@ body:             |
 name:            test_address_gpr_vla
 frameInfo:
   maxAlignment:  16
-fixedStack:
-  - { id: 0, stack-id: sve-vec, size: 16, alignment: 8, offset: -16 }
 stack:
-  - { id: 0, stack-id: default, size: 16, alignment: 8 }
-  - { id: 1, stack-id: default, type: variable-sized }
+  - { id: 0, stack-id: sve-vec, size: 16, alignment: 8 }
+  - { id: 1, stack-id: default, size: 16, alignment: 8 }
+  - { id: 2, stack-id: default, type: variable-sized }
 body:             |
   bb.0.entry:
     liveins: $xzr
 
-    STRXui $xzr, %stack.0, 0
+    STRXui $xzr, %stack.1, 0
 
     RET_ReallyLR
 ---
@@ -429,7 +421,7 @@ body:             |
 # CHECK-LABEL: name: save_restore_sve
 # CHECK: $sp = frame-setup STPXpre killed ${{[a-z0-9]+}}, killed $x21, $sp, -4
 # CHECK: frame-setup STPXi killed $x20, killed $x19, $sp, 2
-# CHECK: $sp = frame-setup ADDVL_XXI $sp, -19
+# CHECK: $sp = frame-setup ADDVL_XXI $sp, -18
 # CHECK: frame-setup STR_PXI killed $p15, $sp, 4
 # CHECK: frame-setup STR_PXI killed $p14, $sp, 5
 # CHECK: frame-setup STR_PXI killed $p5, $sp, 14
@@ -438,9 +430,11 @@ body:             |
 # CHECK: frame-setup STR_ZXI killed $z22, $sp, 3
 # CHECK: frame-setup STR_ZXI killed $z9, $sp, 16
 # CHECK: frame-setup STR_ZXI killed $z8, $sp, 17
+# CHECK: $sp = frame-setup ADDVL_XXI $sp, -1
 # CHECK: $sp = frame-setup SUBXri $sp, 32, 0
 
 # CHECK: $sp = frame-destroy ADDXri $sp, 32, 0
+# CHECK: $sp = frame-destroy ADDVL_XXI $sp, 1
 # CHECK: $p15 = frame-destroy LDR_PXI $sp, 4
 # CHECK: $p14 = frame-destroy LDR_PXI $sp, 5
 # CHECK: $p5 = frame-destroy LDR_PXI $sp, 14
@@ -449,15 +443,14 @@ body:             |
 # CHECK: $z22 = frame-destroy LDR_ZXI $sp, 3
 # CHECK: $z9 = frame-destroy LDR_ZXI $sp, 16
 # CHECK: $z8 = frame-destroy LDR_ZXI $sp, 17
-# CHECK: $sp = frame-destroy ADDVL_XXI $sp, 19
+# CHECK: $sp = frame-destroy ADDVL_XXI $sp, 18
 # CHECK: $x20, $x19 = frame-destroy LDPXi $sp, 2
 # CHECK: $sp, ${{[a-z0-9]+}}, $x21 = frame-destroy LDPXpost $sp, 4
 # CHECK: RET_ReallyLR
 name: save_restore_sve
-fixedStack:
-  - { id: 0, stack-id: sve-vec, size: 16, alignment: 16, offset: -16 }
 stack:
-  - { id: 0, stack-id: default, size: 32, alignment: 16 }
+  - { id: 0, stack-id: sve-vec, size: 16, alignment: 16 }
+  - { id: 1, stack-id: default, size: 32, alignment: 16 }
 body:             |
   bb.0.entry:
 
@@ -494,7 +487,7 @@ body:             |
 # CHECK-LABEL: name: save_restore_sve_realign
 # CHECK:      $sp = frame-setup STPXpre killed $fp, killed $lr, $sp, -2
 # CHECK-NEXT: $fp = frame-setup ADDXri $sp, 0, 0
-# CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -19
+# CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -18
 # CHECK-NEXT: STR_PXI killed $p15, $sp, 4
 # CHECK-NEXT: STR_PXI killed $p14, $sp, 5
 # CHECK:      STR_PXI killed $p5, $sp, 14
@@ -503,6 +496,7 @@ body:             |
 # CHECK-NEXT: STR_ZXI killed $z22, $sp, 3
 # CHECK:      STR_ZXI killed $z9, $sp, 16
 # CHECK-NEXT: STR_ZXI killed $z8, $sp, 17
+# CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -1
 # CHECK-NEXT: $[[TMP:x[0-9]+]] = frame-setup SUBXri $sp, 16, 0
 # CHECK-NEXT: $sp = ANDXri killed $[[TMP]]
 
@@ -519,10 +513,9 @@ body:             |
 # CHECK-NEXT: $sp, $fp, $lr = frame-destroy LDPXpost $sp, 2
 # CHECK-NEXT: RET_ReallyLR
 name: save_restore_sve_realign
-fixedStack:
-  - { id: 0, stack-id: sve-vec, size: 16, alignment: 16, offset: -16 }
 stack:
-  - { id: 0, stack-id: default, size: 16, alignment: 32 }
+  - { id: 0, stack-id: sve-vec, size: 16, alignment: 16 }
+  - { id: 1, stack-id: default, size: 16, alignment: 32 }
 body:             |
   bb.0.entry:
 

From 26b4ef3694973ea2fa656d3d3a7f67f16f135654 Mon Sep 17 00:00:00 2001
From: Sander de Smalen <sander.desmalen@arm.com>
Date: Mon, 27 Jul 2020 14:16:55 +0100
Subject: [PATCH 0306/1035] [AArch64][SVE] Don't align the last SVE callee
 save.

Instead of aligning the last callee-saved-register slot to the stack
alignment (16 bytes), just align the SVE callee-saved block. This also
simplifies the code that allocates space for the callee-saves.

This change is needed to make sure the offset to which the callee-saved
register is spilled, corresponds to the offset used for e.g. unwind call
frame instructions.

Reviewers: efriedma, paulwalker-arm, david-arm, rengolin

Reviewed By: efriedma

Differential Revision: https://reviews.llvm.org/D84042
---
 .../Target/AArch64/AArch64FrameLowering.cpp   | 25 +++++++------------
 llvm/test/CodeGen/AArch64/framelayout-sve.mir |  2 +-
 llvm/test/CodeGen/AArch64/sve-trunc.ll        |  2 +-
 3 files changed, 11 insertions(+), 18 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
index cc563dd706326..1b49c692f2931 100644
--- a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
@@ -1192,7 +1192,7 @@ void AArch64FrameLowering::emitPrologue(MachineFunction &MF,
 
   // Process the SVE callee-saves to determine what space needs to be
   // allocated.
-  if (AFI->getSVECalleeSavedStackSize()) {
+  if (int64_t CalleeSavedSize = AFI->getSVECalleeSavedStackSize()) {
     // Find callee save instructions in frame.
     CalleeSavesBegin = MBBI;
     assert(IsSVECalleeSave(CalleeSavesBegin) && "Unexpected instruction");
@@ -1200,11 +1200,7 @@ void AArch64FrameLowering::emitPrologue(MachineFunction &MF,
       ++MBBI;
     CalleeSavesEnd = MBBI;
 
-    int64_t OffsetToFirstCalleeSaveFromSP =
-        MFI.getObjectOffset(AFI->getMaxSVECSFrameIndex());
-    StackOffset OffsetToCalleeSavesFromSP =
-        StackOffset(OffsetToFirstCalleeSaveFromSP, MVT::nxv1i8) + SVEStackSize;
-    AllocateBefore -= OffsetToCalleeSavesFromSP;
+    AllocateBefore = {CalleeSavedSize, MVT::nxv1i8};
     AllocateAfter = SVEStackSize - AllocateBefore;
   }
 
@@ -1582,7 +1578,7 @@ void AArch64FrameLowering::emitEpilogue(MachineFunction &MF,
   // deallocated.
   StackOffset DeallocateBefore = {}, DeallocateAfter = SVEStackSize;
   MachineBasicBlock::iterator RestoreBegin = LastPopI, RestoreEnd = LastPopI;
-  if (AFI->getSVECalleeSavedStackSize()) {
+  if (int64_t CalleeSavedSize = AFI->getSVECalleeSavedStackSize()) {
     RestoreBegin = std::prev(RestoreEnd);;
     while (IsSVECalleeSave(RestoreBegin) &&
            RestoreBegin != MBB.begin())
@@ -1592,12 +1588,9 @@ void AArch64FrameLowering::emitEpilogue(MachineFunction &MF,
     assert(IsSVECalleeSave(RestoreBegin) &&
            IsSVECalleeSave(std::prev(RestoreEnd)) && "Unexpected instruction");
 
-    int64_t OffsetToFirstCalleeSaveFromSP =
-        MFI.getObjectOffset(AFI->getMaxSVECSFrameIndex());
-    StackOffset OffsetToCalleeSavesFromSP =
-        StackOffset(OffsetToFirstCalleeSaveFromSP, MVT::nxv1i8) + SVEStackSize;
-    DeallocateBefore = OffsetToCalleeSavesFromSP;
-    DeallocateAfter = SVEStackSize - DeallocateBefore;
+    StackOffset CalleeSavedSizeAsOffset = {CalleeSavedSize, MVT::nxv1i8};
+    DeallocateBefore = SVEStackSize - CalleeSavedSizeAsOffset;
+    DeallocateAfter = CalleeSavedSizeAsOffset;
   }
 
   // Deallocate the SVE area.
@@ -2612,9 +2605,6 @@ static int64_t determineSVEStackObjectOffsets(MachineFrameInfo &MFI,
 
   // Then process all callee saved slots.
   if (getSVECalleeSaveSlotRange(MFI, MinCSFrameIndex, MaxCSFrameIndex)) {
-    // Make sure to align the last callee save slot.
-    MFI.setObjectAlignment(MaxCSFrameIndex, Align(16));
-
     // Assign offsets to the callee save slots.
     for (int I = MinCSFrameIndex; I <= MaxCSFrameIndex; ++I) {
       Offset += MFI.getObjectSize(I);
@@ -2624,6 +2614,9 @@ static int64_t determineSVEStackObjectOffsets(MachineFrameInfo &MFI,
     }
   }
 
+  // Ensure that the Callee-save area is aligned to 16bytes.
+  Offset = alignTo(Offset, Align(16U));
+
   // Create a buffer of SVE objects to allocate and sort it.
   SmallVector<int, 8> ObjectsToAllocate;
   for (int I = 0, E = MFI.getObjectIndexEnd(); I != E; ++I) {
diff --git a/llvm/test/CodeGen/AArch64/framelayout-sve.mir b/llvm/test/CodeGen/AArch64/framelayout-sve.mir
index 575c839fbd151..75a65a6ad5226 100644
--- a/llvm/test/CodeGen/AArch64/framelayout-sve.mir
+++ b/llvm/test/CodeGen/AArch64/framelayout-sve.mir
@@ -573,7 +573,7 @@ body:             |
 # CHECK-NEXT:       stack-id: sve-vec, callee-saved-register: '$z23',
 # CHECK:        - { id: 8, name: '', type: spill-slot, offset: -34, size: 2, alignment: 2,
 # CHECK-NEXT:       stack-id: sve-vec, callee-saved-register: '$p4',
-# CHECK:        - { id: 9, name: '', type: spill-slot, offset: -48, size: 2, alignment: 16,
+# CHECK:        - { id: 9, name: '', type: spill-slot, offset: -36, size: 2, alignment: 2,
 # CHECK-NEXT:       stack-id: sve-vec, callee-saved-register: '$p15',
 # CHECK:        - { id: 10, name: '', type: spill-slot, offset: -16, size: 8, alignment: 16,
 # CHECK-NEXT:       stack-id: default, callee-saved-register: '$fp',
diff --git a/llvm/test/CodeGen/AArch64/sve-trunc.ll b/llvm/test/CodeGen/AArch64/sve-trunc.ll
index 46d152bbf7acb..af50176f6b101 100644
--- a/llvm/test/CodeGen/AArch64/sve-trunc.ll
+++ b/llvm/test/CodeGen/AArch64/sve-trunc.ll
@@ -117,7 +117,7 @@ define <vscale x 16 x i1> @trunc_i64toi1_split3(<vscale x 16 x i64> %in) {
 ; CHECK-NEXT:    addvl sp, sp, #-1
 ; CHECK-NEXT:    str p4, [sp, #7, mul vl] // 2-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-NEXT:    .cfi_offset p4, -16
+; CHECK-NEXT:    .cfi_offset p4, -2
 ; CHECK-NEXT:    .cfi_offset w29, -16
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    and z7.d, z7.d, #0x1

From cda2eb3ad2bbe923e74d6eb083af196a0622d800 Mon Sep 17 00:00:00 2001
From: Sander de Smalen <sander.desmalen@arm.com>
Date: Tue, 28 Jul 2020 12:11:09 +0100
Subject: [PATCH 0307/1035] [AArch64][SVE] Fix epilogue for SVE when the stack
 is realigned.

While deallocating the stackframe, the offset used to reload the
callee-saved registers was not pointing to the SVE callee-saves,
but rather to the whole SVE area.

   +--------------+
   | GRP callee   |
   |     saves    |
   +--------------+ <- FP
   | SVE callee   |
   |     saves    |
   +--------------+ <- Should restore SVE callee saves from here
   |  SVE Spills  |
   |  and Locals  |
   +--------------+ <- instead of from here.
   |              |
   :              :
   |              |
   +--------------+ <- SP

Reviewed By: paulwalker-arm

Differential Revision: https://reviews.llvm.org/D84539
---
 llvm/lib/Target/AArch64/AArch64FrameLowering.cpp | 9 +++++----
 llvm/test/CodeGen/AArch64/framelayout-sve.mir    | 2 +-
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
index 1b49c692f2931..4789a9f02937a 100644
--- a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
@@ -1596,12 +1596,13 @@ void AArch64FrameLowering::emitEpilogue(MachineFunction &MF,
   // Deallocate the SVE area.
   if (SVEStackSize) {
     if (AFI->isStackRealigned()) {
-      if (AFI->getSVECalleeSavedStackSize())
-        // Set SP to start of SVE area, from which the callee-save reloads
-        // can be done. The code below will deallocate the stack space
+      if (int64_t CalleeSavedSize = AFI->getSVECalleeSavedStackSize())
+        // Set SP to start of SVE callee-save area from which they can
+        // be reloaded. The code below will deallocate the stack space
         // space by moving FP -> SP.
         emitFrameOffset(MBB, RestoreBegin, DL, AArch64::SP, AArch64::FP,
-                        -SVEStackSize, TII, MachineInstr::FrameDestroy);
+                        {-CalleeSavedSize, MVT::nxv1i8}, TII,
+                        MachineInstr::FrameDestroy);
     } else {
       if (AFI->getSVECalleeSavedStackSize()) {
         // Deallocate the non-SVE locals first before we can deallocate (and
diff --git a/llvm/test/CodeGen/AArch64/framelayout-sve.mir b/llvm/test/CodeGen/AArch64/framelayout-sve.mir
index 75a65a6ad5226..668b243dd79e0 100644
--- a/llvm/test/CodeGen/AArch64/framelayout-sve.mir
+++ b/llvm/test/CodeGen/AArch64/framelayout-sve.mir
@@ -500,7 +500,7 @@ body:             |
 # CHECK-NEXT: $[[TMP:x[0-9]+]] = frame-setup SUBXri $sp, 16, 0
 # CHECK-NEXT: $sp = ANDXri killed $[[TMP]]
 
-# CHECK:      $sp = frame-destroy ADDVL_XXI $fp, -19
+# CHECK:      $sp = frame-destroy ADDVL_XXI $fp, -18
 # CHECK-NEXT: $p15 = frame-destroy LDR_PXI $sp, 4
 # CHECK-NEXT: $p14 = frame-destroy LDR_PXI $sp, 5
 # CHECK:      $p5 = frame-destroy LDR_PXI $sp, 14

From 6784d82d5b859cc14a63358d34ed50e6e2f9bf43 Mon Sep 17 00:00:00 2001
From: Xing GUO <higuoxing@gmail.com>
Date: Tue, 28 Jul 2020 22:20:10 +0800
Subject: [PATCH 0308/1035] [DWARFYAML] Rename checkListEntryOperands() to
 checkOperandCount(). NFC.

This patch renames checkListEntryOperands() to checkOperandCount(), so
that we are able to check DWARF expression operands using the same
function.

Reviewed By: jhenderson, labath

Differential Revision: https://reviews.llvm.org/D84624
---
 llvm/lib/ObjectYAML/DWARFEmitter.cpp | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/ObjectYAML/DWARFEmitter.cpp b/llvm/lib/ObjectYAML/DWARFEmitter.cpp
index b39bb003db9a5..66fabe875f7b0 100644
--- a/llvm/lib/ObjectYAML/DWARFEmitter.cpp
+++ b/llvm/lib/ObjectYAML/DWARFEmitter.cpp
@@ -543,9 +543,9 @@ Error DWARFYAML::emitDebugStrOffsets(raw_ostream &OS, const Data &DI) {
   return Error::success();
 }
 
-static Error checkListEntryOperands(StringRef EncodingString,
-                                    ArrayRef<yaml::Hex64> Values,
-                                    uint64_t ExpectedOperands) {
+static Error checkOperandCount(StringRef EncodingString,
+                               ArrayRef<yaml::Hex64> Values,
+                               uint64_t ExpectedOperands) {
   if (Values.size() != ExpectedOperands)
     return createStringError(
         errc::invalid_argument,
@@ -578,7 +578,7 @@ static Expected<uint64_t> writeListEntry(raw_ostream &OS,
   StringRef EncodingName = dwarf::RangeListEncodingString(Entry.Operator);
 
   auto CheckOperands = [&](uint64_t ExpectedOperands) -> Error {
-    return checkListEntryOperands(EncodingName, Entry.Values, ExpectedOperands);
+    return checkOperandCount(EncodingName, Entry.Values, ExpectedOperands);
   };
 
   auto WriteAddress = [&](uint64_t Addr) -> Error {

From ca0bf440dbf9977340db4a32ba61740930c2be03 Mon Sep 17 00:00:00 2001
From: Camille Coti <coti@cs.uoregon.edu>
Date: Mon, 27 Jul 2020 16:58:39 -0600
Subject: [PATCH 0309/1035] Order of libraries and source files in the f18
 frontend

When the f18 frontend calls the link editor, put the libraries and object files in the correct order.

Fixes the issues reported here https://github.com/flang-compiler/flang/issues/897

Reviewed By: sscalpone, AlexisPerry

Differential Revision: https://reviews.llvm.org/D84340
---
 flang/tools/f18/f18.cpp | 34 ++++++++++++++++++++++------------
 1 file changed, 22 insertions(+), 12 deletions(-)

diff --git a/flang/tools/f18/f18.cpp b/flang/tools/f18/f18.cpp
index 2b2eacc2e6f11..dc5ddd12295f6 100644
--- a/flang/tools/f18/f18.cpp
+++ b/flang/tools/f18/f18.cpp
@@ -369,20 +369,24 @@ std::string CompileOtherLanguage(std::string path, DriverOptions &driver) {
   return {};
 }
 
-void Link(std::vector<std::string> &relocatables, DriverOptions &driver) {
+void Link(std::vector<std::string> &liblist, std::vector<std::string> &objects,
+    DriverOptions &driver) {
   if (!ParentProcess()) {
     std::vector<char *> argv;
     for (size_t j{0}; j < driver.F18_FCArgs.size(); ++j) {
       argv.push_back(driver.F18_FCArgs[j].data());
     }
-    for (auto &relo : relocatables) {
-      argv.push_back(relo.data());
+    for (auto &obj : objects) {
+      argv.push_back(obj.data());
     }
     if (!driver.outputPath.empty()) {
       char dashO[3] = "-o";
       argv.push_back(dashO);
       argv.push_back(driver.outputPath.data());
     }
+    for (auto &lib : liblist) {
+      argv.push_back(lib.data());
+    }
     Exec(argv, driver.verbose);
   }
 }
@@ -397,6 +401,7 @@ int main(int argc, char *const argv[]) {
   bool isPGF90{driver.F18_FCArgs.back().rfind("pgf90") != std::string::npos};
 
   std::list<std::string> args{argList(argc, argv)};
+  std::vector<std::string> objlist, liblist;
   std::string prefix{args.front()};
   args.pop_front();
   prefix += ": ";
@@ -413,32 +418,37 @@ int main(int argc, char *const argv[]) {
 
   Fortran::common::IntrinsicTypeDefaultKinds defaultKinds;
 
-  std::vector<std::string> fortranSources, otherSources, relocatables;
+  std::vector<std::string> fortranSources, otherSources;
   bool anyFiles{false};
 
   while (!args.empty()) {
     std::string arg{std::move(args.front())};
+    auto dot{arg.rfind(".")};
+    std::string suffix{arg.substr(dot + 1)};
+    std::string prefix{arg.substr(0, 2)};
     args.pop_front();
     if (arg.empty()) {
     } else if (arg.at(0) != '-') {
       anyFiles = true;
-      auto dot{arg.rfind(".")};
       if (dot == std::string::npos) {
         driver.F18_FCArgs.push_back(arg);
       } else {
-        std::string suffix{arg.substr(dot + 1)};
         if (suffix == "f" || suffix == "F" || suffix == "ff" ||
             suffix == "f90" || suffix == "F90" || suffix == "ff90" ||
             suffix == "f95" || suffix == "F95" || suffix == "ff95" ||
             suffix == "cuf" || suffix == "CUF" || suffix == "f18" ||
             suffix == "F18" || suffix == "ff18") {
           fortranSources.push_back(arg);
-        } else if (suffix == "o" || suffix == "a") {
-          relocatables.push_back(arg);
+        } else if (suffix == "o" || suffix == "so") {
+          objlist.push_back(arg);
+        } else if (suffix == "a") {
+          liblist.push_back(arg);
         } else {
           otherSources.push_back(arg);
         }
       }
+    } else if (prefix == "-l" || suffix == "a") {
+      liblist.push_back(arg);
     } else if (arg == "-") {
       fortranSources.push_back("-");
     } else if (arg == "--") {
@@ -682,17 +692,17 @@ int main(int argc, char *const argv[]) {
   for (const auto &path : fortranSources) {
     std::string relo{CompileFortran(path, options, driver, defaultKinds)};
     if (!driver.compileOnly && !relo.empty()) {
-      relocatables.push_back(relo);
+      objlist.push_back(relo);
     }
   }
   for (const auto &path : otherSources) {
     std::string relo{CompileOtherLanguage(path, driver)};
     if (!driver.compileOnly && !relo.empty()) {
-      relocatables.push_back(relo);
+      objlist.push_back(relo);
     }
   }
-  if (!relocatables.empty()) {
-    Link(relocatables, driver);
+  if (!objlist.empty()) {
+    Link(liblist, objlist, driver);
   }
   return exitStatus;
 }

From be2ea29ee16bc132626cba07559e9f023ad6ac13 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Sun, 12 Jul 2020 23:05:43 +0100
Subject: [PATCH 0310/1035] [SCEV] Add additional tests.

Increase test coverage for upcoming changes to how SCEV deals with LCSSA
phis.
---
 .../Analysis/ScalarEvolution/trivial-phis.ll  | 191 ++++++
 .../scev-expander-preserve-lcssa.ll           | 587 ++++++++++++++++++
 .../Transforms/LoopStrengthReduce/funclet.ll  | 212 +++++--
 .../LoopStrengthReduce/scev-expander-lcssa.ll | 144 +++++
 4 files changed, 1098 insertions(+), 36 deletions(-)
 create mode 100644 llvm/test/Analysis/ScalarEvolution/trivial-phis.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/scev-expander-preserve-lcssa.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/scev-expander-lcssa.ll

diff --git a/llvm/test/Analysis/ScalarEvolution/trivial-phis.ll b/llvm/test/Analysis/ScalarEvolution/trivial-phis.ll
new file mode 100644
index 0000000000000..3a897e957d1b2
--- /dev/null
+++ b/llvm/test/Analysis/ScalarEvolution/trivial-phis.ll
@@ -0,0 +1,191 @@
+; RUN: opt < %s -analyze -scalar-evolution | FileCheck %s
+
+; CHECK-LABEL @test1
+; CHECK       %add.lcssa.wide = phi i64 [ %indvars.iv.next, %do.body ]
+; CHECK-NEXT  -->  %add.lcssa.wide U: [1,2147483648) S: [1,2147483648)
+
+define i64 @test1(i32 signext %n, float* %A) {
+entry:
+  %0 = sext i32 %n to i64
+  br label %do.body
+
+do.body:                                          ; preds = %do.body, %entry
+  %indvars.iv = phi i64 [ %indvars.iv.next, %do.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %A, i64 %indvars.iv
+  store float 1.000000e+00, float* %arrayidx, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %cmp = icmp slt i64 %indvars.iv.next, %0
+  br i1 %cmp, label %do.body, label %do.end
+
+do.end:                                           ; preds = %do.body
+  %add.lcssa.wide = phi i64 [ %indvars.iv.next, %do.body ]
+  ret i64 %add.lcssa.wide
+}
+
+; CHECK-LABEL @test2
+; CHECK:      %tmp24 = phi i64 [ %tmp14, %bb22 ], [ %tmp14, %bb13 ]
+; CHECK-NEXT: -->  %tmp24 U: full-set S: full-set       Exits: <<Unknown>>      LoopDispositions: { %bb13: Variant, %bb8: Variant, %bb17: Invariant, %bb27: Invariant }
+
+define void @test2(i64 %arg, i32* noalias %arg1) {
+bb:
+  %tmp = icmp slt i64 0, %arg
+  br i1 %tmp, label %bb7, label %bb48
+
+bb7:                                              ; preds = %bb
+  br label %bb8
+
+bb8:                                              ; preds = %bb44, %bb7
+  %tmp9 = phi i64 [ 0, %bb7 ], [ %tmp45, %bb44 ]
+  %tmp10 = add nsw i64 %arg, -1
+  %tmp11 = icmp slt i64 1, %tmp10
+  br i1 %tmp11, label %bb12, label %bb43
+
+bb12:                                             ; preds = %bb8
+  br label %bb13
+
+bb13:                                             ; preds = %bb39, %bb12
+  %tmp14 = phi i64 [ 1, %bb12 ], [ %tmp40, %bb39 ]
+  %tmp15 = icmp slt i64 0, %arg
+  br i1 %tmp15, label %bb16, label %bb23
+
+bb16:                                             ; preds = %bb13
+  br label %bb17
+
+bb17:                                             ; preds = %bb19, %bb16
+  %tmp18 = phi i64 [ 0, %bb16 ], [ %tmp20, %bb19 ]
+  br label %bb19
+
+bb19:                                             ; preds = %bb17
+  %tmp20 = add nuw nsw i64 %tmp18, 1
+  %tmp21 = icmp slt i64 %tmp20, %arg
+  br i1 %tmp21, label %bb17, label %bb22
+
+bb22:                                             ; preds = %bb19
+  br label %bb23
+
+bb23:                                             ; preds = %bb22, %bb13
+  %tmp24 = phi i64 [ %tmp14, %bb22 ], [ %tmp14, %bb13 ]
+  %tmp25 = icmp slt i64 0, %arg
+  br i1 %tmp25, label %bb26, label %bb37
+
+bb26:                                             ; preds = %bb23
+  br label %bb27
+
+bb27:                                             ; preds = %bb33, %bb26
+  %tmp28 = phi i64 [ 0, %bb26 ], [ %tmp34, %bb33 ]
+  %tmp29 = mul nsw i64 %tmp9, %arg
+  %tmp30 = getelementptr inbounds i32, i32* %arg1, i64 %tmp24
+  %tmp31 = getelementptr inbounds i32, i32* %tmp30, i64 %tmp29
+  %tmp32 = load i32, i32* %tmp31, align 4
+  br label %bb33
+
+bb33:                                             ; preds = %bb27
+  %tmp34 = add nuw nsw i64 %tmp28, 1
+  %tmp35 = icmp slt i64 %tmp34, %arg
+  br i1 %tmp35, label %bb27, label %bb36
+
+bb36:                                             ; preds = %bb33
+  br label %bb37
+
+bb37:                                             ; preds = %bb36, %bb23
+  %tmp38 = phi i64 [ %tmp24, %bb36 ], [ %tmp24, %bb23 ]
+  br label %bb39
+
+bb39:                                             ; preds = %bb37
+  %tmp40 = add nuw nsw i64 %tmp38, 1
+  %tmp41 = icmp slt i64 %tmp40, %tmp10
+  br i1 %tmp41, label %bb13, label %bb42
+
+bb42:                                             ; preds = %bb39
+  br label %bb43
+
+bb43:                                             ; preds = %bb42, %bb8
+  br label %bb44
+
+bb44:                                             ; preds = %bb43
+  %tmp45 = add nuw nsw i64 %tmp9, 1
+  %tmp46 = icmp slt i64 %tmp45, %arg
+  br i1 %tmp46, label %bb8, label %bb47
+
+bb47:                                             ; preds = %bb44
+  br label %bb48
+
+bb48:                                             ; preds = %bb47, %bb
+  ret void
+}
+
+; CHECK-LABEL @test3
+
+; CHECK:      %tmp14 = phi i64 [ %tmp40, %bb39 ], [ 1, %bb8 ]
+; CHECK-NEXT: -->  {1,+,1}<%bb13> U: [1,9223372036854775807) S: [1,9223372036854775807)
+; CHECK-SAME:      Exits: (-2 + %arg)       LoopDispositions: { %bb13: Computable, %bb8: Variant, %bb17_a: Invariant, %bb27: Invariant }
+; CHECK:      %tmp18 = phi i64 [ %tmp20, %bb17 ], [ 0, %bb13 ]
+; CHECK-NEXT: -->  {0,+,1}<nuw><nsw><%bb17_a> U: [0,9223372036854775807) S: [0,9223372036854775807)
+; CHECK-SAME:      Exits: (-1 + %arg)       LoopDispositions: { %bb17_a: Computable, %bb13: Variant, %bb8: Variant }
+
+; CHECK:      %tmp24 = phi i64 [ %tmp14, %bb13 ], [ %tmp14, %bb17 ]
+; CHECK-NEXT: -->  {1,+,1}<%bb13> U: [1,9223372036854775807) S: [1,9223372036854775807)
+; CHECK-SAME:      Exits: (-2 + %arg)       LoopDispositions: { %bb13: Computable, %bb8: Variant, %bb17_a: Invariant, %bb27: Invariant }
+; CHECK:       %tmp28 = phi i64 [ %tmp34, %bb27 ], [ 0, %bb23 ]
+; CHECK-NEXT:  -->  {0,+,1}<nuw><nsw><%bb27> U: [0,9223372036854775807) S: [0,9223372036854775807)
+; CHECK-SAME:       Exits: (-1 + %arg)      LoopDispositions: { %bb27: Computable, %bb13: Variant, %bb8: Variant }
+
+; CHECK:      %tmp38 = phi i64 [ %tmp24, %bb23 ], [ %tmp24, %bb27 ]
+; CHECK-NEXT: -->  {1,+,1}<%bb13> U: [1,9223372036854775807) S: [1,9223372036854775807)
+; CHECK-SAME:      Exits: (-2 + %arg)       LoopDispositions: { %bb13: Computable, %bb8: Variant, %bb17_a: Invariant, %bb27: Invariant }
+
+define void @test3(i64 %arg, i32* %arg1) {
+bb:
+  %tmp = icmp slt i64 0, %arg
+  br i1 %tmp, label %bb8, label %bb48
+
+bb8:                                              ; preds = %bb, %bb44
+  %tmp9 = phi i64 [ %tmp45, %bb44 ], [ 0, %bb ]
+  %tmp10 = add nsw i64 %arg, -1
+  %tmp11 = icmp slt i64 1, %tmp10
+  br i1 %tmp11, label %bb13, label %bb44
+
+bb13:                                             ; preds = %bb8, %bb39
+  %tmp14 = phi i64 [ %tmp40, %bb39 ], [ 1, %bb8 ]
+  %tmp15 = icmp slt i64 0, %arg
+  br i1 %tmp15, label %bb17_a, label %bb23
+
+bb17_a:
+  %tmp18 = phi i64 [ %tmp20, %bb17 ], [ 0, %bb13 ]
+  %tmp20 = add nuw nsw i64 %tmp18, 1
+
+  br label %bb17
+
+bb17:                                             ; preds = %bb13, %bb17
+  %tmp21 = icmp slt i64 %tmp20, %arg
+  br i1 %tmp21, label %bb17_a, label %bb23
+
+bb23:                                             ; preds = %bb17, %bb13
+  %tmp24 = phi i64 [ %tmp14, %bb13 ], [ %tmp14, %bb17 ]
+  %tmp25 = icmp slt i64 0, %arg
+  br i1 %tmp25, label %bb27, label %bb39
+
+bb27:                                             ; preds = %bb23, %bb27
+  %tmp28 = phi i64 [ %tmp34, %bb27 ], [ 0, %bb23 ]
+  %tmp29 = mul nsw i64 %tmp9, %arg
+  %tmp30 = getelementptr inbounds i32, i32* %arg1, i64 %tmp24
+  %tmp31 = getelementptr inbounds i32, i32* %tmp30, i64 %tmp29
+  %tmp32 = load i32, i32* %tmp31, align 4
+  %tmp34 = add nuw nsw i64 %tmp28, 1
+  %tmp35 = icmp slt i64 %tmp34, %arg
+  br i1 %tmp35, label %bb27, label %bb39
+
+bb39:                                             ; preds = %bb23, %bb27
+  %tmp38 = phi i64 [ %tmp24, %bb23 ], [ %tmp24, %bb27 ]
+  %tmp40 = add nuw nsw i64 %tmp38, 1
+  %tmp41 = icmp slt i64 %tmp40, %tmp10
+  br i1 %tmp41, label %bb13, label %bb44
+
+bb44:                                             ; preds = %bb8, %bb39
+  %tmp45 = add nuw nsw i64 %tmp9, 1
+  %tmp46 = icmp slt i64 %tmp45, %arg
+  br i1 %tmp46, label %bb8, label %bb48
+
+bb48:                                             ; preds = %bb44, %bb
+  ret void
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/scev-expander-preserve-lcssa.ll b/llvm/test/Transforms/IndVarSimplify/scev-expander-preserve-lcssa.ll
new file mode 100644
index 0000000000000..d26d3b23b7724
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/scev-expander-preserve-lcssa.ll
@@ -0,0 +1,587 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -indvars -S -indvars -verify-loop-lcssa %s | FileCheck %s
+
+; Make sure SCEVExpander does not crash and introduce unnecessary LCSSA PHI nodes.
+; The tests are a collection of cases with crashes when preserving LCSSA PHI
+; nodes directly in SCEVExpander.
+
+declare i1 @cond() readnone
+
+define void @test1(i8 %x, [512 x i8]* %ptr) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LAND_LHS_TRUE:%.*]]
+; CHECK:       land.lhs.true:
+; CHECK-NEXT:    br label [[WHILE_COND22:%.*]]
+; CHECK:       while.cond22:
+; CHECK-NEXT:    [[C_1:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[C_1]], label [[WHILE_COND22]], label [[WHILE_COND29_PREHEADER:%.*]]
+; CHECK:       while.cond29.preheader:
+; CHECK-NEXT:    br label [[WHILE_BODY35:%.*]]
+; CHECK:       while.body35:
+; CHECK-NEXT:    [[I_1107:%.*]] = phi i32 [ [[I_9:%.*]], [[IF_END224:%.*]] ], [ 0, [[WHILE_COND29_PREHEADER]] ]
+; CHECK-NEXT:    br label [[WHILE_COND192:%.*]]
+; CHECK:       while.cond192:
+; CHECK-NEXT:    switch i8 [[X:%.*]], label [[WHILE_BODY205:%.*]] [
+; CHECK-NEXT:    i8 59, label [[WHILE_COND215_PREHEADER:%.*]]
+; CHECK-NEXT:    i8 10, label [[IF_END224_LOOPEXIT1:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       while.cond215.preheader:
+; CHECK-NEXT:    br label [[WHILE_COND215:%.*]]
+; CHECK:       while.body205:
+; CHECK-NEXT:    br label [[WHILE_COND192]]
+; CHECK:       while.cond215:
+; CHECK-NEXT:    [[I_8_IN:%.*]] = phi i32 [ [[I_8:%.*]], [[WHILE_COND215]] ], [ [[I_1107]], [[WHILE_COND215_PREHEADER]] ]
+; CHECK-NEXT:    [[I_8]] = add nsw i32 [[I_8_IN]], 1
+; CHECK-NEXT:    [[IDXPROM216:%.*]] = sext i32 [[I_8]] to i64
+; CHECK-NEXT:    [[ARRAYIDX217:%.*]] = getelementptr inbounds [512 x i8], [512 x i8]* [[PTR:%.*]], i64 0, i64 [[IDXPROM216]]
+; CHECK-NEXT:    [[C_2:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[C_2]], label [[WHILE_COND215]], label [[IF_END224_LOOPEXIT:%.*]]
+; CHECK:       if.end224.loopexit:
+; CHECK-NEXT:    [[I_8_LCSSA:%.*]] = phi i32 [ [[I_8]], [[WHILE_COND215]] ]
+; CHECK-NEXT:    br label [[IF_END224]]
+; CHECK:       if.end224.loopexit1:
+; CHECK-NEXT:    br label [[IF_END224]]
+; CHECK:       if.end224:
+; CHECK-NEXT:    [[I_9]] = phi i32 [ [[I_8_LCSSA]], [[IF_END224_LOOPEXIT]] ], [ [[I_1107]], [[IF_END224_LOOPEXIT1]] ]
+; CHECK-NEXT:    [[C_3:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[C_3]], label [[WHILE_END225:%.*]], label [[WHILE_BODY35]]
+; CHECK:       while.end225:
+; CHECK-NEXT:    br label [[LAND_LHS_TRUE]]
+;
+entry:
+  br label %land.lhs.true
+
+land.lhs.true:                                    ; preds = %while.end225, %entry
+  br label %while.cond22
+
+while.cond22:                                     ; preds = %while.cond22, %land.lhs.true
+  %c.1 = call i1 @cond()
+  br i1 %c.1, label %while.cond22, label %while.cond29.preheader
+
+while.cond29.preheader:                           ; preds = %while.cond22
+  br label %while.body35
+
+while.body35:                                     ; preds = %if.end224, %while.cond29.preheader
+  %i.1107 = phi i32 [ %i.9, %if.end224 ], [ 0, %while.cond29.preheader ]
+  br label %while.cond192
+
+while.cond192:                                    ; preds = %while.body205, %while.body35
+  %i.7 = phi i32 [ %i.1107, %while.body35 ], [ %inc206, %while.body205 ]
+  switch i8 %x, label %while.body205 [
+  i8 59, label %while.cond215
+  i8 10, label %if.end224
+  ]
+
+while.body205:                                    ; preds = %while.cond192
+  %inc206 = add nsw i32 %i.7, 1
+  br label %while.cond192
+
+while.cond215:                                    ; preds = %while.cond215, %while.cond192
+  %i.8.in = phi i32 [ %i.8, %while.cond215 ], [ %i.7, %while.cond192 ]
+  %i.8 = add nsw i32 %i.8.in, 1
+  %idxprom216 = sext i32 %i.8 to i64
+  %arrayidx217 = getelementptr inbounds [512 x i8], [512 x i8]* %ptr, i64 0, i64 %idxprom216
+  %c.2 = call i1 @cond()
+  br i1 %c.2, label %while.cond215, label %if.end224
+
+if.end224:                                        ; preds = %while.cond215, %while.cond192
+  %i.9 = phi i32 [ %i.8, %while.cond215 ], [ %i.7, %while.cond192 ]
+  %c.3 = call i1 @cond()
+  br i1 %c.3, label %while.end225, label %while.body35
+
+while.end225:                                     ; preds = %if.end224
+  br label %land.lhs.true
+}
+
+define void @test2(i16 %x)  {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[C_1:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[C_1]], label [[FOR_COND_PREHEADER:%.*]], label [[RETURN:%.*]]
+; CHECK:       for.cond.preheader:
+; CHECK-NEXT:    br label [[FOR_COND:%.*]]
+; CHECK:       for.cond:
+; CHECK-NEXT:    switch i16 [[X:%.*]], label [[RETURN_LOOPEXIT1:%.*]] [
+; CHECK-NEXT:    i16 41, label [[FOR_END:%.*]]
+; CHECK-NEXT:    i16 43, label [[FOR_COND]]
+; CHECK-NEXT:    ]
+; CHECK:       for.end:
+; CHECK-NEXT:    [[I_0_LCSSA2:%.*]] = phi i32 [ 0, [[FOR_COND]] ]
+; CHECK-NEXT:    [[CMP8243:%.*]] = icmp sgt i32 [[I_0_LCSSA2]], 0
+; CHECK-NEXT:    br i1 [[CMP8243]], label [[FOR_BODY84_PREHEADER:%.*]], label [[RETURN]]
+; CHECK:       for.body84.preheader:
+; CHECK-NEXT:    br label [[FOR_BODY84:%.*]]
+; CHECK:       for.body84:
+; CHECK-NEXT:    [[I_144:%.*]] = phi i32 [ [[INC:%.*]], [[IF_END106:%.*]] ], [ 0, [[FOR_BODY84_PREHEADER]] ]
+; CHECK-NEXT:    [[C_2:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[C_2]], label [[IF_END106]], label [[RETURN_LOOPEXIT:%.*]]
+; CHECK:       if.end106:
+; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_144]], 1
+; CHECK-NEXT:    [[CMP82:%.*]] = icmp slt i32 [[INC]], [[I_0_LCSSA2]]
+; CHECK-NEXT:    br i1 [[CMP82]], label [[FOR_BODY84]], label [[RETURN_LOOPEXIT]]
+; CHECK:       return.loopexit:
+; CHECK-NEXT:    br label [[RETURN]]
+; CHECK:       return.loopexit1:
+; CHECK-NEXT:    br label [[RETURN]]
+; CHECK:       return:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %c.1 = call i1 @cond()
+  br i1 %c.1, label %for.cond, label %return
+
+for.cond:                                         ; preds = %for.cond, %entry
+  %i.0 = phi i32 [ %sub, %for.cond ], [ 0, %entry ]
+  %sub = add nsw i32 %i.0, -1
+  switch i16 %x, label %return [
+  i16 41, label %for.end
+  i16 43, label %for.cond
+  ]
+
+for.end:                                          ; preds = %for.cond
+  %cmp8243 = icmp sgt i32 %i.0, 0
+  br i1 %cmp8243, label %for.body84, label %return
+
+for.body84:                                       ; preds = %if.end106, %for.end
+  %i.144 = phi i32 [ %inc, %if.end106 ], [ 0, %for.end ]
+  %c.2 = call i1 @cond()
+  br i1 %c.2, label %if.end106, label %return
+
+if.end106:                                        ; preds = %for.body84
+  %inc = add nuw nsw i32 %i.144, 1
+  %cmp82 = icmp slt i32 %inc, %i.0
+  br i1 %cmp82, label %for.body84, label %return
+
+return:                                           ; preds = %if.end106, %for.body84, %for.end, %for.cond, %entry
+  ret void
+}
+
+declare i32 @get.i32() readnone
+
+define void @test3(i32* %ptr) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[WHILE_BODY:%.*]]
+; CHECK:       while.body:
+; CHECK-NEXT:    br label [[FOR_BODY1208:%.*]]
+; CHECK:       for.body1208:
+; CHECK-NEXT:    [[M_0804:%.*]] = phi i32 [ 1, [[WHILE_BODY]] ], [ [[INC1499:%.*]], [[FOR_INC1498:%.*]] ]
+; CHECK-NEXT:    [[V:%.*]] = call i32 @get.i32()
+; CHECK-NEXT:    [[CMP1358:%.*]] = icmp eq i32 [[V]], 0
+; CHECK-NEXT:    br i1 [[CMP1358]], label [[IF_THEN1360:%.*]], label [[FOR_INC1498]]
+; CHECK:       if.then1360:
+; CHECK-NEXT:    [[M_0804_LCSSA:%.*]] = phi i32 [ [[M_0804]], [[FOR_BODY1208]] ]
+; CHECK-NEXT:    br label [[FOR_COND1390:%.*]]
+; CHECK:       for.cond1390:
+; CHECK-NEXT:    [[M_2_IN:%.*]] = phi i32 [ [[M_0804_LCSSA]], [[IF_THEN1360]] ], [ 0, [[FOR_BODY1394:%.*]] ]
+; CHECK-NEXT:    [[C_2:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[C_2]], label [[FOR_BODY1394]], label [[FOR_END1469:%.*]]
+; CHECK:       for.body1394:
+; CHECK-NEXT:    br label [[FOR_COND1390]]
+; CHECK:       for.end1469:
+; CHECK-NEXT:    [[M_2_IN_LCSSA:%.*]] = phi i32 [ [[M_2_IN]], [[FOR_COND1390]] ]
+; CHECK-NEXT:    store i32 [[M_2_IN_LCSSA]], i32* [[PTR:%.*]], align 4
+; CHECK-NEXT:    br label [[WHILE_BODY]]
+; CHECK:       for.inc1498:
+; CHECK-NEXT:    [[INC1499]] = add nuw nsw i32 [[M_0804]], 1
+; CHECK-NEXT:    br label [[FOR_BODY1208]]
+;
+entry:
+  br label %while.body
+
+while.body:                                       ; preds = %for.end1469, %entry
+  br label %for.body1208
+
+for.body1208:                                     ; preds = %for.inc1498, %while.body
+  %m.0804 = phi i32 [ 1, %while.body ], [ %inc1499, %for.inc1498 ]
+  %v = call i32 @get.i32()
+  %cmp1358 = icmp eq i32 %v, 0
+  br i1 %cmp1358, label %if.then1360, label %for.inc1498
+
+if.then1360:                                      ; preds = %for.body1208
+  br label %for.cond1390
+
+for.cond1390:                                     ; preds = %for.body1394, %if.then1360
+  %m.2.in = phi i32 [ %m.0804, %if.then1360 ], [ 0, %for.body1394 ]
+  %c.2 = call i1 @cond()
+  br i1 %c.2, label %for.body1394, label %for.end1469
+
+for.body1394:                                     ; preds = %for.cond1390
+  br label %for.cond1390
+
+for.end1469:                                      ; preds = %for.cond1390
+  store i32 %m.2.in, i32* %ptr, align 4
+  br label %while.body
+
+for.inc1498:                                      ; preds = %for.body1208
+  %inc1499 = add nuw nsw i32 %m.0804, 1
+  br label %for.body1208
+}
+
+define void @test4(i32* %ptr) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[WHILE_BODY:%.*]]
+; CHECK:       while.body:
+; CHECK-NEXT:    br label [[FOR_COND1204_PREHEADER:%.*]]
+; CHECK:       for.cond1204.preheader:
+; CHECK-NEXT:    [[C_1:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[C_1]], label [[IF_THEN1504:%.*]], label [[FOR_BODY1208_LR_PH:%.*]]
+; CHECK:       for.body1208.lr.ph:
+; CHECK-NEXT:    br label [[FOR_BODY1208:%.*]]
+; CHECK:       for.body1208:
+; CHECK-NEXT:    [[TMP0:%.*]] = phi i32 [ 0, [[FOR_BODY1208_LR_PH]] ], [ [[TMP1:%.*]], [[FOR_INC1498:%.*]] ]
+; CHECK-NEXT:    [[M_0804:%.*]] = phi i32 [ 1, [[FOR_BODY1208_LR_PH]] ], [ [[INC1499:%.*]], [[FOR_INC1498]] ]
+; CHECK-NEXT:    [[IDXPROM1212:%.*]] = zext i32 [[M_0804]] to i64
+; CHECK-NEXT:    [[V:%.*]] = call i32 @get.i32()
+; CHECK-NEXT:    [[CMP1215:%.*]] = icmp eq i32 0, [[V]]
+; CHECK-NEXT:    [[YPOS1223:%.*]] = getelementptr inbounds i32, i32* [[PTR:%.*]], i64 [[IDXPROM1212]]
+; CHECK-NEXT:    br i1 [[CMP1215]], label [[IF_THEN1217:%.*]], label [[IF_ELSE1351:%.*]]
+; CHECK:       if.then1217:
+; CHECK-NEXT:    [[M_0804_LCSSA:%.*]] = phi i32 [ [[M_0804]], [[FOR_BODY1208]] ]
+; CHECK-NEXT:    br label [[FOR_COND1247:%.*]]
+; CHECK:       for.cond1247:
+; CHECK-NEXT:    [[M_1_IN:%.*]] = phi i32 [ [[M_0804_LCSSA]], [[IF_THEN1217]] ], [ [[M_1:%.*]], [[IF_THEN1260:%.*]] ]
+; CHECK-NEXT:    [[M_1]] = add nuw nsw i32 [[M_1_IN]], 1
+; CHECK-NEXT:    br label [[FOR_BODY1251:%.*]]
+; CHECK:       for.body1251:
+; CHECK-NEXT:    [[IDXPROM1255:%.*]] = zext i32 [[M_1]] to i64
+; CHECK-NEXT:    [[XPOS1257:%.*]] = getelementptr inbounds i32, i32* [[PTR]], i64 [[IDXPROM1255]]
+; CHECK-NEXT:    [[C_2:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[C_2]], label [[IF_THEN1260]], label [[FOR_END1326:%.*]]
+; CHECK:       if.then1260:
+; CHECK-NEXT:    br label [[FOR_COND1247]]
+; CHECK:       for.end1326:
+; CHECK-NEXT:    br label [[IF_END1824:%.*]]
+; CHECK:       if.else1351:
+; CHECK-NEXT:    [[V_2:%.*]] = call i32 @get.i32()
+; CHECK-NEXT:    [[CMP1358:%.*]] = icmp eq i32 [[V_2]], 0
+; CHECK-NEXT:    br i1 [[CMP1358]], label [[IF_THEN1360:%.*]], label [[FOR_INC1498]]
+; CHECK:       if.then1360:
+; CHECK-NEXT:    [[DOTLCSSA2:%.*]] = phi i32 [ [[TMP0]], [[IF_ELSE1351]] ]
+; CHECK-NEXT:    [[M_0804_LCSSA1:%.*]] = phi i32 [ [[M_0804]], [[IF_ELSE1351]] ]
+; CHECK-NEXT:    [[CMP1392:%.*]] = icmp slt i32 [[M_0804_LCSSA1]], [[DOTLCSSA2]]
+; CHECK-NEXT:    unreachable
+; CHECK:       for.inc1498:
+; CHECK-NEXT:    [[INC1499]] = add nuw nsw i32 [[M_0804]], 1
+; CHECK-NEXT:    [[TMP1]] = load i32, i32* [[PTR]], align 8
+; CHECK-NEXT:    br label [[FOR_BODY1208]]
+; CHECK:       if.then1504:
+; CHECK-NEXT:    unreachable
+; CHECK:       if.end1824:
+; CHECK-NEXT:    br label [[WHILE_BODY]]
+;
+entry:
+  br label %while.body
+
+while.body:                                       ; preds = %if.end1824, %entry
+  br label %for.cond1204.preheader
+
+for.cond1204.preheader:                           ; preds = %while.body
+  %c.1 = call i1 @cond()
+  br i1 %c.1, label %if.then1504, label %for.body1208.lr.ph
+
+for.body1208.lr.ph:                               ; preds = %for.cond1204.preheader
+  br label %for.body1208
+
+for.body1208:                                     ; preds = %for.inc1498, %for.body1208.lr.ph
+  %0 = phi i32 [ 0, %for.body1208.lr.ph ], [ %1, %for.inc1498 ]
+  %m.0804 = phi i32 [ 1, %for.body1208.lr.ph ], [ %inc1499, %for.inc1498 ]
+  %idxprom1212 = zext i32 %m.0804 to i64
+  %v = call i32 @get.i32()
+  %cmp1215 = icmp eq i32 0, %v
+  %ypos1223 = getelementptr inbounds i32, i32* %ptr , i64 %idxprom1212
+  br i1 %cmp1215, label %if.then1217, label %if.else1351
+
+if.then1217:                                      ; preds = %for.body1208
+  br label %for.cond1247
+
+for.cond1247:                                     ; preds = %if.then1260, %if.then1217
+  %m.1.in = phi i32 [ %m.0804, %if.then1217 ], [ %m.1, %if.then1260 ]
+  %m.1 = add nuw nsw i32 %m.1.in, 1
+  %cmp1249 = icmp slt i32 %m.1.in, %0
+  br label %for.body1251
+
+for.body1251:                                     ; preds = %for.cond1247
+  %idxprom1255 = zext i32 %m.1 to i64
+  %xpos1257 = getelementptr inbounds i32, i32* %ptr, i64 %idxprom1255
+  %c.2 = call i1 @cond()
+  br i1 %c.2, label %if.then1260, label %for.end1326
+
+if.then1260:                                      ; preds = %for.body1251
+  br label %for.cond1247
+
+for.end1326:                                      ; preds = %for.body1251
+  br label %if.end1824
+
+if.else1351:                                      ; preds = %for.body1208
+  %v.2 = call i32 @get.i32()
+  %cmp1358 = icmp eq i32 %v.2, 0
+  br i1 %cmp1358, label %if.then1360, label %for.inc1498
+
+if.then1360:                                      ; preds = %if.else1351
+  %cmp1392 = icmp slt i32 %m.0804, %0
+  unreachable
+
+for.inc1498:                                      ; preds = %if.else1351
+  %inc1499 = add nuw nsw i32 %m.0804, 1
+  %1 = load i32, i32* %ptr, align 8
+  br label %for.body1208
+
+if.then1504:                                      ; preds = %for.cond1204.preheader
+  unreachable
+
+if.end1824:                                       ; preds = %for.end1326
+  br label %while.body
+}
+
+define void @test5(i8* %header, i32 %conv, i8 %n) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[POS_42:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[ADD85:%.*]], [[FOR_INC:%.*]] ]
+; CHECK-NEXT:    br label [[FOR_INNER:%.*]]
+; CHECK:       for.inner:
+; CHECK-NEXT:    [[I_0_I:%.*]] = phi i32 [ 0, [[FOR_BODY]] ], [ [[INC_I:%.*]], [[FOR_INNER]] ]
+; CHECK-NEXT:    [[INC_I]] = add nuw nsw i32 [[I_0_I]], 1
+; CHECK-NEXT:    [[CMP7_I:%.*]] = icmp slt i8 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP7_I]], label [[FOR_INNER]], label [[FOR_INNER_EXIT:%.*]]
+; CHECK:       for.inner.exit:
+; CHECK-NEXT:    [[INC_I_LCSSA:%.*]] = phi i32 [ [[INC_I]], [[FOR_INNER]] ]
+; CHECK-NEXT:    br label [[FOR_INNER_2:%.*]]
+; CHECK:       for.inner.2:
+; CHECK-NEXT:    [[I_0_I1:%.*]] = phi i32 [ 0, [[FOR_INNER_EXIT]] ], [ [[INC_I3:%.*]], [[FOR_INNER_2]] ]
+; CHECK-NEXT:    [[INC_I3]] = add nuw nsw i32 [[I_0_I1]], 1
+; CHECK-NEXT:    [[CMP7_I4:%.*]] = icmp slt i8 [[N]], 0
+; CHECK-NEXT:    br i1 [[CMP7_I4]], label [[FOR_INNER_2]], label [[FOR_INC]]
+; CHECK:       for.inc:
+; CHECK-NEXT:    [[INC_I3_LCSSA:%.*]] = phi i32 [ [[INC_I3]], [[FOR_INNER_2]] ]
+; CHECK-NEXT:    [[ADD71:%.*]] = add i32 [[POS_42]], [[INC_I_LCSSA]]
+; CHECK-NEXT:    [[ADD85]] = add i32 [[ADD71]], [[INC_I3_LCSSA]]
+; CHECK-NEXT:    br i1 false, label [[FOR_BODY]], label [[WHILE_COND_PREHEADER:%.*]]
+; CHECK:       while.cond.preheader:
+; CHECK-NEXT:    [[ADD85_LCSSA:%.*]] = phi i32 [ [[ADD85]], [[FOR_INC]] ]
+; CHECK-NEXT:    [[SHL:%.*]] = shl nuw nsw i32 [[CONV:%.*]], 2
+; CHECK-NEXT:    br label [[WHILE_COND:%.*]]
+; CHECK:       while.cond:
+; CHECK-NEXT:    [[POS_8:%.*]] = phi i32 [ [[INC114:%.*]], [[WHILE_BODY:%.*]] ], [ [[ADD85_LCSSA]], [[WHILE_COND_PREHEADER]] ]
+; CHECK-NEXT:    [[CMP112:%.*]] = icmp ult i32 [[POS_8]], [[SHL]]
+; CHECK-NEXT:    br i1 [[CMP112]], label [[WHILE_BODY]], label [[CLEANUP122:%.*]]
+; CHECK:       while.body:
+; CHECK-NEXT:    [[INC114]] = add nuw i32 [[POS_8]], 1
+; CHECK-NEXT:    [[C_1:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[C_1]], label [[WHILE_COND]], label [[CLEANUP122]]
+; CHECK:       cleanup122:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %shl = shl nuw nsw i32 %conv, 2
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.inc
+  %pos.42 = phi i32 [ 0, %entry ], [ %add85, %for.inc ]
+  br label %for.inner
+
+for.inner:                                       ; preds = %for.body.i, %for.body
+  %i.0.i = phi i32 [ 0, %for.body ], [ %inc.i, %for.inner ]
+  %inc.i = add nuw nsw i32 %i.0.i, 1
+  %cmp7.i = icmp slt i8 %n, 0
+  br i1 %cmp7.i, label %for.inner, label %for.inner.exit
+
+for.inner.exit:                                   ; preds = %for.body.i
+  %add71 = add i32 %pos.42, %inc.i
+  br label %for.inner.2
+
+for.inner.2:                                      ; preds = %for.body.i6, %cleanup.cont74
+  %i.0.i1 = phi i32 [ 0, %for.inner.exit ], [ %inc.i3, %for.inner.2]
+  %inc.i3 = add nuw nsw i32 %i.0.i1, 1
+  %cmp7.i4 = icmp slt i8 %n, 0
+  br i1 %cmp7.i4, label %for.inner.2, label %for.inc
+
+for.inc:                                          ; preds = %for.body.i6
+  %add85 = add i32 %add71, %inc.i3
+  br i1 false, label %for.body, label %while.cond.preheader
+
+while.cond.preheader:                             ; preds = %for.inc
+  br label %while.cond
+
+while.cond:                                       ; preds = %while.cond.preheader, %while.body
+  %pos.8 = phi i32 [ %inc114, %while.body ], [ %add85, %while.cond.preheader ]
+  %cmp112 = icmp ult i32 %pos.8, %shl
+  br i1 %cmp112, label %while.body, label %cleanup122
+
+while.body:                                       ; preds = %while.cond
+  %inc114 = add nuw i32 %pos.8, 1
+  %c.1 = call i1 @cond()
+  br i1 %c.1, label %while.cond, label %cleanup122
+
+cleanup122:                                       ; preds = %while.body, %while.cond
+  ret void
+}
+
+define void @test6(i8 %x) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[C_1:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[C_1]], label [[LAND_RHS:%.*]], label [[WHILE_END316:%.*]]
+; CHECK:       land.rhs:
+; CHECK-NEXT:    br label [[WHILE_BODY35:%.*]]
+; CHECK:       while.body35:
+; CHECK-NEXT:    br label [[WHILE_COND192:%.*]]
+; CHECK:       while.cond192:
+; CHECK-NEXT:    switch i8 [[X:%.*]], label [[WHILE_BODY205:%.*]] [
+; CHECK-NEXT:    i8 59, label [[WHILE_COND215_PREHEADER:%.*]]
+; CHECK-NEXT:    i8 10, label [[IF_END224:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       while.cond215.preheader:
+; CHECK-NEXT:    [[I_7_LCSSA:%.*]] = phi i32 [ 0, [[WHILE_COND192]] ]
+; CHECK-NEXT:    br label [[WHILE_COND215:%.*]]
+; CHECK:       while.body205:
+; CHECK-NEXT:    br label [[WHILE_COND192]]
+; CHECK:       while.cond215:
+; CHECK-NEXT:    [[I_8_IN:%.*]] = phi i32 [ [[I_8:%.*]], [[WHILE_COND215]] ], [ [[I_7_LCSSA]], [[WHILE_COND215_PREHEADER]] ]
+; CHECK-NEXT:    [[I_8]] = add nuw nsw i32 [[I_8_IN]], 1
+; CHECK-NEXT:    [[IDXPROM216:%.*]] = sext i32 [[I_8]] to i64
+; CHECK-NEXT:    [[ARRAYIDX217:%.*]] = getelementptr inbounds [512 x i8], [512 x i8]* null, i64 0, i64 [[IDXPROM216]]
+; CHECK-NEXT:    br label [[WHILE_COND215]]
+; CHECK:       if.end224:
+; CHECK-NEXT:    [[C_2:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[C_2]], label [[WHILE_END225:%.*]], label [[WHILE_BODY35]]
+; CHECK:       while.end225:
+; CHECK-NEXT:    unreachable
+; CHECK:       while.end316:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %c.1 = call i1 @cond()
+  br i1 %c.1, label %land.rhs, label %while.end316
+
+land.rhs:                                         ; preds = %entry
+  br label %while.body35
+
+while.body35:                                     ; preds = %if.end224, %land.rhs
+  br label %while.cond192
+
+while.cond192:                                    ; preds = %while.body205, %while.body35
+  %i.7 = phi i32 [ 0, %while.body35 ], [ %inc206, %while.body205 ]
+  switch i8 %x, label %while.body205 [
+  i8 59, label %while.cond215
+  i8 10, label %if.end224
+  ]
+
+while.body205:                                    ; preds = %while.cond192
+  %inc206 = add nsw i32 %i.7, 1
+  br label %while.cond192
+
+while.cond215:                                    ; preds = %while.cond215, %while.cond192
+  %i.8.in = phi i32 [ %i.8, %while.cond215 ], [ %i.7, %while.cond192 ]
+  %i.8 = add nsw i32 %i.8.in, 1
+  %idxprom216 = sext i32 %i.8 to i64
+  %arrayidx217 = getelementptr inbounds [512 x i8], [512 x i8]* null, i64 0, i64 %idxprom216
+  br label %while.cond215
+
+if.end224:                                        ; preds = %while.cond192
+  %c.2 = call i1 @cond()
+  br i1 %c.2, label %while.end225, label %while.body35
+
+while.end225:                                     ; preds = %if.end224
+  unreachable
+
+while.end316:                                     ; preds = %entry
+  ret void
+}
+
+define void @test7(i32* %ptr) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[WHILE_BODY:%.*]]
+; CHECK:       while.body:
+; CHECK-NEXT:    br label [[FOR_BODY1208:%.*]]
+; CHECK:       for.body1208:
+; CHECK-NEXT:    [[TMP0:%.*]] = phi i32 [ undef, [[WHILE_BODY]] ], [ [[TMP1:%.*]], [[FOR_INC1498:%.*]] ]
+; CHECK-NEXT:    [[M_048:%.*]] = phi i32 [ 1, [[WHILE_BODY]] ], [ [[INC1499:%.*]], [[FOR_INC1498]] ]
+; CHECK-NEXT:    [[IDXPROM1212:%.*]] = zext i32 [[M_048]] to i64
+; CHECK-NEXT:    [[XPOS1214:%.*]] = getelementptr inbounds i32, i32* [[PTR:%.*]], i64 [[IDXPROM1212]]
+; CHECK-NEXT:    [[V_1:%.*]] = call i32 @get.i32()
+; CHECK-NEXT:    [[CMP1215:%.*]] = icmp eq i32 0, [[V_1]]
+; CHECK-NEXT:    br i1 [[CMP1215]], label [[IF_THEN1217:%.*]], label [[IF_ELSE1351:%.*]]
+; CHECK:       if.then1217:
+; CHECK-NEXT:    [[DOTLCSSA:%.*]] = phi i32 [ [[TMP0]], [[FOR_BODY1208]] ]
+; CHECK-NEXT:    [[M_048_LCSSA:%.*]] = phi i32 [ [[M_048]], [[FOR_BODY1208]] ]
+; CHECK-NEXT:    [[CMP1249_NOT_NOT:%.*]] = icmp slt i32 [[M_048_LCSSA]], [[DOTLCSSA]]
+; CHECK-NEXT:    unreachable
+; CHECK:       if.else1351:
+; CHECK-NEXT:    [[CMP1358:%.*]] = icmp eq i32 0, undef
+; CHECK-NEXT:    br i1 [[CMP1358]], label [[IF_THEN1360:%.*]], label [[FOR_INC1498]]
+; CHECK:       if.then1360:
+; CHECK-NEXT:    [[M_048_LCSSA1:%.*]] = phi i32 [ [[M_048]], [[IF_ELSE1351]] ]
+; CHECK-NEXT:    br label [[FOR_COND1390:%.*]]
+; CHECK:       for.cond1390:
+; CHECK-NEXT:    [[M_2_IN:%.*]] = phi i32 [ [[M_048_LCSSA1]], [[IF_THEN1360]] ], [ [[M_2:%.*]], [[IF_THEN1403:%.*]] ]
+; CHECK-NEXT:    [[M_2]] = add nuw nsw i32 [[M_2_IN]], 1
+; CHECK-NEXT:    [[IDXPROM1398:%.*]] = zext i32 [[M_2]] to i64
+; CHECK-NEXT:    br label [[IF_THEN1403]]
+; CHECK:       if.then1403:
+; CHECK-NEXT:    [[XPOS1409:%.*]] = getelementptr inbounds i32, i32* [[PTR]], i64 [[IDXPROM1398]]
+; CHECK-NEXT:    [[C_1:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[C_1]], label [[FOR_COND1390]], label [[FOR_END1469:%.*]]
+; CHECK:       for.end1469:
+; CHECK-NEXT:    br label [[IF_END1824:%.*]]
+; CHECK:       for.inc1498:
+; CHECK-NEXT:    [[INC1499]] = add nuw nsw i32 [[M_048]], 1
+; CHECK-NEXT:    [[TMP1]] = load i32, i32* undef, align 8
+; CHECK-NEXT:    br label [[FOR_BODY1208]]
+; CHECK:       if.end1824:
+; CHECK-NEXT:    br label [[WHILE_BODY]]
+;
+entry:
+  br label %while.body
+
+while.body:                                       ; preds = %if.end1824, %entry
+  br label %for.body1208
+
+for.body1208:                                     ; preds = %for.inc1498, %while.body
+  %0 = phi i32 [ undef, %while.body ], [ %1, %for.inc1498 ]
+  %m.048 = phi i32 [ 1, %while.body ], [ %inc1499, %for.inc1498 ]
+  %idxprom1212 = zext i32 %m.048 to i64
+  %xpos1214 = getelementptr inbounds i32, i32* %ptr, i64 %idxprom1212
+  %v.1 = call i32 @get.i32()
+  %cmp1215 = icmp eq i32 0, %v.1
+  br i1 %cmp1215, label %if.then1217, label %if.else1351
+
+if.then1217:                                      ; preds = %for.body1208
+  %cmp1249.not.not = icmp slt i32 %m.048, %0
+  unreachable
+
+if.else1351:                                      ; preds = %for.body1208
+  %cmp1358 = icmp eq i32 0, undef
+  br i1 %cmp1358, label %if.then1360, label %for.inc1498
+
+if.then1360:                                      ; preds = %if.else1351
+  br label %for.cond1390
+
+for.cond1390:                                     ; preds = %if.then1403, %if.then1360
+  %m.2.in = phi i32 [ %m.048, %if.then1360 ], [ %m.2, %if.then1403 ]
+  %m.2 = add nuw nsw i32 %m.2.in, 1
+  %cmp1392.not.not = icmp slt i32 %m.2.in, %0
+  %idxprom1398 = zext i32 %m.2 to i64
+  br label %if.then1403
+
+if.then1403:                                      ; preds = %for.cond1390
+  %xpos1409 = getelementptr inbounds i32, i32* %ptr, i64 %idxprom1398
+  %c.1 = call i1 @cond()
+  br i1 %c.1, label %for.cond1390, label %for.end1469
+
+for.end1469:                                      ; preds = %if.then1403
+  br label %if.end1824
+
+for.inc1498:                                      ; preds = %if.else1351
+  %inc1499 = add nuw nsw i32 %m.048, 1
+  %1 = load i32, i32* undef, align 8
+  br label %for.body1208
+
+if.end1824:                                       ; preds = %for.end1469
+  br label %while.body
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/funclet.ll b/llvm/test/Transforms/LoopStrengthReduce/funclet.ll
index 1bee3706cafab..0f725a13b2078 100644
--- a/llvm/test/Transforms/LoopStrengthReduce/funclet.ll
+++ b/llvm/test/Transforms/LoopStrengthReduce/funclet.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -loop-reduce -S | FileCheck %s
 
 target datalayout = "e-m:x-p:32:32-i64:64-f80:32-n8:16:32-a:0:32-S32"
@@ -10,13 +11,42 @@ declare void @external(i32*)
 declare void @reserve()
 
 define void @f() personality i32 (...)* @_except_handler3 {
+; CHECK-LABEL: @f(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[THROW:%.*]]
+; CHECK:       throw:
+; CHECK-NEXT:    [[TMP96:%.*]] = getelementptr inbounds i8, i8* undef, i32 1
+; CHECK-NEXT:    invoke void @reserve()
+; CHECK-NEXT:    to label [[THROW]] unwind label [[PAD:%.*]]
+; CHECK:       pad:
+; CHECK-NEXT:    [[PHI2:%.*]] = phi i8* [ [[TMP96]], [[THROW]] ]
+; CHECK-NEXT:    [[CS:%.*]] = catchswitch within none [label %unreachable] unwind label [[BLAH2:%.*]]
+; CHECK:       unreachable:
+; CHECK-NEXT:    [[TMP0:%.*]] = catchpad within [[CS]] []
+; CHECK-NEXT:    unreachable
+; CHECK:       blah2:
+; CHECK-NEXT:    [[CLEANUPPADI4_I_I_I:%.*]] = cleanuppad within none []
+; CHECK-NEXT:    [[PHI21:%.*]] = ptrtoint i8* [[PHI2]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i32 1, [[PHI21]]
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, i8* undef, i32 [[TMP1]]
+; CHECK-NEXT:    br label [[LOOP_BODY:%.*]]
+; CHECK:       loop_body:
+; CHECK-NEXT:    [[LSR_IV:%.*]] = phi i8* [ [[SCEVGEP2:%.*]], [[ITER:%.*]] ], [ [[SCEVGEP]], [[BLAH2]] ]
+; CHECK-NEXT:    [[SCEVGEP2]] = getelementptr i8, i8* [[LSR_IV]], i32 -1
+; CHECK-NEXT:    [[TMP100:%.*]] = icmp eq i8* [[SCEVGEP2]], null
+; CHECK-NEXT:    br i1 [[TMP100]], label [[UNWIND_OUT:%.*]], label [[ITER]]
+; CHECK:       iter:
+; CHECK-NEXT:    br i1 true, label [[UNWIND_OUT]], label [[LOOP_BODY]]
+; CHECK:       unwind_out:
+; CHECK-NEXT:    cleanupret from [[CLEANUPPADI4_I_I_I]] unwind to caller
+;
 entry:
   br label %throw
 
 throw:                                            ; preds = %throw, %entry
   %tmp96 = getelementptr inbounds i8, i8* undef, i32 1
   invoke void @reserve()
-          to label %throw unwind label %pad
+  to label %throw unwind label %pad
 
 pad:                                              ; preds = %throw
   %phi2 = phi i8* [ %tmp96, %throw ]
@@ -43,18 +73,45 @@ unwind_out:                                       ; preds = %iter, %loop_body
   cleanupret from %cleanuppadi4.i.i.i unwind to caller
 }
 
-; CHECK-LABEL: define void @f(
-; CHECK: cleanuppad within none []
-; CHECK-NEXT: ptrtoint i8* %phi2 to i32
-
 define void @g() personality i32 (...)* @_except_handler3 {
+; CHECK-LABEL: @g(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[THROW:%.*]]
+; CHECK:       throw:
+; CHECK-NEXT:    [[TMP96:%.*]] = getelementptr inbounds i8, i8* undef, i32 1
+; CHECK-NEXT:    invoke void @reserve()
+; CHECK-NEXT:    to label [[THROW]] unwind label [[PAD:%.*]]
+; CHECK:       pad:
+; CHECK-NEXT:    [[PHI2:%.*]] = phi i8* [ [[TMP96]], [[THROW]] ]
+; CHECK-NEXT:    [[CS:%.*]] = catchswitch within none [label [[UNREACHABLE:%.*]], label %blah] unwind to caller
+; CHECK:       unreachable:
+; CHECK-NEXT:    [[TMP0:%.*]] = catchpad within [[CS]] []
+; CHECK-NEXT:    unreachable
+; CHECK:       blah:
+; CHECK-NEXT:    [[CATCHPAD:%.*]] = catchpad within [[CS]] []
+; CHECK-NEXT:    [[PHI21:%.*]] = ptrtoint i8* [[PHI2]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i32 1, [[PHI21]]
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, i8* undef, i32 [[TMP1]]
+; CHECK-NEXT:    br label [[LOOP_BODY:%.*]]
+; CHECK:       unwind_out:
+; CHECK-NEXT:    catchret from [[CATCHPAD]] to label [[LEAVE:%.*]]
+; CHECK:       leave:
+; CHECK-NEXT:    ret void
+; CHECK:       loop_body:
+; CHECK-NEXT:    [[LSR_IV:%.*]] = phi i8* [ [[SCEVGEP2:%.*]], [[ITER:%.*]] ], [ [[SCEVGEP]], [[BLAH:%.*]] ]
+; CHECK-NEXT:    [[SCEVGEP2]] = getelementptr i8, i8* [[LSR_IV]], i32 -1
+; CHECK-NEXT:    [[TMP100:%.*]] = icmp eq i8* [[SCEVGEP2]], null
+; CHECK-NEXT:    br i1 [[TMP100]], label [[UNWIND_OUT:%.*]], label [[ITER]]
+; CHECK:       iter:
+; CHECK-NEXT:    br i1 true, label [[UNWIND_OUT]], label [[LOOP_BODY]]
+;
 entry:
   br label %throw
 
 throw:                                            ; preds = %throw, %entry
   %tmp96 = getelementptr inbounds i8, i8* undef, i32 1
   invoke void @reserve()
-          to label %throw unwind label %pad
+  to label %throw unwind label %pad
 
 pad:
   %phi2 = phi i8* [ %tmp96, %throw ]
@@ -84,20 +141,45 @@ iter:                                             ; preds = %loop_body
   br i1 undef, label %unwind_out, label %loop_body
 }
 
-; CHECK-LABEL: define void @g(
-; CHECK: blah:
-; CHECK-NEXT: catchpad within %cs []
-; CHECK-NEXT: ptrtoint i8* %phi2 to i32
-
-
 define void @h() personality i32 (...)* @_except_handler3 {
+; CHECK-LABEL: @h(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[THROW:%.*]]
+; CHECK:       throw:
+; CHECK-NEXT:    [[TMP96:%.*]] = getelementptr inbounds i8, i8* undef, i32 1
+; CHECK-NEXT:    invoke void @reserve()
+; CHECK-NEXT:    to label [[THROW]] unwind label [[PAD:%.*]]
+; CHECK:       pad:
+; CHECK-NEXT:    [[CS:%.*]] = catchswitch within none [label [[UNREACHABLE:%.*]], label %blug] unwind to caller
+; CHECK:       unreachable:
+; CHECK-NEXT:    [[TMP0:%.*]] = catchpad within [[CS]] []
+; CHECK-NEXT:    unreachable
+; CHECK:       blug:
+; CHECK-NEXT:    [[PHI2:%.*]] = phi i8* [ [[TMP96]], [[PAD]] ]
+; CHECK-NEXT:    [[CATCHPAD:%.*]] = catchpad within [[CS]] []
+; CHECK-NEXT:    [[PHI21:%.*]] = ptrtoint i8* [[PHI2]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i32 1, [[PHI21]]
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, i8* undef, i32 [[TMP1]]
+; CHECK-NEXT:    br label [[LOOP_BODY:%.*]]
+; CHECK:       unwind_out:
+; CHECK-NEXT:    catchret from [[CATCHPAD]] to label [[LEAVE:%.*]]
+; CHECK:       leave:
+; CHECK-NEXT:    ret void
+; CHECK:       loop_body:
+; CHECK-NEXT:    [[LSR_IV:%.*]] = phi i8* [ [[SCEVGEP2:%.*]], [[ITER:%.*]] ], [ [[SCEVGEP]], [[BLUG:%.*]] ]
+; CHECK-NEXT:    [[SCEVGEP2]] = getelementptr i8, i8* [[LSR_IV]], i32 -1
+; CHECK-NEXT:    [[TMP100:%.*]] = icmp eq i8* [[SCEVGEP2]], null
+; CHECK-NEXT:    br i1 [[TMP100]], label [[UNWIND_OUT:%.*]], label [[ITER]]
+; CHECK:       iter:
+; CHECK-NEXT:    br i1 true, label [[UNWIND_OUT]], label [[LOOP_BODY]]
+;
 entry:
   br label %throw
 
 throw:                                            ; preds = %throw, %entry
   %tmp96 = getelementptr inbounds i8, i8* undef, i32 1
   invoke void @reserve()
-          to label %throw unwind label %pad
+  to label %throw unwind label %pad
 
 pad:
   %cs = catchswitch within none [label %unreachable, label %blug] unwind to caller
@@ -127,19 +209,45 @@ iter:                                             ; preds = %loop_body
   br i1 undef, label %unwind_out, label %loop_body
 }
 
-; CHECK-LABEL: define void @h(
-; CHECK: blug:
-; CHECK: catchpad within %cs []
-; CHECK-NEXT: ptrtoint i8* %phi2 to i32
-
 define void @i() personality i32 (...)* @_except_handler3 {
+; CHECK-LABEL: @i(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[THROW:%.*]]
+; CHECK:       throw:
+; CHECK-NEXT:    [[TMP96:%.*]] = getelementptr inbounds i8, i8* undef, i32 1
+; CHECK-NEXT:    invoke void @reserve()
+; CHECK-NEXT:    to label [[THROW]] unwind label [[CATCHPAD:%.*]]
+; CHECK:       catchpad:
+; CHECK-NEXT:    [[PHI2:%.*]] = phi i8* [ [[TMP96]], [[THROW]] ]
+; CHECK-NEXT:    [[CS:%.*]] = catchswitch within none [label %cp_body] unwind label [[CLEANUPPAD:%.*]]
+; CHECK:       cp_body:
+; CHECK-NEXT:    [[TMP0:%.*]] = catchpad within [[CS]] []
+; CHECK-NEXT:    br label [[LOOP_HEAD:%.*]]
+; CHECK:       cleanuppad:
+; CHECK-NEXT:    [[TMP1:%.*]] = cleanuppad within none []
+; CHECK-NEXT:    br label [[LOOP_HEAD]]
+; CHECK:       loop_head:
+; CHECK-NEXT:    [[PHI21:%.*]] = ptrtoint i8* [[PHI2]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = sub i32 1, [[PHI21]]
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, i8* undef, i32 [[TMP2]]
+; CHECK-NEXT:    br label [[LOOP_BODY:%.*]]
+; CHECK:       loop_body:
+; CHECK-NEXT:    [[LSR_IV:%.*]] = phi i8* [ [[SCEVGEP2:%.*]], [[ITER:%.*]] ], [ [[SCEVGEP]], [[LOOP_HEAD]] ]
+; CHECK-NEXT:    [[SCEVGEP2]] = getelementptr i8, i8* [[LSR_IV]], i32 -1
+; CHECK-NEXT:    [[TMP100:%.*]] = icmp eq i8* [[SCEVGEP2]], null
+; CHECK-NEXT:    br i1 [[TMP100]], label [[UNWIND_OUT:%.*]], label [[ITER]]
+; CHECK:       iter:
+; CHECK-NEXT:    br i1 true, label [[UNWIND_OUT]], label [[LOOP_BODY]]
+; CHECK:       unwind_out:
+; CHECK-NEXT:    unreachable
+;
 entry:
   br label %throw
 
 throw:                                            ; preds = %throw, %entry
   %tmp96 = getelementptr inbounds i8, i8* undef, i32 1
   invoke void @reserve()
-          to label %throw unwind label %catchpad
+  to label %throw unwind label %catchpad
 
 catchpad:                                              ; preds = %throw
   %phi2 = phi i8* [ %tmp96, %throw ]
@@ -169,17 +277,41 @@ unwind_out:                                       ; preds = %iter, %loop_body
   unreachable
 }
 
-; CHECK-LABEL: define void @i(
-; CHECK: ptrtoint i8* %phi2 to i32
-
 define void @test1(i32* %b, i32* %c) personality i32 (...)* @__CxxFrameHandler3 {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_COND:%.*]]
+; CHECK:       for.cond:
+; CHECK-NEXT:    [[D_0:%.*]] = phi i32* [ [[B:%.*]], [[ENTRY:%.*]] ], [ [[INCDEC_PTR:%.*]], [[FOR_INC:%.*]] ]
+; CHECK-NEXT:    invoke void @external(i32* [[D_0]])
+; CHECK-NEXT:    to label [[FOR_INC]] unwind label [[CATCH_DISPATCH:%.*]]
+; CHECK:       for.inc:
+; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds i32, i32* [[D_0]], i32 1
+; CHECK-NEXT:    br label [[FOR_COND]]
+; CHECK:       catch.dispatch:
+; CHECK-NEXT:    [[CS:%.*]] = catchswitch within none [label %catch] unwind label [[CATCH_DISPATCH_2:%.*]]
+; CHECK:       catch:
+; CHECK-NEXT:    [[TMP0:%.*]] = catchpad within [[CS]] [i8* null, i32 64, i8* null]
+; CHECK-NEXT:    catchret from [[TMP0]] to label [[TRY_CONT:%.*]]
+; CHECK:       try.cont:
+; CHECK-NEXT:    invoke void @external(i32* [[C:%.*]])
+; CHECK-NEXT:    to label [[TRY_CONT_7:%.*]] unwind label [[CATCH_DISPATCH_2]]
+; CHECK:       catch.dispatch.2:
+; CHECK-NEXT:    [[E_0:%.*]] = phi i32* [ [[C]], [[TRY_CONT]] ], [ [[B]], [[CATCH_DISPATCH]] ]
+; CHECK-NEXT:    [[CS2:%.*]] = catchswitch within none [label %catch.4] unwind to caller
+; CHECK:       catch.4:
+; CHECK-NEXT:    [[TMP1:%.*]] = catchpad within [[CS2]] [i8* null, i32 64, i8* null]
+; CHECK-NEXT:    unreachable
+; CHECK:       try.cont.7:
+; CHECK-NEXT:    ret void
+;
 entry:
   br label %for.cond
 
 for.cond:                                         ; preds = %for.inc, %entry
   %d.0 = phi i32* [ %b, %entry ], [ %incdec.ptr, %for.inc ]
   invoke void @external(i32* %d.0)
-          to label %for.inc unwind label %catch.dispatch
+  to label %for.inc unwind label %catch.dispatch
 
 for.inc:                                          ; preds = %for.cond
   %incdec.ptr = getelementptr inbounds i32, i32* %d.0, i32 1
@@ -194,7 +326,7 @@ catch:                                            ; preds = %catch.dispatch
 
 try.cont:                                         ; preds = %catch
   invoke void @external(i32* %c)
-          to label %try.cont.7 unwind label %catch.dispatch.2
+  to label %try.cont.7 unwind label %catch.dispatch.2
 
 catch.dispatch.2:                                 ; preds = %try.cont, %catchendblock
   %e.0 = phi i32* [ %c, %try.cont ], [ %b, %catch.dispatch ]
@@ -208,21 +340,33 @@ try.cont.7:                                       ; preds = %try.cont
   ret void
 }
 
-; CHECK-LABEL: define void @test1(
-; CHECK: for.cond:
-; CHECK:   %d.0 = phi i32* [ %b, %entry ], [ %incdec.ptr, %for.inc ]
-
-; CHECK: catch.dispatch.2:
-; CHECK: %e.0 = phi i32* [ %c, %try.cont ], [ %b, %catch.dispatch ]
-
 define i32 @test2() personality i32 (...)* @_except_handler3 {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[PHI:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_INC:%.*]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    invoke void @reserve()
+; CHECK-NEXT:    to label [[FOR_INC]] unwind label [[CATCH_DISPATCH:%.*]]
+; CHECK:       catch.dispatch:
+; CHECK-NEXT:    [[TMP18:%.*]] = catchswitch within none [label %catch.handler] unwind to caller
+; CHECK:       catch.handler:
+; CHECK-NEXT:    [[PHI_LCSSA:%.*]] = phi i32 [ [[PHI]], [[CATCH_DISPATCH]] ]
+; CHECK-NEXT:    [[TMP19:%.*]] = catchpad within [[TMP18]] [i8* null]
+; CHECK-NEXT:    catchret from [[TMP19]] to label [[DONE:%.*]]
+; CHECK:       done:
+; CHECK-NEXT:    ret i32 [[PHI_LCSSA]]
+; CHECK:       for.inc:
+; CHECK-NEXT:    [[INC]] = add i32 [[PHI]], 1
+; CHECK-NEXT:    br label [[FOR_BODY]]
+;
 entry:
   br label %for.body
 
 for.body:                                         ; preds = %for.inc, %entry
   %phi = phi i32 [ %inc, %for.inc ], [ 0, %entry ]
   invoke void @reserve()
-          to label %for.inc unwind label %catch.dispatch
+  to label %for.inc unwind label %catch.dispatch
 
 catch.dispatch:                                   ; preds = %for.body
   %tmp18 = catchswitch within none [label %catch.handler] unwind to caller
@@ -239,7 +383,3 @@ for.inc:                                          ; preds = %for.body
   %inc = add i32 %phi, 1
   br label %for.body
 }
-
-; CHECK-LABEL: define i32 @test2(
-; CHECK:      %phi.lcssa = phi i32 [ %phi, %catch.dispatch ]
-; CHECK-NEXT: catchpad within
diff --git a/llvm/test/Transforms/LoopStrengthReduce/scev-expander-lcssa.ll b/llvm/test/Transforms/LoopStrengthReduce/scev-expander-lcssa.ll
new file mode 100644
index 0000000000000..e05dcda85138a
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/scev-expander-lcssa.ll
@@ -0,0 +1,144 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -loop-reduce -S %s | FileCheck %s
+
+; Make sure SCEVExpander does not crash and introduce unnecessary LCSSA PHI nodes.
+
+define void @schedule_block() {
+; CHECK-LABEL: @schedule_block(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    switch i16 undef, label [[IF_END156_I:%.*]] [
+; CHECK-NEXT:    i16 27, label [[IF_THEN_I:%.*]]
+; CHECK-NEXT:    i16 28, label [[IF_THEN_I]]
+; CHECK-NEXT:    i16 29, label [[IF_THEN13_I:%.*]]
+; CHECK-NEXT:    i16 32, label [[LAND_LHS_TRUE136_I:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       if.then.i:
+; CHECK-NEXT:    unreachable
+; CHECK:       if.then13.i:
+; CHECK-NEXT:    unreachable
+; CHECK:       land.lhs.true136.i:
+; CHECK-NEXT:    unreachable
+; CHECK:       if.end156.i:
+; CHECK-NEXT:    switch i16 undef, label [[WHILE_END256:%.*]] [
+; CHECK-NEXT:    i16 29, label [[IF_THEN210:%.*]]
+; CHECK-NEXT:    i16 28, label [[IF_THEN210]]
+; CHECK-NEXT:    i16 27, label [[LAND_LHS_TRUE191:%.*]]
+; CHECK-NEXT:    i16 32, label [[IF_END248:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       land.lhs.true191:
+; CHECK-NEXT:    unreachable
+; CHECK:       if.then210:
+; CHECK-NEXT:    unreachable
+; CHECK:       if.end248:
+; CHECK-NEXT:    br label [[FOR_END:%.*]]
+; CHECK:       while.end256:
+; CHECK-NEXT:    unreachable
+; CHECK:       for.end:
+; CHECK-NEXT:    br label [[WHILE_BODY1013:%.*]]
+; CHECK:       while.body1013:
+; CHECK-NEXT:    br label [[FOR_COND_I2472:%.*]]
+; CHECK:       for.cond.i2472:
+; CHECK-NEXT:    [[I_0_I:%.*]] = phi i32 [ 0, [[WHILE_BODY1013]] ], [ [[TMP2:%.*]], [[FOR_END34_I:%.*]] ]
+; CHECK-NEXT:    br i1 false, label [[FOR_COND3_PREHEADER_I:%.*]], label [[IF_END107_I_LOOPEXIT:%.*]]
+; CHECK:       for.cond3.preheader.i:
+; CHECK-NEXT:    [[TMP0:%.*]] = sext i32 [[I_0_I]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = add nsw i64 [[TMP0]], 1
+; CHECK-NEXT:    br label [[FOR_COND3_I:%.*]]
+; CHECK:       for.cond3.i:
+; CHECK-NEXT:    [[INDVARS_IV301_I2691:%.*]] = phi i64 [ [[INDVARS_IV_NEXT302_I:%.*]], [[FOR_BODY5_I:%.*]] ], [ [[TMP1]], [[FOR_COND3_PREHEADER_I]] ]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT302_I]] = add nsw i64 [[INDVARS_IV301_I2691]], 1
+; CHECK-NEXT:    br label [[FOR_BODY5_I]]
+; CHECK:       for.body5.i:
+; CHECK-NEXT:    br i1 false, label [[FOR_COND3_I]], label [[FOR_BODY5_I_FOR_END_I2475_LOOPEXIT_CRIT_EDGE:%.*]]
+; CHECK:       for.body5.i.for.end.i2475.loopexit_crit_edge:
+; CHECK-NEXT:    [[TMP2]] = trunc i64 [[INDVARS_IV_NEXT302_I]] to i32
+; CHECK-NEXT:    br label [[FOR_END34_I]]
+; CHECK:       for.end34.i:
+; CHECK-NEXT:    br i1 false, label [[FOR_COND_I2472]], label [[IF_ELSE_I2488:%.*]]
+; CHECK:       if.else.i2488:
+; CHECK-NEXT:    br i1 undef, label [[IF_END107_I:%.*]], label [[FOR_BODY45_PREHEADER_I:%.*]]
+; CHECK:       for.body45.preheader.i:
+; CHECK-NEXT:    [[TMP3:%.*]] = sext i32 [[I_0_I]] to i64
+; CHECK-NEXT:    unreachable
+; CHECK:       if.end107.i.loopexit:
+; CHECK-NEXT:    br label [[IF_END107_I]]
+; CHECK:       if.end107.i:
+; CHECK-NEXT:    unreachable
+;
+entry:
+  switch i16 undef, label %if.end156.i [
+  i16 27, label %if.then.i
+  i16 28, label %if.then.i
+  i16 29, label %if.then13.i
+  i16 32, label %land.lhs.true136.i
+  ]
+
+if.then.i:                                        ; preds = %entry, %entry
+  unreachable
+
+if.then13.i:                                      ; preds = %entry
+  unreachable
+
+land.lhs.true136.i:                               ; preds = %entry
+  unreachable
+
+if.end156.i:                                      ; preds = %entry
+  switch i16 undef, label %while.end256 [
+  i16 29, label %if.then210
+  i16 28, label %if.then210
+  i16 27, label %land.lhs.true191
+  i16 32, label %if.end248
+  ]
+
+land.lhs.true191:                                 ; preds = %if.end156.i
+  unreachable
+
+if.then210:                                       ; preds = %if.end156.i, %if.end156.i
+  unreachable
+
+if.end248:                                        ; preds = %if.end156.i
+  br label %for.end
+
+while.end256:                                     ; preds = %if.end156.i
+  unreachable
+
+for.end:                                          ; preds = %if.end248
+  br label %while.body1013
+
+while.body1013:                                   ; preds = %for.end
+  br label %for.cond.i2472
+
+for.cond.i2472:                                   ; preds = %for.end34.i, %while.body1013
+  %i.0.i = phi i32 [ 0, %while.body1013 ], [ %2, %for.end34.i ]
+  br i1 undef, label %for.cond3.preheader.i, label %if.end107.i
+
+for.cond3.preheader.i:                            ; preds = %for.cond.i2472
+  %0 = sext i32 %i.0.i to i64
+  %1 = add nsw i64 %0, 1
+  br label %for.cond3.i
+
+for.cond3.i:                                      ; preds = %for.body5.i, %for.cond3.preheader.i
+  %indvars.iv301.i2691 = phi i64 [ %indvars.iv.next302.i, %for.body5.i ], [ %1, %for.cond3.preheader.i ]
+  %indvars.iv.next302.i = add nsw i64 %indvars.iv301.i2691, 1
+  br label %for.body5.i
+
+for.body5.i:                                      ; preds = %for.cond3.i
+  br i1 undef, label %for.cond3.i, label %for.body5.i.for.end.i2475.loopexit_crit_edge
+
+for.body5.i.for.end.i2475.loopexit_crit_edge:     ; preds = %for.body5.i
+  %2 = trunc i64 %indvars.iv.next302.i to i32
+  br label %for.end34.i
+
+for.end34.i:                                      ; preds = %for.body5.i.for.end.i2475.loopexit_crit_edge
+  br i1 undef, label %for.cond.i2472, label %if.else.i2488
+
+if.else.i2488:                                    ; preds = %for.end34.i
+  br i1 undef, label %if.end107.i, label %for.body45.preheader.i
+
+for.body45.preheader.i:                           ; preds = %if.else.i2488
+  %3 = sext i32 %i.0.i to i64
+  unreachable
+
+if.end107.i:                                      ; preds = %if.else.i2488, %for.cond.i2472
+  unreachable
+}

From a4edc04693f76eec9068db0556d6533e4c201d74 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Sun, 12 Jul 2020 14:16:36 -0400
Subject: [PATCH 0311/1035] AMDGPU/GlobalISel: Use clamp modifier for
 [us]addsat/[us]subsat

We also have never handled this for SelectionDAG, which needs
additional work.
---
 llvm/lib/Target/AMDGPU/AMDGPUGISel.td         |    5 +
 .../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp |   11 +-
 llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h      |    4 +
 llvm/lib/Target/AMDGPU/VOP2Instructions.td    |   18 +
 llvm/lib/Target/AMDGPU/VOP3Instructions.td    |   14 +
 llvm/lib/Target/AMDGPU/VOP3PInstructions.td   |   15 +
 llvm/lib/Target/AMDGPU/VOPInstructions.td     |   13 +
 .../AMDGPU/GlobalISel/legalize-saddsat.mir    |  172 +-
 .../AMDGPU/GlobalISel/legalize-ssubsat.mir    |  172 +-
 .../AMDGPU/GlobalISel/legalize-uaddsat.mir    |  211 +-
 .../AMDGPU/GlobalISel/legalize-usubsat.mir    |  142 +-
 .../test/CodeGen/AMDGPU/GlobalISel/saddsat.ll | 3401 +++--------------
 .../test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll | 3399 +++-------------
 .../test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll | 2550 ++++--------
 .../test/CodeGen/AMDGPU/GlobalISel/usubsat.ll | 2161 ++++-------
 15 files changed, 2714 insertions(+), 9574 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUGISel.td b/llvm/lib/Target/AMDGPU/AMDGPUGISel.td
index 3f12addbcc79b..056f91db24ff6 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUGISel.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUGISel.td
@@ -51,6 +51,11 @@ def gi_vop3opselmods :
     GIComplexOperandMatcher<s32, "selectVOP3OpSelMods">,
     GIComplexPatternEquiv<VOP3OpSelMods>;
 
+// FIXME: Why do we have both VOP3OpSel and VOP3OpSelMods?
+def gi_vop3opsel :
+    GIComplexOperandMatcher<s32, "selectVOP3OpSelMods">,
+    GIComplexPatternEquiv<VOP3OpSel>;
+
 def gi_smrd_imm :
     GIComplexOperandMatcher<s64, "selectSmrdImm">,
     GIComplexPatternEquiv<SMRDImm>;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index bf0ebd322aa9e..e14623e650d42 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -422,7 +422,8 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
     .moreElementsIf(isSmallOddVector(0), oneMoreElement(0))
     .scalarize(0);
 
-  if (ST.hasVOP3PInsts()) {
+  if (ST.hasVOP3PInsts() && ST.hasAddNoCarry() && ST.hasIntClamp()) {
+    // Full set of gfx9 features.
     getActionDefinitionsBuilder({G_ADD, G_SUB, G_MUL})
       .legalFor({S32, S16, V2S16})
       .clampScalar(0, S16, S32)
@@ -431,7 +432,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
       .widenScalarToNextPow2(0, 32);
 
     getActionDefinitionsBuilder({G_UADDSAT, G_USUBSAT, G_SADDSAT, G_SSUBSAT})
-      .lowerFor({S32, S16, V2S16}) // FIXME: legal and merge with add/sub/mul
+      .legalFor({S32, S16, V2S16}) // Clamp modifier
       .minScalar(0, S16)
       .clampMaxNumElements(0, S16, 2)
       .scalarize(0)
@@ -447,7 +448,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
     // Technically the saturating operations require clamp bit support, but this
     // was introduced at the same time as 16-bit operations.
     getActionDefinitionsBuilder({G_UADDSAT, G_USUBSAT})
-      .lowerFor({S32, S16}) // FIXME: legal with clamp modifier
+      .legalFor({S32, S16}) // Clamp modifier
       .minScalar(0, S16)
       .scalarize(0)
       .widenScalarToNextPow2(0, 16)
@@ -467,7 +468,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
 
     if (ST.hasIntClamp()) {
       getActionDefinitionsBuilder({G_UADDSAT, G_USUBSAT})
-        .lowerFor({S32}) // FIXME: legal with clamp modifier.
+        .legalFor({S32}) // Clamp modifier.
         .scalarize(0)
         .minScalarOrElt(0, S32)
         .lower();
@@ -479,6 +480,8 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
         .lower();
     }
 
+    // FIXME: DAG expansion gets better results. The widening uses the smaller
+    // range values and goes for the min/max lowering directly.
     getActionDefinitionsBuilder({G_SADDSAT, G_SSUBSAT})
       .minScalar(0, S32)
       .scalarize(0)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h
index 6b23830491235..6848f762fc276 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h
@@ -791,6 +791,10 @@ class GCNSubtarget : public AMDGPUGenSubtargetInfo,
     return CIInsts;
   }
 
+  /// \returns true if the target has integer add/sub instructions that do not
+  /// produce a carry-out. This includes v_add_[iu]32, v_sub_[iu]32,
+  /// v_add_[iu]16, and v_sub_[iu]16, all of which support the clamp modifier
+  /// for saturation.
   bool hasAddNoCarry() const {
     return AddNoCarryInsts;
   }
diff --git a/llvm/lib/Target/AMDGPU/VOP2Instructions.td b/llvm/lib/Target/AMDGPU/VOP2Instructions.td
index d9dcfdca23b35..3451c23891811 100644
--- a/llvm/lib/Target/AMDGPU/VOP2Instructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP2Instructions.td
@@ -828,6 +828,24 @@ def : GCNPat <
 } // End Predicates = [Has16BitInsts]
 
 
+let SubtargetPredicate = HasIntClamp in {
+// Set clamp bit for saturation.
+def : VOPBinOpClampPat<uaddsat, V_ADD_CO_U32_e64, i32>;
+def : VOPBinOpClampPat<usubsat, V_SUB_CO_U32_e64, i32>;
+}
+
+let SubtargetPredicate = HasAddNoCarryInsts, OtherPredicates = [HasIntClamp] in {
+let AddedComplexity = 1 in { // Prefer over form with carry-out.
+def : VOPBinOpClampPat<uaddsat, V_ADD_U32_e64, i32>;
+def : VOPBinOpClampPat<usubsat, V_SUB_U32_e64, i32>;
+}
+}
+
+let SubtargetPredicate = Has16BitInsts, OtherPredicates = [HasIntClamp] in {
+def : VOPBinOpClampPat<uaddsat, V_ADD_U16_e64, i16>;
+def : VOPBinOpClampPat<usubsat, V_SUB_U16_e64, i16>;
+}
+
 //===----------------------------------------------------------------------===//
 // Target-specific instruction encodings.
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/AMDGPU/VOP3Instructions.td b/llvm/lib/Target/AMDGPU/VOP3Instructions.td
index dcbfeb547a32d..3048bcc610c76 100644
--- a/llvm/lib/Target/AMDGPU/VOP3Instructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP3Instructions.td
@@ -667,6 +667,20 @@ def : ThreeOp_i32_Pats<and, or, V_AND_OR_B32>;
 def : ThreeOp_i32_Pats<or, or, V_OR3_B32>;
 def : ThreeOp_i32_Pats<xor, add, V_XAD_U32>;
 
+def : VOPBinOpClampPat<saddsat, V_ADD_I32, i32>;
+def : VOPBinOpClampPat<ssubsat, V_SUB_I32, i32>;
+
+
+// FIXME: Probably should hardcode clamp bit in pseudo and avoid this.
+class OpSelBinOpClampPat<SDPatternOperator node,
+                         Instruction inst> : GCNPat<
+ (node (i16 (VOP3OpSel i16:$src0, i32:$src0_modifiers)),
+       (i16 (VOP3OpSel i16:$src1, i32:$src1_modifiers))),
+  (inst $src0_modifiers, $src0, $src1_modifiers, $src1, DSTCLAMP.ENABLE, 0)
+>;
+
+def : OpSelBinOpClampPat<saddsat, V_ADD_I16>;
+def : OpSelBinOpClampPat<ssubsat, V_SUB_I16>;
 } // End SubtargetPredicate = isGFX9Plus
 
 def VOP3_PERMLANE_Profile : VOP3_Profile<VOPProfile <[i32, i32, i32, i32]>, VOP3_OPSEL> {
diff --git a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
index fc457ad212d48..446e87ab3fc98 100644
--- a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
@@ -77,6 +77,8 @@ def V_PK_ASHRREV_I16 : VOP3PInst<"v_pk_ashrrev_i16", VOP3_Profile<VOP_V2I16_V2I1
 def V_PK_LSHRREV_B16 : VOP3PInst<"v_pk_lshrrev_b16", VOP3_Profile<VOP_V2I16_V2I16_V2I16>, lshr_rev>;
 
 
+let SubtargetPredicate = HasVOP3PInsts in {
+
 // Undo sub x, c -> add x, -c canonicalization since c is more likely
 // an inline immediate than -c.
 // The constant will be emitted as a mov, and folded later.
@@ -86,6 +88,19 @@ def : GCNPat<
   (V_PK_SUB_U16 $src0_modifiers, $src0, SRCMODS.OP_SEL_1, NegSubInlineConstV216:$src1)
 >;
 
+// Integer operations with clamp bit set.
+class VOP3PSatPat<SDPatternOperator pat, Instruction inst> : GCNPat<
+  (pat (v2i16 (VOP3PMods v2i16:$src0, i32:$src0_modifiers)),
+       (v2i16 (VOP3PMods v2i16:$src1, i32:$src1_modifiers))),
+  (inst $src0_modifiers, $src0, $src1_modifiers, $src1, DSTCLAMP.ENABLE)
+>;
+
+def : VOP3PSatPat<uaddsat, V_PK_ADD_U16>;
+def : VOP3PSatPat<saddsat, V_PK_ADD_I16>;
+def : VOP3PSatPat<usubsat, V_PK_SUB_U16>;
+def : VOP3PSatPat<ssubsat, V_PK_SUB_I16>;
+} // End SubtargetPredicate = HasVOP3PInsts
+
 multiclass MadFmaMixPats<SDPatternOperator fma_like,
                          Instruction mix_inst,
                          Instruction mixlo_inst,
diff --git a/llvm/lib/Target/AMDGPU/VOPInstructions.td b/llvm/lib/Target/AMDGPU/VOPInstructions.td
index f8a83e5f74c0b..ab1915de0c734 100644
--- a/llvm/lib/Target/AMDGPU/VOPInstructions.td
+++ b/llvm/lib/Target/AMDGPU/VOPInstructions.td
@@ -776,6 +776,19 @@ class DivergentFragOrOp<SDPatternOperator Op, VOPProfile P> {
    !if(!isa<SDNode>(Op), getDivergentFrag<Op>.ret, Op), Op);
 }
 
+class getVSrcOp<ValueType vt> {
+  RegisterOperand ret = !if(!eq(vt.Size, 32), VSrc_b32, VSrc_b16);
+}
+
+// Class for binary integer operations with the clamp bit set for saturation
+// TODO: Add sub with negated inline constant pattern.
+class VOPBinOpClampPat<SDPatternOperator node, Instruction inst, ValueType vt> :
+  GCNPat<(node vt:$src0, vt:$src1),
+         (inst getVSrcOp<vt>.ret:$src0, getVSrcOp<vt>.ret:$src1,
+               DSTCLAMP.ENABLE)
+>;
+
+
 include "VOPCInstructions.td"
 include "VOP1Instructions.td"
 include "VOP2Instructions.td"
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-saddsat.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-saddsat.mir
index 28a8efad1d102..51b6e014a9376 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-saddsat.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-saddsat.mir
@@ -59,17 +59,8 @@ body: |
     ; GFX9: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 9
     ; GFX9: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[C]](s16)
     ; GFX9: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[C]](s16)
-    ; GFX9: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 32767
-    ; GFX9: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 -32768
-    ; GFX9: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 0
-    ; GFX9: [[SMAX:%[0-9]+]]:_(s16) = G_SMAX [[SHL]], [[C3]]
-    ; GFX9: [[SUB:%[0-9]+]]:_(s16) = G_SUB [[C1]], [[SMAX]]
-    ; GFX9: [[SMIN:%[0-9]+]]:_(s16) = G_SMIN [[SHL]], [[C3]]
-    ; GFX9: [[SUB1:%[0-9]+]]:_(s16) = G_SUB [[C2]], [[SMIN]]
-    ; GFX9: [[SMAX1:%[0-9]+]]:_(s16) = G_SMAX [[SUB1]], [[SHL1]]
-    ; GFX9: [[SMIN1:%[0-9]+]]:_(s16) = G_SMIN [[SMAX1]], [[SUB]]
-    ; GFX9: [[ADD:%[0-9]+]]:_(s16) = G_ADD [[SHL]], [[SMIN1]]
-    ; GFX9: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[ADD]], [[C]](s16)
+    ; GFX9: [[SADDSAT:%[0-9]+]]:_(s16) = G_SADDSAT [[SHL]], [[SHL1]]
+    ; GFX9: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[SADDSAT]], [[C]](s16)
     ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ASHR]](s16)
     ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
     %0:_(s32) = COPY $vgpr0
@@ -137,17 +128,8 @@ body: |
     ; GFX9: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
     ; GFX9: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[C]](s16)
     ; GFX9: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[C]](s16)
-    ; GFX9: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 32767
-    ; GFX9: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 -32768
-    ; GFX9: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 0
-    ; GFX9: [[SMAX:%[0-9]+]]:_(s16) = G_SMAX [[SHL]], [[C3]]
-    ; GFX9: [[SUB:%[0-9]+]]:_(s16) = G_SUB [[C1]], [[SMAX]]
-    ; GFX9: [[SMIN:%[0-9]+]]:_(s16) = G_SMIN [[SHL]], [[C3]]
-    ; GFX9: [[SUB1:%[0-9]+]]:_(s16) = G_SUB [[C2]], [[SMIN]]
-    ; GFX9: [[SMAX1:%[0-9]+]]:_(s16) = G_SMAX [[SUB1]], [[SHL1]]
-    ; GFX9: [[SMIN1:%[0-9]+]]:_(s16) = G_SMIN [[SMAX1]], [[SUB]]
-    ; GFX9: [[ADD:%[0-9]+]]:_(s16) = G_ADD [[SHL]], [[SMIN1]]
-    ; GFX9: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[ADD]], [[C]](s16)
+    ; GFX9: [[SADDSAT:%[0-9]+]]:_(s16) = G_SADDSAT [[SHL]], [[SHL1]]
+    ; GFX9: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[SADDSAT]], [[C]](s16)
     ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ASHR]](s16)
     ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
     %0:_(s32) = COPY $vgpr0
@@ -282,34 +264,19 @@ body: |
     ; GFX9: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
     ; GFX9: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[C3]](s16)
     ; GFX9: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[C3]](s16)
-    ; GFX9: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 32767
-    ; GFX9: [[C5:%[0-9]+]]:_(s16) = G_CONSTANT i16 -32768
-    ; GFX9: [[C6:%[0-9]+]]:_(s16) = G_CONSTANT i16 0
-    ; GFX9: [[SMAX:%[0-9]+]]:_(s16) = G_SMAX [[SHL]], [[C6]]
-    ; GFX9: [[SUB:%[0-9]+]]:_(s16) = G_SUB [[C4]], [[SMAX]]
-    ; GFX9: [[SMIN:%[0-9]+]]:_(s16) = G_SMIN [[SHL]], [[C6]]
-    ; GFX9: [[SUB1:%[0-9]+]]:_(s16) = G_SUB [[C5]], [[SMIN]]
-    ; GFX9: [[SMAX1:%[0-9]+]]:_(s16) = G_SMAX [[SUB1]], [[SHL1]]
-    ; GFX9: [[SMIN1:%[0-9]+]]:_(s16) = G_SMIN [[SMAX1]], [[SUB]]
-    ; GFX9: [[ADD:%[0-9]+]]:_(s16) = G_ADD [[SHL]], [[SMIN1]]
-    ; GFX9: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[ADD]], [[C3]](s16)
+    ; GFX9: [[SADDSAT:%[0-9]+]]:_(s16) = G_SADDSAT [[SHL]], [[SHL1]]
+    ; GFX9: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[SADDSAT]], [[C3]](s16)
     ; GFX9: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
     ; GFX9: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
     ; GFX9: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[TRUNC2]], [[C3]](s16)
     ; GFX9: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[TRUNC3]], [[C3]](s16)
-    ; GFX9: [[SMAX2:%[0-9]+]]:_(s16) = G_SMAX [[SHL2]], [[C6]]
-    ; GFX9: [[SUB2:%[0-9]+]]:_(s16) = G_SUB [[C4]], [[SMAX2]]
-    ; GFX9: [[SMIN2:%[0-9]+]]:_(s16) = G_SMIN [[SHL2]], [[C6]]
-    ; GFX9: [[SUB3:%[0-9]+]]:_(s16) = G_SUB [[C5]], [[SMIN2]]
-    ; GFX9: [[SMAX3:%[0-9]+]]:_(s16) = G_SMAX [[SUB3]], [[SHL3]]
-    ; GFX9: [[SMIN3:%[0-9]+]]:_(s16) = G_SMIN [[SMAX3]], [[SUB2]]
-    ; GFX9: [[ADD1:%[0-9]+]]:_(s16) = G_ADD [[SHL2]], [[SMIN3]]
-    ; GFX9: [[ASHR1:%[0-9]+]]:_(s16) = G_ASHR [[ADD1]], [[C3]](s16)
-    ; GFX9: [[C7:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; GFX9: [[SADDSAT1:%[0-9]+]]:_(s16) = G_SADDSAT [[SHL2]], [[SHL3]]
+    ; GFX9: [[ASHR1:%[0-9]+]]:_(s16) = G_ASHR [[SADDSAT1]], [[C3]](s16)
+    ; GFX9: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
     ; GFX9: [[COPY2:%[0-9]+]]:_(s16) = COPY [[ASHR]](s16)
-    ; GFX9: [[AND:%[0-9]+]]:_(s16) = G_AND [[COPY2]], [[C7]]
+    ; GFX9: [[AND:%[0-9]+]]:_(s16) = G_AND [[COPY2]], [[C4]]
     ; GFX9: [[COPY3:%[0-9]+]]:_(s16) = COPY [[ASHR1]](s16)
-    ; GFX9: [[AND1:%[0-9]+]]:_(s16) = G_AND [[COPY3]], [[C7]]
+    ; GFX9: [[AND1:%[0-9]+]]:_(s16) = G_AND [[COPY3]], [[C4]]
     ; GFX9: [[SHL4:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C3]](s16)
     ; GFX9: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL4]]
     ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
@@ -375,17 +342,8 @@ body: |
     ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
     ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
     ; GFX9: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
-    ; GFX9: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 32767
-    ; GFX9: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 -32768
-    ; GFX9: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 0
-    ; GFX9: [[SMAX:%[0-9]+]]:_(s16) = G_SMAX [[TRUNC]], [[C2]]
-    ; GFX9: [[SUB:%[0-9]+]]:_(s16) = G_SUB [[C]], [[SMAX]]
-    ; GFX9: [[SMIN:%[0-9]+]]:_(s16) = G_SMIN [[TRUNC]], [[C2]]
-    ; GFX9: [[SUB1:%[0-9]+]]:_(s16) = G_SUB [[C1]], [[SMIN]]
-    ; GFX9: [[SMAX1:%[0-9]+]]:_(s16) = G_SMAX [[SUB1]], [[TRUNC1]]
-    ; GFX9: [[SMIN1:%[0-9]+]]:_(s16) = G_SMIN [[SMAX1]], [[SUB]]
-    ; GFX9: [[ADD:%[0-9]+]]:_(s16) = G_ADD [[TRUNC]], [[SMIN1]]
-    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ADD]](s16)
+    ; GFX9: [[SADDSAT:%[0-9]+]]:_(s16) = G_SADDSAT [[TRUNC]], [[TRUNC1]]
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[SADDSAT]](s16)
     ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
@@ -484,20 +442,8 @@ body: |
     ; GFX9-LABEL: name: saddsat_v2s16
     ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
     ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
-    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 32767
-    ; GFX9: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C]](s32), [[C]](s32)
-    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -32768
-    ; GFX9: [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C1]](s32), [[C1]](s32)
-    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX9: [[BUILD_VECTOR_TRUNC2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C2]](s32), [[C2]](s32)
-    ; GFX9: [[SMAX:%[0-9]+]]:_(<2 x s16>) = G_SMAX [[COPY]], [[BUILD_VECTOR_TRUNC2]]
-    ; GFX9: [[SUB:%[0-9]+]]:_(<2 x s16>) = G_SUB [[BUILD_VECTOR_TRUNC]], [[SMAX]]
-    ; GFX9: [[SMIN:%[0-9]+]]:_(<2 x s16>) = G_SMIN [[COPY]], [[BUILD_VECTOR_TRUNC2]]
-    ; GFX9: [[SUB1:%[0-9]+]]:_(<2 x s16>) = G_SUB [[BUILD_VECTOR_TRUNC1]], [[SMIN]]
-    ; GFX9: [[SMAX1:%[0-9]+]]:_(<2 x s16>) = G_SMAX [[SUB1]], [[COPY1]]
-    ; GFX9: [[SMIN1:%[0-9]+]]:_(<2 x s16>) = G_SMIN [[SMAX1]], [[SUB]]
-    ; GFX9: [[ADD:%[0-9]+]]:_(<2 x s16>) = G_ADD [[COPY]], [[SMIN1]]
-    ; GFX9: $vgpr0 = COPY [[ADD]](<2 x s16>)
+    ; GFX9: [[SADDSAT:%[0-9]+]]:_(<2 x s16>) = G_SADDSAT [[COPY]], [[COPY1]]
+    ; GFX9: $vgpr0 = COPY [[SADDSAT]](<2 x s16>)
     %0:_(<2 x s16>) = COPY $vgpr0
     %1:_(<2 x s16>) = COPY $vgpr1
     %2:_(<2 x s16>) = G_SADDSAT %0, %1
@@ -676,30 +622,9 @@ body: |
     ; GFX9: [[BUILD_VECTOR_TRUNC2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY4]](s32), [[COPY5]](s32)
     ; GFX9: [[COPY6:%[0-9]+]]:_(s32) = COPY [[BITCAST3]](s32)
     ; GFX9: [[BUILD_VECTOR_TRUNC3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY6]](s32), [[DEF1]](s32)
-    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 32767
-    ; GFX9: [[BUILD_VECTOR_TRUNC4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C1]](s32), [[C1]](s32)
-    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -32768
-    ; GFX9: [[BUILD_VECTOR_TRUNC5:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C2]](s32), [[C2]](s32)
-    ; GFX9: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX9: [[BUILD_VECTOR_TRUNC6:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C3]](s32), [[C3]](s32)
-    ; GFX9: [[SMAX:%[0-9]+]]:_(<2 x s16>) = G_SMAX [[BUILD_VECTOR_TRUNC]], [[BUILD_VECTOR_TRUNC6]]
-    ; GFX9: [[SUB:%[0-9]+]]:_(<2 x s16>) = G_SUB [[BUILD_VECTOR_TRUNC4]], [[SMAX]]
-    ; GFX9: [[SMIN:%[0-9]+]]:_(<2 x s16>) = G_SMIN [[BUILD_VECTOR_TRUNC]], [[BUILD_VECTOR_TRUNC6]]
-    ; GFX9: [[SUB1:%[0-9]+]]:_(<2 x s16>) = G_SUB [[BUILD_VECTOR_TRUNC5]], [[SMIN]]
-    ; GFX9: [[SMAX1:%[0-9]+]]:_(<2 x s16>) = G_SMAX [[SUB1]], [[BUILD_VECTOR_TRUNC2]]
-    ; GFX9: [[SMIN1:%[0-9]+]]:_(<2 x s16>) = G_SMIN [[SMAX1]], [[SUB]]
-    ; GFX9: [[ADD:%[0-9]+]]:_(<2 x s16>) = G_ADD [[BUILD_VECTOR_TRUNC]], [[SMIN1]]
-    ; GFX9: [[BUILD_VECTOR_TRUNC7:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C1]](s32), [[C1]](s32)
-    ; GFX9: [[BUILD_VECTOR_TRUNC8:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C2]](s32), [[C2]](s32)
-    ; GFX9: [[BUILD_VECTOR_TRUNC9:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C3]](s32), [[C3]](s32)
-    ; GFX9: [[SMAX2:%[0-9]+]]:_(<2 x s16>) = G_SMAX [[BUILD_VECTOR_TRUNC1]], [[BUILD_VECTOR_TRUNC9]]
-    ; GFX9: [[SUB2:%[0-9]+]]:_(<2 x s16>) = G_SUB [[BUILD_VECTOR_TRUNC7]], [[SMAX2]]
-    ; GFX9: [[SMIN2:%[0-9]+]]:_(<2 x s16>) = G_SMIN [[BUILD_VECTOR_TRUNC1]], [[BUILD_VECTOR_TRUNC9]]
-    ; GFX9: [[SUB3:%[0-9]+]]:_(<2 x s16>) = G_SUB [[BUILD_VECTOR_TRUNC8]], [[SMIN2]]
-    ; GFX9: [[SMAX3:%[0-9]+]]:_(<2 x s16>) = G_SMAX [[SUB3]], [[BUILD_VECTOR_TRUNC3]]
-    ; GFX9: [[SMIN3:%[0-9]+]]:_(<2 x s16>) = G_SMIN [[SMAX3]], [[SUB2]]
-    ; GFX9: [[ADD1:%[0-9]+]]:_(<2 x s16>) = G_ADD [[BUILD_VECTOR_TRUNC1]], [[SMIN3]]
-    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[ADD]](<2 x s16>), [[ADD1]](<2 x s16>), [[DEF2]](<2 x s16>)
+    ; GFX9: [[SADDSAT:%[0-9]+]]:_(<2 x s16>) = G_SADDSAT [[BUILD_VECTOR_TRUNC]], [[BUILD_VECTOR_TRUNC2]]
+    ; GFX9: [[SADDSAT1:%[0-9]+]]:_(<2 x s16>) = G_SADDSAT [[BUILD_VECTOR_TRUNC1]], [[BUILD_VECTOR_TRUNC3]]
+    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[SADDSAT]](<2 x s16>), [[SADDSAT1]](<2 x s16>), [[DEF2]](<2 x s16>)
     ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<6 x s16>), 0
     ; GFX9: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
     ; GFX9: [[CONCAT_VECTORS1:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[EXTRACT]](<3 x s16>), [[EXTRACT1]](<3 x s16>)
@@ -870,30 +795,9 @@ body: |
     ; GFX9: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
     ; GFX9: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
     ; GFX9: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
-    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 32767
-    ; GFX9: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C]](s32), [[C]](s32)
-    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -32768
-    ; GFX9: [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C1]](s32), [[C1]](s32)
-    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX9: [[BUILD_VECTOR_TRUNC2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C2]](s32), [[C2]](s32)
-    ; GFX9: [[SMAX:%[0-9]+]]:_(<2 x s16>) = G_SMAX [[UV]], [[BUILD_VECTOR_TRUNC2]]
-    ; GFX9: [[SUB:%[0-9]+]]:_(<2 x s16>) = G_SUB [[BUILD_VECTOR_TRUNC]], [[SMAX]]
-    ; GFX9: [[SMIN:%[0-9]+]]:_(<2 x s16>) = G_SMIN [[UV]], [[BUILD_VECTOR_TRUNC2]]
-    ; GFX9: [[SUB1:%[0-9]+]]:_(<2 x s16>) = G_SUB [[BUILD_VECTOR_TRUNC1]], [[SMIN]]
-    ; GFX9: [[SMAX1:%[0-9]+]]:_(<2 x s16>) = G_SMAX [[SUB1]], [[UV2]]
-    ; GFX9: [[SMIN1:%[0-9]+]]:_(<2 x s16>) = G_SMIN [[SMAX1]], [[SUB]]
-    ; GFX9: [[ADD:%[0-9]+]]:_(<2 x s16>) = G_ADD [[UV]], [[SMIN1]]
-    ; GFX9: [[BUILD_VECTOR_TRUNC3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C]](s32), [[C]](s32)
-    ; GFX9: [[BUILD_VECTOR_TRUNC4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C1]](s32), [[C1]](s32)
-    ; GFX9: [[BUILD_VECTOR_TRUNC5:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C2]](s32), [[C2]](s32)
-    ; GFX9: [[SMAX2:%[0-9]+]]:_(<2 x s16>) = G_SMAX [[UV1]], [[BUILD_VECTOR_TRUNC5]]
-    ; GFX9: [[SUB2:%[0-9]+]]:_(<2 x s16>) = G_SUB [[BUILD_VECTOR_TRUNC3]], [[SMAX2]]
-    ; GFX9: [[SMIN2:%[0-9]+]]:_(<2 x s16>) = G_SMIN [[UV1]], [[BUILD_VECTOR_TRUNC5]]
-    ; GFX9: [[SUB3:%[0-9]+]]:_(<2 x s16>) = G_SUB [[BUILD_VECTOR_TRUNC4]], [[SMIN2]]
-    ; GFX9: [[SMAX3:%[0-9]+]]:_(<2 x s16>) = G_SMAX [[SUB3]], [[UV3]]
-    ; GFX9: [[SMIN3:%[0-9]+]]:_(<2 x s16>) = G_SMIN [[SMAX3]], [[SUB2]]
-    ; GFX9: [[ADD1:%[0-9]+]]:_(<2 x s16>) = G_ADD [[UV1]], [[SMIN3]]
-    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[ADD]](<2 x s16>), [[ADD1]](<2 x s16>)
+    ; GFX9: [[SADDSAT:%[0-9]+]]:_(<2 x s16>) = G_SADDSAT [[UV]], [[UV2]]
+    ; GFX9: [[SADDSAT1:%[0-9]+]]:_(<2 x s16>) = G_SADDSAT [[UV1]], [[UV3]]
+    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[SADDSAT]](<2 x s16>), [[SADDSAT1]](<2 x s16>)
     ; GFX9: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x s16>)
     %0:_(<4 x s16>) = COPY $vgpr0_vgpr1
     %1:_(<4 x s16>) = COPY $vgpr2_vgpr3
@@ -938,17 +842,8 @@ body: |
     ; GFX9-LABEL: name: saddsat_s32
     ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2147483647
-    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
-    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX9: [[SMAX:%[0-9]+]]:_(s32) = G_SMAX [[COPY]], [[C2]]
-    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C]], [[SMAX]]
-    ; GFX9: [[SMIN:%[0-9]+]]:_(s32) = G_SMIN [[COPY]], [[C2]]
-    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[SMIN]]
-    ; GFX9: [[SMAX1:%[0-9]+]]:_(s32) = G_SMAX [[SUB1]], [[COPY1]]
-    ; GFX9: [[SMIN1:%[0-9]+]]:_(s32) = G_SMIN [[SMAX1]], [[SUB]]
-    ; GFX9: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[COPY]], [[SMIN1]]
-    ; GFX9: $vgpr0 = COPY [[ADD]](s32)
+    ; GFX9: [[SADDSAT:%[0-9]+]]:_(s32) = G_SADDSAT [[COPY]], [[COPY1]]
+    ; GFX9: $vgpr0 = COPY [[SADDSAT]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
     %2:_(s32) = G_SADDSAT %0, %1
@@ -1014,24 +909,9 @@ body: |
     ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
     ; GFX9: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
     ; GFX9: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
-    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2147483647
-    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
-    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX9: [[SMAX:%[0-9]+]]:_(s32) = G_SMAX [[UV]], [[C2]]
-    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C]], [[SMAX]]
-    ; GFX9: [[SMIN:%[0-9]+]]:_(s32) = G_SMIN [[UV]], [[C2]]
-    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[SMIN]]
-    ; GFX9: [[SMAX1:%[0-9]+]]:_(s32) = G_SMAX [[SUB1]], [[UV2]]
-    ; GFX9: [[SMIN1:%[0-9]+]]:_(s32) = G_SMIN [[SMAX1]], [[SUB]]
-    ; GFX9: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[UV]], [[SMIN1]]
-    ; GFX9: [[SMAX2:%[0-9]+]]:_(s32) = G_SMAX [[UV1]], [[C2]]
-    ; GFX9: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[C]], [[SMAX2]]
-    ; GFX9: [[SMIN2:%[0-9]+]]:_(s32) = G_SMIN [[UV1]], [[C2]]
-    ; GFX9: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[SMIN2]]
-    ; GFX9: [[SMAX3:%[0-9]+]]:_(s32) = G_SMAX [[SUB3]], [[UV3]]
-    ; GFX9: [[SMIN3:%[0-9]+]]:_(s32) = G_SMIN [[SMAX3]], [[SUB2]]
-    ; GFX9: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[UV1]], [[SMIN3]]
-    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[ADD]](s32), [[ADD1]](s32)
+    ; GFX9: [[SADDSAT:%[0-9]+]]:_(s32) = G_SADDSAT [[UV]], [[UV2]]
+    ; GFX9: [[SADDSAT1:%[0-9]+]]:_(s32) = G_SADDSAT [[UV1]], [[UV3]]
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SADDSAT]](s32), [[SADDSAT1]](s32)
     ; GFX9: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
     %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
     %1:_(<2 x s32>) = COPY $vgpr2_vgpr3
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ssubsat.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ssubsat.mir
index 40eb12034c97b..f38da863cba90 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ssubsat.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ssubsat.mir
@@ -59,17 +59,8 @@ body: |
     ; GFX9: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 9
     ; GFX9: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[C]](s16)
     ; GFX9: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[C]](s16)
-    ; GFX9: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 32767
-    ; GFX9: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 -32768
-    ; GFX9: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
-    ; GFX9: [[SMAX:%[0-9]+]]:_(s16) = G_SMAX [[SHL]], [[C3]]
-    ; GFX9: [[SUB:%[0-9]+]]:_(s16) = G_SUB [[SMAX]], [[C1]]
-    ; GFX9: [[SMIN:%[0-9]+]]:_(s16) = G_SMIN [[SHL]], [[C3]]
-    ; GFX9: [[SUB1:%[0-9]+]]:_(s16) = G_SUB [[SMIN]], [[C2]]
-    ; GFX9: [[SMAX1:%[0-9]+]]:_(s16) = G_SMAX [[SUB]], [[SHL1]]
-    ; GFX9: [[SMIN1:%[0-9]+]]:_(s16) = G_SMIN [[SMAX1]], [[SUB1]]
-    ; GFX9: [[SUB2:%[0-9]+]]:_(s16) = G_SUB [[SHL]], [[SMIN1]]
-    ; GFX9: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[SUB2]], [[C]](s16)
+    ; GFX9: [[SSUBSAT:%[0-9]+]]:_(s16) = G_SSUBSAT [[SHL]], [[SHL1]]
+    ; GFX9: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[SSUBSAT]], [[C]](s16)
     ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ASHR]](s16)
     ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
     %0:_(s32) = COPY $vgpr0
@@ -137,17 +128,8 @@ body: |
     ; GFX9: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
     ; GFX9: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[C]](s16)
     ; GFX9: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[C]](s16)
-    ; GFX9: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 32767
-    ; GFX9: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 -32768
-    ; GFX9: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
-    ; GFX9: [[SMAX:%[0-9]+]]:_(s16) = G_SMAX [[SHL]], [[C3]]
-    ; GFX9: [[SUB:%[0-9]+]]:_(s16) = G_SUB [[SMAX]], [[C1]]
-    ; GFX9: [[SMIN:%[0-9]+]]:_(s16) = G_SMIN [[SHL]], [[C3]]
-    ; GFX9: [[SUB1:%[0-9]+]]:_(s16) = G_SUB [[SMIN]], [[C2]]
-    ; GFX9: [[SMAX1:%[0-9]+]]:_(s16) = G_SMAX [[SUB]], [[SHL1]]
-    ; GFX9: [[SMIN1:%[0-9]+]]:_(s16) = G_SMIN [[SMAX1]], [[SUB1]]
-    ; GFX9: [[SUB2:%[0-9]+]]:_(s16) = G_SUB [[SHL]], [[SMIN1]]
-    ; GFX9: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[SUB2]], [[C]](s16)
+    ; GFX9: [[SSUBSAT:%[0-9]+]]:_(s16) = G_SSUBSAT [[SHL]], [[SHL1]]
+    ; GFX9: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[SSUBSAT]], [[C]](s16)
     ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ASHR]](s16)
     ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
     %0:_(s32) = COPY $vgpr0
@@ -282,34 +264,19 @@ body: |
     ; GFX9: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
     ; GFX9: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[C3]](s16)
     ; GFX9: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[C3]](s16)
-    ; GFX9: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 32767
-    ; GFX9: [[C5:%[0-9]+]]:_(s16) = G_CONSTANT i16 -32768
-    ; GFX9: [[C6:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
-    ; GFX9: [[SMAX:%[0-9]+]]:_(s16) = G_SMAX [[SHL]], [[C6]]
-    ; GFX9: [[SUB:%[0-9]+]]:_(s16) = G_SUB [[SMAX]], [[C4]]
-    ; GFX9: [[SMIN:%[0-9]+]]:_(s16) = G_SMIN [[SHL]], [[C6]]
-    ; GFX9: [[SUB1:%[0-9]+]]:_(s16) = G_SUB [[SMIN]], [[C5]]
-    ; GFX9: [[SMAX1:%[0-9]+]]:_(s16) = G_SMAX [[SUB]], [[SHL1]]
-    ; GFX9: [[SMIN1:%[0-9]+]]:_(s16) = G_SMIN [[SMAX1]], [[SUB1]]
-    ; GFX9: [[SUB2:%[0-9]+]]:_(s16) = G_SUB [[SHL]], [[SMIN1]]
-    ; GFX9: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[SUB2]], [[C3]](s16)
+    ; GFX9: [[SSUBSAT:%[0-9]+]]:_(s16) = G_SSUBSAT [[SHL]], [[SHL1]]
+    ; GFX9: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[SSUBSAT]], [[C3]](s16)
     ; GFX9: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
     ; GFX9: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
     ; GFX9: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[TRUNC2]], [[C3]](s16)
     ; GFX9: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[TRUNC3]], [[C3]](s16)
-    ; GFX9: [[SMAX2:%[0-9]+]]:_(s16) = G_SMAX [[SHL2]], [[C6]]
-    ; GFX9: [[SUB3:%[0-9]+]]:_(s16) = G_SUB [[SMAX2]], [[C4]]
-    ; GFX9: [[SMIN2:%[0-9]+]]:_(s16) = G_SMIN [[SHL2]], [[C6]]
-    ; GFX9: [[SUB4:%[0-9]+]]:_(s16) = G_SUB [[SMIN2]], [[C5]]
-    ; GFX9: [[SMAX3:%[0-9]+]]:_(s16) = G_SMAX [[SUB3]], [[SHL3]]
-    ; GFX9: [[SMIN3:%[0-9]+]]:_(s16) = G_SMIN [[SMAX3]], [[SUB4]]
-    ; GFX9: [[SUB5:%[0-9]+]]:_(s16) = G_SUB [[SHL2]], [[SMIN3]]
-    ; GFX9: [[ASHR1:%[0-9]+]]:_(s16) = G_ASHR [[SUB5]], [[C3]](s16)
-    ; GFX9: [[C7:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; GFX9: [[SSUBSAT1:%[0-9]+]]:_(s16) = G_SSUBSAT [[SHL2]], [[SHL3]]
+    ; GFX9: [[ASHR1:%[0-9]+]]:_(s16) = G_ASHR [[SSUBSAT1]], [[C3]](s16)
+    ; GFX9: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
     ; GFX9: [[COPY2:%[0-9]+]]:_(s16) = COPY [[ASHR]](s16)
-    ; GFX9: [[AND:%[0-9]+]]:_(s16) = G_AND [[COPY2]], [[C7]]
+    ; GFX9: [[AND:%[0-9]+]]:_(s16) = G_AND [[COPY2]], [[C4]]
     ; GFX9: [[COPY3:%[0-9]+]]:_(s16) = COPY [[ASHR1]](s16)
-    ; GFX9: [[AND1:%[0-9]+]]:_(s16) = G_AND [[COPY3]], [[C7]]
+    ; GFX9: [[AND1:%[0-9]+]]:_(s16) = G_AND [[COPY3]], [[C4]]
     ; GFX9: [[SHL4:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C3]](s16)
     ; GFX9: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL4]]
     ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
@@ -375,17 +342,8 @@ body: |
     ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
     ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
     ; GFX9: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
-    ; GFX9: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 32767
-    ; GFX9: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 -32768
-    ; GFX9: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
-    ; GFX9: [[SMAX:%[0-9]+]]:_(s16) = G_SMAX [[TRUNC]], [[C2]]
-    ; GFX9: [[SUB:%[0-9]+]]:_(s16) = G_SUB [[SMAX]], [[C]]
-    ; GFX9: [[SMIN:%[0-9]+]]:_(s16) = G_SMIN [[TRUNC]], [[C2]]
-    ; GFX9: [[SUB1:%[0-9]+]]:_(s16) = G_SUB [[SMIN]], [[C1]]
-    ; GFX9: [[SMAX1:%[0-9]+]]:_(s16) = G_SMAX [[SUB]], [[TRUNC1]]
-    ; GFX9: [[SMIN1:%[0-9]+]]:_(s16) = G_SMIN [[SMAX1]], [[SUB1]]
-    ; GFX9: [[SUB2:%[0-9]+]]:_(s16) = G_SUB [[TRUNC]], [[SMIN1]]
-    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[SUB2]](s16)
+    ; GFX9: [[SSUBSAT:%[0-9]+]]:_(s16) = G_SSUBSAT [[TRUNC]], [[TRUNC1]]
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[SSUBSAT]](s16)
     ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
@@ -484,20 +442,8 @@ body: |
     ; GFX9-LABEL: name: ssubsat_v2s16
     ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
     ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
-    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 32767
-    ; GFX9: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C]](s32), [[C]](s32)
-    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -32768
-    ; GFX9: [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C1]](s32), [[C1]](s32)
-    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
-    ; GFX9: [[BUILD_VECTOR_TRUNC2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C2]](s32), [[C2]](s32)
-    ; GFX9: [[SMAX:%[0-9]+]]:_(<2 x s16>) = G_SMAX [[COPY]], [[BUILD_VECTOR_TRUNC2]]
-    ; GFX9: [[SUB:%[0-9]+]]:_(<2 x s16>) = G_SUB [[SMAX]], [[BUILD_VECTOR_TRUNC]]
-    ; GFX9: [[SMIN:%[0-9]+]]:_(<2 x s16>) = G_SMIN [[COPY]], [[BUILD_VECTOR_TRUNC2]]
-    ; GFX9: [[SUB1:%[0-9]+]]:_(<2 x s16>) = G_SUB [[SMIN]], [[BUILD_VECTOR_TRUNC1]]
-    ; GFX9: [[SMAX1:%[0-9]+]]:_(<2 x s16>) = G_SMAX [[SUB]], [[COPY1]]
-    ; GFX9: [[SMIN1:%[0-9]+]]:_(<2 x s16>) = G_SMIN [[SMAX1]], [[SUB1]]
-    ; GFX9: [[SUB2:%[0-9]+]]:_(<2 x s16>) = G_SUB [[COPY]], [[SMIN1]]
-    ; GFX9: $vgpr0 = COPY [[SUB2]](<2 x s16>)
+    ; GFX9: [[SSUBSAT:%[0-9]+]]:_(<2 x s16>) = G_SSUBSAT [[COPY]], [[COPY1]]
+    ; GFX9: $vgpr0 = COPY [[SSUBSAT]](<2 x s16>)
     %0:_(<2 x s16>) = COPY $vgpr0
     %1:_(<2 x s16>) = COPY $vgpr1
     %2:_(<2 x s16>) = G_SSUBSAT %0, %1
@@ -676,30 +622,9 @@ body: |
     ; GFX9: [[BUILD_VECTOR_TRUNC2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY4]](s32), [[COPY5]](s32)
     ; GFX9: [[COPY6:%[0-9]+]]:_(s32) = COPY [[BITCAST3]](s32)
     ; GFX9: [[BUILD_VECTOR_TRUNC3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY6]](s32), [[DEF1]](s32)
-    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 32767
-    ; GFX9: [[BUILD_VECTOR_TRUNC4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C1]](s32), [[C1]](s32)
-    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -32768
-    ; GFX9: [[BUILD_VECTOR_TRUNC5:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C2]](s32), [[C2]](s32)
-    ; GFX9: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
-    ; GFX9: [[BUILD_VECTOR_TRUNC6:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C3]](s32), [[C3]](s32)
-    ; GFX9: [[SMAX:%[0-9]+]]:_(<2 x s16>) = G_SMAX [[BUILD_VECTOR_TRUNC]], [[BUILD_VECTOR_TRUNC6]]
-    ; GFX9: [[SUB:%[0-9]+]]:_(<2 x s16>) = G_SUB [[SMAX]], [[BUILD_VECTOR_TRUNC4]]
-    ; GFX9: [[SMIN:%[0-9]+]]:_(<2 x s16>) = G_SMIN [[BUILD_VECTOR_TRUNC]], [[BUILD_VECTOR_TRUNC6]]
-    ; GFX9: [[SUB1:%[0-9]+]]:_(<2 x s16>) = G_SUB [[SMIN]], [[BUILD_VECTOR_TRUNC5]]
-    ; GFX9: [[SMAX1:%[0-9]+]]:_(<2 x s16>) = G_SMAX [[SUB]], [[BUILD_VECTOR_TRUNC2]]
-    ; GFX9: [[SMIN1:%[0-9]+]]:_(<2 x s16>) = G_SMIN [[SMAX1]], [[SUB1]]
-    ; GFX9: [[SUB2:%[0-9]+]]:_(<2 x s16>) = G_SUB [[BUILD_VECTOR_TRUNC]], [[SMIN1]]
-    ; GFX9: [[BUILD_VECTOR_TRUNC7:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C1]](s32), [[C1]](s32)
-    ; GFX9: [[BUILD_VECTOR_TRUNC8:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C2]](s32), [[C2]](s32)
-    ; GFX9: [[BUILD_VECTOR_TRUNC9:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C3]](s32), [[C3]](s32)
-    ; GFX9: [[SMAX2:%[0-9]+]]:_(<2 x s16>) = G_SMAX [[BUILD_VECTOR_TRUNC1]], [[BUILD_VECTOR_TRUNC9]]
-    ; GFX9: [[SUB3:%[0-9]+]]:_(<2 x s16>) = G_SUB [[SMAX2]], [[BUILD_VECTOR_TRUNC7]]
-    ; GFX9: [[SMIN2:%[0-9]+]]:_(<2 x s16>) = G_SMIN [[BUILD_VECTOR_TRUNC1]], [[BUILD_VECTOR_TRUNC9]]
-    ; GFX9: [[SUB4:%[0-9]+]]:_(<2 x s16>) = G_SUB [[SMIN2]], [[BUILD_VECTOR_TRUNC8]]
-    ; GFX9: [[SMAX3:%[0-9]+]]:_(<2 x s16>) = G_SMAX [[SUB3]], [[BUILD_VECTOR_TRUNC3]]
-    ; GFX9: [[SMIN3:%[0-9]+]]:_(<2 x s16>) = G_SMIN [[SMAX3]], [[SUB4]]
-    ; GFX9: [[SUB5:%[0-9]+]]:_(<2 x s16>) = G_SUB [[BUILD_VECTOR_TRUNC1]], [[SMIN3]]
-    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[SUB2]](<2 x s16>), [[SUB5]](<2 x s16>), [[DEF2]](<2 x s16>)
+    ; GFX9: [[SSUBSAT:%[0-9]+]]:_(<2 x s16>) = G_SSUBSAT [[BUILD_VECTOR_TRUNC]], [[BUILD_VECTOR_TRUNC2]]
+    ; GFX9: [[SSUBSAT1:%[0-9]+]]:_(<2 x s16>) = G_SSUBSAT [[BUILD_VECTOR_TRUNC1]], [[BUILD_VECTOR_TRUNC3]]
+    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[SSUBSAT]](<2 x s16>), [[SSUBSAT1]](<2 x s16>), [[DEF2]](<2 x s16>)
     ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<6 x s16>), 0
     ; GFX9: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
     ; GFX9: [[CONCAT_VECTORS1:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[EXTRACT]](<3 x s16>), [[EXTRACT1]](<3 x s16>)
@@ -870,30 +795,9 @@ body: |
     ; GFX9: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
     ; GFX9: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
     ; GFX9: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
-    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 32767
-    ; GFX9: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C]](s32), [[C]](s32)
-    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -32768
-    ; GFX9: [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C1]](s32), [[C1]](s32)
-    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
-    ; GFX9: [[BUILD_VECTOR_TRUNC2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C2]](s32), [[C2]](s32)
-    ; GFX9: [[SMAX:%[0-9]+]]:_(<2 x s16>) = G_SMAX [[UV]], [[BUILD_VECTOR_TRUNC2]]
-    ; GFX9: [[SUB:%[0-9]+]]:_(<2 x s16>) = G_SUB [[SMAX]], [[BUILD_VECTOR_TRUNC]]
-    ; GFX9: [[SMIN:%[0-9]+]]:_(<2 x s16>) = G_SMIN [[UV]], [[BUILD_VECTOR_TRUNC2]]
-    ; GFX9: [[SUB1:%[0-9]+]]:_(<2 x s16>) = G_SUB [[SMIN]], [[BUILD_VECTOR_TRUNC1]]
-    ; GFX9: [[SMAX1:%[0-9]+]]:_(<2 x s16>) = G_SMAX [[SUB]], [[UV2]]
-    ; GFX9: [[SMIN1:%[0-9]+]]:_(<2 x s16>) = G_SMIN [[SMAX1]], [[SUB1]]
-    ; GFX9: [[SUB2:%[0-9]+]]:_(<2 x s16>) = G_SUB [[UV]], [[SMIN1]]
-    ; GFX9: [[BUILD_VECTOR_TRUNC3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C]](s32), [[C]](s32)
-    ; GFX9: [[BUILD_VECTOR_TRUNC4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C1]](s32), [[C1]](s32)
-    ; GFX9: [[BUILD_VECTOR_TRUNC5:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C2]](s32), [[C2]](s32)
-    ; GFX9: [[SMAX2:%[0-9]+]]:_(<2 x s16>) = G_SMAX [[UV1]], [[BUILD_VECTOR_TRUNC5]]
-    ; GFX9: [[SUB3:%[0-9]+]]:_(<2 x s16>) = G_SUB [[SMAX2]], [[BUILD_VECTOR_TRUNC3]]
-    ; GFX9: [[SMIN2:%[0-9]+]]:_(<2 x s16>) = G_SMIN [[UV1]], [[BUILD_VECTOR_TRUNC5]]
-    ; GFX9: [[SUB4:%[0-9]+]]:_(<2 x s16>) = G_SUB [[SMIN2]], [[BUILD_VECTOR_TRUNC4]]
-    ; GFX9: [[SMAX3:%[0-9]+]]:_(<2 x s16>) = G_SMAX [[SUB3]], [[UV3]]
-    ; GFX9: [[SMIN3:%[0-9]+]]:_(<2 x s16>) = G_SMIN [[SMAX3]], [[SUB4]]
-    ; GFX9: [[SUB5:%[0-9]+]]:_(<2 x s16>) = G_SUB [[UV1]], [[SMIN3]]
-    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[SUB2]](<2 x s16>), [[SUB5]](<2 x s16>)
+    ; GFX9: [[SSUBSAT:%[0-9]+]]:_(<2 x s16>) = G_SSUBSAT [[UV]], [[UV2]]
+    ; GFX9: [[SSUBSAT1:%[0-9]+]]:_(<2 x s16>) = G_SSUBSAT [[UV1]], [[UV3]]
+    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[SSUBSAT]](<2 x s16>), [[SSUBSAT1]](<2 x s16>)
     ; GFX9: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x s16>)
     %0:_(<4 x s16>) = COPY $vgpr0_vgpr1
     %1:_(<4 x s16>) = COPY $vgpr2_vgpr3
@@ -938,17 +842,8 @@ body: |
     ; GFX9-LABEL: name: ssubsat_s32
     ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2147483647
-    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
-    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
-    ; GFX9: [[SMAX:%[0-9]+]]:_(s32) = G_SMAX [[COPY]], [[C2]]
-    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[SMAX]], [[C]]
-    ; GFX9: [[SMIN:%[0-9]+]]:_(s32) = G_SMIN [[COPY]], [[C2]]
-    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[SMIN]], [[C1]]
-    ; GFX9: [[SMAX1:%[0-9]+]]:_(s32) = G_SMAX [[SUB]], [[COPY1]]
-    ; GFX9: [[SMIN1:%[0-9]+]]:_(s32) = G_SMIN [[SMAX1]], [[SUB1]]
-    ; GFX9: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[COPY]], [[SMIN1]]
-    ; GFX9: $vgpr0 = COPY [[SUB2]](s32)
+    ; GFX9: [[SSUBSAT:%[0-9]+]]:_(s32) = G_SSUBSAT [[COPY]], [[COPY1]]
+    ; GFX9: $vgpr0 = COPY [[SSUBSAT]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
     %2:_(s32) = G_SSUBSAT %0, %1
@@ -1014,24 +909,9 @@ body: |
     ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
     ; GFX9: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
     ; GFX9: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
-    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2147483647
-    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
-    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
-    ; GFX9: [[SMAX:%[0-9]+]]:_(s32) = G_SMAX [[UV]], [[C2]]
-    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[SMAX]], [[C]]
-    ; GFX9: [[SMIN:%[0-9]+]]:_(s32) = G_SMIN [[UV]], [[C2]]
-    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[SMIN]], [[C1]]
-    ; GFX9: [[SMAX1:%[0-9]+]]:_(s32) = G_SMAX [[SUB]], [[UV2]]
-    ; GFX9: [[SMIN1:%[0-9]+]]:_(s32) = G_SMIN [[SMAX1]], [[SUB1]]
-    ; GFX9: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[UV]], [[SMIN1]]
-    ; GFX9: [[SMAX2:%[0-9]+]]:_(s32) = G_SMAX [[UV1]], [[C2]]
-    ; GFX9: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SMAX2]], [[C]]
-    ; GFX9: [[SMIN2:%[0-9]+]]:_(s32) = G_SMIN [[UV1]], [[C2]]
-    ; GFX9: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[SMIN2]], [[C1]]
-    ; GFX9: [[SMAX3:%[0-9]+]]:_(s32) = G_SMAX [[SUB3]], [[UV3]]
-    ; GFX9: [[SMIN3:%[0-9]+]]:_(s32) = G_SMIN [[SMAX3]], [[SUB4]]
-    ; GFX9: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[UV1]], [[SMIN3]]
-    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SUB2]](s32), [[SUB5]](s32)
+    ; GFX9: [[SSUBSAT:%[0-9]+]]:_(s32) = G_SSUBSAT [[UV]], [[UV2]]
+    ; GFX9: [[SSUBSAT1:%[0-9]+]]:_(s32) = G_SSUBSAT [[UV1]], [[UV3]]
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SSUBSAT]](s32), [[SSUBSAT1]](s32)
     ; GFX9: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
     %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
     %1:_(<2 x s32>) = COPY $vgpr2_vgpr3
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-uaddsat.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-uaddsat.mir
index 690bf34482ddd..9d51870b4fed2 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-uaddsat.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-uaddsat.mir
@@ -32,11 +32,8 @@ body: |
     ; GFX8: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 9
     ; GFX8: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[C]](s16)
     ; GFX8: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[C]](s16)
-    ; GFX8: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
-    ; GFX8: [[XOR:%[0-9]+]]:_(s16) = G_XOR [[SHL]], [[C1]]
-    ; GFX8: [[UMIN:%[0-9]+]]:_(s16) = G_UMIN [[XOR]], [[SHL1]]
-    ; GFX8: [[ADD:%[0-9]+]]:_(s16) = G_ADD [[SHL]], [[UMIN]]
-    ; GFX8: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[ADD]], [[C]](s16)
+    ; GFX8: [[UADDSAT:%[0-9]+]]:_(s16) = G_UADDSAT [[SHL]], [[SHL1]]
+    ; GFX8: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[UADDSAT]], [[C]](s16)
     ; GFX8: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR]](s16)
     ; GFX8: $vgpr0 = COPY [[ANYEXT]](s32)
     ; GFX9-LABEL: name: uaddsat_s7
@@ -47,11 +44,8 @@ body: |
     ; GFX9: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 9
     ; GFX9: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[C]](s16)
     ; GFX9: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[C]](s16)
-    ; GFX9: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
-    ; GFX9: [[XOR:%[0-9]+]]:_(s16) = G_XOR [[SHL]], [[C1]]
-    ; GFX9: [[UMIN:%[0-9]+]]:_(s16) = G_UMIN [[XOR]], [[SHL1]]
-    ; GFX9: [[ADD:%[0-9]+]]:_(s16) = G_ADD [[SHL]], [[UMIN]]
-    ; GFX9: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[ADD]], [[C]](s16)
+    ; GFX9: [[UADDSAT:%[0-9]+]]:_(s16) = G_UADDSAT [[SHL]], [[SHL1]]
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[UADDSAT]], [[C]](s16)
     ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR]](s16)
     ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
     %0:_(s32) = COPY $vgpr0
@@ -92,11 +86,8 @@ body: |
     ; GFX8: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
     ; GFX8: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[C]](s16)
     ; GFX8: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[C]](s16)
-    ; GFX8: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
-    ; GFX8: [[XOR:%[0-9]+]]:_(s16) = G_XOR [[SHL]], [[C1]]
-    ; GFX8: [[UMIN:%[0-9]+]]:_(s16) = G_UMIN [[XOR]], [[SHL1]]
-    ; GFX8: [[ADD:%[0-9]+]]:_(s16) = G_ADD [[SHL]], [[UMIN]]
-    ; GFX8: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[ADD]], [[C]](s16)
+    ; GFX8: [[UADDSAT:%[0-9]+]]:_(s16) = G_UADDSAT [[SHL]], [[SHL1]]
+    ; GFX8: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[UADDSAT]], [[C]](s16)
     ; GFX8: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR]](s16)
     ; GFX8: $vgpr0 = COPY [[ANYEXT]](s32)
     ; GFX9-LABEL: name: uaddsat_s8
@@ -107,11 +98,8 @@ body: |
     ; GFX9: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
     ; GFX9: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[C]](s16)
     ; GFX9: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[C]](s16)
-    ; GFX9: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
-    ; GFX9: [[XOR:%[0-9]+]]:_(s16) = G_XOR [[SHL]], [[C1]]
-    ; GFX9: [[UMIN:%[0-9]+]]:_(s16) = G_UMIN [[XOR]], [[SHL1]]
-    ; GFX9: [[ADD:%[0-9]+]]:_(s16) = G_ADD [[SHL]], [[UMIN]]
-    ; GFX9: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[ADD]], [[C]](s16)
+    ; GFX9: [[UADDSAT:%[0-9]+]]:_(s16) = G_UADDSAT [[SHL]], [[SHL1]]
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[UADDSAT]], [[C]](s16)
     ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR]](s16)
     ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
     %0:_(s32) = COPY $vgpr0
@@ -187,24 +175,19 @@ body: |
     ; GFX8: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
     ; GFX8: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[C3]](s16)
     ; GFX8: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[C3]](s16)
-    ; GFX8: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
-    ; GFX8: [[XOR:%[0-9]+]]:_(s16) = G_XOR [[SHL]], [[C4]]
-    ; GFX8: [[UMIN:%[0-9]+]]:_(s16) = G_UMIN [[XOR]], [[SHL1]]
-    ; GFX8: [[ADD:%[0-9]+]]:_(s16) = G_ADD [[SHL]], [[UMIN]]
-    ; GFX8: [[LSHR6:%[0-9]+]]:_(s16) = G_LSHR [[ADD]], [[C3]](s16)
+    ; GFX8: [[UADDSAT:%[0-9]+]]:_(s16) = G_UADDSAT [[SHL]], [[SHL1]]
+    ; GFX8: [[LSHR6:%[0-9]+]]:_(s16) = G_LSHR [[UADDSAT]], [[C3]](s16)
     ; GFX8: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
     ; GFX8: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
     ; GFX8: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[TRUNC2]], [[C3]](s16)
     ; GFX8: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[TRUNC3]], [[C3]](s16)
-    ; GFX8: [[XOR1:%[0-9]+]]:_(s16) = G_XOR [[SHL2]], [[C4]]
-    ; GFX8: [[UMIN1:%[0-9]+]]:_(s16) = G_UMIN [[XOR1]], [[SHL3]]
-    ; GFX8: [[ADD1:%[0-9]+]]:_(s16) = G_ADD [[SHL2]], [[UMIN1]]
-    ; GFX8: [[LSHR7:%[0-9]+]]:_(s16) = G_LSHR [[ADD1]], [[C3]](s16)
-    ; GFX8: [[C5:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; GFX8: [[UADDSAT1:%[0-9]+]]:_(s16) = G_UADDSAT [[SHL2]], [[SHL3]]
+    ; GFX8: [[LSHR7:%[0-9]+]]:_(s16) = G_LSHR [[UADDSAT1]], [[C3]](s16)
+    ; GFX8: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
     ; GFX8: [[COPY2:%[0-9]+]]:_(s16) = COPY [[LSHR6]](s16)
-    ; GFX8: [[AND:%[0-9]+]]:_(s16) = G_AND [[COPY2]], [[C5]]
+    ; GFX8: [[AND:%[0-9]+]]:_(s16) = G_AND [[COPY2]], [[C4]]
     ; GFX8: [[COPY3:%[0-9]+]]:_(s16) = COPY [[LSHR7]](s16)
-    ; GFX8: [[AND1:%[0-9]+]]:_(s16) = G_AND [[COPY3]], [[C5]]
+    ; GFX8: [[AND1:%[0-9]+]]:_(s16) = G_AND [[COPY3]], [[C4]]
     ; GFX8: [[SHL4:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C3]](s16)
     ; GFX8: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL4]]
     ; GFX8: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
@@ -226,24 +209,19 @@ body: |
     ; GFX9: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
     ; GFX9: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[C3]](s16)
     ; GFX9: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[C3]](s16)
-    ; GFX9: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
-    ; GFX9: [[XOR:%[0-9]+]]:_(s16) = G_XOR [[SHL]], [[C4]]
-    ; GFX9: [[UMIN:%[0-9]+]]:_(s16) = G_UMIN [[XOR]], [[SHL1]]
-    ; GFX9: [[ADD:%[0-9]+]]:_(s16) = G_ADD [[SHL]], [[UMIN]]
-    ; GFX9: [[LSHR6:%[0-9]+]]:_(s16) = G_LSHR [[ADD]], [[C3]](s16)
+    ; GFX9: [[UADDSAT:%[0-9]+]]:_(s16) = G_UADDSAT [[SHL]], [[SHL1]]
+    ; GFX9: [[LSHR6:%[0-9]+]]:_(s16) = G_LSHR [[UADDSAT]], [[C3]](s16)
     ; GFX9: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
     ; GFX9: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
     ; GFX9: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[TRUNC2]], [[C3]](s16)
     ; GFX9: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[TRUNC3]], [[C3]](s16)
-    ; GFX9: [[XOR1:%[0-9]+]]:_(s16) = G_XOR [[SHL2]], [[C4]]
-    ; GFX9: [[UMIN1:%[0-9]+]]:_(s16) = G_UMIN [[XOR1]], [[SHL3]]
-    ; GFX9: [[ADD1:%[0-9]+]]:_(s16) = G_ADD [[SHL2]], [[UMIN1]]
-    ; GFX9: [[LSHR7:%[0-9]+]]:_(s16) = G_LSHR [[ADD1]], [[C3]](s16)
-    ; GFX9: [[C5:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; GFX9: [[UADDSAT1:%[0-9]+]]:_(s16) = G_UADDSAT [[SHL2]], [[SHL3]]
+    ; GFX9: [[LSHR7:%[0-9]+]]:_(s16) = G_LSHR [[UADDSAT1]], [[C3]](s16)
+    ; GFX9: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
     ; GFX9: [[COPY2:%[0-9]+]]:_(s16) = COPY [[LSHR6]](s16)
-    ; GFX9: [[AND:%[0-9]+]]:_(s16) = G_AND [[COPY2]], [[C5]]
+    ; GFX9: [[AND:%[0-9]+]]:_(s16) = G_AND [[COPY2]], [[C4]]
     ; GFX9: [[COPY3:%[0-9]+]]:_(s16) = COPY [[LSHR7]](s16)
-    ; GFX9: [[AND1:%[0-9]+]]:_(s16) = G_AND [[COPY3]], [[C5]]
+    ; GFX9: [[AND1:%[0-9]+]]:_(s16) = G_AND [[COPY3]], [[C4]]
     ; GFX9: [[SHL4:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C3]](s16)
     ; GFX9: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL4]]
     ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
@@ -286,22 +264,16 @@ body: |
     ; GFX8: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
     ; GFX8: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
     ; GFX8: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
-    ; GFX8: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
-    ; GFX8: [[XOR:%[0-9]+]]:_(s16) = G_XOR [[TRUNC]], [[C]]
-    ; GFX8: [[UMIN:%[0-9]+]]:_(s16) = G_UMIN [[XOR]], [[TRUNC1]]
-    ; GFX8: [[ADD:%[0-9]+]]:_(s16) = G_ADD [[TRUNC]], [[UMIN]]
-    ; GFX8: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ADD]](s16)
+    ; GFX8: [[UADDSAT:%[0-9]+]]:_(s16) = G_UADDSAT [[TRUNC]], [[TRUNC1]]
+    ; GFX8: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[UADDSAT]](s16)
     ; GFX8: $vgpr0 = COPY [[ANYEXT]](s32)
     ; GFX9-LABEL: name: uaddsat_s16
     ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
     ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
     ; GFX9: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
-    ; GFX9: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
-    ; GFX9: [[XOR:%[0-9]+]]:_(s16) = G_XOR [[TRUNC]], [[C]]
-    ; GFX9: [[UMIN:%[0-9]+]]:_(s16) = G_UMIN [[XOR]], [[TRUNC1]]
-    ; GFX9: [[ADD:%[0-9]+]]:_(s16) = G_ADD [[TRUNC]], [[UMIN]]
-    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ADD]](s16)
+    ; GFX9: [[UADDSAT:%[0-9]+]]:_(s16) = G_UADDSAT [[TRUNC]], [[TRUNC1]]
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[UADDSAT]](s16)
     ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
@@ -364,15 +336,10 @@ body: |
     ; GFX8: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
     ; GFX8: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
     ; GFX8: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
-    ; GFX8: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
-    ; GFX8: [[XOR:%[0-9]+]]:_(s16) = G_XOR [[TRUNC]], [[C1]]
-    ; GFX8: [[UMIN:%[0-9]+]]:_(s16) = G_UMIN [[XOR]], [[TRUNC2]]
-    ; GFX8: [[ADD:%[0-9]+]]:_(s16) = G_ADD [[TRUNC]], [[UMIN]]
-    ; GFX8: [[XOR1:%[0-9]+]]:_(s16) = G_XOR [[TRUNC1]], [[C1]]
-    ; GFX8: [[UMIN1:%[0-9]+]]:_(s16) = G_UMIN [[XOR1]], [[TRUNC3]]
-    ; GFX8: [[ADD1:%[0-9]+]]:_(s16) = G_ADD [[TRUNC1]], [[UMIN1]]
-    ; GFX8: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ADD]](s16)
-    ; GFX8: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[ADD1]](s16)
+    ; GFX8: [[UADDSAT:%[0-9]+]]:_(s16) = G_UADDSAT [[TRUNC]], [[TRUNC2]]
+    ; GFX8: [[UADDSAT1:%[0-9]+]]:_(s16) = G_UADDSAT [[TRUNC1]], [[TRUNC3]]
+    ; GFX8: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDSAT]](s16)
+    ; GFX8: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UADDSAT1]](s16)
     ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
     ; GFX8: [[OR:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL]]
     ; GFX8: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
@@ -380,12 +347,8 @@ body: |
     ; GFX9-LABEL: name: uaddsat_v2s16
     ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
     ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
-    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
-    ; GFX9: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C]](s32), [[C]](s32)
-    ; GFX9: [[XOR:%[0-9]+]]:_(<2 x s16>) = G_XOR [[COPY]], [[BUILD_VECTOR_TRUNC]]
-    ; GFX9: [[UMIN:%[0-9]+]]:_(<2 x s16>) = G_UMIN [[XOR]], [[COPY1]]
-    ; GFX9: [[ADD:%[0-9]+]]:_(<2 x s16>) = G_ADD [[COPY]], [[UMIN]]
-    ; GFX9: $vgpr0 = COPY [[ADD]](<2 x s16>)
+    ; GFX9: [[UADDSAT:%[0-9]+]]:_(<2 x s16>) = G_UADDSAT [[COPY]], [[COPY1]]
+    ; GFX9: $vgpr0 = COPY [[UADDSAT]](<2 x s16>)
     %0:_(<2 x s16>) = COPY $vgpr0
     %1:_(<2 x s16>) = COPY $vgpr1
     %2:_(<2 x s16>) = G_UADDSAT %0, %1
@@ -482,24 +445,17 @@ body: |
     ; GFX8: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
     ; GFX8: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
     ; GFX8: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
-    ; GFX8: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
-    ; GFX8: [[XOR:%[0-9]+]]:_(s16) = G_XOR [[TRUNC]], [[C1]]
-    ; GFX8: [[UMIN:%[0-9]+]]:_(s16) = G_UMIN [[XOR]], [[TRUNC3]]
-    ; GFX8: [[ADD:%[0-9]+]]:_(s16) = G_ADD [[TRUNC]], [[UMIN]]
-    ; GFX8: [[XOR1:%[0-9]+]]:_(s16) = G_XOR [[TRUNC1]], [[C1]]
-    ; GFX8: [[UMIN1:%[0-9]+]]:_(s16) = G_UMIN [[XOR1]], [[TRUNC4]]
-    ; GFX8: [[ADD1:%[0-9]+]]:_(s16) = G_ADD [[TRUNC1]], [[UMIN1]]
-    ; GFX8: [[XOR2:%[0-9]+]]:_(s16) = G_XOR [[TRUNC2]], [[C1]]
-    ; GFX8: [[UMIN2:%[0-9]+]]:_(s16) = G_UMIN [[XOR2]], [[TRUNC5]]
-    ; GFX8: [[ADD2:%[0-9]+]]:_(s16) = G_ADD [[TRUNC2]], [[UMIN2]]
-    ; GFX8: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ADD]](s16)
-    ; GFX8: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[ADD1]](s16)
+    ; GFX8: [[UADDSAT:%[0-9]+]]:_(s16) = G_UADDSAT [[TRUNC]], [[TRUNC3]]
+    ; GFX8: [[UADDSAT1:%[0-9]+]]:_(s16) = G_UADDSAT [[TRUNC1]], [[TRUNC4]]
+    ; GFX8: [[UADDSAT2:%[0-9]+]]:_(s16) = G_UADDSAT [[TRUNC2]], [[TRUNC5]]
+    ; GFX8: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDSAT]](s16)
+    ; GFX8: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UADDSAT1]](s16)
     ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
     ; GFX8: [[OR:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL]]
     ; GFX8: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
-    ; GFX8: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[ADD2]](s16)
-    ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GFX8: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C]](s32)
+    ; GFX8: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[UADDSAT2]](s16)
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX8: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C]](s32)
     ; GFX8: [[OR1:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[SHL1]]
     ; GFX8: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
     ; GFX8: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -536,16 +492,9 @@ body: |
     ; GFX9: [[BUILD_VECTOR_TRUNC2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY4]](s32), [[COPY5]](s32)
     ; GFX9: [[COPY6:%[0-9]+]]:_(s32) = COPY [[BITCAST3]](s32)
     ; GFX9: [[BUILD_VECTOR_TRUNC3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY6]](s32), [[DEF1]](s32)
-    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
-    ; GFX9: [[BUILD_VECTOR_TRUNC4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C1]](s32), [[C1]](s32)
-    ; GFX9: [[XOR:%[0-9]+]]:_(<2 x s16>) = G_XOR [[BUILD_VECTOR_TRUNC]], [[BUILD_VECTOR_TRUNC4]]
-    ; GFX9: [[UMIN:%[0-9]+]]:_(<2 x s16>) = G_UMIN [[XOR]], [[BUILD_VECTOR_TRUNC2]]
-    ; GFX9: [[ADD:%[0-9]+]]:_(<2 x s16>) = G_ADD [[BUILD_VECTOR_TRUNC]], [[UMIN]]
-    ; GFX9: [[BUILD_VECTOR_TRUNC5:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C1]](s32), [[C1]](s32)
-    ; GFX9: [[XOR1:%[0-9]+]]:_(<2 x s16>) = G_XOR [[BUILD_VECTOR_TRUNC1]], [[BUILD_VECTOR_TRUNC5]]
-    ; GFX9: [[UMIN1:%[0-9]+]]:_(<2 x s16>) = G_UMIN [[XOR1]], [[BUILD_VECTOR_TRUNC3]]
-    ; GFX9: [[ADD1:%[0-9]+]]:_(<2 x s16>) = G_ADD [[BUILD_VECTOR_TRUNC1]], [[UMIN1]]
-    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[ADD]](<2 x s16>), [[ADD1]](<2 x s16>), [[DEF2]](<2 x s16>)
+    ; GFX9: [[UADDSAT:%[0-9]+]]:_(<2 x s16>) = G_UADDSAT [[BUILD_VECTOR_TRUNC]], [[BUILD_VECTOR_TRUNC2]]
+    ; GFX9: [[UADDSAT1:%[0-9]+]]:_(<2 x s16>) = G_UADDSAT [[BUILD_VECTOR_TRUNC1]], [[BUILD_VECTOR_TRUNC3]]
+    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[UADDSAT]](<2 x s16>), [[UADDSAT1]](<2 x s16>), [[DEF2]](<2 x s16>)
     ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<6 x s16>), 0
     ; GFX9: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
     ; GFX9: [[CONCAT_VECTORS1:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[EXTRACT]](<3 x s16>), [[EXTRACT1]](<3 x s16>)
@@ -650,26 +599,17 @@ body: |
     ; GFX8: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
     ; GFX8: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
     ; GFX8: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
-    ; GFX8: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
-    ; GFX8: [[XOR:%[0-9]+]]:_(s16) = G_XOR [[TRUNC]], [[C1]]
-    ; GFX8: [[UMIN:%[0-9]+]]:_(s16) = G_UMIN [[XOR]], [[TRUNC4]]
-    ; GFX8: [[ADD:%[0-9]+]]:_(s16) = G_ADD [[TRUNC]], [[UMIN]]
-    ; GFX8: [[XOR1:%[0-9]+]]:_(s16) = G_XOR [[TRUNC1]], [[C1]]
-    ; GFX8: [[UMIN1:%[0-9]+]]:_(s16) = G_UMIN [[XOR1]], [[TRUNC5]]
-    ; GFX8: [[ADD1:%[0-9]+]]:_(s16) = G_ADD [[TRUNC1]], [[UMIN1]]
-    ; GFX8: [[XOR2:%[0-9]+]]:_(s16) = G_XOR [[TRUNC2]], [[C1]]
-    ; GFX8: [[UMIN2:%[0-9]+]]:_(s16) = G_UMIN [[XOR2]], [[TRUNC6]]
-    ; GFX8: [[ADD2:%[0-9]+]]:_(s16) = G_ADD [[TRUNC2]], [[UMIN2]]
-    ; GFX8: [[XOR3:%[0-9]+]]:_(s16) = G_XOR [[TRUNC3]], [[C1]]
-    ; GFX8: [[UMIN3:%[0-9]+]]:_(s16) = G_UMIN [[XOR3]], [[TRUNC7]]
-    ; GFX8: [[ADD3:%[0-9]+]]:_(s16) = G_ADD [[TRUNC3]], [[UMIN3]]
-    ; GFX8: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ADD]](s16)
-    ; GFX8: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[ADD1]](s16)
+    ; GFX8: [[UADDSAT:%[0-9]+]]:_(s16) = G_UADDSAT [[TRUNC]], [[TRUNC4]]
+    ; GFX8: [[UADDSAT1:%[0-9]+]]:_(s16) = G_UADDSAT [[TRUNC1]], [[TRUNC5]]
+    ; GFX8: [[UADDSAT2:%[0-9]+]]:_(s16) = G_UADDSAT [[TRUNC2]], [[TRUNC6]]
+    ; GFX8: [[UADDSAT3:%[0-9]+]]:_(s16) = G_UADDSAT [[TRUNC3]], [[TRUNC7]]
+    ; GFX8: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDSAT]](s16)
+    ; GFX8: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UADDSAT1]](s16)
     ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
     ; GFX8: [[OR:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL]]
     ; GFX8: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
-    ; GFX8: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[ADD2]](s16)
-    ; GFX8: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[ADD3]](s16)
+    ; GFX8: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[UADDSAT2]](s16)
+    ; GFX8: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[UADDSAT3]](s16)
     ; GFX8: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
     ; GFX8: [[OR1:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[SHL1]]
     ; GFX8: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
@@ -680,16 +620,9 @@ body: |
     ; GFX9: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
     ; GFX9: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
     ; GFX9: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
-    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
-    ; GFX9: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C]](s32), [[C]](s32)
-    ; GFX9: [[XOR:%[0-9]+]]:_(<2 x s16>) = G_XOR [[UV]], [[BUILD_VECTOR_TRUNC]]
-    ; GFX9: [[UMIN:%[0-9]+]]:_(<2 x s16>) = G_UMIN [[XOR]], [[UV2]]
-    ; GFX9: [[ADD:%[0-9]+]]:_(<2 x s16>) = G_ADD [[UV]], [[UMIN]]
-    ; GFX9: [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C]](s32), [[C]](s32)
-    ; GFX9: [[XOR1:%[0-9]+]]:_(<2 x s16>) = G_XOR [[UV1]], [[BUILD_VECTOR_TRUNC1]]
-    ; GFX9: [[UMIN1:%[0-9]+]]:_(<2 x s16>) = G_UMIN [[XOR1]], [[UV3]]
-    ; GFX9: [[ADD1:%[0-9]+]]:_(<2 x s16>) = G_ADD [[UV1]], [[UMIN1]]
-    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[ADD]](<2 x s16>), [[ADD1]](<2 x s16>)
+    ; GFX9: [[UADDSAT:%[0-9]+]]:_(<2 x s16>) = G_UADDSAT [[UV]], [[UV2]]
+    ; GFX9: [[UADDSAT1:%[0-9]+]]:_(<2 x s16>) = G_UADDSAT [[UV1]], [[UV3]]
+    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[UADDSAT]](<2 x s16>), [[UADDSAT1]](<2 x s16>)
     ; GFX9: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x s16>)
     %0:_(<4 x s16>) = COPY $vgpr0_vgpr1
     %1:_(<4 x s16>) = COPY $vgpr2_vgpr3
@@ -714,19 +647,13 @@ body: |
     ; GFX8-LABEL: name: uaddsat_s32
     ; GFX8: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX8: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
-    ; GFX8: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[COPY]], [[C]]
-    ; GFX8: [[UMIN:%[0-9]+]]:_(s32) = G_UMIN [[XOR]], [[COPY1]]
-    ; GFX8: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[COPY]], [[UMIN]]
-    ; GFX8: $vgpr0 = COPY [[ADD]](s32)
+    ; GFX8: [[UADDSAT:%[0-9]+]]:_(s32) = G_UADDSAT [[COPY]], [[COPY1]]
+    ; GFX8: $vgpr0 = COPY [[UADDSAT]](s32)
     ; GFX9-LABEL: name: uaddsat_s32
     ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
-    ; GFX9: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[COPY]], [[C]]
-    ; GFX9: [[UMIN:%[0-9]+]]:_(s32) = G_UMIN [[XOR]], [[COPY1]]
-    ; GFX9: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[COPY]], [[UMIN]]
-    ; GFX9: $vgpr0 = COPY [[ADD]](s32)
+    ; GFX9: [[UADDSAT:%[0-9]+]]:_(s32) = G_UADDSAT [[COPY]], [[COPY1]]
+    ; GFX9: $vgpr0 = COPY [[UADDSAT]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
     %2:_(s32) = G_UADDSAT %0, %1
@@ -758,28 +685,18 @@ body: |
     ; GFX8: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
     ; GFX8: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
     ; GFX8: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
-    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
-    ; GFX8: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[UV]], [[C]]
-    ; GFX8: [[UMIN:%[0-9]+]]:_(s32) = G_UMIN [[XOR]], [[UV2]]
-    ; GFX8: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[UV]], [[UMIN]]
-    ; GFX8: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[UV1]], [[C]]
-    ; GFX8: [[UMIN1:%[0-9]+]]:_(s32) = G_UMIN [[XOR1]], [[UV3]]
-    ; GFX8: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[UV1]], [[UMIN1]]
-    ; GFX8: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[ADD]](s32), [[ADD1]](s32)
+    ; GFX8: [[UADDSAT:%[0-9]+]]:_(s32) = G_UADDSAT [[UV]], [[UV2]]
+    ; GFX8: [[UADDSAT1:%[0-9]+]]:_(s32) = G_UADDSAT [[UV1]], [[UV3]]
+    ; GFX8: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[UADDSAT]](s32), [[UADDSAT1]](s32)
     ; GFX8: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
     ; GFX9-LABEL: name: uaddsat_v2s32
     ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
     ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
     ; GFX9: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
     ; GFX9: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
-    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
-    ; GFX9: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[UV]], [[C]]
-    ; GFX9: [[UMIN:%[0-9]+]]:_(s32) = G_UMIN [[XOR]], [[UV2]]
-    ; GFX9: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[UV]], [[UMIN]]
-    ; GFX9: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[UV1]], [[C]]
-    ; GFX9: [[UMIN1:%[0-9]+]]:_(s32) = G_UMIN [[XOR1]], [[UV3]]
-    ; GFX9: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[UV1]], [[UMIN1]]
-    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[ADD]](s32), [[ADD1]](s32)
+    ; GFX9: [[UADDSAT:%[0-9]+]]:_(s32) = G_UADDSAT [[UV]], [[UV2]]
+    ; GFX9: [[UADDSAT1:%[0-9]+]]:_(s32) = G_UADDSAT [[UV1]], [[UV3]]
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[UADDSAT]](s32), [[UADDSAT1]](s32)
     ; GFX9: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
     %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
     %1:_(<2 x s32>) = COPY $vgpr2_vgpr3
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-usubsat.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-usubsat.mir
index 356bb38456ea6..5bb430cf4a062 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-usubsat.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-usubsat.mir
@@ -30,9 +30,8 @@ body: |
     ; GFX8: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 9
     ; GFX8: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[C]](s16)
     ; GFX8: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[C]](s16)
-    ; GFX8: [[UMIN:%[0-9]+]]:_(s16) = G_UMIN [[SHL]], [[SHL1]]
-    ; GFX8: [[SUB:%[0-9]+]]:_(s16) = G_SUB [[SHL]], [[UMIN]]
-    ; GFX8: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[SUB]], [[C]](s16)
+    ; GFX8: [[USUBSAT:%[0-9]+]]:_(s16) = G_USUBSAT [[SHL]], [[SHL1]]
+    ; GFX8: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[USUBSAT]], [[C]](s16)
     ; GFX8: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR]](s16)
     ; GFX8: $vgpr0 = COPY [[ANYEXT]](s32)
     ; GFX9-LABEL: name: usubsat_s7
@@ -43,9 +42,8 @@ body: |
     ; GFX9: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 9
     ; GFX9: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[C]](s16)
     ; GFX9: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[C]](s16)
-    ; GFX9: [[UMIN:%[0-9]+]]:_(s16) = G_UMIN [[SHL]], [[SHL1]]
-    ; GFX9: [[SUB:%[0-9]+]]:_(s16) = G_SUB [[SHL]], [[UMIN]]
-    ; GFX9: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[SUB]], [[C]](s16)
+    ; GFX9: [[USUBSAT:%[0-9]+]]:_(s16) = G_USUBSAT [[SHL]], [[SHL1]]
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[USUBSAT]], [[C]](s16)
     ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR]](s16)
     ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
     %0:_(s32) = COPY $vgpr0
@@ -84,9 +82,8 @@ body: |
     ; GFX8: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
     ; GFX8: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[C]](s16)
     ; GFX8: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[C]](s16)
-    ; GFX8: [[UMIN:%[0-9]+]]:_(s16) = G_UMIN [[SHL]], [[SHL1]]
-    ; GFX8: [[SUB:%[0-9]+]]:_(s16) = G_SUB [[SHL]], [[UMIN]]
-    ; GFX8: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[SUB]], [[C]](s16)
+    ; GFX8: [[USUBSAT:%[0-9]+]]:_(s16) = G_USUBSAT [[SHL]], [[SHL1]]
+    ; GFX8: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[USUBSAT]], [[C]](s16)
     ; GFX8: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR]](s16)
     ; GFX8: $vgpr0 = COPY [[ANYEXT]](s32)
     ; GFX9-LABEL: name: usubsat_s8
@@ -97,9 +94,8 @@ body: |
     ; GFX9: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
     ; GFX9: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[C]](s16)
     ; GFX9: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[C]](s16)
-    ; GFX9: [[UMIN:%[0-9]+]]:_(s16) = G_UMIN [[SHL]], [[SHL1]]
-    ; GFX9: [[SUB:%[0-9]+]]:_(s16) = G_SUB [[SHL]], [[UMIN]]
-    ; GFX9: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[SUB]], [[C]](s16)
+    ; GFX9: [[USUBSAT:%[0-9]+]]:_(s16) = G_USUBSAT [[SHL]], [[SHL1]]
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[USUBSAT]], [[C]](s16)
     ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR]](s16)
     ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
     %0:_(s32) = COPY $vgpr0
@@ -172,16 +168,14 @@ body: |
     ; GFX8: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
     ; GFX8: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[C3]](s16)
     ; GFX8: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[C3]](s16)
-    ; GFX8: [[UMIN:%[0-9]+]]:_(s16) = G_UMIN [[SHL]], [[SHL1]]
-    ; GFX8: [[SUB:%[0-9]+]]:_(s16) = G_SUB [[SHL]], [[UMIN]]
-    ; GFX8: [[LSHR6:%[0-9]+]]:_(s16) = G_LSHR [[SUB]], [[C3]](s16)
+    ; GFX8: [[USUBSAT:%[0-9]+]]:_(s16) = G_USUBSAT [[SHL]], [[SHL1]]
+    ; GFX8: [[LSHR6:%[0-9]+]]:_(s16) = G_LSHR [[USUBSAT]], [[C3]](s16)
     ; GFX8: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
     ; GFX8: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
     ; GFX8: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[TRUNC2]], [[C3]](s16)
     ; GFX8: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[TRUNC3]], [[C3]](s16)
-    ; GFX8: [[UMIN1:%[0-9]+]]:_(s16) = G_UMIN [[SHL2]], [[SHL3]]
-    ; GFX8: [[SUB1:%[0-9]+]]:_(s16) = G_SUB [[SHL2]], [[UMIN1]]
-    ; GFX8: [[LSHR7:%[0-9]+]]:_(s16) = G_LSHR [[SUB1]], [[C3]](s16)
+    ; GFX8: [[USUBSAT1:%[0-9]+]]:_(s16) = G_USUBSAT [[SHL2]], [[SHL3]]
+    ; GFX8: [[LSHR7:%[0-9]+]]:_(s16) = G_LSHR [[USUBSAT1]], [[C3]](s16)
     ; GFX8: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
     ; GFX8: [[COPY2:%[0-9]+]]:_(s16) = COPY [[LSHR6]](s16)
     ; GFX8: [[AND:%[0-9]+]]:_(s16) = G_AND [[COPY2]], [[C4]]
@@ -208,16 +202,14 @@ body: |
     ; GFX9: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
     ; GFX9: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[C3]](s16)
     ; GFX9: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[C3]](s16)
-    ; GFX9: [[UMIN:%[0-9]+]]:_(s16) = G_UMIN [[SHL]], [[SHL1]]
-    ; GFX9: [[SUB:%[0-9]+]]:_(s16) = G_SUB [[SHL]], [[UMIN]]
-    ; GFX9: [[LSHR6:%[0-9]+]]:_(s16) = G_LSHR [[SUB]], [[C3]](s16)
+    ; GFX9: [[USUBSAT:%[0-9]+]]:_(s16) = G_USUBSAT [[SHL]], [[SHL1]]
+    ; GFX9: [[LSHR6:%[0-9]+]]:_(s16) = G_LSHR [[USUBSAT]], [[C3]](s16)
     ; GFX9: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
     ; GFX9: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
     ; GFX9: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[TRUNC2]], [[C3]](s16)
     ; GFX9: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[TRUNC3]], [[C3]](s16)
-    ; GFX9: [[UMIN1:%[0-9]+]]:_(s16) = G_UMIN [[SHL2]], [[SHL3]]
-    ; GFX9: [[SUB1:%[0-9]+]]:_(s16) = G_SUB [[SHL2]], [[UMIN1]]
-    ; GFX9: [[LSHR7:%[0-9]+]]:_(s16) = G_LSHR [[SUB1]], [[C3]](s16)
+    ; GFX9: [[USUBSAT1:%[0-9]+]]:_(s16) = G_USUBSAT [[SHL2]], [[SHL3]]
+    ; GFX9: [[LSHR7:%[0-9]+]]:_(s16) = G_LSHR [[USUBSAT1]], [[C3]](s16)
     ; GFX9: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
     ; GFX9: [[COPY2:%[0-9]+]]:_(s16) = COPY [[LSHR6]](s16)
     ; GFX9: [[AND:%[0-9]+]]:_(s16) = G_AND [[COPY2]], [[C4]]
@@ -263,18 +255,16 @@ body: |
     ; GFX8: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
     ; GFX8: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
     ; GFX8: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
-    ; GFX8: [[UMIN:%[0-9]+]]:_(s16) = G_UMIN [[TRUNC]], [[TRUNC1]]
-    ; GFX8: [[SUB:%[0-9]+]]:_(s16) = G_SUB [[TRUNC]], [[UMIN]]
-    ; GFX8: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[SUB]](s16)
+    ; GFX8: [[USUBSAT:%[0-9]+]]:_(s16) = G_USUBSAT [[TRUNC]], [[TRUNC1]]
+    ; GFX8: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[USUBSAT]](s16)
     ; GFX8: $vgpr0 = COPY [[ANYEXT]](s32)
     ; GFX9-LABEL: name: usubsat_s16
     ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
     ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
     ; GFX9: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
-    ; GFX9: [[UMIN:%[0-9]+]]:_(s16) = G_UMIN [[TRUNC]], [[TRUNC1]]
-    ; GFX9: [[SUB:%[0-9]+]]:_(s16) = G_SUB [[TRUNC]], [[UMIN]]
-    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[SUB]](s16)
+    ; GFX9: [[USUBSAT:%[0-9]+]]:_(s16) = G_USUBSAT [[TRUNC]], [[TRUNC1]]
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[USUBSAT]](s16)
     ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
@@ -334,12 +324,10 @@ body: |
     ; GFX8: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
     ; GFX8: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
     ; GFX8: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
-    ; GFX8: [[UMIN:%[0-9]+]]:_(s16) = G_UMIN [[TRUNC]], [[TRUNC2]]
-    ; GFX8: [[SUB:%[0-9]+]]:_(s16) = G_SUB [[TRUNC]], [[UMIN]]
-    ; GFX8: [[UMIN1:%[0-9]+]]:_(s16) = G_UMIN [[TRUNC1]], [[TRUNC3]]
-    ; GFX8: [[SUB1:%[0-9]+]]:_(s16) = G_SUB [[TRUNC1]], [[UMIN1]]
-    ; GFX8: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[SUB]](s16)
-    ; GFX8: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[SUB1]](s16)
+    ; GFX8: [[USUBSAT:%[0-9]+]]:_(s16) = G_USUBSAT [[TRUNC]], [[TRUNC2]]
+    ; GFX8: [[USUBSAT1:%[0-9]+]]:_(s16) = G_USUBSAT [[TRUNC1]], [[TRUNC3]]
+    ; GFX8: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[USUBSAT]](s16)
+    ; GFX8: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[USUBSAT1]](s16)
     ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
     ; GFX8: [[OR:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL]]
     ; GFX8: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
@@ -347,9 +335,8 @@ body: |
     ; GFX9-LABEL: name: usubsat_v2s16
     ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
     ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
-    ; GFX9: [[UMIN:%[0-9]+]]:_(<2 x s16>) = G_UMIN [[COPY]], [[COPY1]]
-    ; GFX9: [[SUB:%[0-9]+]]:_(<2 x s16>) = G_SUB [[COPY]], [[UMIN]]
-    ; GFX9: $vgpr0 = COPY [[SUB]](<2 x s16>)
+    ; GFX9: [[USUBSAT:%[0-9]+]]:_(<2 x s16>) = G_USUBSAT [[COPY]], [[COPY1]]
+    ; GFX9: $vgpr0 = COPY [[USUBSAT]](<2 x s16>)
     %0:_(<2 x s16>) = COPY $vgpr0
     %1:_(<2 x s16>) = COPY $vgpr1
     %2:_(<2 x s16>) = G_USUBSAT %0, %1
@@ -442,18 +429,15 @@ body: |
     ; GFX8: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
     ; GFX8: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
     ; GFX8: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
-    ; GFX8: [[UMIN:%[0-9]+]]:_(s16) = G_UMIN [[TRUNC]], [[TRUNC3]]
-    ; GFX8: [[SUB:%[0-9]+]]:_(s16) = G_SUB [[TRUNC]], [[UMIN]]
-    ; GFX8: [[UMIN1:%[0-9]+]]:_(s16) = G_UMIN [[TRUNC1]], [[TRUNC4]]
-    ; GFX8: [[SUB1:%[0-9]+]]:_(s16) = G_SUB [[TRUNC1]], [[UMIN1]]
-    ; GFX8: [[UMIN2:%[0-9]+]]:_(s16) = G_UMIN [[TRUNC2]], [[TRUNC5]]
-    ; GFX8: [[SUB2:%[0-9]+]]:_(s16) = G_SUB [[TRUNC2]], [[UMIN2]]
-    ; GFX8: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[SUB]](s16)
-    ; GFX8: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[SUB1]](s16)
+    ; GFX8: [[USUBSAT:%[0-9]+]]:_(s16) = G_USUBSAT [[TRUNC]], [[TRUNC3]]
+    ; GFX8: [[USUBSAT1:%[0-9]+]]:_(s16) = G_USUBSAT [[TRUNC1]], [[TRUNC4]]
+    ; GFX8: [[USUBSAT2:%[0-9]+]]:_(s16) = G_USUBSAT [[TRUNC2]], [[TRUNC5]]
+    ; GFX8: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[USUBSAT]](s16)
+    ; GFX8: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[USUBSAT1]](s16)
     ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
     ; GFX8: [[OR:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL]]
     ; GFX8: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
-    ; GFX8: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[SUB2]](s16)
+    ; GFX8: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[USUBSAT2]](s16)
     ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; GFX8: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C]](s32)
     ; GFX8: [[OR1:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[SHL1]]
@@ -492,11 +476,9 @@ body: |
     ; GFX9: [[BUILD_VECTOR_TRUNC2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY4]](s32), [[COPY5]](s32)
     ; GFX9: [[COPY6:%[0-9]+]]:_(s32) = COPY [[BITCAST3]](s32)
     ; GFX9: [[BUILD_VECTOR_TRUNC3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY6]](s32), [[DEF1]](s32)
-    ; GFX9: [[UMIN:%[0-9]+]]:_(<2 x s16>) = G_UMIN [[BUILD_VECTOR_TRUNC]], [[BUILD_VECTOR_TRUNC2]]
-    ; GFX9: [[SUB:%[0-9]+]]:_(<2 x s16>) = G_SUB [[BUILD_VECTOR_TRUNC]], [[UMIN]]
-    ; GFX9: [[UMIN1:%[0-9]+]]:_(<2 x s16>) = G_UMIN [[BUILD_VECTOR_TRUNC1]], [[BUILD_VECTOR_TRUNC3]]
-    ; GFX9: [[SUB1:%[0-9]+]]:_(<2 x s16>) = G_SUB [[BUILD_VECTOR_TRUNC1]], [[UMIN1]]
-    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[SUB]](<2 x s16>), [[SUB1]](<2 x s16>), [[DEF2]](<2 x s16>)
+    ; GFX9: [[USUBSAT:%[0-9]+]]:_(<2 x s16>) = G_USUBSAT [[BUILD_VECTOR_TRUNC]], [[BUILD_VECTOR_TRUNC2]]
+    ; GFX9: [[USUBSAT1:%[0-9]+]]:_(<2 x s16>) = G_USUBSAT [[BUILD_VECTOR_TRUNC1]], [[BUILD_VECTOR_TRUNC3]]
+    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[USUBSAT]](<2 x s16>), [[USUBSAT1]](<2 x s16>), [[DEF2]](<2 x s16>)
     ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<6 x s16>), 0
     ; GFX9: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
     ; GFX9: [[CONCAT_VECTORS1:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[EXTRACT]](<3 x s16>), [[EXTRACT1]](<3 x s16>)
@@ -596,21 +578,17 @@ body: |
     ; GFX8: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
     ; GFX8: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
     ; GFX8: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
-    ; GFX8: [[UMIN:%[0-9]+]]:_(s16) = G_UMIN [[TRUNC]], [[TRUNC4]]
-    ; GFX8: [[SUB:%[0-9]+]]:_(s16) = G_SUB [[TRUNC]], [[UMIN]]
-    ; GFX8: [[UMIN1:%[0-9]+]]:_(s16) = G_UMIN [[TRUNC1]], [[TRUNC5]]
-    ; GFX8: [[SUB1:%[0-9]+]]:_(s16) = G_SUB [[TRUNC1]], [[UMIN1]]
-    ; GFX8: [[UMIN2:%[0-9]+]]:_(s16) = G_UMIN [[TRUNC2]], [[TRUNC6]]
-    ; GFX8: [[SUB2:%[0-9]+]]:_(s16) = G_SUB [[TRUNC2]], [[UMIN2]]
-    ; GFX8: [[UMIN3:%[0-9]+]]:_(s16) = G_UMIN [[TRUNC3]], [[TRUNC7]]
-    ; GFX8: [[SUB3:%[0-9]+]]:_(s16) = G_SUB [[TRUNC3]], [[UMIN3]]
-    ; GFX8: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[SUB]](s16)
-    ; GFX8: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[SUB1]](s16)
+    ; GFX8: [[USUBSAT:%[0-9]+]]:_(s16) = G_USUBSAT [[TRUNC]], [[TRUNC4]]
+    ; GFX8: [[USUBSAT1:%[0-9]+]]:_(s16) = G_USUBSAT [[TRUNC1]], [[TRUNC5]]
+    ; GFX8: [[USUBSAT2:%[0-9]+]]:_(s16) = G_USUBSAT [[TRUNC2]], [[TRUNC6]]
+    ; GFX8: [[USUBSAT3:%[0-9]+]]:_(s16) = G_USUBSAT [[TRUNC3]], [[TRUNC7]]
+    ; GFX8: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[USUBSAT]](s16)
+    ; GFX8: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[USUBSAT1]](s16)
     ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
     ; GFX8: [[OR:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL]]
     ; GFX8: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
-    ; GFX8: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[SUB2]](s16)
-    ; GFX8: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[SUB3]](s16)
+    ; GFX8: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[USUBSAT2]](s16)
+    ; GFX8: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[USUBSAT3]](s16)
     ; GFX8: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
     ; GFX8: [[OR1:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[SHL1]]
     ; GFX8: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
@@ -621,11 +599,9 @@ body: |
     ; GFX9: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
     ; GFX9: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
     ; GFX9: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
-    ; GFX9: [[UMIN:%[0-9]+]]:_(<2 x s16>) = G_UMIN [[UV]], [[UV2]]
-    ; GFX9: [[SUB:%[0-9]+]]:_(<2 x s16>) = G_SUB [[UV]], [[UMIN]]
-    ; GFX9: [[UMIN1:%[0-9]+]]:_(<2 x s16>) = G_UMIN [[UV1]], [[UV3]]
-    ; GFX9: [[SUB1:%[0-9]+]]:_(<2 x s16>) = G_SUB [[UV1]], [[UMIN1]]
-    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[SUB]](<2 x s16>), [[SUB1]](<2 x s16>)
+    ; GFX9: [[USUBSAT:%[0-9]+]]:_(<2 x s16>) = G_USUBSAT [[UV]], [[UV2]]
+    ; GFX9: [[USUBSAT1:%[0-9]+]]:_(<2 x s16>) = G_USUBSAT [[UV1]], [[UV3]]
+    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[USUBSAT]](<2 x s16>), [[USUBSAT1]](<2 x s16>)
     ; GFX9: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x s16>)
     %0:_(<4 x s16>) = COPY $vgpr0_vgpr1
     %1:_(<4 x s16>) = COPY $vgpr2_vgpr3
@@ -648,15 +624,13 @@ body: |
     ; GFX8-LABEL: name: usubsat_s32
     ; GFX8: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX8: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-    ; GFX8: [[UMIN:%[0-9]+]]:_(s32) = G_UMIN [[COPY]], [[COPY1]]
-    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[COPY]], [[UMIN]]
-    ; GFX8: $vgpr0 = COPY [[SUB]](s32)
+    ; GFX8: [[USUBSAT:%[0-9]+]]:_(s32) = G_USUBSAT [[COPY]], [[COPY1]]
+    ; GFX8: $vgpr0 = COPY [[USUBSAT]](s32)
     ; GFX9-LABEL: name: usubsat_s32
     ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-    ; GFX9: [[UMIN:%[0-9]+]]:_(s32) = G_UMIN [[COPY]], [[COPY1]]
-    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[COPY]], [[UMIN]]
-    ; GFX9: $vgpr0 = COPY [[SUB]](s32)
+    ; GFX9: [[USUBSAT:%[0-9]+]]:_(s32) = G_USUBSAT [[COPY]], [[COPY1]]
+    ; GFX9: $vgpr0 = COPY [[USUBSAT]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
     %2:_(s32) = G_USUBSAT %0, %1
@@ -685,22 +659,18 @@ body: |
     ; GFX8: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
     ; GFX8: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
     ; GFX8: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
-    ; GFX8: [[UMIN:%[0-9]+]]:_(s32) = G_UMIN [[UV]], [[UV2]]
-    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[UV]], [[UMIN]]
-    ; GFX8: [[UMIN1:%[0-9]+]]:_(s32) = G_UMIN [[UV1]], [[UV3]]
-    ; GFX8: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[UV1]], [[UMIN1]]
-    ; GFX8: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SUB]](s32), [[SUB1]](s32)
+    ; GFX8: [[USUBSAT:%[0-9]+]]:_(s32) = G_USUBSAT [[UV]], [[UV2]]
+    ; GFX8: [[USUBSAT1:%[0-9]+]]:_(s32) = G_USUBSAT [[UV1]], [[UV3]]
+    ; GFX8: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[USUBSAT]](s32), [[USUBSAT1]](s32)
     ; GFX8: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
     ; GFX9-LABEL: name: usubsat_v2s32
     ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
     ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
     ; GFX9: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
     ; GFX9: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
-    ; GFX9: [[UMIN:%[0-9]+]]:_(s32) = G_UMIN [[UV]], [[UV2]]
-    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[UV]], [[UMIN]]
-    ; GFX9: [[UMIN1:%[0-9]+]]:_(s32) = G_UMIN [[UV1]], [[UV3]]
-    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[UV1]], [[UMIN1]]
-    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SUB]](s32), [[SUB1]](s32)
+    ; GFX9: [[USUBSAT:%[0-9]+]]:_(s32) = G_USUBSAT [[UV]], [[UV2]]
+    ; GFX9: [[USUBSAT1:%[0-9]+]]:_(s32) = G_USUBSAT [[UV1]], [[UV3]]
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[USUBSAT]](s32), [[USUBSAT1]](s32)
     ; GFX9: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
     %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
     %1:_(<2 x s32>) = COPY $vgpr2_vgpr3
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/saddsat.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/saddsat.ll
index ba672883fa562..7b88123b2c9b9 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/saddsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/saddsat.ll
@@ -39,14 +39,8 @@ define i7 @v_saddsat_i7(i7 %lhs, i7 %rhs) {
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 9, v0
-; GFX9-NEXT:    v_min_i16_e32 v3, 0, v0
-; GFX9-NEXT:    v_max_i16_e32 v2, 0, v0
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 9, v1
-; GFX9-NEXT:    v_sub_u16_e32 v3, 0x8000, v3
-; GFX9-NEXT:    v_sub_u16_e32 v2, 0x7fff, v2
-; GFX9-NEXT:    v_max_i16_e32 v1, v3, v1
-; GFX9-NEXT:    v_min_i16_e32 v1, v1, v2
-; GFX9-NEXT:    v_add_u16_e32 v0, v0, v1
+; GFX9-NEXT:    v_add_i16 v0, v0, v1 clamp
 ; GFX9-NEXT:    v_ashrrev_i16_e32 v0, 9, v0
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -57,13 +51,7 @@ define i7 @v_saddsat_i7(i7 %lhs, i7 %rhs) {
 ; GFX10-NEXT:    v_lshlrev_b16_e64 v0, 9, v0
 ; GFX10-NEXT:    v_lshlrev_b16_e64 v1, 9, v1
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_min_i16_e64 v2, v0, 0
-; GFX10-NEXT:    v_max_i16_e64 v3, v0, 0
-; GFX10-NEXT:    v_sub_nc_u16_e64 v2, 0x8000, v2
-; GFX10-NEXT:    v_sub_nc_u16_e64 v3, 0x7fff, v3
-; GFX10-NEXT:    v_max_i16_e64 v1, v2, v1
-; GFX10-NEXT:    v_min_i16_e64 v1, v1, v3
-; GFX10-NEXT:    v_add_nc_u16_e64 v0, v0, v1
+; GFX10-NEXT:    v_add_nc_i16 v0, v0, v1 clamp
 ; GFX10-NEXT:    v_ashrrev_i16_e64 v0, 9, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call i7 @llvm.sadd.sat.i7(i7 %lhs, i7 %rhs)
@@ -118,54 +106,23 @@ define amdgpu_ps i7 @s_saddsat_i7(i7 inreg %lhs, i7 inreg %rhs) {
 ; GFX9-LABEL: s_saddsat_i7:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_bfe_u32 s2, 9, 0x100000
-; GFX9-NEXT:    s_lshl_b32 s0, s0, s2
 ; GFX9-NEXT:    s_lshl_b32 s1, s1, s2
-; GFX9-NEXT:    s_sext_i32_i16 s3, s0
-; GFX9-NEXT:    s_sext_i32_i16 s4, 0
-; GFX9-NEXT:    s_cmp_gt_i32 s3, s4
-; GFX9-NEXT:    s_cselect_b32 s5, s3, s4
-; GFX9-NEXT:    s_sub_i32 s5, 0x7fff, s5
-; GFX9-NEXT:    s_cmp_lt_i32 s3, s4
-; GFX9-NEXT:    s_cselect_b32 s3, s3, s4
-; GFX9-NEXT:    s_sub_i32 s3, 0xffff8000, s3
-; GFX9-NEXT:    s_sext_i32_i16 s3, s3
-; GFX9-NEXT:    s_sext_i32_i16 s1, s1
-; GFX9-NEXT:    s_cmp_gt_i32 s3, s1
-; GFX9-NEXT:    s_cselect_b32 s1, s3, s1
-; GFX9-NEXT:    s_sext_i32_i16 s1, s1
-; GFX9-NEXT:    s_sext_i32_i16 s3, s5
-; GFX9-NEXT:    s_cmp_lt_i32 s1, s3
-; GFX9-NEXT:    s_cselect_b32 s1, s1, s3
-; GFX9-NEXT:    s_add_i32 s0, s0, s1
-; GFX9-NEXT:    s_sext_i32_i16 s0, s0
-; GFX9-NEXT:    s_ashr_i32 s0, s0, s2
+; GFX9-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX9-NEXT:    v_mov_b32_e32 v0, s1
+; GFX9-NEXT:    v_add_i16 v0, s0, v0 clamp
+; GFX9-NEXT:    v_ashrrev_i16_e32 v0, 9, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_saddsat_i7:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_bfe_u32 s2, 9, 0x100000
-; GFX10-NEXT:    s_sext_i32_i16 s4, 0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_lshl_b32 s0, s0, s2
 ; GFX10-NEXT:    s_lshl_b32 s1, s1, s2
-; GFX10-NEXT:    s_sext_i32_i16 s3, s0
-; GFX10-NEXT:    s_sext_i32_i16 s1, s1
-; GFX10-NEXT:    s_cmp_gt_i32 s3, s4
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cselect_b32 s5, s3, s4
-; GFX10-NEXT:    s_sub_i32 s5, 0x7fff, s5
-; GFX10-NEXT:    s_cmp_lt_i32 s3, s4
-; GFX10-NEXT:    s_cselect_b32 s3, s3, s4
-; GFX10-NEXT:    s_sub_i32 s3, 0xffff8000, s3
-; GFX10-NEXT:    s_sext_i32_i16 s3, s3
-; GFX10-NEXT:    s_cmp_gt_i32 s3, s1
-; GFX10-NEXT:    s_cselect_b32 s1, s3, s1
-; GFX10-NEXT:    s_sext_i32_i16 s3, s5
-; GFX10-NEXT:    s_sext_i32_i16 s1, s1
-; GFX10-NEXT:    s_cmp_lt_i32 s1, s3
-; GFX10-NEXT:    s_cselect_b32 s1, s1, s3
-; GFX10-NEXT:    s_add_i32 s0, s0, s1
-; GFX10-NEXT:    s_sext_i32_i16 s0, s0
-; GFX10-NEXT:    s_ashr_i32 s0, s0, s2
+; GFX10-NEXT:    v_add_nc_i16 v0, s0, s1 clamp
+; GFX10-NEXT:    v_ashrrev_i16_e64 v0, 9, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call i7 @llvm.sadd.sat.i7(i7 %lhs, i7 %rhs)
   ret i7 %result
@@ -206,14 +163,8 @@ define i8 @v_saddsat_i8(i8 %lhs, i8 %rhs) {
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; GFX9-NEXT:    v_min_i16_e32 v3, 0, v0
-; GFX9-NEXT:    v_max_i16_e32 v2, 0, v0
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
-; GFX9-NEXT:    v_sub_u16_e32 v3, 0x8000, v3
-; GFX9-NEXT:    v_sub_u16_e32 v2, 0x7fff, v2
-; GFX9-NEXT:    v_max_i16_e32 v1, v3, v1
-; GFX9-NEXT:    v_min_i16_e32 v1, v1, v2
-; GFX9-NEXT:    v_add_u16_e32 v0, v0, v1
+; GFX9-NEXT:    v_add_i16 v0, v0, v1 clamp
 ; GFX9-NEXT:    v_ashrrev_i16_e32 v0, 8, v0
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -224,13 +175,7 @@ define i8 @v_saddsat_i8(i8 %lhs, i8 %rhs) {
 ; GFX10-NEXT:    v_lshlrev_b16_e64 v0, 8, v0
 ; GFX10-NEXT:    v_lshlrev_b16_e64 v1, 8, v1
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_min_i16_e64 v2, v0, 0
-; GFX10-NEXT:    v_max_i16_e64 v3, v0, 0
-; GFX10-NEXT:    v_sub_nc_u16_e64 v2, 0x8000, v2
-; GFX10-NEXT:    v_sub_nc_u16_e64 v3, 0x7fff, v3
-; GFX10-NEXT:    v_max_i16_e64 v1, v2, v1
-; GFX10-NEXT:    v_min_i16_e64 v1, v1, v3
-; GFX10-NEXT:    v_add_nc_u16_e64 v0, v0, v1
+; GFX10-NEXT:    v_add_nc_i16 v0, v0, v1 clamp
 ; GFX10-NEXT:    v_ashrrev_i16_e64 v0, 8, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call i8 @llvm.sadd.sat.i8(i8 %lhs, i8 %rhs)
@@ -285,54 +230,23 @@ define amdgpu_ps i8 @s_saddsat_i8(i8 inreg %lhs, i8 inreg %rhs) {
 ; GFX9-LABEL: s_saddsat_i8:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_bfe_u32 s2, 8, 0x100000
-; GFX9-NEXT:    s_lshl_b32 s0, s0, s2
 ; GFX9-NEXT:    s_lshl_b32 s1, s1, s2
-; GFX9-NEXT:    s_sext_i32_i16 s3, s0
-; GFX9-NEXT:    s_sext_i32_i16 s4, 0
-; GFX9-NEXT:    s_cmp_gt_i32 s3, s4
-; GFX9-NEXT:    s_cselect_b32 s5, s3, s4
-; GFX9-NEXT:    s_sub_i32 s5, 0x7fff, s5
-; GFX9-NEXT:    s_cmp_lt_i32 s3, s4
-; GFX9-NEXT:    s_cselect_b32 s3, s3, s4
-; GFX9-NEXT:    s_sub_i32 s3, 0xffff8000, s3
-; GFX9-NEXT:    s_sext_i32_i16 s3, s3
-; GFX9-NEXT:    s_sext_i32_i16 s1, s1
-; GFX9-NEXT:    s_cmp_gt_i32 s3, s1
-; GFX9-NEXT:    s_cselect_b32 s1, s3, s1
-; GFX9-NEXT:    s_sext_i32_i16 s1, s1
-; GFX9-NEXT:    s_sext_i32_i16 s3, s5
-; GFX9-NEXT:    s_cmp_lt_i32 s1, s3
-; GFX9-NEXT:    s_cselect_b32 s1, s1, s3
-; GFX9-NEXT:    s_add_i32 s0, s0, s1
-; GFX9-NEXT:    s_sext_i32_i16 s0, s0
-; GFX9-NEXT:    s_ashr_i32 s0, s0, s2
+; GFX9-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX9-NEXT:    v_mov_b32_e32 v0, s1
+; GFX9-NEXT:    v_add_i16 v0, s0, v0 clamp
+; GFX9-NEXT:    v_ashrrev_i16_e32 v0, 8, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_saddsat_i8:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_bfe_u32 s2, 8, 0x100000
-; GFX10-NEXT:    s_sext_i32_i16 s4, 0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_lshl_b32 s0, s0, s2
 ; GFX10-NEXT:    s_lshl_b32 s1, s1, s2
-; GFX10-NEXT:    s_sext_i32_i16 s3, s0
-; GFX10-NEXT:    s_sext_i32_i16 s1, s1
-; GFX10-NEXT:    s_cmp_gt_i32 s3, s4
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cselect_b32 s5, s3, s4
-; GFX10-NEXT:    s_sub_i32 s5, 0x7fff, s5
-; GFX10-NEXT:    s_cmp_lt_i32 s3, s4
-; GFX10-NEXT:    s_cselect_b32 s3, s3, s4
-; GFX10-NEXT:    s_sub_i32 s3, 0xffff8000, s3
-; GFX10-NEXT:    s_sext_i32_i16 s3, s3
-; GFX10-NEXT:    s_cmp_gt_i32 s3, s1
-; GFX10-NEXT:    s_cselect_b32 s1, s3, s1
-; GFX10-NEXT:    s_sext_i32_i16 s3, s5
-; GFX10-NEXT:    s_sext_i32_i16 s1, s1
-; GFX10-NEXT:    s_cmp_lt_i32 s1, s3
-; GFX10-NEXT:    s_cselect_b32 s1, s1, s3
-; GFX10-NEXT:    s_add_i32 s0, s0, s1
-; GFX10-NEXT:    s_sext_i32_i16 s0, s0
-; GFX10-NEXT:    s_ashr_i32 s0, s0, s2
+; GFX10-NEXT:    v_add_nc_i16 v0, s0, s1 clamp
+; GFX10-NEXT:    v_ashrrev_i16_e64 v0, 8, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call i8 @llvm.sadd.sat.i8(i8 %lhs, i8 %rhs)
   ret i8 %result
@@ -408,26 +322,12 @@ define i16 @v_saddsat_v2i8(i16 %lhs.arg, i16 %rhs.arg) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    s_mov_b32 s4, 8
 ; GFX9-NEXT:    v_lshrrev_b32_sdwa v2, s4, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; GFX9-NEXT:    s_movk_i32 s5, 0x8000
-; GFX9-NEXT:    v_min_i16_e32 v5, 0, v0
 ; GFX9-NEXT:    v_lshrrev_b32_sdwa v3, s4, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
-; GFX9-NEXT:    v_sub_u16_e32 v5, s5, v5
-; GFX9-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX9-NEXT:    v_max_i16_e32 v4, 0, v0
-; GFX9-NEXT:    v_sub_u16_e32 v4, s4, v4
-; GFX9-NEXT:    v_max_i16_e32 v1, v5, v1
-; GFX9-NEXT:    v_min_i16_e32 v1, v1, v4
-; GFX9-NEXT:    v_min_i16_e32 v4, 0, v2
-; GFX9-NEXT:    v_add_u16_e32 v0, v0, v1
-; GFX9-NEXT:    v_max_i16_e32 v1, 0, v2
-; GFX9-NEXT:    v_sub_u16_e32 v4, s5, v4
-; GFX9-NEXT:    v_sub_u16_e32 v1, s4, v1
-; GFX9-NEXT:    v_max_i16_e32 v3, v4, v3
-; GFX9-NEXT:    v_min_i16_e32 v1, v3, v1
+; GFX9-NEXT:    v_add_i16 v0, v0, v1 clamp
 ; GFX9-NEXT:    s_movk_i32 s4, 0xff
-; GFX9-NEXT:    v_add_u16_e32 v1, v2, v1
+; GFX9-NEXT:    v_add_i16 v1, v2, v3 clamp
 ; GFX9-NEXT:    v_and_b32_sdwa v0, sext(v0), s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
 ; GFX9-NEXT:    v_and_b32_sdwa v1, sext(v1), s4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v0, v0, v1
@@ -438,31 +338,17 @@ define i16 @v_saddsat_v2i8(i16 %lhs.arg, i16 %rhs.arg) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    s_mov_b32 s4, 8
-; GFX10-NEXT:    v_lshlrev_b16_e64 v2, 8, v0
-; GFX10-NEXT:    v_lshrrev_b32_sdwa v0, s4, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT:    s_movk_i32 s5, 0x8000
-; GFX10-NEXT:    v_lshrrev_b32_sdwa v3, s4, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT:    v_lshlrev_b16_e64 v1, 8, v1
-; GFX10-NEXT:    v_min_i16_e64 v4, v2, 0
-; GFX10-NEXT:    v_min_i16_e64 v5, v0, 0
-; GFX10-NEXT:    v_max_i16_e64 v6, v2, 0
-; GFX10-NEXT:    v_max_i16_e64 v7, v0, 0
-; GFX10-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX10-NEXT:    v_sub_nc_u16_e64 v4, s5, v4
-; GFX10-NEXT:    v_sub_nc_u16_e64 v5, s5, v5
-; GFX10-NEXT:    v_sub_nc_u16_e64 v6, s4, v6
-; GFX10-NEXT:    v_sub_nc_u16_e64 v7, s4, v7
+; GFX10-NEXT:    v_lshlrev_b16_e64 v3, 8, v1
+; GFX10-NEXT:    v_lshrrev_b32_sdwa v2, s4, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    v_lshrrev_b32_sdwa v1, s4, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    v_lshlrev_b16_e64 v0, 8, v0
 ; GFX10-NEXT:    s_movk_i32 s4, 0xff
-; GFX10-NEXT:    v_max_i16_e64 v1, v4, v1
-; GFX10-NEXT:    v_max_i16_e64 v10, v5, v3
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_min_i16_e64 v1, v1, v6
-; GFX10-NEXT:    v_min_i16_e64 v3, v10, v7
-; GFX10-NEXT:    v_add_nc_u16_e64 v1, v2, v1
-; GFX10-NEXT:    v_add_nc_u16_e64 v0, v0, v3
-; GFX10-NEXT:    v_and_b32_sdwa v1, sext(v1), s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
-; GFX10-NEXT:    v_and_b32_sdwa v0, sext(v0), s4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
-; GFX10-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX10-NEXT:    v_add_nc_i16 v1, v2, v1 clamp
+; GFX10-NEXT:    v_add_nc_i16 v0, v0, v3 clamp
+; GFX10-NEXT:    v_and_b32_sdwa v1, sext(v1), s4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_and_b32_sdwa v0, sext(v0), s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %lhs = bitcast i16 %lhs.arg to <2 x i8>
   %rhs = bitcast i16 %rhs.arg to <2 x i8>
@@ -571,112 +457,40 @@ define amdgpu_ps i16 @s_saddsat_v2i8(i16 inreg %lhs.arg, i16 inreg %rhs.arg) {
 ; GFX9-LABEL: s_saddsat_v2i8:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_bfe_u32 s4, 8, 0x100000
-; GFX9-NEXT:    s_lshr_b32 s2, s0, 8
-; GFX9-NEXT:    s_lshl_b32 s0, s0, s4
 ; GFX9-NEXT:    s_lshr_b32 s3, s1, 8
 ; GFX9-NEXT:    s_lshl_b32 s1, s1, s4
-; GFX9-NEXT:    s_sext_i32_i16 s7, s0
-; GFX9-NEXT:    s_sext_i32_i16 s8, 0
-; GFX9-NEXT:    s_cmp_gt_i32 s7, s8
-; GFX9-NEXT:    s_movk_i32 s5, 0x7fff
-; GFX9-NEXT:    s_cselect_b32 s9, s7, s8
-; GFX9-NEXT:    s_sub_i32 s9, s5, s9
-; GFX9-NEXT:    s_cmp_lt_i32 s7, s8
-; GFX9-NEXT:    s_movk_i32 s6, 0x8000
-; GFX9-NEXT:    s_cselect_b32 s7, s7, s8
-; GFX9-NEXT:    s_sub_i32 s7, s6, s7
-; GFX9-NEXT:    s_sext_i32_i16 s7, s7
-; GFX9-NEXT:    s_sext_i32_i16 s1, s1
-; GFX9-NEXT:    s_cmp_gt_i32 s7, s1
-; GFX9-NEXT:    s_cselect_b32 s1, s7, s1
-; GFX9-NEXT:    s_sext_i32_i16 s1, s1
-; GFX9-NEXT:    s_sext_i32_i16 s7, s9
-; GFX9-NEXT:    s_cmp_lt_i32 s1, s7
-; GFX9-NEXT:    s_cselect_b32 s1, s1, s7
-; GFX9-NEXT:    s_add_i32 s0, s0, s1
-; GFX9-NEXT:    s_sext_i32_i16 s0, s0
-; GFX9-NEXT:    s_lshl_b32 s1, s2, s4
-; GFX9-NEXT:    s_lshl_b32 s2, s3, s4
-; GFX9-NEXT:    s_ashr_i32 s0, s0, s4
-; GFX9-NEXT:    s_sext_i32_i16 s3, s1
-; GFX9-NEXT:    s_cmp_gt_i32 s3, s8
-; GFX9-NEXT:    s_cselect_b32 s7, s3, s8
-; GFX9-NEXT:    s_sub_i32 s5, s5, s7
-; GFX9-NEXT:    s_cmp_lt_i32 s3, s8
-; GFX9-NEXT:    s_cselect_b32 s3, s3, s8
-; GFX9-NEXT:    s_sub_i32 s3, s6, s3
-; GFX9-NEXT:    s_sext_i32_i16 s3, s3
-; GFX9-NEXT:    s_sext_i32_i16 s2, s2
-; GFX9-NEXT:    s_cmp_gt_i32 s3, s2
-; GFX9-NEXT:    s_cselect_b32 s2, s3, s2
-; GFX9-NEXT:    s_sext_i32_i16 s2, s2
-; GFX9-NEXT:    s_sext_i32_i16 s3, s5
-; GFX9-NEXT:    s_cmp_lt_i32 s2, s3
-; GFX9-NEXT:    s_cselect_b32 s2, s2, s3
-; GFX9-NEXT:    s_add_i32 s1, s1, s2
-; GFX9-NEXT:    s_sext_i32_i16 s1, s1
-; GFX9-NEXT:    s_movk_i32 s2, 0xff
-; GFX9-NEXT:    s_ashr_i32 s1, s1, s4
-; GFX9-NEXT:    s_and_b32 s1, s1, s2
-; GFX9-NEXT:    s_and_b32 s0, s0, s2
-; GFX9-NEXT:    s_lshl_b32 s1, s1, s4
-; GFX9-NEXT:    s_or_b32 s0, s0, s1
+; GFX9-NEXT:    s_lshr_b32 s2, s0, 8
+; GFX9-NEXT:    v_mov_b32_e32 v0, s1
+; GFX9-NEXT:    s_lshl_b32 s0, s0, s4
+; GFX9-NEXT:    s_lshl_b32 s1, s3, s4
+; GFX9-NEXT:    v_add_i16 v0, s0, v0 clamp
+; GFX9-NEXT:    s_lshl_b32 s0, s2, s4
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_add_i16 v1, s0, v1 clamp
+; GFX9-NEXT:    s_movk_i32 s0, 0xff
+; GFX9-NEXT:    v_and_b32_sdwa v0, sext(v0), s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-NEXT:    v_and_b32_sdwa v1, sext(v1), s0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_saddsat_v2i8:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_bfe_u32 s2, 8, 0x100000
-; GFX10-NEXT:    s_lshr_b32 s3, s0, 8
-; GFX10-NEXT:    s_lshl_b32 s0, s0, s2
-; GFX10-NEXT:    s_sext_i32_i16 s6, 0
-; GFX10-NEXT:    s_sext_i32_i16 s5, s0
+; GFX10-NEXT:    s_lshr_b32 s2, s0, 8
+; GFX10-NEXT:    s_bfe_u32 s3, 8, 0x100000
 ; GFX10-NEXT:    s_lshr_b32 s4, s1, 8
-; GFX10-NEXT:    s_lshl_b32 s1, s1, s2
-; GFX10-NEXT:    s_cmp_gt_i32 s5, s6
-; GFX10-NEXT:    s_movk_i32 s7, 0x7fff
-; GFX10-NEXT:    s_cselect_b32 s8, s5, s6
-; GFX10-NEXT:    s_movk_i32 s9, 0x8000
-; GFX10-NEXT:    s_sub_i32 s8, s7, s8
-; GFX10-NEXT:    s_cmp_lt_i32 s5, s6
-; GFX10-NEXT:    s_sext_i32_i16 s1, s1
-; GFX10-NEXT:    s_cselect_b32 s5, s5, s6
+; GFX10-NEXT:    s_lshl_b32 s0, s0, s3
+; GFX10-NEXT:    s_lshl_b32 s1, s1, s3
+; GFX10-NEXT:    s_lshl_b32 s2, s2, s3
+; GFX10-NEXT:    s_lshl_b32 s3, s4, s3
+; GFX10-NEXT:    v_add_nc_i16 v0, s0, s1 clamp
+; GFX10-NEXT:    v_add_nc_i16 v1, s2, s3 clamp
+; GFX10-NEXT:    s_movk_i32 s0, 0xff
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_sub_i32 s5, s9, s5
-; GFX10-NEXT:    s_sext_i32_i16 s5, s5
-; GFX10-NEXT:    s_cmp_gt_i32 s5, s1
-; GFX10-NEXT:    s_cselect_b32 s1, s5, s1
-; GFX10-NEXT:    s_sext_i32_i16 s5, s8
-; GFX10-NEXT:    s_sext_i32_i16 s1, s1
-; GFX10-NEXT:    s_cmp_lt_i32 s1, s5
-; GFX10-NEXT:    s_cselect_b32 s1, s1, s5
-; GFX10-NEXT:    s_lshl_b32 s3, s3, s2
-; GFX10-NEXT:    s_add_i32 s0, s0, s1
-; GFX10-NEXT:    s_lshl_b32 s1, s4, s2
-; GFX10-NEXT:    s_sext_i32_i16 s0, s0
-; GFX10-NEXT:    s_sext_i32_i16 s4, s3
-; GFX10-NEXT:    s_ashr_i32 s0, s0, s2
-; GFX10-NEXT:    s_cmp_gt_i32 s4, s6
-; GFX10-NEXT:    s_sext_i32_i16 s1, s1
-; GFX10-NEXT:    s_cselect_b32 s5, s4, s6
-; GFX10-NEXT:    s_sub_i32 s5, s7, s5
-; GFX10-NEXT:    s_cmp_lt_i32 s4, s6
-; GFX10-NEXT:    s_cselect_b32 s4, s4, s6
-; GFX10-NEXT:    s_sub_i32 s4, s9, s4
-; GFX10-NEXT:    s_sext_i32_i16 s4, s4
-; GFX10-NEXT:    s_cmp_gt_i32 s4, s1
-; GFX10-NEXT:    s_cselect_b32 s1, s4, s1
-; GFX10-NEXT:    s_sext_i32_i16 s4, s5
-; GFX10-NEXT:    s_sext_i32_i16 s1, s1
-; GFX10-NEXT:    s_cmp_lt_i32 s1, s4
-; GFX10-NEXT:    s_cselect_b32 s1, s1, s4
-; GFX10-NEXT:    s_add_i32 s3, s3, s1
-; GFX10-NEXT:    s_sext_i32_i16 s1, s3
-; GFX10-NEXT:    s_movk_i32 s3, 0xff
-; GFX10-NEXT:    s_ashr_i32 s1, s1, s2
-; GFX10-NEXT:    s_and_b32 s0, s0, s3
-; GFX10-NEXT:    s_and_b32 s1, s1, s3
-; GFX10-NEXT:    s_lshl_b32 s1, s1, s2
-; GFX10-NEXT:    s_or_b32 s0, s0, s1
+; GFX10-NEXT:    v_and_b32_sdwa v0, sext(v0), s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_and_b32_sdwa v1, sext(v1), s0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
   %lhs = bitcast i16 %lhs.arg to <2 x i8>
   %rhs = bitcast i16 %rhs.arg to <2 x i8>
@@ -815,52 +629,25 @@ define i32 @v_saddsat_v4i8(i32 %lhs.arg, i32 %rhs.arg) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    s_mov_b32 s4, 8
 ; GFX9-NEXT:    v_lshrrev_b32_sdwa v2, s4, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT:    v_lshrrev_b32_sdwa v5, s4, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v4, 24, v0
-; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; GFX9-NEXT:    s_movk_i32 s5, 0x8000
-; GFX9-NEXT:    v_min_i16_e32 v10, 0, v0
-; GFX9-NEXT:    v_lshrrev_b32_sdwa v5, s4, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v6, 16, v1
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v7, 24, v1
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
-; GFX9-NEXT:    v_sub_u16_e32 v10, s5, v10
-; GFX9-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX9-NEXT:    v_max_i16_e32 v8, 0, v0
-; GFX9-NEXT:    v_sub_u16_e32 v8, s4, v8
-; GFX9-NEXT:    v_max_i16_e32 v1, v10, v1
-; GFX9-NEXT:    v_min_i16_e32 v1, v1, v8
-; GFX9-NEXT:    v_min_i16_e32 v8, 0, v2
-; GFX9-NEXT:    v_add_u16_e32 v0, v0, v1
-; GFX9-NEXT:    v_max_i16_e32 v1, 0, v2
-; GFX9-NEXT:    v_sub_u16_e32 v8, s5, v8
-; GFX9-NEXT:    v_sub_u16_e32 v1, s4, v1
-; GFX9-NEXT:    v_max_i16_e32 v5, v8, v5
-; GFX9-NEXT:    v_min_i16_e32 v1, v5, v1
-; GFX9-NEXT:    v_add_u16_e32 v1, v2, v1
+; GFX9-NEXT:    v_add_i16 v0, v0, v1 clamp
+; GFX9-NEXT:    v_add_i16 v1, v2, v5 clamp
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v2, 8, v3
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v3, 8, v6
-; GFX9-NEXT:    v_min_i16_e32 v6, 0, v2
-; GFX9-NEXT:    v_sub_u16_e32 v6, s5, v6
-; GFX9-NEXT:    v_mov_b32_e32 v9, 0x7fff
-; GFX9-NEXT:    v_max_i16_e32 v5, 0, v2
-; GFX9-NEXT:    v_sub_u16_e32 v5, v9, v5
-; GFX9-NEXT:    v_max_i16_e32 v3, v6, v3
-; GFX9-NEXT:    v_min_i16_e32 v3, v3, v5
-; GFX9-NEXT:    v_add_u16_e32 v2, v2, v3
-; GFX9-NEXT:    v_lshlrev_b16_e32 v3, 8, v4
-; GFX9-NEXT:    v_min_i16_e32 v6, 0, v3
-; GFX9-NEXT:    v_max_i16_e32 v5, 0, v3
-; GFX9-NEXT:    v_lshlrev_b16_e32 v4, 8, v7
-; GFX9-NEXT:    v_sub_u16_e32 v6, 0x8000, v6
 ; GFX9-NEXT:    s_movk_i32 s4, 0xff
-; GFX9-NEXT:    v_sub_u16_e32 v5, v9, v5
-; GFX9-NEXT:    v_max_i16_e32 v4, v6, v4
 ; GFX9-NEXT:    v_and_b32_sdwa v1, sext(v1), s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
-; GFX9-NEXT:    v_min_i16_e32 v4, v4, v5
+; GFX9-NEXT:    v_add_i16 v2, v2, v3 clamp
+; GFX9-NEXT:    v_lshlrev_b16_e32 v3, 8, v4
+; GFX9-NEXT:    v_lshlrev_b16_e32 v4, 8, v7
 ; GFX9-NEXT:    v_ashrrev_i16_e32 v0, 8, v0
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
-; GFX9-NEXT:    v_add_u16_e32 v3, v3, v4
+; GFX9-NEXT:    v_add_i16 v3, v3, v4 clamp
 ; GFX9-NEXT:    v_and_or_b32 v0, v0, s4, v1
 ; GFX9-NEXT:    v_and_b32_sdwa v1, sext(v2), s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
 ; GFX9-NEXT:    v_and_b32_sdwa v2, sext(v3), s4 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
@@ -871,57 +658,30 @@ define i32 @v_saddsat_v4i8(i32 %lhs.arg, i32 %rhs.arg) {
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_lshlrev_b16_e64 v4, 8, v0
 ; GFX10-NEXT:    s_mov_b32 s4, 8
-; GFX10-NEXT:    s_mov_b32 s5, 16
+; GFX10-NEXT:    v_lshlrev_b16_e64 v5, 8, v0
 ; GFX10-NEXT:    v_lshrrev_b32_sdwa v2, s4, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT:    v_lshrrev_b32_sdwa v3, s5, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT:    v_min_i16_e64 v8, v4, 0
-; GFX10-NEXT:    v_lshrrev_b32_sdwa v6, s5, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT:    s_movk_i32 s5, 0x8000
-; GFX10-NEXT:    v_min_i16_e64 v9, v2, 0
-; GFX10-NEXT:    v_lshlrev_b16_e64 v7, 8, v1
-; GFX10-NEXT:    v_sub_nc_u16_e64 v8, s5, v8
-; GFX10-NEXT:    v_max_i16_e64 v10, v4, 0
-; GFX10-NEXT:    s_mov_b32 s6, 24
-; GFX10-NEXT:    v_lshrrev_b32_sdwa v5, s4, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX10-NEXT:    v_sub_nc_u16_e64 v15, s5, v9
-; GFX10-NEXT:    v_lshrrev_b32_sdwa v0, s6, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT:    v_max_i16_e64 v11, v2, 0
-; GFX10-NEXT:    v_max_i16_e64 v7, v8, v7
-; GFX10-NEXT:    v_sub_nc_u16_e64 v10, s4, v10
-; GFX10-NEXT:    v_max_i16_e64 v5, v15, v5
-; GFX10-NEXT:    v_lshrrev_b32_sdwa v1, s6, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT:    v_sub_nc_u16_e64 v8, s4, v11
-; GFX10-NEXT:    v_min_i16_e64 v11, v3, 0
-; GFX10-NEXT:    v_min_i16_e64 v7, v7, v10
-; GFX10-NEXT:    v_min_i16_e64 v10, v0, 0
-; GFX10-NEXT:    v_mov_b32_e32 v9, 0x7fff
-; GFX10-NEXT:    v_min_i16_e64 v5, v5, v8
-; GFX10-NEXT:    v_sub_nc_u16_e64 v11, s5, v11
-; GFX10-NEXT:    v_max_i16_e64 v8, v3, 0
-; GFX10-NEXT:    v_sub_nc_u16_e64 v10, 0x8000, v10
-; GFX10-NEXT:    v_max_i16_e64 v12, v0, 0
-; GFX10-NEXT:    v_add_nc_u16_e64 v2, v2, v5
-; GFX10-NEXT:    v_max_i16_e64 v6, v11, v6
-; GFX10-NEXT:    v_sub_nc_u16_e64 v5, v9, v8
-; GFX10-NEXT:    v_max_i16_e64 v1, v10, v1
-; GFX10-NEXT:    v_sub_nc_u16_e64 v8, v9, v12
-; GFX10-NEXT:    s_movk_i32 s4, 0xff
-; GFX10-NEXT:    v_add_nc_u16_e64 v4, v4, v7
-; GFX10-NEXT:    v_and_b32_sdwa v2, sext(v2), s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
-; GFX10-NEXT:    v_min_i16_e64 v5, v6, v5
-; GFX10-NEXT:    v_min_i16_e64 v1, v1, v8
+; GFX10-NEXT:    v_lshrrev_b32_sdwa v3, s4, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    v_lshlrev_b16_e64 v6, 8, v1
+; GFX10-NEXT:    s_mov_b32 s5, 16
+; GFX10-NEXT:    s_mov_b32 s4, 24
+; GFX10-NEXT:    v_lshrrev_b32_sdwa v4, s5, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    v_add_nc_i16 v2, v2, v3 clamp
+; GFX10-NEXT:    v_lshrrev_b32_sdwa v3, s5, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    s_movk_i32 s5, 0xff
+; GFX10-NEXT:    v_lshrrev_b32_sdwa v0, s4, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    v_lshrrev_b32_sdwa v1, s4, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    v_and_b32_sdwa v2, sext(v2), s5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_add_nc_i16 v5, v5, v6 clamp
+; GFX10-NEXT:    v_add_nc_i16 v3, v4, v3 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_ashrrev_i16_e64 v4, 8, v4
+; GFX10-NEXT:    v_add_nc_i16 v0, v0, v1 clamp
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
-; GFX10-NEXT:    v_add_nc_u16_e64 v3, v3, v5
-; GFX10-NEXT:    v_add_nc_u16_e64 v0, v0, v1
-; GFX10-NEXT:    v_and_or_b32 v1, v4, s4, v2
-; GFX10-NEXT:    v_and_b32_sdwa v2, sext(v3), s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
-; GFX10-NEXT:    v_and_b32_sdwa v0, sext(v0), s4 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
-; GFX10-NEXT:    v_or3_b32 v0, v1, v2, v0
+; GFX10-NEXT:    v_ashrrev_i16_e64 v4, 8, v5
+; GFX10-NEXT:    v_and_b32_sdwa v1, sext(v3), s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_and_b32_sdwa v0, sext(v0), s5 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_and_or_b32 v2, v4, s5, v2
+; GFX10-NEXT:    v_or3_b32 v0, v2, v1, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %lhs = bitcast i32 %lhs.arg to <4 x i8>
   %rhs = bitcast i32 %rhs.arg to <4 x i8>
@@ -1118,212 +878,70 @@ define amdgpu_ps i32 @s_saddsat_v4i8(i32 inreg %lhs.arg, i32 inreg %rhs.arg) {
 ; GFX9-LABEL: s_saddsat_v4i8:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_bfe_u32 s8, 8, 0x100000
-; GFX9-NEXT:    s_lshr_b32 s2, s0, 8
-; GFX9-NEXT:    s_lshr_b32 s3, s0, 16
-; GFX9-NEXT:    s_lshr_b32 s4, s0, 24
-; GFX9-NEXT:    s_lshl_b32 s0, s0, s8
 ; GFX9-NEXT:    s_lshr_b32 s5, s1, 8
 ; GFX9-NEXT:    s_lshr_b32 s6, s1, 16
 ; GFX9-NEXT:    s_lshr_b32 s7, s1, 24
 ; GFX9-NEXT:    s_lshl_b32 s1, s1, s8
-; GFX9-NEXT:    s_sext_i32_i16 s11, s0
-; GFX9-NEXT:    s_sext_i32_i16 s12, 0
-; GFX9-NEXT:    s_cmp_gt_i32 s11, s12
-; GFX9-NEXT:    s_movk_i32 s9, 0x7fff
-; GFX9-NEXT:    s_cselect_b32 s13, s11, s12
-; GFX9-NEXT:    s_sub_i32 s13, s9, s13
-; GFX9-NEXT:    s_cmp_lt_i32 s11, s12
-; GFX9-NEXT:    s_movk_i32 s10, 0x8000
-; GFX9-NEXT:    s_cselect_b32 s11, s11, s12
-; GFX9-NEXT:    s_sub_i32 s11, s10, s11
-; GFX9-NEXT:    s_sext_i32_i16 s11, s11
-; GFX9-NEXT:    s_sext_i32_i16 s1, s1
-; GFX9-NEXT:    s_cmp_gt_i32 s11, s1
-; GFX9-NEXT:    s_cselect_b32 s1, s11, s1
-; GFX9-NEXT:    s_sext_i32_i16 s1, s1
-; GFX9-NEXT:    s_sext_i32_i16 s11, s13
-; GFX9-NEXT:    s_cmp_lt_i32 s1, s11
-; GFX9-NEXT:    s_cselect_b32 s1, s1, s11
-; GFX9-NEXT:    s_add_i32 s0, s0, s1
-; GFX9-NEXT:    s_sext_i32_i16 s0, s0
-; GFX9-NEXT:    s_lshl_b32 s1, s2, s8
-; GFX9-NEXT:    s_lshl_b32 s2, s5, s8
-; GFX9-NEXT:    s_ashr_i32 s0, s0, s8
-; GFX9-NEXT:    s_sext_i32_i16 s5, s1
-; GFX9-NEXT:    s_cmp_gt_i32 s5, s12
-; GFX9-NEXT:    s_cselect_b32 s11, s5, s12
-; GFX9-NEXT:    s_sub_i32 s11, s9, s11
-; GFX9-NEXT:    s_cmp_lt_i32 s5, s12
-; GFX9-NEXT:    s_cselect_b32 s5, s5, s12
-; GFX9-NEXT:    s_sub_i32 s5, s10, s5
-; GFX9-NEXT:    s_sext_i32_i16 s5, s5
-; GFX9-NEXT:    s_sext_i32_i16 s2, s2
-; GFX9-NEXT:    s_cmp_gt_i32 s5, s2
-; GFX9-NEXT:    s_cselect_b32 s2, s5, s2
-; GFX9-NEXT:    s_sext_i32_i16 s2, s2
-; GFX9-NEXT:    s_sext_i32_i16 s5, s11
-; GFX9-NEXT:    s_cmp_lt_i32 s2, s5
-; GFX9-NEXT:    s_cselect_b32 s2, s2, s5
-; GFX9-NEXT:    s_add_i32 s1, s1, s2
-; GFX9-NEXT:    s_sext_i32_i16 s1, s1
-; GFX9-NEXT:    s_lshl_b32 s2, s3, s8
-; GFX9-NEXT:    s_lshl_b32 s3, s6, s8
-; GFX9-NEXT:    s_ashr_i32 s1, s1, s8
-; GFX9-NEXT:    s_sext_i32_i16 s5, s2
-; GFX9-NEXT:    s_cmp_gt_i32 s5, s12
-; GFX9-NEXT:    s_cselect_b32 s6, s5, s12
-; GFX9-NEXT:    s_sub_i32 s6, s9, s6
-; GFX9-NEXT:    s_cmp_lt_i32 s5, s12
-; GFX9-NEXT:    s_cselect_b32 s5, s5, s12
-; GFX9-NEXT:    s_sub_i32 s5, s10, s5
-; GFX9-NEXT:    s_sext_i32_i16 s5, s5
-; GFX9-NEXT:    s_sext_i32_i16 s3, s3
-; GFX9-NEXT:    s_cmp_gt_i32 s5, s3
-; GFX9-NEXT:    s_cselect_b32 s3, s5, s3
-; GFX9-NEXT:    s_sext_i32_i16 s3, s3
-; GFX9-NEXT:    s_sext_i32_i16 s5, s6
-; GFX9-NEXT:    s_cmp_lt_i32 s3, s5
-; GFX9-NEXT:    s_cselect_b32 s3, s3, s5
-; GFX9-NEXT:    s_add_i32 s2, s2, s3
-; GFX9-NEXT:    s_sext_i32_i16 s2, s2
-; GFX9-NEXT:    s_lshl_b32 s3, s4, s8
-; GFX9-NEXT:    s_lshl_b32 s4, s7, s8
-; GFX9-NEXT:    s_ashr_i32 s2, s2, s8
-; GFX9-NEXT:    s_sext_i32_i16 s5, s3
-; GFX9-NEXT:    s_cmp_gt_i32 s5, s12
-; GFX9-NEXT:    s_cselect_b32 s6, s5, s12
-; GFX9-NEXT:    s_sub_i32 s6, s9, s6
-; GFX9-NEXT:    s_cmp_lt_i32 s5, s12
-; GFX9-NEXT:    s_cselect_b32 s5, s5, s12
-; GFX9-NEXT:    s_sub_i32 s5, s10, s5
-; GFX9-NEXT:    s_sext_i32_i16 s5, s5
-; GFX9-NEXT:    s_sext_i32_i16 s4, s4
-; GFX9-NEXT:    s_cmp_gt_i32 s5, s4
-; GFX9-NEXT:    s_cselect_b32 s4, s5, s4
-; GFX9-NEXT:    s_sext_i32_i16 s4, s4
-; GFX9-NEXT:    s_sext_i32_i16 s5, s6
-; GFX9-NEXT:    s_cmp_lt_i32 s4, s5
-; GFX9-NEXT:    s_cselect_b32 s4, s4, s5
-; GFX9-NEXT:    s_add_i32 s3, s3, s4
-; GFX9-NEXT:    s_movk_i32 s4, 0xff
-; GFX9-NEXT:    s_and_b32 s1, s1, s4
-; GFX9-NEXT:    s_and_b32 s0, s0, s4
-; GFX9-NEXT:    s_lshl_b32 s1, s1, 8
-; GFX9-NEXT:    s_sext_i32_i16 s3, s3
-; GFX9-NEXT:    s_or_b32 s0, s0, s1
-; GFX9-NEXT:    s_and_b32 s1, s2, s4
-; GFX9-NEXT:    s_lshl_b32 s1, s1, 16
-; GFX9-NEXT:    s_ashr_i32 s3, s3, s8
-; GFX9-NEXT:    s_or_b32 s0, s0, s1
-; GFX9-NEXT:    s_and_b32 s1, s3, s4
-; GFX9-NEXT:    s_lshl_b32 s1, s1, 24
-; GFX9-NEXT:    s_or_b32 s0, s0, s1
+; GFX9-NEXT:    v_mov_b32_e32 v0, s1
+; GFX9-NEXT:    s_lshl_b32 s1, s5, s8
+; GFX9-NEXT:    s_lshr_b32 s2, s0, 8
+; GFX9-NEXT:    s_lshr_b32 s3, s0, 16
+; GFX9-NEXT:    s_lshr_b32 s4, s0, 24
+; GFX9-NEXT:    s_lshl_b32 s0, s0, s8
+; GFX9-NEXT:    v_add_i16 v0, s0, v0 clamp
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    s_lshl_b32 s0, s2, s8
+; GFX9-NEXT:    s_lshl_b32 s1, s6, s8
+; GFX9-NEXT:    v_add_i16 v1, s0, v1 clamp
+; GFX9-NEXT:    v_mov_b32_e32 v2, s1
+; GFX9-NEXT:    s_lshl_b32 s0, s3, s8
+; GFX9-NEXT:    s_lshl_b32 s1, s7, s8
+; GFX9-NEXT:    v_add_i16 v2, s0, v2 clamp
+; GFX9-NEXT:    s_lshl_b32 s0, s4, s8
+; GFX9-NEXT:    v_mov_b32_e32 v3, s1
+; GFX9-NEXT:    v_add_i16 v3, s0, v3 clamp
+; GFX9-NEXT:    s_movk_i32 s0, 0xff
+; GFX9-NEXT:    v_and_b32_sdwa v1, sext(v1), s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-NEXT:    v_ashrrev_i16_e32 v0, 8, v0
+; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX9-NEXT:    v_and_or_b32 v0, v0, s0, v1
+; GFX9-NEXT:    v_and_b32_sdwa v1, sext(v2), s0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-NEXT:    v_and_b32_sdwa v2, sext(v3), s0 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-NEXT:    v_or3_b32 v0, v0, v1, v2
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_saddsat_v4i8:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_bfe_u32 s6, 8, 0x100000
+; GFX10-NEXT:    s_bfe_u32 s5, 8, 0x100000
 ; GFX10-NEXT:    s_lshr_b32 s2, s0, 8
+; GFX10-NEXT:    s_lshr_b32 s6, s1, 8
+; GFX10-NEXT:    s_lshl_b32 s2, s2, s5
+; GFX10-NEXT:    s_lshl_b32 s6, s6, s5
 ; GFX10-NEXT:    s_lshr_b32 s3, s0, 16
+; GFX10-NEXT:    v_add_nc_i16 v1, s2, s6 clamp
 ; GFX10-NEXT:    s_lshr_b32 s4, s0, 24
-; GFX10-NEXT:    s_lshl_b32 s0, s0, s6
-; GFX10-NEXT:    s_sext_i32_i16 s10, 0
-; GFX10-NEXT:    s_sext_i32_i16 s9, s0
-; GFX10-NEXT:    s_lshr_b32 s5, s1, 8
-; GFX10-NEXT:    s_lshr_b32 s7, s1, 16
-; GFX10-NEXT:    s_lshr_b32 s8, s1, 24
-; GFX10-NEXT:    s_lshl_b32 s1, s1, s6
-; GFX10-NEXT:    s_cmp_gt_i32 s9, s10
-; GFX10-NEXT:    s_movk_i32 s11, 0x7fff
-; GFX10-NEXT:    s_cselect_b32 s12, s9, s10
-; GFX10-NEXT:    s_movk_i32 s13, 0x8000
-; GFX10-NEXT:    s_sub_i32 s12, s11, s12
-; GFX10-NEXT:    s_cmp_lt_i32 s9, s10
-; GFX10-NEXT:    s_sext_i32_i16 s1, s1
-; GFX10-NEXT:    s_cselect_b32 s9, s9, s10
+; GFX10-NEXT:    s_movk_i32 s2, 0xff
+; GFX10-NEXT:    s_lshl_b32 s0, s0, s5
+; GFX10-NEXT:    s_lshl_b32 s7, s1, s5
+; GFX10-NEXT:    v_and_b32_sdwa v1, sext(v1), s2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_add_nc_i16 v0, s0, s7 clamp
+; GFX10-NEXT:    s_lshr_b32 s0, s1, 16
+; GFX10-NEXT:    s_lshr_b32 s1, s1, 24
+; GFX10-NEXT:    s_lshl_b32 s3, s3, s5
+; GFX10-NEXT:    s_lshl_b32 s0, s0, s5
+; GFX10-NEXT:    s_lshl_b32 s4, s4, s5
+; GFX10-NEXT:    s_lshl_b32 s1, s1, s5
+; GFX10-NEXT:    v_ashrrev_i16_e64 v0, 8, v0
+; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX10-NEXT:    v_add_nc_i16 v2, s3, s0 clamp
+; GFX10-NEXT:    v_add_nc_i16 v3, s4, s1 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_sub_i32 s9, s13, s9
-; GFX10-NEXT:    s_sext_i32_i16 s9, s9
-; GFX10-NEXT:    s_cmp_gt_i32 s9, s1
-; GFX10-NEXT:    s_cselect_b32 s1, s9, s1
-; GFX10-NEXT:    s_sext_i32_i16 s9, s12
-; GFX10-NEXT:    s_sext_i32_i16 s1, s1
-; GFX10-NEXT:    s_cmp_lt_i32 s1, s9
-; GFX10-NEXT:    s_cselect_b32 s1, s1, s9
-; GFX10-NEXT:    s_lshl_b32 s5, s5, s6
-; GFX10-NEXT:    s_add_i32 s0, s0, s1
-; GFX10-NEXT:    s_lshl_b32 s1, s2, s6
-; GFX10-NEXT:    s_sext_i32_i16 s0, s0
-; GFX10-NEXT:    s_sext_i32_i16 s2, s1
-; GFX10-NEXT:    s_ashr_i32 s0, s0, s6
-; GFX10-NEXT:    s_cmp_gt_i32 s2, s10
-; GFX10-NEXT:    s_sext_i32_i16 s5, s5
-; GFX10-NEXT:    s_cselect_b32 s9, s2, s10
-; GFX10-NEXT:    s_sub_i32 s9, s11, s9
-; GFX10-NEXT:    s_cmp_lt_i32 s2, s10
-; GFX10-NEXT:    s_cselect_b32 s2, s2, s10
-; GFX10-NEXT:    s_sub_i32 s2, s13, s2
-; GFX10-NEXT:    s_sext_i32_i16 s2, s2
-; GFX10-NEXT:    s_cmp_gt_i32 s2, s5
-; GFX10-NEXT:    s_cselect_b32 s2, s2, s5
-; GFX10-NEXT:    s_sext_i32_i16 s5, s9
-; GFX10-NEXT:    s_sext_i32_i16 s2, s2
-; GFX10-NEXT:    s_cmp_lt_i32 s2, s5
-; GFX10-NEXT:    s_cselect_b32 s2, s2, s5
-; GFX10-NEXT:    s_lshl_b32 s3, s3, s6
-; GFX10-NEXT:    s_add_i32 s1, s1, s2
-; GFX10-NEXT:    s_sext_i32_i16 s5, s3
-; GFX10-NEXT:    s_sext_i32_i16 s1, s1
-; GFX10-NEXT:    s_lshl_b32 s2, s7, s6
-; GFX10-NEXT:    s_ashr_i32 s1, s1, s6
-; GFX10-NEXT:    s_cmp_gt_i32 s5, s10
-; GFX10-NEXT:    s_sext_i32_i16 s2, s2
-; GFX10-NEXT:    s_cselect_b32 s7, s5, s10
-; GFX10-NEXT:    s_sub_i32 s7, s11, s7
-; GFX10-NEXT:    s_cmp_lt_i32 s5, s10
-; GFX10-NEXT:    s_cselect_b32 s5, s5, s10
-; GFX10-NEXT:    s_sub_i32 s5, s13, s5
-; GFX10-NEXT:    s_sext_i32_i16 s5, s5
-; GFX10-NEXT:    s_cmp_gt_i32 s5, s2
-; GFX10-NEXT:    s_cselect_b32 s2, s5, s2
-; GFX10-NEXT:    s_sext_i32_i16 s5, s7
-; GFX10-NEXT:    s_sext_i32_i16 s2, s2
-; GFX10-NEXT:    s_cmp_lt_i32 s2, s5
-; GFX10-NEXT:    s_cselect_b32 s2, s2, s5
-; GFX10-NEXT:    s_lshl_b32 s4, s4, s6
-; GFX10-NEXT:    s_add_i32 s3, s3, s2
-; GFX10-NEXT:    s_sext_i32_i16 s5, s4
-; GFX10-NEXT:    s_sext_i32_i16 s3, s3
-; GFX10-NEXT:    s_lshl_b32 s2, s8, s6
-; GFX10-NEXT:    s_ashr_i32 s3, s3, s6
-; GFX10-NEXT:    s_cmp_gt_i32 s5, s10
-; GFX10-NEXT:    s_sext_i32_i16 s2, s2
-; GFX10-NEXT:    s_cselect_b32 s7, s5, s10
-; GFX10-NEXT:    s_sub_i32 s7, s11, s7
-; GFX10-NEXT:    s_cmp_lt_i32 s5, s10
-; GFX10-NEXT:    s_cselect_b32 s5, s5, s10
-; GFX10-NEXT:    s_sub_i32 s5, s13, s5
-; GFX10-NEXT:    s_sext_i32_i16 s5, s5
-; GFX10-NEXT:    s_cmp_gt_i32 s5, s2
-; GFX10-NEXT:    s_cselect_b32 s2, s5, s2
-; GFX10-NEXT:    s_sext_i32_i16 s5, s7
-; GFX10-NEXT:    s_sext_i32_i16 s2, s2
-; GFX10-NEXT:    s_movk_i32 s7, 0xff
-; GFX10-NEXT:    s_cmp_lt_i32 s2, s5
-; GFX10-NEXT:    s_cselect_b32 s2, s2, s5
-; GFX10-NEXT:    s_and_b32 s1, s1, s7
-; GFX10-NEXT:    s_add_i32 s4, s4, s2
-; GFX10-NEXT:    s_and_b32 s2, s3, s7
-; GFX10-NEXT:    s_sext_i32_i16 s3, s4
-; GFX10-NEXT:    s_and_b32 s0, s0, s7
-; GFX10-NEXT:    s_lshl_b32 s1, s1, 8
-; GFX10-NEXT:    s_ashr_i32 s3, s3, s6
-; GFX10-NEXT:    s_or_b32 s0, s0, s1
-; GFX10-NEXT:    s_lshl_b32 s1, s2, 16
-; GFX10-NEXT:    s_and_b32 s2, s3, s7
-; GFX10-NEXT:    s_or_b32 s0, s0, s1
-; GFX10-NEXT:    s_lshl_b32 s1, s2, 24
-; GFX10-NEXT:    s_or_b32 s0, s0, s1
+; GFX10-NEXT:    v_and_or_b32 v0, v0, s2, v1
+; GFX10-NEXT:    v_and_b32_sdwa v1, sext(v2), s2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_and_b32_sdwa v2, sext(v3), s2 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_or3_b32 v0, v0, v1, v2
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
   %lhs = bitcast i32 %lhs.arg to <4 x i8>
   %rhs = bitcast i32 %rhs.arg to <4 x i8>
@@ -1368,14 +986,8 @@ define i24 @v_saddsat_i24(i24 %lhs, i24 %rhs) {
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
-; GFX9-NEXT:    v_min_i32_e32 v3, 0, v0
-; GFX9-NEXT:    v_max_i32_e32 v2, 0, v0
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
-; GFX9-NEXT:    v_sub_u32_e32 v3, 0x80000000, v3
-; GFX9-NEXT:    v_sub_u32_e32 v2, 0x7fffffff, v2
-; GFX9-NEXT:    v_max_i32_e32 v1, v3, v1
-; GFX9-NEXT:    v_min_i32_e32 v1, v1, v2
-; GFX9-NEXT:    v_add_u32_e32 v0, v0, v1
+; GFX9-NEXT:    v_add_i32 v0, v0, v1 clamp
 ; GFX9-NEXT:    v_ashrrev_i32_e32 v0, 8, v0
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -1386,13 +998,7 @@ define i24 @v_saddsat_i24(i24 %lhs, i24 %rhs) {
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_min_i32_e32 v2, 0, v0
-; GFX10-NEXT:    v_max_i32_e32 v3, 0, v0
-; GFX10-NEXT:    v_sub_nc_u32_e32 v2, 0x80000000, v2
-; GFX10-NEXT:    v_sub_nc_u32_e32 v3, 0x7fffffff, v3
-; GFX10-NEXT:    v_max_i32_e32 v1, v2, v1
-; GFX10-NEXT:    v_min_i32_e32 v1, v1, v3
-; GFX10-NEXT:    v_add_nc_u32_e32 v0, v0, v1
+; GFX10-NEXT:    v_add_nc_i32 v0, v0, v1 clamp
 ; GFX10-NEXT:    v_ashrrev_i32_e32 v0, 8, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call i24 @llvm.sadd.sat.i24(i24 %lhs, i24 %rhs)
@@ -1439,39 +1045,22 @@ define amdgpu_ps i24 @s_saddsat_i24(i24 inreg %lhs, i24 inreg %rhs) {
 ;
 ; GFX9-LABEL: s_saddsat_i24:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_lshl_b32 s0, s0, 8
 ; GFX9-NEXT:    s_lshl_b32 s1, s1, 8
-; GFX9-NEXT:    s_cmp_gt_i32 s0, 0
-; GFX9-NEXT:    s_cselect_b32 s2, s0, 0
-; GFX9-NEXT:    s_sub_i32 s2, 0x7fffffff, s2
-; GFX9-NEXT:    s_cmp_lt_i32 s0, 0
-; GFX9-NEXT:    s_cselect_b32 s3, s0, 0
-; GFX9-NEXT:    s_sub_i32 s3, 0x80000000, s3
-; GFX9-NEXT:    s_cmp_gt_i32 s3, s1
-; GFX9-NEXT:    s_cselect_b32 s1, s3, s1
-; GFX9-NEXT:    s_cmp_lt_i32 s1, s2
-; GFX9-NEXT:    s_cselect_b32 s1, s1, s2
-; GFX9-NEXT:    s_add_i32 s0, s0, s1
-; GFX9-NEXT:    s_ashr_i32 s0, s0, 8
+; GFX9-NEXT:    s_lshl_b32 s0, s0, 8
+; GFX9-NEXT:    v_mov_b32_e32 v0, s1
+; GFX9-NEXT:    v_add_i32 v0, s0, v0 clamp
+; GFX9-NEXT:    v_ashrrev_i32_e32 v0, 8, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_saddsat_i24:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_lshl_b32 s0, s0, 8
 ; GFX10-NEXT:    s_lshl_b32 s1, s1, 8
-; GFX10-NEXT:    s_cmp_gt_i32 s0, 0
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cselect_b32 s2, s0, 0
-; GFX10-NEXT:    s_sub_i32 s2, 0x7fffffff, s2
-; GFX10-NEXT:    s_cmp_lt_i32 s0, 0
-; GFX10-NEXT:    s_cselect_b32 s3, s0, 0
-; GFX10-NEXT:    s_sub_i32 s3, 0x80000000, s3
-; GFX10-NEXT:    s_cmp_gt_i32 s3, s1
-; GFX10-NEXT:    s_cselect_b32 s1, s3, s1
-; GFX10-NEXT:    s_cmp_lt_i32 s1, s2
-; GFX10-NEXT:    s_cselect_b32 s1, s1, s2
-; GFX10-NEXT:    s_add_i32 s0, s0, s1
-; GFX10-NEXT:    s_ashr_i32 s0, s0, 8
+; GFX10-NEXT:    v_add_nc_i32 v0, s0, s1 clamp
+; GFX10-NEXT:    v_ashrrev_i32_e32 v0, 8, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call i24 @llvm.sadd.sat.i24(i24 %lhs, i24 %rhs)
   ret i24 %result
@@ -1505,27 +1094,15 @@ define i32 @v_saddsat_i32(i32 %lhs, i32 %rhs) {
 ; GFX9-LABEL: v_saddsat_i32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_min_i32_e32 v3, 0, v0
-; GFX9-NEXT:    v_max_i32_e32 v2, 0, v0
-; GFX9-NEXT:    v_sub_u32_e32 v3, 0x80000000, v3
-; GFX9-NEXT:    v_sub_u32_e32 v2, 0x7fffffff, v2
-; GFX9-NEXT:    v_max_i32_e32 v1, v3, v1
-; GFX9-NEXT:    v_min_i32_e32 v1, v1, v2
-; GFX9-NEXT:    v_add_u32_e32 v0, v0, v1
+; GFX9-NEXT:    v_add_i32 v0, v0, v1 clamp
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_saddsat_i32:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_min_i32_e32 v2, 0, v0
-; GFX10-NEXT:    v_max_i32_e32 v3, 0, v0
+; GFX10-NEXT:    v_add_nc_i32 v0, v0, v1 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_sub_nc_u32_e32 v2, 0x80000000, v2
-; GFX10-NEXT:    v_sub_nc_u32_e32 v3, 0x7fffffff, v3
-; GFX10-NEXT:    v_max_i32_e32 v1, v2, v1
-; GFX10-NEXT:    v_min_i32_e32 v1, v1, v3
-; GFX10-NEXT:    v_add_nc_u32_e32 v0, v0, v1
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call i32 @llvm.sadd.sat.i32(i32 %lhs, i32 %rhs)
   ret i32 %result
@@ -1579,33 +1156,16 @@ define amdgpu_ps i32 @s_saddsat_i32(i32 inreg %lhs, i32 inreg %rhs) {
 ;
 ; GFX9-LABEL: s_saddsat_i32:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_cmp_gt_i32 s0, 0
-; GFX9-NEXT:    s_cselect_b32 s2, s0, 0
-; GFX9-NEXT:    s_sub_i32 s2, 0x7fffffff, s2
-; GFX9-NEXT:    s_cmp_lt_i32 s0, 0
-; GFX9-NEXT:    s_cselect_b32 s3, s0, 0
-; GFX9-NEXT:    s_sub_i32 s3, 0x80000000, s3
-; GFX9-NEXT:    s_cmp_gt_i32 s3, s1
-; GFX9-NEXT:    s_cselect_b32 s1, s3, s1
-; GFX9-NEXT:    s_cmp_lt_i32 s1, s2
-; GFX9-NEXT:    s_cselect_b32 s1, s1, s2
-; GFX9-NEXT:    s_add_i32 s0, s0, s1
+; GFX9-NEXT:    v_mov_b32_e32 v0, s1
+; GFX9-NEXT:    v_add_i32 v0, s0, v0 clamp
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_saddsat_i32:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_cmp_gt_i32 s0, 0
+; GFX10-NEXT:    v_add_nc_i32 v0, s0, s1 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cselect_b32 s2, s0, 0
-; GFX10-NEXT:    s_sub_i32 s2, 0x7fffffff, s2
-; GFX10-NEXT:    s_cmp_lt_i32 s0, 0
-; GFX10-NEXT:    s_cselect_b32 s3, s0, 0
-; GFX10-NEXT:    s_sub_i32 s3, 0x80000000, s3
-; GFX10-NEXT:    s_cmp_gt_i32 s3, s1
-; GFX10-NEXT:    s_cselect_b32 s1, s3, s1
-; GFX10-NEXT:    s_cmp_lt_i32 s1, s2
-; GFX10-NEXT:    s_cselect_b32 s1, s1, s2
-; GFX10-NEXT:    s_add_i32 s0, s0, s1
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call i32 @llvm.sadd.sat.i32(i32 %lhs, i32 %rhs)
   ret i32 %result
@@ -1640,29 +1200,13 @@ define amdgpu_ps float @saddsat_i32_sv(i32 inreg %lhs, i32 %rhs) {
 ;
 ; GFX9-LABEL: saddsat_i32_sv:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_cmp_gt_i32 s0, 0
-; GFX9-NEXT:    s_cselect_b32 s1, s0, 0
-; GFX9-NEXT:    s_sub_i32 s1, 0x7fffffff, s1
-; GFX9-NEXT:    s_cmp_lt_i32 s0, 0
-; GFX9-NEXT:    s_cselect_b32 s2, s0, 0
-; GFX9-NEXT:    s_sub_i32 s2, 0x80000000, s2
-; GFX9-NEXT:    v_max_i32_e32 v0, s2, v0
-; GFX9-NEXT:    v_min_i32_e32 v0, s1, v0
-; GFX9-NEXT:    v_add_u32_e32 v0, s0, v0
+; GFX9-NEXT:    v_add_i32 v0, s0, v0 clamp
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: saddsat_i32_sv:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_cmp_gt_i32 s0, 0
+; GFX10-NEXT:    v_add_nc_i32 v0, s0, v0 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cselect_b32 s1, s0, 0
-; GFX10-NEXT:    s_sub_i32 s1, 0x7fffffff, s1
-; GFX10-NEXT:    s_cmp_lt_i32 s0, 0
-; GFX10-NEXT:    s_cselect_b32 s2, s0, 0
-; GFX10-NEXT:    s_sub_i32 s2, 0x80000000, s2
-; GFX10-NEXT:    v_max_i32_e32 v0, s2, v0
-; GFX10-NEXT:    v_min_i32_e32 v0, s1, v0
-; GFX10-NEXT:    v_add_nc_u32_e32 v0, s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call i32 @llvm.sadd.sat.i32(i32 %lhs, i32 %rhs)
   %cast = bitcast i32 %result to float
@@ -1694,25 +1238,13 @@ define amdgpu_ps float @saddsat_i32_vs(i32 %lhs, i32 inreg %rhs) {
 ;
 ; GFX9-LABEL: saddsat_i32_vs:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    v_min_i32_e32 v2, 0, v0
-; GFX9-NEXT:    v_max_i32_e32 v1, 0, v0
-; GFX9-NEXT:    v_sub_u32_e32 v2, 0x80000000, v2
-; GFX9-NEXT:    v_sub_u32_e32 v1, 0x7fffffff, v1
-; GFX9-NEXT:    v_max_i32_e32 v2, s0, v2
-; GFX9-NEXT:    v_min_i32_e32 v1, v2, v1
-; GFX9-NEXT:    v_add_u32_e32 v0, v0, v1
+; GFX9-NEXT:    v_add_i32 v0, v0, s0 clamp
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: saddsat_i32_vs:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    v_min_i32_e32 v1, 0, v0
-; GFX10-NEXT:    v_max_i32_e32 v2, 0, v0
+; GFX10-NEXT:    v_add_nc_i32 v0, v0, s0 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_sub_nc_u32_e32 v1, 0x80000000, v1
-; GFX10-NEXT:    v_sub_nc_u32_e32 v2, 0x7fffffff, v2
-; GFX10-NEXT:    v_max_i32_e32 v1, s0, v1
-; GFX10-NEXT:    v_min_i32_e32 v1, v1, v2
-; GFX10-NEXT:    v_add_nc_u32_e32 v0, v0, v1
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call i32 @llvm.sadd.sat.i32(i32 %lhs, i32 %rhs)
   %cast = bitcast i32 %result to float
@@ -1765,45 +1297,17 @@ define <2 x i32> @v_saddsat_v2i32(<2 x i32> %lhs, <2 x i32> %rhs) {
 ; GFX9-LABEL: v_saddsat_v2i32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    s_brev_b32 s5, 1
-; GFX9-NEXT:    v_min_i32_e32 v5, 0, v0
-; GFX9-NEXT:    v_sub_u32_e32 v5, s5, v5
-; GFX9-NEXT:    s_brev_b32 s4, -2
-; GFX9-NEXT:    v_max_i32_e32 v4, 0, v0
-; GFX9-NEXT:    v_sub_u32_e32 v4, s4, v4
-; GFX9-NEXT:    v_max_i32_e32 v2, v5, v2
-; GFX9-NEXT:    v_min_i32_e32 v2, v2, v4
-; GFX9-NEXT:    v_min_i32_e32 v4, 0, v1
-; GFX9-NEXT:    v_add_u32_e32 v0, v0, v2
-; GFX9-NEXT:    v_max_i32_e32 v2, 0, v1
-; GFX9-NEXT:    v_sub_u32_e32 v4, s5, v4
-; GFX9-NEXT:    v_sub_u32_e32 v2, s4, v2
-; GFX9-NEXT:    v_max_i32_e32 v3, v4, v3
-; GFX9-NEXT:    v_min_i32_e32 v2, v3, v2
-; GFX9-NEXT:    v_add_u32_e32 v1, v1, v2
+; GFX9-NEXT:    v_add_i32 v0, v0, v2 clamp
+; GFX9-NEXT:    v_add_i32 v1, v1, v3 clamp
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_saddsat_v2i32:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_min_i32_e32 v4, 0, v0
-; GFX10-NEXT:    v_min_i32_e32 v5, 0, v1
-; GFX10-NEXT:    s_brev_b32 s4, 1
-; GFX10-NEXT:    v_max_i32_e32 v6, 0, v0
-; GFX10-NEXT:    v_max_i32_e32 v7, 0, v1
-; GFX10-NEXT:    v_sub_nc_u32_e32 v4, s4, v4
-; GFX10-NEXT:    v_sub_nc_u32_e32 v5, s4, v5
-; GFX10-NEXT:    s_brev_b32 s4, -2
+; GFX10-NEXT:    v_add_nc_i32 v0, v0, v2 clamp
+; GFX10-NEXT:    v_add_nc_i32 v1, v1, v3 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_sub_nc_u32_e32 v6, s4, v6
-; GFX10-NEXT:    v_max_i32_e32 v11, v4, v2
-; GFX10-NEXT:    v_sub_nc_u32_e32 v7, s4, v7
-; GFX10-NEXT:    v_max_i32_e32 v10, v5, v3
-; GFX10-NEXT:    v_min_i32_e32 v2, v11, v6
-; GFX10-NEXT:    v_min_i32_e32 v3, v10, v7
-; GFX10-NEXT:    v_add_nc_u32_e32 v0, v0, v2
-; GFX10-NEXT:    v_add_nc_u32_e32 v1, v1, v3
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <2 x i32> @llvm.sadd.sat.v2i32(<2 x i32> %lhs, <2 x i32> %rhs)
   ret <2 x i32> %result
@@ -1868,59 +1372,21 @@ define amdgpu_ps <2 x i32> @s_saddsat_v2i32(<2 x i32> inreg %lhs, <2 x i32> inre
 ;
 ; GFX9-LABEL: s_saddsat_v2i32:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_cmp_gt_i32 s0, 0
-; GFX9-NEXT:    s_brev_b32 s4, -2
-; GFX9-NEXT:    s_cselect_b32 s6, s0, 0
-; GFX9-NEXT:    s_sub_i32 s6, s4, s6
-; GFX9-NEXT:    s_cmp_lt_i32 s0, 0
-; GFX9-NEXT:    s_brev_b32 s5, 1
-; GFX9-NEXT:    s_cselect_b32 s7, s0, 0
-; GFX9-NEXT:    s_sub_i32 s7, s5, s7
-; GFX9-NEXT:    s_cmp_gt_i32 s7, s2
-; GFX9-NEXT:    s_cselect_b32 s2, s7, s2
-; GFX9-NEXT:    s_cmp_lt_i32 s2, s6
-; GFX9-NEXT:    s_cselect_b32 s2, s2, s6
-; GFX9-NEXT:    s_add_i32 s0, s0, s2
-; GFX9-NEXT:    s_cmp_gt_i32 s1, 0
-; GFX9-NEXT:    s_cselect_b32 s2, s1, 0
-; GFX9-NEXT:    s_sub_i32 s2, s4, s2
-; GFX9-NEXT:    s_cmp_lt_i32 s1, 0
-; GFX9-NEXT:    s_cselect_b32 s4, s1, 0
-; GFX9-NEXT:    s_sub_i32 s4, s5, s4
-; GFX9-NEXT:    s_cmp_gt_i32 s4, s3
-; GFX9-NEXT:    s_cselect_b32 s3, s4, s3
-; GFX9-NEXT:    s_cmp_lt_i32 s3, s2
-; GFX9-NEXT:    s_cselect_b32 s2, s3, s2
-; GFX9-NEXT:    s_add_i32 s1, s1, s2
+; GFX9-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    v_add_i32 v0, s0, v0 clamp
+; GFX9-NEXT:    v_add_i32 v1, s1, v1 clamp
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_saddsat_v2i32:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_cmp_gt_i32 s0, 0
-; GFX10-NEXT:    s_brev_b32 s4, -2
-; GFX10-NEXT:    s_cselect_b32 s5, s0, 0
-; GFX10-NEXT:    s_brev_b32 s6, 1
-; GFX10-NEXT:    s_sub_i32 s5, s4, s5
-; GFX10-NEXT:    s_cmp_lt_i32 s0, 0
+; GFX10-NEXT:    v_add_nc_i32 v0, s0, s2 clamp
+; GFX10-NEXT:    v_add_nc_i32 v1, s1, s3 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cselect_b32 s7, s0, 0
-; GFX10-NEXT:    s_sub_i32 s7, s6, s7
-; GFX10-NEXT:    s_cmp_gt_i32 s7, s2
-; GFX10-NEXT:    s_cselect_b32 s2, s7, s2
-; GFX10-NEXT:    s_cmp_lt_i32 s2, s5
-; GFX10-NEXT:    s_cselect_b32 s2, s2, s5
-; GFX10-NEXT:    s_add_i32 s0, s0, s2
-; GFX10-NEXT:    s_cmp_gt_i32 s1, 0
-; GFX10-NEXT:    s_cselect_b32 s2, s1, 0
-; GFX10-NEXT:    s_sub_i32 s2, s4, s2
-; GFX10-NEXT:    s_cmp_lt_i32 s1, 0
-; GFX10-NEXT:    s_cselect_b32 s4, s1, 0
-; GFX10-NEXT:    s_sub_i32 s4, s6, s4
-; GFX10-NEXT:    s_cmp_gt_i32 s4, s3
-; GFX10-NEXT:    s_cselect_b32 s3, s4, s3
-; GFX10-NEXT:    s_cmp_lt_i32 s3, s2
-; GFX10-NEXT:    s_cselect_b32 s2, s3, s2
-; GFX10-NEXT:    s_add_i32 s1, s1, s2
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <2 x i32> @llvm.sadd.sat.v2i32(<2 x i32> %lhs, <2 x i32> %rhs)
   ret <2 x i32> %result
@@ -1986,59 +1452,19 @@ define <3 x i32> @v_saddsat_v3i32(<3 x i32> %lhs, <3 x i32> %rhs) {
 ; GFX9-LABEL: v_saddsat_v3i32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    s_brev_b32 s5, 1
-; GFX9-NEXT:    v_min_i32_e32 v7, 0, v0
-; GFX9-NEXT:    v_sub_u32_e32 v7, s5, v7
-; GFX9-NEXT:    s_brev_b32 s4, -2
-; GFX9-NEXT:    v_max_i32_e32 v6, 0, v0
-; GFX9-NEXT:    v_sub_u32_e32 v6, s4, v6
-; GFX9-NEXT:    v_max_i32_e32 v3, v7, v3
-; GFX9-NEXT:    v_min_i32_e32 v3, v3, v6
-; GFX9-NEXT:    v_min_i32_e32 v6, 0, v1
-; GFX9-NEXT:    v_add_u32_e32 v0, v0, v3
-; GFX9-NEXT:    v_max_i32_e32 v3, 0, v1
-; GFX9-NEXT:    v_sub_u32_e32 v6, s5, v6
-; GFX9-NEXT:    v_sub_u32_e32 v3, s4, v3
-; GFX9-NEXT:    v_max_i32_e32 v4, v6, v4
-; GFX9-NEXT:    v_min_i32_e32 v3, v4, v3
-; GFX9-NEXT:    v_min_i32_e32 v4, 0, v2
-; GFX9-NEXT:    v_sub_u32_e32 v4, s5, v4
-; GFX9-NEXT:    v_add_u32_e32 v1, v1, v3
-; GFX9-NEXT:    v_max_i32_e32 v3, 0, v2
-; GFX9-NEXT:    v_sub_u32_e32 v3, s4, v3
-; GFX9-NEXT:    v_max_i32_e32 v4, v4, v5
-; GFX9-NEXT:    v_min_i32_e32 v3, v4, v3
-; GFX9-NEXT:    v_add_u32_e32 v2, v2, v3
+; GFX9-NEXT:    v_add_i32 v0, v0, v3 clamp
+; GFX9-NEXT:    v_add_i32 v1, v1, v4 clamp
+; GFX9-NEXT:    v_add_i32 v2, v2, v5 clamp
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_saddsat_v3i32:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_min_i32_e32 v7, 0, v0
-; GFX10-NEXT:    v_min_i32_e32 v8, 0, v1
-; GFX10-NEXT:    v_min_i32_e32 v9, 0, v2
-; GFX10-NEXT:    s_brev_b32 s5, 1
-; GFX10-NEXT:    v_max_i32_e32 v6, 0, v0
-; GFX10-NEXT:    v_sub_nc_u32_e32 v14, s5, v7
-; GFX10-NEXT:    v_sub_nc_u32_e32 v15, s5, v8
-; GFX10-NEXT:    v_sub_nc_u32_e32 v19, s5, v9
-; GFX10-NEXT:    v_max_i32_e32 v10, 0, v1
-; GFX10-NEXT:    v_max_i32_e32 v11, 0, v2
-; GFX10-NEXT:    s_brev_b32 s4, -2
-; GFX10-NEXT:    v_max_i32_e32 v3, v14, v3
-; GFX10-NEXT:    v_sub_nc_u32_e32 v6, s4, v6
-; GFX10-NEXT:    v_sub_nc_u32_e32 v7, s4, v10
-; GFX10-NEXT:    v_max_i32_e32 v4, v15, v4
-; GFX10-NEXT:    v_sub_nc_u32_e32 v8, s4, v11
-; GFX10-NEXT:    v_max_i32_e32 v5, v19, v5
-; GFX10-NEXT:    v_min_i32_e32 v3, v3, v6
+; GFX10-NEXT:    v_add_nc_i32 v0, v0, v3 clamp
+; GFX10-NEXT:    v_add_nc_i32 v1, v1, v4 clamp
+; GFX10-NEXT:    v_add_nc_i32 v2, v2, v5 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_min_i32_e32 v4, v4, v7
-; GFX10-NEXT:    v_min_i32_e32 v5, v5, v8
-; GFX10-NEXT:    v_add_nc_u32_e32 v0, v0, v3
-; GFX10-NEXT:    v_add_nc_u32_e32 v1, v1, v4
-; GFX10-NEXT:    v_add_nc_u32_e32 v2, v2, v5
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <3 x i32> @llvm.sadd.sat.v3i32(<3 x i32> %lhs, <3 x i32> %rhs)
   ret <3 x i32> %result
@@ -2125,81 +1551,26 @@ define amdgpu_ps <3 x i32> @s_saddsat_v3i32(<3 x i32> inreg %lhs, <3 x i32> inre
 ;
 ; GFX9-LABEL: s_saddsat_v3i32:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_cmp_gt_i32 s0, 0
-; GFX9-NEXT:    s_brev_b32 s6, -2
-; GFX9-NEXT:    s_cselect_b32 s8, s0, 0
-; GFX9-NEXT:    s_sub_i32 s8, s6, s8
-; GFX9-NEXT:    s_cmp_lt_i32 s0, 0
-; GFX9-NEXT:    s_brev_b32 s7, 1
-; GFX9-NEXT:    s_cselect_b32 s9, s0, 0
-; GFX9-NEXT:    s_sub_i32 s9, s7, s9
-; GFX9-NEXT:    s_cmp_gt_i32 s9, s3
-; GFX9-NEXT:    s_cselect_b32 s3, s9, s3
-; GFX9-NEXT:    s_cmp_lt_i32 s3, s8
-; GFX9-NEXT:    s_cselect_b32 s3, s3, s8
-; GFX9-NEXT:    s_add_i32 s0, s0, s3
-; GFX9-NEXT:    s_cmp_gt_i32 s1, 0
-; GFX9-NEXT:    s_cselect_b32 s3, s1, 0
-; GFX9-NEXT:    s_sub_i32 s3, s6, s3
-; GFX9-NEXT:    s_cmp_lt_i32 s1, 0
-; GFX9-NEXT:    s_cselect_b32 s8, s1, 0
-; GFX9-NEXT:    s_sub_i32 s8, s7, s8
-; GFX9-NEXT:    s_cmp_gt_i32 s8, s4
-; GFX9-NEXT:    s_cselect_b32 s4, s8, s4
-; GFX9-NEXT:    s_cmp_lt_i32 s4, s3
-; GFX9-NEXT:    s_cselect_b32 s3, s4, s3
-; GFX9-NEXT:    s_add_i32 s1, s1, s3
-; GFX9-NEXT:    s_cmp_gt_i32 s2, 0
-; GFX9-NEXT:    s_cselect_b32 s3, s2, 0
-; GFX9-NEXT:    s_sub_i32 s3, s6, s3
-; GFX9-NEXT:    s_cmp_lt_i32 s2, 0
-; GFX9-NEXT:    s_cselect_b32 s4, s2, 0
-; GFX9-NEXT:    s_sub_i32 s4, s7, s4
-; GFX9-NEXT:    s_cmp_gt_i32 s4, s5
-; GFX9-NEXT:    s_cselect_b32 s4, s4, s5
-; GFX9-NEXT:    s_cmp_lt_i32 s4, s3
-; GFX9-NEXT:    s_cselect_b32 s3, s4, s3
-; GFX9-NEXT:    s_add_i32 s2, s2, s3
+; GFX9-NEXT:    v_mov_b32_e32 v0, s3
+; GFX9-NEXT:    v_mov_b32_e32 v1, s4
+; GFX9-NEXT:    v_mov_b32_e32 v2, s5
+; GFX9-NEXT:    v_add_i32 v0, s0, v0 clamp
+; GFX9-NEXT:    v_add_i32 v1, s1, v1 clamp
+; GFX9-NEXT:    v_add_i32 v2, s2, v2 clamp
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v2
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_saddsat_v3i32:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_cmp_gt_i32 s0, 0
-; GFX10-NEXT:    s_brev_b32 s6, -2
-; GFX10-NEXT:    s_cselect_b32 s7, s0, 0
-; GFX10-NEXT:    s_brev_b32 s8, 1
-; GFX10-NEXT:    s_sub_i32 s7, s6, s7
-; GFX10-NEXT:    s_cmp_lt_i32 s0, 0
+; GFX10-NEXT:    v_add_nc_i32 v0, s0, s3 clamp
+; GFX10-NEXT:    v_add_nc_i32 v1, s1, s4 clamp
+; GFX10-NEXT:    v_add_nc_i32 v2, s2, s5 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cselect_b32 s9, s0, 0
-; GFX10-NEXT:    s_sub_i32 s9, s8, s9
-; GFX10-NEXT:    s_cmp_gt_i32 s9, s3
-; GFX10-NEXT:    s_cselect_b32 s3, s9, s3
-; GFX10-NEXT:    s_cmp_lt_i32 s3, s7
-; GFX10-NEXT:    s_cselect_b32 s3, s3, s7
-; GFX10-NEXT:    s_add_i32 s0, s0, s3
-; GFX10-NEXT:    s_cmp_gt_i32 s1, 0
-; GFX10-NEXT:    s_cselect_b32 s3, s1, 0
-; GFX10-NEXT:    s_sub_i32 s3, s6, s3
-; GFX10-NEXT:    s_cmp_lt_i32 s1, 0
-; GFX10-NEXT:    s_cselect_b32 s7, s1, 0
-; GFX10-NEXT:    s_sub_i32 s7, s8, s7
-; GFX10-NEXT:    s_cmp_gt_i32 s7, s4
-; GFX10-NEXT:    s_cselect_b32 s4, s7, s4
-; GFX10-NEXT:    s_cmp_lt_i32 s4, s3
-; GFX10-NEXT:    s_cselect_b32 s3, s4, s3
-; GFX10-NEXT:    s_add_i32 s1, s1, s3
-; GFX10-NEXT:    s_cmp_gt_i32 s2, 0
-; GFX10-NEXT:    s_cselect_b32 s3, s2, 0
-; GFX10-NEXT:    s_sub_i32 s3, s6, s3
-; GFX10-NEXT:    s_cmp_lt_i32 s2, 0
-; GFX10-NEXT:    s_cselect_b32 s4, s2, 0
-; GFX10-NEXT:    s_sub_i32 s4, s8, s4
-; GFX10-NEXT:    s_cmp_gt_i32 s4, s5
-; GFX10-NEXT:    s_cselect_b32 s4, s4, s5
-; GFX10-NEXT:    s_cmp_lt_i32 s4, s3
-; GFX10-NEXT:    s_cselect_b32 s3, s4, s3
-; GFX10-NEXT:    s_add_i32 s2, s2, s3
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <3 x i32> @llvm.sadd.sat.v3i32(<3 x i32> %lhs, <3 x i32> %rhs)
   ret <3 x i32> %result
@@ -2279,73 +1650,21 @@ define <4 x i32> @v_saddsat_v4i32(<4 x i32> %lhs, <4 x i32> %rhs) {
 ; GFX9-LABEL: v_saddsat_v4i32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    s_brev_b32 s5, 1
-; GFX9-NEXT:    v_min_i32_e32 v9, 0, v0
-; GFX9-NEXT:    v_sub_u32_e32 v9, s5, v9
-; GFX9-NEXT:    s_brev_b32 s4, -2
-; GFX9-NEXT:    v_max_i32_e32 v8, 0, v0
-; GFX9-NEXT:    v_sub_u32_e32 v8, s4, v8
-; GFX9-NEXT:    v_max_i32_e32 v4, v9, v4
-; GFX9-NEXT:    v_min_i32_e32 v4, v4, v8
-; GFX9-NEXT:    v_min_i32_e32 v8, 0, v1
-; GFX9-NEXT:    v_add_u32_e32 v0, v0, v4
-; GFX9-NEXT:    v_max_i32_e32 v4, 0, v1
-; GFX9-NEXT:    v_sub_u32_e32 v8, s5, v8
-; GFX9-NEXT:    v_sub_u32_e32 v4, s4, v4
-; GFX9-NEXT:    v_max_i32_e32 v5, v8, v5
-; GFX9-NEXT:    v_min_i32_e32 v4, v5, v4
-; GFX9-NEXT:    v_min_i32_e32 v5, 0, v2
-; GFX9-NEXT:    v_sub_u32_e32 v5, s5, v5
-; GFX9-NEXT:    v_add_u32_e32 v1, v1, v4
-; GFX9-NEXT:    v_max_i32_e32 v4, 0, v2
-; GFX9-NEXT:    v_sub_u32_e32 v4, s4, v4
-; GFX9-NEXT:    v_max_i32_e32 v5, v5, v6
-; GFX9-NEXT:    v_min_i32_e32 v4, v5, v4
-; GFX9-NEXT:    v_min_i32_e32 v5, 0, v3
-; GFX9-NEXT:    v_sub_u32_e32 v5, 0x80000000, v5
-; GFX9-NEXT:    v_add_u32_e32 v2, v2, v4
-; GFX9-NEXT:    v_max_i32_e32 v4, 0, v3
-; GFX9-NEXT:    v_sub_u32_e32 v4, 0x7fffffff, v4
-; GFX9-NEXT:    v_max_i32_e32 v5, v5, v7
-; GFX9-NEXT:    v_min_i32_e32 v4, v5, v4
-; GFX9-NEXT:    v_add_u32_e32 v3, v3, v4
+; GFX9-NEXT:    v_add_i32 v0, v0, v4 clamp
+; GFX9-NEXT:    v_add_i32 v1, v1, v5 clamp
+; GFX9-NEXT:    v_add_i32 v2, v2, v6 clamp
+; GFX9-NEXT:    v_add_i32 v3, v3, v7 clamp
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_saddsat_v4i32:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_min_i32_e32 v8, 0, v0
-; GFX10-NEXT:    s_brev_b32 s4, 1
-; GFX10-NEXT:    v_min_i32_e32 v11, 0, v1
-; GFX10-NEXT:    v_min_i32_e32 v12, 0, v3
-; GFX10-NEXT:    v_max_i32_e32 v9, 0, v0
-; GFX10-NEXT:    v_sub_nc_u32_e32 v15, s4, v8
-; GFX10-NEXT:    v_min_i32_e32 v8, 0, v2
-; GFX10-NEXT:    v_sub_nc_u32_e32 v11, s4, v11
-; GFX10-NEXT:    v_sub_nc_u32_e32 v12, 0x80000000, v12
-; GFX10-NEXT:    v_max_i32_e32 v10, 0, v1
-; GFX10-NEXT:    v_max_i32_e32 v13, 0, v2
-; GFX10-NEXT:    v_sub_nc_u32_e32 v8, s4, v8
-; GFX10-NEXT:    v_max_i32_e32 v14, 0, v3
-; GFX10-NEXT:    s_brev_b32 s5, -2
-; GFX10-NEXT:    v_max_i32_e32 v5, v11, v5
-; GFX10-NEXT:    v_sub_nc_u32_e32 v10, s5, v10
-; GFX10-NEXT:    v_max_i32_e32 v6, v8, v6
-; GFX10-NEXT:    v_sub_nc_u32_e32 v11, s5, v13
-; GFX10-NEXT:    v_sub_nc_u32_e32 v9, s5, v9
-; GFX10-NEXT:    v_max_i32_e32 v4, v15, v4
-; GFX10-NEXT:    v_sub_nc_u32_e32 v8, 0x7fffffff, v14
-; GFX10-NEXT:    v_max_i32_e32 v7, v12, v7
-; GFX10-NEXT:    v_min_i32_e32 v11, v6, v11
-; GFX10-NEXT:    v_min_i32_e32 v19, v5, v10
-; GFX10-NEXT:    v_min_i32_e32 v15, v4, v9
+; GFX10-NEXT:    v_add_nc_i32 v0, v0, v4 clamp
+; GFX10-NEXT:    v_add_nc_i32 v1, v1, v5 clamp
+; GFX10-NEXT:    v_add_nc_i32 v2, v2, v6 clamp
+; GFX10-NEXT:    v_add_nc_i32 v3, v3, v7 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_min_i32_e32 v6, v7, v8
-; GFX10-NEXT:    v_add_nc_u32_e32 v2, v2, v11
-; GFX10-NEXT:    v_add_nc_u32_e32 v1, v1, v19
-; GFX10-NEXT:    v_add_nc_u32_e32 v0, v0, v15
-; GFX10-NEXT:    v_add_nc_u32_e32 v3, v3, v6
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> %lhs, <4 x i32> %rhs)
   ret <4 x i32> %result
@@ -2454,103 +1773,31 @@ define amdgpu_ps <4 x i32> @s_saddsat_v4i32(<4 x i32> inreg %lhs, <4 x i32> inre
 ;
 ; GFX9-LABEL: s_saddsat_v4i32:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_cmp_gt_i32 s0, 0
-; GFX9-NEXT:    s_brev_b32 s8, -2
-; GFX9-NEXT:    s_cselect_b32 s10, s0, 0
-; GFX9-NEXT:    s_sub_i32 s10, s8, s10
-; GFX9-NEXT:    s_cmp_lt_i32 s0, 0
-; GFX9-NEXT:    s_brev_b32 s9, 1
-; GFX9-NEXT:    s_cselect_b32 s11, s0, 0
-; GFX9-NEXT:    s_sub_i32 s11, s9, s11
-; GFX9-NEXT:    s_cmp_gt_i32 s11, s4
-; GFX9-NEXT:    s_cselect_b32 s4, s11, s4
-; GFX9-NEXT:    s_cmp_lt_i32 s4, s10
-; GFX9-NEXT:    s_cselect_b32 s4, s4, s10
-; GFX9-NEXT:    s_add_i32 s0, s0, s4
-; GFX9-NEXT:    s_cmp_gt_i32 s1, 0
-; GFX9-NEXT:    s_cselect_b32 s4, s1, 0
-; GFX9-NEXT:    s_sub_i32 s4, s8, s4
-; GFX9-NEXT:    s_cmp_lt_i32 s1, 0
-; GFX9-NEXT:    s_cselect_b32 s10, s1, 0
-; GFX9-NEXT:    s_sub_i32 s10, s9, s10
-; GFX9-NEXT:    s_cmp_gt_i32 s10, s5
-; GFX9-NEXT:    s_cselect_b32 s5, s10, s5
-; GFX9-NEXT:    s_cmp_lt_i32 s5, s4
-; GFX9-NEXT:    s_cselect_b32 s4, s5, s4
-; GFX9-NEXT:    s_add_i32 s1, s1, s4
-; GFX9-NEXT:    s_cmp_gt_i32 s2, 0
-; GFX9-NEXT:    s_cselect_b32 s4, s2, 0
-; GFX9-NEXT:    s_sub_i32 s4, s8, s4
-; GFX9-NEXT:    s_cmp_lt_i32 s2, 0
-; GFX9-NEXT:    s_cselect_b32 s5, s2, 0
-; GFX9-NEXT:    s_sub_i32 s5, s9, s5
-; GFX9-NEXT:    s_cmp_gt_i32 s5, s6
-; GFX9-NEXT:    s_cselect_b32 s5, s5, s6
-; GFX9-NEXT:    s_cmp_lt_i32 s5, s4
-; GFX9-NEXT:    s_cselect_b32 s4, s5, s4
-; GFX9-NEXT:    s_add_i32 s2, s2, s4
-; GFX9-NEXT:    s_cmp_gt_i32 s3, 0
-; GFX9-NEXT:    s_cselect_b32 s4, s3, 0
-; GFX9-NEXT:    s_sub_i32 s4, s8, s4
-; GFX9-NEXT:    s_cmp_lt_i32 s3, 0
-; GFX9-NEXT:    s_cselect_b32 s5, s3, 0
-; GFX9-NEXT:    s_sub_i32 s5, s9, s5
-; GFX9-NEXT:    s_cmp_gt_i32 s5, s7
-; GFX9-NEXT:    s_cselect_b32 s5, s5, s7
-; GFX9-NEXT:    s_cmp_lt_i32 s5, s4
-; GFX9-NEXT:    s_cselect_b32 s4, s5, s4
-; GFX9-NEXT:    s_add_i32 s3, s3, s4
+; GFX9-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-NEXT:    v_mov_b32_e32 v2, s6
+; GFX9-NEXT:    v_mov_b32_e32 v3, s7
+; GFX9-NEXT:    v_add_i32 v0, s0, v0 clamp
+; GFX9-NEXT:    v_add_i32 v1, s1, v1 clamp
+; GFX9-NEXT:    v_add_i32 v2, s2, v2 clamp
+; GFX9-NEXT:    v_add_i32 v3, s3, v3 clamp
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX9-NEXT:    v_readfirstlane_b32 s3, v3
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_saddsat_v4i32:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_cmp_gt_i32 s0, 0
-; GFX10-NEXT:    s_brev_b32 s8, -2
-; GFX10-NEXT:    s_cselect_b32 s9, s0, 0
-; GFX10-NEXT:    s_brev_b32 s10, 1
-; GFX10-NEXT:    s_sub_i32 s9, s8, s9
-; GFX10-NEXT:    s_cmp_lt_i32 s0, 0
+; GFX10-NEXT:    v_add_nc_i32 v0, s0, s4 clamp
+; GFX10-NEXT:    v_add_nc_i32 v1, s1, s5 clamp
+; GFX10-NEXT:    v_add_nc_i32 v2, s2, s6 clamp
+; GFX10-NEXT:    v_add_nc_i32 v3, s3, s7 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cselect_b32 s11, s0, 0
-; GFX10-NEXT:    s_sub_i32 s11, s10, s11
-; GFX10-NEXT:    s_cmp_gt_i32 s11, s4
-; GFX10-NEXT:    s_cselect_b32 s4, s11, s4
-; GFX10-NEXT:    s_cmp_lt_i32 s4, s9
-; GFX10-NEXT:    s_cselect_b32 s4, s4, s9
-; GFX10-NEXT:    s_add_i32 s0, s0, s4
-; GFX10-NEXT:    s_cmp_gt_i32 s1, 0
-; GFX10-NEXT:    s_cselect_b32 s4, s1, 0
-; GFX10-NEXT:    s_sub_i32 s4, s8, s4
-; GFX10-NEXT:    s_cmp_lt_i32 s1, 0
-; GFX10-NEXT:    s_cselect_b32 s9, s1, 0
-; GFX10-NEXT:    s_sub_i32 s9, s10, s9
-; GFX10-NEXT:    s_cmp_gt_i32 s9, s5
-; GFX10-NEXT:    s_cselect_b32 s5, s9, s5
-; GFX10-NEXT:    s_cmp_lt_i32 s5, s4
-; GFX10-NEXT:    s_cselect_b32 s4, s5, s4
-; GFX10-NEXT:    s_add_i32 s1, s1, s4
-; GFX10-NEXT:    s_cmp_gt_i32 s2, 0
-; GFX10-NEXT:    s_cselect_b32 s4, s2, 0
-; GFX10-NEXT:    s_sub_i32 s4, s8, s4
-; GFX10-NEXT:    s_cmp_lt_i32 s2, 0
-; GFX10-NEXT:    s_cselect_b32 s5, s2, 0
-; GFX10-NEXT:    s_sub_i32 s5, s10, s5
-; GFX10-NEXT:    s_cmp_gt_i32 s5, s6
-; GFX10-NEXT:    s_cselect_b32 s5, s5, s6
-; GFX10-NEXT:    s_cmp_lt_i32 s5, s4
-; GFX10-NEXT:    s_cselect_b32 s4, s5, s4
-; GFX10-NEXT:    s_add_i32 s2, s2, s4
-; GFX10-NEXT:    s_cmp_gt_i32 s3, 0
-; GFX10-NEXT:    s_cselect_b32 s4, s3, 0
-; GFX10-NEXT:    s_sub_i32 s4, s8, s4
-; GFX10-NEXT:    s_cmp_lt_i32 s3, 0
-; GFX10-NEXT:    s_cselect_b32 s5, s3, 0
-; GFX10-NEXT:    s_sub_i32 s5, s10, s5
-; GFX10-NEXT:    s_cmp_gt_i32 s5, s7
-; GFX10-NEXT:    s_cselect_b32 s5, s5, s7
-; GFX10-NEXT:    s_cmp_lt_i32 s5, s4
-; GFX10-NEXT:    s_cselect_b32 s4, s5, s4
-; GFX10-NEXT:    s_add_i32 s3, s3, s4
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX10-NEXT:    v_readfirstlane_b32 s3, v3
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> %lhs, <4 x i32> %rhs)
   ret <4 x i32> %result
@@ -2648,90 +1895,22 @@ define <5 x i32> @v_saddsat_v5i32(<5 x i32> %lhs, <5 x i32> %rhs) {
 ; GFX9-LABEL: v_saddsat_v5i32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    s_brev_b32 s5, 1
-; GFX9-NEXT:    v_min_i32_e32 v12, 0, v0
-; GFX9-NEXT:    v_sub_u32_e32 v12, s5, v12
-; GFX9-NEXT:    s_brev_b32 s4, -2
-; GFX9-NEXT:    v_max_i32_e32 v10, 0, v0
-; GFX9-NEXT:    v_sub_u32_e32 v10, s4, v10
-; GFX9-NEXT:    v_max_i32_e32 v5, v12, v5
-; GFX9-NEXT:    v_min_i32_e32 v5, v5, v10
-; GFX9-NEXT:    v_min_i32_e32 v10, 0, v1
-; GFX9-NEXT:    v_add_u32_e32 v0, v0, v5
-; GFX9-NEXT:    v_max_i32_e32 v5, 0, v1
-; GFX9-NEXT:    v_sub_u32_e32 v10, s5, v10
-; GFX9-NEXT:    v_sub_u32_e32 v5, s4, v5
-; GFX9-NEXT:    v_max_i32_e32 v6, v10, v6
-; GFX9-NEXT:    v_min_i32_e32 v5, v6, v5
-; GFX9-NEXT:    v_min_i32_e32 v6, 0, v2
-; GFX9-NEXT:    v_sub_u32_e32 v6, s5, v6
-; GFX9-NEXT:    v_add_u32_e32 v1, v1, v5
-; GFX9-NEXT:    v_max_i32_e32 v5, 0, v2
-; GFX9-NEXT:    v_sub_u32_e32 v5, s4, v5
-; GFX9-NEXT:    v_max_i32_e32 v6, v6, v7
-; GFX9-NEXT:    v_min_i32_e32 v5, v6, v5
-; GFX9-NEXT:    v_bfrev_b32_e32 v13, 1
-; GFX9-NEXT:    v_min_i32_e32 v6, 0, v3
-; GFX9-NEXT:    v_sub_u32_e32 v6, v13, v6
-; GFX9-NEXT:    v_add_u32_e32 v2, v2, v5
-; GFX9-NEXT:    v_bfrev_b32_e32 v11, -2
-; GFX9-NEXT:    v_max_i32_e32 v5, 0, v3
-; GFX9-NEXT:    v_sub_u32_e32 v5, v11, v5
-; GFX9-NEXT:    v_max_i32_e32 v6, v6, v8
-; GFX9-NEXT:    v_min_i32_e32 v5, v6, v5
-; GFX9-NEXT:    v_min_i32_e32 v6, 0, v4
-; GFX9-NEXT:    v_sub_u32_e32 v6, v13, v6
-; GFX9-NEXT:    v_add_u32_e32 v3, v3, v5
-; GFX9-NEXT:    v_max_i32_e32 v5, 0, v4
-; GFX9-NEXT:    v_sub_u32_e32 v5, v11, v5
-; GFX9-NEXT:    v_max_i32_e32 v6, v6, v9
-; GFX9-NEXT:    v_min_i32_e32 v5, v6, v5
-; GFX9-NEXT:    v_add_u32_e32 v4, v4, v5
+; GFX9-NEXT:    v_add_i32 v0, v0, v5 clamp
+; GFX9-NEXT:    v_add_i32 v1, v1, v6 clamp
+; GFX9-NEXT:    v_add_i32 v2, v2, v7 clamp
+; GFX9-NEXT:    v_add_i32 v3, v3, v8 clamp
+; GFX9-NEXT:    v_add_i32 v4, v4, v9 clamp
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_saddsat_v5i32:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_min_i32_e32 v13, 0, v1
-; GFX10-NEXT:    s_brev_b32 s5, 1
-; GFX10-NEXT:    v_min_i32_e32 v10, 0, v0
-; GFX10-NEXT:    v_min_i32_e32 v16, 0, v2
-; GFX10-NEXT:    v_bfrev_b32_e32 v15, 1
-; GFX10-NEXT:    v_sub_nc_u32_e32 v13, s5, v13
-; GFX10-NEXT:    v_min_i32_e32 v17, 0, v4
-; GFX10-NEXT:    v_sub_nc_u32_e32 v10, s5, v10
-; GFX10-NEXT:    v_sub_nc_u32_e32 v16, s5, v16
-; GFX10-NEXT:    v_max_i32_e32 v11, 0, v0
-; GFX10-NEXT:    v_max_i32_e32 v23, v13, v6
-; GFX10-NEXT:    v_min_i32_e32 v13, 0, v3
-; GFX10-NEXT:    v_max_i32_e32 v5, v10, v5
-; GFX10-NEXT:    v_bfrev_b32_e32 v12, -2
-; GFX10-NEXT:    v_max_i32_e32 v14, 0, v1
-; GFX10-NEXT:    v_max_i32_e32 v10, 0, v2
-; GFX10-NEXT:    v_sub_nc_u32_e32 v13, v15, v13
-; GFX10-NEXT:    v_sub_nc_u32_e32 v15, v15, v17
-; GFX10-NEXT:    v_max_i32_e32 v18, 0, v3
-; GFX10-NEXT:    v_max_i32_e32 v19, 0, v4
-; GFX10-NEXT:    s_brev_b32 s4, -2
-; GFX10-NEXT:    v_max_i32_e32 v7, v16, v7
-; GFX10-NEXT:    v_sub_nc_u32_e32 v11, s4, v11
-; GFX10-NEXT:    v_sub_nc_u32_e32 v14, s4, v14
-; GFX10-NEXT:    v_sub_nc_u32_e32 v10, s4, v10
-; GFX10-NEXT:    v_sub_nc_u32_e32 v16, v12, v18
-; GFX10-NEXT:    v_max_i32_e32 v27, v13, v8
-; GFX10-NEXT:    v_sub_nc_u32_e32 v12, v12, v19
-; GFX10-NEXT:    v_max_i32_e32 v9, v15, v9
-; GFX10-NEXT:    v_min_i32_e32 v5, v5, v11
-; GFX10-NEXT:    v_min_i32_e32 v6, v23, v14
-; GFX10-NEXT:    v_min_i32_e32 v7, v7, v10
-; GFX10-NEXT:    v_min_i32_e32 v8, v27, v16
-; GFX10-NEXT:    v_min_i32_e32 v9, v9, v12
-; GFX10-NEXT:    v_add_nc_u32_e32 v0, v0, v5
-; GFX10-NEXT:    v_add_nc_u32_e32 v1, v1, v6
-; GFX10-NEXT:    v_add_nc_u32_e32 v2, v2, v7
-; GFX10-NEXT:    v_add_nc_u32_e32 v3, v3, v8
-; GFX10-NEXT:    v_add_nc_u32_e32 v4, v4, v9
+; GFX10-NEXT:    v_add_nc_i32 v0, v0, v5 clamp
+; GFX10-NEXT:    v_add_nc_i32 v1, v1, v6 clamp
+; GFX10-NEXT:    v_add_nc_i32 v2, v2, v7 clamp
+; GFX10-NEXT:    v_add_nc_i32 v3, v3, v8 clamp
+; GFX10-NEXT:    v_add_nc_i32 v4, v4, v9 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <5 x i32> @llvm.sadd.sat.v5i32(<5 x i32> %lhs, <5 x i32> %rhs)
@@ -2863,125 +2042,36 @@ define amdgpu_ps <5 x i32> @s_saddsat_v5i32(<5 x i32> inreg %lhs, <5 x i32> inre
 ;
 ; GFX9-LABEL: s_saddsat_v5i32:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_cmp_gt_i32 s0, 0
-; GFX9-NEXT:    s_brev_b32 s10, -2
-; GFX9-NEXT:    s_cselect_b32 s12, s0, 0
-; GFX9-NEXT:    s_sub_i32 s12, s10, s12
-; GFX9-NEXT:    s_cmp_lt_i32 s0, 0
-; GFX9-NEXT:    s_brev_b32 s11, 1
-; GFX9-NEXT:    s_cselect_b32 s13, s0, 0
-; GFX9-NEXT:    s_sub_i32 s13, s11, s13
-; GFX9-NEXT:    s_cmp_gt_i32 s13, s5
-; GFX9-NEXT:    s_cselect_b32 s5, s13, s5
-; GFX9-NEXT:    s_cmp_lt_i32 s5, s12
-; GFX9-NEXT:    s_cselect_b32 s5, s5, s12
-; GFX9-NEXT:    s_add_i32 s0, s0, s5
-; GFX9-NEXT:    s_cmp_gt_i32 s1, 0
-; GFX9-NEXT:    s_cselect_b32 s5, s1, 0
-; GFX9-NEXT:    s_sub_i32 s5, s10, s5
-; GFX9-NEXT:    s_cmp_lt_i32 s1, 0
-; GFX9-NEXT:    s_cselect_b32 s12, s1, 0
-; GFX9-NEXT:    s_sub_i32 s12, s11, s12
-; GFX9-NEXT:    s_cmp_gt_i32 s12, s6
-; GFX9-NEXT:    s_cselect_b32 s6, s12, s6
-; GFX9-NEXT:    s_cmp_lt_i32 s6, s5
-; GFX9-NEXT:    s_cselect_b32 s5, s6, s5
-; GFX9-NEXT:    s_add_i32 s1, s1, s5
-; GFX9-NEXT:    s_cmp_gt_i32 s2, 0
-; GFX9-NEXT:    s_cselect_b32 s5, s2, 0
-; GFX9-NEXT:    s_sub_i32 s5, s10, s5
-; GFX9-NEXT:    s_cmp_lt_i32 s2, 0
-; GFX9-NEXT:    s_cselect_b32 s6, s2, 0
-; GFX9-NEXT:    s_sub_i32 s6, s11, s6
-; GFX9-NEXT:    s_cmp_gt_i32 s6, s7
-; GFX9-NEXT:    s_cselect_b32 s6, s6, s7
-; GFX9-NEXT:    s_cmp_lt_i32 s6, s5
-; GFX9-NEXT:    s_cselect_b32 s5, s6, s5
-; GFX9-NEXT:    s_add_i32 s2, s2, s5
-; GFX9-NEXT:    s_cmp_gt_i32 s3, 0
-; GFX9-NEXT:    s_cselect_b32 s5, s3, 0
-; GFX9-NEXT:    s_sub_i32 s5, s10, s5
-; GFX9-NEXT:    s_cmp_lt_i32 s3, 0
-; GFX9-NEXT:    s_cselect_b32 s6, s3, 0
-; GFX9-NEXT:    s_sub_i32 s6, s11, s6
-; GFX9-NEXT:    s_cmp_gt_i32 s6, s8
-; GFX9-NEXT:    s_cselect_b32 s6, s6, s8
-; GFX9-NEXT:    s_cmp_lt_i32 s6, s5
-; GFX9-NEXT:    s_cselect_b32 s5, s6, s5
-; GFX9-NEXT:    s_add_i32 s3, s3, s5
-; GFX9-NEXT:    s_cmp_gt_i32 s4, 0
-; GFX9-NEXT:    s_cselect_b32 s5, s4, 0
-; GFX9-NEXT:    s_sub_i32 s5, s10, s5
-; GFX9-NEXT:    s_cmp_lt_i32 s4, 0
-; GFX9-NEXT:    s_cselect_b32 s6, s4, 0
-; GFX9-NEXT:    s_sub_i32 s6, s11, s6
-; GFX9-NEXT:    s_cmp_gt_i32 s6, s9
-; GFX9-NEXT:    s_cselect_b32 s6, s6, s9
-; GFX9-NEXT:    s_cmp_lt_i32 s6, s5
-; GFX9-NEXT:    s_cselect_b32 s5, s6, s5
-; GFX9-NEXT:    s_add_i32 s4, s4, s5
+; GFX9-NEXT:    v_mov_b32_e32 v0, s5
+; GFX9-NEXT:    v_mov_b32_e32 v1, s6
+; GFX9-NEXT:    v_mov_b32_e32 v2, s7
+; GFX9-NEXT:    v_mov_b32_e32 v3, s8
+; GFX9-NEXT:    v_mov_b32_e32 v4, s9
+; GFX9-NEXT:    v_add_i32 v0, s0, v0 clamp
+; GFX9-NEXT:    v_add_i32 v1, s1, v1 clamp
+; GFX9-NEXT:    v_add_i32 v2, s2, v2 clamp
+; GFX9-NEXT:    v_add_i32 v3, s3, v3 clamp
+; GFX9-NEXT:    v_add_i32 v4, s4, v4 clamp
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX9-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX9-NEXT:    v_readfirstlane_b32 s4, v4
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_saddsat_v5i32:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_cmp_gt_i32 s0, 0
-; GFX10-NEXT:    s_brev_b32 s10, -2
-; GFX10-NEXT:    s_cselect_b32 s11, s0, 0
-; GFX10-NEXT:    s_brev_b32 s12, 1
-; GFX10-NEXT:    s_sub_i32 s11, s10, s11
-; GFX10-NEXT:    s_cmp_lt_i32 s0, 0
+; GFX10-NEXT:    v_add_nc_i32 v0, s0, s5 clamp
+; GFX10-NEXT:    v_add_nc_i32 v1, s1, s6 clamp
+; GFX10-NEXT:    v_add_nc_i32 v2, s2, s7 clamp
+; GFX10-NEXT:    v_add_nc_i32 v3, s3, s8 clamp
+; GFX10-NEXT:    v_add_nc_i32 v4, s4, s9 clamp
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX10-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX10-NEXT:    v_readfirstlane_b32 s4, v4
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cselect_b32 s13, s0, 0
-; GFX10-NEXT:    s_sub_i32 s13, s12, s13
-; GFX10-NEXT:    s_cmp_gt_i32 s13, s5
-; GFX10-NEXT:    s_cselect_b32 s5, s13, s5
-; GFX10-NEXT:    s_cmp_lt_i32 s5, s11
-; GFX10-NEXT:    s_cselect_b32 s5, s5, s11
-; GFX10-NEXT:    s_add_i32 s0, s0, s5
-; GFX10-NEXT:    s_cmp_gt_i32 s1, 0
-; GFX10-NEXT:    s_cselect_b32 s5, s1, 0
-; GFX10-NEXT:    s_sub_i32 s5, s10, s5
-; GFX10-NEXT:    s_cmp_lt_i32 s1, 0
-; GFX10-NEXT:    s_cselect_b32 s11, s1, 0
-; GFX10-NEXT:    s_sub_i32 s11, s12, s11
-; GFX10-NEXT:    s_cmp_gt_i32 s11, s6
-; GFX10-NEXT:    s_cselect_b32 s6, s11, s6
-; GFX10-NEXT:    s_cmp_lt_i32 s6, s5
-; GFX10-NEXT:    s_cselect_b32 s5, s6, s5
-; GFX10-NEXT:    s_add_i32 s1, s1, s5
-; GFX10-NEXT:    s_cmp_gt_i32 s2, 0
-; GFX10-NEXT:    s_cselect_b32 s5, s2, 0
-; GFX10-NEXT:    s_sub_i32 s5, s10, s5
-; GFX10-NEXT:    s_cmp_lt_i32 s2, 0
-; GFX10-NEXT:    s_cselect_b32 s6, s2, 0
-; GFX10-NEXT:    s_sub_i32 s6, s12, s6
-; GFX10-NEXT:    s_cmp_gt_i32 s6, s7
-; GFX10-NEXT:    s_cselect_b32 s6, s6, s7
-; GFX10-NEXT:    s_cmp_lt_i32 s6, s5
-; GFX10-NEXT:    s_cselect_b32 s5, s6, s5
-; GFX10-NEXT:    s_add_i32 s2, s2, s5
-; GFX10-NEXT:    s_cmp_gt_i32 s3, 0
-; GFX10-NEXT:    s_cselect_b32 s5, s3, 0
-; GFX10-NEXT:    s_sub_i32 s5, s10, s5
-; GFX10-NEXT:    s_cmp_lt_i32 s3, 0
-; GFX10-NEXT:    s_cselect_b32 s6, s3, 0
-; GFX10-NEXT:    s_sub_i32 s6, s12, s6
-; GFX10-NEXT:    s_cmp_gt_i32 s6, s8
-; GFX10-NEXT:    s_cselect_b32 s6, s6, s8
-; GFX10-NEXT:    s_cmp_lt_i32 s6, s5
-; GFX10-NEXT:    s_cselect_b32 s5, s6, s5
-; GFX10-NEXT:    s_add_i32 s3, s3, s5
-; GFX10-NEXT:    s_cmp_gt_i32 s4, 0
-; GFX10-NEXT:    s_cselect_b32 s5, s4, 0
-; GFX10-NEXT:    s_sub_i32 s5, s10, s5
-; GFX10-NEXT:    s_cmp_lt_i32 s4, 0
-; GFX10-NEXT:    s_cselect_b32 s6, s4, 0
-; GFX10-NEXT:    s_sub_i32 s6, s12, s6
-; GFX10-NEXT:    s_cmp_gt_i32 s6, s9
-; GFX10-NEXT:    s_cselect_b32 s6, s6, s9
-; GFX10-NEXT:    s_cmp_lt_i32 s6, s5
-; GFX10-NEXT:    s_cselect_b32 s5, s6, s5
-; GFX10-NEXT:    s_add_i32 s4, s4, s5
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <5 x i32> @llvm.sadd.sat.v5i32(<5 x i32> %lhs, <5 x i32> %rhs)
   ret <5 x i32> %result
@@ -3233,244 +2323,44 @@ define <16 x i32> @v_saddsat_v16i32(<16 x i32> %lhs, <16 x i32> %rhs) {
 ; GFX9-LABEL: v_saddsat_v16i32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    s_brev_b32 s4, 1
-; GFX9-NEXT:    v_min_i32_e32 v32, 0, v0
-; GFX9-NEXT:    v_sub_u32_e32 v32, s4, v32
-; GFX9-NEXT:    v_max_i32_e32 v16, v32, v16
-; GFX9-NEXT:    s_brev_b32 s5, -2
-; GFX9-NEXT:    v_max_i32_e32 v32, 0, v0
-; GFX9-NEXT:    v_sub_u32_e32 v32, s5, v32
-; GFX9-NEXT:    v_min_i32_e32 v16, v16, v32
-; GFX9-NEXT:    v_add_u32_e32 v0, v0, v16
-; GFX9-NEXT:    v_min_i32_e32 v16, 0, v1
-; GFX9-NEXT:    v_sub_u32_e32 v16, s4, v16
-; GFX9-NEXT:    v_max_i32_e32 v16, v16, v17
-; GFX9-NEXT:    v_max_i32_e32 v17, 0, v1
-; GFX9-NEXT:    v_sub_u32_e32 v17, s5, v17
-; GFX9-NEXT:    v_min_i32_e32 v16, v16, v17
-; GFX9-NEXT:    v_add_u32_e32 v1, v1, v16
-; GFX9-NEXT:    v_min_i32_e32 v16, 0, v2
-; GFX9-NEXT:    v_sub_u32_e32 v16, s4, v16
-; GFX9-NEXT:    v_max_i32_e32 v17, 0, v2
-; GFX9-NEXT:    v_max_i32_e32 v16, v16, v18
-; GFX9-NEXT:    v_sub_u32_e32 v17, s5, v17
-; GFX9-NEXT:    v_min_i32_e32 v16, v16, v17
-; GFX9-NEXT:    v_add_u32_e32 v2, v2, v16
-; GFX9-NEXT:    v_bfrev_b32_e32 v16, 1
-; GFX9-NEXT:    v_min_i32_e32 v17, 0, v3
-; GFX9-NEXT:    v_sub_u32_e32 v17, v16, v17
-; GFX9-NEXT:    v_max_i32_e32 v17, v17, v19
-; GFX9-NEXT:    v_bfrev_b32_e32 v18, -2
-; GFX9-NEXT:    v_max_i32_e32 v19, 0, v3
-; GFX9-NEXT:    v_sub_u32_e32 v19, v18, v19
-; GFX9-NEXT:    v_min_i32_e32 v17, v17, v19
-; GFX9-NEXT:    v_add_u32_e32 v3, v3, v17
-; GFX9-NEXT:    v_min_i32_e32 v17, 0, v4
-; GFX9-NEXT:    v_sub_u32_e32 v17, v16, v17
-; GFX9-NEXT:    v_max_i32_e32 v19, 0, v4
-; GFX9-NEXT:    v_max_i32_e32 v17, v17, v20
-; GFX9-NEXT:    v_sub_u32_e32 v19, v18, v19
-; GFX9-NEXT:    v_min_i32_e32 v17, v17, v19
-; GFX9-NEXT:    v_add_u32_e32 v4, v4, v17
-; GFX9-NEXT:    v_min_i32_e32 v17, 0, v5
-; GFX9-NEXT:    v_sub_u32_e32 v17, v16, v17
-; GFX9-NEXT:    v_max_i32_e32 v19, 0, v5
-; GFX9-NEXT:    v_max_i32_e32 v17, v17, v21
-; GFX9-NEXT:    v_sub_u32_e32 v19, v18, v19
-; GFX9-NEXT:    v_min_i32_e32 v17, v17, v19
-; GFX9-NEXT:    v_add_u32_e32 v5, v5, v17
-; GFX9-NEXT:    v_min_i32_e32 v17, 0, v6
-; GFX9-NEXT:    v_sub_u32_e32 v17, v16, v17
-; GFX9-NEXT:    v_max_i32_e32 v19, 0, v6
-; GFX9-NEXT:    v_max_i32_e32 v17, v17, v22
-; GFX9-NEXT:    v_sub_u32_e32 v19, v18, v19
-; GFX9-NEXT:    v_min_i32_e32 v17, v17, v19
-; GFX9-NEXT:    v_add_u32_e32 v6, v6, v17
-; GFX9-NEXT:    v_min_i32_e32 v17, 0, v7
-; GFX9-NEXT:    v_sub_u32_e32 v17, v16, v17
-; GFX9-NEXT:    v_max_i32_e32 v19, 0, v7
-; GFX9-NEXT:    v_max_i32_e32 v17, v17, v23
-; GFX9-NEXT:    v_sub_u32_e32 v19, v18, v19
-; GFX9-NEXT:    v_min_i32_e32 v17, v17, v19
-; GFX9-NEXT:    v_add_u32_e32 v7, v7, v17
-; GFX9-NEXT:    v_min_i32_e32 v17, 0, v8
-; GFX9-NEXT:    v_sub_u32_e32 v17, v16, v17
-; GFX9-NEXT:    v_max_i32_e32 v19, 0, v8
-; GFX9-NEXT:    v_max_i32_e32 v17, v17, v24
-; GFX9-NEXT:    v_sub_u32_e32 v19, v18, v19
-; GFX9-NEXT:    v_min_i32_e32 v17, v17, v19
-; GFX9-NEXT:    v_add_u32_e32 v8, v8, v17
-; GFX9-NEXT:    v_min_i32_e32 v17, 0, v9
-; GFX9-NEXT:    v_sub_u32_e32 v17, v16, v17
-; GFX9-NEXT:    v_max_i32_e32 v19, 0, v9
-; GFX9-NEXT:    v_max_i32_e32 v17, v17, v25
-; GFX9-NEXT:    v_sub_u32_e32 v19, v18, v19
-; GFX9-NEXT:    v_min_i32_e32 v17, v17, v19
-; GFX9-NEXT:    v_add_u32_e32 v9, v9, v17
-; GFX9-NEXT:    v_min_i32_e32 v17, 0, v10
-; GFX9-NEXT:    v_sub_u32_e32 v17, v16, v17
-; GFX9-NEXT:    v_max_i32_e32 v19, 0, v10
-; GFX9-NEXT:    v_max_i32_e32 v17, v17, v26
-; GFX9-NEXT:    v_sub_u32_e32 v19, v18, v19
-; GFX9-NEXT:    v_min_i32_e32 v17, v17, v19
-; GFX9-NEXT:    v_add_u32_e32 v10, v10, v17
-; GFX9-NEXT:    v_min_i32_e32 v17, 0, v11
-; GFX9-NEXT:    v_sub_u32_e32 v17, v16, v17
-; GFX9-NEXT:    v_max_i32_e32 v19, 0, v11
-; GFX9-NEXT:    v_max_i32_e32 v17, v17, v27
-; GFX9-NEXT:    v_sub_u32_e32 v19, v18, v19
-; GFX9-NEXT:    v_min_i32_e32 v17, v17, v19
-; GFX9-NEXT:    v_add_u32_e32 v11, v11, v17
-; GFX9-NEXT:    v_min_i32_e32 v17, 0, v12
-; GFX9-NEXT:    v_sub_u32_e32 v17, v16, v17
-; GFX9-NEXT:    v_max_i32_e32 v19, 0, v12
-; GFX9-NEXT:    v_max_i32_e32 v17, v17, v28
-; GFX9-NEXT:    v_sub_u32_e32 v19, v18, v19
-; GFX9-NEXT:    v_min_i32_e32 v17, v17, v19
-; GFX9-NEXT:    v_add_u32_e32 v12, v12, v17
-; GFX9-NEXT:    v_min_i32_e32 v17, 0, v13
-; GFX9-NEXT:    v_sub_u32_e32 v17, v16, v17
-; GFX9-NEXT:    v_max_i32_e32 v19, 0, v13
-; GFX9-NEXT:    v_max_i32_e32 v17, v17, v29
-; GFX9-NEXT:    v_sub_u32_e32 v19, v18, v19
-; GFX9-NEXT:    v_min_i32_e32 v17, v17, v19
-; GFX9-NEXT:    v_add_u32_e32 v13, v13, v17
-; GFX9-NEXT:    v_min_i32_e32 v17, 0, v14
-; GFX9-NEXT:    v_sub_u32_e32 v17, v16, v17
-; GFX9-NEXT:    v_max_i32_e32 v19, 0, v14
-; GFX9-NEXT:    v_max_i32_e32 v17, v17, v30
-; GFX9-NEXT:    v_sub_u32_e32 v19, v18, v19
-; GFX9-NEXT:    v_min_i32_e32 v17, v17, v19
-; GFX9-NEXT:    v_add_u32_e32 v14, v14, v17
-; GFX9-NEXT:    v_max_i32_e32 v17, 0, v15
-; GFX9-NEXT:    v_sub_u32_e32 v17, v18, v17
-; GFX9-NEXT:    v_min_i32_e32 v18, 0, v15
-; GFX9-NEXT:    v_sub_u32_e32 v16, v16, v18
-; GFX9-NEXT:    v_max_i32_e32 v16, v16, v31
-; GFX9-NEXT:    v_min_i32_e32 v16, v16, v17
-; GFX9-NEXT:    v_add_u32_e32 v15, v15, v16
+; GFX9-NEXT:    v_add_i32 v0, v0, v16 clamp
+; GFX9-NEXT:    v_add_i32 v1, v1, v17 clamp
+; GFX9-NEXT:    v_add_i32 v2, v2, v18 clamp
+; GFX9-NEXT:    v_add_i32 v3, v3, v19 clamp
+; GFX9-NEXT:    v_add_i32 v4, v4, v20 clamp
+; GFX9-NEXT:    v_add_i32 v5, v5, v21 clamp
+; GFX9-NEXT:    v_add_i32 v6, v6, v22 clamp
+; GFX9-NEXT:    v_add_i32 v7, v7, v23 clamp
+; GFX9-NEXT:    v_add_i32 v8, v8, v24 clamp
+; GFX9-NEXT:    v_add_i32 v9, v9, v25 clamp
+; GFX9-NEXT:    v_add_i32 v10, v10, v26 clamp
+; GFX9-NEXT:    v_add_i32 v11, v11, v27 clamp
+; GFX9-NEXT:    v_add_i32 v12, v12, v28 clamp
+; GFX9-NEXT:    v_add_i32 v13, v13, v29 clamp
+; GFX9-NEXT:    v_add_i32 v14, v14, v30 clamp
+; GFX9-NEXT:    v_add_i32 v15, v15, v31 clamp
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_saddsat_v16i32:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_min_i32_e32 v32, 0, v0
-; GFX10-NEXT:    s_brev_b32 s4, 1
-; GFX10-NEXT:    v_max_i32_e32 v33, 0, v0
-; GFX10-NEXT:    s_brev_b32 s5, -2
-; GFX10-NEXT:    v_min_i32_e32 v36, 0, v2
-; GFX10-NEXT:    v_sub_nc_u32_e32 v35, s4, v32
-; GFX10-NEXT:    v_min_i32_e32 v32, 0, v1
-; GFX10-NEXT:    v_sub_nc_u32_e32 v33, s5, v33
-; GFX10-NEXT:    v_max_i32_e32 v37, 0, v1
-; GFX10-NEXT:    v_sub_nc_u32_e32 v36, s4, v36
-; GFX10-NEXT:    v_max_i32_e32 v16, v35, v16
-; GFX10-NEXT:    v_sub_nc_u32_e32 v32, s4, v32
-; GFX10-NEXT:    v_bfrev_b32_e32 v35, 1
-; GFX10-NEXT:    v_min_i32_e32 v38, 0, v3
-; GFX10-NEXT:    v_max_i32_e32 v18, v36, v18
-; GFX10-NEXT:    v_min_i32_e32 v16, v16, v33
-; GFX10-NEXT:    v_max_i32_e32 v33, 0, v2
-; GFX10-NEXT:    v_max_i32_e32 v39, v32, v17
-; GFX10-NEXT:    v_sub_nc_u32_e32 v36, v35, v38
-; GFX10-NEXT:    v_sub_nc_u32_e32 v37, s5, v37
-; GFX10-NEXT:    v_bfrev_b32_e32 v34, -2
-; GFX10-NEXT:    v_sub_nc_u32_e32 v32, s5, v33
-; GFX10-NEXT:    v_add_nc_u32_e32 v0, v0, v16
-; GFX10-NEXT:    v_max_i32_e32 v33, 0, v3
-; GFX10-NEXT:    v_min_i32_e32 v39, v39, v37
-; GFX10-NEXT:    v_max_i32_e32 v19, v36, v19
-; GFX10-NEXT:    v_min_i32_e32 v16, v18, v32
-; GFX10-NEXT:    v_min_i32_e32 v32, 0, v6
-; GFX10-NEXT:    v_sub_nc_u32_e32 v18, v34, v33
-; GFX10-NEXT:    v_min_i32_e32 v38, 0, v5
-; GFX10-NEXT:    v_max_i32_e32 v17, 0, v4
-; GFX10-NEXT:    v_add_nc_u32_e32 v2, v2, v16
-; GFX10-NEXT:    v_min_i32_e32 v16, 0, v4
-; GFX10-NEXT:    v_min_i32_e32 v18, v19, v18
-; GFX10-NEXT:    v_sub_nc_u32_e32 v19, v35, v38
-; GFX10-NEXT:    v_add_nc_u32_e32 v1, v1, v39
-; GFX10-NEXT:    v_sub_nc_u32_e32 v32, v35, v32
-; GFX10-NEXT:    v_sub_nc_u32_e32 v39, v35, v16
-; GFX10-NEXT:    v_max_i32_e32 v33, 0, v5
-; GFX10-NEXT:    v_max_i32_e32 v36, 0, v6
-; GFX10-NEXT:    v_max_i32_e32 v19, v19, v21
-; GFX10-NEXT:    v_add_nc_u32_e32 v3, v3, v18
-; GFX10-NEXT:    v_max_i32_e32 v16, v39, v20
-; GFX10-NEXT:    v_sub_nc_u32_e32 v17, v34, v17
-; GFX10-NEXT:    v_sub_nc_u32_e32 v20, v34, v33
-; GFX10-NEXT:    v_sub_nc_u32_e32 v21, v34, v36
-; GFX10-NEXT:    v_max_i32_e32 v22, v32, v22
-; GFX10-NEXT:    v_min_i32_e32 v18, 0, v7
-; GFX10-NEXT:    v_min_i32_e32 v39, v16, v17
-; GFX10-NEXT:    v_min_i32_e32 v38, v19, v20
-; GFX10-NEXT:    v_max_i32_e32 v16, 0, v7
-; GFX10-NEXT:    v_min_i32_e32 v19, v22, v21
-; GFX10-NEXT:    v_sub_nc_u32_e32 v17, v35, v18
-; GFX10-NEXT:    v_min_i32_e32 v18, 0, v8
-; GFX10-NEXT:    v_min_i32_e32 v20, 0, v9
-; GFX10-NEXT:    v_sub_nc_u32_e32 v16, v34, v16
-; GFX10-NEXT:    v_add_nc_u32_e32 v6, v6, v19
-; GFX10-NEXT:    v_max_i32_e32 v19, 0, v8
-; GFX10-NEXT:    v_max_i32_e32 v17, v17, v23
-; GFX10-NEXT:    v_sub_nc_u32_e32 v18, v35, v18
-; GFX10-NEXT:    v_min_i32_e32 v22, 0, v10
-; GFX10-NEXT:    v_max_i32_e32 v21, 0, v9
-; GFX10-NEXT:    v_sub_nc_u32_e32 v20, v35, v20
-; GFX10-NEXT:    v_add_nc_u32_e32 v4, v4, v39
-; GFX10-NEXT:    v_max_i32_e32 v18, v18, v24
-; GFX10-NEXT:    v_sub_nc_u32_e32 v39, v35, v22
-; GFX10-NEXT:    v_min_i32_e32 v16, v17, v16
-; GFX10-NEXT:    v_sub_nc_u32_e32 v19, v34, v19
-; GFX10-NEXT:    v_max_i32_e32 v23, 0, v10
-; GFX10-NEXT:    v_max_i32_e32 v20, v20, v25
-; GFX10-NEXT:    v_sub_nc_u32_e32 v21, v34, v21
-; GFX10-NEXT:    v_add_nc_u32_e32 v7, v7, v16
-; GFX10-NEXT:    v_min_i32_e32 v17, v18, v19
-; GFX10-NEXT:    v_min_i32_e32 v16, 0, v11
-; GFX10-NEXT:    v_sub_nc_u32_e32 v18, v34, v23
-; GFX10-NEXT:    v_max_i32_e32 v19, v39, v26
-; GFX10-NEXT:    v_min_i32_e32 v22, 0, v12
-; GFX10-NEXT:    v_min_i32_e32 v20, v20, v21
-; GFX10-NEXT:    v_sub_nc_u32_e32 v16, v35, v16
-; GFX10-NEXT:    v_min_i32_e32 v26, 0, v15
-; GFX10-NEXT:    v_add_nc_u32_e32 v8, v8, v17
-; GFX10-NEXT:    v_min_i32_e32 v17, v19, v18
-; GFX10-NEXT:    v_sub_nc_u32_e32 v19, v35, v22
-; GFX10-NEXT:    v_min_i32_e32 v22, 0, v14
-; GFX10-NEXT:    v_min_i32_e32 v21, 0, v13
-; GFX10-NEXT:    v_max_i32_e32 v24, 0, v14
-; GFX10-NEXT:    v_max_i32_e32 v25, 0, v15
-; GFX10-NEXT:    v_add_nc_u32_e32 v9, v9, v20
-; GFX10-NEXT:    v_max_i32_e32 v20, 0, v13
-; GFX10-NEXT:    v_sub_nc_u32_e32 v39, v35, v22
-; GFX10-NEXT:    v_max_i32_e32 v23, 0, v11
-; GFX10-NEXT:    v_add_nc_u32_e32 v10, v10, v17
-; GFX10-NEXT:    v_max_i32_e32 v17, 0, v12
-; GFX10-NEXT:    v_max_i32_e32 v16, v16, v27
-; GFX10-NEXT:    v_sub_nc_u32_e32 v27, v35, v21
-; GFX10-NEXT:    v_sub_nc_u32_e32 v26, v35, v26
-; GFX10-NEXT:    v_sub_nc_u32_e32 v18, v34, v23
-; GFX10-NEXT:    v_sub_nc_u32_e32 v17, v34, v17
-; GFX10-NEXT:    v_max_i32_e32 v19, v19, v28
-; GFX10-NEXT:    v_sub_nc_u32_e32 v20, v34, v20
-; GFX10-NEXT:    v_max_i32_e32 v21, v27, v29
-; GFX10-NEXT:    v_sub_nc_u32_e32 v24, v34, v24
-; GFX10-NEXT:    v_max_i32_e32 v22, v39, v30
-; GFX10-NEXT:    v_sub_nc_u32_e32 v25, v34, v25
-; GFX10-NEXT:    v_max_i32_e32 v23, v26, v31
-; GFX10-NEXT:    v_min_i32_e32 v16, v16, v18
-; GFX10-NEXT:    v_min_i32_e32 v17, v19, v17
-; GFX10-NEXT:    v_min_i32_e32 v18, v21, v20
-; GFX10-NEXT:    v_min_i32_e32 v19, v22, v24
-; GFX10-NEXT:    v_min_i32_e32 v20, v23, v25
-; GFX10-NEXT:    v_add_nc_u32_e32 v5, v5, v38
-; GFX10-NEXT:    v_add_nc_u32_e32 v11, v11, v16
-; GFX10-NEXT:    v_add_nc_u32_e32 v12, v12, v17
-; GFX10-NEXT:    v_add_nc_u32_e32 v13, v13, v18
-; GFX10-NEXT:    v_add_nc_u32_e32 v14, v14, v19
-; GFX10-NEXT:    v_add_nc_u32_e32 v15, v15, v20
+; GFX10-NEXT:    v_add_nc_i32 v0, v0, v16 clamp
+; GFX10-NEXT:    v_add_nc_i32 v1, v1, v17 clamp
+; GFX10-NEXT:    v_add_nc_i32 v2, v2, v18 clamp
+; GFX10-NEXT:    v_add_nc_i32 v3, v3, v19 clamp
+; GFX10-NEXT:    v_add_nc_i32 v4, v4, v20 clamp
+; GFX10-NEXT:    v_add_nc_i32 v5, v5, v21 clamp
+; GFX10-NEXT:    v_add_nc_i32 v6, v6, v22 clamp
+; GFX10-NEXT:    v_add_nc_i32 v7, v7, v23 clamp
+; GFX10-NEXT:    v_add_nc_i32 v8, v8, v24 clamp
+; GFX10-NEXT:    v_add_nc_i32 v9, v9, v25 clamp
+; GFX10-NEXT:    v_add_nc_i32 v10, v10, v26 clamp
+; GFX10-NEXT:    v_add_nc_i32 v11, v11, v27 clamp
+; GFX10-NEXT:    v_add_nc_i32 v12, v12, v28 clamp
+; GFX10-NEXT:    v_add_nc_i32 v13, v13, v29 clamp
+; GFX10-NEXT:    v_add_nc_i32 v14, v14, v30 clamp
+; GFX10-NEXT:    v_add_nc_i32 v15, v15, v31 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <16 x i32> @llvm.sadd.sat.v16i32(<16 x i32> %lhs, <16 x i32> %rhs)
@@ -3844,367 +2734,91 @@ define amdgpu_ps <16 x i32> @s_saddsat_v16i32(<16 x i32> inreg %lhs, <16 x i32>
 ;
 ; GFX9-LABEL: s_saddsat_v16i32:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_cmp_gt_i32 s0, 0
-; GFX9-NEXT:    s_brev_b32 s32, -2
-; GFX9-NEXT:    s_cselect_b32 s34, s0, 0
-; GFX9-NEXT:    s_sub_i32 s34, s32, s34
-; GFX9-NEXT:    s_cmp_lt_i32 s0, 0
-; GFX9-NEXT:    s_brev_b32 s33, 1
-; GFX9-NEXT:    s_cselect_b32 s35, s0, 0
-; GFX9-NEXT:    s_sub_i32 s35, s33, s35
-; GFX9-NEXT:    s_cmp_gt_i32 s35, s16
-; GFX9-NEXT:    s_cselect_b32 s16, s35, s16
-; GFX9-NEXT:    s_cmp_lt_i32 s16, s34
-; GFX9-NEXT:    s_cselect_b32 s16, s16, s34
-; GFX9-NEXT:    s_add_i32 s0, s0, s16
-; GFX9-NEXT:    s_cmp_gt_i32 s1, 0
-; GFX9-NEXT:    s_cselect_b32 s16, s1, 0
-; GFX9-NEXT:    s_sub_i32 s16, s32, s16
-; GFX9-NEXT:    s_cmp_lt_i32 s1, 0
-; GFX9-NEXT:    s_cselect_b32 s34, s1, 0
-; GFX9-NEXT:    s_sub_i32 s34, s33, s34
-; GFX9-NEXT:    s_cmp_gt_i32 s34, s17
-; GFX9-NEXT:    s_cselect_b32 s17, s34, s17
-; GFX9-NEXT:    s_cmp_lt_i32 s17, s16
-; GFX9-NEXT:    s_cselect_b32 s16, s17, s16
-; GFX9-NEXT:    s_add_i32 s1, s1, s16
-; GFX9-NEXT:    s_cmp_gt_i32 s2, 0
-; GFX9-NEXT:    s_cselect_b32 s16, s2, 0
-; GFX9-NEXT:    s_sub_i32 s16, s32, s16
-; GFX9-NEXT:    s_cmp_lt_i32 s2, 0
-; GFX9-NEXT:    s_cselect_b32 s17, s2, 0
-; GFX9-NEXT:    s_sub_i32 s17, s33, s17
-; GFX9-NEXT:    s_cmp_gt_i32 s17, s18
-; GFX9-NEXT:    s_cselect_b32 s17, s17, s18
-; GFX9-NEXT:    s_cmp_lt_i32 s17, s16
-; GFX9-NEXT:    s_cselect_b32 s16, s17, s16
-; GFX9-NEXT:    s_add_i32 s2, s2, s16
-; GFX9-NEXT:    s_cmp_gt_i32 s3, 0
-; GFX9-NEXT:    s_cselect_b32 s16, s3, 0
-; GFX9-NEXT:    s_sub_i32 s16, s32, s16
-; GFX9-NEXT:    s_cmp_lt_i32 s3, 0
-; GFX9-NEXT:    s_cselect_b32 s17, s3, 0
-; GFX9-NEXT:    s_sub_i32 s17, s33, s17
-; GFX9-NEXT:    s_cmp_gt_i32 s17, s19
-; GFX9-NEXT:    s_cselect_b32 s17, s17, s19
-; GFX9-NEXT:    s_cmp_lt_i32 s17, s16
-; GFX9-NEXT:    s_cselect_b32 s16, s17, s16
-; GFX9-NEXT:    s_add_i32 s3, s3, s16
-; GFX9-NEXT:    s_cmp_gt_i32 s4, 0
-; GFX9-NEXT:    s_cselect_b32 s16, s4, 0
-; GFX9-NEXT:    s_sub_i32 s16, s32, s16
-; GFX9-NEXT:    s_cmp_lt_i32 s4, 0
-; GFX9-NEXT:    s_cselect_b32 s17, s4, 0
-; GFX9-NEXT:    s_sub_i32 s17, s33, s17
-; GFX9-NEXT:    s_cmp_gt_i32 s17, s20
-; GFX9-NEXT:    s_cselect_b32 s17, s17, s20
-; GFX9-NEXT:    s_cmp_lt_i32 s17, s16
-; GFX9-NEXT:    s_cselect_b32 s16, s17, s16
-; GFX9-NEXT:    s_add_i32 s4, s4, s16
-; GFX9-NEXT:    s_cmp_gt_i32 s5, 0
-; GFX9-NEXT:    s_cselect_b32 s16, s5, 0
-; GFX9-NEXT:    s_sub_i32 s16, s32, s16
-; GFX9-NEXT:    s_cmp_lt_i32 s5, 0
-; GFX9-NEXT:    s_cselect_b32 s17, s5, 0
-; GFX9-NEXT:    s_sub_i32 s17, s33, s17
-; GFX9-NEXT:    s_cmp_gt_i32 s17, s21
-; GFX9-NEXT:    s_cselect_b32 s17, s17, s21
-; GFX9-NEXT:    s_cmp_lt_i32 s17, s16
-; GFX9-NEXT:    s_cselect_b32 s16, s17, s16
-; GFX9-NEXT:    s_add_i32 s5, s5, s16
-; GFX9-NEXT:    s_cmp_gt_i32 s6, 0
-; GFX9-NEXT:    s_cselect_b32 s16, s6, 0
-; GFX9-NEXT:    s_sub_i32 s16, s32, s16
-; GFX9-NEXT:    s_cmp_lt_i32 s6, 0
-; GFX9-NEXT:    s_cselect_b32 s17, s6, 0
-; GFX9-NEXT:    s_sub_i32 s17, s33, s17
-; GFX9-NEXT:    s_cmp_gt_i32 s17, s22
-; GFX9-NEXT:    s_cselect_b32 s17, s17, s22
-; GFX9-NEXT:    s_cmp_lt_i32 s17, s16
-; GFX9-NEXT:    s_cselect_b32 s16, s17, s16
-; GFX9-NEXT:    s_add_i32 s6, s6, s16
-; GFX9-NEXT:    s_cmp_gt_i32 s7, 0
-; GFX9-NEXT:    s_cselect_b32 s16, s7, 0
-; GFX9-NEXT:    s_sub_i32 s16, s32, s16
-; GFX9-NEXT:    s_cmp_lt_i32 s7, 0
-; GFX9-NEXT:    s_cselect_b32 s17, s7, 0
-; GFX9-NEXT:    s_sub_i32 s17, s33, s17
-; GFX9-NEXT:    s_cmp_gt_i32 s17, s23
-; GFX9-NEXT:    s_cselect_b32 s17, s17, s23
-; GFX9-NEXT:    s_cmp_lt_i32 s17, s16
-; GFX9-NEXT:    s_cselect_b32 s16, s17, s16
-; GFX9-NEXT:    s_add_i32 s7, s7, s16
-; GFX9-NEXT:    s_cmp_gt_i32 s8, 0
-; GFX9-NEXT:    s_cselect_b32 s16, s8, 0
-; GFX9-NEXT:    s_sub_i32 s16, s32, s16
-; GFX9-NEXT:    s_cmp_lt_i32 s8, 0
-; GFX9-NEXT:    s_cselect_b32 s17, s8, 0
-; GFX9-NEXT:    s_sub_i32 s17, s33, s17
-; GFX9-NEXT:    s_cmp_gt_i32 s17, s24
-; GFX9-NEXT:    s_cselect_b32 s17, s17, s24
-; GFX9-NEXT:    s_cmp_lt_i32 s17, s16
-; GFX9-NEXT:    s_cselect_b32 s16, s17, s16
-; GFX9-NEXT:    s_add_i32 s8, s8, s16
-; GFX9-NEXT:    s_cmp_gt_i32 s9, 0
-; GFX9-NEXT:    s_cselect_b32 s16, s9, 0
-; GFX9-NEXT:    s_sub_i32 s16, s32, s16
-; GFX9-NEXT:    s_cmp_lt_i32 s9, 0
-; GFX9-NEXT:    s_cselect_b32 s17, s9, 0
-; GFX9-NEXT:    s_sub_i32 s17, s33, s17
-; GFX9-NEXT:    s_cmp_gt_i32 s17, s25
-; GFX9-NEXT:    s_cselect_b32 s17, s17, s25
-; GFX9-NEXT:    s_cmp_lt_i32 s17, s16
-; GFX9-NEXT:    s_cselect_b32 s16, s17, s16
-; GFX9-NEXT:    s_add_i32 s9, s9, s16
-; GFX9-NEXT:    s_cmp_gt_i32 s10, 0
-; GFX9-NEXT:    s_cselect_b32 s16, s10, 0
-; GFX9-NEXT:    s_sub_i32 s16, s32, s16
-; GFX9-NEXT:    s_cmp_lt_i32 s10, 0
-; GFX9-NEXT:    s_cselect_b32 s17, s10, 0
-; GFX9-NEXT:    s_sub_i32 s17, s33, s17
-; GFX9-NEXT:    s_cmp_gt_i32 s17, s26
-; GFX9-NEXT:    s_cselect_b32 s17, s17, s26
-; GFX9-NEXT:    s_cmp_lt_i32 s17, s16
-; GFX9-NEXT:    s_cselect_b32 s16, s17, s16
-; GFX9-NEXT:    s_add_i32 s10, s10, s16
-; GFX9-NEXT:    s_cmp_gt_i32 s11, 0
-; GFX9-NEXT:    s_cselect_b32 s16, s11, 0
-; GFX9-NEXT:    s_sub_i32 s16, s32, s16
-; GFX9-NEXT:    s_cmp_lt_i32 s11, 0
-; GFX9-NEXT:    s_cselect_b32 s17, s11, 0
-; GFX9-NEXT:    s_sub_i32 s17, s33, s17
-; GFX9-NEXT:    s_cmp_gt_i32 s17, s27
-; GFX9-NEXT:    s_cselect_b32 s17, s17, s27
-; GFX9-NEXT:    s_cmp_lt_i32 s17, s16
-; GFX9-NEXT:    s_cselect_b32 s16, s17, s16
-; GFX9-NEXT:    s_add_i32 s11, s11, s16
-; GFX9-NEXT:    s_cmp_gt_i32 s12, 0
-; GFX9-NEXT:    s_cselect_b32 s16, s12, 0
-; GFX9-NEXT:    s_sub_i32 s16, s32, s16
-; GFX9-NEXT:    s_cmp_lt_i32 s12, 0
-; GFX9-NEXT:    s_cselect_b32 s17, s12, 0
-; GFX9-NEXT:    s_sub_i32 s17, s33, s17
-; GFX9-NEXT:    s_cmp_gt_i32 s17, s28
-; GFX9-NEXT:    s_cselect_b32 s17, s17, s28
-; GFX9-NEXT:    s_cmp_lt_i32 s17, s16
-; GFX9-NEXT:    s_cselect_b32 s16, s17, s16
-; GFX9-NEXT:    s_add_i32 s12, s12, s16
-; GFX9-NEXT:    s_cmp_gt_i32 s13, 0
-; GFX9-NEXT:    s_cselect_b32 s16, s13, 0
-; GFX9-NEXT:    s_sub_i32 s16, s32, s16
-; GFX9-NEXT:    s_cmp_lt_i32 s13, 0
-; GFX9-NEXT:    s_cselect_b32 s17, s13, 0
-; GFX9-NEXT:    s_sub_i32 s17, s33, s17
-; GFX9-NEXT:    s_cmp_gt_i32 s17, s29
-; GFX9-NEXT:    s_cselect_b32 s17, s17, s29
-; GFX9-NEXT:    s_cmp_lt_i32 s17, s16
-; GFX9-NEXT:    s_cselect_b32 s16, s17, s16
-; GFX9-NEXT:    s_add_i32 s13, s13, s16
-; GFX9-NEXT:    s_cmp_gt_i32 s14, 0
-; GFX9-NEXT:    s_cselect_b32 s16, s14, 0
-; GFX9-NEXT:    s_sub_i32 s16, s32, s16
-; GFX9-NEXT:    s_cmp_lt_i32 s14, 0
-; GFX9-NEXT:    s_cselect_b32 s17, s14, 0
-; GFX9-NEXT:    s_sub_i32 s17, s33, s17
-; GFX9-NEXT:    s_cmp_gt_i32 s17, s30
-; GFX9-NEXT:    s_cselect_b32 s17, s17, s30
-; GFX9-NEXT:    s_cmp_lt_i32 s17, s16
-; GFX9-NEXT:    s_cselect_b32 s16, s17, s16
-; GFX9-NEXT:    s_add_i32 s14, s14, s16
-; GFX9-NEXT:    s_cmp_gt_i32 s15, 0
-; GFX9-NEXT:    s_cselect_b32 s16, s15, 0
-; GFX9-NEXT:    s_sub_i32 s16, s32, s16
-; GFX9-NEXT:    s_cmp_lt_i32 s15, 0
-; GFX9-NEXT:    s_cselect_b32 s17, s15, 0
-; GFX9-NEXT:    s_sub_i32 s17, s33, s17
-; GFX9-NEXT:    s_cmp_gt_i32 s17, s31
-; GFX9-NEXT:    s_cselect_b32 s17, s17, s31
-; GFX9-NEXT:    s_cmp_lt_i32 s17, s16
-; GFX9-NEXT:    s_cselect_b32 s16, s17, s16
-; GFX9-NEXT:    s_add_i32 s15, s15, s16
+; GFX9-NEXT:    v_mov_b32_e32 v0, s16
+; GFX9-NEXT:    v_mov_b32_e32 v1, s17
+; GFX9-NEXT:    v_mov_b32_e32 v2, s18
+; GFX9-NEXT:    v_mov_b32_e32 v3, s19
+; GFX9-NEXT:    v_mov_b32_e32 v4, s20
+; GFX9-NEXT:    v_mov_b32_e32 v5, s21
+; GFX9-NEXT:    v_mov_b32_e32 v6, s22
+; GFX9-NEXT:    v_mov_b32_e32 v7, s23
+; GFX9-NEXT:    v_mov_b32_e32 v8, s24
+; GFX9-NEXT:    v_mov_b32_e32 v9, s25
+; GFX9-NEXT:    v_mov_b32_e32 v10, s26
+; GFX9-NEXT:    v_mov_b32_e32 v11, s27
+; GFX9-NEXT:    v_mov_b32_e32 v12, s28
+; GFX9-NEXT:    v_mov_b32_e32 v13, s29
+; GFX9-NEXT:    v_mov_b32_e32 v14, s30
+; GFX9-NEXT:    v_mov_b32_e32 v15, s31
+; GFX9-NEXT:    v_add_i32 v0, s0, v0 clamp
+; GFX9-NEXT:    v_add_i32 v1, s1, v1 clamp
+; GFX9-NEXT:    v_add_i32 v2, s2, v2 clamp
+; GFX9-NEXT:    v_add_i32 v3, s3, v3 clamp
+; GFX9-NEXT:    v_add_i32 v4, s4, v4 clamp
+; GFX9-NEXT:    v_add_i32 v5, s5, v5 clamp
+; GFX9-NEXT:    v_add_i32 v6, s6, v6 clamp
+; GFX9-NEXT:    v_add_i32 v7, s7, v7 clamp
+; GFX9-NEXT:    v_add_i32 v8, s8, v8 clamp
+; GFX9-NEXT:    v_add_i32 v9, s9, v9 clamp
+; GFX9-NEXT:    v_add_i32 v10, s10, v10 clamp
+; GFX9-NEXT:    v_add_i32 v11, s11, v11 clamp
+; GFX9-NEXT:    v_add_i32 v12, s12, v12 clamp
+; GFX9-NEXT:    v_add_i32 v13, s13, v13 clamp
+; GFX9-NEXT:    v_add_i32 v14, s14, v14 clamp
+; GFX9-NEXT:    v_add_i32 v15, s15, v15 clamp
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX9-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX9-NEXT:    v_readfirstlane_b32 s4, v4
+; GFX9-NEXT:    v_readfirstlane_b32 s5, v5
+; GFX9-NEXT:    v_readfirstlane_b32 s6, v6
+; GFX9-NEXT:    v_readfirstlane_b32 s7, v7
+; GFX9-NEXT:    v_readfirstlane_b32 s8, v8
+; GFX9-NEXT:    v_readfirstlane_b32 s9, v9
+; GFX9-NEXT:    v_readfirstlane_b32 s10, v10
+; GFX9-NEXT:    v_readfirstlane_b32 s11, v11
+; GFX9-NEXT:    v_readfirstlane_b32 s12, v12
+; GFX9-NEXT:    v_readfirstlane_b32 s13, v13
+; GFX9-NEXT:    v_readfirstlane_b32 s14, v14
+; GFX9-NEXT:    v_readfirstlane_b32 s15, v15
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_saddsat_v16i32:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_cmp_gt_i32 s0, 0
-; GFX10-NEXT:    s_brev_b32 s32, -2
-; GFX10-NEXT:    s_cselect_b32 s33, s0, 0
-; GFX10-NEXT:    s_brev_b32 s34, 1
-; GFX10-NEXT:    s_sub_i32 s46, s32, s33
-; GFX10-NEXT:    s_cmp_lt_i32 s0, 0
+; GFX10-NEXT:    v_add_nc_i32 v0, s0, s16 clamp
+; GFX10-NEXT:    v_add_nc_i32 v1, s1, s17 clamp
+; GFX10-NEXT:    v_add_nc_i32 v2, s2, s18 clamp
+; GFX10-NEXT:    v_add_nc_i32 v3, s3, s19 clamp
+; GFX10-NEXT:    v_add_nc_i32 v4, s4, s20 clamp
+; GFX10-NEXT:    v_add_nc_i32 v5, s5, s21 clamp
+; GFX10-NEXT:    v_add_nc_i32 v6, s6, s22 clamp
+; GFX10-NEXT:    v_add_nc_i32 v7, s7, s23 clamp
+; GFX10-NEXT:    v_add_nc_i32 v8, s8, s24 clamp
+; GFX10-NEXT:    v_add_nc_i32 v9, s9, s25 clamp
+; GFX10-NEXT:    v_add_nc_i32 v10, s10, s26 clamp
+; GFX10-NEXT:    v_add_nc_i32 v11, s11, s27 clamp
+; GFX10-NEXT:    v_add_nc_i32 v12, s12, s28 clamp
+; GFX10-NEXT:    v_add_nc_i32 v13, s13, s29 clamp
+; GFX10-NEXT:    v_add_nc_i32 v14, s14, s30 clamp
+; GFX10-NEXT:    v_add_nc_i32 v15, s15, s31 clamp
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX10-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX10-NEXT:    v_readfirstlane_b32 s4, v4
+; GFX10-NEXT:    v_readfirstlane_b32 s5, v5
+; GFX10-NEXT:    v_readfirstlane_b32 s6, v6
+; GFX10-NEXT:    v_readfirstlane_b32 s7, v7
+; GFX10-NEXT:    v_readfirstlane_b32 s8, v8
+; GFX10-NEXT:    v_readfirstlane_b32 s9, v9
+; GFX10-NEXT:    v_readfirstlane_b32 s10, v10
+; GFX10-NEXT:    v_readfirstlane_b32 s11, v11
+; GFX10-NEXT:    v_readfirstlane_b32 s12, v12
+; GFX10-NEXT:    v_readfirstlane_b32 s13, v13
+; GFX10-NEXT:    v_readfirstlane_b32 s14, v14
+; GFX10-NEXT:    v_readfirstlane_b32 s15, v15
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cselect_b32 s35, s0, 0
-; GFX10-NEXT:    s_sub_i32 s35, s34, s35
-; GFX10-NEXT:    s_cmp_gt_i32 s35, s16
-; GFX10-NEXT:    s_cselect_b32 s16, s35, s16
-; GFX10-NEXT:    s_cmp_lt_i32 s16, s46
-; GFX10-NEXT:    s_cselect_b32 s46, s16, s46
-; GFX10-NEXT:    s_add_i32 s0, s0, s46
-; GFX10-NEXT:    s_cmp_gt_i32 s1, 0
-; GFX10-NEXT:    s_cselect_b32 s46, s1, 0
-; GFX10-NEXT:    s_sub_i32 s16, s32, s46
-; GFX10-NEXT:    s_cmp_lt_i32 s1, 0
-; GFX10-NEXT:    s_cselect_b32 s33, s1, 0
-; GFX10-NEXT:    s_sub_i32 s46, s34, s33
-; GFX10-NEXT:    s_cmp_gt_i32 s46, s17
-; GFX10-NEXT:    s_cselect_b32 s17, s46, s17
-; GFX10-NEXT:    s_cmp_lt_i32 s17, s16
-; GFX10-NEXT:    s_cselect_b32 s46, s17, s16
-; GFX10-NEXT:    s_add_i32 s1, s1, s46
-; GFX10-NEXT:    s_cmp_gt_i32 s2, 0
-; GFX10-NEXT:    s_cselect_b32 s46, s2, 0
-; GFX10-NEXT:    s_sub_i32 s16, s32, s46
-; GFX10-NEXT:    s_cmp_lt_i32 s2, 0
-; GFX10-NEXT:    s_cselect_b32 s17, s2, 0
-; GFX10-NEXT:    s_sub_i32 s17, s34, s17
-; GFX10-NEXT:    s_cmp_gt_i32 s17, s18
-; GFX10-NEXT:    s_cselect_b32 s17, s17, s18
-; GFX10-NEXT:    s_cmp_lt_i32 s17, s16
-; GFX10-NEXT:    s_cselect_b32 s16, s17, s16
-; GFX10-NEXT:    s_add_i32 s2, s2, s16
-; GFX10-NEXT:    s_cmp_gt_i32 s3, 0
-; GFX10-NEXT:    s_cselect_b32 s46, s3, 0
-; GFX10-NEXT:    s_sub_i32 s16, s32, s46
-; GFX10-NEXT:    s_cmp_lt_i32 s3, 0
-; GFX10-NEXT:    s_cselect_b32 s17, s3, 0
-; GFX10-NEXT:    s_sub_i32 s17, s34, s17
-; GFX10-NEXT:    s_cmp_gt_i32 s17, s19
-; GFX10-NEXT:    s_cselect_b32 s17, s17, s19
-; GFX10-NEXT:    s_cmp_lt_i32 s17, s16
-; GFX10-NEXT:    s_cselect_b32 s16, s17, s16
-; GFX10-NEXT:    s_add_i32 s3, s3, s16
-; GFX10-NEXT:    s_cmp_gt_i32 s4, 0
-; GFX10-NEXT:    s_cselect_b32 s46, s4, 0
-; GFX10-NEXT:    s_sub_i32 s16, s32, s46
-; GFX10-NEXT:    s_cmp_lt_i32 s4, 0
-; GFX10-NEXT:    s_cselect_b32 s17, s4, 0
-; GFX10-NEXT:    s_sub_i32 s17, s34, s17
-; GFX10-NEXT:    s_cmp_gt_i32 s17, s20
-; GFX10-NEXT:    s_cselect_b32 s17, s17, s20
-; GFX10-NEXT:    s_cmp_lt_i32 s17, s16
-; GFX10-NEXT:    s_cselect_b32 s16, s17, s16
-; GFX10-NEXT:    s_add_i32 s4, s4, s16
-; GFX10-NEXT:    s_cmp_gt_i32 s5, 0
-; GFX10-NEXT:    s_cselect_b32 s46, s5, 0
-; GFX10-NEXT:    s_sub_i32 s16, s32, s46
-; GFX10-NEXT:    s_cmp_lt_i32 s5, 0
-; GFX10-NEXT:    s_cselect_b32 s17, s5, 0
-; GFX10-NEXT:    s_sub_i32 s17, s34, s17
-; GFX10-NEXT:    s_cmp_gt_i32 s17, s21
-; GFX10-NEXT:    s_cselect_b32 s17, s17, s21
-; GFX10-NEXT:    s_cmp_lt_i32 s17, s16
-; GFX10-NEXT:    s_cselect_b32 s16, s17, s16
-; GFX10-NEXT:    s_add_i32 s5, s5, s16
-; GFX10-NEXT:    s_cmp_gt_i32 s6, 0
-; GFX10-NEXT:    s_cselect_b32 s46, s6, 0
-; GFX10-NEXT:    s_sub_i32 s16, s32, s46
-; GFX10-NEXT:    s_cmp_lt_i32 s6, 0
-; GFX10-NEXT:    s_cselect_b32 s17, s6, 0
-; GFX10-NEXT:    s_sub_i32 s17, s34, s17
-; GFX10-NEXT:    s_cmp_gt_i32 s17, s22
-; GFX10-NEXT:    s_cselect_b32 s17, s17, s22
-; GFX10-NEXT:    s_cmp_lt_i32 s17, s16
-; GFX10-NEXT:    s_cselect_b32 s16, s17, s16
-; GFX10-NEXT:    s_add_i32 s6, s6, s16
-; GFX10-NEXT:    s_cmp_gt_i32 s7, 0
-; GFX10-NEXT:    s_cselect_b32 s46, s7, 0
-; GFX10-NEXT:    s_sub_i32 s16, s32, s46
-; GFX10-NEXT:    s_cmp_lt_i32 s7, 0
-; GFX10-NEXT:    s_cselect_b32 s17, s7, 0
-; GFX10-NEXT:    s_sub_i32 s17, s34, s17
-; GFX10-NEXT:    s_cmp_gt_i32 s17, s23
-; GFX10-NEXT:    s_cselect_b32 s17, s17, s23
-; GFX10-NEXT:    s_cmp_lt_i32 s17, s16
-; GFX10-NEXT:    s_cselect_b32 s16, s17, s16
-; GFX10-NEXT:    s_add_i32 s7, s7, s16
-; GFX10-NEXT:    s_cmp_gt_i32 s8, 0
-; GFX10-NEXT:    s_cselect_b32 s46, s8, 0
-; GFX10-NEXT:    s_sub_i32 s16, s32, s46
-; GFX10-NEXT:    s_cmp_lt_i32 s8, 0
-; GFX10-NEXT:    s_cselect_b32 s17, s8, 0
-; GFX10-NEXT:    s_sub_i32 s17, s34, s17
-; GFX10-NEXT:    s_cmp_gt_i32 s17, s24
-; GFX10-NEXT:    s_cselect_b32 s17, s17, s24
-; GFX10-NEXT:    s_cmp_lt_i32 s17, s16
-; GFX10-NEXT:    s_cselect_b32 s16, s17, s16
-; GFX10-NEXT:    s_add_i32 s8, s8, s16
-; GFX10-NEXT:    s_cmp_gt_i32 s9, 0
-; GFX10-NEXT:    s_cselect_b32 s46, s9, 0
-; GFX10-NEXT:    s_sub_i32 s16, s32, s46
-; GFX10-NEXT:    s_cmp_lt_i32 s9, 0
-; GFX10-NEXT:    s_cselect_b32 s17, s9, 0
-; GFX10-NEXT:    s_sub_i32 s17, s34, s17
-; GFX10-NEXT:    s_cmp_gt_i32 s17, s25
-; GFX10-NEXT:    s_cselect_b32 s17, s17, s25
-; GFX10-NEXT:    s_cmp_lt_i32 s17, s16
-; GFX10-NEXT:    s_cselect_b32 s16, s17, s16
-; GFX10-NEXT:    s_add_i32 s9, s9, s16
-; GFX10-NEXT:    s_cmp_gt_i32 s10, 0
-; GFX10-NEXT:    s_cselect_b32 s46, s10, 0
-; GFX10-NEXT:    s_sub_i32 s16, s32, s46
-; GFX10-NEXT:    s_cmp_lt_i32 s10, 0
-; GFX10-NEXT:    s_cselect_b32 s17, s10, 0
-; GFX10-NEXT:    s_sub_i32 s17, s34, s17
-; GFX10-NEXT:    s_cmp_gt_i32 s17, s26
-; GFX10-NEXT:    s_cselect_b32 s17, s17, s26
-; GFX10-NEXT:    s_cmp_lt_i32 s17, s16
-; GFX10-NEXT:    s_cselect_b32 s16, s17, s16
-; GFX10-NEXT:    s_add_i32 s10, s10, s16
-; GFX10-NEXT:    s_cmp_gt_i32 s11, 0
-; GFX10-NEXT:    s_cselect_b32 s46, s11, 0
-; GFX10-NEXT:    s_sub_i32 s16, s32, s46
-; GFX10-NEXT:    s_cmp_lt_i32 s11, 0
-; GFX10-NEXT:    s_cselect_b32 s17, s11, 0
-; GFX10-NEXT:    s_sub_i32 s17, s34, s17
-; GFX10-NEXT:    s_cmp_gt_i32 s17, s27
-; GFX10-NEXT:    s_cselect_b32 s17, s17, s27
-; GFX10-NEXT:    s_cmp_lt_i32 s17, s16
-; GFX10-NEXT:    s_cselect_b32 s16, s17, s16
-; GFX10-NEXT:    s_add_i32 s11, s11, s16
-; GFX10-NEXT:    s_cmp_gt_i32 s12, 0
-; GFX10-NEXT:    s_cselect_b32 s46, s12, 0
-; GFX10-NEXT:    s_sub_i32 s16, s32, s46
-; GFX10-NEXT:    s_cmp_lt_i32 s12, 0
-; GFX10-NEXT:    s_cselect_b32 s17, s12, 0
-; GFX10-NEXT:    s_sub_i32 s17, s34, s17
-; GFX10-NEXT:    s_cmp_gt_i32 s17, s28
-; GFX10-NEXT:    s_cselect_b32 s17, s17, s28
-; GFX10-NEXT:    s_cmp_lt_i32 s17, s16
-; GFX10-NEXT:    s_cselect_b32 s16, s17, s16
-; GFX10-NEXT:    s_add_i32 s12, s12, s16
-; GFX10-NEXT:    s_cmp_gt_i32 s13, 0
-; GFX10-NEXT:    s_cselect_b32 s46, s13, 0
-; GFX10-NEXT:    s_sub_i32 s16, s32, s46
-; GFX10-NEXT:    s_cmp_lt_i32 s13, 0
-; GFX10-NEXT:    s_cselect_b32 s17, s13, 0
-; GFX10-NEXT:    s_sub_i32 s17, s34, s17
-; GFX10-NEXT:    s_cmp_gt_i32 s17, s29
-; GFX10-NEXT:    s_cselect_b32 s17, s17, s29
-; GFX10-NEXT:    s_cmp_lt_i32 s17, s16
-; GFX10-NEXT:    s_cselect_b32 s16, s17, s16
-; GFX10-NEXT:    s_add_i32 s13, s13, s16
-; GFX10-NEXT:    s_cmp_gt_i32 s14, 0
-; GFX10-NEXT:    s_cselect_b32 s46, s14, 0
-; GFX10-NEXT:    s_sub_i32 s16, s32, s46
-; GFX10-NEXT:    s_cmp_lt_i32 s14, 0
-; GFX10-NEXT:    s_cselect_b32 s17, s14, 0
-; GFX10-NEXT:    s_sub_i32 s17, s34, s17
-; GFX10-NEXT:    s_cmp_gt_i32 s17, s30
-; GFX10-NEXT:    s_cselect_b32 s17, s17, s30
-; GFX10-NEXT:    s_cmp_lt_i32 s17, s16
-; GFX10-NEXT:    s_cselect_b32 s16, s17, s16
-; GFX10-NEXT:    s_add_i32 s14, s14, s16
-; GFX10-NEXT:    s_cmp_gt_i32 s15, 0
-; GFX10-NEXT:    s_cselect_b32 s30, s15, 0
-; GFX10-NEXT:    s_sub_i32 s16, s32, s30
-; GFX10-NEXT:    s_cmp_lt_i32 s15, 0
-; GFX10-NEXT:    s_cselect_b32 s17, s15, 0
-; GFX10-NEXT:    s_sub_i32 s17, s34, s17
-; GFX10-NEXT:    s_cmp_gt_i32 s17, s31
-; GFX10-NEXT:    s_cselect_b32 s17, s17, s31
-; GFX10-NEXT:    s_cmp_lt_i32 s17, s16
-; GFX10-NEXT:    s_cselect_b32 s16, s17, s16
-; GFX10-NEXT:    s_add_i32 s15, s15, s16
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <16 x i32> @llvm.sadd.sat.v16i32(<16 x i32> %lhs, <16 x i32> %rhs)
   ret <16 x i32> %result
@@ -4241,27 +2855,15 @@ define i16 @v_saddsat_i16(i16 %lhs, i16 %rhs) {
 ; GFX9-LABEL: v_saddsat_i16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_min_i16_e32 v3, 0, v0
-; GFX9-NEXT:    v_max_i16_e32 v2, 0, v0
-; GFX9-NEXT:    v_sub_u16_e32 v3, 0x8000, v3
-; GFX9-NEXT:    v_sub_u16_e32 v2, 0x7fff, v2
-; GFX9-NEXT:    v_max_i16_e32 v1, v3, v1
-; GFX9-NEXT:    v_min_i16_e32 v1, v1, v2
-; GFX9-NEXT:    v_add_u16_e32 v0, v0, v1
+; GFX9-NEXT:    v_add_i16 v0, v0, v1 clamp
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_saddsat_i16:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_min_i16_e64 v2, v0, 0
-; GFX10-NEXT:    v_max_i16_e64 v3, v0, 0
+; GFX10-NEXT:    v_add_nc_i16 v0, v0, v1 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_sub_nc_u16_e64 v2, 0x8000, v2
-; GFX10-NEXT:    v_sub_nc_u16_e64 v3, 0x7fff, v3
-; GFX10-NEXT:    v_max_i16_e64 v1, v2, v1
-; GFX10-NEXT:    v_min_i16_e64 v1, v1, v3
-; GFX10-NEXT:    v_add_nc_u16_e64 v0, v0, v1
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call i16 @llvm.sadd.sat.i16(i16 %lhs, i16 %rhs)
   ret i16 %result
@@ -4309,45 +2911,16 @@ define amdgpu_ps i16 @s_saddsat_i16(i16 inreg %lhs, i16 inreg %rhs) {
 ;
 ; GFX9-LABEL: s_saddsat_i16:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_sext_i32_i16 s2, s0
-; GFX9-NEXT:    s_sext_i32_i16 s3, 0
-; GFX9-NEXT:    s_cmp_gt_i32 s2, s3
-; GFX9-NEXT:    s_cselect_b32 s4, s2, s3
-; GFX9-NEXT:    s_sub_i32 s4, 0x7fff, s4
-; GFX9-NEXT:    s_cmp_lt_i32 s2, s3
-; GFX9-NEXT:    s_cselect_b32 s2, s2, s3
-; GFX9-NEXT:    s_sub_i32 s2, 0xffff8000, s2
-; GFX9-NEXT:    s_sext_i32_i16 s2, s2
-; GFX9-NEXT:    s_sext_i32_i16 s1, s1
-; GFX9-NEXT:    s_cmp_gt_i32 s2, s1
-; GFX9-NEXT:    s_cselect_b32 s1, s2, s1
-; GFX9-NEXT:    s_sext_i32_i16 s1, s1
-; GFX9-NEXT:    s_sext_i32_i16 s2, s4
-; GFX9-NEXT:    s_cmp_lt_i32 s1, s2
-; GFX9-NEXT:    s_cselect_b32 s1, s1, s2
-; GFX9-NEXT:    s_add_i32 s0, s0, s1
+; GFX9-NEXT:    v_mov_b32_e32 v0, s1
+; GFX9-NEXT:    v_add_i16 v0, s0, v0 clamp
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_saddsat_i16:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_sext_i32_i16 s2, 0
-; GFX10-NEXT:    s_sext_i32_i16 s3, s0
-; GFX10-NEXT:    s_sext_i32_i16 s1, s1
-; GFX10-NEXT:    s_cmp_gt_i32 s3, s2
+; GFX10-NEXT:    v_add_nc_i16 v0, s0, s1 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cselect_b32 s4, s3, s2
-; GFX10-NEXT:    s_sub_i32 s4, 0x7fff, s4
-; GFX10-NEXT:    s_cmp_lt_i32 s3, s2
-; GFX10-NEXT:    s_cselect_b32 s2, s3, s2
-; GFX10-NEXT:    s_sub_i32 s2, 0xffff8000, s2
-; GFX10-NEXT:    s_sext_i32_i16 s2, s2
-; GFX10-NEXT:    s_cmp_gt_i32 s2, s1
-; GFX10-NEXT:    s_cselect_b32 s1, s2, s1
-; GFX10-NEXT:    s_sext_i32_i16 s2, s4
-; GFX10-NEXT:    s_sext_i32_i16 s1, s1
-; GFX10-NEXT:    s_cmp_lt_i32 s1, s2
-; GFX10-NEXT:    s_cselect_b32 s1, s1, s2
-; GFX10-NEXT:    s_add_i32 s0, s0, s1
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call i16 @llvm.sadd.sat.i16(i16 %lhs, i16 %rhs)
   ret i16 %result
@@ -4387,33 +2960,13 @@ define amdgpu_ps half @saddsat_i16_sv(i16 inreg %lhs, i16 %rhs) {
 ;
 ; GFX9-LABEL: saddsat_i16_sv:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_sext_i32_i16 s1, s0
-; GFX9-NEXT:    s_sext_i32_i16 s2, 0
-; GFX9-NEXT:    s_cmp_gt_i32 s1, s2
-; GFX9-NEXT:    s_cselect_b32 s3, s1, s2
-; GFX9-NEXT:    s_sub_i32 s3, 0x7fff, s3
-; GFX9-NEXT:    s_cmp_lt_i32 s1, s2
-; GFX9-NEXT:    s_cselect_b32 s1, s1, s2
-; GFX9-NEXT:    s_sub_i32 s1, 0xffff8000, s1
-; GFX9-NEXT:    v_max_i16_e32 v0, s1, v0
-; GFX9-NEXT:    v_min_i16_e32 v0, s3, v0
-; GFX9-NEXT:    v_add_u16_e32 v0, s0, v0
+; GFX9-NEXT:    v_add_i16 v0, s0, v0 clamp
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: saddsat_i16_sv:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_sext_i32_i16 s1, s0
-; GFX10-NEXT:    s_sext_i32_i16 s2, 0
+; GFX10-NEXT:    v_add_nc_i16 v0, s0, v0 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cmp_gt_i32 s1, s2
-; GFX10-NEXT:    s_cselect_b32 s3, s1, s2
-; GFX10-NEXT:    s_sub_i32 s3, 0x7fff, s3
-; GFX10-NEXT:    s_cmp_lt_i32 s1, s2
-; GFX10-NEXT:    s_cselect_b32 s1, s1, s2
-; GFX10-NEXT:    s_sub_i32 s1, 0xffff8000, s1
-; GFX10-NEXT:    v_max_i16_e64 v0, s1, v0
-; GFX10-NEXT:    v_min_i16_e64 v0, v0, s3
-; GFX10-NEXT:    v_add_nc_u16_e64 v0, s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call i16 @llvm.sadd.sat.i16(i16 %lhs, i16 %rhs)
   %cast = bitcast i16 %result to half
@@ -4448,25 +3001,13 @@ define amdgpu_ps half @saddsat_i16_vs(i16 %lhs, i16 inreg %rhs) {
 ;
 ; GFX9-LABEL: saddsat_i16_vs:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    v_min_i16_e32 v2, 0, v0
-; GFX9-NEXT:    v_max_i16_e32 v1, 0, v0
-; GFX9-NEXT:    v_sub_u16_e32 v2, 0x8000, v2
-; GFX9-NEXT:    v_sub_u16_e32 v1, 0x7fff, v1
-; GFX9-NEXT:    v_max_i16_e32 v2, s0, v2
-; GFX9-NEXT:    v_min_i16_e32 v1, v2, v1
-; GFX9-NEXT:    v_add_u16_e32 v0, v0, v1
+; GFX9-NEXT:    v_add_i16 v0, v0, s0 clamp
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: saddsat_i16_vs:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    v_min_i16_e64 v1, v0, 0
-; GFX10-NEXT:    v_max_i16_e64 v2, v0, 0
+; GFX10-NEXT:    v_add_nc_i16 v0, v0, s0 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_sub_nc_u16_e64 v1, 0x8000, v1
-; GFX10-NEXT:    v_sub_nc_u16_e64 v2, 0x7fff, v2
-; GFX10-NEXT:    v_max_i16_e64 v1, v1, s0
-; GFX10-NEXT:    v_min_i16_e64 v1, v1, v2
-; GFX10-NEXT:    v_add_nc_u16_e64 v0, v0, v1
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call i16 @llvm.sadd.sat.i16(i16 %lhs, i16 %rhs)
   %cast = bitcast i16 %result to half
@@ -4527,29 +3068,15 @@ define <2 x i16> @v_saddsat_v2i16(<2 x i16> %lhs, <2 x i16> %rhs) {
 ; GFX9-LABEL: v_saddsat_v2i16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_pk_max_i16 v2, v0, 0
-; GFX9-NEXT:    v_mov_b32_e32 v3, 0x7fff7fff
-; GFX9-NEXT:    v_pk_sub_i16 v2, v3, v2
-; GFX9-NEXT:    v_pk_min_i16 v3, v0, 0
-; GFX9-NEXT:    v_mov_b32_e32 v4, 0x80008000
-; GFX9-NEXT:    v_pk_sub_i16 v3, v4, v3
-; GFX9-NEXT:    v_pk_max_i16 v1, v3, v1
-; GFX9-NEXT:    v_pk_min_i16 v1, v1, v2
-; GFX9-NEXT:    v_pk_add_u16 v0, v0, v1
+; GFX9-NEXT:    v_pk_add_i16 v0, v0, v1 clamp
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_saddsat_v2i16:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_pk_min_i16 v2, v0, 0
-; GFX10-NEXT:    v_pk_max_i16 v3, v0, 0
+; GFX10-NEXT:    v_pk_add_i16 v0, v0, v1 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_pk_sub_i16 v2, 0x80008000, v2
-; GFX10-NEXT:    v_pk_sub_i16 v3, 0x7fff7fff, v3
-; GFX10-NEXT:    v_pk_max_i16 v1, v2, v1
-; GFX10-NEXT:    v_pk_min_i16 v1, v1, v3
-; GFX10-NEXT:    v_pk_add_u16 v0, v0, v1
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <2 x i16> @llvm.sadd.sat.v2i16(<2 x i16> %lhs, <2 x i16> %rhs)
   ret <2 x i16> %result
@@ -4642,99 +3169,16 @@ define amdgpu_ps i32 @s_saddsat_v2i16(<2 x i16> inreg %lhs, <2 x i16> inreg %rhs
 ;
 ; GFX9-LABEL: s_saddsat_v2i16:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_sext_i32_i16 s2, s0
-; GFX9-NEXT:    s_ashr_i32 s3, s0, 16
-; GFX9-NEXT:    s_sext_i32_i16 s4, 0
-; GFX9-NEXT:    s_cmp_gt_i32 s2, s4
-; GFX9-NEXT:    s_cselect_b32 s5, s2, s4
-; GFX9-NEXT:    s_cmp_gt_i32 s3, 0
-; GFX9-NEXT:    s_cselect_b32 s6, s3, 0
-; GFX9-NEXT:    s_pack_ll_b32_b16 s5, s5, s6
-; GFX9-NEXT:    s_lshr_b32 s6, s5, 16
-; GFX9-NEXT:    s_sub_i32 s5, 0x7fff7fff, s5
-; GFX9-NEXT:    s_sub_i32 s6, 0x7fff, s6
-; GFX9-NEXT:    s_cmp_lt_i32 s2, s4
-; GFX9-NEXT:    s_cselect_b32 s2, s2, s4
-; GFX9-NEXT:    s_cmp_lt_i32 s3, 0
-; GFX9-NEXT:    s_cselect_b32 s3, s3, 0
-; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s3
-; GFX9-NEXT:    s_lshr_b32 s3, s2, 16
-; GFX9-NEXT:    s_sub_i32 s2, 0x80008000, s2
-; GFX9-NEXT:    s_sub_i32 s3, 0x8000, s3
-; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s3
-; GFX9-NEXT:    s_sext_i32_i16 s3, s2
-; GFX9-NEXT:    s_sext_i32_i16 s4, s1
-; GFX9-NEXT:    s_ashr_i32 s2, s2, 16
-; GFX9-NEXT:    s_ashr_i32 s1, s1, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s3, s4
-; GFX9-NEXT:    s_cselect_b32 s3, s3, s4
-; GFX9-NEXT:    s_cmp_gt_i32 s2, s1
-; GFX9-NEXT:    s_cselect_b32 s1, s2, s1
-; GFX9-NEXT:    s_pack_ll_b32_b16 s5, s5, s6
-; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s3, s1
-; GFX9-NEXT:    s_sext_i32_i16 s2, s1
-; GFX9-NEXT:    s_ashr_i32 s1, s1, 16
-; GFX9-NEXT:    s_sext_i32_i16 s3, s5
-; GFX9-NEXT:    s_ashr_i32 s4, s5, 16
-; GFX9-NEXT:    s_cmp_lt_i32 s2, s3
-; GFX9-NEXT:    s_cselect_b32 s2, s2, s3
-; GFX9-NEXT:    s_cmp_lt_i32 s1, s4
-; GFX9-NEXT:    s_cselect_b32 s1, s1, s4
-; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s2, s1
-; GFX9-NEXT:    s_lshr_b32 s2, s0, 16
-; GFX9-NEXT:    s_lshr_b32 s3, s1, 16
-; GFX9-NEXT:    s_add_i32 s0, s0, s1
-; GFX9-NEXT:    s_add_i32 s2, s2, s3
-; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s2
+; GFX9-NEXT:    v_mov_b32_e32 v0, s1
+; GFX9-NEXT:    v_pk_add_i16 v0, s0, v0 clamp
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_saddsat_v2i16:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_sext_i32_i16 s2, s0
-; GFX10-NEXT:    s_sext_i32_i16 s3, 0
-; GFX10-NEXT:    s_ashr_i32 s4, s0, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s2, s3
+; GFX10-NEXT:    v_pk_add_i16 v0, s0, s1 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cselect_b32 s5, s2, s3
-; GFX10-NEXT:    s_cmp_gt_i32 s4, 0
-; GFX10-NEXT:    s_cselect_b32 s6, s4, 0
-; GFX10-NEXT:    s_pack_ll_b32_b16 s5, s5, s6
-; GFX10-NEXT:    s_lshr_b32 s6, s5, 16
-; GFX10-NEXT:    s_sub_i32 s5, 0x7fff7fff, s5
-; GFX10-NEXT:    s_sub_i32 s6, 0x7fff, s6
-; GFX10-NEXT:    s_cmp_lt_i32 s2, s3
-; GFX10-NEXT:    s_cselect_b32 s2, s2, s3
-; GFX10-NEXT:    s_cmp_lt_i32 s4, 0
-; GFX10-NEXT:    s_cselect_b32 s3, s4, 0
-; GFX10-NEXT:    s_sext_i32_i16 s4, s1
-; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s2, s3
-; GFX10-NEXT:    s_ashr_i32 s1, s1, 16
-; GFX10-NEXT:    s_lshr_b32 s3, s2, 16
-; GFX10-NEXT:    s_sub_i32 s2, 0x80008000, s2
-; GFX10-NEXT:    s_sub_i32 s3, 0x8000, s3
-; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s2, s3
-; GFX10-NEXT:    s_sext_i32_i16 s3, s2
-; GFX10-NEXT:    s_ashr_i32 s2, s2, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s3, s4
-; GFX10-NEXT:    s_cselect_b32 s3, s3, s4
-; GFX10-NEXT:    s_cmp_gt_i32 s2, s1
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s5, s6
-; GFX10-NEXT:    s_cselect_b32 s1, s2, s1
-; GFX10-NEXT:    s_sext_i32_i16 s2, s4
-; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s3, s1
-; GFX10-NEXT:    s_ashr_i32 s3, s4, 16
-; GFX10-NEXT:    s_sext_i32_i16 s4, s1
-; GFX10-NEXT:    s_ashr_i32 s1, s1, 16
-; GFX10-NEXT:    s_cmp_lt_i32 s4, s2
-; GFX10-NEXT:    s_cselect_b32 s2, s4, s2
-; GFX10-NEXT:    s_cmp_lt_i32 s1, s3
-; GFX10-NEXT:    s_cselect_b32 s1, s1, s3
-; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s2, s1
-; GFX10-NEXT:    s_lshr_b32 s2, s0, 16
-; GFX10-NEXT:    s_lshr_b32 s3, s1, 16
-; GFX10-NEXT:    s_add_i32 s0, s0, s1
-; GFX10-NEXT:    s_add_i32 s2, s2, s3
-; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s2
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <2 x i16> @llvm.sadd.sat.v2i16(<2 x i16> %lhs, <2 x i16> %rhs)
   %cast = bitcast <2 x i16> %result to i32
@@ -4810,59 +3254,13 @@ define amdgpu_ps float @saddsat_v2i16_sv(<2 x i16> inreg %lhs, <2 x i16> %rhs) {
 ;
 ; GFX9-LABEL: saddsat_v2i16_sv:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_sext_i32_i16 s1, s0
-; GFX9-NEXT:    s_ashr_i32 s2, s0, 16
-; GFX9-NEXT:    s_sext_i32_i16 s3, 0
-; GFX9-NEXT:    s_cmp_gt_i32 s1, s3
-; GFX9-NEXT:    s_cselect_b32 s4, s1, s3
-; GFX9-NEXT:    s_cmp_gt_i32 s2, 0
-; GFX9-NEXT:    s_cselect_b32 s5, s2, 0
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
-; GFX9-NEXT:    s_lshr_b32 s5, s4, 16
-; GFX9-NEXT:    s_sub_i32 s4, 0x7fff7fff, s4
-; GFX9-NEXT:    s_sub_i32 s5, 0x7fff, s5
-; GFX9-NEXT:    s_cmp_lt_i32 s1, s3
-; GFX9-NEXT:    s_cselect_b32 s1, s1, s3
-; GFX9-NEXT:    s_cmp_lt_i32 s2, 0
-; GFX9-NEXT:    s_cselect_b32 s2, s2, 0
-; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s2
-; GFX9-NEXT:    s_lshr_b32 s2, s1, 16
-; GFX9-NEXT:    s_sub_i32 s1, 0x80008000, s1
-; GFX9-NEXT:    s_sub_i32 s2, 0x8000, s2
-; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s2
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
-; GFX9-NEXT:    v_pk_max_i16 v0, s1, v0
-; GFX9-NEXT:    v_pk_min_i16 v0, v0, s4
-; GFX9-NEXT:    v_pk_add_u16 v0, s0, v0
+; GFX9-NEXT:    v_pk_add_i16 v0, s0, v0 clamp
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: saddsat_v2i16_sv:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_sext_i32_i16 s1, s0
-; GFX10-NEXT:    s_sext_i32_i16 s2, 0
-; GFX10-NEXT:    s_ashr_i32 s3, s0, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s1, s2
+; GFX10-NEXT:    v_pk_add_i16 v0, s0, v0 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cselect_b32 s4, s1, s2
-; GFX10-NEXT:    s_cmp_gt_i32 s3, 0
-; GFX10-NEXT:    s_cselect_b32 s5, s3, 0
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
-; GFX10-NEXT:    s_lshr_b32 s5, s4, 16
-; GFX10-NEXT:    s_sub_i32 s4, 0x7fff7fff, s4
-; GFX10-NEXT:    s_sub_i32 s5, 0x7fff, s5
-; GFX10-NEXT:    s_cmp_lt_i32 s1, s2
-; GFX10-NEXT:    s_cselect_b32 s1, s1, s2
-; GFX10-NEXT:    s_cmp_lt_i32 s3, 0
-; GFX10-NEXT:    s_cselect_b32 s2, s3, 0
-; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s1, s2
-; GFX10-NEXT:    s_lshr_b32 s2, s1, 16
-; GFX10-NEXT:    s_sub_i32 s1, 0x80008000, s1
-; GFX10-NEXT:    s_sub_i32 s2, 0x8000, s2
-; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s1, s2
-; GFX10-NEXT:    v_pk_max_i16 v0, s1, v0
-; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s4, s5
-; GFX10-NEXT:    v_pk_min_i16 v0, v0, s1
-; GFX10-NEXT:    v_pk_add_u16 v0, s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <2 x i16> @llvm.sadd.sat.v2i16(<2 x i16> %lhs, <2 x i16> %rhs)
   %cast = bitcast <2 x i16> %result to float
@@ -4926,27 +3324,13 @@ define amdgpu_ps float @saddsat_v2i16_vs(<2 x i16> %lhs, <2 x i16> inreg %rhs) {
 ;
 ; GFX9-LABEL: saddsat_v2i16_vs:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    v_pk_max_i16 v1, v0, 0
-; GFX9-NEXT:    v_mov_b32_e32 v2, 0x7fff7fff
-; GFX9-NEXT:    v_pk_sub_i16 v1, v2, v1
-; GFX9-NEXT:    v_pk_min_i16 v2, v0, 0
-; GFX9-NEXT:    v_mov_b32_e32 v3, 0x80008000
-; GFX9-NEXT:    v_pk_sub_i16 v2, v3, v2
-; GFX9-NEXT:    v_pk_max_i16 v2, v2, s0
-; GFX9-NEXT:    v_pk_min_i16 v1, v2, v1
-; GFX9-NEXT:    v_pk_add_u16 v0, v0, v1
+; GFX9-NEXT:    v_pk_add_i16 v0, v0, s0 clamp
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: saddsat_v2i16_vs:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    v_pk_min_i16 v1, v0, 0
-; GFX10-NEXT:    v_pk_max_i16 v2, v0, 0
+; GFX10-NEXT:    v_pk_add_i16 v0, v0, s0 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_pk_sub_i16 v1, 0x80008000, v1
-; GFX10-NEXT:    v_pk_sub_i16 v2, 0x7fff7fff, v2
-; GFX10-NEXT:    v_pk_max_i16 v1, v1, s0
-; GFX10-NEXT:    v_pk_min_i16 v1, v1, v2
-; GFX10-NEXT:    v_pk_add_u16 v0, v0, v1
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <2 x i16> @llvm.sadd.sat.v2i16(<2 x i16> %lhs, <2 x i16> %rhs)
   %cast = bitcast <2 x i16> %result to float
@@ -5065,43 +3449,17 @@ define <2 x float> @v_saddsat_v4i16(<4 x i16> %lhs, <4 x i16> %rhs) {
 ; GFX9-LABEL: v_saddsat_v4i16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_pk_min_i16 v6, v0, 0
-; GFX9-NEXT:    v_mov_b32_e32 v7, 0x80008000
-; GFX9-NEXT:    v_pk_sub_i16 v6, v7, v6
-; GFX9-NEXT:    v_pk_max_i16 v4, v0, 0
-; GFX9-NEXT:    v_mov_b32_e32 v5, 0x7fff7fff
-; GFX9-NEXT:    v_pk_sub_i16 v4, v5, v4
-; GFX9-NEXT:    v_pk_max_i16 v2, v6, v2
-; GFX9-NEXT:    v_pk_min_i16 v2, v2, v4
-; GFX9-NEXT:    v_pk_min_i16 v4, v1, 0
-; GFX9-NEXT:    v_pk_add_u16 v0, v0, v2
-; GFX9-NEXT:    v_pk_max_i16 v2, v1, 0
-; GFX9-NEXT:    v_pk_sub_i16 v4, v7, v4
-; GFX9-NEXT:    v_pk_sub_i16 v2, v5, v2
-; GFX9-NEXT:    v_pk_max_i16 v3, v4, v3
-; GFX9-NEXT:    v_pk_min_i16 v2, v3, v2
-; GFX9-NEXT:    v_pk_add_u16 v1, v1, v2
+; GFX9-NEXT:    v_pk_add_i16 v0, v0, v2 clamp
+; GFX9-NEXT:    v_pk_add_i16 v1, v1, v3 clamp
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_saddsat_v4i16:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_pk_min_i16 v4, v0, 0
-; GFX10-NEXT:    v_pk_min_i16 v5, v1, 0
-; GFX10-NEXT:    v_pk_max_i16 v6, v0, 0
-; GFX10-NEXT:    v_pk_max_i16 v7, v1, 0
+; GFX10-NEXT:    v_pk_add_i16 v0, v0, v2 clamp
+; GFX10-NEXT:    v_pk_add_i16 v1, v1, v3 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_pk_sub_i16 v4, 0x80008000, v4
-; GFX10-NEXT:    v_pk_sub_i16 v5, 0x80008000, v5
-; GFX10-NEXT:    v_pk_sub_i16 v6, 0x7fff7fff, v6
-; GFX10-NEXT:    v_pk_sub_i16 v7, 0x7fff7fff, v7
-; GFX10-NEXT:    v_pk_max_i16 v11, v4, v2
-; GFX10-NEXT:    v_pk_max_i16 v10, v5, v3
-; GFX10-NEXT:    v_pk_min_i16 v2, v11, v6
-; GFX10-NEXT:    v_pk_min_i16 v3, v10, v7
-; GFX10-NEXT:    v_pk_add_u16 v0, v0, v2
-; GFX10-NEXT:    v_pk_add_u16 v1, v1, v3
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <4 x i16> @llvm.sadd.sat.v4i16(<4 x i16> %lhs, <4 x i16> %rhs)
   %cast = bitcast <4 x i16> %result to <2 x float>
@@ -5265,193 +3623,21 @@ define amdgpu_ps <2 x i32> @s_saddsat_v4i16(<4 x i16> inreg %lhs, <4 x i16> inre
 ;
 ; GFX9-LABEL: s_saddsat_v4i16:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_sext_i32_i16 s6, s0
-; GFX9-NEXT:    s_ashr_i32 s7, s0, 16
-; GFX9-NEXT:    s_sext_i32_i16 s8, 0
-; GFX9-NEXT:    s_cmp_gt_i32 s6, s8
-; GFX9-NEXT:    s_cselect_b32 s9, s6, s8
-; GFX9-NEXT:    s_cmp_gt_i32 s7, 0
-; GFX9-NEXT:    s_cselect_b32 s10, s7, 0
-; GFX9-NEXT:    s_pack_ll_b32_b16 s9, s9, s10
-; GFX9-NEXT:    s_mov_b32 s4, 0x7fff7fff
-; GFX9-NEXT:    s_lshr_b32 s11, s9, 16
-; GFX9-NEXT:    s_movk_i32 s10, 0x7fff
-; GFX9-NEXT:    s_sub_i32 s9, s4, s9
-; GFX9-NEXT:    s_sub_i32 s11, s10, s11
-; GFX9-NEXT:    s_cmp_lt_i32 s6, s8
-; GFX9-NEXT:    s_cselect_b32 s6, s6, s8
-; GFX9-NEXT:    s_cmp_lt_i32 s7, 0
-; GFX9-NEXT:    s_cselect_b32 s7, s7, 0
-; GFX9-NEXT:    s_pack_ll_b32_b16 s6, s6, s7
-; GFX9-NEXT:    s_pack_ll_b32_b16 s9, s9, s11
-; GFX9-NEXT:    s_mov_b32 s5, 0x80008000
-; GFX9-NEXT:    s_lshr_b32 s11, s6, 16
-; GFX9-NEXT:    s_mov_b32 s7, 0x8000
-; GFX9-NEXT:    s_sub_i32 s6, s5, s6
-; GFX9-NEXT:    s_sub_i32 s11, s7, s11
-; GFX9-NEXT:    s_pack_ll_b32_b16 s6, s6, s11
-; GFX9-NEXT:    s_sext_i32_i16 s11, s6
-; GFX9-NEXT:    s_sext_i32_i16 s12, s2
-; GFX9-NEXT:    s_ashr_i32 s6, s6, 16
-; GFX9-NEXT:    s_ashr_i32 s2, s2, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s11, s12
-; GFX9-NEXT:    s_cselect_b32 s11, s11, s12
-; GFX9-NEXT:    s_cmp_gt_i32 s6, s2
-; GFX9-NEXT:    s_cselect_b32 s2, s6, s2
-; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s11, s2
-; GFX9-NEXT:    s_sext_i32_i16 s6, s2
-; GFX9-NEXT:    s_sext_i32_i16 s11, s9
-; GFX9-NEXT:    s_ashr_i32 s2, s2, 16
-; GFX9-NEXT:    s_ashr_i32 s9, s9, 16
-; GFX9-NEXT:    s_cmp_lt_i32 s6, s11
-; GFX9-NEXT:    s_cselect_b32 s6, s6, s11
-; GFX9-NEXT:    s_cmp_lt_i32 s2, s9
-; GFX9-NEXT:    s_cselect_b32 s2, s2, s9
-; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s6, s2
-; GFX9-NEXT:    s_lshr_b32 s6, s0, 16
-; GFX9-NEXT:    s_lshr_b32 s9, s2, 16
-; GFX9-NEXT:    s_add_i32 s0, s0, s2
-; GFX9-NEXT:    s_add_i32 s6, s6, s9
-; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s6
-; GFX9-NEXT:    s_sext_i32_i16 s2, s1
-; GFX9-NEXT:    s_ashr_i32 s6, s1, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s2, s8
-; GFX9-NEXT:    s_cselect_b32 s9, s2, s8
-; GFX9-NEXT:    s_cmp_gt_i32 s6, 0
-; GFX9-NEXT:    s_cselect_b32 s11, s6, 0
-; GFX9-NEXT:    s_pack_ll_b32_b16 s9, s9, s11
-; GFX9-NEXT:    s_lshr_b32 s11, s9, 16
-; GFX9-NEXT:    s_sub_i32 s4, s4, s9
-; GFX9-NEXT:    s_sub_i32 s9, s10, s11
-; GFX9-NEXT:    s_cmp_lt_i32 s2, s8
-; GFX9-NEXT:    s_cselect_b32 s2, s2, s8
-; GFX9-NEXT:    s_cmp_lt_i32 s6, 0
-; GFX9-NEXT:    s_cselect_b32 s6, s6, 0
-; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s6
-; GFX9-NEXT:    s_lshr_b32 s6, s2, 16
-; GFX9-NEXT:    s_sub_i32 s2, s5, s2
-; GFX9-NEXT:    s_sub_i32 s5, s7, s6
-; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s5
-; GFX9-NEXT:    s_sext_i32_i16 s5, s2
-; GFX9-NEXT:    s_sext_i32_i16 s6, s3
-; GFX9-NEXT:    s_ashr_i32 s2, s2, 16
-; GFX9-NEXT:    s_ashr_i32 s3, s3, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s5, s6
-; GFX9-NEXT:    s_cselect_b32 s5, s5, s6
-; GFX9-NEXT:    s_cmp_gt_i32 s2, s3
-; GFX9-NEXT:    s_cselect_b32 s2, s2, s3
-; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s5, s2
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s9
-; GFX9-NEXT:    s_sext_i32_i16 s3, s2
-; GFX9-NEXT:    s_sext_i32_i16 s5, s4
-; GFX9-NEXT:    s_ashr_i32 s2, s2, 16
-; GFX9-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX9-NEXT:    s_cmp_lt_i32 s3, s5
-; GFX9-NEXT:    s_cselect_b32 s3, s3, s5
-; GFX9-NEXT:    s_cmp_lt_i32 s2, s4
-; GFX9-NEXT:    s_cselect_b32 s2, s2, s4
-; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s3, s2
-; GFX9-NEXT:    s_lshr_b32 s3, s1, 16
-; GFX9-NEXT:    s_lshr_b32 s4, s2, 16
-; GFX9-NEXT:    s_add_i32 s1, s1, s2
-; GFX9-NEXT:    s_add_i32 s3, s3, s4
-; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s3
+; GFX9-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    v_pk_add_i16 v0, s0, v0 clamp
+; GFX9-NEXT:    v_pk_add_i16 v1, s1, v1 clamp
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_saddsat_v4i16:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_sext_i32_i16 s4, s0
-; GFX10-NEXT:    s_sext_i32_i16 s5, 0
-; GFX10-NEXT:    s_ashr_i32 s6, s0, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s4, s5
-; GFX10-NEXT:    s_mov_b32 s9, 0x7fff7fff
-; GFX10-NEXT:    s_cselect_b32 s7, s4, s5
-; GFX10-NEXT:    s_cmp_gt_i32 s6, 0
-; GFX10-NEXT:    s_mov_b32 s11, 0x80008000
-; GFX10-NEXT:    s_cselect_b32 s8, s6, 0
-; GFX10-NEXT:    s_sext_i32_i16 s13, s2
-; GFX10-NEXT:    s_pack_ll_b32_b16 s7, s7, s8
-; GFX10-NEXT:    s_movk_i32 s8, 0x7fff
-; GFX10-NEXT:    s_lshr_b32 s10, s7, 16
-; GFX10-NEXT:    s_sub_i32 s7, s9, s7
-; GFX10-NEXT:    s_sub_i32 s10, s8, s10
-; GFX10-NEXT:    s_cmp_lt_i32 s4, s5
+; GFX10-NEXT:    v_pk_add_i16 v0, s0, s2 clamp
+; GFX10-NEXT:    v_pk_add_i16 v1, s1, s3 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cselect_b32 s4, s4, s5
-; GFX10-NEXT:    s_cmp_lt_i32 s6, 0
-; GFX10-NEXT:    s_cselect_b32 s6, s6, 0
-; GFX10-NEXT:    s_ashr_i32 s2, s2, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s4, s6
-; GFX10-NEXT:    s_mov_b32 s6, 0x8000
-; GFX10-NEXT:    s_lshr_b32 s12, s4, 16
-; GFX10-NEXT:    s_sub_i32 s4, s11, s4
-; GFX10-NEXT:    s_sub_i32 s12, s6, s12
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s4, s12
-; GFX10-NEXT:    s_sext_i32_i16 s12, s4
-; GFX10-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s12, s13
-; GFX10-NEXT:    s_cselect_b32 s12, s12, s13
-; GFX10-NEXT:    s_cmp_gt_i32 s4, s2
-; GFX10-NEXT:    s_cselect_b32 s2, s4, s2
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s7, s10
-; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s12, s2
-; GFX10-NEXT:    s_sext_i32_i16 s10, s4
-; GFX10-NEXT:    s_sext_i32_i16 s7, s2
-; GFX10-NEXT:    s_ashr_i32 s2, s2, 16
-; GFX10-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX10-NEXT:    s_cmp_lt_i32 s7, s10
-; GFX10-NEXT:    s_cselect_b32 s7, s7, s10
-; GFX10-NEXT:    s_cmp_lt_i32 s2, s4
-; GFX10-NEXT:    s_cselect_b32 s2, s2, s4
-; GFX10-NEXT:    s_sext_i32_i16 s4, s1
-; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s7, s2
-; GFX10-NEXT:    s_lshr_b32 s7, s0, 16
-; GFX10-NEXT:    s_lshr_b32 s10, s2, 16
-; GFX10-NEXT:    s_add_i32 s0, s0, s2
-; GFX10-NEXT:    s_add_i32 s7, s7, s10
-; GFX10-NEXT:    s_ashr_i32 s2, s1, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s4, s5
-; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s7
-; GFX10-NEXT:    s_cselect_b32 s10, s4, s5
-; GFX10-NEXT:    s_cmp_gt_i32 s2, 0
-; GFX10-NEXT:    s_cselect_b32 s12, s2, 0
-; GFX10-NEXT:    s_pack_ll_b32_b16 s10, s10, s12
-; GFX10-NEXT:    s_lshr_b32 s12, s10, 16
-; GFX10-NEXT:    s_sub_i32 s9, s9, s10
-; GFX10-NEXT:    s_sub_i32 s8, s8, s12
-; GFX10-NEXT:    s_cmp_lt_i32 s4, s5
-; GFX10-NEXT:    s_cselect_b32 s4, s4, s5
-; GFX10-NEXT:    s_cmp_lt_i32 s2, 0
-; GFX10-NEXT:    s_sext_i32_i16 s5, s3
-; GFX10-NEXT:    s_cselect_b32 s2, s2, 0
-; GFX10-NEXT:    s_ashr_i32 s3, s3, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s4, s2
-; GFX10-NEXT:    s_lshr_b32 s4, s2, 16
-; GFX10-NEXT:    s_sub_i32 s2, s11, s2
-; GFX10-NEXT:    s_sub_i32 s4, s6, s4
-; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s2, s4
-; GFX10-NEXT:    s_sext_i32_i16 s4, s2
-; GFX10-NEXT:    s_ashr_i32 s2, s2, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s4, s5
-; GFX10-NEXT:    s_cselect_b32 s4, s4, s5
-; GFX10-NEXT:    s_cmp_gt_i32 s2, s3
-; GFX10-NEXT:    s_pack_ll_b32_b16 s5, s9, s8
-; GFX10-NEXT:    s_cselect_b32 s2, s2, s3
-; GFX10-NEXT:    s_sext_i32_i16 s3, s5
-; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s4, s2
-; GFX10-NEXT:    s_ashr_i32 s4, s5, 16
-; GFX10-NEXT:    s_sext_i32_i16 s5, s2
-; GFX10-NEXT:    s_ashr_i32 s2, s2, 16
-; GFX10-NEXT:    s_cmp_lt_i32 s5, s3
-; GFX10-NEXT:    s_cselect_b32 s3, s5, s3
-; GFX10-NEXT:    s_cmp_lt_i32 s2, s4
-; GFX10-NEXT:    s_cselect_b32 s2, s2, s4
-; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s3, s2
-; GFX10-NEXT:    s_lshr_b32 s3, s1, 16
-; GFX10-NEXT:    s_lshr_b32 s4, s2, 16
-; GFX10-NEXT:    s_add_i32 s1, s1, s2
-; GFX10-NEXT:    s_add_i32 s3, s3, s4
-; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s1, s3
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <4 x i16> @llvm.sadd.sat.v4i16(<4 x i16> %lhs, <4 x i16> %rhs)
   %cast = bitcast <4 x i16> %result to <2 x i32>
@@ -5612,57 +3798,19 @@ define <3 x float> @v_saddsat_v6i16(<6 x i16> %lhs, <6 x i16> %rhs) {
 ; GFX9-LABEL: v_saddsat_v6i16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_pk_min_i16 v8, v0, 0
-; GFX9-NEXT:    v_mov_b32_e32 v9, 0x80008000
-; GFX9-NEXT:    v_pk_sub_i16 v8, v9, v8
-; GFX9-NEXT:    v_pk_max_i16 v6, v0, 0
-; GFX9-NEXT:    v_mov_b32_e32 v7, 0x7fff7fff
-; GFX9-NEXT:    v_pk_sub_i16 v6, v7, v6
-; GFX9-NEXT:    v_pk_max_i16 v3, v8, v3
-; GFX9-NEXT:    v_pk_min_i16 v3, v3, v6
-; GFX9-NEXT:    v_pk_min_i16 v6, v1, 0
-; GFX9-NEXT:    v_pk_add_u16 v0, v0, v3
-; GFX9-NEXT:    v_pk_max_i16 v3, v1, 0
-; GFX9-NEXT:    v_pk_sub_i16 v6, v9, v6
-; GFX9-NEXT:    v_pk_sub_i16 v3, v7, v3
-; GFX9-NEXT:    v_pk_max_i16 v4, v6, v4
-; GFX9-NEXT:    v_pk_min_i16 v3, v4, v3
-; GFX9-NEXT:    v_pk_min_i16 v4, v2, 0
-; GFX9-NEXT:    v_pk_sub_i16 v4, v9, v4
-; GFX9-NEXT:    v_pk_add_u16 v1, v1, v3
-; GFX9-NEXT:    v_pk_max_i16 v3, v2, 0
-; GFX9-NEXT:    v_pk_sub_i16 v3, v7, v3
-; GFX9-NEXT:    v_pk_max_i16 v4, v4, v5
-; GFX9-NEXT:    v_pk_min_i16 v3, v4, v3
-; GFX9-NEXT:    v_pk_add_u16 v2, v2, v3
+; GFX9-NEXT:    v_pk_add_i16 v0, v0, v3 clamp
+; GFX9-NEXT:    v_pk_add_i16 v1, v1, v4 clamp
+; GFX9-NEXT:    v_pk_add_i16 v2, v2, v5 clamp
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_saddsat_v6i16:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_pk_min_i16 v7, v0, 0
-; GFX10-NEXT:    v_pk_min_i16 v8, v1, 0
-; GFX10-NEXT:    v_pk_min_i16 v9, v2, 0
-; GFX10-NEXT:    v_pk_max_i16 v6, v0, 0
-; GFX10-NEXT:    v_pk_max_i16 v10, v1, 0
-; GFX10-NEXT:    v_pk_sub_i16 v14, 0x80008000, v7
-; GFX10-NEXT:    v_pk_sub_i16 v15, 0x80008000, v8
-; GFX10-NEXT:    v_pk_max_i16 v11, v2, 0
-; GFX10-NEXT:    v_pk_sub_i16 v19, 0x80008000, v9
-; GFX10-NEXT:    v_pk_sub_i16 v6, 0x7fff7fff, v6
-; GFX10-NEXT:    v_pk_max_i16 v3, v14, v3
-; GFX10-NEXT:    v_pk_sub_i16 v7, 0x7fff7fff, v10
-; GFX10-NEXT:    v_pk_max_i16 v4, v15, v4
-; GFX10-NEXT:    v_pk_sub_i16 v8, 0x7fff7fff, v11
-; GFX10-NEXT:    v_pk_max_i16 v5, v19, v5
-; GFX10-NEXT:    v_pk_min_i16 v3, v3, v6
+; GFX10-NEXT:    v_pk_add_i16 v0, v0, v3 clamp
+; GFX10-NEXT:    v_pk_add_i16 v1, v1, v4 clamp
+; GFX10-NEXT:    v_pk_add_i16 v2, v2, v5 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_pk_min_i16 v4, v4, v7
-; GFX10-NEXT:    v_pk_min_i16 v5, v5, v8
-; GFX10-NEXT:    v_pk_add_u16 v0, v0, v3
-; GFX10-NEXT:    v_pk_add_u16 v1, v1, v4
-; GFX10-NEXT:    v_pk_add_u16 v2, v2, v5
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <6 x i16> @llvm.sadd.sat.v6i16(<6 x i16> %lhs, <6 x i16> %rhs)
   %cast = bitcast <6 x i16> %result to <3 x float>
@@ -5896,279 +4044,26 @@ define amdgpu_ps <3 x i32> @s_saddsat_v6i16(<6 x i16> inreg %lhs, <6 x i16> inre
 ;
 ; GFX9-LABEL: s_saddsat_v6i16:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_sext_i32_i16 s8, s0
-; GFX9-NEXT:    s_ashr_i32 s9, s0, 16
-; GFX9-NEXT:    s_sext_i32_i16 s10, 0
-; GFX9-NEXT:    s_cmp_gt_i32 s8, s10
-; GFX9-NEXT:    s_cselect_b32 s11, s8, s10
-; GFX9-NEXT:    s_cmp_gt_i32 s9, 0
-; GFX9-NEXT:    s_cselect_b32 s12, s9, 0
-; GFX9-NEXT:    s_pack_ll_b32_b16 s11, s11, s12
-; GFX9-NEXT:    s_mov_b32 s6, 0x7fff7fff
-; GFX9-NEXT:    s_lshr_b32 s13, s11, 16
-; GFX9-NEXT:    s_movk_i32 s12, 0x7fff
-; GFX9-NEXT:    s_sub_i32 s11, s6, s11
-; GFX9-NEXT:    s_sub_i32 s13, s12, s13
-; GFX9-NEXT:    s_cmp_lt_i32 s8, s10
-; GFX9-NEXT:    s_cselect_b32 s8, s8, s10
-; GFX9-NEXT:    s_cmp_lt_i32 s9, 0
-; GFX9-NEXT:    s_cselect_b32 s9, s9, 0
-; GFX9-NEXT:    s_pack_ll_b32_b16 s8, s8, s9
-; GFX9-NEXT:    s_pack_ll_b32_b16 s11, s11, s13
-; GFX9-NEXT:    s_mov_b32 s7, 0x80008000
-; GFX9-NEXT:    s_lshr_b32 s13, s8, 16
-; GFX9-NEXT:    s_mov_b32 s9, 0x8000
-; GFX9-NEXT:    s_sub_i32 s8, s7, s8
-; GFX9-NEXT:    s_sub_i32 s13, s9, s13
-; GFX9-NEXT:    s_pack_ll_b32_b16 s8, s8, s13
-; GFX9-NEXT:    s_sext_i32_i16 s13, s8
-; GFX9-NEXT:    s_sext_i32_i16 s14, s3
-; GFX9-NEXT:    s_ashr_i32 s8, s8, 16
-; GFX9-NEXT:    s_ashr_i32 s3, s3, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s13, s14
-; GFX9-NEXT:    s_cselect_b32 s13, s13, s14
-; GFX9-NEXT:    s_cmp_gt_i32 s8, s3
-; GFX9-NEXT:    s_cselect_b32 s3, s8, s3
-; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s13, s3
-; GFX9-NEXT:    s_sext_i32_i16 s8, s3
-; GFX9-NEXT:    s_sext_i32_i16 s13, s11
-; GFX9-NEXT:    s_ashr_i32 s3, s3, 16
-; GFX9-NEXT:    s_ashr_i32 s11, s11, 16
-; GFX9-NEXT:    s_cmp_lt_i32 s8, s13
-; GFX9-NEXT:    s_cselect_b32 s8, s8, s13
-; GFX9-NEXT:    s_cmp_lt_i32 s3, s11
-; GFX9-NEXT:    s_cselect_b32 s3, s3, s11
-; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s8, s3
-; GFX9-NEXT:    s_lshr_b32 s8, s0, 16
-; GFX9-NEXT:    s_lshr_b32 s11, s3, 16
-; GFX9-NEXT:    s_add_i32 s0, s0, s3
-; GFX9-NEXT:    s_add_i32 s8, s8, s11
-; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s8
-; GFX9-NEXT:    s_sext_i32_i16 s3, s1
-; GFX9-NEXT:    s_ashr_i32 s8, s1, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s3, s10
-; GFX9-NEXT:    s_cselect_b32 s11, s3, s10
-; GFX9-NEXT:    s_cmp_gt_i32 s8, 0
-; GFX9-NEXT:    s_cselect_b32 s13, s8, 0
-; GFX9-NEXT:    s_pack_ll_b32_b16 s11, s11, s13
-; GFX9-NEXT:    s_lshr_b32 s13, s11, 16
-; GFX9-NEXT:    s_sub_i32 s11, s6, s11
-; GFX9-NEXT:    s_sub_i32 s13, s12, s13
-; GFX9-NEXT:    s_cmp_lt_i32 s3, s10
-; GFX9-NEXT:    s_cselect_b32 s3, s3, s10
-; GFX9-NEXT:    s_cmp_lt_i32 s8, 0
-; GFX9-NEXT:    s_cselect_b32 s8, s8, 0
-; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s3, s8
-; GFX9-NEXT:    s_lshr_b32 s8, s3, 16
-; GFX9-NEXT:    s_sub_i32 s3, s7, s3
-; GFX9-NEXT:    s_sub_i32 s8, s9, s8
-; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s3, s8
-; GFX9-NEXT:    s_pack_ll_b32_b16 s11, s11, s13
-; GFX9-NEXT:    s_sext_i32_i16 s8, s3
-; GFX9-NEXT:    s_sext_i32_i16 s13, s4
-; GFX9-NEXT:    s_ashr_i32 s3, s3, 16
-; GFX9-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s8, s13
-; GFX9-NEXT:    s_cselect_b32 s8, s8, s13
-; GFX9-NEXT:    s_cmp_gt_i32 s3, s4
-; GFX9-NEXT:    s_cselect_b32 s3, s3, s4
-; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s8, s3
-; GFX9-NEXT:    s_sext_i32_i16 s4, s3
-; GFX9-NEXT:    s_sext_i32_i16 s8, s11
-; GFX9-NEXT:    s_ashr_i32 s3, s3, 16
-; GFX9-NEXT:    s_ashr_i32 s11, s11, 16
-; GFX9-NEXT:    s_cmp_lt_i32 s4, s8
-; GFX9-NEXT:    s_cselect_b32 s4, s4, s8
-; GFX9-NEXT:    s_cmp_lt_i32 s3, s11
-; GFX9-NEXT:    s_cselect_b32 s3, s3, s11
-; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s4, s3
-; GFX9-NEXT:    s_lshr_b32 s4, s1, 16
-; GFX9-NEXT:    s_lshr_b32 s8, s3, 16
-; GFX9-NEXT:    s_add_i32 s1, s1, s3
-; GFX9-NEXT:    s_add_i32 s4, s4, s8
-; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s4
-; GFX9-NEXT:    s_sext_i32_i16 s3, s2
-; GFX9-NEXT:    s_ashr_i32 s4, s2, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s3, s10
-; GFX9-NEXT:    s_cselect_b32 s8, s3, s10
-; GFX9-NEXT:    s_cmp_gt_i32 s4, 0
-; GFX9-NEXT:    s_cselect_b32 s11, s4, 0
-; GFX9-NEXT:    s_pack_ll_b32_b16 s8, s8, s11
-; GFX9-NEXT:    s_lshr_b32 s11, s8, 16
-; GFX9-NEXT:    s_sub_i32 s6, s6, s8
-; GFX9-NEXT:    s_sub_i32 s8, s12, s11
-; GFX9-NEXT:    s_cmp_lt_i32 s3, s10
-; GFX9-NEXT:    s_cselect_b32 s3, s3, s10
-; GFX9-NEXT:    s_cmp_lt_i32 s4, 0
-; GFX9-NEXT:    s_cselect_b32 s4, s4, 0
-; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s3, s4
-; GFX9-NEXT:    s_lshr_b32 s4, s3, 16
-; GFX9-NEXT:    s_sub_i32 s3, s7, s3
-; GFX9-NEXT:    s_sub_i32 s4, s9, s4
-; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s3, s4
-; GFX9-NEXT:    s_sext_i32_i16 s4, s3
-; GFX9-NEXT:    s_sext_i32_i16 s7, s5
-; GFX9-NEXT:    s_ashr_i32 s3, s3, 16
-; GFX9-NEXT:    s_ashr_i32 s5, s5, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s4, s7
-; GFX9-NEXT:    s_cselect_b32 s4, s4, s7
-; GFX9-NEXT:    s_cmp_gt_i32 s3, s5
-; GFX9-NEXT:    s_cselect_b32 s3, s3, s5
-; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s4, s3
-; GFX9-NEXT:    s_pack_ll_b32_b16 s6, s6, s8
-; GFX9-NEXT:    s_sext_i32_i16 s4, s3
-; GFX9-NEXT:    s_sext_i32_i16 s5, s6
-; GFX9-NEXT:    s_ashr_i32 s3, s3, 16
-; GFX9-NEXT:    s_ashr_i32 s6, s6, 16
-; GFX9-NEXT:    s_cmp_lt_i32 s4, s5
-; GFX9-NEXT:    s_cselect_b32 s4, s4, s5
-; GFX9-NEXT:    s_cmp_lt_i32 s3, s6
-; GFX9-NEXT:    s_cselect_b32 s3, s3, s6
-; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s4, s3
-; GFX9-NEXT:    s_lshr_b32 s4, s2, 16
-; GFX9-NEXT:    s_lshr_b32 s5, s3, 16
-; GFX9-NEXT:    s_add_i32 s2, s2, s3
-; GFX9-NEXT:    s_add_i32 s4, s4, s5
-; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s4
+; GFX9-NEXT:    v_mov_b32_e32 v0, s3
+; GFX9-NEXT:    v_mov_b32_e32 v1, s4
+; GFX9-NEXT:    v_mov_b32_e32 v2, s5
+; GFX9-NEXT:    v_pk_add_i16 v0, s0, v0 clamp
+; GFX9-NEXT:    v_pk_add_i16 v1, s1, v1 clamp
+; GFX9-NEXT:    v_pk_add_i16 v2, s2, v2 clamp
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v2
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_saddsat_v6i16:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_sext_i32_i16 s6, s0
-; GFX10-NEXT:    s_sext_i32_i16 s7, 0
-; GFX10-NEXT:    s_ashr_i32 s8, s0, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s6, s7
-; GFX10-NEXT:    s_mov_b32 s11, 0x7fff7fff
-; GFX10-NEXT:    s_cselect_b32 s9, s6, s7
-; GFX10-NEXT:    s_cmp_gt_i32 s8, 0
-; GFX10-NEXT:    s_mov_b32 s13, 0x80008000
-; GFX10-NEXT:    s_cselect_b32 s10, s8, 0
-; GFX10-NEXT:    s_sext_i32_i16 s15, s3
-; GFX10-NEXT:    s_pack_ll_b32_b16 s9, s9, s10
-; GFX10-NEXT:    s_movk_i32 s10, 0x7fff
-; GFX10-NEXT:    s_lshr_b32 s12, s9, 16
-; GFX10-NEXT:    s_sub_i32 s9, s11, s9
-; GFX10-NEXT:    s_sub_i32 s12, s10, s12
-; GFX10-NEXT:    s_cmp_lt_i32 s6, s7
+; GFX10-NEXT:    v_pk_add_i16 v0, s0, s3 clamp
+; GFX10-NEXT:    v_pk_add_i16 v1, s1, s4 clamp
+; GFX10-NEXT:    v_pk_add_i16 v2, s2, s5 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cselect_b32 s6, s6, s7
-; GFX10-NEXT:    s_cmp_lt_i32 s8, 0
-; GFX10-NEXT:    s_cselect_b32 s8, s8, 0
-; GFX10-NEXT:    s_ashr_i32 s3, s3, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s6, s8
-; GFX10-NEXT:    s_mov_b32 s8, 0x8000
-; GFX10-NEXT:    s_lshr_b32 s14, s6, 16
-; GFX10-NEXT:    s_sub_i32 s6, s13, s6
-; GFX10-NEXT:    s_sub_i32 s14, s8, s14
-; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s6, s14
-; GFX10-NEXT:    s_sext_i32_i16 s14, s6
-; GFX10-NEXT:    s_ashr_i32 s6, s6, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s14, s15
-; GFX10-NEXT:    s_cselect_b32 s14, s14, s15
-; GFX10-NEXT:    s_cmp_gt_i32 s6, s3
-; GFX10-NEXT:    s_sext_i32_i16 s15, s4
-; GFX10-NEXT:    s_cselect_b32 s3, s6, s3
-; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s9, s12
-; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s14, s3
-; GFX10-NEXT:    s_sext_i32_i16 s12, s6
-; GFX10-NEXT:    s_sext_i32_i16 s9, s3
-; GFX10-NEXT:    s_ashr_i32 s3, s3, 16
-; GFX10-NEXT:    s_ashr_i32 s6, s6, 16
-; GFX10-NEXT:    s_cmp_lt_i32 s9, s12
-; GFX10-NEXT:    s_cselect_b32 s9, s9, s12
-; GFX10-NEXT:    s_cmp_lt_i32 s3, s6
-; GFX10-NEXT:    s_cselect_b32 s3, s3, s6
-; GFX10-NEXT:    s_lshr_b32 s6, s0, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s9, s3
-; GFX10-NEXT:    s_lshr_b32 s9, s3, 16
-; GFX10-NEXT:    s_add_i32 s0, s0, s3
-; GFX10-NEXT:    s_sext_i32_i16 s3, s1
-; GFX10-NEXT:    s_add_i32 s6, s6, s9
-; GFX10-NEXT:    s_ashr_i32 s9, s1, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s3, s7
-; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s6
-; GFX10-NEXT:    s_cselect_b32 s12, s3, s7
-; GFX10-NEXT:    s_cmp_gt_i32 s9, 0
-; GFX10-NEXT:    s_cselect_b32 s14, s9, 0
-; GFX10-NEXT:    s_pack_ll_b32_b16 s12, s12, s14
-; GFX10-NEXT:    s_lshr_b32 s14, s12, 16
-; GFX10-NEXT:    s_sub_i32 s12, s11, s12
-; GFX10-NEXT:    s_sub_i32 s14, s10, s14
-; GFX10-NEXT:    s_cmp_lt_i32 s3, s7
-; GFX10-NEXT:    s_pack_ll_b32_b16 s12, s12, s14
-; GFX10-NEXT:    s_cselect_b32 s3, s3, s7
-; GFX10-NEXT:    s_cmp_lt_i32 s9, 0
-; GFX10-NEXT:    s_cselect_b32 s9, s9, 0
-; GFX10-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s3, s9
-; GFX10-NEXT:    s_lshr_b32 s9, s3, 16
-; GFX10-NEXT:    s_sub_i32 s3, s13, s3
-; GFX10-NEXT:    s_sub_i32 s9, s8, s9
-; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s3, s9
-; GFX10-NEXT:    s_sext_i32_i16 s9, s3
-; GFX10-NEXT:    s_ashr_i32 s3, s3, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s9, s15
-; GFX10-NEXT:    s_cselect_b32 s9, s9, s15
-; GFX10-NEXT:    s_cmp_gt_i32 s3, s4
-; GFX10-NEXT:    s_cselect_b32 s3, s3, s4
-; GFX10-NEXT:    s_sext_i32_i16 s4, s12
-; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s9, s3
-; GFX10-NEXT:    s_ashr_i32 s9, s12, 16
-; GFX10-NEXT:    s_sext_i32_i16 s12, s3
-; GFX10-NEXT:    s_ashr_i32 s3, s3, 16
-; GFX10-NEXT:    s_cmp_lt_i32 s12, s4
-; GFX10-NEXT:    s_cselect_b32 s4, s12, s4
-; GFX10-NEXT:    s_cmp_lt_i32 s3, s9
-; GFX10-NEXT:    s_sext_i32_i16 s12, s2
-; GFX10-NEXT:    s_cselect_b32 s3, s3, s9
-; GFX10-NEXT:    s_lshr_b32 s9, s1, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s4, s3
-; GFX10-NEXT:    s_ashr_i32 s4, s2, 16
-; GFX10-NEXT:    s_lshr_b32 s14, s3, 16
-; GFX10-NEXT:    s_add_i32 s1, s1, s3
-; GFX10-NEXT:    s_add_i32 s9, s9, s14
-; GFX10-NEXT:    s_cmp_gt_i32 s12, s7
-; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s1, s9
-; GFX10-NEXT:    s_cselect_b32 s3, s12, s7
-; GFX10-NEXT:    s_cmp_gt_i32 s4, 0
-; GFX10-NEXT:    s_cselect_b32 s14, s4, 0
-; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s3, s14
-; GFX10-NEXT:    s_lshr_b32 s14, s3, 16
-; GFX10-NEXT:    s_sub_i32 s3, s11, s3
-; GFX10-NEXT:    s_sub_i32 s10, s10, s14
-; GFX10-NEXT:    s_cmp_lt_i32 s12, s7
-; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s3, s10
-; GFX10-NEXT:    s_cselect_b32 s7, s12, s7
-; GFX10-NEXT:    s_cmp_lt_i32 s4, 0
-; GFX10-NEXT:    s_cselect_b32 s4, s4, 0
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s7, s4
-; GFX10-NEXT:    s_lshr_b32 s7, s4, 16
-; GFX10-NEXT:    s_sub_i32 s4, s13, s4
-; GFX10-NEXT:    s_sub_i32 s7, s8, s7
-; GFX10-NEXT:    s_sext_i32_i16 s8, s5
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s4, s7
-; GFX10-NEXT:    s_ashr_i32 s5, s5, 16
-; GFX10-NEXT:    s_sext_i32_i16 s7, s4
-; GFX10-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s7, s8
-; GFX10-NEXT:    s_cselect_b32 s7, s7, s8
-; GFX10-NEXT:    s_cmp_gt_i32 s4, s5
-; GFX10-NEXT:    s_cselect_b32 s4, s4, s5
-; GFX10-NEXT:    s_sext_i32_i16 s5, s3
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s7, s4
-; GFX10-NEXT:    s_ashr_i32 s3, s3, 16
-; GFX10-NEXT:    s_sext_i32_i16 s7, s4
-; GFX10-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX10-NEXT:    s_cmp_lt_i32 s7, s5
-; GFX10-NEXT:    s_cselect_b32 s5, s7, s5
-; GFX10-NEXT:    s_cmp_lt_i32 s4, s3
-; GFX10-NEXT:    s_cselect_b32 s3, s4, s3
-; GFX10-NEXT:    s_lshr_b32 s4, s2, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s5, s3
-; GFX10-NEXT:    s_lshr_b32 s5, s3, 16
-; GFX10-NEXT:    s_add_i32 s2, s2, s3
-; GFX10-NEXT:    s_add_i32 s4, s4, s5
-; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s2, s4
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <6 x i16> @llvm.sadd.sat.v6i16(<6 x i16> %lhs, <6 x i16> %rhs)
   %cast = bitcast <6 x i16> %result to <3 x i32>
@@ -6358,71 +4253,21 @@ define <4 x float> @v_saddsat_v8i16(<8 x i16> %lhs, <8 x i16> %rhs) {
 ; GFX9-LABEL: v_saddsat_v8i16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_pk_min_i16 v10, v0, 0
-; GFX9-NEXT:    v_mov_b32_e32 v11, 0x80008000
-; GFX9-NEXT:    v_pk_sub_i16 v10, v11, v10
-; GFX9-NEXT:    v_pk_max_i16 v8, v0, 0
-; GFX9-NEXT:    v_mov_b32_e32 v9, 0x7fff7fff
-; GFX9-NEXT:    v_pk_sub_i16 v8, v9, v8
-; GFX9-NEXT:    v_pk_max_i16 v4, v10, v4
-; GFX9-NEXT:    v_pk_min_i16 v4, v4, v8
-; GFX9-NEXT:    v_pk_min_i16 v8, v1, 0
-; GFX9-NEXT:    v_pk_add_u16 v0, v0, v4
-; GFX9-NEXT:    v_pk_max_i16 v4, v1, 0
-; GFX9-NEXT:    v_pk_sub_i16 v8, v11, v8
-; GFX9-NEXT:    v_pk_sub_i16 v4, v9, v4
-; GFX9-NEXT:    v_pk_max_i16 v5, v8, v5
-; GFX9-NEXT:    v_pk_min_i16 v4, v5, v4
-; GFX9-NEXT:    v_pk_min_i16 v5, v2, 0
-; GFX9-NEXT:    v_pk_sub_i16 v5, v11, v5
-; GFX9-NEXT:    v_pk_add_u16 v1, v1, v4
-; GFX9-NEXT:    v_pk_max_i16 v4, v2, 0
-; GFX9-NEXT:    v_pk_sub_i16 v4, v9, v4
-; GFX9-NEXT:    v_pk_max_i16 v5, v5, v6
-; GFX9-NEXT:    v_pk_min_i16 v4, v5, v4
-; GFX9-NEXT:    v_pk_min_i16 v5, v3, 0
-; GFX9-NEXT:    v_pk_sub_i16 v5, v11, v5
-; GFX9-NEXT:    v_pk_add_u16 v2, v2, v4
-; GFX9-NEXT:    v_pk_max_i16 v4, v3, 0
-; GFX9-NEXT:    v_pk_sub_i16 v4, v9, v4
-; GFX9-NEXT:    v_pk_max_i16 v5, v5, v7
-; GFX9-NEXT:    v_pk_min_i16 v4, v5, v4
-; GFX9-NEXT:    v_pk_add_u16 v3, v3, v4
+; GFX9-NEXT:    v_pk_add_i16 v0, v0, v4 clamp
+; GFX9-NEXT:    v_pk_add_i16 v1, v1, v5 clamp
+; GFX9-NEXT:    v_pk_add_i16 v2, v2, v6 clamp
+; GFX9-NEXT:    v_pk_add_i16 v3, v3, v7 clamp
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_saddsat_v8i16:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_pk_min_i16 v8, v0, 0
-; GFX10-NEXT:    v_pk_min_i16 v11, v1, 0
-; GFX10-NEXT:    v_pk_min_i16 v12, v3, 0
-; GFX10-NEXT:    v_pk_max_i16 v9, v0, 0
-; GFX10-NEXT:    v_pk_max_i16 v10, v1, 0
-; GFX10-NEXT:    v_pk_sub_i16 v15, 0x80008000, v8
-; GFX10-NEXT:    v_pk_min_i16 v8, v2, 0
-; GFX10-NEXT:    v_pk_sub_i16 v11, 0x80008000, v11
-; GFX10-NEXT:    v_pk_sub_i16 v12, 0x80008000, v12
-; GFX10-NEXT:    v_pk_max_i16 v13, v2, 0
-; GFX10-NEXT:    v_pk_max_i16 v14, v3, 0
-; GFX10-NEXT:    v_pk_sub_i16 v8, 0x80008000, v8
-; GFX10-NEXT:    v_pk_max_i16 v5, v11, v5
-; GFX10-NEXT:    v_pk_sub_i16 v10, 0x7fff7fff, v10
-; GFX10-NEXT:    v_pk_sub_i16 v9, 0x7fff7fff, v9
-; GFX10-NEXT:    v_pk_max_i16 v4, v15, v4
-; GFX10-NEXT:    v_pk_max_i16 v6, v8, v6
-; GFX10-NEXT:    v_pk_sub_i16 v11, 0x7fff7fff, v13
-; GFX10-NEXT:    v_pk_sub_i16 v8, 0x7fff7fff, v14
-; GFX10-NEXT:    v_pk_max_i16 v7, v12, v7
-; GFX10-NEXT:    v_pk_min_i16 v15, v4, v9
-; GFX10-NEXT:    v_pk_min_i16 v19, v5, v10
-; GFX10-NEXT:    v_pk_min_i16 v11, v6, v11
+; GFX10-NEXT:    v_pk_add_i16 v0, v0, v4 clamp
+; GFX10-NEXT:    v_pk_add_i16 v1, v1, v5 clamp
+; GFX10-NEXT:    v_pk_add_i16 v2, v2, v6 clamp
+; GFX10-NEXT:    v_pk_add_i16 v3, v3, v7 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_pk_min_i16 v6, v7, v8
-; GFX10-NEXT:    v_pk_add_u16 v0, v0, v15
-; GFX10-NEXT:    v_pk_add_u16 v1, v1, v19
-; GFX10-NEXT:    v_pk_add_u16 v2, v2, v11
-; GFX10-NEXT:    v_pk_add_u16 v3, v3, v6
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> %lhs, <8 x i16> %rhs)
   %cast = bitcast <8 x i16> %result to <4 x float>
@@ -6726,365 +4571,31 @@ define amdgpu_ps <4 x i32> @s_saddsat_v8i16(<8 x i16> inreg %lhs, <8 x i16> inre
 ;
 ; GFX9-LABEL: s_saddsat_v8i16:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_sext_i32_i16 s10, s0
-; GFX9-NEXT:    s_ashr_i32 s11, s0, 16
-; GFX9-NEXT:    s_sext_i32_i16 s12, 0
-; GFX9-NEXT:    s_cmp_gt_i32 s10, s12
-; GFX9-NEXT:    s_cselect_b32 s13, s10, s12
-; GFX9-NEXT:    s_cmp_gt_i32 s11, 0
-; GFX9-NEXT:    s_cselect_b32 s14, s11, 0
-; GFX9-NEXT:    s_pack_ll_b32_b16 s13, s13, s14
-; GFX9-NEXT:    s_mov_b32 s8, 0x7fff7fff
-; GFX9-NEXT:    s_lshr_b32 s15, s13, 16
-; GFX9-NEXT:    s_movk_i32 s14, 0x7fff
-; GFX9-NEXT:    s_sub_i32 s13, s8, s13
-; GFX9-NEXT:    s_sub_i32 s15, s14, s15
-; GFX9-NEXT:    s_cmp_lt_i32 s10, s12
-; GFX9-NEXT:    s_cselect_b32 s10, s10, s12
-; GFX9-NEXT:    s_cmp_lt_i32 s11, 0
-; GFX9-NEXT:    s_cselect_b32 s11, s11, 0
-; GFX9-NEXT:    s_pack_ll_b32_b16 s10, s10, s11
-; GFX9-NEXT:    s_pack_ll_b32_b16 s13, s13, s15
-; GFX9-NEXT:    s_mov_b32 s9, 0x80008000
-; GFX9-NEXT:    s_lshr_b32 s15, s10, 16
-; GFX9-NEXT:    s_mov_b32 s11, 0x8000
-; GFX9-NEXT:    s_sub_i32 s10, s9, s10
-; GFX9-NEXT:    s_sub_i32 s15, s11, s15
-; GFX9-NEXT:    s_pack_ll_b32_b16 s10, s10, s15
-; GFX9-NEXT:    s_sext_i32_i16 s15, s10
-; GFX9-NEXT:    s_sext_i32_i16 s16, s4
-; GFX9-NEXT:    s_ashr_i32 s10, s10, 16
-; GFX9-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s15, s16
-; GFX9-NEXT:    s_cselect_b32 s15, s15, s16
-; GFX9-NEXT:    s_cmp_gt_i32 s10, s4
-; GFX9-NEXT:    s_cselect_b32 s4, s10, s4
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s15, s4
-; GFX9-NEXT:    s_sext_i32_i16 s10, s4
-; GFX9-NEXT:    s_sext_i32_i16 s15, s13
-; GFX9-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX9-NEXT:    s_ashr_i32 s13, s13, 16
-; GFX9-NEXT:    s_cmp_lt_i32 s10, s15
-; GFX9-NEXT:    s_cselect_b32 s10, s10, s15
-; GFX9-NEXT:    s_cmp_lt_i32 s4, s13
-; GFX9-NEXT:    s_cselect_b32 s4, s4, s13
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s10, s4
-; GFX9-NEXT:    s_lshr_b32 s10, s0, 16
-; GFX9-NEXT:    s_lshr_b32 s13, s4, 16
-; GFX9-NEXT:    s_add_i32 s0, s0, s4
-; GFX9-NEXT:    s_add_i32 s10, s10, s13
-; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s10
-; GFX9-NEXT:    s_sext_i32_i16 s4, s1
-; GFX9-NEXT:    s_ashr_i32 s10, s1, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s4, s12
-; GFX9-NEXT:    s_cselect_b32 s13, s4, s12
-; GFX9-NEXT:    s_cmp_gt_i32 s10, 0
-; GFX9-NEXT:    s_cselect_b32 s15, s10, 0
-; GFX9-NEXT:    s_pack_ll_b32_b16 s13, s13, s15
-; GFX9-NEXT:    s_lshr_b32 s15, s13, 16
-; GFX9-NEXT:    s_sub_i32 s13, s8, s13
-; GFX9-NEXT:    s_sub_i32 s15, s14, s15
-; GFX9-NEXT:    s_cmp_lt_i32 s4, s12
-; GFX9-NEXT:    s_cselect_b32 s4, s4, s12
-; GFX9-NEXT:    s_cmp_lt_i32 s10, 0
-; GFX9-NEXT:    s_cselect_b32 s10, s10, 0
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s10
-; GFX9-NEXT:    s_lshr_b32 s10, s4, 16
-; GFX9-NEXT:    s_sub_i32 s4, s9, s4
-; GFX9-NEXT:    s_sub_i32 s10, s11, s10
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s10
-; GFX9-NEXT:    s_pack_ll_b32_b16 s13, s13, s15
-; GFX9-NEXT:    s_sext_i32_i16 s10, s4
-; GFX9-NEXT:    s_sext_i32_i16 s15, s5
-; GFX9-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX9-NEXT:    s_ashr_i32 s5, s5, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s10, s15
-; GFX9-NEXT:    s_cselect_b32 s10, s10, s15
-; GFX9-NEXT:    s_cmp_gt_i32 s4, s5
-; GFX9-NEXT:    s_cselect_b32 s4, s4, s5
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s10, s4
-; GFX9-NEXT:    s_sext_i32_i16 s5, s4
-; GFX9-NEXT:    s_sext_i32_i16 s10, s13
-; GFX9-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX9-NEXT:    s_ashr_i32 s13, s13, 16
-; GFX9-NEXT:    s_cmp_lt_i32 s5, s10
-; GFX9-NEXT:    s_cselect_b32 s5, s5, s10
-; GFX9-NEXT:    s_cmp_lt_i32 s4, s13
-; GFX9-NEXT:    s_cselect_b32 s4, s4, s13
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s5, s4
-; GFX9-NEXT:    s_lshr_b32 s5, s1, 16
-; GFX9-NEXT:    s_lshr_b32 s10, s4, 16
-; GFX9-NEXT:    s_add_i32 s1, s1, s4
-; GFX9-NEXT:    s_add_i32 s5, s5, s10
-; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s5
-; GFX9-NEXT:    s_sext_i32_i16 s4, s2
-; GFX9-NEXT:    s_ashr_i32 s5, s2, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s4, s12
-; GFX9-NEXT:    s_cselect_b32 s10, s4, s12
-; GFX9-NEXT:    s_cmp_gt_i32 s5, 0
-; GFX9-NEXT:    s_cselect_b32 s13, s5, 0
-; GFX9-NEXT:    s_pack_ll_b32_b16 s10, s10, s13
-; GFX9-NEXT:    s_lshr_b32 s13, s10, 16
-; GFX9-NEXT:    s_sub_i32 s10, s8, s10
-; GFX9-NEXT:    s_sub_i32 s13, s14, s13
-; GFX9-NEXT:    s_cmp_lt_i32 s4, s12
-; GFX9-NEXT:    s_cselect_b32 s4, s4, s12
-; GFX9-NEXT:    s_cmp_lt_i32 s5, 0
-; GFX9-NEXT:    s_cselect_b32 s5, s5, 0
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
-; GFX9-NEXT:    s_lshr_b32 s5, s4, 16
-; GFX9-NEXT:    s_sub_i32 s4, s9, s4
-; GFX9-NEXT:    s_sub_i32 s5, s11, s5
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
-; GFX9-NEXT:    s_pack_ll_b32_b16 s10, s10, s13
-; GFX9-NEXT:    s_sext_i32_i16 s5, s4
-; GFX9-NEXT:    s_sext_i32_i16 s13, s6
-; GFX9-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX9-NEXT:    s_ashr_i32 s6, s6, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s5, s13
-; GFX9-NEXT:    s_cselect_b32 s5, s5, s13
-; GFX9-NEXT:    s_cmp_gt_i32 s4, s6
-; GFX9-NEXT:    s_cselect_b32 s4, s4, s6
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s5, s4
-; GFX9-NEXT:    s_sext_i32_i16 s5, s4
-; GFX9-NEXT:    s_sext_i32_i16 s6, s10
-; GFX9-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX9-NEXT:    s_ashr_i32 s10, s10, 16
-; GFX9-NEXT:    s_cmp_lt_i32 s5, s6
-; GFX9-NEXT:    s_cselect_b32 s5, s5, s6
-; GFX9-NEXT:    s_cmp_lt_i32 s4, s10
-; GFX9-NEXT:    s_cselect_b32 s4, s4, s10
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s5, s4
-; GFX9-NEXT:    s_lshr_b32 s5, s2, 16
-; GFX9-NEXT:    s_lshr_b32 s6, s4, 16
-; GFX9-NEXT:    s_add_i32 s2, s2, s4
-; GFX9-NEXT:    s_add_i32 s5, s5, s6
-; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s5
-; GFX9-NEXT:    s_sext_i32_i16 s4, s3
-; GFX9-NEXT:    s_ashr_i32 s5, s3, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s4, s12
-; GFX9-NEXT:    s_cselect_b32 s6, s4, s12
-; GFX9-NEXT:    s_cmp_gt_i32 s5, 0
-; GFX9-NEXT:    s_cselect_b32 s10, s5, 0
-; GFX9-NEXT:    s_pack_ll_b32_b16 s6, s6, s10
-; GFX9-NEXT:    s_lshr_b32 s10, s6, 16
-; GFX9-NEXT:    s_sub_i32 s6, s8, s6
-; GFX9-NEXT:    s_sub_i32 s8, s14, s10
-; GFX9-NEXT:    s_cmp_lt_i32 s4, s12
-; GFX9-NEXT:    s_cselect_b32 s4, s4, s12
-; GFX9-NEXT:    s_cmp_lt_i32 s5, 0
-; GFX9-NEXT:    s_cselect_b32 s5, s5, 0
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
-; GFX9-NEXT:    s_lshr_b32 s5, s4, 16
-; GFX9-NEXT:    s_sub_i32 s4, s9, s4
-; GFX9-NEXT:    s_sub_i32 s5, s11, s5
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
-; GFX9-NEXT:    s_pack_ll_b32_b16 s6, s6, s8
-; GFX9-NEXT:    s_sext_i32_i16 s5, s4
-; GFX9-NEXT:    s_sext_i32_i16 s8, s7
-; GFX9-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX9-NEXT:    s_ashr_i32 s7, s7, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s5, s8
-; GFX9-NEXT:    s_cselect_b32 s5, s5, s8
-; GFX9-NEXT:    s_cmp_gt_i32 s4, s7
-; GFX9-NEXT:    s_cselect_b32 s4, s4, s7
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s5, s4
-; GFX9-NEXT:    s_sext_i32_i16 s5, s4
-; GFX9-NEXT:    s_sext_i32_i16 s7, s6
-; GFX9-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX9-NEXT:    s_ashr_i32 s6, s6, 16
-; GFX9-NEXT:    s_cmp_lt_i32 s5, s7
-; GFX9-NEXT:    s_cselect_b32 s5, s5, s7
-; GFX9-NEXT:    s_cmp_lt_i32 s4, s6
-; GFX9-NEXT:    s_cselect_b32 s4, s4, s6
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s5, s4
-; GFX9-NEXT:    s_lshr_b32 s5, s3, 16
-; GFX9-NEXT:    s_lshr_b32 s6, s4, 16
-; GFX9-NEXT:    s_add_i32 s3, s3, s4
-; GFX9-NEXT:    s_add_i32 s5, s5, s6
-; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s3, s5
+; GFX9-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-NEXT:    v_mov_b32_e32 v2, s6
+; GFX9-NEXT:    v_mov_b32_e32 v3, s7
+; GFX9-NEXT:    v_pk_add_i16 v0, s0, v0 clamp
+; GFX9-NEXT:    v_pk_add_i16 v1, s1, v1 clamp
+; GFX9-NEXT:    v_pk_add_i16 v2, s2, v2 clamp
+; GFX9-NEXT:    v_pk_add_i16 v3, s3, v3 clamp
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX9-NEXT:    v_readfirstlane_b32 s3, v3
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_saddsat_v8i16:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_sext_i32_i16 s8, s0
-; GFX10-NEXT:    s_sext_i32_i16 s9, 0
-; GFX10-NEXT:    s_ashr_i32 s10, s0, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s8, s9
-; GFX10-NEXT:    s_mov_b32 s13, 0x7fff7fff
-; GFX10-NEXT:    s_cselect_b32 s11, s8, s9
-; GFX10-NEXT:    s_cmp_gt_i32 s10, 0
-; GFX10-NEXT:    s_mov_b32 s15, 0x80008000
-; GFX10-NEXT:    s_cselect_b32 s12, s10, 0
-; GFX10-NEXT:    s_sext_i32_i16 s17, s4
-; GFX10-NEXT:    s_pack_ll_b32_b16 s11, s11, s12
-; GFX10-NEXT:    s_movk_i32 s12, 0x7fff
-; GFX10-NEXT:    s_lshr_b32 s14, s11, 16
-; GFX10-NEXT:    s_sub_i32 s11, s13, s11
-; GFX10-NEXT:    s_sub_i32 s14, s12, s14
-; GFX10-NEXT:    s_cmp_lt_i32 s8, s9
+; GFX10-NEXT:    v_pk_add_i16 v0, s0, s4 clamp
+; GFX10-NEXT:    v_pk_add_i16 v1, s1, s5 clamp
+; GFX10-NEXT:    v_pk_add_i16 v2, s2, s6 clamp
+; GFX10-NEXT:    v_pk_add_i16 v3, s3, s7 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cselect_b32 s8, s8, s9
-; GFX10-NEXT:    s_cmp_lt_i32 s10, 0
-; GFX10-NEXT:    s_cselect_b32 s10, s10, 0
-; GFX10-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s8, s8, s10
-; GFX10-NEXT:    s_mov_b32 s10, 0x8000
-; GFX10-NEXT:    s_lshr_b32 s16, s8, 16
-; GFX10-NEXT:    s_sub_i32 s8, s15, s8
-; GFX10-NEXT:    s_sub_i32 s16, s10, s16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s8, s8, s16
-; GFX10-NEXT:    s_sext_i32_i16 s16, s8
-; GFX10-NEXT:    s_ashr_i32 s8, s8, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s16, s17
-; GFX10-NEXT:    s_cselect_b32 s16, s16, s17
-; GFX10-NEXT:    s_cmp_gt_i32 s8, s4
-; GFX10-NEXT:    s_sext_i32_i16 s17, s5
-; GFX10-NEXT:    s_cselect_b32 s4, s8, s4
-; GFX10-NEXT:    s_pack_ll_b32_b16 s8, s11, s14
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s16, s4
-; GFX10-NEXT:    s_sext_i32_i16 s14, s8
-; GFX10-NEXT:    s_sext_i32_i16 s11, s4
-; GFX10-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX10-NEXT:    s_ashr_i32 s8, s8, 16
-; GFX10-NEXT:    s_cmp_lt_i32 s11, s14
-; GFX10-NEXT:    s_cselect_b32 s11, s11, s14
-; GFX10-NEXT:    s_cmp_lt_i32 s4, s8
-; GFX10-NEXT:    s_cselect_b32 s4, s4, s8
-; GFX10-NEXT:    s_lshr_b32 s8, s0, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s11, s4
-; GFX10-NEXT:    s_lshr_b32 s11, s4, 16
-; GFX10-NEXT:    s_add_i32 s0, s0, s4
-; GFX10-NEXT:    s_sext_i32_i16 s4, s1
-; GFX10-NEXT:    s_add_i32 s8, s8, s11
-; GFX10-NEXT:    s_ashr_i32 s11, s1, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s4, s9
-; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s8
-; GFX10-NEXT:    s_cselect_b32 s14, s4, s9
-; GFX10-NEXT:    s_cmp_gt_i32 s11, 0
-; GFX10-NEXT:    s_cselect_b32 s16, s11, 0
-; GFX10-NEXT:    s_pack_ll_b32_b16 s14, s14, s16
-; GFX10-NEXT:    s_lshr_b32 s16, s14, 16
-; GFX10-NEXT:    s_sub_i32 s14, s13, s14
-; GFX10-NEXT:    s_sub_i32 s16, s12, s16
-; GFX10-NEXT:    s_cmp_lt_i32 s4, s9
-; GFX10-NEXT:    s_pack_ll_b32_b16 s14, s14, s16
-; GFX10-NEXT:    s_cselect_b32 s4, s4, s9
-; GFX10-NEXT:    s_cmp_lt_i32 s11, 0
-; GFX10-NEXT:    s_cselect_b32 s11, s11, 0
-; GFX10-NEXT:    s_ashr_i32 s5, s5, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s4, s11
-; GFX10-NEXT:    s_lshr_b32 s11, s4, 16
-; GFX10-NEXT:    s_sub_i32 s4, s15, s4
-; GFX10-NEXT:    s_sub_i32 s11, s10, s11
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s4, s11
-; GFX10-NEXT:    s_sext_i32_i16 s11, s4
-; GFX10-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s11, s17
-; GFX10-NEXT:    s_cselect_b32 s11, s11, s17
-; GFX10-NEXT:    s_cmp_gt_i32 s4, s5
-; GFX10-NEXT:    s_sext_i32_i16 s17, s6
-; GFX10-NEXT:    s_cselect_b32 s4, s4, s5
-; GFX10-NEXT:    s_sext_i32_i16 s5, s14
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s11, s4
-; GFX10-NEXT:    s_ashr_i32 s11, s14, 16
-; GFX10-NEXT:    s_sext_i32_i16 s14, s4
-; GFX10-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX10-NEXT:    s_cmp_lt_i32 s14, s5
-; GFX10-NEXT:    s_cselect_b32 s5, s14, s5
-; GFX10-NEXT:    s_cmp_lt_i32 s4, s11
-; GFX10-NEXT:    s_sext_i32_i16 s14, s2
-; GFX10-NEXT:    s_cselect_b32 s4, s4, s11
-; GFX10-NEXT:    s_lshr_b32 s11, s1, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s5, s4
-; GFX10-NEXT:    s_ashr_i32 s5, s2, 16
-; GFX10-NEXT:    s_lshr_b32 s16, s4, 16
-; GFX10-NEXT:    s_add_i32 s1, s1, s4
-; GFX10-NEXT:    s_add_i32 s11, s11, s16
-; GFX10-NEXT:    s_cmp_gt_i32 s14, s9
-; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s1, s11
-; GFX10-NEXT:    s_cselect_b32 s4, s14, s9
-; GFX10-NEXT:    s_cmp_gt_i32 s5, 0
-; GFX10-NEXT:    s_cselect_b32 s16, s5, 0
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s4, s16
-; GFX10-NEXT:    s_lshr_b32 s16, s4, 16
-; GFX10-NEXT:    s_sub_i32 s4, s13, s4
-; GFX10-NEXT:    s_sub_i32 s16, s12, s16
-; GFX10-NEXT:    s_cmp_lt_i32 s14, s9
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s4, s16
-; GFX10-NEXT:    s_cselect_b32 s14, s14, s9
-; GFX10-NEXT:    s_cmp_lt_i32 s5, 0
-; GFX10-NEXT:    s_cselect_b32 s5, s5, 0
-; GFX10-NEXT:    s_ashr_i32 s6, s6, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s5, s14, s5
-; GFX10-NEXT:    s_lshr_b32 s14, s5, 16
-; GFX10-NEXT:    s_sub_i32 s5, s15, s5
-; GFX10-NEXT:    s_sub_i32 s14, s10, s14
-; GFX10-NEXT:    s_pack_ll_b32_b16 s5, s5, s14
-; GFX10-NEXT:    s_sext_i32_i16 s14, s5
-; GFX10-NEXT:    s_ashr_i32 s5, s5, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s14, s17
-; GFX10-NEXT:    s_cselect_b32 s14, s14, s17
-; GFX10-NEXT:    s_cmp_gt_i32 s5, s6
-; GFX10-NEXT:    s_cselect_b32 s5, s5, s6
-; GFX10-NEXT:    s_sext_i32_i16 s6, s4
-; GFX10-NEXT:    s_pack_ll_b32_b16 s5, s14, s5
-; GFX10-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX10-NEXT:    s_sext_i32_i16 s14, s5
-; GFX10-NEXT:    s_ashr_i32 s5, s5, 16
-; GFX10-NEXT:    s_cmp_lt_i32 s14, s6
-; GFX10-NEXT:    s_cselect_b32 s6, s14, s6
-; GFX10-NEXT:    s_cmp_lt_i32 s5, s4
-; GFX10-NEXT:    s_sext_i32_i16 s14, s3
-; GFX10-NEXT:    s_cselect_b32 s4, s5, s4
-; GFX10-NEXT:    s_lshr_b32 s5, s2, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s6, s4
-; GFX10-NEXT:    s_ashr_i32 s6, s3, 16
-; GFX10-NEXT:    s_lshr_b32 s16, s4, 16
-; GFX10-NEXT:    s_add_i32 s2, s2, s4
-; GFX10-NEXT:    s_add_i32 s5, s5, s16
-; GFX10-NEXT:    s_cmp_gt_i32 s14, s9
-; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s2, s5
-; GFX10-NEXT:    s_cselect_b32 s4, s14, s9
-; GFX10-NEXT:    s_cmp_gt_i32 s6, 0
-; GFX10-NEXT:    s_cselect_b32 s16, s6, 0
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s4, s16
-; GFX10-NEXT:    s_lshr_b32 s16, s4, 16
-; GFX10-NEXT:    s_sub_i32 s4, s13, s4
-; GFX10-NEXT:    s_sub_i32 s12, s12, s16
-; GFX10-NEXT:    s_cmp_lt_i32 s14, s9
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s4, s12
-; GFX10-NEXT:    s_cselect_b32 s9, s14, s9
-; GFX10-NEXT:    s_cmp_lt_i32 s6, 0
-; GFX10-NEXT:    s_cselect_b32 s6, s6, 0
-; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s9, s6
-; GFX10-NEXT:    s_lshr_b32 s9, s6, 16
-; GFX10-NEXT:    s_sub_i32 s6, s15, s6
-; GFX10-NEXT:    s_sub_i32 s9, s10, s9
-; GFX10-NEXT:    s_sext_i32_i16 s10, s7
-; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s6, s9
-; GFX10-NEXT:    s_ashr_i32 s7, s7, 16
-; GFX10-NEXT:    s_sext_i32_i16 s9, s6
-; GFX10-NEXT:    s_ashr_i32 s6, s6, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s9, s10
-; GFX10-NEXT:    s_cselect_b32 s9, s9, s10
-; GFX10-NEXT:    s_cmp_gt_i32 s6, s7
-; GFX10-NEXT:    s_cselect_b32 s6, s6, s7
-; GFX10-NEXT:    s_sext_i32_i16 s7, s4
-; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s9, s6
-; GFX10-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX10-NEXT:    s_sext_i32_i16 s9, s6
-; GFX10-NEXT:    s_ashr_i32 s6, s6, 16
-; GFX10-NEXT:    s_cmp_lt_i32 s9, s7
-; GFX10-NEXT:    s_cselect_b32 s7, s9, s7
-; GFX10-NEXT:    s_cmp_lt_i32 s6, s4
-; GFX10-NEXT:    s_cselect_b32 s4, s6, s4
-; GFX10-NEXT:    s_lshr_b32 s6, s3, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s7, s4
-; GFX10-NEXT:    s_lshr_b32 s7, s4, 16
-; GFX10-NEXT:    s_add_i32 s3, s3, s4
-; GFX10-NEXT:    s_add_i32 s6, s6, s7
-; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s3, s6
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX10-NEXT:    v_readfirstlane_b32 s3, v3
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> %lhs, <8 x i16> %rhs)
   %cast = bitcast <8 x i16> %result to <4 x i32>
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll
index ac2a75383cba3..f9e4ccd03955e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll
@@ -39,14 +39,8 @@ define i7 @v_ssubsat_i7(i7 %lhs, i7 %rhs) {
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 9, v0
-; GFX9-NEXT:    v_max_i16_e32 v2, -1, v0
-; GFX9-NEXT:    v_min_i16_e32 v3, -1, v0
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 9, v1
-; GFX9-NEXT:    v_subrev_u16_e32 v2, 0x7fff, v2
-; GFX9-NEXT:    v_subrev_u16_e32 v3, 0x8000, v3
-; GFX9-NEXT:    v_max_i16_e32 v1, v2, v1
-; GFX9-NEXT:    v_min_i16_e32 v1, v1, v3
-; GFX9-NEXT:    v_sub_u16_e32 v0, v0, v1
+; GFX9-NEXT:    v_sub_i16 v0, v0, v1 clamp
 ; GFX9-NEXT:    v_ashrrev_i16_e32 v0, 9, v0
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -57,13 +51,7 @@ define i7 @v_ssubsat_i7(i7 %lhs, i7 %rhs) {
 ; GFX10-NEXT:    v_lshlrev_b16_e64 v0, 9, v0
 ; GFX10-NEXT:    v_lshlrev_b16_e64 v1, 9, v1
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_max_i16_e64 v2, v0, -1
-; GFX10-NEXT:    v_min_i16_e64 v3, v0, -1
-; GFX10-NEXT:    v_sub_nc_u16_e64 v2, v2, 0x7fff
-; GFX10-NEXT:    v_sub_nc_u16_e64 v3, v3, 0x8000
-; GFX10-NEXT:    v_max_i16_e64 v1, v2, v1
-; GFX10-NEXT:    v_min_i16_e64 v1, v1, v3
-; GFX10-NEXT:    v_sub_nc_u16_e64 v0, v0, v1
+; GFX10-NEXT:    v_sub_nc_i16 v0, v0, v1 clamp
 ; GFX10-NEXT:    v_ashrrev_i16_e64 v0, 9, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call i7 @llvm.ssub.sat.i7(i7 %lhs, i7 %rhs)
@@ -118,54 +106,23 @@ define amdgpu_ps i7 @s_ssubsat_i7(i7 inreg %lhs, i7 inreg %rhs) {
 ; GFX9-LABEL: s_ssubsat_i7:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_bfe_u32 s2, 9, 0x100000
-; GFX9-NEXT:    s_lshl_b32 s0, s0, s2
 ; GFX9-NEXT:    s_lshl_b32 s1, s1, s2
-; GFX9-NEXT:    s_sext_i32_i16 s3, s0
-; GFX9-NEXT:    s_sext_i32_i16 s4, -1
-; GFX9-NEXT:    s_cmp_gt_i32 s3, s4
-; GFX9-NEXT:    s_cselect_b32 s5, s3, s4
-; GFX9-NEXT:    s_sub_i32 s5, s5, 0x7fff
-; GFX9-NEXT:    s_cmp_lt_i32 s3, s4
-; GFX9-NEXT:    s_cselect_b32 s3, s3, s4
-; GFX9-NEXT:    s_sub_i32 s3, s3, 0xffff8000
-; GFX9-NEXT:    s_sext_i32_i16 s4, s5
-; GFX9-NEXT:    s_sext_i32_i16 s1, s1
-; GFX9-NEXT:    s_cmp_gt_i32 s4, s1
-; GFX9-NEXT:    s_cselect_b32 s1, s4, s1
-; GFX9-NEXT:    s_sext_i32_i16 s1, s1
-; GFX9-NEXT:    s_sext_i32_i16 s3, s3
-; GFX9-NEXT:    s_cmp_lt_i32 s1, s3
-; GFX9-NEXT:    s_cselect_b32 s1, s1, s3
-; GFX9-NEXT:    s_sub_i32 s0, s0, s1
-; GFX9-NEXT:    s_sext_i32_i16 s0, s0
-; GFX9-NEXT:    s_ashr_i32 s0, s0, s2
+; GFX9-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX9-NEXT:    v_mov_b32_e32 v0, s1
+; GFX9-NEXT:    v_sub_i16 v0, s0, v0 clamp
+; GFX9-NEXT:    v_ashrrev_i16_e32 v0, 9, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_ssubsat_i7:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_bfe_u32 s2, 9, 0x100000
-; GFX10-NEXT:    s_sext_i32_i16 s4, -1
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_lshl_b32 s0, s0, s2
 ; GFX10-NEXT:    s_lshl_b32 s1, s1, s2
-; GFX10-NEXT:    s_sext_i32_i16 s3, s0
-; GFX10-NEXT:    s_sext_i32_i16 s1, s1
-; GFX10-NEXT:    s_cmp_gt_i32 s3, s4
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cselect_b32 s5, s3, s4
-; GFX10-NEXT:    s_sub_i32 s5, s5, 0x7fff
-; GFX10-NEXT:    s_cmp_lt_i32 s3, s4
-; GFX10-NEXT:    s_cselect_b32 s3, s3, s4
-; GFX10-NEXT:    s_sext_i32_i16 s4, s5
-; GFX10-NEXT:    s_sub_i32 s3, s3, 0xffff8000
-; GFX10-NEXT:    s_cmp_gt_i32 s4, s1
-; GFX10-NEXT:    s_sext_i32_i16 s3, s3
-; GFX10-NEXT:    s_cselect_b32 s1, s4, s1
-; GFX10-NEXT:    s_sext_i32_i16 s1, s1
-; GFX10-NEXT:    s_cmp_lt_i32 s1, s3
-; GFX10-NEXT:    s_cselect_b32 s1, s1, s3
-; GFX10-NEXT:    s_sub_i32 s0, s0, s1
-; GFX10-NEXT:    s_sext_i32_i16 s0, s0
-; GFX10-NEXT:    s_ashr_i32 s0, s0, s2
+; GFX10-NEXT:    v_sub_nc_i16 v0, s0, s1 clamp
+; GFX10-NEXT:    v_ashrrev_i16_e64 v0, 9, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call i7 @llvm.ssub.sat.i7(i7 %lhs, i7 %rhs)
   ret i7 %result
@@ -206,14 +163,8 @@ define i8 @v_ssubsat_i8(i8 %lhs, i8 %rhs) {
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; GFX9-NEXT:    v_max_i16_e32 v2, -1, v0
-; GFX9-NEXT:    v_min_i16_e32 v3, -1, v0
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
-; GFX9-NEXT:    v_subrev_u16_e32 v2, 0x7fff, v2
-; GFX9-NEXT:    v_subrev_u16_e32 v3, 0x8000, v3
-; GFX9-NEXT:    v_max_i16_e32 v1, v2, v1
-; GFX9-NEXT:    v_min_i16_e32 v1, v1, v3
-; GFX9-NEXT:    v_sub_u16_e32 v0, v0, v1
+; GFX9-NEXT:    v_sub_i16 v0, v0, v1 clamp
 ; GFX9-NEXT:    v_ashrrev_i16_e32 v0, 8, v0
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -224,13 +175,7 @@ define i8 @v_ssubsat_i8(i8 %lhs, i8 %rhs) {
 ; GFX10-NEXT:    v_lshlrev_b16_e64 v0, 8, v0
 ; GFX10-NEXT:    v_lshlrev_b16_e64 v1, 8, v1
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_max_i16_e64 v2, v0, -1
-; GFX10-NEXT:    v_min_i16_e64 v3, v0, -1
-; GFX10-NEXT:    v_sub_nc_u16_e64 v2, v2, 0x7fff
-; GFX10-NEXT:    v_sub_nc_u16_e64 v3, v3, 0x8000
-; GFX10-NEXT:    v_max_i16_e64 v1, v2, v1
-; GFX10-NEXT:    v_min_i16_e64 v1, v1, v3
-; GFX10-NEXT:    v_sub_nc_u16_e64 v0, v0, v1
+; GFX10-NEXT:    v_sub_nc_i16 v0, v0, v1 clamp
 ; GFX10-NEXT:    v_ashrrev_i16_e64 v0, 8, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call i8 @llvm.ssub.sat.i8(i8 %lhs, i8 %rhs)
@@ -285,54 +230,23 @@ define amdgpu_ps i8 @s_ssubsat_i8(i8 inreg %lhs, i8 inreg %rhs) {
 ; GFX9-LABEL: s_ssubsat_i8:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_bfe_u32 s2, 8, 0x100000
-; GFX9-NEXT:    s_lshl_b32 s0, s0, s2
 ; GFX9-NEXT:    s_lshl_b32 s1, s1, s2
-; GFX9-NEXT:    s_sext_i32_i16 s3, s0
-; GFX9-NEXT:    s_sext_i32_i16 s4, -1
-; GFX9-NEXT:    s_cmp_gt_i32 s3, s4
-; GFX9-NEXT:    s_cselect_b32 s5, s3, s4
-; GFX9-NEXT:    s_sub_i32 s5, s5, 0x7fff
-; GFX9-NEXT:    s_cmp_lt_i32 s3, s4
-; GFX9-NEXT:    s_cselect_b32 s3, s3, s4
-; GFX9-NEXT:    s_sub_i32 s3, s3, 0xffff8000
-; GFX9-NEXT:    s_sext_i32_i16 s4, s5
-; GFX9-NEXT:    s_sext_i32_i16 s1, s1
-; GFX9-NEXT:    s_cmp_gt_i32 s4, s1
-; GFX9-NEXT:    s_cselect_b32 s1, s4, s1
-; GFX9-NEXT:    s_sext_i32_i16 s1, s1
-; GFX9-NEXT:    s_sext_i32_i16 s3, s3
-; GFX9-NEXT:    s_cmp_lt_i32 s1, s3
-; GFX9-NEXT:    s_cselect_b32 s1, s1, s3
-; GFX9-NEXT:    s_sub_i32 s0, s0, s1
-; GFX9-NEXT:    s_sext_i32_i16 s0, s0
-; GFX9-NEXT:    s_ashr_i32 s0, s0, s2
+; GFX9-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX9-NEXT:    v_mov_b32_e32 v0, s1
+; GFX9-NEXT:    v_sub_i16 v0, s0, v0 clamp
+; GFX9-NEXT:    v_ashrrev_i16_e32 v0, 8, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_ssubsat_i8:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_bfe_u32 s2, 8, 0x100000
-; GFX10-NEXT:    s_sext_i32_i16 s4, -1
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_lshl_b32 s0, s0, s2
 ; GFX10-NEXT:    s_lshl_b32 s1, s1, s2
-; GFX10-NEXT:    s_sext_i32_i16 s3, s0
-; GFX10-NEXT:    s_sext_i32_i16 s1, s1
-; GFX10-NEXT:    s_cmp_gt_i32 s3, s4
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cselect_b32 s5, s3, s4
-; GFX10-NEXT:    s_sub_i32 s5, s5, 0x7fff
-; GFX10-NEXT:    s_cmp_lt_i32 s3, s4
-; GFX10-NEXT:    s_cselect_b32 s3, s3, s4
-; GFX10-NEXT:    s_sext_i32_i16 s4, s5
-; GFX10-NEXT:    s_sub_i32 s3, s3, 0xffff8000
-; GFX10-NEXT:    s_cmp_gt_i32 s4, s1
-; GFX10-NEXT:    s_sext_i32_i16 s3, s3
-; GFX10-NEXT:    s_cselect_b32 s1, s4, s1
-; GFX10-NEXT:    s_sext_i32_i16 s1, s1
-; GFX10-NEXT:    s_cmp_lt_i32 s1, s3
-; GFX10-NEXT:    s_cselect_b32 s1, s1, s3
-; GFX10-NEXT:    s_sub_i32 s0, s0, s1
-; GFX10-NEXT:    s_sext_i32_i16 s0, s0
-; GFX10-NEXT:    s_ashr_i32 s0, s0, s2
+; GFX10-NEXT:    v_sub_nc_i16 v0, s0, s1 clamp
+; GFX10-NEXT:    v_ashrrev_i16_e64 v0, 8, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call i8 @llvm.ssub.sat.i8(i8 %lhs, i8 %rhs)
   ret i8 %result
@@ -408,26 +322,12 @@ define i16 @v_ssubsat_v2i8(i16 %lhs.arg, i16 %rhs.arg) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    s_mov_b32 s4, 8
 ; GFX9-NEXT:    v_lshrrev_b32_sdwa v2, s4, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
 ; GFX9-NEXT:    v_lshrrev_b32_sdwa v3, s4, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX9-NEXT:    v_max_i16_e32 v4, -1, v0
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
-; GFX9-NEXT:    v_subrev_u16_e32 v4, s4, v4
-; GFX9-NEXT:    s_movk_i32 s5, 0x8000
-; GFX9-NEXT:    v_min_i16_e32 v5, -1, v0
-; GFX9-NEXT:    v_max_i16_e32 v1, v4, v1
-; GFX9-NEXT:    v_subrev_u16_e32 v5, s5, v5
-; GFX9-NEXT:    v_min_i16_e32 v1, v1, v5
-; GFX9-NEXT:    v_sub_u16_e32 v0, v0, v1
-; GFX9-NEXT:    v_max_i16_e32 v1, -1, v2
-; GFX9-NEXT:    v_subrev_u16_e32 v1, s4, v1
-; GFX9-NEXT:    v_min_i16_e32 v4, -1, v2
-; GFX9-NEXT:    v_subrev_u16_e32 v4, s5, v4
-; GFX9-NEXT:    v_max_i16_e32 v1, v1, v3
-; GFX9-NEXT:    v_min_i16_e32 v1, v1, v4
+; GFX9-NEXT:    v_sub_i16 v0, v0, v1 clamp
 ; GFX9-NEXT:    s_movk_i32 s4, 0xff
-; GFX9-NEXT:    v_sub_u16_e32 v1, v2, v1
+; GFX9-NEXT:    v_sub_i16 v1, v2, v3 clamp
 ; GFX9-NEXT:    v_and_b32_sdwa v0, sext(v0), s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
 ; GFX9-NEXT:    v_and_b32_sdwa v1, sext(v1), s4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v0, v0, v1
@@ -438,31 +338,17 @@ define i16 @v_ssubsat_v2i8(i16 %lhs.arg, i16 %rhs.arg) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    s_mov_b32 s4, 8
-; GFX10-NEXT:    v_lshlrev_b16_e64 v2, 8, v0
-; GFX10-NEXT:    v_lshrrev_b32_sdwa v0, s4, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT:    s_movk_i32 s5, 0x7fff
-; GFX10-NEXT:    v_lshrrev_b32_sdwa v3, s4, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT:    v_lshlrev_b16_e64 v1, 8, v1
-; GFX10-NEXT:    v_max_i16_e64 v4, v2, -1
-; GFX10-NEXT:    v_max_i16_e64 v5, v0, -1
-; GFX10-NEXT:    v_min_i16_e64 v6, v2, -1
-; GFX10-NEXT:    v_min_i16_e64 v7, v0, -1
-; GFX10-NEXT:    s_movk_i32 s4, 0x8000
-; GFX10-NEXT:    v_sub_nc_u16_e64 v4, v4, s5
-; GFX10-NEXT:    v_sub_nc_u16_e64 v5, v5, s5
-; GFX10-NEXT:    v_sub_nc_u16_e64 v6, v6, s4
-; GFX10-NEXT:    v_sub_nc_u16_e64 v7, v7, s4
+; GFX10-NEXT:    v_lshlrev_b16_e64 v3, 8, v1
+; GFX10-NEXT:    v_lshrrev_b32_sdwa v2, s4, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    v_lshrrev_b32_sdwa v1, s4, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    v_lshlrev_b16_e64 v0, 8, v0
 ; GFX10-NEXT:    s_movk_i32 s4, 0xff
-; GFX10-NEXT:    v_max_i16_e64 v1, v4, v1
-; GFX10-NEXT:    v_max_i16_e64 v10, v5, v3
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_min_i16_e64 v1, v1, v6
-; GFX10-NEXT:    v_min_i16_e64 v3, v10, v7
-; GFX10-NEXT:    v_sub_nc_u16_e64 v1, v2, v1
-; GFX10-NEXT:    v_sub_nc_u16_e64 v0, v0, v3
-; GFX10-NEXT:    v_and_b32_sdwa v1, sext(v1), s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
-; GFX10-NEXT:    v_and_b32_sdwa v0, sext(v0), s4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
-; GFX10-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX10-NEXT:    v_sub_nc_i16 v1, v2, v1 clamp
+; GFX10-NEXT:    v_sub_nc_i16 v0, v0, v3 clamp
+; GFX10-NEXT:    v_and_b32_sdwa v1, sext(v1), s4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_and_b32_sdwa v0, sext(v0), s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %lhs = bitcast i16 %lhs.arg to <2 x i8>
   %rhs = bitcast i16 %rhs.arg to <2 x i8>
@@ -571,112 +457,40 @@ define amdgpu_ps i16 @s_ssubsat_v2i8(i16 inreg %lhs.arg, i16 inreg %rhs.arg) {
 ; GFX9-LABEL: s_ssubsat_v2i8:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_bfe_u32 s4, 8, 0x100000
-; GFX9-NEXT:    s_lshr_b32 s2, s0, 8
-; GFX9-NEXT:    s_lshl_b32 s0, s0, s4
 ; GFX9-NEXT:    s_lshr_b32 s3, s1, 8
 ; GFX9-NEXT:    s_lshl_b32 s1, s1, s4
-; GFX9-NEXT:    s_sext_i32_i16 s7, s0
-; GFX9-NEXT:    s_sext_i32_i16 s8, -1
-; GFX9-NEXT:    s_cmp_gt_i32 s7, s8
-; GFX9-NEXT:    s_movk_i32 s5, 0x7fff
-; GFX9-NEXT:    s_cselect_b32 s9, s7, s8
-; GFX9-NEXT:    s_sub_i32 s9, s9, s5
-; GFX9-NEXT:    s_cmp_lt_i32 s7, s8
-; GFX9-NEXT:    s_movk_i32 s6, 0x8000
-; GFX9-NEXT:    s_cselect_b32 s7, s7, s8
-; GFX9-NEXT:    s_sub_i32 s7, s7, s6
-; GFX9-NEXT:    s_sext_i32_i16 s9, s9
-; GFX9-NEXT:    s_sext_i32_i16 s1, s1
-; GFX9-NEXT:    s_cmp_gt_i32 s9, s1
-; GFX9-NEXT:    s_cselect_b32 s1, s9, s1
-; GFX9-NEXT:    s_sext_i32_i16 s1, s1
-; GFX9-NEXT:    s_sext_i32_i16 s7, s7
-; GFX9-NEXT:    s_cmp_lt_i32 s1, s7
-; GFX9-NEXT:    s_cselect_b32 s1, s1, s7
-; GFX9-NEXT:    s_sub_i32 s0, s0, s1
-; GFX9-NEXT:    s_sext_i32_i16 s0, s0
-; GFX9-NEXT:    s_lshl_b32 s1, s2, s4
-; GFX9-NEXT:    s_lshl_b32 s2, s3, s4
-; GFX9-NEXT:    s_ashr_i32 s0, s0, s4
-; GFX9-NEXT:    s_sext_i32_i16 s3, s1
-; GFX9-NEXT:    s_cmp_gt_i32 s3, s8
-; GFX9-NEXT:    s_cselect_b32 s7, s3, s8
-; GFX9-NEXT:    s_sub_i32 s5, s7, s5
-; GFX9-NEXT:    s_cmp_lt_i32 s3, s8
-; GFX9-NEXT:    s_cselect_b32 s3, s3, s8
-; GFX9-NEXT:    s_sub_i32 s3, s3, s6
-; GFX9-NEXT:    s_sext_i32_i16 s5, s5
-; GFX9-NEXT:    s_sext_i32_i16 s2, s2
-; GFX9-NEXT:    s_cmp_gt_i32 s5, s2
-; GFX9-NEXT:    s_cselect_b32 s2, s5, s2
-; GFX9-NEXT:    s_sext_i32_i16 s2, s2
-; GFX9-NEXT:    s_sext_i32_i16 s3, s3
-; GFX9-NEXT:    s_cmp_lt_i32 s2, s3
-; GFX9-NEXT:    s_cselect_b32 s2, s2, s3
-; GFX9-NEXT:    s_sub_i32 s1, s1, s2
-; GFX9-NEXT:    s_sext_i32_i16 s1, s1
-; GFX9-NEXT:    s_movk_i32 s2, 0xff
-; GFX9-NEXT:    s_ashr_i32 s1, s1, s4
-; GFX9-NEXT:    s_and_b32 s1, s1, s2
-; GFX9-NEXT:    s_and_b32 s0, s0, s2
-; GFX9-NEXT:    s_lshl_b32 s1, s1, s4
-; GFX9-NEXT:    s_or_b32 s0, s0, s1
+; GFX9-NEXT:    s_lshr_b32 s2, s0, 8
+; GFX9-NEXT:    v_mov_b32_e32 v0, s1
+; GFX9-NEXT:    s_lshl_b32 s0, s0, s4
+; GFX9-NEXT:    s_lshl_b32 s1, s3, s4
+; GFX9-NEXT:    v_sub_i16 v0, s0, v0 clamp
+; GFX9-NEXT:    s_lshl_b32 s0, s2, s4
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_sub_i16 v1, s0, v1 clamp
+; GFX9-NEXT:    s_movk_i32 s0, 0xff
+; GFX9-NEXT:    v_and_b32_sdwa v0, sext(v0), s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-NEXT:    v_and_b32_sdwa v1, sext(v1), s0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_ssubsat_v2i8:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_bfe_u32 s2, 8, 0x100000
-; GFX10-NEXT:    s_lshr_b32 s3, s0, 8
-; GFX10-NEXT:    s_lshl_b32 s0, s0, s2
-; GFX10-NEXT:    s_sext_i32_i16 s6, -1
-; GFX10-NEXT:    s_sext_i32_i16 s5, s0
+; GFX10-NEXT:    s_lshr_b32 s2, s0, 8
+; GFX10-NEXT:    s_bfe_u32 s3, 8, 0x100000
 ; GFX10-NEXT:    s_lshr_b32 s4, s1, 8
-; GFX10-NEXT:    s_lshl_b32 s1, s1, s2
-; GFX10-NEXT:    s_cmp_gt_i32 s5, s6
-; GFX10-NEXT:    s_movk_i32 s7, 0x7fff
-; GFX10-NEXT:    s_cselect_b32 s8, s5, s6
-; GFX10-NEXT:    s_movk_i32 s9, 0x8000
-; GFX10-NEXT:    s_sub_i32 s8, s8, s7
-; GFX10-NEXT:    s_cmp_lt_i32 s5, s6
-; GFX10-NEXT:    s_sext_i32_i16 s8, s8
-; GFX10-NEXT:    s_cselect_b32 s5, s5, s6
-; GFX10-NEXT:    s_sext_i32_i16 s1, s1
-; GFX10-NEXT:    s_sub_i32 s5, s5, s9
-; GFX10-NEXT:    s_cmp_gt_i32 s8, s1
-; GFX10-NEXT:    s_sext_i32_i16 s5, s5
-; GFX10-NEXT:    s_cselect_b32 s1, s8, s1
+; GFX10-NEXT:    s_lshl_b32 s0, s0, s3
+; GFX10-NEXT:    s_lshl_b32 s1, s1, s3
+; GFX10-NEXT:    s_lshl_b32 s2, s2, s3
+; GFX10-NEXT:    s_lshl_b32 s3, s4, s3
+; GFX10-NEXT:    v_sub_nc_i16 v0, s0, s1 clamp
+; GFX10-NEXT:    v_sub_nc_i16 v1, s2, s3 clamp
+; GFX10-NEXT:    s_movk_i32 s0, 0xff
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_sext_i32_i16 s1, s1
-; GFX10-NEXT:    s_cmp_lt_i32 s1, s5
-; GFX10-NEXT:    s_cselect_b32 s1, s1, s5
-; GFX10-NEXT:    s_lshl_b32 s3, s3, s2
-; GFX10-NEXT:    s_sub_i32 s0, s0, s1
-; GFX10-NEXT:    s_lshl_b32 s1, s4, s2
-; GFX10-NEXT:    s_sext_i32_i16 s0, s0
-; GFX10-NEXT:    s_sext_i32_i16 s4, s3
-; GFX10-NEXT:    s_ashr_i32 s0, s0, s2
-; GFX10-NEXT:    s_cmp_gt_i32 s4, s6
-; GFX10-NEXT:    s_sext_i32_i16 s1, s1
-; GFX10-NEXT:    s_cselect_b32 s5, s4, s6
-; GFX10-NEXT:    s_sub_i32 s5, s5, s7
-; GFX10-NEXT:    s_cmp_lt_i32 s4, s6
-; GFX10-NEXT:    s_sext_i32_i16 s5, s5
-; GFX10-NEXT:    s_cselect_b32 s4, s4, s6
-; GFX10-NEXT:    s_sub_i32 s4, s4, s9
-; GFX10-NEXT:    s_cmp_gt_i32 s5, s1
-; GFX10-NEXT:    s_sext_i32_i16 s4, s4
-; GFX10-NEXT:    s_cselect_b32 s1, s5, s1
-; GFX10-NEXT:    s_sext_i32_i16 s1, s1
-; GFX10-NEXT:    s_cmp_lt_i32 s1, s4
-; GFX10-NEXT:    s_cselect_b32 s1, s1, s4
-; GFX10-NEXT:    s_sub_i32 s1, s3, s1
-; GFX10-NEXT:    s_movk_i32 s3, 0xff
-; GFX10-NEXT:    s_sext_i32_i16 s1, s1
-; GFX10-NEXT:    s_and_b32 s0, s0, s3
-; GFX10-NEXT:    s_ashr_i32 s1, s1, s2
-; GFX10-NEXT:    s_and_b32 s1, s1, s3
-; GFX10-NEXT:    s_lshl_b32 s1, s1, s2
-; GFX10-NEXT:    s_or_b32 s0, s0, s1
+; GFX10-NEXT:    v_and_b32_sdwa v0, sext(v0), s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_and_b32_sdwa v1, sext(v1), s0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
   %lhs = bitcast i16 %lhs.arg to <2 x i8>
   %rhs = bitcast i16 %rhs.arg to <2 x i8>
@@ -815,52 +629,25 @@ define i32 @v_ssubsat_v4i8(i32 %lhs.arg, i32 %rhs.arg) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    s_mov_b32 s4, 8
 ; GFX9-NEXT:    v_lshrrev_b32_sdwa v2, s4, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT:    v_lshrrev_b32_sdwa v5, s4, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v4, 24, v0
-; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; GFX9-NEXT:    v_lshrrev_b32_sdwa v5, s4, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX9-NEXT:    v_max_i16_e32 v8, -1, v0
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v6, 16, v1
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v7, 24, v1
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
-; GFX9-NEXT:    v_subrev_u16_e32 v8, s4, v8
-; GFX9-NEXT:    s_movk_i32 s5, 0x8000
-; GFX9-NEXT:    v_min_i16_e32 v10, -1, v0
-; GFX9-NEXT:    v_max_i16_e32 v1, v8, v1
-; GFX9-NEXT:    v_subrev_u16_e32 v10, s5, v10
-; GFX9-NEXT:    v_min_i16_e32 v1, v1, v10
-; GFX9-NEXT:    v_sub_u16_e32 v0, v0, v1
-; GFX9-NEXT:    v_max_i16_e32 v1, -1, v2
-; GFX9-NEXT:    v_subrev_u16_e32 v1, s4, v1
-; GFX9-NEXT:    v_min_i16_e32 v8, -1, v2
-; GFX9-NEXT:    v_subrev_u16_e32 v8, s5, v8
-; GFX9-NEXT:    v_max_i16_e32 v1, v1, v5
-; GFX9-NEXT:    v_min_i16_e32 v1, v1, v8
-; GFX9-NEXT:    v_sub_u16_e32 v1, v2, v1
+; GFX9-NEXT:    v_sub_i16 v0, v0, v1 clamp
+; GFX9-NEXT:    v_sub_i16 v1, v2, v5 clamp
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v2, 8, v3
-; GFX9-NEXT:    v_mov_b32_e32 v9, 0x7fff
-; GFX9-NEXT:    v_max_i16_e32 v5, -1, v2
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v3, 8, v6
-; GFX9-NEXT:    v_min_i16_e32 v6, -1, v2
-; GFX9-NEXT:    v_sub_u16_e32 v5, v5, v9
-; GFX9-NEXT:    v_subrev_u16_e32 v6, s5, v6
-; GFX9-NEXT:    v_max_i16_e32 v3, v5, v3
-; GFX9-NEXT:    v_min_i16_e32 v3, v3, v6
-; GFX9-NEXT:    v_sub_u16_e32 v2, v2, v3
-; GFX9-NEXT:    v_lshlrev_b16_e32 v3, 8, v4
-; GFX9-NEXT:    v_max_i16_e32 v5, -1, v3
-; GFX9-NEXT:    v_min_i16_e32 v6, -1, v3
-; GFX9-NEXT:    v_lshlrev_b16_e32 v4, 8, v7
-; GFX9-NEXT:    v_sub_u16_e32 v5, v5, v9
 ; GFX9-NEXT:    s_movk_i32 s4, 0xff
-; GFX9-NEXT:    v_subrev_u16_e32 v6, 0x8000, v6
-; GFX9-NEXT:    v_max_i16_e32 v4, v5, v4
 ; GFX9-NEXT:    v_and_b32_sdwa v1, sext(v1), s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
-; GFX9-NEXT:    v_min_i16_e32 v4, v4, v6
+; GFX9-NEXT:    v_sub_i16 v2, v2, v3 clamp
+; GFX9-NEXT:    v_lshlrev_b16_e32 v3, 8, v4
+; GFX9-NEXT:    v_lshlrev_b16_e32 v4, 8, v7
 ; GFX9-NEXT:    v_ashrrev_i16_e32 v0, 8, v0
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
-; GFX9-NEXT:    v_sub_u16_e32 v3, v3, v4
+; GFX9-NEXT:    v_sub_i16 v3, v3, v4 clamp
 ; GFX9-NEXT:    v_and_or_b32 v0, v0, s4, v1
 ; GFX9-NEXT:    v_and_b32_sdwa v1, sext(v2), s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
 ; GFX9-NEXT:    v_and_b32_sdwa v2, sext(v3), s4 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
@@ -871,57 +658,30 @@ define i32 @v_ssubsat_v4i8(i32 %lhs.arg, i32 %rhs.arg) {
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_lshlrev_b16_e64 v4, 8, v0
 ; GFX10-NEXT:    s_mov_b32 s4, 8
-; GFX10-NEXT:    v_lshlrev_b16_e64 v7, 8, v1
+; GFX10-NEXT:    v_lshlrev_b16_e64 v5, 8, v0
 ; GFX10-NEXT:    v_lshrrev_b32_sdwa v2, s4, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT:    v_lshrrev_b32_sdwa v5, s4, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT:    v_max_i16_e64 v8, v4, -1
-; GFX10-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX10-NEXT:    v_min_i16_e64 v10, v4, -1
-; GFX10-NEXT:    v_max_i16_e64 v9, v2, -1
+; GFX10-NEXT:    v_lshrrev_b32_sdwa v3, s4, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    v_lshlrev_b16_e64 v6, 8, v1
 ; GFX10-NEXT:    s_mov_b32 s5, 16
-; GFX10-NEXT:    v_sub_nc_u16_e64 v8, v8, s4
-; GFX10-NEXT:    v_lshrrev_b32_sdwa v3, s5, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT:    s_mov_b32 s6, 24
-; GFX10-NEXT:    v_lshrrev_b32_sdwa v6, s5, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT:    s_movk_i32 s5, 0x8000
-; GFX10-NEXT:    v_sub_nc_u16_e64 v15, v9, s4
-; GFX10-NEXT:    v_lshrrev_b32_sdwa v0, s6, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT:    v_min_i16_e64 v11, v2, -1
-; GFX10-NEXT:    v_max_i16_e64 v7, v8, v7
-; GFX10-NEXT:    v_sub_nc_u16_e64 v10, v10, s5
-; GFX10-NEXT:    v_max_i16_e64 v5, v15, v5
-; GFX10-NEXT:    v_mov_b32_e32 v9, 0x7fff
-; GFX10-NEXT:    v_sub_nc_u16_e64 v8, v11, s5
-; GFX10-NEXT:    v_max_i16_e64 v11, v3, -1
-; GFX10-NEXT:    v_min_i16_e64 v7, v7, v10
-; GFX10-NEXT:    v_max_i16_e64 v10, v0, -1
-; GFX10-NEXT:    v_lshrrev_b32_sdwa v1, s6, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT:    v_min_i16_e64 v5, v5, v8
-; GFX10-NEXT:    v_sub_nc_u16_e64 v11, v11, v9
-; GFX10-NEXT:    v_min_i16_e64 v8, v3, -1
-; GFX10-NEXT:    v_sub_nc_u16_e64 v15, v10, v9
-; GFX10-NEXT:    v_min_i16_e64 v12, v0, -1
-; GFX10-NEXT:    v_sub_nc_u16_e64 v2, v2, v5
-; GFX10-NEXT:    v_max_i16_e64 v6, v11, v6
-; GFX10-NEXT:    v_sub_nc_u16_e64 v5, v8, s5
-; GFX10-NEXT:    v_max_i16_e64 v1, v15, v1
-; GFX10-NEXT:    v_sub_nc_u16_e64 v8, v12, 0x8000
-; GFX10-NEXT:    s_movk_i32 s4, 0xff
-; GFX10-NEXT:    v_sub_nc_u16_e64 v4, v4, v7
-; GFX10-NEXT:    v_and_b32_sdwa v2, sext(v2), s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
-; GFX10-NEXT:    v_min_i16_e64 v5, v6, v5
-; GFX10-NEXT:    v_min_i16_e64 v1, v1, v8
+; GFX10-NEXT:    s_mov_b32 s4, 24
+; GFX10-NEXT:    v_lshrrev_b32_sdwa v4, s5, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    v_sub_nc_i16 v2, v2, v3 clamp
+; GFX10-NEXT:    v_lshrrev_b32_sdwa v3, s5, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    s_movk_i32 s5, 0xff
+; GFX10-NEXT:    v_lshrrev_b32_sdwa v0, s4, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    v_lshrrev_b32_sdwa v1, s4, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    v_and_b32_sdwa v2, sext(v2), s5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_sub_nc_i16 v5, v5, v6 clamp
+; GFX10-NEXT:    v_sub_nc_i16 v3, v4, v3 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_ashrrev_i16_e64 v4, 8, v4
+; GFX10-NEXT:    v_sub_nc_i16 v0, v0, v1 clamp
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
-; GFX10-NEXT:    v_sub_nc_u16_e64 v3, v3, v5
-; GFX10-NEXT:    v_sub_nc_u16_e64 v0, v0, v1
-; GFX10-NEXT:    v_and_or_b32 v1, v4, s4, v2
-; GFX10-NEXT:    v_and_b32_sdwa v2, sext(v3), s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
-; GFX10-NEXT:    v_and_b32_sdwa v0, sext(v0), s4 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
-; GFX10-NEXT:    v_or3_b32 v0, v1, v2, v0
+; GFX10-NEXT:    v_ashrrev_i16_e64 v4, 8, v5
+; GFX10-NEXT:    v_and_b32_sdwa v1, sext(v3), s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_and_b32_sdwa v0, sext(v0), s5 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_and_or_b32 v2, v4, s5, v2
+; GFX10-NEXT:    v_or3_b32 v0, v2, v1, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %lhs = bitcast i32 %lhs.arg to <4 x i8>
   %rhs = bitcast i32 %rhs.arg to <4 x i8>
@@ -1118,212 +878,70 @@ define amdgpu_ps i32 @s_ssubsat_v4i8(i32 inreg %lhs.arg, i32 inreg %rhs.arg) {
 ; GFX9-LABEL: s_ssubsat_v4i8:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_bfe_u32 s8, 8, 0x100000
-; GFX9-NEXT:    s_lshr_b32 s2, s0, 8
-; GFX9-NEXT:    s_lshr_b32 s3, s0, 16
-; GFX9-NEXT:    s_lshr_b32 s4, s0, 24
-; GFX9-NEXT:    s_lshl_b32 s0, s0, s8
 ; GFX9-NEXT:    s_lshr_b32 s5, s1, 8
 ; GFX9-NEXT:    s_lshr_b32 s6, s1, 16
 ; GFX9-NEXT:    s_lshr_b32 s7, s1, 24
 ; GFX9-NEXT:    s_lshl_b32 s1, s1, s8
-; GFX9-NEXT:    s_sext_i32_i16 s11, s0
-; GFX9-NEXT:    s_sext_i32_i16 s12, -1
-; GFX9-NEXT:    s_cmp_gt_i32 s11, s12
-; GFX9-NEXT:    s_movk_i32 s9, 0x7fff
-; GFX9-NEXT:    s_cselect_b32 s13, s11, s12
-; GFX9-NEXT:    s_sub_i32 s13, s13, s9
-; GFX9-NEXT:    s_cmp_lt_i32 s11, s12
-; GFX9-NEXT:    s_movk_i32 s10, 0x8000
-; GFX9-NEXT:    s_cselect_b32 s11, s11, s12
-; GFX9-NEXT:    s_sub_i32 s11, s11, s10
-; GFX9-NEXT:    s_sext_i32_i16 s13, s13
-; GFX9-NEXT:    s_sext_i32_i16 s1, s1
-; GFX9-NEXT:    s_cmp_gt_i32 s13, s1
-; GFX9-NEXT:    s_cselect_b32 s1, s13, s1
-; GFX9-NEXT:    s_sext_i32_i16 s1, s1
-; GFX9-NEXT:    s_sext_i32_i16 s11, s11
-; GFX9-NEXT:    s_cmp_lt_i32 s1, s11
-; GFX9-NEXT:    s_cselect_b32 s1, s1, s11
-; GFX9-NEXT:    s_sub_i32 s0, s0, s1
-; GFX9-NEXT:    s_sext_i32_i16 s0, s0
-; GFX9-NEXT:    s_lshl_b32 s1, s2, s8
-; GFX9-NEXT:    s_lshl_b32 s2, s5, s8
-; GFX9-NEXT:    s_ashr_i32 s0, s0, s8
-; GFX9-NEXT:    s_sext_i32_i16 s5, s1
-; GFX9-NEXT:    s_cmp_gt_i32 s5, s12
-; GFX9-NEXT:    s_cselect_b32 s11, s5, s12
-; GFX9-NEXT:    s_sub_i32 s11, s11, s9
-; GFX9-NEXT:    s_cmp_lt_i32 s5, s12
-; GFX9-NEXT:    s_cselect_b32 s5, s5, s12
-; GFX9-NEXT:    s_sub_i32 s5, s5, s10
-; GFX9-NEXT:    s_sext_i32_i16 s11, s11
-; GFX9-NEXT:    s_sext_i32_i16 s2, s2
-; GFX9-NEXT:    s_cmp_gt_i32 s11, s2
-; GFX9-NEXT:    s_cselect_b32 s2, s11, s2
-; GFX9-NEXT:    s_sext_i32_i16 s2, s2
-; GFX9-NEXT:    s_sext_i32_i16 s5, s5
-; GFX9-NEXT:    s_cmp_lt_i32 s2, s5
-; GFX9-NEXT:    s_cselect_b32 s2, s2, s5
-; GFX9-NEXT:    s_sub_i32 s1, s1, s2
-; GFX9-NEXT:    s_sext_i32_i16 s1, s1
-; GFX9-NEXT:    s_lshl_b32 s2, s3, s8
-; GFX9-NEXT:    s_lshl_b32 s3, s6, s8
-; GFX9-NEXT:    s_ashr_i32 s1, s1, s8
-; GFX9-NEXT:    s_sext_i32_i16 s5, s2
-; GFX9-NEXT:    s_cmp_gt_i32 s5, s12
-; GFX9-NEXT:    s_cselect_b32 s6, s5, s12
-; GFX9-NEXT:    s_sub_i32 s6, s6, s9
-; GFX9-NEXT:    s_cmp_lt_i32 s5, s12
-; GFX9-NEXT:    s_cselect_b32 s5, s5, s12
-; GFX9-NEXT:    s_sub_i32 s5, s5, s10
-; GFX9-NEXT:    s_sext_i32_i16 s6, s6
-; GFX9-NEXT:    s_sext_i32_i16 s3, s3
-; GFX9-NEXT:    s_cmp_gt_i32 s6, s3
-; GFX9-NEXT:    s_cselect_b32 s3, s6, s3
-; GFX9-NEXT:    s_sext_i32_i16 s3, s3
-; GFX9-NEXT:    s_sext_i32_i16 s5, s5
-; GFX9-NEXT:    s_cmp_lt_i32 s3, s5
-; GFX9-NEXT:    s_cselect_b32 s3, s3, s5
-; GFX9-NEXT:    s_sub_i32 s2, s2, s3
-; GFX9-NEXT:    s_sext_i32_i16 s2, s2
-; GFX9-NEXT:    s_lshl_b32 s3, s4, s8
-; GFX9-NEXT:    s_lshl_b32 s4, s7, s8
-; GFX9-NEXT:    s_ashr_i32 s2, s2, s8
-; GFX9-NEXT:    s_sext_i32_i16 s5, s3
-; GFX9-NEXT:    s_cmp_gt_i32 s5, s12
-; GFX9-NEXT:    s_cselect_b32 s6, s5, s12
-; GFX9-NEXT:    s_sub_i32 s6, s6, s9
-; GFX9-NEXT:    s_cmp_lt_i32 s5, s12
-; GFX9-NEXT:    s_cselect_b32 s5, s5, s12
-; GFX9-NEXT:    s_sub_i32 s5, s5, s10
-; GFX9-NEXT:    s_sext_i32_i16 s6, s6
-; GFX9-NEXT:    s_sext_i32_i16 s4, s4
-; GFX9-NEXT:    s_cmp_gt_i32 s6, s4
-; GFX9-NEXT:    s_cselect_b32 s4, s6, s4
-; GFX9-NEXT:    s_sext_i32_i16 s4, s4
-; GFX9-NEXT:    s_sext_i32_i16 s5, s5
-; GFX9-NEXT:    s_cmp_lt_i32 s4, s5
-; GFX9-NEXT:    s_cselect_b32 s4, s4, s5
-; GFX9-NEXT:    s_sub_i32 s3, s3, s4
-; GFX9-NEXT:    s_movk_i32 s4, 0xff
-; GFX9-NEXT:    s_and_b32 s1, s1, s4
-; GFX9-NEXT:    s_and_b32 s0, s0, s4
-; GFX9-NEXT:    s_lshl_b32 s1, s1, 8
-; GFX9-NEXT:    s_sext_i32_i16 s3, s3
-; GFX9-NEXT:    s_or_b32 s0, s0, s1
-; GFX9-NEXT:    s_and_b32 s1, s2, s4
-; GFX9-NEXT:    s_lshl_b32 s1, s1, 16
-; GFX9-NEXT:    s_ashr_i32 s3, s3, s8
-; GFX9-NEXT:    s_or_b32 s0, s0, s1
-; GFX9-NEXT:    s_and_b32 s1, s3, s4
-; GFX9-NEXT:    s_lshl_b32 s1, s1, 24
-; GFX9-NEXT:    s_or_b32 s0, s0, s1
+; GFX9-NEXT:    v_mov_b32_e32 v0, s1
+; GFX9-NEXT:    s_lshl_b32 s1, s5, s8
+; GFX9-NEXT:    s_lshr_b32 s2, s0, 8
+; GFX9-NEXT:    s_lshr_b32 s3, s0, 16
+; GFX9-NEXT:    s_lshr_b32 s4, s0, 24
+; GFX9-NEXT:    s_lshl_b32 s0, s0, s8
+; GFX9-NEXT:    v_sub_i16 v0, s0, v0 clamp
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    s_lshl_b32 s0, s2, s8
+; GFX9-NEXT:    s_lshl_b32 s1, s6, s8
+; GFX9-NEXT:    v_sub_i16 v1, s0, v1 clamp
+; GFX9-NEXT:    v_mov_b32_e32 v2, s1
+; GFX9-NEXT:    s_lshl_b32 s0, s3, s8
+; GFX9-NEXT:    s_lshl_b32 s1, s7, s8
+; GFX9-NEXT:    v_sub_i16 v2, s0, v2 clamp
+; GFX9-NEXT:    s_lshl_b32 s0, s4, s8
+; GFX9-NEXT:    v_mov_b32_e32 v3, s1
+; GFX9-NEXT:    v_sub_i16 v3, s0, v3 clamp
+; GFX9-NEXT:    s_movk_i32 s0, 0xff
+; GFX9-NEXT:    v_and_b32_sdwa v1, sext(v1), s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-NEXT:    v_ashrrev_i16_e32 v0, 8, v0
+; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX9-NEXT:    v_and_or_b32 v0, v0, s0, v1
+; GFX9-NEXT:    v_and_b32_sdwa v1, sext(v2), s0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-NEXT:    v_and_b32_sdwa v2, sext(v3), s0 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-NEXT:    v_or3_b32 v0, v0, v1, v2
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_ssubsat_v4i8:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_bfe_u32 s6, 8, 0x100000
+; GFX10-NEXT:    s_bfe_u32 s5, 8, 0x100000
 ; GFX10-NEXT:    s_lshr_b32 s2, s0, 8
+; GFX10-NEXT:    s_lshr_b32 s6, s1, 8
+; GFX10-NEXT:    s_lshl_b32 s2, s2, s5
+; GFX10-NEXT:    s_lshl_b32 s6, s6, s5
 ; GFX10-NEXT:    s_lshr_b32 s3, s0, 16
+; GFX10-NEXT:    v_sub_nc_i16 v1, s2, s6 clamp
 ; GFX10-NEXT:    s_lshr_b32 s4, s0, 24
-; GFX10-NEXT:    s_lshl_b32 s0, s0, s6
-; GFX10-NEXT:    s_sext_i32_i16 s10, -1
-; GFX10-NEXT:    s_sext_i32_i16 s9, s0
-; GFX10-NEXT:    s_lshr_b32 s5, s1, 8
-; GFX10-NEXT:    s_lshr_b32 s7, s1, 16
-; GFX10-NEXT:    s_lshr_b32 s8, s1, 24
-; GFX10-NEXT:    s_lshl_b32 s1, s1, s6
-; GFX10-NEXT:    s_cmp_gt_i32 s9, s10
-; GFX10-NEXT:    s_movk_i32 s11, 0x7fff
-; GFX10-NEXT:    s_cselect_b32 s12, s9, s10
-; GFX10-NEXT:    s_movk_i32 s13, 0x8000
-; GFX10-NEXT:    s_sub_i32 s12, s12, s11
-; GFX10-NEXT:    s_cmp_lt_i32 s9, s10
-; GFX10-NEXT:    s_sext_i32_i16 s12, s12
-; GFX10-NEXT:    s_cselect_b32 s9, s9, s10
-; GFX10-NEXT:    s_sext_i32_i16 s1, s1
-; GFX10-NEXT:    s_sub_i32 s9, s9, s13
-; GFX10-NEXT:    s_cmp_gt_i32 s12, s1
-; GFX10-NEXT:    s_sext_i32_i16 s9, s9
-; GFX10-NEXT:    s_cselect_b32 s1, s12, s1
+; GFX10-NEXT:    s_movk_i32 s2, 0xff
+; GFX10-NEXT:    s_lshl_b32 s0, s0, s5
+; GFX10-NEXT:    s_lshl_b32 s7, s1, s5
+; GFX10-NEXT:    v_and_b32_sdwa v1, sext(v1), s2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_sub_nc_i16 v0, s0, s7 clamp
+; GFX10-NEXT:    s_lshr_b32 s0, s1, 16
+; GFX10-NEXT:    s_lshr_b32 s1, s1, 24
+; GFX10-NEXT:    s_lshl_b32 s3, s3, s5
+; GFX10-NEXT:    s_lshl_b32 s0, s0, s5
+; GFX10-NEXT:    s_lshl_b32 s4, s4, s5
+; GFX10-NEXT:    s_lshl_b32 s1, s1, s5
+; GFX10-NEXT:    v_ashrrev_i16_e64 v0, 8, v0
+; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX10-NEXT:    v_sub_nc_i16 v2, s3, s0 clamp
+; GFX10-NEXT:    v_sub_nc_i16 v3, s4, s1 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_sext_i32_i16 s1, s1
-; GFX10-NEXT:    s_cmp_lt_i32 s1, s9
-; GFX10-NEXT:    s_cselect_b32 s1, s1, s9
-; GFX10-NEXT:    s_lshl_b32 s5, s5, s6
-; GFX10-NEXT:    s_sub_i32 s0, s0, s1
-; GFX10-NEXT:    s_lshl_b32 s1, s2, s6
-; GFX10-NEXT:    s_sext_i32_i16 s0, s0
-; GFX10-NEXT:    s_sext_i32_i16 s2, s1
-; GFX10-NEXT:    s_ashr_i32 s0, s0, s6
-; GFX10-NEXT:    s_cmp_gt_i32 s2, s10
-; GFX10-NEXT:    s_sext_i32_i16 s5, s5
-; GFX10-NEXT:    s_cselect_b32 s9, s2, s10
-; GFX10-NEXT:    s_sub_i32 s9, s9, s11
-; GFX10-NEXT:    s_cmp_lt_i32 s2, s10
-; GFX10-NEXT:    s_sext_i32_i16 s9, s9
-; GFX10-NEXT:    s_cselect_b32 s2, s2, s10
-; GFX10-NEXT:    s_sub_i32 s2, s2, s13
-; GFX10-NEXT:    s_cmp_gt_i32 s9, s5
-; GFX10-NEXT:    s_sext_i32_i16 s2, s2
-; GFX10-NEXT:    s_cselect_b32 s5, s9, s5
-; GFX10-NEXT:    s_sext_i32_i16 s5, s5
-; GFX10-NEXT:    s_cmp_lt_i32 s5, s2
-; GFX10-NEXT:    s_cselect_b32 s2, s5, s2
-; GFX10-NEXT:    s_lshl_b32 s3, s3, s6
-; GFX10-NEXT:    s_sub_i32 s1, s1, s2
-; GFX10-NEXT:    s_sext_i32_i16 s5, s3
-; GFX10-NEXT:    s_sext_i32_i16 s1, s1
-; GFX10-NEXT:    s_lshl_b32 s2, s7, s6
-; GFX10-NEXT:    s_ashr_i32 s1, s1, s6
-; GFX10-NEXT:    s_cmp_gt_i32 s5, s10
-; GFX10-NEXT:    s_sext_i32_i16 s2, s2
-; GFX10-NEXT:    s_cselect_b32 s7, s5, s10
-; GFX10-NEXT:    s_sub_i32 s7, s7, s11
-; GFX10-NEXT:    s_cmp_lt_i32 s5, s10
-; GFX10-NEXT:    s_sext_i32_i16 s7, s7
-; GFX10-NEXT:    s_cselect_b32 s5, s5, s10
-; GFX10-NEXT:    s_sub_i32 s5, s5, s13
-; GFX10-NEXT:    s_cmp_gt_i32 s7, s2
-; GFX10-NEXT:    s_sext_i32_i16 s5, s5
-; GFX10-NEXT:    s_cselect_b32 s2, s7, s2
-; GFX10-NEXT:    s_sext_i32_i16 s2, s2
-; GFX10-NEXT:    s_cmp_lt_i32 s2, s5
-; GFX10-NEXT:    s_cselect_b32 s2, s2, s5
-; GFX10-NEXT:    s_lshl_b32 s4, s4, s6
-; GFX10-NEXT:    s_sub_i32 s2, s3, s2
-; GFX10-NEXT:    s_sext_i32_i16 s5, s4
-; GFX10-NEXT:    s_sext_i32_i16 s2, s2
-; GFX10-NEXT:    s_lshl_b32 s3, s8, s6
-; GFX10-NEXT:    s_ashr_i32 s2, s2, s6
-; GFX10-NEXT:    s_cmp_gt_i32 s5, s10
-; GFX10-NEXT:    s_sext_i32_i16 s3, s3
-; GFX10-NEXT:    s_cselect_b32 s7, s5, s10
-; GFX10-NEXT:    s_sub_i32 s7, s7, s11
-; GFX10-NEXT:    s_cmp_lt_i32 s5, s10
-; GFX10-NEXT:    s_sext_i32_i16 s7, s7
-; GFX10-NEXT:    s_cselect_b32 s5, s5, s10
-; GFX10-NEXT:    s_sub_i32 s5, s5, s13
-; GFX10-NEXT:    s_cmp_gt_i32 s7, s3
-; GFX10-NEXT:    s_sext_i32_i16 s5, s5
-; GFX10-NEXT:    s_cselect_b32 s3, s7, s3
-; GFX10-NEXT:    s_movk_i32 s7, 0xff
-; GFX10-NEXT:    s_sext_i32_i16 s3, s3
-; GFX10-NEXT:    s_cmp_lt_i32 s3, s5
-; GFX10-NEXT:    s_cselect_b32 s3, s3, s5
-; GFX10-NEXT:    s_and_b32 s1, s1, s7
-; GFX10-NEXT:    s_sub_i32 s3, s4, s3
-; GFX10-NEXT:    s_and_b32 s0, s0, s7
-; GFX10-NEXT:    s_sext_i32_i16 s3, s3
-; GFX10-NEXT:    s_lshl_b32 s1, s1, 8
-; GFX10-NEXT:    s_and_b32 s2, s2, s7
-; GFX10-NEXT:    s_ashr_i32 s3, s3, s6
-; GFX10-NEXT:    s_or_b32 s0, s0, s1
-; GFX10-NEXT:    s_lshl_b32 s1, s2, 16
-; GFX10-NEXT:    s_and_b32 s2, s3, s7
-; GFX10-NEXT:    s_or_b32 s0, s0, s1
-; GFX10-NEXT:    s_lshl_b32 s1, s2, 24
-; GFX10-NEXT:    s_or_b32 s0, s0, s1
+; GFX10-NEXT:    v_and_or_b32 v0, v0, s2, v1
+; GFX10-NEXT:    v_and_b32_sdwa v1, sext(v2), s2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_and_b32_sdwa v2, sext(v3), s2 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_or3_b32 v0, v0, v1, v2
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
   %lhs = bitcast i32 %lhs.arg to <4 x i8>
   %rhs = bitcast i32 %rhs.arg to <4 x i8>
@@ -1368,14 +986,8 @@ define i24 @v_ssubsat_i24(i24 %lhs, i24 %rhs) {
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
-; GFX9-NEXT:    v_max_i32_e32 v2, -1, v0
-; GFX9-NEXT:    v_min_i32_e32 v3, -1, v0
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
-; GFX9-NEXT:    v_subrev_u32_e32 v2, 0x7fffffff, v2
-; GFX9-NEXT:    v_subrev_u32_e32 v3, 0x80000000, v3
-; GFX9-NEXT:    v_max_i32_e32 v1, v2, v1
-; GFX9-NEXT:    v_min_i32_e32 v1, v1, v3
-; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v1
+; GFX9-NEXT:    v_sub_i32 v0, v0, v1 clamp
 ; GFX9-NEXT:    v_ashrrev_i32_e32 v0, 8, v0
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -1386,13 +998,7 @@ define i24 @v_ssubsat_i24(i24 %lhs, i24 %rhs) {
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_max_i32_e32 v2, -1, v0
-; GFX10-NEXT:    v_min_i32_e32 v3, -1, v0
-; GFX10-NEXT:    v_subrev_nc_u32_e32 v2, 0x7fffffff, v2
-; GFX10-NEXT:    v_subrev_nc_u32_e32 v3, 0x80000000, v3
-; GFX10-NEXT:    v_max_i32_e32 v1, v2, v1
-; GFX10-NEXT:    v_min_i32_e32 v1, v1, v3
-; GFX10-NEXT:    v_sub_nc_u32_e32 v0, v0, v1
+; GFX10-NEXT:    v_sub_nc_i32 v0, v0, v1 clamp
 ; GFX10-NEXT:    v_ashrrev_i32_e32 v0, 8, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call i24 @llvm.ssub.sat.i24(i24 %lhs, i24 %rhs)
@@ -1439,39 +1045,22 @@ define amdgpu_ps i24 @s_ssubsat_i24(i24 inreg %lhs, i24 inreg %rhs) {
 ;
 ; GFX9-LABEL: s_ssubsat_i24:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_lshl_b32 s0, s0, 8
 ; GFX9-NEXT:    s_lshl_b32 s1, s1, 8
-; GFX9-NEXT:    s_cmp_gt_i32 s0, -1
-; GFX9-NEXT:    s_cselect_b32 s2, s0, -1
-; GFX9-NEXT:    s_sub_i32 s2, s2, 0x7fffffff
-; GFX9-NEXT:    s_cmp_lt_i32 s0, -1
-; GFX9-NEXT:    s_cselect_b32 s3, s0, -1
-; GFX9-NEXT:    s_sub_i32 s3, s3, 0x80000000
-; GFX9-NEXT:    s_cmp_gt_i32 s2, s1
-; GFX9-NEXT:    s_cselect_b32 s1, s2, s1
-; GFX9-NEXT:    s_cmp_lt_i32 s1, s3
-; GFX9-NEXT:    s_cselect_b32 s1, s1, s3
-; GFX9-NEXT:    s_sub_i32 s0, s0, s1
-; GFX9-NEXT:    s_ashr_i32 s0, s0, 8
+; GFX9-NEXT:    s_lshl_b32 s0, s0, 8
+; GFX9-NEXT:    v_mov_b32_e32 v0, s1
+; GFX9-NEXT:    v_sub_i32 v0, s0, v0 clamp
+; GFX9-NEXT:    v_ashrrev_i32_e32 v0, 8, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_ssubsat_i24:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_lshl_b32 s0, s0, 8
 ; GFX10-NEXT:    s_lshl_b32 s1, s1, 8
-; GFX10-NEXT:    s_cmp_gt_i32 s0, -1
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cselect_b32 s2, s0, -1
-; GFX10-NEXT:    s_sub_i32 s2, s2, 0x7fffffff
-; GFX10-NEXT:    s_cmp_lt_i32 s0, -1
-; GFX10-NEXT:    s_cselect_b32 s3, s0, -1
-; GFX10-NEXT:    s_sub_i32 s3, s3, 0x80000000
-; GFX10-NEXT:    s_cmp_gt_i32 s2, s1
-; GFX10-NEXT:    s_cselect_b32 s1, s2, s1
-; GFX10-NEXT:    s_cmp_lt_i32 s1, s3
-; GFX10-NEXT:    s_cselect_b32 s1, s1, s3
-; GFX10-NEXT:    s_sub_i32 s0, s0, s1
-; GFX10-NEXT:    s_ashr_i32 s0, s0, 8
+; GFX10-NEXT:    v_sub_nc_i32 v0, s0, s1 clamp
+; GFX10-NEXT:    v_ashrrev_i32_e32 v0, 8, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call i24 @llvm.ssub.sat.i24(i24 %lhs, i24 %rhs)
   ret i24 %result
@@ -1505,27 +1094,15 @@ define i32 @v_ssubsat_i32(i32 %lhs, i32 %rhs) {
 ; GFX9-LABEL: v_ssubsat_i32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_max_i32_e32 v2, -1, v0
-; GFX9-NEXT:    v_min_i32_e32 v3, -1, v0
-; GFX9-NEXT:    v_subrev_u32_e32 v2, 0x7fffffff, v2
-; GFX9-NEXT:    v_subrev_u32_e32 v3, 0x80000000, v3
-; GFX9-NEXT:    v_max_i32_e32 v1, v2, v1
-; GFX9-NEXT:    v_min_i32_e32 v1, v1, v3
-; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v1
+; GFX9-NEXT:    v_sub_i32 v0, v0, v1 clamp
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_ssubsat_i32:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_max_i32_e32 v2, -1, v0
-; GFX10-NEXT:    v_min_i32_e32 v3, -1, v0
+; GFX10-NEXT:    v_sub_nc_i32 v0, v0, v1 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_subrev_nc_u32_e32 v2, 0x7fffffff, v2
-; GFX10-NEXT:    v_subrev_nc_u32_e32 v3, 0x80000000, v3
-; GFX10-NEXT:    v_max_i32_e32 v1, v2, v1
-; GFX10-NEXT:    v_min_i32_e32 v1, v1, v3
-; GFX10-NEXT:    v_sub_nc_u32_e32 v0, v0, v1
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call i32 @llvm.ssub.sat.i32(i32 %lhs, i32 %rhs)
   ret i32 %result
@@ -1564,33 +1141,16 @@ define amdgpu_ps i32 @s_ssubsat_i32(i32 inreg %lhs, i32 inreg %rhs) {
 ;
 ; GFX9-LABEL: s_ssubsat_i32:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_cmp_gt_i32 s0, -1
-; GFX9-NEXT:    s_cselect_b32 s2, s0, -1
-; GFX9-NEXT:    s_sub_i32 s2, s2, 0x7fffffff
-; GFX9-NEXT:    s_cmp_lt_i32 s0, -1
-; GFX9-NEXT:    s_cselect_b32 s3, s0, -1
-; GFX9-NEXT:    s_sub_i32 s3, s3, 0x80000000
-; GFX9-NEXT:    s_cmp_gt_i32 s2, s1
-; GFX9-NEXT:    s_cselect_b32 s1, s2, s1
-; GFX9-NEXT:    s_cmp_lt_i32 s1, s3
-; GFX9-NEXT:    s_cselect_b32 s1, s1, s3
-; GFX9-NEXT:    s_sub_i32 s0, s0, s1
+; GFX9-NEXT:    v_mov_b32_e32 v0, s1
+; GFX9-NEXT:    v_sub_i32 v0, s0, v0 clamp
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_ssubsat_i32:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_cmp_gt_i32 s0, -1
+; GFX10-NEXT:    v_sub_nc_i32 v0, s0, s1 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cselect_b32 s2, s0, -1
-; GFX10-NEXT:    s_sub_i32 s2, s2, 0x7fffffff
-; GFX10-NEXT:    s_cmp_lt_i32 s0, -1
-; GFX10-NEXT:    s_cselect_b32 s3, s0, -1
-; GFX10-NEXT:    s_sub_i32 s3, s3, 0x80000000
-; GFX10-NEXT:    s_cmp_gt_i32 s2, s1
-; GFX10-NEXT:    s_cselect_b32 s1, s2, s1
-; GFX10-NEXT:    s_cmp_lt_i32 s1, s3
-; GFX10-NEXT:    s_cselect_b32 s1, s1, s3
-; GFX10-NEXT:    s_sub_i32 s0, s0, s1
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call i32 @llvm.ssub.sat.i32(i32 %lhs, i32 %rhs)
   ret i32 %result
@@ -1625,29 +1185,13 @@ define amdgpu_ps float @ssubsat_i32_sv(i32 inreg %lhs, i32 %rhs) {
 ;
 ; GFX9-LABEL: ssubsat_i32_sv:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_cmp_gt_i32 s0, -1
-; GFX9-NEXT:    s_cselect_b32 s1, s0, -1
-; GFX9-NEXT:    s_sub_i32 s1, s1, 0x7fffffff
-; GFX9-NEXT:    s_cmp_lt_i32 s0, -1
-; GFX9-NEXT:    s_cselect_b32 s2, s0, -1
-; GFX9-NEXT:    s_sub_i32 s2, s2, 0x80000000
-; GFX9-NEXT:    v_max_i32_e32 v0, s1, v0
-; GFX9-NEXT:    v_min_i32_e32 v0, s2, v0
-; GFX9-NEXT:    v_sub_u32_e32 v0, s0, v0
+; GFX9-NEXT:    v_sub_i32 v0, s0, v0 clamp
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: ssubsat_i32_sv:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_cmp_gt_i32 s0, -1
+; GFX10-NEXT:    v_sub_nc_i32 v0, s0, v0 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cselect_b32 s1, s0, -1
-; GFX10-NEXT:    s_sub_i32 s1, s1, 0x7fffffff
-; GFX10-NEXT:    s_cmp_lt_i32 s0, -1
-; GFX10-NEXT:    v_max_i32_e32 v0, s1, v0
-; GFX10-NEXT:    s_cselect_b32 s1, s0, -1
-; GFX10-NEXT:    s_sub_i32 s1, s1, 0x80000000
-; GFX10-NEXT:    v_min_i32_e32 v0, s1, v0
-; GFX10-NEXT:    v_sub_nc_u32_e32 v0, s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call i32 @llvm.ssub.sat.i32(i32 %lhs, i32 %rhs)
   %cast = bitcast i32 %result to float
@@ -1679,25 +1223,13 @@ define amdgpu_ps float @ssubsat_i32_vs(i32 %lhs, i32 inreg %rhs) {
 ;
 ; GFX9-LABEL: ssubsat_i32_vs:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    v_max_i32_e32 v1, -1, v0
-; GFX9-NEXT:    v_subrev_u32_e32 v1, 0x7fffffff, v1
-; GFX9-NEXT:    v_min_i32_e32 v2, -1, v0
-; GFX9-NEXT:    v_subrev_u32_e32 v2, 0x80000000, v2
-; GFX9-NEXT:    v_max_i32_e32 v1, s0, v1
-; GFX9-NEXT:    v_min_i32_e32 v1, v1, v2
-; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v1
+; GFX9-NEXT:    v_sub_i32 v0, v0, s0 clamp
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: ssubsat_i32_vs:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    v_max_i32_e32 v1, -1, v0
-; GFX10-NEXT:    v_min_i32_e32 v2, -1, v0
+; GFX10-NEXT:    v_sub_nc_i32 v0, v0, s0 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_subrev_nc_u32_e32 v1, 0x7fffffff, v1
-; GFX10-NEXT:    v_subrev_nc_u32_e32 v2, 0x80000000, v2
-; GFX10-NEXT:    v_max_i32_e32 v1, s0, v1
-; GFX10-NEXT:    v_min_i32_e32 v1, v1, v2
-; GFX10-NEXT:    v_sub_nc_u32_e32 v0, v0, v1
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call i32 @llvm.ssub.sat.i32(i32 %lhs, i32 %rhs)
   %cast = bitcast i32 %result to float
@@ -1750,45 +1282,17 @@ define <2 x i32> @v_ssubsat_v2i32(<2 x i32> %lhs, <2 x i32> %rhs) {
 ; GFX9-LABEL: v_ssubsat_v2i32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    s_brev_b32 s4, -2
-; GFX9-NEXT:    v_max_i32_e32 v4, -1, v0
-; GFX9-NEXT:    v_subrev_u32_e32 v4, s4, v4
-; GFX9-NEXT:    s_brev_b32 s5, 1
-; GFX9-NEXT:    v_min_i32_e32 v5, -1, v0
-; GFX9-NEXT:    v_max_i32_e32 v2, v4, v2
-; GFX9-NEXT:    v_subrev_u32_e32 v5, s5, v5
-; GFX9-NEXT:    v_min_i32_e32 v2, v2, v5
-; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v2
-; GFX9-NEXT:    v_max_i32_e32 v2, -1, v1
-; GFX9-NEXT:    v_subrev_u32_e32 v2, s4, v2
-; GFX9-NEXT:    v_min_i32_e32 v4, -1, v1
-; GFX9-NEXT:    v_subrev_u32_e32 v4, s5, v4
-; GFX9-NEXT:    v_max_i32_e32 v2, v2, v3
-; GFX9-NEXT:    v_min_i32_e32 v2, v2, v4
-; GFX9-NEXT:    v_sub_u32_e32 v1, v1, v2
+; GFX9-NEXT:    v_sub_i32 v0, v0, v2 clamp
+; GFX9-NEXT:    v_sub_i32 v1, v1, v3 clamp
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_ssubsat_v2i32:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_max_i32_e32 v4, -1, v0
-; GFX10-NEXT:    v_max_i32_e32 v5, -1, v1
-; GFX10-NEXT:    s_brev_b32 s4, -2
-; GFX10-NEXT:    v_min_i32_e32 v6, -1, v0
-; GFX10-NEXT:    v_min_i32_e32 v7, -1, v1
-; GFX10-NEXT:    v_subrev_nc_u32_e32 v4, s4, v4
-; GFX10-NEXT:    v_subrev_nc_u32_e32 v5, s4, v5
-; GFX10-NEXT:    s_brev_b32 s4, 1
+; GFX10-NEXT:    v_sub_nc_i32 v0, v0, v2 clamp
+; GFX10-NEXT:    v_sub_nc_i32 v1, v1, v3 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_subrev_nc_u32_e32 v6, s4, v6
-; GFX10-NEXT:    v_max_i32_e32 v11, v4, v2
-; GFX10-NEXT:    v_subrev_nc_u32_e32 v7, s4, v7
-; GFX10-NEXT:    v_max_i32_e32 v10, v5, v3
-; GFX10-NEXT:    v_min_i32_e32 v2, v11, v6
-; GFX10-NEXT:    v_min_i32_e32 v3, v10, v7
-; GFX10-NEXT:    v_sub_nc_u32_e32 v0, v0, v2
-; GFX10-NEXT:    v_sub_nc_u32_e32 v1, v1, v3
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <2 x i32> @llvm.ssub.sat.v2i32(<2 x i32> %lhs, <2 x i32> %rhs)
   ret <2 x i32> %result
@@ -1853,59 +1357,21 @@ define amdgpu_ps <2 x i32> @s_ssubsat_v2i32(<2 x i32> inreg %lhs, <2 x i32> inre
 ;
 ; GFX9-LABEL: s_ssubsat_v2i32:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_cmp_gt_i32 s0, -1
-; GFX9-NEXT:    s_brev_b32 s4, -2
-; GFX9-NEXT:    s_cselect_b32 s6, s0, -1
-; GFX9-NEXT:    s_sub_i32 s6, s6, s4
-; GFX9-NEXT:    s_cmp_lt_i32 s0, -1
-; GFX9-NEXT:    s_brev_b32 s5, 1
-; GFX9-NEXT:    s_cselect_b32 s7, s0, -1
-; GFX9-NEXT:    s_sub_i32 s7, s7, s5
-; GFX9-NEXT:    s_cmp_gt_i32 s6, s2
-; GFX9-NEXT:    s_cselect_b32 s2, s6, s2
-; GFX9-NEXT:    s_cmp_lt_i32 s2, s7
-; GFX9-NEXT:    s_cselect_b32 s2, s2, s7
-; GFX9-NEXT:    s_sub_i32 s0, s0, s2
-; GFX9-NEXT:    s_cmp_gt_i32 s1, -1
-; GFX9-NEXT:    s_cselect_b32 s2, s1, -1
-; GFX9-NEXT:    s_sub_i32 s2, s2, s4
-; GFX9-NEXT:    s_cmp_lt_i32 s1, -1
-; GFX9-NEXT:    s_cselect_b32 s4, s1, -1
-; GFX9-NEXT:    s_sub_i32 s4, s4, s5
-; GFX9-NEXT:    s_cmp_gt_i32 s2, s3
-; GFX9-NEXT:    s_cselect_b32 s2, s2, s3
-; GFX9-NEXT:    s_cmp_lt_i32 s2, s4
-; GFX9-NEXT:    s_cselect_b32 s2, s2, s4
-; GFX9-NEXT:    s_sub_i32 s1, s1, s2
+; GFX9-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    v_sub_i32 v0, s0, v0 clamp
+; GFX9-NEXT:    v_sub_i32 v1, s1, v1 clamp
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_ssubsat_v2i32:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_cmp_gt_i32 s0, -1
-; GFX10-NEXT:    s_brev_b32 s4, -2
-; GFX10-NEXT:    s_cselect_b32 s5, s0, -1
-; GFX10-NEXT:    s_brev_b32 s6, 1
-; GFX10-NEXT:    s_sub_i32 s5, s5, s4
-; GFX10-NEXT:    s_cmp_lt_i32 s0, -1
+; GFX10-NEXT:    v_sub_nc_i32 v0, s0, s2 clamp
+; GFX10-NEXT:    v_sub_nc_i32 v1, s1, s3 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cselect_b32 s7, s0, -1
-; GFX10-NEXT:    s_sub_i32 s7, s7, s6
-; GFX10-NEXT:    s_cmp_gt_i32 s5, s2
-; GFX10-NEXT:    s_cselect_b32 s2, s5, s2
-; GFX10-NEXT:    s_cmp_lt_i32 s2, s7
-; GFX10-NEXT:    s_cselect_b32 s2, s2, s7
-; GFX10-NEXT:    s_sub_i32 s0, s0, s2
-; GFX10-NEXT:    s_cmp_gt_i32 s1, -1
-; GFX10-NEXT:    s_cselect_b32 s2, s1, -1
-; GFX10-NEXT:    s_sub_i32 s2, s2, s4
-; GFX10-NEXT:    s_cmp_lt_i32 s1, -1
-; GFX10-NEXT:    s_cselect_b32 s4, s1, -1
-; GFX10-NEXT:    s_sub_i32 s4, s4, s6
-; GFX10-NEXT:    s_cmp_gt_i32 s2, s3
-; GFX10-NEXT:    s_cselect_b32 s2, s2, s3
-; GFX10-NEXT:    s_cmp_lt_i32 s2, s4
-; GFX10-NEXT:    s_cselect_b32 s2, s2, s4
-; GFX10-NEXT:    s_sub_i32 s1, s1, s2
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <2 x i32> @llvm.ssub.sat.v2i32(<2 x i32> %lhs, <2 x i32> %rhs)
   ret <2 x i32> %result
@@ -1971,59 +1437,19 @@ define <3 x i32> @v_ssubsat_v3i32(<3 x i32> %lhs, <3 x i32> %rhs) {
 ; GFX9-LABEL: v_ssubsat_v3i32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    s_brev_b32 s4, -2
-; GFX9-NEXT:    v_max_i32_e32 v6, -1, v0
-; GFX9-NEXT:    v_subrev_u32_e32 v6, s4, v6
-; GFX9-NEXT:    s_brev_b32 s5, 1
-; GFX9-NEXT:    v_min_i32_e32 v7, -1, v0
-; GFX9-NEXT:    v_max_i32_e32 v3, v6, v3
-; GFX9-NEXT:    v_subrev_u32_e32 v7, s5, v7
-; GFX9-NEXT:    v_min_i32_e32 v3, v3, v7
-; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v3
-; GFX9-NEXT:    v_max_i32_e32 v3, -1, v1
-; GFX9-NEXT:    v_subrev_u32_e32 v3, s4, v3
-; GFX9-NEXT:    v_min_i32_e32 v6, -1, v1
-; GFX9-NEXT:    v_max_i32_e32 v3, v3, v4
-; GFX9-NEXT:    v_subrev_u32_e32 v6, s5, v6
-; GFX9-NEXT:    v_min_i32_e32 v3, v3, v6
-; GFX9-NEXT:    v_sub_u32_e32 v1, v1, v3
-; GFX9-NEXT:    v_max_i32_e32 v3, -1, v2
-; GFX9-NEXT:    v_subrev_u32_e32 v3, s4, v3
-; GFX9-NEXT:    v_min_i32_e32 v4, -1, v2
-; GFX9-NEXT:    v_subrev_u32_e32 v4, s5, v4
-; GFX9-NEXT:    v_max_i32_e32 v3, v3, v5
-; GFX9-NEXT:    v_min_i32_e32 v3, v3, v4
-; GFX9-NEXT:    v_sub_u32_e32 v2, v2, v3
+; GFX9-NEXT:    v_sub_i32 v0, v0, v3 clamp
+; GFX9-NEXT:    v_sub_i32 v1, v1, v4 clamp
+; GFX9-NEXT:    v_sub_i32 v2, v2, v5 clamp
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_ssubsat_v3i32:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_max_i32_e32 v6, -1, v0
-; GFX10-NEXT:    v_max_i32_e32 v8, -1, v1
-; GFX10-NEXT:    v_max_i32_e32 v9, -1, v2
-; GFX10-NEXT:    s_brev_b32 s4, -2
-; GFX10-NEXT:    v_min_i32_e32 v7, -1, v0
-; GFX10-NEXT:    v_subrev_nc_u32_e32 v6, s4, v6
-; GFX10-NEXT:    v_subrev_nc_u32_e32 v15, s4, v8
-; GFX10-NEXT:    v_subrev_nc_u32_e32 v19, s4, v9
-; GFX10-NEXT:    v_min_i32_e32 v10, -1, v1
-; GFX10-NEXT:    v_min_i32_e32 v11, -1, v2
-; GFX10-NEXT:    s_brev_b32 s5, 1
-; GFX10-NEXT:    v_max_i32_e32 v14, v6, v3
-; GFX10-NEXT:    v_subrev_nc_u32_e32 v7, s5, v7
-; GFX10-NEXT:    v_subrev_nc_u32_e32 v6, s5, v10
-; GFX10-NEXT:    v_max_i32_e32 v4, v15, v4
-; GFX10-NEXT:    v_subrev_nc_u32_e32 v8, s5, v11
-; GFX10-NEXT:    v_max_i32_e32 v5, v19, v5
-; GFX10-NEXT:    v_min_i32_e32 v3, v14, v7
+; GFX10-NEXT:    v_sub_nc_i32 v0, v0, v3 clamp
+; GFX10-NEXT:    v_sub_nc_i32 v1, v1, v4 clamp
+; GFX10-NEXT:    v_sub_nc_i32 v2, v2, v5 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_min_i32_e32 v4, v4, v6
-; GFX10-NEXT:    v_min_i32_e32 v5, v5, v8
-; GFX10-NEXT:    v_sub_nc_u32_e32 v0, v0, v3
-; GFX10-NEXT:    v_sub_nc_u32_e32 v1, v1, v4
-; GFX10-NEXT:    v_sub_nc_u32_e32 v2, v2, v5
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <3 x i32> @llvm.ssub.sat.v3i32(<3 x i32> %lhs, <3 x i32> %rhs)
   ret <3 x i32> %result
@@ -2110,81 +1536,26 @@ define amdgpu_ps <3 x i32> @s_ssubsat_v3i32(<3 x i32> inreg %lhs, <3 x i32> inre
 ;
 ; GFX9-LABEL: s_ssubsat_v3i32:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_cmp_gt_i32 s0, -1
-; GFX9-NEXT:    s_brev_b32 s6, -2
-; GFX9-NEXT:    s_cselect_b32 s8, s0, -1
-; GFX9-NEXT:    s_sub_i32 s8, s8, s6
-; GFX9-NEXT:    s_cmp_lt_i32 s0, -1
-; GFX9-NEXT:    s_brev_b32 s7, 1
-; GFX9-NEXT:    s_cselect_b32 s9, s0, -1
-; GFX9-NEXT:    s_sub_i32 s9, s9, s7
-; GFX9-NEXT:    s_cmp_gt_i32 s8, s3
-; GFX9-NEXT:    s_cselect_b32 s3, s8, s3
-; GFX9-NEXT:    s_cmp_lt_i32 s3, s9
-; GFX9-NEXT:    s_cselect_b32 s3, s3, s9
-; GFX9-NEXT:    s_sub_i32 s0, s0, s3
-; GFX9-NEXT:    s_cmp_gt_i32 s1, -1
-; GFX9-NEXT:    s_cselect_b32 s3, s1, -1
-; GFX9-NEXT:    s_sub_i32 s3, s3, s6
-; GFX9-NEXT:    s_cmp_lt_i32 s1, -1
-; GFX9-NEXT:    s_cselect_b32 s8, s1, -1
-; GFX9-NEXT:    s_sub_i32 s8, s8, s7
-; GFX9-NEXT:    s_cmp_gt_i32 s3, s4
-; GFX9-NEXT:    s_cselect_b32 s3, s3, s4
-; GFX9-NEXT:    s_cmp_lt_i32 s3, s8
-; GFX9-NEXT:    s_cselect_b32 s3, s3, s8
-; GFX9-NEXT:    s_sub_i32 s1, s1, s3
-; GFX9-NEXT:    s_cmp_gt_i32 s2, -1
-; GFX9-NEXT:    s_cselect_b32 s3, s2, -1
-; GFX9-NEXT:    s_sub_i32 s3, s3, s6
-; GFX9-NEXT:    s_cmp_lt_i32 s2, -1
-; GFX9-NEXT:    s_cselect_b32 s4, s2, -1
-; GFX9-NEXT:    s_sub_i32 s4, s4, s7
-; GFX9-NEXT:    s_cmp_gt_i32 s3, s5
-; GFX9-NEXT:    s_cselect_b32 s3, s3, s5
-; GFX9-NEXT:    s_cmp_lt_i32 s3, s4
-; GFX9-NEXT:    s_cselect_b32 s3, s3, s4
-; GFX9-NEXT:    s_sub_i32 s2, s2, s3
+; GFX9-NEXT:    v_mov_b32_e32 v0, s3
+; GFX9-NEXT:    v_mov_b32_e32 v1, s4
+; GFX9-NEXT:    v_mov_b32_e32 v2, s5
+; GFX9-NEXT:    v_sub_i32 v0, s0, v0 clamp
+; GFX9-NEXT:    v_sub_i32 v1, s1, v1 clamp
+; GFX9-NEXT:    v_sub_i32 v2, s2, v2 clamp
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v2
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_ssubsat_v3i32:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_cmp_gt_i32 s0, -1
-; GFX10-NEXT:    s_brev_b32 s6, -2
-; GFX10-NEXT:    s_cselect_b32 s7, s0, -1
-; GFX10-NEXT:    s_brev_b32 s8, 1
-; GFX10-NEXT:    s_sub_i32 s7, s7, s6
-; GFX10-NEXT:    s_cmp_lt_i32 s0, -1
+; GFX10-NEXT:    v_sub_nc_i32 v0, s0, s3 clamp
+; GFX10-NEXT:    v_sub_nc_i32 v1, s1, s4 clamp
+; GFX10-NEXT:    v_sub_nc_i32 v2, s2, s5 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cselect_b32 s9, s0, -1
-; GFX10-NEXT:    s_sub_i32 s9, s9, s8
-; GFX10-NEXT:    s_cmp_gt_i32 s7, s3
-; GFX10-NEXT:    s_cselect_b32 s3, s7, s3
-; GFX10-NEXT:    s_cmp_lt_i32 s3, s9
-; GFX10-NEXT:    s_cselect_b32 s3, s3, s9
-; GFX10-NEXT:    s_sub_i32 s0, s0, s3
-; GFX10-NEXT:    s_cmp_gt_i32 s1, -1
-; GFX10-NEXT:    s_cselect_b32 s3, s1, -1
-; GFX10-NEXT:    s_sub_i32 s3, s3, s6
-; GFX10-NEXT:    s_cmp_lt_i32 s1, -1
-; GFX10-NEXT:    s_cselect_b32 s7, s1, -1
-; GFX10-NEXT:    s_sub_i32 s7, s7, s8
-; GFX10-NEXT:    s_cmp_gt_i32 s3, s4
-; GFX10-NEXT:    s_cselect_b32 s3, s3, s4
-; GFX10-NEXT:    s_cmp_lt_i32 s3, s7
-; GFX10-NEXT:    s_cselect_b32 s3, s3, s7
-; GFX10-NEXT:    s_sub_i32 s1, s1, s3
-; GFX10-NEXT:    s_cmp_gt_i32 s2, -1
-; GFX10-NEXT:    s_cselect_b32 s3, s2, -1
-; GFX10-NEXT:    s_sub_i32 s3, s3, s6
-; GFX10-NEXT:    s_cmp_lt_i32 s2, -1
-; GFX10-NEXT:    s_cselect_b32 s4, s2, -1
-; GFX10-NEXT:    s_sub_i32 s4, s4, s8
-; GFX10-NEXT:    s_cmp_gt_i32 s3, s5
-; GFX10-NEXT:    s_cselect_b32 s3, s3, s5
-; GFX10-NEXT:    s_cmp_lt_i32 s3, s4
-; GFX10-NEXT:    s_cselect_b32 s3, s3, s4
-; GFX10-NEXT:    s_sub_i32 s2, s2, s3
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <3 x i32> @llvm.ssub.sat.v3i32(<3 x i32> %lhs, <3 x i32> %rhs)
   ret <3 x i32> %result
@@ -2264,73 +1635,21 @@ define <4 x i32> @v_ssubsat_v4i32(<4 x i32> %lhs, <4 x i32> %rhs) {
 ; GFX9-LABEL: v_ssubsat_v4i32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    s_brev_b32 s4, -2
-; GFX9-NEXT:    v_max_i32_e32 v8, -1, v0
-; GFX9-NEXT:    v_subrev_u32_e32 v8, s4, v8
-; GFX9-NEXT:    s_brev_b32 s5, 1
-; GFX9-NEXT:    v_min_i32_e32 v9, -1, v0
-; GFX9-NEXT:    v_max_i32_e32 v4, v8, v4
-; GFX9-NEXT:    v_subrev_u32_e32 v9, s5, v9
-; GFX9-NEXT:    v_min_i32_e32 v4, v4, v9
-; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v4
-; GFX9-NEXT:    v_max_i32_e32 v4, -1, v1
-; GFX9-NEXT:    v_subrev_u32_e32 v4, s4, v4
-; GFX9-NEXT:    v_min_i32_e32 v8, -1, v1
-; GFX9-NEXT:    v_max_i32_e32 v4, v4, v5
-; GFX9-NEXT:    v_subrev_u32_e32 v8, s5, v8
-; GFX9-NEXT:    v_min_i32_e32 v4, v4, v8
-; GFX9-NEXT:    v_sub_u32_e32 v1, v1, v4
-; GFX9-NEXT:    v_max_i32_e32 v4, -1, v2
-; GFX9-NEXT:    v_subrev_u32_e32 v4, s4, v4
-; GFX9-NEXT:    v_min_i32_e32 v5, -1, v2
-; GFX9-NEXT:    v_subrev_u32_e32 v5, s5, v5
-; GFX9-NEXT:    v_max_i32_e32 v4, v4, v6
-; GFX9-NEXT:    v_min_i32_e32 v4, v4, v5
-; GFX9-NEXT:    v_sub_u32_e32 v2, v2, v4
-; GFX9-NEXT:    v_max_i32_e32 v4, -1, v3
-; GFX9-NEXT:    v_subrev_u32_e32 v4, 0x7fffffff, v4
-; GFX9-NEXT:    v_min_i32_e32 v5, -1, v3
-; GFX9-NEXT:    v_subrev_u32_e32 v5, 0x80000000, v5
-; GFX9-NEXT:    v_max_i32_e32 v4, v4, v7
-; GFX9-NEXT:    v_min_i32_e32 v4, v4, v5
-; GFX9-NEXT:    v_sub_u32_e32 v3, v3, v4
+; GFX9-NEXT:    v_sub_i32 v0, v0, v4 clamp
+; GFX9-NEXT:    v_sub_i32 v1, v1, v5 clamp
+; GFX9-NEXT:    v_sub_i32 v2, v2, v6 clamp
+; GFX9-NEXT:    v_sub_i32 v3, v3, v7 clamp
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_ssubsat_v4i32:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_max_i32_e32 v8, -1, v0
-; GFX10-NEXT:    s_brev_b32 s4, -2
-; GFX10-NEXT:    v_max_i32_e32 v10, -1, v1
-; GFX10-NEXT:    v_max_i32_e32 v12, -1, v3
-; GFX10-NEXT:    v_min_i32_e32 v9, -1, v0
-; GFX10-NEXT:    v_subrev_nc_u32_e32 v15, s4, v8
-; GFX10-NEXT:    v_max_i32_e32 v8, -1, v2
-; GFX10-NEXT:    v_subrev_nc_u32_e32 v10, s4, v10
-; GFX10-NEXT:    v_subrev_nc_u32_e32 v12, 0x7fffffff, v12
-; GFX10-NEXT:    v_min_i32_e32 v11, -1, v1
-; GFX10-NEXT:    v_min_i32_e32 v13, -1, v2
-; GFX10-NEXT:    v_subrev_nc_u32_e32 v8, s4, v8
-; GFX10-NEXT:    v_min_i32_e32 v14, -1, v3
-; GFX10-NEXT:    s_brev_b32 s5, 1
-; GFX10-NEXT:    v_max_i32_e32 v4, v15, v4
-; GFX10-NEXT:    v_subrev_nc_u32_e32 v9, s5, v9
-; GFX10-NEXT:    v_max_i32_e32 v5, v10, v5
-; GFX10-NEXT:    v_subrev_nc_u32_e32 v11, s5, v11
-; GFX10-NEXT:    v_max_i32_e32 v15, v8, v6
-; GFX10-NEXT:    v_subrev_nc_u32_e32 v10, s5, v13
-; GFX10-NEXT:    v_subrev_nc_u32_e32 v8, 0x80000000, v14
-; GFX10-NEXT:    v_max_i32_e32 v7, v12, v7
-; GFX10-NEXT:    v_min_i32_e32 v19, v4, v9
-; GFX10-NEXT:    v_min_i32_e32 v11, v5, v11
-; GFX10-NEXT:    v_min_i32_e32 v15, v15, v10
+; GFX10-NEXT:    v_sub_nc_i32 v0, v0, v4 clamp
+; GFX10-NEXT:    v_sub_nc_i32 v1, v1, v5 clamp
+; GFX10-NEXT:    v_sub_nc_i32 v2, v2, v6 clamp
+; GFX10-NEXT:    v_sub_nc_i32 v3, v3, v7 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_min_i32_e32 v6, v7, v8
-; GFX10-NEXT:    v_sub_nc_u32_e32 v0, v0, v19
-; GFX10-NEXT:    v_sub_nc_u32_e32 v1, v1, v11
-; GFX10-NEXT:    v_sub_nc_u32_e32 v2, v2, v15
-; GFX10-NEXT:    v_sub_nc_u32_e32 v3, v3, v6
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> %lhs, <4 x i32> %rhs)
   ret <4 x i32> %result
@@ -2439,103 +1758,31 @@ define amdgpu_ps <4 x i32> @s_ssubsat_v4i32(<4 x i32> inreg %lhs, <4 x i32> inre
 ;
 ; GFX9-LABEL: s_ssubsat_v4i32:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_cmp_gt_i32 s0, -1
-; GFX9-NEXT:    s_brev_b32 s8, -2
-; GFX9-NEXT:    s_cselect_b32 s10, s0, -1
-; GFX9-NEXT:    s_sub_i32 s10, s10, s8
-; GFX9-NEXT:    s_cmp_lt_i32 s0, -1
-; GFX9-NEXT:    s_brev_b32 s9, 1
-; GFX9-NEXT:    s_cselect_b32 s11, s0, -1
-; GFX9-NEXT:    s_sub_i32 s11, s11, s9
-; GFX9-NEXT:    s_cmp_gt_i32 s10, s4
-; GFX9-NEXT:    s_cselect_b32 s4, s10, s4
-; GFX9-NEXT:    s_cmp_lt_i32 s4, s11
-; GFX9-NEXT:    s_cselect_b32 s4, s4, s11
-; GFX9-NEXT:    s_sub_i32 s0, s0, s4
-; GFX9-NEXT:    s_cmp_gt_i32 s1, -1
-; GFX9-NEXT:    s_cselect_b32 s4, s1, -1
-; GFX9-NEXT:    s_sub_i32 s4, s4, s8
-; GFX9-NEXT:    s_cmp_lt_i32 s1, -1
-; GFX9-NEXT:    s_cselect_b32 s10, s1, -1
-; GFX9-NEXT:    s_sub_i32 s10, s10, s9
-; GFX9-NEXT:    s_cmp_gt_i32 s4, s5
-; GFX9-NEXT:    s_cselect_b32 s4, s4, s5
-; GFX9-NEXT:    s_cmp_lt_i32 s4, s10
-; GFX9-NEXT:    s_cselect_b32 s4, s4, s10
-; GFX9-NEXT:    s_sub_i32 s1, s1, s4
-; GFX9-NEXT:    s_cmp_gt_i32 s2, -1
-; GFX9-NEXT:    s_cselect_b32 s4, s2, -1
-; GFX9-NEXT:    s_sub_i32 s4, s4, s8
-; GFX9-NEXT:    s_cmp_lt_i32 s2, -1
-; GFX9-NEXT:    s_cselect_b32 s5, s2, -1
-; GFX9-NEXT:    s_sub_i32 s5, s5, s9
-; GFX9-NEXT:    s_cmp_gt_i32 s4, s6
-; GFX9-NEXT:    s_cselect_b32 s4, s4, s6
-; GFX9-NEXT:    s_cmp_lt_i32 s4, s5
-; GFX9-NEXT:    s_cselect_b32 s4, s4, s5
-; GFX9-NEXT:    s_sub_i32 s2, s2, s4
-; GFX9-NEXT:    s_cmp_gt_i32 s3, -1
-; GFX9-NEXT:    s_cselect_b32 s4, s3, -1
-; GFX9-NEXT:    s_sub_i32 s4, s4, s8
-; GFX9-NEXT:    s_cmp_lt_i32 s3, -1
-; GFX9-NEXT:    s_cselect_b32 s5, s3, -1
-; GFX9-NEXT:    s_sub_i32 s5, s5, s9
-; GFX9-NEXT:    s_cmp_gt_i32 s4, s7
-; GFX9-NEXT:    s_cselect_b32 s4, s4, s7
-; GFX9-NEXT:    s_cmp_lt_i32 s4, s5
-; GFX9-NEXT:    s_cselect_b32 s4, s4, s5
-; GFX9-NEXT:    s_sub_i32 s3, s3, s4
+; GFX9-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-NEXT:    v_mov_b32_e32 v2, s6
+; GFX9-NEXT:    v_mov_b32_e32 v3, s7
+; GFX9-NEXT:    v_sub_i32 v0, s0, v0 clamp
+; GFX9-NEXT:    v_sub_i32 v1, s1, v1 clamp
+; GFX9-NEXT:    v_sub_i32 v2, s2, v2 clamp
+; GFX9-NEXT:    v_sub_i32 v3, s3, v3 clamp
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX9-NEXT:    v_readfirstlane_b32 s3, v3
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_ssubsat_v4i32:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_cmp_gt_i32 s0, -1
-; GFX10-NEXT:    s_brev_b32 s8, -2
-; GFX10-NEXT:    s_cselect_b32 s9, s0, -1
-; GFX10-NEXT:    s_brev_b32 s10, 1
-; GFX10-NEXT:    s_sub_i32 s9, s9, s8
-; GFX10-NEXT:    s_cmp_lt_i32 s0, -1
+; GFX10-NEXT:    v_sub_nc_i32 v0, s0, s4 clamp
+; GFX10-NEXT:    v_sub_nc_i32 v1, s1, s5 clamp
+; GFX10-NEXT:    v_sub_nc_i32 v2, s2, s6 clamp
+; GFX10-NEXT:    v_sub_nc_i32 v3, s3, s7 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cselect_b32 s11, s0, -1
-; GFX10-NEXT:    s_sub_i32 s11, s11, s10
-; GFX10-NEXT:    s_cmp_gt_i32 s9, s4
-; GFX10-NEXT:    s_cselect_b32 s4, s9, s4
-; GFX10-NEXT:    s_cmp_lt_i32 s4, s11
-; GFX10-NEXT:    s_cselect_b32 s4, s4, s11
-; GFX10-NEXT:    s_sub_i32 s0, s0, s4
-; GFX10-NEXT:    s_cmp_gt_i32 s1, -1
-; GFX10-NEXT:    s_cselect_b32 s4, s1, -1
-; GFX10-NEXT:    s_sub_i32 s4, s4, s8
-; GFX10-NEXT:    s_cmp_lt_i32 s1, -1
-; GFX10-NEXT:    s_cselect_b32 s9, s1, -1
-; GFX10-NEXT:    s_sub_i32 s9, s9, s10
-; GFX10-NEXT:    s_cmp_gt_i32 s4, s5
-; GFX10-NEXT:    s_cselect_b32 s4, s4, s5
-; GFX10-NEXT:    s_cmp_lt_i32 s4, s9
-; GFX10-NEXT:    s_cselect_b32 s4, s4, s9
-; GFX10-NEXT:    s_sub_i32 s1, s1, s4
-; GFX10-NEXT:    s_cmp_gt_i32 s2, -1
-; GFX10-NEXT:    s_cselect_b32 s4, s2, -1
-; GFX10-NEXT:    s_sub_i32 s4, s4, s8
-; GFX10-NEXT:    s_cmp_lt_i32 s2, -1
-; GFX10-NEXT:    s_cselect_b32 s5, s2, -1
-; GFX10-NEXT:    s_sub_i32 s5, s5, s10
-; GFX10-NEXT:    s_cmp_gt_i32 s4, s6
-; GFX10-NEXT:    s_cselect_b32 s4, s4, s6
-; GFX10-NEXT:    s_cmp_lt_i32 s4, s5
-; GFX10-NEXT:    s_cselect_b32 s4, s4, s5
-; GFX10-NEXT:    s_sub_i32 s2, s2, s4
-; GFX10-NEXT:    s_cmp_gt_i32 s3, -1
-; GFX10-NEXT:    s_cselect_b32 s4, s3, -1
-; GFX10-NEXT:    s_sub_i32 s4, s4, s8
-; GFX10-NEXT:    s_cmp_lt_i32 s3, -1
-; GFX10-NEXT:    s_cselect_b32 s5, s3, -1
-; GFX10-NEXT:    s_sub_i32 s5, s5, s10
-; GFX10-NEXT:    s_cmp_gt_i32 s4, s7
-; GFX10-NEXT:    s_cselect_b32 s4, s4, s7
-; GFX10-NEXT:    s_cmp_lt_i32 s4, s5
-; GFX10-NEXT:    s_cselect_b32 s4, s4, s5
-; GFX10-NEXT:    s_sub_i32 s3, s3, s4
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX10-NEXT:    v_readfirstlane_b32 s3, v3
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> %lhs, <4 x i32> %rhs)
   ret <4 x i32> %result
@@ -2633,90 +1880,22 @@ define <5 x i32> @v_ssubsat_v5i32(<5 x i32> %lhs, <5 x i32> %rhs) {
 ; GFX9-LABEL: v_ssubsat_v5i32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    s_brev_b32 s4, -2
-; GFX9-NEXT:    v_max_i32_e32 v10, -1, v0
-; GFX9-NEXT:    v_subrev_u32_e32 v10, s4, v10
-; GFX9-NEXT:    s_brev_b32 s5, 1
-; GFX9-NEXT:    v_min_i32_e32 v12, -1, v0
-; GFX9-NEXT:    v_max_i32_e32 v5, v10, v5
-; GFX9-NEXT:    v_subrev_u32_e32 v12, s5, v12
-; GFX9-NEXT:    v_min_i32_e32 v5, v5, v12
-; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v5
-; GFX9-NEXT:    v_max_i32_e32 v5, -1, v1
-; GFX9-NEXT:    v_subrev_u32_e32 v5, s4, v5
-; GFX9-NEXT:    v_min_i32_e32 v10, -1, v1
-; GFX9-NEXT:    v_max_i32_e32 v5, v5, v6
-; GFX9-NEXT:    v_subrev_u32_e32 v10, s5, v10
-; GFX9-NEXT:    v_min_i32_e32 v5, v5, v10
-; GFX9-NEXT:    v_sub_u32_e32 v1, v1, v5
-; GFX9-NEXT:    v_max_i32_e32 v5, -1, v2
-; GFX9-NEXT:    v_subrev_u32_e32 v5, s4, v5
-; GFX9-NEXT:    v_min_i32_e32 v6, -1, v2
-; GFX9-NEXT:    v_subrev_u32_e32 v6, s5, v6
-; GFX9-NEXT:    v_max_i32_e32 v5, v5, v7
-; GFX9-NEXT:    v_min_i32_e32 v5, v5, v6
-; GFX9-NEXT:    v_sub_u32_e32 v2, v2, v5
-; GFX9-NEXT:    v_bfrev_b32_e32 v11, -2
-; GFX9-NEXT:    v_max_i32_e32 v5, -1, v3
-; GFX9-NEXT:    v_sub_u32_e32 v5, v5, v11
-; GFX9-NEXT:    v_bfrev_b32_e32 v13, 1
-; GFX9-NEXT:    v_min_i32_e32 v6, -1, v3
-; GFX9-NEXT:    v_sub_u32_e32 v6, v6, v13
-; GFX9-NEXT:    v_max_i32_e32 v5, v5, v8
-; GFX9-NEXT:    v_min_i32_e32 v5, v5, v6
-; GFX9-NEXT:    v_sub_u32_e32 v3, v3, v5
-; GFX9-NEXT:    v_max_i32_e32 v5, -1, v4
-; GFX9-NEXT:    v_sub_u32_e32 v5, v5, v11
-; GFX9-NEXT:    v_min_i32_e32 v6, -1, v4
-; GFX9-NEXT:    v_sub_u32_e32 v6, v6, v13
-; GFX9-NEXT:    v_max_i32_e32 v5, v5, v9
-; GFX9-NEXT:    v_min_i32_e32 v5, v5, v6
-; GFX9-NEXT:    v_sub_u32_e32 v4, v4, v5
+; GFX9-NEXT:    v_sub_i32 v0, v0, v5 clamp
+; GFX9-NEXT:    v_sub_i32 v1, v1, v6 clamp
+; GFX9-NEXT:    v_sub_i32 v2, v2, v7 clamp
+; GFX9-NEXT:    v_sub_i32 v3, v3, v8 clamp
+; GFX9-NEXT:    v_sub_i32 v4, v4, v9 clamp
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_ssubsat_v5i32:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_max_i32_e32 v10, -1, v0
-; GFX10-NEXT:    v_max_i32_e32 v13, -1, v1
-; GFX10-NEXT:    s_brev_b32 s4, -2
-; GFX10-NEXT:    v_bfrev_b32_e32 v11, -2
-; GFX10-NEXT:    v_max_i32_e32 v17, -1, v4
-; GFX10-NEXT:    v_subrev_nc_u32_e32 v10, s4, v10
-; GFX10-NEXT:    v_subrev_nc_u32_e32 v13, s4, v13
-; GFX10-NEXT:    v_min_i32_e32 v12, -1, v0
-; GFX10-NEXT:    v_bfrev_b32_e32 v14, 1
-; GFX10-NEXT:    v_min_i32_e32 v15, -1, v1
-; GFX10-NEXT:    v_max_i32_e32 v5, v10, v5
-; GFX10-NEXT:    v_max_i32_e32 v10, -1, v2
-; GFX10-NEXT:    v_max_i32_e32 v6, v13, v6
-; GFX10-NEXT:    v_max_i32_e32 v13, -1, v3
-; GFX10-NEXT:    v_min_i32_e32 v16, -1, v2
-; GFX10-NEXT:    v_min_i32_e32 v23, -1, v3
-; GFX10-NEXT:    v_subrev_nc_u32_e32 v10, s4, v10
-; GFX10-NEXT:    v_min_i32_e32 v19, -1, v4
-; GFX10-NEXT:    v_sub_nc_u32_e32 v13, v13, v11
-; GFX10-NEXT:    v_sub_nc_u32_e32 v11, v17, v11
-; GFX10-NEXT:    s_brev_b32 s5, 1
-; GFX10-NEXT:    v_max_i32_e32 v7, v10, v7
-; GFX10-NEXT:    v_subrev_nc_u32_e32 v12, s5, v12
-; GFX10-NEXT:    v_subrev_nc_u32_e32 v15, s5, v15
-; GFX10-NEXT:    v_subrev_nc_u32_e32 v16, s5, v16
-; GFX10-NEXT:    v_max_i32_e32 v8, v13, v8
-; GFX10-NEXT:    v_sub_nc_u32_e32 v10, v23, v14
-; GFX10-NEXT:    v_sub_nc_u32_e32 v13, v19, v14
-; GFX10-NEXT:    v_max_i32_e32 v11, v11, v9
-; GFX10-NEXT:    v_min_i32_e32 v5, v5, v12
-; GFX10-NEXT:    v_min_i32_e32 v6, v6, v15
-; GFX10-NEXT:    v_min_i32_e32 v7, v7, v16
-; GFX10-NEXT:    v_min_i32_e32 v8, v8, v10
-; GFX10-NEXT:    v_min_i32_e32 v9, v11, v13
-; GFX10-NEXT:    v_sub_nc_u32_e32 v0, v0, v5
-; GFX10-NEXT:    v_sub_nc_u32_e32 v1, v1, v6
-; GFX10-NEXT:    v_sub_nc_u32_e32 v2, v2, v7
-; GFX10-NEXT:    v_sub_nc_u32_e32 v3, v3, v8
-; GFX10-NEXT:    v_sub_nc_u32_e32 v4, v4, v9
+; GFX10-NEXT:    v_sub_nc_i32 v0, v0, v5 clamp
+; GFX10-NEXT:    v_sub_nc_i32 v1, v1, v6 clamp
+; GFX10-NEXT:    v_sub_nc_i32 v2, v2, v7 clamp
+; GFX10-NEXT:    v_sub_nc_i32 v3, v3, v8 clamp
+; GFX10-NEXT:    v_sub_nc_i32 v4, v4, v9 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <5 x i32> @llvm.ssub.sat.v5i32(<5 x i32> %lhs, <5 x i32> %rhs)
@@ -2848,125 +2027,36 @@ define amdgpu_ps <5 x i32> @s_ssubsat_v5i32(<5 x i32> inreg %lhs, <5 x i32> inre
 ;
 ; GFX9-LABEL: s_ssubsat_v5i32:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_cmp_gt_i32 s0, -1
-; GFX9-NEXT:    s_brev_b32 s10, -2
-; GFX9-NEXT:    s_cselect_b32 s12, s0, -1
-; GFX9-NEXT:    s_sub_i32 s12, s12, s10
-; GFX9-NEXT:    s_cmp_lt_i32 s0, -1
-; GFX9-NEXT:    s_brev_b32 s11, 1
-; GFX9-NEXT:    s_cselect_b32 s13, s0, -1
-; GFX9-NEXT:    s_sub_i32 s13, s13, s11
-; GFX9-NEXT:    s_cmp_gt_i32 s12, s5
-; GFX9-NEXT:    s_cselect_b32 s5, s12, s5
-; GFX9-NEXT:    s_cmp_lt_i32 s5, s13
-; GFX9-NEXT:    s_cselect_b32 s5, s5, s13
-; GFX9-NEXT:    s_sub_i32 s0, s0, s5
-; GFX9-NEXT:    s_cmp_gt_i32 s1, -1
-; GFX9-NEXT:    s_cselect_b32 s5, s1, -1
-; GFX9-NEXT:    s_sub_i32 s5, s5, s10
-; GFX9-NEXT:    s_cmp_lt_i32 s1, -1
-; GFX9-NEXT:    s_cselect_b32 s12, s1, -1
-; GFX9-NEXT:    s_sub_i32 s12, s12, s11
-; GFX9-NEXT:    s_cmp_gt_i32 s5, s6
-; GFX9-NEXT:    s_cselect_b32 s5, s5, s6
-; GFX9-NEXT:    s_cmp_lt_i32 s5, s12
-; GFX9-NEXT:    s_cselect_b32 s5, s5, s12
-; GFX9-NEXT:    s_sub_i32 s1, s1, s5
-; GFX9-NEXT:    s_cmp_gt_i32 s2, -1
-; GFX9-NEXT:    s_cselect_b32 s5, s2, -1
-; GFX9-NEXT:    s_sub_i32 s5, s5, s10
-; GFX9-NEXT:    s_cmp_lt_i32 s2, -1
-; GFX9-NEXT:    s_cselect_b32 s6, s2, -1
-; GFX9-NEXT:    s_sub_i32 s6, s6, s11
-; GFX9-NEXT:    s_cmp_gt_i32 s5, s7
-; GFX9-NEXT:    s_cselect_b32 s5, s5, s7
-; GFX9-NEXT:    s_cmp_lt_i32 s5, s6
-; GFX9-NEXT:    s_cselect_b32 s5, s5, s6
-; GFX9-NEXT:    s_sub_i32 s2, s2, s5
-; GFX9-NEXT:    s_cmp_gt_i32 s3, -1
-; GFX9-NEXT:    s_cselect_b32 s5, s3, -1
-; GFX9-NEXT:    s_sub_i32 s5, s5, s10
-; GFX9-NEXT:    s_cmp_lt_i32 s3, -1
-; GFX9-NEXT:    s_cselect_b32 s6, s3, -1
-; GFX9-NEXT:    s_sub_i32 s6, s6, s11
-; GFX9-NEXT:    s_cmp_gt_i32 s5, s8
-; GFX9-NEXT:    s_cselect_b32 s5, s5, s8
-; GFX9-NEXT:    s_cmp_lt_i32 s5, s6
-; GFX9-NEXT:    s_cselect_b32 s5, s5, s6
-; GFX9-NEXT:    s_sub_i32 s3, s3, s5
-; GFX9-NEXT:    s_cmp_gt_i32 s4, -1
-; GFX9-NEXT:    s_cselect_b32 s5, s4, -1
-; GFX9-NEXT:    s_sub_i32 s5, s5, s10
-; GFX9-NEXT:    s_cmp_lt_i32 s4, -1
-; GFX9-NEXT:    s_cselect_b32 s6, s4, -1
-; GFX9-NEXT:    s_sub_i32 s6, s6, s11
-; GFX9-NEXT:    s_cmp_gt_i32 s5, s9
-; GFX9-NEXT:    s_cselect_b32 s5, s5, s9
-; GFX9-NEXT:    s_cmp_lt_i32 s5, s6
-; GFX9-NEXT:    s_cselect_b32 s5, s5, s6
-; GFX9-NEXT:    s_sub_i32 s4, s4, s5
+; GFX9-NEXT:    v_mov_b32_e32 v0, s5
+; GFX9-NEXT:    v_mov_b32_e32 v1, s6
+; GFX9-NEXT:    v_mov_b32_e32 v2, s7
+; GFX9-NEXT:    v_mov_b32_e32 v3, s8
+; GFX9-NEXT:    v_mov_b32_e32 v4, s9
+; GFX9-NEXT:    v_sub_i32 v0, s0, v0 clamp
+; GFX9-NEXT:    v_sub_i32 v1, s1, v1 clamp
+; GFX9-NEXT:    v_sub_i32 v2, s2, v2 clamp
+; GFX9-NEXT:    v_sub_i32 v3, s3, v3 clamp
+; GFX9-NEXT:    v_sub_i32 v4, s4, v4 clamp
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX9-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX9-NEXT:    v_readfirstlane_b32 s4, v4
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_ssubsat_v5i32:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_cmp_gt_i32 s0, -1
-; GFX10-NEXT:    s_brev_b32 s10, -2
-; GFX10-NEXT:    s_cselect_b32 s11, s0, -1
-; GFX10-NEXT:    s_brev_b32 s12, 1
-; GFX10-NEXT:    s_sub_i32 s11, s11, s10
-; GFX10-NEXT:    s_cmp_lt_i32 s0, -1
+; GFX10-NEXT:    v_sub_nc_i32 v0, s0, s5 clamp
+; GFX10-NEXT:    v_sub_nc_i32 v1, s1, s6 clamp
+; GFX10-NEXT:    v_sub_nc_i32 v2, s2, s7 clamp
+; GFX10-NEXT:    v_sub_nc_i32 v3, s3, s8 clamp
+; GFX10-NEXT:    v_sub_nc_i32 v4, s4, s9 clamp
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX10-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX10-NEXT:    v_readfirstlane_b32 s4, v4
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cselect_b32 s13, s0, -1
-; GFX10-NEXT:    s_sub_i32 s13, s13, s12
-; GFX10-NEXT:    s_cmp_gt_i32 s11, s5
-; GFX10-NEXT:    s_cselect_b32 s5, s11, s5
-; GFX10-NEXT:    s_cmp_lt_i32 s5, s13
-; GFX10-NEXT:    s_cselect_b32 s5, s5, s13
-; GFX10-NEXT:    s_sub_i32 s0, s0, s5
-; GFX10-NEXT:    s_cmp_gt_i32 s1, -1
-; GFX10-NEXT:    s_cselect_b32 s5, s1, -1
-; GFX10-NEXT:    s_sub_i32 s5, s5, s10
-; GFX10-NEXT:    s_cmp_lt_i32 s1, -1
-; GFX10-NEXT:    s_cselect_b32 s11, s1, -1
-; GFX10-NEXT:    s_sub_i32 s11, s11, s12
-; GFX10-NEXT:    s_cmp_gt_i32 s5, s6
-; GFX10-NEXT:    s_cselect_b32 s5, s5, s6
-; GFX10-NEXT:    s_cmp_lt_i32 s5, s11
-; GFX10-NEXT:    s_cselect_b32 s5, s5, s11
-; GFX10-NEXT:    s_sub_i32 s1, s1, s5
-; GFX10-NEXT:    s_cmp_gt_i32 s2, -1
-; GFX10-NEXT:    s_cselect_b32 s5, s2, -1
-; GFX10-NEXT:    s_sub_i32 s5, s5, s10
-; GFX10-NEXT:    s_cmp_lt_i32 s2, -1
-; GFX10-NEXT:    s_cselect_b32 s6, s2, -1
-; GFX10-NEXT:    s_sub_i32 s6, s6, s12
-; GFX10-NEXT:    s_cmp_gt_i32 s5, s7
-; GFX10-NEXT:    s_cselect_b32 s5, s5, s7
-; GFX10-NEXT:    s_cmp_lt_i32 s5, s6
-; GFX10-NEXT:    s_cselect_b32 s5, s5, s6
-; GFX10-NEXT:    s_sub_i32 s2, s2, s5
-; GFX10-NEXT:    s_cmp_gt_i32 s3, -1
-; GFX10-NEXT:    s_cselect_b32 s5, s3, -1
-; GFX10-NEXT:    s_sub_i32 s5, s5, s10
-; GFX10-NEXT:    s_cmp_lt_i32 s3, -1
-; GFX10-NEXT:    s_cselect_b32 s6, s3, -1
-; GFX10-NEXT:    s_sub_i32 s6, s6, s12
-; GFX10-NEXT:    s_cmp_gt_i32 s5, s8
-; GFX10-NEXT:    s_cselect_b32 s5, s5, s8
-; GFX10-NEXT:    s_cmp_lt_i32 s5, s6
-; GFX10-NEXT:    s_cselect_b32 s5, s5, s6
-; GFX10-NEXT:    s_sub_i32 s3, s3, s5
-; GFX10-NEXT:    s_cmp_gt_i32 s4, -1
-; GFX10-NEXT:    s_cselect_b32 s5, s4, -1
-; GFX10-NEXT:    s_sub_i32 s5, s5, s10
-; GFX10-NEXT:    s_cmp_lt_i32 s4, -1
-; GFX10-NEXT:    s_cselect_b32 s6, s4, -1
-; GFX10-NEXT:    s_sub_i32 s6, s6, s12
-; GFX10-NEXT:    s_cmp_gt_i32 s5, s9
-; GFX10-NEXT:    s_cselect_b32 s5, s5, s9
-; GFX10-NEXT:    s_cmp_lt_i32 s5, s6
-; GFX10-NEXT:    s_cselect_b32 s5, s5, s6
-; GFX10-NEXT:    s_sub_i32 s4, s4, s5
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <5 x i32> @llvm.ssub.sat.v5i32(<5 x i32> %lhs, <5 x i32> %rhs)
   ret <5 x i32> %result
@@ -3218,244 +2308,44 @@ define <16 x i32> @v_ssubsat_v16i32(<16 x i32> %lhs, <16 x i32> %rhs) {
 ; GFX9-LABEL: v_ssubsat_v16i32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    s_brev_b32 s4, -2
-; GFX9-NEXT:    v_max_i32_e32 v32, -1, v0
-; GFX9-NEXT:    v_subrev_u32_e32 v32, s4, v32
-; GFX9-NEXT:    v_max_i32_e32 v16, v32, v16
-; GFX9-NEXT:    s_brev_b32 s5, 1
-; GFX9-NEXT:    v_min_i32_e32 v32, -1, v0
-; GFX9-NEXT:    v_subrev_u32_e32 v32, s5, v32
-; GFX9-NEXT:    v_min_i32_e32 v16, v16, v32
-; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v16
-; GFX9-NEXT:    v_max_i32_e32 v16, -1, v1
-; GFX9-NEXT:    v_subrev_u32_e32 v16, s4, v16
-; GFX9-NEXT:    v_max_i32_e32 v16, v16, v17
-; GFX9-NEXT:    v_min_i32_e32 v17, -1, v1
-; GFX9-NEXT:    v_subrev_u32_e32 v17, s5, v17
-; GFX9-NEXT:    v_min_i32_e32 v16, v16, v17
-; GFX9-NEXT:    v_sub_u32_e32 v1, v1, v16
-; GFX9-NEXT:    v_max_i32_e32 v16, -1, v2
-; GFX9-NEXT:    v_subrev_u32_e32 v16, s4, v16
-; GFX9-NEXT:    v_min_i32_e32 v17, -1, v2
-; GFX9-NEXT:    v_max_i32_e32 v16, v16, v18
-; GFX9-NEXT:    v_subrev_u32_e32 v17, s5, v17
-; GFX9-NEXT:    v_min_i32_e32 v16, v16, v17
-; GFX9-NEXT:    v_sub_u32_e32 v2, v2, v16
-; GFX9-NEXT:    v_bfrev_b32_e32 v16, -2
-; GFX9-NEXT:    v_max_i32_e32 v17, -1, v3
-; GFX9-NEXT:    v_sub_u32_e32 v17, v17, v16
-; GFX9-NEXT:    v_max_i32_e32 v17, v17, v19
-; GFX9-NEXT:    v_bfrev_b32_e32 v18, 1
-; GFX9-NEXT:    v_min_i32_e32 v19, -1, v3
-; GFX9-NEXT:    v_sub_u32_e32 v19, v19, v18
-; GFX9-NEXT:    v_min_i32_e32 v17, v17, v19
-; GFX9-NEXT:    v_sub_u32_e32 v3, v3, v17
-; GFX9-NEXT:    v_max_i32_e32 v17, -1, v4
-; GFX9-NEXT:    v_sub_u32_e32 v17, v17, v16
-; GFX9-NEXT:    v_min_i32_e32 v19, -1, v4
-; GFX9-NEXT:    v_max_i32_e32 v17, v17, v20
-; GFX9-NEXT:    v_sub_u32_e32 v19, v19, v18
-; GFX9-NEXT:    v_min_i32_e32 v17, v17, v19
-; GFX9-NEXT:    v_sub_u32_e32 v4, v4, v17
-; GFX9-NEXT:    v_max_i32_e32 v17, -1, v5
-; GFX9-NEXT:    v_sub_u32_e32 v17, v17, v16
-; GFX9-NEXT:    v_min_i32_e32 v19, -1, v5
-; GFX9-NEXT:    v_max_i32_e32 v17, v17, v21
-; GFX9-NEXT:    v_sub_u32_e32 v19, v19, v18
-; GFX9-NEXT:    v_min_i32_e32 v17, v17, v19
-; GFX9-NEXT:    v_sub_u32_e32 v5, v5, v17
-; GFX9-NEXT:    v_max_i32_e32 v17, -1, v6
-; GFX9-NEXT:    v_sub_u32_e32 v17, v17, v16
-; GFX9-NEXT:    v_min_i32_e32 v19, -1, v6
-; GFX9-NEXT:    v_max_i32_e32 v17, v17, v22
-; GFX9-NEXT:    v_sub_u32_e32 v19, v19, v18
-; GFX9-NEXT:    v_min_i32_e32 v17, v17, v19
-; GFX9-NEXT:    v_sub_u32_e32 v6, v6, v17
-; GFX9-NEXT:    v_max_i32_e32 v17, -1, v7
-; GFX9-NEXT:    v_sub_u32_e32 v17, v17, v16
-; GFX9-NEXT:    v_min_i32_e32 v19, -1, v7
-; GFX9-NEXT:    v_max_i32_e32 v17, v17, v23
-; GFX9-NEXT:    v_sub_u32_e32 v19, v19, v18
-; GFX9-NEXT:    v_min_i32_e32 v17, v17, v19
-; GFX9-NEXT:    v_sub_u32_e32 v7, v7, v17
-; GFX9-NEXT:    v_max_i32_e32 v17, -1, v8
-; GFX9-NEXT:    v_sub_u32_e32 v17, v17, v16
-; GFX9-NEXT:    v_min_i32_e32 v19, -1, v8
-; GFX9-NEXT:    v_max_i32_e32 v17, v17, v24
-; GFX9-NEXT:    v_sub_u32_e32 v19, v19, v18
-; GFX9-NEXT:    v_min_i32_e32 v17, v17, v19
-; GFX9-NEXT:    v_sub_u32_e32 v8, v8, v17
-; GFX9-NEXT:    v_max_i32_e32 v17, -1, v9
-; GFX9-NEXT:    v_sub_u32_e32 v17, v17, v16
-; GFX9-NEXT:    v_min_i32_e32 v19, -1, v9
-; GFX9-NEXT:    v_max_i32_e32 v17, v17, v25
-; GFX9-NEXT:    v_sub_u32_e32 v19, v19, v18
-; GFX9-NEXT:    v_min_i32_e32 v17, v17, v19
-; GFX9-NEXT:    v_sub_u32_e32 v9, v9, v17
-; GFX9-NEXT:    v_max_i32_e32 v17, -1, v10
-; GFX9-NEXT:    v_sub_u32_e32 v17, v17, v16
-; GFX9-NEXT:    v_min_i32_e32 v19, -1, v10
-; GFX9-NEXT:    v_max_i32_e32 v17, v17, v26
-; GFX9-NEXT:    v_sub_u32_e32 v19, v19, v18
-; GFX9-NEXT:    v_min_i32_e32 v17, v17, v19
-; GFX9-NEXT:    v_sub_u32_e32 v10, v10, v17
-; GFX9-NEXT:    v_max_i32_e32 v17, -1, v11
-; GFX9-NEXT:    v_sub_u32_e32 v17, v17, v16
-; GFX9-NEXT:    v_min_i32_e32 v19, -1, v11
-; GFX9-NEXT:    v_max_i32_e32 v17, v17, v27
-; GFX9-NEXT:    v_sub_u32_e32 v19, v19, v18
-; GFX9-NEXT:    v_min_i32_e32 v17, v17, v19
-; GFX9-NEXT:    v_sub_u32_e32 v11, v11, v17
-; GFX9-NEXT:    v_max_i32_e32 v17, -1, v12
-; GFX9-NEXT:    v_sub_u32_e32 v17, v17, v16
-; GFX9-NEXT:    v_min_i32_e32 v19, -1, v12
-; GFX9-NEXT:    v_max_i32_e32 v17, v17, v28
-; GFX9-NEXT:    v_sub_u32_e32 v19, v19, v18
-; GFX9-NEXT:    v_min_i32_e32 v17, v17, v19
-; GFX9-NEXT:    v_sub_u32_e32 v12, v12, v17
-; GFX9-NEXT:    v_max_i32_e32 v17, -1, v13
-; GFX9-NEXT:    v_sub_u32_e32 v17, v17, v16
-; GFX9-NEXT:    v_min_i32_e32 v19, -1, v13
-; GFX9-NEXT:    v_max_i32_e32 v17, v17, v29
-; GFX9-NEXT:    v_sub_u32_e32 v19, v19, v18
-; GFX9-NEXT:    v_min_i32_e32 v17, v17, v19
-; GFX9-NEXT:    v_sub_u32_e32 v13, v13, v17
-; GFX9-NEXT:    v_max_i32_e32 v17, -1, v14
-; GFX9-NEXT:    v_sub_u32_e32 v17, v17, v16
-; GFX9-NEXT:    v_min_i32_e32 v19, -1, v14
-; GFX9-NEXT:    v_max_i32_e32 v17, v17, v30
-; GFX9-NEXT:    v_sub_u32_e32 v19, v19, v18
-; GFX9-NEXT:    v_min_i32_e32 v17, v17, v19
-; GFX9-NEXT:    v_sub_u32_e32 v14, v14, v17
-; GFX9-NEXT:    v_max_i32_e32 v17, -1, v15
-; GFX9-NEXT:    v_sub_u32_e32 v16, v17, v16
-; GFX9-NEXT:    v_min_i32_e32 v17, -1, v15
-; GFX9-NEXT:    v_sub_u32_e32 v17, v17, v18
-; GFX9-NEXT:    v_max_i32_e32 v16, v16, v31
-; GFX9-NEXT:    v_min_i32_e32 v16, v16, v17
-; GFX9-NEXT:    v_sub_u32_e32 v15, v15, v16
+; GFX9-NEXT:    v_sub_i32 v0, v0, v16 clamp
+; GFX9-NEXT:    v_sub_i32 v1, v1, v17 clamp
+; GFX9-NEXT:    v_sub_i32 v2, v2, v18 clamp
+; GFX9-NEXT:    v_sub_i32 v3, v3, v19 clamp
+; GFX9-NEXT:    v_sub_i32 v4, v4, v20 clamp
+; GFX9-NEXT:    v_sub_i32 v5, v5, v21 clamp
+; GFX9-NEXT:    v_sub_i32 v6, v6, v22 clamp
+; GFX9-NEXT:    v_sub_i32 v7, v7, v23 clamp
+; GFX9-NEXT:    v_sub_i32 v8, v8, v24 clamp
+; GFX9-NEXT:    v_sub_i32 v9, v9, v25 clamp
+; GFX9-NEXT:    v_sub_i32 v10, v10, v26 clamp
+; GFX9-NEXT:    v_sub_i32 v11, v11, v27 clamp
+; GFX9-NEXT:    v_sub_i32 v12, v12, v28 clamp
+; GFX9-NEXT:    v_sub_i32 v13, v13, v29 clamp
+; GFX9-NEXT:    v_sub_i32 v14, v14, v30 clamp
+; GFX9-NEXT:    v_sub_i32 v15, v15, v31 clamp
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_ssubsat_v16i32:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_max_i32_e32 v32, -1, v0
-; GFX10-NEXT:    s_brev_b32 s4, -2
-; GFX10-NEXT:    v_min_i32_e32 v33, -1, v0
-; GFX10-NEXT:    s_brev_b32 s5, 1
-; GFX10-NEXT:    v_max_i32_e32 v36, -1, v2
-; GFX10-NEXT:    v_subrev_nc_u32_e32 v35, s4, v32
-; GFX10-NEXT:    v_max_i32_e32 v32, -1, v1
-; GFX10-NEXT:    v_subrev_nc_u32_e32 v33, s5, v33
-; GFX10-NEXT:    v_bfrev_b32_e32 v34, -2
-; GFX10-NEXT:    v_subrev_nc_u32_e32 v36, s4, v36
-; GFX10-NEXT:    v_max_i32_e32 v16, v35, v16
-; GFX10-NEXT:    v_subrev_nc_u32_e32 v32, s4, v32
-; GFX10-NEXT:    v_max_i32_e32 v39, -1, v3
-; GFX10-NEXT:    v_min_i32_e32 v37, -1, v1
-; GFX10-NEXT:    v_max_i32_e32 v18, v36, v18
-; GFX10-NEXT:    v_min_i32_e32 v16, v16, v33
-; GFX10-NEXT:    v_min_i32_e32 v33, -1, v2
-; GFX10-NEXT:    v_max_i32_e32 v38, v32, v17
-; GFX10-NEXT:    v_max_i32_e32 v17, -1, v4
-; GFX10-NEXT:    v_sub_nc_u32_e32 v36, v39, v34
-; GFX10-NEXT:    v_bfrev_b32_e32 v35, 1
-; GFX10-NEXT:    v_subrev_nc_u32_e32 v32, s5, v33
-; GFX10-NEXT:    v_sub_nc_u32_e32 v0, v0, v16
-; GFX10-NEXT:    v_min_i32_e32 v33, -1, v3
-; GFX10-NEXT:    v_sub_nc_u32_e32 v17, v17, v34
-; GFX10-NEXT:    v_subrev_nc_u32_e32 v37, s5, v37
-; GFX10-NEXT:    v_min_i32_e32 v16, v18, v32
-; GFX10-NEXT:    v_max_i32_e32 v19, v36, v19
-; GFX10-NEXT:    v_sub_nc_u32_e32 v18, v33, v35
-; GFX10-NEXT:    v_max_i32_e32 v17, v17, v20
-; GFX10-NEXT:    v_min_i32_e32 v39, v38, v37
-; GFX10-NEXT:    v_sub_nc_u32_e32 v2, v2, v16
-; GFX10-NEXT:    v_min_i32_e32 v16, -1, v4
-; GFX10-NEXT:    v_min_i32_e32 v18, v19, v18
-; GFX10-NEXT:    v_max_i32_e32 v19, -1, v5
-; GFX10-NEXT:    v_max_i32_e32 v32, -1, v6
-; GFX10-NEXT:    v_min_i32_e32 v33, -1, v5
-; GFX10-NEXT:    v_sub_nc_u32_e32 v16, v16, v35
-; GFX10-NEXT:    v_sub_nc_u32_e32 v1, v1, v39
-; GFX10-NEXT:    v_sub_nc_u32_e32 v19, v19, v34
-; GFX10-NEXT:    v_sub_nc_u32_e32 v32, v32, v34
-; GFX10-NEXT:    v_min_i32_e32 v36, -1, v6
-; GFX10-NEXT:    v_min_i32_e32 v39, v17, v16
-; GFX10-NEXT:    v_max_i32_e32 v17, -1, v7
-; GFX10-NEXT:    v_min_i32_e32 v16, -1, v7
-; GFX10-NEXT:    v_max_i32_e32 v19, v19, v21
-; GFX10-NEXT:    v_sub_nc_u32_e32 v20, v33, v35
-; GFX10-NEXT:    v_sub_nc_u32_e32 v4, v4, v39
-; GFX10-NEXT:    v_sub_nc_u32_e32 v17, v17, v34
-; GFX10-NEXT:    v_sub_nc_u32_e32 v21, v36, v35
-; GFX10-NEXT:    v_max_i32_e32 v22, v32, v22
-; GFX10-NEXT:    v_min_i32_e32 v38, v19, v20
-; GFX10-NEXT:    v_max_i32_e32 v20, -1, v9
-; GFX10-NEXT:    v_max_i32_e32 v39, -1, v8
-; GFX10-NEXT:    v_sub_nc_u32_e32 v16, v16, v35
-; GFX10-NEXT:    v_max_i32_e32 v17, v17, v23
-; GFX10-NEXT:    v_min_i32_e32 v19, v22, v21
-; GFX10-NEXT:    v_min_i32_e32 v21, -1, v9
-; GFX10-NEXT:    v_sub_nc_u32_e32 v20, v20, v34
-; GFX10-NEXT:    v_sub_nc_u32_e32 v3, v3, v18
-; GFX10-NEXT:    v_sub_nc_u32_e32 v18, v39, v34
-; GFX10-NEXT:    v_max_i32_e32 v39, -1, v10
-; GFX10-NEXT:    v_min_i32_e32 v16, v17, v16
-; GFX10-NEXT:    v_min_i32_e32 v22, -1, v8
-; GFX10-NEXT:    v_sub_nc_u32_e32 v5, v5, v38
-; GFX10-NEXT:    v_sub_nc_u32_e32 v6, v6, v19
-; GFX10-NEXT:    v_max_i32_e32 v18, v18, v24
-; GFX10-NEXT:    v_max_i32_e32 v20, v20, v25
-; GFX10-NEXT:    v_sub_nc_u32_e32 v19, v22, v35
-; GFX10-NEXT:    v_sub_nc_u32_e32 v21, v21, v35
-; GFX10-NEXT:    v_sub_nc_u32_e32 v7, v7, v16
-; GFX10-NEXT:    v_max_i32_e32 v16, -1, v11
-; GFX10-NEXT:    v_min_i32_e32 v38, -1, v10
-; GFX10-NEXT:    v_sub_nc_u32_e32 v23, v39, v34
-; GFX10-NEXT:    v_min_i32_e32 v17, v18, v19
-; GFX10-NEXT:    v_min_i32_e32 v20, v20, v21
-; GFX10-NEXT:    v_sub_nc_u32_e32 v16, v16, v34
-; GFX10-NEXT:    v_sub_nc_u32_e32 v18, v38, v35
-; GFX10-NEXT:    v_max_i32_e32 v19, v23, v26
-; GFX10-NEXT:    v_sub_nc_u32_e32 v8, v8, v17
-; GFX10-NEXT:    v_sub_nc_u32_e32 v9, v9, v20
-; GFX10-NEXT:    v_max_i32_e32 v20, -1, v13
-; GFX10-NEXT:    v_max_i32_e32 v16, v16, v27
-; GFX10-NEXT:    v_min_i32_e32 v17, v19, v18
-; GFX10-NEXT:    v_max_i32_e32 v19, -1, v12
-; GFX10-NEXT:    v_max_i32_e32 v27, -1, v14
-; GFX10-NEXT:    v_max_i32_e32 v23, -1, v15
-; GFX10-NEXT:    v_min_i32_e32 v18, -1, v11
-; GFX10-NEXT:    v_min_i32_e32 v21, -1, v13
-; GFX10-NEXT:    v_sub_nc_u32_e32 v19, v19, v34
-; GFX10-NEXT:    v_sub_nc_u32_e32 v20, v20, v34
-; GFX10-NEXT:    v_min_i32_e32 v24, -1, v14
-; GFX10-NEXT:    v_min_i32_e32 v25, -1, v15
-; GFX10-NEXT:    v_sub_nc_u32_e32 v26, v23, v34
-; GFX10-NEXT:    v_sub_nc_u32_e32 v10, v10, v17
-; GFX10-NEXT:    v_min_i32_e32 v17, -1, v12
-; GFX10-NEXT:    v_sub_nc_u32_e32 v27, v27, v34
-; GFX10-NEXT:    v_sub_nc_u32_e32 v18, v18, v35
-; GFX10-NEXT:    v_max_i32_e32 v19, v19, v28
-; GFX10-NEXT:    v_sub_nc_u32_e32 v21, v21, v35
-; GFX10-NEXT:    v_sub_nc_u32_e32 v17, v17, v35
-; GFX10-NEXT:    v_max_i32_e32 v20, v20, v29
-; GFX10-NEXT:    v_sub_nc_u32_e32 v24, v24, v35
-; GFX10-NEXT:    v_max_i32_e32 v22, v27, v30
-; GFX10-NEXT:    v_sub_nc_u32_e32 v25, v25, v35
-; GFX10-NEXT:    v_max_i32_e32 v23, v26, v31
-; GFX10-NEXT:    v_min_i32_e32 v16, v16, v18
-; GFX10-NEXT:    v_min_i32_e32 v17, v19, v17
-; GFX10-NEXT:    v_min_i32_e32 v18, v20, v21
-; GFX10-NEXT:    v_min_i32_e32 v19, v22, v24
-; GFX10-NEXT:    v_min_i32_e32 v20, v23, v25
-; GFX10-NEXT:    v_sub_nc_u32_e32 v11, v11, v16
-; GFX10-NEXT:    v_sub_nc_u32_e32 v12, v12, v17
-; GFX10-NEXT:    v_sub_nc_u32_e32 v13, v13, v18
-; GFX10-NEXT:    v_sub_nc_u32_e32 v14, v14, v19
-; GFX10-NEXT:    v_sub_nc_u32_e32 v15, v15, v20
+; GFX10-NEXT:    v_sub_nc_i32 v0, v0, v16 clamp
+; GFX10-NEXT:    v_sub_nc_i32 v1, v1, v17 clamp
+; GFX10-NEXT:    v_sub_nc_i32 v2, v2, v18 clamp
+; GFX10-NEXT:    v_sub_nc_i32 v3, v3, v19 clamp
+; GFX10-NEXT:    v_sub_nc_i32 v4, v4, v20 clamp
+; GFX10-NEXT:    v_sub_nc_i32 v5, v5, v21 clamp
+; GFX10-NEXT:    v_sub_nc_i32 v6, v6, v22 clamp
+; GFX10-NEXT:    v_sub_nc_i32 v7, v7, v23 clamp
+; GFX10-NEXT:    v_sub_nc_i32 v8, v8, v24 clamp
+; GFX10-NEXT:    v_sub_nc_i32 v9, v9, v25 clamp
+; GFX10-NEXT:    v_sub_nc_i32 v10, v10, v26 clamp
+; GFX10-NEXT:    v_sub_nc_i32 v11, v11, v27 clamp
+; GFX10-NEXT:    v_sub_nc_i32 v12, v12, v28 clamp
+; GFX10-NEXT:    v_sub_nc_i32 v13, v13, v29 clamp
+; GFX10-NEXT:    v_sub_nc_i32 v14, v14, v30 clamp
+; GFX10-NEXT:    v_sub_nc_i32 v15, v15, v31 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <16 x i32> @llvm.ssub.sat.v16i32(<16 x i32> %lhs, <16 x i32> %rhs)
@@ -3829,367 +2719,91 @@ define amdgpu_ps <16 x i32> @s_ssubsat_v16i32(<16 x i32> inreg %lhs, <16 x i32>
 ;
 ; GFX9-LABEL: s_ssubsat_v16i32:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_cmp_gt_i32 s0, -1
-; GFX9-NEXT:    s_brev_b32 s32, -2
-; GFX9-NEXT:    s_cselect_b32 s34, s0, -1
-; GFX9-NEXT:    s_sub_i32 s34, s34, s32
-; GFX9-NEXT:    s_cmp_lt_i32 s0, -1
-; GFX9-NEXT:    s_brev_b32 s33, 1
-; GFX9-NEXT:    s_cselect_b32 s35, s0, -1
-; GFX9-NEXT:    s_sub_i32 s35, s35, s33
-; GFX9-NEXT:    s_cmp_gt_i32 s34, s16
-; GFX9-NEXT:    s_cselect_b32 s16, s34, s16
-; GFX9-NEXT:    s_cmp_lt_i32 s16, s35
-; GFX9-NEXT:    s_cselect_b32 s16, s16, s35
-; GFX9-NEXT:    s_sub_i32 s0, s0, s16
-; GFX9-NEXT:    s_cmp_gt_i32 s1, -1
-; GFX9-NEXT:    s_cselect_b32 s16, s1, -1
-; GFX9-NEXT:    s_sub_i32 s16, s16, s32
-; GFX9-NEXT:    s_cmp_lt_i32 s1, -1
-; GFX9-NEXT:    s_cselect_b32 s34, s1, -1
-; GFX9-NEXT:    s_sub_i32 s34, s34, s33
-; GFX9-NEXT:    s_cmp_gt_i32 s16, s17
-; GFX9-NEXT:    s_cselect_b32 s16, s16, s17
-; GFX9-NEXT:    s_cmp_lt_i32 s16, s34
-; GFX9-NEXT:    s_cselect_b32 s16, s16, s34
-; GFX9-NEXT:    s_sub_i32 s1, s1, s16
-; GFX9-NEXT:    s_cmp_gt_i32 s2, -1
-; GFX9-NEXT:    s_cselect_b32 s16, s2, -1
-; GFX9-NEXT:    s_sub_i32 s16, s16, s32
-; GFX9-NEXT:    s_cmp_lt_i32 s2, -1
-; GFX9-NEXT:    s_cselect_b32 s17, s2, -1
-; GFX9-NEXT:    s_sub_i32 s17, s17, s33
-; GFX9-NEXT:    s_cmp_gt_i32 s16, s18
-; GFX9-NEXT:    s_cselect_b32 s16, s16, s18
-; GFX9-NEXT:    s_cmp_lt_i32 s16, s17
-; GFX9-NEXT:    s_cselect_b32 s16, s16, s17
-; GFX9-NEXT:    s_sub_i32 s2, s2, s16
-; GFX9-NEXT:    s_cmp_gt_i32 s3, -1
-; GFX9-NEXT:    s_cselect_b32 s16, s3, -1
-; GFX9-NEXT:    s_sub_i32 s16, s16, s32
-; GFX9-NEXT:    s_cmp_lt_i32 s3, -1
-; GFX9-NEXT:    s_cselect_b32 s17, s3, -1
-; GFX9-NEXT:    s_sub_i32 s17, s17, s33
-; GFX9-NEXT:    s_cmp_gt_i32 s16, s19
-; GFX9-NEXT:    s_cselect_b32 s16, s16, s19
-; GFX9-NEXT:    s_cmp_lt_i32 s16, s17
-; GFX9-NEXT:    s_cselect_b32 s16, s16, s17
-; GFX9-NEXT:    s_sub_i32 s3, s3, s16
-; GFX9-NEXT:    s_cmp_gt_i32 s4, -1
-; GFX9-NEXT:    s_cselect_b32 s16, s4, -1
-; GFX9-NEXT:    s_sub_i32 s16, s16, s32
-; GFX9-NEXT:    s_cmp_lt_i32 s4, -1
-; GFX9-NEXT:    s_cselect_b32 s17, s4, -1
-; GFX9-NEXT:    s_sub_i32 s17, s17, s33
-; GFX9-NEXT:    s_cmp_gt_i32 s16, s20
-; GFX9-NEXT:    s_cselect_b32 s16, s16, s20
-; GFX9-NEXT:    s_cmp_lt_i32 s16, s17
-; GFX9-NEXT:    s_cselect_b32 s16, s16, s17
-; GFX9-NEXT:    s_sub_i32 s4, s4, s16
-; GFX9-NEXT:    s_cmp_gt_i32 s5, -1
-; GFX9-NEXT:    s_cselect_b32 s16, s5, -1
-; GFX9-NEXT:    s_sub_i32 s16, s16, s32
-; GFX9-NEXT:    s_cmp_lt_i32 s5, -1
-; GFX9-NEXT:    s_cselect_b32 s17, s5, -1
-; GFX9-NEXT:    s_sub_i32 s17, s17, s33
-; GFX9-NEXT:    s_cmp_gt_i32 s16, s21
-; GFX9-NEXT:    s_cselect_b32 s16, s16, s21
-; GFX9-NEXT:    s_cmp_lt_i32 s16, s17
-; GFX9-NEXT:    s_cselect_b32 s16, s16, s17
-; GFX9-NEXT:    s_sub_i32 s5, s5, s16
-; GFX9-NEXT:    s_cmp_gt_i32 s6, -1
-; GFX9-NEXT:    s_cselect_b32 s16, s6, -1
-; GFX9-NEXT:    s_sub_i32 s16, s16, s32
-; GFX9-NEXT:    s_cmp_lt_i32 s6, -1
-; GFX9-NEXT:    s_cselect_b32 s17, s6, -1
-; GFX9-NEXT:    s_sub_i32 s17, s17, s33
-; GFX9-NEXT:    s_cmp_gt_i32 s16, s22
-; GFX9-NEXT:    s_cselect_b32 s16, s16, s22
-; GFX9-NEXT:    s_cmp_lt_i32 s16, s17
-; GFX9-NEXT:    s_cselect_b32 s16, s16, s17
-; GFX9-NEXT:    s_sub_i32 s6, s6, s16
-; GFX9-NEXT:    s_cmp_gt_i32 s7, -1
-; GFX9-NEXT:    s_cselect_b32 s16, s7, -1
-; GFX9-NEXT:    s_sub_i32 s16, s16, s32
-; GFX9-NEXT:    s_cmp_lt_i32 s7, -1
-; GFX9-NEXT:    s_cselect_b32 s17, s7, -1
-; GFX9-NEXT:    s_sub_i32 s17, s17, s33
-; GFX9-NEXT:    s_cmp_gt_i32 s16, s23
-; GFX9-NEXT:    s_cselect_b32 s16, s16, s23
-; GFX9-NEXT:    s_cmp_lt_i32 s16, s17
-; GFX9-NEXT:    s_cselect_b32 s16, s16, s17
-; GFX9-NEXT:    s_sub_i32 s7, s7, s16
-; GFX9-NEXT:    s_cmp_gt_i32 s8, -1
-; GFX9-NEXT:    s_cselect_b32 s16, s8, -1
-; GFX9-NEXT:    s_sub_i32 s16, s16, s32
-; GFX9-NEXT:    s_cmp_lt_i32 s8, -1
-; GFX9-NEXT:    s_cselect_b32 s17, s8, -1
-; GFX9-NEXT:    s_sub_i32 s17, s17, s33
-; GFX9-NEXT:    s_cmp_gt_i32 s16, s24
-; GFX9-NEXT:    s_cselect_b32 s16, s16, s24
-; GFX9-NEXT:    s_cmp_lt_i32 s16, s17
-; GFX9-NEXT:    s_cselect_b32 s16, s16, s17
-; GFX9-NEXT:    s_sub_i32 s8, s8, s16
-; GFX9-NEXT:    s_cmp_gt_i32 s9, -1
-; GFX9-NEXT:    s_cselect_b32 s16, s9, -1
-; GFX9-NEXT:    s_sub_i32 s16, s16, s32
-; GFX9-NEXT:    s_cmp_lt_i32 s9, -1
-; GFX9-NEXT:    s_cselect_b32 s17, s9, -1
-; GFX9-NEXT:    s_sub_i32 s17, s17, s33
-; GFX9-NEXT:    s_cmp_gt_i32 s16, s25
-; GFX9-NEXT:    s_cselect_b32 s16, s16, s25
-; GFX9-NEXT:    s_cmp_lt_i32 s16, s17
-; GFX9-NEXT:    s_cselect_b32 s16, s16, s17
-; GFX9-NEXT:    s_sub_i32 s9, s9, s16
-; GFX9-NEXT:    s_cmp_gt_i32 s10, -1
-; GFX9-NEXT:    s_cselect_b32 s16, s10, -1
-; GFX9-NEXT:    s_sub_i32 s16, s16, s32
-; GFX9-NEXT:    s_cmp_lt_i32 s10, -1
-; GFX9-NEXT:    s_cselect_b32 s17, s10, -1
-; GFX9-NEXT:    s_sub_i32 s17, s17, s33
-; GFX9-NEXT:    s_cmp_gt_i32 s16, s26
-; GFX9-NEXT:    s_cselect_b32 s16, s16, s26
-; GFX9-NEXT:    s_cmp_lt_i32 s16, s17
-; GFX9-NEXT:    s_cselect_b32 s16, s16, s17
-; GFX9-NEXT:    s_sub_i32 s10, s10, s16
-; GFX9-NEXT:    s_cmp_gt_i32 s11, -1
-; GFX9-NEXT:    s_cselect_b32 s16, s11, -1
-; GFX9-NEXT:    s_sub_i32 s16, s16, s32
-; GFX9-NEXT:    s_cmp_lt_i32 s11, -1
-; GFX9-NEXT:    s_cselect_b32 s17, s11, -1
-; GFX9-NEXT:    s_sub_i32 s17, s17, s33
-; GFX9-NEXT:    s_cmp_gt_i32 s16, s27
-; GFX9-NEXT:    s_cselect_b32 s16, s16, s27
-; GFX9-NEXT:    s_cmp_lt_i32 s16, s17
-; GFX9-NEXT:    s_cselect_b32 s16, s16, s17
-; GFX9-NEXT:    s_sub_i32 s11, s11, s16
-; GFX9-NEXT:    s_cmp_gt_i32 s12, -1
-; GFX9-NEXT:    s_cselect_b32 s16, s12, -1
-; GFX9-NEXT:    s_sub_i32 s16, s16, s32
-; GFX9-NEXT:    s_cmp_lt_i32 s12, -1
-; GFX9-NEXT:    s_cselect_b32 s17, s12, -1
-; GFX9-NEXT:    s_sub_i32 s17, s17, s33
-; GFX9-NEXT:    s_cmp_gt_i32 s16, s28
-; GFX9-NEXT:    s_cselect_b32 s16, s16, s28
-; GFX9-NEXT:    s_cmp_lt_i32 s16, s17
-; GFX9-NEXT:    s_cselect_b32 s16, s16, s17
-; GFX9-NEXT:    s_sub_i32 s12, s12, s16
-; GFX9-NEXT:    s_cmp_gt_i32 s13, -1
-; GFX9-NEXT:    s_cselect_b32 s16, s13, -1
-; GFX9-NEXT:    s_sub_i32 s16, s16, s32
-; GFX9-NEXT:    s_cmp_lt_i32 s13, -1
-; GFX9-NEXT:    s_cselect_b32 s17, s13, -1
-; GFX9-NEXT:    s_sub_i32 s17, s17, s33
-; GFX9-NEXT:    s_cmp_gt_i32 s16, s29
-; GFX9-NEXT:    s_cselect_b32 s16, s16, s29
-; GFX9-NEXT:    s_cmp_lt_i32 s16, s17
-; GFX9-NEXT:    s_cselect_b32 s16, s16, s17
-; GFX9-NEXT:    s_sub_i32 s13, s13, s16
-; GFX9-NEXT:    s_cmp_gt_i32 s14, -1
-; GFX9-NEXT:    s_cselect_b32 s16, s14, -1
-; GFX9-NEXT:    s_sub_i32 s16, s16, s32
-; GFX9-NEXT:    s_cmp_lt_i32 s14, -1
-; GFX9-NEXT:    s_cselect_b32 s17, s14, -1
-; GFX9-NEXT:    s_sub_i32 s17, s17, s33
-; GFX9-NEXT:    s_cmp_gt_i32 s16, s30
-; GFX9-NEXT:    s_cselect_b32 s16, s16, s30
-; GFX9-NEXT:    s_cmp_lt_i32 s16, s17
-; GFX9-NEXT:    s_cselect_b32 s16, s16, s17
-; GFX9-NEXT:    s_sub_i32 s14, s14, s16
-; GFX9-NEXT:    s_cmp_gt_i32 s15, -1
-; GFX9-NEXT:    s_cselect_b32 s16, s15, -1
-; GFX9-NEXT:    s_sub_i32 s16, s16, s32
-; GFX9-NEXT:    s_cmp_lt_i32 s15, -1
-; GFX9-NEXT:    s_cselect_b32 s17, s15, -1
-; GFX9-NEXT:    s_sub_i32 s17, s17, s33
-; GFX9-NEXT:    s_cmp_gt_i32 s16, s31
-; GFX9-NEXT:    s_cselect_b32 s16, s16, s31
-; GFX9-NEXT:    s_cmp_lt_i32 s16, s17
-; GFX9-NEXT:    s_cselect_b32 s16, s16, s17
-; GFX9-NEXT:    s_sub_i32 s15, s15, s16
+; GFX9-NEXT:    v_mov_b32_e32 v0, s16
+; GFX9-NEXT:    v_mov_b32_e32 v1, s17
+; GFX9-NEXT:    v_mov_b32_e32 v2, s18
+; GFX9-NEXT:    v_mov_b32_e32 v3, s19
+; GFX9-NEXT:    v_mov_b32_e32 v4, s20
+; GFX9-NEXT:    v_mov_b32_e32 v5, s21
+; GFX9-NEXT:    v_mov_b32_e32 v6, s22
+; GFX9-NEXT:    v_mov_b32_e32 v7, s23
+; GFX9-NEXT:    v_mov_b32_e32 v8, s24
+; GFX9-NEXT:    v_mov_b32_e32 v9, s25
+; GFX9-NEXT:    v_mov_b32_e32 v10, s26
+; GFX9-NEXT:    v_mov_b32_e32 v11, s27
+; GFX9-NEXT:    v_mov_b32_e32 v12, s28
+; GFX9-NEXT:    v_mov_b32_e32 v13, s29
+; GFX9-NEXT:    v_mov_b32_e32 v14, s30
+; GFX9-NEXT:    v_mov_b32_e32 v15, s31
+; GFX9-NEXT:    v_sub_i32 v0, s0, v0 clamp
+; GFX9-NEXT:    v_sub_i32 v1, s1, v1 clamp
+; GFX9-NEXT:    v_sub_i32 v2, s2, v2 clamp
+; GFX9-NEXT:    v_sub_i32 v3, s3, v3 clamp
+; GFX9-NEXT:    v_sub_i32 v4, s4, v4 clamp
+; GFX9-NEXT:    v_sub_i32 v5, s5, v5 clamp
+; GFX9-NEXT:    v_sub_i32 v6, s6, v6 clamp
+; GFX9-NEXT:    v_sub_i32 v7, s7, v7 clamp
+; GFX9-NEXT:    v_sub_i32 v8, s8, v8 clamp
+; GFX9-NEXT:    v_sub_i32 v9, s9, v9 clamp
+; GFX9-NEXT:    v_sub_i32 v10, s10, v10 clamp
+; GFX9-NEXT:    v_sub_i32 v11, s11, v11 clamp
+; GFX9-NEXT:    v_sub_i32 v12, s12, v12 clamp
+; GFX9-NEXT:    v_sub_i32 v13, s13, v13 clamp
+; GFX9-NEXT:    v_sub_i32 v14, s14, v14 clamp
+; GFX9-NEXT:    v_sub_i32 v15, s15, v15 clamp
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX9-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX9-NEXT:    v_readfirstlane_b32 s4, v4
+; GFX9-NEXT:    v_readfirstlane_b32 s5, v5
+; GFX9-NEXT:    v_readfirstlane_b32 s6, v6
+; GFX9-NEXT:    v_readfirstlane_b32 s7, v7
+; GFX9-NEXT:    v_readfirstlane_b32 s8, v8
+; GFX9-NEXT:    v_readfirstlane_b32 s9, v9
+; GFX9-NEXT:    v_readfirstlane_b32 s10, v10
+; GFX9-NEXT:    v_readfirstlane_b32 s11, v11
+; GFX9-NEXT:    v_readfirstlane_b32 s12, v12
+; GFX9-NEXT:    v_readfirstlane_b32 s13, v13
+; GFX9-NEXT:    v_readfirstlane_b32 s14, v14
+; GFX9-NEXT:    v_readfirstlane_b32 s15, v15
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_ssubsat_v16i32:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_cmp_gt_i32 s0, -1
-; GFX10-NEXT:    s_brev_b32 s46, -2
-; GFX10-NEXT:    s_cselect_b32 s33, s0, -1
-; GFX10-NEXT:    s_brev_b32 s34, 1
-; GFX10-NEXT:    s_sub_i32 s47, s33, s46
-; GFX10-NEXT:    s_cmp_lt_i32 s0, -1
+; GFX10-NEXT:    v_sub_nc_i32 v0, s0, s16 clamp
+; GFX10-NEXT:    v_sub_nc_i32 v1, s1, s17 clamp
+; GFX10-NEXT:    v_sub_nc_i32 v2, s2, s18 clamp
+; GFX10-NEXT:    v_sub_nc_i32 v3, s3, s19 clamp
+; GFX10-NEXT:    v_sub_nc_i32 v4, s4, s20 clamp
+; GFX10-NEXT:    v_sub_nc_i32 v5, s5, s21 clamp
+; GFX10-NEXT:    v_sub_nc_i32 v6, s6, s22 clamp
+; GFX10-NEXT:    v_sub_nc_i32 v7, s7, s23 clamp
+; GFX10-NEXT:    v_sub_nc_i32 v8, s8, s24 clamp
+; GFX10-NEXT:    v_sub_nc_i32 v9, s9, s25 clamp
+; GFX10-NEXT:    v_sub_nc_i32 v10, s10, s26 clamp
+; GFX10-NEXT:    v_sub_nc_i32 v11, s11, s27 clamp
+; GFX10-NEXT:    v_sub_nc_i32 v12, s12, s28 clamp
+; GFX10-NEXT:    v_sub_nc_i32 v13, s13, s29 clamp
+; GFX10-NEXT:    v_sub_nc_i32 v14, s14, s30 clamp
+; GFX10-NEXT:    v_sub_nc_i32 v15, s15, s31 clamp
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX10-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX10-NEXT:    v_readfirstlane_b32 s4, v4
+; GFX10-NEXT:    v_readfirstlane_b32 s5, v5
+; GFX10-NEXT:    v_readfirstlane_b32 s6, v6
+; GFX10-NEXT:    v_readfirstlane_b32 s7, v7
+; GFX10-NEXT:    v_readfirstlane_b32 s8, v8
+; GFX10-NEXT:    v_readfirstlane_b32 s9, v9
+; GFX10-NEXT:    v_readfirstlane_b32 s10, v10
+; GFX10-NEXT:    v_readfirstlane_b32 s11, v11
+; GFX10-NEXT:    v_readfirstlane_b32 s12, v12
+; GFX10-NEXT:    v_readfirstlane_b32 s13, v13
+; GFX10-NEXT:    v_readfirstlane_b32 s14, v14
+; GFX10-NEXT:    v_readfirstlane_b32 s15, v15
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cselect_b32 s35, s0, -1
-; GFX10-NEXT:    s_sub_i32 s35, s35, s34
-; GFX10-NEXT:    s_cmp_gt_i32 s47, s16
-; GFX10-NEXT:    s_cselect_b32 s16, s47, s16
-; GFX10-NEXT:    s_cmp_lt_i32 s16, s35
-; GFX10-NEXT:    s_cselect_b32 s47, s16, s35
-; GFX10-NEXT:    s_sub_i32 s0, s0, s47
-; GFX10-NEXT:    s_cmp_gt_i32 s1, -1
-; GFX10-NEXT:    s_cselect_b32 s16, s1, -1
-; GFX10-NEXT:    s_sub_i32 s16, s16, s46
-; GFX10-NEXT:    s_cmp_lt_i32 s1, -1
-; GFX10-NEXT:    s_cselect_b32 s33, s1, -1
-; GFX10-NEXT:    s_sub_i32 s47, s33, s34
-; GFX10-NEXT:    s_cmp_gt_i32 s16, s17
-; GFX10-NEXT:    s_cselect_b32 s16, s16, s17
-; GFX10-NEXT:    s_cmp_lt_i32 s16, s47
-; GFX10-NEXT:    s_cselect_b32 s47, s16, s47
-; GFX10-NEXT:    s_sub_i32 s1, s1, s47
-; GFX10-NEXT:    s_cmp_gt_i32 s2, -1
-; GFX10-NEXT:    s_cselect_b32 s16, s2, -1
-; GFX10-NEXT:    s_sub_i32 s16, s16, s46
-; GFX10-NEXT:    s_cmp_lt_i32 s2, -1
-; GFX10-NEXT:    s_cselect_b32 s17, s2, -1
-; GFX10-NEXT:    s_sub_i32 s17, s17, s34
-; GFX10-NEXT:    s_cmp_gt_i32 s16, s18
-; GFX10-NEXT:    s_cselect_b32 s16, s16, s18
-; GFX10-NEXT:    s_cmp_lt_i32 s16, s17
-; GFX10-NEXT:    s_cselect_b32 s16, s16, s17
-; GFX10-NEXT:    s_sub_i32 s2, s2, s16
-; GFX10-NEXT:    s_cmp_gt_i32 s3, -1
-; GFX10-NEXT:    s_cselect_b32 s16, s3, -1
-; GFX10-NEXT:    s_sub_i32 s16, s16, s46
-; GFX10-NEXT:    s_cmp_lt_i32 s3, -1
-; GFX10-NEXT:    s_cselect_b32 s17, s3, -1
-; GFX10-NEXT:    s_sub_i32 s17, s17, s34
-; GFX10-NEXT:    s_cmp_gt_i32 s16, s19
-; GFX10-NEXT:    s_cselect_b32 s16, s16, s19
-; GFX10-NEXT:    s_cmp_lt_i32 s16, s17
-; GFX10-NEXT:    s_cselect_b32 s16, s16, s17
-; GFX10-NEXT:    s_sub_i32 s3, s3, s16
-; GFX10-NEXT:    s_cmp_gt_i32 s4, -1
-; GFX10-NEXT:    s_cselect_b32 s16, s4, -1
-; GFX10-NEXT:    s_sub_i32 s16, s16, s46
-; GFX10-NEXT:    s_cmp_lt_i32 s4, -1
-; GFX10-NEXT:    s_cselect_b32 s17, s4, -1
-; GFX10-NEXT:    s_sub_i32 s17, s17, s34
-; GFX10-NEXT:    s_cmp_gt_i32 s16, s20
-; GFX10-NEXT:    s_cselect_b32 s16, s16, s20
-; GFX10-NEXT:    s_cmp_lt_i32 s16, s17
-; GFX10-NEXT:    s_cselect_b32 s16, s16, s17
-; GFX10-NEXT:    s_sub_i32 s4, s4, s16
-; GFX10-NEXT:    s_cmp_gt_i32 s5, -1
-; GFX10-NEXT:    s_cselect_b32 s16, s5, -1
-; GFX10-NEXT:    s_sub_i32 s16, s16, s46
-; GFX10-NEXT:    s_cmp_lt_i32 s5, -1
-; GFX10-NEXT:    s_cselect_b32 s17, s5, -1
-; GFX10-NEXT:    s_sub_i32 s17, s17, s34
-; GFX10-NEXT:    s_cmp_gt_i32 s16, s21
-; GFX10-NEXT:    s_cselect_b32 s16, s16, s21
-; GFX10-NEXT:    s_cmp_lt_i32 s16, s17
-; GFX10-NEXT:    s_cselect_b32 s16, s16, s17
-; GFX10-NEXT:    s_sub_i32 s5, s5, s16
-; GFX10-NEXT:    s_cmp_gt_i32 s6, -1
-; GFX10-NEXT:    s_cselect_b32 s16, s6, -1
-; GFX10-NEXT:    s_sub_i32 s16, s16, s46
-; GFX10-NEXT:    s_cmp_lt_i32 s6, -1
-; GFX10-NEXT:    s_cselect_b32 s17, s6, -1
-; GFX10-NEXT:    s_sub_i32 s17, s17, s34
-; GFX10-NEXT:    s_cmp_gt_i32 s16, s22
-; GFX10-NEXT:    s_cselect_b32 s16, s16, s22
-; GFX10-NEXT:    s_cmp_lt_i32 s16, s17
-; GFX10-NEXT:    s_cselect_b32 s16, s16, s17
-; GFX10-NEXT:    s_sub_i32 s6, s6, s16
-; GFX10-NEXT:    s_cmp_gt_i32 s7, -1
-; GFX10-NEXT:    s_cselect_b32 s16, s7, -1
-; GFX10-NEXT:    s_sub_i32 s16, s16, s46
-; GFX10-NEXT:    s_cmp_lt_i32 s7, -1
-; GFX10-NEXT:    s_cselect_b32 s17, s7, -1
-; GFX10-NEXT:    s_sub_i32 s17, s17, s34
-; GFX10-NEXT:    s_cmp_gt_i32 s16, s23
-; GFX10-NEXT:    s_cselect_b32 s16, s16, s23
-; GFX10-NEXT:    s_cmp_lt_i32 s16, s17
-; GFX10-NEXT:    s_cselect_b32 s16, s16, s17
-; GFX10-NEXT:    s_sub_i32 s7, s7, s16
-; GFX10-NEXT:    s_cmp_gt_i32 s8, -1
-; GFX10-NEXT:    s_cselect_b32 s16, s8, -1
-; GFX10-NEXT:    s_sub_i32 s16, s16, s46
-; GFX10-NEXT:    s_cmp_lt_i32 s8, -1
-; GFX10-NEXT:    s_cselect_b32 s17, s8, -1
-; GFX10-NEXT:    s_sub_i32 s17, s17, s34
-; GFX10-NEXT:    s_cmp_gt_i32 s16, s24
-; GFX10-NEXT:    s_cselect_b32 s16, s16, s24
-; GFX10-NEXT:    s_cmp_lt_i32 s16, s17
-; GFX10-NEXT:    s_cselect_b32 s16, s16, s17
-; GFX10-NEXT:    s_sub_i32 s8, s8, s16
-; GFX10-NEXT:    s_cmp_gt_i32 s9, -1
-; GFX10-NEXT:    s_cselect_b32 s16, s9, -1
-; GFX10-NEXT:    s_sub_i32 s16, s16, s46
-; GFX10-NEXT:    s_cmp_lt_i32 s9, -1
-; GFX10-NEXT:    s_cselect_b32 s17, s9, -1
-; GFX10-NEXT:    s_sub_i32 s17, s17, s34
-; GFX10-NEXT:    s_cmp_gt_i32 s16, s25
-; GFX10-NEXT:    s_cselect_b32 s16, s16, s25
-; GFX10-NEXT:    s_cmp_lt_i32 s16, s17
-; GFX10-NEXT:    s_cselect_b32 s16, s16, s17
-; GFX10-NEXT:    s_sub_i32 s9, s9, s16
-; GFX10-NEXT:    s_cmp_gt_i32 s10, -1
-; GFX10-NEXT:    s_cselect_b32 s16, s10, -1
-; GFX10-NEXT:    s_sub_i32 s16, s16, s46
-; GFX10-NEXT:    s_cmp_lt_i32 s10, -1
-; GFX10-NEXT:    s_cselect_b32 s17, s10, -1
-; GFX10-NEXT:    s_sub_i32 s17, s17, s34
-; GFX10-NEXT:    s_cmp_gt_i32 s16, s26
-; GFX10-NEXT:    s_cselect_b32 s16, s16, s26
-; GFX10-NEXT:    s_cmp_lt_i32 s16, s17
-; GFX10-NEXT:    s_cselect_b32 s16, s16, s17
-; GFX10-NEXT:    s_sub_i32 s10, s10, s16
-; GFX10-NEXT:    s_cmp_gt_i32 s11, -1
-; GFX10-NEXT:    s_cselect_b32 s16, s11, -1
-; GFX10-NEXT:    s_sub_i32 s16, s16, s46
-; GFX10-NEXT:    s_cmp_lt_i32 s11, -1
-; GFX10-NEXT:    s_cselect_b32 s17, s11, -1
-; GFX10-NEXT:    s_sub_i32 s17, s17, s34
-; GFX10-NEXT:    s_cmp_gt_i32 s16, s27
-; GFX10-NEXT:    s_cselect_b32 s16, s16, s27
-; GFX10-NEXT:    s_cmp_lt_i32 s16, s17
-; GFX10-NEXT:    s_cselect_b32 s16, s16, s17
-; GFX10-NEXT:    s_sub_i32 s11, s11, s16
-; GFX10-NEXT:    s_cmp_gt_i32 s12, -1
-; GFX10-NEXT:    s_cselect_b32 s16, s12, -1
-; GFX10-NEXT:    s_sub_i32 s16, s16, s46
-; GFX10-NEXT:    s_cmp_lt_i32 s12, -1
-; GFX10-NEXT:    s_cselect_b32 s17, s12, -1
-; GFX10-NEXT:    s_sub_i32 s17, s17, s34
-; GFX10-NEXT:    s_cmp_gt_i32 s16, s28
-; GFX10-NEXT:    s_cselect_b32 s16, s16, s28
-; GFX10-NEXT:    s_cmp_lt_i32 s16, s17
-; GFX10-NEXT:    s_cselect_b32 s16, s16, s17
-; GFX10-NEXT:    s_sub_i32 s12, s12, s16
-; GFX10-NEXT:    s_cmp_gt_i32 s13, -1
-; GFX10-NEXT:    s_cselect_b32 s16, s13, -1
-; GFX10-NEXT:    s_sub_i32 s16, s16, s46
-; GFX10-NEXT:    s_cmp_lt_i32 s13, -1
-; GFX10-NEXT:    s_cselect_b32 s17, s13, -1
-; GFX10-NEXT:    s_sub_i32 s17, s17, s34
-; GFX10-NEXT:    s_cmp_gt_i32 s16, s29
-; GFX10-NEXT:    s_cselect_b32 s16, s16, s29
-; GFX10-NEXT:    s_cmp_lt_i32 s16, s17
-; GFX10-NEXT:    s_cselect_b32 s16, s16, s17
-; GFX10-NEXT:    s_sub_i32 s13, s13, s16
-; GFX10-NEXT:    s_cmp_gt_i32 s14, -1
-; GFX10-NEXT:    s_cselect_b32 s16, s14, -1
-; GFX10-NEXT:    s_sub_i32 s16, s16, s46
-; GFX10-NEXT:    s_cmp_lt_i32 s14, -1
-; GFX10-NEXT:    s_cselect_b32 s17, s14, -1
-; GFX10-NEXT:    s_sub_i32 s17, s17, s34
-; GFX10-NEXT:    s_cmp_gt_i32 s16, s30
-; GFX10-NEXT:    s_cselect_b32 s16, s16, s30
-; GFX10-NEXT:    s_cmp_lt_i32 s16, s17
-; GFX10-NEXT:    s_cselect_b32 s16, s16, s17
-; GFX10-NEXT:    s_sub_i32 s14, s14, s16
-; GFX10-NEXT:    s_cmp_gt_i32 s15, -1
-; GFX10-NEXT:    s_cselect_b32 s16, s15, -1
-; GFX10-NEXT:    s_sub_i32 s16, s16, s46
-; GFX10-NEXT:    s_cmp_lt_i32 s15, -1
-; GFX10-NEXT:    s_cselect_b32 s17, s15, -1
-; GFX10-NEXT:    s_sub_i32 s17, s17, s34
-; GFX10-NEXT:    s_cmp_gt_i32 s16, s31
-; GFX10-NEXT:    s_cselect_b32 s16, s16, s31
-; GFX10-NEXT:    s_cmp_lt_i32 s16, s17
-; GFX10-NEXT:    s_cselect_b32 s16, s16, s17
-; GFX10-NEXT:    s_sub_i32 s15, s15, s16
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <16 x i32> @llvm.ssub.sat.v16i32(<16 x i32> %lhs, <16 x i32> %rhs)
   ret <16 x i32> %result
@@ -4226,27 +2840,15 @@ define i16 @v_ssubsat_i16(i16 %lhs, i16 %rhs) {
 ; GFX9-LABEL: v_ssubsat_i16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_max_i16_e32 v2, -1, v0
-; GFX9-NEXT:    v_min_i16_e32 v3, -1, v0
-; GFX9-NEXT:    v_subrev_u16_e32 v2, 0x7fff, v2
-; GFX9-NEXT:    v_subrev_u16_e32 v3, 0x8000, v3
-; GFX9-NEXT:    v_max_i16_e32 v1, v2, v1
-; GFX9-NEXT:    v_min_i16_e32 v1, v1, v3
-; GFX9-NEXT:    v_sub_u16_e32 v0, v0, v1
+; GFX9-NEXT:    v_sub_i16 v0, v0, v1 clamp
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_ssubsat_i16:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_max_i16_e64 v2, v0, -1
-; GFX10-NEXT:    v_min_i16_e64 v3, v0, -1
+; GFX10-NEXT:    v_sub_nc_i16 v0, v0, v1 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_sub_nc_u16_e64 v2, v2, 0x7fff
-; GFX10-NEXT:    v_sub_nc_u16_e64 v3, v3, 0x8000
-; GFX10-NEXT:    v_max_i16_e64 v1, v2, v1
-; GFX10-NEXT:    v_min_i16_e64 v1, v1, v3
-; GFX10-NEXT:    v_sub_nc_u16_e64 v0, v0, v1
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call i16 @llvm.ssub.sat.i16(i16 %lhs, i16 %rhs)
   ret i16 %result
@@ -4294,45 +2896,16 @@ define amdgpu_ps i16 @s_ssubsat_i16(i16 inreg %lhs, i16 inreg %rhs) {
 ;
 ; GFX9-LABEL: s_ssubsat_i16:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_sext_i32_i16 s2, s0
-; GFX9-NEXT:    s_sext_i32_i16 s3, -1
-; GFX9-NEXT:    s_cmp_gt_i32 s2, s3
-; GFX9-NEXT:    s_cselect_b32 s4, s2, s3
-; GFX9-NEXT:    s_sub_i32 s4, s4, 0x7fff
-; GFX9-NEXT:    s_cmp_lt_i32 s2, s3
-; GFX9-NEXT:    s_cselect_b32 s2, s2, s3
-; GFX9-NEXT:    s_sub_i32 s2, s2, 0xffff8000
-; GFX9-NEXT:    s_sext_i32_i16 s3, s4
-; GFX9-NEXT:    s_sext_i32_i16 s1, s1
-; GFX9-NEXT:    s_cmp_gt_i32 s3, s1
-; GFX9-NEXT:    s_cselect_b32 s1, s3, s1
-; GFX9-NEXT:    s_sext_i32_i16 s1, s1
-; GFX9-NEXT:    s_sext_i32_i16 s2, s2
-; GFX9-NEXT:    s_cmp_lt_i32 s1, s2
-; GFX9-NEXT:    s_cselect_b32 s1, s1, s2
-; GFX9-NEXT:    s_sub_i32 s0, s0, s1
+; GFX9-NEXT:    v_mov_b32_e32 v0, s1
+; GFX9-NEXT:    v_sub_i16 v0, s0, v0 clamp
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_ssubsat_i16:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_sext_i32_i16 s2, -1
-; GFX10-NEXT:    s_sext_i32_i16 s3, s0
-; GFX10-NEXT:    s_sext_i32_i16 s1, s1
-; GFX10-NEXT:    s_cmp_gt_i32 s3, s2
+; GFX10-NEXT:    v_sub_nc_i16 v0, s0, s1 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cselect_b32 s4, s3, s2
-; GFX10-NEXT:    s_sub_i32 s4, s4, 0x7fff
-; GFX10-NEXT:    s_cmp_lt_i32 s3, s2
-; GFX10-NEXT:    s_cselect_b32 s2, s3, s2
-; GFX10-NEXT:    s_sext_i32_i16 s3, s4
-; GFX10-NEXT:    s_sub_i32 s2, s2, 0xffff8000
-; GFX10-NEXT:    s_cmp_gt_i32 s3, s1
-; GFX10-NEXT:    s_sext_i32_i16 s2, s2
-; GFX10-NEXT:    s_cselect_b32 s1, s3, s1
-; GFX10-NEXT:    s_sext_i32_i16 s1, s1
-; GFX10-NEXT:    s_cmp_lt_i32 s1, s2
-; GFX10-NEXT:    s_cselect_b32 s1, s1, s2
-; GFX10-NEXT:    s_sub_i32 s0, s0, s1
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call i16 @llvm.ssub.sat.i16(i16 %lhs, i16 %rhs)
   ret i16 %result
@@ -4372,33 +2945,13 @@ define amdgpu_ps half @ssubsat_i16_sv(i16 inreg %lhs, i16 %rhs) {
 ;
 ; GFX9-LABEL: ssubsat_i16_sv:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_sext_i32_i16 s1, s0
-; GFX9-NEXT:    s_sext_i32_i16 s2, -1
-; GFX9-NEXT:    s_cmp_gt_i32 s1, s2
-; GFX9-NEXT:    s_cselect_b32 s3, s1, s2
-; GFX9-NEXT:    s_sub_i32 s3, s3, 0x7fff
-; GFX9-NEXT:    s_cmp_lt_i32 s1, s2
-; GFX9-NEXT:    s_cselect_b32 s1, s1, s2
-; GFX9-NEXT:    s_sub_i32 s1, s1, 0xffff8000
-; GFX9-NEXT:    v_max_i16_e32 v0, s3, v0
-; GFX9-NEXT:    v_min_i16_e32 v0, s1, v0
-; GFX9-NEXT:    v_sub_u16_e32 v0, s0, v0
+; GFX9-NEXT:    v_sub_i16 v0, s0, v0 clamp
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: ssubsat_i16_sv:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_sext_i32_i16 s1, s0
-; GFX10-NEXT:    s_sext_i32_i16 s2, -1
+; GFX10-NEXT:    v_sub_nc_i16 v0, s0, v0 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cmp_gt_i32 s1, s2
-; GFX10-NEXT:    s_cselect_b32 s3, s1, s2
-; GFX10-NEXT:    s_sub_i32 s3, s3, 0x7fff
-; GFX10-NEXT:    s_cmp_lt_i32 s1, s2
-; GFX10-NEXT:    v_max_i16_e64 v0, s3, v0
-; GFX10-NEXT:    s_cselect_b32 s1, s1, s2
-; GFX10-NEXT:    s_sub_i32 s1, s1, 0xffff8000
-; GFX10-NEXT:    v_min_i16_e64 v0, v0, s1
-; GFX10-NEXT:    v_sub_nc_u16_e64 v0, s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call i16 @llvm.ssub.sat.i16(i16 %lhs, i16 %rhs)
   %cast = bitcast i16 %result to half
@@ -4433,25 +2986,13 @@ define amdgpu_ps half @ssubsat_i16_vs(i16 %lhs, i16 inreg %rhs) {
 ;
 ; GFX9-LABEL: ssubsat_i16_vs:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    v_max_i16_e32 v1, -1, v0
-; GFX9-NEXT:    v_subrev_u16_e32 v1, 0x7fff, v1
-; GFX9-NEXT:    v_min_i16_e32 v2, -1, v0
-; GFX9-NEXT:    v_subrev_u16_e32 v2, 0x8000, v2
-; GFX9-NEXT:    v_max_i16_e32 v1, s0, v1
-; GFX9-NEXT:    v_min_i16_e32 v1, v1, v2
-; GFX9-NEXT:    v_sub_u16_e32 v0, v0, v1
+; GFX9-NEXT:    v_sub_i16 v0, v0, s0 clamp
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: ssubsat_i16_vs:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    v_max_i16_e64 v1, v0, -1
-; GFX10-NEXT:    v_min_i16_e64 v2, v0, -1
+; GFX10-NEXT:    v_sub_nc_i16 v0, v0, s0 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_sub_nc_u16_e64 v1, v1, 0x7fff
-; GFX10-NEXT:    v_sub_nc_u16_e64 v2, v2, 0x8000
-; GFX10-NEXT:    v_max_i16_e64 v1, v1, s0
-; GFX10-NEXT:    v_min_i16_e64 v1, v1, v2
-; GFX10-NEXT:    v_sub_nc_u16_e64 v0, v0, v1
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call i16 @llvm.ssub.sat.i16(i16 %lhs, i16 %rhs)
   %cast = bitcast i16 %result to half
@@ -4512,29 +3053,15 @@ define <2 x i16> @v_ssubsat_v2i16(<2 x i16> %lhs, <2 x i16> %rhs) {
 ; GFX9-LABEL: v_ssubsat_v2i16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_pk_max_i16 v2, v0, -1 op_sel_hi:[1,0]
-; GFX9-NEXT:    v_mov_b32_e32 v3, 0x7fff7fff
-; GFX9-NEXT:    v_pk_sub_i16 v2, v2, v3
-; GFX9-NEXT:    v_pk_min_i16 v3, v0, -1 op_sel_hi:[1,0]
-; GFX9-NEXT:    v_mov_b32_e32 v4, 0x80008000
-; GFX9-NEXT:    v_pk_sub_i16 v3, v3, v4
-; GFX9-NEXT:    v_pk_max_i16 v1, v2, v1
-; GFX9-NEXT:    v_pk_min_i16 v1, v1, v3
-; GFX9-NEXT:    v_pk_sub_i16 v0, v0, v1
+; GFX9-NEXT:    v_pk_sub_i16 v0, v0, v1 clamp
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_ssubsat_v2i16:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_pk_max_i16 v2, v0, -1 op_sel_hi:[1,0]
-; GFX10-NEXT:    v_pk_min_i16 v3, v0, -1 op_sel_hi:[1,0]
+; GFX10-NEXT:    v_pk_sub_i16 v0, v0, v1 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_pk_sub_i16 v2, v2, 0x7fff7fff
-; GFX10-NEXT:    v_pk_sub_i16 v3, v3, 0x80008000
-; GFX10-NEXT:    v_pk_max_i16 v1, v2, v1
-; GFX10-NEXT:    v_pk_min_i16 v1, v1, v3
-; GFX10-NEXT:    v_pk_sub_i16 v0, v0, v1
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <2 x i16> @llvm.ssub.sat.v2i16(<2 x i16> %lhs, <2 x i16> %rhs)
   ret <2 x i16> %result
@@ -4627,99 +3154,16 @@ define amdgpu_ps i32 @s_ssubsat_v2i16(<2 x i16> inreg %lhs, <2 x i16> inreg %rhs
 ;
 ; GFX9-LABEL: s_ssubsat_v2i16:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_sext_i32_i16 s2, s0
-; GFX9-NEXT:    s_ashr_i32 s3, s0, 16
-; GFX9-NEXT:    s_sext_i32_i16 s4, -1
-; GFX9-NEXT:    s_cmp_gt_i32 s2, s4
-; GFX9-NEXT:    s_cselect_b32 s5, s2, s4
-; GFX9-NEXT:    s_cmp_gt_i32 s3, -1
-; GFX9-NEXT:    s_cselect_b32 s6, s3, -1
-; GFX9-NEXT:    s_pack_ll_b32_b16 s5, s5, s6
-; GFX9-NEXT:    s_lshr_b32 s6, s5, 16
-; GFX9-NEXT:    s_sub_i32 s5, s5, 0x7fff7fff
-; GFX9-NEXT:    s_sub_i32 s6, s6, 0x7fff
-; GFX9-NEXT:    s_cmp_lt_i32 s2, s4
-; GFX9-NEXT:    s_cselect_b32 s2, s2, s4
-; GFX9-NEXT:    s_cmp_lt_i32 s3, -1
-; GFX9-NEXT:    s_cselect_b32 s3, s3, -1
-; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s3
-; GFX9-NEXT:    s_lshr_b32 s3, s2, 16
-; GFX9-NEXT:    s_pack_ll_b32_b16 s5, s5, s6
-; GFX9-NEXT:    s_sub_i32 s2, s2, 0x80008000
-; GFX9-NEXT:    s_sub_i32 s3, s3, 0x8000
-; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s3
-; GFX9-NEXT:    s_sext_i32_i16 s3, s5
-; GFX9-NEXT:    s_ashr_i32 s4, s5, 16
-; GFX9-NEXT:    s_sext_i32_i16 s5, s1
-; GFX9-NEXT:    s_ashr_i32 s1, s1, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s3, s5
-; GFX9-NEXT:    s_cselect_b32 s3, s3, s5
-; GFX9-NEXT:    s_cmp_gt_i32 s4, s1
-; GFX9-NEXT:    s_cselect_b32 s1, s4, s1
-; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s3, s1
-; GFX9-NEXT:    s_sext_i32_i16 s3, s1
-; GFX9-NEXT:    s_sext_i32_i16 s4, s2
-; GFX9-NEXT:    s_ashr_i32 s1, s1, 16
-; GFX9-NEXT:    s_ashr_i32 s2, s2, 16
-; GFX9-NEXT:    s_cmp_lt_i32 s3, s4
-; GFX9-NEXT:    s_cselect_b32 s3, s3, s4
-; GFX9-NEXT:    s_cmp_lt_i32 s1, s2
-; GFX9-NEXT:    s_cselect_b32 s1, s1, s2
-; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s3, s1
-; GFX9-NEXT:    s_lshr_b32 s2, s0, 16
-; GFX9-NEXT:    s_lshr_b32 s3, s1, 16
-; GFX9-NEXT:    s_sub_i32 s0, s0, s1
-; GFX9-NEXT:    s_sub_i32 s1, s2, s3
-; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s1
+; GFX9-NEXT:    v_mov_b32_e32 v0, s1
+; GFX9-NEXT:    v_pk_sub_i16 v0, s0, v0 clamp
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_ssubsat_v2i16:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_sext_i32_i16 s2, s0
-; GFX10-NEXT:    s_sext_i32_i16 s3, -1
-; GFX10-NEXT:    s_ashr_i32 s4, s0, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s2, s3
+; GFX10-NEXT:    v_pk_sub_i16 v0, s0, s1 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cselect_b32 s5, s2, s3
-; GFX10-NEXT:    s_cmp_gt_i32 s4, -1
-; GFX10-NEXT:    s_cselect_b32 s6, s4, -1
-; GFX10-NEXT:    s_pack_ll_b32_b16 s5, s5, s6
-; GFX10-NEXT:    s_sext_i32_i16 s6, s1
-; GFX10-NEXT:    s_lshr_b32 s7, s5, 16
-; GFX10-NEXT:    s_sub_i32 s5, s5, 0x7fff7fff
-; GFX10-NEXT:    s_sub_i32 s7, s7, 0x7fff
-; GFX10-NEXT:    s_cmp_lt_i32 s2, s3
-; GFX10-NEXT:    s_pack_ll_b32_b16 s5, s5, s7
-; GFX10-NEXT:    s_cselect_b32 s2, s2, s3
-; GFX10-NEXT:    s_cmp_lt_i32 s4, -1
-; GFX10-NEXT:    s_sext_i32_i16 s3, s5
-; GFX10-NEXT:    s_cselect_b32 s4, s4, -1
-; GFX10-NEXT:    s_ashr_i32 s5, s5, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s2, s4
-; GFX10-NEXT:    s_ashr_i32 s1, s1, 16
-; GFX10-NEXT:    s_lshr_b32 s4, s2, 16
-; GFX10-NEXT:    s_sub_i32 s2, s2, 0x80008000
-; GFX10-NEXT:    s_sub_i32 s4, s4, 0x8000
-; GFX10-NEXT:    s_cmp_gt_i32 s3, s6
-; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s2, s4
-; GFX10-NEXT:    s_cselect_b32 s3, s3, s6
-; GFX10-NEXT:    s_cmp_gt_i32 s5, s1
-; GFX10-NEXT:    s_sext_i32_i16 s4, s2
-; GFX10-NEXT:    s_cselect_b32 s1, s5, s1
-; GFX10-NEXT:    s_ashr_i32 s2, s2, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s3, s1
-; GFX10-NEXT:    s_sext_i32_i16 s3, s1
-; GFX10-NEXT:    s_ashr_i32 s1, s1, 16
-; GFX10-NEXT:    s_cmp_lt_i32 s3, s4
-; GFX10-NEXT:    s_cselect_b32 s3, s3, s4
-; GFX10-NEXT:    s_cmp_lt_i32 s1, s2
-; GFX10-NEXT:    s_cselect_b32 s1, s1, s2
-; GFX10-NEXT:    s_lshr_b32 s2, s0, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s3, s1
-; GFX10-NEXT:    s_lshr_b32 s3, s1, 16
-; GFX10-NEXT:    s_sub_i32 s0, s0, s1
-; GFX10-NEXT:    s_sub_i32 s1, s2, s3
-; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s1
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <2 x i16> @llvm.ssub.sat.v2i16(<2 x i16> %lhs, <2 x i16> %rhs)
   %cast = bitcast <2 x i16> %result to i32
@@ -4795,59 +3239,13 @@ define amdgpu_ps float @ssubsat_v2i16_sv(<2 x i16> inreg %lhs, <2 x i16> %rhs) {
 ;
 ; GFX9-LABEL: ssubsat_v2i16_sv:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_sext_i32_i16 s1, s0
-; GFX9-NEXT:    s_ashr_i32 s2, s0, 16
-; GFX9-NEXT:    s_sext_i32_i16 s3, -1
-; GFX9-NEXT:    s_cmp_gt_i32 s1, s3
-; GFX9-NEXT:    s_cselect_b32 s4, s1, s3
-; GFX9-NEXT:    s_cmp_gt_i32 s2, -1
-; GFX9-NEXT:    s_cselect_b32 s5, s2, -1
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
-; GFX9-NEXT:    s_lshr_b32 s5, s4, 16
-; GFX9-NEXT:    s_sub_i32 s4, s4, 0x7fff7fff
-; GFX9-NEXT:    s_sub_i32 s5, s5, 0x7fff
-; GFX9-NEXT:    s_cmp_lt_i32 s1, s3
-; GFX9-NEXT:    s_cselect_b32 s1, s1, s3
-; GFX9-NEXT:    s_cmp_lt_i32 s2, -1
-; GFX9-NEXT:    s_cselect_b32 s2, s2, -1
-; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s2
-; GFX9-NEXT:    s_lshr_b32 s2, s1, 16
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
-; GFX9-NEXT:    s_sub_i32 s1, s1, 0x80008000
-; GFX9-NEXT:    s_sub_i32 s2, s2, 0x8000
-; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s2
-; GFX9-NEXT:    v_pk_max_i16 v0, s4, v0
-; GFX9-NEXT:    v_pk_min_i16 v0, v0, s1
-; GFX9-NEXT:    v_pk_sub_i16 v0, s0, v0
+; GFX9-NEXT:    v_pk_sub_i16 v0, s0, v0 clamp
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: ssubsat_v2i16_sv:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_sext_i32_i16 s1, s0
-; GFX10-NEXT:    s_sext_i32_i16 s2, -1
-; GFX10-NEXT:    s_ashr_i32 s3, s0, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s1, s2
+; GFX10-NEXT:    v_pk_sub_i16 v0, s0, v0 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cselect_b32 s4, s1, s2
-; GFX10-NEXT:    s_cmp_gt_i32 s3, -1
-; GFX10-NEXT:    s_cselect_b32 s5, s3, -1
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
-; GFX10-NEXT:    s_lshr_b32 s5, s4, 16
-; GFX10-NEXT:    s_sub_i32 s4, s4, 0x7fff7fff
-; GFX10-NEXT:    s_sub_i32 s5, s5, 0x7fff
-; GFX10-NEXT:    s_cmp_lt_i32 s1, s2
-; GFX10-NEXT:    s_cselect_b32 s1, s1, s2
-; GFX10-NEXT:    s_cmp_lt_i32 s3, -1
-; GFX10-NEXT:    s_cselect_b32 s2, s3, -1
-; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s1, s2
-; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s4, s5
-; GFX10-NEXT:    s_lshr_b32 s3, s1, 16
-; GFX10-NEXT:    v_pk_max_i16 v0, s2, v0
-; GFX10-NEXT:    s_sub_i32 s1, s1, 0x80008000
-; GFX10-NEXT:    s_sub_i32 s2, s3, 0x8000
-; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s1, s2
-; GFX10-NEXT:    v_pk_min_i16 v0, v0, s1
-; GFX10-NEXT:    v_pk_sub_i16 v0, s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <2 x i16> @llvm.ssub.sat.v2i16(<2 x i16> %lhs, <2 x i16> %rhs)
   %cast = bitcast <2 x i16> %result to float
@@ -4911,27 +3309,13 @@ define amdgpu_ps float @ssubsat_v2i16_vs(<2 x i16> %lhs, <2 x i16> inreg %rhs) {
 ;
 ; GFX9-LABEL: ssubsat_v2i16_vs:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    v_pk_max_i16 v1, v0, -1 op_sel_hi:[1,0]
-; GFX9-NEXT:    v_mov_b32_e32 v2, 0x7fff7fff
-; GFX9-NEXT:    v_pk_sub_i16 v1, v1, v2
-; GFX9-NEXT:    v_pk_min_i16 v2, v0, -1 op_sel_hi:[1,0]
-; GFX9-NEXT:    v_mov_b32_e32 v3, 0x80008000
-; GFX9-NEXT:    v_pk_sub_i16 v2, v2, v3
-; GFX9-NEXT:    v_pk_max_i16 v1, v1, s0
-; GFX9-NEXT:    v_pk_min_i16 v1, v1, v2
-; GFX9-NEXT:    v_pk_sub_i16 v0, v0, v1
+; GFX9-NEXT:    v_pk_sub_i16 v0, v0, s0 clamp
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: ssubsat_v2i16_vs:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    v_pk_max_i16 v1, v0, -1 op_sel_hi:[1,0]
-; GFX10-NEXT:    v_pk_min_i16 v2, v0, -1 op_sel_hi:[1,0]
+; GFX10-NEXT:    v_pk_sub_i16 v0, v0, s0 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_pk_sub_i16 v1, v1, 0x7fff7fff
-; GFX10-NEXT:    v_pk_sub_i16 v2, v2, 0x80008000
-; GFX10-NEXT:    v_pk_max_i16 v1, v1, s0
-; GFX10-NEXT:    v_pk_min_i16 v1, v1, v2
-; GFX10-NEXT:    v_pk_sub_i16 v0, v0, v1
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <2 x i16> @llvm.ssub.sat.v2i16(<2 x i16> %lhs, <2 x i16> %rhs)
   %cast = bitcast <2 x i16> %result to float
@@ -5050,43 +3434,17 @@ define <2 x float> @v_ssubsat_v4i16(<4 x i16> %lhs, <4 x i16> %rhs) {
 ; GFX9-LABEL: v_ssubsat_v4i16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_pk_max_i16 v4, v0, -1 op_sel_hi:[1,0]
-; GFX9-NEXT:    v_mov_b32_e32 v5, 0x7fff7fff
-; GFX9-NEXT:    v_pk_sub_i16 v4, v4, v5
-; GFX9-NEXT:    v_pk_min_i16 v6, v0, -1 op_sel_hi:[1,0]
-; GFX9-NEXT:    v_mov_b32_e32 v7, 0x80008000
-; GFX9-NEXT:    v_pk_max_i16 v2, v4, v2
-; GFX9-NEXT:    v_pk_sub_i16 v6, v6, v7
-; GFX9-NEXT:    v_pk_min_i16 v2, v2, v6
-; GFX9-NEXT:    v_pk_sub_i16 v0, v0, v2
-; GFX9-NEXT:    v_pk_max_i16 v2, v1, -1 op_sel_hi:[1,0]
-; GFX9-NEXT:    v_pk_sub_i16 v2, v2, v5
-; GFX9-NEXT:    v_pk_min_i16 v4, v1, -1 op_sel_hi:[1,0]
-; GFX9-NEXT:    v_pk_sub_i16 v4, v4, v7
-; GFX9-NEXT:    v_pk_max_i16 v2, v2, v3
-; GFX9-NEXT:    v_pk_min_i16 v2, v2, v4
-; GFX9-NEXT:    v_pk_sub_i16 v1, v1, v2
+; GFX9-NEXT:    v_pk_sub_i16 v0, v0, v2 clamp
+; GFX9-NEXT:    v_pk_sub_i16 v1, v1, v3 clamp
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_ssubsat_v4i16:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_pk_max_i16 v4, v0, -1 op_sel_hi:[1,0]
-; GFX10-NEXT:    v_pk_max_i16 v5, v1, -1 op_sel_hi:[1,0]
-; GFX10-NEXT:    v_pk_min_i16 v6, v0, -1 op_sel_hi:[1,0]
-; GFX10-NEXT:    v_pk_min_i16 v7, v1, -1 op_sel_hi:[1,0]
+; GFX10-NEXT:    v_pk_sub_i16 v0, v0, v2 clamp
+; GFX10-NEXT:    v_pk_sub_i16 v1, v1, v3 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_pk_sub_i16 v4, v4, 0x7fff7fff
-; GFX10-NEXT:    v_pk_sub_i16 v5, v5, 0x7fff7fff
-; GFX10-NEXT:    v_pk_sub_i16 v6, v6, 0x80008000
-; GFX10-NEXT:    v_pk_sub_i16 v7, v7, 0x80008000
-; GFX10-NEXT:    v_pk_max_i16 v11, v4, v2
-; GFX10-NEXT:    v_pk_max_i16 v10, v5, v3
-; GFX10-NEXT:    v_pk_min_i16 v2, v11, v6
-; GFX10-NEXT:    v_pk_min_i16 v3, v10, v7
-; GFX10-NEXT:    v_pk_sub_i16 v0, v0, v2
-; GFX10-NEXT:    v_pk_sub_i16 v1, v1, v3
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <4 x i16> @llvm.ssub.sat.v4i16(<4 x i16> %lhs, <4 x i16> %rhs)
   %cast = bitcast <4 x i16> %result to <2 x float>
@@ -5250,193 +3608,21 @@ define amdgpu_ps <2 x i32> @s_ssubsat_v4i16(<4 x i16> inreg %lhs, <4 x i16> inre
 ;
 ; GFX9-LABEL: s_ssubsat_v4i16:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_sext_i32_i16 s6, s0
-; GFX9-NEXT:    s_ashr_i32 s7, s0, 16
-; GFX9-NEXT:    s_sext_i32_i16 s8, -1
-; GFX9-NEXT:    s_cmp_gt_i32 s6, s8
-; GFX9-NEXT:    s_cselect_b32 s9, s6, s8
-; GFX9-NEXT:    s_cmp_gt_i32 s7, -1
-; GFX9-NEXT:    s_cselect_b32 s10, s7, -1
-; GFX9-NEXT:    s_pack_ll_b32_b16 s9, s9, s10
-; GFX9-NEXT:    s_mov_b32 s4, 0x7fff7fff
-; GFX9-NEXT:    s_lshr_b32 s10, s9, 16
-; GFX9-NEXT:    s_movk_i32 s11, 0x7fff
-; GFX9-NEXT:    s_sub_i32 s9, s9, s4
-; GFX9-NEXT:    s_sub_i32 s10, s10, s11
-; GFX9-NEXT:    s_cmp_lt_i32 s6, s8
-; GFX9-NEXT:    s_cselect_b32 s6, s6, s8
-; GFX9-NEXT:    s_cmp_lt_i32 s7, -1
-; GFX9-NEXT:    s_cselect_b32 s7, s7, -1
-; GFX9-NEXT:    s_pack_ll_b32_b16 s6, s6, s7
-; GFX9-NEXT:    s_pack_ll_b32_b16 s9, s9, s10
-; GFX9-NEXT:    s_mov_b32 s5, 0x80008000
-; GFX9-NEXT:    s_lshr_b32 s7, s6, 16
-; GFX9-NEXT:    s_mov_b32 s10, 0x8000
-; GFX9-NEXT:    s_sub_i32 s6, s6, s5
-; GFX9-NEXT:    s_sub_i32 s7, s7, s10
-; GFX9-NEXT:    s_pack_ll_b32_b16 s6, s6, s7
-; GFX9-NEXT:    s_sext_i32_i16 s7, s9
-; GFX9-NEXT:    s_sext_i32_i16 s12, s2
-; GFX9-NEXT:    s_ashr_i32 s9, s9, 16
-; GFX9-NEXT:    s_ashr_i32 s2, s2, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s7, s12
-; GFX9-NEXT:    s_cselect_b32 s7, s7, s12
-; GFX9-NEXT:    s_cmp_gt_i32 s9, s2
-; GFX9-NEXT:    s_cselect_b32 s2, s9, s2
-; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s7, s2
-; GFX9-NEXT:    s_sext_i32_i16 s7, s2
-; GFX9-NEXT:    s_sext_i32_i16 s9, s6
-; GFX9-NEXT:    s_ashr_i32 s2, s2, 16
-; GFX9-NEXT:    s_ashr_i32 s6, s6, 16
-; GFX9-NEXT:    s_cmp_lt_i32 s7, s9
-; GFX9-NEXT:    s_cselect_b32 s7, s7, s9
-; GFX9-NEXT:    s_cmp_lt_i32 s2, s6
-; GFX9-NEXT:    s_cselect_b32 s2, s2, s6
-; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s7, s2
-; GFX9-NEXT:    s_lshr_b32 s6, s0, 16
-; GFX9-NEXT:    s_lshr_b32 s7, s2, 16
-; GFX9-NEXT:    s_sub_i32 s0, s0, s2
-; GFX9-NEXT:    s_sub_i32 s2, s6, s7
-; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s2
-; GFX9-NEXT:    s_sext_i32_i16 s2, s1
-; GFX9-NEXT:    s_ashr_i32 s6, s1, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s2, s8
-; GFX9-NEXT:    s_cselect_b32 s7, s2, s8
-; GFX9-NEXT:    s_cmp_gt_i32 s6, -1
-; GFX9-NEXT:    s_cselect_b32 s9, s6, -1
-; GFX9-NEXT:    s_pack_ll_b32_b16 s7, s7, s9
-; GFX9-NEXT:    s_lshr_b32 s9, s7, 16
-; GFX9-NEXT:    s_sub_i32 s4, s7, s4
-; GFX9-NEXT:    s_sub_i32 s7, s9, s11
-; GFX9-NEXT:    s_cmp_lt_i32 s2, s8
-; GFX9-NEXT:    s_cselect_b32 s2, s2, s8
-; GFX9-NEXT:    s_cmp_lt_i32 s6, -1
-; GFX9-NEXT:    s_cselect_b32 s6, s6, -1
-; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s6
-; GFX9-NEXT:    s_lshr_b32 s6, s2, 16
-; GFX9-NEXT:    s_sub_i32 s2, s2, s5
-; GFX9-NEXT:    s_sub_i32 s5, s6, s10
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s7
-; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s5
-; GFX9-NEXT:    s_sext_i32_i16 s5, s4
-; GFX9-NEXT:    s_sext_i32_i16 s6, s3
-; GFX9-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX9-NEXT:    s_ashr_i32 s3, s3, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s5, s6
-; GFX9-NEXT:    s_cselect_b32 s5, s5, s6
-; GFX9-NEXT:    s_cmp_gt_i32 s4, s3
-; GFX9-NEXT:    s_cselect_b32 s3, s4, s3
-; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s5, s3
-; GFX9-NEXT:    s_sext_i32_i16 s4, s3
-; GFX9-NEXT:    s_sext_i32_i16 s5, s2
-; GFX9-NEXT:    s_ashr_i32 s3, s3, 16
-; GFX9-NEXT:    s_ashr_i32 s2, s2, 16
-; GFX9-NEXT:    s_cmp_lt_i32 s4, s5
-; GFX9-NEXT:    s_cselect_b32 s4, s4, s5
-; GFX9-NEXT:    s_cmp_lt_i32 s3, s2
-; GFX9-NEXT:    s_cselect_b32 s2, s3, s2
-; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s4, s2
-; GFX9-NEXT:    s_lshr_b32 s3, s1, 16
-; GFX9-NEXT:    s_lshr_b32 s4, s2, 16
-; GFX9-NEXT:    s_sub_i32 s1, s1, s2
-; GFX9-NEXT:    s_sub_i32 s2, s3, s4
-; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s2
+; GFX9-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    v_pk_sub_i16 v0, s0, v0 clamp
+; GFX9-NEXT:    v_pk_sub_i16 v1, s1, v1 clamp
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_ssubsat_v4i16:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_sext_i32_i16 s4, s0
-; GFX10-NEXT:    s_sext_i32_i16 s5, -1
-; GFX10-NEXT:    s_ashr_i32 s6, s0, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s4, s5
-; GFX10-NEXT:    s_movk_i32 s10, 0x7fff
-; GFX10-NEXT:    s_cselect_b32 s7, s4, s5
-; GFX10-NEXT:    s_cmp_gt_i32 s6, -1
-; GFX10-NEXT:    s_mov_b32 s11, 0x80008000
-; GFX10-NEXT:    s_cselect_b32 s8, s6, -1
-; GFX10-NEXT:    s_sext_i32_i16 s13, s2
-; GFX10-NEXT:    s_pack_ll_b32_b16 s7, s7, s8
-; GFX10-NEXT:    s_mov_b32 s8, 0x7fff7fff
-; GFX10-NEXT:    s_lshr_b32 s9, s7, 16
-; GFX10-NEXT:    s_sub_i32 s7, s7, s8
-; GFX10-NEXT:    s_sub_i32 s9, s9, s10
-; GFX10-NEXT:    s_cmp_lt_i32 s4, s5
+; GFX10-NEXT:    v_pk_sub_i16 v0, s0, s2 clamp
+; GFX10-NEXT:    v_pk_sub_i16 v1, s1, s3 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cselect_b32 s4, s4, s5
-; GFX10-NEXT:    s_cmp_lt_i32 s6, -1
-; GFX10-NEXT:    s_cselect_b32 s6, s6, -1
-; GFX10-NEXT:    s_ashr_i32 s2, s2, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s4, s6
-; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s7, s9
-; GFX10-NEXT:    s_lshr_b32 s7, s4, 16
-; GFX10-NEXT:    s_mov_b32 s9, 0x8000
-; GFX10-NEXT:    s_sext_i32_i16 s12, s6
-; GFX10-NEXT:    s_sub_i32 s4, s4, s11
-; GFX10-NEXT:    s_sub_i32 s7, s7, s9
-; GFX10-NEXT:    s_ashr_i32 s6, s6, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s12, s13
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s4, s7
-; GFX10-NEXT:    s_cselect_b32 s12, s12, s13
-; GFX10-NEXT:    s_cmp_gt_i32 s6, s2
-; GFX10-NEXT:    s_sext_i32_i16 s7, s4
-; GFX10-NEXT:    s_cselect_b32 s2, s6, s2
-; GFX10-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s12, s2
-; GFX10-NEXT:    s_sext_i32_i16 s6, s2
-; GFX10-NEXT:    s_ashr_i32 s2, s2, 16
-; GFX10-NEXT:    s_cmp_lt_i32 s6, s7
-; GFX10-NEXT:    s_cselect_b32 s6, s6, s7
-; GFX10-NEXT:    s_cmp_lt_i32 s2, s4
-; GFX10-NEXT:    s_cselect_b32 s2, s2, s4
-; GFX10-NEXT:    s_lshr_b32 s4, s0, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s6, s2
-; GFX10-NEXT:    s_lshr_b32 s6, s2, 16
-; GFX10-NEXT:    s_sub_i32 s0, s0, s2
-; GFX10-NEXT:    s_sub_i32 s2, s4, s6
-; GFX10-NEXT:    s_sext_i32_i16 s4, s1
-; GFX10-NEXT:    s_ashr_i32 s6, s1, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s4, s5
-; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s2
-; GFX10-NEXT:    s_cselect_b32 s7, s4, s5
-; GFX10-NEXT:    s_cmp_gt_i32 s6, -1
-; GFX10-NEXT:    s_cselect_b32 s12, s6, -1
-; GFX10-NEXT:    s_pack_ll_b32_b16 s7, s7, s12
-; GFX10-NEXT:    s_lshr_b32 s12, s7, 16
-; GFX10-NEXT:    s_sub_i32 s7, s7, s8
-; GFX10-NEXT:    s_sub_i32 s8, s12, s10
-; GFX10-NEXT:    s_cmp_lt_i32 s4, s5
-; GFX10-NEXT:    s_cselect_b32 s4, s4, s5
-; GFX10-NEXT:    s_cmp_lt_i32 s6, -1
-; GFX10-NEXT:    s_cselect_b32 s5, s6, -1
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
-; GFX10-NEXT:    s_pack_ll_b32_b16 s5, s7, s8
-; GFX10-NEXT:    s_lshr_b32 s6, s4, 16
-; GFX10-NEXT:    s_sext_i32_i16 s7, s5
-; GFX10-NEXT:    s_sext_i32_i16 s8, s3
-; GFX10-NEXT:    s_sub_i32 s4, s4, s11
-; GFX10-NEXT:    s_sub_i32 s6, s6, s9
-; GFX10-NEXT:    s_ashr_i32 s5, s5, 16
-; GFX10-NEXT:    s_ashr_i32 s3, s3, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s7, s8
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s4, s6
-; GFX10-NEXT:    s_cselect_b32 s7, s7, s8
-; GFX10-NEXT:    s_cmp_gt_i32 s5, s3
-; GFX10-NEXT:    s_sext_i32_i16 s6, s4
-; GFX10-NEXT:    s_cselect_b32 s3, s5, s3
-; GFX10-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s7, s3
-; GFX10-NEXT:    s_sext_i32_i16 s5, s3
-; GFX10-NEXT:    s_ashr_i32 s3, s3, 16
-; GFX10-NEXT:    s_cmp_lt_i32 s5, s6
-; GFX10-NEXT:    s_cselect_b32 s5, s5, s6
-; GFX10-NEXT:    s_cmp_lt_i32 s3, s4
-; GFX10-NEXT:    s_cselect_b32 s3, s3, s4
-; GFX10-NEXT:    s_lshr_b32 s4, s1, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s5, s3
-; GFX10-NEXT:    s_lshr_b32 s5, s3, 16
-; GFX10-NEXT:    s_sub_i32 s1, s1, s3
-; GFX10-NEXT:    s_sub_i32 s3, s4, s5
-; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s1, s3
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <4 x i16> @llvm.ssub.sat.v4i16(<4 x i16> %lhs, <4 x i16> %rhs)
   %cast = bitcast <4 x i16> %result to <2 x i32>
@@ -5597,57 +3783,19 @@ define <3 x float> @v_ssubsat_v6i16(<6 x i16> %lhs, <6 x i16> %rhs) {
 ; GFX9-LABEL: v_ssubsat_v6i16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_pk_max_i16 v6, v0, -1 op_sel_hi:[1,0]
-; GFX9-NEXT:    v_mov_b32_e32 v7, 0x7fff7fff
-; GFX9-NEXT:    v_pk_sub_i16 v6, v6, v7
-; GFX9-NEXT:    v_pk_min_i16 v8, v0, -1 op_sel_hi:[1,0]
-; GFX9-NEXT:    v_mov_b32_e32 v9, 0x80008000
-; GFX9-NEXT:    v_pk_max_i16 v3, v6, v3
-; GFX9-NEXT:    v_pk_sub_i16 v8, v8, v9
-; GFX9-NEXT:    v_pk_min_i16 v3, v3, v8
-; GFX9-NEXT:    v_pk_sub_i16 v0, v0, v3
-; GFX9-NEXT:    v_pk_max_i16 v3, v1, -1 op_sel_hi:[1,0]
-; GFX9-NEXT:    v_pk_sub_i16 v3, v3, v7
-; GFX9-NEXT:    v_pk_min_i16 v6, v1, -1 op_sel_hi:[1,0]
-; GFX9-NEXT:    v_pk_max_i16 v3, v3, v4
-; GFX9-NEXT:    v_pk_sub_i16 v6, v6, v9
-; GFX9-NEXT:    v_pk_min_i16 v3, v3, v6
-; GFX9-NEXT:    v_pk_sub_i16 v1, v1, v3
-; GFX9-NEXT:    v_pk_max_i16 v3, v2, -1 op_sel_hi:[1,0]
-; GFX9-NEXT:    v_pk_sub_i16 v3, v3, v7
-; GFX9-NEXT:    v_pk_min_i16 v4, v2, -1 op_sel_hi:[1,0]
-; GFX9-NEXT:    v_pk_sub_i16 v4, v4, v9
-; GFX9-NEXT:    v_pk_max_i16 v3, v3, v5
-; GFX9-NEXT:    v_pk_min_i16 v3, v3, v4
-; GFX9-NEXT:    v_pk_sub_i16 v2, v2, v3
+; GFX9-NEXT:    v_pk_sub_i16 v0, v0, v3 clamp
+; GFX9-NEXT:    v_pk_sub_i16 v1, v1, v4 clamp
+; GFX9-NEXT:    v_pk_sub_i16 v2, v2, v5 clamp
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_ssubsat_v6i16:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_pk_max_i16 v6, v0, -1 op_sel_hi:[1,0]
-; GFX10-NEXT:    v_pk_max_i16 v8, v1, -1 op_sel_hi:[1,0]
-; GFX10-NEXT:    v_pk_max_i16 v9, v2, -1 op_sel_hi:[1,0]
-; GFX10-NEXT:    v_pk_min_i16 v7, v0, -1 op_sel_hi:[1,0]
-; GFX10-NEXT:    v_pk_min_i16 v10, v1, -1 op_sel_hi:[1,0]
-; GFX10-NEXT:    v_pk_sub_i16 v6, v6, 0x7fff7fff
-; GFX10-NEXT:    v_pk_sub_i16 v15, v8, 0x7fff7fff
-; GFX10-NEXT:    v_pk_min_i16 v11, v2, -1 op_sel_hi:[1,0]
-; GFX10-NEXT:    v_pk_sub_i16 v19, v9, 0x7fff7fff
-; GFX10-NEXT:    v_pk_sub_i16 v7, v7, 0x80008000
-; GFX10-NEXT:    v_pk_max_i16 v14, v6, v3
-; GFX10-NEXT:    v_pk_sub_i16 v6, v10, 0x80008000
-; GFX10-NEXT:    v_pk_max_i16 v4, v15, v4
-; GFX10-NEXT:    v_pk_sub_i16 v8, v11, 0x80008000
-; GFX10-NEXT:    v_pk_max_i16 v5, v19, v5
-; GFX10-NEXT:    v_pk_min_i16 v3, v14, v7
+; GFX10-NEXT:    v_pk_sub_i16 v0, v0, v3 clamp
+; GFX10-NEXT:    v_pk_sub_i16 v1, v1, v4 clamp
+; GFX10-NEXT:    v_pk_sub_i16 v2, v2, v5 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_pk_min_i16 v4, v4, v6
-; GFX10-NEXT:    v_pk_min_i16 v5, v5, v8
-; GFX10-NEXT:    v_pk_sub_i16 v0, v0, v3
-; GFX10-NEXT:    v_pk_sub_i16 v1, v1, v4
-; GFX10-NEXT:    v_pk_sub_i16 v2, v2, v5
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <6 x i16> @llvm.ssub.sat.v6i16(<6 x i16> %lhs, <6 x i16> %rhs)
   %cast = bitcast <6 x i16> %result to <3 x float>
@@ -5881,279 +4029,26 @@ define amdgpu_ps <3 x i32> @s_ssubsat_v6i16(<6 x i16> inreg %lhs, <6 x i16> inre
 ;
 ; GFX9-LABEL: s_ssubsat_v6i16:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_sext_i32_i16 s8, s0
-; GFX9-NEXT:    s_ashr_i32 s9, s0, 16
-; GFX9-NEXT:    s_sext_i32_i16 s10, -1
-; GFX9-NEXT:    s_cmp_gt_i32 s8, s10
-; GFX9-NEXT:    s_cselect_b32 s11, s8, s10
-; GFX9-NEXT:    s_cmp_gt_i32 s9, -1
-; GFX9-NEXT:    s_cselect_b32 s12, s9, -1
-; GFX9-NEXT:    s_pack_ll_b32_b16 s11, s11, s12
-; GFX9-NEXT:    s_mov_b32 s6, 0x7fff7fff
-; GFX9-NEXT:    s_lshr_b32 s12, s11, 16
-; GFX9-NEXT:    s_movk_i32 s13, 0x7fff
-; GFX9-NEXT:    s_sub_i32 s11, s11, s6
-; GFX9-NEXT:    s_sub_i32 s12, s12, s13
-; GFX9-NEXT:    s_cmp_lt_i32 s8, s10
-; GFX9-NEXT:    s_cselect_b32 s8, s8, s10
-; GFX9-NEXT:    s_cmp_lt_i32 s9, -1
-; GFX9-NEXT:    s_cselect_b32 s9, s9, -1
-; GFX9-NEXT:    s_pack_ll_b32_b16 s8, s8, s9
-; GFX9-NEXT:    s_pack_ll_b32_b16 s11, s11, s12
-; GFX9-NEXT:    s_mov_b32 s7, 0x80008000
-; GFX9-NEXT:    s_lshr_b32 s9, s8, 16
-; GFX9-NEXT:    s_mov_b32 s12, 0x8000
-; GFX9-NEXT:    s_sub_i32 s8, s8, s7
-; GFX9-NEXT:    s_sub_i32 s9, s9, s12
-; GFX9-NEXT:    s_pack_ll_b32_b16 s8, s8, s9
-; GFX9-NEXT:    s_sext_i32_i16 s9, s11
-; GFX9-NEXT:    s_sext_i32_i16 s14, s3
-; GFX9-NEXT:    s_ashr_i32 s11, s11, 16
-; GFX9-NEXT:    s_ashr_i32 s3, s3, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s9, s14
-; GFX9-NEXT:    s_cselect_b32 s9, s9, s14
-; GFX9-NEXT:    s_cmp_gt_i32 s11, s3
-; GFX9-NEXT:    s_cselect_b32 s3, s11, s3
-; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s9, s3
-; GFX9-NEXT:    s_sext_i32_i16 s9, s3
-; GFX9-NEXT:    s_sext_i32_i16 s11, s8
-; GFX9-NEXT:    s_ashr_i32 s3, s3, 16
-; GFX9-NEXT:    s_ashr_i32 s8, s8, 16
-; GFX9-NEXT:    s_cmp_lt_i32 s9, s11
-; GFX9-NEXT:    s_cselect_b32 s9, s9, s11
-; GFX9-NEXT:    s_cmp_lt_i32 s3, s8
-; GFX9-NEXT:    s_cselect_b32 s3, s3, s8
-; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s9, s3
-; GFX9-NEXT:    s_lshr_b32 s8, s0, 16
-; GFX9-NEXT:    s_lshr_b32 s9, s3, 16
-; GFX9-NEXT:    s_sub_i32 s0, s0, s3
-; GFX9-NEXT:    s_sub_i32 s3, s8, s9
-; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s3
-; GFX9-NEXT:    s_sext_i32_i16 s3, s1
-; GFX9-NEXT:    s_ashr_i32 s8, s1, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s3, s10
-; GFX9-NEXT:    s_cselect_b32 s9, s3, s10
-; GFX9-NEXT:    s_cmp_gt_i32 s8, -1
-; GFX9-NEXT:    s_cselect_b32 s11, s8, -1
-; GFX9-NEXT:    s_pack_ll_b32_b16 s9, s9, s11
-; GFX9-NEXT:    s_lshr_b32 s11, s9, 16
-; GFX9-NEXT:    s_sub_i32 s9, s9, s6
-; GFX9-NEXT:    s_sub_i32 s11, s11, s13
-; GFX9-NEXT:    s_cmp_lt_i32 s3, s10
-; GFX9-NEXT:    s_cselect_b32 s3, s3, s10
-; GFX9-NEXT:    s_cmp_lt_i32 s8, -1
-; GFX9-NEXT:    s_cselect_b32 s8, s8, -1
-; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s3, s8
-; GFX9-NEXT:    s_lshr_b32 s8, s3, 16
-; GFX9-NEXT:    s_pack_ll_b32_b16 s9, s9, s11
-; GFX9-NEXT:    s_sub_i32 s3, s3, s7
-; GFX9-NEXT:    s_sub_i32 s8, s8, s12
-; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s3, s8
-; GFX9-NEXT:    s_sext_i32_i16 s8, s9
-; GFX9-NEXT:    s_sext_i32_i16 s11, s4
-; GFX9-NEXT:    s_ashr_i32 s9, s9, 16
-; GFX9-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s8, s11
-; GFX9-NEXT:    s_cselect_b32 s8, s8, s11
-; GFX9-NEXT:    s_cmp_gt_i32 s9, s4
-; GFX9-NEXT:    s_cselect_b32 s4, s9, s4
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s8, s4
-; GFX9-NEXT:    s_sext_i32_i16 s8, s4
-; GFX9-NEXT:    s_sext_i32_i16 s9, s3
-; GFX9-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX9-NEXT:    s_ashr_i32 s3, s3, 16
-; GFX9-NEXT:    s_cmp_lt_i32 s8, s9
-; GFX9-NEXT:    s_cselect_b32 s8, s8, s9
-; GFX9-NEXT:    s_cmp_lt_i32 s4, s3
-; GFX9-NEXT:    s_cselect_b32 s3, s4, s3
-; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s8, s3
-; GFX9-NEXT:    s_lshr_b32 s4, s1, 16
-; GFX9-NEXT:    s_lshr_b32 s8, s3, 16
-; GFX9-NEXT:    s_sub_i32 s1, s1, s3
-; GFX9-NEXT:    s_sub_i32 s3, s4, s8
-; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s3
-; GFX9-NEXT:    s_sext_i32_i16 s3, s2
-; GFX9-NEXT:    s_ashr_i32 s4, s2, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s3, s10
-; GFX9-NEXT:    s_cselect_b32 s8, s3, s10
-; GFX9-NEXT:    s_cmp_gt_i32 s4, -1
-; GFX9-NEXT:    s_cselect_b32 s9, s4, -1
-; GFX9-NEXT:    s_pack_ll_b32_b16 s8, s8, s9
-; GFX9-NEXT:    s_lshr_b32 s9, s8, 16
-; GFX9-NEXT:    s_sub_i32 s6, s8, s6
-; GFX9-NEXT:    s_sub_i32 s8, s9, s13
-; GFX9-NEXT:    s_cmp_lt_i32 s3, s10
-; GFX9-NEXT:    s_cselect_b32 s3, s3, s10
-; GFX9-NEXT:    s_cmp_lt_i32 s4, -1
-; GFX9-NEXT:    s_cselect_b32 s4, s4, -1
-; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s3, s4
-; GFX9-NEXT:    s_lshr_b32 s4, s3, 16
-; GFX9-NEXT:    s_sub_i32 s3, s3, s7
-; GFX9-NEXT:    s_sub_i32 s4, s4, s12
-; GFX9-NEXT:    s_pack_ll_b32_b16 s6, s6, s8
-; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s3, s4
-; GFX9-NEXT:    s_sext_i32_i16 s4, s6
-; GFX9-NEXT:    s_sext_i32_i16 s7, s5
-; GFX9-NEXT:    s_ashr_i32 s6, s6, 16
-; GFX9-NEXT:    s_ashr_i32 s5, s5, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s4, s7
-; GFX9-NEXT:    s_cselect_b32 s4, s4, s7
-; GFX9-NEXT:    s_cmp_gt_i32 s6, s5
-; GFX9-NEXT:    s_cselect_b32 s5, s6, s5
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
-; GFX9-NEXT:    s_sext_i32_i16 s5, s4
-; GFX9-NEXT:    s_sext_i32_i16 s6, s3
-; GFX9-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX9-NEXT:    s_ashr_i32 s3, s3, 16
-; GFX9-NEXT:    s_cmp_lt_i32 s5, s6
-; GFX9-NEXT:    s_cselect_b32 s5, s5, s6
-; GFX9-NEXT:    s_cmp_lt_i32 s4, s3
-; GFX9-NEXT:    s_cselect_b32 s3, s4, s3
-; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s5, s3
-; GFX9-NEXT:    s_lshr_b32 s4, s2, 16
-; GFX9-NEXT:    s_lshr_b32 s5, s3, 16
-; GFX9-NEXT:    s_sub_i32 s2, s2, s3
-; GFX9-NEXT:    s_sub_i32 s3, s4, s5
-; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s3
+; GFX9-NEXT:    v_mov_b32_e32 v0, s3
+; GFX9-NEXT:    v_mov_b32_e32 v1, s4
+; GFX9-NEXT:    v_mov_b32_e32 v2, s5
+; GFX9-NEXT:    v_pk_sub_i16 v0, s0, v0 clamp
+; GFX9-NEXT:    v_pk_sub_i16 v1, s1, v1 clamp
+; GFX9-NEXT:    v_pk_sub_i16 v2, s2, v2 clamp
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v2
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_ssubsat_v6i16:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_sext_i32_i16 s6, s0
-; GFX10-NEXT:    s_sext_i32_i16 s7, -1
-; GFX10-NEXT:    s_ashr_i32 s8, s0, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s6, s7
-; GFX10-NEXT:    s_movk_i32 s12, 0x7fff
-; GFX10-NEXT:    s_cselect_b32 s9, s6, s7
-; GFX10-NEXT:    s_cmp_gt_i32 s8, -1
-; GFX10-NEXT:    s_mov_b32 s13, 0x80008000
-; GFX10-NEXT:    s_cselect_b32 s10, s8, -1
-; GFX10-NEXT:    s_sext_i32_i16 s15, s3
-; GFX10-NEXT:    s_pack_ll_b32_b16 s9, s9, s10
-; GFX10-NEXT:    s_mov_b32 s10, 0x7fff7fff
-; GFX10-NEXT:    s_lshr_b32 s11, s9, 16
-; GFX10-NEXT:    s_sub_i32 s9, s9, s10
-; GFX10-NEXT:    s_sub_i32 s11, s11, s12
-; GFX10-NEXT:    s_cmp_lt_i32 s6, s7
+; GFX10-NEXT:    v_pk_sub_i16 v0, s0, s3 clamp
+; GFX10-NEXT:    v_pk_sub_i16 v1, s1, s4 clamp
+; GFX10-NEXT:    v_pk_sub_i16 v2, s2, s5 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cselect_b32 s6, s6, s7
-; GFX10-NEXT:    s_cmp_lt_i32 s8, -1
-; GFX10-NEXT:    s_cselect_b32 s8, s8, -1
-; GFX10-NEXT:    s_ashr_i32 s3, s3, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s6, s8
-; GFX10-NEXT:    s_pack_ll_b32_b16 s8, s9, s11
-; GFX10-NEXT:    s_lshr_b32 s9, s6, 16
-; GFX10-NEXT:    s_mov_b32 s11, 0x8000
-; GFX10-NEXT:    s_sext_i32_i16 s14, s8
-; GFX10-NEXT:    s_sub_i32 s6, s6, s13
-; GFX10-NEXT:    s_sub_i32 s9, s9, s11
-; GFX10-NEXT:    s_ashr_i32 s8, s8, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s14, s15
-; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s6, s9
-; GFX10-NEXT:    s_cselect_b32 s14, s14, s15
-; GFX10-NEXT:    s_cmp_gt_i32 s8, s3
-; GFX10-NEXT:    s_sext_i32_i16 s9, s6
-; GFX10-NEXT:    s_cselect_b32 s3, s8, s3
-; GFX10-NEXT:    s_ashr_i32 s6, s6, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s14, s3
-; GFX10-NEXT:    s_sext_i32_i16 s15, s4
-; GFX10-NEXT:    s_sext_i32_i16 s8, s3
-; GFX10-NEXT:    s_ashr_i32 s3, s3, 16
-; GFX10-NEXT:    s_cmp_lt_i32 s8, s9
-; GFX10-NEXT:    s_cselect_b32 s8, s8, s9
-; GFX10-NEXT:    s_cmp_lt_i32 s3, s6
-; GFX10-NEXT:    s_cselect_b32 s3, s3, s6
-; GFX10-NEXT:    s_lshr_b32 s6, s0, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s8, s3
-; GFX10-NEXT:    s_lshr_b32 s8, s3, 16
-; GFX10-NEXT:    s_sub_i32 s0, s0, s3
-; GFX10-NEXT:    s_sub_i32 s3, s6, s8
-; GFX10-NEXT:    s_sext_i32_i16 s6, s1
-; GFX10-NEXT:    s_ashr_i32 s8, s1, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s6, s7
-; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s3
-; GFX10-NEXT:    s_cselect_b32 s9, s6, s7
-; GFX10-NEXT:    s_cmp_gt_i32 s8, -1
-; GFX10-NEXT:    s_cselect_b32 s14, s8, -1
-; GFX10-NEXT:    s_pack_ll_b32_b16 s9, s9, s14
-; GFX10-NEXT:    s_lshr_b32 s14, s9, 16
-; GFX10-NEXT:    s_sub_i32 s9, s9, s10
-; GFX10-NEXT:    s_sub_i32 s14, s14, s12
-; GFX10-NEXT:    s_cmp_lt_i32 s6, s7
-; GFX10-NEXT:    s_cselect_b32 s6, s6, s7
-; GFX10-NEXT:    s_cmp_lt_i32 s8, -1
-; GFX10-NEXT:    s_cselect_b32 s8, s8, -1
-; GFX10-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s6, s8
-; GFX10-NEXT:    s_pack_ll_b32_b16 s8, s9, s14
-; GFX10-NEXT:    s_lshr_b32 s9, s6, 16
-; GFX10-NEXT:    s_sext_i32_i16 s14, s8
-; GFX10-NEXT:    s_sub_i32 s6, s6, s13
-; GFX10-NEXT:    s_sub_i32 s9, s9, s11
-; GFX10-NEXT:    s_ashr_i32 s8, s8, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s14, s15
-; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s6, s9
-; GFX10-NEXT:    s_cselect_b32 s14, s14, s15
-; GFX10-NEXT:    s_cmp_gt_i32 s8, s4
-; GFX10-NEXT:    s_sext_i32_i16 s9, s6
-; GFX10-NEXT:    s_cselect_b32 s4, s8, s4
-; GFX10-NEXT:    s_ashr_i32 s6, s6, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s14, s4
-; GFX10-NEXT:    s_sext_i32_i16 s8, s4
-; GFX10-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX10-NEXT:    s_cmp_lt_i32 s8, s9
-; GFX10-NEXT:    s_cselect_b32 s8, s8, s9
-; GFX10-NEXT:    s_cmp_lt_i32 s4, s6
-; GFX10-NEXT:    s_cselect_b32 s4, s4, s6
-; GFX10-NEXT:    s_lshr_b32 s6, s1, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s8, s4
-; GFX10-NEXT:    s_lshr_b32 s8, s4, 16
-; GFX10-NEXT:    s_sub_i32 s1, s1, s4
-; GFX10-NEXT:    s_sub_i32 s4, s6, s8
-; GFX10-NEXT:    s_sext_i32_i16 s6, s2
-; GFX10-NEXT:    s_ashr_i32 s8, s2, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s6, s7
-; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s1, s4
-; GFX10-NEXT:    s_cselect_b32 s9, s6, s7
-; GFX10-NEXT:    s_cmp_gt_i32 s8, -1
-; GFX10-NEXT:    s_cselect_b32 s14, s8, -1
-; GFX10-NEXT:    s_pack_ll_b32_b16 s9, s9, s14
-; GFX10-NEXT:    s_lshr_b32 s14, s9, 16
-; GFX10-NEXT:    s_sub_i32 s9, s9, s10
-; GFX10-NEXT:    s_sub_i32 s10, s14, s12
-; GFX10-NEXT:    s_cmp_lt_i32 s6, s7
-; GFX10-NEXT:    s_cselect_b32 s6, s6, s7
-; GFX10-NEXT:    s_cmp_lt_i32 s8, -1
-; GFX10-NEXT:    s_cselect_b32 s7, s8, -1
-; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s6, s7
-; GFX10-NEXT:    s_pack_ll_b32_b16 s7, s9, s10
-; GFX10-NEXT:    s_lshr_b32 s8, s6, 16
-; GFX10-NEXT:    s_sext_i32_i16 s9, s7
-; GFX10-NEXT:    s_sext_i32_i16 s10, s5
-; GFX10-NEXT:    s_sub_i32 s6, s6, s13
-; GFX10-NEXT:    s_sub_i32 s8, s8, s11
-; GFX10-NEXT:    s_ashr_i32 s7, s7, 16
-; GFX10-NEXT:    s_ashr_i32 s5, s5, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s9, s10
-; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s6, s8
-; GFX10-NEXT:    s_cselect_b32 s9, s9, s10
-; GFX10-NEXT:    s_cmp_gt_i32 s7, s5
-; GFX10-NEXT:    s_sext_i32_i16 s8, s6
-; GFX10-NEXT:    s_cselect_b32 s5, s7, s5
-; GFX10-NEXT:    s_ashr_i32 s6, s6, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s5, s9, s5
-; GFX10-NEXT:    s_sext_i32_i16 s7, s5
-; GFX10-NEXT:    s_ashr_i32 s5, s5, 16
-; GFX10-NEXT:    s_cmp_lt_i32 s7, s8
-; GFX10-NEXT:    s_cselect_b32 s7, s7, s8
-; GFX10-NEXT:    s_cmp_lt_i32 s5, s6
-; GFX10-NEXT:    s_cselect_b32 s5, s5, s6
-; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s7, s5
-; GFX10-NEXT:    s_lshr_b32 s5, s2, 16
-; GFX10-NEXT:    s_lshr_b32 s6, s3, 16
-; GFX10-NEXT:    s_sub_i32 s2, s2, s3
-; GFX10-NEXT:    s_sub_i32 s3, s5, s6
-; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s2, s3
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <6 x i16> @llvm.ssub.sat.v6i16(<6 x i16> %lhs, <6 x i16> %rhs)
   %cast = bitcast <6 x i16> %result to <3 x i32>
@@ -6343,71 +4238,21 @@ define <4 x float> @v_ssubsat_v8i16(<8 x i16> %lhs, <8 x i16> %rhs) {
 ; GFX9-LABEL: v_ssubsat_v8i16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_pk_max_i16 v8, v0, -1 op_sel_hi:[1,0]
-; GFX9-NEXT:    v_mov_b32_e32 v9, 0x7fff7fff
-; GFX9-NEXT:    v_pk_sub_i16 v8, v8, v9
-; GFX9-NEXT:    v_pk_min_i16 v10, v0, -1 op_sel_hi:[1,0]
-; GFX9-NEXT:    v_mov_b32_e32 v11, 0x80008000
-; GFX9-NEXT:    v_pk_max_i16 v4, v8, v4
-; GFX9-NEXT:    v_pk_sub_i16 v10, v10, v11
-; GFX9-NEXT:    v_pk_min_i16 v4, v4, v10
-; GFX9-NEXT:    v_pk_sub_i16 v0, v0, v4
-; GFX9-NEXT:    v_pk_max_i16 v4, v1, -1 op_sel_hi:[1,0]
-; GFX9-NEXT:    v_pk_sub_i16 v4, v4, v9
-; GFX9-NEXT:    v_pk_min_i16 v8, v1, -1 op_sel_hi:[1,0]
-; GFX9-NEXT:    v_pk_max_i16 v4, v4, v5
-; GFX9-NEXT:    v_pk_sub_i16 v8, v8, v11
-; GFX9-NEXT:    v_pk_min_i16 v4, v4, v8
-; GFX9-NEXT:    v_pk_sub_i16 v1, v1, v4
-; GFX9-NEXT:    v_pk_max_i16 v4, v2, -1 op_sel_hi:[1,0]
-; GFX9-NEXT:    v_pk_sub_i16 v4, v4, v9
-; GFX9-NEXT:    v_pk_min_i16 v5, v2, -1 op_sel_hi:[1,0]
-; GFX9-NEXT:    v_pk_sub_i16 v5, v5, v11
-; GFX9-NEXT:    v_pk_max_i16 v4, v4, v6
-; GFX9-NEXT:    v_pk_min_i16 v4, v4, v5
-; GFX9-NEXT:    v_pk_sub_i16 v2, v2, v4
-; GFX9-NEXT:    v_pk_max_i16 v4, v3, -1 op_sel_hi:[1,0]
-; GFX9-NEXT:    v_pk_sub_i16 v4, v4, v9
-; GFX9-NEXT:    v_pk_min_i16 v5, v3, -1 op_sel_hi:[1,0]
-; GFX9-NEXT:    v_pk_sub_i16 v5, v5, v11
-; GFX9-NEXT:    v_pk_max_i16 v4, v4, v7
-; GFX9-NEXT:    v_pk_min_i16 v4, v4, v5
-; GFX9-NEXT:    v_pk_sub_i16 v3, v3, v4
+; GFX9-NEXT:    v_pk_sub_i16 v0, v0, v4 clamp
+; GFX9-NEXT:    v_pk_sub_i16 v1, v1, v5 clamp
+; GFX9-NEXT:    v_pk_sub_i16 v2, v2, v6 clamp
+; GFX9-NEXT:    v_pk_sub_i16 v3, v3, v7 clamp
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_ssubsat_v8i16:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_pk_max_i16 v8, v0, -1 op_sel_hi:[1,0]
-; GFX10-NEXT:    v_pk_max_i16 v10, v1, -1 op_sel_hi:[1,0]
-; GFX10-NEXT:    v_pk_max_i16 v12, v3, -1 op_sel_hi:[1,0]
-; GFX10-NEXT:    v_pk_min_i16 v9, v0, -1 op_sel_hi:[1,0]
-; GFX10-NEXT:    v_pk_min_i16 v11, v1, -1 op_sel_hi:[1,0]
-; GFX10-NEXT:    v_pk_sub_i16 v15, v8, 0x7fff7fff
-; GFX10-NEXT:    v_pk_max_i16 v8, v2, -1 op_sel_hi:[1,0]
-; GFX10-NEXT:    v_pk_sub_i16 v10, v10, 0x7fff7fff
-; GFX10-NEXT:    v_pk_sub_i16 v12, v12, 0x7fff7fff
-; GFX10-NEXT:    v_pk_min_i16 v13, v2, -1 op_sel_hi:[1,0]
-; GFX10-NEXT:    v_pk_min_i16 v14, v3, -1 op_sel_hi:[1,0]
-; GFX10-NEXT:    v_pk_sub_i16 v8, v8, 0x7fff7fff
-; GFX10-NEXT:    v_pk_max_i16 v4, v15, v4
-; GFX10-NEXT:    v_pk_sub_i16 v9, v9, 0x80008000
-; GFX10-NEXT:    v_pk_max_i16 v5, v10, v5
-; GFX10-NEXT:    v_pk_sub_i16 v11, v11, 0x80008000
-; GFX10-NEXT:    v_pk_max_i16 v15, v8, v6
-; GFX10-NEXT:    v_pk_sub_i16 v10, v13, 0x80008000
-; GFX10-NEXT:    v_pk_sub_i16 v8, v14, 0x80008000
-; GFX10-NEXT:    v_pk_max_i16 v7, v12, v7
-; GFX10-NEXT:    v_pk_min_i16 v19, v4, v9
-; GFX10-NEXT:    v_pk_min_i16 v11, v5, v11
-; GFX10-NEXT:    v_pk_min_i16 v15, v15, v10
+; GFX10-NEXT:    v_pk_sub_i16 v0, v0, v4 clamp
+; GFX10-NEXT:    v_pk_sub_i16 v1, v1, v5 clamp
+; GFX10-NEXT:    v_pk_sub_i16 v2, v2, v6 clamp
+; GFX10-NEXT:    v_pk_sub_i16 v3, v3, v7 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_pk_min_i16 v6, v7, v8
-; GFX10-NEXT:    v_pk_sub_i16 v0, v0, v19
-; GFX10-NEXT:    v_pk_sub_i16 v1, v1, v11
-; GFX10-NEXT:    v_pk_sub_i16 v2, v2, v15
-; GFX10-NEXT:    v_pk_sub_i16 v3, v3, v6
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> %lhs, <8 x i16> %rhs)
   %cast = bitcast <8 x i16> %result to <4 x float>
@@ -6711,365 +4556,31 @@ define amdgpu_ps <4 x i32> @s_ssubsat_v8i16(<8 x i16> inreg %lhs, <8 x i16> inre
 ;
 ; GFX9-LABEL: s_ssubsat_v8i16:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_sext_i32_i16 s10, s0
-; GFX9-NEXT:    s_ashr_i32 s11, s0, 16
-; GFX9-NEXT:    s_sext_i32_i16 s12, -1
-; GFX9-NEXT:    s_cmp_gt_i32 s10, s12
-; GFX9-NEXT:    s_cselect_b32 s13, s10, s12
-; GFX9-NEXT:    s_cmp_gt_i32 s11, -1
-; GFX9-NEXT:    s_cselect_b32 s14, s11, -1
-; GFX9-NEXT:    s_pack_ll_b32_b16 s13, s13, s14
-; GFX9-NEXT:    s_mov_b32 s8, 0x7fff7fff
-; GFX9-NEXT:    s_lshr_b32 s14, s13, 16
-; GFX9-NEXT:    s_movk_i32 s15, 0x7fff
-; GFX9-NEXT:    s_sub_i32 s13, s13, s8
-; GFX9-NEXT:    s_sub_i32 s14, s14, s15
-; GFX9-NEXT:    s_cmp_lt_i32 s10, s12
-; GFX9-NEXT:    s_cselect_b32 s10, s10, s12
-; GFX9-NEXT:    s_cmp_lt_i32 s11, -1
-; GFX9-NEXT:    s_cselect_b32 s11, s11, -1
-; GFX9-NEXT:    s_pack_ll_b32_b16 s10, s10, s11
-; GFX9-NEXT:    s_pack_ll_b32_b16 s13, s13, s14
-; GFX9-NEXT:    s_mov_b32 s9, 0x80008000
-; GFX9-NEXT:    s_lshr_b32 s11, s10, 16
-; GFX9-NEXT:    s_mov_b32 s14, 0x8000
-; GFX9-NEXT:    s_sub_i32 s10, s10, s9
-; GFX9-NEXT:    s_sub_i32 s11, s11, s14
-; GFX9-NEXT:    s_pack_ll_b32_b16 s10, s10, s11
-; GFX9-NEXT:    s_sext_i32_i16 s11, s13
-; GFX9-NEXT:    s_sext_i32_i16 s16, s4
-; GFX9-NEXT:    s_ashr_i32 s13, s13, 16
-; GFX9-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s11, s16
-; GFX9-NEXT:    s_cselect_b32 s11, s11, s16
-; GFX9-NEXT:    s_cmp_gt_i32 s13, s4
-; GFX9-NEXT:    s_cselect_b32 s4, s13, s4
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s11, s4
-; GFX9-NEXT:    s_sext_i32_i16 s11, s4
-; GFX9-NEXT:    s_sext_i32_i16 s13, s10
-; GFX9-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX9-NEXT:    s_ashr_i32 s10, s10, 16
-; GFX9-NEXT:    s_cmp_lt_i32 s11, s13
-; GFX9-NEXT:    s_cselect_b32 s11, s11, s13
-; GFX9-NEXT:    s_cmp_lt_i32 s4, s10
-; GFX9-NEXT:    s_cselect_b32 s4, s4, s10
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s11, s4
-; GFX9-NEXT:    s_lshr_b32 s10, s0, 16
-; GFX9-NEXT:    s_lshr_b32 s11, s4, 16
-; GFX9-NEXT:    s_sub_i32 s0, s0, s4
-; GFX9-NEXT:    s_sub_i32 s4, s10, s11
-; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s4
-; GFX9-NEXT:    s_sext_i32_i16 s4, s1
-; GFX9-NEXT:    s_ashr_i32 s10, s1, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s4, s12
-; GFX9-NEXT:    s_cselect_b32 s11, s4, s12
-; GFX9-NEXT:    s_cmp_gt_i32 s10, -1
-; GFX9-NEXT:    s_cselect_b32 s13, s10, -1
-; GFX9-NEXT:    s_pack_ll_b32_b16 s11, s11, s13
-; GFX9-NEXT:    s_lshr_b32 s13, s11, 16
-; GFX9-NEXT:    s_sub_i32 s11, s11, s8
-; GFX9-NEXT:    s_sub_i32 s13, s13, s15
-; GFX9-NEXT:    s_cmp_lt_i32 s4, s12
-; GFX9-NEXT:    s_cselect_b32 s4, s4, s12
-; GFX9-NEXT:    s_cmp_lt_i32 s10, -1
-; GFX9-NEXT:    s_cselect_b32 s10, s10, -1
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s10
-; GFX9-NEXT:    s_lshr_b32 s10, s4, 16
-; GFX9-NEXT:    s_pack_ll_b32_b16 s11, s11, s13
-; GFX9-NEXT:    s_sub_i32 s4, s4, s9
-; GFX9-NEXT:    s_sub_i32 s10, s10, s14
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s10
-; GFX9-NEXT:    s_sext_i32_i16 s10, s11
-; GFX9-NEXT:    s_sext_i32_i16 s13, s5
-; GFX9-NEXT:    s_ashr_i32 s11, s11, 16
-; GFX9-NEXT:    s_ashr_i32 s5, s5, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s10, s13
-; GFX9-NEXT:    s_cselect_b32 s10, s10, s13
-; GFX9-NEXT:    s_cmp_gt_i32 s11, s5
-; GFX9-NEXT:    s_cselect_b32 s5, s11, s5
-; GFX9-NEXT:    s_pack_ll_b32_b16 s5, s10, s5
-; GFX9-NEXT:    s_sext_i32_i16 s10, s5
-; GFX9-NEXT:    s_sext_i32_i16 s11, s4
-; GFX9-NEXT:    s_ashr_i32 s5, s5, 16
-; GFX9-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX9-NEXT:    s_cmp_lt_i32 s10, s11
-; GFX9-NEXT:    s_cselect_b32 s10, s10, s11
-; GFX9-NEXT:    s_cmp_lt_i32 s5, s4
-; GFX9-NEXT:    s_cselect_b32 s4, s5, s4
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s10, s4
-; GFX9-NEXT:    s_lshr_b32 s5, s1, 16
-; GFX9-NEXT:    s_lshr_b32 s10, s4, 16
-; GFX9-NEXT:    s_sub_i32 s1, s1, s4
-; GFX9-NEXT:    s_sub_i32 s4, s5, s10
-; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s4
-; GFX9-NEXT:    s_sext_i32_i16 s4, s2
-; GFX9-NEXT:    s_ashr_i32 s5, s2, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s4, s12
-; GFX9-NEXT:    s_cselect_b32 s10, s4, s12
-; GFX9-NEXT:    s_cmp_gt_i32 s5, -1
-; GFX9-NEXT:    s_cselect_b32 s11, s5, -1
-; GFX9-NEXT:    s_pack_ll_b32_b16 s10, s10, s11
-; GFX9-NEXT:    s_lshr_b32 s11, s10, 16
-; GFX9-NEXT:    s_sub_i32 s10, s10, s8
-; GFX9-NEXT:    s_sub_i32 s11, s11, s15
-; GFX9-NEXT:    s_cmp_lt_i32 s4, s12
-; GFX9-NEXT:    s_cselect_b32 s4, s4, s12
-; GFX9-NEXT:    s_cmp_lt_i32 s5, -1
-; GFX9-NEXT:    s_cselect_b32 s5, s5, -1
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
-; GFX9-NEXT:    s_lshr_b32 s5, s4, 16
-; GFX9-NEXT:    s_pack_ll_b32_b16 s10, s10, s11
-; GFX9-NEXT:    s_sub_i32 s4, s4, s9
-; GFX9-NEXT:    s_sub_i32 s5, s5, s14
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
-; GFX9-NEXT:    s_sext_i32_i16 s5, s10
-; GFX9-NEXT:    s_sext_i32_i16 s11, s6
-; GFX9-NEXT:    s_ashr_i32 s10, s10, 16
-; GFX9-NEXT:    s_ashr_i32 s6, s6, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s5, s11
-; GFX9-NEXT:    s_cselect_b32 s5, s5, s11
-; GFX9-NEXT:    s_cmp_gt_i32 s10, s6
-; GFX9-NEXT:    s_cselect_b32 s6, s10, s6
-; GFX9-NEXT:    s_pack_ll_b32_b16 s5, s5, s6
-; GFX9-NEXT:    s_sext_i32_i16 s6, s5
-; GFX9-NEXT:    s_sext_i32_i16 s10, s4
-; GFX9-NEXT:    s_ashr_i32 s5, s5, 16
-; GFX9-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX9-NEXT:    s_cmp_lt_i32 s6, s10
-; GFX9-NEXT:    s_cselect_b32 s6, s6, s10
-; GFX9-NEXT:    s_cmp_lt_i32 s5, s4
-; GFX9-NEXT:    s_cselect_b32 s4, s5, s4
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s6, s4
-; GFX9-NEXT:    s_lshr_b32 s5, s2, 16
-; GFX9-NEXT:    s_lshr_b32 s6, s4, 16
-; GFX9-NEXT:    s_sub_i32 s2, s2, s4
-; GFX9-NEXT:    s_sub_i32 s4, s5, s6
-; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s4
-; GFX9-NEXT:    s_sext_i32_i16 s4, s3
-; GFX9-NEXT:    s_ashr_i32 s5, s3, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s4, s12
-; GFX9-NEXT:    s_cselect_b32 s6, s4, s12
-; GFX9-NEXT:    s_cmp_gt_i32 s5, -1
-; GFX9-NEXT:    s_cselect_b32 s10, s5, -1
-; GFX9-NEXT:    s_pack_ll_b32_b16 s6, s6, s10
-; GFX9-NEXT:    s_lshr_b32 s10, s6, 16
-; GFX9-NEXT:    s_sub_i32 s6, s6, s8
-; GFX9-NEXT:    s_sub_i32 s8, s10, s15
-; GFX9-NEXT:    s_cmp_lt_i32 s4, s12
-; GFX9-NEXT:    s_cselect_b32 s4, s4, s12
-; GFX9-NEXT:    s_cmp_lt_i32 s5, -1
-; GFX9-NEXT:    s_cselect_b32 s5, s5, -1
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
-; GFX9-NEXT:    s_lshr_b32 s5, s4, 16
-; GFX9-NEXT:    s_pack_ll_b32_b16 s6, s6, s8
-; GFX9-NEXT:    s_sub_i32 s4, s4, s9
-; GFX9-NEXT:    s_sub_i32 s5, s5, s14
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
-; GFX9-NEXT:    s_sext_i32_i16 s5, s6
-; GFX9-NEXT:    s_sext_i32_i16 s8, s7
-; GFX9-NEXT:    s_ashr_i32 s6, s6, 16
-; GFX9-NEXT:    s_ashr_i32 s7, s7, 16
-; GFX9-NEXT:    s_cmp_gt_i32 s5, s8
-; GFX9-NEXT:    s_cselect_b32 s5, s5, s8
-; GFX9-NEXT:    s_cmp_gt_i32 s6, s7
-; GFX9-NEXT:    s_cselect_b32 s6, s6, s7
-; GFX9-NEXT:    s_pack_ll_b32_b16 s5, s5, s6
-; GFX9-NEXT:    s_sext_i32_i16 s6, s5
-; GFX9-NEXT:    s_sext_i32_i16 s7, s4
-; GFX9-NEXT:    s_ashr_i32 s5, s5, 16
-; GFX9-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX9-NEXT:    s_cmp_lt_i32 s6, s7
-; GFX9-NEXT:    s_cselect_b32 s6, s6, s7
-; GFX9-NEXT:    s_cmp_lt_i32 s5, s4
-; GFX9-NEXT:    s_cselect_b32 s4, s5, s4
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s6, s4
-; GFX9-NEXT:    s_lshr_b32 s5, s3, 16
-; GFX9-NEXT:    s_lshr_b32 s6, s4, 16
-; GFX9-NEXT:    s_sub_i32 s3, s3, s4
-; GFX9-NEXT:    s_sub_i32 s4, s5, s6
-; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s3, s4
+; GFX9-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-NEXT:    v_mov_b32_e32 v2, s6
+; GFX9-NEXT:    v_mov_b32_e32 v3, s7
+; GFX9-NEXT:    v_pk_sub_i16 v0, s0, v0 clamp
+; GFX9-NEXT:    v_pk_sub_i16 v1, s1, v1 clamp
+; GFX9-NEXT:    v_pk_sub_i16 v2, s2, v2 clamp
+; GFX9-NEXT:    v_pk_sub_i16 v3, s3, v3 clamp
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX9-NEXT:    v_readfirstlane_b32 s3, v3
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_ssubsat_v8i16:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_sext_i32_i16 s8, s0
-; GFX10-NEXT:    s_sext_i32_i16 s9, -1
-; GFX10-NEXT:    s_ashr_i32 s10, s0, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s8, s9
-; GFX10-NEXT:    s_movk_i32 s14, 0x7fff
-; GFX10-NEXT:    s_cselect_b32 s11, s8, s9
-; GFX10-NEXT:    s_cmp_gt_i32 s10, -1
-; GFX10-NEXT:    s_mov_b32 s15, 0x80008000
-; GFX10-NEXT:    s_cselect_b32 s12, s10, -1
-; GFX10-NEXT:    s_sext_i32_i16 s17, s4
-; GFX10-NEXT:    s_pack_ll_b32_b16 s11, s11, s12
-; GFX10-NEXT:    s_mov_b32 s12, 0x7fff7fff
-; GFX10-NEXT:    s_lshr_b32 s13, s11, 16
-; GFX10-NEXT:    s_sub_i32 s11, s11, s12
-; GFX10-NEXT:    s_sub_i32 s13, s13, s14
-; GFX10-NEXT:    s_cmp_lt_i32 s8, s9
+; GFX10-NEXT:    v_pk_sub_i16 v0, s0, s4 clamp
+; GFX10-NEXT:    v_pk_sub_i16 v1, s1, s5 clamp
+; GFX10-NEXT:    v_pk_sub_i16 v2, s2, s6 clamp
+; GFX10-NEXT:    v_pk_sub_i16 v3, s3, s7 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cselect_b32 s8, s8, s9
-; GFX10-NEXT:    s_cmp_lt_i32 s10, -1
-; GFX10-NEXT:    s_cselect_b32 s10, s10, -1
-; GFX10-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s8, s8, s10
-; GFX10-NEXT:    s_pack_ll_b32_b16 s10, s11, s13
-; GFX10-NEXT:    s_lshr_b32 s11, s8, 16
-; GFX10-NEXT:    s_mov_b32 s13, 0x8000
-; GFX10-NEXT:    s_sext_i32_i16 s16, s10
-; GFX10-NEXT:    s_sub_i32 s8, s8, s15
-; GFX10-NEXT:    s_sub_i32 s11, s11, s13
-; GFX10-NEXT:    s_ashr_i32 s10, s10, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s16, s17
-; GFX10-NEXT:    s_pack_ll_b32_b16 s8, s8, s11
-; GFX10-NEXT:    s_cselect_b32 s16, s16, s17
-; GFX10-NEXT:    s_cmp_gt_i32 s10, s4
-; GFX10-NEXT:    s_sext_i32_i16 s11, s8
-; GFX10-NEXT:    s_cselect_b32 s4, s10, s4
-; GFX10-NEXT:    s_ashr_i32 s8, s8, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s16, s4
-; GFX10-NEXT:    s_sext_i32_i16 s17, s5
-; GFX10-NEXT:    s_sext_i32_i16 s10, s4
-; GFX10-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX10-NEXT:    s_cmp_lt_i32 s10, s11
-; GFX10-NEXT:    s_cselect_b32 s10, s10, s11
-; GFX10-NEXT:    s_cmp_lt_i32 s4, s8
-; GFX10-NEXT:    s_cselect_b32 s4, s4, s8
-; GFX10-NEXT:    s_lshr_b32 s8, s0, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s10, s4
-; GFX10-NEXT:    s_lshr_b32 s10, s4, 16
-; GFX10-NEXT:    s_sub_i32 s0, s0, s4
-; GFX10-NEXT:    s_sub_i32 s4, s8, s10
-; GFX10-NEXT:    s_sext_i32_i16 s8, s1
-; GFX10-NEXT:    s_ashr_i32 s10, s1, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s8, s9
-; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s4
-; GFX10-NEXT:    s_cselect_b32 s11, s8, s9
-; GFX10-NEXT:    s_cmp_gt_i32 s10, -1
-; GFX10-NEXT:    s_cselect_b32 s16, s10, -1
-; GFX10-NEXT:    s_pack_ll_b32_b16 s11, s11, s16
-; GFX10-NEXT:    s_lshr_b32 s16, s11, 16
-; GFX10-NEXT:    s_sub_i32 s11, s11, s12
-; GFX10-NEXT:    s_sub_i32 s16, s16, s14
-; GFX10-NEXT:    s_cmp_lt_i32 s8, s9
-; GFX10-NEXT:    s_cselect_b32 s8, s8, s9
-; GFX10-NEXT:    s_cmp_lt_i32 s10, -1
-; GFX10-NEXT:    s_cselect_b32 s10, s10, -1
-; GFX10-NEXT:    s_ashr_i32 s5, s5, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s8, s8, s10
-; GFX10-NEXT:    s_pack_ll_b32_b16 s10, s11, s16
-; GFX10-NEXT:    s_lshr_b32 s11, s8, 16
-; GFX10-NEXT:    s_sext_i32_i16 s16, s10
-; GFX10-NEXT:    s_sub_i32 s8, s8, s15
-; GFX10-NEXT:    s_sub_i32 s11, s11, s13
-; GFX10-NEXT:    s_ashr_i32 s10, s10, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s16, s17
-; GFX10-NEXT:    s_pack_ll_b32_b16 s8, s8, s11
-; GFX10-NEXT:    s_cselect_b32 s16, s16, s17
-; GFX10-NEXT:    s_cmp_gt_i32 s10, s5
-; GFX10-NEXT:    s_sext_i32_i16 s11, s8
-; GFX10-NEXT:    s_cselect_b32 s5, s10, s5
-; GFX10-NEXT:    s_ashr_i32 s8, s8, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s5, s16, s5
-; GFX10-NEXT:    s_sext_i32_i16 s17, s6
-; GFX10-NEXT:    s_sext_i32_i16 s10, s5
-; GFX10-NEXT:    s_ashr_i32 s5, s5, 16
-; GFX10-NEXT:    s_cmp_lt_i32 s10, s11
-; GFX10-NEXT:    s_cselect_b32 s10, s10, s11
-; GFX10-NEXT:    s_cmp_lt_i32 s5, s8
-; GFX10-NEXT:    s_cselect_b32 s5, s5, s8
-; GFX10-NEXT:    s_lshr_b32 s8, s1, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s5, s10, s5
-; GFX10-NEXT:    s_lshr_b32 s10, s5, 16
-; GFX10-NEXT:    s_sub_i32 s1, s1, s5
-; GFX10-NEXT:    s_sub_i32 s5, s8, s10
-; GFX10-NEXT:    s_sext_i32_i16 s8, s2
-; GFX10-NEXT:    s_ashr_i32 s10, s2, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s8, s9
-; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s1, s5
-; GFX10-NEXT:    s_cselect_b32 s11, s8, s9
-; GFX10-NEXT:    s_cmp_gt_i32 s10, -1
-; GFX10-NEXT:    s_cselect_b32 s16, s10, -1
-; GFX10-NEXT:    s_pack_ll_b32_b16 s11, s11, s16
-; GFX10-NEXT:    s_lshr_b32 s16, s11, 16
-; GFX10-NEXT:    s_sub_i32 s11, s11, s12
-; GFX10-NEXT:    s_sub_i32 s16, s16, s14
-; GFX10-NEXT:    s_cmp_lt_i32 s8, s9
-; GFX10-NEXT:    s_cselect_b32 s8, s8, s9
-; GFX10-NEXT:    s_cmp_lt_i32 s10, -1
-; GFX10-NEXT:    s_cselect_b32 s10, s10, -1
-; GFX10-NEXT:    s_ashr_i32 s6, s6, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s8, s8, s10
-; GFX10-NEXT:    s_pack_ll_b32_b16 s10, s11, s16
-; GFX10-NEXT:    s_lshr_b32 s11, s8, 16
-; GFX10-NEXT:    s_sext_i32_i16 s16, s10
-; GFX10-NEXT:    s_sub_i32 s8, s8, s15
-; GFX10-NEXT:    s_sub_i32 s11, s11, s13
-; GFX10-NEXT:    s_ashr_i32 s10, s10, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s16, s17
-; GFX10-NEXT:    s_pack_ll_b32_b16 s8, s8, s11
-; GFX10-NEXT:    s_cselect_b32 s16, s16, s17
-; GFX10-NEXT:    s_cmp_gt_i32 s10, s6
-; GFX10-NEXT:    s_sext_i32_i16 s11, s8
-; GFX10-NEXT:    s_cselect_b32 s6, s10, s6
-; GFX10-NEXT:    s_ashr_i32 s8, s8, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s16, s6
-; GFX10-NEXT:    s_sext_i32_i16 s10, s6
-; GFX10-NEXT:    s_ashr_i32 s6, s6, 16
-; GFX10-NEXT:    s_cmp_lt_i32 s10, s11
-; GFX10-NEXT:    s_cselect_b32 s10, s10, s11
-; GFX10-NEXT:    s_cmp_lt_i32 s6, s8
-; GFX10-NEXT:    s_cselect_b32 s6, s6, s8
-; GFX10-NEXT:    s_lshr_b32 s8, s2, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s10, s6
-; GFX10-NEXT:    s_lshr_b32 s10, s6, 16
-; GFX10-NEXT:    s_sub_i32 s2, s2, s6
-; GFX10-NEXT:    s_sub_i32 s6, s8, s10
-; GFX10-NEXT:    s_sext_i32_i16 s8, s3
-; GFX10-NEXT:    s_ashr_i32 s10, s3, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s8, s9
-; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s2, s6
-; GFX10-NEXT:    s_cselect_b32 s11, s8, s9
-; GFX10-NEXT:    s_cmp_gt_i32 s10, -1
-; GFX10-NEXT:    s_cselect_b32 s16, s10, -1
-; GFX10-NEXT:    s_pack_ll_b32_b16 s11, s11, s16
-; GFX10-NEXT:    s_lshr_b32 s16, s11, 16
-; GFX10-NEXT:    s_sub_i32 s11, s11, s12
-; GFX10-NEXT:    s_sub_i32 s12, s16, s14
-; GFX10-NEXT:    s_cmp_lt_i32 s8, s9
-; GFX10-NEXT:    s_cselect_b32 s8, s8, s9
-; GFX10-NEXT:    s_cmp_lt_i32 s10, -1
-; GFX10-NEXT:    s_cselect_b32 s9, s10, -1
-; GFX10-NEXT:    s_pack_ll_b32_b16 s8, s8, s9
-; GFX10-NEXT:    s_pack_ll_b32_b16 s9, s11, s12
-; GFX10-NEXT:    s_lshr_b32 s10, s8, 16
-; GFX10-NEXT:    s_sext_i32_i16 s11, s9
-; GFX10-NEXT:    s_sext_i32_i16 s12, s7
-; GFX10-NEXT:    s_sub_i32 s8, s8, s15
-; GFX10-NEXT:    s_sub_i32 s10, s10, s13
-; GFX10-NEXT:    s_ashr_i32 s9, s9, 16
-; GFX10-NEXT:    s_ashr_i32 s7, s7, 16
-; GFX10-NEXT:    s_cmp_gt_i32 s11, s12
-; GFX10-NEXT:    s_pack_ll_b32_b16 s8, s8, s10
-; GFX10-NEXT:    s_cselect_b32 s11, s11, s12
-; GFX10-NEXT:    s_cmp_gt_i32 s9, s7
-; GFX10-NEXT:    s_sext_i32_i16 s10, s8
-; GFX10-NEXT:    s_cselect_b32 s7, s9, s7
-; GFX10-NEXT:    s_ashr_i32 s8, s8, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s7, s11, s7
-; GFX10-NEXT:    s_sext_i32_i16 s9, s7
-; GFX10-NEXT:    s_ashr_i32 s7, s7, 16
-; GFX10-NEXT:    s_cmp_lt_i32 s9, s10
-; GFX10-NEXT:    s_cselect_b32 s9, s9, s10
-; GFX10-NEXT:    s_cmp_lt_i32 s7, s8
-; GFX10-NEXT:    s_cselect_b32 s4, s7, s8
-; GFX10-NEXT:    s_lshr_b32 s5, s3, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s9, s4
-; GFX10-NEXT:    s_lshr_b32 s7, s4, 16
-; GFX10-NEXT:    s_sub_i32 s3, s3, s4
-; GFX10-NEXT:    s_sub_i32 s4, s5, s7
-; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s3, s4
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX10-NEXT:    v_readfirstlane_b32 s3, v3
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> %lhs, <8 x i16> %rhs)
   %cast = bitcast <8 x i16> %result to <4 x i32>
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll
index 3a742fbcbd919..cbabb07aa9361 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll
@@ -21,9 +21,7 @@ define i7 @v_uaddsat_i7(i7 %lhs, i7 %rhs) {
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    v_lshlrev_b16_e32 v0, 9, v0
 ; GFX8-NEXT:    v_lshlrev_b16_e32 v1, 9, v1
-; GFX8-NEXT:    v_xor_b32_e32 v2, -1, v0
-; GFX8-NEXT:    v_min_u16_e32 v1, v2, v1
-; GFX8-NEXT:    v_add_u16_e32 v0, v0, v1
+; GFX8-NEXT:    v_add_u16_e64 v0, v0, v1 clamp
 ; GFX8-NEXT:    v_lshrrev_b16_e32 v0, 9, v0
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -32,9 +30,7 @@ define i7 @v_uaddsat_i7(i7 %lhs, i7 %rhs) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 9, v0
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 9, v1
-; GFX9-NEXT:    v_xor_b32_e32 v2, -1, v0
-; GFX9-NEXT:    v_min_u16_e32 v1, v2, v1
-; GFX9-NEXT:    v_add_u16_e32 v0, v0, v1
+; GFX9-NEXT:    v_add_u16_e64 v0, v0, v1 clamp
 ; GFX9-NEXT:    v_lshrrev_b16_e32 v0, 9, v0
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -45,9 +41,7 @@ define i7 @v_uaddsat_i7(i7 %lhs, i7 %rhs) {
 ; GFX10-NEXT:    v_lshlrev_b16_e64 v0, 9, v0
 ; GFX10-NEXT:    v_lshlrev_b16_e64 v1, 9, v1
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_xor_b32_e32 v2, -1, v0
-; GFX10-NEXT:    v_min_u16_e64 v1, v2, v1
-; GFX10-NEXT:    v_add_nc_u16_e64 v0, v0, v1
+; GFX10-NEXT:    v_add_nc_u16_e64 v0, v0, v1 clamp
 ; GFX10-NEXT:    v_lshrrev_b16_e64 v0, 9, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call i7 @llvm.uadd.sat.i7(i7 %lhs, i7 %rhs)
@@ -69,31 +63,23 @@ define amdgpu_ps i7 @s_uaddsat_i7(i7 inreg %lhs, i7 inreg %rhs) {
 ; GFX8-LABEL: s_uaddsat_i7:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_bfe_u32 s2, 9, 0x100000
-; GFX8-NEXT:    s_lshl_b32 s0, s0, s2
 ; GFX8-NEXT:    s_lshl_b32 s1, s1, s2
-; GFX8-NEXT:    s_xor_b32 s3, s0, -1
-; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s3, s1
-; GFX8-NEXT:    s_cselect_b32 s1, s3, s1
-; GFX8-NEXT:    s_add_i32 s0, s0, s1
-; GFX8-NEXT:    s_bfe_u32 s0, s0, 0x100000
-; GFX8-NEXT:    s_lshr_b32 s0, s0, s2
+; GFX8-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX8-NEXT:    v_mov_b32_e32 v0, s1
+; GFX8-NEXT:    v_add_u16_e64 v0, s0, v0 clamp
+; GFX8-NEXT:    v_lshrrev_b16_e32 v0, 9, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: s_uaddsat_i7:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_bfe_u32 s2, 9, 0x100000
-; GFX9-NEXT:    s_lshl_b32 s0, s0, s2
 ; GFX9-NEXT:    s_lshl_b32 s1, s1, s2
-; GFX9-NEXT:    s_xor_b32 s3, s0, -1
-; GFX9-NEXT:    s_bfe_u32 s3, s3, 0x100000
-; GFX9-NEXT:    s_bfe_u32 s1, s1, 0x100000
-; GFX9-NEXT:    s_cmp_lt_u32 s3, s1
-; GFX9-NEXT:    s_cselect_b32 s1, s3, s1
-; GFX9-NEXT:    s_add_i32 s0, s0, s1
-; GFX9-NEXT:    s_bfe_u32 s0, s0, 0x100000
-; GFX9-NEXT:    s_lshr_b32 s0, s0, s2
+; GFX9-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX9-NEXT:    v_mov_b32_e32 v0, s1
+; GFX9-NEXT:    v_add_u16_e64 v0, s0, v0 clamp
+; GFX9-NEXT:    v_lshrrev_b16_e32 v0, 9, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_uaddsat_i7:
@@ -102,14 +88,9 @@ define amdgpu_ps i7 @s_uaddsat_i7(i7 inreg %lhs, i7 inreg %rhs) {
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_lshl_b32 s0, s0, s2
 ; GFX10-NEXT:    s_lshl_b32 s1, s1, s2
-; GFX10-NEXT:    s_xor_b32 s3, s0, -1
-; GFX10-NEXT:    s_bfe_u32 s1, s1, 0x100000
-; GFX10-NEXT:    s_bfe_u32 s3, s3, 0x100000
-; GFX10-NEXT:    s_cmp_lt_u32 s3, s1
-; GFX10-NEXT:    s_cselect_b32 s1, s3, s1
-; GFX10-NEXT:    s_add_i32 s0, s0, s1
-; GFX10-NEXT:    s_bfe_u32 s0, s0, 0x100000
-; GFX10-NEXT:    s_lshr_b32 s0, s0, s2
+; GFX10-NEXT:    v_add_nc_u16_e64 v0, s0, s1 clamp
+; GFX10-NEXT:    v_lshrrev_b16_e64 v0, 9, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call i7 @llvm.uadd.sat.i7(i7 %lhs, i7 %rhs)
   ret i7 %result
@@ -132,9 +113,7 @@ define i8 @v_uaddsat_i8(i8 %lhs, i8 %rhs) {
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
 ; GFX8-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
-; GFX8-NEXT:    v_xor_b32_e32 v2, -1, v0
-; GFX8-NEXT:    v_min_u16_e32 v1, v2, v1
-; GFX8-NEXT:    v_add_u16_e32 v0, v0, v1
+; GFX8-NEXT:    v_add_u16_e64 v0, v0, v1 clamp
 ; GFX8-NEXT:    v_lshrrev_b16_e32 v0, 8, v0
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -143,9 +122,7 @@ define i8 @v_uaddsat_i8(i8 %lhs, i8 %rhs) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
-; GFX9-NEXT:    v_xor_b32_e32 v2, -1, v0
-; GFX9-NEXT:    v_min_u16_e32 v1, v2, v1
-; GFX9-NEXT:    v_add_u16_e32 v0, v0, v1
+; GFX9-NEXT:    v_add_u16_e64 v0, v0, v1 clamp
 ; GFX9-NEXT:    v_lshrrev_b16_e32 v0, 8, v0
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -156,9 +133,7 @@ define i8 @v_uaddsat_i8(i8 %lhs, i8 %rhs) {
 ; GFX10-NEXT:    v_lshlrev_b16_e64 v0, 8, v0
 ; GFX10-NEXT:    v_lshlrev_b16_e64 v1, 8, v1
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_xor_b32_e32 v2, -1, v0
-; GFX10-NEXT:    v_min_u16_e64 v1, v2, v1
-; GFX10-NEXT:    v_add_nc_u16_e64 v0, v0, v1
+; GFX10-NEXT:    v_add_nc_u16_e64 v0, v0, v1 clamp
 ; GFX10-NEXT:    v_lshrrev_b16_e64 v0, 8, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call i8 @llvm.uadd.sat.i8(i8 %lhs, i8 %rhs)
@@ -180,31 +155,23 @@ define amdgpu_ps i8 @s_uaddsat_i8(i8 inreg %lhs, i8 inreg %rhs) {
 ; GFX8-LABEL: s_uaddsat_i8:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_bfe_u32 s2, 8, 0x100000
-; GFX8-NEXT:    s_lshl_b32 s0, s0, s2
 ; GFX8-NEXT:    s_lshl_b32 s1, s1, s2
-; GFX8-NEXT:    s_xor_b32 s3, s0, -1
-; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s3, s1
-; GFX8-NEXT:    s_cselect_b32 s1, s3, s1
-; GFX8-NEXT:    s_add_i32 s0, s0, s1
-; GFX8-NEXT:    s_bfe_u32 s0, s0, 0x100000
-; GFX8-NEXT:    s_lshr_b32 s0, s0, s2
+; GFX8-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX8-NEXT:    v_mov_b32_e32 v0, s1
+; GFX8-NEXT:    v_add_u16_e64 v0, s0, v0 clamp
+; GFX8-NEXT:    v_lshrrev_b16_e32 v0, 8, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: s_uaddsat_i8:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_bfe_u32 s2, 8, 0x100000
-; GFX9-NEXT:    s_lshl_b32 s0, s0, s2
 ; GFX9-NEXT:    s_lshl_b32 s1, s1, s2
-; GFX9-NEXT:    s_xor_b32 s3, s0, -1
-; GFX9-NEXT:    s_bfe_u32 s3, s3, 0x100000
-; GFX9-NEXT:    s_bfe_u32 s1, s1, 0x100000
-; GFX9-NEXT:    s_cmp_lt_u32 s3, s1
-; GFX9-NEXT:    s_cselect_b32 s1, s3, s1
-; GFX9-NEXT:    s_add_i32 s0, s0, s1
-; GFX9-NEXT:    s_bfe_u32 s0, s0, 0x100000
-; GFX9-NEXT:    s_lshr_b32 s0, s0, s2
+; GFX9-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX9-NEXT:    v_mov_b32_e32 v0, s1
+; GFX9-NEXT:    v_add_u16_e64 v0, s0, v0 clamp
+; GFX9-NEXT:    v_lshrrev_b16_e32 v0, 8, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_uaddsat_i8:
@@ -213,14 +180,9 @@ define amdgpu_ps i8 @s_uaddsat_i8(i8 inreg %lhs, i8 inreg %rhs) {
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_lshl_b32 s0, s0, s2
 ; GFX10-NEXT:    s_lshl_b32 s1, s1, s2
-; GFX10-NEXT:    s_xor_b32 s3, s0, -1
-; GFX10-NEXT:    s_bfe_u32 s1, s1, 0x100000
-; GFX10-NEXT:    s_bfe_u32 s3, s3, 0x100000
-; GFX10-NEXT:    s_cmp_lt_u32 s3, s1
-; GFX10-NEXT:    s_cselect_b32 s1, s3, s1
-; GFX10-NEXT:    s_add_i32 s0, s0, s1
-; GFX10-NEXT:    s_bfe_u32 s0, s0, 0x100000
-; GFX10-NEXT:    s_lshr_b32 s0, s0, s2
+; GFX10-NEXT:    v_add_nc_u16_e64 v0, s0, s1 clamp
+; GFX10-NEXT:    v_lshrrev_b16_e64 v0, 8, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call i8 @llvm.uadd.sat.i8(i8 %lhs, i8 %rhs)
   ret i8 %result
@@ -256,16 +218,12 @@ define i16 @v_uaddsat_v2i8(i16 %lhs.arg, i16 %rhs.arg) {
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    v_mov_b32_e32 v2, 8
 ; GFX8-NEXT:    v_lshrrev_b32_sdwa v3, v2, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
 ; GFX8-NEXT:    v_lshrrev_b32_sdwa v2, v2, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
 ; GFX8-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
-; GFX8-NEXT:    v_xor_b32_e32 v4, -1, v0
-; GFX8-NEXT:    v_min_u16_e32 v1, v4, v1
-; GFX8-NEXT:    v_add_u16_e32 v0, v0, v1
-; GFX8-NEXT:    v_xor_b32_e32 v1, -1, v3
-; GFX8-NEXT:    v_min_u16_e32 v1, v1, v2
+; GFX8-NEXT:    v_add_u16_e64 v0, v0, v1 clamp
+; GFX8-NEXT:    v_add_u16_e64 v1, v3, v2 clamp
 ; GFX8-NEXT:    v_mov_b32_e32 v2, 0xff
-; GFX8-NEXT:    v_add_u16_e32 v1, v3, v1
 ; GFX8-NEXT:    v_and_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
 ; GFX8-NEXT:    v_and_b32_sdwa v1, v1, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
 ; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
@@ -276,16 +234,12 @@ define i16 @v_uaddsat_v2i8(i16 %lhs.arg, i16 %rhs.arg) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    s_mov_b32 s4, 8
 ; GFX9-NEXT:    v_lshrrev_b32_sdwa v2, s4, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
 ; GFX9-NEXT:    v_lshrrev_b32_sdwa v3, s4, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
-; GFX9-NEXT:    v_xor_b32_e32 v4, -1, v0
-; GFX9-NEXT:    v_min_u16_e32 v1, v4, v1
-; GFX9-NEXT:    v_add_u16_e32 v0, v0, v1
-; GFX9-NEXT:    v_xor_b32_e32 v1, -1, v2
-; GFX9-NEXT:    v_min_u16_e32 v1, v1, v3
+; GFX9-NEXT:    v_add_u16_e64 v0, v0, v1 clamp
 ; GFX9-NEXT:    s_movk_i32 s4, 0xff
-; GFX9-NEXT:    v_add_u16_e32 v1, v2, v1
+; GFX9-NEXT:    v_add_u16_e64 v1, v2, v3 clamp
 ; GFX9-NEXT:    v_and_b32_sdwa v0, v0, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
 ; GFX9-NEXT:    v_and_b32_sdwa v1, v1, s4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v0, v0, v1
@@ -296,21 +250,17 @@ define i16 @v_uaddsat_v2i8(i16 %lhs.arg, i16 %rhs.arg) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    s_mov_b32 s4, 8
-; GFX10-NEXT:    v_lshlrev_b16_e64 v2, 8, v0
-; GFX10-NEXT:    v_lshrrev_b32_sdwa v0, s4, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT:    v_lshrrev_b32_sdwa v3, s4, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT:    v_lshlrev_b16_e64 v1, 8, v1
+; GFX10-NEXT:    v_lshlrev_b16_e64 v3, 8, v1
+; GFX10-NEXT:    v_lshrrev_b32_sdwa v2, s4, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    v_lshrrev_b32_sdwa v1, s4, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    v_lshlrev_b16_e64 v0, 8, v0
 ; GFX10-NEXT:    s_movk_i32 s4, 0xff
-; GFX10-NEXT:    v_xor_b32_e32 v4, -1, v2
-; GFX10-NEXT:    v_xor_b32_e32 v5, -1, v0
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_min_u16_e64 v1, v4, v1
-; GFX10-NEXT:    v_min_u16_e64 v3, v5, v3
-; GFX10-NEXT:    v_add_nc_u16_e64 v1, v2, v1
-; GFX10-NEXT:    v_add_nc_u16_e64 v0, v0, v3
-; GFX10-NEXT:    v_and_b32_sdwa v1, v1, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
-; GFX10-NEXT:    v_and_b32_sdwa v0, v0, s4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
-; GFX10-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX10-NEXT:    v_add_nc_u16_e64 v1, v2, v1 clamp
+; GFX10-NEXT:    v_add_nc_u16_e64 v0, v0, v3 clamp
+; GFX10-NEXT:    v_and_b32_sdwa v1, v1, s4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_and_b32_sdwa v0, v0, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %lhs = bitcast i16 %lhs.arg to <2 x i8>
   %rhs = bitcast i16 %rhs.arg to <2 x i8>
@@ -348,98 +298,60 @@ define amdgpu_ps i16 @s_uaddsat_v2i8(i16 inreg %lhs.arg, i16 inreg %rhs.arg) {
 ; GFX8-LABEL: s_uaddsat_v2i8:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_bfe_u32 s4, 8, 0x100000
-; GFX8-NEXT:    s_lshr_b32 s2, s0, 8
-; GFX8-NEXT:    s_lshl_b32 s0, s0, s4
-; GFX8-NEXT:    s_xor_b32 s5, s0, -1
 ; GFX8-NEXT:    s_lshr_b32 s3, s1, 8
 ; GFX8-NEXT:    s_lshl_b32 s1, s1, s4
-; GFX8-NEXT:    s_bfe_u32 s5, s5, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s5, s1
-; GFX8-NEXT:    s_cselect_b32 s1, s5, s1
-; GFX8-NEXT:    s_add_i32 s0, s0, s1
-; GFX8-NEXT:    s_lshl_b32 s1, s2, s4
-; GFX8-NEXT:    s_lshl_b32 s2, s3, s4
-; GFX8-NEXT:    s_bfe_u32 s0, s0, 0x100000
-; GFX8-NEXT:    s_xor_b32 s3, s1, -1
-; GFX8-NEXT:    s_lshr_b32 s0, s0, s4
-; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s3, s2
-; GFX8-NEXT:    s_cselect_b32 s2, s3, s2
-; GFX8-NEXT:    s_add_i32 s1, s1, s2
-; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
-; GFX8-NEXT:    s_movk_i32 s2, 0xff
-; GFX8-NEXT:    s_lshr_b32 s1, s1, s4
-; GFX8-NEXT:    s_and_b32 s1, s1, s2
-; GFX8-NEXT:    s_and_b32 s0, s0, s2
-; GFX8-NEXT:    s_lshl_b32 s1, s1, s4
-; GFX8-NEXT:    s_or_b32 s0, s0, s1
+; GFX8-NEXT:    s_lshr_b32 s2, s0, 8
+; GFX8-NEXT:    v_mov_b32_e32 v0, s1
+; GFX8-NEXT:    s_lshl_b32 s0, s0, s4
+; GFX8-NEXT:    s_lshl_b32 s1, s3, s4
+; GFX8-NEXT:    v_add_u16_e64 v0, s0, v0 clamp
+; GFX8-NEXT:    s_lshl_b32 s0, s2, s4
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    v_mov_b32_e32 v2, 0xff
+; GFX8-NEXT:    v_add_u16_e64 v1, s0, v1 clamp
+; GFX8-NEXT:    v_and_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX8-NEXT:    v_and_b32_sdwa v1, v1, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: s_uaddsat_v2i8:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_bfe_u32 s4, 8, 0x100000
-; GFX9-NEXT:    s_lshr_b32 s2, s0, 8
-; GFX9-NEXT:    s_lshl_b32 s0, s0, s4
-; GFX9-NEXT:    s_xor_b32 s5, s0, -1
 ; GFX9-NEXT:    s_lshr_b32 s3, s1, 8
 ; GFX9-NEXT:    s_lshl_b32 s1, s1, s4
-; GFX9-NEXT:    s_bfe_u32 s5, s5, 0x100000
-; GFX9-NEXT:    s_bfe_u32 s1, s1, 0x100000
-; GFX9-NEXT:    s_cmp_lt_u32 s5, s1
-; GFX9-NEXT:    s_cselect_b32 s1, s5, s1
-; GFX9-NEXT:    s_add_i32 s0, s0, s1
-; GFX9-NEXT:    s_lshl_b32 s1, s2, s4
-; GFX9-NEXT:    s_lshl_b32 s2, s3, s4
-; GFX9-NEXT:    s_bfe_u32 s0, s0, 0x100000
-; GFX9-NEXT:    s_xor_b32 s3, s1, -1
-; GFX9-NEXT:    s_lshr_b32 s0, s0, s4
-; GFX9-NEXT:    s_bfe_u32 s3, s3, 0x100000
-; GFX9-NEXT:    s_bfe_u32 s2, s2, 0x100000
-; GFX9-NEXT:    s_cmp_lt_u32 s3, s2
-; GFX9-NEXT:    s_cselect_b32 s2, s3, s2
-; GFX9-NEXT:    s_add_i32 s1, s1, s2
-; GFX9-NEXT:    s_bfe_u32 s1, s1, 0x100000
-; GFX9-NEXT:    s_movk_i32 s2, 0xff
-; GFX9-NEXT:    s_lshr_b32 s1, s1, s4
-; GFX9-NEXT:    s_and_b32 s1, s1, s2
-; GFX9-NEXT:    s_and_b32 s0, s0, s2
-; GFX9-NEXT:    s_lshl_b32 s1, s1, s4
-; GFX9-NEXT:    s_or_b32 s0, s0, s1
+; GFX9-NEXT:    s_lshr_b32 s2, s0, 8
+; GFX9-NEXT:    v_mov_b32_e32 v0, s1
+; GFX9-NEXT:    s_lshl_b32 s0, s0, s4
+; GFX9-NEXT:    s_lshl_b32 s1, s3, s4
+; GFX9-NEXT:    v_add_u16_e64 v0, s0, v0 clamp
+; GFX9-NEXT:    s_lshl_b32 s0, s2, s4
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_add_u16_e64 v1, s0, v1 clamp
+; GFX9-NEXT:    s_movk_i32 s0, 0xff
+; GFX9-NEXT:    v_and_b32_sdwa v0, v0, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-NEXT:    v_and_b32_sdwa v1, v1, s0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_uaddsat_v2i8:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_bfe_u32 s2, 8, 0x100000
+; GFX10-NEXT:    s_lshr_b32 s2, s0, 8
+; GFX10-NEXT:    s_bfe_u32 s3, 8, 0x100000
+; GFX10-NEXT:    s_lshr_b32 s4, s1, 8
+; GFX10-NEXT:    s_lshl_b32 s0, s0, s3
+; GFX10-NEXT:    s_lshl_b32 s1, s1, s3
+; GFX10-NEXT:    s_lshl_b32 s2, s2, s3
+; GFX10-NEXT:    s_lshl_b32 s3, s4, s3
+; GFX10-NEXT:    v_add_nc_u16_e64 v0, s0, s1 clamp
+; GFX10-NEXT:    v_add_nc_u16_e64 v1, s2, s3 clamp
+; GFX10-NEXT:    s_movk_i32 s0, 0xff
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_lshl_b32 s3, s0, s2
-; GFX10-NEXT:    s_lshl_b32 s5, s1, s2
-; GFX10-NEXT:    s_xor_b32 s4, s3, -1
-; GFX10-NEXT:    s_bfe_u32 s5, s5, 0x100000
-; GFX10-NEXT:    s_bfe_u32 s4, s4, 0x100000
-; GFX10-NEXT:    s_lshr_b32 s0, s0, 8
-; GFX10-NEXT:    s_lshr_b32 s1, s1, 8
-; GFX10-NEXT:    s_cmp_lt_u32 s4, s5
-; GFX10-NEXT:    s_cselect_b32 s4, s4, s5
-; GFX10-NEXT:    s_lshl_b32 s0, s0, s2
-; GFX10-NEXT:    s_add_i32 s3, s3, s4
-; GFX10-NEXT:    s_xor_b32 s4, s0, -1
-; GFX10-NEXT:    s_lshl_b32 s1, s1, s2
-; GFX10-NEXT:    s_bfe_u32 s3, s3, 0x100000
-; GFX10-NEXT:    s_bfe_u32 s4, s4, 0x100000
-; GFX10-NEXT:    s_bfe_u32 s1, s1, 0x100000
-; GFX10-NEXT:    s_lshr_b32 s3, s3, s2
-; GFX10-NEXT:    s_cmp_lt_u32 s4, s1
-; GFX10-NEXT:    s_cselect_b32 s1, s4, s1
-; GFX10-NEXT:    s_add_i32 s0, s0, s1
-; GFX10-NEXT:    s_movk_i32 s1, 0xff
-; GFX10-NEXT:    s_bfe_u32 s0, s0, 0x100000
-; GFX10-NEXT:    s_lshr_b32 s0, s0, s2
-; GFX10-NEXT:    s_and_b32 s0, s0, s1
-; GFX10-NEXT:    s_and_b32 s1, s3, s1
-; GFX10-NEXT:    s_lshl_b32 s0, s0, s2
-; GFX10-NEXT:    s_or_b32 s0, s1, s0
+; GFX10-NEXT:    v_and_b32_sdwa v0, v0, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_and_b32_sdwa v1, v1, s0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
   %lhs = bitcast i16 %lhs.arg to <2 x i8>
   %rhs = bitcast i16 %rhs.arg to <2 x i8>
@@ -502,27 +414,19 @@ define i32 @v_uaddsat_v4i8(i32 %lhs.arg, i32 %rhs.arg) {
 ; GFX8-NEXT:    v_lshrrev_b32_sdwa v3, v2, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v4, 16, v0
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v5, 24, v0
-; GFX8-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
 ; GFX8-NEXT:    v_lshrrev_b32_sdwa v2, v2, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v6, 16, v1
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v7, 24, v1
+; GFX8-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
 ; GFX8-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
-; GFX8-NEXT:    v_xor_b32_e32 v8, -1, v0
-; GFX8-NEXT:    v_min_u16_e32 v1, v8, v1
-; GFX8-NEXT:    v_add_u16_e32 v0, v0, v1
-; GFX8-NEXT:    v_xor_b32_e32 v1, -1, v3
-; GFX8-NEXT:    v_min_u16_e32 v1, v1, v2
+; GFX8-NEXT:    v_add_u16_e64 v0, v0, v1 clamp
+; GFX8-NEXT:    v_add_u16_e64 v1, v3, v2 clamp
 ; GFX8-NEXT:    v_lshlrev_b16_e32 v2, 8, v4
-; GFX8-NEXT:    v_add_u16_e32 v1, v3, v1
 ; GFX8-NEXT:    v_lshlrev_b16_e32 v3, 8, v6
-; GFX8-NEXT:    v_xor_b32_e32 v4, -1, v2
-; GFX8-NEXT:    v_min_u16_e32 v3, v4, v3
-; GFX8-NEXT:    v_add_u16_e32 v2, v2, v3
+; GFX8-NEXT:    v_add_u16_e64 v2, v2, v3 clamp
 ; GFX8-NEXT:    v_lshlrev_b16_e32 v3, 8, v5
 ; GFX8-NEXT:    v_lshlrev_b16_e32 v4, 8, v7
-; GFX8-NEXT:    v_xor_b32_e32 v5, -1, v3
-; GFX8-NEXT:    v_min_u16_e32 v4, v5, v4
-; GFX8-NEXT:    v_add_u16_e32 v3, v3, v4
+; GFX8-NEXT:    v_add_u16_e64 v3, v3, v4 clamp
 ; GFX8-NEXT:    v_mov_b32_e32 v4, 0xff
 ; GFX8-NEXT:    v_and_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
 ; GFX8-NEXT:    v_and_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
@@ -539,33 +443,25 @@ define i32 @v_uaddsat_v4i8(i32 %lhs.arg, i32 %rhs.arg) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    s_mov_b32 s4, 8
 ; GFX9-NEXT:    v_lshrrev_b32_sdwa v2, s4, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT:    v_lshrrev_b32_sdwa v5, s4, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v4, 24, v0
-; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
-; GFX9-NEXT:    v_lshrrev_b32_sdwa v5, s4, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v6, 16, v1
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v7, 24, v1
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
-; GFX9-NEXT:    v_xor_b32_e32 v8, -1, v0
-; GFX9-NEXT:    v_min_u16_e32 v1, v8, v1
-; GFX9-NEXT:    v_add_u16_e32 v0, v0, v1
-; GFX9-NEXT:    v_xor_b32_e32 v1, -1, v2
-; GFX9-NEXT:    v_min_u16_e32 v1, v1, v5
-; GFX9-NEXT:    v_add_u16_e32 v1, v2, v1
+; GFX9-NEXT:    v_add_u16_e64 v0, v0, v1 clamp
+; GFX9-NEXT:    v_add_u16_e64 v1, v2, v5 clamp
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v2, 8, v3
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v3, 8, v6
-; GFX9-NEXT:    v_xor_b32_e32 v5, -1, v2
-; GFX9-NEXT:    v_min_u16_e32 v3, v5, v3
-; GFX9-NEXT:    v_add_u16_e32 v2, v2, v3
-; GFX9-NEXT:    v_lshlrev_b16_e32 v3, 8, v4
 ; GFX9-NEXT:    s_movk_i32 s4, 0xff
-; GFX9-NEXT:    v_lshlrev_b16_e32 v4, 8, v7
-; GFX9-NEXT:    v_xor_b32_e32 v5, -1, v3
 ; GFX9-NEXT:    v_and_b32_sdwa v1, v1, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
-; GFX9-NEXT:    v_min_u16_e32 v4, v5, v4
+; GFX9-NEXT:    v_add_u16_e64 v2, v2, v3 clamp
+; GFX9-NEXT:    v_lshlrev_b16_e32 v3, 8, v4
+; GFX9-NEXT:    v_lshlrev_b16_e32 v4, 8, v7
 ; GFX9-NEXT:    v_lshrrev_b16_e32 v0, 8, v0
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
-; GFX9-NEXT:    v_add_u16_e32 v3, v3, v4
+; GFX9-NEXT:    v_add_u16_e64 v3, v3, v4 clamp
 ; GFX9-NEXT:    v_and_or_b32 v0, v0, s4, v1
 ; GFX9-NEXT:    v_and_b32_sdwa v1, v2, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
 ; GFX9-NEXT:    v_and_b32_sdwa v2, v3, s4 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
@@ -577,36 +473,28 @@ define i32 @v_uaddsat_v4i8(i32 %lhs.arg, i32 %rhs.arg) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    s_mov_b32 s4, 8
-; GFX10-NEXT:    v_lshlrev_b16_e64 v4, 8, v0
+; GFX10-NEXT:    v_lshlrev_b16_e64 v5, 8, v0
 ; GFX10-NEXT:    v_lshrrev_b32_sdwa v2, s4, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX10-NEXT:    v_lshrrev_b32_sdwa v3, s4, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    v_lshlrev_b16_e64 v6, 8, v1
 ; GFX10-NEXT:    s_mov_b32 s5, 16
-; GFX10-NEXT:    s_mov_b32 s6, 24
-; GFX10-NEXT:    v_lshrrev_b32_sdwa v6, s5, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT:    v_xor_b32_e32 v5, -1, v2
-; GFX10-NEXT:    v_lshrrev_b32_sdwa v0, s6, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT:    v_lshlrev_b16_e64 v7, 8, v1
-; GFX10-NEXT:    v_xor_b32_e32 v8, -1, v4
-; GFX10-NEXT:    v_xor_b32_e32 v11, -1, v6
-; GFX10-NEXT:    v_min_u16_e64 v3, v5, v3
-; GFX10-NEXT:    v_lshrrev_b32_sdwa v5, s5, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT:    v_lshrrev_b32_sdwa v1, s6, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT:    v_min_u16_e64 v7, v8, v7
-; GFX10-NEXT:    s_movk_i32 s4, 0xff
-; GFX10-NEXT:    v_add_nc_u16_e64 v2, v2, v3
-; GFX10-NEXT:    v_xor_b32_e32 v3, -1, v0
-; GFX10-NEXT:    v_min_u16_e64 v5, v11, v5
-; GFX10-NEXT:    v_add_nc_u16_e64 v4, v4, v7
+; GFX10-NEXT:    s_mov_b32 s4, 24
+; GFX10-NEXT:    v_lshrrev_b32_sdwa v4, s5, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    v_add_nc_u16_e64 v2, v2, v3 clamp
+; GFX10-NEXT:    v_lshrrev_b32_sdwa v3, s5, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    s_movk_i32 s5, 0xff
+; GFX10-NEXT:    v_lshrrev_b32_sdwa v0, s4, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    v_lshrrev_b32_sdwa v1, s4, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    v_and_b32_sdwa v2, v2, s5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_add_nc_u16_e64 v5, v5, v6 clamp
+; GFX10-NEXT:    v_add_nc_u16_e64 v3, v4, v3 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_and_b32_sdwa v2, v2, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
-; GFX10-NEXT:    v_min_u16_e64 v1, v3, v1
-; GFX10-NEXT:    v_add_nc_u16_e64 v3, v6, v5
-; GFX10-NEXT:    v_lshrrev_b16_e64 v4, 8, v4
+; GFX10-NEXT:    v_add_nc_u16_e64 v0, v0, v1 clamp
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
-; GFX10-NEXT:    v_add_nc_u16_e64 v0, v0, v1
-; GFX10-NEXT:    v_and_b32_sdwa v1, v3, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
-; GFX10-NEXT:    v_and_or_b32 v2, v4, s4, v2
-; GFX10-NEXT:    v_and_b32_sdwa v0, v0, s4 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_lshrrev_b16_e64 v4, 8, v5
+; GFX10-NEXT:    v_and_b32_sdwa v1, v3, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_and_b32_sdwa v0, v0, s5 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_and_or_b32 v2, v4, s5, v2
 ; GFX10-NEXT:    v_or3_b32 v0, v2, v1, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %lhs = bitcast i32 %lhs.arg to <4 x i8>
@@ -669,188 +557,107 @@ define amdgpu_ps i32 @s_uaddsat_v4i8(i32 inreg %lhs.arg, i32 inreg %rhs.arg) {
 ; GFX8-LABEL: s_uaddsat_v4i8:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_bfe_u32 s8, 8, 0x100000
-; GFX8-NEXT:    s_lshr_b32 s2, s0, 8
-; GFX8-NEXT:    s_lshr_b32 s3, s0, 16
-; GFX8-NEXT:    s_lshr_b32 s4, s0, 24
-; GFX8-NEXT:    s_lshl_b32 s0, s0, s8
-; GFX8-NEXT:    s_xor_b32 s9, s0, -1
 ; GFX8-NEXT:    s_lshr_b32 s5, s1, 8
 ; GFX8-NEXT:    s_lshr_b32 s6, s1, 16
 ; GFX8-NEXT:    s_lshr_b32 s7, s1, 24
 ; GFX8-NEXT:    s_lshl_b32 s1, s1, s8
-; GFX8-NEXT:    s_bfe_u32 s9, s9, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s9, s1
-; GFX8-NEXT:    s_cselect_b32 s1, s9, s1
-; GFX8-NEXT:    s_add_i32 s0, s0, s1
-; GFX8-NEXT:    s_lshl_b32 s1, s2, s8
-; GFX8-NEXT:    s_lshl_b32 s2, s5, s8
-; GFX8-NEXT:    s_bfe_u32 s0, s0, 0x100000
-; GFX8-NEXT:    s_xor_b32 s5, s1, -1
-; GFX8-NEXT:    s_lshr_b32 s0, s0, s8
-; GFX8-NEXT:    s_bfe_u32 s5, s5, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s5, s2
-; GFX8-NEXT:    s_cselect_b32 s2, s5, s2
-; GFX8-NEXT:    s_add_i32 s1, s1, s2
-; GFX8-NEXT:    s_lshl_b32 s2, s3, s8
-; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
-; GFX8-NEXT:    s_lshl_b32 s3, s6, s8
-; GFX8-NEXT:    s_xor_b32 s5, s2, -1
-; GFX8-NEXT:    s_lshr_b32 s1, s1, s8
-; GFX8-NEXT:    s_bfe_u32 s5, s5, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s5, s3
-; GFX8-NEXT:    s_cselect_b32 s3, s5, s3
-; GFX8-NEXT:    s_add_i32 s2, s2, s3
-; GFX8-NEXT:    s_lshl_b32 s3, s4, s8
-; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
-; GFX8-NEXT:    s_lshl_b32 s4, s7, s8
-; GFX8-NEXT:    s_xor_b32 s5, s3, -1
-; GFX8-NEXT:    s_lshr_b32 s2, s2, s8
-; GFX8-NEXT:    s_bfe_u32 s5, s5, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s4, s4, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s5, s4
-; GFX8-NEXT:    s_cselect_b32 s4, s5, s4
-; GFX8-NEXT:    s_add_i32 s3, s3, s4
-; GFX8-NEXT:    s_movk_i32 s4, 0xff
-; GFX8-NEXT:    s_and_b32 s1, s1, s4
-; GFX8-NEXT:    s_and_b32 s0, s0, s4
-; GFX8-NEXT:    s_lshl_b32 s1, s1, 8
-; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
-; GFX8-NEXT:    s_or_b32 s0, s0, s1
-; GFX8-NEXT:    s_and_b32 s1, s2, s4
-; GFX8-NEXT:    s_lshl_b32 s1, s1, 16
-; GFX8-NEXT:    s_lshr_b32 s3, s3, s8
-; GFX8-NEXT:    s_or_b32 s0, s0, s1
-; GFX8-NEXT:    s_and_b32 s1, s3, s4
-; GFX8-NEXT:    s_lshl_b32 s1, s1, 24
-; GFX8-NEXT:    s_or_b32 s0, s0, s1
+; GFX8-NEXT:    v_mov_b32_e32 v0, s1
+; GFX8-NEXT:    s_lshl_b32 s1, s5, s8
+; GFX8-NEXT:    s_lshr_b32 s2, s0, 8
+; GFX8-NEXT:    s_lshr_b32 s3, s0, 16
+; GFX8-NEXT:    s_lshr_b32 s4, s0, 24
+; GFX8-NEXT:    s_lshl_b32 s0, s0, s8
+; GFX8-NEXT:    v_add_u16_e64 v0, s0, v0 clamp
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    s_lshl_b32 s0, s2, s8
+; GFX8-NEXT:    v_add_u16_e64 v1, s0, v1 clamp
+; GFX8-NEXT:    v_mov_b32_e32 v4, 0xff
+; GFX8-NEXT:    s_lshl_b32 s1, s6, s8
+; GFX8-NEXT:    v_and_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX8-NEXT:    v_mov_b32_e32 v2, s1
+; GFX8-NEXT:    s_lshl_b32 s0, s3, s8
+; GFX8-NEXT:    s_lshl_b32 s1, s7, s8
+; GFX8-NEXT:    v_add_u16_e64 v2, s0, v2 clamp
+; GFX8-NEXT:    s_lshl_b32 s0, s4, s8
+; GFX8-NEXT:    v_mov_b32_e32 v3, s1
+; GFX8-NEXT:    v_and_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX8-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    v_and_b32_sdwa v1, v2, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX8-NEXT:    v_add_u16_e64 v3, s0, v3 clamp
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    v_and_b32_sdwa v1, v3, v4 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: s_uaddsat_v4i8:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_bfe_u32 s8, 8, 0x100000
-; GFX9-NEXT:    s_lshr_b32 s2, s0, 8
-; GFX9-NEXT:    s_lshr_b32 s3, s0, 16
-; GFX9-NEXT:    s_lshr_b32 s4, s0, 24
-; GFX9-NEXT:    s_lshl_b32 s0, s0, s8
-; GFX9-NEXT:    s_xor_b32 s9, s0, -1
 ; GFX9-NEXT:    s_lshr_b32 s5, s1, 8
 ; GFX9-NEXT:    s_lshr_b32 s6, s1, 16
 ; GFX9-NEXT:    s_lshr_b32 s7, s1, 24
 ; GFX9-NEXT:    s_lshl_b32 s1, s1, s8
-; GFX9-NEXT:    s_bfe_u32 s9, s9, 0x100000
-; GFX9-NEXT:    s_bfe_u32 s1, s1, 0x100000
-; GFX9-NEXT:    s_cmp_lt_u32 s9, s1
-; GFX9-NEXT:    s_cselect_b32 s1, s9, s1
-; GFX9-NEXT:    s_add_i32 s0, s0, s1
-; GFX9-NEXT:    s_lshl_b32 s1, s2, s8
-; GFX9-NEXT:    s_lshl_b32 s2, s5, s8
-; GFX9-NEXT:    s_bfe_u32 s0, s0, 0x100000
-; GFX9-NEXT:    s_xor_b32 s5, s1, -1
-; GFX9-NEXT:    s_lshr_b32 s0, s0, s8
-; GFX9-NEXT:    s_bfe_u32 s5, s5, 0x100000
-; GFX9-NEXT:    s_bfe_u32 s2, s2, 0x100000
-; GFX9-NEXT:    s_cmp_lt_u32 s5, s2
-; GFX9-NEXT:    s_cselect_b32 s2, s5, s2
-; GFX9-NEXT:    s_add_i32 s1, s1, s2
-; GFX9-NEXT:    s_lshl_b32 s2, s3, s8
-; GFX9-NEXT:    s_bfe_u32 s1, s1, 0x100000
-; GFX9-NEXT:    s_lshl_b32 s3, s6, s8
-; GFX9-NEXT:    s_xor_b32 s5, s2, -1
-; GFX9-NEXT:    s_lshr_b32 s1, s1, s8
-; GFX9-NEXT:    s_bfe_u32 s5, s5, 0x100000
-; GFX9-NEXT:    s_bfe_u32 s3, s3, 0x100000
-; GFX9-NEXT:    s_cmp_lt_u32 s5, s3
-; GFX9-NEXT:    s_cselect_b32 s3, s5, s3
-; GFX9-NEXT:    s_add_i32 s2, s2, s3
-; GFX9-NEXT:    s_lshl_b32 s3, s4, s8
-; GFX9-NEXT:    s_bfe_u32 s2, s2, 0x100000
-; GFX9-NEXT:    s_lshl_b32 s4, s7, s8
-; GFX9-NEXT:    s_xor_b32 s5, s3, -1
-; GFX9-NEXT:    s_lshr_b32 s2, s2, s8
-; GFX9-NEXT:    s_bfe_u32 s5, s5, 0x100000
-; GFX9-NEXT:    s_bfe_u32 s4, s4, 0x100000
-; GFX9-NEXT:    s_cmp_lt_u32 s5, s4
-; GFX9-NEXT:    s_cselect_b32 s4, s5, s4
-; GFX9-NEXT:    s_add_i32 s3, s3, s4
-; GFX9-NEXT:    s_movk_i32 s4, 0xff
-; GFX9-NEXT:    s_and_b32 s1, s1, s4
-; GFX9-NEXT:    s_and_b32 s0, s0, s4
-; GFX9-NEXT:    s_lshl_b32 s1, s1, 8
-; GFX9-NEXT:    s_bfe_u32 s3, s3, 0x100000
-; GFX9-NEXT:    s_or_b32 s0, s0, s1
-; GFX9-NEXT:    s_and_b32 s1, s2, s4
-; GFX9-NEXT:    s_lshl_b32 s1, s1, 16
-; GFX9-NEXT:    s_lshr_b32 s3, s3, s8
-; GFX9-NEXT:    s_or_b32 s0, s0, s1
-; GFX9-NEXT:    s_and_b32 s1, s3, s4
-; GFX9-NEXT:    s_lshl_b32 s1, s1, 24
-; GFX9-NEXT:    s_or_b32 s0, s0, s1
+; GFX9-NEXT:    v_mov_b32_e32 v0, s1
+; GFX9-NEXT:    s_lshl_b32 s1, s5, s8
+; GFX9-NEXT:    s_lshr_b32 s2, s0, 8
+; GFX9-NEXT:    s_lshr_b32 s3, s0, 16
+; GFX9-NEXT:    s_lshr_b32 s4, s0, 24
+; GFX9-NEXT:    s_lshl_b32 s0, s0, s8
+; GFX9-NEXT:    v_add_u16_e64 v0, s0, v0 clamp
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    s_lshl_b32 s0, s2, s8
+; GFX9-NEXT:    s_lshl_b32 s1, s6, s8
+; GFX9-NEXT:    v_add_u16_e64 v1, s0, v1 clamp
+; GFX9-NEXT:    v_mov_b32_e32 v2, s1
+; GFX9-NEXT:    s_lshl_b32 s0, s3, s8
+; GFX9-NEXT:    s_lshl_b32 s1, s7, s8
+; GFX9-NEXT:    v_add_u16_e64 v2, s0, v2 clamp
+; GFX9-NEXT:    s_lshl_b32 s0, s4, s8
+; GFX9-NEXT:    v_mov_b32_e32 v3, s1
+; GFX9-NEXT:    v_add_u16_e64 v3, s0, v3 clamp
+; GFX9-NEXT:    s_movk_i32 s0, 0xff
+; GFX9-NEXT:    v_and_b32_sdwa v1, v1, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-NEXT:    v_lshrrev_b16_e32 v0, 8, v0
+; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX9-NEXT:    v_and_or_b32 v0, v0, s0, v1
+; GFX9-NEXT:    v_and_b32_sdwa v1, v2, s0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-NEXT:    v_and_b32_sdwa v2, v3, s0 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-NEXT:    v_or3_b32 v0, v0, v1, v2
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_uaddsat_v4i8:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_bfe_u32 s5, 8, 0x100000
 ; GFX10-NEXT:    s_lshr_b32 s2, s0, 8
+; GFX10-NEXT:    s_lshr_b32 s6, s1, 8
+; GFX10-NEXT:    s_lshl_b32 s2, s2, s5
+; GFX10-NEXT:    s_lshl_b32 s6, s6, s5
 ; GFX10-NEXT:    s_lshr_b32 s3, s0, 16
+; GFX10-NEXT:    v_add_nc_u16_e64 v1, s2, s6 clamp
 ; GFX10-NEXT:    s_lshr_b32 s4, s0, 24
+; GFX10-NEXT:    s_movk_i32 s2, 0xff
 ; GFX10-NEXT:    s_lshl_b32 s0, s0, s5
-; GFX10-NEXT:    s_lshl_b32 s9, s1, s5
-; GFX10-NEXT:    s_xor_b32 s8, s0, -1
-; GFX10-NEXT:    s_bfe_u32 s9, s9, 0x100000
-; GFX10-NEXT:    s_bfe_u32 s8, s8, 0x100000
-; GFX10-NEXT:    s_lshr_b32 s6, s1, 8
-; GFX10-NEXT:    s_lshr_b32 s7, s1, 16
+; GFX10-NEXT:    s_lshl_b32 s7, s1, s5
+; GFX10-NEXT:    v_and_b32_sdwa v1, v1, s2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_add_nc_u16_e64 v0, s0, s7 clamp
+; GFX10-NEXT:    s_lshr_b32 s0, s1, 16
 ; GFX10-NEXT:    s_lshr_b32 s1, s1, 24
-; GFX10-NEXT:    s_cmp_lt_u32 s8, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cselect_b32 s8, s8, s9
-; GFX10-NEXT:    s_lshl_b32 s2, s2, s5
-; GFX10-NEXT:    s_add_i32 s0, s0, s8
-; GFX10-NEXT:    s_xor_b32 s8, s2, -1
-; GFX10-NEXT:    s_lshl_b32 s6, s6, s5
-; GFX10-NEXT:    s_bfe_u32 s0, s0, 0x100000
-; GFX10-NEXT:    s_bfe_u32 s8, s8, 0x100000
-; GFX10-NEXT:    s_bfe_u32 s6, s6, 0x100000
-; GFX10-NEXT:    s_lshr_b32 s0, s0, s5
-; GFX10-NEXT:    s_cmp_lt_u32 s8, s6
-; GFX10-NEXT:    s_cselect_b32 s6, s8, s6
 ; GFX10-NEXT:    s_lshl_b32 s3, s3, s5
-; GFX10-NEXT:    s_add_i32 s2, s2, s6
-; GFX10-NEXT:    s_xor_b32 s6, s3, -1
-; GFX10-NEXT:    s_lshl_b32 s7, s7, s5
-; GFX10-NEXT:    s_bfe_u32 s2, s2, 0x100000
-; GFX10-NEXT:    s_bfe_u32 s6, s6, 0x100000
-; GFX10-NEXT:    s_bfe_u32 s7, s7, 0x100000
-; GFX10-NEXT:    s_lshr_b32 s2, s2, s5
-; GFX10-NEXT:    s_cmp_lt_u32 s6, s7
-; GFX10-NEXT:    s_cselect_b32 s6, s6, s7
+; GFX10-NEXT:    s_lshl_b32 s0, s0, s5
 ; GFX10-NEXT:    s_lshl_b32 s4, s4, s5
-; GFX10-NEXT:    s_add_i32 s3, s3, s6
-; GFX10-NEXT:    s_xor_b32 s6, s4, -1
 ; GFX10-NEXT:    s_lshl_b32 s1, s1, s5
-; GFX10-NEXT:    s_bfe_u32 s3, s3, 0x100000
-; GFX10-NEXT:    s_bfe_u32 s6, s6, 0x100000
-; GFX10-NEXT:    s_bfe_u32 s1, s1, 0x100000
-; GFX10-NEXT:    s_lshr_b32 s3, s3, s5
-; GFX10-NEXT:    s_cmp_lt_u32 s6, s1
-; GFX10-NEXT:    s_cselect_b32 s1, s6, s1
-; GFX10-NEXT:    s_add_i32 s4, s4, s1
-; GFX10-NEXT:    s_bfe_u32 s1, s4, 0x100000
-; GFX10-NEXT:    s_movk_i32 s4, 0xff
-; GFX10-NEXT:    s_lshr_b32 s1, s1, s5
-; GFX10-NEXT:    s_and_b32 s2, s2, s4
-; GFX10-NEXT:    s_and_b32 s0, s0, s4
-; GFX10-NEXT:    s_lshl_b32 s2, s2, 8
-; GFX10-NEXT:    s_and_b32 s3, s3, s4
-; GFX10-NEXT:    s_and_b32 s1, s1, s4
-; GFX10-NEXT:    s_or_b32 s0, s0, s2
-; GFX10-NEXT:    s_lshl_b32 s2, s3, 16
-; GFX10-NEXT:    s_lshl_b32 s1, s1, 24
-; GFX10-NEXT:    s_or_b32 s0, s0, s2
-; GFX10-NEXT:    s_or_b32 s0, s0, s1
+; GFX10-NEXT:    v_lshrrev_b16_e64 v0, 8, v0
+; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX10-NEXT:    v_add_nc_u16_e64 v2, s3, s0 clamp
+; GFX10-NEXT:    v_add_nc_u16_e64 v3, s4, s1 clamp
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_and_or_b32 v0, v0, s2, v1
+; GFX10-NEXT:    v_and_b32_sdwa v1, v2, s2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_and_b32_sdwa v2, v3, s2 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_or3_b32 v0, v0, v1, v2
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
   %lhs = bitcast i32 %lhs.arg to <4 x i8>
   %rhs = bitcast i32 %rhs.arg to <4 x i8>
@@ -876,9 +683,7 @@ define i24 @v_uaddsat_i24(i24 %lhs, i24 %rhs) {
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
-; GFX8-NEXT:    v_xor_b32_e32 v2, -1, v0
-; GFX8-NEXT:    v_min_u32_e32 v1, v2, v1
-; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v0, v1
+; GFX8-NEXT:    v_add_u32_e64 v0, s[4:5], v0, v1 clamp
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v0, 8, v0
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -887,9 +692,7 @@ define i24 @v_uaddsat_i24(i24 %lhs, i24 %rhs) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
-; GFX9-NEXT:    v_xor_b32_e32 v2, -1, v0
-; GFX9-NEXT:    v_min_u32_e32 v1, v2, v1
-; GFX9-NEXT:    v_add_u32_e32 v0, v0, v1
+; GFX9-NEXT:    v_add_u32_e64 v0, v0, v1 clamp
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 8, v0
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -900,9 +703,7 @@ define i24 @v_uaddsat_i24(i24 %lhs, i24 %rhs) {
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_xor_b32_e32 v2, -1, v0
-; GFX10-NEXT:    v_min_u32_e32 v1, v2, v1
-; GFX10-NEXT:    v_add_nc_u32_e32 v0, v0, v1
+; GFX10-NEXT:    v_add_nc_u32_e64 v0, v0, v1 clamp
 ; GFX10-NEXT:    v_lshrrev_b32_e32 v0, 8, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call i24 @llvm.uadd.sat.i24(i24 %lhs, i24 %rhs)
@@ -923,36 +724,32 @@ define amdgpu_ps i24 @s_uaddsat_i24(i24 inreg %lhs, i24 inreg %rhs) {
 ;
 ; GFX8-LABEL: s_uaddsat_i24:
 ; GFX8:       ; %bb.0:
-; GFX8-NEXT:    s_lshl_b32 s0, s0, 8
 ; GFX8-NEXT:    s_lshl_b32 s1, s1, 8
-; GFX8-NEXT:    s_not_b32 s2, s0
-; GFX8-NEXT:    s_cmp_lt_u32 s2, s1
-; GFX8-NEXT:    s_cselect_b32 s1, s2, s1
-; GFX8-NEXT:    s_add_i32 s0, s0, s1
-; GFX8-NEXT:    s_lshr_b32 s0, s0, 8
+; GFX8-NEXT:    s_lshl_b32 s0, s0, 8
+; GFX8-NEXT:    v_mov_b32_e32 v0, s1
+; GFX8-NEXT:    v_add_u32_e64 v0, s[0:1], s0, v0 clamp
+; GFX8-NEXT:    v_lshrrev_b32_e32 v0, 8, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: s_uaddsat_i24:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_lshl_b32 s0, s0, 8
 ; GFX9-NEXT:    s_lshl_b32 s1, s1, 8
-; GFX9-NEXT:    s_not_b32 s2, s0
-; GFX9-NEXT:    s_cmp_lt_u32 s2, s1
-; GFX9-NEXT:    s_cselect_b32 s1, s2, s1
-; GFX9-NEXT:    s_add_i32 s0, s0, s1
-; GFX9-NEXT:    s_lshr_b32 s0, s0, 8
+; GFX9-NEXT:    s_lshl_b32 s0, s0, 8
+; GFX9-NEXT:    v_mov_b32_e32 v0, s1
+; GFX9-NEXT:    v_add_u32_e64 v0, s0, v0 clamp
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 8, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_uaddsat_i24:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_lshl_b32 s0, s0, 8
 ; GFX10-NEXT:    s_lshl_b32 s1, s1, 8
-; GFX10-NEXT:    s_not_b32 s2, s0
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cmp_lt_u32 s2, s1
-; GFX10-NEXT:    s_cselect_b32 s1, s2, s1
-; GFX10-NEXT:    s_add_i32 s0, s0, s1
-; GFX10-NEXT:    s_lshr_b32 s0, s0, 8
+; GFX10-NEXT:    v_add_nc_u32_e64 v0, s0, s1 clamp
+; GFX10-NEXT:    v_lshrrev_b32_e32 v0, 8, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call i24 @llvm.uadd.sat.i24(i24 %lhs, i24 %rhs)
   ret i24 %result
@@ -970,27 +767,21 @@ define i32 @v_uaddsat_i32(i32 %lhs, i32 %rhs) {
 ; GFX8-LABEL: v_uaddsat_i32:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_xor_b32_e32 v2, -1, v0
-; GFX8-NEXT:    v_min_u32_e32 v1, v2, v1
-; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v0, v1
+; GFX8-NEXT:    v_add_u32_e64 v0, s[4:5], v0, v1 clamp
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_uaddsat_i32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_xor_b32_e32 v2, -1, v0
-; GFX9-NEXT:    v_min_u32_e32 v1, v2, v1
-; GFX9-NEXT:    v_add_u32_e32 v0, v0, v1
+; GFX9-NEXT:    v_add_u32_e64 v0, v0, v1 clamp
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_uaddsat_i32:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_xor_b32_e32 v2, -1, v0
+; GFX10-NEXT:    v_add_nc_u32_e64 v0, v0, v1 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_min_u32_e32 v1, v2, v1
-; GFX10-NEXT:    v_add_nc_u32_e32 v0, v0, v1
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call i32 @llvm.uadd.sat.i32(i32 %lhs, i32 %rhs)
   ret i32 %result
@@ -1007,27 +798,23 @@ define amdgpu_ps i32 @s_uaddsat_i32(i32 inreg %lhs, i32 inreg %rhs) {
 ;
 ; GFX8-LABEL: s_uaddsat_i32:
 ; GFX8:       ; %bb.0:
-; GFX8-NEXT:    s_not_b32 s2, s0
-; GFX8-NEXT:    s_cmp_lt_u32 s2, s1
-; GFX8-NEXT:    s_cselect_b32 s1, s2, s1
-; GFX8-NEXT:    s_add_i32 s0, s0, s1
+; GFX8-NEXT:    v_mov_b32_e32 v0, s1
+; GFX8-NEXT:    v_add_u32_e64 v0, s[0:1], s0, v0 clamp
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: s_uaddsat_i32:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_not_b32 s2, s0
-; GFX9-NEXT:    s_cmp_lt_u32 s2, s1
-; GFX9-NEXT:    s_cselect_b32 s1, s2, s1
-; GFX9-NEXT:    s_add_i32 s0, s0, s1
+; GFX9-NEXT:    v_mov_b32_e32 v0, s1
+; GFX9-NEXT:    v_add_u32_e64 v0, s0, v0 clamp
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_uaddsat_i32:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_not_b32 s2, s0
+; GFX10-NEXT:    v_add_nc_u32_e64 v0, s0, s1 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cmp_lt_u32 s2, s1
-; GFX10-NEXT:    s_cselect_b32 s1, s2, s1
-; GFX10-NEXT:    s_add_i32 s0, s0, s1
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call i32 @llvm.uadd.sat.i32(i32 %lhs, i32 %rhs)
   ret i32 %result
@@ -1043,24 +830,18 @@ define amdgpu_ps float @uaddsat_i32_sv(i32 inreg %lhs, i32 %rhs) {
 ;
 ; GFX8-LABEL: uaddsat_i32_sv:
 ; GFX8:       ; %bb.0:
-; GFX8-NEXT:    s_not_b32 s1, s0
-; GFX8-NEXT:    v_min_u32_e32 v0, s1, v0
-; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; GFX8-NEXT:    v_add_u32_e64 v0, s[0:1], s0, v0 clamp
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: uaddsat_i32_sv:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_not_b32 s1, s0
-; GFX9-NEXT:    v_min_u32_e32 v0, s1, v0
-; GFX9-NEXT:    v_add_u32_e32 v0, s0, v0
+; GFX9-NEXT:    v_add_u32_e64 v0, s0, v0 clamp
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: uaddsat_i32_sv:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_not_b32 s1, s0
+; GFX10-NEXT:    v_add_nc_u32_e64 v0, s0, v0 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_min_u32_e32 v0, s1, v0
-; GFX10-NEXT:    v_add_nc_u32_e32 v0, s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call i32 @llvm.uadd.sat.i32(i32 %lhs, i32 %rhs)
   %cast = bitcast i32 %result to float
@@ -1077,24 +858,18 @@ define amdgpu_ps float @uaddsat_i32_vs(i32 %lhs, i32 inreg %rhs) {
 ;
 ; GFX8-LABEL: uaddsat_i32_vs:
 ; GFX8:       ; %bb.0:
-; GFX8-NEXT:    v_xor_b32_e32 v1, -1, v0
-; GFX8-NEXT:    v_min_u32_e32 v1, s0, v1
-; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v0, v1
+; GFX8-NEXT:    v_add_u32_e64 v0, s[0:1], v0, s0 clamp
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: uaddsat_i32_vs:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    v_xor_b32_e32 v1, -1, v0
-; GFX9-NEXT:    v_min_u32_e32 v1, s0, v1
-; GFX9-NEXT:    v_add_u32_e32 v0, v0, v1
+; GFX9-NEXT:    v_add_u32_e64 v0, v0, s0 clamp
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: uaddsat_i32_vs:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    v_xor_b32_e32 v1, -1, v0
+; GFX10-NEXT:    v_add_nc_u32_e64 v0, v0, s0 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_min_u32_e32 v1, s0, v1
-; GFX10-NEXT:    v_add_nc_u32_e32 v0, v0, v1
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call i32 @llvm.uadd.sat.i32(i32 %lhs, i32 %rhs)
   %cast = bitcast i32 %result to float
@@ -1116,36 +891,24 @@ define <2 x i32> @v_uaddsat_v2i32(<2 x i32> %lhs, <2 x i32> %rhs) {
 ; GFX8-LABEL: v_uaddsat_v2i32:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_xor_b32_e32 v4, -1, v0
-; GFX8-NEXT:    v_min_u32_e32 v2, v4, v2
-; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v0, v2
-; GFX8-NEXT:    v_xor_b32_e32 v2, -1, v1
-; GFX8-NEXT:    v_min_u32_e32 v2, v2, v3
-; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v1, v2
+; GFX8-NEXT:    v_add_u32_e64 v0, s[4:5], v0, v2 clamp
+; GFX8-NEXT:    v_add_u32_e64 v1, s[4:5], v1, v3 clamp
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_uaddsat_v2i32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_xor_b32_e32 v4, -1, v0
-; GFX9-NEXT:    v_min_u32_e32 v2, v4, v2
-; GFX9-NEXT:    v_add_u32_e32 v0, v0, v2
-; GFX9-NEXT:    v_xor_b32_e32 v2, -1, v1
-; GFX9-NEXT:    v_min_u32_e32 v2, v2, v3
-; GFX9-NEXT:    v_add_u32_e32 v1, v1, v2
+; GFX9-NEXT:    v_add_u32_e64 v0, v0, v2 clamp
+; GFX9-NEXT:    v_add_u32_e64 v1, v1, v3 clamp
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_uaddsat_v2i32:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_xor_b32_e32 v4, -1, v0
-; GFX10-NEXT:    v_xor_b32_e32 v5, -1, v1
+; GFX10-NEXT:    v_add_nc_u32_e64 v0, v0, v2 clamp
+; GFX10-NEXT:    v_add_nc_u32_e64 v1, v1, v3 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_min_u32_e32 v2, v4, v2
-; GFX10-NEXT:    v_min_u32_e32 v3, v5, v3
-; GFX10-NEXT:    v_add_nc_u32_e32 v0, v0, v2
-; GFX10-NEXT:    v_add_nc_u32_e32 v1, v1, v3
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <2 x i32> @llvm.uadd.sat.v2i32(<2 x i32> %lhs, <2 x i32> %rhs)
   ret <2 x i32> %result
@@ -1166,39 +929,31 @@ define amdgpu_ps <2 x i32> @s_uaddsat_v2i32(<2 x i32> inreg %lhs, <2 x i32> inre
 ;
 ; GFX8-LABEL: s_uaddsat_v2i32:
 ; GFX8:       ; %bb.0:
-; GFX8-NEXT:    s_not_b32 s4, s0
-; GFX8-NEXT:    s_cmp_lt_u32 s4, s2
-; GFX8-NEXT:    s_cselect_b32 s2, s4, s2
-; GFX8-NEXT:    s_add_i32 s0, s0, s2
-; GFX8-NEXT:    s_not_b32 s2, s1
-; GFX8-NEXT:    s_cmp_lt_u32 s2, s3
-; GFX8-NEXT:    s_cselect_b32 s2, s2, s3
-; GFX8-NEXT:    s_add_i32 s1, s1, s2
+; GFX8-NEXT:    v_mov_b32_e32 v0, s2
+; GFX8-NEXT:    v_mov_b32_e32 v1, s3
+; GFX8-NEXT:    v_add_u32_e64 v0, s[4:5], s0, v0 clamp
+; GFX8-NEXT:    v_add_u32_e64 v1, s[0:1], s1, v1 clamp
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: s_uaddsat_v2i32:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_not_b32 s4, s0
-; GFX9-NEXT:    s_cmp_lt_u32 s4, s2
-; GFX9-NEXT:    s_cselect_b32 s2, s4, s2
-; GFX9-NEXT:    s_add_i32 s0, s0, s2
-; GFX9-NEXT:    s_not_b32 s2, s1
-; GFX9-NEXT:    s_cmp_lt_u32 s2, s3
-; GFX9-NEXT:    s_cselect_b32 s2, s2, s3
-; GFX9-NEXT:    s_add_i32 s1, s1, s2
+; GFX9-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    v_add_u32_e64 v0, s0, v0 clamp
+; GFX9-NEXT:    v_add_u32_e64 v1, s1, v1 clamp
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_uaddsat_v2i32:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_not_b32 s4, s0
+; GFX10-NEXT:    v_add_nc_u32_e64 v0, s0, s2 clamp
+; GFX10-NEXT:    v_add_nc_u32_e64 v1, s1, s3 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cmp_lt_u32 s4, s2
-; GFX10-NEXT:    s_cselect_b32 s2, s4, s2
-; GFX10-NEXT:    s_not_b32 s4, s1
-; GFX10-NEXT:    s_add_i32 s0, s0, s2
-; GFX10-NEXT:    s_cmp_lt_u32 s4, s3
-; GFX10-NEXT:    s_cselect_b32 s2, s4, s3
-; GFX10-NEXT:    s_add_i32 s1, s1, s2
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <2 x i32> @llvm.uadd.sat.v2i32(<2 x i32> %lhs, <2 x i32> %rhs)
   ret <2 x i32> %result
@@ -1222,45 +977,27 @@ define <3 x i32> @v_uaddsat_v3i32(<3 x i32> %lhs, <3 x i32> %rhs) {
 ; GFX8-LABEL: v_uaddsat_v3i32:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_xor_b32_e32 v6, -1, v0
-; GFX8-NEXT:    v_min_u32_e32 v3, v6, v3
-; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v0, v3
-; GFX8-NEXT:    v_xor_b32_e32 v3, -1, v1
-; GFX8-NEXT:    v_min_u32_e32 v3, v3, v4
-; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v1, v3
-; GFX8-NEXT:    v_xor_b32_e32 v3, -1, v2
-; GFX8-NEXT:    v_min_u32_e32 v3, v3, v5
-; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v3
+; GFX8-NEXT:    v_add_u32_e64 v0, s[4:5], v0, v3 clamp
+; GFX8-NEXT:    v_add_u32_e64 v1, s[4:5], v1, v4 clamp
+; GFX8-NEXT:    v_add_u32_e64 v2, s[4:5], v2, v5 clamp
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_uaddsat_v3i32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_xor_b32_e32 v6, -1, v0
-; GFX9-NEXT:    v_min_u32_e32 v3, v6, v3
-; GFX9-NEXT:    v_add_u32_e32 v0, v0, v3
-; GFX9-NEXT:    v_xor_b32_e32 v3, -1, v1
-; GFX9-NEXT:    v_min_u32_e32 v3, v3, v4
-; GFX9-NEXT:    v_add_u32_e32 v1, v1, v3
-; GFX9-NEXT:    v_xor_b32_e32 v3, -1, v2
-; GFX9-NEXT:    v_min_u32_e32 v3, v3, v5
-; GFX9-NEXT:    v_add_u32_e32 v2, v2, v3
+; GFX9-NEXT:    v_add_u32_e64 v0, v0, v3 clamp
+; GFX9-NEXT:    v_add_u32_e64 v1, v1, v4 clamp
+; GFX9-NEXT:    v_add_u32_e64 v2, v2, v5 clamp
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_uaddsat_v3i32:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_xor_b32_e32 v6, -1, v0
-; GFX10-NEXT:    v_xor_b32_e32 v7, -1, v1
-; GFX10-NEXT:    v_xor_b32_e32 v8, -1, v2
+; GFX10-NEXT:    v_add_nc_u32_e64 v0, v0, v3 clamp
+; GFX10-NEXT:    v_add_nc_u32_e64 v1, v1, v4 clamp
+; GFX10-NEXT:    v_add_nc_u32_e64 v2, v2, v5 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_min_u32_e32 v3, v6, v3
-; GFX10-NEXT:    v_min_u32_e32 v4, v7, v4
-; GFX10-NEXT:    v_min_u32_e32 v5, v8, v5
-; GFX10-NEXT:    v_add_nc_u32_e32 v0, v0, v3
-; GFX10-NEXT:    v_add_nc_u32_e32 v1, v1, v4
-; GFX10-NEXT:    v_add_nc_u32_e32 v2, v2, v5
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <3 x i32> @llvm.uadd.sat.v3i32(<3 x i32> %lhs, <3 x i32> %rhs)
   ret <3 x i32> %result
@@ -1285,51 +1022,39 @@ define amdgpu_ps <3 x i32> @s_uaddsat_v3i32(<3 x i32> inreg %lhs, <3 x i32> inre
 ;
 ; GFX8-LABEL: s_uaddsat_v3i32:
 ; GFX8:       ; %bb.0:
-; GFX8-NEXT:    s_not_b32 s6, s0
-; GFX8-NEXT:    s_cmp_lt_u32 s6, s3
-; GFX8-NEXT:    s_cselect_b32 s3, s6, s3
-; GFX8-NEXT:    s_add_i32 s0, s0, s3
-; GFX8-NEXT:    s_not_b32 s3, s1
-; GFX8-NEXT:    s_cmp_lt_u32 s3, s4
-; GFX8-NEXT:    s_cselect_b32 s3, s3, s4
-; GFX8-NEXT:    s_add_i32 s1, s1, s3
-; GFX8-NEXT:    s_not_b32 s3, s2
-; GFX8-NEXT:    s_cmp_lt_u32 s3, s5
-; GFX8-NEXT:    s_cselect_b32 s3, s3, s5
-; GFX8-NEXT:    s_add_i32 s2, s2, s3
+; GFX8-NEXT:    v_mov_b32_e32 v0, s3
+; GFX8-NEXT:    v_mov_b32_e32 v1, s4
+; GFX8-NEXT:    v_mov_b32_e32 v2, s5
+; GFX8-NEXT:    v_add_u32_e64 v0, s[6:7], s0, v0 clamp
+; GFX8-NEXT:    v_add_u32_e64 v1, s[0:1], s1, v1 clamp
+; GFX8-NEXT:    v_add_u32_e64 v2, s[0:1], s2, v2 clamp
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX8-NEXT:    v_readfirstlane_b32 s2, v2
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: s_uaddsat_v3i32:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_not_b32 s6, s0
-; GFX9-NEXT:    s_cmp_lt_u32 s6, s3
-; GFX9-NEXT:    s_cselect_b32 s3, s6, s3
-; GFX9-NEXT:    s_add_i32 s0, s0, s3
-; GFX9-NEXT:    s_not_b32 s3, s1
-; GFX9-NEXT:    s_cmp_lt_u32 s3, s4
-; GFX9-NEXT:    s_cselect_b32 s3, s3, s4
-; GFX9-NEXT:    s_add_i32 s1, s1, s3
-; GFX9-NEXT:    s_not_b32 s3, s2
-; GFX9-NEXT:    s_cmp_lt_u32 s3, s5
-; GFX9-NEXT:    s_cselect_b32 s3, s3, s5
-; GFX9-NEXT:    s_add_i32 s2, s2, s3
+; GFX9-NEXT:    v_mov_b32_e32 v0, s3
+; GFX9-NEXT:    v_mov_b32_e32 v1, s4
+; GFX9-NEXT:    v_mov_b32_e32 v2, s5
+; GFX9-NEXT:    v_add_u32_e64 v0, s0, v0 clamp
+; GFX9-NEXT:    v_add_u32_e64 v1, s1, v1 clamp
+; GFX9-NEXT:    v_add_u32_e64 v2, s2, v2 clamp
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v2
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_uaddsat_v3i32:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_not_b32 s6, s0
+; GFX10-NEXT:    v_add_nc_u32_e64 v0, s0, s3 clamp
+; GFX10-NEXT:    v_add_nc_u32_e64 v1, s1, s4 clamp
+; GFX10-NEXT:    v_add_nc_u32_e64 v2, s2, s5 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cmp_lt_u32 s6, s3
-; GFX10-NEXT:    s_cselect_b32 s3, s6, s3
-; GFX10-NEXT:    s_not_b32 s6, s1
-; GFX10-NEXT:    s_add_i32 s0, s0, s3
-; GFX10-NEXT:    s_cmp_lt_u32 s6, s4
-; GFX10-NEXT:    s_cselect_b32 s3, s6, s4
-; GFX10-NEXT:    s_not_b32 s4, s2
-; GFX10-NEXT:    s_add_i32 s1, s1, s3
-; GFX10-NEXT:    s_cmp_lt_u32 s4, s5
-; GFX10-NEXT:    s_cselect_b32 s3, s4, s5
-; GFX10-NEXT:    s_add_i32 s2, s2, s3
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <3 x i32> @llvm.uadd.sat.v3i32(<3 x i32> %lhs, <3 x i32> %rhs)
   ret <3 x i32> %result
@@ -1356,54 +1081,30 @@ define <4 x i32> @v_uaddsat_v4i32(<4 x i32> %lhs, <4 x i32> %rhs) {
 ; GFX8-LABEL: v_uaddsat_v4i32:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_xor_b32_e32 v8, -1, v0
-; GFX8-NEXT:    v_min_u32_e32 v4, v8, v4
-; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v0, v4
-; GFX8-NEXT:    v_xor_b32_e32 v4, -1, v1
-; GFX8-NEXT:    v_min_u32_e32 v4, v4, v5
-; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v1, v4
-; GFX8-NEXT:    v_xor_b32_e32 v4, -1, v2
-; GFX8-NEXT:    v_min_u32_e32 v4, v4, v6
-; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v4
-; GFX8-NEXT:    v_xor_b32_e32 v4, -1, v3
-; GFX8-NEXT:    v_min_u32_e32 v4, v4, v7
-; GFX8-NEXT:    v_add_u32_e32 v3, vcc, v3, v4
+; GFX8-NEXT:    v_add_u32_e64 v0, s[4:5], v0, v4 clamp
+; GFX8-NEXT:    v_add_u32_e64 v1, s[4:5], v1, v5 clamp
+; GFX8-NEXT:    v_add_u32_e64 v2, s[4:5], v2, v6 clamp
+; GFX8-NEXT:    v_add_u32_e64 v3, s[4:5], v3, v7 clamp
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_uaddsat_v4i32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_xor_b32_e32 v8, -1, v0
-; GFX9-NEXT:    v_min_u32_e32 v4, v8, v4
-; GFX9-NEXT:    v_add_u32_e32 v0, v0, v4
-; GFX9-NEXT:    v_xor_b32_e32 v4, -1, v1
-; GFX9-NEXT:    v_min_u32_e32 v4, v4, v5
-; GFX9-NEXT:    v_add_u32_e32 v1, v1, v4
-; GFX9-NEXT:    v_xor_b32_e32 v4, -1, v2
-; GFX9-NEXT:    v_min_u32_e32 v4, v4, v6
-; GFX9-NEXT:    v_add_u32_e32 v2, v2, v4
-; GFX9-NEXT:    v_xor_b32_e32 v4, -1, v3
-; GFX9-NEXT:    v_min_u32_e32 v4, v4, v7
-; GFX9-NEXT:    v_add_u32_e32 v3, v3, v4
+; GFX9-NEXT:    v_add_u32_e64 v0, v0, v4 clamp
+; GFX9-NEXT:    v_add_u32_e64 v1, v1, v5 clamp
+; GFX9-NEXT:    v_add_u32_e64 v2, v2, v6 clamp
+; GFX9-NEXT:    v_add_u32_e64 v3, v3, v7 clamp
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_uaddsat_v4i32:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_xor_b32_e32 v15, -1, v0
-; GFX10-NEXT:    v_xor_b32_e32 v19, -1, v1
-; GFX10-NEXT:    v_xor_b32_e32 v23, -1, v2
-; GFX10-NEXT:    v_xor_b32_e32 v10, -1, v3
+; GFX10-NEXT:    v_add_nc_u32_e64 v0, v0, v4 clamp
+; GFX10-NEXT:    v_add_nc_u32_e64 v1, v1, v5 clamp
+; GFX10-NEXT:    v_add_nc_u32_e64 v2, v2, v6 clamp
+; GFX10-NEXT:    v_add_nc_u32_e64 v3, v3, v7 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_min_u32_e32 v11, v15, v4
-; GFX10-NEXT:    v_min_u32_e32 v15, v19, v5
-; GFX10-NEXT:    v_min_u32_e32 v19, v23, v6
-; GFX10-NEXT:    v_min_u32_e32 v6, v10, v7
-; GFX10-NEXT:    v_add_nc_u32_e32 v0, v0, v11
-; GFX10-NEXT:    v_add_nc_u32_e32 v1, v1, v15
-; GFX10-NEXT:    v_add_nc_u32_e32 v2, v2, v19
-; GFX10-NEXT:    v_add_nc_u32_e32 v3, v3, v6
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> %lhs, <4 x i32> %rhs)
   ret <4 x i32> %result
@@ -1432,63 +1133,47 @@ define amdgpu_ps <4 x i32> @s_uaddsat_v4i32(<4 x i32> inreg %lhs, <4 x i32> inre
 ;
 ; GFX8-LABEL: s_uaddsat_v4i32:
 ; GFX8:       ; %bb.0:
-; GFX8-NEXT:    s_not_b32 s8, s0
-; GFX8-NEXT:    s_cmp_lt_u32 s8, s4
-; GFX8-NEXT:    s_cselect_b32 s4, s8, s4
-; GFX8-NEXT:    s_add_i32 s0, s0, s4
-; GFX8-NEXT:    s_not_b32 s4, s1
-; GFX8-NEXT:    s_cmp_lt_u32 s4, s5
-; GFX8-NEXT:    s_cselect_b32 s4, s4, s5
-; GFX8-NEXT:    s_add_i32 s1, s1, s4
-; GFX8-NEXT:    s_not_b32 s4, s2
-; GFX8-NEXT:    s_cmp_lt_u32 s4, s6
-; GFX8-NEXT:    s_cselect_b32 s4, s4, s6
-; GFX8-NEXT:    s_add_i32 s2, s2, s4
-; GFX8-NEXT:    s_not_b32 s4, s3
-; GFX8-NEXT:    s_cmp_lt_u32 s4, s7
-; GFX8-NEXT:    s_cselect_b32 s4, s4, s7
-; GFX8-NEXT:    s_add_i32 s3, s3, s4
+; GFX8-NEXT:    v_mov_b32_e32 v0, s4
+; GFX8-NEXT:    v_mov_b32_e32 v1, s5
+; GFX8-NEXT:    v_mov_b32_e32 v2, s6
+; GFX8-NEXT:    v_mov_b32_e32 v3, s7
+; GFX8-NEXT:    v_add_u32_e64 v0, s[8:9], s0, v0 clamp
+; GFX8-NEXT:    v_add_u32_e64 v1, s[0:1], s1, v1 clamp
+; GFX8-NEXT:    v_add_u32_e64 v2, s[0:1], s2, v2 clamp
+; GFX8-NEXT:    v_add_u32_e64 v3, s[0:1], s3, v3 clamp
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX8-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX8-NEXT:    v_readfirstlane_b32 s3, v3
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: s_uaddsat_v4i32:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_not_b32 s8, s0
-; GFX9-NEXT:    s_cmp_lt_u32 s8, s4
-; GFX9-NEXT:    s_cselect_b32 s4, s8, s4
-; GFX9-NEXT:    s_add_i32 s0, s0, s4
-; GFX9-NEXT:    s_not_b32 s4, s1
-; GFX9-NEXT:    s_cmp_lt_u32 s4, s5
-; GFX9-NEXT:    s_cselect_b32 s4, s4, s5
-; GFX9-NEXT:    s_add_i32 s1, s1, s4
-; GFX9-NEXT:    s_not_b32 s4, s2
-; GFX9-NEXT:    s_cmp_lt_u32 s4, s6
-; GFX9-NEXT:    s_cselect_b32 s4, s4, s6
-; GFX9-NEXT:    s_add_i32 s2, s2, s4
-; GFX9-NEXT:    s_not_b32 s4, s3
-; GFX9-NEXT:    s_cmp_lt_u32 s4, s7
-; GFX9-NEXT:    s_cselect_b32 s4, s4, s7
-; GFX9-NEXT:    s_add_i32 s3, s3, s4
+; GFX9-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-NEXT:    v_mov_b32_e32 v2, s6
+; GFX9-NEXT:    v_mov_b32_e32 v3, s7
+; GFX9-NEXT:    v_add_u32_e64 v0, s0, v0 clamp
+; GFX9-NEXT:    v_add_u32_e64 v1, s1, v1 clamp
+; GFX9-NEXT:    v_add_u32_e64 v2, s2, v2 clamp
+; GFX9-NEXT:    v_add_u32_e64 v3, s3, v3 clamp
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX9-NEXT:    v_readfirstlane_b32 s3, v3
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_uaddsat_v4i32:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_not_b32 s8, s0
+; GFX10-NEXT:    v_add_nc_u32_e64 v0, s0, s4 clamp
+; GFX10-NEXT:    v_add_nc_u32_e64 v1, s1, s5 clamp
+; GFX10-NEXT:    v_add_nc_u32_e64 v2, s2, s6 clamp
+; GFX10-NEXT:    v_add_nc_u32_e64 v3, s3, s7 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cmp_lt_u32 s8, s4
-; GFX10-NEXT:    s_cselect_b32 s4, s8, s4
-; GFX10-NEXT:    s_not_b32 s8, s1
-; GFX10-NEXT:    s_add_i32 s0, s0, s4
-; GFX10-NEXT:    s_cmp_lt_u32 s8, s5
-; GFX10-NEXT:    s_cselect_b32 s4, s8, s5
-; GFX10-NEXT:    s_not_b32 s5, s2
-; GFX10-NEXT:    s_add_i32 s1, s1, s4
-; GFX10-NEXT:    s_cmp_lt_u32 s5, s6
-; GFX10-NEXT:    s_cselect_b32 s4, s5, s6
-; GFX10-NEXT:    s_not_b32 s5, s3
-; GFX10-NEXT:    s_add_i32 s2, s2, s4
-; GFX10-NEXT:    s_cmp_lt_u32 s5, s7
-; GFX10-NEXT:    s_cselect_b32 s4, s5, s7
-; GFX10-NEXT:    s_add_i32 s3, s3, s4
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX10-NEXT:    v_readfirstlane_b32 s3, v3
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> %lhs, <4 x i32> %rhs)
   ret <4 x i32> %result
@@ -1518,62 +1203,32 @@ define <5 x i32> @v_uaddsat_v5i32(<5 x i32> %lhs, <5 x i32> %rhs) {
 ; GFX8-LABEL: v_uaddsat_v5i32:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_xor_b32_e32 v10, -1, v0
-; GFX8-NEXT:    v_min_u32_e32 v5, v10, v5
-; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v0, v5
-; GFX8-NEXT:    v_xor_b32_e32 v5, -1, v1
-; GFX8-NEXT:    v_min_u32_e32 v5, v5, v6
-; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v1, v5
-; GFX8-NEXT:    v_xor_b32_e32 v5, -1, v2
-; GFX8-NEXT:    v_min_u32_e32 v5, v5, v7
-; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v5
-; GFX8-NEXT:    v_xor_b32_e32 v5, -1, v3
-; GFX8-NEXT:    v_min_u32_e32 v5, v5, v8
-; GFX8-NEXT:    v_add_u32_e32 v3, vcc, v3, v5
-; GFX8-NEXT:    v_xor_b32_e32 v5, -1, v4
-; GFX8-NEXT:    v_min_u32_e32 v5, v5, v9
-; GFX8-NEXT:    v_add_u32_e32 v4, vcc, v4, v5
+; GFX8-NEXT:    v_add_u32_e64 v0, s[4:5], v0, v5 clamp
+; GFX8-NEXT:    v_add_u32_e64 v1, s[4:5], v1, v6 clamp
+; GFX8-NEXT:    v_add_u32_e64 v2, s[4:5], v2, v7 clamp
+; GFX8-NEXT:    v_add_u32_e64 v3, s[4:5], v3, v8 clamp
+; GFX8-NEXT:    v_add_u32_e64 v4, s[4:5], v4, v9 clamp
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_uaddsat_v5i32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_xor_b32_e32 v10, -1, v0
-; GFX9-NEXT:    v_min_u32_e32 v5, v10, v5
-; GFX9-NEXT:    v_add_u32_e32 v0, v0, v5
-; GFX9-NEXT:    v_xor_b32_e32 v5, -1, v1
-; GFX9-NEXT:    v_min_u32_e32 v5, v5, v6
-; GFX9-NEXT:    v_add_u32_e32 v1, v1, v5
-; GFX9-NEXT:    v_xor_b32_e32 v5, -1, v2
-; GFX9-NEXT:    v_min_u32_e32 v5, v5, v7
-; GFX9-NEXT:    v_add_u32_e32 v2, v2, v5
-; GFX9-NEXT:    v_xor_b32_e32 v5, -1, v3
-; GFX9-NEXT:    v_min_u32_e32 v5, v5, v8
-; GFX9-NEXT:    v_add_u32_e32 v3, v3, v5
-; GFX9-NEXT:    v_xor_b32_e32 v5, -1, v4
-; GFX9-NEXT:    v_min_u32_e32 v5, v5, v9
-; GFX9-NEXT:    v_add_u32_e32 v4, v4, v5
+; GFX9-NEXT:    v_add_u32_e64 v0, v0, v5 clamp
+; GFX9-NEXT:    v_add_u32_e64 v1, v1, v6 clamp
+; GFX9-NEXT:    v_add_u32_e64 v2, v2, v7 clamp
+; GFX9-NEXT:    v_add_u32_e64 v3, v3, v8 clamp
+; GFX9-NEXT:    v_add_u32_e64 v4, v4, v9 clamp
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_uaddsat_v5i32:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_xor_b32_e32 v10, -1, v0
-; GFX10-NEXT:    v_xor_b32_e32 v11, -1, v1
-; GFX10-NEXT:    v_xor_b32_e32 v12, -1, v2
-; GFX10-NEXT:    v_xor_b32_e32 v13, -1, v3
-; GFX10-NEXT:    v_xor_b32_e32 v14, -1, v4
-; GFX10-NEXT:    v_min_u32_e32 v5, v10, v5
-; GFX10-NEXT:    v_min_u32_e32 v6, v11, v6
-; GFX10-NEXT:    v_min_u32_e32 v7, v12, v7
-; GFX10-NEXT:    v_min_u32_e32 v8, v13, v8
-; GFX10-NEXT:    v_min_u32_e32 v9, v14, v9
-; GFX10-NEXT:    v_add_nc_u32_e32 v0, v0, v5
-; GFX10-NEXT:    v_add_nc_u32_e32 v1, v1, v6
-; GFX10-NEXT:    v_add_nc_u32_e32 v2, v2, v7
-; GFX10-NEXT:    v_add_nc_u32_e32 v3, v3, v8
-; GFX10-NEXT:    v_add_nc_u32_e32 v4, v4, v9
+; GFX10-NEXT:    v_add_nc_u32_e64 v0, v0, v5 clamp
+; GFX10-NEXT:    v_add_nc_u32_e64 v1, v1, v6 clamp
+; GFX10-NEXT:    v_add_nc_u32_e64 v2, v2, v7 clamp
+; GFX10-NEXT:    v_add_nc_u32_e64 v3, v3, v8 clamp
+; GFX10-NEXT:    v_add_nc_u32_e64 v4, v4, v9 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <5 x i32> @llvm.uadd.sat.v5i32(<5 x i32> %lhs, <5 x i32> %rhs)
@@ -1607,75 +1262,55 @@ define amdgpu_ps <5 x i32> @s_uaddsat_v5i32(<5 x i32> inreg %lhs, <5 x i32> inre
 ;
 ; GFX8-LABEL: s_uaddsat_v5i32:
 ; GFX8:       ; %bb.0:
-; GFX8-NEXT:    s_not_b32 s10, s0
-; GFX8-NEXT:    s_cmp_lt_u32 s10, s5
-; GFX8-NEXT:    s_cselect_b32 s5, s10, s5
-; GFX8-NEXT:    s_add_i32 s0, s0, s5
-; GFX8-NEXT:    s_not_b32 s5, s1
-; GFX8-NEXT:    s_cmp_lt_u32 s5, s6
-; GFX8-NEXT:    s_cselect_b32 s5, s5, s6
-; GFX8-NEXT:    s_add_i32 s1, s1, s5
-; GFX8-NEXT:    s_not_b32 s5, s2
-; GFX8-NEXT:    s_cmp_lt_u32 s5, s7
-; GFX8-NEXT:    s_cselect_b32 s5, s5, s7
-; GFX8-NEXT:    s_add_i32 s2, s2, s5
-; GFX8-NEXT:    s_not_b32 s5, s3
-; GFX8-NEXT:    s_cmp_lt_u32 s5, s8
-; GFX8-NEXT:    s_cselect_b32 s5, s5, s8
-; GFX8-NEXT:    s_add_i32 s3, s3, s5
-; GFX8-NEXT:    s_not_b32 s5, s4
-; GFX8-NEXT:    s_cmp_lt_u32 s5, s9
-; GFX8-NEXT:    s_cselect_b32 s5, s5, s9
-; GFX8-NEXT:    s_add_i32 s4, s4, s5
+; GFX8-NEXT:    v_mov_b32_e32 v0, s5
+; GFX8-NEXT:    v_mov_b32_e32 v1, s6
+; GFX8-NEXT:    v_mov_b32_e32 v2, s7
+; GFX8-NEXT:    v_mov_b32_e32 v3, s8
+; GFX8-NEXT:    v_mov_b32_e32 v4, s9
+; GFX8-NEXT:    v_add_u32_e64 v0, s[10:11], s0, v0 clamp
+; GFX8-NEXT:    v_add_u32_e64 v1, s[0:1], s1, v1 clamp
+; GFX8-NEXT:    v_add_u32_e64 v2, s[0:1], s2, v2 clamp
+; GFX8-NEXT:    v_add_u32_e64 v3, s[0:1], s3, v3 clamp
+; GFX8-NEXT:    v_add_u32_e64 v4, s[0:1], s4, v4 clamp
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX8-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX8-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX8-NEXT:    v_readfirstlane_b32 s4, v4
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: s_uaddsat_v5i32:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_not_b32 s10, s0
-; GFX9-NEXT:    s_cmp_lt_u32 s10, s5
-; GFX9-NEXT:    s_cselect_b32 s5, s10, s5
-; GFX9-NEXT:    s_add_i32 s0, s0, s5
-; GFX9-NEXT:    s_not_b32 s5, s1
-; GFX9-NEXT:    s_cmp_lt_u32 s5, s6
-; GFX9-NEXT:    s_cselect_b32 s5, s5, s6
-; GFX9-NEXT:    s_add_i32 s1, s1, s5
-; GFX9-NEXT:    s_not_b32 s5, s2
-; GFX9-NEXT:    s_cmp_lt_u32 s5, s7
-; GFX9-NEXT:    s_cselect_b32 s5, s5, s7
-; GFX9-NEXT:    s_add_i32 s2, s2, s5
-; GFX9-NEXT:    s_not_b32 s5, s3
-; GFX9-NEXT:    s_cmp_lt_u32 s5, s8
-; GFX9-NEXT:    s_cselect_b32 s5, s5, s8
-; GFX9-NEXT:    s_add_i32 s3, s3, s5
-; GFX9-NEXT:    s_not_b32 s5, s4
-; GFX9-NEXT:    s_cmp_lt_u32 s5, s9
-; GFX9-NEXT:    s_cselect_b32 s5, s5, s9
-; GFX9-NEXT:    s_add_i32 s4, s4, s5
+; GFX9-NEXT:    v_mov_b32_e32 v0, s5
+; GFX9-NEXT:    v_mov_b32_e32 v1, s6
+; GFX9-NEXT:    v_mov_b32_e32 v2, s7
+; GFX9-NEXT:    v_mov_b32_e32 v3, s8
+; GFX9-NEXT:    v_mov_b32_e32 v4, s9
+; GFX9-NEXT:    v_add_u32_e64 v0, s0, v0 clamp
+; GFX9-NEXT:    v_add_u32_e64 v1, s1, v1 clamp
+; GFX9-NEXT:    v_add_u32_e64 v2, s2, v2 clamp
+; GFX9-NEXT:    v_add_u32_e64 v3, s3, v3 clamp
+; GFX9-NEXT:    v_add_u32_e64 v4, s4, v4 clamp
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX9-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX9-NEXT:    v_readfirstlane_b32 s4, v4
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_uaddsat_v5i32:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_not_b32 s10, s0
+; GFX10-NEXT:    v_add_nc_u32_e64 v0, s0, s5 clamp
+; GFX10-NEXT:    v_add_nc_u32_e64 v1, s1, s6 clamp
+; GFX10-NEXT:    v_add_nc_u32_e64 v2, s2, s7 clamp
+; GFX10-NEXT:    v_add_nc_u32_e64 v3, s3, s8 clamp
+; GFX10-NEXT:    v_add_nc_u32_e64 v4, s4, s9 clamp
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX10-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX10-NEXT:    v_readfirstlane_b32 s4, v4
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cmp_lt_u32 s10, s5
-; GFX10-NEXT:    s_cselect_b32 s5, s10, s5
-; GFX10-NEXT:    s_not_b32 s10, s1
-; GFX10-NEXT:    s_add_i32 s0, s0, s5
-; GFX10-NEXT:    s_cmp_lt_u32 s10, s6
-; GFX10-NEXT:    s_cselect_b32 s5, s10, s6
-; GFX10-NEXT:    s_not_b32 s6, s2
-; GFX10-NEXT:    s_add_i32 s1, s1, s5
-; GFX10-NEXT:    s_cmp_lt_u32 s6, s7
-; GFX10-NEXT:    s_cselect_b32 s5, s6, s7
-; GFX10-NEXT:    s_not_b32 s6, s3
-; GFX10-NEXT:    s_add_i32 s2, s2, s5
-; GFX10-NEXT:    s_cmp_lt_u32 s6, s8
-; GFX10-NEXT:    s_cselect_b32 s5, s6, s8
-; GFX10-NEXT:    s_not_b32 s6, s4
-; GFX10-NEXT:    s_add_i32 s3, s3, s5
-; GFX10-NEXT:    s_cmp_lt_u32 s6, s9
-; GFX10-NEXT:    s_cselect_b32 s5, s6, s9
-; GFX10-NEXT:    s_add_i32 s4, s4, s5
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <5 x i32> @llvm.uadd.sat.v5i32(<5 x i32> %lhs, <5 x i32> %rhs)
   ret <5 x i32> %result
@@ -1738,162 +1373,66 @@ define <16 x i32> @v_uaddsat_v16i32(<16 x i32> %lhs, <16 x i32> %rhs) {
 ; GFX8-LABEL: v_uaddsat_v16i32:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_xor_b32_e32 v32, -1, v0
-; GFX8-NEXT:    v_min_u32_e32 v16, v32, v16
-; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v0, v16
-; GFX8-NEXT:    v_xor_b32_e32 v16, -1, v1
-; GFX8-NEXT:    v_min_u32_e32 v16, v16, v17
-; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v1, v16
-; GFX8-NEXT:    v_xor_b32_e32 v16, -1, v2
-; GFX8-NEXT:    v_min_u32_e32 v16, v16, v18
-; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v16
-; GFX8-NEXT:    v_xor_b32_e32 v16, -1, v3
-; GFX8-NEXT:    v_min_u32_e32 v16, v16, v19
-; GFX8-NEXT:    v_add_u32_e32 v3, vcc, v3, v16
-; GFX8-NEXT:    v_xor_b32_e32 v16, -1, v4
-; GFX8-NEXT:    v_min_u32_e32 v16, v16, v20
-; GFX8-NEXT:    v_add_u32_e32 v4, vcc, v4, v16
-; GFX8-NEXT:    v_xor_b32_e32 v16, -1, v5
-; GFX8-NEXT:    v_min_u32_e32 v16, v16, v21
-; GFX8-NEXT:    v_add_u32_e32 v5, vcc, v5, v16
-; GFX8-NEXT:    v_xor_b32_e32 v16, -1, v6
-; GFX8-NEXT:    v_min_u32_e32 v16, v16, v22
-; GFX8-NEXT:    v_add_u32_e32 v6, vcc, v6, v16
-; GFX8-NEXT:    v_xor_b32_e32 v16, -1, v7
-; GFX8-NEXT:    v_min_u32_e32 v16, v16, v23
-; GFX8-NEXT:    v_add_u32_e32 v7, vcc, v7, v16
-; GFX8-NEXT:    v_xor_b32_e32 v16, -1, v8
-; GFX8-NEXT:    v_min_u32_e32 v16, v16, v24
-; GFX8-NEXT:    v_add_u32_e32 v8, vcc, v8, v16
-; GFX8-NEXT:    v_xor_b32_e32 v16, -1, v9
-; GFX8-NEXT:    v_min_u32_e32 v16, v16, v25
-; GFX8-NEXT:    v_add_u32_e32 v9, vcc, v9, v16
-; GFX8-NEXT:    v_xor_b32_e32 v16, -1, v10
-; GFX8-NEXT:    v_min_u32_e32 v16, v16, v26
-; GFX8-NEXT:    v_add_u32_e32 v10, vcc, v10, v16
-; GFX8-NEXT:    v_xor_b32_e32 v16, -1, v11
-; GFX8-NEXT:    v_min_u32_e32 v16, v16, v27
-; GFX8-NEXT:    v_add_u32_e32 v11, vcc, v11, v16
-; GFX8-NEXT:    v_xor_b32_e32 v16, -1, v12
-; GFX8-NEXT:    v_min_u32_e32 v16, v16, v28
-; GFX8-NEXT:    v_add_u32_e32 v12, vcc, v12, v16
-; GFX8-NEXT:    v_xor_b32_e32 v16, -1, v13
-; GFX8-NEXT:    v_min_u32_e32 v16, v16, v29
-; GFX8-NEXT:    v_add_u32_e32 v13, vcc, v13, v16
-; GFX8-NEXT:    v_xor_b32_e32 v16, -1, v14
-; GFX8-NEXT:    v_min_u32_e32 v16, v16, v30
-; GFX8-NEXT:    v_add_u32_e32 v14, vcc, v14, v16
-; GFX8-NEXT:    v_xor_b32_e32 v16, -1, v15
-; GFX8-NEXT:    v_min_u32_e32 v16, v16, v31
-; GFX8-NEXT:    v_add_u32_e32 v15, vcc, v15, v16
+; GFX8-NEXT:    v_add_u32_e64 v0, s[4:5], v0, v16 clamp
+; GFX8-NEXT:    v_add_u32_e64 v1, s[4:5], v1, v17 clamp
+; GFX8-NEXT:    v_add_u32_e64 v2, s[4:5], v2, v18 clamp
+; GFX8-NEXT:    v_add_u32_e64 v3, s[4:5], v3, v19 clamp
+; GFX8-NEXT:    v_add_u32_e64 v4, s[4:5], v4, v20 clamp
+; GFX8-NEXT:    v_add_u32_e64 v5, s[4:5], v5, v21 clamp
+; GFX8-NEXT:    v_add_u32_e64 v6, s[4:5], v6, v22 clamp
+; GFX8-NEXT:    v_add_u32_e64 v7, s[4:5], v7, v23 clamp
+; GFX8-NEXT:    v_add_u32_e64 v8, s[4:5], v8, v24 clamp
+; GFX8-NEXT:    v_add_u32_e64 v9, s[4:5], v9, v25 clamp
+; GFX8-NEXT:    v_add_u32_e64 v10, s[4:5], v10, v26 clamp
+; GFX8-NEXT:    v_add_u32_e64 v11, s[4:5], v11, v27 clamp
+; GFX8-NEXT:    v_add_u32_e64 v12, s[4:5], v12, v28 clamp
+; GFX8-NEXT:    v_add_u32_e64 v13, s[4:5], v13, v29 clamp
+; GFX8-NEXT:    v_add_u32_e64 v14, s[4:5], v14, v30 clamp
+; GFX8-NEXT:    v_add_u32_e64 v15, s[4:5], v15, v31 clamp
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_uaddsat_v16i32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_xor_b32_e32 v32, -1, v0
-; GFX9-NEXT:    v_min_u32_e32 v16, v32, v16
-; GFX9-NEXT:    v_add_u32_e32 v0, v0, v16
-; GFX9-NEXT:    v_xor_b32_e32 v16, -1, v1
-; GFX9-NEXT:    v_min_u32_e32 v16, v16, v17
-; GFX9-NEXT:    v_add_u32_e32 v1, v1, v16
-; GFX9-NEXT:    v_xor_b32_e32 v16, -1, v2
-; GFX9-NEXT:    v_min_u32_e32 v16, v16, v18
-; GFX9-NEXT:    v_add_u32_e32 v2, v2, v16
-; GFX9-NEXT:    v_xor_b32_e32 v16, -1, v3
-; GFX9-NEXT:    v_min_u32_e32 v16, v16, v19
-; GFX9-NEXT:    v_add_u32_e32 v3, v3, v16
-; GFX9-NEXT:    v_xor_b32_e32 v16, -1, v4
-; GFX9-NEXT:    v_min_u32_e32 v16, v16, v20
-; GFX9-NEXT:    v_add_u32_e32 v4, v4, v16
-; GFX9-NEXT:    v_xor_b32_e32 v16, -1, v5
-; GFX9-NEXT:    v_min_u32_e32 v16, v16, v21
-; GFX9-NEXT:    v_add_u32_e32 v5, v5, v16
-; GFX9-NEXT:    v_xor_b32_e32 v16, -1, v6
-; GFX9-NEXT:    v_min_u32_e32 v16, v16, v22
-; GFX9-NEXT:    v_add_u32_e32 v6, v6, v16
-; GFX9-NEXT:    v_xor_b32_e32 v16, -1, v7
-; GFX9-NEXT:    v_min_u32_e32 v16, v16, v23
-; GFX9-NEXT:    v_add_u32_e32 v7, v7, v16
-; GFX9-NEXT:    v_xor_b32_e32 v16, -1, v8
-; GFX9-NEXT:    v_min_u32_e32 v16, v16, v24
-; GFX9-NEXT:    v_add_u32_e32 v8, v8, v16
-; GFX9-NEXT:    v_xor_b32_e32 v16, -1, v9
-; GFX9-NEXT:    v_min_u32_e32 v16, v16, v25
-; GFX9-NEXT:    v_add_u32_e32 v9, v9, v16
-; GFX9-NEXT:    v_xor_b32_e32 v16, -1, v10
-; GFX9-NEXT:    v_min_u32_e32 v16, v16, v26
-; GFX9-NEXT:    v_add_u32_e32 v10, v10, v16
-; GFX9-NEXT:    v_xor_b32_e32 v16, -1, v11
-; GFX9-NEXT:    v_min_u32_e32 v16, v16, v27
-; GFX9-NEXT:    v_add_u32_e32 v11, v11, v16
-; GFX9-NEXT:    v_xor_b32_e32 v16, -1, v12
-; GFX9-NEXT:    v_min_u32_e32 v16, v16, v28
-; GFX9-NEXT:    v_add_u32_e32 v12, v12, v16
-; GFX9-NEXT:    v_xor_b32_e32 v16, -1, v13
-; GFX9-NEXT:    v_min_u32_e32 v16, v16, v29
-; GFX9-NEXT:    v_add_u32_e32 v13, v13, v16
-; GFX9-NEXT:    v_xor_b32_e32 v16, -1, v14
-; GFX9-NEXT:    v_min_u32_e32 v16, v16, v30
-; GFX9-NEXT:    v_add_u32_e32 v14, v14, v16
-; GFX9-NEXT:    v_xor_b32_e32 v16, -1, v15
-; GFX9-NEXT:    v_min_u32_e32 v16, v16, v31
-; GFX9-NEXT:    v_add_u32_e32 v15, v15, v16
+; GFX9-NEXT:    v_add_u32_e64 v0, v0, v16 clamp
+; GFX9-NEXT:    v_add_u32_e64 v1, v1, v17 clamp
+; GFX9-NEXT:    v_add_u32_e64 v2, v2, v18 clamp
+; GFX9-NEXT:    v_add_u32_e64 v3, v3, v19 clamp
+; GFX9-NEXT:    v_add_u32_e64 v4, v4, v20 clamp
+; GFX9-NEXT:    v_add_u32_e64 v5, v5, v21 clamp
+; GFX9-NEXT:    v_add_u32_e64 v6, v6, v22 clamp
+; GFX9-NEXT:    v_add_u32_e64 v7, v7, v23 clamp
+; GFX9-NEXT:    v_add_u32_e64 v8, v8, v24 clamp
+; GFX9-NEXT:    v_add_u32_e64 v9, v9, v25 clamp
+; GFX9-NEXT:    v_add_u32_e64 v10, v10, v26 clamp
+; GFX9-NEXT:    v_add_u32_e64 v11, v11, v27 clamp
+; GFX9-NEXT:    v_add_u32_e64 v12, v12, v28 clamp
+; GFX9-NEXT:    v_add_u32_e64 v13, v13, v29 clamp
+; GFX9-NEXT:    v_add_u32_e64 v14, v14, v30 clamp
+; GFX9-NEXT:    v_add_u32_e64 v15, v15, v31 clamp
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_uaddsat_v16i32:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_xor_b32_e32 v35, -1, v0
-; GFX10-NEXT:    v_xor_b32_e32 v32, -1, v2
-; GFX10-NEXT:    v_xor_b32_e32 v33, -1, v3
-; GFX10-NEXT:    v_xor_b32_e32 v34, -1, v4
+; GFX10-NEXT:    v_add_nc_u32_e64 v0, v0, v16 clamp
+; GFX10-NEXT:    v_add_nc_u32_e64 v1, v1, v17 clamp
+; GFX10-NEXT:    v_add_nc_u32_e64 v2, v2, v18 clamp
+; GFX10-NEXT:    v_add_nc_u32_e64 v3, v3, v19 clamp
+; GFX10-NEXT:    v_add_nc_u32_e64 v4, v4, v20 clamp
+; GFX10-NEXT:    v_add_nc_u32_e64 v5, v5, v21 clamp
+; GFX10-NEXT:    v_add_nc_u32_e64 v6, v6, v22 clamp
+; GFX10-NEXT:    v_add_nc_u32_e64 v7, v7, v23 clamp
+; GFX10-NEXT:    v_add_nc_u32_e64 v8, v8, v24 clamp
+; GFX10-NEXT:    v_add_nc_u32_e64 v9, v9, v25 clamp
+; GFX10-NEXT:    v_add_nc_u32_e64 v10, v10, v26 clamp
+; GFX10-NEXT:    v_add_nc_u32_e64 v11, v11, v27 clamp
+; GFX10-NEXT:    v_add_nc_u32_e64 v12, v12, v28 clamp
+; GFX10-NEXT:    v_add_nc_u32_e64 v13, v13, v29 clamp
+; GFX10-NEXT:    v_add_nc_u32_e64 v14, v14, v30 clamp
+; GFX10-NEXT:    v_add_nc_u32_e64 v15, v15, v31 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_min_u32_e32 v35, v35, v16
-; GFX10-NEXT:    v_xor_b32_e32 v16, -1, v1
-; GFX10-NEXT:    v_add_nc_u32_e32 v0, v0, v35
-; GFX10-NEXT:    v_xor_b32_e32 v35, -1, v5
-; GFX10-NEXT:    v_min_u32_e32 v16, v16, v17
-; GFX10-NEXT:    v_min_u32_e32 v17, v32, v18
-; GFX10-NEXT:    v_min_u32_e32 v18, v33, v19
-; GFX10-NEXT:    v_min_u32_e32 v19, v34, v20
-; GFX10-NEXT:    v_min_u32_e32 v20, v35, v21
-; GFX10-NEXT:    v_add_nc_u32_e32 v1, v1, v16
-; GFX10-NEXT:    v_xor_b32_e32 v16, -1, v6
-; GFX10-NEXT:    v_add_nc_u32_e32 v2, v2, v17
-; GFX10-NEXT:    v_xor_b32_e32 v17, -1, v7
-; GFX10-NEXT:    v_add_nc_u32_e32 v3, v3, v18
-; GFX10-NEXT:    v_xor_b32_e32 v18, -1, v8
-; GFX10-NEXT:    v_add_nc_u32_e32 v4, v4, v19
-; GFX10-NEXT:    v_xor_b32_e32 v19, -1, v9
-; GFX10-NEXT:    v_add_nc_u32_e32 v5, v5, v20
-; GFX10-NEXT:    v_xor_b32_e32 v20, -1, v10
-; GFX10-NEXT:    v_min_u32_e32 v16, v16, v22
-; GFX10-NEXT:    v_min_u32_e32 v17, v17, v23
-; GFX10-NEXT:    v_min_u32_e32 v18, v18, v24
-; GFX10-NEXT:    v_min_u32_e32 v19, v19, v25
-; GFX10-NEXT:    v_min_u32_e32 v20, v20, v26
-; GFX10-NEXT:    v_add_nc_u32_e32 v6, v6, v16
-; GFX10-NEXT:    v_xor_b32_e32 v16, -1, v11
-; GFX10-NEXT:    v_add_nc_u32_e32 v7, v7, v17
-; GFX10-NEXT:    v_xor_b32_e32 v17, -1, v12
-; GFX10-NEXT:    v_add_nc_u32_e32 v8, v8, v18
-; GFX10-NEXT:    v_xor_b32_e32 v18, -1, v13
-; GFX10-NEXT:    v_add_nc_u32_e32 v9, v9, v19
-; GFX10-NEXT:    v_xor_b32_e32 v19, -1, v14
-; GFX10-NEXT:    v_add_nc_u32_e32 v10, v10, v20
-; GFX10-NEXT:    v_xor_b32_e32 v20, -1, v15
-; GFX10-NEXT:    v_min_u32_e32 v16, v16, v27
-; GFX10-NEXT:    v_min_u32_e32 v17, v17, v28
-; GFX10-NEXT:    v_min_u32_e32 v18, v18, v29
-; GFX10-NEXT:    v_min_u32_e32 v19, v19, v30
-; GFX10-NEXT:    v_min_u32_e32 v20, v20, v31
-; GFX10-NEXT:    v_add_nc_u32_e32 v11, v11, v16
-; GFX10-NEXT:    v_add_nc_u32_e32 v12, v12, v17
-; GFX10-NEXT:    v_add_nc_u32_e32 v13, v13, v18
-; GFX10-NEXT:    v_add_nc_u32_e32 v14, v14, v19
-; GFX10-NEXT:    v_add_nc_u32_e32 v15, v15, v20
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <16 x i32> @llvm.uadd.sat.v16i32(<16 x i32> %lhs, <16 x i32> %rhs)
   ret <16 x i32> %result
@@ -1970,207 +1509,143 @@ define amdgpu_ps <16 x i32> @s_uaddsat_v16i32(<16 x i32> inreg %lhs, <16 x i32>
 ;
 ; GFX8-LABEL: s_uaddsat_v16i32:
 ; GFX8:       ; %bb.0:
-; GFX8-NEXT:    s_not_b32 s32, s0
-; GFX8-NEXT:    s_cmp_lt_u32 s32, s16
-; GFX8-NEXT:    s_cselect_b32 s16, s32, s16
-; GFX8-NEXT:    s_add_i32 s0, s0, s16
-; GFX8-NEXT:    s_not_b32 s16, s1
-; GFX8-NEXT:    s_cmp_lt_u32 s16, s17
-; GFX8-NEXT:    s_cselect_b32 s16, s16, s17
-; GFX8-NEXT:    s_add_i32 s1, s1, s16
-; GFX8-NEXT:    s_not_b32 s16, s2
-; GFX8-NEXT:    s_cmp_lt_u32 s16, s18
-; GFX8-NEXT:    s_cselect_b32 s16, s16, s18
-; GFX8-NEXT:    s_add_i32 s2, s2, s16
-; GFX8-NEXT:    s_not_b32 s16, s3
-; GFX8-NEXT:    s_cmp_lt_u32 s16, s19
-; GFX8-NEXT:    s_cselect_b32 s16, s16, s19
-; GFX8-NEXT:    s_add_i32 s3, s3, s16
-; GFX8-NEXT:    s_not_b32 s16, s4
-; GFX8-NEXT:    s_cmp_lt_u32 s16, s20
-; GFX8-NEXT:    s_cselect_b32 s16, s16, s20
-; GFX8-NEXT:    s_add_i32 s4, s4, s16
-; GFX8-NEXT:    s_not_b32 s16, s5
-; GFX8-NEXT:    s_cmp_lt_u32 s16, s21
-; GFX8-NEXT:    s_cselect_b32 s16, s16, s21
-; GFX8-NEXT:    s_add_i32 s5, s5, s16
-; GFX8-NEXT:    s_not_b32 s16, s6
-; GFX8-NEXT:    s_cmp_lt_u32 s16, s22
-; GFX8-NEXT:    s_cselect_b32 s16, s16, s22
-; GFX8-NEXT:    s_add_i32 s6, s6, s16
-; GFX8-NEXT:    s_not_b32 s16, s7
-; GFX8-NEXT:    s_cmp_lt_u32 s16, s23
-; GFX8-NEXT:    s_cselect_b32 s16, s16, s23
-; GFX8-NEXT:    s_add_i32 s7, s7, s16
-; GFX8-NEXT:    s_not_b32 s16, s8
-; GFX8-NEXT:    s_cmp_lt_u32 s16, s24
-; GFX8-NEXT:    s_cselect_b32 s16, s16, s24
-; GFX8-NEXT:    s_add_i32 s8, s8, s16
-; GFX8-NEXT:    s_not_b32 s16, s9
-; GFX8-NEXT:    s_cmp_lt_u32 s16, s25
-; GFX8-NEXT:    s_cselect_b32 s16, s16, s25
-; GFX8-NEXT:    s_add_i32 s9, s9, s16
-; GFX8-NEXT:    s_not_b32 s16, s10
-; GFX8-NEXT:    s_cmp_lt_u32 s16, s26
-; GFX8-NEXT:    s_cselect_b32 s16, s16, s26
-; GFX8-NEXT:    s_add_i32 s10, s10, s16
-; GFX8-NEXT:    s_not_b32 s16, s11
-; GFX8-NEXT:    s_cmp_lt_u32 s16, s27
-; GFX8-NEXT:    s_cselect_b32 s16, s16, s27
-; GFX8-NEXT:    s_add_i32 s11, s11, s16
-; GFX8-NEXT:    s_not_b32 s16, s12
-; GFX8-NEXT:    s_cmp_lt_u32 s16, s28
-; GFX8-NEXT:    s_cselect_b32 s16, s16, s28
-; GFX8-NEXT:    s_add_i32 s12, s12, s16
-; GFX8-NEXT:    s_not_b32 s16, s13
-; GFX8-NEXT:    s_cmp_lt_u32 s16, s29
-; GFX8-NEXT:    s_cselect_b32 s16, s16, s29
-; GFX8-NEXT:    s_add_i32 s13, s13, s16
-; GFX8-NEXT:    s_not_b32 s16, s14
-; GFX8-NEXT:    s_cmp_lt_u32 s16, s30
-; GFX8-NEXT:    s_cselect_b32 s16, s16, s30
-; GFX8-NEXT:    s_add_i32 s14, s14, s16
-; GFX8-NEXT:    s_not_b32 s16, s15
-; GFX8-NEXT:    s_cmp_lt_u32 s16, s31
-; GFX8-NEXT:    s_cselect_b32 s16, s16, s31
-; GFX8-NEXT:    s_add_i32 s15, s15, s16
+; GFX8-NEXT:    v_mov_b32_e32 v0, s16
+; GFX8-NEXT:    v_mov_b32_e32 v1, s17
+; GFX8-NEXT:    v_mov_b32_e32 v2, s18
+; GFX8-NEXT:    v_mov_b32_e32 v3, s19
+; GFX8-NEXT:    v_mov_b32_e32 v4, s20
+; GFX8-NEXT:    v_mov_b32_e32 v5, s21
+; GFX8-NEXT:    v_mov_b32_e32 v6, s22
+; GFX8-NEXT:    v_mov_b32_e32 v7, s23
+; GFX8-NEXT:    v_mov_b32_e32 v8, s24
+; GFX8-NEXT:    v_mov_b32_e32 v9, s25
+; GFX8-NEXT:    v_mov_b32_e32 v10, s26
+; GFX8-NEXT:    v_mov_b32_e32 v11, s27
+; GFX8-NEXT:    v_mov_b32_e32 v12, s28
+; GFX8-NEXT:    v_mov_b32_e32 v13, s29
+; GFX8-NEXT:    v_mov_b32_e32 v14, s30
+; GFX8-NEXT:    v_mov_b32_e32 v15, s31
+; GFX8-NEXT:    v_add_u32_e64 v0, s[32:33], s0, v0 clamp
+; GFX8-NEXT:    v_add_u32_e64 v1, s[16:17], s1, v1 clamp
+; GFX8-NEXT:    v_add_u32_e64 v2, s[16:17], s2, v2 clamp
+; GFX8-NEXT:    v_add_u32_e64 v3, s[2:3], s3, v3 clamp
+; GFX8-NEXT:    v_add_u32_e64 v4, s[2:3], s4, v4 clamp
+; GFX8-NEXT:    v_add_u32_e64 v5, s[2:3], s5, v5 clamp
+; GFX8-NEXT:    v_add_u32_e64 v6, s[2:3], s6, v6 clamp
+; GFX8-NEXT:    v_add_u32_e64 v7, s[2:3], s7, v7 clamp
+; GFX8-NEXT:    v_add_u32_e64 v8, s[2:3], s8, v8 clamp
+; GFX8-NEXT:    v_add_u32_e64 v9, s[2:3], s9, v9 clamp
+; GFX8-NEXT:    v_add_u32_e64 v10, s[2:3], s10, v10 clamp
+; GFX8-NEXT:    v_add_u32_e64 v11, s[2:3], s11, v11 clamp
+; GFX8-NEXT:    v_add_u32_e64 v12, s[2:3], s12, v12 clamp
+; GFX8-NEXT:    v_add_u32_e64 v13, s[2:3], s13, v13 clamp
+; GFX8-NEXT:    v_add_u32_e64 v14, s[2:3], s14, v14 clamp
+; GFX8-NEXT:    v_add_u32_e64 v15, s[2:3], s15, v15 clamp
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX8-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX8-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX8-NEXT:    v_readfirstlane_b32 s4, v4
+; GFX8-NEXT:    v_readfirstlane_b32 s5, v5
+; GFX8-NEXT:    v_readfirstlane_b32 s6, v6
+; GFX8-NEXT:    v_readfirstlane_b32 s7, v7
+; GFX8-NEXT:    v_readfirstlane_b32 s8, v8
+; GFX8-NEXT:    v_readfirstlane_b32 s9, v9
+; GFX8-NEXT:    v_readfirstlane_b32 s10, v10
+; GFX8-NEXT:    v_readfirstlane_b32 s11, v11
+; GFX8-NEXT:    v_readfirstlane_b32 s12, v12
+; GFX8-NEXT:    v_readfirstlane_b32 s13, v13
+; GFX8-NEXT:    v_readfirstlane_b32 s14, v14
+; GFX8-NEXT:    v_readfirstlane_b32 s15, v15
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: s_uaddsat_v16i32:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_not_b32 s32, s0
-; GFX9-NEXT:    s_cmp_lt_u32 s32, s16
-; GFX9-NEXT:    s_cselect_b32 s16, s32, s16
-; GFX9-NEXT:    s_add_i32 s0, s0, s16
-; GFX9-NEXT:    s_not_b32 s16, s1
-; GFX9-NEXT:    s_cmp_lt_u32 s16, s17
-; GFX9-NEXT:    s_cselect_b32 s16, s16, s17
-; GFX9-NEXT:    s_add_i32 s1, s1, s16
-; GFX9-NEXT:    s_not_b32 s16, s2
-; GFX9-NEXT:    s_cmp_lt_u32 s16, s18
-; GFX9-NEXT:    s_cselect_b32 s16, s16, s18
-; GFX9-NEXT:    s_add_i32 s2, s2, s16
-; GFX9-NEXT:    s_not_b32 s16, s3
-; GFX9-NEXT:    s_cmp_lt_u32 s16, s19
-; GFX9-NEXT:    s_cselect_b32 s16, s16, s19
-; GFX9-NEXT:    s_add_i32 s3, s3, s16
-; GFX9-NEXT:    s_not_b32 s16, s4
-; GFX9-NEXT:    s_cmp_lt_u32 s16, s20
-; GFX9-NEXT:    s_cselect_b32 s16, s16, s20
-; GFX9-NEXT:    s_add_i32 s4, s4, s16
-; GFX9-NEXT:    s_not_b32 s16, s5
-; GFX9-NEXT:    s_cmp_lt_u32 s16, s21
-; GFX9-NEXT:    s_cselect_b32 s16, s16, s21
-; GFX9-NEXT:    s_add_i32 s5, s5, s16
-; GFX9-NEXT:    s_not_b32 s16, s6
-; GFX9-NEXT:    s_cmp_lt_u32 s16, s22
-; GFX9-NEXT:    s_cselect_b32 s16, s16, s22
-; GFX9-NEXT:    s_add_i32 s6, s6, s16
-; GFX9-NEXT:    s_not_b32 s16, s7
-; GFX9-NEXT:    s_cmp_lt_u32 s16, s23
-; GFX9-NEXT:    s_cselect_b32 s16, s16, s23
-; GFX9-NEXT:    s_add_i32 s7, s7, s16
-; GFX9-NEXT:    s_not_b32 s16, s8
-; GFX9-NEXT:    s_cmp_lt_u32 s16, s24
-; GFX9-NEXT:    s_cselect_b32 s16, s16, s24
-; GFX9-NEXT:    s_add_i32 s8, s8, s16
-; GFX9-NEXT:    s_not_b32 s16, s9
-; GFX9-NEXT:    s_cmp_lt_u32 s16, s25
-; GFX9-NEXT:    s_cselect_b32 s16, s16, s25
-; GFX9-NEXT:    s_add_i32 s9, s9, s16
-; GFX9-NEXT:    s_not_b32 s16, s10
-; GFX9-NEXT:    s_cmp_lt_u32 s16, s26
-; GFX9-NEXT:    s_cselect_b32 s16, s16, s26
-; GFX9-NEXT:    s_add_i32 s10, s10, s16
-; GFX9-NEXT:    s_not_b32 s16, s11
-; GFX9-NEXT:    s_cmp_lt_u32 s16, s27
-; GFX9-NEXT:    s_cselect_b32 s16, s16, s27
-; GFX9-NEXT:    s_add_i32 s11, s11, s16
-; GFX9-NEXT:    s_not_b32 s16, s12
-; GFX9-NEXT:    s_cmp_lt_u32 s16, s28
-; GFX9-NEXT:    s_cselect_b32 s16, s16, s28
-; GFX9-NEXT:    s_add_i32 s12, s12, s16
-; GFX9-NEXT:    s_not_b32 s16, s13
-; GFX9-NEXT:    s_cmp_lt_u32 s16, s29
-; GFX9-NEXT:    s_cselect_b32 s16, s16, s29
-; GFX9-NEXT:    s_add_i32 s13, s13, s16
-; GFX9-NEXT:    s_not_b32 s16, s14
-; GFX9-NEXT:    s_cmp_lt_u32 s16, s30
-; GFX9-NEXT:    s_cselect_b32 s16, s16, s30
-; GFX9-NEXT:    s_add_i32 s14, s14, s16
-; GFX9-NEXT:    s_not_b32 s16, s15
-; GFX9-NEXT:    s_cmp_lt_u32 s16, s31
-; GFX9-NEXT:    s_cselect_b32 s16, s16, s31
-; GFX9-NEXT:    s_add_i32 s15, s15, s16
+; GFX9-NEXT:    v_mov_b32_e32 v0, s16
+; GFX9-NEXT:    v_mov_b32_e32 v1, s17
+; GFX9-NEXT:    v_mov_b32_e32 v2, s18
+; GFX9-NEXT:    v_mov_b32_e32 v3, s19
+; GFX9-NEXT:    v_mov_b32_e32 v4, s20
+; GFX9-NEXT:    v_mov_b32_e32 v5, s21
+; GFX9-NEXT:    v_mov_b32_e32 v6, s22
+; GFX9-NEXT:    v_mov_b32_e32 v7, s23
+; GFX9-NEXT:    v_mov_b32_e32 v8, s24
+; GFX9-NEXT:    v_mov_b32_e32 v9, s25
+; GFX9-NEXT:    v_mov_b32_e32 v10, s26
+; GFX9-NEXT:    v_mov_b32_e32 v11, s27
+; GFX9-NEXT:    v_mov_b32_e32 v12, s28
+; GFX9-NEXT:    v_mov_b32_e32 v13, s29
+; GFX9-NEXT:    v_mov_b32_e32 v14, s30
+; GFX9-NEXT:    v_mov_b32_e32 v15, s31
+; GFX9-NEXT:    v_add_u32_e64 v0, s0, v0 clamp
+; GFX9-NEXT:    v_add_u32_e64 v1, s1, v1 clamp
+; GFX9-NEXT:    v_add_u32_e64 v2, s2, v2 clamp
+; GFX9-NEXT:    v_add_u32_e64 v3, s3, v3 clamp
+; GFX9-NEXT:    v_add_u32_e64 v4, s4, v4 clamp
+; GFX9-NEXT:    v_add_u32_e64 v5, s5, v5 clamp
+; GFX9-NEXT:    v_add_u32_e64 v6, s6, v6 clamp
+; GFX9-NEXT:    v_add_u32_e64 v7, s7, v7 clamp
+; GFX9-NEXT:    v_add_u32_e64 v8, s8, v8 clamp
+; GFX9-NEXT:    v_add_u32_e64 v9, s9, v9 clamp
+; GFX9-NEXT:    v_add_u32_e64 v10, s10, v10 clamp
+; GFX9-NEXT:    v_add_u32_e64 v11, s11, v11 clamp
+; GFX9-NEXT:    v_add_u32_e64 v12, s12, v12 clamp
+; GFX9-NEXT:    v_add_u32_e64 v13, s13, v13 clamp
+; GFX9-NEXT:    v_add_u32_e64 v14, s14, v14 clamp
+; GFX9-NEXT:    v_add_u32_e64 v15, s15, v15 clamp
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX9-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX9-NEXT:    v_readfirstlane_b32 s4, v4
+; GFX9-NEXT:    v_readfirstlane_b32 s5, v5
+; GFX9-NEXT:    v_readfirstlane_b32 s6, v6
+; GFX9-NEXT:    v_readfirstlane_b32 s7, v7
+; GFX9-NEXT:    v_readfirstlane_b32 s8, v8
+; GFX9-NEXT:    v_readfirstlane_b32 s9, v9
+; GFX9-NEXT:    v_readfirstlane_b32 s10, v10
+; GFX9-NEXT:    v_readfirstlane_b32 s11, v11
+; GFX9-NEXT:    v_readfirstlane_b32 s12, v12
+; GFX9-NEXT:    v_readfirstlane_b32 s13, v13
+; GFX9-NEXT:    v_readfirstlane_b32 s14, v14
+; GFX9-NEXT:    v_readfirstlane_b32 s15, v15
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_uaddsat_v16i32:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_not_b32 s46, s0
+; GFX10-NEXT:    v_add_nc_u32_e64 v0, s0, s16 clamp
+; GFX10-NEXT:    v_add_nc_u32_e64 v1, s1, s17 clamp
+; GFX10-NEXT:    v_add_nc_u32_e64 v2, s2, s18 clamp
+; GFX10-NEXT:    v_add_nc_u32_e64 v3, s3, s19 clamp
+; GFX10-NEXT:    v_add_nc_u32_e64 v4, s4, s20 clamp
+; GFX10-NEXT:    v_add_nc_u32_e64 v5, s5, s21 clamp
+; GFX10-NEXT:    v_add_nc_u32_e64 v6, s6, s22 clamp
+; GFX10-NEXT:    v_add_nc_u32_e64 v7, s7, s23 clamp
+; GFX10-NEXT:    v_add_nc_u32_e64 v8, s8, s24 clamp
+; GFX10-NEXT:    v_add_nc_u32_e64 v9, s9, s25 clamp
+; GFX10-NEXT:    v_add_nc_u32_e64 v10, s10, s26 clamp
+; GFX10-NEXT:    v_add_nc_u32_e64 v11, s11, s27 clamp
+; GFX10-NEXT:    v_add_nc_u32_e64 v12, s12, s28 clamp
+; GFX10-NEXT:    v_add_nc_u32_e64 v13, s13, s29 clamp
+; GFX10-NEXT:    v_add_nc_u32_e64 v14, s14, s30 clamp
+; GFX10-NEXT:    v_add_nc_u32_e64 v15, s15, s31 clamp
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX10-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX10-NEXT:    v_readfirstlane_b32 s4, v4
+; GFX10-NEXT:    v_readfirstlane_b32 s5, v5
+; GFX10-NEXT:    v_readfirstlane_b32 s6, v6
+; GFX10-NEXT:    v_readfirstlane_b32 s7, v7
+; GFX10-NEXT:    v_readfirstlane_b32 s8, v8
+; GFX10-NEXT:    v_readfirstlane_b32 s9, v9
+; GFX10-NEXT:    v_readfirstlane_b32 s10, v10
+; GFX10-NEXT:    v_readfirstlane_b32 s11, v11
+; GFX10-NEXT:    v_readfirstlane_b32 s12, v12
+; GFX10-NEXT:    v_readfirstlane_b32 s13, v13
+; GFX10-NEXT:    v_readfirstlane_b32 s14, v14
+; GFX10-NEXT:    v_readfirstlane_b32 s15, v15
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cmp_lt_u32 s46, s16
-; GFX10-NEXT:    s_cselect_b32 s46, s46, s16
-; GFX10-NEXT:    s_not_b32 s47, s1
-; GFX10-NEXT:    s_add_i32 s0, s0, s46
-; GFX10-NEXT:    s_cmp_lt_u32 s47, s17
-; GFX10-NEXT:    s_cselect_b32 s46, s47, s17
-; GFX10-NEXT:    s_not_b32 s17, s2
-; GFX10-NEXT:    s_add_i32 s1, s1, s46
-; GFX10-NEXT:    s_cmp_lt_u32 s17, s18
-; GFX10-NEXT:    s_cselect_b32 s16, s17, s18
-; GFX10-NEXT:    s_not_b32 s17, s3
-; GFX10-NEXT:    s_add_i32 s2, s2, s16
-; GFX10-NEXT:    s_cmp_lt_u32 s17, s19
-; GFX10-NEXT:    s_cselect_b32 s16, s17, s19
-; GFX10-NEXT:    s_not_b32 s17, s4
-; GFX10-NEXT:    s_add_i32 s3, s3, s16
-; GFX10-NEXT:    s_cmp_lt_u32 s17, s20
-; GFX10-NEXT:    s_cselect_b32 s16, s17, s20
-; GFX10-NEXT:    s_not_b32 s17, s5
-; GFX10-NEXT:    s_add_i32 s4, s4, s16
-; GFX10-NEXT:    s_cmp_lt_u32 s17, s21
-; GFX10-NEXT:    s_cselect_b32 s16, s17, s21
-; GFX10-NEXT:    s_not_b32 s17, s6
-; GFX10-NEXT:    s_add_i32 s5, s5, s16
-; GFX10-NEXT:    s_cmp_lt_u32 s17, s22
-; GFX10-NEXT:    s_cselect_b32 s16, s17, s22
-; GFX10-NEXT:    s_not_b32 s17, s7
-; GFX10-NEXT:    s_add_i32 s6, s6, s16
-; GFX10-NEXT:    s_cmp_lt_u32 s17, s23
-; GFX10-NEXT:    s_cselect_b32 s16, s17, s23
-; GFX10-NEXT:    s_not_b32 s17, s8
-; GFX10-NEXT:    s_add_i32 s7, s7, s16
-; GFX10-NEXT:    s_cmp_lt_u32 s17, s24
-; GFX10-NEXT:    s_cselect_b32 s16, s17, s24
-; GFX10-NEXT:    s_not_b32 s17, s9
-; GFX10-NEXT:    s_add_i32 s8, s8, s16
-; GFX10-NEXT:    s_cmp_lt_u32 s17, s25
-; GFX10-NEXT:    s_cselect_b32 s16, s17, s25
-; GFX10-NEXT:    s_not_b32 s17, s10
-; GFX10-NEXT:    s_add_i32 s9, s9, s16
-; GFX10-NEXT:    s_cmp_lt_u32 s17, s26
-; GFX10-NEXT:    s_cselect_b32 s16, s17, s26
-; GFX10-NEXT:    s_not_b32 s17, s11
-; GFX10-NEXT:    s_add_i32 s10, s10, s16
-; GFX10-NEXT:    s_cmp_lt_u32 s17, s27
-; GFX10-NEXT:    s_cselect_b32 s16, s17, s27
-; GFX10-NEXT:    s_not_b32 s17, s12
-; GFX10-NEXT:    s_add_i32 s11, s11, s16
-; GFX10-NEXT:    s_cmp_lt_u32 s17, s28
-; GFX10-NEXT:    s_cselect_b32 s16, s17, s28
-; GFX10-NEXT:    s_not_b32 s17, s13
-; GFX10-NEXT:    s_add_i32 s12, s12, s16
-; GFX10-NEXT:    s_cmp_lt_u32 s17, s29
-; GFX10-NEXT:    s_cselect_b32 s16, s17, s29
-; GFX10-NEXT:    s_not_b32 s17, s14
-; GFX10-NEXT:    s_add_i32 s13, s13, s16
-; GFX10-NEXT:    s_cmp_lt_u32 s17, s30
-; GFX10-NEXT:    s_cselect_b32 s16, s17, s30
-; GFX10-NEXT:    s_not_b32 s17, s15
-; GFX10-NEXT:    s_add_i32 s14, s14, s16
-; GFX10-NEXT:    s_cmp_lt_u32 s17, s31
-; GFX10-NEXT:    s_cselect_b32 s16, s17, s31
-; GFX10-NEXT:    s_add_i32 s15, s15, s16
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <16 x i32> @llvm.uadd.sat.v16i32(<16 x i32> %lhs, <16 x i32> %rhs)
   ret <16 x i32> %result
@@ -2191,27 +1666,21 @@ define i16 @v_uaddsat_i16(i16 %lhs, i16 %rhs) {
 ; GFX8-LABEL: v_uaddsat_i16:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_xor_b32_e32 v2, -1, v0
-; GFX8-NEXT:    v_min_u16_e32 v1, v2, v1
-; GFX8-NEXT:    v_add_u16_e32 v0, v0, v1
+; GFX8-NEXT:    v_add_u16_e64 v0, v0, v1 clamp
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_uaddsat_i16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_xor_b32_e32 v2, -1, v0
-; GFX9-NEXT:    v_min_u16_e32 v1, v2, v1
-; GFX9-NEXT:    v_add_u16_e32 v0, v0, v1
+; GFX9-NEXT:    v_add_u16_e64 v0, v0, v1 clamp
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_uaddsat_i16:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_xor_b32_e32 v2, -1, v0
+; GFX10-NEXT:    v_add_nc_u16_e64 v0, v0, v1 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_min_u16_e64 v1, v2, v1
-; GFX10-NEXT:    v_add_nc_u16_e64 v0, v0, v1
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call i16 @llvm.uadd.sat.i16(i16 %lhs, i16 %rhs)
   ret i16 %result
@@ -2231,33 +1700,23 @@ define amdgpu_ps i16 @s_uaddsat_i16(i16 inreg %lhs, i16 inreg %rhs) {
 ;
 ; GFX8-LABEL: s_uaddsat_i16:
 ; GFX8:       ; %bb.0:
-; GFX8-NEXT:    s_xor_b32 s2, s0, -1
-; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s2, s1
-; GFX8-NEXT:    s_cselect_b32 s1, s2, s1
-; GFX8-NEXT:    s_add_i32 s0, s0, s1
+; GFX8-NEXT:    v_mov_b32_e32 v0, s1
+; GFX8-NEXT:    v_add_u16_e64 v0, s0, v0 clamp
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: s_uaddsat_i16:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_xor_b32 s2, s0, -1
-; GFX9-NEXT:    s_bfe_u32 s2, s2, 0x100000
-; GFX9-NEXT:    s_bfe_u32 s1, s1, 0x100000
-; GFX9-NEXT:    s_cmp_lt_u32 s2, s1
-; GFX9-NEXT:    s_cselect_b32 s1, s2, s1
-; GFX9-NEXT:    s_add_i32 s0, s0, s1
+; GFX9-NEXT:    v_mov_b32_e32 v0, s1
+; GFX9-NEXT:    v_add_u16_e64 v0, s0, v0 clamp
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_uaddsat_i16:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_xor_b32 s2, s0, -1
-; GFX10-NEXT:    s_bfe_u32 s1, s1, 0x100000
-; GFX10-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX10-NEXT:    v_add_nc_u16_e64 v0, s0, s1 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cmp_lt_u32 s2, s1
-; GFX10-NEXT:    s_cselect_b32 s1, s2, s1
-; GFX10-NEXT:    s_add_i32 s0, s0, s1
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call i16 @llvm.uadd.sat.i16(i16 %lhs, i16 %rhs)
   ret i16 %result
@@ -2276,24 +1735,18 @@ define amdgpu_ps half @uaddsat_i16_sv(i16 inreg %lhs, i16 %rhs) {
 ;
 ; GFX8-LABEL: uaddsat_i16_sv:
 ; GFX8:       ; %bb.0:
-; GFX8-NEXT:    s_xor_b32 s1, s0, -1
-; GFX8-NEXT:    v_min_u16_e32 v0, s1, v0
-; GFX8-NEXT:    v_add_u16_e32 v0, s0, v0
+; GFX8-NEXT:    v_add_u16_e64 v0, s0, v0 clamp
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: uaddsat_i16_sv:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_xor_b32 s1, s0, -1
-; GFX9-NEXT:    v_min_u16_e32 v0, s1, v0
-; GFX9-NEXT:    v_add_u16_e32 v0, s0, v0
+; GFX9-NEXT:    v_add_u16_e64 v0, s0, v0 clamp
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: uaddsat_i16_sv:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_xor_b32 s1, s0, -1
+; GFX10-NEXT:    v_add_nc_u16_e64 v0, s0, v0 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_min_u16_e64 v0, s1, v0
-; GFX10-NEXT:    v_add_nc_u16_e64 v0, s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call i16 @llvm.uadd.sat.i16(i16 %lhs, i16 %rhs)
   %cast = bitcast i16 %result to half
@@ -2313,24 +1766,18 @@ define amdgpu_ps half @uaddsat_i16_vs(i16 %lhs, i16 inreg %rhs) {
 ;
 ; GFX8-LABEL: uaddsat_i16_vs:
 ; GFX8:       ; %bb.0:
-; GFX8-NEXT:    v_xor_b32_e32 v1, -1, v0
-; GFX8-NEXT:    v_min_u16_e32 v1, s0, v1
-; GFX8-NEXT:    v_add_u16_e32 v0, v0, v1
+; GFX8-NEXT:    v_add_u16_e64 v0, v0, s0 clamp
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: uaddsat_i16_vs:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    v_xor_b32_e32 v1, -1, v0
-; GFX9-NEXT:    v_min_u16_e32 v1, s0, v1
-; GFX9-NEXT:    v_add_u16_e32 v0, v0, v1
+; GFX9-NEXT:    v_add_u16_e64 v0, v0, s0 clamp
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: uaddsat_i16_vs:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    v_xor_b32_e32 v1, -1, v0
+; GFX10-NEXT:    v_add_nc_u16_e64 v0, v0, s0 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_min_u16_e64 v1, v1, s0
-; GFX10-NEXT:    v_add_nc_u16_e64 v0, v0, v1
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call i16 @llvm.uadd.sat.i16(i16 %lhs, i16 %rhs)
   %cast = bitcast i16 %result to half
@@ -2358,32 +1805,25 @@ define <2 x i16> @v_uaddsat_v2i16(<2 x i16> %lhs, <2 x i16> %rhs) {
 ; GFX8-LABEL: v_uaddsat_v2i16:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
-; GFX8-NEXT:    v_xor_b32_e32 v3, -1, v0
-; GFX8-NEXT:    v_xor_b32_e32 v4, -1, v2
-; GFX8-NEXT:    v_min_u16_e32 v3, v3, v1
-; GFX8-NEXT:    v_min_u16_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT:    v_add_u16_e32 v0, v0, v3
-; GFX8-NEXT:    v_add_u16_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    v_add_u16_e64 v2, v0, v1 clamp
+; GFX8-NEXT:    v_add_u16_sdwa v0, v0, v1 clamp dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_mov_b32_e32 v1, 16
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_uaddsat_v2i16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_xor_b32_e32 v2, -1, v0
-; GFX9-NEXT:    v_pk_min_u16 v1, v2, v1
-; GFX9-NEXT:    v_pk_add_u16 v0, v0, v1
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, v1 clamp
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_uaddsat_v2i16:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_xor_b32_e32 v2, -1, v0
+; GFX10-NEXT:    v_pk_add_u16 v0, v0, v1 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_pk_min_u16 v1, v2, v1
-; GFX10-NEXT:    v_pk_add_u16 v0, v0, v1
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> %lhs, <2 x i16> %rhs)
   ret <2 x i16> %result
@@ -2415,65 +1855,30 @@ define amdgpu_ps i32 @s_uaddsat_v2i16(<2 x i16> inreg %lhs, <2 x i16> inreg %rhs
 ;
 ; GFX8-LABEL: s_uaddsat_v2i16:
 ; GFX8:       ; %bb.0:
-; GFX8-NEXT:    s_xor_b32 s4, s0, -1
 ; GFX8-NEXT:    s_lshr_b32 s3, s1, 16
 ; GFX8-NEXT:    s_lshr_b32 s2, s0, 16
-; GFX8-NEXT:    s_bfe_u32 s4, s4, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s4, s1
-; GFX8-NEXT:    s_cselect_b32 s1, s4, s1
-; GFX8-NEXT:    s_add_i32 s0, s0, s1
-; GFX8-NEXT:    s_xor_b32 s1, s2, -1
-; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s1, s3
-; GFX8-NEXT:    s_cselect_b32 s1, s1, s3
-; GFX8-NEXT:    s_add_i32 s2, s2, s1
-; GFX8-NEXT:    s_bfe_u32 s1, s2, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s0, s0, 0x100000
-; GFX8-NEXT:    s_lshl_b32 s1, s1, 16
-; GFX8-NEXT:    s_or_b32 s0, s0, s1
+; GFX8-NEXT:    v_mov_b32_e32 v1, s3
+; GFX8-NEXT:    v_mov_b32_e32 v0, s1
+; GFX8-NEXT:    v_add_u16_e64 v1, s2, v1 clamp
+; GFX8-NEXT:    v_mov_b32_e32 v2, 16
+; GFX8-NEXT:    v_add_u16_e64 v0, s0, v0 clamp
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: s_uaddsat_v2i16:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_xor_b32 s2, s0, -1
-; GFX9-NEXT:    s_mov_b32 s4, 0xffff
-; GFX9-NEXT:    s_lshr_b32 s3, s2, 16
-; GFX9-NEXT:    s_lshr_b32 s5, s1, 16
-; GFX9-NEXT:    s_and_b32 s2, s2, s4
-; GFX9-NEXT:    s_and_b32 s1, s1, s4
-; GFX9-NEXT:    s_cmp_lt_u32 s2, s1
-; GFX9-NEXT:    s_cselect_b32 s1, s2, s1
-; GFX9-NEXT:    s_cmp_lt_u32 s3, s5
-; GFX9-NEXT:    s_cselect_b32 s2, s3, s5
-; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s2
-; GFX9-NEXT:    s_lshr_b32 s2, s0, 16
-; GFX9-NEXT:    s_lshr_b32 s3, s1, 16
-; GFX9-NEXT:    s_add_i32 s0, s0, s1
-; GFX9-NEXT:    s_add_i32 s2, s2, s3
-; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s2
+; GFX9-NEXT:    v_mov_b32_e32 v0, s1
+; GFX9-NEXT:    v_pk_add_u16 v0, s0, v0 clamp
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_uaddsat_v2i16:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_xor_b32 s2, s0, -1
-; GFX10-NEXT:    s_mov_b32 s3, 0xffff
-; GFX10-NEXT:    s_lshr_b32 s4, s2, 16
-; GFX10-NEXT:    s_and_b32 s2, s2, s3
-; GFX10-NEXT:    s_and_b32 s3, s1, s3
-; GFX10-NEXT:    s_lshr_b32 s1, s1, 16
-; GFX10-NEXT:    s_cmp_lt_u32 s2, s3
+; GFX10-NEXT:    v_pk_add_u16 v0, s0, s1 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cselect_b32 s2, s2, s3
-; GFX10-NEXT:    s_cmp_lt_u32 s4, s1
-; GFX10-NEXT:    s_cselect_b32 s1, s4, s1
-; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s2, s1
-; GFX10-NEXT:    s_lshr_b32 s2, s0, 16
-; GFX10-NEXT:    s_lshr_b32 s3, s1, 16
-; GFX10-NEXT:    s_add_i32 s0, s0, s1
-; GFX10-NEXT:    s_add_i32 s2, s2, s3
-; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s2
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> %lhs, <2 x i16> %rhs)
   %cast = bitcast <2 x i16> %result to i32
@@ -2504,31 +1909,24 @@ define amdgpu_ps float @uaddsat_v2i16_sv(<2 x i16> inreg %lhs, <2 x i16> %rhs) {
 ;
 ; GFX8-LABEL: uaddsat_v2i16_sv:
 ; GFX8:       ; %bb.0:
-; GFX8-NEXT:    s_xor_b32 s2, s0, -1
 ; GFX8-NEXT:    s_lshr_b32 s1, s0, 16
-; GFX8-NEXT:    v_min_u16_e32 v1, s2, v0
-; GFX8-NEXT:    s_xor_b32 s2, s1, -1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s2
-; GFX8-NEXT:    v_min_u16_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
 ; GFX8-NEXT:    v_mov_b32_e32 v2, s1
-; GFX8-NEXT:    v_add_u16_e32 v1, s0, v1
-; GFX8-NEXT:    v_add_u16_sdwa v0, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX8-NEXT:    v_add_u16_e64 v1, s0, v0 clamp
+; GFX8-NEXT:    v_add_u16_sdwa v0, v2, v0 clamp dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_mov_b32_e32 v2, 16
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: uaddsat_v2i16_sv:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_xor_b32 s1, s0, -1
-; GFX9-NEXT:    v_pk_min_u16 v0, s1, v0
-; GFX9-NEXT:    v_pk_add_u16 v0, s0, v0
+; GFX9-NEXT:    v_pk_add_u16 v0, s0, v0 clamp
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: uaddsat_v2i16_sv:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_xor_b32 s1, s0, -1
+; GFX10-NEXT:    v_pk_add_u16 v0, s0, v0 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_pk_min_u16 v0, s1, v0
-; GFX10-NEXT:    v_pk_add_u16 v0, s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> %lhs, <2 x i16> %rhs)
   %cast = bitcast <2 x i16> %result to float
@@ -2559,30 +1957,24 @@ define amdgpu_ps float @uaddsat_v2i16_vs(<2 x i16> %lhs, <2 x i16> inreg %rhs) {
 ;
 ; GFX8-LABEL: uaddsat_v2i16_vs:
 ; GFX8:       ; %bb.0:
-; GFX8-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
-; GFX8-NEXT:    v_xor_b32_e32 v2, -1, v0
 ; GFX8-NEXT:    s_lshr_b32 s1, s0, 16
-; GFX8-NEXT:    v_xor_b32_e32 v3, -1, v1
-; GFX8-NEXT:    v_min_u16_e32 v2, s0, v2
-; GFX8-NEXT:    v_min_u16_e32 v3, s1, v3
-; GFX8-NEXT:    v_add_u16_e32 v0, v0, v2
-; GFX8-NEXT:    v_add_u16_sdwa v1, v1, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    v_mov_b32_e32 v2, s1
+; GFX8-NEXT:    v_add_u16_e64 v1, v0, s0 clamp
+; GFX8-NEXT:    v_add_u16_sdwa v0, v0, v2 clamp dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT:    v_mov_b32_e32 v2, 16
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: uaddsat_v2i16_vs:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    v_xor_b32_e32 v1, -1, v0
-; GFX9-NEXT:    v_pk_min_u16 v1, v1, s0
-; GFX9-NEXT:    v_pk_add_u16 v0, v0, v1
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, s0 clamp
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: uaddsat_v2i16_vs:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    v_xor_b32_e32 v1, -1, v0
+; GFX10-NEXT:    v_pk_add_u16 v0, v0, s0 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_pk_min_u16 v1, v1, s0
-; GFX10-NEXT:    v_pk_add_u16 v0, v0, v1
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> %lhs, <2 x i16> %rhs)
   %cast = bitcast <2 x i16> %result to float
@@ -2642,46 +2034,31 @@ define <2 x float> @v_uaddsat_v4i16(<4 x i16> %lhs, <4 x i16> %rhs) {
 ; GFX8-LABEL: v_uaddsat_v4i16:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_lshrrev_b32_e32 v4, 16, v0
-; GFX8-NEXT:    v_xor_b32_e32 v6, -1, v0
-; GFX8-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
-; GFX8-NEXT:    v_xor_b32_e32 v7, -1, v4
-; GFX8-NEXT:    v_min_u16_e32 v6, v6, v2
-; GFX8-NEXT:    v_min_u16_sdwa v2, v7, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT:    v_xor_b32_e32 v7, -1, v1
-; GFX8-NEXT:    v_xor_b32_e32 v8, -1, v5
-; GFX8-NEXT:    v_min_u16_e32 v7, v7, v3
-; GFX8-NEXT:    v_min_u16_sdwa v3, v8, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT:    v_add_u16_e32 v0, v0, v6
-; GFX8-NEXT:    v_add_u16_sdwa v2, v4, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT:    v_or_b32_e32 v0, v0, v2
-; GFX8-NEXT:    v_add_u16_e32 v1, v1, v7
-; GFX8-NEXT:    v_add_u16_sdwa v2, v5, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT:    v_or_b32_e32 v1, v1, v2
+; GFX8-NEXT:    v_add_u16_e64 v4, v0, v2 clamp
+; GFX8-NEXT:    v_add_u16_sdwa v0, v0, v2 clamp dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_add_u16_e64 v2, v1, v3 clamp
+; GFX8-NEXT:    v_add_u16_sdwa v1, v1, v3 clamp dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_mov_b32_e32 v3, 16
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_uaddsat_v4i16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_xor_b32_e32 v4, -1, v0
-; GFX9-NEXT:    v_pk_min_u16 v2, v4, v2
-; GFX9-NEXT:    v_pk_add_u16 v0, v0, v2
-; GFX9-NEXT:    v_xor_b32_e32 v2, -1, v1
-; GFX9-NEXT:    v_pk_min_u16 v2, v2, v3
-; GFX9-NEXT:    v_pk_add_u16 v1, v1, v2
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, v2 clamp
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, v3 clamp
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_uaddsat_v4i16:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_xor_b32_e32 v4, -1, v0
-; GFX10-NEXT:    v_xor_b32_e32 v5, -1, v1
+; GFX10-NEXT:    v_pk_add_u16 v0, v0, v2 clamp
+; GFX10-NEXT:    v_pk_add_u16 v1, v1, v3 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_pk_min_u16 v2, v4, v2
-; GFX10-NEXT:    v_pk_min_u16 v3, v5, v3
-; GFX10-NEXT:    v_pk_add_u16 v0, v0, v2
-; GFX10-NEXT:    v_pk_add_u16 v1, v1, v3
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <4 x i16> @llvm.uadd.sat.v4i16(<4 x i16> %lhs, <4 x i16> %rhs)
   %cast = bitcast <4 x i16> %result to <2 x float>
@@ -2732,113 +2109,44 @@ define amdgpu_ps <2 x i32> @s_uaddsat_v4i16(<4 x i16> inreg %lhs, <4 x i16> inre
 ;
 ; GFX8-LABEL: s_uaddsat_v4i16:
 ; GFX8:       ; %bb.0:
-; GFX8-NEXT:    s_xor_b32 s8, s0, -1
 ; GFX8-NEXT:    s_lshr_b32 s6, s2, 16
+; GFX8-NEXT:    s_lshr_b32 s7, s3, 16
 ; GFX8-NEXT:    s_lshr_b32 s4, s0, 16
+; GFX8-NEXT:    v_mov_b32_e32 v1, s6
+; GFX8-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX8-NEXT:    s_lshr_b32 s5, s1, 16
-; GFX8-NEXT:    s_lshr_b32 s7, s3, 16
-; GFX8-NEXT:    s_bfe_u32 s8, s8, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s8, s2
-; GFX8-NEXT:    s_cselect_b32 s2, s8, s2
-; GFX8-NEXT:    s_add_i32 s0, s0, s2
-; GFX8-NEXT:    s_xor_b32 s2, s4, -1
-; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s6, s6, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s2, s6
-; GFX8-NEXT:    s_cselect_b32 s2, s2, s6
-; GFX8-NEXT:    s_add_i32 s4, s4, s2
-; GFX8-NEXT:    s_xor_b32 s2, s1, -1
-; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s2, s3
-; GFX8-NEXT:    s_cselect_b32 s2, s2, s3
-; GFX8-NEXT:    s_add_i32 s1, s1, s2
-; GFX8-NEXT:    s_xor_b32 s2, s5, -1
-; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s3, s7, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s2, s3
-; GFX8-NEXT:    s_cselect_b32 s2, s2, s3
-; GFX8-NEXT:    s_add_i32 s5, s5, s2
-; GFX8-NEXT:    s_bfe_u32 s2, s4, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s0, s0, 0x100000
-; GFX8-NEXT:    s_lshl_b32 s2, s2, 16
-; GFX8-NEXT:    s_or_b32 s0, s0, s2
-; GFX8-NEXT:    s_bfe_u32 s2, s5, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
-; GFX8-NEXT:    s_lshl_b32 s2, s2, 16
-; GFX8-NEXT:    s_or_b32 s1, s1, s2
+; GFX8-NEXT:    v_mov_b32_e32 v3, s7
+; GFX8-NEXT:    v_add_u16_e64 v1, s4, v1 clamp
+; GFX8-NEXT:    v_mov_b32_e32 v4, 16
+; GFX8-NEXT:    v_mov_b32_e32 v2, s3
+; GFX8-NEXT:    v_add_u16_e64 v0, s0, v0 clamp
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_add_u16_e64 v3, s5, v3 clamp
+; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    v_add_u16_e64 v2, s1, v2 clamp
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v1, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: s_uaddsat_v4i16:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_xor_b32 s4, s0, -1
-; GFX9-NEXT:    s_mov_b32 s6, 0xffff
-; GFX9-NEXT:    s_lshr_b32 s5, s4, 16
-; GFX9-NEXT:    s_lshr_b32 s7, s2, 16
-; GFX9-NEXT:    s_and_b32 s4, s4, s6
-; GFX9-NEXT:    s_and_b32 s2, s2, s6
-; GFX9-NEXT:    s_cmp_lt_u32 s4, s2
-; GFX9-NEXT:    s_cselect_b32 s2, s4, s2
-; GFX9-NEXT:    s_cmp_lt_u32 s5, s7
-; GFX9-NEXT:    s_cselect_b32 s4, s5, s7
-; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s4
-; GFX9-NEXT:    s_lshr_b32 s4, s0, 16
-; GFX9-NEXT:    s_lshr_b32 s5, s2, 16
-; GFX9-NEXT:    s_add_i32 s0, s0, s2
-; GFX9-NEXT:    s_add_i32 s4, s4, s5
-; GFX9-NEXT:    s_xor_b32 s2, s1, -1
-; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s4
-; GFX9-NEXT:    s_lshr_b32 s4, s2, 16
-; GFX9-NEXT:    s_lshr_b32 s5, s3, 16
-; GFX9-NEXT:    s_and_b32 s2, s2, s6
-; GFX9-NEXT:    s_and_b32 s3, s3, s6
-; GFX9-NEXT:    s_cmp_lt_u32 s2, s3
-; GFX9-NEXT:    s_cselect_b32 s2, s2, s3
-; GFX9-NEXT:    s_cmp_lt_u32 s4, s5
-; GFX9-NEXT:    s_cselect_b32 s3, s4, s5
-; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s3
-; GFX9-NEXT:    s_lshr_b32 s3, s1, 16
-; GFX9-NEXT:    s_lshr_b32 s4, s2, 16
-; GFX9-NEXT:    s_add_i32 s1, s1, s2
-; GFX9-NEXT:    s_add_i32 s3, s3, s4
-; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s3
+; GFX9-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    v_pk_add_u16 v0, s0, v0 clamp
+; GFX9-NEXT:    v_pk_add_u16 v1, s1, v1 clamp
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_uaddsat_v4i16:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_xor_b32 s4, s0, -1
-; GFX10-NEXT:    s_mov_b32 s5, 0xffff
-; GFX10-NEXT:    s_lshr_b32 s6, s4, 16
-; GFX10-NEXT:    s_and_b32 s7, s2, s5
-; GFX10-NEXT:    s_and_b32 s4, s4, s5
-; GFX10-NEXT:    s_lshr_b32 s2, s2, 16
-; GFX10-NEXT:    s_cmp_lt_u32 s4, s7
+; GFX10-NEXT:    v_pk_add_u16 v0, s0, s2 clamp
+; GFX10-NEXT:    v_pk_add_u16 v1, s1, s3 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cselect_b32 s4, s4, s7
-; GFX10-NEXT:    s_cmp_lt_u32 s6, s2
-; GFX10-NEXT:    s_cselect_b32 s2, s6, s2
-; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s4, s2
-; GFX10-NEXT:    s_lshr_b32 s4, s0, 16
-; GFX10-NEXT:    s_lshr_b32 s6, s2, 16
-; GFX10-NEXT:    s_add_i32 s0, s0, s2
-; GFX10-NEXT:    s_xor_b32 s2, s1, -1
-; GFX10-NEXT:    s_add_i32 s4, s4, s6
-; GFX10-NEXT:    s_lshr_b32 s6, s2, 16
-; GFX10-NEXT:    s_and_b32 s2, s2, s5
-; GFX10-NEXT:    s_and_b32 s5, s3, s5
-; GFX10-NEXT:    s_lshr_b32 s3, s3, 16
-; GFX10-NEXT:    s_cmp_lt_u32 s2, s5
-; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s4
-; GFX10-NEXT:    s_cselect_b32 s2, s2, s5
-; GFX10-NEXT:    s_cmp_lt_u32 s6, s3
-; GFX10-NEXT:    s_cselect_b32 s3, s6, s3
-; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s2, s3
-; GFX10-NEXT:    s_lshr_b32 s3, s1, 16
-; GFX10-NEXT:    s_lshr_b32 s5, s2, 16
-; GFX10-NEXT:    s_add_i32 s1, s1, s2
-; GFX10-NEXT:    s_add_i32 s3, s3, s5
-; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s1, s3
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <4 x i16> @llvm.uadd.sat.v4i16(<4 x i16> %lhs, <4 x i16> %rhs)
   %cast = bitcast <4 x i16> %result to <2 x i32>
@@ -2914,60 +2222,38 @@ define <3 x float> @v_uaddsat_v6i16(<6 x i16> %lhs, <6 x i16> %rhs) {
 ; GFX8-LABEL: v_uaddsat_v6i16:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_lshrrev_b32_e32 v6, 16, v0
-; GFX8-NEXT:    v_xor_b32_e32 v9, -1, v0
-; GFX8-NEXT:    v_lshrrev_b32_e32 v7, 16, v1
-; GFX8-NEXT:    v_xor_b32_e32 v10, -1, v6
-; GFX8-NEXT:    v_min_u16_e32 v9, v9, v3
-; GFX8-NEXT:    v_min_u16_sdwa v3, v10, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT:    v_xor_b32_e32 v10, -1, v1
-; GFX8-NEXT:    v_lshrrev_b32_e32 v8, 16, v2
-; GFX8-NEXT:    v_xor_b32_e32 v11, -1, v7
-; GFX8-NEXT:    v_min_u16_e32 v10, v10, v4
-; GFX8-NEXT:    v_min_u16_sdwa v4, v11, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT:    v_xor_b32_e32 v11, -1, v2
-; GFX8-NEXT:    v_xor_b32_e32 v12, -1, v8
-; GFX8-NEXT:    v_add_u16_e32 v0, v0, v9
-; GFX8-NEXT:    v_add_u16_sdwa v3, v6, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT:    v_min_u16_e32 v11, v11, v5
-; GFX8-NEXT:    v_or_b32_e32 v0, v0, v3
-; GFX8-NEXT:    v_min_u16_sdwa v5, v12, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT:    v_add_u16_e32 v1, v1, v10
-; GFX8-NEXT:    v_add_u16_sdwa v3, v7, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT:    v_or_b32_e32 v1, v1, v3
-; GFX8-NEXT:    v_add_u16_e32 v2, v2, v11
-; GFX8-NEXT:    v_add_u16_sdwa v3, v8, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT:    v_or_b32_e32 v2, v2, v3
+; GFX8-NEXT:    v_add_u16_e64 v6, v0, v3 clamp
+; GFX8-NEXT:    v_add_u16_sdwa v0, v0, v3 clamp dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_add_u16_e64 v3, v1, v4 clamp
+; GFX8-NEXT:    v_add_u16_sdwa v1, v1, v4 clamp dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_add_u16_e64 v4, v2, v5 clamp
+; GFX8-NEXT:    v_add_u16_sdwa v2, v2, v5 clamp dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_mov_b32_e32 v5, 16
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v1, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    v_mov_b32_e32 v3, 16
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v0, v5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_or_b32_sdwa v0, v6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_uaddsat_v6i16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_xor_b32_e32 v6, -1, v0
-; GFX9-NEXT:    v_pk_min_u16 v3, v6, v3
-; GFX9-NEXT:    v_pk_add_u16 v0, v0, v3
-; GFX9-NEXT:    v_xor_b32_e32 v3, -1, v1
-; GFX9-NEXT:    v_pk_min_u16 v3, v3, v4
-; GFX9-NEXT:    v_pk_add_u16 v1, v1, v3
-; GFX9-NEXT:    v_xor_b32_e32 v3, -1, v2
-; GFX9-NEXT:    v_pk_min_u16 v3, v3, v5
-; GFX9-NEXT:    v_pk_add_u16 v2, v2, v3
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, v3 clamp
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, v4 clamp
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, v5 clamp
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_uaddsat_v6i16:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_xor_b32_e32 v6, -1, v0
-; GFX10-NEXT:    v_xor_b32_e32 v7, -1, v1
-; GFX10-NEXT:    v_xor_b32_e32 v8, -1, v2
+; GFX10-NEXT:    v_pk_add_u16 v0, v0, v3 clamp
+; GFX10-NEXT:    v_pk_add_u16 v1, v1, v4 clamp
+; GFX10-NEXT:    v_pk_add_u16 v2, v2, v5 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_pk_min_u16 v3, v6, v3
-; GFX10-NEXT:    v_pk_min_u16 v4, v7, v4
-; GFX10-NEXT:    v_pk_min_u16 v5, v8, v5
-; GFX10-NEXT:    v_pk_add_u16 v0, v0, v3
-; GFX10-NEXT:    v_pk_add_u16 v1, v1, v4
-; GFX10-NEXT:    v_pk_add_u16 v2, v2, v5
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <6 x i16> @llvm.uadd.sat.v6i16(<6 x i16> %lhs, <6 x i16> %rhs)
   %cast = bitcast <6 x i16> %result to <3 x float>
@@ -3036,161 +2322,58 @@ define amdgpu_ps <3 x i32> @s_uaddsat_v6i16(<6 x i16> inreg %lhs, <6 x i16> inre
 ;
 ; GFX8-LABEL: s_uaddsat_v6i16:
 ; GFX8:       ; %bb.0:
-; GFX8-NEXT:    s_xor_b32 s12, s0, -1
 ; GFX8-NEXT:    s_lshr_b32 s9, s3, 16
+; GFX8-NEXT:    s_lshr_b32 s10, s4, 16
 ; GFX8-NEXT:    s_lshr_b32 s6, s0, 16
+; GFX8-NEXT:    v_mov_b32_e32 v1, s9
+; GFX8-NEXT:    v_mov_b32_e32 v0, s3
+; GFX8-NEXT:    s_lshr_b32 s11, s5, 16
 ; GFX8-NEXT:    s_lshr_b32 s7, s1, 16
+; GFX8-NEXT:    v_mov_b32_e32 v3, s10
+; GFX8-NEXT:    v_add_u16_e64 v1, s6, v1 clamp
+; GFX8-NEXT:    v_mov_b32_e32 v6, 16
+; GFX8-NEXT:    v_mov_b32_e32 v2, s4
 ; GFX8-NEXT:    s_lshr_b32 s8, s2, 16
-; GFX8-NEXT:    s_lshr_b32 s10, s4, 16
-; GFX8-NEXT:    s_lshr_b32 s11, s5, 16
-; GFX8-NEXT:    s_bfe_u32 s12, s12, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s12, s3
-; GFX8-NEXT:    s_cselect_b32 s3, s12, s3
-; GFX8-NEXT:    s_add_i32 s0, s0, s3
-; GFX8-NEXT:    s_xor_b32 s3, s6, -1
-; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s9, s9, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s3, s9
-; GFX8-NEXT:    s_cselect_b32 s3, s3, s9
-; GFX8-NEXT:    s_add_i32 s6, s6, s3
-; GFX8-NEXT:    s_xor_b32 s3, s1, -1
-; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s4, s4, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s3, s4
-; GFX8-NEXT:    s_cselect_b32 s3, s3, s4
-; GFX8-NEXT:    s_add_i32 s1, s1, s3
-; GFX8-NEXT:    s_xor_b32 s3, s7, -1
-; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s4, s10, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s3, s4
-; GFX8-NEXT:    s_cselect_b32 s3, s3, s4
-; GFX8-NEXT:    s_add_i32 s7, s7, s3
-; GFX8-NEXT:    s_xor_b32 s3, s2, -1
-; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s4, s5, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s3, s4
-; GFX8-NEXT:    s_cselect_b32 s3, s3, s4
-; GFX8-NEXT:    s_add_i32 s2, s2, s3
-; GFX8-NEXT:    s_xor_b32 s3, s8, -1
-; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s4, s11, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s3, s4
-; GFX8-NEXT:    s_cselect_b32 s3, s3, s4
-; GFX8-NEXT:    s_add_i32 s8, s8, s3
-; GFX8-NEXT:    s_bfe_u32 s3, s6, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s0, s0, 0x100000
-; GFX8-NEXT:    s_lshl_b32 s3, s3, 16
-; GFX8-NEXT:    s_or_b32 s0, s0, s3
-; GFX8-NEXT:    s_bfe_u32 s3, s7, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
-; GFX8-NEXT:    s_lshl_b32 s3, s3, 16
-; GFX8-NEXT:    s_or_b32 s1, s1, s3
-; GFX8-NEXT:    s_bfe_u32 s3, s8, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
-; GFX8-NEXT:    s_lshl_b32 s3, s3, 16
-; GFX8-NEXT:    s_or_b32 s2, s2, s3
+; GFX8-NEXT:    v_mov_b32_e32 v5, s11
+; GFX8-NEXT:    v_add_u16_e64 v0, s0, v0 clamp
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v1, v6, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_add_u16_e64 v3, s7, v3 clamp
+; GFX8-NEXT:    v_mov_b32_e32 v4, s5
+; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    v_add_u16_e64 v2, s1, v2 clamp
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v1, v6, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_add_u16_e64 v5, s8, v5 clamp
+; GFX8-NEXT:    v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    v_add_u16_e64 v4, s2, v4 clamp
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v2, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_or_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX8-NEXT:    v_readfirstlane_b32 s2, v2
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: s_uaddsat_v6i16:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_xor_b32 s6, s0, -1
-; GFX9-NEXT:    s_mov_b32 s8, 0xffff
-; GFX9-NEXT:    s_lshr_b32 s7, s6, 16
-; GFX9-NEXT:    s_lshr_b32 s9, s3, 16
-; GFX9-NEXT:    s_and_b32 s6, s6, s8
-; GFX9-NEXT:    s_and_b32 s3, s3, s8
-; GFX9-NEXT:    s_cmp_lt_u32 s6, s3
-; GFX9-NEXT:    s_cselect_b32 s3, s6, s3
-; GFX9-NEXT:    s_cmp_lt_u32 s7, s9
-; GFX9-NEXT:    s_cselect_b32 s6, s7, s9
-; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s3, s6
-; GFX9-NEXT:    s_lshr_b32 s6, s0, 16
-; GFX9-NEXT:    s_lshr_b32 s7, s3, 16
-; GFX9-NEXT:    s_add_i32 s0, s0, s3
-; GFX9-NEXT:    s_add_i32 s6, s6, s7
-; GFX9-NEXT:    s_xor_b32 s3, s1, -1
-; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s6
-; GFX9-NEXT:    s_lshr_b32 s6, s3, 16
-; GFX9-NEXT:    s_lshr_b32 s7, s4, 16
-; GFX9-NEXT:    s_and_b32 s3, s3, s8
-; GFX9-NEXT:    s_and_b32 s4, s4, s8
-; GFX9-NEXT:    s_cmp_lt_u32 s3, s4
-; GFX9-NEXT:    s_cselect_b32 s3, s3, s4
-; GFX9-NEXT:    s_cmp_lt_u32 s6, s7
-; GFX9-NEXT:    s_cselect_b32 s4, s6, s7
-; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s3, s4
-; GFX9-NEXT:    s_lshr_b32 s4, s1, 16
-; GFX9-NEXT:    s_lshr_b32 s6, s3, 16
-; GFX9-NEXT:    s_add_i32 s1, s1, s3
-; GFX9-NEXT:    s_add_i32 s4, s4, s6
-; GFX9-NEXT:    s_xor_b32 s3, s2, -1
-; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s4
-; GFX9-NEXT:    s_lshr_b32 s4, s3, 16
-; GFX9-NEXT:    s_lshr_b32 s6, s5, 16
-; GFX9-NEXT:    s_and_b32 s3, s3, s8
-; GFX9-NEXT:    s_and_b32 s5, s5, s8
-; GFX9-NEXT:    s_cmp_lt_u32 s3, s5
-; GFX9-NEXT:    s_cselect_b32 s3, s3, s5
-; GFX9-NEXT:    s_cmp_lt_u32 s4, s6
-; GFX9-NEXT:    s_cselect_b32 s4, s4, s6
-; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s3, s4
-; GFX9-NEXT:    s_lshr_b32 s4, s2, 16
-; GFX9-NEXT:    s_lshr_b32 s5, s3, 16
-; GFX9-NEXT:    s_add_i32 s2, s2, s3
-; GFX9-NEXT:    s_add_i32 s4, s4, s5
-; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s4
+; GFX9-NEXT:    v_mov_b32_e32 v0, s3
+; GFX9-NEXT:    v_mov_b32_e32 v1, s4
+; GFX9-NEXT:    v_mov_b32_e32 v2, s5
+; GFX9-NEXT:    v_pk_add_u16 v0, s0, v0 clamp
+; GFX9-NEXT:    v_pk_add_u16 v1, s1, v1 clamp
+; GFX9-NEXT:    v_pk_add_u16 v2, s2, v2 clamp
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v2
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_uaddsat_v6i16:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_xor_b32 s6, s0, -1
-; GFX10-NEXT:    s_mov_b32 s7, 0xffff
-; GFX10-NEXT:    s_lshr_b32 s8, s6, 16
-; GFX10-NEXT:    s_and_b32 s9, s3, s7
-; GFX10-NEXT:    s_and_b32 s6, s6, s7
-; GFX10-NEXT:    s_lshr_b32 s3, s3, 16
-; GFX10-NEXT:    s_cmp_lt_u32 s6, s9
+; GFX10-NEXT:    v_pk_add_u16 v0, s0, s3 clamp
+; GFX10-NEXT:    v_pk_add_u16 v1, s1, s4 clamp
+; GFX10-NEXT:    v_pk_add_u16 v2, s2, s5 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cselect_b32 s6, s6, s9
-; GFX10-NEXT:    s_cmp_lt_u32 s8, s3
-; GFX10-NEXT:    s_cselect_b32 s3, s8, s3
-; GFX10-NEXT:    s_and_b32 s9, s4, s7
-; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s6, s3
-; GFX10-NEXT:    s_lshr_b32 s6, s0, 16
-; GFX10-NEXT:    s_lshr_b32 s8, s3, 16
-; GFX10-NEXT:    s_add_i32 s0, s0, s3
-; GFX10-NEXT:    s_xor_b32 s3, s1, -1
-; GFX10-NEXT:    s_add_i32 s6, s6, s8
-; GFX10-NEXT:    s_lshr_b32 s8, s3, 16
-; GFX10-NEXT:    s_and_b32 s3, s3, s7
-; GFX10-NEXT:    s_lshr_b32 s4, s4, 16
-; GFX10-NEXT:    s_cmp_lt_u32 s3, s9
-; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s6
-; GFX10-NEXT:    s_cselect_b32 s3, s3, s9
-; GFX10-NEXT:    s_cmp_lt_u32 s8, s4
-; GFX10-NEXT:    s_cselect_b32 s4, s8, s4
-; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s3, s4
-; GFX10-NEXT:    s_lshr_b32 s4, s1, 16
-; GFX10-NEXT:    s_lshr_b32 s8, s3, 16
-; GFX10-NEXT:    s_add_i32 s1, s1, s3
-; GFX10-NEXT:    s_xor_b32 s3, s2, -1
-; GFX10-NEXT:    s_add_i32 s4, s4, s8
-; GFX10-NEXT:    s_lshr_b32 s8, s3, 16
-; GFX10-NEXT:    s_and_b32 s3, s3, s7
-; GFX10-NEXT:    s_and_b32 s7, s5, s7
-; GFX10-NEXT:    s_lshr_b32 s5, s5, 16
-; GFX10-NEXT:    s_cmp_lt_u32 s3, s7
-; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s1, s4
-; GFX10-NEXT:    s_cselect_b32 s3, s3, s7
-; GFX10-NEXT:    s_cmp_lt_u32 s8, s5
-; GFX10-NEXT:    s_cselect_b32 s5, s8, s5
-; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s3, s5
-; GFX10-NEXT:    s_lshr_b32 s5, s2, 16
-; GFX10-NEXT:    s_lshr_b32 s6, s3, 16
-; GFX10-NEXT:    s_add_i32 s2, s2, s3
-; GFX10-NEXT:    s_add_i32 s5, s5, s6
-; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s2, s5
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <6 x i16> @llvm.uadd.sat.v6i16(<6 x i16> %lhs, <6 x i16> %rhs)
   %cast = bitcast <6 x i16> %result to <3 x i32>
@@ -3271,74 +2454,44 @@ define <4 x float> @v_uaddsat_v8i16(<8 x i16> %lhs, <8 x i16> %rhs) {
 ; GFX8-LABEL: v_uaddsat_v8i16:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_lshrrev_b32_e32 v8, 16, v0
-; GFX8-NEXT:    v_xor_b32_e32 v12, -1, v0
-; GFX8-NEXT:    v_lshrrev_b32_e32 v9, 16, v1
-; GFX8-NEXT:    v_xor_b32_e32 v13, -1, v8
-; GFX8-NEXT:    v_min_u16_e32 v12, v12, v4
-; GFX8-NEXT:    v_min_u16_sdwa v4, v13, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT:    v_xor_b32_e32 v13, -1, v1
-; GFX8-NEXT:    v_lshrrev_b32_e32 v10, 16, v2
-; GFX8-NEXT:    v_xor_b32_e32 v14, -1, v9
-; GFX8-NEXT:    v_min_u16_e32 v13, v13, v5
-; GFX8-NEXT:    v_lshrrev_b32_e32 v11, 16, v3
-; GFX8-NEXT:    v_min_u16_sdwa v5, v14, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT:    v_xor_b32_e32 v14, -1, v2
-; GFX8-NEXT:    v_xor_b32_e32 v15, -1, v10
-; GFX8-NEXT:    v_add_u16_e32 v0, v0, v12
-; GFX8-NEXT:    v_add_u16_sdwa v4, v8, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT:    v_min_u16_e32 v14, v14, v6
-; GFX8-NEXT:    v_min_u16_sdwa v6, v15, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT:    v_xor_b32_e32 v15, -1, v3
-; GFX8-NEXT:    v_or_b32_e32 v0, v0, v4
-; GFX8-NEXT:    v_xor_b32_e32 v16, -1, v11
-; GFX8-NEXT:    v_add_u16_e32 v1, v1, v13
-; GFX8-NEXT:    v_add_u16_sdwa v4, v9, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT:    v_min_u16_e32 v15, v15, v7
-; GFX8-NEXT:    v_or_b32_e32 v1, v1, v4
-; GFX8-NEXT:    v_min_u16_sdwa v7, v16, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT:    v_add_u16_e32 v2, v2, v14
-; GFX8-NEXT:    v_add_u16_sdwa v4, v10, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT:    v_or_b32_e32 v2, v2, v4
-; GFX8-NEXT:    v_add_u16_e32 v3, v3, v15
-; GFX8-NEXT:    v_add_u16_sdwa v4, v11, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT:    v_or_b32_e32 v3, v3, v4
+; GFX8-NEXT:    v_add_u16_e64 v8, v0, v4 clamp
+; GFX8-NEXT:    v_add_u16_sdwa v0, v0, v4 clamp dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_add_u16_e64 v4, v1, v5 clamp
+; GFX8-NEXT:    v_add_u16_sdwa v1, v1, v5 clamp dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_add_u16_e64 v5, v2, v6 clamp
+; GFX8-NEXT:    v_add_u16_sdwa v2, v2, v6 clamp dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_add_u16_e64 v6, v3, v7 clamp
+; GFX8-NEXT:    v_add_u16_sdwa v3, v3, v7 clamp dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_mov_b32_e32 v7, 16
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v0, v7, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_mov_b32_e32 v7, 16
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v1, v7, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v2, v7, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v3, v7, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_or_b32_sdwa v0, v8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v2, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v3, v6, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_uaddsat_v8i16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_xor_b32_e32 v8, -1, v0
-; GFX9-NEXT:    v_pk_min_u16 v4, v8, v4
-; GFX9-NEXT:    v_pk_add_u16 v0, v0, v4
-; GFX9-NEXT:    v_xor_b32_e32 v4, -1, v1
-; GFX9-NEXT:    v_pk_min_u16 v4, v4, v5
-; GFX9-NEXT:    v_pk_add_u16 v1, v1, v4
-; GFX9-NEXT:    v_xor_b32_e32 v4, -1, v2
-; GFX9-NEXT:    v_pk_min_u16 v4, v4, v6
-; GFX9-NEXT:    v_pk_add_u16 v2, v2, v4
-; GFX9-NEXT:    v_xor_b32_e32 v4, -1, v3
-; GFX9-NEXT:    v_pk_min_u16 v4, v4, v7
-; GFX9-NEXT:    v_pk_add_u16 v3, v3, v4
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, v4 clamp
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, v5 clamp
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, v6 clamp
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, v7 clamp
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_uaddsat_v8i16:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_xor_b32_e32 v15, -1, v0
-; GFX10-NEXT:    v_xor_b32_e32 v19, -1, v1
-; GFX10-NEXT:    v_xor_b32_e32 v23, -1, v2
-; GFX10-NEXT:    v_xor_b32_e32 v10, -1, v3
+; GFX10-NEXT:    v_pk_add_u16 v0, v0, v4 clamp
+; GFX10-NEXT:    v_pk_add_u16 v1, v1, v5 clamp
+; GFX10-NEXT:    v_pk_add_u16 v2, v2, v6 clamp
+; GFX10-NEXT:    v_pk_add_u16 v3, v3, v7 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_pk_min_u16 v11, v15, v4
-; GFX10-NEXT:    v_pk_min_u16 v15, v19, v5
-; GFX10-NEXT:    v_pk_min_u16 v19, v23, v6
-; GFX10-NEXT:    v_pk_min_u16 v6, v10, v7
-; GFX10-NEXT:    v_pk_add_u16 v0, v0, v11
-; GFX10-NEXT:    v_pk_add_u16 v1, v1, v15
-; GFX10-NEXT:    v_pk_add_u16 v2, v2, v19
-; GFX10-NEXT:    v_pk_add_u16 v3, v3, v6
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> %lhs, <8 x i16> %rhs)
   %cast = bitcast <8 x i16> %result to <4 x float>
@@ -3425,209 +2578,72 @@ define amdgpu_ps <4 x i32> @s_uaddsat_v8i16(<8 x i16> inreg %lhs, <8 x i16> inre
 ;
 ; GFX8-LABEL: s_uaddsat_v8i16:
 ; GFX8:       ; %bb.0:
-; GFX8-NEXT:    s_xor_b32 s16, s0, -1
 ; GFX8-NEXT:    s_lshr_b32 s12, s4, 16
+; GFX8-NEXT:    s_lshr_b32 s13, s5, 16
 ; GFX8-NEXT:    s_lshr_b32 s8, s0, 16
+; GFX8-NEXT:    v_mov_b32_e32 v1, s12
+; GFX8-NEXT:    v_mov_b32_e32 v0, s4
+; GFX8-NEXT:    s_lshr_b32 s14, s6, 16
+; GFX8-NEXT:    s_lshr_b32 s15, s7, 16
 ; GFX8-NEXT:    s_lshr_b32 s9, s1, 16
+; GFX8-NEXT:    v_mov_b32_e32 v3, s13
+; GFX8-NEXT:    v_add_u16_e64 v1, s8, v1 clamp
+; GFX8-NEXT:    v_mov_b32_e32 v8, 16
+; GFX8-NEXT:    v_mov_b32_e32 v2, s5
 ; GFX8-NEXT:    s_lshr_b32 s10, s2, 16
+; GFX8-NEXT:    v_mov_b32_e32 v5, s14
 ; GFX8-NEXT:    s_lshr_b32 s11, s3, 16
-; GFX8-NEXT:    s_lshr_b32 s13, s5, 16
-; GFX8-NEXT:    s_lshr_b32 s14, s6, 16
-; GFX8-NEXT:    s_lshr_b32 s15, s7, 16
-; GFX8-NEXT:    s_bfe_u32 s16, s16, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s4, s4, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s16, s4
-; GFX8-NEXT:    s_cselect_b32 s4, s16, s4
-; GFX8-NEXT:    s_add_i32 s0, s0, s4
-; GFX8-NEXT:    s_xor_b32 s4, s8, -1
-; GFX8-NEXT:    s_bfe_u32 s4, s4, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s12, s12, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s4, s12
-; GFX8-NEXT:    s_cselect_b32 s4, s4, s12
-; GFX8-NEXT:    s_add_i32 s8, s8, s4
-; GFX8-NEXT:    s_xor_b32 s4, s1, -1
-; GFX8-NEXT:    s_bfe_u32 s4, s4, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s5, s5, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s4, s5
-; GFX8-NEXT:    s_cselect_b32 s4, s4, s5
-; GFX8-NEXT:    s_add_i32 s1, s1, s4
-; GFX8-NEXT:    s_xor_b32 s4, s9, -1
-; GFX8-NEXT:    s_bfe_u32 s4, s4, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s5, s13, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s4, s5
-; GFX8-NEXT:    s_cselect_b32 s4, s4, s5
-; GFX8-NEXT:    s_add_i32 s9, s9, s4
-; GFX8-NEXT:    s_xor_b32 s4, s2, -1
-; GFX8-NEXT:    s_bfe_u32 s4, s4, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s5, s6, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s4, s5
-; GFX8-NEXT:    s_cselect_b32 s4, s4, s5
-; GFX8-NEXT:    s_add_i32 s2, s2, s4
-; GFX8-NEXT:    s_xor_b32 s4, s10, -1
-; GFX8-NEXT:    s_bfe_u32 s4, s4, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s5, s14, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s4, s5
-; GFX8-NEXT:    s_cselect_b32 s4, s4, s5
-; GFX8-NEXT:    s_add_i32 s10, s10, s4
-; GFX8-NEXT:    s_xor_b32 s4, s3, -1
-; GFX8-NEXT:    s_bfe_u32 s4, s4, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s5, s7, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s4, s5
-; GFX8-NEXT:    s_cselect_b32 s4, s4, s5
-; GFX8-NEXT:    s_add_i32 s3, s3, s4
-; GFX8-NEXT:    s_xor_b32 s4, s11, -1
-; GFX8-NEXT:    s_bfe_u32 s4, s4, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s5, s15, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s4, s5
-; GFX8-NEXT:    s_cselect_b32 s4, s4, s5
-; GFX8-NEXT:    s_add_i32 s11, s11, s4
-; GFX8-NEXT:    s_bfe_u32 s4, s8, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s0, s0, 0x100000
-; GFX8-NEXT:    s_lshl_b32 s4, s4, 16
-; GFX8-NEXT:    s_or_b32 s0, s0, s4
-; GFX8-NEXT:    s_bfe_u32 s4, s9, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
-; GFX8-NEXT:    s_lshl_b32 s4, s4, 16
-; GFX8-NEXT:    s_or_b32 s1, s1, s4
-; GFX8-NEXT:    s_bfe_u32 s4, s10, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
-; GFX8-NEXT:    s_lshl_b32 s4, s4, 16
-; GFX8-NEXT:    s_or_b32 s2, s2, s4
-; GFX8-NEXT:    s_bfe_u32 s4, s11, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
-; GFX8-NEXT:    s_lshl_b32 s4, s4, 16
-; GFX8-NEXT:    s_or_b32 s3, s3, s4
+; GFX8-NEXT:    v_mov_b32_e32 v7, s15
+; GFX8-NEXT:    v_add_u16_e64 v0, s0, v0 clamp
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v1, v8, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_add_u16_e64 v3, s9, v3 clamp
+; GFX8-NEXT:    v_mov_b32_e32 v4, s6
+; GFX8-NEXT:    v_mov_b32_e32 v6, s7
+; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    v_add_u16_e64 v2, s1, v2 clamp
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v1, v8, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_add_u16_e64 v7, s11, v7 clamp
+; GFX8-NEXT:    v_add_u16_e64 v5, s10, v5 clamp
+; GFX8-NEXT:    v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    v_add_u16_e64 v4, s2, v4 clamp
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v2, v8, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_add_u16_e64 v6, s3, v6 clamp
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v3, v8, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_or_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v3, v6, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX8-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX8-NEXT:    v_readfirstlane_b32 s3, v3
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: s_uaddsat_v8i16:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_xor_b32 s8, s0, -1
-; GFX9-NEXT:    s_mov_b32 s10, 0xffff
-; GFX9-NEXT:    s_lshr_b32 s9, s8, 16
-; GFX9-NEXT:    s_lshr_b32 s11, s4, 16
-; GFX9-NEXT:    s_and_b32 s8, s8, s10
-; GFX9-NEXT:    s_and_b32 s4, s4, s10
-; GFX9-NEXT:    s_cmp_lt_u32 s8, s4
-; GFX9-NEXT:    s_cselect_b32 s4, s8, s4
-; GFX9-NEXT:    s_cmp_lt_u32 s9, s11
-; GFX9-NEXT:    s_cselect_b32 s8, s9, s11
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s8
-; GFX9-NEXT:    s_lshr_b32 s8, s0, 16
-; GFX9-NEXT:    s_lshr_b32 s9, s4, 16
-; GFX9-NEXT:    s_add_i32 s0, s0, s4
-; GFX9-NEXT:    s_add_i32 s8, s8, s9
-; GFX9-NEXT:    s_xor_b32 s4, s1, -1
-; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s8
-; GFX9-NEXT:    s_lshr_b32 s8, s4, 16
-; GFX9-NEXT:    s_lshr_b32 s9, s5, 16
-; GFX9-NEXT:    s_and_b32 s4, s4, s10
-; GFX9-NEXT:    s_and_b32 s5, s5, s10
-; GFX9-NEXT:    s_cmp_lt_u32 s4, s5
-; GFX9-NEXT:    s_cselect_b32 s4, s4, s5
-; GFX9-NEXT:    s_cmp_lt_u32 s8, s9
-; GFX9-NEXT:    s_cselect_b32 s5, s8, s9
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
-; GFX9-NEXT:    s_lshr_b32 s5, s1, 16
-; GFX9-NEXT:    s_lshr_b32 s8, s4, 16
-; GFX9-NEXT:    s_add_i32 s1, s1, s4
-; GFX9-NEXT:    s_add_i32 s5, s5, s8
-; GFX9-NEXT:    s_xor_b32 s4, s2, -1
-; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s5
-; GFX9-NEXT:    s_lshr_b32 s5, s4, 16
-; GFX9-NEXT:    s_lshr_b32 s8, s6, 16
-; GFX9-NEXT:    s_and_b32 s4, s4, s10
-; GFX9-NEXT:    s_and_b32 s6, s6, s10
-; GFX9-NEXT:    s_cmp_lt_u32 s4, s6
-; GFX9-NEXT:    s_cselect_b32 s4, s4, s6
-; GFX9-NEXT:    s_cmp_lt_u32 s5, s8
-; GFX9-NEXT:    s_cselect_b32 s5, s5, s8
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
-; GFX9-NEXT:    s_lshr_b32 s5, s2, 16
-; GFX9-NEXT:    s_lshr_b32 s6, s4, 16
-; GFX9-NEXT:    s_add_i32 s2, s2, s4
-; GFX9-NEXT:    s_add_i32 s5, s5, s6
-; GFX9-NEXT:    s_xor_b32 s4, s3, -1
-; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s5
-; GFX9-NEXT:    s_lshr_b32 s5, s4, 16
-; GFX9-NEXT:    s_lshr_b32 s6, s7, 16
-; GFX9-NEXT:    s_and_b32 s4, s4, s10
-; GFX9-NEXT:    s_and_b32 s7, s7, s10
-; GFX9-NEXT:    s_cmp_lt_u32 s4, s7
-; GFX9-NEXT:    s_cselect_b32 s4, s4, s7
-; GFX9-NEXT:    s_cmp_lt_u32 s5, s6
-; GFX9-NEXT:    s_cselect_b32 s5, s5, s6
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
-; GFX9-NEXT:    s_lshr_b32 s5, s3, 16
-; GFX9-NEXT:    s_lshr_b32 s6, s4, 16
-; GFX9-NEXT:    s_add_i32 s3, s3, s4
-; GFX9-NEXT:    s_add_i32 s5, s5, s6
-; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s3, s5
+; GFX9-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-NEXT:    v_mov_b32_e32 v2, s6
+; GFX9-NEXT:    v_mov_b32_e32 v3, s7
+; GFX9-NEXT:    v_pk_add_u16 v0, s0, v0 clamp
+; GFX9-NEXT:    v_pk_add_u16 v1, s1, v1 clamp
+; GFX9-NEXT:    v_pk_add_u16 v2, s2, v2 clamp
+; GFX9-NEXT:    v_pk_add_u16 v3, s3, v3 clamp
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX9-NEXT:    v_readfirstlane_b32 s3, v3
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_uaddsat_v8i16:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_xor_b32 s8, s0, -1
-; GFX10-NEXT:    s_mov_b32 s9, 0xffff
-; GFX10-NEXT:    s_lshr_b32 s10, s8, 16
-; GFX10-NEXT:    s_and_b32 s11, s4, s9
-; GFX10-NEXT:    s_and_b32 s8, s8, s9
-; GFX10-NEXT:    s_lshr_b32 s4, s4, 16
-; GFX10-NEXT:    s_cmp_lt_u32 s8, s11
+; GFX10-NEXT:    v_pk_add_u16 v0, s0, s4 clamp
+; GFX10-NEXT:    v_pk_add_u16 v1, s1, s5 clamp
+; GFX10-NEXT:    v_pk_add_u16 v2, s2, s6 clamp
+; GFX10-NEXT:    v_pk_add_u16 v3, s3, s7 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cselect_b32 s8, s8, s11
-; GFX10-NEXT:    s_cmp_lt_u32 s10, s4
-; GFX10-NEXT:    s_cselect_b32 s4, s10, s4
-; GFX10-NEXT:    s_and_b32 s11, s5, s9
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s8, s4
-; GFX10-NEXT:    s_lshr_b32 s8, s0, 16
-; GFX10-NEXT:    s_lshr_b32 s10, s4, 16
-; GFX10-NEXT:    s_add_i32 s0, s0, s4
-; GFX10-NEXT:    s_xor_b32 s4, s1, -1
-; GFX10-NEXT:    s_add_i32 s8, s8, s10
-; GFX10-NEXT:    s_lshr_b32 s10, s4, 16
-; GFX10-NEXT:    s_and_b32 s4, s4, s9
-; GFX10-NEXT:    s_lshr_b32 s5, s5, 16
-; GFX10-NEXT:    s_cmp_lt_u32 s4, s11
-; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s8
-; GFX10-NEXT:    s_cselect_b32 s4, s4, s11
-; GFX10-NEXT:    s_cmp_lt_u32 s10, s5
-; GFX10-NEXT:    s_cselect_b32 s5, s10, s5
-; GFX10-NEXT:    s_and_b32 s11, s6, s9
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
-; GFX10-NEXT:    s_lshr_b32 s5, s1, 16
-; GFX10-NEXT:    s_lshr_b32 s10, s4, 16
-; GFX10-NEXT:    s_add_i32 s1, s1, s4
-; GFX10-NEXT:    s_xor_b32 s4, s2, -1
-; GFX10-NEXT:    s_add_i32 s5, s5, s10
-; GFX10-NEXT:    s_lshr_b32 s10, s4, 16
-; GFX10-NEXT:    s_and_b32 s4, s4, s9
-; GFX10-NEXT:    s_lshr_b32 s6, s6, 16
-; GFX10-NEXT:    s_cmp_lt_u32 s4, s11
-; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s1, s5
-; GFX10-NEXT:    s_cselect_b32 s4, s4, s11
-; GFX10-NEXT:    s_cmp_lt_u32 s10, s6
-; GFX10-NEXT:    s_cselect_b32 s6, s10, s6
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s4, s6
-; GFX10-NEXT:    s_lshr_b32 s6, s2, 16
-; GFX10-NEXT:    s_lshr_b32 s10, s4, 16
-; GFX10-NEXT:    s_add_i32 s2, s2, s4
-; GFX10-NEXT:    s_xor_b32 s4, s3, -1
-; GFX10-NEXT:    s_add_i32 s6, s6, s10
-; GFX10-NEXT:    s_lshr_b32 s10, s4, 16
-; GFX10-NEXT:    s_and_b32 s4, s4, s9
-; GFX10-NEXT:    s_and_b32 s9, s7, s9
-; GFX10-NEXT:    s_lshr_b32 s7, s7, 16
-; GFX10-NEXT:    s_cmp_lt_u32 s4, s9
-; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s2, s6
-; GFX10-NEXT:    s_cselect_b32 s4, s4, s9
-; GFX10-NEXT:    s_cmp_lt_u32 s10, s7
-; GFX10-NEXT:    s_cselect_b32 s7, s10, s7
-; GFX10-NEXT:    s_lshr_b32 s5, s3, 16
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s4, s7
-; GFX10-NEXT:    s_lshr_b32 s7, s4, 16
-; GFX10-NEXT:    s_add_i32 s3, s3, s4
-; GFX10-NEXT:    s_add_i32 s5, s5, s7
-; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s3, s5
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX10-NEXT:    v_readfirstlane_b32 s3, v3
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> %lhs, <8 x i16> %rhs)
   %cast = bitcast <8 x i16> %result to <4 x i32>
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/usubsat.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/usubsat.ll
index b111fd31851cf..8553853ff00c2 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/usubsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/usubsat.ll
@@ -20,8 +20,7 @@ define i7 @v_usubsat_i7(i7 %lhs, i7 %rhs) {
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    v_lshlrev_b16_e32 v0, 9, v0
 ; GFX8-NEXT:    v_lshlrev_b16_e32 v1, 9, v1
-; GFX8-NEXT:    v_min_u16_e32 v1, v0, v1
-; GFX8-NEXT:    v_sub_u16_e32 v0, v0, v1
+; GFX8-NEXT:    v_sub_u16_e64 v0, v0, v1 clamp
 ; GFX8-NEXT:    v_lshrrev_b16_e32 v0, 9, v0
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -30,8 +29,7 @@ define i7 @v_usubsat_i7(i7 %lhs, i7 %rhs) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 9, v0
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 9, v1
-; GFX9-NEXT:    v_min_u16_e32 v1, v0, v1
-; GFX9-NEXT:    v_sub_u16_e32 v0, v0, v1
+; GFX9-NEXT:    v_sub_u16_e64 v0, v0, v1 clamp
 ; GFX9-NEXT:    v_lshrrev_b16_e32 v0, 9, v0
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -42,8 +40,7 @@ define i7 @v_usubsat_i7(i7 %lhs, i7 %rhs) {
 ; GFX10-NEXT:    v_lshlrev_b16_e64 v0, 9, v0
 ; GFX10-NEXT:    v_lshlrev_b16_e64 v1, 9, v1
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_min_u16_e64 v1, v0, v1
-; GFX10-NEXT:    v_sub_nc_u16_e64 v0, v0, v1
+; GFX10-NEXT:    v_sub_nc_u16_e64 v0, v0, v1 clamp
 ; GFX10-NEXT:    v_lshrrev_b16_e64 v0, 9, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call i7 @llvm.usub.sat.i7(i7 %lhs, i7 %rhs)
@@ -66,13 +63,10 @@ define amdgpu_ps i7 @s_usubsat_i7(i7 inreg %lhs, i7 inreg %rhs) {
 ; GFX8-NEXT:    s_bfe_u32 s2, 9, 0x100000
 ; GFX8-NEXT:    s_lshl_b32 s1, s1, s2
 ; GFX8-NEXT:    s_lshl_b32 s0, s0, s2
-; GFX8-NEXT:    s_bfe_u32 s3, s0, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s3, s1
-; GFX8-NEXT:    s_cselect_b32 s1, s3, s1
-; GFX8-NEXT:    s_sub_i32 s0, s0, s1
-; GFX8-NEXT:    s_bfe_u32 s0, s0, 0x100000
-; GFX8-NEXT:    s_lshr_b32 s0, s0, s2
+; GFX8-NEXT:    v_mov_b32_e32 v0, s1
+; GFX8-NEXT:    v_sub_u16_e64 v0, s0, v0 clamp
+; GFX8-NEXT:    v_lshrrev_b16_e32 v0, 9, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: s_usubsat_i7:
@@ -80,28 +74,21 @@ define amdgpu_ps i7 @s_usubsat_i7(i7 inreg %lhs, i7 inreg %rhs) {
 ; GFX9-NEXT:    s_bfe_u32 s2, 9, 0x100000
 ; GFX9-NEXT:    s_lshl_b32 s1, s1, s2
 ; GFX9-NEXT:    s_lshl_b32 s0, s0, s2
-; GFX9-NEXT:    s_bfe_u32 s3, s0, 0x100000
-; GFX9-NEXT:    s_bfe_u32 s1, s1, 0x100000
-; GFX9-NEXT:    s_cmp_lt_u32 s3, s1
-; GFX9-NEXT:    s_cselect_b32 s1, s3, s1
-; GFX9-NEXT:    s_sub_i32 s0, s0, s1
-; GFX9-NEXT:    s_bfe_u32 s0, s0, 0x100000
-; GFX9-NEXT:    s_lshr_b32 s0, s0, s2
+; GFX9-NEXT:    v_mov_b32_e32 v0, s1
+; GFX9-NEXT:    v_sub_u16_e64 v0, s0, v0 clamp
+; GFX9-NEXT:    v_lshrrev_b16_e32 v0, 9, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_usubsat_i7:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_bfe_u32 s2, 9, 0x100000
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_lshl_b32 s1, s1, s2
 ; GFX10-NEXT:    s_lshl_b32 s0, s0, s2
-; GFX10-NEXT:    s_bfe_u32 s1, s1, 0x100000
-; GFX10-NEXT:    s_bfe_u32 s3, s0, 0x100000
-; GFX10-NEXT:    s_cmp_lt_u32 s3, s1
-; GFX10-NEXT:    s_cselect_b32 s1, s3, s1
-; GFX10-NEXT:    s_sub_i32 s0, s0, s1
-; GFX10-NEXT:    s_bfe_u32 s0, s0, 0x100000
-; GFX10-NEXT:    s_lshr_b32 s0, s0, s2
+; GFX10-NEXT:    s_lshl_b32 s1, s1, s2
+; GFX10-NEXT:    v_sub_nc_u16_e64 v0, s0, s1 clamp
+; GFX10-NEXT:    v_lshrrev_b16_e64 v0, 9, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call i7 @llvm.usub.sat.i7(i7 %lhs, i7 %rhs)
   ret i7 %result
@@ -123,8 +110,7 @@ define i8 @v_usubsat_i8(i8 %lhs, i8 %rhs) {
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
 ; GFX8-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
-; GFX8-NEXT:    v_min_u16_e32 v1, v0, v1
-; GFX8-NEXT:    v_sub_u16_e32 v0, v0, v1
+; GFX8-NEXT:    v_sub_u16_e64 v0, v0, v1 clamp
 ; GFX8-NEXT:    v_lshrrev_b16_e32 v0, 8, v0
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -133,8 +119,7 @@ define i8 @v_usubsat_i8(i8 %lhs, i8 %rhs) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
-; GFX9-NEXT:    v_min_u16_e32 v1, v0, v1
-; GFX9-NEXT:    v_sub_u16_e32 v0, v0, v1
+; GFX9-NEXT:    v_sub_u16_e64 v0, v0, v1 clamp
 ; GFX9-NEXT:    v_lshrrev_b16_e32 v0, 8, v0
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -145,8 +130,7 @@ define i8 @v_usubsat_i8(i8 %lhs, i8 %rhs) {
 ; GFX10-NEXT:    v_lshlrev_b16_e64 v0, 8, v0
 ; GFX10-NEXT:    v_lshlrev_b16_e64 v1, 8, v1
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_min_u16_e64 v1, v0, v1
-; GFX10-NEXT:    v_sub_nc_u16_e64 v0, v0, v1
+; GFX10-NEXT:    v_sub_nc_u16_e64 v0, v0, v1 clamp
 ; GFX10-NEXT:    v_lshrrev_b16_e64 v0, 8, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call i8 @llvm.usub.sat.i8(i8 %lhs, i8 %rhs)
@@ -169,13 +153,10 @@ define amdgpu_ps i8 @s_usubsat_i8(i8 inreg %lhs, i8 inreg %rhs) {
 ; GFX8-NEXT:    s_bfe_u32 s2, 8, 0x100000
 ; GFX8-NEXT:    s_lshl_b32 s1, s1, s2
 ; GFX8-NEXT:    s_lshl_b32 s0, s0, s2
-; GFX8-NEXT:    s_bfe_u32 s3, s0, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s3, s1
-; GFX8-NEXT:    s_cselect_b32 s1, s3, s1
-; GFX8-NEXT:    s_sub_i32 s0, s0, s1
-; GFX8-NEXT:    s_bfe_u32 s0, s0, 0x100000
-; GFX8-NEXT:    s_lshr_b32 s0, s0, s2
+; GFX8-NEXT:    v_mov_b32_e32 v0, s1
+; GFX8-NEXT:    v_sub_u16_e64 v0, s0, v0 clamp
+; GFX8-NEXT:    v_lshrrev_b16_e32 v0, 8, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: s_usubsat_i8:
@@ -183,28 +164,21 @@ define amdgpu_ps i8 @s_usubsat_i8(i8 inreg %lhs, i8 inreg %rhs) {
 ; GFX9-NEXT:    s_bfe_u32 s2, 8, 0x100000
 ; GFX9-NEXT:    s_lshl_b32 s1, s1, s2
 ; GFX9-NEXT:    s_lshl_b32 s0, s0, s2
-; GFX9-NEXT:    s_bfe_u32 s3, s0, 0x100000
-; GFX9-NEXT:    s_bfe_u32 s1, s1, 0x100000
-; GFX9-NEXT:    s_cmp_lt_u32 s3, s1
-; GFX9-NEXT:    s_cselect_b32 s1, s3, s1
-; GFX9-NEXT:    s_sub_i32 s0, s0, s1
-; GFX9-NEXT:    s_bfe_u32 s0, s0, 0x100000
-; GFX9-NEXT:    s_lshr_b32 s0, s0, s2
+; GFX9-NEXT:    v_mov_b32_e32 v0, s1
+; GFX9-NEXT:    v_sub_u16_e64 v0, s0, v0 clamp
+; GFX9-NEXT:    v_lshrrev_b16_e32 v0, 8, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_usubsat_i8:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_bfe_u32 s2, 8, 0x100000
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_lshl_b32 s1, s1, s2
 ; GFX10-NEXT:    s_lshl_b32 s0, s0, s2
-; GFX10-NEXT:    s_bfe_u32 s1, s1, 0x100000
-; GFX10-NEXT:    s_bfe_u32 s3, s0, 0x100000
-; GFX10-NEXT:    s_cmp_lt_u32 s3, s1
-; GFX10-NEXT:    s_cselect_b32 s1, s3, s1
-; GFX10-NEXT:    s_sub_i32 s0, s0, s1
-; GFX10-NEXT:    s_bfe_u32 s0, s0, 0x100000
-; GFX10-NEXT:    s_lshr_b32 s0, s0, s2
+; GFX10-NEXT:    s_lshl_b32 s1, s1, s2
+; GFX10-NEXT:    v_sub_nc_u16_e64 v0, s0, s1 clamp
+; GFX10-NEXT:    v_lshrrev_b16_e64 v0, 8, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call i8 @llvm.usub.sat.i8(i8 %lhs, i8 %rhs)
   ret i8 %result
@@ -241,11 +215,9 @@ define i16 @v_usubsat_v2i8(i16 %lhs.arg, i16 %rhs.arg) {
 ; GFX8-NEXT:    v_lshrrev_b32_sdwa v2, v2, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX8-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
 ; GFX8-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
-; GFX8-NEXT:    v_min_u16_e32 v1, v0, v1
-; GFX8-NEXT:    v_sub_u16_e32 v0, v0, v1
-; GFX8-NEXT:    v_min_u16_e32 v1, v3, v2
+; GFX8-NEXT:    v_sub_u16_e64 v0, v0, v1 clamp
+; GFX8-NEXT:    v_sub_u16_e64 v1, v3, v2 clamp
 ; GFX8-NEXT:    v_mov_b32_e32 v2, 0xff
-; GFX8-NEXT:    v_sub_u16_e32 v1, v3, v1
 ; GFX8-NEXT:    v_and_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
 ; GFX8-NEXT:    v_and_b32_sdwa v1, v1, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
 ; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
@@ -259,11 +231,9 @@ define i16 @v_usubsat_v2i8(i16 %lhs.arg, i16 %rhs.arg) {
 ; GFX9-NEXT:    v_lshrrev_b32_sdwa v3, s4, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
-; GFX9-NEXT:    v_min_u16_e32 v1, v0, v1
-; GFX9-NEXT:    v_sub_u16_e32 v0, v0, v1
-; GFX9-NEXT:    v_min_u16_e32 v1, v2, v3
+; GFX9-NEXT:    v_sub_u16_e64 v0, v0, v1 clamp
 ; GFX9-NEXT:    s_movk_i32 s4, 0xff
-; GFX9-NEXT:    v_sub_u16_e32 v1, v2, v1
+; GFX9-NEXT:    v_sub_u16_e64 v1, v2, v3 clamp
 ; GFX9-NEXT:    v_and_b32_sdwa v0, v0, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
 ; GFX9-NEXT:    v_and_b32_sdwa v1, v1, s4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
 ; GFX9-NEXT:    v_or_b32_e32 v0, v0, v1
@@ -280,10 +250,8 @@ define i16 @v_usubsat_v2i8(i16 %lhs.arg, i16 %rhs.arg) {
 ; GFX10-NEXT:    v_lshlrev_b16_e64 v0, 8, v0
 ; GFX10-NEXT:    s_movk_i32 s4, 0xff
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_min_u16_e64 v1, v2, v1
-; GFX10-NEXT:    v_min_u16_e64 v3, v0, v3
-; GFX10-NEXT:    v_sub_nc_u16_e64 v1, v2, v1
-; GFX10-NEXT:    v_sub_nc_u16_e64 v0, v0, v3
+; GFX10-NEXT:    v_sub_nc_u16_e64 v1, v2, v1 clamp
+; GFX10-NEXT:    v_sub_nc_u16_e64 v0, v0, v3 clamp
 ; GFX10-NEXT:    v_and_b32_sdwa v1, v1, s4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
 ; GFX10-NEXT:    v_and_b32_sdwa v0, v0, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
 ; GFX10-NEXT:    v_or_b32_e32 v0, v0, v1
@@ -322,92 +290,60 @@ define amdgpu_ps i16 @s_usubsat_v2i8(i16 inreg %lhs.arg, i16 inreg %rhs.arg) {
 ; GFX8-LABEL: s_usubsat_v2i8:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_bfe_u32 s4, 8, 0x100000
-; GFX8-NEXT:    s_lshr_b32 s2, s0, 8
 ; GFX8-NEXT:    s_lshr_b32 s3, s1, 8
 ; GFX8-NEXT:    s_lshl_b32 s1, s1, s4
+; GFX8-NEXT:    s_lshr_b32 s2, s0, 8
+; GFX8-NEXT:    v_mov_b32_e32 v0, s1
 ; GFX8-NEXT:    s_lshl_b32 s0, s0, s4
-; GFX8-NEXT:    s_bfe_u32 s5, s0, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s5, s1
-; GFX8-NEXT:    s_cselect_b32 s1, s5, s1
-; GFX8-NEXT:    s_sub_i32 s0, s0, s1
-; GFX8-NEXT:    s_bfe_u32 s0, s0, 0x100000
-; GFX8-NEXT:    s_lshl_b32 s1, s2, s4
-; GFX8-NEXT:    s_lshl_b32 s2, s3, s4
-; GFX8-NEXT:    s_lshr_b32 s0, s0, s4
-; GFX8-NEXT:    s_bfe_u32 s3, s1, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s3, s2
-; GFX8-NEXT:    s_cselect_b32 s2, s3, s2
-; GFX8-NEXT:    s_sub_i32 s1, s1, s2
-; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
-; GFX8-NEXT:    s_movk_i32 s2, 0xff
-; GFX8-NEXT:    s_lshr_b32 s1, s1, s4
-; GFX8-NEXT:    s_and_b32 s1, s1, s2
-; GFX8-NEXT:    s_and_b32 s0, s0, s2
-; GFX8-NEXT:    s_lshl_b32 s1, s1, s4
-; GFX8-NEXT:    s_or_b32 s0, s0, s1
+; GFX8-NEXT:    s_lshl_b32 s1, s3, s4
+; GFX8-NEXT:    v_sub_u16_e64 v0, s0, v0 clamp
+; GFX8-NEXT:    s_lshl_b32 s0, s2, s4
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    v_mov_b32_e32 v2, 0xff
+; GFX8-NEXT:    v_sub_u16_e64 v1, s0, v1 clamp
+; GFX8-NEXT:    v_and_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX8-NEXT:    v_and_b32_sdwa v1, v1, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: s_usubsat_v2i8:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_bfe_u32 s4, 8, 0x100000
-; GFX9-NEXT:    s_lshr_b32 s2, s0, 8
 ; GFX9-NEXT:    s_lshr_b32 s3, s1, 8
 ; GFX9-NEXT:    s_lshl_b32 s1, s1, s4
+; GFX9-NEXT:    s_lshr_b32 s2, s0, 8
+; GFX9-NEXT:    v_mov_b32_e32 v0, s1
 ; GFX9-NEXT:    s_lshl_b32 s0, s0, s4
-; GFX9-NEXT:    s_bfe_u32 s5, s0, 0x100000
-; GFX9-NEXT:    s_bfe_u32 s1, s1, 0x100000
-; GFX9-NEXT:    s_cmp_lt_u32 s5, s1
-; GFX9-NEXT:    s_cselect_b32 s1, s5, s1
-; GFX9-NEXT:    s_sub_i32 s0, s0, s1
-; GFX9-NEXT:    s_bfe_u32 s0, s0, 0x100000
-; GFX9-NEXT:    s_lshl_b32 s1, s2, s4
-; GFX9-NEXT:    s_lshl_b32 s2, s3, s4
-; GFX9-NEXT:    s_lshr_b32 s0, s0, s4
-; GFX9-NEXT:    s_bfe_u32 s3, s1, 0x100000
-; GFX9-NEXT:    s_bfe_u32 s2, s2, 0x100000
-; GFX9-NEXT:    s_cmp_lt_u32 s3, s2
-; GFX9-NEXT:    s_cselect_b32 s2, s3, s2
-; GFX9-NEXT:    s_sub_i32 s1, s1, s2
-; GFX9-NEXT:    s_bfe_u32 s1, s1, 0x100000
-; GFX9-NEXT:    s_movk_i32 s2, 0xff
-; GFX9-NEXT:    s_lshr_b32 s1, s1, s4
-; GFX9-NEXT:    s_and_b32 s1, s1, s2
-; GFX9-NEXT:    s_and_b32 s0, s0, s2
-; GFX9-NEXT:    s_lshl_b32 s1, s1, s4
-; GFX9-NEXT:    s_or_b32 s0, s0, s1
+; GFX9-NEXT:    s_lshl_b32 s1, s3, s4
+; GFX9-NEXT:    v_sub_u16_e64 v0, s0, v0 clamp
+; GFX9-NEXT:    s_lshl_b32 s0, s2, s4
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_sub_u16_e64 v1, s0, v1 clamp
+; GFX9-NEXT:    s_movk_i32 s0, 0xff
+; GFX9-NEXT:    v_and_b32_sdwa v0, v0, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-NEXT:    v_and_b32_sdwa v1, v1, s0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_usubsat_v2i8:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_bfe_u32 s2, 8, 0x100000
-; GFX10-NEXT:    s_lshr_b32 s3, s0, 8
-; GFX10-NEXT:    s_lshl_b32 s4, s1, s2
-; GFX10-NEXT:    s_lshl_b32 s0, s0, s2
-; GFX10-NEXT:    s_bfe_u32 s4, s4, 0x100000
-; GFX10-NEXT:    s_bfe_u32 s5, s0, 0x100000
-; GFX10-NEXT:    s_lshr_b32 s1, s1, 8
-; GFX10-NEXT:    s_cmp_lt_u32 s5, s4
+; GFX10-NEXT:    s_lshr_b32 s2, s0, 8
+; GFX10-NEXT:    s_bfe_u32 s3, 8, 0x100000
+; GFX10-NEXT:    s_lshr_b32 s4, s1, 8
+; GFX10-NEXT:    s_lshl_b32 s0, s0, s3
+; GFX10-NEXT:    s_lshl_b32 s1, s1, s3
+; GFX10-NEXT:    s_lshl_b32 s2, s2, s3
+; GFX10-NEXT:    s_lshl_b32 s3, s4, s3
+; GFX10-NEXT:    v_sub_nc_u16_e64 v0, s0, s1 clamp
+; GFX10-NEXT:    v_sub_nc_u16_e64 v1, s2, s3 clamp
+; GFX10-NEXT:    s_movk_i32 s0, 0xff
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cselect_b32 s4, s5, s4
-; GFX10-NEXT:    s_lshl_b32 s1, s1, s2
-; GFX10-NEXT:    s_sub_i32 s0, s0, s4
-; GFX10-NEXT:    s_lshl_b32 s3, s3, s2
-; GFX10-NEXT:    s_bfe_u32 s0, s0, 0x100000
-; GFX10-NEXT:    s_bfe_u32 s4, s3, 0x100000
-; GFX10-NEXT:    s_bfe_u32 s1, s1, 0x100000
-; GFX10-NEXT:    s_lshr_b32 s0, s0, s2
-; GFX10-NEXT:    s_cmp_lt_u32 s4, s1
-; GFX10-NEXT:    s_cselect_b32 s1, s4, s1
-; GFX10-NEXT:    s_sub_i32 s1, s3, s1
-; GFX10-NEXT:    s_movk_i32 s3, 0xff
-; GFX10-NEXT:    s_bfe_u32 s1, s1, 0x100000
-; GFX10-NEXT:    s_and_b32 s0, s0, s3
-; GFX10-NEXT:    s_lshr_b32 s1, s1, s2
-; GFX10-NEXT:    s_and_b32 s1, s1, s3
-; GFX10-NEXT:    s_lshl_b32 s1, s1, s2
-; GFX10-NEXT:    s_or_b32 s0, s0, s1
+; GFX10-NEXT:    v_and_b32_sdwa v0, v0, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_and_b32_sdwa v1, v1, s0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
   %lhs = bitcast i16 %lhs.arg to <2 x i8>
   %rhs = bitcast i16 %rhs.arg to <2 x i8>
@@ -471,18 +407,14 @@ define i32 @v_usubsat_v4i8(i32 %lhs.arg, i32 %rhs.arg) {
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v7, 24, v1
 ; GFX8-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
 ; GFX8-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
-; GFX8-NEXT:    v_min_u16_e32 v1, v0, v1
-; GFX8-NEXT:    v_sub_u16_e32 v0, v0, v1
-; GFX8-NEXT:    v_min_u16_e32 v1, v3, v2
-; GFX8-NEXT:    v_sub_u16_e32 v1, v3, v1
+; GFX8-NEXT:    v_sub_u16_e64 v0, v0, v1 clamp
+; GFX8-NEXT:    v_sub_u16_e64 v1, v3, v2 clamp
 ; GFX8-NEXT:    v_lshlrev_b16_e32 v2, 8, v4
 ; GFX8-NEXT:    v_lshlrev_b16_e32 v3, 8, v6
-; GFX8-NEXT:    v_min_u16_e32 v3, v2, v3
-; GFX8-NEXT:    v_sub_u16_e32 v2, v2, v3
+; GFX8-NEXT:    v_sub_u16_e64 v2, v2, v3 clamp
 ; GFX8-NEXT:    v_lshlrev_b16_e32 v3, 8, v5
 ; GFX8-NEXT:    v_lshlrev_b16_e32 v4, 8, v7
-; GFX8-NEXT:    v_min_u16_e32 v4, v3, v4
-; GFX8-NEXT:    v_sub_u16_e32 v3, v3, v4
+; GFX8-NEXT:    v_sub_u16_e64 v3, v3, v4 clamp
 ; GFX8-NEXT:    v_mov_b32_e32 v4, 0xff
 ; GFX8-NEXT:    v_and_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
 ; GFX8-NEXT:    v_and_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
@@ -506,22 +438,18 @@ define i32 @v_usubsat_v4i8(i32 %lhs.arg, i32 %rhs.arg) {
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v7, 24, v1
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
-; GFX9-NEXT:    v_min_u16_e32 v1, v0, v1
-; GFX9-NEXT:    v_sub_u16_e32 v0, v0, v1
-; GFX9-NEXT:    v_min_u16_e32 v1, v2, v5
-; GFX9-NEXT:    v_sub_u16_e32 v1, v2, v1
+; GFX9-NEXT:    v_sub_u16_e64 v0, v0, v1 clamp
+; GFX9-NEXT:    v_sub_u16_e64 v1, v2, v5 clamp
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v2, 8, v3
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v3, 8, v6
-; GFX9-NEXT:    v_min_u16_e32 v3, v2, v3
-; GFX9-NEXT:    v_sub_u16_e32 v2, v2, v3
-; GFX9-NEXT:    v_lshlrev_b16_e32 v3, 8, v4
-; GFX9-NEXT:    v_lshlrev_b16_e32 v4, 8, v7
 ; GFX9-NEXT:    s_movk_i32 s4, 0xff
-; GFX9-NEXT:    v_min_u16_e32 v4, v3, v4
 ; GFX9-NEXT:    v_and_b32_sdwa v1, v1, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-NEXT:    v_sub_u16_e64 v2, v2, v3 clamp
+; GFX9-NEXT:    v_lshlrev_b16_e32 v3, 8, v4
+; GFX9-NEXT:    v_lshlrev_b16_e32 v4, 8, v7
 ; GFX9-NEXT:    v_lshrrev_b16_e32 v0, 8, v0
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
-; GFX9-NEXT:    v_sub_u16_e32 v3, v3, v4
+; GFX9-NEXT:    v_sub_u16_e64 v3, v3, v4 clamp
 ; GFX9-NEXT:    v_and_or_b32 v0, v0, s4, v1
 ; GFX9-NEXT:    v_and_b32_sdwa v1, v2, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
 ; GFX9-NEXT:    v_and_b32_sdwa v2, v3, s4 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
@@ -533,32 +461,28 @@ define i32 @v_usubsat_v4i8(i32 %lhs.arg, i32 %rhs.arg) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    s_mov_b32 s4, 8
-; GFX10-NEXT:    v_lshlrev_b16_e64 v4, 8, v0
+; GFX10-NEXT:    v_lshlrev_b16_e64 v5, 8, v0
 ; GFX10-NEXT:    v_lshrrev_b32_sdwa v2, s4, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX10-NEXT:    v_lshrrev_b32_sdwa v3, s4, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT:    v_lshlrev_b16_e64 v5, 8, v1
-; GFX10-NEXT:    s_mov_b32 s4, 16
-; GFX10-NEXT:    s_mov_b32 s5, 24
-; GFX10-NEXT:    v_lshrrev_b32_sdwa v6, s4, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT:    v_min_u16_e64 v3, v2, v3
-; GFX10-NEXT:    v_lshrrev_b32_sdwa v7, s4, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT:    v_min_u16_e64 v5, v4, v5
-; GFX10-NEXT:    v_lshrrev_b32_sdwa v0, s5, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT:    v_lshrrev_b32_sdwa v1, s5, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT:    v_sub_nc_u16_e64 v2, v2, v3
-; GFX10-NEXT:    s_movk_i32 s4, 0xff
-; GFX10-NEXT:    v_min_u16_e64 v3, v6, v7
-; GFX10-NEXT:    v_sub_nc_u16_e64 v4, v4, v5
-; GFX10-NEXT:    v_min_u16_e64 v1, v0, v1
-; GFX10-NEXT:    v_and_b32_sdwa v2, v2, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_lshlrev_b16_e64 v6, 8, v1
+; GFX10-NEXT:    s_mov_b32 s5, 16
+; GFX10-NEXT:    s_mov_b32 s4, 24
+; GFX10-NEXT:    v_lshrrev_b32_sdwa v4, s5, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    v_sub_nc_u16_e64 v2, v2, v3 clamp
+; GFX10-NEXT:    v_lshrrev_b32_sdwa v3, s5, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    s_movk_i32 s5, 0xff
+; GFX10-NEXT:    v_lshrrev_b32_sdwa v0, s4, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    v_lshrrev_b32_sdwa v1, s4, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    v_and_b32_sdwa v2, v2, s5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_sub_nc_u16_e64 v5, v5, v6 clamp
+; GFX10-NEXT:    v_sub_nc_u16_e64 v3, v4, v3 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_sub_nc_u16_e64 v3, v6, v3
-; GFX10-NEXT:    v_lshrrev_b16_e64 v4, 8, v4
-; GFX10-NEXT:    v_sub_nc_u16_e64 v0, v0, v1
+; GFX10-NEXT:    v_sub_nc_u16_e64 v0, v0, v1 clamp
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
-; GFX10-NEXT:    v_and_b32_sdwa v1, v3, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
-; GFX10-NEXT:    v_and_b32_sdwa v0, v0, s4 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
-; GFX10-NEXT:    v_and_or_b32 v2, v4, s4, v2
+; GFX10-NEXT:    v_lshrrev_b16_e64 v4, 8, v5
+; GFX10-NEXT:    v_and_b32_sdwa v1, v3, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_and_b32_sdwa v0, v0, s5 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_and_or_b32 v2, v4, s5, v2
 ; GFX10-NEXT:    v_or3_b32 v0, v2, v1, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %lhs = bitcast i32 %lhs.arg to <4 x i8>
@@ -617,176 +541,107 @@ define amdgpu_ps i32 @s_usubsat_v4i8(i32 inreg %lhs.arg, i32 inreg %rhs.arg) {
 ; GFX8-LABEL: s_usubsat_v4i8:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_bfe_u32 s8, 8, 0x100000
-; GFX8-NEXT:    s_lshr_b32 s2, s0, 8
-; GFX8-NEXT:    s_lshr_b32 s3, s0, 16
-; GFX8-NEXT:    s_lshr_b32 s4, s0, 24
-; GFX8-NEXT:    s_lshl_b32 s0, s0, s8
 ; GFX8-NEXT:    s_lshr_b32 s5, s1, 8
 ; GFX8-NEXT:    s_lshr_b32 s6, s1, 16
 ; GFX8-NEXT:    s_lshr_b32 s7, s1, 24
 ; GFX8-NEXT:    s_lshl_b32 s1, s1, s8
-; GFX8-NEXT:    s_bfe_u32 s9, s0, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s9, s1
-; GFX8-NEXT:    s_cselect_b32 s1, s9, s1
-; GFX8-NEXT:    s_sub_i32 s0, s0, s1
-; GFX8-NEXT:    s_bfe_u32 s0, s0, 0x100000
-; GFX8-NEXT:    s_lshl_b32 s1, s2, s8
-; GFX8-NEXT:    s_lshl_b32 s2, s5, s8
-; GFX8-NEXT:    s_lshr_b32 s0, s0, s8
-; GFX8-NEXT:    s_bfe_u32 s5, s1, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s5, s2
-; GFX8-NEXT:    s_cselect_b32 s2, s5, s2
-; GFX8-NEXT:    s_sub_i32 s1, s1, s2
-; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
-; GFX8-NEXT:    s_lshl_b32 s2, s3, s8
-; GFX8-NEXT:    s_lshl_b32 s3, s6, s8
-; GFX8-NEXT:    s_lshr_b32 s1, s1, s8
-; GFX8-NEXT:    s_bfe_u32 s5, s2, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s5, s3
-; GFX8-NEXT:    s_cselect_b32 s3, s5, s3
-; GFX8-NEXT:    s_sub_i32 s2, s2, s3
-; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
-; GFX8-NEXT:    s_lshl_b32 s3, s4, s8
-; GFX8-NEXT:    s_lshl_b32 s4, s7, s8
-; GFX8-NEXT:    s_lshr_b32 s2, s2, s8
-; GFX8-NEXT:    s_bfe_u32 s5, s3, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s4, s4, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s5, s4
-; GFX8-NEXT:    s_cselect_b32 s4, s5, s4
-; GFX8-NEXT:    s_sub_i32 s3, s3, s4
-; GFX8-NEXT:    s_movk_i32 s4, 0xff
-; GFX8-NEXT:    s_and_b32 s1, s1, s4
-; GFX8-NEXT:    s_and_b32 s0, s0, s4
-; GFX8-NEXT:    s_lshl_b32 s1, s1, 8
-; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
-; GFX8-NEXT:    s_or_b32 s0, s0, s1
-; GFX8-NEXT:    s_and_b32 s1, s2, s4
-; GFX8-NEXT:    s_lshl_b32 s1, s1, 16
-; GFX8-NEXT:    s_lshr_b32 s3, s3, s8
-; GFX8-NEXT:    s_or_b32 s0, s0, s1
-; GFX8-NEXT:    s_and_b32 s1, s3, s4
-; GFX8-NEXT:    s_lshl_b32 s1, s1, 24
-; GFX8-NEXT:    s_or_b32 s0, s0, s1
+; GFX8-NEXT:    v_mov_b32_e32 v0, s1
+; GFX8-NEXT:    s_lshl_b32 s1, s5, s8
+; GFX8-NEXT:    s_lshr_b32 s2, s0, 8
+; GFX8-NEXT:    s_lshr_b32 s3, s0, 16
+; GFX8-NEXT:    s_lshr_b32 s4, s0, 24
+; GFX8-NEXT:    s_lshl_b32 s0, s0, s8
+; GFX8-NEXT:    v_sub_u16_e64 v0, s0, v0 clamp
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    s_lshl_b32 s0, s2, s8
+; GFX8-NEXT:    v_sub_u16_e64 v1, s0, v1 clamp
+; GFX8-NEXT:    v_mov_b32_e32 v4, 0xff
+; GFX8-NEXT:    s_lshl_b32 s1, s6, s8
+; GFX8-NEXT:    v_and_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX8-NEXT:    v_mov_b32_e32 v2, s1
+; GFX8-NEXT:    s_lshl_b32 s0, s3, s8
+; GFX8-NEXT:    s_lshl_b32 s1, s7, s8
+; GFX8-NEXT:    v_sub_u16_e64 v2, s0, v2 clamp
+; GFX8-NEXT:    s_lshl_b32 s0, s4, s8
+; GFX8-NEXT:    v_mov_b32_e32 v3, s1
+; GFX8-NEXT:    v_and_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX8-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    v_and_b32_sdwa v1, v2, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX8-NEXT:    v_sub_u16_e64 v3, s0, v3 clamp
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    v_and_b32_sdwa v1, v3, v4 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: s_usubsat_v4i8:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_bfe_u32 s8, 8, 0x100000
-; GFX9-NEXT:    s_lshr_b32 s2, s0, 8
-; GFX9-NEXT:    s_lshr_b32 s3, s0, 16
-; GFX9-NEXT:    s_lshr_b32 s4, s0, 24
-; GFX9-NEXT:    s_lshl_b32 s0, s0, s8
 ; GFX9-NEXT:    s_lshr_b32 s5, s1, 8
 ; GFX9-NEXT:    s_lshr_b32 s6, s1, 16
 ; GFX9-NEXT:    s_lshr_b32 s7, s1, 24
 ; GFX9-NEXT:    s_lshl_b32 s1, s1, s8
-; GFX9-NEXT:    s_bfe_u32 s9, s0, 0x100000
-; GFX9-NEXT:    s_bfe_u32 s1, s1, 0x100000
-; GFX9-NEXT:    s_cmp_lt_u32 s9, s1
-; GFX9-NEXT:    s_cselect_b32 s1, s9, s1
-; GFX9-NEXT:    s_sub_i32 s0, s0, s1
-; GFX9-NEXT:    s_bfe_u32 s0, s0, 0x100000
-; GFX9-NEXT:    s_lshl_b32 s1, s2, s8
-; GFX9-NEXT:    s_lshl_b32 s2, s5, s8
-; GFX9-NEXT:    s_lshr_b32 s0, s0, s8
-; GFX9-NEXT:    s_bfe_u32 s5, s1, 0x100000
-; GFX9-NEXT:    s_bfe_u32 s2, s2, 0x100000
-; GFX9-NEXT:    s_cmp_lt_u32 s5, s2
-; GFX9-NEXT:    s_cselect_b32 s2, s5, s2
-; GFX9-NEXT:    s_sub_i32 s1, s1, s2
-; GFX9-NEXT:    s_bfe_u32 s1, s1, 0x100000
-; GFX9-NEXT:    s_lshl_b32 s2, s3, s8
-; GFX9-NEXT:    s_lshl_b32 s3, s6, s8
-; GFX9-NEXT:    s_lshr_b32 s1, s1, s8
-; GFX9-NEXT:    s_bfe_u32 s5, s2, 0x100000
-; GFX9-NEXT:    s_bfe_u32 s3, s3, 0x100000
-; GFX9-NEXT:    s_cmp_lt_u32 s5, s3
-; GFX9-NEXT:    s_cselect_b32 s3, s5, s3
-; GFX9-NEXT:    s_sub_i32 s2, s2, s3
-; GFX9-NEXT:    s_bfe_u32 s2, s2, 0x100000
-; GFX9-NEXT:    s_lshl_b32 s3, s4, s8
-; GFX9-NEXT:    s_lshl_b32 s4, s7, s8
-; GFX9-NEXT:    s_lshr_b32 s2, s2, s8
-; GFX9-NEXT:    s_bfe_u32 s5, s3, 0x100000
-; GFX9-NEXT:    s_bfe_u32 s4, s4, 0x100000
-; GFX9-NEXT:    s_cmp_lt_u32 s5, s4
-; GFX9-NEXT:    s_cselect_b32 s4, s5, s4
-; GFX9-NEXT:    s_sub_i32 s3, s3, s4
-; GFX9-NEXT:    s_movk_i32 s4, 0xff
-; GFX9-NEXT:    s_and_b32 s1, s1, s4
-; GFX9-NEXT:    s_and_b32 s0, s0, s4
-; GFX9-NEXT:    s_lshl_b32 s1, s1, 8
-; GFX9-NEXT:    s_bfe_u32 s3, s3, 0x100000
-; GFX9-NEXT:    s_or_b32 s0, s0, s1
-; GFX9-NEXT:    s_and_b32 s1, s2, s4
-; GFX9-NEXT:    s_lshl_b32 s1, s1, 16
-; GFX9-NEXT:    s_lshr_b32 s3, s3, s8
-; GFX9-NEXT:    s_or_b32 s0, s0, s1
-; GFX9-NEXT:    s_and_b32 s1, s3, s4
-; GFX9-NEXT:    s_lshl_b32 s1, s1, 24
-; GFX9-NEXT:    s_or_b32 s0, s0, s1
+; GFX9-NEXT:    v_mov_b32_e32 v0, s1
+; GFX9-NEXT:    s_lshl_b32 s1, s5, s8
+; GFX9-NEXT:    s_lshr_b32 s2, s0, 8
+; GFX9-NEXT:    s_lshr_b32 s3, s0, 16
+; GFX9-NEXT:    s_lshr_b32 s4, s0, 24
+; GFX9-NEXT:    s_lshl_b32 s0, s0, s8
+; GFX9-NEXT:    v_sub_u16_e64 v0, s0, v0 clamp
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    s_lshl_b32 s0, s2, s8
+; GFX9-NEXT:    s_lshl_b32 s1, s6, s8
+; GFX9-NEXT:    v_sub_u16_e64 v1, s0, v1 clamp
+; GFX9-NEXT:    v_mov_b32_e32 v2, s1
+; GFX9-NEXT:    s_lshl_b32 s0, s3, s8
+; GFX9-NEXT:    s_lshl_b32 s1, s7, s8
+; GFX9-NEXT:    v_sub_u16_e64 v2, s0, v2 clamp
+; GFX9-NEXT:    s_lshl_b32 s0, s4, s8
+; GFX9-NEXT:    v_mov_b32_e32 v3, s1
+; GFX9-NEXT:    v_sub_u16_e64 v3, s0, v3 clamp
+; GFX9-NEXT:    s_movk_i32 s0, 0xff
+; GFX9-NEXT:    v_and_b32_sdwa v1, v1, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-NEXT:    v_lshrrev_b16_e32 v0, 8, v0
+; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX9-NEXT:    v_and_or_b32 v0, v0, s0, v1
+; GFX9-NEXT:    v_and_b32_sdwa v1, v2, s0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-NEXT:    v_and_b32_sdwa v2, v3, s0 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-NEXT:    v_or3_b32 v0, v0, v1, v2
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_usubsat_v4i8:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_bfe_u32 s6, 8, 0x100000
+; GFX10-NEXT:    s_bfe_u32 s5, 8, 0x100000
 ; GFX10-NEXT:    s_lshr_b32 s2, s0, 8
-; GFX10-NEXT:    s_lshl_b32 s8, s1, s6
+; GFX10-NEXT:    s_lshr_b32 s6, s1, 8
+; GFX10-NEXT:    s_lshl_b32 s2, s2, s5
+; GFX10-NEXT:    s_lshl_b32 s6, s6, s5
 ; GFX10-NEXT:    s_lshr_b32 s3, s0, 16
+; GFX10-NEXT:    v_sub_nc_u16_e64 v1, s2, s6 clamp
 ; GFX10-NEXT:    s_lshr_b32 s4, s0, 24
-; GFX10-NEXT:    s_lshl_b32 s0, s0, s6
-; GFX10-NEXT:    s_bfe_u32 s8, s8, 0x100000
-; GFX10-NEXT:    s_bfe_u32 s9, s0, 0x100000
-; GFX10-NEXT:    s_lshr_b32 s5, s1, 8
-; GFX10-NEXT:    s_lshr_b32 s7, s1, 16
+; GFX10-NEXT:    s_movk_i32 s2, 0xff
+; GFX10-NEXT:    s_lshl_b32 s0, s0, s5
+; GFX10-NEXT:    s_lshl_b32 s7, s1, s5
+; GFX10-NEXT:    v_and_b32_sdwa v1, v1, s2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_sub_nc_u16_e64 v0, s0, s7 clamp
+; GFX10-NEXT:    s_lshr_b32 s0, s1, 16
 ; GFX10-NEXT:    s_lshr_b32 s1, s1, 24
-; GFX10-NEXT:    s_cmp_lt_u32 s9, s8
+; GFX10-NEXT:    s_lshl_b32 s3, s3, s5
+; GFX10-NEXT:    s_lshl_b32 s0, s0, s5
+; GFX10-NEXT:    s_lshl_b32 s4, s4, s5
+; GFX10-NEXT:    s_lshl_b32 s1, s1, s5
+; GFX10-NEXT:    v_lshrrev_b16_e64 v0, 8, v0
+; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX10-NEXT:    v_sub_nc_u16_e64 v2, s3, s0 clamp
+; GFX10-NEXT:    v_sub_nc_u16_e64 v3, s4, s1 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cselect_b32 s8, s9, s8
-; GFX10-NEXT:    s_lshl_b32 s5, s5, s6
-; GFX10-NEXT:    s_sub_i32 s0, s0, s8
-; GFX10-NEXT:    s_lshl_b32 s2, s2, s6
-; GFX10-NEXT:    s_bfe_u32 s0, s0, 0x100000
-; GFX10-NEXT:    s_bfe_u32 s8, s2, 0x100000
-; GFX10-NEXT:    s_bfe_u32 s5, s5, 0x100000
-; GFX10-NEXT:    s_lshr_b32 s0, s0, s6
-; GFX10-NEXT:    s_cmp_lt_u32 s8, s5
-; GFX10-NEXT:    s_cselect_b32 s5, s8, s5
-; GFX10-NEXT:    s_lshl_b32 s3, s3, s6
-; GFX10-NEXT:    s_sub_i32 s2, s2, s5
-; GFX10-NEXT:    s_lshl_b32 s5, s7, s6
-; GFX10-NEXT:    s_bfe_u32 s2, s2, 0x100000
-; GFX10-NEXT:    s_bfe_u32 s7, s3, 0x100000
-; GFX10-NEXT:    s_bfe_u32 s5, s5, 0x100000
-; GFX10-NEXT:    s_lshr_b32 s2, s2, s6
-; GFX10-NEXT:    s_cmp_lt_u32 s7, s5
-; GFX10-NEXT:    s_cselect_b32 s5, s7, s5
-; GFX10-NEXT:    s_lshl_b32 s1, s1, s6
-; GFX10-NEXT:    s_sub_i32 s3, s3, s5
-; GFX10-NEXT:    s_lshl_b32 s4, s4, s6
-; GFX10-NEXT:    s_bfe_u32 s3, s3, 0x100000
-; GFX10-NEXT:    s_bfe_u32 s5, s4, 0x100000
-; GFX10-NEXT:    s_bfe_u32 s1, s1, 0x100000
-; GFX10-NEXT:    s_lshr_b32 s3, s3, s6
-; GFX10-NEXT:    s_cmp_lt_u32 s5, s1
-; GFX10-NEXT:    s_cselect_b32 s1, s5, s1
-; GFX10-NEXT:    s_sub_i32 s1, s4, s1
-; GFX10-NEXT:    s_movk_i32 s4, 0xff
-; GFX10-NEXT:    s_bfe_u32 s1, s1, 0x100000
-; GFX10-NEXT:    s_and_b32 s2, s2, s4
-; GFX10-NEXT:    s_lshr_b32 s1, s1, s6
-; GFX10-NEXT:    s_and_b32 s0, s0, s4
-; GFX10-NEXT:    s_lshl_b32 s2, s2, 8
-; GFX10-NEXT:    s_and_b32 s3, s3, s4
-; GFX10-NEXT:    s_and_b32 s1, s1, s4
-; GFX10-NEXT:    s_or_b32 s0, s0, s2
-; GFX10-NEXT:    s_lshl_b32 s2, s3, 16
-; GFX10-NEXT:    s_lshl_b32 s1, s1, 24
-; GFX10-NEXT:    s_or_b32 s0, s0, s2
-; GFX10-NEXT:    s_or_b32 s0, s0, s1
+; GFX10-NEXT:    v_and_or_b32 v0, v0, s2, v1
+; GFX10-NEXT:    v_and_b32_sdwa v1, v2, s2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_and_b32_sdwa v2, v3, s2 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_or3_b32 v0, v0, v1, v2
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
   %lhs = bitcast i32 %lhs.arg to <4 x i8>
   %rhs = bitcast i32 %rhs.arg to <4 x i8>
@@ -811,8 +666,7 @@ define i24 @v_usubsat_i24(i24 %lhs, i24 %rhs) {
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
-; GFX8-NEXT:    v_min_u32_e32 v1, v0, v1
-; GFX8-NEXT:    v_sub_u32_e32 v0, vcc, v0, v1
+; GFX8-NEXT:    v_sub_u32_e64 v0, s[4:5], v0, v1 clamp
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v0, 8, v0
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -821,8 +675,7 @@ define i24 @v_usubsat_i24(i24 %lhs, i24 %rhs) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
-; GFX9-NEXT:    v_min_u32_e32 v1, v0, v1
-; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v1
+; GFX9-NEXT:    v_sub_u32_e64 v0, v0, v1 clamp
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 8, v0
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -833,8 +686,7 @@ define i24 @v_usubsat_i24(i24 %lhs, i24 %rhs) {
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_min_u32_e32 v1, v0, v1
-; GFX10-NEXT:    v_sub_nc_u32_e32 v0, v0, v1
+; GFX10-NEXT:    v_sub_nc_u32_e64 v0, v0, v1 clamp
 ; GFX10-NEXT:    v_lshrrev_b32_e32 v0, 8, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call i24 @llvm.usub.sat.i24(i24 %lhs, i24 %rhs)
@@ -854,22 +706,22 @@ define amdgpu_ps i24 @s_usubsat_i24(i24 inreg %lhs, i24 inreg %rhs) {
 ;
 ; GFX8-LABEL: s_usubsat_i24:
 ; GFX8:       ; %bb.0:
-; GFX8-NEXT:    s_lshl_b32 s0, s0, 8
 ; GFX8-NEXT:    s_lshl_b32 s1, s1, 8
-; GFX8-NEXT:    s_cmp_lt_u32 s0, s1
-; GFX8-NEXT:    s_cselect_b32 s1, s0, s1
-; GFX8-NEXT:    s_sub_i32 s0, s0, s1
-; GFX8-NEXT:    s_lshr_b32 s0, s0, 8
+; GFX8-NEXT:    s_lshl_b32 s0, s0, 8
+; GFX8-NEXT:    v_mov_b32_e32 v0, s1
+; GFX8-NEXT:    v_sub_u32_e64 v0, s[0:1], s0, v0 clamp
+; GFX8-NEXT:    v_lshrrev_b32_e32 v0, 8, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: s_usubsat_i24:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_lshl_b32 s0, s0, 8
 ; GFX9-NEXT:    s_lshl_b32 s1, s1, 8
-; GFX9-NEXT:    s_cmp_lt_u32 s0, s1
-; GFX9-NEXT:    s_cselect_b32 s1, s0, s1
-; GFX9-NEXT:    s_sub_i32 s0, s0, s1
-; GFX9-NEXT:    s_lshr_b32 s0, s0, 8
+; GFX9-NEXT:    s_lshl_b32 s0, s0, 8
+; GFX9-NEXT:    v_mov_b32_e32 v0, s1
+; GFX9-NEXT:    v_sub_u32_e64 v0, s0, v0 clamp
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 8, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_usubsat_i24:
@@ -877,10 +729,9 @@ define amdgpu_ps i24 @s_usubsat_i24(i24 inreg %lhs, i24 inreg %rhs) {
 ; GFX10-NEXT:    s_lshl_b32 s0, s0, 8
 ; GFX10-NEXT:    s_lshl_b32 s1, s1, 8
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cmp_lt_u32 s0, s1
-; GFX10-NEXT:    s_cselect_b32 s1, s0, s1
-; GFX10-NEXT:    s_sub_i32 s0, s0, s1
-; GFX10-NEXT:    s_lshr_b32 s0, s0, 8
+; GFX10-NEXT:    v_sub_nc_u32_e64 v0, s0, s1 clamp
+; GFX10-NEXT:    v_lshrrev_b32_e32 v0, 8, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call i24 @llvm.usub.sat.i24(i24 %lhs, i24 %rhs)
   ret i24 %result
@@ -897,24 +748,21 @@ define i32 @v_usubsat_i32(i32 %lhs, i32 %rhs) {
 ; GFX8-LABEL: v_usubsat_i32:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_min_u32_e32 v1, v0, v1
-; GFX8-NEXT:    v_sub_u32_e32 v0, vcc, v0, v1
+; GFX8-NEXT:    v_sub_u32_e64 v0, s[4:5], v0, v1 clamp
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_usubsat_i32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_min_u32_e32 v1, v0, v1
-; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v1
+; GFX9-NEXT:    v_sub_u32_e64 v0, v0, v1 clamp
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_usubsat_i32:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_min_u32_e32 v1, v0, v1
+; GFX10-NEXT:    v_sub_nc_u32_e64 v0, v0, v1 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_sub_nc_u32_e32 v0, v0, v1
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call i32 @llvm.usub.sat.i32(i32 %lhs, i32 %rhs)
   ret i32 %result
@@ -930,24 +778,23 @@ define amdgpu_ps i32 @s_usubsat_i32(i32 inreg %lhs, i32 inreg %rhs) {
 ;
 ; GFX8-LABEL: s_usubsat_i32:
 ; GFX8:       ; %bb.0:
-; GFX8-NEXT:    s_cmp_lt_u32 s0, s1
-; GFX8-NEXT:    s_cselect_b32 s1, s0, s1
-; GFX8-NEXT:    s_sub_i32 s0, s0, s1
+; GFX8-NEXT:    v_mov_b32_e32 v0, s1
+; GFX8-NEXT:    v_sub_u32_e64 v0, s[0:1], s0, v0 clamp
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: s_usubsat_i32:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_cmp_lt_u32 s0, s1
-; GFX9-NEXT:    s_cselect_b32 s1, s0, s1
-; GFX9-NEXT:    s_sub_i32 s0, s0, s1
+; GFX9-NEXT:    v_mov_b32_e32 v0, s1
+; GFX9-NEXT:    v_sub_u32_e64 v0, s0, v0 clamp
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_usubsat_i32:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_cmp_lt_u32 s0, s1
+; GFX10-NEXT:    v_sub_nc_u32_e64 v0, s0, s1 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cselect_b32 s1, s0, s1
-; GFX10-NEXT:    s_sub_i32 s0, s0, s1
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call i32 @llvm.usub.sat.i32(i32 %lhs, i32 %rhs)
   ret i32 %result
@@ -962,21 +809,18 @@ define amdgpu_ps float @usubsat_i32_sv(i32 inreg %lhs, i32 %rhs) {
 ;
 ; GFX8-LABEL: usubsat_i32_sv:
 ; GFX8:       ; %bb.0:
-; GFX8-NEXT:    v_min_u32_e32 v0, s0, v0
-; GFX8-NEXT:    v_sub_u32_e32 v0, vcc, s0, v0
+; GFX8-NEXT:    v_sub_u32_e64 v0, s[0:1], s0, v0 clamp
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: usubsat_i32_sv:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    v_min_u32_e32 v0, s0, v0
-; GFX9-NEXT:    v_sub_u32_e32 v0, s0, v0
+; GFX9-NEXT:    v_sub_u32_e64 v0, s0, v0 clamp
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: usubsat_i32_sv:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    v_min_u32_e32 v0, s0, v0
+; GFX10-NEXT:    v_sub_nc_u32_e64 v0, s0, v0 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_sub_nc_u32_e32 v0, s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call i32 @llvm.usub.sat.i32(i32 %lhs, i32 %rhs)
   %cast = bitcast i32 %result to float
@@ -992,21 +836,18 @@ define amdgpu_ps float @usubsat_i32_vs(i32 %lhs, i32 inreg %rhs) {
 ;
 ; GFX8-LABEL: usubsat_i32_vs:
 ; GFX8:       ; %bb.0:
-; GFX8-NEXT:    v_min_u32_e32 v1, s0, v0
-; GFX8-NEXT:    v_sub_u32_e32 v0, vcc, v0, v1
+; GFX8-NEXT:    v_sub_u32_e64 v0, s[0:1], v0, s0 clamp
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: usubsat_i32_vs:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    v_min_u32_e32 v1, s0, v0
-; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v1
+; GFX9-NEXT:    v_sub_u32_e64 v0, v0, s0 clamp
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: usubsat_i32_vs:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    v_min_u32_e32 v1, s0, v0
+; GFX10-NEXT:    v_sub_nc_u32_e64 v0, v0, s0 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_sub_nc_u32_e32 v0, v0, v1
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call i32 @llvm.usub.sat.i32(i32 %lhs, i32 %rhs)
   %cast = bitcast i32 %result to float
@@ -1026,30 +867,24 @@ define <2 x i32> @v_usubsat_v2i32(<2 x i32> %lhs, <2 x i32> %rhs) {
 ; GFX8-LABEL: v_usubsat_v2i32:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_min_u32_e32 v2, v0, v2
-; GFX8-NEXT:    v_sub_u32_e32 v0, vcc, v0, v2
-; GFX8-NEXT:    v_min_u32_e32 v2, v1, v3
-; GFX8-NEXT:    v_sub_u32_e32 v1, vcc, v1, v2
+; GFX8-NEXT:    v_sub_u32_e64 v0, s[4:5], v0, v2 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v1, s[4:5], v1, v3 clamp
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_usubsat_v2i32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_min_u32_e32 v2, v0, v2
-; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v2
-; GFX9-NEXT:    v_min_u32_e32 v2, v1, v3
-; GFX9-NEXT:    v_sub_u32_e32 v1, v1, v2
+; GFX9-NEXT:    v_sub_u32_e64 v0, v0, v2 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v1, v1, v3 clamp
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_usubsat_v2i32:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_min_u32_e32 v2, v0, v2
-; GFX10-NEXT:    v_min_u32_e32 v3, v1, v3
+; GFX10-NEXT:    v_sub_nc_u32_e64 v0, v0, v2 clamp
+; GFX10-NEXT:    v_sub_nc_u32_e64 v1, v1, v3 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_sub_nc_u32_e32 v0, v0, v2
-; GFX10-NEXT:    v_sub_nc_u32_e32 v1, v1, v3
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <2 x i32> @llvm.usub.sat.v2i32(<2 x i32> %lhs, <2 x i32> %rhs)
   ret <2 x i32> %result
@@ -1068,33 +903,31 @@ define amdgpu_ps <2 x i32> @s_usubsat_v2i32(<2 x i32> inreg %lhs, <2 x i32> inre
 ;
 ; GFX8-LABEL: s_usubsat_v2i32:
 ; GFX8:       ; %bb.0:
-; GFX8-NEXT:    s_cmp_lt_u32 s0, s2
-; GFX8-NEXT:    s_cselect_b32 s2, s0, s2
-; GFX8-NEXT:    s_sub_i32 s0, s0, s2
-; GFX8-NEXT:    s_cmp_lt_u32 s1, s3
-; GFX8-NEXT:    s_cselect_b32 s2, s1, s3
-; GFX8-NEXT:    s_sub_i32 s1, s1, s2
+; GFX8-NEXT:    v_mov_b32_e32 v0, s2
+; GFX8-NEXT:    v_mov_b32_e32 v1, s3
+; GFX8-NEXT:    v_sub_u32_e64 v0, s[4:5], s0, v0 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v1, s[0:1], s1, v1 clamp
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: s_usubsat_v2i32:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_cmp_lt_u32 s0, s2
-; GFX9-NEXT:    s_cselect_b32 s2, s0, s2
-; GFX9-NEXT:    s_sub_i32 s0, s0, s2
-; GFX9-NEXT:    s_cmp_lt_u32 s1, s3
-; GFX9-NEXT:    s_cselect_b32 s2, s1, s3
-; GFX9-NEXT:    s_sub_i32 s1, s1, s2
+; GFX9-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    v_sub_u32_e64 v0, s0, v0 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v1, s1, v1 clamp
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_usubsat_v2i32:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_cmp_lt_u32 s0, s2
+; GFX10-NEXT:    v_sub_nc_u32_e64 v0, s0, s2 clamp
+; GFX10-NEXT:    v_sub_nc_u32_e64 v1, s1, s3 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cselect_b32 s2, s0, s2
-; GFX10-NEXT:    s_sub_i32 s0, s0, s2
-; GFX10-NEXT:    s_cmp_lt_u32 s1, s3
-; GFX10-NEXT:    s_cselect_b32 s2, s1, s3
-; GFX10-NEXT:    s_sub_i32 s1, s1, s2
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <2 x i32> @llvm.usub.sat.v2i32(<2 x i32> %lhs, <2 x i32> %rhs)
   ret <2 x i32> %result
@@ -1115,36 +948,27 @@ define <3 x i32> @v_usubsat_v3i32(<3 x i32> %lhs, <3 x i32> %rhs) {
 ; GFX8-LABEL: v_usubsat_v3i32:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_min_u32_e32 v3, v0, v3
-; GFX8-NEXT:    v_sub_u32_e32 v0, vcc, v0, v3
-; GFX8-NEXT:    v_min_u32_e32 v3, v1, v4
-; GFX8-NEXT:    v_sub_u32_e32 v1, vcc, v1, v3
-; GFX8-NEXT:    v_min_u32_e32 v3, v2, v5
-; GFX8-NEXT:    v_sub_u32_e32 v2, vcc, v2, v3
+; GFX8-NEXT:    v_sub_u32_e64 v0, s[4:5], v0, v3 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v1, s[4:5], v1, v4 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v2, s[4:5], v2, v5 clamp
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_usubsat_v3i32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_min_u32_e32 v3, v0, v3
-; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v3
-; GFX9-NEXT:    v_min_u32_e32 v3, v1, v4
-; GFX9-NEXT:    v_sub_u32_e32 v1, v1, v3
-; GFX9-NEXT:    v_min_u32_e32 v3, v2, v5
-; GFX9-NEXT:    v_sub_u32_e32 v2, v2, v3
+; GFX9-NEXT:    v_sub_u32_e64 v0, v0, v3 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v1, v1, v4 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v2, v2, v5 clamp
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_usubsat_v3i32:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_min_u32_e32 v3, v0, v3
-; GFX10-NEXT:    v_min_u32_e32 v4, v1, v4
-; GFX10-NEXT:    v_min_u32_e32 v5, v2, v5
+; GFX10-NEXT:    v_sub_nc_u32_e64 v0, v0, v3 clamp
+; GFX10-NEXT:    v_sub_nc_u32_e64 v1, v1, v4 clamp
+; GFX10-NEXT:    v_sub_nc_u32_e64 v2, v2, v5 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_sub_nc_u32_e32 v0, v0, v3
-; GFX10-NEXT:    v_sub_nc_u32_e32 v1, v1, v4
-; GFX10-NEXT:    v_sub_nc_u32_e32 v2, v2, v5
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <3 x i32> @llvm.usub.sat.v3i32(<3 x i32> %lhs, <3 x i32> %rhs)
   ret <3 x i32> %result
@@ -1166,42 +990,39 @@ define amdgpu_ps <3 x i32> @s_usubsat_v3i32(<3 x i32> inreg %lhs, <3 x i32> inre
 ;
 ; GFX8-LABEL: s_usubsat_v3i32:
 ; GFX8:       ; %bb.0:
-; GFX8-NEXT:    s_cmp_lt_u32 s0, s3
-; GFX8-NEXT:    s_cselect_b32 s3, s0, s3
-; GFX8-NEXT:    s_sub_i32 s0, s0, s3
-; GFX8-NEXT:    s_cmp_lt_u32 s1, s4
-; GFX8-NEXT:    s_cselect_b32 s3, s1, s4
-; GFX8-NEXT:    s_sub_i32 s1, s1, s3
-; GFX8-NEXT:    s_cmp_lt_u32 s2, s5
-; GFX8-NEXT:    s_cselect_b32 s3, s2, s5
-; GFX8-NEXT:    s_sub_i32 s2, s2, s3
+; GFX8-NEXT:    v_mov_b32_e32 v0, s3
+; GFX8-NEXT:    v_mov_b32_e32 v1, s4
+; GFX8-NEXT:    v_mov_b32_e32 v2, s5
+; GFX8-NEXT:    v_sub_u32_e64 v0, s[6:7], s0, v0 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v1, s[0:1], s1, v1 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v2, s[0:1], s2, v2 clamp
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX8-NEXT:    v_readfirstlane_b32 s2, v2
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: s_usubsat_v3i32:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_cmp_lt_u32 s0, s3
-; GFX9-NEXT:    s_cselect_b32 s3, s0, s3
-; GFX9-NEXT:    s_sub_i32 s0, s0, s3
-; GFX9-NEXT:    s_cmp_lt_u32 s1, s4
-; GFX9-NEXT:    s_cselect_b32 s3, s1, s4
-; GFX9-NEXT:    s_sub_i32 s1, s1, s3
-; GFX9-NEXT:    s_cmp_lt_u32 s2, s5
-; GFX9-NEXT:    s_cselect_b32 s3, s2, s5
-; GFX9-NEXT:    s_sub_i32 s2, s2, s3
+; GFX9-NEXT:    v_mov_b32_e32 v0, s3
+; GFX9-NEXT:    v_mov_b32_e32 v1, s4
+; GFX9-NEXT:    v_mov_b32_e32 v2, s5
+; GFX9-NEXT:    v_sub_u32_e64 v0, s0, v0 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v1, s1, v1 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v2, s2, v2 clamp
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v2
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_usubsat_v3i32:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_cmp_lt_u32 s0, s3
+; GFX10-NEXT:    v_sub_nc_u32_e64 v0, s0, s3 clamp
+; GFX10-NEXT:    v_sub_nc_u32_e64 v1, s1, s4 clamp
+; GFX10-NEXT:    v_sub_nc_u32_e64 v2, s2, s5 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cselect_b32 s3, s0, s3
-; GFX10-NEXT:    s_sub_i32 s0, s0, s3
-; GFX10-NEXT:    s_cmp_lt_u32 s1, s4
-; GFX10-NEXT:    s_cselect_b32 s3, s1, s4
-; GFX10-NEXT:    s_sub_i32 s1, s1, s3
-; GFX10-NEXT:    s_cmp_lt_u32 s2, s5
-; GFX10-NEXT:    s_cselect_b32 s3, s2, s5
-; GFX10-NEXT:    s_sub_i32 s2, s2, s3
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <3 x i32> @llvm.usub.sat.v3i32(<3 x i32> %lhs, <3 x i32> %rhs)
   ret <3 x i32> %result
@@ -1224,42 +1045,30 @@ define <4 x i32> @v_usubsat_v4i32(<4 x i32> %lhs, <4 x i32> %rhs) {
 ; GFX8-LABEL: v_usubsat_v4i32:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_min_u32_e32 v4, v0, v4
-; GFX8-NEXT:    v_sub_u32_e32 v0, vcc, v0, v4
-; GFX8-NEXT:    v_min_u32_e32 v4, v1, v5
-; GFX8-NEXT:    v_sub_u32_e32 v1, vcc, v1, v4
-; GFX8-NEXT:    v_min_u32_e32 v4, v2, v6
-; GFX8-NEXT:    v_sub_u32_e32 v2, vcc, v2, v4
-; GFX8-NEXT:    v_min_u32_e32 v4, v3, v7
-; GFX8-NEXT:    v_sub_u32_e32 v3, vcc, v3, v4
+; GFX8-NEXT:    v_sub_u32_e64 v0, s[4:5], v0, v4 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v1, s[4:5], v1, v5 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v2, s[4:5], v2, v6 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v3, s[4:5], v3, v7 clamp
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_usubsat_v4i32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_min_u32_e32 v4, v0, v4
-; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v4
-; GFX9-NEXT:    v_min_u32_e32 v4, v1, v5
-; GFX9-NEXT:    v_sub_u32_e32 v1, v1, v4
-; GFX9-NEXT:    v_min_u32_e32 v4, v2, v6
-; GFX9-NEXT:    v_sub_u32_e32 v2, v2, v4
-; GFX9-NEXT:    v_min_u32_e32 v4, v3, v7
-; GFX9-NEXT:    v_sub_u32_e32 v3, v3, v4
+; GFX9-NEXT:    v_sub_u32_e64 v0, v0, v4 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v1, v1, v5 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v2, v2, v6 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v3, v3, v7 clamp
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_usubsat_v4i32:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_min_u32_e32 v19, v2, v6
-; GFX10-NEXT:    v_min_u32_e32 v11, v0, v4
-; GFX10-NEXT:    v_min_u32_e32 v15, v1, v5
-; GFX10-NEXT:    v_min_u32_e32 v6, v3, v7
+; GFX10-NEXT:    v_sub_nc_u32_e64 v0, v0, v4 clamp
+; GFX10-NEXT:    v_sub_nc_u32_e64 v1, v1, v5 clamp
+; GFX10-NEXT:    v_sub_nc_u32_e64 v2, v2, v6 clamp
+; GFX10-NEXT:    v_sub_nc_u32_e64 v3, v3, v7 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_sub_nc_u32_e32 v2, v2, v19
-; GFX10-NEXT:    v_sub_nc_u32_e32 v0, v0, v11
-; GFX10-NEXT:    v_sub_nc_u32_e32 v1, v1, v15
-; GFX10-NEXT:    v_sub_nc_u32_e32 v3, v3, v6
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> %lhs, <4 x i32> %rhs)
   ret <4 x i32> %result
@@ -1284,51 +1093,47 @@ define amdgpu_ps <4 x i32> @s_usubsat_v4i32(<4 x i32> inreg %lhs, <4 x i32> inre
 ;
 ; GFX8-LABEL: s_usubsat_v4i32:
 ; GFX8:       ; %bb.0:
-; GFX8-NEXT:    s_cmp_lt_u32 s0, s4
-; GFX8-NEXT:    s_cselect_b32 s4, s0, s4
-; GFX8-NEXT:    s_sub_i32 s0, s0, s4
-; GFX8-NEXT:    s_cmp_lt_u32 s1, s5
-; GFX8-NEXT:    s_cselect_b32 s4, s1, s5
-; GFX8-NEXT:    s_sub_i32 s1, s1, s4
-; GFX8-NEXT:    s_cmp_lt_u32 s2, s6
-; GFX8-NEXT:    s_cselect_b32 s4, s2, s6
-; GFX8-NEXT:    s_sub_i32 s2, s2, s4
-; GFX8-NEXT:    s_cmp_lt_u32 s3, s7
-; GFX8-NEXT:    s_cselect_b32 s4, s3, s7
-; GFX8-NEXT:    s_sub_i32 s3, s3, s4
+; GFX8-NEXT:    v_mov_b32_e32 v0, s4
+; GFX8-NEXT:    v_mov_b32_e32 v1, s5
+; GFX8-NEXT:    v_mov_b32_e32 v2, s6
+; GFX8-NEXT:    v_mov_b32_e32 v3, s7
+; GFX8-NEXT:    v_sub_u32_e64 v0, s[8:9], s0, v0 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v1, s[0:1], s1, v1 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v2, s[0:1], s2, v2 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v3, s[0:1], s3, v3 clamp
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX8-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX8-NEXT:    v_readfirstlane_b32 s3, v3
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: s_usubsat_v4i32:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_cmp_lt_u32 s0, s4
-; GFX9-NEXT:    s_cselect_b32 s4, s0, s4
-; GFX9-NEXT:    s_sub_i32 s0, s0, s4
-; GFX9-NEXT:    s_cmp_lt_u32 s1, s5
-; GFX9-NEXT:    s_cselect_b32 s4, s1, s5
-; GFX9-NEXT:    s_sub_i32 s1, s1, s4
-; GFX9-NEXT:    s_cmp_lt_u32 s2, s6
-; GFX9-NEXT:    s_cselect_b32 s4, s2, s6
-; GFX9-NEXT:    s_sub_i32 s2, s2, s4
-; GFX9-NEXT:    s_cmp_lt_u32 s3, s7
-; GFX9-NEXT:    s_cselect_b32 s4, s3, s7
-; GFX9-NEXT:    s_sub_i32 s3, s3, s4
+; GFX9-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-NEXT:    v_mov_b32_e32 v2, s6
+; GFX9-NEXT:    v_mov_b32_e32 v3, s7
+; GFX9-NEXT:    v_sub_u32_e64 v0, s0, v0 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v1, s1, v1 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v2, s2, v2 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v3, s3, v3 clamp
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX9-NEXT:    v_readfirstlane_b32 s3, v3
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_usubsat_v4i32:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_cmp_lt_u32 s0, s4
+; GFX10-NEXT:    v_sub_nc_u32_e64 v0, s0, s4 clamp
+; GFX10-NEXT:    v_sub_nc_u32_e64 v1, s1, s5 clamp
+; GFX10-NEXT:    v_sub_nc_u32_e64 v2, s2, s6 clamp
+; GFX10-NEXT:    v_sub_nc_u32_e64 v3, s3, s7 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cselect_b32 s4, s0, s4
-; GFX10-NEXT:    s_sub_i32 s0, s0, s4
-; GFX10-NEXT:    s_cmp_lt_u32 s1, s5
-; GFX10-NEXT:    s_cselect_b32 s4, s1, s5
-; GFX10-NEXT:    s_sub_i32 s1, s1, s4
-; GFX10-NEXT:    s_cmp_lt_u32 s2, s6
-; GFX10-NEXT:    s_cselect_b32 s4, s2, s6
-; GFX10-NEXT:    s_sub_i32 s2, s2, s4
-; GFX10-NEXT:    s_cmp_lt_u32 s3, s7
-; GFX10-NEXT:    s_cselect_b32 s4, s3, s7
-; GFX10-NEXT:    s_sub_i32 s3, s3, s4
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX10-NEXT:    v_readfirstlane_b32 s3, v3
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> %lhs, <4 x i32> %rhs)
   ret <4 x i32> %result
@@ -1353,47 +1158,32 @@ define <5 x i32> @v_usubsat_v5i32(<5 x i32> %lhs, <5 x i32> %rhs) {
 ; GFX8-LABEL: v_usubsat_v5i32:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_min_u32_e32 v5, v0, v5
-; GFX8-NEXT:    v_sub_u32_e32 v0, vcc, v0, v5
-; GFX8-NEXT:    v_min_u32_e32 v5, v1, v6
-; GFX8-NEXT:    v_sub_u32_e32 v1, vcc, v1, v5
-; GFX8-NEXT:    v_min_u32_e32 v5, v2, v7
-; GFX8-NEXT:    v_sub_u32_e32 v2, vcc, v2, v5
-; GFX8-NEXT:    v_min_u32_e32 v5, v3, v8
-; GFX8-NEXT:    v_sub_u32_e32 v3, vcc, v3, v5
-; GFX8-NEXT:    v_min_u32_e32 v5, v4, v9
-; GFX8-NEXT:    v_sub_u32_e32 v4, vcc, v4, v5
+; GFX8-NEXT:    v_sub_u32_e64 v0, s[4:5], v0, v5 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v1, s[4:5], v1, v6 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v2, s[4:5], v2, v7 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v3, s[4:5], v3, v8 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v4, s[4:5], v4, v9 clamp
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_usubsat_v5i32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_min_u32_e32 v5, v0, v5
-; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v5
-; GFX9-NEXT:    v_min_u32_e32 v5, v1, v6
-; GFX9-NEXT:    v_sub_u32_e32 v1, v1, v5
-; GFX9-NEXT:    v_min_u32_e32 v5, v2, v7
-; GFX9-NEXT:    v_sub_u32_e32 v2, v2, v5
-; GFX9-NEXT:    v_min_u32_e32 v5, v3, v8
-; GFX9-NEXT:    v_sub_u32_e32 v3, v3, v5
-; GFX9-NEXT:    v_min_u32_e32 v5, v4, v9
-; GFX9-NEXT:    v_sub_u32_e32 v4, v4, v5
+; GFX9-NEXT:    v_sub_u32_e64 v0, v0, v5 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v1, v1, v6 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v2, v2, v7 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v3, v3, v8 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v4, v4, v9 clamp
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_usubsat_v5i32:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_min_u32_e32 v5, v0, v5
-; GFX10-NEXT:    v_min_u32_e32 v6, v1, v6
-; GFX10-NEXT:    v_min_u32_e32 v7, v2, v7
-; GFX10-NEXT:    v_min_u32_e32 v8, v3, v8
-; GFX10-NEXT:    v_min_u32_e32 v9, v4, v9
-; GFX10-NEXT:    v_sub_nc_u32_e32 v0, v0, v5
-; GFX10-NEXT:    v_sub_nc_u32_e32 v1, v1, v6
-; GFX10-NEXT:    v_sub_nc_u32_e32 v2, v2, v7
-; GFX10-NEXT:    v_sub_nc_u32_e32 v3, v3, v8
-; GFX10-NEXT:    v_sub_nc_u32_e32 v4, v4, v9
+; GFX10-NEXT:    v_sub_nc_u32_e64 v0, v0, v5 clamp
+; GFX10-NEXT:    v_sub_nc_u32_e64 v1, v1, v6 clamp
+; GFX10-NEXT:    v_sub_nc_u32_e64 v2, v2, v7 clamp
+; GFX10-NEXT:    v_sub_nc_u32_e64 v3, v3, v8 clamp
+; GFX10-NEXT:    v_sub_nc_u32_e64 v4, v4, v9 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <5 x i32> @llvm.usub.sat.v5i32(<5 x i32> %lhs, <5 x i32> %rhs)
@@ -1422,60 +1212,55 @@ define amdgpu_ps <5 x i32> @s_usubsat_v5i32(<5 x i32> inreg %lhs, <5 x i32> inre
 ;
 ; GFX8-LABEL: s_usubsat_v5i32:
 ; GFX8:       ; %bb.0:
-; GFX8-NEXT:    s_cmp_lt_u32 s0, s5
-; GFX8-NEXT:    s_cselect_b32 s5, s0, s5
-; GFX8-NEXT:    s_sub_i32 s0, s0, s5
-; GFX8-NEXT:    s_cmp_lt_u32 s1, s6
-; GFX8-NEXT:    s_cselect_b32 s5, s1, s6
-; GFX8-NEXT:    s_sub_i32 s1, s1, s5
-; GFX8-NEXT:    s_cmp_lt_u32 s2, s7
-; GFX8-NEXT:    s_cselect_b32 s5, s2, s7
-; GFX8-NEXT:    s_sub_i32 s2, s2, s5
-; GFX8-NEXT:    s_cmp_lt_u32 s3, s8
-; GFX8-NEXT:    s_cselect_b32 s5, s3, s8
-; GFX8-NEXT:    s_sub_i32 s3, s3, s5
-; GFX8-NEXT:    s_cmp_lt_u32 s4, s9
-; GFX8-NEXT:    s_cselect_b32 s5, s4, s9
-; GFX8-NEXT:    s_sub_i32 s4, s4, s5
+; GFX8-NEXT:    v_mov_b32_e32 v0, s5
+; GFX8-NEXT:    v_mov_b32_e32 v1, s6
+; GFX8-NEXT:    v_mov_b32_e32 v2, s7
+; GFX8-NEXT:    v_mov_b32_e32 v3, s8
+; GFX8-NEXT:    v_mov_b32_e32 v4, s9
+; GFX8-NEXT:    v_sub_u32_e64 v0, s[10:11], s0, v0 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v1, s[0:1], s1, v1 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v2, s[0:1], s2, v2 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v3, s[0:1], s3, v3 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v4, s[0:1], s4, v4 clamp
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX8-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX8-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX8-NEXT:    v_readfirstlane_b32 s4, v4
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: s_usubsat_v5i32:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_cmp_lt_u32 s0, s5
-; GFX9-NEXT:    s_cselect_b32 s5, s0, s5
-; GFX9-NEXT:    s_sub_i32 s0, s0, s5
-; GFX9-NEXT:    s_cmp_lt_u32 s1, s6
-; GFX9-NEXT:    s_cselect_b32 s5, s1, s6
-; GFX9-NEXT:    s_sub_i32 s1, s1, s5
-; GFX9-NEXT:    s_cmp_lt_u32 s2, s7
-; GFX9-NEXT:    s_cselect_b32 s5, s2, s7
-; GFX9-NEXT:    s_sub_i32 s2, s2, s5
-; GFX9-NEXT:    s_cmp_lt_u32 s3, s8
-; GFX9-NEXT:    s_cselect_b32 s5, s3, s8
-; GFX9-NEXT:    s_sub_i32 s3, s3, s5
-; GFX9-NEXT:    s_cmp_lt_u32 s4, s9
-; GFX9-NEXT:    s_cselect_b32 s5, s4, s9
-; GFX9-NEXT:    s_sub_i32 s4, s4, s5
+; GFX9-NEXT:    v_mov_b32_e32 v0, s5
+; GFX9-NEXT:    v_mov_b32_e32 v1, s6
+; GFX9-NEXT:    v_mov_b32_e32 v2, s7
+; GFX9-NEXT:    v_mov_b32_e32 v3, s8
+; GFX9-NEXT:    v_mov_b32_e32 v4, s9
+; GFX9-NEXT:    v_sub_u32_e64 v0, s0, v0 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v1, s1, v1 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v2, s2, v2 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v3, s3, v3 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v4, s4, v4 clamp
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX9-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX9-NEXT:    v_readfirstlane_b32 s4, v4
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_usubsat_v5i32:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_cmp_lt_u32 s0, s5
+; GFX10-NEXT:    v_sub_nc_u32_e64 v0, s0, s5 clamp
+; GFX10-NEXT:    v_sub_nc_u32_e64 v1, s1, s6 clamp
+; GFX10-NEXT:    v_sub_nc_u32_e64 v2, s2, s7 clamp
+; GFX10-NEXT:    v_sub_nc_u32_e64 v3, s3, s8 clamp
+; GFX10-NEXT:    v_sub_nc_u32_e64 v4, s4, s9 clamp
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX10-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX10-NEXT:    v_readfirstlane_b32 s4, v4
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cselect_b32 s5, s0, s5
-; GFX10-NEXT:    s_sub_i32 s0, s0, s5
-; GFX10-NEXT:    s_cmp_lt_u32 s1, s6
-; GFX10-NEXT:    s_cselect_b32 s5, s1, s6
-; GFX10-NEXT:    s_sub_i32 s1, s1, s5
-; GFX10-NEXT:    s_cmp_lt_u32 s2, s7
-; GFX10-NEXT:    s_cselect_b32 s5, s2, s7
-; GFX10-NEXT:    s_sub_i32 s2, s2, s5
-; GFX10-NEXT:    s_cmp_lt_u32 s3, s8
-; GFX10-NEXT:    s_cselect_b32 s5, s3, s8
-; GFX10-NEXT:    s_sub_i32 s3, s3, s5
-; GFX10-NEXT:    s_cmp_lt_u32 s4, s9
-; GFX10-NEXT:    s_cselect_b32 s5, s4, s9
-; GFX10-NEXT:    s_sub_i32 s4, s4, s5
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <5 x i32> @llvm.usub.sat.v5i32(<5 x i32> %lhs, <5 x i32> %rhs)
   ret <5 x i32> %result
@@ -1522,113 +1307,65 @@ define <16 x i32> @v_usubsat_v16i32(<16 x i32> %lhs, <16 x i32> %rhs) {
 ; GFX8-LABEL: v_usubsat_v16i32:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_min_u32_e32 v16, v0, v16
-; GFX8-NEXT:    v_sub_u32_e32 v0, vcc, v0, v16
-; GFX8-NEXT:    v_min_u32_e32 v16, v1, v17
-; GFX8-NEXT:    v_sub_u32_e32 v1, vcc, v1, v16
-; GFX8-NEXT:    v_min_u32_e32 v16, v2, v18
-; GFX8-NEXT:    v_sub_u32_e32 v2, vcc, v2, v16
-; GFX8-NEXT:    v_min_u32_e32 v16, v3, v19
-; GFX8-NEXT:    v_sub_u32_e32 v3, vcc, v3, v16
-; GFX8-NEXT:    v_min_u32_e32 v16, v4, v20
-; GFX8-NEXT:    v_sub_u32_e32 v4, vcc, v4, v16
-; GFX8-NEXT:    v_min_u32_e32 v16, v5, v21
-; GFX8-NEXT:    v_sub_u32_e32 v5, vcc, v5, v16
-; GFX8-NEXT:    v_min_u32_e32 v16, v6, v22
-; GFX8-NEXT:    v_sub_u32_e32 v6, vcc, v6, v16
-; GFX8-NEXT:    v_min_u32_e32 v16, v7, v23
-; GFX8-NEXT:    v_sub_u32_e32 v7, vcc, v7, v16
-; GFX8-NEXT:    v_min_u32_e32 v16, v8, v24
-; GFX8-NEXT:    v_sub_u32_e32 v8, vcc, v8, v16
-; GFX8-NEXT:    v_min_u32_e32 v16, v9, v25
-; GFX8-NEXT:    v_sub_u32_e32 v9, vcc, v9, v16
-; GFX8-NEXT:    v_min_u32_e32 v16, v10, v26
-; GFX8-NEXT:    v_sub_u32_e32 v10, vcc, v10, v16
-; GFX8-NEXT:    v_min_u32_e32 v16, v11, v27
-; GFX8-NEXT:    v_sub_u32_e32 v11, vcc, v11, v16
-; GFX8-NEXT:    v_min_u32_e32 v16, v12, v28
-; GFX8-NEXT:    v_sub_u32_e32 v12, vcc, v12, v16
-; GFX8-NEXT:    v_min_u32_e32 v16, v13, v29
-; GFX8-NEXT:    v_sub_u32_e32 v13, vcc, v13, v16
-; GFX8-NEXT:    v_min_u32_e32 v16, v14, v30
-; GFX8-NEXT:    v_sub_u32_e32 v14, vcc, v14, v16
-; GFX8-NEXT:    v_min_u32_e32 v16, v15, v31
-; GFX8-NEXT:    v_sub_u32_e32 v15, vcc, v15, v16
+; GFX8-NEXT:    v_sub_u32_e64 v0, s[4:5], v0, v16 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v1, s[4:5], v1, v17 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v2, s[4:5], v2, v18 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v3, s[4:5], v3, v19 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v4, s[4:5], v4, v20 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v5, s[4:5], v5, v21 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v6, s[4:5], v6, v22 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v7, s[4:5], v7, v23 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v8, s[4:5], v8, v24 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v9, s[4:5], v9, v25 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v10, s[4:5], v10, v26 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v11, s[4:5], v11, v27 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v12, s[4:5], v12, v28 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v13, s[4:5], v13, v29 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v14, s[4:5], v14, v30 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v15, s[4:5], v15, v31 clamp
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_usubsat_v16i32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_min_u32_e32 v16, v0, v16
-; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v16
-; GFX9-NEXT:    v_min_u32_e32 v16, v1, v17
-; GFX9-NEXT:    v_sub_u32_e32 v1, v1, v16
-; GFX9-NEXT:    v_min_u32_e32 v16, v2, v18
-; GFX9-NEXT:    v_sub_u32_e32 v2, v2, v16
-; GFX9-NEXT:    v_min_u32_e32 v16, v3, v19
-; GFX9-NEXT:    v_sub_u32_e32 v3, v3, v16
-; GFX9-NEXT:    v_min_u32_e32 v16, v4, v20
-; GFX9-NEXT:    v_sub_u32_e32 v4, v4, v16
-; GFX9-NEXT:    v_min_u32_e32 v16, v5, v21
-; GFX9-NEXT:    v_sub_u32_e32 v5, v5, v16
-; GFX9-NEXT:    v_min_u32_e32 v16, v6, v22
-; GFX9-NEXT:    v_sub_u32_e32 v6, v6, v16
-; GFX9-NEXT:    v_min_u32_e32 v16, v7, v23
-; GFX9-NEXT:    v_sub_u32_e32 v7, v7, v16
-; GFX9-NEXT:    v_min_u32_e32 v16, v8, v24
-; GFX9-NEXT:    v_sub_u32_e32 v8, v8, v16
-; GFX9-NEXT:    v_min_u32_e32 v16, v9, v25
-; GFX9-NEXT:    v_sub_u32_e32 v9, v9, v16
-; GFX9-NEXT:    v_min_u32_e32 v16, v10, v26
-; GFX9-NEXT:    v_sub_u32_e32 v10, v10, v16
-; GFX9-NEXT:    v_min_u32_e32 v16, v11, v27
-; GFX9-NEXT:    v_sub_u32_e32 v11, v11, v16
-; GFX9-NEXT:    v_min_u32_e32 v16, v12, v28
-; GFX9-NEXT:    v_sub_u32_e32 v12, v12, v16
-; GFX9-NEXT:    v_min_u32_e32 v16, v13, v29
-; GFX9-NEXT:    v_sub_u32_e32 v13, v13, v16
-; GFX9-NEXT:    v_min_u32_e32 v16, v14, v30
-; GFX9-NEXT:    v_sub_u32_e32 v14, v14, v16
-; GFX9-NEXT:    v_min_u32_e32 v16, v15, v31
-; GFX9-NEXT:    v_sub_u32_e32 v15, v15, v16
+; GFX9-NEXT:    v_sub_u32_e64 v0, v0, v16 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v1, v1, v17 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v2, v2, v18 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v3, v3, v19 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v4, v4, v20 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v5, v5, v21 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v6, v6, v22 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v7, v7, v23 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v8, v8, v24 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v9, v9, v25 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v10, v10, v26 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v11, v11, v27 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v12, v12, v28 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v13, v13, v29 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v14, v14, v30 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v15, v15, v31 clamp
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_usubsat_v16i32:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_min_u32_e32 v35, v0, v16
-; GFX10-NEXT:    v_min_u32_e32 v16, v1, v17
-; GFX10-NEXT:    v_min_u32_e32 v17, v2, v18
-; GFX10-NEXT:    v_min_u32_e32 v18, v3, v19
-; GFX10-NEXT:    v_min_u32_e32 v19, v4, v20
-; GFX10-NEXT:    v_min_u32_e32 v20, v5, v21
-; GFX10-NEXT:    v_sub_nc_u32_e32 v1, v1, v16
-; GFX10-NEXT:    v_min_u32_e32 v16, v6, v22
-; GFX10-NEXT:    v_sub_nc_u32_e32 v2, v2, v17
-; GFX10-NEXT:    v_min_u32_e32 v17, v7, v23
-; GFX10-NEXT:    v_sub_nc_u32_e32 v3, v3, v18
-; GFX10-NEXT:    v_min_u32_e32 v18, v8, v24
-; GFX10-NEXT:    v_sub_nc_u32_e32 v4, v4, v19
-; GFX10-NEXT:    v_min_u32_e32 v19, v9, v25
-; GFX10-NEXT:    v_sub_nc_u32_e32 v5, v5, v20
-; GFX10-NEXT:    v_min_u32_e32 v20, v10, v26
-; GFX10-NEXT:    v_sub_nc_u32_e32 v6, v6, v16
-; GFX10-NEXT:    v_min_u32_e32 v16, v11, v27
-; GFX10-NEXT:    v_sub_nc_u32_e32 v7, v7, v17
-; GFX10-NEXT:    v_min_u32_e32 v17, v12, v28
-; GFX10-NEXT:    v_sub_nc_u32_e32 v8, v8, v18
-; GFX10-NEXT:    v_min_u32_e32 v18, v13, v29
-; GFX10-NEXT:    v_sub_nc_u32_e32 v9, v9, v19
-; GFX10-NEXT:    v_min_u32_e32 v19, v14, v30
-; GFX10-NEXT:    v_sub_nc_u32_e32 v10, v10, v20
-; GFX10-NEXT:    v_min_u32_e32 v20, v15, v31
-; GFX10-NEXT:    v_sub_nc_u32_e32 v0, v0, v35
-; GFX10-NEXT:    v_sub_nc_u32_e32 v11, v11, v16
-; GFX10-NEXT:    v_sub_nc_u32_e32 v12, v12, v17
-; GFX10-NEXT:    v_sub_nc_u32_e32 v13, v13, v18
-; GFX10-NEXT:    v_sub_nc_u32_e32 v14, v14, v19
-; GFX10-NEXT:    v_sub_nc_u32_e32 v15, v15, v20
+; GFX10-NEXT:    v_sub_nc_u32_e64 v0, v0, v16 clamp
+; GFX10-NEXT:    v_sub_nc_u32_e64 v1, v1, v17 clamp
+; GFX10-NEXT:    v_sub_nc_u32_e64 v2, v2, v18 clamp
+; GFX10-NEXT:    v_sub_nc_u32_e64 v3, v3, v19 clamp
+; GFX10-NEXT:    v_sub_nc_u32_e64 v4, v4, v20 clamp
+; GFX10-NEXT:    v_sub_nc_u32_e64 v5, v5, v21 clamp
+; GFX10-NEXT:    v_sub_nc_u32_e64 v6, v6, v22 clamp
+; GFX10-NEXT:    v_sub_nc_u32_e64 v7, v7, v23 clamp
+; GFX10-NEXT:    v_sub_nc_u32_e64 v8, v8, v24 clamp
+; GFX10-NEXT:    v_sub_nc_u32_e64 v9, v9, v25 clamp
+; GFX10-NEXT:    v_sub_nc_u32_e64 v10, v10, v26 clamp
+; GFX10-NEXT:    v_sub_nc_u32_e64 v11, v11, v27 clamp
+; GFX10-NEXT:    v_sub_nc_u32_e64 v12, v12, v28 clamp
+; GFX10-NEXT:    v_sub_nc_u32_e64 v13, v13, v29 clamp
+; GFX10-NEXT:    v_sub_nc_u32_e64 v14, v14, v30 clamp
+; GFX10-NEXT:    v_sub_nc_u32_e64 v15, v15, v31 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <16 x i32> @llvm.usub.sat.v16i32(<16 x i32> %lhs, <16 x i32> %rhs)
@@ -1690,159 +1427,143 @@ define amdgpu_ps <16 x i32> @s_usubsat_v16i32(<16 x i32> inreg %lhs, <16 x i32>
 ;
 ; GFX8-LABEL: s_usubsat_v16i32:
 ; GFX8:       ; %bb.0:
-; GFX8-NEXT:    s_cmp_lt_u32 s0, s16
-; GFX8-NEXT:    s_cselect_b32 s16, s0, s16
-; GFX8-NEXT:    s_sub_i32 s0, s0, s16
-; GFX8-NEXT:    s_cmp_lt_u32 s1, s17
-; GFX8-NEXT:    s_cselect_b32 s16, s1, s17
-; GFX8-NEXT:    s_sub_i32 s1, s1, s16
-; GFX8-NEXT:    s_cmp_lt_u32 s2, s18
-; GFX8-NEXT:    s_cselect_b32 s16, s2, s18
-; GFX8-NEXT:    s_sub_i32 s2, s2, s16
-; GFX8-NEXT:    s_cmp_lt_u32 s3, s19
-; GFX8-NEXT:    s_cselect_b32 s16, s3, s19
-; GFX8-NEXT:    s_sub_i32 s3, s3, s16
-; GFX8-NEXT:    s_cmp_lt_u32 s4, s20
-; GFX8-NEXT:    s_cselect_b32 s16, s4, s20
-; GFX8-NEXT:    s_sub_i32 s4, s4, s16
-; GFX8-NEXT:    s_cmp_lt_u32 s5, s21
-; GFX8-NEXT:    s_cselect_b32 s16, s5, s21
-; GFX8-NEXT:    s_sub_i32 s5, s5, s16
-; GFX8-NEXT:    s_cmp_lt_u32 s6, s22
-; GFX8-NEXT:    s_cselect_b32 s16, s6, s22
-; GFX8-NEXT:    s_sub_i32 s6, s6, s16
-; GFX8-NEXT:    s_cmp_lt_u32 s7, s23
-; GFX8-NEXT:    s_cselect_b32 s16, s7, s23
-; GFX8-NEXT:    s_sub_i32 s7, s7, s16
-; GFX8-NEXT:    s_cmp_lt_u32 s8, s24
-; GFX8-NEXT:    s_cselect_b32 s16, s8, s24
-; GFX8-NEXT:    s_sub_i32 s8, s8, s16
-; GFX8-NEXT:    s_cmp_lt_u32 s9, s25
-; GFX8-NEXT:    s_cselect_b32 s16, s9, s25
-; GFX8-NEXT:    s_sub_i32 s9, s9, s16
-; GFX8-NEXT:    s_cmp_lt_u32 s10, s26
-; GFX8-NEXT:    s_cselect_b32 s16, s10, s26
-; GFX8-NEXT:    s_sub_i32 s10, s10, s16
-; GFX8-NEXT:    s_cmp_lt_u32 s11, s27
-; GFX8-NEXT:    s_cselect_b32 s16, s11, s27
-; GFX8-NEXT:    s_sub_i32 s11, s11, s16
-; GFX8-NEXT:    s_cmp_lt_u32 s12, s28
-; GFX8-NEXT:    s_cselect_b32 s16, s12, s28
-; GFX8-NEXT:    s_sub_i32 s12, s12, s16
-; GFX8-NEXT:    s_cmp_lt_u32 s13, s29
-; GFX8-NEXT:    s_cselect_b32 s16, s13, s29
-; GFX8-NEXT:    s_sub_i32 s13, s13, s16
-; GFX8-NEXT:    s_cmp_lt_u32 s14, s30
-; GFX8-NEXT:    s_cselect_b32 s16, s14, s30
-; GFX8-NEXT:    s_sub_i32 s14, s14, s16
-; GFX8-NEXT:    s_cmp_lt_u32 s15, s31
-; GFX8-NEXT:    s_cselect_b32 s16, s15, s31
-; GFX8-NEXT:    s_sub_i32 s15, s15, s16
+; GFX8-NEXT:    v_mov_b32_e32 v0, s16
+; GFX8-NEXT:    v_mov_b32_e32 v1, s17
+; GFX8-NEXT:    v_mov_b32_e32 v2, s18
+; GFX8-NEXT:    v_mov_b32_e32 v3, s19
+; GFX8-NEXT:    v_mov_b32_e32 v4, s20
+; GFX8-NEXT:    v_mov_b32_e32 v5, s21
+; GFX8-NEXT:    v_mov_b32_e32 v6, s22
+; GFX8-NEXT:    v_mov_b32_e32 v7, s23
+; GFX8-NEXT:    v_mov_b32_e32 v8, s24
+; GFX8-NEXT:    v_mov_b32_e32 v9, s25
+; GFX8-NEXT:    v_mov_b32_e32 v10, s26
+; GFX8-NEXT:    v_mov_b32_e32 v11, s27
+; GFX8-NEXT:    v_mov_b32_e32 v12, s28
+; GFX8-NEXT:    v_mov_b32_e32 v13, s29
+; GFX8-NEXT:    v_mov_b32_e32 v14, s30
+; GFX8-NEXT:    v_mov_b32_e32 v15, s31
+; GFX8-NEXT:    v_sub_u32_e64 v0, s[32:33], s0, v0 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v1, s[16:17], s1, v1 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v2, s[16:17], s2, v2 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v3, s[2:3], s3, v3 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v4, s[2:3], s4, v4 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v5, s[2:3], s5, v5 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v6, s[2:3], s6, v6 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v7, s[2:3], s7, v7 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v8, s[2:3], s8, v8 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v9, s[2:3], s9, v9 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v10, s[2:3], s10, v10 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v11, s[2:3], s11, v11 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v12, s[2:3], s12, v12 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v13, s[2:3], s13, v13 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v14, s[2:3], s14, v14 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v15, s[2:3], s15, v15 clamp
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX8-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX8-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX8-NEXT:    v_readfirstlane_b32 s4, v4
+; GFX8-NEXT:    v_readfirstlane_b32 s5, v5
+; GFX8-NEXT:    v_readfirstlane_b32 s6, v6
+; GFX8-NEXT:    v_readfirstlane_b32 s7, v7
+; GFX8-NEXT:    v_readfirstlane_b32 s8, v8
+; GFX8-NEXT:    v_readfirstlane_b32 s9, v9
+; GFX8-NEXT:    v_readfirstlane_b32 s10, v10
+; GFX8-NEXT:    v_readfirstlane_b32 s11, v11
+; GFX8-NEXT:    v_readfirstlane_b32 s12, v12
+; GFX8-NEXT:    v_readfirstlane_b32 s13, v13
+; GFX8-NEXT:    v_readfirstlane_b32 s14, v14
+; GFX8-NEXT:    v_readfirstlane_b32 s15, v15
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: s_usubsat_v16i32:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_cmp_lt_u32 s0, s16
-; GFX9-NEXT:    s_cselect_b32 s16, s0, s16
-; GFX9-NEXT:    s_sub_i32 s0, s0, s16
-; GFX9-NEXT:    s_cmp_lt_u32 s1, s17
-; GFX9-NEXT:    s_cselect_b32 s16, s1, s17
-; GFX9-NEXT:    s_sub_i32 s1, s1, s16
-; GFX9-NEXT:    s_cmp_lt_u32 s2, s18
-; GFX9-NEXT:    s_cselect_b32 s16, s2, s18
-; GFX9-NEXT:    s_sub_i32 s2, s2, s16
-; GFX9-NEXT:    s_cmp_lt_u32 s3, s19
-; GFX9-NEXT:    s_cselect_b32 s16, s3, s19
-; GFX9-NEXT:    s_sub_i32 s3, s3, s16
-; GFX9-NEXT:    s_cmp_lt_u32 s4, s20
-; GFX9-NEXT:    s_cselect_b32 s16, s4, s20
-; GFX9-NEXT:    s_sub_i32 s4, s4, s16
-; GFX9-NEXT:    s_cmp_lt_u32 s5, s21
-; GFX9-NEXT:    s_cselect_b32 s16, s5, s21
-; GFX9-NEXT:    s_sub_i32 s5, s5, s16
-; GFX9-NEXT:    s_cmp_lt_u32 s6, s22
-; GFX9-NEXT:    s_cselect_b32 s16, s6, s22
-; GFX9-NEXT:    s_sub_i32 s6, s6, s16
-; GFX9-NEXT:    s_cmp_lt_u32 s7, s23
-; GFX9-NEXT:    s_cselect_b32 s16, s7, s23
-; GFX9-NEXT:    s_sub_i32 s7, s7, s16
-; GFX9-NEXT:    s_cmp_lt_u32 s8, s24
-; GFX9-NEXT:    s_cselect_b32 s16, s8, s24
-; GFX9-NEXT:    s_sub_i32 s8, s8, s16
-; GFX9-NEXT:    s_cmp_lt_u32 s9, s25
-; GFX9-NEXT:    s_cselect_b32 s16, s9, s25
-; GFX9-NEXT:    s_sub_i32 s9, s9, s16
-; GFX9-NEXT:    s_cmp_lt_u32 s10, s26
-; GFX9-NEXT:    s_cselect_b32 s16, s10, s26
-; GFX9-NEXT:    s_sub_i32 s10, s10, s16
-; GFX9-NEXT:    s_cmp_lt_u32 s11, s27
-; GFX9-NEXT:    s_cselect_b32 s16, s11, s27
-; GFX9-NEXT:    s_sub_i32 s11, s11, s16
-; GFX9-NEXT:    s_cmp_lt_u32 s12, s28
-; GFX9-NEXT:    s_cselect_b32 s16, s12, s28
-; GFX9-NEXT:    s_sub_i32 s12, s12, s16
-; GFX9-NEXT:    s_cmp_lt_u32 s13, s29
-; GFX9-NEXT:    s_cselect_b32 s16, s13, s29
-; GFX9-NEXT:    s_sub_i32 s13, s13, s16
-; GFX9-NEXT:    s_cmp_lt_u32 s14, s30
-; GFX9-NEXT:    s_cselect_b32 s16, s14, s30
-; GFX9-NEXT:    s_sub_i32 s14, s14, s16
-; GFX9-NEXT:    s_cmp_lt_u32 s15, s31
-; GFX9-NEXT:    s_cselect_b32 s16, s15, s31
-; GFX9-NEXT:    s_sub_i32 s15, s15, s16
+; GFX9-NEXT:    v_mov_b32_e32 v0, s16
+; GFX9-NEXT:    v_mov_b32_e32 v1, s17
+; GFX9-NEXT:    v_mov_b32_e32 v2, s18
+; GFX9-NEXT:    v_mov_b32_e32 v3, s19
+; GFX9-NEXT:    v_mov_b32_e32 v4, s20
+; GFX9-NEXT:    v_mov_b32_e32 v5, s21
+; GFX9-NEXT:    v_mov_b32_e32 v6, s22
+; GFX9-NEXT:    v_mov_b32_e32 v7, s23
+; GFX9-NEXT:    v_mov_b32_e32 v8, s24
+; GFX9-NEXT:    v_mov_b32_e32 v9, s25
+; GFX9-NEXT:    v_mov_b32_e32 v10, s26
+; GFX9-NEXT:    v_mov_b32_e32 v11, s27
+; GFX9-NEXT:    v_mov_b32_e32 v12, s28
+; GFX9-NEXT:    v_mov_b32_e32 v13, s29
+; GFX9-NEXT:    v_mov_b32_e32 v14, s30
+; GFX9-NEXT:    v_mov_b32_e32 v15, s31
+; GFX9-NEXT:    v_sub_u32_e64 v0, s0, v0 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v1, s1, v1 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v2, s2, v2 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v3, s3, v3 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v4, s4, v4 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v5, s5, v5 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v6, s6, v6 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v7, s7, v7 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v8, s8, v8 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v9, s9, v9 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v10, s10, v10 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v11, s11, v11 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v12, s12, v12 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v13, s13, v13 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v14, s14, v14 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v15, s15, v15 clamp
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX9-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX9-NEXT:    v_readfirstlane_b32 s4, v4
+; GFX9-NEXT:    v_readfirstlane_b32 s5, v5
+; GFX9-NEXT:    v_readfirstlane_b32 s6, v6
+; GFX9-NEXT:    v_readfirstlane_b32 s7, v7
+; GFX9-NEXT:    v_readfirstlane_b32 s8, v8
+; GFX9-NEXT:    v_readfirstlane_b32 s9, v9
+; GFX9-NEXT:    v_readfirstlane_b32 s10, v10
+; GFX9-NEXT:    v_readfirstlane_b32 s11, v11
+; GFX9-NEXT:    v_readfirstlane_b32 s12, v12
+; GFX9-NEXT:    v_readfirstlane_b32 s13, v13
+; GFX9-NEXT:    v_readfirstlane_b32 s14, v14
+; GFX9-NEXT:    v_readfirstlane_b32 s15, v15
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_usubsat_v16i32:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_cmp_lt_u32 s0, s16
+; GFX10-NEXT:    v_sub_nc_u32_e64 v0, s0, s16 clamp
+; GFX10-NEXT:    v_sub_nc_u32_e64 v1, s1, s17 clamp
+; GFX10-NEXT:    v_sub_nc_u32_e64 v2, s2, s18 clamp
+; GFX10-NEXT:    v_sub_nc_u32_e64 v3, s3, s19 clamp
+; GFX10-NEXT:    v_sub_nc_u32_e64 v4, s4, s20 clamp
+; GFX10-NEXT:    v_sub_nc_u32_e64 v5, s5, s21 clamp
+; GFX10-NEXT:    v_sub_nc_u32_e64 v6, s6, s22 clamp
+; GFX10-NEXT:    v_sub_nc_u32_e64 v7, s7, s23 clamp
+; GFX10-NEXT:    v_sub_nc_u32_e64 v8, s8, s24 clamp
+; GFX10-NEXT:    v_sub_nc_u32_e64 v9, s9, s25 clamp
+; GFX10-NEXT:    v_sub_nc_u32_e64 v10, s10, s26 clamp
+; GFX10-NEXT:    v_sub_nc_u32_e64 v11, s11, s27 clamp
+; GFX10-NEXT:    v_sub_nc_u32_e64 v12, s12, s28 clamp
+; GFX10-NEXT:    v_sub_nc_u32_e64 v13, s13, s29 clamp
+; GFX10-NEXT:    v_sub_nc_u32_e64 v14, s14, s30 clamp
+; GFX10-NEXT:    v_sub_nc_u32_e64 v15, s15, s31 clamp
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX10-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX10-NEXT:    v_readfirstlane_b32 s4, v4
+; GFX10-NEXT:    v_readfirstlane_b32 s5, v5
+; GFX10-NEXT:    v_readfirstlane_b32 s6, v6
+; GFX10-NEXT:    v_readfirstlane_b32 s7, v7
+; GFX10-NEXT:    v_readfirstlane_b32 s8, v8
+; GFX10-NEXT:    v_readfirstlane_b32 s9, v9
+; GFX10-NEXT:    v_readfirstlane_b32 s10, v10
+; GFX10-NEXT:    v_readfirstlane_b32 s11, v11
+; GFX10-NEXT:    v_readfirstlane_b32 s12, v12
+; GFX10-NEXT:    v_readfirstlane_b32 s13, v13
+; GFX10-NEXT:    v_readfirstlane_b32 s14, v14
+; GFX10-NEXT:    v_readfirstlane_b32 s15, v15
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cselect_b32 s46, s0, s16
-; GFX10-NEXT:    s_sub_i32 s0, s0, s46
-; GFX10-NEXT:    s_cmp_lt_u32 s1, s17
-; GFX10-NEXT:    s_cselect_b32 s46, s1, s17
-; GFX10-NEXT:    s_sub_i32 s1, s1, s46
-; GFX10-NEXT:    s_cmp_lt_u32 s2, s18
-; GFX10-NEXT:    s_cselect_b32 s16, s2, s18
-; GFX10-NEXT:    s_sub_i32 s2, s2, s16
-; GFX10-NEXT:    s_cmp_lt_u32 s3, s19
-; GFX10-NEXT:    s_cselect_b32 s16, s3, s19
-; GFX10-NEXT:    s_sub_i32 s3, s3, s16
-; GFX10-NEXT:    s_cmp_lt_u32 s4, s20
-; GFX10-NEXT:    s_cselect_b32 s16, s4, s20
-; GFX10-NEXT:    s_sub_i32 s4, s4, s16
-; GFX10-NEXT:    s_cmp_lt_u32 s5, s21
-; GFX10-NEXT:    s_cselect_b32 s16, s5, s21
-; GFX10-NEXT:    s_sub_i32 s5, s5, s16
-; GFX10-NEXT:    s_cmp_lt_u32 s6, s22
-; GFX10-NEXT:    s_cselect_b32 s16, s6, s22
-; GFX10-NEXT:    s_sub_i32 s6, s6, s16
-; GFX10-NEXT:    s_cmp_lt_u32 s7, s23
-; GFX10-NEXT:    s_cselect_b32 s16, s7, s23
-; GFX10-NEXT:    s_sub_i32 s7, s7, s16
-; GFX10-NEXT:    s_cmp_lt_u32 s8, s24
-; GFX10-NEXT:    s_cselect_b32 s16, s8, s24
-; GFX10-NEXT:    s_sub_i32 s8, s8, s16
-; GFX10-NEXT:    s_cmp_lt_u32 s9, s25
-; GFX10-NEXT:    s_cselect_b32 s16, s9, s25
-; GFX10-NEXT:    s_sub_i32 s9, s9, s16
-; GFX10-NEXT:    s_cmp_lt_u32 s10, s26
-; GFX10-NEXT:    s_cselect_b32 s16, s10, s26
-; GFX10-NEXT:    s_sub_i32 s10, s10, s16
-; GFX10-NEXT:    s_cmp_lt_u32 s11, s27
-; GFX10-NEXT:    s_cselect_b32 s16, s11, s27
-; GFX10-NEXT:    s_sub_i32 s11, s11, s16
-; GFX10-NEXT:    s_cmp_lt_u32 s12, s28
-; GFX10-NEXT:    s_cselect_b32 s16, s12, s28
-; GFX10-NEXT:    s_sub_i32 s12, s12, s16
-; GFX10-NEXT:    s_cmp_lt_u32 s13, s29
-; GFX10-NEXT:    s_cselect_b32 s16, s13, s29
-; GFX10-NEXT:    s_sub_i32 s13, s13, s16
-; GFX10-NEXT:    s_cmp_lt_u32 s14, s30
-; GFX10-NEXT:    s_cselect_b32 s16, s14, s30
-; GFX10-NEXT:    s_sub_i32 s14, s14, s16
-; GFX10-NEXT:    s_cmp_lt_u32 s15, s31
-; GFX10-NEXT:    s_cselect_b32 s16, s15, s31
-; GFX10-NEXT:    s_sub_i32 s15, s15, s16
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <16 x i32> @llvm.usub.sat.v16i32(<16 x i32> %lhs, <16 x i32> %rhs)
   ret <16 x i32> %result
@@ -1862,24 +1583,21 @@ define i16 @v_usubsat_i16(i16 %lhs, i16 %rhs) {
 ; GFX8-LABEL: v_usubsat_i16:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_min_u16_e32 v1, v0, v1
-; GFX8-NEXT:    v_sub_u16_e32 v0, v0, v1
+; GFX8-NEXT:    v_sub_u16_e64 v0, v0, v1 clamp
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_usubsat_i16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_min_u16_e32 v1, v0, v1
-; GFX9-NEXT:    v_sub_u16_e32 v0, v0, v1
+; GFX9-NEXT:    v_sub_u16_e64 v0, v0, v1 clamp
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_usubsat_i16:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_min_u16_e64 v1, v0, v1
+; GFX10-NEXT:    v_sub_nc_u16_e64 v0, v0, v1 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_sub_nc_u16_e64 v0, v0, v1
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call i16 @llvm.usub.sat.i16(i16 %lhs, i16 %rhs)
   ret i16 %result
@@ -1898,30 +1616,23 @@ define amdgpu_ps i16 @s_usubsat_i16(i16 inreg %lhs, i16 inreg %rhs) {
 ;
 ; GFX8-LABEL: s_usubsat_i16:
 ; GFX8:       ; %bb.0:
-; GFX8-NEXT:    s_bfe_u32 s2, s0, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s2, s1
-; GFX8-NEXT:    s_cselect_b32 s1, s2, s1
-; GFX8-NEXT:    s_sub_i32 s0, s0, s1
+; GFX8-NEXT:    v_mov_b32_e32 v0, s1
+; GFX8-NEXT:    v_sub_u16_e64 v0, s0, v0 clamp
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: s_usubsat_i16:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_bfe_u32 s2, s0, 0x100000
-; GFX9-NEXT:    s_bfe_u32 s1, s1, 0x100000
-; GFX9-NEXT:    s_cmp_lt_u32 s2, s1
-; GFX9-NEXT:    s_cselect_b32 s1, s2, s1
-; GFX9-NEXT:    s_sub_i32 s0, s0, s1
+; GFX9-NEXT:    v_mov_b32_e32 v0, s1
+; GFX9-NEXT:    v_sub_u16_e64 v0, s0, v0 clamp
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_usubsat_i16:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_bfe_u32 s2, s0, 0x100000
-; GFX10-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX10-NEXT:    v_sub_nc_u16_e64 v0, s0, s1 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cmp_lt_u32 s2, s1
-; GFX10-NEXT:    s_cselect_b32 s1, s2, s1
-; GFX10-NEXT:    s_sub_i32 s0, s0, s1
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call i16 @llvm.usub.sat.i16(i16 %lhs, i16 %rhs)
   ret i16 %result
@@ -1939,21 +1650,18 @@ define amdgpu_ps half @usubsat_i16_sv(i16 inreg %lhs, i16 %rhs) {
 ;
 ; GFX8-LABEL: usubsat_i16_sv:
 ; GFX8:       ; %bb.0:
-; GFX8-NEXT:    v_min_u16_e32 v0, s0, v0
-; GFX8-NEXT:    v_sub_u16_e32 v0, s0, v0
+; GFX8-NEXT:    v_sub_u16_e64 v0, s0, v0 clamp
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: usubsat_i16_sv:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    v_min_u16_e32 v0, s0, v0
-; GFX9-NEXT:    v_sub_u16_e32 v0, s0, v0
+; GFX9-NEXT:    v_sub_u16_e64 v0, s0, v0 clamp
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: usubsat_i16_sv:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    v_min_u16_e64 v0, s0, v0
+; GFX10-NEXT:    v_sub_nc_u16_e64 v0, s0, v0 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_sub_nc_u16_e64 v0, s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call i16 @llvm.usub.sat.i16(i16 %lhs, i16 %rhs)
   %cast = bitcast i16 %result to half
@@ -1972,21 +1680,18 @@ define amdgpu_ps half @usubsat_i16_vs(i16 %lhs, i16 inreg %rhs) {
 ;
 ; GFX8-LABEL: usubsat_i16_vs:
 ; GFX8:       ; %bb.0:
-; GFX8-NEXT:    v_min_u16_e32 v1, s0, v0
-; GFX8-NEXT:    v_sub_u16_e32 v0, v0, v1
+; GFX8-NEXT:    v_sub_u16_e64 v0, v0, s0 clamp
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: usubsat_i16_vs:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    v_min_u16_e32 v1, s0, v0
-; GFX9-NEXT:    v_sub_u16_e32 v0, v0, v1
+; GFX9-NEXT:    v_sub_u16_e64 v0, v0, s0 clamp
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: usubsat_i16_vs:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    v_min_u16_e64 v1, v0, s0
+; GFX10-NEXT:    v_sub_nc_u16_e64 v0, v0, s0 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_sub_nc_u16_e64 v0, v0, v1
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call i16 @llvm.usub.sat.i16(i16 %lhs, i16 %rhs)
   %cast = bitcast i16 %result to half
@@ -2012,28 +1717,25 @@ define <2 x i16> @v_usubsat_v2i16(<2 x i16> %lhs, <2 x i16> %rhs) {
 ; GFX8-LABEL: v_usubsat_v2i16:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
-; GFX8-NEXT:    v_min_u16_e32 v3, v0, v1
-; GFX8-NEXT:    v_min_u16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT:    v_sub_u16_e32 v0, v0, v3
-; GFX8-NEXT:    v_sub_u16_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    v_sub_u16_e64 v2, v0, v1 clamp
+; GFX8-NEXT:    v_sub_u16_sdwa v0, v0, v1 clamp dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_mov_b32_e32 v1, 16
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_usubsat_v2i16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_pk_min_u16 v1, v0, v1
-; GFX9-NEXT:    v_pk_sub_i16 v0, v0, v1
+; GFX9-NEXT:    v_pk_sub_u16 v0, v0, v1 clamp
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_usubsat_v2i16:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_pk_min_u16 v1, v0, v1
+; GFX10-NEXT:    v_pk_sub_u16 v0, v0, v1 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_pk_sub_i16 v0, v0, v1
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <2 x i16> @llvm.usub.sat.v2i16(<2 x i16> %lhs, <2 x i16> %rhs)
   ret <2 x i16> %result
@@ -2065,57 +1767,28 @@ define amdgpu_ps i32 @s_usubsat_v2i16(<2 x i16> inreg %lhs, <2 x i16> inreg %rhs
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_lshr_b32 s3, s1, 16
 ; GFX8-NEXT:    s_lshr_b32 s2, s0, 16
-; GFX8-NEXT:    s_bfe_u32 s4, s0, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s4, s1
-; GFX8-NEXT:    s_cselect_b32 s1, s4, s1
-; GFX8-NEXT:    s_sub_i32 s0, s0, s1
-; GFX8-NEXT:    s_bfe_u32 s1, s2, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s1, s3
-; GFX8-NEXT:    s_cselect_b32 s1, s1, s3
-; GFX8-NEXT:    s_sub_i32 s1, s2, s1
-; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s0, s0, 0x100000
-; GFX8-NEXT:    s_lshl_b32 s1, s1, 16
-; GFX8-NEXT:    s_or_b32 s0, s0, s1
+; GFX8-NEXT:    v_mov_b32_e32 v1, s3
+; GFX8-NEXT:    v_mov_b32_e32 v0, s1
+; GFX8-NEXT:    v_sub_u16_e64 v1, s2, v1 clamp
+; GFX8-NEXT:    v_mov_b32_e32 v2, 16
+; GFX8-NEXT:    v_sub_u16_e64 v0, s0, v0 clamp
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: s_usubsat_v2i16:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_mov_b32 s3, 0xffff
-; GFX9-NEXT:    s_lshr_b32 s5, s1, 16
-; GFX9-NEXT:    s_lshr_b32 s2, s0, 16
-; GFX9-NEXT:    s_and_b32 s4, s0, s3
-; GFX9-NEXT:    s_and_b32 s1, s1, s3
-; GFX9-NEXT:    s_cmp_lt_u32 s4, s1
-; GFX9-NEXT:    s_cselect_b32 s1, s4, s1
-; GFX9-NEXT:    s_cmp_lt_u32 s2, s5
-; GFX9-NEXT:    s_cselect_b32 s3, s2, s5
-; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s3
-; GFX9-NEXT:    s_lshr_b32 s3, s1, 16
-; GFX9-NEXT:    s_sub_i32 s0, s0, s1
-; GFX9-NEXT:    s_sub_i32 s1, s2, s3
-; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s1
+; GFX9-NEXT:    v_mov_b32_e32 v0, s1
+; GFX9-NEXT:    v_pk_sub_u16 v0, s0, v0 clamp
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_usubsat_v2i16:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_mov_b32 s2, 0xffff
-; GFX10-NEXT:    s_lshr_b32 s3, s0, 16
-; GFX10-NEXT:    s_and_b32 s4, s0, s2
-; GFX10-NEXT:    s_and_b32 s2, s1, s2
-; GFX10-NEXT:    s_lshr_b32 s1, s1, 16
-; GFX10-NEXT:    s_cmp_lt_u32 s4, s2
+; GFX10-NEXT:    v_pk_sub_u16 v0, s0, s1 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cselect_b32 s2, s4, s2
-; GFX10-NEXT:    s_cmp_lt_u32 s3, s1
-; GFX10-NEXT:    s_cselect_b32 s1, s3, s1
-; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s2, s1
-; GFX10-NEXT:    s_lshr_b32 s2, s1, 16
-; GFX10-NEXT:    s_sub_i32 s0, s0, s1
-; GFX10-NEXT:    s_sub_i32 s1, s3, s2
-; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s1
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <2 x i16> @llvm.usub.sat.v2i16(<2 x i16> %lhs, <2 x i16> %rhs)
   %cast = bitcast <2 x i16> %result to i32
@@ -2146,24 +1819,22 @@ define amdgpu_ps float @usubsat_v2i16_sv(<2 x i16> inreg %lhs, <2 x i16> %rhs) {
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_lshr_b32 s1, s0, 16
 ; GFX8-NEXT:    v_mov_b32_e32 v2, s1
-; GFX8-NEXT:    v_min_u16_e32 v1, s0, v0
-; GFX8-NEXT:    v_min_u16_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT:    v_sub_u16_e32 v1, s0, v1
-; GFX8-NEXT:    v_sub_u16_sdwa v0, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX8-NEXT:    v_sub_u16_e64 v1, s0, v0 clamp
+; GFX8-NEXT:    v_sub_u16_sdwa v0, v2, v0 clamp dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_mov_b32_e32 v2, 16
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: usubsat_v2i16_sv:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    v_pk_min_u16 v0, s0, v0
-; GFX9-NEXT:    v_pk_sub_i16 v0, s0, v0
+; GFX9-NEXT:    v_pk_sub_u16 v0, s0, v0 clamp
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: usubsat_v2i16_sv:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    v_pk_min_u16 v0, s0, v0
+; GFX10-NEXT:    v_pk_sub_u16 v0, s0, v0 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_pk_sub_i16 v0, s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <2 x i16> @llvm.usub.sat.v2i16(<2 x i16> %lhs, <2 x i16> %rhs)
   %cast = bitcast <2 x i16> %result to float
@@ -2192,26 +1863,24 @@ define amdgpu_ps float @usubsat_v2i16_vs(<2 x i16> %lhs, <2 x i16> inreg %rhs) {
 ;
 ; GFX8-LABEL: usubsat_v2i16_vs:
 ; GFX8:       ; %bb.0:
-; GFX8-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
 ; GFX8-NEXT:    s_lshr_b32 s1, s0, 16
-; GFX8-NEXT:    v_min_u16_e32 v2, s0, v0
-; GFX8-NEXT:    v_min_u16_e32 v3, s1, v1
-; GFX8-NEXT:    v_sub_u16_e32 v0, v0, v2
-; GFX8-NEXT:    v_sub_u16_sdwa v1, v1, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    v_mov_b32_e32 v2, s1
+; GFX8-NEXT:    v_sub_u16_e64 v1, v0, s0 clamp
+; GFX8-NEXT:    v_sub_u16_sdwa v0, v0, v2 clamp dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT:    v_mov_b32_e32 v2, 16
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: usubsat_v2i16_vs:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    v_pk_min_u16 v1, v0, s0
-; GFX9-NEXT:    v_pk_sub_i16 v0, v0, v1
+; GFX9-NEXT:    v_pk_sub_u16 v0, v0, s0 clamp
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: usubsat_v2i16_vs:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    v_pk_min_u16 v1, v0, s0
+; GFX10-NEXT:    v_pk_sub_u16 v0, v0, s0 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_pk_sub_i16 v0, v0, v1
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <2 x i16> @llvm.usub.sat.v2i16(<2 x i16> %lhs, <2 x i16> %rhs)
   %cast = bitcast <2 x i16> %result to float
@@ -2267,38 +1936,31 @@ define <2 x float> @v_usubsat_v4i16(<4 x i16> %lhs, <4 x i16> %rhs) {
 ; GFX8-LABEL: v_usubsat_v4i16:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_lshrrev_b32_e32 v4, 16, v0
-; GFX8-NEXT:    v_min_u16_e32 v6, v0, v2
-; GFX8-NEXT:    v_min_u16_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
-; GFX8-NEXT:    v_min_u16_e32 v7, v1, v3
-; GFX8-NEXT:    v_min_u16_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT:    v_sub_u16_e32 v0, v0, v6
-; GFX8-NEXT:    v_sub_u16_sdwa v2, v4, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT:    v_or_b32_e32 v0, v0, v2
-; GFX8-NEXT:    v_sub_u16_e32 v1, v1, v7
-; GFX8-NEXT:    v_sub_u16_sdwa v2, v5, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT:    v_or_b32_e32 v1, v1, v2
+; GFX8-NEXT:    v_sub_u16_e64 v4, v0, v2 clamp
+; GFX8-NEXT:    v_sub_u16_sdwa v0, v0, v2 clamp dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_sub_u16_e64 v2, v1, v3 clamp
+; GFX8-NEXT:    v_sub_u16_sdwa v1, v1, v3 clamp dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_mov_b32_e32 v3, 16
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_usubsat_v4i16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_pk_min_u16 v2, v0, v2
-; GFX9-NEXT:    v_pk_sub_i16 v0, v0, v2
-; GFX9-NEXT:    v_pk_min_u16 v2, v1, v3
-; GFX9-NEXT:    v_pk_sub_i16 v1, v1, v2
+; GFX9-NEXT:    v_pk_sub_u16 v0, v0, v2 clamp
+; GFX9-NEXT:    v_pk_sub_u16 v1, v1, v3 clamp
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_usubsat_v4i16:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_pk_min_u16 v2, v0, v2
-; GFX10-NEXT:    v_pk_min_u16 v3, v1, v3
+; GFX10-NEXT:    v_pk_sub_u16 v0, v0, v2 clamp
+; GFX10-NEXT:    v_pk_sub_u16 v1, v1, v3 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_pk_sub_i16 v0, v0, v2
-; GFX10-NEXT:    v_pk_sub_i16 v1, v1, v3
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <4 x i16> @llvm.usub.sat.v4i16(<4 x i16> %lhs, <4 x i16> %rhs)
   %cast = bitcast <4 x i16> %result to <2 x float>
@@ -2346,100 +2008,43 @@ define amdgpu_ps <2 x i32> @s_usubsat_v4i16(<4 x i16> inreg %lhs, <4 x i16> inre
 ; GFX8-LABEL: s_usubsat_v4i16:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_lshr_b32 s6, s2, 16
+; GFX8-NEXT:    s_lshr_b32 s7, s3, 16
 ; GFX8-NEXT:    s_lshr_b32 s4, s0, 16
+; GFX8-NEXT:    v_mov_b32_e32 v1, s6
+; GFX8-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX8-NEXT:    s_lshr_b32 s5, s1, 16
-; GFX8-NEXT:    s_lshr_b32 s7, s3, 16
-; GFX8-NEXT:    s_bfe_u32 s8, s0, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s8, s2
-; GFX8-NEXT:    s_cselect_b32 s2, s8, s2
-; GFX8-NEXT:    s_sub_i32 s0, s0, s2
-; GFX8-NEXT:    s_bfe_u32 s2, s4, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s6, s6, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s2, s6
-; GFX8-NEXT:    s_cselect_b32 s2, s2, s6
-; GFX8-NEXT:    s_sub_i32 s2, s4, s2
-; GFX8-NEXT:    s_bfe_u32 s4, s1, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s4, s3
-; GFX8-NEXT:    s_cselect_b32 s3, s4, s3
-; GFX8-NEXT:    s_sub_i32 s1, s1, s3
-; GFX8-NEXT:    s_bfe_u32 s3, s5, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s4, s7, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s3, s4
-; GFX8-NEXT:    s_cselect_b32 s3, s3, s4
-; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
-; GFX8-NEXT:    s_sub_i32 s3, s5, s3
-; GFX8-NEXT:    s_bfe_u32 s0, s0, 0x100000
-; GFX8-NEXT:    s_lshl_b32 s2, s2, 16
-; GFX8-NEXT:    s_or_b32 s0, s0, s2
-; GFX8-NEXT:    s_bfe_u32 s2, s3, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
-; GFX8-NEXT:    s_lshl_b32 s2, s2, 16
-; GFX8-NEXT:    s_or_b32 s1, s1, s2
+; GFX8-NEXT:    v_mov_b32_e32 v3, s7
+; GFX8-NEXT:    v_sub_u16_e64 v1, s4, v1 clamp
+; GFX8-NEXT:    v_mov_b32_e32 v4, 16
+; GFX8-NEXT:    v_mov_b32_e32 v2, s3
+; GFX8-NEXT:    v_sub_u16_e64 v0, s0, v0 clamp
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_sub_u16_e64 v3, s5, v3 clamp
+; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    v_sub_u16_e64 v2, s1, v2 clamp
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v1, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: s_usubsat_v4i16:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_mov_b32 s5, 0xffff
-; GFX9-NEXT:    s_lshr_b32 s7, s2, 16
-; GFX9-NEXT:    s_lshr_b32 s4, s0, 16
-; GFX9-NEXT:    s_and_b32 s6, s0, s5
-; GFX9-NEXT:    s_and_b32 s2, s2, s5
-; GFX9-NEXT:    s_cmp_lt_u32 s6, s2
-; GFX9-NEXT:    s_cselect_b32 s2, s6, s2
-; GFX9-NEXT:    s_cmp_lt_u32 s4, s7
-; GFX9-NEXT:    s_cselect_b32 s6, s4, s7
-; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s6
-; GFX9-NEXT:    s_lshr_b32 s6, s2, 16
-; GFX9-NEXT:    s_sub_i32 s0, s0, s2
-; GFX9-NEXT:    s_sub_i32 s2, s4, s6
-; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s2
-; GFX9-NEXT:    s_lshr_b32 s6, s3, 16
-; GFX9-NEXT:    s_lshr_b32 s2, s1, 16
-; GFX9-NEXT:    s_and_b32 s4, s1, s5
-; GFX9-NEXT:    s_and_b32 s3, s3, s5
-; GFX9-NEXT:    s_cmp_lt_u32 s4, s3
-; GFX9-NEXT:    s_cselect_b32 s3, s4, s3
-; GFX9-NEXT:    s_cmp_lt_u32 s2, s6
-; GFX9-NEXT:    s_cselect_b32 s4, s2, s6
-; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s3, s4
-; GFX9-NEXT:    s_lshr_b32 s4, s3, 16
-; GFX9-NEXT:    s_sub_i32 s1, s1, s3
-; GFX9-NEXT:    s_sub_i32 s2, s2, s4
-; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s2
+; GFX9-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    v_pk_sub_u16 v0, s0, v0 clamp
+; GFX9-NEXT:    v_pk_sub_u16 v1, s1, v1 clamp
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_usubsat_v4i16:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_mov_b32 s4, 0xffff
-; GFX10-NEXT:    s_lshr_b32 s5, s0, 16
-; GFX10-NEXT:    s_and_b32 s7, s2, s4
-; GFX10-NEXT:    s_and_b32 s6, s0, s4
-; GFX10-NEXT:    s_lshr_b32 s2, s2, 16
-; GFX10-NEXT:    s_cmp_lt_u32 s6, s7
+; GFX10-NEXT:    v_pk_sub_u16 v0, s0, s2 clamp
+; GFX10-NEXT:    v_pk_sub_u16 v1, s1, s3 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cselect_b32 s6, s6, s7
-; GFX10-NEXT:    s_cmp_lt_u32 s5, s2
-; GFX10-NEXT:    s_cselect_b32 s2, s5, s2
-; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s6, s2
-; GFX10-NEXT:    s_lshr_b32 s6, s2, 16
-; GFX10-NEXT:    s_sub_i32 s0, s0, s2
-; GFX10-NEXT:    s_sub_i32 s2, s5, s6
-; GFX10-NEXT:    s_and_b32 s6, s1, s4
-; GFX10-NEXT:    s_and_b32 s4, s3, s4
-; GFX10-NEXT:    s_lshr_b32 s5, s1, 16
-; GFX10-NEXT:    s_lshr_b32 s3, s3, 16
-; GFX10-NEXT:    s_cmp_lt_u32 s6, s4
-; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s2
-; GFX10-NEXT:    s_cselect_b32 s4, s6, s4
-; GFX10-NEXT:    s_cmp_lt_u32 s5, s3
-; GFX10-NEXT:    s_cselect_b32 s3, s5, s3
-; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s4, s3
-; GFX10-NEXT:    s_lshr_b32 s4, s3, 16
-; GFX10-NEXT:    s_sub_i32 s1, s1, s3
-; GFX10-NEXT:    s_sub_i32 s3, s5, s4
-; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s1, s3
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <4 x i16> @llvm.usub.sat.v4i16(<4 x i16> %lhs, <4 x i16> %rhs)
   %cast = bitcast <4 x i16> %result to <2 x i32>
@@ -2509,48 +2114,38 @@ define <3 x float> @v_usubsat_v6i16(<6 x i16> %lhs, <6 x i16> %rhs) {
 ; GFX8-LABEL: v_usubsat_v6i16:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_lshrrev_b32_e32 v6, 16, v0
-; GFX8-NEXT:    v_min_u16_e32 v9, v0, v3
-; GFX8-NEXT:    v_min_u16_sdwa v3, v6, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT:    v_lshrrev_b32_e32 v7, 16, v1
-; GFX8-NEXT:    v_min_u16_e32 v10, v1, v4
-; GFX8-NEXT:    v_lshrrev_b32_e32 v8, 16, v2
-; GFX8-NEXT:    v_min_u16_sdwa v4, v7, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT:    v_sub_u16_e32 v0, v0, v9
-; GFX8-NEXT:    v_sub_u16_sdwa v3, v6, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT:    v_min_u16_e32 v11, v2, v5
-; GFX8-NEXT:    v_or_b32_e32 v0, v0, v3
-; GFX8-NEXT:    v_min_u16_sdwa v5, v8, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT:    v_sub_u16_e32 v1, v1, v10
-; GFX8-NEXT:    v_sub_u16_sdwa v3, v7, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT:    v_or_b32_e32 v1, v1, v3
-; GFX8-NEXT:    v_sub_u16_e32 v2, v2, v11
-; GFX8-NEXT:    v_sub_u16_sdwa v3, v8, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT:    v_or_b32_e32 v2, v2, v3
+; GFX8-NEXT:    v_sub_u16_e64 v6, v0, v3 clamp
+; GFX8-NEXT:    v_sub_u16_sdwa v0, v0, v3 clamp dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_sub_u16_e64 v3, v1, v4 clamp
+; GFX8-NEXT:    v_sub_u16_sdwa v1, v1, v4 clamp dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_sub_u16_e64 v4, v2, v5 clamp
+; GFX8-NEXT:    v_sub_u16_sdwa v2, v2, v5 clamp dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_mov_b32_e32 v5, 16
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v1, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    v_mov_b32_e32 v3, 16
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v0, v5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_or_b32_sdwa v0, v6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_usubsat_v6i16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_pk_min_u16 v3, v0, v3
-; GFX9-NEXT:    v_pk_sub_i16 v0, v0, v3
-; GFX9-NEXT:    v_pk_min_u16 v3, v1, v4
-; GFX9-NEXT:    v_pk_sub_i16 v1, v1, v3
-; GFX9-NEXT:    v_pk_min_u16 v3, v2, v5
-; GFX9-NEXT:    v_pk_sub_i16 v2, v2, v3
+; GFX9-NEXT:    v_pk_sub_u16 v0, v0, v3 clamp
+; GFX9-NEXT:    v_pk_sub_u16 v1, v1, v4 clamp
+; GFX9-NEXT:    v_pk_sub_u16 v2, v2, v5 clamp
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_usubsat_v6i16:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_pk_min_u16 v3, v0, v3
-; GFX10-NEXT:    v_pk_min_u16 v4, v1, v4
-; GFX10-NEXT:    v_pk_min_u16 v5, v2, v5
+; GFX10-NEXT:    v_pk_sub_u16 v0, v0, v3 clamp
+; GFX10-NEXT:    v_pk_sub_u16 v1, v1, v4 clamp
+; GFX10-NEXT:    v_pk_sub_u16 v2, v2, v5 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_pk_sub_i16 v0, v0, v3
-; GFX10-NEXT:    v_pk_sub_i16 v1, v1, v4
-; GFX10-NEXT:    v_pk_sub_i16 v2, v2, v5
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <6 x i16> @llvm.usub.sat.v6i16(<6 x i16> %lhs, <6 x i16> %rhs)
   %cast = bitcast <6 x i16> %result to <3 x float>
@@ -2614,142 +2209,57 @@ define amdgpu_ps <3 x i32> @s_usubsat_v6i16(<6 x i16> inreg %lhs, <6 x i16> inre
 ; GFX8-LABEL: s_usubsat_v6i16:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_lshr_b32 s9, s3, 16
+; GFX8-NEXT:    s_lshr_b32 s10, s4, 16
 ; GFX8-NEXT:    s_lshr_b32 s6, s0, 16
+; GFX8-NEXT:    v_mov_b32_e32 v1, s9
+; GFX8-NEXT:    v_mov_b32_e32 v0, s3
+; GFX8-NEXT:    s_lshr_b32 s11, s5, 16
 ; GFX8-NEXT:    s_lshr_b32 s7, s1, 16
+; GFX8-NEXT:    v_mov_b32_e32 v3, s10
+; GFX8-NEXT:    v_sub_u16_e64 v1, s6, v1 clamp
+; GFX8-NEXT:    v_mov_b32_e32 v6, 16
+; GFX8-NEXT:    v_mov_b32_e32 v2, s4
 ; GFX8-NEXT:    s_lshr_b32 s8, s2, 16
-; GFX8-NEXT:    s_lshr_b32 s10, s4, 16
-; GFX8-NEXT:    s_lshr_b32 s11, s5, 16
-; GFX8-NEXT:    s_bfe_u32 s12, s0, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s12, s3
-; GFX8-NEXT:    s_cselect_b32 s3, s12, s3
-; GFX8-NEXT:    s_sub_i32 s0, s0, s3
-; GFX8-NEXT:    s_bfe_u32 s3, s6, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s9, s9, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s3, s9
-; GFX8-NEXT:    s_cselect_b32 s3, s3, s9
-; GFX8-NEXT:    s_sub_i32 s3, s6, s3
-; GFX8-NEXT:    s_bfe_u32 s6, s1, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s4, s4, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s6, s4
-; GFX8-NEXT:    s_cselect_b32 s4, s6, s4
-; GFX8-NEXT:    s_sub_i32 s1, s1, s4
-; GFX8-NEXT:    s_bfe_u32 s4, s7, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s6, s10, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s4, s6
-; GFX8-NEXT:    s_cselect_b32 s4, s4, s6
-; GFX8-NEXT:    s_sub_i32 s4, s7, s4
-; GFX8-NEXT:    s_bfe_u32 s6, s2, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s5, s5, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s6, s5
-; GFX8-NEXT:    s_cselect_b32 s5, s6, s5
-; GFX8-NEXT:    s_sub_i32 s2, s2, s5
-; GFX8-NEXT:    s_bfe_u32 s5, s8, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s6, s11, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s5, s6
-; GFX8-NEXT:    s_cselect_b32 s5, s5, s6
-; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s0, s0, 0x100000
-; GFX8-NEXT:    s_lshl_b32 s3, s3, 16
-; GFX8-NEXT:    s_or_b32 s0, s0, s3
-; GFX8-NEXT:    s_bfe_u32 s3, s4, 0x100000
-; GFX8-NEXT:    s_sub_i32 s5, s8, s5
-; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
-; GFX8-NEXT:    s_lshl_b32 s3, s3, 16
-; GFX8-NEXT:    s_or_b32 s1, s1, s3
-; GFX8-NEXT:    s_bfe_u32 s3, s5, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
-; GFX8-NEXT:    s_lshl_b32 s3, s3, 16
-; GFX8-NEXT:    s_or_b32 s2, s2, s3
+; GFX8-NEXT:    v_mov_b32_e32 v5, s11
+; GFX8-NEXT:    v_sub_u16_e64 v0, s0, v0 clamp
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v1, v6, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_sub_u16_e64 v3, s7, v3 clamp
+; GFX8-NEXT:    v_mov_b32_e32 v4, s5
+; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    v_sub_u16_e64 v2, s1, v2 clamp
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v1, v6, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_sub_u16_e64 v5, s8, v5 clamp
+; GFX8-NEXT:    v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    v_sub_u16_e64 v4, s2, v4 clamp
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v2, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_or_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX8-NEXT:    v_readfirstlane_b32 s2, v2
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: s_usubsat_v6i16:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_mov_b32 s7, 0xffff
-; GFX9-NEXT:    s_lshr_b32 s9, s3, 16
-; GFX9-NEXT:    s_lshr_b32 s6, s0, 16
-; GFX9-NEXT:    s_and_b32 s8, s0, s7
-; GFX9-NEXT:    s_and_b32 s3, s3, s7
-; GFX9-NEXT:    s_cmp_lt_u32 s8, s3
-; GFX9-NEXT:    s_cselect_b32 s3, s8, s3
-; GFX9-NEXT:    s_cmp_lt_u32 s6, s9
-; GFX9-NEXT:    s_cselect_b32 s8, s6, s9
-; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s3, s8
-; GFX9-NEXT:    s_lshr_b32 s8, s3, 16
-; GFX9-NEXT:    s_sub_i32 s0, s0, s3
-; GFX9-NEXT:    s_sub_i32 s3, s6, s8
-; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s3
-; GFX9-NEXT:    s_lshr_b32 s8, s4, 16
-; GFX9-NEXT:    s_lshr_b32 s3, s1, 16
-; GFX9-NEXT:    s_and_b32 s6, s1, s7
-; GFX9-NEXT:    s_and_b32 s4, s4, s7
-; GFX9-NEXT:    s_cmp_lt_u32 s6, s4
-; GFX9-NEXT:    s_cselect_b32 s4, s6, s4
-; GFX9-NEXT:    s_cmp_lt_u32 s3, s8
-; GFX9-NEXT:    s_cselect_b32 s6, s3, s8
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s6
-; GFX9-NEXT:    s_lshr_b32 s6, s4, 16
-; GFX9-NEXT:    s_sub_i32 s1, s1, s4
-; GFX9-NEXT:    s_sub_i32 s3, s3, s6
-; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s3
-; GFX9-NEXT:    s_lshr_b32 s6, s5, 16
-; GFX9-NEXT:    s_lshr_b32 s3, s2, 16
-; GFX9-NEXT:    s_and_b32 s4, s2, s7
-; GFX9-NEXT:    s_and_b32 s5, s5, s7
-; GFX9-NEXT:    s_cmp_lt_u32 s4, s5
-; GFX9-NEXT:    s_cselect_b32 s4, s4, s5
-; GFX9-NEXT:    s_cmp_lt_u32 s3, s6
-; GFX9-NEXT:    s_cselect_b32 s5, s3, s6
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
-; GFX9-NEXT:    s_lshr_b32 s5, s4, 16
-; GFX9-NEXT:    s_sub_i32 s2, s2, s4
-; GFX9-NEXT:    s_sub_i32 s3, s3, s5
-; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s3
+; GFX9-NEXT:    v_mov_b32_e32 v0, s3
+; GFX9-NEXT:    v_mov_b32_e32 v1, s4
+; GFX9-NEXT:    v_mov_b32_e32 v2, s5
+; GFX9-NEXT:    v_pk_sub_u16 v0, s0, v0 clamp
+; GFX9-NEXT:    v_pk_sub_u16 v1, s1, v1 clamp
+; GFX9-NEXT:    v_pk_sub_u16 v2, s2, v2 clamp
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v2
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_usubsat_v6i16:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_mov_b32 s6, 0xffff
-; GFX10-NEXT:    s_lshr_b32 s7, s0, 16
-; GFX10-NEXT:    s_and_b32 s9, s3, s6
-; GFX10-NEXT:    s_and_b32 s8, s0, s6
-; GFX10-NEXT:    s_lshr_b32 s3, s3, 16
-; GFX10-NEXT:    s_cmp_lt_u32 s8, s9
+; GFX10-NEXT:    v_pk_sub_u16 v0, s0, s3 clamp
+; GFX10-NEXT:    v_pk_sub_u16 v1, s1, s4 clamp
+; GFX10-NEXT:    v_pk_sub_u16 v2, s2, s5 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cselect_b32 s8, s8, s9
-; GFX10-NEXT:    s_cmp_lt_u32 s7, s3
-; GFX10-NEXT:    s_cselect_b32 s3, s7, s3
-; GFX10-NEXT:    s_and_b32 s9, s4, s6
-; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s8, s3
-; GFX10-NEXT:    s_lshr_b32 s4, s4, 16
-; GFX10-NEXT:    s_lshr_b32 s8, s3, 16
-; GFX10-NEXT:    s_sub_i32 s0, s0, s3
-; GFX10-NEXT:    s_sub_i32 s3, s7, s8
-; GFX10-NEXT:    s_and_b32 s8, s1, s6
-; GFX10-NEXT:    s_lshr_b32 s7, s1, 16
-; GFX10-NEXT:    s_cmp_lt_u32 s8, s9
-; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s3
-; GFX10-NEXT:    s_cselect_b32 s8, s8, s9
-; GFX10-NEXT:    s_cmp_lt_u32 s7, s4
-; GFX10-NEXT:    s_cselect_b32 s4, s7, s4
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s8, s4
-; GFX10-NEXT:    s_lshr_b32 s8, s4, 16
-; GFX10-NEXT:    s_sub_i32 s1, s1, s4
-; GFX10-NEXT:    s_sub_i32 s4, s7, s8
-; GFX10-NEXT:    s_and_b32 s8, s2, s6
-; GFX10-NEXT:    s_and_b32 s6, s5, s6
-; GFX10-NEXT:    s_lshr_b32 s7, s2, 16
-; GFX10-NEXT:    s_lshr_b32 s5, s5, 16
-; GFX10-NEXT:    s_cmp_lt_u32 s8, s6
-; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s1, s4
-; GFX10-NEXT:    s_cselect_b32 s6, s8, s6
-; GFX10-NEXT:    s_cmp_lt_u32 s7, s5
-; GFX10-NEXT:    s_cselect_b32 s5, s7, s5
-; GFX10-NEXT:    s_pack_ll_b32_b16 s5, s6, s5
-; GFX10-NEXT:    s_lshr_b32 s3, s5, 16
-; GFX10-NEXT:    s_sub_i32 s2, s2, s5
-; GFX10-NEXT:    s_sub_i32 s3, s7, s3
-; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s2, s3
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <6 x i16> @llvm.usub.sat.v6i16(<6 x i16> %lhs, <6 x i16> %rhs)
   %cast = bitcast <6 x i16> %result to <3 x i32>
@@ -2822,58 +2332,44 @@ define <4 x float> @v_usubsat_v8i16(<8 x i16> %lhs, <8 x i16> %rhs) {
 ; GFX8-LABEL: v_usubsat_v8i16:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_lshrrev_b32_e32 v8, 16, v0
-; GFX8-NEXT:    v_min_u16_e32 v12, v0, v4
-; GFX8-NEXT:    v_min_u16_sdwa v4, v8, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT:    v_lshrrev_b32_e32 v9, 16, v1
-; GFX8-NEXT:    v_min_u16_e32 v13, v1, v5
-; GFX8-NEXT:    v_lshrrev_b32_e32 v10, 16, v2
-; GFX8-NEXT:    v_min_u16_sdwa v5, v9, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT:    v_sub_u16_e32 v0, v0, v12
-; GFX8-NEXT:    v_sub_u16_sdwa v4, v8, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT:    v_min_u16_e32 v14, v2, v6
-; GFX8-NEXT:    v_or_b32_e32 v0, v0, v4
-; GFX8-NEXT:    v_lshrrev_b32_e32 v11, 16, v3
-; GFX8-NEXT:    v_min_u16_sdwa v6, v10, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT:    v_sub_u16_e32 v1, v1, v13
-; GFX8-NEXT:    v_sub_u16_sdwa v4, v9, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT:    v_min_u16_e32 v15, v3, v7
-; GFX8-NEXT:    v_or_b32_e32 v1, v1, v4
-; GFX8-NEXT:    v_min_u16_sdwa v7, v11, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT:    v_sub_u16_e32 v2, v2, v14
-; GFX8-NEXT:    v_sub_u16_sdwa v4, v10, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT:    v_or_b32_e32 v2, v2, v4
-; GFX8-NEXT:    v_sub_u16_e32 v3, v3, v15
-; GFX8-NEXT:    v_sub_u16_sdwa v4, v11, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT:    v_or_b32_e32 v3, v3, v4
+; GFX8-NEXT:    v_sub_u16_e64 v8, v0, v4 clamp
+; GFX8-NEXT:    v_sub_u16_sdwa v0, v0, v4 clamp dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_sub_u16_e64 v4, v1, v5 clamp
+; GFX8-NEXT:    v_sub_u16_sdwa v1, v1, v5 clamp dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_sub_u16_e64 v5, v2, v6 clamp
+; GFX8-NEXT:    v_sub_u16_sdwa v2, v2, v6 clamp dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_sub_u16_e64 v6, v3, v7 clamp
+; GFX8-NEXT:    v_sub_u16_sdwa v3, v3, v7 clamp dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_mov_b32_e32 v7, 16
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v0, v7, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_mov_b32_e32 v7, 16
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v1, v7, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v2, v7, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v3, v7, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_or_b32_sdwa v0, v8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v2, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v3, v6, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_usubsat_v8i16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_pk_min_u16 v4, v0, v4
-; GFX9-NEXT:    v_pk_sub_i16 v0, v0, v4
-; GFX9-NEXT:    v_pk_min_u16 v4, v1, v5
-; GFX9-NEXT:    v_pk_sub_i16 v1, v1, v4
-; GFX9-NEXT:    v_pk_min_u16 v4, v2, v6
-; GFX9-NEXT:    v_pk_sub_i16 v2, v2, v4
-; GFX9-NEXT:    v_pk_min_u16 v4, v3, v7
-; GFX9-NEXT:    v_pk_sub_i16 v3, v3, v4
+; GFX9-NEXT:    v_pk_sub_u16 v0, v0, v4 clamp
+; GFX9-NEXT:    v_pk_sub_u16 v1, v1, v5 clamp
+; GFX9-NEXT:    v_pk_sub_u16 v2, v2, v6 clamp
+; GFX9-NEXT:    v_pk_sub_u16 v3, v3, v7 clamp
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_usubsat_v8i16:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_pk_min_u16 v19, v2, v6
-; GFX10-NEXT:    v_pk_min_u16 v11, v0, v4
-; GFX10-NEXT:    v_pk_min_u16 v15, v1, v5
-; GFX10-NEXT:    v_pk_min_u16 v6, v3, v7
+; GFX10-NEXT:    v_pk_sub_u16 v0, v0, v4 clamp
+; GFX10-NEXT:    v_pk_sub_u16 v1, v1, v5 clamp
+; GFX10-NEXT:    v_pk_sub_u16 v2, v2, v6 clamp
+; GFX10-NEXT:    v_pk_sub_u16 v3, v3, v7 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    v_pk_sub_i16 v2, v2, v19
-; GFX10-NEXT:    v_pk_sub_i16 v0, v0, v11
-; GFX10-NEXT:    v_pk_sub_i16 v1, v1, v15
-; GFX10-NEXT:    v_pk_sub_i16 v3, v3, v6
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> %lhs, <8 x i16> %rhs)
   %cast = bitcast <8 x i16> %result to <4 x float>
@@ -2953,184 +2449,71 @@ define amdgpu_ps <4 x i32> @s_usubsat_v8i16(<8 x i16> inreg %lhs, <8 x i16> inre
 ; GFX8-LABEL: s_usubsat_v8i16:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_lshr_b32 s12, s4, 16
+; GFX8-NEXT:    s_lshr_b32 s13, s5, 16
 ; GFX8-NEXT:    s_lshr_b32 s8, s0, 16
+; GFX8-NEXT:    v_mov_b32_e32 v1, s12
+; GFX8-NEXT:    v_mov_b32_e32 v0, s4
+; GFX8-NEXT:    s_lshr_b32 s14, s6, 16
+; GFX8-NEXT:    s_lshr_b32 s15, s7, 16
 ; GFX8-NEXT:    s_lshr_b32 s9, s1, 16
+; GFX8-NEXT:    v_mov_b32_e32 v3, s13
+; GFX8-NEXT:    v_sub_u16_e64 v1, s8, v1 clamp
+; GFX8-NEXT:    v_mov_b32_e32 v8, 16
+; GFX8-NEXT:    v_mov_b32_e32 v2, s5
 ; GFX8-NEXT:    s_lshr_b32 s10, s2, 16
+; GFX8-NEXT:    v_mov_b32_e32 v5, s14
 ; GFX8-NEXT:    s_lshr_b32 s11, s3, 16
-; GFX8-NEXT:    s_lshr_b32 s13, s5, 16
-; GFX8-NEXT:    s_lshr_b32 s14, s6, 16
-; GFX8-NEXT:    s_lshr_b32 s15, s7, 16
-; GFX8-NEXT:    s_bfe_u32 s16, s0, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s4, s4, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s16, s4
-; GFX8-NEXT:    s_cselect_b32 s4, s16, s4
-; GFX8-NEXT:    s_sub_i32 s0, s0, s4
-; GFX8-NEXT:    s_bfe_u32 s4, s8, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s12, s12, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s4, s12
-; GFX8-NEXT:    s_cselect_b32 s4, s4, s12
-; GFX8-NEXT:    s_sub_i32 s4, s8, s4
-; GFX8-NEXT:    s_bfe_u32 s8, s1, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s5, s5, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s8, s5
-; GFX8-NEXT:    s_cselect_b32 s5, s8, s5
-; GFX8-NEXT:    s_sub_i32 s1, s1, s5
-; GFX8-NEXT:    s_bfe_u32 s5, s9, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s8, s13, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s5, s8
-; GFX8-NEXT:    s_cselect_b32 s5, s5, s8
-; GFX8-NEXT:    s_sub_i32 s5, s9, s5
-; GFX8-NEXT:    s_bfe_u32 s8, s2, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s6, s6, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s8, s6
-; GFX8-NEXT:    s_cselect_b32 s6, s8, s6
-; GFX8-NEXT:    s_sub_i32 s2, s2, s6
-; GFX8-NEXT:    s_bfe_u32 s6, s10, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s8, s14, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s6, s8
-; GFX8-NEXT:    s_cselect_b32 s6, s6, s8
-; GFX8-NEXT:    s_sub_i32 s6, s10, s6
-; GFX8-NEXT:    s_bfe_u32 s8, s3, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s7, s7, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s8, s7
-; GFX8-NEXT:    s_cselect_b32 s7, s8, s7
-; GFX8-NEXT:    s_sub_i32 s3, s3, s7
-; GFX8-NEXT:    s_bfe_u32 s7, s11, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s8, s15, 0x100000
-; GFX8-NEXT:    s_cmp_lt_u32 s7, s8
-; GFX8-NEXT:    s_cselect_b32 s7, s7, s8
-; GFX8-NEXT:    s_bfe_u32 s4, s4, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s0, s0, 0x100000
-; GFX8-NEXT:    s_lshl_b32 s4, s4, 16
-; GFX8-NEXT:    s_or_b32 s0, s0, s4
-; GFX8-NEXT:    s_bfe_u32 s4, s5, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
-; GFX8-NEXT:    s_lshl_b32 s4, s4, 16
-; GFX8-NEXT:    s_or_b32 s1, s1, s4
-; GFX8-NEXT:    s_bfe_u32 s4, s6, 0x100000
-; GFX8-NEXT:    s_sub_i32 s7, s11, s7
-; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
-; GFX8-NEXT:    s_lshl_b32 s4, s4, 16
-; GFX8-NEXT:    s_or_b32 s2, s2, s4
-; GFX8-NEXT:    s_bfe_u32 s4, s7, 0x100000
-; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
-; GFX8-NEXT:    s_lshl_b32 s4, s4, 16
-; GFX8-NEXT:    s_or_b32 s3, s3, s4
+; GFX8-NEXT:    v_mov_b32_e32 v7, s15
+; GFX8-NEXT:    v_sub_u16_e64 v0, s0, v0 clamp
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v1, v8, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_sub_u16_e64 v3, s9, v3 clamp
+; GFX8-NEXT:    v_mov_b32_e32 v4, s6
+; GFX8-NEXT:    v_mov_b32_e32 v6, s7
+; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    v_sub_u16_e64 v2, s1, v2 clamp
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v1, v8, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_sub_u16_e64 v7, s11, v7 clamp
+; GFX8-NEXT:    v_sub_u16_e64 v5, s10, v5 clamp
+; GFX8-NEXT:    v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    v_sub_u16_e64 v4, s2, v4 clamp
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v2, v8, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_sub_u16_e64 v6, s3, v6 clamp
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v3, v8, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_or_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v3, v6, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX8-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX8-NEXT:    v_readfirstlane_b32 s3, v3
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: s_usubsat_v8i16:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_mov_b32 s9, 0xffff
-; GFX9-NEXT:    s_lshr_b32 s11, s4, 16
-; GFX9-NEXT:    s_lshr_b32 s8, s0, 16
-; GFX9-NEXT:    s_and_b32 s10, s0, s9
-; GFX9-NEXT:    s_and_b32 s4, s4, s9
-; GFX9-NEXT:    s_cmp_lt_u32 s10, s4
-; GFX9-NEXT:    s_cselect_b32 s4, s10, s4
-; GFX9-NEXT:    s_cmp_lt_u32 s8, s11
-; GFX9-NEXT:    s_cselect_b32 s10, s8, s11
-; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s10
-; GFX9-NEXT:    s_lshr_b32 s10, s4, 16
-; GFX9-NEXT:    s_sub_i32 s0, s0, s4
-; GFX9-NEXT:    s_sub_i32 s4, s8, s10
-; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s4
-; GFX9-NEXT:    s_lshr_b32 s10, s5, 16
-; GFX9-NEXT:    s_lshr_b32 s4, s1, 16
-; GFX9-NEXT:    s_and_b32 s8, s1, s9
-; GFX9-NEXT:    s_and_b32 s5, s5, s9
-; GFX9-NEXT:    s_cmp_lt_u32 s8, s5
-; GFX9-NEXT:    s_cselect_b32 s5, s8, s5
-; GFX9-NEXT:    s_cmp_lt_u32 s4, s10
-; GFX9-NEXT:    s_cselect_b32 s8, s4, s10
-; GFX9-NEXT:    s_pack_ll_b32_b16 s5, s5, s8
-; GFX9-NEXT:    s_lshr_b32 s8, s5, 16
-; GFX9-NEXT:    s_sub_i32 s1, s1, s5
-; GFX9-NEXT:    s_sub_i32 s4, s4, s8
-; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s4
-; GFX9-NEXT:    s_lshr_b32 s8, s6, 16
-; GFX9-NEXT:    s_lshr_b32 s4, s2, 16
-; GFX9-NEXT:    s_and_b32 s5, s2, s9
-; GFX9-NEXT:    s_and_b32 s6, s6, s9
-; GFX9-NEXT:    s_cmp_lt_u32 s5, s6
-; GFX9-NEXT:    s_cselect_b32 s5, s5, s6
-; GFX9-NEXT:    s_cmp_lt_u32 s4, s8
-; GFX9-NEXT:    s_cselect_b32 s6, s4, s8
-; GFX9-NEXT:    s_pack_ll_b32_b16 s5, s5, s6
-; GFX9-NEXT:    s_lshr_b32 s6, s5, 16
-; GFX9-NEXT:    s_sub_i32 s2, s2, s5
-; GFX9-NEXT:    s_sub_i32 s4, s4, s6
-; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s4
-; GFX9-NEXT:    s_lshr_b32 s6, s7, 16
-; GFX9-NEXT:    s_lshr_b32 s4, s3, 16
-; GFX9-NEXT:    s_and_b32 s5, s3, s9
-; GFX9-NEXT:    s_and_b32 s7, s7, s9
-; GFX9-NEXT:    s_cmp_lt_u32 s5, s7
-; GFX9-NEXT:    s_cselect_b32 s5, s5, s7
-; GFX9-NEXT:    s_cmp_lt_u32 s4, s6
-; GFX9-NEXT:    s_cselect_b32 s6, s4, s6
-; GFX9-NEXT:    s_pack_ll_b32_b16 s5, s5, s6
-; GFX9-NEXT:    s_lshr_b32 s6, s5, 16
-; GFX9-NEXT:    s_sub_i32 s3, s3, s5
-; GFX9-NEXT:    s_sub_i32 s4, s4, s6
-; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s3, s4
+; GFX9-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-NEXT:    v_mov_b32_e32 v2, s6
+; GFX9-NEXT:    v_mov_b32_e32 v3, s7
+; GFX9-NEXT:    v_pk_sub_u16 v0, s0, v0 clamp
+; GFX9-NEXT:    v_pk_sub_u16 v1, s1, v1 clamp
+; GFX9-NEXT:    v_pk_sub_u16 v2, s2, v2 clamp
+; GFX9-NEXT:    v_pk_sub_u16 v3, s3, v3 clamp
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX9-NEXT:    v_readfirstlane_b32 s3, v3
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_usubsat_v8i16:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_mov_b32 s8, 0xffff
-; GFX10-NEXT:    s_lshr_b32 s9, s0, 16
-; GFX10-NEXT:    s_and_b32 s11, s4, s8
-; GFX10-NEXT:    s_and_b32 s10, s0, s8
-; GFX10-NEXT:    s_lshr_b32 s4, s4, 16
-; GFX10-NEXT:    s_cmp_lt_u32 s10, s11
+; GFX10-NEXT:    v_pk_sub_u16 v0, s0, s4 clamp
+; GFX10-NEXT:    v_pk_sub_u16 v1, s1, s5 clamp
+; GFX10-NEXT:    v_pk_sub_u16 v2, s2, s6 clamp
+; GFX10-NEXT:    v_pk_sub_u16 v3, s3, s7 clamp
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_cselect_b32 s10, s10, s11
-; GFX10-NEXT:    s_cmp_lt_u32 s9, s4
-; GFX10-NEXT:    s_cselect_b32 s4, s9, s4
-; GFX10-NEXT:    s_and_b32 s11, s5, s8
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s10, s4
-; GFX10-NEXT:    s_lshr_b32 s5, s5, 16
-; GFX10-NEXT:    s_lshr_b32 s10, s4, 16
-; GFX10-NEXT:    s_sub_i32 s0, s0, s4
-; GFX10-NEXT:    s_sub_i32 s4, s9, s10
-; GFX10-NEXT:    s_and_b32 s10, s1, s8
-; GFX10-NEXT:    s_lshr_b32 s9, s1, 16
-; GFX10-NEXT:    s_cmp_lt_u32 s10, s11
-; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s4
-; GFX10-NEXT:    s_cselect_b32 s10, s10, s11
-; GFX10-NEXT:    s_cmp_lt_u32 s9, s5
-; GFX10-NEXT:    s_cselect_b32 s5, s9, s5
-; GFX10-NEXT:    s_and_b32 s11, s6, s8
-; GFX10-NEXT:    s_pack_ll_b32_b16 s5, s10, s5
-; GFX10-NEXT:    s_lshr_b32 s6, s6, 16
-; GFX10-NEXT:    s_lshr_b32 s10, s5, 16
-; GFX10-NEXT:    s_sub_i32 s1, s1, s5
-; GFX10-NEXT:    s_sub_i32 s5, s9, s10
-; GFX10-NEXT:    s_and_b32 s10, s2, s8
-; GFX10-NEXT:    s_lshr_b32 s9, s2, 16
-; GFX10-NEXT:    s_cmp_lt_u32 s10, s11
-; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s1, s5
-; GFX10-NEXT:    s_cselect_b32 s10, s10, s11
-; GFX10-NEXT:    s_cmp_lt_u32 s9, s6
-; GFX10-NEXT:    s_cselect_b32 s6, s9, s6
-; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s10, s6
-; GFX10-NEXT:    s_lshr_b32 s10, s6, 16
-; GFX10-NEXT:    s_sub_i32 s2, s2, s6
-; GFX10-NEXT:    s_sub_i32 s6, s9, s10
-; GFX10-NEXT:    s_and_b32 s10, s3, s8
-; GFX10-NEXT:    s_and_b32 s8, s7, s8
-; GFX10-NEXT:    s_lshr_b32 s9, s3, 16
-; GFX10-NEXT:    s_lshr_b32 s7, s7, 16
-; GFX10-NEXT:    s_cmp_lt_u32 s10, s8
-; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s2, s6
-; GFX10-NEXT:    s_cselect_b32 s8, s10, s8
-; GFX10-NEXT:    s_cmp_lt_u32 s9, s7
-; GFX10-NEXT:    s_cselect_b32 s7, s9, s7
-; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s8, s7
-; GFX10-NEXT:    s_lshr_b32 s5, s4, 16
-; GFX10-NEXT:    s_sub_i32 s3, s3, s4
-; GFX10-NEXT:    s_sub_i32 s4, s9, s5
-; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s3, s4
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX10-NEXT:    v_readfirstlane_b32 s3, v3
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> %lhs, <8 x i16> %rhs)
   %cast = bitcast <8 x i16> %result to <4 x i32>

From dfcc09890a91b1085139fee175936b0e67824e47 Mon Sep 17 00:00:00 2001
From: Frederik Gossen <frgossen@google.com>
Date: Tue, 28 Jul 2020 15:39:49 +0000
Subject: [PATCH 0312/1035] [MLIR][Shape] Lower `shape.const_shape` to
 `tensor_from_elements`

Differential Revision: https://reviews.llvm.org/D82848
---
 .../ShapeToStandard/ShapeToStandard.cpp       | 34 +++++++++++++++++++
 .../ShapeToStandard/shape-to-standard.mlir    | 16 +++++++++
 2 files changed, 50 insertions(+)

diff --git a/mlir/lib/Conversion/ShapeToStandard/ShapeToStandard.cpp b/mlir/lib/Conversion/ShapeToStandard/ShapeToStandard.cpp
index f239d1cfb4f0b..b84b6ba3b5d67 100644
--- a/mlir/lib/Conversion/ShapeToStandard/ShapeToStandard.cpp
+++ b/mlir/lib/Conversion/ShapeToStandard/ShapeToStandard.cpp
@@ -103,6 +103,39 @@ LogicalResult ShapeOfOpConversion::matchAndRewrite(
   return success();
 }
 
+namespace {
+class ConstShapeOpConverter : public OpConversionPattern<ConstShapeOp> {
+public:
+  using OpConversionPattern<ConstShapeOp>::OpConversionPattern;
+
+  LogicalResult
+  matchAndRewrite(ConstShapeOp op, ArrayRef<Value> operands,
+                  ConversionPatternRewriter &rewriter) const override;
+};
+} // namespace
+
+LogicalResult ConstShapeOpConverter::matchAndRewrite(
+    ConstShapeOp op, ArrayRef<Value> operands,
+    ConversionPatternRewriter &rewriter) const {
+
+  // For now, this lowering supports only extent tensors, not `shape.shape`
+  // types.
+  if (op.getType().isa<ShapeType>())
+    return failure();
+
+  auto loc = op.getLoc();
+  SmallVector<Value, 4> extentOperands;
+  for (auto extent : op.shape()) {
+    extentOperands.push_back(
+        rewriter.create<ConstantIndexOp>(loc, extent.getLimitedValue()));
+  }
+  Value tensor = rewriter.create<TensorFromElementsOp>(loc, extentOperands);
+  Type indexTy = rewriter.getIndexType();
+  Type resultTy = RankedTensorType::get({ShapedType::kDynamicSize}, indexTy);
+  rewriter.replaceOpWithNewOp<TensorCastOp>(op, tensor, resultTy);
+  return success();
+}
+
 namespace {
 class GetExtentOpConverter : public OpConversionPattern<GetExtentOp> {
   using OpConversionPattern<GetExtentOp>::OpConversionPattern;
@@ -209,6 +242,7 @@ void mlir::populateShapeToStandardConversionPatterns(
   patterns.insert<
       AnyOpConversion,
       BinaryOpConversion<AddOp, AddIOp>,
+      ConstShapeOpConverter,
       BinaryOpConversion<MulOp, MulIOp>,
       GetExtentOpConverter,
       RankOpConverter,
diff --git a/mlir/test/Conversion/ShapeToStandard/shape-to-standard.mlir b/mlir/test/Conversion/ShapeToStandard/shape-to-standard.mlir
index 9336402d86da4..7f875f3bb19f9 100644
--- a/mlir/test/Conversion/ShapeToStandard/shape-to-standard.mlir
+++ b/mlir/test/Conversion/ShapeToStandard/shape-to-standard.mlir
@@ -111,6 +111,22 @@ func @get_extent_from_extent_tensor(%extents : tensor<?xindex>, %idx : index)
 
 // -----
 
+// Lower `const_shape` to `tensor_from_elements`.
+// CHECK-LABEL: @const_shape
+// CHECK-SAME: () -> tensor<?xindex>
+func @const_shape() -> tensor<?xindex> {
+  // CHECK: %[[C1:.*]] = constant 1 : index
+  // CHECK: %[[C2:.*]] = constant 2 : index
+  // CHECK: %[[C3:.*]] = constant 3 : index
+  // CHECK: %[[TENSOR3:.*]] = tensor_from_elements(%[[C1]], %[[C2]], %[[C3]])
+  // CHECK: %[[RESULT:.*]] = tensor_cast %[[TENSOR3]] : tensor<3xindex> to tensor<?xindex>
+  // CHECK: return %[[RESULT]] : tensor<?xindex>
+  %shape = shape.const_shape [1, 2, 3] : tensor<?xindex>
+  return %shape : tensor<?xindex>
+}
+
+// -----
+
 // Lower `any` to its first operand.
 // CHECK-LABEL: @any_of_three
 // CHECK-SAME:  (%[[A:.*]]: tensor<?xindex>, %[[B:.*]]: tensor<?xindex>, %[[C:.*]]: tensor<?xindex>) -> tensor<?xindex>

From bb23b5cfe0c4391576541ffe8a4f966155d2b608 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Sun, 26 Jul 2020 12:20:29 -0400
Subject: [PATCH 0313/1035] AMDGPU/GlobalISel: Merge identical select cases

---
 llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
index 777c8c6c2ee69..ff83e1ef404cf 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
@@ -2846,6 +2846,8 @@ bool AMDGPUInstructionSelector::select(MachineInstr &I) {
   case TargetOpcode::G_ATOMICRMW_UMIN:
   case TargetOpcode::G_ATOMICRMW_UMAX:
   case TargetOpcode::G_ATOMICRMW_FADD:
+  case AMDGPU::G_AMDGPU_ATOMIC_INC:
+  case AMDGPU::G_AMDGPU_ATOMIC_DEC:
     return selectG_LOAD_ATOMICRMW(I);
   case AMDGPU::G_AMDGPU_ATOMIC_CMPXCHG:
     return selectG_AMDGPU_ATOMIC_CMPXCHG(I);
@@ -2875,10 +2877,6 @@ bool AMDGPUInstructionSelector::select(MachineInstr &I) {
     return selectG_INSERT_VECTOR_ELT(I);
   case TargetOpcode::G_SHUFFLE_VECTOR:
     return selectG_SHUFFLE_VECTOR(I);
-  case AMDGPU::G_AMDGPU_ATOMIC_INC:
-  case AMDGPU::G_AMDGPU_ATOMIC_DEC:
-    initM0(I);
-    return selectImpl(I, *CoverageInfo);
   case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD:
   case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE: {
     const AMDGPU::ImageDimIntrinsicInfo *Intr

From 16bcd54570328c90182edb56d4593ff19207cbc7 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Sun, 26 Jul 2020 10:52:51 -0400
Subject: [PATCH 0314/1035] AMDGPU/GlobalISel: Mark GlobalISel classes as final

---
 llvm/lib/Target/AMDGPU/AMDGPUCallLowering.h            | 2 +-
 llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h     | 2 +-
 llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h           | 2 +-
 llvm/lib/Target/AMDGPU/AMDGPUPostLegalizerCombiner.cpp | 2 +-
 llvm/lib/Target/AMDGPU/AMDGPUPreLegalizerCombiner.cpp  | 2 +-
 llvm/lib/Target/AMDGPU/AMDGPURegBankCombiner.cpp       | 2 +-
 llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.h        | 2 +-
 7 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.h b/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.h
index d11b9801b9716..4d78a4f063209 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.h
@@ -22,7 +22,7 @@ namespace llvm {
 class AMDGPUTargetLowering;
 class MachineInstrBuilder;
 
-class AMDGPUCallLowering: public CallLowering {
+class AMDGPUCallLowering final : public CallLowering {
   void lowerParameterPtr(Register DstReg, MachineIRBuilder &B, Type *ParamTy,
                          uint64_t Offset) const;
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
index 1fe80958917d6..1fb64442347c9 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
@@ -47,7 +47,7 @@ class SIInstrInfo;
 class SIMachineFunctionInfo;
 class SIRegisterInfo;
 
-class AMDGPUInstructionSelector : public InstructionSelector {
+class AMDGPUInstructionSelector final : public InstructionSelector {
 private:
   MachineRegisterInfo *MRI;
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h
index d932cab4659ab..43be02c661ebe 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h
@@ -25,7 +25,7 @@ class LLVMContext;
 class GCNSubtarget;
 
 /// This class provides the information for the target register banks.
-class AMDGPULegalizerInfo : public LegalizerInfo {
+class AMDGPULegalizerInfo final : public LegalizerInfo {
   const GCNSubtarget &ST;
 
 public:
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPostLegalizerCombiner.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPostLegalizerCombiner.cpp
index 098b0e9938861..8a48ea5bd30c5 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPostLegalizerCombiner.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPostLegalizerCombiner.cpp
@@ -234,7 +234,7 @@ namespace {
 #include "AMDGPUGenPostLegalizeGICombiner.inc"
 #undef AMDGPUPOSTLEGALIZERCOMBINERHELPER_GENCOMBINERHELPER_H
 
-class AMDGPUPostLegalizerCombinerInfo : public CombinerInfo {
+class AMDGPUPostLegalizerCombinerInfo final : public CombinerInfo {
   GISelKnownBits *KB;
   MachineDominatorTree *MDT;
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPreLegalizerCombiner.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPreLegalizerCombiner.cpp
index 800ad2039f0e9..aea148e910710 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPreLegalizerCombiner.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPreLegalizerCombiner.cpp
@@ -37,7 +37,7 @@ namespace {
 #include "AMDGPUGenPreLegalizeGICombiner.inc"
 #undef AMDGPUPRELEGALIZERCOMBINERHELPER_GENCOMBINERHELPER_H
 
-class AMDGPUPreLegalizerCombinerInfo : public CombinerInfo {
+class AMDGPUPreLegalizerCombinerInfo final : public CombinerInfo {
   GISelKnownBits *KB;
   MachineDominatorTree *MDT;
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankCombiner.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankCombiner.cpp
index 71d82679b3ff1..a99dbeebfbf8c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankCombiner.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankCombiner.cpp
@@ -39,7 +39,7 @@ namespace {
 #include "AMDGPUGenRegBankGICombiner.inc"
 #undef AMDGPUREGBANKCOMBINERHELPER_GENCOMBINERHELPER_H
 
-class AMDGPURegBankCombinerInfo : public CombinerInfo {
+class AMDGPURegBankCombinerInfo final : public CombinerInfo {
   GISelKnownBits *KB;
   MachineDominatorTree *MDT;
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.h b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.h
index 23a466e9bd85b..2cfb32201a6a3 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.h
@@ -39,7 +39,7 @@ class AMDGPUGenRegisterBankInfo : public RegisterBankInfo {
 #define GET_TARGET_REGBANK_CLASS
 #include "AMDGPUGenRegisterBank.inc"
 };
-class AMDGPURegisterBankInfo : public AMDGPUGenRegisterBankInfo {
+class AMDGPURegisterBankInfo final : public AMDGPUGenRegisterBankInfo {
 public:
   const GCNSubtarget &Subtarget;
   const SIRegisterInfo *TRI;

From 97b5fb78d137a44bec104ba073dd620008ed7abb Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Sat, 25 Jul 2020 16:58:15 -0400
Subject: [PATCH 0315/1035] GlobalISel: Translate llvm.convert.{to|from}.fp16
 intrinsics

I think these were added as a workaround for SelectionDAG lacking half
legalization support in the past. I think they should probably be
removed from the IR, but clang does still have a target control to
emit these instead of the native half fpext/fptrunc.
---
 llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp  | 12 +++++++
 .../irtranslator-convert-fp16-intrinsics.ll   | 31 +++++++++++++++++++
 2 files changed, 43 insertions(+)
 create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-convert-fp16-intrinsics.ll

diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 6433c13e990bf..dba341660a550 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -1543,6 +1543,18 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
     }
     return true;
   }
+  case Intrinsic::convert_from_fp16:
+    // FIXME: This intrinsic should probably be removed from the IR.
+    MIRBuilder.buildFPExt(getOrCreateVReg(CI),
+                          getOrCreateVReg(*CI.getArgOperand(0)),
+                          MachineInstr::copyFlagsFromInstruction(CI));
+    return true;
+  case Intrinsic::convert_to_fp16:
+    // FIXME: This intrinsic should probably be removed from the IR.
+    MIRBuilder.buildFPTrunc(getOrCreateVReg(CI),
+                            getOrCreateVReg(*CI.getArgOperand(0)),
+                            MachineInstr::copyFlagsFromInstruction(CI));
+    return true;
   case Intrinsic::memcpy:
   case Intrinsic::memmove:
   case Intrinsic::memset:
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-convert-fp16-intrinsics.ll b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-convert-fp16-intrinsics.ll
new file mode 100644
index 0000000000000..065a3d8e4dd25
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-convert-fp16-intrinsics.ll
@@ -0,0 +1,31 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -mtriple=aarch64-- -mcpu=falkor -mattr=+lse -O0 -aarch64-enable-atomic-cfg-tidy=0 -stop-after=irtranslator -global-isel -verify-machineinstrs %s -o - | FileCheck %s
+
+define i16 @convert_to_fp16(float %src) {
+  ; CHECK-LABEL: name: convert_to_fp16
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $s0
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $s0
+  ; CHECK:   [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[COPY]](s32)
+  ; CHECK:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FPTRUNC]](s16)
+  ; CHECK:   $w0 = COPY [[ANYEXT]](s32)
+  ; CHECK:   RET_ReallyLR implicit $w0
+  %cvt = call i16 @llvm.convert.to.fp16.f32(float %src)
+  ret i16 %cvt
+}
+
+define float @convert_from_fp16(i16 %src) {
+  ; CHECK-LABEL: name: convert_from_fp16
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $w0
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $w0
+  ; CHECK:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+  ; CHECK:   [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
+  ; CHECK:   $s0 = COPY [[FPEXT]](s32)
+  ; CHECK:   RET_ReallyLR implicit $s0
+  %cvt = call float @llvm.convert.from.fp16.f32(i16 %src)
+  ret float %cvt
+}
+
+declare i16 @llvm.convert.to.fp16.f32(float) nounwind readnone
+declare float @llvm.convert.from.fp16.f32(i16) nounwind readnone

From 736423af53d707e097a174c3a91b75132b8dc6b1 Mon Sep 17 00:00:00 2001
From: Jon Roelofs <jonathan_roelofs@apple.com>
Date: Mon, 27 Jul 2020 12:34:36 -0600
Subject: [PATCH 0316/1035] [OldPM] Print out a bit more when passes lie about
 changing IR

https://reviews.llvm.org/D84686
---
 llvm/lib/IR/LegacyPassManager.cpp | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/llvm/lib/IR/LegacyPassManager.cpp b/llvm/lib/IR/LegacyPassManager.cpp
index c01696e4e575e..96434ae3306b3 100644
--- a/llvm/lib/IR/LegacyPassManager.cpp
+++ b/llvm/lib/IR/LegacyPassManager.cpp
@@ -1586,9 +1586,12 @@ bool FPPassManager::runOnFunction(Function &F) {
 #endif
       LocalChanged |= FP->runOnFunction(F);
 
-#ifdef EXPENSIVE_CHECKS
-      assert((LocalChanged || (RefHash == StructuralHash(F))) &&
-             "Pass modifies its input and doesn't report it.");
+#if defined(EXPENSIVE_CHECKS) && !defined(NDEBUG)
+      if (!LocalChanged && (RefHash != StructuralHash(F))) {
+        llvm::errs() << "Pass modifies its input and doesn't report it: "
+                     << FP->getPassName() << "\n";
+        assert(false && "Pass modifies its input and doesn't report it.");
+      }
 #endif
 
       if (EmitICRemark) {

From 2ca6c422d2d025821390260232307567191a7deb Mon Sep 17 00:00:00 2001
From: Arthur Eubanks <aeubanks@google.com>
Date: Tue, 28 Jul 2020 09:08:08 -0700
Subject: [PATCH 0317/1035] [FunctionAttrs] Rename functionattrs ->
 function-attrs

To match NewPM pass name, and also for readability.
Also rename rpo-functionattrs -> rpo-function-attrs while we're here.

Reviewed By: arsenm

Differential Revision: https://reviews.llvm.org/D84694
---
 llvm/docs/Passes.rst                             |  6 +++---
 llvm/lib/Passes/PassRegistry.def                 |  2 +-
 llvm/lib/Target/README.txt                       |  6 +++---
 llvm/lib/Transforms/IPO/FunctionAttrs.cpp        | 16 +++++++++-------
 llvm/test/Analysis/MemorySSA/pr39197.ll          |  2 +-
 .../TypeBasedAliasAnalysis/functionattrs.ll      |  2 +-
 llvm/test/Analysis/alias-analysis-uses.ll        |  2 +-
 .../check-debugify-preserves-analyses.ll         |  4 ++--
 .../Feature/OperandBundles/function-attrs.ll     |  2 +-
 llvm/test/Feature/OperandBundles/pr26510.ll      |  2 +-
 .../FunctionAttrs/2008-09-03-Mutual.ll           |  2 +-
 .../FunctionAttrs/2008-09-03-ReadNone.ll         |  2 +-
 .../FunctionAttrs/2008-09-03-ReadOnly.ll         |  2 +-
 .../FunctionAttrs/2008-09-13-VolatileRead.ll     |  2 +-
 .../FunctionAttrs/2008-12-29-Constant.ll         |  2 +-
 .../FunctionAttrs/2009-01-02-LocalStores.ll      |  2 +-
 .../FunctionAttrs/2010-10-30-volatile.ll         |  2 +-
 .../Transforms/FunctionAttrs/arg_returned.ll     |  2 +-
 llvm/test/Transforms/FunctionAttrs/assume.ll     |  2 +-
 llvm/test/Transforms/FunctionAttrs/atomic.ll     |  2 +-
 llvm/test/Transforms/FunctionAttrs/comdat-ipo.ll |  2 +-
 llvm/test/Transforms/FunctionAttrs/convergent.ll |  2 +-
 .../FunctionAttrs/incompatible_fn_attrs.ll       |  2 +-
 .../Transforms/FunctionAttrs/int_sideeffect.ll   |  2 +-
 .../Transforms/FunctionAttrs/naked_functions.ll  |  2 +-
 llvm/test/Transforms/FunctionAttrs/nocapture.ll  |  2 +-
 .../FunctionAttrs/nofree-attributor.ll           |  2 +-
 llvm/test/Transforms/FunctionAttrs/nofree.ll     |  2 +-
 .../Transforms/FunctionAttrs/nonnull-global.ll   |  2 +-
 llvm/test/Transforms/FunctionAttrs/nonnull.ll    |  2 +-
 llvm/test/Transforms/FunctionAttrs/norecurse.ll  |  4 ++--
 llvm/test/Transforms/FunctionAttrs/nounwind.ll   |  2 +-
 .../FunctionAttrs/operand-bundles-scc.ll         |  2 +-
 llvm/test/Transforms/FunctionAttrs/optnone.ll    |  2 +-
 .../FunctionAttrs/out-of-bounds-iterator-bug.ll  |  2 +-
 .../Transforms/FunctionAttrs/read-write-scc.ll   |  2 +-
 llvm/test/Transforms/FunctionAttrs/readattrs.ll  |  4 ++--
 llvm/test/Transforms/FunctionAttrs/readnone.ll   |  2 +-
 llvm/test/Transforms/FunctionAttrs/returned.ll   |  2 +-
 llvm/test/Transforms/FunctionAttrs/writeonly.ll  |  2 +-
 .../Transforms/GlobalDCE/crash-assertingvh.ll    |  2 +-
 llvm/test/Transforms/IndVarSimplify/pr38855.ll   |  2 +-
 llvm/test/Transforms/Inline/delete-call.ll       |  2 +-
 .../InstCombine/2009-02-11-NotInitialized.ll     |  2 +-
 .../2012-04-30-LoopUnswitch-LPad-Crash.ll        |  2 +-
 .../Reassociate/reassociate-deadinst.ll          |  2 +-
 .../2012-04-30-LoopUnswitch-LPad-Crash.ll        |  2 +-
 47 files changed, 62 insertions(+), 60 deletions(-)

diff --git a/llvm/docs/Passes.rst b/llvm/docs/Passes.rst
index 9a6c6944b96ef..e45adad98c157 100644
--- a/llvm/docs/Passes.rst
+++ b/llvm/docs/Passes.rst
@@ -522,9 +522,9 @@ instructions that are obviously dead.
 A trivial dead store elimination that only considers basic-block local
 redundant stores.
 
-.. _passes-functionattrs:
+.. _passes-function-attrs:
 
-``-functionattrs``: Deduce function attributes
+``-function-attrs``: Deduce function attributes
 ----------------------------------------------
 
 A simple interprocedural pass which walks the call-graph, looking for functions
@@ -651,7 +651,7 @@ This pass can also simplify calls to specific well-known function calls (e.g.
 runtime library functions).  For example, a call ``exit(3)`` that occurs within
 the ``main()`` function can be transformed into simply ``return 3``. Whether or
 not library calls are simplified is controlled by the
-:ref:`-functionattrs <passes-functionattrs>` pass and LLVM's knowledge of
+:ref:`-function-attrs <passes-function-attrs>` pass and LLVM's knowledge of
 library calls on different targets.
 
 .. _passes-aggressive-instcombine:
diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def
index edaca9ebf6090..11154686fa77a 100644
--- a/llvm/lib/Passes/PassRegistry.def
+++ b/llvm/lib/Passes/PassRegistry.def
@@ -81,7 +81,7 @@ MODULE_PASS("print-lcg-dot", LazyCallGraphDOTPrinterPass(dbgs()))
 MODULE_PASS("print-stack-safety", StackSafetyGlobalPrinterPass(dbgs()))
 MODULE_PASS("rewrite-statepoints-for-gc", RewriteStatepointsForGC())
 MODULE_PASS("rewrite-symbols", RewriteSymbolPass())
-MODULE_PASS("rpo-functionattrs", ReversePostOrderFunctionAttrsPass())
+MODULE_PASS("rpo-function-attrs", ReversePostOrderFunctionAttrsPass())
 MODULE_PASS("sample-profile", SampleProfileLoaderPass())
 MODULE_PASS("scc-oz-module-inliner",
   buildInlinerPipeline(OptimizationLevel::Oz, ThinLTOPhase::None, DebugLogging))
diff --git a/llvm/lib/Target/README.txt b/llvm/lib/Target/README.txt
index 563aee9e1a780..a4876f715c64b 100644
--- a/llvm/lib/Target/README.txt
+++ b/llvm/lib/Target/README.txt
@@ -1541,9 +1541,9 @@ int bar() { return foo("abcd"); }
 
 //===---------------------------------------------------------------------===//
 
-functionattrs doesn't know much about memcpy/memset.  This function should be
+function-attrs doesn't know much about memcpy/memset.  This function should be
 marked readnone rather than readonly, since it only twiddles local memory, but
-functionattrs doesn't handle memset/memcpy/memmove aggressively:
+function-attrs doesn't handle memset/memcpy/memmove aggressively:
 
 struct X { int *p; int *q; };
 int foo() {
@@ -1557,7 +1557,7 @@ int foo() {
 }
 
 This can be seen at:
-$ clang t.c -S -o - -mkernel -O0 -emit-llvm | opt -functionattrs -S
+$ clang t.c -S -o - -mkernel -O0 -emit-llvm | opt -function-attrs -S
 
 
 //===---------------------------------------------------------------------===//
diff --git a/llvm/lib/Transforms/IPO/FunctionAttrs.cpp b/llvm/lib/Transforms/IPO/FunctionAttrs.cpp
index 4baeaa6e16304..5f70f8eaebb65 100644
--- a/llvm/lib/Transforms/IPO/FunctionAttrs.cpp
+++ b/llvm/lib/Transforms/IPO/FunctionAttrs.cpp
@@ -63,7 +63,7 @@
 
 using namespace llvm;
 
-#define DEBUG_TYPE "functionattrs"
+#define DEBUG_TYPE "function-attrs"
 
 STATISTIC(NumReadNone, "Number of functions marked readnone");
 STATISTIC(NumReadOnly, "Number of functions marked readonly");
@@ -1477,11 +1477,11 @@ struct PostOrderFunctionAttrsLegacyPass : public CallGraphSCCPass {
 } // end anonymous namespace
 
 char PostOrderFunctionAttrsLegacyPass::ID = 0;
-INITIALIZE_PASS_BEGIN(PostOrderFunctionAttrsLegacyPass, "functionattrs",
+INITIALIZE_PASS_BEGIN(PostOrderFunctionAttrsLegacyPass, "function-attrs",
                       "Deduce function attributes", false, false)
 INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
 INITIALIZE_PASS_DEPENDENCY(CallGraphWrapperPass)
-INITIALIZE_PASS_END(PostOrderFunctionAttrsLegacyPass, "functionattrs",
+INITIALIZE_PASS_END(PostOrderFunctionAttrsLegacyPass, "function-attrs",
                     "Deduce function attributes", false, false)
 
 Pass *llvm::createPostOrderFunctionAttrsLegacyPass() {
@@ -1542,11 +1542,13 @@ struct ReversePostOrderFunctionAttrsLegacyPass : public ModulePass {
 
 char ReversePostOrderFunctionAttrsLegacyPass::ID = 0;
 
-INITIALIZE_PASS_BEGIN(ReversePostOrderFunctionAttrsLegacyPass, "rpo-functionattrs",
-                      "Deduce function attributes in RPO", false, false)
+INITIALIZE_PASS_BEGIN(ReversePostOrderFunctionAttrsLegacyPass,
+                      "rpo-function-attrs", "Deduce function attributes in RPO",
+                      false, false)
 INITIALIZE_PASS_DEPENDENCY(CallGraphWrapperPass)
-INITIALIZE_PASS_END(ReversePostOrderFunctionAttrsLegacyPass, "rpo-functionattrs",
-                    "Deduce function attributes in RPO", false, false)
+INITIALIZE_PASS_END(ReversePostOrderFunctionAttrsLegacyPass,
+                    "rpo-function-attrs", "Deduce function attributes in RPO",
+                    false, false)
 
 Pass *llvm::createReversePostOrderFunctionAttrsPass() {
   return new ReversePostOrderFunctionAttrsLegacyPass();
diff --git a/llvm/test/Analysis/MemorySSA/pr39197.ll b/llvm/test/Analysis/MemorySSA/pr39197.ll
index 16a321a8108b2..717d92471406a 100644
--- a/llvm/test/Analysis/MemorySSA/pr39197.ll
+++ b/llvm/test/Analysis/MemorySSA/pr39197.ll
@@ -1,4 +1,4 @@
-; RUN: opt -mtriple=s390x-linux-gnu -mcpu=z13 -enable-mssa-loop-dependency -verify-memoryssa -sroa -globalopt -functionattrs -simplifycfg -licm -loop-unswitch %s -S | FileCheck %s
+; RUN: opt -mtriple=s390x-linux-gnu -mcpu=z13 -enable-mssa-loop-dependency -verify-memoryssa -sroa -globalopt -function-attrs -simplifycfg -licm -loop-unswitch %s -S | FileCheck %s
 ; REQUIRES: asserts
 
 target datalayout = "E-m:e-i1:8:16-i8:8:16-i64:64-f128:64-v128:64-a:8:16-n32:64"
diff --git a/llvm/test/Analysis/TypeBasedAliasAnalysis/functionattrs.ll b/llvm/test/Analysis/TypeBasedAliasAnalysis/functionattrs.ll
index ba893fedee269..fca330a1029f3 100644
--- a/llvm/test/Analysis/TypeBasedAliasAnalysis/functionattrs.ll
+++ b/llvm/test/Analysis/TypeBasedAliasAnalysis/functionattrs.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -tbaa -basic-aa -functionattrs -S | FileCheck %s
+; RUN: opt < %s -tbaa -basic-aa -function-attrs -S | FileCheck %s
 
 ; FunctionAttrs should make use of TBAA.
 
diff --git a/llvm/test/Analysis/alias-analysis-uses.ll b/llvm/test/Analysis/alias-analysis-uses.ll
index 4163ec25584c6..8f13148b20117 100644
--- a/llvm/test/Analysis/alias-analysis-uses.ll
+++ b/llvm/test/Analysis/alias-analysis-uses.ll
@@ -1,4 +1,4 @@
-; RUN: opt -debug-pass=Executions -globals-aa -functionattrs -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt -debug-pass=Executions -globals-aa -function-attrs -disable-output < %s 2>&1 | FileCheck %s
 
 ; CHECK: Executing Pass 'Globals Alias Analysis'
 ; CHECK-NOT: Freeing Pass 'Globals Alias Analysis'
diff --git a/llvm/test/DebugInfo/check-debugify-preserves-analyses.ll b/llvm/test/DebugInfo/check-debugify-preserves-analyses.ll
index 08195da75e853..74f564356d2dd 100644
--- a/llvm/test/DebugInfo/check-debugify-preserves-analyses.ll
+++ b/llvm/test/DebugInfo/check-debugify-preserves-analyses.ll
@@ -1,7 +1,7 @@
-; RUN: opt < %s -globals-aa -functionattrs | \
+; RUN: opt < %s -globals-aa -function-attrs | \
 ; RUN:   opt -S -strip -strip-dead-prototypes -strip-named-metadata > %t.no_dbg
 
-; RUN: opt < %s -debugify-each -globals-aa -functionattrs | \
+; RUN: opt < %s -debugify-each -globals-aa -function-attrs | \
 ; RUN:   opt -S -strip -strip-dead-prototypes -strip-named-metadata > %t.with_dbg
 
 ; RUN: diff %t.no_dbg %t.with_dbg
diff --git a/llvm/test/Feature/OperandBundles/function-attrs.ll b/llvm/test/Feature/OperandBundles/function-attrs.ll
index 6e1b255039928..94f21bdffb244 100644
--- a/llvm/test/Feature/OperandBundles/function-attrs.ll
+++ b/llvm/test/Feature/OperandBundles/function-attrs.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -functionattrs < %s | FileCheck %s
+; RUN: opt -S -function-attrs < %s | FileCheck %s
 
 declare void @f_readonly() readonly
 declare void @f_readnone() readnone
diff --git a/llvm/test/Feature/OperandBundles/pr26510.ll b/llvm/test/Feature/OperandBundles/pr26510.ll
index 08bd92aa6fa30..1877f35a40067 100644
--- a/llvm/test/Feature/OperandBundles/pr26510.ll
+++ b/llvm/test/Feature/OperandBundles/pr26510.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -globals-aa -functionattrs < %s | FileCheck %s
+; RUN: opt -S -globals-aa -function-attrs < %s | FileCheck %s
 ; RUN: opt -S -O3 < %s | FileCheck %s
 
 ; Apart from checking for the direct cause of the bug, we also check
diff --git a/llvm/test/Transforms/FunctionAttrs/2008-09-03-Mutual.ll b/llvm/test/Transforms/FunctionAttrs/2008-09-03-Mutual.ll
index 6bbd99951adfc..1d75248f41f8b 100644
--- a/llvm/test/Transforms/FunctionAttrs/2008-09-03-Mutual.ll
+++ b/llvm/test/Transforms/FunctionAttrs/2008-09-03-Mutual.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -functionattrs -S | FileCheck %s
+; RUN: opt < %s -function-attrs -S | FileCheck %s
 ; RUN: opt < %s -passes=function-attrs -S | FileCheck %s
 
 ; CHECK: Function Attrs
diff --git a/llvm/test/Transforms/FunctionAttrs/2008-09-03-ReadNone.ll b/llvm/test/Transforms/FunctionAttrs/2008-09-03-ReadNone.ll
index b3035a67abaca..51a9a04eff366 100644
--- a/llvm/test/Transforms/FunctionAttrs/2008-09-03-ReadNone.ll
+++ b/llvm/test/Transforms/FunctionAttrs/2008-09-03-ReadNone.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basic-aa -functionattrs -S | FileCheck %s
+; RUN: opt < %s -basic-aa -function-attrs -S | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=basic-aa -passes=function-attrs -S | FileCheck %s
 
 @x = global i32 0
diff --git a/llvm/test/Transforms/FunctionAttrs/2008-09-03-ReadOnly.ll b/llvm/test/Transforms/FunctionAttrs/2008-09-03-ReadOnly.ll
index 1df26459e89f4..4ea6fdc87dfa3 100644
--- a/llvm/test/Transforms/FunctionAttrs/2008-09-03-ReadOnly.ll
+++ b/llvm/test/Transforms/FunctionAttrs/2008-09-03-ReadOnly.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basic-aa -functionattrs -S | FileCheck %s
+; RUN: opt < %s -basic-aa -function-attrs -S | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=basic-aa -passes=function-attrs -S | FileCheck %s
 
 ; CHECK: define i32 @f() #0
diff --git a/llvm/test/Transforms/FunctionAttrs/2008-09-13-VolatileRead.ll b/llvm/test/Transforms/FunctionAttrs/2008-09-13-VolatileRead.ll
index 8212e8945ec6f..a296c78570f84 100644
--- a/llvm/test/Transforms/FunctionAttrs/2008-09-13-VolatileRead.ll
+++ b/llvm/test/Transforms/FunctionAttrs/2008-09-13-VolatileRead.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -functionattrs -S | FileCheck %s
+; RUN: opt < %s -function-attrs -S | FileCheck %s
 ; RUN: opt < %s -passes=function-attrs -S | FileCheck %s
 ; PR2792
 
diff --git a/llvm/test/Transforms/FunctionAttrs/2008-12-29-Constant.ll b/llvm/test/Transforms/FunctionAttrs/2008-12-29-Constant.ll
index 69ff6cc0a6eb0..d814666b1b64e 100644
--- a/llvm/test/Transforms/FunctionAttrs/2008-12-29-Constant.ll
+++ b/llvm/test/Transforms/FunctionAttrs/2008-12-29-Constant.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basic-aa -functionattrs -S | FileCheck %s
+; RUN: opt < %s -basic-aa -function-attrs -S | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=basic-aa -passes=function-attrs -S | FileCheck %s
 
 @s = external constant i8		; <i8*> [#uses=1]
diff --git a/llvm/test/Transforms/FunctionAttrs/2009-01-02-LocalStores.ll b/llvm/test/Transforms/FunctionAttrs/2009-01-02-LocalStores.ll
index ce72c41656333..435f7810fbde6 100644
--- a/llvm/test/Transforms/FunctionAttrs/2009-01-02-LocalStores.ll
+++ b/llvm/test/Transforms/FunctionAttrs/2009-01-02-LocalStores.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -functionattrs -S | FileCheck %s
+; RUN: opt < %s -function-attrs -S | FileCheck %s
 ; RUN: opt < %s -passes=function-attrs -S | FileCheck %s
 
 ; CHECK: define i32* @a(i32** nocapture readonly %p)
diff --git a/llvm/test/Transforms/FunctionAttrs/2010-10-30-volatile.ll b/llvm/test/Transforms/FunctionAttrs/2010-10-30-volatile.ll
index b9536dce8a489..c5361b995482a 100644
--- a/llvm/test/Transforms/FunctionAttrs/2010-10-30-volatile.ll
+++ b/llvm/test/Transforms/FunctionAttrs/2010-10-30-volatile.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -functionattrs -S | FileCheck %s
+; RUN: opt < %s -function-attrs -S | FileCheck %s
 ; RUN: opt < %s -passes=function-attrs -S | FileCheck %s
 ; PR8279
 
diff --git a/llvm/test/Transforms/FunctionAttrs/arg_returned.ll b/llvm/test/Transforms/FunctionAttrs/arg_returned.ll
index 0adf91cd9aa1d..9ee90f9719e79 100644
--- a/llvm/test/Transforms/FunctionAttrs/arg_returned.ll
+++ b/llvm/test/Transforms/FunctionAttrs/arg_returned.ll
@@ -1,4 +1,4 @@
-; RUN: opt -functionattrs -S < %s | FileCheck %s --check-prefix=FNATTR
+; RUN: opt -function-attrs -S < %s | FileCheck %s --check-prefix=FNATTR
 ;
 ; Test cases specifically designed for the "returned" argument attribute.
 ; We use FIXME's to indicate problems and missing attributes.
diff --git a/llvm/test/Transforms/FunctionAttrs/assume.ll b/llvm/test/Transforms/FunctionAttrs/assume.ll
index d6296624a2d2f..d94c3705a88e7 100644
--- a/llvm/test/Transforms/FunctionAttrs/assume.ll
+++ b/llvm/test/Transforms/FunctionAttrs/assume.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -o - -functionattrs %s | FileCheck %s
+; RUN: opt -S -o - -function-attrs %s | FileCheck %s
 ; RUN: opt -S -o - -passes=function-attrs %s | FileCheck %s
 
 ; CHECK-NOT: readnone
diff --git a/llvm/test/Transforms/FunctionAttrs/atomic.ll b/llvm/test/Transforms/FunctionAttrs/atomic.ll
index a0a08890cb65a..8112996404a54 100644
--- a/llvm/test/Transforms/FunctionAttrs/atomic.ll
+++ b/llvm/test/Transforms/FunctionAttrs/atomic.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basic-aa -functionattrs -S < %s | FileCheck %s
+; RUN: opt -basic-aa -function-attrs -S < %s | FileCheck %s
 ; RUN: opt -aa-pipeline=basic-aa -passes=function-attrs -S < %s | FileCheck %s
 
 ; Atomic load/store to local doesn't affect whether a function is
diff --git a/llvm/test/Transforms/FunctionAttrs/comdat-ipo.ll b/llvm/test/Transforms/FunctionAttrs/comdat-ipo.ll
index d2f194facba8b..4ad080bff425f 100644
--- a/llvm/test/Transforms/FunctionAttrs/comdat-ipo.ll
+++ b/llvm/test/Transforms/FunctionAttrs/comdat-ipo.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -functionattrs -S | FileCheck %s
+; RUN: opt < %s -function-attrs -S | FileCheck %s
 ; RUN: opt < %s -passes=function-attrs -S | FileCheck %s
 
 ; See PR26774
diff --git a/llvm/test/Transforms/FunctionAttrs/convergent.ll b/llvm/test/Transforms/FunctionAttrs/convergent.ll
index 0e4b7515d0187..8b764f502307f 100644
--- a/llvm/test/Transforms/FunctionAttrs/convergent.ll
+++ b/llvm/test/Transforms/FunctionAttrs/convergent.ll
@@ -1,7 +1,7 @@
 ; FIXME: convert CHECK-INDIRECT into CHECK (and remove -check-prefixes) as soon
 ; FIXME: as new-pass-manager's handling of indirect_non_convergent_call is fixed
 ;
-; RUN: opt -functionattrs -S < %s | FileCheck %s --check-prefixes=CHECK,CHECK-INDIRECT
+; RUN: opt -function-attrs -S < %s | FileCheck %s --check-prefixes=CHECK,CHECK-INDIRECT
 ; RUN: opt -passes=function-attrs -S < %s | FileCheck %s
 
 ; CHECK: Function Attrs
diff --git a/llvm/test/Transforms/FunctionAttrs/incompatible_fn_attrs.ll b/llvm/test/Transforms/FunctionAttrs/incompatible_fn_attrs.ll
index 79af817ff0351..906ae01422c1f 100644
--- a/llvm/test/Transforms/FunctionAttrs/incompatible_fn_attrs.ll
+++ b/llvm/test/Transforms/FunctionAttrs/incompatible_fn_attrs.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -o - -functionattrs %s | FileCheck %s
+; RUN: opt -S -o - -function-attrs %s | FileCheck %s
 ; RUN: opt -S -o - -passes=function-attrs %s | FileCheck %s
 
 ; Verify we remove argmemonly/inaccessiblememonly/inaccessiblemem_or_argmemonly
diff --git a/llvm/test/Transforms/FunctionAttrs/int_sideeffect.ll b/llvm/test/Transforms/FunctionAttrs/int_sideeffect.ll
index 24a145908e00c..4c33116b57c98 100644
--- a/llvm/test/Transforms/FunctionAttrs/int_sideeffect.ll
+++ b/llvm/test/Transforms/FunctionAttrs/int_sideeffect.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S < %s -functionattrs | FileCheck %s
+; RUN: opt -S < %s -function-attrs | FileCheck %s
 ; RUN: opt -S < %s -passes=function-attrs | FileCheck %s
 
 ; CHECK: Function Attrs
diff --git a/llvm/test/Transforms/FunctionAttrs/naked_functions.ll b/llvm/test/Transforms/FunctionAttrs/naked_functions.ll
index c4996d4e7e909..20df047ce0f46 100644
--- a/llvm/test/Transforms/FunctionAttrs/naked_functions.ll
+++ b/llvm/test/Transforms/FunctionAttrs/naked_functions.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -functionattrs %s | FileCheck %s
+; RUN: opt -S -function-attrs %s | FileCheck %s
 ; RUN: opt -S -passes='function-attrs' %s | FileCheck %s
 
 ; Don't change the attributes of parameters of naked functions, in particular
diff --git a/llvm/test/Transforms/FunctionAttrs/nocapture.ll b/llvm/test/Transforms/FunctionAttrs/nocapture.ll
index ba43f9637b885..370a74d469600 100644
--- a/llvm/test/Transforms/FunctionAttrs/nocapture.ll
+++ b/llvm/test/Transforms/FunctionAttrs/nocapture.ll
@@ -1,4 +1,4 @@
-; RUN: opt -functionattrs -S < %s | FileCheck %s --check-prefixes=FNATTR
+; RUN: opt -function-attrs -S < %s | FileCheck %s --check-prefixes=FNATTR
 ; RUN: opt -passes=function-attrs -S < %s | FileCheck %s --check-prefixes=FNATTR
 
 @g = global i32* null		; <i32**> [#uses=1]
diff --git a/llvm/test/Transforms/FunctionAttrs/nofree-attributor.ll b/llvm/test/Transforms/FunctionAttrs/nofree-attributor.ll
index 8ac037e5cd8dc..ef9d086f8f17a 100644
--- a/llvm/test/Transforms/FunctionAttrs/nofree-attributor.ll
+++ b/llvm/test/Transforms/FunctionAttrs/nofree-attributor.ll
@@ -1,4 +1,4 @@
-; RUN: opt -functionattrs --disable-nofree-inference=false -S < %s | FileCheck %s --check-prefix=FNATTR
+; RUN: opt -function-attrs --disable-nofree-inference=false -S < %s | FileCheck %s --check-prefix=FNATTR
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/llvm/test/Transforms/FunctionAttrs/nofree.ll b/llvm/test/Transforms/FunctionAttrs/nofree.ll
index e72ff2f532543..4d36cc82bae2d 100644
--- a/llvm/test/Transforms/FunctionAttrs/nofree.ll
+++ b/llvm/test/Transforms/FunctionAttrs/nofree.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -functionattrs -S | FileCheck %s
+; RUN: opt < %s -function-attrs -S | FileCheck %s
 ; RUN: opt < %s -passes=function-attrs -S | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/FunctionAttrs/nonnull-global.ll b/llvm/test/Transforms/FunctionAttrs/nonnull-global.ll
index d79a7ae290a97..e0c8ae465935d 100644
--- a/llvm/test/Transforms/FunctionAttrs/nonnull-global.ll
+++ b/llvm/test/Transforms/FunctionAttrs/nonnull-global.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -functionattrs %s | FileCheck %s
+; RUN: opt -S -function-attrs %s | FileCheck %s
 ; RUN: opt -S -passes=function-attrs %s | FileCheck %s
 
 @a = external global i8, !absolute_symbol !0
diff --git a/llvm/test/Transforms/FunctionAttrs/nonnull.ll b/llvm/test/Transforms/FunctionAttrs/nonnull.ll
index a71d656882041..9e39580229950 100644
--- a/llvm/test/Transforms/FunctionAttrs/nonnull.ll
+++ b/llvm/test/Transforms/FunctionAttrs/nonnull.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S -functionattrs -enable-nonnull-arg-prop %s | FileCheck %s --check-prefixes=FNATTR
+; RUN: opt -S -function-attrs -enable-nonnull-arg-prop %s | FileCheck %s --check-prefixes=FNATTR
 ; RUN: opt -S -passes=function-attrs -enable-nonnull-arg-prop %s | FileCheck %s --check-prefixes=FNATTR
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/FunctionAttrs/norecurse.ll b/llvm/test/Transforms/FunctionAttrs/norecurse.ll
index ad03694c32850..cc48dda663c5e 100644
--- a/llvm/test/Transforms/FunctionAttrs/norecurse.ll
+++ b/llvm/test/Transforms/FunctionAttrs/norecurse.ll
@@ -1,5 +1,5 @@
-; RUN: opt < %s -basic-aa -functionattrs -rpo-functionattrs -S | FileCheck %s 
-; RUN: opt < %s -aa-pipeline=basic-aa -passes='cgscc(function-attrs),rpo-functionattrs' -S | FileCheck %s
+; RUN: opt < %s -basic-aa -function-attrs -rpo-function-attrs -S | FileCheck %s 
+; RUN: opt < %s -aa-pipeline=basic-aa -passes='cgscc(function-attrs),rpo-function-attrs' -S | FileCheck %s
 
 ; CHECK: Function Attrs
 ; CHECK-SAME: norecurse nounwind readnone
diff --git a/llvm/test/Transforms/FunctionAttrs/nounwind.ll b/llvm/test/Transforms/FunctionAttrs/nounwind.ll
index 6d5e3a2ea5b24..57518f4870cc6 100644
--- a/llvm/test/Transforms/FunctionAttrs/nounwind.ll
+++ b/llvm/test/Transforms/FunctionAttrs/nounwind.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -functionattrs -S | FileCheck %s
+; RUN: opt < %s -function-attrs -S | FileCheck %s
 
 ; TEST 1
 ; CHECK: Function Attrs: norecurse nounwind readnone
diff --git a/llvm/test/Transforms/FunctionAttrs/operand-bundles-scc.ll b/llvm/test/Transforms/FunctionAttrs/operand-bundles-scc.ll
index 4ad195ea2b5ff..78cfd03d5ac5c 100644
--- a/llvm/test/Transforms/FunctionAttrs/operand-bundles-scc.ll
+++ b/llvm/test/Transforms/FunctionAttrs/operand-bundles-scc.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -functionattrs < %s | FileCheck %s
+; RUN: opt -S -function-attrs < %s | FileCheck %s
 ; RUN: opt -S -passes=function-attrs < %s | FileCheck %s
 
 define void @f() {
diff --git a/llvm/test/Transforms/FunctionAttrs/optnone.ll b/llvm/test/Transforms/FunctionAttrs/optnone.ll
index 586a6d4a081f9..b7e9ea3636c3b 100644
--- a/llvm/test/Transforms/FunctionAttrs/optnone.ll
+++ b/llvm/test/Transforms/FunctionAttrs/optnone.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -functionattrs -S | FileCheck %s
+; RUN: opt < %s -function-attrs -S | FileCheck %s
 ; RUN: opt < %s -passes=function-attrs -S | FileCheck %s
 
 @x = global i32 0
diff --git a/llvm/test/Transforms/FunctionAttrs/out-of-bounds-iterator-bug.ll b/llvm/test/Transforms/FunctionAttrs/out-of-bounds-iterator-bug.ll
index f2294fe22ef48..c87e591898952 100644
--- a/llvm/test/Transforms/FunctionAttrs/out-of-bounds-iterator-bug.ll
+++ b/llvm/test/Transforms/FunctionAttrs/out-of-bounds-iterator-bug.ll
@@ -1,4 +1,4 @@
-; RUN: opt -functionattrs -S < %s | FileCheck %s
+; RUN: opt -function-attrs -S < %s | FileCheck %s
 ; RUN: opt -passes=function-attrs -S < %s | FileCheck %s
 
 ; This checks for an iterator wraparound bug in FunctionAttrs.  The previous
diff --git a/llvm/test/Transforms/FunctionAttrs/read-write-scc.ll b/llvm/test/Transforms/FunctionAttrs/read-write-scc.ll
index 319aaf0136233..968a290b988c5 100644
--- a/llvm/test/Transforms/FunctionAttrs/read-write-scc.ll
+++ b/llvm/test/Transforms/FunctionAttrs/read-write-scc.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -functionattrs < %s | FileCheck %s
+; RUN: opt -S -function-attrs < %s | FileCheck %s
 ; RUN: opt -S -passes=function-attrs < %s | FileCheck %s
 
 @i = global i32 0
diff --git a/llvm/test/Transforms/FunctionAttrs/readattrs.ll b/llvm/test/Transforms/FunctionAttrs/readattrs.ll
index e566c96d42b00..ae34219bd011e 100644
--- a/llvm/test/Transforms/FunctionAttrs/readattrs.ll
+++ b/llvm/test/Transforms/FunctionAttrs/readattrs.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -functionattrs -S | FileCheck %s
+; RUN: opt < %s -function-attrs -S | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=basic-aa -passes='cgscc(function-attrs)' -S | FileCheck %s
 
 @x = global i32 0
@@ -130,7 +130,7 @@ declare void @escape_readonly_ptr(i8** %addr, i8* readonly %ptr)
 ; is marked as readnone/only. However, the functions can write the pointer into
 ; %addr, causing the store to write to %escaped_then_written.
 ;
-; FIXME: This test currently exposes a bug in functionattrs!
+; FIXME: This test currently exposes a bug in function-attrs!
 ;
 ; CHECK: define void @unsound_readnone(i8* nocapture readnone %ignored, i8* readnone %escaped_then_written)
 ; CHECK: define void @unsound_readonly(i8* nocapture readnone %ignored, i8* readonly %escaped_then_written)
diff --git a/llvm/test/Transforms/FunctionAttrs/readnone.ll b/llvm/test/Transforms/FunctionAttrs/readnone.ll
index b18aab539b62b..aa074b3da11fa 100644
--- a/llvm/test/Transforms/FunctionAttrs/readnone.ll
+++ b/llvm/test/Transforms/FunctionAttrs/readnone.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -functionattrs -S | FileCheck %s
+; RUN: opt < %s -function-attrs -S | FileCheck %s
 ; RUN: opt < %s -passes=function-attrs -S | FileCheck %s
 
 ; CHECK: define void @bar(i8* nocapture readnone %0)
diff --git a/llvm/test/Transforms/FunctionAttrs/returned.ll b/llvm/test/Transforms/FunctionAttrs/returned.ll
index 04ddb7b5ac03a..451b95074b630 100644
--- a/llvm/test/Transforms/FunctionAttrs/returned.ll
+++ b/llvm/test/Transforms/FunctionAttrs/returned.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -functionattrs -S | FileCheck %s
+; RUN: opt < %s -function-attrs -S | FileCheck %s
 ; RUN: opt < %s -passes=function-attrs -S | FileCheck %s
 
 ; CHECK: define i32 @test1(i32 %p, i32 %q)
diff --git a/llvm/test/Transforms/FunctionAttrs/writeonly.ll b/llvm/test/Transforms/FunctionAttrs/writeonly.ll
index 6514cd9d10641..9be998787466a 100644
--- a/llvm/test/Transforms/FunctionAttrs/writeonly.ll
+++ b/llvm/test/Transforms/FunctionAttrs/writeonly.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -functionattrs         -S | FileCheck %s
+; RUN: opt < %s -function-attrs         -S | FileCheck %s
 ; RUN: opt < %s -passes=function-attrs -S | FileCheck %s
 
 ; CHECK: define void @nouses-argworn-funrn(i32* nocapture readnone %.aaa) #0 {
diff --git a/llvm/test/Transforms/GlobalDCE/crash-assertingvh.ll b/llvm/test/Transforms/GlobalDCE/crash-assertingvh.ll
index 2919999d5e288..08230ef55fa73 100644
--- a/llvm/test/Transforms/GlobalDCE/crash-assertingvh.ll
+++ b/llvm/test/Transforms/GlobalDCE/crash-assertingvh.ll
@@ -3,7 +3,7 @@
 ; to assert when global DCE deletes the body of the function.
 ;
 ; RUN: opt -disable-output < %s -passes='module(function(jump-threading),globaldce)'
-; RUN: opt -disable-output < %s -passes='module(rpo-functionattrs,globaldce)'
+; RUN: opt -disable-output < %s -passes='module(rpo-function-attrs,globaldce)'
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/Transforms/IndVarSimplify/pr38855.ll b/llvm/test/Transforms/IndVarSimplify/pr38855.ll
index 67887f5146c85..4088584f6f0af 100644
--- a/llvm/test/Transforms/IndVarSimplify/pr38855.ll
+++ b/llvm/test/Transforms/IndVarSimplify/pr38855.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S -disable-nounwind-inference=false -inline -functionattrs -indvars  < %s | FileCheck %s
+; RUN: opt -S -disable-nounwind-inference=false -inline -function-attrs -indvars  < %s | FileCheck %s
 
 ; Check that the invalidation happens correctly and the test does not crash.
 define void @f2() {
diff --git a/llvm/test/Transforms/Inline/delete-call.ll b/llvm/test/Transforms/Inline/delete-call.ll
index 7f30ffb306b41..9d6140a3f78e5 100644
--- a/llvm/test/Transforms/Inline/delete-call.ll
+++ b/llvm/test/Transforms/Inline/delete-call.ll
@@ -2,7 +2,7 @@
 ; RUN: opt -S -inline -stats < %s 2>&1 | FileCheck %s
 ; CHECK: Number of functions inlined
 
-; RUN: opt -S -inline -functionattrs -stats < %s 2>&1 | FileCheck -check-prefix=CHECK-FUNCTIONATTRS %s
+; RUN: opt -S -inline -function-attrs -stats < %s 2>&1 | FileCheck -check-prefix=CHECK-FUNCTIONATTRS %s
 ; CHECK-FUNCTIONATTRS: Number of call sites deleted, not inlined
 
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128-n8:16:32"
diff --git a/llvm/test/Transforms/InstCombine/2009-02-11-NotInitialized.ll b/llvm/test/Transforms/InstCombine/2009-02-11-NotInitialized.ll
index b66495d9cbaa3..ceffb8941de86 100644
--- a/llvm/test/Transforms/InstCombine/2009-02-11-NotInitialized.ll
+++ b/llvm/test/Transforms/InstCombine/2009-02-11-NotInitialized.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -inline -instcombine -functionattrs | llvm-dis
+; RUN: opt < %s -inline -instcombine -function-attrs | llvm-dis
 ;
 ; Check that nocapture attributes are added when run after an SCC pass.
 ; PR3520
diff --git a/llvm/test/Transforms/LoopUnswitch/2012-04-30-LoopUnswitch-LPad-Crash.ll b/llvm/test/Transforms/LoopUnswitch/2012-04-30-LoopUnswitch-LPad-Crash.ll
index 67742fcab46ae..14b6484618f17 100644
--- a/llvm/test/Transforms/LoopUnswitch/2012-04-30-LoopUnswitch-LPad-Crash.ll
+++ b/llvm/test/Transforms/LoopUnswitch/2012-04-30-LoopUnswitch-LPad-Crash.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basic-aa -instcombine -inline -functionattrs -licm -loop-unswitch -gvn -verify
+; RUN: opt < %s -basic-aa -instcombine -inline -function-attrs -licm -loop-unswitch -gvn -verify
 ; PR12573
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.7.0"
diff --git a/llvm/test/Transforms/Reassociate/reassociate-deadinst.ll b/llvm/test/Transforms/Reassociate/reassociate-deadinst.ll
index df314d571d37a..9266fce48e8ff 100644
--- a/llvm/test/Transforms/Reassociate/reassociate-deadinst.ll
+++ b/llvm/test/Transforms/Reassociate/reassociate-deadinst.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -inline -functionattrs -reassociate -S | FileCheck %s
+; RUN: opt < %s -inline -function-attrs -reassociate -S | FileCheck %s
 
 ; CHECK-NOT: func1
 ; CHECK-LABEL: main
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/2012-04-30-LoopUnswitch-LPad-Crash.ll b/llvm/test/Transforms/SimpleLoopUnswitch/2012-04-30-LoopUnswitch-LPad-Crash.ll
index c9837c5cc141d..66176ed717532 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/2012-04-30-LoopUnswitch-LPad-Crash.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/2012-04-30-LoopUnswitch-LPad-Crash.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basic-aa -instcombine -inline -functionattrs -licm -simple-loop-unswitch -gvn -verify
+; RUN: opt < %s -basic-aa -instcombine -inline -function-attrs -licm -simple-loop-unswitch -gvn -verify
 ; PR12573
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.7.0"

From 9b1539be406da714ab7cbb379575f67d82b874b2 Mon Sep 17 00:00:00 2001
From: Arthur Eubanks <aeubanks@google.com>
Date: Mon, 27 Jul 2020 11:31:34 -0700
Subject: [PATCH 0318/1035] [NewPM][Sancov] Pin RUN lines with -sancov to
 legacy PM

Since the NPM pass is named sancov-module, not sancov.
This makes all tests under Instrumentation/SanitizerCoverage pass when
-enable-new-pm is on by default.

Reviewed By: vitalybuka

Differential Revision: https://reviews.llvm.org/D84687
---
 .../SanitizerCoverage/abort-in-entry-block.ll               | 2 +-
 .../Instrumentation/SanitizerCoverage/backedge-pruning.ll   | 4 ++--
 llvm/test/Instrumentation/SanitizerCoverage/chains.ll       | 2 +-
 .../SanitizerCoverage/cmp-tracing-api-x86_32.ll             | 2 +-
 .../SanitizerCoverage/cmp-tracing-api-x86_64.ll             | 2 +-
 llvm/test/Instrumentation/SanitizerCoverage/cmp-tracing.ll  | 2 +-
 llvm/test/Instrumentation/SanitizerCoverage/coff-comdat.ll  | 2 +-
 .../SanitizerCoverage/coff-pc-table-inline-8bit-counters.ll | 2 +-
 .../SanitizerCoverage/coff-pc-table-inline-bool-flag.ll     | 2 +-
 .../Instrumentation/SanitizerCoverage/coff-used-ctor.ll     | 2 +-
 .../Instrumentation/SanitizerCoverage/const-cmp-tracing.ll  | 2 +-
 llvm/test/Instrumentation/SanitizerCoverage/coverage-dbg.ll | 2 +-
 llvm/test/Instrumentation/SanitizerCoverage/coverage.ll     | 4 ++--
 .../test/Instrumentation/SanitizerCoverage/coverage2-dbg.ll | 2 +-
 llvm/test/Instrumentation/SanitizerCoverage/div-tracing.ll  | 2 +-
 llvm/test/Instrumentation/SanitizerCoverage/gep-tracing.ll  | 2 +-
 .../SanitizerCoverage/inline-8bit-counters.ll               | 2 +-
 .../Instrumentation/SanitizerCoverage/inline-bool-flag.ll   | 2 +-
 .../SanitizerCoverage/interposable-symbol-nocomdat.ll       | 4 ++--
 llvm/test/Instrumentation/SanitizerCoverage/no-func.ll      | 2 +-
 llvm/test/Instrumentation/SanitizerCoverage/pc-table.ll     | 6 +++---
 .../SanitizerCoverage/postdominator_check.ll                | 4 ++--
 llvm/test/Instrumentation/SanitizerCoverage/seh.ll          | 6 +++---
 .../stack-depth-variable-declared-by-user.ll                | 2 +-
 llvm/test/Instrumentation/SanitizerCoverage/stack-depth.ll  | 4 ++--
 .../Instrumentation/SanitizerCoverage/switch-tracing.ll     | 2 +-
 .../SanitizerCoverage/trace-pc-guard-comdat.ll              | 2 +-
 .../trace-pc-guard-inline-8bit-counters.ll                  | 2 +-
 .../SanitizerCoverage/trace-pc-guard-inline-bool-flag.ll    | 2 +-
 .../SanitizerCoverage/trace-pc-guard-nocomdat.ll            | 2 +-
 .../Instrumentation/SanitizerCoverage/tracing-comdat.ll     | 4 ++--
 llvm/test/Instrumentation/SanitizerCoverage/tracing.ll      | 6 +++---
 .../SanitizerCoverage/unreachable-critedge.ll               | 2 +-
 llvm/test/Instrumentation/SanitizerCoverage/wineh.ll        | 2 +-
 34 files changed, 46 insertions(+), 46 deletions(-)

diff --git a/llvm/test/Instrumentation/SanitizerCoverage/abort-in-entry-block.ll b/llvm/test/Instrumentation/SanitizerCoverage/abort-in-entry-block.ll
index 1a44e9b76f553..79a46d09e447f 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/abort-in-entry-block.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/abort-in-entry-block.ll
@@ -1,5 +1,5 @@
 ; Checks that a function with no-return in the entry block is not instrumented.
-; RUN: opt < %s -sancov -sanitizer-coverage-level=3 -sanitizer-coverage-trace-pc-guard -S | FileCheck %s
+; RUN: opt < %s -sancov -sanitizer-coverage-level=3 -sanitizer-coverage-trace-pc-guard -S -enable-new-pm=0 | FileCheck %s
 ; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=3 -sanitizer-coverage-trace-pc-guard -S | FileCheck %s
 ; CHECK-NOT: call void @__sanitizer_cov_trace_pc_guard
 
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/backedge-pruning.ll b/llvm/test/Instrumentation/SanitizerCoverage/backedge-pruning.ll
index c3a13b090dc0e..6b5ba8bc2cf17 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/backedge-pruning.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/backedge-pruning.ll
@@ -1,6 +1,6 @@
 ; Test -sanitizer-coverage-trace-compares=1 and how it prunes backedge compares.
-; RUN: opt < %s -sancov -sanitizer-coverage-level=1 -sanitizer-coverage-trace-compares=1  -sanitizer-coverage-prune-blocks=1 -S | FileCheck %s --check-prefix=PRUNE
-; RUN: opt < %s -sancov -sanitizer-coverage-level=1 -sanitizer-coverage-trace-compares=1  -sanitizer-coverage-prune-blocks=0 -S | FileCheck %s --check-prefix=NOPRUNE
+; RUN: opt < %s -sancov -sanitizer-coverage-level=1 -sanitizer-coverage-trace-compares=1  -sanitizer-coverage-prune-blocks=1 -S -enable-new-pm=0 | FileCheck %s --check-prefix=PRUNE
+; RUN: opt < %s -sancov -sanitizer-coverage-level=1 -sanitizer-coverage-trace-compares=1  -sanitizer-coverage-prune-blocks=0 -S -enable-new-pm=0 | FileCheck %s --check-prefix=NOPRUNE
 ; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=1 -sanitizer-coverage-trace-compares=1  -sanitizer-coverage-prune-blocks=1 -S | FileCheck %s --check-prefix=PRUNE
 ; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=1 -sanitizer-coverage-trace-compares=1  -sanitizer-coverage-prune-blocks=0 -S | FileCheck %s --check-prefix=NOPRUNE
 
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/chains.ll b/llvm/test/Instrumentation/SanitizerCoverage/chains.ll
index ba83c85c83611..ffb46264b226e 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/chains.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/chains.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -sancov -sanitizer-coverage-level=4 -sanitizer-coverage-trace-pc -sanitizer-coverage-prune-blocks=1  -S | FileCheck %s
+; RUN: opt < %s -sancov -sanitizer-coverage-level=4 -sanitizer-coverage-trace-pc -sanitizer-coverage-prune-blocks=1  -S -enable-new-pm=0 | FileCheck %s
 ; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=4 -sanitizer-coverage-trace-pc -sanitizer-coverage-prune-blocks=1  -S | FileCheck %s
 
 define i32 @blah(i32) #0 {
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/cmp-tracing-api-x86_32.ll b/llvm/test/Instrumentation/SanitizerCoverage/cmp-tracing-api-x86_32.ll
index a0406f833a623..25e7a6ece8444 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/cmp-tracing-api-x86_32.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/cmp-tracing-api-x86_32.ll
@@ -1,5 +1,5 @@
 ; Test -sanitizer-coverage-trace-compares=1 API declarations on a non-x86_64 arch
-; RUN: opt < %s -sancov -sanitizer-coverage-level=1 -sanitizer-coverage-trace-compares=1  -S | FileCheck %s
+; RUN: opt < %s -sancov -sanitizer-coverage-level=1 -sanitizer-coverage-trace-compares=1  -S -enable-new-pm=0 | FileCheck %s
 ; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=1 -sanitizer-coverage-trace-compares=1  -S | FileCheck %s
 
 target triple = "i386-unknown-linux-gnu"
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/cmp-tracing-api-x86_64.ll b/llvm/test/Instrumentation/SanitizerCoverage/cmp-tracing-api-x86_64.ll
index e2645d19a340d..cd18aeed28a18 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/cmp-tracing-api-x86_64.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/cmp-tracing-api-x86_64.ll
@@ -1,5 +1,5 @@
 ; Test -sanitizer-coverage-trace-compares=1 API declarations on x86_64
-; RUN: opt < %s -sancov -sanitizer-coverage-level=1 -sanitizer-coverage-trace-compares=1  -S | FileCheck %s
+; RUN: opt < %s -sancov -sanitizer-coverage-level=1 -sanitizer-coverage-trace-compares=1  -S -enable-new-pm=0 | FileCheck %s
 ; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=1 -sanitizer-coverage-trace-compares=1  -S | FileCheck %s
 
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/cmp-tracing.ll b/llvm/test/Instrumentation/SanitizerCoverage/cmp-tracing.ll
index df329fbe85eca..79c682630e65a 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/cmp-tracing.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/cmp-tracing.ll
@@ -1,5 +1,5 @@
 ; Test -sanitizer-coverage-trace-compares=1
-; RUN: opt < %s -sancov -sanitizer-coverage-level=1 -sanitizer-coverage-trace-compares=1  -S | FileCheck %s
+; RUN: opt < %s -sancov -sanitizer-coverage-level=1 -sanitizer-coverage-trace-compares=1  -S -enable-new-pm=0 | FileCheck %s
 ; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=1 -sanitizer-coverage-trace-compares=1  -S | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/coff-comdat.ll b/llvm/test/Instrumentation/SanitizerCoverage/coff-comdat.ll
index 240c6c50ecd8d..b104796a2ebba 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/coff-comdat.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/coff-comdat.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -sancov -sanitizer-coverage-level=1 -sanitizer-coverage-inline-8bit-counters=1 -sanitizer-coverage-pc-table=1 -S | FileCheck %s
+; RUN: opt < %s -sancov -sanitizer-coverage-level=1 -sanitizer-coverage-inline-8bit-counters=1 -sanitizer-coverage-pc-table=1 -S -enable-new-pm=0 | FileCheck %s
 ; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=1 -sanitizer-coverage-inline-8bit-counters=1 -sanitizer-coverage-pc-table=1 -S | FileCheck %s
 
 ; Make sure we use the right comdat groups for COFF to avoid relocations
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/coff-pc-table-inline-8bit-counters.ll b/llvm/test/Instrumentation/SanitizerCoverage/coff-pc-table-inline-8bit-counters.ll
index 3c279ebeaef34..0134c7ea92c14 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/coff-pc-table-inline-8bit-counters.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/coff-pc-table-inline-8bit-counters.ll
@@ -1,5 +1,5 @@
 ; Checks that the PC and 8-bit Counter Arrays are placed in their own sections in COFF binaries.
-; RUN: opt < %s -sancov -sanitizer-coverage-level=1 -sanitizer-coverage-inline-8bit-counters=1 -sanitizer-coverage-pc-table=1 -S | FileCheck %s
+; RUN: opt < %s -sancov -sanitizer-coverage-level=1 -sanitizer-coverage-inline-8bit-counters=1 -sanitizer-coverage-pc-table=1 -S -enable-new-pm=0 | FileCheck %s
 ; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=1 -sanitizer-coverage-inline-8bit-counters=1 -sanitizer-coverage-pc-table=1 -S | FileCheck %s
 target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-pc-windows-msvc19.14.26433"
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/coff-pc-table-inline-bool-flag.ll b/llvm/test/Instrumentation/SanitizerCoverage/coff-pc-table-inline-bool-flag.ll
index 9fcafa9a8b9a3..d91714ba8fcf6 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/coff-pc-table-inline-bool-flag.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/coff-pc-table-inline-bool-flag.ll
@@ -1,5 +1,5 @@
 ; Checks that the PC and 8-bit Counter Arrays are placed in their own sections in COFF binaries.
-; RUN: opt < %s -sancov -sanitizer-coverage-level=1 -sanitizer-coverage-inline-bool-flag=1 -sanitizer-coverage-pc-table=1 -S | FileCheck %s
+; RUN: opt < %s -sancov -sanitizer-coverage-level=1 -sanitizer-coverage-inline-bool-flag=1 -sanitizer-coverage-pc-table=1 -S -enable-new-pm=0 | FileCheck %s
 ; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=1 -sanitizer-coverage-inline-bool-flag=1 -sanitizer-coverage-pc-table=1 -S | FileCheck %s
 target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-pc-windows-msvc19.14.26433"
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/coff-used-ctor.ll b/llvm/test/Instrumentation/SanitizerCoverage/coff-used-ctor.ll
index d89375af982f2..2484756458860 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/coff-used-ctor.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/coff-used-ctor.ll
@@ -1,5 +1,5 @@
 ; Checks that sancov.module_ctor is marked used.
-; RUN: opt < %s -sancov -sanitizer-coverage-level=1 -sanitizer-coverage-inline-8bit-counters=1 -sanitizer-coverage-pc-table=1 -S | FileCheck %s
+; RUN: opt < %s -sancov -sanitizer-coverage-level=1 -sanitizer-coverage-inline-8bit-counters=1 -sanitizer-coverage-pc-table=1 -S -enable-new-pm=0 | FileCheck %s
 ; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=1 -sanitizer-coverage-inline-8bit-counters=1 -sanitizer-coverage-pc-table=1 -S | FileCheck %s
 target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-pc-windows-msvc19.14.26433"
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/const-cmp-tracing.ll b/llvm/test/Instrumentation/SanitizerCoverage/const-cmp-tracing.ll
index c5fbea3d0a9b7..36919275488b3 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/const-cmp-tracing.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/const-cmp-tracing.ll
@@ -1,5 +1,5 @@
 ; Test -sanitizer-coverage-trace-compares=1
-; RUN: opt < %s -sancov -sanitizer-coverage-level=1 -sanitizer-coverage-trace-compares=1  -S | FileCheck %s
+; RUN: opt < %s -sancov -sanitizer-coverage-level=1 -sanitizer-coverage-trace-compares=1  -S -enable-new-pm=0 | FileCheck %s
 ; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=1 -sanitizer-coverage-trace-compares=1  -S | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/coverage-dbg.ll b/llvm/test/Instrumentation/SanitizerCoverage/coverage-dbg.ll
index f870b6eaba36a..7c7932d430d33 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/coverage-dbg.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/coverage-dbg.ll
@@ -1,6 +1,6 @@
 ; Test that coverage instrumentation does not lose debug location.
 
-; RUN: opt < %s -sancov -sanitizer-coverage-level=1 -S | FileCheck %s
+; RUN: opt < %s -sancov -sanitizer-coverage-level=1 -S -enable-new-pm=0 | FileCheck %s
 ; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=1 -S | FileCheck %s
 
 ; C++ source:
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/coverage.ll b/llvm/test/Instrumentation/SanitizerCoverage/coverage.ll
index bb0bed06a47ef..3b3d8d9f04c48 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/coverage.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/coverage.ll
@@ -1,5 +1,5 @@
-; RUN: opt < %s -sancov -sanitizer-coverage-level=4 -sanitizer-coverage-trace-pc  -S | FileCheck %s --check-prefix=CHECK_TRACE_PC
-; RUN: opt < %s -sancov -sanitizer-coverage-level=3 -sanitizer-coverage-prune-blocks=1 -S | FileCheck %s --check-prefix=CHECKPRUNE
+; RUN: opt < %s -sancov -sanitizer-coverage-level=4 -sanitizer-coverage-trace-pc  -S -enable-new-pm=0 | FileCheck %s --check-prefix=CHECK_TRACE_PC
+; RUN: opt < %s -sancov -sanitizer-coverage-level=3 -sanitizer-coverage-prune-blocks=1 -S -enable-new-pm=0 | FileCheck %s --check-prefix=CHECKPRUNE
 
 ; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=4 -sanitizer-coverage-trace-pc  -S | FileCheck %s --check-prefix=CHECK_TRACE_PC
 ; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=3 -sanitizer-coverage-prune-blocks=1 -S | FileCheck %s --check-prefix=CHECKPRUNE
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/coverage2-dbg.ll b/llvm/test/Instrumentation/SanitizerCoverage/coverage2-dbg.ll
index 11ec286ce1ee0..59db418a6f81a 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/coverage2-dbg.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/coverage2-dbg.ll
@@ -1,6 +1,6 @@
 ; Test that coverage instrumentation does not lose debug location.
 
-; RUN: opt < %s -sancov  -sanitizer-coverage-level=2 -S | FileCheck %s
+; RUN: opt < %s -sancov  -sanitizer-coverage-level=2 -S -enable-new-pm=0 | FileCheck %s
 ; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=2 -S | FileCheck %s
 
 ; C++ source:
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/div-tracing.ll b/llvm/test/Instrumentation/SanitizerCoverage/div-tracing.ll
index 35162bc279450..12394159510a9 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/div-tracing.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/div-tracing.ll
@@ -1,5 +1,5 @@
 ; Test -sanitizer-coverage-trace-divs=1
-; RUN: opt < %s -sancov -sanitizer-coverage-level=1 -sanitizer-coverage-trace-divs=1  -S | FileCheck %s
+; RUN: opt < %s -sancov -sanitizer-coverage-level=1 -sanitizer-coverage-trace-divs=1  -S -enable-new-pm=0 | FileCheck %s
 ; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=1 -sanitizer-coverage-trace-divs=1  -S | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/gep-tracing.ll b/llvm/test/Instrumentation/SanitizerCoverage/gep-tracing.ll
index 04ff415c235a1..ad8544ad8a6f2 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/gep-tracing.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/gep-tracing.ll
@@ -1,5 +1,5 @@
 ; Test -sanitizer-coverage-trace-geps=1
-; RUN: opt < %s -sancov -sanitizer-coverage-level=1 -sanitizer-coverage-trace-geps=1  -S | FileCheck %s
+; RUN: opt < %s -sancov -sanitizer-coverage-level=1 -sanitizer-coverage-trace-geps=1  -S -enable-new-pm=0 | FileCheck %s
 ; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=1 -sanitizer-coverage-trace-geps=1  -S | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/inline-8bit-counters.ll b/llvm/test/Instrumentation/SanitizerCoverage/inline-8bit-counters.ll
index b3269b8799884..775ce4fd772d3 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/inline-8bit-counters.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/inline-8bit-counters.ll
@@ -1,5 +1,5 @@
 ; Test -sanitizer-coverage-inline-8bit-counters=1
-; RUN: opt < %s -sancov -sanitizer-coverage-level=1 -sanitizer-coverage-inline-8bit-counters=1  -S | FileCheck %s
+; RUN: opt < %s -sancov -sanitizer-coverage-level=1 -sanitizer-coverage-inline-8bit-counters=1  -S -enable-new-pm=0 | FileCheck %s
 ; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=1 -sanitizer-coverage-inline-8bit-counters=1  -S | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/inline-bool-flag.ll b/llvm/test/Instrumentation/SanitizerCoverage/inline-bool-flag.ll
index 65bee698e23dd..8b05aac813c6a 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/inline-bool-flag.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/inline-bool-flag.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; Test -sanitizer-coverage-inline-bool-flag=1
-; RUN: opt < %s -sancov -sanitizer-coverage-level=1 -sanitizer-coverage-inline-bool-flag=1  -S | FileCheck %s
+; RUN: opt < %s -sancov -sanitizer-coverage-level=1 -sanitizer-coverage-inline-bool-flag=1  -S -enable-new-pm=0 | FileCheck %s
 ; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=1 -sanitizer-coverage-inline-bool-flag=1  -S | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/interposable-symbol-nocomdat.ll b/llvm/test/Instrumentation/SanitizerCoverage/interposable-symbol-nocomdat.ll
index fcdea5535d4fb..d6a1593c7b3be 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/interposable-symbol-nocomdat.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/interposable-symbol-nocomdat.ll
@@ -1,6 +1,6 @@
 ; Test that interposable symbols do not get put in comdats.
-; RUN: opt < %s -sancov -sanitizer-coverage-level=3 -sanitizer-coverage-trace-pc-guard -mtriple x86_64-linux-gnu -S | FileCheck %s
-; RUN: opt < %s -sancov -sanitizer-coverage-level=3 -sanitizer-coverage-trace-pc-guard -mtriple x86_64-windows-msvc -S | FileCheck %s
+; RUN: opt < %s -sancov -sanitizer-coverage-level=3 -sanitizer-coverage-trace-pc-guard -mtriple x86_64-linux-gnu -S -enable-new-pm=0 | FileCheck %s
+; RUN: opt < %s -sancov -sanitizer-coverage-level=3 -sanitizer-coverage-trace-pc-guard -mtriple x86_64-windows-msvc -S -enable-new-pm=0 | FileCheck %s
 ; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=3 -sanitizer-coverage-trace-pc-guard -mtriple x86_64-linux-gnu -S | FileCheck %s
 ; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=3 -sanitizer-coverage-trace-pc-guard -mtriple x86_64-windows-msvc -S | FileCheck %s
 
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/no-func.ll b/llvm/test/Instrumentation/SanitizerCoverage/no-func.ll
index 6dfecb87d62c8..219b82c9d364b 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/no-func.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/no-func.ll
@@ -1,6 +1,6 @@
 ; Tests that we don't insert __sanitizer_cov_trace_pc_guard_init or some such
 ; when there is no instrumentation.
-; RUN: opt < %s -sancov -sanitizer-coverage-level=3 -sanitizer-coverage-trace-pc-guard  -S | FileCheck %s
+; RUN: opt < %s -sancov -sanitizer-coverage-level=3 -sanitizer-coverage-trace-pc-guard  -S -enable-new-pm=0 | FileCheck %s
 ; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=3 -sanitizer-coverage-trace-pc-guard  -S | FileCheck %s
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/pc-table.ll b/llvm/test/Instrumentation/SanitizerCoverage/pc-table.ll
index 509abab591873..eeeb56bfb2abd 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/pc-table.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/pc-table.ll
@@ -1,7 +1,7 @@
 ; Test -sanitizer-coverage-pc-table=1
-; RUN: opt < %s -sancov -sanitizer-coverage-level=3 -sanitizer-coverage-trace-pc-guard       -sanitizer-coverage-pc-table=1 -S | FileCheck %s
-; RUN: opt < %s -sancov -sanitizer-coverage-level=3 -sanitizer-coverage-inline-8bit-counters -sanitizer-coverage-pc-table=1 -S | FileCheck %s
-; RUN: opt < %s -sancov -sanitizer-coverage-level=3 -sanitizer-coverage-inline-bool-flag -sanitizer-coverage-pc-table=1 -S | FileCheck %s
+; RUN: opt < %s -sancov -sanitizer-coverage-level=3 -sanitizer-coverage-trace-pc-guard       -sanitizer-coverage-pc-table=1 -S -enable-new-pm=0 | FileCheck %s
+; RUN: opt < %s -sancov -sanitizer-coverage-level=3 -sanitizer-coverage-inline-8bit-counters -sanitizer-coverage-pc-table=1 -S -enable-new-pm=0 | FileCheck %s
+; RUN: opt < %s -sancov -sanitizer-coverage-level=3 -sanitizer-coverage-inline-bool-flag -sanitizer-coverage-pc-table=1 -S -enable-new-pm=0 | FileCheck %s
 ; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=3 -sanitizer-coverage-trace-pc-guard       -sanitizer-coverage-pc-table=1 -S | FileCheck %s
 ; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=3 -sanitizer-coverage-inline-8bit-counters -sanitizer-coverage-pc-table=1 -S | FileCheck %s
 ; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=3 -sanitizer-coverage-inline-bool-flag -sanitizer-coverage-pc-table=1 -S | FileCheck %s
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/postdominator_check.ll b/llvm/test/Instrumentation/SanitizerCoverage/postdominator_check.ll
index 50b411de9395a..32479d5f1037d 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/postdominator_check.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/postdominator_check.ll
@@ -1,5 +1,5 @@
-; RUN: opt < %s -sancov -sanitizer-coverage-level=4 -sanitizer-coverage-trace-pc -sanitizer-coverage-prune-blocks=1 -S | FileCheck %s
-; RUN: opt < %s -sancov -sanitizer-coverage-level=4 -sanitizer-coverage-trace-pc -sanitizer-coverage-prune-blocks=0 -S | FileCheck %s --check-prefix=CHECK_NO_PRUNE
+; RUN: opt < %s -sancov -sanitizer-coverage-level=4 -sanitizer-coverage-trace-pc -sanitizer-coverage-prune-blocks=1 -S -enable-new-pm=0 | FileCheck %s
+; RUN: opt < %s -sancov -sanitizer-coverage-level=4 -sanitizer-coverage-trace-pc -sanitizer-coverage-prune-blocks=0 -S -enable-new-pm=0 | FileCheck %s --check-prefix=CHECK_NO_PRUNE
 ; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=4 -sanitizer-coverage-trace-pc -sanitizer-coverage-prune-blocks=1 -S | FileCheck %s
 ; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=4 -sanitizer-coverage-trace-pc -sanitizer-coverage-prune-blocks=0 -S | FileCheck %s --check-prefix=CHECK_NO_PRUNE
 
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/seh.ll b/llvm/test/Instrumentation/SanitizerCoverage/seh.ll
index f88ccbe2d5312..da91f394d2e6c 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/seh.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/seh.ll
@@ -1,6 +1,6 @@
-; RUN: opt < %s -sancov -sanitizer-coverage-level=0 -S | FileCheck %s
-; RUN: opt < %s -sancov -sanitizer-coverage-level=1 -S | FileCheck %s
-; RUN: opt < %s -sancov -sanitizer-coverage-level=2 -S | FileCheck %s
+; RUN: opt < %s -sancov -sanitizer-coverage-level=0 -S -enable-new-pm=0 | FileCheck %s
+; RUN: opt < %s -sancov -sanitizer-coverage-level=1 -S -enable-new-pm=0 | FileCheck %s
+; RUN: opt < %s -sancov -sanitizer-coverage-level=2 -S -enable-new-pm=0 | FileCheck %s
 ; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=0 -S | FileCheck %s
 ; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=1 -S | FileCheck %s
 ; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=2 -S | FileCheck %s
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/stack-depth-variable-declared-by-user.ll b/llvm/test/Instrumentation/SanitizerCoverage/stack-depth-variable-declared-by-user.ll
index b93d6e91a2eb5..f44c3e0c458f5 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/stack-depth-variable-declared-by-user.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/stack-depth-variable-declared-by-user.ll
@@ -1,7 +1,7 @@
 ; Ensure that we terminate with a useful error message (instead of crash) if the
 ; user declares `__sancov_lowest_stack` with an unexpected type.
 ; RUN: not opt < %s -sancov -sanitizer-coverage-level=1 \
-; RUN:         -sanitizer-coverage-stack-depth -S 2>&1 | FileCheck %s
+; RUN:         -sanitizer-coverage-stack-depth -S 2>&1 -enable-new-pm=0 | FileCheck %s
 ; RUN: not opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=1 \
 ; RUN:         -sanitizer-coverage-stack-depth -S 2>&1 | FileCheck %s
 
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/stack-depth.ll b/llvm/test/Instrumentation/SanitizerCoverage/stack-depth.ll
index bba3e3b7d2805..cc696779795e7 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/stack-depth.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/stack-depth.ll
@@ -1,9 +1,9 @@
 ; This check verifies that stack depth instrumentation works correctly.
 ; RUN: opt < %s -sancov -sanitizer-coverage-level=1 \
-; RUN:     -sanitizer-coverage-stack-depth -S | FileCheck %s
+; RUN:     -sanitizer-coverage-stack-depth -S -enable-new-pm=0 | FileCheck %s
 ; RUN: opt < %s -sancov -sanitizer-coverage-level=3 \
 ; RUN:     -sanitizer-coverage-stack-depth -sanitizer-coverage-trace-pc-guard \
-; RUN:     -S | FileCheck %s
+; RUN:     -S -enable-new-pm=0 | FileCheck %s
 ; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=1 \
 ; RUN:     -sanitizer-coverage-stack-depth -S | FileCheck %s
 ; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=3 \
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/switch-tracing.ll b/llvm/test/Instrumentation/SanitizerCoverage/switch-tracing.ll
index 2d4eb4ca1514b..3b5e762956205 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/switch-tracing.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/switch-tracing.ll
@@ -1,5 +1,5 @@
 ; Test -sanitizer-coverage-trace-compares=1 (instrumenting a switch)
-; RUN: opt < %s -sancov -sanitizer-coverage-level=1 -sanitizer-coverage-trace-compares=1  -S | FileCheck %s
+; RUN: opt < %s -sancov -sanitizer-coverage-level=1 -sanitizer-coverage-trace-compares=1  -S -enable-new-pm=0 | FileCheck %s
 ; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=1 -sanitizer-coverage-trace-compares=1  -S | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/trace-pc-guard-comdat.ll b/llvm/test/Instrumentation/SanitizerCoverage/trace-pc-guard-comdat.ll
index a278dbcfdcab4..0765d737f143f 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/trace-pc-guard-comdat.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/trace-pc-guard-comdat.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -sancov -sanitizer-coverage-level=4 -sanitizer-coverage-trace-pc-guard  -S | FileCheck %s --check-prefix=CHECK_TRACE_PC_GUARD
+; RUN: opt < %s -sancov -sanitizer-coverage-level=4 -sanitizer-coverage-trace-pc-guard  -S -enable-new-pm=0 | FileCheck %s --check-prefix=CHECK_TRACE_PC_GUARD
 ; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=4 -sanitizer-coverage-trace-pc-guard  -S | FileCheck %s --check-prefix=CHECK_TRACE_PC_GUARD
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/trace-pc-guard-inline-8bit-counters.ll b/llvm/test/Instrumentation/SanitizerCoverage/trace-pc-guard-inline-8bit-counters.ll
index b3b2cc18916fd..2cce6e76f4f38 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/trace-pc-guard-inline-8bit-counters.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/trace-pc-guard-inline-8bit-counters.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -sancov -sanitizer-coverage-level=1 -sanitizer-coverage-trace-pc-guard -sanitizer-coverage-inline-8bit-counters -S | FileCheck %s
+; RUN: opt < %s -sancov -sanitizer-coverage-level=1 -sanitizer-coverage-trace-pc-guard -sanitizer-coverage-inline-8bit-counters -S -enable-new-pm=0 | FileCheck %s
 ; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=1 -sanitizer-coverage-trace-pc-guard -sanitizer-coverage-inline-8bit-counters -S | FileCheck %s
 
 ; Module ctors should have stable names across modules, not something like
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/trace-pc-guard-inline-bool-flag.ll b/llvm/test/Instrumentation/SanitizerCoverage/trace-pc-guard-inline-bool-flag.ll
index 76f9dc9c1ccbf..14198c77d90b2 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/trace-pc-guard-inline-bool-flag.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/trace-pc-guard-inline-bool-flag.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -sancov -sanitizer-coverage-level=1 -sanitizer-coverage-trace-pc-guard -sanitizer-coverage-inline-bool-flag -S | FileCheck %s
+; RUN: opt < %s -sancov -sanitizer-coverage-level=1 -sanitizer-coverage-trace-pc-guard -sanitizer-coverage-inline-bool-flag -S -enable-new-pm=0 | FileCheck %s
 ; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=1 -sanitizer-coverage-trace-pc-guard -sanitizer-coverage-inline-bool-flag -S | FileCheck %s
 
 ; Module ctors should have stable names across modules, not something like
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/trace-pc-guard-nocomdat.ll b/llvm/test/Instrumentation/SanitizerCoverage/trace-pc-guard-nocomdat.ll
index 1cfbfebdf2689..a1aeb7e5b41c2 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/trace-pc-guard-nocomdat.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/trace-pc-guard-nocomdat.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -sancov -sanitizer-coverage-level=4 -sanitizer-coverage-trace-pc-guard  -S | FileCheck %s --check-prefix=CHECK_TRACE_PC_GUARD
+; RUN: opt < %s -sancov -sanitizer-coverage-level=4 -sanitizer-coverage-trace-pc-guard  -S -enable-new-pm=0 | FileCheck %s --check-prefix=CHECK_TRACE_PC_GUARD
 ; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=4 -sanitizer-coverage-trace-pc-guard  -S | FileCheck %s --check-prefix=CHECK_TRACE_PC_GUARD
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/tracing-comdat.ll b/llvm/test/Instrumentation/SanitizerCoverage/tracing-comdat.ll
index 709108434975d..2f7d468406412 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/tracing-comdat.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/tracing-comdat.ll
@@ -1,7 +1,7 @@
 ; Test that the coverage guards have proper comdat
-; RUN: opt < %s -sancov                    -sanitizer-coverage-level=3 -sanitizer-coverage-trace-pc-guard  -S  | FileCheck %s
+; RUN: opt < %s -sancov                    -sanitizer-coverage-level=3 -sanitizer-coverage-trace-pc-guard  -S  -enable-new-pm=0 | FileCheck %s
 ; Make sure asan does not instrument __sancov_gen_
-; RUN: opt < %s -sancov -asan -asan-module -sanitizer-coverage-level=3 -sanitizer-coverage-trace-pc-guard  -S  | FileCheck %s
+; RUN: opt < %s -sancov -asan -asan-module -sanitizer-coverage-level=3 -sanitizer-coverage-trace-pc-guard  -S  -enable-new-pm=0 | FileCheck %s
 
 ; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=3 -sanitizer-coverage-trace-pc-guard  -S  | FileCheck %s
 ; RUN: opt < %s -passes='module(require<asan-globals-md>,sancov-module,asan-module),function(asan)' -sanitizer-coverage-level=3 -sanitizer-coverage-trace-pc-guard  -S  | FileCheck %s
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/tracing.ll b/llvm/test/Instrumentation/SanitizerCoverage/tracing.ll
index ddc0e354ec59d..75a30d6b2b2b2 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/tracing.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/tracing.ll
@@ -1,7 +1,7 @@
 ; Test -sanitizer-coverage-experimental-tracing
-; RUN: opt < %s -sancov -sanitizer-coverage-level=3 -sanitizer-coverage-trace-pc  -S | FileCheck %s --check-prefix=CHECK_PC
-; RUN: opt < %s -sancov -sanitizer-coverage-level=3 -sanitizer-coverage-trace-pc-guard  -S | FileCheck %s --check-prefix=CHECK_PC_GUARD
-; RUN: opt < %s -sancov -sanitizer-coverage-level=3 -sanitizer-coverage-trace-pc-guard  -S -mtriple=x86_64-apple-macosx | FileCheck %s --check-prefix=CHECK_PC_GUARD_DARWIN
+; RUN: opt < %s -sancov -sanitizer-coverage-level=3 -sanitizer-coverage-trace-pc  -S -enable-new-pm=0 | FileCheck %s --check-prefix=CHECK_PC
+; RUN: opt < %s -sancov -sanitizer-coverage-level=3 -sanitizer-coverage-trace-pc-guard  -S -enable-new-pm=0 | FileCheck %s --check-prefix=CHECK_PC_GUARD
+; RUN: opt < %s -sancov -sanitizer-coverage-level=3 -sanitizer-coverage-trace-pc-guard  -S -mtriple=x86_64-apple-macosx -enable-new-pm=0 | FileCheck %s --check-prefix=CHECK_PC_GUARD_DARWIN
 
 ; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=3 -sanitizer-coverage-trace-pc  -S | FileCheck %s --check-prefix=CHECK_PC
 ; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=3 -sanitizer-coverage-trace-pc-guard  -S | FileCheck %s --check-prefix=CHECK_PC_GUARD
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/unreachable-critedge.ll b/llvm/test/Instrumentation/SanitizerCoverage/unreachable-critedge.ll
index 99e3383db9a7f..bc9083fcf6cac 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/unreachable-critedge.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/unreachable-critedge.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -S -sancov -sanitizer-coverage-level=3 | FileCheck %s
+; RUN: opt < %s -S -sancov -sanitizer-coverage-level=3 -enable-new-pm=0 | FileCheck %s
 ; RUN: opt < %s -S -passes='module(sancov-module)' -sanitizer-coverage-level=3 | FileCheck %s
 
 ; The critical edges to unreachable_bb should not be split.
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/wineh.ll b/llvm/test/Instrumentation/SanitizerCoverage/wineh.ll
index fbe9d89f085ee..63120cc29d65f 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/wineh.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/wineh.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -sancov -sanitizer-coverage-level=3 -sanitizer-coverage-trace-pc -S | FileCheck %s --check-prefix=CHECK
+; RUN: opt < %s -sancov -sanitizer-coverage-level=3 -sanitizer-coverage-trace-pc -S -enable-new-pm=0 | FileCheck %s --check-prefix=CHECK
 ; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=3 -sanitizer-coverage-trace-pc -S | FileCheck %s --check-prefix=CHECK
 
 ; Generated from this C++ source:

From 46ebb619bf0fb98d94175dca2a06ead27318002f Mon Sep 17 00:00:00 2001
From: Wei Wang <apollo.mobility@gmail.com>
Date: Tue, 28 Jul 2020 09:12:31 -0700
Subject: [PATCH 0319/1035] [FIX] Resolve test failure in
 polly/test/ScopInfo/memcpy-raw-source.ll

scoped-noalias -> scoped-noalias-aa

reference: https://reviews.llvm.org/D84542

Reviewed By: aeubanks

Differential Revision: https://reviews.llvm.org/D84720
---
 polly/test/ScopInfo/memcpy-raw-source.ll | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/polly/test/ScopInfo/memcpy-raw-source.ll b/polly/test/ScopInfo/memcpy-raw-source.ll
index 69a12f5cb8346..f558bfe997276 100644
--- a/polly/test/ScopInfo/memcpy-raw-source.ll
+++ b/polly/test/ScopInfo/memcpy-raw-source.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basic-aa -scoped-noalias -tbaa -polly-scops -analyze < %s
+; RUN: opt %loadPolly -basic-aa -scoped-noalias-aa -tbaa -polly-scops -analyze < %s
 ;
 ; Ensure that ScopInfo's alias analysis llvm.memcpy for,
 ; like the AliasSetTracker, preserves bitcasts.

From 8b2fcc42b895575d46dbd9252df566938cf68a69 Mon Sep 17 00:00:00 2001
From: Arthur Eubanks <aeubanks@google.com>
Date: Thu, 23 Jul 2020 14:17:37 -0700
Subject: [PATCH 0320/1035] [CompilerRT] Don't pass global compile test flags
 in non-standalone build

In a build with -DLLVM_ENABLE_LTO=Thin:

$ ninja TSanitizer-x86_64-Test-Nolibc
[1/1] Generating Sanitizer-x86_64-Test-Nolibc
FAILED: projects/compiler-rt/lib/sanitizer_common/tests/Sanitizer-x86_64-Test-Nolibc
sanitizer_nolibc_test_main.x86_64.o: file not recognized: file format not recognized

because -flto=thin is getting passed to the clang_compile step.

For non-standalone builds, global compilation flags shouldn't be passed to compiler-rt tests, only the flags the test specifies.

Reviewed By: vitalybuka

Differential Revision: https://reviews.llvm.org/D84466
---
 .../cmake/Modules/CompilerRTCompile.cmake     | 45 ++++++++++---------
 1 file changed, 25 insertions(+), 20 deletions(-)

diff --git a/compiler-rt/cmake/Modules/CompilerRTCompile.cmake b/compiler-rt/cmake/Modules/CompilerRTCompile.cmake
index 07b589beb2d10..3330038f80688 100644
--- a/compiler-rt/cmake/Modules/CompilerRTCompile.cmake
+++ b/compiler-rt/cmake/Modules/CompilerRTCompile.cmake
@@ -70,29 +70,34 @@ function(clang_compile object_file source)
   if (TARGET CompilerRTUnitTestCheckCxx)
     list(APPEND SOURCE_DEPS CompilerRTUnitTestCheckCxx)
   endif()
-  string(REGEX MATCH "[.](cc|cpp)$" is_cxx ${source_rpath})
-  string(REGEX MATCH "[.](m|mm)$" is_objc ${source_rpath})
-  if(is_cxx)
-    string(REPLACE " " ";" global_flags "${CMAKE_CXX_FLAGS}")
-  else()
-    string(REPLACE " " ";" global_flags "${CMAKE_C_FLAGS}")
-  endif()
+  if(COMPILER_RT_STANDALONE_BUILD)
+    # Only add global flags in standalone build.
+    string(REGEX MATCH "[.](cc|cpp)$" is_cxx ${source_rpath})
+    string(REGEX MATCH "[.](m|mm)$" is_objc ${source_rpath})
+    if(is_cxx)
+      string(REPLACE " " ";" global_flags "${CMAKE_CXX_FLAGS}")
+    else()
+      string(REPLACE " " ";" global_flags "${CMAKE_C_FLAGS}")
+    endif()
 
-  if (MSVC)
-    translate_msvc_cflags(global_flags "${global_flags}")
-  endif()
+    if (MSVC)
+      translate_msvc_cflags(global_flags "${global_flags}")
+    endif()
 
-  if (APPLE)
-    set(global_flags ${OSX_SYSROOT_FLAG} ${global_flags})
-  endif()
-  if (is_objc)
-    list(APPEND global_flags -ObjC)
-  endif()
+    if (APPLE)
+      set(global_flags ${OSX_SYSROOT_FLAG} ${global_flags})
+    endif()
+    if (is_objc)
+      list(APPEND global_flags -ObjC)
+    endif()
 
-  # Ignore unknown warnings. CMAKE_CXX_FLAGS may contain GCC-specific options
-  # which are not supported by Clang.
-  list(APPEND global_flags -Wno-unknown-warning-option)
-  set(compile_flags ${global_flags} ${SOURCE_CFLAGS})
+    # Ignore unknown warnings. CMAKE_CXX_FLAGS may contain GCC-specific options
+    # which are not supported by Clang.
+    list(APPEND global_flags -Wno-unknown-warning-option)
+    set(compile_flags ${global_flags} ${SOURCE_CFLAGS})
+  else()
+    set(compile_flags ${SOURCE_CFLAGS})
+  endif()
   add_custom_command(
     OUTPUT ${object_file}
     COMMAND ${COMPILER_RT_TEST_COMPILER} ${compile_flags} -c

From 3fb0de820796cc6e322c8378713d375d9870a353 Mon Sep 17 00:00:00 2001
From: Julian Lettner <julian.lettner@apple.com>
Date: Fri, 24 Jul 2020 11:24:34 -0700
Subject: [PATCH 0321/1035] [Darwin] Fix OS version checks inside simulators

compiler-rt checks OS versions by querying the Darwin kernel version.
This is not necessarily correct inside the simulators if the simulator
runtime is not aligned with the host macOS.  Let's instead check the
`SIMULATOR_RUNTIME_VERSION` env var.

rdar://63031937

Reviewed By: delcypher

Differential Revision: https://reviews.llvm.org/D83977
---
 .../lib/sanitizer_common/sanitizer_mac.cpp    | 89 +++++++++++++------
 .../tests/sanitizer_mac_test.cpp              | 42 +++++++--
 2 files changed, 95 insertions(+), 36 deletions(-)

diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp
index db8a09e6f0de3..522a909e95285 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp
@@ -606,21 +606,65 @@ HandleSignalMode GetHandleSignalMode(int signum) {
   return result;
 }
 
-// This corresponds to Triple::getMacOSXVersion() in the Clang driver.
-static MacosVersion GetMacosAlignedVersionInternal() {
-  u16 kernel_major = GetDarwinKernelVersion().major;
-  // Darwin 0-3  -> unsupported
-  // Darwin 4-19 -> macOS 10.x
-  // Darwin 20+  -> macOS 11+
-  CHECK_GE(kernel_major, 4);
-  u16 major, minor;
-  if (kernel_major < 20) {
-    major = 10;
-    minor = kernel_major - 4;
+using VersStr = char[64];
+
+static void GetOSVersion(VersStr vers) {
+  uptr len = sizeof(VersStr);
+  if (SANITIZER_IOSSIM) {
+    const char *vers_env = GetEnv("SIMULATOR_RUNTIME_VERSION");
+    if (!vers_env) {
+      Report("ERROR: Running in simulator but SIMULATOR_RUNTIME_VERSION env "
+          "var is not set.\n");
+      Die();
+    }
+    len = internal_strlcpy(vers, vers_env, len);
   } else {
-    major = 11 + kernel_major - 20;
-    minor = 0;
+    int res =
+        internal_sysctlbyname("kern.osproductversion", vers, &len, nullptr, 0);
+    CHECK_EQ(res, 0);
+  }
+  CHECK_LT(len, sizeof(VersStr));
+}
+
+void ParseVersion(const char *vers, u16 *major, u16 *minor) {
+  // Format: <major>.<minor>[.<patch>]\0
+  CHECK_GE(internal_strlen(vers), 3);
+  const char *p = vers;
+  *major = internal_simple_strtoll(p, &p, /*base=*/10);
+  CHECK_EQ(*p, '.');
+  p += 1;
+  *minor = internal_simple_strtoll(p, &p, /*base=*/10);
+}
+
+// Aligned versions example:
+// macOS 10.15 -- iOS 13 -- tvOS 13 -- watchOS 6
+static void MapToMacos(u16 *major, u16 *minor) {
+  if (TARGET_OS_OSX)
+    return;
+
+  if (SANITIZER_IOS || SANITIZER_TVOS)
+    *major += 2;
+  else if (SANITIZER_WATCHOS)
+    *major += 9;
+  else
+    UNREACHABLE("unsupported platform");
+
+  if (*major >= 16) {  // macOS 11+
+    *major -= 5;
+  } else {  // macOS 10.15 and below
+    *minor = *major;
+    *major = 10;
   }
+}
+
+static MacosVersion GetMacosAlignedVersionInternal() {
+  VersStr vers;
+  GetOSVersion(vers);
+
+  u16 major, minor;
+  ParseVersion(vers, &major, &minor);
+  MapToMacos(&major, &minor);
+
   return MacosVersion(major, minor);
 }
 
@@ -639,24 +683,15 @@ MacosVersion GetMacosAlignedVersion() {
   return *reinterpret_cast<MacosVersion *>(&result);
 }
 
-void ParseVersion(const char *vers, u16 *major, u16 *minor) {
-  // Format: <major>.<minor>.<patch>\0
-  CHECK_GE(internal_strlen(vers), 5);
-  const char *p = vers;
-  *major = internal_simple_strtoll(p, &p, /*base=*/10);
-  CHECK_EQ(*p, '.');
-  p += 1;
-  *minor = internal_simple_strtoll(p, &p, /*base=*/10);
-}
-
 DarwinKernelVersion GetDarwinKernelVersion() {
-  char buf[100];
-  size_t len = sizeof(buf);
-  int res = internal_sysctlbyname("kern.osrelease", buf, &len, nullptr, 0);
+  VersStr vers;
+  uptr len = sizeof(VersStr);
+  int res = internal_sysctlbyname("kern.osrelease", vers, &len, nullptr, 0);
   CHECK_EQ(res, 0);
+  CHECK_LT(len, sizeof(VersStr));
 
   u16 major, minor;
-  ParseVersion(buf, &major, &minor);
+  ParseVersion(vers, &major, &minor);
 
   return DarwinKernelVersion(major, minor);
 }
diff --git a/compiler-rt/lib/sanitizer_common/tests/sanitizer_mac_test.cpp b/compiler-rt/lib/sanitizer_common/tests/sanitizer_mac_test.cpp
index c8658ea55d034..090947eceb4a7 100644
--- a/compiler-rt/lib/sanitizer_common/tests/sanitizer_mac_test.cpp
+++ b/compiler-rt/lib/sanitizer_common/tests/sanitizer_mac_test.cpp
@@ -22,6 +22,38 @@
 
 namespace __sanitizer {
 
+void ParseVersion(const char *vers, u16 *major, u16 *minor);
+
+TEST(SanitizerMac, ParseVersion) {
+  u16 major, minor;
+
+  ParseVersion("11.22.33", &major, &minor);
+  EXPECT_EQ(major, 11);
+  EXPECT_EQ(minor, 22);
+
+  ParseVersion("1.2", &major, &minor);
+  EXPECT_EQ(major, 1);
+  EXPECT_EQ(minor, 2);
+}
+
+// TODO(yln): Run sanitizer unit tests for the simulators (rdar://65680742)
+#if SANITIZER_IOSSIM
+TEST(SanitizerMac, GetMacosAlignedVersion) {
+  const char *vers_str;
+  if (SANITIZER_IOS || SANITIZER_TVOS) {
+    vers_str = "13.0";
+  } else if (SANITIZER_WATCHOS) {
+    vers_str = "6.5";
+  } else {
+    FAIL() << "unsupported simulator runtime";
+  }
+  setenv("SIMULATOR_RUNTIME_VERSION", vers_str, /*overwrite=*/1);
+
+  MacosVersion vers = GetMacosAlignedVersion();
+  EXPECT_EQ(vers.major, 10);
+  EXPECT_EQ(vers.minor, 15);
+}
+#else
 TEST(SanitizerMac, GetMacosAlignedVersion) {
   MacosVersion vers = GetMacosAlignedVersion();
   u16 kernel_major = GetDarwinKernelVersion().major;
@@ -31,15 +63,7 @@ TEST(SanitizerMac, GetMacosAlignedVersion) {
   EXPECT_EQ(vers.major, expected_major);
   EXPECT_EQ(vers.minor, expected_minor);
 }
-
-void ParseVersion(const char *vers, u16 *major, u16 *minor);
-
-TEST(SanitizerMac, ParseVersion) {
-  u16 major, minor;
-  ParseVersion("11.22.33", &major, &minor);
-  EXPECT_EQ(major, 11);
-  EXPECT_EQ(minor, 22);
-}
+#endif
 
 TEST(SanitizerMac, GetDarwinKernelVersion) {
   DarwinKernelVersion vers = GetDarwinKernelVersion();

From 0c64233bb7ae760d36a0af3a22324810787fa4ca Mon Sep 17 00:00:00 2001
From: Anna Welker <anna.welker@arm.com>
Date: Tue, 28 Jul 2020 17:28:19 +0100
Subject: [PATCH 0322/1035] [ARM][MVE] Teach MVEGatherScatterLowering to merge
 successive getelementpointers

A patch following up on the introduction of pointer induction variables, adding
a preprocessing step to the address optimisation in the MVEGatherScatterLowering
pass. If the getelementpointer that is the address is itself using a
getelementpointer as base, they will be merged into one by summing up the
offsets, after checking that this will not cause an overflow (this can be
repeated recursively).

Differential Revision: https://reviews.llvm.org/D84027
---
 .../Target/ARM/MVEGatherScatterLowering.cpp   |  243 +++-
 .../CodeGen/Thumb2/mve-gather-ind16-scaled.ll |  424 +++++++
 .../CodeGen/Thumb2/mve-gather-ind32-scaled.ll |   39 +
 .../Thumb2/mve-gather-ind8-unscaled.ll        |  806 +++++++++++++
 .../Thumb2/mve-gather-scatter-ptr-address.ll  | 1012 +++++++++++++++++
 .../Thumb2/mve-scatter-ind16-scaled.ll        |   70 ++
 .../Thumb2/mve-scatter-ind32-scaled.ll        |   52 +
 .../Thumb2/mve-scatter-ind8-unscaled.ll       |  109 ++
 8 files changed, 2718 insertions(+), 37 deletions(-)
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-gather-scatter-ptr-address.ll

diff --git a/llvm/lib/Target/ARM/MVEGatherScatterLowering.cpp b/llvm/lib/Target/ARM/MVEGatherScatterLowering.cpp
index e0ba7f88db951..2746d4d456e4f 100644
--- a/llvm/lib/Target/ARM/MVEGatherScatterLowering.cpp
+++ b/llvm/lib/Target/ARM/MVEGatherScatterLowering.cpp
@@ -84,7 +84,7 @@ class MVEGatherScatterLowering : public FunctionPass {
   // Check for a getelementptr and deduce base and offsets from it, on success
   // returning the base directly and the offsets indirectly using the Offsets
   // argument
-  Value *checkGEP(Value *&Offsets, Type *Ty, GetElementPtrInst *GEP,
+  Value *checkGEP(Value *&Offsets, FixedVectorType *Ty, GetElementPtrInst *GEP,
                   IRBuilder<> &Builder);
   // Compute the scale of this gather/scatter instruction
   int computeScale(unsigned GEPElemSize, unsigned MemoryElemSize);
@@ -132,6 +132,11 @@ class MVEGatherScatterLowering : public FunctionPass {
   Value *tryCreateIncrementingWBGatScat(IntrinsicInst *I, Value *BasePtr,
                                         Value *Ptr, unsigned TypeScale,
                                         IRBuilder<> &Builder);
+
+  // Optimise the base and offsets of the given address
+  bool optimiseAddress(Value *Address, BasicBlock *BB, LoopInfo *LI);
+  // Try to fold consecutive geps together into one
+  Value *foldGEP(GetElementPtrInst *GEP, Value *&Offsets, IRBuilder<> &Builder);
   // Check whether these offsets could be moved out of the loop they're in
   bool optimiseOffsets(Value *Offsets, BasicBlock *BB, LoopInfo *LI);
   // Pushes the given add out of the loop
@@ -167,7 +172,49 @@ bool MVEGatherScatterLowering::isLegalTypeAndAlignment(unsigned NumElements,
   return false;
 }
 
-Value *MVEGatherScatterLowering::checkGEP(Value *&Offsets, Type *Ty,
+bool checkOffsetSize(Value *Offsets, unsigned TargetElemCount) {
+  // Offsets that are not of type <N x i32> are sign extended by the
+  // getelementptr instruction, and MVE gathers/scatters treat the offset as
+  // unsigned. Thus, if the element size is smaller than 32, we can only allow
+  // positive offsets - i.e., the offsets are not allowed to be variables we
+  // can't look into.
+  // Additionally, <N x i32> offsets have to either originate from a zext of a
+  // vector with element types smaller or equal the type of the gather we're
+  // looking at, or consist of constants that we can check are small enough
+  // to fit into the gather type.
+  // Thus we check that 0 < value < 2^TargetElemSize.
+  unsigned TargetElemSize = 128 / TargetElemCount;
+  unsigned OffsetElemSize = cast<FixedVectorType>(Offsets->getType())
+                                ->getElementType()
+                                ->getScalarSizeInBits();
+  if (OffsetElemSize != TargetElemSize || OffsetElemSize != 32) {
+    Constant *ConstOff = dyn_cast<Constant>(Offsets);
+    if (!ConstOff)
+      return false;
+    int64_t TargetElemMaxSize = (1ULL << TargetElemSize);
+    auto CheckValueSize = [TargetElemMaxSize](Value *OffsetElem) {
+      ConstantInt *OConst = dyn_cast<ConstantInt>(OffsetElem);
+      if (!OConst)
+        return false;
+      int SExtValue = OConst->getSExtValue();
+      if (SExtValue >= TargetElemMaxSize || SExtValue < 0)
+        return false;
+      return true;
+    };
+    if (isa<FixedVectorType>(ConstOff->getType())) {
+      for (unsigned i = 0; i < TargetElemCount; i++) {
+        if (!CheckValueSize(ConstOff->getAggregateElement(i)))
+          return false;
+      }
+    } else {
+      if (!CheckValueSize(ConstOff))
+        return false;
+    }
+  }
+  return true;
+}
+
+Value *MVEGatherScatterLowering::checkGEP(Value *&Offsets, FixedVectorType *Ty,
                                           GetElementPtrInst *GEP,
                                           IRBuilder<> &Builder) {
   if (!GEP) {
@@ -178,40 +225,43 @@ Value *MVEGatherScatterLowering::checkGEP(Value *&Offsets, Type *Ty,
   LLVM_DEBUG(dbgs() << "masked gathers/scatters: getelementpointer found."
                     << " Looking at intrinsic for base + vector of offsets\n");
   Value *GEPPtr = GEP->getPointerOperand();
-  if (GEPPtr->getType()->isVectorTy()) {
+  Offsets = GEP->getOperand(1);
+  if (GEPPtr->getType()->isVectorTy() ||
+      !isa<FixedVectorType>(Offsets->getType()))
     return nullptr;
-  }
+
   if (GEP->getNumOperands() != 2) {
     LLVM_DEBUG(dbgs() << "masked gathers/scatters: getelementptr with too many"
                       << " operands. Expanding.\n");
     return nullptr;
   }
   Offsets = GEP->getOperand(1);
+  unsigned OffsetsElemCount =
+      cast<FixedVectorType>(Offsets->getType())->getNumElements();
   // Paranoid check whether the number of parallel lanes is the same
-  assert(cast<FixedVectorType>(Ty)->getNumElements() ==
-         cast<FixedVectorType>(Offsets->getType())->getNumElements());
-  // Only <N x i32> offsets can be integrated into an arm gather, any smaller
-  // type would have to be sign extended by the gep - and arm gathers can only
-  // zero extend. Additionally, the offsets do have to originate from a zext of
-  // a vector with element types smaller or equal the type of the gather we're
-  // looking at
-  if (Offsets->getType()->getScalarSizeInBits() != 32)
-    return nullptr;
-  if (ZExtInst *ZextOffs = dyn_cast<ZExtInst>(Offsets))
+  assert(Ty->getNumElements() == OffsetsElemCount);
+
+  ZExtInst *ZextOffs = dyn_cast<ZExtInst>(Offsets);
+  if (ZextOffs)
     Offsets = ZextOffs->getOperand(0);
-  else if (!(cast<FixedVectorType>(Offsets->getType())->getNumElements() == 4 &&
-             Offsets->getType()->getScalarSizeInBits() == 32))
-    return nullptr;
+  FixedVectorType *OffsetType = cast<FixedVectorType>(Offsets->getType());
+
+  // If the offsets are already being zext-ed to <N x i32>, that relieves us of
+  // having to make sure that they won't overflow.
+  if (!ZextOffs || cast<FixedVectorType>(ZextOffs->getDestTy())
+                           ->getElementType()
+                           ->getScalarSizeInBits() != 32)
+    if (!checkOffsetSize(Offsets, OffsetsElemCount))
+      return nullptr;
 
+  // The offset sizes have been checked; if any truncating or zext-ing is
+  // required to fix them, do that now
   if (Ty != Offsets->getType()) {
-    if ((Ty->getScalarSizeInBits() <
-         Offsets->getType()->getScalarSizeInBits())) {
-      LLVM_DEBUG(dbgs() << "masked gathers/scatters: no correct offset type."
-                        << " Can't create intrinsic.\n");
-      return nullptr;
+    if ((Ty->getElementType()->getScalarSizeInBits() <
+         OffsetType->getElementType()->getScalarSizeInBits())) {
+      Offsets = Builder.CreateTrunc(Offsets, Ty);
     } else {
-      Offsets = Builder.CreateZExt(
-          Offsets, VectorType::getInteger(cast<VectorType>(Ty)));
+      Offsets = Builder.CreateZExt(Offsets, VectorType::getInteger(Ty));
     }
   }
   // If none of the checks failed, return the gep's base pointer
@@ -426,7 +476,8 @@ Value *MVEGatherScatterLowering::tryCreateMaskedGatherOffset(
 
   GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Ptr);
   Value *Offsets;
-  Value *BasePtr = checkGEP(Offsets, ResultTy, GEP, Builder);
+  Value *BasePtr =
+      checkGEP(Offsets, cast<FixedVectorType>(ResultTy), GEP, Builder);
   if (!BasePtr)
     return nullptr;
   // Check whether the offset is a constant increment that could be merged into
@@ -566,7 +617,8 @@ Value *MVEGatherScatterLowering::tryCreateMaskedScatterOffset(
 
   GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Ptr);
   Value *Offsets;
-  Value *BasePtr = checkGEP(Offsets, InputTy, GEP, Builder);
+  Value *BasePtr =
+      checkGEP(Offsets, cast<FixedVectorType>(InputTy), GEP, Builder);
   if (!BasePtr)
     return nullptr;
   // Check whether the offset is a constant increment that could be merged into
@@ -978,6 +1030,128 @@ bool MVEGatherScatterLowering::optimiseOffsets(Value *Offsets, BasicBlock *BB,
   return true;
 }
 
+Value *CheckAndCreateOffsetAdd(Value *X, Value *Y, Value *GEP,
+                               IRBuilder<> &Builder) {
+
+  // Splat the non-vector value to a vector of the given type - if the value is
+  // a constant (and its value isn't too big), we can even use this opportunity
+  // to scale it to the size of the vector elements
+  auto FixSummands = [&Builder](FixedVectorType *&VT, Value *&NonVectorVal) {
+    ConstantInt *Const;
+    if ((Const = dyn_cast<ConstantInt>(NonVectorVal)) &&
+        VT->getElementType() != NonVectorVal->getType()) {
+      unsigned TargetElemSize = VT->getElementType()->getPrimitiveSizeInBits();
+      uint64_t N = Const->getZExtValue();
+      if (N < (unsigned)(1 << (TargetElemSize - 1))) {
+        NonVectorVal = Builder.CreateVectorSplat(
+            VT->getNumElements(), Builder.getIntN(TargetElemSize, N));
+        return;
+      }
+    }
+    NonVectorVal =
+        Builder.CreateVectorSplat(VT->getNumElements(), NonVectorVal);
+  };
+
+  FixedVectorType *XElType = dyn_cast<FixedVectorType>(X->getType());
+  FixedVectorType *YElType = dyn_cast<FixedVectorType>(Y->getType());
+  // If one of X, Y is not a vector, we have to splat it in order
+  // to add the two of them.
+  if (XElType && !YElType) {
+    FixSummands(XElType, Y);
+    YElType = cast<FixedVectorType>(Y->getType());
+  } else if (YElType && !XElType) {
+    FixSummands(YElType, X);
+    XElType = cast<FixedVectorType>(X->getType());
+  }
+  // Check that the summands are of compatible types
+  if (XElType != YElType) {
+    LLVM_DEBUG(dbgs() << "masked gathers/scatters: incompatible gep offsets\n");
+    return nullptr;
+  }
+
+  if (XElType->getElementType()->getScalarSizeInBits() != 32) {
+    // Check that by adding the vectors we do not accidentally
+    // create an overflow
+    Constant *ConstX = dyn_cast<Constant>(X);
+    Constant *ConstY = dyn_cast<Constant>(Y);
+    if (!ConstX || !ConstY)
+      return nullptr;
+    unsigned TargetElemSize = 128 / XElType->getNumElements();
+    for (unsigned i = 0; i < XElType->getNumElements(); i++) {
+      ConstantInt *ConstXEl =
+          dyn_cast<ConstantInt>(ConstX->getAggregateElement(i));
+      ConstantInt *ConstYEl =
+          dyn_cast<ConstantInt>(ConstY->getAggregateElement(i));
+      if (!ConstXEl || !ConstYEl ||
+          ConstXEl->getZExtValue() + ConstYEl->getZExtValue() >=
+              (unsigned)(1 << (TargetElemSize - 1)))
+        return nullptr;
+    }
+  }
+
+  Value *Add = Builder.CreateAdd(X, Y);
+
+  FixedVectorType *GEPType = cast<FixedVectorType>(GEP->getType());
+  if (checkOffsetSize(Add, GEPType->getNumElements()))
+    return Add;
+  else
+    return nullptr;
+}
+
+Value *MVEGatherScatterLowering::foldGEP(GetElementPtrInst *GEP,
+                                         Value *&Offsets,
+                                         IRBuilder<> &Builder) {
+  Value *GEPPtr = GEP->getPointerOperand();
+  Offsets = GEP->getOperand(1);
+  // We only merge geps with constant offsets, because only for those
+  // we can make sure that we do not cause an overflow
+  if (!isa<Constant>(Offsets))
+    return nullptr;
+  GetElementPtrInst *BaseGEP;
+  if ((BaseGEP = dyn_cast<GetElementPtrInst>(GEPPtr))) {
+    // Merge the two geps into one
+    Value *BaseBasePtr = foldGEP(BaseGEP, Offsets, Builder);
+    if (!BaseBasePtr)
+      return nullptr;
+    Offsets =
+        CheckAndCreateOffsetAdd(Offsets, GEP->getOperand(1), GEP, Builder);
+    if (Offsets == nullptr)
+      return nullptr;
+    return BaseBasePtr;
+  }
+  return GEPPtr;
+}
+
+bool MVEGatherScatterLowering::optimiseAddress(Value *Address, BasicBlock *BB,
+                                               LoopInfo *LI) {
+  GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Address);
+  if (!GEP)
+    return false;
+  bool Changed = false;
+  if (GEP->hasOneUse() &&
+      dyn_cast<GetElementPtrInst>(GEP->getPointerOperand())) {
+    IRBuilder<> Builder(GEP->getContext());
+    Builder.SetInsertPoint(GEP);
+    Builder.SetCurrentDebugLocation(GEP->getDebugLoc());
+    Value *Offsets;
+    Value *Base = foldGEP(GEP, Offsets, Builder);
+    // We only want to merge the geps if there is a real chance that they can be
+    // used by an MVE gather; thus the offset has to have the correct size
+    // (always i32 if it is not of vector type) and the base has to be a
+    // pointer.
+    if (Offsets && Base && Base != GEP) {
+      PointerType *BaseType = cast<PointerType>(Base->getType());
+      GetElementPtrInst *NewAddress = GetElementPtrInst::Create(
+          BaseType->getPointerElementType(), Base, Offsets, "gep.merged", GEP);
+      GEP->replaceAllUsesWith(NewAddress);
+      GEP = NewAddress;
+      Changed = true;
+    }
+  }
+  Changed |= optimiseOffsets(GEP->getOperand(1), GEP->getParent(), LI);
+  return Changed;
+}
+
 bool MVEGatherScatterLowering::runOnFunction(Function &F) {
   if (!EnableMaskedGatherScatters)
     return false;
@@ -995,22 +1169,17 @@ bool MVEGatherScatterLowering::runOnFunction(Function &F) {
   for (BasicBlock &BB : F) {
     for (Instruction &I : BB) {
       IntrinsicInst *II = dyn_cast<IntrinsicInst>(&I);
-      if (II && II->getIntrinsicID() == Intrinsic::masked_gather) {
+      if (II && II->getIntrinsicID() == Intrinsic::masked_gather &&
+          isa<FixedVectorType>(II->getType())) {
         Gathers.push_back(II);
-        if (isa<GetElementPtrInst>(II->getArgOperand(0)))
-          Changed |= optimiseOffsets(
-              cast<Instruction>(II->getArgOperand(0))->getOperand(1),
-              II->getParent(), LI);
-      } else if (II && II->getIntrinsicID() == Intrinsic::masked_scatter) {
+        Changed |= optimiseAddress(II->getArgOperand(0), II->getParent(), LI);
+      } else if (II && II->getIntrinsicID() == Intrinsic::masked_scatter &&
+                 isa<FixedVectorType>(II->getArgOperand(0)->getType())) {
         Scatters.push_back(II);
-        if (isa<GetElementPtrInst>(II->getArgOperand(1)))
-          Changed |= optimiseOffsets(
-              cast<Instruction>(II->getArgOperand(1))->getOperand(1),
-              II->getParent(), LI);
+        Changed |= optimiseAddress(II->getArgOperand(1), II->getParent(), LI);
       }
     }
   }
-
   for (unsigned i = 0; i < Gathers.size(); i++) {
     IntrinsicInst *I = Gathers[i];
     Value *L = lowerGather(I);
diff --git a/llvm/test/CodeGen/Thumb2/mve-gather-ind16-scaled.ll b/llvm/test/CodeGen/Thumb2/mve-gather-ind16-scaled.ll
index 30ce13b850736..63990baf7fc84 100644
--- a/llvm/test/CodeGen/Thumb2/mve-gather-ind16-scaled.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-gather-ind16-scaled.ll
@@ -266,6 +266,430 @@ entry:
   ret <8 x i16> %gather
 }
 
+define arm_aapcs_vfpcc <8 x i16> @scaled_v8i16_i16_2gep(i16* %base, <8 x i16>* %offptr) {
+; CHECK-LABEL: scaled_v8i16_i16_2gep:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, r5, r7, lr}
+; CHECK-NEXT:    push {r4, r5, r7, lr}
+; CHECK-NEXT:    vldrh.s32 q0, [r1]
+; CHECK-NEXT:    vmov.i32 q1, #0x28
+; CHECK-NEXT:    vldrh.s32 q2, [r1, #8]
+; CHECK-NEXT:    vshl.i32 q0, q0, #1
+; CHECK-NEXT:    vadd.i32 q0, q0, r0
+; CHECK-NEXT:    vshl.i32 q2, q2, #1
+; CHECK-NEXT:    vadd.i32 q0, q0, q1
+; CHECK-NEXT:    vadd.i32 q2, q2, r0
+; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    vadd.i32 q1, q2, q1
+; CHECK-NEXT:    vmov r3, s3
+; CHECK-NEXT:    vmov r5, s1
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov r1, s5
+; CHECK-NEXT:    vmov r4, s7
+; CHECK-NEXT:    ldrh.w r12, [r2]
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    ldrh.w lr, [r3]
+; CHECK-NEXT:    vmov r3, s6
+; CHECK-NEXT:    ldrh r5, [r5]
+; CHECK-NEXT:    ldrh r0, [r0]
+; CHECK-NEXT:    ldrh r1, [r1]
+; CHECK-NEXT:    ldrh r4, [r4]
+; CHECK-NEXT:    ldrh r2, [r2]
+; CHECK-NEXT:    ldrh r3, [r3]
+; CHECK-NEXT:    vmov.16 q0[0], r2
+; CHECK-NEXT:    vmov.16 q0[1], r5
+; CHECK-NEXT:    vmov.16 q0[2], r12
+; CHECK-NEXT:    vmov.16 q0[3], lr
+; CHECK-NEXT:    vmov.16 q0[4], r0
+; CHECK-NEXT:    vmov.16 q0[5], r1
+; CHECK-NEXT:    vmov.16 q0[6], r3
+; CHECK-NEXT:    vmov.16 q0[7], r4
+; CHECK-NEXT:    pop {r4, r5, r7, pc}
+entry:
+  %offs = load <8 x i16>, <8 x i16>* %offptr, align 2
+  %ptrs = getelementptr inbounds i16, i16* %base, <8 x i16> %offs
+  %ptrs2 = getelementptr inbounds i16, <8 x i16*> %ptrs, i16 20
+  %gather = call <8 x i16> @llvm.masked.gather.v8i16.v8p0i16(<8 x i16*> %ptrs2, i32 2, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i16> undef)
+  ret <8 x i16> %gather
+}
+
+define arm_aapcs_vfpcc <8 x i16> @scaled_v8i16_i16_2gep2(i16* %base, <8 x i16>* %offptr) {
+; CHECK-LABEL: scaled_v8i16_i16_2gep2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    adr r1, .LCPI14_0
+; CHECK-NEXT:    vldrw.u32 q1, [r1]
+; CHECK-NEXT:    vldrh.u16 q0, [r0, q1, uxtw #1]
+; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI14_0:
+; CHECK-NEXT:    .short 20 @ 0x14
+; CHECK-NEXT:    .short 23 @ 0x17
+; CHECK-NEXT:    .short 26 @ 0x1a
+; CHECK-NEXT:    .short 29 @ 0x1d
+; CHECK-NEXT:    .short 32 @ 0x20
+; CHECK-NEXT:    .short 35 @ 0x23
+; CHECK-NEXT:    .short 38 @ 0x26
+; CHECK-NEXT:    .short 41 @ 0x29
+entry:
+  %ptrs = getelementptr inbounds i16, i16* %base, <8 x i16> <i16 0, i16 3, i16 6, i16 9, i16 12, i16 15, i16 18, i16 21>
+  %ptrs2 = getelementptr inbounds i16,<8 x i16*> %ptrs, i16 20
+  %gather = call <8 x i16> @llvm.masked.gather.v8i16.v8p0i16(<8 x i16*> %ptrs2, i32 2, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i16> undef)
+  ret <8 x i16> %gather
+}
+
+define arm_aapcs_vfpcc <8 x i16> @scaled_v8i16_i16_biggep(i16* %base) {
+; CHECK-LABEL: scaled_v8i16_i16_biggep:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    adr r1, .LCPI15_0
+; CHECK-NEXT:    vldrw.u32 q1, [r1]
+; CHECK-NEXT:    vldrh.u16 q0, [r0, q1, uxtw #1]
+; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI15_0:
+; CHECK-NEXT:    .short 20 @ 0x14
+; CHECK-NEXT:    .short 23 @ 0x17
+; CHECK-NEXT:    .short 26 @ 0x1a
+; CHECK-NEXT:    .short 29 @ 0x1d
+; CHECK-NEXT:    .short 32 @ 0x20
+; CHECK-NEXT:    .short 35 @ 0x23
+; CHECK-NEXT:    .short 38 @ 0x26
+; CHECK-NEXT:    .short 41 @ 0x29
+entry:
+  %ptrs = getelementptr inbounds i16, i16* %base, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
+  %ptrs2 = getelementptr inbounds i16,<8 x i16*> %ptrs, i32 20
+  %gather = call <8 x i16> @llvm.masked.gather.v8i16.v8p0i16(<8 x i16*> %ptrs2, i32 2, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i16> undef)
+  ret <8 x i16> %gather
+}
+
+define arm_aapcs_vfpcc <8 x i16> @scaled_v8i16_i16_biggep2(i16* %base) {
+; CHECK-LABEL: scaled_v8i16_i16_biggep2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    adr r1, .LCPI16_0
+; CHECK-NEXT:    vldrw.u32 q1, [r1]
+; CHECK-NEXT:    vldrh.u16 q0, [r0, q1, uxtw #1]
+; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI16_0:
+; CHECK-NEXT:    .short 0 @ 0x0
+; CHECK-NEXT:    .short 3 @ 0x3
+; CHECK-NEXT:    .short 6 @ 0x6
+; CHECK-NEXT:    .short 9 @ 0x9
+; CHECK-NEXT:    .short 12 @ 0xc
+; CHECK-NEXT:    .short 15 @ 0xf
+; CHECK-NEXT:    .short 18 @ 0x12
+; CHECK-NEXT:    .short 21 @ 0x15
+entry:
+  %ptrs = getelementptr inbounds i16, i16* %base, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
+  %gather = call <8 x i16> @llvm.masked.gather.v8i16.v8p0i16(<8 x i16*> %ptrs, i32 2, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i16> undef)
+  ret <8 x i16> %gather
+}
+
+define arm_aapcs_vfpcc <8 x i16> @scaled_v8i16_i16_biggep3(i16* %base) {
+; CHECK-LABEL: scaled_v8i16_i16_biggep3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, r5, r6, lr}
+; CHECK-NEXT:    push {r4, r5, r6, lr}
+; CHECK-NEXT:    adr r1, .LCPI17_0
+; CHECK-NEXT:    adr.w r12, .LCPI17_1
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r12]
+; CHECK-NEXT:    vadd.i32 q0, q0, r0
+; CHECK-NEXT:    vadd.i32 q1, q1, r0
+; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    vmov r2, s3
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    vmov r5, s1
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov r4, s7
+; CHECK-NEXT:    ldrh.w lr, [r1]
+; CHECK-NEXT:    vmov r1, s5
+; CHECK-NEXT:    ldrh r6, [r2]
+; CHECK-NEXT:    vmov r2, s6
+; CHECK-NEXT:    ldrh r3, [r3]
+; CHECK-NEXT:    ldrh r5, [r5]
+; CHECK-NEXT:    vmov.16 q0[0], r3
+; CHECK-NEXT:    ldrh r0, [r0]
+; CHECK-NEXT:    vmov.16 q0[1], r5
+; CHECK-NEXT:    ldrh r4, [r4]
+; CHECK-NEXT:    vmov.16 q0[2], lr
+; CHECK-NEXT:    vmov.16 q0[3], r6
+; CHECK-NEXT:    vmov.16 q0[4], r0
+; CHECK-NEXT:    ldrh r1, [r1]
+; CHECK-NEXT:    ldrh r2, [r2]
+; CHECK-NEXT:    vmov.16 q0[5], r1
+; CHECK-NEXT:    vmov.16 q0[6], r2
+; CHECK-NEXT:    vmov.16 q0[7], r4
+; CHECK-NEXT:    pop {r4, r5, r6, pc}
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI17_0:
+; CHECK-NEXT:    .long 131072 @ 0x20000
+; CHECK-NEXT:    .long 131078 @ 0x20006
+; CHECK-NEXT:    .long 131084 @ 0x2000c
+; CHECK-NEXT:    .long 131090 @ 0x20012
+; CHECK-NEXT:  .LCPI17_1:
+; CHECK-NEXT:    .long 131096 @ 0x20018
+; CHECK-NEXT:    .long 131102 @ 0x2001e
+; CHECK-NEXT:    .long 131108 @ 0x20024
+; CHECK-NEXT:    .long 131114 @ 0x2002a
+entry:
+  %ptrs = getelementptr inbounds i16, i16* %base, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
+  %ptrs2 = getelementptr inbounds i16,<8 x i16*> %ptrs, i32 65536
+  %gather = call <8 x i16> @llvm.masked.gather.v8i16.v8p0i16(<8 x i16*> %ptrs2, i32 2, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i16> undef)
+  ret <8 x i16> %gather
+}
+
+define arm_aapcs_vfpcc <8 x i16> @scaled_v8i16_i16_biggep4(i16* %base) {
+; CHECK-LABEL: scaled_v8i16_i16_biggep4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, r5, r6, lr}
+; CHECK-NEXT:    push {r4, r5, r6, lr}
+; CHECK-NEXT:    adr r1, .LCPI18_0
+; CHECK-NEXT:    adr.w r12, .LCPI18_1
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r12]
+; CHECK-NEXT:    vadd.i32 q0, q0, r0
+; CHECK-NEXT:    vadd.i32 q1, q1, r0
+; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    vmov r2, s3
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    vmov r5, s1
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov r4, s7
+; CHECK-NEXT:    ldrh.w lr, [r1]
+; CHECK-NEXT:    vmov r1, s5
+; CHECK-NEXT:    ldrh r6, [r2]
+; CHECK-NEXT:    vmov r2, s6
+; CHECK-NEXT:    ldrh r3, [r3]
+; CHECK-NEXT:    ldrh r5, [r5]
+; CHECK-NEXT:    vmov.16 q0[0], r3
+; CHECK-NEXT:    ldrh r0, [r0]
+; CHECK-NEXT:    vmov.16 q0[1], r5
+; CHECK-NEXT:    ldrh r4, [r4]
+; CHECK-NEXT:    vmov.16 q0[2], lr
+; CHECK-NEXT:    vmov.16 q0[3], r6
+; CHECK-NEXT:    vmov.16 q0[4], r0
+; CHECK-NEXT:    ldrh r1, [r1]
+; CHECK-NEXT:    ldrh r2, [r2]
+; CHECK-NEXT:    vmov.16 q0[5], r1
+; CHECK-NEXT:    vmov.16 q0[6], r2
+; CHECK-NEXT:    vmov.16 q0[7], r4
+; CHECK-NEXT:    pop {r4, r5, r6, pc}
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI18_0:
+; CHECK-NEXT:    .long 0 @ 0x0
+; CHECK-NEXT:    .long 6 @ 0x6
+; CHECK-NEXT:    .long 12 @ 0xc
+; CHECK-NEXT:    .long 18 @ 0x12
+; CHECK-NEXT:  .LCPI18_1:
+; CHECK-NEXT:    .long 24 @ 0x18
+; CHECK-NEXT:    .long 131072 @ 0x20000
+; CHECK-NEXT:    .long 36 @ 0x24
+; CHECK-NEXT:    .long 42 @ 0x2a
+entry:
+  %ptrs = getelementptr inbounds i16, i16* %base, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 65536, i32 18, i32 21>
+  %gather = call <8 x i16> @llvm.masked.gather.v8i16.v8p0i16(<8 x i16*> %ptrs, i32 2, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i16> undef)
+  ret <8 x i16> %gather
+}
+
+define arm_aapcs_vfpcc <8 x i16> @scaled_v8i16_i16_biggep5(<8 x i16*> %base) {
+; CHECK-LABEL: scaled_v8i16_i16_biggep5:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, r5, r7, lr}
+; CHECK-NEXT:    push {r4, r5, r7, lr}
+; CHECK-NEXT:    vmov.i32 q2, #0x20000
+; CHECK-NEXT:    vadd.i32 q0, q0, q2
+; CHECK-NEXT:    vadd.i32 q1, q1, q2
+; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmov r5, s1
+; CHECK-NEXT:    vmov r3, s4
+; CHECK-NEXT:    vmov r4, s7
+; CHECK-NEXT:    ldrh.w r12, [r0]
+; CHECK-NEXT:    vmov r0, s5
+; CHECK-NEXT:    ldrh.w lr, [r1]
+; CHECK-NEXT:    vmov r1, s6
+; CHECK-NEXT:    ldrh r2, [r2]
+; CHECK-NEXT:    ldrh r5, [r5]
+; CHECK-NEXT:    vmov.16 q0[0], r2
+; CHECK-NEXT:    ldrh r3, [r3]
+; CHECK-NEXT:    vmov.16 q0[1], r5
+; CHECK-NEXT:    ldrh r4, [r4]
+; CHECK-NEXT:    vmov.16 q0[2], r12
+; CHECK-NEXT:    vmov.16 q0[3], lr
+; CHECK-NEXT:    vmov.16 q0[4], r3
+; CHECK-NEXT:    ldrh r0, [r0]
+; CHECK-NEXT:    ldrh r1, [r1]
+; CHECK-NEXT:    vmov.16 q0[5], r0
+; CHECK-NEXT:    vmov.16 q0[6], r1
+; CHECK-NEXT:    vmov.16 q0[7], r4
+; CHECK-NEXT:    pop {r4, r5, r7, pc}
+entry:
+  %ptrs2 = getelementptr inbounds i16,<8 x i16*> %base, i32 65536
+  %gather = call <8 x i16> @llvm.masked.gather.v8i16.v8p0i16(<8 x i16*> %ptrs2, i32 2, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i16> undef)
+  ret <8 x i16> %gather
+}
+
+define arm_aapcs_vfpcc <8 x i16> @scaled_v8i16_i16_biggep6(i16* %base) {
+; CHECK-LABEL: scaled_v8i16_i16_biggep6:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, r5, r6, lr}
+; CHECK-NEXT:    push {r4, r5, r6, lr}
+; CHECK-NEXT:    adr r1, .LCPI20_0
+; CHECK-NEXT:    adr.w r12, .LCPI20_1
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r12]
+; CHECK-NEXT:    vadd.i32 q0, q0, r0
+; CHECK-NEXT:    vadd.i32 q1, q1, r0
+; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    vmov r2, s3
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    vmov r5, s1
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov r4, s7
+; CHECK-NEXT:    ldrh.w lr, [r1]
+; CHECK-NEXT:    vmov r1, s5
+; CHECK-NEXT:    ldrh r6, [r2]
+; CHECK-NEXT:    vmov r2, s6
+; CHECK-NEXT:    ldrh r3, [r3]
+; CHECK-NEXT:    ldrh r5, [r5]
+; CHECK-NEXT:    vmov.16 q0[0], r3
+; CHECK-NEXT:    ldrh r0, [r0]
+; CHECK-NEXT:    vmov.16 q0[1], r5
+; CHECK-NEXT:    ldrh r4, [r4]
+; CHECK-NEXT:    vmov.16 q0[2], lr
+; CHECK-NEXT:    vmov.16 q0[3], r6
+; CHECK-NEXT:    vmov.16 q0[4], r0
+; CHECK-NEXT:    ldrh r1, [r1]
+; CHECK-NEXT:    ldrh r2, [r2]
+; CHECK-NEXT:    vmov.16 q0[5], r1
+; CHECK-NEXT:    vmov.16 q0[6], r2
+; CHECK-NEXT:    vmov.16 q0[7], r4
+; CHECK-NEXT:    pop {r4, r5, r6, pc}
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI20_0:
+; CHECK-NEXT:    .long 2 @ 0x2
+; CHECK-NEXT:    .long 8 @ 0x8
+; CHECK-NEXT:    .long 14 @ 0xe
+; CHECK-NEXT:    .long 20 @ 0x14
+; CHECK-NEXT:  .LCPI20_1:
+; CHECK-NEXT:    .long 131074 @ 0x20002
+; CHECK-NEXT:    .long 32 @ 0x20
+; CHECK-NEXT:    .long 38 @ 0x26
+; CHECK-NEXT:    .long 44 @ 0x2c
+entry:
+  %ptrs = getelementptr inbounds i16, i16* %base, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 65536, i32 15, i32 18, i32 21>
+  %ptrs2 = getelementptr inbounds i16,<8 x i16*> %ptrs, i32 1
+  %gather = call <8 x i16> @llvm.masked.gather.v8i16.v8p0i16(<8 x i16*> %ptrs2, i32 2, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i16> undef)
+  ret <8 x i16> %gather
+}
+
+define arm_aapcs_vfpcc <8 x i16> @scaled_v8i16_i16_biggep7(i16* %base, <8 x i16>* %offptr) {
+; CHECK-LABEL: scaled_v8i16_i16_biggep7:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, r5, r6, lr}
+; CHECK-NEXT:    push {r4, r5, r6, lr}
+; CHECK-NEXT:    adr r1, .LCPI21_0
+; CHECK-NEXT:    adr.w r12, .LCPI21_1
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r12]
+; CHECK-NEXT:    vadd.i32 q0, q0, r0
+; CHECK-NEXT:    vadd.i32 q1, q1, r0
+; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    vmov r2, s3
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    vmov r5, s1
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov r4, s7
+; CHECK-NEXT:    ldrh.w lr, [r1]
+; CHECK-NEXT:    vmov r1, s5
+; CHECK-NEXT:    ldrh r6, [r2]
+; CHECK-NEXT:    vmov r2, s6
+; CHECK-NEXT:    ldrh r3, [r3]
+; CHECK-NEXT:    ldrh r5, [r5]
+; CHECK-NEXT:    vmov.16 q0[0], r3
+; CHECK-NEXT:    ldrh r0, [r0]
+; CHECK-NEXT:    vmov.16 q0[1], r5
+; CHECK-NEXT:    ldrh r4, [r4]
+; CHECK-NEXT:    vmov.16 q0[2], lr
+; CHECK-NEXT:    vmov.16 q0[3], r6
+; CHECK-NEXT:    vmov.16 q0[4], r0
+; CHECK-NEXT:    ldrh r1, [r1]
+; CHECK-NEXT:    ldrh r2, [r2]
+; CHECK-NEXT:    vmov.16 q0[5], r1
+; CHECK-NEXT:    vmov.16 q0[6], r2
+; CHECK-NEXT:    vmov.16 q0[7], r4
+; CHECK-NEXT:    pop {r4, r5, r6, pc}
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI21_0:
+; CHECK-NEXT:    .long 128 @ 0x80
+; CHECK-NEXT:    .long 1206 @ 0x4b6
+; CHECK-NEXT:    .long 1212 @ 0x4bc
+; CHECK-NEXT:    .long 1218 @ 0x4c2
+; CHECK-NEXT:  .LCPI21_1:
+; CHECK-NEXT:    .long 1224 @ 0x4c8
+; CHECK-NEXT:    .long 1230 @ 0x4ce
+; CHECK-NEXT:    .long 1236 @ 0x4d4
+; CHECK-NEXT:    .long 1242 @ 0x4da
+entry:
+  %ptrs = getelementptr inbounds i16, i16* %base, <8 x i16> <i16 65000, i16 3, i16 6, i16 9, i16 12, i16 15, i16 18, i16 21>
+  %ptrs2 = getelementptr inbounds i16,<8 x i16*> %ptrs, i16 600
+  %gather = call <8 x i16> @llvm.masked.gather.v8i16.v8p0i16(<8 x i16*> %ptrs2, i32 2, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i16> undef)
+  ret <8 x i16> %gather
+}
+
+define arm_aapcs_vfpcc <8 x i16> @scaled_v8i16_i16_basei32(i32* %base, <8 x i16>* %offptr) {
+; CHECK-LABEL: scaled_v8i16_i16_basei32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, r5, r7, lr}
+; CHECK-NEXT:    push {r4, r5, r7, lr}
+; CHECK-NEXT:    vldrh.u32 q0, [r1]
+; CHECK-NEXT:    vldrh.u32 q1, [r1, #8]
+; CHECK-NEXT:    vshl.i32 q0, q0, #2
+; CHECK-NEXT:    vshl.i32 q1, q1, #2
+; CHECK-NEXT:    vadd.i32 q0, q0, r0
+; CHECK-NEXT:    vadd.i32 q1, q1, r0
+; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    vmov r3, s3
+; CHECK-NEXT:    vmov r5, s1
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov r1, s5
+; CHECK-NEXT:    vmov r4, s7
+; CHECK-NEXT:    ldrh.w r12, [r2]
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    ldrh.w lr, [r3]
+; CHECK-NEXT:    vmov r3, s6
+; CHECK-NEXT:    ldrh r5, [r5]
+; CHECK-NEXT:    ldrh r0, [r0]
+; CHECK-NEXT:    ldrh r1, [r1]
+; CHECK-NEXT:    ldrh r4, [r4]
+; CHECK-NEXT:    ldrh r2, [r2]
+; CHECK-NEXT:    ldrh r3, [r3]
+; CHECK-NEXT:    vmov.16 q0[0], r2
+; CHECK-NEXT:    vmov.16 q0[1], r5
+; CHECK-NEXT:    vmov.16 q0[2], r12
+; CHECK-NEXT:    vmov.16 q0[3], lr
+; CHECK-NEXT:    vmov.16 q0[4], r0
+; CHECK-NEXT:    vmov.16 q0[5], r1
+; CHECK-NEXT:    vmov.16 q0[6], r3
+; CHECK-NEXT:    vmov.16 q0[7], r4
+; CHECK-NEXT:    pop {r4, r5, r7, pc}
+entry:
+  %offs = load <8 x i16>, <8 x i16>* %offptr, align 2
+  %offs.zext = zext <8 x i16> %offs to <8 x i32>
+  %ptrs = getelementptr inbounds i32, i32* %base, <8 x i32> %offs.zext
+  %ptrs.cast = bitcast <8 x i32*> %ptrs to <8 x i16*>
+  %gather = call <8 x i16> @llvm.masked.gather.v8i16.v8p0i16(<8 x i16*> %ptrs.cast, i32 2, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i16> undef)
+  ret <8 x i16> %gather
+}
+
 declare <8 x i8> @llvm.masked.gather.v8i8.v8p0i8(<8 x i8*>, i32, <8 x i1>, <8 x i8>) #1
 declare <8 x i16> @llvm.masked.gather.v8i16.v8p0i16(<8 x i16*>, i32, <8 x i1>, <8 x i16>) #1
 declare <8 x half> @llvm.masked.gather.v8f16.v8p0f16(<8 x half*>, i32, <8 x i1>, <8 x half>) #1
diff --git a/llvm/test/CodeGen/Thumb2/mve-gather-ind32-scaled.ll b/llvm/test/CodeGen/Thumb2/mve-gather-ind32-scaled.ll
index c674ffbf51bd3..4c32200ee4a53 100644
--- a/llvm/test/CodeGen/Thumb2/mve-gather-ind32-scaled.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-gather-ind32-scaled.ll
@@ -294,6 +294,45 @@ entry:
   ret <4 x i32> %gather.sext
 }
 
+define arm_aapcs_vfpcc <4 x i32> @scaled_i32_i32_2gep(i32* %base, <4 x i32>* %offptr) {
+; CHECK-LABEL: scaled_i32_i32_2gep:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q1, [r1]
+; CHECK-NEXT:    vmov.i32 q0, #0x14
+; CHECK-NEXT:    vshl.i32 q1, q1, #2
+; CHECK-NEXT:    vadd.i32 q1, q1, r0
+; CHECK-NEXT:    vadd.i32 q1, q1, q0
+; CHECK-NEXT:    vldrw.u32 q0, [q1]
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <4 x i32>, <4 x i32>* %offptr, align 4
+  %ptrs = getelementptr inbounds i32, i32* %base, <4 x i32> %offs
+  %ptrs2 = getelementptr inbounds i32, <4 x i32*> %ptrs, i32 5
+  %gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %ptrs2, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
+  ret <4 x i32> %gather
+}
+
+define arm_aapcs_vfpcc <4 x i32> @scaled_i32_i32_2gep2(i32* %base, <4 x i32>* %offptr) {
+; CHECK-LABEL: scaled_i32_i32_2gep2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    adr r1, .LCPI21_0
+; CHECK-NEXT:    vldrw.u32 q1, [r1]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, q1, uxtw #2]
+; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI21_0:
+; CHECK-NEXT:    .long 5 @ 0x5
+; CHECK-NEXT:    .long 8 @ 0x8
+; CHECK-NEXT:    .long 11 @ 0xb
+; CHECK-NEXT:    .long 14 @ 0xe
+entry:
+  %ptrs = getelementptr inbounds i32, i32* %base, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
+  %ptrs2 = getelementptr inbounds i32, <4 x i32*> %ptrs, i32 5
+  %gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %ptrs2, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
+  ret <4 x i32> %gather
+}
+
 declare <4 x i8>  @llvm.masked.gather.v4i8.v4p0i8(<4 x i8*>, i32, <4 x i1>, <4 x i8>)
 declare <4 x i16> @llvm.masked.gather.v4i16.v4p0i16(<4 x i16*>, i32, <4 x i1>, <4 x i16>)
 declare <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*>, i32, <4 x i1>, <4 x i32>)
diff --git a/llvm/test/CodeGen/Thumb2/mve-gather-ind8-unscaled.ll b/llvm/test/CodeGen/Thumb2/mve-gather-ind8-unscaled.ll
index 60bffc5a31dbd..c7d29af67b5b1 100644
--- a/llvm/test/CodeGen/Thumb2/mve-gather-ind8-unscaled.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-gather-ind8-unscaled.ll
@@ -365,6 +365,812 @@ entry:
   ret <16 x i8> %gather
 }
 
+define arm_aapcs_vfpcc <16 x i8> @unscaled_v16i8_i8_2gep(i8* %base, <16 x i8>* %offptr) {
+; CHECK-LABEL: unscaled_v16i8_i8_2gep:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, r5, r6, r7, lr}
+; CHECK-NEXT:    push {r4, r5, r6, r7, lr}
+; CHECK-NEXT:    vldrb.s32 q0, [r1, #12]
+; CHECK-NEXT:    vmov.i32 q2, #0x5
+; CHECK-NEXT:    vadd.i32 q0, q0, r0
+; CHECK-NEXT:    vadd.i32 q0, q0, q2
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    ldrb r3, [r2]
+; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    ldrb.w r12, [r2]
+; CHECK-NEXT:    vmov r2, s3
+; CHECK-NEXT:    ldrb.w lr, [r2]
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    vldrb.s32 q0, [r1, #8]
+; CHECK-NEXT:    vadd.i32 q0, q0, r0
+; CHECK-NEXT:    vadd.i32 q1, q0, q2
+; CHECK-NEXT:    vldrb.s32 q0, [r1]
+; CHECK-NEXT:    vmov r6, s4
+; CHECK-NEXT:    vadd.i32 q0, q0, r0
+; CHECK-NEXT:    vmov r4, s6
+; CHECK-NEXT:    vadd.i32 q3, q0, q2
+; CHECK-NEXT:    vmov r5, s12
+; CHECK-NEXT:    vmov r7, s15
+; CHECK-NEXT:    ldrb r2, [r2]
+; CHECK-NEXT:    ldrb r6, [r6]
+; CHECK-NEXT:    ldrb r4, [r4]
+; CHECK-NEXT:    ldrb r5, [r5]
+; CHECK-NEXT:    ldrb r7, [r7]
+; CHECK-NEXT:    vmov.8 q0[0], r5
+; CHECK-NEXT:    vmov r5, s13
+; CHECK-NEXT:    ldrb r5, [r5]
+; CHECK-NEXT:    vmov.8 q0[1], r5
+; CHECK-NEXT:    vmov r5, s14
+; CHECK-NEXT:    vldrb.s32 q3, [r1, #4]
+; CHECK-NEXT:    vadd.i32 q3, q3, r0
+; CHECK-NEXT:    vadd.i32 q2, q3, q2
+; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    ldrb r5, [r5]
+; CHECK-NEXT:    vmov.8 q0[2], r5
+; CHECK-NEXT:    vmov r5, s7
+; CHECK-NEXT:    vmov.8 q0[3], r7
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[4], r0
+; CHECK-NEXT:    vmov r0, s9
+; CHECK-NEXT:    ldrb r5, [r5]
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[5], r0
+; CHECK-NEXT:    vmov r0, s10
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[6], r0
+; CHECK-NEXT:    vmov r0, s11
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[7], r0
+; CHECK-NEXT:    vmov r0, s5
+; CHECK-NEXT:    vmov.8 q0[8], r6
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[9], r0
+; CHECK-NEXT:    vmov.8 q0[10], r4
+; CHECK-NEXT:    vmov.8 q0[11], r5
+; CHECK-NEXT:    vmov.8 q0[12], r3
+; CHECK-NEXT:    vmov.8 q0[13], r2
+; CHECK-NEXT:    vmov.8 q0[14], r12
+; CHECK-NEXT:    vmov.8 q0[15], lr
+; CHECK-NEXT:    pop {r4, r5, r6, r7, pc}
+entry:
+	%offs = load <16 x i8>, <16 x i8>* %offptr, align 1
+	%ptrs = getelementptr inbounds i8, i8* %base, <16 x i8> %offs
+	%ptrs2 = getelementptr inbounds i8, <16 x i8*> %ptrs, i8 5
+	%gather = call <16 x i8> @llvm.masked.gather.v16i8.v16p0i8(<16 x i8*> %ptrs2, i32 1, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i8> undef)
+	ret <16 x i8> %gather
+}
+
+
+define arm_aapcs_vfpcc <16 x i8> @unscaled_v16i8_i8_2gep2(i8* %base, <16 x i8>* %offptr) {
+; CHECK-LABEL: unscaled_v16i8_i8_2gep2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    adr r1, .LCPI8_0
+; CHECK-NEXT:    vldrw.u32 q1, [r1]
+; CHECK-NEXT:    vldrb.u8 q0, [r0, q1]
+; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI8_0:
+; CHECK-NEXT:    .byte 5 @ 0x5
+; CHECK-NEXT:    .byte 8 @ 0x8
+; CHECK-NEXT:    .byte 11 @ 0xb
+; CHECK-NEXT:    .byte 14 @ 0xe
+; CHECK-NEXT:    .byte 17 @ 0x11
+; CHECK-NEXT:    .byte 20 @ 0x14
+; CHECK-NEXT:    .byte 23 @ 0x17
+; CHECK-NEXT:    .byte 26 @ 0x1a
+; CHECK-NEXT:    .byte 29 @ 0x1d
+; CHECK-NEXT:    .byte 32 @ 0x20
+; CHECK-NEXT:    .byte 35 @ 0x23
+; CHECK-NEXT:    .byte 38 @ 0x26
+; CHECK-NEXT:    .byte 41 @ 0x29
+; CHECK-NEXT:    .byte 44 @ 0x2c
+; CHECK-NEXT:    .byte 47 @ 0x2f
+; CHECK-NEXT:    .byte 50 @ 0x32
+entry:
+	%ptrs = getelementptr inbounds i8, i8* %base, <16 x i8> <i8 0, i8 3, i8 6, i8 9, i8 12, i8 15, i8 18, i8 21, i8 24, i8 27, i8 30, i8 33, i8 36, i8 39, i8 42, i8 45>
+	%ptrs2 = getelementptr inbounds i8, <16 x i8*> %ptrs, i8 5
+	%gather = call <16 x i8> @llvm.masked.gather.v16i8.v16p0i8(<16 x i8*> %ptrs2, i32 1, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i8> undef)
+	ret <16 x i8> %gather
+}
+
+
+define arm_aapcs_vfpcc <16 x i8> @unscaled_v16i8_i8_biggep(i8* %base) {
+; CHECK-LABEL: unscaled_v16i8_i8_biggep:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    adr r1, .LCPI9_0
+; CHECK-NEXT:    vldrw.u32 q1, [r1]
+; CHECK-NEXT:    vldrb.u8 q0, [r0, q1]
+; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI9_0:
+; CHECK-NEXT:    .byte 5 @ 0x5
+; CHECK-NEXT:    .byte 8 @ 0x8
+; CHECK-NEXT:    .byte 11 @ 0xb
+; CHECK-NEXT:    .byte 14 @ 0xe
+; CHECK-NEXT:    .byte 17 @ 0x11
+; CHECK-NEXT:    .byte 20 @ 0x14
+; CHECK-NEXT:    .byte 23 @ 0x17
+; CHECK-NEXT:    .byte 26 @ 0x1a
+; CHECK-NEXT:    .byte 29 @ 0x1d
+; CHECK-NEXT:    .byte 32 @ 0x20
+; CHECK-NEXT:    .byte 35 @ 0x23
+; CHECK-NEXT:    .byte 38 @ 0x26
+; CHECK-NEXT:    .byte 41 @ 0x29
+; CHECK-NEXT:    .byte 44 @ 0x2c
+; CHECK-NEXT:    .byte 47 @ 0x2f
+; CHECK-NEXT:    .byte 50 @ 0x32
+entry:
+  %ptrs = getelementptr inbounds i8, i8* %base, <16 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21, i32 24, i32 27, i32 30, i32 33, i32 36, i32 39, i32 42, i32 45>
+	%ptrs2 = getelementptr inbounds i8, <16 x i8*> %ptrs, i32 5
+	%gather = call <16 x i8> @llvm.masked.gather.v16i8.v16p0i8(<16 x i8*> %ptrs2, i32 1, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i8> undef)
+	ret <16 x i8> %gather
+}
+
+
+define arm_aapcs_vfpcc <16 x i8> @unscaled_v16i8_i8_biggep2(i8* %base) {
+; CHECK-LABEL: unscaled_v16i8_i8_biggep2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    adr r1, .LCPI10_0
+; CHECK-NEXT:    vldrw.u32 q1, [r1]
+; CHECK-NEXT:    vldrb.u8 q0, [r0, q1]
+; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI10_0:
+; CHECK-NEXT:    .byte 0 @ 0x0
+; CHECK-NEXT:    .byte 3 @ 0x3
+; CHECK-NEXT:    .byte 6 @ 0x6
+; CHECK-NEXT:    .byte 9 @ 0x9
+; CHECK-NEXT:    .byte 12 @ 0xc
+; CHECK-NEXT:    .byte 15 @ 0xf
+; CHECK-NEXT:    .byte 18 @ 0x12
+; CHECK-NEXT:    .byte 21 @ 0x15
+; CHECK-NEXT:    .byte 24 @ 0x18
+; CHECK-NEXT:    .byte 27 @ 0x1b
+; CHECK-NEXT:    .byte 30 @ 0x1e
+; CHECK-NEXT:    .byte 33 @ 0x21
+; CHECK-NEXT:    .byte 36 @ 0x24
+; CHECK-NEXT:    .byte 39 @ 0x27
+; CHECK-NEXT:    .byte 42 @ 0x2a
+; CHECK-NEXT:    .byte 45 @ 0x2d
+entry:
+	%ptrs = getelementptr inbounds i8, i8* %base, <16 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21, i32 24, i32 27, i32 30, i32 33, i32 36, i32 39, i32 42, i32 45>
+	%gather = call <16 x i8> @llvm.masked.gather.v16i8.v16p0i8(<16 x i8*> %ptrs, i32 1, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i8> undef)
+	ret <16 x i8> %gather
+}
+
+
+define arm_aapcs_vfpcc <16 x i8> @unscaled_v16i8_i8_biggep3(i8* %base) {
+; CHECK-LABEL: unscaled_v16i8_i8_biggep3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, r5, r6, lr}
+; CHECK-NEXT:    push {r4, r5, r6, lr}
+; CHECK-NEXT:    adr r1, .LCPI11_0
+; CHECK-NEXT:    adr r2, .LCPI11_1
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    adr r6, .LCPI11_2
+; CHECK-NEXT:    vadd.i32 q1, q0, r0
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vmov r1, s6
+; CHECK-NEXT:    vadd.i32 q0, q0, r0
+; CHECK-NEXT:    vmov r5, s4
+; CHECK-NEXT:    vmov r2, s3
+; CHECK-NEXT:    vmov r4, s7
+; CHECK-NEXT:    ldrb.w r12, [r1]
+; CHECK-NEXT:    vmov r1, s0
+; CHECK-NEXT:    ldrb r5, [r5]
+; CHECK-NEXT:    ldrb r2, [r2]
+; CHECK-NEXT:    ldrb r4, [r4]
+; CHECK-NEXT:    ldrb.w lr, [r1]
+; CHECK-NEXT:    vmov r1, s1
+; CHECK-NEXT:    ldrb r3, [r1]
+; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    vldrw.u32 q0, [r6]
+; CHECK-NEXT:    adr r6, .LCPI11_3
+; CHECK-NEXT:    vldrw.u32 q2, [r6]
+; CHECK-NEXT:    vadd.i32 q3, q0, r0
+; CHECK-NEXT:    vadd.i32 q2, q2, r0
+; CHECK-NEXT:    vmov r0, s12
+; CHECK-NEXT:    ldrb r1, [r1]
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[0], r0
+; CHECK-NEXT:    vmov r0, s13
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[1], r0
+; CHECK-NEXT:    vmov r0, s14
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[2], r0
+; CHECK-NEXT:    vmov r0, s15
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[3], r0
+; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[4], r0
+; CHECK-NEXT:    vmov r0, s9
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[5], r0
+; CHECK-NEXT:    vmov r0, s10
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[6], r0
+; CHECK-NEXT:    vmov r0, s11
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[7], r0
+; CHECK-NEXT:    vmov r0, s5
+; CHECK-NEXT:    vmov.8 q0[8], r5
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[9], r0
+; CHECK-NEXT:    vmov.8 q0[10], r12
+; CHECK-NEXT:    vmov.8 q0[11], r4
+; CHECK-NEXT:    vmov.8 q0[12], lr
+; CHECK-NEXT:    vmov.8 q0[13], r3
+; CHECK-NEXT:    vmov.8 q0[14], r1
+; CHECK-NEXT:    vmov.8 q0[15], r2
+; CHECK-NEXT:    pop {r4, r5, r6, pc}
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI11_0:
+; CHECK-NEXT:    .long 280 @ 0x118
+; CHECK-NEXT:    .long 283 @ 0x11b
+; CHECK-NEXT:    .long 286 @ 0x11e
+; CHECK-NEXT:    .long 289 @ 0x121
+; CHECK-NEXT:  .LCPI11_1:
+; CHECK-NEXT:    .long 292 @ 0x124
+; CHECK-NEXT:    .long 295 @ 0x127
+; CHECK-NEXT:    .long 298 @ 0x12a
+; CHECK-NEXT:    .long 301 @ 0x12d
+; CHECK-NEXT:  .LCPI11_2:
+; CHECK-NEXT:    .long 256 @ 0x100
+; CHECK-NEXT:    .long 259 @ 0x103
+; CHECK-NEXT:    .long 262 @ 0x106
+; CHECK-NEXT:    .long 265 @ 0x109
+; CHECK-NEXT:  .LCPI11_3:
+; CHECK-NEXT:    .long 268 @ 0x10c
+; CHECK-NEXT:    .long 271 @ 0x10f
+; CHECK-NEXT:    .long 274 @ 0x112
+; CHECK-NEXT:    .long 277 @ 0x115
+entry:
+	%ptrs = getelementptr inbounds i8, i8* %base, <16 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21, i32 24, i32 27, i32 30, i32 33, i32 36, i32 39, i32 42, i32 45>
+	%ptrs2 = getelementptr inbounds i8, <16 x i8*> %ptrs, i32 256
+	%gather = call <16 x i8> @llvm.masked.gather.v16i8.v16p0i8(<16 x i8*> %ptrs2, i32 1, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i8> undef)
+	ret <16 x i8> %gather
+}
+
+
+define arm_aapcs_vfpcc <16 x i8> @unscaled_v16i8_i8_biggep4(i8* %base) {
+; CHECK-LABEL: unscaled_v16i8_i8_biggep4:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, r5, r6, lr}
+; CHECK-NEXT:    push {r4, r5, r6, lr}
+; CHECK-NEXT:    adr r1, .LCPI12_0
+; CHECK-NEXT:    adr r2, .LCPI12_1
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    adr r6, .LCPI12_2
+; CHECK-NEXT:    vadd.i32 q1, q0, r0
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vmov r1, s6
+; CHECK-NEXT:    vadd.i32 q0, q0, r0
+; CHECK-NEXT:    vmov r5, s4
+; CHECK-NEXT:    vmov r2, s3
+; CHECK-NEXT:    vmov r4, s7
+; CHECK-NEXT:    ldrb.w r12, [r1]
+; CHECK-NEXT:    vmov r1, s0
+; CHECK-NEXT:    ldrb r5, [r5]
+; CHECK-NEXT:    ldrb r2, [r2]
+; CHECK-NEXT:    ldrb r4, [r4]
+; CHECK-NEXT:    ldrb.w lr, [r1]
+; CHECK-NEXT:    vmov r1, s1
+; CHECK-NEXT:    ldrb r3, [r1]
+; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    vldrw.u32 q0, [r6]
+; CHECK-NEXT:    adr r6, .LCPI12_3
+; CHECK-NEXT:    vldrw.u32 q2, [r6]
+; CHECK-NEXT:    vadd.i32 q3, q0, r0
+; CHECK-NEXT:    vadd.i32 q2, q2, r0
+; CHECK-NEXT:    vmov r0, s12
+; CHECK-NEXT:    ldrb r1, [r1]
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[0], r0
+; CHECK-NEXT:    vmov r0, s13
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[1], r0
+; CHECK-NEXT:    vmov r0, s14
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[2], r0
+; CHECK-NEXT:    vmov r0, s15
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[3], r0
+; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[4], r0
+; CHECK-NEXT:    vmov r0, s9
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[5], r0
+; CHECK-NEXT:    vmov r0, s10
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[6], r0
+; CHECK-NEXT:    vmov r0, s11
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[7], r0
+; CHECK-NEXT:    vmov r0, s5
+; CHECK-NEXT:    vmov.8 q0[8], r5
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[9], r0
+; CHECK-NEXT:    vmov.8 q0[10], r12
+; CHECK-NEXT:    vmov.8 q0[11], r4
+; CHECK-NEXT:    vmov.8 q0[12], lr
+; CHECK-NEXT:    vmov.8 q0[13], r3
+; CHECK-NEXT:    vmov.8 q0[14], r1
+; CHECK-NEXT:    vmov.8 q0[15], r2
+; CHECK-NEXT:    pop {r4, r5, r6, pc}
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI12_0:
+; CHECK-NEXT:    .long 256 @ 0x100
+; CHECK-NEXT:    .long 27 @ 0x1b
+; CHECK-NEXT:    .long 30 @ 0x1e
+; CHECK-NEXT:    .long 33 @ 0x21
+; CHECK-NEXT:  .LCPI12_1:
+; CHECK-NEXT:    .long 36 @ 0x24
+; CHECK-NEXT:    .long 39 @ 0x27
+; CHECK-NEXT:    .long 42 @ 0x2a
+; CHECK-NEXT:    .long 45 @ 0x2d
+; CHECK-NEXT:  .LCPI12_2:
+; CHECK-NEXT:    .long 0 @ 0x0
+; CHECK-NEXT:    .long 3 @ 0x3
+; CHECK-NEXT:    .long 6 @ 0x6
+; CHECK-NEXT:    .long 9 @ 0x9
+; CHECK-NEXT:  .LCPI12_3:
+; CHECK-NEXT:    .long 12 @ 0xc
+; CHECK-NEXT:    .long 15 @ 0xf
+; CHECK-NEXT:    .long 18 @ 0x12
+; CHECK-NEXT:    .long 21 @ 0x15
+entry:
+	%ptrs = getelementptr inbounds i8, i8* %base, <16 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21, i32 256, i32 27, i32 30, i32 33, i32 36, i32 39, i32 42, i32 45>
+	%gather = call <16 x i8> @llvm.masked.gather.v16i8.v16p0i8(<16 x i8*> %ptrs, i32 1, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i8> undef)
+	ret <16 x i8> %gather
+}
+
+
+define arm_aapcs_vfpcc <16 x i8> @unscaled_v16i8_i8_biggep5(<16 x i8*> %base) {
+; CHECK-LABEL: unscaled_v16i8_i8_biggep5:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, r5, r7, lr}
+; CHECK-NEXT:    push {r4, r5, r7, lr}
+; CHECK-NEXT:    .vsave {d8, d9}
+; CHECK-NEXT:    vpush {d8, d9}
+; CHECK-NEXT:    vmov.i32 q4, #0x100
+; CHECK-NEXT:    vadd.i32 q2, q2, q4
+; CHECK-NEXT:    vadd.i32 q3, q3, q4
+; CHECK-NEXT:    vmov r0, s10
+; CHECK-NEXT:    vadd.i32 q1, q1, q4
+; CHECK-NEXT:    vmov r1, s15
+; CHECK-NEXT:    vmov r4, s8
+; CHECK-NEXT:    ldrb.w r12, [r0]
+; CHECK-NEXT:    vmov r0, s11
+; CHECK-NEXT:    ldrb r1, [r1]
+; CHECK-NEXT:    ldrb r4, [r4]
+; CHECK-NEXT:    ldrb.w lr, [r0]
+; CHECK-NEXT:    vmov r0, s12
+; CHECK-NEXT:    ldrb r2, [r0]
+; CHECK-NEXT:    vmov r0, s13
+; CHECK-NEXT:    ldrb r3, [r0]
+; CHECK-NEXT:    vmov r0, s14
+; CHECK-NEXT:    vadd.i32 q3, q0, q4
+; CHECK-NEXT:    vmov r5, s12
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    ldrb r5, [r5]
+; CHECK-NEXT:    vmov.8 q0[0], r5
+; CHECK-NEXT:    vmov r5, s13
+; CHECK-NEXT:    ldrb r5, [r5]
+; CHECK-NEXT:    vmov.8 q0[1], r5
+; CHECK-NEXT:    vmov r5, s14
+; CHECK-NEXT:    ldrb r5, [r5]
+; CHECK-NEXT:    vmov.8 q0[2], r5
+; CHECK-NEXT:    vmov r5, s15
+; CHECK-NEXT:    ldrb r5, [r5]
+; CHECK-NEXT:    vmov.8 q0[3], r5
+; CHECK-NEXT:    vmov r5, s4
+; CHECK-NEXT:    ldrb r5, [r5]
+; CHECK-NEXT:    vmov.8 q0[4], r5
+; CHECK-NEXT:    vmov r5, s5
+; CHECK-NEXT:    ldrb r5, [r5]
+; CHECK-NEXT:    vmov.8 q0[5], r5
+; CHECK-NEXT:    vmov r5, s6
+; CHECK-NEXT:    ldrb r5, [r5]
+; CHECK-NEXT:    vmov.8 q0[6], r5
+; CHECK-NEXT:    vmov r5, s7
+; CHECK-NEXT:    ldrb r5, [r5]
+; CHECK-NEXT:    vmov.8 q0[7], r5
+; CHECK-NEXT:    vmov r5, s9
+; CHECK-NEXT:    vmov.8 q0[8], r4
+; CHECK-NEXT:    ldrb r5, [r5]
+; CHECK-NEXT:    vmov.8 q0[9], r5
+; CHECK-NEXT:    vmov.8 q0[10], r12
+; CHECK-NEXT:    vmov.8 q0[11], lr
+; CHECK-NEXT:    vmov.8 q0[12], r2
+; CHECK-NEXT:    vmov.8 q0[13], r3
+; CHECK-NEXT:    vmov.8 q0[14], r0
+; CHECK-NEXT:    vmov.8 q0[15], r1
+; CHECK-NEXT:    vpop {d8, d9}
+; CHECK-NEXT:    pop {r4, r5, r7, pc}
+entry:
+	%ptrs2 = getelementptr inbounds i8, <16 x i8*> %base, i32 256
+  %gather = call <16 x i8> @llvm.masked.gather.v16i8.v16p0i8(<16 x i8*> %ptrs2, i32 1, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i8> undef)
+	ret <16 x i8> %gather
+}
+
+
+define arm_aapcs_vfpcc <16 x i8> @unscaled_v16i8_i8_biggep6(i8* %base) {
+; CHECK-LABEL: unscaled_v16i8_i8_biggep6:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, r5, r6, lr}
+; CHECK-NEXT:    push {r4, r5, r6, lr}
+; CHECK-NEXT:    adr r1, .LCPI14_0
+; CHECK-NEXT:    adr r2, .LCPI14_1
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    adr r6, .LCPI14_2
+; CHECK-NEXT:    vadd.i32 q1, q0, r0
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vmov r1, s6
+; CHECK-NEXT:    vadd.i32 q0, q0, r0
+; CHECK-NEXT:    vmov r5, s4
+; CHECK-NEXT:    vmov r2, s3
+; CHECK-NEXT:    vmov r4, s7
+; CHECK-NEXT:    ldrb.w r12, [r1]
+; CHECK-NEXT:    vmov r1, s0
+; CHECK-NEXT:    ldrb r5, [r5]
+; CHECK-NEXT:    ldrb r2, [r2]
+; CHECK-NEXT:    ldrb r4, [r4]
+; CHECK-NEXT:    ldrb.w lr, [r1]
+; CHECK-NEXT:    vmov r1, s1
+; CHECK-NEXT:    ldrb r3, [r1]
+; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    vldrw.u32 q0, [r6]
+; CHECK-NEXT:    adr r6, .LCPI14_3
+; CHECK-NEXT:    vldrw.u32 q2, [r6]
+; CHECK-NEXT:    vadd.i32 q3, q0, r0
+; CHECK-NEXT:    vadd.i32 q2, q2, r0
+; CHECK-NEXT:    vmov r0, s12
+; CHECK-NEXT:    ldrb r1, [r1]
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[0], r0
+; CHECK-NEXT:    vmov r0, s13
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[1], r0
+; CHECK-NEXT:    vmov r0, s14
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[2], r0
+; CHECK-NEXT:    vmov r0, s15
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[3], r0
+; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[4], r0
+; CHECK-NEXT:    vmov r0, s9
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[5], r0
+; CHECK-NEXT:    vmov r0, s10
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[6], r0
+; CHECK-NEXT:    vmov r0, s11
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[7], r0
+; CHECK-NEXT:    vmov r0, s5
+; CHECK-NEXT:    vmov.8 q0[8], r5
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[9], r0
+; CHECK-NEXT:    vmov.8 q0[10], r12
+; CHECK-NEXT:    vmov.8 q0[11], r4
+; CHECK-NEXT:    vmov.8 q0[12], lr
+; CHECK-NEXT:    vmov.8 q0[13], r3
+; CHECK-NEXT:    vmov.8 q0[14], r1
+; CHECK-NEXT:    vmov.8 q0[15], r2
+; CHECK-NEXT:    pop {r4, r5, r6, pc}
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI14_0:
+; CHECK-NEXT:    .long 257 @ 0x101
+; CHECK-NEXT:    .long 28 @ 0x1c
+; CHECK-NEXT:    .long 31 @ 0x1f
+; CHECK-NEXT:    .long 34 @ 0x22
+; CHECK-NEXT:  .LCPI14_1:
+; CHECK-NEXT:    .long 37 @ 0x25
+; CHECK-NEXT:    .long 40 @ 0x28
+; CHECK-NEXT:    .long 43 @ 0x2b
+; CHECK-NEXT:    .long 46 @ 0x2e
+; CHECK-NEXT:  .LCPI14_2:
+; CHECK-NEXT:    .long 1 @ 0x1
+; CHECK-NEXT:    .long 4 @ 0x4
+; CHECK-NEXT:    .long 7 @ 0x7
+; CHECK-NEXT:    .long 10 @ 0xa
+; CHECK-NEXT:  .LCPI14_3:
+; CHECK-NEXT:    .long 13 @ 0xd
+; CHECK-NEXT:    .long 16 @ 0x10
+; CHECK-NEXT:    .long 19 @ 0x13
+; CHECK-NEXT:    .long 22 @ 0x16
+entry:
+	%ptrs = getelementptr inbounds i8, i8* %base, <16 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21, i32 256, i32 27, i32 30, i32 33, i32 36, i32 39, i32 42, i32 45>
+	%ptrs2 = getelementptr inbounds i8, <16 x i8*> %ptrs, i32 1
+	%gather = call <16 x i8> @llvm.masked.gather.v16i8.v16p0i8(<16 x i8*> %ptrs2, i32 1, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i8> undef)
+	ret <16 x i8> %gather
+}
+
+
+define arm_aapcs_vfpcc <16 x i8> @unscaled_v16i8_i8_biggep7(i8* %base) {
+; CHECK-LABEL: unscaled_v16i8_i8_biggep7:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, r5, r6, lr}
+; CHECK-NEXT:    push {r4, r5, r6, lr}
+; CHECK-NEXT:    adr r1, .LCPI15_0
+; CHECK-NEXT:    adr r2, .LCPI15_1
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    adr r6, .LCPI15_2
+; CHECK-NEXT:    vadd.i32 q1, q0, r0
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    vmov r1, s6
+; CHECK-NEXT:    vadd.i32 q0, q0, r0
+; CHECK-NEXT:    vmov r5, s4
+; CHECK-NEXT:    vmov r2, s3
+; CHECK-NEXT:    vmov r4, s7
+; CHECK-NEXT:    ldrb.w r12, [r1]
+; CHECK-NEXT:    vmov r1, s0
+; CHECK-NEXT:    ldrb r5, [r5]
+; CHECK-NEXT:    ldrb r2, [r2]
+; CHECK-NEXT:    ldrb r4, [r4]
+; CHECK-NEXT:    ldrb.w lr, [r1]
+; CHECK-NEXT:    vmov r1, s1
+; CHECK-NEXT:    ldrb r3, [r1]
+; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    vldrw.u32 q0, [r6]
+; CHECK-NEXT:    adr r6, .LCPI15_3
+; CHECK-NEXT:    vldrw.u32 q2, [r6]
+; CHECK-NEXT:    vadd.i32 q3, q0, r0
+; CHECK-NEXT:    vadd.i32 q2, q2, r0
+; CHECK-NEXT:    vmov r0, s12
+; CHECK-NEXT:    ldrb r1, [r1]
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[0], r0
+; CHECK-NEXT:    vmov r0, s13
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[1], r0
+; CHECK-NEXT:    vmov r0, s14
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[2], r0
+; CHECK-NEXT:    vmov r0, s15
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[3], r0
+; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[4], r0
+; CHECK-NEXT:    vmov r0, s9
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[5], r0
+; CHECK-NEXT:    vmov r0, s10
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[6], r0
+; CHECK-NEXT:    vmov r0, s11
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[7], r0
+; CHECK-NEXT:    vmov r0, s5
+; CHECK-NEXT:    vmov.8 q0[8], r5
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[9], r0
+; CHECK-NEXT:    vmov.8 q0[10], r12
+; CHECK-NEXT:    vmov.8 q0[11], r4
+; CHECK-NEXT:    vmov.8 q0[12], lr
+; CHECK-NEXT:    vmov.8 q0[13], r3
+; CHECK-NEXT:    vmov.8 q0[14], r1
+; CHECK-NEXT:    vmov.8 q0[15], r2
+; CHECK-NEXT:    pop {r4, r5, r6, pc}
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI15_0:
+; CHECK-NEXT:    .long 224 @ 0xe0
+; CHECK-NEXT:    .long 227 @ 0xe3
+; CHECK-NEXT:    .long 230 @ 0xe6
+; CHECK-NEXT:    .long 233 @ 0xe9
+; CHECK-NEXT:  .LCPI15_1:
+; CHECK-NEXT:    .long 236 @ 0xec
+; CHECK-NEXT:    .long 239 @ 0xef
+; CHECK-NEXT:    .long 242 @ 0xf2
+; CHECK-NEXT:    .long 245 @ 0xf5
+; CHECK-NEXT:  .LCPI15_2:
+; CHECK-NEXT:    .long 300 @ 0x12c
+; CHECK-NEXT:    .long 203 @ 0xcb
+; CHECK-NEXT:    .long 206 @ 0xce
+; CHECK-NEXT:    .long 209 @ 0xd1
+; CHECK-NEXT:  .LCPI15_3:
+; CHECK-NEXT:    .long 212 @ 0xd4
+; CHECK-NEXT:    .long 215 @ 0xd7
+; CHECK-NEXT:    .long 218 @ 0xda
+; CHECK-NEXT:    .long 221 @ 0xdd
+entry:
+  %ptrs = getelementptr inbounds i8, i8* %base, <16 x i32> <i32 100, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21, i32 24, i32 27, i32 30, i32 33, i32 36, i32 39, i32 42, i32 45>
+	%ptrs2 = getelementptr inbounds i8, <16 x i8*> %ptrs, i32 200
+	%gather = call <16 x i8> @llvm.masked.gather.v16i8.v16p0i8(<16 x i8*> %ptrs2, i32 1, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i8> undef)
+	ret <16 x i8> %gather
+}
+
+
+define arm_aapcs_vfpcc <16 x i8> @unscaled_v16i8_i8_2(i8* %base, <16 x i8>* %offptr) {
+; CHECK-LABEL: unscaled_v16i8_i8_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, r5, r6, lr}
+; CHECK-NEXT:    push {r4, r5, r6, lr}
+; CHECK-NEXT:    vldrb.s32 q0, [r1, #8]
+; CHECK-NEXT:    vldrb.s32 q2, [r1, #4]
+; CHECK-NEXT:    vadd.i32 q1, q0, r0
+; CHECK-NEXT:    vldrb.s32 q0, [r1, #12]
+; CHECK-NEXT:    vmov r2, s6
+; CHECK-NEXT:    vadd.i32 q2, q2, r0
+; CHECK-NEXT:    vadd.i32 q0, q0, r0
+; CHECK-NEXT:    vmov r6, s4
+; CHECK-NEXT:    vmov r3, s2
+; CHECK-NEXT:    vmov r4, s3
+; CHECK-NEXT:    vmov r5, s7
+; CHECK-NEXT:    ldrb.w r12, [r2]
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    ldrb r6, [r6]
+; CHECK-NEXT:    ldrb r3, [r3]
+; CHECK-NEXT:    ldrb r4, [r4]
+; CHECK-NEXT:    ldrb r5, [r5]
+; CHECK-NEXT:    ldrb.w lr, [r2]
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    vldrb.s32 q0, [r1]
+; CHECK-NEXT:    vadd.i32 q3, q0, r0
+; CHECK-NEXT:    vmov r0, s12
+; CHECK-NEXT:    ldrb r2, [r2]
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[0], r0
+; CHECK-NEXT:    vmov r0, s13
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[1], r0
+; CHECK-NEXT:    vmov r0, s14
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[2], r0
+; CHECK-NEXT:    vmov r0, s15
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[3], r0
+; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[4], r0
+; CHECK-NEXT:    vmov r0, s9
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[5], r0
+; CHECK-NEXT:    vmov r0, s10
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[6], r0
+; CHECK-NEXT:    vmov r0, s11
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[7], r0
+; CHECK-NEXT:    vmov r0, s5
+; CHECK-NEXT:    vmov.8 q0[8], r6
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[9], r0
+; CHECK-NEXT:    vmov.8 q0[10], r12
+; CHECK-NEXT:    vmov.8 q0[11], r5
+; CHECK-NEXT:    vmov.8 q0[12], lr
+; CHECK-NEXT:    vmov.8 q0[13], r2
+; CHECK-NEXT:    vmov.8 q0[14], r3
+; CHECK-NEXT:    vmov.8 q0[15], r4
+; CHECK-NEXT:    pop {r4, r5, r6, pc}
+entry:
+  %offs = load <16 x i8>, <16 x i8>* %offptr, align 1
+  %ptrs = getelementptr inbounds i8, i8* %base, <16 x i8> %offs
+  %gather = call <16 x i8> @llvm.masked.gather.v16i8.v16p0i8(<16 x i8*> %ptrs, i32 1, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i8> undef)
+	ret <16 x i8> %gather
+}
+
+
+define arm_aapcs_vfpcc <16 x i8> @unscaled_v16i8_i8_3(i8* %base, <16 x i8>* %offptr) {
+; CHECK-LABEL: unscaled_v16i8_i8_3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    adr r1, .LCPI17_0
+; CHECK-NEXT:    vldrw.u32 q1, [r1]
+; CHECK-NEXT:    vldrb.u8 q0, [r0, q1]
+; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI17_0:
+; CHECK-NEXT:    .byte 0 @ 0x0
+; CHECK-NEXT:    .byte 3 @ 0x3
+; CHECK-NEXT:    .byte 6 @ 0x6
+; CHECK-NEXT:    .byte 9 @ 0x9
+; CHECK-NEXT:    .byte 12 @ 0xc
+; CHECK-NEXT:    .byte 15 @ 0xf
+; CHECK-NEXT:    .byte 18 @ 0x12
+; CHECK-NEXT:    .byte 21 @ 0x15
+; CHECK-NEXT:    .byte 24 @ 0x18
+; CHECK-NEXT:    .byte 27 @ 0x1b
+; CHECK-NEXT:    .byte 30 @ 0x1e
+; CHECK-NEXT:    .byte 33 @ 0x21
+; CHECK-NEXT:    .byte 36 @ 0x24
+; CHECK-NEXT:    .byte 39 @ 0x27
+; CHECK-NEXT:    .byte 42 @ 0x2a
+; CHECK-NEXT:    .byte 45 @ 0x2d
+entry:
+  %ptrs = getelementptr inbounds i8, i8* %base, <16 x i8> <i8 0, i8 3, i8 6, i8 9, i8 12, i8 15, i8 18, i8 21, i8 24, i8 27, i8 30, i8 33, i8 36, i8 39, i8 42, i8 45>
+  %gather = call <16 x i8> @llvm.masked.gather.v16i8.v16p0i8(<16 x i8*> %ptrs, i32 1, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i8> undef)
+	ret <16 x i8> %gather
+}
+
+define arm_aapcs_vfpcc <16 x i8> @unscaled_v16i8_basei16(i16* %base, <16 x i8>* %offptr) {
+; CHECK-LABEL: unscaled_v16i8_basei16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, r5, r6, lr}
+; CHECK-NEXT:    push {r4, r5, r6, lr}
+; CHECK-NEXT:    vldrb.u32 q0, [r1, #8]
+; CHECK-NEXT:    vldrb.u32 q2, [r1, #4]
+; CHECK-NEXT:    vshl.i32 q0, q0, #1
+; CHECK-NEXT:    vshl.i32 q2, q2, #1
+; CHECK-NEXT:    vadd.i32 q1, q0, r0
+; CHECK-NEXT:    vldrb.u32 q0, [r1, #12]
+; CHECK-NEXT:    vmov r2, s6
+; CHECK-NEXT:    vadd.i32 q2, q2, r0
+; CHECK-NEXT:    vshl.i32 q0, q0, #1
+; CHECK-NEXT:    vmov r6, s4
+; CHECK-NEXT:    vadd.i32 q0, q0, r0
+; CHECK-NEXT:    vmov r5, s7
+; CHECK-NEXT:    vmov r3, s2
+; CHECK-NEXT:    vmov r4, s3
+; CHECK-NEXT:    ldrb.w r12, [r2]
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    ldrb r6, [r6]
+; CHECK-NEXT:    ldrb r5, [r5]
+; CHECK-NEXT:    ldrb r3, [r3]
+; CHECK-NEXT:    ldrb r4, [r4]
+; CHECK-NEXT:    ldrb.w lr, [r2]
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    vldrb.u32 q0, [r1]
+; CHECK-NEXT:    vshl.i32 q0, q0, #1
+; CHECK-NEXT:    vadd.i32 q3, q0, r0
+; CHECK-NEXT:    vmov r0, s12
+; CHECK-NEXT:    ldrb r2, [r2]
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[0], r0
+; CHECK-NEXT:    vmov r0, s13
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[1], r0
+; CHECK-NEXT:    vmov r0, s14
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[2], r0
+; CHECK-NEXT:    vmov r0, s15
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[3], r0
+; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[4], r0
+; CHECK-NEXT:    vmov r0, s9
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[5], r0
+; CHECK-NEXT:    vmov r0, s10
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[6], r0
+; CHECK-NEXT:    vmov r0, s11
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[7], r0
+; CHECK-NEXT:    vmov r0, s5
+; CHECK-NEXT:    vmov.8 q0[8], r6
+; CHECK-NEXT:    ldrb r0, [r0]
+; CHECK-NEXT:    vmov.8 q0[9], r0
+; CHECK-NEXT:    vmov.8 q0[10], r12
+; CHECK-NEXT:    vmov.8 q0[11], r5
+; CHECK-NEXT:    vmov.8 q0[12], lr
+; CHECK-NEXT:    vmov.8 q0[13], r2
+; CHECK-NEXT:    vmov.8 q0[14], r3
+; CHECK-NEXT:    vmov.8 q0[15], r4
+; CHECK-NEXT:    pop {r4, r5, r6, pc}
+entry:
+  %offs = load <16 x i8>, <16 x i8>* %offptr, align 1
+  %offs.zext = zext <16 x i8> %offs to <16 x i32>
+  %ptrs = getelementptr inbounds i16, i16* %base, <16 x i32> %offs.zext
+  %ptrs.cast = bitcast <16 x i16*> %ptrs to <16 x i8*>
+  %gather = call <16 x i8> @llvm.masked.gather.v16i8.v16p0i8(<16 x i8*> %ptrs.cast, i32 1, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i8> undef)
+  ret <16 x i8> %gather
+}
+
 declare <16 x i8> @llvm.masked.gather.v16i8.v16p0i8(<16 x i8*>, i32, <16 x i1>, <16 x i8>)
 declare <8 x i8> @llvm.masked.gather.v8i8.v8p0i8(<8 x i8*>, i32, <8 x i1>, <8 x i8>)
 declare <2 x i8> @llvm.masked.gather.v2i8.v2p0i8(<2 x i8*>, i32, <2 x i1>, <2 x i8>)
diff --git a/llvm/test/CodeGen/Thumb2/mve-gather-scatter-ptr-address.ll b/llvm/test/CodeGen/Thumb2/mve-gather-scatter-ptr-address.ll
new file mode 100644
index 0000000000000..7e01d24d006aa
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-gather-scatter-ptr-address.ll
@@ -0,0 +1,1012 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+
+; RUN: llc -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve.fp -enable-arm-maskedgatscat %s -o 2>/dev/null - | FileCheck %s
+
+define void @ptr_iv_v4i32(i32* noalias nocapture readonly %A, i32* noalias nocapture %B, i32 %y) {
+; CHECK-LABEL: ptr_iv_v4i32:
+; CHECK:       @ %bb.0: @ %vector.ph
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    adr r3, .LCPI0_0
+; CHECK-NEXT:    mov.w lr, #249
+; CHECK-NEXT:    vldrw.u32 q0, [r3]
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB0_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrw.u32 q1, [r0, q0, uxtw #2]
+; CHECK-NEXT:    adds r0, #64
+; CHECK-NEXT:    vadd.i32 q1, q1, r2
+; CHECK-NEXT:    vstrw.32 q1, [r1, q0, uxtw #2]
+; CHECK-NEXT:    adds r1, #64
+; CHECK-NEXT:    le lr, .LBB0_1
+; CHECK-NEXT:  @ %bb.2: @ %end
+; CHECK-NEXT:    pop {r7, pc}
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  @ %bb.3:
+; CHECK-NEXT:  .LCPI0_0:
+; CHECK-NEXT:    .long 0 @ 0x0
+; CHECK-NEXT:    .long 4 @ 0x4
+; CHECK-NEXT:    .long 8 @ 0x8
+; CHECK-NEXT:    .long 12 @ 0xc
+vector.ph:
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %y, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:
+  %pointer.phi = phi i32* [ %A, %vector.ph ], [ %0, %vector.body ]
+  %pointer.phi13 = phi i32* [ %B, %vector.ph ], [ %2, %vector.body ]
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %0 = getelementptr i32, i32* %pointer.phi, i32 16
+  %1 = getelementptr i32, i32* %pointer.phi, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+  %2 = getelementptr i32, i32* %pointer.phi13, i32 16
+  %3 = getelementptr i32, i32* %pointer.phi13, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+  %wide.masked.gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %1, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
+  %4 = add nsw <4 x i32> %wide.masked.gather, %broadcast.splat
+  call void @llvm.masked.scatter.v4i32.v4p0i32(<4 x i32> %4, <4 x i32*> %3, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
+  %index.next = add i32 %index, 4
+  %5 = icmp eq i32 %index.next, 996
+  br i1 %5, label %end, label %vector.body
+
+end:
+  ret void
+}
+
+define void @ptr_iv_v4i32_mult(i32* noalias nocapture readonly %A, i32* noalias nocapture %B, i32 %y) {
+; CHECK-LABEL: ptr_iv_v4i32_mult:
+; CHECK:       @ %bb.0: @ %vector.ph
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    adr r1, .LCPI1_0
+; CHECK-NEXT:    adr r3, .LCPI1_1
+; CHECK-NEXT:    vldrw.u32 q0, [r3]
+; CHECK-NEXT:    vldrw.u32 q1, [r1]
+; CHECK-NEXT:    mov.w lr, #249
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB1_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrw.u32 q2, [r0, q0, uxtw #2]
+; CHECK-NEXT:    vadd.i32 q2, q2, r2
+; CHECK-NEXT:    vstrw.32 q2, [r0, q1, uxtw #2]
+; CHECK-NEXT:    adds r0, #64
+; CHECK-NEXT:    le lr, .LBB1_1
+; CHECK-NEXT:  @ %bb.2: @ %end
+; CHECK-NEXT:    pop {r7, pc}
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  @ %bb.3:
+; CHECK-NEXT:  .LCPI1_0:
+; CHECK-NEXT:    .long 5 @ 0x5
+; CHECK-NEXT:    .long 9 @ 0x9
+; CHECK-NEXT:    .long 13 @ 0xd
+; CHECK-NEXT:    .long 17 @ 0x11
+; CHECK-NEXT:  .LCPI1_1:
+; CHECK-NEXT:    .long 3 @ 0x3
+; CHECK-NEXT:    .long 7 @ 0x7
+; CHECK-NEXT:    .long 11 @ 0xb
+; CHECK-NEXT:    .long 15 @ 0xf
+vector.ph:
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %y, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:
+  %pointer.phi = phi i32* [ %A, %vector.ph ], [ %0, %vector.body ]
+  %pointer.phi13 = phi i32* [ %B, %vector.ph ], [ %2, %vector.body ]
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %0 = getelementptr i32, i32* %pointer.phi, i32 16
+  %1 = getelementptr i32, i32* %pointer.phi, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+  %gather.address = getelementptr i32, <4 x i32*> %1, i32 3
+  %2 = getelementptr i32, i32* %pointer.phi13, i32 16
+  %3 = getelementptr i32, i32* %pointer.phi13, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+  %scatter.address = getelementptr i32, <4 x i32*> %1, i32 5
+  %wide.masked.gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %gather.address, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
+  %4 = add nsw <4 x i32> %wide.masked.gather, %broadcast.splat
+  call void @llvm.masked.scatter.v4i32.v4p0i32(<4 x i32> %4, <4 x i32*> %scatter.address, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
+  %index.next = add i32 %index, 4
+  %5 = icmp eq i32 %index.next, 996
+  br i1 %5, label %end, label %vector.body
+
+end:
+  ret void
+}
+
+define void @ptr_iv_v8i16(i16* noalias nocapture readonly %A, i16* noalias nocapture %B, i16 %y) {
+; CHECK-LABEL: ptr_iv_v8i16:
+; CHECK:       @ %bb.0: @ %vector.ph
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    adr r3, .LCPI2_0
+; CHECK-NEXT:    mov.w lr, #249
+; CHECK-NEXT:    vldrw.u32 q0, [r3]
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB2_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrh.u16 q1, [r0, q0, uxtw #1]
+; CHECK-NEXT:    adds r0, #64
+; CHECK-NEXT:    vadd.i16 q1, q1, r2
+; CHECK-NEXT:    vstrh.16 q1, [r1, q0, uxtw #1]
+; CHECK-NEXT:    adds r1, #64
+; CHECK-NEXT:    le lr, .LBB2_1
+; CHECK-NEXT:  @ %bb.2: @ %end
+; CHECK-NEXT:    pop {r7, pc}
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  @ %bb.3:
+; CHECK-NEXT:  .LCPI2_0:
+; CHECK-NEXT:    .short 0 @ 0x0
+; CHECK-NEXT:    .short 4 @ 0x4
+; CHECK-NEXT:    .short 8 @ 0x8
+; CHECK-NEXT:    .short 12 @ 0xc
+; CHECK-NEXT:    .short 16 @ 0x10
+; CHECK-NEXT:    .short 20 @ 0x14
+; CHECK-NEXT:    .short 24 @ 0x18
+; CHECK-NEXT:    .short 28 @ 0x1c
+vector.ph:
+  %broadcast.splatinsert = insertelement <8 x i16> undef, i16 %y, i32 0
+  %broadcast.splat = shufflevector <8 x i16> %broadcast.splatinsert, <8 x i16> undef, <8 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:
+  %pointer.phi = phi i16* [ %A, %vector.ph ], [ %0, %vector.body ]
+  %pointer.phi13 = phi i16* [ %B, %vector.ph ], [ %2, %vector.body ]
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %0 = getelementptr i16, i16* %pointer.phi, i32 32
+  %1 = getelementptr i16, i16* %pointer.phi, <8 x i16> <i16 0, i16 4, i16 8, i16 12, i16 16, i16 20, i16 24, i16 28>
+  %2 = getelementptr i16, i16* %pointer.phi13, i32 32
+  %3 = getelementptr i16, i16* %pointer.phi13, <8 x i16> <i16 0, i16 4, i16 8, i16 12, i16 16, i16 20, i16 24, i16 28>
+  %wide.masked.gather = call <8 x i16> @llvm.masked.gather.v8i16.v8p0i16(<8 x i16*> %1, i32 4, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i16> undef)
+  %4 = add nsw <8 x i16> %wide.masked.gather, %broadcast.splat
+  call void @llvm.masked.scatter.v8i16.v8p0i16(<8 x i16> %4, <8 x i16*> %3, i32 4, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  %index.next = add i32 %index, 4
+  %5 = icmp eq i32 %index.next, 996
+  br i1 %5, label %end, label %vector.body
+
+end:
+  ret void
+}
+
+
+define void @ptr_iv_v8i16_mult(i16* noalias nocapture readonly %A, i16* noalias nocapture %B, i16 %y) {
+; CHECK-LABEL: ptr_iv_v8i16_mult:
+; CHECK:       @ %bb.0: @ %vector.ph
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    adr.w r12, .LCPI3_0
+; CHECK-NEXT:    adr r3, .LCPI3_1
+; CHECK-NEXT:    vldrw.u32 q0, [r3]
+; CHECK-NEXT:    vldrw.u32 q1, [r12]
+; CHECK-NEXT:    mov.w lr, #249
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB3_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrh.u16 q2, [r0, q0, uxtw #1]
+; CHECK-NEXT:    adds r0, #64
+; CHECK-NEXT:    vadd.i16 q2, q2, r2
+; CHECK-NEXT:    vstrh.16 q2, [r1, q1, uxtw #1]
+; CHECK-NEXT:    adds r1, #64
+; CHECK-NEXT:    le lr, .LBB3_1
+; CHECK-NEXT:  @ %bb.2: @ %end
+; CHECK-NEXT:    pop {r7, pc}
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  @ %bb.3:
+; CHECK-NEXT:  .LCPI3_0:
+; CHECK-NEXT:    .short 5 @ 0x5
+; CHECK-NEXT:    .short 9 @ 0x9
+; CHECK-NEXT:    .short 13 @ 0xd
+; CHECK-NEXT:    .short 17 @ 0x11
+; CHECK-NEXT:    .short 21 @ 0x15
+; CHECK-NEXT:    .short 25 @ 0x19
+; CHECK-NEXT:    .short 29 @ 0x1d
+; CHECK-NEXT:    .short 33 @ 0x21
+; CHECK-NEXT:  .LCPI3_1:
+; CHECK-NEXT:    .short 3 @ 0x3
+; CHECK-NEXT:    .short 7 @ 0x7
+; CHECK-NEXT:    .short 11 @ 0xb
+; CHECK-NEXT:    .short 15 @ 0xf
+; CHECK-NEXT:    .short 19 @ 0x13
+; CHECK-NEXT:    .short 23 @ 0x17
+; CHECK-NEXT:    .short 27 @ 0x1b
+; CHECK-NEXT:    .short 31 @ 0x1f
+vector.ph:
+  %broadcast.splatinsert = insertelement <8 x i16> undef, i16 %y, i32 0
+  %broadcast.splat = shufflevector <8 x i16> %broadcast.splatinsert, <8 x i16> undef, <8 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:
+  %pointer.phi = phi i16* [ %A, %vector.ph ], [ %0, %vector.body ]
+  %pointer.phi13 = phi i16* [ %B, %vector.ph ], [ %2, %vector.body ]
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %0 = getelementptr i16, i16* %pointer.phi, i32 32
+  %1 = getelementptr i16, i16* %pointer.phi, <8 x i16> <i16 0, i16 4, i16 8, i16 12, i16 16, i16 20, i16 24, i16 28>
+  %gather.address = getelementptr i16, <8 x i16*> %1, i16 3
+  %2 = getelementptr i16, i16* %pointer.phi13, i32 32
+  %3 = getelementptr i16, i16* %pointer.phi13, <8 x i16> <i16 0, i16 4, i16 8, i16 12, i16 16, i16 20, i16 24, i16 28>
+  %scatter.address = getelementptr i16, <8 x i16*> %3, i16 5
+  %wide.masked.gather = call <8 x i16> @llvm.masked.gather.v8i16.v8p0i16(<8 x i16*> %gather.address, i32 4, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i16> undef)
+  %4 = add nsw <8 x i16> %wide.masked.gather, %broadcast.splat
+  call void @llvm.masked.scatter.v8i16.v8p0i16(<8 x i16> %4, <8 x i16*> %scatter.address, i32 4, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  %index.next = add i32 %index, 4
+  %5 = icmp eq i32 %index.next, 996
+  br i1 %5, label %end, label %vector.body
+
+end:
+  ret void
+}
+
+define void @ptr_iv_v16i8(i8* noalias nocapture readonly %A, i8* noalias nocapture %B, i8 %y) {
+; CHECK-LABEL: ptr_iv_v16i8:
+; CHECK:       @ %bb.0: @ %vector.ph
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    adr r3, .LCPI4_0
+; CHECK-NEXT:    mov.w lr, #249
+; CHECK-NEXT:    vldrw.u32 q0, [r3]
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB4_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrb.u8 q1, [r0, q0]
+; CHECK-NEXT:    adds r0, #64
+; CHECK-NEXT:    vadd.i8 q1, q1, r2
+; CHECK-NEXT:    vstrb.8 q1, [r1, q0]
+; CHECK-NEXT:    adds r1, #64
+; CHECK-NEXT:    le lr, .LBB4_1
+; CHECK-NEXT:  @ %bb.2: @ %end
+; CHECK-NEXT:    pop {r7, pc}
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  @ %bb.3:
+; CHECK-NEXT:  .LCPI4_0:
+; CHECK-NEXT:    .byte 0 @ 0x0
+; CHECK-NEXT:    .byte 4 @ 0x4
+; CHECK-NEXT:    .byte 8 @ 0x8
+; CHECK-NEXT:    .byte 12 @ 0xc
+; CHECK-NEXT:    .byte 16 @ 0x10
+; CHECK-NEXT:    .byte 20 @ 0x14
+; CHECK-NEXT:    .byte 24 @ 0x18
+; CHECK-NEXT:    .byte 28 @ 0x1c
+; CHECK-NEXT:    .byte 32 @ 0x20
+; CHECK-NEXT:    .byte 36 @ 0x24
+; CHECK-NEXT:    .byte 40 @ 0x28
+; CHECK-NEXT:    .byte 44 @ 0x2c
+; CHECK-NEXT:    .byte 48 @ 0x30
+; CHECK-NEXT:    .byte 52 @ 0x34
+; CHECK-NEXT:    .byte 56 @ 0x38
+; CHECK-NEXT:    .byte 60 @ 0x3c
+vector.ph:                                        ; preds = %entry
+  %broadcast.splatinsert = insertelement <16 x i8> undef, i8 %y, i32 0
+  %broadcast.splat = shufflevector <16 x i8> %broadcast.splatinsert, <16 x i8> undef, <16 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:
+  %pointer.phi = phi i8* [ %A, %vector.ph ], [ %0, %vector.body ]
+  %pointer.phi13 = phi i8* [ %B, %vector.ph ], [ %2, %vector.body ]
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %0 = getelementptr i8, i8* %pointer.phi, i32 64
+  %1 = getelementptr i8, i8* %pointer.phi, <16 x i8> <i8 0, i8 4, i8 8, i8 12, i8 16, i8 20, i8 24, i8 28, i8 32, i8 36, i8 40, i8 44, i8 48, i8 52, i8 56, i8 60>
+  %2 = getelementptr i8, i8* %pointer.phi13, i32 64
+  %3 = getelementptr i8, i8* %pointer.phi13, <16 x i8> <i8 0, i8 4, i8 8, i8 12, i8 16, i8 20, i8 24, i8 28, i8 32, i8 36, i8 40, i8 44, i8 48, i8 52, i8 56, i8 60>
+  %wide.masked.gather = call <16 x i8> @llvm.masked.gather.v16i8.v16p0i8(<16 x i8*> %1, i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i8> undef)
+  %4 = add nsw <16 x i8> %wide.masked.gather, %broadcast.splat
+  call void @llvm.masked.scatter.v16i8.v16p0i8(<16 x i8> %4, <16 x i8*> %3, i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  %index.next = add i32 %index, 4
+  %5 = icmp eq i32 %index.next, 996
+  br i1 %5, label %end, label %vector.body
+
+end:
+  ret void
+}
+
+
+define void @ptr_iv_v16i8_mult(i8* noalias nocapture readonly %A, i8* noalias nocapture %B, i8 %y) {
+; CHECK-LABEL: ptr_iv_v16i8_mult:
+; CHECK:       @ %bb.0: @ %vector.ph
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    adr.w r12, .LCPI5_0
+; CHECK-NEXT:    adr r3, .LCPI5_1
+; CHECK-NEXT:    vldrw.u32 q0, [r3]
+; CHECK-NEXT:    vldrw.u32 q1, [r12]
+; CHECK-NEXT:    mov.w lr, #249
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB5_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrb.u8 q2, [r0, q0]
+; CHECK-NEXT:    adds r0, #64
+; CHECK-NEXT:    vadd.i8 q2, q2, r2
+; CHECK-NEXT:    vstrb.8 q2, [r1, q1]
+; CHECK-NEXT:    adds r1, #64
+; CHECK-NEXT:    le lr, .LBB5_1
+; CHECK-NEXT:  @ %bb.2: @ %end
+; CHECK-NEXT:    pop {r7, pc}
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  @ %bb.3:
+; CHECK-NEXT:  .LCPI5_0:
+; CHECK-NEXT:    .byte 5 @ 0x5
+; CHECK-NEXT:    .byte 9 @ 0x9
+; CHECK-NEXT:    .byte 13 @ 0xd
+; CHECK-NEXT:    .byte 17 @ 0x11
+; CHECK-NEXT:    .byte 21 @ 0x15
+; CHECK-NEXT:    .byte 25 @ 0x19
+; CHECK-NEXT:    .byte 29 @ 0x1d
+; CHECK-NEXT:    .byte 33 @ 0x21
+; CHECK-NEXT:    .byte 37 @ 0x25
+; CHECK-NEXT:    .byte 41 @ 0x29
+; CHECK-NEXT:    .byte 45 @ 0x2d
+; CHECK-NEXT:    .byte 49 @ 0x31
+; CHECK-NEXT:    .byte 53 @ 0x35
+; CHECK-NEXT:    .byte 57 @ 0x39
+; CHECK-NEXT:    .byte 61 @ 0x3d
+; CHECK-NEXT:    .byte 65 @ 0x41
+; CHECK-NEXT:  .LCPI5_1:
+; CHECK-NEXT:    .byte 3 @ 0x3
+; CHECK-NEXT:    .byte 7 @ 0x7
+; CHECK-NEXT:    .byte 11 @ 0xb
+; CHECK-NEXT:    .byte 15 @ 0xf
+; CHECK-NEXT:    .byte 19 @ 0x13
+; CHECK-NEXT:    .byte 23 @ 0x17
+; CHECK-NEXT:    .byte 27 @ 0x1b
+; CHECK-NEXT:    .byte 31 @ 0x1f
+; CHECK-NEXT:    .byte 35 @ 0x23
+; CHECK-NEXT:    .byte 39 @ 0x27
+; CHECK-NEXT:    .byte 43 @ 0x2b
+; CHECK-NEXT:    .byte 47 @ 0x2f
+; CHECK-NEXT:    .byte 51 @ 0x33
+; CHECK-NEXT:    .byte 55 @ 0x37
+; CHECK-NEXT:    .byte 59 @ 0x3b
+; CHECK-NEXT:    .byte 63 @ 0x3f
+vector.ph:                                        ; preds = %entry
+  %broadcast.splatinsert = insertelement <16 x i8> undef, i8 %y, i32 0
+  %broadcast.splat = shufflevector <16 x i8> %broadcast.splatinsert, <16 x i8> undef, <16 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:
+  %pointer.phi = phi i8* [ %A, %vector.ph ], [ %0, %vector.body ]
+  %pointer.phi13 = phi i8* [ %B, %vector.ph ], [ %2, %vector.body ]
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %0 = getelementptr i8, i8* %pointer.phi, i32 64
+  %1 = getelementptr i8, i8* %pointer.phi, <16 x i8> <i8 0, i8 4, i8 8, i8 12, i8 16, i8 20, i8 24, i8 28, i8 32, i8 36, i8 40, i8 44, i8 48, i8 52, i8 56, i8 60>
+  %gather.address = getelementptr i8, <16 x i8*> %1, i8 3
+  %2 = getelementptr i8, i8* %pointer.phi13, i32 64
+  %3 = getelementptr i8, i8* %pointer.phi13, <16 x i8> <i8 0, i8 4, i8 8, i8 12, i8 16, i8 20, i8 24, i8 28, i8 32, i8 36, i8 40, i8 44, i8 48, i8 52, i8 56, i8 60>
+  %scatter.address = getelementptr i8, <16 x i8*> %3, i8 5
+  %wide.masked.gather = call <16 x i8> @llvm.masked.gather.v16i8.v16p0i8(<16 x i8*> %gather.address, i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i8> undef)
+  %4 = add nsw <16 x i8> %wide.masked.gather, %broadcast.splat
+  call void @llvm.masked.scatter.v16i8.v16p0i8(<16 x i8> %4, <16 x i8*> %scatter.address, i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  %index.next = add i32 %index, 4
+  %5 = icmp eq i32 %index.next, 996
+  br i1 %5, label %end, label %vector.body
+
+end:
+  ret void
+}
+
+define void @ptr_iv_v4f32(float* noalias nocapture readonly %A, float* noalias nocapture %B, float %y) {
+; CHECK-LABEL: ptr_iv_v4f32:
+; CHECK:       @ %bb.0: @ %vector.ph
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    adr r3, .LCPI6_0
+; CHECK-NEXT:    mov.w lr, #249
+; CHECK-NEXT:    vldrw.u32 q0, [r3]
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB6_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrw.u32 q1, [r0, q0, uxtw #2]
+; CHECK-NEXT:    adds r0, #64
+; CHECK-NEXT:    vadd.f32 q1, q1, r2
+; CHECK-NEXT:    vstrw.32 q1, [r1, q0, uxtw #2]
+; CHECK-NEXT:    adds r1, #64
+; CHECK-NEXT:    le lr, .LBB6_1
+; CHECK-NEXT:  @ %bb.2: @ %end
+; CHECK-NEXT:    pop {r7, pc}
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  @ %bb.3:
+; CHECK-NEXT:  .LCPI6_0:
+; CHECK-NEXT:    .long 0 @ 0x0
+; CHECK-NEXT:    .long 4 @ 0x4
+; CHECK-NEXT:    .long 8 @ 0x8
+; CHECK-NEXT:    .long 12 @ 0xc
+vector.ph:                                        ; preds = %entry
+  %broadcast.splatinsert = insertelement <4 x float> undef, float %y, i32 0
+  %broadcast.splat = shufflevector <4 x float> %broadcast.splatinsert, <4 x float> undef, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:
+  %pointer.phi = phi float* [ %A, %vector.ph ], [ %0, %vector.body ]
+  %pointer.phi13 = phi float* [ %B, %vector.ph ], [ %2, %vector.body ]
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %0 = getelementptr float, float* %pointer.phi, i32 16
+  %1 = getelementptr float, float* %pointer.phi, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+  %2 = getelementptr float, float* %pointer.phi13, i32 16
+  %3 = getelementptr float, float* %pointer.phi13, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+  %wide.masked.gather = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> %1, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x float> undef)
+  %4 = fadd <4 x float> %wide.masked.gather, %broadcast.splat
+  call void @llvm.masked.scatter.v4f32.v4p0f32(<4 x float> %4, <4 x float*> %3, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
+  %index.next = add i32 %index, 4
+  %5 = icmp eq i32 %index.next, 996
+  br i1 %5, label %end, label %vector.body
+
+end:
+  ret void
+}
+
+define void @ptr_iv_v4f32_mult(float* noalias nocapture readonly %A, float* noalias nocapture %B, float %y) {
+; CHECK-LABEL: ptr_iv_v4f32_mult:
+; CHECK:       @ %bb.0: @ %vector.ph
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    adr r1, .LCPI7_0
+; CHECK-NEXT:    adr r3, .LCPI7_1
+; CHECK-NEXT:    vldrw.u32 q0, [r3]
+; CHECK-NEXT:    vldrw.u32 q1, [r1]
+; CHECK-NEXT:    mov.w lr, #249
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB7_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrw.u32 q2, [r0, q0, uxtw #2]
+; CHECK-NEXT:    vadd.f32 q2, q2, r2
+; CHECK-NEXT:    vstrw.32 q2, [r0, q1, uxtw #2]
+; CHECK-NEXT:    adds r0, #64
+; CHECK-NEXT:    le lr, .LBB7_1
+; CHECK-NEXT:  @ %bb.2: @ %end
+; CHECK-NEXT:    pop {r7, pc}
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  @ %bb.3:
+; CHECK-NEXT:  .LCPI7_0:
+; CHECK-NEXT:    .long 5 @ 0x5
+; CHECK-NEXT:    .long 9 @ 0x9
+; CHECK-NEXT:    .long 13 @ 0xd
+; CHECK-NEXT:    .long 17 @ 0x11
+; CHECK-NEXT:  .LCPI7_1:
+; CHECK-NEXT:    .long 3 @ 0x3
+; CHECK-NEXT:    .long 7 @ 0x7
+; CHECK-NEXT:    .long 11 @ 0xb
+; CHECK-NEXT:    .long 15 @ 0xf
+vector.ph:                                        ; preds = %entry
+  %broadcast.splatinsert = insertelement <4 x float> undef, float %y, i32 0
+  %broadcast.splat = shufflevector <4 x float> %broadcast.splatinsert, <4 x float> undef, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:
+  %pointer.phi = phi float* [ %A, %vector.ph ], [ %0, %vector.body ]
+  %pointer.phi13 = phi float* [ %B, %vector.ph ], [ %2, %vector.body ]
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %0 = getelementptr float, float* %pointer.phi, i32 16
+  %1 = getelementptr float, float* %pointer.phi, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+  %gather.address = getelementptr float, <4 x float*> %1, i32 3
+  %2 = getelementptr float, float* %pointer.phi13, i32 16
+  %3 = getelementptr float, float* %pointer.phi13, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+  %scatter.address = getelementptr float, <4 x float*> %1, i32 5
+  %wide.masked.gather = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> %gather.address, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x float> undef)
+  %4 = fadd <4 x float> %wide.masked.gather, %broadcast.splat
+  call void @llvm.masked.scatter.v4f32.v4p0f32(<4 x float> %4, <4 x float*> %scatter.address, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
+  %index.next = add i32 %index, 4
+  %5 = icmp eq i32 %index.next, 996
+  br i1 %5, label %end, label %vector.body
+
+end:
+  ret void
+}
+
+define void @ptr_iv_v8f16(half* noalias nocapture readonly %A, half* noalias nocapture %B, float %y) {
+; CHECK-LABEL: ptr_iv_v8f16:
+; CHECK:       @ %bb.0: @ %vector.ph
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    vmov s0, r2
+; CHECK-NEXT:    adr r3, .LCPI8_0
+; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
+; CHECK-NEXT:    mov.w lr, #249
+; CHECK-NEXT:    vmov.f16 r2, s0
+; CHECK-NEXT:    vldrw.u32 q0, [r3]
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB8_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrh.u16 q1, [r0, q0, uxtw #1]
+; CHECK-NEXT:    adds r0, #64
+; CHECK-NEXT:    vadd.f16 q1, q1, r2
+; CHECK-NEXT:    vstrh.16 q1, [r1, q0, uxtw #1]
+; CHECK-NEXT:    adds r1, #64
+; CHECK-NEXT:    le lr, .LBB8_1
+; CHECK-NEXT:  @ %bb.2: @ %end
+; CHECK-NEXT:    pop {r7, pc}
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  @ %bb.3:
+; CHECK-NEXT:  .LCPI8_0:
+; CHECK-NEXT:    .short 0 @ 0x0
+; CHECK-NEXT:    .short 4 @ 0x4
+; CHECK-NEXT:    .short 8 @ 0x8
+; CHECK-NEXT:    .short 12 @ 0xc
+; CHECK-NEXT:    .short 16 @ 0x10
+; CHECK-NEXT:    .short 20 @ 0x14
+; CHECK-NEXT:    .short 24 @ 0x18
+; CHECK-NEXT:    .short 28 @ 0x1c
+vector.ph:
+  %y.trunc = fptrunc float %y to half
+  %broadcast.splatinsert = insertelement <8 x half> undef, half %y.trunc, i32 0
+  %broadcast.splat = shufflevector <8 x half> %broadcast.splatinsert, <8 x half> undef, <8 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:
+  %pointer.phi = phi half* [ %A, %vector.ph ], [ %0, %vector.body ]
+  %pointer.phi13 = phi half* [ %B, %vector.ph ], [ %2, %vector.body ]
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %0 = getelementptr half, half* %pointer.phi, i32 32
+  %1 = getelementptr half, half* %pointer.phi, <8 x i16> <i16 0, i16 4, i16 8, i16 12, i16 16, i16 20, i16 24, i16 28>
+  %2 = getelementptr half, half* %pointer.phi13, i32 32
+  %3 = getelementptr half, half* %pointer.phi13, <8 x i16> <i16 0, i16 4, i16 8, i16 12, i16 16, i16 20, i16 24, i16 28>
+  %wide.masked.gather = call <8 x half> @llvm.masked.gather.v8f16.v8p0f16(<8 x half*> %1, i32 4, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x half> undef)
+  %4 = fadd <8 x half> %wide.masked.gather, %broadcast.splat
+  call void @llvm.masked.scatter.v8f16.v8p0f16(<8 x half> %4, <8 x half*> %3, i32 4, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  %index.next = add i32 %index, 4
+  %5 = icmp eq i32 %index.next, 996
+  br i1 %5, label %end, label %vector.body
+
+end:
+  ret void
+}
+
+define void @ptr_iv_v8f16_mult(half* noalias nocapture readonly %A, half* noalias nocapture %B, float %y) {
+; CHECK-LABEL: ptr_iv_v8f16_mult:
+; CHECK:       @ %bb.0: @ %vector.ph
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    vmov s0, r2
+; CHECK-NEXT:    adr r2, .LCPI9_0
+; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
+; CHECK-NEXT:    mov.w lr, #249
+; CHECK-NEXT:    vmov.f16 r1, s0
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    adr r2, .LCPI9_1
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:  .LBB9_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrh.u16 q2, [r0, q0, uxtw #1]
+; CHECK-NEXT:    vadd.f16 q2, q2, r1
+; CHECK-NEXT:    vstrh.16 q2, [r0, q1, uxtw #1]
+; CHECK-NEXT:    adds r0, #64
+; CHECK-NEXT:    le lr, .LBB9_1
+; CHECK-NEXT:  @ %bb.2: @ %end
+; CHECK-NEXT:    pop {r7, pc}
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  @ %bb.3:
+; CHECK-NEXT:  .LCPI9_0:
+; CHECK-NEXT:    .short 3 @ 0x3
+; CHECK-NEXT:    .short 7 @ 0x7
+; CHECK-NEXT:    .short 11 @ 0xb
+; CHECK-NEXT:    .short 15 @ 0xf
+; CHECK-NEXT:    .short 19 @ 0x13
+; CHECK-NEXT:    .short 23 @ 0x17
+; CHECK-NEXT:    .short 27 @ 0x1b
+; CHECK-NEXT:    .short 31 @ 0x1f
+; CHECK-NEXT:  .LCPI9_1:
+; CHECK-NEXT:    .short 5 @ 0x5
+; CHECK-NEXT:    .short 9 @ 0x9
+; CHECK-NEXT:    .short 13 @ 0xd
+; CHECK-NEXT:    .short 17 @ 0x11
+; CHECK-NEXT:    .short 21 @ 0x15
+; CHECK-NEXT:    .short 25 @ 0x19
+; CHECK-NEXT:    .short 29 @ 0x1d
+; CHECK-NEXT:    .short 33 @ 0x21
+vector.ph:
+  %y.trunc = fptrunc float %y to half
+  %broadcast.splatinsert = insertelement <8 x half> undef, half %y.trunc, i32 0
+  %broadcast.splat = shufflevector <8 x half> %broadcast.splatinsert, <8 x half> undef, <8 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:
+  %pointer.phi = phi half* [ %A, %vector.ph ], [ %0, %vector.body ]
+  %pointer.phi13 = phi half* [ %B, %vector.ph ], [ %2, %vector.body ]
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %0 = getelementptr half, half* %pointer.phi, i32 32
+  %1 = getelementptr half, half* %pointer.phi, <8 x i16> <i16 0, i16 4, i16 8, i16 12, i16 16, i16 20, i16 24, i16 28>
+  %gather.address = getelementptr half, <8 x half*> %1, i32 3
+  %2 = getelementptr half, half* %pointer.phi13, i32 32
+  %3 = getelementptr half, half* %pointer.phi13, <8 x i16> <i16 0, i16 4, i16 8, i16 12, i16 16, i16 20, i16 24, i16 28>
+  %scatter.address = getelementptr half, <8 x half*> %1, i32 5
+  %wide.masked.gather = call <8 x half> @llvm.masked.gather.v8f16.v8p0f16(<8 x half*> %gather.address, i32 4, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x half> undef)
+  %4 = fadd <8 x half> %wide.masked.gather, %broadcast.splat
+  call void @llvm.masked.scatter.v8f16.v8p0f16(<8 x half> %4, <8 x half*> %scatter.address, i32 4, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  %index.next = add i32 %index, 4
+  %5 = icmp eq i32 %index.next, 996
+  br i1 %5, label %end, label %vector.body
+
+end:
+  ret void
+}
+
+
+define arm_aapcs_vfpcc void @three_pointer_iv_v4i32(i32* nocapture readonly %x, i32* nocapture %z, i32 %n) {
+; CHECK-LABEL: three_pointer_iv_v4i32:
+; CHECK:       @ %bb.0: @ %vector.ph
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT:    adr.w r12, .LCPI10_0
+; CHECK-NEXT:    adr.w lr, .LCPI10_1
+; CHECK-NEXT:    adr r3, .LCPI10_2
+; CHECK-NEXT:    vldrw.u32 q2, [lr]
+; CHECK-NEXT:    vldrw.u32 q1, [r3]
+; CHECK-NEXT:    vldrw.u32 q3, [r12]
+; CHECK-NEXT:    vmov.i32 q0, #0xa
+; CHECK-NEXT:  .LBB10_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrw.u32 q4, [r0, q1, uxtw #2]
+; CHECK-NEXT:    vldrw.u32 q5, [r0, q2, uxtw #2]
+; CHECK-NEXT:    vldrw.u32 q6, [r0, q3, uxtw #2]
+; CHECK-NEXT:    subs r2, #4
+; CHECK-NEXT:    vmul.i32 q4, q5, q4
+; CHECK-NEXT:    add.w r0, r0, #48
+; CHECK-NEXT:    vmul.i32 q6, q5, q6
+; CHECK-NEXT:    vmul.i32 q5, q5, q0
+; CHECK-NEXT:    vstrw.32 q5, [r1, q2, uxtw #2]
+; CHECK-NEXT:    vstrw.32 q6, [r1, q3, uxtw #2]
+; CHECK-NEXT:    vstrw.32 q4, [r1, q1, uxtw #2]
+; CHECK-NEXT:    add.w r1, r1, #48
+; CHECK-NEXT:    bne .LBB10_1
+; CHECK-NEXT:  @ %bb.2: @ %end
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT:    pop {r7, pc}
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  @ %bb.3:
+; CHECK-NEXT:  .LCPI10_0:
+; CHECK-NEXT:    .long 1 @ 0x1
+; CHECK-NEXT:    .long 4 @ 0x4
+; CHECK-NEXT:    .long 7 @ 0x7
+; CHECK-NEXT:    .long 10 @ 0xa
+; CHECK-NEXT:  .LCPI10_1:
+; CHECK-NEXT:    .long 0 @ 0x0
+; CHECK-NEXT:    .long 3 @ 0x3
+; CHECK-NEXT:    .long 6 @ 0x6
+; CHECK-NEXT:    .long 9 @ 0x9
+; CHECK-NEXT:  .LCPI10_2:
+; CHECK-NEXT:    .long 2 @ 0x2
+; CHECK-NEXT:    .long 5 @ 0x5
+; CHECK-NEXT:    .long 8 @ 0x8
+; CHECK-NEXT:    .long 11 @ 0xb
+vector.ph:
+  br label %vector.body
+
+vector.body:
+  %pointer.phi = phi i32* [ %x, %vector.ph ], [ %v3, %vector.body ]
+  %pointer.phi55 = phi i32* [ %z, %vector.ph ], [ %v4, %vector.body ]
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %vector.gep = getelementptr i32, i32* %pointer.phi, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
+  %v3 = getelementptr i32, i32* %pointer.phi, i32 12
+  %vector.gep56 = getelementptr i32, i32* %pointer.phi55, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
+  %v4 = getelementptr i32, i32* %pointer.phi55, i32 12
+  %v5 = add i32 %index, 0
+  %v6 = getelementptr inbounds i32, <4 x i32*> %vector.gep, i32 1
+  %wide.masked.gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %vector.gep, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
+  %v7 = getelementptr inbounds i32, <4 x i32*> %vector.gep, i32 2
+  %wide.masked.gather57 = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %v6, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
+  %wide.masked.gather58 = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %v7, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
+  %v11 = mul nuw nsw <4 x i32> %wide.masked.gather, <i32 10, i32 10, i32 10, i32 10>
+  %v13 = mul nuw nsw <4 x i32> %wide.masked.gather, %wide.masked.gather57
+  %v15 = mul nuw nsw <4 x i32> %wide.masked.gather, %wide.masked.gather58
+  %v17 = getelementptr inbounds i32, <4 x i32*> %vector.gep56, i32 1
+  call void @llvm.masked.scatter.v4i32.v4p0i32(<4 x i32> %v11, <4 x i32*> %vector.gep56, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
+  %v18 = getelementptr inbounds i32, <4 x i32*> %vector.gep56, i32 2
+  call void @llvm.masked.scatter.v4i32.v4p0i32(<4 x i32> %v13, <4 x i32*> %v17, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
+  call void @llvm.masked.scatter.v4i32.v4p0i32(<4 x i32> %v15, <4 x i32*> %v18, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
+  %index.next = add i32 %index, 4
+  %v37 = icmp eq i32 %index.next, %n
+  br i1 %v37, label %end, label %vector.body
+
+end:
+  ret void;
+}
+
+define arm_aapcs_vfpcc void @three_pointer_iv_v4i8(i8* nocapture readonly %x, i8* nocapture %z, i32 %n) {
+; CHECK-LABEL: three_pointer_iv_v4i8:
+; CHECK:       @ %bb.0: @ %vector.ph
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT:    adr.w r12, .LCPI11_0
+; CHECK-NEXT:    adr.w lr, .LCPI11_1
+; CHECK-NEXT:    adr r3, .LCPI11_2
+; CHECK-NEXT:    vldrw.u32 q2, [lr]
+; CHECK-NEXT:    vldrw.u32 q1, [r3]
+; CHECK-NEXT:    vldrw.u32 q3, [r12]
+; CHECK-NEXT:    vmov.i32 q0, #0xa
+; CHECK-NEXT:  .LBB11_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrb.u32 q4, [r0, q1]
+; CHECK-NEXT:    vldrb.u32 q5, [r0, q2]
+; CHECK-NEXT:    vldrb.u32 q6, [r0, q3]
+; CHECK-NEXT:    subs r2, #4
+; CHECK-NEXT:    vmul.i32 q4, q5, q4
+; CHECK-NEXT:    add.w r0, r0, #12
+; CHECK-NEXT:    vmul.i32 q6, q5, q6
+; CHECK-NEXT:    vmul.i32 q5, q5, q0
+; CHECK-NEXT:    vstrb.32 q5, [r1, q2]
+; CHECK-NEXT:    vstrb.32 q6, [r1, q3]
+; CHECK-NEXT:    vstrb.32 q4, [r1, q1]
+; CHECK-NEXT:    add.w r1, r1, #12
+; CHECK-NEXT:    bne .LBB11_1
+; CHECK-NEXT:  @ %bb.2: @ %end
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT:    pop {r7, pc}
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  @ %bb.3:
+; CHECK-NEXT:  .LCPI11_0:
+; CHECK-NEXT:    .long 1 @ 0x1
+; CHECK-NEXT:    .long 4 @ 0x4
+; CHECK-NEXT:    .long 7 @ 0x7
+; CHECK-NEXT:    .long 10 @ 0xa
+; CHECK-NEXT:  .LCPI11_1:
+; CHECK-NEXT:    .long 0 @ 0x0
+; CHECK-NEXT:    .long 3 @ 0x3
+; CHECK-NEXT:    .long 6 @ 0x6
+; CHECK-NEXT:    .long 9 @ 0x9
+; CHECK-NEXT:  .LCPI11_2:
+; CHECK-NEXT:    .long 2 @ 0x2
+; CHECK-NEXT:    .long 5 @ 0x5
+; CHECK-NEXT:    .long 8 @ 0x8
+; CHECK-NEXT:    .long 11 @ 0xb
+vector.ph:
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %pointer.phi = phi i8* [ %x, %vector.ph ], [ %v3, %vector.body ]
+  %pointer.phi55 = phi i8* [ %z, %vector.ph ], [ %v4, %vector.body ]
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %vector.gep = getelementptr i8, i8* %pointer.phi, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
+  %v3 = getelementptr i8, i8* %pointer.phi, i32 12
+  %vector.gep56 = getelementptr i8, i8* %pointer.phi55, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
+  %v4 = getelementptr i8, i8* %pointer.phi55, i32 12
+  %v5 = add i32 %index, 0
+  %v6 = getelementptr inbounds i8, <4 x i8*> %vector.gep, i32 1
+  %wide.masked.gather = call <4 x i8> @llvm.masked.gather.v4i8.v4p0i8(<4 x i8*> %vector.gep, i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i8> undef)
+  %v7 = getelementptr inbounds i8, <4 x i8*> %vector.gep, i32 2
+  %wide.masked.gather57 = call <4 x i8> @llvm.masked.gather.v4i8.v4p0i8(<4 x i8*> %v6, i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i8> undef)
+  %wide.masked.gather58 = call <4 x i8> @llvm.masked.gather.v4i8.v4p0i8(<4 x i8*> %v7, i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i8> undef)
+  %v8 = zext <4 x i8> %wide.masked.gather to <4 x i32>
+  %v9 = zext <4 x i8> %wide.masked.gather57 to <4 x i32>
+  %v10 = zext <4 x i8> %wide.masked.gather58 to <4 x i32>
+  %v11 = mul nuw nsw <4 x i32> %v8, <i32 10, i32 10, i32 10, i32 10>
+  %v12 = trunc <4 x i32> %v11 to <4 x i8>
+  %v13 = mul nuw nsw <4 x i32> %v8, %v9
+  %v14 = trunc <4 x i32> %v13 to <4 x i8>
+  %v15 = mul nuw nsw <4 x i32> %v8, %v10
+  %v16 = trunc <4 x i32> %v15 to <4 x i8>
+  %v17 = getelementptr inbounds i8, <4 x i8*> %vector.gep56, i32 1
+  call void @llvm.masked.scatter.v4i8.v4p0i8(<4 x i8> %v12, <4 x i8*> %vector.gep56, i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
+  %v18 = getelementptr inbounds i8, <4 x i8*> %vector.gep56, i32 2
+  call void @llvm.masked.scatter.v4i8.v4p0i8(<4 x i8> %v14, <4 x i8*> %v17, i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
+  call void @llvm.masked.scatter.v4i8.v4p0i8(<4 x i8> %v16, <4 x i8*> %v18, i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
+  %index.next = add i32 %index, 4
+  %v37 = icmp eq i32 %index.next, %n
+  br i1 %v37, label %end, label %vector.body
+
+end:
+  ret void;
+}
+
+define arm_aapcs_vfpcc void @three_pointer_iv_v8i16(i16* nocapture readonly %x, i16* nocapture %z, i32 %n) {
+; CHECK-LABEL: three_pointer_iv_v8i16:
+; CHECK:       @ %bb.0: @ %vector.ph
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT:    adr.w r12, .LCPI12_0
+; CHECK-NEXT:    adr.w lr, .LCPI12_1
+; CHECK-NEXT:    adr r3, .LCPI12_2
+; CHECK-NEXT:    vldrw.u32 q2, [lr]
+; CHECK-NEXT:    vldrw.u32 q1, [r3]
+; CHECK-NEXT:    vldrw.u32 q3, [r12]
+; CHECK-NEXT:    vmov.i16 q0, #0xa
+; CHECK-NEXT:  .LBB12_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrh.u16 q4, [r0, q1, uxtw #1]
+; CHECK-NEXT:    vldrh.u16 q5, [r0, q2, uxtw #1]
+; CHECK-NEXT:    vldrh.u16 q6, [r0, q3, uxtw #1]
+; CHECK-NEXT:    subs r2, #4
+; CHECK-NEXT:    vmul.i16 q4, q5, q4
+; CHECK-NEXT:    add.w r0, r0, #48
+; CHECK-NEXT:    vmul.i16 q6, q5, q6
+; CHECK-NEXT:    vmul.i16 q5, q5, q0
+; CHECK-NEXT:    vstrh.16 q5, [r1, q2, uxtw #1]
+; CHECK-NEXT:    vstrh.16 q6, [r1, q3, uxtw #1]
+; CHECK-NEXT:    vstrh.16 q4, [r1, q1, uxtw #1]
+; CHECK-NEXT:    add.w r1, r1, #48
+; CHECK-NEXT:    bne .LBB12_1
+; CHECK-NEXT:  @ %bb.2: @ %end
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT:    pop {r7, pc}
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  @ %bb.3:
+; CHECK-NEXT:  .LCPI12_0:
+; CHECK-NEXT:    .short 1 @ 0x1
+; CHECK-NEXT:    .short 4 @ 0x4
+; CHECK-NEXT:    .short 7 @ 0x7
+; CHECK-NEXT:    .short 10 @ 0xa
+; CHECK-NEXT:    .short 13 @ 0xd
+; CHECK-NEXT:    .short 16 @ 0x10
+; CHECK-NEXT:    .short 19 @ 0x13
+; CHECK-NEXT:    .short 22 @ 0x16
+; CHECK-NEXT:  .LCPI12_1:
+; CHECK-NEXT:    .short 0 @ 0x0
+; CHECK-NEXT:    .short 3 @ 0x3
+; CHECK-NEXT:    .short 6 @ 0x6
+; CHECK-NEXT:    .short 9 @ 0x9
+; CHECK-NEXT:    .short 12 @ 0xc
+; CHECK-NEXT:    .short 15 @ 0xf
+; CHECK-NEXT:    .short 18 @ 0x12
+; CHECK-NEXT:    .short 21 @ 0x15
+; CHECK-NEXT:  .LCPI12_2:
+; CHECK-NEXT:    .short 2 @ 0x2
+; CHECK-NEXT:    .short 5 @ 0x5
+; CHECK-NEXT:    .short 8 @ 0x8
+; CHECK-NEXT:    .short 11 @ 0xb
+; CHECK-NEXT:    .short 14 @ 0xe
+; CHECK-NEXT:    .short 17 @ 0x11
+; CHECK-NEXT:    .short 20 @ 0x14
+; CHECK-NEXT:    .short 23 @ 0x17
+vector.ph:
+  br label %vector.body
+
+vector.body:
+  %pointer.phi = phi i16* [ %x, %vector.ph ], [ %v3, %vector.body ]
+  %pointer.phi55 = phi i16* [ %z, %vector.ph ], [ %v4, %vector.body ]
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %vector.gep = getelementptr i16, i16* %pointer.phi, <8 x i16> <i16 0, i16 3, i16 6, i16 9, i16 12, i16 15, i16 18, i16 21>
+  %v3 = getelementptr i16, i16* %pointer.phi, i32 24
+  %vector.gep56 = getelementptr i16, i16* %pointer.phi55, <8 x i16> <i16 0, i16 3, i16 6, i16 9, i16 12, i16 15, i16 18, i16 21>
+  %v4 = getelementptr i16, i16* %pointer.phi55, i32 24
+  %v5 = add i32 %index, 0
+  %v6 = getelementptr inbounds i16, <8 x i16*> %vector.gep, i16 1
+  %wide.masked.gather = call <8 x i16> @llvm.masked.gather.v8i16.v8p0i16(<8 x i16*> %vector.gep, i32 2, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i16> undef)
+  %v7 = getelementptr inbounds i16, <8 x i16*> %vector.gep, i16 2
+  %wide.masked.gather57 = call <8 x i16> @llvm.masked.gather.v8i16.v8p0i16(<8 x i16*> %v6, i32 2, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i16> undef)
+  %wide.masked.gather58 = call <8 x i16> @llvm.masked.gather.v8i16.v8p0i16(<8 x i16*> %v7, i32 2, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i16> undef)
+  %v11 = mul nuw nsw <8 x i16> %wide.masked.gather, <i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10>
+  %v13 = mul nuw nsw <8 x i16> %wide.masked.gather, %wide.masked.gather57
+  %v15 = mul nuw nsw <8 x i16> %wide.masked.gather, %wide.masked.gather58
+  %v17 = getelementptr inbounds i16, <8 x i16*> %vector.gep56, i32 1
+  call void @llvm.masked.scatter.v8i16.v8p0i16(<8 x i16> %v11, <8 x i16*> %vector.gep56, i32 2, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  %v18 = getelementptr inbounds i16, <8 x i16*> %vector.gep56, i32 2
+  call void @llvm.masked.scatter.v8i16.v8p0i16(<8 x i16> %v13, <8 x i16*> %v17, i32 2, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  call void @llvm.masked.scatter.v8i16.v8p0i16(<8 x i16> %v15, <8 x i16*> %v18, i32 2, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  %index.next = add i32 %index, 4
+  %v37 = icmp eq i32 %index.next, %n
+  br i1 %v37, label %end, label %vector.body
+
+end:
+  ret void;
+}
+
+define arm_aapcs_vfpcc void @three_pointer_iv_v16i8(i8* nocapture readonly %x, i8* nocapture %z, i32 %n) {
+; CHECK-LABEL: three_pointer_iv_v16i8:
+; CHECK:       @ %bb.0: @ %vector.ph
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT:    adr.w r12, .LCPI13_0
+; CHECK-NEXT:    adr.w lr, .LCPI13_1
+; CHECK-NEXT:    adr r3, .LCPI13_2
+; CHECK-NEXT:    vldrw.u32 q2, [lr]
+; CHECK-NEXT:    vldrw.u32 q1, [r3]
+; CHECK-NEXT:    vldrw.u32 q3, [r12]
+; CHECK-NEXT:    vmov.i8 q0, #0xa
+; CHECK-NEXT:  .LBB13_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrb.u8 q4, [r0, q1]
+; CHECK-NEXT:    vldrb.u8 q5, [r0, q2]
+; CHECK-NEXT:    vldrb.u8 q6, [r0, q3]
+; CHECK-NEXT:    subs r2, #4
+; CHECK-NEXT:    vmul.i8 q4, q5, q4
+; CHECK-NEXT:    add.w r0, r0, #48
+; CHECK-NEXT:    vmul.i8 q6, q5, q6
+; CHECK-NEXT:    vmul.i8 q5, q5, q0
+; CHECK-NEXT:    vstrb.8 q5, [r1, q2]
+; CHECK-NEXT:    vstrb.8 q6, [r1, q3]
+; CHECK-NEXT:    vstrb.8 q4, [r1, q1]
+; CHECK-NEXT:    add.w r1, r1, #48
+; CHECK-NEXT:    bne .LBB13_1
+; CHECK-NEXT:  @ %bb.2: @ %end
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT:    pop {r7, pc}
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  @ %bb.3:
+; CHECK-NEXT:  .LCPI13_0:
+; CHECK-NEXT:    .byte 1 @ 0x1
+; CHECK-NEXT:    .byte 4 @ 0x4
+; CHECK-NEXT:    .byte 7 @ 0x7
+; CHECK-NEXT:    .byte 10 @ 0xa
+; CHECK-NEXT:    .byte 13 @ 0xd
+; CHECK-NEXT:    .byte 16 @ 0x10
+; CHECK-NEXT:    .byte 19 @ 0x13
+; CHECK-NEXT:    .byte 22 @ 0x16
+; CHECK-NEXT:    .byte 25 @ 0x19
+; CHECK-NEXT:    .byte 28 @ 0x1c
+; CHECK-NEXT:    .byte 31 @ 0x1f
+; CHECK-NEXT:    .byte 34 @ 0x22
+; CHECK-NEXT:    .byte 37 @ 0x25
+; CHECK-NEXT:    .byte 40 @ 0x28
+; CHECK-NEXT:    .byte 43 @ 0x2b
+; CHECK-NEXT:    .byte 46 @ 0x2e
+; CHECK-NEXT:  .LCPI13_1:
+; CHECK-NEXT:    .byte 0 @ 0x0
+; CHECK-NEXT:    .byte 3 @ 0x3
+; CHECK-NEXT:    .byte 6 @ 0x6
+; CHECK-NEXT:    .byte 9 @ 0x9
+; CHECK-NEXT:    .byte 12 @ 0xc
+; CHECK-NEXT:    .byte 15 @ 0xf
+; CHECK-NEXT:    .byte 18 @ 0x12
+; CHECK-NEXT:    .byte 21 @ 0x15
+; CHECK-NEXT:    .byte 24 @ 0x18
+; CHECK-NEXT:    .byte 27 @ 0x1b
+; CHECK-NEXT:    .byte 30 @ 0x1e
+; CHECK-NEXT:    .byte 33 @ 0x21
+; CHECK-NEXT:    .byte 36 @ 0x24
+; CHECK-NEXT:    .byte 39 @ 0x27
+; CHECK-NEXT:    .byte 42 @ 0x2a
+; CHECK-NEXT:    .byte 45 @ 0x2d
+; CHECK-NEXT:  .LCPI13_2:
+; CHECK-NEXT:    .byte 2 @ 0x2
+; CHECK-NEXT:    .byte 5 @ 0x5
+; CHECK-NEXT:    .byte 8 @ 0x8
+; CHECK-NEXT:    .byte 11 @ 0xb
+; CHECK-NEXT:    .byte 14 @ 0xe
+; CHECK-NEXT:    .byte 17 @ 0x11
+; CHECK-NEXT:    .byte 20 @ 0x14
+; CHECK-NEXT:    .byte 23 @ 0x17
+; CHECK-NEXT:    .byte 26 @ 0x1a
+; CHECK-NEXT:    .byte 29 @ 0x1d
+; CHECK-NEXT:    .byte 32 @ 0x20
+; CHECK-NEXT:    .byte 35 @ 0x23
+; CHECK-NEXT:    .byte 38 @ 0x26
+; CHECK-NEXT:    .byte 41 @ 0x29
+; CHECK-NEXT:    .byte 44 @ 0x2c
+; CHECK-NEXT:    .byte 47 @ 0x2f
+vector.ph:
+  br label %vector.body
+
+vector.body:
+  %pointer.phi = phi i8* [ %x, %vector.ph ], [ %v3, %vector.body ]
+  %pointer.phi55 = phi i8* [ %z, %vector.ph ], [ %v4, %vector.body ]
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %vector.gep = getelementptr i8, i8* %pointer.phi, <16 x i8> <i8 0, i8 3, i8 6, i8 9, i8 12, i8 15, i8 18, i8 21, i8 24, i8 27, i8 30, i8 33, i8 36, i8 39, i8 42, i8 45>
+  %v3 = getelementptr i8, i8* %pointer.phi, i32 48
+  %vector.gep56 = getelementptr i8, i8* %pointer.phi55, <16 x i8> <i8 0, i8 3, i8 6, i8 9, i8 12, i8 15, i8 18, i8 21, i8 24, i8 27, i8 30, i8 33, i8 36, i8 39, i8 42, i8 45>
+  %v4 = getelementptr i8, i8* %pointer.phi55, i32 48
+  %v5 = add i32 %index, 0
+  %v6 = getelementptr inbounds i8, <16 x i8*> %vector.gep, i8 1
+  %wide.masked.gather = call <16 x i8> @llvm.masked.gather.v16i8.v16p0i8(<16 x i8*> %vector.gep, i32 1, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i8> undef)
+  %v7 = getelementptr inbounds i8, <16 x i8*> %vector.gep, i8 2
+  %wide.masked.gather57 = call <16 x i8> @llvm.masked.gather.v16i8.v16p0i8(<16 x i8*> %v6, i32 1, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i8> undef)
+  %wide.masked.gather58 = call <16 x i8> @llvm.masked.gather.v16i8.v16p0i8(<16 x i8*> %v7, i32 1, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i8> undef)
+  %v11 = mul nuw nsw <16 x i8> %wide.masked.gather, <i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10>
+  %v13 = mul nuw nsw <16 x i8> %wide.masked.gather, %wide.masked.gather57
+  %v15 = mul nuw nsw <16 x i8> %wide.masked.gather, %wide.masked.gather58
+  %v17 = getelementptr inbounds i8, <16 x i8*> %vector.gep56, i32 1
+  call void @llvm.masked.scatter.v16i8.v16p0i8(<16 x i8> %v11, <16 x i8*> %vector.gep56, i32 1, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  %v18 = getelementptr inbounds i8, <16 x i8*> %vector.gep56, i32 2
+  call void @llvm.masked.scatter.v16i8.v16p0i8(<16 x i8> %v13, <16 x i8*> %v17, i32 1, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  call void @llvm.masked.scatter.v16i8.v16p0i8(<16 x i8> %v15, <16 x i8*> %v18, i32 1, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  %index.next = add i32 %index, 4
+  %v37 = icmp eq i32 %index.next, %n
+  br i1 %v37, label %end, label %vector.body
+
+end:
+  ret void;
+}
+
+declare <4 x i8> @llvm.masked.gather.v4i8.v4p0i8(<4 x i8*>, i32, <4 x i1>, <4 x i8>)
+declare <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*>, i32, <4 x i1>, <4 x i32>)
+declare <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*>, i32, <4 x i1>, <4 x float>)
+declare <8 x i16> @llvm.masked.gather.v8i16.v8p0i16(<8 x i16*>, i32, <8 x i1>, <8 x i16>)
+declare <8 x half> @llvm.masked.gather.v8f16.v8p0f16(<8 x half*>, i32, <8 x i1>, <8 x half>)
+declare <16 x i8> @llvm.masked.gather.v16i8.v16p0i8(<16 x i8*>, i32, <16 x i1>, <16 x i8>)
+
+declare void @llvm.masked.scatter.v4i8.v4p0i8(<4 x i8>, <4 x i8*>, i32, <4 x i1>)
+declare void @llvm.masked.scatter.v4i32.v4p0i32(<4 x i32>, <4 x i32*>, i32, <4 x i1>)
+declare void @llvm.masked.scatter.v4f32.v4p0f32(<4 x float>, <4 x float*>, i32, <4 x i1>)
+declare void @llvm.masked.scatter.v8i16.v8p0i16(<8 x i16>, <8 x i16*>, i32, <8 x i1>)
+declare void @llvm.masked.scatter.v8f16.v8p0f16(<8 x half>, <8 x half*>, i32, <8 x i1>)
+declare void @llvm.masked.scatter.v16i8.v16p0i8(<16 x i8>, <16 x i8*>, i32, <16 x i1>)
diff --git a/llvm/test/CodeGen/Thumb2/mve-scatter-ind16-scaled.ll b/llvm/test/CodeGen/Thumb2/mve-scatter-ind16-scaled.ll
index 2d9e2047e08cb..ac615a1d57a1c 100644
--- a/llvm/test/CodeGen/Thumb2/mve-scatter-ind16-scaled.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-scatter-ind16-scaled.ll
@@ -177,5 +177,75 @@ entry:
   ret void
 }
 
+define arm_aapcs_vfpcc void @scaled_v8i16_i16_2gep(i16* %base, <8 x i16>* %offptr, <8 x i16> %input) {
+; CHECK-LABEL: scaled_v8i16_i16_2gep:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.s32 q1, [r1, #8]
+; CHECK-NEXT:    vldrh.s32 q3, [r1]
+; CHECK-NEXT:    vmov.i32 q2, #0x28
+; CHECK-NEXT:    vmov.u16 r1, q0[0]
+; CHECK-NEXT:    vshl.i32 q1, q1, #1
+; CHECK-NEXT:    vshl.i32 q3, q3, #1
+; CHECK-NEXT:    vadd.i32 q1, q1, r0
+; CHECK-NEXT:    vadd.i32 q3, q3, r0
+; CHECK-NEXT:    vadd.i32 q1, q1, q2
+; CHECK-NEXT:    vadd.i32 q2, q3, q2
+; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    strh r1, [r0]
+; CHECK-NEXT:    vmov r0, s9
+; CHECK-NEXT:    vmov.u16 r1, q0[1]
+; CHECK-NEXT:    strh r1, [r0]
+; CHECK-NEXT:    vmov r0, s10
+; CHECK-NEXT:    vmov.u16 r1, q0[2]
+; CHECK-NEXT:    strh r1, [r0]
+; CHECK-NEXT:    vmov r0, s11
+; CHECK-NEXT:    vmov.u16 r1, q0[3]
+; CHECK-NEXT:    strh r1, [r0]
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov.u16 r1, q0[4]
+; CHECK-NEXT:    strh r1, [r0]
+; CHECK-NEXT:    vmov r0, s5
+; CHECK-NEXT:    vmov.u16 r1, q0[5]
+; CHECK-NEXT:    strh r1, [r0]
+; CHECK-NEXT:    vmov r0, s6
+; CHECK-NEXT:    vmov.u16 r1, q0[6]
+; CHECK-NEXT:    strh r1, [r0]
+; CHECK-NEXT:    vmov r0, s7
+; CHECK-NEXT:    vmov.u16 r1, q0[7]
+; CHECK-NEXT:    strh r1, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <8 x i16>, <8 x i16>* %offptr, align 2
+  %ptrs = getelementptr inbounds i16, i16* %base, <8 x i16> %offs
+  %ptrs2 = getelementptr inbounds i16, <8 x i16*> %ptrs, i16 20
+  call void @llvm.masked.scatter.v8i16.v8p0i16(<8 x i16> %input, <8 x i16*> %ptrs2, i32 2, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @scaled_v8i16_i16_2gep2(i16* %base, <8 x i16>* %offptr, <8 x i16> %input) {
+; CHECK-LABEL: scaled_v8i16_i16_2gep2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    adr r1, .LCPI9_0
+; CHECK-NEXT:    vldrw.u32 q1, [r1]
+; CHECK-NEXT:    vstrh.16 q0, [r0, q1, uxtw #1]
+; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI9_0:
+; CHECK-NEXT:    .short 20 @ 0x14
+; CHECK-NEXT:    .short 23 @ 0x17
+; CHECK-NEXT:    .short 26 @ 0x1a
+; CHECK-NEXT:    .short 29 @ 0x1d
+; CHECK-NEXT:    .short 32 @ 0x20
+; CHECK-NEXT:    .short 35 @ 0x23
+; CHECK-NEXT:    .short 38 @ 0x26
+; CHECK-NEXT:    .short 41 @ 0x29
+entry:
+  %ptrs = getelementptr inbounds i16, i16* %base, <8 x i16> <i16 0, i16 3, i16 6, i16 9, i16 12, i16 15, i16 18, i16 21>
+  %ptrs2 = getelementptr inbounds i16, <8 x i16*> %ptrs, i16 20
+  call void @llvm.masked.scatter.v8i16.v8p0i16(<8 x i16> %input, <8 x i16*> %ptrs2, i32 2, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  ret void
+}
+
 declare void @llvm.masked.scatter.v8i16.v8p0i16(<8 x i16>, <8 x i16*>, i32, <8 x i1>)
 declare void @llvm.masked.scatter.v8f16.v8p0f16(<8 x half>, <8 x half*>, i32, <8 x i1>)
diff --git a/llvm/test/CodeGen/Thumb2/mve-scatter-ind32-scaled.ll b/llvm/test/CodeGen/Thumb2/mve-scatter-ind32-scaled.ll
index 25a9cea5f5be9..1c9871e54b10d 100644
--- a/llvm/test/CodeGen/Thumb2/mve-scatter-ind32-scaled.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-scatter-ind32-scaled.ll
@@ -233,6 +233,58 @@ entry:
   ret void
 }
 
+define arm_aapcs_vfpcc void @ext_scaled_i16_i32_2gep(i16* %base, <4 x i32>* %offptr, <4 x i32> %input) {
+; CHECK-LABEL: ext_scaled_i16_i32_2gep:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q2, [r1]
+; CHECK-NEXT:    vmov.i32 q1, #0xa
+; CHECK-NEXT:    vmov r1, s0
+; CHECK-NEXT:    vshl.i32 q2, q2, #1
+; CHECK-NEXT:    vadd.i32 q2, q2, r0
+; CHECK-NEXT:    vadd.i32 q1, q2, q1
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    strh r1, [r0]
+; CHECK-NEXT:    vmov r0, s5
+; CHECK-NEXT:    vmov r1, s1
+; CHECK-NEXT:    strh r1, [r0]
+; CHECK-NEXT:    vmov r0, s6
+; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    strh r1, [r0]
+; CHECK-NEXT:    vmov r0, s7
+; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    strh r1, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <4 x i32>, <4 x i32>* %offptr, align 4
+  %ptrs = getelementptr inbounds i16, i16* %base, <4 x i32> %offs
+  %ptrs2 = getelementptr inbounds i16, <4 x i16*> %ptrs, i16 5
+  %t = trunc <4 x i32> %input to <4 x i16>
+  call void @llvm.masked.scatter.v4i16.v4p0i16(<4 x i16> %t, <4 x i16*> %ptrs2, i32 2, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @ext_scaled_i16_i32_2gep2(i16* %base, <4 x i32>* %offptr, <4 x i32> %input) {
+; CHECK-LABEL: ext_scaled_i16_i32_2gep2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    adr r1, .LCPI16_0
+; CHECK-NEXT:    vldrw.u32 q1, [r1]
+; CHECK-NEXT:    vstrh.32 q0, [r0, q1, uxtw #1]
+; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI16_0:
+; CHECK-NEXT:    .long 5 @ 0x5
+; CHECK-NEXT:    .long 8 @ 0x8
+; CHECK-NEXT:    .long 11 @ 0xb
+; CHECK-NEXT:    .long 14 @ 0xe
+entry:
+  %ptrs = getelementptr inbounds i16, i16* %base, <4 x i16> <i16 0, i16 3, i16 6, i16 9>
+  %ptrs2 = getelementptr inbounds i16, <4 x i16*> %ptrs, i16 5
+  %t = trunc <4 x i32> %input to <4 x i16>
+  call void @llvm.masked.scatter.v4i16.v4p0i16(<4 x i16> %t, <4 x i16*> %ptrs2, i32 2, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
+  ret void
+}
+
 declare void @llvm.masked.scatter.v4i8.v4p0i8(<4 x i8>, <4 x i8*>, i32, <4 x i1>)
 declare void @llvm.masked.scatter.v4i16.v4p0i16(<4 x i16>, <4 x i16*>, i32, <4 x i1>)
 declare void @llvm.masked.scatter.v4f16.v4p0f16(<4 x half>, <4 x half*>, i32, <4 x i1>)
diff --git a/llvm/test/CodeGen/Thumb2/mve-scatter-ind8-unscaled.ll b/llvm/test/CodeGen/Thumb2/mve-scatter-ind8-unscaled.ll
index d84066cdc5bdd..7fde69ef836c8 100644
--- a/llvm/test/CodeGen/Thumb2/mve-scatter-ind8-unscaled.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-scatter-ind8-unscaled.ll
@@ -534,6 +534,115 @@ entry:
   ret void
 }
 
+define arm_aapcs_vfpcc void @unscaled_v16i8_i8_2gep(i8* %base, <16 x i8>* %offptr, <16 x i8> %input) {
+; CHECK-LABEL: unscaled_v16i8_i8_2gep:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11}
+; CHECK-NEXT:    vldrb.s32 q1, [r1, #12]
+; CHECK-NEXT:    vldrb.s32 q2, [r1, #8]
+; CHECK-NEXT:    vldrb.s32 q3, [r1, #4]
+; CHECK-NEXT:    vldrb.s32 q5, [r1]
+; CHECK-NEXT:    vmov.i32 q4, #0x5
+; CHECK-NEXT:    vadd.i32 q1, q1, r0
+; CHECK-NEXT:    vadd.i32 q2, q2, r0
+; CHECK-NEXT:    vadd.i32 q3, q3, r0
+; CHECK-NEXT:    vadd.i32 q5, q5, r0
+; CHECK-NEXT:    vadd.i32 q1, q1, q4
+; CHECK-NEXT:    vadd.i32 q2, q2, q4
+; CHECK-NEXT:    vadd.i32 q3, q3, q4
+; CHECK-NEXT:    vadd.i32 q4, q5, q4
+; CHECK-NEXT:    vmov.u8 r1, q0[0]
+; CHECK-NEXT:    vmov r0, s16
+; CHECK-NEXT:    strb r1, [r0]
+; CHECK-NEXT:    vmov r0, s17
+; CHECK-NEXT:    vmov.u8 r1, q0[1]
+; CHECK-NEXT:    strb r1, [r0]
+; CHECK-NEXT:    vmov r0, s18
+; CHECK-NEXT:    vmov.u8 r1, q0[2]
+; CHECK-NEXT:    strb r1, [r0]
+; CHECK-NEXT:    vmov r0, s19
+; CHECK-NEXT:    vmov.u8 r1, q0[3]
+; CHECK-NEXT:    strb r1, [r0]
+; CHECK-NEXT:    vmov r0, s12
+; CHECK-NEXT:    vmov.u8 r1, q0[4]
+; CHECK-NEXT:    strb r1, [r0]
+; CHECK-NEXT:    vmov r0, s13
+; CHECK-NEXT:    vmov.u8 r1, q0[5]
+; CHECK-NEXT:    strb r1, [r0]
+; CHECK-NEXT:    vmov r0, s14
+; CHECK-NEXT:    vmov.u8 r1, q0[6]
+; CHECK-NEXT:    strb r1, [r0]
+; CHECK-NEXT:    vmov r0, s15
+; CHECK-NEXT:    vmov.u8 r1, q0[7]
+; CHECK-NEXT:    strb r1, [r0]
+; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    vmov.u8 r1, q0[8]
+; CHECK-NEXT:    strb r1, [r0]
+; CHECK-NEXT:    vmov r0, s9
+; CHECK-NEXT:    vmov.u8 r1, q0[9]
+; CHECK-NEXT:    strb r1, [r0]
+; CHECK-NEXT:    vmov r0, s10
+; CHECK-NEXT:    vmov.u8 r1, q0[10]
+; CHECK-NEXT:    strb r1, [r0]
+; CHECK-NEXT:    vmov r0, s11
+; CHECK-NEXT:    vmov.u8 r1, q0[11]
+; CHECK-NEXT:    strb r1, [r0]
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov.u8 r1, q0[12]
+; CHECK-NEXT:    strb r1, [r0]
+; CHECK-NEXT:    vmov r0, s5
+; CHECK-NEXT:    vmov.u8 r1, q0[13]
+; CHECK-NEXT:    strb r1, [r0]
+; CHECK-NEXT:    vmov r0, s6
+; CHECK-NEXT:    vmov.u8 r1, q0[14]
+; CHECK-NEXT:    strb r1, [r0]
+; CHECK-NEXT:    vmov r0, s7
+; CHECK-NEXT:    vmov.u8 r1, q0[15]
+; CHECK-NEXT:    strb r1, [r0]
+; CHECK-NEXT:    vpop {d8, d9, d10, d11}
+; CHECK-NEXT:    bx lr
+entry:
+  %offs = load <16 x i8>, <16 x i8>* %offptr, align 1
+  %ptrs = getelementptr inbounds i8, i8* %base, <16 x i8> %offs
+  %ptrs2 = getelementptr inbounds i8, <16 x i8*> %ptrs, i8 5
+  call void @llvm.masked.scatter.v16i8.v16p0i8(<16 x i8> %input, <16 x i8*> %ptrs2, i32 1, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @unscaled_v16i8_i8_2gep2(i8* %base, <16 x i8>* %offptr, <16 x i8> %input) {
+; CHECK-LABEL: unscaled_v16i8_i8_2gep2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    adr r1, .LCPI11_0
+; CHECK-NEXT:    vldrw.u32 q1, [r1]
+; CHECK-NEXT:    vstrb.8 q0, [r0, q1]
+; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI11_0:
+; CHECK-NEXT:    .byte 5 @ 0x5
+; CHECK-NEXT:    .byte 8 @ 0x8
+; CHECK-NEXT:    .byte 11 @ 0xb
+; CHECK-NEXT:    .byte 14 @ 0xe
+; CHECK-NEXT:    .byte 17 @ 0x11
+; CHECK-NEXT:    .byte 20 @ 0x14
+; CHECK-NEXT:    .byte 23 @ 0x17
+; CHECK-NEXT:    .byte 26 @ 0x1a
+; CHECK-NEXT:    .byte 29 @ 0x1d
+; CHECK-NEXT:    .byte 32 @ 0x20
+; CHECK-NEXT:    .byte 35 @ 0x23
+; CHECK-NEXT:    .byte 38 @ 0x26
+; CHECK-NEXT:    .byte 41 @ 0x29
+; CHECK-NEXT:    .byte 44 @ 0x2c
+; CHECK-NEXT:    .byte 47 @ 0x2f
+; CHECK-NEXT:    .byte 50 @ 0x32
+entry:
+  %ptrs = getelementptr inbounds i8, i8* %base, <16 x i8> <i8 0, i8 3, i8 6, i8 9, i8 12, i8 15, i8 18, i8 21, i8 24, i8 27, i8 30, i8 33, i8 36, i8 39, i8 42, i8 45>
+  %ptrs2 = getelementptr inbounds i8, <16 x i8*> %ptrs, i8 5
+  call void @llvm.masked.scatter.v16i8.v16p0i8(<16 x i8> %input, <16 x i8*> %ptrs2, i32 1, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  ret void
+}
+
 
 declare void @llvm.masked.scatter.v2i8.v2p0i8(<2 x i8>, <2 x i8*>, i32, <2 x i1>)
 declare void @llvm.masked.scatter.v8i8.v8p0i8(<8 x i8>, <8 x i8*>, i32, <8 x i1>)

From bd4757cc4e40cb1b916c2c39f7bac8abc5adc294 Mon Sep 17 00:00:00 2001
From: Christy Lee <christylee@fb.com>
Date: Tue, 28 Jul 2020 09:41:27 -0700
Subject: [PATCH 0323/1035] [ELF] --reproduce should include lto sample profile

Reviewed By: MaskRay

Differential Revision: https://reviews.llvm.org/D84569
---
 lld/ELF/Driver.cpp           |  3 +++
 lld/test/ELF/reproduce-lto.s | 10 ++++++++++
 2 files changed, 13 insertions(+)
 create mode 100644 lld/test/ELF/reproduce-lto.s

diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp
index 30bff945f1a54..64a41ba77ba2f 100644
--- a/lld/ELF/Driver.cpp
+++ b/lld/ELF/Driver.cpp
@@ -495,6 +495,9 @@ void LinkerDriver::main(ArrayRef<const char *> argsArr) {
       tar = std::move(*errOrWriter);
       tar->append("response.txt", createResponseFile(args));
       tar->append("version.txt", getLLDVersion() + "\n");
+      StringRef ltoSampleProfile = args.getLastArgValue(OPT_lto_sample_profile);
+      if (!ltoSampleProfile.empty())
+        readFile(ltoSampleProfile);
     } else {
       error("--reproduce: " + toString(errOrWriter.takeError()));
     }
diff --git a/lld/test/ELF/reproduce-lto.s b/lld/test/ELF/reproduce-lto.s
new file mode 100644
index 0000000000000..e7c3ece61085d
--- /dev/null
+++ b/lld/test/ELF/reproduce-lto.s
@@ -0,0 +1,10 @@
+# REQUIRES: x86
+
+# RUN: rm -rf %t.dir
+# RUN: mkdir -p %t.dir/build1
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o %t.dir/build1/foo.o
+# RUN: echo > %t.dir/build1/empty_profile.txt
+# RUN: cd %t.dir
+# RUN: ld.lld build1/foo.o -o /dev/null --reproduce repro1.tar --lto-sample-profile=%t.dir/build1/empty_profile.txt
+# RUN: tar tvf repro1.tar | FileCheck %s
+# CHECK: repro1/{{.*}}/empty_profile.txt

From 12f27fc4b505da848a06b37488c5717bf9e3b85d Mon Sep 17 00:00:00 2001
From: Julian Lettner <julian.lettner@apple.com>
Date: Tue, 28 Jul 2020 09:44:02 -0700
Subject: [PATCH 0324/1035] [Darwin] Cleanup code via improved
 GetMacosAlignedVersion()

Checking the OS version via `GetMacosAlignedVersion()` now works in
simulators [1].  Let's use it to simplify `DyldNeedsEnvVariable()`.

[1] 3fb0de820796cc6e322c8378713d375d9870a353

Reviewed By: delcypher

Differential Revision: https://reviews.llvm.org/D81197
---
 .../lib/sanitizer_common/sanitizer_mac.cpp       | 16 +++-------------
 compiler-rt/lib/sanitizer_common/sanitizer_mac.h |  1 +
 2 files changed, 4 insertions(+), 13 deletions(-)

diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp
index 522a909e95285..f96f187131972 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp
@@ -879,20 +879,10 @@ bool ReexecDisabled() {
   return false;
 }
 
-extern "C" SANITIZER_WEAK_ATTRIBUTE double dyldVersionNumber;
-static const double kMinDyldVersionWithAutoInterposition = 360.0;
-
-bool DyldNeedsEnvVariable() {
-  // Although sanitizer support was added to LLVM on OS X 10.7+, GCC users
-  // still may want use them on older systems. On older Darwin platforms, dyld
-  // doesn't export dyldVersionNumber symbol and we simply return true.
-  if (!&dyldVersionNumber) return true;
+static bool DyldNeedsEnvVariable() {
   // If running on OS X 10.11+ or iOS 9.0+, dyld will interpose even if
-  // DYLD_INSERT_LIBRARIES is not set. However, checking OS version via
-  // GetMacosAlignedVersion() doesn't work for the simulator. Let's instead
-  // check `dyldVersionNumber`, which is exported by dyld, against a known
-  // version number from the first OS release where this appeared.
-  return dyldVersionNumber < kMinDyldVersionWithAutoInterposition;
+  // DYLD_INSERT_LIBRARIES is not set.
+  return GetMacosAlignedVersion() < MacosVersion(10, 11);
 }
 
 void MaybeReexec() {
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_mac.h b/compiler-rt/lib/sanitizer_common/sanitizer_mac.h
index 90ecff4815c2e..f61ebe2566e5f 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_mac.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_mac.h
@@ -44,6 +44,7 @@ struct VersionBase {
     return major > other.major ||
            (major == other.major && minor >= other.minor);
   }
+  bool operator<(const VersionType &other) const { return !(*this >= other); }
 };
 
 struct MacosVersion : VersionBase<MacosVersion> {

From f761acfb1a737d8a631a5e55b58cdb7c2215baad Mon Sep 17 00:00:00 2001
From: Vince Bridgers <vince.a.bridgers@gmail.com>
Date: Thu, 16 Jul 2020 17:46:48 -0500
Subject: [PATCH 0325/1035] [ASTImporter] Add Visitor for TypedefNameDecl's

We found a case where Typedef Name Declarations were not being added
correctly when importing builtin types. This exposed the need for a
TypedefNameDecl visitor so these types can be added by RecordDecl and
fields.

This code is covered by the ASTImporterTest cases that use the implicit
struct __NSConstantString_tag definitions.

Thanks to @martong for the debugging assist!

Depends on D83970.

Reviewed By: martong

Differential Revision: https://reviews.llvm.org/D83992
---
 clang/lib/AST/ASTImporterLookupTable.cpp      | 14 +++++++++++++
 clang/test/Analysis/Inputs/ctu-import.c       | 15 ++++++++++++++
 .../ctu-import.c.externalDefMap.ast-dump.txt  |  1 +
 clang/test/Analysis/ctu-implicit.c            | 20 +++++++++++++++++++
 4 files changed, 50 insertions(+)
 create mode 100644 clang/test/Analysis/Inputs/ctu-import.c
 create mode 100644 clang/test/Analysis/Inputs/ctu-import.c.externalDefMap.ast-dump.txt
 create mode 100644 clang/test/Analysis/ctu-implicit.c

diff --git a/clang/lib/AST/ASTImporterLookupTable.cpp b/clang/lib/AST/ASTImporterLookupTable.cpp
index 4d6fff8f34191..e17d6082dcdcc 100644
--- a/clang/lib/AST/ASTImporterLookupTable.cpp
+++ b/clang/lib/AST/ASTImporterLookupTable.cpp
@@ -22,6 +22,20 @@ namespace {
 struct Builder : RecursiveASTVisitor<Builder> {
   ASTImporterLookupTable &LT;
   Builder(ASTImporterLookupTable &LT) : LT(LT) {}
+
+  bool VisitTypedefNameDecl(TypedefNameDecl *D) {
+    QualType Ty = D->getUnderlyingType();
+    Ty = Ty.getCanonicalType();
+    if (const auto *RTy = dyn_cast<RecordType>(Ty)) {
+      LT.add(RTy->getAsRecordDecl());
+      // iterate over the field decls, adding them
+      for (auto *it : RTy->getAsRecordDecl()->fields()) {
+        LT.add(it);
+      }
+    }
+    return true;
+  }
+
   bool VisitNamedDecl(NamedDecl *D) {
     LT.add(D);
     return true;
diff --git a/clang/test/Analysis/Inputs/ctu-import.c b/clang/test/Analysis/Inputs/ctu-import.c
new file mode 100644
index 0000000000000..6c99a36427978
--- /dev/null
+++ b/clang/test/Analysis/Inputs/ctu-import.c
@@ -0,0 +1,15 @@
+
+// Use an internal, implicitly defined type, called by
+// a function imported for CTU. This should not crash.
+int foo(void);
+int foobar(int skip) {
+  __NSConstantString str = {.flags = 1};
+
+  if (str.flags >= 0)
+    str.flags = 0;
+  return 4;
+}
+
+int testStaticImplicit(void) {
+  return foobar(3);
+}
diff --git a/clang/test/Analysis/Inputs/ctu-import.c.externalDefMap.ast-dump.txt b/clang/test/Analysis/Inputs/ctu-import.c.externalDefMap.ast-dump.txt
new file mode 100644
index 0000000000000..83d3b4ca451e8
--- /dev/null
+++ b/clang/test/Analysis/Inputs/ctu-import.c.externalDefMap.ast-dump.txt
@@ -0,0 +1 @@
+c:@F@testStaticImplicit ctu-import.c.ast
diff --git a/clang/test/Analysis/ctu-implicit.c b/clang/test/Analysis/ctu-implicit.c
new file mode 100644
index 0000000000000..925044845e090
--- /dev/null
+++ b/clang/test/Analysis/ctu-implicit.c
@@ -0,0 +1,20 @@
+// RUN: rm -rf %t && mkdir %t
+// RUN: mkdir -p %t/ctudir2
+// RUN: %clang_cc1  \
+// RUN:   -emit-pch -o %t/ctudir2/ctu-import.c.ast %S/Inputs/ctu-import.c
+// RUN: cp %S/Inputs/ctu-import.c.externalDefMap.ast-dump.txt %t/ctudir2/externalDefMap.txt
+// RUN: %clang_cc1 -analyze \
+// RUN:   -analyzer-checker=core,debug.ExprInspection \
+// RUN:   -analyzer-config experimental-enable-naive-ctu-analysis=true \
+// RUN:   -analyzer-config  display-ctu-progress=true \
+// RUN:   -analyzer-config ctu-dir=%t/ctudir2 \
+// RUN:   -verify %s
+
+void clang_analyzer_eval(int);
+
+int testStaticImplicit(void);
+int func(void) {
+  int ret = testStaticImplicit();
+  clang_analyzer_eval(ret == 4); // expected-warning{{TRUE}}
+  return testStaticImplicit();
+}

From 4853a86022feee1b16249cc7a0ea12c842fa5986 Mon Sep 17 00:00:00 2001
From: Jez Ng <jezng@fb.com>
Date: Tue, 28 Jul 2020 09:56:50 -0700
Subject: [PATCH 0326/1035] [lld-macho] Support -filelist

XCode passes files in using this flag

Reviewed By: #lld-macho, compnerd

Differential Revision: https://reviews.llvm.org/D84486
---
 lld/MachO/Driver.cpp                 | 12 +++++++++
 lld/test/MachO/filelist.s            | 40 ++++++++++++++++++++++++++++
 lld/test/MachO/invalid/no-filelist.s |  9 +++++++
 3 files changed, 61 insertions(+)
 create mode 100644 lld/test/MachO/filelist.s
 create mode 100644 lld/test/MachO/invalid/no-filelist.s

diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp
index ee794129e1fc6..d76e0115d10fc 100644
--- a/lld/MachO/Driver.cpp
+++ b/lld/MachO/Driver.cpp
@@ -213,6 +213,15 @@ static void addFile(StringRef path) {
   }
 }
 
+static void addFileList(StringRef path) {
+  Optional<MemoryBufferRef> buffer = readFile(path);
+  if (!buffer)
+    return;
+  MemoryBufferRef mbref = *buffer;
+  for (StringRef path : args::getLines(mbref))
+    addFile(path);
+}
+
 static std::array<StringRef, 6> archNames{"arm",    "arm64", "i386",
                                           "x86_64", "ppc",   "ppc64"};
 static bool isArchString(StringRef s) {
@@ -411,6 +420,9 @@ bool macho::link(llvm::ArrayRef<const char *> argsArr, bool canExitEarly,
     case OPT_INPUT:
       addFile(arg->getValue());
       break;
+    case OPT_filelist:
+      addFileList(arg->getValue());
+      break;
     case OPT_l: {
       StringRef name = arg->getValue();
       if (Optional<std::string> path = findLibrary(name)) {
diff --git a/lld/test/MachO/filelist.s b/lld/test/MachO/filelist.s
new file mode 100644
index 0000000000000..673740ebf06df
--- /dev/null
+++ b/lld/test/MachO/filelist.s
@@ -0,0 +1,40 @@
+# REQUIRES: x86
+
+## This test verifies that the paths in -filelist get processed in command-line
+## order.
+
+# RUN: mkdir -p %t
+# RUN: echo ".globl _foo; .weak_definition _foo; .section __TEXT,first; _foo:" | llvm-mc -filetype=obj -triple=x86_64-apple-darwin -o %t/first.o
+# RUN: echo ".globl _foo; .weak_definition _foo; .section __TEXT,second; _foo:" | llvm-mc -filetype=obj -triple=x86_64-apple-darwin -o %t/second.o
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %s -o %t/test.o
+
+# FIRST: __TEXT,first _foo
+# SECOND: __TEXT,second _foo
+
+# RUN: echo "%t/first.o" > filelist
+# RUN: echo "%t/second.o" >> filelist
+# RUN: lld -flavor darwinnew -Z -filelist filelist %t/test.o -o %t/test
+# RUN: llvm-objdump --syms %t/test | FileCheck %s --check-prefix=FIRST
+
+# RUN: echo "%t/second.o" > filelist
+# RUN: echo "%t/first.o" >> filelist
+# RUN: lld -flavor darwinnew -Z -filelist filelist %t/test.o -o %t/test
+# RUN: llvm-objdump --syms %t/test | FileCheck %s --check-prefix=SECOND
+
+# RUN: echo "%t/first.o" > filelist
+# RUN: lld -flavor darwinnew -Z -filelist filelist %t/second.o %t/test.o -o %t/test
+# RUN: llvm-objdump --syms %t/test | FileCheck %s --check-prefix=FIRST
+# RUN: lld -flavor darwinnew -Z %t/second.o -filelist filelist %t/test.o -o %t/test
+# RUN: llvm-objdump --syms %t/test | FileCheck %s --check-prefix=SECOND
+
+# RUN: echo "%t/first.o" > filelist-1
+# RUN: echo "%t/second.o" > filelist-2
+# RUN: lld -flavor darwinnew -Z -filelist filelist-1 -filelist filelist-2 %t/test.o -o %t/test
+# RUN: llvm-objdump --syms %t/test | FileCheck %s --check-prefix=FIRST
+# RUN: lld -flavor darwinnew -Z -filelist filelist-2 -filelist filelist-1 %t/test.o -o %t/test
+# RUN: llvm-objdump --syms %t/test | FileCheck %s --check-prefix=SECOND
+
+.globl _main
+
+_main:
+  ret
diff --git a/lld/test/MachO/invalid/no-filelist.s b/lld/test/MachO/invalid/no-filelist.s
new file mode 100644
index 0000000000000..fb80185c66aef
--- /dev/null
+++ b/lld/test/MachO/invalid/no-filelist.s
@@ -0,0 +1,9 @@
+# REQUIRES: x86
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %s -o %t.o
+# RUN: not lld -flavor darwinnew -Z -filelist nonexistent %t.o -o %t 2>&1 | FileCheck %s
+# CHECK: cannot open nonexistent: No such file or directory
+
+.globl _main
+
+_main:
+  ret

From d32e32500f92602ccedcf967df2915da6f3803d2 Mon Sep 17 00:00:00 2001
From: Jez Ng <jezng@fb.com>
Date: Tue, 28 Jul 2020 09:56:55 -0700
Subject: [PATCH 0327/1035] [lld-macho] Fix segment filesize calculation

The previous approach of adding up the file sizes of the
component sections ignored the fact that the sections did not have to be
contiguous in the file. As such, it was underestimating the true size.

I discovered this issue because `codesign` checks whether `__LINKEDIT`
extends to the end of the file. Since we were underestimating segment
sizes, this check failed.

Reviewed By: #lld-macho, compnerd

Differential Revision: https://reviews.llvm.org/D84574
---
 lld/MachO/Writer.cpp             |  8 +++-
 lld/test/MachO/section-headers.s | 16 ++++++--
 lld/test/MachO/segments.s        | 69 ++++++++++++++++++--------------
 3 files changed, 59 insertions(+), 34 deletions(-)

diff --git a/lld/MachO/Writer.cpp b/lld/MachO/Writer.cpp
index 03000a7f437e0..c9070e90f97e0 100644
--- a/lld/MachO/Writer.cpp
+++ b/lld/MachO/Writer.cpp
@@ -134,7 +134,11 @@ class LCSegment : public LoadCommand {
     c->nsects = seg->numNonHiddenSections();
 
     for (OutputSection *osec : seg->getSections()) {
-      c->filesize += osec->getFileSize();
+      if (!isZeroFill(osec->flags)) {
+        assert(osec->fileOff >= seg->fileOff);
+        c->filesize = std::max(
+            c->filesize, osec->fileOff + osec->getFileSize() - seg->fileOff);
+      }
 
       if (osec->isHidden())
         continue;
@@ -454,6 +458,8 @@ void Writer::assignAddresses(OutputSegment *seg) {
   seg->fileOff = fileOff;
 
   for (auto *osec : seg->getSections()) {
+    if (!osec->isNeeded())
+      continue;
     addr = alignTo(addr, osec->align);
     fileOff = alignTo(fileOff, osec->align);
     osec->addr = addr;
diff --git a/lld/test/MachO/section-headers.s b/lld/test/MachO/section-headers.s
index 9fafc5a912b0f..fdfdbed632450 100644
--- a/lld/test/MachO/section-headers.s
+++ b/lld/test/MachO/section-headers.s
@@ -1,7 +1,7 @@
 # REQUIRES: x86
 # RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %s -o %t.o
 # RUN: lld -flavor darwinnew -o %t %t.o
-# RUN: llvm-readobj --section-headers %t | FileCheck %s
+# RUN: llvm-readobj --section-headers --macho-segment %t | FileCheck %s
 
 # CHECK:      Name: __text
 # CHECK-NEXT: Segment: __TEXT
@@ -25,11 +25,21 @@
 
 # CHECK:      Name: maxlen_16ch_name
 # CHECK-NEXT: Segment: __TEXT
-# CHECK-NOT:  }
-# CHECK:      Alignment: 3
+# CHECK-NEXT: Address:
+# CHECK-NEXT: Size: [[#%x, LAST_SEC_SIZE:]]
+# CHECK-NEXT: Offset: [[#%u, LAST_SEC_OFF:]]
+# CHECK-NEXT: Alignment: 3
 # CHECK-NOT:  }
 # CHECK:      Type: Regular (0x0)
 
+# CHECK-LABEL: Segment {
+# CHECK:       Name: __TEXT
+# CHECK-NEXT:  Size:
+# CHECK-NEXT:  vmaddr:
+# CHECK-NEXT:  vmsize:
+# CHECK-NEXT:  fileoff: 0
+# CHECK-NEXT:  filesize: [[#%u, LAST_SEC_SIZE + LAST_SEC_OFF]]
+
 .text
 .align 1
 .global _main
diff --git a/lld/test/MachO/segments.s b/lld/test/MachO/segments.s
index acb0f1e90101a..e0f127fabe55f 100644
--- a/lld/test/MachO/segments.s
+++ b/lld/test/MachO/segments.s
@@ -1,49 +1,58 @@
-# REQUIRES: x86
+# REQUIRES: x86, shell
 # RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %s -o %t.o
 # RUN: lld -flavor darwinnew -o %t %t.o
-# RUN: llvm-readobj --macho-segment %t | FileCheck %s
+# RUN: (llvm-readobj --macho-segment %t; echo "Total file size"; wc -c %t) | FileCheck %s
 
 ## These two segments must always be present at the start of an executable.
 # CHECK-NOT:  Segment {
 # CHECK:      Segment {
-# CHECK:        Cmd: LC_SEGMENT_64
-# CHECK:        Name: __PAGEZERO
-# CHECK:        Size: 72
-# CHECK:        vmaddr: 0x0
-# CHECK:        vmsize: 0x100000000
-# CHECK:        fileoff: 0
-# CHECK:        filesize: 0
+# CHECK-NEXT:   Cmd: LC_SEGMENT_64
+# CHECK-NEXT:   Name: __PAGEZERO
+# CHECK-NEXT:   Size: 72
+# CHECK-NEXT:   vmaddr: 0x0
+# CHECK-NEXT:   vmsize: 0x100000000
+# CHECK-NEXT:   fileoff: 0
+# CHECK-NEXT:   filesize: 0
 ## The kernel won't execute a binary with the wrong protections for __PAGEZERO.
-# CHECK:        maxprot: ---
-# CHECK:        initprot: ---
-# CHECK:        nsects: 0
-# CHECK:        flags: 0x0
-# CHECK:      }
-# CHECK:      Segment {
-# CHECK:        Cmd: LC_SEGMENT_64
-# CHECK:        Name: __TEXT
-# CHECK:        Size: 152
-# CHECK:        vmaddr: 0x100000000
-# CHECK:        vmsize:
+# CHECK-NEXT:   maxprot: ---
+# CHECK-NEXT:   initprot: ---
+# CHECK-NEXT:   nsects: 0
+# CHECK-NEXT:   flags: 0x0
+# CHECK-NEXT: }
+# CHECK-NEXT: Segment {
+# CHECK-NEXT:   Cmd: LC_SEGMENT_64
+# CHECK-NEXT:   Name: __TEXT
+# CHECK-NEXT:   Size: 152
+# CHECK-NEXT:   vmaddr: 0x100000000
+# CHECK-NEXT:   vmsize:
 ## dyld3 assumes that the __TEXT segment starts from the file header
-# CHECK:        fileoff: 0
-# CHECK:        filesize:
-# CHECK:        maxprot: rwx
-# CHECK:        initprot: r-x
-# CHECK:        nsects: 1
-# CHECK:        flags: 0x0
-# CHECK:      }
+# CHECK-NEXT:   fileoff: 0
+# CHECK-NEXT:   filesize:
+# CHECK-NEXT:   maxprot: rwx
+# CHECK-NEXT:   initprot: r-x
+# CHECK-NEXT:   nsects: 1
+# CHECK-NEXT:   flags: 0x0
+# CHECK-NEXT: }
 
 ## Check that we handle max-length names correctly.
 # CHECK:      Cmd: LC_SEGMENT_64
 # CHECK-NEXT: Name: maxlen_16ch_name
 
-## This segment must always be present at the end of an executable.
+## This segment must always be present at the end of an executable, and cover
+## its last byte.
 # CHECK:      Name: __LINKEDIT
-# CHECK:      maxprot: rwx
-# CHECK:      initprot: r--
+# CHECK-NEXT: Size:
+# CHECK-NEXT: vmaddr:
+# CHECK-NEXT: vmsize:
+# CHECK-NEXT: fileoff: [[#%u, LINKEDIT_OFF:]]
+# CHECK-NEXT: filesize: [[#%u, LINKEDIT_SIZE:]]
+# CHECK-NEXT: maxprot: rwx
+# CHECK-NEXT: initprot: r--
 # CHECK-NOT:  Cmd: LC_SEGMENT_64
 
+# CHECK-LABEL: Total file size
+# CHECK-NEXT:  [[#%u, LINKEDIT_OFF + LINKEDIT_SIZE]]
+
 .text
 .global _main
 _main:

From 4b56238b136ba88b9e7794deca86bab7ccbe3ae3 Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne@apple.com>
Date: Tue, 28 Jul 2020 13:09:32 -0400
Subject: [PATCH 0328/1035] NFC: Add whitespace changing revisions to
 .git-blame-ignore-revs

---
 .git-blame-ignore-revs | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs
index 71d0488b32790..7c759a1adc950 100644
--- a/.git-blame-ignore-revs
+++ b/.git-blame-ignore-revs
@@ -22,3 +22,12 @@ b9c1b51e45b845debb76d8658edabca70ca56079
 
 # r302496: That is the revert of r302421
 ff63090b0e1072bd398b8efef8ae2291613a6ec9
+
+# Fix more line endings changed in r320089. NFC.
+d8f0e6caa91e230a486c948ab643174e40bdf215
+
+# Correct line endings that got mixed up in r320089; NFC.
+29dc5ded45447915d96ef7ca3f02acf2232282e0
+
+# Remove line-endings added by r320089. NFC.
+100a0eedc00b2bf48bcdc6c209c000745a4a0e48

From 64cdd5b3da1abce63838bfc5bd32398c834a32e3 Mon Sep 17 00:00:00 2001
From: Nicolas Vasilache <ntv@google.com>
Date: Tue, 28 Jul 2020 09:58:44 -0400
Subject: [PATCH 0329/1035] [mlir][Vector] Drop declarative transforms

For the purpose of vector transforms, the Tablegen-based infra is subsumed by simple C++ pattern application. Deprecate declarative transforms whose complexity does not pay for itself.

Differential Revision: https://reviews.llvm.org/D84753
---
 .../mlir/Dialect/Vector/CMakeLists.txt        |  4 ---
 .../Dialect/Vector/VectorTransformPatterns.td | 26 --------------
 mlir/lib/Dialect/Vector/CMakeLists.txt        |  1 -
 mlir/test/lib/CMakeLists.txt                  |  1 -
 .../lib/DeclarativeTransforms/CMakeLists.txt  |  3 --
 .../TestVectorTransformPatterns.td            | 34 -------------------
 .../lib/DeclarativeTransforms/lit.local.cfg   |  1 -
 mlir/test/lib/Transforms/CMakeLists.txt       |  3 --
 .../lib/Transforms/TestVectorTransforms.cpp   | 12 +++----
 9 files changed, 6 insertions(+), 79 deletions(-)
 delete mode 100644 mlir/include/mlir/Dialect/Vector/VectorTransformPatterns.td
 delete mode 100644 mlir/test/lib/DeclarativeTransforms/CMakeLists.txt
 delete mode 100644 mlir/test/lib/DeclarativeTransforms/TestVectorTransformPatterns.td
 delete mode 100644 mlir/test/lib/DeclarativeTransforms/lit.local.cfg

diff --git a/mlir/include/mlir/Dialect/Vector/CMakeLists.txt b/mlir/include/mlir/Dialect/Vector/CMakeLists.txt
index a27eef693a288..23ad74e0cb72f 100644
--- a/mlir/include/mlir/Dialect/Vector/CMakeLists.txt
+++ b/mlir/include/mlir/Dialect/Vector/CMakeLists.txt
@@ -1,6 +1,2 @@
 add_mlir_dialect(VectorOps vector)
 add_mlir_doc(VectorOps -gen-op-doc VectorOps Dialects/)
-
-set(LLVM_TARGET_DEFINITIONS VectorTransformPatterns.td)
-mlir_tablegen(VectorTransformPatterns.h.inc -gen-rewriters)
-add_public_tablegen_target(MLIRVectorTransformPatternsIncGen)
diff --git a/mlir/include/mlir/Dialect/Vector/VectorTransformPatterns.td b/mlir/include/mlir/Dialect/Vector/VectorTransformPatterns.td
deleted file mode 100644
index ef8118ec64704..0000000000000
--- a/mlir/include/mlir/Dialect/Vector/VectorTransformPatterns.td
+++ /dev/null
@@ -1,26 +0,0 @@
-//===- VectorTransformPatterns.td - Vector-Vector patterns -*- tablegen -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This is the pattern definition file for declarative Vector transformations.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef VECTOR_TRANSFORM_PATTERNS
-#define VECTOR_TRANSFORM_PATTERNS
-
-include "mlir/IR/OpBase.td"
-
-class HasShape<list<int> shape> :
-  CPred<"$0.getType().cast<ShapedType>().hasStaticShape({" #
-    StrJoinInt<shape>.result # "})">;
-
-class UnrollVectorOp<list<int> factors> : NativeCodeCall<
-  "unrollSingleResultVectorOp($_builder, $0.getDefiningOp(), " #
-    "{" # StrJoinInt<factors>.result # "})">;
-
-#endif // VECTOR_TRANSFORM_PATTERNS
diff --git a/mlir/lib/Dialect/Vector/CMakeLists.txt b/mlir/lib/Dialect/Vector/CMakeLists.txt
index d6ba987e6622f..7b34f1933c42c 100644
--- a/mlir/lib/Dialect/Vector/CMakeLists.txt
+++ b/mlir/lib/Dialect/Vector/CMakeLists.txt
@@ -9,7 +9,6 @@ add_mlir_dialect_library(MLIRVector
 
   DEPENDS
   MLIRVectorOpsIncGen
-  MLIRVectorTransformPatternsIncGen
 
   LINK_LIBS PUBLIC
   MLIREDSC
diff --git a/mlir/test/lib/CMakeLists.txt b/mlir/test/lib/CMakeLists.txt
index 641a6218d1ccd..0df357c8c355e 100644
--- a/mlir/test/lib/CMakeLists.txt
+++ b/mlir/test/lib/CMakeLists.txt
@@ -1,4 +1,3 @@
-add_subdirectory(DeclarativeTransforms)
 add_subdirectory(Dialect)
 add_subdirectory(IR)
 add_subdirectory(Pass)
diff --git a/mlir/test/lib/DeclarativeTransforms/CMakeLists.txt b/mlir/test/lib/DeclarativeTransforms/CMakeLists.txt
deleted file mode 100644
index 67d194ff868a5..0000000000000
--- a/mlir/test/lib/DeclarativeTransforms/CMakeLists.txt
+++ /dev/null
@@ -1,3 +0,0 @@
-set(LLVM_TARGET_DEFINITIONS TestVectorTransformPatterns.td)
-mlir_tablegen(TestVectorTransformPatterns.h.inc -gen-rewriters)
-add_public_tablegen_target(MLIRTestVectorTransformPatternsIncGen)
diff --git a/mlir/test/lib/DeclarativeTransforms/TestVectorTransformPatterns.td b/mlir/test/lib/DeclarativeTransforms/TestVectorTransformPatterns.td
deleted file mode 100644
index 2c6ca1a05733e..0000000000000
--- a/mlir/test/lib/DeclarativeTransforms/TestVectorTransformPatterns.td
+++ /dev/null
@@ -1,34 +0,0 @@
-//===- TestVectorTransformPatterns.td - Test patterns ---*- tablegen ----*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This is the pattern definition file for declarative Vector transformations
-// tests.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef TEST_VECTOR_TRANSFORMS_PATTERNS
-#define TEST_VECTOR_TRANSFORMS_PATTERNS
-
-include "mlir/Dialect/StandardOps/IR/Ops.td"
-include "mlir/Dialect/Vector/VectorOps.td"
-include "mlir/Dialect/Vector/VectorTransformPatterns.td"
-
-def : Pat<(AddFOp:$op_results $a, $b),
-          (UnrollVectorOp<[2, 2]> $op_results, $a, $b),
-          [(Constraint<HasShape<[4, 2]>> $a)]>;
-
-def : Pat<(AddFOp:$op_results $a, $b),
-          (UnrollVectorOp<[2, 2]> $op_results, $a, $b),
-          [(Constraint<HasShape<[4, 4]>> $a)]>;
-
-// TODO: Add Constraints on lhs/rhs shapes.
-def : Pat<(Vector_ContractionOp:$op_results $a, $b, $c, $masks, $attr0, $attr1),
-          (UnrollVectorOp<[2, 2, 2]> $op_results, $a, $b, $c),
-          [(Constraint<HasShape<[4, 4]>> $c)]>;
-
-#endif // TEST_VECTOR_TRANSFORMS_PATTERNS
diff --git a/mlir/test/lib/DeclarativeTransforms/lit.local.cfg b/mlir/test/lib/DeclarativeTransforms/lit.local.cfg
deleted file mode 100644
index edb5b44b2e2fe..0000000000000
--- a/mlir/test/lib/DeclarativeTransforms/lit.local.cfg
+++ /dev/null
@@ -1 +0,0 @@
-config.suffixes.remove('.td')
\ No newline at end of file
diff --git a/mlir/test/lib/Transforms/CMakeLists.txt b/mlir/test/lib/Transforms/CMakeLists.txt
index cdfd4e8a815b0..c3318316c5085 100644
--- a/mlir/test/lib/Transforms/CMakeLists.txt
+++ b/mlir/test/lib/Transforms/CMakeLists.txt
@@ -32,7 +32,6 @@ add_mlir_library(MLIRTestTransforms
 
   DEPENDS
   MLIRStandardOpsIncGen
-  MLIRTestVectorTransformPatternsIncGen
 
   LINK_LIBS PUBLIC
   MLIRAffineOps
@@ -59,5 +58,3 @@ add_mlir_library(MLIRTestTransforms
 
 include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../Dialect/Test)
 include_directories(${CMAKE_CURRENT_BINARY_DIR}/../Dialect/Test)
-include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../DeclarativeTransforms)
-include_directories(${CMAKE_CURRENT_BINARY_DIR}/../DeclarativeTransforms)
diff --git a/mlir/test/lib/Transforms/TestVectorTransforms.cpp b/mlir/test/lib/Transforms/TestVectorTransforms.cpp
index 2dffd88ed7093..2058706dcbdd3 100644
--- a/mlir/test/lib/Transforms/TestVectorTransforms.cpp
+++ b/mlir/test/lib/Transforms/TestVectorTransforms.cpp
@@ -18,16 +18,16 @@ using namespace mlir;
 using namespace mlir::vector;
 namespace {
 
-#include "TestVectorTransformPatterns.h.inc"
-
 struct TestVectorToVectorConversion
     : public PassWrapper<TestVectorToVectorConversion, FunctionPass> {
   void runOnFunction() override {
     OwningRewritePatternList patterns;
-    auto *context = &getContext();
-    populateWithGenerated(context, &patterns);
-    populateVectorToVectorCanonicalizationPatterns(patterns, context);
-    populateVectorToVectorTransformationPatterns(patterns, context);
+    auto *ctx = &getContext();
+    patterns.insert<UnrollVectorPattern<AddFOp>>(ArrayRef<int64_t>{2, 2}, ctx);
+    patterns.insert<UnrollVectorPattern<vector::ContractionOp>>(
+        ArrayRef<int64_t>{2, 2, 2}, ctx);
+    populateVectorToVectorCanonicalizationPatterns(patterns, ctx);
+    populateVectorToVectorTransformationPatterns(patterns, ctx);
     applyPatternsAndFoldGreedily(getFunction(), patterns);
   }
 };

From 4887495a3e0b97de4e38232e9f65b9454434a818 Mon Sep 17 00:00:00 2001
From: Juneyoung Lee <aqjune@gmail.com>
Date: Wed, 29 Jul 2020 02:16:34 +0900
Subject: [PATCH 0330/1035] [JumpThreading] Add tests that have a cast of
 freeze and vice versa

---
 llvm/test/Transforms/JumpThreading/freeze.ll | 92 ++++++++++++++++++++
 1 file changed, 92 insertions(+)

diff --git a/llvm/test/Transforms/JumpThreading/freeze.ll b/llvm/test/Transforms/JumpThreading/freeze.ll
index 3c6aa98c32b84..e08aabf5be170 100644
--- a/llvm/test/Transforms/JumpThreading/freeze.ll
+++ b/llvm/test/Transforms/JumpThreading/freeze.ll
@@ -49,6 +49,98 @@ F2:
   ret i32 %B
 }
 
+define i32 @test1_cast(i1 %cond) {
+; CHECK-LABEL: @test1_cast(
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[T1:%.*]], label [[F1:%.*]]
+; CHECK:       T1:
+; CHECK-NEXT:    [[V1:%.*]] = call i32 @f1()
+; CHECK-NEXT:    br label [[MERGE:%.*]]
+; CHECK:       F1:
+; CHECK-NEXT:    [[V2:%.*]] = call i32 @f2()
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       Merge:
+; CHECK-NEXT:    [[A0:%.*]] = phi i32 [ 1, [[T1]] ], [ 0, [[F1]] ]
+; CHECK-NEXT:    [[B:%.*]] = phi i32 [ [[V1]], [[T1]] ], [ [[V2]], [[F1]] ]
+; CHECK-NEXT:    [[A:%.*]] = trunc i32 [[A0]] to i1
+; CHECK-NEXT:    [[A_FR:%.*]] = freeze i1 [[A]]
+; CHECK-NEXT:    br i1 [[A_FR]], label [[T2:%.*]], label [[F2:%.*]]
+; CHECK:       T2:
+; CHECK-NEXT:    call void @f3()
+; CHECK-NEXT:    ret i32 [[B]]
+; CHECK:       F2:
+; CHECK-NEXT:    ret i32 [[B]]
+;
+  br i1 %cond, label %T1, label %F1
+
+T1:
+  %v1 = call i32 @f1()
+  br label %Merge
+
+F1:
+  %v2 = call i32 @f2()
+  br label %Merge
+
+Merge:
+  %A0 = phi i32 [1, %T1], [0, %F1]
+  %B = phi i32 [%v1, %T1], [%v2, %F1]
+  %A = trunc i32 %A0 to i1
+  %A.fr = freeze i1 %A
+  br i1 %A.fr, label %T2, label %F2
+
+T2:
+  call void @f3()
+  ret i32 %B
+
+F2:
+  ret i32 %B
+}
+
+define i32 @test1_cast2(i1 %cond) {
+; CHECK-LABEL: @test1_cast2(
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[T1:%.*]], label [[F1:%.*]]
+; CHECK:       T1:
+; CHECK-NEXT:    [[V1:%.*]] = call i32 @f1()
+; CHECK-NEXT:    br label [[MERGE:%.*]]
+; CHECK:       F1:
+; CHECK-NEXT:    [[V2:%.*]] = call i32 @f2()
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       Merge:
+; CHECK-NEXT:    [[A0:%.*]] = phi i32 [ 1, [[T1]] ], [ 0, [[F1]] ]
+; CHECK-NEXT:    [[B:%.*]] = phi i32 [ [[V1]], [[T1]] ], [ [[V2]], [[F1]] ]
+; CHECK-NEXT:    [[A0_FR:%.*]] = freeze i32 [[A0]]
+; CHECK-NEXT:    [[A_FR:%.*]] = trunc i32 [[A0_FR]] to i1
+; CHECK-NEXT:    br i1 [[A_FR]], label [[T2:%.*]], label [[F2:%.*]]
+; CHECK:       T2:
+; CHECK-NEXT:    call void @f3()
+; CHECK-NEXT:    ret i32 [[B]]
+; CHECK:       F2:
+; CHECK-NEXT:    ret i32 [[B]]
+;
+  br i1 %cond, label %T1, label %F1
+
+T1:
+  %v1 = call i32 @f1()
+  br label %Merge
+
+F1:
+  %v2 = call i32 @f2()
+  br label %Merge
+
+Merge:
+  %A0 = phi i32 [1, %T1], [0, %F1]
+  %B = phi i32 [%v1, %T1], [%v2, %F1]
+  %A0.fr = freeze i32 %A0
+  %A.fr = trunc i32 %A0.fr to i1
+  br i1 %A.fr, label %T2, label %F2
+
+T2:
+  call void @f3()
+  ret i32 %B
+
+F2:
+  ret i32 %B
+}
+
 define i32 @test1_undef(i1 %cond) {
 ; CHECK-LABEL: @test1_undef(
 ; CHECK-NEXT:    br i1 [[COND:%.*]], label [[T1:%.*]], label [[F1:%.*]]

From 04a21318b55756d50836f6e40f2d209f18cce417 Mon Sep 17 00:00:00 2001
From: Yitzhak Mandelbaum <yitzhakm@google.com>
Date: Tue, 28 Jul 2020 17:26:12 +0000
Subject: [PATCH 0331/1035] [libTooling] Add a `between` range-selector
 combinator.

Adds the `between` combinator and registers it with the parser. As a driveby, updates some deprecated names to their current versions.

Reviewed By: gribozavr2

Differential Revision: https://reviews.llvm.org/D84315
---
 .../clang/Tooling/Transformer/RangeSelector.h |  5 +++
 clang/lib/Tooling/Transformer/Parsing.cpp     |  4 +--
 clang/unittests/Tooling/RangeSelectorTest.cpp | 33 ++++++++++++++++---
 3 files changed, 36 insertions(+), 6 deletions(-)

diff --git a/clang/include/clang/Tooling/Transformer/RangeSelector.h b/clang/include/clang/Tooling/Transformer/RangeSelector.h
index 2807037bc208f..e070c0e7e2e6b 100644
--- a/clang/include/clang/Tooling/Transformer/RangeSelector.h
+++ b/clang/include/clang/Tooling/Transformer/RangeSelector.h
@@ -56,6 +56,11 @@ RangeSelector before(RangeSelector Selector);
 /// * the TokenRange [B,E'] where the token at E' spans the range [E',E).
 RangeSelector after(RangeSelector Selector);
 
+/// Selects the range between `R1` and `R2.
+inline RangeSelector between(RangeSelector R1, RangeSelector R2) {
+  return enclose(after(std::move(R1)), before(std::move(R2)));
+}
+
 /// Selects a node, including trailing semicolon (for non-expression
 /// statements). \p ID is the node's binding in the match result.
 RangeSelector node(std::string ID);
diff --git a/clang/lib/Tooling/Transformer/Parsing.cpp b/clang/lib/Tooling/Transformer/Parsing.cpp
index 1579115b93138..fb5fd4a800bbb 100644
--- a/clang/lib/Tooling/Transformer/Parsing.cpp
+++ b/clang/lib/Tooling/Transformer/Parsing.cpp
@@ -109,14 +109,14 @@ getUnaryRangeSelectors() {
 static const llvm::StringMap<RangeSelectorOp<std::string, std::string>> &
 getBinaryStringSelectors() {
   static const llvm::StringMap<RangeSelectorOp<std::string, std::string>> M = {
-      {"encloseNodes", range}};
+      {"encloseNodes", encloseNodes}};
   return M;
 }
 
 static const llvm::StringMap<RangeSelectorOp<RangeSelector, RangeSelector>> &
 getBinaryRangeSelectors() {
   static const llvm::StringMap<RangeSelectorOp<RangeSelector, RangeSelector>>
-      M = {{"enclose", range}};
+      M = {{"enclose", enclose}, {"between", between}};
   return M;
 }
 
diff --git a/clang/unittests/Tooling/RangeSelectorTest.cpp b/clang/unittests/Tooling/RangeSelectorTest.cpp
index e2d7723eab116..64ddee7894eb9 100644
--- a/clang/unittests/Tooling/RangeSelectorTest.cpp
+++ b/clang/unittests/Tooling/RangeSelectorTest.cpp
@@ -193,8 +193,33 @@ TEST(RangeSelectorTest, AfterOp) {
                        HasValue(EqualsCharSourceRange(ExpectedAfter)));
 }
 
+TEST(RangeSelectorTest, BetweenOp) {
+  StringRef Code = R"cc(
+    int f(int x, int y, int z) { return 3; }
+    int g() { return f(3, /* comment */ 7 /* comment */, 9); }
+  )cc";
+  auto Matcher = callExpr(hasArgument(0, expr().bind("a0")),
+                          hasArgument(1, expr().bind("a1")));
+  RangeSelector R = between(node("a0"), node("a1"));
+  TestMatch Match = matchCode(Code, Matcher);
+  EXPECT_THAT_EXPECTED(select(R, Match), HasValue(", /* comment */ "));
+}
+
+TEST(RangeSelectorTest, BetweenOpParsed) {
+  StringRef Code = R"cc(
+    int f(int x, int y, int z) { return 3; }
+    int g() { return f(3, /* comment */ 7 /* comment */, 9); }
+  )cc";
+  auto Matcher = callExpr(hasArgument(0, expr().bind("a0")),
+                          hasArgument(1, expr().bind("a1")));
+  auto R = parseRangeSelector(R"rs(between(node("a0"), node("a1")))rs");
+  ASSERT_THAT_EXPECTED(R, llvm::Succeeded());
+  TestMatch Match = matchCode(Code, Matcher);
+  EXPECT_THAT_EXPECTED(select(*R, Match), HasValue(", /* comment */ "));
+}
+
 // Node-id specific version.
-TEST(RangeSelectorTest, RangeOpNodes) {
+TEST(RangeSelectorTest, EncloseOpNodes) {
   StringRef Code = R"cc(
     int f(int x, int y, int z) { return 3; }
     int g() { return f(/* comment */ 3, 7 /* comment */, 9); }
@@ -206,7 +231,7 @@ TEST(RangeSelectorTest, RangeOpNodes) {
   EXPECT_THAT_EXPECTED(select(R, Match), HasValue("3, 7"));
 }
 
-TEST(RangeSelectorTest, RangeOpGeneral) {
+TEST(RangeSelectorTest, EncloseOpGeneral) {
   StringRef Code = R"cc(
     int f(int x, int y, int z) { return 3; }
     int g() { return f(/* comment */ 3, 7 /* comment */, 9); }
@@ -218,7 +243,7 @@ TEST(RangeSelectorTest, RangeOpGeneral) {
   EXPECT_THAT_EXPECTED(select(R, Match), HasValue("3, 7"));
 }
 
-TEST(RangeSelectorTest, RangeOpNodesParsed) {
+TEST(RangeSelectorTest, EncloseOpNodesParsed) {
   StringRef Code = R"cc(
     int f(int x, int y, int z) { return 3; }
     int g() { return f(/* comment */ 3, 7 /* comment */, 9); }
@@ -231,7 +256,7 @@ TEST(RangeSelectorTest, RangeOpNodesParsed) {
   EXPECT_THAT_EXPECTED(select(*R, Match), HasValue("3, 7"));
 }
 
-TEST(RangeSelectorTest, RangeOpGeneralParsed) {
+TEST(RangeSelectorTest, EncloseOpGeneralParsed) {
   StringRef Code = R"cc(
     int f(int x, int y, int z) { return 3; }
     int g() { return f(/* comment */ 3, 7 /* comment */, 9); }

From 6c3dc6e1d57d15568ebcb3b725fa65a7ba4462de Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@intel.com>
Date: Tue, 28 Jul 2020 09:53:42 -0700
Subject: [PATCH 0332/1035] [X86] Merge disp8 and cdisp8 handling into a single
 helper function to reduce some code.

We currently handle EVEX and non-EVEX separately in two places. By sinking the EVEX
check into the existing helper for CDisp8 we can simplify these two places.

Differential Revision: https://reviews.llvm.org/D84730
---
 .../X86/MCTargetDesc/X86MCCodeEmitter.cpp     | 63 +++++++------------
 1 file changed, 22 insertions(+), 41 deletions(-)

diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp
index 5f1b5b5e2b96e..900b4d78b6f27 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp
@@ -113,33 +113,28 @@ static void emitConstant(uint64_t Val, unsigned Size, raw_ostream &OS) {
   }
 }
 
-/// \returns true if this signed displacement fits in a 8-bit sign-extended
-/// field.
-static bool isDisp8(int Value) { return Value == (int8_t)Value; }
-
-/// \returns true if this signed displacement fits in a 8-bit compressed
-/// dispacement field.
-static bool isCDisp8(uint64_t TSFlags, int Value, int &CValue) {
-  assert(((TSFlags & X86II::EncodingMask) == X86II::EVEX) &&
-         "Compressed 8-bit displacement is only valid for EVEX inst.");
+/// Determine if this immediate can fit in a disp8 or a compressed disp8 for
+/// EVEX instructions. \p will be set to the value to pass to the ImmOffset
+/// parameter of emitImmediate.
+static bool isDispOrCDisp8(uint64_t TSFlags, int Value, int &ImmOffset) {
+  bool HasEVEX = (TSFlags & X86II::EncodingMask) == X86II::EVEX;
 
-  unsigned CD8_Scale =
+  int CD8_Scale =
       (TSFlags & X86II::CD8_Scale_Mask) >> X86II::CD8_Scale_Shift;
-  if (CD8_Scale == 0) {
-    CValue = Value;
-    return isDisp8(Value);
-  }
+  if (!HasEVEX || CD8_Scale == 0)
+    return isInt<8>(Value);
 
-  unsigned Mask = CD8_Scale - 1;
-  assert((CD8_Scale & Mask) == 0 && "Invalid memory object size.");
-  if (Value & Mask) // Unaligned offset
+  assert(isPowerOf2_32(CD8_Scale) && "Unexpected CD8 scale!");
+  if (Value & (CD8_Scale - 1)) // Unaligned offset
     return false;
-  Value /= (int)CD8_Scale;
-  bool Ret = (Value == (int8_t)Value);
 
-  if (Ret)
-    CValue = Value;
-  return Ret;
+  int CDisp8 = Value / CD8_Scale;
+  if (!isInt<8>(CDisp8))
+    return false;
+
+  // ImmOffset will be added to Value in emitImmediate leaving just CDisp8.
+  ImmOffset = CDisp8 - Value;
+  return true;
 }
 
 /// \returns the appropriate fixup kind to use for an immediate in an
@@ -393,7 +388,6 @@ void X86MCCodeEmitter::emitMemModRMByte(const MCInst &MI, unsigned Op,
   const MCOperand &Scale = MI.getOperand(Op + X86::AddrScaleAmt);
   const MCOperand &IndexReg = MI.getOperand(Op + X86::AddrIndexReg);
   unsigned BaseReg = Base.getReg();
-  bool HasEVEX = (TSFlags & X86II::EncodingMask) == X86II::EVEX;
 
   // Handle %rip relative addressing.
   if (BaseReg == X86::RIP ||
@@ -487,7 +481,7 @@ void X86MCCodeEmitter::emitMemModRMByte(const MCInst &MI, unsigned Op,
           RMfield = (IndexReg16 & 1) | ((7 - RMfield) << 1);
       }
 
-      if (Disp.isImm() && isDisp8(Disp.getImm())) {
+      if (Disp.isImm() && isInt<8>(Disp.getImm())) {
         if (Disp.getImm() == 0 && RMfield != 6) {
           // There is no displacement; just the register.
           emitByte(modRMByte(0, RegOpcodeField, RMfield), OS);
@@ -557,18 +551,11 @@ void X86MCCodeEmitter::emitMemModRMByte(const MCInst &MI, unsigned Op,
 
     // Otherwise, if the displacement fits in a byte, encode as [REG+disp8].
     if (Disp.isImm()) {
-      if (!HasEVEX && isDisp8(Disp.getImm())) {
-        emitByte(modRMByte(1, RegOpcodeField, BaseRegNo), OS);
-        emitImmediate(Disp, MI.getLoc(), 1, FK_Data_1, StartByte, OS, Fixups);
-        return;
-      }
-      // Try EVEX compressed 8-bit displacement first; if failed, fall back to
-      // 32-bit displacement.
-      int CDisp8 = 0;
-      if (HasEVEX && isCDisp8(TSFlags, Disp.getImm(), CDisp8)) {
+      int ImmOffset = 0;
+      if (isDispOrCDisp8(TSFlags, Disp.getImm(), ImmOffset)) {
         emitByte(modRMByte(1, RegOpcodeField, BaseRegNo), OS);
         emitImmediate(Disp, MI.getLoc(), 1, FK_Data_1, StartByte, OS, Fixups,
-                      CDisp8 - Disp.getImm());
+                      ImmOffset);
         return;
       }
     }
@@ -589,7 +576,6 @@ void X86MCCodeEmitter::emitMemModRMByte(const MCInst &MI, unsigned Op,
 
   bool ForceDisp32 = false;
   bool ForceDisp8 = false;
-  int CDisp8 = 0;
   int ImmOffset = 0;
   if (BaseReg == 0) {
     // If there is no base register, we emit the special case SIB byte with
@@ -606,15 +592,10 @@ void X86MCCodeEmitter::emitMemModRMByte(const MCInst &MI, unsigned Op,
              BaseRegNo != N86::EBP) {
     // Emit no displacement ModR/M byte
     emitByte(modRMByte(0, RegOpcodeField, 4), OS);
-  } else if (!HasEVEX && isDisp8(Disp.getImm())) {
-    // Emit the disp8 encoding.
-    emitByte(modRMByte(1, RegOpcodeField, 4), OS);
-    ForceDisp8 = true; // Make sure to force 8 bit disp if Base=EBP
-  } else if (HasEVEX && isCDisp8(TSFlags, Disp.getImm(), CDisp8)) {
+  } else if (isDispOrCDisp8(TSFlags, Disp.getImm(), ImmOffset)) {
     // Emit the disp8 encoding.
     emitByte(modRMByte(1, RegOpcodeField, 4), OS);
     ForceDisp8 = true; // Make sure to force 8 bit disp if Base=EBP
-    ImmOffset = CDisp8 - Disp.getImm();
   } else {
     // Emit the normal disp32 encoding.
     emitByte(modRMByte(2, RegOpcodeField, 4), OS);

From 91b8c1fd0f31a548b75acb3de4dbee09847e19e5 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@intel.com>
Date: Tue, 28 Jul 2020 10:46:04 -0700
Subject: [PATCH 0333/1035] [X86] Simplify some code in emitMemModRMByte. NFCI

---
 .../X86/MCTargetDesc/X86MCCodeEmitter.cpp     | 25 ++++++-------------
 1 file changed, 8 insertions(+), 17 deletions(-)

diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp
index 900b4d78b6f27..25f1089912639 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp
@@ -605,23 +605,14 @@ void X86MCCodeEmitter::emitMemModRMByte(const MCInst &MI, unsigned Op,
   static const unsigned SSTable[] = {~0U, 0, 1, ~0U, 2, ~0U, ~0U, ~0U, 3};
   unsigned SS = SSTable[Scale.getImm()];
 
-  if (BaseReg == 0) {
-    // Handle the SIB byte for the case where there is no base, see Intel
-    // Manual 2A, table 2-7. The displacement has already been output.
-    unsigned IndexRegNo;
-    if (IndexReg.getReg())
-      IndexRegNo = getX86RegNum(IndexReg);
-    else // Examples: [ESP+1*<noreg>+4] or [scaled idx]+disp32 (MOD=0,BASE=5)
-      IndexRegNo = 4;
-    emitSIBByte(SS, IndexRegNo, 5, OS);
-  } else {
-    unsigned IndexRegNo;
-    if (IndexReg.getReg())
-      IndexRegNo = getX86RegNum(IndexReg);
-    else
-      IndexRegNo = 4; // For example [ESP+1*<noreg>+4]
-    emitSIBByte(SS, IndexRegNo, getX86RegNum(Base), OS);
-  }
+  unsigned IndexRegNo = IndexReg.getReg() ? getX86RegNum(IndexReg) : 4;
+
+  // Handle the SIB byte for the case where there is no base, see Intel
+  // Manual 2A, table 2-7. The displacement has already been output.
+  if (BaseReg == 0)
+    BaseRegNo = 5;
+
+  emitSIBByte(SS, IndexRegNo, BaseRegNo, OS);
 
   // Do we need to output a displacement?
   if (ForceDisp8)

From a5b89c285329dbf160d501bf81fcc3765fde219d Mon Sep 17 00:00:00 2001
From: Jez Ng <jezng@fb.com>
Date: Tue, 28 Jul 2020 11:04:43 -0700
Subject: [PATCH 0334/1035] [lld-macho] Fix no-filelist test on Windows

---
 lld/test/MachO/invalid/no-filelist.s | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lld/test/MachO/invalid/no-filelist.s b/lld/test/MachO/invalid/no-filelist.s
index fb80185c66aef..980814cf6eeb1 100644
--- a/lld/test/MachO/invalid/no-filelist.s
+++ b/lld/test/MachO/invalid/no-filelist.s
@@ -1,7 +1,7 @@
 # REQUIRES: x86
 # RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %s -o %t.o
 # RUN: not lld -flavor darwinnew -Z -filelist nonexistent %t.o -o %t 2>&1 | FileCheck %s
-# CHECK: cannot open nonexistent: No such file or directory
+# CHECK: cannot open nonexistent: {{N|n}}o such file or directory
 
 .globl _main
 

From acca2980a33e182dd6f4c71554ff2130f260463e Mon Sep 17 00:00:00 2001
From: Siva Chandra Reddy <sivachandra@google.com>
Date: Tue, 28 Jul 2020 10:57:00 -0700
Subject: [PATCH 0335/1035] [libc][obvious] Move ErrnoSetterMatcher to
 test/ErrnoSetterMetcher.h.

---
 libc/test/CMakeLists.txt                           |  6 ++++++
 libc/{utils/UnitTest => test}/ErrnoSetterMatcher.h | 12 ++++--------
 libc/test/src/signal/CMakeLists.txt                |  6 ++++++
 libc/test/src/signal/sigaction_test.cpp            |  2 +-
 libc/test/src/signal/sigaddset_test.cpp            |  2 +-
 libc/test/src/signal/sigdelset_test.cpp            |  2 +-
 libc/test/src/signal/sigfillset_test.cpp           |  2 +-
 libc/test/src/signal/signal_test.cpp               |  2 +-
 libc/test/src/signal/sigprocmask_test.cpp          |  2 +-
 libc/test/src/sys/mman/linux/CMakeLists.txt        |  1 +
 libc/test/src/sys/mman/linux/mmap_test.cpp         |  2 +-
 libc/test/src/unistd/CMakeLists.txt                |  3 ++-
 libc/test/src/unistd/write_test.cpp                |  2 +-
 libc/utils/UnitTest/CMakeLists.txt                 |  1 -
 14 files changed, 27 insertions(+), 18 deletions(-)
 rename libc/{utils/UnitTest => test}/ErrnoSetterMatcher.h (84%)

diff --git a/libc/test/CMakeLists.txt b/libc/test/CMakeLists.txt
index 500b294ad3866..edce4bbf78cc0 100644
--- a/libc/test/CMakeLists.txt
+++ b/libc/test/CMakeLists.txt
@@ -1,3 +1,9 @@
+add_header_library(
+  errno_setter_matcher
+  HDRS
+    ErrnoSetterMatcher.h
+)
+
 add_custom_target(check-libc)
 
 add_subdirectory(config)
diff --git a/libc/utils/UnitTest/ErrnoSetterMatcher.h b/libc/test/ErrnoSetterMatcher.h
similarity index 84%
rename from libc/utils/UnitTest/ErrnoSetterMatcher.h
rename to libc/test/ErrnoSetterMatcher.h
index d676ee9cbc917..7f8311bfd5e6c 100644
--- a/libc/utils/UnitTest/ErrnoSetterMatcher.h
+++ b/libc/test/ErrnoSetterMatcher.h
@@ -6,15 +6,11 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_LIBC_UTILS_UNITTEST_ERRNOSETTERMATCHER_H
-#define LLVM_LIBC_UTILS_UNITTEST_ERRNOSETTERMATCHER_H
+#ifndef LLVM_LIBC_TEST_ERRNOSETTERMATCHER_H
+#define LLVM_LIBC_TEST_ERRNOSETTERMATCHER_H
 
-#include "Test.h"
-
-// Using LLVM libc headers in UnitTest is not ideal however we also want the
-// test/ directory to have the same layout as libc/ so there is no clean place
-// to put this file except for in utils/UnitTest/.
 #include "src/errno/llvmlibc_errno.h"
+#include "utils/UnitTest/Test.h"
 
 namespace __llvm_libc {
 namespace testing {
@@ -73,4 +69,4 @@ static internal::ErrnoSetterMatcher<RetT> Fails(int ExpectedErrno,
 } // namespace testing
 } // namespace __llvm_libc
 
-#endif // LLVM_LIBC_UTILS_UNITTEST_ERRNOSETTERMATCHER_H
+#endif // LLVM_LIBC_TEST_ERRNOSETTERMATCHER_H
diff --git a/libc/test/src/signal/CMakeLists.txt b/libc/test/src/signal/CMakeLists.txt
index b5c1281b81023..c5baaf7d9d550 100644
--- a/libc/test/src/signal/CMakeLists.txt
+++ b/libc/test/src/signal/CMakeLists.txt
@@ -22,6 +22,7 @@ add_libc_unittest(
     libc.include.signal
     libc.src.signal.raise
     libc.src.signal.sigaction
+    libc.test.errno_setter_matcher
 )
 
 add_libc_unittest(
@@ -37,6 +38,7 @@ add_libc_unittest(
     libc.src.signal.sigaddset
     libc.src.signal.sigemptyset
     libc.src.signal.sigprocmask
+    libc.test.errno_setter_matcher
 )
 
 add_libc_unittest(
@@ -49,6 +51,7 @@ add_libc_unittest(
     libc.include.errno
     libc.include.signal
     libc.src.signal.sigaddset
+    libc.test.errno_setter_matcher
 )
 
 add_libc_unittest(
@@ -63,6 +66,7 @@ add_libc_unittest(
     libc.src.errno.__errno_location
     libc.src.signal.raise
     libc.src.signal.signal
+    libc.test.errno_setter_matcher
 )
 
 add_libc_unittest(
@@ -77,6 +81,7 @@ add_libc_unittest(
     libc.src.signal.raise
     libc.src.signal.sigfillset
     libc.src.signal.sigprocmask
+    libc.test.errno_setter_matcher
 )
 
 add_libc_unittest(
@@ -92,4 +97,5 @@ add_libc_unittest(
     libc.src.signal.sigdelset
     libc.src.signal.sigfillset
     libc.src.signal.sigprocmask
+    libc.test.errno_setter_matcher
 )
diff --git a/libc/test/src/signal/sigaction_test.cpp b/libc/test/src/signal/sigaction_test.cpp
index 726d76b3795de..a473b646072f6 100644
--- a/libc/test/src/signal/sigaction_test.cpp
+++ b/libc/test/src/signal/sigaction_test.cpp
@@ -12,7 +12,7 @@
 #include "src/signal/raise.h"
 #include "src/signal/sigaction.h"
 
-#include "utils/UnitTest/ErrnoSetterMatcher.h"
+#include "test/ErrnoSetterMatcher.h"
 #include "utils/UnitTest/Test.h"
 
 using __llvm_libc::testing::ErrnoSetterMatcher::Fails;
diff --git a/libc/test/src/signal/sigaddset_test.cpp b/libc/test/src/signal/sigaddset_test.cpp
index f106edb57f905..922110b397b6c 100644
--- a/libc/test/src/signal/sigaddset_test.cpp
+++ b/libc/test/src/signal/sigaddset_test.cpp
@@ -10,7 +10,7 @@
 #include "include/signal.h"
 #include "src/signal/sigaddset.h"
 
-#include "utils/UnitTest/ErrnoSetterMatcher.h"
+#include "test/ErrnoSetterMatcher.h"
 #include "utils/UnitTest/Test.h"
 
 // This tests invalid inputs and ensures errno is properly set.
diff --git a/libc/test/src/signal/sigdelset_test.cpp b/libc/test/src/signal/sigdelset_test.cpp
index 48e0e6f3f5c4a..42ba9335e219d 100644
--- a/libc/test/src/signal/sigdelset_test.cpp
+++ b/libc/test/src/signal/sigdelset_test.cpp
@@ -13,7 +13,7 @@
 #include "src/signal/sigfillset.h"
 #include "src/signal/sigprocmask.h"
 
-#include "utils/UnitTest/ErrnoSetterMatcher.h"
+#include "test/ErrnoSetterMatcher.h"
 #include "utils/UnitTest/Test.h"
 
 TEST(Sigdelset, Invalid) {
diff --git a/libc/test/src/signal/sigfillset_test.cpp b/libc/test/src/signal/sigfillset_test.cpp
index c21bf137d2836..16f87e641fbe5 100644
--- a/libc/test/src/signal/sigfillset_test.cpp
+++ b/libc/test/src/signal/sigfillset_test.cpp
@@ -12,7 +12,7 @@
 #include "src/signal/sigfillset.h"
 #include "src/signal/sigprocmask.h"
 
-#include "utils/UnitTest/ErrnoSetterMatcher.h"
+#include "test/ErrnoSetterMatcher.h"
 #include "utils/UnitTest/Test.h"
 
 TEST(Sigfillset, Invalid) {
diff --git a/libc/test/src/signal/signal_test.cpp b/libc/test/src/signal/signal_test.cpp
index 8db26e91d670b..03bf22a5f3eb9 100644
--- a/libc/test/src/signal/signal_test.cpp
+++ b/libc/test/src/signal/signal_test.cpp
@@ -12,7 +12,7 @@
 #include "src/signal/raise.h"
 #include "src/signal/signal.h"
 
-#include "utils/UnitTest/ErrnoSetterMatcher.h"
+#include "test/ErrnoSetterMatcher.h"
 #include "utils/UnitTest/Test.h"
 
 using __llvm_libc::testing::ErrnoSetterMatcher::Fails;
diff --git a/libc/test/src/signal/sigprocmask_test.cpp b/libc/test/src/signal/sigprocmask_test.cpp
index 824415b7267cd..90b57e9e0f2f0 100644
--- a/libc/test/src/signal/sigprocmask_test.cpp
+++ b/libc/test/src/signal/sigprocmask_test.cpp
@@ -14,7 +14,7 @@
 #include "src/signal/sigemptyset.h"
 #include "src/signal/sigprocmask.h"
 
-#include "utils/UnitTest/ErrnoSetterMatcher.h"
+#include "test/ErrnoSetterMatcher.h"
 #include "utils/UnitTest/Test.h"
 
 class SignalTest : public __llvm_libc::testing::Test {
diff --git a/libc/test/src/sys/mman/linux/CMakeLists.txt b/libc/test/src/sys/mman/linux/CMakeLists.txt
index 3c53e4b60107e..c7d3dde7d9751 100644
--- a/libc/test/src/sys/mman/linux/CMakeLists.txt
+++ b/libc/test/src/sys/mman/linux/CMakeLists.txt
@@ -12,4 +12,5 @@ add_libc_unittest(
     libc.src.errno.__errno_location
     libc.src.sys.mman.mmap
     libc.src.sys.mman.munmap
+    libc.test.errno_setter_matcher
 )
diff --git a/libc/test/src/sys/mman/linux/mmap_test.cpp b/libc/test/src/sys/mman/linux/mmap_test.cpp
index 38478c77b18bd..8f91c2b548e01 100644
--- a/libc/test/src/sys/mman/linux/mmap_test.cpp
+++ b/libc/test/src/sys/mman/linux/mmap_test.cpp
@@ -11,7 +11,7 @@
 #include "src/errno/llvmlibc_errno.h"
 #include "src/sys/mman/mmap.h"
 #include "src/sys/mman/munmap.h"
-#include "utils/UnitTest/ErrnoSetterMatcher.h"
+#include "test/ErrnoSetterMatcher.h"
 #include "utils/UnitTest/Test.h"
 
 using __llvm_libc::testing::ErrnoSetterMatcher::Fails;
diff --git a/libc/test/src/unistd/CMakeLists.txt b/libc/test/src/unistd/CMakeLists.txt
index 5eae9c8b478c0..54e513504b3bf 100644
--- a/libc/test/src/unistd/CMakeLists.txt
+++ b/libc/test/src/unistd/CMakeLists.txt
@@ -7,7 +7,8 @@ add_libc_unittest(
   SRCS
     write_test.cpp
   DEPENDS
-    libc.src.unistd.write
     libc.include.errno
     libc.include.unistd
+    libc.src.unistd.write
+    libc.test.errno_setter_matcher
 )
diff --git a/libc/test/src/unistd/write_test.cpp b/libc/test/src/unistd/write_test.cpp
index 2a91ef6fc277f..a2c4cfca05c75 100644
--- a/libc/test/src/unistd/write_test.cpp
+++ b/libc/test/src/unistd/write_test.cpp
@@ -8,7 +8,7 @@
 
 #include "include/errno.h"
 #include "src/unistd/write.h"
-#include "utils/UnitTest/ErrnoSetterMatcher.h"
+#include "test/ErrnoSetterMatcher.h"
 #include "utils/UnitTest/Test.h"
 #include "utils/testutils/FDReader.h"
 
diff --git a/libc/utils/UnitTest/CMakeLists.txt b/libc/utils/UnitTest/CMakeLists.txt
index 0837f3d8895fc..9c7b48aea4f94 100644
--- a/libc/utils/UnitTest/CMakeLists.txt
+++ b/libc/utils/UnitTest/CMakeLists.txt
@@ -2,7 +2,6 @@ add_llvm_library(
   LibcUnitTest
   Test.cpp
   Test.h
-  ErrnoSetterMatcher.h
   LINK_COMPONENTS Support
 )
 target_include_directories(LibcUnitTest PUBLIC ${LIBC_SOURCE_DIR})

From 5608f28f552793d115a7f8682559ab053f961924 Mon Sep 17 00:00:00 2001
From: Shu Anzai <shu.anzai@gmail.com>
Date: Tue, 28 Jul 2020 11:05:44 -0700
Subject: [PATCH 0336/1035] [lldb] Change the definition of ANSI_UNFAINT

Change the definition of ANSI_UNFAINT in Editline.cpp.

Differential revision: https://reviews.llvm.org/D84695
---
 lldb/source/Host/common/Editline.cpp | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/lldb/source/Host/common/Editline.cpp b/lldb/source/Host/common/Editline.cpp
index 226e638aba250..49b7a38d8dae4 100644
--- a/lldb/source/Host/common/Editline.cpp
+++ b/lldb/source/Host/common/Editline.cpp
@@ -48,9 +48,12 @@ int setupterm(char *term, int fildes, int *errret);
 // understand the relationship between DisplayInput(), MoveCursor(),
 // SetCurrentLine(), and SaveEditedLine() before making changes.
 
+/// https://www.ecma-international.org/publications/files/ECMA-ST/Ecma-048.pdf
 #define ESCAPE "\x1b"
+/// Faint, decreased intensity or second colour.
 #define ANSI_FAINT ESCAPE "[2m"
-#define ANSI_UNFAINT ESCAPE "[22m"
+/// Normal colour or normal intensity (neither bold nor faint).
+#define ANSI_UNFAINT ESCAPE "[0m"
 #define ANSI_CLEAR_BELOW ESCAPE "[J"
 #define ANSI_CLEAR_RIGHT ESCAPE "[K"
 #define ANSI_SET_COLUMN_N ESCAPE "[%dG"

From 745eb02496b515cc8292dd7f9d7f0db43e162013 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Martin=20Storsj=C3=B6?= <martin@martin.st>
Date: Mon, 27 Jul 2020 23:44:41 +0300
Subject: [PATCH 0337/1035] [LLD] [MinGW] Implement the --no-seh flag

Previously this flag was just ignored. If set, set the
IMAGE_DLL_CHARACTERISTICS_NO_SEH bit, regardless of the normal safeSEH
machinery.

In mingw configurations, the safeSEH bit might not be set in e.g. object
files built from handwritten assembly, making it impossible to use the
normal safeseh flag. As mingw setups don't generally use SEH on 32 bit
x86 at all, it should be fine to set that flag bit though - hook up
the existing GNU ld flag for controlling that.

Differential Revision: https://reviews.llvm.org/D84701
---
 lld/COFF/Config.h          |  1 +
 lld/COFF/Driver.cpp        |  7 ++++---
 lld/COFF/Options.td        |  1 +
 lld/COFF/Writer.cpp        |  2 +-
 lld/MinGW/Driver.cpp       |  2 ++
 lld/MinGW/Options.td       |  2 +-
 lld/test/COFF/noseh.s      | 19 +++++++++++++++++++
 lld/test/MinGW/driver.test |  4 ++++
 8 files changed, 33 insertions(+), 5 deletions(-)
 create mode 100644 lld/test/COFF/noseh.s

diff --git a/lld/COFF/Config.h b/lld/COFF/Config.h
index 72d826b8bd173..7c439176f3a45 100644
--- a/lld/COFF/Config.h
+++ b/lld/COFF/Config.h
@@ -140,6 +140,7 @@ struct Configuration {
   bool safeSEH = false;
   Symbol *sehTable = nullptr;
   Symbol *sehCount = nullptr;
+  bool noSEH = false;
 
   // Used for /opt:lldlto=N
   unsigned ltoo = 2;
diff --git a/lld/COFF/Driver.cpp b/lld/COFF/Driver.cpp
index 7372505bb6161..9ceccef867797 100644
--- a/lld/COFF/Driver.cpp
+++ b/lld/COFF/Driver.cpp
@@ -1700,9 +1700,10 @@ void LinkerDriver::link(ArrayRef<const char *> argsArr) {
   config->wordsize = config->is64() ? 8 : 4;
 
   // Handle /safeseh, x86 only, on by default, except for mingw.
-  if (config->machine == I386 &&
-      args.hasFlag(OPT_safeseh, OPT_safeseh_no, !config->mingw))
-    config->safeSEH = true;
+  if (config->machine == I386) {
+    config->safeSEH = args.hasFlag(OPT_safeseh, OPT_safeseh_no, !config->mingw);
+    config->noSEH = args.hasArg(OPT_noseh);
+  }
 
   // Handle /functionpadmin
   for (auto *arg : args.filtered(OPT_functionpadmin, OPT_functionpadmin_opt))
diff --git a/lld/COFF/Options.td b/lld/COFF/Options.td
index 212879e1d60bd..087d53b5d2ddd 100644
--- a/lld/COFF/Options.td
+++ b/lld/COFF/Options.td
@@ -204,6 +204,7 @@ def include_optional : Joined<["/", "-", "/?", "-?"], "includeoptional:">,
     HelpText<"Add symbol as undefined, but allow it to remain undefined">;
 def kill_at : F<"kill-at">;
 def lldmingw : F<"lldmingw">;
+def noseh : F<"noseh">;
 def output_def : Joined<["/", "-", "/?", "-?"], "output-def:">;
 def pdb_source_path : P<"pdbsourcepath",
     "Base path used to make relative source file path absolute in PDB">;
diff --git a/lld/COFF/Writer.cpp b/lld/COFF/Writer.cpp
index 3bcc1777f7ac8..082de5b8c1d62 100644
--- a/lld/COFF/Writer.cpp
+++ b/lld/COFF/Writer.cpp
@@ -1393,7 +1393,7 @@ template <typename PEHeaderTy> void Writer::writeHeader() {
     pe->DLLCharacteristics |= IMAGE_DLL_CHARACTERISTICS_GUARD_CF;
   if (config->integrityCheck)
     pe->DLLCharacteristics |= IMAGE_DLL_CHARACTERISTICS_FORCE_INTEGRITY;
-  if (setNoSEHCharacteristic)
+  if (setNoSEHCharacteristic || config->noSEH)
     pe->DLLCharacteristics |= IMAGE_DLL_CHARACTERISTICS_NO_SEH;
   if (config->terminalServerAware)
     pe->DLLCharacteristics |= IMAGE_DLL_CHARACTERISTICS_TERMINAL_SERVER_AWARE;
diff --git a/lld/MinGW/Driver.cpp b/lld/MinGW/Driver.cpp
index f33b5e19502c6..d60765c70c095 100644
--- a/lld/MinGW/Driver.cpp
+++ b/lld/MinGW/Driver.cpp
@@ -288,6 +288,8 @@ bool mingw::link(ArrayRef<const char *> argsArr, bool canExitEarly,
     add("-kill-at");
   if (args.hasArg(OPT_appcontainer))
     add("-appcontainer");
+  if (args.hasArg(OPT_no_seh))
+    add("-noseh");
 
   if (args.getLastArgValue(OPT_m) != "thumb2pe" &&
       args.getLastArgValue(OPT_m) != "arm64pe" && !args.hasArg(OPT_dynamicbase))
diff --git a/lld/MinGW/Options.td b/lld/MinGW/Options.td
index 3281951dc89dd..fe44166600509 100644
--- a/lld/MinGW/Options.td
+++ b/lld/MinGW/Options.td
@@ -56,6 +56,7 @@ defm minor_subsystem_version: EqLong<"minor-subsystem-version",
      "Set the OS and subsystem minor version">;
 def no_insert_timestamp: F<"no-insert-timestamp">,
     HelpText<"Don't include PE header timestamp">;
+def no_seh: F<"no-seh">, HelpText<"Set the 'no SEH' flag in the executable">;
 def no_whole_archive: F<"no-whole-archive">,
     HelpText<"No longer include all object files for following archives">;
 def large_address_aware: Flag<["--"], "large-address-aware">,
@@ -111,7 +112,6 @@ def: Flag<["--"], "full-shutdown">;
 def: F<"high-entropy-va">;
 def: S<"major-image-version">;
 def: S<"minor-image-version">;
-def: F<"no-seh">;
 def: F<"nxcompat">;
 def: F<"pic-executable">;
 def: S<"plugin">;
diff --git a/lld/test/COFF/noseh.s b/lld/test/COFF/noseh.s
new file mode 100644
index 0000000000000..442952286229c
--- /dev/null
+++ b/lld/test/COFF/noseh.s
@@ -0,0 +1,19 @@
+# REQUIRES: x86
+# RUN: llvm-mc -triple i686-w64-mingw32 %s -filetype=obj -o %t.obj
+# RUN: lld-link -lldmingw %t.obj -out:%t.exe -entry:main
+# RUN: llvm-readobj --file-headers %t.exe | FileCheck %s --check-prefix=DEFAULT
+# RUN: lld-link -lldmingw %t.obj -out:%t.noseh.exe -entry:main -noseh
+# RUN: llvm-readobj --file-headers %t.noseh.exe | FileCheck %s --check-prefix=NOSEH
+
+# DEFAULT: Characteristics [
+# DEFAULT-NOT:   IMAGE_DLL_CHARACTERISTICS_NO_SEH
+# DEFAULT: ]
+
+# NOSEH: Characteristics [
+# NOSEH:   IMAGE_DLL_CHARACTERISTICS_NO_SEH
+# NOSEH: ]
+
+        .text
+        .globl  _main
+_main:
+        ret
diff --git a/lld/test/MinGW/driver.test b/lld/test/MinGW/driver.test
index 385822c7e1f70..faac3a0be57d0 100644
--- a/lld/test/MinGW/driver.test
+++ b/lld/test/MinGW/driver.test
@@ -256,3 +256,7 @@ RUN: ld.lld -### -m i386pep foo.o -section-alignment 0x2000 | FileCheck -check-p
 RUN: ld.lld -### -m i386pep foo.o --section-alignment=0x2000 | FileCheck -check-prefix ALIGN %s
 RUN: ld.lld -### -m i386pep foo.o -section-alignment=0x2000 | FileCheck -check-prefix ALIGN %s
 ALIGN: -align:0x2000
+
+RUN: ld.lld -### -m i386pe foo.o -no-seh | FileCheck -check-prefix NOSEH %s
+RUN: ld.lld -### -m i386pe foo.o --no-seh | FileCheck -check-prefix NOSEH %s
+NOSEH: -noseh

From 4c9af6d0e001bf76007527899df7a9d8860c9a5a Mon Sep 17 00:00:00 2001
From: Juneyoung Lee <aqjune@gmail.com>
Date: Wed, 29 Jul 2020 03:11:57 +0900
Subject: [PATCH 0338/1035] [JumpThreading] Add a basic support for freeze
 instruction

This patch adds a basic support for freeze instruction to JumpThreading
by making ComputeValueKnownInPredecessorsImpl look into its operand.

Reviewed By: efriedma, nikic

Differential Revision: https://reviews.llvm.org/D84598
---
 llvm/lib/Transforms/Scalar/JumpThreading.cpp |  21 +++-
 llvm/test/Transforms/JumpThreading/freeze.ll | 103 ++++++++-----------
 2 files changed, 60 insertions(+), 64 deletions(-)

diff --git a/llvm/lib/Transforms/Scalar/JumpThreading.cpp b/llvm/lib/Transforms/Scalar/JumpThreading.cpp
index 2f379b7f61608..7399c7ab609e5 100644
--- a/llvm/lib/Transforms/Scalar/JumpThreading.cpp
+++ b/llvm/lib/Transforms/Scalar/JumpThreading.cpp
@@ -675,10 +675,11 @@ bool JumpThreadingPass::ComputeValueKnownInPredecessorsImpl(
   }
 
   // Handle Cast instructions.  Only see through Cast when the source operand is
-  // PHI or Cmp to save the compilation time.
+  // PHI, Cmp, or Freeze to save the compilation time.
   if (CastInst *CI = dyn_cast<CastInst>(I)) {
     Value *Source = CI->getOperand(0);
-    if (!isa<PHINode>(Source) && !isa<CmpInst>(Source))
+    if (!isa<PHINode>(Source) && !isa<CmpInst>(Source) &&
+        !isa<FreezeInst>(Source))
       return false;
     ComputeValueKnownInPredecessorsImpl(Source, BB, Result, Preference,
                                         RecursionSet, CxtI);
@@ -692,6 +693,22 @@ bool JumpThreadingPass::ComputeValueKnownInPredecessorsImpl(
     return true;
   }
 
+  // Handle Freeze instructions, in a manner similar to Cast.
+  if (FreezeInst *FI = dyn_cast<FreezeInst>(I)) {
+    Value *Source = FI->getOperand(0);
+    if (!isa<PHINode>(Source) && !isa<CmpInst>(Source) &&
+        !isa<CastInst>(Source))
+      return false;
+    ComputeValueKnownInPredecessorsImpl(Source, BB, Result, Preference,
+                                        RecursionSet, CxtI);
+
+    erase_if(Result, [](auto &Pair) {
+      return !isGuaranteedNotToBeUndefOrPoison(Pair.first);
+    });
+
+    return !Result.empty();
+  }
+
   // Handle some boolean conditions.
   if (I->getType()->getPrimitiveSizeInBits() == 1) {
     assert(Preference == WantInteger && "One-bit non-integer type?");
diff --git a/llvm/test/Transforms/JumpThreading/freeze.ll b/llvm/test/Transforms/JumpThreading/freeze.ll
index e08aabf5be170..8f8f3f5566568 100644
--- a/llvm/test/Transforms/JumpThreading/freeze.ll
+++ b/llvm/test/Transforms/JumpThreading/freeze.ll
@@ -7,23 +7,14 @@ declare void @f3()
 
 define i32 @test1(i1 %cond) {
 ; CHECK-LABEL: @test1(
-; CHECK-NEXT:    br i1 [[COND:%.*]], label [[T1:%.*]], label [[F1:%.*]]
-; CHECK:       T1:
-; CHECK-NEXT:    [[V1:%.*]] = call i32 @f1()
-; CHECK-NEXT:    br label [[MERGE:%.*]]
-; CHECK:       F1:
-; CHECK-NEXT:    [[V2:%.*]] = call i32 @f2()
-; CHECK-NEXT:    br label [[MERGE]]
-; CHECK:       Merge:
-; CHECK-NEXT:    [[A:%.*]] = phi i1 [ true, [[T1]] ], [ false, [[F1]] ]
-; CHECK-NEXT:    [[B:%.*]] = phi i32 [ [[V1]], [[T1]] ], [ [[V2]], [[F1]] ]
-; CHECK-NEXT:    [[A_FR:%.*]] = freeze i1 [[A]]
-; CHECK-NEXT:    br i1 [[A_FR]], label [[T2:%.*]], label [[F2:%.*]]
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[T2:%.*]], label [[F2:%.*]]
 ; CHECK:       T2:
+; CHECK-NEXT:    [[V1:%.*]] = call i32 @f1()
 ; CHECK-NEXT:    call void @f3()
-; CHECK-NEXT:    ret i32 [[B]]
+; CHECK-NEXT:    ret i32 [[V1]]
 ; CHECK:       F2:
-; CHECK-NEXT:    ret i32 [[B]]
+; CHECK-NEXT:    [[V2:%.*]] = call i32 @f2()
+; CHECK-NEXT:    ret i32 [[V2]]
 ;
   br i1 %cond, label %T1, label %F1
 
@@ -51,24 +42,21 @@ F2:
 
 define i32 @test1_cast(i1 %cond) {
 ; CHECK-LABEL: @test1_cast(
-; CHECK-NEXT:    br i1 [[COND:%.*]], label [[T1:%.*]], label [[F1:%.*]]
-; CHECK:       T1:
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[MERGE_THREAD:%.*]], label [[MERGE:%.*]]
+; CHECK:       Merge.thread:
 ; CHECK-NEXT:    [[V1:%.*]] = call i32 @f1()
-; CHECK-NEXT:    br label [[MERGE:%.*]]
-; CHECK:       F1:
-; CHECK-NEXT:    [[V2:%.*]] = call i32 @f2()
-; CHECK-NEXT:    br label [[MERGE]]
+; CHECK-NEXT:    br label [[T2:%.*]]
 ; CHECK:       Merge:
-; CHECK-NEXT:    [[A0:%.*]] = phi i32 [ 1, [[T1]] ], [ 0, [[F1]] ]
-; CHECK-NEXT:    [[B:%.*]] = phi i32 [ [[V1]], [[T1]] ], [ [[V2]], [[F1]] ]
-; CHECK-NEXT:    [[A:%.*]] = trunc i32 [[A0]] to i1
+; CHECK-NEXT:    [[V2:%.*]] = call i32 @f2()
+; CHECK-NEXT:    [[A:%.*]] = trunc i32 0 to i1
 ; CHECK-NEXT:    [[A_FR:%.*]] = freeze i1 [[A]]
-; CHECK-NEXT:    br i1 [[A_FR]], label [[T2:%.*]], label [[F2:%.*]]
+; CHECK-NEXT:    br i1 [[A_FR]], label [[T2]], label [[F2:%.*]]
 ; CHECK:       T2:
+; CHECK-NEXT:    [[B5:%.*]] = phi i32 [ [[V1]], [[MERGE_THREAD]] ], [ [[V2]], [[MERGE]] ]
 ; CHECK-NEXT:    call void @f3()
-; CHECK-NEXT:    ret i32 [[B]]
+; CHECK-NEXT:    ret i32 [[B5]]
 ; CHECK:       F2:
-; CHECK-NEXT:    ret i32 [[B]]
+; CHECK-NEXT:    ret i32 [[V2]]
 ;
   br i1 %cond, label %T1, label %F1
 
@@ -97,24 +85,21 @@ F2:
 
 define i32 @test1_cast2(i1 %cond) {
 ; CHECK-LABEL: @test1_cast2(
-; CHECK-NEXT:    br i1 [[COND:%.*]], label [[T1:%.*]], label [[F1:%.*]]
-; CHECK:       T1:
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[MERGE_THREAD:%.*]], label [[MERGE:%.*]]
+; CHECK:       Merge.thread:
 ; CHECK-NEXT:    [[V1:%.*]] = call i32 @f1()
-; CHECK-NEXT:    br label [[MERGE:%.*]]
-; CHECK:       F1:
-; CHECK-NEXT:    [[V2:%.*]] = call i32 @f2()
-; CHECK-NEXT:    br label [[MERGE]]
+; CHECK-NEXT:    br label [[T2:%.*]]
 ; CHECK:       Merge:
-; CHECK-NEXT:    [[A0:%.*]] = phi i32 [ 1, [[T1]] ], [ 0, [[F1]] ]
-; CHECK-NEXT:    [[B:%.*]] = phi i32 [ [[V1]], [[T1]] ], [ [[V2]], [[F1]] ]
-; CHECK-NEXT:    [[A0_FR:%.*]] = freeze i32 [[A0]]
+; CHECK-NEXT:    [[V2:%.*]] = call i32 @f2()
+; CHECK-NEXT:    [[A0_FR:%.*]] = freeze i32 0
 ; CHECK-NEXT:    [[A_FR:%.*]] = trunc i32 [[A0_FR]] to i1
-; CHECK-NEXT:    br i1 [[A_FR]], label [[T2:%.*]], label [[F2:%.*]]
+; CHECK-NEXT:    br i1 [[A_FR]], label [[T2]], label [[F2:%.*]]
 ; CHECK:       T2:
+; CHECK-NEXT:    [[B5:%.*]] = phi i32 [ [[V1]], [[MERGE_THREAD]] ], [ [[V2]], [[MERGE]] ]
 ; CHECK-NEXT:    call void @f3()
-; CHECK-NEXT:    ret i32 [[B]]
+; CHECK-NEXT:    ret i32 [[B5]]
 ; CHECK:       F2:
-; CHECK-NEXT:    ret i32 [[B]]
+; CHECK-NEXT:    ret i32 [[V2]]
 ;
   br i1 %cond, label %T1, label %F1
 
@@ -143,23 +128,20 @@ F2:
 
 define i32 @test1_undef(i1 %cond) {
 ; CHECK-LABEL: @test1_undef(
-; CHECK-NEXT:    br i1 [[COND:%.*]], label [[T1:%.*]], label [[F1:%.*]]
-; CHECK:       T1:
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[MERGE_THREAD:%.*]], label [[MERGE:%.*]]
+; CHECK:       Merge.thread:
 ; CHECK-NEXT:    [[V1:%.*]] = call i32 @f1()
-; CHECK-NEXT:    br label [[MERGE:%.*]]
-; CHECK:       F1:
-; CHECK-NEXT:    [[V2:%.*]] = call i32 @f2()
-; CHECK-NEXT:    br label [[MERGE]]
+; CHECK-NEXT:    br label [[T2:%.*]]
 ; CHECK:       Merge:
-; CHECK-NEXT:    [[A:%.*]] = phi i1 [ true, [[T1]] ], [ undef, [[F1]] ]
-; CHECK-NEXT:    [[B:%.*]] = phi i32 [ [[V1]], [[T1]] ], [ [[V2]], [[F1]] ]
-; CHECK-NEXT:    [[A_FR:%.*]] = freeze i1 [[A]]
-; CHECK-NEXT:    br i1 [[A_FR]], label [[T2:%.*]], label [[F2:%.*]]
+; CHECK-NEXT:    [[V2:%.*]] = call i32 @f2()
+; CHECK-NEXT:    [[A_FR:%.*]] = freeze i1 undef
+; CHECK-NEXT:    br i1 [[A_FR]], label [[T2]], label [[F2:%.*]]
 ; CHECK:       T2:
+; CHECK-NEXT:    [[B4:%.*]] = phi i32 [ [[V1]], [[MERGE_THREAD]] ], [ [[V2]], [[MERGE]] ]
 ; CHECK-NEXT:    call void @f3()
-; CHECK-NEXT:    ret i32 [[B]]
+; CHECK-NEXT:    ret i32 [[B4]]
 ; CHECK:       F2:
-; CHECK-NEXT:    ret i32 [[B]]
+; CHECK-NEXT:    ret i32 [[V2]]
 ;
   br i1 %cond, label %T1, label %F1
 
@@ -187,23 +169,20 @@ F2:
 
 define i32 @test2(i1 %cond, i1 %cond2) {
 ; CHECK-LABEL: @test2(
-; CHECK-NEXT:    br i1 [[COND:%.*]], label [[T1:%.*]], label [[F1:%.*]]
-; CHECK:       T1:
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[MERGE_THREAD:%.*]], label [[MERGE:%.*]]
+; CHECK:       Merge.thread:
 ; CHECK-NEXT:    [[V1:%.*]] = call i32 @f1()
-; CHECK-NEXT:    br label [[MERGE:%.*]]
-; CHECK:       F1:
-; CHECK-NEXT:    [[V2:%.*]] = call i32 @f2()
-; CHECK-NEXT:    br label [[MERGE]]
+; CHECK-NEXT:    br label [[T2:%.*]]
 ; CHECK:       Merge:
-; CHECK-NEXT:    [[A:%.*]] = phi i1 [ true, [[T1]] ], [ [[COND2:%.*]], [[F1]] ]
-; CHECK-NEXT:    [[B:%.*]] = phi i32 [ [[V1]], [[T1]] ], [ [[V2]], [[F1]] ]
-; CHECK-NEXT:    [[A_FR:%.*]] = freeze i1 [[A]]
-; CHECK-NEXT:    br i1 [[A_FR]], label [[T2:%.*]], label [[F2:%.*]]
+; CHECK-NEXT:    [[V2:%.*]] = call i32 @f2()
+; CHECK-NEXT:    [[A_FR:%.*]] = freeze i1 [[COND2:%.*]]
+; CHECK-NEXT:    br i1 [[A_FR]], label [[T2]], label [[F2:%.*]]
 ; CHECK:       T2:
+; CHECK-NEXT:    [[B4:%.*]] = phi i32 [ [[V1]], [[MERGE_THREAD]] ], [ [[V2]], [[MERGE]] ]
 ; CHECK-NEXT:    call void @f3()
-; CHECK-NEXT:    ret i32 [[B]]
+; CHECK-NEXT:    ret i32 [[B4]]
 ; CHECK:       F2:
-; CHECK-NEXT:    ret i32 [[B]]
+; CHECK-NEXT:    ret i32 [[V2]]
 ;
   br i1 %cond, label %T1, label %F1
 

From 394db2259575ef3cac8d3d37836b11eb2373c435 Mon Sep 17 00:00:00 2001
From: Amy Huang <akhuang@google.com>
Date: Tue, 28 Jul 2020 11:23:59 -0700
Subject: [PATCH 0339/1035] Revert "Switch to using
 -debug-info-kind=constructor as default (from =limited)"

This reverts commit 227db86a1b7dd6f96f7df14890fcd071bc4fe1f5.

Causing debug info errors in google3 LTO builds; also causes a
debuginfo-test failure.
---
 clang/lib/Driver/ToolChains/Clang.cpp            | 16 +++++++---------
 clang/test/Driver/cl-options.c                   |  6 +++---
 clang/test/Driver/clang-g-opts.c                 |  2 +-
 clang/test/Driver/cuda-dwarf-2.cu                |  2 +-
 clang/test/Driver/debug-options-as.c             |  2 +-
 clang/test/Driver/debug-options.c                |  8 ++++----
 clang/test/Driver/integrated-as.s                | 10 +++++-----
 clang/test/Driver/myriad-toolchain.c             |  2 +-
 clang/test/Driver/openmp-offload-gpu.c           |  2 +-
 clang/test/Driver/split-debug.c                  | 10 +++++-----
 .../SymbolFile/PDB/Inputs/ClassLayoutTest.cpp    |  1 -
 11 files changed, 29 insertions(+), 32 deletions(-)

diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index b0de225f8abf5..68e4eb0eedda1 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -498,7 +498,7 @@ static codegenoptions::DebugInfoKind DebugLevelToInfoKind(const Arg &A) {
     return codegenoptions::DebugLineTablesOnly;
   if (A.getOption().matches(options::OPT_gline_directives_only))
     return codegenoptions::DebugDirectivesOnly;
-  return codegenoptions::DebugInfoConstructor;
+  return codegenoptions::LimitedDebugInfo;
 }
 
 static bool mustUseNonLeafFramePointerForTarget(const llvm::Triple &Triple) {
@@ -2383,7 +2383,7 @@ static void CollectArgsForIntegratedAssembler(Compilation &C,
           CmdArgs.push_back(Value.data());
         } else {
           RenderDebugEnablingArgs(Args, CmdArgs,
-                                  codegenoptions::DebugInfoConstructor,
+                                  codegenoptions::LimitedDebugInfo,
                                   DwarfVersion, llvm::DebuggerKind::Default);
         }
       } else if (Value.startswith("-mcpu") || Value.startswith("-mfpu") ||
@@ -3656,7 +3656,7 @@ static void RenderDebugOptions(const ToolChain &TC, const Driver &D,
   if (const Arg *A =
           Args.getLastArg(options::OPT_g_Group, options::OPT_gsplit_dwarf,
                           options::OPT_gsplit_dwarf_EQ)) {
-    DebugInfoKind = codegenoptions::DebugInfoConstructor;
+    DebugInfoKind = codegenoptions::LimitedDebugInfo;
 
     // If the last option explicitly specified a debug-info level, use it.
     if (checkDebugInfoOption(A, Args, D, TC) &&
@@ -3761,7 +3761,7 @@ static void RenderDebugOptions(const ToolChain &TC, const Driver &D,
     if (checkDebugInfoOption(A, Args, D, TC)) {
       if (DebugInfoKind != codegenoptions::DebugLineTablesOnly &&
           DebugInfoKind != codegenoptions::DebugDirectivesOnly) {
-        DebugInfoKind = codegenoptions::DebugInfoConstructor;
+        DebugInfoKind = codegenoptions::LimitedDebugInfo;
         CmdArgs.push_back("-dwarf-ext-refs");
         CmdArgs.push_back("-fmodule-format=obj");
       }
@@ -3781,9 +3781,7 @@ static void RenderDebugOptions(const ToolChain &TC, const Driver &D,
           TC.GetDefaultStandaloneDebug());
   if (const Arg *A = Args.getLastArg(options::OPT_fstandalone_debug))
     (void)checkDebugInfoOption(A, Args, D, TC);
-  if ((DebugInfoKind == codegenoptions::LimitedDebugInfo ||
-       DebugInfoKind == codegenoptions::DebugInfoConstructor) &&
-      NeedFullDebug)
+  if (DebugInfoKind == codegenoptions::LimitedDebugInfo && NeedFullDebug)
     DebugInfoKind = codegenoptions::FullDebugInfo;
 
   if (Args.hasFlag(options::OPT_gembed_source, options::OPT_gno_embed_source,
@@ -6569,7 +6567,7 @@ void Clang::AddClangCLArgs(const ArgList &Args, types::ID InputType,
                           options::OPT_gline_tables_only)) {
     *EmitCodeView = true;
     if (DebugInfoArg->getOption().matches(options::OPT__SLASH_Z7))
-      *DebugInfoKind = codegenoptions::DebugInfoConstructor;
+      *DebugInfoKind = codegenoptions::LimitedDebugInfo;
     else
       *DebugInfoKind = codegenoptions::DebugLineTablesOnly;
   } else {
@@ -6866,7 +6864,7 @@ void ClangAs::ConstructJob(Compilation &C, const JobAction &JA,
     // the guard for source type, however there is a test which asserts
     // that some assembler invocation receives no -debug-info-kind,
     // and it's not clear whether that test is just overly restrictive.
-    DebugInfoKind = (WantDebug ? codegenoptions::DebugInfoConstructor
+    DebugInfoKind = (WantDebug ? codegenoptions::LimitedDebugInfo
                                : codegenoptions::NoDebugInfo);
     // Add the -fdebug-compilation-dir flag if needed.
     addDebugCompDirArg(Args, CmdArgs, C.getDriver().getVFS());
diff --git a/clang/test/Driver/cl-options.c b/clang/test/Driver/cl-options.c
index 0dcaf61088069..d0c48ae41d9a2 100644
--- a/clang/test/Driver/cl-options.c
+++ b/clang/test/Driver/cl-options.c
@@ -524,11 +524,11 @@
 
 // RUN: %clang_cl /Zi /c -### -- %s 2>&1 | FileCheck -check-prefix=Zi %s
 // Zi: "-gcodeview"
-// Zi: "-debug-info-kind=constructor"
+// Zi: "-debug-info-kind=limited"
 
 // RUN: %clang_cl /Z7 /c -### -- %s 2>&1 | FileCheck -check-prefix=Z7 %s
 // Z7: "-gcodeview"
-// Z7: "-debug-info-kind=constructor"
+// Z7: "-debug-info-kind=limited"
 
 // RUN: %clang_cl /Zd /c -### -- %s 2>&1 | FileCheck -check-prefix=Z7GMLT %s
 // Z7GMLT: "-gcodeview"
@@ -557,7 +557,7 @@
 // which made it "win". This test could not detect that bug.
 // RUN: %clang_cl /Z7 -gdwarf /c -### -- %s 2>&1 | FileCheck -check-prefix=Z7_gdwarf %s
 // Z7_gdwarf: "-gcodeview"
-// Z7_gdwarf: "-debug-info-kind=constructor"
+// Z7_gdwarf: "-debug-info-kind=limited"
 // Z7_gdwarf: "-dwarf-version=4"
 
 // RUN: %clang_cl -fmsc-version=1800 -TP -### -- %s 2>&1 | FileCheck -check-prefix=CXX11 %s
diff --git a/clang/test/Driver/clang-g-opts.c b/clang/test/Driver/clang-g-opts.c
index 60c97790b7dae..bc714b6c93791 100644
--- a/clang/test/Driver/clang-g-opts.c
+++ b/clang/test/Driver/clang-g-opts.c
@@ -31,7 +31,7 @@
 // RUN:             | FileCheck --check-prefix=CHECK-WITH-G-DWARF2 %s
 
 // CHECK-WITHOUT-G-NOT: -debug-info-kind
-// CHECK-WITH-G: "-debug-info-kind=constructor"
+// CHECK-WITH-G: "-debug-info-kind=limited"
 // CHECK-WITH-G: "-dwarf-version=4"
 // CHECK-WITH-G-DWARF2: "-dwarf-version=2"
 
diff --git a/clang/test/Driver/cuda-dwarf-2.cu b/clang/test/Driver/cuda-dwarf-2.cu
index 92b8919729fc4..bcfb2444bc516 100644
--- a/clang/test/Driver/cuda-dwarf-2.cu
+++ b/clang/test/Driver/cuda-dwarf-2.cu
@@ -49,7 +49,7 @@
 
 // HAS_DEBUG-NOT: warning: debug
 // HAS_DEBUG: "-fcuda-is-device"
-// HAS_DEBUG-SAME: "-debug-info-kind={{constructor|line-tables-only}}"
+// HAS_DEBUG-SAME: "-debug-info-kind={{limited|line-tables-only}}"
 // HAS_DEBUG-SAME: "-dwarf-version=2"
 // HAS_DEBUG: ptxas
 // HAS_DEBUG-SAME: "-g"
diff --git a/clang/test/Driver/debug-options-as.c b/clang/test/Driver/debug-options-as.c
index 4808219702e76..51475680e9b18 100644
--- a/clang/test/Driver/debug-options-as.c
+++ b/clang/test/Driver/debug-options-as.c
@@ -23,7 +23,7 @@
 // RUN:   | FileCheck %s
 //
 // CHECK: "-cc1as"
-// CHECK: "-debug-info-kind=constructor"
+// CHECK: "-debug-info-kind=limited"
 
 // Check to make sure clang with -g on a .s file gets passed -dwarf-debug-producer.
 // rdar://12955296
diff --git a/clang/test/Driver/debug-options.c b/clang/test/Driver/debug-options.c
index 2d1a0b2d5cd8f..189c1f9addeb9 100644
--- a/clang/test/Driver/debug-options.c
+++ b/clang/test/Driver/debug-options.c
@@ -274,18 +274,18 @@
 // GLIO_ONLY_DWARF2: "-dwarf-version=2"
 //
 // G_ONLY: "-cc1"
-// G_ONLY: "-debug-info-kind=constructor"
+// G_ONLY: "-debug-info-kind=limited"
 //
 // These tests assert that "-gline-tables-only" "-g" uses the latter,
 // but otherwise not caring about the DebugInfoKind.
 // G_ONLY_DWARF2: "-cc1"
-// G_ONLY_DWARF2: "-debug-info-kind={{standalone|constructor}}"
+// G_ONLY_DWARF2: "-debug-info-kind={{standalone|limited}}"
 // G_ONLY_DWARF2: "-dwarf-version=2"
 //
 // G_STANDALONE: "-cc1"
 // G_STANDALONE: "-debug-info-kind=standalone"
 // G_LIMITED: "-cc1"
-// G_LIMITED: "-debug-info-kind=constructor"
+// G_LIMITED: "-debug-info-kind=limited"
 // G_DWARF2: "-dwarf-version=2"
 // G_DWARF4: "-dwarf-version=4"
 //
@@ -339,7 +339,7 @@
 // NOCI: "-gno-column-info"
 //
 // GEXTREFS: "-dwarf-ext-refs" "-fmodule-format=obj"
-// GEXTREFS: "-debug-info-kind={{standalone|constructor}}"
+// GEXTREFS: "-debug-info-kind={{standalone|limited}}"
 
 // RUN: not %clang -cc1 -debug-info-kind=watkind 2>&1 | FileCheck -check-prefix=BADSTRING1 %s
 // BADSTRING1: error: invalid value 'watkind' in '-debug-info-kind=watkind'
diff --git a/clang/test/Driver/integrated-as.s b/clang/test/Driver/integrated-as.s
index 05999cfe002b5..0194a3d5a4382 100644
--- a/clang/test/Driver/integrated-as.s
+++ b/clang/test/Driver/integrated-as.s
@@ -27,19 +27,19 @@
 // XA_INCLUDE2: "-Ifoo_dir"
 
 // RUN: %clang -### -target x86_64--- -c -integrated-as %s -gdwarf-4 -gdwarf-2 2>&1 | FileCheck --check-prefix=DWARF2 %s
-// DWARF2: "-debug-info-kind=constructor" "-dwarf-version=2"
+// DWARF2: "-debug-info-kind=limited" "-dwarf-version=2"
 
 // RUN: %clang -### -target x86_64--- -c -integrated-as %s -gdwarf-3 2>&1 | FileCheck --check-prefix=DWARF3 %s
-// DWARF3: "-debug-info-kind=constructor" "-dwarf-version=3"
+// DWARF3: "-debug-info-kind=limited" "-dwarf-version=3"
 
 // RUN: %clang -### -target x86_64--- -c -integrated-as %s -gdwarf-4 2>&1 | FileCheck --check-prefix=DWARF4 %s
-// DWARF4: "-debug-info-kind=constructor" "-dwarf-version=4"
+// DWARF4: "-debug-info-kind=limited" "-dwarf-version=4"
 
 // RUN: %clang -### -target x86_64--- -c -integrated-as %s -Xassembler -gdwarf-2 2>&1 | FileCheck --check-prefix=DWARF2XASSEMBLER %s
-// DWARF2XASSEMBLER: "-debug-info-kind=constructor" "-dwarf-version=2"
+// DWARF2XASSEMBLER: "-debug-info-kind=limited" "-dwarf-version=2"
 
 // RUN: %clang -### -target x86_64--- -c -integrated-as %s -Wa,-gdwarf-2 2>&1 | FileCheck --check-prefix=DWARF2WA %s
-// DWARF2WA: "-debug-info-kind=constructor" "-dwarf-version=2"
+// DWARF2WA: "-debug-info-kind=limited" "-dwarf-version=2"
 
 // A dwarf version number that driver can't parse is just stuffed in.
 // RUN: %clang -### -target x86_64--- -c -integrated-as %s -Wa,-gdwarf-huh 2>&1 | FileCheck --check-prefix=BOGODWARF %s
diff --git a/clang/test/Driver/myriad-toolchain.c b/clang/test/Driver/myriad-toolchain.c
index a4bd260a14986..215a02fd0dec1 100644
--- a/clang/test/Driver/myriad-toolchain.c
+++ b/clang/test/Driver/myriad-toolchain.c
@@ -83,7 +83,7 @@
 // NOSTDLIB-NOT: "-lc"
 
 // RUN: %clang -### -c -g %s -target sparc-myriad 2>&1 | FileCheck -check-prefix=G_SPARC %s
-// G_SPARC: "-debug-info-kind=constructor" "-dwarf-version=2"
+// G_SPARC: "-debug-info-kind=limited" "-dwarf-version=2"
 
 // RUN: %clang -### -c %s -target sparc-myriad-rtems -fuse-init-array 2>&1 \
 // RUN: | FileCheck -check-prefix=USE-INIT-ARRAY %s
diff --git a/clang/test/Driver/openmp-offload-gpu.c b/clang/test/Driver/openmp-offload-gpu.c
index 3ddd6446d1176..6415f1d61b720 100644
--- a/clang/test/Driver/openmp-offload-gpu.c
+++ b/clang/test/Driver/openmp-offload-gpu.c
@@ -241,7 +241,7 @@
 
 // HAS_DEBUG-NOT: warning: debug
 // HAS_DEBUG: "-triple" "nvptx64-nvidia-cuda"
-// HAS_DEBUG-SAME: "-debug-info-kind={{constructor|line-tables-only}}"
+// HAS_DEBUG-SAME: "-debug-info-kind={{limited|line-tables-only}}"
 // HAS_DEBUG-SAME: "-dwarf-version=2"
 // HAS_DEBUG-SAME: "-fopenmp-is-device"
 // HAS_DEBUG: ptxas
diff --git a/clang/test/Driver/split-debug.c b/clang/test/Driver/split-debug.c
index 70f8d91d48e01..d40207d5ae3b6 100644
--- a/clang/test/Driver/split-debug.c
+++ b/clang/test/Driver/split-debug.c
@@ -68,18 +68,18 @@
 // RUN: FileCheck -check-prefix=CHECK-NOINLINE-WITHOUT-SPLIT < %t %s
 //
 // CHECK-NOINLINE-WITHOUT-SPLIT: "-fno-split-dwarf-inlining"
-// CHECK-NOINLINE-WITHOUT-SPLIT: "-debug-info-kind=constructor"
+// CHECK-NOINLINE-WITHOUT-SPLIT: "-debug-info-kind=limited"
 
 // RUN: %clang -target x86_64-unknown-linux-gnu -gmlt -gsplit-dwarf -fno-split-dwarf-inlining -S -### %s 2> %t
 // RUN: FileCheck -check-prefix=CHECK-SPLIT-WITH-GMLT < %t %s
 //
-// CHECK-SPLIT-WITH-GMLT: "-debug-info-kind=constructor"
+// CHECK-SPLIT-WITH-GMLT: "-debug-info-kind=limited"
 // CHECK-SPLIT-WITH-GMLT: "-split-dwarf-output"
 
 // RUN: %clang -target x86_64-unknown-linux-gnu -gsplit-dwarf -fno-split-dwarf-inlining -S -### %s 2> %t
 // RUN: FileCheck -check-prefix=CHECK-SPLIT-WITH-NOINL < %t %s
 //
-// CHECK-SPLIT-WITH-NOINL: "-debug-info-kind=constructor"
+// CHECK-SPLIT-WITH-NOINL: "-debug-info-kind=limited"
 // CHECK-SPLIT-WITH-NOINL: "-split-dwarf-output"
 
 // RUN: %clang -target x86_64-unknown-linux-gnu -gsplit-dwarf -gmlt -fsplit-dwarf-inlining -S -### %s 2> %t
@@ -92,7 +92,7 @@
 // RUN: %clang -target x86_64-unknown-linux-gnu -gmlt -gsplit-dwarf -S -### %s 2> %t
 // RUN: FileCheck -check-prefix=CHECK-SPLIT-OVER-GMLT < %t %s
 //
-// CHECK-SPLIT-OVER-GMLT: "-debug-info-kind=constructor"
+// CHECK-SPLIT-OVER-GMLT: "-debug-info-kind=limited"
 // CHECK-SPLIT-OVER-GMLT: "-split-dwarf-file"
 // CHECK-SPLIT-OVER-GMLT: "-split-dwarf-output"
 
@@ -117,6 +117,6 @@
 // RUN: %clang -target x86_64-unknown-linux-gnu -g0 -gsplit-dwarf=split -S -### %s 2> %t
 // RUN: FileCheck -check-prefix=CHECK-SPLIT-OVER-G0 < %t %s
 //
-// CHECK-SPLIT-OVER-G0: "-debug-info-kind=constructor"
+// CHECK-SPLIT-OVER-G0: "-debug-info-kind=limited"
 // CHECK-SPLIT-OVER-G0: "-split-dwarf-file"
 // CHECK-SPLIT-OVER-G0: "-split-dwarf-output"
diff --git a/lldb/test/Shell/SymbolFile/PDB/Inputs/ClassLayoutTest.cpp b/lldb/test/Shell/SymbolFile/PDB/Inputs/ClassLayoutTest.cpp
index 503939680c500..3c4b005cdf1be 100644
--- a/lldb/test/Shell/SymbolFile/PDB/Inputs/ClassLayoutTest.cpp
+++ b/lldb/test/Shell/SymbolFile/PDB/Inputs/ClassLayoutTest.cpp
@@ -106,7 +106,6 @@ class Class : public Base { // Test base class.
 int main() {
   MemberTest::Base B1;
   B1.Get();
-  MemberTest::Class C1;
   MemberTest::Class::StaticMemberFunc(1, 10, 2);
   return 0;
 }

From 834133c950fce120d0378d09718d32a320cbcd72 Mon Sep 17 00:00:00 2001
From: Anand Kodnani <anand.kodnani@intel.com>
Date: Tue, 28 Jul 2020 10:37:16 -0700
Subject: [PATCH 0340/1035] [MLIR] Vector store to load forwarding

The MemRefDataFlow pass does store to load forwarding
only for affine store/loads. This patch updates the pass
to use affine read/write interface which enables vector
forwarding.

Reviewed By: dcaballe, bondhugula, ftynse

Differential Revision: https://reviews.llvm.org/D84302
---
 .../Affine/IR/AffineMemoryOpInterfaces.td     | 21 +++++++++++++++++++
 .../mlir/Dialect/Affine/IR/AffineOps.td       |  4 ++--
 mlir/lib/Transforms/MemRefDataFlowOpt.cpp     | 15 ++++++-------
 mlir/test/Transforms/memref-dataflow-opt.mlir | 20 ++++++++++++++++++
 4 files changed, 51 insertions(+), 9 deletions(-)

diff --git a/mlir/include/mlir/Dialect/Affine/IR/AffineMemoryOpInterfaces.td b/mlir/include/mlir/Dialect/Affine/IR/AffineMemoryOpInterfaces.td
index a093cb9fd4be4..1f25073f07e36 100644
--- a/mlir/include/mlir/Dialect/Affine/IR/AffineMemoryOpInterfaces.td
+++ b/mlir/include/mlir/Dialect/Affine/IR/AffineMemoryOpInterfaces.td
@@ -81,6 +81,16 @@ def AffineReadOpInterface : OpInterface<"AffineReadOpInterface"> {
                 op.getAffineMapAttr()};
       }]
     >,
+    InterfaceMethod<
+      /*desc=*/"Returns the value read by this operation.",
+      /*retTy=*/"Value",
+      /*methodName=*/"getValue",
+      /*args=*/(ins),
+      /*methodBody=*/[{}],
+      /*defaultImplementation=*/[{
+        return cast<ConcreteOp>(this->getOperation());
+      }]
+    >,
   ];
 }
 
@@ -150,6 +160,17 @@ def AffineWriteOpInterface : OpInterface<"AffineWriteOpInterface"> {
                 op.getAffineMapAttr()};
       }]
     >,
+    InterfaceMethod<
+      /*desc=*/"Returns the value to store.",
+      /*retTy=*/"Value",
+      /*methodName=*/"getValueToStore",
+      /*args=*/(ins),
+      /*methodBody=*/[{}],
+      /*defaultImplementation=*/[{
+        ConcreteOp op = cast<ConcreteOp>(this->getOperation());
+        return op.getOperand(op.getStoredValOperandIndex());
+      }]
+    >,
   ];
 }
 
diff --git a/mlir/include/mlir/Dialect/Affine/IR/AffineOps.td b/mlir/include/mlir/Dialect/Affine/IR/AffineOps.td
index ab7d96f7cafa3..95e17aa1eec87 100644
--- a/mlir/include/mlir/Dialect/Affine/IR/AffineOps.td
+++ b/mlir/include/mlir/Dialect/Affine/IR/AffineOps.td
@@ -725,8 +725,8 @@ class AffineStoreOpBase<string mnemonic, list<OpTrait> traits = []> :
     Affine_Op<mnemonic, !listconcat(traits,
         [DeclareOpInterfaceMethods<AffineWriteOpInterface>])> {
   code extraClassDeclarationBase = [{
-    /// Get value to be stored by store operation.
-    Value getValueToStore() { return getOperand(0); }
+    /// Returns the operand index of the value to be stored.
+    unsigned getStoredValOperandIndex() { return 0; }
 
     /// Returns the operand index of the memref.
     unsigned getMemRefOperandIndex() { return 1; }
diff --git a/mlir/lib/Transforms/MemRefDataFlowOpt.cpp b/mlir/lib/Transforms/MemRefDataFlowOpt.cpp
index 7220fd1f3dc6a..7924b46a83d58 100644
--- a/mlir/lib/Transforms/MemRefDataFlowOpt.cpp
+++ b/mlir/lib/Transforms/MemRefDataFlowOpt.cpp
@@ -63,7 +63,7 @@ namespace {
 struct MemRefDataFlowOpt : public MemRefDataFlowOptBase<MemRefDataFlowOpt> {
   void runOnFunction() override;
 
-  void forwardStoreToLoad(AffineLoadOp loadOp);
+  void forwardStoreToLoad(AffineReadOpInterface loadOp);
 
   // A list of memref's that are potentially dead / could be eliminated.
   SmallPtrSet<Value, 4> memrefsToErase;
@@ -84,14 +84,14 @@ std::unique_ptr<OperationPass<FuncOp>> mlir::createMemRefDataFlowOptPass() {
 
 // This is a straightforward implementation not optimized for speed. Optimize
 // if needed.
-void MemRefDataFlowOpt::forwardStoreToLoad(AffineLoadOp loadOp) {
+void MemRefDataFlowOpt::forwardStoreToLoad(AffineReadOpInterface loadOp) {
   // First pass over the use list to get the minimum number of surrounding
   // loops common between the load op and the store op, with min taken across
   // all store ops.
   SmallVector<Operation *, 8> storeOps;
   unsigned minSurroundingLoops = getNestingDepth(loadOp);
   for (auto *user : loadOp.getMemRef().getUsers()) {
-    auto storeOp = dyn_cast<AffineStoreOp>(user);
+    auto storeOp = dyn_cast<AffineWriteOpInterface>(user);
     if (!storeOp)
       continue;
     unsigned nsLoops = getNumCommonSurroundingLoops(*loadOp, *storeOp);
@@ -167,8 +167,9 @@ void MemRefDataFlowOpt::forwardStoreToLoad(AffineLoadOp loadOp) {
     return;
 
   // Perform the actual store to load forwarding.
-  Value storeVal = cast<AffineStoreOp>(lastWriteStoreOp).getValueToStore();
-  loadOp.replaceAllUsesWith(storeVal);
+  Value storeVal =
+    cast<AffineWriteOpInterface>(lastWriteStoreOp).getValueToStore();
+  loadOp.getValue().replaceAllUsesWith(storeVal);
   // Record the memref for a later sweep to optimize away.
   memrefsToErase.insert(loadOp.getMemRef());
   // Record this to erase later.
@@ -190,7 +191,7 @@ void MemRefDataFlowOpt::runOnFunction() {
   memrefsToErase.clear();
 
   // Walk all load's and perform store to load forwarding.
-  f.walk([&](AffineLoadOp loadOp) { forwardStoreToLoad(loadOp); });
+  f.walk([&](AffineReadOpInterface loadOp) { forwardStoreToLoad(loadOp); });
 
   // Erase all load op's whose results were replaced with store fwd'ed ones.
   for (auto *loadOp : loadOpsToErase)
@@ -207,7 +208,7 @@ void MemRefDataFlowOpt::runOnFunction() {
       // could still erase it if the call had no side-effects.
       continue;
     if (llvm::any_of(memref.getUsers(), [&](Operation *ownerOp) {
-          return !isa<AffineStoreOp, DeallocOp>(ownerOp);
+          return !isa<AffineWriteOpInterface, DeallocOp>(ownerOp);
         }))
       continue;
 
diff --git a/mlir/test/Transforms/memref-dataflow-opt.mlir b/mlir/test/Transforms/memref-dataflow-opt.mlir
index 6d5288c775c7f..dfda19372813a 100644
--- a/mlir/test/Transforms/memref-dataflow-opt.mlir
+++ b/mlir/test/Transforms/memref-dataflow-opt.mlir
@@ -280,3 +280,23 @@ func @refs_not_known_to_be_equal(%A : memref<100 x 100 x f32>, %M : index) {
   }
   return
 }
+
+// The test checks for value forwarding from vector stores to vector loads.
+// The value loaded from %in can directly be stored to %out by eliminating
+// store and load from %tmp.
+func @vector_forwarding(%in : memref<512xf32>, %out : memref<512xf32>) {
+  %tmp = alloc() : memref<512xf32>
+  affine.for %i = 0 to 16 {
+    %ld0 = affine.vector_load %in[32*%i] : memref<512xf32>, vector<32xf32>
+    affine.vector_store %ld0, %tmp[32*%i] : memref<512xf32>, vector<32xf32>
+    %ld1 = affine.vector_load %tmp[32*%i] : memref<512xf32>, vector<32xf32>
+    affine.vector_store %ld1, %out[32*%i] : memref<512xf32>, vector<32xf32>
+  }
+  return
+}
+
+// CHECK-LABEL: func @vector_forwarding
+// CHECK:      affine.for %{{.*}} = 0 to 16 {
+// CHECK-NEXT:   %[[LDVAL:.*]] = affine.vector_load
+// CHECK-NEXT:   affine.vector_store %[[LDVAL]],{{.*}}
+// CHECK-NEXT: }

From adeeac9d5a322a751011d4624152317cb43419d9 Mon Sep 17 00:00:00 2001
From: Austin Kerbow <Austin.Kerbow@amd.com>
Date: Mon, 13 Jul 2020 15:38:29 -0700
Subject: [PATCH 0341/1035] [AMDGPU] Spill CSR VGPR which is reserved for SGPR
 spills

Update logic for reserving VGPR for SGPR spills. A CSR VGPR being reserved for
SGPR spills could be clobbered if there were no free lower VGPR's available.
Create a stack object so that it will be spilled in the prologue. Also
adds more tests.

Differential Revision: https://reviews.llvm.org/D83730
---
 llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp  |  49 +++---
 .../AMDGPU/reserve-vgpr-for-sgpr-spill.ll     | 158 +++++++++++++++++-
 2 files changed, 179 insertions(+), 28 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp b/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp
index 1349d3b6bf3f6..a7e963e043f46 100644
--- a/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp
+++ b/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp
@@ -233,10 +233,18 @@ bool SILowerSGPRSpills::spillCalleeSavedRegs(MachineFunction &MF) {
 // Find lowest available VGPR and use it as VGPR reserved for SGPR spills.
 static bool lowerShiftReservedVGPR(MachineFunction &MF,
                                    const GCNSubtarget &ST) {
+  SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
+  const Register PreReservedVGPR = FuncInfo->VGPRReservedForSGPRSpill;
+  // Early out if pre-reservation of a VGPR for SGPR spilling is disabled.
+  if (!PreReservedVGPR)
+    return false;
+
+  // If there are no free lower VGPRs available, default to using the
+  // pre-reserved register instead.
+  Register LowestAvailableVGPR = PreReservedVGPR;
+
   MachineRegisterInfo &MRI = MF.getRegInfo();
   MachineFrameInfo &FrameInfo = MF.getFrameInfo();
-  SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
-  Register LowestAvailableVGPR, ReservedVGPR;
   ArrayRef<MCPhysReg> AllVGPR32s = ST.getRegisterInfo()->getAllVGPR32(MF);
   for (MCPhysReg Reg : AllVGPR32s) {
     if (MRI.isAllocatable(Reg) && !MRI.isPhysRegUsed(Reg)) {
@@ -245,26 +253,29 @@ static bool lowerShiftReservedVGPR(MachineFunction &MF,
     }
   }
 
-  if (!LowestAvailableVGPR)
-    return false;
-
-  ReservedVGPR = FuncInfo->VGPRReservedForSGPRSpill;
   const MCPhysReg *CSRegs = MF.getRegInfo().getCalleeSavedRegs();
-  int i = 0;
+  Optional<int> FI;
+  // Check if we are reserving a CSR. Create a stack object for a possible spill
+  // in the function prologue.
+  if (FuncInfo->isCalleeSavedReg(CSRegs, LowestAvailableVGPR))
+    FI = FrameInfo.CreateSpillStackObject(4, Align(4));
+
+  // Find saved info about the pre-reserved register.
+  const auto *ReservedVGPRInfoItr =
+      std::find_if(FuncInfo->getSGPRSpillVGPRs().begin(),
+                   FuncInfo->getSGPRSpillVGPRs().end(),
+                   [PreReservedVGPR](const auto &SpillRegInfo) {
+                     return SpillRegInfo.VGPR == PreReservedVGPR;
+                   });
+
+  assert(ReservedVGPRInfoItr != FuncInfo->getSGPRSpillVGPRs().end());
+  auto Index =
+      std::distance(FuncInfo->getSGPRSpillVGPRs().begin(), ReservedVGPRInfoItr);
+
+  FuncInfo->setSGPRSpillVGPRs(LowestAvailableVGPR, FI, Index);
 
   for (MachineBasicBlock &MBB : MF) {
-    for (auto Reg : FuncInfo->getSGPRSpillVGPRs()) {
-      if (Reg.VGPR == ReservedVGPR) {
-        MBB.removeLiveIn(ReservedVGPR);
-        MBB.addLiveIn(LowestAvailableVGPR);
-        Optional<int> FI;
-        if (FuncInfo->isCalleeSavedReg(CSRegs, LowestAvailableVGPR))
-          FI = FrameInfo.CreateSpillStackObject(4, Align(4));
-
-        FuncInfo->setSGPRSpillVGPRs(LowestAvailableVGPR, FI, i);
-      }
-      ++i;
-    }
+    MBB.addLiveIn(LowestAvailableVGPR);
     MBB.sortUniqueLiveIns();
   }
 
diff --git a/llvm/test/CodeGen/AMDGPU/reserve-vgpr-for-sgpr-spill.ll b/llvm/test/CodeGen/AMDGPU/reserve-vgpr-for-sgpr-spill.ll
index 9df99aae15d49..73d837efa9f44 100644
--- a/llvm/test/CodeGen/AMDGPU/reserve-vgpr-for-sgpr-spill.ll
+++ b/llvm/test/CodeGen/AMDGPU/reserve-vgpr-for-sgpr-spill.ll
@@ -5,17 +5,21 @@ define void @child_function() #0 {
   ret void
 }
 
-; GCN-LABEL: {{^}}parent_func:
-; CHECK: 	v_writelane_b32 v255, s33, 2
-; CHECK: 	v_writelane_b32 v255, s30, 0
-; CHECK: 	v_writelane_b32 v255, s31, 1
-; CHECK:  s_swappc_b64 s[30:31], s[4:5]
-; CHECK: 	v_readlane_b32 s4, v255, 0
-; CHECK:  v_readlane_b32 s5, v255, 1
-; CHECK: 	v_readlane_b32 s33, v255, 2
+; GCN-LABEL: {{^}}reserve_vgpr_with_no_lower_vgpr_available:
+; GCN:  buffer_store_dword v255, off, s[0:3], s32
+; GCN:  v_writelane_b32 v255, s33, 2
+; GCN:  v_writelane_b32 v255, s30, 0
+; GCN:  v_writelane_b32 v255, s31, 1
+; GCN:  s_swappc_b64 s[30:31], s[4:5]
+; GCN:  v_readlane_b32 s4, v255, 0
+; GCN:  v_readlane_b32 s5, v255, 1
+; GCN:  v_readlane_b32 s33, v255, 2
 ; GCN: ; NumVgprs: 256
 
-define void @parent_func() #0 {
+define void @reserve_vgpr_with_no_lower_vgpr_available() #0 {
+  %alloca = alloca i32, align 4, addrspace(5)
+  store volatile i32 0, i32 addrspace(5)* %alloca
+
   call void asm sideeffect "",
   "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7},~{v8},~{v9}
   ,~{v10},~{v11},~{v12},~{v13},~{v14},~{v15},~{v16},~{v17},~{v18},~{v19}
@@ -47,4 +51,140 @@ define void @parent_func() #0 {
   ret void
 }
 
+; GCN-LABEL: {{^}}reserve_lowest_available_vgpr:
+; GCN:  buffer_store_dword v254, off, s[0:3], s32
+; GCN:  v_writelane_b32 v254, s33, 2
+; GCN:  v_writelane_b32 v254, s30, 0
+; GCN:  v_writelane_b32 v254, s31, 1
+; GCN:  s_swappc_b64 s[30:31], s[4:5]
+; GCN:  v_readlane_b32 s4, v254, 0
+; GCN:  v_readlane_b32 s5, v254, 1
+; GCN:  v_readlane_b32 s33, v254, 2
+
+define void @reserve_lowest_available_vgpr() #0 {
+  %alloca = alloca i32, align 4, addrspace(5)
+  store volatile i32 0, i32 addrspace(5)* %alloca
+
+  call void asm sideeffect "",
+  "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7},~{v8},~{v9}
+  ,~{v10},~{v11},~{v12},~{v13},~{v14},~{v15},~{v16},~{v17},~{v18},~{v19}
+  ,~{v20},~{v21},~{v22},~{v23},~{v24},~{v25},~{v26},~{v27},~{v28},~{v29}
+  ,~{v30},~{v31},~{v32},~{v33},~{v34},~{v35},~{v36},~{v37},~{v38},~{v39}
+  ,~{v40},~{v41},~{v42},~{v43},~{v44},~{v45},~{v46},~{v47},~{v48},~{v49}
+  ,~{v50},~{v51},~{v52},~{v53},~{v54},~{v55},~{v56},~{v57},~{v58},~{v59}
+  ,~{v60},~{v61},~{v62},~{v63},~{v64},~{v65},~{v66},~{v67},~{v68},~{v69}
+  ,~{v70},~{v71},~{v72},~{v73},~{v74},~{v75},~{v76},~{v77},~{v78},~{v79}
+  ,~{v80},~{v81},~{v82},~{v83},~{v84},~{v85},~{v86},~{v87},~{v88},~{v89}
+  ,~{v90},~{v91},~{v92},~{v93},~{v94},~{v95},~{v96},~{v97},~{v98},~{v99}
+  ,~{v100},~{v101},~{v102},~{v103},~{v104},~{v105},~{v106},~{v107},~{v108},~{v109}
+  ,~{v110},~{v111},~{v112},~{v113},~{v114},~{v115},~{v116},~{v117},~{v118},~{v119}
+  ,~{v120},~{v121},~{v122},~{v123},~{v124},~{v125},~{v126},~{v127},~{v128},~{v129}
+  ,~{v130},~{v131},~{v132},~{v133},~{v134},~{v135},~{v136},~{v137},~{v138},~{v139}
+  ,~{v140},~{v141},~{v142},~{v143},~{v144},~{v145},~{v146},~{v147},~{v148},~{v149}
+  ,~{v150},~{v151},~{v152},~{v153},~{v154},~{v155},~{v156},~{v157},~{v158},~{v159}
+  ,~{v160},~{v161},~{v162},~{v163},~{v164},~{v165},~{v166},~{v167},~{v168},~{v169}
+  ,~{v170},~{v171},~{v172},~{v173},~{v174},~{v175},~{v176},~{v177},~{v178},~{v179}
+  ,~{v180},~{v181},~{v182},~{v183},~{v184},~{v185},~{v186},~{v187},~{v188},~{v189}
+  ,~{v190},~{v191},~{v192},~{v193},~{v194},~{v195},~{v196},~{v197},~{v198},~{v199}
+  ,~{v200},~{v201},~{v202},~{v203},~{v204},~{v205},~{v206},~{v207},~{v208},~{v209}
+  ,~{v210},~{v211},~{v212},~{v213},~{v214},~{v215},~{v216},~{v217},~{v218},~{v219}
+  ,~{v220},~{v221},~{v222},~{v223},~{v224},~{v225},~{v226},~{v227},~{v228},~{v229}
+  ,~{v230},~{v231},~{v232},~{v233},~{v234},~{v235},~{v236},~{v237},~{v238},~{v239}
+  ,~{v240},~{v241},~{v242},~{v243},~{v244},~{v245},~{v246},~{v247},~{v248},~{v249}
+  ,~{v250},~{v251},~{v252},~{v253}" () #0
+  call void @child_function()
+  ret void
+}
+
+; GCN-LABEL: {{^}}reserve_vgpr_with_sgpr_spills:
+; GCN-NOT:  buffer_store_dword v255, off, s[0:3], s32
+; GCN: ; def s4
+; GCN: v_writelane_b32 v254, s4, 2
+; GCN: v_readlane_b32 s4, v254, 2
+; GCN: ; use s4
+
+define void @reserve_vgpr_with_sgpr_spills() #0 {
+  %alloca = alloca i32, align 4, addrspace(5)
+  store volatile i32 0, i32 addrspace(5)* %alloca
+
+  call void asm sideeffect "",
+  "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7},~{v8},~{v9}
+  ,~{v10},~{v11},~{v12},~{v13},~{v14},~{v15},~{v16},~{v17},~{v18},~{v19}
+  ,~{v20},~{v21},~{v22},~{v23},~{v24},~{v25},~{v26},~{v27},~{v28},~{v29}
+  ,~{v30},~{v31},~{v32},~{v33},~{v34},~{v35},~{v36},~{v37},~{v38},~{v39}
+  ,~{v40},~{v41},~{v42},~{v43},~{v44},~{v45},~{v46},~{v47},~{v48},~{v49}
+  ,~{v50},~{v51},~{v52},~{v53},~{v54},~{v55},~{v56},~{v57},~{v58},~{v59}
+  ,~{v60},~{v61},~{v62},~{v63},~{v64},~{v65},~{v66},~{v67},~{v68},~{v69}
+  ,~{v70},~{v71},~{v72},~{v73},~{v74},~{v75},~{v76},~{v77},~{v78},~{v79}
+  ,~{v80},~{v81},~{v82},~{v83},~{v84},~{v85},~{v86},~{v87},~{v88},~{v89}
+  ,~{v90},~{v91},~{v92},~{v93},~{v94},~{v95},~{v96},~{v97},~{v98},~{v99}
+  ,~{v100},~{v101},~{v102},~{v103},~{v104},~{v105},~{v106},~{v107},~{v108},~{v109}
+  ,~{v110},~{v111},~{v112},~{v113},~{v114},~{v115},~{v116},~{v117},~{v118},~{v119}
+  ,~{v120},~{v121},~{v122},~{v123},~{v124},~{v125},~{v126},~{v127},~{v128},~{v129}
+  ,~{v130},~{v131},~{v132},~{v133},~{v134},~{v135},~{v136},~{v137},~{v138},~{v139}
+  ,~{v140},~{v141},~{v142},~{v143},~{v144},~{v145},~{v146},~{v147},~{v148},~{v149}
+  ,~{v150},~{v151},~{v152},~{v153},~{v154},~{v155},~{v156},~{v157},~{v158},~{v159}
+  ,~{v160},~{v161},~{v162},~{v163},~{v164},~{v165},~{v166},~{v167},~{v168},~{v169}
+  ,~{v170},~{v171},~{v172},~{v173},~{v174},~{v175},~{v176},~{v177},~{v178},~{v179}
+  ,~{v180},~{v181},~{v182},~{v183},~{v184},~{v185},~{v186},~{v187},~{v188},~{v189}
+  ,~{v190},~{v191},~{v192},~{v193},~{v194},~{v195},~{v196},~{v197},~{v198},~{v199}
+  ,~{v200},~{v201},~{v202},~{v203},~{v204},~{v205},~{v206},~{v207},~{v208},~{v209}
+  ,~{v210},~{v211},~{v212},~{v213},~{v214},~{v215},~{v216},~{v217},~{v218},~{v219}
+  ,~{v220},~{v221},~{v222},~{v223},~{v224},~{v225},~{v226},~{v227},~{v228},~{v229}
+  ,~{v230},~{v231},~{v232},~{v233},~{v234},~{v235},~{v236},~{v237},~{v238},~{v239}
+  ,~{v240},~{v241},~{v242},~{v243},~{v244},~{v245},~{v246},~{v247},~{v248},~{v249}
+  ,~{v250},~{v251},~{v252},~{v253}" () #0
+
+  %sgpr = call i32 asm sideeffect "; def $0", "=s" () #0
+  %cmp = icmp eq i32 undef, 0
+  br i1 %cmp, label %bb0, label %ret
+
+bb0:
+  call void asm sideeffect "; use $0", "s"(i32 %sgpr) #0
+  br label %ret
+
+ret:
+  ret void
+}
+
+; GCN-LABEL: {{^}}reserve_vgpr_with_tail_call
+; GCN-NOT:  buffer_store_dword v255, off, s[0:3], s32
+; GCN-NOT:  v_writelane
+; GCN:  s_setpc_b64 s[4:5]
+
+define void @reserve_vgpr_with_tail_call() #0 {
+  %alloca = alloca i32, align 4, addrspace(5)
+  store volatile i32 0, i32 addrspace(5)* %alloca
+
+  call void asm sideeffect "",
+  "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7},~{v8},~{v9}
+  ,~{v10},~{v11},~{v12},~{v13},~{v14},~{v15},~{v16},~{v17},~{v18},~{v19}
+  ,~{v20},~{v21},~{v22},~{v23},~{v24},~{v25},~{v26},~{v27},~{v28},~{v29}
+  ,~{v30},~{v31},~{v32},~{v33},~{v34},~{v35},~{v36},~{v37},~{v38},~{v39}
+  ,~{v40},~{v41},~{v42},~{v43},~{v44},~{v45},~{v46},~{v47},~{v48},~{v49}
+  ,~{v50},~{v51},~{v52},~{v53},~{v54},~{v55},~{v56},~{v57},~{v58},~{v59}
+  ,~{v60},~{v61},~{v62},~{v63},~{v64},~{v65},~{v66},~{v67},~{v68},~{v69}
+  ,~{v70},~{v71},~{v72},~{v73},~{v74},~{v75},~{v76},~{v77},~{v78},~{v79}
+  ,~{v80},~{v81},~{v82},~{v83},~{v84},~{v85},~{v86},~{v87},~{v88},~{v89}
+  ,~{v90},~{v91},~{v92},~{v93},~{v94},~{v95},~{v96},~{v97},~{v98},~{v99}
+  ,~{v100},~{v101},~{v102},~{v103},~{v104},~{v105},~{v106},~{v107},~{v108},~{v109}
+  ,~{v110},~{v111},~{v112},~{v113},~{v114},~{v115},~{v116},~{v117},~{v118},~{v119}
+  ,~{v120},~{v121},~{v122},~{v123},~{v124},~{v125},~{v126},~{v127},~{v128},~{v129}
+  ,~{v130},~{v131},~{v132},~{v133},~{v134},~{v135},~{v136},~{v137},~{v138},~{v139}
+  ,~{v140},~{v141},~{v142},~{v143},~{v144},~{v145},~{v146},~{v147},~{v148},~{v149}
+  ,~{v150},~{v151},~{v152},~{v153},~{v154},~{v155},~{v156},~{v157},~{v158},~{v159}
+  ,~{v160},~{v161},~{v162},~{v163},~{v164},~{v165},~{v166},~{v167},~{v168},~{v169}
+  ,~{v170},~{v171},~{v172},~{v173},~{v174},~{v175},~{v176},~{v177},~{v178},~{v179}
+  ,~{v180},~{v181},~{v182},~{v183},~{v184},~{v185},~{v186},~{v187},~{v188},~{v189}
+  ,~{v190},~{v191},~{v192},~{v193},~{v194},~{v195},~{v196},~{v197},~{v198},~{v199}
+  ,~{v200},~{v201},~{v202},~{v203},~{v204},~{v205},~{v206},~{v207},~{v208},~{v209}
+  ,~{v210},~{v211},~{v212},~{v213},~{v214},~{v215},~{v216},~{v217},~{v218},~{v219}
+  ,~{v220},~{v221},~{v222},~{v223},~{v224},~{v225},~{v226},~{v227},~{v228},~{v229}
+  ,~{v230},~{v231},~{v232},~{v233},~{v234},~{v235},~{v236},~{v237},~{v238},~{v239}
+  ,~{v240},~{v241},~{v242},~{v243},~{v244},~{v245},~{v246},~{v247},~{v248},~{v249}
+  ,~{v250},~{v251},~{v252},~{v253},~{v254}" () #0
+  musttail call void @child_function()
+  ret void
+}
+
 attributes #0 = { nounwind noinline norecurse "amdgpu-flat-work-group-size"="1,256" }

From 4838cd46a90931af3905233a86a7ceda6df7ac69 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Tue, 28 Jul 2020 13:08:22 +0100
Subject: [PATCH 0342/1035] [X86][XOP] Shuffle v16i8 using VPPERM(X,Y) instead
 of OR(PSHUFB(X),PSHUFB(Y))

---
 llvm/lib/Target/X86/X86ISelLowering.cpp       |  6 +++++
 llvm/test/CodeGen/X86/oddshuffles.ll          | 24 ++++++-------------
 .../CodeGen/X86/vector-shuffle-128-v16.ll     | 24 ++++++++++++++-----
 .../CodeGen/X86/vector-shuffle-256-v32.ll     | 20 ++++++----------
 llvm/test/CodeGen/X86/vector-shuffle-v48.ll   | 10 +++-----
 5 files changed, 41 insertions(+), 43 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 5eadd9c287c79..9dd4dfdfea49a 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -15057,6 +15057,12 @@ static SDValue lowerV16I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
       if (Subtarget.hasVBMI() && Subtarget.hasVLX())
         return lowerShuffleWithPERMV(DL, MVT::v16i8, Mask, V1, V2, DAG);
 
+      // If we have XOP we can use one VPPERM instead of multiple PSHUFBs.
+      if (Subtarget.hasXOP()) {
+        SDValue MaskNode = getConstVector(Mask, MVT::v16i8, DAG, DL, true);
+        return DAG.getNode(X86ISD::VPPERM, DL, MVT::v16i8, V1, V2, MaskNode);
+      }
+
       // Use PALIGNR+Permute if possible - permute might become PSHUFB but the
       // PALIGNR will be cheaper than the second PSHUFB+OR.
       if (SDValue V = lowerShuffleAsByteRotateAndPermute(
diff --git a/llvm/test/CodeGen/X86/oddshuffles.ll b/llvm/test/CodeGen/X86/oddshuffles.ll
index d24fd3f024d49..abec83fec594d 100644
--- a/llvm/test/CodeGen/X86/oddshuffles.ll
+++ b/llvm/test/CodeGen/X86/oddshuffles.ll
@@ -775,17 +775,11 @@ define void @interleave_24i8_out(<24 x i8>* %p, <8 x i8>* %q1, <8 x i8>* %q2, <8
 ;
 ; XOP-LABEL: interleave_24i8_out:
 ; XOP:       # %bb.0:
-; XOP-NEXT:    vmovdqu (%rdi), %xmm0
-; XOP-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
-; XOP-NEXT:    vpshufb {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,xmm1[2,5,u,u,u,u,u,u,u,u]
-; XOP-NEXT:    vpshufb {{.*#+}} xmm3 = xmm0[0,3,6,9,12,15],zero,zero,xmm0[u,u,u,u,u,u,u,u]
-; XOP-NEXT:    vpor %xmm2, %xmm3, %xmm2
-; XOP-NEXT:    vpshufb {{.*#+}} xmm3 = zero,zero,zero,zero,zero,xmm1[0,3,6,u,u,u,u,u,u,u,u]
-; XOP-NEXT:    vpshufb {{.*#+}} xmm4 = xmm0[1,4,7,10,13],zero,zero,zero,xmm0[u,u,u,u,u,u,u,u]
-; XOP-NEXT:    vpor %xmm3, %xmm4, %xmm3
-; XOP-NEXT:    vpshufb {{.*#+}} xmm1 = zero,zero,zero,zero,zero,xmm1[1,4,7,u,u,u,u,u,u,u,u]
-; XOP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[2,5,8,11,14],zero,zero,zero,xmm0[u,u,u,u,u,u,u,u]
-; XOP-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; XOP-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
+; XOP-NEXT:    vmovdqu (%rdi), %xmm1
+; XOP-NEXT:    vpperm {{.*#+}} xmm2 = xmm1[0,3,6,9,12,15],xmm0[2,5],xmm1[u,u,u,u,u,u,u,u]
+; XOP-NEXT:    vpperm {{.*#+}} xmm3 = xmm1[1,4,7,10,13],xmm0[0,3,6],xmm1[u,u,u,u,u,u,u,u]
+; XOP-NEXT:    vpperm {{.*#+}} xmm0 = xmm1[2,5,8,11,14],xmm0[1,4,7],xmm1[u,u,u,u,u,u,u,u]
 ; XOP-NEXT:    vmovq %xmm2, (%rsi)
 ; XOP-NEXT:    vmovq %xmm3, (%rdx)
 ; XOP-NEXT:    vmovq %xmm0, (%rcx)
@@ -883,12 +877,8 @@ define void @interleave_24i8_in(<24 x i8>* %p, <8 x i8>* %q1, <8 x i8>* %q2, <8
 ; XOP-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
 ; XOP-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; XOP-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
-; XOP-NEXT:    vpshufb {{.*#+}} xmm2 = xmm0[0,8],zero,xmm0[1,9],zero,xmm0[2,10],zero,xmm0[3,11],zero,xmm0[4,12],zero,xmm0[5]
-; XOP-NEXT:    vpshufb {{.*#+}} xmm3 = zero,zero,xmm1[0],zero,zero,xmm1[1],zero,zero,xmm1[2],zero,zero,xmm1[3],zero,zero,xmm1[4],zero
-; XOP-NEXT:    vpor %xmm3, %xmm2, %xmm2
-; XOP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[13],zero,xmm0[6,14],zero,xmm0[7,15],zero,xmm0[u,u,u,u,u,u,u,u]
-; XOP-NEXT:    vpshufb {{.*#+}} xmm1 = zero,xmm1[5],zero,zero,xmm1[6],zero,zero,xmm1[7,u,u,u,u,u,u,u,u]
-; XOP-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; XOP-NEXT:    vpperm {{.*#+}} xmm2 = xmm0[0,8],xmm1[0],xmm0[1,9],xmm1[1],xmm0[2,10],xmm1[2],xmm0[3,11],xmm1[3],xmm0[4,12],xmm1[4],xmm0[5]
+; XOP-NEXT:    vpperm {{.*#+}} xmm0 = xmm0[13],xmm1[5],xmm0[6,14],xmm1[6],xmm0[7,15],xmm1[7],xmm0[u,u,u,u,u,u,u,u]
 ; XOP-NEXT:    vmovq %xmm0, 16(%rdi)
 ; XOP-NEXT:    vmovdqu %xmm2, (%rdi)
 ; XOP-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-128-v16.ll b/llvm/test/CodeGen/X86/vector-shuffle-128-v16.ll
index 5ba0efdf7f50b..8e9b991c34bf5 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-128-v16.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-128-v16.ll
@@ -1577,12 +1577,19 @@ define <16 x i8> @shuffle_v16i8_uu_10_02_07_22_14_07_02_18_03_01_14_18_09_11_00(
 ; SSE41-NEXT:    por %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX1OR2-LABEL: shuffle_v16i8_uu_10_02_07_22_14_07_02_18_03_01_14_18_09_11_00:
-; AVX1OR2:       # %bb.0: # %entry
-; AVX1OR2-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[u],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[2],zero,zero,zero
-; AVX1OR2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,10,2,7],zero,xmm0[14,7,2],zero,xmm0[3,1,14],zero,xmm0[9,11,0]
-; AVX1OR2-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX1OR2-NEXT:    retq
+; AVX1-LABEL: shuffle_v16i8_uu_10_02_07_22_14_07_02_18_03_01_14_18_09_11_00:
+; AVX1:       # %bb.0: # %entry
+; AVX1-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[u],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[2],zero,zero,zero
+; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,10,2,7],zero,xmm0[14,7,2],zero,xmm0[3,1,14],zero,xmm0[9,11,0]
+; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: shuffle_v16i8_uu_10_02_07_22_14_07_02_18_03_01_14_18_09_11_00:
+; AVX2:       # %bb.0: # %entry
+; AVX2-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[u],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[2],zero,zero,zero
+; AVX2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,10,2,7],zero,xmm0[14,7,2],zero,xmm0[3,1,14],zero,xmm0[9,11,0]
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: shuffle_v16i8_uu_10_02_07_22_14_07_02_18_03_01_14_18_09_11_00:
 ; AVX512VLBW:       # %bb.0: # %entry
@@ -1596,6 +1603,11 @@ define <16 x i8> @shuffle_v16i8_uu_10_02_07_22_14_07_02_18_03_01_14_18_09_11_00(
 ; AVX512VLVBMI-NEXT:    vmovdqa {{.*#+}} xmm2 = <u,10,2,7,22,14,7,2,18,3,1,14,18,9,11,0>
 ; AVX512VLVBMI-NEXT:    vpermt2b %xmm1, %xmm2, %xmm0
 ; AVX512VLVBMI-NEXT:    retq
+;
+; XOP-LABEL: shuffle_v16i8_uu_10_02_07_22_14_07_02_18_03_01_14_18_09_11_00:
+; XOP:       # %bb.0: # %entry
+; XOP-NEXT:    vpperm {{.*#+}} xmm0 = xmm0[u,10,2,7],xmm1[6],xmm0[14,7,2],xmm1[2],xmm0[3,1,14],xmm1[2],xmm0[9,11,0]
+; XOP-NEXT:    retq
 entry:
   %shuffle = shufflevector <16 x i8> %a, <16 x i8> %b, <16 x i32> <i32 undef, i32 10, i32 2, i32 7, i32 22, i32 14, i32 7, i32 2, i32 18, i32 3, i32 1, i32 14, i32 18, i32 9, i32 11, i32 0>
 
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-256-v32.ll b/llvm/test/CodeGen/X86/vector-shuffle-256-v32.ll
index 0508c6bac2bb1..ff2654e800e23 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-256-v32.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-256-v32.ll
@@ -3407,20 +3407,14 @@ define <32 x i8> @shuffle_v32i8_42_45_12_13_35_35_60_40_17_22_29_44_33_12_48_51_
 ;
 ; XOPAVX1-LABEL: shuffle_v32i8_42_45_12_13_35_35_60_40_17_22_29_44_33_12_48_51_20_19_52_19_49_54_37_32_48_42_59_07_36_34_36_39:
 ; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; XOPAVX1-NEXT:    vpperm {{.*#+}} xmm3 = xmm2[4,3,u,3,u,u,u,u,u,u,u],xmm0[7],xmm2[u,u,u,u]
-; XOPAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm4
-; XOPAVX1-NEXT:    vpshufb {{.*#+}} xmm5 = xmm4[u,u,4,u,1,6],zero,zero,xmm4[0],zero,xmm4[11,u],zero,zero,zero,zero
-; XOPAVX1-NEXT:    vpshufb {{.*#+}} xmm6 = xmm1[u,u],zero,xmm1[u],zero,zero,xmm1[5,0],zero,xmm1[10],zero,xmm1[u,4,2,4,7]
-; XOPAVX1-NEXT:    vpor %xmm5, %xmm6, %xmm5
+; XOPAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; XOPAVX1-NEXT:    vpperm {{.*#+}} xmm3 = xmm1[u,u],xmm2[4],xmm1[u],xmm2[1,6],xmm1[5,0],xmm2[0],xmm1[10],xmm2[11],xmm1[u,4,2,4,7]
+; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
+; XOPAVX1-NEXT:    vpperm {{.*#+}} xmm5 = xmm4[4,3,u,3,u,u,u,u,u,u,u],xmm0[7],xmm4[u,u,u,u]
 ; XOPAVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [0,0,255,0,255,255,255,255,255,255,255,0,255,255,255,255]
-; XOPAVX1-NEXT:    vpblendvb %xmm6, %xmm5, %xmm3, %xmm3
-; XOPAVX1-NEXT:    vpshufb {{.*#+}} xmm4 = zero,zero,xmm4[u,u],zero,zero,xmm4[12],zero,xmm4[u,u,u],zero,zero,xmm4[u,0,3]
-; XOPAVX1-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[10,13,u,u,3,3],zero,xmm1[8,u,u,u,12,1,u],zero,zero
-; XOPAVX1-NEXT:    vpor %xmm4, %xmm1, %xmm1
-; XOPAVX1-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,u],zero,zero,xmm2[u,u,u,u,1,6,13,u,u],zero,xmm2[u,u]
-; XOPAVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,12,13,u,u,u,u],zero,zero,zero,xmm0[u,u,12,u,u]
-; XOPAVX1-NEXT:    vpor %xmm2, %xmm0, %xmm0
+; XOPAVX1-NEXT:    vpblendvb %xmm6, %xmm3, %xmm5, %xmm3
+; XOPAVX1-NEXT:    vpperm {{.*#+}} xmm1 = xmm1[10,13,u,u,3,3],xmm2[12],xmm1[8,u,u,u,12,1,u],xmm2[0,3]
+; XOPAVX1-NEXT:    vpperm {{.*#+}} xmm0 = xmm0[u,u,12,13,u,u,u,u],xmm4[1,6,13],xmm0[u,u,12,u,u]
 ; XOPAVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [255,255,0,0,255,255,255,255,0,0,0,255,255,0,255,255]
 ; XOPAVX1-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-v48.ll b/llvm/test/CodeGen/X86/vector-shuffle-v48.ll
index e854923258257..1cf66bb08bb3c 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-v48.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-v48.ll
@@ -26,13 +26,9 @@ define <32 x i8> @foo(<48 x i8>* %x0) {
 ; XOP-NEXT:    vmovdqu (%rdi), %xmm0
 ; XOP-NEXT:    vmovdqu 16(%rdi), %xmm1
 ; XOP-NEXT:    vmovdqu 32(%rdi), %xmm2
-; XOP-NEXT:    vpshufb {{.*#+}} xmm3 = xmm1[8,9,11,12,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; XOP-NEXT:    vpshufb {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,xmm2[1,2,4,5,7,8,10,11,13,14]
-; XOP-NEXT:    vpor %xmm3, %xmm2, %xmm2
-; XOP-NEXT:    vpshufb {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,2,3,5,6]
-; XOP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,3,4,6,7,9,10,12,13,15],zero,zero,zero,zero,zero
-; XOP-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; XOP-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; XOP-NEXT:    vpperm {{.*#+}} xmm0 = xmm0[0,1,3,4,6,7,9,10,12,13,15],xmm1[0,2,3,5,6]
+; XOP-NEXT:    vpperm {{.*#+}} xmm1 = xmm1[8,9,11,12,14,15],xmm2[1,2,4,5,7,8,10,11,13,14]
+; XOP-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; XOP-NEXT:    retq
 ;
 ; AVX2-LABEL: foo:

From c6920081a8b97a8a911803d2bac50fa1db7a8123 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Tue, 28 Jul 2020 19:37:40 +0100
Subject: [PATCH 0343/1035] [CostModel][X86] Add abs intrinsics cost model
 tests

abs costs currently falls back in scalar generic intrinsic calls
---
 llvm/test/Analysis/CostModel/X86/abs.ll | 354 ++++++++++++++++++++++++
 1 file changed, 354 insertions(+)
 create mode 100644 llvm/test/Analysis/CostModel/X86/abs.ll

diff --git a/llvm/test/Analysis/CostModel/X86/abs.ll b/llvm/test/Analysis/CostModel/X86/abs.ll
new file mode 100644
index 0000000000000..ee61f5c07a393
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/X86/abs.ll
@@ -0,0 +1,354 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+sse2 | FileCheck %s -check-prefixes=CHECK,SSE,SSE2
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+sse2 | FileCheck %s -check-prefixes=CHECK,SSE,SSE2
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+ssse3 | FileCheck %s -check-prefixes=CHECK,SSE,SSSE3
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+sse4.2 | FileCheck %s -check-prefixes=CHECK,SSE,SSE42
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+avx | FileCheck %s -check-prefixes=CHECK,AVX,AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+avx2 | FileCheck %s -check-prefixes=CHECK,AVX,AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+avx512f | FileCheck %s -check-prefixes=CHECK,AVX512,AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+avx512vl,+avx512dq | FileCheck %s -check-prefixes=CHECK,AVX512,AVX512DQ
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+avx512vl,+avx512bw | FileCheck %s -check-prefixes=CHECK,AVX512,AVX512BW
+
+;
+; abs(X, not_posion)
+;
+
+define void @cost_abs_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 x i64> %a512) {
+; SSE2-LABEL: 'cost_abs_i64'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.abs.i64(i64 %a64, i1 false)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2I64 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %a128, i1 false)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %a256, i1 false)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V8I64 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> %a512, i1 false)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; SSSE3-LABEL: 'cost_abs_i64'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.abs.i64(i64 %a64, i1 false)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2I64 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %a128, i1 false)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %a256, i1 false)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V8I64 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> %a512, i1 false)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; SSE42-LABEL: 'cost_abs_i64'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.abs.i64(i64 %a64, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %a128, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %a256, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I64 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> %a512, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX-LABEL: 'cost_abs_i64'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.abs.i64(i64 %a64, i1 false)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %a128, i1 false)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %a256, i1 false)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V8I64 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> %a512, i1 false)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX512-LABEL: 'cost_abs_i64'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.abs.i64(i64 %a64, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %a128, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %a256, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V8I64 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> %a512, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %I64    = call i64 @llvm.abs.i64(i64 %a64, i1 0)
+  %V2I64  = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %a128, i1 0)
+  %V4I64  = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %a256, i1 0)
+  %V8I64  = call <8 x i64> @llvm.abs.v8i64(<8 x i64> %a512, i1 0)
+  ret void
+}
+
+define void @cost_abs_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 x i32> %a512) {
+; SSE2-LABEL: 'cost_abs_i32'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.abs.i32(i32 %a32, i1 false)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V2I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %a128, i1 false)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V4I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %a256, i1 false)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V8I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> %a512, i1 false)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; SSSE3-LABEL: 'cost_abs_i32'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.abs.i32(i32 %a32, i1 false)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V2I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %a128, i1 false)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V4I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %a256, i1 false)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V8I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> %a512, i1 false)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; SSE42-LABEL: 'cost_abs_i32'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.abs.i32(i32 %a32, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %a128, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V4I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %a256, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V8I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> %a512, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX-LABEL: 'cost_abs_i32'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.abs.i32(i32 %a32, i1 false)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %a128, i1 false)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V4I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %a256, i1 false)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 58 for instruction: %V8I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> %a512, i1 false)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX512-LABEL: 'cost_abs_i32'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.abs.i32(i32 %a32, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %a128, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V4I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %a256, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %V8I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> %a512, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %I32   = call i32 @llvm.abs.i32(i32 %a32, i1 0)
+  %V2I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %a128, i1 0)
+  %V4I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %a256, i1 0)
+  %V8I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> %a512, i1 0)
+  ret void
+}
+
+define void @cost_abs_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <32 x i16> %a512) {
+; SSE-LABEL: 'cost_abs_i16'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.abs.i16(i16 %a16, i1 false)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 false)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a256, i1 false)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %a512, i1 false)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX-LABEL: 'cost_abs_i16'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.abs.i16(i16 %a16, i1 false)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 false)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 57 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a256, i1 false)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 114 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %a512, i1 false)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX512-LABEL: 'cost_abs_i16'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.abs.i16(i16 %a16, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 57 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a256, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 123 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %a512, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %I16    = call i16 @llvm.abs.i16(i16 %a16, i1 0)
+  %V8I16  = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 0)
+  %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a256, i1 0)
+  %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %a512, i1 0)
+  ret void
+}
+
+define void @cost_abs_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x i8> %a512) {
+; SSE2-LABEL: 'cost_abs_i8'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.abs.i8(i8 %a8, i1 false)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 78 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 false)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 156 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 false)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 312 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 false)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; SSSE3-LABEL: 'cost_abs_i8'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.abs.i8(i8 %a8, i1 false)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 78 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 false)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 156 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 false)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 312 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 false)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; SSE42-LABEL: 'cost_abs_i8'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.abs.i8(i8 %a8, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX-LABEL: 'cost_abs_i8'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.abs.i8(i8 %a8, i1 false)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 false)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 113 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 false)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 226 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 false)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX512-LABEL: 'cost_abs_i8'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.abs.i8(i8 %a8, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 113 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 243 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %I8    = call i8 @llvm.abs.i8(i8 %a8, i1 0)
+  %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 0)
+  %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 0)
+  %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 0)
+  ret void
+}
+
+;
+; abs(X, posion)
+;
+
+define void @cost_abs_i64_poison(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 x i64> %a512) {
+; SSE2-LABEL: 'cost_abs_i64_poison'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.abs.i64(i64 %a64, i1 true)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2I64 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %a128, i1 true)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %a256, i1 true)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V8I64 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> %a512, i1 true)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; SSSE3-LABEL: 'cost_abs_i64_poison'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.abs.i64(i64 %a64, i1 true)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2I64 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %a128, i1 true)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %a256, i1 true)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V8I64 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> %a512, i1 true)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; SSE42-LABEL: 'cost_abs_i64_poison'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.abs.i64(i64 %a64, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %a128, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %a256, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I64 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> %a512, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX-LABEL: 'cost_abs_i64_poison'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.abs.i64(i64 %a64, i1 true)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %a128, i1 true)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %a256, i1 true)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V8I64 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> %a512, i1 true)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX512-LABEL: 'cost_abs_i64_poison'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.abs.i64(i64 %a64, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %a128, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %a256, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V8I64 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> %a512, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %I64    = call i64 @llvm.abs.i64(i64 %a64, i1 -1)
+  %V2I64  = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %a128, i1 -1)
+  %V4I64  = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %a256, i1 -1)
+  %V8I64  = call <8 x i64> @llvm.abs.v8i64(<8 x i64> %a512, i1 -1)
+  ret void
+}
+
+define void @cost_abs_i32_poison(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 x i32> %a512) {
+; SSE2-LABEL: 'cost_abs_i32_poison'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.abs.i32(i32 %a32, i1 true)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V2I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %a128, i1 true)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V4I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %a256, i1 true)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V8I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> %a512, i1 true)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; SSSE3-LABEL: 'cost_abs_i32_poison'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.abs.i32(i32 %a32, i1 true)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V2I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %a128, i1 true)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V4I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %a256, i1 true)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V8I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> %a512, i1 true)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; SSE42-LABEL: 'cost_abs_i32_poison'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.abs.i32(i32 %a32, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %a128, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V4I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %a256, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V8I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> %a512, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX-LABEL: 'cost_abs_i32_poison'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.abs.i32(i32 %a32, i1 true)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %a128, i1 true)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V4I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %a256, i1 true)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 58 for instruction: %V8I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> %a512, i1 true)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX512-LABEL: 'cost_abs_i32_poison'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.abs.i32(i32 %a32, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %a128, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V4I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %a256, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %V8I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> %a512, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %I32   = call i32 @llvm.abs.i32(i32 %a32, i1 -1)
+  %V2I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %a128, i1 -1)
+  %V4I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %a256, i1 -1)
+  %V8I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> %a512, i1 -1)
+  ret void
+}
+
+define void @cost_abs_i16_poison(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <32 x i16> %a512) {
+; SSE-LABEL: 'cost_abs_i16_poison'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.abs.i16(i16 %a16, i1 true)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 true)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a256, i1 true)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %a512, i1 true)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX-LABEL: 'cost_abs_i16_poison'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.abs.i16(i16 %a16, i1 true)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 true)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 57 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a256, i1 true)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 114 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %a512, i1 true)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX512-LABEL: 'cost_abs_i16_poison'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.abs.i16(i16 %a16, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 57 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a256, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 123 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %a512, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %I16    = call i16 @llvm.abs.i16(i16 %a16, i1 -1)
+  %V8I16  = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 -1)
+  %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a256, i1 -1)
+  %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %a512, i1 -1)
+  ret void
+}
+
+define void @cost_abs_i8_poison(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x i8> %a512) {
+; SSE2-LABEL: 'cost_abs_i8_poison'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.abs.i8(i8 %a8, i1 true)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 78 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 true)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 156 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 true)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 312 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 true)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; SSSE3-LABEL: 'cost_abs_i8_poison'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.abs.i8(i8 %a8, i1 true)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 78 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 true)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 156 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 true)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 312 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 true)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; SSE42-LABEL: 'cost_abs_i8_poison'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.abs.i8(i8 %a8, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX-LABEL: 'cost_abs_i8_poison'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.abs.i8(i8 %a8, i1 true)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 true)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 113 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 true)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 226 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 true)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX512-LABEL: 'cost_abs_i8_poison'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.abs.i8(i8 %a8, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 113 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 243 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %I8    = call i8 @llvm.abs.i8(i8 %a8, i1 -1)
+  %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 -1)
+  %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 -1)
+  %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 -1)
+  ret void
+}
+
+declare i64 @llvm.abs.i64(i64, i1)
+declare i32 @llvm.abs.i32(i32, i1)
+declare i16 @llvm.abs.i16(i16, i1)
+declare  i8 @llvm.abs.i8(i8, i1)
+
+declare <2 x i64>  @llvm.abs.v2i64(<2 x i64>, i1)
+declare <4 x i32>  @llvm.abs.v4i32(<4 x i32>, i1)
+declare <8 x i16>  @llvm.abs.v8i16(<8 x i16>, i1)
+declare <16 x i8>  @llvm.abs.v16i8(<16 x i8>, i1)
+
+declare <4 x i64>  @llvm.abs.v4i64(<4 x i64>, i1)
+declare <8 x i32>  @llvm.abs.v8i32(<8 x i32>, i1)
+declare <16 x i16> @llvm.abs.v16i16(<16 x i16>, i1)
+declare <32 x i8>  @llvm.abs.v32i8(<32 x i8>, i1)
+
+declare <8 x i64>  @llvm.abs.v8i64(<8 x i64>, i1)
+declare <16 x i32> @llvm.abs.v16i32(<16 x i32>, i1)
+declare <32 x i16> @llvm.abs.v32i16(<32 x i16>, i1)
+declare <64 x i8>  @llvm.abs.v64i8(<64 x i8>, i1)

From b4b6e77454b6822e1ba7bdaa1b5bde8654c3e87f Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Tue, 28 Jul 2020 19:48:35 +0100
Subject: [PATCH 0344/1035] [DAG] isSplatValue - add support for
 TRUNCATE/SIGN_EXTEND/ZERO_EXTEND

These are just pass-throughs to the source operand - we can't assume that ANY_EXTEND(splat) will still be a splat though.
---
 .../lib/CodeGen/SelectionDAG/SelectionDAG.cpp |   4 +
 llvm/test/CodeGen/X86/vector-fshl-128.ll      |  96 ++++++++---------
 llvm/test/CodeGen/X86/vector-fshl-256.ll      |  48 ++++-----
 llvm/test/CodeGen/X86/vector-fshl-rot-128.ll  |  38 +++----
 llvm/test/CodeGen/X86/vector-fshl-rot-256.ll  |  36 +++----
 llvm/test/CodeGen/X86/vector-fshl-rot-512.ll  |  64 +++++------
 llvm/test/CodeGen/X86/vector-fshr-128.ll      | 100 +++++++++---------
 llvm/test/CodeGen/X86/vector-fshr-256.ll      |  48 ++++-----
 llvm/test/CodeGen/X86/vector-fshr-rot-128.ll  |  38 +++----
 llvm/test/CodeGen/X86/vector-fshr-rot-256.ll  |  36 +++----
 llvm/test/CodeGen/X86/vector-fshr-rot-512.ll  |  64 +++++------
 llvm/test/CodeGen/X86/vector-rotate-128.ll    |  36 +++----
 llvm/test/CodeGen/X86/vector-rotate-256.ll    |  32 +++---
 .../test/CodeGen/X86/vector-shift-ashr-128.ll |  20 ++--
 .../test/CodeGen/X86/vector-shift-ashr-256.ll |  10 +-
 .../CodeGen/X86/vector-shift-ashr-sub128.ll   |  60 +++++------
 .../test/CodeGen/X86/vector-shift-lshr-128.ll |  20 ++--
 .../test/CodeGen/X86/vector-shift-lshr-256.ll |  10 +-
 .../CodeGen/X86/vector-shift-lshr-sub128.ll   |  60 +++++------
 llvm/test/CodeGen/X86/vector-shift-shl-128.ll |  20 ++--
 llvm/test/CodeGen/X86/vector-shift-shl-256.ll |  10 +-
 .../CodeGen/X86/vector-shift-shl-sub128.ll    |  60 +++++------
 22 files changed, 424 insertions(+), 486 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 592c09c10fb08..87d2fa15d0377 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -2323,6 +2323,10 @@ bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts,
     }
     break;
   }
+  case ISD::TRUNCATE:
+  case ISD::SIGN_EXTEND:
+  case ISD::ZERO_EXTEND:
+    return isSplatValue(V.getOperand(0), DemandedElts, UndefElts);
   }
 
   // We don't support other cases than those above for scalable vectors at
diff --git a/llvm/test/CodeGen/X86/vector-fshl-128.ll b/llvm/test/CodeGen/X86/vector-fshl-128.ll
index 0192d1e8137c1..81c291fa602b8 100644
--- a/llvm/test/CodeGen/X86/vector-fshl-128.ll
+++ b/llvm/test/CodeGen/X86/vector-fshl-128.ll
@@ -1947,16 +1947,16 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %
 ;
 ; AVX512F-LABEL: splatvar_funnnel_v16i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpmovzxbd {{.*#+}} zmm3 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
 ; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
-; AVX512F-NEXT:    vpbroadcastb %xmm2, %xmm2
-; AVX512F-NEXT:    vpmovzxbd {{.*#+}} zmm4 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
-; AVX512F-NEXT:    vpsllvd %zmm4, %zmm3, %zmm3
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512F-NEXT:    vpsubb %xmm2, %xmm4, %xmm4
-; AVX512F-NEXT:    vpmovzxbd {{.*#+}} zmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero,xmm4[4],zero,zero,zero,xmm4[5],zero,zero,zero,xmm4[6],zero,zero,zero,xmm4[7],zero,zero,zero,xmm4[8],zero,zero,zero,xmm4[9],zero,zero,zero,xmm4[10],zero,zero,zero,xmm4[11],zero,zero,zero,xmm4[12],zero,zero,zero,xmm4[13],zero,zero,zero,xmm4[14],zero,zero,zero,xmm4[15],zero,zero,zero
+; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512F-NEXT:    vpsubb %xmm2, %xmm3, %xmm3
+; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512F-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
-; AVX512F-NEXT:    vpsrlvd %zmm4, %zmm1, %zmm1
+; AVX512F-NEXT:    vpsrld %xmm3, %zmm1, %zmm1
+; AVX512F-NEXT:    vpmovzxbd {{.*#+}} zmm3 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512F-NEXT:    vpbroadcastb %xmm2, %xmm2
+; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512F-NEXT:    vpslld %xmm4, %zmm3, %zmm3
 ; AVX512F-NEXT:    vpord %zmm1, %zmm3, %zmm1
 ; AVX512F-NEXT:    vpmovdb %zmm1, %xmm1
 ; AVX512F-NEXT:    vpxor %xmm3, %xmm3, %xmm3
@@ -1967,16 +1967,16 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %
 ;
 ; AVX512VL-LABEL: splatvar_funnnel_v16i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpmovzxbd {{.*#+}} zmm3 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
 ; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
-; AVX512VL-NEXT:    vpbroadcastb %xmm2, %xmm2
-; AVX512VL-NEXT:    vpmovzxbd {{.*#+}} zmm4 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
-; AVX512VL-NEXT:    vpsllvd %zmm4, %zmm3, %zmm3
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512VL-NEXT:    vpsubb %xmm2, %xmm4, %xmm4
-; AVX512VL-NEXT:    vpmovzxbd {{.*#+}} zmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero,xmm4[4],zero,zero,zero,xmm4[5],zero,zero,zero,xmm4[6],zero,zero,zero,xmm4[7],zero,zero,zero,xmm4[8],zero,zero,zero,xmm4[9],zero,zero,zero,xmm4[10],zero,zero,zero,xmm4[11],zero,zero,zero,xmm4[12],zero,zero,zero,xmm4[13],zero,zero,zero,xmm4[14],zero,zero,zero,xmm4[15],zero,zero,zero
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VL-NEXT:    vpsubb %xmm2, %xmm3, %xmm3
+; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512VL-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
-; AVX512VL-NEXT:    vpsrlvd %zmm4, %zmm1, %zmm1
+; AVX512VL-NEXT:    vpsrld %xmm3, %zmm1, %zmm1
+; AVX512VL-NEXT:    vpmovzxbd {{.*#+}} zmm3 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512VL-NEXT:    vpbroadcastb %xmm2, %xmm2
+; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VL-NEXT:    vpslld %xmm4, %zmm3, %zmm3
 ; AVX512VL-NEXT:    vpord %zmm1, %zmm3, %zmm1
 ; AVX512VL-NEXT:    vpmovdb %zmm1, %xmm1
 ; AVX512VL-NEXT:    vpxor %xmm3, %xmm3, %xmm3
@@ -1988,16 +1988,16 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %
 ; AVX512BW-LABEL: splatvar_funnnel_v16i8:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
 ; AVX512BW-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
-; AVX512BW-NEXT:    vpbroadcastb %xmm2, %xmm2
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero,xmm2[8],zero,xmm2[9],zero,xmm2[10],zero,xmm2[11],zero,xmm2[12],zero,xmm2[13],zero,xmm2[14],zero,xmm2[15],zero
-; AVX512BW-NEXT:    vpsllvw %zmm4, %zmm3, %zmm3
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512BW-NEXT:    vpsubb %xmm2, %xmm3, %xmm3
+; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512BW-NEXT:    vpsubb %xmm2, %xmm4, %xmm4
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero,xmm4[8],zero,xmm4[9],zero,xmm4[10],zero,xmm4[11],zero,xmm4[12],zero,xmm4[13],zero,xmm4[14],zero,xmm4[15],zero
-; AVX512BW-NEXT:    vpsrlvw %zmm4, %zmm1, %zmm1
+; AVX512BW-NEXT:    vpsrlw %xmm3, %ymm1, %ymm1
+; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512BW-NEXT:    vpbroadcastb %xmm2, %xmm2
+; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512BW-NEXT:    vpsllw %xmm4, %ymm3, %ymm3
 ; AVX512BW-NEXT:    vpor %ymm1, %ymm3, %ymm1
 ; AVX512BW-NEXT:    vpmovwb %zmm1, %ymm1
 ; AVX512BW-NEXT:    vptestnmb %zmm2, %zmm2, %k1
@@ -2009,16 +2009,16 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %
 ; AVX512VBMI2-LABEL: splatvar_funnnel_v16i8:
 ; AVX512VBMI2:       # %bb.0:
 ; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
 ; AVX512VBMI2-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
-; AVX512VBMI2-NEXT:    vpbroadcastb %xmm2, %xmm2
-; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero,xmm2[8],zero,xmm2[9],zero,xmm2[10],zero,xmm2[11],zero,xmm2[12],zero,xmm2[13],zero,xmm2[14],zero,xmm2[15],zero
-; AVX512VBMI2-NEXT:    vpsllvw %zmm4, %zmm3, %zmm3
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VBMI2-NEXT:    vpsubb %xmm2, %xmm3, %xmm3
+; AVX512VBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512VBMI2-NEXT:    vpsubb %xmm2, %xmm4, %xmm4
-; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero,xmm4[8],zero,xmm4[9],zero,xmm4[10],zero,xmm4[11],zero,xmm4[12],zero,xmm4[13],zero,xmm4[14],zero,xmm4[15],zero
-; AVX512VBMI2-NEXT:    vpsrlvw %zmm4, %zmm1, %zmm1
+; AVX512VBMI2-NEXT:    vpsrlw %xmm3, %ymm1, %ymm1
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512VBMI2-NEXT:    vpbroadcastb %xmm2, %xmm2
+; AVX512VBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VBMI2-NEXT:    vpsllw %xmm4, %ymm3, %ymm3
 ; AVX512VBMI2-NEXT:    vpor %ymm1, %ymm3, %ymm1
 ; AVX512VBMI2-NEXT:    vpmovwb %zmm1, %ymm1
 ; AVX512VBMI2-NEXT:    vptestnmb %zmm2, %zmm2, %k1
@@ -2029,16 +2029,16 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %
 ;
 ; AVX512VLBW-LABEL: splatvar_funnnel_v16i8:
 ; AVX512VLBW:       # %bb.0:
-; AVX512VLBW-NEXT:    vpmovzxbw {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
 ; AVX512VLBW-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
-; AVX512VLBW-NEXT:    vpbroadcastb %xmm2, %xmm2
-; AVX512VLBW-NEXT:    vpmovzxbw {{.*#+}} ymm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero,xmm2[8],zero,xmm2[9],zero,xmm2[10],zero,xmm2[11],zero,xmm2[12],zero,xmm2[13],zero,xmm2[14],zero,xmm2[15],zero
-; AVX512VLBW-NEXT:    vpsllvw %ymm4, %ymm3, %ymm3
-; AVX512VLBW-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512VLBW-NEXT:    vpsubb %xmm2, %xmm4, %xmm4
-; AVX512VLBW-NEXT:    vpmovzxbw {{.*#+}} ymm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero,xmm4[8],zero,xmm4[9],zero,xmm4[10],zero,xmm4[11],zero,xmm4[12],zero,xmm4[13],zero,xmm4[14],zero,xmm4[15],zero
+; AVX512VLBW-NEXT:    vmovdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VLBW-NEXT:    vpsubb %xmm2, %xmm3, %xmm3
+; AVX512VLBW-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512VLBW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512VLBW-NEXT:    vpsrlvw %ymm4, %ymm1, %ymm1
+; AVX512VLBW-NEXT:    vpsrlw %xmm3, %ymm1, %ymm1
+; AVX512VLBW-NEXT:    vpmovzxbw {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512VLBW-NEXT:    vpbroadcastb %xmm2, %xmm2
+; AVX512VLBW-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VLBW-NEXT:    vpsllw %xmm4, %ymm3, %ymm3
 ; AVX512VLBW-NEXT:    vpor %ymm1, %ymm3, %ymm1
 ; AVX512VLBW-NEXT:    vpmovwb %ymm1, %xmm1
 ; AVX512VLBW-NEXT:    vptestnmb %xmm2, %xmm2, %k1
@@ -2049,16 +2049,16 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %
 ;
 ; AVX512VLVBMI2-LABEL: splatvar_funnnel_v16i8:
 ; AVX512VLVBMI2:       # %bb.0:
-; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
 ; AVX512VLVBMI2-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
-; AVX512VLVBMI2-NEXT:    vpbroadcastb %xmm2, %xmm2
-; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero,xmm2[8],zero,xmm2[9],zero,xmm2[10],zero,xmm2[11],zero,xmm2[12],zero,xmm2[13],zero,xmm2[14],zero,xmm2[15],zero
-; AVX512VLVBMI2-NEXT:    vpsllvw %ymm4, %ymm3, %ymm3
-; AVX512VLVBMI2-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512VLVBMI2-NEXT:    vpsubb %xmm2, %xmm4, %xmm4
-; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero,xmm4[8],zero,xmm4[9],zero,xmm4[10],zero,xmm4[11],zero,xmm4[12],zero,xmm4[13],zero,xmm4[14],zero,xmm4[15],zero
+; AVX512VLVBMI2-NEXT:    vmovdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VLVBMI2-NEXT:    vpsubb %xmm2, %xmm3, %xmm3
+; AVX512VLVBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512VLVBMI2-NEXT:    vpsrlvw %ymm4, %ymm1, %ymm1
+; AVX512VLVBMI2-NEXT:    vpsrlw %xmm3, %ymm1, %ymm1
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512VLVBMI2-NEXT:    vpbroadcastb %xmm2, %xmm2
+; AVX512VLVBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VLVBMI2-NEXT:    vpsllw %xmm4, %ymm3, %ymm3
 ; AVX512VLVBMI2-NEXT:    vpor %ymm1, %ymm3, %ymm1
 ; AVX512VLVBMI2-NEXT:    vpmovwb %ymm1, %xmm1
 ; AVX512VLVBMI2-NEXT:    vptestnmb %xmm2, %xmm2, %k1
diff --git a/llvm/test/CodeGen/X86/vector-fshl-256.ll b/llvm/test/CodeGen/X86/vector-fshl-256.ll
index 0cf4c172412a6..afe7716a36e49 100644
--- a/llvm/test/CodeGen/X86/vector-fshl-256.ll
+++ b/llvm/test/CodeGen/X86/vector-fshl-256.ll
@@ -1526,13 +1526,13 @@ define <32 x i8> @splatvar_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y, <32 x i8> %
 ; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm3 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
 ; AVX512BW-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
 ; AVX512BW-NEXT:    vpbroadcastb %xmm2, %ymm2
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm4 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero,ymm2[16],zero,ymm2[17],zero,ymm2[18],zero,ymm2[19],zero,ymm2[20],zero,ymm2[21],zero,ymm2[22],zero,ymm2[23],zero,ymm2[24],zero,ymm2[25],zero,ymm2[26],zero,ymm2[27],zero,ymm2[28],zero,ymm2[29],zero,ymm2[30],zero,ymm2[31],zero
-; AVX512BW-NEXT:    vpsllvw %zmm4, %zmm3, %zmm3
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512BW-NEXT:    vpsubb %ymm2, %ymm4, %ymm4
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm4 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero,ymm4[16],zero,ymm4[17],zero,ymm4[18],zero,ymm4[19],zero,ymm4[20],zero,ymm4[21],zero,ymm4[22],zero,ymm4[23],zero,ymm4[24],zero,ymm4[25],zero,ymm4[26],zero,ymm4[27],zero,ymm4[28],zero,ymm4[29],zero,ymm4[30],zero,ymm4[31],zero
+; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512BW-NEXT:    vpsllw %xmm4, %zmm3, %zmm3
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512BW-NEXT:    vpsubb %xmm2, %xmm4, %xmm4
+; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
-; AVX512BW-NEXT:    vpsrlvw %zmm4, %zmm1, %zmm1
+; AVX512BW-NEXT:    vpsrlw %xmm4, %zmm1, %zmm1
 ; AVX512BW-NEXT:    vporq %zmm1, %zmm3, %zmm1
 ; AVX512BW-NEXT:    vpmovwb %zmm1, %ymm1
 ; AVX512BW-NEXT:    vptestnmb %zmm2, %zmm2, %k1
@@ -1546,13 +1546,13 @@ define <32 x i8> @splatvar_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y, <32 x i8> %
 ; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm3 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
 ; AVX512VBMI2-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
 ; AVX512VBMI2-NEXT:    vpbroadcastb %xmm2, %ymm2
-; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm4 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero,ymm2[16],zero,ymm2[17],zero,ymm2[18],zero,ymm2[19],zero,ymm2[20],zero,ymm2[21],zero,ymm2[22],zero,ymm2[23],zero,ymm2[24],zero,ymm2[25],zero,ymm2[26],zero,ymm2[27],zero,ymm2[28],zero,ymm2[29],zero,ymm2[30],zero,ymm2[31],zero
-; AVX512VBMI2-NEXT:    vpsllvw %zmm4, %zmm3, %zmm3
-; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} ymm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512VBMI2-NEXT:    vpsubb %ymm2, %ymm4, %ymm4
-; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm4 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero,ymm4[16],zero,ymm4[17],zero,ymm4[18],zero,ymm4[19],zero,ymm4[20],zero,ymm4[21],zero,ymm4[22],zero,ymm4[23],zero,ymm4[24],zero,ymm4[25],zero,ymm4[26],zero,ymm4[27],zero,ymm4[28],zero,ymm4[29],zero,ymm4[30],zero,ymm4[31],zero
+; AVX512VBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VBMI2-NEXT:    vpsllw %xmm4, %zmm3, %zmm3
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VBMI2-NEXT:    vpsubb %xmm2, %xmm4, %xmm4
+; AVX512VBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
-; AVX512VBMI2-NEXT:    vpsrlvw %zmm4, %zmm1, %zmm1
+; AVX512VBMI2-NEXT:    vpsrlw %xmm4, %zmm1, %zmm1
 ; AVX512VBMI2-NEXT:    vporq %zmm1, %zmm3, %zmm1
 ; AVX512VBMI2-NEXT:    vpmovwb %zmm1, %ymm1
 ; AVX512VBMI2-NEXT:    vptestnmb %zmm2, %zmm2, %k1
@@ -1565,13 +1565,13 @@ define <32 x i8> @splatvar_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y, <32 x i8> %
 ; AVX512VLBW-NEXT:    vpmovzxbw {{.*#+}} zmm3 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
 ; AVX512VLBW-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
 ; AVX512VLBW-NEXT:    vpbroadcastb %xmm2, %ymm2
-; AVX512VLBW-NEXT:    vpmovzxbw {{.*#+}} zmm4 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero,ymm2[16],zero,ymm2[17],zero,ymm2[18],zero,ymm2[19],zero,ymm2[20],zero,ymm2[21],zero,ymm2[22],zero,ymm2[23],zero,ymm2[24],zero,ymm2[25],zero,ymm2[26],zero,ymm2[27],zero,ymm2[28],zero,ymm2[29],zero,ymm2[30],zero,ymm2[31],zero
-; AVX512VLBW-NEXT:    vpsllvw %zmm4, %zmm3, %zmm3
-; AVX512VLBW-NEXT:    vmovdqa {{.*#+}} ymm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512VLBW-NEXT:    vpsubb %ymm2, %ymm4, %ymm4
-; AVX512VLBW-NEXT:    vpmovzxbw {{.*#+}} zmm4 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero,ymm4[16],zero,ymm4[17],zero,ymm4[18],zero,ymm4[19],zero,ymm4[20],zero,ymm4[21],zero,ymm4[22],zero,ymm4[23],zero,ymm4[24],zero,ymm4[25],zero,ymm4[26],zero,ymm4[27],zero,ymm4[28],zero,ymm4[29],zero,ymm4[30],zero,ymm4[31],zero
+; AVX512VLBW-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VLBW-NEXT:    vpsllw %xmm4, %zmm3, %zmm3
+; AVX512VLBW-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VLBW-NEXT:    vpsubb %xmm2, %xmm4, %xmm4
+; AVX512VLBW-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512VLBW-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
-; AVX512VLBW-NEXT:    vpsrlvw %zmm4, %zmm1, %zmm1
+; AVX512VLBW-NEXT:    vpsrlw %xmm4, %zmm1, %zmm1
 ; AVX512VLBW-NEXT:    vporq %zmm1, %zmm3, %zmm1
 ; AVX512VLBW-NEXT:    vpmovwb %zmm1, %ymm1
 ; AVX512VLBW-NEXT:    vptestnmb %ymm2, %ymm2, %k1
@@ -1584,13 +1584,13 @@ define <32 x i8> @splatvar_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y, <32 x i8> %
 ; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm3 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
 ; AVX512VLVBMI2-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
 ; AVX512VLVBMI2-NEXT:    vpbroadcastb %xmm2, %ymm2
-; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm4 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero,ymm2[16],zero,ymm2[17],zero,ymm2[18],zero,ymm2[19],zero,ymm2[20],zero,ymm2[21],zero,ymm2[22],zero,ymm2[23],zero,ymm2[24],zero,ymm2[25],zero,ymm2[26],zero,ymm2[27],zero,ymm2[28],zero,ymm2[29],zero,ymm2[30],zero,ymm2[31],zero
-; AVX512VLVBMI2-NEXT:    vpsllvw %zmm4, %zmm3, %zmm3
-; AVX512VLVBMI2-NEXT:    vmovdqa {{.*#+}} ymm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512VLVBMI2-NEXT:    vpsubb %ymm2, %ymm4, %ymm4
-; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm4 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero,ymm4[16],zero,ymm4[17],zero,ymm4[18],zero,ymm4[19],zero,ymm4[20],zero,ymm4[21],zero,ymm4[22],zero,ymm4[23],zero,ymm4[24],zero,ymm4[25],zero,ymm4[26],zero,ymm4[27],zero,ymm4[28],zero,ymm4[29],zero,ymm4[30],zero,ymm4[31],zero
+; AVX512VLVBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VLVBMI2-NEXT:    vpsllw %xmm4, %zmm3, %zmm3
+; AVX512VLVBMI2-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VLVBMI2-NEXT:    vpsubb %xmm2, %xmm4, %xmm4
+; AVX512VLVBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
-; AVX512VLVBMI2-NEXT:    vpsrlvw %zmm4, %zmm1, %zmm1
+; AVX512VLVBMI2-NEXT:    vpsrlw %xmm4, %zmm1, %zmm1
 ; AVX512VLVBMI2-NEXT:    vporq %zmm1, %zmm3, %zmm1
 ; AVX512VLVBMI2-NEXT:    vpmovwb %zmm1, %ymm1
 ; AVX512VLVBMI2-NEXT:    vptestnmb %ymm2, %ymm2, %k1
diff --git a/llvm/test/CodeGen/X86/vector-fshl-rot-128.ll b/llvm/test/CodeGen/X86/vector-fshl-rot-128.ll
index 59bef3a97b1fa..b1eb2fdbe7af8 100644
--- a/llvm/test/CodeGen/X86/vector-fshl-rot-128.ll
+++ b/llvm/test/CodeGen/X86/vector-fshl-rot-128.ll
@@ -1056,17 +1056,16 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %amt) nounwind
 ;
 ; AVX512F-LABEL: splatvar_funnnel_v16i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpbroadcastb %xmm1, %xmm1
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
 ; AVX512F-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX512F-NEXT:    vpmovzxbd {{.*#+}} zmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero,xmm3[4],zero,zero,zero,xmm3[5],zero,zero,zero,xmm3[6],zero,zero,zero,xmm3[7],zero,zero,zero,xmm3[8],zero,zero,zero,xmm3[9],zero,zero,zero,xmm3[10],zero,zero,zero,xmm3[11],zero,zero,zero,xmm3[12],zero,zero,zero,xmm3[13],zero,zero,zero,xmm3[14],zero,zero,zero,xmm3[15],zero,zero,zero
+; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512F-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
-; AVX512F-NEXT:    vpsllvd %zmm3, %zmm0, %zmm3
+; AVX512F-NEXT:    vpslld %xmm3, %zmm0, %zmm3
 ; AVX512F-NEXT:    vpxor %xmm4, %xmm4, %xmm4
 ; AVX512F-NEXT:    vpsubb %xmm1, %xmm4, %xmm1
 ; AVX512F-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; AVX512F-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
-; AVX512F-NEXT:    vpsrlvd %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512F-NEXT:    vpsrld %xmm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    vpord %zmm0, %zmm3, %zmm0
 ; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
 ; AVX512F-NEXT:    vzeroupper
@@ -1074,17 +1073,16 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %amt) nounwind
 ;
 ; AVX512VL-LABEL: splatvar_funnnel_v16i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpbroadcastb %xmm1, %xmm1
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
 ; AVX512VL-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX512VL-NEXT:    vpmovzxbd {{.*#+}} zmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero,xmm3[4],zero,zero,zero,xmm3[5],zero,zero,zero,xmm3[6],zero,zero,zero,xmm3[7],zero,zero,zero,xmm3[8],zero,zero,zero,xmm3[9],zero,zero,zero,xmm3[10],zero,zero,zero,xmm3[11],zero,zero,zero,xmm3[12],zero,zero,zero,xmm3[13],zero,zero,zero,xmm3[14],zero,zero,zero,xmm3[15],zero,zero,zero
+; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512VL-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
-; AVX512VL-NEXT:    vpsllvd %zmm3, %zmm0, %zmm3
+; AVX512VL-NEXT:    vpslld %xmm3, %zmm0, %zmm3
 ; AVX512VL-NEXT:    vpxor %xmm4, %xmm4, %xmm4
 ; AVX512VL-NEXT:    vpsubb %xmm1, %xmm4, %xmm1
 ; AVX512VL-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
-; AVX512VL-NEXT:    vpsrlvd %zmm1, %zmm0, %zmm0
+; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VL-NEXT:    vpsrld %xmm1, %zmm0, %zmm0
 ; AVX512VL-NEXT:    vpord %zmm0, %zmm3, %zmm0
 ; AVX512VL-NEXT:    vpmovdb %zmm0, %xmm0
 ; AVX512VL-NEXT:    vzeroupper
@@ -1092,17 +1090,16 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %amt) nounwind
 ;
 ; AVX512BW-LABEL: splatvar_funnnel_v16i8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpbroadcastb %xmm1, %xmm1
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
 ; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
 ; AVX512BW-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero,xmm3[8],zero,xmm3[9],zero,xmm3[10],zero,xmm3[11],zero,xmm3[12],zero,xmm3[13],zero,xmm3[14],zero,xmm3[15],zero
-; AVX512BW-NEXT:    vpsllvw %zmm3, %zmm0, %zmm3
+; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512BW-NEXT:    vpsllw %xmm3, %ymm0, %ymm3
 ; AVX512BW-NEXT:    vpxor %xmm4, %xmm4, %xmm4
 ; AVX512BW-NEXT:    vpsubb %xmm1, %xmm4, %xmm1
 ; AVX512BW-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512BW-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512BW-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
 ; AVX512BW-NEXT:    vpor %ymm0, %ymm3, %ymm0
 ; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
@@ -1111,17 +1108,16 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %amt) nounwind
 ;
 ; AVX512VLBW-LABEL: splatvar_funnnel_v16i8:
 ; AVX512VLBW:       # %bb.0:
-; AVX512VLBW-NEXT:    vpbroadcastb %xmm1, %xmm1
 ; AVX512VLBW-NEXT:    vmovdqa {{.*#+}} xmm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
 ; AVX512VLBW-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX512VLBW-NEXT:    vpmovzxbw {{.*#+}} ymm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero,xmm3[8],zero,xmm3[9],zero,xmm3[10],zero,xmm3[11],zero,xmm3[12],zero,xmm3[13],zero,xmm3[14],zero,xmm3[15],zero
+; AVX512VLBW-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512VLBW-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
-; AVX512VLBW-NEXT:    vpsllvw %ymm3, %ymm0, %ymm3
+; AVX512VLBW-NEXT:    vpsllw %xmm3, %ymm0, %ymm3
 ; AVX512VLBW-NEXT:    vpxor %xmm4, %xmm4, %xmm4
 ; AVX512VLBW-NEXT:    vpsubb %xmm1, %xmm4, %xmm1
 ; AVX512VLBW-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; AVX512VLBW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512VLBW-NEXT:    vpsrlvw %ymm1, %ymm0, %ymm0
+; AVX512VLBW-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VLBW-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
 ; AVX512VLBW-NEXT:    vpor %ymm0, %ymm3, %ymm0
 ; AVX512VLBW-NEXT:    vpmovwb %ymm0, %xmm0
 ; AVX512VLBW-NEXT:    vzeroupper
diff --git a/llvm/test/CodeGen/X86/vector-fshl-rot-256.ll b/llvm/test/CodeGen/X86/vector-fshl-rot-256.ll
index fd0e1c7e2f3ab..7db474fc39da9 100644
--- a/llvm/test/CodeGen/X86/vector-fshl-rot-256.ll
+++ b/llvm/test/CodeGen/X86/vector-fshl-rot-256.ll
@@ -841,34 +841,34 @@ define <32 x i8> @splatvar_funnnel_v32i8(<32 x i8> %x, <32 x i8> %amt) nounwind
 ;
 ; AVX512BW-LABEL: splatvar_funnnel_v32i8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpbroadcastb %xmm1, %ymm1
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; AVX512BW-NEXT:    vpand %ymm2, %ymm1, %ymm3
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero,ymm3[16],zero,ymm3[17],zero,ymm3[18],zero,ymm3[19],zero,ymm3[20],zero,ymm3[21],zero,ymm3[22],zero,ymm3[23],zero,ymm3[24],zero,ymm3[25],zero,ymm3[26],zero,ymm3[27],zero,ymm3[28],zero,ymm3[29],zero,ymm3[30],zero,ymm3[31],zero
+; AVX512BW-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512BW-NEXT:    vpand %xmm2, %xmm1, %xmm3
+; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
-; AVX512BW-NEXT:    vpsllvw %zmm3, %zmm0, %zmm3
+; AVX512BW-NEXT:    vpsllw %xmm3, %zmm0, %zmm3
 ; AVX512BW-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512BW-NEXT:    vpsubb %ymm1, %ymm4, %ymm1
-; AVX512BW-NEXT:    vpand %ymm2, %ymm1, %ymm1
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
-; AVX512BW-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsubb %xmm1, %xmm4, %xmm1
+; AVX512BW-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512BW-NEXT:    vpsrlw %xmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vporq %zmm0, %zmm3, %zmm0
 ; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: splatvar_funnnel_v32i8:
 ; AVX512VLBW:       # %bb.0:
-; AVX512VLBW-NEXT:    vpbroadcastb %xmm1, %ymm1
-; AVX512VLBW-NEXT:    vmovdqa {{.*#+}} ymm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; AVX512VLBW-NEXT:    vpand %ymm2, %ymm1, %ymm3
-; AVX512VLBW-NEXT:    vpmovzxbw {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero,ymm3[16],zero,ymm3[17],zero,ymm3[18],zero,ymm3[19],zero,ymm3[20],zero,ymm3[21],zero,ymm3[22],zero,ymm3[23],zero,ymm3[24],zero,ymm3[25],zero,ymm3[26],zero,ymm3[27],zero,ymm3[28],zero,ymm3[29],zero,ymm3[30],zero,ymm3[31],zero
+; AVX512VLBW-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512VLBW-NEXT:    vmovdqa {{.*#+}} xmm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512VLBW-NEXT:    vpand %xmm2, %xmm1, %xmm3
+; AVX512VLBW-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512VLBW-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
-; AVX512VLBW-NEXT:    vpsllvw %zmm3, %zmm0, %zmm3
+; AVX512VLBW-NEXT:    vpsllw %xmm3, %zmm0, %zmm3
 ; AVX512VLBW-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512VLBW-NEXT:    vpsubb %ymm1, %ymm4, %ymm1
-; AVX512VLBW-NEXT:    vpand %ymm2, %ymm1, %ymm1
-; AVX512VLBW-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
-; AVX512VLBW-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm0
+; AVX512VLBW-NEXT:    vpsubb %xmm1, %xmm4, %xmm1
+; AVX512VLBW-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX512VLBW-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VLBW-NEXT:    vpsrlw %xmm1, %zmm0, %zmm0
 ; AVX512VLBW-NEXT:    vporq %zmm0, %zmm3, %zmm0
 ; AVX512VLBW-NEXT:    vpmovwb %zmm0, %ymm0
 ; AVX512VLBW-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/vector-fshl-rot-512.ll b/llvm/test/CodeGen/X86/vector-fshl-rot-512.ll
index ff177678bed6c..2481b8ebfe25d 100644
--- a/llvm/test/CodeGen/X86/vector-fshl-rot-512.ll
+++ b/llvm/test/CodeGen/X86/vector-fshl-rot-512.ll
@@ -366,50 +366,50 @@ define <16 x i32> @splatvar_funnnel_v16i32(<16 x i32> %x, <16 x i32> %amt) nounw
 define <32 x i16> @splatvar_funnnel_v32i16(<32 x i16> %x, <32 x i16> %amt) nounwind {
 ; AVX512F-LABEL: splatvar_funnnel_v32i16:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpbroadcastw %xmm1, %ymm1
-; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm1, %zmm1
-; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm2
-; AVX512F-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
-; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
-; AVX512F-NEXT:    vpsllw %xmm2, %ymm3, %ymm4
-; AVX512F-NEXT:    vpsllw %xmm2, %ymm0, %ymm2
-; AVX512F-NEXT:    vinserti64x4 $1, %ymm4, %zmm2, %zmm2
-; AVX512F-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512F-NEXT:    vpsubw %ymm1, %ymm4, %ymm1
-; AVX512F-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
-; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
+; AVX512F-NEXT:    vpbroadcastw %xmm1, %xmm1
+; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15]
+; AVX512F-NEXT:    vpand %xmm2, %xmm1, %xmm3
+; AVX512F-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm4
+; AVX512F-NEXT:    vpsllw %xmm3, %ymm4, %ymm5
+; AVX512F-NEXT:    vpsllw %xmm3, %ymm0, %ymm3
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm5, %zmm3, %zmm3
+; AVX512F-NEXT:    vpxor %xmm5, %xmm5, %xmm5
+; AVX512F-NEXT:    vpsubw %xmm1, %xmm5, %xmm1
+; AVX512F-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX512F-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
 ; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
-; AVX512F-NEXT:    vpsrlvd %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vpsrld %xmm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
-; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
-; AVX512F-NEXT:    vpsrlvd %zmm1, %zmm3, %zmm1
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero
+; AVX512F-NEXT:    vpsrld %xmm1, %zmm2, %zmm1
 ; AVX512F-NEXT:    vpmovdw %zmm1, %ymm1
 ; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512F-NEXT:    vporq %zmm0, %zmm2, %zmm0
+; AVX512F-NEXT:    vporq %zmm0, %zmm3, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatvar_funnnel_v32i16:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpbroadcastw %xmm1, %ymm1
-; AVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm1, %zmm1
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm2
-; AVX512VL-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
-; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
-; AVX512VL-NEXT:    vpsllw %xmm2, %ymm3, %ymm4
-; AVX512VL-NEXT:    vpsllw %xmm2, %ymm0, %ymm2
-; AVX512VL-NEXT:    vinserti64x4 $1, %ymm4, %zmm2, %zmm2
-; AVX512VL-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512VL-NEXT:    vpsubw %ymm1, %ymm4, %ymm1
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
-; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
+; AVX512VL-NEXT:    vpbroadcastw %xmm1, %xmm1
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15]
+; AVX512VL-NEXT:    vpand %xmm2, %xmm1, %xmm3
+; AVX512VL-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm4
+; AVX512VL-NEXT:    vpsllw %xmm3, %ymm4, %ymm5
+; AVX512VL-NEXT:    vpsllw %xmm3, %ymm0, %ymm3
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm5, %zmm3, %zmm3
+; AVX512VL-NEXT:    vpxor %xmm5, %xmm5, %xmm5
+; AVX512VL-NEXT:    vpsubw %xmm1, %xmm5, %xmm1
+; AVX512VL-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX512VL-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
 ; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
-; AVX512VL-NEXT:    vpsrlvd %zmm1, %zmm0, %zmm0
+; AVX512VL-NEXT:    vpsrld %xmm1, %zmm0, %zmm0
 ; AVX512VL-NEXT:    vpmovdw %zmm0, %ymm0
-; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
-; AVX512VL-NEXT:    vpsrlvd %zmm1, %zmm3, %zmm1
+; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero
+; AVX512VL-NEXT:    vpsrld %xmm1, %zmm2, %zmm1
 ; AVX512VL-NEXT:    vpmovdw %zmm1, %ymm1
 ; AVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512VL-NEXT:    vporq %zmm0, %zmm2, %zmm0
+; AVX512VL-NEXT:    vporq %zmm0, %zmm3, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatvar_funnnel_v32i16:
diff --git a/llvm/test/CodeGen/X86/vector-fshr-128.ll b/llvm/test/CodeGen/X86/vector-fshr-128.ll
index 817bca051e0ae..db76d3eef282d 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-128.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-128.ll
@@ -1967,18 +1967,18 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpmovzxbd {{.*#+}} zmm3 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
 ; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
-; AVX512F-NEXT:    vpbroadcastb %xmm2, %xmm2
-; AVX512F-NEXT:    vpmovzxbd {{.*#+}} zmm4 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
-; AVX512F-NEXT:    vpsrlvd %zmm4, %zmm3, %zmm3
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512F-NEXT:    vpsubb %xmm2, %xmm4, %xmm4
-; AVX512F-NEXT:    vpmovzxbd {{.*#+}} zmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero,xmm4[4],zero,zero,zero,xmm4[5],zero,zero,zero,xmm4[6],zero,zero,zero,xmm4[7],zero,zero,zero,xmm4[8],zero,zero,zero,xmm4[9],zero,zero,zero,xmm4[10],zero,zero,zero,xmm4[11],zero,zero,zero,xmm4[12],zero,zero,zero,xmm4[13],zero,zero,zero,xmm4[14],zero,zero,zero,xmm4[15],zero,zero,zero
+; AVX512F-NEXT:    vpbroadcastb %xmm2, %xmm4
+; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm5 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512F-NEXT:    vpsrld %xmm5, %zmm3, %zmm3
+; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm5 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512F-NEXT:    vpsubb %xmm2, %xmm5, %xmm2
+; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512F-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
-; AVX512F-NEXT:    vpsllvd %zmm4, %zmm0, %zmm0
+; AVX512F-NEXT:    vpslld %xmm2, %zmm0, %zmm0
 ; AVX512F-NEXT:    vpord %zmm3, %zmm0, %zmm0
 ; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
-; AVX512F-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX512F-NEXT:    vpcmpeqb %xmm3, %xmm2, %xmm2
+; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm4, %xmm2
 ; AVX512F-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
@@ -1987,18 +1987,18 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpmovzxbd {{.*#+}} zmm3 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
 ; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
-; AVX512VL-NEXT:    vpbroadcastb %xmm2, %xmm2
-; AVX512VL-NEXT:    vpmovzxbd {{.*#+}} zmm4 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
-; AVX512VL-NEXT:    vpsrlvd %zmm4, %zmm3, %zmm3
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512VL-NEXT:    vpsubb %xmm2, %xmm4, %xmm4
-; AVX512VL-NEXT:    vpmovzxbd {{.*#+}} zmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero,xmm4[4],zero,zero,zero,xmm4[5],zero,zero,zero,xmm4[6],zero,zero,zero,xmm4[7],zero,zero,zero,xmm4[8],zero,zero,zero,xmm4[9],zero,zero,zero,xmm4[10],zero,zero,zero,xmm4[11],zero,zero,zero,xmm4[12],zero,zero,zero,xmm4[13],zero,zero,zero,xmm4[14],zero,zero,zero,xmm4[15],zero,zero,zero
+; AVX512VL-NEXT:    vpbroadcastb %xmm2, %xmm4
+; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm5 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VL-NEXT:    vpsrld %xmm5, %zmm3, %zmm3
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm5 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VL-NEXT:    vpsubb %xmm2, %xmm5, %xmm2
+; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512VL-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
-; AVX512VL-NEXT:    vpsllvd %zmm4, %zmm0, %zmm0
+; AVX512VL-NEXT:    vpslld %xmm2, %zmm0, %zmm0
 ; AVX512VL-NEXT:    vpord %zmm3, %zmm0, %zmm0
 ; AVX512VL-NEXT:    vpmovdb %zmm0, %xmm0
-; AVX512VL-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX512VL-NEXT:    vpcmpeqb %xmm3, %xmm2, %xmm2
+; AVX512VL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512VL-NEXT:    vpcmpeqb %xmm2, %xmm4, %xmm2
 ; AVX512VL-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
@@ -2008,17 +2008,17 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %
 ; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm3 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
 ; AVX512BW-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
-; AVX512BW-NEXT:    vpbroadcastb %xmm2, %xmm2
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero,xmm2[8],zero,xmm2[9],zero,xmm2[10],zero,xmm2[11],zero,xmm2[12],zero,xmm2[13],zero,xmm2[14],zero,xmm2[15],zero
-; AVX512BW-NEXT:    vpsrlvw %zmm4, %zmm3, %zmm3
+; AVX512BW-NEXT:    vpbroadcastb %xmm2, %xmm4
+; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm5 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512BW-NEXT:    vpsrlw %xmm5, %ymm3, %ymm3
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm5 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512BW-NEXT:    vpsubb %xmm2, %xmm5, %xmm2
+; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512BW-NEXT:    vpsubb %xmm2, %xmm4, %xmm4
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero,xmm4[8],zero,xmm4[9],zero,xmm4[10],zero,xmm4[11],zero,xmm4[12],zero,xmm4[13],zero,xmm4[14],zero,xmm4[15],zero
-; AVX512BW-NEXT:    vpsllvw %zmm4, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsllw %xmm2, %ymm0, %ymm0
 ; AVX512BW-NEXT:    vpor %ymm3, %ymm0, %ymm0
 ; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
-; AVX512BW-NEXT:    vptestnmb %zmm2, %zmm2, %k1
+; AVX512BW-NEXT:    vptestnmb %zmm4, %zmm4, %k1
 ; AVX512BW-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k1}
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512BW-NEXT:    vzeroupper
@@ -2029,17 +2029,17 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %
 ; AVX512VBMI2-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm3 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
 ; AVX512VBMI2-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
-; AVX512VBMI2-NEXT:    vpbroadcastb %xmm2, %xmm2
-; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero,xmm2[8],zero,xmm2[9],zero,xmm2[10],zero,xmm2[11],zero,xmm2[12],zero,xmm2[13],zero,xmm2[14],zero,xmm2[15],zero
-; AVX512VBMI2-NEXT:    vpsrlvw %zmm4, %zmm3, %zmm3
+; AVX512VBMI2-NEXT:    vpbroadcastb %xmm2, %xmm4
+; AVX512VBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm5 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VBMI2-NEXT:    vpsrlw %xmm5, %ymm3, %ymm3
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm5 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VBMI2-NEXT:    vpsubb %xmm2, %xmm5, %xmm2
+; AVX512VBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
-; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512VBMI2-NEXT:    vpsubb %xmm2, %xmm4, %xmm4
-; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero,xmm4[8],zero,xmm4[9],zero,xmm4[10],zero,xmm4[11],zero,xmm4[12],zero,xmm4[13],zero,xmm4[14],zero,xmm4[15],zero
-; AVX512VBMI2-NEXT:    vpsllvw %zmm4, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vpsllw %xmm2, %ymm0, %ymm0
 ; AVX512VBMI2-NEXT:    vpor %ymm3, %ymm0, %ymm0
 ; AVX512VBMI2-NEXT:    vpmovwb %zmm0, %ymm0
-; AVX512VBMI2-NEXT:    vptestnmb %zmm2, %zmm2, %k1
+; AVX512VBMI2-NEXT:    vptestnmb %zmm4, %zmm4, %k1
 ; AVX512VBMI2-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k1}
 ; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512VBMI2-NEXT:    vzeroupper
@@ -2049,17 +2049,17 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpmovzxbw {{.*#+}} ymm3 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
 ; AVX512VLBW-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
-; AVX512VLBW-NEXT:    vpbroadcastb %xmm2, %xmm2
-; AVX512VLBW-NEXT:    vpmovzxbw {{.*#+}} ymm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero,xmm2[8],zero,xmm2[9],zero,xmm2[10],zero,xmm2[11],zero,xmm2[12],zero,xmm2[13],zero,xmm2[14],zero,xmm2[15],zero
-; AVX512VLBW-NEXT:    vpsrlvw %ymm4, %ymm3, %ymm3
-; AVX512VLBW-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512VLBW-NEXT:    vpsubb %xmm2, %xmm4, %xmm4
-; AVX512VLBW-NEXT:    vpmovzxbw {{.*#+}} ymm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero,xmm4[8],zero,xmm4[9],zero,xmm4[10],zero,xmm4[11],zero,xmm4[12],zero,xmm4[13],zero,xmm4[14],zero,xmm4[15],zero
+; AVX512VLBW-NEXT:    vpbroadcastb %xmm2, %xmm4
+; AVX512VLBW-NEXT:    vpmovzxbq {{.*#+}} xmm5 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VLBW-NEXT:    vpsrlw %xmm5, %ymm3, %ymm3
+; AVX512VLBW-NEXT:    vmovdqa {{.*#+}} xmm5 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VLBW-NEXT:    vpsubb %xmm2, %xmm5, %xmm2
+; AVX512VLBW-NEXT:    vpmovzxbq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512VLBW-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
-; AVX512VLBW-NEXT:    vpsllvw %ymm4, %ymm0, %ymm0
+; AVX512VLBW-NEXT:    vpsllw %xmm2, %ymm0, %ymm0
 ; AVX512VLBW-NEXT:    vpor %ymm3, %ymm0, %ymm0
 ; AVX512VLBW-NEXT:    vpmovwb %ymm0, %xmm0
-; AVX512VLBW-NEXT:    vptestnmb %xmm2, %xmm2, %k1
+; AVX512VLBW-NEXT:    vptestnmb %xmm4, %xmm4, %k1
 ; AVX512VLBW-NEXT:    vmovdqu8 %xmm1, %xmm0 {%k1}
 ; AVX512VLBW-NEXT:    vzeroupper
 ; AVX512VLBW-NEXT:    retq
@@ -2068,17 +2068,17 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %
 ; AVX512VLVBMI2:       # %bb.0:
 ; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm3 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
 ; AVX512VLVBMI2-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
-; AVX512VLVBMI2-NEXT:    vpbroadcastb %xmm2, %xmm2
-; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero,xmm2[8],zero,xmm2[9],zero,xmm2[10],zero,xmm2[11],zero,xmm2[12],zero,xmm2[13],zero,xmm2[14],zero,xmm2[15],zero
-; AVX512VLVBMI2-NEXT:    vpsrlvw %ymm4, %ymm3, %ymm3
-; AVX512VLVBMI2-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512VLVBMI2-NEXT:    vpsubb %xmm2, %xmm4, %xmm4
-; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero,xmm4[8],zero,xmm4[9],zero,xmm4[10],zero,xmm4[11],zero,xmm4[12],zero,xmm4[13],zero,xmm4[14],zero,xmm4[15],zero
+; AVX512VLVBMI2-NEXT:    vpbroadcastb %xmm2, %xmm4
+; AVX512VLVBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm5 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VLVBMI2-NEXT:    vpsrlw %xmm5, %ymm3, %ymm3
+; AVX512VLVBMI2-NEXT:    vmovdqa {{.*#+}} xmm5 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VLVBMI2-NEXT:    vpsubb %xmm2, %xmm5, %xmm2
+; AVX512VLVBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
-; AVX512VLVBMI2-NEXT:    vpsllvw %ymm4, %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    vpsllw %xmm2, %ymm0, %ymm0
 ; AVX512VLVBMI2-NEXT:    vpor %ymm3, %ymm0, %ymm0
 ; AVX512VLVBMI2-NEXT:    vpmovwb %ymm0, %xmm0
-; AVX512VLVBMI2-NEXT:    vptestnmb %xmm2, %xmm2, %k1
+; AVX512VLVBMI2-NEXT:    vptestnmb %xmm4, %xmm4, %k1
 ; AVX512VLVBMI2-NEXT:    vmovdqu8 %xmm1, %xmm0 {%k1}
 ; AVX512VLVBMI2-NEXT:    vzeroupper
 ; AVX512VLVBMI2-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/vector-fshr-256.ll b/llvm/test/CodeGen/X86/vector-fshr-256.ll
index f0848cfd2e49a..6449d58fa38c5 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-256.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-256.ll
@@ -1528,13 +1528,13 @@ define <32 x i8> @splatvar_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y, <32 x i8> %
 ; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm3 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
 ; AVX512BW-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
 ; AVX512BW-NEXT:    vpbroadcastb %xmm2, %ymm2
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm4 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero,ymm2[16],zero,ymm2[17],zero,ymm2[18],zero,ymm2[19],zero,ymm2[20],zero,ymm2[21],zero,ymm2[22],zero,ymm2[23],zero,ymm2[24],zero,ymm2[25],zero,ymm2[26],zero,ymm2[27],zero,ymm2[28],zero,ymm2[29],zero,ymm2[30],zero,ymm2[31],zero
-; AVX512BW-NEXT:    vpsrlvw %zmm4, %zmm3, %zmm3
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512BW-NEXT:    vpsubb %ymm2, %ymm4, %ymm4
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm4 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero,ymm4[16],zero,ymm4[17],zero,ymm4[18],zero,ymm4[19],zero,ymm4[20],zero,ymm4[21],zero,ymm4[22],zero,ymm4[23],zero,ymm4[24],zero,ymm4[25],zero,ymm4[26],zero,ymm4[27],zero,ymm4[28],zero,ymm4[29],zero,ymm4[30],zero,ymm4[31],zero
+; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512BW-NEXT:    vpsrlw %xmm4, %zmm3, %zmm3
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512BW-NEXT:    vpsubb %xmm2, %xmm4, %xmm4
+; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
-; AVX512BW-NEXT:    vpsllvw %zmm4, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsllw %xmm4, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vporq %zmm3, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
 ; AVX512BW-NEXT:    vptestnmb %zmm2, %zmm2, %k1
@@ -1548,13 +1548,13 @@ define <32 x i8> @splatvar_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y, <32 x i8> %
 ; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm3 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
 ; AVX512VBMI2-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
 ; AVX512VBMI2-NEXT:    vpbroadcastb %xmm2, %ymm2
-; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm4 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero,ymm2[16],zero,ymm2[17],zero,ymm2[18],zero,ymm2[19],zero,ymm2[20],zero,ymm2[21],zero,ymm2[22],zero,ymm2[23],zero,ymm2[24],zero,ymm2[25],zero,ymm2[26],zero,ymm2[27],zero,ymm2[28],zero,ymm2[29],zero,ymm2[30],zero,ymm2[31],zero
-; AVX512VBMI2-NEXT:    vpsrlvw %zmm4, %zmm3, %zmm3
-; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} ymm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512VBMI2-NEXT:    vpsubb %ymm2, %ymm4, %ymm4
-; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm4 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero,ymm4[16],zero,ymm4[17],zero,ymm4[18],zero,ymm4[19],zero,ymm4[20],zero,ymm4[21],zero,ymm4[22],zero,ymm4[23],zero,ymm4[24],zero,ymm4[25],zero,ymm4[26],zero,ymm4[27],zero,ymm4[28],zero,ymm4[29],zero,ymm4[30],zero,ymm4[31],zero
+; AVX512VBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VBMI2-NEXT:    vpsrlw %xmm4, %zmm3, %zmm3
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VBMI2-NEXT:    vpsubb %xmm2, %xmm4, %xmm4
+; AVX512VBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
-; AVX512VBMI2-NEXT:    vpsllvw %zmm4, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vpsllw %xmm4, %zmm0, %zmm0
 ; AVX512VBMI2-NEXT:    vporq %zmm3, %zmm0, %zmm0
 ; AVX512VBMI2-NEXT:    vpmovwb %zmm0, %ymm0
 ; AVX512VBMI2-NEXT:    vptestnmb %zmm2, %zmm2, %k1
@@ -1567,13 +1567,13 @@ define <32 x i8> @splatvar_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y, <32 x i8> %
 ; AVX512VLBW-NEXT:    vpmovzxbw {{.*#+}} zmm3 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
 ; AVX512VLBW-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
 ; AVX512VLBW-NEXT:    vpbroadcastb %xmm2, %ymm2
-; AVX512VLBW-NEXT:    vpmovzxbw {{.*#+}} zmm4 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero,ymm2[16],zero,ymm2[17],zero,ymm2[18],zero,ymm2[19],zero,ymm2[20],zero,ymm2[21],zero,ymm2[22],zero,ymm2[23],zero,ymm2[24],zero,ymm2[25],zero,ymm2[26],zero,ymm2[27],zero,ymm2[28],zero,ymm2[29],zero,ymm2[30],zero,ymm2[31],zero
-; AVX512VLBW-NEXT:    vpsrlvw %zmm4, %zmm3, %zmm3
-; AVX512VLBW-NEXT:    vmovdqa {{.*#+}} ymm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512VLBW-NEXT:    vpsubb %ymm2, %ymm4, %ymm4
-; AVX512VLBW-NEXT:    vpmovzxbw {{.*#+}} zmm4 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero,ymm4[16],zero,ymm4[17],zero,ymm4[18],zero,ymm4[19],zero,ymm4[20],zero,ymm4[21],zero,ymm4[22],zero,ymm4[23],zero,ymm4[24],zero,ymm4[25],zero,ymm4[26],zero,ymm4[27],zero,ymm4[28],zero,ymm4[29],zero,ymm4[30],zero,ymm4[31],zero
+; AVX512VLBW-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VLBW-NEXT:    vpsrlw %xmm4, %zmm3, %zmm3
+; AVX512VLBW-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VLBW-NEXT:    vpsubb %xmm2, %xmm4, %xmm4
+; AVX512VLBW-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512VLBW-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
-; AVX512VLBW-NEXT:    vpsllvw %zmm4, %zmm0, %zmm0
+; AVX512VLBW-NEXT:    vpsllw %xmm4, %zmm0, %zmm0
 ; AVX512VLBW-NEXT:    vporq %zmm3, %zmm0, %zmm0
 ; AVX512VLBW-NEXT:    vpmovwb %zmm0, %ymm0
 ; AVX512VLBW-NEXT:    vptestnmb %ymm2, %ymm2, %k1
@@ -1585,13 +1585,13 @@ define <32 x i8> @splatvar_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y, <32 x i8> %
 ; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm3 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
 ; AVX512VLVBMI2-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
 ; AVX512VLVBMI2-NEXT:    vpbroadcastb %xmm2, %ymm2
-; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm4 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero,ymm2[16],zero,ymm2[17],zero,ymm2[18],zero,ymm2[19],zero,ymm2[20],zero,ymm2[21],zero,ymm2[22],zero,ymm2[23],zero,ymm2[24],zero,ymm2[25],zero,ymm2[26],zero,ymm2[27],zero,ymm2[28],zero,ymm2[29],zero,ymm2[30],zero,ymm2[31],zero
-; AVX512VLVBMI2-NEXT:    vpsrlvw %zmm4, %zmm3, %zmm3
-; AVX512VLVBMI2-NEXT:    vmovdqa {{.*#+}} ymm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512VLVBMI2-NEXT:    vpsubb %ymm2, %ymm4, %ymm4
-; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm4 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero,ymm4[16],zero,ymm4[17],zero,ymm4[18],zero,ymm4[19],zero,ymm4[20],zero,ymm4[21],zero,ymm4[22],zero,ymm4[23],zero,ymm4[24],zero,ymm4[25],zero,ymm4[26],zero,ymm4[27],zero,ymm4[28],zero,ymm4[29],zero,ymm4[30],zero,ymm4[31],zero
+; AVX512VLVBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VLVBMI2-NEXT:    vpsrlw %xmm4, %zmm3, %zmm3
+; AVX512VLVBMI2-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VLVBMI2-NEXT:    vpsubb %xmm2, %xmm4, %xmm4
+; AVX512VLVBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
-; AVX512VLVBMI2-NEXT:    vpsllvw %zmm4, %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    vpsllw %xmm4, %zmm0, %zmm0
 ; AVX512VLVBMI2-NEXT:    vporq %zmm3, %zmm0, %zmm0
 ; AVX512VLVBMI2-NEXT:    vpmovwb %zmm0, %ymm0
 ; AVX512VLVBMI2-NEXT:    vptestnmb %ymm2, %ymm2, %k1
diff --git a/llvm/test/CodeGen/X86/vector-fshr-rot-128.ll b/llvm/test/CodeGen/X86/vector-fshr-rot-128.ll
index 8fe7ba9e471a9..6cda97e1ebd39 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-rot-128.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-rot-128.ll
@@ -1132,17 +1132,16 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %amt) nounwind
 ;
 ; AVX512F-LABEL: splatvar_funnnel_v16i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpbroadcastb %xmm1, %xmm1
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
 ; AVX512F-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX512F-NEXT:    vpmovzxbd {{.*#+}} zmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero,xmm3[4],zero,zero,zero,xmm3[5],zero,zero,zero,xmm3[6],zero,zero,zero,xmm3[7],zero,zero,zero,xmm3[8],zero,zero,zero,xmm3[9],zero,zero,zero,xmm3[10],zero,zero,zero,xmm3[11],zero,zero,zero,xmm3[12],zero,zero,zero,xmm3[13],zero,zero,zero,xmm3[14],zero,zero,zero,xmm3[15],zero,zero,zero
+; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512F-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
-; AVX512F-NEXT:    vpsrlvd %zmm3, %zmm0, %zmm3
+; AVX512F-NEXT:    vpsrld %xmm3, %zmm0, %zmm3
 ; AVX512F-NEXT:    vpxor %xmm4, %xmm4, %xmm4
 ; AVX512F-NEXT:    vpsubb %xmm1, %xmm4, %xmm1
 ; AVX512F-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; AVX512F-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
-; AVX512F-NEXT:    vpsllvd %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512F-NEXT:    vpslld %xmm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    vpord %zmm3, %zmm0, %zmm0
 ; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
 ; AVX512F-NEXT:    vzeroupper
@@ -1150,17 +1149,16 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %amt) nounwind
 ;
 ; AVX512VL-LABEL: splatvar_funnnel_v16i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpbroadcastb %xmm1, %xmm1
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
 ; AVX512VL-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX512VL-NEXT:    vpmovzxbd {{.*#+}} zmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero,xmm3[4],zero,zero,zero,xmm3[5],zero,zero,zero,xmm3[6],zero,zero,zero,xmm3[7],zero,zero,zero,xmm3[8],zero,zero,zero,xmm3[9],zero,zero,zero,xmm3[10],zero,zero,zero,xmm3[11],zero,zero,zero,xmm3[12],zero,zero,zero,xmm3[13],zero,zero,zero,xmm3[14],zero,zero,zero,xmm3[15],zero,zero,zero
+; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512VL-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
-; AVX512VL-NEXT:    vpsrlvd %zmm3, %zmm0, %zmm3
+; AVX512VL-NEXT:    vpsrld %xmm3, %zmm0, %zmm3
 ; AVX512VL-NEXT:    vpxor %xmm4, %xmm4, %xmm4
 ; AVX512VL-NEXT:    vpsubb %xmm1, %xmm4, %xmm1
 ; AVX512VL-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
-; AVX512VL-NEXT:    vpsllvd %zmm1, %zmm0, %zmm0
+; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VL-NEXT:    vpslld %xmm1, %zmm0, %zmm0
 ; AVX512VL-NEXT:    vpord %zmm3, %zmm0, %zmm0
 ; AVX512VL-NEXT:    vpmovdb %zmm0, %xmm0
 ; AVX512VL-NEXT:    vzeroupper
@@ -1168,17 +1166,16 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %amt) nounwind
 ;
 ; AVX512BW-LABEL: splatvar_funnnel_v16i8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpbroadcastb %xmm1, %xmm1
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
 ; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
 ; AVX512BW-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero,xmm3[8],zero,xmm3[9],zero,xmm3[10],zero,xmm3[11],zero,xmm3[12],zero,xmm3[13],zero,xmm3[14],zero,xmm3[15],zero
-; AVX512BW-NEXT:    vpsrlvw %zmm3, %zmm0, %zmm3
+; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512BW-NEXT:    vpsrlw %xmm3, %ymm0, %ymm3
 ; AVX512BW-NEXT:    vpxor %xmm4, %xmm4, %xmm4
 ; AVX512BW-NEXT:    vpsubb %xmm1, %xmm4, %xmm1
 ; AVX512BW-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512BW-NEXT:    vpsllvw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512BW-NEXT:    vpsllw %xmm1, %ymm0, %ymm0
 ; AVX512BW-NEXT:    vpor %ymm3, %ymm0, %ymm0
 ; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
@@ -1187,17 +1184,16 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %amt) nounwind
 ;
 ; AVX512VLBW-LABEL: splatvar_funnnel_v16i8:
 ; AVX512VLBW:       # %bb.0:
-; AVX512VLBW-NEXT:    vpbroadcastb %xmm1, %xmm1
 ; AVX512VLBW-NEXT:    vmovdqa {{.*#+}} xmm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
 ; AVX512VLBW-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX512VLBW-NEXT:    vpmovzxbw {{.*#+}} ymm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero,xmm3[8],zero,xmm3[9],zero,xmm3[10],zero,xmm3[11],zero,xmm3[12],zero,xmm3[13],zero,xmm3[14],zero,xmm3[15],zero
+; AVX512VLBW-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512VLBW-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
-; AVX512VLBW-NEXT:    vpsrlvw %ymm3, %ymm0, %ymm3
+; AVX512VLBW-NEXT:    vpsrlw %xmm3, %ymm0, %ymm3
 ; AVX512VLBW-NEXT:    vpxor %xmm4, %xmm4, %xmm4
 ; AVX512VLBW-NEXT:    vpsubb %xmm1, %xmm4, %xmm1
 ; AVX512VLBW-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; AVX512VLBW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512VLBW-NEXT:    vpsllvw %ymm1, %ymm0, %ymm0
+; AVX512VLBW-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VLBW-NEXT:    vpsllw %xmm1, %ymm0, %ymm0
 ; AVX512VLBW-NEXT:    vpor %ymm3, %ymm0, %ymm0
 ; AVX512VLBW-NEXT:    vpmovwb %ymm0, %xmm0
 ; AVX512VLBW-NEXT:    vzeroupper
diff --git a/llvm/test/CodeGen/X86/vector-fshr-rot-256.ll b/llvm/test/CodeGen/X86/vector-fshr-rot-256.ll
index 4e92bfc4f9136..fffa286b79196 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-rot-256.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-rot-256.ll
@@ -916,34 +916,34 @@ define <32 x i8> @splatvar_funnnel_v32i8(<32 x i8> %x, <32 x i8> %amt) nounwind
 ;
 ; AVX512BW-LABEL: splatvar_funnnel_v32i8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpbroadcastb %xmm1, %ymm1
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; AVX512BW-NEXT:    vpand %ymm2, %ymm1, %ymm3
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero,ymm3[16],zero,ymm3[17],zero,ymm3[18],zero,ymm3[19],zero,ymm3[20],zero,ymm3[21],zero,ymm3[22],zero,ymm3[23],zero,ymm3[24],zero,ymm3[25],zero,ymm3[26],zero,ymm3[27],zero,ymm3[28],zero,ymm3[29],zero,ymm3[30],zero,ymm3[31],zero
+; AVX512BW-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512BW-NEXT:    vpand %xmm2, %xmm1, %xmm3
+; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
-; AVX512BW-NEXT:    vpsrlvw %zmm3, %zmm0, %zmm3
+; AVX512BW-NEXT:    vpsrlw %xmm3, %zmm0, %zmm3
 ; AVX512BW-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512BW-NEXT:    vpsubb %ymm1, %ymm4, %ymm1
-; AVX512BW-NEXT:    vpand %ymm2, %ymm1, %ymm1
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
-; AVX512BW-NEXT:    vpsllvw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsubb %xmm1, %xmm4, %xmm1
+; AVX512BW-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512BW-NEXT:    vpsllw %xmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vporq %zmm3, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: splatvar_funnnel_v32i8:
 ; AVX512VLBW:       # %bb.0:
-; AVX512VLBW-NEXT:    vpbroadcastb %xmm1, %ymm1
-; AVX512VLBW-NEXT:    vmovdqa {{.*#+}} ymm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; AVX512VLBW-NEXT:    vpand %ymm2, %ymm1, %ymm3
-; AVX512VLBW-NEXT:    vpmovzxbw {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero,ymm3[16],zero,ymm3[17],zero,ymm3[18],zero,ymm3[19],zero,ymm3[20],zero,ymm3[21],zero,ymm3[22],zero,ymm3[23],zero,ymm3[24],zero,ymm3[25],zero,ymm3[26],zero,ymm3[27],zero,ymm3[28],zero,ymm3[29],zero,ymm3[30],zero,ymm3[31],zero
+; AVX512VLBW-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512VLBW-NEXT:    vmovdqa {{.*#+}} xmm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512VLBW-NEXT:    vpand %xmm2, %xmm1, %xmm3
+; AVX512VLBW-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512VLBW-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
-; AVX512VLBW-NEXT:    vpsrlvw %zmm3, %zmm0, %zmm3
+; AVX512VLBW-NEXT:    vpsrlw %xmm3, %zmm0, %zmm3
 ; AVX512VLBW-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512VLBW-NEXT:    vpsubb %ymm1, %ymm4, %ymm1
-; AVX512VLBW-NEXT:    vpand %ymm2, %ymm1, %ymm1
-; AVX512VLBW-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
-; AVX512VLBW-NEXT:    vpsllvw %zmm1, %zmm0, %zmm0
+; AVX512VLBW-NEXT:    vpsubb %xmm1, %xmm4, %xmm1
+; AVX512VLBW-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX512VLBW-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VLBW-NEXT:    vpsllw %xmm1, %zmm0, %zmm0
 ; AVX512VLBW-NEXT:    vporq %zmm3, %zmm0, %zmm0
 ; AVX512VLBW-NEXT:    vpmovwb %zmm0, %ymm0
 ; AVX512VLBW-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/vector-fshr-rot-512.ll b/llvm/test/CodeGen/X86/vector-fshr-rot-512.ll
index fda0dacedf85b..04a883171a7ce 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-rot-512.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-rot-512.ll
@@ -366,50 +366,50 @@ define <16 x i32> @splatvar_funnnel_v16i32(<16 x i32> %x, <16 x i32> %amt) nounw
 define <32 x i16> @splatvar_funnnel_v32i16(<32 x i16> %x, <32 x i16> %amt) nounwind {
 ; AVX512F-LABEL: splatvar_funnnel_v32i16:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpbroadcastw %xmm1, %ymm1
-; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm1, %zmm1
-; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm2
-; AVX512F-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
-; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
-; AVX512F-NEXT:    vpsrlw %xmm2, %ymm3, %ymm4
-; AVX512F-NEXT:    vpsrlw %xmm2, %ymm0, %ymm2
-; AVX512F-NEXT:    vinserti64x4 $1, %ymm4, %zmm2, %zmm2
-; AVX512F-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512F-NEXT:    vpsubw %ymm1, %ymm4, %ymm1
-; AVX512F-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
-; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
+; AVX512F-NEXT:    vpbroadcastw %xmm1, %xmm1
+; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15]
+; AVX512F-NEXT:    vpand %xmm2, %xmm1, %xmm3
+; AVX512F-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm4
+; AVX512F-NEXT:    vpsrlw %xmm3, %ymm4, %ymm5
+; AVX512F-NEXT:    vpsrlw %xmm3, %ymm0, %ymm3
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm5, %zmm3, %zmm3
+; AVX512F-NEXT:    vpxor %xmm5, %xmm5, %xmm5
+; AVX512F-NEXT:    vpsubw %xmm1, %xmm5, %xmm1
+; AVX512F-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX512F-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
 ; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
-; AVX512F-NEXT:    vpsllvd %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vpslld %xmm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
-; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
-; AVX512F-NEXT:    vpsllvd %zmm1, %zmm3, %zmm1
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero
+; AVX512F-NEXT:    vpslld %xmm1, %zmm2, %zmm1
 ; AVX512F-NEXT:    vpmovdw %zmm1, %ymm1
 ; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512F-NEXT:    vporq %zmm2, %zmm0, %zmm0
+; AVX512F-NEXT:    vporq %zmm3, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatvar_funnnel_v32i16:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpbroadcastw %xmm1, %ymm1
-; AVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm1, %zmm1
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm2
-; AVX512VL-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
-; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
-; AVX512VL-NEXT:    vpsrlw %xmm2, %ymm3, %ymm4
-; AVX512VL-NEXT:    vpsrlw %xmm2, %ymm0, %ymm2
-; AVX512VL-NEXT:    vinserti64x4 $1, %ymm4, %zmm2, %zmm2
-; AVX512VL-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512VL-NEXT:    vpsubw %ymm1, %ymm4, %ymm1
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
-; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
+; AVX512VL-NEXT:    vpbroadcastw %xmm1, %xmm1
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15]
+; AVX512VL-NEXT:    vpand %xmm2, %xmm1, %xmm3
+; AVX512VL-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm4
+; AVX512VL-NEXT:    vpsrlw %xmm3, %ymm4, %ymm5
+; AVX512VL-NEXT:    vpsrlw %xmm3, %ymm0, %ymm3
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm5, %zmm3, %zmm3
+; AVX512VL-NEXT:    vpxor %xmm5, %xmm5, %xmm5
+; AVX512VL-NEXT:    vpsubw %xmm1, %xmm5, %xmm1
+; AVX512VL-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX512VL-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
 ; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
-; AVX512VL-NEXT:    vpsllvd %zmm1, %zmm0, %zmm0
+; AVX512VL-NEXT:    vpslld %xmm1, %zmm0, %zmm0
 ; AVX512VL-NEXT:    vpmovdw %zmm0, %ymm0
-; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
-; AVX512VL-NEXT:    vpsllvd %zmm1, %zmm3, %zmm1
+; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero
+; AVX512VL-NEXT:    vpslld %xmm1, %zmm2, %zmm1
 ; AVX512VL-NEXT:    vpmovdw %zmm1, %ymm1
 ; AVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512VL-NEXT:    vporq %zmm2, %zmm0, %zmm0
+; AVX512VL-NEXT:    vporq %zmm3, %zmm0, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatvar_funnnel_v32i16:
diff --git a/llvm/test/CodeGen/X86/vector-rotate-128.ll b/llvm/test/CodeGen/X86/vector-rotate-128.ll
index d140fb5c09295..6f11c5cbd7567 100644
--- a/llvm/test/CodeGen/X86/vector-rotate-128.ll
+++ b/llvm/test/CodeGen/X86/vector-rotate-128.ll
@@ -1032,14 +1032,13 @@ define <16 x i8> @splatvar_rotate_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
 ;
 ; AVX512F-LABEL: splatvar_rotate_v16i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpbroadcastb %xmm1, %xmm1
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm2 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
 ; AVX512F-NEXT:    vpsubb %xmm1, %xmm2, %xmm2
+; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512F-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
-; AVX512F-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
-; AVX512F-NEXT:    vpsllvd %zmm1, %zmm0, %zmm1
-; AVX512F-NEXT:    vpmovzxbd {{.*#+}} zmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
-; AVX512F-NEXT:    vpsrlvd %zmm2, %zmm0, %zmm0
+; AVX512F-NEXT:    vpslld %xmm1, %zmm0, %zmm1
+; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512F-NEXT:    vpsrld %xmm2, %zmm0, %zmm0
 ; AVX512F-NEXT:    vpord %zmm0, %zmm1, %zmm0
 ; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
 ; AVX512F-NEXT:    vzeroupper
@@ -1047,14 +1046,13 @@ define <16 x i8> @splatvar_rotate_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
 ;
 ; AVX512VL-LABEL: splatvar_rotate_v16i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpbroadcastb %xmm1, %xmm1
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm2 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
 ; AVX512VL-NEXT:    vpsubb %xmm1, %xmm2, %xmm2
+; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512VL-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
-; AVX512VL-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
-; AVX512VL-NEXT:    vpsllvd %zmm1, %zmm0, %zmm1
-; AVX512VL-NEXT:    vpmovzxbd {{.*#+}} zmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
-; AVX512VL-NEXT:    vpsrlvd %zmm2, %zmm0, %zmm0
+; AVX512VL-NEXT:    vpslld %xmm1, %zmm0, %zmm1
+; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VL-NEXT:    vpsrld %xmm2, %zmm0, %zmm0
 ; AVX512VL-NEXT:    vpord %zmm0, %zmm1, %zmm0
 ; AVX512VL-NEXT:    vpmovdb %zmm0, %xmm0
 ; AVX512VL-NEXT:    vzeroupper
@@ -1062,14 +1060,13 @@ define <16 x i8> @splatvar_rotate_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
 ;
 ; AVX512BW-LABEL: splatvar_rotate_v16i8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpbroadcastb %xmm1, %xmm1
 ; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm2 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
 ; AVX512BW-NEXT:    vpsubb %xmm1, %xmm2, %xmm2
+; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512BW-NEXT:    vpsllvw %zmm1, %zmm0, %zmm1
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero,xmm2[8],zero,xmm2[9],zero,xmm2[10],zero,xmm2[11],zero,xmm2[12],zero,xmm2[13],zero,xmm2[14],zero,xmm2[15],zero
-; AVX512BW-NEXT:    vpsrlvw %zmm2, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsllw %xmm1, %ymm0, %ymm1
+; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512BW-NEXT:    vpsrlw %xmm2, %ymm0, %ymm0
 ; AVX512BW-NEXT:    vpor %ymm0, %ymm1, %ymm0
 ; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
@@ -1078,14 +1075,13 @@ define <16 x i8> @splatvar_rotate_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
 ;
 ; AVX512VLBW-LABEL: splatvar_rotate_v16i8:
 ; AVX512VLBW:       # %bb.0:
-; AVX512VLBW-NEXT:    vpbroadcastb %xmm1, %xmm1
 ; AVX512VLBW-NEXT:    vmovdqa {{.*#+}} xmm2 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
 ; AVX512VLBW-NEXT:    vpsubb %xmm1, %xmm2, %xmm2
+; AVX512VLBW-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512VLBW-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
-; AVX512VLBW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512VLBW-NEXT:    vpsllvw %ymm1, %ymm0, %ymm1
-; AVX512VLBW-NEXT:    vpmovzxbw {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero,xmm2[8],zero,xmm2[9],zero,xmm2[10],zero,xmm2[11],zero,xmm2[12],zero,xmm2[13],zero,xmm2[14],zero,xmm2[15],zero
-; AVX512VLBW-NEXT:    vpsrlvw %ymm2, %ymm0, %ymm0
+; AVX512VLBW-NEXT:    vpsllw %xmm1, %ymm0, %ymm1
+; AVX512VLBW-NEXT:    vpmovzxbq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VLBW-NEXT:    vpsrlw %xmm2, %ymm0, %ymm0
 ; AVX512VLBW-NEXT:    vpor %ymm0, %ymm1, %ymm0
 ; AVX512VLBW-NEXT:    vpmovwb %ymm0, %xmm0
 ; AVX512VLBW-NEXT:    vzeroupper
diff --git a/llvm/test/CodeGen/X86/vector-rotate-256.ll b/llvm/test/CodeGen/X86/vector-rotate-256.ll
index a850ab5ba7822..f54aea8d86ddf 100644
--- a/llvm/test/CodeGen/X86/vector-rotate-256.ll
+++ b/llvm/test/CodeGen/X86/vector-rotate-256.ll
@@ -831,29 +831,29 @@ define <32 x i8> @splatvar_rotate_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
 ;
 ; AVX512BW-LABEL: splatvar_rotate_v32i8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpbroadcastb %xmm1, %ymm1
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm2 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512BW-NEXT:    vpsubb %ymm1, %ymm2, %ymm2
 ; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
-; AVX512BW-NEXT:    vpsllvw %zmm1, %zmm0, %zmm1
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero,ymm2[16],zero,ymm2[17],zero,ymm2[18],zero,ymm2[19],zero,ymm2[20],zero,ymm2[21],zero,ymm2[22],zero,ymm2[23],zero,ymm2[24],zero,ymm2[25],zero,ymm2[26],zero,ymm2[27],zero,ymm2[28],zero,ymm2[29],zero,ymm2[30],zero,ymm2[31],zero
-; AVX512BW-NEXT:    vpsrlvw %zmm2, %zmm0, %zmm0
-; AVX512BW-NEXT:    vporq %zmm0, %zmm1, %zmm0
+; AVX512BW-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512BW-NEXT:    vpsllw %xmm2, %zmm0, %zmm2
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512BW-NEXT:    vpsubb %xmm1, %xmm3, %xmm1
+; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512BW-NEXT:    vpsrlw %xmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vporq %zmm0, %zmm2, %zmm0
 ; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: splatvar_rotate_v32i8:
 ; AVX512VLBW:       # %bb.0:
-; AVX512VLBW-NEXT:    vpbroadcastb %xmm1, %ymm1
-; AVX512VLBW-NEXT:    vmovdqa {{.*#+}} ymm2 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512VLBW-NEXT:    vpsubb %ymm1, %ymm2, %ymm2
 ; AVX512VLBW-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
-; AVX512VLBW-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
-; AVX512VLBW-NEXT:    vpsllvw %zmm1, %zmm0, %zmm1
-; AVX512VLBW-NEXT:    vpmovzxbw {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero,ymm2[16],zero,ymm2[17],zero,ymm2[18],zero,ymm2[19],zero,ymm2[20],zero,ymm2[21],zero,ymm2[22],zero,ymm2[23],zero,ymm2[24],zero,ymm2[25],zero,ymm2[26],zero,ymm2[27],zero,ymm2[28],zero,ymm2[29],zero,ymm2[30],zero,ymm2[31],zero
-; AVX512VLBW-NEXT:    vpsrlvw %zmm2, %zmm0, %zmm0
-; AVX512VLBW-NEXT:    vporq %zmm0, %zmm1, %zmm0
+; AVX512VLBW-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512VLBW-NEXT:    vpmovzxbq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VLBW-NEXT:    vpsllw %xmm2, %zmm0, %zmm2
+; AVX512VLBW-NEXT:    vmovdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VLBW-NEXT:    vpsubb %xmm1, %xmm3, %xmm1
+; AVX512VLBW-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VLBW-NEXT:    vpsrlw %xmm1, %zmm0, %zmm0
+; AVX512VLBW-NEXT:    vporq %zmm0, %zmm2, %zmm0
 ; AVX512VLBW-NEXT:    vpmovwb %zmm0, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/vector-shift-ashr-128.ll b/llvm/test/CodeGen/X86/vector-shift-ashr-128.ll
index 9b1fb29cb0297..b4bcca54b1f73 100644
--- a/llvm/test/CodeGen/X86/vector-shift-ashr-128.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-ashr-128.ll
@@ -863,20 +863,18 @@ define <16 x i8> @splatvar_shift_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
 ;
 ; AVX512DQ-LABEL: splatvar_shift_v16i8:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512DQ-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512DQ-NEXT:    vpmovsxbd %xmm0, %zmm0
-; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
-; AVX512DQ-NEXT:    vpsravd %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpsrad %xmm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    vpmovdb %zmm0, %xmm0
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatvar_shift_v16i8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512BW-NEXT:    vpmovsxbw %xmm0, %ymm0
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512BW-NEXT:    vpsravw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsraw %xmm1, %ymm0, %ymm0
 ; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
 ; AVX512BW-NEXT:    vzeroupper
@@ -884,20 +882,18 @@ define <16 x i8> @splatvar_shift_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
 ;
 ; AVX512DQVL-LABEL: splatvar_shift_v16i8:
 ; AVX512DQVL:       # %bb.0:
-; AVX512DQVL-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512DQVL-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512DQVL-NEXT:    vpmovsxbd %xmm0, %zmm0
-; AVX512DQVL-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
-; AVX512DQVL-NEXT:    vpsravd %zmm1, %zmm0, %zmm0
+; AVX512DQVL-NEXT:    vpsrad %xmm1, %zmm0, %zmm0
 ; AVX512DQVL-NEXT:    vpmovdb %zmm0, %xmm0
 ; AVX512DQVL-NEXT:    vzeroupper
 ; AVX512DQVL-NEXT:    retq
 ;
 ; AVX512BWVL-LABEL: splatvar_shift_v16i8:
 ; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512BWVL-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512BWVL-NEXT:    vpmovsxbw %xmm0, %ymm0
-; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512BWVL-NEXT:    vpsravw %ymm1, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpsraw %xmm1, %ymm0, %ymm0
 ; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm0
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/vector-shift-ashr-256.ll b/llvm/test/CodeGen/X86/vector-shift-ashr-256.ll
index ae9c375eec254..141b68db31173 100644
--- a/llvm/test/CodeGen/X86/vector-shift-ashr-256.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-ashr-256.ll
@@ -933,10 +933,9 @@ define <32 x i8> @splatvar_shift_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
 ;
 ; AVX512BW-LABEL: splatvar_shift_v32i8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpbroadcastb %xmm1, %ymm1
 ; AVX512BW-NEXT:    vpmovsxbw %ymm0, %zmm0
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
-; AVX512BW-NEXT:    vpsravw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512BW-NEXT:    vpsraw %xmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
 ; AVX512BW-NEXT:    retq
 ;
@@ -956,10 +955,9 @@ define <32 x i8> @splatvar_shift_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
 ;
 ; AVX512BWVL-LABEL: splatvar_shift_v32i8:
 ; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vpbroadcastb %xmm1, %ymm1
 ; AVX512BWVL-NEXT:    vpmovsxbw %ymm0, %zmm0
-; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
-; AVX512BWVL-NEXT:    vpsravw %zmm1, %zmm0, %zmm0
+; AVX512BWVL-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512BWVL-NEXT:    vpsraw %xmm1, %zmm0, %zmm0
 ; AVX512BWVL-NEXT:    vpmovwb %zmm0, %ymm0
 ; AVX512BWVL-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/vector-shift-ashr-sub128.ll b/llvm/test/CodeGen/X86/vector-shift-ashr-sub128.ll
index a994d6610d73d..bac8f615116e7 100644
--- a/llvm/test/CodeGen/X86/vector-shift-ashr-sub128.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-ashr-sub128.ll
@@ -1385,20 +1385,18 @@ define <8 x i8> @splatvar_shift_v8i8(<8 x i8> %a, <8 x i8> %b) nounwind {
 ;
 ; AVX512DQ-LABEL: splatvar_shift_v8i8:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512DQ-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512DQ-NEXT:    vpmovsxbd %xmm0, %zmm0
-; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
-; AVX512DQ-NEXT:    vpsravd %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpsrad %xmm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    vpmovdb %zmm0, %xmm0
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatvar_shift_v8i8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512BW-NEXT:    vpmovsxbw %xmm0, %ymm0
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512BW-NEXT:    vpsravw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsraw %xmm1, %ymm0, %ymm0
 ; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
 ; AVX512BW-NEXT:    vzeroupper
@@ -1406,20 +1404,18 @@ define <8 x i8> @splatvar_shift_v8i8(<8 x i8> %a, <8 x i8> %b) nounwind {
 ;
 ; AVX512DQVL-LABEL: splatvar_shift_v8i8:
 ; AVX512DQVL:       # %bb.0:
-; AVX512DQVL-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512DQVL-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512DQVL-NEXT:    vpmovsxbd %xmm0, %zmm0
-; AVX512DQVL-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
-; AVX512DQVL-NEXT:    vpsravd %zmm1, %zmm0, %zmm0
+; AVX512DQVL-NEXT:    vpsrad %xmm1, %zmm0, %zmm0
 ; AVX512DQVL-NEXT:    vpmovdb %zmm0, %xmm0
 ; AVX512DQVL-NEXT:    vzeroupper
 ; AVX512DQVL-NEXT:    retq
 ;
 ; AVX512BWVL-LABEL: splatvar_shift_v8i8:
 ; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512BWVL-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512BWVL-NEXT:    vpmovsxbw %xmm0, %ymm0
-; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512BWVL-NEXT:    vpsravw %ymm1, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpsraw %xmm1, %ymm0, %ymm0
 ; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm0
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
@@ -1527,20 +1523,18 @@ define <4 x i8> @splatvar_shift_v4i8(<4 x i8> %a, <4 x i8> %b) nounwind {
 ;
 ; AVX512DQ-LABEL: splatvar_shift_v4i8:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512DQ-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512DQ-NEXT:    vpmovsxbd %xmm0, %zmm0
-; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
-; AVX512DQ-NEXT:    vpsravd %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpsrad %xmm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    vpmovdb %zmm0, %xmm0
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatvar_shift_v4i8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512BW-NEXT:    vpmovsxbw %xmm0, %ymm0
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512BW-NEXT:    vpsravw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsraw %xmm1, %ymm0, %ymm0
 ; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
 ; AVX512BW-NEXT:    vzeroupper
@@ -1548,20 +1542,18 @@ define <4 x i8> @splatvar_shift_v4i8(<4 x i8> %a, <4 x i8> %b) nounwind {
 ;
 ; AVX512DQVL-LABEL: splatvar_shift_v4i8:
 ; AVX512DQVL:       # %bb.0:
-; AVX512DQVL-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512DQVL-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512DQVL-NEXT:    vpmovsxbd %xmm0, %zmm0
-; AVX512DQVL-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
-; AVX512DQVL-NEXT:    vpsravd %zmm1, %zmm0, %zmm0
+; AVX512DQVL-NEXT:    vpsrad %xmm1, %zmm0, %zmm0
 ; AVX512DQVL-NEXT:    vpmovdb %zmm0, %xmm0
 ; AVX512DQVL-NEXT:    vzeroupper
 ; AVX512DQVL-NEXT:    retq
 ;
 ; AVX512BWVL-LABEL: splatvar_shift_v4i8:
 ; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512BWVL-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512BWVL-NEXT:    vpmovsxbw %xmm0, %ymm0
-; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512BWVL-NEXT:    vpsravw %ymm1, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpsraw %xmm1, %ymm0, %ymm0
 ; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm0
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
@@ -1660,20 +1652,18 @@ define <2 x i8> @splatvar_shift_v2i8(<2 x i8> %a, <2 x i8> %b) nounwind {
 ;
 ; AVX512DQ-LABEL: splatvar_shift_v2i8:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512DQ-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512DQ-NEXT:    vpmovsxbd %xmm0, %zmm0
-; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
-; AVX512DQ-NEXT:    vpsravd %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpsrad %xmm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    vpmovdb %zmm0, %xmm0
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatvar_shift_v2i8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512BW-NEXT:    vpmovsxbw %xmm0, %ymm0
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512BW-NEXT:    vpsravw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsraw %xmm1, %ymm0, %ymm0
 ; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
 ; AVX512BW-NEXT:    vzeroupper
@@ -1681,20 +1671,18 @@ define <2 x i8> @splatvar_shift_v2i8(<2 x i8> %a, <2 x i8> %b) nounwind {
 ;
 ; AVX512DQVL-LABEL: splatvar_shift_v2i8:
 ; AVX512DQVL:       # %bb.0:
-; AVX512DQVL-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512DQVL-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512DQVL-NEXT:    vpmovsxbd %xmm0, %zmm0
-; AVX512DQVL-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
-; AVX512DQVL-NEXT:    vpsravd %zmm1, %zmm0, %zmm0
+; AVX512DQVL-NEXT:    vpsrad %xmm1, %zmm0, %zmm0
 ; AVX512DQVL-NEXT:    vpmovdb %zmm0, %xmm0
 ; AVX512DQVL-NEXT:    vzeroupper
 ; AVX512DQVL-NEXT:    retq
 ;
 ; AVX512BWVL-LABEL: splatvar_shift_v2i8:
 ; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512BWVL-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512BWVL-NEXT:    vpmovsxbw %xmm0, %ymm0
-; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512BWVL-NEXT:    vpsravw %ymm1, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpsraw %xmm1, %ymm0, %ymm0
 ; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm0
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/vector-shift-lshr-128.ll b/llvm/test/CodeGen/X86/vector-shift-lshr-128.ll
index 2e19f753722d3..7dab12f3f03e8 100644
--- a/llvm/test/CodeGen/X86/vector-shift-lshr-128.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-lshr-128.ll
@@ -713,20 +713,18 @@ define <16 x i8> @splatvar_shift_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
 ;
 ; AVX512DQ-LABEL: splatvar_shift_v16i8:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512DQ-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
-; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
-; AVX512DQ-NEXT:    vpsrlvd %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpsrld %xmm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    vpmovdb %zmm0, %xmm0
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatvar_shift_v16i8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512BW-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
 ; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
 ; AVX512BW-NEXT:    vzeroupper
@@ -734,20 +732,18 @@ define <16 x i8> @splatvar_shift_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
 ;
 ; AVX512DQVL-LABEL: splatvar_shift_v16i8:
 ; AVX512DQVL:       # %bb.0:
-; AVX512DQVL-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512DQVL-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512DQVL-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
-; AVX512DQVL-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
-; AVX512DQVL-NEXT:    vpsrlvd %zmm1, %zmm0, %zmm0
+; AVX512DQVL-NEXT:    vpsrld %xmm1, %zmm0, %zmm0
 ; AVX512DQVL-NEXT:    vpmovdb %zmm0, %xmm0
 ; AVX512DQVL-NEXT:    vzeroupper
 ; AVX512DQVL-NEXT:    retq
 ;
 ; AVX512BWVL-LABEL: splatvar_shift_v16i8:
 ; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512BWVL-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
-; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512BWVL-NEXT:    vpsrlvw %ymm1, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
 ; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm0
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/vector-shift-lshr-256.ll b/llvm/test/CodeGen/X86/vector-shift-lshr-256.ll
index 9119e32bda375..9ba3e4a451ecf 100644
--- a/llvm/test/CodeGen/X86/vector-shift-lshr-256.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-lshr-256.ll
@@ -762,10 +762,9 @@ define <32 x i8> @splatvar_shift_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
 ;
 ; AVX512BW-LABEL: splatvar_shift_v32i8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpbroadcastb %xmm1, %ymm1
 ; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
-; AVX512BW-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512BW-NEXT:    vpsrlw %xmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
 ; AVX512BW-NEXT:    retq
 ;
@@ -782,10 +781,9 @@ define <32 x i8> @splatvar_shift_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
 ;
 ; AVX512BWVL-LABEL: splatvar_shift_v32i8:
 ; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vpbroadcastb %xmm1, %ymm1
 ; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
-; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
-; AVX512BWVL-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm0
+; AVX512BWVL-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512BWVL-NEXT:    vpsrlw %xmm1, %zmm0, %zmm0
 ; AVX512BWVL-NEXT:    vpmovwb %zmm0, %ymm0
 ; AVX512BWVL-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/vector-shift-lshr-sub128.ll b/llvm/test/CodeGen/X86/vector-shift-lshr-sub128.ll
index 8a843ef652e7f..6b9ac09658e9e 100644
--- a/llvm/test/CodeGen/X86/vector-shift-lshr-sub128.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-lshr-sub128.ll
@@ -1135,20 +1135,18 @@ define <8 x i8> @splatvar_shift_v8i8(<8 x i8> %a, <8 x i8> %b) nounwind {
 ;
 ; AVX512DQ-LABEL: splatvar_shift_v8i8:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512DQ-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
-; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
-; AVX512DQ-NEXT:    vpsrlvd %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpsrld %xmm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    vpmovdb %zmm0, %xmm0
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatvar_shift_v8i8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512BW-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
 ; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
 ; AVX512BW-NEXT:    vzeroupper
@@ -1156,20 +1154,18 @@ define <8 x i8> @splatvar_shift_v8i8(<8 x i8> %a, <8 x i8> %b) nounwind {
 ;
 ; AVX512DQVL-LABEL: splatvar_shift_v8i8:
 ; AVX512DQVL:       # %bb.0:
-; AVX512DQVL-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512DQVL-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512DQVL-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
-; AVX512DQVL-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
-; AVX512DQVL-NEXT:    vpsrlvd %zmm1, %zmm0, %zmm0
+; AVX512DQVL-NEXT:    vpsrld %xmm1, %zmm0, %zmm0
 ; AVX512DQVL-NEXT:    vpmovdb %zmm0, %xmm0
 ; AVX512DQVL-NEXT:    vzeroupper
 ; AVX512DQVL-NEXT:    retq
 ;
 ; AVX512BWVL-LABEL: splatvar_shift_v8i8:
 ; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512BWVL-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
-; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512BWVL-NEXT:    vpsrlvw %ymm1, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
 ; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm0
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
@@ -1257,20 +1253,18 @@ define <4 x i8> @splatvar_shift_v4i8(<4 x i8> %a, <4 x i8> %b) nounwind {
 ;
 ; AVX512DQ-LABEL: splatvar_shift_v4i8:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512DQ-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
-; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
-; AVX512DQ-NEXT:    vpsrlvd %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpsrld %xmm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    vpmovdb %zmm0, %xmm0
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatvar_shift_v4i8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512BW-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
 ; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
 ; AVX512BW-NEXT:    vzeroupper
@@ -1278,20 +1272,18 @@ define <4 x i8> @splatvar_shift_v4i8(<4 x i8> %a, <4 x i8> %b) nounwind {
 ;
 ; AVX512DQVL-LABEL: splatvar_shift_v4i8:
 ; AVX512DQVL:       # %bb.0:
-; AVX512DQVL-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512DQVL-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512DQVL-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
-; AVX512DQVL-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
-; AVX512DQVL-NEXT:    vpsrlvd %zmm1, %zmm0, %zmm0
+; AVX512DQVL-NEXT:    vpsrld %xmm1, %zmm0, %zmm0
 ; AVX512DQVL-NEXT:    vpmovdb %zmm0, %xmm0
 ; AVX512DQVL-NEXT:    vzeroupper
 ; AVX512DQVL-NEXT:    retq
 ;
 ; AVX512BWVL-LABEL: splatvar_shift_v4i8:
 ; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512BWVL-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
-; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512BWVL-NEXT:    vpsrlvw %ymm1, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
 ; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm0
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
@@ -1370,20 +1362,18 @@ define <2 x i8> @splatvar_shift_v2i8(<2 x i8> %a, <2 x i8> %b) nounwind {
 ;
 ; AVX512DQ-LABEL: splatvar_shift_v2i8:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512DQ-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
-; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
-; AVX512DQ-NEXT:    vpsrlvd %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpsrld %xmm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    vpmovdb %zmm0, %xmm0
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatvar_shift_v2i8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512BW-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
 ; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
 ; AVX512BW-NEXT:    vzeroupper
@@ -1391,20 +1381,18 @@ define <2 x i8> @splatvar_shift_v2i8(<2 x i8> %a, <2 x i8> %b) nounwind {
 ;
 ; AVX512DQVL-LABEL: splatvar_shift_v2i8:
 ; AVX512DQVL:       # %bb.0:
-; AVX512DQVL-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512DQVL-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512DQVL-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
-; AVX512DQVL-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
-; AVX512DQVL-NEXT:    vpsrlvd %zmm1, %zmm0, %zmm0
+; AVX512DQVL-NEXT:    vpsrld %xmm1, %zmm0, %zmm0
 ; AVX512DQVL-NEXT:    vpmovdb %zmm0, %xmm0
 ; AVX512DQVL-NEXT:    vzeroupper
 ; AVX512DQVL-NEXT:    retq
 ;
 ; AVX512BWVL-LABEL: splatvar_shift_v2i8:
 ; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512BWVL-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
-; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512BWVL-NEXT:    vpsrlvw %ymm1, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
 ; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm0
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/vector-shift-shl-128.ll b/llvm/test/CodeGen/X86/vector-shift-shl-128.ll
index 5f1325aacb4d1..4b83700e3a85f 100644
--- a/llvm/test/CodeGen/X86/vector-shift-shl-128.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-shl-128.ll
@@ -623,20 +623,18 @@ define <16 x i8> @splatvar_shift_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
 ;
 ; AVX512DQ-LABEL: splatvar_shift_v16i8:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512DQ-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
-; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
-; AVX512DQ-NEXT:    vpsllvd %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpslld %xmm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    vpmovdb %zmm0, %xmm0
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatvar_shift_v16i8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512BW-NEXT:    vpsllvw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsllw %xmm1, %ymm0, %ymm0
 ; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
 ; AVX512BW-NEXT:    vzeroupper
@@ -644,20 +642,18 @@ define <16 x i8> @splatvar_shift_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
 ;
 ; AVX512DQVL-LABEL: splatvar_shift_v16i8:
 ; AVX512DQVL:       # %bb.0:
-; AVX512DQVL-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512DQVL-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512DQVL-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
-; AVX512DQVL-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
-; AVX512DQVL-NEXT:    vpsllvd %zmm1, %zmm0, %zmm0
+; AVX512DQVL-NEXT:    vpslld %xmm1, %zmm0, %zmm0
 ; AVX512DQVL-NEXT:    vpmovdb %zmm0, %xmm0
 ; AVX512DQVL-NEXT:    vzeroupper
 ; AVX512DQVL-NEXT:    retq
 ;
 ; AVX512BWVL-LABEL: splatvar_shift_v16i8:
 ; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512BWVL-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
-; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512BWVL-NEXT:    vpsllvw %ymm1, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpsllw %xmm1, %ymm0, %ymm0
 ; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm0
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/vector-shift-shl-256.ll b/llvm/test/CodeGen/X86/vector-shift-shl-256.ll
index 1296fcf8bb902..8ed95bae2ebd2 100644
--- a/llvm/test/CodeGen/X86/vector-shift-shl-256.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-shl-256.ll
@@ -688,10 +688,9 @@ define <32 x i8> @splatvar_shift_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
 ;
 ; AVX512BW-LABEL: splatvar_shift_v32i8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpbroadcastb %xmm1, %ymm1
 ; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
-; AVX512BW-NEXT:    vpsllvw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512BW-NEXT:    vpsllw %xmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
 ; AVX512BW-NEXT:    retq
 ;
@@ -707,10 +706,9 @@ define <32 x i8> @splatvar_shift_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
 ;
 ; AVX512BWVL-LABEL: splatvar_shift_v32i8:
 ; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vpbroadcastb %xmm1, %ymm1
 ; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
-; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
-; AVX512BWVL-NEXT:    vpsllvw %zmm1, %zmm0, %zmm0
+; AVX512BWVL-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512BWVL-NEXT:    vpsllw %xmm1, %zmm0, %zmm0
 ; AVX512BWVL-NEXT:    vpmovwb %zmm0, %ymm0
 ; AVX512BWVL-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/vector-shift-shl-sub128.ll b/llvm/test/CodeGen/X86/vector-shift-shl-sub128.ll
index 808033e0fb7ce..1c688089af494 100644
--- a/llvm/test/CodeGen/X86/vector-shift-shl-sub128.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-shl-sub128.ll
@@ -988,20 +988,18 @@ define <8 x i8> @splatvar_shift_v8i8(<8 x i8> %a, <8 x i8> %b) nounwind {
 ;
 ; AVX512DQ-LABEL: splatvar_shift_v8i8:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512DQ-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
-; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
-; AVX512DQ-NEXT:    vpsllvd %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpslld %xmm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    vpmovdb %zmm0, %xmm0
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatvar_shift_v8i8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512BW-NEXT:    vpsllvw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsllw %xmm1, %ymm0, %ymm0
 ; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
 ; AVX512BW-NEXT:    vzeroupper
@@ -1009,20 +1007,18 @@ define <8 x i8> @splatvar_shift_v8i8(<8 x i8> %a, <8 x i8> %b) nounwind {
 ;
 ; AVX512DQVL-LABEL: splatvar_shift_v8i8:
 ; AVX512DQVL:       # %bb.0:
-; AVX512DQVL-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512DQVL-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512DQVL-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
-; AVX512DQVL-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
-; AVX512DQVL-NEXT:    vpsllvd %zmm1, %zmm0, %zmm0
+; AVX512DQVL-NEXT:    vpslld %xmm1, %zmm0, %zmm0
 ; AVX512DQVL-NEXT:    vpmovdb %zmm0, %xmm0
 ; AVX512DQVL-NEXT:    vzeroupper
 ; AVX512DQVL-NEXT:    retq
 ;
 ; AVX512BWVL-LABEL: splatvar_shift_v8i8:
 ; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512BWVL-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
-; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512BWVL-NEXT:    vpsllvw %ymm1, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpsllw %xmm1, %ymm0, %ymm0
 ; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm0
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
@@ -1105,20 +1101,18 @@ define <4 x i8> @splatvar_shift_v4i8(<4 x i8> %a, <4 x i8> %b) nounwind {
 ;
 ; AVX512DQ-LABEL: splatvar_shift_v4i8:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512DQ-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
-; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
-; AVX512DQ-NEXT:    vpsllvd %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpslld %xmm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    vpmovdb %zmm0, %xmm0
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatvar_shift_v4i8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512BW-NEXT:    vpsllvw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsllw %xmm1, %ymm0, %ymm0
 ; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
 ; AVX512BW-NEXT:    vzeroupper
@@ -1126,20 +1120,18 @@ define <4 x i8> @splatvar_shift_v4i8(<4 x i8> %a, <4 x i8> %b) nounwind {
 ;
 ; AVX512DQVL-LABEL: splatvar_shift_v4i8:
 ; AVX512DQVL:       # %bb.0:
-; AVX512DQVL-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512DQVL-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512DQVL-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
-; AVX512DQVL-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
-; AVX512DQVL-NEXT:    vpsllvd %zmm1, %zmm0, %zmm0
+; AVX512DQVL-NEXT:    vpslld %xmm1, %zmm0, %zmm0
 ; AVX512DQVL-NEXT:    vpmovdb %zmm0, %xmm0
 ; AVX512DQVL-NEXT:    vzeroupper
 ; AVX512DQVL-NEXT:    retq
 ;
 ; AVX512BWVL-LABEL: splatvar_shift_v4i8:
 ; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512BWVL-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
-; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512BWVL-NEXT:    vpsllvw %ymm1, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpsllw %xmm1, %ymm0, %ymm0
 ; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm0
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
@@ -1215,20 +1207,18 @@ define <2 x i8> @splatvar_shift_v2i8(<2 x i8> %a, <2 x i8> %b) nounwind {
 ;
 ; AVX512DQ-LABEL: splatvar_shift_v2i8:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512DQ-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
-; AVX512DQ-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
-; AVX512DQ-NEXT:    vpsllvd %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpslld %xmm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    vpmovdb %zmm0, %xmm0
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatvar_shift_v2i8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512BW-NEXT:    vpsllvw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsllw %xmm1, %ymm0, %ymm0
 ; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
 ; AVX512BW-NEXT:    vzeroupper
@@ -1236,20 +1226,18 @@ define <2 x i8> @splatvar_shift_v2i8(<2 x i8> %a, <2 x i8> %b) nounwind {
 ;
 ; AVX512DQVL-LABEL: splatvar_shift_v2i8:
 ; AVX512DQVL:       # %bb.0:
-; AVX512DQVL-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512DQVL-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512DQVL-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
-; AVX512DQVL-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
-; AVX512DQVL-NEXT:    vpsllvd %zmm1, %zmm0, %zmm0
+; AVX512DQVL-NEXT:    vpslld %xmm1, %zmm0, %zmm0
 ; AVX512DQVL-NEXT:    vpmovdb %zmm0, %xmm0
 ; AVX512DQVL-NEXT:    vzeroupper
 ; AVX512DQVL-NEXT:    retq
 ;
 ; AVX512BWVL-LABEL: splatvar_shift_v2i8:
 ; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512BWVL-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
-; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512BWVL-NEXT:    vpsllvw %ymm1, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpsllw %xmm1, %ymm0, %ymm0
 ; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm0
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq

From 3f7249046a87e08272957d12bff73295fc4f0e8c Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Tue, 28 Jul 2020 19:55:55 +0100
Subject: [PATCH 0345/1035] [CostModel][X86] Add smax/smin/umin/umax intrinsics
 cost model tests

Costs currently fall back to scalar generic intrinsic calls
---
 .../Analysis/CostModel/X86/arith-sminmax.ll   | 287 ++++++++++++++++++
 .../Analysis/CostModel/X86/arith-uminmax.ll   | 287 ++++++++++++++++++
 2 files changed, 574 insertions(+)
 create mode 100644 llvm/test/Analysis/CostModel/X86/arith-sminmax.ll
 create mode 100644 llvm/test/Analysis/CostModel/X86/arith-uminmax.ll

diff --git a/llvm/test/Analysis/CostModel/X86/arith-sminmax.ll b/llvm/test/Analysis/CostModel/X86/arith-sminmax.ll
new file mode 100644
index 0000000000000..96180d01aae09
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/X86/arith-sminmax.ll
@@ -0,0 +1,287 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+sse2 | FileCheck %s -check-prefixes=CHECK,SSE,SSE2
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+ssse3 | FileCheck %s -check-prefixes=CHECK,SSE,SSSE3
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+sse4.2 | FileCheck %s -check-prefixes=CHECK,SSE,SSE42
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+avx | FileCheck %s -check-prefixes=CHECK,AVX,AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+avx2 | FileCheck %s -check-prefixes=CHECK,AVX,AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+avx512f | FileCheck %s -check-prefixes=CHECK,AVX512,AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+avx512vl,+avx512dq | FileCheck %s -check-prefixes=CHECK,AVX512,AVX512DQ
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+avx512vl,+avx512bw | FileCheck %s -check-prefixes=CHECK,AVX512,AVX512BW
+
+declare i64        @llvm.smax.i64(i64, i64)
+declare <2 x i64>  @llvm.smax.v2i64(<2 x i64>, <2 x i64>)
+declare <4 x i64>  @llvm.smax.v4i64(<4 x i64>, <4 x i64>)
+declare <8 x i64>  @llvm.smax.v8i64(<8 x i64>, <8 x i64>)
+
+declare i32        @llvm.smax.i32(i32, i32)
+declare <4 x i32>  @llvm.smax.v4i32(<4 x i32>, <4 x i32>)
+declare <8 x i32>  @llvm.smax.v8i32(<8 x i32>, <8 x i32>)
+declare <16 x i32> @llvm.smax.v16i32(<16 x i32>, <16 x i32>)
+
+declare i16        @llvm.smax.i16(i16, i16)
+declare <8 x i16>  @llvm.smax.v8i16(<8 x i16>, <8 x i16>)
+declare <16 x i16> @llvm.smax.v16i16(<16 x i16>, <16 x i16>)
+declare <32 x i16> @llvm.smax.v32i16(<32 x i16>, <32 x i16>)
+
+declare i8         @llvm.smax.i8(i8,  i8)
+declare <16 x i8>  @llvm.smax.v16i8(<16 x i8>, <16 x i8>)
+declare <32 x i8>  @llvm.smax.v32i8(<32 x i8>, <32 x i8>)
+declare <64 x i8>  @llvm.smax.v64i8(<64 x i8>, <64 x i8>)
+
+define i32 @smax(i32 %arg) {
+; SSE2-LABEL: 'smax'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.smax.i64(i64 undef, i64 undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I64 = call <2 x i64> @llvm.smax.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I64 = call <4 x i64> @llvm.smax.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V8I64 = call <8 x i64> @llvm.smax.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.smax.i32(i32 undef, i32 undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4I32 = call <4 x i32> @llvm.smax.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V8I32 = call <8 x i32> @llvm.smax.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V16I32 = call <16 x i32> @llvm.smax.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.smax.i16(i16 undef, i16 undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I16 = call <8 x i16> @llvm.smax.v8i16(<8 x i16> undef, <8 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I16 = call <16 x i16> @llvm.smax.v16i16(<16 x i16> undef, <16 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V32I16 = call <32 x i16> @llvm.smax.v32i16(<32 x i16> undef, <32 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.smax.i8(i8 undef, i8 undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V16I8 = call <16 x i8> @llvm.smax.v16i8(<16 x i8> undef, <16 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 94 for instruction: %V32I8 = call <32 x i8> @llvm.smax.v32i8(<32 x i8> undef, <32 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 188 for instruction: %V64I8 = call <64 x i8> @llvm.smax.v64i8(<64 x i8> undef, <64 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSSE3-LABEL: 'smax'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.smax.i64(i64 undef, i64 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I64 = call <2 x i64> @llvm.smax.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I64 = call <4 x i64> @llvm.smax.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V8I64 = call <8 x i64> @llvm.smax.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.smax.i32(i32 undef, i32 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4I32 = call <4 x i32> @llvm.smax.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V8I32 = call <8 x i32> @llvm.smax.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V16I32 = call <16 x i32> @llvm.smax.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.smax.i16(i16 undef, i16 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I16 = call <8 x i16> @llvm.smax.v8i16(<8 x i16> undef, <8 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I16 = call <16 x i16> @llvm.smax.v16i16(<16 x i16> undef, <16 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V32I16 = call <32 x i16> @llvm.smax.v32i16(<32 x i16> undef, <32 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.smax.i8(i8 undef, i8 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V16I8 = call <16 x i8> @llvm.smax.v16i8(<16 x i8> undef, <16 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 94 for instruction: %V32I8 = call <32 x i8> @llvm.smax.v32i8(<32 x i8> undef, <32 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 188 for instruction: %V64I8 = call <64 x i8> @llvm.smax.v64i8(<64 x i8> undef, <64 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE42-LABEL: 'smax'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.smax.i64(i64 undef, i64 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I64 = call <2 x i64> @llvm.smax.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4I64 = call <4 x i64> @llvm.smax.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I64 = call <8 x i64> @llvm.smax.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.smax.i32(i32 undef, i32 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4I32 = call <4 x i32> @llvm.smax.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I32 = call <8 x i32> @llvm.smax.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I32 = call <16 x i32> @llvm.smax.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.smax.i16(i16 undef, i16 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I16 = call <8 x i16> @llvm.smax.v8i16(<8 x i16> undef, <8 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I16 = call <16 x i16> @llvm.smax.v16i16(<16 x i16> undef, <16 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V32I16 = call <32 x i16> @llvm.smax.v32i16(<32 x i16> undef, <32 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.smax.i8(i8 undef, i8 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I8 = call <16 x i8> @llvm.smax.v16i8(<16 x i8> undef, <16 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V32I8 = call <32 x i8> @llvm.smax.v32i8(<32 x i8> undef, <32 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %V64I8 = call <64 x i8> @llvm.smax.v64i8(<64 x i8> undef, <64 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX-LABEL: 'smax'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.smax.i64(i64 undef, i64 undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I64 = call <2 x i64> @llvm.smax.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I64 = call <4 x i64> @llvm.smax.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V8I64 = call <8 x i64> @llvm.smax.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.smax.i32(i32 undef, i32 undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4I32 = call <4 x i32> @llvm.smax.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I32 = call <8 x i32> @llvm.smax.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V16I32 = call <16 x i32> @llvm.smax.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.smax.i16(i16 undef, i16 undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I16 = call <8 x i16> @llvm.smax.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V16I16 = call <16 x i16> @llvm.smax.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %V32I16 = call <32 x i16> @llvm.smax.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.smax.i8(i8 undef, i8 undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I8 = call <16 x i8> @llvm.smax.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: %V32I8 = call <32 x i8> @llvm.smax.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %V64I8 = call <64 x i8> @llvm.smax.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512-LABEL: 'smax'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.smax.i64(i64 undef, i64 undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I64 = call <2 x i64> @llvm.smax.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I64 = call <4 x i64> @llvm.smax.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V8I64 = call <8 x i64> @llvm.smax.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.smax.i32(i32 undef, i32 undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4I32 = call <4 x i32> @llvm.smax.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I32 = call <8 x i32> @llvm.smax.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V16I32 = call <16 x i32> @llvm.smax.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.smax.i16(i16 undef, i16 undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I16 = call <8 x i16> @llvm.smax.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V16I16 = call <16 x i16> @llvm.smax.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %V32I16 = call <32 x i16> @llvm.smax.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.smax.i8(i8 undef, i8 undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I8 = call <16 x i8> @llvm.smax.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: %V32I8 = call <32 x i8> @llvm.smax.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 131 for instruction: %V64I8 = call <64 x i8> @llvm.smax.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %I64 = call i64 @llvm.smax.i64(i64 undef, i64 undef)
+  %V2I64 = call <2 x i64> @llvm.smax.v2i64(<2 x i64> undef, <2 x i64> undef)
+  %V4I64 = call <4 x i64> @llvm.smax.v4i64(<4 x i64> undef, <4 x i64> undef)
+  %V8I64 = call <8 x i64> @llvm.smax.v8i64(<8 x i64> undef, <8 x i64> undef)
+
+  %I32 = call i32 @llvm.smax.i32(i32 undef, i32 undef)
+  %V4I32  = call <4 x i32>  @llvm.smax.v4i32(<4 x i32> undef, <4 x i32> undef)
+  %V8I32  = call <8 x i32>  @llvm.smax.v8i32(<8 x i32> undef, <8 x i32> undef)
+  %V16I32 = call <16 x i32> @llvm.smax.v16i32(<16 x i32> undef, <16 x i32> undef)
+
+  %I16 = call i16 @llvm.smax.i16(i16 undef, i16 undef)
+  %V8I16  = call <8 x i16>  @llvm.smax.v8i16(<8 x i16> undef, <8 x i16> undef)
+  %V16I16 = call <16 x i16> @llvm.smax.v16i16(<16 x i16> undef, <16 x i16> undef)
+  %V32I16 = call <32 x i16> @llvm.smax.v32i16(<32 x i16> undef, <32 x i16> undef)
+
+  %I8 = call i8 @llvm.smax.i8(i8 undef, i8 undef)
+  %V16I8 = call <16 x i8> @llvm.smax.v16i8(<16 x i8> undef, <16 x i8> undef)
+  %V32I8 = call <32 x i8> @llvm.smax.v32i8(<32 x i8> undef, <32 x i8> undef)
+  %V64I8 = call <64 x i8> @llvm.smax.v64i8(<64 x i8> undef, <64 x i8> undef)
+
+  ret i32 undef
+}
+
+declare i64        @llvm.smin.i64(i64, i64)
+declare <2 x i64>  @llvm.smin.v2i64(<2 x i64>, <2 x i64>)
+declare <4 x i64>  @llvm.smin.v4i64(<4 x i64>, <4 x i64>)
+declare <8 x i64>  @llvm.smin.v8i64(<8 x i64>, <8 x i64>)
+
+declare i32        @llvm.smin.i32(i32, i32)
+declare <4 x i32>  @llvm.smin.v4i32(<4 x i32>, <4 x i32>)
+declare <8 x i32>  @llvm.smin.v8i32(<8 x i32>, <8 x i32>)
+declare <16 x i32> @llvm.smin.v16i32(<16 x i32>, <16 x i32>)
+
+declare i16        @llvm.smin.i16(i16, i16)
+declare <8 x i16>  @llvm.smin.v8i16(<8 x i16>, <8 x i16>)
+declare <16 x i16> @llvm.smin.v16i16(<16 x i16>, <16 x i16>)
+declare <32 x i16> @llvm.smin.v32i16(<32 x i16>, <32 x i16>)
+
+declare i8         @llvm.smin.i8(i8,  i8)
+declare <16 x i8>  @llvm.smin.v16i8(<16 x i8>, <16 x i8>)
+declare <32 x i8>  @llvm.smin.v32i8(<32 x i8>, <32 x i8>)
+declare <64 x i8>  @llvm.smin.v64i8(<64 x i8>, <64 x i8>)
+
+define i32 @smin(i32 %arg) {
+; SSE2-LABEL: 'smin'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.smin.i64(i64 undef, i64 undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I64 = call <2 x i64> @llvm.smin.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I64 = call <4 x i64> @llvm.smin.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V8I64 = call <8 x i64> @llvm.smin.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.smin.i32(i32 undef, i32 undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4I32 = call <4 x i32> @llvm.smin.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V8I32 = call <8 x i32> @llvm.smin.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V16I32 = call <16 x i32> @llvm.smin.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.smin.i16(i16 undef, i16 undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I16 = call <8 x i16> @llvm.smin.v8i16(<8 x i16> undef, <8 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I16 = call <16 x i16> @llvm.smin.v16i16(<16 x i16> undef, <16 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V32I16 = call <32 x i16> @llvm.smin.v32i16(<32 x i16> undef, <32 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.smin.i8(i8 undef, i8 undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V16I8 = call <16 x i8> @llvm.smin.v16i8(<16 x i8> undef, <16 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 94 for instruction: %V32I8 = call <32 x i8> @llvm.smin.v32i8(<32 x i8> undef, <32 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 188 for instruction: %V64I8 = call <64 x i8> @llvm.smin.v64i8(<64 x i8> undef, <64 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSSE3-LABEL: 'smin'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.smin.i64(i64 undef, i64 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I64 = call <2 x i64> @llvm.smin.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I64 = call <4 x i64> @llvm.smin.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V8I64 = call <8 x i64> @llvm.smin.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.smin.i32(i32 undef, i32 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4I32 = call <4 x i32> @llvm.smin.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V8I32 = call <8 x i32> @llvm.smin.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V16I32 = call <16 x i32> @llvm.smin.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.smin.i16(i16 undef, i16 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I16 = call <8 x i16> @llvm.smin.v8i16(<8 x i16> undef, <8 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I16 = call <16 x i16> @llvm.smin.v16i16(<16 x i16> undef, <16 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V32I16 = call <32 x i16> @llvm.smin.v32i16(<32 x i16> undef, <32 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.smin.i8(i8 undef, i8 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V16I8 = call <16 x i8> @llvm.smin.v16i8(<16 x i8> undef, <16 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 94 for instruction: %V32I8 = call <32 x i8> @llvm.smin.v32i8(<32 x i8> undef, <32 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 188 for instruction: %V64I8 = call <64 x i8> @llvm.smin.v64i8(<64 x i8> undef, <64 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE42-LABEL: 'smin'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.smin.i64(i64 undef, i64 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I64 = call <2 x i64> @llvm.smin.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4I64 = call <4 x i64> @llvm.smin.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I64 = call <8 x i64> @llvm.smin.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.smin.i32(i32 undef, i32 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4I32 = call <4 x i32> @llvm.smin.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I32 = call <8 x i32> @llvm.smin.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I32 = call <16 x i32> @llvm.smin.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.smin.i16(i16 undef, i16 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I16 = call <8 x i16> @llvm.smin.v8i16(<8 x i16> undef, <8 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I16 = call <16 x i16> @llvm.smin.v16i16(<16 x i16> undef, <16 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V32I16 = call <32 x i16> @llvm.smin.v32i16(<32 x i16> undef, <32 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.smin.i8(i8 undef, i8 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I8 = call <16 x i8> @llvm.smin.v16i8(<16 x i8> undef, <16 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V32I8 = call <32 x i8> @llvm.smin.v32i8(<32 x i8> undef, <32 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %V64I8 = call <64 x i8> @llvm.smin.v64i8(<64 x i8> undef, <64 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX-LABEL: 'smin'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.smin.i64(i64 undef, i64 undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I64 = call <2 x i64> @llvm.smin.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I64 = call <4 x i64> @llvm.smin.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V8I64 = call <8 x i64> @llvm.smin.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.smin.i32(i32 undef, i32 undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4I32 = call <4 x i32> @llvm.smin.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I32 = call <8 x i32> @llvm.smin.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V16I32 = call <16 x i32> @llvm.smin.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.smin.i16(i16 undef, i16 undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I16 = call <8 x i16> @llvm.smin.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V16I16 = call <16 x i16> @llvm.smin.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %V32I16 = call <32 x i16> @llvm.smin.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.smin.i8(i8 undef, i8 undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I8 = call <16 x i8> @llvm.smin.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: %V32I8 = call <32 x i8> @llvm.smin.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %V64I8 = call <64 x i8> @llvm.smin.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512-LABEL: 'smin'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.smin.i64(i64 undef, i64 undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I64 = call <2 x i64> @llvm.smin.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I64 = call <4 x i64> @llvm.smin.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V8I64 = call <8 x i64> @llvm.smin.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.smin.i32(i32 undef, i32 undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4I32 = call <4 x i32> @llvm.smin.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I32 = call <8 x i32> @llvm.smin.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V16I32 = call <16 x i32> @llvm.smin.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.smin.i16(i16 undef, i16 undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I16 = call <8 x i16> @llvm.smin.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V16I16 = call <16 x i16> @llvm.smin.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %V32I16 = call <32 x i16> @llvm.smin.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.smin.i8(i8 undef, i8 undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I8 = call <16 x i8> @llvm.smin.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: %V32I8 = call <32 x i8> @llvm.smin.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 131 for instruction: %V64I8 = call <64 x i8> @llvm.smin.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %I64 = call i64 @llvm.smin.i64(i64 undef, i64 undef)
+  %V2I64 = call <2 x i64> @llvm.smin.v2i64(<2 x i64> undef, <2 x i64> undef)
+  %V4I64 = call <4 x i64> @llvm.smin.v4i64(<4 x i64> undef, <4 x i64> undef)
+  %V8I64 = call <8 x i64> @llvm.smin.v8i64(<8 x i64> undef, <8 x i64> undef)
+
+  %I32 = call i32 @llvm.smin.i32(i32 undef, i32 undef)
+  %V4I32  = call <4 x i32>  @llvm.smin.v4i32(<4 x i32> undef, <4 x i32> undef)
+  %V8I32  = call <8 x i32>  @llvm.smin.v8i32(<8 x i32> undef, <8 x i32> undef)
+  %V16I32 = call <16 x i32> @llvm.smin.v16i32(<16 x i32> undef, <16 x i32> undef)
+
+  %I16 = call i16 @llvm.smin.i16(i16 undef, i16 undef)
+  %V8I16  = call <8 x i16>  @llvm.smin.v8i16(<8 x i16> undef, <8 x i16> undef)
+  %V16I16 = call <16 x i16> @llvm.smin.v16i16(<16 x i16> undef, <16 x i16> undef)
+  %V32I16 = call <32 x i16> @llvm.smin.v32i16(<32 x i16> undef, <32 x i16> undef)
+
+  %I8 = call i8 @llvm.smin.i8(i8 undef, i8 undef)
+  %V16I8 = call <16 x i8> @llvm.smin.v16i8(<16 x i8> undef, <16 x i8> undef)
+  %V32I8 = call <32 x i8> @llvm.smin.v32i8(<32 x i8> undef, <32 x i8> undef)
+  %V64I8 = call <64 x i8> @llvm.smin.v64i8(<64 x i8> undef, <64 x i8> undef)
+
+  ret i32 undef
+}
diff --git a/llvm/test/Analysis/CostModel/X86/arith-uminmax.ll b/llvm/test/Analysis/CostModel/X86/arith-uminmax.ll
new file mode 100644
index 0000000000000..74aaa3b03a49e
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/X86/arith-uminmax.ll
@@ -0,0 +1,287 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+sse2 | FileCheck %s -check-prefixes=CHECK,SSE,SSE2
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+ssse3 | FileCheck %s -check-prefixes=CHECK,SSE,SSSE3
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+sse4.2 | FileCheck %s -check-prefixes=CHECK,SSE,SSE42
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+avx | FileCheck %s -check-prefixes=CHECK,AVX,AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+avx2 | FileCheck %s -check-prefixes=CHECK,AVX,AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+avx512f | FileCheck %s -check-prefixes=CHECK,AVX512,AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+avx512vl,+avx512dq | FileCheck %s -check-prefixes=CHECK,AVX512,AVX512DQ
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+avx512vl,+avx512bw | FileCheck %s -check-prefixes=CHECK,AVX512,AVX512BW
+
+declare i64        @llvm.umax.i64(i64, i64)
+declare <2 x i64>  @llvm.umax.v2i64(<2 x i64>, <2 x i64>)
+declare <4 x i64>  @llvm.umax.v4i64(<4 x i64>, <4 x i64>)
+declare <8 x i64>  @llvm.umax.v8i64(<8 x i64>, <8 x i64>)
+
+declare i32        @llvm.umax.i32(i32, i32)
+declare <4 x i32>  @llvm.umax.v4i32(<4 x i32>, <4 x i32>)
+declare <8 x i32>  @llvm.umax.v8i32(<8 x i32>, <8 x i32>)
+declare <16 x i32> @llvm.umax.v16i32(<16 x i32>, <16 x i32>)
+
+declare i16        @llvm.umax.i16(i16, i16)
+declare <8 x i16>  @llvm.umax.v8i16(<8 x i16>, <8 x i16>)
+declare <16 x i16> @llvm.umax.v16i16(<16 x i16>, <16 x i16>)
+declare <32 x i16> @llvm.umax.v32i16(<32 x i16>, <32 x i16>)
+
+declare i8         @llvm.umax.i8(i8,  i8)
+declare <16 x i8>  @llvm.umax.v16i8(<16 x i8>, <16 x i8>)
+declare <32 x i8>  @llvm.umax.v32i8(<32 x i8>, <32 x i8>)
+declare <64 x i8>  @llvm.umax.v64i8(<64 x i8>, <64 x i8>)
+
+define i32 @umax(i32 %arg) {
+; SSE2-LABEL: 'umax'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.umax.i64(i64 undef, i64 undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I64 = call <2 x i64> @llvm.umax.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I64 = call <4 x i64> @llvm.umax.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V8I64 = call <8 x i64> @llvm.umax.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.umax.i32(i32 undef, i32 undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4I32 = call <4 x i32> @llvm.umax.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V8I32 = call <8 x i32> @llvm.umax.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V16I32 = call <16 x i32> @llvm.umax.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.umax.i16(i16 undef, i16 undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I16 = call <8 x i16> @llvm.umax.v8i16(<8 x i16> undef, <8 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I16 = call <16 x i16> @llvm.umax.v16i16(<16 x i16> undef, <16 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V32I16 = call <32 x i16> @llvm.umax.v32i16(<32 x i16> undef, <32 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.umax.i8(i8 undef, i8 undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V16I8 = call <16 x i8> @llvm.umax.v16i8(<16 x i8> undef, <16 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 94 for instruction: %V32I8 = call <32 x i8> @llvm.umax.v32i8(<32 x i8> undef, <32 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 188 for instruction: %V64I8 = call <64 x i8> @llvm.umax.v64i8(<64 x i8> undef, <64 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSSE3-LABEL: 'umax'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.umax.i64(i64 undef, i64 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I64 = call <2 x i64> @llvm.umax.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I64 = call <4 x i64> @llvm.umax.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V8I64 = call <8 x i64> @llvm.umax.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.umax.i32(i32 undef, i32 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4I32 = call <4 x i32> @llvm.umax.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V8I32 = call <8 x i32> @llvm.umax.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V16I32 = call <16 x i32> @llvm.umax.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.umax.i16(i16 undef, i16 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I16 = call <8 x i16> @llvm.umax.v8i16(<8 x i16> undef, <8 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I16 = call <16 x i16> @llvm.umax.v16i16(<16 x i16> undef, <16 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V32I16 = call <32 x i16> @llvm.umax.v32i16(<32 x i16> undef, <32 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.umax.i8(i8 undef, i8 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V16I8 = call <16 x i8> @llvm.umax.v16i8(<16 x i8> undef, <16 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 94 for instruction: %V32I8 = call <32 x i8> @llvm.umax.v32i8(<32 x i8> undef, <32 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 188 for instruction: %V64I8 = call <64 x i8> @llvm.umax.v64i8(<64 x i8> undef, <64 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE42-LABEL: 'umax'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.umax.i64(i64 undef, i64 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I64 = call <2 x i64> @llvm.umax.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4I64 = call <4 x i64> @llvm.umax.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I64 = call <8 x i64> @llvm.umax.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.umax.i32(i32 undef, i32 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4I32 = call <4 x i32> @llvm.umax.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I32 = call <8 x i32> @llvm.umax.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I32 = call <16 x i32> @llvm.umax.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.umax.i16(i16 undef, i16 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I16 = call <8 x i16> @llvm.umax.v8i16(<8 x i16> undef, <8 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I16 = call <16 x i16> @llvm.umax.v16i16(<16 x i16> undef, <16 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V32I16 = call <32 x i16> @llvm.umax.v32i16(<32 x i16> undef, <32 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.umax.i8(i8 undef, i8 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I8 = call <16 x i8> @llvm.umax.v16i8(<16 x i8> undef, <16 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V32I8 = call <32 x i8> @llvm.umax.v32i8(<32 x i8> undef, <32 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %V64I8 = call <64 x i8> @llvm.umax.v64i8(<64 x i8> undef, <64 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX-LABEL: 'umax'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.umax.i64(i64 undef, i64 undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I64 = call <2 x i64> @llvm.umax.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I64 = call <4 x i64> @llvm.umax.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V8I64 = call <8 x i64> @llvm.umax.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.umax.i32(i32 undef, i32 undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4I32 = call <4 x i32> @llvm.umax.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I32 = call <8 x i32> @llvm.umax.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V16I32 = call <16 x i32> @llvm.umax.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.umax.i16(i16 undef, i16 undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I16 = call <8 x i16> @llvm.umax.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V16I16 = call <16 x i16> @llvm.umax.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %V32I16 = call <32 x i16> @llvm.umax.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.umax.i8(i8 undef, i8 undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I8 = call <16 x i8> @llvm.umax.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: %V32I8 = call <32 x i8> @llvm.umax.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %V64I8 = call <64 x i8> @llvm.umax.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512-LABEL: 'umax'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.umax.i64(i64 undef, i64 undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I64 = call <2 x i64> @llvm.umax.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I64 = call <4 x i64> @llvm.umax.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V8I64 = call <8 x i64> @llvm.umax.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.umax.i32(i32 undef, i32 undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4I32 = call <4 x i32> @llvm.umax.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I32 = call <8 x i32> @llvm.umax.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V16I32 = call <16 x i32> @llvm.umax.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.umax.i16(i16 undef, i16 undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I16 = call <8 x i16> @llvm.umax.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V16I16 = call <16 x i16> @llvm.umax.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %V32I16 = call <32 x i16> @llvm.umax.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.umax.i8(i8 undef, i8 undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I8 = call <16 x i8> @llvm.umax.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: %V32I8 = call <32 x i8> @llvm.umax.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 131 for instruction: %V64I8 = call <64 x i8> @llvm.umax.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %I64 = call i64 @llvm.umax.i64(i64 undef, i64 undef)
+  %V2I64 = call <2 x i64> @llvm.umax.v2i64(<2 x i64> undef, <2 x i64> undef)
+  %V4I64 = call <4 x i64> @llvm.umax.v4i64(<4 x i64> undef, <4 x i64> undef)
+  %V8I64 = call <8 x i64> @llvm.umax.v8i64(<8 x i64> undef, <8 x i64> undef)
+
+  %I32 = call i32 @llvm.umax.i32(i32 undef, i32 undef)
+  %V4I32  = call <4 x i32>  @llvm.umax.v4i32(<4 x i32> undef, <4 x i32> undef)
+  %V8I32  = call <8 x i32>  @llvm.umax.v8i32(<8 x i32> undef, <8 x i32> undef)
+  %V16I32 = call <16 x i32> @llvm.umax.v16i32(<16 x i32> undef, <16 x i32> undef)
+
+  %I16 = call i16 @llvm.umax.i16(i16 undef, i16 undef)
+  %V8I16  = call <8 x i16>  @llvm.umax.v8i16(<8 x i16> undef, <8 x i16> undef)
+  %V16I16 = call <16 x i16> @llvm.umax.v16i16(<16 x i16> undef, <16 x i16> undef)
+  %V32I16 = call <32 x i16> @llvm.umax.v32i16(<32 x i16> undef, <32 x i16> undef)
+
+  %I8 = call i8 @llvm.umax.i8(i8 undef, i8 undef)
+  %V16I8 = call <16 x i8> @llvm.umax.v16i8(<16 x i8> undef, <16 x i8> undef)
+  %V32I8 = call <32 x i8> @llvm.umax.v32i8(<32 x i8> undef, <32 x i8> undef)
+  %V64I8 = call <64 x i8> @llvm.umax.v64i8(<64 x i8> undef, <64 x i8> undef)
+
+  ret i32 undef
+}
+
+declare i64        @llvm.umin.i64(i64, i64)
+declare <2 x i64>  @llvm.umin.v2i64(<2 x i64>, <2 x i64>)
+declare <4 x i64>  @llvm.umin.v4i64(<4 x i64>, <4 x i64>)
+declare <8 x i64>  @llvm.umin.v8i64(<8 x i64>, <8 x i64>)
+
+declare i32        @llvm.umin.i32(i32, i32)
+declare <4 x i32>  @llvm.umin.v4i32(<4 x i32>, <4 x i32>)
+declare <8 x i32>  @llvm.umin.v8i32(<8 x i32>, <8 x i32>)
+declare <16 x i32> @llvm.umin.v16i32(<16 x i32>, <16 x i32>)
+
+declare i16        @llvm.umin.i16(i16, i16)
+declare <8 x i16>  @llvm.umin.v8i16(<8 x i16>, <8 x i16>)
+declare <16 x i16> @llvm.umin.v16i16(<16 x i16>, <16 x i16>)
+declare <32 x i16> @llvm.umin.v32i16(<32 x i16>, <32 x i16>)
+
+declare i8         @llvm.umin.i8(i8,  i8)
+declare <16 x i8>  @llvm.umin.v16i8(<16 x i8>, <16 x i8>)
+declare <32 x i8>  @llvm.umin.v32i8(<32 x i8>, <32 x i8>)
+declare <64 x i8>  @llvm.umin.v64i8(<64 x i8>, <64 x i8>)
+
+define i32 @umin(i32 %arg) {
+; SSE2-LABEL: 'umin'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.umin.i64(i64 undef, i64 undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I64 = call <2 x i64> @llvm.umin.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I64 = call <4 x i64> @llvm.umin.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V8I64 = call <8 x i64> @llvm.umin.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.umin.i32(i32 undef, i32 undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4I32 = call <4 x i32> @llvm.umin.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V8I32 = call <8 x i32> @llvm.umin.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V16I32 = call <16 x i32> @llvm.umin.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.umin.i16(i16 undef, i16 undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I16 = call <8 x i16> @llvm.umin.v8i16(<8 x i16> undef, <8 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I16 = call <16 x i16> @llvm.umin.v16i16(<16 x i16> undef, <16 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V32I16 = call <32 x i16> @llvm.umin.v32i16(<32 x i16> undef, <32 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.umin.i8(i8 undef, i8 undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V16I8 = call <16 x i8> @llvm.umin.v16i8(<16 x i8> undef, <16 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 94 for instruction: %V32I8 = call <32 x i8> @llvm.umin.v32i8(<32 x i8> undef, <32 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 188 for instruction: %V64I8 = call <64 x i8> @llvm.umin.v64i8(<64 x i8> undef, <64 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSSE3-LABEL: 'umin'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.umin.i64(i64 undef, i64 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I64 = call <2 x i64> @llvm.umin.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I64 = call <4 x i64> @llvm.umin.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V8I64 = call <8 x i64> @llvm.umin.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.umin.i32(i32 undef, i32 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4I32 = call <4 x i32> @llvm.umin.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V8I32 = call <8 x i32> @llvm.umin.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V16I32 = call <16 x i32> @llvm.umin.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.umin.i16(i16 undef, i16 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I16 = call <8 x i16> @llvm.umin.v8i16(<8 x i16> undef, <8 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I16 = call <16 x i16> @llvm.umin.v16i16(<16 x i16> undef, <16 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V32I16 = call <32 x i16> @llvm.umin.v32i16(<32 x i16> undef, <32 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.umin.i8(i8 undef, i8 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V16I8 = call <16 x i8> @llvm.umin.v16i8(<16 x i8> undef, <16 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 94 for instruction: %V32I8 = call <32 x i8> @llvm.umin.v32i8(<32 x i8> undef, <32 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 188 for instruction: %V64I8 = call <64 x i8> @llvm.umin.v64i8(<64 x i8> undef, <64 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE42-LABEL: 'umin'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.umin.i64(i64 undef, i64 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I64 = call <2 x i64> @llvm.umin.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4I64 = call <4 x i64> @llvm.umin.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I64 = call <8 x i64> @llvm.umin.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.umin.i32(i32 undef, i32 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4I32 = call <4 x i32> @llvm.umin.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I32 = call <8 x i32> @llvm.umin.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I32 = call <16 x i32> @llvm.umin.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.umin.i16(i16 undef, i16 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I16 = call <8 x i16> @llvm.umin.v8i16(<8 x i16> undef, <8 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I16 = call <16 x i16> @llvm.umin.v16i16(<16 x i16> undef, <16 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V32I16 = call <32 x i16> @llvm.umin.v32i16(<32 x i16> undef, <32 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.umin.i8(i8 undef, i8 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I8 = call <16 x i8> @llvm.umin.v16i8(<16 x i8> undef, <16 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V32I8 = call <32 x i8> @llvm.umin.v32i8(<32 x i8> undef, <32 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %V64I8 = call <64 x i8> @llvm.umin.v64i8(<64 x i8> undef, <64 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX-LABEL: 'umin'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.umin.i64(i64 undef, i64 undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I64 = call <2 x i64> @llvm.umin.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I64 = call <4 x i64> @llvm.umin.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V8I64 = call <8 x i64> @llvm.umin.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.umin.i32(i32 undef, i32 undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4I32 = call <4 x i32> @llvm.umin.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I32 = call <8 x i32> @llvm.umin.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V16I32 = call <16 x i32> @llvm.umin.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.umin.i16(i16 undef, i16 undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I16 = call <8 x i16> @llvm.umin.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V16I16 = call <16 x i16> @llvm.umin.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %V32I16 = call <32 x i16> @llvm.umin.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.umin.i8(i8 undef, i8 undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I8 = call <16 x i8> @llvm.umin.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: %V32I8 = call <32 x i8> @llvm.umin.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %V64I8 = call <64 x i8> @llvm.umin.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512-LABEL: 'umin'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.umin.i64(i64 undef, i64 undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I64 = call <2 x i64> @llvm.umin.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I64 = call <4 x i64> @llvm.umin.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V8I64 = call <8 x i64> @llvm.umin.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.umin.i32(i32 undef, i32 undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4I32 = call <4 x i32> @llvm.umin.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I32 = call <8 x i32> @llvm.umin.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V16I32 = call <16 x i32> @llvm.umin.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.umin.i16(i16 undef, i16 undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I16 = call <8 x i16> @llvm.umin.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V16I16 = call <16 x i16> @llvm.umin.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %V32I16 = call <32 x i16> @llvm.umin.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.umin.i8(i8 undef, i8 undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I8 = call <16 x i8> @llvm.umin.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: %V32I8 = call <32 x i8> @llvm.umin.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 131 for instruction: %V64I8 = call <64 x i8> @llvm.umin.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %I64 = call i64 @llvm.umin.i64(i64 undef, i64 undef)
+  %V2I64 = call <2 x i64> @llvm.umin.v2i64(<2 x i64> undef, <2 x i64> undef)
+  %V4I64 = call <4 x i64> @llvm.umin.v4i64(<4 x i64> undef, <4 x i64> undef)
+  %V8I64 = call <8 x i64> @llvm.umin.v8i64(<8 x i64> undef, <8 x i64> undef)
+
+  %I32 = call i32 @llvm.umin.i32(i32 undef, i32 undef)
+  %V4I32  = call <4 x i32>  @llvm.umin.v4i32(<4 x i32> undef, <4 x i32> undef)
+  %V8I32  = call <8 x i32>  @llvm.umin.v8i32(<8 x i32> undef, <8 x i32> undef)
+  %V16I32 = call <16 x i32> @llvm.umin.v16i32(<16 x i32> undef, <16 x i32> undef)
+
+  %I16 = call i16 @llvm.umin.i16(i16 undef, i16 undef)
+  %V8I16  = call <8 x i16>  @llvm.umin.v8i16(<8 x i16> undef, <8 x i16> undef)
+  %V16I16 = call <16 x i16> @llvm.umin.v16i16(<16 x i16> undef, <16 x i16> undef)
+  %V32I16 = call <32 x i16> @llvm.umin.v32i16(<32 x i16> undef, <32 x i16> undef)
+
+  %I8 = call i8 @llvm.umin.i8(i8 undef, i8 undef)
+  %V16I8 = call <16 x i8> @llvm.umin.v16i8(<16 x i8> undef, <16 x i8> undef)
+  %V32I8 = call <32 x i8> @llvm.umin.v32i8(<32 x i8> undef, <32 x i8> undef)
+  %V64I8 = call <64 x i8> @llvm.umin.v64i8(<64 x i8> undef, <64 x i8> undef)
+
+  ret i32 undef
+}

From 6f00f3b56e5a13286142facd929be15ab7b17aa3 Mon Sep 17 00:00:00 2001
From: Kostya Kortchinsky <kostyak@google.com>
Date: Mon, 27 Jul 2020 09:13:42 -0700
Subject: [PATCH 0346/1035] [scudo][standalone] mallopt runtime configuration
 options

Summary:
Partners have requested the ability to configure more parts of Scudo
at runtime, notably the Secondary cache options (maximum number of
blocks cached, maximum size) as well as the TSD registry options
(the maximum number of TSDs in use).

This CL adds a few more Scudo specific `mallopt` parameters that are
passed down to the various subcomponents of the Combined allocator.

- `M_CACHE_COUNT_MAX`: sets the maximum number of Secondary cached items
- `M_CACHE_SIZE_MAX`: sets the maximum size of a cacheable item in the Secondary
- `M_TSDS_COUNT_MAX`: sets the maximum number of TSDs that can be used (Shared Registry only)

Regarding the TSDs maximum count, this is a one way option, only
allowing to increase the count.

In order to allow for this, I rearranged the code to have some `setOption`
member function to the relevant classes, using the `scudo::Option` class
enum to determine what is to be set.

This also fixes an issue where a static variable (`Ready`) was used in
templated functions without being set back to `false` every time.

Reviewers: pcc, eugenis, hctim, cferris

Subscribers: jfb, llvm-commits, #sanitizers

Tags: #sanitizers

Differential Revision: https://reviews.llvm.org/D84667
---
 .../lib/scudo/standalone/allocator_config.h   |  11 +-
 compiler-rt/lib/scudo/standalone/combined.h   |  46 ++++----
 compiler-rt/lib/scudo/standalone/common.h     |   8 ++
 .../standalone/include/scudo/interface.h      |  12 ++
 compiler-rt/lib/scudo/standalone/primary32.h  |  24 ++--
 compiler-rt/lib/scudo/standalone/primary64.h  |  24 ++--
 compiler-rt/lib/scudo/standalone/secondary.h  |  89 +++++++++------
 .../scudo/standalone/tests/combined_test.cpp  |   5 +-
 .../scudo/standalone/tests/primary_test.cpp   |   3 +-
 .../scudo/standalone/tests/secondary_test.cpp |  40 +++++--
 .../lib/scudo/standalone/tests/tsd_test.cpp   |  78 ++++++++++++-
 .../standalone/tests/wrappers_c_test.cpp      |   1 +
 .../standalone/tests/wrappers_cpp_test.cpp    |   3 +-
 .../lib/scudo/standalone/tsd_exclusive.h      |   6 +
 compiler-rt/lib/scudo/standalone/tsd_shared.h | 105 ++++++++++++------
 15 files changed, 318 insertions(+), 137 deletions(-)

diff --git a/compiler-rt/lib/scudo/standalone/allocator_config.h b/compiler-rt/lib/scudo/standalone/allocator_config.h
index ad2a17ef7014a..cf362da4e5be5 100644
--- a/compiler-rt/lib/scudo/standalone/allocator_config.h
+++ b/compiler-rt/lib/scudo/standalone/allocator_config.h
@@ -48,9 +48,10 @@ struct AndroidConfig {
   typedef SizeClassAllocator32<SizeClassMap, 18U, 1000, 1000> Primary;
 #endif
   // Cache blocks up to 2MB
-  typedef MapAllocator<MapAllocatorCache<32U, 2UL << 20, 0, 1000>> Secondary;
+  typedef MapAllocator<MapAllocatorCache<256U, 32U, 2UL << 20, 0, 1000>>
+      Secondary;
   template <class A>
-  using TSDRegistryT = TSDRegistrySharedT<A, 2U>; // Shared, max 2 TSDs.
+  using TSDRegistryT = TSDRegistrySharedT<A, 8U, 2U>; // Shared, max 8 TSDs.
 };
 
 struct AndroidSvelteConfig {
@@ -62,9 +63,9 @@ struct AndroidSvelteConfig {
   // 64KB regions
   typedef SizeClassAllocator32<SizeClassMap, 16U, 1000, 1000> Primary;
 #endif
-  typedef MapAllocator<MapAllocatorCache<4U, 1UL << 18, 0, 0>> Secondary;
+  typedef MapAllocator<MapAllocatorCache<16U, 4U, 1UL << 18, 0, 0>> Secondary;
   template <class A>
-  using TSDRegistryT = TSDRegistrySharedT<A, 1U>; // Shared, only 1 TSD.
+  using TSDRegistryT = TSDRegistrySharedT<A, 2U, 1U>; // Shared, max 2 TSDs.
 };
 
 #if SCUDO_CAN_USE_PRIMARY64
@@ -73,7 +74,7 @@ struct FuchsiaConfig {
   typedef SizeClassAllocator64<DefaultSizeClassMap, 30U> Primary;
   typedef MapAllocator<MapAllocatorNoCache> Secondary;
   template <class A>
-  using TSDRegistryT = TSDRegistrySharedT<A, 8U>; // Shared, max 8 TSDs.
+  using TSDRegistryT = TSDRegistrySharedT<A, 8U, 4U>; // Shared, max 8 TSDs.
 };
 #endif
 
diff --git a/compiler-rt/lib/scudo/standalone/combined.h b/compiler-rt/lib/scudo/standalone/combined.h
index 582178ee36b40..6ca00c29ab732 100644
--- a/compiler-rt/lib/scudo/standalone/combined.h
+++ b/compiler-rt/lib/scudo/standalone/combined.h
@@ -41,8 +41,6 @@ extern "C" size_t android_unsafe_frame_pointer_chase(scudo::uptr *buf,
 
 namespace scudo {
 
-enum class Option { ReleaseInterval, MemtagTuning };
-
 template <class Params, void (*PostInitCallback)(void) = EmptyCallback>
 class Allocator {
 public:
@@ -277,7 +275,7 @@ class Allocator {
     }
 #endif // GWP_ASAN_HOOKS
 
-    FillContentsMode FillContents =
+    const FillContentsMode FillContents =
         ZeroContents ? ZeroFill : Options.FillContents;
 
     if (UNLIKELY(Alignment > MaxAlignment)) {
@@ -285,7 +283,7 @@ class Allocator {
         return nullptr;
       reportAlignmentTooBig(Alignment, MaxAlignment);
     }
-    if (Alignment < MinAlignment)
+    if (UNLIKELY(Alignment < MinAlignment))
       Alignment = MinAlignment;
 
     // If the requested size happens to be 0 (more common than you might think),
@@ -322,13 +320,11 @@ class Allocator {
       if (UNLIKELY(!Block)) {
         while (ClassId < SizeClassMap::LargestClassId) {
           Block = TSD->Cache.allocate(++ClassId);
-          if (LIKELY(Block)) {
+          if (LIKELY(Block))
             break;
-          }
         }
-        if (UNLIKELY(!Block)) {
+        if (UNLIKELY(!Block))
           ClassId = 0;
-        }
       }
       if (UnlockRequired)
         TSD->unlock();
@@ -349,7 +345,7 @@ class Allocator {
 
     void *Ptr = reinterpret_cast<void *>(UserPtr);
     void *TaggedPtr = Ptr;
-    if (ClassId) {
+    if (LIKELY(ClassId)) {
       // We only need to zero or tag the contents for Primary backed
       // allocations. We only set tags for primary allocations in order to avoid
       // faulting potentially large numbers of pages for large secondary
@@ -692,11 +688,7 @@ class Allocator {
   }
 
   bool setOption(Option O, sptr Value) {
-    if (O == Option::ReleaseInterval) {
-      Primary.setReleaseToOsIntervalMs(static_cast<s32>(Value));
-      Secondary.setReleaseToOsIntervalMs(static_cast<s32>(Value));
-      return true;
-    }
+    initThreadMaybe();
     if (O == Option::MemtagTuning) {
       // Enabling odd/even tags involves a tradeoff between use-after-free
       // detection and buffer overflow detection. Odd/even tags make it more
@@ -705,14 +697,19 @@ class Allocator {
       // use-after-free is less likely to be detected because the tag space for
       // any particular chunk is cut in half. Therefore we use this tuning
       // setting to control whether odd/even tags are enabled.
-      if (Value == M_MEMTAG_TUNING_BUFFER_OVERFLOW) {
+      if (Value == M_MEMTAG_TUNING_BUFFER_OVERFLOW)
         Options.UseOddEvenTags = true;
-        return true;
-      }
-      if (Value == M_MEMTAG_TUNING_UAF) {
+      else if (Value == M_MEMTAG_TUNING_UAF)
         Options.UseOddEvenTags = false;
-        return true;
-      }
+      return true;
+    } else {
+      // We leave it to the various sub-components to decide whether or not they
+      // want to handle the option, but we do not want to short-circuit
+      // execution if one of the setOption was to return false.
+      const bool PrimaryResult = Primary.setOption(O, Value);
+      const bool SecondaryResult = Secondary.setOption(O, Value);
+      const bool RegistryResult = TSDRegistry.setOption(O, Value);
+      return PrimaryResult && SecondaryResult && RegistryResult;
     }
     return false;
   }
@@ -805,8 +802,7 @@ class Allocator {
         PrimaryT::findNearestBlock(RegionInfoPtr, UntaggedFaultAddr);
 
     auto GetGranule = [&](uptr Addr, const char **Data, uint8_t *Tag) -> bool {
-      if (Addr < MemoryAddr ||
-          Addr + archMemoryTagGranuleSize() < Addr ||
+      if (Addr < MemoryAddr || Addr + archMemoryTagGranuleSize() < Addr ||
           Addr + archMemoryTagGranuleSize() > MemoryAddr + MemorySize)
         return false;
       *Data = &Memory[Addr - MemoryAddr];
@@ -950,10 +946,10 @@ class Allocator {
   u32 Cookie;
 
   struct {
-    u8 MayReturnNull : 1;       // may_return_null
+    u8 MayReturnNull : 1;              // may_return_null
     FillContentsMode FillContents : 2; // zero_contents, pattern_fill_contents
-    u8 DeallocTypeMismatch : 1; // dealloc_type_mismatch
-    u8 DeleteSizeMismatch : 1;  // delete_size_mismatch
+    u8 DeallocTypeMismatch : 1;        // dealloc_type_mismatch
+    u8 DeleteSizeMismatch : 1;         // delete_size_mismatch
     u8 TrackAllocationStacks : 1;
     u8 UseOddEvenTags : 1;
     u32 QuarantineMaxChunkSize; // quarantine_max_chunk_size
diff --git a/compiler-rt/lib/scudo/standalone/common.h b/compiler-rt/lib/scudo/standalone/common.h
index 9037f92b4976c..b3bce6ee291aa 100644
--- a/compiler-rt/lib/scudo/standalone/common.h
+++ b/compiler-rt/lib/scudo/standalone/common.h
@@ -182,6 +182,14 @@ struct BlockInfo {
   uptr RegionEnd;
 };
 
+enum class Option : u8 {
+  ReleaseInterval,      // Release to OS interval in milliseconds.
+  MemtagTuning,         // Whether to tune tagging for UAF or overflow.
+  MaxCacheEntriesCount, // Maximum number of blocks that can be cached.
+  MaxCacheEntrySize,    // Maximum size of a block that can be cached.
+  MaxTSDsCount,         // Number of usable TSDs for the shared registry.
+};
+
 constexpr unsigned char PatternFillByte = 0xAB;
 
 enum FillContentsMode {
diff --git a/compiler-rt/lib/scudo/standalone/include/scudo/interface.h b/compiler-rt/lib/scudo/standalone/include/scudo/interface.h
index 27c1684596c44..7e65b68ab36d0 100644
--- a/compiler-rt/lib/scudo/standalone/include/scudo/interface.h
+++ b/compiler-rt/lib/scudo/standalone/include/scudo/interface.h
@@ -121,6 +121,18 @@ size_t __scudo_get_region_info_size();
 #define M_MEMTAG_TUNING -102
 #endif
 
+#ifndef M_CACHE_COUNT_MAX
+#define M_CACHE_COUNT_MAX -200
+#endif
+
+#ifndef M_CACHE_SIZE_MAX
+#define M_CACHE_SIZE_MAX -201
+#endif
+
+#ifndef M_TSDS_COUNT_MAX
+#define M_TSDS_COUNT_MAX -202
+#endif
+
 enum scudo_memtag_tuning {
   // Tune for buffer overflows.
   M_MEMTAG_TUNING_BUFFER_OVERFLOW,
diff --git a/compiler-rt/lib/scudo/standalone/primary32.h b/compiler-rt/lib/scudo/standalone/primary32.h
index 321cf92fae30e..e41b949d8d0e6 100644
--- a/compiler-rt/lib/scudo/standalone/primary32.h
+++ b/compiler-rt/lib/scudo/standalone/primary32.h
@@ -86,7 +86,7 @@ class SizeClassAllocator32 {
       if (Sci->CanRelease)
         Sci->ReleaseInfo.LastReleaseAtNs = Time;
     }
-    setReleaseToOsIntervalMs(ReleaseToOsInterval);
+    setOption(Option::ReleaseInterval, static_cast<sptr>(ReleaseToOsInterval));
   }
   void init(s32 ReleaseToOsInterval) {
     memset(this, 0, sizeof(*this));
@@ -184,13 +184,16 @@ class SizeClassAllocator32 {
       getStats(Str, I, 0);
   }
 
-  void setReleaseToOsIntervalMs(s32 Interval) {
-    if (Interval >= MaxReleaseToOsIntervalMs) {
-      Interval = MaxReleaseToOsIntervalMs;
-    } else if (Interval <= MinReleaseToOsIntervalMs) {
-      Interval = MinReleaseToOsIntervalMs;
+  bool setOption(Option O, sptr Value) {
+    if (O == Option::ReleaseInterval) {
+      const s32 Interval =
+          Max(Min(static_cast<s32>(Value), MaxReleaseToOsIntervalMs),
+              MinReleaseToOsIntervalMs);
+      atomic_store(&ReleaseToOsIntervalMs, Interval, memory_order_relaxed);
+      return true;
     }
-    atomic_store(&ReleaseToOsIntervalMs, Interval, memory_order_relaxed);
+    // Not supported by the Primary, but not an error either.
+    return true;
   }
 
   uptr releaseToOS() {
@@ -423,10 +426,6 @@ class SizeClassAllocator32 {
                 AvailableChunks, Rss >> 10, Sci->ReleaseInfo.RangesReleased);
   }
 
-  s32 getReleaseToOsIntervalMs() {
-    return atomic_load(&ReleaseToOsIntervalMs, memory_order_relaxed);
-  }
-
   NOINLINE uptr releaseToOSMaybe(SizeClassInfo *Sci, uptr ClassId,
                                  bool Force = false) {
     const uptr BlockSize = getSizeByClassId(ClassId);
@@ -457,7 +456,8 @@ class SizeClassAllocator32 {
     }
 
     if (!Force) {
-      const s32 IntervalMs = getReleaseToOsIntervalMs();
+      const s32 IntervalMs =
+          atomic_load(&ReleaseToOsIntervalMs, memory_order_relaxed);
       if (IntervalMs < 0)
         return 0;
       if (Sci->ReleaseInfo.LastReleaseAtNs +
diff --git a/compiler-rt/lib/scudo/standalone/primary64.h b/compiler-rt/lib/scudo/standalone/primary64.h
index e37dc4951f238..ad92ae250e1f4 100644
--- a/compiler-rt/lib/scudo/standalone/primary64.h
+++ b/compiler-rt/lib/scudo/standalone/primary64.h
@@ -91,7 +91,7 @@ class SizeClassAllocator64 {
       if (Region->CanRelease)
         Region->ReleaseInfo.LastReleaseAtNs = Time;
     }
-    setReleaseToOsIntervalMs(ReleaseToOsInterval);
+    setOption(Option::ReleaseInterval, static_cast<sptr>(ReleaseToOsInterval));
 
     if (SupportsMemoryTagging)
       UseMemoryTagging = systemSupportsMemoryTagging();
@@ -185,13 +185,16 @@ class SizeClassAllocator64 {
       getStats(Str, I, 0);
   }
 
-  void setReleaseToOsIntervalMs(s32 Interval) {
-    if (Interval >= MaxReleaseToOsIntervalMs) {
-      Interval = MaxReleaseToOsIntervalMs;
-    } else if (Interval <= MinReleaseToOsIntervalMs) {
-      Interval = MinReleaseToOsIntervalMs;
+  bool setOption(Option O, sptr Value) {
+    if (O == Option::ReleaseInterval) {
+      const s32 Interval =
+          Max(Min(static_cast<s32>(Value), MaxReleaseToOsIntervalMs),
+              MinReleaseToOsIntervalMs);
+      atomic_store(&ReleaseToOsIntervalMs, Interval, memory_order_relaxed);
+      return true;
     }
-    atomic_store(&ReleaseToOsIntervalMs, Interval, memory_order_relaxed);
+    // Not supported by the Primary, but not an error either.
+    return true;
   }
 
   uptr releaseToOS() {
@@ -435,10 +438,6 @@ class SizeClassAllocator64 {
                 getRegionBaseByClassId(ClassId));
   }
 
-  s32 getReleaseToOsIntervalMs() {
-    return atomic_load(&ReleaseToOsIntervalMs, memory_order_relaxed);
-  }
-
   NOINLINE uptr releaseToOSMaybe(RegionInfo *Region, uptr ClassId,
                                  bool Force = false) {
     const uptr BlockSize = getSizeByClassId(ClassId);
@@ -469,7 +468,8 @@ class SizeClassAllocator64 {
     }
 
     if (!Force) {
-      const s32 IntervalMs = getReleaseToOsIntervalMs();
+      const s32 IntervalMs =
+          atomic_load(&ReleaseToOsIntervalMs, memory_order_relaxed);
       if (IntervalMs < 0)
         return 0;
       if (Region->ReleaseInfo.LastReleaseAtNs +
diff --git a/compiler-rt/lib/scudo/standalone/secondary.h b/compiler-rt/lib/scudo/standalone/secondary.h
index 84eaa5091b434..b5bb53ddcf2d9 100644
--- a/compiler-rt/lib/scudo/standalone/secondary.h
+++ b/compiler-rt/lib/scudo/standalone/secondary.h
@@ -56,14 +56,21 @@ class MapAllocatorNoCache {
     return false;
   }
   bool store(UNUSED LargeBlock::Header *H) { return false; }
-  static bool canCache(UNUSED uptr Size) { return false; }
+  bool canCache(UNUSED uptr Size) { return false; }
   void disable() {}
   void enable() {}
   void releaseToOS() {}
-  void setReleaseToOsIntervalMs(UNUSED s32 Interval) {}
+  bool setOption(Option O, UNUSED sptr Value) {
+    if (O == Option::ReleaseInterval || O == Option::MaxCacheEntriesCount ||
+        O == Option::MaxCacheEntrySize)
+      return false;
+    // Not supported by the Secondary Cache, but not an error either.
+    return true;
+  }
 };
 
-template <uptr MaxEntriesCount = 32U, uptr MaxEntrySize = 1UL << 19,
+template <u32 EntriesArraySize = 32U, u32 DefaultMaxEntriesCount = 32U,
+          uptr DefaultMaxEntrySize = 1UL << 19,
           s32 MinReleaseToOsIntervalMs = INT32_MIN,
           s32 MaxReleaseToOsIntervalMs = INT32_MAX>
 class MapAllocatorCache {
@@ -71,10 +78,17 @@ class MapAllocatorCache {
   // Fuchsia doesn't allow releasing Secondary blocks yet. Note that 0 length
   // arrays are an extension for some compilers.
   // FIXME(kostyak): support (partially) the cache on Fuchsia.
-  static_assert(!SCUDO_FUCHSIA || MaxEntriesCount == 0U, "");
+  static_assert(!SCUDO_FUCHSIA || EntriesArraySize == 0U, "");
+
+  // Ensure the default maximum specified fits the array.
+  static_assert(DefaultMaxEntriesCount <= EntriesArraySize, "");
 
   void initLinkerInitialized(s32 ReleaseToOsInterval) {
-    setReleaseToOsIntervalMs(ReleaseToOsInterval);
+    setOption(Option::MaxCacheEntriesCount,
+              static_cast<sptr>(DefaultMaxEntriesCount));
+    setOption(Option::MaxCacheEntrySize,
+              static_cast<sptr>(DefaultMaxEntrySize));
+    setOption(Option::ReleaseInterval, static_cast<sptr>(ReleaseToOsInterval));
   }
   void init(s32 ReleaseToOsInterval) {
     memset(this, 0, sizeof(*this));
@@ -85,13 +99,14 @@ class MapAllocatorCache {
     bool EntryCached = false;
     bool EmptyCache = false;
     const u64 Time = getMonotonicTime();
+    const u32 MaxCount = atomic_load(&MaxEntriesCount, memory_order_relaxed);
     {
       ScopedLock L(Mutex);
-      if (EntriesCount == MaxEntriesCount) {
+      if (EntriesCount >= MaxCount) {
         if (IsFullEvents++ == 4U)
           EmptyCache = true;
       } else {
-        for (uptr I = 0; I < MaxEntriesCount; I++) {
+        for (u32 I = 0; I < MaxCount; I++) {
           if (Entries[I].Block)
             continue;
           if (I != 0)
@@ -111,17 +126,19 @@ class MapAllocatorCache {
     s32 Interval;
     if (EmptyCache)
       empty();
-    else if ((Interval = getReleaseToOsIntervalMs()) >= 0)
+    else if ((Interval = atomic_load(&ReleaseToOsIntervalMs,
+                                     memory_order_relaxed)) >= 0)
       releaseOlderThan(Time - static_cast<u64>(Interval) * 1000000);
     return EntryCached;
   }
 
   bool retrieve(uptr Size, LargeBlock::Header **H) {
     const uptr PageSize = getPageSizeCached();
+    const u32 MaxCount = atomic_load(&MaxEntriesCount, memory_order_relaxed);
     ScopedLock L(Mutex);
     if (EntriesCount == 0)
       return false;
-    for (uptr I = 0; I < MaxEntriesCount; I++) {
+    for (u32 I = 0; I < MaxCount; I++) {
       if (!Entries[I].Block)
         continue;
       const uptr BlockSize = Entries[I].BlockEnd - Entries[I].Block;
@@ -141,17 +158,31 @@ class MapAllocatorCache {
     return false;
   }
 
-  static bool canCache(uptr Size) {
-    return MaxEntriesCount != 0U && Size <= MaxEntrySize;
+  bool canCache(uptr Size) {
+    return atomic_load(&MaxEntriesCount, memory_order_relaxed) != 0U &&
+           Size <= atomic_load(&MaxEntrySize, memory_order_relaxed);
   }
 
-  void setReleaseToOsIntervalMs(s32 Interval) {
-    if (Interval >= MaxReleaseToOsIntervalMs) {
-      Interval = MaxReleaseToOsIntervalMs;
-    } else if (Interval <= MinReleaseToOsIntervalMs) {
-      Interval = MinReleaseToOsIntervalMs;
+  bool setOption(Option O, sptr Value) {
+    if (O == Option::ReleaseInterval) {
+      const s32 Interval =
+          Max(Min(static_cast<s32>(Value), MaxReleaseToOsIntervalMs),
+              MinReleaseToOsIntervalMs);
+      atomic_store(&ReleaseToOsIntervalMs, Interval, memory_order_relaxed);
+      return true;
+    } else if (O == Option::MaxCacheEntriesCount) {
+      const u32 MaxCount = static_cast<u32>(Value);
+      if (MaxCount > EntriesArraySize)
+        return false;
+      atomic_store(&MaxEntriesCount, MaxCount, memory_order_relaxed);
+      return true;
+    } else if (O == Option::MaxCacheEntrySize) {
+      atomic_store(&MaxEntrySize, static_cast<uptr>(Value),
+                   memory_order_relaxed);
+      return true;
     }
-    atomic_store(&ReleaseToOsIntervalMs, Interval, memory_order_relaxed);
+    // Not supported by the Secondary Cache, but not an error either.
+    return true;
   }
 
   void releaseToOS() { releaseOlderThan(UINT64_MAX); }
@@ -166,11 +197,11 @@ class MapAllocatorCache {
       void *MapBase;
       uptr MapSize;
       MapPlatformData Data;
-    } MapInfo[MaxEntriesCount];
+    } MapInfo[EntriesArraySize];
     uptr N = 0;
     {
       ScopedLock L(Mutex);
-      for (uptr I = 0; I < MaxEntriesCount; I++) {
+      for (uptr I = 0; I < EntriesArraySize; I++) {
         if (!Entries[I].Block)
           continue;
         MapInfo[N].MapBase = reinterpret_cast<void *>(Entries[I].MapBase);
@@ -191,7 +222,7 @@ class MapAllocatorCache {
     ScopedLock L(Mutex);
     if (!EntriesCount)
       return;
-    for (uptr I = 0; I < MaxEntriesCount; I++) {
+    for (uptr I = 0; I < EntriesArraySize; I++) {
       if (!Entries[I].Block || !Entries[I].Time || Entries[I].Time > Time)
         continue;
       releasePagesToOS(Entries[I].Block, 0,
@@ -201,10 +232,6 @@ class MapAllocatorCache {
     }
   }
 
-  s32 getReleaseToOsIntervalMs() {
-    return atomic_load(&ReleaseToOsIntervalMs, memory_order_relaxed);
-  }
-
   struct CachedBlock {
     uptr Block;
     uptr BlockEnd;
@@ -215,8 +242,10 @@ class MapAllocatorCache {
   };
 
   HybridMutex Mutex;
-  CachedBlock Entries[MaxEntriesCount];
+  CachedBlock Entries[EntriesArraySize];
   u32 EntriesCount;
+  atomic_u32 MaxEntriesCount;
+  atomic_uptr MaxEntrySize;
   uptr LargestSize;
   u32 IsFullEvents;
   atomic_s32 ReleaseToOsIntervalMs;
@@ -265,11 +294,9 @@ template <class CacheT> class MapAllocator {
       Callback(reinterpret_cast<uptr>(&H) + LargeBlock::getHeaderSize());
   }
 
-  static uptr canCache(uptr Size) { return CacheT::canCache(Size); }
+  uptr canCache(uptr Size) { return Cache.canCache(Size); }
 
-  void setReleaseToOsIntervalMs(s32 Interval) {
-    Cache.setReleaseToOsIntervalMs(Interval);
-  }
+  bool setOption(Option O, sptr Value) { return Cache.setOption(O, Value); }
 
   void releaseToOS() { Cache.releaseToOS(); }
 
@@ -306,7 +333,7 @@ void *MapAllocator<CacheT>::allocate(uptr Size, uptr AlignmentHint,
   const uptr RoundedSize =
       roundUpTo(Size + LargeBlock::getHeaderSize(), PageSize);
 
-  if (AlignmentHint < PageSize && CacheT::canCache(RoundedSize)) {
+  if (AlignmentHint < PageSize && Cache.canCache(RoundedSize)) {
     LargeBlock::Header *H;
     if (Cache.retrieve(RoundedSize, &H)) {
       if (BlockEnd)
@@ -400,7 +427,7 @@ template <class CacheT> void MapAllocator<CacheT>::deallocate(void *Ptr) {
     Stats.sub(StatAllocated, CommitSize);
     Stats.sub(StatMapped, H->MapSize);
   }
-  if (CacheT::canCache(CommitSize) && Cache.store(H))
+  if (Cache.canCache(CommitSize) && Cache.store(H))
     return;
   void *Addr = reinterpret_cast<void *>(H->MapBase);
   const uptr Size = H->MapSize;
diff --git a/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp b/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp
index 6cefe18b8f15c..9689c4265e06c 100644
--- a/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp
+++ b/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp
@@ -19,7 +19,7 @@
 
 static std::mutex Mutex;
 static std::condition_variable Cv;
-static bool Ready = false;
+static bool Ready;
 
 static constexpr scudo::Chunk::Origin Origin = scudo::Chunk::Origin::Malloc;
 
@@ -351,6 +351,7 @@ template <typename AllocatorT> static void stressAllocator(AllocatorT *A) {
 }
 
 template <class Config> static void testAllocatorThreaded() {
+  Ready = false;
   using AllocatorT = TestAllocator<Config>;
   auto Allocator = std::unique_ptr<AllocatorT>(new AllocatorT());
   std::thread Threads[32];
@@ -394,7 +395,7 @@ struct DeathConfig {
   typedef scudo::SizeClassAllocator64<DeathSizeClassMap, DeathRegionSizeLog>
       Primary;
   typedef scudo::MapAllocator<scudo::MapAllocatorNoCache> Secondary;
-  template <class A> using TSDRegistryT = scudo::TSDRegistrySharedT<A, 1U>;
+  template <class A> using TSDRegistryT = scudo::TSDRegistrySharedT<A, 1U, 1U>;
 };
 
 TEST(ScudoCombinedTest, DeathCombined) {
diff --git a/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp b/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp
index 010bf84490e0e..a7a2b3160611e 100644
--- a/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp
+++ b/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp
@@ -149,7 +149,7 @@ TEST(ScudoPrimaryTest, PrimaryIterate) {
 
 static std::mutex Mutex;
 static std::condition_variable Cv;
-static bool Ready = false;
+static bool Ready;
 
 template <typename Primary> static void performAllocations(Primary *Allocator) {
   static THREADLOCAL typename Primary::CacheT Cache;
@@ -176,6 +176,7 @@ template <typename Primary> static void performAllocations(Primary *Allocator) {
 }
 
 template <typename Primary> static void testPrimaryThreaded() {
+  Ready = false;
   auto Deleter = [](Primary *P) {
     P->unmapTestOnly();
     delete P;
diff --git a/compiler-rt/lib/scudo/standalone/tests/secondary_test.cpp b/compiler-rt/lib/scudo/standalone/tests/secondary_test.cpp
index d2260b9c15b07..29efdb3060128 100644
--- a/compiler-rt/lib/scudo/standalone/tests/secondary_test.cpp
+++ b/compiler-rt/lib/scudo/standalone/tests/secondary_test.cpp
@@ -21,7 +21,7 @@
 template <class SecondaryT> static void testSecondaryBasic(void) {
   scudo::GlobalStats S;
   S.init();
-  SecondaryT *L = new SecondaryT;
+  std::unique_ptr<SecondaryT> L(new SecondaryT);
   L->init(&S);
   const scudo::uptr Size = 1U << 16;
   void *P = L->allocate(Size);
@@ -30,7 +30,7 @@ template <class SecondaryT> static void testSecondaryBasic(void) {
   EXPECT_GE(SecondaryT::getBlockSize(P), Size);
   L->deallocate(P);
   // If the Secondary can't cache that pointer, it will be unmapped.
-  if (!SecondaryT::canCache(Size))
+  if (!L->canCache(Size))
     EXPECT_DEATH(memset(P, 'A', Size), "");
 
   const scudo::uptr Align = 1U << 16;
@@ -59,7 +59,7 @@ TEST(ScudoSecondaryTest, SecondaryBasic) {
 #if !SCUDO_FUCHSIA
   testSecondaryBasic<scudo::MapAllocator<scudo::MapAllocatorCache<>>>();
   testSecondaryBasic<
-      scudo::MapAllocator<scudo::MapAllocatorCache<64U, 1UL << 20>>>();
+      scudo::MapAllocator<scudo::MapAllocatorCache<128U, 64U, 1UL << 20>>>();
 #endif
 }
 
@@ -75,7 +75,7 @@ using LargeAllocator = scudo::MapAllocator<scudo::MapAllocatorCache<>>;
 TEST(ScudoSecondaryTest, SecondaryCombinations) {
   constexpr scudo::uptr MinAlign = FIRST_32_SECOND_64(8, 16);
   constexpr scudo::uptr HeaderSize = scudo::roundUpTo(8, MinAlign);
-  LargeAllocator *L = new LargeAllocator;
+  std::unique_ptr<LargeAllocator> L(new LargeAllocator);
   L->init(nullptr);
   for (scudo::uptr SizeLog = 0; SizeLog <= 20; SizeLog++) {
     for (scudo::uptr AlignLog = FIRST_32_SECOND_64(3, 4); AlignLog <= 16;
@@ -103,7 +103,7 @@ TEST(ScudoSecondaryTest, SecondaryCombinations) {
 }
 
 TEST(ScudoSecondaryTest, SecondaryIterate) {
-  LargeAllocator *L = new LargeAllocator;
+  std::unique_ptr<LargeAllocator> L(new LargeAllocator);
   L->init(nullptr);
   std::vector<void *> V;
   const scudo::uptr PageSize = scudo::getPageSizeCached();
@@ -125,9 +125,32 @@ TEST(ScudoSecondaryTest, SecondaryIterate) {
   Str.output();
 }
 
+TEST(ScudoSecondaryTest, SecondaryOptions) {
+  std::unique_ptr<LargeAllocator> L(new LargeAllocator);
+  L->init(nullptr);
+  // Attempt to set a maximum number of entries higher than the array size.
+  EXPECT_FALSE(L->setOption(scudo::Option::MaxCacheEntriesCount, 4096U));
+  // A negative number will be cast to a scudo::u32, and fail.
+  EXPECT_FALSE(L->setOption(scudo::Option::MaxCacheEntriesCount, -1));
+  if (L->canCache(0U)) {
+    // Various valid combinations.
+    EXPECT_TRUE(L->setOption(scudo::Option::MaxCacheEntriesCount, 4U));
+    EXPECT_TRUE(L->setOption(scudo::Option::MaxCacheEntrySize, 1UL << 20));
+    EXPECT_TRUE(L->canCache(1UL << 18));
+    EXPECT_TRUE(L->setOption(scudo::Option::MaxCacheEntrySize, 1UL << 17));
+    EXPECT_FALSE(L->canCache(1UL << 18));
+    EXPECT_TRUE(L->canCache(1UL << 16));
+    EXPECT_TRUE(L->setOption(scudo::Option::MaxCacheEntriesCount, 0U));
+    EXPECT_FALSE(L->canCache(1UL << 16));
+    EXPECT_TRUE(L->setOption(scudo::Option::MaxCacheEntriesCount, 4U));
+    EXPECT_TRUE(L->setOption(scudo::Option::MaxCacheEntrySize, 1UL << 20));
+    EXPECT_TRUE(L->canCache(1UL << 16));
+  }
+}
+
 static std::mutex Mutex;
 static std::condition_variable Cv;
-static bool Ready = false;
+static bool Ready;
 
 static void performAllocations(LargeAllocator *L) {
   std::vector<void *> V;
@@ -153,11 +176,12 @@ static void performAllocations(LargeAllocator *L) {
 }
 
 TEST(ScudoSecondaryTest, SecondaryThreadsRace) {
-  LargeAllocator *L = new LargeAllocator;
+  Ready = false;
+  std::unique_ptr<LargeAllocator> L(new LargeAllocator);
   L->init(nullptr, /*ReleaseToOsInterval=*/0);
   std::thread Threads[16];
   for (scudo::uptr I = 0; I < ARRAY_SIZE(Threads); I++)
-    Threads[I] = std::thread(performAllocations, L);
+    Threads[I] = std::thread(performAllocations, L.get());
   {
     std::unique_lock<std::mutex> Lock(Mutex);
     Ready = true;
diff --git a/compiler-rt/lib/scudo/standalone/tests/tsd_test.cpp b/compiler-rt/lib/scudo/standalone/tests/tsd_test.cpp
index 4a3cf1cd0fc31..561bda47e24c8 100644
--- a/compiler-rt/lib/scudo/standalone/tests/tsd_test.cpp
+++ b/compiler-rt/lib/scudo/standalone/tests/tsd_test.cpp
@@ -13,6 +13,7 @@
 
 #include <condition_variable>
 #include <mutex>
+#include <set>
 #include <thread>
 
 // We mock out an allocator with a TSD registry, mostly using empty stubs. The
@@ -47,12 +48,12 @@ template <class Config> class MockAllocator {
 
 struct OneCache {
   template <class Allocator>
-  using TSDRegistryT = scudo::TSDRegistrySharedT<Allocator, 1U>;
+  using TSDRegistryT = scudo::TSDRegistrySharedT<Allocator, 1U, 1U>;
 };
 
 struct SharedCaches {
   template <class Allocator>
-  using TSDRegistryT = scudo::TSDRegistrySharedT<Allocator, 16U>;
+  using TSDRegistryT = scudo::TSDRegistrySharedT<Allocator, 16U, 8U>;
 };
 
 struct ExclusiveCaches {
@@ -116,7 +117,7 @@ TEST(ScudoTSDTest, TSDRegistryBasic) {
 
 static std::mutex Mutex;
 static std::condition_variable Cv;
-static bool Ready = false;
+static bool Ready;
 
 template <typename AllocatorT> static void stressCache(AllocatorT *Allocator) {
   auto Registry = Allocator->getTSDRegistry();
@@ -145,6 +146,7 @@ template <typename AllocatorT> static void stressCache(AllocatorT *Allocator) {
 }
 
 template <class AllocatorT> static void testRegistryThreaded() {
+  Ready = false;
   auto Deleter = [](AllocatorT *A) {
     A->unmapTestOnly();
     delete A;
@@ -171,3 +173,73 @@ TEST(ScudoTSDTest, TSDRegistryThreaded) {
   testRegistryThreaded<MockAllocator<ExclusiveCaches>>();
 #endif
 }
+
+static std::set<void *> Pointers;
+
+static void stressSharedRegistry(MockAllocator<SharedCaches> *Allocator) {
+  std::set<void *> Set;
+  auto Registry = Allocator->getTSDRegistry();
+  {
+    std::unique_lock<std::mutex> Lock(Mutex);
+    while (!Ready)
+      Cv.wait(Lock);
+  }
+  Registry->initThreadMaybe(Allocator, /*MinimalInit=*/false);
+  bool UnlockRequired;
+  for (scudo::uptr I = 0; I < 4096U; I++) {
+    auto TSD = Registry->getTSDAndLock(&UnlockRequired);
+    EXPECT_NE(TSD, nullptr);
+    Set.insert(reinterpret_cast<void *>(TSD));
+    if (UnlockRequired)
+      TSD->unlock();
+  }
+  {
+    std::unique_lock<std::mutex> Lock(Mutex);
+    Pointers.insert(Set.begin(), Set.end());
+  }
+}
+
+TEST(ScudoTSDTest, TSDRegistryTSDsCount) {
+  Ready = false;
+  using AllocatorT = MockAllocator<SharedCaches>;
+  auto Deleter = [](AllocatorT *A) {
+    A->unmapTestOnly();
+    delete A;
+  };
+  std::unique_ptr<AllocatorT, decltype(Deleter)> Allocator(new AllocatorT,
+                                                           Deleter);
+  Allocator->reset();
+  // We attempt to use as many TSDs as the shared cache offers by creating a
+  // decent amount of threads that will be run concurrently and attempt to get
+  // and lock TSDs. We put them all in a set and count the number of entries
+  // after we are done.
+  std::thread Threads[32];
+  for (scudo::uptr I = 0; I < ARRAY_SIZE(Threads); I++)
+    Threads[I] = std::thread(stressSharedRegistry, Allocator.get());
+  {
+    std::unique_lock<std::mutex> Lock(Mutex);
+    Ready = true;
+    Cv.notify_all();
+  }
+  for (auto &T : Threads)
+    T.join();
+  // The initial number of TSDs we get will be the minimum of the default count
+  // and the number of CPUs.
+  EXPECT_LE(Pointers.size(), 8U);
+  Pointers.clear();
+  auto Registry = Allocator->getTSDRegistry();
+  // Increase the number of TSDs to 16.
+  Registry->setOption(scudo::Option::MaxTSDsCount, 16);
+  Ready = false;
+  for (scudo::uptr I = 0; I < ARRAY_SIZE(Threads); I++)
+    Threads[I] = std::thread(stressSharedRegistry, Allocator.get());
+  {
+    std::unique_lock<std::mutex> Lock(Mutex);
+    Ready = true;
+    Cv.notify_all();
+  }
+  for (auto &T : Threads)
+    T.join();
+  // We should get 16 distinct TSDs back.
+  EXPECT_EQ(Pointers.size(), 16U);
+}
diff --git a/compiler-rt/lib/scudo/standalone/tests/wrappers_c_test.cpp b/compiler-rt/lib/scudo/standalone/tests/wrappers_c_test.cpp
index 4b851a7af181f..ed5b64c203364 100644
--- a/compiler-rt/lib/scudo/standalone/tests/wrappers_c_test.cpp
+++ b/compiler-rt/lib/scudo/standalone/tests/wrappers_c_test.cpp
@@ -389,6 +389,7 @@ static void *enableMalloc(void *Unused) {
 
 TEST(ScudoWrappersCTest, DisableForkEnable) {
   pthread_t ThreadId;
+  Ready = false;
   EXPECT_EQ(pthread_create(&ThreadId, nullptr, &enableMalloc, nullptr), 0);
 
   // Wait for the thread to be warmed up.
diff --git a/compiler-rt/lib/scudo/standalone/tests/wrappers_cpp_test.cpp b/compiler-rt/lib/scudo/standalone/tests/wrappers_cpp_test.cpp
index 4ccef5bb0deec..d24b6651d95e3 100644
--- a/compiler-rt/lib/scudo/standalone/tests/wrappers_cpp_test.cpp
+++ b/compiler-rt/lib/scudo/standalone/tests/wrappers_cpp_test.cpp
@@ -79,7 +79,7 @@ TEST(ScudoWrappersCppTest, New) {
 
 static std::mutex Mutex;
 static std::condition_variable Cv;
-static bool Ready = false;
+static bool Ready;
 
 static void stressNew() {
   std::vector<uintptr_t *> V;
@@ -103,6 +103,7 @@ static void stressNew() {
 }
 
 TEST(ScudoWrappersCppTest, ThreadedNew) {
+  Ready = false;
   std::thread Threads[32];
   for (size_t I = 0U; I < sizeof(Threads) / sizeof(Threads[0]); I++)
     Threads[I] = std::thread(stressNew);
diff --git a/compiler-rt/lib/scudo/standalone/tsd_exclusive.h b/compiler-rt/lib/scudo/standalone/tsd_exclusive.h
index 3492509b5a8eb..ac5a22c970701 100644
--- a/compiler-rt/lib/scudo/standalone/tsd_exclusive.h
+++ b/compiler-rt/lib/scudo/standalone/tsd_exclusive.h
@@ -66,6 +66,12 @@ template <class Allocator> struct TSDRegistryExT {
     Mutex.unlock();
   }
 
+  bool setOption(Option O, UNUSED sptr Value) {
+    if (O == Option::MaxTSDsCount)
+      return false;
+    return true;
+  }
+
 private:
   void initOnceMaybe(Allocator *Instance) {
     ScopedLock L(Mutex);
diff --git a/compiler-rt/lib/scudo/standalone/tsd_shared.h b/compiler-rt/lib/scudo/standalone/tsd_shared.h
index 038a5905ff489..25ba191826c3f 100644
--- a/compiler-rt/lib/scudo/standalone/tsd_shared.h
+++ b/compiler-rt/lib/scudo/standalone/tsd_shared.h
@@ -14,31 +14,16 @@
 
 namespace scudo {
 
-template <class Allocator, u32 MaxTSDCount> struct TSDRegistrySharedT {
+template <class Allocator, u32 TSDsArraySize, u32 DefaultTSDCount>
+struct TSDRegistrySharedT {
   void initLinkerInitialized(Allocator *Instance) {
     Instance->initLinkerInitialized();
     CHECK_EQ(pthread_key_create(&PThreadKey, nullptr), 0); // For non-TLS
-    const u32 NumberOfCPUs = getNumberOfCPUs();
-    NumberOfTSDs = (SCUDO_ANDROID || NumberOfCPUs == 0)
-                       ? MaxTSDCount
-                       : Min(NumberOfCPUs, MaxTSDCount);
-    for (u32 I = 0; I < NumberOfTSDs; I++)
+    for (u32 I = 0; I < TSDsArraySize; I++)
       TSDs[I].initLinkerInitialized(Instance);
-    // Compute all the coprimes of NumberOfTSDs. This will be used to walk the
-    // array of TSDs in a random order. For details, see:
-    // https://lemire.me/blog/2017/09/18/visiting-all-values-in-an-array-exactly-once-in-random-order/
-    for (u32 I = 0; I < NumberOfTSDs; I++) {
-      u32 A = I + 1;
-      u32 B = NumberOfTSDs;
-      // Find the GCD between I + 1 and NumberOfTSDs. If 1, they are coprimes.
-      while (B != 0) {
-        const u32 T = A;
-        A = B;
-        B = T % B;
-      }
-      if (A == 1)
-        CoPrimes[NumberOfCoPrimes++] = I + 1;
-    }
+    const u32 NumberOfCPUs = getNumberOfCPUs();
+    setNumberOfTSDs((NumberOfCPUs == 0) ? DefaultTSDCount
+                                        : Min(NumberOfCPUs, DefaultTSDCount));
     Initialized = true;
   }
   void init(Allocator *Instance) {
@@ -66,21 +51,34 @@ template <class Allocator, u32 MaxTSDCount> struct TSDRegistrySharedT {
     if (TSD->tryLock())
       return TSD;
     // If that fails, go down the slow path.
+    if (TSDsArraySize == 1U) {
+      // Only 1 TSD, not need to go any further.
+      // The compiler will optimize this one way or the other.
+      TSD->lock();
+      return TSD;
+    }
     return getTSDAndLockSlow(TSD);
   }
 
   void disable() {
     Mutex.lock();
-    for (u32 I = 0; I < NumberOfTSDs; I++)
+    for (u32 I = 0; I < TSDsArraySize; I++)
       TSDs[I].lock();
   }
 
   void enable() {
-    for (s32 I = static_cast<s32>(NumberOfTSDs - 1); I >= 0; I--)
+    for (s32 I = static_cast<s32>(TSDsArraySize - 1); I >= 0; I--)
       TSDs[I].unlock();
     Mutex.unlock();
   }
 
+  bool setOption(Option O, sptr Value) {
+    if (O == Option::MaxTSDsCount)
+      return setNumberOfTSDs(static_cast<u32>(Value));
+    // Not supported by the TSD Registry, but not an error either.
+    return true;
+  }
+
 private:
   ALWAYS_INLINE void setCurrentTSD(TSD<Allocator> *CurrentTSD) {
 #if _BIONIC
@@ -104,6 +102,32 @@ template <class Allocator, u32 MaxTSDCount> struct TSDRegistrySharedT {
 #endif
   }
 
+  bool setNumberOfTSDs(u32 N) {
+    ScopedLock L(MutexTSDs);
+    if (N < NumberOfTSDs)
+      return false;
+    if (N > TSDsArraySize)
+      N = TSDsArraySize;
+    NumberOfTSDs = N;
+    NumberOfCoPrimes = 0;
+    // Compute all the coprimes of NumberOfTSDs. This will be used to walk the
+    // array of TSDs in a random order. For details, see:
+    // https://lemire.me/blog/2017/09/18/visiting-all-values-in-an-array-exactly-once-in-random-order/
+    for (u32 I = 0; I < N; I++) {
+      u32 A = I + 1;
+      u32 B = N;
+      // Find the GCD between I + 1 and N. If 1, they are coprimes.
+      while (B != 0) {
+        const u32 T = A;
+        A = B;
+        B = T % B;
+      }
+      if (A == 1)
+        CoPrimes[NumberOfCoPrimes++] = I + 1;
+    }
+    return true;
+  }
+
   void initOnceMaybe(Allocator *Instance) {
     ScopedLock L(Mutex);
     if (LIKELY(Initialized))
@@ -120,17 +144,23 @@ template <class Allocator, u32 MaxTSDCount> struct TSDRegistrySharedT {
   }
 
   NOINLINE TSD<Allocator> *getTSDAndLockSlow(TSD<Allocator> *CurrentTSD) {
-    if (MaxTSDCount > 1U && NumberOfTSDs > 1U) {
-      // Use the Precedence of the current TSD as our random seed. Since we are
-      // in the slow path, it means that tryLock failed, and as a result it's
-      // very likely that said Precedence is non-zero.
-      const u32 R = static_cast<u32>(CurrentTSD->getPrecedence());
-      const u32 Inc = CoPrimes[R % NumberOfCoPrimes];
-      u32 Index = R % NumberOfTSDs;
+    // Use the Precedence of the current TSD as our random seed. Since we are
+    // in the slow path, it means that tryLock failed, and as a result it's
+    // very likely that said Precedence is non-zero.
+    const u32 R = static_cast<u32>(CurrentTSD->getPrecedence());
+    u32 N, Inc;
+    {
+      ScopedLock L(MutexTSDs);
+      N = NumberOfTSDs;
+      DCHECK_NE(NumberOfCoPrimes, 0U);
+      Inc = CoPrimes[R % NumberOfCoPrimes];
+    }
+    if (N > 1U) {
+      u32 Index = R % N;
       uptr LowestPrecedence = UINTPTR_MAX;
       TSD<Allocator> *CandidateTSD = nullptr;
       // Go randomly through at most 4 contexts and find a candidate.
-      for (u32 I = 0; I < Min(4U, NumberOfTSDs); I++) {
+      for (u32 I = 0; I < Min(4U, N); I++) {
         if (TSDs[Index].tryLock()) {
           setCurrentTSD(&TSDs[Index]);
           return &TSDs[Index];
@@ -142,8 +172,8 @@ template <class Allocator, u32 MaxTSDCount> struct TSDRegistrySharedT {
           LowestPrecedence = Precedence;
         }
         Index += Inc;
-        if (Index >= NumberOfTSDs)
-          Index -= NumberOfTSDs;
+        if (Index >= N)
+          Index -= N;
       }
       if (CandidateTSD) {
         CandidateTSD->lock();
@@ -160,19 +190,20 @@ template <class Allocator, u32 MaxTSDCount> struct TSDRegistrySharedT {
   atomic_u32 CurrentIndex;
   u32 NumberOfTSDs;
   u32 NumberOfCoPrimes;
-  u32 CoPrimes[MaxTSDCount];
+  u32 CoPrimes[TSDsArraySize];
   bool Initialized;
   HybridMutex Mutex;
-  TSD<Allocator> TSDs[MaxTSDCount];
+  HybridMutex MutexTSDs;
+  TSD<Allocator> TSDs[TSDsArraySize];
 #if SCUDO_LINUX && !_BIONIC
   static THREADLOCAL TSD<Allocator> *ThreadTSD;
 #endif
 };
 
 #if SCUDO_LINUX && !_BIONIC
-template <class Allocator, u32 MaxTSDCount>
+template <class Allocator, u32 TSDsArraySize, u32 DefaultTSDCount>
 THREADLOCAL TSD<Allocator>
-    *TSDRegistrySharedT<Allocator, MaxTSDCount>::ThreadTSD;
+    *TSDRegistrySharedT<Allocator, TSDsArraySize, DefaultTSDCount>::ThreadTSD;
 #endif
 
 } // namespace scudo

From 9cc70e047c2892a318ade3afacab7faffa4f49cc Mon Sep 17 00:00:00 2001
From: Siva Chandra Reddy <sivachandra@google.com>
Date: Tue, 28 Jul 2020 11:43:04 -0700
Subject: [PATCH 0347/1035] [libc][NFC] Zero out padding bits in the uint form
 of x86 FPBits<long double>.

---
 libc/utils/FPUtil/LongDoubleBitsX86.h | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/libc/utils/FPUtil/LongDoubleBitsX86.h b/libc/utils/FPUtil/LongDoubleBitsX86.h
index 5438e0b2b6edb..4d64490c5a11d 100644
--- a/libc/utils/FPUtil/LongDoubleBitsX86.h
+++ b/libc/utils/FPUtil/LongDoubleBitsX86.h
@@ -23,10 +23,10 @@ template <> struct MantissaWidth<long double> {
 template <unsigned Width> struct Padding;
 
 // i386 padding.
-template <> struct Padding<4> { static constexpr unsigned Value = 16; };
+template <> struct Padding<4> { static constexpr unsigned value = 16; };
 
 // x86_64 padding.
-template <> struct Padding<8> { static constexpr unsigned Value = 48; };
+template <> struct Padding<8> { static constexpr unsigned value = 48; };
 
 template <> struct __attribute__((packed)) FPBits<long double> {
   using UIntType = __uint128_t;
@@ -38,7 +38,7 @@ template <> struct __attribute__((packed)) FPBits<long double> {
   uint8_t implicitBit : 1;
   uint16_t exponent : ExponentWidth<long double>::value;
   uint8_t sign : 1;
-  uint64_t padding : Padding<sizeof(uintptr_t)>::Value;
+  uint64_t padding : Padding<sizeof(uintptr_t)>::value;
 
   template <typename XType,
             cpp::EnableIfType<cpp::IsSame<long double, XType>::Value, int> = 0>
@@ -91,7 +91,15 @@ template <> struct __attribute__((packed)) FPBits<long double> {
     // zero in case i386.
     UIntType result = UIntType(0);
     *reinterpret_cast<FPBits<long double> *>(&result) = *this;
-    return result;
+
+    // Even though we zero out |result| before copying the long double value,
+    // there can be garbage bits in the padding. So, we zero the padding bits
+    // in |result|.
+    static constexpr UIntType mask =
+        (UIntType(1) << (sizeof(long double) -
+                         Padding<sizeof(uintptr_t)>::value / 8)) -
+        1;
+    return result & mask;
   }
 
   static FPBits<long double> zero() { return FPBits<long double>(0.0l); }

From f8ab66538c4c8dc60cd458024639772a164ef9c5 Mon Sep 17 00:00:00 2001
From: jasonliu <jasonliu.development@gmail.com>
Date: Tue, 28 Jul 2020 17:56:13 +0000
Subject: [PATCH 0348/1035] [NFC][XCOFF] Use getFunctionEntryPointSymbol from
 TLOF to simplify logic

Reviewed By: Xiangling_L

Differential Revision: https://reviews.llvm.org/D84693
---
 llvm/lib/Target/PowerPC/PPCISelLowering.cpp | 87 ++++++++++-----------
 1 file changed, 41 insertions(+), 46 deletions(-)

diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index db3833d595797..29a6d54d3296c 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -74,6 +74,7 @@
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCRegisterInfo.h"
+#include "llvm/MC/MCSectionXCOFF.h"
 #include "llvm/MC/MCSymbolXCOFF.h"
 #include "llvm/Support/AtomicOrdering.h"
 #include "llvm/Support/BranchProbability.h"
@@ -5120,50 +5121,38 @@ static SDValue transformCallee(const SDValue &Callee, SelectionDAG &DAG,
       Subtarget.is32BitELFABI() && !isLocalCallee() &&
       Subtarget.getTargetMachine().getRelocationModel() == Reloc::PIC_;
 
-  // On AIX, direct function calls reference the symbol for the function's
-  // entry point, which is named by prepending a "." before the function's
-  // C-linkage name.
-  const auto getFunctionEntryPointSymbol = [&](StringRef SymName) {
-    auto &Context = DAG.getMachineFunction().getMMI().getContext();
-    return cast<MCSymbolXCOFF>(
-        Context.getOrCreateSymbol(Twine(".") + Twine(SymName)));
+  const auto getAIXFuncEntryPointSymbolSDNode = [&](const GlobalValue *GV) {
+    const TargetMachine &TM = Subtarget.getTargetMachine();
+    const TargetLoweringObjectFile *TLOF = TM.getObjFileLowering();
+    MCSymbolXCOFF *S =
+        cast<MCSymbolXCOFF>(TLOF->getFunctionEntryPointSymbol(GV, TM));
+
+    if (GV->isDeclaration() && !S->hasRepresentedCsectSet()) {
+      // On AIX, an undefined symbol needs to be associated with a
+      // MCSectionXCOFF to get the correct storage mapping class.
+      // In this case, XCOFF::XMC_PR.
+      const XCOFF::StorageClass SC =
+          TargetLoweringObjectFileXCOFF::getStorageClassForGlobal(GV);
+      auto &Context = DAG.getMachineFunction().getMMI().getContext();
+      MCSectionXCOFF *Sec = Context.getXCOFFSection(
+          S->getSymbolTableName(), XCOFF::XMC_PR, XCOFF::XTY_ER, SC,
+          SectionKind::getMetadata());
+      S->setRepresentedCsect(Sec);
+    }
+
+    MVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
+    return DAG.getMCSymbol(S, PtrVT);
   };
 
-  const auto getAIXFuncEntryPointSymbolSDNode =
-      [&](StringRef FuncName, bool IsDeclaration,
-          const XCOFF::StorageClass &SC) {
-        MCSymbolXCOFF *S = getFunctionEntryPointSymbol(FuncName);
-
-        auto &Context = DAG.getMachineFunction().getMMI().getContext();
-
-        if (IsDeclaration && !S->hasRepresentedCsectSet()) {
-          // On AIX, an undefined symbol needs to be associated with a
-          // MCSectionXCOFF to get the correct storage mapping class.
-          // In this case, XCOFF::XMC_PR.
-          MCSectionXCOFF *Sec = Context.getXCOFFSection(
-              S->getSymbolTableName(), XCOFF::XMC_PR, XCOFF::XTY_ER, SC,
-              SectionKind::getMetadata());
-          S->setRepresentedCsect(Sec);
-        }
-
-        MVT PtrVT =
-            DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
-        return DAG.getMCSymbol(S, PtrVT);
-      };
-
   if (isFunctionGlobalAddress(Callee)) {
-    const GlobalAddressSDNode *G = cast<GlobalAddressSDNode>(Callee);
-    const GlobalValue *GV = G->getGlobal();
-
-    if (!Subtarget.isAIXABI())
-      return DAG.getTargetGlobalAddress(GV, dl, Callee.getValueType(), 0,
-                                        UsePlt ? PPCII::MO_PLT : 0);
+    const GlobalValue *GV = cast<GlobalAddressSDNode>(Callee)->getGlobal();
 
-    assert(!isa<GlobalIFunc>(GV) && "IFunc is not supported on AIX.");
-    const XCOFF::StorageClass SC =
-        TargetLoweringObjectFileXCOFF::getStorageClassForGlobal(GV);
-    return getAIXFuncEntryPointSymbolSDNode(GV->getName(), GV->isDeclaration(),
-                                            SC);
+    if (Subtarget.isAIXABI()) {
+      assert(!isa<GlobalIFunc>(GV) && "IFunc is not supported on AIX.");
+      return getAIXFuncEntryPointSymbolSDNode(GV);
+    }
+    return DAG.getTargetGlobalAddress(GV, dl, Callee.getValueType(), 0,
+                                      UsePlt ? PPCII::MO_PLT : 0);
   }
 
   if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) {
@@ -5173,12 +5162,18 @@ static SDValue transformCallee(const SDValue &Callee, SelectionDAG &DAG,
       // ExternalSymbol's, then we pick up the user-declared version.
       const Module *Mod = DAG.getMachineFunction().getFunction().getParent();
       if (const Function *F =
-              dyn_cast_or_null<Function>(Mod->getNamedValue(SymName))) {
-        const XCOFF::StorageClass SC =
-            TargetLoweringObjectFileXCOFF::getStorageClassForGlobal(F);
-        return getAIXFuncEntryPointSymbolSDNode(F->getName(),
-                                                F->isDeclaration(), SC);
-      }
+              dyn_cast_or_null<Function>(Mod->getNamedValue(SymName)))
+        return getAIXFuncEntryPointSymbolSDNode(F);
+
+      // On AIX, direct function calls reference the symbol for the function's
+      // entry point, which is named by prepending a "." before the function's
+      // C-linkage name.
+      const auto getFunctionEntryPointSymbol = [&](StringRef SymName) {
+        auto &Context = DAG.getMachineFunction().getMMI().getContext();
+        return cast<MCSymbolXCOFF>(
+            Context.getOrCreateSymbol(Twine(".") + Twine(SymName)));
+      };
+
       SymName = getFunctionEntryPointSymbol(SymName)->getName().data();
     }
     return DAG.getTargetExternalSymbol(SymName, Callee.getValueType(),

From 496fc3f196bca77d48804ab4bc9343c3d82b97bf Mon Sep 17 00:00:00 2001
From: Sanjay Patel <spatel@rotateright.com>
Date: Tue, 28 Jul 2020 11:13:05 -0400
Subject: [PATCH 0349/1035] [InstSimplify] add tests for icmp with partial
 undef constant; NFC

---
 .../Transforms/InstSimplify/icmp-constant.ll  | 62 +++++++++++++++++++
 1 file changed, 62 insertions(+)

diff --git a/llvm/test/Transforms/InstSimplify/icmp-constant.ll b/llvm/test/Transforms/InstSimplify/icmp-constant.ll
index 3ebaca7c8aafb..8f51ba3c4c0e5 100644
--- a/llvm/test/Transforms/InstSimplify/icmp-constant.ll
+++ b/llvm/test/Transforms/InstSimplify/icmp-constant.ll
@@ -19,6 +19,15 @@ define <2 x i1> @tautological_ule_vec(<2 x i8> %x) {
   ret <2 x i1> %cmp
 }
 
+define <2 x i1> @tautological_ule_vec_partial_undef(<2 x i8> %x) {
+; CHECK-LABEL: @tautological_ule_vec_partial_undef(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ule <2 x i8> [[X:%.*]], <i8 -1, i8 undef>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %cmp = icmp ule <2 x i8> %x, <i8 255, i8 undef>
+  ret <2 x i1> %cmp
+}
+
 define i1 @tautological_ugt(i8 %x) {
 ; CHECK-LABEL: @tautological_ugt(
 ; CHECK-NEXT:    ret i1 false
@@ -35,6 +44,15 @@ define <2 x i1> @tautological_ugt_vec(<2 x i8> %x) {
   ret <2 x i1> %cmp
 }
 
+define <2 x i1> @tautological_ugt_vec_partial_undef(<2 x i8> %x) {
+; CHECK-LABEL: @tautological_ugt_vec_partial_undef(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <2 x i8> [[X:%.*]], <i8 undef, i8 -1>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %cmp = icmp ugt <2 x i8> %x, <i8 undef, i8 255>
+  ret <2 x i1> %cmp
+}
+
 ; 'urem x, C2' produces [0, C2)
 define i1 @urem3(i32 %X) {
 ; CHECK-LABEL: @urem3(
@@ -54,6 +72,17 @@ define <2 x i1> @urem3_vec(<2 x i32> %X) {
   ret <2 x i1> %B
 }
 
+define <2 x i1> @urem3_vec_partial_undef(<2 x i32> %X) {
+; CHECK-LABEL: @urem3_vec_partial_undef(
+; CHECK-NEXT:    [[A:%.*]] = urem <2 x i32> [[X:%.*]], <i32 10, i32 10>
+; CHECK-NEXT:    [[B:%.*]] = icmp ult <2 x i32> [[A]], <i32 undef, i32 15>
+; CHECK-NEXT:    ret <2 x i1> [[B]]
+;
+  %A = urem <2 x i32> %X, <i32 10, i32 10>
+  %B = icmp ult <2 x i32> %A, <i32 undef, i32 15>
+  ret <2 x i1> %B
+}
+
 ;'srem x, C2' produces (-|C2|, |C2|)
 define i1 @srem1(i32 %X) {
 ; CHECK-LABEL: @srem1(
@@ -73,6 +102,17 @@ define <2 x i1> @srem1_vec(<2 x i32> %X) {
   ret <2 x i1> %B
 }
 
+define <2 x i1> @srem1_vec_partial_undef(<2 x i32> %X) {
+; CHECK-LABEL: @srem1_vec_partial_undef(
+; CHECK-NEXT:    [[A:%.*]] = srem <2 x i32> [[X:%.*]], <i32 -5, i32 -5>
+; CHECK-NEXT:    [[B:%.*]] = icmp sgt <2 x i32> [[A]], <i32 5, i32 undef>
+; CHECK-NEXT:    ret <2 x i1> [[B]]
+;
+  %A = srem <2 x i32> %X, <i32 -5, i32 -5>
+  %B = icmp sgt <2 x i32> %A, <i32 5, i32 undef>
+  ret <2 x i1> %B
+}
+
 ;'udiv C2, x' produces [0, C2]
 define i1 @udiv5(i32 %X) {
 ; CHECK-LABEL: @udiv5(
@@ -169,6 +209,17 @@ define <2 x i1> @shl5_vec(<2 x i32> %X) {
   ret <2 x i1> %cmp
 }
 
+define <2 x i1> @shl5_vec_partial_undef(<2 x i32> %X) {
+; CHECK-LABEL: @shl5_vec_partial_undef(
+; CHECK-NEXT:    [[SUB:%.*]] = shl nuw <2 x i32> <i32 4, i32 4>, [[X:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <2 x i32> [[SUB]], <i32 undef, i32 3>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %sub = shl nuw <2 x i32> <i32 4, i32 4>, %X
+  %cmp = icmp ugt <2 x i32> %sub, <i32 undef, i32 3>
+  ret <2 x i1> %cmp
+}
+
 ; 'shl nsw C2, x' produces [C2 << CLO(C2)-1, C2]
 define i1 @shl2(i32 %X) {
 ; CHECK-LABEL: @shl2(
@@ -378,6 +429,17 @@ define <2 x i1> @or1_vec(<2 x i32> %X) {
   ret <2 x i1> %B
 }
 
+define <2 x i1> @or1_vec_partial_undef(<2 x i32> %X) {
+; CHECK-LABEL: @or1_vec_partial_undef(
+; CHECK-NEXT:    [[A:%.*]] = or <2 x i32> [[X:%.*]], <i32 62, i32 62>
+; CHECK-NEXT:    [[B:%.*]] = icmp ult <2 x i32> [[A]], <i32 undef, i32 50>
+; CHECK-NEXT:    ret <2 x i1> [[B]]
+;
+  %A = or <2 x i32> %X, <i32 62, i32 62>
+  %B = icmp ult <2 x i32> %A, <i32 undef, i32 50>
+  ret <2 x i1> %B
+}
+
 ; Single bit OR.
 define i1 @or2_true(i8 %x) {
 ; CHECK-LABEL: @or2_true(

From f75cf240d6ed528e1ce7770bbe09b417338b40ef Mon Sep 17 00:00:00 2001
From: Sanjay Patel <spatel@rotateright.com>
Date: Tue, 28 Jul 2020 15:02:36 -0400
Subject: [PATCH 0350/1035] [InstCombine] avoid crashing on vector constant
 expression (PR46872)

---
 .../InstCombine/InstructionCombining.cpp          |  2 +-
 llvm/test/Transforms/InstCombine/vec_shuffle.ll   | 15 +++++++++++++++
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
index 965a72d3a6500..ceeddda05fd9a 100644
--- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
@@ -1599,7 +1599,7 @@ Instruction *InstCombinerImpl::foldVectorBinop(BinaryOperator &Inst) {
   Constant *C;
   if (match(&Inst,
             m_c_BinOp(m_OneUse(m_Shuffle(m_Value(V1), m_Undef(), m_Mask(Mask))),
-                      m_Constant(C))) &&
+                      m_Constant(C))) && !isa<ConstantExpr>(C) &&
       cast<FixedVectorType>(V1->getType())->getNumElements() <= NumElts) {
     assert(Inst.getType()->getScalarType() == V1->getType()->getScalarType() &&
            "Shuffle should not change scalar type");
diff --git a/llvm/test/Transforms/InstCombine/vec_shuffle.ll b/llvm/test/Transforms/InstCombine/vec_shuffle.ll
index e7e55b07b7cd1..3f3431c5d9045 100644
--- a/llvm/test/Transforms/InstCombine/vec_shuffle.ll
+++ b/llvm/test/Transforms/InstCombine/vec_shuffle.ll
@@ -1745,3 +1745,18 @@ define <4 x i32> @splat_assoc_add_mul(<4 x i32> %x, <4 x i32> %y) {
   %r = mul <4 x i32> %splatx, %a
   ret <4 x i32> %r
 }
+
+
+; Do not crash on constant expressions.
+
+define <4 x i32> @PR46872(<4 x i32> %x) {
+; CHECK-LABEL: @PR46872(
+; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <4 x i32> <i32 undef, i32 0, i32 1, i32 1>
+; CHECK-NEXT:    [[A:%.*]] = and <4 x i32> [[S]], bitcast (<2 x i64> <i64 ptrtoint (<4 x i32> (<4 x i32>)* @PR46872 to i64), i64 ptrtoint (<4 x i32> (<4 x i32>)* @PR46872 to i64)> to <4 x i32>)
+; CHECK-NEXT:    ret <4 x i32> [[A]]
+;
+  %s = shufflevector <4 x i32> %x, <4 x i32> undef, <4 x i32> <i32 undef, i32 0, i32 1, i32 1>
+  %a = and <4 x i32> %s, bitcast (<2 x i64> <i64 ptrtoint (<4 x i32> (<4 x i32>)* @PR46872 to i64), i64 ptrtoint (<4 x i32> (<4 x i32>)* @PR46872 to i64)> to <4 x i32>)
+  ret <4 x i32> %a
+}
+

From e5608cacfd60bb28685206ca96a8f3ceeee1e8a6 Mon Sep 17 00:00:00 2001
From: MaheshRavishankar <ravishankarm@google.com>
Date: Tue, 28 Jul 2020 11:47:30 -0700
Subject: [PATCH 0351/1035] [mlir][GPUToSPIRV] Add a test pass to set workgroup
 size for kernel functions.

This allows using command line flags to lowere from GPU to SPIR-V. The
pass added is only for testing/example purposes. Most uses cases will
need more fine-grained control on setting workgroup sizes for kernel
functions.

Differential Revision: https://reviews.llvm.org/D84619
---
 .../GPUToSPIRV/test_spirv_entry_point.mlir    | 14 ++++
 mlir/test/lib/Dialect/SPIRV/CMakeLists.txt    |  2 +
 .../lib/Dialect/SPIRV/TestEntryPointAbi.cpp   | 64 +++++++++++++++++++
 mlir/tools/mlir-opt/mlir-opt.cpp              |  2 +
 4 files changed, 82 insertions(+)
 create mode 100644 mlir/test/Conversion/GPUToSPIRV/test_spirv_entry_point.mlir
 create mode 100644 mlir/test/lib/Dialect/SPIRV/TestEntryPointAbi.cpp

diff --git a/mlir/test/Conversion/GPUToSPIRV/test_spirv_entry_point.mlir b/mlir/test/Conversion/GPUToSPIRV/test_spirv_entry_point.mlir
new file mode 100644
index 0000000000000..26556a41c9be1
--- /dev/null
+++ b/mlir/test/Conversion/GPUToSPIRV/test_spirv_entry_point.mlir
@@ -0,0 +1,14 @@
+// RUN: mlir-opt -test-spirv-entry-point-abi %s | FileCheck %s -check-prefix=DEFAULT
+// RUN: mlir-opt -test-spirv-entry-point-abi="workgroup-size=32" %s | FileCheck %s -check-prefix=WG32
+
+//      DEFAULT: gpu.func @foo()
+// DEFAULT-SAME: spv.entry_point_abi = {local_size = dense<1> : vector<3xi32>}
+
+//      WG32: gpu.func @foo()
+// WG32-SAME:  spv.entry_point_abi = {local_size = dense<[32, 1, 1]> : vector<3xi32>}
+
+gpu.module @kernels {
+  gpu.func @foo() kernel {
+    gpu.return
+  }
+}
diff --git a/mlir/test/lib/Dialect/SPIRV/CMakeLists.txt b/mlir/test/lib/Dialect/SPIRV/CMakeLists.txt
index 15d4673f381f3..204a633377307 100644
--- a/mlir/test/lib/Dialect/SPIRV/CMakeLists.txt
+++ b/mlir/test/lib/Dialect/SPIRV/CMakeLists.txt
@@ -1,6 +1,7 @@
 # Exclude tests from libMLIR.so
 add_mlir_library(MLIRSPIRVTestPasses
   TestAvailability.cpp
+  TestEntryPointAbi.cpp
 
   EXCLUDE_FROM_LIBMLIR
 
@@ -9,6 +10,7 @@ add_mlir_library(MLIRSPIRVTestPasses
   ${MLIR_MAIN_INCLUDE_DIR}/mlir/IR
 
   LINK_LIBS PUBLIC
+  MLIRGPU
   MLIRIR
   MLIRPass
   MLIRSPIRV
diff --git a/mlir/test/lib/Dialect/SPIRV/TestEntryPointAbi.cpp b/mlir/test/lib/Dialect/SPIRV/TestEntryPointAbi.cpp
new file mode 100644
index 0000000000000..bcbdb828523d4
--- /dev/null
+++ b/mlir/test/lib/Dialect/SPIRV/TestEntryPointAbi.cpp
@@ -0,0 +1,64 @@
+//===- TestAvailability.cpp - Test pass for setting Entry point ABI info --===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements a pass that sets the spv.entry_point_abi attribute on
+// functions that are to be lowered as entry point functions.
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/GPU/GPUDialect.h"
+#include "mlir/Dialect/SPIRV/SPIRVDialect.h"
+#include "mlir/Dialect/SPIRV/TargetAndABI.h"
+#include "mlir/Pass/Pass.h"
+
+using namespace mlir;
+
+namespace {
+/// Pass to set the spv.entry_point_abi
+class TestSpirvEntryPointABIPass
+    : public PassWrapper<TestSpirvEntryPointABIPass,
+                         OperationPass<gpu::GPUModuleOp>> {
+public:
+  TestSpirvEntryPointABIPass() = default;
+  TestSpirvEntryPointABIPass(const TestSpirvEntryPointABIPass &) {}
+  void runOnOperation() override;
+
+private:
+  Pass::ListOption<int32_t> workgroupSize{
+      *this, "workgroup-size",
+      llvm::cl::desc(
+          "Workgroup size to use for all gpu.func kernels in the module, "
+          "specified with x-dimension first, y-dimension next and z-dimension "
+          "last. Unspecified dimensions will be set to 1"),
+      llvm::cl::ZeroOrMore, llvm::cl::MiscFlags::CommaSeparated};
+};
+} // namespace
+
+void TestSpirvEntryPointABIPass::runOnOperation() {
+  gpu::GPUModuleOp gpuModule = getOperation();
+  MLIRContext *context = &getContext();
+  StringRef attrName = spirv::getEntryPointABIAttrName();
+  for (gpu::GPUFuncOp gpuFunc : gpuModule.getOps<gpu::GPUFuncOp>()) {
+    if (!gpu::GPUDialect::isKernel(gpuFunc) || gpuFunc.getAttr(attrName))
+      continue;
+    SmallVector<int32_t, 3> workgroupSizeVec(workgroupSize.begin(),
+                                             workgroupSize.end());
+    workgroupSizeVec.resize(3, 1);
+    gpuFunc.setAttr(attrName,
+                    spirv::getEntryPointABIAttr(workgroupSizeVec, context));
+  }
+}
+
+namespace mlir {
+void registerTestSpirvEntryPointABIPass() {
+  PassRegistration<TestSpirvEntryPointABIPass> registration(
+      "test-spirv-entry-point-abi",
+      "Set the spv.entry_point_abi attribute on GPU kernel function within the "
+      "module, intended for testing only");
+}
+} // namespace mlir
diff --git a/mlir/tools/mlir-opt/mlir-opt.cpp b/mlir/tools/mlir-opt/mlir-opt.cpp
index f60864a6a371b..620c5871a420c 100644
--- a/mlir/tools/mlir-opt/mlir-opt.cpp
+++ b/mlir/tools/mlir-opt/mlir-opt.cpp
@@ -66,6 +66,7 @@ void registerTestPreparationPassWithAllowedMemrefResults();
 void registerTestRecursiveTypesPass();
 void registerTestReducer();
 void registerTestGpuParallelLoopMappingPass();
+void registerTestSpirvEntryPointABIPass();
 void registerTestSCFUtilsPass();
 void registerTestVectorConversions();
 void registerVectorizerTestPass();
@@ -142,6 +143,7 @@ void registerTestPasses() {
   registerTestRecursiveTypesPass();
   registerTestReducer();
   registerTestGpuParallelLoopMappingPass();
+  registerTestSpirvEntryPointABIPass();
   registerTestSCFUtilsPass();
   registerTestVectorConversions();
   registerVectorizerTestPass();

From 69152a11cf181d9c1859947f0f2c2f1554a891f2 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@intel.com>
Date: Tue, 28 Jul 2020 12:03:54 -0700
Subject: [PATCH 0352/1035] [X86] Merge the two 'Emit the normal disp32
 encoding' cases in SIB byte handling in emitMemModRMByte. NFCI

By repeating the Disp.isImm() check in a couple spots we can
make the normal case for immediate and for expression the same.
And then always rely on the ForceDisp32 flag to remove a later
non-zero immediate check.

This should make {disp32} pseudo prefix handling
slightly easier as we need the normal disp32 handler to handle a
immediate of 0.
---
 .../lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp
index 25f1089912639..abdc0f156b9f9 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp
@@ -582,23 +582,21 @@ void X86MCCodeEmitter::emitMemModRMByte(const MCInst &MI, unsigned Op,
     // MOD=0, BASE=5, to JUST get the index, scale, and displacement.
     emitByte(modRMByte(0, RegOpcodeField, 4), OS);
     ForceDisp32 = true;
-  } else if (!Disp.isImm()) {
-    // Emit the normal disp32 encoding.
-    emitByte(modRMByte(2, RegOpcodeField, 4), OS);
-    ForceDisp32 = true;
-  } else if (Disp.getImm() == 0 &&
+  } else if (Disp.isImm() && Disp.getImm() == 0 &&
              // Base reg can't be anything that ends up with '5' as the base
              // reg, it is the magic [*] nomenclature that indicates no base.
              BaseRegNo != N86::EBP) {
     // Emit no displacement ModR/M byte
     emitByte(modRMByte(0, RegOpcodeField, 4), OS);
-  } else if (isDispOrCDisp8(TSFlags, Disp.getImm(), ImmOffset)) {
+  } else if (Disp.isImm() &&
+             isDispOrCDisp8(TSFlags, Disp.getImm(), ImmOffset)) {
     // Emit the disp8 encoding.
     emitByte(modRMByte(1, RegOpcodeField, 4), OS);
     ForceDisp8 = true; // Make sure to force 8 bit disp if Base=EBP
   } else {
     // Emit the normal disp32 encoding.
     emitByte(modRMByte(2, RegOpcodeField, 4), OS);
+    ForceDisp32 = true;
   }
 
   // Calculate what the SS field value should be...
@@ -618,7 +616,7 @@ void X86MCCodeEmitter::emitMemModRMByte(const MCInst &MI, unsigned Op,
   if (ForceDisp8)
     emitImmediate(Disp, MI.getLoc(), 1, FK_Data_1, StartByte, OS, Fixups,
                   ImmOffset);
-  else if (ForceDisp32 || Disp.getImm() != 0)
+  else if (ForceDisp32)
     emitImmediate(Disp, MI.getLoc(), 4, MCFixupKind(X86::reloc_signed_4byte),
                   StartByte, OS, Fixups);
 }

From 1bd7c02233969b430b2d49e95345f507fdcc9f30 Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne@apple.com>
Date: Tue, 28 Jul 2020 13:41:12 -0400
Subject: [PATCH 0353/1035] [libc++] Clean up tests for "optional" C11 features

First, add a TEST_HAS_QUICK_EXIT macro to mirror other C11 features like
TEST_HAS_ALIGNED_ALLOC, and update the tests for that.

Second, get rid of TEST_HAS_C11_FEATURES and _LIBCPP_HAS_C11_FEATURES,
which were only used to ensure that feature macros don't get out of
sync between <__config> and "test_macros.h". This is not necessary
anymore, since we have tests for each individual macro now.
---
 libcxx/include/__config                       |  5 ---
 .../has_aligned_alloc.compile.pass.cpp        | 19 +++++++++++
 .../has_c11_features.pass.cpp                 | 33 -------------------
 .../has_quick_exit.compile.pass.cpp           | 19 +++++++++++
 .../has_timespec_get.compile.pass.cpp         | 19 +++++++++++
 .../std/depr/depr.c.headers/float_h.pass.cpp  |  6 ++--
 .../support.limits/c.limits/cfloat.pass.cpp   |  6 ++--
 ...ail.cpp => at_quick_exit.compile.fail.cpp} | 18 +++++-----
 ...e.fail.cpp => quick_exit.compile.fail.cpp} | 15 ++++-----
 .../support.start.term/quick_exit.pass.cpp    |  2 +-
 .../test/support/msvc_stdlib_force_include.h  |  3 --
 libcxx/test/support/test_macros.h             | 16 +++++----
 12 files changed, 90 insertions(+), 71 deletions(-)
 create mode 100644 libcxx/test/libcxx/language.support/has_aligned_alloc.compile.pass.cpp
 delete mode 100644 libcxx/test/libcxx/language.support/has_c11_features.pass.cpp
 create mode 100644 libcxx/test/libcxx/language.support/has_quick_exit.compile.pass.cpp
 create mode 100644 libcxx/test/libcxx/language.support/has_timespec_get.compile.pass.cpp
 rename libcxx/test/std/language.support/support.start.term/{quick_exit_check1.compile.fail.cpp => at_quick_exit.compile.fail.cpp} (70%)
 rename libcxx/test/std/language.support/support.start.term/{quick_exit_check2.compile.fail.cpp => quick_exit.compile.fail.cpp} (72%)

diff --git a/libcxx/include/__config b/libcxx/include/__config
index ebdd64ed4292d..a9eca04959bf4 100644
--- a/libcxx/include/__config
+++ b/libcxx/include/__config
@@ -348,13 +348,11 @@
 #  if defined(__FreeBSD__)
 #    define _LIBCPP_HAS_ALIGNED_ALLOC
 #    define _LIBCPP_HAS_QUICK_EXIT
-#    define _LIBCPP_HAS_C11_FEATURES
 #    if __FreeBSD_version >= 1300064 || \
        (__FreeBSD_version >= 1201504 && __FreeBSD_version < 1300000)
 #      define _LIBCPP_HAS_TIMESPEC_GET
 #    endif
 #  elif defined(__BIONIC__)
-#    define _LIBCPP_HAS_C11_FEATURES
 #    if __ANDROID_API__ >= 21
 #      define _LIBCPP_HAS_QUICK_EXIT
 #    endif
@@ -368,7 +366,6 @@
 #    define _LIBCPP_HAS_ALIGNED_ALLOC
 #    define _LIBCPP_HAS_QUICK_EXIT
 #    define _LIBCPP_HAS_TIMESPEC_GET
-#    define _LIBCPP_HAS_C11_FEATURES
 #  elif defined(__linux__)
 #    if !defined(_LIBCPP_HAS_MUSL_LIBC)
 #      if _LIBCPP_GLIBC_PREREQ(2, 15) || defined(__BIONIC__)
@@ -376,14 +373,12 @@
 #      endif
 #      if _LIBCPP_GLIBC_PREREQ(2, 17)
 #        define _LIBCPP_HAS_ALIGNED_ALLOC
-#        define _LIBCPP_HAS_C11_FEATURES
 #        define _LIBCPP_HAS_TIMESPEC_GET
 #      endif
 #    else // defined(_LIBCPP_HAS_MUSL_LIBC)
 #      define _LIBCPP_HAS_ALIGNED_ALLOC
 #      define _LIBCPP_HAS_QUICK_EXIT
 #      define _LIBCPP_HAS_TIMESPEC_GET
-#      define _LIBCPP_HAS_C11_FEATURES
 #    endif
 #  endif // __linux__
 #endif
diff --git a/libcxx/test/libcxx/language.support/has_aligned_alloc.compile.pass.cpp b/libcxx/test/libcxx/language.support/has_aligned_alloc.compile.pass.cpp
new file mode 100644
index 0000000000000..d1b41de5dc1a9
--- /dev/null
+++ b/libcxx/test/libcxx/language.support/has_aligned_alloc.compile.pass.cpp
@@ -0,0 +1,19 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14
+
+// Make sure TEST_HAS_ALIGNED_ALLOC (defined by the test suite) and
+// _LIBCPP_HAS_ALIGNED_ALLOC (defined by libc++) stay in sync.
+
+#include <__config>
+#include "test_macros.h"
+
+#if defined(TEST_HAS_ALIGNED_ALLOC) != defined(_LIBCPP_HAS_ALIGNED_ALLOC)
+#   error "TEST_HAS_ALIGNED_ALLOC and _LIBCPP_HAS_ALIGNED_ALLOC are out of sync"
+#endif
diff --git a/libcxx/test/libcxx/language.support/has_c11_features.pass.cpp b/libcxx/test/libcxx/language.support/has_c11_features.pass.cpp
deleted file mode 100644
index ab1e6ea619252..0000000000000
--- a/libcxx/test/libcxx/language.support/has_c11_features.pass.cpp
+++ /dev/null
@@ -1,33 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-// UNSUPPORTED: c++03, c++11, c++14
-
-//	We have two macros for checking whether or not the underlying C library
-//	 has C11 features:
-//		TEST_HAS_C11_FEATURES    - which is defined in "test_macros.h"
-//		_LIBCPP_HAS_C11_FEATURES - which is defined in <__config>
-//	They should always be the same
-
-#include <__config>
-#include "test_macros.h"
-
-#ifdef TEST_HAS_C11_FEATURES
-# ifndef _LIBCPP_HAS_C11_FEATURES
-#  error "TEST_HAS_C11_FEATURES is defined, but _LIBCPP_HAS_C11_FEATURES is not"
-# endif
-#endif
-
-#ifdef _LIBCPP_HAS_C11_FEATURES
-# ifndef TEST_HAS_C11_FEATURES
-#  error "_LIBCPP_HAS_C11_FEATURES is defined, but TEST_HAS_C11_FEATURES is not"
-# endif
-#endif
-
-int main(int, char**) {
-  return 0;
-}
diff --git a/libcxx/test/libcxx/language.support/has_quick_exit.compile.pass.cpp b/libcxx/test/libcxx/language.support/has_quick_exit.compile.pass.cpp
new file mode 100644
index 0000000000000..ad0fe6fb18fd1
--- /dev/null
+++ b/libcxx/test/libcxx/language.support/has_quick_exit.compile.pass.cpp
@@ -0,0 +1,19 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14
+
+// Make sure TEST_HAS_QUICK_EXIT (defined by the test suite) and
+// _LIBCPP_HAS_QUICK_EXIT (defined by libc++) stay in sync.
+
+#include <__config>
+#include "test_macros.h"
+
+#if defined(TEST_HAS_QUICK_EXIT) != defined(_LIBCPP_HAS_QUICK_EXIT)
+#   error "TEST_HAS_QUICK_EXIT and _LIBCPP_HAS_QUICK_EXIT are out of sync"
+#endif
diff --git a/libcxx/test/libcxx/language.support/has_timespec_get.compile.pass.cpp b/libcxx/test/libcxx/language.support/has_timespec_get.compile.pass.cpp
new file mode 100644
index 0000000000000..8b86a5ef97195
--- /dev/null
+++ b/libcxx/test/libcxx/language.support/has_timespec_get.compile.pass.cpp
@@ -0,0 +1,19 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14
+
+// Make sure TEST_HAS_TIMESPEC_GET (defined by the test suite) and
+// _LIBCPP_HAS_TIMESPEC_GET (defined by libc++) stay in sync.
+
+#include <__config>
+#include "test_macros.h"
+
+#if defined(TEST_HAS_TIMESPEC_GET) != defined(_LIBCPP_HAS_TIMESPEC_GET)
+#   error "TEST_HAS_TIMESPEC_GET and _LIBCPP_HAS_TIMESPEC_GET are out of sync"
+#endif
diff --git a/libcxx/test/std/depr/depr.c.headers/float_h.pass.cpp b/libcxx/test/std/depr/depr.c.headers/float_h.pass.cpp
index 779fbc66f5d02..1069a679263fb 100644
--- a/libcxx/test/std/depr/depr.c.headers/float_h.pass.cpp
+++ b/libcxx/test/std/depr/depr.c.headers/float_h.pass.cpp
@@ -24,7 +24,7 @@
 #error FLT_RADIX not defined
 #endif
 
-#if TEST_STD_VER > 14 && defined(TEST_HAS_C11_FEATURES) && 0
+#if TEST_STD_VER > 14 && 0
 #ifndef FLT_HAS_SUBNORM
 #error FLT_HAS_SUBNORM not defined
 #endif
@@ -54,7 +54,7 @@
 #error DECIMAL_DIG not defined
 #endif
 
-#if TEST_STD_VER > 14 && defined(TEST_HAS_C11_FEATURES) && 0
+#if TEST_STD_VER > 14 && 0
 #ifndef FLT_DECIMAL_DIG
 #error FLT_DECIMAL_DIG not defined
 #endif
@@ -164,7 +164,7 @@
 #error LDBL_MIN not defined
 #endif
 
-#if TEST_STD_VER > 14 && defined(TEST_HAS_C11_FEATURES) && 0
+#if TEST_STD_VER > 14 && 0
 #ifndef FLT_TRUE_MIN
 #error FLT_TRUE_MIN not defined
 #endif
diff --git a/libcxx/test/std/language.support/support.limits/c.limits/cfloat.pass.cpp b/libcxx/test/std/language.support/support.limits/c.limits/cfloat.pass.cpp
index 12b80adf01169..6ddd8c60834b0 100644
--- a/libcxx/test/std/language.support/support.limits/c.limits/cfloat.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/c.limits/cfloat.pass.cpp
@@ -24,7 +24,7 @@
 #error FLT_RADIX not defined
 #endif
 
-#if TEST_STD_VER > 14 && defined(TEST_HAS_C11_FEATURES) && 0
+#if TEST_STD_VER > 14 && 0
 #ifndef FLT_HAS_SUBNORM
 #error FLT_HAS_SUBNORM not defined
 #endif
@@ -54,7 +54,7 @@
 #error DECIMAL_DIG not defined
 #endif
 
-#if TEST_STD_VER > 14 && defined(TEST_HAS_C11_FEATURES) && 0
+#if TEST_STD_VER > 14 && 0
 #ifndef FLT_DECIMAL_DIG
 #error FLT_DECIMAL_DIG not defined
 #endif
@@ -164,7 +164,7 @@
 #error LDBL_MIN not defined
 #endif
 
-#if TEST_STD_VER > 14 && defined(TEST_HAS_C11_FEATURES) && 0
+#if TEST_STD_VER > 14 && 0
 #ifndef FLT_TRUE_MIN
 #error FLT_TRUE_MIN not defined
 #endif
diff --git a/libcxx/test/std/language.support/support.start.term/quick_exit_check1.compile.fail.cpp b/libcxx/test/std/language.support/support.start.term/at_quick_exit.compile.fail.cpp
similarity index 70%
rename from libcxx/test/std/language.support/support.start.term/quick_exit_check1.compile.fail.cpp
rename to libcxx/test/std/language.support/support.start.term/at_quick_exit.compile.fail.cpp
index 10bd5767de0e8..79051fb5167d4 100644
--- a/libcxx/test/std/language.support/support.start.term/quick_exit_check1.compile.fail.cpp
+++ b/libcxx/test/std/language.support/support.start.term/at_quick_exit.compile.fail.cpp
@@ -5,23 +5,23 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
-//
+
 // UNSUPPORTED: c++03
 
-// test that referencing at_quick_exit when _LIBCPP_HAS_QUICK_EXIT is not defined
+// test that referencing at_quick_exit when TEST_HAS_QUICK_EXIT is not defined
 // results in a compile error.
 
 #include <cstdlib>
 
-void f() {}
+#include "test_macros.h"
 
-int main(int, char**)
-{
-#ifndef _LIBCPP_HAS_QUICK_EXIT
+void f() { }
+
+int main(int, char**) {
+#if !defined(TEST_HAS_QUICK_EXIT)
     std::at_quick_exit(f);
 #else
-#error
+#   error
 #endif
-
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/language.support/support.start.term/quick_exit_check2.compile.fail.cpp b/libcxx/test/std/language.support/support.start.term/quick_exit.compile.fail.cpp
similarity index 72%
rename from libcxx/test/std/language.support/support.start.term/quick_exit_check2.compile.fail.cpp
rename to libcxx/test/std/language.support/support.start.term/quick_exit.compile.fail.cpp
index cb0dbd35c5575..f9c5bdc835694 100644
--- a/libcxx/test/std/language.support/support.start.term/quick_exit_check2.compile.fail.cpp
+++ b/libcxx/test/std/language.support/support.start.term/quick_exit.compile.fail.cpp
@@ -5,22 +5,21 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
+
 // UNSUPPORTED: c++03
 
-// test that referencing quick_exit when _LIBCPP_HAS_QUICK_EXIT is not defined
+// test that referencing quick_exit when TEST_HAS_QUICK_EXIT is not defined
 // results in a compile error.
 
 #include <cstdlib>
 
-void f() {}
+#include "test_macros.h"
 
-int main(int, char**)
-{
-#ifndef _LIBCPP_HAS_QUICK_EXIT
+int main(int, char**) {
+#if !defined(TEST_HAS_QUICK_EXIT)
     std::quick_exit(0);
 #else
-#error
+#   error
 #endif
-
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/language.support/support.start.term/quick_exit.pass.cpp b/libcxx/test/std/language.support/support.start.term/quick_exit.pass.cpp
index b45b3391ba5c3..16f68d435640c 100644
--- a/libcxx/test/std/language.support/support.start.term/quick_exit.pass.cpp
+++ b/libcxx/test/std/language.support/support.start.term/quick_exit.pass.cpp
@@ -17,7 +17,7 @@ void f() {}
 
 int main(int, char**)
 {
-#ifdef _LIBCPP_HAS_QUICK_EXIT
+#ifdef TEST_HAS_QUICK_EXIT
     std::at_quick_exit(f);
     std::quick_exit(0);
 #endif
diff --git a/libcxx/test/support/msvc_stdlib_force_include.h b/libcxx/test/support/msvc_stdlib_force_include.h
index cf8696af5588a..0869be349e23d 100644
--- a/libcxx/test/support/msvc_stdlib_force_include.h
+++ b/libcxx/test/support/msvc_stdlib_force_include.h
@@ -65,9 +65,6 @@ const AssertionDialogAvoider assertion_dialog_avoider{};
 // MSVC doesn't have __int128_t.
 #define _LIBCPP_HAS_NO_INT128
 
-// MSVC has quick_exit() and at_quick_exit().
-#define _LIBCPP_HAS_QUICK_EXIT
-
 #ifndef _LIBCXX_IN_DEVCRT
     // atomic_is_lock_free.pass.cpp needs this VS 2015 Update 2 fix.
     #define _ENABLE_ATOMIC_ALIGNMENT_FIX
diff --git a/libcxx/test/support/test_macros.h b/libcxx/test/support/test_macros.h
index 63aa4f47a70f0..15c6a280dc192 100644
--- a/libcxx/test/support/test_macros.h
+++ b/libcxx/test/support/test_macros.h
@@ -169,9 +169,11 @@
 //  Specifically, FreeBSD does NOT have timespec_get, even though they have all
 //  the rest of C11 - this is PR#38495
 #    define TEST_HAS_ALIGNED_ALLOC
-#    define TEST_HAS_C11_FEATURES
+#    define TEST_HAS_QUICK_EXIT
 #  elif defined(__BIONIC__)
-#    define TEST_HAS_C11_FEATURES
+#    if __ANDROID_API__ >= 21
+#      define TEST_HAS_QUICK_EXIT
+#    endif
 #    if __ANDROID_API__ >= 28
 #      define TEST_HAS_ALIGNED_ALLOC
 #    endif
@@ -179,8 +181,8 @@
 #      define TEST_HAS_TIMESPEC_GET
 #    endif
 #  elif defined(__Fuchsia__) || defined(__wasi__) || defined(__NetBSD__)
+#    define TEST_HAS_QUICK_EXIT
 #    define TEST_HAS_ALIGNED_ALLOC
-#    define TEST_HAS_C11_FEATURES
 #    define TEST_HAS_TIMESPEC_GET
 #  elif defined(__linux__)
 // This block preserves the old behavior used by include/__config:
@@ -188,20 +190,22 @@
 // available. The configuration here may be too vague though, as Bionic, uClibc,
 // newlib, etc may all support these features but need to be configured.
 #    if defined(TEST_GLIBC_PREREQ)
+#      if TEST_GLIBC_PREREQ(2, 15)
+#        define TEST_HAS_QUICK_EXIT
+#      endif
 #      if TEST_GLIBC_PREREQ(2, 17)
 #        define TEST_HAS_ALIGNED_ALLOC
 #        define TEST_HAS_TIMESPEC_GET
-#        define TEST_HAS_C11_FEATURES
 #      endif
 #    elif defined(_LIBCPP_HAS_MUSL_LIBC)
+#      define TEST_HAS_QUICK_EXIT
 #      define TEST_HAS_ALIGNED_ALLOC
-#      define TEST_HAS_C11_FEATURES
 #      define TEST_HAS_TIMESPEC_GET
 #    endif
 #  elif defined(_WIN32)
 #    if defined(_MSC_VER) && !defined(__MINGW32__)
+#      define TEST_HAS_QUICK_EXIT
 #      define TEST_HAS_ALIGNED_ALLOC
-#      define TEST_HAS_C11_FEATURES // Using Microsoft's C Runtime library
 #      define TEST_HAS_TIMESPEC_GET
 #    endif
 #  endif

From 67dfba96296b37f7bac9b4a68572288bc44b63b2 Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne@apple.com>
Date: Mon, 27 Jul 2020 14:25:03 -0400
Subject: [PATCH 0354/1035] [libc++] Provide std::aligned_alloc and
 std::timespec_get on Apple platforms

rdar://66113878
---
 libcxx/include/__config           | 12 +++++++++++-
 libcxx/test/support/test_macros.h | 12 +++++++++++-
 2 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/libcxx/include/__config b/libcxx/include/__config
index a9eca04959bf4..8c22323c2babe 100644
--- a/libcxx/include/__config
+++ b/libcxx/include/__config
@@ -380,7 +380,17 @@
 #      define _LIBCPP_HAS_QUICK_EXIT
 #      define _LIBCPP_HAS_TIMESPEC_GET
 #    endif
-#  endif // __linux__
+#  elif defined(__APPLE__)
+     // timespec_get and aligned_alloc were introduced in macOS 10.15 and
+     // aligned releases
+#    if (__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 101500 || \
+         __ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__ >= 130000 || \
+         __ENVIRONMENT_TV_OS_VERSION_MIN_REQUIRED__ >= 130000 || \
+         __ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__ >= 60000)
+#      define _LIBCPP_HAS_ALIGNED_ALLOC
+#      define _LIBCPP_HAS_TIMESPEC_GET
+#    endif
+#  endif // __APPLE__
 #endif
 
 #ifndef _LIBCPP_CXX03_LANG
diff --git a/libcxx/test/support/test_macros.h b/libcxx/test/support/test_macros.h
index 15c6a280dc192..6fe1c56a5588e 100644
--- a/libcxx/test/support/test_macros.h
+++ b/libcxx/test/support/test_macros.h
@@ -208,7 +208,17 @@
 #      define TEST_HAS_ALIGNED_ALLOC
 #      define TEST_HAS_TIMESPEC_GET
 #    endif
-#  endif
+#  elif defined(__APPLE__)
+     // timespec_get and aligned_alloc were introduced in macOS 10.15 and
+     // aligned releases
+#    if (__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 101500 || \
+         __ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__ >= 130000 || \
+         __ENVIRONMENT_TV_OS_VERSION_MIN_REQUIRED__ >= 130000 || \
+         __ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__ >= 60000)
+#      define TEST_HAS_ALIGNED_ALLOC
+#      define TEST_HAS_TIMESPEC_GET
+#    endif
+#  endif // __APPLE__
 #endif
 
 /* Features that were introduced in C++14 */

From 3fb13b8484dcbec085da047879bf89ccb1b65b12 Mon Sep 17 00:00:00 2001
From: Sanjay Patel <spatel@rotateright.com>
Date: Tue, 28 Jul 2020 15:13:19 -0400
Subject: [PATCH 0355/1035] [InstSimplify] allow undefs in icmp with vector
 constant folds

This is the main icmp simplification shortcoming seen in D84655.

Alive2 agrees that the basic examples are correct at least:

define <2 x i1> @src(<2 x i8> %x) {
%0:
  %r = icmp sle <2 x i8> { undef, 128 }, %x
  ret <2 x i1> %r
}
=>
define <2 x i1> @tgt(<2 x i8> %x) {
%0:
  ret <2 x i1> { 1, 1 }
}
Transformation seems to be correct!

define <2 x i1> @src(<2 x i32> %X) {
%0:
  %A = or <2 x i32> %X, { 63, 63 }
  %B = icmp ult <2 x i32> %A, { undef, 50 }
  ret <2 x i1> %B
}
=>
define <2 x i1> @tgt(<2 x i32> %X) {
%0:
  ret <2 x i1> { 0, 0 }
}
Transformation seems to be correct!

https://alive2.llvm.org/ce/z/omt2ee
https://alive2.llvm.org/ce/z/GW4nP_

Differential Revision: https://reviews.llvm.org/D84762
---
 llvm/lib/Analysis/InstructionSimplify.cpp     |  2 +-
 .../Transforms/InstSimplify/icmp-constant.ll  | 22 +++++--------------
 2 files changed, 7 insertions(+), 17 deletions(-)

diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp
index c920fb3f52bea..01b2eb14a5b49 100644
--- a/llvm/lib/Analysis/InstructionSimplify.cpp
+++ b/llvm/lib/Analysis/InstructionSimplify.cpp
@@ -2732,7 +2732,7 @@ static Value *simplifyICmpWithConstant(CmpInst::Predicate Pred, Value *LHS,
   }
 
   const APInt *C;
-  if (!match(RHS, m_APInt(C)))
+  if (!match(RHS, m_APIntAllowUndef(C)))
     return nullptr;
 
   // Rule out tautological comparisons (eg., ult 0 or uge 0).
diff --git a/llvm/test/Transforms/InstSimplify/icmp-constant.ll b/llvm/test/Transforms/InstSimplify/icmp-constant.ll
index 8f51ba3c4c0e5..94126bf158487 100644
--- a/llvm/test/Transforms/InstSimplify/icmp-constant.ll
+++ b/llvm/test/Transforms/InstSimplify/icmp-constant.ll
@@ -21,8 +21,7 @@ define <2 x i1> @tautological_ule_vec(<2 x i8> %x) {
 
 define <2 x i1> @tautological_ule_vec_partial_undef(<2 x i8> %x) {
 ; CHECK-LABEL: @tautological_ule_vec_partial_undef(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ule <2 x i8> [[X:%.*]], <i8 -1, i8 undef>
-; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+; CHECK-NEXT:    ret <2 x i1> <i1 true, i1 true>
 ;
   %cmp = icmp ule <2 x i8> %x, <i8 255, i8 undef>
   ret <2 x i1> %cmp
@@ -46,8 +45,7 @@ define <2 x i1> @tautological_ugt_vec(<2 x i8> %x) {
 
 define <2 x i1> @tautological_ugt_vec_partial_undef(<2 x i8> %x) {
 ; CHECK-LABEL: @tautological_ugt_vec_partial_undef(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <2 x i8> [[X:%.*]], <i8 undef, i8 -1>
-; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
 ;
   %cmp = icmp ugt <2 x i8> %x, <i8 undef, i8 255>
   ret <2 x i1> %cmp
@@ -74,9 +72,7 @@ define <2 x i1> @urem3_vec(<2 x i32> %X) {
 
 define <2 x i1> @urem3_vec_partial_undef(<2 x i32> %X) {
 ; CHECK-LABEL: @urem3_vec_partial_undef(
-; CHECK-NEXT:    [[A:%.*]] = urem <2 x i32> [[X:%.*]], <i32 10, i32 10>
-; CHECK-NEXT:    [[B:%.*]] = icmp ult <2 x i32> [[A]], <i32 undef, i32 15>
-; CHECK-NEXT:    ret <2 x i1> [[B]]
+; CHECK-NEXT:    ret <2 x i1> <i1 true, i1 true>
 ;
   %A = urem <2 x i32> %X, <i32 10, i32 10>
   %B = icmp ult <2 x i32> %A, <i32 undef, i32 15>
@@ -104,9 +100,7 @@ define <2 x i1> @srem1_vec(<2 x i32> %X) {
 
 define <2 x i1> @srem1_vec_partial_undef(<2 x i32> %X) {
 ; CHECK-LABEL: @srem1_vec_partial_undef(
-; CHECK-NEXT:    [[A:%.*]] = srem <2 x i32> [[X:%.*]], <i32 -5, i32 -5>
-; CHECK-NEXT:    [[B:%.*]] = icmp sgt <2 x i32> [[A]], <i32 5, i32 undef>
-; CHECK-NEXT:    ret <2 x i1> [[B]]
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
 ;
   %A = srem <2 x i32> %X, <i32 -5, i32 -5>
   %B = icmp sgt <2 x i32> %A, <i32 5, i32 undef>
@@ -211,9 +205,7 @@ define <2 x i1> @shl5_vec(<2 x i32> %X) {
 
 define <2 x i1> @shl5_vec_partial_undef(<2 x i32> %X) {
 ; CHECK-LABEL: @shl5_vec_partial_undef(
-; CHECK-NEXT:    [[SUB:%.*]] = shl nuw <2 x i32> <i32 4, i32 4>, [[X:%.*]]
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <2 x i32> [[SUB]], <i32 undef, i32 3>
-; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+; CHECK-NEXT:    ret <2 x i1> <i1 true, i1 true>
 ;
   %sub = shl nuw <2 x i32> <i32 4, i32 4>, %X
   %cmp = icmp ugt <2 x i32> %sub, <i32 undef, i32 3>
@@ -431,9 +423,7 @@ define <2 x i1> @or1_vec(<2 x i32> %X) {
 
 define <2 x i1> @or1_vec_partial_undef(<2 x i32> %X) {
 ; CHECK-LABEL: @or1_vec_partial_undef(
-; CHECK-NEXT:    [[A:%.*]] = or <2 x i32> [[X:%.*]], <i32 62, i32 62>
-; CHECK-NEXT:    [[B:%.*]] = icmp ult <2 x i32> [[A]], <i32 undef, i32 50>
-; CHECK-NEXT:    ret <2 x i1> [[B]]
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
 ;
   %A = or <2 x i32> %X, <i32 62, i32 62>
   %B = icmp ult <2 x i32> %A, <i32 undef, i32 50>

From 811828a01d64868329de279a7117e12d10e9da2a Mon Sep 17 00:00:00 2001
From: Ahsan Saghir <saghir@ca.ibm.com>
Date: Tue, 28 Jul 2020 13:29:25 -0500
Subject: [PATCH 0356/1035] [PowerPC] Mark allocator_oom_test.cpp unsupported
 on PowerPC

This patch marks compiler-rt/test/asan/TestCases/Linux/allocator_oom_test.cpp
unsupported on PowerPC 64bit-LE architecture since this test fails when run
on a machine with larger system memory.

Reviewed By: #powerpc, nemanjai

Differential Revision: https://reviews.llvm.org/D84786
---
 compiler-rt/test/asan/TestCases/Linux/allocator_oom_test.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/compiler-rt/test/asan/TestCases/Linux/allocator_oom_test.cpp b/compiler-rt/test/asan/TestCases/Linux/allocator_oom_test.cpp
index c450ae5bb8e91..c294ca53b6be5 100644
--- a/compiler-rt/test/asan/TestCases/Linux/allocator_oom_test.cpp
+++ b/compiler-rt/test/asan/TestCases/Linux/allocator_oom_test.cpp
@@ -32,7 +32,7 @@
 // AArch64 bots fail on this test.
 // TODO(alekseys): Android lit do not run ulimit on device.
 // REQUIRES: shadow-scale-3
-// UNSUPPORTED: s390,android,aarch64
+// UNSUPPORTED: s390,android,aarch64,powerpc64le
 
 #include <stdlib.h>
 #include <string.h>

From 3fac05e49fe3eb4d6becf1761b4df01dbd871aa4 Mon Sep 17 00:00:00 2001
From: clementval <clementval@gmail.com>
Date: Tue, 28 Jul 2020 15:26:27 -0400
Subject: [PATCH 0357/1035] [openacc] Add missing newline at end of file

---
 llvm/include/llvm/Frontend/OpenACC/ACC.td | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/include/llvm/Frontend/OpenACC/ACC.td b/llvm/include/llvm/Frontend/OpenACC/ACC.td
index e96b7e8466628..4dbf4bbc0fb0b 100644
--- a/llvm/include/llvm/Frontend/OpenACC/ACC.td
+++ b/llvm/include/llvm/Frontend/OpenACC/ACC.td
@@ -601,4 +601,4 @@ def ACC_SerialLoop : Directive<"serial loop"> {
 
 def ACC_Unknown : Directive<"unknown"> {
   let isDefault = 1;
-}
\ No newline at end of file
+}

From 8e67982384d4a11892c04d16c2d10d7533e56094 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Tue, 28 Jul 2020 20:31:57 +0100
Subject: [PATCH 0358/1035] [NewGVN] Add test cases for remaining known issues.

This patch adds IR tests for the known NewGVN issues. The intention is
that adding them now will make it easier to keep track of fixes.
---
 .../NewGVN/todo-pr33165-distribute-undef.ll   | 17 ++++
 .../NewGVN/todo-pr35074-phi-of-ops.ll         | 35 ++++++++
 .../NewGVN/todo-pr36335-phi-undef.ll          | 32 +++++++
 .../todo-pr37121-seens-this-value-a-lot.ll    | 25 ++++++
 .../NewGVN/todo-pr42422-phi-of-ops.ll         | 84 +++++++++++++++++++
 5 files changed, 193 insertions(+)
 create mode 100644 llvm/test/Transforms/NewGVN/todo-pr33165-distribute-undef.ll
 create mode 100644 llvm/test/Transforms/NewGVN/todo-pr35074-phi-of-ops.ll
 create mode 100644 llvm/test/Transforms/NewGVN/todo-pr36335-phi-undef.ll
 create mode 100644 llvm/test/Transforms/NewGVN/todo-pr37121-seens-this-value-a-lot.ll
 create mode 100644 llvm/test/Transforms/NewGVN/todo-pr42422-phi-of-ops.ll

diff --git a/llvm/test/Transforms/NewGVN/todo-pr33165-distribute-undef.ll b/llvm/test/Transforms/NewGVN/todo-pr33165-distribute-undef.ll
new file mode 100644
index 0000000000000..0a9255c448e56
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/todo-pr33165-distribute-undef.ll
@@ -0,0 +1,17 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -newgvn -S %s | FileCheck %s
+
+; Test for PR33165.
+
+; TODO: Currently NewGVN miscompiles the function.
+define i2 @f(i2, i1) {
+; CHECK-LABEL: @f(
+; CHECK-NEXT:    [[A:%.*]] = xor i2 [[TMP0:%.*]], -1
+; CHECK-NEXT:    [[B:%.*]] = select i1 [[TMP1:%.*]], i2 [[A]], i2 undef
+; CHECK-NEXT:    ret i2 [[B]]
+;
+  %a = xor i2 %0, -1
+  %b = select i1 %1, i2 %a, i2 undef
+  %c = and i2 %a, %b
+  ret i2 %c
+}
diff --git a/llvm/test/Transforms/NewGVN/todo-pr35074-phi-of-ops.ll b/llvm/test/Transforms/NewGVN/todo-pr35074-phi-of-ops.ll
new file mode 100644
index 0000000000000..e80292e94fd1c
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/todo-pr35074-phi-of-ops.ll
@@ -0,0 +1,35 @@
+; RUN: opt -newgvn -S %s | FileCheck %s
+
+XFAIL: *
+
+; TODO: Test case for PR35074. Crashes caused by phi-of-ops.
+define void @crash1_pr35074(i32 %this, i1 %c) {
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc6, %entry
+  %y.0 = phi i32 [ 1, %entry ], [ %inc7, %for.inc6 ]
+  br i1 %c, label %for.inc6, label %for.body.lr.ph
+
+for.body.lr.ph:                                   ; preds = %for.cond
+  %sub = add nsw i32 %y.0, -1
+  br label %for.body4
+
+for.body4:                                        ; preds = %for.body.lr.ph
+  %cmp = icmp ugt i32 %sub, %y.0
+  br i1 %cmp, label %for.end, label %for.body4.1
+
+for.end:                                          ; preds = %for.body4.1, %for.body4
+  ret void
+
+for.inc6:                                         ; preds = %for.cond
+  %inc7 = add nuw nsw i32 %y.0, 1
+  br label %for.cond
+
+for.body4.1:                                      ; preds = %for.body4
+  %inc.1 = add nuw nsw i32 %y.0, 1
+  tail call void @_blah(i32 %inc.1)
+  br label %for.end
+}
+
+declare void @_blah(i32)
diff --git a/llvm/test/Transforms/NewGVN/todo-pr36335-phi-undef.ll b/llvm/test/Transforms/NewGVN/todo-pr36335-phi-undef.ll
new file mode 100644
index 0000000000000..6208f4eb19346
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/todo-pr36335-phi-undef.ll
@@ -0,0 +1,32 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -newgvn -S %s | FileCheck %s
+
+; TODO: NewGVN currently miscomiles the function below. PR36335.
+
+declare void @foo(i32)
+
+define void @main(i1 %c1, i1 %c2, i32 %x) {
+; CHECK-LABEL: @main(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[C1:%.*]], label [[L:%.*]], label [[END:%.*]]
+; CHECK:       L:
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[X:%.*]], -1
+; CHECK-NEXT:    call void @foo(i32 [[XOR]])
+; CHECK-NEXT:    br label [[L]]
+; CHECK:       end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br i1 %c1, label %L, label %end
+
+L:
+  %d.1 = phi i8 [ undef, %entry ], [ -1, %L ]
+  %conv = sext i8 %d.1 to i32
+  %xor = xor i32 %x, %conv
+  %neg = xor i32 %xor, -1
+  call void @foo(i32 %neg)
+  br label %L
+
+end:
+  ret void
+}
diff --git a/llvm/test/Transforms/NewGVN/todo-pr37121-seens-this-value-a-lot.ll b/llvm/test/Transforms/NewGVN/todo-pr37121-seens-this-value-a-lot.ll
new file mode 100644
index 0000000000000..b4961fa32ef00
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/todo-pr37121-seens-this-value-a-lot.ll
@@ -0,0 +1,25 @@
+; RUN: opt -newgvn -S %s | FileCheck %s
+
+; XFAIL: *
+
+; TODO: Current NewGVN crashes on the function below. See PR37121.
+
+define hidden void @foo() {
+top:
+  %.promoted = load i8, i8* undef, align 8
+  br label %if
+
+;; This is really a multi-valued phi, because the phi is defined by an expression of the phi.
+;; This means that we can't propagate the value over the backedge, because we'll just cycle
+;; through every value.
+
+if:                                               ; preds = %if, %top
+  %0 = phi i8 [ %1, %if ], [ %.promoted, %top ]
+  %1 = xor i8 %0, undef
+  br i1 false, label %L50, label %if
+
+L50:                                              ; preds = %if
+  %.lcssa = phi i8 [ %1, %if ]
+  store i8 %.lcssa, i8* undef, align 8
+  ret void
+}
diff --git a/llvm/test/Transforms/NewGVN/todo-pr42422-phi-of-ops.ll b/llvm/test/Transforms/NewGVN/todo-pr42422-phi-of-ops.ll
new file mode 100644
index 0000000000000..6c4cf5f82314b
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/todo-pr42422-phi-of-ops.ll
@@ -0,0 +1,84 @@
+; RUN: opt -newgvn -S %s | FileChecks %s
+
+; XFAIL: *
+
+; TODO: Currently NewGVN crashes on the function below, see PR42422.
+
+define void @d() {
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %cleanup20, %entry
+  br label %for.cond1
+
+for.cond1:                                        ; preds = %for.inc17, %for.cond
+  %0 = phi i32 [ %inc18, %for.inc17 ], [ 0, %for.cond ]
+  %cmp = icmp sle i32 %0, 1
+  br i1 %cmp, label %for.body, label %for.end19
+
+for.body:                                         ; preds = %for.cond1
+  br i1 undef, label %for.body3, label %for.body.for.cond4_crit_edge
+
+for.body.for.cond4_crit_edge:                     ; preds = %for.body
+  br label %for.cond4
+
+for.body3:                                        ; preds = %for.body
+  br label %cleanup14
+
+for.cond4:                                        ; preds = %cleanup, %for.body.for.cond4_crit_edge
+  br i1 undef, label %if.then, label %if.end
+
+if.then:                                          ; preds = %for.cond4
+  br label %cleanup
+
+if.end:                                           ; preds = %for.cond4
+  br label %for.cond6
+
+for.cond6:                                        ; preds = %for.inc, %if.end
+  %1 = phi i64 [ %inc, %for.inc ], [ 0, %if.end ]
+  %cmp7 = icmp sle i64 %1, 1
+  br i1 %cmp7, label %for.inc, label %for.end9
+
+for.inc:                                          ; preds = %for.cond6
+  %inc = add nsw i64 %1, 1
+  br label %for.cond6
+
+for.end9:                                         ; preds = %for.cond6
+  br i1 true, label %if.then11, label %if.end12
+
+if.then11:                                        ; preds = %for.end9
+  br label %cleanup
+
+if.end12:                                         ; preds = %for.end9
+  br label %cleanup
+
+cleanup:                                          ; preds = %if.end12, %if.then11, %if.then
+  %cleanup.dest = phi i32 [ undef, %if.end12 ], [ 1, %if.then11 ], [ 9, %if.then ]
+  switch i32 %cleanup.dest, label %cleanup14 [
+    i32 0, label %for.cond4
+    i32 9, label %for.end13
+  ]
+
+for.end13:                                        ; preds = %cleanup
+  br label %cleanup14
+
+cleanup14:                                        ; preds = %for.end13, %cleanup, %for.body3
+  %cleanup.dest15 = phi i32 [ 0, %for.end13 ], [ %cleanup.dest, %cleanup ], [ 1, %for.body3 ]
+  %cond1 = icmp eq i32 %cleanup.dest15, 0
+  br i1 %cond1, label %for.inc17, label %cleanup20
+
+for.inc17:                                        ; preds = %cleanup14
+  %inc18 = add nsw i32 %0, 1
+  br label %for.cond1
+
+for.end19:                                        ; preds = %for.cond1
+  br label %cleanup20
+
+cleanup20:                                        ; preds = %for.end19, %cleanup14
+  %cleanup.dest21 = phi i32 [ %cleanup.dest15, %cleanup14 ], [ 0, %for.end19 ]
+  %cond = icmp eq i32 %cleanup.dest21, 0
+  br i1 %cond, label %for.cond, label %cleanup23
+
+cleanup23:                                        ; preds = %cleanup20
+  ret void
+}

From b8943e7cea7cafebe26dd96e1a0d98e1f68a1386 Mon Sep 17 00:00:00 2001
From: Vincent Zhao <vincentzhaorz@gmail.com>
Date: Tue, 28 Jul 2020 19:40:55 +0000
Subject: [PATCH 0359/1035] [MLIR][Linalg] Fixed obsolete examples in the MLIR
 Linalg Dialect doc

This diff fixes some obsolete examples in the Linalg dialect documentation: https://mlir.llvm.org/docs/Dialects/Linalg/

These examples are used to explain the basic properties of the Linalg dialect, which are not automatically generated from TableGen and are using out-of-date MLIR/Linalg syntax.

This diff extends each example by adding essential attributes and changing its syntax to make it processible by `mlir-opt`. There is also a command attached to each example that says how the example can be processed.

Reviewed By: ftynse

Differential Revision: https://reviews.llvm.org/D84229
---
 mlir/docs/Dialects/Linalg.md | 247 ++++++++++++++++++++++-------------
 1 file changed, 157 insertions(+), 90 deletions(-)

diff --git a/mlir/docs/Dialects/Linalg.md b/mlir/docs/Dialects/Linalg.md
index b383294ffd741..edf5eb217799a 100644
--- a/mlir/docs/Dialects/Linalg.md
+++ b/mlir/docs/Dialects/Linalg.md
@@ -60,32 +60,55 @@ needed to synthesize the control-flow required to iterate over its operands,
 according to their type. This notion of IR localization bears some resemblance
 to [URUK](http://icps.u-strasbg.fr/~bastoul/research/papers/GVBCPST06-IJPP.pdf).
 
-Consider the following, partially specified, `linalg.generic` example:
-```
-#attrs = {args_in: 1, args_out: 1}
-func @example(%A: memref<?xf32, layout1>,
-              %B: memref<?xvector<4xf32, layout2>>) {
-  linalg.generic #attrs (%2, %3): memref<?xf32, layout1>,
-                                  memref<?xvector<4xf32, layout2>>
+Consider the following fully specified `linalg.generic` example.
+Here, the first operand is a `memref` of `f32` scalar elements that
+has an ordinary identity layout, and the second one is a `memref` of
+4-element vectors with a 2-strided, 1-offset layout.
+
+```mlir
+// File name: example1.mlir
+#accesses = [
+  affine_map<(m) -> (m)>,
+  affine_map<(m) -> (m)>
+]
+#attrs = {
+  args_in = 1,
+  args_out = 1,
+  indexing_maps = #accesses,
+  iterator_types = ["parallel"]
+}
+// memory layouts
+#identity = affine_map<(d0) -> (d0)>
+
+func @example(%A: memref<?xf32, #identity>,
+              %B: memref<?xvector<4xf32>, offset: 1, strides: [2]>) {
+  linalg.generic #attrs %A, %B {
+  ^bb0(%a: f32, %b: vector<4xf32>):
+    %c = "some_compute"(%a, %b): (f32, vector<4xf32>) -> (vector<4xf32>)
+    linalg.yield %c: vector<4xf32>
+  } : memref<?xf32, #identity>, memref<?xvector<4xf32>, offset: 1, strides: [2]>
   return
 }
 ```
 
 The property "*Input and Output Operands Define The Iteration Space*" is
 materialized by a lowering into a form that will resemble:
-```
-func @example(%A: memref<?xf32, layout1>,
-              %B: memref<?xvector<4xf32, layout2>>) {
-  %M = "dim" %A, 0: index
-  %N = "dim" %B, 0: index
-  %eq = eq %M, %N: i1   // iteration space is consistent with data
-  assert(%eq): (i1) -> ()
-  for %i = 0 to %M {
-    %a = load %A[%i]: memref<?xf32, layout1>
-    %b = load %B[%i]: memref<?xvector<4xf32>, layout2>
-    // compute arg types match elemental tensor types
-    %c = "some_compute"(%a, %b): (f32, vector<4xf32>) -> (vector<4xf32>)
-    store %c, %B[%i]: memref<?xvector<4xf32>, layout2>
+
+```mlir
+// Run: mlir-opt example1.mlir -allow-unregistered-dialect -convert-linalg-to-loops
+// This converted representation is in the `scf` dialect.
+// It's syntax can be found here: https://mlir.llvm.org/docs/Dialects/SCFDialect/
+#map0 = affine_map<(d0) -> (d0 * 2 + 1)>
+
+func @example(%arg0: memref<?xf32>, %arg1: memref<?xvector<4xf32>, #map0>) {
+  %c0 = constant 0 : index
+  %c1 = constant 1 : index
+  %0 = dim %arg0, %c0 : memref<?xf32>
+  scf.for %arg2 = %c0 to %0 step %c1 {
+    %1 = load %arg0[%arg2] : memref<?xf32>
+    %2 = load %arg1[%arg2] : memref<?xvector<4xf32>, #map0>
+    %3 = "some_compute"(%1, %2) : (f32, vector<4xf32>) -> vector<4xf32>
+    store %3, %arg1[%arg2] : memref<?xvector<4xf32>, #map0>
   }
   return
 }
@@ -123,17 +146,30 @@ as well as [TACO](http://tensor-compiler.org/), has shown.
 A `linalg.generic` *defines* the mapping between the iteration space (i.e. the
 loops) and the data.
 
-Consider the following, partially specified, `linalg.generic` example:
+Consider the following fully specified `linalg.generic` example.
+Here, the first `memref` is a 2-strided one on both of its dimensions,
+and the second `memref` uses an identity layout.
+
 ```
-#indexing_maps = {
-  (i, j) -> (j, i),
-  (i, j) -> (j)
+// File name: example2.mlir
+#indexing_maps = [
+  affine_map<(i, j) -> (j, i)>,
+  affine_map<(i, j) -> (j)>
+]
+#attrs = {
+  args_in = 1,
+  args_out = 1,
+  indexing_maps = #indexing_maps,
+  iterator_types = ["parallel", "parallel"]
 }
-#attrs = {args_in: 1, args_out: 1, indexings: indexing_maps}
-func @example(%A: memref<8x?xf32, layout1>,
-              %B: memref<?xvector<4xf32, layout2>>) {
-  linalg.generic #attrs (%A, %B): memref<8x?xf32, layout1>,
-                                  memref<?xvector<4xf32, layout2>>
+
+func @example(%A: memref<8x?xf32, offset: 0, strides: [2, 2]>,
+              %B: memref<?xvector<4xf32>>) {
+  linalg.generic #attrs %A, %B {
+  ^bb0(%a: f32, %b: vector<4xf32>):
+    %c = "some_compute"(%a, %b): (f32, vector<4xf32>) -> (vector<4xf32>)
+    linalg.yield %c: vector<4xf32>
+  }: memref<8x?xf32 , offset: 0, strides: [2, 2]>, memref<?xvector<4xf32>>
   return
 }
 ```
@@ -141,22 +177,20 @@ func @example(%A: memref<8x?xf32, layout1>,
 The property "*Reversible Mappings Between Control and Data Structures*" is
 materialized by a lowering into a form that will resemble:
 ```
-#attrs = {args_in: 1, args_out: 1, indexings: indexing_maps}
-func @example(%A: memref<8x?xf32, layout1>,
-              %B: memref<?xvector<4xf32, layout2>>) {
-  // loop bounds determined from data sizes by “inverting the map”
-  %J = "dim" %A, 0: index
-  %I = "dim" %A, 1: index
-  %J2 = "dim" %B, 0: index
-  // iteration space is consistent with data + mapping inference
-  %eq = "eq" %J, %J2: i1
-  "assert" %eq: (i1) -> ()
-  for %i = 0 to %I {           // loop order is fully defined by indexing maps
-    for %j = 0 to %J {         // arbitrary permutations are possible
-      %a = "load" %A, %j, %i: memref<8x?xf32>
-      %b = "load" %B, %j: memref<?xvector<4xf32>>
-      %c = "some_compute"(%a, %b): (f32, vector<4xf32>) -> (vector<4xf32>)
-      "store" %c, %B, %j: memref<?xvector<4xf32>>
+// Run: mlir-opt example2.mlir -allow-unregistered-dialect -convert-linalg-to-loops
+#map0 = affine_map<(d0, d1) -> (d0 * 2 + d1 * 2)>
+
+func @example(%arg0: memref<8x?xf32, #map0>, %arg1: memref<?xvector<4xf32>>) {
+  %c8 = constant 8 : index
+  %c0 = constant 0 : index
+  %c1 = constant 1 : index
+  %0 = dim %arg0, %c1 : memref<8x?xf32, #map0>
+  scf.for %arg2 = %c0 to %0 step %c1 {
+    scf.for %arg3 = %c0 to %c8 step %c1 {
+      %1 = load %arg0[%arg3, %arg2] : memref<8x?xf32, #map0>
+      %2 = load %arg1[%arg3] : memref<?xvector<4xf32>>
+      %3 = "some_compute"(%1, %2) : (f32, vector<4xf32>) -> vector<4xf32>
+      store %3, %arg1[%arg3] : memref<?xvector<4xf32>>
     }
   }
   return
@@ -174,7 +208,7 @@ Answering these `2` questions is one of the main analyses that Linalg uses to
 implement transformations such as tiling, tiled producer-consumer fusion, and
 promotion to temporary buffers in fast memory.
 
-In the current implementation, `linalg.generic` uses a list of [AffineMaps]().
+In the current implementation, `linalg.generic` uses a list of [AffineMaps](https://mlir.llvm.org/docs/LangRef/#affinemap-attribute) (see the `#indexing_maps` attribute in the previous examples).
 This is a pragmatic short-term solution, but in the longer term note that
 this property could be even evaluated dynamically, similarly to
 inspector-executor algorithms.
@@ -234,38 +268,53 @@ to correspond to the operations inside the region: the region can capture
 buffers arbitrarily and write into them. If this conflicts with some parallel
 iterator requirement, this is undefined behavior.
 
-Concretely, consider the following, partially specified, `linalg.generic`
-example:
+Previous examples already ellaborate compute payloads with an unregistered function `"some_compute"`. The following code snippet shows what the result will be when using a concrete operation `addf`:
 ```
-#indexing_maps = {
-  (i, j) -> (i, j),
-  (i, j) -> (i, j)
+// File name: example3.mlir
+#indexing_maps = [
+  affine_map<(i, j) -> (i, j)>,
+  affine_map<(i, j) -> (i, j)>,
+  affine_map<(i, j) -> (i, j)>
+]
+#attrs = {
+  args_in = 2,
+  args_out = 1,
+  indexing_maps = #indexing_maps,
+  iterator_types = ["parallel", "parallel"]
 }
-#attrs = {args_in: 2, args_out: 1, indexings: #indexing_maps}
 func @example(%A: memref<?x?xf32>, %B: memref<?x?xf32>, %C: memref<?x?xf32>) {
-  linalg.generic #attrs (%A, %B, %C) {
-    ^bb0(%a: f32, %b: f32):
-      %c = addf %a, %b : f32
-      return %c : f32
+  linalg.generic #attrs %A, %B, %C {
+  ^bb0(%a: f32, %b: f32, %c: f32):
+    %d = addf %a, %b : f32
+    linalg.yield %d : f32
   }: memref<?x?xf32>, memref<?x?xf32>, memref<?x?xf32>
   return
 }
 ```
 
+This function basically element-wise adds up two matrices (`%A` and `%B`) and stores the result into another one (`%C`).
+
 The property "*The Compute Payload is Specified With a Region*" is
 materialized by a lowering into a form that will resemble:
 ```
+// Run: mlir-opt example3.mlir -convert-linalg-to-loops
+#indexing_maps = [
+  affine_map<(i, j) -> (i, j)>,
+  affine_map<(i, j) -> (i, j)>,
+  affine_map<(i, j) -> (i, j)>
+]
+#attrs = {
+  args_in = 2,
+  args_out = 1,
+  indexing_maps = #indexing_maps,
+  iterator_types = ["parallel", "parallel"]
+}
 func @example(%A: memref<?x?xf32>, %B: memref<?x?xf32>, %C: memref<?x?xf32>) {
-  %M = dim %A, 0: index
-  %N = dim %B, 1: index
-  for %i = 0 to %M {
-    for %j = 0 to %N {
-      %a = load %A[%i, %j]: memref<?x?xf32>
-      %b = load %B[%i, %j]: memref<?x?xf32>>
-      %c = addf %a, %b : f32
-      store %c, %C[%i, %j]: memref<?x?xf32>
-    }
-  }
+  linalg.generic #attrs %A, %B, %C {
+  ^bb0(%a: f32, %b: f32, %c: f32):
+    %d = addf %a, %b : f32
+    linalg.yield %d : f32
+  }: memref<?x?xf32>, memref<?x?xf32>, memref<?x?xf32>
   return
 }
 ```
@@ -287,20 +336,27 @@ and integration at the ABI level. Regardless of whether one wants to use
 external library calls or a custom ISA, the problem for codegen is similar:
 preservation of a fixed granularity.
 
-Consider the following, partially specified, `linalg.generic`
-example:
+Consider the following example that adds an additional attribute `library_call="pointwise_add"`
+that specifies the name of an external library call we intend to use:
 ```
-#fun_attr = "pointwise_add"
-#indexing_maps = {
-  (i, j) -> (i, j),
-  (i, j) -> (i, j)
+// File name: example4.mlir
+#indexing_maps = [
+  affine_map<(i, j) -> (i, j)>,
+  affine_map<(i, j) -> (i, j)>,
+  affine_map<(i, j) -> (i, j)>
+]
+#attrs = {
+  args_in = 2,
+  args_out = 1,
+  indexing_maps = #indexing_maps,
+  iterator_types = ["parallel", "parallel"],
+  library_call = "pointwise_add"
 }
-#attrs = {args_in: 2, args_out: 1, indexings: #indexing_maps, fun: #fun_attr}
 func @example(%A: memref<?x?xf32>, %B: memref<?x?xf32>, %C: memref<?x?xf32>) {
-  linalg.generic #attrs (%A, %B, %C) {
-    ^bb0(%a: f32, %b: f32):
-      %c = addf %a, %b : f32
-      return %c : f32
+  linalg.generic #attrs %A, %B, %C {
+  ^bb0(%a: f32, %b: f32, %c: f32):
+    %d = addf %a, %b : f32
+    linalg.yield %d : f32
   }: memref<?x?xf32>, memref<?x?xf32>, memref<?x?xf32>
   return
 }
@@ -310,28 +366,39 @@ The property "*Map To an External Library Call*" is
 materialized by a lowering into a form that will resemble:
 
 ```
-func @pointwise_add_sxsxf32_sxsxf32(memref<?x?xf32>, memref<?x?xf32>, memref<?x?xf32>) -> ()
+// Run: mlir-opt example4.mlir -convert-linalg-to-std
+// Note that we lower the Linalg dialect directly to the Standard dialect.
+// See this doc: https://mlir.llvm.org/docs/Dialects/Standard/
 
-func @example(%A: memref<?x?xf32>, %B: memref<?x?xf32>, %C: memref<?x?xf32>) {
-  call @pointwise_add_sxsxf32_sxsxf32 (%A, %B, %C):
-    (memref<?x?xf32>, memref<?x?xf32>, memref<?x?xf32>) -> ()
+#map0 = affine_map<(d0, d1)[s0, s1, s2] -> (d0 * s1 + s0 + d1 * s2)>
+
+func @example(%arg0: memref<?x?xf32>, %arg1: memref<?x?xf32>, %arg2: memref<?x?xf32>) {
+  %0 = memref_cast %arg0 : memref<?x?xf32> to memref<?x?xf32, #map0>
+  %1 = memref_cast %arg1 : memref<?x?xf32> to memref<?x?xf32, #map0>
+  %2 = memref_cast %arg2 : memref<?x?xf32> to memref<?x?xf32, #map0>
+  call @pointwise_add(%0, %1, %2) : (memref<?x?xf32, #map0>, memref<?x?xf32, #map0>, memref<?x?xf32, #map0>) -> ()
   return
 }
+func @pointwise_add(memref<?x?xf32, #map0>, memref<?x?xf32, #map0>, memref<?x?xf32, #map0>) attributes {llvm.emit_c_interface}
 ```
 
 Which, after lowering to LLVM resembles:
 ```
-func @pointwise_add_sxsxf32_sxsxf32(!llvm<"{ float*, i64, [2 x i64], [3 x i64] }*">,
-                                    !llvm<"{ float*, i64, [2 x i64], [3 x i64] }*">,
-                                    !llvm<"{ float*, i64, [2 x i64], [3 x i64] }*">) -> ()
-
-func @example(%A: !llvm<"{ float*, i64, [2 x i64], [3 x i64] }*">,
-              %B: !llvm<"{ float*, i64, [2 x i64], [3 x i64] }*">,
-              %C: !llvm<"{ float*, i64, [2 x i64], [3 x i64] }*">) {
-  llvm.call @pointwise_add_sxsxf32_sxsxf32 (%A, %B, %C):
-    (!llvm<"{ float*, i64, [2 x i64], [3 x i64] }*">...) -> ()
+// Run: mlir-opt example4.mlir -convert-linalg-to-std | mlir-opt -convert-std-to-llvm
+// Some generated code are omitted here.
+func @example(%arg0: !llvm<"float*">, ...) {
+  ...
+  llvm.call @pointwise_add(...) : (!llvm<"float*">, ...) -> ()
   return
 }
+
+llvm.func @pointwise_add(%arg0: !llvm<"float*">, ...) attributes {llvm.emit_c_interface} {
+  ...
+  llvm.call @_mlir_ciface_pointwise_add(%9, %19, %29) : (!llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }*">, !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }*">, !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }
+*">) -> ()
+  llvm.return
+}
+llvm.func @_mlir_ciface_pointwise_add(!llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }*">, !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }*">, !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }*">) attributes {llvm.emit_c_interface}
 ```
 
 ##### Convention For External Library Interoperability

From 1e027b77f056fb50e50d9f743e21eb41151c32da Mon Sep 17 00:00:00 2001
From: Mircea Trofin <mtrofin@google.com>
Date: Tue, 28 Jul 2020 13:04:11 -0700
Subject: [PATCH 0360/1035] [llvm][NFC] refactor setBlockFrequency for clarity.

The refactoring encapsulates frequency calculation in MachineBlockFrequencyInfo,
and renames the API to clarify its motivation. It should clarify
frequencies may not be reset 'freely' by users of the analysis, as the
API serves as a partial update to avoid a full analysis recomputation.

Differential Revision: https://reviews.llvm.org/D84427
---
 llvm/include/llvm/CodeGen/MachineBlockFrequencyInfo.h |  6 +++++-
 llvm/lib/CodeGen/MachineBlockFrequencyInfo.cpp        | 11 ++++++++---
 llvm/lib/CodeGen/MachineSink.cpp                      |  8 +++-----
 3 files changed, 16 insertions(+), 9 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/MachineBlockFrequencyInfo.h b/llvm/include/llvm/CodeGen/MachineBlockFrequencyInfo.h
index 7ce11c784b08f..6c442d3d07bdb 100644
--- a/llvm/include/llvm/CodeGen/MachineBlockFrequencyInfo.h
+++ b/llvm/include/llvm/CodeGen/MachineBlockFrequencyInfo.h
@@ -74,7 +74,11 @@ class MachineBlockFrequencyInfo : public MachineFunctionPass {
 
   bool isIrrLoopHeader(const MachineBasicBlock *MBB) const;
 
-  void setBlockFreq(const MachineBasicBlock *MBB, uint64_t Freq);
+  /// incrementally calculate block frequencies when we split edges, to avoid
+  /// full CFG traversal.
+  void onEdgeSplit(const MachineBasicBlock &NewPredecessor,
+                   const MachineBasicBlock &NewSuccessor,
+                   const MachineBranchProbabilityInfo &MBPI);
 
   const MachineFunction *getFunction() const;
   const MachineBranchProbabilityInfo *getMBPI() const;
diff --git a/llvm/lib/CodeGen/MachineBlockFrequencyInfo.cpp b/llvm/lib/CodeGen/MachineBlockFrequencyInfo.cpp
index c836997060149..54e0a14e05557 100644
--- a/llvm/lib/CodeGen/MachineBlockFrequencyInfo.cpp
+++ b/llvm/lib/CodeGen/MachineBlockFrequencyInfo.cpp
@@ -247,10 +247,15 @@ bool MachineBlockFrequencyInfo::isIrrLoopHeader(
   return MBFI->isIrrLoopHeader(MBB);
 }
 
-void MachineBlockFrequencyInfo::setBlockFreq(const MachineBasicBlock *MBB,
-                                             uint64_t Freq) {
+void MachineBlockFrequencyInfo::onEdgeSplit(
+    const MachineBasicBlock &NewPredecessor,
+    const MachineBasicBlock &NewSuccessor,
+    const MachineBranchProbabilityInfo &MBPI) {
   assert(MBFI && "Expected analysis to be available");
-  MBFI->setBlockFreq(MBB, Freq);
+  auto NewSuccFreq = MBFI->getBlockFreq(&NewPredecessor) *
+                     MBPI.getEdgeProbability(&NewPredecessor, &NewSuccessor);
+
+  MBFI->setBlockFreq(&NewSuccessor, NewSuccFreq.getFrequency());
 }
 
 const MachineFunction *MachineBlockFrequencyInfo::getFunction() const {
diff --git a/llvm/lib/CodeGen/MachineSink.cpp b/llvm/lib/CodeGen/MachineSink.cpp
index 5f958bbc31b7a..dfb88f6bf5170 100644
--- a/llvm/lib/CodeGen/MachineSink.cpp
+++ b/llvm/lib/CodeGen/MachineSink.cpp
@@ -347,11 +347,9 @@ bool MachineSinking::runOnMachineFunction(MachineFunction &MF) {
                           << printMBBReference(*Pair.first) << " -- "
                           << printMBBReference(*NewSucc) << " -- "
                           << printMBBReference(*Pair.second) << '\n');
-        if (MBFI) {
-          auto NewSuccFreq = MBFI->getBlockFreq(Pair.first) *
-                             MBPI->getEdgeProbability(Pair.first, NewSucc);
-          MBFI->setBlockFreq(NewSucc, NewSuccFreq.getFrequency());
-        }
+        if (MBFI)
+          MBFI->onEdgeSplit(*Pair.first, *NewSucc, *MBPI);
+
         MadeChange = true;
         ++NumSplit;
       } else

From 1f166edeb47ea3584f4f6a267a9054af994af45c Mon Sep 17 00:00:00 2001
From: Hafiz Abid Qadeer <abidh@codesourcery.com>
Date: Thu, 16 Jul 2020 21:40:31 +0100
Subject: [PATCH 0361/1035] [lld][linkerscript] Fix handling of DEFINED.

Current implementation did not check that symbols is actually defined. Only checked for presence.  GNU ld documentation says,

"Return 1 if symbol is in the linker global symbol table and is defined before the statement using DEFINED in the script, otherwise return 0."

https://sourceware.org/binutils/docs/ld/Builtin-Functions.html#Builtin-Functions

Reviewed By: MaskRay

Differential Revision: https://reviews.llvm.org/D83758
---
 lld/ELF/ScriptParser.cpp                  | 5 ++++-
 lld/test/ELF/linkerscript/Inputs/define.s | 3 +++
 lld/test/ELF/linkerscript/define.test     | 6 +++++-
 3 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/lld/ELF/ScriptParser.cpp b/lld/ELF/ScriptParser.cpp
index 17ac7ff6d5f4a..eae1d17b2f43e 100644
--- a/lld/ELF/ScriptParser.cpp
+++ b/lld/ELF/ScriptParser.cpp
@@ -1311,7 +1311,10 @@ Expr ScriptParser::readPrimary() {
   }
   if (tok == "DEFINED") {
     StringRef name = readParenLiteral();
-    return [=] { return symtab->find(name) ? 1 : 0; };
+    return [=] {
+      Symbol *b = symtab->find(name);
+      return (b && b->isDefined()) ? 1 : 0;
+    };
   }
   if (tok == "LENGTH") {
     StringRef name = readParenLiteral();
diff --git a/lld/test/ELF/linkerscript/Inputs/define.s b/lld/test/ELF/linkerscript/Inputs/define.s
index bc60a233dcb4b..69f47a8803164 100644
--- a/lld/test/ELF/linkerscript/Inputs/define.s
+++ b/lld/test/ELF/linkerscript/Inputs/define.s
@@ -6,3 +6,6 @@ defined = 0
 
 .section .bar,"a"
 .quad 1
+
+.section .test,"a"
+.quad 1
diff --git a/lld/test/ELF/linkerscript/define.test b/lld/test/ELF/linkerscript/define.test
index 689476ba32ad0..3ecaa11cc5b69 100644
--- a/lld/test/ELF/linkerscript/define.test
+++ b/lld/test/ELF/linkerscript/define.test
@@ -3,13 +3,17 @@
 # RUN: ld.lld -o %t --script %s %t.o
 # RUN: llvm-objdump --section-headers %t | FileCheck %s
 
+EXTERN(extern_defined)
 SECTIONS {
   . = DEFINED(defined) ? 0x11000 : .;
   .foo : { *(.foo*) }
   . = DEFINED(notdefined) ? 0x12000 : 0x13000;
   .bar : { *(.bar*) }
+  . = DEFINED(extern_defined) ? 0x14000 : 0x15000;
+  .test : { *(.test*) }
 }
 
 # CHECK: 1 .foo  00000008 0000000000011000 DATA
 # CHECK: 2 .bar  00000008 0000000000013000 DATA
-# CHECK: 3 .text 00000000 0000000000013008 TEXT
+# CHECK: 3 .test  00000008 0000000000015000 DATA
+# CHECK: 4 .text 00000000 0000000000015008 TEXT

From b46176bbb094610460667edad950a9c99f844118 Mon Sep 17 00:00:00 2001
From: Zequan Wu <zequanwu@google.com>
Date: Wed, 22 Jul 2020 19:04:59 -0700
Subject: [PATCH 0362/1035] Reland [Coverage] Add comment to skipped regions

Bug filled here: https://bugs.llvm.org/show_bug.cgi?id=45757.
Add comment to skipped regions so we don't track execution count for lines containing only comments.

Differential Revision: https://reviews.llvm.org/D83592
---
 clang/include/clang/Lex/Preprocessor.h        |  5 ++
 clang/lib/CodeGen/CodeGenAction.cpp           |  8 +--
 clang/lib/CodeGen/CoverageMappingGen.cpp      | 69 ++++++++++++++++--
 clang/lib/CodeGen/CoverageMappingGen.h        | 34 ++++++++-
 clang/lib/Lex/Preprocessor.cpp                |  6 +-
 clang/test/CoverageMapping/break.c            |  3 +-
 clang/test/CoverageMapping/builtinmacro.c     |  3 +-
 clang/test/CoverageMapping/classtemplate.cpp  |  3 +-
 clang/test/CoverageMapping/comment-in-macro.c |  7 +-
 clang/test/CoverageMapping/continue.c         |  3 +-
 clang/test/CoverageMapping/coroutine.cpp      |  3 +-
 .../test/CoverageMapping/deferred-region.cpp  |  3 +-
 clang/test/CoverageMapping/if.cpp             |  3 +-
 clang/test/CoverageMapping/includehell.cpp    |  2 +
 clang/test/CoverageMapping/label.cpp          |  5 +-
 clang/test/CoverageMapping/logical.cpp        |  3 +-
 clang/test/CoverageMapping/loops.cpp          |  5 +-
 .../CoverageMapping/macro-expressions.cpp     |  4 +-
 clang/test/CoverageMapping/macroparams2.c     |  4 +-
 clang/test/CoverageMapping/macros.c           |  4 +-
 clang/test/CoverageMapping/macroscopes.cpp    |  4 +-
 clang/test/CoverageMapping/moremacros.c       |  4 +-
 clang/test/CoverageMapping/objc.m             |  3 +-
 clang/test/CoverageMapping/pr32679.cpp        |  5 +-
 clang/test/CoverageMapping/preprocessor.c     | 11 +--
 clang/test/CoverageMapping/return.c           |  3 +-
 clang/test/CoverageMapping/switch.cpp         |  5 +-
 clang/test/CoverageMapping/switchmacro.c      |  3 +-
 clang/test/CoverageMapping/test.c             |  3 +-
 clang/test/CoverageMapping/trycatch.cpp       |  3 +-
 .../test/CoverageMapping/unreachable-macro.c  |  4 +-
 clang/test/CoverageMapping/while.c            |  5 +-
 clang/test/lit.cfg.py                         |  5 ++
 .../test/profile/Inputs/instrprof-comdat.h    |  2 +-
 .../test/profile/coverage_comments.cpp        | 71 +++++++++++++++++++
 .../instrprof-set-file-object-merging.c       |  2 +-
 36 files changed, 250 insertions(+), 60 deletions(-)
 create mode 100644 compiler-rt/test/profile/coverage_comments.cpp

diff --git a/clang/include/clang/Lex/Preprocessor.h b/clang/include/clang/Lex/Preprocessor.h
index 5cd017fa925fd..b0dd363555ab1 100644
--- a/clang/include/clang/Lex/Preprocessor.h
+++ b/clang/include/clang/Lex/Preprocessor.h
@@ -419,6 +419,9 @@ class Preprocessor {
   /// The number of (LexLevel 0) preprocessor tokens.
   unsigned TokenCount = 0;
 
+  /// Preprocess every token regardless of LexLevel.
+  bool PreprocessToken = false;
+
   /// The maximum number of (LexLevel 0) tokens before issuing a -Wmax-tokens
   /// warning, or zero for unlimited.
   unsigned MaxTokens = 0;
@@ -1038,6 +1041,8 @@ class Preprocessor {
     OnToken = std::move(F);
   }
 
+  void setPreprocessToken(bool Preprocess) { PreprocessToken = Preprocess; }
+
   bool isMacroDefined(StringRef Id) {
     return isMacroDefined(&Identifiers.get(Id));
   }
diff --git a/clang/lib/CodeGen/CodeGenAction.cpp b/clang/lib/CodeGen/CodeGenAction.cpp
index 55925110708ec..5a6ce0f5dbd50 100644
--- a/clang/lib/CodeGen/CodeGenAction.cpp
+++ b/clang/lib/CodeGen/CodeGenAction.cpp
@@ -990,11 +990,9 @@ CodeGenAction::CreateASTConsumer(CompilerInstance &CI, StringRef InFile) {
 
   CoverageSourceInfo *CoverageInfo = nullptr;
   // Add the preprocessor callback only when the coverage mapping is generated.
-  if (CI.getCodeGenOpts().CoverageMapping) {
-    CoverageInfo = new CoverageSourceInfo;
-    CI.getPreprocessor().addPPCallbacks(
-                                    std::unique_ptr<PPCallbacks>(CoverageInfo));
-  }
+  if (CI.getCodeGenOpts().CoverageMapping)
+    CoverageInfo = CodeGen::CoverageMappingModuleGen::setUpCoverageCallbacks(
+        CI.getPreprocessor());
 
   std::unique_ptr<BackendConsumer> Result(new BackendConsumer(
       BA, CI.getDiagnostics(), CI.getHeaderSearchOpts(),
diff --git a/clang/lib/CodeGen/CoverageMappingGen.cpp b/clang/lib/CodeGen/CoverageMappingGen.cpp
index 78b268f423cbf..9a7096b8d1d02 100644
--- a/clang/lib/CodeGen/CoverageMappingGen.cpp
+++ b/clang/lib/CodeGen/CoverageMappingGen.cpp
@@ -35,8 +35,35 @@ using namespace clang;
 using namespace CodeGen;
 using namespace llvm::coverage;
 
+CoverageSourceInfo *
+CoverageMappingModuleGen::setUpCoverageCallbacks(Preprocessor &PP) {
+  CoverageSourceInfo *CoverageInfo = new CoverageSourceInfo();
+  PP.addPPCallbacks(std::unique_ptr<PPCallbacks>(CoverageInfo));
+  PP.addCommentHandler(CoverageInfo);
+  PP.setPreprocessToken(true);
+  PP.setTokenWatcher([CoverageInfo](clang::Token Tok) {
+    // Update previous token location.
+    CoverageInfo->PrevTokLoc = Tok.getLocation();
+    CoverageInfo->updateNextTokLoc(Tok.getLocation());
+  });
+  return CoverageInfo;
+}
+
 void CoverageSourceInfo::SourceRangeSkipped(SourceRange Range, SourceLocation) {
-  SkippedRanges.push_back(Range);
+  SkippedRanges.push_back({Range});
+}
+
+bool CoverageSourceInfo::HandleComment(Preprocessor &PP, SourceRange Range) {
+  SkippedRanges.push_back({Range, PrevTokLoc});
+  AfterComment = true;
+  return false;
+}
+
+void CoverageSourceInfo::updateNextTokLoc(SourceLocation Loc) {
+  if (AfterComment) {
+    SkippedRanges.back().NextTokLoc = Loc;
+    AfterComment = false;
+  }
 }
 
 namespace {
@@ -274,8 +301,34 @@ class CoverageMappingBuilder {
     return None;
   }
 
+  /// This shrinks the skipped range if it spans a line that contains a
+  /// non-comment token. If shrinking the skipped range would make it empty,
+  /// this returns None.
+  Optional<SpellingRegion> adjustSkippedRange(SourceManager &SM,
+                                              SpellingRegion SR,
+                                              SourceLocation PrevTokLoc,
+                                              SourceLocation NextTokLoc) {
+    // If Range begin location is invalid, it's not a comment region.
+    if (PrevTokLoc.isInvalid())
+      return SR;
+    unsigned PrevTokLine = SM.getSpellingLineNumber(PrevTokLoc);
+    unsigned NextTokLine = SM.getSpellingLineNumber(NextTokLoc);
+    SpellingRegion newSR(SR);
+    if (SR.LineStart == PrevTokLine) {
+      newSR.LineStart = SR.LineStart + 1;
+      newSR.ColumnStart = 1;
+    }
+    if (SR.LineEnd == NextTokLine) {
+      newSR.LineEnd = SR.LineEnd - 1;
+      newSR.ColumnEnd = SR.ColumnStart + 1;
+    }
+    if (newSR.isInSourceOrder())
+      return newSR;
+    return None;
+  }
+
   /// Gather all the regions that were skipped by the preprocessor
-  /// using the constructs like #if.
+  /// using the constructs like #if or comments.
   void gatherSkippedRegions() {
     /// An array of the minimum lineStarts and the maximum lineEnds
     /// for mapping regions from the appropriate source files.
@@ -291,9 +344,10 @@ class CoverageMappingBuilder {
     }
 
     auto SkippedRanges = CVM.getSourceInfo().getSkippedRanges();
-    for (const auto &I : SkippedRanges) {
-      auto LocStart = I.getBegin();
-      auto LocEnd = I.getEnd();
+    for (auto &I : SkippedRanges) {
+      SourceRange Range = I.Range;
+      auto LocStart = Range.getBegin();
+      auto LocEnd = Range.getEnd();
       assert(SM.isWrittenInSameFile(LocStart, LocEnd) &&
              "region spans multiple files");
 
@@ -301,6 +355,11 @@ class CoverageMappingBuilder {
       if (!CovFileID)
         continue;
       SpellingRegion SR{SM, LocStart, LocEnd};
+      if (Optional<SpellingRegion> res =
+              adjustSkippedRange(SM, SR, I.PrevTokLoc, I.NextTokLoc))
+        SR = res.getValue();
+      else
+        continue;
       auto Region = CounterMappingRegion::makeSkipped(
           *CovFileID, SR.LineStart, SR.ColumnStart, SR.LineEnd, SR.ColumnEnd);
       // Make sure that we only collect the regions that are inside
diff --git a/clang/lib/CodeGen/CoverageMappingGen.h b/clang/lib/CodeGen/CoverageMappingGen.h
index 5d79d1e656703..d3eec226d67ca 100644
--- a/clang/lib/CodeGen/CoverageMappingGen.h
+++ b/clang/lib/CodeGen/CoverageMappingGen.h
@@ -16,6 +16,7 @@
 #include "clang/Basic/LLVM.h"
 #include "clang/Basic/SourceLocation.h"
 #include "clang/Lex/PPCallbacks.h"
+#include "clang/Lex/Preprocessor.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/IR/GlobalValue.h"
 #include "llvm/Support/raw_ostream.h"
@@ -29,15 +30,40 @@ class Preprocessor;
 class Decl;
 class Stmt;
 
+struct SkippedRange {
+  SourceRange Range;
+  // The location of token before the skipped source range.
+  SourceLocation PrevTokLoc;
+  // The location of token after the skipped source range.
+  SourceLocation NextTokLoc;
+
+  SkippedRange(SourceRange Range, SourceLocation PrevTokLoc = SourceLocation(),
+               SourceLocation NextTokLoc = SourceLocation())
+      : Range(Range), PrevTokLoc(PrevTokLoc), NextTokLoc(NextTokLoc) {}
+};
+
 /// Stores additional source code information like skipped ranges which
 /// is required by the coverage mapping generator and is obtained from
 /// the preprocessor.
-class CoverageSourceInfo : public PPCallbacks {
-  std::vector<SourceRange> SkippedRanges;
+class CoverageSourceInfo : public PPCallbacks, public CommentHandler {
+  // A vector of skipped source ranges and PrevTokLoc with NextTokLoc.
+  std::vector<SkippedRange> SkippedRanges;
+  bool AfterComment = false;
+
 public:
-  ArrayRef<SourceRange> getSkippedRanges() const { return SkippedRanges; }
+  // Location of the token parsed before HandleComment is called. This is
+  // updated every time Preprocessor::Lex lexes a new token.
+  SourceLocation PrevTokLoc;
+  // The location of token before comment.
+  SourceLocation BeforeCommentLoc;
+
+  std::vector<SkippedRange> &getSkippedRanges() { return SkippedRanges; }
 
   void SourceRangeSkipped(SourceRange Range, SourceLocation EndifLoc) override;
+
+  bool HandleComment(Preprocessor &PP, SourceRange Range) override;
+
+  void updateNextTokLoc(SourceLocation Loc);
 };
 
 namespace CodeGen {
@@ -66,6 +92,8 @@ class CoverageMappingModuleGen {
                                  uint64_t FilenamesRef);
 
 public:
+  static CoverageSourceInfo *setUpCoverageCallbacks(Preprocessor &PP);
+
   CoverageMappingModuleGen(CodeGenModule &CGM, CoverageSourceInfo &SourceInfo)
       : CGM(CGM), SourceInfo(SourceInfo) {}
 
diff --git a/clang/lib/Lex/Preprocessor.cpp b/clang/lib/Lex/Preprocessor.cpp
index 160e2b6ed8846..58c28cea30a18 100644
--- a/clang/lib/Lex/Preprocessor.cpp
+++ b/clang/lib/Lex/Preprocessor.cpp
@@ -969,8 +969,10 @@ void Preprocessor::Lex(Token &Result) {
   LastTokenWasAt = Result.is(tok::at);
   --LexLevel;
 
-  if (LexLevel == 0 && !Result.getFlag(Token::IsReinjected)) {
-    ++TokenCount;
+  if ((LexLevel == 0 || PreprocessToken) &&
+      !Result.getFlag(Token::IsReinjected)) {
+    if (LexLevel == 0)
+      ++TokenCount;
     if (OnToken)
       OnToken(Result);
   }
diff --git a/clang/test/CoverageMapping/break.c b/clang/test/CoverageMapping/break.c
index 08461d7ed2de2..191f34765a65b 100644
--- a/clang/test/CoverageMapping/break.c
+++ b/clang/test/CoverageMapping/break.c
@@ -1,4 +1,5 @@
-// RUN: %clang_cc1 -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only -main-file-name break.c %s | FileCheck %s
+// RUN: %strip_comments > %t.stripped.c
+// RUN: %clang_cc1 -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only -main-file-name break.c %t.stripped.c | FileCheck %s
 
 int main() {         // CHECK: File 0, [[@LINE]]:12 -> {{[0-9]+}}:2 = #0
   int cnt = 0;       // CHECK-NEXT: File 0, [[@LINE+1]]:9 -> [[@LINE+1]]:18 = #0
diff --git a/clang/test/CoverageMapping/builtinmacro.c b/clang/test/CoverageMapping/builtinmacro.c
index 63f5584d40c2c..0de0e8e6a1252 100644
--- a/clang/test/CoverageMapping/builtinmacro.c
+++ b/clang/test/CoverageMapping/builtinmacro.c
@@ -1,4 +1,5 @@
-// RUN: %clang_cc1 -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only -main-file-name builtinmacro.c %s | FileCheck %s
+// RUN: %strip_comments > %t.stripped.c
+// RUN: %clang_cc1 -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only -main-file-name builtinmacro.c %t.stripped.c | FileCheck %s
 
 // Test the coverage mapping generation for built-in macroes.
 
diff --git a/clang/test/CoverageMapping/classtemplate.cpp b/clang/test/CoverageMapping/classtemplate.cpp
index 0dbb0c0ede841..9250069e04f65 100644
--- a/clang/test/CoverageMapping/classtemplate.cpp
+++ b/clang/test/CoverageMapping/classtemplate.cpp
@@ -1,4 +1,5 @@
-// RUN: %clang_cc1 -triple %itanium_abi_triple -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only -main-file-name classtemplate.cpp %s > %tmapping
+// RUN: %strip_comments > %t.stripped.cpp
+// RUN: %clang_cc1 -triple %itanium_abi_triple -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only -main-file-name classtemplate.cpp %t.stripped.cpp > %tmapping
 // RUN: FileCheck -input-file %tmapping %s --check-prefix=CHECK-CONSTRUCTOR
 // RUN: FileCheck -input-file %tmapping %s --check-prefix=CHECK-GETTER
 // RUN: FileCheck -input-file %tmapping %s --check-prefix=CHECK-SETTER
diff --git a/clang/test/CoverageMapping/comment-in-macro.c b/clang/test/CoverageMapping/comment-in-macro.c
index 06e8adbc41ee2..86b554f8bd50c 100644
--- a/clang/test/CoverageMapping/comment-in-macro.c
+++ b/clang/test/CoverageMapping/comment-in-macro.c
@@ -1,4 +1,5 @@
-// RUN: %clang_cc1 -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only %s | FileCheck %s
+// RUN: %strip_comments > %t.stripped.c
+// RUN: %clang_cc1 -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only %t.stripped.c | FileCheck %s
 
 #define x1 "" // ...
 #define x2 return 0
@@ -7,5 +8,5 @@ int main() { // CHECK-NEXT: File 0, [[@LINE]]:12 -> [[@LINE+3]]:2 = #0
   x1;        // CHECK-NEXT: Expansion,File 0, [[@LINE]]:3 -> [[@LINE]]:5 = #0
   x2;        // CHECK-NEXT: Expansion,File 0, [[@LINE]]:3 -> [[@LINE]]:5 = #0
 }
-// CHECK-NEXT: File 1, 3:12 -> 3:14 = #0
-// CHECK-NEXT: File 2, 4:12 -> 4:20 = #0
+// CHECK-NEXT: File 1, 4:12 -> 4:14 = #0
+// CHECK-NEXT: File 2, 5:12 -> 5:20 = #0
diff --git a/clang/test/CoverageMapping/continue.c b/clang/test/CoverageMapping/continue.c
index 9864c912f239c..774fe2ee6fffd 100644
--- a/clang/test/CoverageMapping/continue.c
+++ b/clang/test/CoverageMapping/continue.c
@@ -1,4 +1,5 @@
-// RUN: %clang_cc1 -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only -main-file-name continue.c %s | FileCheck %s
+// RUN: %strip_comments > %t.stripped.c
+// RUN: %clang_cc1 -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only -main-file-name continue.c %t.stripped.c | FileCheck %s
 
 int main() {                    // CHECK: File 0, [[@LINE]]:12 -> [[@LINE+21]]:2 = #0
   int j = 0;                    // CHECK-NEXT: File 0, [[@LINE+2]]:18 -> [[@LINE+2]]:24 = (#0 + #1)
diff --git a/clang/test/CoverageMapping/coroutine.cpp b/clang/test/CoverageMapping/coroutine.cpp
index dc9473348fc90..a614e6e8ea402 100644
--- a/clang/test/CoverageMapping/coroutine.cpp
+++ b/clang/test/CoverageMapping/coroutine.cpp
@@ -1,6 +1,7 @@
 // fixme: the following line is added to cleanup bots, will be removed in weeks.
 // RUN: rm -f %S/coroutine.ll
-// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fcoroutines-ts -std=c++14 -emit-llvm -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping %s -o - | FileCheck %s
+// RUN: %strip_comments > %t.stripped.cpp
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fcoroutines-ts -std=c++14 -emit-llvm -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping %t.stripped.cpp -o - | FileCheck %s
 
 namespace std::experimental {
 template <typename... T>
diff --git a/clang/test/CoverageMapping/deferred-region.cpp b/clang/test/CoverageMapping/deferred-region.cpp
index 45113e46f02b2..8db6119472dbc 100644
--- a/clang/test/CoverageMapping/deferred-region.cpp
+++ b/clang/test/CoverageMapping/deferred-region.cpp
@@ -1,4 +1,5 @@
-// RUN: %clang_cc1 -std=c++11 -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -fexceptions -fcxx-exceptions -emit-llvm-only -triple %itanium_abi_triple -main-file-name deferred-region.cpp -I %S/Inputs %s | FileCheck %s
+// RUN: %strip_comments > %t.stripped.cpp
+// RUN: %clang_cc1 -std=c++11 -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -fexceptions -fcxx-exceptions -emit-llvm-only -triple %itanium_abi_triple -main-file-name deferred-region.cpp -I %S/Inputs %t.stripped.cpp | FileCheck %s
 
 #define IF if
 #define STMT(S) S
diff --git a/clang/test/CoverageMapping/if.cpp b/clang/test/CoverageMapping/if.cpp
index e3d6f4e25e573..8ffc09d29a3c7 100644
--- a/clang/test/CoverageMapping/if.cpp
+++ b/clang/test/CoverageMapping/if.cpp
@@ -1,4 +1,5 @@
-// RUN: %clang_cc1 -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only -std=c++1z -triple %itanium_abi_triple -main-file-name if.cpp %s | FileCheck %s
+// RUN: %strip_comments > %t.stripped.cpp
+// RUN: %clang_cc1 -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only -std=c++1z -triple %itanium_abi_triple -main-file-name if.cpp %t.stripped.cpp | FileCheck %s
 
 int nop() { return 0; }
 
diff --git a/clang/test/CoverageMapping/includehell.cpp b/clang/test/CoverageMapping/includehell.cpp
index fd08d6af7f3da..c92f12e5e80d0 100644
--- a/clang/test/CoverageMapping/includehell.cpp
+++ b/clang/test/CoverageMapping/includehell.cpp
@@ -51,6 +51,7 @@ int main() {
 // CHECK-START: File [[START3]], 4:29 -> 5:1 = #9
 
 // CHECK-CODE:      File [[CODE1:[0-9]]], 1:1 -> 14:1 = #1
+// CHECK-CODE:      Skipped,File [[CODE1]], 1:1 -> 1:41 = 0
 // CHECK-CODE-NEXT: File [[CODE1]], 4:5 -> 4:11 = #1
 // CHECK-CODE: File [[CODE1]], 4:13 -> 6:2 = #2
 // CHECK-CODE: File [[CODE1]], 6:8 -> 8:2 = (#1 - #2)
@@ -58,6 +59,7 @@ int main() {
 // CHECK-CODE: File [[CODE1]], 9:11 -> 11:2 = #3
 // CHECK-CODE: File [[CODE1]], 11:8 -> 13:2 = (#1 - #3)
 // CHECK-CODE:      File [[CODE2:[0-9]]], 1:1 -> 14:1 = #5
+// CHECK-CODE:      Skipped,File [[CODE2]], 1:1 -> 1:41 = 0
 // CHECK-CODE-NEXT: File [[CODE2]], 4:5 -> 4:11 = #5
 // CHECK-CODE: File [[CODE2]], 4:13 -> 6:2 = #6
 // CHECK-CODE: File [[CODE2]], 6:8 -> 8:2 = (#5 - #6)
diff --git a/clang/test/CoverageMapping/label.cpp b/clang/test/CoverageMapping/label.cpp
index 995835ad6b2ff..e77372d325e5a 100644
--- a/clang/test/CoverageMapping/label.cpp
+++ b/clang/test/CoverageMapping/label.cpp
@@ -1,6 +1,7 @@
-// RUN: %clang_cc1 -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only -main-file-name label.cpp %s | FileCheck %s
+// RUN: %strip_comments > %t.stripped.cpp
+// RUN: %clang_cc1 -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only -main-file-name label.cpp %t.stripped.cpp | FileCheck %s
 
-                             // CHECK: func
+// CHECK: func
 void func() {                // CHECK-NEXT: File 0, [[@LINE]]:13 -> {{[0-9]+}}:2 = #0
   int i = 0;                 // CHECK-NEXT: File 0, [[@LINE+2]]:14 -> [[@LINE+2]]:20 = (#0 + #3)
                              // CHECK-NEXT: File 0, [[@LINE+1]]:22 -> [[@LINE+1]]:25 = #3
diff --git a/clang/test/CoverageMapping/logical.cpp b/clang/test/CoverageMapping/logical.cpp
index bc7c785b7b550..cca03377c98a4 100644
--- a/clang/test/CoverageMapping/logical.cpp
+++ b/clang/test/CoverageMapping/logical.cpp
@@ -1,4 +1,5 @@
-// RUN: %clang_cc1 -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only -main-file-name logical.cpp %s | FileCheck %s
+// RUN: %strip_comments > %t.stripped.cpp
+// RUN: %clang_cc1 -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only -main-file-name logical.cpp %t.stripped.cpp | FileCheck %s
 
 int main() {                        // CHECK: File 0, [[@LINE]]:12 -> [[@LINE+15]]:2 = #0
   bool bt = true;
diff --git a/clang/test/CoverageMapping/loops.cpp b/clang/test/CoverageMapping/loops.cpp
index ff7aafd66d944..498d214e69d9c 100644
--- a/clang/test/CoverageMapping/loops.cpp
+++ b/clang/test/CoverageMapping/loops.cpp
@@ -1,6 +1,7 @@
-// RUN: %clang_cc1 -std=c++11 -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only -main-file-name loops.cpp %s | FileCheck %s
+// RUN: %strip_comments > %t.stripped.cpp
+// RUN: %clang_cc1 -std=c++11 -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only -main-file-name loops.cpp %t.stripped.cpp | FileCheck %s
 
-                                    // CHECK: rangedFor
+// CHECK: rangedFor
 void rangedFor() {                  // CHECK-NEXT: File 0, [[@LINE]]:18 -> {{[0-9]+}}:2 = #0
   int arr[] = { 1, 2, 3, 4, 5 };
   int sum = 0;                      // CHECK: Gap,File 0, [[@LINE+1]]:20 -> [[@LINE+1]]:21 = #1
diff --git a/clang/test/CoverageMapping/macro-expressions.cpp b/clang/test/CoverageMapping/macro-expressions.cpp
index 26d70c67fca08..60afc5238b9eb 100644
--- a/clang/test/CoverageMapping/macro-expressions.cpp
+++ b/clang/test/CoverageMapping/macro-expressions.cpp
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -std=c++11 -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only -main-file-name macro-expressions.cpp -w %s | FileCheck %s
-
+// RUN: %strip_comments > %t.stripped.cpp
+// RUN: %clang_cc1 -std=c++11 -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only -main-file-name macro-expressions.cpp -w %t.stripped.cpp | FileCheck %s
 #define EXPR(x) (x)
 #define NEXPR(x) (!x)
 #define DECL(T, x) T x
diff --git a/clang/test/CoverageMapping/macroparams2.c b/clang/test/CoverageMapping/macroparams2.c
index 4e04581b725e8..30ce25d9accda 100644
--- a/clang/test/CoverageMapping/macroparams2.c
+++ b/clang/test/CoverageMapping/macroparams2.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only -main-file-name macroparams2.c %s | FileCheck %s
-
+// RUN: %strip_comments > %t.stripped.c
+// RUN: %clang_cc1 -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only -main-file-name macroparams2.c %t.stripped.c | FileCheck %s
 #define MACRO(REFS, CALLS)  (4 * (CALLS) < (REFS))
 
 struct S {
diff --git a/clang/test/CoverageMapping/macros.c b/clang/test/CoverageMapping/macros.c
index 39cd190b2a882..83e2029be5612 100644
--- a/clang/test/CoverageMapping/macros.c
+++ b/clang/test/CoverageMapping/macros.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only -main-file-name macros.c %s | FileCheck %s
-
+// RUN: %strip_comments > %t.stripped.c
+// RUN: %clang_cc1 -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only -main-file-name macros.c %t.stripped.c | FileCheck %s
 #define MACRO return; bar()
 #define MACRO_2 bar()
 #define MACRO_1 return; MACRO_2
diff --git a/clang/test/CoverageMapping/macroscopes.cpp b/clang/test/CoverageMapping/macroscopes.cpp
index 3f5f65e5ad7b5..62f5dbe77981f 100644
--- a/clang/test/CoverageMapping/macroscopes.cpp
+++ b/clang/test/CoverageMapping/macroscopes.cpp
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only -main-file-name macroscopes.cpp %s | FileCheck %s
-
+// RUN: %strip_comments > %t.stripped.cpp
+// RUN: %clang_cc1 -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only -main-file-name macroscopes.cpp %t.stripped.cpp | FileCheck %s
 #define starts_a_scope for (int i = 0; i < 2; ++i) {
 
 #define ends_a_scope \
diff --git a/clang/test/CoverageMapping/moremacros.c b/clang/test/CoverageMapping/moremacros.c
index 5e0d4a6abf257..ed89dcafd6723 100644
--- a/clang/test/CoverageMapping/moremacros.c
+++ b/clang/test/CoverageMapping/moremacros.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only -main-file-name macro-expansion.c %s | FileCheck %s
-
+// RUN: %strip_comments > %t.stripped.c
+// RUN: %clang_cc1 -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only -main-file-name macro-expansion.c %t.stripped.c | FileCheck %s
 #define LBRAC {
 #define RBRAC }
 
diff --git a/clang/test/CoverageMapping/objc.m b/clang/test/CoverageMapping/objc.m
index 4e4c184f0a887..008d291aee31a 100644
--- a/clang/test/CoverageMapping/objc.m
+++ b/clang/test/CoverageMapping/objc.m
@@ -1,4 +1,5 @@
-// RUN: %clang_cc1 -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only -main-file-name objc.m -triple x86_64-apple-darwin -fobjc-runtime=macosx-fragile-10.5 -w %s | FileCheck %s
+// RUN: %strip_comments > %t.stripped.m
+// RUN: %clang_cc1 -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only -main-file-name objc.m -triple x86_64-apple-darwin -fobjc-runtime=macosx-fragile-10.5 -w %t.stripped.m | FileCheck %s
 
 @interface A
 - (void)bork:(int)msg;
diff --git a/clang/test/CoverageMapping/pr32679.cpp b/clang/test/CoverageMapping/pr32679.cpp
index eac3afb6efb6c..c93133f8c2f2d 100644
--- a/clang/test/CoverageMapping/pr32679.cpp
+++ b/clang/test/CoverageMapping/pr32679.cpp
@@ -1,5 +1,6 @@
-// RUN: %clang_cc1 -cc1 -triple i686-pc-windows-msvc19.0.0 -emit-obj -fprofile-instrument=clang -std=c++14 -fdelayed-template-parsing -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only -main-file-name pr32679.cpp -o - %s | FileCheck %s -check-prefix=MSABI -implicit-check-not=f2
-// RUN: %clang_cc1 -cc1 -triple %itanium_abi_triple -emit-obj -fprofile-instrument=clang -std=c++14 -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only -main-file-name pr32679.cpp -o - %s | FileCheck %s -check-prefix=ITANIUM -implicit-check-not=f2
+// RUN: %strip_comments > %t.stripped.cpp
+// RUN: %clang_cc1 -cc1 -triple i686-pc-windows-msvc19.0.0 -emit-obj -fprofile-instrument=clang -std=c++14 -fdelayed-template-parsing -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only -main-file-name pr32679.cpp -o - %t.stripped.cpp | FileCheck %s -check-prefix=MSABI -implicit-check-not=f2
+// RUN: %clang_cc1 -cc1 -triple %itanium_abi_triple -emit-obj -fprofile-instrument=clang -std=c++14 -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only -main-file-name pr32679.cpp -o - %t.stripped.cpp | FileCheck %s -check-prefix=ITANIUM -implicit-check-not=f2
 
 template <typename T, int S1>
 struct CreateSpecialization;
diff --git a/clang/test/CoverageMapping/preprocessor.c b/clang/test/CoverageMapping/preprocessor.c
index 9225c9f162a20..b68aa2e0e93c6 100644
--- a/clang/test/CoverageMapping/preprocessor.c
+++ b/clang/test/CoverageMapping/preprocessor.c
@@ -1,6 +1,7 @@
-// RUN: %clang_cc1 -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only -main-file-name preprocessor.c %s | FileCheck %s
+// RUN: %strip_comments > %t.stripped.c
+// RUN: %clang_cc1 -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only -main-file-name preprocessor.c %t.stripped.c | FileCheck %s
 
-                 // CHECK: func
+// CHECK: func
 void func() {    // CHECK: File 0, [[@LINE]]:13 -> [[@LINE+5]]:2 = #0
   int i = 0;
 #ifdef MACRO     // CHECK-NEXT: Skipped,File 0, [[@LINE]]:1 -> [[@LINE+2]]:7 = 0
@@ -11,7 +12,7 @@ void func() {    // CHECK: File 0, [[@LINE]]:13 -> [[@LINE+5]]:2 = #0
                  // CHECK: main
 int main() {     // CHECK-NEXT: File 0, [[@LINE]]:12 -> {{[0-9]+}}:2 = #0
   int i = 0;
-#  if 0            // CHECK-NEXT: Skipped,File 0, [[@LINE]]:1 -> [[@LINE+4]]:29 = 0
+#if 0 // CHECK-NEXT: Skipped,File 0, [[@LINE]]:1 -> [[@LINE+4]]:9 = 0
   if(i == 0) {
     i = 1;
   }
@@ -29,7 +30,7 @@ int main() {     // CHECK-NEXT: File 0, [[@LINE]]:12 -> {{[0-9]+}}:2 = #0
 }
 #endif
 
-  // CHECK-NEXT: Skipped,File 0, [[@LINE+1]]:1 -> [[@LINE+4]]:24
+  // CHECK-NEXT: Skipped,File 0, [[@LINE+1]]:1 -> [[@LINE+4]]:8
 #\
   if 0
 #\
@@ -59,7 +60,7 @@ int main() {     // CHECK-NEXT: File 0, [[@LINE]]:12 -> {{[0-9]+}}:2 = #0
 #\
   endif
 
-  // CHECK-NEXT: Skipped,File 0, [[@LINE+1]]:1 -> [[@LINE+6]]:26
+  // CHECK-NEXT: Skipped,File 0, [[@LINE+1]]:1 -> [[@LINE+6]]:10
 #\
   ifdef NOT_DEFINED
 #\
diff --git a/clang/test/CoverageMapping/return.c b/clang/test/CoverageMapping/return.c
index 440acb569b8fa..fae2e9f761b7a 100644
--- a/clang/test/CoverageMapping/return.c
+++ b/clang/test/CoverageMapping/return.c
@@ -1,4 +1,5 @@
-// RUN: %clang_cc1 -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only -main-file-name return.c %s | FileCheck %s
+// RUN: %strip_comments > %t.stripped.c
+// RUN: %clang_cc1 -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only -main-file-name return.c %t.stripped.c | FileCheck %s
 
                                 // CHECK: func
 void func() {                   // CHECK: File 0, [[@LINE]]:13 -> [[@LINE+3]]:2 = #0
diff --git a/clang/test/CoverageMapping/switch.cpp b/clang/test/CoverageMapping/switch.cpp
index 25ea4053f4e2c..1a8db09b3ff14 100644
--- a/clang/test/CoverageMapping/switch.cpp
+++ b/clang/test/CoverageMapping/switch.cpp
@@ -1,6 +1,7 @@
-// RUN: %clang_cc1 -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only -std=c++1z -triple %itanium_abi_triple -main-file-name switch.cpp %s | FileCheck %s
+// RUN: %strip_comments > %t.stripped.cpp
+// RUN: %clang_cc1 -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only -std=c++1z -triple %itanium_abi_triple -main-file-name switch.cpp %t.stripped.cpp | FileCheck %s
 
-                    // CHECK: foo
+// CHECK: foo
 void foo(int i) {   // CHECK-NEXT: File 0, [[@LINE]]:17 -> [[@LINE+8]]:2 = #0
   switch(i) {       // CHECK-NEXT: Gap,File 0, [[@LINE]]:13 -> [[@LINE+4]]:10 = 0
   case 1:           // CHECK-NEXT: File 0, [[@LINE]]:3 -> [[@LINE+1]]:11 = #2
diff --git a/clang/test/CoverageMapping/switchmacro.c b/clang/test/CoverageMapping/switchmacro.c
index fc0392fb91e53..5c6a37e77f944 100644
--- a/clang/test/CoverageMapping/switchmacro.c
+++ b/clang/test/CoverageMapping/switchmacro.c
@@ -1,4 +1,5 @@
-// RUN: %clang_cc1 -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only -main-file-name switchmacro.c %s | FileCheck %s
+// RUN: %strip_comments > %t.stripped.c
+// RUN: %clang_cc1 -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only -main-file-name switchmacro.c %t.stripped.c | FileCheck %s
 
 #define FOO(x) (void)x
 
diff --git a/clang/test/CoverageMapping/test.c b/clang/test/CoverageMapping/test.c
index ae73fcb3bbab2..559036a3be9be 100644
--- a/clang/test/CoverageMapping/test.c
+++ b/clang/test/CoverageMapping/test.c
@@ -1,4 +1,5 @@
-// RUN: %clang_cc1 -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only -main-file-name test.c %s | FileCheck %s
+// RUN: %strip_comments > %t.stripped.c
+// RUN: %clang_cc1 -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only -main-file-name test.c %t.stripped.c | FileCheck %s
 
 void bar();
 static void static_func();
diff --git a/clang/test/CoverageMapping/trycatch.cpp b/clang/test/CoverageMapping/trycatch.cpp
index 5d284daaca017..ba1b26b7acee7 100644
--- a/clang/test/CoverageMapping/trycatch.cpp
+++ b/clang/test/CoverageMapping/trycatch.cpp
@@ -1,4 +1,5 @@
-// RUN: %clang_cc1 -triple %itanium_abi_triple -std=c++11 -fexceptions -fcxx-exceptions -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only -main-file-name trycatch.cpp %s | FileCheck %s
+// RUN: %strip_comments > %t.stripped.cpp
+// RUN: %clang_cc1 -triple %itanium_abi_triple -std=c++11 -fexceptions -fcxx-exceptions -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only -main-file-name trycatch.cpp %t.stripped.cpp | FileCheck %s
 
 class Error {
 };
diff --git a/clang/test/CoverageMapping/unreachable-macro.c b/clang/test/CoverageMapping/unreachable-macro.c
index b9d4f3616ffa5..b84acca33f938 100644
--- a/clang/test/CoverageMapping/unreachable-macro.c
+++ b/clang/test/CoverageMapping/unreachable-macro.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only %s | FileCheck %s
-
+// RUN: %strip_comments > %t.stripped.c
+// RUN: %clang_cc1 -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only %t.stripped.c | FileCheck %s
 #define WHILE while (0) {}
 
 // CHECK: counters_in_macro_following_unreachable
diff --git a/clang/test/CoverageMapping/while.c b/clang/test/CoverageMapping/while.c
index 616ecf69020d7..e0c3ca35537f8 100644
--- a/clang/test/CoverageMapping/while.c
+++ b/clang/test/CoverageMapping/while.c
@@ -1,6 +1,7 @@
-// RUN: %clang_cc1 -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only -main-file-name loops.cpp %s | FileCheck %s
+// RUN: %strip_comments > %t.stripped.c
+// RUN: %clang_cc1 -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only -main-file-name loops.cpp %t.stripped.c | FileCheck %s
 
-                                    // CHECK: main
+// CHECK: main
 int main() {                        // CHECK-NEXT: File 0, [[@LINE]]:12 -> [[@LINE+8]]:2 = #0
   int j = 0;                        // CHECK-NEXT: File 0, [[@LINE+1]]:9 -> [[@LINE+1]]:14 = (#0 + #1)
   while(j < 5) ++j;                 // CHECK-NEXT: File 0, [[@LINE]]:15 -> [[@LINE]]:16 = #1
diff --git a/clang/test/lit.cfg.py b/clang/test/lit.cfg.py
index dacda6894a045..6c677eda9a093 100644
--- a/clang/test/lit.cfg.py
+++ b/clang/test/lit.cfg.py
@@ -91,6 +91,11 @@
     ('%hmaptool', "'%s' %s" % (config.python_executable,
                              os.path.join(config.clang_tools_dir, 'hmaptool'))))
 
+# Strip C++ comments "//"" from tests
+config.substitutions.append(
+    ('%strip_comments', "sed 's/[ \t]*\/\/.*//' %s")
+)
+
 # Plugins (loadable modules)
 if config.has_plugins and config.llvm_plugin_ext:
     config.available_features.add('plugins')
diff --git a/compiler-rt/test/profile/Inputs/instrprof-comdat.h b/compiler-rt/test/profile/Inputs/instrprof-comdat.h
index 61e283cc878ed..956496ee1361a 100644
--- a/compiler-rt/test/profile/Inputs/instrprof-comdat.h
+++ b/compiler-rt/test/profile/Inputs/instrprof-comdat.h
@@ -18,6 +18,6 @@ template <class T> T FOO<T>::DoIt(T ti) { // HEADER: [[@LINE]]|  2|template
     if (I > ti / 2)                       // HEADER: [[@LINE]]| 20|    if (I > ti
       t -= 1;                             // HEADER: [[@LINE]]|  8|      t -= 1;
   }                                       // HEADER: [[@LINE]]| 10|  }
-                                          // HEADER: [[@LINE]]|  1|
+                                          // HEADER: [[@LINE]]|   |
   return t;                               // HEADER: [[@LINE]]|  1|  return t;
 }
diff --git a/compiler-rt/test/profile/coverage_comments.cpp b/compiler-rt/test/profile/coverage_comments.cpp
new file mode 100644
index 0000000000000..0cf78add7913b
--- /dev/null
+++ b/compiler-rt/test/profile/coverage_comments.cpp
@@ -0,0 +1,71 @@
+// RUN: %clangxx_profgen -fcoverage-mapping -Wno-comment -o %t %s
+// RUN: env LLVM_PROFILE_FILE=%t.profraw %run %t
+// RUN: llvm-profdata merge -o %t.profdata %t.profraw
+// RUN: llvm-cov show %t -instr-profile %t.profdata -path-equivalence=/tmp,%S 2>&1 | FileCheck %s
+
+int main() {                           // CHECK:       [[# @LINE]]| 1|int main() {
+    /* comment */ int x = 0;           // CHECK-NEXT:  [[# @LINE]]| 1|
+    int y = 0; /* comment */           // CHECK-NEXT:  [[# @LINE]]| 1|
+    int z = 0; // comment              // CHECK-NEXT:  [[# @LINE]]| 1|
+    // comment                         // CHECK-NEXT:  [[# @LINE]]|  |
+                                       // CHECK-NEXT:  [[# @LINE]]|  |
+    x = 0; /*                          // CHECK-NEXT:  [[# @LINE]]| 1|
+    comment                            // CHECK-NEXT:  [[# @LINE]]|  |
+    */                                 // CHECK-NEXT:  [[# @LINE]]|  |
+                                       // CHECK-NEXT:  [[# @LINE]]|  |
+    /*                                 // CHECK-NEXT:  [[# @LINE]]|  |
+    comment                            // CHECK-NEXT:  [[# @LINE]]|  |
+    */ x = 0;                          // CHECK-NEXT:  [[# @LINE]]| 1|
+                                       // CHECK-NEXT:  [[# @LINE]]|  |
+    /* comment */                      // CHECK-NEXT:  [[# @LINE]]|  |
+    // comment                         // CHECK-NEXT:  [[# @LINE]]|  |
+    /* comment */                      // CHECK-NEXT:  [[# @LINE]]|  |
+    z =                                // CHECK-NEXT:  [[# @LINE]]| 1|
+    x // comment                       // CHECK-NEXT:  [[# @LINE]]| 1|
+    // comment                         // CHECK-NEXT:  [[# @LINE]]|  |
+    + /*                               // CHECK-NEXT:  [[# @LINE]]| 1|
+    comment                            // CHECK-NEXT:  [[# @LINE]]|  |
+    */                                 // CHECK-NEXT:  [[# @LINE]]|  |
+    /*                                 // CHECK-NEXT:  [[# @LINE]]|  |
+    comment                            // CHECK-NEXT:  [[# @LINE]]|  |
+    */y;                               // CHECK-NEXT:  [[# @LINE]]| 1|
+                                       // CHECK-NEXT:  [[# @LINE]]|  |
+    // Comments inside directives.     // CHECK-NEXT:  [[# @LINE]]|  |
+    #if 0 //comment                    // CHECK-NEXT:  [[# @LINE]]|  |
+    /* comment */ x = 0;               // CHECK-NEXT:  [[# @LINE]]|  |
+    y = 0; /* comment */               // CHECK-NEXT:  [[# @LINE]]|  |
+    z = 0; // comment                  // CHECK-NEXT:  [[# @LINE]]|  |
+    // comment                         // CHECK-NEXT:  [[# @LINE]]|  |
+                                       // CHECK-NEXT:  [[# @LINE]]|  |
+    x = 0; /*                          // CHECK-NEXT:  [[# @LINE]]|  |
+    comment                            // CHECK-NEXT:  [[# @LINE]]|  |
+    */                                 // CHECK-NEXT:  [[# @LINE]]|  |
+                                       // CHECK-NEXT:  [[# @LINE]]|  | 
+    /*                                 // CHECK-NEXT:  [[# @LINE]]|  |
+    comment                            // CHECK-NEXT:  [[# @LINE]]|  |
+    */ x = 0;                          // CHECK-NEXT:  [[# @LINE]]|  |
+                                       // CHECK-NEXT:  [[# @LINE]]|  |
+    /* comment */                      // CHECK-NEXT:  [[# @LINE]]|  |
+    // comment                         // CHECK-NEXT:  [[# @LINE]]|  |
+    /* comment */                      // CHECK-NEXT:  [[# @LINE]]|  |
+    #endif // comment                  // CHECK-NEXT:  [[# @LINE]]|  |
+    #if 1 // comment                   // CHECK-NEXT:  [[# @LINE]]| 1|
+    /* comment */ x = 0;               // CHECK-NEXT:  [[# @LINE]]| 1|
+    y = 0; /* comment */               // CHECK-NEXT:  [[# @LINE]]| 1|
+    z = 0; // comment                  // CHECK-NEXT:  [[# @LINE]]| 1|
+    // comment                         // CHECK-NEXT:  [[# @LINE]]|  |
+                                       // CHECK-NEXT:  [[# @LINE]]|  |
+    x = 0; /*                          // CHECK-NEXT:  [[# @LINE]]| 1|
+    comment                            // CHECK-NEXT:  [[# @LINE]]|  |
+    */                                 // CHECK-NEXT:  [[# @LINE]]|  |
+                                       // CHECK-NEXT:  [[# @LINE]]|  |
+    /*                                 // CHECK-NEXT:  [[# @LINE]]|  |
+    comment                            // CHECK-NEXT:  [[# @LINE]]|  |
+    */ x = 0;                          // CHECK-NEXT:  [[# @LINE]]| 1|
+                                       // CHECK-NEXT:  [[# @LINE]]|  |
+    /* comment */                      // CHECK-NEXT:  [[# @LINE]]|  |
+    // comment                         // CHECK-NEXT:  [[# @LINE]]|  |
+    /* comment */                      // CHECK-NEXT:  [[# @LINE]]|  |
+    #endif //comment                   // CHECK-NEXT:  [[# @LINE]]| 1|
+    return 0;                          // CHECK-NEXT:  [[# @LINE]]| 1|
+}                                      // CHECK-NEXT:  [[# @LINE]]| 1|
diff --git a/compiler-rt/test/profile/instrprof-set-file-object-merging.c b/compiler-rt/test/profile/instrprof-set-file-object-merging.c
index 0ca5f6ff9ed95..35e9becf228f0 100644
--- a/compiler-rt/test/profile/instrprof-set-file-object-merging.c
+++ b/compiler-rt/test/profile/instrprof-set-file-object-merging.c
@@ -34,7 +34,7 @@ int main(int argc, const char *argv[]) {
 // CHECK:   17|      2|
 // CHECK:   18|      2|  FILE *F = fopen(argv[1], "r+b");
 // CHECK:   19|      2|  if (!F) {
-// CHECK:   20|      1|    // File might not exist, try opening with truncation
+// CHECK:   20|       |    // File might not exist, try opening with truncation
 // CHECK:   21|      1|    F = fopen(argv[1], "w+b");
 // CHECK:   22|      1|  }
 // CHECK:   23|      2|  __llvm_profile_set_file_object(F, 1);

From 740a164dec483225cbd02ab6c82199e2747ffacb Mon Sep 17 00:00:00 2001
From: Richard Smith <richard@metafoo.co.uk>
Date: Tue, 28 Jul 2020 12:09:16 -0700
Subject: [PATCH 0363/1035] PR46377: Fix dependence calculation for function
 types and typedef types.

We previously did not treat a function type as dependent if it had a
parameter pack with a non-dependent type -- such a function type depends
on the arity of the pack so is dependent even though none of the
parameter types is dependent. In order to properly handle this, we now
treat pack expansion types as always being dependent types (depending on
at least the pack arity), and always canonically being pack expansion
types, even in the unusual case when the pattern is not a dependent
type. This does mean that we can have canonical types that are pack
expansions that contain no unexpanded packs, which is unfortunate but
not inaccurate.

We also previously did not treat a typedef type as
instantiation-dependent if its canonical type was not
instantiation-dependent. That's wrong because instantiation-dependence
is a property of the type sugar, not of the type; an
instantiation-dependent type can have a non-instantiation-dependent
canonical type.
---
 clang/include/clang/AST/ASTContext.h          | 10 +++++-
 clang/include/clang/AST/Type.h                | 13 +++----
 clang/include/clang/Basic/TypeNodes.td        |  2 +-
 clang/lib/AST/ASTContext.cpp                  | 34 +++++++------------
 clang/lib/AST/Type.cpp                        |  9 +++--
 clang/lib/CodeGen/CGDebugInfo.cpp             |  1 -
 clang/lib/CodeGen/CodeGenFunction.cpp         |  1 -
 clang/lib/Sema/SemaExpr.cpp                   |  1 -
 clang/lib/Sema/SemaLambda.cpp                 |  3 +-
 clang/lib/Sema/SemaTemplateVariadic.cpp       |  3 +-
 clang/lib/Sema/SemaType.cpp                   |  2 +-
 .../alias-template-nondependent.cpp           | 24 +++++++++++++
 12 files changed, 62 insertions(+), 41 deletions(-)
 create mode 100644 clang/test/SemaTemplate/alias-template-nondependent.cpp

diff --git a/clang/include/clang/AST/ASTContext.h b/clang/include/clang/AST/ASTContext.h
index 59e2679ddded7..6c00fe86f282d 100644
--- a/clang/include/clang/AST/ASTContext.h
+++ b/clang/include/clang/AST/ASTContext.h
@@ -1459,8 +1459,16 @@ class ASTContext : public RefCountedBase<ASTContext> {
   void getInjectedTemplateArgs(const TemplateParameterList *Params,
                                SmallVectorImpl<TemplateArgument> &Args);
 
+  /// Form a pack expansion type with the given pattern.
+  /// \param NumExpansions The number of expansions for the pack, if known.
+  /// \param ExpectPackInType If \c false, we should not expect \p Pattern to
+  ///        contain an unexpanded pack. This only makes sense if the pack
+  ///        expansion is used in a context where the arity is inferred from
+  ///        elsewhere, such as if the pattern contains a placeholder type or
+  ///        if this is the canonical type of another pack expansion type.
   QualType getPackExpansionType(QualType Pattern,
-                                Optional<unsigned> NumExpansions);
+                                Optional<unsigned> NumExpansions,
+                                bool ExpectPackInType = true);
 
   QualType getObjCInterfaceType(const ObjCInterfaceDecl *Decl,
                                 ObjCInterfaceDecl *PrevDecl = nullptr) const;
diff --git a/clang/include/clang/AST/Type.h b/clang/include/clang/AST/Type.h
index 9a745ef20fac3..7fe652492b0e0 100644
--- a/clang/include/clang/AST/Type.h
+++ b/clang/include/clang/AST/Type.h
@@ -4383,11 +4383,7 @@ class TypedefType : public Type {
 protected:
   friend class ASTContext; // ASTContext creates these.
 
-  TypedefType(TypeClass tc, const TypedefNameDecl *D, QualType can)
-      : Type(tc, can, can->getDependence() & ~TypeDependence::UnexpandedPack),
-        Decl(const_cast<TypedefNameDecl *>(D)) {
-    assert(!isa<TypedefType>(can) && "Invalid canonical type");
-  }
+  TypedefType(TypeClass tc, const TypedefNameDecl *D, QualType can);
 
 public:
   TypedefNameDecl *getDecl() const { return Decl; }
@@ -5624,7 +5620,8 @@ class PackExpansionType : public Type, public llvm::FoldingSetNode {
   PackExpansionType(QualType Pattern, QualType Canon,
                     Optional<unsigned> NumExpansions)
       : Type(PackExpansion, Canon,
-             (Pattern->getDependence() | TypeDependence::Instantiation) &
+             (Pattern->getDependence() | TypeDependence::Dependent |
+              TypeDependence::Instantiation) &
                  ~TypeDependence::UnexpandedPack),
         Pattern(Pattern) {
     PackExpansionTypeBits.NumExpansions =
@@ -5645,8 +5642,8 @@ class PackExpansionType : public Type, public llvm::FoldingSetNode {
     return None;
   }
 
-  bool isSugared() const { return !Pattern->isDependentType(); }
-  QualType desugar() const { return isSugared() ? Pattern : QualType(this, 0); }
+  bool isSugared() const { return false; }
+  QualType desugar() const { return QualType(this, 0); }
 
   void Profile(llvm::FoldingSetNodeID &ID) {
     Profile(ID, getPattern(), getNumExpansions());
diff --git a/clang/include/clang/Basic/TypeNodes.td b/clang/include/clang/Basic/TypeNodes.td
index a4e3002b90753..011394c3ef455 100644
--- a/clang/include/clang/Basic/TypeNodes.td
+++ b/clang/include/clang/Basic/TypeNodes.td
@@ -100,7 +100,7 @@ def DeducedTemplateSpecializationType : TypeNode<DeducedType>;
 def InjectedClassNameType : TypeNode<Type>, AlwaysDependent, LeafType;
 def DependentNameType : TypeNode<Type>, AlwaysDependent;
 def DependentTemplateSpecializationType : TypeNode<Type>, AlwaysDependent;
-def PackExpansionType : TypeNode<Type>, NeverCanonicalUnlessDependent;
+def PackExpansionType : TypeNode<Type>, AlwaysDependent;
 def ObjCTypeParamType : TypeNode<Type>, NeverCanonical;
 def ObjCObjectType : TypeNode<Type>;
 def ObjCInterfaceType : TypeNode<ObjCObjectType>, LeafType;
diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index e7518a538fe67..25bf71519d1c3 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -4817,37 +4817,27 @@ ASTContext::getInjectedTemplateArgs(const TemplateParameterList *Params,
 }
 
 QualType ASTContext::getPackExpansionType(QualType Pattern,
-                                          Optional<unsigned> NumExpansions) {
+                                          Optional<unsigned> NumExpansions,
+                                          bool ExpectPackInType) {
+  assert((!ExpectPackInType || Pattern->containsUnexpandedParameterPack()) &&
+         "Pack expansions must expand one or more parameter packs");
+
   llvm::FoldingSetNodeID ID;
   PackExpansionType::Profile(ID, Pattern, NumExpansions);
 
-  // A deduced type can deduce to a pack, eg
-  //   auto ...x = some_pack;
-  // That declaration isn't (yet) valid, but is created as part of building an
-  // init-capture pack:
-  //   [...x = some_pack] {}
-  assert((Pattern->containsUnexpandedParameterPack() ||
-          Pattern->getContainedDeducedType()) &&
-         "Pack expansions must expand one or more parameter packs");
   void *InsertPos = nullptr;
-  PackExpansionType *T
-    = PackExpansionTypes.FindNodeOrInsertPos(ID, InsertPos);
+  PackExpansionType *T = PackExpansionTypes.FindNodeOrInsertPos(ID, InsertPos);
   if (T)
     return QualType(T, 0);
 
   QualType Canon;
   if (!Pattern.isCanonical()) {
-    Canon = getCanonicalType(Pattern);
-    // The canonical type might not contain an unexpanded parameter pack, if it
-    // contains an alias template specialization which ignores one of its
-    // parameters.
-    if (Canon->containsUnexpandedParameterPack()) {
-      Canon = getPackExpansionType(Canon, NumExpansions);
-
-      // Find the insert position again, in case we inserted an element into
-      // PackExpansionTypes and invalidated our insert position.
-      PackExpansionTypes.FindNodeOrInsertPos(ID, InsertPos);
-    }
+    Canon = getPackExpansionType(getCanonicalType(Pattern), NumExpansions,
+                                 /*ExpectPackInType=*/false);
+
+    // Find the insert position again, in case we inserted an element into
+    // PackExpansionTypes and invalidated our insert position.
+    PackExpansionTypes.FindNodeOrInsertPos(ID, InsertPos);
   }
 
   T = new (*this, TypeAlignment)
diff --git a/clang/lib/AST/Type.cpp b/clang/lib/AST/Type.cpp
index 0122d2e7de52d..d40ba4c648c4c 100644
--- a/clang/lib/AST/Type.cpp
+++ b/clang/lib/AST/Type.cpp
@@ -1187,9 +1187,6 @@ struct SimpleTransformVisitor : public TypeVisitor<Derived, QualType> {
                            T->getTypeConstraintArguments());
   }
 
-  // FIXME: Non-trivial to implement, but important for C++
-  SUGARED_TYPE_CLASS(PackExpansion)
-
   QualType VisitObjCObjectType(const ObjCObjectType *T) {
     QualType baseType = recurse(T->getBaseType());
     if (baseType.isNull())
@@ -3348,6 +3345,12 @@ void FunctionProtoType::Profile(llvm::FoldingSetNodeID &ID,
           getExtProtoInfo(), Ctx, isCanonicalUnqualified());
 }
 
+TypedefType::TypedefType(TypeClass tc, const TypedefNameDecl *D, QualType can)
+    : Type(tc, can, D->getUnderlyingType()->getDependence()),
+      Decl(const_cast<TypedefNameDecl *>(D)) {
+  assert(!isa<TypedefType>(can) && "Invalid canonical type");
+}
+
 QualType TypedefType::desugar() const {
   return getDecl()->getUnderlyingType();
 }
diff --git a/clang/lib/CodeGen/CGDebugInfo.cpp b/clang/lib/CodeGen/CGDebugInfo.cpp
index 6965c4a1209c2..780e0c692c051 100644
--- a/clang/lib/CodeGen/CGDebugInfo.cpp
+++ b/clang/lib/CodeGen/CGDebugInfo.cpp
@@ -3252,7 +3252,6 @@ llvm::DIType *CGDebugInfo::CreateTypeNode(QualType Ty, llvm::DIFile *Unit) {
   case Type::TypeOf:
   case Type::Decltype:
   case Type::UnaryTransform:
-  case Type::PackExpansion:
     break;
   }
 
diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp
index 8ce488f35dd32..8f79cc77f0e64 100644
--- a/clang/lib/CodeGen/CodeGenFunction.cpp
+++ b/clang/lib/CodeGen/CodeGenFunction.cpp
@@ -2075,7 +2075,6 @@ void CodeGenFunction::EmitVariablyModifiedType(QualType type) {
     case Type::UnaryTransform:
     case Type::Attributed:
     case Type::SubstTemplateTypeParm:
-    case Type::PackExpansion:
     case Type::MacroQualified:
       // Keep walking after single level desugaring.
       type = type.getSingleStepDesugaredType(getContext());
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index 21d3bbf419a9a..bb0b1fa49851d 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -4345,7 +4345,6 @@ static void captureVariablyModifiedType(ASTContext &Context, QualType T,
     case Type::UnaryTransform:
     case Type::Attributed:
     case Type::SubstTemplateTypeParm:
-    case Type::PackExpansion:
     case Type::MacroQualified:
       // Keep walking after single level desugaring.
       T = T.getSingleStepDesugaredType(Context);
diff --git a/clang/lib/Sema/SemaLambda.cpp b/clang/lib/Sema/SemaLambda.cpp
index 657ed13f207ad..dc74f6e2f7dc5 100644
--- a/clang/lib/Sema/SemaLambda.cpp
+++ b/clang/lib/Sema/SemaLambda.cpp
@@ -803,7 +803,8 @@ QualType Sema::buildLambdaInitCaptureInitialization(
       Diag(EllipsisLoc, getLangOpts().CPlusPlus20
                             ? diag::warn_cxx17_compat_init_capture_pack
                             : diag::ext_init_capture_pack);
-      DeductType = Context.getPackExpansionType(DeductType, NumExpansions);
+      DeductType = Context.getPackExpansionType(DeductType, NumExpansions,
+                                                /*ExpectPackInType=*/false);
       TLB.push<PackExpansionTypeLoc>(DeductType).setEllipsisLoc(EllipsisLoc);
     } else {
       // Just ignore the ellipsis for now and form a non-pack variable. We'll
diff --git a/clang/lib/Sema/SemaTemplateVariadic.cpp b/clang/lib/Sema/SemaTemplateVariadic.cpp
index 7b77d1cb482ae..259cc51657763 100644
--- a/clang/lib/Sema/SemaTemplateVariadic.cpp
+++ b/clang/lib/Sema/SemaTemplateVariadic.cpp
@@ -614,7 +614,8 @@ QualType Sema::CheckPackExpansion(QualType Pattern, SourceRange PatternRange,
     return QualType();
   }
 
-  return Context.getPackExpansionType(Pattern, NumExpansions);
+  return Context.getPackExpansionType(Pattern, NumExpansions,
+                                      /*ExpectPackInType=*/false);
 }
 
 ExprResult Sema::ActOnPackExpansion(Expr *Pattern, SourceLocation EllipsisLoc) {
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index 3eabe7ca6ffe7..4c7eece68bca3 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -5516,7 +5516,7 @@ static TypeSourceInfo *GetFullTypeForDeclarator(TypeProcessingState &state,
           << T <<  D.getSourceRange();
         D.setEllipsisLoc(SourceLocation());
       } else {
-        T = Context.getPackExpansionType(T, None);
+        T = Context.getPackExpansionType(T, None, /*ExpectPackInType=*/false);
       }
       break;
     case DeclaratorContext::TemplateParamContext:
diff --git a/clang/test/SemaTemplate/alias-template-nondependent.cpp b/clang/test/SemaTemplate/alias-template-nondependent.cpp
new file mode 100644
index 0000000000000..e8ea16483a09f
--- /dev/null
+++ b/clang/test/SemaTemplate/alias-template-nondependent.cpp
@@ -0,0 +1,24 @@
+// RUN: %clang_cc1 -std=c++20 -verify %s
+
+namespace PR46377 {
+  template<typename> using IntPtr = int*;
+  template<typename ...T> auto non_dependent_typedef() {
+    typedef int(*P)(IntPtr<T>...);
+    return P();
+  }
+  template<typename ...T> auto non_dependent_alias() {
+    using P = int(*)(IntPtr<T>...);
+    return P();
+  }
+  template<typename ...T> auto non_dependent_via_sizeof() {
+    using P = int(*)(int(...pack)[sizeof(sizeof(T))]); // expected-error {{invalid application of 'sizeof'}}
+    return P();
+  }
+
+  using a = int (*)(int*, int*);
+  using a = decltype(non_dependent_typedef<void, void>());
+  using a = decltype(non_dependent_alias<void, void>());
+  using a = decltype(non_dependent_via_sizeof<float, float>());
+
+  using b = decltype(non_dependent_via_sizeof<float, void>()); // expected-note {{instantiation of}}
+}

From dd405f1a5397d3e7595458144ccb0bcec192a3bf Mon Sep 17 00:00:00 2001
From: Fangrui Song <maskray@google.com>
Date: Tue, 28 Jul 2020 13:26:09 -0700
Subject: [PATCH 0364/1035] Revert D83834 "Add test utility 'extract'"

This reverts commit d054c7ee2e9f4f98af7f22a5b00a941eb919bd59.

There are discussions about the utility name, its functionality and user interface.
Revert before we reach consensus.
---
 lld/test/CMakeLists.txt                       |   2 +-
 lld/test/ELF/linkerscript/noload.s            |  19 ++-
 lld/test/lit.cfg.py                           |   8 +-
 llvm/docs/TestingGuide.rst                    |  23 +---
 llvm/test/CMakeLists.txt                      |   1 -
 llvm/test/lit.cfg.py                          |   1 -
 llvm/test/tools/extract/Inputs/basic-aa.txt   |   6 -
 llvm/test/tools/extract/Inputs/basic-bb.txt   |  10 --
 llvm/test/tools/extract/basic.test            |  32 -----
 llvm/test/tools/extract/help.test             |   5 -
 llvm/test/tools/extract/no-leading-lines.test |  10 --
 llvm/test/tools/gold/X86/multiple-sections.ll |  14 +--
 .../tools/llvm-objcopy/ELF/strip-symbol.test  |  19 ++-
 llvm/test/tools/llvm-strings/radix.test       |  23 ++--
 llvm/tools/extract/.clang-tidy                |  19 ---
 llvm/tools/extract/CMakeLists.txt             |   7 --
 llvm/tools/extract/extract.cpp                | 113 ------------------
 17 files changed, 36 insertions(+), 276 deletions(-)
 delete mode 100644 llvm/test/tools/extract/Inputs/basic-aa.txt
 delete mode 100644 llvm/test/tools/extract/Inputs/basic-bb.txt
 delete mode 100644 llvm/test/tools/extract/basic.test
 delete mode 100644 llvm/test/tools/extract/help.test
 delete mode 100644 llvm/test/tools/extract/no-leading-lines.test
 delete mode 100644 llvm/tools/extract/.clang-tidy
 delete mode 100644 llvm/tools/extract/CMakeLists.txt
 delete mode 100644 llvm/tools/extract/extract.cpp

diff --git a/lld/test/CMakeLists.txt b/lld/test/CMakeLists.txt
index 753eacf4d255b..e7d1133307393 100644
--- a/lld/test/CMakeLists.txt
+++ b/lld/test/CMakeLists.txt
@@ -25,7 +25,7 @@ configure_lit_site_cfg(
 set(LLD_TEST_DEPS lld)
 if (NOT LLD_BUILT_STANDALONE)
   list(APPEND LLD_TEST_DEPS
-    FileCheck count extract llc llvm-ar llvm-as llvm-bcanalyzer llvm-config llvm-cvtres
+    FileCheck count llc llvm-ar llvm-as llvm-bcanalyzer llvm-config llvm-cvtres
     llvm-dis llvm-dwarfdump llvm-lib llvm-lipo llvm-mc llvm-nm llvm-objcopy
     llvm-objdump llvm-pdbutil llvm-readelf llvm-readobj llvm-strip not obj2yaml
     opt yaml2obj
diff --git a/lld/test/ELF/linkerscript/noload.s b/lld/test/ELF/linkerscript/noload.s
index c2014722985d3..2f52b465854e2 100644
--- a/lld/test/ELF/linkerscript/noload.s
+++ b/lld/test/ELF/linkerscript/noload.s
@@ -1,7 +1,11 @@
 # REQUIRES: x86
-# RUN: extract asm %s -o %t.s && extract lds %s -o %t.lds
-# RUN: llvm-mc -filetype=obj -triple=x86_64 %t.s -o %t.o
-# RUN: ld.lld -o %t --script %t.lds %t.o
+# RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %s -o %t.o
+# RUN: echo "SECTIONS { \
+# RUN:        .data_noload_a (NOLOAD) : { *(.data_noload_a) } \
+# RUN:        .data_noload_b (0x10000) (NOLOAD) : { *(.data_noload_b) } \
+# RUN:        .no_input_sec_noload (NOLOAD) : { . += 1; } \
+# RUN:        .text (0x20000) : { *(.text) } };" > %t.script
+# RUN: ld.lld -o %t --script %t.script %t.o
 # RUN: llvm-readelf -S -l %t | FileCheck %s
 
 # CHECK:      Name                 Type   Address          Off               Size
@@ -12,7 +16,6 @@
 # CHECK:      Type Offset   VirtAddr           PhysAddr
 # CHECK-NEXT: LOAD 0x001000 0x0000000000020000 0x0000000000020000
 
-#--- asm
 .section .text,"ax",@progbits
   nop
 
@@ -21,11 +24,3 @@
 
 .section .data_noload_b,"aw",@progbits
 .zero 4096
-
-#--- lds
-SECTIONS {
-  .data_noload_a (NOLOAD) : { *(.data_noload_a) }
-  .data_noload_b (0x10000) (NOLOAD) : { *(.data_noload_b) }
-  .no_input_sec_noload (NOLOAD) : { . += 1; }
-  .text (0x20000) : { *(.text) }
-}
diff --git a/lld/test/lit.cfg.py b/lld/test/lit.cfg.py
index 0fa9b48c3c792..267f8c5178584 100644
--- a/lld/test/lit.cfg.py
+++ b/lld/test/lit.cfg.py
@@ -39,9 +39,9 @@
 llvm_config.use_lld()
 
 tool_patterns = [
-    'extract', 'llc', 'llvm-as', 'llvm-mc', 'llvm-nm', 'llvm-objdump',
-    'llvm-pdbutil', 'llvm-dwarfdump', 'llvm-readelf', 'llvm-readobj',
-    'obj2yaml', 'yaml2obj', 'opt', 'llvm-dis']
+    'llc', 'llvm-as', 'llvm-mc', 'llvm-nm', 'llvm-objdump', 'llvm-pdbutil',
+    'llvm-dwarfdump', 'llvm-readelf', 'llvm-readobj', 'obj2yaml', 'yaml2obj',
+    'opt', 'llvm-dis']
 
 llvm_config.add_tool_substitutions(tool_patterns)
 
@@ -87,7 +87,7 @@
 # Indirectly check if the mt.exe Microsoft utility exists by searching for
 # cvtres, which always accompanies it.  Alternatively, check if we can use
 # libxml2 to merge manifests.
-if (lit.util.which('cvtres', config.environment['PATH']) or
+if (lit.util.which('cvtres', config.environment['PATH']) or 
         config.llvm_libxml2_enabled):
     config.available_features.add('manifest_tool')
 
diff --git a/llvm/docs/TestingGuide.rst b/llvm/docs/TestingGuide.rst
index 6fd9ab2d24ca4..2e937f0006272 100644
--- a/llvm/docs/TestingGuide.rst
+++ b/llvm/docs/TestingGuide.rst
@@ -271,27 +271,8 @@ adding your code there instead of creating a new file.
 Extra files
 -----------
 
-If your test requires extra files besides the file containing the ``RUN:`` lines
-and the extra files are small, consider specifying them in the same file and
-using ``extract`` to extract them. For example,
-
-.. code-block:: llvm
-
-  ; RUN: extract b %s -o %tb.ll
-  ; RUN: extract a %s | llvm-link - %tb.ll -S | FileCheck %s
-
-  ; CHECK: ...
-
-  ;--- a
-  ...
-  ;--- b
-  ...
-
-The parts are separated by the regex ``^(.|//)--- <part>``. By default the
-extracted content has leading empty lines to preserve line numbers. Specify
-``--no-leading-lines`` to drop leading lines.
-
-If the extra files are large, the idiomatic place to put them is in a subdirectory ``Inputs``.
+If your test requires extra files besides the file containing the ``RUN:``
+lines, the idiomatic place to put them is in a subdirectory ``Inputs``.
 You can then refer to the extra files as ``%S/Inputs/foo.bar``.
 
 For example, consider ``test/Linker/ident.ll``. The directory structure is
diff --git a/llvm/test/CMakeLists.txt b/llvm/test/CMakeLists.txt
index 7ec0453fb131e..91215b3ca0ef0 100644
--- a/llvm/test/CMakeLists.txt
+++ b/llvm/test/CMakeLists.txt
@@ -52,7 +52,6 @@ set(LLVM_TEST_DEPENDS
           UnitTests
           bugpoint
           count
-          extract
           llc
           lli
           lli-child-target
diff --git a/llvm/test/lit.cfg.py b/llvm/test/lit.cfg.py
index 49bd8ddfb2dc5..0a3289fcc4ad4 100644
--- a/llvm/test/lit.cfg.py
+++ b/llvm/test/lit.cfg.py
@@ -130,7 +130,6 @@ def get_asan_rtlib():
 config.llvm_locstats_used = os.path.exists(llvm_locstats_tool)
 
 tools = [
-    ToolSubst('%extract', FindTool('extract')),
     ToolSubst('%lli', FindTool('lli'), post='.', extra_args=lli_args),
     ToolSubst('%llc_dwarf', FindTool('llc'), extra_args=llc_args),
     ToolSubst('%go', config.go_executable, unresolved='ignore'),
diff --git a/llvm/test/tools/extract/Inputs/basic-aa.txt b/llvm/test/tools/extract/Inputs/basic-aa.txt
deleted file mode 100644
index 9eac3fdccbee4..0000000000000
--- a/llvm/test/tools/extract/Inputs/basic-aa.txt
+++ /dev/null
@@ -1,6 +0,0 @@
-
-
-
-aa
-; BB-NOT: {{.}}
-; BB: {{^}}bb{{$}}
diff --git a/llvm/test/tools/extract/Inputs/basic-bb.txt b/llvm/test/tools/extract/Inputs/basic-bb.txt
deleted file mode 100644
index de17efab6fb6b..0000000000000
--- a/llvm/test/tools/extract/Inputs/basic-bb.txt
+++ /dev/null
@@ -1,10 +0,0 @@
-
-
-
-
-
-
-
-bb
-
-// CC: // Comments are preserved.
diff --git a/llvm/test/tools/extract/basic.test b/llvm/test/tools/extract/basic.test
deleted file mode 100644
index 9f9413106cc75..0000000000000
--- a/llvm/test/tools/extract/basic.test
+++ /dev/null
@@ -1,32 +0,0 @@
-# AA-NOT: {{.}}
-# AA: {{^}}aa{{$}}
-#--- aa
-aa
-; BB-NOT: {{.}}
-; BB: {{^}}bb{{$}}
-;--- bb
-bb
-
-// CC: // Comments are preserved.
-//--- cc
-cc
-// Comments are preserved.
-;--- dup
-;--- dup
-
-# RUN: extract aa %s | diff %S/Inputs/basic-aa.txt -
-# RUN: extract bb - < %s | diff %S/Inputs/basic-bb.txt -
-# RUN: extract cc %s -o %t
-# RUN: FileCheck %s --check-prefix=CC < %t
-
-# RUN: not %extract aa 2>&1 | FileCheck %s --check-prefix=NO_INPUT
-
-# NO_INPUT: extract: error: input filename is not specified
-
-# RUN: not %extract dup %s 2>&1 | FileCheck %s --check-prefix=DUP
-
-# DUP: extract: error: {{.*}}.test: ';--- dup' occurs more than once
-
-# RUN: not %extract not_exist %s 2>&1 | FileCheck %s --check-prefix=NOT_EXIST
-
-# NOT_EXIST: extract: error: {{.*}}.test: ';--- not_exist' was not found
diff --git a/llvm/test/tools/extract/help.test b/llvm/test/tools/extract/help.test
deleted file mode 100644
index 282052869116c..0000000000000
--- a/llvm/test/tools/extract/help.test
+++ /dev/null
@@ -1,5 +0,0 @@
-RUN: extract --help 2>&1 | FileCheck --implicit-check-not='General Options:' %s
-CHECK: OVERVIEW: Split input {{.*}}
-CHECK: Generic Options:
-CHECK: extract Options:
-CHECK:   -o
diff --git a/llvm/test/tools/extract/no-leading-lines.test b/llvm/test/tools/extract/no-leading-lines.test
deleted file mode 100644
index f0efff5475afb..0000000000000
--- a/llvm/test/tools/extract/no-leading-lines.test
+++ /dev/null
@@ -1,10 +0,0 @@
-## With --no-leading-lines, don't add leading lines (which is used to preserve line numbers).
-
-# RUN: extract --no-leading-lines input %s -o %t
-# RUN: count 1 < %t
-# RUN: FileCheck %s < %t
-
-# CHECK: input
-
-#--- input
-input
diff --git a/llvm/test/tools/gold/X86/multiple-sections.ll b/llvm/test/tools/gold/X86/multiple-sections.ll
index 31a89a9d3b484..facbd8d992ed7 100644
--- a/llvm/test/tools/gold/X86/multiple-sections.ll
+++ b/llvm/test/tools/gold/X86/multiple-sections.ll
@@ -1,8 +1,10 @@
-; RUN: extract order %s -o %t.order
-; RUN: extract ir %s | llvm-as -o %t.o
+; RUN: echo ".text.tin" > %t_order_lto.txt
+; RUN: echo ".text._start" >> %t_order_lto.txt
+; RUN: echo ".text.pat" >> %t_order_lto.txt
+; RUN: llvm-as %s -o %t.o
 ; RUN: %gold -plugin %llvmshlibdir/LLVMgold%shlibext \
 ; RUN:     -m elf_x86_64 -o %t.exe %t.o \
-; RUN:     --section-ordering-file=%t.order
+; RUN:     --section-ordering-file=%t_order_lto.txt
 ; RUN: llvm-readelf -s %t.exe | FileCheck %s
 
 ; Check that the order of the sections is tin -> _start -> pat.
@@ -11,12 +13,6 @@
 ; CHECK:      00000000004000b0     1 FUNC    LOCAL  DEFAULT    1 tin
 ; CHECK:      00000000004000c0    15 FUNC    GLOBAL DEFAULT    1 _start
 
-;--- order
-.text.tin
-.text._start
-.text.pat
-
-;--- ir
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
diff --git a/llvm/test/tools/llvm-objcopy/ELF/strip-symbol.test b/llvm/test/tools/llvm-objcopy/ELF/strip-symbol.test
index ad71e81eab830..78de46cc47b5d 100644
--- a/llvm/test/tools/llvm-objcopy/ELF/strip-symbol.test
+++ b/llvm/test/tools/llvm-objcopy/ELF/strip-symbol.test
@@ -1,24 +1,19 @@
-# RUN: extract yaml %s | yaml2obj - -o %t
+# RUN: yaml2obj %s -o %t
 # RUN: llvm-objcopy --strip-symbol baz -N bar %t %t2
 # RUN: llvm-readobj --symbols --sections %t2 | FileCheck %s
 # RUN: llvm-strip --strip-symbol baz -N bar %t -o %t3
 # RUN: cmp %t2 %t3
 # RUN: llvm-strip --regex --strip-symbol '^b.*' -N bar %t -o %t4
 # RUN: cmp %t3 %t4
-# RUN: extract list1 %s -o %t-list.txt && llvm-objcopy --strip-symbols %t-list.txt %t %t5
+# RUN: echo " bar # bar" > %t-list.txt
+# RUN: echo " baz # baz" >> %t-list.txt
+# RUN: echo " # no symbol" >> %t-list.txt
+# RUN: llvm-objcopy --strip-symbols %t-list.txt %t %t5
 # RUN: cmp %t3 %t5
-# RUN: extract list2 %s -o %t-list2.txt && llvm-objcopy --regex --strip-symbols %t-list2.txt %t %t6
+# RUN: echo "b.* # bar & baz" > %t-list2.txt
+# RUN: llvm-objcopy --regex --strip-symbols %t-list2.txt %t %t6
 # RUN: cmp %t3 %t6
 
-#--- list1
-bar # bar
-baz # baz
-# no symbol
-
-#--- list2
-b.* # bar & baz
-
-#--- yaml
 !ELF
 FileHeader:
   Class:           ELFCLASS64
diff --git a/llvm/test/tools/llvm-strings/radix.test b/llvm/test/tools/llvm-strings/radix.test
index d9796a937d905..d23fb3cddc8f8 100644
--- a/llvm/test/tools/llvm-strings/radix.test
+++ b/llvm/test/tools/llvm-strings/radix.test
@@ -1,18 +1,15 @@
 ## Show that llvm-strings can handle the -t/--radix switch properly.
 
-RUN: extract --no-leading-lines input %s -o %t
-#--- input
-one
-two
-three
-four
-five
-six
-seven
-eight
-nine
-ten
-#--- end
+RUN: echo one > %t
+RUN: echo two >> %t
+RUN: echo three >> %t
+RUN: echo four >> %t
+RUN: echo five >> %t
+RUN: echo six >> %t
+RUN: echo seven >> %t
+RUN: echo eight >> %t
+RUN: echo nine >> %t
+RUN: echo ten >> %t
 
 RUN: llvm-strings %t | FileCheck %s -check-prefix CHECK-NONE --implicit-check-not={{.}}
 RUN: llvm-strings -t d %t | FileCheck %s -check-prefix CHECK-DEC --strict-whitespace --implicit-check-not={{.}}
diff --git a/llvm/tools/extract/.clang-tidy b/llvm/tools/extract/.clang-tidy
deleted file mode 100644
index 87ec2ff53af6e..0000000000000
--- a/llvm/tools/extract/.clang-tidy
+++ /dev/null
@@ -1,19 +0,0 @@
-# Almost identical to the top-level .clang-tidy, except that {Member,Parameter,Variable}Case use camelBack.
-Checks: '-*,clang-diagnostic-*,llvm-*,misc-*,-misc-unused-parameters,-misc-non-private-member-variables-in-classes,readability-identifier-naming'
-CheckOptions:
-  - key:             readability-identifier-naming.ClassCase
-    value:           CamelCase
-  - key:             readability-identifier-naming.EnumCase
-    value:           CamelCase
-  - key:             readability-identifier-naming.FunctionCase
-    value:           camelBack
-  - key:             readability-identifier-naming.MemberCase
-    value:           camelBack
-  - key:             readability-identifier-naming.ParameterCase
-    value:           camelBack
-  - key:             readability-identifier-naming.UnionCase
-    value:           CamelCase
-  - key:             readability-identifier-naming.VariableCase
-    value:           camelBack
-  - key:             readability-identifier-naming.IgnoreMainLikeFunctions
-    value:           1
diff --git a/llvm/tools/extract/CMakeLists.txt b/llvm/tools/extract/CMakeLists.txt
deleted file mode 100644
index dae1f463f0666..0000000000000
--- a/llvm/tools/extract/CMakeLists.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-set(LLVM_LINK_COMPONENTS
-  Support
-  )
-
-add_llvm_tool(extract
-  extract.cpp
-  )
diff --git a/llvm/tools/extract/extract.cpp b/llvm/tools/extract/extract.cpp
deleted file mode 100644
index 8ccb539156145..0000000000000
--- a/llvm/tools/extract/extract.cpp
+++ /dev/null
@@ -1,113 +0,0 @@
-//===- extract.cpp - Input splitting utility ------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// Split input into multipe parts separated by regex '^(.|//)--- ' and extract
-// the specified part.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/ADT/StringRef.h"
-#include "llvm/Support/CommandLine.h"
-#include "llvm/Support/FileOutputBuffer.h"
-#include "llvm/Support/LineIterator.h"
-#include "llvm/Support/MemoryBuffer.h"
-#include "llvm/Support/Path.h"
-#include "llvm/Support/WithColor.h"
-#include <string>
-
-using namespace llvm;
-
-static cl::OptionCategory cat("extract Options");
-
-static cl::opt<std::string> part(cl::Positional, cl::desc("part"),
-                                 cl::cat(cat));
-
-static cl::opt<std::string> input(cl::Positional, cl::desc("filename"),
-                                  cl::cat(cat));
-
-static cl::opt<std::string> output("o", cl::desc("Output filename"),
-                                   cl::value_desc("filename"), cl::init("-"),
-                                   cl::cat(cat));
-
-static cl::opt<bool> noLeadingLines("no-leading-lines",
-                                    cl::desc("Don't preserve line numbers"),
-                                    cl::cat(cat));
-
-static StringRef toolName;
-
-LLVM_ATTRIBUTE_NORETURN static void error(StringRef filename,
-                                          const Twine &message) {
-  if (filename.empty())
-    WithColor::error(errs(), toolName) << message << '\n';
-  else
-    WithColor::error(errs(), toolName) << filename << ": " << message << '\n';
-  exit(1);
-}
-
-static void handle(MemoryBuffer &inputBuf, StringRef input) {
-  const char *partBegin = nullptr, *partEnd = nullptr;
-  int numEmptyLines = 0;
-  StringRef separator;
-  for (line_iterator i(inputBuf, /*SkipBlanks=*/false, '\0'); !i.is_at_eof();) {
-    StringRef line = *i++;
-    size_t markerLen = line.startswith("//") ? 6 : 5;
-    if (!(line.size() > markerLen &&
-          line.substr(markerLen - 4).startswith("--- ")))
-      continue;
-    separator = line.substr(0, markerLen);
-    StringRef cur = line.substr(markerLen);
-    if (cur == part) {
-      if (partBegin)
-        error(input, "'" + separator + cur + "' occurs more than once");
-      if (!noLeadingLines)
-        numEmptyLines = i.line_number() - 1;
-      if (i.is_at_eof())
-        break;
-      partBegin = i->data();
-    } else if (partBegin && !partEnd) {
-      partEnd = line.data();
-    }
-  }
-  if (!partBegin)
-    error(input, "'" + separator + part + "' was not found");
-  if (!partEnd)
-    partEnd = inputBuf.getBufferEnd();
-
-  Expected<std::unique_ptr<FileOutputBuffer>> outputBuf =
-      FileOutputBuffer::create(output, numEmptyLines + (partEnd - partBegin));
-  if (!outputBuf)
-    error(input, toString(outputBuf.takeError()));
-  uint8_t *buf = (*outputBuf)->getBufferStart();
-
-  // If --no-leading-lines is not specified, numEmptyLines is 0. Append newlines
-  // so that the extracted part preserves line numbers.
-  std::fill_n(buf, numEmptyLines, '\n');
-  std::copy(partBegin, partEnd, buf + numEmptyLines);
-  if (Error e = (*outputBuf)->commit())
-    error(input, toString(std::move(e)));
-}
-
-int main(int argc, const char **argv) {
-  toolName = sys::path::stem(argv[0]);
-  cl::HideUnrelatedOptions({&cat});
-  cl::ParseCommandLineOptions(
-      argc, argv,
-      "Split input into multiple parts separated by regex '^(.|//)--- ' and "
-      "extract the part specified by '^(.|//)--- <part>'\n",
-      nullptr,
-      /*EnvVar=*/nullptr,
-      /*LongOptionsUseDoubleDash=*/true);
-
-  if (input.empty())
-    error("", "input filename is not specified");
-  ErrorOr<std::unique_ptr<MemoryBuffer>> bufferOrErr =
-      MemoryBuffer::getFileOrSTDIN(input);
-  if (std::error_code ec = bufferOrErr.getError())
-    error(input, ec.message());
-  handle(**bufferOrErr, input);
-}

From 26fc91eb5d26fb8bc3ab5328e698c9c1d9546b44 Mon Sep 17 00:00:00 2001
From: Fangrui Song <maskray@google.com>
Date: Tue, 28 Jul 2020 13:30:29 -0700
Subject: [PATCH 0365/1035] Revert "[gn build] (manually) merge d054c7ee2e9"

This reverts commit ab73b6da95750164daac4cfbd351ca96e1084117.
---
 llvm/utils/gn/secondary/lld/test/BUILD.gn           | 1 -
 llvm/utils/gn/secondary/llvm/test/BUILD.gn          | 1 -
 llvm/utils/gn/secondary/llvm/tools/extract/BUILD.gn | 4 ----
 3 files changed, 6 deletions(-)
 delete mode 100644 llvm/utils/gn/secondary/llvm/tools/extract/BUILD.gn

diff --git a/llvm/utils/gn/secondary/lld/test/BUILD.gn b/llvm/utils/gn/secondary/lld/test/BUILD.gn
index a6fb457cff6ac..581cc5482578c 100644
--- a/llvm/utils/gn/secondary/lld/test/BUILD.gn
+++ b/llvm/utils/gn/secondary/lld/test/BUILD.gn
@@ -78,7 +78,6 @@ group("test") {
     ":lit_unit_site_cfg",
     "//lld/tools/lld:symlinks",
     "//lld/unittests",
-    "//llvm/tools/extract",
     "//llvm/tools/llc",
     "//llvm/tools/llvm-ar:symlinks",
     "//llvm/tools/llvm-as",
diff --git a/llvm/utils/gn/secondary/llvm/test/BUILD.gn b/llvm/utils/gn/secondary/llvm/test/BUILD.gn
index 3ad1d0ba4f443..2c4a23ffbaacb 100644
--- a/llvm/utils/gn/secondary/llvm/test/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/test/BUILD.gn
@@ -203,7 +203,6 @@ group("test") {
     "//llvm/lib/Testing/Support",
     "//llvm/tools/bugpoint",
     "//llvm/tools/dsymutil",
-    "//llvm/tools/extract",
     "//llvm/tools/llc",
     "//llvm/tools/lli",
     "//llvm/tools/lli/ChildTarget:lli-child-target",
diff --git a/llvm/utils/gn/secondary/llvm/tools/extract/BUILD.gn b/llvm/utils/gn/secondary/llvm/tools/extract/BUILD.gn
deleted file mode 100644
index f4553476f1e8b..0000000000000
--- a/llvm/utils/gn/secondary/llvm/tools/extract/BUILD.gn
+++ /dev/null
@@ -1,4 +0,0 @@
-executable("extract") {
-  deps = [ "//llvm/lib/Support" ]
-  sources = [ "extract.cpp" ]
-}

From e1dd212c874c2ff01b72e9e60db6dbded9e2e6d1 Mon Sep 17 00:00:00 2001
From: Roman Lebedev <lebedev.ri@gmail.com>
Date: Tue, 28 Jul 2020 23:35:04 +0300
Subject: [PATCH 0366/1035] [X86] Remove disabled miscompiling X86CondBrFolding
 pass

As briefly discussed in IRC with @craig.topper,
the pass is disabled basically since it's original introduction (nov 2018)
due to  known correctness issues (miscompilations),
and there hasn't been much work done to fix that.

While i won't promise that i will "fix" the pass,
i have looked at it previously, and i'm sure i won't try to fix it
if that requires actually fixing this existing code.

Reviewed By: craig.topper

Differential Revision: https://reviews.llvm.org/D84775
---
 llvm/lib/Target/X86/CMakeLists.txt            |   1 -
 llvm/lib/Target/X86/X86.h                     |   4 -
 llvm/lib/Target/X86/X86.td                    |   7 -
 llvm/lib/Target/X86/X86CondBrFolding.cpp      | 579 ------------------
 llvm/lib/Target/X86/X86Subtarget.h            |   4 -
 llvm/lib/Target/X86/X86TargetMachine.cpp      |   8 -
 llvm/lib/Target/X86/X86TargetTransformInfo.h  |   1 -
 llvm/test/CodeGen/X86/condbr_if.ll            | 178 ------
 llvm/test/CodeGen/X86/condbr_switch.ll        | 167 -----
 .../CodeGen/X86/test_x86condbr_globaladdr.mir |  30 -
 .../gn/secondary/llvm/lib/Target/X86/BUILD.gn |   1 -
 11 files changed, 980 deletions(-)
 delete mode 100644 llvm/lib/Target/X86/X86CondBrFolding.cpp
 delete mode 100644 llvm/test/CodeGen/X86/condbr_if.ll
 delete mode 100644 llvm/test/CodeGen/X86/condbr_switch.ll
 delete mode 100644 llvm/test/CodeGen/X86/test_x86condbr_globaladdr.mir

diff --git a/llvm/lib/Target/X86/CMakeLists.txt b/llvm/lib/Target/X86/CMakeLists.txt
index 2b0cbb676c129..677d0a55336a1 100644
--- a/llvm/lib/Target/X86/CMakeLists.txt
+++ b/llvm/lib/Target/X86/CMakeLists.txt
@@ -28,7 +28,6 @@ set(sources
   X86CallingConv.cpp
   X86CallLowering.cpp
   X86CmovConversion.cpp
-  X86CondBrFolding.cpp
   X86DomainReassignment.cpp
   X86DiscriminateMemOps.cpp
   X86ExpandPseudo.cpp
diff --git a/llvm/lib/Target/X86/X86.h b/llvm/lib/Target/X86/X86.h
index 91ba4e3d091e8..d4ad10d79bab2 100644
--- a/llvm/lib/Target/X86/X86.h
+++ b/llvm/lib/Target/X86/X86.h
@@ -67,9 +67,6 @@ FunctionPass *createX86OptimizeLEAs();
 /// Return a pass that transforms setcc + movzx pairs into xor + setcc.
 FunctionPass *createX86FixupSetCC();
 
-/// Return a pass that folds conditional branch jumps.
-FunctionPass *createX86CondBrFolding();
-
 /// Return a pass that avoids creating store forward block issues in the hardware.
 FunctionPass *createX86AvoidStoreForwardingBlocks();
 
@@ -154,7 +151,6 @@ void initializeX86AvoidSFBPassPass(PassRegistry &);
 void initializeX86AvoidTrailingCallPassPass(PassRegistry &);
 void initializeX86CallFrameOptimizationPass(PassRegistry &);
 void initializeX86CmovConverterPassPass(PassRegistry &);
-void initializeX86CondBrFoldingPassPass(PassRegistry &);
 void initializeX86DomainReassignmentPass(PassRegistry &);
 void initializeX86ExecutionDomainFixPass(PassRegistry &);
 void initializeX86ExpandPseudoPass(PassRegistry &);
diff --git a/llvm/lib/Target/X86/X86.td b/llvm/lib/Target/X86/X86.td
index 69d83ab1528cb..bc8927899d687 100644
--- a/llvm/lib/Target/X86/X86.td
+++ b/llvm/lib/Target/X86/X86.td
@@ -499,12 +499,6 @@ def FeatureUseGLMDivSqrtCosts
     : SubtargetFeature<"use-glm-div-sqrt-costs", "UseGLMDivSqrtCosts", "true",
         "Use Goldmont specific floating point div/sqrt costs">;
 
-// Merge branches using three-way conditional code.
-def FeatureMergeToThreeWayBranch : SubtargetFeature<"merge-to-threeway-branch",
-                                        "ThreewayBranchProfitable", "true",
-                                        "Merge branches to a three-way "
-                                        "conditional branch">;
-
 // Enable use of alias analysis during code generation.
 def FeatureUseAA : SubtargetFeature<"use-aa", "UseAA", "true",
                                     "Use alias analysis during codegen">;
@@ -586,7 +580,6 @@ def ProcessorFeatures {
                                                   FeatureSlow3OpsLEA,
                                                   FeatureFastScalarFSQRT,
                                                   FeatureFastSHLDRotate,
-                                                  FeatureMergeToThreeWayBranch,
                                                   FeatureFast15ByteNOP];
   list<SubtargetFeature> SNBSpecificFeatures = [FeatureSlowUAMem32,
                                                 FeaturePOPCNTFalseDeps];
diff --git a/llvm/lib/Target/X86/X86CondBrFolding.cpp b/llvm/lib/Target/X86/X86CondBrFolding.cpp
deleted file mode 100644
index 7ede94664bf62..0000000000000
--- a/llvm/lib/Target/X86/X86CondBrFolding.cpp
+++ /dev/null
@@ -1,579 +0,0 @@
-//===---- X86CondBrFolding.cpp - optimize conditional branches ------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-// This file defines a pass that optimizes condition branches on x86 by taking
-// advantage of the three-way conditional code generated by compare
-// instructions.
-// Currently, it tries to hoisting EQ and NE conditional branch to a dominant
-// conditional branch condition where the same EQ/NE conditional code is
-// computed. An example:
-//   bb_0:
-//     cmp %0, 19
-//     jg bb_1
-//     jmp bb_2
-//   bb_1:
-//     cmp %0, 40
-//     jg bb_3
-//     jmp bb_4
-//   bb_4:
-//     cmp %0, 20
-//     je bb_5
-//     jmp bb_6
-// Here we could combine the two compares in bb_0 and bb_4 and have the
-// following code:
-//   bb_0:
-//     cmp %0, 20
-//     jg bb_1
-//     jl bb_2
-//     jmp bb_5
-//   bb_1:
-//     cmp %0, 40
-//     jg bb_3
-//     jmp bb_6
-// For the case of %0 == 20 (bb_5), we eliminate two jumps, and the control
-// height for bb_6 is also reduced. bb_4 is gone after the optimization.
-//
-// There are plenty of this code patterns, especially from the switch case
-// lowing where we generate compare of "pivot-1" for the inner nodes in the
-// binary search tree.
-//===----------------------------------------------------------------------===//
-
-#include "X86.h"
-#include "X86InstrInfo.h"
-#include "X86Subtarget.h"
-#include "llvm/ADT/Statistic.h"
-#include "llvm/CodeGen/MachineBranchProbabilityInfo.h"
-#include "llvm/CodeGen/MachineFunctionPass.h"
-#include "llvm/CodeGen/MachineInstrBuilder.h"
-#include "llvm/CodeGen/MachineRegisterInfo.h"
-#include "llvm/Support/BranchProbability.h"
-
-using namespace llvm;
-
-#define DEBUG_TYPE "x86-condbr-folding"
-
-STATISTIC(NumFixedCondBrs, "Number of x86 condbr folded");
-
-namespace {
-class X86CondBrFoldingPass : public MachineFunctionPass {
-public:
-  X86CondBrFoldingPass() : MachineFunctionPass(ID) { }
-  StringRef getPassName() const override { return "X86 CondBr Folding"; }
-
-  bool runOnMachineFunction(MachineFunction &MF) override;
-
-  void getAnalysisUsage(AnalysisUsage &AU) const override {
-    MachineFunctionPass::getAnalysisUsage(AU);
-    AU.addRequired<MachineBranchProbabilityInfo>();
-  }
-
-public:
-  static char ID;
-};
-} // namespace
-
-char X86CondBrFoldingPass::ID = 0;
-INITIALIZE_PASS(X86CondBrFoldingPass, "X86CondBrFolding", "X86CondBrFolding", false, false)
-
-FunctionPass *llvm::createX86CondBrFolding() {
-  return new X86CondBrFoldingPass();
-}
-
-namespace {
-// A class the stores the auxiliary information for each MBB.
-struct TargetMBBInfo {
-  MachineBasicBlock *TBB;
-  MachineBasicBlock *FBB;
-  MachineInstr *BrInstr;
-  MachineInstr *CmpInstr;
-  X86::CondCode BranchCode;
-  unsigned SrcReg;
-  int CmpValue;
-  bool Modified;
-  bool CmpBrOnly;
-};
-
-// A class that optimizes the conditional branch by hoisting and merge CondCode.
-class X86CondBrFolding {
-public:
-  X86CondBrFolding(const X86InstrInfo *TII,
-                   const MachineBranchProbabilityInfo *MBPI,
-                   MachineFunction &MF)
-      : TII(TII), MBPI(MBPI), MF(MF) {}
-  bool optimize();
-
-private:
-  const X86InstrInfo *TII;
-  const MachineBranchProbabilityInfo *MBPI;
-  MachineFunction &MF;
-  std::vector<std::unique_ptr<TargetMBBInfo>> MBBInfos;
-  SmallVector<MachineBasicBlock *, 4> RemoveList;
-
-  void optimizeCondBr(MachineBasicBlock &MBB,
-                      SmallVectorImpl<MachineBasicBlock *> &BranchPath);
-  void replaceBrDest(MachineBasicBlock *MBB, MachineBasicBlock *OrigDest,
-                     MachineBasicBlock *NewDest);
-  void fixupModifiedCond(MachineBasicBlock *MBB);
-  std::unique_ptr<TargetMBBInfo> analyzeMBB(MachineBasicBlock &MBB);
-  static bool analyzeCompare(const MachineInstr &MI, unsigned &SrcReg,
-                             int &CmpValue);
-  bool findPath(MachineBasicBlock *MBB,
-                SmallVectorImpl<MachineBasicBlock *> &BranchPath);
-  TargetMBBInfo *getMBBInfo(MachineBasicBlock *MBB) const {
-    return MBBInfos[MBB->getNumber()].get();
-  }
-};
-} // namespace
-
-// Find a valid path that we can reuse the CondCode.
-// The resulted path (if return true) is stored in BranchPath.
-// Return value:
-//  false: is no valid path is found.
-//  true: a valid path is found and the targetBB can be reached.
-bool X86CondBrFolding::findPath(
-    MachineBasicBlock *MBB, SmallVectorImpl<MachineBasicBlock *> &BranchPath) {
-  TargetMBBInfo *MBBInfo = getMBBInfo(MBB);
-  assert(MBBInfo && "Expecting a candidate MBB");
-  int CmpValue = MBBInfo->CmpValue;
-
-  MachineBasicBlock *PredMBB = *MBB->pred_begin();
-  MachineBasicBlock *SaveMBB = MBB;
-  while (PredMBB) {
-    TargetMBBInfo *PredMBBInfo = getMBBInfo(PredMBB);
-    if (!PredMBBInfo || PredMBBInfo->SrcReg != MBBInfo->SrcReg)
-      return false;
-
-    assert(SaveMBB == PredMBBInfo->TBB || SaveMBB == PredMBBInfo->FBB);
-    bool IsFalseBranch = (SaveMBB == PredMBBInfo->FBB);
-
-    X86::CondCode CC = PredMBBInfo->BranchCode;
-    assert(CC == X86::COND_L || CC == X86::COND_G || CC == X86::COND_E);
-    int PredCmpValue = PredMBBInfo->CmpValue;
-    bool ValueCmpTrue = ((CmpValue < PredCmpValue && CC == X86::COND_L) ||
-                         (CmpValue > PredCmpValue && CC == X86::COND_G) ||
-                         (CmpValue == PredCmpValue && CC == X86::COND_E));
-    // Check if both the result of value compare and the branch target match.
-    if (!(ValueCmpTrue ^ IsFalseBranch)) {
-      LLVM_DEBUG(dbgs() << "Dead BB detected!\n");
-      return false;
-    }
-
-    BranchPath.push_back(PredMBB);
-    // These are the conditions on which we could combine the compares.
-    if ((CmpValue == PredCmpValue) ||
-        (CmpValue == PredCmpValue - 1 && CC == X86::COND_L) ||
-        (CmpValue == PredCmpValue + 1 && CC == X86::COND_G))
-      return true;
-
-    // If PredMBB has more than on preds, or not a pure cmp and br, we bailout.
-    if (PredMBB->pred_size() != 1 || !PredMBBInfo->CmpBrOnly)
-      return false;
-
-    SaveMBB = PredMBB;
-    PredMBB = *PredMBB->pred_begin();
-  }
-  return false;
-}
-
-// Fix up any PHI node in the successor of MBB.
-static void fixPHIsInSucc(MachineBasicBlock *MBB, MachineBasicBlock *OldMBB,
-                          MachineBasicBlock *NewMBB) {
-  if (NewMBB == OldMBB)
-    return;
-  for (auto MI = MBB->instr_begin(), ME = MBB->instr_end();
-       MI != ME && MI->isPHI(); ++MI)
-    for (unsigned i = 2, e = MI->getNumOperands() + 1; i != e; i += 2) {
-      MachineOperand &MO = MI->getOperand(i);
-      if (MO.getMBB() == OldMBB)
-        MO.setMBB(NewMBB);
-    }
-}
-
-// Utility function to set branch probability for edge MBB->SuccMBB.
-static inline bool setBranchProb(MachineBasicBlock *MBB,
-                                 MachineBasicBlock *SuccMBB,
-                                 BranchProbability Prob) {
-  auto MBBI = std::find(MBB->succ_begin(), MBB->succ_end(), SuccMBB);
-  if (MBBI == MBB->succ_end())
-    return false;
-  MBB->setSuccProbability(MBBI, Prob);
-  return true;
-}
-
-// Utility function to find the unconditional br instruction in MBB.
-static inline MachineBasicBlock::iterator
-findUncondBrI(MachineBasicBlock *MBB) {
-  return std::find_if(MBB->begin(), MBB->end(), [](MachineInstr &MI) -> bool {
-    return MI.getOpcode() == X86::JMP_1;
-  });
-}
-
-// Replace MBB's original successor, OrigDest, with NewDest.
-// Also update the MBBInfo for MBB.
-void X86CondBrFolding::replaceBrDest(MachineBasicBlock *MBB,
-                                     MachineBasicBlock *OrigDest,
-                                     MachineBasicBlock *NewDest) {
-  TargetMBBInfo *MBBInfo = getMBBInfo(MBB);
-  MachineInstr *BrMI;
-  if (MBBInfo->TBB == OrigDest) {
-    BrMI = MBBInfo->BrInstr;
-    MachineInstrBuilder MIB =
-        BuildMI(*MBB, BrMI, MBB->findDebugLoc(BrMI), TII->get(X86::JCC_1))
-            .addMBB(NewDest).addImm(MBBInfo->BranchCode);
-    MBBInfo->TBB = NewDest;
-    MBBInfo->BrInstr = MIB.getInstr();
-  } else { // Should be the unconditional jump stmt.
-    MachineBasicBlock::iterator UncondBrI = findUncondBrI(MBB);
-    BuildMI(*MBB, UncondBrI, MBB->findDebugLoc(UncondBrI), TII->get(X86::JMP_1))
-        .addMBB(NewDest);
-    MBBInfo->FBB = NewDest;
-    BrMI = &*UncondBrI;
-  }
-  fixPHIsInSucc(NewDest, OrigDest, MBB);
-  BrMI->eraseFromParent();
-  MBB->addSuccessor(NewDest);
-  setBranchProb(MBB, NewDest, MBPI->getEdgeProbability(MBB, OrigDest));
-  MBB->removeSuccessor(OrigDest);
-}
-
-// Change the CondCode and BrInstr according to MBBInfo.
-void X86CondBrFolding::fixupModifiedCond(MachineBasicBlock *MBB) {
-  TargetMBBInfo *MBBInfo = getMBBInfo(MBB);
-  if (!MBBInfo->Modified)
-    return;
-
-  MachineInstr *BrMI = MBBInfo->BrInstr;
-  X86::CondCode CC = MBBInfo->BranchCode;
-  MachineInstrBuilder MIB = BuildMI(*MBB, BrMI, MBB->findDebugLoc(BrMI),
-                                    TII->get(X86::JCC_1))
-                                .addMBB(MBBInfo->TBB).addImm(CC);
-  BrMI->eraseFromParent();
-  MBBInfo->BrInstr = MIB.getInstr();
-
-  MachineBasicBlock::iterator UncondBrI = findUncondBrI(MBB);
-  BuildMI(*MBB, UncondBrI, MBB->findDebugLoc(UncondBrI), TII->get(X86::JMP_1))
-      .addMBB(MBBInfo->FBB);
-  MBB->erase(UncondBrI);
-  MBBInfo->Modified = false;
-}
-
-//
-// Apply the transformation:
-//  RootMBB -1-> ... PredMBB -3-> MBB -5-> TargetMBB
-//     \-2->           \-4->       \-6-> FalseMBB
-// ==>
-//             RootMBB -1-> ... PredMBB -7-> FalseMBB
-// TargetMBB <-8-/ \-2->           \-4->
-//
-// Note that PredMBB and RootMBB could be the same.
-// And in the case of dead TargetMBB, we will not have TargetMBB and edge 8.
-//
-// There are some special handling where the RootMBB is COND_E in which case
-// we directly short-cycle the brinstr.
-//
-void X86CondBrFolding::optimizeCondBr(
-    MachineBasicBlock &MBB, SmallVectorImpl<MachineBasicBlock *> &BranchPath) {
-
-  X86::CondCode CC;
-  TargetMBBInfo *MBBInfo = getMBBInfo(&MBB);
-  assert(MBBInfo && "Expecting a candidate MBB");
-  MachineBasicBlock *TargetMBB = MBBInfo->TBB;
-  BranchProbability TargetProb = MBPI->getEdgeProbability(&MBB, MBBInfo->TBB);
-
-  // Forward the jump from MBB's predecessor to MBB's false target.
-  MachineBasicBlock *PredMBB = BranchPath.front();
-  TargetMBBInfo *PredMBBInfo = getMBBInfo(PredMBB);
-  assert(PredMBBInfo && "Expecting a candidate MBB");
-  if (PredMBBInfo->Modified)
-    fixupModifiedCond(PredMBB);
-  CC = PredMBBInfo->BranchCode;
-  // Don't do this if depth of BranchPath is 1 and PredMBB is of COND_E.
-  // We will short-cycle directly for this case.
-  if (!(CC == X86::COND_E && BranchPath.size() == 1))
-    replaceBrDest(PredMBB, &MBB, MBBInfo->FBB);
-
-  MachineBasicBlock *RootMBB = BranchPath.back();
-  TargetMBBInfo *RootMBBInfo = getMBBInfo(RootMBB);
-  assert(RootMBBInfo && "Expecting a candidate MBB");
-  if (RootMBBInfo->Modified)
-    fixupModifiedCond(RootMBB);
-  CC = RootMBBInfo->BranchCode;
-
-  if (CC != X86::COND_E) {
-    MachineBasicBlock::iterator UncondBrI = findUncondBrI(RootMBB);
-    // RootMBB: Cond jump to the original not-taken MBB.
-    X86::CondCode NewCC;
-    switch (CC) {
-    case X86::COND_L:
-      NewCC = X86::COND_G;
-      break;
-    case X86::COND_G:
-      NewCC = X86::COND_L;
-      break;
-    default:
-      llvm_unreachable("unexpected condtional code.");
-    }
-    BuildMI(*RootMBB, UncondBrI, RootMBB->findDebugLoc(UncondBrI),
-            TII->get(X86::JCC_1))
-        .addMBB(RootMBBInfo->FBB).addImm(NewCC);
-
-    // RootMBB: Jump to TargetMBB
-    BuildMI(*RootMBB, UncondBrI, RootMBB->findDebugLoc(UncondBrI),
-            TII->get(X86::JMP_1))
-        .addMBB(TargetMBB);
-    RootMBB->addSuccessor(TargetMBB);
-    fixPHIsInSucc(TargetMBB, &MBB, RootMBB);
-    RootMBB->erase(UncondBrI);
-  } else {
-    replaceBrDest(RootMBB, RootMBBInfo->TBB, TargetMBB);
-  }
-
-  // Fix RootMBB's CmpValue to MBB's CmpValue to TargetMBB. Don't set Imm
-  // directly. Move MBB's stmt to here as the opcode might be different.
-  if (RootMBBInfo->CmpValue != MBBInfo->CmpValue) {
-    MachineInstr *NewCmp = MBBInfo->CmpInstr;
-    NewCmp->removeFromParent();
-    RootMBB->insert(RootMBBInfo->CmpInstr, NewCmp);
-    RootMBBInfo->CmpInstr->eraseFromParent();
-  }
-
-  // Fix branch Probabilities.
-  auto fixBranchProb = [&](MachineBasicBlock *NextMBB) {
-    BranchProbability Prob;
-    for (auto &I : BranchPath) {
-      MachineBasicBlock *ThisMBB = I;
-      if (!ThisMBB->hasSuccessorProbabilities() ||
-          !ThisMBB->isSuccessor(NextMBB))
-        break;
-      Prob = MBPI->getEdgeProbability(ThisMBB, NextMBB);
-      if (Prob.isUnknown())
-        break;
-      TargetProb = Prob * TargetProb;
-      Prob = Prob - TargetProb;
-      setBranchProb(ThisMBB, NextMBB, Prob);
-      if (ThisMBB == RootMBB) {
-        setBranchProb(ThisMBB, TargetMBB, TargetProb);
-      }
-      ThisMBB->normalizeSuccProbs();
-      if (ThisMBB == RootMBB)
-        break;
-      NextMBB = ThisMBB;
-    }
-    return true;
-  };
-  if (CC != X86::COND_E && !TargetProb.isUnknown())
-    fixBranchProb(MBBInfo->FBB);
-
-  if (CC != X86::COND_E)
-    RemoveList.push_back(&MBB);
-
-  // Invalidate MBBInfo just in case.
-  MBBInfos[MBB.getNumber()] = nullptr;
-  MBBInfos[RootMBB->getNumber()] = nullptr;
-
-  LLVM_DEBUG(dbgs() << "After optimization:\nRootMBB is: " << *RootMBB << "\n");
-  if (BranchPath.size() > 1)
-    LLVM_DEBUG(dbgs() << "PredMBB is: " << *(BranchPath[0]) << "\n");
-}
-
-// Driver function for optimization: find the valid candidate and apply
-// the transformation.
-bool X86CondBrFolding::optimize() {
-  bool Changed = false;
-  LLVM_DEBUG(dbgs() << "***** X86CondBr Folding on Function: " << MF.getName()
-                    << " *****\n");
-  // Setup data structures.
-  MBBInfos.resize(MF.getNumBlockIDs());
-  for (auto &MBB : MF)
-    MBBInfos[MBB.getNumber()] = analyzeMBB(MBB);
-
-  for (auto &MBB : MF) {
-    TargetMBBInfo *MBBInfo = getMBBInfo(&MBB);
-    if (!MBBInfo || !MBBInfo->CmpBrOnly)
-      continue;
-    if (MBB.pred_size() != 1)
-      continue;
-    LLVM_DEBUG(dbgs() << "Work on MBB." << MBB.getNumber()
-                      << " CmpValue: " << MBBInfo->CmpValue << "\n");
-    SmallVector<MachineBasicBlock *, 4> BranchPath;
-    if (!findPath(&MBB, BranchPath))
-      continue;
-
-#ifndef NDEBUG
-    LLVM_DEBUG(dbgs() << "Found one path (len=" << BranchPath.size() << "):\n");
-    int Index = 1;
-    LLVM_DEBUG(dbgs() << "Target MBB is: " << MBB << "\n");
-    for (auto I = BranchPath.rbegin(); I != BranchPath.rend(); ++I, ++Index) {
-      MachineBasicBlock *PMBB = *I;
-      TargetMBBInfo *PMBBInfo = getMBBInfo(PMBB);
-      LLVM_DEBUG(dbgs() << "Path MBB (" << Index << " of " << BranchPath.size()
-                        << ") is " << *PMBB);
-      LLVM_DEBUG(dbgs() << "CC=" << PMBBInfo->BranchCode
-                        << "  Val=" << PMBBInfo->CmpValue
-                        << "  CmpBrOnly=" << PMBBInfo->CmpBrOnly << "\n\n");
-    }
-#endif
-    optimizeCondBr(MBB, BranchPath);
-    Changed = true;
-  }
-  NumFixedCondBrs += RemoveList.size();
-  for (auto MBBI : RemoveList) {
-    while (!MBBI->succ_empty())
-      MBBI->removeSuccessor(MBBI->succ_end() - 1);
-
-    MBBI->eraseFromParent();
-  }
-
-  return Changed;
-}
-
-// Analyze instructions that generate CondCode and extract information.
-bool X86CondBrFolding::analyzeCompare(const MachineInstr &MI, unsigned &SrcReg,
-                                      int &CmpValue) {
-  unsigned SrcRegIndex = 0;
-  unsigned ValueIndex = 0;
-  switch (MI.getOpcode()) {
-  // TODO: handle test instructions.
-  default:
-    return false;
-  case X86::CMP64ri32:
-  case X86::CMP64ri8:
-  case X86::CMP32ri:
-  case X86::CMP32ri8:
-  case X86::CMP16ri:
-  case X86::CMP16ri8:
-  case X86::CMP8ri:
-    SrcRegIndex = 0;
-    ValueIndex = 1;
-    break;
-  case X86::SUB64ri32:
-  case X86::SUB64ri8:
-  case X86::SUB32ri:
-  case X86::SUB32ri8:
-  case X86::SUB16ri:
-  case X86::SUB16ri8:
-  case X86::SUB8ri:
-    SrcRegIndex = 1;
-    ValueIndex = 2;
-    break;
-  }
-  SrcReg = MI.getOperand(SrcRegIndex).getReg();
-  if (!MI.getOperand(ValueIndex).isImm())
-    return false;
-  CmpValue = MI.getOperand(ValueIndex).getImm();
-  return true;
-}
-
-// Analyze a candidate MBB and set the extract all the information needed.
-// The valid candidate will have two successors.
-// It also should have a sequence of
-//  Branch_instr,
-//  CondBr,
-//  UnCondBr.
-// Return TargetMBBInfo if MBB is a valid candidate and nullptr otherwise.
-std::unique_ptr<TargetMBBInfo>
-X86CondBrFolding::analyzeMBB(MachineBasicBlock &MBB) {
-  MachineBasicBlock *TBB;
-  MachineBasicBlock *FBB;
-  MachineInstr *BrInstr;
-  MachineInstr *CmpInstr;
-  X86::CondCode CC;
-  unsigned SrcReg;
-  int CmpValue;
-  bool Modified;
-  bool CmpBrOnly;
-
-  if (MBB.succ_size() != 2)
-    return nullptr;
-
-  CmpBrOnly = true;
-  FBB = TBB = nullptr;
-  CmpInstr = nullptr;
-  MachineBasicBlock::iterator I = MBB.end();
-  while (I != MBB.begin()) {
-    --I;
-    if (I->isDebugValue())
-      continue;
-    if (I->getOpcode() == X86::JMP_1) {
-      if (FBB)
-        return nullptr;
-      FBB = I->getOperand(0).getMBB();
-      continue;
-    }
-    if (I->isBranch()) {
-      if (TBB)
-        return nullptr;
-      CC = X86::getCondFromBranch(*I);
-      switch (CC) {
-      default:
-        return nullptr;
-      case X86::COND_E:
-      case X86::COND_L:
-      case X86::COND_G:
-      case X86::COND_NE:
-      case X86::COND_LE:
-      case X86::COND_GE:
-        break;
-      }
-      TBB = I->getOperand(0).getMBB();
-      BrInstr = &*I;
-      continue;
-    }
-    if (analyzeCompare(*I, SrcReg, CmpValue)) {
-      if (CmpInstr)
-        return nullptr;
-      CmpInstr = &*I;
-      continue;
-    }
-    CmpBrOnly = false;
-    break;
-  }
-
-  if (!TBB || !FBB || !CmpInstr)
-    return nullptr;
-
-  // Simplify CondCode. Note this is only to simplify the findPath logic
-  // and will not change the instruction here.
-  switch (CC) {
-  case X86::COND_NE:
-    CC = X86::COND_E;
-    std::swap(TBB, FBB);
-    Modified = true;
-    break;
-  case X86::COND_LE:
-    if (CmpValue == INT_MAX)
-      return nullptr;
-    CC = X86::COND_L;
-    CmpValue += 1;
-    Modified = true;
-    break;
-  case X86::COND_GE:
-    if (CmpValue == INT_MIN)
-      return nullptr;
-    CC = X86::COND_G;
-    CmpValue -= 1;
-    Modified = true;
-    break;
-  default:
-    Modified = false;
-    break;
-  }
-  return std::make_unique<TargetMBBInfo>(TargetMBBInfo{
-      TBB, FBB, BrInstr, CmpInstr, CC, SrcReg, CmpValue, Modified, CmpBrOnly});
-}
-
-bool X86CondBrFoldingPass::runOnMachineFunction(MachineFunction &MF) {
-  const X86Subtarget &ST = MF.getSubtarget<X86Subtarget>();
-  if (!ST.threewayBranchProfitable())
-    return false;
-  const X86InstrInfo *TII = ST.getInstrInfo();
-  const MachineBranchProbabilityInfo *MBPI =
-      &getAnalysis<MachineBranchProbabilityInfo>();
-
-  X86CondBrFolding CondBr(TII, MBPI, MF);
-  return CondBr.optimize();
-}
diff --git a/llvm/lib/Target/X86/X86Subtarget.h b/llvm/lib/Target/X86/X86Subtarget.h
index e555dfdd638ac..c93fa082eeb60 100644
--- a/llvm/lib/Target/X86/X86Subtarget.h
+++ b/llvm/lib/Target/X86/X86Subtarget.h
@@ -468,9 +468,6 @@ class X86Subtarget final : public X86GenSubtargetInfo {
   /// Indicates target prefers AVX512 mask registers.
   bool PreferMaskRegisters = false;
 
-  /// Threeway branch is profitable in this subtarget.
-  bool ThreewayBranchProfitable = false;
-
   /// Use Goldmont specific floating point div/sqrt costs.
   bool UseGLMDivSqrtCosts = false;
 
@@ -723,7 +720,6 @@ class X86Subtarget final : public X86GenSubtargetInfo {
   bool hasWAITPKG() const { return HasWAITPKG; }
   bool hasPCONFIG() const { return HasPCONFIG; }
   bool hasSGX() const { return HasSGX; }
-  bool threewayBranchProfitable() const { return ThreewayBranchProfitable; }
   bool hasINVPCID() const { return HasINVPCID; }
   bool hasENQCMD() const { return HasENQCMD; }
   bool hasSERIALIZE() const { return HasSERIALIZE; }
diff --git a/llvm/lib/Target/X86/X86TargetMachine.cpp b/llvm/lib/Target/X86/X86TargetMachine.cpp
index 9a9ea245f7027..f660b99a4511d 100644
--- a/llvm/lib/Target/X86/X86TargetMachine.cpp
+++ b/llvm/lib/Target/X86/X86TargetMachine.cpp
@@ -56,11 +56,6 @@ static cl::opt<bool> EnableMachineCombinerPass("x86-machine-combiner",
                                cl::desc("Enable the machine combiner pass"),
                                cl::init(true), cl::Hidden);
 
-static cl::opt<bool> EnableCondBrFoldingPass("x86-condbr-folding",
-                               cl::desc("Enable the conditional branch "
-                                        "folding pass"),
-                               cl::init(false), cl::Hidden);
-
 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeX86Target() {
   // Register the target.
   RegisterTargetMachine<X86TargetMachine> X(getTheX86_32Target());
@@ -84,7 +79,6 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeX86Target() {
   initializeX86SpeculativeLoadHardeningPassPass(PR);
   initializeX86SpeculativeExecutionSideEffectSuppressionPass(PR);
   initializeX86FlagsCopyLoweringPassPass(PR);
-  initializeX86CondBrFoldingPassPass(PR);
   initializeX86LoadValueInjectionLoadHardeningPassPass(PR);
   initializeX86LoadValueInjectionRetHardeningPassPass(PR);
   initializeX86OptimizeLEAPassPass(PR);
@@ -456,8 +450,6 @@ bool X86PassConfig::addGlobalInstructionSelect() {
 }
 
 bool X86PassConfig::addILPOpts() {
-  if (EnableCondBrFoldingPass)
-    addPass(createX86CondBrFolding());
   addPass(&EarlyIfConverterID);
   if (EnableMachineCombinerPass)
     addPass(&MachineCombinerID);
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.h b/llvm/lib/Target/X86/X86TargetTransformInfo.h
index 5ccaf409790db..fb4719064613e 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.h
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.h
@@ -62,7 +62,6 @@ class X86TTIImpl : public BasicTTIImplBase<X86TTIImpl> {
       X86::FeatureLZCNTFalseDeps,
       X86::FeatureBranchFusion,
       X86::FeatureMacroFusion,
-      X86::FeatureMergeToThreeWayBranch,
       X86::FeaturePadShortFunctions,
       X86::FeaturePOPCNTFalseDeps,
       X86::FeatureSSEUnalignedMem,
diff --git a/llvm/test/CodeGen/X86/condbr_if.ll b/llvm/test/CodeGen/X86/condbr_if.ll
deleted file mode 100644
index 7b92f712be6dd..0000000000000
--- a/llvm/test/CodeGen/X86/condbr_if.ll
+++ /dev/null
@@ -1,178 +0,0 @@
-; RUN: llc -x86-condbr-folding=true -mtriple=x86_64-linux-gnu -mcpu=sandybridge %s -o - -verify-machineinstrs | FileCheck %s --check-prefix=MERGE
-; RUN: llc -x86-condbr-folding=true -mtriple=x86_64-linux-gnu -mcpu=ivybridge %s -o - -verify-machineinstrs | FileCheck %s --check-prefix=MERGE
-; RUN: llc -x86-condbr-folding=true -mtriple=x86_64-linux-gnu -mcpu=haswell %s -o - -verify-machineinstrs | FileCheck %s --check-prefix=MERGE
-; RUN: llc -x86-condbr-folding=true -mtriple=x86_64-linux-gnu -mcpu=broadwell %s -o - -verify-machineinstrs | FileCheck %s --check-prefix=MERGE
-; RUN: llc -x86-condbr-folding=true -mtriple=x86_64-linux-gnu -mcpu=skylake %s -o - -verify-machineinstrs | FileCheck %s --check-prefix=MERGE
-; RUN: llc -x86-condbr-folding=true -mtriple=x86_64-linux-gnu -mcpu=skx %s -o - -verify-machineinstrs | FileCheck %s --check-prefix=MERGE
-; RUN: llc -x86-condbr-folding=true -mtriple=x86_64-linux-gnu %s -o - -verify-machineinstrs | FileCheck %s --check-prefix=NOTMERGE
-
-define i32 @length2_1(i32) {
-  %2 = icmp slt i32 %0, 3
-  br i1 %2, label %3, label %5
-
-; <label>:3:
-  %4 = tail call i32 (...) @f1()
-  br label %13
-
-; <label>:5:
-  %6 = icmp slt i32 %0, 40
-  br i1 %6, label %7, label %13
-
-; <label>:7:
-  %8 = icmp eq i32 %0, 3
-  br i1 %8, label %9, label %11
-
-; <label>:9:
-  %10 = tail call i32 (...) @f2()
-  br label %11
-
-; <label>:11:
-  %12 = tail call i32 (...) @f3() #2
-  br label %13
-
-; <label>:13:
-  ret i32 0
-}
-; MERGE-LABEL: length2_1
-; MERGE: cmpl $3
-; MERGE-NEXT: jg
-; MERGE-NEXT: jge
-; NOTMERGE-LABEL: length2_1
-; NOTMERGE: cmpl $2
-; NOTMERGE-NEXT: jg
-
-define i32 @length2_2(i32) {
-  %2 = icmp sle i32 %0, 2
-  br i1 %2, label %3, label %5
-
-; <label>:3:
-  %4 = tail call i32 (...) @f1()
-  br label %13
-
-; <label>:5:
-  %6 = icmp slt i32 %0, 40
-  br i1 %6, label %7, label %13
-
-; <label>:7:
-  %8 = icmp eq i32 %0, 3
-  br i1 %8, label %9, label %11
-
-; <label>:9:
-  %10 = tail call i32 (...) @f2()
-  br label %11
-
-; <label>:11:
-  %12 = tail call i32 (...) @f3() #2
-  br label %13
-
-; <label>:13:
-  ret i32 0
-}
-; MERGE-LABEL: length2_2
-; MERGE: cmpl $3
-; MERGE-NEXT: jg
-; MERGE-NEXT: jge
-; NOTMERGE-LABEL: length2_2
-; NOTMERGE: cmpl $2
-; NOTMERGE-NEXT: jg
-
-define i32 @length2_3(i32) {
-  %2 = icmp sgt i32 %0, 3
-  br i1 %2, label %3, label %5
-
-; <label>:3:
-  %4 = tail call i32 (...) @f1()
-  br label %13
-
-; <label>:5:
-  %6 = icmp sgt i32 %0, -40
-  br i1 %6, label %7, label %13
-
-; <label>:7:
-  %8 = icmp eq i32 %0, 3
-  br i1 %8, label %9, label %11
-
-; <label>:9:
-  %10 = tail call i32 (...) @f2()
-  br label %11
-
-; <label>:11:
-  %12 = tail call i32 (...) @f3() #2
-  br label %13
-
-; <label>:13:
-  ret i32 0
-}
-; MERGE-LABEL: length2_3
-; MERGE: cmpl $3
-; MERGE-NEXT: jl
-; MERGE-NEXT: jle
-; NOTMERGE-LABEL: length2_3
-; NOTMERGE: cmpl $4
-; NOTMERGE-NEXT: jl
-
-define i32 @length2_4(i32) {
-  %2 = icmp sge i32 %0, 4
-  br i1 %2, label %3, label %5
-
-; <label>:3:
-  %4 = tail call i32 (...) @f1()
-  br label %13
-
-; <label>:5:
-  %6 = icmp sgt i32 %0, -40
-  br i1 %6, label %7, label %13
-
-; <label>:7:
-  %8 = icmp eq i32 %0, 3
-  br i1 %8, label %9, label %11
-
-; <label>:9:
-  %10 = tail call i32 (...) @f2()
-  br label %11
-
-; <label>:11:
-  %12 = tail call i32 (...) @f3() #2
-  br label %13
-
-; <label>:13:
-  ret i32 0
-}
-; MERGE-LABEL: length2_4
-; MERGE: cmpl $3
-; MERGE-NEXT: jl
-; MERGE-NEXT: jle
-; NOTMERGE-LABEL: length2_4
-; NOTMERGE: cmpl $4
-; NOTMERGE-NEXT: jl
-
-declare i32 @f1(...)
-declare i32 @f2(...)
-declare i32 @f3(...)
-
-define i32 @length1_1(i32) {
-  %2 = icmp sgt i32 %0, 5
-  br i1 %2, label %3, label %5
-
-; <label>:3:
-  %4 = tail call i32 (...) @f1()
-  br label %9
-
-; <label>:5:
-  %6 = icmp eq i32 %0, 5
-  br i1 %6, label %7, label %9
-
-; <label>:7:
-  %8 = tail call i32 (...) @f2()
-  br label %9
-
-; <label>:9:
-  ret i32 0
-}
-; MERGE-LABEL: length1_1
-; MERGE: cmpl $5
-; MERGE-NEXT: jl
-; MERGE-NEXT: jle
-; NOTMERGE-LABEL: length1_1
-; NOTMERGE: cmpl $6
-; NOTMERGE-NEXT: jl
diff --git a/llvm/test/CodeGen/X86/condbr_switch.ll b/llvm/test/CodeGen/X86/condbr_switch.ll
deleted file mode 100644
index 079c0e2ab2026..0000000000000
--- a/llvm/test/CodeGen/X86/condbr_switch.ll
+++ /dev/null
@@ -1,167 +0,0 @@
-; RUN: llc -x86-condbr-folding=true -mtriple=x86_64-linux-gnu -mcpu=sandybridge %s -o - -verify-machineinstrs | FileCheck %s --check-prefix=MERGE
-; RUN: llc -x86-condbr-folding=true -mtriple=x86_64-linux-gnu -mcpu=ivybridge %s -o - -verify-machineinstrs | FileCheck %s --check-prefix=MERGE
-; RUN: llc -x86-condbr-folding=true -mtriple=x86_64-linux-gnu -mcpu=haswell %s -o - -verify-machineinstrs | FileCheck %s --check-prefix=MERGE
-; RUN: llc -x86-condbr-folding=true -mtriple=x86_64-linux-gnu -mcpu=broadwell %s -o - -verify-machineinstrs | FileCheck %s --check-prefix=MERGE
-; RUN: llc -x86-condbr-folding=true -mtriple=x86_64-linux-gnu -mcpu=skylake %s -o - -verify-machineinstrs | FileCheck %s --check-prefix=MERGE
-; RUN: llc -x86-condbr-folding=true -mtriple=x86_64-linux-gnu -mcpu=skx %s -o - -verify-machineinstrs | FileCheck %s --check-prefix=MERGE
-; RUN: llc -x86-condbr-folding=true -mtriple=x86_64-linux-gnu %s -o - -verify-machineinstrs | FileCheck %s --check-prefix=NOTMERGE
-
-@v1 = common dso_local local_unnamed_addr global i32 0, align 4
-@v2 = common dso_local local_unnamed_addr global i32 0, align 4
-@v3 = common dso_local local_unnamed_addr global i32 0, align 4
-@v4 = common dso_local local_unnamed_addr global i32 0, align 4
-@v5 = common dso_local local_unnamed_addr global i32 0, align 4
-@v6 = common dso_local local_unnamed_addr global i32 0, align 4
-@v7 = common dso_local local_unnamed_addr global i32 0, align 4
-@v8 = common dso_local local_unnamed_addr global i32 0, align 4
-@v9 = common dso_local local_unnamed_addr global i32 0, align 4
-@v10 = common dso_local local_unnamed_addr global i32 0, align 4
-@v11 = common dso_local local_unnamed_addr global i32 0, align 4
-@v12 = common dso_local local_unnamed_addr global i32 0, align 4
-@v13 = common dso_local local_unnamed_addr global i32 0, align 4
-@v14 = common dso_local local_unnamed_addr global i32 0, align 4
-@v15 = common dso_local local_unnamed_addr global i32 0, align 4
-
-define dso_local i32 @fourcases(i32 %n) {
-entry:
-  switch i32 %n, label %return [
-    i32 111, label %sw.bb
-    i32 222, label %sw.bb1
-    i32 3665, label %sw.bb2
-    i32 4444, label %sw.bb4
-  ]
-
-sw.bb:
-  %0 = load i32, i32* @v1, align 4
-  br label %return
-
-sw.bb1:
-  %1 = load i32, i32* @v2, align 4
-  %add = add nsw i32 %1, 12
-  br label %return
-
-sw.bb2:
-  %2 = load i32, i32* @v3, align 4
-  %add3 = add nsw i32 %2, 13
-  br label %return
-
-sw.bb4:
-  %3 = load i32, i32* @v1, align 4
-  %4 = load i32, i32* @v2, align 4
-  %add5 = add nsw i32 %4, %3
-  br label %return
-
-return:
-  %retval.0 = phi i32 [ %add5, %sw.bb4 ], [ %add3, %sw.bb2 ], [ %add, %sw.bb1 ], [ %0, %sw.bb ], [ 0, %entry ]
-  ret i32 %retval.0
-}
-; MERGE-LABEL: fourcases
-; MERGE: cmpl $3665
-; MERGE-NEXT: jg
-; MERGE-NEXT: jge
-; NOTMERGE: cmpl $3664
-; NOTMERGE-NEXT: jg
-
-define dso_local i32 @fifteencases(i32) {
-  switch i32 %0, label %32 [
-    i32 -111, label %2
-    i32 -13, label %4
-    i32 25, label %6
-    i32 37, label %8
-    i32 89, label %10
-    i32 111, label %12
-    i32 213, label %14
-    i32 271, label %16
-    i32 283, label %18
-    i32 325, label %20
-    i32 327, label %22
-    i32 429, label %24
-    i32 500, label %26
-    i32 603, label %28
-    i32 605, label %30
-  ]
-
-; <label>:2
-  %3 = load i32, i32* @v1, align 4
-  br label %32
-
-; <label>:4
-  %5 = load i32, i32* @v2, align 4
-  br label %32
-
-; <label>:6
-  %7 = load i32, i32* @v3, align 4
-  br label %32
-
-; <label>:8
-  %9 = load i32, i32* @v4, align 4
-  br label %32
-
-; <label>:10
-  %11 = load i32, i32* @v5, align 4
-  br label %32
-
-; <label>:12
-  %13 = load i32, i32* @v6, align 4
-  br label %32
-
-; <label>:14
-  %15 = load i32, i32* @v7, align 4
-  br label %32
-
-; <label>:16
-  %17 = load i32, i32* @v8, align 4
-  br label %32
-
-; <label>:18
-  %19 = load i32, i32* @v9, align 4
-  br label %32
-
-; <label>:20
-  %21 = load i32, i32* @v10, align 4
-  br label %32
-
-; <label>:22
-  %23 = load i32, i32* @v11, align 4
-  br label %32
-
-; <label>:24
-  %25 = load i32, i32* @v12, align 4
-  br label %32
-
-; <label>:26
-  %27 = load i32, i32* @v13, align 4
-  br label %32
-
-; <label>:28:
-  %29 = load i32, i32* @v14, align 4
-  br label %32
-
-; <label>:30:
-  %31 = load i32, i32* @v15, align 4
-  br label %32
-
-; <label>:32:
-  %33 = phi i32 [ %31, %30 ], [ %29, %28 ], [ %27, %26 ], [ %25, %24 ], [ %23, %22 ], [ %21, %20 ], [ %19, %18 ], [ %17, %16 ], [ %15, %14 ], [ %13, %12 ], [ %11, %10 ], [ %9, %8 ], [ %7, %6 ], [ %5, %4 ], [ %3, %2 ], [ 0, %1 ]
-  ret i32 %33
-}
-; MERGE-LABEL: fifteencases
-; MERGE: cmpl $271
-; MERGE-NEXT: jg
-; MERGE-NEXT: jge
-; MERGE: cmpl $37
-; MERGE-NEXT: jg
-; MERGE-NEXT: jge
-; MERGE: cmpl $429
-; MERGE-NEXT: jg
-; MERGE-NEXT: jge
-; MERGE: cmpl $325
-; MERGE-NEXT: jg
-; MERGE-NEXT: jge
-; MERGE: cmpl $603
-; MERGE-NEXT: jg
-; MERGE-NEXT: jge
-; NOTMERGE-LABEL: fifteencases
-; NOTMERGE: cmpl $270
-; NOTMERGE-NEXT: jle
-
diff --git a/llvm/test/CodeGen/X86/test_x86condbr_globaladdr.mir b/llvm/test/CodeGen/X86/test_x86condbr_globaladdr.mir
deleted file mode 100644
index 4abe7d5e048c6..0000000000000
--- a/llvm/test/CodeGen/X86/test_x86condbr_globaladdr.mir
+++ /dev/null
@@ -1,30 +0,0 @@
-# RUN: llc -o - %s -mtriple=i686-- -mcpu=ivybridge --run-pass X86CondBrFolding | FileCheck %s
-
-# Test wrong assertion when meet SUB32ri with global address
-# in X86CondBrFoldingiPass
---- |
-  @img2buf_normal = external global i32
-  define void @func() { ret void }
-...
----
-# CHECK: bb.0:
-# CHECK: %2:gr32 = SUB32ri %1, @img2buf_normal, implicit-def $eflags
-
-name: func
-tracksRegLiveness: true
-body: |
-  bb.0:
-    liveins: $edx
-    %1:gr32 = COPY $edx
-    %2:gr32 = MOV32rm %1:gr32, 1, $noreg, 850256, $noreg
-    %3:gr32 = SUB32ri %2:gr32, @img2buf_normal, implicit-def $eflags
-    JCC_1 %bb.2, 4, implicit $eflags
-    JMP_1 %bb.3
-
-  bb.2:
-    RET 0, undef $eax
-
-  bb.3:
-    $eax = MOV32rr %3:gr32
-    RET 0, $eax
-...
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/X86/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/X86/BUILD.gn
index 80098c996f870..66e67af338bde 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/X86/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/X86/BUILD.gn
@@ -82,7 +82,6 @@ static_library("LLVMX86CodeGen") {
     "X86CallLowering.cpp",
     "X86CallingConv.cpp",
     "X86CmovConversion.cpp",
-    "X86CondBrFolding.cpp",
     "X86DiscriminateMemOps.cpp",
     "X86DomainReassignment.cpp",
     "X86EvexToVex.cpp",

From e9b236f411c5683d270a381bf810ba3c8f3ed12c Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Fri, 24 Jul 2020 20:13:04 -0400
Subject: [PATCH 0367/1035] AMDGPU: Check for other defs when folding
 conditions into s_andn2_b64

We can't fold the masked compare value through the select if the
select condition is re-defed after the and instruction. Fixes a
verifier error and trying to use the outgoing value defined in the
block.

I'm not sure why this pass is bothering to handle physregs. It's
making this more complex and forces extra liveness computation.
---
 .../AMDGPU/SIOptimizeExecMaskingPreRA.cpp     |  34 ++-
 .../optimize-exec-mask-pre-ra-loop-phi.mir    | 201 ++++++++++++++++++
 2 files changed, 234 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/CodeGen/AMDGPU/optimize-exec-mask-pre-ra-loop-phi.mir

diff --git a/llvm/lib/Target/AMDGPU/SIOptimizeExecMaskingPreRA.cpp b/llvm/lib/Target/AMDGPU/SIOptimizeExecMaskingPreRA.cpp
index 8af00fcf62a82..74546befbb596 100644
--- a/llvm/lib/Target/AMDGPU/SIOptimizeExecMaskingPreRA.cpp
+++ b/llvm/lib/Target/AMDGPU/SIOptimizeExecMaskingPreRA.cpp
@@ -77,6 +77,32 @@ static bool isFullExecCopy(const MachineInstr& MI, const GCNSubtarget& ST) {
   return false;
 }
 
+// See if there is a def between \p AndIdx and \p SelIdx that needs to live
+// beyond \p AndIdx.
+static bool isDefBetween(const LiveRange &LR, SlotIndex AndIdx,
+                         SlotIndex SelIdx) {
+  LiveQueryResult AndLRQ = LR.Query(AndIdx);
+  return (!AndLRQ.isKill() && AndLRQ.valueIn() != LR.Query(SelIdx).valueOut());
+}
+
+// FIXME: Why do we bother trying to handle physical registers here?
+static bool isDefBetween(const SIRegisterInfo &TRI,
+                         LiveIntervals *LIS, Register Reg,
+                         const MachineInstr &Sel, const MachineInstr &And) {
+  SlotIndex AndIdx = LIS->getInstructionIndex(And);
+  SlotIndex SelIdx = LIS->getInstructionIndex(Sel);
+
+  if (Reg.isVirtual())
+    return isDefBetween(LIS->getInterval(Reg), AndIdx, SelIdx);
+
+  for (MCRegUnitIterator UI(Reg, &TRI); UI.isValid(); ++UI) {
+    if (isDefBetween(LIS->getRegUnit(*UI), AndIdx, SelIdx))
+      return true;
+  }
+
+  return false;
+}
+
 // Optimize sequence
 //    %sel = V_CNDMASK_B32_e64 0, 1, %cc
 //    %cmp = V_CMP_NE_U32 1, %1
@@ -158,10 +184,16 @@ static unsigned optimizeVcndVcmpPair(MachineBasicBlock &MBB,
       Op1->getImm() != 0 || Op2->getImm() != 1)
     return AMDGPU::NoRegister;
 
+  Register CCReg = CC->getReg();
+
+  // If there was a def between the select and the and, we would need to move it
+  // to fold this.
+  if (isDefBetween(*TRI, LIS, CCReg, *Sel, *And))
+    return AMDGPU::NoRegister;
+
   LLVM_DEBUG(dbgs() << "Folding sequence:\n\t" << *Sel << '\t' << *Cmp << '\t'
                     << *And);
 
-  Register CCReg = CC->getReg();
   LIS->RemoveMachineInstrFromMaps(*And);
   MachineInstr *Andn2 =
       BuildMI(MBB, *And, And->getDebugLoc(), TII->get(Andn2Opc),
diff --git a/llvm/test/CodeGen/AMDGPU/optimize-exec-mask-pre-ra-loop-phi.mir b/llvm/test/CodeGen/AMDGPU/optimize-exec-mask-pre-ra-loop-phi.mir
new file mode 100644
index 0000000000000..0c62d666e7296
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/optimize-exec-mask-pre-ra-loop-phi.mir
@@ -0,0 +1,201 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx906 -verify-machineinstrs -run-pass=si-optimize-exec-masking-pre-ra,si-optimize-exec-masking-pre-ra -o - %s | FileCheck %s
+
+# FIXME: Second run of the pass is a workaround for a bug in
+# -run-pass. The verifier doesn't detect broken LiveIntervals, see bug
+# 46873
+
+
+# Cannot fold this without moving the def of %7 after the and.
+---
+name:            no_fold_andn2_select_condition_live_out_phi
+tracksRegLiveness: true
+body:             |
+  ; CHECK-LABEL: name: no_fold_andn2_select_condition_live_out_phi
+  ; CHECK: bb.0:
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 -1
+  ; CHECK:   undef %1.sub0:vreg_64 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK:   S_BRANCH %bb.2
+  ; CHECK: bb.1:
+  ; CHECK:   S_ENDPGM 0
+  ; CHECK: bb.2:
+  ; CHECK:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, [[S_MOV_B64_]], implicit $exec
+  ; CHECK:   V_CMP_NE_U32_e32 1, [[V_CNDMASK_B32_e64_]], implicit-def $vcc, implicit $exec
+  ; CHECK:   %1.sub1:vreg_64 = COPY %1.sub0
+  ; CHECK:   DS_WRITE_B64_gfx9 undef %3:vgpr_32, %1, 0, 0, implicit $exec :: (store 8, addrspace 3)
+  ; CHECK:   ATOMIC_FENCE 4, 2
+  ; CHECK:   [[S_MOV_B64_1:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 0
+  ; CHECK:   $vcc = S_AND_B64 $exec, $vcc, implicit-def dead $scc
+  ; CHECK:   S_CBRANCH_VCCNZ %bb.1, implicit $vcc
+  ; CHECK:   S_BRANCH %bb.2
+  bb.0:
+    successors: %bb.2
+
+    %7:sreg_64_xexec = S_MOV_B64 -1
+    undef %5.sub0:vreg_64 = V_MOV_B32_e32 0, implicit $exec
+    S_BRANCH %bb.2
+
+  bb.1:
+    S_ENDPGM 0
+
+  bb.2:
+    successors: %bb.1, %bb.2
+
+    %4:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, %7, implicit $exec
+    V_CMP_NE_U32_e32 1, %4, implicit-def $vcc, implicit $exec
+    %5.sub1:vreg_64 = COPY %5.sub0
+    DS_WRITE_B64_gfx9 undef %6:vgpr_32, %5, 0, 0, implicit $exec :: (store 8, addrspace 3)
+    ATOMIC_FENCE 4, 2
+    %7:sreg_64_xexec = S_MOV_B64 0
+    $vcc = S_AND_B64 $exec, killed $vcc, implicit-def dead $scc
+    S_CBRANCH_VCCNZ %bb.1, implicit killed $vcc
+    S_BRANCH %bb.2
+
+...
+
+# It's OK to fold this, since the phi def is after the andn2 insert point.
+---
+name:            fold_andn2_select_condition_live_out_phi_reorder
+tracksRegLiveness: true
+body:             |
+  ; CHECK-LABEL: name: fold_andn2_select_condition_live_out_phi_reorder
+  ; CHECK: bb.0:
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 -1
+  ; CHECK:   undef %1.sub0:vreg_64 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK:   S_BRANCH %bb.2
+  ; CHECK: bb.1:
+  ; CHECK:   S_ENDPGM 0
+  ; CHECK: bb.2:
+  ; CHECK:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   %1.sub1:vreg_64 = COPY %1.sub0
+  ; CHECK:   DS_WRITE_B64_gfx9 undef %3:vgpr_32, %1, 0, 0, implicit $exec :: (store 8, addrspace 3)
+  ; CHECK:   ATOMIC_FENCE 4, 2
+  ; CHECK:   $vcc = S_ANDN2_B64 $exec, [[S_MOV_B64_]], implicit-def dead $scc
+  ; CHECK:   [[S_MOV_B64_1:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 0
+  ; CHECK:   S_CBRANCH_VCCNZ %bb.1, implicit $vcc
+  ; CHECK:   S_BRANCH %bb.2
+  bb.0:
+    successors: %bb.2
+
+    %7:sreg_64_xexec = S_MOV_B64 -1
+    undef %5.sub0:vreg_64 = V_MOV_B32_e32 0, implicit $exec
+    S_BRANCH %bb.2
+
+  bb.1:
+    S_ENDPGM 0
+
+  bb.2:
+    successors: %bb.1, %bb.2
+
+    %4:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, %7, implicit $exec
+    V_CMP_NE_U32_e32 1, %4, implicit-def $vcc, implicit $exec
+    %5.sub1:vreg_64 = COPY %5.sub0
+    DS_WRITE_B64_gfx9 undef %6:vgpr_32, %5, 0, 0, implicit $exec :: (store 8, addrspace 3)
+    ATOMIC_FENCE 4, 2
+    $vcc = S_AND_B64 $exec, killed $vcc, implicit-def dead $scc
+    %7:sreg_64_xexec = S_MOV_B64 0
+    S_CBRANCH_VCCNZ %bb.1, implicit killed $vcc
+    S_BRANCH %bb.2
+
+...
+
+---
+name:            no_fold_andn2_select_condition_live_out_phi_physreg
+tracksRegLiveness: true
+body:             |
+  ; CHECK-LABEL: name: no_fold_andn2_select_condition_live_out_phi_physreg
+  ; CHECK: bb.0:
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   $sgpr4_sgpr5 = S_MOV_B64 -1
+  ; CHECK:   undef %0.sub0:vreg_64 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK:   S_BRANCH %bb.2
+  ; CHECK: bb.1:
+  ; CHECK:   S_ENDPGM 0
+  ; CHECK: bb.2:
+  ; CHECK:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   liveins: $sgpr4_sgpr5
+  ; CHECK:   [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, $sgpr4_sgpr5, implicit $exec
+  ; CHECK:   V_CMP_NE_U32_e32 1, [[V_CNDMASK_B32_e64_]], implicit-def $vcc, implicit $exec
+  ; CHECK:   %0.sub1:vreg_64 = COPY %0.sub0
+  ; CHECK:   DS_WRITE_B64_gfx9 undef %2:vgpr_32, %0, 0, 0, implicit $exec :: (store 8, addrspace 3)
+  ; CHECK:   ATOMIC_FENCE 4, 2
+  ; CHECK:   $sgpr4_sgpr5 = S_MOV_B64 0
+  ; CHECK:   $vcc = S_AND_B64 $exec, $vcc, implicit-def dead $scc
+  ; CHECK:   S_CBRANCH_VCCNZ %bb.1, implicit $vcc
+  ; CHECK:   S_BRANCH %bb.2
+  bb.0:
+    successors: %bb.2
+
+    $sgpr4_sgpr5 = S_MOV_B64 -1
+    undef %5.sub0:vreg_64 = V_MOV_B32_e32 0, implicit $exec
+    S_BRANCH %bb.2
+
+  bb.1:
+    S_ENDPGM 0
+
+  bb.2:
+    successors: %bb.1, %bb.2
+    liveins: $sgpr4_sgpr5
+
+    %4:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, $sgpr4_sgpr5, implicit $exec
+    V_CMP_NE_U32_e32 1, %4, implicit-def $vcc, implicit $exec
+    %5.sub1:vreg_64 = COPY %5.sub0
+    DS_WRITE_B64_gfx9 undef %6:vgpr_32, %5, 0, 0, implicit $exec :: (store 8, addrspace 3)
+    ATOMIC_FENCE 4, 2
+    $sgpr4_sgpr5 = S_MOV_B64 0
+    $vcc = S_AND_B64 $exec, killed $vcc, implicit-def dead $scc
+    S_CBRANCH_VCCNZ %bb.1, implicit killed $vcc
+    S_BRANCH %bb.2
+
+...
+
+---
+name:            fold_andn2_select_condition_live_out_phi_physreg_reorder
+tracksRegLiveness: true
+body:             |
+  ; CHECK-LABEL: name: fold_andn2_select_condition_live_out_phi_physreg_reorder
+  ; CHECK: bb.0:
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   $sgpr4_sgpr5 = S_MOV_B64 -1
+  ; CHECK:   undef %0.sub0:vreg_64 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK:   S_BRANCH %bb.2
+  ; CHECK: bb.1:
+  ; CHECK:   S_ENDPGM 0
+  ; CHECK: bb.2:
+  ; CHECK:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   liveins: $sgpr4_sgpr5
+  ; CHECK:   %0.sub1:vreg_64 = COPY %0.sub0
+  ; CHECK:   DS_WRITE_B64_gfx9 undef %2:vgpr_32, %0, 0, 0, implicit $exec :: (store 8, addrspace 3)
+  ; CHECK:   ATOMIC_FENCE 4, 2
+  ; CHECK:   $vcc = S_ANDN2_B64 $exec, $sgpr4_sgpr5, implicit-def dead $scc
+  ; CHECK:   $sgpr4_sgpr5 = S_MOV_B64 0
+  ; CHECK:   S_CBRANCH_VCCNZ %bb.1, implicit $vcc
+  ; CHECK:   S_BRANCH %bb.2
+  bb.0:
+    successors: %bb.2
+
+    $sgpr4_sgpr5 = S_MOV_B64 -1
+    undef %5.sub0:vreg_64 = V_MOV_B32_e32 0, implicit $exec
+    S_BRANCH %bb.2
+
+  bb.1:
+    S_ENDPGM 0
+
+  bb.2:
+    successors: %bb.1, %bb.2
+    liveins: $sgpr4_sgpr5
+
+    %4:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, $sgpr4_sgpr5, implicit $exec
+    V_CMP_NE_U32_e32 1, %4, implicit-def $vcc, implicit $exec
+    %5.sub1:vreg_64 = COPY %5.sub0
+    DS_WRITE_B64_gfx9 undef %6:vgpr_32, %5, 0, 0, implicit $exec :: (store 8, addrspace 3)
+    ATOMIC_FENCE 4, 2
+    $vcc = S_AND_B64 $exec, killed $vcc, implicit-def dead $scc
+    $sgpr4_sgpr5 = S_MOV_B64 0
+    S_CBRANCH_VCCNZ %bb.1, implicit killed $vcc
+    S_BRANCH %bb.2
+
+...

From 389f009c5757cf09c0b2d77a15b3b541fb0f2a1c Mon Sep 17 00:00:00 2001
From: JF Bastien <jfbastien@apple.com>
Date: Tue, 28 Jul 2020 13:35:03 -0700
Subject: [PATCH 0368/1035] [NFC] Sema: use checkArgCount instead of custom
 checking

 As requested in D79279.

Differential Revision: https://reviews.llvm.org/D84666
---
 .../clang/Basic/DiagnosticSemaKinds.td        |  2 -
 clang/lib/Sema/SemaChecking.cpp               | 55 +++----------------
 clang/test/CodeGen/builtins-ppc-error.c       |  8 +--
 clang/test/SemaOpenCL/to_addr_builtin.cl      |  2 +-
 4 files changed, 13 insertions(+), 54 deletions(-)

diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index adfd4c207da8b..8093e7ed3fbe1 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -9724,8 +9724,6 @@ def err_opencl_block_ref_block : Error<
   "cannot refer to a block inside block">;
 
 // OpenCL v2.0 s6.13.9 - Address space qualifier functions.
-def err_opencl_builtin_to_addr_arg_num : Error<
-  "invalid number of arguments to function: %0">;
 def err_opencl_builtin_to_addr_invalid_arg : Error<
   "invalid argument %0 to function: %1, expecting a generic pointer argument">;
 
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index 77d5f3ff816ed..ccdb277dda1a0 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -1274,11 +1274,8 @@ static bool SemaBuiltinPipePackets(Sema &S, CallExpr *Call) {
 // \return True if a semantic error has been found, false otherwise.
 static bool SemaOpenCLBuiltinToAddr(Sema &S, unsigned BuiltinID,
                                     CallExpr *Call) {
-  if (Call->getNumArgs() != 1) {
-    S.Diag(Call->getBeginLoc(), diag::err_opencl_builtin_to_addr_arg_num)
-        << Call->getDirectCallee() << Call->getSourceRange();
+  if (checkArgCount(S, Call, 1))
     return true;
-  }
 
   auto RT = Call->getArg(0)->getType();
   if (!RT->isPointerType() || RT->getPointeeType()
@@ -5572,21 +5569,8 @@ bool Sema::SemaBuiltinVAStart(unsigned BuiltinID, CallExpr *TheCall) {
   if (checkVAStartABI(*this, BuiltinID, Fn))
     return true;
 
-  if (TheCall->getNumArgs() > 2) {
-    Diag(TheCall->getArg(2)->getBeginLoc(),
-         diag::err_typecheck_call_too_many_args)
-        << 0 /*function call*/ << 2 << TheCall->getNumArgs()
-        << Fn->getSourceRange()
-        << SourceRange(TheCall->getArg(2)->getBeginLoc(),
-                       (*(TheCall->arg_end() - 1))->getEndLoc());
+  if (checkArgCount(*this, TheCall, 2))
     return true;
-  }
-
-  if (TheCall->getNumArgs() < 2) {
-    return Diag(TheCall->getEndLoc(),
-                diag::err_typecheck_call_too_few_args_at_least)
-           << 0 /*function call*/ << 2 << TheCall->getNumArgs();
-  }
 
   // Type-check the first argument normally.
   if (checkBuiltinArgument(*this, TheCall, 0))
@@ -5696,15 +5680,8 @@ bool Sema::SemaBuiltinVAStartARMMicrosoft(CallExpr *Call) {
 /// SemaBuiltinUnorderedCompare - Handle functions like __builtin_isgreater and
 /// friends.  This is declared to take (...), so we have to check everything.
 bool Sema::SemaBuiltinUnorderedCompare(CallExpr *TheCall) {
-  if (TheCall->getNumArgs() < 2)
-    return Diag(TheCall->getEndLoc(), diag::err_typecheck_call_too_few_args)
-           << 0 << 2 << TheCall->getNumArgs() /*function call*/;
-  if (TheCall->getNumArgs() > 2)
-    return Diag(TheCall->getArg(2)->getBeginLoc(),
-                diag::err_typecheck_call_too_many_args)
-           << 0 /*function call*/ << 2 << TheCall->getNumArgs()
-           << SourceRange(TheCall->getArg(2)->getBeginLoc(),
-                          (*(TheCall->arg_end() - 1))->getEndLoc());
+  if (checkArgCount(*this, TheCall, 2))
+    return true;
 
   ExprResult OrigArg0 = TheCall->getArg(0);
   ExprResult OrigArg1 = TheCall->getArg(1);
@@ -5742,15 +5719,8 @@ bool Sema::SemaBuiltinUnorderedCompare(CallExpr *TheCall) {
 /// to check everything. We expect the last argument to be a floating point
 /// value.
 bool Sema::SemaBuiltinFPClassification(CallExpr *TheCall, unsigned NumArgs) {
-  if (TheCall->getNumArgs() < NumArgs)
-    return Diag(TheCall->getEndLoc(), diag::err_typecheck_call_too_few_args)
-           << 0 << NumArgs << TheCall->getNumArgs() /*function call*/;
-  if (TheCall->getNumArgs() > NumArgs)
-    return Diag(TheCall->getArg(NumArgs)->getBeginLoc(),
-                diag::err_typecheck_call_too_many_args)
-           << 0 /*function call*/ << NumArgs << TheCall->getNumArgs()
-           << SourceRange(TheCall->getArg(NumArgs)->getBeginLoc(),
-                          (*(TheCall->arg_end() - 1))->getEndLoc());
+  if (checkArgCount(*this, TheCall, NumArgs))
+    return true;
 
   // __builtin_fpclassify is the only case where NumArgs != 1, so we can count
   // on all preceding parameters just being int.  Try all of those.
@@ -5854,17 +5824,8 @@ bool Sema::SemaBuiltinComplex(CallExpr *TheCall) {
 // vector short vec_xxsldwi(vector short, vector short, int);
 bool Sema::SemaBuiltinVSX(CallExpr *TheCall) {
   unsigned ExpectedNumArgs = 3;
-  if (TheCall->getNumArgs() < ExpectedNumArgs)
-    return Diag(TheCall->getEndLoc(),
-                diag::err_typecheck_call_too_few_args_at_least)
-           << 0 /*function call*/ << ExpectedNumArgs << TheCall->getNumArgs()
-           << TheCall->getSourceRange();
-
-  if (TheCall->getNumArgs() > ExpectedNumArgs)
-    return Diag(TheCall->getEndLoc(),
-                diag::err_typecheck_call_too_many_args_at_most)
-           << 0 /*function call*/ << ExpectedNumArgs << TheCall->getNumArgs()
-           << TheCall->getSourceRange();
+  if (checkArgCount(*this, TheCall, ExpectedNumArgs))
+    return true;
 
   // Check the third argument is a compile time constant
   if (!TheCall->getArg(2)->isIntegerConstantExpr(Context))
diff --git a/clang/test/CodeGen/builtins-ppc-error.c b/clang/test/CodeGen/builtins-ppc-error.c
index 80ca227eeb24c..6557565938b0c 100644
--- a/clang/test/CodeGen/builtins-ppc-error.c
+++ b/clang/test/CodeGen/builtins-ppc-error.c
@@ -32,16 +32,16 @@ void testInsertWord(void) {
 }
 
 void testXXPERMDI(int index) {
-  vec_xxpermdi(vsi); //expected-error {{too few arguments to function call, expected at least 3, have 1}}
-  vec_xxpermdi(vsi, vsi, 2, 4); //expected-error {{too many arguments to function call, expected at most 3, have 4}}
+  vec_xxpermdi(vsi); //expected-error {{too few arguments to function call, expected 3, have 1}}
+  vec_xxpermdi(vsi, vsi, 2, 4); //expected-error {{too many arguments to function call, expected 3, have 4}}
   vec_xxpermdi(vsi, vsi, index); //expected-error {{argument 3 to '__builtin_vsx_xxpermdi' must be a 2-bit unsigned literal (i.e. 0, 1, 2 or 3)}}
   vec_xxpermdi(1, 2, 3); //expected-error {{first two arguments to '__builtin_vsx_xxpermdi' must be vectors}}
   vec_xxpermdi(vsi, vuc, 2); //expected-error {{first two arguments to '__builtin_vsx_xxpermdi' must have the same type}}
 }
 
 void testXXSLDWI(int index) {
-  vec_xxsldwi(vsi); //expected-error {{too few arguments to function call, expected at least 3, have 1}}
-  vec_xxsldwi(vsi, vsi, 2, 4); //expected-error {{too many arguments to function call, expected at most 3, have 4}}
+  vec_xxsldwi(vsi); //expected-error {{too few arguments to function call, expected 3, have 1}}
+  vec_xxsldwi(vsi, vsi, 2, 4); //expected-error {{too many arguments to function call, expected 3, have 4}}
   vec_xxsldwi(vsi, vsi, index); //expected-error {{argument 3 to '__builtin_vsx_xxsldwi' must be a 2-bit unsigned literal (i.e. 0, 1, 2 or 3)}}
   vec_xxsldwi(1, 2, 3); //expected-error {{first two arguments to '__builtin_vsx_xxsldwi' must be vectors}}
   vec_xxsldwi(vsi, vuc, 2); //expected-error {{first two arguments to '__builtin_vsx_xxsldwi' must have the same type}}
diff --git a/clang/test/SemaOpenCL/to_addr_builtin.cl b/clang/test/SemaOpenCL/to_addr_builtin.cl
index ff2d7807356a2..8f4a20ac7662a 100644
--- a/clang/test/SemaOpenCL/to_addr_builtin.cl
+++ b/clang/test/SemaOpenCL/to_addr_builtin.cl
@@ -15,7 +15,7 @@ void test(void) {
   // expected-error@-2{{implicit declaration of function 'to_global' is invalid in OpenCL}}
   // expected-warning@-3{{incompatible integer to pointer conversion assigning to '__global int *__private' from 'int'}}
 #else
-  // expected-error@-5{{invalid number of arguments to function: 'to_global'}}
+  // expected-error@-5{{too many arguments to function call, expected 1, have 2}}
 #endif
 
   int x;

From 9731ef3ec578a20be3d81d0ef340faf61bf797d4 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Sun, 26 Jul 2020 11:04:37 -0400
Subject: [PATCH 0369/1035] AMDGPU/GlobalISel: Add SReg_96 to SGPRRegBank

---
 llvm/lib/Target/AMDGPU/AMDGPURegisterBanks.td | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBanks.td b/llvm/lib/Target/AMDGPU/AMDGPURegisterBanks.td
index 9f6ebd00cd97b..6c70b53b23c11 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBanks.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBanks.td
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 def SGPRRegBank : RegisterBank<"SGPR",
-  [SReg_LO16, SReg_32, SReg_64, SReg_128, SReg_160, SReg_192, SReg_256, SReg_512, SReg_1024]
+  [SReg_LO16, SReg_32, SReg_64, SReg_96, SReg_128, SReg_160, SReg_192, SReg_256, SReg_512, SReg_1024]
 >;
 
 def VGPRRegBank : RegisterBank<"VGPR",

From ee713a2d2854c56886ebfee165fe2d32905c4ef7 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Sun, 26 Jul 2020 11:04:07 -0400
Subject: [PATCH 0370/1035] AMDGPU/GlobalISel: Add some missing tests for
 extract selection

---
 .../AMDGPU/GlobalISel/inst-select-extract.mir | 39 +++++++++++++++++++
 1 file changed, 39 insertions(+)

diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-extract.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-extract.mir
index df16e9c1f0917..d6d2a74cbcc21 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-extract.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-extract.mir
@@ -199,6 +199,45 @@ body: |
 
 ...
 
+---
+name:            extract_sgpr_s32_from_v3s32
+legalized:       true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1_sgpr2
+    ; CHECK-LABEL: name: extract_sgpr_s32_from_v3s32
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr_96 = COPY $sgpr0_sgpr1_sgpr2
+    ; CHECK: [[COPY1:%[0-9]+]]:sreg_32 = COPY [[COPY]].sub1
+    ; CHECK: [[COPY2:%[0-9]+]]:sreg_32 = COPY [[COPY]].sub2
+    ; CHECK: S_ENDPGM 0, implicit [[COPY]], implicit [[COPY1]], implicit [[COPY2]]
+    %0:sgpr(<3 x s32>) = COPY $sgpr0_sgpr1_sgpr2
+    %1:sgpr(s32) = G_EXTRACT %0, 0
+    %2:sgpr(s32) = G_EXTRACT %0, 32
+    %3:sgpr(s32) = G_EXTRACT %0, 64
+    S_ENDPGM 0, implicit %0, implicit %2, implicit %3
+
+...
+
+---
+name:            extract_sgpr_v2s32_from_v3s32
+legalized:       true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1_sgpr2
+    ; CHECK-LABEL: name: extract_sgpr_v2s32_from_v3s32
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr_96_with_sub0_sub1 = COPY $sgpr0_sgpr1_sgpr2
+    ; CHECK: [[COPY1:%[0-9]+]]:sreg_64 = COPY [[COPY]].sub0_sub1
+    ; CHECK: S_ENDPGM 0, implicit [[COPY1]]
+    %0:sgpr(<3 x s32>) = COPY $sgpr0_sgpr1_sgpr2
+    %1:sgpr(<2 x s32>) = G_EXTRACT %0, 0
+    S_ENDPGM 0, implicit %1
+
+...
+
 ---
 name:            extract_sgpr_v3s32_from_v4s32
 legalized:       true

From 5174e7b443d815df6707addbd4fe172418d6f4a2 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Sat, 25 Jul 2020 11:12:29 -0400
Subject: [PATCH 0371/1035] GlobalISel: Add typeIsNot LegalityPredicate

This allows sorting the legal/custom rules first as is recommended
---
 llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h | 10 +++++++++-
 llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp       |  4 ++--
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h
index b50d7cd7a5f91..7123aff36cd14 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h
@@ -218,11 +218,19 @@ Predicate any(Predicate P0, Predicate P1, Args... args) {
   return any(any(P0, P1), args...);
 }
 
-/// True iff the given type index is the specified types.
+/// True iff the given type index is the specified type.
 LegalityPredicate typeIs(unsigned TypeIdx, LLT TypesInit);
 /// True iff the given type index is one of the specified types.
 LegalityPredicate typeInSet(unsigned TypeIdx,
                             std::initializer_list<LLT> TypesInit);
+
+/// True iff the given type index is not the specified type.
+inline LegalityPredicate typeIsNot(unsigned TypeIdx, LLT Type) {
+  return [=](const LegalityQuery &Query) {
+           return Query.Types[TypeIdx] != Type;
+         };
+}
+
 /// True iff the given types for the given pair of type indexes is one of the
 /// specified type pairs.
 LegalityPredicate
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index e14623e650d42..27031d789aac5 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -551,8 +551,8 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
     .legalFor({{PrivatePtr, S32}});
 
   getActionDefinitionsBuilder(G_GLOBAL_VALUE)
-    .unsupportedFor({PrivatePtr})
-    .custom();
+    .customIf(typeIsNot(0, PrivatePtr));
+
   setAction({G_BLOCK_ADDR, CodePtr}, Legal);
 
   auto &FPOpActions = getActionDefinitionsBuilder(

From e87356b4981bb0b504c106fd70fbde31e84447e6 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Sat, 25 Jul 2020 10:27:16 -0400
Subject: [PATCH 0372/1035] GlobalISel: Don't assert on operations with no type
 indices

Fix not marking G_FENCE as legal on AMDGPU This was apparently
defaulting to legal using the "legacy" rules, whatever those are.
---
 llvm/lib/CodeGen/GlobalISel/LegalizerInfo.cpp  | 3 ++-
 llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 3 +++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerInfo.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerInfo.cpp
index 4abd0c4df97a2..d14682ccc054b 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerInfo.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerInfo.cpp
@@ -105,6 +105,7 @@ raw_ostream &LegalityQuery::print(raw_ostream &OS) const {
 static bool hasNoSimpleLoops(const LegalizeRule &Rule, const LegalityQuery &Q,
                              const std::pair<unsigned, LLT> &Mutation) {
   switch (Rule.getAction()) {
+  case Legal:
   case Custom:
   case Lower:
   case MoreElements:
@@ -122,7 +123,7 @@ static bool mutationIsSane(const LegalizeRule &Rule,
                            std::pair<unsigned, LLT> Mutation) {
   // If the user wants a custom mutation, then we can't really say much about
   // it. Return true, and trust that they're doing the right thing.
-  if (Rule.getAction() == Custom)
+  if (Rule.getAction() == Custom || Rule.getAction() == Legal)
     return true;
 
   const unsigned TypeIdx = Mutation.first;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 27031d789aac5..aa437638cec59 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -1491,6 +1491,9 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
   getActionDefinitionsBuilder(G_READCYCLECOUNTER)
     .legalFor({S64});
 
+  getActionDefinitionsBuilder(G_FENCE)
+    .alwaysLegal();
+
   getActionDefinitionsBuilder({
       // TODO: Verify V_BFI_B32 is generated from expanded bit ops
       G_FCOPYSIGN,

From abf1ed70d60f4e8dffc53c9bb2cdb1e4be9fe888 Mon Sep 17 00:00:00 2001
From: Daniel Sanders <daniel_l_sanders@apple.com>
Date: Tue, 28 Jul 2020 13:56:51 -0700
Subject: [PATCH 0373/1035] [globalisel][cse] Merge debug locations when
 CSE'ing

Reviewed By: aditya_nandakumar

Differential Revision: https://reviews.llvm.org/D78388
---
 llvm/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/llvm/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp b/llvm/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp
index 6f8fd309e1b58..9048583ff7281 100644
--- a/llvm/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp
@@ -13,6 +13,7 @@
 
 #include "llvm/CodeGen/GlobalISel/CSEMIRBuilder.h"
 #include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
+#include "llvm/IR/DebugInfoMetadata.h"
 
 using namespace llvm;
 
@@ -134,6 +135,21 @@ CSEMIRBuilder::generateCopiesIfRequired(ArrayRef<DstOp> DstOps,
     if (Op.getDstOpKind() == DstOp::DstType::Ty_Reg)
       return buildCopy(Op.getReg(), MIB.getReg(0));
   }
+
+  // If we didn't generate a copy then we're re-using an existing node directly
+  // instead of emitting any code. Merge the debug location we wanted to emit
+  // into the instruction we're CSE'ing with. Debug locations arent part of the
+  // profile so we don't need to recompute it.
+  if (getDebugLoc()) {
+    GISelChangeObserver *Observer = getState().Observer;
+    if (Observer)
+      Observer->changingInstr(*MIB);
+    MIB->setDebugLoc(
+        DILocation::getMergedLocation(MIB->getDebugLoc(), getDebugLoc()));
+    if (Observer)
+      Observer->changedInstr(*MIB);
+  }
+
   return MIB;
 }
 

From 417d3d495f1cfb0a2f7b60d00829925126fdcfd9 Mon Sep 17 00:00:00 2001
From: Varun Gandhi <varun_gandhi@apple.com>
Date: Tue, 28 Jul 2020 14:35:06 -0700
Subject: [PATCH 0374/1035] [docs] [lit] Add a more helpful description for
 lit.py's -s flag.

Reviewed By: yln

Differential Revision: https://reviews.llvm.org/D82808
---
 llvm/docs/CommandGuide/lit.rst     | 1 +
 llvm/utils/lit/lit/cl_arguments.py | 4 +++-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/llvm/docs/CommandGuide/lit.rst b/llvm/docs/CommandGuide/lit.rst
index d29d293f32d8d..c9ad808e13500 100644
--- a/llvm/docs/CommandGuide/lit.rst
+++ b/llvm/docs/CommandGuide/lit.rst
@@ -81,6 +81,7 @@ OUTPUT OPTIONS
 .. option:: -s, --succinct
 
  Show less output, for example don't show information on tests that pass.
+ Also show a progress bar, unless ``--no-progress-bar`` is specificed.
 
 .. option:: -v, --verbose
 
diff --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py
index 966863624feb9..baeb3635298f5 100644
--- a/llvm/utils/lit/lit/cl_arguments.py
+++ b/llvm/utils/lit/lit/cl_arguments.py
@@ -42,7 +42,9 @@ def parse_args():
             help="Suppress no error output",
             action="store_true")
     format_group.add_argument("-s", "--succinct",
-            help="Reduce amount of output",
+            help="Reduce amount of output."
+                 " Additionally, show a progress bar,"
+                 " unless --no-progress-bar is specified.",
             action="store_true")
     format_group.add_argument("-v", "--verbose",
             dest="showOutput",

From 450dc09d69df343386865d279983de9d51d1e984 Mon Sep 17 00:00:00 2001
From: Johannes Doerfert <johannes@jdoerfert.de>
Date: Tue, 28 Jul 2020 09:48:47 -0500
Subject: [PATCH 0375/1035] [SROA][Mem2Reg] Use efficient droppable use API
 (after D83976)

Reviewed By: efriedma

Differential Revision: https://reviews.llvm.org/D84804
---
 llvm/include/llvm/IR/Value.h                  |  7 +++-
 llvm/lib/IR/Value.cpp                         | 41 +++++++++++--------
 llvm/lib/Transforms/Scalar/SROA.cpp           |  2 +-
 .../Utils/PromoteMemoryToRegister.cpp         | 30 ++++----------
 4 files changed, 39 insertions(+), 41 deletions(-)

diff --git a/llvm/include/llvm/IR/Value.h b/llvm/include/llvm/IR/Value.h
index fa706a1b64086..eec96373a1a9f 100644
--- a/llvm/include/llvm/IR/Value.h
+++ b/llvm/include/llvm/IR/Value.h
@@ -470,8 +470,11 @@ class Value {
   void dropDroppableUses(llvm::function_ref<bool(const Use *)> ShouldDrop =
                              [](const Use *) { return true; });
 
-  /// Remove every use of \p User that can safely be removed.
-  void dropDroppableUsesByUser(const User &Usr);
+  /// Remove every use of this value in \p User that can safely be removed.
+  void dropDroppableUsesIn(User &Usr);
+
+  /// Remove the droppable use \p U.
+  void dropDroppableUse(Use &U);
 
   /// Check if this value is used in the specified basic block.
   bool isUsedInBasicBlock(const BasicBlock *BB) const;
diff --git a/llvm/lib/IR/Value.cpp b/llvm/lib/IR/Value.cpp
index 8c1f9c5a3b36f..26a983596ab24 100644
--- a/llvm/lib/IR/Value.cpp
+++ b/llvm/lib/IR/Value.cpp
@@ -175,25 +175,34 @@ void Value::dropDroppableUses(
   for (Use &U : uses())
     if (U.getUser()->isDroppable() && ShouldDrop(&U))
       ToBeEdited.push_back(&U);
-  for (Use *U : ToBeEdited) {
-    U->removeFromList();
-    if (auto *Assume = dyn_cast<IntrinsicInst>(U->getUser())) {
-      assert(Assume->getIntrinsicID() == Intrinsic::assume);
-      unsigned OpNo = U->getOperandNo();
-      if (OpNo == 0)
-        Assume->setOperand(0, ConstantInt::getTrue(Assume->getContext()));
-      else {
-        Assume->setOperand(OpNo, UndefValue::get(U->get()->getType()));
-        CallInst::BundleOpInfo &BOI = Assume->getBundleOpInfoForOperand(OpNo);
-        BOI.Tag = getContext().pImpl->getOrInsertBundleTag("ignore");
-      }
-    } else
-      llvm_unreachable("unkown droppable use");
+  for (Use *U : ToBeEdited)
+    dropDroppableUse(*U);
+}
+
+void Value::dropDroppableUsesIn(User &Usr) {
+  assert(Usr.isDroppable() && "Expected a droppable user!");
+  for (Use &UsrOp : Usr.operands()) {
+    if (UsrOp.get() == this)
+      dropDroppableUse(UsrOp);
   }
 }
 
-void Value::dropDroppableUsesByUser(const User &Usr) {
-  dropDroppableUses([&](const Use *U) { return U->getUser() == &Usr; });
+void Value::dropDroppableUse(Use &U) {
+  U.removeFromList();
+  if (auto *Assume = dyn_cast<IntrinsicInst>(U.getUser())) {
+    assert(Assume->getIntrinsicID() == Intrinsic::assume);
+    unsigned OpNo = U.getOperandNo();
+    if (OpNo == 0)
+      U.set(ConstantInt::getTrue(Assume->getContext()));
+    else {
+      U.set(UndefValue::get(U.get()->getType()));
+      CallInst::BundleOpInfo &BOI = Assume->getBundleOpInfoForOperand(OpNo);
+      BOI.Tag = getContext().pImpl->getOrInsertBundleTag("ignore");
+    }
+    return;
+  }
+
+  llvm_unreachable("unkown droppable use");
 }
 
 bool Value::isUsedInBasicBlock(const BasicBlock *BB) const {
diff --git a/llvm/lib/Transforms/Scalar/SROA.cpp b/llvm/lib/Transforms/Scalar/SROA.cpp
index 8eb2853347bdb..a95a7aba0d38d 100644
--- a/llvm/lib/Transforms/Scalar/SROA.cpp
+++ b/llvm/lib/Transforms/Scalar/SROA.cpp
@@ -3087,7 +3087,7 @@ class llvm::sroa::AllocaSliceRewriter
     if (II.isDroppable()) {
       assert(II.getIntrinsicID() == Intrinsic::assume && "Expected assume");
       // TODO For now we forget assumed information, this can be improved.
-      OldPtr->dropDroppableUsesByUser(II);
+      OldPtr->dropDroppableUsesIn(II);
       return true;
     }
 
diff --git a/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp b/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
index 33904e54ac237..bfb915eca1f3a 100644
--- a/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
+++ b/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
@@ -311,31 +311,16 @@ static void removeIntrinsicUsers(AllocaInst *AI) {
   // Knowing that this alloca is promotable, we know that it's safe to kill all
   // instructions except for load and store.
 
-  // Helper to drop the uses of \p I in \p UserI.
-  auto DropUsesIn = [](Instruction *UserI, Instruction *I,
-                       Instruction::user_iterator &UI,
-                       const Instruction::user_iterator &UE) {
-    // TODO For now we forget assumed information, this can be improved.
-    assert(isa<IntrinsicInst>(UserI) &&
-           cast<IntrinsicInst>(UserI)->getIntrinsicID() == Intrinsic::assume &&
-           "Expected assume");
-
-    // Skip ahead if User has multiple uses of I.
-    while (UI != UE && *UI == UserI)
-      ++UI;
-
-    I->dropDroppableUsesByUser(*UserI);
-  };
-
-  for (auto UI = AI->user_begin(), UE = AI->user_end(); UI != UE;) {
-    Instruction *I = cast<Instruction>(*UI);
+  for (auto UI = AI->use_begin(), UE = AI->use_end(); UI != UE;) {
+    Instruction *I = cast<Instruction>(UI->getUser());
+    Use &U = *UI;
     ++UI;
     if (isa<LoadInst>(I) || isa<StoreInst>(I))
       continue;
 
     // Drop the use of AI in droppable instructions.
     if (I->isDroppable()) {
-      DropUsesIn(I, AI, UI, UE);
+      I->dropDroppableUse(U);
       continue;
     }
 
@@ -343,13 +328,14 @@ static void removeIntrinsicUsers(AllocaInst *AI) {
       // The only users of this bitcast/GEP instruction are lifetime intrinsics.
       // Follow the use/def chain to erase them now instead of leaving it for
       // dead code elimination later.
-      for (auto UUI = I->user_begin(), UUE = I->user_end(); UUI != UUE;) {
-        Instruction *Inst = cast<Instruction>(*UUI);
+      for (auto UUI = I->use_begin(), UUE = I->use_end(); UUI != UUE;) {
+        Instruction *Inst = cast<Instruction>(UUI->getUser());
+        Use &UU = *UUI;
         ++UUI;
 
         // Drop the use of I in droppable instructions.
         if (Inst->isDroppable()) {
-          DropUsesIn(Inst, I, UUI, UUE);
+          Inst->dropDroppableUse(UU);
           continue;
         }
         Inst->eraseFromParent();

From 0b161def6cacff1a63d3cf1a1efe95b550814d7a Mon Sep 17 00:00:00 2001
From: Rahul Joshi <jurahul@google.com>
Date: Tue, 28 Jul 2020 10:29:43 -0700
Subject: [PATCH 0376/1035] [MLIR] Add unit test for tblgen Op build methods

- Initiate the unit test with a test that tests variants of build() methods
  generated for ops with variadic operands and results.
- The intent is to migrate unit .td tests in mlir/test/mlir-tblgen that check for
  generated C++ code to these unit tests which test both that the generated code
  compiles and also is functionally correct.

Differential Revision: https://reviews.llvm.org/D84074
---
 mlir/test/lib/Dialect/Test/TestOps.td  |  32 +++++
 mlir/unittests/TableGen/CMakeLists.txt |   9 +-
 mlir/unittests/TableGen/OpBuildGen.cpp | 181 +++++++++++++++++++++++++
 3 files changed, 221 insertions(+), 1 deletion(-)
 create mode 100644 mlir/unittests/TableGen/OpBuildGen.cpp

diff --git a/mlir/test/lib/Dialect/Test/TestOps.td b/mlir/test/lib/Dialect/Test/TestOps.td
index 528d219f70c74..af20034abd9ba 100644
--- a/mlir/test/lib/Dialect/Test/TestOps.td
+++ b/mlir/test/lib/Dialect/Test/TestOps.td
@@ -1473,4 +1473,36 @@ def RegionIfOp : TEST_Op<"region_if",
   }];
 }
 
+//===----------------------------------------------------------------------===//
+// Test TableGen generated build() methods
+//===----------------------------------------------------------------------===//
+
+def TableGenConstant : TEST_Op<"tblgen_constant"> {
+  let results = (outs AnyType);
+}
+
+// No variadic args or results.
+def TableGenBuildOp0 : TEST_Op<"tblgen_build_0"> {
+  let arguments = (ins AnyType:$value);
+  let results = (outs AnyType:$result);
+}
+
+// Sigle variadic arg and single variadic results.
+def TableGenBuildOp1 : TEST_Op<"tblgen_build_1"> {
+  let arguments = (ins Variadic<AnyType>:$inputs);
+  let results = (outs Variadic<AnyType>:$results);
+}
+
+// Single variadic arg and non-variadic results.
+def TableGenBuildOp2 : TEST_Op<"tblgen_build_2"> {
+  let arguments = (ins Variadic<AnyType>:$inputs);
+  let results = (outs AnyType:$result);
+}
+
+// Single variadic arg and multiple variadic results.
+def TableGenBuildOp3 : TEST_Op<"tblgen_build_3", [SameVariadicResultSize]> {
+  let arguments = (ins Variadic<AnyType>:$inputs);
+  let results = (outs Variadic<AnyType>:$resultA, Variadic<AnyType>:$resultB);
+}
+
 #endif // TEST_OPS
diff --git a/mlir/unittests/TableGen/CMakeLists.txt b/mlir/unittests/TableGen/CMakeLists.txt
index cb4dea6918b87..7cee691de4006 100644
--- a/mlir/unittests/TableGen/CMakeLists.txt
+++ b/mlir/unittests/TableGen/CMakeLists.txt
@@ -12,10 +12,17 @@ add_mlir_unittest(MLIRTableGenTests
   EnumsGenTest.cpp
   StructsGenTest.cpp
   FormatTest.cpp
+  OpBuildGen.cpp
 )
 
 add_dependencies(MLIRTableGenTests MLIRTableGenEnumsIncGen)
 add_dependencies(MLIRTableGenTests MLIRTableGenStructAttrIncGen)
+add_dependencies(MLIRTableGenTests MLIRTestDialect)
+
+include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../../test/lib/Dialect/Test)
+include_directories(${CMAKE_CURRENT_BINARY_DIR}/../../test/lib/Dialect/Test)
 
 target_link_libraries(MLIRTableGenTests
-  PRIVATE MLIRTableGen MLIRIR)
+  PRIVATE MLIRTableGen MLIRIR
+  PUBLIC MLIRTestDialect
+)
diff --git a/mlir/unittests/TableGen/OpBuildGen.cpp b/mlir/unittests/TableGen/OpBuildGen.cpp
new file mode 100644
index 0000000000000..e90f96b87d634
--- /dev/null
+++ b/mlir/unittests/TableGen/OpBuildGen.cpp
@@ -0,0 +1,181 @@
+//===- OpBuildGen.cpp - TableGen OpBuildGen Tests -------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Test TableGen generated build() methods on Operations.
+//
+//===----------------------------------------------------------------------===//
+
+#include "TestDialect.h"
+#include "mlir/IR/Attributes.h"
+#include "mlir/IR/Builders.h"
+#include "mlir/IR/Dialect.h"
+#include "mlir/IR/Identifier.h"
+#include "mlir/IR/StandardTypes.h"
+#include "gmock/gmock.h"
+#include <vector>
+
+namespace mlir {
+
+//===----------------------------------------------------------------------===//
+// Test Fixture
+//===----------------------------------------------------------------------===//
+
+/// Test fixture for providing basic utilities for testing.
+class OpBuildGenTest : public ::testing::Test {
+protected:
+  OpBuildGenTest()
+      : ctx{}, builder(&ctx), loc(builder.getUnknownLoc()),
+        i32Ty(builder.getI32Type()), f32Ty(builder.getF32Type()),
+        cstI32(builder.create<TableGenConstant>(loc, i32Ty)),
+        cstF32(builder.create<TableGenConstant>(loc, f32Ty)),
+        noAttrs(), attrStorage{builder.getNamedAttr("attr0",
+                                                    builder.getBoolAttr(true)),
+                               builder.getNamedAttr(
+                                   "attr1", builder.getI32IntegerAttr(33))},
+        attrs(attrStorage) {}
+
+  // Verify that `op` has the given set of result types, operands, and
+  // attributes.
+  template <typename OpTy>
+  void verifyOp(OpTy &&concreteOp, std::vector<Type> resultTypes,
+                std::vector<Value> operands,
+                std::vector<NamedAttribute> attrs) {
+    ASSERT_NE(concreteOp, nullptr);
+    Operation *op = concreteOp.getOperation();
+
+    EXPECT_EQ(op->getNumResults(), resultTypes.size());
+    for (unsigned idx : llvm::seq(0U, op->getNumResults()))
+      EXPECT_EQ(op->getResult(idx).getType(), resultTypes[idx]);
+
+    EXPECT_EQ(op->getNumOperands(), operands.size());
+    for (unsigned idx : llvm::seq(0U, op->getNumOperands()))
+      EXPECT_EQ(op->getOperand(idx), operands[idx]);
+
+    EXPECT_EQ(op->getAttrs().size(), attrs.size());
+    for (unsigned idx : llvm::seq<unsigned>(0U, attrs.size()))
+      EXPECT_EQ(op->getAttr(attrs[idx].first.strref()), attrs[idx].second);
+
+    concreteOp.erase();
+  }
+
+protected:
+  MLIRContext ctx;
+  OpBuilder builder;
+  Location loc;
+  Type i32Ty;
+  Type f32Ty;
+  TableGenConstant cstI32;
+  TableGenConstant cstF32;
+
+  ArrayRef<NamedAttribute> noAttrs;
+  std::vector<NamedAttribute> attrStorage;
+  ArrayRef<NamedAttribute> attrs;
+};
+
+/// Test basic build methods.
+TEST_F(OpBuildGenTest, BasicBuildMethods) {
+  // Test separate args, separate results build method.
+  auto op = builder.create<TableGenBuildOp0>(loc, i32Ty, cstI32);
+  verifyOp(op, {i32Ty}, {cstI32}, noAttrs);
+
+  // Test separate args, collective results build method.
+  op = builder.create<TableGenBuildOp0>(loc, ArrayRef<Type>{i32Ty}, cstI32);
+  verifyOp(op, {i32Ty}, {cstI32}, noAttrs);
+
+  // Test collective args, collective params build method.
+  op = builder.create<TableGenBuildOp0>(loc, ArrayRef<Type>{i32Ty},
+                                        ArrayRef<Value>{cstI32});
+  verifyOp(op, {i32Ty}, {cstI32}, noAttrs);
+
+  // Test collective args, collective results, non-empty attributes
+  op = builder.create<TableGenBuildOp0>(loc, ArrayRef<Type>{i32Ty},
+                                        ArrayRef<Value>{cstI32}, attrs);
+  verifyOp(op, {i32Ty}, {cstI32}, attrs);
+}
+
+/// The following 3 tests exercise build methods generated for operations
+/// with a combination of:
+///
+/// single variadic arg x
+/// {single variadic result, non-variadic result, multiple variadic results}
+///
+/// Specifically to test that that ODS framework does not generate ambiguous
+/// build() methods that fail to compile.
+
+/// Test build methods for an Op with a single varadic arg and a single
+/// variadic result.
+TEST_F(OpBuildGenTest, BuildMethodsSingleVariadicArgAndResult) {
+  // Test collective args, collective results method, building a unary op.
+  auto op = builder.create<TableGenBuildOp1>(loc, ArrayRef<Type>{i32Ty},
+                                             ArrayRef<Value>{cstI32});
+  verifyOp(std::move(op), {i32Ty}, {cstI32}, noAttrs);
+
+  // Test collective args, collective results method, building a unary op with
+  // named attributes.
+  op = builder.create<TableGenBuildOp1>(loc, ArrayRef<Type>{i32Ty},
+                                        ArrayRef<Value>{cstI32}, attrs);
+  verifyOp(std::move(op), {i32Ty}, {cstI32}, attrs);
+
+  // Test collective args, collective results method, building a binary op.
+  op = builder.create<TableGenBuildOp1>(loc, ArrayRef<Type>{i32Ty, f32Ty},
+                                        ArrayRef<Value>{cstI32, cstF32});
+  verifyOp(std::move(op), {i32Ty, f32Ty}, {cstI32, cstF32}, noAttrs);
+
+  // Test collective args, collective results method, building a binary op with
+  // named attributes.
+  op = builder.create<TableGenBuildOp1>(loc, ArrayRef<Type>{i32Ty, f32Ty},
+                                        ArrayRef<Value>{cstI32, cstF32}, attrs);
+  verifyOp(std::move(op), {i32Ty, f32Ty}, {cstI32, cstF32}, attrs);
+}
+
+/// Test build methods for an Op with a single varadic arg and a non-variadic
+/// result.
+TEST_F(OpBuildGenTest, BuildMethodsSingleVariadicArgNonVariadicResults) {
+  // Test separate arg, separate param build method.
+  auto op =
+      builder.create<TableGenBuildOp1>(loc, i32Ty, ArrayRef<Value>{cstI32});
+  verifyOp(std::move(op), {i32Ty}, {cstI32}, noAttrs);
+
+  // Test collective params build method, no attributes.
+  op = builder.create<TableGenBuildOp1>(loc, ArrayRef<Type>{i32Ty},
+                                        ArrayRef<Value>{cstI32});
+  verifyOp(std::move(op), {i32Ty}, {cstI32}, noAttrs);
+
+  // Test collective params build method no attributes, 2 inputs.
+  op = builder.create<TableGenBuildOp1>(loc, ArrayRef<Type>{i32Ty},
+                                        ArrayRef<Value>{cstI32, cstF32});
+  verifyOp(std::move(op), {i32Ty}, {cstI32, cstF32}, noAttrs);
+
+  // Test collective params build method, non-empty attributes.
+  op = builder.create<TableGenBuildOp1>(loc, ArrayRef<Type>{i32Ty},
+                                        ArrayRef<Value>{cstI32, cstF32}, attrs);
+  verifyOp(std::move(op), {i32Ty}, {cstI32, cstF32}, attrs);
+}
+
+/// Test build methods for an Op with a single varadic arg and multiple variadic
+/// result.
+TEST_F(OpBuildGenTest,
+       BuildMethodsSingleVariadicArgAndMultipleVariadicResults) {
+  // Test separate arg, separate param build method.
+  auto op = builder.create<TableGenBuildOp3>(loc, ArrayRef<Type>{i32Ty},
+                                             ArrayRef<Type>{f32Ty},
+                                             ArrayRef<Value>{cstI32});
+  verifyOp(std::move(op), {i32Ty, f32Ty}, {cstI32}, noAttrs);
+
+  // Test collective params build method, no attributes.
+  op = builder.create<TableGenBuildOp3>(loc, ArrayRef<Type>{i32Ty, f32Ty},
+                                        ArrayRef<Value>{cstI32});
+  verifyOp(std::move(op), {i32Ty, f32Ty}, {cstI32}, noAttrs);
+
+  // Test collective params build method, with attributes.
+  op = builder.create<TableGenBuildOp3>(loc, ArrayRef<Type>{i32Ty, f32Ty},
+                                        ArrayRef<Value>{cstI32}, attrs);
+  verifyOp(std::move(op), {i32Ty, f32Ty}, {cstI32}, attrs);
+}
+
+} // namespace mlir

From e22de4e46d1dd1aacc3a7060d24bcbe89908ba6c Mon Sep 17 00:00:00 2001
From: Alina Sbirlea <asbirlea@google.com>
Date: Fri, 24 Jul 2020 18:36:18 -0700
Subject: [PATCH 0377/1035] [DominatorTree] Simplify ChildrenGetter.

Summary:
Simplify ChildrenGetter to a simple wrapper around a GraphDiff call.
GraphDiff already handles nullptr in children, so the special casing in
clang can also be removed.

Reviewers: kuhar, dblaikie

Subscribers: llvm-commits, cfe-commits

Tags: #clang, #llvm

Differential Revision: https://reviews.llvm.org/D84713
---
 .../clang/Analysis/Analyses/Dominators.h      | 70 -------------------
 .../llvm/Support/GenericDomTreeConstruction.h | 42 +++--------
 2 files changed, 11 insertions(+), 101 deletions(-)

diff --git a/clang/include/clang/Analysis/Analyses/Dominators.h b/clang/include/clang/Analysis/Analyses/Dominators.h
index 95a661138df4d..25a5ba9d83fe3 100644
--- a/clang/include/clang/Analysis/Analyses/Dominators.h
+++ b/clang/include/clang/Analysis/Analyses/Dominators.h
@@ -273,76 +273,6 @@ class ControlDependencyCalculator : public ManagedAnalysis {
 
 namespace llvm {
 
-/// Clang's CFG contains nullpointers for unreachable succesors, e.g. when an
-/// if statement's condition is always false, it's 'then' branch is represented
-/// with a nullptr. This however will result in a nullpointer derefernece for
-/// dominator tree calculation.
-///
-/// To circumvent this, let's just crudely specialize the children getters
-/// used in LLVM's dominator tree builder.
-namespace DomTreeBuilder {
-
-using ClangCFGDomChildrenGetter =
-SemiNCAInfo<clang::CFGDomTree::DominatorTreeBase>::ChildrenGetter<
-                                                             /*Inverse=*/false>;
-
-template <>
-template <>
-inline ClangCFGDomChildrenGetter::ResultTy ClangCFGDomChildrenGetter::Get(
-    clang::CFGBlock *N, std::integral_constant<bool, /*Inverse=*/false>) {
-  auto RChildren = reverse(children<NodePtr>(N));
-  ResultTy Ret(RChildren.begin(), RChildren.end());
-  Ret.erase(std::remove(Ret.begin(), Ret.end(), nullptr), Ret.end());
-  return Ret;
-}
-
-using ClangCFGDomReverseChildrenGetter =
-SemiNCAInfo<clang::CFGDomTree::DominatorTreeBase>::ChildrenGetter<
-                                                              /*Inverse=*/true>;
-
-template <>
-template <>
-inline ClangCFGDomReverseChildrenGetter::ResultTy
-ClangCFGDomReverseChildrenGetter::Get(
-    clang::CFGBlock *N, std::integral_constant<bool, /*Inverse=*/true>) {
-  auto IChildren = inverse_children<NodePtr>(N);
-  ResultTy Ret(IChildren.begin(), IChildren.end());
-  Ret.erase(std::remove(Ret.begin(), Ret.end(), nullptr), Ret.end());
-  return Ret;
-}
-
-using ClangCFGPostDomChildrenGetter =
-SemiNCAInfo<clang::CFGPostDomTree::DominatorTreeBase>::ChildrenGetter<
-                                                             /*Inverse=*/false>;
-
-template <>
-template <>
-inline ClangCFGPostDomChildrenGetter::ResultTy
-ClangCFGPostDomChildrenGetter::Get(
-    clang::CFGBlock *N, std::integral_constant<bool, /*Inverse=*/false>) {
-  auto RChildren = reverse(children<NodePtr>(N));
-  ResultTy Ret(RChildren.begin(), RChildren.end());
-  Ret.erase(std::remove(Ret.begin(), Ret.end(), nullptr), Ret.end());
-  return Ret;
-}
-
-using ClangCFGPostDomReverseChildrenGetter =
-SemiNCAInfo<clang::CFGPostDomTree::DominatorTreeBase>::ChildrenGetter<
-                                                              /*Inverse=*/true>;
-
-template <>
-template <>
-inline ClangCFGPostDomReverseChildrenGetter::ResultTy
-ClangCFGPostDomReverseChildrenGetter::Get(
-    clang::CFGBlock *N, std::integral_constant<bool, /*Inverse=*/true>) {
-  auto IChildren = inverse_children<NodePtr>(N);
-  ResultTy Ret(IChildren.begin(), IChildren.end());
-  Ret.erase(std::remove(Ret.begin(), Ret.end(), nullptr), Ret.end());
-  return Ret;
-}
-
-} // end of namespace DomTreeBuilder
-
 //===-------------------------------------
 /// DominatorTree GraphTraits specialization so the DominatorTree can be
 /// iterable by generic graph iterators.
diff --git a/llvm/include/llvm/Support/GenericDomTreeConstruction.h b/llvm/include/llvm/Support/GenericDomTreeConstruction.h
index 5a1f03c879db4..6a9d38bceb388 100644
--- a/llvm/include/llvm/Support/GenericDomTreeConstruction.h
+++ b/llvm/include/llvm/Support/GenericDomTreeConstruction.h
@@ -103,31 +103,13 @@ struct SemiNCAInfo {
     // in progress, we need this information to continue it.
   }
 
-  template <bool Inversed> struct ChildrenGetter {
-    using ResultTy = SmallVector<NodePtr, 8>;
-
-    static ResultTy Get(NodePtr N, std::integral_constant<bool, false>) {
-      auto RChildren = reverse(children<NodePtr>(N));
-      return ResultTy(RChildren.begin(), RChildren.end());
-    }
-
-    static ResultTy Get(NodePtr N, std::integral_constant<bool, true>) {
-      auto IChildren = inverse_children<NodePtr>(N);
-      return ResultTy(IChildren.begin(), IChildren.end());
-    }
-
-    using Tag = std::integral_constant<bool, Inversed>;
-
-    // The function below is the core part of the batch updater. It allows the
-    // Depth Based Search algorithm to perform incremental updates in lockstep
-    // with updates to the CFG. We emulated lockstep CFG updates by getting its
-    // next snapshots by reverse-applying future updates.
-    static ResultTy Get(NodePtr N, BatchUpdatePtr BUI) {
-      if (!BUI)
-        return Get(N, Tag());
+  template <bool Inversed>
+  static SmallVector<NodePtr, 8> getChildren(NodePtr N, BatchUpdatePtr BUI) {
+    if (BUI)
       return BUI->PreViewCFG.template getChildren<Inversed>(N);
-    }
-  };
+    GraphDiffT GD;
+    return GD.template getChildren<Inversed>(N);
+  }
 
   NodePtr getIDom(NodePtr BB) const {
     auto InfoIt = NodeToInfo.find(BB);
@@ -194,8 +176,7 @@ struct SemiNCAInfo {
       NumToNode.push_back(BB);
 
       constexpr bool Direction = IsReverse != IsPostDom;  // XOR.
-      for (const NodePtr Succ :
-           ChildrenGetter<Direction>::Get(BB, BatchUpdates)) {
+      for (const NodePtr Succ : getChildren<Direction>(BB, BatchUpdates)) {
         const auto SIT = NodeToInfo.find(Succ);
         // Don't visit nodes more than once but remember to collect
         // ReverseChildren.
@@ -330,7 +311,7 @@ struct SemiNCAInfo {
   // to CFG nodes within infinite loops.
   static bool HasForwardSuccessors(const NodePtr N, BatchUpdatePtr BUI) {
     assert(N && "N must be a valid node");
-    return !ChildrenGetter<false>::Get(N, BUI).empty();
+    return !getChildren<false>(N, BUI).empty();
   }
 
   static NodePtr GetEntryNode(const DomTreeT &DT) {
@@ -748,8 +729,7 @@ struct SemiNCAInfo {
         //
         // Invariant: there is an optimal path from `To` to TN with the minimum
         // depth being CurrentLevel.
-        for (const NodePtr Succ :
-             ChildrenGetter<IsPostDom>::Get(TN->getBlock(), BUI)) {
+        for (const NodePtr Succ : getChildren<IsPostDom>(TN->getBlock(), BUI)) {
           const TreeNodePtr SuccTN = DT.getNode(Succ);
           assert(SuccTN &&
                  "Unreachable successor found at reachable insertion");
@@ -879,7 +859,7 @@ struct SemiNCAInfo {
     // the DomTree about it.
     // The check is O(N), so run it only in debug configuration.
     auto IsSuccessor = [BUI](const NodePtr SuccCandidate, const NodePtr Of) {
-      auto Successors = ChildrenGetter<IsPostDom>::Get(Of, BUI);
+      auto Successors = getChildren<IsPostDom>(Of, BUI);
       return llvm::find(Successors, SuccCandidate) != Successors.end();
     };
     (void)IsSuccessor;
@@ -967,7 +947,7 @@ struct SemiNCAInfo {
     LLVM_DEBUG(dbgs() << "IsReachableFromIDom " << BlockNamePrinter(TN)
                       << "\n");
     auto TNB = TN->getBlock();
-    for (const NodePtr Pred : ChildrenGetter<!IsPostDom>::Get(TNB, BUI)) {
+    for (const NodePtr Pred : getChildren<!IsPostDom>(TNB, BUI)) {
       LLVM_DEBUG(dbgs() << "\tPred " << BlockNamePrinter(Pred) << "\n");
       if (!DT.getNode(Pred)) continue;
 

From 9cf98d26e7b1204478cc13ae3df44a6843965c11 Mon Sep 17 00:00:00 2001
From: Richard Smith <richard@metafoo.co.uk>
Date: Tue, 28 Jul 2020 15:51:13 -0700
Subject: [PATCH 0378/1035] PR46637: Fix handling of placeholder types in
 trailing-return-types.

Only permit a placeholder type in a trailing-return-type if it would
also have been permitted in the decl-specifier sequence of a
corresponding declaration with no trailing-return-type. The standard
doesn't actually say this, but this is the only thing that makes sense.

Also fix handling of an 'auto' in a trailing-return-type in a parameter
of a generic lambda. We used to crash if we saw such a thing.
---
 clang/include/clang/Sema/DeclSpec.h           |   9 ++
 clang/include/clang/Sema/Sema.h               |   2 +
 clang/lib/Sema/SemaTemplateDeduction.cpp      |   7 ++
 clang/lib/Sema/SemaType.cpp                   | 117 +++++++++---------
 clang/test/SemaCXX/auto-type-from-cxx.cpp     |   3 +-
 .../SemaCXX/cxx1y-deduced-return-type.cpp     |   8 ++
 clang/test/SemaCXX/cxx1y-generic-lambdas.cpp  |   8 ++
 clang/test/SemaCXX/trailing-return-0x.cpp     |   7 ++
 .../SemaTemplate/temp_arg_nontype_cxx1z.cpp   |  11 ++
 9 files changed, 115 insertions(+), 57 deletions(-)

diff --git a/clang/include/clang/Sema/DeclSpec.h b/clang/include/clang/Sema/DeclSpec.h
index 8db03babfb1e9..0a22b5af7c644 100644
--- a/clang/include/clang/Sema/DeclSpec.h
+++ b/clang/include/clang/Sema/DeclSpec.h
@@ -2435,6 +2435,15 @@ class Declarator {
         return true;
     return false;
   }
+  /// Get the trailing return type appearing (at any level) within this
+  /// declarator.
+  ParsedType getTrailingReturnType() const {
+    for (const auto &Chunk : type_objects())
+      if (Chunk.Kind == DeclaratorChunk::Function &&
+          Chunk.Fun.hasTrailingReturnType())
+        return Chunk.Fun.getTrailingReturnType();
+    return ParsedType();
+  }
 
   /// \brief Sets a trailing requires clause for this declarator.
   void setTrailingRequiresClause(Expr *TRC) {
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 63e2d0d17fca2..fd4300f563a98 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -8179,6 +8179,8 @@ class Sema final {
   /// Completely replace the \c auto in \p TypeWithAuto by
   /// \p Replacement. This does not retain any \c auto type sugar.
   QualType ReplaceAutoType(QualType TypeWithAuto, QualType Replacement);
+  TypeSourceInfo *ReplaceAutoTypeSourceInfo(TypeSourceInfo *TypeWithAuto,
+                                            QualType Replacement);
 
   /// Result type of DeduceAutoType.
   enum DeduceAutoResult {
diff --git a/clang/lib/Sema/SemaTemplateDeduction.cpp b/clang/lib/Sema/SemaTemplateDeduction.cpp
index 8e7b4e1655ea9..1f7d0f0e8d973 100644
--- a/clang/lib/Sema/SemaTemplateDeduction.cpp
+++ b/clang/lib/Sema/SemaTemplateDeduction.cpp
@@ -4896,6 +4896,13 @@ QualType Sema::ReplaceAutoType(QualType TypeWithAuto,
       .TransformType(TypeWithAuto);
 }
 
+TypeSourceInfo *Sema::ReplaceAutoTypeSourceInfo(TypeSourceInfo *TypeWithAuto,
+                                                QualType TypeToReplaceAuto) {
+  return SubstituteDeducedTypeTransform(*this, TypeToReplaceAuto,
+                                        /*UseTypeSugar*/ false)
+      .TransformType(TypeWithAuto);
+}
+
 void Sema::DiagnoseAutoDeductionFailure(VarDecl *VDecl, Expr *Init) {
   if (isa<InitListExpr>(Init))
     Diag(VDecl->getLocation(),
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index 4c7eece68bca3..78c65ad01e475 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -3099,24 +3099,10 @@ static void diagnoseRedundantReturnTypeQualifiers(Sema &S, QualType RetTy,
                               D.getDeclSpec().getUnalignedSpecLoc());
 }
 
-static void CopyTypeConstraintFromAutoType(Sema &SemaRef, const AutoType *Auto,
-                                           AutoTypeLoc AutoLoc,
-                                           TemplateTypeParmDecl *TP,
-                                           SourceLocation EllipsisLoc) {
-
-  TemplateArgumentListInfo TAL(AutoLoc.getLAngleLoc(), AutoLoc.getRAngleLoc());
-  for (unsigned Idx = 0; Idx < AutoLoc.getNumArgs(); ++Idx)
-    TAL.addArgument(AutoLoc.getArgLoc(Idx));
-
-  SemaRef.AttachTypeConstraint(
-      AutoLoc.getNestedNameSpecifierLoc(), AutoLoc.getConceptNameInfo(),
-      AutoLoc.getNamedConcept(),
-      AutoLoc.hasExplicitTemplateArgs() ? &TAL : nullptr, TP, EllipsisLoc);
-}
-
-static QualType InventTemplateParameter(
-    TypeProcessingState &state, QualType T, TypeSourceInfo *TSI, AutoType *Auto,
-    InventedTemplateParameterInfo &Info) {
+static std::pair<QualType, TypeSourceInfo *>
+InventTemplateParameter(TypeProcessingState &state, QualType T,
+                        TypeSourceInfo *TrailingTSI, AutoType *Auto,
+                        InventedTemplateParameterInfo &Info) {
   Sema &S = state.getSema();
   Declarator &D = state.getDeclarator();
 
@@ -3141,13 +3127,25 @@ static QualType InventTemplateParameter(
           IsParameterPack, /*HasTypeConstraint=*/Auto->isConstrained());
   InventedTemplateParam->setImplicit();
   Info.TemplateParams.push_back(InventedTemplateParam);
-  // Attach type constraints
+
+  // Attach type constraints to the new parameter.
   if (Auto->isConstrained()) {
-    if (TSI) {
-      CopyTypeConstraintFromAutoType(
-          S, Auto, TSI->getTypeLoc().getContainedAutoTypeLoc(),
-          InventedTemplateParam, D.getEllipsisLoc());
+    if (TrailingTSI) {
+      // The 'auto' appears in a trailing return type we've already built;
+      // extract its type constraints to attach to the template parameter.
+      AutoTypeLoc AutoLoc = TrailingTSI->getTypeLoc().getContainedAutoTypeLoc();
+      TemplateArgumentListInfo TAL(AutoLoc.getLAngleLoc(), AutoLoc.getRAngleLoc());
+      for (unsigned Idx = 0; Idx < AutoLoc.getNumArgs(); ++Idx)
+        TAL.addArgument(AutoLoc.getArgLoc(Idx));
+
+      S.AttachTypeConstraint(AutoLoc.getNestedNameSpecifierLoc(),
+                             AutoLoc.getConceptNameInfo(),
+                             AutoLoc.getNamedConcept(),
+                             AutoLoc.hasExplicitTemplateArgs() ? &TAL : nullptr,
+                             InventedTemplateParam, D.getEllipsisLoc());
     } else {
+      // The 'auto' appears in the decl-specifiers; we've not finished forming
+      // TypeSourceInfo for it yet.
       TemplateIdAnnotation *TemplateId = D.getDeclSpec().getRepAsTemplateId();
       TemplateArgumentListInfo TemplateArgsInfo;
       if (TemplateId->LAngleLoc.isValid()) {
@@ -3165,15 +3163,16 @@ static QualType InventTemplateParameter(
     }
   }
 
-  // If TSI is nullptr, this is a constrained declspec auto and the type
-  // constraint will be attached later in TypeSpecLocFiller
-
   // Replace the 'auto' in the function parameter with this invented
   // template type parameter.
   // FIXME: Retain some type sugar to indicate that this was written
   //  as 'auto'?
-  return state.ReplaceAutoType(
-      T, QualType(InventedTemplateParam->getTypeForDecl(), 0));
+  QualType Replacement(InventedTemplateParam->getTypeForDecl(), 0);
+  QualType NewT = state.ReplaceAutoType(T, Replacement);
+  TypeSourceInfo *NewTSI =
+      TrailingTSI ? S.ReplaceAutoTypeSourceInfo(TrailingTSI, Replacement)
+                  : nullptr;
+  return {NewT, NewTSI};
 }
 
 static TypeSourceInfo *
@@ -3232,8 +3231,19 @@ static QualType GetDeclSpecTypeForDeclarator(TypeProcessingState &state,
   if (!D.getAttributes().empty())
     distributeTypeAttrsFromDeclarator(state, T);
 
+  // Find the deduced type in this type. Look in the trailing return type if we
+  // have one, otherwise in the DeclSpec type.
+  // FIXME: The standard wording doesn't currently describe this.
+  DeducedType *Deduced = T->getContainedDeducedType();
+  bool DeducedIsTrailingReturnType = false;
+  if (Deduced && isa<AutoType>(Deduced) && D.hasTrailingReturnType()) {
+    QualType T = SemaRef.GetTypeFromParser(D.getTrailingReturnType());
+    Deduced = T.isNull() ? nullptr : T->getContainedDeducedType();
+    DeducedIsTrailingReturnType = true;
+  }
+
   // C++11 [dcl.spec.auto]p5: reject 'auto' if it is not in an allowed context.
-  if (DeducedType *Deduced = T->getContainedDeducedType()) {
+  if (Deduced) {
     AutoType *Auto = dyn_cast<AutoType>(Deduced);
     int Error = -1;
 
@@ -3267,10 +3277,6 @@ static QualType GetDeclSpecTypeForDeclarator(TypeProcessingState &state,
         } else if (!SemaRef.getCurScope()->isFunctionDeclarationScope()) {
           Error = 21;
           break;
-        } else if (D.hasTrailingReturnType()) {
-          // This might be OK, but we'll need to convert the trailing return
-          // type later.
-          break;
         }
 
         Info = &SemaRef.InventedParameterInfos.back();
@@ -3284,7 +3290,12 @@ static QualType GetDeclSpecTypeForDeclarator(TypeProcessingState &state,
         Info = SemaRef.getCurLambda();
         assert(Info && "No LambdaScopeInfo on the stack!");
       }
-      T = InventTemplateParameter(state, T, nullptr, Auto, *Info);
+
+      // We'll deal with inventing template parameters for 'auto' in trailing
+      // return types when we pick up the trailing return type when processing
+      // the function chunk.
+      if (!DeducedIsTrailingReturnType)
+        T = InventTemplateParameter(state, T, nullptr, Auto, *Info).first;
       break;
     }
     case DeclaratorContext::MemberContext: {
@@ -3382,20 +3393,6 @@ static QualType GetDeclSpecTypeForDeclarator(TypeProcessingState &state,
         (!SemaRef.getLangOpts().CPlusPlus11 || !IsCXXAutoType))
       Error = 13;
 
-    bool HaveTrailing = false;
-
-    // C++11 [dcl.spec.auto]p2: 'auto' is always fine if the declarator
-    // contains a trailing return type. That is only legal at the outermost
-    // level. Check all declarator chunks (outermost first) anyway, to give
-    // better diagnostics.
-    // We don't support '__auto_type' with trailing return types.
-    // FIXME: Should we only do this for 'auto' and not 'decltype(auto)'?
-    if (SemaRef.getLangOpts().CPlusPlus11 && IsCXXAutoType &&
-        D.hasTrailingReturnType()) {
-      HaveTrailing = true;
-      Error = -1;
-    }
-
     SourceRange AutoRange = D.getDeclSpec().getTypeSpecTypeLoc();
     if (D.getName().getKind() == UnqualifiedIdKind::IK_ConversionFunctionId)
       AutoRange = D.getName().getSourceRange();
@@ -3425,8 +3422,7 @@ static QualType GetDeclSpecTypeForDeclarator(TypeProcessingState &state,
 
       T = SemaRef.Context.IntTy;
       D.setInvalidType(true);
-    } else if (Auto && !HaveTrailing &&
-               D.getContext() != DeclaratorContext::LambdaExprContext) {
+    } else if (Auto && D.getContext() != DeclaratorContext::LambdaExprContext) {
       // If there was a trailing return type, we already got
       // warn_cxx98_compat_trailing_return_type in the parser.
       SemaRef.Diag(AutoRange.getBegin(),
@@ -4879,12 +4875,21 @@ static TypeSourceInfo *GetFullTypeForDeclarator(TypeProcessingState &state,
             // An error occurred parsing the trailing return type.
             T = Context.IntTy;
             D.setInvalidType(true);
-          } else if (S.getLangOpts().CPlusPlus20)
-            // Handle cases like: `auto f() -> auto` or `auto f() -> C auto`.
-            if (AutoType *Auto = T->getContainedAutoType())
-              if (S.getCurScope()->isFunctionDeclarationScope())
-                T = InventTemplateParameter(state, T, TInfo, Auto,
-                                            S.InventedParameterInfos.back());
+          } else if (AutoType *Auto = T->getContainedAutoType()) {
+            // If the trailing return type contains an `auto`, we may need to
+            // invent a template parameter for it, for cases like
+            // `auto f() -> C auto` or `[](auto (*p) -> auto) {}`.
+            InventedTemplateParameterInfo *InventedParamInfo = nullptr;
+            if (D.getContext() == DeclaratorContext::PrototypeContext)
+              InventedParamInfo = &S.InventedParameterInfos.back();
+            else if (D.getContext() ==
+                     DeclaratorContext::LambdaExprParameterContext)
+              InventedParamInfo = S.getCurLambda();
+            if (InventedParamInfo) {
+              std::tie(T, TInfo) = InventTemplateParameter(
+                  state, T, TInfo, Auto, *InventedParamInfo);
+            }
+          }
         } else {
           // This function type is not the type of the entity being declared,
           // so checking the 'auto' is not the responsibility of this chunk.
diff --git a/clang/test/SemaCXX/auto-type-from-cxx.cpp b/clang/test/SemaCXX/auto-type-from-cxx.cpp
index 961402f7d9331..21620f73be7e9 100644
--- a/clang/test/SemaCXX/auto-type-from-cxx.cpp
+++ b/clang/test/SemaCXX/auto-type-from-cxx.cpp
@@ -4,7 +4,8 @@ struct A {
     operator __auto_type() {} // expected-error {{'__auto_type' not allowed in conversion function type}}
 };
 
-__auto_type a() -> int; // expected-error {{'__auto_type' not allowed in function return type}}
+__auto_type a() -> int; // expected-error {{function with trailing return type must specify return type 'auto'}}
+__auto_type a2(); // expected-error {{'__auto_type' not allowed in function return type}}
 template <typename T>
 __auto_type b() { return T::x; } // expected-error {{'__auto_type' not allowed in function return type}}
 auto c() -> __auto_type { __builtin_unreachable(); } // expected-error {{'__auto_type' not allowed in function return type}}
diff --git a/clang/test/SemaCXX/cxx1y-deduced-return-type.cpp b/clang/test/SemaCXX/cxx1y-deduced-return-type.cpp
index 687ecf28ee3d7..958728b104876 100644
--- a/clang/test/SemaCXX/cxx1y-deduced-return-type.cpp
+++ b/clang/test/SemaCXX/cxx1y-deduced-return-type.cpp
@@ -620,3 +620,11 @@ namespace PR33222 {
   // FIXME: suppress this follow-on error: expected-error@-1 {{cannot initialize}}
   template<> int B<char[3]>::q() { return 0; } // expected-error {{return type}}
 }
+
+namespace PR46637 {
+  using A = auto () -> auto; // expected-error {{'auto' not allowed in type alias}}
+  using B = auto (*)() -> auto; // expected-error {{'auto' not allowed in type alias}}
+  template<auto (*)() -> auto> struct X {}; // expected-error {{'auto' not allowed in template parameter until C++17}}
+  template<typename T> struct Y { T x; };
+  Y<auto() -> auto> y; // expected-error {{'auto' not allowed in template argument}}
+}
diff --git a/clang/test/SemaCXX/cxx1y-generic-lambdas.cpp b/clang/test/SemaCXX/cxx1y-generic-lambdas.cpp
index 13ab7aae6c32b..52caaa59dd30f 100644
--- a/clang/test/SemaCXX/cxx1y-generic-lambdas.cpp
+++ b/clang/test/SemaCXX/cxx1y-generic-lambdas.cpp
@@ -1012,3 +1012,11 @@ namespace PR32638 {
     [](auto x) noexcept(noexcept(x)) { } (0);
  }
 }
+
+namespace PR46637 {
+  auto x = [](auto (*p)()) { return p(); };
+  auto y = [](auto (*p)() -> auto) { return p(); };
+  int f();
+  void *v = x(f); // expected-error {{cannot initialize a variable of type 'void *' with an rvalue of type 'int'}}
+  void *w = y(f); // expected-error {{cannot initialize a variable of type 'void *' with an rvalue of type 'int'}}
+}
diff --git a/clang/test/SemaCXX/trailing-return-0x.cpp b/clang/test/SemaCXX/trailing-return-0x.cpp
index c6b22c54be0c1..4834aaf5277a8 100644
--- a/clang/test/SemaCXX/trailing-return-0x.cpp
+++ b/clang/test/SemaCXX/trailing-return-0x.cpp
@@ -104,3 +104,10 @@ namespace PR16273 {
   };
 }
 
+namespace PR46637 {
+  using A = auto () -> auto; // expected-error {{'auto' not allowed in function return type}}
+  using B = auto (*)() -> auto; // expected-error {{'auto' not allowed in function return type}}
+  template<auto (*)() -> auto> struct X {}; // expected-error {{'auto' not allowed in function return type}}
+  template<typename T> struct Y { T x; };
+  Y<auto() -> auto> y; // expected-error {{'auto' not allowed in function return type}}
+}
diff --git a/clang/test/SemaTemplate/temp_arg_nontype_cxx1z.cpp b/clang/test/SemaTemplate/temp_arg_nontype_cxx1z.cpp
index fdcc5006f5890..7538de3309027 100644
--- a/clang/test/SemaTemplate/temp_arg_nontype_cxx1z.cpp
+++ b/clang/test/SemaTemplate/temp_arg_nontype_cxx1z.cpp
@@ -448,3 +448,14 @@ namespace PR42108 {
     A<T{}>(); // expected-error {{non-type template argument is not a constant expression}} expected-note 2{{temporary}}
   }
 }
+
+namespace PR46637 {
+  template<auto (*f)() -> auto> struct X { // expected-note {{here}}
+    auto call() { return f(); }
+  };
+  X<nullptr> x; // expected-error {{incompatible initializer}}
+
+  void *f();
+  X<f> y;
+  int n = y.call(); // expected-error {{cannot initialize a variable of type 'int' with an rvalue of type 'void *'}}
+}

From 80bd6ae13ea23d453a1f45d6ccdbfc7d0c877bb0 Mon Sep 17 00:00:00 2001
From: Zahira Ammarguellat <Zahira.Ammarguellat@intel.com>
Date: Tue, 28 Jul 2020 17:59:33 -0500
Subject: [PATCH 0379/1035] On Windows build, making the /bigobj flag global ,
 instead of passing it per file.

To avoid having this flag be passed in per/file manner, we are instead
passing it globally.

This fixes this bug: https://bugs.llvm.org/show_bug.cgi?id=46733

Reviewed-by: aaron.ballman, beanz, meinersbur

Differential Revision: https://reviews.llvm.org/D84038
---
 clang/lib/ASTMatchers/Dynamic/CMakeLists.txt | 9 ---------
 clang/lib/CodeGen/CMakeLists.txt             | 4 ----
 clang/lib/Sema/CMakeLists.txt                | 9 ---------
 clang/unittests/AST/CMakeLists.txt           | 3 ---
 clang/unittests/ASTMatchers/CMakeLists.txt   | 9 ---------
 clang/unittests/Tooling/CMakeLists.txt       | 8 --------
 lldb/source/API/CMakeLists.txt               | 3 ---
 llvm/cmake/modules/HandleLLVMOptions.cmake   | 4 ++++
 llvm/lib/Passes/CMakeLists.txt               | 4 ----
 mlir/lib/Dialect/SPIRV/CMakeLists.txt        | 3 ---
 10 files changed, 4 insertions(+), 52 deletions(-)

diff --git a/clang/lib/ASTMatchers/Dynamic/CMakeLists.txt b/clang/lib/ASTMatchers/Dynamic/CMakeLists.txt
index adb891575b308..7110a503869e6 100644
--- a/clang/lib/ASTMatchers/Dynamic/CMakeLists.txt
+++ b/clang/lib/ASTMatchers/Dynamic/CMakeLists.txt
@@ -3,15 +3,6 @@ set(LLVM_LINK_COMPONENTS
   Support
 )
 
-# The registry source file ends up generating a lot of sections for each
-# matcher. Each matcher appears to get a vtable and several methods. Each
-# method needs .text, .pdata, .xdata, and .debug sections, adding to the
-# section multiplier. By default MSVC has a 2^16 limit on the number of
-# sections in an object file, and this needs more than that.
-if (MSVC)
-  set_source_files_properties(Registry.cpp PROPERTIES COMPILE_FLAGS /bigobj)
-endif()
-
 add_clang_library(clangDynamicASTMatchers
   Diagnostics.cpp
   Marshallers.cpp
diff --git a/clang/lib/CodeGen/CMakeLists.txt b/clang/lib/CodeGen/CMakeLists.txt
index 8afd7219fbe1e..88647a2007fb0 100644
--- a/clang/lib/CodeGen/CMakeLists.txt
+++ b/clang/lib/CodeGen/CMakeLists.txt
@@ -26,10 +26,6 @@ set(LLVM_LINK_COMPONENTS
   TransformUtils
   )
 
-if (MSVC)
-  set_source_files_properties(CodeGenModule.cpp PROPERTIES COMPILE_FLAGS /bigobj)
-endif()
-
 add_clang_library(clangCodeGen
   BackendUtil.cpp
   CGAtomic.cpp
diff --git a/clang/lib/Sema/CMakeLists.txt b/clang/lib/Sema/CMakeLists.txt
index adadc06e209c3..042ff561664d7 100644
--- a/clang/lib/Sema/CMakeLists.txt
+++ b/clang/lib/Sema/CMakeLists.txt
@@ -3,15 +3,6 @@ set(LLVM_LINK_COMPONENTS
   Support
   )
 
-if (MSVC)
-  set_source_files_properties(SemaDeclAttr.cpp PROPERTIES COMPILE_FLAGS /bigobj)
-  set_source_files_properties(SemaExpr.cpp PROPERTIES COMPILE_FLAGS /bigobj)
-  set_source_files_properties(SemaExprCXX.cpp PROPERTIES COMPILE_FLAGS /bigobj)
-  set_source_files_properties(SemaTemplate.cpp PROPERTIES COMPILE_FLAGS /bigobj)
-  set_source_files_properties(SemaTemplateDeduction.cpp PROPERTIES COMPILE_FLAGS /bigobj)
-  set_source_files_properties(SemaOpenMP.cpp PROPERTIES COMPILE_FLAGS /bigobj)
-endif()
-
 clang_tablegen(OpenCLBuiltins.inc -gen-clang-opencl-builtins
   SOURCE OpenCLBuiltins.td
   TARGET ClangOpenCLBuiltinsImpl
diff --git a/clang/unittests/AST/CMakeLists.txt b/clang/unittests/AST/CMakeLists.txt
index 2e750ac9ea925..2d5d0172afedc 100644
--- a/clang/unittests/AST/CMakeLists.txt
+++ b/clang/unittests/AST/CMakeLists.txt
@@ -3,9 +3,6 @@ set(LLVM_LINK_COMPONENTS
   Support
   )
 
-if (MSVC)
-  set_source_files_properties(ASTImporterTest.cpp PROPERTIES COMPILE_FLAGS /bigobj)
-endif()
 
 add_clang_unittest(ASTTests
   ASTContextParentMapTest.cpp
diff --git a/clang/unittests/ASTMatchers/CMakeLists.txt b/clang/unittests/ASTMatchers/CMakeLists.txt
index c9ac6c57d6897..b40b3886dcd2b 100644
--- a/clang/unittests/ASTMatchers/CMakeLists.txt
+++ b/clang/unittests/ASTMatchers/CMakeLists.txt
@@ -3,15 +3,6 @@ set(LLVM_LINK_COMPONENTS
   Support
   )
 
-# By default MSVC has a 2^16 limit on the number of sections in an object file,
-# and this needs more than that.
-if (MSVC)
-  set_source_files_properties(InternalASTMatchersTest.cpp PROPERTIES COMPILE_FLAGS /bigobj)
-  set_source_files_properties(NodeMatchersTest.cpp PROPERTIES COMPILE_FLAGS /bigobj)
-  set_source_files_properties(NarrowingMatchersTest.cpp PROPERTIES COMPILE_FLAGS /bigobj)
-  set_source_files_properties(ASTTraversalMatchersTest.cpp PROPERTIES COMPILE_FLAGS /bigobj)
-endif()
-
 add_clang_unittest(ASTMatchersTests
   ASTMatchersInternalTest.cpp
   ASTMatchersNodeTest.cpp
diff --git a/clang/unittests/Tooling/CMakeLists.txt b/clang/unittests/Tooling/CMakeLists.txt
index c439f5a786375..f290c3d2bedee 100644
--- a/clang/unittests/Tooling/CMakeLists.txt
+++ b/clang/unittests/Tooling/CMakeLists.txt
@@ -4,14 +4,6 @@ set(LLVM_LINK_COMPONENTS
   Support
   )
 
-# By default MSVC has a 2^16 limit on the number of sections in an object file,
-# and this needs more than that.
-if (MSVC)
-  set_source_files_properties(RecursiveASTVisitorTest.cpp PROPERTIES COMPILE_FLAGS /bigobj)
-  set_source_files_properties(RecursiveASTVisitorTestExprVisitor.cpp PROPERTIES COMPILE_FLAGS /bigobj)
-  set_source_files_properties(RecursiveASTVisitorTests/Callbacks.cpp PROPERTIES COMPILE_FLAGS /bigobj)
-  set_source_files_properties(SourceCodeTest.cpp PROPERTIES COMPILE_FLAGS /bigobj)
-endif()
 
 add_clang_unittest(ToolingTests
   ASTSelectionTest.cpp
diff --git a/lldb/source/API/CMakeLists.txt b/lldb/source/API/CMakeLists.txt
index ce6a7ec830fce..34f03e11c007a 100644
--- a/lldb/source/API/CMakeLists.txt
+++ b/lldb/source/API/CMakeLists.txt
@@ -126,9 +126,6 @@ if(PYTHON_RPATH)
   set_property(TARGET liblldb APPEND PROPERTY BUILD_RPATH   "${PYTHON_RPATH}")
 endif()
 
-if (MSVC)
-  set_source_files_properties(SBReproducer.cpp PROPERTIES COMPILE_FLAGS /bigobj)
-endif()
 
 if(lldb_python_wrapper)
   add_dependencies(liblldb swig_wrapper)
diff --git a/llvm/cmake/modules/HandleLLVMOptions.cmake b/llvm/cmake/modules/HandleLLVMOptions.cmake
index 89f7016a7db44..0df9334363b43 100644
--- a/llvm/cmake/modules/HandleLLVMOptions.cmake
+++ b/llvm/cmake/modules/HandleLLVMOptions.cmake
@@ -473,6 +473,10 @@ if( MSVC )
       endif()
     endif()
   endif()
+  # By default MSVC has a 2^16 limit on the number of sections in an object file,
+  # but in many objects files need more than that. This flag is to increase the
+  # number of sections.
+  append("/bigobj" CMAKE_CXX_FLAGS)
 endif( MSVC )
 
 # Warnings-as-errors handling for GCC-compatible compilers:
diff --git a/llvm/lib/Passes/CMakeLists.txt b/llvm/lib/Passes/CMakeLists.txt
index 371a21c113dcb..35fde7ba4e28d 100644
--- a/llvm/lib/Passes/CMakeLists.txt
+++ b/llvm/lib/Passes/CMakeLists.txt
@@ -1,7 +1,3 @@
-if (MSVC)
-  set_source_files_properties(PassBuilder.cpp PROPERTIES COMPILE_FLAGS /bigobj)
-endif()
-
 add_llvm_component_library(LLVMPasses
   PassBuilder.cpp
   PassPlugin.cpp
diff --git a/mlir/lib/Dialect/SPIRV/CMakeLists.txt b/mlir/lib/Dialect/SPIRV/CMakeLists.txt
index fd34c14628041..10f06fdb88615 100644
--- a/mlir/lib/Dialect/SPIRV/CMakeLists.txt
+++ b/mlir/lib/Dialect/SPIRV/CMakeLists.txt
@@ -1,6 +1,3 @@
-if (MSVC)
-  set_source_files_properties(SPIRVDialect.cpp PROPERTIES COMPILE_FLAGS /bigobj)
-endif()
 
 set(LLVM_TARGET_DEFINITIONS SPIRVCanonicalization.td)
 mlir_tablegen(SPIRVCanonicalization.inc -gen-rewriters)

From a3d1f88fa7da3dfc0b4319f2e4eb7374fa60b819 Mon Sep 17 00:00:00 2001
From: "Joel E. Denny" <jdenny.ornl@gmail.com>
Date: Tue, 28 Jul 2020 18:06:05 -0400
Subject: [PATCH 0380/1035] [OpenMP][NFC] Consolidate `to` and `from` clause
 modifiers

`to` and `from` clauses take the same modifiers, which are called
"motion modifiers" in TR8, so implement handling of their modifiers
once not twice.  This will make it easier to implement additional
motion modifiers in the future.

Reviewed By: ABataev

Differential Revision: https://reviews.llvm.org/D84710
---
 clang/include/clang/Basic/OpenMPKinds.def | 17 ++++---------
 clang/include/clang/Basic/OpenMPKinds.h   | 18 ++++----------
 clang/lib/Basic/OpenMPKinds.cpp           | 30 ++++++-----------------
 clang/lib/Parse/ParseOpenMP.cpp           | 19 +++-----------
 4 files changed, 21 insertions(+), 63 deletions(-)

diff --git a/clang/include/clang/Basic/OpenMPKinds.def b/clang/include/clang/Basic/OpenMPKinds.def
index 275c4fcdabb25..04ecbeaaa03e2 100644
--- a/clang/include/clang/Basic/OpenMPKinds.def
+++ b/clang/include/clang/Basic/OpenMPKinds.def
@@ -29,11 +29,8 @@
 #ifndef OPENMP_MAP_MODIFIER_KIND
 #define OPENMP_MAP_MODIFIER_KIND(Name)
 #endif
-#ifndef OPENMP_TO_MODIFIER_KIND
-#define OPENMP_TO_MODIFIER_KIND(Name)
-#endif
-#ifndef OPENMP_FROM_MODIFIER_KIND
-#define OPENMP_FROM_MODIFIER_KIND(Name)
+#ifndef OPENMP_MOTION_MODIFIER_KIND
+#define OPENMP_MOTION_MODIFIER_KIND(Name)
 #endif
 #ifndef OPENMP_DIST_SCHEDULE_KIND
 #define OPENMP_DIST_SCHEDULE_KIND(Name)
@@ -126,11 +123,8 @@ OPENMP_MAP_MODIFIER_KIND(close)
 OPENMP_MAP_MODIFIER_KIND(mapper)
 OPENMP_MAP_MODIFIER_KIND(present)
 
-// Modifiers for 'to' clause.
-OPENMP_TO_MODIFIER_KIND(mapper)
-
-// Modifiers for 'from' clause.
-OPENMP_FROM_MODIFIER_KIND(mapper)
+// Modifiers for 'to' or 'from' clause.
+OPENMP_MOTION_MODIFIER_KIND(mapper)
 
 // Static attributes for 'dist_schedule' clause.
 OPENMP_DIST_SCHEDULE_KIND(static)
@@ -163,8 +157,7 @@ OPENMP_REDUCTION_MODIFIER(task)
 #undef OPENMP_ATOMIC_DEFAULT_MEM_ORDER_KIND
 #undef OPENMP_MAP_KIND
 #undef OPENMP_MAP_MODIFIER_KIND
-#undef OPENMP_TO_MODIFIER_KIND
-#undef OPENMP_FROM_MODIFIER_KIND
+#undef OPENMP_MOTION_MODIFIER_KIND
 #undef OPENMP_DIST_SCHEDULE_KIND
 #undef OPENMP_DEFAULTMAP_KIND
 #undef OPENMP_DEFAULTMAP_MODIFIER
diff --git a/clang/include/clang/Basic/OpenMPKinds.h b/clang/include/clang/Basic/OpenMPKinds.h
index dc6198f93f9d8..3e9b5c4a8b14a 100644
--- a/clang/include/clang/Basic/OpenMPKinds.h
+++ b/clang/include/clang/Basic/OpenMPKinds.h
@@ -86,20 +86,12 @@ enum OpenMPMapModifierKind {
 static constexpr unsigned NumberOfOMPMapClauseModifiers =
     OMPC_MAP_MODIFIER_last - OMPC_MAP_MODIFIER_unknown - 1;
 
-/// OpenMP modifier kind for 'to' clause.
-enum OpenMPToModifierKind {
-#define OPENMP_TO_MODIFIER_KIND(Name) \
-  OMPC_TO_MODIFIER_##Name,
+/// OpenMP modifier kind for 'to' or 'from' clause.
+enum OpenMPMotionModifierKind {
+#define OPENMP_MOTION_MODIFIER_KIND(Name) \
+  OMPC_MOTION_MODIFIER_##Name,
 #include "clang/Basic/OpenMPKinds.def"
-  OMPC_TO_MODIFIER_unknown
-};
-
-/// OpenMP modifier kind for 'from' clause.
-enum OpenMPFromModifierKind {
-#define OPENMP_FROM_MODIFIER_KIND(Name) \
-  OMPC_FROM_MODIFIER_##Name,
-#include "clang/Basic/OpenMPKinds.def"
-  OMPC_FROM_MODIFIER_unknown
+  OMPC_MOTION_MODIFIER_unknown
 };
 
 /// OpenMP attributes for 'dist_schedule' clause.
diff --git a/clang/lib/Basic/OpenMPKinds.cpp b/clang/lib/Basic/OpenMPKinds.cpp
index 4807702e896e8..da362f99ed297 100644
--- a/clang/lib/Basic/OpenMPKinds.cpp
+++ b/clang/lib/Basic/OpenMPKinds.cpp
@@ -64,17 +64,12 @@ unsigned clang::getOpenMPSimpleClauseType(OpenMPClauseKind Kind, StringRef Str,
     return Type;
   }
   case OMPC_to:
-    return llvm::StringSwitch<unsigned>(Str)
-#define OPENMP_TO_MODIFIER_KIND(Name)                                          \
-  .Case(#Name, static_cast<unsigned>(OMPC_TO_MODIFIER_##Name))
-#include "clang/Basic/OpenMPKinds.def"
-        .Default(OMPC_TO_MODIFIER_unknown);
   case OMPC_from:
     return llvm::StringSwitch<unsigned>(Str)
-#define OPENMP_FROM_MODIFIER_KIND(Name)                                     \
-  .Case(#Name, static_cast<unsigned>(OMPC_FROM_MODIFIER_##Name))
+#define OPENMP_MOTION_MODIFIER_KIND(Name)                                      \
+  .Case(#Name, static_cast<unsigned>(OMPC_MOTION_MODIFIER_##Name))
 #include "clang/Basic/OpenMPKinds.def"
-        .Default(OMPC_FROM_MODIFIER_unknown);
+        .Default(OMPC_MOTION_MODIFIER_unknown);
   case OMPC_dist_schedule:
     return llvm::StringSwitch<OpenMPDistScheduleClauseKind>(Str)
 #define OPENMP_DIST_SCHEDULE_KIND(Name) .Case(#Name, OMPC_DIST_SCHEDULE_##Name)
@@ -258,29 +253,18 @@ const char *clang::getOpenMPSimpleClauseTypeName(OpenMPClauseKind Kind,
     }
     llvm_unreachable("Invalid OpenMP 'map' clause type");
   case OMPC_to:
-    switch (Type) {
-    case OMPC_TO_MODIFIER_unknown:
-      return "unknown";
-#define OPENMP_TO_MODIFIER_KIND(Name)                                          \
-  case OMPC_TO_MODIFIER_##Name:                                                \
-    return #Name;
-#include "clang/Basic/OpenMPKinds.def"
-    default:
-      break;
-    }
-    llvm_unreachable("Invalid OpenMP 'to' clause type");
   case OMPC_from:
     switch (Type) {
-    case OMPC_FROM_MODIFIER_unknown:
+    case OMPC_MOTION_MODIFIER_unknown:
       return "unknown";
-#define OPENMP_FROM_MODIFIER_KIND(Name)                                        \
-  case OMPC_FROM_MODIFIER_##Name:                                              \
+#define OPENMP_MOTION_MODIFIER_KIND(Name)                                      \
+  case OMPC_MOTION_MODIFIER_##Name:                                            \
     return #Name;
 #include "clang/Basic/OpenMPKinds.def"
     default:
       break;
     }
-    llvm_unreachable("Invalid OpenMP 'from' clause type");
+    llvm_unreachable("Invalid OpenMP 'to' or 'from' clause type");
   case OMPC_dist_schedule:
     switch (Type) {
     case OMPC_DIST_SCHEDULE_unknown:
diff --git a/clang/lib/Parse/ParseOpenMP.cpp b/clang/lib/Parse/ParseOpenMP.cpp
index 79744a7519e04..6a5d7c604c6bb 100644
--- a/clang/lib/Parse/ParseOpenMP.cpp
+++ b/clang/lib/Parse/ParseOpenMP.cpp
@@ -3442,21 +3442,10 @@ bool Parser::ParseOpenMPVarList(OpenMPDirectiveKind DKind,
       Data.ColonLoc = ConsumeToken();
   } else if (Kind == OMPC_to || Kind == OMPC_from) {
     if (Tok.is(tok::identifier)) {
-      bool IsMapperModifier = false;
-      if (Kind == OMPC_to) {
-        auto Modifier =
-            static_cast<OpenMPToModifierKind>(getOpenMPSimpleClauseType(
-                Kind, PP.getSpelling(Tok), getLangOpts().OpenMP));
-        if (Modifier == OMPC_TO_MODIFIER_mapper)
-          IsMapperModifier = true;
-      } else {
-        auto Modifier =
-            static_cast<OpenMPFromModifierKind>(getOpenMPSimpleClauseType(
-                Kind, PP.getSpelling(Tok), getLangOpts().OpenMP));
-        if (Modifier == OMPC_FROM_MODIFIER_mapper)
-          IsMapperModifier = true;
-      }
-      if (IsMapperModifier) {
+      auto Modifier =
+          static_cast<OpenMPMotionModifierKind>(getOpenMPSimpleClauseType(
+              Kind, PP.getSpelling(Tok), getLangOpts().OpenMP));
+      if (Modifier == OMPC_MOTION_MODIFIER_mapper) {
         // Parse the mapper modifier.
         ConsumeToken();
         IsInvalidMapperModifier = parseMapperModifier(Data);

From 3c3faae497046be706df29e16c9fbccb7e1fce09 Mon Sep 17 00:00:00 2001
From: "Joel E. Denny" <jdenny.ornl@gmail.com>
Date: Tue, 28 Jul 2020 18:06:24 -0400
Subject: [PATCH 0381/1035] [OpenMP] Implement TR8 `present` motion modifier in
 Clang (1/2)

This patch implements Clang front end support for the OpenMP TR8
`present` motion modifier for `omp target update` directives.  The
next patch in this series implements OpenMP runtime support.

Reviewed By: ABataev

Differential Revision: https://reviews.llvm.org/D84711
---
 clang/include/clang/AST/OpenMPClause.h        | 187 +++++++++++++-
 .../clang/Basic/DiagnosticSemaKinds.td        |   2 +
 clang/include/clang/Basic/OpenMPKinds.def     |   1 +
 clang/include/clang/Basic/OpenMPKinds.h       |   4 +
 clang/include/clang/Parse/Parser.h            |   3 +
 clang/include/clang/Sema/Sema.h               |  23 +-
 clang/lib/AST/OpenMPClause.cpp                |  94 +++----
 clang/lib/Basic/OpenMPKinds.cpp               |   8 +-
 clang/lib/CodeGen/CGOpenMPRuntime.cpp         |  22 +-
 clang/lib/Parse/ParseOpenMP.cpp               |  45 +++-
 clang/lib/Sema/SemaOpenMP.cpp                 |  78 ++++--
 clang/lib/Sema/TreeTransform.h                |  42 ++--
 clang/lib/Serialization/ASTReader.cpp         |  12 +
 clang/lib/Serialization/ASTWriter.cpp         |  10 +
 clang/test/OpenMP/declare_mapper_ast_print.c  |  14 ++
 clang/test/OpenMP/declare_mapper_codegen.cpp  | 238 ++++++++++++++++++
 clang/test/OpenMP/target_update_ast_print.cpp |  24 ++
 clang/test/OpenMP/target_update_codegen.cpp   |  63 +++++
 clang/test/OpenMP/target_update_messages.cpp  | 117 ++++++++-
 19 files changed, 868 insertions(+), 119 deletions(-)

diff --git a/clang/include/clang/AST/OpenMPClause.h b/clang/include/clang/AST/OpenMPClause.h
index c649502f765b4..46c26c6c77de2 100644
--- a/clang/include/clang/AST/OpenMPClause.h
+++ b/clang/include/clang/AST/OpenMPClause.h
@@ -6329,8 +6329,20 @@ class OMPToClause final : public OMPMappableExprListClause<OMPToClause>,
   friend OMPVarListClause;
   friend TrailingObjects;
 
+  /// Motion-modifiers for the 'to' clause.
+  OpenMPMotionModifierKind MotionModifiers[NumberOfOMPMotionModifiers] = {
+      OMPC_MOTION_MODIFIER_unknown, OMPC_MOTION_MODIFIER_unknown};
+
+  /// Location of motion-modifiers for the 'to' clause.
+  SourceLocation MotionModifiersLoc[NumberOfOMPMotionModifiers];
+
+  /// Colon location.
+  SourceLocation ColonLoc;
+
   /// Build clause with number of variables \a NumVars.
   ///
+  /// \param TheMotionModifiers Motion-modifiers.
+  /// \param TheMotionModifiersLoc Locations of motion-modifiers.
   /// \param MapperQualifierLoc C++ nested name specifier for the associated
   /// user-defined mapper.
   /// \param MapperIdInfo The identifier of associated user-defined mapper.
@@ -6342,13 +6354,24 @@ class OMPToClause final : public OMPMappableExprListClause<OMPToClause>,
   /// NumUniqueDeclarations: number of unique base declarations in this clause;
   /// 3) NumComponentLists: number of component lists in this clause; and 4)
   /// NumComponents: total number of expression components in the clause.
-  explicit OMPToClause(NestedNameSpecifierLoc MapperQualifierLoc,
+  explicit OMPToClause(ArrayRef<OpenMPMotionModifierKind> TheMotionModifiers,
+                       ArrayRef<SourceLocation> TheMotionModifiersLoc,
+                       NestedNameSpecifierLoc MapperQualifierLoc,
                        DeclarationNameInfo MapperIdInfo,
                        const OMPVarListLocTy &Locs,
                        const OMPMappableExprListSizeTy &Sizes)
       : OMPMappableExprListClause(llvm::omp::OMPC_to, Locs, Sizes,
                                   /*SupportsMapper=*/true, &MapperQualifierLoc,
-                                  &MapperIdInfo) {}
+                                  &MapperIdInfo) {
+    assert(llvm::array_lengthof(MotionModifiers) == TheMotionModifiers.size() &&
+           "Unexpected number of motion modifiers.");
+    llvm::copy(TheMotionModifiers, std::begin(MotionModifiers));
+
+    assert(llvm::array_lengthof(MotionModifiersLoc) ==
+               TheMotionModifiersLoc.size() &&
+           "Unexpected number of motion modifier locations.");
+    llvm::copy(TheMotionModifiersLoc, std::begin(MotionModifiersLoc));
+  }
 
   /// Build an empty clause.
   ///
@@ -6361,6 +6384,29 @@ class OMPToClause final : public OMPMappableExprListClause<OMPToClause>,
       : OMPMappableExprListClause(llvm::omp::OMPC_to, OMPVarListLocTy(), Sizes,
                                   /*SupportsMapper=*/true) {}
 
+  /// Set motion-modifier for the clause.
+  ///
+  /// \param I index for motion-modifier.
+  /// \param T motion-modifier for the clause.
+  void setMotionModifier(unsigned I, OpenMPMotionModifierKind T) {
+    assert(I < NumberOfOMPMotionModifiers &&
+           "Unexpected index to store motion modifier, exceeds array size.");
+    MotionModifiers[I] = T;
+  }
+
+  /// Set location for the motion-modifier.
+  ///
+  /// \param I index for motion-modifier location.
+  /// \param TLoc motion-modifier location.
+  void setMotionModifierLoc(unsigned I, SourceLocation TLoc) {
+    assert(I < NumberOfOMPMotionModifiers &&
+           "Index to store motion modifier location exceeds array size.");
+    MotionModifiersLoc[I] = TLoc;
+  }
+
+  /// Set colon location.
+  void setColonLoc(SourceLocation Loc) { ColonLoc = Loc; }
+
   /// Define the sizes of each trailing object array except the last one. This
   /// is required for TrailingObjects to work properly.
   size_t numTrailingObjects(OverloadToken<Expr *>) const {
@@ -6385,6 +6431,8 @@ class OMPToClause final : public OMPMappableExprListClause<OMPToClause>,
   /// \param Vars The original expression used in the clause.
   /// \param Declarations Declarations used in the clause.
   /// \param ComponentLists Component lists used in the clause.
+  /// \param MotionModifiers Motion-modifiers.
+  /// \param MotionModifiersLoc Location of motion-modifiers.
   /// \param UDMapperRefs References to user-defined mappers associated with
   /// expressions used in the clause.
   /// \param UDMQualifierLoc C++ nested name specifier for the associated
@@ -6395,6 +6443,8 @@ class OMPToClause final : public OMPMappableExprListClause<OMPToClause>,
                              ArrayRef<ValueDecl *> Declarations,
                              MappableExprComponentListsRef ComponentLists,
                              ArrayRef<Expr *> UDMapperRefs,
+                             ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
+                             ArrayRef<SourceLocation> MotionModifiersLoc,
                              NestedNameSpecifierLoc UDMQualifierLoc,
                              DeclarationNameInfo MapperId);
 
@@ -6409,6 +6459,38 @@ class OMPToClause final : public OMPMappableExprListClause<OMPToClause>,
   static OMPToClause *CreateEmpty(const ASTContext &C,
                                   const OMPMappableExprListSizeTy &Sizes);
 
+  /// Fetches the motion-modifier at 'Cnt' index of array of modifiers.
+  ///
+  /// \param Cnt index for motion-modifier.
+  OpenMPMotionModifierKind getMotionModifier(unsigned Cnt) const LLVM_READONLY {
+    assert(Cnt < NumberOfOMPMotionModifiers &&
+           "Requested modifier exceeds the total number of modifiers.");
+    return MotionModifiers[Cnt];
+  }
+
+  /// Fetches the motion-modifier location at 'Cnt' index of array of modifiers'
+  /// locations.
+  ///
+  /// \param Cnt index for motion-modifier location.
+  SourceLocation getMotionModifierLoc(unsigned Cnt) const LLVM_READONLY {
+    assert(Cnt < NumberOfOMPMotionModifiers &&
+           "Requested modifier location exceeds total number of modifiers.");
+    return MotionModifiersLoc[Cnt];
+  }
+
+  /// Fetches ArrayRef of motion-modifiers.
+  ArrayRef<OpenMPMotionModifierKind> getMotionModifiers() const LLVM_READONLY {
+    return llvm::makeArrayRef(MotionModifiers);
+  }
+
+  /// Fetches ArrayRef of location of motion-modifiers.
+  ArrayRef<SourceLocation> getMotionModifiersLoc() const LLVM_READONLY {
+    return llvm::makeArrayRef(MotionModifiersLoc);
+  }
+
+  /// Get colon location.
+  SourceLocation getColonLoc() const { return ColonLoc; }
+
   child_range children() {
     return child_range(reinterpret_cast<Stmt **>(varlist_begin()),
                        reinterpret_cast<Stmt **>(varlist_end()));
@@ -6449,8 +6531,20 @@ class OMPFromClause final
   friend OMPVarListClause;
   friend TrailingObjects;
 
+  /// Motion-modifiers for the 'from' clause.
+  OpenMPMotionModifierKind MotionModifiers[NumberOfOMPMotionModifiers] = {
+      OMPC_MOTION_MODIFIER_unknown, OMPC_MOTION_MODIFIER_unknown};
+
+  /// Location of motion-modifiers for the 'from' clause.
+  SourceLocation MotionModifiersLoc[NumberOfOMPMotionModifiers];
+
+  /// Colon location.
+  SourceLocation ColonLoc;
+
   /// Build clause with number of variables \a NumVars.
   ///
+  /// \param TheMotionModifiers Motion-modifiers.
+  /// \param TheMotionModifiersLoc Locations of motion-modifiers.
   /// \param MapperQualifierLoc C++ nested name specifier for the associated
   /// user-defined mapper.
   /// \param MapperIdInfo The identifier of associated user-defined mapper.
@@ -6462,13 +6556,24 @@ class OMPFromClause final
   /// NumUniqueDeclarations: number of unique base declarations in this clause;
   /// 3) NumComponentLists: number of component lists in this clause; and 4)
   /// NumComponents: total number of expression components in the clause.
-  explicit OMPFromClause(NestedNameSpecifierLoc MapperQualifierLoc,
+  explicit OMPFromClause(ArrayRef<OpenMPMotionModifierKind> TheMotionModifiers,
+                         ArrayRef<SourceLocation> TheMotionModifiersLoc,
+                         NestedNameSpecifierLoc MapperQualifierLoc,
                          DeclarationNameInfo MapperIdInfo,
                          const OMPVarListLocTy &Locs,
                          const OMPMappableExprListSizeTy &Sizes)
       : OMPMappableExprListClause(llvm::omp::OMPC_from, Locs, Sizes,
                                   /*SupportsMapper=*/true, &MapperQualifierLoc,
-                                  &MapperIdInfo) {}
+                                  &MapperIdInfo) {
+    assert(llvm::array_lengthof(MotionModifiers) == TheMotionModifiers.size() &&
+           "Unexpected number of motion modifiers.");
+    llvm::copy(TheMotionModifiers, std::begin(MotionModifiers));
+
+    assert(llvm::array_lengthof(MotionModifiersLoc) ==
+               TheMotionModifiersLoc.size() &&
+           "Unexpected number of motion modifier locations.");
+    llvm::copy(TheMotionModifiersLoc, std::begin(MotionModifiersLoc));
+  }
 
   /// Build an empty clause.
   ///
@@ -6481,6 +6586,29 @@ class OMPFromClause final
       : OMPMappableExprListClause(llvm::omp::OMPC_from, OMPVarListLocTy(),
                                   Sizes, /*SupportsMapper=*/true) {}
 
+  /// Set motion-modifier for the clause.
+  ///
+  /// \param I index for motion-modifier.
+  /// \param T motion-modifier for the clause.
+  void setMotionModifier(unsigned I, OpenMPMotionModifierKind T) {
+    assert(I < NumberOfOMPMotionModifiers &&
+           "Unexpected index to store motion modifier, exceeds array size.");
+    MotionModifiers[I] = T;
+  }
+
+  /// Set location for the motion-modifier.
+  ///
+  /// \param I index for motion-modifier location.
+  /// \param TLoc motion-modifier location.
+  void setMotionModifierLoc(unsigned I, SourceLocation TLoc) {
+    assert(I < NumberOfOMPMotionModifiers &&
+           "Index to store motion modifier location exceeds array size.");
+    MotionModifiersLoc[I] = TLoc;
+  }
+
+  /// Set colon location.
+  void setColonLoc(SourceLocation Loc) { ColonLoc = Loc; }
+
   /// Define the sizes of each trailing object array except the last one. This
   /// is required for TrailingObjects to work properly.
   size_t numTrailingObjects(OverloadToken<Expr *>) const {
@@ -6505,18 +6633,21 @@ class OMPFromClause final
   /// \param Vars The original expression used in the clause.
   /// \param Declarations Declarations used in the clause.
   /// \param ComponentLists Component lists used in the clause.
+  /// \param MotionModifiers Motion-modifiers.
+  /// \param MotionModifiersLoc Location of motion-modifiers.
   /// \param UDMapperRefs References to user-defined mappers associated with
   /// expressions used in the clause.
   /// \param UDMQualifierLoc C++ nested name specifier for the associated
   /// user-defined mapper.
   /// \param MapperId The identifier of associated user-defined mapper.
-  static OMPFromClause *Create(const ASTContext &C, const OMPVarListLocTy &Locs,
-                               ArrayRef<Expr *> Vars,
-                               ArrayRef<ValueDecl *> Declarations,
-                               MappableExprComponentListsRef ComponentLists,
-                               ArrayRef<Expr *> UDMapperRefs,
-                               NestedNameSpecifierLoc UDMQualifierLoc,
-                               DeclarationNameInfo MapperId);
+  static OMPFromClause *
+  Create(const ASTContext &C, const OMPVarListLocTy &Locs,
+         ArrayRef<Expr *> Vars, ArrayRef<ValueDecl *> Declarations,
+         MappableExprComponentListsRef ComponentLists,
+         ArrayRef<Expr *> UDMapperRefs,
+         ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
+         ArrayRef<SourceLocation> MotionModifiersLoc,
+         NestedNameSpecifierLoc UDMQualifierLoc, DeclarationNameInfo MapperId);
 
   /// Creates an empty clause with the place for \a NumVars variables.
   ///
@@ -6529,6 +6660,38 @@ class OMPFromClause final
   static OMPFromClause *CreateEmpty(const ASTContext &C,
                                     const OMPMappableExprListSizeTy &Sizes);
 
+  /// Fetches the motion-modifier at 'Cnt' index of array of modifiers.
+  ///
+  /// \param Cnt index for motion-modifier.
+  OpenMPMotionModifierKind getMotionModifier(unsigned Cnt) const LLVM_READONLY {
+    assert(Cnt < NumberOfOMPMotionModifiers &&
+           "Requested modifier exceeds the total number of modifiers.");
+    return MotionModifiers[Cnt];
+  }
+
+  /// Fetches the motion-modifier location at 'Cnt' index of array of modifiers'
+  /// locations.
+  ///
+  /// \param Cnt index for motion-modifier location.
+  SourceLocation getMotionModifierLoc(unsigned Cnt) const LLVM_READONLY {
+    assert(Cnt < NumberOfOMPMotionModifiers &&
+           "Requested modifier location exceeds total number of modifiers.");
+    return MotionModifiersLoc[Cnt];
+  }
+
+  /// Fetches ArrayRef of motion-modifiers.
+  ArrayRef<OpenMPMotionModifierKind> getMotionModifiers() const LLVM_READONLY {
+    return llvm::makeArrayRef(MotionModifiers);
+  }
+
+  /// Fetches ArrayRef of location of motion-modifiers.
+  ArrayRef<SourceLocation> getMotionModifiersLoc() const LLVM_READONLY {
+    return llvm::makeArrayRef(MotionModifiersLoc);
+  }
+
+  /// Get colon location.
+  SourceLocation getColonLoc() const { return ColonLoc; }
+
   child_range children() {
     return child_range(reinterpret_cast<Stmt **>(varlist_begin()),
                        reinterpret_cast<Stmt **>(varlist_end()));
@@ -7623,6 +7786,8 @@ class OMPClausePrinter final : public OMPClauseVisitor<OMPClausePrinter> {
 
   /// Process clauses with list of variables.
   template <typename T> void VisitOMPClauseList(T *Node, char StartSym);
+  /// Process motion clauses.
+  template <typename T> void VisitOMPMotionClause(T *Node);
 
 public:
   OMPClausePrinter(raw_ostream &OS, const PrintingPolicy &Policy)
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 8093e7ed3fbe1..8a9cada75e351 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -9805,6 +9805,8 @@ def err_omp_multiple_array_items_in_map_clause : Error<
   "multiple array elements associated with the same variable are not allowed in map clauses of the same construct">;
 def err_omp_duplicate_map_type_modifier : Error<
   "same map type modifier has been specified more than once">;
+def err_omp_duplicate_motion_modifier : Error<
+  "same motion modifier has been specified more than once">;
 def err_omp_pointer_mapped_along_with_derived_section : Error<
   "pointer cannot be mapped along with a section derived from itself">;
 def err_omp_original_storage_is_shared_and_does_not_contain : Error<
diff --git a/clang/include/clang/Basic/OpenMPKinds.def b/clang/include/clang/Basic/OpenMPKinds.def
index 04ecbeaaa03e2..59bb331a7a8a6 100644
--- a/clang/include/clang/Basic/OpenMPKinds.def
+++ b/clang/include/clang/Basic/OpenMPKinds.def
@@ -125,6 +125,7 @@ OPENMP_MAP_MODIFIER_KIND(present)
 
 // Modifiers for 'to' or 'from' clause.
 OPENMP_MOTION_MODIFIER_KIND(mapper)
+OPENMP_MOTION_MODIFIER_KIND(present)
 
 // Static attributes for 'dist_schedule' clause.
 OPENMP_DIST_SCHEDULE_KIND(static)
diff --git a/clang/include/clang/Basic/OpenMPKinds.h b/clang/include/clang/Basic/OpenMPKinds.h
index 3e9b5c4a8b14a..0f37dc9ad997d 100644
--- a/clang/include/clang/Basic/OpenMPKinds.h
+++ b/clang/include/clang/Basic/OpenMPKinds.h
@@ -94,6 +94,10 @@ enum OpenMPMotionModifierKind {
   OMPC_MOTION_MODIFIER_unknown
 };
 
+/// Number of allowed motion-modifiers.
+static constexpr unsigned NumberOfOMPMotionModifiers =
+    OMPC_MOTION_MODIFIER_unknown;
+
 /// OpenMP attributes for 'dist_schedule' clause.
 enum OpenMPDistScheduleClauseKind {
 #define OPENMP_DIST_SCHEDULE_KIND(Name) OMPC_DIST_SCHEDULE_##Name,
diff --git a/clang/include/clang/Parse/Parser.h b/clang/include/clang/Parse/Parser.h
index e809d87b59a0f..4068e6a444c9c 100644
--- a/clang/include/clang/Parse/Parser.h
+++ b/clang/include/clang/Parse/Parser.h
@@ -3221,6 +3221,9 @@ class Parser : public CodeCompletionHandler {
     MapTypeModifiers;
     SmallVector<SourceLocation, NumberOfOMPMapClauseModifiers>
     MapTypeModifiersLoc;
+    SmallVector<OpenMPMotionModifierKind, NumberOfOMPMotionModifiers>
+        MotionModifiers;
+    SmallVector<SourceLocation, NumberOfOMPMotionModifiers> MotionModifiersLoc;
     bool IsMapTypeImplicit = false;
     SourceLocation ExtraModifierLoc;
   };
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index fd4300f563a98..4721ccdf14590 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -10725,7 +10725,9 @@ class Sema final {
       DeclarationNameInfo &ReductionOrMapperId, int ExtraModifier,
       ArrayRef<OpenMPMapModifierKind> MapTypeModifiers,
       ArrayRef<SourceLocation> MapTypeModifiersLoc, bool IsMapTypeImplicit,
-      SourceLocation ExtraModifierLoc);
+      SourceLocation ExtraModifierLoc,
+      ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
+      ArrayRef<SourceLocation> MotionModifiersLoc);
   /// Called on well-formed 'inclusive' clause.
   OMPClause *ActOnOpenMPInclusiveClause(ArrayRef<Expr *> VarList,
                                         SourceLocation StartLoc,
@@ -10862,15 +10864,20 @@ class Sema final {
       SourceLocation KindLoc, SourceLocation EndLoc);
   /// Called on well-formed 'to' clause.
   OMPClause *
-  ActOnOpenMPToClause(ArrayRef<Expr *> VarList, CXXScopeSpec &MapperIdScopeSpec,
-                      DeclarationNameInfo &MapperId,
-                      const OMPVarListLocTy &Locs,
+  ActOnOpenMPToClause(ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
+                      ArrayRef<SourceLocation> MotionModifiersLoc,
+                      CXXScopeSpec &MapperIdScopeSpec,
+                      DeclarationNameInfo &MapperId, SourceLocation ColonLoc,
+                      ArrayRef<Expr *> VarList, const OMPVarListLocTy &Locs,
                       ArrayRef<Expr *> UnresolvedMappers = llvm::None);
   /// Called on well-formed 'from' clause.
-  OMPClause *ActOnOpenMPFromClause(
-      ArrayRef<Expr *> VarList, CXXScopeSpec &MapperIdScopeSpec,
-      DeclarationNameInfo &MapperId, const OMPVarListLocTy &Locs,
-      ArrayRef<Expr *> UnresolvedMappers = llvm::None);
+  OMPClause *
+  ActOnOpenMPFromClause(ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
+                        ArrayRef<SourceLocation> MotionModifiersLoc,
+                        CXXScopeSpec &MapperIdScopeSpec,
+                        DeclarationNameInfo &MapperId, SourceLocation ColonLoc,
+                        ArrayRef<Expr *> VarList, const OMPVarListLocTy &Locs,
+                        ArrayRef<Expr *> UnresolvedMappers = llvm::None);
   /// Called on well-formed 'use_device_ptr' clause.
   OMPClause *ActOnOpenMPUseDevicePtrClause(ArrayRef<Expr *> VarList,
                                            const OMPVarListLocTy &Locs);
diff --git a/clang/lib/AST/OpenMPClause.cpp b/clang/lib/AST/OpenMPClause.cpp
index 6933c57425520..cf548c2ab4741 100644
--- a/clang/lib/AST/OpenMPClause.cpp
+++ b/clang/lib/AST/OpenMPClause.cpp
@@ -1097,6 +1097,8 @@ OMPToClause *OMPToClause::Create(
     const ASTContext &C, const OMPVarListLocTy &Locs, ArrayRef<Expr *> Vars,
     ArrayRef<ValueDecl *> Declarations,
     MappableExprComponentListsRef ComponentLists, ArrayRef<Expr *> UDMapperRefs,
+    ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
+    ArrayRef<SourceLocation> MotionModifiersLoc,
     NestedNameSpecifierLoc UDMQualifierLoc, DeclarationNameInfo MapperId) {
   OMPMappableExprListSizeTy Sizes;
   Sizes.NumVars = Vars.size();
@@ -1121,7 +1123,8 @@ OMPToClause *OMPToClause::Create(
           Sizes.NumUniqueDeclarations + Sizes.NumComponentLists,
           Sizes.NumComponents));
 
-  auto *Clause = new (Mem) OMPToClause(UDMQualifierLoc, MapperId, Locs, Sizes);
+  auto *Clause = new (Mem) OMPToClause(MotionModifiers, MotionModifiersLoc,
+                                       UDMQualifierLoc, MapperId, Locs, Sizes);
 
   Clause->setVarRefs(Vars);
   Clause->setUDMapperRefs(UDMapperRefs);
@@ -1144,6 +1147,8 @@ OMPFromClause *OMPFromClause::Create(
     const ASTContext &C, const OMPVarListLocTy &Locs, ArrayRef<Expr *> Vars,
     ArrayRef<ValueDecl *> Declarations,
     MappableExprComponentListsRef ComponentLists, ArrayRef<Expr *> UDMapperRefs,
+    ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
+    ArrayRef<SourceLocation> MotionModifiersLoc,
     NestedNameSpecifierLoc UDMQualifierLoc, DeclarationNameInfo MapperId) {
   OMPMappableExprListSizeTy Sizes;
   Sizes.NumVars = Vars.size();
@@ -1169,7 +1174,8 @@ OMPFromClause *OMPFromClause::Create(
           Sizes.NumComponents));
 
   auto *Clause =
-      new (Mem) OMPFromClause(UDMQualifierLoc, MapperId, Locs, Sizes);
+      new (Mem) OMPFromClause(MotionModifiers, MotionModifiersLoc,
+                              UDMQualifierLoc, MapperId, Locs, Sizes);
 
   Clause->setVarRefs(Vars);
   Clause->setUDMapperRefs(UDMapperRefs);
@@ -1936,6 +1942,17 @@ void OMPClausePrinter::VisitOMPDependClause(OMPDependClause *Node) {
   OS << ")";
 }
 
+template <typename T>
+static void PrintMapper(raw_ostream &OS, T *Node,
+                        const PrintingPolicy &Policy) {
+  OS << '(';
+  NestedNameSpecifier *MapperNNS =
+      Node->getMapperQualifierLoc().getNestedNameSpecifier();
+  if (MapperNNS)
+    MapperNNS->print(OS, Policy);
+  OS << Node->getMapperIdInfo() << ')';
+}
+
 void OMPClausePrinter::VisitOMPMapClause(OMPMapClause *Node) {
   if (!Node->varlist_empty()) {
     OS << "map(";
@@ -1944,14 +1961,8 @@ void OMPClausePrinter::VisitOMPMapClause(OMPMapClause *Node) {
         if (Node->getMapTypeModifier(I) != OMPC_MAP_MODIFIER_unknown) {
           OS << getOpenMPSimpleClauseTypeName(OMPC_map,
                                               Node->getMapTypeModifier(I));
-          if (Node->getMapTypeModifier(I) == OMPC_MAP_MODIFIER_mapper) {
-            OS << '(';
-            NestedNameSpecifier *MapperNNS =
-                Node->getMapperQualifierLoc().getNestedNameSpecifier();
-            if (MapperNNS)
-              MapperNNS->print(OS, Policy);
-            OS << Node->getMapperIdInfo() << ')';
-          }
+          if (Node->getMapTypeModifier(I) == OMPC_MAP_MODIFIER_mapper)
+            PrintMapper(OS, Node, Policy);
           OS << ',';
         }
       }
@@ -1963,44 +1974,41 @@ void OMPClausePrinter::VisitOMPMapClause(OMPMapClause *Node) {
   }
 }
 
-void OMPClausePrinter::VisitOMPToClause(OMPToClause *Node) {
-  if (!Node->varlist_empty()) {
-    OS << "to";
-    DeclarationNameInfo MapperId = Node->getMapperIdInfo();
-    if (MapperId.getName() && !MapperId.getName().isEmpty()) {
-      OS << '(';
-      OS << "mapper(";
-      NestedNameSpecifier *MapperNNS =
-          Node->getMapperQualifierLoc().getNestedNameSpecifier();
-      if (MapperNNS)
-        MapperNNS->print(OS, Policy);
-      OS << MapperId << "):";
-      VisitOMPClauseList(Node, ' ');
-    } else {
-      VisitOMPClauseList(Node, '(');
+template <typename T> void OMPClausePrinter::VisitOMPMotionClause(T *Node) {
+  if (Node->varlist_empty())
+    return;
+  OS << getOpenMPClauseName(Node->getClauseKind());
+  unsigned ModifierCount = 0;
+  for (unsigned I = 0; I < NumberOfOMPMotionModifiers; ++I) {
+    if (Node->getMotionModifier(I) != OMPC_MOTION_MODIFIER_unknown)
+      ++ModifierCount;
+  }
+  if (ModifierCount) {
+    OS << '(';
+    for (unsigned I = 0; I < NumberOfOMPMotionModifiers; ++I) {
+      if (Node->getMotionModifier(I) != OMPC_MOTION_MODIFIER_unknown) {
+        OS << getOpenMPSimpleClauseTypeName(Node->getClauseKind(),
+                                            Node->getMotionModifier(I));
+        if (Node->getMotionModifier(I) == OMPC_MOTION_MODIFIER_mapper)
+          PrintMapper(OS, Node, Policy);
+        if (I < ModifierCount - 1)
+          OS << ", ";
+      }
     }
-    OS << ")";
+    OS << ':';
+    VisitOMPClauseList(Node, ' ');
+  } else {
+    VisitOMPClauseList(Node, '(');
   }
+  OS << ")";
+}
+
+void OMPClausePrinter::VisitOMPToClause(OMPToClause *Node) {
+  VisitOMPMotionClause(Node);
 }
 
 void OMPClausePrinter::VisitOMPFromClause(OMPFromClause *Node) {
-  if (!Node->varlist_empty()) {
-    OS << "from";
-    DeclarationNameInfo MapperId = Node->getMapperIdInfo();
-    if (MapperId.getName() && !MapperId.getName().isEmpty()) {
-      OS << '(';
-      OS << "mapper(";
-      NestedNameSpecifier *MapperNNS =
-          Node->getMapperQualifierLoc().getNestedNameSpecifier();
-      if (MapperNNS)
-        MapperNNS->print(OS, Policy);
-      OS << MapperId << "):";
-      VisitOMPClauseList(Node, ' ');
-    } else {
-      VisitOMPClauseList(Node, '(');
-    }
-    OS << ")";
-  }
+  VisitOMPMotionClause(Node);
 }
 
 void OMPClausePrinter::VisitOMPDistScheduleClause(OMPDistScheduleClause *Node) {
diff --git a/clang/lib/Basic/OpenMPKinds.cpp b/clang/lib/Basic/OpenMPKinds.cpp
index da362f99ed297..5c19d60cbd6e3 100644
--- a/clang/lib/Basic/OpenMPKinds.cpp
+++ b/clang/lib/Basic/OpenMPKinds.cpp
@@ -64,12 +64,16 @@ unsigned clang::getOpenMPSimpleClauseType(OpenMPClauseKind Kind, StringRef Str,
     return Type;
   }
   case OMPC_to:
-  case OMPC_from:
-    return llvm::StringSwitch<unsigned>(Str)
+  case OMPC_from: {
+    unsigned Type = llvm::StringSwitch<unsigned>(Str)
 #define OPENMP_MOTION_MODIFIER_KIND(Name)                                      \
   .Case(#Name, static_cast<unsigned>(OMPC_MOTION_MODIFIER_##Name))
 #include "clang/Basic/OpenMPKinds.def"
         .Default(OMPC_MOTION_MODIFIER_unknown);
+    if (OpenMPVersion < 51 && Type == OMPC_MOTION_MODIFIER_present)
+      return OMPC_MOTION_MODIFIER_unknown;
+    return Type;
+  }
   case OMPC_dist_schedule:
     return llvm::StringSwitch<OpenMPDistScheduleClauseKind>(Str)
 #define OPENMP_DIST_SCHEDULE_KIND(Name) .Case(#Name, OMPC_DIST_SCHEDULE_##Name)
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
index f16f2ed89de89..f7e0d7a56d5ac 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -7920,6 +7920,20 @@ class MappableExprsHandler {
         Layout.push_back(Data.get<const FieldDecl *>());
     }
   }
+  static void translateMotionModifiers(
+      ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
+      SmallVectorImpl<OpenMPMapModifierKind> &MapModifiers) {
+    for (OpenMPMotionModifierKind MotionMod : MotionModifiers) {
+      switch (MotionMod) {
+      case OMPC_MOTION_MODIFIER_present:
+        MapModifiers.push_back(OMPC_MAP_MODIFIER_present);
+        break;
+      case OMPC_MOTION_MODIFIER_mapper:
+      case OMPC_MOTION_MODIFIER_unknown:
+        break;
+      }
+    }
+  }
 
 public:
   MappableExprsHandler(const OMPExecutableDirective &Dir, CodeGenFunction &CGF)
@@ -8038,12 +8052,16 @@ class MappableExprsHandler {
       }
     for (const auto *C : CurExecDir->getClausesOfKind<OMPToClause>())
       for (const auto L : C->component_lists()) {
-        InfoGen(std::get<0>(L), std::get<1>(L), OMPC_MAP_to, llvm::None,
+        SmallVector<OpenMPMapModifierKind, 2> MapModifiers;
+        translateMotionModifiers(C->getMotionModifiers(), MapModifiers);
+        InfoGen(std::get<0>(L), std::get<1>(L), OMPC_MAP_to, MapModifiers,
                 /*ReturnDevicePointer=*/false, C->isImplicit(), std::get<2>(L));
       }
     for (const auto *C : CurExecDir->getClausesOfKind<OMPFromClause>())
       for (const auto L : C->component_lists()) {
-        InfoGen(std::get<0>(L), std::get<1>(L), OMPC_MAP_from, llvm::None,
+        SmallVector<OpenMPMapModifierKind, 2> MapModifiers;
+        translateMotionModifiers(C->getMotionModifiers(), MapModifiers);
+        InfoGen(std::get<0>(L), std::get<1>(L), OMPC_MAP_from, MapModifiers,
                 /*ReturnDevicePointer=*/false, C->isImplicit(), std::get<2>(L));
       }
 
diff --git a/clang/lib/Parse/ParseOpenMP.cpp b/clang/lib/Parse/ParseOpenMP.cpp
index 6a5d7c604c6bb..cafe4191b2074 100644
--- a/clang/lib/Parse/ParseOpenMP.cpp
+++ b/clang/lib/Parse/ParseOpenMP.cpp
@@ -3441,25 +3441,43 @@ bool Parser::ParseOpenMPVarList(OpenMPDirectiveKind DKind,
     if (Tok.is(tok::colon))
       Data.ColonLoc = ConsumeToken();
   } else if (Kind == OMPC_to || Kind == OMPC_from) {
-    if (Tok.is(tok::identifier)) {
+    while (Tok.is(tok::identifier)) {
       auto Modifier =
           static_cast<OpenMPMotionModifierKind>(getOpenMPSimpleClauseType(
               Kind, PP.getSpelling(Tok), getLangOpts().OpenMP));
+      if (Modifier == OMPC_MOTION_MODIFIER_unknown)
+        break;
+      Data.MotionModifiers.push_back(Modifier);
+      Data.MotionModifiersLoc.push_back(Tok.getLocation());
+      ConsumeToken();
       if (Modifier == OMPC_MOTION_MODIFIER_mapper) {
-        // Parse the mapper modifier.
-        ConsumeToken();
         IsInvalidMapperModifier = parseMapperModifier(Data);
-        if (Tok.isNot(tok::colon)) {
-          if (!IsInvalidMapperModifier)
-            Diag(Tok, diag::warn_pragma_expected_colon) << ")";
-          SkipUntil(tok::colon, tok::r_paren, tok::annot_pragma_openmp_end,
-                    StopBeforeMatch);
-        }
-        // Consume ':'.
-        if (Tok.is(tok::colon))
-          ConsumeToken();
+        if (IsInvalidMapperModifier)
+          break;
       }
+      // OpenMP < 5.1 doesn't permit a ',' or additional modifiers.
+      if (getLangOpts().OpenMP < 51)
+        break;
+      // OpenMP 5.1 accepts an optional ',' even if the next character is ':'.
+      // TODO: Is that intentional?
+      if (Tok.is(tok::comma))
+        ConsumeToken();
+    }
+    if (!Data.MotionModifiers.empty() && Tok.isNot(tok::colon)) {
+      if (!IsInvalidMapperModifier) {
+        if (getLangOpts().OpenMP < 51)
+          Diag(Tok, diag::warn_pragma_expected_colon) << ")";
+        else
+          Diag(Tok, diag::warn_pragma_expected_colon) << "motion modifier";
+      }
+      SkipUntil(tok::colon, tok::r_paren, tok::annot_pragma_openmp_end,
+                StopBeforeMatch);
     }
+    // OpenMP 5.1 permits a ':' even without a preceding modifier.  TODO: Is
+    // that intentional?
+    if ((!Data.MotionModifiers.empty() || getLangOpts().OpenMP >= 51) &&
+        Tok.is(tok::colon))
+      Data.ColonLoc = ConsumeToken();
   } else if (Kind == OMPC_allocate ||
              (Kind == OMPC_affinity && Tok.is(tok::identifier) &&
               PP.getSpelling(Tok) == "iterator")) {
@@ -3635,6 +3653,7 @@ OMPClause *Parser::ParseOpenMPVarListClause(OpenMPDirectiveKind DKind,
       Kind, Vars, Data.DepModOrTailExpr, Locs, Data.ColonLoc,
       Data.ReductionOrMapperIdScopeSpec, Data.ReductionOrMapperId,
       Data.ExtraModifier, Data.MapTypeModifiers, Data.MapTypeModifiersLoc,
-      Data.IsMapTypeImplicit, Data.ExtraModifierLoc);
+      Data.IsMapTypeImplicit, Data.ExtraModifierLoc, Data.MotionModifiers,
+      Data.MotionModifiersLoc);
 }
 
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index 0192df3bd170d..d7f618a2fa601 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -13536,7 +13536,9 @@ OMPClause *Sema::ActOnOpenMPVarListClause(
     DeclarationNameInfo &ReductionOrMapperId, int ExtraModifier,
     ArrayRef<OpenMPMapModifierKind> MapTypeModifiers,
     ArrayRef<SourceLocation> MapTypeModifiersLoc, bool IsMapTypeImplicit,
-    SourceLocation ExtraModifierLoc) {
+    SourceLocation ExtraModifierLoc,
+    ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
+    ArrayRef<SourceLocation> MotionModifiersLoc) {
   SourceLocation StartLoc = Locs.StartLoc;
   SourceLocation LParenLoc = Locs.LParenLoc;
   SourceLocation EndLoc = Locs.EndLoc;
@@ -13613,12 +13615,14 @@ OMPClause *Sema::ActOnOpenMPVarListClause(
         IsMapTypeImplicit, ExtraModifierLoc, ColonLoc, VarList, Locs);
     break;
   case OMPC_to:
-    Res = ActOnOpenMPToClause(VarList, ReductionOrMapperIdScopeSpec,
-                              ReductionOrMapperId, Locs);
+    Res = ActOnOpenMPToClause(MotionModifiers, MotionModifiersLoc,
+                              ReductionOrMapperIdScopeSpec, ReductionOrMapperId,
+                              ColonLoc, VarList, Locs);
     break;
   case OMPC_from:
-    Res = ActOnOpenMPFromClause(VarList, ReductionOrMapperIdScopeSpec,
-                                ReductionOrMapperId, Locs);
+    Res = ActOnOpenMPFromClause(MotionModifiers, MotionModifiersLoc,
+                                ReductionOrMapperIdScopeSpec,
+                                ReductionOrMapperId, ColonLoc, VarList, Locs);
     break;
   case OMPC_use_device_ptr:
     Res = ActOnOpenMPUseDevicePtrClause(VarList, Locs);
@@ -18576,11 +18580,31 @@ void Sema::checkDeclIsAllowedInOpenMPTarget(Expr *E, Decl *D,
   checkDeclInTargetContext(E->getExprLoc(), E->getSourceRange(), *this, D);
 }
 
-OMPClause *Sema::ActOnOpenMPToClause(ArrayRef<Expr *> VarList,
-                                     CXXScopeSpec &MapperIdScopeSpec,
-                                     DeclarationNameInfo &MapperId,
-                                     const OMPVarListLocTy &Locs,
-                                     ArrayRef<Expr *> UnresolvedMappers) {
+OMPClause *Sema::ActOnOpenMPToClause(
+    ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
+    ArrayRef<SourceLocation> MotionModifiersLoc,
+    CXXScopeSpec &MapperIdScopeSpec, DeclarationNameInfo &MapperId,
+    SourceLocation ColonLoc, ArrayRef<Expr *> VarList,
+    const OMPVarListLocTy &Locs, ArrayRef<Expr *> UnresolvedMappers) {
+  OpenMPMotionModifierKind Modifiers[] = {OMPC_MOTION_MODIFIER_unknown,
+                                          OMPC_MOTION_MODIFIER_unknown};
+  SourceLocation ModifiersLoc[NumberOfOMPMotionModifiers];
+
+  // Process motion-modifiers, flag errors for duplicate modifiers.
+  unsigned Count = 0;
+  for (unsigned I = 0, E = MotionModifiers.size(); I < E; ++I) {
+    if (MotionModifiers[I] != OMPC_MOTION_MODIFIER_unknown &&
+        llvm::find(Modifiers, MotionModifiers[I]) != std::end(Modifiers)) {
+      Diag(MotionModifiersLoc[I], diag::err_omp_duplicate_motion_modifier);
+      continue;
+    }
+    assert(Count < NumberOfOMPMotionModifiers &&
+           "Modifiers exceed the allowed number of motion modifiers");
+    Modifiers[Count] = MotionModifiers[I];
+    ModifiersLoc[Count] = MotionModifiersLoc[I];
+    ++Count;
+  }
+
   MappableVarListInfo MVLI(VarList);
   checkMappableExpressionList(*this, DSAStack, OMPC_to, MVLI, Locs.StartLoc,
                               MapperIdScopeSpec, MapperId, UnresolvedMappers);
@@ -18589,15 +18613,35 @@ OMPClause *Sema::ActOnOpenMPToClause(ArrayRef<Expr *> VarList,
 
   return OMPToClause::Create(
       Context, Locs, MVLI.ProcessedVarList, MVLI.VarBaseDeclarations,
-      MVLI.VarComponents, MVLI.UDMapperList,
+      MVLI.VarComponents, MVLI.UDMapperList, Modifiers, ModifiersLoc,
       MapperIdScopeSpec.getWithLocInContext(Context), MapperId);
 }
 
-OMPClause *Sema::ActOnOpenMPFromClause(ArrayRef<Expr *> VarList,
-                                       CXXScopeSpec &MapperIdScopeSpec,
-                                       DeclarationNameInfo &MapperId,
-                                       const OMPVarListLocTy &Locs,
-                                       ArrayRef<Expr *> UnresolvedMappers) {
+OMPClause *Sema::ActOnOpenMPFromClause(
+    ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
+    ArrayRef<SourceLocation> MotionModifiersLoc,
+    CXXScopeSpec &MapperIdScopeSpec, DeclarationNameInfo &MapperId,
+    SourceLocation ColonLoc, ArrayRef<Expr *> VarList,
+    const OMPVarListLocTy &Locs, ArrayRef<Expr *> UnresolvedMappers) {
+  OpenMPMotionModifierKind Modifiers[] = {OMPC_MOTION_MODIFIER_unknown,
+                                          OMPC_MOTION_MODIFIER_unknown};
+  SourceLocation ModifiersLoc[NumberOfOMPMotionModifiers];
+
+  // Process motion-modifiers, flag errors for duplicate modifiers.
+  unsigned Count = 0;
+  for (unsigned I = 0, E = MotionModifiers.size(); I < E; ++I) {
+    if (MotionModifiers[I] != OMPC_MOTION_MODIFIER_unknown &&
+        llvm::find(Modifiers, MotionModifiers[I]) != std::end(Modifiers)) {
+      Diag(MotionModifiersLoc[I], diag::err_omp_duplicate_motion_modifier);
+      continue;
+    }
+    assert(Count < NumberOfOMPMotionModifiers &&
+           "Modifiers exceed the allowed number of motion modifiers");
+    Modifiers[Count] = MotionModifiers[I];
+    ModifiersLoc[Count] = MotionModifiersLoc[I];
+    ++Count;
+  }
+
   MappableVarListInfo MVLI(VarList);
   checkMappableExpressionList(*this, DSAStack, OMPC_from, MVLI, Locs.StartLoc,
                               MapperIdScopeSpec, MapperId, UnresolvedMappers);
@@ -18606,7 +18650,7 @@ OMPClause *Sema::ActOnOpenMPFromClause(ArrayRef<Expr *> VarList,
 
   return OMPFromClause::Create(
       Context, Locs, MVLI.ProcessedVarList, MVLI.VarBaseDeclarations,
-      MVLI.VarComponents, MVLI.UDMapperList,
+      MVLI.VarComponents, MVLI.UDMapperList, Modifiers, ModifiersLoc,
       MapperIdScopeSpec.getWithLocInContext(Context), MapperId);
 }
 
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index 0f83431c94cf7..53064d28dd991 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -2006,26 +2006,32 @@ class TreeTransform {
   ///
   /// By default, performs semantic analysis to build the new statement.
   /// Subclasses may override this routine to provide different behavior.
-  OMPClause *RebuildOMPToClause(ArrayRef<Expr *> VarList,
-                                CXXScopeSpec &MapperIdScopeSpec,
-                                DeclarationNameInfo &MapperId,
-                                const OMPVarListLocTy &Locs,
-                                ArrayRef<Expr *> UnresolvedMappers) {
-    return getSema().ActOnOpenMPToClause(VarList, MapperIdScopeSpec, MapperId,
-                                         Locs, UnresolvedMappers);
+  OMPClause *
+  RebuildOMPToClause(ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
+                     ArrayRef<SourceLocation> MotionModifiersLoc,
+                     CXXScopeSpec &MapperIdScopeSpec,
+                     DeclarationNameInfo &MapperId, SourceLocation ColonLoc,
+                     ArrayRef<Expr *> VarList, const OMPVarListLocTy &Locs,
+                     ArrayRef<Expr *> UnresolvedMappers) {
+    return getSema().ActOnOpenMPToClause(MotionModifiers, MotionModifiersLoc,
+                                         MapperIdScopeSpec, MapperId, ColonLoc,
+                                         VarList, Locs, UnresolvedMappers);
   }
 
   /// Build a new OpenMP 'from' clause.
   ///
   /// By default, performs semantic analysis to build the new statement.
   /// Subclasses may override this routine to provide different behavior.
-  OMPClause *RebuildOMPFromClause(ArrayRef<Expr *> VarList,
-                                  CXXScopeSpec &MapperIdScopeSpec,
-                                  DeclarationNameInfo &MapperId,
-                                  const OMPVarListLocTy &Locs,
-                                  ArrayRef<Expr *> UnresolvedMappers) {
-    return getSema().ActOnOpenMPFromClause(VarList, MapperIdScopeSpec, MapperId,
-                                           Locs, UnresolvedMappers);
+  OMPClause *
+  RebuildOMPFromClause(ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
+                       ArrayRef<SourceLocation> MotionModifiersLoc,
+                       CXXScopeSpec &MapperIdScopeSpec,
+                       DeclarationNameInfo &MapperId, SourceLocation ColonLoc,
+                       ArrayRef<Expr *> VarList, const OMPVarListLocTy &Locs,
+                       ArrayRef<Expr *> UnresolvedMappers) {
+    return getSema().ActOnOpenMPFromClause(
+        MotionModifiers, MotionModifiersLoc, MapperIdScopeSpec, MapperId,
+        ColonLoc, VarList, Locs, UnresolvedMappers);
   }
 
   /// Build a new OpenMP 'use_device_ptr' clause.
@@ -9738,8 +9744,9 @@ OMPClause *TreeTransform<Derived>::TransformOMPToClause(OMPToClause *C) {
   if (transformOMPMappableExprListClause<Derived, OMPToClause>(
           *this, C, Vars, MapperIdScopeSpec, MapperIdInfo, UnresolvedMappers))
     return nullptr;
-  return getDerived().RebuildOMPToClause(Vars, MapperIdScopeSpec, MapperIdInfo,
-                                         Locs, UnresolvedMappers);
+  return getDerived().RebuildOMPToClause(
+      C->getMotionModifiers(), C->getMotionModifiersLoc(), MapperIdScopeSpec,
+      MapperIdInfo, C->getColonLoc(), Vars, Locs, UnresolvedMappers);
 }
 
 template <typename Derived>
@@ -9753,7 +9760,8 @@ OMPClause *TreeTransform<Derived>::TransformOMPFromClause(OMPFromClause *C) {
           *this, C, Vars, MapperIdScopeSpec, MapperIdInfo, UnresolvedMappers))
     return nullptr;
   return getDerived().RebuildOMPFromClause(
-      Vars, MapperIdScopeSpec, MapperIdInfo, Locs, UnresolvedMappers);
+      C->getMotionModifiers(), C->getMotionModifiersLoc(), MapperIdScopeSpec,
+      MapperIdInfo, C->getColonLoc(), Vars, Locs, UnresolvedMappers);
 }
 
 template <typename Derived>
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 6b96b4ff59b8e..a27c36420fdbc 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -12601,8 +12601,14 @@ void OMPClauseReader::VisitOMPDefaultmapClause(OMPDefaultmapClause *C) {
 
 void OMPClauseReader::VisitOMPToClause(OMPToClause *C) {
   C->setLParenLoc(Record.readSourceLocation());
+  for (unsigned I = 0; I < NumberOfOMPMotionModifiers; ++I) {
+    C->setMotionModifier(
+        I, static_cast<OpenMPMotionModifierKind>(Record.readInt()));
+    C->setMotionModifierLoc(I, Record.readSourceLocation());
+  }
   C->setMapperQualifierLoc(Record.readNestedNameSpecifierLoc());
   C->setMapperIdInfo(Record.readDeclarationNameInfo());
+  C->setColonLoc(Record.readSourceLocation());
   auto NumVars = C->varlist_size();
   auto UniqueDecls = C->getUniqueDeclarationsNum();
   auto TotalLists = C->getTotalComponentListNum();
@@ -12651,8 +12657,14 @@ void OMPClauseReader::VisitOMPToClause(OMPToClause *C) {
 
 void OMPClauseReader::VisitOMPFromClause(OMPFromClause *C) {
   C->setLParenLoc(Record.readSourceLocation());
+  for (unsigned I = 0; I < NumberOfOMPMotionModifiers; ++I) {
+    C->setMotionModifier(
+        I, static_cast<OpenMPMotionModifierKind>(Record.readInt()));
+    C->setMotionModifierLoc(I, Record.readSourceLocation());
+  }
   C->setMapperQualifierLoc(Record.readNestedNameSpecifierLoc());
   C->setMapperIdInfo(Record.readDeclarationNameInfo());
+  C->setColonLoc(Record.readSourceLocation());
   auto NumVars = C->varlist_size();
   auto UniqueDecls = C->getUniqueDeclarationsNum();
   auto TotalLists = C->getTotalComponentListNum();
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index 9ea90b2a0212a..8c693b7d34fdd 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -6581,8 +6581,13 @@ void OMPClauseWriter::VisitOMPToClause(OMPToClause *C) {
   Record.push_back(C->getTotalComponentListNum());
   Record.push_back(C->getTotalComponentsNum());
   Record.AddSourceLocation(C->getLParenLoc());
+  for (unsigned I = 0; I < NumberOfOMPMotionModifiers; ++I) {
+    Record.push_back(C->getMotionModifier(I));
+    Record.AddSourceLocation(C->getMotionModifierLoc(I));
+  }
   Record.AddNestedNameSpecifierLoc(C->getMapperQualifierLoc());
   Record.AddDeclarationNameInfo(C->getMapperIdInfo());
+  Record.AddSourceLocation(C->getColonLoc());
   for (auto *E : C->varlists())
     Record.AddStmt(E);
   for (auto *E : C->mapperlists())
@@ -6605,8 +6610,13 @@ void OMPClauseWriter::VisitOMPFromClause(OMPFromClause *C) {
   Record.push_back(C->getTotalComponentListNum());
   Record.push_back(C->getTotalComponentsNum());
   Record.AddSourceLocation(C->getLParenLoc());
+  for (unsigned I = 0; I < NumberOfOMPMotionModifiers; ++I) {
+    Record.push_back(C->getMotionModifier(I));
+    Record.AddSourceLocation(C->getMotionModifierLoc(I));
+  }
   Record.AddNestedNameSpecifierLoc(C->getMapperQualifierLoc());
   Record.AddDeclarationNameInfo(C->getMapperIdInfo());
+  Record.AddSourceLocation(C->getColonLoc());
   for (auto *E : C->varlists())
     Record.AddStmt(E);
   for (auto *E : C->mapperlists())
diff --git a/clang/test/OpenMP/declare_mapper_ast_print.c b/clang/test/OpenMP/declare_mapper_ast_print.c
index e82bc4c2d3109..b7c9976c122af 100644
--- a/clang/test/OpenMP/declare_mapper_ast_print.c
+++ b/clang/test/OpenMP/declare_mapper_ast_print.c
@@ -5,6 +5,14 @@
 // RUN: %clang_cc1 -verify -fopenmp-simd -ast-print %s | FileCheck %s
 // RUN: %clang_cc1 -fopenmp-simd -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp-simd -include-pch %t -fsyntax-only -verify %s -ast-print | FileCheck %s
+
+// RUN: %clang_cc1 -DOMP51 -verify -fopenmp -fopenmp-version=51 -ast-print %s | FileCheck -check-prefixes=CHECK,OMP51 %s
+// RUN: %clang_cc1 -DOMP51 -fopenmp -fopenmp-version=51 -emit-pch -o %t %s
+// RUN: %clang_cc1 -DOMP51 -fopenmp -fopenmp-version=51 -include-pch %t -fsyntax-only -verify %s -ast-print | FileCheck -check-prefixes=CHECK,OMP51 %s
+
+// RUN: %clang_cc1 -DOMP51 -verify -fopenmp-simd -fopenmp-version=51 -ast-print %s | FileCheck -check-prefixes=CHECK,OMP51 %s
+// RUN: %clang_cc1 -DOMP51 -fopenmp-simd -fopenmp-version=51 -emit-pch -o %t %s
+// RUN: %clang_cc1 -DOMP51 -fopenmp-simd -fopenmp-version=51 -include-pch %t -fsyntax-only -verify %s -ast-print | FileCheck -check-prefixes=CHECK,OMP51 %s
 // expected-no-diagnostics
 
 #ifndef HEADER
@@ -50,6 +58,12 @@ int main() {
     { dd[0].i++; }
 #pragma omp target update to(mapper(id): vv) from(mapper(default): dd[0:10])
 // CHECK: #pragma omp target update to(mapper(id): vv) from(mapper(default): dd[0:10])
+#ifdef OMP51
+#pragma omp target update to(mapper(id) present: vv) from(mapper(default), present: dd[0:10])
+// OMP51: #pragma omp target update to(mapper(id), present: vv) from(mapper(default), present: dd[0:10])
+#pragma omp target update to(present mapper(id): vv) from(present, mapper(default): dd[0:10])
+// OMP51: #pragma omp target update to(present, mapper(id): vv) from(present, mapper(default): dd[0:10])
+#endif
   }
   return 0;
 }
diff --git a/clang/test/OpenMP/declare_mapper_codegen.cpp b/clang/test/OpenMP/declare_mapper_codegen.cpp
index ee64fe2099007..2fd4b3cb7ed07 100644
--- a/clang/test/OpenMP/declare_mapper_codegen.cpp
+++ b/clang/test/OpenMP/declare_mapper_codegen.cpp
@@ -839,4 +839,242 @@ void foo(int a){
 
 #endif // CK3
 
+///==========================================================================///
+// RUN: %clang_cc1 -DCK4 -verify -fopenmp -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm -femit-all-decls -disable-llvm-passes %s -o - | FileCheck --check-prefix CK4 --check-prefix CK4-64 %s
+// RUN: %clang_cc1 -DCK4 -fopenmp -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -femit-all-decls -disable-llvm-passes -o %t %s
+// RUN: %clang_cc1 -DCK4 -fopenmp -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -femit-all-decls -disable-llvm-passes -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix CK4 --check-prefix CK4-64 %s
+// RUN: %clang_cc1 -DCK4 -verify -fopenmp -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm -femit-all-decls -disable-llvm-passes %s -o - | FileCheck --check-prefix CK4 --check-prefix CK4-32 %s
+// RUN: %clang_cc1 -DCK4 -fopenmp -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -femit-all-decls -disable-llvm-passes -o %t %s
+// RUN: %clang_cc1 -DCK4 -fopenmp -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -femit-all-decls -disable-llvm-passes -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix CK4 --check-prefix CK4-32 %s
+
+// RUN: %clang_cc1 -DCK4 -verify -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm -femit-all-decls -disable-llvm-passes %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK4 -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -femit-all-decls -disable-llvm-passes -o %t %s
+// RUN: %clang_cc1 -DCK4 -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -femit-all-decls -disable-llvm-passes -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK4 -verify -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm -femit-all-decls -disable-llvm-passes %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK4 -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -femit-all-decls -disable-llvm-passes -o %t %s
+// RUN: %clang_cc1 -DCK4 -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -femit-all-decls -disable-llvm-passes -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+
+#ifdef CK4
+// Mapper function code generation and runtime interface.
+
+// CK4-64: [[TSIZES:@.+]] = {{.+}}constant [1 x i64] [i64 16]
+// CK4-32: [[TSIZES:@.+]] = {{.+}}constant [1 x i64] [i64 8]
+// PRESENT=0x1000 | TARGET_PARAM=0x20 | TO=0x1 = 0x1021
+// CK4: [[TTYPES:@.+]] = {{.+}}constant [1 x i64] [i64 [[#0x1021]]]
+
+// CK4-64: [[FSIZES:@.+]] = {{.+}}constant [1 x i64] [i64 16]
+// CK4-32: [[FSIZES:@.+]] = {{.+}}constant [1 x i64] [i64 8]
+// PRESENT=0x1000 | TARGET_PARAM=0x20 | FROM=0x2 = 0x1022
+// CK4: [[FTYPES:@.+]] = {{.+}}constant [1 x i64] [i64 [[#0x1022]]]
+
+class C {
+public:
+  int a;
+  double *b;
+};
+
+#pragma omp declare mapper(id: C s) map(s.a, s.b[0:2])
+
+// CK4: define {{.*}}void [[MPRFUNC:@[.]omp_mapper[.].*C[.]id]](i8*{{.*}}, i8*{{.*}}, i8*{{.*}}, i64{{.*}}, i64{{.*}})
+// CK4: store i8* %{{[^,]+}}, i8** [[HANDLEADDR:%[^,]+]]
+// CK4: store i8* %{{[^,]+}}, i8** [[BPTRADDR:%[^,]+]]
+// CK4: store i8* %{{[^,]+}}, i8** [[VPTRADDR:%[^,]+]]
+// CK4: store i64 %{{[^,]+}}, i{{64|32}}* [[SIZEADDR:%[^,]+]]
+// CK4: store i64 %{{[^,]+}}, i64* [[TYPEADDR:%[^,]+]]
+// CK4-DAG: [[BYTESIZE:%.+]] = load i64, i64* [[SIZEADDR]]
+// CK4-64-DAG: [[SIZE:%.+]] = udiv exact i64 [[BYTESIZE]], 16
+// CK4-32-DAG: [[SIZE:%.+]] = udiv exact i64 [[BYTESIZE]], 8
+// CK4-DAG: [[TYPE:%.+]] = load i64, i64* [[TYPEADDR]]
+// CK4-DAG: [[HANDLE:%.+]] = load i8*, i8** [[HANDLEADDR]]
+// CK4-DAG: [[PTRBEGIN:%.+]] = bitcast i8** [[VPTRADDR]] to %class.C**
+// CK4-DAG: [[PTREND:%.+]] = getelementptr %class.C*, %class.C** [[PTRBEGIN]], i64 [[SIZE]]
+// CK4-DAG: [[BPTR:%.+]] = load i8*, i8** [[BPTRADDR]]
+// CK4-DAG: [[BEGIN:%.+]] = load i8*, i8** [[VPTRADDR]]
+// CK4: [[ISARRAY:%.+]] = icmp sge i64 [[SIZE]], 1
+// CK4: br i1 [[ISARRAY]], label %[[INITEVALDEL:[^,]+]], label %[[LHEAD:[^,]+]]
+
+// CK4: [[INITEVALDEL]]
+// CK4: [[TYPEDEL:%.+]] = and i64 [[TYPE]], 8
+// CK4: [[ISNOTDEL:%.+]] = icmp eq i64 [[TYPEDEL]], 0
+// CK4: br i1 [[ISNOTDEL]], label %[[INIT:[^,]+]], label %[[LHEAD:[^,]+]]
+// CK4: [[INIT]]
+// CK4-64-DAG: [[ARRSIZE:%.+]] = mul nuw i64 [[SIZE]], 16
+// CK4-32-DAG: [[ARRSIZE:%.+]] = mul nuw i64 [[SIZE]], 8
+// CK4-DAG: [[ITYPE:%.+]] = and i64 [[TYPE]], -4
+// CK4: call void @__tgt_push_mapper_component(i8* [[HANDLE]], i8* [[BPTR]], i8* [[BEGIN]], i64 [[ARRSIZE]], i64 [[ITYPE]])
+// CK4: br label %[[LHEAD:[^,]+]]
+
+// CK4: [[LHEAD]]
+// CK4: [[ISEMPTY:%.+]] = icmp eq %class.C** [[PTRBEGIN]], [[PTREND]]
+// CK4: br i1 [[ISEMPTY]], label %[[DONE:[^,]+]], label %[[LBODY:[^,]+]]
+// CK4: [[LBODY]]
+// CK4: [[PTR:%.+]] = phi %class.C** [ [[PTRBEGIN]], %[[LHEAD]] ], [ [[PTRNEXT:%.+]], %[[LCORRECT:[^,]+]] ]
+// CK4: [[OBJ:%.+]] = load %class.C*, %class.C** [[PTR]]
+// CK4-DAG: [[ABEGIN:%.+]] = getelementptr inbounds %class.C, %class.C* [[OBJ]], i32 0, i32 0
+// CK4-DAG: [[BBEGIN:%.+]] = getelementptr inbounds %class.C, %class.C* [[OBJ]], i32 0, i32 1
+// CK4-DAG: [[BBEGIN2:%.+]] = getelementptr inbounds %class.C, %class.C* [[OBJ]], i32 0, i32 1
+// CK4-DAG: [[BARRBEGIN:%.+]] = load double*, double** [[BBEGIN2]]
+// CK4-DAG: [[BARRBEGINGEP:%.+]] = getelementptr inbounds double, double* [[BARRBEGIN]], i[[sz:64|32]] 0
+// CK4-DAG: [[BEND:%.+]] = getelementptr double*, double** [[BBEGIN]], i32 1
+// CK4-DAG: [[ABEGINV:%.+]] = bitcast i32* [[ABEGIN]] to i8*
+// CK4-DAG: [[BENDV:%.+]] = bitcast double** [[BEND]] to i8*
+// CK4-DAG: [[ABEGINI:%.+]] = ptrtoint i8* [[ABEGINV]] to i64
+// CK4-DAG: [[BENDI:%.+]] = ptrtoint i8* [[BENDV]] to i64
+// CK4-DAG: [[CSIZE:%.+]] = sub i64 [[BENDI]], [[ABEGINI]]
+// CK4-DAG: [[CUSIZE:%.+]] = sdiv exact i64 [[CSIZE]], ptrtoint (i8* getelementptr (i8, i8* null, i32 1) to i64)
+// CK4-DAG: [[BPTRADDR0BC:%.+]] = bitcast %class.C* [[OBJ]] to i8*
+// CK4-DAG: [[PTRADDR0BC:%.+]] = bitcast i32* [[ABEGIN]] to i8*
+// CK4-DAG: [[PRESIZE:%.+]] = call i64 @__tgt_mapper_num_components(i8* [[HANDLE]])
+// CK4-DAG: [[SHIPRESIZE:%.+]] = shl i64 [[PRESIZE]], 48
+// CK4-DAG: br label %[[MEMBER:[^,]+]]
+// CK4-DAG: [[MEMBER]]
+// CK4-DAG: br i1 true, label %[[LTYPE:[^,]+]], label %[[MEMBERCOM:[^,]+]]
+// CK4-DAG: [[MEMBERCOM]]
+// CK4-DAG: [[MEMBERCOMTYPE:%.+]] = add nuw i64 32, [[SHIPRESIZE]]
+// CK4-DAG: br label %[[LTYPE]]
+// CK4-DAG: [[LTYPE]]
+// CK4-DAG: [[MEMBERTYPE:%.+]] = phi i64 [ 32, %[[MEMBER]] ], [ [[MEMBERCOMTYPE]], %[[MEMBERCOM]] ]
+// CK4-DAG: [[TYPETF:%.+]] = and i64 [[TYPE]], 3
+// CK4-DAG: [[ISALLOC:%.+]] = icmp eq i64 [[TYPETF]], 0
+// CK4-DAG: br i1 [[ISALLOC]], label %[[ALLOC:[^,]+]], label %[[ALLOCELSE:[^,]+]]
+// CK4-DAG: [[ALLOC]]
+// CK4-DAG: [[ALLOCTYPE:%.+]] = and i64 [[MEMBERTYPE]], -4
+// CK4-DAG: br label %[[TYEND:[^,]+]]
+// CK4-DAG: [[ALLOCELSE]]
+// CK4-DAG: [[ISTO:%.+]] = icmp eq i64 [[TYPETF]], 1
+// CK4-DAG: br i1 [[ISTO]], label %[[TO:[^,]+]], label %[[TOELSE:[^,]+]]
+// CK4-DAG: [[TO]]
+// CK4-DAG: [[TOTYPE:%.+]] = and i64 [[MEMBERTYPE]], -3
+// CK4-DAG: br label %[[TYEND]]
+// CK4-DAG: [[TOELSE]]
+// CK4-DAG: [[ISFROM:%.+]] = icmp eq i64 [[TYPETF]], 2
+// CK4-DAG: br i1 [[ISFROM]], label %[[FROM:[^,]+]], label %[[TYEND]]
+// CK4-DAG: [[FROM]]
+// CK4-DAG: [[FROMTYPE:%.+]] = and i64 [[MEMBERTYPE]], -2
+// CK4-DAG: br label %[[TYEND]]
+// CK4-DAG: [[TYEND]]
+// CK4-DAG: [[PHITYPE0:%.+]] = phi i64 [ [[ALLOCTYPE]], %[[ALLOC]] ], [ [[TOTYPE]], %[[TO]] ], [ [[FROMTYPE]], %[[FROM]] ], [ [[MEMBERTYPE]], %[[TOELSE]] ]
+// CK4: call void @__tgt_push_mapper_component(i8* [[HANDLE]], i8* [[BPTRADDR0BC]], i8* [[PTRADDR0BC]], i64 [[CUSIZE]], i64 [[PHITYPE0]])
+// CK4-DAG: [[BPTRADDR1BC:%.+]] = bitcast %class.C* [[OBJ]] to i8*
+// CK4-DAG: [[PTRADDR1BC:%.+]] = bitcast i32* [[ABEGIN]] to i8*
+// CK4-DAG: br label %[[MEMBER:[^,]+]]
+// CK4-DAG: [[MEMBER]]
+// CK4-DAG: br i1 false, label %[[LTYPE:[^,]+]], label %[[MEMBERCOM:[^,]+]]
+// CK4-DAG: [[MEMBERCOM]]
+// 281474976710659 == 0x1,000,000,003
+// CK4-DAG: [[MEMBERCOMTYPE:%.+]] = add nuw i64 281474976710659, [[SHIPRESIZE]]
+// CK4-DAG: br label %[[LTYPE]]
+// CK4-DAG: [[LTYPE]]
+// CK4-DAG: [[MEMBERTYPE:%.+]] = phi i64 [ 281474976710659, %[[MEMBER]] ], [ [[MEMBERCOMTYPE]], %[[MEMBERCOM]] ]
+// CK4-DAG: [[TYPETF:%.+]] = and i64 [[TYPE]], 3
+// CK4-DAG: [[ISALLOC:%.+]] = icmp eq i64 [[TYPETF]], 0
+// CK4-DAG: br i1 [[ISALLOC]], label %[[ALLOC:[^,]+]], label %[[ALLOCELSE:[^,]+]]
+// CK4-DAG: [[ALLOC]]
+// CK4-DAG: [[ALLOCTYPE:%.+]] = and i64 [[MEMBERTYPE]], -4
+// CK4-DAG: br label %[[TYEND:[^,]+]]
+// CK4-DAG: [[ALLOCELSE]]
+// CK4-DAG: [[ISTO:%.+]] = icmp eq i64 [[TYPETF]], 1
+// CK4-DAG: br i1 [[ISTO]], label %[[TO:[^,]+]], label %[[TOELSE:[^,]+]]
+// CK4-DAG: [[TO]]
+// CK4-DAG: [[TOTYPE:%.+]] = and i64 [[MEMBERTYPE]], -3
+// CK4-DAG: br label %[[TYEND]]
+// CK4-DAG: [[TOELSE]]
+// CK4-DAG: [[ISFROM:%.+]] = icmp eq i64 [[TYPETF]], 2
+// CK4-DAG: br i1 [[ISFROM]], label %[[FROM:[^,]+]], label %[[TYEND]]
+// CK4-DAG: [[FROM]]
+// CK4-DAG: [[FROMTYPE:%.+]] = and i64 [[MEMBERTYPE]], -2
+// CK4-DAG: br label %[[TYEND]]
+// CK4-DAG: [[TYEND]]
+// CK4-DAG: [[TYPE1:%.+]] = phi i64 [ [[ALLOCTYPE]], %[[ALLOC]] ], [ [[TOTYPE]], %[[TO]] ], [ [[FROMTYPE]], %[[FROM]] ], [ [[MEMBERTYPE]], %[[TOELSE]] ]
+// CK4: call void @__tgt_push_mapper_component(i8* [[HANDLE]], i8* [[BPTRADDR1BC]], i8* [[PTRADDR1BC]], i64 4, i64 [[TYPE1]])
+// CK4-DAG: [[BPTRADDR2BC:%.+]] = bitcast double** [[BBEGIN]] to i8*
+// CK4-DAG: [[PTRADDR2BC:%.+]] = bitcast double* [[BARRBEGINGEP]] to i8*
+// CK4-DAG: br label %[[MEMBER:[^,]+]]
+// CK4-DAG: [[MEMBER]]
+// CK4-DAG: br i1 false, label %[[LTYPE:[^,]+]], label %[[MEMBERCOM:[^,]+]]
+// CK4-DAG: [[MEMBERCOM]]
+// 281474976710675 == 0x1,000,000,013
+// CK4-DAG: [[MEMBERCOMTYPE:%.+]] = add nuw i64 281474976710675, [[SHIPRESIZE]]
+// CK4-DAG: br label %[[LTYPE]]
+// CK4-DAG: [[LTYPE]]
+// CK4-DAG: [[MEMBERTYPE:%.+]] = phi i64 [ 281474976710675, %[[MEMBER]] ], [ [[MEMBERCOMTYPE]], %[[MEMBERCOM]] ]
+// CK4-DAG: [[TYPETF:%.+]] = and i64 [[TYPE]], 3
+// CK4-DAG: [[ISALLOC:%.+]] = icmp eq i64 [[TYPETF]], 0
+// CK4-DAG: br i1 [[ISALLOC]], label %[[ALLOC:[^,]+]], label %[[ALLOCELSE:[^,]+]]
+// CK4-DAG: [[ALLOC]]
+// CK4-DAG: [[ALLOCTYPE:%.+]] = and i64 [[MEMBERTYPE]], -4
+// CK4-DAG: br label %[[TYEND:[^,]+]]
+// CK4-DAG: [[ALLOCELSE]]
+// CK4-DAG: [[ISTO:%.+]] = icmp eq i64 [[TYPETF]], 1
+// CK4-DAG: br i1 [[ISTO]], label %[[TO:[^,]+]], label %[[TOELSE:[^,]+]]
+// CK4-DAG: [[TO]]
+// CK4-DAG: [[TOTYPE:%.+]] = and i64 [[MEMBERTYPE]], -3
+// CK4-DAG: br label %[[TYEND]]
+// CK4-DAG: [[TOELSE]]
+// CK4-DAG: [[ISFROM:%.+]] = icmp eq i64 [[TYPETF]], 2
+// CK4-DAG: br i1 [[ISFROM]], label %[[FROM:[^,]+]], label %[[TYEND]]
+// CK4-DAG: [[FROM]]
+// CK4-DAG: [[FROMTYPE:%.+]] = and i64 [[MEMBERTYPE]], -2
+// CK4-DAG: br label %[[TYEND]]
+// CK4-DAG: [[TYEND]]
+// CK4-DAG: [[TYPE2:%.+]] = phi i64 [ [[ALLOCTYPE]], %[[ALLOC]] ], [ [[TOTYPE]], %[[TO]] ], [ [[FROMTYPE]], %[[FROM]] ], [ [[MEMBERTYPE]], %[[TOELSE]] ]
+// CK4: call void @__tgt_push_mapper_component(i8* [[HANDLE]], i8* [[BPTRADDR2BC]], i8* [[PTRADDR2BC]], i64 16, i64 [[TYPE2]])
+// CK4: [[PTRNEXT]] = getelementptr %class.C*, %class.C** [[PTR]], i32 1
+// CK4: [[ISDONE:%.+]] = icmp eq %class.C** [[PTRNEXT]], [[PTREND]]
+// CK4: br i1 [[ISDONE]], label %[[LEXIT:[^,]+]], label %[[LBODY]]
+
+// CK4: [[LEXIT]]
+// CK4: [[ISARRAY:%.+]] = icmp sge i64 [[SIZE]], 1
+// CK4: br i1 [[ISARRAY]], label %[[EVALDEL:[^,]+]], label %[[DONE]]
+// CK4: [[EVALDEL]]
+// CK4: [[TYPEDEL:%.+]] = and i64 [[TYPE]], 8
+// CK4: [[ISDEL:%.+]] = icmp ne i64 [[TYPEDEL]], 0
+// CK4: br i1 [[ISDEL]], label %[[DEL:[^,]+]], label %[[DONE]]
+// CK4: [[DEL]]
+// CK4-64-DAG: [[ARRSIZE:%.+]] = mul nuw i64 [[SIZE]], 16
+// CK4-32-DAG: [[ARRSIZE:%.+]] = mul nuw i64 [[SIZE]], 8
+// CK4-DAG: [[DTYPE:%.+]] = and i64 [[TYPE]], -4
+// CK4: call void @__tgt_push_mapper_component(i8* [[HANDLE]], i8* [[BPTR]], i8* [[BEGIN]], i64 [[ARRSIZE]], i64 [[DTYPE]])
+// CK4: br label %[[DONE]]
+// CK4: [[DONE]]
+// CK4: ret void
+
+
+// CK4-LABEL: define {{.*}}void @{{.*}}foo{{.*}}
+void foo(int a){
+  int i = a;
+  C c;
+  c.a = a;
+
+  // CK4-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 1, i8** [[TGEPBP:%.+]], i8** [[TGEPP:%.+]], i64* getelementptr {{.+}}[1 x i64]* [[TSIZES]], i32 0, i32 0), {{.+}}getelementptr {{.+}}[1 x i64]* [[TTYPES]]{{.+}}, i8** [[TMPRGEP:%.+]])
+  // CK4-DAG: [[TGEPBP]] = getelementptr inbounds {{.+}}[[TBP:%[^,]+]], i{{.+}} 0, i{{.+}} 0
+  // CK4-DAG: [[TGEPP]] = getelementptr inbounds {{.+}}[[TP:%[^,]+]], i{{.+}} 0, i{{.+}} 0
+  // CK4-DAG: [[TMPRGEP]] = bitcast [1 x i8*]* [[TMPR:%[^,]+]] to i8**
+  // CK4-DAG: [[TBP0:%.+]] = getelementptr inbounds {{.+}}[[TBP]], i{{.+}} 0, i{{.+}} 0
+  // CK4-DAG: [[TP0:%.+]] = getelementptr inbounds {{.+}}[[TP]], i{{.+}} 0, i{{.+}} 0
+  // CK4-DAG: [[TMPR1:%.+]] = getelementptr inbounds {{.+}}[[TMPR]], i[[sz]] 0, i[[sz]] 0
+  // CK4-DAG: [[TCBP0:%.+]] = bitcast i8** [[TBP0]] to %class.C**
+  // CK4-DAG: [[TCP0:%.+]] = bitcast i8** [[TP0]] to %class.C**
+  // CK4-DAG: store %class.C* [[VAL:%[^,]+]], %class.C** [[TCBP0]]
+  // CK4-DAG: store %class.C* [[VAL]], %class.C** [[TCP0]]
+  // CK4-DAG: store i8* bitcast (void (i8*, i8*, i8*, i64, i64)* [[MPRFUNC]] to i8*), i8** [[TMPR1]]
+  #pragma omp target update to(present, mapper(id): c)
+
+  // CK4-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 1, i8** [[FGEPBP:%.+]], i8** [[FGEPP:%.+]], i64* getelementptr {{.+}}[1 x i64]* [[FSIZES]], i32 0, i32 0), {{.+}}getelementptr {{.+}}[1 x i64]* [[FTYPES]]{{.+}}, i8** [[FMPRGEP:%.+]])
+  // CK4-DAG: [[FGEPBP]] = getelementptr inbounds {{.+}}[[FBP:%[^,]+]], i{{.+}} 0, i{{.+}} 0
+  // CK4-DAG: [[FGEPP]] = getelementptr inbounds {{.+}}[[FP:%[^,]+]], i{{.+}} 0, i{{.+}} 0
+  // CK4-DAG: [[FMPRGEP]] = bitcast [1 x i8*]* [[FMPR:%[^,]+]] to i8**
+  // CK4-DAG: [[FBP0:%.+]] = getelementptr inbounds {{.+}}[[FBP]], i{{.+}} 0, i{{.+}} 0
+  // CK4-DAG: [[FP0:%.+]] = getelementptr inbounds {{.+}}[[FP]], i{{.+}} 0, i{{.+}} 0
+  // CK4-DAG: [[FMPR1:%.+]] = getelementptr inbounds {{.+}}[[FMPR]], i[[sz]] 0, i[[sz]] 0
+  // CK4-DAG: [[FCBP0:%.+]] = bitcast i8** [[FBP0]] to %class.C**
+  // CK4-DAG: [[FCP0:%.+]] = bitcast i8** [[FP0]] to %class.C**
+  // CK4-DAG: store %class.C* [[VAL]], %class.C** [[FCBP0]]
+  // CK4-DAG: store %class.C* [[VAL]], %class.C** [[FCP0]]
+  // CK4-DAG: store i8* bitcast (void (i8*, i8*, i8*, i64, i64)* [[MPRFUNC]] to i8*), i8** [[FMPR1]]
+  #pragma omp target update from(mapper(id), present: c)
+}
+
+#endif // CK4
+
 #endif // HEADER
diff --git a/clang/test/OpenMP/target_update_ast_print.cpp b/clang/test/OpenMP/target_update_ast_print.cpp
index 0111432fde8f6..745233c430eb1 100644
--- a/clang/test/OpenMP/target_update_ast_print.cpp
+++ b/clang/test/OpenMP/target_update_ast_print.cpp
@@ -5,6 +5,14 @@
 // RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=50 -ast-print %s | FileCheck %s
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=50 -x c++ -std=c++11 -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=50 -std=c++11 -include-pch %t -fsyntax-only -verify %s -ast-print | FileCheck %s
+
+// RUN: %clang_cc1 -DOMP51 -verify -fopenmp -fopenmp-version=51 -ast-print %s | FileCheck -check-prefixes=CHECK,OMP51 %s
+// RUN: %clang_cc1 -DOMP51 -fopenmp -fopenmp-version=51 -x c++ -std=c++11 -emit-pch -o %t %s
+// RUN: %clang_cc1 -DOMP51 -fopenmp -fopenmp-version=51 -std=c++11 -include-pch %t -fsyntax-only -verify %s -ast-print | FileCheck -check-prefixes=CHECK,OMP51 %s
+
+// RUN: %clang_cc1 -DOMP51 -verify -fopenmp-simd -fopenmp-version=51 -ast-print %s | FileCheck -check-prefixes=CHECK,OMP51 %s
+// RUN: %clang_cc1 -DOMP51 -fopenmp-simd -fopenmp-version=51 -x c++ -std=c++11 -emit-pch -o %t %s
+// RUN: %clang_cc1 -DOMP51 -fopenmp-simd -fopenmp-version=51 -std=c++11 -include-pch %t -fsyntax-only -verify %s -ast-print | FileCheck -check-prefixes=CHECK,OMP51 %s
 // expected-no-diagnostics
 
 #ifndef HEADER
@@ -25,6 +33,13 @@ T foo(T targ, U uarg) {
 #pragma omp target update to(arr[2][0:1:2])
 
 #pragma omp target update from(arr[2][0:1:2])
+
+#ifdef OMP51
+#pragma omp target update to(present: arr[2][0:1:2])
+
+#pragma omp target update from(present: arr[2][0:1:2], a)
+#endif
+
   return a + targ + (T)b;
 }
 // CHECK:      static T a, *p;
@@ -45,6 +60,8 @@ T foo(T targ, U uarg) {
 // CHECK:      int arr[100][100];
 // CHECK-NEXT: #pragma omp target update to(arr[2][0:1:2])
 // CHECK-NEXT: #pragma omp target update from(arr[2][0:1:2])
+// OMP5-NEXT: #pragma omp target update to(present: arr[2][0:1:2])
+// OMP5-NEXT: #pragma omp target update from(present: arr[2][0:1:2], a)
 
 int main(int argc, char **argv) {
   static int a;
@@ -62,6 +79,13 @@ int main(int argc, char **argv) {
 // CHECK-NEXT: #pragma omp target update to(argv[2][0:1:2])
 #pragma omp target update from(argv[2][0:1:2])
 // CHECK-NEXT: #pragma omp target update from(argv[2][0:1:2])
+#ifdef OMP51
+#pragma omp target update to(present: argv[2][0:1:2])
+// OMP5-NEXT: #pragma omp target update to(present: arr[2][0:1:2])
+#pragma omp target update from(argv[2][0:1:2], a)
+// OMP5-NEXT: #pragma omp target update from(present: arr[2][0:1:2], a)
+#endif
+
 
   return foo(argc, f) + foo(argv[0][0], f) + a;
 }
diff --git a/clang/test/OpenMP/target_update_codegen.cpp b/clang/test/OpenMP/target_update_codegen.cpp
index 637b580d754f2..feea1129be4cc 100644
--- a/clang/test/OpenMP/target_update_codegen.cpp
+++ b/clang/test/OpenMP/target_update_codegen.cpp
@@ -1084,5 +1084,68 @@ void array_shaping(float *f, int sa) {
   #pragma omp target update from(([sa][5])f)
 }
 
+#endif
+///==========================================================================///
+// RUN: %clang_cc1 -DCK19 -verify -fopenmp -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK19 --check-prefix CK19-64
+// RUN: %clang_cc1 -DCK19 -fopenmp -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK19 --check-prefix CK19-64
+// RUN: %clang_cc1 -DCK19 -verify -fopenmp -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK19 --check-prefix CK19-32
+// RUN: %clang_cc1 -DCK19 -fopenmp -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK19 --check-prefix CK19-32
+
+// RUN: %clang_cc1 -DCK19 -verify -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK19 -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK19 -verify -fopenmp-version=51 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK19 -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
+#ifdef CK19
+
+// PRESENT=0x1000 | TARGET_PARAM=0x20 | TO=0x1 = 0x1021
+// CK19: [[MTYPE00:@.+]] = {{.+}}constant [1 x i64] [i64 [[#0x1021]]]
+
+// PRESENT=0x1000 | TARGET_PARAM=0x20 | FROM=0x2 = 0x1022
+// CK19: [[MTYPE01:@.+]] = {{.+}}constant [1 x i64] [i64 [[#0x1022]]]
+
+// CK19-LABEL: _Z13check_presenti
+void check_present(int arg) {
+  int la;
+  float lb[arg];
+
+  // Region 00
+  // CK19-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 1, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i64* [[GEPS:%.+]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
+  // CK19-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
+  // CK19-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
+  // CK19-DAG: [[GEPS]] = getelementptr inbounds {{.+}}[[S:%[^,]+]]
+
+  // CK19-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 0
+  // CK19-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 0
+  // CK19-DAG: [[S0:%.+]] = getelementptr inbounds {{.+}}[[S]], i{{.+}} 0, i{{.+}} 0
+  // CK19-DAG: [[CBP0:%.+]] = bitcast i8** [[BP0]] to float**
+  // CK19-DAG: [[CP0:%.+]] = bitcast i8** [[P0]] to float**
+  // CK19-DAG: store float* [[VAL0:%[^,]+]], float** [[CBP0]]
+  // CK19-DAG: store float* [[VAL0]], float** [[CP0]]
+  // CK19-DAG: store i64 [[CSVAL0:%[^,]+]], i64* [[S0]]
+  #pragma omp target update to(present: lb)
+  ;
+
+  // Region 01
+  // CK19-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 1, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i64* [[GEPS:%.+]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[MTYPE01]]{{.+}}, i8** null)
+  // CK19-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
+  // CK19-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
+  // CK19-DAG: [[GEPS]] = getelementptr inbounds {{.+}}[[S:%[^,]+]]
+
+  // CK19-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 0
+  // CK19-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 0
+  // CK19-DAG: [[S0:%.+]] = getelementptr inbounds {{.+}}[[S]], i{{.+}} 0, i{{.+}} 0
+  // CK19-DAG: [[CBP0:%.+]] = bitcast i8** [[BP0]] to float**
+  // CK19-DAG: [[CP0:%.+]] = bitcast i8** [[P0]] to float**
+  // CK19-DAG: store float* [[VAL0:%[^,]+]], float** [[CBP0]]
+  // CK19-DAG: store float* [[VAL0]], float** [[CP0]]
+  // CK19-DAG: store i64 [[CSVAL0:%[^,]+]], i64* [[S0]]
+  #pragma omp target update from(present: lb)
+  ;
+}
 #endif
 #endif
diff --git a/clang/test/OpenMP/target_update_messages.cpp b/clang/test/OpenMP/target_update_messages.cpp
index 4092b623bdc97..401941b63028c 100644
--- a/clang/test/OpenMP/target_update_messages.cpp
+++ b/clang/test/OpenMP/target_update_messages.cpp
@@ -1,8 +1,10 @@
-// RUN: %clang_cc1 -verify=expected,omp45 -fopenmp -fopenmp-version=45 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
-// RUN: %clang_cc1 -verify=expected,omp50 -fopenmp -fopenmp-version=50 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,lt50,lt51 -fopenmp -fopenmp-version=45 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,ge50,lt51 -fopenmp -fopenmp-version=50 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,ge50,ge51 -fopenmp -fopenmp-version=51 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
 
-// RUN: %clang_cc1 -verify=expected,omp45 -fopenmp-simd -fopenmp-version=45 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
-// RUN: %clang_cc1 -verify=expected,omp50 -fopenmp-simd -fopenmp-version=50 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,lt50,lt51 -fopenmp-simd -fopenmp-version=45 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,ge50,lt51 -fopenmp-simd -fopenmp-version=50 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,ge50,ge51 -fopenmp-simd -fopenmp-version=51 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
 
 void xxx(int argc) {
   int x; // expected-note {{initialize the variable 'x' to silence this warning}}
@@ -25,6 +27,10 @@ int tmain(T argc, S **argv) {
   return 0;
 }
 
+struct S {
+  int i;
+};
+
 int main(int argc, char **argv) {
   int m;
   #pragma omp target update // expected-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
@@ -34,15 +40,114 @@ int main(int argc, char **argv) {
   #pragma omp target update to(m) ] // expected-warning {{extra tokens at the end of '#pragma omp target update' are ignored}}
   #pragma omp target update to(m) ) // expected-warning {{extra tokens at the end of '#pragma omp target update' are ignored}}
 
+  #pragma omp declare mapper(id: S s) map(s.i)
+  S s;
+
+  // Check parsing with no modifiers.
+  // lt51-error@+2 {{expected expression}}
+  // lt51-error@+1 {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
+  #pragma omp target update to(: s)
+  // expected-error@+2 {{expected expression}}
+  // expected-error@+1 {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
+  #pragma omp target update to(:)
+  // expected-error@+2 2 {{expected expression}}
+  // expected-error@+1 {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
+  #pragma omp target update to(,:)
+
+  // Check parsing with one modifier.
+  // expected-error@+2 {{use of undeclared identifier 'foobar'}}
+  // expected-error@+1 {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
+  #pragma omp target update to(foobar: s)
+  // expected-error@+3 {{expected ',' or ')' in 'to' clause}}
+  // expected-error@+2 {{expected ')'}}
+  // expected-note@+1 {{to match this '('}}
+  #pragma omp target update to(m: s)
+  #pragma omp target update to(mapper(id): s)
+  // lt51-error@+2 {{use of undeclared identifier 'present'}}
+  // lt51-error@+1 {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
+  #pragma omp target update to(present: s)
+  // ge51-warning@+4 {{missing ':' after motion modifier - ignoring}}
+  // lt51-warning@+3 {{missing ':' after ) - ignoring}}
+  // expected-error@+2 {{expected expression}}
+  // expected-error@+1 {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
+  #pragma omp target update to(mapper(id) s)
+  // ge51-warning@+4 {{missing ':' after motion modifier - ignoring}}
+  // ge51-error@+3 {{expected expression}}
+  // lt51-error@+2 {{use of undeclared identifier 'present'}}
+  // expected-error@+1 {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
+  #pragma omp target update to(present s)
+  // ge51-warning@+4 {{missing ':' after motion modifier - ignoring}}
+  // lt51-warning@+3 {{missing ':' after ) - ignoring}}
+  // expected-error@+2 {{expected expression}}
+  // expected-error@+1 {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
+  #pragma omp target update to(mapper(id))
+  // ge51-warning@+4 {{missing ':' after motion modifier - ignoring}}
+  // ge51-error@+3 {{expected expression}}
+  // lt51-error@+2 {{use of undeclared identifier 'present'}}
+  // expected-error@+1 {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
+  #pragma omp target update to(present)
+  // expected-error@+2 {{expected expression}}
+  // expected-error@+1 {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
+  #pragma omp target update to(mapper(id):)
+  // ge51-error@+3 {{expected expression}}
+  // lt51-error@+2 {{use of undeclared identifier 'present'}}
+  // expected-error@+1 {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
+  #pragma omp target update to(present:)
+
+  // Check parsing with two modifiers.
+  // lt51-warning@+1 {{missing ':' after ) - ignoring}}
+  #pragma omp target update to(mapper(id), present: s)
+  // lt51-error@+3 {{use of undeclared identifier 'present'}}
+  // lt51-error@+2 {{use of undeclared identifier 'id'}}
+  // lt51-error@+1 {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
+  #pragma omp target update to(present, mapper(id): s)
+  // lt51-warning@+1 {{missing ':' after ) - ignoring}}
+  #pragma omp target update to(mapper(id) present: s)
+  // lt51-error@+2 {{use of undeclared identifier 'present'}}
+  // lt51-error@+1 {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
+  #pragma omp target update to(present mapper(id): s)
+
+  // Check parsing with unnecessary commas.
+  // lt51-warning@+1 {{missing ':' after ) - ignoring}}
+  #pragma omp target update to(mapper(id),: s)
+  // lt51-error@+3 {{use of undeclared identifier 'present'}}
+  // lt51-error@+2 {{expected expression}}
+  // lt51-error@+1 {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
+  #pragma omp target update to(present , : s)
+  // ge51-warning@+2 {{missing ':' after motion modifier - ignoring}}
+  // lt51-warning@+1 {{missing ':' after ) - ignoring}}
+  #pragma omp target update to(mapper(id),,: s)
+  // ge51-warning@+5 {{missing ':' after motion modifier - ignoring}}
+  // lt51-error@+4 {{use of undeclared identifier 'present'}}
+  // lt51-error@+3 {{expected expression}}
+  // lt51-error@+2 {{expected expression}}
+  // lt51-error@+1 {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
+  #pragma omp target update to(present,,: s)
+  // lt51-warning@+1 {{missing ':' after ) - ignoring}}
+  #pragma omp target update to(mapper(id), present,: s)
+  // lt51-error@+4 {{use of undeclared identifier 'present'}}
+  // lt51-error@+3 {{use of undeclared identifier 'id'}}
+  // lt51-error@+2 {{expected expression}}
+  // lt51-error@+1 {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
+  #pragma omp target update to(present, mapper(id),: s)
+
   #pragma omp target update from(m) allocate(m) // expected-error {{unexpected OpenMP clause 'allocate' in directive '#pragma omp target update'}}
   {
     foo();
   }
 
   int iarr[5][5];
-#pragma omp target update to(iarr[0:][1:2:-1]) // omp50-error {{section stride is evaluated to a non-positive value -1}} omp45-error {{expected ']'}} omp45-note {{to match this '['}} expected-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
+// ge50-error@+4 {{section stride is evaluated to a non-positive value -1}}
+// lt50-error@+3 {{expected ']'}}
+// lt50-note@+2 {{to match this '['}}
+// expected-error@+1 {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
+#pragma omp target update to(iarr[0:][1:2:-1])
   {}
-#pragma omp target update from(iarr[0:][1:2:-1]) // omp50-error {{section stride is evaluated to a non-positive value -1}} omp45-error {{expected ']'}} omp45-note {{to match this '['}} expected-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
+// ge50-error@+4 {{section stride is evaluated to a non-positive value -1}}
+// lt50-error@+3 {{expected ']'}}
+// lt50-note@+2 {{to match this '['}}
+// expected-error@+1 {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
+#pragma omp target update from(iarr[0:][1:2:-1])
 
   return tmain(argc, argv);
 }

From 2cb926a447d22166dac0d6e6dceaf5b3616ef6a1 Mon Sep 17 00:00:00 2001
From: "Joel E. Denny" <jdenny.ornl@gmail.com>
Date: Tue, 28 Jul 2020 18:06:42 -0400
Subject: [PATCH 0382/1035] [OpenMP] Implement TR8 `present` motion modifier in
 runtime (2/2)

This patch implements OpenMP runtime support for the OpenMP TR8
`present` motion modifier for `omp target update` directives.  The
previous patch in this series implements Clang front end support.

Reviewed By: grokos

Differential Revision: https://reviews.llvm.org/D84712
---
 openmp/libomptarget/src/omptarget.cpp         |  6 ++
 .../test/mapping/present/target_update.c      | 73 +++++++++++++++++++
 2 files changed, 79 insertions(+)
 create mode 100644 openmp/libomptarget/test/mapping/present/target_update.c

diff --git a/openmp/libomptarget/src/omptarget.cpp b/openmp/libomptarget/src/omptarget.cpp
index 25722d75f6ea4..aa2bebdebd80f 100644
--- a/openmp/libomptarget/src/omptarget.cpp
+++ b/openmp/libomptarget/src/omptarget.cpp
@@ -627,6 +627,12 @@ int target_data_update(DeviceTy &Device, int32_t arg_num,
         false, IsHostPtr);
     if (!TgtPtrBegin) {
       DP("hst data:" DPxMOD " not found, becomes a noop\n", DPxPTR(HstPtrBegin));
+      if (arg_types[i] & OMP_TGT_MAPTYPE_PRESENT) {
+        MESSAGE("device mapping required by 'present' motion modifier does not "
+                "exist for host address " DPxMOD " (%ld bytes)",
+                DPxPTR(HstPtrBegin), MapSize);
+        return OFFLOAD_FAIL;
+      }
       continue;
     }
 
diff --git a/openmp/libomptarget/test/mapping/present/target_update.c b/openmp/libomptarget/test/mapping/present/target_update.c
new file mode 100644
index 0000000000000..3a3575a27d015
--- /dev/null
+++ b/openmp/libomptarget/test/mapping/present/target_update.c
@@ -0,0 +1,73 @@
+// --------------------------------------------------
+// Check 'to'
+// --------------------------------------------------
+
+// RUN: %libomptarget-compile-aarch64-unknown-linux-gnu \
+// RUN:   -fopenmp-version=51 -DCLAUSE=to
+// RUN: %libomptarget-run-fail-aarch64-unknown-linux-gnu 2>&1 \
+// RUN: | %fcheck-aarch64-unknown-linux-gnu
+
+// RUN: %libomptarget-compile-powerpc64-ibm-linux-gnu \
+// RUN:   -fopenmp-version=51 -DCLAUSE=to
+// RUN: %libomptarget-run-fail-powerpc64-ibm-linux-gnu 2>&1 \
+// RUN: | %fcheck-powerpc64-ibm-linux-gnu
+
+// RUN: %libomptarget-compile-powerpc64le-ibm-linux-gnu \
+// RUN:   -fopenmp-version=51 -DCLAUSE=to
+// RUN: %libomptarget-run-fail-powerpc64le-ibm-linux-gnu 2>&1 \
+// RUN: | %fcheck-powerpc64le-ibm-linux-gnu
+
+// RUN: %libomptarget-compile-x86_64-pc-linux-gnu \
+// RUN:   -fopenmp-version=51 -DCLAUSE=to
+// RUN: %libomptarget-run-fail-x86_64-pc-linux-gnu 2>&1 \
+// RUN: | %fcheck-x86_64-pc-linux-gnu
+
+// --------------------------------------------------
+// Check 'from'
+// --------------------------------------------------
+
+// RUN: %libomptarget-compile-aarch64-unknown-linux-gnu \
+// RUN:   -fopenmp-version=51 -DCLAUSE=from
+// RUN: %libomptarget-run-fail-aarch64-unknown-linux-gnu 2>&1 \
+// RUN: | %fcheck-aarch64-unknown-linux-gnu
+
+// RUN: %libomptarget-compile-powerpc64-ibm-linux-gnu \
+// RUN:   -fopenmp-version=51 -DCLAUSE=from
+// RUN: %libomptarget-run-fail-powerpc64-ibm-linux-gnu 2>&1 \
+// RUN: | %fcheck-powerpc64-ibm-linux-gnu
+
+// RUN: %libomptarget-compile-powerpc64le-ibm-linux-gnu \
+// RUN:   -fopenmp-version=51 -DCLAUSE=from
+// RUN: %libomptarget-run-fail-powerpc64le-ibm-linux-gnu 2>&1 \
+// RUN: | %fcheck-powerpc64le-ibm-linux-gnu
+
+// RUN: %libomptarget-compile-x86_64-pc-linux-gnu \
+// RUN:   -fopenmp-version=51 -DCLAUSE=from
+// RUN: %libomptarget-run-fail-x86_64-pc-linux-gnu 2>&1 \
+// RUN: | %fcheck-x86_64-pc-linux-gnu
+
+#include <stdio.h>
+
+int main() {
+  int i;
+
+  // CHECK: addr=0x[[#%x,HOST_ADDR:]], size=[[#%u,SIZE:]]
+  fprintf(stderr, "addr=%p, size=%ld\n", &i, sizeof i);
+
+  // CHECK-NOT: Libomptarget
+#pragma omp target enter data map(alloc: i)
+#pragma omp target update CLAUSE(present: i)
+#pragma omp target exit data map(delete: i)
+
+  // CHECK: i is present
+  fprintf(stderr, "i is present\n");
+
+  // CHECK: Libomptarget message: device mapping required by 'present' motion modifier does not exist for host address 0x{{0*}}[[#HOST_ADDR]] ([[#SIZE]] bytes)
+  // CHECK: Libomptarget fatal error 1: failure of target construct while offloading is mandatory
+#pragma omp target update CLAUSE(present: i)
+
+  // CHECK-NOT: i is present
+  fprintf(stderr, "i is present\n");
+
+  return 0;
+}

From d680711b94e94e9387076a0daf2a329e304e6751 Mon Sep 17 00:00:00 2001
From: "Joel E. Denny" <jdenny.ornl@gmail.com>
Date: Tue, 28 Jul 2020 18:09:05 -0400
Subject: [PATCH 0383/1035] [FileCheck] Extend -dump-input with substitutions

Substitutions are already reported in the diagnostics appearing before
the input dump in the case of failed directives, and they're reported
in traces (produced by `-vv -dump-input=never`) in the case of
successful directives.  However, those reports are not always
convenient to view while investigating the input dump, so this patch
adds the substitution report to the input dump too.  For example:

```
$ cat check
CHECK: hello [[WHAT:[a-z]+]]
CHECK: [[VERB]] [[WHAT]]

$ FileCheck -vv -DVERB=goodbye check < input |& tail -8
<<<<<<
           1: hello world
check:1       ^~~~~~~~~~~
           2: goodbye word
check:2'0     X~~~~~~~~~~~ error: no match found
check:2'1                  with "VERB" equal to "goodbye"
check:2'2                  with "WHAT" equal to "world"
>>>>>>
```

Without this patch, the location reported for a substitution for a
directive match is the directive's full match range.  This location is
misleading as it implies the substitution itself matches that range.
This patch changes the reported location to just the match range start
to suggest the substitution is known at the start of the match.  (As
in the above example, input dumps don't mark any range for
substitutions.  The location info in that case simply identifies the
right line for the annotation.)

Reviewed By: mehdi_amini, thopre

Differential Revision: https://reviews.llvm.org/D83650
---
 llvm/include/llvm/Support/FileCheck.h         |  11 +-
 llvm/lib/Support/FileCheck.cpp                |  66 +++++---
 llvm/lib/Support/FileCheckImpl.h              |   3 +-
 .../test/FileCheck/dump-input-annotations.txt | 149 ++++++++++++++++++
 llvm/test/FileCheck/verbose.txt               |   2 +-
 llvm/utils/FileCheck/FileCheck.cpp            |  12 ++
 6 files changed, 212 insertions(+), 31 deletions(-)

diff --git a/llvm/include/llvm/Support/FileCheck.h b/llvm/include/llvm/Support/FileCheck.h
index 2f0e641394d57..1eb609ef725eb 100644
--- a/llvm/include/llvm/Support/FileCheck.h
+++ b/llvm/include/llvm/Support/FileCheck.h
@@ -39,10 +39,6 @@ struct FileCheckRequest {
   bool VerboseVerbose = false;
 };
 
-//===----------------------------------------------------------------------===//
-// Summary of a FileCheck diagnostic.
-//===----------------------------------------------------------------------===//
-
 namespace Check {
 
 enum FileCheckKind {
@@ -86,6 +82,7 @@ class FileCheckType {
 };
 } // namespace Check
 
+/// Summary of a FileCheck diagnostic.
 struct FileCheckDiag {
   /// What is the FileCheck directive for this diagnostic?
   Check::FileCheckType CheckTy;
@@ -131,8 +128,12 @@ struct FileCheckDiag {
   unsigned InputStartCol;
   unsigned InputEndLine;
   unsigned InputEndCol;
+  /// A note to replace the one normally indicated by MatchTy, or the empty
+  /// string if none.
+  std::string Note;
   FileCheckDiag(const SourceMgr &SM, const Check::FileCheckType &CheckTy,
-                SMLoc CheckLoc, MatchType MatchTy, SMRange InputRange);
+                SMLoc CheckLoc, MatchType MatchTy, SMRange InputRange,
+                StringRef Note = "");
 };
 
 class FileCheckPatternContext;
diff --git a/llvm/lib/Support/FileCheck.cpp b/llvm/lib/Support/FileCheck.cpp
index d0e79c675bcbd..fd426a16d02eb 100644
--- a/llvm/lib/Support/FileCheck.cpp
+++ b/llvm/lib/Support/FileCheck.cpp
@@ -1247,7 +1247,9 @@ unsigned Pattern::computeMatchDistance(StringRef Buffer) const {
 }
 
 void Pattern::printSubstitutions(const SourceMgr &SM, StringRef Buffer,
-                                 SMRange MatchRange) const {
+                                 SMRange Range,
+                                 FileCheckDiag::MatchType MatchTy,
+                                 std::vector<FileCheckDiag> *Diags) const {
   // Print what we know about substitutions.
   if (!Substitutions.empty()) {
     for (const auto &Substitution : Substitutions) {
@@ -1280,12 +1282,15 @@ void Pattern::printSubstitutions(const SourceMgr &SM, StringRef Buffer,
         OS.write_escaped(*MatchedValue) << "\"";
       }
 
-      if (MatchRange.isValid())
-        SM.PrintMessage(MatchRange.Start, SourceMgr::DK_Note, OS.str(),
-                        {MatchRange});
+      // We report only the start of the match/search range to suggest we are
+      // reporting the substitutions as set at the start of the match/search.
+      // Indicating a non-zero-length range might instead seem to imply that the
+      // substitution matches or was captured from exactly that range.
+      if (Diags)
+        Diags->emplace_back(SM, CheckTy, getLoc(), MatchTy,
+                            SMRange(Range.Start, Range.Start), OS.str());
       else
-        SM.PrintMessage(SMLoc::getFromPointer(Buffer.data()),
-                        SourceMgr::DK_Note, OS.str());
+        SM.PrintMessage(Range.Start, SourceMgr::DK_Note, OS.str());
     }
   }
 }
@@ -1295,14 +1300,17 @@ static SMRange ProcessMatchResult(FileCheckDiag::MatchType MatchTy,
                                   Check::FileCheckType CheckTy,
                                   StringRef Buffer, size_t Pos, size_t Len,
                                   std::vector<FileCheckDiag> *Diags,
-                                  bool AdjustPrevDiag = false) {
+                                  bool AdjustPrevDiags = false) {
   SMLoc Start = SMLoc::getFromPointer(Buffer.data() + Pos);
   SMLoc End = SMLoc::getFromPointer(Buffer.data() + Pos + Len);
   SMRange Range(Start, End);
   if (Diags) {
-    if (AdjustPrevDiag)
-      Diags->rbegin()->MatchTy = MatchTy;
-    else
+    if (AdjustPrevDiags) {
+      SMLoc CheckLoc = Diags->rbegin()->CheckLoc;
+      for (auto I = Diags->rbegin(), E = Diags->rend();
+           I != E && I->CheckLoc == CheckLoc; ++I)
+        I->MatchTy = MatchTy;
+    } else
       Diags->emplace_back(SM, CheckTy, Loc, MatchTy, Range);
   }
   return Range;
@@ -1455,8 +1463,8 @@ StringRef FileCheck::CanonicalizeFile(MemoryBuffer &MB,
 FileCheckDiag::FileCheckDiag(const SourceMgr &SM,
                              const Check::FileCheckType &CheckTy,
                              SMLoc CheckLoc, MatchType MatchTy,
-                             SMRange InputRange)
-    : CheckTy(CheckTy), CheckLoc(CheckLoc), MatchTy(MatchTy) {
+                             SMRange InputRange, StringRef Note)
+    : CheckTy(CheckTy), CheckLoc(CheckLoc), MatchTy(MatchTy), Note(Note) {
   auto Start = SM.getLineAndColumn(InputRange.Start);
   auto End = SM.getLineAndColumn(InputRange.End);
   InputStartLine = Start.first;
@@ -1863,10 +1871,13 @@ static void PrintMatch(bool ExpectedMatch, const SourceMgr &SM,
     // diagnostics.
     PrintDiag = !Diags;
   }
-  SMRange MatchRange = ProcessMatchResult(
-      ExpectedMatch ? FileCheckDiag::MatchFoundAndExpected
-                    : FileCheckDiag::MatchFoundButExcluded,
-      SM, Loc, Pat.getCheckTy(), Buffer, MatchPos, MatchLen, Diags);
+  FileCheckDiag::MatchType MatchTy = ExpectedMatch
+                                         ? FileCheckDiag::MatchFoundAndExpected
+                                         : FileCheckDiag::MatchFoundButExcluded;
+  SMRange MatchRange = ProcessMatchResult(MatchTy, SM, Loc, Pat.getCheckTy(),
+                                          Buffer, MatchPos, MatchLen, Diags);
+  if (Diags)
+    Pat.printSubstitutions(SM, Buffer, MatchRange, MatchTy, Diags);
   if (!PrintDiag)
     return;
 
@@ -1881,7 +1892,7 @@ static void PrintMatch(bool ExpectedMatch, const SourceMgr &SM,
       Loc, ExpectedMatch ? SourceMgr::DK_Remark : SourceMgr::DK_Error, Message);
   SM.PrintMessage(MatchRange.Start, SourceMgr::DK_Note, "found here",
                   {MatchRange});
-  Pat.printSubstitutions(SM, Buffer, MatchRange);
+  Pat.printSubstitutions(SM, Buffer, MatchRange, MatchTy, nullptr);
 }
 
 static void PrintMatch(bool ExpectedMatch, const SourceMgr &SM,
@@ -1914,10 +1925,13 @@ static void PrintNoMatch(bool ExpectedMatch, const SourceMgr &SM,
   // If the current position is at the end of a line, advance to the start of
   // the next line.
   Buffer = Buffer.substr(Buffer.find_first_not_of(" \t\n\r"));
-  SMRange SearchRange = ProcessMatchResult(
-      ExpectedMatch ? FileCheckDiag::MatchNoneButExpected
-                    : FileCheckDiag::MatchNoneAndExcluded,
-      SM, Loc, Pat.getCheckTy(), Buffer, 0, Buffer.size(), Diags);
+  FileCheckDiag::MatchType MatchTy = ExpectedMatch
+                                         ? FileCheckDiag::MatchNoneButExpected
+                                         : FileCheckDiag::MatchNoneAndExcluded;
+  SMRange SearchRange = ProcessMatchResult(MatchTy, SM, Loc, Pat.getCheckTy(),
+                                           Buffer, 0, Buffer.size(), Diags);
+  if (Diags)
+    Pat.printSubstitutions(SM, Buffer, SearchRange, MatchTy, Diags);
   if (!PrintDiag) {
     consumeError(std::move(MatchErrors));
     return;
@@ -1945,7 +1959,7 @@ static void PrintNoMatch(bool ExpectedMatch, const SourceMgr &SM,
   SM.PrintMessage(SearchRange.Start, SourceMgr::DK_Note, "scanning from here");
 
   // Allow the pattern to print additional information if desired.
-  Pat.printSubstitutions(SM, Buffer);
+  Pat.printSubstitutions(SM, Buffer, SearchRange, MatchTy, nullptr);
 
   if (ExpectedMatch)
     Pat.printFuzzyMatch(SM, Buffer, Diags);
@@ -2248,8 +2262,12 @@ size_t FileCheckString::CheckDag(const SourceMgr &SM, StringRef Buffer,
           SM.PrintMessage(OldStart, SourceMgr::DK_Note,
                           "match discarded, overlaps earlier DAG match here",
                           {OldRange});
-        } else
-          Diags->rbegin()->MatchTy = FileCheckDiag::MatchFoundButDiscarded;
+        } else {
+          SMLoc CheckLoc = Diags->rbegin()->CheckLoc;
+          for (auto I = Diags->rbegin(), E = Diags->rend();
+               I != E && I->CheckLoc == CheckLoc; ++I)
+            I->MatchTy = FileCheckDiag::MatchFoundButDiscarded;
+        }
       }
       MatchPos = MI->End;
     }
diff --git a/llvm/lib/Support/FileCheckImpl.h b/llvm/lib/Support/FileCheckImpl.h
index 6ca67ec2964c2..d3f4384e0df0c 100644
--- a/llvm/lib/Support/FileCheckImpl.h
+++ b/llvm/lib/Support/FileCheckImpl.h
@@ -683,7 +683,8 @@ class Pattern {
   /// Prints the value of successful substitutions or the name of the undefined
   /// string or numeric variables preventing a successful substitution.
   void printSubstitutions(const SourceMgr &SM, StringRef Buffer,
-                          SMRange MatchRange = None) const;
+                          SMRange MatchRange, FileCheckDiag::MatchType MatchTy,
+                          std::vector<FileCheckDiag> *Diags) const;
   void printFuzzyMatch(const SourceMgr &SM, StringRef Buffer,
                        std::vector<FileCheckDiag> *Diags) const;
 
diff --git a/llvm/test/FileCheck/dump-input-annotations.txt b/llvm/test/FileCheck/dump-input-annotations.txt
index 785ee23d03dd5..fe39f39186bf1 100644
--- a/llvm/test/FileCheck/dump-input-annotations.txt
+++ b/llvm/test/FileCheck/dump-input-annotations.txt
@@ -599,3 +599,152 @@
 ;    IMPNOT-NEXT:not:imp3                 !~~~~  error: no match expected
 ;    IMPNOT-NEXT:>>>>>>
 ;     IMPNOT-NOT:{{.}}
+
+;--------------------------------------------------
+; Substitutions: successful and failed positive directives.
+;--------------------------------------------------
+
+; RUN: echo 'def-match1 def-match2'  >  %t.in
+; RUN: echo 'def-match1 def-nomatch' >> %t.in
+
+; RUN: echo 'CHECK: [[DEF_MATCH1]] [[DEF_MATCH2]]'            >  %t.chk
+; RUN: echo 'CHECK: [[DEF_MATCH1]] [[UNDEF]] [[DEF_NOMATCH]]' >> %t.chk
+
+; RUN: %ProtectFileCheckOutput \
+; RUN: not FileCheck -dump-input=always -vv -input-file=%t.in %t.chk 2>&1 \
+; RUN:               -DDEF_MATCH1=def-match1 -DDEF_MATCH2=def-match2 \
+; RUN:               -DDEF_NOMATCH=foobar \
+; RUN: | FileCheck -match-full-lines %s -check-prefix=SUBST-POS
+
+;      SUBST-POS:<<<<<<
+; SUBST-POS-NEXT:           1: def-match1 def-match2
+; SUBST-POS-NEXT:check:1'0     ^~~~~~~~~~~~~~~~~~~~~
+; SUBST-POS-NEXT:check:1'1                           with "DEF_MATCH1" equal to "def-match1"
+; SUBST-POS-NEXT:check:1'2                           with "DEF_MATCH2" equal to "def-match2"
+; SUBST-POS-NEXT:           2: def-match1 def-nomatch
+; SUBST-POS-NEXT:check:2'0     X~~~~~~~~~~~~~~~~~~~~~ error: no match found
+; SUBST-POS-NEXT:check:2'1                            with "DEF_MATCH1" equal to "def-match1"
+; SUBST-POS-NEXT:check:2'2                            uses undefined variable(s): "UNDEF"
+; SUBST-POS-NEXT:check:2'3                            with "DEF_NOMATCH" equal to "foobar"
+; SUBST-POS-NEXT:check:2'4              ?             possible intended match
+; SUBST-POS-NEXT:>>>>>>
+
+;--------------------------------------------------
+; Substitutions: successful and failed negative directives.
+;
+; FIXME: The first CHECK-NOT directive below uses an undefined variable.
+; Because it therefore cannot match regardless of the input, the directive
+; succeeds.  However, it should fail in this case.  Update the example below
+; once that's fixed.
+;--------------------------------------------------
+
+; RUN: echo 'def-match1 def-nomatch' >  %t.in
+; RUN: echo 'def-match1 def-match2'  >> %t.in
+; RUN: echo 'END'                    >> %t.in
+
+; RUN: echo 'CHECK-NOT: [[DEF_MATCH1]] [[UNDEF]] [[DEF_NOMATCH]]' >  %t.chk
+; RUN: echo 'CHECK-NOT: [[DEF_MATCH1]] [[DEF_MATCH2]]'            >> %t.chk
+; RUN: echo 'CHECK: END'                                          >> %t.chk
+
+; RUN: %ProtectFileCheckOutput \
+; RUN: not FileCheck -dump-input=always -vv -input-file=%t.in %t.chk 2>&1 \
+; RUN:               -DDEF_MATCH1=def-match1 -DDEF_MATCH2=def-match2 \
+; RUN:               -DDEF_NOMATCH=foobar \
+; RUN: | FileCheck -match-full-lines %s -check-prefix=SUBST-NEG
+
+;      SUBST-NEG:<<<<<<
+; SUBST-NEG-NEXT:         1: def-match1 def-nomatch
+; SUBST-NEG-NEXT:not:1'0     X~~~~~~~~~~~~~~~~~~~~~
+; SUBST-NEG-NEXT:not:1'1                            with "DEF_MATCH1" equal to "def-match1"
+; SUBST-NEG-NEXT:not:1'2                            uses undefined variable(s): "UNDEF"
+; SUBST-NEG-NEXT:not:1'3                            with "DEF_NOMATCH" equal to "foobar"
+; SUBST-NEG-NEXT:         2: def-match1 def-match2
+; SUBST-NEG-NEXT:not:1'0     ~~~~~~~~~~~~~~~~~~~~~
+; SUBST-NEG-NEXT:not:2'0     !~~~~~~~~~~~~~~~~~~~~ error: no match expected
+; SUBST-NEG-NEXT:not:2'1                           with "DEF_MATCH1" equal to "def-match1"
+; SUBST-NEG-NEXT:not:2'2                           with "DEF_MATCH2" equal to "def-match2"
+; SUBST-NEG-NEXT:         3: END
+; SUBST-NEG-NEXT:check:3     ^~~
+; SUBST-NEG-NEXT:>>>>>>
+
+;--------------------------------------------------
+; Substitutions: CHECK-NEXT, CHECK-SAME, CHECK-DAG fixups.
+;
+; When CHECK-NEXT or CHECK-SAME fails for the wrong line, or when a CHECK-DAG
+; match is discarded, the associated diagnostic type must be converted from
+; successful to failed or discarded.  However, any substitution diagnostics must
+; be traversed to find that diagnostic.
+;--------------------------------------------------
+
+;- - - - - - - - - - - - - - - - - - - - - - - - -
+; CHECK-NEXT.
+;- - - - - - - - - - - - - - - - - - - - - - - - -
+
+; RUN: echo 'pre var' > %t.in
+
+; RUN: echo 'CHECK: pre'          >  %t.chk
+; RUN: echo 'CHECK-NEXT: [[VAR]]' >> %t.chk
+
+; RUN: %ProtectFileCheckOutput \
+; RUN: not FileCheck -dump-input=always -vv -input-file=%t.in %t.chk 2>&1 \
+; RUN:               -DVAR=var \
+; RUN: | FileCheck -match-full-lines %s -check-prefix=SUBST_NEXT
+
+;      SUBST_NEXT:<<<<<<
+; SUBST_NEXT-NEXT:          1: pre var
+; SUBST_NEXT-NEXT:check:1      ^~~
+; SUBST_NEXT-NEXT:next:2'0         !~~ error: match on wrong line
+; SUBST_NEXT-NEXT:next:2'1             with "VAR" equal to "var"
+; SUBST_NEXT-NEXT:>>>>>>
+
+;- - - - - - - - - - - - - - - - - - - - - - - - -
+; CHECK-SAME.
+;- - - - - - - - - - - - - - - - - - - - - - - - -
+
+; RUN: echo 'pre' >  %t.in
+; RUN: echo 'var' >> %t.in
+
+; RUN: echo 'CHECK: pre'          >  %t.chk
+; RUN: echo 'CHECK-SAME: [[VAR]]' >> %t.chk
+
+; RUN: %ProtectFileCheckOutput \
+; RUN: not FileCheck -dump-input=always -vv -input-file=%t.in %t.chk 2>&1 \
+; RUN:               -DVAR=var \
+; RUN: | FileCheck -match-full-lines %s -check-prefix=SUBST_SAME
+
+;      SUBST_SAME:<<<<<<
+; SUBST_SAME-NEXT:          1: pre
+; SUBST_SAME-NEXT:check:1      ^~~
+; SUBST_SAME-NEXT:          2: var
+; SUBST_SAME-NEXT:same:2'0     !~~ error: match on wrong line
+; SUBST_SAME-NEXT:same:2'1         with "VAR" equal to "var"
+; SUBST_SAME-NEXT:>>>>>>
+
+;- - - - - - - - - - - - - - - - - - - - - - - - -
+; CHECK-DAG.
+;- - - - - - - - - - - - - - - - - - - - - - - - -
+
+; RUN: echo 'var' >  %t.in
+; RUN: echo 'var' >> %t.in
+; RUN: echo 'END' >> %t.in
+
+; RUN: echo 'CHECK-DAG: var'      >  %t.chk
+; RUN: echo 'CHECK-DAG: [[VAR]]'  >> %t.chk
+; RUN: echo 'CHECK: END'          >> %t.chk
+
+; RUN: %ProtectFileCheckOutput \
+; RUN: FileCheck -dump-input=always -vv -input-file=%t.in %t.chk 2>&1 \
+; RUN:           -DVAR=var \
+; RUN: | FileCheck -match-full-lines %s -check-prefix=SUBST_DAG
+
+;      SUBST_DAG:<<<<<<
+; SUBST_DAG-NEXT:         1: var
+; SUBST_DAG-NEXT:dag:1       ^~~
+; SUBST_DAG-NEXT:dag:2'0     !~~ discard: overlaps earlier match
+; SUBST_DAG-NEXT:dag:2'1         with "VAR" equal to "var"
+; SUBST_DAG-NEXT:         2: var
+; SUBST_DAG-NEXT:dag:2'2     ^~~
+; SUBST_DAG-NEXT:dag:2'3         with "VAR" equal to "var"
+; SUBST_DAG-NEXT:         3: END
+; SUBST_DAG-NEXT:check:3     ^~~
+; SUBST_DAG-NEXT:>>>>>>
diff --git a/llvm/test/FileCheck/verbose.txt b/llvm/test/FileCheck/verbose.txt
index d0532d02f9fc6..2098ddc4a2d86 100644
--- a/llvm/test/FileCheck/verbose.txt
+++ b/llvm/test/FileCheck/verbose.txt
@@ -76,7 +76,7 @@ V-NEXT: {{^}}NUMVAR - 1:41{{$}}
 V-NEXT: {{^}}^~~~~~~~~~~~~{{$}}
 V-NEXT: verbose.txt:[[#@LINE-18]]:1: note: with "NUMVAR - 1" equal to "41"
 V-NEXT: {{^}}NUMVAR - 1:41{{$}}
-V-NEXT: {{^}}^~~~~~~~~~~~~{{$}}
+V-NEXT: {{^}}^{{$}}
 
 VV-NEXT: verbose.txt:[[#@LINE-20]]:12: remark: {{C}}HECK-NOT: excluded string not found in input
 VV-NEXT: {{C}}HECK-NOT: {{[[][[]#NUMVAR [+] 1[]][]]$}}
diff --git a/llvm/utils/FileCheck/FileCheck.cpp b/llvm/utils/FileCheck/FileCheck.cpp
index fa79c5e894894..81889b2540f9b 100644
--- a/llvm/utils/FileCheck/FileCheck.cpp
+++ b/llvm/utils/FileCheck/FileCheck.cpp
@@ -402,6 +402,18 @@ BuildInputAnnotations(const SourceMgr &SM, unsigned CheckFileBufferID,
     LabelWidth = std::max((std::string::size_type)LabelWidth, A.Label.size());
 
     A.Marker = GetMarker(DiagItr->MatchTy);
+    if (!DiagItr->Note.empty()) {
+      A.Marker.Note = DiagItr->Note;
+      // It's less confusing if notes that don't actually have ranges don't have
+      // markers.  For example, a marker for 'with "VAR" equal to "5"' would
+      // seem to indicate where "VAR" matches, but the location we actually have
+      // for the marker simply points to the start of the match/search range for
+      // the full pattern of which the substitution is potentially just one
+      // component.
+      if (DiagItr->InputStartLine == DiagItr->InputEndLine &&
+          DiagItr->InputStartCol == DiagItr->InputEndCol)
+        A.Marker.Lead = ' ';
+    }
     A.FoundAndExpectedMatch =
         DiagItr->MatchTy == FileCheckDiag::MatchFoundAndExpected;
 

From 9f86b8ec41f04fa6f087264008317515a612a922 Mon Sep 17 00:00:00 2001
From: "Joel E. Denny" <jdenny.ornl@gmail.com>
Date: Tue, 28 Jul 2020 18:09:47 -0400
Subject: [PATCH 0384/1035] [FileCheck] Report captured variables

Report captured variables in input dumps and traces.  For example:

```
$ cat check
CHECK: hello [[WHAT:[a-z]+]]
CHECK: goodbye [[WHAT]]

$ FileCheck -dump-input=always -vv check < input |& tail -8
<<<<<<
           1: hello world
check:1'0     ^~~~~~~~~~~
check:1'1           ^~~~~ captured var "WHAT"
           2: goodbye world
check:2'0     ^~~~~~~~~~~~~
check:2'1                   with "WHAT" equal to "world"
>>>>>>

$ FileCheck -dump-input=never -vv check < input
check2:1:8: remark: CHECK: expected string found in input
CHECK: hello [[WHAT:[a-z]+]]
       ^
<stdin>:1:1: note: found here
hello world
^~~~~~~~~~~
<stdin>:1:7: note: captured var "WHAT"
hello world
      ^~~~~
check2:2:8: remark: CHECK: expected string found in input
CHECK: goodbye [[WHAT]]
       ^
<stdin>:2:1: note: found here
goodbye world
^~~~~~~~~~~~~
<stdin>:2:1: note: with "WHAT" equal to "world"
goodbye world
^
```

Reviewed By: thopre

Differential Revision: https://reviews.llvm.org/D83651
---
 llvm/lib/Support/FileCheck.cpp                | 58 ++++++++++++++++++-
 llvm/lib/Support/FileCheckImpl.h              | 28 ++++++++-
 .../test/FileCheck/dump-input-annotations.txt | 58 ++++++++++++++++++-
 llvm/test/FileCheck/verbose.txt               | 38 ++++++++++--
 llvm/unittests/Support/FileCheckTest.cpp      | 46 ++++++++++++++-
 5 files changed, 213 insertions(+), 15 deletions(-)

diff --git a/llvm/lib/Support/FileCheck.cpp b/llvm/lib/Support/FileCheck.cpp
index fd426a16d02eb..29a5a1345a58a 100644
--- a/llvm/lib/Support/FileCheck.cpp
+++ b/llvm/lib/Support/FileCheck.cpp
@@ -1218,7 +1218,7 @@ Expected<size_t> Pattern::match(StringRef Buffer, size_t &MatchLen,
         Format.valueFromStringRepr(MatchedValue, SM);
     if (!Value)
       return Value.takeError();
-    DefinedNumericVariable->setValue(*Value);
+    DefinedNumericVariable->setValue(*Value, MatchedValue);
   }
 
   // Like CHECK-NEXT, CHECK-EMPTY's match range is considered to start after
@@ -1295,6 +1295,57 @@ void Pattern::printSubstitutions(const SourceMgr &SM, StringRef Buffer,
   }
 }
 
+void Pattern::printVariableDefs(const SourceMgr &SM,
+                                FileCheckDiag::MatchType MatchTy,
+                                std::vector<FileCheckDiag> *Diags) const {
+  if (VariableDefs.empty() && NumericVariableDefs.empty())
+    return;
+  // Build list of variable captures.
+  struct VarCapture {
+    StringRef Name;
+    SMRange Range;
+  };
+  SmallVector<VarCapture, 2> VarCaptures;
+  for (const auto &VariableDef : VariableDefs) {
+    VarCapture VC;
+    VC.Name = VariableDef.first;
+    StringRef Value = Context->GlobalVariableTable[VC.Name];
+    SMLoc Start = SMLoc::getFromPointer(Value.data());
+    SMLoc End = SMLoc::getFromPointer(Value.data() + Value.size());
+    VC.Range = SMRange(Start, End);
+    VarCaptures.push_back(VC);
+  }
+  for (const auto &VariableDef : NumericVariableDefs) {
+    VarCapture VC;
+    VC.Name = VariableDef.getKey();
+    StringRef StrValue = VariableDef.getValue()
+                             .DefinedNumericVariable->getStringValue()
+                             .getValue();
+    SMLoc Start = SMLoc::getFromPointer(StrValue.data());
+    SMLoc End = SMLoc::getFromPointer(StrValue.data() + StrValue.size());
+    VC.Range = SMRange(Start, End);
+    VarCaptures.push_back(VC);
+  }
+  // Sort variable captures by the order in which they matched the input.
+  // Ranges shouldn't be overlapping, so we can just compare the start.
+  std::sort(VarCaptures.begin(), VarCaptures.end(),
+            [](const VarCapture &A, const VarCapture &B) {
+              assert(A.Range.Start != B.Range.Start &&
+                     "unexpected overlapping variable captures");
+              return A.Range.Start.getPointer() < B.Range.Start.getPointer();
+            });
+  // Create notes for the sorted captures.
+  for (const VarCapture &VC : VarCaptures) {
+    SmallString<256> Msg;
+    raw_svector_ostream OS(Msg);
+    OS << "captured var \"" << VC.Name << "\"";
+    if (Diags)
+      Diags->emplace_back(SM, CheckTy, getLoc(), MatchTy, VC.Range, OS.str());
+    else
+      SM.PrintMessage(VC.Range.Start, SourceMgr::DK_Note, OS.str(), VC.Range);
+  }
+}
+
 static SMRange ProcessMatchResult(FileCheckDiag::MatchType MatchTy,
                                   const SourceMgr &SM, SMLoc Loc,
                                   Check::FileCheckType CheckTy,
@@ -1876,8 +1927,10 @@ static void PrintMatch(bool ExpectedMatch, const SourceMgr &SM,
                                          : FileCheckDiag::MatchFoundButExcluded;
   SMRange MatchRange = ProcessMatchResult(MatchTy, SM, Loc, Pat.getCheckTy(),
                                           Buffer, MatchPos, MatchLen, Diags);
-  if (Diags)
+  if (Diags) {
     Pat.printSubstitutions(SM, Buffer, MatchRange, MatchTy, Diags);
+    Pat.printVariableDefs(SM, MatchTy, Diags);
+  }
   if (!PrintDiag)
     return;
 
@@ -1893,6 +1946,7 @@ static void PrintMatch(bool ExpectedMatch, const SourceMgr &SM,
   SM.PrintMessage(MatchRange.Start, SourceMgr::DK_Note, "found here",
                   {MatchRange});
   Pat.printSubstitutions(SM, Buffer, MatchRange, MatchTy, nullptr);
+  Pat.printVariableDefs(SM, MatchTy, nullptr);
 }
 
 static void PrintMatch(bool ExpectedMatch, const SourceMgr &SM,
diff --git a/llvm/lib/Support/FileCheckImpl.h b/llvm/lib/Support/FileCheckImpl.h
index d3f4384e0df0c..edb0c2548308d 100644
--- a/llvm/lib/Support/FileCheckImpl.h
+++ b/llvm/lib/Support/FileCheckImpl.h
@@ -261,6 +261,10 @@ class NumericVariable {
   /// Value of numeric variable, if defined, or None otherwise.
   Optional<ExpressionValue> Value;
 
+  /// The input buffer's string from which Value was parsed, or None.  See
+  /// comments on getStringValue for a discussion of the None case.
+  Optional<StringRef> StrValue;
+
   /// Line number where this variable is defined, or None if defined before
   /// input is parsed. Used to determine whether a variable is defined on the
   /// same line as a given use.
@@ -284,12 +288,28 @@ class NumericVariable {
   /// \returns this variable's value.
   Optional<ExpressionValue> getValue() const { return Value; }
 
-  /// Sets value of this numeric variable to \p NewValue.
-  void setValue(ExpressionValue NewValue) { Value = NewValue; }
+  /// \returns the input buffer's string from which this variable's value was
+  /// parsed, or None if the value is not yet defined or was not parsed from the
+  /// input buffer.  For example, the value of @LINE is not parsed from the
+  /// input buffer, and some numeric variables are parsed from the command
+  /// line instead.
+  Optional<StringRef> getStringValue() const { return StrValue; }
+
+  /// Sets value of this numeric variable to \p NewValue, and sets the input
+  /// buffer string from which it was parsed to \p NewStrValue.  See comments on
+  /// getStringValue for a discussion of when the latter can be None.
+  void setValue(ExpressionValue NewValue,
+                Optional<StringRef> NewStrValue = None) {
+    Value = NewValue;
+    StrValue = NewStrValue;
+  }
 
   /// Clears value of this numeric variable, regardless of whether it is
   /// currently defined or not.
-  void clearValue() { Value = None; }
+  void clearValue() {
+    Value = None;
+    StrValue = None;
+  }
 
   /// \returns the line number where this variable is defined, if any, or None
   /// if defined before input is parsed.
@@ -691,6 +711,8 @@ class Pattern {
   bool hasVariable() const {
     return !(Substitutions.empty() && VariableDefs.empty());
   }
+  void printVariableDefs(const SourceMgr &SM, FileCheckDiag::MatchType MatchTy,
+                         std::vector<FileCheckDiag> *Diags) const;
 
   Check::FileCheckType getCheckTy() const { return CheckTy; }
 
diff --git a/llvm/test/FileCheck/dump-input-annotations.txt b/llvm/test/FileCheck/dump-input-annotations.txt
index fe39f39186bf1..80d9b1d3c4e82 100644
--- a/llvm/test/FileCheck/dump-input-annotations.txt
+++ b/llvm/test/FileCheck/dump-input-annotations.txt
@@ -668,12 +668,64 @@
 ; SUBST-NEG-NEXT:>>>>>>
 
 ;--------------------------------------------------
-; Substitutions: CHECK-NEXT, CHECK-SAME, CHECK-DAG fixups.
+; Captured variables
+;--------------------------------------------------
+
+; RUN: echo 'strvar: foo'           >  %t.in
+; RUN: echo 'numvar no expr: 51'    >> %t.in
+; RUN: echo 'numvar expr: -49'      >> %t.in
+; RUN: echo 'many: foo 100 8 bar'   >> %t.in
+; RUN: echo 'var in neg match: foo' >> %t.in
+; RUN: echo 'END'                   >> %t.in
+
+; RUN: echo 'CHECK: strvar: [[STRVAR:[a-z]+]]'                         >  %t.chk
+; RUN: echo 'CHECK: numvar no expr: [[#NUMVAR_NO_EXPR:]]'              >> %t.chk
+; RUN: echo 'CHECK: numvar expr: [[#%d,NUMVAR_EXPR:2-NUMVAR_NO_EXPR]]' >> %t.chk
+
+; Capture many variables of different kinds in a different order than their
+; names sort alphabetically to ensure they're sorted in capture order.
+; RUN: echo 'CHECK: many: [[VAR1:foo]] [[#%d,VAR3:]] [[#VAR2:]] [[VAR4:bar]]' \
+; RUN:      >> %t.chk
+
+; RUN: echo 'CHECK-NOT: var in neg match: [[VAR:foo]]' >> %t.chk
+; RUN: echo 'CHECK: END'                               >> %t.chk
+
+; RUN: %ProtectFileCheckOutput \
+; RUN: not FileCheck -dump-input=always -vv -input-file=%t.in %t.chk 2>&1 \
+; RUN: | FileCheck -strict-whitespace -match-full-lines %s -check-prefix=CAPTURE-NEG
+
+;      CAPTURE-NEG:<<<<<<
+; CAPTURE-NEG-NEXT:           1: strvar: foo
+; CAPTURE-NEG-NEXT:check:1'0     ^~~~~~~~~~~
+; CAPTURE-NEG-NEXT:check:1'1             ^~~ captured var "STRVAR"
+; CAPTURE-NEG-NEXT:           2: numvar no expr: 51
+; CAPTURE-NEG-NEXT:check:2'0     ^~~~~~~~~~~~~~~~~~
+; CAPTURE-NEG-NEXT:check:2'1                     ^~ captured var "NUMVAR_NO_EXPR"
+; CAPTURE-NEG-NEXT:           3: numvar expr: -49
+; CAPTURE-NEG-NEXT:check:3'0     ^~~~~~~~~~~~~~~~
+; CAPTURE-NEG-NEXT:check:3'1                      with "%d,NUMVAR_EXPR:2-NUMVAR_NO_EXPR" equal to "-49"
+; CAPTURE-NEG-NEXT:check:3'2                  ^~~ captured var "NUMVAR_EXPR"
+; CAPTURE-NEG-NEXT:           4: many: foo 100 8 bar
+; CAPTURE-NEG-NEXT:check:4'0     ^~~~~~~~~~~~~~~~~~~
+; CAPTURE-NEG-NEXT:check:4'1           ^~~           captured var "VAR1"
+; CAPTURE-NEG-NEXT:check:4'2               ^~~       captured var "VAR3"
+; CAPTURE-NEG-NEXT:check:4'3                   ^     captured var "VAR2"
+; CAPTURE-NEG-NEXT:check:4'4                     ^~~ captured var "VAR4"
+; CAPTURE-NEG-NEXT:           5: var in neg match: foo
+; CAPTURE-NEG-NEXT:not:5'0       !~~~~~~~~~~~~~~~~~~~~ error: no match expected
+; CAPTURE-NEG-NEXT:not:5'1                         !~~ captured var "VAR"
+; CAPTURE-NEG-NEXT:           6: END
+; CAPTURE-NEG-NEXT:check:6       ^~~
+; CAPTURE-NEG-NEXT:>>>>>>
+
+;--------------------------------------------------
+; CHECK-NEXT, CHECK-SAME, CHECK-DAG note fixups.
 ;
 ; When CHECK-NEXT or CHECK-SAME fails for the wrong line, or when a CHECK-DAG
 ; match is discarded, the associated diagnostic type must be converted from
-; successful to failed or discarded.  However, any substitution diagnostics must
-; be traversed to find that diagnostic.
+; successful to failed or discarded.  However, any note annotation must be
+; traversed to find that diagnostic.  We check this behavior here only for
+; substitutions, but it's the same mechanism for all note annotations.
 ;--------------------------------------------------
 
 ;- - - - - - - - - - - - - - - - - - - - - - - - -
diff --git a/llvm/test/FileCheck/verbose.txt b/llvm/test/FileCheck/verbose.txt
index 2098ddc4a2d86..6baf07e39c675 100644
--- a/llvm/test/FileCheck/verbose.txt
+++ b/llvm/test/FileCheck/verbose.txt
@@ -55,6 +55,31 @@ VV-NEXT: verbose.txt:[[@LINE-22]]:1: note: scanning from here
 VV-NEXT: {{^}}bar{{$}}
 VV-NEXT: {{^}}^{{$}}
 
+STRVAR=foobar
+STRVAR:foobar
+CHECK: STRVAR=[[STRVAR:[a-z]+]]
+CHECK-NEXT: STRVAR:[[STRVAR]]
+
+     V: verbose.txt:[[#@LINE-3]]:8: remark: {{C}}HECK: expected string found in input
+V-NEXT: {{C}}HECK: {{STRVAR=\[\[STRVAR:\[a-z\]\+\]\]}}
+V-NEXT:  {{^}}       ^{{$}}
+V-NEXT: verbose.txt:[[#@LINE-8]]:1: note: found here
+V-NEXT: {{^}}STRVAR=foobar{{$}}
+V-NEXT: {{^}}^~~~~~~~~~~~~{{$}}
+V-NEXT: verbose.txt:[[#@LINE-11]]:8: note: captured var "STRVAR"
+V-NEXT: {{^}}STRVAR=foobar{{$}}
+V-NEXT: {{^}}       ^~~~~~{{$}}
+
+V-NEXT: verbose.txt:[[#@LINE-12]]:13: remark: {{C}}HECK-NEXT: expected string found in input
+V-NEXT: {{C}}HECK-NEXT: {{STRVAR:\[\[STRVAR\]\]}}
+V-NEXT:  {{^}}            ^{{$}}
+V-NEXT: verbose.txt:[[#@LINE-17]]:1: note: found here
+V-NEXT: {{^}}STRVAR:foobar{{$}}
+V-NEXT: {{^}}^~~~~~~~~~~~~{{$}}
+V-NEXT: verbose.txt:[[#@LINE-20]]:1: note: with "STRVAR" equal to "foobar"
+V-NEXT: {{^}}STRVAR:foobar{{$}}
+V-NEXT: {{^}}^{{$}}
+
 NUMVAR=42
 NUMVAR - 1:41
 CHECK: NUMVAR=[[#NUMVAR:]]
@@ -67,21 +92,24 @@ V-NEXT:  {{^}}       ^{{$}}
 V-NEXT: verbose.txt:[[#@LINE-9]]:1: note: found here
 V-NEXT: {{^}}NUMVAR=42{{$}}
 V-NEXT: {{^}}^~~~~~~~~{{$}}
+V-NEXT: verbose.txt:[[#@LINE-12]]:8: note: captured var "NUMVAR"
+V-NEXT: NUMVAR=42
+V-NEXT:        ^~
 
-V-NEXT: verbose.txt:[[#@LINE-9]]:13: remark: {{C}}HECK-NEXT: expected string found in input
+V-NEXT: verbose.txt:[[#@LINE-12]]:13: remark: {{C}}HECK-NEXT: expected string found in input
 V-NEXT: {{C}}HECK-NEXT: {{NUMVAR - 1:[[][[]#NUMVAR - 1[]][]]$}}
 V-NEXT:  {{^}}            ^{{$}}
-V-NEXT: verbose.txt:[[#@LINE-15]]:1: note: found here
+V-NEXT: verbose.txt:[[#@LINE-18]]:1: note: found here
 V-NEXT: {{^}}NUMVAR - 1:41{{$}}
 V-NEXT: {{^}}^~~~~~~~~~~~~{{$}}
-V-NEXT: verbose.txt:[[#@LINE-18]]:1: note: with "NUMVAR - 1" equal to "41"
+V-NEXT: verbose.txt:[[#@LINE-21]]:1: note: with "NUMVAR - 1" equal to "41"
 V-NEXT: {{^}}NUMVAR - 1:41{{$}}
 V-NEXT: {{^}}^{{$}}
 
-VV-NEXT: verbose.txt:[[#@LINE-20]]:12: remark: {{C}}HECK-NOT: excluded string not found in input
+VV-NEXT: verbose.txt:[[#@LINE-23]]:12: remark: {{C}}HECK-NOT: excluded string not found in input
 VV-NEXT: {{C}}HECK-NOT: {{[[][[]#NUMVAR [+] 1[]][]]$}}
 VV-NEXT:   {{^}}           ^{{$}}
-VV-NEXT: verbose.txt:[[#@LINE-25]]:1: note: scanning from here
+VV-NEXT: verbose.txt:[[#@LINE-28]]:1: note: scanning from here
 VV-NEXT: {{^}}NUMVAR - 1:41{{$}}
 VV-NEXT: {{^}}^{{$}}
 
diff --git a/llvm/unittests/Support/FileCheckTest.cpp b/llvm/unittests/Support/FileCheckTest.cpp
index 8cf823425fc37..9f286849d66be 100644
--- a/llvm/unittests/Support/FileCheckTest.cpp
+++ b/llvm/unittests/Support/FileCheckTest.cpp
@@ -655,20 +655,36 @@ TEST_F(FileCheckTest, NumericVariable) {
   Expected<ExpressionValue> EvalResult = FooVarUse.eval();
   expectUndefErrors({"FOO"}, EvalResult.takeError());
 
+  // Defined variable without string: only getValue and eval return value set.
   FooVar.setValue(ExpressionValue(42u));
-
-  // Defined variable: getValue and eval return value set.
   Optional<ExpressionValue> Value = FooVar.getValue();
   ASSERT_TRUE(Value);
   EXPECT_EQ(42, cantFail(Value->getSignedValue()));
+  EXPECT_FALSE(FooVar.getStringValue());
   EvalResult = FooVarUse.eval();
   ASSERT_THAT_EXPECTED(EvalResult, Succeeded());
   EXPECT_EQ(42, cantFail(EvalResult->getSignedValue()));
 
+  // Defined variable with string: getValue, eval, and getStringValue return
+  // value set.
+  StringRef StringValue = "925";
+  FooVar.setValue(ExpressionValue(925u), StringValue);
+  Value = FooVar.getValue();
+  ASSERT_TRUE(Value);
+  EXPECT_EQ(925, cantFail(Value->getSignedValue()));
+  // getStringValue should return the same memory not just the same characters.
+  EXPECT_EQ(StringValue.begin(), FooVar.getStringValue().getValue().begin());
+  EXPECT_EQ(StringValue.end(), FooVar.getStringValue().getValue().end());
+  EvalResult = FooVarUse.eval();
+  ASSERT_THAT_EXPECTED(EvalResult, Succeeded());
+  EXPECT_EQ(925, cantFail(EvalResult->getSignedValue()));
+  EXPECT_EQ(925, cantFail(EvalResult->getSignedValue()));
+
   // Clearing variable: getValue and eval fail. Error returned by eval holds
   // the name of the cleared variable.
   FooVar.clearValue();
   EXPECT_FALSE(FooVar.getValue());
+  EXPECT_FALSE(FooVar.getStringValue());
   EvalResult = FooVarUse.eval();
   expectUndefErrors({"FOO"}, EvalResult.takeError());
 }
@@ -921,6 +937,11 @@ class PatternTester {
     size_t MatchLen;
     return P.match(BufferRef, MatchLen, SM);
   }
+
+  void printVariableDefs(FileCheckDiag::MatchType MatchTy,
+                         std::vector<FileCheckDiag> &Diags) {
+    P.printVariableDefs(SM, MatchTy, &Diags);
+  }
 };
 
 TEST_F(FileCheckTest, ParseNumericSubstitutionBlock) {
@@ -1617,4 +1638,25 @@ TEST_F(FileCheckTest, FileCheckContext) {
   ASSERT_THAT_EXPECTED(ExpressionVal, Succeeded());
   EXPECT_EQ(cantFail(ExpressionVal->getSignedValue()), 36);
 }
+
+TEST_F(FileCheckTest, CapturedVarDiags) {
+  PatternTester Tester;
+  ASSERT_FALSE(Tester.parsePattern("[[STRVAR:[a-z]+]] [[#NUMVAR:@LINE]]"));
+  EXPECT_THAT_EXPECTED(Tester.match("foobar 2"), Succeeded());
+  std::vector<FileCheckDiag> Diags;
+  Tester.printVariableDefs(FileCheckDiag::MatchFoundAndExpected, Diags);
+  EXPECT_EQ(Diags.size(), 2ul);
+  for (FileCheckDiag Diag : Diags) {
+    EXPECT_EQ(Diag.CheckTy, Check::CheckPlain);
+    EXPECT_EQ(Diag.MatchTy, FileCheckDiag::MatchFoundAndExpected);
+    EXPECT_EQ(Diag.InputStartLine, 1u);
+    EXPECT_EQ(Diag.InputEndLine, 1u);
+  }
+  EXPECT_EQ(Diags[0].InputStartCol, 1u);
+  EXPECT_EQ(Diags[0].InputEndCol, 7u);
+  EXPECT_EQ(Diags[1].InputStartCol, 8u);
+  EXPECT_EQ(Diags[1].InputEndCol, 9u);
+  EXPECT_EQ(Diags[0].Note, "captured var \"STRVAR\"");
+  EXPECT_EQ(Diags[1].Note, "captured var \"NUMVAR\"");
+}
 } // namespace

From 2fa3da7dcdd021e05899770c02e08a20c19a98b3 Mon Sep 17 00:00:00 2001
From: Jonas Devlieghere <jonas@devlieghere.com>
Date: Tue, 28 Jul 2020 14:21:03 -0700
Subject: [PATCH 0385/1035] [lldb] Remove unused option '--platform-path' for
 'target create'

Differential revision: https://reviews.llvm.org/D84800
---
 lldb/source/Commands/CommandObjectTarget.cpp | 15 ---------------
 1 file changed, 15 deletions(-)

diff --git a/lldb/source/Commands/CommandObjectTarget.cpp b/lldb/source/Commands/CommandObjectTarget.cpp
index 0a14a25107a46..e975d99ee0246 100644
--- a/lldb/source/Commands/CommandObjectTarget.cpp
+++ b/lldb/source/Commands/CommandObjectTarget.cpp
@@ -206,8 +206,6 @@ class OptionGroupDependents : public OptionGroup {
 
 #pragma mark CommandObjectTargetCreate
 
-// "target create"
-
 class CommandObjectTargetCreate : public CommandObjectParsed {
 public:
   CommandObjectTargetCreate(CommandInterpreter &interpreter)
@@ -218,9 +216,6 @@ class CommandObjectTargetCreate : public CommandObjectParsed {
         m_option_group(), m_arch_option(),
         m_core_file(LLDB_OPT_SET_1, false, "core", 'c', 0, eArgTypeFilename,
                     "Fullpath to a core file to use for this target."),
-        m_platform_path(LLDB_OPT_SET_1, false, "platform-path", 'P', 0,
-                        eArgTypePath,
-                        "Path to the remote file to use for this target."),
         m_symbol_file(LLDB_OPT_SET_1, false, "symfile", 's', 0,
                       eArgTypeFilename,
                       "Fullpath to a stand alone debug "
@@ -246,7 +241,6 @@ class CommandObjectTargetCreate : public CommandObjectParsed {
 
     m_option_group.Append(&m_arch_option, LLDB_OPT_SET_ALL, LLDB_OPT_SET_1);
     m_option_group.Append(&m_core_file, LLDB_OPT_SET_ALL, LLDB_OPT_SET_1);
-    m_option_group.Append(&m_platform_path, LLDB_OPT_SET_ALL, LLDB_OPT_SET_1);
     m_option_group.Append(&m_symbol_file, LLDB_OPT_SET_ALL, LLDB_OPT_SET_1);
     m_option_group.Append(&m_remote_file, LLDB_OPT_SET_ALL, LLDB_OPT_SET_1);
     m_option_group.Append(&m_add_dependents, LLDB_OPT_SET_ALL, LLDB_OPT_SET_1);
@@ -449,7 +443,6 @@ class CommandObjectTargetCreate : public CommandObjectParsed {
   OptionGroupOptions m_option_group;
   OptionGroupArchitecture m_arch_option;
   OptionGroupFile m_core_file;
-  OptionGroupFile m_platform_path;
   OptionGroupFile m_symbol_file;
   OptionGroupFile m_remote_file;
   OptionGroupDependents m_add_dependents;
@@ -457,8 +450,6 @@ class CommandObjectTargetCreate : public CommandObjectParsed {
 
 #pragma mark CommandObjectTargetList
 
-// "target list"
-
 class CommandObjectTargetList : public CommandObjectParsed {
 public:
   CommandObjectTargetList(CommandInterpreter &interpreter)
@@ -490,8 +481,6 @@ class CommandObjectTargetList : public CommandObjectParsed {
 
 #pragma mark CommandObjectTargetSelect
 
-// "target select"
-
 class CommandObjectTargetSelect : public CommandObjectParsed {
 public:
   CommandObjectTargetSelect(CommandInterpreter &interpreter)
@@ -551,8 +540,6 @@ class CommandObjectTargetSelect : public CommandObjectParsed {
 
 #pragma mark CommandObjectTargetDelete
 
-// "target delete"
-
 class CommandObjectTargetDelete : public CommandObjectParsed {
 public:
   CommandObjectTargetDelete(CommandInterpreter &interpreter)
@@ -697,8 +684,6 @@ class CommandObjectTargetShowLaunchEnvironment : public CommandObjectParsed {
 
 #pragma mark CommandObjectTargetVariable
 
-// "target variable"
-
 class CommandObjectTargetVariable : public CommandObjectParsed {
   static const uint32_t SHORT_OPTION_FILE = 0x66696c65; // 'file'
   static const uint32_t SHORT_OPTION_SHLB = 0x73686c62; // 'shlb'

From 686c82b9745b207227a83aa87000a6cbfe32fc20 Mon Sep 17 00:00:00 2001
From: cgyurgyik <gyurgyikcp@gmail.com>
Date: Tue, 28 Jul 2020 19:12:17 -0400
Subject: [PATCH 0386/1035] [libc] Add scaffolding for ctype and implementation
 of isalpha

Reviewed By: sivachandra

Differential Revision: https://reviews.llvm.org/D84575
---
 libc/config/linux/aarch64/entrypoints.txt |  3 +++
 libc/config/linux/api.td                  |  6 ++++++
 libc/config/linux/x86_64/entrypoints.txt  |  3 +++
 libc/include/CMakeLists.txt               |  6 +++---
 libc/include/ctype.h.def                  | 16 ++++++++++++++
 libc/spec/stdc.td                         | 17 ++++++++++++++-
 libc/src/CMakeLists.txt                   |  1 +
 libc/src/ctype/CMakeLists.txt             |  7 ++++++
 libc/src/ctype/isalpha.cpp                | 22 +++++++++++++++++++
 libc/src/ctype/isalpha.h                  | 18 ++++++++++++++++
 libc/test/src/CMakeLists.txt              |  3 ++-
 libc/test/src/ctype/CMakeLists.txt        | 11 ++++++++++
 libc/test/src/ctype/isalpha_test.cpp      | 26 +++++++++++++++++++++++
 13 files changed, 134 insertions(+), 5 deletions(-)
 create mode 100644 libc/include/ctype.h.def
 create mode 100644 libc/src/ctype/CMakeLists.txt
 create mode 100644 libc/src/ctype/isalpha.cpp
 create mode 100644 libc/src/ctype/isalpha.h
 create mode 100644 libc/test/src/ctype/CMakeLists.txt
 create mode 100644 libc/test/src/ctype/isalpha_test.cpp

diff --git a/libc/config/linux/aarch64/entrypoints.txt b/libc/config/linux/aarch64/entrypoints.txt
index 8cdc6ca68621f..7e3589bcf0ecc 100644
--- a/libc/config/linux/aarch64/entrypoints.txt
+++ b/libc/config/linux/aarch64/entrypoints.txt
@@ -1,4 +1,7 @@
 set(TARGET_LIBC_ENTRYPOINTS
+    # ctype.h entrypoints
+    libc.src.ctype.isalpha
+    
     # errno.h entrypoints
     libc.src.errno.__errno_location
 
diff --git a/libc/config/linux/api.td b/libc/config/linux/api.td
index 3abdf9f55b07a..aea3fed1b7c3a 100644
--- a/libc/config/linux/api.td
+++ b/libc/config/linux/api.td
@@ -86,6 +86,12 @@ def AssertAPI : PublicAPI<"assert.h"> {
   ];
 }
 
+def CTypeAPI : PublicAPI<"ctype.h"> {
+  let Functions = [
+    "isalpha",
+  ];
+}
+
 def MathErrHandlingMacro : MacroDef<"math_errhandling"> {
   let Defn = [{
     #ifndef math_errhandling
diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt
index 16dd702d0eb56..c361fce3dbec0 100644
--- a/libc/config/linux/x86_64/entrypoints.txt
+++ b/libc/config/linux/x86_64/entrypoints.txt
@@ -1,6 +1,9 @@
 set(TARGET_LIBC_ENTRYPOINTS
     # assert.h entrypoints
     libc.src.assert.__assert_fail
+    
+    # ctype.h entrypoints
+    libc.src.ctype.isalpha
 
     # errno.h entrypoints
     libc.src.errno.__errno_location
diff --git a/libc/include/CMakeLists.txt b/libc/include/CMakeLists.txt
index 4dcd69a62bfbf..f676342a134e5 100644
--- a/libc/include/CMakeLists.txt
+++ b/libc/include/CMakeLists.txt
@@ -17,10 +17,10 @@ add_header(
     __llvm-libc-stdc-types.h
 )
 
-add_header(
+add_gen_header(
   ctype
-  HDR
-   ctype.h
+  DEF_FILE ctype.h.def
+  GEN_HDR ctype.h
   DEPENDS
     .llvm_libc_common_h
 )
diff --git a/libc/include/ctype.h.def b/libc/include/ctype.h.def
new file mode 100644
index 0000000000000..ac52a36bf72f5
--- /dev/null
+++ b/libc/include/ctype.h.def
@@ -0,0 +1,16 @@
+//===-- C standard library header ctype.h --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_CTYPE_H
+#define LLVM_LIBC_CTYPE_H
+
+#include <__llvm-libc-common.h>
+
+%%public_api()
+
+#endif // LLVM_LIBC_CTYPE_H
diff --git a/libc/spec/stdc.td b/libc/spec/stdc.td
index 6a11b002d8742..1b93960c9a4f0 100644
--- a/libc/spec/stdc.td
+++ b/libc/spec/stdc.td
@@ -39,7 +39,21 @@ def StdC : StandardSpec<"stdc"> {
       [], // Enumerations
       []
   >;
-
+  
+  HeaderSpec CType = HeaderSpec<
+      "ctype.h",
+      [], // Macros
+      [], // Types
+      [], // Enumerations
+      [
+          FunctionSpec<
+              "isalpha",
+              RetValSpec<IntType>,
+              [ArgSpec<IntType>]
+          >,
+      ]
+  >;
+     
   HeaderSpec String = HeaderSpec<
       "string.h",
       [
@@ -386,6 +400,7 @@ def StdC : StandardSpec<"stdc"> {
 
   let Headers = [
     Assert,
+    CType,
     Errno,
     Math,
     String,
diff --git a/libc/src/CMakeLists.txt b/libc/src/CMakeLists.txt
index 88d2829d656a0..9e5b06f201ce2 100644
--- a/libc/src/CMakeLists.txt
+++ b/libc/src/CMakeLists.txt
@@ -1,4 +1,5 @@
 add_subdirectory(assert)
+add_subdirectory(ctype)
 add_subdirectory(errno)
 add_subdirectory(math)
 add_subdirectory(signal)
diff --git a/libc/src/ctype/CMakeLists.txt b/libc/src/ctype/CMakeLists.txt
new file mode 100644
index 0000000000000..686ec78785123
--- /dev/null
+++ b/libc/src/ctype/CMakeLists.txt
@@ -0,0 +1,7 @@
+add_entrypoint_object(
+  isalpha
+  SRCS
+    isalpha.cpp
+  HDRS
+    isalpha.h
+)
diff --git a/libc/src/ctype/isalpha.cpp b/libc/src/ctype/isalpha.cpp
new file mode 100644
index 0000000000000..e945ca98ffb1c
--- /dev/null
+++ b/libc/src/ctype/isalpha.cpp
@@ -0,0 +1,22 @@
+//===-- Implementation of isalpha------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/ctype/isalpha.h"
+
+#include "src/__support/common.h"
+
+namespace __llvm_libc {
+
+// TODO: Currently restricted to default locale.
+// These should be extended using locale information.
+int LLVM_LIBC_ENTRYPOINT(isalpha)(int c) {
+  const unsigned ch = c;
+  return (ch | 32) - 'a' < 26;
+}
+
+} // namespace __llvm_libc
diff --git a/libc/src/ctype/isalpha.h b/libc/src/ctype/isalpha.h
new file mode 100644
index 0000000000000..d5697a39e9aa5
--- /dev/null
+++ b/libc/src/ctype/isalpha.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for isalpha -------------------------*-C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_CTYPE_ISALPHA_H
+#define LLVM_LIBC_SRC_CTYPE_ISALPHA_H
+
+namespace __llvm_libc {
+
+int isalpha(int c);
+
+} // namespace __llvm_libc
+
+#endif //  LLVM_LIBC_SRC_CTYPE_ISALPHA_H
diff --git a/libc/test/src/CMakeLists.txt b/libc/test/src/CMakeLists.txt
index 3e7bad72482be..e6390fc7a1d65 100644
--- a/libc/test/src/CMakeLists.txt
+++ b/libc/test/src/CMakeLists.txt
@@ -1,4 +1,5 @@
 add_subdirectory(assert)
+add_subdirectory(ctype)
 add_subdirectory(errno)
 add_subdirectory(math)
 add_subdirectory(signal)
@@ -91,4 +92,4 @@ endif()
 target_link_libraries(libc-integration-test
   PRIVATE
   ${library_files}
-)
\ No newline at end of file
+)
diff --git a/libc/test/src/ctype/CMakeLists.txt b/libc/test/src/ctype/CMakeLists.txt
new file mode 100644
index 0000000000000..b7572d3c5bfad
--- /dev/null
+++ b/libc/test/src/ctype/CMakeLists.txt
@@ -0,0 +1,11 @@
+add_libc_testsuite(libc_ctype_unittests)
+
+add_libc_unittest(
+  isalpha
+  SUITE
+    libc_ctype_unittests
+  SRCS
+    isalpha_test.cpp
+  DEPENDS
+    libc.src.ctype.isalpha
+)
diff --git a/libc/test/src/ctype/isalpha_test.cpp b/libc/test/src/ctype/isalpha_test.cpp
new file mode 100644
index 0000000000000..81fc7248f8714
--- /dev/null
+++ b/libc/test/src/ctype/isalpha_test.cpp
@@ -0,0 +1,26 @@
+//===-- Unittests for isalpha----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/ctype/isalpha.h"
+#include "utils/UnitTest/Test.h"
+
+// Helper function that makes a call to isalpha a bit cleaner
+// for use with testing utilities, since it explicitly requires
+// a boolean value for EXPECT_TRUE and EXPECT_FALSE.
+bool call_isalpha(int c) { return __llvm_libc::isalpha(c); }
+
+TEST(IsAlpha, DefaultLocale) {
+  // Loops through all characters, verifying that letters return true
+  // and everything else returns false.
+  for (int ch = 0; ch < 255; ++ch) {
+    if (('a' <= ch && ch <= 'z') || ('A' <= ch && ch <= 'Z'))
+      EXPECT_TRUE(call_isalpha(ch));
+    else
+      EXPECT_FALSE(call_isalpha(ch));
+  }
+}

From 75f222b4762a660506325850bfbd3e4241a7e347 Mon Sep 17 00:00:00 2001
From: cgyurgyik <gyurgyikcp@gmail.com>
Date: Tue, 28 Jul 2020 19:19:01 -0400
Subject: [PATCH 0387/1035] [libc] [Obvious] Delete ctype.h, since ctype.h.def
 is used.

---
 libc/include/ctype.h | 46 --------------------------------------------
 1 file changed, 46 deletions(-)
 delete mode 100644 libc/include/ctype.h

diff --git a/libc/include/ctype.h b/libc/include/ctype.h
deleted file mode 100644
index 862a1fb71c17f..0000000000000
--- a/libc/include/ctype.h
+++ /dev/null
@@ -1,46 +0,0 @@
-//===-- C standard library header ctype.h ---------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIBC_CTYPE_H
-#define LLVM_LIBC_CTYPE_H
-
-#include <__llvm-libc-common.h>
-
-__BEGIN_C_DECLS
-
-int isalnum(int);
-
-int isalpha(int);
-
-int isblank(int);
-
-int iscntrl(int);
-
-int isdigit(int);
-
-int isgraph(int);
-
-int islower(int);
-
-int isprint(int);
-
-int ispunct(int);
-
-int isspace(int);
-
-int isupper(int);
-
-int isxdigit(int);
-
-int tolower(int);
-
-int toupper(int);
-
-__END_C_DECLS
-
-#endif // LLVM_LIBC_CTYPE_H

From f14472a2e327a393d4795a141b901be1dd397505 Mon Sep 17 00:00:00 2001
From: Christopher Ferris <cferris@google.com>
Date: Tue, 28 Jul 2020 16:25:49 -0700
Subject: [PATCH 0388/1035] [scudo][standalone] Add new mallopt options.

This adds the code to support calling mallopt and converting the
options to the internal Option enum.

Reviewed By: cryptoad

Differential Revision: https://reviews.llvm.org/D84806
---
 .../standalone/tests/wrappers_c_test.cpp      |  6 +++++
 .../lib/scudo/standalone/wrappers_c.inc       | 25 +++++++++++++++----
 2 files changed, 26 insertions(+), 5 deletions(-)

diff --git a/compiler-rt/lib/scudo/standalone/tests/wrappers_c_test.cpp b/compiler-rt/lib/scudo/standalone/tests/wrappers_c_test.cpp
index ed5b64c203364..3ddc4ec818327 100644
--- a/compiler-rt/lib/scudo/standalone/tests/wrappers_c_test.cpp
+++ b/compiler-rt/lib/scudo/standalone/tests/wrappers_c_test.cpp
@@ -202,6 +202,12 @@ TEST(ScudoWrappersCTest, MallOpt) {
   EXPECT_EQ(mallopt(M_DECAY_TIME, 0), 1);
   EXPECT_EQ(mallopt(M_DECAY_TIME, 1), 1);
   EXPECT_EQ(mallopt(M_DECAY_TIME, 0), 1);
+
+  if (SCUDO_ANDROID) {
+    EXPECT_EQ(mallopt(M_CACHE_COUNT_MAX, 100), 1);
+    EXPECT_EQ(mallopt(M_CACHE_SIZE_MAX, 1024 * 1024 * 2), 1);
+    EXPECT_EQ(mallopt(M_TSDS_COUNT_MAX, 10), 1);
+  }
 }
 #endif
 
diff --git a/compiler-rt/lib/scudo/standalone/wrappers_c.inc b/compiler-rt/lib/scudo/standalone/wrappers_c.inc
index da5a170569d7c..b25135b1ce1b9 100644
--- a/compiler-rt/lib/scudo/standalone/wrappers_c.inc
+++ b/compiler-rt/lib/scudo/standalone/wrappers_c.inc
@@ -155,7 +155,7 @@ void SCUDO_PREFIX(malloc_postinit)() {
                  SCUDO_PREFIX(malloc_enable));
 }
 
-INTERFACE WEAK int SCUDO_PREFIX(mallopt)(int param, UNUSED int value) {
+INTERFACE WEAK int SCUDO_PREFIX(mallopt)(int param, int value) {
   if (param == M_DECAY_TIME) {
     if (SCUDO_ANDROID) {
       if (value == 0) {
@@ -173,11 +173,26 @@ INTERFACE WEAK int SCUDO_PREFIX(mallopt)(int param, UNUSED int value) {
   } else if (param == M_PURGE) {
     SCUDO_ALLOCATOR.releaseToOS();
     return 1;
-  } else if (param == M_MEMTAG_TUNING) {
-    return SCUDO_ALLOCATOR.setOption(scudo::Option::MemtagTuning,
-                                     static_cast<scudo::sptr>(value));
+  } else {
+    scudo::Option option;
+    switch (param) {
+    case M_MEMTAG_TUNING:
+      option = scudo::Option::MemtagTuning;
+      break;
+    case M_CACHE_COUNT_MAX:
+      option = scudo::Option::MaxCacheEntriesCount;
+      break;
+    case M_CACHE_SIZE_MAX:
+      option = scudo::Option::MaxCacheEntrySize;
+      break;
+    case M_TSDS_COUNT_MAX:
+      option = scudo::Option::MaxTSDsCount;
+      break;
+    default:
+      return 0;
+    }
+    return SCUDO_ALLOCATOR.setOption(option, static_cast<scudo::sptr>(value));
   }
-  return 0;
 }
 
 INTERFACE WEAK void *SCUDO_PREFIX(aligned_alloc)(size_t alignment,

From 59f17a7d4f42fa330557e36d18f8b5f82a2c5e30 Mon Sep 17 00:00:00 2001
From: Siva Chandra Reddy <sivachandra@google.com>
Date: Tue, 28 Jul 2020 16:07:32 -0700
Subject: [PATCH 0389/1035] [libc][NFC] Move tablegen indexer class into a util
 library of its own.

This class is currently used by two tools: HdrGen and PrototypeTestGen.
We will be adding more tools based on this class so it is convenient to
keep it in a util library of its own.
---
 libc/utils/CMakeLists.txt                     |   1 +
 libc/utils/HdrGen/CMakeLists.txt              |   3 +
 .../HdrGen/PrototypeTestGen/CMakeLists.txt    |   4 +-
 .../PrototypeTestGen/PrototypeTestGen.cpp     |   2 +-
 libc/utils/HdrGen/PublicAPICommand.cpp        | 146 +----------------
 libc/utils/HdrGen/PublicAPICommand.h          |  61 +------
 libc/utils/LibcTableGenUtil/APIIndexer.cpp    | 154 ++++++++++++++++++
 libc/utils/LibcTableGenUtil/APIIndexer.h      |  66 ++++++++
 libc/utils/LibcTableGenUtil/CMakeLists.txt    |   7 +
 9 files changed, 238 insertions(+), 206 deletions(-)
 create mode 100644 libc/utils/LibcTableGenUtil/APIIndexer.cpp
 create mode 100644 libc/utils/LibcTableGenUtil/APIIndexer.h
 create mode 100644 libc/utils/LibcTableGenUtil/CMakeLists.txt

diff --git a/libc/utils/CMakeLists.txt b/libc/utils/CMakeLists.txt
index c6e877fceef7c..ec1308927bf92 100644
--- a/libc/utils/CMakeLists.txt
+++ b/libc/utils/CMakeLists.txt
@@ -1,5 +1,6 @@
 add_subdirectory(CPP)
 add_subdirectory(FPUtil)
+add_subdirectory(LibcTableGenUtil)
 add_subdirectory(HdrGen)
 add_subdirectory(MPFRWrapper)
 add_subdirectory(testutils)
diff --git a/libc/utils/HdrGen/CMakeLists.txt b/libc/utils/HdrGen/CMakeLists.txt
index 65c2d65c79181..8aaaa965acbfe 100644
--- a/libc/utils/HdrGen/CMakeLists.txt
+++ b/libc/utils/HdrGen/CMakeLists.txt
@@ -12,4 +12,7 @@ add_tablegen(libc-hdrgen llvm-libc
   PublicAPICommand.h
 )
 
+target_include_directories(libc-hdrgen PRIVATE ${LIBC_SOURCE_DIR})
+target_link_libraries(libc-hdrgen PRIVATE LibcTableGenUtil)
+
 add_subdirectory(PrototypeTestGen)
diff --git a/libc/utils/HdrGen/PrototypeTestGen/CMakeLists.txt b/libc/utils/HdrGen/PrototypeTestGen/CMakeLists.txt
index e4ad5c209cd9e..c90fde76dd58c 100644
--- a/libc/utils/HdrGen/PrototypeTestGen/CMakeLists.txt
+++ b/libc/utils/HdrGen/PrototypeTestGen/CMakeLists.txt
@@ -1,5 +1,5 @@
 add_tablegen(libc-prototype-testgen llvm-libc
   PrototypeTestGen.cpp
-  ../PublicAPICommand.cpp
-  ../Command.cpp
 )
+target_link_libraries(libc-prototype-testgen PRIVATE LibcTableGenUtil)
+target_include_directories(libc-prototype-testgen PRIVATE ${LIBC_SOURCE_DIR})
diff --git a/libc/utils/HdrGen/PrototypeTestGen/PrototypeTestGen.cpp b/libc/utils/HdrGen/PrototypeTestGen/PrototypeTestGen.cpp
index aad451a8de225..918b520d3910a 100644
--- a/libc/utils/HdrGen/PrototypeTestGen/PrototypeTestGen.cpp
+++ b/libc/utils/HdrGen/PrototypeTestGen/PrototypeTestGen.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "../PublicAPICommand.h"
+#include "utils/LibcTableGenUtil/APIIndexer.h"
 
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/CommandLine.h"
diff --git a/libc/utils/HdrGen/PublicAPICommand.cpp b/libc/utils/HdrGen/PublicAPICommand.cpp
index 11b825831e883..87b188ca8c02b 100644
--- a/libc/utils/HdrGen/PublicAPICommand.cpp
+++ b/libc/utils/HdrGen/PublicAPICommand.cpp
@@ -8,32 +8,13 @@
 
 #include "PublicAPICommand.h"
 
+#include "utils/LibcTableGenUtil/APIIndexer.h"
+
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/SourceMgr.h"
-#include "llvm/TableGen/Error.h"
 #include "llvm/TableGen/Record.h"
 
-static const char NamedTypeClassName[] = "NamedType";
-static const char PtrTypeClassName[] = "PtrType";
-static const char RestrictedPtrTypeClassName[] = "RestrictedPtrType";
-static const char ConstTypeClassName[] = "ConstType";
-static const char StructTypeClassName[] = "Struct";
-
-static const char StandardSpecClassName[] = "StandardSpec";
-static const char PublicAPIClassName[] = "PublicAPI";
-
-static bool isa(llvm::Record *Def, llvm::Record *TypeClass) {
-  llvm::RecordRecTy *RecordType = Def->getType();
-  llvm::ArrayRef<llvm::Record *> Classes = RecordType->getClasses();
-  // We want exact types. That is, we don't want the classes listed in
-  // spec.td to be subclassed. Hence, we do not want the record |Def|
-  // to be of more than one class type..
-  if (Classes.size() != 1)
-    return false;
-  return Classes[0] == TypeClass;
-}
-
 // Text blocks for macro definitions and type decls can be indented to
 // suit the surrounding tablegen listing. We need to dedent such blocks
 // before writing them out.
@@ -59,129 +40,6 @@ static void dedentAndWrite(llvm::StringRef Text, llvm::raw_ostream &OS) {
 
 namespace llvm_libc {
 
-bool APIIndexer::isaNamedType(llvm::Record *Def) {
-  return isa(Def, NamedTypeClass);
-}
-
-bool APIIndexer::isaStructType(llvm::Record *Def) {
-  return isa(Def, StructClass);
-}
-
-bool APIIndexer::isaPtrType(llvm::Record *Def) {
-  return isa(Def, PtrTypeClass);
-}
-
-bool APIIndexer::isaConstType(llvm::Record *Def) {
-  return isa(Def, ConstTypeClass);
-}
-
-bool APIIndexer::isaRestrictedPtrType(llvm::Record *Def) {
-  return isa(Def, RestrictedPtrTypeClass);
-}
-
-bool APIIndexer::isaStandardSpec(llvm::Record *Def) {
-  return isa(Def, StandardSpecClass);
-}
-
-bool APIIndexer::isaPublicAPI(llvm::Record *Def) {
-  return isa(Def, PublicAPIClass);
-}
-
-std::string APIIndexer::getTypeAsString(llvm::Record *TypeRecord) {
-  if (isaNamedType(TypeRecord) || isaStructType(TypeRecord)) {
-    return std::string(TypeRecord->getValueAsString("Name"));
-  } else if (isaPtrType(TypeRecord)) {
-    return getTypeAsString(TypeRecord->getValueAsDef("PointeeType")) + " *";
-  } else if (isaConstType(TypeRecord)) {
-    return std::string("const ") +
-           getTypeAsString(TypeRecord->getValueAsDef("UnqualifiedType"));
-  } else if (isaRestrictedPtrType(TypeRecord)) {
-    return getTypeAsString(TypeRecord->getValueAsDef("PointeeType")) +
-           " *__restrict";
-  } else {
-    llvm::PrintFatalError(TypeRecord->getLoc(), "Invalid type.\n");
-  }
-}
-
-void APIIndexer::indexStandardSpecDef(llvm::Record *StandardSpec) {
-  auto HeaderSpecList = StandardSpec->getValueAsListOfDefs("Headers");
-  for (llvm::Record *HeaderSpec : HeaderSpecList) {
-    llvm::StringRef Header = HeaderSpec->getValueAsString("Name");
-    if (!StdHeader.hasValue() || Header == StdHeader) {
-      PublicHeaders.emplace(Header);
-      auto MacroSpecList = HeaderSpec->getValueAsListOfDefs("Macros");
-      // TODO: Trigger a fatal error on duplicate specs.
-      for (llvm::Record *MacroSpec : MacroSpecList)
-        MacroSpecMap[std::string(MacroSpec->getValueAsString("Name"))] =
-            MacroSpec;
-
-      auto TypeSpecList = HeaderSpec->getValueAsListOfDefs("Types");
-      for (llvm::Record *TypeSpec : TypeSpecList)
-        TypeSpecMap[std::string(TypeSpec->getValueAsString("Name"))] = TypeSpec;
-
-      auto FunctionSpecList = HeaderSpec->getValueAsListOfDefs("Functions");
-      for (llvm::Record *FunctionSpec : FunctionSpecList) {
-        FunctionSpecMap[std::string(FunctionSpec->getValueAsString("Name"))] =
-            FunctionSpec;
-      }
-
-      auto EnumerationSpecList =
-          HeaderSpec->getValueAsListOfDefs("Enumerations");
-      for (llvm::Record *EnumerationSpec : EnumerationSpecList) {
-        EnumerationSpecMap[std::string(
-            EnumerationSpec->getValueAsString("Name"))] = EnumerationSpec;
-      }
-    }
-  }
-}
-
-void APIIndexer::indexPublicAPIDef(llvm::Record *PublicAPI) {
-  // While indexing the public API, we do not check if any of the entities
-  // requested is from an included standard. Such a check is done while
-  // generating the API.
-  auto MacroDefList = PublicAPI->getValueAsListOfDefs("Macros");
-  for (llvm::Record *MacroDef : MacroDefList)
-    MacroDefsMap[std::string(MacroDef->getValueAsString("Name"))] = MacroDef;
-
-  auto TypeDeclList = PublicAPI->getValueAsListOfDefs("TypeDeclarations");
-  for (llvm::Record *TypeDecl : TypeDeclList)
-    TypeDeclsMap[std::string(TypeDecl->getValueAsString("Name"))] = TypeDecl;
-
-  auto StructList = PublicAPI->getValueAsListOfStrings("Structs");
-  for (llvm::StringRef StructName : StructList)
-    Structs.insert(std::string(StructName));
-
-  auto FunctionList = PublicAPI->getValueAsListOfStrings("Functions");
-  for (llvm::StringRef FunctionName : FunctionList)
-    Functions.insert(std::string(FunctionName));
-
-  auto EnumerationList = PublicAPI->getValueAsListOfStrings("Enumerations");
-  for (llvm::StringRef EnumerationName : EnumerationList)
-    Enumerations.insert(std::string(EnumerationName));
-}
-
-void APIIndexer::index(llvm::RecordKeeper &Records) {
-  NamedTypeClass = Records.getClass(NamedTypeClassName);
-  PtrTypeClass = Records.getClass(PtrTypeClassName);
-  RestrictedPtrTypeClass = Records.getClass(RestrictedPtrTypeClassName);
-  StructClass = Records.getClass(StructTypeClassName);
-  ConstTypeClass = Records.getClass(ConstTypeClassName);
-  StandardSpecClass = Records.getClass(StandardSpecClassName);
-  PublicAPIClass = Records.getClass(PublicAPIClassName);
-
-  const auto &DefsMap = Records.getDefs();
-  for (auto &Pair : DefsMap) {
-    llvm::Record *Def = Pair.second.get();
-    if (isaStandardSpec(Def))
-      indexStandardSpecDef(Def);
-    if (isaPublicAPI(Def)) {
-      if (!StdHeader.hasValue() ||
-          Def->getValueAsString("HeaderName") == StdHeader)
-        indexPublicAPIDef(Def);
-    }
-  }
-}
-
 void writeAPIFromIndex(APIIndexer &G, llvm::raw_ostream &OS) {
   for (auto &Pair : G.MacroDefsMap) {
     const std::string &Name = Pair.first;
diff --git a/libc/utils/HdrGen/PublicAPICommand.h b/libc/utils/HdrGen/PublicAPICommand.h
index c95ad525647a7..bfe2e5f5d0075 100644
--- a/libc/utils/HdrGen/PublicAPICommand.h
+++ b/libc/utils/HdrGen/PublicAPICommand.h
@@ -12,10 +12,8 @@
 #include "Command.h"
 
 #include "llvm/ADT/StringRef.h"
-
-#include <string>
-#include <unordered_map>
-#include <unordered_set>
+#include "llvm/TableGen/Error.h"
+#include "llvm/TableGen/Record.h"
 
 namespace llvm {
 
@@ -36,61 +34,6 @@ class PublicAPICommand : public Command {
            const Command::ErrorReporter &Reporter) const override;
 };
 
-class APIIndexer {
-private:
-  llvm::Optional<llvm::StringRef> StdHeader;
-
-  // TableGen classes in spec.td.
-  llvm::Record *NamedTypeClass;
-  llvm::Record *PtrTypeClass;
-  llvm::Record *RestrictedPtrTypeClass;
-  llvm::Record *ConstTypeClass;
-  llvm::Record *StructClass;
-  llvm::Record *StandardSpecClass;
-  llvm::Record *PublicAPIClass;
-
-  bool isaNamedType(llvm::Record *Def);
-  bool isaStructType(llvm::Record *Def);
-  bool isaPtrType(llvm::Record *Def);
-  bool isaConstType(llvm::Record *Def);
-  bool isaRestrictedPtrType(llvm::Record *Def);
-  bool isaStandardSpec(llvm::Record *Def);
-  bool isaPublicAPI(llvm::Record *Def);
-
-  void indexStandardSpecDef(llvm::Record *StandardSpec);
-  void indexPublicAPIDef(llvm::Record *PublicAPI);
-  void index(llvm::RecordKeeper &Records);
-
-public:
-  using NameToRecordMapping = std::unordered_map<std::string, llvm::Record *>;
-  using NameSet = std::unordered_set<std::string>;
-
-  // This indexes all headers, not just a specified one.
-  explicit APIIndexer(llvm::RecordKeeper &Records) : StdHeader(llvm::None) {
-    index(Records);
-  }
-
-  APIIndexer(llvm::StringRef Header, llvm::RecordKeeper &Records)
-      : StdHeader(Header) {
-    index(Records);
-  }
-
-  // Mapping from names to records defining them.
-  NameToRecordMapping MacroSpecMap;
-  NameToRecordMapping TypeSpecMap;
-  NameToRecordMapping EnumerationSpecMap;
-  NameToRecordMapping FunctionSpecMap;
-  NameToRecordMapping MacroDefsMap;
-  NameToRecordMapping TypeDeclsMap;
-
-  NameSet Structs;
-  NameSet Enumerations;
-  NameSet Functions;
-  NameSet PublicHeaders;
-
-  std::string getTypeAsString(llvm::Record *TypeRecord);
-};
-
 } // namespace llvm_libc
 
 #endif // LLVM_LIBC_UTILS_HDRGEN_PUBLICAPICOMMAND_H
diff --git a/libc/utils/LibcTableGenUtil/APIIndexer.cpp b/libc/utils/LibcTableGenUtil/APIIndexer.cpp
new file mode 100644
index 0000000000000..934d713f905e5
--- /dev/null
+++ b/libc/utils/LibcTableGenUtil/APIIndexer.cpp
@@ -0,0 +1,154 @@
+
+#include "APIIndexer.h"
+
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/TableGen/Error.h"
+#include "llvm/TableGen/Record.h"
+
+namespace llvm_libc {
+
+static const char NamedTypeClassName[] = "NamedType";
+static const char PtrTypeClassName[] = "PtrType";
+static const char RestrictedPtrTypeClassName[] = "RestrictedPtrType";
+static const char ConstTypeClassName[] = "ConstType";
+static const char StructTypeClassName[] = "Struct";
+
+static const char StandardSpecClassName[] = "StandardSpec";
+static const char PublicAPIClassName[] = "PublicAPI";
+
+static bool isa(llvm::Record *Def, llvm::Record *TypeClass) {
+  llvm::RecordRecTy *RecordType = Def->getType();
+  llvm::ArrayRef<llvm::Record *> Classes = RecordType->getClasses();
+  // We want exact types. That is, we don't want the classes listed in
+  // spec.td to be subclassed. Hence, we do not want the record |Def|
+  // to be of more than one class type..
+  if (Classes.size() != 1)
+    return false;
+  return Classes[0] == TypeClass;
+}
+
+bool APIIndexer::isaNamedType(llvm::Record *Def) {
+  return isa(Def, NamedTypeClass);
+}
+
+bool APIIndexer::isaStructType(llvm::Record *Def) {
+  return isa(Def, StructClass);
+}
+
+bool APIIndexer::isaPtrType(llvm::Record *Def) {
+  return isa(Def, PtrTypeClass);
+}
+
+bool APIIndexer::isaConstType(llvm::Record *Def) {
+  return isa(Def, ConstTypeClass);
+}
+
+bool APIIndexer::isaRestrictedPtrType(llvm::Record *Def) {
+  return isa(Def, RestrictedPtrTypeClass);
+}
+
+bool APIIndexer::isaStandardSpec(llvm::Record *Def) {
+  return isa(Def, StandardSpecClass);
+}
+
+bool APIIndexer::isaPublicAPI(llvm::Record *Def) {
+  return isa(Def, PublicAPIClass);
+}
+
+std::string APIIndexer::getTypeAsString(llvm::Record *TypeRecord) {
+  if (isaNamedType(TypeRecord) || isaStructType(TypeRecord)) {
+    return std::string(TypeRecord->getValueAsString("Name"));
+  } else if (isaPtrType(TypeRecord)) {
+    return getTypeAsString(TypeRecord->getValueAsDef("PointeeType")) + " *";
+  } else if (isaConstType(TypeRecord)) {
+    return std::string("const ") +
+           getTypeAsString(TypeRecord->getValueAsDef("UnqualifiedType"));
+  } else if (isaRestrictedPtrType(TypeRecord)) {
+    return getTypeAsString(TypeRecord->getValueAsDef("PointeeType")) +
+           " *__restrict";
+  } else {
+    llvm::PrintFatalError(TypeRecord->getLoc(), "Invalid type.\n");
+  }
+}
+
+void APIIndexer::indexStandardSpecDef(llvm::Record *StandardSpec) {
+  auto HeaderSpecList = StandardSpec->getValueAsListOfDefs("Headers");
+  for (llvm::Record *HeaderSpec : HeaderSpecList) {
+    llvm::StringRef Header = HeaderSpec->getValueAsString("Name");
+    if (!StdHeader.hasValue() || Header == StdHeader) {
+      PublicHeaders.emplace(Header);
+      auto MacroSpecList = HeaderSpec->getValueAsListOfDefs("Macros");
+      // TODO: Trigger a fatal error on duplicate specs.
+      for (llvm::Record *MacroSpec : MacroSpecList)
+        MacroSpecMap[std::string(MacroSpec->getValueAsString("Name"))] =
+            MacroSpec;
+
+      auto TypeSpecList = HeaderSpec->getValueAsListOfDefs("Types");
+      for (llvm::Record *TypeSpec : TypeSpecList)
+        TypeSpecMap[std::string(TypeSpec->getValueAsString("Name"))] = TypeSpec;
+
+      auto FunctionSpecList = HeaderSpec->getValueAsListOfDefs("Functions");
+      for (llvm::Record *FunctionSpec : FunctionSpecList) {
+        FunctionSpecMap[std::string(FunctionSpec->getValueAsString("Name"))] =
+            FunctionSpec;
+      }
+
+      auto EnumerationSpecList =
+          HeaderSpec->getValueAsListOfDefs("Enumerations");
+      for (llvm::Record *EnumerationSpec : EnumerationSpecList) {
+        EnumerationSpecMap[std::string(
+            EnumerationSpec->getValueAsString("Name"))] = EnumerationSpec;
+      }
+    }
+  }
+}
+
+void APIIndexer::indexPublicAPIDef(llvm::Record *PublicAPI) {
+  // While indexing the public API, we do not check if any of the entities
+  // requested is from an included standard. Such a check is done while
+  // generating the API.
+  auto MacroDefList = PublicAPI->getValueAsListOfDefs("Macros");
+  for (llvm::Record *MacroDef : MacroDefList)
+    MacroDefsMap[std::string(MacroDef->getValueAsString("Name"))] = MacroDef;
+
+  auto TypeDeclList = PublicAPI->getValueAsListOfDefs("TypeDeclarations");
+  for (llvm::Record *TypeDecl : TypeDeclList)
+    TypeDeclsMap[std::string(TypeDecl->getValueAsString("Name"))] = TypeDecl;
+
+  auto StructList = PublicAPI->getValueAsListOfStrings("Structs");
+  for (llvm::StringRef StructName : StructList)
+    Structs.insert(std::string(StructName));
+
+  auto FunctionList = PublicAPI->getValueAsListOfStrings("Functions");
+  for (llvm::StringRef FunctionName : FunctionList)
+    Functions.insert(std::string(FunctionName));
+
+  auto EnumerationList = PublicAPI->getValueAsListOfStrings("Enumerations");
+  for (llvm::StringRef EnumerationName : EnumerationList)
+    Enumerations.insert(std::string(EnumerationName));
+}
+
+void APIIndexer::index(llvm::RecordKeeper &Records) {
+  NamedTypeClass = Records.getClass(NamedTypeClassName);
+  PtrTypeClass = Records.getClass(PtrTypeClassName);
+  RestrictedPtrTypeClass = Records.getClass(RestrictedPtrTypeClassName);
+  StructClass = Records.getClass(StructTypeClassName);
+  ConstTypeClass = Records.getClass(ConstTypeClassName);
+  StandardSpecClass = Records.getClass(StandardSpecClassName);
+  PublicAPIClass = Records.getClass(PublicAPIClassName);
+
+  const auto &DefsMap = Records.getDefs();
+  for (auto &Pair : DefsMap) {
+    llvm::Record *Def = Pair.second.get();
+    if (isaStandardSpec(Def))
+      indexStandardSpecDef(Def);
+    if (isaPublicAPI(Def)) {
+      if (!StdHeader.hasValue() ||
+          Def->getValueAsString("HeaderName") == StdHeader)
+        indexPublicAPIDef(Def);
+    }
+  }
+}
+
+} // namespace llvm_libc
diff --git a/libc/utils/LibcTableGenUtil/APIIndexer.h b/libc/utils/LibcTableGenUtil/APIIndexer.h
new file mode 100644
index 0000000000000..a0f0acf600d81
--- /dev/null
+++ b/libc/utils/LibcTableGenUtil/APIIndexer.h
@@ -0,0 +1,66 @@
+
+#include "llvm/ADT/StringRef.h"
+#include "llvm/TableGen/Record.h"
+
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+
+namespace llvm_libc {
+
+class APIIndexer {
+private:
+  llvm::Optional<llvm::StringRef> StdHeader;
+
+  // TableGen classes in spec.td.
+  llvm::Record *NamedTypeClass;
+  llvm::Record *PtrTypeClass;
+  llvm::Record *RestrictedPtrTypeClass;
+  llvm::Record *ConstTypeClass;
+  llvm::Record *StructClass;
+  llvm::Record *StandardSpecClass;
+  llvm::Record *PublicAPIClass;
+
+  bool isaNamedType(llvm::Record *Def);
+  bool isaStructType(llvm::Record *Def);
+  bool isaPtrType(llvm::Record *Def);
+  bool isaConstType(llvm::Record *Def);
+  bool isaRestrictedPtrType(llvm::Record *Def);
+  bool isaStandardSpec(llvm::Record *Def);
+  bool isaPublicAPI(llvm::Record *Def);
+
+  void indexStandardSpecDef(llvm::Record *StandardSpec);
+  void indexPublicAPIDef(llvm::Record *PublicAPI);
+  void index(llvm::RecordKeeper &Records);
+
+public:
+  using NameToRecordMapping = std::unordered_map<std::string, llvm::Record *>;
+  using NameSet = std::unordered_set<std::string>;
+
+  // This indexes all headers, not just a specified one.
+  explicit APIIndexer(llvm::RecordKeeper &Records) : StdHeader(llvm::None) {
+    index(Records);
+  }
+
+  APIIndexer(llvm::StringRef Header, llvm::RecordKeeper &Records)
+      : StdHeader(Header) {
+    index(Records);
+  }
+
+  // Mapping from names to records defining them.
+  NameToRecordMapping MacroSpecMap;
+  NameToRecordMapping TypeSpecMap;
+  NameToRecordMapping EnumerationSpecMap;
+  NameToRecordMapping FunctionSpecMap;
+  NameToRecordMapping MacroDefsMap;
+  NameToRecordMapping TypeDeclsMap;
+
+  NameSet Structs;
+  NameSet Enumerations;
+  NameSet Functions;
+  NameSet PublicHeaders;
+
+  std::string getTypeAsString(llvm::Record *TypeRecord);
+};
+
+} // namespace llvm_libc
diff --git a/libc/utils/LibcTableGenUtil/CMakeLists.txt b/libc/utils/LibcTableGenUtil/CMakeLists.txt
new file mode 100644
index 0000000000000..ae887a8bdb03a
--- /dev/null
+++ b/libc/utils/LibcTableGenUtil/CMakeLists.txt
@@ -0,0 +1,7 @@
+add_llvm_library(
+  LibcTableGenUtil
+  APIIndexer.cpp
+  APIIndexer.h
+  LINK_COMPONENTS Support
+)
+target_include_directories(LibcTableGenUtil PUBLIC ${LIBC_SOURCE_DIR})

From 9500d48de367d969dc944445297ae5bdfaa14079 Mon Sep 17 00:00:00 2001
From: Tim Keith <tkeith@nvidia.com>
Date: Tue, 28 Jul 2020 16:38:34 -0700
Subject: [PATCH 0390/1035] [flang][NFC] Extract name resolution for OpenACC &
 OpenMP into new file

Move `ResolveAccParts` and `ResolveOmpParts` from resolve-names.cpp to
resolve-directives.{h,cpp}. Move the implementation in the classes
`DirectiveAttributeVisitor`, `AccAttributeVisitor`, and
`OmpAttributeVisitor` to resolve-directives.cpp as well.

To allow this to happen, move `EvaluateIntExpr` and introduce
`EvaluateInt64` to resolve-names-utils.h. The latter is also useful
elsewhere in resolve-names.cpp for converting an Expr to std::int64_t.

The other problem was that `ResolveDesignator` was called from the code
that was moved. At the moment it doesn't seem to be doing anything so I
removed the calls (and no tests failed). If it proves to be needed, we
can either resolve those designators in resolve-names.cpp or pass the
`ResolveDesignator` function in to the code that needs to call it.

Differential Revision: https://reviews.llvm.org/D84768
---
 flang/lib/Semantics/CMakeLists.txt         |    3 +-
 flang/lib/Semantics/resolve-directives.cpp | 1004 +++++++++++++++++++
 flang/lib/Semantics/resolve-directives.h   |   26 +
 flang/lib/Semantics/resolve-names-utils.h  |   21 +
 flang/lib/Semantics/resolve-names.cpp      | 1050 +-------------------
 flang/test/Semantics/acc-resolve01.f90     |    2 +-
 6 files changed, 1061 insertions(+), 1045 deletions(-)
 create mode 100644 flang/lib/Semantics/resolve-directives.cpp
 create mode 100644 flang/lib/Semantics/resolve-directives.h

diff --git a/flang/lib/Semantics/CMakeLists.txt b/flang/lib/Semantics/CMakeLists.txt
index 2bdc5f9582819..5d84061437728 100644
--- a/flang/lib/Semantics/CMakeLists.txt
+++ b/flang/lib/Semantics/CMakeLists.txt
@@ -31,8 +31,9 @@ add_flang_library(FortranSemantics
   pointer-assignment.cpp
   program-tree.cpp
   resolve-labels.cpp
-  resolve-names.cpp
+  resolve-directives.cpp
   resolve-names-utils.cpp
+  resolve-names.cpp
   rewrite-parse-tree.cpp
   scope.cpp
   semantics.cpp
diff --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
new file mode 100644
index 0000000000000..d05eb5385ba87
--- /dev/null
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -0,0 +1,1004 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "resolve-directives.h"
+
+#include "check-acc-structure.h"
+#include "check-omp-structure.h"
+#include "resolve-names-utils.h"
+#include "flang/Common/idioms.h"
+#include "flang/Evaluate/fold.h"
+#include "flang/Parser/parse-tree-visitor.h"
+#include "flang/Parser/parse-tree.h"
+#include "flang/Parser/tools.h"
+#include "flang/Semantics/expression.h"
+#include <list>
+#include <map>
+
+namespace Fortran::semantics {
+
+template <typename T> class DirectiveAttributeVisitor {
+public:
+  explicit DirectiveAttributeVisitor(SemanticsContext &context)
+      : context_{context} {}
+
+  template <typename A> bool Pre(const A &) { return true; }
+  template <typename A> void Post(const A &) {}
+
+protected:
+  struct DirContext {
+    DirContext(const parser::CharBlock &source, T d, Scope &s)
+        : directiveSource{source}, directive{d}, scope{s} {}
+    parser::CharBlock directiveSource;
+    T directive;
+    Scope &scope;
+    Symbol::Flag defaultDSA{Symbol::Flag::AccShared}; // TODOACC
+    std::map<const Symbol *, Symbol::Flag> objectWithDSA;
+    bool withinConstruct{false};
+    std::int64_t associatedLoopLevel{0};
+  };
+
+  DirContext &GetContext() {
+    CHECK(!dirContext_.empty());
+    return dirContext_.back();
+  }
+  void PushContext(const parser::CharBlock &source, T dir) {
+    dirContext_.emplace_back(source, dir, context_.FindScope(source));
+  }
+  void PopContext() { dirContext_.pop_back(); }
+  void SetContextDirectiveSource(parser::CharBlock &dir) {
+    GetContext().directiveSource = dir;
+  }
+  Scope &currScope() { return GetContext().scope; }
+  void SetContextDefaultDSA(Symbol::Flag flag) {
+    GetContext().defaultDSA = flag;
+  }
+  void AddToContextObjectWithDSA(
+      const Symbol &symbol, Symbol::Flag flag, DirContext &context) {
+    context.objectWithDSA.emplace(&symbol, flag);
+  }
+  void AddToContextObjectWithDSA(const Symbol &symbol, Symbol::Flag flag) {
+    AddToContextObjectWithDSA(symbol, flag, GetContext());
+  }
+  bool IsObjectWithDSA(const Symbol &symbol) {
+    auto it{GetContext().objectWithDSA.find(&symbol)};
+    return it != GetContext().objectWithDSA.end();
+  }
+  void SetContextAssociatedLoopLevel(std::int64_t level) {
+    GetContext().associatedLoopLevel = level;
+  }
+  Symbol &MakeAssocSymbol(const SourceName &name, Symbol &prev, Scope &scope) {
+    const auto pair{scope.try_emplace(name, Attrs{}, HostAssocDetails{prev})};
+    return *pair.first->second;
+  }
+  Symbol &MakeAssocSymbol(const SourceName &name, Symbol &prev) {
+    return MakeAssocSymbol(name, prev, currScope());
+  }
+  static const parser::Name *GetDesignatorNameIfDataRef(
+      const parser::Designator &designator) {
+    const auto *dataRef{std::get_if<parser::DataRef>(&designator.u)};
+    return dataRef ? std::get_if<parser::Name>(&dataRef->u) : nullptr;
+  }
+  void AddDataSharingAttributeObject(SymbolRef object) {
+    dataSharingAttributeObjects_.insert(object);
+  }
+  void ClearDataSharingAttributeObjects() {
+    dataSharingAttributeObjects_.clear();
+  }
+  bool HasDataSharingAttributeObject(const Symbol &);
+  const parser::Name &GetLoopIndex(const parser::DoConstruct &);
+  const parser::DoConstruct *GetDoConstructIf(
+      const parser::ExecutionPartConstruct &);
+  Symbol *DeclarePrivateAccessEntity(
+      const parser::Name &, Symbol::Flag, Scope &);
+  Symbol *DeclarePrivateAccessEntity(Symbol &, Symbol::Flag, Scope &);
+  Symbol *DeclareOrMarkOtherAccessEntity(const parser::Name &, Symbol::Flag);
+
+  SymbolSet dataSharingAttributeObjects_; // on one directive
+  SemanticsContext &context_;
+  std::vector<DirContext> dirContext_; // used as a stack
+};
+
+class AccAttributeVisitor : DirectiveAttributeVisitor<llvm::acc::Directive> {
+public:
+  explicit AccAttributeVisitor(SemanticsContext &context)
+      : DirectiveAttributeVisitor(context) {}
+
+  template <typename A> void Walk(const A &x) { parser::Walk(x, *this); }
+  template <typename A> bool Pre(const A &) { return true; }
+  template <typename A> void Post(const A &) {}
+
+  bool Pre(const parser::SpecificationPart &x) {
+    Walk(std::get<std::list<parser::OpenACCDeclarativeConstruct>>(x.t));
+    return false;
+  }
+
+  bool Pre(const parser::OpenACCBlockConstruct &);
+  void Post(const parser::OpenACCBlockConstruct &) { PopContext(); }
+  bool Pre(const parser::OpenACCCombinedConstruct &);
+  void Post(const parser::OpenACCCombinedConstruct &) { PopContext(); }
+
+  void Post(const parser::AccBeginBlockDirective &) {
+    GetContext().withinConstruct = true;
+  }
+
+  bool Pre(const parser::OpenACCLoopConstruct &);
+  void Post(const parser::OpenACCLoopConstruct &) { PopContext(); }
+  void Post(const parser::AccLoopDirective &) {
+    GetContext().withinConstruct = true;
+  }
+
+  bool Pre(const parser::OpenACCStandaloneConstruct &);
+  void Post(const parser::OpenACCStandaloneConstruct &) { PopContext(); }
+  void Post(const parser::AccStandaloneDirective &) {
+    GetContext().withinConstruct = true;
+  }
+
+  void Post(const parser::AccDefaultClause &);
+
+  bool Pre(const parser::AccClause::Copy &x) {
+    ResolveAccObjectList(x.v, Symbol::Flag::AccCopyIn);
+    ResolveAccObjectList(x.v, Symbol::Flag::AccCopyOut);
+    return false;
+  }
+
+  bool Pre(const parser::AccClause::Create &x) {
+    const auto &objectList{std::get<parser::AccObjectList>(x.v.t)};
+    ResolveAccObjectList(objectList, Symbol::Flag::AccCreate);
+    return false;
+  }
+
+  bool Pre(const parser::AccClause::Copyin &x) {
+    const auto &objectList{std::get<parser::AccObjectList>(x.v.t)};
+    ResolveAccObjectList(objectList, Symbol::Flag::AccCopyIn);
+    return false;
+  }
+
+  bool Pre(const parser::AccClause::Copyout &x) {
+    const auto &objectList{std::get<parser::AccObjectList>(x.v.t)};
+    ResolveAccObjectList(objectList, Symbol::Flag::AccCopyOut);
+    return false;
+  }
+
+  bool Pre(const parser::AccClause::Present &x) {
+    ResolveAccObjectList(x.v, Symbol::Flag::AccPresent);
+    return false;
+  }
+  bool Pre(const parser::AccClause::Private &x) {
+    ResolveAccObjectList(x.v, Symbol::Flag::AccPrivate);
+    return false;
+  }
+  bool Pre(const parser::AccClause::FirstPrivate &x) {
+    ResolveAccObjectList(x.v, Symbol::Flag::AccFirstPrivate);
+    return false;
+  }
+
+  void Post(const parser::Name &);
+
+private:
+  std::int64_t GetAssociatedLoopLevelFromClauses(const parser::AccClauseList &);
+
+  static constexpr Symbol::Flags dataSharingAttributeFlags{
+      Symbol::Flag::AccShared, Symbol::Flag::AccPrivate,
+      Symbol::Flag::AccPresent, Symbol::Flag::AccFirstPrivate,
+      Symbol::Flag::AccReduction};
+
+  static constexpr Symbol::Flags dataMappingAttributeFlags{
+      Symbol::Flag::AccCreate, Symbol::Flag::AccCopyIn,
+      Symbol::Flag::AccCopyOut, Symbol::Flag::AccDelete};
+
+  static constexpr Symbol::Flags accFlagsRequireNewSymbol{
+      Symbol::Flag::AccPrivate, Symbol::Flag::AccFirstPrivate,
+      Symbol::Flag::AccReduction};
+
+  static constexpr Symbol::Flags accFlagsRequireMark{};
+
+  void PrivatizeAssociatedLoopIndex(const parser::OpenACCLoopConstruct &);
+  void ResolveAccObjectList(const parser::AccObjectList &, Symbol::Flag);
+  void ResolveAccObject(const parser::AccObject &, Symbol::Flag);
+  Symbol *ResolveAcc(const parser::Name &, Symbol::Flag, Scope &);
+  Symbol *ResolveAcc(Symbol &, Symbol::Flag, Scope &);
+  Symbol *ResolveAccCommonBlockName(const parser::Name *);
+  Symbol *DeclareOrMarkOtherAccessEntity(const parser::Name &, Symbol::Flag);
+  Symbol *DeclareOrMarkOtherAccessEntity(Symbol &, Symbol::Flag);
+  void CheckMultipleAppearances(
+      const parser::Name &, const Symbol &, Symbol::Flag);
+};
+
+// Data-sharing and Data-mapping attributes for data-refs in OpenMP construct
+class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
+public:
+  explicit OmpAttributeVisitor(SemanticsContext &context)
+      : DirectiveAttributeVisitor(context) {}
+
+  template <typename A> void Walk(const A &x) { parser::Walk(x, *this); }
+  template <typename A> bool Pre(const A &) { return true; }
+  template <typename A> void Post(const A &) {}
+
+  bool Pre(const parser::SpecificationPart &x) {
+    Walk(std::get<std::list<parser::OpenMPDeclarativeConstruct>>(x.t));
+    return false;
+  }
+
+  bool Pre(const parser::OpenMPBlockConstruct &);
+  void Post(const parser::OpenMPBlockConstruct &) { PopContext(); }
+  void Post(const parser::OmpBeginBlockDirective &) {
+    GetContext().withinConstruct = true;
+  }
+
+  bool Pre(const parser::OpenMPLoopConstruct &);
+  void Post(const parser::OpenMPLoopConstruct &) { PopContext(); }
+  void Post(const parser::OmpBeginLoopDirective &) {
+    GetContext().withinConstruct = true;
+  }
+  bool Pre(const parser::DoConstruct &);
+
+  bool Pre(const parser::OpenMPSectionsConstruct &);
+  void Post(const parser::OpenMPSectionsConstruct &) { PopContext(); }
+
+  bool Pre(const parser::OpenMPThreadprivate &);
+  void Post(const parser::OpenMPThreadprivate &) { PopContext(); }
+
+  // 2.15.3 Data-Sharing Attribute Clauses
+  void Post(const parser::OmpDefaultClause &);
+  bool Pre(const parser::OmpClause::Shared &x) {
+    ResolveOmpObjectList(x.v, Symbol::Flag::OmpShared);
+    return false;
+  }
+  bool Pre(const parser::OmpClause::Private &x) {
+    ResolveOmpObjectList(x.v, Symbol::Flag::OmpPrivate);
+    return false;
+  }
+  bool Pre(const parser::OmpClause::Firstprivate &x) {
+    ResolveOmpObjectList(x.v, Symbol::Flag::OmpFirstPrivate);
+    return false;
+  }
+  bool Pre(const parser::OmpClause::Lastprivate &x) {
+    ResolveOmpObjectList(x.v, Symbol::Flag::OmpLastPrivate);
+    return false;
+  }
+
+  void Post(const parser::Name &);
+
+private:
+  std::int64_t GetAssociatedLoopLevelFromClauses(const parser::OmpClauseList &);
+
+  static constexpr Symbol::Flags dataSharingAttributeFlags{
+      Symbol::Flag::OmpShared, Symbol::Flag::OmpPrivate,
+      Symbol::Flag::OmpFirstPrivate, Symbol::Flag::OmpLastPrivate,
+      Symbol::Flag::OmpReduction, Symbol::Flag::OmpLinear};
+
+  static constexpr Symbol::Flags ompFlagsRequireNewSymbol{
+      Symbol::Flag::OmpPrivate, Symbol::Flag::OmpLinear,
+      Symbol::Flag::OmpFirstPrivate, Symbol::Flag::OmpLastPrivate,
+      Symbol::Flag::OmpReduction};
+
+  static constexpr Symbol::Flags ompFlagsRequireMark{
+      Symbol::Flag::OmpThreadprivate};
+
+  // Predetermined DSA rules
+  void PrivatizeAssociatedLoopIndex(const parser::OpenMPLoopConstruct &);
+  void ResolveSeqLoopIndexInParallelOrTaskConstruct(const parser::Name &);
+
+  void ResolveOmpObjectList(const parser::OmpObjectList &, Symbol::Flag);
+  void ResolveOmpObject(const parser::OmpObject &, Symbol::Flag);
+  Symbol *ResolveOmp(const parser::Name &, Symbol::Flag, Scope &);
+  Symbol *ResolveOmp(Symbol &, Symbol::Flag, Scope &);
+  Symbol *ResolveOmpCommonBlockName(const parser::Name *);
+  Symbol *DeclareOrMarkOtherAccessEntity(const parser::Name &, Symbol::Flag);
+  Symbol *DeclareOrMarkOtherAccessEntity(Symbol &, Symbol::Flag);
+  void CheckMultipleAppearances(
+      const parser::Name &, const Symbol &, Symbol::Flag);
+};
+
+template <typename T>
+bool DirectiveAttributeVisitor<T>::HasDataSharingAttributeObject(
+    const Symbol &object) {
+  auto it{dataSharingAttributeObjects_.find(object)};
+  return it != dataSharingAttributeObjects_.end();
+}
+
+template <typename T>
+const parser::Name &DirectiveAttributeVisitor<T>::GetLoopIndex(
+    const parser::DoConstruct &x) {
+  using Bounds = parser::LoopControl::Bounds;
+  return std::get<Bounds>(x.GetLoopControl()->u).name.thing;
+}
+
+template <typename T>
+const parser::DoConstruct *DirectiveAttributeVisitor<T>::GetDoConstructIf(
+    const parser::ExecutionPartConstruct &x) {
+  return parser::Unwrap<parser::DoConstruct>(x);
+}
+
+template <typename T>
+Symbol *DirectiveAttributeVisitor<T>::DeclarePrivateAccessEntity(
+    const parser::Name &name, Symbol::Flag flag, Scope &scope) {
+  if (!name.symbol) {
+    return nullptr; // not resolved by Name Resolution step, do nothing
+  }
+  name.symbol = DeclarePrivateAccessEntity(*name.symbol, flag, scope);
+  return name.symbol;
+}
+
+template <typename T>
+Symbol *DirectiveAttributeVisitor<T>::DeclarePrivateAccessEntity(
+    Symbol &object, Symbol::Flag flag, Scope &scope) {
+  if (object.owner() != currScope()) {
+    auto &symbol{MakeAssocSymbol(object.name(), object, scope)};
+    symbol.set(flag);
+    return &symbol;
+  } else {
+    object.set(flag);
+    return &object;
+  }
+}
+
+bool AccAttributeVisitor::Pre(const parser::OpenACCBlockConstruct &x) {
+  const auto &beginBlockDir{std::get<parser::AccBeginBlockDirective>(x.t)};
+  const auto &blockDir{std::get<parser::AccBlockDirective>(beginBlockDir.t)};
+  switch (blockDir.v) {
+  case llvm::acc::Directive::ACCD_data:
+  case llvm::acc::Directive::ACCD_host_data:
+  case llvm::acc::Directive::ACCD_kernels:
+  case llvm::acc::Directive::ACCD_parallel:
+  case llvm::acc::Directive::ACCD_serial:
+    PushContext(blockDir.source, blockDir.v);
+    break;
+  default:
+    break;
+  }
+  ClearDataSharingAttributeObjects();
+  return true;
+}
+
+bool AccAttributeVisitor::Pre(const parser::OpenACCLoopConstruct &x) {
+  const auto &beginDir{std::get<parser::AccBeginLoopDirective>(x.t)};
+  const auto &loopDir{std::get<parser::AccLoopDirective>(beginDir.t)};
+  const auto &clauseList{std::get<parser::AccClauseList>(beginDir.t)};
+  if (loopDir.v == llvm::acc::Directive::ACCD_loop) {
+    PushContext(loopDir.source, loopDir.v);
+  }
+  ClearDataSharingAttributeObjects();
+  SetContextAssociatedLoopLevel(GetAssociatedLoopLevelFromClauses(clauseList));
+  PrivatizeAssociatedLoopIndex(x);
+  return true;
+}
+
+bool AccAttributeVisitor::Pre(const parser::OpenACCStandaloneConstruct &x) {
+  const auto &standaloneDir{std::get<parser::AccStandaloneDirective>(x.t)};
+  switch (standaloneDir.v) {
+  case llvm::acc::Directive::ACCD_cache:
+  case llvm::acc::Directive::ACCD_enter_data:
+  case llvm::acc::Directive::ACCD_exit_data:
+  case llvm::acc::Directive::ACCD_init:
+  case llvm::acc::Directive::ACCD_set:
+  case llvm::acc::Directive::ACCD_shutdown:
+  case llvm::acc::Directive::ACCD_update:
+    PushContext(standaloneDir.source, standaloneDir.v);
+    break;
+  default:
+    break;
+  }
+  ClearDataSharingAttributeObjects();
+  return true;
+}
+
+bool AccAttributeVisitor::Pre(const parser::OpenACCCombinedConstruct &x) {
+  const auto &beginBlockDir{std::get<parser::AccBeginCombinedDirective>(x.t)};
+  const auto &combinedDir{
+      std::get<parser::AccCombinedDirective>(beginBlockDir.t)};
+  switch (combinedDir.v) {
+  case llvm::acc::Directive::ACCD_kernels_loop:
+  case llvm::acc::Directive::ACCD_parallel_loop:
+  case llvm::acc::Directive::ACCD_serial_loop:
+    PushContext(combinedDir.source, combinedDir.v);
+    break;
+  default:
+    break;
+  }
+  ClearDataSharingAttributeObjects();
+  return true;
+}
+
+std::int64_t AccAttributeVisitor::GetAssociatedLoopLevelFromClauses(
+    const parser::AccClauseList &x) {
+  std::int64_t collapseLevel{0};
+  for (const auto &clause : x.v) {
+    if (const auto *collapseClause{
+            std::get_if<parser::AccClause::Collapse>(&clause.u)}) {
+      if (const auto v{EvaluateInt64(context_, collapseClause->v)}) {
+        collapseLevel = *v;
+      }
+    }
+  }
+
+  if (collapseLevel) {
+    return collapseLevel;
+  }
+  return 1; // default is outermost loop
+}
+
+void AccAttributeVisitor::PrivatizeAssociatedLoopIndex(
+    const parser::OpenACCLoopConstruct &x) {
+  std::int64_t level{GetContext().associatedLoopLevel};
+  if (level <= 0) { // collpase value was negative or 0
+    return;
+  }
+  Symbol::Flag ivDSA{Symbol::Flag::AccPrivate};
+
+  const auto &outer{std::get<std::optional<parser::DoConstruct>>(x.t)};
+  for (const parser::DoConstruct *loop{&*outer}; loop && level > 0; --level) {
+    // go through all the nested do-loops and resolve index variables
+    const parser::Name &iv{GetLoopIndex(*loop)};
+    if (auto *symbol{ResolveAcc(iv, ivDSA, currScope())}) {
+      symbol->set(Symbol::Flag::AccPreDetermined);
+      iv.symbol = symbol; // adjust the symbol within region
+      AddToContextObjectWithDSA(*symbol, ivDSA);
+    }
+
+    const auto &block{std::get<parser::Block>(loop->t)};
+    const auto it{block.begin()};
+    loop = it != block.end() ? GetDoConstructIf(*it) : nullptr;
+  }
+  CHECK(level == 0);
+}
+
+void AccAttributeVisitor::Post(const parser::AccDefaultClause &x) {
+  if (!dirContext_.empty()) {
+    switch (x.v) {
+    case parser::AccDefaultClause::Arg::Present:
+      SetContextDefaultDSA(Symbol::Flag::AccPresent);
+      break;
+    case parser::AccDefaultClause::Arg::None:
+      SetContextDefaultDSA(Symbol::Flag::AccNone);
+      break;
+    }
+  }
+}
+
+// For OpenACC constructs, check all the data-refs within the constructs
+// and adjust the symbol for each Name if necessary
+void AccAttributeVisitor::Post(const parser::Name &name) {
+  auto *symbol{name.symbol};
+  if (symbol && !dirContext_.empty() && GetContext().withinConstruct) {
+    if (!symbol->owner().IsDerivedType() && !symbol->has<ProcEntityDetails>() &&
+        !IsObjectWithDSA(*symbol)) {
+      if (Symbol * found{currScope().FindSymbol(name.source)}) {
+        if (symbol != found) {
+          name.symbol = found; // adjust the symbol within region
+        } else if (GetContext().defaultDSA == Symbol::Flag::AccNone) {
+          // 2.5.14.
+          context_.Say(name.source,
+              "The DEFAULT(NONE) clause requires that '%s' must be listed in "
+              "a data-mapping clause"_err_en_US,
+              symbol->name());
+        }
+      }
+    }
+  } // within OpenACC construct
+}
+
+Symbol *AccAttributeVisitor::ResolveAccCommonBlockName(
+    const parser::Name *name) {
+  if (!name) {
+    return nullptr;
+  } else if (auto *prev{
+                 GetContext().scope.parent().FindCommonBlock(name->source)}) {
+    name->symbol = prev;
+    return prev;
+  } else {
+    return nullptr;
+  }
+}
+
+void AccAttributeVisitor::ResolveAccObjectList(
+    const parser::AccObjectList &accObjectList, Symbol::Flag accFlag) {
+  for (const auto &accObject : accObjectList.v) {
+    ResolveAccObject(accObject, accFlag);
+  }
+}
+
+void AccAttributeVisitor::ResolveAccObject(
+    const parser::AccObject &accObject, Symbol::Flag accFlag) {
+  std::visit(
+      common::visitors{
+          [&](const parser::Designator &designator) {
+            if (const auto *name{GetDesignatorNameIfDataRef(designator)}) {
+              if (auto *symbol{ResolveAcc(*name, accFlag, currScope())}) {
+                AddToContextObjectWithDSA(*symbol, accFlag);
+                if (dataSharingAttributeFlags.test(accFlag)) {
+                  CheckMultipleAppearances(*name, *symbol, accFlag);
+                }
+              }
+            } else {
+              // Array sections to be changed to substrings as needed
+              if (AnalyzeExpr(context_, designator)) {
+                if (std::holds_alternative<parser::Substring>(designator.u)) {
+                  context_.Say(designator.source,
+                      "Substrings are not allowed on OpenACC "
+                      "directives or clauses"_err_en_US);
+                }
+              }
+              // other checks, more TBD
+            }
+          },
+          [&](const parser::Name &name) { // common block
+            if (auto *symbol{ResolveAccCommonBlockName(&name)}) {
+              CheckMultipleAppearances(
+                  name, *symbol, Symbol::Flag::AccCommonBlock);
+              for (auto &object : symbol->get<CommonBlockDetails>().objects()) {
+                if (auto *resolvedObject{
+                        ResolveAcc(*object, accFlag, currScope())}) {
+                  AddToContextObjectWithDSA(*resolvedObject, accFlag);
+                }
+              }
+            } else {
+              context_.Say(name.source,
+                  "COMMON block must be declared in the same scoping unit "
+                  "in which the OpenACC directive or clause appears"_err_en_US);
+            }
+          },
+      },
+      accObject.u);
+}
+
+Symbol *AccAttributeVisitor::ResolveAcc(
+    const parser::Name &name, Symbol::Flag accFlag, Scope &scope) {
+  if (accFlagsRequireNewSymbol.test(accFlag)) {
+    return DeclarePrivateAccessEntity(name, accFlag, scope);
+  } else {
+    return DeclareOrMarkOtherAccessEntity(name, accFlag);
+  }
+}
+
+Symbol *AccAttributeVisitor::ResolveAcc(
+    Symbol &symbol, Symbol::Flag accFlag, Scope &scope) {
+  if (accFlagsRequireNewSymbol.test(accFlag)) {
+    return DeclarePrivateAccessEntity(symbol, accFlag, scope);
+  } else {
+    return DeclareOrMarkOtherAccessEntity(symbol, accFlag);
+  }
+}
+
+Symbol *AccAttributeVisitor::DeclareOrMarkOtherAccessEntity(
+    const parser::Name &name, Symbol::Flag accFlag) {
+  Symbol *prev{currScope().FindSymbol(name.source)};
+  if (!name.symbol || !prev) {
+    return nullptr;
+  } else if (prev != name.symbol) {
+    name.symbol = prev;
+  }
+  return DeclareOrMarkOtherAccessEntity(*prev, accFlag);
+}
+
+Symbol *AccAttributeVisitor::DeclareOrMarkOtherAccessEntity(
+    Symbol &object, Symbol::Flag accFlag) {
+  if (accFlagsRequireMark.test(accFlag)) {
+    object.set(accFlag);
+  }
+  return &object;
+}
+
+static bool WithMultipleAppearancesAccException(
+    const Symbol &symbol, Symbol::Flag flag) {
+  return false; // Place holder
+}
+
+void AccAttributeVisitor::CheckMultipleAppearances(
+    const parser::Name &name, const Symbol &symbol, Symbol::Flag accFlag) {
+  const auto *target{&symbol};
+  if (accFlagsRequireNewSymbol.test(accFlag)) {
+    if (const auto *details{symbol.detailsIf<HostAssocDetails>()}) {
+      target = &details->symbol();
+    }
+  }
+  if (HasDataSharingAttributeObject(*target) &&
+      !WithMultipleAppearancesAccException(symbol, accFlag)) {
+    context_.Say(name.source,
+        "'%s' appears in more than one data-sharing clause "
+        "on the same OpenACC directive"_err_en_US,
+        name.ToString());
+  } else {
+    AddDataSharingAttributeObject(*target);
+  }
+}
+
+bool OmpAttributeVisitor::Pre(const parser::OpenMPBlockConstruct &x) {
+  const auto &beginBlockDir{std::get<parser::OmpBeginBlockDirective>(x.t)};
+  const auto &beginDir{std::get<parser::OmpBlockDirective>(beginBlockDir.t)};
+  switch (beginDir.v) {
+  case llvm::omp::Directive::OMPD_master:
+  case llvm::omp::Directive::OMPD_ordered:
+  case llvm::omp::Directive::OMPD_parallel:
+  case llvm::omp::Directive::OMPD_single:
+  case llvm::omp::Directive::OMPD_target:
+  case llvm::omp::Directive::OMPD_target_data:
+  case llvm::omp::Directive::OMPD_task:
+  case llvm::omp::Directive::OMPD_teams:
+  case llvm::omp::Directive::OMPD_workshare:
+  case llvm::omp::Directive::OMPD_parallel_workshare:
+  case llvm::omp::Directive::OMPD_target_teams:
+  case llvm::omp::Directive::OMPD_target_parallel:
+    PushContext(beginDir.source, beginDir.v);
+    break;
+  default:
+    // TODO others
+    break;
+  }
+  ClearDataSharingAttributeObjects();
+  return true;
+}
+
+bool OmpAttributeVisitor::Pre(const parser::OpenMPLoopConstruct &x) {
+  const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
+  const auto &beginDir{std::get<parser::OmpLoopDirective>(beginLoopDir.t)};
+  const auto &clauseList{std::get<parser::OmpClauseList>(beginLoopDir.t)};
+  switch (beginDir.v) {
+  case llvm::omp::Directive::OMPD_distribute:
+  case llvm::omp::Directive::OMPD_distribute_parallel_do:
+  case llvm::omp::Directive::OMPD_distribute_parallel_do_simd:
+  case llvm::omp::Directive::OMPD_distribute_simd:
+  case llvm::omp::Directive::OMPD_do:
+  case llvm::omp::Directive::OMPD_do_simd:
+  case llvm::omp::Directive::OMPD_parallel_do:
+  case llvm::omp::Directive::OMPD_parallel_do_simd:
+  case llvm::omp::Directive::OMPD_simd:
+  case llvm::omp::Directive::OMPD_target_parallel_do:
+  case llvm::omp::Directive::OMPD_target_parallel_do_simd:
+  case llvm::omp::Directive::OMPD_target_teams_distribute:
+  case llvm::omp::Directive::OMPD_target_teams_distribute_parallel_do:
+  case llvm::omp::Directive::OMPD_target_teams_distribute_parallel_do_simd:
+  case llvm::omp::Directive::OMPD_target_teams_distribute_simd:
+  case llvm::omp::Directive::OMPD_target_simd:
+  case llvm::omp::Directive::OMPD_taskloop:
+  case llvm::omp::Directive::OMPD_taskloop_simd:
+  case llvm::omp::Directive::OMPD_teams_distribute:
+  case llvm::omp::Directive::OMPD_teams_distribute_parallel_do:
+  case llvm::omp::Directive::OMPD_teams_distribute_parallel_do_simd:
+  case llvm::omp::Directive::OMPD_teams_distribute_simd:
+    PushContext(beginDir.source, beginDir.v);
+    break;
+  default:
+    break;
+  }
+  ClearDataSharingAttributeObjects();
+  SetContextAssociatedLoopLevel(GetAssociatedLoopLevelFromClauses(clauseList));
+  PrivatizeAssociatedLoopIndex(x);
+  return true;
+}
+
+void OmpAttributeVisitor::ResolveSeqLoopIndexInParallelOrTaskConstruct(
+    const parser::Name &iv) {
+  auto targetIt{dirContext_.rbegin()};
+  for (;; ++targetIt) {
+    if (targetIt == dirContext_.rend()) {
+      return;
+    }
+    if (llvm::omp::parallelSet.test(targetIt->directive) ||
+        llvm::omp::taskGeneratingSet.test(targetIt->directive)) {
+      break;
+    }
+  }
+  if (auto *symbol{ResolveOmp(iv, Symbol::Flag::OmpPrivate, targetIt->scope)}) {
+    targetIt++;
+    symbol->set(Symbol::Flag::OmpPreDetermined);
+    iv.symbol = symbol; // adjust the symbol within region
+    for (auto it{dirContext_.rbegin()}; it != targetIt; ++it) {
+      AddToContextObjectWithDSA(*symbol, Symbol::Flag::OmpPrivate, *it);
+    }
+  }
+}
+
+// 2.15.1.1 Data-sharing Attribute Rules - Predetermined
+//   - A loop iteration variable for a sequential loop in a parallel
+//     or task generating construct is private in the innermost such
+//     construct that encloses the loop
+bool OmpAttributeVisitor::Pre(const parser::DoConstruct &x) {
+  if (!dirContext_.empty() && GetContext().withinConstruct) {
+    if (const auto &iv{GetLoopIndex(x)}; iv.symbol) {
+      if (!iv.symbol->test(Symbol::Flag::OmpPreDetermined)) {
+        ResolveSeqLoopIndexInParallelOrTaskConstruct(iv);
+      } else {
+        // TODO: conflict checks with explicitly determined DSA
+      }
+    }
+  }
+  return true;
+}
+
+std::int64_t OmpAttributeVisitor::GetAssociatedLoopLevelFromClauses(
+    const parser::OmpClauseList &x) {
+  std::int64_t orderedLevel{0};
+  std::int64_t collapseLevel{0};
+  for (const auto &clause : x.v) {
+    if (const auto *orderedClause{
+            std::get_if<parser::OmpClause::Ordered>(&clause.u)}) {
+      if (const auto v{EvaluateInt64(context_, orderedClause->v)}) {
+        orderedLevel = *v;
+      }
+    }
+    if (const auto *collapseClause{
+            std::get_if<parser::OmpClause::Collapse>(&clause.u)}) {
+      if (const auto v{EvaluateInt64(context_, collapseClause->v)}) {
+        collapseLevel = *v;
+      }
+    }
+  }
+
+  if (orderedLevel && (!collapseLevel || orderedLevel >= collapseLevel)) {
+    return orderedLevel;
+  } else if (!orderedLevel && collapseLevel) {
+    return collapseLevel;
+  } // orderedLevel < collapseLevel is an error handled in structural checks
+  return 1; // default is outermost loop
+}
+
+// 2.15.1.1 Data-sharing Attribute Rules - Predetermined
+//   - The loop iteration variable(s) in the associated do-loop(s) of a do,
+//     parallel do, taskloop, or distribute construct is (are) private.
+//   - The loop iteration variable in the associated do-loop of a simd construct
+//     with just one associated do-loop is linear with a linear-step that is the
+//     increment of the associated do-loop.
+//   - The loop iteration variables in the associated do-loops of a simd
+//     construct with multiple associated do-loops are lastprivate.
+//
+// TODO: revisit after semantics checks are completed for do-loop association of
+//       collapse and ordered
+void OmpAttributeVisitor::PrivatizeAssociatedLoopIndex(
+    const parser::OpenMPLoopConstruct &x) {
+  std::int64_t level{GetContext().associatedLoopLevel};
+  if (level <= 0) {
+    return;
+  }
+  Symbol::Flag ivDSA;
+  if (!llvm::omp::simdSet.test(GetContext().directive)) {
+    ivDSA = Symbol::Flag::OmpPrivate;
+  } else if (level == 1) {
+    ivDSA = Symbol::Flag::OmpLinear;
+  } else {
+    ivDSA = Symbol::Flag::OmpLastPrivate;
+  }
+
+  const auto &outer{std::get<std::optional<parser::DoConstruct>>(x.t)};
+  for (const parser::DoConstruct *loop{&*outer}; loop && level > 0; --level) {
+    // go through all the nested do-loops and resolve index variables
+    const parser::Name &iv{GetLoopIndex(*loop)};
+    if (auto *symbol{ResolveOmp(iv, ivDSA, currScope())}) {
+      symbol->set(Symbol::Flag::OmpPreDetermined);
+      iv.symbol = symbol; // adjust the symbol within region
+      AddToContextObjectWithDSA(*symbol, ivDSA);
+    }
+
+    const auto &block{std::get<parser::Block>(loop->t)};
+    const auto it{block.begin()};
+    loop = it != block.end() ? GetDoConstructIf(*it) : nullptr;
+  }
+  CHECK(level == 0);
+}
+
+bool OmpAttributeVisitor::Pre(const parser::OpenMPSectionsConstruct &x) {
+  const auto &beginSectionsDir{
+      std::get<parser::OmpBeginSectionsDirective>(x.t)};
+  const auto &beginDir{
+      std::get<parser::OmpSectionsDirective>(beginSectionsDir.t)};
+  switch (beginDir.v) {
+  case llvm::omp::Directive::OMPD_parallel_sections:
+  case llvm::omp::Directive::OMPD_sections:
+    PushContext(beginDir.source, beginDir.v);
+    break;
+  default:
+    break;
+  }
+  ClearDataSharingAttributeObjects();
+  return true;
+}
+
+bool OmpAttributeVisitor::Pre(const parser::OpenMPThreadprivate &x) {
+  PushContext(x.source, llvm::omp::Directive::OMPD_threadprivate);
+  const auto &list{std::get<parser::OmpObjectList>(x.t)};
+  ResolveOmpObjectList(list, Symbol::Flag::OmpThreadprivate);
+  return false;
+}
+
+void OmpAttributeVisitor::Post(const parser::OmpDefaultClause &x) {
+  if (!dirContext_.empty()) {
+    switch (x.v) {
+    case parser::OmpDefaultClause::Type::Private:
+      SetContextDefaultDSA(Symbol::Flag::OmpPrivate);
+      break;
+    case parser::OmpDefaultClause::Type::Firstprivate:
+      SetContextDefaultDSA(Symbol::Flag::OmpFirstPrivate);
+      break;
+    case parser::OmpDefaultClause::Type::Shared:
+      SetContextDefaultDSA(Symbol::Flag::OmpShared);
+      break;
+    case parser::OmpDefaultClause::Type::None:
+      SetContextDefaultDSA(Symbol::Flag::OmpNone);
+      break;
+    }
+  }
+}
+
+// For OpenMP constructs, check all the data-refs within the constructs
+// and adjust the symbol for each Name if necessary
+void OmpAttributeVisitor::Post(const parser::Name &name) {
+  auto *symbol{name.symbol};
+  if (symbol && !dirContext_.empty() && GetContext().withinConstruct) {
+    if (!symbol->owner().IsDerivedType() && !symbol->has<ProcEntityDetails>() &&
+        !IsObjectWithDSA(*symbol)) {
+      // TODO: create a separate function to go through the rules for
+      //       predetermined, explicitly determined, and implicitly
+      //       determined data-sharing attributes (2.15.1.1).
+      if (Symbol * found{currScope().FindSymbol(name.source)}) {
+        if (symbol != found) {
+          name.symbol = found; // adjust the symbol within region
+        } else if (GetContext().defaultDSA == Symbol::Flag::OmpNone) {
+          context_.Say(name.source,
+              "The DEFAULT(NONE) clause requires that '%s' must be listed in "
+              "a data-sharing attribute clause"_err_en_US,
+              symbol->name());
+        }
+      }
+    }
+  } // within OpenMP construct
+}
+
+Symbol *OmpAttributeVisitor::ResolveOmpCommonBlockName(
+    const parser::Name *name) {
+  if (auto *prev{name
+              ? GetContext().scope.parent().FindCommonBlock(name->source)
+              : nullptr}) {
+    name->symbol = prev;
+    return prev;
+  } else {
+    return nullptr;
+  }
+}
+
+void OmpAttributeVisitor::ResolveOmpObjectList(
+    const parser::OmpObjectList &ompObjectList, Symbol::Flag ompFlag) {
+  for (const auto &ompObject : ompObjectList.v) {
+    ResolveOmpObject(ompObject, ompFlag);
+  }
+}
+
+void OmpAttributeVisitor::ResolveOmpObject(
+    const parser::OmpObject &ompObject, Symbol::Flag ompFlag) {
+  std::visit(
+      common::visitors{
+          [&](const parser::Designator &designator) {
+            if (const auto *name{GetDesignatorNameIfDataRef(designator)}) {
+              if (auto *symbol{ResolveOmp(*name, ompFlag, currScope())}) {
+                AddToContextObjectWithDSA(*symbol, ompFlag);
+                if (dataSharingAttributeFlags.test(ompFlag)) {
+                  CheckMultipleAppearances(*name, *symbol, ompFlag);
+                }
+              }
+            } else {
+              // Array sections to be changed to substrings as needed
+              if (AnalyzeExpr(context_, designator)) {
+                if (std::holds_alternative<parser::Substring>(designator.u)) {
+                  context_.Say(designator.source,
+                      "Substrings are not allowed on OpenMP "
+                      "directives or clauses"_err_en_US);
+                }
+              }
+              // other checks, more TBD
+            }
+          },
+          [&](const parser::Name &name) { // common block
+            if (auto *symbol{ResolveOmpCommonBlockName(&name)}) {
+              CheckMultipleAppearances(
+                  name, *symbol, Symbol::Flag::OmpCommonBlock);
+              // 2.15.3 When a named common block appears in a list, it has the
+              // same meaning as if every explicit member of the common block
+              // appeared in the list
+              for (auto &object : symbol->get<CommonBlockDetails>().objects()) {
+                if (auto *resolvedObject{
+                        ResolveOmp(*object, ompFlag, currScope())}) {
+                  AddToContextObjectWithDSA(*resolvedObject, ompFlag);
+                }
+              }
+            } else {
+              context_.Say(name.source, // 2.15.3
+                  "COMMON block must be declared in the same scoping unit "
+                  "in which the OpenMP directive or clause appears"_err_en_US);
+            }
+          },
+      },
+      ompObject.u);
+}
+
+Symbol *OmpAttributeVisitor::ResolveOmp(
+    const parser::Name &name, Symbol::Flag ompFlag, Scope &scope) {
+  if (ompFlagsRequireNewSymbol.test(ompFlag)) {
+    return DeclarePrivateAccessEntity(name, ompFlag, scope);
+  } else {
+    return DeclareOrMarkOtherAccessEntity(name, ompFlag);
+  }
+}
+
+Symbol *OmpAttributeVisitor::ResolveOmp(
+    Symbol &symbol, Symbol::Flag ompFlag, Scope &scope) {
+  if (ompFlagsRequireNewSymbol.test(ompFlag)) {
+    return DeclarePrivateAccessEntity(symbol, ompFlag, scope);
+  } else {
+    return DeclareOrMarkOtherAccessEntity(symbol, ompFlag);
+  }
+}
+
+Symbol *OmpAttributeVisitor::DeclareOrMarkOtherAccessEntity(
+    const parser::Name &name, Symbol::Flag ompFlag) {
+  Symbol *prev{currScope().FindSymbol(name.source)};
+  if (!name.symbol || !prev) {
+    return nullptr;
+  } else if (prev != name.symbol) {
+    name.symbol = prev;
+  }
+  return DeclareOrMarkOtherAccessEntity(*prev, ompFlag);
+}
+
+Symbol *OmpAttributeVisitor::DeclareOrMarkOtherAccessEntity(
+    Symbol &object, Symbol::Flag ompFlag) {
+  if (ompFlagsRequireMark.test(ompFlag)) {
+    object.set(ompFlag);
+  }
+  return &object;
+}
+
+static bool WithMultipleAppearancesOmpException(
+    const Symbol &symbol, Symbol::Flag flag) {
+  return (flag == Symbol::Flag::OmpFirstPrivate &&
+             symbol.test(Symbol::Flag::OmpLastPrivate)) ||
+      (flag == Symbol::Flag::OmpLastPrivate &&
+          symbol.test(Symbol::Flag::OmpFirstPrivate));
+}
+
+void OmpAttributeVisitor::CheckMultipleAppearances(
+    const parser::Name &name, const Symbol &symbol, Symbol::Flag ompFlag) {
+  const auto *target{&symbol};
+  if (ompFlagsRequireNewSymbol.test(ompFlag)) {
+    if (const auto *details{symbol.detailsIf<HostAssocDetails>()}) {
+      target = &details->symbol();
+    }
+  }
+  if (HasDataSharingAttributeObject(*target) &&
+      !WithMultipleAppearancesOmpException(symbol, ompFlag)) {
+    context_.Say(name.source,
+        "'%s' appears in more than one data-sharing clause "
+        "on the same OpenMP directive"_err_en_US,
+        name.ToString());
+  } else {
+    AddDataSharingAttributeObject(*target);
+  }
+}
+
+void ResolveAccParts(
+    SemanticsContext &context, const parser::ProgramUnit &node) {
+  if (context.IsEnabled(common::LanguageFeature::OpenACC)) {
+    AccAttributeVisitor{context}.Walk(node);
+  }
+}
+
+void ResolveOmpParts(
+    SemanticsContext &context, const parser::ProgramUnit &node) {
+  if (context.IsEnabled(common::LanguageFeature::OpenMP)) {
+    OmpAttributeVisitor{context}.Walk(node);
+    if (!context.AnyFatalError()) {
+      // The data-sharing attribute of the loop iteration variable for a
+      // sequential loop (2.15.1.1) can only be determined when visiting
+      // the corresponding DoConstruct, a second walk is to adjust the
+      // symbols for all the data-refs of that loop iteration variable
+      // prior to the DoConstruct.
+      OmpAttributeVisitor{context}.Walk(node);
+    }
+  }
+}
+
+} // namespace Fortran::semantics
diff --git a/flang/lib/Semantics/resolve-directives.h b/flang/lib/Semantics/resolve-directives.h
new file mode 100644
index 0000000000000..6ba7a06252942
--- /dev/null
+++ b/flang/lib/Semantics/resolve-directives.h
@@ -0,0 +1,26 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef FORTRAN_SEMANTICS_RESOLVE_DIRECTIVES_H_
+#define FORTRAN_SEMANTICS_RESOLVE_DIRECTIVES_H_
+
+namespace Fortran::parser {
+struct Name;
+struct ProgramUnit;
+} // namespace Fortran::parser
+
+namespace Fortran::semantics {
+
+class SemanticsContext;
+
+// Name resolution for OpenACC and OpenMP directives
+void ResolveAccParts(SemanticsContext &, const parser::ProgramUnit &);
+void ResolveOmpParts(SemanticsContext &, const parser::ProgramUnit &);
+
+} // namespace Fortran::semantics
+#endif
diff --git a/flang/lib/Semantics/resolve-names-utils.h b/flang/lib/Semantics/resolve-names-utils.h
index e04f58f1cc576..08db70345f152 100644
--- a/flang/lib/Semantics/resolve-names-utils.h
+++ b/flang/lib/Semantics/resolve-names-utils.h
@@ -11,8 +11,12 @@
 
 // Utility functions and class for use in resolve-names.cpp.
 
+#include "flang/Evaluate/fold.h"
 #include "flang/Parser/message.h"
+#include "flang/Parser/tools.h"
+#include "flang/Semantics/expression.h"
 #include "flang/Semantics/scope.h"
+#include "flang/Semantics/semantics.h"
 #include "flang/Semantics/symbol.h"
 #include "flang/Semantics/type.h"
 #include <forward_list>
@@ -48,6 +52,23 @@ bool IsDefinedOperator(const SourceName &);
 bool IsIntrinsicOperator(const SemanticsContext &, const SourceName &);
 bool IsLogicalConstant(const SemanticsContext &, const SourceName &);
 
+template <typename T>
+MaybeIntExpr EvaluateIntExpr(SemanticsContext &context, const T &expr) {
+  if (MaybeExpr maybeExpr{
+          Fold(context.foldingContext(), AnalyzeExpr(context, expr))}) {
+    if (auto *intExpr{evaluate::UnwrapExpr<SomeIntExpr>(*maybeExpr)}) {
+      return std::move(*intExpr);
+    }
+  }
+  return std::nullopt;
+}
+
+template <typename T>
+std::optional<std::int64_t> EvaluateInt64(
+    SemanticsContext &context, const T &expr) {
+  return evaluate::ToInt64(EvaluateIntExpr(context, expr));
+}
+
 // Analyze a generic-spec and generate a symbol name and GenericKind for it.
 class GenericSpecInfo {
 public:
diff --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp
index 8d3e97d20521d..c1aef4b9b34c3 100644
--- a/flang/lib/Semantics/resolve-names.cpp
+++ b/flang/lib/Semantics/resolve-names.cpp
@@ -7,11 +7,10 @@
 
 #include "resolve-names.h"
 #include "assignment.h"
-#include "check-acc-structure.h"
-#include "check-omp-structure.h"
 #include "mod-file.h"
 #include "pointer-assignment.h"
 #include "program-tree.h"
+#include "resolve-directives.h"
 #include "resolve-names-utils.h"
 #include "rewrite-parse-tree.h"
 #include "flang/Common/Fortran.h"
@@ -198,12 +197,7 @@ class BaseVisitor {
   }
 
   template <typename T> MaybeIntExpr EvaluateIntExpr(const T &expr) {
-    if (MaybeExpr maybeExpr{EvaluateExpr(expr)}) {
-      if (auto *intExpr{evaluate::UnwrapExpr<SomeIntExpr>(*maybeExpr)}) {
-        return std::move(*intExpr);
-      }
-    }
-    return std::nullopt;
+    return semantics::EvaluateIntExpr(*context_, expr);
   }
 
   template <typename T>
@@ -846,7 +840,6 @@ class DeclarationVisitor : public ArraySpecVisitor,
   const parser::Name *ResolveStructureComponent(
       const parser::StructureComponent &);
   const parser::Name *ResolveDataRef(const parser::DataRef &);
-  const parser::Name *ResolveVariable(const parser::Variable &);
   const parser::Name *ResolveName(const parser::Name &);
   bool PassesSharedLocalityChecks(const parser::Name &name, Symbol &symbol);
   Symbol *NoteInterfaceName(const parser::Name &);
@@ -1082,133 +1075,6 @@ class ConstructVisitor : public virtual DeclarationVisitor {
   void PopAssociation();
 };
 
-template <typename T> class DirectiveAttributeVisitor {
-public:
-  explicit DirectiveAttributeVisitor(
-      SemanticsContext &context, ResolveNamesVisitor &resolver)
-      : context_{context}, resolver_{resolver} {}
-
-protected:
-  struct DirContext {
-    DirContext(const parser::CharBlock &source, T d, Scope &s)
-        : directiveSource{source}, directive{d}, scope{s} {}
-    parser::CharBlock directiveSource;
-    T directive;
-    Scope &scope;
-    Symbol::Flag defaultDSA{Symbol::Flag::AccShared}; // TODOACC
-    std::map<const Symbol *, Symbol::Flag> objectWithDSA;
-    bool withinConstruct{false};
-    int64_t associatedLoopLevel{0};
-  };
-
-  DirContext &GetContext() {
-    CHECK(!dirContext_.empty());
-    return dirContext_.back();
-  }
-  void PushContext(const parser::CharBlock &source, T dir) {
-    dirContext_.emplace_back(source, dir, context_.FindScope(source));
-  }
-  void PopContext() { dirContext_.pop_back(); }
-  void SetContextDirectiveSource(parser::CharBlock &dir) {
-    GetContext().directiveSource = dir;
-  }
-  void SetContextDirectiveEnum(T dir) { GetContext().directive = dir; }
-  Scope &currScope() { return GetContext().scope; }
-  void SetContextDefaultDSA(Symbol::Flag flag) {
-    GetContext().defaultDSA = flag;
-  }
-  void AddToContextObjectWithDSA(
-      const Symbol &symbol, Symbol::Flag flag, DirContext &context) {
-    context.objectWithDSA.emplace(&symbol, flag);
-  }
-  void AddToContextObjectWithDSA(const Symbol &symbol, Symbol::Flag flag) {
-    AddToContextObjectWithDSA(symbol, flag, GetContext());
-  }
-  bool IsObjectWithDSA(const Symbol &symbol) {
-    auto it{GetContext().objectWithDSA.find(&symbol)};
-    return it != GetContext().objectWithDSA.end();
-  }
-  void SetContextAssociatedLoopLevel(int64_t level) {
-    GetContext().associatedLoopLevel = level;
-  }
-  Symbol &MakeAssocSymbol(const SourceName &name, Symbol &prev, Scope &scope) {
-    const auto pair{scope.try_emplace(name, Attrs{}, HostAssocDetails{prev})};
-    return *pair.first->second;
-  }
-  Symbol &MakeAssocSymbol(const SourceName &name, Symbol &prev) {
-    return MakeAssocSymbol(name, prev, currScope());
-  }
-  static const parser::Name *GetDesignatorNameIfDataRef(
-      const parser::Designator &designator) {
-    const auto *dataRef{std::get_if<parser::DataRef>(&designator.u)};
-    return dataRef ? std::get_if<parser::Name>(&dataRef->u) : nullptr;
-  }
-  void AddDataSharingAttributeObject(SymbolRef object) {
-    dataSharingAttributeObjects_.insert(object);
-  }
-  void ClearDataSharingAttributeObjects() {
-    dataSharingAttributeObjects_.clear();
-  }
-  bool HasDataSharingAttributeObject(const Symbol &);
-  const parser::Name &GetLoopIndex(const parser::DoConstruct &);
-  const parser::DoConstruct *GetDoConstructIf(
-      const parser::ExecutionPartConstruct &);
-  Symbol *DeclarePrivateAccessEntity(
-      const parser::Name &, Symbol::Flag, Scope &);
-  Symbol *DeclarePrivateAccessEntity(Symbol &, Symbol::Flag, Scope &);
-  Symbol *DeclareOrMarkOtherAccessEntity(const parser::Name &, Symbol::Flag);
-
-  SymbolSet dataSharingAttributeObjects_; // on one directive
-  SemanticsContext &context_;
-  ResolveNamesVisitor &resolver_;
-  std::vector<DirContext> dirContext_; // used as a stack
-};
-
-template <typename T>
-bool DirectiveAttributeVisitor<T>::HasDataSharingAttributeObject(
-    const Symbol &object) {
-  auto it{dataSharingAttributeObjects_.find(object)};
-  return it != dataSharingAttributeObjects_.end();
-}
-
-template <typename T>
-const parser::Name &DirectiveAttributeVisitor<T>::GetLoopIndex(
-    const parser::DoConstruct &x) {
-  auto &loopControl{x.GetLoopControl().value()};
-  using Bounds = parser::LoopControl::Bounds;
-  const Bounds &bounds{std::get<Bounds>(loopControl.u)};
-  return bounds.name.thing;
-}
-
-template <typename T>
-const parser::DoConstruct *DirectiveAttributeVisitor<T>::GetDoConstructIf(
-    const parser::ExecutionPartConstruct &x) {
-  return parser::Unwrap<parser::DoConstruct>(x);
-}
-
-template <typename T>
-Symbol *DirectiveAttributeVisitor<T>::DeclarePrivateAccessEntity(
-    const parser::Name &name, Symbol::Flag flag, Scope &scope) {
-  if (!name.symbol) {
-    return nullptr; // not resolved by Name Resolution step, do nothing
-  }
-  name.symbol = DeclarePrivateAccessEntity(*name.symbol, flag, scope);
-  return name.symbol;
-}
-
-template <typename T>
-Symbol *DirectiveAttributeVisitor<T>::DeclarePrivateAccessEntity(
-    Symbol &object, Symbol::Flag flag, Scope &scope) {
-  if (object.owner() != currScope()) {
-    auto &symbol{MakeAssocSymbol(object.name(), object, scope)};
-    symbol.set(flag);
-    return &symbol;
-  } else {
-    object.set(flag);
-    return &object;
-  }
-}
-
 // Create scopes for OpenACC constructs
 class AccVisitor : public virtual DeclarationVisitor {
 public:
@@ -1274,113 +1140,6 @@ void AccVisitor::Post(const parser::OpenACCBlockConstruct &x) {
   }
 }
 
-class AccAttributeVisitor : DirectiveAttributeVisitor<llvm::acc::Directive> {
-public:
-  explicit AccAttributeVisitor(
-      SemanticsContext &context, ResolveNamesVisitor &resolver)
-      : DirectiveAttributeVisitor(context, resolver) {}
-
-  template <typename A> void Walk(const A &x) { parser::Walk(x, *this); }
-  template <typename A> bool Pre(const A &) { return true; }
-  template <typename A> void Post(const A &) {}
-
-  bool Pre(const parser::SpecificationPart &x) {
-    Walk(std::get<std::list<parser::OpenACCDeclarativeConstruct>>(x.t));
-    return false;
-  }
-
-  bool Pre(const parser::OpenACCBlockConstruct &);
-  void Post(const parser::OpenACCBlockConstruct &) { PopContext(); }
-  bool Pre(const parser::OpenACCCombinedConstruct &);
-  void Post(const parser::OpenACCCombinedConstruct &) { PopContext(); }
-
-  void Post(const parser::AccBeginBlockDirective &) {
-    GetContext().withinConstruct = true;
-  }
-
-  bool Pre(const parser::OpenACCLoopConstruct &);
-  void Post(const parser::OpenACCLoopConstruct &) { PopContext(); }
-  void Post(const parser::AccLoopDirective &) {
-    GetContext().withinConstruct = true;
-  }
-
-  bool Pre(const parser::OpenACCStandaloneConstruct &);
-  void Post(const parser::OpenACCStandaloneConstruct &) { PopContext(); }
-  void Post(const parser::AccStandaloneDirective &) {
-    GetContext().withinConstruct = true;
-  }
-
-  void Post(const parser::AccDefaultClause &);
-
-  bool Pre(const parser::AccClause::Copy &x) {
-    ResolveAccObjectList(x.v, Symbol::Flag::AccCopyIn);
-    ResolveAccObjectList(x.v, Symbol::Flag::AccCopyOut);
-    return false;
-  }
-
-  bool Pre(const parser::AccClause::Create &x) {
-    const auto &objectList{std::get<parser::AccObjectList>(x.v.t)};
-    ResolveAccObjectList(objectList, Symbol::Flag::AccCreate);
-    return false;
-  }
-
-  bool Pre(const parser::AccClause::Copyin &x) {
-    const auto &objectList{std::get<parser::AccObjectList>(x.v.t)};
-    ResolveAccObjectList(objectList, Symbol::Flag::AccCopyIn);
-    return false;
-  }
-
-  bool Pre(const parser::AccClause::Copyout &x) {
-    const auto &objectList{std::get<parser::AccObjectList>(x.v.t)};
-    ResolveAccObjectList(objectList, Symbol::Flag::AccCopyOut);
-    return false;
-  }
-
-  bool Pre(const parser::AccClause::Present &x) {
-    ResolveAccObjectList(x.v, Symbol::Flag::AccPresent);
-    return false;
-  }
-  bool Pre(const parser::AccClause::Private &x) {
-    ResolveAccObjectList(x.v, Symbol::Flag::AccPrivate);
-    return false;
-  }
-  bool Pre(const parser::AccClause::FirstPrivate &x) {
-    ResolveAccObjectList(x.v, Symbol::Flag::AccFirstPrivate);
-    return false;
-  }
-
-  void Post(const parser::Name &);
-
-private:
-  int64_t GetAssociatedLoopLevelFromClauses(const parser::AccClauseList &);
-
-  static constexpr Symbol::Flags dataSharingAttributeFlags{
-      Symbol::Flag::AccShared, Symbol::Flag::AccPrivate,
-      Symbol::Flag::AccPresent, Symbol::Flag::AccFirstPrivate,
-      Symbol::Flag::AccReduction};
-
-  static constexpr Symbol::Flags dataMappingAttributeFlags{
-      Symbol::Flag::AccCreate, Symbol::Flag::AccCopyIn,
-      Symbol::Flag::AccCopyOut, Symbol::Flag::AccDelete};
-
-  static constexpr Symbol::Flags accFlagsRequireNewSymbol{
-      Symbol::Flag::AccPrivate, Symbol::Flag::AccFirstPrivate,
-      Symbol::Flag::AccReduction};
-
-  static constexpr Symbol::Flags accFlagsRequireMark{};
-
-  void PrivatizeAssociatedLoopIndex(const parser::OpenACCLoopConstruct &);
-  void ResolveAccObjectList(const parser::AccObjectList &, Symbol::Flag);
-  void ResolveAccObject(const parser::AccObject &, Symbol::Flag);
-  Symbol *ResolveAcc(const parser::Name &, Symbol::Flag, Scope &);
-  Symbol *ResolveAcc(Symbol &, Symbol::Flag, Scope &);
-  Symbol *ResolveAccCommonBlockName(const parser::Name *);
-  Symbol *DeclareOrMarkOtherAccessEntity(const parser::Name &, Symbol::Flag);
-  Symbol *DeclareOrMarkOtherAccessEntity(Symbol &, Symbol::Flag);
-  void CheckMultipleAppearances(
-      const parser::Name &, const Symbol &, Symbol::Flag);
-};
-
 // Create scopes for OpenMP constructs
 class OmpVisitor : public virtual DeclarationVisitor {
 public:
@@ -1477,94 +1236,6 @@ void OmpVisitor::Post(const parser::OpenMPBlockConstruct &x) {
   }
 }
 
-// Data-sharing and Data-mapping attributes for data-refs in OpenMP construct
-class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
-public:
-  explicit OmpAttributeVisitor(
-      SemanticsContext &context, ResolveNamesVisitor &resolver)
-      : DirectiveAttributeVisitor(context, resolver) {}
-
-  template <typename A> void Walk(const A &x) { parser::Walk(x, *this); }
-
-  template <typename A> bool Pre(const A &) { return true; }
-  template <typename A> void Post(const A &) {}
-
-  bool Pre(const parser::SpecificationPart &x) {
-    Walk(std::get<std::list<parser::OpenMPDeclarativeConstruct>>(x.t));
-    return false;
-  }
-
-  bool Pre(const parser::OpenMPBlockConstruct &);
-  void Post(const parser::OpenMPBlockConstruct &) { PopContext(); }
-  void Post(const parser::OmpBeginBlockDirective &) {
-    GetContext().withinConstruct = true;
-  }
-
-  bool Pre(const parser::OpenMPLoopConstruct &);
-  void Post(const parser::OpenMPLoopConstruct &) { PopContext(); }
-  void Post(const parser::OmpBeginLoopDirective &) {
-    GetContext().withinConstruct = true;
-  }
-  bool Pre(const parser::DoConstruct &);
-
-  bool Pre(const parser::OpenMPSectionsConstruct &);
-  void Post(const parser::OpenMPSectionsConstruct &) { PopContext(); }
-
-  bool Pre(const parser::OpenMPThreadprivate &);
-  void Post(const parser::OpenMPThreadprivate &) { PopContext(); }
-
-  // 2.15.3 Data-Sharing Attribute Clauses
-  void Post(const parser::OmpDefaultClause &);
-  bool Pre(const parser::OmpClause::Shared &x) {
-    ResolveOmpObjectList(x.v, Symbol::Flag::OmpShared);
-    return false;
-  }
-  bool Pre(const parser::OmpClause::Private &x) {
-    ResolveOmpObjectList(x.v, Symbol::Flag::OmpPrivate);
-    return false;
-  }
-  bool Pre(const parser::OmpClause::Firstprivate &x) {
-    ResolveOmpObjectList(x.v, Symbol::Flag::OmpFirstPrivate);
-    return false;
-  }
-  bool Pre(const parser::OmpClause::Lastprivate &x) {
-    ResolveOmpObjectList(x.v, Symbol::Flag::OmpLastPrivate);
-    return false;
-  }
-
-  void Post(const parser::Name &);
-
-private:
-  std::int64_t GetAssociatedLoopLevelFromClauses(const parser::OmpClauseList &);
-
-  static constexpr Symbol::Flags dataSharingAttributeFlags{
-      Symbol::Flag::OmpShared, Symbol::Flag::OmpPrivate,
-      Symbol::Flag::OmpFirstPrivate, Symbol::Flag::OmpLastPrivate,
-      Symbol::Flag::OmpReduction, Symbol::Flag::OmpLinear};
-
-  static constexpr Symbol::Flags ompFlagsRequireNewSymbol{
-      Symbol::Flag::OmpPrivate, Symbol::Flag::OmpLinear,
-      Symbol::Flag::OmpFirstPrivate, Symbol::Flag::OmpLastPrivate,
-      Symbol::Flag::OmpReduction};
-
-  static constexpr Symbol::Flags ompFlagsRequireMark{
-      Symbol::Flag::OmpThreadprivate};
-
-  // Predetermined DSA rules
-  void PrivatizeAssociatedLoopIndex(const parser::OpenMPLoopConstruct &);
-  void ResolveSeqLoopIndexInParallelOrTaskConstruct(const parser::Name &);
-
-  void ResolveOmpObjectList(const parser::OmpObjectList &, Symbol::Flag);
-  void ResolveOmpObject(const parser::OmpObject &, Symbol::Flag);
-  Symbol *ResolveOmp(const parser::Name &, Symbol::Flag, Scope &);
-  Symbol *ResolveOmp(Symbol &, Symbol::Flag, Scope &);
-  Symbol *ResolveOmpCommonBlockName(const parser::Name *);
-  Symbol *DeclareOrMarkOtherAccessEntity(const parser::Name &, Symbol::Flag);
-  Symbol *DeclareOrMarkOtherAccessEntity(Symbol &, Symbol::Flag);
-  void CheckMultipleAppearances(
-      const parser::Name &, const Symbol &, Symbol::Flag);
-};
-
 // Walk the parse tree and resolve names to symbols.
 class ResolveNamesVisitor : public virtual ScopeHandler,
                             public ModuleVisitor,
@@ -1672,8 +1343,6 @@ class ResolveNamesVisitor : public virtual ScopeHandler,
   void FinishSpecificationParts(const ProgramTree &);
   void FinishDerivedTypeInstantiation(Scope &);
   void ResolveExecutionParts(const ProgramTree &);
-  void ResolveAccParts(const parser::ProgramUnit &);
-  void ResolveOmpParts(const parser::ProgramUnit &);
 };
 
 // ImplicitRules implementation
@@ -3510,8 +3179,7 @@ bool DeclarationVisitor::Pre(const parser::Enumerator &enumerator) {
   if (auto &init{std::get<std::optional<parser::ScalarIntConstantExpr>>(
           enumerator.t)}) {
     Walk(*init); // Resolve names in expression before evaluation.
-    MaybeIntExpr expr{EvaluateIntExpr(*init)};
-    if (auto value{evaluate::ToInt64(expr)}) {
+    if (auto value{EvaluateInt64(context(), *init)}) {
       // Cast all init expressions to C_INT so that they can then be
       // safely incremented (see 7.6 Note 2).
       enumerationState_.value = static_cast<int>(*value);
@@ -5475,8 +5143,7 @@ void ConstructVisitor::Post(const parser::SelectRankCaseStmt::Rank &x) {
     SetTypeFromAssociation(*symbol);
     SetAttrsFromAssociation(*symbol);
     if (const auto *init{std::get_if<parser::ScalarIntConstantExpr>(&x.u)}) {
-      MaybeIntExpr expr{EvaluateIntExpr(*init)};
-      if (auto val{evaluate::ToInt64(expr)}) {
+      if (auto val{EvaluateInt64(context(), *init)}) {
         auto &details{symbol->get<AssocEntityDetails>()};
         details.set_rank(*val);
       }
@@ -5752,28 +5419,6 @@ const parser::Name *DeclarationVisitor::ResolveDataRef(
       x.u);
 }
 
-const parser::Name *DeclarationVisitor::ResolveVariable(
-    const parser::Variable &x) {
-  return std::visit(
-      common::visitors{
-          [&](const Indirection<parser::Designator> &y) {
-            return ResolveDesignator(y.value());
-          },
-          [&](const Indirection<parser::FunctionReference> &y) {
-            const auto &proc{
-                std::get<parser::ProcedureDesignator>(y.value().v.t)};
-            return std::visit(common::visitors{
-                                  [&](const parser::Name &z) { return &z; },
-                                  [&](const parser::ProcComponentRef &z) {
-                                    return ResolveStructureComponent(z.v.thing);
-                                  },
-                              },
-                proc.u);
-          },
-      },
-      x.u);
-}
-
 // If implicit types are allowed, ensure name is in the symbol table.
 // Otherwise, report an error if it hasn't been declared.
 const parser::Name *DeclarationVisitor::ResolveName(const parser::Name &name) {
@@ -6498,12 +6143,8 @@ bool ResolveNamesVisitor::Pre(const parser::ProgramUnit &x) {
   inExecutionPart_ = true;
   ResolveExecutionParts(root);
   inExecutionPart_ = false;
-  if (context().IsEnabled(common::LanguageFeature::OpenACC)) {
-    ResolveAccParts(x);
-  }
-  if (context().IsEnabled(common::LanguageFeature::OpenMP)) {
-    ResolveOmpParts(x);
-  }
+  ResolveAccParts(context(), x);
+  ResolveOmpParts(context(), x);
   return false;
 }
 
@@ -6696,668 +6337,6 @@ class DeferredCheckVisitor {
   bool pushedScope_{false};
 };
 
-bool AccAttributeVisitor::Pre(const parser::OpenACCBlockConstruct &x) {
-  const auto &beginBlockDir{std::get<parser::AccBeginBlockDirective>(x.t)};
-  const auto &blockDir{std::get<parser::AccBlockDirective>(beginBlockDir.t)};
-  switch (blockDir.v) {
-  case llvm::acc::Directive::ACCD_data:
-  case llvm::acc::Directive::ACCD_host_data:
-  case llvm::acc::Directive::ACCD_kernels:
-  case llvm::acc::Directive::ACCD_parallel:
-  case llvm::acc::Directive::ACCD_serial:
-    PushContext(blockDir.source, blockDir.v);
-    break;
-  default:
-    break;
-  }
-  ClearDataSharingAttributeObjects();
-  return true;
-}
-
-bool AccAttributeVisitor::Pre(const parser::OpenACCLoopConstruct &x) {
-  const auto &beginDir{std::get<parser::AccBeginLoopDirective>(x.t)};
-  const auto &loopDir{std::get<parser::AccLoopDirective>(beginDir.t)};
-  const auto &clauseList{std::get<parser::AccClauseList>(beginDir.t)};
-  if (loopDir.v == llvm::acc::Directive::ACCD_loop) {
-    PushContext(loopDir.source, loopDir.v);
-  }
-  ClearDataSharingAttributeObjects();
-  SetContextAssociatedLoopLevel(GetAssociatedLoopLevelFromClauses(clauseList));
-  PrivatizeAssociatedLoopIndex(x);
-  return true;
-}
-
-bool AccAttributeVisitor::Pre(const parser::OpenACCStandaloneConstruct &x) {
-  const auto &standaloneDir{std::get<parser::AccStandaloneDirective>(x.t)};
-  switch (standaloneDir.v) {
-  case llvm::acc::Directive::ACCD_cache:
-  case llvm::acc::Directive::ACCD_enter_data:
-  case llvm::acc::Directive::ACCD_exit_data:
-  case llvm::acc::Directive::ACCD_init:
-  case llvm::acc::Directive::ACCD_set:
-  case llvm::acc::Directive::ACCD_shutdown:
-  case llvm::acc::Directive::ACCD_update:
-    PushContext(standaloneDir.source, standaloneDir.v);
-    break;
-  default:
-    break;
-  }
-  ClearDataSharingAttributeObjects();
-  return true;
-}
-
-bool AccAttributeVisitor::Pre(const parser::OpenACCCombinedConstruct &x) {
-  const auto &beginBlockDir{std::get<parser::AccBeginCombinedDirective>(x.t)};
-  const auto &combinedDir{
-      std::get<parser::AccCombinedDirective>(beginBlockDir.t)};
-  switch (combinedDir.v) {
-  case llvm::acc::Directive::ACCD_kernels_loop:
-  case llvm::acc::Directive::ACCD_parallel_loop:
-  case llvm::acc::Directive::ACCD_serial_loop:
-    PushContext(combinedDir.source, combinedDir.v);
-    break;
-  default:
-    break;
-  }
-  ClearDataSharingAttributeObjects();
-  return true;
-}
-
-int64_t AccAttributeVisitor::GetAssociatedLoopLevelFromClauses(
-    const parser::AccClauseList &x) {
-  int64_t collapseLevel{0};
-  for (const auto &clause : x.v) {
-    if (const auto *collapseClause{
-            std::get_if<parser::AccClause::Collapse>(&clause.u)}) {
-      if (const auto v{evaluate::ToInt64(
-              resolver_.EvaluateIntExpr(collapseClause->v))}) {
-        collapseLevel = *v;
-      }
-    }
-  }
-
-  if (collapseLevel) {
-    return collapseLevel;
-  }
-  return 1; // default is outermost loop
-}
-
-void AccAttributeVisitor::PrivatizeAssociatedLoopIndex(
-    const parser::OpenACCLoopConstruct &x) {
-  int64_t level{GetContext().associatedLoopLevel};
-  if (level <= 0) { // collpase value was negative or 0
-    return;
-  }
-  Symbol::Flag ivDSA{Symbol::Flag::AccPrivate};
-
-  const auto &outer{std::get<std::optional<parser::DoConstruct>>(x.t)};
-  for (const parser::DoConstruct *loop{&*outer}; loop && level > 0; --level) {
-    // go through all the nested do-loops and resolve index variables
-    const parser::Name &iv{GetLoopIndex(*loop)};
-    if (auto *symbol{ResolveAcc(iv, ivDSA, currScope())}) {
-      symbol->set(Symbol::Flag::AccPreDetermined);
-      iv.symbol = symbol; // adjust the symbol within region
-      AddToContextObjectWithDSA(*symbol, ivDSA);
-    }
-
-    const auto &block{std::get<parser::Block>(loop->t)};
-    const auto it{block.begin()};
-    loop = it != block.end() ? GetDoConstructIf(*it) : nullptr;
-  }
-  CHECK(level == 0);
-}
-
-void AccAttributeVisitor::Post(const parser::AccDefaultClause &x) {
-  if (!dirContext_.empty()) {
-    switch (x.v) {
-    case parser::AccDefaultClause::Arg::Present:
-      SetContextDefaultDSA(Symbol::Flag::AccPresent);
-      break;
-    case parser::AccDefaultClause::Arg::None:
-      SetContextDefaultDSA(Symbol::Flag::AccNone);
-      break;
-    }
-  }
-}
-
-// For OpenACC constructs, check all the data-refs within the constructs
-// and adjust the symbol for each Name if necessary
-void AccAttributeVisitor::Post(const parser::Name &name) {
-  auto *symbol{name.symbol};
-  if (symbol && !dirContext_.empty() && GetContext().withinConstruct) {
-    if (!symbol->owner().IsDerivedType() && !symbol->has<ProcEntityDetails>() &&
-        !IsObjectWithDSA(*symbol)) {
-      if (Symbol * found{currScope().FindSymbol(name.source)}) {
-        if (symbol != found) {
-          name.symbol = found; // adjust the symbol within region
-        } else if (GetContext().defaultDSA == Symbol::Flag::AccNone) {
-          // 2.5.14.
-          context_.Say(name.source,
-              "The DEFAULT(NONE) clause requires that '%s' must be listed in "
-              "a data-mapping clause"_err_en_US,
-              symbol->name());
-        }
-      }
-    }
-  } // within OpenACC construct
-}
-
-Symbol *AccAttributeVisitor::ResolveAccCommonBlockName(
-    const parser::Name *name) {
-  if (!name) {
-    return nullptr;
-  } else if (auto *prev{
-                 GetContext().scope.parent().FindCommonBlock(name->source)}) {
-    name->symbol = prev;
-    return prev;
-  } else {
-    return nullptr;
-  }
-}
-
-void AccAttributeVisitor::ResolveAccObjectList(
-    const parser::AccObjectList &accObjectList, Symbol::Flag accFlag) {
-  for (const auto &accObject : accObjectList.v) {
-    ResolveAccObject(accObject, accFlag);
-  }
-}
-
-void AccAttributeVisitor::ResolveAccObject(
-    const parser::AccObject &accObject, Symbol::Flag accFlag) {
-  std::visit(
-      common::visitors{
-          [&](const parser::Designator &designator) {
-            if (const auto *name{GetDesignatorNameIfDataRef(designator)}) {
-              if (auto *symbol{ResolveAcc(*name, accFlag, currScope())}) {
-                AddToContextObjectWithDSA(*symbol, accFlag);
-                if (dataSharingAttributeFlags.test(accFlag)) {
-                  CheckMultipleAppearances(*name, *symbol, accFlag);
-                }
-              }
-            } else if (const auto *designatorName{
-                           resolver_.ResolveDesignator(designator)};
-                       designatorName->symbol) {
-              // Array sections to be changed to substrings as needed
-              if (AnalyzeExpr(context_, designator)) {
-                if (std::holds_alternative<parser::Substring>(designator.u)) {
-                  context_.Say(designator.source,
-                      "Substrings are not allowed on OpenACC "
-                      "directives or clauses"_err_en_US);
-                }
-              }
-              // other checks, more TBD
-              if (const auto *details{designatorName->symbol
-                                          ->detailsIf<ObjectEntityDetails>()}) {
-                if (details->IsArray()) {
-                  // TODO: check Array Sections
-                } else if (designatorName->symbol->owner().IsDerivedType()) {
-                  // TODO: check Structure Component
-                }
-              }
-            }
-          },
-          [&](const parser::Name &name) { // common block
-            if (auto *symbol{ResolveAccCommonBlockName(&name)}) {
-              CheckMultipleAppearances(
-                  name, *symbol, Symbol::Flag::AccCommonBlock);
-              for (auto &object : symbol->get<CommonBlockDetails>().objects()) {
-                if (auto *resolvedObject{
-                        ResolveAcc(*object, accFlag, currScope())}) {
-                  AddToContextObjectWithDSA(*resolvedObject, accFlag);
-                }
-              }
-            } else {
-              context_.Say(name.source,
-                  "COMMON block must be declared in the same scoping unit "
-                  "in which the OpenACC directive or clause appears"_err_en_US);
-            }
-          },
-      },
-      accObject.u);
-}
-
-Symbol *AccAttributeVisitor::ResolveAcc(
-    const parser::Name &name, Symbol::Flag accFlag, Scope &scope) {
-  if (accFlagsRequireNewSymbol.test(accFlag)) {
-    return DeclarePrivateAccessEntity(name, accFlag, scope);
-  } else {
-    return DeclareOrMarkOtherAccessEntity(name, accFlag);
-  }
-}
-
-Symbol *AccAttributeVisitor::ResolveAcc(
-    Symbol &symbol, Symbol::Flag accFlag, Scope &scope) {
-  if (accFlagsRequireNewSymbol.test(accFlag)) {
-    return DeclarePrivateAccessEntity(symbol, accFlag, scope);
-  } else {
-    return DeclareOrMarkOtherAccessEntity(symbol, accFlag);
-  }
-}
-
-Symbol *AccAttributeVisitor::DeclareOrMarkOtherAccessEntity(
-    const parser::Name &name, Symbol::Flag accFlag) {
-  Symbol *prev{currScope().FindSymbol(name.source)};
-  if (!name.symbol || !prev) {
-    return nullptr;
-  } else if (prev != name.symbol) {
-    name.symbol = prev;
-  }
-  return DeclareOrMarkOtherAccessEntity(*prev, accFlag);
-}
-
-Symbol *AccAttributeVisitor::DeclareOrMarkOtherAccessEntity(
-    Symbol &object, Symbol::Flag accFlag) {
-  if (accFlagsRequireMark.test(accFlag)) {
-    object.set(accFlag);
-  }
-  return &object;
-}
-
-static bool WithMultipleAppearancesAccException(
-    const Symbol &symbol, Symbol::Flag flag) {
-  return false; // Place holder
-}
-
-void AccAttributeVisitor::CheckMultipleAppearances(
-    const parser::Name &name, const Symbol &symbol, Symbol::Flag accFlag) {
-  const auto *target{&symbol};
-  if (accFlagsRequireNewSymbol.test(accFlag)) {
-    if (const auto *details{symbol.detailsIf<HostAssocDetails>()}) {
-      target = &details->symbol();
-    }
-  }
-  if (HasDataSharingAttributeObject(*target) &&
-      !WithMultipleAppearancesAccException(symbol, accFlag)) {
-    context_.Say(name.source,
-        "'%s' appears in more than one data-sharing clause "
-        "on the same OpenACC directive"_err_en_US,
-        name.ToString());
-  } else {
-    AddDataSharingAttributeObject(*target);
-  }
-}
-
-bool OmpAttributeVisitor::Pre(const parser::OpenMPBlockConstruct &x) {
-  const auto &beginBlockDir{std::get<parser::OmpBeginBlockDirective>(x.t)};
-  const auto &beginDir{std::get<parser::OmpBlockDirective>(beginBlockDir.t)};
-  switch (beginDir.v) {
-  case llvm::omp::Directive::OMPD_master:
-  case llvm::omp::Directive::OMPD_ordered:
-  case llvm::omp::Directive::OMPD_parallel:
-  case llvm::omp::Directive::OMPD_single:
-  case llvm::omp::Directive::OMPD_target:
-  case llvm::omp::Directive::OMPD_target_data:
-  case llvm::omp::Directive::OMPD_task:
-  case llvm::omp::Directive::OMPD_teams:
-  case llvm::omp::Directive::OMPD_workshare:
-  case llvm::omp::Directive::OMPD_parallel_workshare:
-  case llvm::omp::Directive::OMPD_target_teams:
-  case llvm::omp::Directive::OMPD_target_parallel:
-    PushContext(beginDir.source, beginDir.v);
-    break;
-  default:
-    // TODO others
-    break;
-  }
-  ClearDataSharingAttributeObjects();
-  return true;
-}
-
-bool OmpAttributeVisitor::Pre(const parser::OpenMPLoopConstruct &x) {
-  const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
-  const auto &beginDir{std::get<parser::OmpLoopDirective>(beginLoopDir.t)};
-  const auto &clauseList{std::get<parser::OmpClauseList>(beginLoopDir.t)};
-  switch (beginDir.v) {
-  case llvm::omp::Directive::OMPD_distribute:
-  case llvm::omp::Directive::OMPD_distribute_parallel_do:
-  case llvm::omp::Directive::OMPD_distribute_parallel_do_simd:
-  case llvm::omp::Directive::OMPD_distribute_simd:
-  case llvm::omp::Directive::OMPD_do:
-  case llvm::omp::Directive::OMPD_do_simd:
-  case llvm::omp::Directive::OMPD_parallel_do:
-  case llvm::omp::Directive::OMPD_parallel_do_simd:
-  case llvm::omp::Directive::OMPD_simd:
-  case llvm::omp::Directive::OMPD_target_parallel_do:
-  case llvm::omp::Directive::OMPD_target_parallel_do_simd:
-  case llvm::omp::Directive::OMPD_target_teams_distribute:
-  case llvm::omp::Directive::OMPD_target_teams_distribute_parallel_do:
-  case llvm::omp::Directive::OMPD_target_teams_distribute_parallel_do_simd:
-  case llvm::omp::Directive::OMPD_target_teams_distribute_simd:
-  case llvm::omp::Directive::OMPD_target_simd:
-  case llvm::omp::Directive::OMPD_taskloop:
-  case llvm::omp::Directive::OMPD_taskloop_simd:
-  case llvm::omp::Directive::OMPD_teams_distribute:
-  case llvm::omp::Directive::OMPD_teams_distribute_parallel_do:
-  case llvm::omp::Directive::OMPD_teams_distribute_parallel_do_simd:
-  case llvm::omp::Directive::OMPD_teams_distribute_simd:
-    PushContext(beginDir.source, beginDir.v);
-    break;
-  default:
-    break;
-  }
-  ClearDataSharingAttributeObjects();
-  SetContextAssociatedLoopLevel(GetAssociatedLoopLevelFromClauses(clauseList));
-  PrivatizeAssociatedLoopIndex(x);
-  return true;
-}
-
-void OmpAttributeVisitor::ResolveSeqLoopIndexInParallelOrTaskConstruct(
-    const parser::Name &iv) {
-  auto targetIt{dirContext_.rbegin()};
-  for (;; ++targetIt) {
-    if (targetIt == dirContext_.rend()) {
-      return;
-    }
-    if (llvm::omp::parallelSet.test(targetIt->directive) ||
-        llvm::omp::taskGeneratingSet.test(targetIt->directive)) {
-      break;
-    }
-  }
-  if (auto *symbol{ResolveOmp(iv, Symbol::Flag::OmpPrivate, targetIt->scope)}) {
-    targetIt++;
-    symbol->set(Symbol::Flag::OmpPreDetermined);
-    iv.symbol = symbol; // adjust the symbol within region
-    for (auto it{dirContext_.rbegin()}; it != targetIt; ++it) {
-      AddToContextObjectWithDSA(*symbol, Symbol::Flag::OmpPrivate, *it);
-    }
-  }
-}
-
-// 2.15.1.1 Data-sharing Attribute Rules - Predetermined
-//   - A loop iteration variable for a sequential loop in a parallel
-//     or task generating construct is private in the innermost such
-//     construct that encloses the loop
-bool OmpAttributeVisitor::Pre(const parser::DoConstruct &x) {
-  if (!dirContext_.empty() && GetContext().withinConstruct) {
-    if (const auto &iv{GetLoopIndex(x)}; iv.symbol) {
-      if (!iv.symbol->test(Symbol::Flag::OmpPreDetermined)) {
-        ResolveSeqLoopIndexInParallelOrTaskConstruct(iv);
-      } else {
-        // TODO: conflict checks with explicitly determined DSA
-      }
-    }
-  }
-  return true;
-}
-
-std::int64_t OmpAttributeVisitor::GetAssociatedLoopLevelFromClauses(
-    const parser::OmpClauseList &x) {
-  std::int64_t orderedLevel{0};
-  std::int64_t collapseLevel{0};
-  for (const auto &clause : x.v) {
-    if (const auto *orderedClause{
-            std::get_if<parser::OmpClause::Ordered>(&clause.u)}) {
-      if (const auto v{
-              evaluate::ToInt64(resolver_.EvaluateIntExpr(orderedClause->v))}) {
-        orderedLevel = *v;
-      }
-    }
-    if (const auto *collapseClause{
-            std::get_if<parser::OmpClause::Collapse>(&clause.u)}) {
-      if (const auto v{evaluate::ToInt64(
-              resolver_.EvaluateIntExpr(collapseClause->v))}) {
-        collapseLevel = *v;
-      }
-    }
-  }
-
-  if (orderedLevel && (!collapseLevel || orderedLevel >= collapseLevel)) {
-    return orderedLevel;
-  } else if (!orderedLevel && collapseLevel) {
-    return collapseLevel;
-  } // orderedLevel < collapseLevel is an error handled in structural checks
-  return 1; // default is outermost loop
-}
-
-// 2.15.1.1 Data-sharing Attribute Rules - Predetermined
-//   - The loop iteration variable(s) in the associated do-loop(s) of a do,
-//     parallel do, taskloop, or distribute construct is (are) private.
-//   - The loop iteration variable in the associated do-loop of a simd construct
-//     with just one associated do-loop is linear with a linear-step that is the
-//     increment of the associated do-loop.
-//   - The loop iteration variables in the associated do-loops of a simd
-//     construct with multiple associated do-loops are lastprivate.
-//
-// TODO: revisit after semantics checks are completed for do-loop association of
-//       collapse and ordered
-void OmpAttributeVisitor::PrivatizeAssociatedLoopIndex(
-    const parser::OpenMPLoopConstruct &x) {
-  std::int64_t level{GetContext().associatedLoopLevel};
-  if (level <= 0)
-    return;
-  Symbol::Flag ivDSA{Symbol::Flag::OmpPrivate};
-  if (llvm::omp::simdSet.test(GetContext().directive)) {
-    if (level == 1) {
-      ivDSA = Symbol::Flag::OmpLinear;
-    } else {
-      ivDSA = Symbol::Flag::OmpLastPrivate;
-    }
-  }
-
-  auto &outer{std::get<std::optional<parser::DoConstruct>>(x.t)};
-  for (const parser::DoConstruct *loop{&*outer}; loop && level > 0; --level) {
-    // go through all the nested do-loops and resolve index variables
-    const parser::Name &iv{GetLoopIndex(*loop)};
-    if (auto *symbol{ResolveOmp(iv, ivDSA, currScope())}) {
-      symbol->set(Symbol::Flag::OmpPreDetermined);
-      iv.symbol = symbol; // adjust the symbol within region
-      AddToContextObjectWithDSA(*symbol, ivDSA);
-    }
-
-    const auto &block{std::get<parser::Block>(loop->t)};
-    const auto it{block.begin()};
-    loop = it != block.end() ? GetDoConstructIf(*it) : nullptr;
-  }
-  CHECK(level == 0);
-}
-
-bool OmpAttributeVisitor::Pre(const parser::OpenMPSectionsConstruct &x) {
-  const auto &beginSectionsDir{
-      std::get<parser::OmpBeginSectionsDirective>(x.t)};
-  const auto &beginDir{
-      std::get<parser::OmpSectionsDirective>(beginSectionsDir.t)};
-  switch (beginDir.v) {
-  case llvm::omp::Directive::OMPD_parallel_sections:
-  case llvm::omp::Directive::OMPD_sections:
-    PushContext(beginDir.source, beginDir.v);
-    break;
-  default:
-    break;
-  }
-  ClearDataSharingAttributeObjects();
-  return true;
-}
-
-bool OmpAttributeVisitor::Pre(const parser::OpenMPThreadprivate &x) {
-  PushContext(x.source, llvm::omp::Directive::OMPD_threadprivate);
-  const auto &list{std::get<parser::OmpObjectList>(x.t)};
-  ResolveOmpObjectList(list, Symbol::Flag::OmpThreadprivate);
-  return false;
-}
-
-void OmpAttributeVisitor::Post(const parser::OmpDefaultClause &x) {
-  if (!dirContext_.empty()) {
-    switch (x.v) {
-    case parser::OmpDefaultClause::Type::Private:
-      SetContextDefaultDSA(Symbol::Flag::OmpPrivate);
-      break;
-    case parser::OmpDefaultClause::Type::Firstprivate:
-      SetContextDefaultDSA(Symbol::Flag::OmpFirstPrivate);
-      break;
-    case parser::OmpDefaultClause::Type::Shared:
-      SetContextDefaultDSA(Symbol::Flag::OmpShared);
-      break;
-    case parser::OmpDefaultClause::Type::None:
-      SetContextDefaultDSA(Symbol::Flag::OmpNone);
-      break;
-    }
-  }
-}
-
-// For OpenMP constructs, check all the data-refs within the constructs
-// and adjust the symbol for each Name if necessary
-void OmpAttributeVisitor::Post(const parser::Name &name) {
-  auto *symbol{name.symbol};
-  if (symbol && !dirContext_.empty() && GetContext().withinConstruct) {
-    if (!symbol->owner().IsDerivedType() && !symbol->has<ProcEntityDetails>() &&
-        !IsObjectWithDSA(*symbol)) {
-      // TODO: create a separate function to go through the rules for
-      //       predetermined, explicitly determined, and implicitly
-      //       determined data-sharing attributes (2.15.1.1).
-      if (Symbol * found{currScope().FindSymbol(name.source)}) {
-        if (symbol != found) {
-          name.symbol = found; // adjust the symbol within region
-        } else if (GetContext().defaultDSA == Symbol::Flag::OmpNone) {
-          context_.Say(name.source,
-              "The DEFAULT(NONE) clause requires that '%s' must be listed in "
-              "a data-sharing attribute clause"_err_en_US,
-              symbol->name());
-        }
-      }
-    }
-  } // within OpenMP construct
-}
-
-Symbol *OmpAttributeVisitor::ResolveOmpCommonBlockName(
-    const parser::Name *name) {
-  if (auto *prev{name
-              ? GetContext().scope.parent().FindCommonBlock(name->source)
-              : nullptr}) {
-    name->symbol = prev;
-    return prev;
-  } else {
-    return nullptr;
-  }
-}
-
-void OmpAttributeVisitor::ResolveOmpObjectList(
-    const parser::OmpObjectList &ompObjectList, Symbol::Flag ompFlag) {
-  for (const auto &ompObject : ompObjectList.v) {
-    ResolveOmpObject(ompObject, ompFlag);
-  }
-}
-
-void OmpAttributeVisitor::ResolveOmpObject(
-    const parser::OmpObject &ompObject, Symbol::Flag ompFlag) {
-  std::visit(
-      common::visitors{
-          [&](const parser::Designator &designator) {
-            if (const auto *name{GetDesignatorNameIfDataRef(designator)}) {
-              if (auto *symbol{ResolveOmp(*name, ompFlag, currScope())}) {
-                AddToContextObjectWithDSA(*symbol, ompFlag);
-                if (dataSharingAttributeFlags.test(ompFlag)) {
-                  CheckMultipleAppearances(*name, *symbol, ompFlag);
-                }
-              }
-            } else if (const auto *designatorName{
-                           resolver_.ResolveDesignator(designator)};
-                       designatorName->symbol) {
-              // Array sections to be changed to substrings as needed
-              if (AnalyzeExpr(context_, designator)) {
-                if (std::holds_alternative<parser::Substring>(designator.u)) {
-                  context_.Say(designator.source,
-                      "Substrings are not allowed on OpenMP "
-                      "directives or clauses"_err_en_US);
-                }
-              }
-              // other checks, more TBD
-              if (const auto *details{designatorName->symbol
-                                          ->detailsIf<ObjectEntityDetails>()}) {
-                if (details->IsArray()) {
-                  // TODO: check Array Sections
-                } else if (designatorName->symbol->owner().IsDerivedType()) {
-                  // TODO: check Structure Component
-                }
-              }
-            }
-          },
-          [&](const parser::Name &name) { // common block
-            if (auto *symbol{ResolveOmpCommonBlockName(&name)}) {
-              CheckMultipleAppearances(
-                  name, *symbol, Symbol::Flag::OmpCommonBlock);
-              // 2.15.3 When a named common block appears in a list, it has the
-              // same meaning as if every explicit member of the common block
-              // appeared in the list
-              for (auto &object : symbol->get<CommonBlockDetails>().objects()) {
-                if (auto *resolvedObject{
-                        ResolveOmp(*object, ompFlag, currScope())}) {
-                  AddToContextObjectWithDSA(*resolvedObject, ompFlag);
-                }
-              }
-            } else {
-              context_.Say(name.source, // 2.15.3
-                  "COMMON block must be declared in the same scoping unit "
-                  "in which the OpenMP directive or clause appears"_err_en_US);
-            }
-          },
-      },
-      ompObject.u);
-}
-
-Symbol *OmpAttributeVisitor::ResolveOmp(
-    const parser::Name &name, Symbol::Flag ompFlag, Scope &scope) {
-  if (ompFlagsRequireNewSymbol.test(ompFlag)) {
-    return DeclarePrivateAccessEntity(name, ompFlag, scope);
-  } else {
-    return DeclareOrMarkOtherAccessEntity(name, ompFlag);
-  }
-}
-
-Symbol *OmpAttributeVisitor::ResolveOmp(
-    Symbol &symbol, Symbol::Flag ompFlag, Scope &scope) {
-  if (ompFlagsRequireNewSymbol.test(ompFlag)) {
-    return DeclarePrivateAccessEntity(symbol, ompFlag, scope);
-  } else {
-    return DeclareOrMarkOtherAccessEntity(symbol, ompFlag);
-  }
-}
-
-Symbol *OmpAttributeVisitor::DeclareOrMarkOtherAccessEntity(
-    const parser::Name &name, Symbol::Flag ompFlag) {
-  Symbol *prev{currScope().FindSymbol(name.source)};
-  if (!name.symbol || !prev) {
-    return nullptr;
-  } else if (prev != name.symbol) {
-    name.symbol = prev;
-  }
-  return DeclareOrMarkOtherAccessEntity(*prev, ompFlag);
-}
-
-Symbol *OmpAttributeVisitor::DeclareOrMarkOtherAccessEntity(
-    Symbol &object, Symbol::Flag ompFlag) {
-  if (ompFlagsRequireMark.test(ompFlag)) {
-    object.set(ompFlag);
-  }
-  return &object;
-}
-
-static bool WithMultipleAppearancesOmpException(
-    const Symbol &symbol, Symbol::Flag flag) {
-  return (flag == Symbol::Flag::OmpFirstPrivate &&
-             symbol.test(Symbol::Flag::OmpLastPrivate)) ||
-      (flag == Symbol::Flag::OmpLastPrivate &&
-          symbol.test(Symbol::Flag::OmpFirstPrivate));
-}
-
-void OmpAttributeVisitor::CheckMultipleAppearances(
-    const parser::Name &name, const Symbol &symbol, Symbol::Flag ompFlag) {
-  const auto *target{&symbol};
-  if (ompFlagsRequireNewSymbol.test(ompFlag)) {
-    if (const auto *details{symbol.detailsIf<HostAssocDetails>()}) {
-      target = &details->symbol();
-    }
-  }
-  if (HasDataSharingAttributeObject(*target) &&
-      !WithMultipleAppearancesOmpException(symbol, ompFlag)) {
-    context_.Say(name.source,
-        "'%s' appears in more than one data-sharing clause "
-        "on the same OpenMP directive"_err_en_US,
-        name.ToString());
-  } else {
-    AddDataSharingAttributeObject(*target);
-  }
-}
-
 // Perform checks and completions that need to happen after all of
 // the specification parts but before any of the execution parts.
 void ResolveNamesVisitor::FinishSpecificationParts(const ProgramTree &node) {
@@ -7427,22 +6406,6 @@ void ResolveNamesVisitor::ResolveExecutionParts(const ProgramTree &node) {
   }
 }
 
-void ResolveNamesVisitor::ResolveAccParts(const parser::ProgramUnit &node) {
-  AccAttributeVisitor{context(), *this}.Walk(node);
-}
-
-void ResolveNamesVisitor::ResolveOmpParts(const parser::ProgramUnit &node) {
-  OmpAttributeVisitor{context(), *this}.Walk(node);
-  if (!context().AnyFatalError()) {
-    // The data-sharing attribute of the loop iteration variable for a
-    // sequential loop (2.15.1.1) can only be determined when visiting
-    // the corresponding DoConstruct, a second walk is to adjust the
-    // symbols for all the data-refs of that loop iteration variable
-    // prior to the DoConstruct.
-    OmpAttributeVisitor{context(), *this}.Walk(node);
-  }
-}
-
 void ResolveNamesVisitor::Post(const parser::Program &) {
   // ensure that all temps were deallocated
   CHECK(!attrs_);
@@ -7474,4 +6437,5 @@ void ResolveSpecificationParts(
   visitor.ResolveSpecificationParts(node);
   context.set_location(std::move(originalLocation));
 }
+
 } // namespace Fortran::semantics
diff --git a/flang/test/Semantics/acc-resolve01.f90 b/flang/test/Semantics/acc-resolve01.f90
index 7e904b525926b..7ca0aa3627cee 100644
--- a/flang/test/Semantics/acc-resolve01.f90
+++ b/flang/test/Semantics/acc-resolve01.f90
@@ -19,4 +19,4 @@ end subroutine default_none
 
 program mm
   call default_none()
-end
\ No newline at end of file
+end

From 462b3564f1bd103b43ccaa2ddd9211badcf0f68d Mon Sep 17 00:00:00 2001
From: Siva Chandra Reddy <sivachandra@google.com>
Date: Tue, 28 Jul 2020 16:39:44 -0700
Subject: [PATCH 0391/1035] [libc][obvious] Add the new ctype.h target to the
 list of x86_64 headers.

This will fix the integration test which is failing currently.
---
 libc/config/linux/x86_64/headers.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libc/config/linux/x86_64/headers.txt b/libc/config/linux/x86_64/headers.txt
index b44681a5db4ec..550f172c32c1e 100644
--- a/libc/config/linux/x86_64/headers.txt
+++ b/libc/config/linux/x86_64/headers.txt
@@ -1,5 +1,6 @@
 set(TARGET_PUBLIC_HEADERS
     libc.include.assert_h
+    libc.include.ctype
     libc.include.errno
     libc.include.math
     libc.include.signal

From fb22678cd67801e41af4917acceb4014ab5d007e Mon Sep 17 00:00:00 2001
From: Kirill Bobyrev <kbobyrev@google.com>
Date: Wed, 29 Jul 2020 01:40:46 +0200
Subject: [PATCH 0392/1035] [clangd] Use elog instead of llvm::errs, log
 instead of llvm::outs

Reviewed By: sammccall

Differential Revision: https://reviews.llvm.org/D84697
---
 .../clangd/index/remote/server/Server.cpp     | 21 ++++++++++++++++---
 .../clangd/indexer/IndexerMain.cpp            |  3 ++-
 clang-tools-extra/clangd/tool/ClangdMain.cpp  | 14 ++++++-------
 3 files changed, 27 insertions(+), 11 deletions(-)

diff --git a/clang-tools-extra/clangd/index/remote/server/Server.cpp b/clang-tools-extra/clangd/index/remote/server/Server.cpp
index 364e4db6503c2..c3f9efeb7ec05 100644
--- a/clang-tools-extra/clangd/index/remote/server/Server.cpp
+++ b/clang-tools-extra/clangd/index/remote/server/Server.cpp
@@ -39,6 +39,15 @@ llvm::cl::opt<std::string> IndexPath(llvm::cl::desc("<INDEX FILE>"),
 llvm::cl::opt<std::string> IndexRoot(llvm::cl::desc("<PROJECT ROOT>"),
                                      llvm::cl::Positional, llvm::cl::Required);
 
+llvm::cl::opt<Logger::Level> LogLevel{
+    "log",
+    llvm::cl::desc("Verbosity of log messages written to stderr"),
+    values(clEnumValN(Logger::Error, "error", "Error messages only"),
+           clEnumValN(Logger::Info, "info", "High level execution tracing"),
+           clEnumValN(Logger::Debug, "verbose", "Low level details")),
+    llvm::cl::init(Logger::Info),
+};
+
 llvm::cl::opt<std::string> TraceFile(
     "trace-file",
     llvm::cl::desc("Path to the file where tracer logs will be stored"));
@@ -173,7 +182,7 @@ void runServer(std::unique_ptr<clangd::SymbolIndex> Index,
   Builder.AddListeningPort(ServerAddress, grpc::InsecureServerCredentials());
   Builder.RegisterService(&Service);
   std::unique_ptr<grpc::Server> Server(Builder.BuildAndStart());
-  llvm::outs() << "Server listening on " << ServerAddress << '\n';
+  log("Server listening on {0}", ServerAddress);
 
   Server->Wait();
 }
@@ -191,10 +200,16 @@ int main(int argc, char *argv[]) {
   llvm::sys::PrintStackTraceOnErrorSignal(argv[0]);
 
   if (!llvm::sys::path::is_absolute(IndexRoot)) {
-    elog("Index root should be an absolute path.");
+    llvm::errs() << "Index root should be an absolute path.\n";
     return -1;
   }
 
+  llvm::errs().SetBuffered();
+  // Don't flush stdout when logging for thread safety.
+  llvm::errs().tie(nullptr);
+  clang::clangd::StreamLogger Logger(llvm::errs(), LogLevel);
+  clang::clangd::LoggingSession LoggingSession(Logger);
+
   llvm::Optional<llvm::raw_fd_ostream> TracerStream;
   std::unique_ptr<clang::clangd::trace::EventTracer> Tracer;
   if (!TraceFile.empty()) {
@@ -220,7 +235,7 @@ int main(int argc, char *argv[]) {
   std::unique_ptr<clang::clangd::SymbolIndex> Index = openIndex(IndexPath);
 
   if (!Index) {
-    elog("Failed to open the index.");
+    llvm::errs() << "Failed to open the index.\n";
     return -1;
   }
 
diff --git a/clang-tools-extra/clangd/indexer/IndexerMain.cpp b/clang-tools-extra/clangd/indexer/IndexerMain.cpp
index dac038308d9e9..46224238c3fca 100644
--- a/clang-tools-extra/clangd/indexer/IndexerMain.cpp
+++ b/clang-tools-extra/clangd/indexer/IndexerMain.cpp
@@ -16,6 +16,7 @@
 #include "index/Serialization.h"
 #include "index/Symbol.h"
 #include "index/SymbolCollector.h"
+#include "support/Logger.h"
 #include "clang/Tooling/ArgumentsAdjusters.h"
 #include "clang/Tooling/CommonOptionsParser.h"
 #include "clang/Tooling/Execution.h"
@@ -122,7 +123,7 @@ int main(int argc, const char **argv) {
       std::make_unique<clang::clangd::IndexActionFactory>(Data),
       clang::tooling::getStripPluginsAdjuster());
   if (Err) {
-    llvm::errs() << llvm::toString(std::move(Err)) << "\n";
+    clang::clangd::elog("{0}", std::move(Err));
   }
 
   // Emit collected data.
diff --git a/clang-tools-extra/clangd/tool/ClangdMain.cpp b/clang-tools-extra/clangd/tool/ClangdMain.cpp
index 0d4267774c92f..daf87d11c3843 100644
--- a/clang-tools-extra/clangd/tool/ClangdMain.cpp
+++ b/clang-tools-extra/clangd/tool/ClangdMain.cpp
@@ -657,16 +657,16 @@ clangd accepts flags on the commandline, and in the CLANGD_FLAGS environment var
       // continuing.
       llvm::SmallString<128> Path(CompileCommandsDir);
       if (std::error_code EC = llvm::sys::fs::make_absolute(Path)) {
-        llvm::errs() << "Error while converting the relative path specified by "
-                        "--compile-commands-dir to an absolute path: "
-                     << EC.message() << ". The argument will be ignored.\n";
+        elog("Error while converting the relative path specified by "
+             "--compile-commands-dir to an absolute path: {0}. The argument "
+             "will be ignored.",
+             EC.message());
       } else {
         CompileCommandsDirPath = std::string(Path.str());
       }
     } else {
-      llvm::errs()
-          << "Path specified by --compile-commands-dir does not exist. The "
-             "argument will be ignored.\n";
+      elog("Path specified by --compile-commands-dir does not exist. The "
+           "argument will be ignored.");
     }
   }
 
@@ -750,7 +750,7 @@ clangd accepts flags on the commandline, and in the CLANGD_FLAGS environment var
       elog("Couldn't determine user config file, not loading");
     }
     std::vector<const config::Provider *> ProviderPointers;
-    for (const auto& P : ProviderStack)
+    for (const auto &P : ProviderStack)
       ProviderPointers.push_back(P.get());
     Config = config::Provider::combine(std::move(ProviderPointers));
     Opts.ConfigProvider = Config.get();

From 66d60e06cbc51b453f9e38ad69795f9487213fe5 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Thu, 23 Jul 2020 21:11:46 -0400
Subject: [PATCH 0393/1035] AMDGPU: Serialize MFI spill fields

These should probably be inferred from the function on parse, but the
target specific infrastructure currently does not give you a way to do
this. SILowerSGPRSpills early exits without this reporting spills,
which makes it difficult to write a MIR test for.
---
 .../Target/AMDGPU/SIMachineFunctionInfo.cpp   |  4 +++
 .../lib/Target/AMDGPU/SIMachineFunctionInfo.h |  4 +++
 .../AMDGPU/machine-function-info-no-ir.mir    | 26 +++++++++++++++++++
 .../MIR/AMDGPU/machine-function-info.ll       |  8 ++++++
 4 files changed, 42 insertions(+)

diff --git a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp
index 788e9873f780f..441bad07f89f2 100644
--- a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp
@@ -546,6 +546,8 @@ yaml::SIMachineFunctionInfo::SIMachineFunctionInfo(
     NoSignedZerosFPMath(MFI.hasNoSignedZerosFPMath()),
     MemoryBound(MFI.isMemoryBound()),
     WaveLimiter(MFI.needsWaveLimiter()),
+    HasSpilledSGPRs(MFI.hasSpilledSGPRs()),
+    HasSpilledVGPRs(MFI.hasSpilledVGPRs()),
     HighBitsOf32BitAddress(MFI.get32BitAddressHighBits()),
     ScratchRSrcReg(regToString(MFI.getScratchRSrcReg(), TRI)),
     FrameOffsetReg(regToString(MFI.getFrameOffsetReg(), TRI)),
@@ -567,6 +569,8 @@ bool SIMachineFunctionInfo::initializeBaseYamlFields(
   NoSignedZerosFPMath = YamlMFI.NoSignedZerosFPMath;
   MemoryBound = YamlMFI.MemoryBound;
   WaveLimiter = YamlMFI.WaveLimiter;
+  HasSpilledSGPRs = YamlMFI.HasSpilledSGPRs;
+  HasSpilledVGPRs = YamlMFI.HasSpilledVGPRs;
   return false;
 }
 
diff --git a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h
index 57a89d9acc815..d9a2d3abb0b88 100644
--- a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h
@@ -281,6 +281,8 @@ struct SIMachineFunctionInfo final : public yaml::MachineFunctionInfo {
   bool NoSignedZerosFPMath = false;
   bool MemoryBound = false;
   bool WaveLimiter = false;
+  bool HasSpilledSGPRs = false;
+  bool HasSpilledVGPRs = false;
   uint32_t HighBitsOf32BitAddress = 0;
 
   StringValue ScratchRSrcReg = "$private_rsrc_reg";
@@ -308,6 +310,8 @@ template <> struct MappingTraits<SIMachineFunctionInfo> {
     YamlIO.mapOptional("noSignedZerosFPMath", MFI.NoSignedZerosFPMath, false);
     YamlIO.mapOptional("memoryBound", MFI.MemoryBound, false);
     YamlIO.mapOptional("waveLimiter", MFI.WaveLimiter, false);
+    YamlIO.mapOptional("hasSpilledSGPRs", MFI.HasSpilledSGPRs, false);
+    YamlIO.mapOptional("hasSpilledVGPRs", MFI.HasSpilledVGPRs, false);
     YamlIO.mapOptional("scratchRSrcReg", MFI.ScratchRSrcReg,
                        StringValue("$private_rsrc_reg"));
     YamlIO.mapOptional("frameOffsetReg", MFI.FrameOffsetReg,
diff --git a/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-no-ir.mir b/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-no-ir.mir
index e39ff3b350c80..5d4eae6fca117 100644
--- a/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-no-ir.mir
+++ b/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-no-ir.mir
@@ -12,6 +12,8 @@
 # FULL-NEXT: noSignedZerosFPMath: false
 # FULL-NEXT: memoryBound:     true
 # FULL-NEXT: waveLimiter:     true
+# FULL-NEXT: hasSpilledSGPRs: false
+# FULL-NEXT: hasSpilledVGPRs: false
 # FULL-NEXT: scratchRSrcReg:  '$sgpr8_sgpr9_sgpr10_sgpr11'
 # FULL-NEXT: frameOffsetReg:  '$sgpr12'
 # FULL-NEXT: stackPtrOffsetReg:  '$sgpr13'
@@ -83,6 +85,8 @@ body:             |
 # FULL-NEXT: noSignedZerosFPMath: false
 # FULL-NEXT: memoryBound:     false
 # FULL-NEXT: waveLimiter:     false
+# FULL-NEXT: hasSpilledSGPRs: false
+# FULL-NEXT: hasSpilledVGPRs: false
 # FULL-NEXT: scratchRSrcReg:  '$private_rsrc_reg'
 # FULL-NEXT: frameOffsetReg:  '$fp_reg'
 # FULL-NEXT: stackPtrOffsetReg:  '$sp_reg'
@@ -121,6 +125,8 @@ body:             |
 # FULL-NEXT: noSignedZerosFPMath: false
 # FULL-NEXT: memoryBound:     false
 # FULL-NEXT: waveLimiter:     false
+# FULL-NEXT: hasSpilledSGPRs: false
+# FULL-NEXT: hasSpilledVGPRs: false
 # FULL-NEXT: scratchRSrcReg:  '$private_rsrc_reg'
 # FULL-NEXT: frameOffsetReg:  '$fp_reg'
 # FULL-NEXT: stackPtrOffsetReg:  '$sp_reg'
@@ -160,6 +166,8 @@ body:             |
 # FULL-NEXT: noSignedZerosFPMath: false
 # FULL-NEXT: memoryBound:     false
 # FULL-NEXT: waveLimiter:     false
+# FULL-NEXT: hasSpilledSGPRs: false
+# FULL-NEXT: hasSpilledVGPRs: false
 # FULL-NEXT: scratchRSrcReg:  '$private_rsrc_reg'
 # FULL-NEXT: frameOffsetReg:  '$fp_reg'
 # FULL-NEXT: stackPtrOffsetReg:  '$sp_reg'
@@ -259,3 +267,21 @@ body:             |
     S_ENDPGM 0
 
 ...
+
+
+---
+# ALL-LABEL: name: parse_spilled_regs
+# ALL: machineFunctionInfo:
+# ALL: hasSpilledSGPRs: true
+# ALL-NEXT: hasSpilledVGPRs: true
+
+name: parse_spilled_regs
+machineFunctionInfo:
+  hasSpilledSGPRs: true
+  hasSpilledVGPRs: true
+
+body:             |
+  bb.0:
+    S_ENDPGM 0
+
+...
diff --git a/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info.ll b/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info.ll
index 94f926e451a9e..b6a0476389429 100644
--- a/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info.ll
+++ b/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info.ll
@@ -15,6 +15,8 @@
 ; CHECK-NEXT: noSignedZerosFPMath: false
 ; CHECK-NEXT: memoryBound: false
 ; CHECK-NEXT: waveLimiter: false
+; CHECK-NEXT: hasSpilledSGPRs: false
+; CHECK-NEXT: hasSpilledVGPRs: false
 ; CHECK-NEXT: scratchRSrcReg:  '$sgpr96_sgpr97_sgpr98_sgpr99'
 ; CHECK-NEXT: frameOffsetReg:  '$fp_reg'
 ; CHECK-NEXT: stackPtrOffsetReg: '$sgpr32'
@@ -48,6 +50,8 @@ define amdgpu_kernel void @kernel(i32 %arg0, i64 %arg1, <16 x i32> %arg2) {
 ; CHECK-NEXT: noSignedZerosFPMath: false
 ; CHECK-NEXT: memoryBound: false
 ; CHECK-NEXT: waveLimiter: false
+; CHECK-NEXT: hasSpilledSGPRs: false
+; CHECK-NEXT: hasSpilledVGPRs: false
 ; CHECK-NEXT: scratchRSrcReg:  '$sgpr96_sgpr97_sgpr98_sgpr99'
 ; CHECK-NEXT: frameOffsetReg:  '$fp_reg'
 ; CHECK-NEXT: stackPtrOffsetReg: '$sgpr32'
@@ -76,6 +80,8 @@ define amdgpu_ps void @ps_shader(i32 %arg0, i32 inreg %arg1) {
 ; CHECK-NEXT: noSignedZerosFPMath: false
 ; CHECK-NEXT: memoryBound: false
 ; CHECK-NEXT: waveLimiter: false
+; CHECK-NEXT: hasSpilledSGPRs: false
+; CHECK-NEXT: hasSpilledVGPRs: false
 ; CHECK-NEXT: scratchRSrcReg: '$sgpr0_sgpr1_sgpr2_sgpr3'
 ; CHECK-NEXT: frameOffsetReg: '$sgpr33'
 ; CHECK-NEXT: stackPtrOffsetReg: '$sgpr32'
@@ -103,6 +109,8 @@ define void @function() {
 ; CHECK-NEXT: noSignedZerosFPMath: true
 ; CHECK-NEXT: memoryBound: false
 ; CHECK-NEXT: waveLimiter: false
+; CHECK-NEXT: hasSpilledSGPRs: false
+; CHECK-NEXT: hasSpilledVGPRs: false
 ; CHECK-NEXT: scratchRSrcReg: '$sgpr0_sgpr1_sgpr2_sgpr3'
 ; CHECK-NEXT: frameOffsetReg: '$sgpr33'
 ; CHECK-NEXT: stackPtrOffsetReg: '$sgpr32'

From 592f2e8d1ceb8ecec6f8b54eeb9fd7e0a099c0fe Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Thu, 23 Jul 2020 21:11:04 -0400
Subject: [PATCH 0394/1035] AMDGPU: Fix verifier error on spilling partially
 defined SGPRs

This needs an implicit def of the super-register in case one of the
lanes isn't defined, similar to copyPhysReg (or the not-VGPR spill
case below). This showed up in GlobalISel testing since it currently
doesn't fold out many undef instructions.
---
 llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp     |  8 ++-
 .../AMDGPU/sgpr-spill-partially-undef.mir     | 52 +++++++++++++++++++
 2 files changed, 59 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/CodeGen/AMDGPU/sgpr-spill-partially-undef.mir

diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
index 956658296a94a..9263be6e1abfd 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
@@ -1029,13 +1029,19 @@ bool SIRegisterInfo::spillSGPR(MachineBasicBlock::iterator MI,
 
       // Mark the "old value of vgpr" input undef only if this is the first sgpr
       // spill to this specific vgpr in the first basic block.
-      BuildMI(*MBB, MI, DL,
+      auto MIB = BuildMI(*MBB, MI, DL,
               TII->getMCOpcodeFromPseudo(AMDGPU::V_WRITELANE_B32),
               Spill.VGPR)
         .addReg(SubReg, getKillRegState(IsKill))
         .addImm(Spill.Lane)
         .addReg(Spill.VGPR, VGPRDefined ? 0 : RegState::Undef);
 
+      if (i == 0 && NumSubRegs > 1) {
+        // We may be spilling a super-register which is only partially defined,
+        // and need to ensure later spills think the value is defined.
+        MIB.addReg(SuperReg, RegState::ImplicitDefine);
+      }
+
       // FIXME: Since this spills to another register instead of an actual
       // frame index, we should delete the frame index when all references to
       // it are fixed.
diff --git a/llvm/test/CodeGen/AMDGPU/sgpr-spill-partially-undef.mir b/llvm/test/CodeGen/AMDGPU/sgpr-spill-partially-undef.mir
new file mode 100644
index 0000000000000..777a692affca0
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/sgpr-spill-partially-undef.mir
@@ -0,0 +1,52 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs -run-pass=si-lower-sgpr-spills -o - %s  | FileCheck %s
+#
+# Check for liveness errors when spilling partially defined super registers.
+
+---
+name:  sgpr_spill_s64_undef_high32
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+  hasSpilledSGPRs:     true
+  scratchRSrcReg:  '$sgpr96_sgpr97_sgpr98_sgpr99'
+  stackPtrOffsetReg: '$sgpr32'
+
+stack:
+  - { id: 0, type: spill-slot, size: 8, alignment: 4, stack-id: sgpr-spill }
+
+body:             |
+  bb.0:
+    liveins: $sgpr4
+
+    ; CHECK-LABEL: name: sgpr_spill_s64_undef_high32
+    ; CHECK: liveins: $sgpr4, $vgpr0
+    ; CHECK: $vgpr0 = V_WRITELANE_B32_gfx6_gfx7 $sgpr4, 0, undef $vgpr0, implicit-def $sgpr4_sgpr5
+    ; CHECK: $vgpr0 = V_WRITELANE_B32_gfx6_gfx7 $sgpr5, 1, $vgpr0
+    SI_SPILL_S64_SAVE renamable $sgpr4_sgpr5, %stack.0, implicit $exec, implicit $sgpr96_sgpr97_sgpr98_sgpr99, implicit $sgpr32 :: (store 8 into %stack.0, align 4, addrspace 5)
+
+...
+
+---
+name:  sgpr_spill_s64_undef_low32
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+  hasSpilledSGPRs:     true
+  scratchRSrcReg:  '$sgpr96_sgpr97_sgpr98_sgpr99'
+  stackPtrOffsetReg: '$sgpr32'
+
+stack:
+  - { id: 0, type: spill-slot, size: 8, alignment: 4, stack-id: sgpr-spill }
+
+body:             |
+  bb.0:
+    liveins: $sgpr5
+
+    ; CHECK-LABEL: name: sgpr_spill_s64_undef_low32
+    ; CHECK: liveins: $sgpr5, $vgpr0
+    ; CHECK: $vgpr0 = V_WRITELANE_B32_gfx6_gfx7 $sgpr4, 0, undef $vgpr0, implicit-def $sgpr4_sgpr5
+    ; CHECK: $vgpr0 = V_WRITELANE_B32_gfx6_gfx7 $sgpr5, 1, $vgpr0
+    SI_SPILL_S64_SAVE renamable $sgpr4_sgpr5, %stack.0, implicit $exec, implicit $sgpr96_sgpr97_sgpr98_sgpr99, implicit $sgpr32 :: (store 8 into %stack.0, align 4, addrspace 5)
+
+...

From cd7a5c3cd9a4891654b1b36f743a250135eb5c5a Mon Sep 17 00:00:00 2001
From: Siva Chandra Reddy <sivachandra@google.com>
Date: Tue, 28 Jul 2020 17:03:16 -0700
Subject: [PATCH 0395/1035] [libc][obvious] Add license headers to
 APIIndexer.[cpp|h].

---
 libc/utils/LibcTableGenUtil/APIIndexer.cpp |  7 +++++++
 libc/utils/LibcTableGenUtil/APIIndexer.h   | 12 ++++++++++++
 2 files changed, 19 insertions(+)

diff --git a/libc/utils/LibcTableGenUtil/APIIndexer.cpp b/libc/utils/LibcTableGenUtil/APIIndexer.cpp
index 934d713f905e5..366dea9ec11c6 100644
--- a/libc/utils/LibcTableGenUtil/APIIndexer.cpp
+++ b/libc/utils/LibcTableGenUtil/APIIndexer.cpp
@@ -1,3 +1,10 @@
+//===-- Implementation of APIIndexer clas ---------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
 
 #include "APIIndexer.h"
 
diff --git a/libc/utils/LibcTableGenUtil/APIIndexer.h b/libc/utils/LibcTableGenUtil/APIIndexer.h
index a0f0acf600d81..8fa87113610e1 100644
--- a/libc/utils/LibcTableGenUtil/APIIndexer.h
+++ b/libc/utils/LibcTableGenUtil/APIIndexer.h
@@ -1,3 +1,13 @@
+//===-- A class to index libc API listed in tablegen files ------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_UTILS_LIBC_TABLE_GEN_UTILS_API_INDEXER_H
+#define LLVM_LIBC_UTILS_LIBC_TABLE_GEN_UTILS_API_INDEXER_H
 
 #include "llvm/ADT/StringRef.h"
 #include "llvm/TableGen/Record.h"
@@ -64,3 +74,5 @@ class APIIndexer {
 };
 
 } // namespace llvm_libc
+
+#endif // LLVM_LIBC_UTILS_LIBC_TABLE_GEN_UTILS_API_INDEXER_H

From 3ce69d4d50a24394eff15f92e3f4a609acc963e7 Mon Sep 17 00:00:00 2001
From: Shilei Tian <tianshilei1992@gmail.com>
Date: Tue, 28 Jul 2020 20:10:59 -0400
Subject: [PATCH 0396/1035] [NFC][OpenMP] Renamed all variable and function
 names in `target` to conform with LLVM code standard

This patch only touched variables and functions in `target`.

Reviewed By: ye-luo

Differential Revision: https://reviews.llvm.org/D84797
---
 openmp/libomptarget/src/api.cpp       |   8 +-
 openmp/libomptarget/src/device.cpp    |  34 +--
 openmp/libomptarget/src/device.h      |  22 +-
 openmp/libomptarget/src/interface.cpp |   8 +-
 openmp/libomptarget/src/omptarget.cpp | 292 +++++++++++++-------------
 openmp/libomptarget/src/private.h     |  26 +--
 6 files changed, 194 insertions(+), 196 deletions(-)

diff --git a/openmp/libomptarget/src/api.cpp b/openmp/libomptarget/src/api.cpp
index 6a5ad86bd18a4..da665dbee1e8d 100644
--- a/openmp/libomptarget/src/api.cpp
+++ b/openmp/libomptarget/src/api.cpp
@@ -57,7 +57,7 @@ EXTERN void *omp_target_alloc(size_t size, int device_num) {
     return NULL;
   }
 
-  rc = Devices[device_num].data_alloc(size);
+  rc = Devices[device_num].allocData(size);
   DP("omp_target_alloc returns device ptr " DPxMOD "\n", DPxPTR(rc));
   return rc;
 }
@@ -82,7 +82,7 @@ EXTERN void omp_target_free(void *device_ptr, int device_num) {
     return;
   }
 
-  Devices[device_num].data_delete(device_ptr);
+  Devices[device_num].deleteData(device_ptr);
   DP("omp_target_free deallocated device ptr\n");
 }
 
@@ -159,7 +159,7 @@ EXTERN int omp_target_memcpy(void *dst, void *src, size_t length,
   } else if (src_device == omp_get_initial_device()) {
     DP("copy from host to device\n");
     DeviceTy& DstDev = Devices[dst_device];
-    rc = DstDev.data_submit(dstAddr, srcAddr, length, nullptr);
+    rc = DstDev.submitData(dstAddr, srcAddr, length, nullptr);
   } else if (dst_device == omp_get_initial_device()) {
     DP("copy from device to host\n");
     DeviceTy& SrcDev = Devices[src_device];
@@ -179,7 +179,7 @@ EXTERN int omp_target_memcpy(void *dst, void *src, size_t length,
     void *buffer = malloc(length);
     rc = SrcDev.data_retrieve(buffer, srcAddr, length, nullptr);
     if (rc == OFFLOAD_SUCCESS)
-      rc = DstDev.data_submit(dstAddr, buffer, length, nullptr);
+      rc = DstDev.submitData(dstAddr, buffer, length, nullptr);
     free(buffer);
   }
 
diff --git a/openmp/libomptarget/src/device.cpp b/openmp/libomptarget/src/device.cpp
index 6da8320462eef..c16a1a8cd764e 100644
--- a/openmp/libomptarget/src/device.cpp
+++ b/openmp/libomptarget/src/device.cpp
@@ -153,7 +153,7 @@ LookupResult DeviceTy::lookupMapping(void *HstPtrBegin, int64_t Size) {
   return lr;
 }
 
-// Used by target_data_begin
+// Used by targetDataBegin
 // Return the target pointer begin (where the data will be moved).
 // Allocate memory if this is the first occurrence of this mapping.
 // Increment the reference counter.
@@ -217,7 +217,7 @@ void *DeviceTy::getOrAllocTgtPtr(void *HstPtrBegin, void *HstPtrBase,
   } else if (Size) {
     // If it is not contained and Size > 0, we should create a new entry for it.
     IsNew = true;
-    uintptr_t tp = (uintptr_t)data_alloc(Size, HstPtrBegin);
+    uintptr_t tp = (uintptr_t)allocData(Size, HstPtrBegin);
     DP("Creating new map entry: HstBase=" DPxMOD ", HstBegin=" DPxMOD ", "
        "HstEnd=" DPxMOD ", TgtBegin=" DPxMOD "\n",
        DPxPTR(HstPtrBase), DPxPTR(HstPtrBegin),
@@ -232,9 +232,9 @@ void *DeviceTy::getOrAllocTgtPtr(void *HstPtrBegin, void *HstPtrBase,
   return rc;
 }
 
-// Used by target_data_begin, target_data_end, target_data_update and target.
+// Used by targetDataBegin, targetDataEnd, target_data_update and target.
 // Return the target pointer begin (where the data will be moved).
-// Decrement the reference counter if called from target_data_end.
+// Decrement the reference counter if called from targetDataEnd.
 void *DeviceTy::getTgtPtrBegin(void *HstPtrBegin, int64_t Size, bool &IsLast,
     bool UpdateRefCount, bool &IsHostPtr) {
   void *rc = NULL;
@@ -299,7 +299,7 @@ int DeviceTy::deallocTgtPtr(void *HstPtrBegin, int64_t Size, bool ForceDelete,
     if (HT.decRefCount() == 0) {
       DP("Deleting tgt data " DPxMOD " of size %ld\n",
           DPxPTR(HT.TgtPtrBegin), Size);
-      data_delete((void *)HT.TgtPtrBegin);
+      deleteData((void *)HT.TgtPtrBegin);
       DP("Removing%s mapping with HstPtrBegin=" DPxMOD ", TgtPtrBegin=" DPxMOD
           ", Size=%ld\n", (ForceDelete ? " (forced)" : ""),
           DPxPTR(HT.HstPtrBegin), DPxPTR(HT.TgtPtrBegin), Size);
@@ -351,17 +351,17 @@ __tgt_target_table *DeviceTy::load_binary(void *Img) {
   return rc;
 }
 
-void *DeviceTy::data_alloc(int64_t Size, void *HstPtr) {
+void *DeviceTy::allocData(int64_t Size, void *HstPtr) {
   return RTL->data_alloc(RTLDeviceID, Size, HstPtr);
 }
 
-int32_t DeviceTy::data_delete(void *TgtPtrBegin) {
+int32_t DeviceTy::deleteData(void *TgtPtrBegin) {
   return RTL->data_delete(RTLDeviceID, TgtPtrBegin);
 }
 
 // Submit data to device
-int32_t DeviceTy::data_submit(void *TgtPtrBegin, void *HstPtrBegin,
-                              int64_t Size, __tgt_async_info *AsyncInfoPtr) {
+int32_t DeviceTy::submitData(void *TgtPtrBegin, void *HstPtrBegin, int64_t Size,
+                             __tgt_async_info *AsyncInfoPtr) {
   if (!AsyncInfoPtr || !RTL->data_submit_async || !RTL->synchronize)
     return RTL->data_submit(RTLDeviceID, TgtPtrBegin, HstPtrBegin, Size);
   else
@@ -392,9 +392,9 @@ int32_t DeviceTy::data_exchange(void *SrcPtr, DeviceTy DstDev, void *DstPtr,
 }
 
 // Run region on device
-int32_t DeviceTy::run_region(void *TgtEntryPtr, void **TgtVarsPtr,
-                             ptrdiff_t *TgtOffsets, int32_t TgtVarsSize,
-                             __tgt_async_info *AsyncInfoPtr) {
+int32_t DeviceTy::runRegion(void *TgtEntryPtr, void **TgtVarsPtr,
+                            ptrdiff_t *TgtOffsets, int32_t TgtVarsSize,
+                            __tgt_async_info *AsyncInfoPtr) {
   if (!AsyncInfoPtr || !RTL->run_region || !RTL->synchronize)
     return RTL->run_region(RTLDeviceID, TgtEntryPtr, TgtVarsPtr, TgtOffsets,
                            TgtVarsSize);
@@ -404,11 +404,11 @@ int32_t DeviceTy::run_region(void *TgtEntryPtr, void **TgtVarsPtr,
 }
 
 // Run team region on device.
-int32_t DeviceTy::run_team_region(void *TgtEntryPtr, void **TgtVarsPtr,
-                                  ptrdiff_t *TgtOffsets, int32_t TgtVarsSize,
-                                  int32_t NumTeams, int32_t ThreadLimit,
-                                  uint64_t LoopTripCount,
-                                  __tgt_async_info *AsyncInfoPtr) {
+int32_t DeviceTy::runTeamRegion(void *TgtEntryPtr, void **TgtVarsPtr,
+                                ptrdiff_t *TgtOffsets, int32_t TgtVarsSize,
+                                int32_t NumTeams, int32_t ThreadLimit,
+                                uint64_t LoopTripCount,
+                                __tgt_async_info *AsyncInfoPtr) {
   if (!AsyncInfoPtr || !RTL->run_team_region_async || !RTL->synchronize)
     return RTL->run_team_region(RTLDeviceID, TgtEntryPtr, TgtVarsPtr,
                                 TgtOffsets, TgtVarsSize, NumTeams, ThreadLimit,
diff --git a/openmp/libomptarget/src/device.h b/openmp/libomptarget/src/device.h
index 72b919c04df01..e4b188c6a8fbc 100644
--- a/openmp/libomptarget/src/device.h
+++ b/openmp/libomptarget/src/device.h
@@ -199,16 +199,16 @@ struct DeviceTy {
   /// default value of \p HstPtr is nullptr. Note: this function doesn't do
   /// pointer association. Actually, all the __tgt_rtl_data_alloc
   /// implementations ignore \p HstPtr.
-  void *data_alloc(int64_t Size, void *HstPtr = nullptr);
+  void *allocData(int64_t Size, void *HstPtr = nullptr);
   /// Deallocates memory which \p TgtPtrBegin points at and returns
   /// OFFLOAD_SUCCESS/OFFLOAD_FAIL when succeeds/fails.
-  int32_t data_delete(void *TgtPtrBegin);
+  int32_t deleteData(void *TgtPtrBegin);
 
   // Data transfer. When AsyncInfoPtr is nullptr, the transfer will be
   // synchronous.
   // Copy data from host to device
-  int32_t data_submit(void *TgtPtrBegin, void *HstPtrBegin, int64_t Size,
-                      __tgt_async_info *AsyncInfoPtr);
+  int32_t submitData(void *TgtPtrBegin, void *HstPtrBegin, int64_t Size,
+                     __tgt_async_info *AsyncInfoPtr);
   // Copy data from device back to host
   int32_t data_retrieve(void *HstPtrBegin, void *TgtPtrBegin, int64_t Size,
                         __tgt_async_info *AsyncInfoPtr);
@@ -216,14 +216,12 @@ struct DeviceTy {
   int32_t data_exchange(void *SrcPtr, DeviceTy DstDev, void *DstPtr,
                         int64_t Size, __tgt_async_info *AsyncInfoPtr);
 
-  int32_t run_region(void *TgtEntryPtr, void **TgtVarsPtr,
-                     ptrdiff_t *TgtOffsets, int32_t TgtVarsSize,
-                     __tgt_async_info *AsyncInfoPtr);
-  int32_t run_team_region(void *TgtEntryPtr, void **TgtVarsPtr,
-                          ptrdiff_t *TgtOffsets, int32_t TgtVarsSize,
-                          int32_t NumTeams, int32_t ThreadLimit,
-                          uint64_t LoopTripCount,
-                          __tgt_async_info *AsyncInfoPtr);
+  int32_t runRegion(void *TgtEntryPtr, void **TgtVarsPtr, ptrdiff_t *TgtOffsets,
+                    int32_t TgtVarsSize, __tgt_async_info *AsyncInfoPtr);
+  int32_t runTeamRegion(void *TgtEntryPtr, void **TgtVarsPtr,
+                        ptrdiff_t *TgtOffsets, int32_t TgtVarsSize,
+                        int32_t NumTeams, int32_t ThreadLimit,
+                        uint64_t LoopTripCount, __tgt_async_info *AsyncInfoPtr);
 
   /// Synchronize device/queue/event based on \p AsyncInfoPtr and return
   /// OFFLOAD_SUCCESS/OFFLOAD_FAIL when succeeds/fails.
diff --git a/openmp/libomptarget/src/interface.cpp b/openmp/libomptarget/src/interface.cpp
index 751641183437d..24b733ca99d24 100644
--- a/openmp/libomptarget/src/interface.cpp
+++ b/openmp/libomptarget/src/interface.cpp
@@ -136,8 +136,8 @@ EXTERN void __tgt_target_data_begin_mapper(int64_t device_id, int32_t arg_num,
   }
 #endif
 
-  int rc = target_data_begin(Device, arg_num, args_base, args, arg_sizes,
-      arg_types, arg_mappers, nullptr);
+  int rc = targetDataBegin(Device, arg_num, args_base, args, arg_sizes,
+                           arg_types, arg_mappers, nullptr);
   HandleTargetOutcome(rc == OFFLOAD_SUCCESS);
 }
 
@@ -207,8 +207,8 @@ EXTERN void __tgt_target_data_end_mapper(int64_t device_id, int32_t arg_num,
   }
 #endif
 
-  int rc = target_data_end(Device, arg_num, args_base, args, arg_sizes,
-      arg_types, arg_mappers, nullptr);
+  int rc = targetDataEnd(Device, arg_num, args_base, args, arg_sizes, arg_types,
+                         arg_mappers, nullptr);
   HandleTargetOutcome(rc == OFFLOAD_SUCCESS);
 }
 
diff --git a/openmp/libomptarget/src/omptarget.cpp b/openmp/libomptarget/src/omptarget.cpp
index aa2bebdebd80f..d057c88bad10b 100644
--- a/openmp/libomptarget/src/omptarget.cpp
+++ b/openmp/libomptarget/src/omptarget.cpp
@@ -210,7 +210,7 @@ int CheckDeviceAndCtors(int64_t device_id) {
   return OFFLOAD_SUCCESS;
 }
 
-static int32_t member_of(int64_t type) {
+static int32_t getParentIndex(int64_t type) {
   return ((type & OMP_TGT_MAPTYPE_MEMBER_OF) >> 48) - 1;
 }
 
@@ -251,9 +251,9 @@ int target_data_mapper(DeviceTy &Device, void *arg_base,
 }
 
 /// Internal function to do the mapping and transfer the data to the device
-int target_data_begin(DeviceTy &Device, int32_t arg_num, void **args_base,
-                      void **args, int64_t *arg_sizes, int64_t *arg_types,
-                      void **arg_mappers, __tgt_async_info *async_info_ptr) {
+int targetDataBegin(DeviceTy &Device, int32_t arg_num, void **args_base,
+                    void **args, int64_t *arg_sizes, int64_t *arg_types,
+                    void **arg_mappers, __tgt_async_info *async_info_ptr) {
   // process each input.
   for (int32_t i = 0; i < arg_num; ++i) {
     // Ignore private variables and arrays - there is no mapping for them.
@@ -262,17 +262,18 @@ int target_data_begin(DeviceTy &Device, int32_t arg_num, void **args_base,
       continue;
 
     if (arg_mappers && arg_mappers[i]) {
-      // Instead of executing the regular path of target_data_begin, call the
-      // target_data_mapper variant which will call target_data_begin again
+      // Instead of executing the regular path of targetDataBegin, call the
+      // target_data_mapper variant which will call targetDataBegin again
       // with new arguments.
       DP("Calling target_data_mapper for the %dth argument\n", i);
 
-      int rc = target_data_mapper(Device, args_base[i], args[i], arg_sizes[i],
-          arg_types[i], arg_mappers[i], target_data_begin);
+      int rc =
+          target_data_mapper(Device, args_base[i], args[i], arg_sizes[i],
+                             arg_types[i], arg_mappers[i], targetDataBegin);
 
       if (rc != OFFLOAD_SUCCESS) {
-        DP("Call to target_data_begin via target_data_mapper for custom mapper"
-            " failed.\n");
+        DP("Call to targetDataBegin via target_data_mapper for custom mapper"
+           " failed.\n");
         return OFFLOAD_FAIL;
       }
 
@@ -289,8 +290,8 @@ int target_data_begin(DeviceTy &Device, int32_t arg_num, void **args_base,
     // is a combined entry.
     int64_t padding = 0;
     const int next_i = i+1;
-    if (member_of(arg_types[i]) < 0 && next_i < arg_num &&
-        member_of(arg_types[next_i]) == i) {
+    if (getParentIndex(arg_types[i]) < 0 && next_i < arg_num &&
+        getParentIndex(arg_types[next_i]) == i) {
       padding = (int64_t)HstPtrBegin % alignment;
       if (padding) {
         DP("Using a padding of %" PRId64 " bytes for begin address " DPxMOD
@@ -301,7 +302,7 @@ int target_data_begin(DeviceTy &Device, int32_t arg_num, void **args_base,
     }
 
     // Address of pointer on the host and device, respectively.
-    void *Pointer_HstPtrBegin, *Pointer_TgtPtrBegin;
+    void *Pointer_HstPtrBegin, *PointerTgtPtrBegin;
     bool IsNew, Pointer_IsNew;
     bool IsHostPtr = false;
     bool IsImplicit = arg_types[i] & OMP_TGT_MAPTYPE_IMPLICIT;
@@ -325,22 +326,22 @@ int target_data_begin(DeviceTy &Device, int32_t arg_num, void **args_base,
       //
       // The map entry for s comes first, and the PTR_AND_OBJ entry comes
       // afterward, so the pointer is already allocated by the time the
-      // PTR_AND_OBJ entry is handled below, and Pointer_TgtPtrBegin is thus
+      // PTR_AND_OBJ entry is handled below, and PointerTgtPtrBegin is thus
       // non-null.  However, "declare target link" can produce a PTR_AND_OBJ
       // entry for a global that might not already be allocated by the time the
       // PTR_AND_OBJ entry is handled below, and so the allocation might fail
       // when HasPresentModifier.
-      Pointer_TgtPtrBegin = Device.getOrAllocTgtPtr(
+      PointerTgtPtrBegin = Device.getOrAllocTgtPtr(
           HstPtrBase, HstPtrBase, sizeof(void *), Pointer_IsNew, IsHostPtr,
           IsImplicit, UpdateRef, HasCloseModifier, HasPresentModifier);
-      if (!Pointer_TgtPtrBegin) {
+      if (!PointerTgtPtrBegin) {
         DP("Call to getOrAllocTgtPtr returned null pointer (%s).\n",
            HasPresentModifier ? "'present' map type modifier"
                               : "device failure or illegal mapping");
         return OFFLOAD_FAIL;
       }
       DP("There are %zu bytes allocated at target address " DPxMOD " - is%s new"
-          "\n", sizeof(void *), DPxPTR(Pointer_TgtPtrBegin),
+          "\n", sizeof(void *), DPxPTR(PointerTgtPtrBegin),
           (Pointer_IsNew ? "" : " not"));
       Pointer_HstPtrBegin = HstPtrBase;
       // modify current entry.
@@ -378,7 +379,7 @@ int target_data_begin(DeviceTy &Device, int32_t arg_num, void **args_base,
           copy = true;
         } else if (arg_types[i] & OMP_TGT_MAPTYPE_MEMBER_OF) {
           // Copy data only if the "parent" struct has RefCount==1.
-          int32_t parent_idx = member_of(arg_types[i]);
+          int32_t parent_idx = getParentIndex(arg_types[i]);
           uint64_t parent_rc = Device.getMapEntryRefCnt(args[parent_idx]);
           assert(parent_rc > 0 && "parent struct not found");
           if (parent_rc == 1) {
@@ -390,8 +391,8 @@ int target_data_begin(DeviceTy &Device, int32_t arg_num, void **args_base,
       if (copy && !IsHostPtr) {
         DP("Moving %" PRId64 " bytes (hst:" DPxMOD ") -> (tgt:" DPxMOD ")\n",
            data_size, DPxPTR(HstPtrBegin), DPxPTR(TgtPtrBegin));
-        int rt = Device.data_submit(TgtPtrBegin, HstPtrBegin, data_size,
-                                    async_info_ptr);
+        int rt = Device.submitData(TgtPtrBegin, HstPtrBegin, data_size,
+                                   async_info_ptr);
         if (rt != OFFLOAD_SUCCESS) {
           DP("Copying data to device failed.\n");
           return OFFLOAD_FAIL;
@@ -401,19 +402,19 @@ int target_data_begin(DeviceTy &Device, int32_t arg_num, void **args_base,
 
     if (arg_types[i] & OMP_TGT_MAPTYPE_PTR_AND_OBJ && !IsHostPtr) {
       DP("Update pointer (" DPxMOD ") -> [" DPxMOD "]\n",
-          DPxPTR(Pointer_TgtPtrBegin), DPxPTR(TgtPtrBegin));
+         DPxPTR(PointerTgtPtrBegin), DPxPTR(TgtPtrBegin));
       uint64_t Delta = (uint64_t)HstPtrBegin - (uint64_t)HstPtrBase;
       void *TgtPtrBase = (void *)((uint64_t)TgtPtrBegin - Delta);
-      int rt = Device.data_submit(Pointer_TgtPtrBegin, &TgtPtrBase,
-                                  sizeof(void *), async_info_ptr);
+      int rt = Device.submitData(PointerTgtPtrBegin, &TgtPtrBase,
+                                 sizeof(void *), async_info_ptr);
       if (rt != OFFLOAD_SUCCESS) {
         DP("Copying data to device failed.\n");
         return OFFLOAD_FAIL;
       }
       // create shadow pointers for this entry
       Device.ShadowMtx.lock();
-      Device.ShadowPtrMap[Pointer_HstPtrBegin] = {HstPtrBase,
-          Pointer_TgtPtrBegin, TgtPtrBase};
+      Device.ShadowPtrMap[Pointer_HstPtrBegin] = {
+          HstPtrBase, PointerTgtPtrBegin, TgtPtrBase};
       Device.ShadowMtx.unlock();
     }
   }
@@ -422,9 +423,9 @@ int target_data_begin(DeviceTy &Device, int32_t arg_num, void **args_base,
 }
 
 /// Internal function to undo the mapping and retrieve the data from the device.
-int target_data_end(DeviceTy &Device, int32_t arg_num, void **args_base,
-                    void **args, int64_t *arg_sizes, int64_t *arg_types,
-                    void **arg_mappers, __tgt_async_info *async_info_ptr) {
+int targetDataEnd(DeviceTy &Device, int32_t arg_num, void **args_base,
+                  void **args, int64_t *arg_sizes, int64_t *arg_types,
+                  void **arg_mappers, __tgt_async_info *async_info_ptr) {
   // process each input.
   for (int32_t i = arg_num - 1; i >= 0; --i) {
     // Ignore private variables and arrays - there is no mapping for them.
@@ -434,17 +435,17 @@ int target_data_end(DeviceTy &Device, int32_t arg_num, void **args_base,
       continue;
 
     if (arg_mappers && arg_mappers[i]) {
-      // Instead of executing the regular path of target_data_end, call the
-      // target_data_mapper variant which will call target_data_end again
+      // Instead of executing the regular path of targetDataEnd, call the
+      // target_data_mapper variant which will call targetDataEnd again
       // with new arguments.
       DP("Calling target_data_mapper for the %dth argument\n", i);
 
       int rc = target_data_mapper(Device, args_base[i], args[i], arg_sizes[i],
-          arg_types[i], arg_mappers[i], target_data_end);
+                                  arg_types[i], arg_mappers[i], targetDataEnd);
 
       if (rc != OFFLOAD_SUCCESS) {
-        DP("Call to target_data_end via target_data_mapper for custom mapper"
-            " failed.\n");
+        DP("Call to targetDataEnd via target_data_mapper for custom mapper"
+           " failed.\n");
         return OFFLOAD_FAIL;
       }
 
@@ -459,8 +460,8 @@ int target_data_end(DeviceTy &Device, int32_t arg_num, void **args_base,
     // is a combined entry.
     int64_t padding = 0;
     const int next_i = i+1;
-    if (member_of(arg_types[i]) < 0 && next_i < arg_num &&
-        member_of(arg_types[next_i]) == i) {
+    if (getParentIndex(arg_types[i]) < 0 && next_i < arg_num &&
+        getParentIndex(arg_types[next_i]) == i) {
       padding = (int64_t)HstPtrBegin % alignment;
       if (padding) {
         DP("Using a padding of %" PRId64 " bytes for begin address " DPxMOD
@@ -514,7 +515,7 @@ int target_data_end(DeviceTy &Device, int32_t arg_num, void **args_base,
           if ((arg_types[i] & OMP_TGT_MAPTYPE_MEMBER_OF) &&
               !(arg_types[i] & OMP_TGT_MAPTYPE_PTR_AND_OBJ)) {
             // Copy data only if the "parent" struct has RefCount==1.
-            int32_t parent_idx = member_of(arg_types[i]);
+            int32_t parent_idx = getParentIndex(arg_types[i]);
             uint64_t parent_rc = Device.getMapEntryRefCnt(args[parent_idx]);
             assert(parent_rc > 0 && "parent struct not found");
             if (parent_rc == 1) {
@@ -591,7 +592,7 @@ int target_data_end(DeviceTy &Device, int32_t arg_num, void **args_base,
 
 /// Internal function to pass data to/from the target.
 // async_info_ptr is currently unused, added here so target_data_update has the
-// same signature as target_data_begin and target_data_end.
+// same signature as targetDataBegin and targetDataEnd.
 int target_data_update(DeviceTy &Device, int32_t arg_num,
     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types,
     void **arg_mappers, __tgt_async_info *async_info_ptr) {
@@ -673,7 +674,7 @@ int target_data_update(DeviceTy &Device, int32_t arg_num,
     if (arg_types[i] & OMP_TGT_MAPTYPE_TO) {
       DP("Moving %" PRId64 " bytes (hst:" DPxMOD ") -> (tgt:" DPxMOD ")\n",
           arg_sizes[i], DPxPTR(HstPtrBegin), DPxPTR(TgtPtrBegin));
-      int rt = Device.data_submit(TgtPtrBegin, HstPtrBegin, MapSize, nullptr);
+      int rt = Device.submitData(TgtPtrBegin, HstPtrBegin, MapSize, nullptr);
       if (rt != OFFLOAD_SUCCESS) {
         DP("Copying data to device failed.\n");
         return OFFLOAD_FAIL;
@@ -684,16 +685,16 @@ int target_data_update(DeviceTy &Device, int32_t arg_num,
       Device.ShadowMtx.lock();
       for (ShadowPtrListTy::iterator it = Device.ShadowPtrMap.begin();
           it != Device.ShadowPtrMap.end(); ++it) {
-        void **ShadowHstPtrAddr = (void**) it->first;
-        if ((uintptr_t) ShadowHstPtrAddr < lb)
+        void **ShadowHstPtrAddr = (void **)it->first;
+        if ((uintptr_t)ShadowHstPtrAddr < lb)
           continue;
-        if ((uintptr_t) ShadowHstPtrAddr >= ub)
+        if ((uintptr_t)ShadowHstPtrAddr >= ub)
           break;
         DP("Restoring original target pointer value " DPxMOD " for target "
-            "pointer " DPxMOD "\n", DPxPTR(it->second.TgtPtrVal),
-            DPxPTR(it->second.TgtPtrAddr));
-        rt = Device.data_submit(it->second.TgtPtrAddr,
-            &it->second.TgtPtrVal, sizeof(void *), nullptr);
+           "pointer " DPxMOD "\n",
+           DPxPTR(it->second.TgtPtrVal), DPxPTR(it->second.TgtPtrAddr));
+        rt = Device.submitData(it->second.TgtPtrAddr, &it->second.TgtPtrVal,
+                               sizeof(void *), nullptr);
         if (rt != OFFLOAD_SUCCESS) {
           DP("Copying data to device failed.\n");
           Device.ShadowMtx.unlock();
@@ -719,39 +720,38 @@ static bool isLambdaMapping(int64_t Mapping) {
 /// performs the same action as data_update and data_end above. This function
 /// returns 0 if it was able to transfer the execution to a target and an
 /// integer different from zero otherwise.
-int target(int64_t device_id, void *host_ptr, int32_t arg_num,
-    void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types,
-    void **arg_mappers, int32_t team_num, int32_t thread_limit,
-    int IsTeamConstruct) {
-  DeviceTy &Device = Devices[device_id];
+int target(int64_t DeviceId, void *HostPtr, int32_t ArgNum, void **ArgBases,
+           void **Args, int64_t *ArgSizes, int64_t *ArgTypes, void **ArgMappers,
+           int32_t TeamNum, int32_t ThreadLimit, int IsTeamConstruct) {
+  DeviceTy &Device = Devices[DeviceId];
 
   // Find the table information in the map or look it up in the translation
   // tables.
   TableMap *TM = 0;
   TblMapMtx->lock();
-  HostPtrToTableMapTy::iterator TableMapIt = HostPtrToTableMap->find(host_ptr);
+  HostPtrToTableMapTy::iterator TableMapIt = HostPtrToTableMap->find(HostPtr);
   if (TableMapIt == HostPtrToTableMap->end()) {
     // We don't have a map. So search all the registered libraries.
     TrlTblMtx->lock();
     for (HostEntriesBeginToTransTableTy::iterator
-             ii = HostEntriesBeginToTransTable->begin(),
-             ie = HostEntriesBeginToTransTable->end();
-         !TM && ii != ie; ++ii) {
+             II = HostEntriesBeginToTransTable->begin(),
+             IE = HostEntriesBeginToTransTable->end();
+         !TM && II != IE; ++II) {
       // get the translation table (which contains all the good info).
-      TranslationTable *TransTable = &ii->second;
+      TranslationTable *TransTable = &II->second;
       // iterate over all the host table entries to see if we can locate the
       // host_ptr.
-      __tgt_offload_entry *begin = TransTable->HostTable.EntriesBegin;
-      __tgt_offload_entry *end = TransTable->HostTable.EntriesEnd;
-      __tgt_offload_entry *cur = begin;
-      for (uint32_t i = 0; cur < end; ++cur, ++i) {
-        if (cur->addr != host_ptr)
+      __tgt_offload_entry *Begin = TransTable->HostTable.EntriesBegin;
+      __tgt_offload_entry *End = TransTable->HostTable.EntriesEnd;
+      __tgt_offload_entry *Cur = Begin;
+      for (uint32_t I = 0; Cur < End; ++Cur, ++I) {
+        if (Cur->addr != HostPtr)
           continue;
         // we got a match, now fill the HostPtrToTableMap so that we
         // may avoid this search next time.
-        TM = &(*HostPtrToTableMap)[host_ptr];
+        TM = &(*HostPtrToTableMap)[HostPtr];
         TM->Table = TransTable;
-        TM->Index = i;
+        TM->Index = I;
         break;
       }
     }
@@ -764,60 +764,59 @@ int target(int64_t device_id, void *host_ptr, int32_t arg_num,
   // No map for this host pointer found!
   if (!TM) {
     DP("Host ptr " DPxMOD " does not have a matching target pointer.\n",
-       DPxPTR(host_ptr));
+       DPxPTR(HostPtr));
     return OFFLOAD_FAIL;
   }
 
   // get target table.
   TrlTblMtx->lock();
-  assert(TM->Table->TargetsTable.size() > (size_t)device_id &&
+  assert(TM->Table->TargetsTable.size() > (size_t)DeviceId &&
          "Not expecting a device ID outside the table's bounds!");
-  __tgt_target_table *TargetTable = TM->Table->TargetsTable[device_id];
+  __tgt_target_table *TargetTable = TM->Table->TargetsTable[DeviceId];
   TrlTblMtx->unlock();
   assert(TargetTable && "Global data has not been mapped\n");
 
   __tgt_async_info AsyncInfo;
 
   // Move data to device.
-  int rc = target_data_begin(Device, arg_num, args_base, args, arg_sizes,
-                             arg_types, arg_mappers, &AsyncInfo);
-  if (rc != OFFLOAD_SUCCESS) {
-    DP("Call to target_data_begin failed, abort target.\n");
+  int Ret = targetDataBegin(Device, ArgNum, ArgBases, Args, ArgSizes, ArgTypes,
+                            ArgMappers, &AsyncInfo);
+  if (Ret != OFFLOAD_SUCCESS) {
+    DP("Call to targetDataBegin failed, abort target.\n");
     return OFFLOAD_FAIL;
   }
 
-  std::vector<void *> tgt_args;
-  std::vector<ptrdiff_t> tgt_offsets;
+  std::vector<void *> TgtArgs;
+  std::vector<ptrdiff_t> TgtOffsets;
 
   // List of (first-)private arrays allocated for this target region
-  std::vector<void *> fpArrays;
-  std::vector<int> tgtArgsPositions(arg_num, -1);
+  std::vector<void *> FPArrays;
+  std::vector<int> TgtArgsPositions(ArgNum, -1);
 
-  for (int32_t i = 0; i < arg_num; ++i) {
-    if (!(arg_types[i] & OMP_TGT_MAPTYPE_TARGET_PARAM)) {
-      // This is not a target parameter, do not push it into tgt_args.
+  for (int32_t I = 0; I < ArgNum; ++I) {
+    if (!(ArgTypes[I] & OMP_TGT_MAPTYPE_TARGET_PARAM)) {
+      // This is not a target parameter, do not push it into TgtArgs.
       // Check for lambda mapping.
-      if (isLambdaMapping(arg_types[i])) {
-        assert((arg_types[i] & OMP_TGT_MAPTYPE_MEMBER_OF) &&
+      if (isLambdaMapping(ArgTypes[I])) {
+        assert((ArgTypes[I] & OMP_TGT_MAPTYPE_MEMBER_OF) &&
                "PTR_AND_OBJ must be also MEMBER_OF.");
-        unsigned idx = member_of(arg_types[i]);
-        int tgtIdx = tgtArgsPositions[idx];
-        assert(tgtIdx != -1 && "Base address must be translated already.");
+        unsigned Idx = getParentIndex(ArgTypes[I]);
+        int TgtIdx = TgtArgsPositions[Idx];
+        assert(TgtIdx != -1 && "Base address must be translated already.");
         // The parent lambda must be processed already and it must be the last
-        // in tgt_args and tgt_offsets arrays.
-        void *HstPtrVal = args[i];
-        void *HstPtrBegin = args_base[i];
-        void *HstPtrBase = args[idx];
+        // in TgtArgs and TgtOffsets arrays.
+        void *HstPtrVal = Args[I];
+        void *HstPtrBegin = ArgBases[I];
+        void *HstPtrBase = Args[Idx];
         bool IsLast, IsHostPtr; // unused.
         void *TgtPtrBase =
-            (void *)((intptr_t)tgt_args[tgtIdx] + tgt_offsets[tgtIdx]);
+            (void *)((intptr_t)TgtArgs[TgtIdx] + TgtOffsets[TgtIdx]);
         DP("Parent lambda base " DPxMOD "\n", DPxPTR(TgtPtrBase));
         uint64_t Delta = (uint64_t)HstPtrBegin - (uint64_t)HstPtrBase;
         void *TgtPtrBegin = (void *)((uintptr_t)TgtPtrBase + Delta);
-        void *Pointer_TgtPtrBegin =
-            Device.getTgtPtrBegin(HstPtrVal, arg_sizes[i], IsLast, false,
-                                  IsHostPtr);
-        if (!Pointer_TgtPtrBegin) {
+        void *PointerTgtPtrBegin = Device.getTgtPtrBegin(
+            HstPtrVal, ArgSizes[I], IsLast, false, IsHostPtr);
+        if (!PointerTgtPtrBegin) {
           DP("No lambda captured variable mapped (" DPxMOD ") - ignored\n",
              DPxPTR(HstPtrVal));
           continue;
@@ -825,127 +824,128 @@ int target(int64_t device_id, void *host_ptr, int32_t arg_num,
         if (RTLs->RequiresFlags & OMP_REQ_UNIFIED_SHARED_MEMORY &&
             TgtPtrBegin == HstPtrBegin) {
           DP("Unified memory is active, no need to map lambda captured"
-             "variable (" DPxMOD ")\n", DPxPTR(HstPtrVal));
+             "variable (" DPxMOD ")\n",
+             DPxPTR(HstPtrVal));
           continue;
         }
         DP("Update lambda reference (" DPxMOD ") -> [" DPxMOD "]\n",
-           DPxPTR(Pointer_TgtPtrBegin), DPxPTR(TgtPtrBegin));
-        int rt = Device.data_submit(TgtPtrBegin, &Pointer_TgtPtrBegin,
-                                    sizeof(void *), &AsyncInfo);
-        if (rt != OFFLOAD_SUCCESS) {
+           DPxPTR(PointerTgtPtrBegin), DPxPTR(TgtPtrBegin));
+        Ret = Device.submitData(TgtPtrBegin, &PointerTgtPtrBegin,
+                                sizeof(void *), &AsyncInfo);
+        if (Ret != OFFLOAD_SUCCESS) {
           DP("Copying data to device failed.\n");
           return OFFLOAD_FAIL;
         }
       }
       continue;
     }
-    void *HstPtrBegin = args[i];
-    void *HstPtrBase = args_base[i];
+    void *HstPtrBegin = Args[I];
+    void *HstPtrBase = ArgBases[I];
     void *TgtPtrBegin;
     ptrdiff_t TgtBaseOffset;
     bool IsLast, IsHostPtr; // unused.
-    if (arg_types[i] & OMP_TGT_MAPTYPE_LITERAL) {
+    if (ArgTypes[I] & OMP_TGT_MAPTYPE_LITERAL) {
       DP("Forwarding first-private value " DPxMOD " to the target construct\n",
-          DPxPTR(HstPtrBase));
+         DPxPTR(HstPtrBase));
       TgtPtrBegin = HstPtrBase;
       TgtBaseOffset = 0;
-    } else if (arg_types[i] & OMP_TGT_MAPTYPE_PRIVATE) {
+    } else if (ArgTypes[I] & OMP_TGT_MAPTYPE_PRIVATE) {
       // Allocate memory for (first-)private array
-      TgtPtrBegin = Device.data_alloc(arg_sizes[i], HstPtrBegin);
+      TgtPtrBegin = Device.allocData(ArgSizes[I], HstPtrBegin);
       if (!TgtPtrBegin) {
-        DP ("Data allocation for %sprivate array " DPxMOD " failed, "
-            "abort target.\n",
-            (arg_types[i] & OMP_TGT_MAPTYPE_TO ? "first-" : ""),
-            DPxPTR(HstPtrBegin));
+        DP("Data allocation for %sprivate array " DPxMOD " failed, "
+           "abort target.\n",
+           (ArgTypes[I] & OMP_TGT_MAPTYPE_TO ? "first-" : ""),
+           DPxPTR(HstPtrBegin));
         return OFFLOAD_FAIL;
       }
-      fpArrays.push_back(TgtPtrBegin);
+      FPArrays.push_back(TgtPtrBegin);
       TgtBaseOffset = (intptr_t)HstPtrBase - (intptr_t)HstPtrBegin;
 #ifdef OMPTARGET_DEBUG
       void *TgtPtrBase = (void *)((intptr_t)TgtPtrBegin + TgtBaseOffset);
       DP("Allocated %" PRId64 " bytes of target memory at " DPxMOD " for "
-          "%sprivate array " DPxMOD " - pushing target argument " DPxMOD "\n",
-          arg_sizes[i], DPxPTR(TgtPtrBegin),
-          (arg_types[i] & OMP_TGT_MAPTYPE_TO ? "first-" : ""),
-          DPxPTR(HstPtrBegin), DPxPTR(TgtPtrBase));
+         "%sprivate array " DPxMOD " - pushing target argument " DPxMOD "\n",
+         ArgSizes[I], DPxPTR(TgtPtrBegin),
+         (ArgTypes[I] & OMP_TGT_MAPTYPE_TO ? "first-" : ""),
+         DPxPTR(HstPtrBegin), DPxPTR(TgtPtrBase));
 #endif
       // If first-private, copy data from host
-      if (arg_types[i] & OMP_TGT_MAPTYPE_TO) {
-        int rt = Device.data_submit(TgtPtrBegin, HstPtrBegin, arg_sizes[i],
-                                    &AsyncInfo);
-        if (rt != OFFLOAD_SUCCESS) {
+      if (ArgTypes[I] & OMP_TGT_MAPTYPE_TO) {
+        Ret = Device.submitData(TgtPtrBegin, HstPtrBegin, ArgSizes[I],
+                                &AsyncInfo);
+        if (Ret != OFFLOAD_SUCCESS) {
           DP("Copying data to device failed, failed.\n");
           return OFFLOAD_FAIL;
         }
       }
-    } else if (arg_types[i] & OMP_TGT_MAPTYPE_PTR_AND_OBJ) {
+    } else if (ArgTypes[I] & OMP_TGT_MAPTYPE_PTR_AND_OBJ) {
       TgtPtrBegin = Device.getTgtPtrBegin(HstPtrBase, sizeof(void *), IsLast,
-          false, IsHostPtr);
+                                          false, IsHostPtr);
       TgtBaseOffset = 0; // no offset for ptrs.
       DP("Obtained target argument " DPxMOD " from host pointer " DPxMOD " to "
-         "object " DPxMOD "\n", DPxPTR(TgtPtrBegin), DPxPTR(HstPtrBase),
-         DPxPTR(HstPtrBase));
+         "object " DPxMOD "\n",
+         DPxPTR(TgtPtrBegin), DPxPTR(HstPtrBase), DPxPTR(HstPtrBase));
     } else {
-      TgtPtrBegin = Device.getTgtPtrBegin(HstPtrBegin, arg_sizes[i], IsLast,
-          false, IsHostPtr);
+      TgtPtrBegin = Device.getTgtPtrBegin(HstPtrBegin, ArgSizes[I], IsLast,
+                                          false, IsHostPtr);
       TgtBaseOffset = (intptr_t)HstPtrBase - (intptr_t)HstPtrBegin;
 #ifdef OMPTARGET_DEBUG
       void *TgtPtrBase = (void *)((intptr_t)TgtPtrBegin + TgtBaseOffset);
       DP("Obtained target argument " DPxMOD " from host pointer " DPxMOD "\n",
-          DPxPTR(TgtPtrBase), DPxPTR(HstPtrBegin));
+         DPxPTR(TgtPtrBase), DPxPTR(HstPtrBegin));
 #endif
     }
-    tgtArgsPositions[i] = tgt_args.size();
-    tgt_args.push_back(TgtPtrBegin);
-    tgt_offsets.push_back(TgtBaseOffset);
+    TgtArgsPositions[I] = TgtArgs.size();
+    TgtArgs.push_back(TgtPtrBegin);
+    TgtOffsets.push_back(TgtBaseOffset);
   }
 
-  assert(tgt_args.size() == tgt_offsets.size() &&
-      "Size mismatch in arguments and offsets");
+  assert(TgtArgs.size() == TgtOffsets.size() &&
+         "Size mismatch in arguments and offsets");
 
   // Pop loop trip count
-  uint64_t ltc = 0;
+  uint64_t LoopTripCount = 0;
   TblMapMtx->lock();
   auto I = Device.LoopTripCnt.find(__kmpc_global_thread_num(NULL));
   if (I != Device.LoopTripCnt.end()) {
-    ltc = I->second;
+    LoopTripCount = I->second;
     Device.LoopTripCnt.erase(I);
-    DP("loop trip count is %lu.\n", ltc);
+    DP("loop trip count is %lu.\n", LoopTripCount);
   }
   TblMapMtx->unlock();
 
   // Launch device execution.
   DP("Launching target execution %s with pointer " DPxMOD " (index=%d).\n",
-      TargetTable->EntriesBegin[TM->Index].name,
-      DPxPTR(TargetTable->EntriesBegin[TM->Index].addr), TM->Index);
+     TargetTable->EntriesBegin[TM->Index].name,
+     DPxPTR(TargetTable->EntriesBegin[TM->Index].addr), TM->Index);
   if (IsTeamConstruct) {
-    rc = Device.run_team_region(TargetTable->EntriesBegin[TM->Index].addr,
-                                &tgt_args[0], &tgt_offsets[0], tgt_args.size(),
-                                team_num, thread_limit, ltc, &AsyncInfo);
+    Ret = Device.runTeamRegion(TargetTable->EntriesBegin[TM->Index].addr,
+                               &TgtArgs[0], &TgtOffsets[0], TgtArgs.size(),
+                               TeamNum, ThreadLimit, LoopTripCount, &AsyncInfo);
   } else {
-    rc = Device.run_region(TargetTable->EntriesBegin[TM->Index].addr,
-                           &tgt_args[0], &tgt_offsets[0], tgt_args.size(),
-                           &AsyncInfo);
+    Ret =
+        Device.runRegion(TargetTable->EntriesBegin[TM->Index].addr, &TgtArgs[0],
+                         &TgtOffsets[0], TgtArgs.size(), &AsyncInfo);
   }
-  if (rc != OFFLOAD_SUCCESS) {
-    DP ("Executing target region abort target.\n");
+  if (Ret != OFFLOAD_SUCCESS) {
+    DP("Executing target region abort target.\n");
     return OFFLOAD_FAIL;
   }
 
   // Deallocate (first-)private arrays
-  for (auto it : fpArrays) {
-    int rt = Device.data_delete(it);
-    if (rt != OFFLOAD_SUCCESS) {
+  for (auto Itr : FPArrays) {
+    Ret = Device.deleteData(Itr);
+    if (Ret != OFFLOAD_SUCCESS) {
       DP("Deallocation of (first-)private arrays failed.\n");
       return OFFLOAD_FAIL;
     }
   }
 
   // Move data from device.
-  int rt = target_data_end(Device, arg_num, args_base, args, arg_sizes,
-                           arg_types, arg_mappers, &AsyncInfo);
-  if (rt != OFFLOAD_SUCCESS) {
-    DP("Call to target_data_end failed, abort targe.\n");
+  Ret = targetDataEnd(Device, ArgNum, ArgBases, Args, ArgSizes, ArgTypes,
+                      ArgMappers, &AsyncInfo);
+  if (Ret != OFFLOAD_SUCCESS) {
+    DP("Call to targetDataEnd failed, abort targe.\n");
     return OFFLOAD_FAIL;
   }
 
diff --git a/openmp/libomptarget/src/private.h b/openmp/libomptarget/src/private.h
index ba2f161c7927a..41da68f24d595 100644
--- a/openmp/libomptarget/src/private.h
+++ b/openmp/libomptarget/src/private.h
@@ -17,26 +17,25 @@
 
 #include <cstdint>
 
-extern int target_data_begin(DeviceTy &Device, int32_t arg_num,
-                             void **args_base, void **args, int64_t *arg_sizes,
-                             int64_t *arg_types, void **arg_mappers,
-                             __tgt_async_info *async_info_ptr);
-
-extern int target_data_end(DeviceTy &Device, int32_t arg_num, void **args_base,
+extern int targetDataBegin(DeviceTy &Device, int32_t arg_num, void **args_base,
                            void **args, int64_t *arg_sizes, int64_t *arg_types,
                            void **arg_mappers,
                            __tgt_async_info *async_info_ptr);
 
+extern int targetDataEnd(DeviceTy &Device, int32_t arg_num, void **args_base,
+                         void **args, int64_t *arg_sizes, int64_t *arg_types,
+                         void **arg_mappers, __tgt_async_info *async_info_ptr);
+
 extern int target_data_update(DeviceTy &Device, int32_t arg_num,
                               void **args_base, void **args,
                               int64_t *arg_sizes, int64_t *arg_types,
                               void **arg_mappers,
                               __tgt_async_info *async_info_ptr = nullptr);
 
-extern int target(int64_t device_id, void *host_ptr, int32_t arg_num,
-                  void **args_base, void **args, int64_t *arg_sizes,
-                  int64_t *arg_types, void **arg_mappers, int32_t team_num,
-                  int32_t thread_limit, int IsTeamConstruct);
+extern int target(int64_t DeviceId, void *HostPtr, int32_t ArgNum,
+                  void **ArgBases, void **Args, int64_t *ArgSizes,
+                  int64_t *ArgTypes, void **ArgMappers, int32_t TeamNum,
+                  int32_t ThreadLimit, int IsTeamConstruct);
 
 extern int CheckDeviceAndCtors(int64_t device_id);
 
@@ -74,10 +73,11 @@ struct MapperComponentsTy {
 //                                           size_t size, int64_t type);
 typedef void (*MapperFuncPtrTy)(void *, void *, void *, int64_t, int64_t);
 
-// Function pointer type for target_data_* functions (target_data_begin,
-// target_data_end and target_data_update).
+// Function pointer type for target_data_* functions (targetDataBegin,
+// targetDataEnd and target_data_update).
 typedef int (*TargetDataFuncPtrTy)(DeviceTy &, int32_t, void **, void **,
-    int64_t *, int64_t *, void **, __tgt_async_info *);
+                                   int64_t *, int64_t *, void **,
+                                   __tgt_async_info *);
 
 ////////////////////////////////////////////////////////////////////////////////
 // implementation for messages

From b3e63aa8a47078e8ac24ceffa7160cc1af657dcb Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Thu, 23 Jul 2020 18:05:16 -0400
Subject: [PATCH 0397/1035] AMDGPU: Don't assume there is only one terminator
 copy

This would stop on the first in reverse order, failing the verifier if
there were more earlier in the block.
---
 .../Target/AMDGPU/SIOptimizeExecMasking.cpp   | 22 +++++++---
 ...ize-exec-masking-strip-terminator-bits.mir | 44 +++++++++++++++++++
 2 files changed, 60 insertions(+), 6 deletions(-)
 create mode 100644 llvm/test/CodeGen/AMDGPU/optimize-exec-masking-strip-terminator-bits.mir

diff --git a/llvm/lib/Target/AMDGPU/SIOptimizeExecMasking.cpp b/llvm/lib/Target/AMDGPU/SIOptimizeExecMasking.cpp
index a9717c6ffb705..2994d1954d710 100644
--- a/llvm/lib/Target/AMDGPU/SIOptimizeExecMasking.cpp
+++ b/llvm/lib/Target/AMDGPU/SIOptimizeExecMasking.cpp
@@ -176,8 +176,7 @@ static unsigned getSaveExecOp(unsigned Opc) {
 }
 
 // These are only terminators to get correct spill code placement during
-// register allocation, so turn them back into normal instructions. Only one of
-// these is expected per block.
+// register allocation, so turn them back into normal instructions.
 static bool removeTerminatorBit(const SIInstrInfo &TII, MachineInstr &MI) {
   switch (MI.getOpcode()) {
   case AMDGPU::S_MOV_B64_term:
@@ -220,19 +219,29 @@ static bool removeTerminatorBit(const SIInstrInfo &TII, MachineInstr &MI) {
   }
 }
 
+// Turn all pseudoterminators in the block into their equivalent non-terminator
+// instructions. Returns the reverse iterator to the first non-terminator
+// instruction in the block.
 static MachineBasicBlock::reverse_iterator fixTerminators(
   const SIInstrInfo &TII,
   MachineBasicBlock &MBB) {
   MachineBasicBlock::reverse_iterator I = MBB.rbegin(), E = MBB.rend();
+
+  bool Seen = false;
+  MachineBasicBlock::reverse_iterator FirstNonTerm = I;
   for (; I != E; ++I) {
     if (!I->isTerminator())
-      return I;
+      return Seen ? FirstNonTerm : I;
 
-    if (removeTerminatorBit(TII, *I))
-      return I;
+    if (removeTerminatorBit(TII, *I)) {
+      if (!Seen) {
+        FirstNonTerm = I;
+        Seen = true;
+      }
+    }
   }
 
-  return E;
+  return FirstNonTerm;
 }
 
 static MachineBasicBlock::reverse_iterator findExecCopy(
@@ -291,6 +300,7 @@ bool SIOptimizeExecMasking::runOnMachineFunction(MachineFunction &MF) {
     if (I == E)
       continue;
 
+    // TODO: It's possible to see other terminator copies after the exec copy.
     Register CopyToExec = isCopyToExec(*I, ST);
     if (!CopyToExec.isValid())
       continue;
diff --git a/llvm/test/CodeGen/AMDGPU/optimize-exec-masking-strip-terminator-bits.mir b/llvm/test/CodeGen/AMDGPU/optimize-exec-masking-strip-terminator-bits.mir
new file mode 100644
index 0000000000000..3e939bd476041
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/optimize-exec-masking-strip-terminator-bits.mir
@@ -0,0 +1,44 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs -run-pass=si-optimize-exec-masking -o - %s | FileCheck %s
+
+# There are multiple _term pseudos here. Starting from the end of the
+# block, they all need to be converted to regular copies removed in
+# order to avoid making the verifier unhappy.
+
+---
+name:  multi_term_pseudos
+tracksRegLiveness: true
+body:             |
+  ; CHECK-LABEL: name: multi_term_pseudos
+  ; CHECK: bb.0:
+  ; CHECK:   successors: %bb.2(0x40000000), %bb.1(0x40000000)
+  ; CHECK:   liveins: $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9
+  ; CHECK:   $exec = COPY killed renamable $sgpr4_sgpr5
+  ; CHECK:   renamable $sgpr10_sgpr11 = COPY killed renamable $sgpr6_sgpr7, implicit $exec
+  ; CHECK:   renamable $sgpr12_sgpr13 = COPY killed renamable $sgpr8_sgpr9, implicit $exec
+  ; CHECK:   S_CBRANCH_EXECZ %bb.1, implicit $exec
+  ; CHECK:   S_BRANCH %bb.2
+  ; CHECK: bb.1:
+  ; CHECK:   liveins: $sgpr12_sgpr13
+  ; CHECK:   S_ENDPGM 0, implicit $sgpr12_sgpr13
+  ; CHECK: bb.2:
+  ; CHECK:   liveins: $sgpr12_sgpr13
+  ; CHECK:   S_ENDPGM 0, implicit $sgpr12_sgpr13
+  bb.0:
+    successors: %bb.2(0x40000000), %bb.1(0x40000000)
+    liveins: $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9
+    $exec = S_MOV_B64_term killed renamable $sgpr4_sgpr5
+    renamable $sgpr10_sgpr11 = S_MOV_B64_term killed renamable $sgpr6_sgpr7, implicit $exec
+    renamable $sgpr12_sgpr13 = S_MOV_B64_term killed renamable $sgpr8_sgpr9, implicit $exec
+    S_CBRANCH_EXECZ %bb.1, implicit $exec
+    S_BRANCH %bb.2
+
+  bb.1:
+    liveins: $sgpr12_sgpr13
+    S_ENDPGM 0, implicit $sgpr12_sgpr13
+
+  bb.2:
+    liveins: $sgpr12_sgpr13
+    S_ENDPGM 0, implicit $sgpr12_sgpr13
+
+...

From 8860daf0ed248960c3bc26208b7a38b3e1aadb1d Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Tue, 21 Jul 2020 21:43:52 -0400
Subject: [PATCH 0398/1035] AMDGPU: Handle a few missing cases in
 getAddrModeArguments

---
 llvm/lib/Target/AMDGPU/SIISelLowering.cpp     |  6 +-
 .../AMDGPU/cgp-addressing-modes-gfx1030.ll    | 75 ++++++++++++++++++
 .../AMDGPU/cgp-addressing-modes-gfx908.ll     | 78 +++++++++++++++++++
 .../CodeGen/AMDGPU/cgp-addressing-modes.ll    | 55 +++++++++++++
 4 files changed, 213 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/CodeGen/AMDGPU/cgp-addressing-modes-gfx1030.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/cgp-addressing-modes-gfx908.ll

diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index c92065f7495a2..4c9fce92f9a26 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -1175,9 +1175,13 @@ bool SITargetLowering::getAddrModeArguments(IntrinsicInst *II,
   case Intrinsic::amdgcn_atomic_dec:
   case Intrinsic::amdgcn_ds_ordered_add:
   case Intrinsic::amdgcn_ds_ordered_swap:
+  case Intrinsic::amdgcn_ds_append:
+  case Intrinsic::amdgcn_ds_consume:
   case Intrinsic::amdgcn_ds_fadd:
   case Intrinsic::amdgcn_ds_fmin:
-  case Intrinsic::amdgcn_ds_fmax: {
+  case Intrinsic::amdgcn_ds_fmax:
+  case Intrinsic::amdgcn_global_atomic_fadd:
+  case Intrinsic::amdgcn_global_atomic_csub: {
     Value *Ptr = II->getArgOperand(0);
     AccessTy = II->getType();
     Ops.push_back(Ptr);
diff --git a/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes-gfx1030.ll b/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes-gfx1030.ll
new file mode 100644
index 0000000000000..0f655dadfa11d
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes-gfx1030.ll
@@ -0,0 +1,75 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: opt -S -codegenprepare -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1030 < %s | FileCheck -check-prefix=OPT %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1030 < %s | FileCheck -check-prefix=GCN %s
+
+; Make sure we match the addressing mode offset of csub intrinsics across blocks.
+
+define amdgpu_kernel void @test_sink_small_offset_global_atomic_csub_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) {
+; OPT-LABEL: @test_sink_small_offset_global_atomic_csub_i32(
+; OPT-NEXT:  entry:
+; OPT-NEXT:    [[OUT_GEP:%.*]] = getelementptr i32, i32 addrspace(1)* [[OUT:%.*]], i32 999999
+; OPT-NEXT:    [[TID:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #3
+; OPT-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TID]], 0
+; OPT-NEXT:    br i1 [[CMP]], label [[ENDIF:%.*]], label [[IF:%.*]]
+; OPT:       if:
+; OPT-NEXT:    [[TMP0:%.*]] = bitcast i32 addrspace(1)* [[IN:%.*]] to i8 addrspace(1)*
+; OPT-NEXT:    [[SUNKADDR:%.*]] = getelementptr i8, i8 addrspace(1)* [[TMP0]], i64 28
+; OPT-NEXT:    [[TMP1:%.*]] = bitcast i8 addrspace(1)* [[SUNKADDR]] to i32 addrspace(1)*
+; OPT-NEXT:    [[VAL:%.*]] = call i32 @llvm.amdgcn.global.atomic.csub.p1i32(i32 addrspace(1)* [[TMP1]], i32 2)
+; OPT-NEXT:    br label [[ENDIF]]
+; OPT:       endif:
+; OPT-NEXT:    [[X:%.*]] = phi i32 [ [[VAL]], [[IF]] ], [ 0, [[ENTRY:%.*]] ]
+; OPT-NEXT:    store i32 [[X]], i32 addrspace(1)* [[OUT_GEP]], align 4
+; OPT-NEXT:    ret void
+;
+; GCN-LABEL: test_sink_small_offset_global_atomic_csub_i32:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GCN-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, -1, 0
+; GCN-NEXT:    ; implicit-def: $vcc_hi
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
+; GCN-NEXT:    v_mov_b32_e32 v0, 0
+; GCN-NEXT:    s_and_saveexec_b32 s4, vcc_lo
+; GCN-NEXT:    s_cbranch_execz BB0_2
+; GCN-NEXT:  ; %bb.1: ; %if
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-NEXT:    v_mov_b32_e32 v2, 2
+; GCN-NEXT:    global_atomic_csub v0, v[0:1], v2, off offset:28 glc
+; GCN-NEXT:  BB0_2: ; %endif
+; GCN-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_add_co_u32_e64 v1, s0, 0x3d0800, s0
+; GCN-NEXT:    v_add_co_ci_u32_e64 v2, s0, 0, s1, s0
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    global_store_dword v[1:2], v0, off offset:252
+; GCN-NEXT:    s_endpgm
+entry:
+  %out.gep = getelementptr i32, i32 addrspace(1)* %out, i32 999999
+  %in.gep = getelementptr i32, i32 addrspace(1)* %in, i32 7
+  %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0
+  %cmp = icmp eq i32 %tid, 0
+  br i1 %cmp, label %endif, label %if
+
+if:
+  %val = call i32 @llvm.amdgcn.global.atomic.csub.p1i32(i32 addrspace(1)* %in.gep, i32 2)
+  br label %endif
+
+endif:
+  %x = phi i32 [ %val, %if ], [ 0, %entry ]
+  store i32 %x, i32 addrspace(1)* %out.gep
+  br label %done
+
+done:
+  ret void
+}
+
+declare i32 @llvm.amdgcn.global.atomic.csub.p1i32(i32 addrspace(1)* nocapture, i32) #0
+declare i32 @llvm.amdgcn.mbcnt.lo(i32, i32) #1
+declare void @llvm.amdgcn.global.atomic.fadd.p1f32.f32(float addrspace(1)* nocapture, float) #2
+
+attributes #0 = { argmemonly nounwind }
+attributes #1 = { nounwind readnone willreturn }
+attributes #2 = { argmemonly nounwind willreturn }
diff --git a/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes-gfx908.ll b/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes-gfx908.ll
new file mode 100644
index 0000000000000..840a4ec3dac8f
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes-gfx908.ll
@@ -0,0 +1,78 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: opt -S -codegenprepare -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 < %s | FileCheck -check-prefix=OPT %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 < %s | FileCheck -check-prefix=GCN %s
+
+; Make sure we match the addressing mode offset of globla.atomic.fadd intrinsics across blocks.
+
+define amdgpu_kernel void @test_sink_small_offset_global_atomic_fadd_f32(float addrspace(1)* %out, float addrspace(1)* %in) {
+; OPT-LABEL: @test_sink_small_offset_global_atomic_fadd_f32(
+; OPT-NEXT:  entry:
+; OPT-NEXT:    [[OUT_GEP:%.*]] = getelementptr float, float addrspace(1)* [[OUT:%.*]], i32 999999
+; OPT-NEXT:    [[TID:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #3
+; OPT-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TID]], 0
+; OPT-NEXT:    br i1 [[CMP]], label [[ENDIF:%.*]], label [[IF:%.*]]
+; OPT:       if:
+; OPT-NEXT:    [[TMP0:%.*]] = bitcast float addrspace(1)* [[IN:%.*]] to i8 addrspace(1)*
+; OPT-NEXT:    [[SUNKADDR:%.*]] = getelementptr i8, i8 addrspace(1)* [[TMP0]], i64 28
+; OPT-NEXT:    [[TMP1:%.*]] = bitcast i8 addrspace(1)* [[SUNKADDR]] to float addrspace(1)*
+; OPT-NEXT:    call void @llvm.amdgcn.global.atomic.fadd.p1f32.f32(float addrspace(1)* [[TMP1]], float 2.000000e+00)
+; OPT-NEXT:    [[VAL:%.*]] = load volatile float, float addrspace(1)* undef, align 4
+; OPT-NEXT:    br label [[ENDIF]]
+; OPT:       endif:
+; OPT-NEXT:    [[X:%.*]] = phi float [ [[VAL]], [[IF]] ], [ 0.000000e+00, [[ENTRY:%.*]] ]
+; OPT-NEXT:    store float [[X]], float addrspace(1)* [[OUT_GEP]], align 4
+; OPT-NEXT:    ret void
+;
+; GCN-LABEL: test_sink_small_offset_global_atomic_fadd_f32:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GCN-NEXT:    v_mbcnt_lo_u32_b32 v0, -1, 0
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GCN-NEXT:    v_mov_b32_e32 v0, 0
+; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT:    s_cbranch_execz BB0_2
+; GCN-NEXT:  ; %bb.1: ; %if
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-NEXT:    v_mov_b32_e32 v2, 2.0
+; GCN-NEXT:    global_atomic_add_f32 v[0:1], v2, off offset:28
+; GCN-NEXT:    global_load_dword v0, v[0:1], off
+; GCN-NEXT:  BB0_2: ; %endif
+; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v1, s0
+; GCN-NEXT:    v_add_co_u32_e32 v1, vcc, 0x3d0000, v1
+; GCN-NEXT:    v_mov_b32_e32 v2, s1
+; GCN-NEXT:    v_addc_co_u32_e32 v2, vcc, 0, v2, vcc
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    global_store_dword v[1:2], v0, off offset:2300
+; GCN-NEXT:    s_endpgm
+entry:
+  %out.gep = getelementptr float, float addrspace(1)* %out, i32 999999
+  %in.gep = getelementptr float, float addrspace(1)* %in, i32 7
+  %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0
+  %cmp = icmp eq i32 %tid, 0
+  br i1 %cmp, label %endif, label %if
+
+if:
+  call void @llvm.amdgcn.global.atomic.fadd.p1f32.f32(float addrspace(1)* %in.gep, float 2.0)
+  %val = load volatile float, float addrspace(1)* undef
+  br label %endif
+
+endif:
+  %x = phi float [ %val, %if ], [ 0.0, %entry ]
+  store float %x, float addrspace(1)* %out.gep
+  br label %done
+
+done:
+  ret void
+}
+
+declare i32 @llvm.amdgcn.mbcnt.lo(i32, i32) #1
+declare void @llvm.amdgcn.global.atomic.fadd.p1f32.f32(float addrspace(1)* nocapture, float) #2
+
+attributes #0 = { argmemonly nounwind }
+attributes #1 = { nounwind readnone willreturn }
+attributes #2 = { argmemonly nounwind willreturn }
diff --git a/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes.ll b/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes.ll
index a2fb893af5404..bb621b294bbc1 100644
--- a/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes.ll
+++ b/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes.ll
@@ -748,10 +748,65 @@ done:
   ret void
 }
 
+; OPT-LABEL: @test_sink_small_offset_ds_append(
+; OPT: %0 = bitcast i32 addrspace(3)* %in to i8 addrspace(3)*
+; OPT: %sunkaddr = getelementptr i8, i8 addrspace(3)* %0, i32 28
+; OPT: %1 = bitcast i8 addrspace(3)* %sunkaddr to i32 addrspace(3)*
+; OPT: %tmp1 = call i32 @llvm.amdgcn.ds.append.p3i32(i32 addrspace(3)* %1, i1 false)
+define amdgpu_kernel void @test_sink_small_offset_ds_append(i32 addrspace(3)* %out, i32 addrspace(3)* %in) {
+entry:
+  %out.gep = getelementptr i32, i32 addrspace(3)* %out, i32 999999
+  %in.gep = getelementptr i32, i32 addrspace(3)* %in, i32 7
+  %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0
+  %tmp0 = icmp eq i32 %tid, 0
+  br i1 %tmp0, label %endif, label %if
+
+if:
+  %tmp1 = call i32 @llvm.amdgcn.ds.append.p3i32(i32 addrspace(3)* %in.gep, i1 false)
+  br label %endif
+
+endif:
+  %x = phi i32 [ %tmp1, %if ], [ 0, %entry ]
+  store i32 %x, i32 addrspace(3)* %out.gep
+  br label %done
+
+done:
+  ret void
+}
+
+; OPT-LABEL: @test_sink_small_offset_ds_consume(
+; OPT: %0 = bitcast i32 addrspace(3)* %in to i8 addrspace(3)*
+; OPT: %sunkaddr = getelementptr i8, i8 addrspace(3)* %0, i32 28
+; OPT: %1 = bitcast i8 addrspace(3)* %sunkaddr to i32 addrspace(3)*
+; OPT: %tmp1 = call i32 @llvm.amdgcn.ds.consume.p3i32(i32 addrspace(3)* %1, i1 false)
+define amdgpu_kernel void @test_sink_small_offset_ds_consume(i32 addrspace(3)* %out, i32 addrspace(3)* %in) {
+entry:
+  %out.gep = getelementptr i32, i32 addrspace(3)* %out, i32 999999
+  %in.gep = getelementptr i32, i32 addrspace(3)* %in, i32 7
+  %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0
+  %tmp0 = icmp eq i32 %tid, 0
+  br i1 %tmp0, label %endif, label %if
+
+if:
+  %tmp1 = call i32 @llvm.amdgcn.ds.consume.p3i32(i32 addrspace(3)* %in.gep, i1 false)
+  br label %endif
+
+endif:
+  %x = phi i32 [ %tmp1, %if ], [ 0, %entry ]
+  store i32 %x, i32 addrspace(3)* %out.gep
+  br label %done
+
+done:
+  ret void
+}
+
 declare i32 @llvm.amdgcn.mbcnt.lo(i32, i32) #0
 declare i32 @llvm.amdgcn.atomic.inc.i32.p3i32(i32 addrspace(3)* nocapture, i32, i32, i32, i1) #2
 declare i32 @llvm.amdgcn.atomic.dec.i32.p3i32(i32 addrspace(3)* nocapture, i32, i32, i32, i1) #2
+declare i32 @llvm.amdgcn.ds.append.p3i32(i32 addrspace(3)* nocapture, i1 immarg) #3
+declare i32 @llvm.amdgcn.ds.consume.p3i32(i32 addrspace(3)* nocapture, i1 immarg) #3
 
 attributes #0 = { nounwind readnone }
 attributes #1 = { nounwind }
 attributes #2 = { nounwind argmemonly }
+attributes #3 = { argmemonly convergent nounwind willreturn }

From 65564e5eaf143cb3c43907cc546d76e2b20e59f3 Mon Sep 17 00:00:00 2001
From: "Joel E. Denny" <jdenny.ornl@gmail.com>
Date: Tue, 28 Jul 2020 20:27:48 -0400
Subject: [PATCH 0399/1035] Revert "[OpenMP] Implement TR8 `present` motion
 modifier in runtime (2/2)"

This reverts commit 2cb926a447d22166dac0d6e6dceaf5b3616ef6a1.

It depends on 3c3faae497046be706df29e16c9fbccb7e1fce09, which is being
reverted.
---
 openmp/libomptarget/src/omptarget.cpp         |  6 --
 .../test/mapping/present/target_update.c      | 73 -------------------
 2 files changed, 79 deletions(-)
 delete mode 100644 openmp/libomptarget/test/mapping/present/target_update.c

diff --git a/openmp/libomptarget/src/omptarget.cpp b/openmp/libomptarget/src/omptarget.cpp
index d057c88bad10b..f109c6acc8ca3 100644
--- a/openmp/libomptarget/src/omptarget.cpp
+++ b/openmp/libomptarget/src/omptarget.cpp
@@ -628,12 +628,6 @@ int target_data_update(DeviceTy &Device, int32_t arg_num,
         false, IsHostPtr);
     if (!TgtPtrBegin) {
       DP("hst data:" DPxMOD " not found, becomes a noop\n", DPxPTR(HstPtrBegin));
-      if (arg_types[i] & OMP_TGT_MAPTYPE_PRESENT) {
-        MESSAGE("device mapping required by 'present' motion modifier does not "
-                "exist for host address " DPxMOD " (%ld bytes)",
-                DPxPTR(HstPtrBegin), MapSize);
-        return OFFLOAD_FAIL;
-      }
       continue;
     }
 
diff --git a/openmp/libomptarget/test/mapping/present/target_update.c b/openmp/libomptarget/test/mapping/present/target_update.c
deleted file mode 100644
index 3a3575a27d015..0000000000000
--- a/openmp/libomptarget/test/mapping/present/target_update.c
+++ /dev/null
@@ -1,73 +0,0 @@
-// --------------------------------------------------
-// Check 'to'
-// --------------------------------------------------
-
-// RUN: %libomptarget-compile-aarch64-unknown-linux-gnu \
-// RUN:   -fopenmp-version=51 -DCLAUSE=to
-// RUN: %libomptarget-run-fail-aarch64-unknown-linux-gnu 2>&1 \
-// RUN: | %fcheck-aarch64-unknown-linux-gnu
-
-// RUN: %libomptarget-compile-powerpc64-ibm-linux-gnu \
-// RUN:   -fopenmp-version=51 -DCLAUSE=to
-// RUN: %libomptarget-run-fail-powerpc64-ibm-linux-gnu 2>&1 \
-// RUN: | %fcheck-powerpc64-ibm-linux-gnu
-
-// RUN: %libomptarget-compile-powerpc64le-ibm-linux-gnu \
-// RUN:   -fopenmp-version=51 -DCLAUSE=to
-// RUN: %libomptarget-run-fail-powerpc64le-ibm-linux-gnu 2>&1 \
-// RUN: | %fcheck-powerpc64le-ibm-linux-gnu
-
-// RUN: %libomptarget-compile-x86_64-pc-linux-gnu \
-// RUN:   -fopenmp-version=51 -DCLAUSE=to
-// RUN: %libomptarget-run-fail-x86_64-pc-linux-gnu 2>&1 \
-// RUN: | %fcheck-x86_64-pc-linux-gnu
-
-// --------------------------------------------------
-// Check 'from'
-// --------------------------------------------------
-
-// RUN: %libomptarget-compile-aarch64-unknown-linux-gnu \
-// RUN:   -fopenmp-version=51 -DCLAUSE=from
-// RUN: %libomptarget-run-fail-aarch64-unknown-linux-gnu 2>&1 \
-// RUN: | %fcheck-aarch64-unknown-linux-gnu
-
-// RUN: %libomptarget-compile-powerpc64-ibm-linux-gnu \
-// RUN:   -fopenmp-version=51 -DCLAUSE=from
-// RUN: %libomptarget-run-fail-powerpc64-ibm-linux-gnu 2>&1 \
-// RUN: | %fcheck-powerpc64-ibm-linux-gnu
-
-// RUN: %libomptarget-compile-powerpc64le-ibm-linux-gnu \
-// RUN:   -fopenmp-version=51 -DCLAUSE=from
-// RUN: %libomptarget-run-fail-powerpc64le-ibm-linux-gnu 2>&1 \
-// RUN: | %fcheck-powerpc64le-ibm-linux-gnu
-
-// RUN: %libomptarget-compile-x86_64-pc-linux-gnu \
-// RUN:   -fopenmp-version=51 -DCLAUSE=from
-// RUN: %libomptarget-run-fail-x86_64-pc-linux-gnu 2>&1 \
-// RUN: | %fcheck-x86_64-pc-linux-gnu
-
-#include <stdio.h>
-
-int main() {
-  int i;
-
-  // CHECK: addr=0x[[#%x,HOST_ADDR:]], size=[[#%u,SIZE:]]
-  fprintf(stderr, "addr=%p, size=%ld\n", &i, sizeof i);
-
-  // CHECK-NOT: Libomptarget
-#pragma omp target enter data map(alloc: i)
-#pragma omp target update CLAUSE(present: i)
-#pragma omp target exit data map(delete: i)
-
-  // CHECK: i is present
-  fprintf(stderr, "i is present\n");
-
-  // CHECK: Libomptarget message: device mapping required by 'present' motion modifier does not exist for host address 0x{{0*}}[[#HOST_ADDR]] ([[#SIZE]] bytes)
-  // CHECK: Libomptarget fatal error 1: failure of target construct while offloading is mandatory
-#pragma omp target update CLAUSE(present: i)
-
-  // CHECK-NOT: i is present
-  fprintf(stderr, "i is present\n");
-
-  return 0;
-}

From 69fc33f0cd130b02a38d2fc582afc7b0fcd6458a Mon Sep 17 00:00:00 2001
From: "Joel E. Denny" <jdenny.ornl@gmail.com>
Date: Tue, 28 Jul 2020 20:26:56 -0400
Subject: [PATCH 0400/1035] Revert "[OpenMP] Implement TR8 `present` motion
 modifier in Clang (1/2)"

This reverts commit 3c3faae497046be706df29e16c9fbccb7e1fce09.

It breaks a number of bots.
---
 clang/include/clang/AST/OpenMPClause.h        | 187 +-------------
 .../clang/Basic/DiagnosticSemaKinds.td        |   2 -
 clang/include/clang/Basic/OpenMPKinds.def     |   1 -
 clang/include/clang/Basic/OpenMPKinds.h       |   4 -
 clang/include/clang/Parse/Parser.h            |   3 -
 clang/include/clang/Sema/Sema.h               |  23 +-
 clang/lib/AST/OpenMPClause.cpp                |  94 ++++---
 clang/lib/Basic/OpenMPKinds.cpp               |   8 +-
 clang/lib/CodeGen/CGOpenMPRuntime.cpp         |  22 +-
 clang/lib/Parse/ParseOpenMP.cpp               |  45 +---
 clang/lib/Sema/SemaOpenMP.cpp                 |  78 ++----
 clang/lib/Sema/TreeTransform.h                |  42 ++--
 clang/lib/Serialization/ASTReader.cpp         |  12 -
 clang/lib/Serialization/ASTWriter.cpp         |  10 -
 clang/test/OpenMP/declare_mapper_ast_print.c  |  14 --
 clang/test/OpenMP/declare_mapper_codegen.cpp  | 238 ------------------
 clang/test/OpenMP/target_update_ast_print.cpp |  24 --
 clang/test/OpenMP/target_update_codegen.cpp   |  63 -----
 clang/test/OpenMP/target_update_messages.cpp  | 117 +--------
 19 files changed, 119 insertions(+), 868 deletions(-)

diff --git a/clang/include/clang/AST/OpenMPClause.h b/clang/include/clang/AST/OpenMPClause.h
index 46c26c6c77de2..c649502f765b4 100644
--- a/clang/include/clang/AST/OpenMPClause.h
+++ b/clang/include/clang/AST/OpenMPClause.h
@@ -6329,20 +6329,8 @@ class OMPToClause final : public OMPMappableExprListClause<OMPToClause>,
   friend OMPVarListClause;
   friend TrailingObjects;
 
-  /// Motion-modifiers for the 'to' clause.
-  OpenMPMotionModifierKind MotionModifiers[NumberOfOMPMotionModifiers] = {
-      OMPC_MOTION_MODIFIER_unknown, OMPC_MOTION_MODIFIER_unknown};
-
-  /// Location of motion-modifiers for the 'to' clause.
-  SourceLocation MotionModifiersLoc[NumberOfOMPMotionModifiers];
-
-  /// Colon location.
-  SourceLocation ColonLoc;
-
   /// Build clause with number of variables \a NumVars.
   ///
-  /// \param TheMotionModifiers Motion-modifiers.
-  /// \param TheMotionModifiersLoc Locations of motion-modifiers.
   /// \param MapperQualifierLoc C++ nested name specifier for the associated
   /// user-defined mapper.
   /// \param MapperIdInfo The identifier of associated user-defined mapper.
@@ -6354,24 +6342,13 @@ class OMPToClause final : public OMPMappableExprListClause<OMPToClause>,
   /// NumUniqueDeclarations: number of unique base declarations in this clause;
   /// 3) NumComponentLists: number of component lists in this clause; and 4)
   /// NumComponents: total number of expression components in the clause.
-  explicit OMPToClause(ArrayRef<OpenMPMotionModifierKind> TheMotionModifiers,
-                       ArrayRef<SourceLocation> TheMotionModifiersLoc,
-                       NestedNameSpecifierLoc MapperQualifierLoc,
+  explicit OMPToClause(NestedNameSpecifierLoc MapperQualifierLoc,
                        DeclarationNameInfo MapperIdInfo,
                        const OMPVarListLocTy &Locs,
                        const OMPMappableExprListSizeTy &Sizes)
       : OMPMappableExprListClause(llvm::omp::OMPC_to, Locs, Sizes,
                                   /*SupportsMapper=*/true, &MapperQualifierLoc,
-                                  &MapperIdInfo) {
-    assert(llvm::array_lengthof(MotionModifiers) == TheMotionModifiers.size() &&
-           "Unexpected number of motion modifiers.");
-    llvm::copy(TheMotionModifiers, std::begin(MotionModifiers));
-
-    assert(llvm::array_lengthof(MotionModifiersLoc) ==
-               TheMotionModifiersLoc.size() &&
-           "Unexpected number of motion modifier locations.");
-    llvm::copy(TheMotionModifiersLoc, std::begin(MotionModifiersLoc));
-  }
+                                  &MapperIdInfo) {}
 
   /// Build an empty clause.
   ///
@@ -6384,29 +6361,6 @@ class OMPToClause final : public OMPMappableExprListClause<OMPToClause>,
       : OMPMappableExprListClause(llvm::omp::OMPC_to, OMPVarListLocTy(), Sizes,
                                   /*SupportsMapper=*/true) {}
 
-  /// Set motion-modifier for the clause.
-  ///
-  /// \param I index for motion-modifier.
-  /// \param T motion-modifier for the clause.
-  void setMotionModifier(unsigned I, OpenMPMotionModifierKind T) {
-    assert(I < NumberOfOMPMotionModifiers &&
-           "Unexpected index to store motion modifier, exceeds array size.");
-    MotionModifiers[I] = T;
-  }
-
-  /// Set location for the motion-modifier.
-  ///
-  /// \param I index for motion-modifier location.
-  /// \param TLoc motion-modifier location.
-  void setMotionModifierLoc(unsigned I, SourceLocation TLoc) {
-    assert(I < NumberOfOMPMotionModifiers &&
-           "Index to store motion modifier location exceeds array size.");
-    MotionModifiersLoc[I] = TLoc;
-  }
-
-  /// Set colon location.
-  void setColonLoc(SourceLocation Loc) { ColonLoc = Loc; }
-
   /// Define the sizes of each trailing object array except the last one. This
   /// is required for TrailingObjects to work properly.
   size_t numTrailingObjects(OverloadToken<Expr *>) const {
@@ -6431,8 +6385,6 @@ class OMPToClause final : public OMPMappableExprListClause<OMPToClause>,
   /// \param Vars The original expression used in the clause.
   /// \param Declarations Declarations used in the clause.
   /// \param ComponentLists Component lists used in the clause.
-  /// \param MotionModifiers Motion-modifiers.
-  /// \param MotionModifiersLoc Location of motion-modifiers.
   /// \param UDMapperRefs References to user-defined mappers associated with
   /// expressions used in the clause.
   /// \param UDMQualifierLoc C++ nested name specifier for the associated
@@ -6443,8 +6395,6 @@ class OMPToClause final : public OMPMappableExprListClause<OMPToClause>,
                              ArrayRef<ValueDecl *> Declarations,
                              MappableExprComponentListsRef ComponentLists,
                              ArrayRef<Expr *> UDMapperRefs,
-                             ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
-                             ArrayRef<SourceLocation> MotionModifiersLoc,
                              NestedNameSpecifierLoc UDMQualifierLoc,
                              DeclarationNameInfo MapperId);
 
@@ -6459,38 +6409,6 @@ class OMPToClause final : public OMPMappableExprListClause<OMPToClause>,
   static OMPToClause *CreateEmpty(const ASTContext &C,
                                   const OMPMappableExprListSizeTy &Sizes);
 
-  /// Fetches the motion-modifier at 'Cnt' index of array of modifiers.
-  ///
-  /// \param Cnt index for motion-modifier.
-  OpenMPMotionModifierKind getMotionModifier(unsigned Cnt) const LLVM_READONLY {
-    assert(Cnt < NumberOfOMPMotionModifiers &&
-           "Requested modifier exceeds the total number of modifiers.");
-    return MotionModifiers[Cnt];
-  }
-
-  /// Fetches the motion-modifier location at 'Cnt' index of array of modifiers'
-  /// locations.
-  ///
-  /// \param Cnt index for motion-modifier location.
-  SourceLocation getMotionModifierLoc(unsigned Cnt) const LLVM_READONLY {
-    assert(Cnt < NumberOfOMPMotionModifiers &&
-           "Requested modifier location exceeds total number of modifiers.");
-    return MotionModifiersLoc[Cnt];
-  }
-
-  /// Fetches ArrayRef of motion-modifiers.
-  ArrayRef<OpenMPMotionModifierKind> getMotionModifiers() const LLVM_READONLY {
-    return llvm::makeArrayRef(MotionModifiers);
-  }
-
-  /// Fetches ArrayRef of location of motion-modifiers.
-  ArrayRef<SourceLocation> getMotionModifiersLoc() const LLVM_READONLY {
-    return llvm::makeArrayRef(MotionModifiersLoc);
-  }
-
-  /// Get colon location.
-  SourceLocation getColonLoc() const { return ColonLoc; }
-
   child_range children() {
     return child_range(reinterpret_cast<Stmt **>(varlist_begin()),
                        reinterpret_cast<Stmt **>(varlist_end()));
@@ -6531,20 +6449,8 @@ class OMPFromClause final
   friend OMPVarListClause;
   friend TrailingObjects;
 
-  /// Motion-modifiers for the 'from' clause.
-  OpenMPMotionModifierKind MotionModifiers[NumberOfOMPMotionModifiers] = {
-      OMPC_MOTION_MODIFIER_unknown, OMPC_MOTION_MODIFIER_unknown};
-
-  /// Location of motion-modifiers for the 'from' clause.
-  SourceLocation MotionModifiersLoc[NumberOfOMPMotionModifiers];
-
-  /// Colon location.
-  SourceLocation ColonLoc;
-
   /// Build clause with number of variables \a NumVars.
   ///
-  /// \param TheMotionModifiers Motion-modifiers.
-  /// \param TheMotionModifiersLoc Locations of motion-modifiers.
   /// \param MapperQualifierLoc C++ nested name specifier for the associated
   /// user-defined mapper.
   /// \param MapperIdInfo The identifier of associated user-defined mapper.
@@ -6556,24 +6462,13 @@ class OMPFromClause final
   /// NumUniqueDeclarations: number of unique base declarations in this clause;
   /// 3) NumComponentLists: number of component lists in this clause; and 4)
   /// NumComponents: total number of expression components in the clause.
-  explicit OMPFromClause(ArrayRef<OpenMPMotionModifierKind> TheMotionModifiers,
-                         ArrayRef<SourceLocation> TheMotionModifiersLoc,
-                         NestedNameSpecifierLoc MapperQualifierLoc,
+  explicit OMPFromClause(NestedNameSpecifierLoc MapperQualifierLoc,
                          DeclarationNameInfo MapperIdInfo,
                          const OMPVarListLocTy &Locs,
                          const OMPMappableExprListSizeTy &Sizes)
       : OMPMappableExprListClause(llvm::omp::OMPC_from, Locs, Sizes,
                                   /*SupportsMapper=*/true, &MapperQualifierLoc,
-                                  &MapperIdInfo) {
-    assert(llvm::array_lengthof(MotionModifiers) == TheMotionModifiers.size() &&
-           "Unexpected number of motion modifiers.");
-    llvm::copy(TheMotionModifiers, std::begin(MotionModifiers));
-
-    assert(llvm::array_lengthof(MotionModifiersLoc) ==
-               TheMotionModifiersLoc.size() &&
-           "Unexpected number of motion modifier locations.");
-    llvm::copy(TheMotionModifiersLoc, std::begin(MotionModifiersLoc));
-  }
+                                  &MapperIdInfo) {}
 
   /// Build an empty clause.
   ///
@@ -6586,29 +6481,6 @@ class OMPFromClause final
       : OMPMappableExprListClause(llvm::omp::OMPC_from, OMPVarListLocTy(),
                                   Sizes, /*SupportsMapper=*/true) {}
 
-  /// Set motion-modifier for the clause.
-  ///
-  /// \param I index for motion-modifier.
-  /// \param T motion-modifier for the clause.
-  void setMotionModifier(unsigned I, OpenMPMotionModifierKind T) {
-    assert(I < NumberOfOMPMotionModifiers &&
-           "Unexpected index to store motion modifier, exceeds array size.");
-    MotionModifiers[I] = T;
-  }
-
-  /// Set location for the motion-modifier.
-  ///
-  /// \param I index for motion-modifier location.
-  /// \param TLoc motion-modifier location.
-  void setMotionModifierLoc(unsigned I, SourceLocation TLoc) {
-    assert(I < NumberOfOMPMotionModifiers &&
-           "Index to store motion modifier location exceeds array size.");
-    MotionModifiersLoc[I] = TLoc;
-  }
-
-  /// Set colon location.
-  void setColonLoc(SourceLocation Loc) { ColonLoc = Loc; }
-
   /// Define the sizes of each trailing object array except the last one. This
   /// is required for TrailingObjects to work properly.
   size_t numTrailingObjects(OverloadToken<Expr *>) const {
@@ -6633,21 +6505,18 @@ class OMPFromClause final
   /// \param Vars The original expression used in the clause.
   /// \param Declarations Declarations used in the clause.
   /// \param ComponentLists Component lists used in the clause.
-  /// \param MotionModifiers Motion-modifiers.
-  /// \param MotionModifiersLoc Location of motion-modifiers.
   /// \param UDMapperRefs References to user-defined mappers associated with
   /// expressions used in the clause.
   /// \param UDMQualifierLoc C++ nested name specifier for the associated
   /// user-defined mapper.
   /// \param MapperId The identifier of associated user-defined mapper.
-  static OMPFromClause *
-  Create(const ASTContext &C, const OMPVarListLocTy &Locs,
-         ArrayRef<Expr *> Vars, ArrayRef<ValueDecl *> Declarations,
-         MappableExprComponentListsRef ComponentLists,
-         ArrayRef<Expr *> UDMapperRefs,
-         ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
-         ArrayRef<SourceLocation> MotionModifiersLoc,
-         NestedNameSpecifierLoc UDMQualifierLoc, DeclarationNameInfo MapperId);
+  static OMPFromClause *Create(const ASTContext &C, const OMPVarListLocTy &Locs,
+                               ArrayRef<Expr *> Vars,
+                               ArrayRef<ValueDecl *> Declarations,
+                               MappableExprComponentListsRef ComponentLists,
+                               ArrayRef<Expr *> UDMapperRefs,
+                               NestedNameSpecifierLoc UDMQualifierLoc,
+                               DeclarationNameInfo MapperId);
 
   /// Creates an empty clause with the place for \a NumVars variables.
   ///
@@ -6660,38 +6529,6 @@ class OMPFromClause final
   static OMPFromClause *CreateEmpty(const ASTContext &C,
                                     const OMPMappableExprListSizeTy &Sizes);
 
-  /// Fetches the motion-modifier at 'Cnt' index of array of modifiers.
-  ///
-  /// \param Cnt index for motion-modifier.
-  OpenMPMotionModifierKind getMotionModifier(unsigned Cnt) const LLVM_READONLY {
-    assert(Cnt < NumberOfOMPMotionModifiers &&
-           "Requested modifier exceeds the total number of modifiers.");
-    return MotionModifiers[Cnt];
-  }
-
-  /// Fetches the motion-modifier location at 'Cnt' index of array of modifiers'
-  /// locations.
-  ///
-  /// \param Cnt index for motion-modifier location.
-  SourceLocation getMotionModifierLoc(unsigned Cnt) const LLVM_READONLY {
-    assert(Cnt < NumberOfOMPMotionModifiers &&
-           "Requested modifier location exceeds total number of modifiers.");
-    return MotionModifiersLoc[Cnt];
-  }
-
-  /// Fetches ArrayRef of motion-modifiers.
-  ArrayRef<OpenMPMotionModifierKind> getMotionModifiers() const LLVM_READONLY {
-    return llvm::makeArrayRef(MotionModifiers);
-  }
-
-  /// Fetches ArrayRef of location of motion-modifiers.
-  ArrayRef<SourceLocation> getMotionModifiersLoc() const LLVM_READONLY {
-    return llvm::makeArrayRef(MotionModifiersLoc);
-  }
-
-  /// Get colon location.
-  SourceLocation getColonLoc() const { return ColonLoc; }
-
   child_range children() {
     return child_range(reinterpret_cast<Stmt **>(varlist_begin()),
                        reinterpret_cast<Stmt **>(varlist_end()));
@@ -7786,8 +7623,6 @@ class OMPClausePrinter final : public OMPClauseVisitor<OMPClausePrinter> {
 
   /// Process clauses with list of variables.
   template <typename T> void VisitOMPClauseList(T *Node, char StartSym);
-  /// Process motion clauses.
-  template <typename T> void VisitOMPMotionClause(T *Node);
 
 public:
   OMPClausePrinter(raw_ostream &OS, const PrintingPolicy &Policy)
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 8a9cada75e351..8093e7ed3fbe1 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -9805,8 +9805,6 @@ def err_omp_multiple_array_items_in_map_clause : Error<
   "multiple array elements associated with the same variable are not allowed in map clauses of the same construct">;
 def err_omp_duplicate_map_type_modifier : Error<
   "same map type modifier has been specified more than once">;
-def err_omp_duplicate_motion_modifier : Error<
-  "same motion modifier has been specified more than once">;
 def err_omp_pointer_mapped_along_with_derived_section : Error<
   "pointer cannot be mapped along with a section derived from itself">;
 def err_omp_original_storage_is_shared_and_does_not_contain : Error<
diff --git a/clang/include/clang/Basic/OpenMPKinds.def b/clang/include/clang/Basic/OpenMPKinds.def
index 59bb331a7a8a6..04ecbeaaa03e2 100644
--- a/clang/include/clang/Basic/OpenMPKinds.def
+++ b/clang/include/clang/Basic/OpenMPKinds.def
@@ -125,7 +125,6 @@ OPENMP_MAP_MODIFIER_KIND(present)
 
 // Modifiers for 'to' or 'from' clause.
 OPENMP_MOTION_MODIFIER_KIND(mapper)
-OPENMP_MOTION_MODIFIER_KIND(present)
 
 // Static attributes for 'dist_schedule' clause.
 OPENMP_DIST_SCHEDULE_KIND(static)
diff --git a/clang/include/clang/Basic/OpenMPKinds.h b/clang/include/clang/Basic/OpenMPKinds.h
index 0f37dc9ad997d..3e9b5c4a8b14a 100644
--- a/clang/include/clang/Basic/OpenMPKinds.h
+++ b/clang/include/clang/Basic/OpenMPKinds.h
@@ -94,10 +94,6 @@ enum OpenMPMotionModifierKind {
   OMPC_MOTION_MODIFIER_unknown
 };
 
-/// Number of allowed motion-modifiers.
-static constexpr unsigned NumberOfOMPMotionModifiers =
-    OMPC_MOTION_MODIFIER_unknown;
-
 /// OpenMP attributes for 'dist_schedule' clause.
 enum OpenMPDistScheduleClauseKind {
 #define OPENMP_DIST_SCHEDULE_KIND(Name) OMPC_DIST_SCHEDULE_##Name,
diff --git a/clang/include/clang/Parse/Parser.h b/clang/include/clang/Parse/Parser.h
index 4068e6a444c9c..e809d87b59a0f 100644
--- a/clang/include/clang/Parse/Parser.h
+++ b/clang/include/clang/Parse/Parser.h
@@ -3221,9 +3221,6 @@ class Parser : public CodeCompletionHandler {
     MapTypeModifiers;
     SmallVector<SourceLocation, NumberOfOMPMapClauseModifiers>
     MapTypeModifiersLoc;
-    SmallVector<OpenMPMotionModifierKind, NumberOfOMPMotionModifiers>
-        MotionModifiers;
-    SmallVector<SourceLocation, NumberOfOMPMotionModifiers> MotionModifiersLoc;
     bool IsMapTypeImplicit = false;
     SourceLocation ExtraModifierLoc;
   };
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 4721ccdf14590..fd4300f563a98 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -10725,9 +10725,7 @@ class Sema final {
       DeclarationNameInfo &ReductionOrMapperId, int ExtraModifier,
       ArrayRef<OpenMPMapModifierKind> MapTypeModifiers,
       ArrayRef<SourceLocation> MapTypeModifiersLoc, bool IsMapTypeImplicit,
-      SourceLocation ExtraModifierLoc,
-      ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
-      ArrayRef<SourceLocation> MotionModifiersLoc);
+      SourceLocation ExtraModifierLoc);
   /// Called on well-formed 'inclusive' clause.
   OMPClause *ActOnOpenMPInclusiveClause(ArrayRef<Expr *> VarList,
                                         SourceLocation StartLoc,
@@ -10864,20 +10862,15 @@ class Sema final {
       SourceLocation KindLoc, SourceLocation EndLoc);
   /// Called on well-formed 'to' clause.
   OMPClause *
-  ActOnOpenMPToClause(ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
-                      ArrayRef<SourceLocation> MotionModifiersLoc,
-                      CXXScopeSpec &MapperIdScopeSpec,
-                      DeclarationNameInfo &MapperId, SourceLocation ColonLoc,
-                      ArrayRef<Expr *> VarList, const OMPVarListLocTy &Locs,
+  ActOnOpenMPToClause(ArrayRef<Expr *> VarList, CXXScopeSpec &MapperIdScopeSpec,
+                      DeclarationNameInfo &MapperId,
+                      const OMPVarListLocTy &Locs,
                       ArrayRef<Expr *> UnresolvedMappers = llvm::None);
   /// Called on well-formed 'from' clause.
-  OMPClause *
-  ActOnOpenMPFromClause(ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
-                        ArrayRef<SourceLocation> MotionModifiersLoc,
-                        CXXScopeSpec &MapperIdScopeSpec,
-                        DeclarationNameInfo &MapperId, SourceLocation ColonLoc,
-                        ArrayRef<Expr *> VarList, const OMPVarListLocTy &Locs,
-                        ArrayRef<Expr *> UnresolvedMappers = llvm::None);
+  OMPClause *ActOnOpenMPFromClause(
+      ArrayRef<Expr *> VarList, CXXScopeSpec &MapperIdScopeSpec,
+      DeclarationNameInfo &MapperId, const OMPVarListLocTy &Locs,
+      ArrayRef<Expr *> UnresolvedMappers = llvm::None);
   /// Called on well-formed 'use_device_ptr' clause.
   OMPClause *ActOnOpenMPUseDevicePtrClause(ArrayRef<Expr *> VarList,
                                            const OMPVarListLocTy &Locs);
diff --git a/clang/lib/AST/OpenMPClause.cpp b/clang/lib/AST/OpenMPClause.cpp
index cf548c2ab4741..6933c57425520 100644
--- a/clang/lib/AST/OpenMPClause.cpp
+++ b/clang/lib/AST/OpenMPClause.cpp
@@ -1097,8 +1097,6 @@ OMPToClause *OMPToClause::Create(
     const ASTContext &C, const OMPVarListLocTy &Locs, ArrayRef<Expr *> Vars,
     ArrayRef<ValueDecl *> Declarations,
     MappableExprComponentListsRef ComponentLists, ArrayRef<Expr *> UDMapperRefs,
-    ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
-    ArrayRef<SourceLocation> MotionModifiersLoc,
     NestedNameSpecifierLoc UDMQualifierLoc, DeclarationNameInfo MapperId) {
   OMPMappableExprListSizeTy Sizes;
   Sizes.NumVars = Vars.size();
@@ -1123,8 +1121,7 @@ OMPToClause *OMPToClause::Create(
           Sizes.NumUniqueDeclarations + Sizes.NumComponentLists,
           Sizes.NumComponents));
 
-  auto *Clause = new (Mem) OMPToClause(MotionModifiers, MotionModifiersLoc,
-                                       UDMQualifierLoc, MapperId, Locs, Sizes);
+  auto *Clause = new (Mem) OMPToClause(UDMQualifierLoc, MapperId, Locs, Sizes);
 
   Clause->setVarRefs(Vars);
   Clause->setUDMapperRefs(UDMapperRefs);
@@ -1147,8 +1144,6 @@ OMPFromClause *OMPFromClause::Create(
     const ASTContext &C, const OMPVarListLocTy &Locs, ArrayRef<Expr *> Vars,
     ArrayRef<ValueDecl *> Declarations,
     MappableExprComponentListsRef ComponentLists, ArrayRef<Expr *> UDMapperRefs,
-    ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
-    ArrayRef<SourceLocation> MotionModifiersLoc,
     NestedNameSpecifierLoc UDMQualifierLoc, DeclarationNameInfo MapperId) {
   OMPMappableExprListSizeTy Sizes;
   Sizes.NumVars = Vars.size();
@@ -1174,8 +1169,7 @@ OMPFromClause *OMPFromClause::Create(
           Sizes.NumComponents));
 
   auto *Clause =
-      new (Mem) OMPFromClause(MotionModifiers, MotionModifiersLoc,
-                              UDMQualifierLoc, MapperId, Locs, Sizes);
+      new (Mem) OMPFromClause(UDMQualifierLoc, MapperId, Locs, Sizes);
 
   Clause->setVarRefs(Vars);
   Clause->setUDMapperRefs(UDMapperRefs);
@@ -1942,17 +1936,6 @@ void OMPClausePrinter::VisitOMPDependClause(OMPDependClause *Node) {
   OS << ")";
 }
 
-template <typename T>
-static void PrintMapper(raw_ostream &OS, T *Node,
-                        const PrintingPolicy &Policy) {
-  OS << '(';
-  NestedNameSpecifier *MapperNNS =
-      Node->getMapperQualifierLoc().getNestedNameSpecifier();
-  if (MapperNNS)
-    MapperNNS->print(OS, Policy);
-  OS << Node->getMapperIdInfo() << ')';
-}
-
 void OMPClausePrinter::VisitOMPMapClause(OMPMapClause *Node) {
   if (!Node->varlist_empty()) {
     OS << "map(";
@@ -1961,8 +1944,14 @@ void OMPClausePrinter::VisitOMPMapClause(OMPMapClause *Node) {
         if (Node->getMapTypeModifier(I) != OMPC_MAP_MODIFIER_unknown) {
           OS << getOpenMPSimpleClauseTypeName(OMPC_map,
                                               Node->getMapTypeModifier(I));
-          if (Node->getMapTypeModifier(I) == OMPC_MAP_MODIFIER_mapper)
-            PrintMapper(OS, Node, Policy);
+          if (Node->getMapTypeModifier(I) == OMPC_MAP_MODIFIER_mapper) {
+            OS << '(';
+            NestedNameSpecifier *MapperNNS =
+                Node->getMapperQualifierLoc().getNestedNameSpecifier();
+            if (MapperNNS)
+              MapperNNS->print(OS, Policy);
+            OS << Node->getMapperIdInfo() << ')';
+          }
           OS << ',';
         }
       }
@@ -1974,41 +1963,44 @@ void OMPClausePrinter::VisitOMPMapClause(OMPMapClause *Node) {
   }
 }
 
-template <typename T> void OMPClausePrinter::VisitOMPMotionClause(T *Node) {
-  if (Node->varlist_empty())
-    return;
-  OS << getOpenMPClauseName(Node->getClauseKind());
-  unsigned ModifierCount = 0;
-  for (unsigned I = 0; I < NumberOfOMPMotionModifiers; ++I) {
-    if (Node->getMotionModifier(I) != OMPC_MOTION_MODIFIER_unknown)
-      ++ModifierCount;
-  }
-  if (ModifierCount) {
-    OS << '(';
-    for (unsigned I = 0; I < NumberOfOMPMotionModifiers; ++I) {
-      if (Node->getMotionModifier(I) != OMPC_MOTION_MODIFIER_unknown) {
-        OS << getOpenMPSimpleClauseTypeName(Node->getClauseKind(),
-                                            Node->getMotionModifier(I));
-        if (Node->getMotionModifier(I) == OMPC_MOTION_MODIFIER_mapper)
-          PrintMapper(OS, Node, Policy);
-        if (I < ModifierCount - 1)
-          OS << ", ";
-      }
+void OMPClausePrinter::VisitOMPToClause(OMPToClause *Node) {
+  if (!Node->varlist_empty()) {
+    OS << "to";
+    DeclarationNameInfo MapperId = Node->getMapperIdInfo();
+    if (MapperId.getName() && !MapperId.getName().isEmpty()) {
+      OS << '(';
+      OS << "mapper(";
+      NestedNameSpecifier *MapperNNS =
+          Node->getMapperQualifierLoc().getNestedNameSpecifier();
+      if (MapperNNS)
+        MapperNNS->print(OS, Policy);
+      OS << MapperId << "):";
+      VisitOMPClauseList(Node, ' ');
+    } else {
+      VisitOMPClauseList(Node, '(');
     }
-    OS << ':';
-    VisitOMPClauseList(Node, ' ');
-  } else {
-    VisitOMPClauseList(Node, '(');
+    OS << ")";
   }
-  OS << ")";
-}
-
-void OMPClausePrinter::VisitOMPToClause(OMPToClause *Node) {
-  VisitOMPMotionClause(Node);
 }
 
 void OMPClausePrinter::VisitOMPFromClause(OMPFromClause *Node) {
-  VisitOMPMotionClause(Node);
+  if (!Node->varlist_empty()) {
+    OS << "from";
+    DeclarationNameInfo MapperId = Node->getMapperIdInfo();
+    if (MapperId.getName() && !MapperId.getName().isEmpty()) {
+      OS << '(';
+      OS << "mapper(";
+      NestedNameSpecifier *MapperNNS =
+          Node->getMapperQualifierLoc().getNestedNameSpecifier();
+      if (MapperNNS)
+        MapperNNS->print(OS, Policy);
+      OS << MapperId << "):";
+      VisitOMPClauseList(Node, ' ');
+    } else {
+      VisitOMPClauseList(Node, '(');
+    }
+    OS << ")";
+  }
 }
 
 void OMPClausePrinter::VisitOMPDistScheduleClause(OMPDistScheduleClause *Node) {
diff --git a/clang/lib/Basic/OpenMPKinds.cpp b/clang/lib/Basic/OpenMPKinds.cpp
index 5c19d60cbd6e3..da362f99ed297 100644
--- a/clang/lib/Basic/OpenMPKinds.cpp
+++ b/clang/lib/Basic/OpenMPKinds.cpp
@@ -64,16 +64,12 @@ unsigned clang::getOpenMPSimpleClauseType(OpenMPClauseKind Kind, StringRef Str,
     return Type;
   }
   case OMPC_to:
-  case OMPC_from: {
-    unsigned Type = llvm::StringSwitch<unsigned>(Str)
+  case OMPC_from:
+    return llvm::StringSwitch<unsigned>(Str)
 #define OPENMP_MOTION_MODIFIER_KIND(Name)                                      \
   .Case(#Name, static_cast<unsigned>(OMPC_MOTION_MODIFIER_##Name))
 #include "clang/Basic/OpenMPKinds.def"
         .Default(OMPC_MOTION_MODIFIER_unknown);
-    if (OpenMPVersion < 51 && Type == OMPC_MOTION_MODIFIER_present)
-      return OMPC_MOTION_MODIFIER_unknown;
-    return Type;
-  }
   case OMPC_dist_schedule:
     return llvm::StringSwitch<OpenMPDistScheduleClauseKind>(Str)
 #define OPENMP_DIST_SCHEDULE_KIND(Name) .Case(#Name, OMPC_DIST_SCHEDULE_##Name)
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
index f7e0d7a56d5ac..f16f2ed89de89 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -7920,20 +7920,6 @@ class MappableExprsHandler {
         Layout.push_back(Data.get<const FieldDecl *>());
     }
   }
-  static void translateMotionModifiers(
-      ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
-      SmallVectorImpl<OpenMPMapModifierKind> &MapModifiers) {
-    for (OpenMPMotionModifierKind MotionMod : MotionModifiers) {
-      switch (MotionMod) {
-      case OMPC_MOTION_MODIFIER_present:
-        MapModifiers.push_back(OMPC_MAP_MODIFIER_present);
-        break;
-      case OMPC_MOTION_MODIFIER_mapper:
-      case OMPC_MOTION_MODIFIER_unknown:
-        break;
-      }
-    }
-  }
 
 public:
   MappableExprsHandler(const OMPExecutableDirective &Dir, CodeGenFunction &CGF)
@@ -8052,16 +8038,12 @@ class MappableExprsHandler {
       }
     for (const auto *C : CurExecDir->getClausesOfKind<OMPToClause>())
       for (const auto L : C->component_lists()) {
-        SmallVector<OpenMPMapModifierKind, 2> MapModifiers;
-        translateMotionModifiers(C->getMotionModifiers(), MapModifiers);
-        InfoGen(std::get<0>(L), std::get<1>(L), OMPC_MAP_to, MapModifiers,
+        InfoGen(std::get<0>(L), std::get<1>(L), OMPC_MAP_to, llvm::None,
                 /*ReturnDevicePointer=*/false, C->isImplicit(), std::get<2>(L));
       }
     for (const auto *C : CurExecDir->getClausesOfKind<OMPFromClause>())
       for (const auto L : C->component_lists()) {
-        SmallVector<OpenMPMapModifierKind, 2> MapModifiers;
-        translateMotionModifiers(C->getMotionModifiers(), MapModifiers);
-        InfoGen(std::get<0>(L), std::get<1>(L), OMPC_MAP_from, MapModifiers,
+        InfoGen(std::get<0>(L), std::get<1>(L), OMPC_MAP_from, llvm::None,
                 /*ReturnDevicePointer=*/false, C->isImplicit(), std::get<2>(L));
       }
 
diff --git a/clang/lib/Parse/ParseOpenMP.cpp b/clang/lib/Parse/ParseOpenMP.cpp
index cafe4191b2074..6a5d7c604c6bb 100644
--- a/clang/lib/Parse/ParseOpenMP.cpp
+++ b/clang/lib/Parse/ParseOpenMP.cpp
@@ -3441,43 +3441,25 @@ bool Parser::ParseOpenMPVarList(OpenMPDirectiveKind DKind,
     if (Tok.is(tok::colon))
       Data.ColonLoc = ConsumeToken();
   } else if (Kind == OMPC_to || Kind == OMPC_from) {
-    while (Tok.is(tok::identifier)) {
+    if (Tok.is(tok::identifier)) {
       auto Modifier =
           static_cast<OpenMPMotionModifierKind>(getOpenMPSimpleClauseType(
               Kind, PP.getSpelling(Tok), getLangOpts().OpenMP));
-      if (Modifier == OMPC_MOTION_MODIFIER_unknown)
-        break;
-      Data.MotionModifiers.push_back(Modifier);
-      Data.MotionModifiersLoc.push_back(Tok.getLocation());
-      ConsumeToken();
       if (Modifier == OMPC_MOTION_MODIFIER_mapper) {
-        IsInvalidMapperModifier = parseMapperModifier(Data);
-        if (IsInvalidMapperModifier)
-          break;
-      }
-      // OpenMP < 5.1 doesn't permit a ',' or additional modifiers.
-      if (getLangOpts().OpenMP < 51)
-        break;
-      // OpenMP 5.1 accepts an optional ',' even if the next character is ':'.
-      // TODO: Is that intentional?
-      if (Tok.is(tok::comma))
+        // Parse the mapper modifier.
         ConsumeToken();
-    }
-    if (!Data.MotionModifiers.empty() && Tok.isNot(tok::colon)) {
-      if (!IsInvalidMapperModifier) {
-        if (getLangOpts().OpenMP < 51)
-          Diag(Tok, diag::warn_pragma_expected_colon) << ")";
-        else
-          Diag(Tok, diag::warn_pragma_expected_colon) << "motion modifier";
+        IsInvalidMapperModifier = parseMapperModifier(Data);
+        if (Tok.isNot(tok::colon)) {
+          if (!IsInvalidMapperModifier)
+            Diag(Tok, diag::warn_pragma_expected_colon) << ")";
+          SkipUntil(tok::colon, tok::r_paren, tok::annot_pragma_openmp_end,
+                    StopBeforeMatch);
+        }
+        // Consume ':'.
+        if (Tok.is(tok::colon))
+          ConsumeToken();
       }
-      SkipUntil(tok::colon, tok::r_paren, tok::annot_pragma_openmp_end,
-                StopBeforeMatch);
     }
-    // OpenMP 5.1 permits a ':' even without a preceding modifier.  TODO: Is
-    // that intentional?
-    if ((!Data.MotionModifiers.empty() || getLangOpts().OpenMP >= 51) &&
-        Tok.is(tok::colon))
-      Data.ColonLoc = ConsumeToken();
   } else if (Kind == OMPC_allocate ||
              (Kind == OMPC_affinity && Tok.is(tok::identifier) &&
               PP.getSpelling(Tok) == "iterator")) {
@@ -3653,7 +3635,6 @@ OMPClause *Parser::ParseOpenMPVarListClause(OpenMPDirectiveKind DKind,
       Kind, Vars, Data.DepModOrTailExpr, Locs, Data.ColonLoc,
       Data.ReductionOrMapperIdScopeSpec, Data.ReductionOrMapperId,
       Data.ExtraModifier, Data.MapTypeModifiers, Data.MapTypeModifiersLoc,
-      Data.IsMapTypeImplicit, Data.ExtraModifierLoc, Data.MotionModifiers,
-      Data.MotionModifiersLoc);
+      Data.IsMapTypeImplicit, Data.ExtraModifierLoc);
 }
 
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index d7f618a2fa601..0192df3bd170d 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -13536,9 +13536,7 @@ OMPClause *Sema::ActOnOpenMPVarListClause(
     DeclarationNameInfo &ReductionOrMapperId, int ExtraModifier,
     ArrayRef<OpenMPMapModifierKind> MapTypeModifiers,
     ArrayRef<SourceLocation> MapTypeModifiersLoc, bool IsMapTypeImplicit,
-    SourceLocation ExtraModifierLoc,
-    ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
-    ArrayRef<SourceLocation> MotionModifiersLoc) {
+    SourceLocation ExtraModifierLoc) {
   SourceLocation StartLoc = Locs.StartLoc;
   SourceLocation LParenLoc = Locs.LParenLoc;
   SourceLocation EndLoc = Locs.EndLoc;
@@ -13615,14 +13613,12 @@ OMPClause *Sema::ActOnOpenMPVarListClause(
         IsMapTypeImplicit, ExtraModifierLoc, ColonLoc, VarList, Locs);
     break;
   case OMPC_to:
-    Res = ActOnOpenMPToClause(MotionModifiers, MotionModifiersLoc,
-                              ReductionOrMapperIdScopeSpec, ReductionOrMapperId,
-                              ColonLoc, VarList, Locs);
+    Res = ActOnOpenMPToClause(VarList, ReductionOrMapperIdScopeSpec,
+                              ReductionOrMapperId, Locs);
     break;
   case OMPC_from:
-    Res = ActOnOpenMPFromClause(MotionModifiers, MotionModifiersLoc,
-                                ReductionOrMapperIdScopeSpec,
-                                ReductionOrMapperId, ColonLoc, VarList, Locs);
+    Res = ActOnOpenMPFromClause(VarList, ReductionOrMapperIdScopeSpec,
+                                ReductionOrMapperId, Locs);
     break;
   case OMPC_use_device_ptr:
     Res = ActOnOpenMPUseDevicePtrClause(VarList, Locs);
@@ -18580,31 +18576,11 @@ void Sema::checkDeclIsAllowedInOpenMPTarget(Expr *E, Decl *D,
   checkDeclInTargetContext(E->getExprLoc(), E->getSourceRange(), *this, D);
 }
 
-OMPClause *Sema::ActOnOpenMPToClause(
-    ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
-    ArrayRef<SourceLocation> MotionModifiersLoc,
-    CXXScopeSpec &MapperIdScopeSpec, DeclarationNameInfo &MapperId,
-    SourceLocation ColonLoc, ArrayRef<Expr *> VarList,
-    const OMPVarListLocTy &Locs, ArrayRef<Expr *> UnresolvedMappers) {
-  OpenMPMotionModifierKind Modifiers[] = {OMPC_MOTION_MODIFIER_unknown,
-                                          OMPC_MOTION_MODIFIER_unknown};
-  SourceLocation ModifiersLoc[NumberOfOMPMotionModifiers];
-
-  // Process motion-modifiers, flag errors for duplicate modifiers.
-  unsigned Count = 0;
-  for (unsigned I = 0, E = MotionModifiers.size(); I < E; ++I) {
-    if (MotionModifiers[I] != OMPC_MOTION_MODIFIER_unknown &&
-        llvm::find(Modifiers, MotionModifiers[I]) != std::end(Modifiers)) {
-      Diag(MotionModifiersLoc[I], diag::err_omp_duplicate_motion_modifier);
-      continue;
-    }
-    assert(Count < NumberOfOMPMotionModifiers &&
-           "Modifiers exceed the allowed number of motion modifiers");
-    Modifiers[Count] = MotionModifiers[I];
-    ModifiersLoc[Count] = MotionModifiersLoc[I];
-    ++Count;
-  }
-
+OMPClause *Sema::ActOnOpenMPToClause(ArrayRef<Expr *> VarList,
+                                     CXXScopeSpec &MapperIdScopeSpec,
+                                     DeclarationNameInfo &MapperId,
+                                     const OMPVarListLocTy &Locs,
+                                     ArrayRef<Expr *> UnresolvedMappers) {
   MappableVarListInfo MVLI(VarList);
   checkMappableExpressionList(*this, DSAStack, OMPC_to, MVLI, Locs.StartLoc,
                               MapperIdScopeSpec, MapperId, UnresolvedMappers);
@@ -18613,35 +18589,15 @@ OMPClause *Sema::ActOnOpenMPToClause(
 
   return OMPToClause::Create(
       Context, Locs, MVLI.ProcessedVarList, MVLI.VarBaseDeclarations,
-      MVLI.VarComponents, MVLI.UDMapperList, Modifiers, ModifiersLoc,
+      MVLI.VarComponents, MVLI.UDMapperList,
       MapperIdScopeSpec.getWithLocInContext(Context), MapperId);
 }
 
-OMPClause *Sema::ActOnOpenMPFromClause(
-    ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
-    ArrayRef<SourceLocation> MotionModifiersLoc,
-    CXXScopeSpec &MapperIdScopeSpec, DeclarationNameInfo &MapperId,
-    SourceLocation ColonLoc, ArrayRef<Expr *> VarList,
-    const OMPVarListLocTy &Locs, ArrayRef<Expr *> UnresolvedMappers) {
-  OpenMPMotionModifierKind Modifiers[] = {OMPC_MOTION_MODIFIER_unknown,
-                                          OMPC_MOTION_MODIFIER_unknown};
-  SourceLocation ModifiersLoc[NumberOfOMPMotionModifiers];
-
-  // Process motion-modifiers, flag errors for duplicate modifiers.
-  unsigned Count = 0;
-  for (unsigned I = 0, E = MotionModifiers.size(); I < E; ++I) {
-    if (MotionModifiers[I] != OMPC_MOTION_MODIFIER_unknown &&
-        llvm::find(Modifiers, MotionModifiers[I]) != std::end(Modifiers)) {
-      Diag(MotionModifiersLoc[I], diag::err_omp_duplicate_motion_modifier);
-      continue;
-    }
-    assert(Count < NumberOfOMPMotionModifiers &&
-           "Modifiers exceed the allowed number of motion modifiers");
-    Modifiers[Count] = MotionModifiers[I];
-    ModifiersLoc[Count] = MotionModifiersLoc[I];
-    ++Count;
-  }
-
+OMPClause *Sema::ActOnOpenMPFromClause(ArrayRef<Expr *> VarList,
+                                       CXXScopeSpec &MapperIdScopeSpec,
+                                       DeclarationNameInfo &MapperId,
+                                       const OMPVarListLocTy &Locs,
+                                       ArrayRef<Expr *> UnresolvedMappers) {
   MappableVarListInfo MVLI(VarList);
   checkMappableExpressionList(*this, DSAStack, OMPC_from, MVLI, Locs.StartLoc,
                               MapperIdScopeSpec, MapperId, UnresolvedMappers);
@@ -18650,7 +18606,7 @@ OMPClause *Sema::ActOnOpenMPFromClause(
 
   return OMPFromClause::Create(
       Context, Locs, MVLI.ProcessedVarList, MVLI.VarBaseDeclarations,
-      MVLI.VarComponents, MVLI.UDMapperList, Modifiers, ModifiersLoc,
+      MVLI.VarComponents, MVLI.UDMapperList,
       MapperIdScopeSpec.getWithLocInContext(Context), MapperId);
 }
 
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index 53064d28dd991..0f83431c94cf7 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -2006,32 +2006,26 @@ class TreeTransform {
   ///
   /// By default, performs semantic analysis to build the new statement.
   /// Subclasses may override this routine to provide different behavior.
-  OMPClause *
-  RebuildOMPToClause(ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
-                     ArrayRef<SourceLocation> MotionModifiersLoc,
-                     CXXScopeSpec &MapperIdScopeSpec,
-                     DeclarationNameInfo &MapperId, SourceLocation ColonLoc,
-                     ArrayRef<Expr *> VarList, const OMPVarListLocTy &Locs,
-                     ArrayRef<Expr *> UnresolvedMappers) {
-    return getSema().ActOnOpenMPToClause(MotionModifiers, MotionModifiersLoc,
-                                         MapperIdScopeSpec, MapperId, ColonLoc,
-                                         VarList, Locs, UnresolvedMappers);
+  OMPClause *RebuildOMPToClause(ArrayRef<Expr *> VarList,
+                                CXXScopeSpec &MapperIdScopeSpec,
+                                DeclarationNameInfo &MapperId,
+                                const OMPVarListLocTy &Locs,
+                                ArrayRef<Expr *> UnresolvedMappers) {
+    return getSema().ActOnOpenMPToClause(VarList, MapperIdScopeSpec, MapperId,
+                                         Locs, UnresolvedMappers);
   }
 
   /// Build a new OpenMP 'from' clause.
   ///
   /// By default, performs semantic analysis to build the new statement.
   /// Subclasses may override this routine to provide different behavior.
-  OMPClause *
-  RebuildOMPFromClause(ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
-                       ArrayRef<SourceLocation> MotionModifiersLoc,
-                       CXXScopeSpec &MapperIdScopeSpec,
-                       DeclarationNameInfo &MapperId, SourceLocation ColonLoc,
-                       ArrayRef<Expr *> VarList, const OMPVarListLocTy &Locs,
-                       ArrayRef<Expr *> UnresolvedMappers) {
-    return getSema().ActOnOpenMPFromClause(
-        MotionModifiers, MotionModifiersLoc, MapperIdScopeSpec, MapperId,
-        ColonLoc, VarList, Locs, UnresolvedMappers);
+  OMPClause *RebuildOMPFromClause(ArrayRef<Expr *> VarList,
+                                  CXXScopeSpec &MapperIdScopeSpec,
+                                  DeclarationNameInfo &MapperId,
+                                  const OMPVarListLocTy &Locs,
+                                  ArrayRef<Expr *> UnresolvedMappers) {
+    return getSema().ActOnOpenMPFromClause(VarList, MapperIdScopeSpec, MapperId,
+                                           Locs, UnresolvedMappers);
   }
 
   /// Build a new OpenMP 'use_device_ptr' clause.
@@ -9744,9 +9738,8 @@ OMPClause *TreeTransform<Derived>::TransformOMPToClause(OMPToClause *C) {
   if (transformOMPMappableExprListClause<Derived, OMPToClause>(
           *this, C, Vars, MapperIdScopeSpec, MapperIdInfo, UnresolvedMappers))
     return nullptr;
-  return getDerived().RebuildOMPToClause(
-      C->getMotionModifiers(), C->getMotionModifiersLoc(), MapperIdScopeSpec,
-      MapperIdInfo, C->getColonLoc(), Vars, Locs, UnresolvedMappers);
+  return getDerived().RebuildOMPToClause(Vars, MapperIdScopeSpec, MapperIdInfo,
+                                         Locs, UnresolvedMappers);
 }
 
 template <typename Derived>
@@ -9760,8 +9753,7 @@ OMPClause *TreeTransform<Derived>::TransformOMPFromClause(OMPFromClause *C) {
           *this, C, Vars, MapperIdScopeSpec, MapperIdInfo, UnresolvedMappers))
     return nullptr;
   return getDerived().RebuildOMPFromClause(
-      C->getMotionModifiers(), C->getMotionModifiersLoc(), MapperIdScopeSpec,
-      MapperIdInfo, C->getColonLoc(), Vars, Locs, UnresolvedMappers);
+      Vars, MapperIdScopeSpec, MapperIdInfo, Locs, UnresolvedMappers);
 }
 
 template <typename Derived>
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index a27c36420fdbc..6b96b4ff59b8e 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -12601,14 +12601,8 @@ void OMPClauseReader::VisitOMPDefaultmapClause(OMPDefaultmapClause *C) {
 
 void OMPClauseReader::VisitOMPToClause(OMPToClause *C) {
   C->setLParenLoc(Record.readSourceLocation());
-  for (unsigned I = 0; I < NumberOfOMPMotionModifiers; ++I) {
-    C->setMotionModifier(
-        I, static_cast<OpenMPMotionModifierKind>(Record.readInt()));
-    C->setMotionModifierLoc(I, Record.readSourceLocation());
-  }
   C->setMapperQualifierLoc(Record.readNestedNameSpecifierLoc());
   C->setMapperIdInfo(Record.readDeclarationNameInfo());
-  C->setColonLoc(Record.readSourceLocation());
   auto NumVars = C->varlist_size();
   auto UniqueDecls = C->getUniqueDeclarationsNum();
   auto TotalLists = C->getTotalComponentListNum();
@@ -12657,14 +12651,8 @@ void OMPClauseReader::VisitOMPToClause(OMPToClause *C) {
 
 void OMPClauseReader::VisitOMPFromClause(OMPFromClause *C) {
   C->setLParenLoc(Record.readSourceLocation());
-  for (unsigned I = 0; I < NumberOfOMPMotionModifiers; ++I) {
-    C->setMotionModifier(
-        I, static_cast<OpenMPMotionModifierKind>(Record.readInt()));
-    C->setMotionModifierLoc(I, Record.readSourceLocation());
-  }
   C->setMapperQualifierLoc(Record.readNestedNameSpecifierLoc());
   C->setMapperIdInfo(Record.readDeclarationNameInfo());
-  C->setColonLoc(Record.readSourceLocation());
   auto NumVars = C->varlist_size();
   auto UniqueDecls = C->getUniqueDeclarationsNum();
   auto TotalLists = C->getTotalComponentListNum();
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index 8c693b7d34fdd..9ea90b2a0212a 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -6581,13 +6581,8 @@ void OMPClauseWriter::VisitOMPToClause(OMPToClause *C) {
   Record.push_back(C->getTotalComponentListNum());
   Record.push_back(C->getTotalComponentsNum());
   Record.AddSourceLocation(C->getLParenLoc());
-  for (unsigned I = 0; I < NumberOfOMPMotionModifiers; ++I) {
-    Record.push_back(C->getMotionModifier(I));
-    Record.AddSourceLocation(C->getMotionModifierLoc(I));
-  }
   Record.AddNestedNameSpecifierLoc(C->getMapperQualifierLoc());
   Record.AddDeclarationNameInfo(C->getMapperIdInfo());
-  Record.AddSourceLocation(C->getColonLoc());
   for (auto *E : C->varlists())
     Record.AddStmt(E);
   for (auto *E : C->mapperlists())
@@ -6610,13 +6605,8 @@ void OMPClauseWriter::VisitOMPFromClause(OMPFromClause *C) {
   Record.push_back(C->getTotalComponentListNum());
   Record.push_back(C->getTotalComponentsNum());
   Record.AddSourceLocation(C->getLParenLoc());
-  for (unsigned I = 0; I < NumberOfOMPMotionModifiers; ++I) {
-    Record.push_back(C->getMotionModifier(I));
-    Record.AddSourceLocation(C->getMotionModifierLoc(I));
-  }
   Record.AddNestedNameSpecifierLoc(C->getMapperQualifierLoc());
   Record.AddDeclarationNameInfo(C->getMapperIdInfo());
-  Record.AddSourceLocation(C->getColonLoc());
   for (auto *E : C->varlists())
     Record.AddStmt(E);
   for (auto *E : C->mapperlists())
diff --git a/clang/test/OpenMP/declare_mapper_ast_print.c b/clang/test/OpenMP/declare_mapper_ast_print.c
index b7c9976c122af..e82bc4c2d3109 100644
--- a/clang/test/OpenMP/declare_mapper_ast_print.c
+++ b/clang/test/OpenMP/declare_mapper_ast_print.c
@@ -5,14 +5,6 @@
 // RUN: %clang_cc1 -verify -fopenmp-simd -ast-print %s | FileCheck %s
 // RUN: %clang_cc1 -fopenmp-simd -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp-simd -include-pch %t -fsyntax-only -verify %s -ast-print | FileCheck %s
-
-// RUN: %clang_cc1 -DOMP51 -verify -fopenmp -fopenmp-version=51 -ast-print %s | FileCheck -check-prefixes=CHECK,OMP51 %s
-// RUN: %clang_cc1 -DOMP51 -fopenmp -fopenmp-version=51 -emit-pch -o %t %s
-// RUN: %clang_cc1 -DOMP51 -fopenmp -fopenmp-version=51 -include-pch %t -fsyntax-only -verify %s -ast-print | FileCheck -check-prefixes=CHECK,OMP51 %s
-
-// RUN: %clang_cc1 -DOMP51 -verify -fopenmp-simd -fopenmp-version=51 -ast-print %s | FileCheck -check-prefixes=CHECK,OMP51 %s
-// RUN: %clang_cc1 -DOMP51 -fopenmp-simd -fopenmp-version=51 -emit-pch -o %t %s
-// RUN: %clang_cc1 -DOMP51 -fopenmp-simd -fopenmp-version=51 -include-pch %t -fsyntax-only -verify %s -ast-print | FileCheck -check-prefixes=CHECK,OMP51 %s
 // expected-no-diagnostics
 
 #ifndef HEADER
@@ -58,12 +50,6 @@ int main() {
     { dd[0].i++; }
 #pragma omp target update to(mapper(id): vv) from(mapper(default): dd[0:10])
 // CHECK: #pragma omp target update to(mapper(id): vv) from(mapper(default): dd[0:10])
-#ifdef OMP51
-#pragma omp target update to(mapper(id) present: vv) from(mapper(default), present: dd[0:10])
-// OMP51: #pragma omp target update to(mapper(id), present: vv) from(mapper(default), present: dd[0:10])
-#pragma omp target update to(present mapper(id): vv) from(present, mapper(default): dd[0:10])
-// OMP51: #pragma omp target update to(present, mapper(id): vv) from(present, mapper(default): dd[0:10])
-#endif
   }
   return 0;
 }
diff --git a/clang/test/OpenMP/declare_mapper_codegen.cpp b/clang/test/OpenMP/declare_mapper_codegen.cpp
index 2fd4b3cb7ed07..ee64fe2099007 100644
--- a/clang/test/OpenMP/declare_mapper_codegen.cpp
+++ b/clang/test/OpenMP/declare_mapper_codegen.cpp
@@ -839,242 +839,4 @@ void foo(int a){
 
 #endif // CK3
 
-///==========================================================================///
-// RUN: %clang_cc1 -DCK4 -verify -fopenmp -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm -femit-all-decls -disable-llvm-passes %s -o - | FileCheck --check-prefix CK4 --check-prefix CK4-64 %s
-// RUN: %clang_cc1 -DCK4 -fopenmp -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -femit-all-decls -disable-llvm-passes -o %t %s
-// RUN: %clang_cc1 -DCK4 -fopenmp -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -femit-all-decls -disable-llvm-passes -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix CK4 --check-prefix CK4-64 %s
-// RUN: %clang_cc1 -DCK4 -verify -fopenmp -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm -femit-all-decls -disable-llvm-passes %s -o - | FileCheck --check-prefix CK4 --check-prefix CK4-32 %s
-// RUN: %clang_cc1 -DCK4 -fopenmp -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -femit-all-decls -disable-llvm-passes -o %t %s
-// RUN: %clang_cc1 -DCK4 -fopenmp -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -femit-all-decls -disable-llvm-passes -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix CK4 --check-prefix CK4-32 %s
-
-// RUN: %clang_cc1 -DCK4 -verify -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm -femit-all-decls -disable-llvm-passes %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -DCK4 -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -femit-all-decls -disable-llvm-passes -o %t %s
-// RUN: %clang_cc1 -DCK4 -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -femit-all-decls -disable-llvm-passes -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -DCK4 -verify -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm -femit-all-decls -disable-llvm-passes %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -DCK4 -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -femit-all-decls -disable-llvm-passes -o %t %s
-// RUN: %clang_cc1 -DCK4 -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -femit-all-decls -disable-llvm-passes -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-
-#ifdef CK4
-// Mapper function code generation and runtime interface.
-
-// CK4-64: [[TSIZES:@.+]] = {{.+}}constant [1 x i64] [i64 16]
-// CK4-32: [[TSIZES:@.+]] = {{.+}}constant [1 x i64] [i64 8]
-// PRESENT=0x1000 | TARGET_PARAM=0x20 | TO=0x1 = 0x1021
-// CK4: [[TTYPES:@.+]] = {{.+}}constant [1 x i64] [i64 [[#0x1021]]]
-
-// CK4-64: [[FSIZES:@.+]] = {{.+}}constant [1 x i64] [i64 16]
-// CK4-32: [[FSIZES:@.+]] = {{.+}}constant [1 x i64] [i64 8]
-// PRESENT=0x1000 | TARGET_PARAM=0x20 | FROM=0x2 = 0x1022
-// CK4: [[FTYPES:@.+]] = {{.+}}constant [1 x i64] [i64 [[#0x1022]]]
-
-class C {
-public:
-  int a;
-  double *b;
-};
-
-#pragma omp declare mapper(id: C s) map(s.a, s.b[0:2])
-
-// CK4: define {{.*}}void [[MPRFUNC:@[.]omp_mapper[.].*C[.]id]](i8*{{.*}}, i8*{{.*}}, i8*{{.*}}, i64{{.*}}, i64{{.*}})
-// CK4: store i8* %{{[^,]+}}, i8** [[HANDLEADDR:%[^,]+]]
-// CK4: store i8* %{{[^,]+}}, i8** [[BPTRADDR:%[^,]+]]
-// CK4: store i8* %{{[^,]+}}, i8** [[VPTRADDR:%[^,]+]]
-// CK4: store i64 %{{[^,]+}}, i{{64|32}}* [[SIZEADDR:%[^,]+]]
-// CK4: store i64 %{{[^,]+}}, i64* [[TYPEADDR:%[^,]+]]
-// CK4-DAG: [[BYTESIZE:%.+]] = load i64, i64* [[SIZEADDR]]
-// CK4-64-DAG: [[SIZE:%.+]] = udiv exact i64 [[BYTESIZE]], 16
-// CK4-32-DAG: [[SIZE:%.+]] = udiv exact i64 [[BYTESIZE]], 8
-// CK4-DAG: [[TYPE:%.+]] = load i64, i64* [[TYPEADDR]]
-// CK4-DAG: [[HANDLE:%.+]] = load i8*, i8** [[HANDLEADDR]]
-// CK4-DAG: [[PTRBEGIN:%.+]] = bitcast i8** [[VPTRADDR]] to %class.C**
-// CK4-DAG: [[PTREND:%.+]] = getelementptr %class.C*, %class.C** [[PTRBEGIN]], i64 [[SIZE]]
-// CK4-DAG: [[BPTR:%.+]] = load i8*, i8** [[BPTRADDR]]
-// CK4-DAG: [[BEGIN:%.+]] = load i8*, i8** [[VPTRADDR]]
-// CK4: [[ISARRAY:%.+]] = icmp sge i64 [[SIZE]], 1
-// CK4: br i1 [[ISARRAY]], label %[[INITEVALDEL:[^,]+]], label %[[LHEAD:[^,]+]]
-
-// CK4: [[INITEVALDEL]]
-// CK4: [[TYPEDEL:%.+]] = and i64 [[TYPE]], 8
-// CK4: [[ISNOTDEL:%.+]] = icmp eq i64 [[TYPEDEL]], 0
-// CK4: br i1 [[ISNOTDEL]], label %[[INIT:[^,]+]], label %[[LHEAD:[^,]+]]
-// CK4: [[INIT]]
-// CK4-64-DAG: [[ARRSIZE:%.+]] = mul nuw i64 [[SIZE]], 16
-// CK4-32-DAG: [[ARRSIZE:%.+]] = mul nuw i64 [[SIZE]], 8
-// CK4-DAG: [[ITYPE:%.+]] = and i64 [[TYPE]], -4
-// CK4: call void @__tgt_push_mapper_component(i8* [[HANDLE]], i8* [[BPTR]], i8* [[BEGIN]], i64 [[ARRSIZE]], i64 [[ITYPE]])
-// CK4: br label %[[LHEAD:[^,]+]]
-
-// CK4: [[LHEAD]]
-// CK4: [[ISEMPTY:%.+]] = icmp eq %class.C** [[PTRBEGIN]], [[PTREND]]
-// CK4: br i1 [[ISEMPTY]], label %[[DONE:[^,]+]], label %[[LBODY:[^,]+]]
-// CK4: [[LBODY]]
-// CK4: [[PTR:%.+]] = phi %class.C** [ [[PTRBEGIN]], %[[LHEAD]] ], [ [[PTRNEXT:%.+]], %[[LCORRECT:[^,]+]] ]
-// CK4: [[OBJ:%.+]] = load %class.C*, %class.C** [[PTR]]
-// CK4-DAG: [[ABEGIN:%.+]] = getelementptr inbounds %class.C, %class.C* [[OBJ]], i32 0, i32 0
-// CK4-DAG: [[BBEGIN:%.+]] = getelementptr inbounds %class.C, %class.C* [[OBJ]], i32 0, i32 1
-// CK4-DAG: [[BBEGIN2:%.+]] = getelementptr inbounds %class.C, %class.C* [[OBJ]], i32 0, i32 1
-// CK4-DAG: [[BARRBEGIN:%.+]] = load double*, double** [[BBEGIN2]]
-// CK4-DAG: [[BARRBEGINGEP:%.+]] = getelementptr inbounds double, double* [[BARRBEGIN]], i[[sz:64|32]] 0
-// CK4-DAG: [[BEND:%.+]] = getelementptr double*, double** [[BBEGIN]], i32 1
-// CK4-DAG: [[ABEGINV:%.+]] = bitcast i32* [[ABEGIN]] to i8*
-// CK4-DAG: [[BENDV:%.+]] = bitcast double** [[BEND]] to i8*
-// CK4-DAG: [[ABEGINI:%.+]] = ptrtoint i8* [[ABEGINV]] to i64
-// CK4-DAG: [[BENDI:%.+]] = ptrtoint i8* [[BENDV]] to i64
-// CK4-DAG: [[CSIZE:%.+]] = sub i64 [[BENDI]], [[ABEGINI]]
-// CK4-DAG: [[CUSIZE:%.+]] = sdiv exact i64 [[CSIZE]], ptrtoint (i8* getelementptr (i8, i8* null, i32 1) to i64)
-// CK4-DAG: [[BPTRADDR0BC:%.+]] = bitcast %class.C* [[OBJ]] to i8*
-// CK4-DAG: [[PTRADDR0BC:%.+]] = bitcast i32* [[ABEGIN]] to i8*
-// CK4-DAG: [[PRESIZE:%.+]] = call i64 @__tgt_mapper_num_components(i8* [[HANDLE]])
-// CK4-DAG: [[SHIPRESIZE:%.+]] = shl i64 [[PRESIZE]], 48
-// CK4-DAG: br label %[[MEMBER:[^,]+]]
-// CK4-DAG: [[MEMBER]]
-// CK4-DAG: br i1 true, label %[[LTYPE:[^,]+]], label %[[MEMBERCOM:[^,]+]]
-// CK4-DAG: [[MEMBERCOM]]
-// CK4-DAG: [[MEMBERCOMTYPE:%.+]] = add nuw i64 32, [[SHIPRESIZE]]
-// CK4-DAG: br label %[[LTYPE]]
-// CK4-DAG: [[LTYPE]]
-// CK4-DAG: [[MEMBERTYPE:%.+]] = phi i64 [ 32, %[[MEMBER]] ], [ [[MEMBERCOMTYPE]], %[[MEMBERCOM]] ]
-// CK4-DAG: [[TYPETF:%.+]] = and i64 [[TYPE]], 3
-// CK4-DAG: [[ISALLOC:%.+]] = icmp eq i64 [[TYPETF]], 0
-// CK4-DAG: br i1 [[ISALLOC]], label %[[ALLOC:[^,]+]], label %[[ALLOCELSE:[^,]+]]
-// CK4-DAG: [[ALLOC]]
-// CK4-DAG: [[ALLOCTYPE:%.+]] = and i64 [[MEMBERTYPE]], -4
-// CK4-DAG: br label %[[TYEND:[^,]+]]
-// CK4-DAG: [[ALLOCELSE]]
-// CK4-DAG: [[ISTO:%.+]] = icmp eq i64 [[TYPETF]], 1
-// CK4-DAG: br i1 [[ISTO]], label %[[TO:[^,]+]], label %[[TOELSE:[^,]+]]
-// CK4-DAG: [[TO]]
-// CK4-DAG: [[TOTYPE:%.+]] = and i64 [[MEMBERTYPE]], -3
-// CK4-DAG: br label %[[TYEND]]
-// CK4-DAG: [[TOELSE]]
-// CK4-DAG: [[ISFROM:%.+]] = icmp eq i64 [[TYPETF]], 2
-// CK4-DAG: br i1 [[ISFROM]], label %[[FROM:[^,]+]], label %[[TYEND]]
-// CK4-DAG: [[FROM]]
-// CK4-DAG: [[FROMTYPE:%.+]] = and i64 [[MEMBERTYPE]], -2
-// CK4-DAG: br label %[[TYEND]]
-// CK4-DAG: [[TYEND]]
-// CK4-DAG: [[PHITYPE0:%.+]] = phi i64 [ [[ALLOCTYPE]], %[[ALLOC]] ], [ [[TOTYPE]], %[[TO]] ], [ [[FROMTYPE]], %[[FROM]] ], [ [[MEMBERTYPE]], %[[TOELSE]] ]
-// CK4: call void @__tgt_push_mapper_component(i8* [[HANDLE]], i8* [[BPTRADDR0BC]], i8* [[PTRADDR0BC]], i64 [[CUSIZE]], i64 [[PHITYPE0]])
-// CK4-DAG: [[BPTRADDR1BC:%.+]] = bitcast %class.C* [[OBJ]] to i8*
-// CK4-DAG: [[PTRADDR1BC:%.+]] = bitcast i32* [[ABEGIN]] to i8*
-// CK4-DAG: br label %[[MEMBER:[^,]+]]
-// CK4-DAG: [[MEMBER]]
-// CK4-DAG: br i1 false, label %[[LTYPE:[^,]+]], label %[[MEMBERCOM:[^,]+]]
-// CK4-DAG: [[MEMBERCOM]]
-// 281474976710659 == 0x1,000,000,003
-// CK4-DAG: [[MEMBERCOMTYPE:%.+]] = add nuw i64 281474976710659, [[SHIPRESIZE]]
-// CK4-DAG: br label %[[LTYPE]]
-// CK4-DAG: [[LTYPE]]
-// CK4-DAG: [[MEMBERTYPE:%.+]] = phi i64 [ 281474976710659, %[[MEMBER]] ], [ [[MEMBERCOMTYPE]], %[[MEMBERCOM]] ]
-// CK4-DAG: [[TYPETF:%.+]] = and i64 [[TYPE]], 3
-// CK4-DAG: [[ISALLOC:%.+]] = icmp eq i64 [[TYPETF]], 0
-// CK4-DAG: br i1 [[ISALLOC]], label %[[ALLOC:[^,]+]], label %[[ALLOCELSE:[^,]+]]
-// CK4-DAG: [[ALLOC]]
-// CK4-DAG: [[ALLOCTYPE:%.+]] = and i64 [[MEMBERTYPE]], -4
-// CK4-DAG: br label %[[TYEND:[^,]+]]
-// CK4-DAG: [[ALLOCELSE]]
-// CK4-DAG: [[ISTO:%.+]] = icmp eq i64 [[TYPETF]], 1
-// CK4-DAG: br i1 [[ISTO]], label %[[TO:[^,]+]], label %[[TOELSE:[^,]+]]
-// CK4-DAG: [[TO]]
-// CK4-DAG: [[TOTYPE:%.+]] = and i64 [[MEMBERTYPE]], -3
-// CK4-DAG: br label %[[TYEND]]
-// CK4-DAG: [[TOELSE]]
-// CK4-DAG: [[ISFROM:%.+]] = icmp eq i64 [[TYPETF]], 2
-// CK4-DAG: br i1 [[ISFROM]], label %[[FROM:[^,]+]], label %[[TYEND]]
-// CK4-DAG: [[FROM]]
-// CK4-DAG: [[FROMTYPE:%.+]] = and i64 [[MEMBERTYPE]], -2
-// CK4-DAG: br label %[[TYEND]]
-// CK4-DAG: [[TYEND]]
-// CK4-DAG: [[TYPE1:%.+]] = phi i64 [ [[ALLOCTYPE]], %[[ALLOC]] ], [ [[TOTYPE]], %[[TO]] ], [ [[FROMTYPE]], %[[FROM]] ], [ [[MEMBERTYPE]], %[[TOELSE]] ]
-// CK4: call void @__tgt_push_mapper_component(i8* [[HANDLE]], i8* [[BPTRADDR1BC]], i8* [[PTRADDR1BC]], i64 4, i64 [[TYPE1]])
-// CK4-DAG: [[BPTRADDR2BC:%.+]] = bitcast double** [[BBEGIN]] to i8*
-// CK4-DAG: [[PTRADDR2BC:%.+]] = bitcast double* [[BARRBEGINGEP]] to i8*
-// CK4-DAG: br label %[[MEMBER:[^,]+]]
-// CK4-DAG: [[MEMBER]]
-// CK4-DAG: br i1 false, label %[[LTYPE:[^,]+]], label %[[MEMBERCOM:[^,]+]]
-// CK4-DAG: [[MEMBERCOM]]
-// 281474976710675 == 0x1,000,000,013
-// CK4-DAG: [[MEMBERCOMTYPE:%.+]] = add nuw i64 281474976710675, [[SHIPRESIZE]]
-// CK4-DAG: br label %[[LTYPE]]
-// CK4-DAG: [[LTYPE]]
-// CK4-DAG: [[MEMBERTYPE:%.+]] = phi i64 [ 281474976710675, %[[MEMBER]] ], [ [[MEMBERCOMTYPE]], %[[MEMBERCOM]] ]
-// CK4-DAG: [[TYPETF:%.+]] = and i64 [[TYPE]], 3
-// CK4-DAG: [[ISALLOC:%.+]] = icmp eq i64 [[TYPETF]], 0
-// CK4-DAG: br i1 [[ISALLOC]], label %[[ALLOC:[^,]+]], label %[[ALLOCELSE:[^,]+]]
-// CK4-DAG: [[ALLOC]]
-// CK4-DAG: [[ALLOCTYPE:%.+]] = and i64 [[MEMBERTYPE]], -4
-// CK4-DAG: br label %[[TYEND:[^,]+]]
-// CK4-DAG: [[ALLOCELSE]]
-// CK4-DAG: [[ISTO:%.+]] = icmp eq i64 [[TYPETF]], 1
-// CK4-DAG: br i1 [[ISTO]], label %[[TO:[^,]+]], label %[[TOELSE:[^,]+]]
-// CK4-DAG: [[TO]]
-// CK4-DAG: [[TOTYPE:%.+]] = and i64 [[MEMBERTYPE]], -3
-// CK4-DAG: br label %[[TYEND]]
-// CK4-DAG: [[TOELSE]]
-// CK4-DAG: [[ISFROM:%.+]] = icmp eq i64 [[TYPETF]], 2
-// CK4-DAG: br i1 [[ISFROM]], label %[[FROM:[^,]+]], label %[[TYEND]]
-// CK4-DAG: [[FROM]]
-// CK4-DAG: [[FROMTYPE:%.+]] = and i64 [[MEMBERTYPE]], -2
-// CK4-DAG: br label %[[TYEND]]
-// CK4-DAG: [[TYEND]]
-// CK4-DAG: [[TYPE2:%.+]] = phi i64 [ [[ALLOCTYPE]], %[[ALLOC]] ], [ [[TOTYPE]], %[[TO]] ], [ [[FROMTYPE]], %[[FROM]] ], [ [[MEMBERTYPE]], %[[TOELSE]] ]
-// CK4: call void @__tgt_push_mapper_component(i8* [[HANDLE]], i8* [[BPTRADDR2BC]], i8* [[PTRADDR2BC]], i64 16, i64 [[TYPE2]])
-// CK4: [[PTRNEXT]] = getelementptr %class.C*, %class.C** [[PTR]], i32 1
-// CK4: [[ISDONE:%.+]] = icmp eq %class.C** [[PTRNEXT]], [[PTREND]]
-// CK4: br i1 [[ISDONE]], label %[[LEXIT:[^,]+]], label %[[LBODY]]
-
-// CK4: [[LEXIT]]
-// CK4: [[ISARRAY:%.+]] = icmp sge i64 [[SIZE]], 1
-// CK4: br i1 [[ISARRAY]], label %[[EVALDEL:[^,]+]], label %[[DONE]]
-// CK4: [[EVALDEL]]
-// CK4: [[TYPEDEL:%.+]] = and i64 [[TYPE]], 8
-// CK4: [[ISDEL:%.+]] = icmp ne i64 [[TYPEDEL]], 0
-// CK4: br i1 [[ISDEL]], label %[[DEL:[^,]+]], label %[[DONE]]
-// CK4: [[DEL]]
-// CK4-64-DAG: [[ARRSIZE:%.+]] = mul nuw i64 [[SIZE]], 16
-// CK4-32-DAG: [[ARRSIZE:%.+]] = mul nuw i64 [[SIZE]], 8
-// CK4-DAG: [[DTYPE:%.+]] = and i64 [[TYPE]], -4
-// CK4: call void @__tgt_push_mapper_component(i8* [[HANDLE]], i8* [[BPTR]], i8* [[BEGIN]], i64 [[ARRSIZE]], i64 [[DTYPE]])
-// CK4: br label %[[DONE]]
-// CK4: [[DONE]]
-// CK4: ret void
-
-
-// CK4-LABEL: define {{.*}}void @{{.*}}foo{{.*}}
-void foo(int a){
-  int i = a;
-  C c;
-  c.a = a;
-
-  // CK4-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 1, i8** [[TGEPBP:%.+]], i8** [[TGEPP:%.+]], i64* getelementptr {{.+}}[1 x i64]* [[TSIZES]], i32 0, i32 0), {{.+}}getelementptr {{.+}}[1 x i64]* [[TTYPES]]{{.+}}, i8** [[TMPRGEP:%.+]])
-  // CK4-DAG: [[TGEPBP]] = getelementptr inbounds {{.+}}[[TBP:%[^,]+]], i{{.+}} 0, i{{.+}} 0
-  // CK4-DAG: [[TGEPP]] = getelementptr inbounds {{.+}}[[TP:%[^,]+]], i{{.+}} 0, i{{.+}} 0
-  // CK4-DAG: [[TMPRGEP]] = bitcast [1 x i8*]* [[TMPR:%[^,]+]] to i8**
-  // CK4-DAG: [[TBP0:%.+]] = getelementptr inbounds {{.+}}[[TBP]], i{{.+}} 0, i{{.+}} 0
-  // CK4-DAG: [[TP0:%.+]] = getelementptr inbounds {{.+}}[[TP]], i{{.+}} 0, i{{.+}} 0
-  // CK4-DAG: [[TMPR1:%.+]] = getelementptr inbounds {{.+}}[[TMPR]], i[[sz]] 0, i[[sz]] 0
-  // CK4-DAG: [[TCBP0:%.+]] = bitcast i8** [[TBP0]] to %class.C**
-  // CK4-DAG: [[TCP0:%.+]] = bitcast i8** [[TP0]] to %class.C**
-  // CK4-DAG: store %class.C* [[VAL:%[^,]+]], %class.C** [[TCBP0]]
-  // CK4-DAG: store %class.C* [[VAL]], %class.C** [[TCP0]]
-  // CK4-DAG: store i8* bitcast (void (i8*, i8*, i8*, i64, i64)* [[MPRFUNC]] to i8*), i8** [[TMPR1]]
-  #pragma omp target update to(present, mapper(id): c)
-
-  // CK4-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 1, i8** [[FGEPBP:%.+]], i8** [[FGEPP:%.+]], i64* getelementptr {{.+}}[1 x i64]* [[FSIZES]], i32 0, i32 0), {{.+}}getelementptr {{.+}}[1 x i64]* [[FTYPES]]{{.+}}, i8** [[FMPRGEP:%.+]])
-  // CK4-DAG: [[FGEPBP]] = getelementptr inbounds {{.+}}[[FBP:%[^,]+]], i{{.+}} 0, i{{.+}} 0
-  // CK4-DAG: [[FGEPP]] = getelementptr inbounds {{.+}}[[FP:%[^,]+]], i{{.+}} 0, i{{.+}} 0
-  // CK4-DAG: [[FMPRGEP]] = bitcast [1 x i8*]* [[FMPR:%[^,]+]] to i8**
-  // CK4-DAG: [[FBP0:%.+]] = getelementptr inbounds {{.+}}[[FBP]], i{{.+}} 0, i{{.+}} 0
-  // CK4-DAG: [[FP0:%.+]] = getelementptr inbounds {{.+}}[[FP]], i{{.+}} 0, i{{.+}} 0
-  // CK4-DAG: [[FMPR1:%.+]] = getelementptr inbounds {{.+}}[[FMPR]], i[[sz]] 0, i[[sz]] 0
-  // CK4-DAG: [[FCBP0:%.+]] = bitcast i8** [[FBP0]] to %class.C**
-  // CK4-DAG: [[FCP0:%.+]] = bitcast i8** [[FP0]] to %class.C**
-  // CK4-DAG: store %class.C* [[VAL]], %class.C** [[FCBP0]]
-  // CK4-DAG: store %class.C* [[VAL]], %class.C** [[FCP0]]
-  // CK4-DAG: store i8* bitcast (void (i8*, i8*, i8*, i64, i64)* [[MPRFUNC]] to i8*), i8** [[FMPR1]]
-  #pragma omp target update from(mapper(id), present: c)
-}
-
-#endif // CK4
-
 #endif // HEADER
diff --git a/clang/test/OpenMP/target_update_ast_print.cpp b/clang/test/OpenMP/target_update_ast_print.cpp
index 745233c430eb1..0111432fde8f6 100644
--- a/clang/test/OpenMP/target_update_ast_print.cpp
+++ b/clang/test/OpenMP/target_update_ast_print.cpp
@@ -5,14 +5,6 @@
 // RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=50 -ast-print %s | FileCheck %s
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=50 -x c++ -std=c++11 -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=50 -std=c++11 -include-pch %t -fsyntax-only -verify %s -ast-print | FileCheck %s
-
-// RUN: %clang_cc1 -DOMP51 -verify -fopenmp -fopenmp-version=51 -ast-print %s | FileCheck -check-prefixes=CHECK,OMP51 %s
-// RUN: %clang_cc1 -DOMP51 -fopenmp -fopenmp-version=51 -x c++ -std=c++11 -emit-pch -o %t %s
-// RUN: %clang_cc1 -DOMP51 -fopenmp -fopenmp-version=51 -std=c++11 -include-pch %t -fsyntax-only -verify %s -ast-print | FileCheck -check-prefixes=CHECK,OMP51 %s
-
-// RUN: %clang_cc1 -DOMP51 -verify -fopenmp-simd -fopenmp-version=51 -ast-print %s | FileCheck -check-prefixes=CHECK,OMP51 %s
-// RUN: %clang_cc1 -DOMP51 -fopenmp-simd -fopenmp-version=51 -x c++ -std=c++11 -emit-pch -o %t %s
-// RUN: %clang_cc1 -DOMP51 -fopenmp-simd -fopenmp-version=51 -std=c++11 -include-pch %t -fsyntax-only -verify %s -ast-print | FileCheck -check-prefixes=CHECK,OMP51 %s
 // expected-no-diagnostics
 
 #ifndef HEADER
@@ -33,13 +25,6 @@ T foo(T targ, U uarg) {
 #pragma omp target update to(arr[2][0:1:2])
 
 #pragma omp target update from(arr[2][0:1:2])
-
-#ifdef OMP51
-#pragma omp target update to(present: arr[2][0:1:2])
-
-#pragma omp target update from(present: arr[2][0:1:2], a)
-#endif
-
   return a + targ + (T)b;
 }
 // CHECK:      static T a, *p;
@@ -60,8 +45,6 @@ T foo(T targ, U uarg) {
 // CHECK:      int arr[100][100];
 // CHECK-NEXT: #pragma omp target update to(arr[2][0:1:2])
 // CHECK-NEXT: #pragma omp target update from(arr[2][0:1:2])
-// OMP5-NEXT: #pragma omp target update to(present: arr[2][0:1:2])
-// OMP5-NEXT: #pragma omp target update from(present: arr[2][0:1:2], a)
 
 int main(int argc, char **argv) {
   static int a;
@@ -79,13 +62,6 @@ int main(int argc, char **argv) {
 // CHECK-NEXT: #pragma omp target update to(argv[2][0:1:2])
 #pragma omp target update from(argv[2][0:1:2])
 // CHECK-NEXT: #pragma omp target update from(argv[2][0:1:2])
-#ifdef OMP51
-#pragma omp target update to(present: argv[2][0:1:2])
-// OMP5-NEXT: #pragma omp target update to(present: arr[2][0:1:2])
-#pragma omp target update from(argv[2][0:1:2], a)
-// OMP5-NEXT: #pragma omp target update from(present: arr[2][0:1:2], a)
-#endif
-
 
   return foo(argc, f) + foo(argv[0][0], f) + a;
 }
diff --git a/clang/test/OpenMP/target_update_codegen.cpp b/clang/test/OpenMP/target_update_codegen.cpp
index feea1129be4cc..637b580d754f2 100644
--- a/clang/test/OpenMP/target_update_codegen.cpp
+++ b/clang/test/OpenMP/target_update_codegen.cpp
@@ -1084,68 +1084,5 @@ void array_shaping(float *f, int sa) {
   #pragma omp target update from(([sa][5])f)
 }
 
-#endif
-///==========================================================================///
-// RUN: %clang_cc1 -DCK19 -verify -fopenmp -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK19 --check-prefix CK19-64
-// RUN: %clang_cc1 -DCK19 -fopenmp -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK19 --check-prefix CK19-64
-// RUN: %clang_cc1 -DCK19 -verify -fopenmp -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK19 --check-prefix CK19-32
-// RUN: %clang_cc1 -DCK19 -fopenmp -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK19 --check-prefix CK19-32
-
-// RUN: %clang_cc1 -DCK19 -verify -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -DCK19 -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -DCK19 -verify -fopenmp-version=51 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -DCK19 -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
-#ifdef CK19
-
-// PRESENT=0x1000 | TARGET_PARAM=0x20 | TO=0x1 = 0x1021
-// CK19: [[MTYPE00:@.+]] = {{.+}}constant [1 x i64] [i64 [[#0x1021]]]
-
-// PRESENT=0x1000 | TARGET_PARAM=0x20 | FROM=0x2 = 0x1022
-// CK19: [[MTYPE01:@.+]] = {{.+}}constant [1 x i64] [i64 [[#0x1022]]]
-
-// CK19-LABEL: _Z13check_presenti
-void check_present(int arg) {
-  int la;
-  float lb[arg];
-
-  // Region 00
-  // CK19-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 1, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i64* [[GEPS:%.+]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
-  // CK19-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
-  // CK19-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
-  // CK19-DAG: [[GEPS]] = getelementptr inbounds {{.+}}[[S:%[^,]+]]
-
-  // CK19-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 0
-  // CK19-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 0
-  // CK19-DAG: [[S0:%.+]] = getelementptr inbounds {{.+}}[[S]], i{{.+}} 0, i{{.+}} 0
-  // CK19-DAG: [[CBP0:%.+]] = bitcast i8** [[BP0]] to float**
-  // CK19-DAG: [[CP0:%.+]] = bitcast i8** [[P0]] to float**
-  // CK19-DAG: store float* [[VAL0:%[^,]+]], float** [[CBP0]]
-  // CK19-DAG: store float* [[VAL0]], float** [[CP0]]
-  // CK19-DAG: store i64 [[CSVAL0:%[^,]+]], i64* [[S0]]
-  #pragma omp target update to(present: lb)
-  ;
-
-  // Region 01
-  // CK19-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 1, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i64* [[GEPS:%.+]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[MTYPE01]]{{.+}}, i8** null)
-  // CK19-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
-  // CK19-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
-  // CK19-DAG: [[GEPS]] = getelementptr inbounds {{.+}}[[S:%[^,]+]]
-
-  // CK19-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 0
-  // CK19-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 0
-  // CK19-DAG: [[S0:%.+]] = getelementptr inbounds {{.+}}[[S]], i{{.+}} 0, i{{.+}} 0
-  // CK19-DAG: [[CBP0:%.+]] = bitcast i8** [[BP0]] to float**
-  // CK19-DAG: [[CP0:%.+]] = bitcast i8** [[P0]] to float**
-  // CK19-DAG: store float* [[VAL0:%[^,]+]], float** [[CBP0]]
-  // CK19-DAG: store float* [[VAL0]], float** [[CP0]]
-  // CK19-DAG: store i64 [[CSVAL0:%[^,]+]], i64* [[S0]]
-  #pragma omp target update from(present: lb)
-  ;
-}
 #endif
 #endif
diff --git a/clang/test/OpenMP/target_update_messages.cpp b/clang/test/OpenMP/target_update_messages.cpp
index 401941b63028c..4092b623bdc97 100644
--- a/clang/test/OpenMP/target_update_messages.cpp
+++ b/clang/test/OpenMP/target_update_messages.cpp
@@ -1,10 +1,8 @@
-// RUN: %clang_cc1 -verify=expected,lt50,lt51 -fopenmp -fopenmp-version=45 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
-// RUN: %clang_cc1 -verify=expected,ge50,lt51 -fopenmp -fopenmp-version=50 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
-// RUN: %clang_cc1 -verify=expected,ge50,ge51 -fopenmp -fopenmp-version=51 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp45 -fopenmp -fopenmp-version=45 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp50 -fopenmp -fopenmp-version=50 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
 
-// RUN: %clang_cc1 -verify=expected,lt50,lt51 -fopenmp-simd -fopenmp-version=45 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
-// RUN: %clang_cc1 -verify=expected,ge50,lt51 -fopenmp-simd -fopenmp-version=50 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
-// RUN: %clang_cc1 -verify=expected,ge50,ge51 -fopenmp-simd -fopenmp-version=51 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp45 -fopenmp-simd -fopenmp-version=45 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,omp50 -fopenmp-simd -fopenmp-version=50 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
 
 void xxx(int argc) {
   int x; // expected-note {{initialize the variable 'x' to silence this warning}}
@@ -27,10 +25,6 @@ int tmain(T argc, S **argv) {
   return 0;
 }
 
-struct S {
-  int i;
-};
-
 int main(int argc, char **argv) {
   int m;
   #pragma omp target update // expected-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
@@ -40,114 +34,15 @@ int main(int argc, char **argv) {
   #pragma omp target update to(m) ] // expected-warning {{extra tokens at the end of '#pragma omp target update' are ignored}}
   #pragma omp target update to(m) ) // expected-warning {{extra tokens at the end of '#pragma omp target update' are ignored}}
 
-  #pragma omp declare mapper(id: S s) map(s.i)
-  S s;
-
-  // Check parsing with no modifiers.
-  // lt51-error@+2 {{expected expression}}
-  // lt51-error@+1 {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
-  #pragma omp target update to(: s)
-  // expected-error@+2 {{expected expression}}
-  // expected-error@+1 {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
-  #pragma omp target update to(:)
-  // expected-error@+2 2 {{expected expression}}
-  // expected-error@+1 {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
-  #pragma omp target update to(,:)
-
-  // Check parsing with one modifier.
-  // expected-error@+2 {{use of undeclared identifier 'foobar'}}
-  // expected-error@+1 {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
-  #pragma omp target update to(foobar: s)
-  // expected-error@+3 {{expected ',' or ')' in 'to' clause}}
-  // expected-error@+2 {{expected ')'}}
-  // expected-note@+1 {{to match this '('}}
-  #pragma omp target update to(m: s)
-  #pragma omp target update to(mapper(id): s)
-  // lt51-error@+2 {{use of undeclared identifier 'present'}}
-  // lt51-error@+1 {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
-  #pragma omp target update to(present: s)
-  // ge51-warning@+4 {{missing ':' after motion modifier - ignoring}}
-  // lt51-warning@+3 {{missing ':' after ) - ignoring}}
-  // expected-error@+2 {{expected expression}}
-  // expected-error@+1 {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
-  #pragma omp target update to(mapper(id) s)
-  // ge51-warning@+4 {{missing ':' after motion modifier - ignoring}}
-  // ge51-error@+3 {{expected expression}}
-  // lt51-error@+2 {{use of undeclared identifier 'present'}}
-  // expected-error@+1 {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
-  #pragma omp target update to(present s)
-  // ge51-warning@+4 {{missing ':' after motion modifier - ignoring}}
-  // lt51-warning@+3 {{missing ':' after ) - ignoring}}
-  // expected-error@+2 {{expected expression}}
-  // expected-error@+1 {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
-  #pragma omp target update to(mapper(id))
-  // ge51-warning@+4 {{missing ':' after motion modifier - ignoring}}
-  // ge51-error@+3 {{expected expression}}
-  // lt51-error@+2 {{use of undeclared identifier 'present'}}
-  // expected-error@+1 {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
-  #pragma omp target update to(present)
-  // expected-error@+2 {{expected expression}}
-  // expected-error@+1 {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
-  #pragma omp target update to(mapper(id):)
-  // ge51-error@+3 {{expected expression}}
-  // lt51-error@+2 {{use of undeclared identifier 'present'}}
-  // expected-error@+1 {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
-  #pragma omp target update to(present:)
-
-  // Check parsing with two modifiers.
-  // lt51-warning@+1 {{missing ':' after ) - ignoring}}
-  #pragma omp target update to(mapper(id), present: s)
-  // lt51-error@+3 {{use of undeclared identifier 'present'}}
-  // lt51-error@+2 {{use of undeclared identifier 'id'}}
-  // lt51-error@+1 {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
-  #pragma omp target update to(present, mapper(id): s)
-  // lt51-warning@+1 {{missing ':' after ) - ignoring}}
-  #pragma omp target update to(mapper(id) present: s)
-  // lt51-error@+2 {{use of undeclared identifier 'present'}}
-  // lt51-error@+1 {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
-  #pragma omp target update to(present mapper(id): s)
-
-  // Check parsing with unnecessary commas.
-  // lt51-warning@+1 {{missing ':' after ) - ignoring}}
-  #pragma omp target update to(mapper(id),: s)
-  // lt51-error@+3 {{use of undeclared identifier 'present'}}
-  // lt51-error@+2 {{expected expression}}
-  // lt51-error@+1 {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
-  #pragma omp target update to(present , : s)
-  // ge51-warning@+2 {{missing ':' after motion modifier - ignoring}}
-  // lt51-warning@+1 {{missing ':' after ) - ignoring}}
-  #pragma omp target update to(mapper(id),,: s)
-  // ge51-warning@+5 {{missing ':' after motion modifier - ignoring}}
-  // lt51-error@+4 {{use of undeclared identifier 'present'}}
-  // lt51-error@+3 {{expected expression}}
-  // lt51-error@+2 {{expected expression}}
-  // lt51-error@+1 {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
-  #pragma omp target update to(present,,: s)
-  // lt51-warning@+1 {{missing ':' after ) - ignoring}}
-  #pragma omp target update to(mapper(id), present,: s)
-  // lt51-error@+4 {{use of undeclared identifier 'present'}}
-  // lt51-error@+3 {{use of undeclared identifier 'id'}}
-  // lt51-error@+2 {{expected expression}}
-  // lt51-error@+1 {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
-  #pragma omp target update to(present, mapper(id),: s)
-
   #pragma omp target update from(m) allocate(m) // expected-error {{unexpected OpenMP clause 'allocate' in directive '#pragma omp target update'}}
   {
     foo();
   }
 
   int iarr[5][5];
-// ge50-error@+4 {{section stride is evaluated to a non-positive value -1}}
-// lt50-error@+3 {{expected ']'}}
-// lt50-note@+2 {{to match this '['}}
-// expected-error@+1 {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
-#pragma omp target update to(iarr[0:][1:2:-1])
+#pragma omp target update to(iarr[0:][1:2:-1]) // omp50-error {{section stride is evaluated to a non-positive value -1}} omp45-error {{expected ']'}} omp45-note {{to match this '['}} expected-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
   {}
-// ge50-error@+4 {{section stride is evaluated to a non-positive value -1}}
-// lt50-error@+3 {{expected ']'}}
-// lt50-note@+2 {{to match this '['}}
-// expected-error@+1 {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
-#pragma omp target update from(iarr[0:][1:2:-1])
+#pragma omp target update from(iarr[0:][1:2:-1]) // omp50-error {{section stride is evaluated to a non-positive value -1}} omp45-error {{expected ']'}} omp45-note {{to match this '['}} expected-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
 
   return tmain(argc, argv);
 }

From 30440924d48cdcbdef1dd7b2791171d821a859ae Mon Sep 17 00:00:00 2001
From: Shilei Tian <tianshilei1992@gmail.com>
Date: Tue, 28 Jul 2020 20:31:34 -0400
Subject: [PATCH 0401/1035] [OpenMP] Replaced mutex lock/unlock in `target`
 with `std::lock_guard`

Reviewed By: ye-luo

Differential Revision: https://reviews.llvm.org/D84799
---
 openmp/libomptarget/src/omptarget.cpp | 85 ++++++++++++++-------------
 1 file changed, 44 insertions(+), 41 deletions(-)

diff --git a/openmp/libomptarget/src/omptarget.cpp b/openmp/libomptarget/src/omptarget.cpp
index f109c6acc8ca3..d391e245fdd66 100644
--- a/openmp/libomptarget/src/omptarget.cpp
+++ b/openmp/libomptarget/src/omptarget.cpp
@@ -722,38 +722,38 @@ int target(int64_t DeviceId, void *HostPtr, int32_t ArgNum, void **ArgBases,
   // Find the table information in the map or look it up in the translation
   // tables.
   TableMap *TM = 0;
-  TblMapMtx->lock();
-  HostPtrToTableMapTy::iterator TableMapIt = HostPtrToTableMap->find(HostPtr);
-  if (TableMapIt == HostPtrToTableMap->end()) {
-    // We don't have a map. So search all the registered libraries.
-    TrlTblMtx->lock();
-    for (HostEntriesBeginToTransTableTy::iterator
-             II = HostEntriesBeginToTransTable->begin(),
-             IE = HostEntriesBeginToTransTable->end();
-         !TM && II != IE; ++II) {
-      // get the translation table (which contains all the good info).
-      TranslationTable *TransTable = &II->second;
-      // iterate over all the host table entries to see if we can locate the
-      // host_ptr.
-      __tgt_offload_entry *Begin = TransTable->HostTable.EntriesBegin;
-      __tgt_offload_entry *End = TransTable->HostTable.EntriesEnd;
-      __tgt_offload_entry *Cur = Begin;
-      for (uint32_t I = 0; Cur < End; ++Cur, ++I) {
-        if (Cur->addr != HostPtr)
-          continue;
-        // we got a match, now fill the HostPtrToTableMap so that we
-        // may avoid this search next time.
-        TM = &(*HostPtrToTableMap)[HostPtr];
-        TM->Table = TransTable;
-        TM->Index = I;
-        break;
+  {
+    std::lock_guard<std::mutex> TblMapLock(*TblMapMtx);
+    HostPtrToTableMapTy::iterator TableMapIt = HostPtrToTableMap->find(HostPtr);
+    if (TableMapIt == HostPtrToTableMap->end()) {
+      // We don't have a map. So search all the registered libraries.
+      std::lock_guard<std::mutex> TrlTblLock(*TrlTblMtx);
+      for (HostEntriesBeginToTransTableTy::iterator
+               II = HostEntriesBeginToTransTable->begin(),
+               IE = HostEntriesBeginToTransTable->end();
+           !TM && II != IE; ++II) {
+        // get the translation table (which contains all the good info).
+        TranslationTable *TransTable = &II->second;
+        // iterate over all the host table entries to see if we can locate the
+        // host_ptr.
+        __tgt_offload_entry *Begin = TransTable->HostTable.EntriesBegin;
+        __tgt_offload_entry *End = TransTable->HostTable.EntriesEnd;
+        __tgt_offload_entry *Cur = Begin;
+        for (uint32_t I = 0; Cur < End; ++Cur, ++I) {
+          if (Cur->addr != HostPtr)
+            continue;
+          // we got a match, now fill the HostPtrToTableMap so that we
+          // may avoid this search next time.
+          TM = &(*HostPtrToTableMap)[HostPtr];
+          TM->Table = TransTable;
+          TM->Index = I;
+          break;
+        }
       }
+    } else {
+      TM = &TableMapIt->second;
     }
-    TrlTblMtx->unlock();
-  } else {
-    TM = &TableMapIt->second;
   }
-  TblMapMtx->unlock();
 
   // No map for this host pointer found!
   if (!TM) {
@@ -763,11 +763,13 @@ int target(int64_t DeviceId, void *HostPtr, int32_t ArgNum, void **ArgBases,
   }
 
   // get target table.
-  TrlTblMtx->lock();
-  assert(TM->Table->TargetsTable.size() > (size_t)DeviceId &&
-         "Not expecting a device ID outside the table's bounds!");
-  __tgt_target_table *TargetTable = TM->Table->TargetsTable[DeviceId];
-  TrlTblMtx->unlock();
+  __tgt_target_table *TargetTable = nullptr;
+  {
+    std::lock_guard<std::mutex> TrlTblLock(*TrlTblMtx);
+    assert(TM->Table->TargetsTable.size() > (size_t)DeviceId &&
+           "Not expecting a device ID outside the table's bounds!");
+    TargetTable = TM->Table->TargetsTable[DeviceId];
+  }
   assert(TargetTable && "Global data has not been mapped\n");
 
   __tgt_async_info AsyncInfo;
@@ -899,14 +901,15 @@ int target(int64_t DeviceId, void *HostPtr, int32_t ArgNum, void **ArgBases,
 
   // Pop loop trip count
   uint64_t LoopTripCount = 0;
-  TblMapMtx->lock();
-  auto I = Device.LoopTripCnt.find(__kmpc_global_thread_num(NULL));
-  if (I != Device.LoopTripCnt.end()) {
-    LoopTripCount = I->second;
-    Device.LoopTripCnt.erase(I);
-    DP("loop trip count is %lu.\n", LoopTripCount);
+  {
+    std::lock_guard<std::mutex> TblMapLock(*TblMapMtx);
+    auto I = Device.LoopTripCnt.find(__kmpc_global_thread_num(NULL));
+    if (I != Device.LoopTripCnt.end()) {
+      LoopTripCount = I->second;
+      Device.LoopTripCnt.erase(I);
+      DP("loop trip count is %lu.\n", LoopTripCount);
+    }
   }
-  TblMapMtx->unlock();
 
   // Launch device execution.
   DP("Launching target execution %s with pointer " DPxMOD " (index=%d).\n",

From 068808d102846d551e683241d0b3b9607b3bdf8d Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Sat, 25 Jul 2020 20:16:15 -0400
Subject: [PATCH 0402/1035] AMDGPU: Don't assume call targets are registers

GlobalISel let through a call to null, which would then fold into the
source operand like any other inline immediate. The SelectionDAG
lowering deletes calls to null and undef as a workaround from before
calls were supported. We should probably drop the special handling
case in the DAG lowering now, since the middle end optimizers delete
null calls anyway.
---
 llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp | 26 +++++++++---------
 llvm/test/CodeGen/AMDGPU/call-constant.ll   | 29 +++++++++++++--------
 2 files changed, 32 insertions(+), 23 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index 2a157eb20ab47..5ab6edf3f606b 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -963,26 +963,28 @@ bool SIInsertWaitcnts::generateWaitcntInstBefore(
 
       int CallAddrOpIdx =
           AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::src0);
-      RegInterval CallAddrOpInterval =
+
+      if (MI.getOperand(CallAddrOpIdx).isReg()) {
+        RegInterval CallAddrOpInterval =
           ScoreBrackets.getRegInterval(&MI, TII, MRI, TRI, CallAddrOpIdx);
 
-      for (int RegNo = CallAddrOpInterval.first;
-           RegNo < CallAddrOpInterval.second; ++RegNo)
-        ScoreBrackets.determineWait(
+        for (int RegNo = CallAddrOpInterval.first;
+             RegNo < CallAddrOpInterval.second; ++RegNo)
+          ScoreBrackets.determineWait(
             LGKM_CNT, ScoreBrackets.getRegScore(RegNo, LGKM_CNT), Wait);
 
-      int RtnAddrOpIdx =
-            AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::dst);
-      if (RtnAddrOpIdx != -1) {
-        RegInterval RtnAddrOpInterval =
+        int RtnAddrOpIdx =
+          AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::dst);
+        if (RtnAddrOpIdx != -1) {
+          RegInterval RtnAddrOpInterval =
             ScoreBrackets.getRegInterval(&MI, TII, MRI, TRI, RtnAddrOpIdx);
 
-        for (int RegNo = RtnAddrOpInterval.first;
-             RegNo < RtnAddrOpInterval.second; ++RegNo)
-          ScoreBrackets.determineWait(
+          for (int RegNo = RtnAddrOpInterval.first;
+               RegNo < RtnAddrOpInterval.second; ++RegNo)
+            ScoreBrackets.determineWait(
               LGKM_CNT, ScoreBrackets.getRegScore(RegNo, LGKM_CNT), Wait);
+        }
       }
-
     } else {
       // FIXME: Should not be relying on memoperands.
       // Look at the source operands of every instruction to see if
diff --git a/llvm/test/CodeGen/AMDGPU/call-constant.ll b/llvm/test/CodeGen/AMDGPU/call-constant.ll
index 11f4b3c0b913a..9a52ccf541dfb 100644
--- a/llvm/test/CodeGen/AMDGPU/call-constant.ll
+++ b/llvm/test/CodeGen/AMDGPU/call-constant.ll
@@ -1,11 +1,12 @@
-; RUN: llc -mtriple=amdgcn-amd-amdhsa < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -global-isel=0 -amdgpu-fixed-function-abi=0 -mtriple=amdgcn-amd-amdhsa < %s | FileCheck -check-prefixes=GCN,SDAG %s
+; RUN: llc -global-isel=1 -amdgpu-fixed-function-abi=1 -mtriple=amdgcn-amd-amdhsa < %s | FileCheck -check-prefixes=GCN,GISEL %s
 
 ; FIXME: Emitting unnecessary flat_scratch setup
 
 ; GCN-LABEL: {{^}}test_call_undef:
-; GCN: s_mov_b32 flat_scratch_lo, s5
-; GCN: s_add_u32 s4, s4, s7
-; GCN: s_lshr_b32
+; SDAG: s_mov_b32 flat_scratch_lo, s5
+; SDAG: s_add_u32 s4, s4, s7
+; SDAG: s_lshr_b32
 ; GCN: s_endpgm
 define amdgpu_kernel void @test_call_undef() #0 {
   %val = call i32 undef(i32 1)
@@ -15,17 +16,21 @@ define amdgpu_kernel void @test_call_undef() #0 {
 }
 
 ; GCN-LABEL: {{^}}test_tail_call_undef:
-; GCN: s_waitcnt
-; GCN-NEXT: .Lfunc_end
+; SDAG: s_waitcnt
+; SDAG-NEXT: .Lfunc_end
+
+; GISEL: s_swappc_b64 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}
 define i32 @test_tail_call_undef() #0 {
   %call = tail call i32 undef(i32 1)
   ret i32 %call
 }
 
 ; GCN-LABEL: {{^}}test_call_null:
-; GCN: s_mov_b32 flat_scratch_lo, s5
-; GCN: s_add_u32 s4, s4, s7
-; GCN: s_lshr_b32
+; SDAG: s_mov_b32 flat_scratch_lo, s5
+; SDAG: s_add_u32 s4, s4, s7
+; SDAG: s_lshr_b32
+
+; GISEL: s_swappc_b64 s{{\[[0-9]+:[0-9]+\]}}, 0{{$}}
 ; GCN: s_endpgm
 define amdgpu_kernel void @test_call_null() #0 {
   %val = call i32 null(i32 1)
@@ -35,8 +40,10 @@ define amdgpu_kernel void @test_call_null() #0 {
 }
 
 ; GCN-LABEL: {{^}}test_tail_call_null:
-; GCN: s_waitcnt
-; GCN-NEXT: .Lfunc_end
+; SDAG: s_waitcnt
+; SDAG-NEXT: .Lfunc_end
+
+; GISEL: s_swappc_b64 s{{\[[0-9]+:[0-9]+\]}}, 0{{$}}
 define i32 @test_tail_call_null() #0 {
   %call = tail call i32 null(i32 1)
   ret i32 %call

From ffd8c23ccb74bbd19ba16e3706a0c5e1e472cca6 Mon Sep 17 00:00:00 2001
From: Thomas Lively <tlively@google.com>
Date: Tue, 28 Jul 2020 17:46:45 -0700
Subject: [PATCH 0403/1035] [WebAssembly] Implement truncating vector stores

Rather than expanding truncating stores so that vectors are stored one
lane at a time, lower them to a sequence of instructions using
narrowing operations instead, when possible. Since the narrowing
operations have saturating semantics, but truncating stores require
truncation, mask the stored value to manually truncate it before
narrowing. Also, since narrowing is a binary operation, pass in the
original vector as the unused second argument.

Differential Revision: https://reviews.llvm.org/D84377
---
 .../WebAssembly/WebAssemblyISelLowering.cpp   |   3 +
 .../WebAssembly/WebAssemblyInstrSIMD.td       | 100 +++++-
 llvm/test/CodeGen/WebAssembly/simd-offset.ll  | 320 ++++++++++++++++++
 3 files changed, 418 insertions(+), 5 deletions(-)

diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
index c6519fafbc491..d6197e9ea84c2 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
@@ -246,6 +246,9 @@ WebAssemblyTargetLowering::WebAssemblyTargetLowering(
       setLoadExtAction(Ext, MVT::v4i32, MVT::v4i16, Legal);
       setLoadExtAction(Ext, MVT::v2i64, MVT::v2i32, Legal);
     }
+    // And some truncating stores are legal as well
+    setTruncStoreAction(MVT::v8i16, MVT::v8i8, Legal);
+    setTruncStoreAction(MVT::v4i32, MVT::v4i16, Legal);
   }
 
   // Don't do anything clever with build_pairs
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td b/llvm/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td
index b603701ab930e..16bfc81af8b72 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td
@@ -885,6 +885,12 @@ defm "" : SIMDConvert<v4i32, v4f32, fp_to_uint, "i32x4.trunc_sat_f32x4_u", 249>;
 defm "" : SIMDConvert<v4f32, v4i32, sint_to_fp, "f32x4.convert_i32x4_s", 250>;
 defm "" : SIMDConvert<v4f32, v4i32, uint_to_fp, "f32x4.convert_i32x4_u", 251>;
 
+// Lower llvm.wasm.trunc.saturate.* to saturating instructions
+def : Pat<(v4i32 (int_wasm_trunc_saturate_signed (v4f32 V128:$src))),
+          (fp_to_sint_v4i32_v4f32 (v4f32 V128:$src))>;
+def : Pat<(v4i32 (int_wasm_trunc_saturate_unsigned (v4f32 V128:$src))),
+          (fp_to_uint_v4i32_v4f32 (v4f32 V128:$src))>;
+
 // Widening operations
 multiclass SIMDWiden<ValueType vec_t, string vec, ValueType arg_t, string arg,
                      bits<32> baseInst> {
@@ -921,11 +927,95 @@ multiclass SIMDNarrow<ValueType vec_t, string vec, ValueType arg_t, string arg,
 defm "" : SIMDNarrow<v16i8, "i8x16", v8i16, "i16x8", 101>;
 defm "" : SIMDNarrow<v8i16, "i16x8", v4i32, "i32x4", 133>;
 
-// Lower llvm.wasm.trunc.saturate.* to saturating instructions
-def : Pat<(v4i32 (int_wasm_trunc_saturate_signed (v4f32 V128:$src))),
-          (fp_to_sint_v4i32_v4f32 (v4f32 V128:$src))>;
-def : Pat<(v4i32 (int_wasm_trunc_saturate_unsigned (v4f32 V128:$src))),
-          (fp_to_uint_v4i32_v4f32 (v4f32 V128:$src))>;
+// Use narrowing operations for truncating stores. Since the narrowing
+// operations are saturating instead of truncating, we need to mask
+// the stored values first.
+// TODO: Use consts instead of splats
+def store_v8i8_trunc_v8i16 :
+  OutPatFrag<(ops node:$val),
+             (EXTRACT_LANE_v2i64
+               (NARROW_U_v16i8
+                 (AND_v4i32 (SPLAT_v4i32 (CONST_I32 0x00ff00ff)), node:$val),
+                 node:$val // Unused input
+               ),
+               0
+             )>;
+
+def store_v4i16_trunc_v4i32 :
+  OutPatFrag<(ops node:$val),
+             (EXTRACT_LANE_v2i64
+               (NARROW_U_v8i16
+                 (AND_v4i32 (SPLAT_v4i32 (CONST_I32 0x0000ffff)), node:$val),
+                 node:$val // Unused input
+               ),
+               0
+             )>;
+
+// Store patterns adapted from WebAssemblyInstrMemory.td
+multiclass NarrowingStorePatNoOffset<ValueType ty, PatFrag node,
+                                     OutPatFrag out> {
+  def : Pat<(node ty:$val, I32:$addr),
+            (STORE_I64_A32 0, 0, I32:$addr, (i64 (out ty:$val)))>,
+        Requires<[HasAddr32]>;
+  def : Pat<(node ty:$val, I64:$addr),
+            (STORE_I64_A64 0, 0, I64:$addr, (i64 (out ty:$val)))>,
+        Requires<[HasAddr64]>;
+}
+
+defm : NarrowingStorePatNoOffset<v8i16, truncstorevi8, store_v8i8_trunc_v8i16>;
+defm : NarrowingStorePatNoOffset<v4i32, truncstorevi16,
+                                 store_v4i16_trunc_v4i32>;
+
+multiclass NarrowingStorePatImmOff<ValueType ty, PatFrag kind,
+                                   PatFrag operand, OutPatFrag out> {
+  def : Pat<(kind ty:$val, (operand I32:$addr, imm:$off)),
+            (STORE_I64_A32 0, imm:$off, I32:$addr, (i64 (out ty:$val)))>,
+        Requires<[HasAddr32]>;
+  def : Pat<(kind ty:$val, (operand I64:$addr, imm:$off)),
+            (STORE_I64_A64 0, imm:$off, I64:$addr, (i64 (out ty:$val)))>,
+        Requires<[HasAddr64]>;
+}
+
+defm : NarrowingStorePatImmOff<v8i16, truncstorevi8, regPlusImm,
+                               store_v8i8_trunc_v8i16>;
+defm : NarrowingStorePatImmOff<v4i32, truncstorevi16, regPlusImm,
+                               store_v4i16_trunc_v4i32>;
+defm : NarrowingStorePatImmOff<v8i16, truncstorevi8, or_is_add,
+                               store_v8i8_trunc_v8i16>;
+defm : NarrowingStorePatImmOff<v4i32, truncstorevi16, or_is_add,
+                               store_v4i16_trunc_v4i32>;
+
+multiclass NarrowingStorePatOffsetOnly<ValueType ty, PatFrag kind,
+                                       OutPatFrag out> {
+  def : Pat<(kind ty:$val, imm:$off),
+            (STORE_I64_A32 0, imm:$off, (CONST_I32 0), (i64 (out ty:$val)))>,
+        Requires<[HasAddr32]>;
+  def : Pat<(kind ty:$val, imm:$off),
+            (STORE_I64_A64 0, imm:$off, (CONST_I64 0), (i64 (out ty:$val)))>,
+        Requires<[HasAddr64]>;
+}
+
+defm : NarrowingStorePatOffsetOnly<v8i16, truncstorevi8,
+                                   store_v8i8_trunc_v8i16>;
+defm : NarrowingStorePatOffsetOnly<v4i32, truncstorevi16,
+                                   store_v4i16_trunc_v4i32>;
+
+multiclass NarrowingStorePatGlobalAddrOffOnly<ValueType ty, PatFrag kind,
+                                              OutPatFrag out> {
+  def : Pat<(kind ty:$val, (WebAssemblywrapper tglobaladdr:$off)),
+            (STORE_I64_A32
+              0, tglobaladdr:$off, (CONST_I32 0), (i64 (out ty:$val)))>,
+        Requires<[IsNotPIC, HasAddr32]>;
+  def : Pat<(kind ty:$val, (WebAssemblywrapper tglobaladdr:$off)),
+            (STORE_I64_A64
+              0, tglobaladdr:$off, (CONST_I64 0), (i64 (out ty:$val)))>,
+        Requires<[IsNotPIC, HasAddr64]>;
+}
+
+defm : NarrowingStorePatGlobalAddrOffOnly<v8i16, truncstorevi8,
+                                          store_v8i8_trunc_v8i16>;
+defm : NarrowingStorePatGlobalAddrOffOnly<v4i32, truncstorevi16,
+                                          store_v4i16_trunc_v4i32>;
 
 // Bitcasts are nops
 // Matching bitcast t1 to t1 causes strange errors, so avoid repeating types
diff --git a/llvm/test/CodeGen/WebAssembly/simd-offset.ll b/llvm/test/CodeGen/WebAssembly/simd-offset.ll
index 8d39ddc20f244..b2d32936df130 100644
--- a/llvm/test/CodeGen/WebAssembly/simd-offset.ll
+++ b/llvm/test/CodeGen/WebAssembly/simd-offset.ll
@@ -918,6 +918,24 @@ define void @store_v8i16(<8 x i16> %v, <8 x i16>* %p) {
   ret void
 }
 
+define void @store_narrowing_v8i16(<8 x i8> %v, <8 x i8>* %p) {
+; CHECK-LABEL: store_narrowing_v8i16:
+; CHECK:         .functype store_narrowing_v8i16 (v128, i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    i32.const 16711935
+; CHECK-NEXT:    i32x4.splat
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.and
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i8x16.narrow_i16x8_u
+; CHECK-NEXT:    i64x2.extract_lane 0
+; CHECK-NEXT:    i64.store 0
+; CHECK-NEXT:    # fallthrough-return
+  store <8 x i8> %v, <8 x i8>* %p
+  ret void
+}
+
 define void @store_v8i16_with_folded_offset(<8 x i16> %v, <8 x i16>* %p) {
 ; CHECK-LABEL: store_v8i16_with_folded_offset:
 ; CHECK:         .functype store_v8i16_with_folded_offset (v128, i32) -> ()
@@ -933,6 +951,27 @@ define void @store_v8i16_with_folded_offset(<8 x i16> %v, <8 x i16>* %p) {
   ret void
 }
 
+define void @store_narrowing_v8i16_with_folded_offset(<8 x i8> %v, <8 x i8>* %p) {
+; CHECK-LABEL: store_narrowing_v8i16_with_folded_offset:
+; CHECK:         .functype store_narrowing_v8i16_with_folded_offset (v128, i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    i32.const 16711935
+; CHECK-NEXT:    i32x4.splat
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.and
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i8x16.narrow_i16x8_u
+; CHECK-NEXT:    i64x2.extract_lane 0
+; CHECK-NEXT:    i64.store 16
+; CHECK-NEXT:    # fallthrough-return
+  %q = ptrtoint <8 x i8>* %p to i32
+  %r = add nuw i32 %q, 16
+  %s = inttoptr i32 %r to <8 x i8>*
+  store <8 x i8> %v , <8 x i8>* %s
+  ret void
+}
+
 define void @store_v8i16_with_folded_gep_offset(<8 x i16> %v, <8 x i16>* %p) {
 ; CHECK-LABEL: store_v8i16_with_folded_gep_offset:
 ; CHECK:         .functype store_v8i16_with_folded_gep_offset (v128, i32) -> ()
@@ -946,6 +985,25 @@ define void @store_v8i16_with_folded_gep_offset(<8 x i16> %v, <8 x i16>* %p) {
   ret void
 }
 
+define void @store_narrowing_v8i16_with_folded_gep_offset(<8 x i8> %v, <8 x i8>* %p) {
+; CHECK-LABEL: store_narrowing_v8i16_with_folded_gep_offset:
+; CHECK:         .functype store_narrowing_v8i16_with_folded_gep_offset (v128, i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    i32.const 16711935
+; CHECK-NEXT:    i32x4.splat
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.and
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i8x16.narrow_i16x8_u
+; CHECK-NEXT:    i64x2.extract_lane 0
+; CHECK-NEXT:    i64.store 8
+; CHECK-NEXT:    # fallthrough-return
+  %s = getelementptr inbounds <8 x i8>, <8 x i8>* %p, i32 1
+  store <8 x i8> %v , <8 x i8>* %s
+  ret void
+}
+
 define void @store_v8i16_with_unfolded_gep_negative_offset(<8 x i16> %v, <8 x i16>* %p) {
 ; CHECK-LABEL: store_v8i16_with_unfolded_gep_negative_offset:
 ; CHECK:         .functype store_v8i16_with_unfolded_gep_negative_offset (v128, i32) -> ()
@@ -961,6 +1019,27 @@ define void @store_v8i16_with_unfolded_gep_negative_offset(<8 x i16> %v, <8 x i1
   ret void
 }
 
+define void @store_narrowing_v8i16_with_unfolded_gep_negative_offset(<8 x i8> %v, <8 x i8>* %p) {
+; CHECK-LABEL: store_narrowing_v8i16_with_unfolded_gep_negative_offset:
+; CHECK:         .functype store_narrowing_v8i16_with_unfolded_gep_negative_offset (v128, i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    i32.const -8
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    i32.const 16711935
+; CHECK-NEXT:    i32x4.splat
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.and
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i8x16.narrow_i16x8_u
+; CHECK-NEXT:    i64x2.extract_lane 0
+; CHECK-NEXT:    i64.store 0
+; CHECK-NEXT:    # fallthrough-return
+  %s = getelementptr inbounds <8 x i8>, <8 x i8>* %p, i32 -1
+  store <8 x i8> %v , <8 x i8>* %s
+  ret void
+}
+
 define void @store_v8i16_with_unfolded_offset(<8 x i16> %v, <8 x i16>* %p) {
 ; CHECK-LABEL: store_v8i16_with_unfolded_offset:
 ; CHECK:         .functype store_v8i16_with_unfolded_offset (v128, i32) -> ()
@@ -978,6 +1057,29 @@ define void @store_v8i16_with_unfolded_offset(<8 x i16> %v, <8 x i16>* %p) {
   ret void
 }
 
+define void @store_narrowing_v8i16_with_unfolded_offset(<8 x i8> %v, <8 x i8>* %p) {
+; CHECK-LABEL: store_narrowing_v8i16_with_unfolded_offset:
+; CHECK:         .functype store_narrowing_v8i16_with_unfolded_offset (v128, i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    i32.const 16711935
+; CHECK-NEXT:    i32x4.splat
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.and
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i8x16.narrow_i16x8_u
+; CHECK-NEXT:    i64x2.extract_lane 0
+; CHECK-NEXT:    i64.store 0
+; CHECK-NEXT:    # fallthrough-return
+  %q = ptrtoint <8 x i8>* %p to i32
+  %r = add nsw i32 %q, 16
+  %s = inttoptr i32 %r to <8 x i8>*
+  store <8 x i8> %v , <8 x i8>* %s
+  ret void
+}
+
 define void @store_v8i16_with_unfolded_gep_offset(<8 x i16> %v, <8 x i16>* %p) {
 ; CHECK-LABEL: store_v8i16_with_unfolded_gep_offset:
 ; CHECK:         .functype store_v8i16_with_unfolded_gep_offset (v128, i32) -> ()
@@ -993,6 +1095,27 @@ define void @store_v8i16_with_unfolded_gep_offset(<8 x i16> %v, <8 x i16>* %p) {
   ret void
 }
 
+define void @store_narrowing_v8i16_with_unfolded_gep_offset(<8 x i8> %v, <8 x i8>* %p) {
+; CHECK-LABEL: store_narrowing_v8i16_with_unfolded_gep_offset:
+; CHECK:         .functype store_narrowing_v8i16_with_unfolded_gep_offset (v128, i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    i32.const 8
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    i32.const 16711935
+; CHECK-NEXT:    i32x4.splat
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.and
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i8x16.narrow_i16x8_u
+; CHECK-NEXT:    i64x2.extract_lane 0
+; CHECK-NEXT:    i64.store 0
+; CHECK-NEXT:    # fallthrough-return
+  %s = getelementptr <8 x i8>, <8 x i8>* %p, i32 1
+  store <8 x i8> %v , <8 x i8>* %s
+  ret void
+}
+
 define void @store_v8i16_to_numeric_address(<8 x i16> %v) {
 ; CHECK-LABEL: store_v8i16_to_numeric_address:
 ; CHECK:         .functype store_v8i16_to_numeric_address (v128) -> ()
@@ -1006,6 +1129,25 @@ define void @store_v8i16_to_numeric_address(<8 x i16> %v) {
   ret void
 }
 
+define void @store_narrowing_v8i16_to_numeric_address(<8 x i8> %v, <8 x i8>* %p) {
+; CHECK-LABEL: store_narrowing_v8i16_to_numeric_address:
+; CHECK:         .functype store_narrowing_v8i16_to_numeric_address (v128, i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    i32.const 16711935
+; CHECK-NEXT:    i32x4.splat
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.and
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i8x16.narrow_i16x8_u
+; CHECK-NEXT:    i64x2.extract_lane 0
+; CHECK-NEXT:    i64.store 32
+; CHECK-NEXT:    # fallthrough-return
+  %s = inttoptr i32 32 to <8 x i8>*
+  store <8 x i8> %v , <8 x i8>* %s
+  ret void
+}
+
 define void @store_v8i16_to_global_address(<8 x i16> %v) {
 ; CHECK-LABEL: store_v8i16_to_global_address:
 ; CHECK:         .functype store_v8i16_to_global_address (v128) -> ()
@@ -1018,6 +1160,24 @@ define void @store_v8i16_to_global_address(<8 x i16> %v) {
   ret void
 }
 
+define void @store_narrowing_v8i16_to_global_address(<8 x i8> %v) {
+; CHECK-LABEL: store_narrowing_v8i16_to_global_address:
+; CHECK:         .functype store_narrowing_v8i16_to_global_address (v128) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    i32.const 16711935
+; CHECK-NEXT:    i32x4.splat
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.and
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i8x16.narrow_i16x8_u
+; CHECK-NEXT:    i64x2.extract_lane 0
+; CHECK-NEXT:    i64.store gv_v8i8
+; CHECK-NEXT:    # fallthrough-return
+  store <8 x i8> %v , <8 x i8>* @gv_v8i8
+  ret void
+}
+
 ; ==============================================================================
 ; 4 x i32
 ; ==============================================================================
@@ -1588,6 +1748,24 @@ define void @store_v4i32(<4 x i32> %v, <4 x i32>* %p) {
   ret void
 }
 
+define void @store_narrowing_v4i32(<4 x i16> %v, <4 x i16>* %p) {
+; CHECK-LABEL: store_narrowing_v4i32:
+; CHECK:         .functype store_narrowing_v4i32 (v128, i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    i32.const 65535
+; CHECK-NEXT:    i32x4.splat
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.and
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i16x8.narrow_i32x4_u
+; CHECK-NEXT:    i64x2.extract_lane 0
+; CHECK-NEXT:    i64.store 0
+; CHECK-NEXT:    # fallthrough-return
+  store <4 x i16> %v , <4 x i16>* %p
+  ret void
+}
+
 define void @store_v4i32_with_folded_offset(<4 x i32> %v, <4 x i32>* %p) {
 ; CHECK-LABEL: store_v4i32_with_folded_offset:
 ; CHECK:         .functype store_v4i32_with_folded_offset (v128, i32) -> ()
@@ -1603,6 +1781,27 @@ define void @store_v4i32_with_folded_offset(<4 x i32> %v, <4 x i32>* %p) {
   ret void
 }
 
+define void @store_narrowing_v4i32_with_folded_offset(<4 x i16> %v, <4 x i16>* %p) {
+; CHECK-LABEL: store_narrowing_v4i32_with_folded_offset:
+; CHECK:         .functype store_narrowing_v4i32_with_folded_offset (v128, i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    i32.const 65535
+; CHECK-NEXT:    i32x4.splat
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.and
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i16x8.narrow_i32x4_u
+; CHECK-NEXT:    i64x2.extract_lane 0
+; CHECK-NEXT:    i64.store 16
+; CHECK-NEXT:    # fallthrough-return
+  %q = ptrtoint <4 x i16>* %p to i32
+  %r = add nuw i32 %q, 16
+  %s = inttoptr i32 %r to <4 x i16>*
+  store <4 x i16> %v , <4 x i16>* %s
+  ret void
+}
+
 define void @store_v4i32_with_folded_gep_offset(<4 x i32> %v, <4 x i32>* %p) {
 ; CHECK-LABEL: store_v4i32_with_folded_gep_offset:
 ; CHECK:         .functype store_v4i32_with_folded_gep_offset (v128, i32) -> ()
@@ -1616,6 +1815,25 @@ define void @store_v4i32_with_folded_gep_offset(<4 x i32> %v, <4 x i32>* %p) {
   ret void
 }
 
+define void @store_narrowing_v4i32_with_folded_gep_offset(<4 x i16> %v, <4 x i16>* %p) {
+; CHECK-LABEL: store_narrowing_v4i32_with_folded_gep_offset:
+; CHECK:         .functype store_narrowing_v4i32_with_folded_gep_offset (v128, i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    i32.const 65535
+; CHECK-NEXT:    i32x4.splat
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.and
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i16x8.narrow_i32x4_u
+; CHECK-NEXT:    i64x2.extract_lane 0
+; CHECK-NEXT:    i64.store 8
+; CHECK-NEXT:    # fallthrough-return
+  %s = getelementptr inbounds <4 x i16>, <4 x i16>* %p, i32 1
+  store <4 x i16> %v , <4 x i16>* %s
+  ret void
+}
+
 define void @store_v4i32_with_unfolded_gep_negative_offset(<4 x i32> %v, <4 x i32>* %p) {
 ; CHECK-LABEL: store_v4i32_with_unfolded_gep_negative_offset:
 ; CHECK:         .functype store_v4i32_with_unfolded_gep_negative_offset (v128, i32) -> ()
@@ -1631,6 +1849,27 @@ define void @store_v4i32_with_unfolded_gep_negative_offset(<4 x i32> %v, <4 x i3
   ret void
 }
 
+define void @store_narrowing_v4i32_with_unfolded_gep_negative_offset(<4 x i16> %v, <4 x i16>* %p) {
+; CHECK-LABEL: store_narrowing_v4i32_with_unfolded_gep_negative_offset:
+; CHECK:         .functype store_narrowing_v4i32_with_unfolded_gep_negative_offset (v128, i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    i32.const -8
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    i32.const 65535
+; CHECK-NEXT:    i32x4.splat
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.and
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i16x8.narrow_i32x4_u
+; CHECK-NEXT:    i64x2.extract_lane 0
+; CHECK-NEXT:    i64.store 0
+; CHECK-NEXT:    # fallthrough-return
+  %s = getelementptr inbounds <4 x i16>, <4 x i16>* %p, i32 -1
+  store <4 x i16> %v , <4 x i16>* %s
+  ret void
+}
+
 define void @store_v4i32_with_unfolded_offset(<4 x i32> %v, <4 x i32>* %p) {
 ; CHECK-LABEL: store_v4i32_with_unfolded_offset:
 ; CHECK:         .functype store_v4i32_with_unfolded_offset (v128, i32) -> ()
@@ -1648,6 +1887,29 @@ define void @store_v4i32_with_unfolded_offset(<4 x i32> %v, <4 x i32>* %p) {
   ret void
 }
 
+define void @store_narrowing_v4i32_with_unfolded_offset(<4 x i16> %v, <4 x i16>* %p) {
+; CHECK-LABEL: store_narrowing_v4i32_with_unfolded_offset:
+; CHECK:         .functype store_narrowing_v4i32_with_unfolded_offset (v128, i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    i32.const 65535
+; CHECK-NEXT:    i32x4.splat
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.and
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i16x8.narrow_i32x4_u
+; CHECK-NEXT:    i64x2.extract_lane 0
+; CHECK-NEXT:    i64.store 0
+; CHECK-NEXT:    # fallthrough-return
+  %q = ptrtoint <4 x i16>* %p to i32
+  %r = add nsw i32 %q, 16
+  %s = inttoptr i32 %r to <4 x i16>*
+  store <4 x i16> %v , <4 x i16>* %s
+  ret void
+}
+
 define void @store_v4i32_with_unfolded_gep_offset(<4 x i32> %v, <4 x i32>* %p) {
 ; CHECK-LABEL: store_v4i32_with_unfolded_gep_offset:
 ; CHECK:         .functype store_v4i32_with_unfolded_gep_offset (v128, i32) -> ()
@@ -1663,6 +1925,27 @@ define void @store_v4i32_with_unfolded_gep_offset(<4 x i32> %v, <4 x i32>* %p) {
   ret void
 }
 
+define void @store_narrowing_v4i32_with_unfolded_gep_offset(<4 x i16> %v, <4 x i16>* %p) {
+; CHECK-LABEL: store_narrowing_v4i32_with_unfolded_gep_offset:
+; CHECK:         .functype store_narrowing_v4i32_with_unfolded_gep_offset (v128, i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    i32.const 8
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    i32.const 65535
+; CHECK-NEXT:    i32x4.splat
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.and
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i16x8.narrow_i32x4_u
+; CHECK-NEXT:    i64x2.extract_lane 0
+; CHECK-NEXT:    i64.store 0
+; CHECK-NEXT:    # fallthrough-return
+  %s = getelementptr <4 x i16>, <4 x i16>* %p, i32 1
+  store <4 x i16> %v , <4 x i16>* %s
+  ret void
+}
+
 define void @store_v4i32_to_numeric_address(<4 x i32> %v) {
 ; CHECK-LABEL: store_v4i32_to_numeric_address:
 ; CHECK:         .functype store_v4i32_to_numeric_address (v128) -> ()
@@ -1676,6 +1959,25 @@ define void @store_v4i32_to_numeric_address(<4 x i32> %v) {
   ret void
 }
 
+define void @store_narrowing_v4i32_to_numeric_address(<4 x i16> %v) {
+; CHECK-LABEL: store_narrowing_v4i32_to_numeric_address:
+; CHECK:         .functype store_narrowing_v4i32_to_numeric_address (v128) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    i32.const 65535
+; CHECK-NEXT:    i32x4.splat
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.and
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i16x8.narrow_i32x4_u
+; CHECK-NEXT:    i64x2.extract_lane 0
+; CHECK-NEXT:    i64.store 32
+; CHECK-NEXT:    # fallthrough-return
+  %s = inttoptr i32 32 to <4 x i16>*
+  store <4 x i16> %v , <4 x i16>* %s
+  ret void
+}
+
 define void @store_v4i32_to_global_address(<4 x i32> %v) {
 ; CHECK-LABEL: store_v4i32_to_global_address:
 ; CHECK:         .functype store_v4i32_to_global_address (v128) -> ()
@@ -1688,6 +1990,24 @@ define void @store_v4i32_to_global_address(<4 x i32> %v) {
   ret void
 }
 
+define void @store_narrowing_v4i32_to_global_address(<4 x i16> %v) {
+; CHECK-LABEL: store_narrowing_v4i32_to_global_address:
+; CHECK:         .functype store_narrowing_v4i32_to_global_address (v128) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    i32.const 65535
+; CHECK-NEXT:    i32x4.splat
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.and
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i16x8.narrow_i32x4_u
+; CHECK-NEXT:    i64x2.extract_lane 0
+; CHECK-NEXT:    i64.store gv_v4i16
+; CHECK-NEXT:    # fallthrough-return
+  store <4 x i16> %v , <4 x i16>* @gv_v4i16
+  ret void
+}
+
 ; ==============================================================================
 ; 2 x i64
 ; ==============================================================================

From e8d4038efb181f886e997ba28da1572e41aa16bd Mon Sep 17 00:00:00 2001
From: Valentin Clement <clementval@gmail.com>
Date: Tue, 28 Jul 2020 20:45:21 -0400
Subject: [PATCH 0404/1035] [openmp][openacc][NFC] Add wrapper for records in
 DirectiveEmitter

Add wrapper classes to to access record's fields. This makes it easier to
pass record information to the diverse functions for code generation.

Reviewed By: jdenny

Differential Revision: https://reviews.llvm.org/D84612
---
 llvm/utils/TableGen/DirectiveEmitter.cpp | 473 +++++++++++++----------
 1 file changed, 265 insertions(+), 208 deletions(-)

diff --git a/llvm/utils/TableGen/DirectiveEmitter.cpp b/llvm/utils/TableGen/DirectiveEmitter.cpp
index 2061ff1fdd1af..0e7f17c35415a 100644
--- a/llvm/utils/TableGen/DirectiveEmitter.cpp
+++ b/llvm/utils/TableGen/DirectiveEmitter.cpp
@@ -40,23 +40,141 @@ class IfDefScope {
 
 namespace llvm {
 
-// Get Directive or Clause name formatted by replacing whitespaces with
-// underscores.
-std::string getFormattedName(StringRef Name) {
-  std::string N = Name.str();
-  std::replace(N.begin(), N.end(), ' ', '_');
-  return N;
-}
+// Wrapper class that contains DirectiveLanguage's information defined in
+// DirectiveBase.td and provides helper methods for accessing it.
+class DirectiveLanguage {
+public:
+  explicit DirectiveLanguage(const llvm::Record *Def) : Def(Def) {}
+
+  StringRef getName() const { return Def->getValueAsString("name"); }
+
+  StringRef getCppNamespace() const {
+    return Def->getValueAsString("cppNamespace");
+  }
+
+  StringRef getDirectivePrefix() const {
+    return Def->getValueAsString("directivePrefix");
+  }
+
+  StringRef getClausePrefix() const {
+    return Def->getValueAsString("clausePrefix");
+  }
+
+  StringRef getIncludeHeader() const {
+    return Def->getValueAsString("includeHeader");
+  }
+
+  StringRef getClauseEnumSetClass() const {
+    return Def->getValueAsString("clauseEnumSetClass");
+  }
+
+  bool hasMakeEnumAvailableInNamespace() const {
+    return Def->getValueAsBit("makeEnumAvailableInNamespace");
+  }
+
+  bool hasEnableBitmaskEnumInNamespace() const {
+    return Def->getValueAsBit("enableBitmaskEnumInNamespace");
+  }
+
+private:
+  const llvm::Record *Def;
+};
+
+// Base record class used for Directive and Clause class defined in
+// DirectiveBase.td.
+class BaseRecord {
+public:
+  explicit BaseRecord(const llvm::Record *Def) : Def(Def) {}
+
+  StringRef getName() const { return Def->getValueAsString("name"); }
+
+  StringRef getAlternativeName() const {
+    return Def->getValueAsString("alternativeName");
+  }
+
+  // Returns the name of the directive formatted for output. Whitespace are
+  // replaced with underscores.
+  std::string getFormattedName() {
+    StringRef Name = Def->getValueAsString("name");
+    std::string N = Name.str();
+    std::replace(N.begin(), N.end(), ' ', '_');
+    return N;
+  }
+
+  bool isDefault() const { return Def->getValueAsBit("isDefault"); }
+
+protected:
+  const llvm::Record *Def;
+};
+
+// Wrapper class that contains a Directive's information defined in
+// DirectiveBase.td and provides helper methods for accessing it.
+class Directive : public BaseRecord {
+public:
+  explicit Directive(const llvm::Record *Def) : BaseRecord(Def) {}
+
+  std::vector<Record *> getAllowedClauses() const {
+    return Def->getValueAsListOfDefs("allowedClauses");
+  }
+
+  std::vector<Record *> getAllowedOnceClauses() const {
+    return Def->getValueAsListOfDefs("allowedOnceClauses");
+  }
+
+  std::vector<Record *> getAllowedExclusiveClauses() const {
+    return Def->getValueAsListOfDefs("allowedExclusiveClauses");
+  }
+
+  std::vector<Record *> getRequiredClauses() const {
+    return Def->getValueAsListOfDefs("requiredClauses");
+  }
+};
+
+// Wrapper class that contains Clause's information defined in DirectiveBase.td
+// and provides helper methods for accessing it.
+class Clause : public BaseRecord {
+public:
+  explicit Clause(const llvm::Record *Def) : BaseRecord(Def) {}
+
+  // Optional field.
+  StringRef getClangClass() const {
+    return Def->getValueAsString("clangClass");
+  }
+
+  // Optional field.
+  StringRef getFlangClass() const {
+    return Def->getValueAsString("flangClass");
+  }
+
+  bool isImplict() const { return Def->getValueAsBit("isImplicit"); }
+};
+
+// Wrapper class that contains VersionedClause's information defined in
+// DirectiveBase.td and provides helper methods for accessing it.
+class VersionedClause {
+public:
+  explicit VersionedClause(const llvm::Record *Def) : Def(Def) {}
+
+  // Return the specific clause record wrapped in the Clause class.
+  Clause getClause() const { return Clause{Def->getValueAsDef("clause")}; }
+
+  int64_t getMinVersion() const { return Def->getValueAsInt("minVersion"); }
+
+  int64_t getMaxVersion() const { return Def->getValueAsInt("maxVersion"); }
+
+private:
+  const llvm::Record *Def;
+};
 
 // Generate enum class
 void GenerateEnumClass(const std::vector<Record *> &Records, raw_ostream &OS,
-                       StringRef Enum, StringRef Prefix, StringRef CppNamespace,
-                       bool MakeEnumAvailableInNamespace) {
+                       StringRef Enum, StringRef Prefix,
+                       DirectiveLanguage &DirLang) {
   OS << "\n";
   OS << "enum class " << Enum << " {\n";
   for (const auto &R : Records) {
-    const auto Name = R->getValueAsString("name");
-    OS << "  " << Prefix << getFormattedName(Name) << ",\n";
+    BaseRecord Rec{R};
+    OS << "  " << Prefix << Rec.getFormattedName() << ",\n";
   }
   OS << "};\n";
   OS << "\n";
@@ -68,13 +186,13 @@ void GenerateEnumClass(const std::vector<Record *> &Records, raw_ostream &OS,
   // At the same time we do not loose the strong type guarantees of the enum
   // class, that is we cannot pass an unsigned as Directive without an explicit
   // cast.
-  if (MakeEnumAvailableInNamespace) {
+  if (DirLang.hasMakeEnumAvailableInNamespace()) {
     OS << "\n";
     for (const auto &R : Records) {
-      const auto FormattedName = getFormattedName(R->getValueAsString("name"));
-      OS << "constexpr auto " << Prefix << FormattedName << " = "
-         << "llvm::" << CppNamespace << "::" << Enum << "::" << Prefix
-         << FormattedName << ";\n";
+      BaseRecord Rec{R};
+      OS << "constexpr auto " << Prefix << Rec.getFormattedName() << " = "
+         << "llvm::" << DirLang.getCppNamespace() << "::" << Enum
+         << "::" << Prefix << Rec.getFormattedName() << ";\n";
     }
   }
 }
@@ -91,21 +209,12 @@ void EmitDirectivesDecl(RecordKeeper &Records, raw_ostream &OS) {
     return;
   }
 
-  const auto &DirectiveLanguage = DirectiveLanguages[0];
-  StringRef LanguageName = DirectiveLanguage->getValueAsString("name");
-  StringRef DirectivePrefix =
-      DirectiveLanguage->getValueAsString("directivePrefix");
-  StringRef ClausePrefix = DirectiveLanguage->getValueAsString("clausePrefix");
-  StringRef CppNamespace = DirectiveLanguage->getValueAsString("cppNamespace");
-  bool MakeEnumAvailableInNamespace =
-      DirectiveLanguage->getValueAsBit("makeEnumAvailableInNamespace");
-  bool EnableBitmaskEnumInNamespace =
-      DirectiveLanguage->getValueAsBit("enableBitmaskEnumInNamespace");
-
-  OS << "#ifndef LLVM_" << LanguageName << "_INC\n";
-  OS << "#define LLVM_" << LanguageName << "_INC\n";
-
-  if (EnableBitmaskEnumInNamespace)
+  DirectiveLanguage DirLang{DirectiveLanguages[0]};
+
+  OS << "#ifndef LLVM_" << DirLang.getName() << "_INC\n";
+  OS << "#define LLVM_" << DirLang.getName() << "_INC\n";
+
+  if (DirLang.hasEnableBitmaskEnumInNamespace())
     OS << "\n#include \"llvm/ADT/BitmaskEnum.h\"\n";
 
   OS << "\n";
@@ -114,35 +223,35 @@ void EmitDirectivesDecl(RecordKeeper &Records, raw_ostream &OS) {
 
   // Open namespaces defined in the directive language
   llvm::SmallVector<StringRef, 2> Namespaces;
-  llvm::SplitString(CppNamespace, Namespaces, "::");
+  llvm::SplitString(DirLang.getCppNamespace(), Namespaces, "::");
   for (auto Ns : Namespaces)
     OS << "namespace " << Ns << " {\n";
 
-  if (EnableBitmaskEnumInNamespace)
+  if (DirLang.hasEnableBitmaskEnumInNamespace())
     OS << "\nLLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE();\n";
 
   // Emit Directive enumeration
   const auto &Directives = Records.getAllDerivedDefinitions("Directive");
-  GenerateEnumClass(Directives, OS, "Directive", DirectivePrefix, CppNamespace,
-                    MakeEnumAvailableInNamespace);
+  GenerateEnumClass(Directives, OS, "Directive", DirLang.getDirectivePrefix(),
+                    DirLang);
 
   // Emit Clause enumeration
   const auto &Clauses = Records.getAllDerivedDefinitions("Clause");
-  GenerateEnumClass(Clauses, OS, "Clause", ClausePrefix, CppNamespace,
-                    MakeEnumAvailableInNamespace);
+  GenerateEnumClass(Clauses, OS, "Clause", DirLang.getClausePrefix(), DirLang);
 
   // Generic function signatures
   OS << "\n";
   OS << "// Enumeration helper functions\n";
-  OS << "Directive get" << LanguageName
+  OS << "Directive get" << DirLang.getName()
      << "DirectiveKind(llvm::StringRef Str);\n";
   OS << "\n";
-  OS << "llvm::StringRef get" << LanguageName
+  OS << "llvm::StringRef get" << DirLang.getName()
      << "DirectiveName(Directive D);\n";
   OS << "\n";
-  OS << "Clause get" << LanguageName << "ClauseKind(llvm::StringRef Str);\n";
+  OS << "Clause get" << DirLang.getName()
+     << "ClauseKind(llvm::StringRef Str);\n";
   OS << "\n";
-  OS << "llvm::StringRef get" << LanguageName << "ClauseName(Clause C);\n";
+  OS << "llvm::StringRef get" << DirLang.getName() << "ClauseName(Clause C);\n";
   OS << "\n";
   OS << "/// Return true if \\p C is a valid clause for \\p D in version \\p "
      << "Version.\n";
@@ -156,38 +265,37 @@ void EmitDirectivesDecl(RecordKeeper &Records, raw_ostream &OS) {
 
   OS << "} // namespace llvm\n";
 
-  OS << "#endif // LLVM_" << LanguageName << "_INC\n";
+  OS << "#endif // LLVM_" << DirLang.getName() << "_INC\n";
 }
 
 // Generate function implementation for get<Enum>Name(StringRef Str)
 void GenerateGetName(const std::vector<Record *> &Records, raw_ostream &OS,
-                     StringRef Enum, StringRef Prefix, StringRef LanguageName,
-                     StringRef Namespace) {
+                     StringRef Enum, DirectiveLanguage &DirLang,
+                     StringRef Prefix) {
   OS << "\n";
-  OS << "llvm::StringRef llvm::" << Namespace << "::get" << LanguageName << Enum
-     << "Name(" << Enum << " Kind) {\n";
+  OS << "llvm::StringRef llvm::" << DirLang.getCppNamespace() << "::get"
+     << DirLang.getName() << Enum << "Name(" << Enum << " Kind) {\n";
   OS << "  switch (Kind) {\n";
   for (const auto &R : Records) {
-    const auto Name = R->getValueAsString("name");
-    const auto AlternativeName = R->getValueAsString("alternativeName");
-    OS << "    case " << Prefix << getFormattedName(Name) << ":\n";
+    BaseRecord Rec{R};
+    OS << "    case " << Prefix << Rec.getFormattedName() << ":\n";
     OS << "      return \"";
-    if (AlternativeName.empty())
-      OS << Name;
+    if (Rec.getAlternativeName().empty())
+      OS << Rec.getName();
     else
-      OS << AlternativeName;
+      OS << Rec.getAlternativeName();
     OS << "\";\n";
   }
   OS << "  }\n"; // switch
-  OS << "  llvm_unreachable(\"Invalid " << LanguageName << " " << Enum
+  OS << "  llvm_unreachable(\"Invalid " << DirLang.getName() << " " << Enum
      << " kind\");\n";
   OS << "}\n";
 }
 
 // Generate function implementation for get<Enum>Kind(StringRef Str)
 void GenerateGetKind(const std::vector<Record *> &Records, raw_ostream &OS,
-                     StringRef Enum, StringRef Prefix, StringRef LanguageName,
-                     StringRef Namespace, bool ImplicitAsUnknown) {
+                     StringRef Enum, DirectiveLanguage &DirLang,
+                     StringRef Prefix, bool ImplicitAsUnknown) {
 
   auto DefaultIt = std::find_if(Records.begin(), Records.end(), [](Record *R) {
     return R->getValueAsBit("isDefault") == true;
@@ -198,96 +306,86 @@ void GenerateGetKind(const std::vector<Record *> &Records, raw_ostream &OS,
     return;
   }
 
-  const auto FormattedDefaultName =
-      getFormattedName((*DefaultIt)->getValueAsString("name"));
+  BaseRecord DefaultRec{(*DefaultIt)};
 
   OS << "\n";
-  OS << Enum << " llvm::" << Namespace << "::get" << LanguageName << Enum
-     << "Kind(llvm::StringRef Str) {\n";
+  OS << Enum << " llvm::" << DirLang.getCppNamespace() << "::get"
+     << DirLang.getName() << Enum << "Kind(llvm::StringRef Str) {\n";
   OS << "  return llvm::StringSwitch<" << Enum << ">(Str)\n";
 
   for (const auto &R : Records) {
-    const auto Name = R->getValueAsString("name");
+    BaseRecord Rec{R};
     if (ImplicitAsUnknown && R->getValueAsBit("isImplicit")) {
-      OS << "    .Case(\"" << Name << "\"," << Prefix << FormattedDefaultName
-         << ")\n";
+      OS << "    .Case(\"" << Rec.getName() << "\"," << Prefix
+         << DefaultRec.getFormattedName() << ")\n";
     } else {
-      OS << "    .Case(\"" << Name << "\"," << Prefix << getFormattedName(Name)
-         << ")\n";
+      OS << "    .Case(\"" << Rec.getName() << "\"," << Prefix
+         << Rec.getFormattedName() << ")\n";
     }
   }
-  OS << "    .Default(" << Prefix << FormattedDefaultName << ");\n";
+  OS << "    .Default(" << Prefix << DefaultRec.getFormattedName() << ");\n";
   OS << "}\n";
 }
 
 void GenerateCaseForVersionedClauses(const std::vector<Record *> &Clauses,
                                      raw_ostream &OS, StringRef DirectiveName,
-                                     StringRef DirectivePrefix,
-                                     StringRef ClausePrefix,
+                                     DirectiveLanguage &DirLang,
                                      llvm::StringSet<> &Cases) {
   for (const auto &C : Clauses) {
-    const auto MinVersion = C->getValueAsInt("minVersion");
-    const auto MaxVersion = C->getValueAsInt("maxVersion");
-    const auto SpecificClause = C->getValueAsDef("clause");
-    const auto ClauseName =
-        getFormattedName(SpecificClause->getValueAsString("name"));
-
-    if (Cases.find(ClauseName) == Cases.end()) {
-      Cases.insert(ClauseName);
-      OS << "        case " << ClausePrefix << ClauseName << ":\n";
-      OS << "          return " << MinVersion << " <= Version && " << MaxVersion
-         << " >= Version;\n";
+    VersionedClause VerClause{C};
+
+    const auto ClauseFormattedName = VerClause.getClause().getFormattedName();
+
+    if (Cases.find(ClauseFormattedName) == Cases.end()) {
+      Cases.insert(ClauseFormattedName);
+      OS << "        case " << DirLang.getClausePrefix() << ClauseFormattedName
+         << ":\n";
+      OS << "          return " << VerClause.getMinVersion()
+         << " <= Version && " << VerClause.getMaxVersion() << " >= Version;\n";
     }
   }
 }
 
 // Generate the isAllowedClauseForDirective function implementation.
 void GenerateIsAllowedClause(const std::vector<Record *> &Directives,
-                             raw_ostream &OS, StringRef LanguageName,
-                             StringRef DirectivePrefix, StringRef ClausePrefix,
-                             StringRef CppNamespace) {
+                             raw_ostream &OS, DirectiveLanguage &DirLang) {
   OS << "\n";
-  OS << "bool llvm::" << CppNamespace << "::isAllowedClauseForDirective("
+  OS << "bool llvm::" << DirLang.getCppNamespace()
+     << "::isAllowedClauseForDirective("
      << "Directive D, Clause C, unsigned Version) {\n";
-  OS << "  assert(unsigned(D) <= llvm::" << CppNamespace
+  OS << "  assert(unsigned(D) <= llvm::" << DirLang.getCppNamespace()
      << "::Directive_enumSize);\n";
-  OS << "  assert(unsigned(C) <= llvm::" << CppNamespace
+  OS << "  assert(unsigned(C) <= llvm::" << DirLang.getCppNamespace()
      << "::Clause_enumSize);\n";
 
   OS << "  switch (D) {\n";
 
   for (const auto &D : Directives) {
+    Directive Dir{D};
 
-    const auto DirectiveName = D->getValueAsString("name");
-    const auto &AllowedClauses = D->getValueAsListOfDefs("allowedClauses");
-    const auto &AllowedOnceClauses =
-        D->getValueAsListOfDefs("allowedOnceClauses");
-    const auto &AllowedExclusiveClauses =
-        D->getValueAsListOfDefs("allowedExclusiveClauses");
-    const auto &RequiredClauses = D->getValueAsListOfDefs("requiredClauses");
-
-    OS << "    case " << DirectivePrefix << getFormattedName(DirectiveName)
+    OS << "    case " << DirLang.getDirectivePrefix() << Dir.getFormattedName()
        << ":\n";
-    if (AllowedClauses.size() == 0 && AllowedOnceClauses.size() == 0 &&
-        AllowedExclusiveClauses.size() == 0 && RequiredClauses.size() == 0) {
+    if (Dir.getAllowedClauses().size() == 0 &&
+        Dir.getAllowedOnceClauses().size() == 0 &&
+        Dir.getAllowedExclusiveClauses().size() == 0 &&
+        Dir.getRequiredClauses().size() == 0) {
       OS << "      return false;\n";
     } else {
       OS << "      switch (C) {\n";
 
       llvm::StringSet<> Cases;
 
-      GenerateCaseForVersionedClauses(AllowedClauses, OS, DirectiveName,
-                                      DirectivePrefix, ClausePrefix, Cases);
+      GenerateCaseForVersionedClauses(Dir.getAllowedClauses(), OS,
+                                      Dir.getName(), DirLang, Cases);
 
-      GenerateCaseForVersionedClauses(AllowedOnceClauses, OS, DirectiveName,
-                                      DirectivePrefix, ClausePrefix, Cases);
+      GenerateCaseForVersionedClauses(Dir.getAllowedOnceClauses(), OS,
+                                      Dir.getName(), DirLang, Cases);
 
-      GenerateCaseForVersionedClauses(AllowedExclusiveClauses, OS,
-                                      DirectiveName, DirectivePrefix,
-                                      ClausePrefix, Cases);
+      GenerateCaseForVersionedClauses(Dir.getAllowedExclusiveClauses(), OS,
+                                      Dir.getName(), DirLang, Cases);
 
-      GenerateCaseForVersionedClauses(RequiredClauses, OS, DirectiveName,
-                                      DirectivePrefix, ClausePrefix, Cases);
+      GenerateCaseForVersionedClauses(Dir.getRequiredClauses(), OS,
+                                      Dir.getName(), DirLang, Cases);
 
       OS << "        default:\n";
       OS << "          return false;\n";
@@ -297,37 +395,32 @@ void GenerateIsAllowedClause(const std::vector<Record *> &Directives,
   }
 
   OS << "  }\n"; // End of directives switch
-  OS << "  llvm_unreachable(\"Invalid " << LanguageName
+  OS << "  llvm_unreachable(\"Invalid " << DirLang.getName()
      << " Directive kind\");\n";
   OS << "}\n"; // End of function isAllowedClauseForDirective
 }
 
 // Generate a simple enum set with the give clauses.
 void GenerateClauseSet(const std::vector<Record *> &Clauses, raw_ostream &OS,
-                       StringRef ClauseEnumSetClass, StringRef ClauseSetPrefix,
-                       StringRef DirectiveName, StringRef DirectivePrefix,
-                       StringRef ClausePrefix, StringRef CppNamespace) {
+                       StringRef ClauseSetPrefix, Directive &Dir,
+                       DirectiveLanguage &DirLang) {
 
   OS << "\n";
-  OS << "  static " << ClauseEnumSetClass << " " << ClauseSetPrefix
-     << DirectivePrefix << getFormattedName(DirectiveName) << " {\n";
+  OS << "  static " << DirLang.getClauseEnumSetClass() << " " << ClauseSetPrefix
+     << DirLang.getDirectivePrefix() << Dir.getFormattedName() << " {\n";
 
   for (const auto &C : Clauses) {
-    const auto SpecificClause = C->getValueAsDef("clause");
-    const auto ClauseName = SpecificClause->getValueAsString("name");
-    OS << "    llvm::" << CppNamespace << "::Clause::" << ClausePrefix
-       << getFormattedName(ClauseName) << ",\n";
+    VersionedClause VerClause{C};
+    OS << "    llvm::" << DirLang.getCppNamespace()
+       << "::Clause::" << DirLang.getClausePrefix()
+       << VerClause.getClause().getFormattedName() << ",\n";
   }
   OS << "  };\n";
 }
 
 // Generate an enum set for the 4 kinds of clauses linked to a directive.
 void GenerateDirectiveClauseSets(const std::vector<Record *> &Directives,
-                                 raw_ostream &OS, StringRef LanguageName,
-                                 StringRef ClauseEnumSetClass,
-                                 StringRef DirectivePrefix,
-                                 StringRef ClausePrefix,
-                                 StringRef CppNamespace) {
+                                 raw_ostream &OS, DirectiveLanguage &DirLang) {
 
   IfDefScope Scope("GEN_FLANG_DIRECTIVE_CLAUSE_SETS", OS);
 
@@ -336,35 +429,24 @@ void GenerateDirectiveClauseSets(const std::vector<Record *> &Directives,
 
   // Open namespaces defined in the directive language.
   llvm::SmallVector<StringRef, 2> Namespaces;
-  llvm::SplitString(CppNamespace, Namespaces, "::");
+  llvm::SplitString(DirLang.getCppNamespace(), Namespaces, "::");
   for (auto Ns : Namespaces)
     OS << "namespace " << Ns << " {\n";
 
   for (const auto &D : Directives) {
-    const auto DirectiveName = D->getValueAsString("name");
-
-    const auto &AllowedClauses = D->getValueAsListOfDefs("allowedClauses");
-    const auto &AllowedOnceClauses =
-        D->getValueAsListOfDefs("allowedOnceClauses");
-    const auto &AllowedExclusiveClauses =
-        D->getValueAsListOfDefs("allowedExclusiveClauses");
-    const auto &RequiredClauses = D->getValueAsListOfDefs("requiredClauses");
+    Directive Dir{D};
 
     OS << "\n";
-    OS << "  // Sets for " << DirectiveName << "\n";
-
-    GenerateClauseSet(AllowedClauses, OS, ClauseEnumSetClass, "allowedClauses_",
-                      DirectiveName, DirectivePrefix, ClausePrefix,
-                      CppNamespace);
-    GenerateClauseSet(AllowedOnceClauses, OS, ClauseEnumSetClass,
-                      "allowedOnceClauses_", DirectiveName, DirectivePrefix,
-                      ClausePrefix, CppNamespace);
-    GenerateClauseSet(AllowedExclusiveClauses, OS, ClauseEnumSetClass,
-                      "allowedExclusiveClauses_", DirectiveName,
-                      DirectivePrefix, ClausePrefix, CppNamespace);
-    GenerateClauseSet(RequiredClauses, OS, ClauseEnumSetClass,
-                      "requiredClauses_", DirectiveName, DirectivePrefix,
-                      ClausePrefix, CppNamespace);
+    OS << "  // Sets for " << Dir.getName() << "\n";
+
+    GenerateClauseSet(Dir.getAllowedClauses(), OS, "allowedClauses_", Dir,
+                      DirLang);
+    GenerateClauseSet(Dir.getAllowedOnceClauses(), OS, "allowedOnceClauses_",
+                      Dir, DirLang);
+    GenerateClauseSet(Dir.getAllowedExclusiveClauses(), OS,
+                      "allowedExclusiveClauses_", Dir, DirLang);
+    GenerateClauseSet(Dir.getRequiredClauses(), OS, "requiredClauses_", Dir,
+                      DirLang);
   }
 
   // Closing namespaces
@@ -378,42 +460,39 @@ void GenerateDirectiveClauseSets(const std::vector<Record *> &Directives,
 // The struct holds the 4 sets of enumeration for the 4 kinds of clauses
 // allowances (allowed, allowed once, allowed exclusive and required).
 void GenerateDirectiveClauseMap(const std::vector<Record *> &Directives,
-                                raw_ostream &OS, StringRef LanguageName,
-                                StringRef ClauseEnumSetClass,
-                                StringRef DirectivePrefix,
-                                StringRef ClausePrefix,
-                                StringRef CppNamespace) {
+                                raw_ostream &OS, DirectiveLanguage &DirLang) {
 
   IfDefScope Scope("GEN_FLANG_DIRECTIVE_CLAUSE_MAP", OS);
 
   OS << "\n";
-  OS << "struct " << LanguageName << "DirectiveClauses {\n";
-  OS << "  const " << ClauseEnumSetClass << " allowed;\n";
-  OS << "  const " << ClauseEnumSetClass << " allowedOnce;\n";
-  OS << "  const " << ClauseEnumSetClass << " allowedExclusive;\n";
-  OS << "  const " << ClauseEnumSetClass << " requiredOneOf;\n";
+  OS << "struct " << DirLang.getName() << "DirectiveClauses {\n";
+  OS << "  const " << DirLang.getClauseEnumSetClass() << " allowed;\n";
+  OS << "  const " << DirLang.getClauseEnumSetClass() << " allowedOnce;\n";
+  OS << "  const " << DirLang.getClauseEnumSetClass() << " allowedExclusive;\n";
+  OS << "  const " << DirLang.getClauseEnumSetClass() << " requiredOneOf;\n";
   OS << "};\n";
 
   OS << "\n";
 
-  OS << "std::unordered_map<llvm::" << CppNamespace << "::Directive, "
-     << LanguageName << "DirectiveClauses>\n";
+  OS << "std::unordered_map<llvm::" << DirLang.getCppNamespace()
+     << "::Directive, " << DirLang.getName() << "DirectiveClauses>\n";
   OS << "    directiveClausesTable = {\n";
 
   for (const auto &D : Directives) {
-    const auto FormattedDirectiveName =
-        getFormattedName(D->getValueAsString("name"));
-    OS << "  {llvm::" << CppNamespace << "::Directive::" << DirectivePrefix
-       << FormattedDirectiveName << ",\n";
+    Directive Dir{D};
+    OS << "  {llvm::" << DirLang.getCppNamespace()
+       << "::Directive::" << DirLang.getDirectivePrefix()
+       << Dir.getFormattedName() << ",\n";
     OS << "    {\n";
-    OS << "      llvm::" << CppNamespace << "::allowedClauses_"
-       << DirectivePrefix << FormattedDirectiveName << ",\n";
-    OS << "      llvm::" << CppNamespace << "::allowedOnceClauses_"
-       << DirectivePrefix << FormattedDirectiveName << ",\n";
-    OS << "      llvm::" << CppNamespace << "::allowedExclusiveClauses_"
-       << DirectivePrefix << FormattedDirectiveName << ",\n";
-    OS << "      llvm::" << CppNamespace << "::requiredClauses_"
-       << DirectivePrefix << FormattedDirectiveName << ",\n";
+    OS << "      llvm::" << DirLang.getCppNamespace() << "::allowedClauses_"
+       << DirLang.getDirectivePrefix() << Dir.getFormattedName() << ",\n";
+    OS << "      llvm::" << DirLang.getCppNamespace() << "::allowedOnceClauses_"
+       << DirLang.getDirectivePrefix() << Dir.getFormattedName() << ",\n";
+    OS << "      llvm::" << DirLang.getCppNamespace()
+       << "::allowedExclusiveClauses_" << DirLang.getDirectivePrefix()
+       << Dir.getFormattedName() << ",\n";
+    OS << "      llvm::" << DirLang.getCppNamespace() << "::requiredClauses_"
+       << DirLang.getDirectivePrefix() << Dir.getFormattedName() << ",\n";
     OS << "    }\n";
     OS << "  },\n";
   }
@@ -424,16 +503,12 @@ void GenerateDirectiveClauseMap(const std::vector<Record *> &Directives,
 // Generate the implemenation section for the enumeration in the directive
 // language
 void EmitDirectivesFlangImpl(const std::vector<Record *> &Directives,
-                             raw_ostream &OS, StringRef LanguageName,
-                             StringRef ClauseEnumSetClass,
-                             StringRef DirectivePrefix, StringRef ClausePrefix,
-                             StringRef CppNamespace) {
+                             raw_ostream &OS,
+                             DirectiveLanguage &DirectiveLanguage) {
 
-  GenerateDirectiveClauseSets(Directives, OS, LanguageName, ClauseEnumSetClass,
-                              DirectivePrefix, ClausePrefix, CppNamespace);
+  GenerateDirectiveClauseSets(Directives, OS, DirectiveLanguage);
 
-  GenerateDirectiveClauseMap(Directives, OS, LanguageName, ClauseEnumSetClass,
-                             DirectivePrefix, ClausePrefix, CppNamespace);
+  GenerateDirectiveClauseMap(Directives, OS, DirectiveLanguage);
 }
 
 // Generate the implemenation section for the enumeration in the directive
@@ -448,19 +523,9 @@ void EmitDirectivesGen(RecordKeeper &Records, raw_ostream &OS) {
     return;
   }
 
-  const auto &DirectiveLanguage = DirectiveLanguages[0];
-  StringRef DirectivePrefix =
-      DirectiveLanguage->getValueAsString("directivePrefix");
-  StringRef LanguageName = DirectiveLanguage->getValueAsString("name");
-  StringRef ClausePrefix = DirectiveLanguage->getValueAsString("clausePrefix");
-  StringRef CppNamespace = DirectiveLanguage->getValueAsString("cppNamespace");
-  StringRef ClauseEnumSetClass =
-      DirectiveLanguage->getValueAsString("clauseEnumSetClass");
-
   const auto &Directives = Records.getAllDerivedDefinitions("Directive");
-
-  EmitDirectivesFlangImpl(Directives, OS, LanguageName, ClauseEnumSetClass,
-                          DirectivePrefix, ClausePrefix, CppNamespace);
+  DirectiveLanguage DirectiveLanguage{DirectiveLanguages[0]};
+  EmitDirectivesFlangImpl(Directives, OS, DirectiveLanguage);
 }
 
 // Generate the implemenation for the enumeration in the directive
@@ -475,20 +540,14 @@ void EmitDirectivesImpl(RecordKeeper &Records, raw_ostream &OS) {
     return;
   }
 
-  const auto &DirectiveLanguage = DirectiveLanguages[0];
-  StringRef DirectivePrefix =
-      DirectiveLanguage->getValueAsString("directivePrefix");
-  StringRef LanguageName = DirectiveLanguage->getValueAsString("name");
-  StringRef ClausePrefix = DirectiveLanguage->getValueAsString("clausePrefix");
-  StringRef CppNamespace = DirectiveLanguage->getValueAsString("cppNamespace");
   const auto &Directives = Records.getAllDerivedDefinitions("Directive");
-  const auto &Clauses = Records.getAllDerivedDefinitions("Clause");
 
-  StringRef IncludeHeader =
-      DirectiveLanguage->getValueAsString("includeHeader");
+  DirectiveLanguage DirLang = DirectiveLanguage{DirectiveLanguages[0]};
+
+  const auto &Clauses = Records.getAllDerivedDefinitions("Clause");
 
-  if (!IncludeHeader.empty())
-    OS << "#include \"" << IncludeHeader << "\"\n\n";
+  if (!DirLang.getIncludeHeader().empty())
+    OS << "#include \"" << DirLang.getIncludeHeader() << "\"\n\n";
 
   OS << "#include \"llvm/ADT/StringRef.h\"\n";
   OS << "#include \"llvm/ADT/StringSwitch.h\"\n";
@@ -496,29 +555,27 @@ void EmitDirectivesImpl(RecordKeeper &Records, raw_ostream &OS) {
   OS << "\n";
   OS << "using namespace llvm;\n";
   llvm::SmallVector<StringRef, 2> Namespaces;
-  llvm::SplitString(CppNamespace, Namespaces, "::");
+  llvm::SplitString(DirLang.getCppNamespace(), Namespaces, "::");
   for (auto Ns : Namespaces)
     OS << "using namespace " << Ns << ";\n";
 
   // getDirectiveKind(StringRef Str)
-  GenerateGetKind(Directives, OS, "Directive", DirectivePrefix, LanguageName,
-                  CppNamespace, /*ImplicitAsUnknown=*/false);
+  GenerateGetKind(Directives, OS, "Directive", DirLang,
+                  DirLang.getDirectivePrefix(), /*ImplicitAsUnknown=*/false);
 
   // getDirectiveName(Directive Kind)
-  GenerateGetName(Directives, OS, "Directive", DirectivePrefix, LanguageName,
-                  CppNamespace);
+  GenerateGetName(Directives, OS, "Directive", DirLang,
+                  DirLang.getDirectivePrefix());
 
   // getClauseKind(StringRef Str)
-  GenerateGetKind(Clauses, OS, "Clause", ClausePrefix, LanguageName,
-                  CppNamespace, /*ImplicitAsUnknown=*/true);
+  GenerateGetKind(Clauses, OS, "Clause", DirLang, DirLang.getClausePrefix(),
+                  /*ImplicitAsUnknown=*/true);
 
   // getClauseName(Clause Kind)
-  GenerateGetName(Clauses, OS, "Clause", ClausePrefix, LanguageName,
-                  CppNamespace);
+  GenerateGetName(Clauses, OS, "Clause", DirLang, DirLang.getClausePrefix());
 
   // isAllowedClauseForDirective(Directive D, Clause C, unsigned Version)
-  GenerateIsAllowedClause(Directives, OS, LanguageName, DirectivePrefix,
-                          ClausePrefix, CppNamespace);
+  GenerateIsAllowedClause(Directives, OS, DirLang);
 }
 
 } // namespace llvm

From 6a7b6dd54b207945ba4b3750496036c6280ab522 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Wed, 22 Jul 2020 12:55:20 -0400
Subject: [PATCH 0405/1035] AMDGPU: Don't assert in canInsertSelect

Currently GlobalISel doesn't force all VGPR phi operands to VGPRs, so
this hit a case where it was queried with a VGPR and SGPR. This could
arguably be a verifier error, but it's currently not.
---
 llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 5d7cd5ffc4ce7..59915cb48324d 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -2392,7 +2392,8 @@ bool SIInstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
   case VCCZ: {
     const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
     const TargetRegisterClass *RC = MRI.getRegClass(TrueReg);
-    assert(MRI.getRegClass(FalseReg) == RC);
+    if (MRI.getRegClass(FalseReg) != RC)
+      return false;
 
     int NumInsts = AMDGPU::getRegBitWidth(RC->getID()) / 32;
     CondCycles = TrueCycles = FalseCycles = NumInsts; // ???
@@ -2406,7 +2407,8 @@ bool SIInstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
     // with a vector one.
     const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
     const TargetRegisterClass *RC = MRI.getRegClass(TrueReg);
-    assert(MRI.getRegClass(FalseReg) == RC);
+    if (MRI.getRegClass(FalseReg) != RC)
+      return false;
 
     int NumInsts = AMDGPU::getRegBitWidth(RC->getID()) / 32;
 

From 06cf6f770d649ed241314881054b7b8d9eee765d Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@intel.com>
Date: Tue, 28 Jul 2020 17:55:50 -0700
Subject: [PATCH 0406/1035] [X86] Add FeatureCMPXCHG8B and FeatureSlowUAMem16
 to 'lakemont' in X86.td

We already had CMPXCH8B feature on this CPU for the frontend so
this doesn't have much effect.

The FeatureSlowUAMem16 only matters if someone compiles with
-march=lakemont -msse which doesn't make sense, but is consistent
with all our pre-sse4.2 CPUs. Maybe the feature flag should be
FeatureFastUAMem16 and set on the newer CPUs instead.
---
 llvm/lib/Target/X86/X86.td         | 3 ++-
 llvm/test/CodeGen/X86/cmpxchg8b.ll | 1 +
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/Target/X86/X86.td b/llvm/lib/Target/X86/X86.td
index bc8927899d687..f9ed55ea37d49 100644
--- a/llvm/lib/Target/X86/X86.td
+++ b/llvm/lib/Target/X86/X86.td
@@ -1081,7 +1081,8 @@ foreach P = ["pentium4", "pentium4m"] in {
 }
 
 // Intel Quark.
-def : Proc<"lakemont",        [FeatureInsertVZEROUPPER]>;
+def : Proc<"lakemont", [FeatureSlowUAMem16, FeatureCMPXCHG8B,
+                        FeatureInsertVZEROUPPER]>;
 
 // Intel Core Duo.
 def : ProcessorModel<"yonah", SandyBridgeModel,
diff --git a/llvm/test/CodeGen/X86/cmpxchg8b.ll b/llvm/test/CodeGen/X86/cmpxchg8b.ll
index caf40c541e280..998b0ad62fc98 100644
--- a/llvm/test/CodeGen/X86/cmpxchg8b.ll
+++ b/llvm/test/CodeGen/X86/cmpxchg8b.ll
@@ -3,6 +3,7 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown- -mcpu=core2 | FileCheck %s --check-prefixes=CHECK,X64
 ; RUN: llc < %s -mtriple=i686-unknown- -mcpu=i486 | FileCheck %s --check-prefixes=I486
 ; RUN: llc < %s -mtriple=i686-unknown- -mcpu=znver1 | FileCheck %s --check-prefixes=CHECK,X86
+; RUN: llc < %s -mtriple=i686-unknown- -mcpu=lakemont | FileCheck %s --check-prefixes=CHECK,X86
 
 ; Basic 64-bit cmpxchg
 define void @t1(i64* nocapture %p) nounwind ssp {

From 11bb7eef4152cab895983f19e638f0cfdf8a580f Mon Sep 17 00:00:00 2001
From: Thomas Lively <tlively@google.com>
Date: Tue, 28 Jul 2020 18:25:55 -0700
Subject: [PATCH 0407/1035] [WebAssembly] Remove intrinsics for SIMD widening
 ops

Instead, pattern match extends of extract_subvectors to generate
widening operations. Since extract_subvector is not a legal node, this
is implemented via a custom combine that recognizes extract_subvector
nodes before they are legalized. The combine produces custom ISD nodes
that are later pattern matched directly, just like the intrinsic was.

Also removes the clang builtins for these operations since the
instructions can now be generated from portable code sequences.

Differential Revision: https://reviews.llvm.org/D84556
---
 .../clang/Basic/BuiltinsWebAssembly.def       |   9 -
 clang/lib/CodeGen/CGBuiltin.cpp               |  34 ----
 clang/lib/Headers/wasm_simd128.h              |  51 ++++-
 clang/test/CodeGen/builtins-wasm.c            |  48 -----
 llvm/include/llvm/IR/IntrinsicsWebAssembly.td |  16 --
 .../lib/Target/WebAssembly/WebAssemblyISD.def |   4 +
 .../WebAssembly/WebAssemblyISelLowering.cpp   |  50 +++++
 .../WebAssembly/WebAssemblyInstrSIMD.td       |  14 +-
 .../CodeGen/WebAssembly/simd-intrinsics.ll    |  80 --------
 .../test/CodeGen/WebAssembly/simd-widening.ll | 180 ++++++++++++++++++
 10 files changed, 287 insertions(+), 199 deletions(-)
 create mode 100644 llvm/test/CodeGen/WebAssembly/simd-widening.ll

diff --git a/clang/include/clang/Basic/BuiltinsWebAssembly.def b/clang/include/clang/Basic/BuiltinsWebAssembly.def
index ecee7782920f6..d0f40f991a4c7 100644
--- a/clang/include/clang/Basic/BuiltinsWebAssembly.def
+++ b/clang/include/clang/Basic/BuiltinsWebAssembly.def
@@ -169,14 +169,5 @@ TARGET_BUILTIN(__builtin_wasm_narrow_u_i8x16_i16x8, "V16cV8sV8s", "nc", "simd128
 TARGET_BUILTIN(__builtin_wasm_narrow_s_i16x8_i32x4, "V8sV4iV4i", "nc", "simd128")
 TARGET_BUILTIN(__builtin_wasm_narrow_u_i16x8_i32x4, "V8sV4iV4i", "nc", "simd128")
 
-TARGET_BUILTIN(__builtin_wasm_widen_low_s_i16x8_i8x16, "V8sV16c", "nc", "simd128")
-TARGET_BUILTIN(__builtin_wasm_widen_high_s_i16x8_i8x16, "V8sV16c", "nc", "simd128")
-TARGET_BUILTIN(__builtin_wasm_widen_low_u_i16x8_i8x16, "V8sV16c", "nc", "simd128")
-TARGET_BUILTIN(__builtin_wasm_widen_high_u_i16x8_i8x16, "V8sV16c", "nc", "simd128")
-TARGET_BUILTIN(__builtin_wasm_widen_low_s_i32x4_i16x8, "V4iV8s", "nc", "simd128")
-TARGET_BUILTIN(__builtin_wasm_widen_high_s_i32x4_i16x8, "V4iV8s", "nc", "simd128")
-TARGET_BUILTIN(__builtin_wasm_widen_low_u_i32x4_i16x8, "V4iV8s", "nc", "simd128")
-TARGET_BUILTIN(__builtin_wasm_widen_high_u_i32x4_i16x8, "V4iV8s", "nc", "simd128")
-
 #undef BUILTIN
 #undef TARGET_BUILTIN
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index d572de1b8b79d..ecc0b5bf2dc44 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -16528,40 +16528,6 @@ Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID,
         CGM.getIntrinsic(IntNo, {ConvertType(E->getType()), Low->getType()});
     return Builder.CreateCall(Callee, {Low, High});
   }
-  case WebAssembly::BI__builtin_wasm_widen_low_s_i16x8_i8x16:
-  case WebAssembly::BI__builtin_wasm_widen_high_s_i16x8_i8x16:
-  case WebAssembly::BI__builtin_wasm_widen_low_u_i16x8_i8x16:
-  case WebAssembly::BI__builtin_wasm_widen_high_u_i16x8_i8x16:
-  case WebAssembly::BI__builtin_wasm_widen_low_s_i32x4_i16x8:
-  case WebAssembly::BI__builtin_wasm_widen_high_s_i32x4_i16x8:
-  case WebAssembly::BI__builtin_wasm_widen_low_u_i32x4_i16x8:
-  case WebAssembly::BI__builtin_wasm_widen_high_u_i32x4_i16x8: {
-    Value *Vec = EmitScalarExpr(E->getArg(0));
-    unsigned IntNo;
-    switch (BuiltinID) {
-    case WebAssembly::BI__builtin_wasm_widen_low_s_i16x8_i8x16:
-    case WebAssembly::BI__builtin_wasm_widen_low_s_i32x4_i16x8:
-      IntNo = Intrinsic::wasm_widen_low_signed;
-      break;
-    case WebAssembly::BI__builtin_wasm_widen_high_s_i16x8_i8x16:
-    case WebAssembly::BI__builtin_wasm_widen_high_s_i32x4_i16x8:
-      IntNo = Intrinsic::wasm_widen_high_signed;
-      break;
-    case WebAssembly::BI__builtin_wasm_widen_low_u_i16x8_i8x16:
-    case WebAssembly::BI__builtin_wasm_widen_low_u_i32x4_i16x8:
-      IntNo = Intrinsic::wasm_widen_low_unsigned;
-      break;
-    case WebAssembly::BI__builtin_wasm_widen_high_u_i16x8_i8x16:
-    case WebAssembly::BI__builtin_wasm_widen_high_u_i32x4_i16x8:
-      IntNo = Intrinsic::wasm_widen_high_unsigned;
-      break;
-    default:
-      llvm_unreachable("unexpected builtin ID");
-    }
-    Function *Callee =
-        CGM.getIntrinsic(IntNo, {ConvertType(E->getType()), Vec->getType()});
-    return Builder.CreateCall(Callee, Vec);
-  }
   case WebAssembly::BI__builtin_wasm_shuffle_v8x16: {
     Value *Ops[18];
     size_t OpIdx = 0;
diff --git a/clang/lib/Headers/wasm_simd128.h b/clang/lib/Headers/wasm_simd128.h
index b78123834b643..967008b555f42 100644
--- a/clang/lib/Headers/wasm_simd128.h
+++ b/clang/lib/Headers/wasm_simd128.h
@@ -35,6 +35,13 @@ typedef unsigned long long __u64x2
 typedef float __f32x4 __attribute__((__vector_size__(16), __aligned__(16)));
 typedef double __f64x2 __attribute__((__vector_size__(16), __aligned__(16)));
 
+typedef signed char __i8x8 __attribute__((__vector_size__(8), __aligned__(8)));
+typedef unsigned char __u8x8
+    __attribute__((__vector_size__(8), __aligned__(8)));
+typedef short __i16x4 __attribute__((__vector_size__(8), __aligned__(8)));
+typedef unsigned short __u16x4
+    __attribute__((__vector_size__(8), __aligned__(8)));
+
 #define __DEFAULT_FN_ATTRS                                                     \
   __attribute__((__always_inline__, __nodebug__, __target__("simd128"),        \
                  __min_vector_width__(128)))
@@ -1089,42 +1096,70 @@ wasm_u16x8_narrow_i32x4(v128_t __a, v128_t __b) {
 
 static __inline__ v128_t __DEFAULT_FN_ATTRS
 wasm_i16x8_widen_low_i8x16(v128_t __a) {
-  return (v128_t)__builtin_wasm_widen_low_s_i16x8_i8x16((__i8x16)__a);
+  return (v128_t) __builtin_convertvector(
+      (__i8x8){((__i8x16)__a)[0], ((__i8x16)__a)[1], ((__i8x16)__a)[2],
+               ((__i8x16)__a)[3], ((__i8x16)__a)[4], ((__i8x16)__a)[5],
+               ((__i8x16)__a)[6], ((__i8x16)__a)[7]},
+      __i16x8);
 }
 
 static __inline__ v128_t __DEFAULT_FN_ATTRS
 wasm_i16x8_widen_high_i8x16(v128_t __a) {
-  return (v128_t)__builtin_wasm_widen_high_s_i16x8_i8x16((__i8x16)__a);
+  return (v128_t) __builtin_convertvector(
+      (__i8x8){((__i8x16)__a)[8], ((__i8x16)__a)[9], ((__i8x16)__a)[10],
+               ((__i8x16)__a)[11], ((__i8x16)__a)[12], ((__i8x16)__a)[13],
+               ((__i8x16)__a)[14], ((__i8x16)__a)[15]},
+      __i16x8);
 }
 
 static __inline__ v128_t __DEFAULT_FN_ATTRS
 wasm_i16x8_widen_low_u8x16(v128_t __a) {
-  return (v128_t)__builtin_wasm_widen_low_u_i16x8_i8x16((__i8x16)__a);
+  return (v128_t) __builtin_convertvector(
+      (__u8x8){((__u8x16)__a)[0], ((__u8x16)__a)[1], ((__u8x16)__a)[2],
+               ((__u8x16)__a)[3], ((__u8x16)__a)[4], ((__u8x16)__a)[5],
+               ((__u8x16)__a)[6], ((__u8x16)__a)[7]},
+      __u16x8);
 }
 
 static __inline__ v128_t __DEFAULT_FN_ATTRS
 wasm_i16x8_widen_high_u8x16(v128_t __a) {
-  return (v128_t)__builtin_wasm_widen_high_u_i16x8_i8x16((__i8x16)__a);
+  return (v128_t) __builtin_convertvector(
+      (__u8x8){((__u8x16)__a)[8], ((__u8x16)__a)[9], ((__u8x16)__a)[10],
+               ((__u8x16)__a)[11], ((__u8x16)__a)[12], ((__u8x16)__a)[13],
+               ((__u8x16)__a)[14], ((__u8x16)__a)[15]},
+      __u16x8);
 }
 
 static __inline__ v128_t __DEFAULT_FN_ATTRS
 wasm_i32x4_widen_low_i16x8(v128_t __a) {
-  return (v128_t)__builtin_wasm_widen_low_s_i32x4_i16x8((__i16x8)__a);
+  return (v128_t) __builtin_convertvector(
+      (__i16x4){((__i16x8)__a)[0], ((__i16x8)__a)[1], ((__i16x8)__a)[2],
+                ((__i16x8)__a)[3]},
+      __i32x4);
 }
 
 static __inline__ v128_t __DEFAULT_FN_ATTRS
 wasm_i32x4_widen_high_i16x8(v128_t __a) {
-  return (v128_t)__builtin_wasm_widen_high_s_i32x4_i16x8((__i16x8)__a);
+  return (v128_t) __builtin_convertvector(
+      (__i16x4){((__i16x8)__a)[4], ((__i16x8)__a)[5], ((__i16x8)__a)[6],
+                ((__i16x8)__a)[7]},
+      __i32x4);
 }
 
 static __inline__ v128_t __DEFAULT_FN_ATTRS
 wasm_i32x4_widen_low_u16x8(v128_t __a) {
-  return (v128_t)__builtin_wasm_widen_low_u_i32x4_i16x8((__i16x8)__a);
+  return (v128_t) __builtin_convertvector(
+      (__u16x4){((__u16x8)__a)[0], ((__u16x8)__a)[1], ((__u16x8)__a)[2],
+                ((__u16x8)__a)[3]},
+      __u32x4);
 }
 
 static __inline__ v128_t __DEFAULT_FN_ATTRS
 wasm_i32x4_widen_high_u16x8(v128_t __a) {
-  return (v128_t)__builtin_wasm_widen_high_u_i32x4_i16x8((__i16x8)__a);
+  return (v128_t) __builtin_convertvector(
+      (__u16x4){((__u16x8)__a)[4], ((__u16x8)__a)[5], ((__u16x8)__a)[6],
+                ((__u16x8)__a)[7]},
+      __u32x4);
 }
 
 // Undefine helper macros
diff --git a/clang/test/CodeGen/builtins-wasm.c b/clang/test/CodeGen/builtins-wasm.c
index f7e3dc1ea5e7a..0f66fceef4cca 100644
--- a/clang/test/CodeGen/builtins-wasm.c
+++ b/clang/test/CodeGen/builtins-wasm.c
@@ -737,54 +737,6 @@ i16x8 narrow_u_i16x8_i32x4(i32x4 low, i32x4 high) {
   // WEBASSEMBLY: ret
 }
 
-i16x8 widen_low_s_i16x8_i8x16(i8x16 v) {
-  return __builtin_wasm_widen_low_s_i16x8_i8x16(v);
-  // WEBASSEMBLY: call <8 x i16> @llvm.wasm.widen.low.signed.v8i16.v16i8(<16 x i8> %v)
-  // WEBASSEMBLY: ret
-}
-
-i16x8 widen_high_s_i16x8_i8x16(i8x16 v) {
-  return __builtin_wasm_widen_high_s_i16x8_i8x16(v);
-  // WEBASSEMBLY: call <8 x i16> @llvm.wasm.widen.high.signed.v8i16.v16i8(<16 x i8> %v)
-  // WEBASSEMBLY: ret
-}
-
-i16x8 widen_low_u_i16x8_i8x16(i8x16 v) {
-  return __builtin_wasm_widen_low_u_i16x8_i8x16(v);
-  // WEBASSEMBLY: call <8 x i16> @llvm.wasm.widen.low.unsigned.v8i16.v16i8(<16 x i8> %v)
-  // WEBASSEMBLY: ret
-}
-
-i16x8 widen_high_u_i16x8_i8x16(i8x16 v) {
-  return __builtin_wasm_widen_high_u_i16x8_i8x16(v);
-  // WEBASSEMBLY: call <8 x i16> @llvm.wasm.widen.high.unsigned.v8i16.v16i8(<16 x i8> %v)
-  // WEBASSEMBLY: ret
-}
-
-i32x4 widen_low_s_i32x4_i16x8(i16x8 v) {
-  return __builtin_wasm_widen_low_s_i32x4_i16x8(v);
-  // WEBASSEMBLY: call <4 x i32> @llvm.wasm.widen.low.signed.v4i32.v8i16(<8 x i16> %v)
-  // WEBASSEMBLY: ret
-}
-
-i32x4 widen_high_s_i32x4_i16x8(i16x8 v) {
-  return __builtin_wasm_widen_high_s_i32x4_i16x8(v);
-  // WEBASSEMBLY: call <4 x i32> @llvm.wasm.widen.high.signed.v4i32.v8i16(<8 x i16> %v)
-  // WEBASSEMBLY: ret
-}
-
-i32x4 widen_low_u_i32x4_i16x8(i16x8 v) {
-  return __builtin_wasm_widen_low_u_i32x4_i16x8(v);
-  // WEBASSEMBLY: call <4 x i32> @llvm.wasm.widen.low.unsigned.v4i32.v8i16(<8 x i16> %v)
-  // WEBASSEMBLY: ret
-}
-
-i32x4 widen_high_u_i32x4_i16x8(i16x8 v) {
-  return __builtin_wasm_widen_high_u_i32x4_i16x8(v);
-  // WEBASSEMBLY: call <4 x i32> @llvm.wasm.widen.high.unsigned.v4i32.v8i16(<8 x i16> %v)
-  // WEBASSEMBLY: ret
-}
-
 i8x16 swizzle_v8x16(i8x16 x, i8x16 y) {
   return __builtin_wasm_swizzle_v8x16(x, y);
   // WEBASSEMBLY: call <16 x i8> @llvm.wasm.swizzle(<16 x i8> %x, <16 x i8> %y)
diff --git a/llvm/include/llvm/IR/IntrinsicsWebAssembly.td b/llvm/include/llvm/IR/IntrinsicsWebAssembly.td
index 7c9ceb148a472..9cc9f9eb6f187 100644
--- a/llvm/include/llvm/IR/IntrinsicsWebAssembly.td
+++ b/llvm/include/llvm/IR/IntrinsicsWebAssembly.td
@@ -159,22 +159,6 @@ def int_wasm_narrow_unsigned :
   Intrinsic<[llvm_anyvector_ty],
             [llvm_anyvector_ty, LLVMMatchType<1>],
             [IntrNoMem, IntrSpeculatable]>;
-def int_wasm_widen_low_signed :
-  Intrinsic<[llvm_anyvector_ty],
-            [llvm_anyvector_ty],
-            [IntrNoMem, IntrSpeculatable]>;
-def int_wasm_widen_high_signed :
-  Intrinsic<[llvm_anyvector_ty],
-            [llvm_anyvector_ty],
-            [IntrNoMem, IntrSpeculatable]>;
-def int_wasm_widen_low_unsigned :
-  Intrinsic<[llvm_anyvector_ty],
-            [llvm_anyvector_ty],
-            [IntrNoMem, IntrSpeculatable]>;
-def int_wasm_widen_high_unsigned :
-  Intrinsic<[llvm_anyvector_ty],
-            [llvm_anyvector_ty],
-            [IntrNoMem, IntrSpeculatable]>;
 
 // TODO: Replace these intrinsics with normal ISel patterns
 def int_wasm_pmin :
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISD.def b/llvm/lib/Target/WebAssembly/WebAssemblyISD.def
index dee1c4e281494..5720d3e5afb02 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyISD.def
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyISD.def
@@ -29,6 +29,10 @@ HANDLE_NODETYPE(SWIZZLE)
 HANDLE_NODETYPE(VEC_SHL)
 HANDLE_NODETYPE(VEC_SHR_S)
 HANDLE_NODETYPE(VEC_SHR_U)
+HANDLE_NODETYPE(WIDEN_LOW_S)
+HANDLE_NODETYPE(WIDEN_LOW_U)
+HANDLE_NODETYPE(WIDEN_HIGH_S)
+HANDLE_NODETYPE(WIDEN_HIGH_U)
 HANDLE_NODETYPE(THROW)
 HANDLE_NODETYPE(MEMORY_COPY)
 HANDLE_NODETYPE(MEMORY_FILL)
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
index d6197e9ea84c2..cdfbfe388abbf 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
@@ -123,6 +123,10 @@ WebAssemblyTargetLowering::WebAssemblyTargetLowering(
     // Hoist bitcasts out of shuffles
     setTargetDAGCombine(ISD::VECTOR_SHUFFLE);
 
+    // Combine extends of extract_subvectors into widening ops
+    setTargetDAGCombine(ISD::SIGN_EXTEND);
+    setTargetDAGCombine(ISD::ZERO_EXTEND);
+
     // Support saturating add for i8x16 and i16x8
     for (auto Op : {ISD::SADDSAT, ISD::UADDSAT})
       for (auto T : {MVT::v16i8, MVT::v8i16})
@@ -1745,6 +1749,49 @@ performVECTOR_SHUFFLECombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
   return DAG.getBitcast(DstType, NewShuffle);
 }
 
+static SDValue performVectorWidenCombine(SDNode *N,
+                                         TargetLowering::DAGCombinerInfo &DCI) {
+  auto &DAG = DCI.DAG;
+  assert(N->getOpcode() == ISD::SIGN_EXTEND ||
+         N->getOpcode() == ISD::ZERO_EXTEND);
+
+  // Combine ({s,z}ext (extract_subvector src, i)) into a widening operation if
+  // possible before the extract_subvector can be expanded.
+  auto Extract = N->getOperand(0);
+  if (Extract.getOpcode() != ISD::EXTRACT_SUBVECTOR)
+    return SDValue();
+  auto Source = Extract.getOperand(0);
+  auto *IndexNode = dyn_cast<ConstantSDNode>(Extract.getOperand(1));
+  if (IndexNode == nullptr)
+    return SDValue();
+  auto Index = IndexNode->getZExtValue();
+
+  // Only v8i8 and v4i16 extracts can be widened, and only if the extracted
+  // subvector is the low or high half of its source.
+  EVT ResVT = N->getValueType(0);
+  if (ResVT == MVT::v8i16) {
+    if (Extract.getValueType() != MVT::v8i8 ||
+        Source.getValueType() != MVT::v16i8 || (Index != 0 && Index != 8))
+      return SDValue();
+  } else if (ResVT == MVT::v4i32) {
+    if (Extract.getValueType() != MVT::v4i16 ||
+        Source.getValueType() != MVT::v8i16 || (Index != 0 && Index != 4))
+      return SDValue();
+  } else {
+    return SDValue();
+  }
+
+  bool IsSext = N->getOpcode() == ISD::SIGN_EXTEND;
+  bool IsLow = Index == 0;
+
+  unsigned Op = IsSext ? (IsLow ? WebAssemblyISD::WIDEN_LOW_S
+                                : WebAssemblyISD::WIDEN_HIGH_S)
+                       : (IsLow ? WebAssemblyISD::WIDEN_LOW_U
+                                : WebAssemblyISD::WIDEN_HIGH_U);
+
+  return DAG.getNode(Op, SDLoc(N), ResVT, Source);
+}
+
 SDValue
 WebAssemblyTargetLowering::PerformDAGCombine(SDNode *N,
                                              DAGCombinerInfo &DCI) const {
@@ -1753,5 +1800,8 @@ WebAssemblyTargetLowering::PerformDAGCombine(SDNode *N,
     return SDValue();
   case ISD::VECTOR_SHUFFLE:
     return performVECTOR_SHUFFLECombine(N, DCI);
+  case ISD::SIGN_EXTEND:
+  case ISD::ZERO_EXTEND:
+    return performVectorWidenCombine(N, DCI);
   }
 }
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td b/llvm/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td
index 16bfc81af8b72..9bbccecffaa09 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td
@@ -892,15 +892,21 @@ def : Pat<(v4i32 (int_wasm_trunc_saturate_unsigned (v4f32 V128:$src))),
           (fp_to_uint_v4i32_v4f32 (v4f32 V128:$src))>;
 
 // Widening operations
+def widen_t : SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisVec<1>]>;
+def widen_low_s : SDNode<"WebAssemblyISD::WIDEN_LOW_S", widen_t>;
+def widen_high_s : SDNode<"WebAssemblyISD::WIDEN_HIGH_S", widen_t>;
+def widen_low_u : SDNode<"WebAssemblyISD::WIDEN_LOW_U", widen_t>;
+def widen_high_u : SDNode<"WebAssemblyISD::WIDEN_HIGH_U", widen_t>;
+
 multiclass SIMDWiden<ValueType vec_t, string vec, ValueType arg_t, string arg,
                      bits<32> baseInst> {
-  defm "" : SIMDConvert<vec_t, arg_t, int_wasm_widen_low_signed,
+  defm "" : SIMDConvert<vec_t, arg_t, widen_low_s,
                         vec#".widen_low_"#arg#"_s", baseInst>;
-  defm "" : SIMDConvert<vec_t, arg_t, int_wasm_widen_high_signed,
+  defm "" : SIMDConvert<vec_t, arg_t, widen_high_s,
                         vec#".widen_high_"#arg#"_s", !add(baseInst, 1)>;
-  defm "" : SIMDConvert<vec_t, arg_t, int_wasm_widen_low_unsigned,
+  defm "" : SIMDConvert<vec_t, arg_t, widen_low_u,
                         vec#".widen_low_"#arg#"_u", !add(baseInst, 2)>;
-  defm "" : SIMDConvert<vec_t, arg_t, int_wasm_widen_high_unsigned,
+  defm "" : SIMDConvert<vec_t, arg_t, widen_high_u,
                         vec#".widen_high_"#arg#"_u", !add(baseInst, 3)>;
 }
 
diff --git a/llvm/test/CodeGen/WebAssembly/simd-intrinsics.ll b/llvm/test/CodeGen/WebAssembly/simd-intrinsics.ll
index 05d256fa5524c..63092a8c51786 100644
--- a/llvm/test/CodeGen/WebAssembly/simd-intrinsics.ll
+++ b/llvm/test/CodeGen/WebAssembly/simd-intrinsics.ll
@@ -294,46 +294,6 @@ define <8 x i16> @narrow_unsigned_v8i16(<4 x i32> %low, <4 x i32> %high) {
   ret <8 x i16> %a
 }
 
-; CHECK-LABEL: widen_low_signed_v8i16:
-; SIMD128-NEXT: .functype widen_low_signed_v8i16 (v128) -> (v128){{$}}
-; SIMD128-NEXT: i16x8.widen_low_i8x16_s $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
-declare <8 x i16> @llvm.wasm.widen.low.signed.v8i16.v16i8(<16 x i8>)
-define <8 x i16> @widen_low_signed_v8i16(<16 x i8> %v) {
-  %a = call <8 x i16> @llvm.wasm.widen.low.signed.v8i16.v16i8(<16 x i8> %v)
-  ret <8 x i16> %a
-}
-
-; CHECK-LABEL: widen_high_signed_v8i16:
-; SIMD128-NEXT: .functype widen_high_signed_v8i16 (v128) -> (v128){{$}}
-; SIMD128-NEXT: i16x8.widen_high_i8x16_s $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
-declare <8 x i16> @llvm.wasm.widen.high.signed.v8i16.v16i8(<16 x i8>)
-define <8 x i16> @widen_high_signed_v8i16(<16 x i8> %v) {
-  %a = call <8 x i16> @llvm.wasm.widen.high.signed.v8i16.v16i8(<16 x i8> %v)
-  ret <8 x i16> %a
-}
-
-; CHECK-LABEL: widen_low_unsigned_v8i16:
-; SIMD128-NEXT: .functype widen_low_unsigned_v8i16 (v128) -> (v128){{$}}
-; SIMD128-NEXT: i16x8.widen_low_i8x16_u $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
-declare <8 x i16> @llvm.wasm.widen.low.unsigned.v8i16.v16i8(<16 x i8>)
-define <8 x i16> @widen_low_unsigned_v8i16(<16 x i8> %v) {
-  %a = call <8 x i16> @llvm.wasm.widen.low.unsigned.v8i16.v16i8(<16 x i8> %v)
-  ret <8 x i16> %a
-}
-
-; CHECK-LABEL: widen_high_unsigned_v8i16:
-; SIMD128-NEXT: .functype widen_high_unsigned_v8i16 (v128) -> (v128){{$}}
-; SIMD128-NEXT: i16x8.widen_high_i8x16_u $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
-declare <8 x i16> @llvm.wasm.widen.high.unsigned.v8i16.v16i8(<16 x i8>)
-define <8 x i16> @widen_high_unsigned_v8i16(<16 x i8> %v) {
-  %a = call <8 x i16> @llvm.wasm.widen.high.unsigned.v8i16.v16i8(<16 x i8> %v)
-  ret <8 x i16> %a
-}
-
 ; ==============================================================================
 ; 4 x i32
 ; ==============================================================================
@@ -411,46 +371,6 @@ define <4 x i32> @trunc_sat_u_v4i32(<4 x float> %x) {
   ret <4 x i32> %a
 }
 
-; CHECK-LABEL: widen_low_signed_v4i32:
-; SIMD128-NEXT: .functype widen_low_signed_v4i32 (v128) -> (v128){{$}}
-; SIMD128-NEXT: i32x4.widen_low_i16x8_s $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
-declare <4 x i32> @llvm.wasm.widen.low.signed.v4i32.v8i16(<8 x i16>)
-define <4 x i32> @widen_low_signed_v4i32(<8 x i16> %v) {
-  %a = call <4 x i32> @llvm.wasm.widen.low.signed.v4i32.v8i16(<8 x i16> %v)
-  ret <4 x i32> %a
-}
-
-; CHECK-LABEL: widen_high_signed_v4i32:
-; SIMD128-NEXT: .functype widen_high_signed_v4i32 (v128) -> (v128){{$}}
-; SIMD128-NEXT: i32x4.widen_high_i16x8_s $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
-declare <4 x i32> @llvm.wasm.widen.high.signed.v4i32.v8i16(<8 x i16>)
-define <4 x i32> @widen_high_signed_v4i32(<8 x i16> %v) {
-  %a = call <4 x i32> @llvm.wasm.widen.high.signed.v4i32.v8i16(<8 x i16> %v)
-  ret <4 x i32> %a
-}
-
-; CHECK-LABEL: widen_low_unsigned_v4i32:
-; SIMD128-NEXT: .functype widen_low_unsigned_v4i32 (v128) -> (v128){{$}}
-; SIMD128-NEXT: i32x4.widen_low_i16x8_u $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
-declare <4 x i32> @llvm.wasm.widen.low.unsigned.v4i32.v8i16(<8 x i16>)
-define <4 x i32> @widen_low_unsigned_v4i32(<8 x i16> %v) {
-  %a = call <4 x i32> @llvm.wasm.widen.low.unsigned.v4i32.v8i16(<8 x i16> %v)
-  ret <4 x i32> %a
-}
-
-; CHECK-LABEL: widen_high_unsigned_v4i32:
-; SIMD128-NEXT: .functype widen_high_unsigned_v4i32 (v128) -> (v128){{$}}
-; SIMD128-NEXT: i32x4.widen_high_i16x8_u $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
-declare <4 x i32> @llvm.wasm.widen.high.unsigned.v4i32.v8i16(<8 x i16>)
-define <4 x i32> @widen_high_unsigned_v4i32(<8 x i16> %v) {
-  %a = call <4 x i32> @llvm.wasm.widen.high.unsigned.v4i32.v8i16(<8 x i16> %v)
-  ret <4 x i32> %a
-}
-
 ; ==============================================================================
 ; 2 x i64
 ; ==============================================================================
diff --git a/llvm/test/CodeGen/WebAssembly/simd-widening.ll b/llvm/test/CodeGen/WebAssembly/simd-widening.ll
new file mode 100644
index 0000000000000..c9a7ffbbfcaf5
--- /dev/null
+++ b/llvm/test/CodeGen/WebAssembly/simd-widening.ll
@@ -0,0 +1,180 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mattr=+simd128 | FileCheck %s
+
+;; Test that SIMD widening operations can be successfully selected
+
+target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
+target triple = "wasm32-unknown-unknown"
+
+define <8 x i16> @widen_low_i8x16_s(<16 x i8> %v) {
+; CHECK-LABEL: widen_low_i8x16_s:
+; CHECK:         .functype widen_low_i8x16_s (v128) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i16x8.widen_low_i8x16_s
+; CHECK-NEXT:    # fallthrough-return
+  %low = shufflevector <16 x i8> %v, <16 x i8> undef,
+           <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %widened = sext <8 x i8> %low to <8 x i16>
+  ret <8 x i16> %widened
+}
+
+define <8 x i16> @widen_low_i8x16_u(<16 x i8> %v) {
+; CHECK-LABEL: widen_low_i8x16_u:
+; CHECK:         .functype widen_low_i8x16_u (v128) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i16x8.widen_low_i8x16_u
+; CHECK-NEXT:    # fallthrough-return
+  %low = shufflevector <16 x i8> %v, <16 x i8> undef,
+           <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %widened = zext <8 x i8> %low to <8 x i16>
+  ret <8 x i16> %widened
+}
+
+define <8 x i16> @widen_high_i8x16_s(<16 x i8> %v) {
+; CHECK-LABEL: widen_high_i8x16_s:
+; CHECK:         .functype widen_high_i8x16_s (v128) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i16x8.widen_high_i8x16_s
+; CHECK-NEXT:    # fallthrough-return
+  %low = shufflevector <16 x i8> %v, <16 x i8> undef,
+           <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %widened = sext <8 x i8> %low to <8 x i16>
+  ret <8 x i16> %widened
+}
+
+define <8 x i16> @widen_high_i8x16_u(<16 x i8> %v) {
+; CHECK-LABEL: widen_high_i8x16_u:
+; CHECK:         .functype widen_high_i8x16_u (v128) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i16x8.widen_high_i8x16_u
+; CHECK-NEXT:    # fallthrough-return
+  %low = shufflevector <16 x i8> %v, <16 x i8> undef,
+           <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %widened = zext <8 x i8> %low to <8 x i16>
+  ret <8 x i16> %widened
+}
+
+define <4 x i32> @widen_low_i16x8_s(<8 x i16> %v) {
+; CHECK-LABEL: widen_low_i16x8_s:
+; CHECK:         .functype widen_low_i16x8_s (v128) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32x4.widen_low_i16x8_s
+; CHECK-NEXT:    # fallthrough-return
+  %low = shufflevector <8 x i16> %v, <8 x i16> undef,
+           <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %widened = sext <4 x i16> %low to <4 x i32>
+  ret <4 x i32> %widened
+}
+
+define <4 x i32> @widen_low_i16x8_u(<8 x i16> %v) {
+; CHECK-LABEL: widen_low_i16x8_u:
+; CHECK:         .functype widen_low_i16x8_u (v128) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32x4.widen_low_i16x8_u
+; CHECK-NEXT:    # fallthrough-return
+  %low = shufflevector <8 x i16> %v, <8 x i16> undef,
+           <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %widened = zext <4 x i16> %low to <4 x i32>
+  ret <4 x i32> %widened
+}
+
+define <4 x i32> @widen_high_i16x8_s(<8 x i16> %v) {
+; CHECK-LABEL: widen_high_i16x8_s:
+; CHECK:         .functype widen_high_i16x8_s (v128) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32x4.widen_high_i16x8_s
+; CHECK-NEXT:    # fallthrough-return
+  %low = shufflevector <8 x i16> %v, <8 x i16> undef,
+           <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %widened = sext <4 x i16> %low to <4 x i32>
+  ret <4 x i32> %widened
+}
+
+define <4 x i32> @widen_high_i16x8_u(<8 x i16> %v) {
+; CHECK-LABEL: widen_high_i16x8_u:
+; CHECK:         .functype widen_high_i16x8_u (v128) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32x4.widen_high_i16x8_u
+; CHECK-NEXT:    # fallthrough-return
+  %low = shufflevector <8 x i16> %v, <8 x i16> undef,
+           <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %widened = zext <4 x i16> %low to <4 x i32>
+  ret <4 x i32> %widened
+}
+
+;; Also test that similar patterns with offsets not corresponding to
+;; the low or high half are correctly expanded.
+
+define <8 x i16> @widen_lowish_i8x16_s(<16 x i8> %v) {
+; CHECK-LABEL: widen_lowish_i8x16_s:
+; CHECK:         .functype widen_lowish_i8x16_s (v128) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i8x16.extract_lane_u 1
+; CHECK-NEXT:    i16x8.splat
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i8x16.extract_lane_u 2
+; CHECK-NEXT:    i16x8.replace_lane 1
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i8x16.extract_lane_u 3
+; CHECK-NEXT:    i16x8.replace_lane 2
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i8x16.extract_lane_u 4
+; CHECK-NEXT:    i16x8.replace_lane 3
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i8x16.extract_lane_u 5
+; CHECK-NEXT:    i16x8.replace_lane 4
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i8x16.extract_lane_u 6
+; CHECK-NEXT:    i16x8.replace_lane 5
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i8x16.extract_lane_u 7
+; CHECK-NEXT:    i16x8.replace_lane 6
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i8x16.extract_lane_u 8
+; CHECK-NEXT:    i16x8.replace_lane 7
+; CHECK-NEXT:    i32.const 8
+; CHECK-NEXT:    i16x8.shl
+; CHECK-NEXT:    i32.const 8
+; CHECK-NEXT:    i16x8.shr_s
+; CHECK-NEXT:    # fallthrough-return
+  %lowish = shufflevector <16 x i8> %v, <16 x i8> undef,
+           <8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
+  %widened = sext <8 x i8> %lowish to <8 x i16>
+  ret <8 x i16> %widened
+}
+
+define <4 x i32> @widen_lowish_i16x8_s(<8 x i16> %v) {
+; CHECK-LABEL: widen_lowish_i16x8_s:
+; CHECK:         .functype widen_lowish_i16x8_s (v128) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i16x8.extract_lane_u 1
+; CHECK-NEXT:    i32x4.splat
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i16x8.extract_lane_u 2
+; CHECK-NEXT:    i32x4.replace_lane 1
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i16x8.extract_lane_u 3
+; CHECK-NEXT:    i32x4.replace_lane 2
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i16x8.extract_lane_u 4
+; CHECK-NEXT:    i32x4.replace_lane 3
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32x4.shl
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32x4.shr_s
+; CHECK-NEXT:    # fallthrough-return
+  %lowish = shufflevector <8 x i16> %v, <8 x i16> undef,
+           <4 x i32> <i32 1, i32 2, i32 3, i32 4>
+  %widened = sext <4 x i16> %lowish to <4 x i32>
+  ret <4 x i32> %widened
+}

From 706d992cedaf2ca3190e4445015da62faf2db544 Mon Sep 17 00:00:00 2001
From: Rahul Joshi <jurahul@google.com>
Date: Tue, 28 Jul 2020 15:26:36 -0700
Subject: [PATCH 0408/1035] [NFC] Add getArgumentTypes() to Region

- Add getArgumentTypes() to Region (missed from before)
- Adopt Region argument API in `hasMultiplyAddBody`
- Fix 2 typos in comments

Differential Revision: https://reviews.llvm.org/D84807
---
 mlir/include/mlir/IR/Region.h                        | 3 +++
 mlir/include/mlir/IR/Visitors.h                      | 2 +-
 mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp | 6 +++---
 mlir/lib/IR/Region.cpp                               | 5 +++++
 mlir/lib/Transforms/Utils/InliningUtils.cpp          | 2 +-
 5 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/mlir/include/mlir/IR/Region.h b/mlir/include/mlir/IR/Region.h
index 5671f2b5581e2..9b6f54fec9363 100644
--- a/mlir/include/mlir/IR/Region.h
+++ b/mlir/include/mlir/IR/Region.h
@@ -74,6 +74,9 @@ class Region {
   BlockArgListType getArguments() {
     return empty() ? BlockArgListType() : front().getArguments();
   }
+
+  ValueTypeRange<BlockArgListType> getArgumentTypes();
+
   using args_iterator = BlockArgListType::iterator;
   using reverse_args_iterator = BlockArgListType::reverse_iterator;
   args_iterator args_begin() { return getArguments().begin(); }
diff --git a/mlir/include/mlir/IR/Visitors.h b/mlir/include/mlir/IR/Visitors.h
index 09b7441aece1f..8554b7a7b8854 100644
--- a/mlir/include/mlir/IR/Visitors.h
+++ b/mlir/include/mlir/IR/Visitors.h
@@ -76,7 +76,7 @@ WalkResult walkOperations(Operation *op,
 // upon the type of the callback function.
 
 /// Walk all of the operations nested under and including the given operation.
-/// This method is selected for callbacks that operation on Operation*.
+/// This method is selected for callbacks that operate on Operation*.
 ///
 /// Example:
 ///   op->walk([](Operation *op) { ... });
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp b/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
index 23d89c21e6e0a..559d411469033 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
@@ -43,9 +43,9 @@ static bool hasMultiplyAddBody(Region &r) {
     return false;
 
   using mlir::matchers::m_Val;
-  auto a = m_Val(r.front().getArgument(0));
-  auto b = m_Val(r.front().getArgument(1));
-  auto c = m_Val(r.front().getArgument(2));
+  auto a = m_Val(r.getArgument(0));
+  auto b = m_Val(r.getArgument(1));
+  auto c = m_Val(r.getArgument(2));
   // TODO: Update this detection once we have  matcher support for specifying
   // that any permutation of operands matches.
   auto pattern1 = m_Op<YieldOp>(m_Op<AddFOp>(m_Op<MulFOp>(a, b), c));
diff --git a/mlir/lib/IR/Region.cpp b/mlir/lib/IR/Region.cpp
index b616eaa15422d..76101382cd585 100644
--- a/mlir/lib/IR/Region.cpp
+++ b/mlir/lib/IR/Region.cpp
@@ -33,6 +33,11 @@ Location Region::getLoc() {
   return container->getLoc();
 }
 
+/// Return a range containing the types of the arguments for this region.
+auto Region::getArgumentTypes() -> ValueTypeRange<BlockArgListType> {
+  return ValueTypeRange<BlockArgListType>(getArguments());
+}
+
 /// Add one argument to the argument list for each type specified in the list.
 iterator_range<Region::args_iterator> Region::addArguments(TypeRange types) {
   return front().addArguments(types);
diff --git a/mlir/lib/Transforms/Utils/InliningUtils.cpp b/mlir/lib/Transforms/Utils/InliningUtils.cpp
index 31f24a43c2b86..4b7ae80241155 100644
--- a/mlir/lib/Transforms/Utils/InliningUtils.cpp
+++ b/mlir/lib/Transforms/Utils/InliningUtils.cpp
@@ -334,7 +334,7 @@ LogicalResult mlir::inlineCall(InlinerInterface &interface,
     mapper.map(regionArg, operand);
   }
 
-  // Ensure that the resultant values of the call, match the callable.
+  // Ensure that the resultant values of the call match the callable.
   castBuilder.setInsertionPointAfter(call);
   for (unsigned i = 0, e = callResults.size(); i != e; ++i) {
     Value callResult = callResults[i];

From b6ebc77326843deeb34006bf278329bcf5e530b9 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Sat, 25 Jul 2020 12:34:59 -0400
Subject: [PATCH 0409/1035] AMDGPU/GlobalISel: Fix selecting
 llvm.amdgcn.s.getreg

This introduces the same bug llvm.amdgcn.s.setreg has where if the
user specified an immediate outside of the valid 16-bit range, it will
select into a verifier error.
---
 llvm/lib/Target/AMDGPU/SIInstrInfo.td            |  6 +++---
 llvm/lib/Target/AMDGPU/SOPInstructions.td        | 16 ++++++----------
 llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.getreg.ll |  7 ++++++-
 3 files changed, 15 insertions(+), 14 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.td b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
index 72feff80ac81c..b6c2082b8d23d 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.td
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
@@ -1389,9 +1389,9 @@ def HWREG {
 }
 
 class getHwRegImm<int Reg, int Offset = 0, int Size = 32> {
-  int ret = !or(Reg,
-                !or(!shl(Offset, 6),
-                    !shl(!add(Size, -1), 11)));
+  int ret = !and(!or(Reg,
+                  !or(!shl(Offset, 6),
+                      !shl(!add(Size, -1), 11))), 65535);
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/AMDGPU/SOPInstructions.td b/llvm/lib/Target/AMDGPU/SOPInstructions.td
index d60fa58a0a74e..92cb2807611af 100644
--- a/llvm/lib/Target/AMDGPU/SOPInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SOPInstructions.td
@@ -806,15 +806,19 @@ let hasSideEffects = 1 in {
 let mayLoad = 1 in {
 // s_getreg_b32 should use hasSideEffects = 1 for tablegen to allow
 // its use in the readcyclecounter selection.
+// FIXME: Need to truncate immediate to 16-bits.
 def S_GETREG_B32 : SOPK_Pseudo <
   "s_getreg_b32",
   (outs SReg_32:$sdst), (ins hwreg:$simm16),
-  "$sdst, $simm16"
->;
+  "$sdst, $simm16",
+  [(set i32:$sdst, (int_amdgcn_s_getreg (i32 timm:$simm16)))]> {
+  let SOPKZext = 1;
+}
 }
 
 let mayLoad = 0, mayStore =0 in {
 
+// FIXME: Need to truncate immediate to 16-bits.
 def S_SETREG_B32 : SOPK_Pseudo <
   "s_setreg_b32",
   (outs), (ins SReg_32:$sdst, hwreg:$simm16),
@@ -1250,14 +1254,6 @@ let SubtargetPredicate = isGFX10Plus in {
     SOPP<0x028, (ins s16imm:$simm16), "s_ttracedata_imm $simm16">;
 } // End SubtargetPredicate = isGFX10Plus
 
-//===----------------------------------------------------------------------===//
-// S_GETREG_B32 Intrinsic Pattern.
-//===----------------------------------------------------------------------===//
-def : GCNPat <
-  (int_amdgcn_s_getreg timm:$simm16),
-  (S_GETREG_B32 (as_i16imm $simm16))
->;
-
 //===----------------------------------------------------------------------===//
 // SOP1 Patterns
 //===----------------------------------------------------------------------===//
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.getreg.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.getreg.ll
index 906a8a3e05f44..69cbf62c538d2 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.getreg.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.getreg.ll
@@ -2,6 +2,11 @@
 ; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=kaveri -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
 ; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
 
+; RUN: llc -global-isel -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -global-isel -mtriple=amdgcn--amdhsa -mcpu=kaveri -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -global-isel -mtriple=amdgcn--amdhsa -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+
+
 ; GCN-LABEL: {{^}}s_getreg_test:
 ; GCN: s_getreg_b32 s{{[0-9]+}}, hwreg(HW_REG_LDS_ALLOC, 8, 23)
 define amdgpu_kernel void @s_getreg_test(i32 addrspace(1)* %out) { ; simm16=45574 for lds size.
@@ -21,7 +26,7 @@ define amdgpu_kernel void @readnone_s_getreg_test(i32 addrspace(1)* %out) { ; si
   ret void
 }
 
-declare i32 @llvm.amdgcn.s.getreg(i32) #0
+declare i32 @llvm.amdgcn.s.getreg(i32 immarg) #0
 
 attributes #0 = { nounwind readonly }
 attributes #1 = { nounwind readnone }

From 44211f20a82418b20a69df9de6291d923dcbf709 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Thu, 23 Jul 2020 19:42:37 -0400
Subject: [PATCH 0410/1035] AMDGPU: Optimize copies to exec with other insts
 after exec def

It's possible to have terminator instructions after a write to exec,
so skip over them to find it.
---
 .../Target/AMDGPU/SIOptimizeExecMasking.cpp   | 17 +++++--
 ...ize-exec-copies-extra-insts-after-copy.mir | 51 +++++++++++++++++++
 2 files changed, 65 insertions(+), 3 deletions(-)
 create mode 100644 llvm/test/CodeGen/AMDGPU/optimize-exec-copies-extra-insts-after-copy.mir

diff --git a/llvm/lib/Target/AMDGPU/SIOptimizeExecMasking.cpp b/llvm/lib/Target/AMDGPU/SIOptimizeExecMasking.cpp
index 2994d1954d710..449b6287a87b7 100644
--- a/llvm/lib/Target/AMDGPU/SIOptimizeExecMasking.cpp
+++ b/llvm/lib/Target/AMDGPU/SIOptimizeExecMasking.cpp
@@ -300,9 +300,20 @@ bool SIOptimizeExecMasking::runOnMachineFunction(MachineFunction &MF) {
     if (I == E)
       continue;
 
-    // TODO: It's possible to see other terminator copies after the exec copy.
-    Register CopyToExec = isCopyToExec(*I, ST);
-    if (!CopyToExec.isValid())
+    // It's possible to see other terminator copies after the exec copy. This
+    // can happen if control flow pseudos had their outputs used by phis.
+    Register CopyToExec;
+
+    unsigned SearchCount = 0;
+    const unsigned SearchLimit = 5;
+    while (I != E && SearchCount++ < SearchLimit) {
+      CopyToExec = isCopyToExec(*I, ST);
+      if (CopyToExec)
+        break;
+      ++I;
+    }
+
+    if (!CopyToExec)
       continue;
 
     // Scan backwards to find the def.
diff --git a/llvm/test/CodeGen/AMDGPU/optimize-exec-copies-extra-insts-after-copy.mir b/llvm/test/CodeGen/AMDGPU/optimize-exec-copies-extra-insts-after-copy.mir
new file mode 100644
index 0000000000000..b0e67034a4031
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/optimize-exec-copies-extra-insts-after-copy.mir
@@ -0,0 +1,51 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs -run-pass=si-optimize-exec-masking -o - %s | FileCheck %s
+
+# Make sure we can still optimize writes to exec when there are
+# additional terminators after the exec write. This can happen with
+# phi users of control flow intrinsics.
+
+---
+name: instructions_after_copy_to_exec
+tracksRegLiveness: true
+body:             |
+  ; CHECK-LABEL: name: instructions_after_copy_to_exec
+  ; CHECK: bb.0:
+  ; CHECK:   successors: %bb.2(0x40000000), %bb.1(0x40000000)
+  ; CHECK:   liveins: $vgpr0
+  ; CHECK:   renamable $vgpr1 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK:   renamable $vcc = V_CMP_EQ_U32_e64 0, killed $vgpr0, implicit $exec
+  ; CHECK:   $sgpr0_sgpr1 = S_AND_SAVEEXEC_B64 $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK:   renamable $sgpr0_sgpr1 = S_XOR_B64 $exec, killed renamable $sgpr0_sgpr1, implicit-def dead $scc
+  ; CHECK:   renamable $sgpr0_sgpr1 = COPY killed renamable $sgpr0_sgpr1, implicit $exec
+  ; CHECK:   S_CBRANCH_EXECZ %bb.2, implicit $exec
+  ; CHECK: bb.1:
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   liveins: $sgpr0_sgpr1
+  ; CHECK:   S_NOP 0, implicit $sgpr0_sgpr1
+  ; CHECK: bb.2:
+  ; CHECK:   liveins: $sgpr0_sgpr1
+  ; CHECK:   S_NOP 0, implicit $sgpr0_sgpr1
+  bb.0:
+    liveins: $vgpr0
+
+    renamable $vgpr1 = V_MOV_B32_e32 0, implicit $exec
+    renamable $vcc = V_CMP_EQ_U32_e64 0, killed $vgpr0, implicit $exec
+    renamable $sgpr0_sgpr1 = COPY $exec, implicit-def $exec
+    renamable $sgpr2_sgpr3 = S_AND_B64 renamable $sgpr0_sgpr1, renamable $vcc, implicit-def dead $scc
+    renamable $sgpr0_sgpr1 = S_XOR_B64 renamable $sgpr2_sgpr3, killed renamable $sgpr0_sgpr1, implicit-def dead $scc
+    $exec = S_MOV_B64_term killed renamable $sgpr2_sgpr3
+    renamable $sgpr0_sgpr1 = S_MOV_B64_term killed renamable $sgpr0_sgpr1, implicit $exec
+    S_CBRANCH_EXECZ %bb.2, implicit $exec
+
+  bb.1:
+    liveins: $sgpr0_sgpr1
+
+    S_NOP 0, implicit $sgpr0_sgpr1
+
+  bb.2:
+    liveins: $sgpr0_sgpr1
+
+    S_NOP 0, implicit $sgpr0_sgpr1
+
+...

From 00046d789c9db444efa952e4b6e8df86990bfdda Mon Sep 17 00:00:00 2001
From: Kang Zhang <shkzhang@cn.ibm.com>
Date: Wed, 29 Jul 2020 01:47:23 +0000
Subject: [PATCH 0411/1035] [PowerPC] Add Def CR1 for MTFSFI_rec and MTFSF_rec

---
 llvm/lib/Target/PowerPC/PPCInstrInfo.td | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.td b/llvm/lib/Target/PowerPC/PPCInstrInfo.td
index 264002c229b9e..e4ec0f1e5ea13 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrInfo.td
+++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.td
@@ -4320,7 +4320,7 @@ def MCRFS : XLForm_3<63, 64, (outs crrc:$BF), (ins crrc:$BFA),
 
 def MTFSFI : XLForm_4<63, 134, (outs crrc:$BF), (ins i32imm:$U, i32imm:$W),
                       "mtfsfi $BF, $U, $W", IIC_IntMFFS>;
-
+let Defs = [CR1] in
 def MTFSFI_rec : XLForm_4<63, 134, (outs crrc:$BF), (ins i32imm:$U, i32imm:$W),
                        "mtfsfi. $BF, $U, $W", IIC_IntMFFS>, isRecordForm;
 
@@ -4331,6 +4331,7 @@ let Predicates = [HasFPU] in {
 def MTFSF : XFLForm_1<63, 711, (outs),
                       (ins i32imm:$FLM, f8rc:$FRB, i32imm:$L, i32imm:$W),
                       "mtfsf $FLM, $FRB, $L, $W", IIC_IntMFFS, []>;
+let Defs = [CR1] in
 def MTFSF_rec : XFLForm_1<63, 711, (outs),
                        (ins i32imm:$FLM, f8rc:$FRB, i32imm:$L, i32imm:$W),
                        "mtfsf. $FLM, $FRB, $L, $W", IIC_IntMFFS, []>, isRecordForm;

From db0d636e52c163196bc8abeb62b649b7de700569 Mon Sep 17 00:00:00 2001
From: Stefanos Baziotis <sdi1600105@di.uoa.gr>
Date: Wed, 29 Jul 2020 04:51:22 +0300
Subject: [PATCH 0412/1035] [ADT][BitVector][NFC] Merge find_first_in() /
 find_first_unset_in()

We can implement find_first_unset_in() in the same function
if every BitWord we use is first flipped.

Differential Revision: https://reviews.llvm.org/D84717
---
 llvm/include/llvm/ADT/BitVector.h | 40 +++++++++----------------------
 1 file changed, 11 insertions(+), 29 deletions(-)

diff --git a/llvm/include/llvm/ADT/BitVector.h b/llvm/include/llvm/ADT/BitVector.h
index a8d0f07af94a2..2a857786f454f 100644
--- a/llvm/include/llvm/ADT/BitVector.h
+++ b/llvm/include/llvm/ADT/BitVector.h
@@ -203,9 +203,10 @@ class BitVector {
     return !any();
   }
 
-  /// find_first_in - Returns the index of the first set bit in the range
-  /// [Begin, End).  Returns -1 if all bits in the range are unset.
-  int find_first_in(unsigned Begin, unsigned End) const {
+  /// find_first_in - Returns the index of the first set / unset bit,
+  /// depending on \p Set, in the range [Begin, End).
+  /// Returns -1 if all bits in the range are unset / set.
+  int find_first_in(unsigned Begin, unsigned End, bool Set = true) const {
     assert(Begin <= End && End <= Size);
     if (Begin == End)
       return -1;
@@ -214,8 +215,14 @@ class BitVector {
     unsigned LastWord = (End - 1) / BITWORD_SIZE;
 
     // Check subsequent words.
+    // The code below is based on search for the first _set_ bit. If
+    // we're searching for the first _unset_, we just take the
+    // complement of each word before we use it and apply
+    // the same method.
     for (unsigned i = FirstWord; i <= LastWord; ++i) {
       BitWord Copy = Bits[i];
+      if (!Set)
+        Copy = ~Copy;
 
       if (i == FirstWord) {
         unsigned FirstBit = Begin % BITWORD_SIZE;
@@ -266,32 +273,7 @@ class BitVector {
   /// find_first_unset_in - Returns the index of the first unset bit in the
   /// range [Begin, End).  Returns -1 if all bits in the range are set.
   int find_first_unset_in(unsigned Begin, unsigned End) const {
-    assert(Begin <= End && End <= Size);
-    if (Begin == End)
-      return -1;
-
-    unsigned FirstWord = Begin / BITWORD_SIZE;
-    unsigned LastWord = (End - 1) / BITWORD_SIZE;
-
-    // Check subsequent words.
-    for (unsigned i = FirstWord; i <= LastWord; ++i) {
-      BitWord Copy = Bits[i];
-
-      if (i == FirstWord) {
-        unsigned FirstBit = Begin % BITWORD_SIZE;
-        Copy |= maskTrailingOnes<BitWord>(FirstBit);
-      }
-
-      if (i == LastWord) {
-        unsigned LastBit = (End - 1) % BITWORD_SIZE;
-        Copy |= maskTrailingZeros<BitWord>(LastBit + 1);
-      }
-      if (Copy != ~BitWord(0)) {
-        unsigned Result = i * BITWORD_SIZE + countTrailingOnes(Copy);
-        return Result < size() ? Result : -1;
-      }
-    }
-    return -1;
+    return find_first_in(Begin, End, /* Set = */ false);
   }
 
   /// find_last_unset_in - Returns the index of the last unset bit in the

From c26f237cef1b33277f072c609c19192c5213f348 Mon Sep 17 00:00:00 2001
From: Artem Dergachev <artem.dergachev@gmail.com>
Date: Tue, 28 Jul 2020 18:54:01 -0700
Subject: [PATCH 0413/1035] [analyzer] FuchsiaHandleChecker: Suppress a
 non-deterministic test failure.

Noticed by Jon Roelofs in https://reviews.llvm.org/D73151#2180499
---
 clang/test/Analysis/fuchsia_handle.cpp | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/clang/test/Analysis/fuchsia_handle.cpp b/clang/test/Analysis/fuchsia_handle.cpp
index dade5261bd78e..d104b13c77abf 100644
--- a/clang/test/Analysis/fuchsia_handle.cpp
+++ b/clang/test/Analysis/fuchsia_handle.cpp
@@ -77,7 +77,9 @@ void handleDieBeforeErrorSymbol01() {
 void handleDieBeforeErrorSymbol02() {
   zx_handle_t sa, sb;
   zx_status_t status = zx_channel_create(0, &sa, &sb);
-  // expected-note@-1 {{Handle allocated through 2nd parameter}}
+  // FIXME: There appears to be non-determinism in choosing
+  // which handle to report.
+  // expected-note-re@-3 {{Handle allocated through {{(2nd|3rd)}} parameter}}
   if (status == 0) { // expected-note {{Assuming 'status' is equal to 0}}
                      // expected-note@-1 {{Taking true branch}}
     return; // expected-warning {{Potential leak of handle}}

From c12394fef3edc74207801adc6f778a0fb422f195 Mon Sep 17 00:00:00 2001
From: Xing GUO <higuoxing@gmail.com>
Date: Wed, 29 Jul 2020 09:52:32 +0800
Subject: [PATCH 0414/1035] [llvm-readelf][test] Improve wording in the
 comments. NFC.

This patch addresses comments in D84640 (https://reviews.llvm.org/D84640#2178475).
---
 llvm/test/tools/llvm-readobj/ELF/hex-dump.test | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/llvm/test/tools/llvm-readobj/ELF/hex-dump.test b/llvm/test/tools/llvm-readobj/ELF/hex-dump.test
index 5954153c63451..278ccf28029d0 100644
--- a/llvm/test/tools/llvm-readobj/ELF/hex-dump.test
+++ b/llvm/test/tools/llvm-readobj/ELF/hex-dump.test
@@ -49,10 +49,10 @@ FileHeader:
   Type:    ET_DYN
   Machine: EM_386
 
-## Test that llvm-readelf emits a correct amount of spaces between the hex data
+## Test that llvm-readelf emits the correct number of spaces between the hex data
 ## and its ascii representation.
 
-## a) When the hex data doesn't fill the column whose index isn't 4.
+## a) When the hex data doesn't fill a column other than the last one.
 # RUN: yaml2obj --docnum=2 -DSIZE=18 %s -o %t2.out1
 # RUN: llvm-readelf --hex-dump=.sec %t2.out1 | \
 # RUN:   FileCheck %s --match-full-lines --strict-whitespace --check-prefix=SPACES1
@@ -61,7 +61,7 @@ FileHeader:
 # SPACES1-NEXT:0x00000000 00000000 00000000 00000000 00000000 ................
 # SPACES1-NEXT:0x00000010 0000                                ..
 
-## b) When the hex data doesn't fill the column whose index is 4.
+## b) When the hex data doesn't fill the last column.
 # RUN: yaml2obj --docnum=2 -DSIZE=30 %s -o %t2.out2
 # RUN: llvm-readelf --hex-dump=.sec %t2.out2 | \
 # RUN:   FileCheck %s --match-full-lines --strict-whitespace --check-prefix=SPACES2
@@ -70,7 +70,7 @@ FileHeader:
 # SPACES2-NEXT:0x00000000 00000000 00000000 00000000 00000000 ................
 # SPACES2-NEXT:0x00000010 00000000 00000000 00000000 0000     ..............
 
-## c) When the hex data fills the column.
+## c) When the hex data fills a column.
 # RUN: yaml2obj --docnum=2 -DSIZE=28 %s -o %t2.out3
 # RUN: llvm-readelf --hex-dump=.sec %t2.out3 | \
 # RUN:   FileCheck %s --match-full-lines --strict-whitespace --check-prefix=SPACES3

From dd4106d22ef6ce51bfac666584b76dd43e98acf6 Mon Sep 17 00:00:00 2001
From: Chuanqi Xu <yedeng.yd@linux.alibaba.com>
Date: Wed, 29 Jul 2020 10:02:04 +0800
Subject: [PATCH 0415/1035] [NFC] Edit the comment in User::replaceUsesOfWith

---
 llvm/lib/IR/User.cpp           |  2 +-
 llvm/unittests/IR/UserTest.cpp | 32 ++++++++++++++++++++++++++++++++
 2 files changed, 33 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/IR/User.cpp b/llvm/lib/IR/User.cpp
index 7da592f401273..9105c6fbd230d 100644
--- a/llvm/lib/IR/User.cpp
+++ b/llvm/lib/IR/User.cpp
@@ -29,7 +29,7 @@ void User::replaceUsesOfWith(Value *From, Value *To) {
       // The side effects of this setOperand call include linking to
       // "To", adding "this" to the uses list of To, and
       // most importantly, removing "this" from the use list of "From".
-      setOperand(i, To); // Fix it now...
+      setOperand(i, To);
     }
 }
 
diff --git a/llvm/unittests/IR/UserTest.cpp b/llvm/unittests/IR/UserTest.cpp
index e495735aec653..8fd435ecbc2b6 100644
--- a/llvm/unittests/IR/UserTest.cpp
+++ b/llvm/unittests/IR/UserTest.cpp
@@ -117,6 +117,38 @@ TEST(UserTest, ValueOpIteration) {
   EXPECT_EQ(IP->value_op_end(), (CI - 2) + 8);
 }
 
+TEST(UserTest, replaceUseOfWith) {
+  LLVMContext C;
+
+  const char *ModuleString = "define void @f(i32 %x) {\n"
+                             "entry:\n"
+                             "  %v0 = add i32 1, 1\n"
+                             "  %v1 = add i32 %x, 2\n"
+                             "  ret void\n"
+                             "}\n";
+  SMDiagnostic Err;
+  std::unique_ptr<Module> M = parseAssemblyString(ModuleString, Err, C);
+  Function *F = M->getFunction("f");
+  EXPECT_TRUE(F);
+  EXPECT_TRUE(F->arg_begin() != F->arg_end());
+  BasicBlock& entryBB = F->front();
+  Instruction& I0 = *(entryBB.begin());
+  Instruction& I1 = *(++(entryBB.begin()));
+
+  Argument &X = *F->arg_begin();
+  EXPECT_EQ("x", X.getName());
+  EXPECT_NE(X.user_begin() ,X.user_end());
+  EXPECT_EQ(I0.user_begin() ,I0.user_end());
+
+
+  auto XUser = find(X.users(), &(I1));
+  EXPECT_NE(XUser, X.user_end());
+ 
+  XUser->replaceUsesOfWith(&X, &I0);
+  EXPECT_EQ(X.user_begin() ,X.user_end());
+  EXPECT_NE(I0.user_begin() ,I0.user_end());
+}
+
 TEST(UserTest, PersonalityUser) {
   LLVMContext Context;
   Module M("", Context);

From d3527052fc2a952d752f82ffbe39220bad33fc8d Mon Sep 17 00:00:00 2001
From: Chuanqi Xu <yedeng.yd@linux.alibaba.com>
Date: Wed, 29 Jul 2020 10:20:55 +0800
Subject: [PATCH 0416/1035] [NFC] Edit the comment for the return type of
 await_suspend

---
 clang/lib/Sema/SemaCoroutine.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/clang/lib/Sema/SemaCoroutine.cpp b/clang/lib/Sema/SemaCoroutine.cpp
index 992cccac64050..990ab26335209 100644
--- a/clang/lib/Sema/SemaCoroutine.cpp
+++ b/clang/lib/Sema/SemaCoroutine.cpp
@@ -449,7 +449,8 @@ static ReadySuspendResumeResult buildCoawaitCalls(Sema &S, VarDecl *CoroPromise,
   if (!AwaitSuspend->getType()->isDependentType()) {
     // [expr.await]p3 [...]
     //   - await-suspend is the expression e.await_suspend(h), which shall be
-    //     a prvalue of type void or bool.
+    //     a prvalue of type void, bool, or std::coroutine_handle<Z> for some
+    //     type Z.
     QualType RetType = AwaitSuspend->getCallReturnType(S.Context);
 
     // Experimental support for coroutine_handle returning await_suspend.

From d50d4c3d440c1d5a5a2003b782f8fbb7cc7b1441 Mon Sep 17 00:00:00 2001
From: Ikhlas Ajbar <iajbar@quicinc.com>
Date: Tue, 28 Jul 2020 16:35:34 -0500
Subject: [PATCH 0417/1035] [Hexagon] Correct the order of operands when
 lowering funnel shift-left

This patch corrects the order of operands in the pattern that lowers fshl
in Hexagon.
---
 llvm/lib/Target/Hexagon/HexagonPatterns.td |  4 +--
 llvm/test/CodeGen/Hexagon/funnel-shift.ll  |  8 +++---
 llvm/test/CodeGen/Hexagon/funnel-shift2.ll | 31 ++++++++++++++++++++++
 3 files changed, 37 insertions(+), 6 deletions(-)
 create mode 100644 llvm/test/CodeGen/Hexagon/funnel-shift2.ll

diff --git a/llvm/lib/Target/Hexagon/HexagonPatterns.td b/llvm/lib/Target/Hexagon/HexagonPatterns.td
index cc10627955fb0..c3422d595c793 100644
--- a/llvm/lib/Target/Hexagon/HexagonPatterns.td
+++ b/llvm/lib/Target/Hexagon/HexagonPatterns.td
@@ -1082,9 +1082,9 @@ def FShl32r: OutPatFrag<(ops node:$Rs, node:$Rt, node:$Ru),
   (HiReg (S2_asl_r_p (Combinew $Rs, $Rt), $Ru))>;
 
 def FShl64i: OutPatFrag<(ops node:$Rs, node:$Rt, node:$S),
-  (S2_lsr_i_p_or (S2_asl_i_p $Rt, $S),  $Rs, (Subi<64> $S))>;
+  (S2_lsr_i_p_or (S2_asl_i_p $Rs, $S),  $Rt, (Subi<64> $S))>;
 def FShl64r: OutPatFrag<(ops node:$Rs, node:$Rt, node:$Ru),
-  (S2_lsr_r_p_or (S2_asl_r_p $Rt, $Ru), $Rs, (A2_subri 64, $Ru))>;
+  (S2_lsr_r_p_or (S2_asl_r_p $Rs, $Ru), $Rt, (A2_subri 64, $Ru))>;
 
 // Combined SDNodeXForm: (Divu8 (Subi<64> $S))
 def Divu64_8: SDNodeXForm<imm, [{
diff --git a/llvm/test/CodeGen/Hexagon/funnel-shift.ll b/llvm/test/CodeGen/Hexagon/funnel-shift.ll
index b4b8f4726021f..6ad3b8a30519e 100644
--- a/llvm/test/CodeGen/Hexagon/funnel-shift.ll
+++ b/llvm/test/CodeGen/Hexagon/funnel-shift.ll
@@ -19,8 +19,8 @@ b0:
 }
 
 ; CHECK-LABEL: f2:
-; CHECK: r[[R20:[0-9]+]]:[[R21:[0-9]+]] = asl(r3:2,#17)
-; CHECK: r[[R20]]:[[R21]] |= lsr(r1:0,#47)
+; CHECK: r[[R20:[0-9]+]]:[[R21:[0-9]+]] = asl(r1:0,#17)
+; CHECK: r[[R20]]:[[R21]] |= lsr(r3:2,#47)
 define i64 @f2(i64 %a0, i64 %a1) #1 {
 b0:
   %v0 = tail call i64 @llvm.fshl.i64(i64 %a0, i64 %a1, i64 17)
@@ -28,9 +28,9 @@ b0:
 }
 
 ; CHECK-LABEL: f3:
-; CHECK: r[[R30:[0-9]+]]:[[R31:[0-9]+]] = asl(r3:2,r4)
+; CHECK: r[[R30:[0-9]+]]:[[R31:[0-9]+]] = asl(r1:0,r4)
 ; CHECK: r[[R32:[0-9]+]] = sub(#64,r4)
-; CHECK: r[[R30]]:[[R31]] |= lsr(r1:0,r[[R32]])
+; CHECK: r[[R30]]:[[R31]] |= lsr(r3:2,r[[R32]])
 define i64 @f3(i64 %a0, i64 %a1, i64 %a2) #1 {
 b0:
   %v0 = tail call i64 @llvm.fshl.i64(i64 %a0, i64 %a1, i64 %a2)
diff --git a/llvm/test/CodeGen/Hexagon/funnel-shift2.ll b/llvm/test/CodeGen/Hexagon/funnel-shift2.ll
new file mode 100644
index 0000000000000..b3032dbb7efd0
--- /dev/null
+++ b/llvm/test/CodeGen/Hexagon/funnel-shift2.ll
@@ -0,0 +1,31 @@
+; RUN: llc -mtriple=hexagon < %s | FileCheck %s
+
+; This patch corrects the order of operands in the pattern that lowers funnel
+; shift-left.
+
+; CHECK: r[[R17:[0-9]+]]:[[R16:[0-9]+]] = combine
+; CHECK: call parity
+; CHECK: r[[R1:[0-9]+]]:[[R0:[0-9]+]] = asl(r[[R1]]:[[R0]],#63)
+; CHECK: r[[R1]]:[[R0]] |= lsr(r[[R17]]:[[R16]],#1)
+
+target triple = "hexagon-unknown-unknown-elf"
+
+define dso_local i64 @fshl(i64 %x, i64 %y) {
+entry:
+  %x.addr = alloca i64, align 8
+  %y.addr = alloca i64, align 8
+  store i64 %x, i64* %x.addr, align 8
+  store i64 %y, i64* %y.addr, align 8
+  %0 = load i64, i64* %x.addr, align 8
+  %shr = lshr i64 %0, 1
+  %1 = load i64, i64* %x.addr, align 8
+  %2 = load i64, i64* %y.addr, align 8
+  %call = call i64 @parity(i64 %1, i64 %2)
+  %shl = shl i64 %call, 63
+  %or = or i64 %shr, %shl
+  store i64 %or, i64* %x.addr, align 8
+  %3 = load i64, i64* %x.addr, align 8
+  ret i64 %3
+}
+
+declare dso_local i64 @parity(i64, i64)

From 60c889cf714cd9488b1a54482b64948dd21a4d5f Mon Sep 17 00:00:00 2001
From: Azharuddin Mohammed <azhar@apple.com>
Date: Tue, 28 Jul 2020 15:52:54 -0700
Subject: [PATCH 0418/1035] [ThinLTO] [test] cache.ll: Prevent Spotlight
 indexing of the output dir

The test output files whose atime is altered in the test were getting
accessed by Spotlight indexing on macOS, causing them to get an updated
atime and leading to the test not behaving as expected.

Reviewed By: jhenderson, steven_wu

Differential Revision: https://reviews.llvm.org/D84700
---
 llvm/test/ThinLTO/X86/cache.ll | 179 +++++++++++++++++----------------
 1 file changed, 91 insertions(+), 88 deletions(-)

diff --git a/llvm/test/ThinLTO/X86/cache.ll b/llvm/test/ThinLTO/X86/cache.ll
index c9285f91b2557..406a1a456f89c 100644
--- a/llvm/test/ThinLTO/X86/cache.ll
+++ b/llvm/test/ThinLTO/X86/cache.ll
@@ -1,23 +1,26 @@
 ; NetBSD: noatime mounts currently inhibit 'touch -a' updates
 ; UNSUPPORTED: system-netbsd
 
+; The .noindex suffix for output dir is to prevent Spotlight on macOS from
+; indexing it.
+
 ; Verify first that *without* hash, we don't use the cache.
 ; RUN: opt -module-summary %s -o %t.bc
 ; RUN: opt -module-summary %p/Inputs/cache.ll -o %t2.bc
 
 ; Verify that enabling caching is ignoring module without hash
-; RUN: rm -Rf %t.cache && mkdir %t.cache
-; RUN: llvm-lto -thinlto-action=run -exported-symbol=globalfunc %t2.bc %t.bc -thinlto-cache-dir %t.cache
-; RUN: ls %t.cache/llvmcache.timestamp
-; RUN: ls %t.cache | count 1
+; RUN: rm -Rf %t.cache.noindex && mkdir %t.cache.noindex
+; RUN: llvm-lto -thinlto-action=run -exported-symbol=globalfunc %t2.bc %t.bc -thinlto-cache-dir %t.cache.noindex
+; RUN: ls %t.cache.noindex/llvmcache.timestamp
+; RUN: ls %t.cache.noindex | count 1
 
 ; Verify that enabling caching is ignoring module without hash with llvm-lto2
-; RUN: rm -Rf %t.cache
-; RUN: llvm-lto2 run -o %t.o %t2.bc  %t.bc -cache-dir %t.cache \
+; RUN: rm -Rf %t.cache.noindex
+; RUN: llvm-lto2 run -o %t.o %t2.bc  %t.bc -cache-dir %t.cache.noindex \
 ; RUN:  -r=%t2.bc,_main,plx \
 ; RUN:  -r=%t2.bc,_globalfunc,lx \
 ; RUN:  -r=%t.bc,_globalfunc,plx
-; RUN: ls %t.cache | count 0
+; RUN: ls %t.cache.noindex | count 0
 
 
 ; Repeat again, *with* hash this time.
@@ -27,138 +30,138 @@
 
 ; Verify that enabling caching is working, and that the pruner only removes
 ; files matching the pattern "llvmcache-*".
-; RUN: rm -Rf %t.cache && mkdir %t.cache
-; RUN: touch -t 197001011200 %t.cache/llvmcache-foo %t.cache/foo
-; RUN: llvm-lto -thinlto-action=run -exported-symbol=globalfunc %t2.bc %t.bc -thinlto-cache-dir %t.cache
-; RUN: ls %t.cache | count 4
-; RUN: ls %t.cache/llvmcache.timestamp
-; RUN: ls %t.cache/foo
-; RUN: not ls %t.cache/llvmcache-foo
-; RUN: ls %t.cache/llvmcache-* | count 2
+; RUN: rm -Rf %t.cache.noindex && mkdir %t.cache.noindex
+; RUN: touch -t 197001011200 %t.cache.noindex/llvmcache-foo %t.cache.noindex/foo
+; RUN: llvm-lto -thinlto-action=run -exported-symbol=globalfunc %t2.bc %t.bc -thinlto-cache-dir %t.cache.noindex
+; RUN: ls %t.cache.noindex | count 4
+; RUN: ls %t.cache.noindex/llvmcache.timestamp
+; RUN: ls %t.cache.noindex/foo
+; RUN: not ls %t.cache.noindex/llvmcache-foo
+; RUN: ls %t.cache.noindex/llvmcache-* | count 2
 
 ; Verify that enabling caching is working with llvm-lto2
-; RUN: rm -Rf %t.cache
-; RUN: llvm-lto2 run -o %t.o %t2.bc %t.bc -cache-dir %t.cache \
+; RUN: rm -Rf %t.cache.noindex
+; RUN: llvm-lto2 run -o %t.o %t2.bc %t.bc -cache-dir %t.cache.noindex \
 ; RUN:  -r=%t2.bc,_main,plx \
 ; RUN:  -r=%t2.bc,_globalfunc,lx \
 ; RUN:  -r=%t.bc,_globalfunc,plx
-; RUN: ls %t.cache | count 2
-; RUN: ls %t.cache/llvmcache-* | count 2
+; RUN: ls %t.cache.noindex | count 2
+; RUN: ls %t.cache.noindex/llvmcache-* | count 2
 
 ; Verify that caches with a timestamp older than the pruning interval
 ; will be pruned
-; RUN: rm -Rf %t.cache && mkdir %t.cache
-; RUN: touch -t 197001011200 %t.cache/llvmcache-foo
-; RUN: touch -t 197001011200 %t.cache/llvmcache.timestamp
-; RUN: llvm-lto -thinlto-action=run -exported-symbol=globalfunc %t2.bc %t.bc -thinlto-cache-dir %t.cache
-; RUN: not ls %t.cache/llvmcache-foo
+; RUN: rm -Rf %t.cache.noindex && mkdir %t.cache.noindex
+; RUN: touch -t 197001011200 %t.cache.noindex/llvmcache-foo
+; RUN: touch -t 197001011200 %t.cache.noindex/llvmcache.timestamp
+; RUN: llvm-lto -thinlto-action=run -exported-symbol=globalfunc %t2.bc %t.bc -thinlto-cache-dir %t.cache.noindex
+; RUN: not ls %t.cache.noindex/llvmcache-foo
 
 ; Verify that specifying a negative number for the pruning interval
 ; effectively disables the pruning
-; RUN: rm -Rf %t.cache && mkdir %t.cache
-; RUN: touch -t 197001011200 %t.cache/llvmcache-foo
-; RUN: touch -t 197001011200 %t.cache/llvmcache.timestamp
-; RUN: llvm-lto -thinlto-action=run -exported-symbol=globalfunc %t2.bc %t.bc -thinlto-cache-dir %t.cache --thinlto-cache-pruning-interval -1
-; RUN: ls %t.cache/llvmcache-foo
+; RUN: rm -Rf %t.cache.noindex && mkdir %t.cache.noindex
+; RUN: touch -t 197001011200 %t.cache.noindex/llvmcache-foo
+; RUN: touch -t 197001011200 %t.cache.noindex/llvmcache.timestamp
+; RUN: llvm-lto -thinlto-action=run -exported-symbol=globalfunc %t2.bc %t.bc -thinlto-cache-dir %t.cache.noindex --thinlto-cache-pruning-interval -1
+; RUN: ls %t.cache.noindex/llvmcache-foo
 
 ; Verify that the pruner doesn't run and a cache file is not deleted when: 
 ; default values for pruning interval and cache expiration are used, 
 ; llvmcache.timestamp is current, 
 ; cache file is older than default cache expiration value.
-; RUN: rm -Rf %t.cache && mkdir %t.cache
-; RUN: touch -t 197001011200 %t.cache/llvmcache-foo
-; RUN: touch %t.cache/llvmcache.timestamp
-; RUN: llvm-lto -thinlto-action=run -exported-symbol=globalfunc %t2.bc %t.bc -thinlto-cache-dir %t.cache
-; RUN: ls %t.cache/llvmcache-foo
+; RUN: rm -Rf %t.cache.noindex && mkdir %t.cache.noindex
+; RUN: touch -t 197001011200 %t.cache.noindex/llvmcache-foo
+; RUN: touch %t.cache.noindex/llvmcache.timestamp
+; RUN: llvm-lto -thinlto-action=run -exported-symbol=globalfunc %t2.bc %t.bc -thinlto-cache-dir %t.cache.noindex
+; RUN: ls %t.cache.noindex/llvmcache-foo
 
 ; Verify that the pruner runs and a cache file is deleted when:
 ; pruning interval has value 0 (i.e. run garbage collector now)
 ; default value for cache expiration is used,
 ; llvmcache.timestamp is current,
 ; cache file is older than default cache expiration value.
-; RUN: rm -Rf %t.cache && mkdir %t.cache
-; RUN: touch -t 197001011200 %t.cache/llvmcache-foo
-; RUN: touch %t.cache/llvmcache.timestamp
-; RUN: llvm-lto -thinlto-action=run -exported-symbol=globalfunc %t2.bc %t.bc -thinlto-cache-dir %t.cache --thinlto-cache-pruning-interval 0
-; RUN: not ls %t.cache/llvmcache-foo
+; RUN: rm -Rf %t.cache.noindex && mkdir %t.cache.noindex
+; RUN: touch -t 197001011200 %t.cache.noindex/llvmcache-foo
+; RUN: touch %t.cache.noindex/llvmcache.timestamp
+; RUN: llvm-lto -thinlto-action=run -exported-symbol=globalfunc %t2.bc %t.bc -thinlto-cache-dir %t.cache.noindex --thinlto-cache-pruning-interval 0
+; RUN: not ls %t.cache.noindex/llvmcache-foo
 
 ; Populate the cache with files with "old" access times, then check llvm-lto updates these file times
 ; A negative pruning interval is used to avoid removing cache entries
-; RUN: rm -Rf %t.cache && mkdir %t.cache
-; RUN: llvm-lto -thinlto-action=run -exported-symbol=globalfunc %t2.bc %t.bc -thinlto-cache-dir %t.cache
-; RUN: touch -a -t 197001011200 %t.cache/llvmcache-*
-; RUN: llvm-lto -thinlto-action=run -exported-symbol=globalfunc %t2.bc %t.bc -thinlto-cache-dir %t.cache --thinlto-cache-pruning-interval -1
-; RUN: ls -ltu %t.cache/* | not grep 1970-01-01
+; RUN: rm -Rf %t.cache.noindex && mkdir %t.cache.noindex
+; RUN: llvm-lto -thinlto-action=run -exported-symbol=globalfunc %t2.bc %t.bc -thinlto-cache-dir %t.cache.noindex
+; RUN: touch -a -t 197001011200 %t.cache.noindex/llvmcache-*
+; RUN: llvm-lto -thinlto-action=run -exported-symbol=globalfunc %t2.bc %t.bc -thinlto-cache-dir %t.cache.noindex --thinlto-cache-pruning-interval -1
+; RUN: ls -ltu %t.cache.noindex/* | not grep 1970-01-01
 
 ; Populate the cache with files with "old" access times, then check llvm-lto2 updates these file times
-; RUN: rm -Rf %t.cache
-; RUN: llvm-lto2 run -o %t.o %t2.bc %t.bc -cache-dir %t.cache \
+; RUN: rm -Rf %t.cache.noindex
+; RUN: llvm-lto2 run -o %t.o %t2.bc %t.bc -cache-dir %t.cache.noindex \
 ; RUN:  -r=%t2.bc,_main,plx \
 ; RUN:  -r=%t2.bc,_globalfunc,lx \
 ; RUN:  -r=%t.bc,_globalfunc,plx
-; RUN: touch -a -t 197001011200 %t.cache/llvmcache-*
-; RUN: llvm-lto2 run -o %t.o %t2.bc %t.bc -cache-dir %t.cache \
+; RUN: touch -a -t 197001011200 %t.cache.noindex/llvmcache-*
+; RUN: llvm-lto2 run -o %t.o %t2.bc %t.bc -cache-dir %t.cache.noindex \
 ; RUN:  -r=%t2.bc,_main,plx \
 ; RUN:  -r=%t2.bc,_globalfunc,lx \
 ; RUN:  -r=%t.bc,_globalfunc,plx
-; RUN: ls -ltu %t.cache/* | not grep 1970-01-01
+; RUN: ls -ltu %t.cache.noindex/* | not grep 1970-01-01
 
 ; Verify that specifying max size for the cache directory prunes it to this
 ; size, removing the oldest files first.
-; RUN: rm -Rf %t.cache && mkdir %t.cache
+; RUN: rm -Rf %t.cache.noindex && mkdir %t.cache.noindex
 ; Create cache files with different sizes.
 ; Only 8B and 76B files should stay after pruning.
-; RUN: %python -c "with open(r'%t.cache/llvmcache-foo-100k', 'w') as file: file.truncate(102400)"
-; RUN: touch -t 198002011200 %t.cache/llvmcache-foo-100k
-; RUN: %python -c "with open(r'%t.cache/llvmcache-foo-16', 'w') as file: file.truncate(16)"
-; RUN: touch -t 198002021200 %t.cache/llvmcache-foo-16
-; RUN: %python -c "with open(r'%t.cache/llvmcache-foo-77k', 'w') as file: file.truncate(78848)"
-; RUN: touch -t 198002031200 %t.cache/llvmcache-foo-77k
-; RUN: %python -c "with open(r'%t.cache/llvmcache-foo-8', 'w') as file: file.truncate(8)"
-; RUN: touch -t 198002041200 %t.cache/llvmcache-foo-8
-; RUN: %python -c "with open(r'%t.cache/llvmcache-foo-76', 'w') as file: file.truncate(76)"
-; RUN: touch -t 198002051200 %t.cache/llvmcache-foo-76
-; RUN: llvm-lto -thinlto-action=run -exported-symbol=globalfunc %t2.bc %t.bc -thinlto-cache-dir %t.cache --thinlto-cache-max-size-bytes 78847 --thinlto-cache-entry-expiration 4294967295
-; RUN: ls %t.cache/llvmcache-foo-8
-; RUN: ls %t.cache/llvmcache-foo-76
-; RUN: not ls %t.cache/llvmcache-foo-16
-; RUN: not ls %t.cache/llvmcache-foo-100k
-; RUN: not ls %t.cache/llvmcache-foo-77k
+; RUN: %python -c "with open(r'%t.cache.noindex/llvmcache-foo-100k', 'w') as file: file.truncate(102400)"
+; RUN: touch -t 198002011200 %t.cache.noindex/llvmcache-foo-100k
+; RUN: %python -c "with open(r'%t.cache.noindex/llvmcache-foo-16', 'w') as file: file.truncate(16)"
+; RUN: touch -t 198002021200 %t.cache.noindex/llvmcache-foo-16
+; RUN: %python -c "with open(r'%t.cache.noindex/llvmcache-foo-77k', 'w') as file: file.truncate(78848)"
+; RUN: touch -t 198002031200 %t.cache.noindex/llvmcache-foo-77k
+; RUN: %python -c "with open(r'%t.cache.noindex/llvmcache-foo-8', 'w') as file: file.truncate(8)"
+; RUN: touch -t 198002041200 %t.cache.noindex/llvmcache-foo-8
+; RUN: %python -c "with open(r'%t.cache.noindex/llvmcache-foo-76', 'w') as file: file.truncate(76)"
+; RUN: touch -t 198002051200 %t.cache.noindex/llvmcache-foo-76
+; RUN: llvm-lto -thinlto-action=run -exported-symbol=globalfunc %t2.bc %t.bc -thinlto-cache-dir %t.cache.noindex --thinlto-cache-max-size-bytes 78847 --thinlto-cache-entry-expiration 4294967295
+; RUN: ls %t.cache.noindex/llvmcache-foo-8
+; RUN: ls %t.cache.noindex/llvmcache-foo-76
+; RUN: not ls %t.cache.noindex/llvmcache-foo-16
+; RUN: not ls %t.cache.noindex/llvmcache-foo-100k
+; RUN: not ls %t.cache.noindex/llvmcache-foo-77k
 
 ; Verify that specifying a max size > 4GB for the cache directory does not
 ; prematurely prune, due to an integer overflow.
-; RUN: rm -Rf %t.cache && mkdir %t.cache
-; RUN: %python -c "with open(r'%t.cache/llvmcache-foo-10', 'w') as file: file.truncate(10)"
-; RUN: llvm-lto -thinlto-action=run -exported-symbol=globalfunc %t2.bc %t.bc -thinlto-cache-dir %t.cache --thinlto-cache-max-size-bytes 4294967297
-; RUN: ls %t.cache/llvmcache-foo-10
+; RUN: rm -Rf %t.cache.noindex && mkdir %t.cache.noindex
+; RUN: %python -c "with open(r'%t.cache.noindex/llvmcache-foo-10', 'w') as file: file.truncate(10)"
+; RUN: llvm-lto -thinlto-action=run -exported-symbol=globalfunc %t2.bc %t.bc -thinlto-cache-dir %t.cache.noindex --thinlto-cache-max-size-bytes 4294967297
+; RUN: ls %t.cache.noindex/llvmcache-foo-10
 
 ; Verify that negative numbers aren't accepted for the
 ; --thinlto-cache-max-size-bytes switch
-; RUN: rm -Rf %t.cache && mkdir %t.cache
+; RUN: rm -Rf %t.cache.noindex && mkdir %t.cache.noindex
 ; RUN: not llvm-lto %t.bc --thinlto-cache-max-size-bytes -1 2>&1 | FileCheck %s
 ; CHECK: -thinlto-cache-max-size-bytes option: '-1' value invalid
 
 ; Verify that specifying max number of files in the cache directory prunes
 ; it to this amount, removing the oldest files first.
-; RUN: rm -Rf %t.cache && mkdir %t.cache
+; RUN: rm -Rf %t.cache.noindex && mkdir %t.cache.noindex
 ; Create cache files with different sizes.
 ; Only 75B and 76B files should stay after pruning.
-; RUN: %python -c "print(' ' * 1023)" > %t.cache/llvmcache-foo-1023
-; RUN: touch -t 198002011200 %t.cache/llvmcache-foo-1023
-; RUN: %python -c "print(' ' * 15)" > %t.cache/llvmcache-foo-15
-; RUN: touch -t 198002021200 %t.cache/llvmcache-foo-15
-; RUN: %python -c "print(' ' * 7)" > %t.cache/llvmcache-foo-7
-; RUN: touch -t 198002031200 %t.cache/llvmcache-foo-7
-; RUN: %python -c "print(' ' * 75)" > %t.cache/llvmcache-foo-75
-; RUN: touch -t 198002041200 %t.cache/llvmcache-foo-75
-; RUN: %python -c "print(' ' * 76)" > %t.cache/llvmcache-foo-76
-; RUN: touch -t 198002051200 %t.cache/llvmcache-foo-76
-; RUN: llvm-lto -thinlto-action=run -exported-symbol=globalfunc %t2.bc %t.bc -thinlto-cache-dir %t.cache --thinlto-cache-max-size-files 4 --thinlto-cache-entry-expiration 4294967295
-; RUN: ls %t.cache/llvmcache-foo-75
-; RUN: ls %t.cache/llvmcache-foo-76
-; RUN: not ls %t.cache/llvmcache-foo-15
-; RUN: not ls %t.cache/llvmcache-foo-1024
-; RUN: not ls %t.cache/llvmcache-foo-7
+; RUN: %python -c "print(' ' * 1023)" > %t.cache.noindex/llvmcache-foo-1023
+; RUN: touch -t 198002011200 %t.cache.noindex/llvmcache-foo-1023
+; RUN: %python -c "print(' ' * 15)" > %t.cache.noindex/llvmcache-foo-15
+; RUN: touch -t 198002021200 %t.cache.noindex/llvmcache-foo-15
+; RUN: %python -c "print(' ' * 7)" > %t.cache.noindex/llvmcache-foo-7
+; RUN: touch -t 198002031200 %t.cache.noindex/llvmcache-foo-7
+; RUN: %python -c "print(' ' * 75)" > %t.cache.noindex/llvmcache-foo-75
+; RUN: touch -t 198002041200 %t.cache.noindex/llvmcache-foo-75
+; RUN: %python -c "print(' ' * 76)" > %t.cache.noindex/llvmcache-foo-76
+; RUN: touch -t 198002051200 %t.cache.noindex/llvmcache-foo-76
+; RUN: llvm-lto -thinlto-action=run -exported-symbol=globalfunc %t2.bc %t.bc -thinlto-cache-dir %t.cache.noindex --thinlto-cache-max-size-files 4 --thinlto-cache-entry-expiration 4294967295
+; RUN: ls %t.cache.noindex/llvmcache-foo-75
+; RUN: ls %t.cache.noindex/llvmcache-foo-76
+; RUN: not ls %t.cache.noindex/llvmcache-foo-15
+; RUN: not ls %t.cache.noindex/llvmcache-foo-1024
+; RUN: not ls %t.cache.noindex/llvmcache-foo-7
 
 target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.11.0"

From 8dbe9b249846be81bc676543698db14ee37e28e2 Mon Sep 17 00:00:00 2001
From: Anh Tuyen Tran <anhtuyen@ca.ibm.com>
Date: Wed, 29 Jul 2020 05:29:45 +0000
Subject: [PATCH 0419/1035] [Clang-tools-extra] Mark override a function which
 overrides a virtual one

Function void run() on line 286 overrides a virtual function on line 92 of
clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp.  Not marking it override will
cause a build failure when we use -Werror (every warning is treated as an error).

Reviewed By: kbobyrev (Kirill Bobyrev)

Differential Revision: https://reviews.llvm.org/D84794
---
 clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp b/clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp
index 80d87aa3f9f51..ca35f620bba18 100644
--- a/clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp
+++ b/clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp
@@ -283,7 +283,7 @@ class Export : public Command {
   };
 
 public:
-  void run() {
+  void run() override {
     using namespace clang::clangd;
     // Read input file (as specified in global option)
     auto Buffer = llvm::MemoryBuffer::getFile(IndexLocation);

From b1e398920f7372db3ebfeeb219d4538d235c7efb Mon Sep 17 00:00:00 2001
From: George Mitenkov <georgemitenk0v@gmail.com>
Date: Wed, 29 Jul 2020 08:13:11 +0300
Subject: [PATCH 0420/1035] [MLIR][SPIRVToLLVM] Support of volatile/nontemporal
 memory access in load/store

This patch adds support of Volatile and Nontemporal
memory accesses to `spv.Load` and `spv.Store`. These attributes are
modelled with a `volatile` and `nontemporal` flags.

Reviewed By: antiagainst

Differential Revision: https://reviews.llvm.org/D84739
---
 .../SPIRVToLLVM/ConvertSPIRVToLLVM.cpp        | 42 ++++++++++++-------
 .../SPIRVToLLVM/memory-ops-to-llvm.mlir       | 28 +++++++++++++
 2 files changed, 56 insertions(+), 14 deletions(-)

diff --git a/mlir/lib/Conversion/SPIRVToLLVM/ConvertSPIRVToLLVM.cpp b/mlir/lib/Conversion/SPIRVToLLVM/ConvertSPIRVToLLVM.cpp
index 12aa6573ef379..161fda0fc3535 100644
--- a/mlir/lib/Conversion/SPIRVToLLVM/ConvertSPIRVToLLVM.cpp
+++ b/mlir/lib/Conversion/SPIRVToLLVM/ConvertSPIRVToLLVM.cpp
@@ -189,18 +189,20 @@ static Value createI32ConstantOf(Location loc, PatternRewriter &rewriter,
 static LogicalResult replaceWithLoadOrStore(Operation *op,
                                             ConversionPatternRewriter &rewriter,
                                             LLVMTypeConverter &typeConverter,
-                                            unsigned alignment) {
+                                            unsigned alignment, bool isVolatile,
+                                            bool isNonTemporal) {
   if (auto loadOp = dyn_cast<spirv::LoadOp>(op)) {
     auto dstType = typeConverter.convertType(loadOp.getType());
     if (!dstType)
       return failure();
-    rewriter.replaceOpWithNewOp<LLVM::LoadOp>(loadOp, dstType, loadOp.ptr(),
-                                              alignment);
+    rewriter.replaceOpWithNewOp<LLVM::LoadOp>(
+        loadOp, dstType, loadOp.ptr(), alignment, isVolatile, isNonTemporal);
     return success();
   }
   auto storeOp = cast<spirv::StoreOp>(op);
   rewriter.replaceOpWithNewOp<LLVM::StoreOp>(storeOp, storeOp.value(),
-                                             storeOp.ptr(), alignment);
+                                             storeOp.ptr(), alignment,
+                                             isVolatile, isNonTemporal);
   return success();
 }
 
@@ -594,19 +596,31 @@ class LoadStorePattern : public SPIRVToLLVMConversion<SPIRVop> {
   LogicalResult
   matchAndRewrite(SPIRVop op, ArrayRef<Value> operands,
                   ConversionPatternRewriter &rewriter) const override {
-    if (op.memory_access().hasValue() &&
-        op.memory_access().getValue() != spirv::MemoryAccess::None) {
-      auto memoryAccess = op.memory_access().getValue();
-      if (memoryAccess == spirv::MemoryAccess::Aligned) {
-        unsigned alignment = op.alignment().getValue().getZExtValue();
-        replaceWithLoadOrStore(op, rewriter, this->typeConverter, alignment);
-        return success();
-      }
+
+    if (!op.memory_access().hasValue()) {
+      replaceWithLoadOrStore(op, rewriter, this->typeConverter, /*alignment=*/0,
+                             /*isVolatile=*/false, /*isNonTemporal=*/ false);
+      return success();
+    }
+    auto memoryAccess = op.memory_access().getValue();
+    switch (memoryAccess) {
+    case spirv::MemoryAccess::Aligned:
+    case spirv::MemoryAccess::None:
+    case spirv::MemoryAccess::Nontemporal:
+    case spirv::MemoryAccess::Volatile: {
+      unsigned alignment = memoryAccess == spirv::MemoryAccess::Aligned
+                               ? op.alignment().getValue().getZExtValue()
+                               : 0;
+      bool isNonTemporal = memoryAccess == spirv::MemoryAccess::Nontemporal;
+      bool isVolatile = memoryAccess == spirv::MemoryAccess::Volatile;
+      replaceWithLoadOrStore(op, rewriter, this->typeConverter, alignment,
+                             isVolatile, isNonTemporal);
+      return success();
+    }
+    default:
       // There is no support of other memory access attributes.
       return failure();
     }
-    replaceWithLoadOrStore(op, rewriter, this->typeConverter, 0);
-    return success();
   }
 };
 
diff --git a/mlir/test/Conversion/SPIRVToLLVM/memory-ops-to-llvm.mlir b/mlir/test/Conversion/SPIRVToLLVM/memory-ops-to-llvm.mlir
index 362c702e8d3c4..f717948f8069d 100644
--- a/mlir/test/Conversion/SPIRVToLLVM/memory-ops-to-llvm.mlir
+++ b/mlir/test/Conversion/SPIRVToLLVM/memory-ops-to-llvm.mlir
@@ -25,6 +25,20 @@ func @load_with_alignment() {
   return
 }
 
+func @load_volatile() {
+  %0 = spv.Variable : !spv.ptr<f32, Function>
+  // CHECK: %{{.*}} = llvm.load volatile %{{.*}} : !llvm<"float*">
+  %1 = spv.Load "Function" %0 ["Volatile"] : f32
+  return
+}
+
+func @load_nontemporal() {
+  %0 = spv.Variable : !spv.ptr<f32, Function>
+  // CHECK: %{{.*}} = llvm.load %{{.*}} {nontemporal} : !llvm<"float*">
+  %1 = spv.Load "Function" %0 ["Nontemporal"] : f32
+  return
+}
+
 //===----------------------------------------------------------------------===//
 // spv.Store
 //===----------------------------------------------------------------------===//
@@ -50,6 +64,20 @@ func @store_with_alignment(%arg0 : f32) -> () {
   return
 }
 
+func @store_volatile(%arg0 : f32) -> () {
+  %0 = spv.Variable : !spv.ptr<f32, Function>
+  // CHECK: llvm.store volatile %{{.*}}, %{{.*}} : !llvm<"float*">
+  spv.Store "Function" %0, %arg0 ["Volatile"] : f32
+  return
+}
+
+func @store_nontemporal(%arg0 : f32) -> () {
+  %0 = spv.Variable : !spv.ptr<f32, Function>
+  // CHECK: llvm.store %{{.*}}, %{{.*}} {nontemporal} : !llvm<"float*">
+  spv.Store "Function" %0, %arg0 ["Nontemporal"] : f32
+  return
+}
+
 //===----------------------------------------------------------------------===//
 // spv.Variable
 //===----------------------------------------------------------------------===//

From 8a66bb7a75f71a749e9ee603c9c5c8d016ed7238 Mon Sep 17 00:00:00 2001
From: George Mitenkov <georgemitenk0v@gmail.com>
Date: Wed, 29 Jul 2020 08:47:22 +0300
Subject: [PATCH 0421/1035] [MLIR][SPIRV] Added storage class constraint on
 global variable

Added a check for 'Function' storage class in `spv.globalVariable`
verifier since it only can be used with `spv.Variable`.

Reviewed By: antiagainst

Differential Revision: https://reviews.llvm.org/D84731
---
 mlir/lib/Dialect/SPIRV/SPIRVOps.cpp        | 9 +++++++--
 mlir/test/Dialect/SPIRV/structure-ops.mlir | 7 +++++++
 2 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/mlir/lib/Dialect/SPIRV/SPIRVOps.cpp b/mlir/lib/Dialect/SPIRV/SPIRVOps.cpp
index 9d0570257d422..b0235d419ebe4 100644
--- a/mlir/lib/Dialect/SPIRV/SPIRVOps.cpp
+++ b/mlir/lib/Dialect/SPIRV/SPIRVOps.cpp
@@ -1955,8 +1955,13 @@ static LogicalResult verify(spirv::GlobalVariableOp varOp) {
   // SPIR-V spec: "Storage Class is the Storage Class of the memory holding the
   // object. It cannot be Generic. It must be the same as the Storage Class
   // operand of the Result Type."
-  if (varOp.storageClass() == spirv::StorageClass::Generic)
-    return varOp.emitOpError("storage class cannot be 'Generic'");
+  // Also, Function storage class is reserved by spv.Variable.
+  auto storageClass = varOp.storageClass();
+  if (storageClass == spirv::StorageClass::Generic ||
+      storageClass == spirv::StorageClass::Function) {
+    return varOp.emitOpError("storage class cannot be '")
+           << stringifyStorageClass(storageClass) << "'";
+  }
 
   if (auto init =
           varOp.getAttrOfType<FlatSymbolRefAttr>(kInitializerAttrName)) {
diff --git a/mlir/test/Dialect/SPIRV/structure-ops.mlir b/mlir/test/Dialect/SPIRV/structure-ops.mlir
index 2d62f64b2479c..e20da2e4e6c97 100644
--- a/mlir/test/Dialect/SPIRV/structure-ops.mlir
+++ b/mlir/test/Dialect/SPIRV/structure-ops.mlir
@@ -347,6 +347,13 @@ spv.module Logical GLSL450 {
 
 // -----
 
+spv.module Logical GLSL450 {
+  // expected-error @+1 {{storage class cannot be 'Function'}}
+  spv.globalVariable @var0 : !spv.ptr<f32, Function>
+}
+
+// -----
+
 spv.module Logical GLSL450 {
   spv.func @foo() "None" {
     // expected-error @+1 {{op must appear in a module-like op's block}}

From 2ead4fca79d087e2d658a3101e3946a54c1d4685 Mon Sep 17 00:00:00 2001
From: Stephan Bergmann <sbergman@redhat.com>
Date: Wed, 29 Jul 2020 08:18:28 +0200
Subject: [PATCH 0422/1035] Test including rpc/xdr.h requires sunrpc

...which is set based on HAVE_RPC_XDR_H.  At least Fedora 32 does not have a
/usr/include/rpc/xdr.h, so failed this test introduced with
<https://reviews.llvm.org/D83358> "[Sanitizers] Add interceptor for
xdrrec_create".

Differential Revision: https://reviews.llvm.org/D84740
---
 compiler-rt/test/sanitizer_common/TestCases/Linux/xdrrec.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/compiler-rt/test/sanitizer_common/TestCases/Linux/xdrrec.cpp b/compiler-rt/test/sanitizer_common/TestCases/Linux/xdrrec.cpp
index f39bed3da3763..cc50a650f751b 100644
--- a/compiler-rt/test/sanitizer_common/TestCases/Linux/xdrrec.cpp
+++ b/compiler-rt/test/sanitizer_common/TestCases/Linux/xdrrec.cpp
@@ -1,5 +1,5 @@
 // RUN: %clangxx -O0 %s -o %t && %run %t | FileCheck %s
-// REQUIRES: !android
+// REQUIRES: sunrpc, !android
 #include <cassert>
 #include <rpc/xdr.h>
 

From 1ae766e3e06a233ce9d60a5734f29e78ccd80639 Mon Sep 17 00:00:00 2001
From: Juneyoung Lee <aqjune@gmail.com>
Date: Wed, 29 Jul 2020 15:26:55 +0900
Subject: [PATCH 0423/1035] [InstCombine] Add tests for select(freeze(undef));
 NFC

---
 llvm/test/Transforms/InstCombine/select.ll | 39 ++++++++++++++++++++++
 1 file changed, 39 insertions(+)

diff --git a/llvm/test/Transforms/InstCombine/select.ll b/llvm/test/Transforms/InstCombine/select.ll
index 93281ce1f799d..c2cacc0b14ee2 100644
--- a/llvm/test/Transforms/InstCombine/select.ll
+++ b/llvm/test/Transforms/InstCombine/select.ll
@@ -2487,3 +2487,42 @@ define <2 x i32> @true_undef_vec(i1 %cond, <2 x i32> %x) {
   %s = select i1 %cond, <2 x i32> undef, <2 x i32> %x
   ret <2 x i32> %s
 }
+
+define i8 @cond_freeze(i8 %x, i8 %y) {
+; CHECK-LABEL: @cond_freeze(
+; CHECK-NEXT:    [[COND_FR:%.*]] = freeze i1 undef
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[COND_FR]], i8 [[X:%.*]], i8 [[Y:%.*]]
+; CHECK-NEXT:    ret i8 [[S]]
+;
+  %cond.fr = freeze i1 undef
+  %s = select i1 %cond.fr, i8 %x, i8 %y
+  ret i8 %s
+}
+
+define i8 @cond_freeze2(i8 %x, i8 %y) {
+; CHECK-LABEL: @cond_freeze2(
+; CHECK-NEXT:    [[COND_FR:%.*]] = freeze i1 undef
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[COND_FR]], i8 [[X:%.*]], i8 1
+; CHECK-NEXT:    ret i8 [[S]]
+;
+  %cond.fr = freeze i1 undef
+  %s = select i1 %cond.fr, i8 %x, i8 1
+  ret i8 %s
+}
+
+declare void @foo2(i8, i8)
+
+define void @cond_freeze_multipleuses(i8 %x, i8 %y) {
+; CHECK-LABEL: @cond_freeze_multipleuses(
+; CHECK-NEXT:    [[COND_FR:%.*]] = freeze i1 undef
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[COND_FR]], i8 [[X:%.*]], i8 [[Y:%.*]]
+; CHECK-NEXT:    [[S2:%.*]] = select i1 [[COND_FR]], i8 [[Y]], i8 [[X]]
+; CHECK-NEXT:    call void @foo2(i8 [[S]], i8 [[S2]])
+; CHECK-NEXT:    ret void
+;
+  %cond.fr = freeze i1 undef
+  %s = select i1 %cond.fr, i8 %x, i8 %y
+  %s2 = select i1 %cond.fr, i8 %y, i8 %x
+  call void @foo2(i8 %s, i8 %s2)
+  ret void
+}

From 89247792c5bdd58500b0e6c5e56532424c2e2ca1 Mon Sep 17 00:00:00 2001
From: Nathan Ridge <zeratul976@hotmail.com>
Date: Sun, 26 Jul 2020 22:43:29 -0400
Subject: [PATCH 0424/1035] [clang] Fix ConceptSpecializationExpr::getEndLoc()

Summary:
It returned an invalid location in case of a constrained-parameter
with no explicit arguments.

Reviewers: hokein

Subscribers: cfe-commits

Tags: #clang

Differential Revision: https://reviews.llvm.org/D84613
---
 clang/include/clang/AST/ExprConcepts.h |  6 +++++-
 clang/test/AST/ast-dump-concepts.cpp   | 10 +++++++++-
 2 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/clang/include/clang/AST/ExprConcepts.h b/clang/include/clang/AST/ExprConcepts.h
index 2a88ed5175d24..1544c498ef662 100644
--- a/clang/include/clang/AST/ExprConcepts.h
+++ b/clang/include/clang/AST/ExprConcepts.h
@@ -126,7 +126,11 @@ class ConceptSpecializationExpr final : public Expr, public ConceptReference,
   }
 
   SourceLocation getEndLoc() const LLVM_READONLY {
-    return ArgsAsWritten->RAngleLoc;
+    // If the ConceptSpecializationExpr is the ImmediatelyDeclaredConstraint
+    // of a TypeConstraint written syntactically as a constrained-parameter,
+    // there may not be a template argument list.
+    return ArgsAsWritten->RAngleLoc.isValid() ? ArgsAsWritten->RAngleLoc
+                                              : ConceptName.getEndLoc();
   }
 
   // Iterators
diff --git a/clang/test/AST/ast-dump-concepts.cpp b/clang/test/AST/ast-dump-concepts.cpp
index 530c1baeffa77..3429fa6b46be5 100644
--- a/clang/test/AST/ast-dump-concepts.cpp
+++ b/clang/test/AST/ast-dump-concepts.cpp
@@ -6,14 +6,22 @@
 // RUN: -ast-dump-all -ast-dump-filter Foo /dev/null \
 // RUN: | FileCheck --strict-whitespace %s
 
+template <typename T>
+concept unary_concept = true;
+
 template <typename T, typename U>
 concept binary_concept = true;
 
 template <typename T>
 struct Foo {
   // CHECK:      TemplateTypeParmDecl {{.*}} referenced Concept {{.*}} 'binary_concept'
-  // CHECK-NEXT: |-ConceptSpecializationExpr {{.*}} 'bool'
+  // CHECK-NEXT: |-ConceptSpecializationExpr {{.*}} <col:13, col:31> 'bool'
   // CHECK-NEXT: `-TemplateArgument {{.*}} type 'int'
   template <binary_concept<int> R>
   Foo(R);
+
+  // CHECK:      TemplateTypeParmDecl {{.*}} referenced Concept {{.*}} 'unary_concept'
+  // CHECK-NEXT: `-ConceptSpecializationExpr {{.*}} <col:13> 'bool'
+  template <unary_concept R>
+  Foo(R);
 };

From 1f4aa30a4f8a3fb869baa741662b9f6b3c73a0e3 Mon Sep 17 00:00:00 2001
From: George Mitenkov <georgemitenk0v@gmail.com>
Date: Wed, 29 Jul 2020 09:16:47 +0300
Subject: [PATCH 0425/1035] [MLIR][SPIRVToLLVM] Branch weights support for
 BranchConditional conversion

Conversion of `spv.BranchConditional` now supports branch weights
that are mapped to weights vector in `llvm.cond_br`.

Reviewed By: antiagainst

Differential Revision: https://reviews.llvm.org/D84657
---
 .../Conversion/SPIRVToLLVM/ConvertSPIRVToLLVM.cpp | 15 ++++++++++-----
 .../SPIRVToLLVM/control-flow-ops-to-llvm.mlir     |  8 +++-----
 2 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/mlir/lib/Conversion/SPIRVToLLVM/ConvertSPIRVToLLVM.cpp b/mlir/lib/Conversion/SPIRVToLLVM/ConvertSPIRVToLLVM.cpp
index 161fda0fc3535..25a3ac07d5f4a 100644
--- a/mlir/lib/Conversion/SPIRVToLLVM/ConvertSPIRVToLLVM.cpp
+++ b/mlir/lib/Conversion/SPIRVToLLVM/ConvertSPIRVToLLVM.cpp
@@ -459,13 +459,18 @@ class BranchConditionalConversionPattern
   LogicalResult
   matchAndRewrite(spirv::BranchConditionalOp op, ArrayRef<Value> operands,
                   ConversionPatternRewriter &rewriter) const override {
-    // There is no support of branch weights in LLVM dialect at the moment.
-    if (auto weights = op.branch_weights())
-      return failure();
+    // If branch weights exist, map them to 32-bit integer vector.
+    ElementsAttr branchWeights = nullptr;
+    if (auto weights = op.branch_weights()) {
+      VectorType weightType = VectorType::get(2, rewriter.getI32Type());
+      branchWeights =
+          DenseElementsAttr::get(weightType, weights.getValue().getValue());
+    }
 
     rewriter.replaceOpWithNewOp<LLVM::CondBrOp>(
-        op, op.condition(), op.getTrueBlock(), op.getTrueBlockArguments(),
-        op.getFalseBlock(), op.getFalseBlockArguments());
+        op, op.condition(), op.getTrueBlockArguments(),
+        op.getFalseBlockArguments(), branchWeights, op.getTrueBlock(),
+        op.getFalseBlock());
     return success();
   }
 };
diff --git a/mlir/test/Conversion/SPIRVToLLVM/control-flow-ops-to-llvm.mlir b/mlir/test/Conversion/SPIRVToLLVM/control-flow-ops-to-llvm.mlir
index 0a2f6d681769d..3c92040a17ed7 100644
--- a/mlir/test/Conversion/SPIRVToLLVM/control-flow-ops-to-llvm.mlir
+++ b/mlir/test/Conversion/SPIRVToLLVM/control-flow-ops-to-llvm.mlir
@@ -66,16 +66,14 @@ spv.module Logical GLSL450 {
   ^inner_false(%arg3: i32, %arg4: i32):
     spv.Return
   }
-}
-
-// -----
 
-spv.module Logical GLSL450 {
   spv.func @cond_branch_with_weights(%cond: i1) -> () "None" {
-    // expected-error@+1 {{failed to legalize operation 'spv.BranchConditional' that was explicitly marked illegal}}
+    // CHECK: llvm.cond_br %{{.*}} weights(dense<[1, 2]> : vector<2xi32>), ^bb1, ^bb2
     spv.BranchConditional %cond [1, 2], ^true, ^false
+  // CHECK: ^bb1:
   ^true:
     spv.Return
+  // CHECK: ^bb2:
   ^false:
     spv.Return
   }

From 1603470e59a99a39ebdc4bf62a3a16c8c4ebea36 Mon Sep 17 00:00:00 2001
From: Kirill Bobyrev <kbobyrev@google.com>
Date: Wed, 29 Jul 2020 10:27:11 +0200
Subject: [PATCH 0426/1035] [clangd] Fix clangd-indexeer builds after D84697

Some buildbots require explicit clangdSupport dependency:

http://lab.llvm.org:8011/builders/llvm-avr-linux/builds/3996/steps/build%20stage%201/logs/stdio
---
 clang-tools-extra/clangd/indexer/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/clang-tools-extra/clangd/indexer/CMakeLists.txt b/clang-tools-extra/clangd/indexer/CMakeLists.txt
index edbced1410bb8..ff110693c7066 100644
--- a/clang-tools-extra/clangd/indexer/CMakeLists.txt
+++ b/clang-tools-extra/clangd/indexer/CMakeLists.txt
@@ -20,4 +20,5 @@ clang_target_link_libraries(clangd-indexer
 target_link_libraries(clangd-indexer
   PRIVATE
   clangDaemon
+  clangdSupport
 )

From 334a7025e0d1dd79e3c6b1976be65296ab3da01d Mon Sep 17 00:00:00 2001
From: Xing GUO <higuoxing@gmail.com>
Date: Wed, 29 Jul 2020 16:29:51 +0800
Subject: [PATCH 0427/1035] [DWARFYAML] Replace uint*_t with yaml::Hex* in the
 'debug_aranges' entry.

Normally, we use yaml::Hex* to describe the length, offsets,
address/segment size. NFC.
---
 llvm/include/llvm/ObjectYAML/DWARFYAML.h            | 8 ++++----
 llvm/test/ObjectYAML/MachO/DWARF-debug_aranges.yaml | 8 ++++----
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/llvm/include/llvm/ObjectYAML/DWARFYAML.h b/llvm/include/llvm/ObjectYAML/DWARFYAML.h
index f4ace738ef8cf..6b8ab07274acc 100644
--- a/llvm/include/llvm/ObjectYAML/DWARFYAML.h
+++ b/llvm/include/llvm/ObjectYAML/DWARFYAML.h
@@ -66,11 +66,11 @@ struct ARangeDescriptor {
 
 struct ARange {
   dwarf::DwarfFormat Format;
-  uint64_t Length;
+  yaml::Hex64 Length;
   uint16_t Version;
-  uint32_t CuOffset;
-  uint8_t AddrSize;
-  uint8_t SegSize;
+  yaml::Hex64 CuOffset;
+  yaml::Hex8 AddrSize;
+  yaml::Hex8 SegSize;
   std::vector<ARangeDescriptor> Descriptors;
 };
 
diff --git a/llvm/test/ObjectYAML/MachO/DWARF-debug_aranges.yaml b/llvm/test/ObjectYAML/MachO/DWARF-debug_aranges.yaml
index 2822c94d77514..b23989ffad074 100644
--- a/llvm/test/ObjectYAML/MachO/DWARF-debug_aranges.yaml
+++ b/llvm/test/ObjectYAML/MachO/DWARF-debug_aranges.yaml
@@ -325,11 +325,11 @@ DWARF:
 
 #CHECK: DWARF:           
 #CHECK:   debug_aranges:   
-#CHECK:     - Length:          44
+#CHECK:     - Length:          0x000000000000002C
 #CHECK:       Version:         2
-#CHECK:       CuOffset:        0
-#CHECK:       AddrSize:        8
-#CHECK:       SegSize:         0
+#CHECK:       CuOffset:        0x0000000000000000
+#CHECK:       AddrSize:        0x08
+#CHECK:       SegSize:         0x00
 #CHECK:       Descriptors:     
 #CHECK:         - Address:         0x0000000100000F50
 #CHECK:           Length:          52

From 2f98eff3457cf08d45ffd709c69238efbecbfa69 Mon Sep 17 00:00:00 2001
From: Xing GUO <higuoxing@gmail.com>
Date: Wed, 29 Jul 2020 17:31:38 +0800
Subject: [PATCH 0428/1035] [DWARFYAML][test] Make the check lines stricter.
 NFC.

This patch makes the check lines stricter.
---
 .../ObjectYAML/MachO/DWARF-debug_aranges.yaml | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/llvm/test/ObjectYAML/MachO/DWARF-debug_aranges.yaml b/llvm/test/ObjectYAML/MachO/DWARF-debug_aranges.yaml
index b23989ffad074..2953317518795 100644
--- a/llvm/test/ObjectYAML/MachO/DWARF-debug_aranges.yaml
+++ b/llvm/test/ObjectYAML/MachO/DWARF-debug_aranges.yaml
@@ -323,13 +323,12 @@ DWARF:
           Length:          52
 ...
 
-#CHECK: DWARF:           
-#CHECK:   debug_aranges:   
-#CHECK:     - Length:          0x000000000000002C
-#CHECK:       Version:         2
-#CHECK:       CuOffset:        0x0000000000000000
-#CHECK:       AddrSize:        0x08
-#CHECK:       SegSize:         0x00
-#CHECK:       Descriptors:     
-#CHECK:         - Address:         0x0000000100000F50
-#CHECK:           Length:          52
+#      CHECK:   debug_aranges:
+# CHECK-NEXT:     - Length:   0x000000000000002C
+# CHECK-NEXT:       Version:  2
+# CHECK-NEXT:       CuOffset: 0x0000000000000000
+# CHECK-NEXT:       AddrSize: 0x08
+# CHECK-NEXT:       SegSize:  0x00
+# CHECK-NEXT:       Descriptors:
+# CHECK-NEXT:         - Address: 0x0000000100000F50
+# CHECK-NEXT:           Length:  52

From 2e7baf61970a08a77540f3afdbefbb20b4a78348 Mon Sep 17 00:00:00 2001
From: Frederik Gossen <frgossen@google.com>
Date: Wed, 29 Jul 2020 10:22:18 +0000
Subject: [PATCH 0429/1035] [MLIR][Shape] Allow `shape.add` to operate on
 indices

Differential Revision: https://reviews.llvm.org/D84441
---
 .../include/mlir/Dialect/Shape/IR/ShapeOps.td | 21 ++++++++++++-------
 mlir/test/Dialect/Shape/invalid.mlir          | 14 ++++++++++---
 mlir/test/Dialect/Shape/ops.mlir              |  9 ++++++++
 3 files changed, 33 insertions(+), 11 deletions(-)

diff --git a/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td b/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td
index 7b676a2b05981..9ba838dbbb26f 100644
--- a/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td
+++ b/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td
@@ -53,18 +53,23 @@ def Shape_WitnessType : DialectType<ShapeDialect,
 class Shape_Op<string mnemonic, list<OpTrait> traits = []> :
     Op<ShapeDialect, mnemonic, traits>;
 
-def Shape_AddOp : Shape_Op<"add", [Commutative, SameOperandsAndResultType]> {
-  let summary = "Addition of sizes";
+def Shape_AddOp : Shape_Op<"add", [Commutative, NoSideEffect]> {
+  let summary = "Addition of sizes and indices";
   let description = [{
-    Adds two valid sizes as follows:
-    * lhs + rhs = unknown if either lhs or rhs unknown;
-    * lhs + rhs = (int)lhs + (int)rhs if known;
+    Adds two sizes or indices. If either operand is an error it will be
+    propagated to the result. The operands can be of type `size` or `index`. If
+    at least one of the operands can hold an error, i.e. if it is of type `size`,
+    then also the result must be of type `size`.
   }];
 
-  let arguments = (ins Shape_SizeType:$lhs, Shape_SizeType:$rhs);
-  let results = (outs Shape_SizeType:$result);
+  let arguments = (ins Shape_SizeOrIndexType:$lhs, Shape_SizeOrIndexType:$rhs);
+  let results = (outs Shape_SizeOrIndexType:$result);
 
-  let assemblyFormat = "$lhs `,` $rhs attr-dict";
+  let assemblyFormat = [{
+    $lhs `,` $rhs attr-dict `:` type($lhs) `,` type($rhs) `->` type($result)
+  }];
+
+  let verifier = [{ return verifySizeOrIndexOp(*this); }];
 }
 
 def Shape_BroadcastOp : Shape_Op<"broadcast", [Commutative]> {
diff --git a/mlir/test/Dialect/Shape/invalid.mlir b/mlir/test/Dialect/Shape/invalid.mlir
index 4a45181d4587c..d804efbd9e4ec 100644
--- a/mlir/test/Dialect/Shape/invalid.mlir
+++ b/mlir/test/Dialect/Shape/invalid.mlir
@@ -105,7 +105,7 @@ func @rank(%arg : !shape.shape) {
 
 // -----
 
-func @get_extent_error_possible(%arg : tensor<?xindex>) -> index {
+func @get_extent(%arg : tensor<?xindex>) -> index {
   %c0 = shape.const_size 0
   // expected-error@+1 {{if at least one of the operands can hold error values then the result must be of type `size` to propagate them}}
   %result = shape.get_extent %arg, %c0 : tensor<?xindex>, !shape.size -> index
@@ -114,7 +114,7 @@ func @get_extent_error_possible(%arg : tensor<?xindex>) -> index {
 
 // -----
 
-func @mul_error_possible(%lhs : !shape.size, %rhs : index) -> index {
+func @mul(%lhs : !shape.size, %rhs : index) -> index {
   // expected-error@+1 {{if at least one of the operands can hold error values then the result must be of type `size` to propagate them}}
   %result = shape.mul %lhs, %rhs : !shape.size, index -> index
   return %result : index
@@ -122,9 +122,17 @@ func @mul_error_possible(%lhs : !shape.size, %rhs : index) -> index {
 
 // -----
 
-func @num_elements_error_possible(%arg : !shape.shape) -> index {
+func @num_elements(%arg : !shape.shape) -> index {
   // expected-error@+1 {{if at least one of the operands can hold error values then the result must be of type `size` to propagate them}}
   %result = shape.num_elements %arg : !shape.shape -> index
   return %result : index
 }
 
+// -----
+
+func @add(%lhs : !shape.size, %rhs : index) -> index {
+  // expected-error@+1 {{if at least one of the operands can hold error values then the result must be of type `size` to propagate them}}
+  %result = shape.add %lhs, %rhs : !shape.size, index -> index
+  return %result : index
+}
+
diff --git a/mlir/test/Dialect/Shape/ops.mlir b/mlir/test/Dialect/Shape/ops.mlir
index 87af623fe0f7b..229f3948d31df 100644
--- a/mlir/test/Dialect/Shape/ops.mlir
+++ b/mlir/test/Dialect/Shape/ops.mlir
@@ -120,6 +120,15 @@ func @mul(%size_arg : !shape.size, %index_arg : index) {
   return
 }
 
+func @add(%size_arg : !shape.size, %index_arg : index) {
+  %size_sum = shape.add %size_arg, %size_arg
+      : !shape.size, !shape.size -> !shape.size
+  %index_sum = shape.add %index_arg, %index_arg : index, index -> index
+  %mixed_sum = shape.add %size_arg, %index_arg
+      : !shape.size, index -> !shape.size
+  return
+}
+
 func @const_size() {
   // CHECK: %c1 = shape.const_size 1
   // CHECK: %c2 = shape.const_size 2

From 0c005be6eb6bc21464d7c1f7d5f44eb1e5369749 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Wed, 29 Jul 2020 11:32:07 +0100
Subject: [PATCH 0430/1035] [X86][SSE] getV4X86ShuffleImm8 - canonicalize
 broadcast masks

If the mask input to getV4X86ShuffleImm8 only refers to a single source element (+ undefs) then canonicalize to a full broadcast.

getV4X86ShuffleImm8 defaults to inline values for undefs, which can be useful for shuffle widening/narrowing but does leave SimplifyDemanded* calls thinking the shuffle depends on unnecessary elements.

I'm still investigating what we should do more generally to avoid these undemanded elements, but broadcast cases was a simpler win.
---
 llvm/lib/Target/X86/X86ISelLowering.cpp       |   9 +
 llvm/test/CodeGen/X86/2011-05-09-loaduse.ll   |   4 +-
 llvm/test/CodeGen/X86/atomic-fp.ll            |  10 +-
 llvm/test/CodeGen/X86/atomic-non-integer.ll   |   4 +-
 llvm/test/CodeGen/X86/avg.ll                  |  44 +--
 .../CodeGen/X86/avx-intrinsics-fast-isel.ll   |   4 +-
 llvm/test/CodeGen/X86/avx-splat.ll            |   6 +-
 llvm/test/CodeGen/X86/avx-vbroadcast.ll       |   4 +-
 llvm/test/CodeGen/X86/avx-vinsertf128.ll      |   2 +-
 llvm/test/CodeGen/X86/avx-vperm2x128.ll       |   2 +-
 .../CodeGen/X86/avx512-any_extend_load.ll     |   2 +-
 llvm/test/CodeGen/X86/avx512-cvt.ll           |   2 +-
 llvm/test/CodeGen/X86/avx512-hadd-hsub.ll     |   8 +-
 .../X86/avx512-intrinsics-fast-isel.ll        |  24 +-
 .../X86/avx512-shuffles/partial_permute.ll    |  19 +-
 .../X86/bitcast-int-to-vector-bool-sext.ll    |  16 +-
 .../X86/bitcast-int-to-vector-bool-zext.ll    |  16 +-
 .../CodeGen/X86/bitcast-int-to-vector-bool.ll |   4 +-
 .../test/CodeGen/X86/bitcast-int-to-vector.ll |   4 +-
 llvm/test/CodeGen/X86/bitcast-vector-bool.ll  |   2 +-
 llvm/test/CodeGen/X86/buildvec-extract.ll     |  10 +-
 llvm/test/CodeGen/X86/buildvec-insertvec.ll   |   4 +-
 llvm/test/CodeGen/X86/cast-vsel.ll            |  12 +-
 llvm/test/CodeGen/X86/combine-fcopysign.ll    |   4 +-
 .../X86/div-rem-pair-recomposition-signed.ll  |  24 +-
 .../div-rem-pair-recomposition-unsigned.ll    |  24 +-
 llvm/test/CodeGen/X86/extract-fp.ll           |   8 +-
 llvm/test/CodeGen/X86/extract-store.ll        |   8 +-
 llvm/test/CodeGen/X86/extractelement-index.ll |   4 +-
 llvm/test/CodeGen/X86/extractelement-load.ll  |   6 +-
 llvm/test/CodeGen/X86/fma.ll                  | 168 +++++------
 llvm/test/CodeGen/X86/fp-intrinsics-fma.ll    |  72 ++---
 llvm/test/CodeGen/X86/fp-round.ll             |  28 +-
 llvm/test/CodeGen/X86/fp-roundeven.ll         |  86 +++---
 llvm/test/CodeGen/X86/gather-addresses.ll     |  16 +-
 llvm/test/CodeGen/X86/haddsub-2.ll            |  80 +++---
 llvm/test/CodeGen/X86/haddsub-3.ll            |   4 +-
 llvm/test/CodeGen/X86/haddsub-undef.ll        |  28 +-
 llvm/test/CodeGen/X86/haddsub.ll              |  36 +--
 llvm/test/CodeGen/X86/half.ll                 |  12 +-
 .../test/CodeGen/X86/horizontal-reduce-add.ll |  40 +--
 .../CodeGen/X86/horizontal-reduce-fadd.ll     |   6 +-
 .../CodeGen/X86/horizontal-reduce-smax.ll     |  94 +++---
 .../CodeGen/X86/horizontal-reduce-smin.ll     |  94 +++---
 .../CodeGen/X86/horizontal-reduce-umax.ll     |  94 +++---
 .../CodeGen/X86/horizontal-reduce-umin.ll     |  94 +++---
 .../X86/insert-into-constant-vector.ll        |   2 +-
 llvm/test/CodeGen/X86/insert-loaded-scalar.ll |   4 +-
 .../CodeGen/X86/insertelement-var-index.ll    |  12 +-
 .../test/CodeGen/X86/known-signbits-vector.ll |   8 +-
 llvm/test/CodeGen/X86/load-partial.ll         |   8 +-
 llvm/test/CodeGen/X86/madd.ll                 | 108 +++----
 llvm/test/CodeGen/X86/masked_compressstore.ll |  64 ++---
 llvm/test/CodeGen/X86/masked_expandload.ll    |   2 +-
 llvm/test/CodeGen/X86/masked_load.ll          |  10 +-
 llvm/test/CodeGen/X86/masked_store.ll         |  58 ++--
 llvm/test/CodeGen/X86/masked_store_trunc.ll   |  14 +-
 .../CodeGen/X86/masked_store_trunc_ssat.ll    |  14 +-
 .../CodeGen/X86/masked_store_trunc_usat.ll    |  14 +-
 llvm/test/CodeGen/X86/memset-nonzero.ll       |  10 +-
 .../X86/merge-consecutive-stores-nt.ll        |  16 +-
 .../CodeGen/X86/min-legal-vector-width.ll     |   8 +-
 llvm/test/CodeGen/X86/mmx-arith.ll            |   2 +-
 llvm/test/CodeGen/X86/nontemporal-2.ll        |   6 +-
 llvm/test/CodeGen/X86/oddshuffles.ll          |  76 ++---
 llvm/test/CodeGen/X86/phaddsub-extract.ll     | 106 +++----
 llvm/test/CodeGen/X86/phaddsub-undef.ll       |   4 +-
 llvm/test/CodeGen/X86/phaddsub.ll             |  50 ++--
 llvm/test/CodeGen/X86/pmaddubsw.ll            |  16 +-
 llvm/test/CodeGen/X86/pmul.ll                 |   4 +-
 llvm/test/CodeGen/X86/pmulh.ll                |  12 +-
 llvm/test/CodeGen/X86/pow.ll                  |   4 +-
 llvm/test/CodeGen/X86/pr14161.ll              |   3 +-
 llvm/test/CodeGen/X86/pr29112.ll              |   2 +-
 llvm/test/CodeGen/X86/pr44976.ll              |  39 ++-
 llvm/test/CodeGen/X86/pr46455.ll              |   4 +-
 llvm/test/CodeGen/X86/pr46527.ll              |   2 +-
 .../CodeGen/X86/prefer-avx256-mask-shuffle.ll |   6 +-
 llvm/test/CodeGen/X86/psubus.ll               |  26 +-
 llvm/test/CodeGen/X86/sad.ll                  |  96 ++++---
 llvm/test/CodeGen/X86/scalarize-fp.ll         |   4 +-
 llvm/test/CodeGen/X86/setcc-combine.ll        |  18 +-
 llvm/test/CodeGen/X86/shrink_vmul.ll          |  20 +-
 .../X86/shuffle-strided-with-offset-128.ll    |   8 +-
 llvm/test/CodeGen/X86/slow-pmulld.ll          |  48 ++--
 llvm/test/CodeGen/X86/smul_fix_sat.ll         |  16 +-
 llvm/test/CodeGen/X86/split-vector-bitcast.ll |   2 +-
 llvm/test/CodeGen/X86/split-vector-rem.ll     |  48 ++--
 llvm/test/CodeGen/X86/sse1.ll                 |   4 +-
 .../CodeGen/X86/sse2-intrinsics-fast-isel.ll  |  24 +-
 llvm/test/CodeGen/X86/sse3-avx-addsub-2.ll    |  44 +--
 llvm/test/CodeGen/X86/sse3.ll                 |   4 +-
 llvm/test/CodeGen/X86/sse41.ll                | 115 ++++----
 llvm/test/CodeGen/X86/umul_fix_sat.ll         |  16 +-
 llvm/test/CodeGen/X86/var-permute-128.ll      |   8 +-
 llvm/test/CodeGen/X86/vec-libcalls.ll         |  12 +-
 llvm/test/CodeGen/X86/vec-strict-128.ll       |  24 +-
 llvm/test/CodeGen/X86/vec-strict-cmp-128.ll   | 128 ++++-----
 .../test/CodeGen/X86/vec-strict-cmp-sub128.ll |  16 +-
 .../CodeGen/X86/vec-strict-fptoint-128.ll     |  28 +-
 .../CodeGen/X86/vec-strict-fptoint-256.ll     |  18 +-
 .../CodeGen/X86/vec-strict-fptoint-512.ll     |  12 +-
 .../CodeGen/X86/vec-strict-inttofp-128.ll     |   8 +-
 .../CodeGen/X86/vec-strict-inttofp-256.ll     |   4 +-
 llvm/test/CodeGen/X86/vec3.ll                 |   4 +-
 llvm/test/CodeGen/X86/vec_cast2.ll            |   4 +-
 llvm/test/CodeGen/X86/vec_extract-mmx.ll      |   4 +-
 llvm/test/CodeGen/X86/vec_extract.ll          |   4 +-
 llvm/test/CodeGen/X86/vec_fp_to_int.ll        |  60 ++--
 llvm/test/CodeGen/X86/vec_int_to_fp.ll        |  48 ++--
 llvm/test/CodeGen/X86/vec_saddo.ll            |  20 +-
 llvm/test/CodeGen/X86/vec_set-H.ll            |   2 +-
 llvm/test/CodeGen/X86/vec_shift7.ll           |   2 +-
 llvm/test/CodeGen/X86/vec_smulo.ll            |  64 ++---
 llvm/test/CodeGen/X86/vec_ssubo.ll            |  20 +-
 llvm/test/CodeGen/X86/vec_uaddo.ll            |  20 +-
 llvm/test/CodeGen/X86/vec_umulo.ll            |  64 ++---
 llvm/test/CodeGen/X86/vec_usubo.ll            |  20 +-
 .../X86/vector-constrained-fp-intrinsics.ll   |   8 +-
 llvm/test/CodeGen/X86/vector-extend-inreg.ll  |   2 +-
 llvm/test/CodeGen/X86/vector-fshl-128.ll      |  22 +-
 llvm/test/CodeGen/X86/vector-fshl-256.ll      |   8 +-
 llvm/test/CodeGen/X86/vector-fshl-rot-128.ll  |  10 +-
 llvm/test/CodeGen/X86/vector-fshl-rot-256.ll  |   4 +-
 llvm/test/CodeGen/X86/vector-fshr-128.ll      |  22 +-
 llvm/test/CodeGen/X86/vector-fshr-256.ll      |   4 +-
 llvm/test/CodeGen/X86/vector-fshr-rot-128.ll  |  10 +-
 llvm/test/CodeGen/X86/vector-fshr-rot-256.ll  |   4 +-
 llvm/test/CodeGen/X86/vector-idiv-v2i32.ll    |  94 +++---
 llvm/test/CodeGen/X86/vector-narrow-binop.ll  |   4 +-
 llvm/test/CodeGen/X86/vector-reduce-add.ll    |  86 +++---
 .../test/CodeGen/X86/vector-reduce-and-cmp.ll |  96 +++----
 llvm/test/CodeGen/X86/vector-reduce-and.ll    |  96 +++----
 .../CodeGen/X86/vector-reduce-fadd-fast.ll    |  24 +-
 llvm/test/CodeGen/X86/vector-reduce-fadd.ll   | 216 +++++++-------
 .../CodeGen/X86/vector-reduce-fmax-nnan.ll    |   8 +-
 llvm/test/CodeGen/X86/vector-reduce-fmax.ll   |   8 +-
 .../CodeGen/X86/vector-reduce-fmin-nnan.ll    |   8 +-
 llvm/test/CodeGen/X86/vector-reduce-fmin.ll   |   8 +-
 .../CodeGen/X86/vector-reduce-fmul-fast.ll    |  24 +-
 llvm/test/CodeGen/X86/vector-reduce-fmul.ll   | 216 +++++++-------
 llvm/test/CodeGen/X86/vector-reduce-mul.ll    | 252 ++++++++--------
 llvm/test/CodeGen/X86/vector-reduce-or.ll     |  96 +++----
 llvm/test/CodeGen/X86/vector-reduce-smax.ll   |  76 ++---
 llvm/test/CodeGen/X86/vector-reduce-smin.ll   |  76 ++---
 llvm/test/CodeGen/X86/vector-reduce-umax.ll   |  76 ++---
 llvm/test/CodeGen/X86/vector-reduce-umin.ll   |  76 ++---
 llvm/test/CodeGen/X86/vector-reduce-xor.ll    |  96 +++----
 llvm/test/CodeGen/X86/vector-rem.ll           |  24 +-
 llvm/test/CodeGen/X86/vector-rotate-128.ll    |  10 +-
 llvm/test/CodeGen/X86/vector-rotate-256.ll    |   4 +-
 llvm/test/CodeGen/X86/vector-sext.ll          | 132 ++++-----
 .../test/CodeGen/X86/vector-shift-ashr-128.ll |   4 +-
 .../CodeGen/X86/vector-shift-ashr-sub128.ll   |  18 +-
 .../X86/vector-shift-by-select-loop.ll        |  24 +-
 .../test/CodeGen/X86/vector-shift-lshr-128.ll |   4 +-
 .../CodeGen/X86/vector-shift-lshr-sub128.ll   |  18 +-
 llvm/test/CodeGen/X86/vector-shift-shl-128.ll |   4 +-
 .../CodeGen/X86/vector-shift-shl-sub128.ll    |  18 +-
 .../CodeGen/X86/vector-shuffle-128-v16.ll     |  30 +-
 .../test/CodeGen/X86/vector-shuffle-128-v4.ll |  40 +--
 .../test/CodeGen/X86/vector-shuffle-128-v8.ll | 100 +++----
 .../CodeGen/X86/vector-shuffle-256-v16.ll     | 271 +++++++++---------
 .../CodeGen/X86/vector-shuffle-256-v32.ll     |  24 +-
 .../test/CodeGen/X86/vector-shuffle-256-v4.ll |  10 +-
 .../test/CodeGen/X86/vector-shuffle-256-v8.ll |  14 +-
 .../CodeGen/X86/vector-shuffle-512-v16.ll     |   2 +-
 .../CodeGen/X86/vector-shuffle-512-v64.ll     |   4 +-
 .../X86/vector-shuffle-combining-avx2.ll      |   2 +-
 .../CodeGen/X86/vector-shuffle-combining.ll   |  14 +-
 llvm/test/CodeGen/X86/vector-shuffle-mmx.ll   |   4 +-
 llvm/test/CodeGen/X86/vector-shuffle-sse1.ll  |   2 +-
 llvm/test/CodeGen/X86/vector-shuffle-v1.ll    |   4 +-
 llvm/test/CodeGen/X86/vector-trunc.ll         |   8 +-
 llvm/test/CodeGen/X86/vector-zext.ll          | 114 ++++----
 llvm/test/CodeGen/X86/vsel-cmp-load.ll        |   8 +-
 llvm/test/CodeGen/X86/vselect.ll              |   6 +-
 llvm/test/CodeGen/X86/vshift-4.ll             |  16 +-
 llvm/test/CodeGen/X86/widen_conv-3.ll         |   2 +-
 llvm/test/CodeGen/X86/widen_conv-4.ll         |   4 +-
 llvm/test/CodeGen/X86/widened-broadcast.ll    |  12 +-
 181 files changed, 2865 insertions(+), 2833 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 9dd4dfdfea49a..94e2b1708b9db 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -10938,6 +10938,15 @@ static unsigned getV4X86ShuffleImm(ArrayRef<int> Mask) {
   assert(Mask[2] >= -1 && Mask[2] < 4 && "Out of bound mask element!");
   assert(Mask[3] >= -1 && Mask[3] < 4 && "Out of bound mask element!");
 
+  // If the mask only uses one non-undef element, then fully 'splat' it to
+  // improve later broadcast matching.
+  int FirstIndex = find_if(Mask, [](int M) { return M >= 0; }) - Mask.begin();
+  assert(0 <= FirstIndex && FirstIndex < 4 && "All undef shuffle mask");
+
+  int FirstElt = Mask[FirstIndex];
+  if (all_of(Mask, [FirstElt](int M) { return M < 0 || M == FirstElt; }))
+    return (FirstElt << 6) | (FirstElt << 4) | (FirstElt << 2) | FirstElt;
+
   unsigned Imm = 0;
   Imm |= (Mask[0] < 0 ? 0 : Mask[0]) << 0;
   Imm |= (Mask[1] < 0 ? 1 : Mask[1]) << 2;
diff --git a/llvm/test/CodeGen/X86/2011-05-09-loaduse.ll b/llvm/test/CodeGen/X86/2011-05-09-loaduse.ll
index 53b710324292d..027e9b5a86f43 100644
--- a/llvm/test/CodeGen/X86/2011-05-09-loaduse.ll
+++ b/llvm/test/CodeGen/X86/2011-05-09-loaduse.ll
@@ -8,7 +8,7 @@ define float @test(<4 x float>* %A) nounwind {
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movaps (%eax), %xmm0
-; X86-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; X86-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; X86-NEXT:    xorps %xmm1, %xmm1
 ; X86-NEXT:    movaps %xmm1, (%eax)
 ; X86-NEXT:    movss %xmm0, (%esp)
@@ -19,7 +19,7 @@ define float @test(<4 x float>* %A) nounwind {
 ; X64-LABEL: test:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    movaps (%rdi), %xmm0
-; X64-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; X64-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; X64-NEXT:    xorps %xmm1, %xmm1
 ; X64-NEXT:    movaps %xmm1, (%rdi)
 ; X64-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/atomic-fp.ll b/llvm/test/CodeGen/X86/atomic-fp.ll
index 17b58a581db1c..7cd38af00079f 100644
--- a/llvm/test/CodeGen/X86/atomic-fp.ll
+++ b/llvm/test/CodeGen/X86/atomic-fp.ll
@@ -113,7 +113,7 @@ define void @fadd_64r(double* %loc, double %val) nounwind {
 ; X86-SSE1-NEXT:    xorps %xmm1, %xmm1
 ; X86-SSE1-NEXT:    movlps {{.*#+}} xmm1 = mem[0,1],xmm1[2,3]
 ; X86-SSE1-NEXT:    movss %xmm1, (%esp)
-; X86-SSE1-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; X86-SSE1-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,1,1]
 ; X86-SSE1-NEXT:    movss %xmm1, {{[0-9]+}}(%esp)
 ; X86-SSE1-NEXT:    fldl (%esp)
 ; X86-SSE1-NEXT:    faddl 12(%ebp)
@@ -278,7 +278,7 @@ define void @fadd_64g() nounwind {
 ; X86-SSE1-NEXT:    xorps %xmm1, %xmm1
 ; X86-SSE1-NEXT:    movlps {{.*#+}} xmm1 = mem[0,1],xmm1[2,3]
 ; X86-SSE1-NEXT:    movss %xmm1, (%esp)
-; X86-SSE1-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; X86-SSE1-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,1,1]
 ; X86-SSE1-NEXT:    movss %xmm1, {{[0-9]+}}(%esp)
 ; X86-SSE1-NEXT:    fld1
 ; X86-SSE1-NEXT:    faddl (%esp)
@@ -441,7 +441,7 @@ define void @fadd_64imm() nounwind {
 ; X86-SSE1-NEXT:    xorps %xmm1, %xmm1
 ; X86-SSE1-NEXT:    movlps {{.*#+}} xmm1 = mem[0,1],xmm1[2,3]
 ; X86-SSE1-NEXT:    movss %xmm1, (%esp)
-; X86-SSE1-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; X86-SSE1-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,1,1]
 ; X86-SSE1-NEXT:    movss %xmm1, {{[0-9]+}}(%esp)
 ; X86-SSE1-NEXT:    fld1
 ; X86-SSE1-NEXT:    faddl (%esp)
@@ -610,7 +610,7 @@ define void @fadd_64stack() nounwind {
 ; X86-SSE1-NEXT:    xorps %xmm1, %xmm1
 ; X86-SSE1-NEXT:    movlps {{.*#+}} xmm1 = mem[0,1],xmm1[2,3]
 ; X86-SSE1-NEXT:    movss %xmm1, (%esp)
-; X86-SSE1-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; X86-SSE1-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,1,1]
 ; X86-SSE1-NEXT:    movss %xmm1, {{[0-9]+}}(%esp)
 ; X86-SSE1-NEXT:    fld1
 ; X86-SSE1-NEXT:    faddl (%esp)
@@ -716,7 +716,7 @@ define void @fadd_array(i64* %arg, double %arg1, i64 %arg2) nounwind {
 ; X86-SSE1-NEXT:    xorps %xmm1, %xmm1
 ; X86-SSE1-NEXT:    movlps {{.*#+}} xmm1 = mem[0,1],xmm1[2,3]
 ; X86-SSE1-NEXT:    movss %xmm1, (%esp)
-; X86-SSE1-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; X86-SSE1-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,1,1]
 ; X86-SSE1-NEXT:    movss %xmm1, {{[0-9]+}}(%esp)
 ; X86-SSE1-NEXT:    fldl (%esp)
 ; X86-SSE1-NEXT:    faddl 12(%ebp)
diff --git a/llvm/test/CodeGen/X86/atomic-non-integer.ll b/llvm/test/CodeGen/X86/atomic-non-integer.ll
index 664f195e3884a..a963865136054 100644
--- a/llvm/test/CodeGen/X86/atomic-non-integer.ll
+++ b/llvm/test/CodeGen/X86/atomic-non-integer.ll
@@ -278,7 +278,7 @@ define double @load_double(double* %fptr) {
 ; X86-SSE1-NEXT:    xorps %xmm0, %xmm0
 ; X86-SSE1-NEXT:    movlps {{.*#+}} xmm0 = mem[0,1],xmm0[2,3]
 ; X86-SSE1-NEXT:    movss %xmm0, (%esp)
-; X86-SSE1-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; X86-SSE1-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; X86-SSE1-NEXT:    movss %xmm0, {{[0-9]+}}(%esp)
 ; X86-SSE1-NEXT:    fldl (%esp)
 ; X86-SSE1-NEXT:    addl $12, %esp
@@ -665,7 +665,7 @@ define double @load_double_seq_cst(double* %fptr) {
 ; X86-SSE1-NEXT:    xorps %xmm0, %xmm0
 ; X86-SSE1-NEXT:    movlps {{.*#+}} xmm0 = mem[0,1],xmm0[2,3]
 ; X86-SSE1-NEXT:    movss %xmm0, (%esp)
-; X86-SSE1-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; X86-SSE1-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; X86-SSE1-NEXT:    movss %xmm0, {{[0-9]+}}(%esp)
 ; X86-SSE1-NEXT:    fldl (%esp)
 ; X86-SSE1-NEXT:    addl $12, %esp
diff --git a/llvm/test/CodeGen/X86/avg.ll b/llvm/test/CodeGen/X86/avg.ll
index 1411318d8176d..8cd4b2f2571f3 100644
--- a/llvm/test/CodeGen/X86/avg.ll
+++ b/llvm/test/CodeGen/X86/avg.ll
@@ -162,12 +162,12 @@ define void @avg_v24i8(<24 x i8>* %a, <24 x i8>* %b) nounwind {
 ; AVX1-NEXT:    vmovdqa 16(%rdi), %xmm1
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[3,3,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[3,3,3,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm1[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm1[1,1,1,1]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm5 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm8 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; AVX1-NEXT:    vmovdqa (%rsi), %xmm6
@@ -175,15 +175,15 @@ define void @avg_v24i8(<24 x i8>* %a, <24 x i8>* %b) nounwind {
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm6[2,3,2,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm6[3,3,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm6[3,3,3,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
 ; AVX1-NEXT:    vpaddd %xmm2, %xmm3, %xmm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm6[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm6[1,1,1,1]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
 ; AVX1-NEXT:    vpaddd %xmm3, %xmm4, %xmm3
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm4 = xmm6[0],zero,zero,zero,xmm6[1],zero,zero,zero,xmm6[2],zero,zero,zero,xmm6[3],zero,zero,zero
 ; AVX1-NEXT:    vpaddd %xmm4, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm7[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm7[1,1,1,1]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero
 ; AVX1-NEXT:    vpaddd %xmm4, %xmm5, %xmm4
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm5 = xmm7[0],zero,zero,zero,xmm7[1],zero,zero,zero,xmm7[2],zero,zero,zero,xmm7[3],zero,zero,zero
@@ -456,25 +456,25 @@ define void @avg_v48i8(<48 x i8>* %a, <48 x i8>* %b) nounwind {
 ; AVX1-NEXT:    vmovdqa 32(%rdi), %xmm1
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm5 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[3,3,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm6 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm7 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm15 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[3,3,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[3,3,3,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm11 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm14 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm13 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm4[2,3,2,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; AVX1-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm4[3,3,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm4[3,3,3,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; AVX1-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm4[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm4[1,1,1,1]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; AVX1-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero
@@ -485,10 +485,10 @@ define void @avg_v48i8(<48 x i8>* %a, <48 x i8>* %b) nounwind {
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
 ; AVX1-NEXT:    vpaddd %xmm2, %xmm5, %xmm12
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm3[3,3,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm3[3,3,3,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm5 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero
 ; AVX1-NEXT:    vpaddd %xmm5, %xmm6, %xmm10
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm3[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm3[1,1,1,1]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm6 = xmm6[0],zero,zero,zero,xmm6[1],zero,zero,zero,xmm6[2],zero,zero,zero,xmm6[3],zero,zero,zero
 ; AVX1-NEXT:    vpaddd %xmm6, %xmm7, %xmm9
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
@@ -496,10 +496,10 @@ define void @avg_v48i8(<48 x i8>* %a, <48 x i8>* %b) nounwind {
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
 ; AVX1-NEXT:    vpaddd %xmm3, %xmm15, %xmm15
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm0[3,3,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm0[3,3,3,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm7 = xmm7[0],zero,zero,zero,xmm7[1],zero,zero,zero,xmm7[2],zero,zero,zero,xmm7[3],zero,zero,zero
 ; AVX1-NEXT:    vpaddd %xmm7, %xmm11, %xmm7
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
 ; AVX1-NEXT:    vpaddd %xmm2, %xmm14, %xmm14
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
@@ -507,10 +507,10 @@ define void @avg_v48i8(<48 x i8>* %a, <48 x i8>* %b) nounwind {
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm4[2,3,2,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm5 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero
 ; AVX1-NEXT:    vpaddd {{[-0-9]+}}(%r{{[sb]}}p), %xmm5, %xmm5 # 16-byte Folded Reload
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm4[3,3,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm4[3,3,3,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm6 = xmm6[0],zero,zero,zero,xmm6[1],zero,zero,zero,xmm6[2],zero,zero,zero,xmm6[3],zero,zero,zero
 ; AVX1-NEXT:    vpaddd {{[-0-9]+}}(%r{{[sb]}}p), %xmm6, %xmm6 # 16-byte Folded Reload
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm4[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm4[1,1,1,1]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; AVX1-NEXT:    vpaddd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero
@@ -2386,12 +2386,12 @@ define void @not_avg_v16i8_wide_constants(<16 x i8>* %a, <16 x i8>* %b) nounwind
 ; SSE2-NEXT:    pand %xmm0, %xmm2
 ; SSE2-NEXT:    pandn %xmm4, %xmm0
 ; SSE2-NEXT:    por %xmm2, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm8 = xmm8[0],xmm0[0],xmm8[1],xmm0[1]
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm12[0],xmm1[1],xmm12[1],xmm1[2],xmm12[2],xmm1[3],xmm12[3],xmm1[4],xmm12[4],xmm1[5],xmm12[5],xmm1[6],xmm12[6],xmm1[7],xmm12[7]
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm13[0],xmm6[1],xmm13[1],xmm6[2],xmm13[2],xmm6[3],xmm13[3],xmm6[4],xmm13[4],xmm6[5],xmm13[5],xmm6[6],xmm13[6],xmm6[7],xmm13[7]
 ; SSE2-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,1,2,0]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,0,0]
 ; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
 ; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm14[0],xmm5[1],xmm14[1],xmm5[2],xmm14[2],xmm5[3],xmm14[3],xmm5[4],xmm14[4],xmm5[5],xmm14[5],xmm5[6],xmm14[6],xmm5[7],xmm14[7]
@@ -2402,7 +2402,7 @@ define void @not_avg_v16i8_wide_constants(<16 x i8>* %a, <16 x i8>* %b) nounwind
 ; SSE2-NEXT:    pand %xmm1, %xmm2
 ; SSE2-NEXT:    pandn %xmm5, %xmm1
 ; SSE2-NEXT:    por %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,1,2,2]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,2,2,2]
 ; SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm0[2],xmm1[3],xmm0[3]
 ; SSE2-NEXT:    movsd {{.*#+}} xmm1 = xmm8[0],xmm1[1]
 ; SSE2-NEXT:    movupd %xmm1, (%rax)
@@ -2558,7 +2558,7 @@ define void @not_avg_v16i8_wide_constants(<16 x i8>* %a, <16 x i8>* %b) nounwind
 ; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm11[0],xmm10[0],xmm11[1],xmm10[1],xmm11[2],xmm10[2],xmm11[3],xmm10[3],xmm11[4],xmm10[4],xmm11[5],xmm10[5],xmm11[6],xmm10[6],xmm11[7],xmm10[7]
 ; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
 ; AVX1-NEXT:    vpslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1]
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[0,1,2,0]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1,2,3,4,5,6],xmm1[7]
 ; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3],xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
 ; AVX1-NEXT:    vpslldq {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm2[0,1,2,3,4,5]
diff --git a/llvm/test/CodeGen/X86/avx-intrinsics-fast-isel.ll b/llvm/test/CodeGen/X86/avx-intrinsics-fast-isel.ll
index 746b068cec2fb..35c449e813c09 100644
--- a/llvm/test/CodeGen/X86/avx-intrinsics-fast-isel.ll
+++ b/llvm/test/CodeGen/X86/avx-intrinsics-fast-isel.ll
@@ -1895,7 +1895,7 @@ define <4 x i64> @test_mm256_set1_epi16(i16 %a0) nounwind {
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    vmovd %eax, %xmm0
-; X86-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; X86-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; X86-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; X86-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; X86-NEXT:    retl
@@ -1903,7 +1903,7 @@ define <4 x i64> @test_mm256_set1_epi16(i16 %a0) nounwind {
 ; X64-LABEL: test_mm256_set1_epi16:
 ; X64:       # %bb.0:
 ; X64-NEXT:    vmovd %edi, %xmm0
-; X64-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; X64-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; X64-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; X64-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; X64-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/avx-splat.ll b/llvm/test/CodeGen/X86/avx-splat.ll
index 26fefb26fff41..3755cf4740abe 100644
--- a/llvm/test/CodeGen/X86/avx-splat.ll
+++ b/llvm/test/CodeGen/X86/avx-splat.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --no_x86_scrub_mem_shuffle
 ; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=CHECK,X86
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=CHECK,X64
 
@@ -16,7 +16,7 @@ entry:
 define <16 x i16> @funcB(<16 x i16> %a) nounwind uwtable readnone ssp {
 ; CHECK-LABEL: funcB:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,6,7]
+; CHECK-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,5,5]
 ; CHECK-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,2,2]
 ; CHECK-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; CHECK-NEXT:    ret{{[l|q]}}
@@ -134,7 +134,7 @@ define <8 x float> @funcF(i32 %val) nounwind {
 ; X64-LABEL: funcF:
 ; X64:       # %bb.0:
 ; X64-NEXT:    vmovd %edi, %xmm0
-; X64-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,0]
+; X64-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; X64-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; X64-NEXT:    retq
   %ret6 = insertelement <8 x i32> undef, i32 %val, i32 6
diff --git a/llvm/test/CodeGen/X86/avx-vbroadcast.ll b/llvm/test/CodeGen/X86/avx-vbroadcast.ll
index 609c02eee0902..89baa50bf0a40 100644
--- a/llvm/test/CodeGen/X86/avx-vbroadcast.ll
+++ b/llvm/test/CodeGen/X86/avx-vbroadcast.ll
@@ -624,12 +624,12 @@ entry:
 define <4 x i32> @H(<4 x i32> %a) {
 ; X32-LABEL: H:
 ; X32:       ## %bb.0: ## %entry
-; X32-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; X32-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: H:
 ; X64:       ## %bb.0: ## %entry
-; X64-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; X64-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; X64-NEXT:    retq
 entry:
   %x = shufflevector <4 x i32> %a, <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
diff --git a/llvm/test/CodeGen/X86/avx-vinsertf128.ll b/llvm/test/CodeGen/X86/avx-vinsertf128.ll
index 89f74a45e6b7b..baef0fb8567bd 100644
--- a/llvm/test/CodeGen/X86/avx-vinsertf128.ll
+++ b/llvm/test/CodeGen/X86/avx-vinsertf128.ll
@@ -29,7 +29,7 @@ define void @insert_crash() nounwind {
 ; CHECK-NEXT:    vminpd %xmm0, %xmm0, %xmm0
 ; CHECK-NEXT:    vminsd %xmm0, %xmm0, %xmm0
 ; CHECK-NEXT:    vcvtsd2ss %xmm0, %xmm0, %xmm0
-; CHECK-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,1,2,0]
+; CHECK-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; CHECK-NEXT:    vmovups %xmm0, (%rax)
 ; CHECK-NEXT:    retq
 allocas:
diff --git a/llvm/test/CodeGen/X86/avx-vperm2x128.ll b/llvm/test/CodeGen/X86/avx-vperm2x128.ll
index 26a5cd328d5c7..a519f55aaafe5 100644
--- a/llvm/test/CodeGen/X86/avx-vperm2x128.ll
+++ b/llvm/test/CodeGen/X86/avx-vperm2x128.ll
@@ -265,7 +265,7 @@ define <8 x float> @shuffle_v8f32_uu67uu67(<8 x float> %a, <8 x float> %b) nounw
 ;
 ; AVX2-LABEL: shuffle_v8f32_uu67uu67:
 ; AVX2:       # %bb.0: # %entry
-; AVX2-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,3,2,3]
+; AVX2-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[3,3,3,3]
 ; AVX2-NEXT:    retq
 entry:
   %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> <i32 undef, i32 undef, i32 6, i32 7, i32 undef, i32 undef, i32 6, i32 7>
diff --git a/llvm/test/CodeGen/X86/avx512-any_extend_load.ll b/llvm/test/CodeGen/X86/avx512-any_extend_load.ll
index 4c257a0edf9d7..99ebd4baffcca 100644
--- a/llvm/test/CodeGen/X86/avx512-any_extend_load.ll
+++ b/llvm/test/CodeGen/X86/avx512-any_extend_load.ll
@@ -8,7 +8,7 @@ define void @any_extend_load_v8i64(<8 x i8> * %ptr) {
 ; KNL:       # %bb.0:
 ; KNL-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
 ; KNL-NEXT:    vpmovzxbq {{.*#+}} ymm1 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero
-; KNL-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; KNL-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; KNL-NEXT:    vpmovzxbq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero
 ; KNL-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; KNL-NEXT:    vpaddq {{.*}}(%rip){1to8}, %zmm0, %zmm0
diff --git a/llvm/test/CodeGen/X86/avx512-cvt.ll b/llvm/test/CodeGen/X86/avx512-cvt.ll
index e92528b9bbcc8..64f281b7ffdf1 100644
--- a/llvm/test/CodeGen/X86/avx512-cvt.ll
+++ b/llvm/test/CodeGen/X86/avx512-cvt.ll
@@ -217,7 +217,7 @@ define <4 x i64> @f64to4sl(<4 x double> %a) {
 define <4 x i64> @f32to4sl(<4 x float> %a) {
 ; NODQ-LABEL: f32to4sl:
 ; NODQ:       # %bb.0:
-; NODQ-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; NODQ-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,3,3,3]
 ; NODQ-NEXT:    vcvttss2si %xmm1, %rax
 ; NODQ-NEXT:    vmovq %rax, %xmm1
 ; NODQ-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
diff --git a/llvm/test/CodeGen/X86/avx512-hadd-hsub.ll b/llvm/test/CodeGen/X86/avx512-hadd-hsub.ll
index b504646336def..9e6c8b8becbc9 100644
--- a/llvm/test/CodeGen/X86/avx512-hadd-hsub.ll
+++ b/llvm/test/CodeGen/X86/avx512-hadd-hsub.ll
@@ -7,7 +7,7 @@ define i32 @hadd_16(<16 x i32> %x225) {
 ; KNL:       # %bb.0:
 ; KNL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; KNL-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; KNL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; KNL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; KNL-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; KNL-NEXT:    vmovd %xmm0, %eax
 ; KNL-NEXT:    retq
@@ -16,7 +16,7 @@ define i32 @hadd_16(<16 x i32> %x225) {
 ; SKX:       # %bb.0:
 ; SKX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SKX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; SKX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SKX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SKX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; SKX-NEXT:    vmovd %xmm0, %eax
 ; SKX-NEXT:    vzeroupper
@@ -34,7 +34,7 @@ define i32 @hsub_16(<16 x i32> %x225) {
 ; KNL:       # %bb.0:
 ; KNL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; KNL-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; KNL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; KNL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; KNL-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
 ; KNL-NEXT:    vmovd %xmm0, %eax
 ; KNL-NEXT:    retq
@@ -43,7 +43,7 @@ define i32 @hsub_16(<16 x i32> %x225) {
 ; SKX:       # %bb.0:
 ; SKX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SKX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; SKX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SKX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SKX-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
 ; SKX-NEXT:    vmovd %xmm0, %eax
 ; SKX-NEXT:    vzeroupper
diff --git a/llvm/test/CodeGen/X86/avx512-intrinsics-fast-isel.ll b/llvm/test/CodeGen/X86/avx512-intrinsics-fast-isel.ll
index 650bbe23b86e5..2bc798c06b777 100644
--- a/llvm/test/CodeGen/X86/avx512-intrinsics-fast-isel.ll
+++ b/llvm/test/CodeGen/X86/avx512-intrinsics-fast-isel.ll
@@ -6972,7 +6972,7 @@ define i32 @test_mm512_reduce_add_epi32(<8 x i64> %__W) {
 ; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; CHECK-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
-; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; CHECK-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
 ; CHECK-NEXT:    vmovd %xmm0, %eax
 ; CHECK-NEXT:    vzeroupper
@@ -7006,7 +7006,7 @@ define i32 @test_mm512_reduce_mul_epi32(<8 x i64> %__W) {
 ; CHECK-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; CHECK-NEXT:    vpmulld %xmm0, %xmm1, %xmm0
-; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; CHECK-NEXT:    vpmulld %xmm0, %xmm1, %xmm0
 ; CHECK-NEXT:    vmovd %xmm0, %eax
 ; CHECK-NEXT:    vzeroupper
@@ -7040,7 +7040,7 @@ define i32 @test_mm512_reduce_or_epi32(<8 x i64> %__W) {
 ; CHECK-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; CHECK-NEXT:    vpor %xmm0, %xmm1, %xmm0
-; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; CHECK-NEXT:    vpor %xmm0, %xmm1, %xmm0
 ; CHECK-NEXT:    vmovd %xmm0, %eax
 ; CHECK-NEXT:    vzeroupper
@@ -7070,7 +7070,7 @@ define i32 @test_mm512_reduce_and_epi32(<8 x i64> %__W) {
 ; CHECK-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; CHECK-NEXT:    vpand %xmm0, %xmm1, %xmm0
-; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; CHECK-NEXT:    vpand %xmm0, %xmm1, %xmm0
 ; CHECK-NEXT:    vmovd %xmm0, %eax
 ; CHECK-NEXT:    vzeroupper
@@ -7103,7 +7103,7 @@ define i32 @test_mm512_mask_reduce_add_epi32(i16 zeroext %__M, <8 x i64> %__W) {
 ; X86-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; X86-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; X86-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
-; X86-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X86-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X86-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
 ; X86-NEXT:    vmovd %xmm0, %eax
 ; X86-NEXT:    vzeroupper
@@ -7119,7 +7119,7 @@ define i32 @test_mm512_mask_reduce_add_epi32(i16 zeroext %__M, <8 x i64> %__W) {
 ; X64-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; X64-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; X64-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
-; X64-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X64-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X64-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
 ; X64-NEXT:    vmovd %xmm0, %eax
 ; X64-NEXT:    vzeroupper
@@ -7161,7 +7161,7 @@ define i32 @test_mm512_mask_reduce_mul_epi32(i16 zeroext %__M, <8 x i64> %__W) {
 ; X86-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; X86-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; X86-NEXT:    vpmulld %xmm0, %xmm1, %xmm0
-; X86-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X86-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X86-NEXT:    vpmulld %xmm0, %xmm1, %xmm0
 ; X86-NEXT:    vmovd %xmm0, %eax
 ; X86-NEXT:    vzeroupper
@@ -7178,7 +7178,7 @@ define i32 @test_mm512_mask_reduce_mul_epi32(i16 zeroext %__M, <8 x i64> %__W) {
 ; X64-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; X64-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; X64-NEXT:    vpmulld %xmm0, %xmm1, %xmm0
-; X64-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X64-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X64-NEXT:    vpmulld %xmm0, %xmm1, %xmm0
 ; X64-NEXT:    vmovd %xmm0, %eax
 ; X64-NEXT:    vzeroupper
@@ -7220,7 +7220,7 @@ define i32 @test_mm512_mask_reduce_and_epi32(i16 zeroext %__M, <8 x i64> %__W) {
 ; X86-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; X86-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; X86-NEXT:    vpand %xmm0, %xmm1, %xmm0
-; X86-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X86-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X86-NEXT:    vpand %xmm0, %xmm1, %xmm0
 ; X86-NEXT:    vmovd %xmm0, %eax
 ; X86-NEXT:    vzeroupper
@@ -7237,7 +7237,7 @@ define i32 @test_mm512_mask_reduce_and_epi32(i16 zeroext %__M, <8 x i64> %__W) {
 ; X64-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; X64-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; X64-NEXT:    vpand %xmm0, %xmm1, %xmm0
-; X64-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X64-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X64-NEXT:    vpand %xmm0, %xmm1, %xmm0
 ; X64-NEXT:    vmovd %xmm0, %eax
 ; X64-NEXT:    vzeroupper
@@ -7274,7 +7274,7 @@ define i32 @test_mm512_mask_reduce_or_epi32(i16 zeroext %__M, <8 x i64> %__W) {
 ; X86-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; X86-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; X86-NEXT:    vpor %xmm0, %xmm1, %xmm0
-; X86-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X86-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X86-NEXT:    vpor %xmm0, %xmm1, %xmm0
 ; X86-NEXT:    vmovd %xmm0, %eax
 ; X86-NEXT:    vzeroupper
@@ -7290,7 +7290,7 @@ define i32 @test_mm512_mask_reduce_or_epi32(i16 zeroext %__M, <8 x i64> %__W) {
 ; X64-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; X64-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; X64-NEXT:    vpor %xmm0, %xmm1, %xmm0
-; X64-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X64-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X64-NEXT:    vpor %xmm0, %xmm1, %xmm0
 ; X64-NEXT:    vmovd %xmm0, %eax
 ; X64-NEXT:    vzeroupper
diff --git a/llvm/test/CodeGen/X86/avx512-shuffles/partial_permute.ll b/llvm/test/CodeGen/X86/avx512-shuffles/partial_permute.ll
index 24b4c322c54b8..050269c2ba2f3 100644
--- a/llvm/test/CodeGen/X86/avx512-shuffles/partial_permute.ll
+++ b/llvm/test/CodeGen/X86/avx512-shuffles/partial_permute.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --no_x86_scrub_mem_shuffle
 ; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512f,+avx512vl,+avx512bw,+fast-variable-shuffle %s -o - | FileCheck %s
 
 ; FIXME: All cases here should be fixed by PR34380
@@ -2143,11 +2143,11 @@ define <4 x i64> @test_masked_z_8xi64_to_4xi64_perm_mask6(<8 x i64> %vec, <4 x i
 define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mask7(<8 x i64> %vec, <4 x i64> %vec2, <4 x i64> %mask) {
 ; CHECK-LABEL: test_masked_8xi64_to_4xi64_perm_mask7:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vbroadcasti64x4 {{.*#+}} zmm3 = [2,0,3,4,2,0,3,4]
-; CHECK-NEXT:    # zmm3 = mem[0,1,2,3,0,1,2,3]
-; CHECK-NEXT:    vpermq %zmm0, %zmm3, %zmm0
+; CHECK-NEXT:    vextracti32x4 $2, %zmm0, %xmm3
+; CHECK-NEXT:    vmovdqa {{.*#+}} ymm4 = [2,0,3,4]
+; CHECK-NEXT:    vpermi2q %ymm3, %ymm0, %ymm4
 ; CHECK-NEXT:    vptestnmq %ymm2, %ymm2, %k1
-; CHECK-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
+; CHECK-NEXT:    vpblendmq %ymm4, %ymm1, %ymm0 {%k1}
 ; CHECK-NEXT:    retq
   %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 2, i32 0, i32 3, i32 4>
   %cmp = icmp eq <4 x i64> %mask, zeroinitializer
@@ -2158,10 +2158,11 @@ define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mask7(<8 x i64> %vec, <4 x i64
 define <4 x i64> @test_masked_z_8xi64_to_4xi64_perm_mask7(<8 x i64> %vec, <4 x i64> %mask) {
 ; CHECK-LABEL: test_masked_z_8xi64_to_4xi64_perm_mask7:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vextracti32x4 $2, %zmm0, %xmm3
 ; CHECK-NEXT:    vmovdqa {{.*#+}} ymm2 = [2,0,3,4]
 ; CHECK-NEXT:    vptestnmq %ymm1, %ymm1, %k1
-; CHECK-NEXT:    vpermq %zmm0, %zmm2, %zmm0 {%k1} {z}
-; CHECK-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; CHECK-NEXT:    vpermi2q %ymm3, %ymm0, %ymm2 {%k1} {z}
+; CHECK-NEXT:    vmovdqa %ymm2, %ymm0
 ; CHECK-NEXT:    retq
   %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 2, i32 0, i32 3, i32 4>
   %cmp = icmp eq <4 x i64> %mask, zeroinitializer
@@ -3426,7 +3427,7 @@ define <4 x float> @test_masked_16xfloat_to_4xfloat_perm_mem_mask2(<16 x float>*
 ; CHECK-LABEL: test_masked_16xfloat_to_4xfloat_perm_mem_mask2:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vmovaps 32(%rdi), %ymm2
-; CHECK-NEXT:    vmovaps {{.*#+}} ymm3 = <4,14,4,14,u,u,u,u>
+; CHECK-NEXT:    vbroadcastsd {{.*#+}} ymm3 = [60129542148,60129542148,60129542148,60129542148]
 ; CHECK-NEXT:    vpermi2ps (%rdi), %ymm2, %ymm3
 ; CHECK-NEXT:    vxorps %xmm2, %xmm2, %xmm2
 ; CHECK-NEXT:    vcmpeqps %xmm2, %xmm1, %k1
@@ -3444,7 +3445,7 @@ define <4 x float> @test_masked_z_16xfloat_to_4xfloat_perm_mem_mask2(<16 x float
 ; CHECK-LABEL: test_masked_z_16xfloat_to_4xfloat_perm_mem_mask2:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vmovaps 32(%rdi), %ymm2
-; CHECK-NEXT:    vmovaps {{.*#+}} ymm1 = <4,14,4,14,u,u,u,u>
+; CHECK-NEXT:    vbroadcastsd {{.*#+}} ymm1 = [60129542148,60129542148,60129542148,60129542148]
 ; CHECK-NEXT:    vxorps %xmm3, %xmm3, %xmm3
 ; CHECK-NEXT:    vcmpeqps %xmm3, %xmm0, %k1
 ; CHECK-NEXT:    vpermi2ps (%rdi), %ymm2, %ymm1 {%k1} {z}
diff --git a/llvm/test/CodeGen/X86/bitcast-int-to-vector-bool-sext.ll b/llvm/test/CodeGen/X86/bitcast-int-to-vector-bool-sext.ll
index f3805163b691a..75e3ab7639a95 100644
--- a/llvm/test/CodeGen/X86/bitcast-int-to-vector-bool-sext.ll
+++ b/llvm/test/CodeGen/X86/bitcast-int-to-vector-bool-sext.ll
@@ -94,7 +94,7 @@ define <8 x i16> @ext_i8_8i16(i8 %a0) {
 ; SSE2-SSSE3-LABEL: ext_i8_8i16:
 ; SSE2-SSSE3:       # %bb.0:
 ; SSE2-SSSE3-NEXT:    movd %edi, %xmm0
-; SSE2-SSSE3-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; SSE2-SSSE3-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; SSE2-SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [1,2,4,8,16,32,64,128]
 ; SSE2-SSSE3-NEXT:    pand %xmm1, %xmm0
@@ -104,7 +104,7 @@ define <8 x i16> @ext_i8_8i16(i8 %a0) {
 ; AVX1-LABEL: ext_i8_8i16:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vmovd %edi, %xmm0
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [1,2,4,8,16,32,64,128]
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -293,7 +293,7 @@ define <16 x i16> @ext_i16_16i16(i16 %a0) {
 ; SSE2-SSSE3-LABEL: ext_i16_16i16:
 ; SSE2-SSSE3:       # %bb.0:
 ; SSE2-SSSE3-NEXT:    movd %edi, %xmm0
-; SSE2-SSSE3-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; SSE2-SSSE3-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,0,0]
 ; SSE2-SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [1,2,4,8,16,32,64,128]
 ; SSE2-SSSE3-NEXT:    movdqa %xmm1, %xmm0
@@ -307,7 +307,7 @@ define <16 x i16> @ext_i16_16i16(i16 %a0) {
 ; AVX1-LABEL: ext_i16_16i16:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vmovd %edi, %xmm0
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
@@ -545,7 +545,7 @@ define <32 x i16> @ext_i32_32i16(i32 %a0) {
 ; SSE2-SSSE3-LABEL: ext_i32_32i16:
 ; SSE2-SSSE3:       # %bb.0:
 ; SSE2-SSSE3-NEXT:    movd %edi, %xmm2
-; SSE2-SSSE3-NEXT:    pshuflw {{.*#+}} xmm0 = xmm2[0,0,2,3,4,5,6,7]
+; SSE2-SSSE3-NEXT:    pshuflw {{.*#+}} xmm0 = xmm2[0,0,0,0,4,5,6,7]
 ; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,0,0]
 ; SSE2-SSSE3-NEXT:    movdqa {{.*#+}} xmm4 = [1,2,4,8,16,32,64,128]
 ; SSE2-SSSE3-NEXT:    movdqa %xmm1, %xmm0
@@ -554,7 +554,7 @@ define <32 x i16> @ext_i32_32i16(i32 %a0) {
 ; SSE2-SSSE3-NEXT:    movdqa {{.*#+}} xmm5 = [256,512,1024,2048,4096,8192,16384,32768]
 ; SSE2-SSSE3-NEXT:    pand %xmm5, %xmm1
 ; SSE2-SSSE3-NEXT:    pcmpeqw %xmm5, %xmm1
-; SSE2-SSSE3-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[1,1,2,3,4,5,6,7]
+; SSE2-SSSE3-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[1,1,1,1,4,5,6,7]
 ; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,0,0]
 ; SSE2-SSSE3-NEXT:    movdqa %xmm3, %xmm2
 ; SSE2-SSSE3-NEXT:    pand %xmm4, %xmm2
@@ -566,7 +566,7 @@ define <32 x i16> @ext_i32_32i16(i32 %a0) {
 ; AVX1-LABEL: ext_i32_32i16:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vmovd %edi, %xmm1
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm1[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm1[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; AVX1-NEXT:    vmovaps {{.*#+}} ymm2 = [1,2,4,8,16,32,64,128,256,512,1024,2048,4096,8192,16384,32768]
@@ -579,7 +579,7 @@ define <32 x i16> @ext_i32_32i16(i32 %a0) {
 ; AVX1-NEXT:    vpcmpeqw %xmm4, %xmm0, %xmm0
 ; AVX1-NEXT:    vpxor %xmm5, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[1,1,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[1,1,1,1,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm1, %ymm1
 ; AVX1-NEXT:    vandps %ymm2, %ymm1, %ymm1
diff --git a/llvm/test/CodeGen/X86/bitcast-int-to-vector-bool-zext.ll b/llvm/test/CodeGen/X86/bitcast-int-to-vector-bool-zext.ll
index 20135e8ca29b3..e7fcb50682464 100644
--- a/llvm/test/CodeGen/X86/bitcast-int-to-vector-bool-zext.ll
+++ b/llvm/test/CodeGen/X86/bitcast-int-to-vector-bool-zext.ll
@@ -119,7 +119,7 @@ define <8 x i16> @ext_i8_8i16(i8 %a0) {
 ; SSE2-SSSE3-LABEL: ext_i8_8i16:
 ; SSE2-SSSE3:       # %bb.0:
 ; SSE2-SSSE3-NEXT:    movd %edi, %xmm0
-; SSE2-SSSE3-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; SSE2-SSSE3-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; SSE2-SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [1,2,4,8,16,32,64,128]
 ; SSE2-SSSE3-NEXT:    pand %xmm1, %xmm0
@@ -130,7 +130,7 @@ define <8 x i16> @ext_i8_8i16(i8 %a0) {
 ; AVX1-LABEL: ext_i8_8i16:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vmovd %edi, %xmm0
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [1,2,4,8,16,32,64,128]
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -373,7 +373,7 @@ define <16 x i16> @ext_i16_16i16(i16 %a0) {
 ; SSE2-SSSE3-LABEL: ext_i16_16i16:
 ; SSE2-SSSE3:       # %bb.0:
 ; SSE2-SSSE3-NEXT:    movd %edi, %xmm0
-; SSE2-SSSE3-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; SSE2-SSSE3-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,0,0]
 ; SSE2-SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [1,2,4,8,16,32,64,128]
 ; SSE2-SSSE3-NEXT:    movdqa %xmm1, %xmm0
@@ -389,7 +389,7 @@ define <16 x i16> @ext_i16_16i16(i16 %a0) {
 ; AVX1-LABEL: ext_i16_16i16:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vmovd %edi, %xmm0
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
@@ -700,7 +700,7 @@ define <32 x i16> @ext_i32_32i16(i32 %a0) {
 ; SSE2-SSSE3-LABEL: ext_i32_32i16:
 ; SSE2-SSSE3:       # %bb.0:
 ; SSE2-SSSE3-NEXT:    movd %edi, %xmm2
-; SSE2-SSSE3-NEXT:    pshuflw {{.*#+}} xmm0 = xmm2[0,0,2,3,4,5,6,7]
+; SSE2-SSSE3-NEXT:    pshuflw {{.*#+}} xmm0 = xmm2[0,0,0,0,4,5,6,7]
 ; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,0,0]
 ; SSE2-SSSE3-NEXT:    movdqa {{.*#+}} xmm4 = [1,2,4,8,16,32,64,128]
 ; SSE2-SSSE3-NEXT:    movdqa %xmm1, %xmm0
@@ -711,7 +711,7 @@ define <32 x i16> @ext_i32_32i16(i32 %a0) {
 ; SSE2-SSSE3-NEXT:    pand %xmm5, %xmm1
 ; SSE2-SSSE3-NEXT:    pcmpeqw %xmm5, %xmm1
 ; SSE2-SSSE3-NEXT:    psrlw $15, %xmm1
-; SSE2-SSSE3-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[1,1,2,3,4,5,6,7]
+; SSE2-SSSE3-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[1,1,1,1,4,5,6,7]
 ; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,0,0]
 ; SSE2-SSSE3-NEXT:    movdqa %xmm3, %xmm2
 ; SSE2-SSSE3-NEXT:    pand %xmm4, %xmm2
@@ -725,7 +725,7 @@ define <32 x i16> @ext_i32_32i16(i32 %a0) {
 ; AVX1-LABEL: ext_i32_32i16:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vmovd %edi, %xmm1
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm1[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm1[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; AVX1-NEXT:    vmovaps {{.*#+}} ymm2 = [1,2,4,8,16,32,64,128,256,512,1024,2048,4096,8192,16384,32768]
@@ -740,7 +740,7 @@ define <32 x i16> @ext_i32_32i16(i32 %a0) {
 ; AVX1-NEXT:    vpxor %xmm5, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrlw $15, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm4, %ymm0
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[1,1,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[1,1,1,1,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm1, %ymm1
 ; AVX1-NEXT:    vandps %ymm2, %ymm1, %ymm1
diff --git a/llvm/test/CodeGen/X86/bitcast-int-to-vector-bool.ll b/llvm/test/CodeGen/X86/bitcast-int-to-vector-bool.ll
index ac2fd448452d4..65bf43ff2ed9e 100644
--- a/llvm/test/CodeGen/X86/bitcast-int-to-vector-bool.ll
+++ b/llvm/test/CodeGen/X86/bitcast-int-to-vector-bool.ll
@@ -93,7 +93,7 @@ define <8 x i1> @bitcast_i8_8i1(i8 zeroext %a0) {
 ; SSE2-SSSE3-LABEL: bitcast_i8_8i1:
 ; SSE2-SSSE3:       # %bb.0:
 ; SSE2-SSSE3-NEXT:    movd %edi, %xmm0
-; SSE2-SSSE3-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; SSE2-SSSE3-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; SSE2-SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [1,2,4,8,16,32,64,128]
 ; SSE2-SSSE3-NEXT:    pand %xmm1, %xmm0
@@ -104,7 +104,7 @@ define <8 x i1> @bitcast_i8_8i1(i8 zeroext %a0) {
 ; AVX1-LABEL: bitcast_i8_8i1:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vmovd %edi, %xmm0
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [1,2,4,8,16,32,64,128]
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
diff --git a/llvm/test/CodeGen/X86/bitcast-int-to-vector.ll b/llvm/test/CodeGen/X86/bitcast-int-to-vector.ll
index e319255e8f0c3..995532cb5b9d7 100644
--- a/llvm/test/CodeGen/X86/bitcast-int-to-vector.ll
+++ b/llvm/test/CodeGen/X86/bitcast-int-to-vector.ll
@@ -19,7 +19,7 @@ define i1 @foo(i64 %a) {
 ; X86-SSE:       # %bb.0:
 ; X86-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; X86-SSE-NEXT:    movaps %xmm0, %xmm1
-; X86-SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[2,3]
+; X86-SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[1,1]
 ; X86-SSE-NEXT:    ucomiss %xmm1, %xmm0
 ; X86-SSE-NEXT:    setp %al
 ; X86-SSE-NEXT:    retl
@@ -27,7 +27,7 @@ define i1 @foo(i64 %a) {
 ; X64-LABEL: foo:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movq %rdi, %xmm0
-; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X64-NEXT:    ucomiss %xmm1, %xmm0
 ; X64-NEXT:    setp %al
 ; X64-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/bitcast-vector-bool.ll b/llvm/test/CodeGen/X86/bitcast-vector-bool.ll
index faccb8158d322..7c61f1cac2d8d 100644
--- a/llvm/test/CodeGen/X86/bitcast-vector-bool.ll
+++ b/llvm/test/CodeGen/X86/bitcast-vector-bool.ll
@@ -515,7 +515,7 @@ define i32 @bitcast_v64i8_to_v2i32(<64 x i8> %a0) nounwind {
 ; SSE2-SSSE3-NEXT:    shlq $32, %rdx
 ; SSE2-SSSE3-NEXT:    orq %rcx, %rdx
 ; SSE2-SSSE3-NEXT:    movq %rdx, %xmm0
-; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE2-SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE2-SSSE3-NEXT:    movd %xmm0, %eax
 ; SSE2-SSSE3-NEXT:    addl %ecx, %eax
 ; SSE2-SSSE3-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/buildvec-extract.ll b/llvm/test/CodeGen/X86/buildvec-extract.ll
index 2195526f94c39..19f9155d851a6 100644
--- a/llvm/test/CodeGen/X86/buildvec-extract.ll
+++ b/llvm/test/CodeGen/X86/buildvec-extract.ll
@@ -69,21 +69,21 @@ define <2 x i64> @extract1_i32_zext_insert0_i64_undef(<4 x i32> %x) {
 define <2 x i64> @extract1_i32_zext_insert0_i64_zero(<4 x i32> %x) {
 ; SSE2-LABEL: extract1_i32_zext_insert0_i64_zero:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
 ; SSE2-NEXT:    movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: extract1_i32_zext_insert0_i64_zero:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
 ; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3,4,5,6,7]
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: extract1_i32_zext_insert0_i64_zero:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
 ; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    retq
@@ -242,7 +242,7 @@ define <2 x i64> @extract1_i32_zext_insert1_i64_undef(<4 x i32> %x) {
 define <2 x i64> @extract1_i32_zext_insert1_i64_zero(<4 x i32> %x) {
 ; SSE2-LABEL: extract1_i32_zext_insert1_i64_zero:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    movd %xmm0, %eax
 ; SSE2-NEXT:    movq %rax, %xmm0
 ; SSE2-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7]
@@ -332,7 +332,7 @@ define <2 x i64> @extract3_i32_zext_insert1_i64_undef(<4 x i32> %x) {
 define <2 x i64> @extract3_i32_zext_insert1_i64_zero(<4 x i32> %x) {
 ; SSE2-LABEL: extract3_i32_zext_insert1_i64_zero:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE2-NEXT:    movd %xmm0, %eax
 ; SSE2-NEXT:    movq %rax, %xmm0
 ; SSE2-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7]
diff --git a/llvm/test/CodeGen/X86/buildvec-insertvec.ll b/llvm/test/CodeGen/X86/buildvec-insertvec.ll
index e428ae8d59194..8758779d97c52 100644
--- a/llvm/test/CodeGen/X86/buildvec-insertvec.ll
+++ b/llvm/test/CodeGen/X86/buildvec-insertvec.ll
@@ -792,9 +792,9 @@ define i32 @PR46586(i8* %p, <4 x i32> %v) {
 ; SSE2-NEXT:    movzbl 3(%rdi), %eax
 ; SSE2-NEXT:    pxor %xmm1, %xmm1
 ; SSE2-NEXT:    pinsrw $6, %eax, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; SSE2-NEXT:    movd %xmm1, %eax
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE2-NEXT:    movd %xmm0, %ecx
 ; SSE2-NEXT:    xorl %edx, %edx
 ; SSE2-NEXT:    divl %ecx
diff --git a/llvm/test/CodeGen/X86/cast-vsel.ll b/llvm/test/CodeGen/X86/cast-vsel.ll
index 6e9e4fd00636d..c2e55cf458b59 100644
--- a/llvm/test/CodeGen/X86/cast-vsel.ll
+++ b/llvm/test/CodeGen/X86/cast-vsel.ll
@@ -411,10 +411,10 @@ define void @example24(i16 signext %x, i16 signext %y) nounwind {
 ; SSE2-LABEL: example24:
 ; SSE2:       # %bb.0: # %vector.ph
 ; SSE2-NEXT:    movd %edi, %xmm0
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; SSE2-NEXT:    movd %esi, %xmm1
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,0,2,3,4,5,6,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,0,0,0,4,5,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; SSE2-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; SSE2-NEXT:    .p2align 4, 0x90
@@ -443,10 +443,10 @@ define void @example24(i16 signext %x, i16 signext %y) nounwind {
 ; SSE41-LABEL: example24:
 ; SSE41:       # %bb.0: # %vector.ph
 ; SSE41-NEXT:    movd %edi, %xmm0
-; SSE41-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; SSE41-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,0,0]
 ; SSE41-NEXT:    movd %esi, %xmm0
-; SSE41-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; SSE41-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,0,0]
 ; SSE41-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; SSE41-NEXT:    .p2align 4, 0x90
@@ -472,10 +472,10 @@ define void @example24(i16 signext %x, i16 signext %y) nounwind {
 ; AVX1-LABEL: example24:
 ; AVX1:       # %bb.0: # %vector.ph
 ; AVX1-NEXT:    vmovd %edi, %xmm0
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    vmovd %esi, %xmm0
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; AVX1-NEXT:    vpmovsxwd %xmm2, %xmm0
diff --git a/llvm/test/CodeGen/X86/combine-fcopysign.ll b/llvm/test/CodeGen/X86/combine-fcopysign.ll
index 05c546c411f5b..6ed5fb5f49b6e 100644
--- a/llvm/test/CodeGen/X86/combine-fcopysign.ll
+++ b/llvm/test/CodeGen/X86/combine-fcopysign.ll
@@ -198,7 +198,7 @@ define <4 x double> @combine_vec_fcopysign_fpext_sgn(<4 x double> %x, <4 x float
 ; SSE-NEXT:    movshdup {{.*#+}} xmm4 = xmm2[1,1,3,3]
 ; SSE-NEXT:    cvtss2sd %xmm2, %xmm5
 ; SSE-NEXT:    movhlps {{.*#+}} xmm2 = xmm2[1,1]
-; SSE-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,1,2,3]
+; SSE-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,3,3,3]
 ; SSE-NEXT:    movaps {{.*#+}} xmm6 = [NaN,NaN]
 ; SSE-NEXT:    cvtss2sd %xmm3, %xmm3
 ; SSE-NEXT:    movaps %xmm6, %xmm7
@@ -267,7 +267,7 @@ define <4 x float> @combine_vec_fcopysign_fptrunc_sgn(<4 x float> %x, <4 x doubl
 ; SSE-NEXT:    orps %xmm5, %xmm1
 ; SSE-NEXT:    unpcklps {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
 ; SSE-NEXT:    insertps {{.*#+}} xmm3 = xmm3[0,1],xmm6[0],xmm3[3]
-; SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE-NEXT:    andps %xmm4, %xmm0
 ; SSE-NEXT:    movhlps {{.*#+}} xmm2 = xmm2[1,1]
 ; SSE-NEXT:    xorps %xmm1, %xmm1
diff --git a/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll b/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll
index 59101503b5a93..789487a12a410 100644
--- a/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll
+++ b/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll
@@ -558,9 +558,9 @@ define <4 x i32> @vector_i128_i32(<4 x i32> %x, <4 x i32> %y, <4 x i32>* %divdst
 ; X86:       # %bb.0:
 ; X86-NEXT:    pushl %esi
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,1,2,3]
+; X86-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
 ; X86-NEXT:    movd %xmm2, %eax
-; X86-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,1,2,3]
+; X86-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
 ; X86-NEXT:    movd %xmm2, %esi
 ; X86-NEXT:    cltd
 ; X86-NEXT:    idivl %esi
@@ -578,9 +578,9 @@ define <4 x i32> @vector_i128_i32(<4 x i32> %x, <4 x i32> %y, <4 x i32>* %divdst
 ; X86-NEXT:    cltd
 ; X86-NEXT:    idivl %esi
 ; X86-NEXT:    movd %eax, %xmm2
-; X86-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,2,3]
+; X86-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,1,1]
 ; X86-NEXT:    movd %xmm4, %eax
-; X86-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,2,3]
+; X86-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,1,1]
 ; X86-NEXT:    movd %xmm4, %esi
 ; X86-NEXT:    cltd
 ; X86-NEXT:    idivl %esi
@@ -601,9 +601,9 @@ define <4 x i32> @vector_i128_i32(<4 x i32> %x, <4 x i32> %y, <4 x i32>* %divdst
 ;
 ; X64-LABEL: vector_i128_i32:
 ; X64:       # %bb.0:
-; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,1,2,3]
+; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
 ; X64-NEXT:    movd %xmm2, %eax
-; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,1,2,3]
+; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
 ; X64-NEXT:    movd %xmm2, %ecx
 ; X64-NEXT:    cltd
 ; X64-NEXT:    idivl %ecx
@@ -621,9 +621,9 @@ define <4 x i32> @vector_i128_i32(<4 x i32> %x, <4 x i32> %y, <4 x i32>* %divdst
 ; X64-NEXT:    cltd
 ; X64-NEXT:    idivl %ecx
 ; X64-NEXT:    movd %eax, %xmm2
-; X64-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,2,3]
+; X64-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,1,1]
 ; X64-NEXT:    movd %xmm4, %eax
-; X64-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,2,3]
+; X64-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,1,1]
 ; X64-NEXT:    movd %xmm4, %ecx
 ; X64-NEXT:    cltd
 ; X64-NEXT:    idivl %ecx
@@ -655,21 +655,21 @@ define <2 x i64> @vector_i128_i64(<2 x i64> %x, <2 x i64> %y, <2 x i64>* %divdst
 ; X86-NEXT:    movdqu %xmm1, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
 ; X86-NEXT:    movdqu %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,1,2,3]
+; X86-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
 ; X86-NEXT:    movd %xmm2, {{[0-9]+}}(%esp)
 ; X86-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; X86-NEXT:    movd %xmm2, {{[0-9]+}}(%esp)
-; X86-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; X86-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
 ; X86-NEXT:    movd %xmm1, {{[0-9]+}}(%esp)
 ; X86-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-NEXT:    movd %xmm1, (%esp)
 ; X86-NEXT:    calll __divdi3
 ; X86-NEXT:    movdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm1 # 16-byte Reload
-; X86-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X86-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X86-NEXT:    movd %xmm0, {{[0-9]+}}(%esp)
 ; X86-NEXT:    movd %xmm1, {{[0-9]+}}(%esp)
 ; X86-NEXT:    movdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm1 # 16-byte Reload
-; X86-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X86-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X86-NEXT:    movd %xmm0, {{[0-9]+}}(%esp)
 ; X86-NEXT:    movd %xmm1, (%esp)
 ; X86-NEXT:    movd %edx, %xmm0
diff --git a/llvm/test/CodeGen/X86/div-rem-pair-recomposition-unsigned.ll b/llvm/test/CodeGen/X86/div-rem-pair-recomposition-unsigned.ll
index d787f91ababba..584dd01b8239d 100644
--- a/llvm/test/CodeGen/X86/div-rem-pair-recomposition-unsigned.ll
+++ b/llvm/test/CodeGen/X86/div-rem-pair-recomposition-unsigned.ll
@@ -558,9 +558,9 @@ define <4 x i32> @vector_i128_i32(<4 x i32> %x, <4 x i32> %y, <4 x i32>* %divdst
 ; X86:       # %bb.0:
 ; X86-NEXT:    pushl %esi
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,1,2,3]
+; X86-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
 ; X86-NEXT:    movd %xmm2, %eax
-; X86-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,1,2,3]
+; X86-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
 ; X86-NEXT:    movd %xmm2, %esi
 ; X86-NEXT:    xorl %edx, %edx
 ; X86-NEXT:    divl %esi
@@ -578,9 +578,9 @@ define <4 x i32> @vector_i128_i32(<4 x i32> %x, <4 x i32> %y, <4 x i32>* %divdst
 ; X86-NEXT:    xorl %edx, %edx
 ; X86-NEXT:    divl %esi
 ; X86-NEXT:    movd %eax, %xmm2
-; X86-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,2,3]
+; X86-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,1,1]
 ; X86-NEXT:    movd %xmm4, %eax
-; X86-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,2,3]
+; X86-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,1,1]
 ; X86-NEXT:    movd %xmm4, %esi
 ; X86-NEXT:    xorl %edx, %edx
 ; X86-NEXT:    divl %esi
@@ -601,9 +601,9 @@ define <4 x i32> @vector_i128_i32(<4 x i32> %x, <4 x i32> %y, <4 x i32>* %divdst
 ;
 ; X64-LABEL: vector_i128_i32:
 ; X64:       # %bb.0:
-; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,1,2,3]
+; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
 ; X64-NEXT:    movd %xmm2, %eax
-; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,1,2,3]
+; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
 ; X64-NEXT:    movd %xmm2, %ecx
 ; X64-NEXT:    xorl %edx, %edx
 ; X64-NEXT:    divl %ecx
@@ -621,9 +621,9 @@ define <4 x i32> @vector_i128_i32(<4 x i32> %x, <4 x i32> %y, <4 x i32>* %divdst
 ; X64-NEXT:    xorl %edx, %edx
 ; X64-NEXT:    divl %ecx
 ; X64-NEXT:    movd %eax, %xmm2
-; X64-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,2,3]
+; X64-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,1,1]
 ; X64-NEXT:    movd %xmm4, %eax
-; X64-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,2,3]
+; X64-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,1,1]
 ; X64-NEXT:    movd %xmm4, %ecx
 ; X64-NEXT:    xorl %edx, %edx
 ; X64-NEXT:    divl %ecx
@@ -655,21 +655,21 @@ define <2 x i64> @vector_i128_i64(<2 x i64> %x, <2 x i64> %y, <2 x i64>* %divdst
 ; X86-NEXT:    movdqu %xmm1, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
 ; X86-NEXT:    movdqu %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,1,2,3]
+; X86-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
 ; X86-NEXT:    movd %xmm2, {{[0-9]+}}(%esp)
 ; X86-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; X86-NEXT:    movd %xmm2, {{[0-9]+}}(%esp)
-; X86-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; X86-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
 ; X86-NEXT:    movd %xmm1, {{[0-9]+}}(%esp)
 ; X86-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-NEXT:    movd %xmm1, (%esp)
 ; X86-NEXT:    calll __udivdi3
 ; X86-NEXT:    movdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm1 # 16-byte Reload
-; X86-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X86-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X86-NEXT:    movd %xmm0, {{[0-9]+}}(%esp)
 ; X86-NEXT:    movd %xmm1, {{[0-9]+}}(%esp)
 ; X86-NEXT:    movdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm1 # 16-byte Reload
-; X86-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X86-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X86-NEXT:    movd %xmm0, {{[0-9]+}}(%esp)
 ; X86-NEXT:    movd %xmm1, (%esp)
 ; X86-NEXT:    movd %edx, %xmm0
diff --git a/llvm/test/CodeGen/X86/extract-fp.ll b/llvm/test/CodeGen/X86/extract-fp.ll
index 27430efa7822c..6281f12979f1e 100644
--- a/llvm/test/CodeGen/X86/extract-fp.ll
+++ b/llvm/test/CodeGen/X86/extract-fp.ll
@@ -15,7 +15,7 @@ define float @ext_fadd_v4f32(<4 x float> %x) {
 define float @ext_fsub_v4f32(<4 x float> %x) {
 ; CHECK-LABEL: ext_fsub_v4f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; CHECK-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; CHECK-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; CHECK-NEXT:    subss %xmm0, %xmm1
 ; CHECK-NEXT:    movaps %xmm1, %xmm0
@@ -28,7 +28,7 @@ define float @ext_fsub_v4f32(<4 x float> %x) {
 define float @ext_fmul_v4f32(<4 x float> %x) {
 ; CHECK-LABEL: ext_fmul_v4f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; CHECK-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; CHECK-NEXT:    mulss {{.*}}(%rip), %xmm0
 ; CHECK-NEXT:    retq
   %bo = fmul <4 x float> %x, <float 1.0, float 2.0, float 3.0, float 42.0>
@@ -50,7 +50,7 @@ define float @ext_fdiv_v4f32(<4 x float> %x) {
 define float @ext_fdiv_v4f32_constant_op0(<4 x float> %x) {
 ; CHECK-LABEL: ext_fdiv_v4f32_constant_op0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; CHECK-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; CHECK-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; CHECK-NEXT:    divss %xmm0, %xmm1
 ; CHECK-NEXT:    movaps %xmm1, %xmm0
@@ -75,7 +75,7 @@ define float @ext_frem_v4f32_constant_op0(<4 x float> %x) {
 ; CHECK-LABEL: ext_frem_v4f32_constant_op0:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movaps %xmm0, %xmm1
-; CHECK-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[2,3]
+; CHECK-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[1,1]
 ; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; CHECK-NEXT:    jmp fmodf # TAILCALL
   %bo = frem <4 x float> <float 1.0, float 2.0, float 3.0, float 42.0>, %x
diff --git a/llvm/test/CodeGen/X86/extract-store.ll b/llvm/test/CodeGen/X86/extract-store.ll
index d50c2ed920753..1160c6e5d21a9 100644
--- a/llvm/test/CodeGen/X86/extract-store.ll
+++ b/llvm/test/CodeGen/X86/extract-store.ll
@@ -241,13 +241,13 @@ define void @extract_i32_3(i32* nocapture %dst, <4 x i32> %foo) nounwind {
 ; SSE2-X32-LABEL: extract_i32_3:
 ; SSE2-X32:       # %bb.0:
 ; SSE2-X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; SSE2-X32-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-X32-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE2-X32-NEXT:    movd %xmm0, (%eax)
 ; SSE2-X32-NEXT:    retl
 ;
 ; SSE2-X64-LABEL: extract_i32_3:
 ; SSE2-X64:       # %bb.0:
-; SSE2-X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE2-X64-NEXT:    movd %xmm0, (%rdi)
 ; SSE2-X64-NEXT:    retq
 ;
@@ -370,13 +370,13 @@ define void @extract_f32_3(float* nocapture %dst, <4 x float> %foo) nounwind {
 ; SSE2-X32-LABEL: extract_f32_3:
 ; SSE2-X32:       # %bb.0:
 ; SSE2-X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; SSE2-X32-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-X32-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE2-X32-NEXT:    movss %xmm0, (%eax)
 ; SSE2-X32-NEXT:    retl
 ;
 ; SSE2-X64-LABEL: extract_f32_3:
 ; SSE2-X64:       # %bb.0:
-; SSE2-X64-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-X64-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE2-X64-NEXT:    movss %xmm0, (%rdi)
 ; SSE2-X64-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/extractelement-index.ll b/llvm/test/CodeGen/X86/extractelement-index.ll
index f2e01e93361ef..99297b435b2a9 100644
--- a/llvm/test/CodeGen/X86/extractelement-index.ll
+++ b/llvm/test/CodeGen/X86/extractelement-index.ll
@@ -249,7 +249,7 @@ define i32 @extractelement_v4i32_0(<4 x i32> %a) nounwind {
 define i32 @extractelement_v4i32_3(<4 x i32> %a) nounwind {
 ; SSE2-LABEL: extractelement_v4i32_3:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE2-NEXT:    movd %xmm0, %eax
 ; SSE2-NEXT:    retq
 ;
@@ -315,7 +315,7 @@ define i32 @extractelement_v8i32_4(<8 x i32> %a) nounwind {
 define i32 @extractelement_v8i32_7(<8 x i32> %a) nounwind {
 ; SSE2-LABEL: extractelement_v8i32_7:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[3,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[3,3,3,3]
 ; SSE2-NEXT:    movd %xmm0, %eax
 ; SSE2-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/extractelement-load.ll b/llvm/test/CodeGen/X86/extractelement-load.ll
index b694859b757c0..9d3e5a88b0376 100644
--- a/llvm/test/CodeGen/X86/extractelement-load.ll
+++ b/llvm/test/CodeGen/X86/extractelement-load.ll
@@ -86,7 +86,7 @@ define i64 @t4(<2 x double>* %a) {
 ; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-SSE2-NEXT:    movdqa (%eax), %xmm0
 ; X32-SSE2-NEXT:    movd %xmm0, %eax
-; X32-SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,0,1]
+; X32-SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; X32-SSE2-NEXT:    movd %xmm0, %edx
 ; X32-SSE2-NEXT:    retl
 ;
@@ -136,7 +136,7 @@ define float @t6(<8 x float> *%a0) {
 ; X32-SSE2-NEXT:    .cfi_def_cfa_offset 8
 ; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-SSE2-NEXT:    movaps (%eax), %xmm0
-; X32-SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; X32-SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; X32-SSE2-NEXT:    xorps %xmm1, %xmm1
 ; X32-SSE2-NEXT:    cmpeqss %xmm0, %xmm1
 ; X32-SSE2-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
@@ -228,7 +228,7 @@ define float @PR43971_1(<8 x float> *%a0) nounwind {
 ; X32-SSE2-NEXT:    pushl %eax
 ; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-SSE2-NEXT:    movaps (%eax), %xmm0
-; X32-SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; X32-SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; X32-SSE2-NEXT:    xorps %xmm1, %xmm1
 ; X32-SSE2-NEXT:    cmpeqss %xmm0, %xmm1
 ; X32-SSE2-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
diff --git a/llvm/test/CodeGen/X86/fma.ll b/llvm/test/CodeGen/X86/fma.ll
index 91ba1c8891409..414a0edf41b76 100644
--- a/llvm/test/CodeGen/X86/fma.ll
+++ b/llvm/test/CodeGen/X86/fma.ll
@@ -315,12 +315,12 @@ define <4 x float> @test_v4f32(<4 x float> %a, <4 x float> %b, <4 x float> %c) #
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x29,0x4c,0x24,0x10]
 ; FMACALL64-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x29,0x44,0x24,0x20]
-; FMACALL64-NEXT:    shufps $231, %xmm0, %xmm0 ## encoding: [0x0f,0xc6,0xc0,0xe7]
-; FMACALL64-NEXT:    ## xmm0 = xmm0[3,1,2,3]
-; FMACALL64-NEXT:    shufps $231, %xmm1, %xmm1 ## encoding: [0x0f,0xc6,0xc9,0xe7]
-; FMACALL64-NEXT:    ## xmm1 = xmm1[3,1,2,3]
-; FMACALL64-NEXT:    shufps $231, %xmm2, %xmm2 ## encoding: [0x0f,0xc6,0xd2,0xe7]
-; FMACALL64-NEXT:    ## xmm2 = xmm2[3,1,2,3]
+; FMACALL64-NEXT:    shufps $255, %xmm0, %xmm0 ## encoding: [0x0f,0xc6,0xc0,0xff]
+; FMACALL64-NEXT:    ## xmm0 = xmm0[3,3,3,3]
+; FMACALL64-NEXT:    shufps $255, %xmm1, %xmm1 ## encoding: [0x0f,0xc6,0xc9,0xff]
+; FMACALL64-NEXT:    ## xmm1 = xmm1[3,3,3,3]
+; FMACALL64-NEXT:    shufps $255, %xmm2, %xmm2 ## encoding: [0x0f,0xc6,0xd2,0xff]
+; FMACALL64-NEXT:    ## xmm2 = xmm2[3,3,3,3]
 ; FMACALL64-NEXT:    callq _fmaf ## encoding: [0xe8,A,A,A,A]
 ; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: reloc_branch_4byte_pcrel
 ; FMACALL64-NEXT:    movaps %xmm0, (%rsp) ## 16-byte Spill
@@ -356,16 +356,16 @@ define <4 x float> @test_v4f32(<4 x float> %a, <4 x float> %b, <4 x float> %c) #
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x29,0x44,0x24,0x40]
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x28,0x44,0x24,0x20]
-; FMACALL64-NEXT:    shufps $229, %xmm0, %xmm0 ## encoding: [0x0f,0xc6,0xc0,0xe5]
-; FMACALL64-NEXT:    ## xmm0 = xmm0[1,1,2,3]
+; FMACALL64-NEXT:    shufps $85, %xmm0, %xmm0 ## encoding: [0x0f,0xc6,0xc0,0x55]
+; FMACALL64-NEXT:    ## xmm0 = xmm0[1,1,1,1]
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x28,0x4c,0x24,0x10]
-; FMACALL64-NEXT:    shufps $229, %xmm1, %xmm1 ## encoding: [0x0f,0xc6,0xc9,0xe5]
-; FMACALL64-NEXT:    ## xmm1 = xmm1[1,1,2,3]
+; FMACALL64-NEXT:    shufps $85, %xmm1, %xmm1 ## encoding: [0x0f,0xc6,0xc9,0x55]
+; FMACALL64-NEXT:    ## xmm1 = xmm1[1,1,1,1]
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 ## 16-byte Reload
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x28,0x54,0x24,0x30]
-; FMACALL64-NEXT:    shufps $229, %xmm2, %xmm2 ## encoding: [0x0f,0xc6,0xd2,0xe5]
-; FMACALL64-NEXT:    ## xmm2 = xmm2[1,1,2,3]
+; FMACALL64-NEXT:    shufps $85, %xmm2, %xmm2 ## encoding: [0x0f,0xc6,0xd2,0x55]
+; FMACALL64-NEXT:    ## xmm2 = xmm2[1,1,1,1]
 ; FMACALL64-NEXT:    callq _fmaf ## encoding: [0xe8,A,A,A,A]
 ; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: reloc_branch_4byte_pcrel
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
@@ -493,14 +493,14 @@ define <8 x float> @test_v8f32(<8 x float> %a, <8 x float> %b, <8 x float> %c) #
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x29,0x4c,0x24,0x30]
 ; FMACALL64-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x29,0x44,0x24,0x10]
-; FMACALL64-NEXT:    shufps $231, %xmm0, %xmm0 ## encoding: [0x0f,0xc6,0xc0,0xe7]
-; FMACALL64-NEXT:    ## xmm0 = xmm0[3,1,2,3]
+; FMACALL64-NEXT:    shufps $255, %xmm0, %xmm0 ## encoding: [0x0f,0xc6,0xc0,0xff]
+; FMACALL64-NEXT:    ## xmm0 = xmm0[3,3,3,3]
 ; FMACALL64-NEXT:    movaps %xmm2, %xmm1 ## encoding: [0x0f,0x28,0xca]
-; FMACALL64-NEXT:    shufps $231, %xmm2, %xmm1 ## encoding: [0x0f,0xc6,0xca,0xe7]
-; FMACALL64-NEXT:    ## xmm1 = xmm1[3,1],xmm2[2,3]
+; FMACALL64-NEXT:    shufps $255, %xmm2, %xmm1 ## encoding: [0x0f,0xc6,0xca,0xff]
+; FMACALL64-NEXT:    ## xmm1 = xmm1[3,3],xmm2[3,3]
 ; FMACALL64-NEXT:    movaps %xmm4, %xmm2 ## encoding: [0x0f,0x28,0xd4]
-; FMACALL64-NEXT:    shufps $231, %xmm4, %xmm2 ## encoding: [0x0f,0xc6,0xd4,0xe7]
-; FMACALL64-NEXT:    ## xmm2 = xmm2[3,1],xmm4[2,3]
+; FMACALL64-NEXT:    shufps $255, %xmm4, %xmm2 ## encoding: [0x0f,0xc6,0xd4,0xff]
+; FMACALL64-NEXT:    ## xmm2 = xmm2[3,3],xmm4[3,3]
 ; FMACALL64-NEXT:    callq _fmaf ## encoding: [0xe8,A,A,A,A]
 ; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: reloc_branch_4byte_pcrel
 ; FMACALL64-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
@@ -536,16 +536,16 @@ define <8 x float> @test_v8f32(<8 x float> %a, <8 x float> %b, <8 x float> %c) #
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x29,0x44,0x24,0x20]
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x28,0x44,0x24,0x10]
-; FMACALL64-NEXT:    shufps $229, %xmm0, %xmm0 ## encoding: [0x0f,0xc6,0xc0,0xe5]
-; FMACALL64-NEXT:    ## xmm0 = xmm0[1,1,2,3]
+; FMACALL64-NEXT:    shufps $85, %xmm0, %xmm0 ## encoding: [0x0f,0xc6,0xc0,0x55]
+; FMACALL64-NEXT:    ## xmm0 = xmm0[1,1,1,1]
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x28,0x4c,0x24,0x60]
-; FMACALL64-NEXT:    shufps $229, %xmm1, %xmm1 ## encoding: [0x0f,0xc6,0xc9,0xe5]
-; FMACALL64-NEXT:    ## xmm1 = xmm1[1,1,2,3]
+; FMACALL64-NEXT:    shufps $85, %xmm1, %xmm1 ## encoding: [0x0f,0xc6,0xc9,0x55]
+; FMACALL64-NEXT:    ## xmm1 = xmm1[1,1,1,1]
 ; FMACALL64-NEXT:    movaps (%rsp), %xmm2 ## 16-byte Reload
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x28,0x14,0x24]
-; FMACALL64-NEXT:    shufps $229, %xmm2, %xmm2 ## encoding: [0x0f,0xc6,0xd2,0xe5]
-; FMACALL64-NEXT:    ## xmm2 = xmm2[1,1,2,3]
+; FMACALL64-NEXT:    shufps $85, %xmm2, %xmm2 ## encoding: [0x0f,0xc6,0xd2,0x55]
+; FMACALL64-NEXT:    ## xmm2 = xmm2[1,1,1,1]
 ; FMACALL64-NEXT:    callq _fmaf ## encoding: [0xe8,A,A,A,A]
 ; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: reloc_branch_4byte_pcrel
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
@@ -559,16 +559,16 @@ define <8 x float> @test_v8f32(<8 x float> %a, <8 x float> %b, <8 x float> %c) #
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x29,0x4c,0x24,0x20]
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x28,0x44,0x24,0x30]
-; FMACALL64-NEXT:    shufps $231, %xmm0, %xmm0 ## encoding: [0x0f,0xc6,0xc0,0xe7]
-; FMACALL64-NEXT:    ## xmm0 = xmm0[3,1,2,3]
+; FMACALL64-NEXT:    shufps $255, %xmm0, %xmm0 ## encoding: [0x0f,0xc6,0xc0,0xff]
+; FMACALL64-NEXT:    ## xmm0 = xmm0[3,3,3,3]
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x28,0x4c,0x24,0x40]
-; FMACALL64-NEXT:    shufps $231, %xmm1, %xmm1 ## encoding: [0x0f,0xc6,0xc9,0xe7]
-; FMACALL64-NEXT:    ## xmm1 = xmm1[3,1,2,3]
+; FMACALL64-NEXT:    shufps $255, %xmm1, %xmm1 ## encoding: [0x0f,0xc6,0xc9,0xff]
+; FMACALL64-NEXT:    ## xmm1 = xmm1[3,3,3,3]
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 ## 16-byte Reload
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x28,0x54,0x24,0x50]
-; FMACALL64-NEXT:    shufps $231, %xmm2, %xmm2 ## encoding: [0x0f,0xc6,0xd2,0xe7]
-; FMACALL64-NEXT:    ## xmm2 = xmm2[3,1,2,3]
+; FMACALL64-NEXT:    shufps $255, %xmm2, %xmm2 ## encoding: [0x0f,0xc6,0xd2,0xff]
+; FMACALL64-NEXT:    ## xmm2 = xmm2[3,3,3,3]
 ; FMACALL64-NEXT:    callq _fmaf ## encoding: [0xe8,A,A,A,A]
 ; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: reloc_branch_4byte_pcrel
 ; FMACALL64-NEXT:    movaps %xmm0, (%rsp) ## 16-byte Spill
@@ -604,16 +604,16 @@ define <8 x float> @test_v8f32(<8 x float> %a, <8 x float> %b, <8 x float> %c) #
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x29,0x44,0x24,0x10]
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x28,0x44,0x24,0x30]
-; FMACALL64-NEXT:    shufps $229, %xmm0, %xmm0 ## encoding: [0x0f,0xc6,0xc0,0xe5]
-; FMACALL64-NEXT:    ## xmm0 = xmm0[1,1,2,3]
+; FMACALL64-NEXT:    shufps $85, %xmm0, %xmm0 ## encoding: [0x0f,0xc6,0xc0,0x55]
+; FMACALL64-NEXT:    ## xmm0 = xmm0[1,1,1,1]
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x28,0x4c,0x24,0x40]
-; FMACALL64-NEXT:    shufps $229, %xmm1, %xmm1 ## encoding: [0x0f,0xc6,0xc9,0xe5]
-; FMACALL64-NEXT:    ## xmm1 = xmm1[1,1,2,3]
+; FMACALL64-NEXT:    shufps $85, %xmm1, %xmm1 ## encoding: [0x0f,0xc6,0xc9,0x55]
+; FMACALL64-NEXT:    ## xmm1 = xmm1[1,1,1,1]
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 ## 16-byte Reload
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x28,0x54,0x24,0x50]
-; FMACALL64-NEXT:    shufps $229, %xmm2, %xmm2 ## encoding: [0x0f,0xc6,0xd2,0xe5]
-; FMACALL64-NEXT:    ## xmm2 = xmm2[1,1,2,3]
+; FMACALL64-NEXT:    shufps $85, %xmm2, %xmm2 ## encoding: [0x0f,0xc6,0xd2,0x55]
+; FMACALL64-NEXT:    ## xmm2 = xmm2[1,1,1,1]
 ; FMACALL64-NEXT:    callq _fmaf ## encoding: [0xe8,A,A,A,A]
 ; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: reloc_branch_4byte_pcrel
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
@@ -846,13 +846,13 @@ define <16 x float> @test_v16f32(<16 x float> %a, <16 x float> %b, <16 x float>
 ; FMACALL64-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x29,0x44,0x24,0x50]
 ; FMACALL64-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm2 ## encoding: [0x0f,0x28,0x94,0x24,0xb0,0x00,0x00,0x00]
-; FMACALL64-NEXT:    shufps $231, %xmm0, %xmm0 ## encoding: [0x0f,0xc6,0xc0,0xe7]
-; FMACALL64-NEXT:    ## xmm0 = xmm0[3,1,2,3]
+; FMACALL64-NEXT:    shufps $255, %xmm0, %xmm0 ## encoding: [0x0f,0xc6,0xc0,0xff]
+; FMACALL64-NEXT:    ## xmm0 = xmm0[3,3,3,3]
 ; FMACALL64-NEXT:    movaps %xmm4, %xmm1 ## encoding: [0x0f,0x28,0xcc]
-; FMACALL64-NEXT:    shufps $231, %xmm4, %xmm1 ## encoding: [0x0f,0xc6,0xcc,0xe7]
-; FMACALL64-NEXT:    ## xmm1 = xmm1[3,1],xmm4[2,3]
-; FMACALL64-NEXT:    shufps $231, %xmm2, %xmm2 ## encoding: [0x0f,0xc6,0xd2,0xe7]
-; FMACALL64-NEXT:    ## xmm2 = xmm2[3,1,2,3]
+; FMACALL64-NEXT:    shufps $255, %xmm4, %xmm1 ## encoding: [0x0f,0xc6,0xcc,0xff]
+; FMACALL64-NEXT:    ## xmm1 = xmm1[3,3],xmm4[3,3]
+; FMACALL64-NEXT:    shufps $255, %xmm2, %xmm2 ## encoding: [0x0f,0xc6,0xd2,0xff]
+; FMACALL64-NEXT:    ## xmm2 = xmm2[3,3,3,3]
 ; FMACALL64-NEXT:    callq _fmaf ## encoding: [0xe8,A,A,A,A]
 ; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: reloc_branch_4byte_pcrel
 ; FMACALL64-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
@@ -886,15 +886,15 @@ define <16 x float> @test_v16f32(<16 x float> %a, <16 x float> %b, <16 x float>
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x29,0x44,0x24,0x60]
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x28,0x44,0x24,0x50]
-; FMACALL64-NEXT:    shufps $229, %xmm0, %xmm0 ## encoding: [0x0f,0xc6,0xc0,0xe5]
-; FMACALL64-NEXT:    ## xmm0 = xmm0[1,1,2,3]
+; FMACALL64-NEXT:    shufps $85, %xmm0, %xmm0 ## encoding: [0x0f,0xc6,0xc0,0x55]
+; FMACALL64-NEXT:    ## xmm0 = xmm0[1,1,1,1]
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x28,0x4c,0x24,0x10]
-; FMACALL64-NEXT:    shufps $229, %xmm1, %xmm1 ## encoding: [0x0f,0xc6,0xc9,0xe5]
-; FMACALL64-NEXT:    ## xmm1 = xmm1[1,1,2,3]
+; FMACALL64-NEXT:    shufps $85, %xmm1, %xmm1 ## encoding: [0x0f,0xc6,0xc9,0x55]
+; FMACALL64-NEXT:    ## xmm1 = xmm1[1,1,1,1]
 ; FMACALL64-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm2 ## encoding: [0x0f,0x28,0x94,0x24,0xb0,0x00,0x00,0x00]
-; FMACALL64-NEXT:    shufps $229, %xmm2, %xmm2 ## encoding: [0x0f,0xc6,0xd2,0xe5]
-; FMACALL64-NEXT:    ## xmm2 = xmm2[1,1,2,3]
+; FMACALL64-NEXT:    shufps $85, %xmm2, %xmm2 ## encoding: [0x0f,0xc6,0xd2,0x55]
+; FMACALL64-NEXT:    ## xmm2 = xmm2[1,1,1,1]
 ; FMACALL64-NEXT:    callq _fmaf ## encoding: [0xe8,A,A,A,A]
 ; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: reloc_branch_4byte_pcrel
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
@@ -908,15 +908,15 @@ define <16 x float> @test_v16f32(<16 x float> %a, <16 x float> %b, <16 x float>
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x29,0x4c,0x24,0x60]
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x28,0x44,0x24,0x30]
-; FMACALL64-NEXT:    shufps $231, %xmm0, %xmm0 ## encoding: [0x0f,0xc6,0xc0,0xe7]
-; FMACALL64-NEXT:    ## xmm0 = xmm0[3,1,2,3]
+; FMACALL64-NEXT:    shufps $255, %xmm0, %xmm0 ## encoding: [0x0f,0xc6,0xc0,0xff]
+; FMACALL64-NEXT:    ## xmm0 = xmm0[3,3,3,3]
 ; FMACALL64-NEXT:    movaps (%rsp), %xmm1 ## 16-byte Reload
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x28,0x0c,0x24]
-; FMACALL64-NEXT:    shufps $231, %xmm1, %xmm1 ## encoding: [0x0f,0xc6,0xc9,0xe7]
-; FMACALL64-NEXT:    ## xmm1 = xmm1[3,1,2,3]
+; FMACALL64-NEXT:    shufps $255, %xmm1, %xmm1 ## encoding: [0x0f,0xc6,0xc9,0xff]
+; FMACALL64-NEXT:    ## xmm1 = xmm1[3,3,3,3]
 ; FMACALL64-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm2 ## encoding: [0x0f,0x28,0x94,0x24,0xc0,0x00,0x00,0x00]
-; FMACALL64-NEXT:    shufps $231, %xmm2, %xmm2 ## encoding: [0x0f,0xc6,0xd2,0xe7]
-; FMACALL64-NEXT:    ## xmm2 = xmm2[3,1,2,3]
+; FMACALL64-NEXT:    shufps $255, %xmm2, %xmm2 ## encoding: [0x0f,0xc6,0xd2,0xff]
+; FMACALL64-NEXT:    ## xmm2 = xmm2[3,3,3,3]
 ; FMACALL64-NEXT:    callq _fmaf ## encoding: [0xe8,A,A,A,A]
 ; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: reloc_branch_4byte_pcrel
 ; FMACALL64-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
@@ -950,15 +950,15 @@ define <16 x float> @test_v16f32(<16 x float> %a, <16 x float> %b, <16 x float>
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x29,0x44,0x24,0x10]
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x28,0x44,0x24,0x30]
-; FMACALL64-NEXT:    shufps $229, %xmm0, %xmm0 ## encoding: [0x0f,0xc6,0xc0,0xe5]
-; FMACALL64-NEXT:    ## xmm0 = xmm0[1,1,2,3]
+; FMACALL64-NEXT:    shufps $85, %xmm0, %xmm0 ## encoding: [0x0f,0xc6,0xc0,0x55]
+; FMACALL64-NEXT:    ## xmm0 = xmm0[1,1,1,1]
 ; FMACALL64-NEXT:    movaps (%rsp), %xmm1 ## 16-byte Reload
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x28,0x0c,0x24]
-; FMACALL64-NEXT:    shufps $229, %xmm1, %xmm1 ## encoding: [0x0f,0xc6,0xc9,0xe5]
-; FMACALL64-NEXT:    ## xmm1 = xmm1[1,1,2,3]
+; FMACALL64-NEXT:    shufps $85, %xmm1, %xmm1 ## encoding: [0x0f,0xc6,0xc9,0x55]
+; FMACALL64-NEXT:    ## xmm1 = xmm1[1,1,1,1]
 ; FMACALL64-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm2 ## encoding: [0x0f,0x28,0x94,0x24,0xc0,0x00,0x00,0x00]
-; FMACALL64-NEXT:    shufps $229, %xmm2, %xmm2 ## encoding: [0x0f,0xc6,0xd2,0xe5]
-; FMACALL64-NEXT:    ## xmm2 = xmm2[1,1,2,3]
+; FMACALL64-NEXT:    shufps $85, %xmm2, %xmm2 ## encoding: [0x0f,0xc6,0xd2,0x55]
+; FMACALL64-NEXT:    ## xmm2 = xmm2[1,1,1,1]
 ; FMACALL64-NEXT:    callq _fmaf ## encoding: [0xe8,A,A,A,A]
 ; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: reloc_branch_4byte_pcrel
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
@@ -972,15 +972,15 @@ define <16 x float> @test_v16f32(<16 x float> %a, <16 x float> %b, <16 x float>
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x29,0x4c,0x24,0x10]
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x28,0x44,0x24,0x40]
-; FMACALL64-NEXT:    shufps $231, %xmm0, %xmm0 ## encoding: [0x0f,0xc6,0xc0,0xe7]
-; FMACALL64-NEXT:    ## xmm0 = xmm0[3,1,2,3]
+; FMACALL64-NEXT:    shufps $255, %xmm0, %xmm0 ## encoding: [0x0f,0xc6,0xc0,0xff]
+; FMACALL64-NEXT:    ## xmm0 = xmm0[3,3,3,3]
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x28,0x4c,0x24,0x20]
-; FMACALL64-NEXT:    shufps $231, %xmm1, %xmm1 ## encoding: [0x0f,0xc6,0xc9,0xe7]
-; FMACALL64-NEXT:    ## xmm1 = xmm1[3,1,2,3]
+; FMACALL64-NEXT:    shufps $255, %xmm1, %xmm1 ## encoding: [0x0f,0xc6,0xc9,0xff]
+; FMACALL64-NEXT:    ## xmm1 = xmm1[3,3,3,3]
 ; FMACALL64-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm2 ## encoding: [0x0f,0x28,0x94,0x24,0xd0,0x00,0x00,0x00]
-; FMACALL64-NEXT:    shufps $231, %xmm2, %xmm2 ## encoding: [0x0f,0xc6,0xd2,0xe7]
-; FMACALL64-NEXT:    ## xmm2 = xmm2[3,1,2,3]
+; FMACALL64-NEXT:    shufps $255, %xmm2, %xmm2 ## encoding: [0x0f,0xc6,0xd2,0xff]
+; FMACALL64-NEXT:    ## xmm2 = xmm2[3,3,3,3]
 ; FMACALL64-NEXT:    callq _fmaf ## encoding: [0xe8,A,A,A,A]
 ; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: reloc_branch_4byte_pcrel
 ; FMACALL64-NEXT:    movaps %xmm0, (%rsp) ## 16-byte Spill
@@ -1014,15 +1014,15 @@ define <16 x float> @test_v16f32(<16 x float> %a, <16 x float> %b, <16 x float>
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x29,0x04,0x24]
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x28,0x44,0x24,0x40]
-; FMACALL64-NEXT:    shufps $229, %xmm0, %xmm0 ## encoding: [0x0f,0xc6,0xc0,0xe5]
-; FMACALL64-NEXT:    ## xmm0 = xmm0[1,1,2,3]
+; FMACALL64-NEXT:    shufps $85, %xmm0, %xmm0 ## encoding: [0x0f,0xc6,0xc0,0x55]
+; FMACALL64-NEXT:    ## xmm0 = xmm0[1,1,1,1]
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x28,0x4c,0x24,0x20]
-; FMACALL64-NEXT:    shufps $229, %xmm1, %xmm1 ## encoding: [0x0f,0xc6,0xc9,0xe5]
-; FMACALL64-NEXT:    ## xmm1 = xmm1[1,1,2,3]
+; FMACALL64-NEXT:    shufps $85, %xmm1, %xmm1 ## encoding: [0x0f,0xc6,0xc9,0x55]
+; FMACALL64-NEXT:    ## xmm1 = xmm1[1,1,1,1]
 ; FMACALL64-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm2 ## encoding: [0x0f,0x28,0x94,0x24,0xd0,0x00,0x00,0x00]
-; FMACALL64-NEXT:    shufps $229, %xmm2, %xmm2 ## encoding: [0x0f,0xc6,0xd2,0xe5]
-; FMACALL64-NEXT:    ## xmm2 = xmm2[1,1,2,3]
+; FMACALL64-NEXT:    shufps $85, %xmm2, %xmm2 ## encoding: [0x0f,0xc6,0xd2,0x55]
+; FMACALL64-NEXT:    ## xmm2 = xmm2[1,1,1,1]
 ; FMACALL64-NEXT:    callq _fmaf ## encoding: [0xe8,A,A,A,A]
 ; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: reloc_branch_4byte_pcrel
 ; FMACALL64-NEXT:    movaps (%rsp), %xmm1 ## 16-byte Reload
@@ -1036,15 +1036,15 @@ define <16 x float> @test_v16f32(<16 x float> %a, <16 x float> %b, <16 x float>
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x29,0x0c,0x24]
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x28,0x44,0x24,0x70]
-; FMACALL64-NEXT:    shufps $231, %xmm0, %xmm0 ## encoding: [0x0f,0xc6,0xc0,0xe7]
-; FMACALL64-NEXT:    ## xmm0 = xmm0[3,1,2,3]
+; FMACALL64-NEXT:    shufps $255, %xmm0, %xmm0 ## encoding: [0x0f,0xc6,0xc0,0xff]
+; FMACALL64-NEXT:    ## xmm0 = xmm0[3,3,3,3]
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x28,0x8c,0x24,0x80,0x00,0x00,0x00]
-; FMACALL64-NEXT:    shufps $231, %xmm1, %xmm1 ## encoding: [0x0f,0xc6,0xc9,0xe7]
-; FMACALL64-NEXT:    ## xmm1 = xmm1[3,1,2,3]
+; FMACALL64-NEXT:    shufps $255, %xmm1, %xmm1 ## encoding: [0x0f,0xc6,0xc9,0xff]
+; FMACALL64-NEXT:    ## xmm1 = xmm1[3,3,3,3]
 ; FMACALL64-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm2 ## encoding: [0x0f,0x28,0x94,0x24,0xe0,0x00,0x00,0x00]
-; FMACALL64-NEXT:    shufps $231, %xmm2, %xmm2 ## encoding: [0x0f,0xc6,0xd2,0xe7]
-; FMACALL64-NEXT:    ## xmm2 = xmm2[3,1,2,3]
+; FMACALL64-NEXT:    shufps $255, %xmm2, %xmm2 ## encoding: [0x0f,0xc6,0xd2,0xff]
+; FMACALL64-NEXT:    ## xmm2 = xmm2[3,3,3,3]
 ; FMACALL64-NEXT:    callq _fmaf ## encoding: [0xe8,A,A,A,A]
 ; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: reloc_branch_4byte_pcrel
 ; FMACALL64-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
@@ -1078,15 +1078,15 @@ define <16 x float> @test_v16f32(<16 x float> %a, <16 x float> %b, <16 x float>
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x29,0x44,0x24,0x40]
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x28,0x44,0x24,0x70]
-; FMACALL64-NEXT:    shufps $229, %xmm0, %xmm0 ## encoding: [0x0f,0xc6,0xc0,0xe5]
-; FMACALL64-NEXT:    ## xmm0 = xmm0[1,1,2,3]
+; FMACALL64-NEXT:    shufps $85, %xmm0, %xmm0 ## encoding: [0x0f,0xc6,0xc0,0x55]
+; FMACALL64-NEXT:    ## xmm0 = xmm0[1,1,1,1]
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x28,0x8c,0x24,0x80,0x00,0x00,0x00]
-; FMACALL64-NEXT:    shufps $229, %xmm1, %xmm1 ## encoding: [0x0f,0xc6,0xc9,0xe5]
-; FMACALL64-NEXT:    ## xmm1 = xmm1[1,1,2,3]
+; FMACALL64-NEXT:    shufps $85, %xmm1, %xmm1 ## encoding: [0x0f,0xc6,0xc9,0x55]
+; FMACALL64-NEXT:    ## xmm1 = xmm1[1,1,1,1]
 ; FMACALL64-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm2 ## encoding: [0x0f,0x28,0x94,0x24,0xe0,0x00,0x00,0x00]
-; FMACALL64-NEXT:    shufps $229, %xmm2, %xmm2 ## encoding: [0x0f,0xc6,0xd2,0xe5]
-; FMACALL64-NEXT:    ## xmm2 = xmm2[1,1,2,3]
+; FMACALL64-NEXT:    shufps $85, %xmm2, %xmm2 ## encoding: [0x0f,0xc6,0xd2,0x55]
+; FMACALL64-NEXT:    ## xmm2 = xmm2[1,1,1,1]
 ; FMACALL64-NEXT:    callq _fmaf ## encoding: [0xe8,A,A,A,A]
 ; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: reloc_branch_4byte_pcrel
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 ## 16-byte Reload
diff --git a/llvm/test/CodeGen/X86/fp-intrinsics-fma.ll b/llvm/test/CodeGen/X86/fp-intrinsics-fma.ll
index c2cbad0694252..04de8ec497178 100644
--- a/llvm/test/CodeGen/X86/fp-intrinsics-fma.ll
+++ b/llvm/test/CodeGen/X86/fp-intrinsics-fma.ll
@@ -525,9 +525,9 @@ define <4 x float> @f19(<4 x float> %0, <4 x float> %1, <4 x float> %2) #0 {
 ; NOFMA-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; NOFMA-NEXT:    pxor {{.*}}(%rip), %xmm0
 ; NOFMA-NEXT:    movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; NOFMA-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1,2,3]
-; NOFMA-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,1,2,3]
-; NOFMA-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; NOFMA-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
+; NOFMA-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,3,3,3]
+; NOFMA-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; NOFMA-NEXT:    callq fmaf
 ; NOFMA-NEXT:    movdqa %xmm0, (%rsp) # 16-byte Spill
 ; NOFMA-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
@@ -546,11 +546,11 @@ define <4 x float> @f19(<4 x float> %0, <4 x float> %1, <4 x float> %2) #0 {
 ; NOFMA-NEXT:    callq fmaf
 ; NOFMA-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; NOFMA-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; NOFMA-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; NOFMA-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,1,1]
 ; NOFMA-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
-; NOFMA-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,2,3]
-; NOFMA-NEXT:    pshufd $229, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
-; NOFMA-NEXT:    # xmm0 = mem[1,1,2,3]
+; NOFMA-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,1,1]
+; NOFMA-NEXT:    pshufd $85, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; NOFMA-NEXT:    # xmm0 = mem[1,1,1,1]
 ; NOFMA-NEXT:    callq fmaf
 ; NOFMA-NEXT:    movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
 ; NOFMA-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
@@ -629,9 +629,9 @@ define <4 x float> @f21(<4 x float> %0, <4 x float> %1, <4 x float> %2) #0 {
 ; NOFMA-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; NOFMA-NEXT:    pxor {{.*}}(%rip), %xmm2
 ; NOFMA-NEXT:    movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; NOFMA-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
-; NOFMA-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1,2,3]
-; NOFMA-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[3,1,2,3]
+; NOFMA-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; NOFMA-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
+; NOFMA-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[3,3,3,3]
 ; NOFMA-NEXT:    callq fmaf
 ; NOFMA-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
 ; NOFMA-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
@@ -650,11 +650,11 @@ define <4 x float> @f21(<4 x float> %0, <4 x float> %1, <4 x float> %2) #0 {
 ; NOFMA-NEXT:    callq fmaf
 ; NOFMA-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; NOFMA-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; NOFMA-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; NOFMA-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; NOFMA-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; NOFMA-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,2,3]
-; NOFMA-NEXT:    pshufd $229, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
-; NOFMA-NEXT:    # xmm2 = mem[1,1,2,3]
+; NOFMA-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,1,1]
+; NOFMA-NEXT:    pshufd $85, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
+; NOFMA-NEXT:    # xmm2 = mem[1,1,1,1]
 ; NOFMA-NEXT:    callq fmaf
 ; NOFMA-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
 ; NOFMA-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
@@ -735,9 +735,9 @@ define <4 x float> @f23(<4 x float> %0, <4 x float> %1, <4 x float> %2) #0 {
 ; NOFMA-NEXT:    movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; NOFMA-NEXT:    pxor %xmm3, %xmm2
 ; NOFMA-NEXT:    movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; NOFMA-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
-; NOFMA-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[3,1,2,3]
-; NOFMA-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; NOFMA-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; NOFMA-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[3,3,3,3]
+; NOFMA-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; NOFMA-NEXT:    callq fmaf
 ; NOFMA-NEXT:    movdqa %xmm0, (%rsp) # 16-byte Spill
 ; NOFMA-NEXT:    pshufd $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
@@ -755,12 +755,12 @@ define <4 x float> @f23(<4 x float> %0, <4 x float> %1, <4 x float> %2) #0 {
 ; NOFMA-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
 ; NOFMA-NEXT:    callq fmaf
 ; NOFMA-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; NOFMA-NEXT:    pshufd $229, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
-; NOFMA-NEXT:    # xmm0 = mem[1,1,2,3]
-; NOFMA-NEXT:    pshufd $229, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
-; NOFMA-NEXT:    # xmm2 = mem[1,1,2,3]
+; NOFMA-NEXT:    pshufd $85, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; NOFMA-NEXT:    # xmm0 = mem[1,1,1,1]
+; NOFMA-NEXT:    pshufd $85, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
+; NOFMA-NEXT:    # xmm2 = mem[1,1,1,1]
 ; NOFMA-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; NOFMA-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; NOFMA-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,1,1]
 ; NOFMA-NEXT:    callq fmaf
 ; NOFMA-NEXT:    movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
 ; NOFMA-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
@@ -842,9 +842,9 @@ define <4 x float> @f25(<4 x float> %0, <4 x float> %1, <4 x float> %2) #0 {
 ; NOFMA-NEXT:    movaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; NOFMA-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; NOFMA-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; NOFMA-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
-; NOFMA-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1,2,3]
-; NOFMA-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,1,2,3]
+; NOFMA-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; NOFMA-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
+; NOFMA-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,3,3,3]
 ; NOFMA-NEXT:    callq fmaf
 ; NOFMA-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
 ; NOFMA-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
@@ -863,11 +863,11 @@ define <4 x float> @f25(<4 x float> %0, <4 x float> %1, <4 x float> %2) #0 {
 ; NOFMA-NEXT:    callq fmaf
 ; NOFMA-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; NOFMA-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; NOFMA-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; NOFMA-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; NOFMA-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; NOFMA-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; NOFMA-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,1,1]
 ; NOFMA-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
-; NOFMA-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,2,3]
+; NOFMA-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,1,1]
 ; NOFMA-NEXT:    callq fmaf
 ; NOFMA-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
 ; NOFMA-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
@@ -960,9 +960,9 @@ define <4 x float> @f27(<4 x float> %0, <4 x float> %1, <4 x float> %2) #0 {
 ; NOFMA-NEXT:    movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; NOFMA-NEXT:    pxor %xmm3, %xmm2
 ; NOFMA-NEXT:    movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; NOFMA-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
-; NOFMA-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[3,1,2,3]
-; NOFMA-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; NOFMA-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; NOFMA-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[3,3,3,3]
+; NOFMA-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; NOFMA-NEXT:    callq fmaf
 ; NOFMA-NEXT:    movdqa %xmm0, (%rsp) # 16-byte Spill
 ; NOFMA-NEXT:    pshufd $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
@@ -980,12 +980,12 @@ define <4 x float> @f27(<4 x float> %0, <4 x float> %1, <4 x float> %2) #0 {
 ; NOFMA-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
 ; NOFMA-NEXT:    callq fmaf
 ; NOFMA-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; NOFMA-NEXT:    pshufd $229, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
-; NOFMA-NEXT:    # xmm0 = mem[1,1,2,3]
-; NOFMA-NEXT:    pshufd $229, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
-; NOFMA-NEXT:    # xmm2 = mem[1,1,2,3]
+; NOFMA-NEXT:    pshufd $85, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; NOFMA-NEXT:    # xmm0 = mem[1,1,1,1]
+; NOFMA-NEXT:    pshufd $85, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
+; NOFMA-NEXT:    # xmm2 = mem[1,1,1,1]
 ; NOFMA-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; NOFMA-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; NOFMA-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,1,1]
 ; NOFMA-NEXT:    callq fmaf
 ; NOFMA-NEXT:    movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
 ; NOFMA-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
diff --git a/llvm/test/CodeGen/X86/fp-round.ll b/llvm/test/CodeGen/X86/fp-round.ll
index f67ddb65be570..9d40593b25744 100644
--- a/llvm/test/CodeGen/X86/fp-round.ll
+++ b/llvm/test/CodeGen/X86/fp-round.ll
@@ -77,7 +77,7 @@ define <4 x float> @round_v4f32(<4 x float> %x) {
 ; SSE2-NEXT:    subq $56, %rsp
 ; SSE2-NEXT:    .cfi_def_cfa_offset 64
 ; SSE2-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE2-NEXT:    callq _roundf
 ; SSE2-NEXT:    movaps %xmm0, (%rsp) ## 16-byte Spill
 ; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
@@ -90,7 +90,7 @@ define <4 x float> @round_v4f32(<4 x float> %x) {
 ; SSE2-NEXT:    callq _roundf
 ; SSE2-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
 ; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    callq _roundf
 ; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
 ; SSE2-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
@@ -174,7 +174,7 @@ define <8 x float> @round_v8f32(<8 x float> %x) {
 ; SSE2-NEXT:    .cfi_def_cfa_offset 80
 ; SSE2-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
 ; SSE2-NEXT:    movaps %xmm0, (%rsp) ## 16-byte Spill
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE2-NEXT:    callq _roundf
 ; SSE2-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
 ; SSE2-NEXT:    movaps (%rsp), %xmm0 ## 16-byte Reload
@@ -187,7 +187,7 @@ define <8 x float> @round_v8f32(<8 x float> %x) {
 ; SSE2-NEXT:    callq _roundf
 ; SSE2-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
 ; SSE2-NEXT:    movaps (%rsp), %xmm0 ## 16-byte Reload
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    callq _roundf
 ; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
 ; SSE2-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
@@ -195,7 +195,7 @@ define <8 x float> @round_v8f32(<8 x float> %x) {
 ; SSE2-NEXT:    ## xmm1 = xmm1[0],mem[0]
 ; SSE2-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
 ; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE2-NEXT:    callq _roundf
 ; SSE2-NEXT:    movaps %xmm0, (%rsp) ## 16-byte Spill
 ; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
@@ -208,7 +208,7 @@ define <8 x float> @round_v8f32(<8 x float> %x) {
 ; SSE2-NEXT:    callq _roundf
 ; SSE2-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
 ; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    callq _roundf
 ; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
 ; SSE2-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
@@ -326,7 +326,7 @@ define <16 x float> @round_v16f32(<16 x float> %x) {
 ; SSE2-NEXT:    movaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
 ; SSE2-NEXT:    movaps %xmm1, (%rsp) ## 16-byte Spill
 ; SSE2-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE2-NEXT:    callq _roundf
 ; SSE2-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
 ; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
@@ -339,7 +339,7 @@ define <16 x float> @round_v16f32(<16 x float> %x) {
 ; SSE2-NEXT:    callq _roundf
 ; SSE2-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
 ; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    callq _roundf
 ; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
 ; SSE2-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
@@ -347,7 +347,7 @@ define <16 x float> @round_v16f32(<16 x float> %x) {
 ; SSE2-NEXT:    ## xmm1 = xmm1[0],mem[0]
 ; SSE2-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
 ; SSE2-NEXT:    movaps (%rsp), %xmm0 ## 16-byte Reload
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE2-NEXT:    callq _roundf
 ; SSE2-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
 ; SSE2-NEXT:    movaps (%rsp), %xmm0 ## 16-byte Reload
@@ -360,7 +360,7 @@ define <16 x float> @round_v16f32(<16 x float> %x) {
 ; SSE2-NEXT:    callq _roundf
 ; SSE2-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
 ; SSE2-NEXT:    movaps (%rsp), %xmm0 ## 16-byte Reload
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    callq _roundf
 ; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
 ; SSE2-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
@@ -368,7 +368,7 @@ define <16 x float> @round_v16f32(<16 x float> %x) {
 ; SSE2-NEXT:    ## xmm1 = xmm1[0],mem[0]
 ; SSE2-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
 ; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE2-NEXT:    callq _roundf
 ; SSE2-NEXT:    movaps %xmm0, (%rsp) ## 16-byte Spill
 ; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
@@ -381,7 +381,7 @@ define <16 x float> @round_v16f32(<16 x float> %x) {
 ; SSE2-NEXT:    callq _roundf
 ; SSE2-NEXT:    movaps %xmm0, (%rsp) ## 16-byte Spill
 ; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    callq _roundf
 ; SSE2-NEXT:    movaps (%rsp), %xmm1 ## 16-byte Reload
 ; SSE2-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
@@ -389,7 +389,7 @@ define <16 x float> @round_v16f32(<16 x float> %x) {
 ; SSE2-NEXT:    ## xmm1 = xmm1[0],mem[0]
 ; SSE2-NEXT:    movaps %xmm1, (%rsp) ## 16-byte Spill
 ; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE2-NEXT:    callq _roundf
 ; SSE2-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
 ; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
@@ -402,7 +402,7 @@ define <16 x float> @round_v16f32(<16 x float> %x) {
 ; SSE2-NEXT:    callq _roundf
 ; SSE2-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
 ; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    callq _roundf
 ; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 ## 16-byte Reload
 ; SSE2-NEXT:    unpcklps {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1]
diff --git a/llvm/test/CodeGen/X86/fp-roundeven.ll b/llvm/test/CodeGen/X86/fp-roundeven.ll
index a3eae0137f3ec..cc6cc35ee2e02 100644
--- a/llvm/test/CodeGen/X86/fp-roundeven.ll
+++ b/llvm/test/CodeGen/X86/fp-roundeven.ll
@@ -42,7 +42,7 @@ define <4 x float> @roundeven_v4f32(<4 x float> %x) {
 ; SSE2-NEXT:    subq $56, %rsp
 ; SSE2-NEXT:    .cfi_def_cfa_offset 64
 ; SSE2-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE2-NEXT:    callq _roundevenf
 ; SSE2-NEXT:    movaps %xmm0, (%rsp) ## 16-byte Spill
 ; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
@@ -55,7 +55,7 @@ define <4 x float> @roundeven_v4f32(<4 x float> %x) {
 ; SSE2-NEXT:    callq _roundevenf
 ; SSE2-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
 ; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    callq _roundevenf
 ; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
 ; SSE2-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
@@ -85,7 +85,7 @@ define <4 x float> @roundeven_v4f32(<4 x float> %x) {
 ; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0,1],xmm0[0],xmm1[3]
 ; SSE41-NEXT:    movaps %xmm1, (%rsp) ## 16-byte Spill
 ; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
-; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE41-NEXT:    callq _roundevenf
 ; SSE41-NEXT:    movaps (%rsp), %xmm1 ## 16-byte Reload
 ; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm0[0]
@@ -112,8 +112,8 @@ define <4 x float> @roundeven_v4f32(<4 x float> %x) {
 ; AVX-NEXT:    vmovaps (%rsp), %xmm1 ## 16-byte Reload
 ; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
 ; AVX-NEXT:    vmovaps %xmm0, (%rsp) ## 16-byte Spill
-; AVX-NEXT:    vpermilps $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
-; AVX-NEXT:    ## xmm0 = mem[3,1,2,3]
+; AVX-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
+; AVX-NEXT:    ## xmm0 = mem[3,3,3,3]
 ; AVX-NEXT:    callq _roundevenf
 ; AVX-NEXT:    vmovaps (%rsp), %xmm1 ## 16-byte Reload
 ; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
@@ -181,7 +181,7 @@ define <8 x float> @roundeven_v8f32(<8 x float> %x) {
 ; SSE2-NEXT:    .cfi_def_cfa_offset 80
 ; SSE2-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
 ; SSE2-NEXT:    movaps %xmm0, (%rsp) ## 16-byte Spill
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE2-NEXT:    callq _roundevenf
 ; SSE2-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
 ; SSE2-NEXT:    movaps (%rsp), %xmm0 ## 16-byte Reload
@@ -194,7 +194,7 @@ define <8 x float> @roundeven_v8f32(<8 x float> %x) {
 ; SSE2-NEXT:    callq _roundevenf
 ; SSE2-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
 ; SSE2-NEXT:    movaps (%rsp), %xmm0 ## 16-byte Reload
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    callq _roundevenf
 ; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
 ; SSE2-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
@@ -202,7 +202,7 @@ define <8 x float> @roundeven_v8f32(<8 x float> %x) {
 ; SSE2-NEXT:    ## xmm1 = xmm1[0],mem[0]
 ; SSE2-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
 ; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE2-NEXT:    callq _roundevenf
 ; SSE2-NEXT:    movaps %xmm0, (%rsp) ## 16-byte Spill
 ; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
@@ -215,7 +215,7 @@ define <8 x float> @roundeven_v8f32(<8 x float> %x) {
 ; SSE2-NEXT:    callq _roundevenf
 ; SSE2-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
 ; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    callq _roundevenf
 ; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
 ; SSE2-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
@@ -246,7 +246,7 @@ define <8 x float> @roundeven_v8f32(<8 x float> %x) {
 ; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0,1],xmm0[0],xmm1[3]
 ; SSE41-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
 ; SSE41-NEXT:    movaps (%rsp), %xmm0 ## 16-byte Reload
-; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE41-NEXT:    callq _roundevenf
 ; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
 ; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm0[0]
@@ -267,7 +267,7 @@ define <8 x float> @roundeven_v8f32(<8 x float> %x) {
 ; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0,1],xmm0[0],xmm1[3]
 ; SSE41-NEXT:    movaps %xmm1, (%rsp) ## 16-byte Spill
 ; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
-; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE41-NEXT:    callq _roundevenf
 ; SSE41-NEXT:    movaps (%rsp), %xmm1 ## 16-byte Reload
 ; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm0[0]
@@ -297,8 +297,8 @@ define <8 x float> @roundeven_v8f32(<8 x float> %x) {
 ; AVX-NEXT:    vmovaps (%rsp), %xmm1 ## 16-byte Reload
 ; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
 ; AVX-NEXT:    vmovaps %xmm0, (%rsp) ## 16-byte Spill
-; AVX-NEXT:    vpermilps $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
-; AVX-NEXT:    ## xmm0 = mem[3,1,2,3]
+; AVX-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
+; AVX-NEXT:    ## xmm0 = mem[3,3,3,3]
 ; AVX-NEXT:    callq _roundevenf
 ; AVX-NEXT:    vmovaps (%rsp), %xmm1 ## 16-byte Reload
 ; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
@@ -320,8 +320,8 @@ define <8 x float> @roundeven_v8f32(<8 x float> %x) {
 ; AVX-NEXT:    vmovaps (%rsp), %xmm1 ## 16-byte Reload
 ; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
 ; AVX-NEXT:    vmovaps %xmm0, (%rsp) ## 16-byte Spill
-; AVX-NEXT:    vpermilps $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
-; AVX-NEXT:    ## xmm0 = mem[3,1,2,3]
+; AVX-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
+; AVX-NEXT:    ## xmm0 = mem[3,3,3,3]
 ; AVX-NEXT:    callq _roundevenf
 ; AVX-NEXT:    vmovaps (%rsp), %xmm1 ## 16-byte Reload
 ; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
@@ -427,7 +427,7 @@ define <16 x float> @roundeven_v16f32(<16 x float> %x) {
 ; SSE2-NEXT:    movaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
 ; SSE2-NEXT:    movaps %xmm1, (%rsp) ## 16-byte Spill
 ; SSE2-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE2-NEXT:    callq _roundevenf
 ; SSE2-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
 ; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
@@ -440,7 +440,7 @@ define <16 x float> @roundeven_v16f32(<16 x float> %x) {
 ; SSE2-NEXT:    callq _roundevenf
 ; SSE2-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
 ; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    callq _roundevenf
 ; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
 ; SSE2-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
@@ -448,7 +448,7 @@ define <16 x float> @roundeven_v16f32(<16 x float> %x) {
 ; SSE2-NEXT:    ## xmm1 = xmm1[0],mem[0]
 ; SSE2-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
 ; SSE2-NEXT:    movaps (%rsp), %xmm0 ## 16-byte Reload
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE2-NEXT:    callq _roundevenf
 ; SSE2-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
 ; SSE2-NEXT:    movaps (%rsp), %xmm0 ## 16-byte Reload
@@ -461,7 +461,7 @@ define <16 x float> @roundeven_v16f32(<16 x float> %x) {
 ; SSE2-NEXT:    callq _roundevenf
 ; SSE2-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
 ; SSE2-NEXT:    movaps (%rsp), %xmm0 ## 16-byte Reload
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    callq _roundevenf
 ; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
 ; SSE2-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
@@ -469,7 +469,7 @@ define <16 x float> @roundeven_v16f32(<16 x float> %x) {
 ; SSE2-NEXT:    ## xmm1 = xmm1[0],mem[0]
 ; SSE2-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
 ; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE2-NEXT:    callq _roundevenf
 ; SSE2-NEXT:    movaps %xmm0, (%rsp) ## 16-byte Spill
 ; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
@@ -482,7 +482,7 @@ define <16 x float> @roundeven_v16f32(<16 x float> %x) {
 ; SSE2-NEXT:    callq _roundevenf
 ; SSE2-NEXT:    movaps %xmm0, (%rsp) ## 16-byte Spill
 ; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    callq _roundevenf
 ; SSE2-NEXT:    movaps (%rsp), %xmm1 ## 16-byte Reload
 ; SSE2-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
@@ -490,7 +490,7 @@ define <16 x float> @roundeven_v16f32(<16 x float> %x) {
 ; SSE2-NEXT:    ## xmm1 = xmm1[0],mem[0]
 ; SSE2-NEXT:    movaps %xmm1, (%rsp) ## 16-byte Spill
 ; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE2-NEXT:    callq _roundevenf
 ; SSE2-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
 ; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
@@ -503,7 +503,7 @@ define <16 x float> @roundeven_v16f32(<16 x float> %x) {
 ; SSE2-NEXT:    callq _roundevenf
 ; SSE2-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
 ; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    callq _roundevenf
 ; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 ## 16-byte Reload
 ; SSE2-NEXT:    unpcklps {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1]
@@ -538,7 +538,7 @@ define <16 x float> @roundeven_v16f32(<16 x float> %x) {
 ; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0,1],xmm0[0],xmm1[3]
 ; SSE41-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
 ; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
-; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE41-NEXT:    callq _roundevenf
 ; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
 ; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm0[0]
@@ -559,7 +559,7 @@ define <16 x float> @roundeven_v16f32(<16 x float> %x) {
 ; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0,1],xmm0[0],xmm1[3]
 ; SSE41-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
 ; SSE41-NEXT:    movaps (%rsp), %xmm0 ## 16-byte Reload
-; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE41-NEXT:    callq _roundevenf
 ; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
 ; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm0[0]
@@ -580,7 +580,7 @@ define <16 x float> @roundeven_v16f32(<16 x float> %x) {
 ; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0,1],xmm0[0],xmm1[3]
 ; SSE41-NEXT:    movaps %xmm1, (%rsp) ## 16-byte Spill
 ; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
-; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE41-NEXT:    callq _roundevenf
 ; SSE41-NEXT:    movaps (%rsp), %xmm1 ## 16-byte Reload
 ; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm0[0]
@@ -601,7 +601,7 @@ define <16 x float> @roundeven_v16f32(<16 x float> %x) {
 ; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0,1],xmm0[0],xmm1[3]
 ; SSE41-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
 ; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
-; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE41-NEXT:    callq _roundevenf
 ; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 ## 16-byte Reload
 ; SSE41-NEXT:    insertps {{.*#+}} xmm3 = xmm3[0,1,2],xmm0[0]
@@ -634,8 +634,8 @@ define <16 x float> @roundeven_v16f32(<16 x float> %x) {
 ; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
 ; AVX1-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; AVX1-NEXT:    vpermilps $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
-; AVX1-NEXT:    ## xmm0 = mem[3,1,2,3]
+; AVX1-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
+; AVX1-NEXT:    ## xmm0 = mem[3,3,3,3]
 ; AVX1-NEXT:    callq _roundevenf
 ; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
@@ -657,8 +657,8 @@ define <16 x float> @roundeven_v16f32(<16 x float> %x) {
 ; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
 ; AVX1-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; AVX1-NEXT:    vpermilps $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
-; AVX1-NEXT:    ## xmm0 = mem[3,1,2,3]
+; AVX1-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
+; AVX1-NEXT:    ## xmm0 = mem[3,3,3,3]
 ; AVX1-NEXT:    callq _roundevenf
 ; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
@@ -682,8 +682,8 @@ define <16 x float> @roundeven_v16f32(<16 x float> %x) {
 ; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
 ; AVX1-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; AVX1-NEXT:    vpermilps $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
-; AVX1-NEXT:    ## xmm0 = mem[3,1,2,3]
+; AVX1-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
+; AVX1-NEXT:    ## xmm0 = mem[3,3,3,3]
 ; AVX1-NEXT:    callq _roundevenf
 ; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
@@ -705,8 +705,8 @@ define <16 x float> @roundeven_v16f32(<16 x float> %x) {
 ; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
 ; AVX1-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; AVX1-NEXT:    vpermilps $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
-; AVX1-NEXT:    ## xmm0 = mem[3,1,2,3]
+; AVX1-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
+; AVX1-NEXT:    ## xmm0 = mem[3,3,3,3]
 ; AVX1-NEXT:    callq _roundevenf
 ; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
@@ -737,8 +737,8 @@ define <16 x float> @roundeven_v16f32(<16 x float> %x) {
 ; AVX512-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
 ; AVX512-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
 ; AVX512-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; AVX512-NEXT:    vpermilps $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
-; AVX512-NEXT:    ## xmm0 = mem[3,1,2,3]
+; AVX512-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
+; AVX512-NEXT:    ## xmm0 = mem[3,3,3,3]
 ; AVX512-NEXT:    callq _roundevenf
 ; AVX512-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
 ; AVX512-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
@@ -761,8 +761,8 @@ define <16 x float> @roundeven_v16f32(<16 x float> %x) {
 ; AVX512-NEXT:    vmovaps (%rsp), %xmm1 ## 16-byte Reload
 ; AVX512-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
 ; AVX512-NEXT:    vmovaps %xmm0, (%rsp) ## 16-byte Spill
-; AVX512-NEXT:    vpermilps $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
-; AVX512-NEXT:    ## xmm0 = mem[3,1,2,3]
+; AVX512-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
+; AVX512-NEXT:    ## xmm0 = mem[3,3,3,3]
 ; AVX512-NEXT:    callq _roundevenf
 ; AVX512-NEXT:    vmovaps (%rsp), %xmm1 ## 16-byte Reload
 ; AVX512-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
@@ -786,8 +786,8 @@ define <16 x float> @roundeven_v16f32(<16 x float> %x) {
 ; AVX512-NEXT:    vmovaps (%rsp), %xmm1 ## 16-byte Reload
 ; AVX512-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
 ; AVX512-NEXT:    vmovaps %xmm0, (%rsp) ## 16-byte Spill
-; AVX512-NEXT:    vpermilps $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
-; AVX512-NEXT:    ## xmm0 = mem[3,1,2,3]
+; AVX512-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
+; AVX512-NEXT:    ## xmm0 = mem[3,3,3,3]
 ; AVX512-NEXT:    callq _roundevenf
 ; AVX512-NEXT:    vmovaps (%rsp), %xmm1 ## 16-byte Reload
 ; AVX512-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
@@ -809,8 +809,8 @@ define <16 x float> @roundeven_v16f32(<16 x float> %x) {
 ; AVX512-NEXT:    vmovaps (%rsp), %xmm1 ## 16-byte Reload
 ; AVX512-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
 ; AVX512-NEXT:    vmovaps %xmm0, (%rsp) ## 16-byte Spill
-; AVX512-NEXT:    vpermilps $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
-; AVX512-NEXT:    ## xmm0 = mem[3,1,2,3]
+; AVX512-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
+; AVX512-NEXT:    ## xmm0 = mem[3,3,3,3]
 ; AVX512-NEXT:    callq _roundevenf
 ; AVX512-NEXT:    vmovaps (%rsp), %xmm1 ## 16-byte Reload
 ; AVX512-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
diff --git a/llvm/test/CodeGen/X86/gather-addresses.ll b/llvm/test/CodeGen/X86/gather-addresses.ll
index 00f84a6e4b159..9043f8e69774f 100644
--- a/llvm/test/CodeGen/X86/gather-addresses.ll
+++ b/llvm/test/CodeGen/X86/gather-addresses.ll
@@ -15,11 +15,11 @@ define <4 x double> @foo(double* %p, <4 x i32>* %i, <4 x i32>* %h) nounwind {
 ; LIN-SSE2-NEXT:    movdqa (%rsi), %xmm0
 ; LIN-SSE2-NEXT:    pand (%rdx), %xmm0
 ; LIN-SSE2-NEXT:    movd %xmm0, %eax
-; LIN-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; LIN-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; LIN-SSE2-NEXT:    movd %xmm1, %ecx
 ; LIN-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; LIN-SSE2-NEXT:    movd %xmm1, %edx
-; LIN-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; LIN-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; LIN-SSE2-NEXT:    movd %xmm0, %esi
 ; LIN-SSE2-NEXT:    cltq
 ; LIN-SSE2-NEXT:    movslq %ecx, %rcx
@@ -54,11 +54,11 @@ define <4 x double> @foo(double* %p, <4 x i32>* %i, <4 x i32>* %h) nounwind {
 ; WIN-SSE2-NEXT:    movdqa (%rdx), %xmm0
 ; WIN-SSE2-NEXT:    pand (%r8), %xmm0
 ; WIN-SSE2-NEXT:    movd %xmm0, %r8d
-; WIN-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; WIN-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; WIN-SSE2-NEXT:    movd %xmm1, %r9d
 ; WIN-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; WIN-SSE2-NEXT:    movd %xmm1, %r10d
-; WIN-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; WIN-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; WIN-SSE2-NEXT:    movd %xmm0, %edx
 ; WIN-SSE2-NEXT:    movslq %r8d, %rax
 ; WIN-SSE2-NEXT:    movslq %r9d, %r8
@@ -139,11 +139,11 @@ define <4 x i64> @old(double* %p, <4 x i32>* %i, <4 x i32>* %h, i64 %f) nounwind
 ; LIN-SSE2-NEXT:    movdqa (%rsi), %xmm0
 ; LIN-SSE2-NEXT:    pand (%rdx), %xmm0
 ; LIN-SSE2-NEXT:    movd %xmm0, %eax
-; LIN-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; LIN-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; LIN-SSE2-NEXT:    movd %xmm1, %edx
 ; LIN-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; LIN-SSE2-NEXT:    movd %xmm1, %esi
-; LIN-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; LIN-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; LIN-SSE2-NEXT:    movd %xmm0, %edi
 ; LIN-SSE2-NEXT:    andl %ecx, %eax
 ; LIN-SSE2-NEXT:    andl %ecx, %edx
@@ -182,11 +182,11 @@ define <4 x i64> @old(double* %p, <4 x i32>* %i, <4 x i32>* %h, i64 %f) nounwind
 ; WIN-SSE2-NEXT:    movdqa (%rdx), %xmm0
 ; WIN-SSE2-NEXT:    pand (%r8), %xmm0
 ; WIN-SSE2-NEXT:    movd %xmm0, %eax
-; WIN-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; WIN-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; WIN-SSE2-NEXT:    movd %xmm1, %ecx
 ; WIN-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; WIN-SSE2-NEXT:    movd %xmm1, %r8d
-; WIN-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; WIN-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; WIN-SSE2-NEXT:    movd %xmm0, %edx
 ; WIN-SSE2-NEXT:    andl %r9d, %eax
 ; WIN-SSE2-NEXT:    andl %r9d, %ecx
diff --git a/llvm/test/CodeGen/X86/haddsub-2.ll b/llvm/test/CodeGen/X86/haddsub-2.ll
index e36c0479448ea..256e1f460ba25 100644
--- a/llvm/test/CodeGen/X86/haddsub-2.ll
+++ b/llvm/test/CodeGen/X86/haddsub-2.ll
@@ -124,21 +124,21 @@ define <4 x i32> @phadd_d_test1(<4 x i32> %A, <4 x i32> %B) {
 ; SSE3-LABEL: phadd_d_test1:
 ; SSE3:       # %bb.0:
 ; SSE3-NEXT:    movd %xmm0, %eax
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,1,1]
 ; SSE3-NEXT:    movd %xmm2, %ecx
 ; SSE3-NEXT:    addl %eax, %ecx
 ; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; SSE3-NEXT:    movd %xmm2, %eax
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE3-NEXT:    movd %xmm0, %edx
 ; SSE3-NEXT:    addl %eax, %edx
 ; SSE3-NEXT:    movd %xmm1, %eax
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE3-NEXT:    movd %xmm0, %esi
 ; SSE3-NEXT:    addl %eax, %esi
 ; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE3-NEXT:    movd %xmm0, %eax
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[3,1,2,3]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[3,3,3,3]
 ; SSE3-NEXT:    movd %xmm0, %edi
 ; SSE3-NEXT:    addl %eax, %edi
 ; SSE3-NEXT:    movd %edi, %xmm0
@@ -183,20 +183,20 @@ define <4 x i32> @phadd_d_test2(<4 x i32> %A, <4 x i32> %B) {
 ; SSE3:       # %bb.0:
 ; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; SSE3-NEXT:    movd %xmm2, %eax
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,1,2,3]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
 ; SSE3-NEXT:    movd %xmm2, %ecx
 ; SSE3-NEXT:    addl %eax, %ecx
 ; SSE3-NEXT:    movd %xmm0, %eax
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE3-NEXT:    movd %xmm0, %edx
 ; SSE3-NEXT:    addl %eax, %edx
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[3,1,2,3]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[3,3,3,3]
 ; SSE3-NEXT:    movd %xmm0, %eax
 ; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE3-NEXT:    movd %xmm0, %esi
 ; SSE3-NEXT:    addl %eax, %esi
 ; SSE3-NEXT:    movd %esi, %xmm0
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,2,3]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
 ; SSE3-NEXT:    movd %xmm2, %eax
 ; SSE3-NEXT:    movd %xmm1, %esi
 ; SSE3-NEXT:    addl %eax, %esi
@@ -240,21 +240,21 @@ define <4 x i32> @phsub_d_test1(<4 x i32> %A, <4 x i32> %B) {
 ; SSE3-LABEL: phsub_d_test1:
 ; SSE3:       # %bb.0:
 ; SSE3-NEXT:    movd %xmm0, %eax
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,1,1]
 ; SSE3-NEXT:    movd %xmm2, %ecx
 ; SSE3-NEXT:    subl %ecx, %eax
 ; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; SSE3-NEXT:    movd %xmm2, %ecx
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE3-NEXT:    movd %xmm0, %edx
 ; SSE3-NEXT:    subl %edx, %ecx
 ; SSE3-NEXT:    movd %xmm1, %edx
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE3-NEXT:    movd %xmm0, %esi
 ; SSE3-NEXT:    subl %esi, %edx
 ; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE3-NEXT:    movd %xmm0, %esi
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[3,1,2,3]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[3,3,3,3]
 ; SSE3-NEXT:    movd %xmm0, %edi
 ; SSE3-NEXT:    subl %edi, %esi
 ; SSE3-NEXT:    movd %esi, %xmm0
@@ -299,21 +299,21 @@ define <4 x i32> @phsub_d_test2(<4 x i32> %A, <4 x i32> %B) {
 ; SSE3:       # %bb.0:
 ; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; SSE3-NEXT:    movd %xmm2, %eax
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,1,2,3]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
 ; SSE3-NEXT:    movd %xmm2, %ecx
 ; SSE3-NEXT:    subl %ecx, %eax
 ; SSE3-NEXT:    movd %xmm0, %ecx
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE3-NEXT:    movd %xmm0, %edx
 ; SSE3-NEXT:    subl %edx, %ecx
 ; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE3-NEXT:    movd %xmm0, %edx
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[3,1,2,3]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[3,3,3,3]
 ; SSE3-NEXT:    movd %xmm0, %esi
 ; SSE3-NEXT:    subl %esi, %edx
 ; SSE3-NEXT:    movd %edx, %xmm0
 ; SSE3-NEXT:    movd %xmm1, %edx
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
 ; SSE3-NEXT:    movd %xmm1, %esi
 ; SSE3-NEXT:    subl %esi, %edx
 ; SSE3-NEXT:    movd %edx, %xmm1
@@ -510,39 +510,39 @@ define <8 x i32> @avx2_vphadd_d_test(<8 x i32> %A, <8 x i32> %B) {
 ; SSE3-LABEL: avx2_vphadd_d_test:
 ; SSE3:       # %bb.0:
 ; SSE3-NEXT:    movd %xmm0, %ecx
-; SSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,2,3]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,1,1]
 ; SSE3-NEXT:    movd %xmm4, %r8d
 ; SSE3-NEXT:    addl %ecx, %r8d
 ; SSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
 ; SSE3-NEXT:    movd %xmm4, %edx
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE3-NEXT:    movd %xmm0, %r9d
 ; SSE3-NEXT:    addl %edx, %r9d
 ; SSE3-NEXT:    movd %xmm1, %edx
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE3-NEXT:    movd %xmm0, %esi
 ; SSE3-NEXT:    addl %edx, %esi
 ; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE3-NEXT:    movd %xmm0, %edx
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[3,1,2,3]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[3,3,3,3]
 ; SSE3-NEXT:    movd %xmm0, %edi
 ; SSE3-NEXT:    addl %edx, %edi
 ; SSE3-NEXT:    movd %xmm2, %eax
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
 ; SSE3-NEXT:    movd %xmm0, %r10d
 ; SSE3-NEXT:    addl %eax, %r10d
 ; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE3-NEXT:    movd %xmm0, %eax
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[3,1,2,3]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[3,3,3,3]
 ; SSE3-NEXT:    movd %xmm0, %ecx
 ; SSE3-NEXT:    addl %eax, %ecx
 ; SSE3-NEXT:    movd %xmm3, %eax
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,2,3]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,1,1]
 ; SSE3-NEXT:    movd %xmm0, %edx
 ; SSE3-NEXT:    addl %eax, %edx
 ; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]
 ; SSE3-NEXT:    movd %xmm0, %r11d
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[3,1,2,3]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[3,3,3,3]
 ; SSE3-NEXT:    movd %xmm0, %eax
 ; SSE3-NEXT:    addl %r11d, %eax
 ; SSE3-NEXT:    movd %edi, %xmm0
@@ -816,19 +816,19 @@ define <4 x i32> @not_a_hsub_1(<4 x i32> %A, <4 x i32> %B) {
 ; SSE-LABEL: not_a_hsub_1:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    movd %xmm0, %eax
-; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,1,1]
 ; SSE-NEXT:    movd %xmm2, %ecx
 ; SSE-NEXT:    subl %ecx, %eax
 ; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; SSE-NEXT:    movd %xmm2, %ecx
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE-NEXT:    movd %xmm0, %edx
 ; SSE-NEXT:    subl %edx, %ecx
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE-NEXT:    movd %xmm0, %edx
 ; SSE-NEXT:    movd %xmm1, %esi
 ; SSE-NEXT:    subl %esi, %edx
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[3,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[3,3,3,3]
 ; SSE-NEXT:    movd %xmm0, %esi
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE-NEXT:    movd %xmm0, %edi
@@ -886,13 +886,13 @@ define <4 x float> @not_a_hsub_2(<4 x float> %A, <4 x float> %B) {
 ; SSE-NEXT:    movaps %xmm0, %xmm2
 ; SSE-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm0[1]
 ; SSE-NEXT:    movaps %xmm0, %xmm3
-; SSE-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,1],xmm0[2,3]
+; SSE-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,3],xmm0[3,3]
 ; SSE-NEXT:    subss %xmm3, %xmm2
 ; SSE-NEXT:    movshdup {{.*#+}} xmm3 = xmm0[1,1,3,3]
 ; SSE-NEXT:    subss %xmm3, %xmm0
 ; SSE-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
 ; SSE-NEXT:    movaps %xmm1, %xmm2
-; SSE-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,1],xmm1[2,3]
+; SSE-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,3],xmm1[3,3]
 ; SSE-NEXT:    movaps %xmm1, %xmm3
 ; SSE-NEXT:    unpckhpd {{.*#+}} xmm3 = xmm3[1],xmm1[1]
 ; SSE-NEXT:    subss %xmm3, %xmm2
@@ -905,12 +905,12 @@ define <4 x float> @not_a_hsub_2(<4 x float> %A, <4 x float> %B) {
 ; AVX-LABEL: not_a_hsub_2:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
-; AVX-NEXT:    vpermilps {{.*#+}} xmm3 = xmm0[3,1,2,3]
+; AVX-NEXT:    vpermilps {{.*#+}} xmm3 = xmm0[3,3,3,3]
 ; AVX-NEXT:    vsubss %xmm3, %xmm2, %xmm2
 ; AVX-NEXT:    vmovshdup {{.*#+}} xmm3 = xmm0[1,1,3,3]
 ; AVX-NEXT:    vsubss %xmm3, %xmm0, %xmm0
 ; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[2,3]
-; AVX-NEXT:    vpermilps {{.*#+}} xmm2 = xmm1[3,1,2,3]
+; AVX-NEXT:    vpermilps {{.*#+}} xmm2 = xmm1[3,3,3,3]
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm3 = xmm1[1,0]
 ; AVX-NEXT:    vsubss %xmm3, %xmm2, %xmm2
 ; AVX-NEXT:    vmovshdup {{.*#+}} xmm3 = xmm1[1,1,3,3]
@@ -1130,39 +1130,39 @@ define <8 x i32> @avx2_hadd_d(<8 x i32> %a, <8 x i32> %b) {
 ; SSE3-LABEL: avx2_hadd_d:
 ; SSE3:       # %bb.0:
 ; SSE3-NEXT:    movd %xmm0, %ecx
-; SSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,2,3]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,1,1]
 ; SSE3-NEXT:    movd %xmm4, %r8d
 ; SSE3-NEXT:    addl %ecx, %r8d
 ; SSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
 ; SSE3-NEXT:    movd %xmm4, %edx
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE3-NEXT:    movd %xmm0, %r9d
 ; SSE3-NEXT:    addl %edx, %r9d
 ; SSE3-NEXT:    movd %xmm2, %edx
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
 ; SSE3-NEXT:    movd %xmm0, %esi
 ; SSE3-NEXT:    addl %edx, %esi
 ; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE3-NEXT:    movd %xmm0, %edx
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[3,1,2,3]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[3,3,3,3]
 ; SSE3-NEXT:    movd %xmm0, %edi
 ; SSE3-NEXT:    addl %edx, %edi
 ; SSE3-NEXT:    movd %xmm1, %eax
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE3-NEXT:    movd %xmm0, %r10d
 ; SSE3-NEXT:    addl %eax, %r10d
 ; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE3-NEXT:    movd %xmm0, %eax
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[3,1,2,3]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[3,3,3,3]
 ; SSE3-NEXT:    movd %xmm0, %ecx
 ; SSE3-NEXT:    addl %eax, %ecx
 ; SSE3-NEXT:    movd %xmm3, %eax
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,2,3]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,1,1]
 ; SSE3-NEXT:    movd %xmm0, %edx
 ; SSE3-NEXT:    addl %eax, %edx
 ; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]
 ; SSE3-NEXT:    movd %xmm0, %r11d
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[3,1,2,3]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[3,3,3,3]
 ; SSE3-NEXT:    movd %xmm0, %eax
 ; SSE3-NEXT:    addl %r11d, %eax
 ; SSE3-NEXT:    movd %edi, %xmm0
diff --git a/llvm/test/CodeGen/X86/haddsub-3.ll b/llvm/test/CodeGen/X86/haddsub-3.ll
index 6abba1bbfe9f3..e0ea725fb84fb 100644
--- a/llvm/test/CodeGen/X86/haddsub-3.ll
+++ b/llvm/test/CodeGen/X86/haddsub-3.ll
@@ -30,7 +30,7 @@ define float @pr26491(<4 x float> %a0) {
 ; SSSE3-FAST:       # %bb.0:
 ; SSSE3-FAST-NEXT:    haddps %xmm0, %xmm0
 ; SSSE3-FAST-NEXT:    movaps %xmm0, %xmm1
-; SSSE3-FAST-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1],xmm0[2,3]
+; SSSE3-FAST-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3],xmm0[3,3]
 ; SSSE3-FAST-NEXT:    addss %xmm0, %xmm1
 ; SSSE3-FAST-NEXT:    movaps %xmm1, %xmm0
 ; SSSE3-FAST-NEXT:    retq
@@ -46,7 +46,7 @@ define float @pr26491(<4 x float> %a0) {
 ; AVX1-FAST-LABEL: pr26491:
 ; AVX1-FAST:       # %bb.0:
 ; AVX1-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
-; AVX1-FAST-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; AVX1-FAST-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,3,3,3]
 ; AVX1-FAST-NEXT:    vaddss %xmm0, %xmm1, %xmm0
 ; AVX1-FAST-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/haddsub-undef.ll b/llvm/test/CodeGen/X86/haddsub-undef.ll
index ae53f2d8905fe..de2408c1f6bda 100644
--- a/llvm/test/CodeGen/X86/haddsub-undef.ll
+++ b/llvm/test/CodeGen/X86/haddsub-undef.ll
@@ -192,7 +192,7 @@ define <4 x float> @test8_undef(<4 x float> %a, <4 x float> %b) {
 ; SSE-SLOW-NEXT:    addss %xmm0, %xmm1
 ; SSE-SLOW-NEXT:    movaps %xmm0, %xmm2
 ; SSE-SLOW-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm0[1]
-; SSE-SLOW-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE-SLOW-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE-SLOW-NEXT:    addss %xmm2, %xmm0
 ; SSE-SLOW-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0]
 ; SSE-SLOW-NEXT:    movaps %xmm1, %xmm0
@@ -209,7 +209,7 @@ define <4 x float> @test8_undef(<4 x float> %a, <4 x float> %b) {
 ; AVX-SLOW-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
 ; AVX-SLOW-NEXT:    vaddss %xmm1, %xmm0, %xmm1
 ; AVX-SLOW-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
-; AVX-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX-SLOW-NEXT:    vaddss %xmm0, %xmm2, %xmm0
 ; AVX-SLOW-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
 ; AVX-SLOW-NEXT:    retq
@@ -378,7 +378,7 @@ define <16 x float> @test13_v16f32_undef(<16 x float> %a, <16 x float> %b) {
 ; AVX512-SLOW-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
 ; AVX512-SLOW-NEXT:    vaddss %xmm1, %xmm0, %xmm1
 ; AVX512-SLOW-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
-; AVX512-SLOW-NEXT:    vpermilps {{.*#+}} xmm3 = xmm0[3,1,2,3]
+; AVX512-SLOW-NEXT:    vpermilps {{.*#+}} xmm3 = xmm0[3,3,3,3]
 ; AVX512-SLOW-NEXT:    vaddss %xmm3, %xmm2, %xmm2
 ; AVX512-SLOW-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[2,3]
 ; AVX512-SLOW-NEXT:    vextractf128 $1, %ymm0, %xmm0
@@ -386,7 +386,7 @@ define <16 x float> @test13_v16f32_undef(<16 x float> %a, <16 x float> %b) {
 ; AVX512-SLOW-NEXT:    vaddss %xmm2, %xmm0, %xmm2
 ; AVX512-SLOW-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0],xmm1[3]
 ; AVX512-SLOW-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
-; AVX512-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX512-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX512-SLOW-NEXT:    vaddss %xmm0, %xmm2, %xmm0
 ; AVX512-SLOW-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
 ; AVX512-SLOW-NEXT:    retq
@@ -580,27 +580,27 @@ define <4 x float> @add_ps_017(<4 x float> %x) {
 ; SSE-SLOW:       # %bb.0:
 ; SSE-SLOW-NEXT:    movsldup {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE-SLOW-NEXT:    addps %xmm0, %xmm1
-; SSE-SLOW-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; SSE-SLOW-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; SSE-SLOW-NEXT:    movaps %xmm1, %xmm0
 ; SSE-SLOW-NEXT:    retq
 ;
 ; SSE-FAST-LABEL: add_ps_017:
 ; SSE-FAST:       # %bb.0:
 ; SSE-FAST-NEXT:    haddps %xmm0, %xmm0
-; SSE-FAST-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE-FAST-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE-FAST-NEXT:    retq
 ;
 ; AVX-SLOW-LABEL: add_ps_017:
 ; AVX-SLOW:       # %bb.0:
 ; AVX-SLOW-NEXT:    vmovsldup {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; AVX-SLOW-NEXT:    vaddps %xmm0, %xmm1, %xmm0
-; AVX-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX-SLOW-NEXT:    retq
 ;
 ; AVX-FAST-LABEL: add_ps_017:
 ; AVX-FAST:       # %bb.0:
 ; AVX-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
-; AVX-FAST-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX-FAST-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX-FAST-NEXT:    retq
   %l = shufflevector <4 x float> %x, <4 x float> undef, <4 x i32> <i32 undef, i32 undef, i32 undef, i32 2>
   %add = fadd <4 x float> %l, %x
@@ -612,13 +612,13 @@ define <4 x float> @add_ps_018(<4 x float> %x) {
 ; SSE-LABEL: add_ps_018:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    haddps %xmm0, %xmm0
-; SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[2,2,2,2]
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: add_ps_018:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
-; AVX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[2,2,2,2]
 ; AVX-NEXT:    retq
   %l = shufflevector <4 x float> %x, <4 x float> undef, <4 x i32> <i32 undef, i32 undef, i32 0, i32 undef>
   %r = shufflevector <4 x float> %x, <4 x float> undef, <4 x i32> <i32 undef, i32 undef, i32 1, i32 undef>
@@ -848,7 +848,7 @@ define <4 x float> @PR45747_1(<4 x float> %a, <4 x float> %b) nounwind {
 ; SSE-SLOW:       # %bb.0:
 ; SSE-SLOW-NEXT:    movshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
 ; SSE-SLOW-NEXT:    addps %xmm0, %xmm1
-; SSE-SLOW-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; SSE-SLOW-NEXT:    shufps {{.*#+}} xmm1 = xmm1[2,2,2,2]
 ; SSE-SLOW-NEXT:    movaps %xmm1, %xmm0
 ; SSE-SLOW-NEXT:    retq
 ;
@@ -861,7 +861,7 @@ define <4 x float> @PR45747_1(<4 x float> %a, <4 x float> %b) nounwind {
 ; AVX-SLOW:       # %bb.0:
 ; AVX-SLOW-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
 ; AVX-SLOW-NEXT:    vaddps %xmm0, %xmm1, %xmm0
-; AVX-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[2,2,2,2]
 ; AVX-SLOW-NEXT:    retq
 ;
 ; AVX-FAST-LABEL: PR45747_1:
@@ -886,7 +886,7 @@ define <4 x float> @PR45747_2(<4 x float> %a, <4 x float> %b) nounwind {
 ; SSE-FAST:       # %bb.0:
 ; SSE-FAST-NEXT:    movaps %xmm1, %xmm0
 ; SSE-FAST-NEXT:    haddps %xmm1, %xmm0
-; SSE-FAST-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE-FAST-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE-FAST-NEXT:    retq
 ;
 ; AVX-SLOW-LABEL: PR45747_2:
@@ -899,7 +899,7 @@ define <4 x float> @PR45747_2(<4 x float> %a, <4 x float> %b) nounwind {
 ; AVX-FAST-LABEL: PR45747_2:
 ; AVX-FAST:       # %bb.0:
 ; AVX-FAST-NEXT:    vhaddps %xmm1, %xmm1, %xmm0
-; AVX-FAST-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX-FAST-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX-FAST-NEXT:    retq
   %t0 = shufflevector <4 x float> %b, <4 x float> undef, <4 x i32> <i32 undef, i32 undef, i32 3, i32 undef>
   %t1 = fadd <4 x float> %t0, %b
diff --git a/llvm/test/CodeGen/X86/haddsub.ll b/llvm/test/CodeGen/X86/haddsub.ll
index 872cc4bf9bced..2ea26b45e8713 100644
--- a/llvm/test/CodeGen/X86/haddsub.ll
+++ b/llvm/test/CodeGen/X86/haddsub.ll
@@ -490,7 +490,7 @@ define float @extract_extract23_v4f32_fadd_f32(<4 x float> %x) {
 ; SSE3-SLOW:       # %bb.0:
 ; SSE3-SLOW-NEXT:    movaps %xmm0, %xmm1
 ; SSE3-SLOW-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
-; SSE3-SLOW-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE3-SLOW-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE3-SLOW-NEXT:    addss %xmm1, %xmm0
 ; SSE3-SLOW-NEXT:    retq
 ;
@@ -503,7 +503,7 @@ define float @extract_extract23_v4f32_fadd_f32(<4 x float> %x) {
 ; AVX-SLOW-LABEL: extract_extract23_v4f32_fadd_f32:
 ; AVX-SLOW:       # %bb.0:
 ; AVX-SLOW-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
-; AVX-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX-SLOW-NEXT:    vaddss %xmm0, %xmm1, %xmm0
 ; AVX-SLOW-NEXT:    retq
 ;
@@ -551,7 +551,7 @@ define float @extract_extract23_v4f32_fadd_f32_commute(<4 x float> %x) {
 ; SSE3-SLOW:       # %bb.0:
 ; SSE3-SLOW-NEXT:    movaps %xmm0, %xmm1
 ; SSE3-SLOW-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
-; SSE3-SLOW-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE3-SLOW-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE3-SLOW-NEXT:    addss %xmm1, %xmm0
 ; SSE3-SLOW-NEXT:    retq
 ;
@@ -564,7 +564,7 @@ define float @extract_extract23_v4f32_fadd_f32_commute(<4 x float> %x) {
 ; AVX-SLOW-LABEL: extract_extract23_v4f32_fadd_f32_commute:
 ; AVX-SLOW:       # %bb.0:
 ; AVX-SLOW-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
-; AVX-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX-SLOW-NEXT:    vaddss %xmm1, %xmm0, %xmm0
 ; AVX-SLOW-NEXT:    retq
 ;
@@ -672,7 +672,7 @@ define float @extract_extract23_v4f32_fsub_f32(<4 x float> %x) {
 ; SSE3-SLOW:       # %bb.0:
 ; SSE3-SLOW-NEXT:    movaps %xmm0, %xmm1
 ; SSE3-SLOW-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
-; SSE3-SLOW-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE3-SLOW-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE3-SLOW-NEXT:    subss %xmm0, %xmm1
 ; SSE3-SLOW-NEXT:    movaps %xmm1, %xmm0
 ; SSE3-SLOW-NEXT:    retq
@@ -686,7 +686,7 @@ define float @extract_extract23_v4f32_fsub_f32(<4 x float> %x) {
 ; AVX-SLOW-LABEL: extract_extract23_v4f32_fsub_f32:
 ; AVX-SLOW:       # %bb.0:
 ; AVX-SLOW-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
-; AVX-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX-SLOW-NEXT:    vsubss %xmm0, %xmm1, %xmm0
 ; AVX-SLOW-NEXT:    retq
 ;
@@ -725,14 +725,14 @@ define float @extract_extract23_v4f32_fsub_f32_commute(<4 x float> %x) {
 ; SSE3:       # %bb.0:
 ; SSE3-NEXT:    movaps %xmm0, %xmm1
 ; SSE3-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
-; SSE3-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE3-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE3-NEXT:    subss %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; AVX-LABEL: extract_extract23_v4f32_fsub_f32_commute:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
-; AVX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX-NEXT:    vsubss %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
   %x0 = extractelement <4 x float> %x, i32 2
@@ -827,7 +827,7 @@ define float @extract_extract23_v8f32_fadd_f32(<8 x float> %x) {
 ; SSE3-SLOW:       # %bb.0:
 ; SSE3-SLOW-NEXT:    movaps %xmm0, %xmm1
 ; SSE3-SLOW-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
-; SSE3-SLOW-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE3-SLOW-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE3-SLOW-NEXT:    addss %xmm1, %xmm0
 ; SSE3-SLOW-NEXT:    retq
 ;
@@ -840,7 +840,7 @@ define float @extract_extract23_v8f32_fadd_f32(<8 x float> %x) {
 ; AVX-SLOW-LABEL: extract_extract23_v8f32_fadd_f32:
 ; AVX-SLOW:       # %bb.0:
 ; AVX-SLOW-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
-; AVX-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX-SLOW-NEXT:    vaddss %xmm0, %xmm1, %xmm0
 ; AVX-SLOW-NEXT:    vzeroupper
 ; AVX-SLOW-NEXT:    retq
@@ -862,7 +862,7 @@ define float @extract_extract67_v8f32_fadd_f32(<8 x float> %x) {
 ; SSE3-SLOW:       # %bb.0:
 ; SSE3-SLOW-NEXT:    movaps %xmm1, %xmm0
 ; SSE3-SLOW-NEXT:    movhlps {{.*#+}} xmm1 = xmm1[1,1]
-; SSE3-SLOW-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE3-SLOW-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE3-SLOW-NEXT:    addss %xmm1, %xmm0
 ; SSE3-SLOW-NEXT:    retq
 ;
@@ -876,7 +876,7 @@ define float @extract_extract67_v8f32_fadd_f32(<8 x float> %x) {
 ; AVX-SLOW:       # %bb.0:
 ; AVX-SLOW-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; AVX-SLOW-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
-; AVX-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX-SLOW-NEXT:    vaddss %xmm0, %xmm1, %xmm0
 ; AVX-SLOW-NEXT:    vzeroupper
 ; AVX-SLOW-NEXT:    retq
@@ -929,7 +929,7 @@ define float @extract_extract23_v8f32_fadd_f32_commute(<8 x float> %x) {
 ; SSE3-SLOW:       # %bb.0:
 ; SSE3-SLOW-NEXT:    movaps %xmm0, %xmm1
 ; SSE3-SLOW-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
-; SSE3-SLOW-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE3-SLOW-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE3-SLOW-NEXT:    addss %xmm1, %xmm0
 ; SSE3-SLOW-NEXT:    retq
 ;
@@ -942,7 +942,7 @@ define float @extract_extract23_v8f32_fadd_f32_commute(<8 x float> %x) {
 ; AVX-SLOW-LABEL: extract_extract23_v8f32_fadd_f32_commute:
 ; AVX-SLOW:       # %bb.0:
 ; AVX-SLOW-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
-; AVX-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX-SLOW-NEXT:    vaddss %xmm1, %xmm0, %xmm0
 ; AVX-SLOW-NEXT:    vzeroupper
 ; AVX-SLOW-NEXT:    retq
@@ -964,7 +964,7 @@ define float @extract_extract67_v8f32_fadd_f32_commute(<8 x float> %x) {
 ; SSE3-SLOW:       # %bb.0:
 ; SSE3-SLOW-NEXT:    movaps %xmm1, %xmm0
 ; SSE3-SLOW-NEXT:    movhlps {{.*#+}} xmm1 = xmm1[1,1]
-; SSE3-SLOW-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE3-SLOW-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE3-SLOW-NEXT:    addss %xmm1, %xmm0
 ; SSE3-SLOW-NEXT:    retq
 ;
@@ -978,7 +978,7 @@ define float @extract_extract67_v8f32_fadd_f32_commute(<8 x float> %x) {
 ; AVX-SLOW:       # %bb.0:
 ; AVX-SLOW-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; AVX-SLOW-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
-; AVX-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX-SLOW-NEXT:    vaddss %xmm1, %xmm0, %xmm0
 ; AVX-SLOW-NEXT:    vzeroupper
 ; AVX-SLOW-NEXT:    retq
@@ -1163,7 +1163,7 @@ define float @extract_extract23_v8f32_fsub_f32(<8 x float> %x) {
 ; SSE3-SLOW:       # %bb.0:
 ; SSE3-SLOW-NEXT:    movaps %xmm0, %xmm1
 ; SSE3-SLOW-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
-; SSE3-SLOW-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE3-SLOW-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE3-SLOW-NEXT:    subss %xmm0, %xmm1
 ; SSE3-SLOW-NEXT:    movaps %xmm1, %xmm0
 ; SSE3-SLOW-NEXT:    retq
@@ -1177,7 +1177,7 @@ define float @extract_extract23_v8f32_fsub_f32(<8 x float> %x) {
 ; AVX-SLOW-LABEL: extract_extract23_v8f32_fsub_f32:
 ; AVX-SLOW:       # %bb.0:
 ; AVX-SLOW-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
-; AVX-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX-SLOW-NEXT:    vsubss %xmm0, %xmm1, %xmm0
 ; AVX-SLOW-NEXT:    vzeroupper
 ; AVX-SLOW-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/half.ll b/llvm/test/CodeGen/X86/half.ll
index 1fabce24cc25b..fe0321eff85be 100644
--- a/llvm/test/CodeGen/X86/half.ll
+++ b/llvm/test/CodeGen/X86/half.ll
@@ -558,7 +558,7 @@ define void @test_trunc32_vec4(<4 x float> %a, <4 x half>* %p) #0 {
 ; BWON-NOF16C-NEXT:    subq $24, %rsp
 ; BWON-NOF16C-NEXT:    movq %rdi, %rbx
 ; BWON-NOF16C-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
-; BWON-NOF16C-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; BWON-NOF16C-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; BWON-NOF16C-NEXT:    callq __gnu_f2h_ieee
 ; BWON-NOF16C-NEXT:    movl %eax, %r14d
 ; BWON-NOF16C-NEXT:    movaps (%rsp), %xmm0 # 16-byte Reload
@@ -566,7 +566,7 @@ define void @test_trunc32_vec4(<4 x float> %a, <4 x half>* %p) #0 {
 ; BWON-NOF16C-NEXT:    callq __gnu_f2h_ieee
 ; BWON-NOF16C-NEXT:    movl %eax, %r15d
 ; BWON-NOF16C-NEXT:    movaps (%rsp), %xmm0 # 16-byte Reload
-; BWON-NOF16C-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; BWON-NOF16C-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; BWON-NOF16C-NEXT:    callq __gnu_f2h_ieee
 ; BWON-NOF16C-NEXT:    movl %eax, %ebp
 ; BWON-NOF16C-NEXT:    movaps (%rsp), %xmm0 # 16-byte Reload
@@ -591,7 +591,7 @@ define void @test_trunc32_vec4(<4 x float> %a, <4 x half>* %p) #0 {
 ; BWOFF-NEXT:    subq $24, %rsp
 ; BWOFF-NEXT:    movq %rdi, %rbx
 ; BWOFF-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
-; BWOFF-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; BWOFF-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; BWOFF-NEXT:    callq __gnu_f2h_ieee
 ; BWOFF-NEXT:    movw %ax, %r14w
 ; BWOFF-NEXT:    movaps (%rsp), %xmm0 # 16-byte Reload
@@ -599,7 +599,7 @@ define void @test_trunc32_vec4(<4 x float> %a, <4 x half>* %p) #0 {
 ; BWOFF-NEXT:    callq __gnu_f2h_ieee
 ; BWOFF-NEXT:    movw %ax, %r15w
 ; BWOFF-NEXT:    movaps (%rsp), %xmm0 # 16-byte Reload
-; BWOFF-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; BWOFF-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; BWOFF-NEXT:    callq __gnu_f2h_ieee
 ; BWOFF-NEXT:    movw %ax, %bp
 ; BWOFF-NEXT:    movaps (%rsp), %xmm0 # 16-byte Reload
@@ -630,7 +630,7 @@ define void @test_trunc32_vec4(<4 x float> %a, <4 x half>* %p) #0 {
 ; CHECK-I686-NEXT:    movaps %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
 ; CHECK-I686-NEXT:    movl {{[0-9]+}}(%esp), %ebp
 ; CHECK-I686-NEXT:    movaps %xmm0, %xmm1
-; CHECK-I686-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[2,3]
+; CHECK-I686-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[1,1]
 ; CHECK-I686-NEXT:    movss %xmm1, (%esp)
 ; CHECK-I686-NEXT:    calll __gnu_f2h_ieee
 ; CHECK-I686-NEXT:    movw %ax, %si
@@ -640,7 +640,7 @@ define void @test_trunc32_vec4(<4 x float> %a, <4 x half>* %p) #0 {
 ; CHECK-I686-NEXT:    calll __gnu_f2h_ieee
 ; CHECK-I686-NEXT:    movw %ax, %di
 ; CHECK-I686-NEXT:    movaps {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
-; CHECK-I686-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; CHECK-I686-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; CHECK-I686-NEXT:    movss %xmm0, (%esp)
 ; CHECK-I686-NEXT:    calll __gnu_f2h_ieee
 ; CHECK-I686-NEXT:    movw %ax, %bx
diff --git a/llvm/test/CodeGen/X86/horizontal-reduce-add.ll b/llvm/test/CodeGen/X86/horizontal-reduce-add.ll
index dab7785c85cc1..f509a2e8c8187 100644
--- a/llvm/test/CodeGen/X86/horizontal-reduce-add.ll
+++ b/llvm/test/CodeGen/X86/horizontal-reduce-add.ll
@@ -13,7 +13,7 @@ define i32 @PR37890_v4i32(<4 x i32> %a)  {
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE2-NEXT:    paddd %xmm1, %xmm0
 ; SSE2-NEXT:    movd %xmm0, %eax
 ; SSE2-NEXT:    retq
@@ -22,7 +22,7 @@ define i32 @PR37890_v4i32(<4 x i32> %a)  {
 ; SSSE3-SLOW:       # %bb.0:
 ; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSSE3-SLOW-NEXT:    paddd %xmm0, %xmm1
-; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSSE3-SLOW-NEXT:    paddd %xmm1, %xmm0
 ; SSSE3-SLOW-NEXT:    movd %xmm0, %eax
 ; SSSE3-SLOW-NEXT:    retq
@@ -38,7 +38,7 @@ define i32 @PR37890_v4i32(<4 x i32> %a)  {
 ; AVX1-SLOW:       # %bb.0:
 ; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX1-SLOW-NEXT:    vmovd %xmm0, %eax
 ; AVX1-SLOW-NEXT:    retq
@@ -54,7 +54,7 @@ define i32 @PR37890_v4i32(<4 x i32> %a)  {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
 ; AVX2-NEXT:    retq
@@ -72,7 +72,7 @@ define i16 @PR37890_v8i16(<8 x i16> %a)  {
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    paddw %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE2-NEXT:    paddw %xmm1, %xmm0
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; SSE2-NEXT:    psrld $16, %xmm1
@@ -85,7 +85,7 @@ define i16 @PR37890_v8i16(<8 x i16> %a)  {
 ; SSSE3-SLOW:       # %bb.0:
 ; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSSE3-SLOW-NEXT:    paddw %xmm0, %xmm1
-; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSSE3-SLOW-NEXT:    paddw %xmm1, %xmm0
 ; SSSE3-SLOW-NEXT:    movdqa %xmm0, %xmm1
 ; SSSE3-SLOW-NEXT:    psrld $16, %xmm1
@@ -107,7 +107,7 @@ define i16 @PR37890_v8i16(<8 x i16> %a)  {
 ; AVX1-SLOW:       # %bb.0:
 ; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-SLOW-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
-; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-SLOW-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX1-SLOW-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX1-SLOW-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
@@ -128,7 +128,7 @@ define i16 @PR37890_v8i16(<8 x i16> %a)  {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
@@ -153,7 +153,7 @@ define i32 @PR37890_v8i32(<8 x i32> %a)  {
 ; SSE2-NEXT:    paddd %xmm1, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE2-NEXT:    paddd %xmm1, %xmm0
 ; SSE2-NEXT:    movd %xmm0, %eax
 ; SSE2-NEXT:    retq
@@ -163,7 +163,7 @@ define i32 @PR37890_v8i32(<8 x i32> %a)  {
 ; SSSE3-SLOW-NEXT:    paddd %xmm1, %xmm0
 ; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSSE3-SLOW-NEXT:    paddd %xmm0, %xmm1
-; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSSE3-SLOW-NEXT:    paddd %xmm1, %xmm0
 ; SSSE3-SLOW-NEXT:    movd %xmm0, %eax
 ; SSSE3-SLOW-NEXT:    retq
@@ -182,7 +182,7 @@ define i32 @PR37890_v8i32(<8 x i32> %a)  {
 ; AVX1-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX1-SLOW-NEXT:    vmovd %xmm0, %eax
 ; AVX1-SLOW-NEXT:    vzeroupper
@@ -204,7 +204,7 @@ define i32 @PR37890_v8i32(<8 x i32> %a)  {
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
 ; AVX2-NEXT:    vzeroupper
@@ -227,7 +227,7 @@ define i16 @PR37890_v16i16(<16 x i16> %a)  {
 ; SSE2-NEXT:    paddw %xmm1, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    paddw %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE2-NEXT:    paddw %xmm1, %xmm0
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; SSE2-NEXT:    psrld $16, %xmm1
@@ -241,7 +241,7 @@ define i16 @PR37890_v16i16(<16 x i16> %a)  {
 ; SSSE3-SLOW-NEXT:    paddw %xmm1, %xmm0
 ; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSSE3-SLOW-NEXT:    paddw %xmm0, %xmm1
-; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSSE3-SLOW-NEXT:    paddw %xmm1, %xmm0
 ; SSSE3-SLOW-NEXT:    movdqa %xmm0, %xmm1
 ; SSSE3-SLOW-NEXT:    psrld $16, %xmm1
@@ -266,7 +266,7 @@ define i16 @PR37890_v16i16(<16 x i16> %a)  {
 ; AVX1-SLOW-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-SLOW-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
-; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-SLOW-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX1-SLOW-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX1-SLOW-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
@@ -293,7 +293,7 @@ define i16 @PR37890_v16i16(<16 x i16> %a)  {
 ; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
@@ -324,7 +324,7 @@ define i32 @PR37890_v16i32(<16 x i32> %a)  {
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    paddd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
 ; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    retq
@@ -336,7 +336,7 @@ define i32 @PR37890_v16i32(<16 x i32> %a)  {
 ; SSSE3-SLOW-NEXT:    paddd %xmm0, %xmm1
 ; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSSE3-SLOW-NEXT:    paddd %xmm1, %xmm0
-; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSSE3-SLOW-NEXT:    paddd %xmm0, %xmm1
 ; SSSE3-SLOW-NEXT:    movd %xmm1, %eax
 ; SSSE3-SLOW-NEXT:    retq
@@ -361,7 +361,7 @@ define i32 @PR37890_v16i32(<16 x i32> %a)  {
 ; AVX1-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX1-SLOW-NEXT:    vmovd %xmm0, %eax
 ; AVX1-SLOW-NEXT:    vzeroupper
@@ -387,7 +387,7 @@ define i32 @PR37890_v16i32(<16 x i32> %a)  {
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
 ; AVX2-NEXT:    vzeroupper
diff --git a/llvm/test/CodeGen/X86/horizontal-reduce-fadd.ll b/llvm/test/CodeGen/X86/horizontal-reduce-fadd.ll
index ff635b94de4df..5e3b2817b13ee 100644
--- a/llvm/test/CodeGen/X86/horizontal-reduce-fadd.ll
+++ b/llvm/test/CodeGen/X86/horizontal-reduce-fadd.ll
@@ -15,7 +15,7 @@ define float @PR37890_v4f32(<4 x float> %a)  {
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
 ; SSE2-NEXT:    addps %xmm0, %xmm1
 ; SSE2-NEXT:    movaps %xmm1, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[1,1]
 ; SSE2-NEXT:    addss %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
@@ -132,7 +132,7 @@ define float @PR37890_v8f32(<8 x float> %a)  {
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
 ; SSE2-NEXT:    addps %xmm0, %xmm1
 ; SSE2-NEXT:    movaps %xmm1, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[1,1]
 ; SSE2-NEXT:    addss %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
@@ -276,7 +276,7 @@ define float @PR37890_v16f32(<16 x float> %a)  {
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm1[1]
 ; SSE2-NEXT:    addps %xmm1, %xmm2
 ; SSE2-NEXT:    movaps %xmm2, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm2[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm2[1,1]
 ; SSE2-NEXT:    addss %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/horizontal-reduce-smax.ll b/llvm/test/CodeGen/X86/horizontal-reduce-smax.ll
index a17b1db2c1783..e816c55a1ac8a 100644
--- a/llvm/test/CodeGen/X86/horizontal-reduce-smax.ll
+++ b/llvm/test/CodeGen/X86/horizontal-reduce-smax.ll
@@ -33,7 +33,7 @@ define i64 @test_reduce_v2i64(<2 x i64> %a0) {
 ; X86-SSE2-NEXT:    pandn %xmm1, %xmm3
 ; X86-SSE2-NEXT:    por %xmm0, %xmm3
 ; X86-SSE2-NEXT:    movd %xmm3, %eax
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,1,1]
 ; X86-SSE2-NEXT:    movd %xmm0, %edx
 ; X86-SSE2-NEXT:    retl
 ;
@@ -124,7 +124,7 @@ define i32 @test_reduce_v4i32(<4 x i32> %a0) {
 ; X86-SSE2-NEXT:    pand %xmm2, %xmm0
 ; X86-SSE2-NEXT:    pandn %xmm1, %xmm2
 ; X86-SSE2-NEXT:    por %xmm0, %xmm2
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
 ; X86-SSE2-NEXT:    movdqa %xmm2, %xmm1
 ; X86-SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
 ; X86-SSE2-NEXT:    pand %xmm1, %xmm2
@@ -137,7 +137,7 @@ define i32 @test_reduce_v4i32(<4 x i32> %a0) {
 ; X86-SSE42:       ## %bb.0:
 ; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE42-NEXT:    pmaxsd %xmm0, %xmm1
-; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X86-SSE42-NEXT:    pmaxsd %xmm1, %xmm0
 ; X86-SSE42-NEXT:    movd %xmm0, %eax
 ; X86-SSE42-NEXT:    retl
@@ -146,7 +146,7 @@ define i32 @test_reduce_v4i32(<4 x i32> %a0) {
 ; X86-AVX:       ## %bb.0:
 ; X86-AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-AVX-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
-; X86-AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X86-AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X86-AVX-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vmovd %xmm0, %eax
 ; X86-AVX-NEXT:    retl
@@ -159,7 +159,7 @@ define i32 @test_reduce_v4i32(<4 x i32> %a0) {
 ; X64-SSE2-NEXT:    pand %xmm2, %xmm0
 ; X64-SSE2-NEXT:    pandn %xmm1, %xmm2
 ; X64-SSE2-NEXT:    por %xmm0, %xmm2
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
 ; X64-SSE2-NEXT:    movdqa %xmm2, %xmm1
 ; X64-SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
 ; X64-SSE2-NEXT:    pand %xmm1, %xmm2
@@ -172,7 +172,7 @@ define i32 @test_reduce_v4i32(<4 x i32> %a0) {
 ; X64-SSE42:       ## %bb.0:
 ; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE42-NEXT:    pmaxsd %xmm0, %xmm1
-; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X64-SSE42-NEXT:    pmaxsd %xmm1, %xmm0
 ; X64-SSE42-NEXT:    movd %xmm0, %eax
 ; X64-SSE42-NEXT:    retq
@@ -181,7 +181,7 @@ define i32 @test_reduce_v4i32(<4 x i32> %a0) {
 ; X64-AVX:       ## %bb.0:
 ; X64-AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
-; X64-AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X64-AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X64-AVX-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; X64-AVX-NEXT:    vmovd %xmm0, %eax
 ; X64-AVX-NEXT:    retq
@@ -200,7 +200,7 @@ define i16 @test_reduce_v8i16(<8 x i16> %a0) {
 ; X86-SSE2:       ## %bb.0:
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    pmaxsw %xmm0, %xmm1
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X86-SSE2-NEXT:    pmaxsw %xmm1, %xmm0
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X86-SSE2-NEXT:    psrld $16, %xmm1
@@ -231,7 +231,7 @@ define i16 @test_reduce_v8i16(<8 x i16> %a0) {
 ; X64-SSE2:       ## %bb.0:
 ; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    pmaxsw %xmm0, %xmm1
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X64-SSE2-NEXT:    pmaxsw %xmm1, %xmm0
 ; X64-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X64-SSE2-NEXT:    psrld $16, %xmm1
@@ -279,7 +279,7 @@ define i8 @test_reduce_v16i8(<16 x i8> %a0) {
 ; X86-SSE2-NEXT:    pand %xmm2, %xmm0
 ; X86-SSE2-NEXT:    pandn %xmm1, %xmm2
 ; X86-SSE2-NEXT:    por %xmm0, %xmm2
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
 ; X86-SSE2-NEXT:    movdqa %xmm2, %xmm1
 ; X86-SSE2-NEXT:    pcmpgtb %xmm0, %xmm1
 ; X86-SSE2-NEXT:    pand %xmm1, %xmm2
@@ -334,7 +334,7 @@ define i8 @test_reduce_v16i8(<16 x i8> %a0) {
 ; X64-SSE2-NEXT:    pand %xmm2, %xmm0
 ; X64-SSE2-NEXT:    pandn %xmm1, %xmm2
 ; X64-SSE2-NEXT:    por %xmm0, %xmm2
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
 ; X64-SSE2-NEXT:    movdqa %xmm2, %xmm1
 ; X64-SSE2-NEXT:    pcmpgtb %xmm0, %xmm1
 ; X64-SSE2-NEXT:    pand %xmm1, %xmm2
@@ -435,7 +435,7 @@ define i64 @test_reduce_v4i64(<4 x i64> %a0) {
 ; X86-SSE2-NEXT:    pandn %xmm0, %xmm2
 ; X86-SSE2-NEXT:    por %xmm4, %xmm2
 ; X86-SSE2-NEXT:    movd %xmm2, %eax
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
 ; X86-SSE2-NEXT:    movd %xmm0, %edx
 ; X86-SSE2-NEXT:    retl
 ;
@@ -583,7 +583,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X86-SSE2-NEXT:    pand %xmm1, %xmm2
 ; X86-SSE2-NEXT:    pandn %xmm0, %xmm1
 ; X86-SSE2-NEXT:    por %xmm2, %xmm1
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X86-SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; X86-SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
 ; X86-SSE2-NEXT:    pand %xmm2, %xmm1
@@ -597,7 +597,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X86-SSE42-NEXT:    pmaxsd %xmm1, %xmm0
 ; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE42-NEXT:    pmaxsd %xmm0, %xmm1
-; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X86-SSE42-NEXT:    pmaxsd %xmm1, %xmm0
 ; X86-SSE42-NEXT:    movd %xmm0, %eax
 ; X86-SSE42-NEXT:    retl
@@ -608,7 +608,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X86-AVX1-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-AVX1-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
-; X86-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X86-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X86-AVX1-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vmovd %xmm0, %eax
 ; X86-AVX1-NEXT:    vzeroupper
@@ -620,7 +620,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X86-AVX2-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; X86-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-AVX2-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
-; X86-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X86-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X86-AVX2-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; X86-AVX2-NEXT:    vmovd %xmm0, %eax
 ; X86-AVX2-NEXT:    vzeroupper
@@ -639,7 +639,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X64-SSE2-NEXT:    pand %xmm1, %xmm2
 ; X64-SSE2-NEXT:    pandn %xmm0, %xmm1
 ; X64-SSE2-NEXT:    por %xmm2, %xmm1
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X64-SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; X64-SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
 ; X64-SSE2-NEXT:    pand %xmm2, %xmm1
@@ -653,7 +653,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X64-SSE42-NEXT:    pmaxsd %xmm1, %xmm0
 ; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE42-NEXT:    pmaxsd %xmm0, %xmm1
-; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X64-SSE42-NEXT:    pmaxsd %xmm1, %xmm0
 ; X64-SSE42-NEXT:    movd %xmm0, %eax
 ; X64-SSE42-NEXT:    retq
@@ -664,7 +664,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X64-AVX1-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX1-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X64-AVX1-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vmovd %xmm0, %eax
 ; X64-AVX1-NEXT:    vzeroupper
@@ -676,7 +676,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X64-AVX2-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; X64-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX2-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
-; X64-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X64-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X64-AVX2-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; X64-AVX2-NEXT:    vmovd %xmm0, %eax
 ; X64-AVX2-NEXT:    vzeroupper
@@ -688,7 +688,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X64-AVX512-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX512-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
-; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X64-AVX512-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vmovd %xmm0, %eax
 ; X64-AVX512-NEXT:    vzeroupper
@@ -712,7 +712,7 @@ define i16 @test_reduce_v16i16(<16 x i16> %a0) {
 ; X86-SSE2-NEXT:    pmaxsw %xmm1, %xmm0
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    pmaxsw %xmm0, %xmm1
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X86-SSE2-NEXT:    pmaxsw %xmm1, %xmm0
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X86-SSE2-NEXT:    psrld $16, %xmm1
@@ -760,7 +760,7 @@ define i16 @test_reduce_v16i16(<16 x i16> %a0) {
 ; X64-SSE2-NEXT:    pmaxsw %xmm1, %xmm0
 ; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    pmaxsw %xmm0, %xmm1
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X64-SSE2-NEXT:    pmaxsw %xmm1, %xmm0
 ; X64-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X64-SSE2-NEXT:    psrld $16, %xmm1
@@ -844,7 +844,7 @@ define i8 @test_reduce_v32i8(<32 x i8> %a0) {
 ; X86-SSE2-NEXT:    pand %xmm1, %xmm2
 ; X86-SSE2-NEXT:    pandn %xmm0, %xmm1
 ; X86-SSE2-NEXT:    por %xmm2, %xmm1
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X86-SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; X86-SSE2-NEXT:    pcmpgtb %xmm0, %xmm2
 ; X86-SSE2-NEXT:    pand %xmm2, %xmm1
@@ -922,7 +922,7 @@ define i8 @test_reduce_v32i8(<32 x i8> %a0) {
 ; X64-SSE2-NEXT:    pand %xmm1, %xmm2
 ; X64-SSE2-NEXT:    pandn %xmm0, %xmm1
 ; X64-SSE2-NEXT:    por %xmm2, %xmm1
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X64-SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; X64-SSE2-NEXT:    pcmpgtb %xmm0, %xmm2
 ; X64-SSE2-NEXT:    pand %xmm2, %xmm1
@@ -1088,7 +1088,7 @@ define i64 @test_reduce_v8i64(<8 x i64> %a0) {
 ; X86-SSE2-NEXT:    pandn %xmm0, %xmm2
 ; X86-SSE2-NEXT:    por %xmm1, %xmm2
 ; X86-SSE2-NEXT:    movd %xmm2, %eax
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
 ; X86-SSE2-NEXT:    movd %xmm0, %edx
 ; X86-SSE2-NEXT:    retl
 ;
@@ -1309,7 +1309,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X86-SSE2-NEXT:    pand %xmm2, %xmm0
 ; X86-SSE2-NEXT:    pandn %xmm1, %xmm2
 ; X86-SSE2-NEXT:    por %xmm0, %xmm2
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
 ; X86-SSE2-NEXT:    movdqa %xmm2, %xmm1
 ; X86-SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
 ; X86-SSE2-NEXT:    pand %xmm1, %xmm2
@@ -1325,7 +1325,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X86-SSE42-NEXT:    pmaxsd %xmm0, %xmm1
 ; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; X86-SSE42-NEXT:    pmaxsd %xmm1, %xmm0
-; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X86-SSE42-NEXT:    pmaxsd %xmm0, %xmm1
 ; X86-SSE42-NEXT:    movd %xmm1, %eax
 ; X86-SSE42-NEXT:    retl
@@ -1339,7 +1339,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X86-AVX1-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-AVX1-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
-; X86-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X86-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X86-AVX1-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vmovd %xmm0, %eax
 ; X86-AVX1-NEXT:    vzeroupper
@@ -1352,7 +1352,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X86-AVX2-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; X86-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-AVX2-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
-; X86-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X86-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X86-AVX2-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; X86-AVX2-NEXT:    vmovd %xmm0, %eax
 ; X86-AVX2-NEXT:    vzeroupper
@@ -1381,7 +1381,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X64-SSE2-NEXT:    pand %xmm2, %xmm0
 ; X64-SSE2-NEXT:    pandn %xmm1, %xmm2
 ; X64-SSE2-NEXT:    por %xmm0, %xmm2
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
 ; X64-SSE2-NEXT:    movdqa %xmm2, %xmm1
 ; X64-SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
 ; X64-SSE2-NEXT:    pand %xmm1, %xmm2
@@ -1397,7 +1397,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X64-SSE42-NEXT:    pmaxsd %xmm0, %xmm1
 ; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; X64-SSE42-NEXT:    pmaxsd %xmm1, %xmm0
-; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X64-SSE42-NEXT:    pmaxsd %xmm0, %xmm1
 ; X64-SSE42-NEXT:    movd %xmm1, %eax
 ; X64-SSE42-NEXT:    retq
@@ -1411,7 +1411,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X64-AVX1-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX1-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X64-AVX1-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vmovd %xmm0, %eax
 ; X64-AVX1-NEXT:    vzeroupper
@@ -1424,7 +1424,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X64-AVX2-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; X64-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX2-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
-; X64-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X64-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X64-AVX2-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; X64-AVX2-NEXT:    vmovd %xmm0, %eax
 ; X64-AVX2-NEXT:    vzeroupper
@@ -1438,7 +1438,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X64-AVX512-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX512-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
-; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X64-AVX512-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vmovd %xmm0, %eax
 ; X64-AVX512-NEXT:    vzeroupper
@@ -1467,7 +1467,7 @@ define i16 @test_reduce_v32i16(<32 x i16> %a0) {
 ; X86-SSE2-NEXT:    pmaxsw %xmm0, %xmm1
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; X86-SSE2-NEXT:    pmaxsw %xmm1, %xmm0
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X86-SSE2-NEXT:    pmaxsw %xmm0, %xmm1
 ; X86-SSE2-NEXT:    movdqa %xmm1, %xmm0
 ; X86-SSE2-NEXT:    psrld $16, %xmm0
@@ -1523,7 +1523,7 @@ define i16 @test_reduce_v32i16(<32 x i16> %a0) {
 ; X64-SSE2-NEXT:    pmaxsw %xmm0, %xmm1
 ; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; X64-SSE2-NEXT:    pmaxsw %xmm1, %xmm0
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X64-SSE2-NEXT:    pmaxsw %xmm0, %xmm1
 ; X64-SSE2-NEXT:    movdqa %xmm1, %xmm0
 ; X64-SSE2-NEXT:    psrld $16, %xmm0
@@ -1628,7 +1628,7 @@ define i8 @test_reduce_v64i8(<64 x i8> %a0) {
 ; X86-SSE2-NEXT:    pand %xmm2, %xmm0
 ; X86-SSE2-NEXT:    pandn %xmm1, %xmm2
 ; X86-SSE2-NEXT:    por %xmm0, %xmm2
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
 ; X86-SSE2-NEXT:    movdqa %xmm2, %xmm1
 ; X86-SSE2-NEXT:    pcmpgtb %xmm0, %xmm1
 ; X86-SSE2-NEXT:    pand %xmm1, %xmm2
@@ -1722,7 +1722,7 @@ define i8 @test_reduce_v64i8(<64 x i8> %a0) {
 ; X64-SSE2-NEXT:    pand %xmm2, %xmm0
 ; X64-SSE2-NEXT:    pandn %xmm1, %xmm2
 ; X64-SSE2-NEXT:    por %xmm0, %xmm2
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
 ; X64-SSE2-NEXT:    movdqa %xmm2, %xmm1
 ; X64-SSE2-NEXT:    pcmpgtb %xmm0, %xmm1
 ; X64-SSE2-NEXT:    pand %xmm1, %xmm2
@@ -1839,7 +1839,7 @@ define i16 @test_reduce_v16i16_v8i16(<16 x i16> %a0) {
 ; X86-SSE2:       ## %bb.0:
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    pmaxsw %xmm0, %xmm1
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X86-SSE2-NEXT:    pmaxsw %xmm1, %xmm0
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X86-SSE2-NEXT:    psrld $16, %xmm1
@@ -1871,7 +1871,7 @@ define i16 @test_reduce_v16i16_v8i16(<16 x i16> %a0) {
 ; X64-SSE2:       ## %bb.0:
 ; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    pmaxsw %xmm0, %xmm1
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X64-SSE2-NEXT:    pmaxsw %xmm1, %xmm0
 ; X64-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X64-SSE2-NEXT:    psrld $16, %xmm1
@@ -1916,7 +1916,7 @@ define i16 @test_reduce_v32i16_v8i16(<32 x i16> %a0) {
 ; X86-SSE2:       ## %bb.0:
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    pmaxsw %xmm0, %xmm1
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X86-SSE2-NEXT:    pmaxsw %xmm1, %xmm0
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X86-SSE2-NEXT:    psrld $16, %xmm1
@@ -1948,7 +1948,7 @@ define i16 @test_reduce_v32i16_v8i16(<32 x i16> %a0) {
 ; X64-SSE2:       ## %bb.0:
 ; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    pmaxsw %xmm0, %xmm1
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X64-SSE2-NEXT:    pmaxsw %xmm1, %xmm0
 ; X64-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X64-SSE2-NEXT:    psrld $16, %xmm1
@@ -1997,7 +1997,7 @@ define i8 @test_reduce_v32i8_v16i8(<32 x i8> %a0) {
 ; X86-SSE2-NEXT:    pand %xmm2, %xmm0
 ; X86-SSE2-NEXT:    pandn %xmm1, %xmm2
 ; X86-SSE2-NEXT:    por %xmm0, %xmm2
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
 ; X86-SSE2-NEXT:    movdqa %xmm2, %xmm1
 ; X86-SSE2-NEXT:    pcmpgtb %xmm0, %xmm1
 ; X86-SSE2-NEXT:    pand %xmm1, %xmm2
@@ -2053,7 +2053,7 @@ define i8 @test_reduce_v32i8_v16i8(<32 x i8> %a0) {
 ; X64-SSE2-NEXT:    pand %xmm2, %xmm0
 ; X64-SSE2-NEXT:    pandn %xmm1, %xmm2
 ; X64-SSE2-NEXT:    por %xmm0, %xmm2
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
 ; X64-SSE2-NEXT:    movdqa %xmm2, %xmm1
 ; X64-SSE2-NEXT:    pcmpgtb %xmm0, %xmm1
 ; X64-SSE2-NEXT:    pand %xmm1, %xmm2
@@ -2125,7 +2125,7 @@ define i8 @test_reduce_v64i8_v16i8(<64 x i8> %a0) {
 ; X86-SSE2-NEXT:    pand %xmm2, %xmm0
 ; X86-SSE2-NEXT:    pandn %xmm1, %xmm2
 ; X86-SSE2-NEXT:    por %xmm0, %xmm2
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
 ; X86-SSE2-NEXT:    movdqa %xmm2, %xmm1
 ; X86-SSE2-NEXT:    pcmpgtb %xmm0, %xmm1
 ; X86-SSE2-NEXT:    pand %xmm1, %xmm2
@@ -2181,7 +2181,7 @@ define i8 @test_reduce_v64i8_v16i8(<64 x i8> %a0) {
 ; X64-SSE2-NEXT:    pand %xmm2, %xmm0
 ; X64-SSE2-NEXT:    pandn %xmm1, %xmm2
 ; X64-SSE2-NEXT:    por %xmm0, %xmm2
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
 ; X64-SSE2-NEXT:    movdqa %xmm2, %xmm1
 ; X64-SSE2-NEXT:    pcmpgtb %xmm0, %xmm1
 ; X64-SSE2-NEXT:    pand %xmm1, %xmm2
diff --git a/llvm/test/CodeGen/X86/horizontal-reduce-smin.ll b/llvm/test/CodeGen/X86/horizontal-reduce-smin.ll
index 17c3a9fd4a010..9564efe2d08fc 100644
--- a/llvm/test/CodeGen/X86/horizontal-reduce-smin.ll
+++ b/llvm/test/CodeGen/X86/horizontal-reduce-smin.ll
@@ -33,7 +33,7 @@ define i64 @test_reduce_v2i64(<2 x i64> %a0) {
 ; X86-SSE2-NEXT:    pandn %xmm1, %xmm3
 ; X86-SSE2-NEXT:    por %xmm0, %xmm3
 ; X86-SSE2-NEXT:    movd %xmm3, %eax
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,1,1]
 ; X86-SSE2-NEXT:    movd %xmm0, %edx
 ; X86-SSE2-NEXT:    retl
 ;
@@ -126,7 +126,7 @@ define i32 @test_reduce_v4i32(<4 x i32> %a0) {
 ; X86-SSE2-NEXT:    pand %xmm2, %xmm0
 ; X86-SSE2-NEXT:    pandn %xmm1, %xmm2
 ; X86-SSE2-NEXT:    por %xmm0, %xmm2
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X86-SSE2-NEXT:    pcmpgtd %xmm2, %xmm1
 ; X86-SSE2-NEXT:    pand %xmm1, %xmm2
@@ -139,7 +139,7 @@ define i32 @test_reduce_v4i32(<4 x i32> %a0) {
 ; X86-SSE42:       ## %bb.0:
 ; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE42-NEXT:    pminsd %xmm0, %xmm1
-; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X86-SSE42-NEXT:    pminsd %xmm1, %xmm0
 ; X86-SSE42-NEXT:    movd %xmm0, %eax
 ; X86-SSE42-NEXT:    retl
@@ -148,7 +148,7 @@ define i32 @test_reduce_v4i32(<4 x i32> %a0) {
 ; X86-AVX:       ## %bb.0:
 ; X86-AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-AVX-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
-; X86-AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X86-AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X86-AVX-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vmovd %xmm0, %eax
 ; X86-AVX-NEXT:    retl
@@ -161,7 +161,7 @@ define i32 @test_reduce_v4i32(<4 x i32> %a0) {
 ; X64-SSE2-NEXT:    pand %xmm2, %xmm0
 ; X64-SSE2-NEXT:    pandn %xmm1, %xmm2
 ; X64-SSE2-NEXT:    por %xmm0, %xmm2
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
 ; X64-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X64-SSE2-NEXT:    pcmpgtd %xmm2, %xmm1
 ; X64-SSE2-NEXT:    pand %xmm1, %xmm2
@@ -174,7 +174,7 @@ define i32 @test_reduce_v4i32(<4 x i32> %a0) {
 ; X64-SSE42:       ## %bb.0:
 ; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE42-NEXT:    pminsd %xmm0, %xmm1
-; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X64-SSE42-NEXT:    pminsd %xmm1, %xmm0
 ; X64-SSE42-NEXT:    movd %xmm0, %eax
 ; X64-SSE42-NEXT:    retq
@@ -183,7 +183,7 @@ define i32 @test_reduce_v4i32(<4 x i32> %a0) {
 ; X64-AVX:       ## %bb.0:
 ; X64-AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
-; X64-AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X64-AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X64-AVX-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; X64-AVX-NEXT:    vmovd %xmm0, %eax
 ; X64-AVX-NEXT:    retq
@@ -202,7 +202,7 @@ define i16 @test_reduce_v8i16(<8 x i16> %a0) {
 ; X86-SSE2:       ## %bb.0:
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    pminsw %xmm0, %xmm1
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X86-SSE2-NEXT:    pminsw %xmm1, %xmm0
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X86-SSE2-NEXT:    psrld $16, %xmm1
@@ -233,7 +233,7 @@ define i16 @test_reduce_v8i16(<8 x i16> %a0) {
 ; X64-SSE2:       ## %bb.0:
 ; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    pminsw %xmm0, %xmm1
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X64-SSE2-NEXT:    pminsw %xmm1, %xmm0
 ; X64-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X64-SSE2-NEXT:    psrld $16, %xmm1
@@ -281,7 +281,7 @@ define i8 @test_reduce_v16i8(<16 x i8> %a0) {
 ; X86-SSE2-NEXT:    pand %xmm2, %xmm0
 ; X86-SSE2-NEXT:    pandn %xmm1, %xmm2
 ; X86-SSE2-NEXT:    por %xmm0, %xmm2
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X86-SSE2-NEXT:    pcmpgtb %xmm2, %xmm1
 ; X86-SSE2-NEXT:    pand %xmm1, %xmm2
@@ -336,7 +336,7 @@ define i8 @test_reduce_v16i8(<16 x i8> %a0) {
 ; X64-SSE2-NEXT:    pand %xmm2, %xmm0
 ; X64-SSE2-NEXT:    pandn %xmm1, %xmm2
 ; X64-SSE2-NEXT:    por %xmm0, %xmm2
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
 ; X64-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X64-SSE2-NEXT:    pcmpgtb %xmm2, %xmm1
 ; X64-SSE2-NEXT:    pand %xmm1, %xmm2
@@ -437,7 +437,7 @@ define i64 @test_reduce_v4i64(<4 x i64> %a0) {
 ; X86-SSE2-NEXT:    pandn %xmm0, %xmm2
 ; X86-SSE2-NEXT:    por %xmm4, %xmm2
 ; X86-SSE2-NEXT:    movd %xmm2, %eax
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
 ; X86-SSE2-NEXT:    movd %xmm0, %edx
 ; X86-SSE2-NEXT:    retl
 ;
@@ -587,7 +587,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X86-SSE2-NEXT:    pand %xmm1, %xmm2
 ; X86-SSE2-NEXT:    pandn %xmm0, %xmm1
 ; X86-SSE2-NEXT:    por %xmm2, %xmm1
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; X86-SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
 ; X86-SSE2-NEXT:    pand %xmm2, %xmm1
@@ -601,7 +601,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X86-SSE42-NEXT:    pminsd %xmm1, %xmm0
 ; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE42-NEXT:    pminsd %xmm0, %xmm1
-; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X86-SSE42-NEXT:    pminsd %xmm1, %xmm0
 ; X86-SSE42-NEXT:    movd %xmm0, %eax
 ; X86-SSE42-NEXT:    retl
@@ -612,7 +612,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X86-AVX1-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-AVX1-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
-; X86-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X86-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X86-AVX1-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vmovd %xmm0, %eax
 ; X86-AVX1-NEXT:    vzeroupper
@@ -624,7 +624,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X86-AVX2-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; X86-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-AVX2-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
-; X86-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X86-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X86-AVX2-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; X86-AVX2-NEXT:    vmovd %xmm0, %eax
 ; X86-AVX2-NEXT:    vzeroupper
@@ -643,7 +643,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X64-SSE2-NEXT:    pand %xmm1, %xmm2
 ; X64-SSE2-NEXT:    pandn %xmm0, %xmm1
 ; X64-SSE2-NEXT:    por %xmm2, %xmm1
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X64-SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; X64-SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
 ; X64-SSE2-NEXT:    pand %xmm2, %xmm1
@@ -657,7 +657,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X64-SSE42-NEXT:    pminsd %xmm1, %xmm0
 ; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE42-NEXT:    pminsd %xmm0, %xmm1
-; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X64-SSE42-NEXT:    pminsd %xmm1, %xmm0
 ; X64-SSE42-NEXT:    movd %xmm0, %eax
 ; X64-SSE42-NEXT:    retq
@@ -668,7 +668,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X64-AVX1-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX1-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X64-AVX1-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vmovd %xmm0, %eax
 ; X64-AVX1-NEXT:    vzeroupper
@@ -680,7 +680,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X64-AVX2-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; X64-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX2-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
-; X64-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X64-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X64-AVX2-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; X64-AVX2-NEXT:    vmovd %xmm0, %eax
 ; X64-AVX2-NEXT:    vzeroupper
@@ -692,7 +692,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X64-AVX512-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX512-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
-; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X64-AVX512-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vmovd %xmm0, %eax
 ; X64-AVX512-NEXT:    vzeroupper
@@ -716,7 +716,7 @@ define i16 @test_reduce_v16i16(<16 x i16> %a0) {
 ; X86-SSE2-NEXT:    pminsw %xmm1, %xmm0
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    pminsw %xmm0, %xmm1
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X86-SSE2-NEXT:    pminsw %xmm1, %xmm0
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X86-SSE2-NEXT:    psrld $16, %xmm1
@@ -764,7 +764,7 @@ define i16 @test_reduce_v16i16(<16 x i16> %a0) {
 ; X64-SSE2-NEXT:    pminsw %xmm1, %xmm0
 ; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    pminsw %xmm0, %xmm1
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X64-SSE2-NEXT:    pminsw %xmm1, %xmm0
 ; X64-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X64-SSE2-NEXT:    psrld $16, %xmm1
@@ -848,7 +848,7 @@ define i8 @test_reduce_v32i8(<32 x i8> %a0) {
 ; X86-SSE2-NEXT:    pand %xmm1, %xmm2
 ; X86-SSE2-NEXT:    pandn %xmm0, %xmm1
 ; X86-SSE2-NEXT:    por %xmm2, %xmm1
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; X86-SSE2-NEXT:    pcmpgtb %xmm1, %xmm2
 ; X86-SSE2-NEXT:    pand %xmm2, %xmm1
@@ -926,7 +926,7 @@ define i8 @test_reduce_v32i8(<32 x i8> %a0) {
 ; X64-SSE2-NEXT:    pand %xmm1, %xmm2
 ; X64-SSE2-NEXT:    pandn %xmm0, %xmm1
 ; X64-SSE2-NEXT:    por %xmm2, %xmm1
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X64-SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; X64-SSE2-NEXT:    pcmpgtb %xmm1, %xmm2
 ; X64-SSE2-NEXT:    pand %xmm2, %xmm1
@@ -1092,7 +1092,7 @@ define i64 @test_reduce_v8i64(<8 x i64> %a0) {
 ; X86-SSE2-NEXT:    pandn %xmm1, %xmm2
 ; X86-SSE2-NEXT:    por %xmm0, %xmm2
 ; X86-SSE2-NEXT:    movd %xmm2, %eax
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
 ; X86-SSE2-NEXT:    movd %xmm0, %edx
 ; X86-SSE2-NEXT:    retl
 ;
@@ -1313,7 +1313,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X86-SSE2-NEXT:    pand %xmm2, %xmm1
 ; X86-SSE2-NEXT:    pandn %xmm0, %xmm2
 ; X86-SSE2-NEXT:    por %xmm1, %xmm2
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X86-SSE2-NEXT:    pcmpgtd %xmm2, %xmm1
 ; X86-SSE2-NEXT:    pand %xmm1, %xmm2
@@ -1329,7 +1329,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X86-SSE42-NEXT:    pminsd %xmm0, %xmm1
 ; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; X86-SSE42-NEXT:    pminsd %xmm1, %xmm0
-; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X86-SSE42-NEXT:    pminsd %xmm0, %xmm1
 ; X86-SSE42-NEXT:    movd %xmm1, %eax
 ; X86-SSE42-NEXT:    retl
@@ -1343,7 +1343,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X86-AVX1-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-AVX1-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
-; X86-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X86-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X86-AVX1-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vmovd %xmm0, %eax
 ; X86-AVX1-NEXT:    vzeroupper
@@ -1356,7 +1356,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X86-AVX2-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; X86-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-AVX2-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
-; X86-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X86-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X86-AVX2-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; X86-AVX2-NEXT:    vmovd %xmm0, %eax
 ; X86-AVX2-NEXT:    vzeroupper
@@ -1385,7 +1385,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X64-SSE2-NEXT:    pand %xmm2, %xmm1
 ; X64-SSE2-NEXT:    pandn %xmm0, %xmm2
 ; X64-SSE2-NEXT:    por %xmm1, %xmm2
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
 ; X64-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X64-SSE2-NEXT:    pcmpgtd %xmm2, %xmm1
 ; X64-SSE2-NEXT:    pand %xmm1, %xmm2
@@ -1401,7 +1401,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X64-SSE42-NEXT:    pminsd %xmm0, %xmm1
 ; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; X64-SSE42-NEXT:    pminsd %xmm1, %xmm0
-; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X64-SSE42-NEXT:    pminsd %xmm0, %xmm1
 ; X64-SSE42-NEXT:    movd %xmm1, %eax
 ; X64-SSE42-NEXT:    retq
@@ -1415,7 +1415,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X64-AVX1-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX1-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X64-AVX1-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vmovd %xmm0, %eax
 ; X64-AVX1-NEXT:    vzeroupper
@@ -1428,7 +1428,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X64-AVX2-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; X64-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX2-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
-; X64-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X64-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X64-AVX2-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; X64-AVX2-NEXT:    vmovd %xmm0, %eax
 ; X64-AVX2-NEXT:    vzeroupper
@@ -1442,7 +1442,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X64-AVX512-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX512-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
-; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X64-AVX512-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vmovd %xmm0, %eax
 ; X64-AVX512-NEXT:    vzeroupper
@@ -1471,7 +1471,7 @@ define i16 @test_reduce_v32i16(<32 x i16> %a0) {
 ; X86-SSE2-NEXT:    pminsw %xmm0, %xmm1
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; X86-SSE2-NEXT:    pminsw %xmm1, %xmm0
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X86-SSE2-NEXT:    pminsw %xmm0, %xmm1
 ; X86-SSE2-NEXT:    movdqa %xmm1, %xmm0
 ; X86-SSE2-NEXT:    psrld $16, %xmm0
@@ -1527,7 +1527,7 @@ define i16 @test_reduce_v32i16(<32 x i16> %a0) {
 ; X64-SSE2-NEXT:    pminsw %xmm0, %xmm1
 ; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; X64-SSE2-NEXT:    pminsw %xmm1, %xmm0
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X64-SSE2-NEXT:    pminsw %xmm0, %xmm1
 ; X64-SSE2-NEXT:    movdqa %xmm1, %xmm0
 ; X64-SSE2-NEXT:    psrld $16, %xmm0
@@ -1632,7 +1632,7 @@ define i8 @test_reduce_v64i8(<64 x i8> %a0) {
 ; X86-SSE2-NEXT:    pand %xmm2, %xmm1
 ; X86-SSE2-NEXT:    pandn %xmm0, %xmm2
 ; X86-SSE2-NEXT:    por %xmm1, %xmm2
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X86-SSE2-NEXT:    pcmpgtb %xmm2, %xmm1
 ; X86-SSE2-NEXT:    pand %xmm1, %xmm2
@@ -1726,7 +1726,7 @@ define i8 @test_reduce_v64i8(<64 x i8> %a0) {
 ; X64-SSE2-NEXT:    pand %xmm2, %xmm1
 ; X64-SSE2-NEXT:    pandn %xmm0, %xmm2
 ; X64-SSE2-NEXT:    por %xmm1, %xmm2
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
 ; X64-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X64-SSE2-NEXT:    pcmpgtb %xmm2, %xmm1
 ; X64-SSE2-NEXT:    pand %xmm1, %xmm2
@@ -1843,7 +1843,7 @@ define i16 @test_reduce_v16i16_v8i16(<16 x i16> %a0) {
 ; X86-SSE2:       ## %bb.0:
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    pminsw %xmm0, %xmm1
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X86-SSE2-NEXT:    pminsw %xmm1, %xmm0
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X86-SSE2-NEXT:    psrld $16, %xmm1
@@ -1875,7 +1875,7 @@ define i16 @test_reduce_v16i16_v8i16(<16 x i16> %a0) {
 ; X64-SSE2:       ## %bb.0:
 ; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    pminsw %xmm0, %xmm1
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X64-SSE2-NEXT:    pminsw %xmm1, %xmm0
 ; X64-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X64-SSE2-NEXT:    psrld $16, %xmm1
@@ -1920,7 +1920,7 @@ define i16 @test_reduce_v32i16_v8i16(<32 x i16> %a0) {
 ; X86-SSE2:       ## %bb.0:
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    pminsw %xmm0, %xmm1
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X86-SSE2-NEXT:    pminsw %xmm1, %xmm0
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X86-SSE2-NEXT:    psrld $16, %xmm1
@@ -1952,7 +1952,7 @@ define i16 @test_reduce_v32i16_v8i16(<32 x i16> %a0) {
 ; X64-SSE2:       ## %bb.0:
 ; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    pminsw %xmm0, %xmm1
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X64-SSE2-NEXT:    pminsw %xmm1, %xmm0
 ; X64-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X64-SSE2-NEXT:    psrld $16, %xmm1
@@ -2001,7 +2001,7 @@ define i8 @test_reduce_v32i8_v16i8(<32 x i8> %a0) {
 ; X86-SSE2-NEXT:    pand %xmm2, %xmm0
 ; X86-SSE2-NEXT:    pandn %xmm1, %xmm2
 ; X86-SSE2-NEXT:    por %xmm0, %xmm2
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X86-SSE2-NEXT:    pcmpgtb %xmm2, %xmm1
 ; X86-SSE2-NEXT:    pand %xmm1, %xmm2
@@ -2057,7 +2057,7 @@ define i8 @test_reduce_v32i8_v16i8(<32 x i8> %a0) {
 ; X64-SSE2-NEXT:    pand %xmm2, %xmm0
 ; X64-SSE2-NEXT:    pandn %xmm1, %xmm2
 ; X64-SSE2-NEXT:    por %xmm0, %xmm2
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
 ; X64-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X64-SSE2-NEXT:    pcmpgtb %xmm2, %xmm1
 ; X64-SSE2-NEXT:    pand %xmm1, %xmm2
@@ -2129,7 +2129,7 @@ define i8 @test_reduce_v64i8_v16i8(<64 x i8> %a0) {
 ; X86-SSE2-NEXT:    pand %xmm2, %xmm0
 ; X86-SSE2-NEXT:    pandn %xmm1, %xmm2
 ; X86-SSE2-NEXT:    por %xmm0, %xmm2
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X86-SSE2-NEXT:    pcmpgtb %xmm2, %xmm1
 ; X86-SSE2-NEXT:    pand %xmm1, %xmm2
@@ -2185,7 +2185,7 @@ define i8 @test_reduce_v64i8_v16i8(<64 x i8> %a0) {
 ; X64-SSE2-NEXT:    pand %xmm2, %xmm0
 ; X64-SSE2-NEXT:    pandn %xmm1, %xmm2
 ; X64-SSE2-NEXT:    por %xmm0, %xmm2
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
 ; X64-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X64-SSE2-NEXT:    pcmpgtb %xmm2, %xmm1
 ; X64-SSE2-NEXT:    pand %xmm1, %xmm2
diff --git a/llvm/test/CodeGen/X86/horizontal-reduce-umax.ll b/llvm/test/CodeGen/X86/horizontal-reduce-umax.ll
index c69551aa3d883..5faf061997786 100644
--- a/llvm/test/CodeGen/X86/horizontal-reduce-umax.ll
+++ b/llvm/test/CodeGen/X86/horizontal-reduce-umax.ll
@@ -33,7 +33,7 @@ define i64 @test_reduce_v2i64(<2 x i64> %a0) {
 ; X86-SSE2-NEXT:    pandn %xmm1, %xmm3
 ; X86-SSE2-NEXT:    por %xmm0, %xmm3
 ; X86-SSE2-NEXT:    movd %xmm3, %eax
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,1,1]
 ; X86-SSE2-NEXT:    movd %xmm0, %edx
 ; X86-SSE2-NEXT:    retl
 ;
@@ -156,7 +156,7 @@ define i32 @test_reduce_v4i32(<4 x i32> %a0) {
 ; X86-SSE2-NEXT:    pand %xmm3, %xmm0
 ; X86-SSE2-NEXT:    pandn %xmm1, %xmm3
 ; X86-SSE2-NEXT:    por %xmm0, %xmm3
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,1,1]
 ; X86-SSE2-NEXT:    movdqa %xmm3, %xmm1
 ; X86-SSE2-NEXT:    pxor %xmm2, %xmm1
 ; X86-SSE2-NEXT:    pxor %xmm0, %xmm2
@@ -171,7 +171,7 @@ define i32 @test_reduce_v4i32(<4 x i32> %a0) {
 ; X86-SSE42:       ## %bb.0:
 ; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE42-NEXT:    pmaxud %xmm0, %xmm1
-; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X86-SSE42-NEXT:    pmaxud %xmm1, %xmm0
 ; X86-SSE42-NEXT:    movd %xmm0, %eax
 ; X86-SSE42-NEXT:    retl
@@ -180,7 +180,7 @@ define i32 @test_reduce_v4i32(<4 x i32> %a0) {
 ; X86-AVX:       ## %bb.0:
 ; X86-AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-AVX-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
-; X86-AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X86-AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X86-AVX-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vmovd %xmm0, %eax
 ; X86-AVX-NEXT:    retl
@@ -197,7 +197,7 @@ define i32 @test_reduce_v4i32(<4 x i32> %a0) {
 ; X64-SSE2-NEXT:    pand %xmm3, %xmm0
 ; X64-SSE2-NEXT:    pandn %xmm1, %xmm3
 ; X64-SSE2-NEXT:    por %xmm0, %xmm3
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,2,3]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,1,1]
 ; X64-SSE2-NEXT:    movdqa %xmm3, %xmm1
 ; X64-SSE2-NEXT:    pxor %xmm2, %xmm1
 ; X64-SSE2-NEXT:    pxor %xmm0, %xmm2
@@ -212,7 +212,7 @@ define i32 @test_reduce_v4i32(<4 x i32> %a0) {
 ; X64-SSE42:       ## %bb.0:
 ; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE42-NEXT:    pmaxud %xmm0, %xmm1
-; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X64-SSE42-NEXT:    pmaxud %xmm1, %xmm0
 ; X64-SSE42-NEXT:    movd %xmm0, %eax
 ; X64-SSE42-NEXT:    retq
@@ -221,7 +221,7 @@ define i32 @test_reduce_v4i32(<4 x i32> %a0) {
 ; X64-AVX:       ## %bb.0:
 ; X64-AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
-; X64-AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X64-AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X64-AVX-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; X64-AVX-NEXT:    vmovd %xmm0, %eax
 ; X64-AVX-NEXT:    retq
@@ -243,7 +243,7 @@ define i16 @test_reduce_v8i16(<8 x i16> %a0) {
 ; X86-SSE2-NEXT:    pxor %xmm2, %xmm0
 ; X86-SSE2-NEXT:    pxor %xmm2, %xmm1
 ; X86-SSE2-NEXT:    pmaxsw %xmm0, %xmm1
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X86-SSE2-NEXT:    pmaxsw %xmm1, %xmm0
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X86-SSE2-NEXT:    psrld $16, %xmm1
@@ -280,7 +280,7 @@ define i16 @test_reduce_v8i16(<8 x i16> %a0) {
 ; X64-SSE2-NEXT:    pxor %xmm2, %xmm0
 ; X64-SSE2-NEXT:    pxor %xmm2, %xmm1
 ; X64-SSE2-NEXT:    pmaxsw %xmm0, %xmm1
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X64-SSE2-NEXT:    pmaxsw %xmm1, %xmm0
 ; X64-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X64-SSE2-NEXT:    psrld $16, %xmm1
@@ -346,7 +346,7 @@ define i8 @test_reduce_v16i8(<16 x i8> %a0) {
 ; X86-SSE2:       ## %bb.0:
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    pmaxub %xmm0, %xmm1
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X86-SSE2-NEXT:    pmaxub %xmm1, %xmm0
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X86-SSE2-NEXT:    psrld $16, %xmm1
@@ -387,7 +387,7 @@ define i8 @test_reduce_v16i8(<16 x i8> %a0) {
 ; X64-SSE2:       ## %bb.0:
 ; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    pmaxub %xmm0, %xmm1
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X64-SSE2-NEXT:    pmaxub %xmm1, %xmm0
 ; X64-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X64-SSE2-NEXT:    psrld $16, %xmm1
@@ -501,7 +501,7 @@ define i64 @test_reduce_v4i64(<4 x i64> %a0) {
 ; X86-SSE2-NEXT:    pandn %xmm0, %xmm2
 ; X86-SSE2-NEXT:    por %xmm4, %xmm2
 ; X86-SSE2-NEXT:    movd %xmm2, %eax
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
 ; X86-SSE2-NEXT:    movd %xmm0, %edx
 ; X86-SSE2-NEXT:    retl
 ;
@@ -689,7 +689,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X86-SSE2-NEXT:    pand %xmm1, %xmm4
 ; X86-SSE2-NEXT:    pandn %xmm0, %xmm1
 ; X86-SSE2-NEXT:    por %xmm4, %xmm1
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X86-SSE2-NEXT:    movdqa %xmm1, %xmm3
 ; X86-SSE2-NEXT:    pxor %xmm2, %xmm3
 ; X86-SSE2-NEXT:    pxor %xmm0, %xmm2
@@ -705,7 +705,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X86-SSE42-NEXT:    pmaxud %xmm1, %xmm0
 ; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE42-NEXT:    pmaxud %xmm0, %xmm1
-; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X86-SSE42-NEXT:    pmaxud %xmm1, %xmm0
 ; X86-SSE42-NEXT:    movd %xmm0, %eax
 ; X86-SSE42-NEXT:    retl
@@ -716,7 +716,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X86-AVX1-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-AVX1-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
-; X86-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X86-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X86-AVX1-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vmovd %xmm0, %eax
 ; X86-AVX1-NEXT:    vzeroupper
@@ -728,7 +728,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X86-AVX2-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; X86-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-AVX2-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
-; X86-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X86-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X86-AVX2-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; X86-AVX2-NEXT:    vmovd %xmm0, %eax
 ; X86-AVX2-NEXT:    vzeroupper
@@ -754,7 +754,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X64-SSE2-NEXT:    pand %xmm1, %xmm4
 ; X64-SSE2-NEXT:    pandn %xmm0, %xmm1
 ; X64-SSE2-NEXT:    por %xmm4, %xmm1
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X64-SSE2-NEXT:    movdqa %xmm1, %xmm3
 ; X64-SSE2-NEXT:    pxor %xmm2, %xmm3
 ; X64-SSE2-NEXT:    pxor %xmm0, %xmm2
@@ -770,7 +770,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X64-SSE42-NEXT:    pmaxud %xmm1, %xmm0
 ; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE42-NEXT:    pmaxud %xmm0, %xmm1
-; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X64-SSE42-NEXT:    pmaxud %xmm1, %xmm0
 ; X64-SSE42-NEXT:    movd %xmm0, %eax
 ; X64-SSE42-NEXT:    retq
@@ -781,7 +781,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X64-AVX1-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX1-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X64-AVX1-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vmovd %xmm0, %eax
 ; X64-AVX1-NEXT:    vzeroupper
@@ -793,7 +793,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X64-AVX2-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; X64-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX2-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
-; X64-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X64-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X64-AVX2-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; X64-AVX2-NEXT:    vmovd %xmm0, %eax
 ; X64-AVX2-NEXT:    vzeroupper
@@ -805,7 +805,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X64-AVX512-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX512-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
-; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X64-AVX512-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vmovd %xmm0, %eax
 ; X64-AVX512-NEXT:    vzeroupper
@@ -832,7 +832,7 @@ define i16 @test_reduce_v16i16(<16 x i16> %a0) {
 ; X86-SSE2-NEXT:    pmaxsw %xmm1, %xmm0
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    pmaxsw %xmm0, %xmm1
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X86-SSE2-NEXT:    pmaxsw %xmm1, %xmm0
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X86-SSE2-NEXT:    psrld $16, %xmm1
@@ -887,7 +887,7 @@ define i16 @test_reduce_v16i16(<16 x i16> %a0) {
 ; X64-SSE2-NEXT:    pmaxsw %xmm1, %xmm0
 ; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    pmaxsw %xmm0, %xmm1
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X64-SSE2-NEXT:    pmaxsw %xmm1, %xmm0
 ; X64-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X64-SSE2-NEXT:    psrld $16, %xmm1
@@ -967,7 +967,7 @@ define i8 @test_reduce_v32i8(<32 x i8> %a0) {
 ; X86-SSE2-NEXT:    pmaxub %xmm1, %xmm0
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    pmaxub %xmm0, %xmm1
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X86-SSE2-NEXT:    pmaxub %xmm1, %xmm0
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X86-SSE2-NEXT:    psrld $16, %xmm1
@@ -1028,7 +1028,7 @@ define i8 @test_reduce_v32i8(<32 x i8> %a0) {
 ; X64-SSE2-NEXT:    pmaxub %xmm1, %xmm0
 ; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    pmaxub %xmm0, %xmm1
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X64-SSE2-NEXT:    pmaxub %xmm1, %xmm0
 ; X64-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X64-SSE2-NEXT:    psrld $16, %xmm1
@@ -1185,7 +1185,7 @@ define i64 @test_reduce_v8i64(<8 x i64> %a0) {
 ; X86-SSE2-NEXT:    pandn %xmm0, %xmm2
 ; X86-SSE2-NEXT:    por %xmm1, %xmm2
 ; X86-SSE2-NEXT:    movd %xmm2, %eax
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
 ; X86-SSE2-NEXT:    movd %xmm0, %edx
 ; X86-SSE2-NEXT:    retl
 ;
@@ -1474,7 +1474,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X86-SSE2-NEXT:    pand %xmm2, %xmm1
 ; X86-SSE2-NEXT:    pandn %xmm0, %xmm2
 ; X86-SSE2-NEXT:    por %xmm1, %xmm2
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
 ; X86-SSE2-NEXT:    movdqa %xmm2, %xmm1
 ; X86-SSE2-NEXT:    pxor %xmm4, %xmm1
 ; X86-SSE2-NEXT:    pxor %xmm0, %xmm4
@@ -1492,7 +1492,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X86-SSE42-NEXT:    pmaxud %xmm0, %xmm1
 ; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; X86-SSE42-NEXT:    pmaxud %xmm1, %xmm0
-; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X86-SSE42-NEXT:    pmaxud %xmm0, %xmm1
 ; X86-SSE42-NEXT:    movd %xmm1, %eax
 ; X86-SSE42-NEXT:    retl
@@ -1506,7 +1506,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X86-AVX1-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-AVX1-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
-; X86-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X86-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X86-AVX1-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vmovd %xmm0, %eax
 ; X86-AVX1-NEXT:    vzeroupper
@@ -1519,7 +1519,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X86-AVX2-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; X86-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-AVX2-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
-; X86-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X86-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X86-AVX2-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; X86-AVX2-NEXT:    vmovd %xmm0, %eax
 ; X86-AVX2-NEXT:    vzeroupper
@@ -1561,7 +1561,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X64-SSE2-NEXT:    pand %xmm2, %xmm1
 ; X64-SSE2-NEXT:    pandn %xmm0, %xmm2
 ; X64-SSE2-NEXT:    por %xmm1, %xmm2
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
 ; X64-SSE2-NEXT:    movdqa %xmm2, %xmm1
 ; X64-SSE2-NEXT:    pxor %xmm4, %xmm1
 ; X64-SSE2-NEXT:    pxor %xmm0, %xmm4
@@ -1579,7 +1579,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X64-SSE42-NEXT:    pmaxud %xmm0, %xmm1
 ; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; X64-SSE42-NEXT:    pmaxud %xmm1, %xmm0
-; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X64-SSE42-NEXT:    pmaxud %xmm0, %xmm1
 ; X64-SSE42-NEXT:    movd %xmm1, %eax
 ; X64-SSE42-NEXT:    retq
@@ -1593,7 +1593,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X64-AVX1-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX1-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X64-AVX1-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vmovd %xmm0, %eax
 ; X64-AVX1-NEXT:    vzeroupper
@@ -1606,7 +1606,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X64-AVX2-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; X64-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX2-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
-; X64-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X64-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X64-AVX2-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; X64-AVX2-NEXT:    vmovd %xmm0, %eax
 ; X64-AVX2-NEXT:    vzeroupper
@@ -1620,7 +1620,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X64-AVX512-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX512-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
-; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X64-AVX512-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vmovd %xmm0, %eax
 ; X64-AVX512-NEXT:    vzeroupper
@@ -1654,7 +1654,7 @@ define i16 @test_reduce_v32i16(<32 x i16> %a0) {
 ; X86-SSE2-NEXT:    pmaxsw %xmm2, %xmm0
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    pmaxsw %xmm0, %xmm1
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X86-SSE2-NEXT:    pmaxsw %xmm1, %xmm0
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X86-SSE2-NEXT:    psrld $16, %xmm1
@@ -1719,7 +1719,7 @@ define i16 @test_reduce_v32i16(<32 x i16> %a0) {
 ; X64-SSE2-NEXT:    pmaxsw %xmm2, %xmm0
 ; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    pmaxsw %xmm0, %xmm1
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X64-SSE2-NEXT:    pmaxsw %xmm1, %xmm0
 ; X64-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X64-SSE2-NEXT:    psrld $16, %xmm1
@@ -1812,7 +1812,7 @@ define i8 @test_reduce_v64i8(<64 x i8> %a0) {
 ; X86-SSE2-NEXT:    pmaxub %xmm0, %xmm1
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; X86-SSE2-NEXT:    pmaxub %xmm1, %xmm0
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X86-SSE2-NEXT:    pmaxub %xmm0, %xmm1
 ; X86-SSE2-NEXT:    movdqa %xmm1, %xmm0
 ; X86-SSE2-NEXT:    psrld $16, %xmm0
@@ -1881,7 +1881,7 @@ define i8 @test_reduce_v64i8(<64 x i8> %a0) {
 ; X64-SSE2-NEXT:    pmaxub %xmm0, %xmm1
 ; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; X64-SSE2-NEXT:    pmaxub %xmm1, %xmm0
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X64-SSE2-NEXT:    pmaxub %xmm0, %xmm1
 ; X64-SSE2-NEXT:    movdqa %xmm1, %xmm0
 ; X64-SSE2-NEXT:    psrld $16, %xmm0
@@ -1992,7 +1992,7 @@ define i16 @test_reduce_v16i16_v8i16(<16 x i16> %a0) {
 ; X86-SSE2-NEXT:    pxor %xmm2, %xmm0
 ; X86-SSE2-NEXT:    pxor %xmm2, %xmm1
 ; X86-SSE2-NEXT:    pmaxsw %xmm0, %xmm1
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X86-SSE2-NEXT:    pmaxsw %xmm1, %xmm0
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X86-SSE2-NEXT:    psrld $16, %xmm1
@@ -2030,7 +2030,7 @@ define i16 @test_reduce_v16i16_v8i16(<16 x i16> %a0) {
 ; X64-SSE2-NEXT:    pxor %xmm2, %xmm0
 ; X64-SSE2-NEXT:    pxor %xmm2, %xmm1
 ; X64-SSE2-NEXT:    pmaxsw %xmm0, %xmm1
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X64-SSE2-NEXT:    pmaxsw %xmm1, %xmm0
 ; X64-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X64-SSE2-NEXT:    psrld $16, %xmm1
@@ -2102,7 +2102,7 @@ define i16 @test_reduce_v32i16_v8i16(<32 x i16> %a0) {
 ; X86-SSE2-NEXT:    pxor %xmm2, %xmm0
 ; X86-SSE2-NEXT:    pxor %xmm2, %xmm1
 ; X86-SSE2-NEXT:    pmaxsw %xmm0, %xmm1
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X86-SSE2-NEXT:    pmaxsw %xmm1, %xmm0
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X86-SSE2-NEXT:    psrld $16, %xmm1
@@ -2140,7 +2140,7 @@ define i16 @test_reduce_v32i16_v8i16(<32 x i16> %a0) {
 ; X64-SSE2-NEXT:    pxor %xmm2, %xmm0
 ; X64-SSE2-NEXT:    pxor %xmm2, %xmm1
 ; X64-SSE2-NEXT:    pmaxsw %xmm0, %xmm1
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X64-SSE2-NEXT:    pmaxsw %xmm1, %xmm0
 ; X64-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X64-SSE2-NEXT:    psrld $16, %xmm1
@@ -2209,7 +2209,7 @@ define i8 @test_reduce_v32i8_v16i8(<32 x i8> %a0) {
 ; X86-SSE2:       ## %bb.0:
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    pmaxub %xmm0, %xmm1
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X86-SSE2-NEXT:    pmaxub %xmm1, %xmm0
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X86-SSE2-NEXT:    psrld $16, %xmm1
@@ -2251,7 +2251,7 @@ define i8 @test_reduce_v32i8_v16i8(<32 x i8> %a0) {
 ; X64-SSE2:       ## %bb.0:
 ; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    pmaxub %xmm0, %xmm1
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X64-SSE2-NEXT:    pmaxub %xmm1, %xmm0
 ; X64-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X64-SSE2-NEXT:    psrld $16, %xmm1
@@ -2334,7 +2334,7 @@ define i8 @test_reduce_v64i8_v16i8(<64 x i8> %a0) {
 ; X86-SSE2:       ## %bb.0:
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    pmaxub %xmm0, %xmm1
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X86-SSE2-NEXT:    pmaxub %xmm1, %xmm0
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X86-SSE2-NEXT:    psrld $16, %xmm1
@@ -2376,7 +2376,7 @@ define i8 @test_reduce_v64i8_v16i8(<64 x i8> %a0) {
 ; X64-SSE2:       ## %bb.0:
 ; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    pmaxub %xmm0, %xmm1
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X64-SSE2-NEXT:    pmaxub %xmm1, %xmm0
 ; X64-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X64-SSE2-NEXT:    psrld $16, %xmm1
diff --git a/llvm/test/CodeGen/X86/horizontal-reduce-umin.ll b/llvm/test/CodeGen/X86/horizontal-reduce-umin.ll
index 5f33520200d24..cd048b8d7659b 100644
--- a/llvm/test/CodeGen/X86/horizontal-reduce-umin.ll
+++ b/llvm/test/CodeGen/X86/horizontal-reduce-umin.ll
@@ -33,7 +33,7 @@ define i64 @test_reduce_v2i64(<2 x i64> %a0) {
 ; X86-SSE2-NEXT:    pandn %xmm1, %xmm3
 ; X86-SSE2-NEXT:    por %xmm0, %xmm3
 ; X86-SSE2-NEXT:    movd %xmm3, %eax
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,1,1]
 ; X86-SSE2-NEXT:    movd %xmm0, %edx
 ; X86-SSE2-NEXT:    retl
 ;
@@ -158,7 +158,7 @@ define i32 @test_reduce_v4i32(<4 x i32> %a0) {
 ; X86-SSE2-NEXT:    pand %xmm4, %xmm0
 ; X86-SSE2-NEXT:    pandn %xmm1, %xmm4
 ; X86-SSE2-NEXT:    por %xmm0, %xmm4
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,1,1,1]
 ; X86-SSE2-NEXT:    movdqa %xmm4, %xmm1
 ; X86-SSE2-NEXT:    pxor %xmm2, %xmm1
 ; X86-SSE2-NEXT:    pxor %xmm0, %xmm2
@@ -173,7 +173,7 @@ define i32 @test_reduce_v4i32(<4 x i32> %a0) {
 ; X86-SSE42:       ## %bb.0:
 ; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE42-NEXT:    pminud %xmm0, %xmm1
-; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X86-SSE42-NEXT:    pminud %xmm1, %xmm0
 ; X86-SSE42-NEXT:    movd %xmm0, %eax
 ; X86-SSE42-NEXT:    retl
@@ -182,7 +182,7 @@ define i32 @test_reduce_v4i32(<4 x i32> %a0) {
 ; X86-AVX:       ## %bb.0:
 ; X86-AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-AVX-NEXT:    vpminud %xmm1, %xmm0, %xmm0
-; X86-AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X86-AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X86-AVX-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vmovd %xmm0, %eax
 ; X86-AVX-NEXT:    retl
@@ -199,7 +199,7 @@ define i32 @test_reduce_v4i32(<4 x i32> %a0) {
 ; X64-SSE2-NEXT:    pand %xmm4, %xmm0
 ; X64-SSE2-NEXT:    pandn %xmm1, %xmm4
 ; X64-SSE2-NEXT:    por %xmm0, %xmm4
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,1,2,3]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,1,1,1]
 ; X64-SSE2-NEXT:    movdqa %xmm4, %xmm1
 ; X64-SSE2-NEXT:    pxor %xmm2, %xmm1
 ; X64-SSE2-NEXT:    pxor %xmm0, %xmm2
@@ -214,7 +214,7 @@ define i32 @test_reduce_v4i32(<4 x i32> %a0) {
 ; X64-SSE42:       ## %bb.0:
 ; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE42-NEXT:    pminud %xmm0, %xmm1
-; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X64-SSE42-NEXT:    pminud %xmm1, %xmm0
 ; X64-SSE42-NEXT:    movd %xmm0, %eax
 ; X64-SSE42-NEXT:    retq
@@ -223,7 +223,7 @@ define i32 @test_reduce_v4i32(<4 x i32> %a0) {
 ; X64-AVX:       ## %bb.0:
 ; X64-AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX-NEXT:    vpminud %xmm1, %xmm0, %xmm0
-; X64-AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X64-AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X64-AVX-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; X64-AVX-NEXT:    vmovd %xmm0, %eax
 ; X64-AVX-NEXT:    retq
@@ -245,7 +245,7 @@ define i16 @test_reduce_v8i16(<8 x i16> %a0) {
 ; X86-SSE2-NEXT:    pxor %xmm2, %xmm0
 ; X86-SSE2-NEXT:    pxor %xmm2, %xmm1
 ; X86-SSE2-NEXT:    pminsw %xmm0, %xmm1
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X86-SSE2-NEXT:    pminsw %xmm1, %xmm0
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X86-SSE2-NEXT:    psrld $16, %xmm1
@@ -276,7 +276,7 @@ define i16 @test_reduce_v8i16(<8 x i16> %a0) {
 ; X64-SSE2-NEXT:    pxor %xmm2, %xmm0
 ; X64-SSE2-NEXT:    pxor %xmm2, %xmm1
 ; X64-SSE2-NEXT:    pminsw %xmm0, %xmm1
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X64-SSE2-NEXT:    pminsw %xmm1, %xmm0
 ; X64-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X64-SSE2-NEXT:    psrld $16, %xmm1
@@ -317,7 +317,7 @@ define i8 @test_reduce_v16i8(<16 x i8> %a0) {
 ; X86-SSE2:       ## %bb.0:
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    pminub %xmm0, %xmm1
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X86-SSE2-NEXT:    pminub %xmm1, %xmm0
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X86-SSE2-NEXT:    psrld $16, %xmm1
@@ -352,7 +352,7 @@ define i8 @test_reduce_v16i8(<16 x i8> %a0) {
 ; X64-SSE2:       ## %bb.0:
 ; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    pminub %xmm0, %xmm1
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X64-SSE2-NEXT:    pminub %xmm1, %xmm0
 ; X64-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X64-SSE2-NEXT:    psrld $16, %xmm1
@@ -437,7 +437,7 @@ define i64 @test_reduce_v4i64(<4 x i64> %a0) {
 ; X86-SSE2-NEXT:    pandn %xmm0, %xmm2
 ; X86-SSE2-NEXT:    por %xmm4, %xmm2
 ; X86-SSE2-NEXT:    movd %xmm2, %eax
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
 ; X86-SSE2-NEXT:    movd %xmm0, %edx
 ; X86-SSE2-NEXT:    retl
 ;
@@ -629,7 +629,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X86-SSE2-NEXT:    pand %xmm3, %xmm4
 ; X86-SSE2-NEXT:    pandn %xmm0, %xmm3
 ; X86-SSE2-NEXT:    por %xmm4, %xmm3
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,1,1]
 ; X86-SSE2-NEXT:    movdqa %xmm3, %xmm1
 ; X86-SSE2-NEXT:    pxor %xmm2, %xmm1
 ; X86-SSE2-NEXT:    pxor %xmm0, %xmm2
@@ -645,7 +645,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X86-SSE42-NEXT:    pminud %xmm1, %xmm0
 ; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE42-NEXT:    pminud %xmm0, %xmm1
-; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X86-SSE42-NEXT:    pminud %xmm1, %xmm0
 ; X86-SSE42-NEXT:    movd %xmm0, %eax
 ; X86-SSE42-NEXT:    retl
@@ -656,7 +656,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X86-AVX1-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-AVX1-NEXT:    vpminud %xmm1, %xmm0, %xmm0
-; X86-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X86-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X86-AVX1-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vmovd %xmm0, %eax
 ; X86-AVX1-NEXT:    vzeroupper
@@ -668,7 +668,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X86-AVX2-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; X86-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-AVX2-NEXT:    vpminud %xmm1, %xmm0, %xmm0
-; X86-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X86-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X86-AVX2-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; X86-AVX2-NEXT:    vmovd %xmm0, %eax
 ; X86-AVX2-NEXT:    vzeroupper
@@ -694,7 +694,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X64-SSE2-NEXT:    pand %xmm3, %xmm4
 ; X64-SSE2-NEXT:    pandn %xmm0, %xmm3
 ; X64-SSE2-NEXT:    por %xmm4, %xmm3
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,2,3]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,1,1]
 ; X64-SSE2-NEXT:    movdqa %xmm3, %xmm1
 ; X64-SSE2-NEXT:    pxor %xmm2, %xmm1
 ; X64-SSE2-NEXT:    pxor %xmm0, %xmm2
@@ -710,7 +710,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X64-SSE42-NEXT:    pminud %xmm1, %xmm0
 ; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE42-NEXT:    pminud %xmm0, %xmm1
-; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X64-SSE42-NEXT:    pminud %xmm1, %xmm0
 ; X64-SSE42-NEXT:    movd %xmm0, %eax
 ; X64-SSE42-NEXT:    retq
@@ -721,7 +721,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X64-AVX1-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX1-NEXT:    vpminud %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X64-AVX1-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vmovd %xmm0, %eax
 ; X64-AVX1-NEXT:    vzeroupper
@@ -733,7 +733,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X64-AVX2-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; X64-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX2-NEXT:    vpminud %xmm1, %xmm0, %xmm0
-; X64-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X64-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X64-AVX2-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; X64-AVX2-NEXT:    vmovd %xmm0, %eax
 ; X64-AVX2-NEXT:    vzeroupper
@@ -745,7 +745,7 @@ define i32 @test_reduce_v8i32(<8 x i32> %a0) {
 ; X64-AVX512-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX512-NEXT:    vpminud %xmm1, %xmm0, %xmm0
-; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X64-AVX512-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vmovd %xmm0, %eax
 ; X64-AVX512-NEXT:    vzeroupper
@@ -772,7 +772,7 @@ define i16 @test_reduce_v16i16(<16 x i16> %a0) {
 ; X86-SSE2-NEXT:    pminsw %xmm1, %xmm0
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    pminsw %xmm0, %xmm1
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X86-SSE2-NEXT:    pminsw %xmm1, %xmm0
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X86-SSE2-NEXT:    psrld $16, %xmm1
@@ -818,7 +818,7 @@ define i16 @test_reduce_v16i16(<16 x i16> %a0) {
 ; X64-SSE2-NEXT:    pminsw %xmm1, %xmm0
 ; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    pminsw %xmm0, %xmm1
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X64-SSE2-NEXT:    pminsw %xmm1, %xmm0
 ; X64-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X64-SSE2-NEXT:    psrld $16, %xmm1
@@ -887,7 +887,7 @@ define i8 @test_reduce_v32i8(<32 x i8> %a0) {
 ; X86-SSE2-NEXT:    pminub %xmm1, %xmm0
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    pminub %xmm0, %xmm1
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X86-SSE2-NEXT:    pminub %xmm1, %xmm0
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X86-SSE2-NEXT:    psrld $16, %xmm1
@@ -939,7 +939,7 @@ define i8 @test_reduce_v32i8(<32 x i8> %a0) {
 ; X64-SSE2-NEXT:    pminub %xmm1, %xmm0
 ; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    pminub %xmm0, %xmm1
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X64-SSE2-NEXT:    pminub %xmm1, %xmm0
 ; X64-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X64-SSE2-NEXT:    psrld $16, %xmm1
@@ -1085,7 +1085,7 @@ define i64 @test_reduce_v8i64(<8 x i64> %a0) {
 ; X86-SSE2-NEXT:    pandn %xmm1, %xmm2
 ; X86-SSE2-NEXT:    por %xmm0, %xmm2
 ; X86-SSE2-NEXT:    movd %xmm2, %eax
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
 ; X86-SSE2-NEXT:    movd %xmm0, %edx
 ; X86-SSE2-NEXT:    retl
 ;
@@ -1378,7 +1378,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X86-SSE2-NEXT:    pand %xmm3, %xmm1
 ; X86-SSE2-NEXT:    pandn %xmm0, %xmm3
 ; X86-SSE2-NEXT:    por %xmm1, %xmm3
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,1,1]
 ; X86-SSE2-NEXT:    movdqa %xmm3, %xmm1
 ; X86-SSE2-NEXT:    pxor %xmm4, %xmm1
 ; X86-SSE2-NEXT:    pxor %xmm0, %xmm4
@@ -1396,7 +1396,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X86-SSE42-NEXT:    pminud %xmm0, %xmm1
 ; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; X86-SSE42-NEXT:    pminud %xmm1, %xmm0
-; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X86-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X86-SSE42-NEXT:    pminud %xmm0, %xmm1
 ; X86-SSE42-NEXT:    movd %xmm1, %eax
 ; X86-SSE42-NEXT:    retl
@@ -1410,7 +1410,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X86-AVX1-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-AVX1-NEXT:    vpminud %xmm1, %xmm0, %xmm0
-; X86-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X86-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X86-AVX1-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vmovd %xmm0, %eax
 ; X86-AVX1-NEXT:    vzeroupper
@@ -1423,7 +1423,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X86-AVX2-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; X86-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-AVX2-NEXT:    vpminud %xmm1, %xmm0, %xmm0
-; X86-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X86-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X86-AVX2-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; X86-AVX2-NEXT:    vmovd %xmm0, %eax
 ; X86-AVX2-NEXT:    vzeroupper
@@ -1465,7 +1465,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X64-SSE2-NEXT:    pand %xmm3, %xmm1
 ; X64-SSE2-NEXT:    pandn %xmm0, %xmm3
 ; X64-SSE2-NEXT:    por %xmm1, %xmm3
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,2,3]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,1,1]
 ; X64-SSE2-NEXT:    movdqa %xmm3, %xmm1
 ; X64-SSE2-NEXT:    pxor %xmm4, %xmm1
 ; X64-SSE2-NEXT:    pxor %xmm0, %xmm4
@@ -1483,7 +1483,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X64-SSE42-NEXT:    pminud %xmm0, %xmm1
 ; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; X64-SSE42-NEXT:    pminud %xmm1, %xmm0
-; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X64-SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X64-SSE42-NEXT:    pminud %xmm0, %xmm1
 ; X64-SSE42-NEXT:    movd %xmm1, %eax
 ; X64-SSE42-NEXT:    retq
@@ -1497,7 +1497,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X64-AVX1-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX1-NEXT:    vpminud %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X64-AVX1-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vmovd %xmm0, %eax
 ; X64-AVX1-NEXT:    vzeroupper
@@ -1510,7 +1510,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X64-AVX2-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; X64-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX2-NEXT:    vpminud %xmm1, %xmm0, %xmm0
-; X64-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X64-AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X64-AVX2-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; X64-AVX2-NEXT:    vmovd %xmm0, %eax
 ; X64-AVX2-NEXT:    vzeroupper
@@ -1524,7 +1524,7 @@ define i32 @test_reduce_v16i32(<16 x i32> %a0) {
 ; X64-AVX512-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-AVX512-NEXT:    vpminud %xmm1, %xmm0, %xmm0
-; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X64-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X64-AVX512-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vmovd %xmm0, %eax
 ; X64-AVX512-NEXT:    vzeroupper
@@ -1558,7 +1558,7 @@ define i16 @test_reduce_v32i16(<32 x i16> %a0) {
 ; X86-SSE2-NEXT:    pminsw %xmm2, %xmm0
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    pminsw %xmm0, %xmm1
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X86-SSE2-NEXT:    pminsw %xmm1, %xmm0
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X86-SSE2-NEXT:    psrld $16, %xmm1
@@ -1614,7 +1614,7 @@ define i16 @test_reduce_v32i16(<32 x i16> %a0) {
 ; X64-SSE2-NEXT:    pminsw %xmm2, %xmm0
 ; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    pminsw %xmm0, %xmm1
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X64-SSE2-NEXT:    pminsw %xmm1, %xmm0
 ; X64-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X64-SSE2-NEXT:    psrld $16, %xmm1
@@ -1696,7 +1696,7 @@ define i8 @test_reduce_v64i8(<64 x i8> %a0) {
 ; X86-SSE2-NEXT:    pminub %xmm0, %xmm1
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; X86-SSE2-NEXT:    pminub %xmm1, %xmm0
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X86-SSE2-NEXT:    pminub %xmm0, %xmm1
 ; X86-SSE2-NEXT:    movdqa %xmm1, %xmm0
 ; X86-SSE2-NEXT:    psrld $16, %xmm0
@@ -1756,7 +1756,7 @@ define i8 @test_reduce_v64i8(<64 x i8> %a0) {
 ; X64-SSE2-NEXT:    pminub %xmm0, %xmm1
 ; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; X64-SSE2-NEXT:    pminub %xmm1, %xmm0
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X64-SSE2-NEXT:    pminub %xmm0, %xmm1
 ; X64-SSE2-NEXT:    movdqa %xmm1, %xmm0
 ; X64-SSE2-NEXT:    psrld $16, %xmm0
@@ -1856,7 +1856,7 @@ define i16 @test_reduce_v16i16_v8i16(<16 x i16> %a0) {
 ; X86-SSE2-NEXT:    pxor %xmm2, %xmm0
 ; X86-SSE2-NEXT:    pxor %xmm2, %xmm1
 ; X86-SSE2-NEXT:    pminsw %xmm0, %xmm1
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X86-SSE2-NEXT:    pminsw %xmm1, %xmm0
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X86-SSE2-NEXT:    psrld $16, %xmm1
@@ -1888,7 +1888,7 @@ define i16 @test_reduce_v16i16_v8i16(<16 x i16> %a0) {
 ; X64-SSE2-NEXT:    pxor %xmm2, %xmm0
 ; X64-SSE2-NEXT:    pxor %xmm2, %xmm1
 ; X64-SSE2-NEXT:    pminsw %xmm0, %xmm1
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X64-SSE2-NEXT:    pminsw %xmm1, %xmm0
 ; X64-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X64-SSE2-NEXT:    psrld $16, %xmm1
@@ -1933,7 +1933,7 @@ define i16 @test_reduce_v32i16_v8i16(<32 x i16> %a0) {
 ; X86-SSE2-NEXT:    pxor %xmm2, %xmm0
 ; X86-SSE2-NEXT:    pxor %xmm2, %xmm1
 ; X86-SSE2-NEXT:    pminsw %xmm0, %xmm1
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X86-SSE2-NEXT:    pminsw %xmm1, %xmm0
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X86-SSE2-NEXT:    psrld $16, %xmm1
@@ -1965,7 +1965,7 @@ define i16 @test_reduce_v32i16_v8i16(<32 x i16> %a0) {
 ; X64-SSE2-NEXT:    pxor %xmm2, %xmm0
 ; X64-SSE2-NEXT:    pxor %xmm2, %xmm1
 ; X64-SSE2-NEXT:    pminsw %xmm0, %xmm1
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X64-SSE2-NEXT:    pminsw %xmm1, %xmm0
 ; X64-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X64-SSE2-NEXT:    psrld $16, %xmm1
@@ -2007,7 +2007,7 @@ define i8 @test_reduce_v32i8_v16i8(<32 x i8> %a0) {
 ; X86-SSE2:       ## %bb.0:
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    pminub %xmm0, %xmm1
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X86-SSE2-NEXT:    pminub %xmm1, %xmm0
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X86-SSE2-NEXT:    psrld $16, %xmm1
@@ -2043,7 +2043,7 @@ define i8 @test_reduce_v32i8_v16i8(<32 x i8> %a0) {
 ; X64-SSE2:       ## %bb.0:
 ; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    pminub %xmm0, %xmm1
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X64-SSE2-NEXT:    pminub %xmm1, %xmm0
 ; X64-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X64-SSE2-NEXT:    psrld $16, %xmm1
@@ -2095,7 +2095,7 @@ define i8 @test_reduce_v64i8_v16i8(<64 x i8> %a0) {
 ; X86-SSE2:       ## %bb.0:
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    pminub %xmm0, %xmm1
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X86-SSE2-NEXT:    pminub %xmm1, %xmm0
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X86-SSE2-NEXT:    psrld $16, %xmm1
@@ -2131,7 +2131,7 @@ define i8 @test_reduce_v64i8_v16i8(<64 x i8> %a0) {
 ; X64-SSE2:       ## %bb.0:
 ; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; X64-SSE2-NEXT:    pminub %xmm0, %xmm1
-; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X64-SSE2-NEXT:    pminub %xmm1, %xmm0
 ; X64-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X64-SSE2-NEXT:    psrld $16, %xmm1
diff --git a/llvm/test/CodeGen/X86/insert-into-constant-vector.ll b/llvm/test/CodeGen/X86/insert-into-constant-vector.ll
index 779c91ab2575c..886fa6d07c5ec 100644
--- a/llvm/test/CodeGen/X86/insert-into-constant-vector.ll
+++ b/llvm/test/CodeGen/X86/insert-into-constant-vector.ll
@@ -276,7 +276,7 @@ define <8 x i32> @elt7_v8i32(i32 %x) {
 ; X64AVX1-LABEL: elt7_v8i32:
 ; X64AVX1:       # %bb.0:
 ; X64AVX1-NEXT:    vmovd %edi, %xmm0
-; X64AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,2,0]
+; X64AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; X64AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; X64AVX1-NEXT:    vblendps {{.*#+}} ymm0 = mem[0,1,2,3,4,5,6],ymm0[7]
 ; X64AVX1-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/insert-loaded-scalar.ll b/llvm/test/CodeGen/X86/insert-loaded-scalar.ll
index fe00e50137bbe..3dc598828e484 100644
--- a/llvm/test/CodeGen/X86/insert-loaded-scalar.ll
+++ b/llvm/test/CodeGen/X86/insert-loaded-scalar.ll
@@ -191,7 +191,7 @@ define <4 x float> @load32_ins_eltc_v4f32(float* %p) nounwind {
 ; SSE-LABEL: load32_ins_eltc_v4f32:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1,2,0]
+; SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: load32_ins_eltc_v4f32:
@@ -376,7 +376,7 @@ define <8 x i32> @load32_ins_eltc_v8i32(i32* %p) nounwind {
 ; SSE-LABEL: load32_ins_eltc_v8i32:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,1,2,0]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,0,0]
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: load32_ins_eltc_v8i32:
diff --git a/llvm/test/CodeGen/X86/insertelement-var-index.ll b/llvm/test/CodeGen/X86/insertelement-var-index.ll
index 2e3824c8f03f4..75ed52588d505 100644
--- a/llvm/test/CodeGen/X86/insertelement-var-index.ll
+++ b/llvm/test/CodeGen/X86/insertelement-var-index.ll
@@ -47,14 +47,14 @@ define <8 x i16> @arg_i16_v8i16(i16 %x, i32 %y) nounwind {
 ; SSE-LABEL: arg_i16_v8i16:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    movd %edi, %xmm0
-; SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: arg_i16_v8i16:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vmovd %edi, %xmm0
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    retq
 ;
@@ -175,7 +175,7 @@ define <8 x i16> @load_i16_v8i16(i16* %p, i32 %y) nounwind {
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    movzwl (%rdi), %eax
 ; SSE-NEXT:    movd %eax, %xmm0
-; SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; SSE-NEXT:    retq
 ;
@@ -183,7 +183,7 @@ define <8 x i16> @load_i16_v8i16(i16* %p, i32 %y) nounwind {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    movzwl (%rdi), %eax
 ; AVX1-NEXT:    vmovd %eax, %xmm0
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    retq
 ;
@@ -299,7 +299,7 @@ define <16 x i16> @arg_i16_v16i16(i16 %x, i32 %y) nounwind {
 ; AVX1-LABEL: arg_i16_v16i16:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vmovd %edi, %xmm0
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
@@ -457,7 +457,7 @@ define <16 x i16> @load_i16_v16i16(i16* %p, i32 %y) nounwind {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    movzwl (%rdi), %eax
 ; AVX1-NEXT:    vmovd %eax, %xmm0
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/known-signbits-vector.ll b/llvm/test/CodeGen/X86/known-signbits-vector.ll
index 6c4d0a919ef0f..1561fbc844d0c 100644
--- a/llvm/test/CodeGen/X86/known-signbits-vector.ll
+++ b/llvm/test/CodeGen/X86/known-signbits-vector.ll
@@ -148,7 +148,7 @@ define float @signbits_ashr_extract_sitofp_0(<2 x i64> %a0) nounwind {
 ; X86-LABEL: signbits_ashr_extract_sitofp_0:
 ; X86:       # %bb.0:
 ; X86-NEXT:    pushl %eax
-; X86-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; X86-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; X86-NEXT:    vcvtdq2ps %xmm0, %xmm0
 ; X86-NEXT:    vmovss %xmm0, (%esp)
 ; X86-NEXT:    flds (%esp)
@@ -157,7 +157,7 @@ define float @signbits_ashr_extract_sitofp_0(<2 x i64> %a0) nounwind {
 ;
 ; X64-LABEL: signbits_ashr_extract_sitofp_0:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; X64-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; X64-NEXT:    vcvtdq2ps %xmm0, %xmm0
 ; X64-NEXT:    retq
   %1 = ashr <2 x i64> %a0, <i64 32, i64 32>
@@ -170,7 +170,7 @@ define float @signbits_ashr_extract_sitofp_1(<2 x i64> %a0) nounwind {
 ; X86-LABEL: signbits_ashr_extract_sitofp_1:
 ; X86:       # %bb.0:
 ; X86-NEXT:    pushl %eax
-; X86-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; X86-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; X86-NEXT:    vcvtdq2ps %xmm0, %xmm0
 ; X86-NEXT:    vmovss %xmm0, (%esp)
 ; X86-NEXT:    flds (%esp)
@@ -179,7 +179,7 @@ define float @signbits_ashr_extract_sitofp_1(<2 x i64> %a0) nounwind {
 ;
 ; X64-LABEL: signbits_ashr_extract_sitofp_1:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; X64-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; X64-NEXT:    vcvtdq2ps %xmm0, %xmm0
 ; X64-NEXT:    retq
   %1 = ashr <2 x i64> %a0, <i64 32, i64 63>
diff --git a/llvm/test/CodeGen/X86/load-partial.ll b/llvm/test/CodeGen/X86/load-partial.ll
index 9bef57bef7989..29984987c7726 100644
--- a/llvm/test/CodeGen/X86/load-partial.ll
+++ b/llvm/test/CodeGen/X86/load-partial.ll
@@ -349,7 +349,7 @@ define void @PR43227(i32* %explicit_0, <8 x i32>* %explicit_1) {
 ; SSE2-LABEL: PR43227:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    xorps %xmm1, %xmm1
 ; SSE2-NEXT:    xorps %xmm2, %xmm2
 ; SSE2-NEXT:    movss {{.*#+}} xmm2 = xmm0[0],xmm2[1,2,3]
@@ -362,7 +362,7 @@ define void @PR43227(i32* %explicit_0, <8 x i32>* %explicit_1) {
 ; SSSE3-LABEL: PR43227:
 ; SSSE3:       # %bb.0:
 ; SSSE3-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSSE3-NEXT:    xorps %xmm1, %xmm1
 ; SSSE3-NEXT:    xorps %xmm2, %xmm2
 ; SSSE3-NEXT:    movss {{.*#+}} xmm2 = xmm0[0],xmm2[1,2,3]
@@ -375,7 +375,7 @@ define void @PR43227(i32* %explicit_0, <8 x i32>* %explicit_1) {
 ; SSE41-LABEL: PR43227:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE41-NEXT:    pxor %xmm1, %xmm1
 ; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]
 ; SSE41-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
@@ -387,7 +387,7 @@ define void @PR43227(i32* %explicit_0, <8 x i32>* %explicit_1) {
 ; AVX-LABEL: PR43227:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; AVX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
 ; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
 ; AVX-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
diff --git a/llvm/test/CodeGen/X86/madd.ll b/llvm/test/CodeGen/X86/madd.ll
index 93097e2b98fb7..a024a04fa37f3 100644
--- a/llvm/test/CodeGen/X86/madd.ll
+++ b/llvm/test/CodeGen/X86/madd.ll
@@ -27,7 +27,7 @@ define i32 @_Z10test_shortPsS_i_128(i16* nocapture readonly, i16* nocapture read
 ; SSE2-NEXT:  # %bb.2: # %middle.block
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE2-NEXT:    paddd %xmm1, %xmm0
 ; SSE2-NEXT:    movd %xmm0, %eax
 ; SSE2-NEXT:    retq
@@ -50,7 +50,7 @@ define i32 @_Z10test_shortPsS_i_128(i16* nocapture readonly, i16* nocapture read
 ; AVX-NEXT:  # %bb.2: # %middle.block
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovd %xmm0, %eax
 ; AVX-NEXT:    retq
@@ -105,7 +105,7 @@ define i32 @_Z10test_shortPsS_i_256(i16* nocapture readonly, i16* nocapture read
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    paddd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
 ; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    retq
@@ -130,7 +130,7 @@ define i32 @_Z10test_shortPsS_i_256(i16* nocapture readonly, i16* nocapture read
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
 ; AVX1-NEXT:    vzeroupper
@@ -155,7 +155,7 @@ define i32 @_Z10test_shortPsS_i_256(i16* nocapture readonly, i16* nocapture read
 ; AVX256-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX256-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX256-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX256-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX256-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX256-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX256-NEXT:    vmovd %xmm0, %eax
 ; AVX256-NEXT:    vzeroupper
@@ -220,7 +220,7 @@ define i32 @_Z10test_shortPsS_i_512(i16* nocapture readonly, i16* nocapture read
 ; SSE2-NEXT:    paddd %xmm2, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    paddd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
 ; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    retq
@@ -253,7 +253,7 @@ define i32 @_Z10test_shortPsS_i_512(i16* nocapture readonly, i16* nocapture read
 ; AVX1-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
 ; AVX1-NEXT:    vzeroupper
@@ -280,7 +280,7 @@ define i32 @_Z10test_shortPsS_i_512(i16* nocapture readonly, i16* nocapture read
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
 ; AVX2-NEXT:    vzeroupper
@@ -307,7 +307,7 @@ define i32 @_Z10test_shortPsS_i_512(i16* nocapture readonly, i16* nocapture read
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
 ; AVX512-NEXT:    vzeroupper
@@ -388,7 +388,7 @@ define i32 @_Z10test_shortPsS_i_1024(i16* nocapture readonly, i16* nocapture rea
 ; SSE2-NEXT:    paddd %xmm2, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    paddd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
 ; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    retq
@@ -435,7 +435,7 @@ define i32 @_Z10test_shortPsS_i_1024(i16* nocapture readonly, i16* nocapture rea
 ; AVX1-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
 ; AVX1-NEXT:    vzeroupper
@@ -468,7 +468,7 @@ define i32 @_Z10test_shortPsS_i_1024(i16* nocapture readonly, i16* nocapture rea
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
 ; AVX2-NEXT:    vzeroupper
@@ -500,7 +500,7 @@ define i32 @_Z10test_shortPsS_i_1024(i16* nocapture readonly, i16* nocapture rea
 ; AVX512F-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512F-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512F-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX512F-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512F-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512F-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512F-NEXT:    vmovd %xmm0, %eax
 ; AVX512F-NEXT:    vzeroupper
@@ -529,7 +529,7 @@ define i32 @_Z10test_shortPsS_i_1024(i16* nocapture readonly, i16* nocapture rea
 ; AVX512BW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512BW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512BW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vmovd %xmm0, %eax
 ; AVX512BW-NEXT:    vzeroupper
@@ -595,7 +595,7 @@ define i32 @_Z9test_charPcS_i_128(i8* nocapture readonly, i8* nocapture readonly
 ; SSE2-NEXT:  # %bb.2: # %middle.block
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE2-NEXT:    paddd %xmm1, %xmm0
 ; SSE2-NEXT:    movd %xmm0, %eax
 ; SSE2-NEXT:    retq
@@ -618,7 +618,7 @@ define i32 @_Z9test_charPcS_i_128(i8* nocapture readonly, i8* nocapture readonly
 ; AVX-NEXT:  # %bb.2: # %middle.block
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovd %xmm0, %eax
 ; AVX-NEXT:    retq
@@ -677,7 +677,7 @@ define i32 @_Z9test_charPcS_i_256(i8* nocapture readonly, i8* nocapture readonly
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    paddd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
 ; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    retq
@@ -703,7 +703,7 @@ define i32 @_Z9test_charPcS_i_256(i8* nocapture readonly, i8* nocapture readonly
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
 ; AVX1-NEXT:    vzeroupper
@@ -729,7 +729,7 @@ define i32 @_Z9test_charPcS_i_256(i8* nocapture readonly, i8* nocapture readonly
 ; AVX256-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX256-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX256-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX256-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX256-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX256-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX256-NEXT:    vmovd %xmm0, %eax
 ; AVX256-NEXT:    vzeroupper
@@ -800,7 +800,7 @@ define i32 @_Z9test_charPcS_i_512(i8* nocapture readonly, i8* nocapture readonly
 ; SSE2-NEXT:    paddd %xmm2, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    paddd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
 ; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    retq
@@ -835,7 +835,7 @@ define i32 @_Z9test_charPcS_i_512(i8* nocapture readonly, i8* nocapture readonly
 ; AVX1-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
 ; AVX1-NEXT:    vzeroupper
@@ -863,7 +863,7 @@ define i32 @_Z9test_charPcS_i_512(i8* nocapture readonly, i8* nocapture readonly
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
 ; AVX2-NEXT:    vzeroupper
@@ -891,7 +891,7 @@ define i32 @_Z9test_charPcS_i_512(i8* nocapture readonly, i8* nocapture readonly
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
 ; AVX512-NEXT:    vzeroupper
@@ -984,7 +984,7 @@ define i32 @_Z9test_charPcS_i_1024(i8* nocapture readonly, i8* nocapture readonl
 ; SSE2-NEXT:    paddd %xmm2, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    paddd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
 ; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    retq
@@ -1035,7 +1035,7 @@ define i32 @_Z9test_charPcS_i_1024(i8* nocapture readonly, i8* nocapture readonl
 ; AVX1-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
 ; AVX1-NEXT:    vzeroupper
@@ -1070,7 +1070,7 @@ define i32 @_Z9test_charPcS_i_1024(i8* nocapture readonly, i8* nocapture readonl
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
 ; AVX2-NEXT:    vzeroupper
@@ -1104,7 +1104,7 @@ define i32 @_Z9test_charPcS_i_1024(i8* nocapture readonly, i8* nocapture readonl
 ; AVX512F-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512F-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512F-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX512F-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512F-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512F-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512F-NEXT:    vmovd %xmm0, %eax
 ; AVX512F-NEXT:    vzeroupper
@@ -1134,7 +1134,7 @@ define i32 @_Z9test_charPcS_i_1024(i8* nocapture readonly, i8* nocapture readonl
 ; AVX512BW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512BW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512BW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vmovd %xmm0, %eax
 ; AVX512BW-NEXT:    vzeroupper
@@ -1197,7 +1197,7 @@ define i32 @test_unsigned_short_128(i16* nocapture readonly, i16* nocapture read
 ; SSE2-NEXT:  # %bb.2: # %middle.block
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE2-NEXT:    paddd %xmm1, %xmm0
 ; SSE2-NEXT:    movd %xmm0, %eax
 ; SSE2-NEXT:    retq
@@ -1220,7 +1220,7 @@ define i32 @test_unsigned_short_128(i16* nocapture readonly, i16* nocapture read
 ; AVX-NEXT:  # %bb.2: # %middle.block
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovd %xmm0, %eax
 ; AVX-NEXT:    retq
@@ -1281,7 +1281,7 @@ define i32 @test_unsigned_short_256(i16* nocapture readonly, i16* nocapture read
 ; SSE2-NEXT:    paddd %xmm1, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE2-NEXT:    paddd %xmm1, %xmm0
 ; SSE2-NEXT:    movd %xmm0, %eax
 ; SSE2-NEXT:    retq
@@ -1312,7 +1312,7 @@ define i32 @test_unsigned_short_256(i16* nocapture readonly, i16* nocapture read
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
 ; AVX1-NEXT:    vzeroupper
@@ -1338,7 +1338,7 @@ define i32 @test_unsigned_short_256(i16* nocapture readonly, i16* nocapture read
 ; AVX256-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX256-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX256-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX256-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX256-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX256-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX256-NEXT:    vmovd %xmm0, %eax
 ; AVX256-NEXT:    vzeroupper
@@ -1416,7 +1416,7 @@ define i32 @test_unsigned_short_512(i16* nocapture readonly, i16* nocapture read
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    paddd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
 ; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    retq
@@ -1461,7 +1461,7 @@ define i32 @test_unsigned_short_512(i16* nocapture readonly, i16* nocapture read
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
 ; AVX1-NEXT:    vzeroupper
@@ -1493,7 +1493,7 @@ define i32 @test_unsigned_short_512(i16* nocapture readonly, i16* nocapture read
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
 ; AVX2-NEXT:    vzeroupper
@@ -1521,7 +1521,7 @@ define i32 @test_unsigned_short_512(i16* nocapture readonly, i16* nocapture read
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
 ; AVX512-NEXT:    vzeroupper
@@ -1629,7 +1629,7 @@ define i32 @test_unsigned_short_1024(i16* nocapture readonly, i16* nocapture rea
 ; SSE2-NEXT:    paddd %xmm8, %xmm9
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm9[2,3,2,3]
 ; SSE2-NEXT:    paddd %xmm9, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
 ; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    retq
@@ -1702,7 +1702,7 @@ define i32 @test_unsigned_short_1024(i16* nocapture readonly, i16* nocapture rea
 ; AVX1-NEXT:    vpaddd %xmm0, %xmm8, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
 ; AVX1-NEXT:    vzeroupper
@@ -1746,7 +1746,7 @@ define i32 @test_unsigned_short_1024(i16* nocapture readonly, i16* nocapture rea
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
 ; AVX2-NEXT:    vzeroupper
@@ -1780,7 +1780,7 @@ define i32 @test_unsigned_short_1024(i16* nocapture readonly, i16* nocapture rea
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
 ; AVX512-NEXT:    vzeroupper
@@ -2649,7 +2649,7 @@ define i32 @madd_double_reduction(<8 x i16>* %arg, <8 x i16>* %arg1, <8 x i16>*
 ; SSE2-NEXT:    paddd %xmm1, %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE2-NEXT:    paddd %xmm2, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
 ; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    retq
@@ -2663,7 +2663,7 @@ define i32 @madd_double_reduction(<8 x i16>* %arg, <8 x i16>* %arg1, <8 x i16>*
 ; AVX-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovd %xmm0, %eax
 ; AVX-NEXT:    retq
@@ -2710,7 +2710,7 @@ define i32 @madd_quad_reduction(<8 x i16>* %arg, <8 x i16>* %arg1, <8 x i16>* %a
 ; SSE2-NEXT:    paddd %xmm2, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    paddd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
 ; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    retq
@@ -2732,7 +2732,7 @@ define i32 @madd_quad_reduction(<8 x i16>* %arg, <8 x i16>* %arg1, <8 x i16>* %a
 ; AVX-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovd %xmm0, %eax
 ; AVX-NEXT:    retq
@@ -2800,13 +2800,13 @@ define i64 @sum_and_sum_of_squares(i8* %a, i32 %n) {
 ; SSE2-NEXT:    paddd %xmm3, %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
 ; SSE2-NEXT:    paddd %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,1,1,1]
 ; SSE2-NEXT:    paddd %xmm3, %xmm2
 ; SSE2-NEXT:    movd %xmm2, %ecx
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    paddd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
 ; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    shlq $32, %rcx
@@ -2841,14 +2841,14 @@ define i64 @sum_and_sum_of_squares(i8* %a, i32 %n) {
 ; AVX1-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; AVX1-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
 ; AVX1-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vmovd %xmm1, %ecx
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
 ; AVX1-NEXT:    shlq $32, %rcx
@@ -2876,14 +2876,14 @@ define i64 @sum_and_sum_of_squares(i8* %a, i32 %n) {
 ; AVX256-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
 ; AVX256-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; AVX256-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
-; AVX256-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,2,3]
+; AVX256-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
 ; AVX256-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
 ; AVX256-NEXT:    vmovd %xmm1, %ecx
 ; AVX256-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX256-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX256-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX256-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX256-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX256-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX256-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX256-NEXT:    vmovd %xmm0, %eax
 ; AVX256-NEXT:    shlq $32, %rcx
@@ -2955,7 +2955,7 @@ define i32 @sum_of_square_differences(i8* %a, i8* %b, i32 %n) {
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    paddd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
 ; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    retq
@@ -2982,7 +2982,7 @@ define i32 @sum_of_square_differences(i8* %a, i8* %b, i32 %n) {
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
 ; AVX1-NEXT:    vzeroupper
@@ -3009,7 +3009,7 @@ define i32 @sum_of_square_differences(i8* %a, i8* %b, i32 %n) {
 ; AVX256-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX256-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX256-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX256-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX256-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX256-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX256-NEXT:    vmovd %xmm0, %eax
 ; AVX256-NEXT:    vzeroupper
diff --git a/llvm/test/CodeGen/X86/masked_compressstore.ll b/llvm/test/CodeGen/X86/masked_compressstore.ll
index c93543ddda7ae..96ca81c228a2f 100644
--- a/llvm/test/CodeGen/X86/masked_compressstore.ll
+++ b/llvm/test/CodeGen/X86/masked_compressstore.ll
@@ -620,7 +620,7 @@ define void @compressstore_v2f32_v2i32(float* %base, <2 x float> %V, <2 x i32> %
 ; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je LBB2_4
 ; SSE2-NEXT:  LBB2_3: ## %cond.store1
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    movss %xmm0, (%rdi)
 ; SSE2-NEXT:    retq
 ;
@@ -724,7 +724,7 @@ define void @compressstore_v4f32_v4i1(float* %base, <4 x float> %V, <4 x i1> %ma
 ; SSE2-NEXT:    je LBB3_4
 ; SSE2-NEXT:  LBB3_3: ## %cond.store1
 ; SSE2-NEXT:    movaps %xmm0, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[1,1]
 ; SSE2-NEXT:    movss %xmm1, (%rdi)
 ; SSE2-NEXT:    addq $4, %rdi
 ; SSE2-NEXT:    testb $4, %al
@@ -737,7 +737,7 @@ define void @compressstore_v4f32_v4i1(float* %base, <4 x float> %V, <4 x i1> %ma
 ; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je LBB3_8
 ; SSE2-NEXT:  LBB3_7: ## %cond.store7
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE2-NEXT:    movss %xmm0, (%rdi)
 ; SSE2-NEXT:    retq
 ;
@@ -879,7 +879,7 @@ define void @compressstore_v8f32_v8i1(float* %base, <8 x float> %V, <8 x i1> %ma
 ; SSE2-NEXT:    je LBB4_4
 ; SSE2-NEXT:  LBB4_3: ## %cond.store1
 ; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1],xmm0[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1],xmm0[1,1]
 ; SSE2-NEXT:    movss %xmm2, (%rdi)
 ; SSE2-NEXT:    addq $4, %rdi
 ; SSE2-NEXT:    testb $4, %al
@@ -892,7 +892,7 @@ define void @compressstore_v8f32_v8i1(float* %base, <8 x float> %V, <8 x i1> %ma
 ; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je LBB4_8
 ; SSE2-NEXT:  LBB4_7: ## %cond.store7
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE2-NEXT:    movss %xmm0, (%rdi)
 ; SSE2-NEXT:    addq $4, %rdi
 ; SSE2-NEXT:    testb $16, %al
@@ -904,7 +904,7 @@ define void @compressstore_v8f32_v8i1(float* %base, <8 x float> %V, <8 x i1> %ma
 ; SSE2-NEXT:    je LBB4_12
 ; SSE2-NEXT:  LBB4_11: ## %cond.store13
 ; SSE2-NEXT:    movaps %xmm1, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[1,1]
 ; SSE2-NEXT:    movss %xmm0, (%rdi)
 ; SSE2-NEXT:    addq $4, %rdi
 ; SSE2-NEXT:    testb $64, %al
@@ -917,7 +917,7 @@ define void @compressstore_v8f32_v8i1(float* %base, <8 x float> %V, <8 x i1> %ma
 ; SSE2-NEXT:    testb $-128, %al
 ; SSE2-NEXT:    je LBB4_16
 ; SSE2-NEXT:  LBB4_15: ## %cond.store19
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; SSE2-NEXT:    movss %xmm1, (%rdi)
 ; SSE2-NEXT:    retq
 ;
@@ -1167,36 +1167,36 @@ define void @compressstore_v16f32_const(float* %base, <16 x float> %V) {
 ; SSE2:       ## %bb.0:
 ; SSE2-NEXT:    movss %xmm0, (%rdi)
 ; SSE2-NEXT:    movaps %xmm0, %xmm4
-; SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[1,1],xmm0[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[1,1],xmm0[1,1]
 ; SSE2-NEXT:    movss %xmm4, 4(%rdi)
 ; SSE2-NEXT:    movaps %xmm0, %xmm4
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm0[1]
 ; SSE2-NEXT:    movss %xmm4, 8(%rdi)
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE2-NEXT:    movss %xmm0, 12(%rdi)
 ; SSE2-NEXT:    movss %xmm1, 16(%rdi)
 ; SSE2-NEXT:    movaps %xmm1, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[1,1]
 ; SSE2-NEXT:    movss %xmm0, 20(%rdi)
 ; SSE2-NEXT:    movaps %xmm1, %xmm0
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1]
 ; SSE2-NEXT:    movss %xmm0, 24(%rdi)
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; SSE2-NEXT:    movss %xmm1, 28(%rdi)
 ; SSE2-NEXT:    movss %xmm2, 32(%rdi)
 ; SSE2-NEXT:    movaps %xmm2, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm2[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm2[1,1]
 ; SSE2-NEXT:    movss %xmm0, 36(%rdi)
 ; SSE2-NEXT:    movhlps {{.*#+}} xmm2 = xmm2[1,1]
 ; SSE2-NEXT:    movss %xmm2, 40(%rdi)
 ; SSE2-NEXT:    movss %xmm3, 44(%rdi)
 ; SSE2-NEXT:    movaps %xmm3, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm3[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm3[1,1]
 ; SSE2-NEXT:    movss %xmm0, 48(%rdi)
 ; SSE2-NEXT:    movaps %xmm3, %xmm0
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm3[1]
 ; SSE2-NEXT:    movss %xmm0, 52(%rdi)
-; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,3,3,3]
 ; SSE2-NEXT:    movss %xmm3, 56(%rdi)
 ; SSE2-NEXT:    retq
 ;
@@ -1396,7 +1396,7 @@ define void @compressstore_v32f32_v32i32(float* %base, <32 x float> %V, <32 x i3
 ; SSE2-NEXT:    je LBB6_4
 ; SSE2-NEXT:  LBB6_3: ## %cond.store1
 ; SSE2-NEXT:    movaps %xmm0, %xmm8
-; SSE2-NEXT:    shufps {{.*#+}} xmm8 = xmm8[1,1],xmm0[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm8 = xmm8[1,1],xmm0[1,1]
 ; SSE2-NEXT:    movss %xmm8, (%rdi)
 ; SSE2-NEXT:    addq $4, %rdi
 ; SSE2-NEXT:    testb $4, %al
@@ -1409,7 +1409,7 @@ define void @compressstore_v32f32_v32i32(float* %base, <32 x float> %V, <32 x i3
 ; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je LBB6_8
 ; SSE2-NEXT:  LBB6_7: ## %cond.store7
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE2-NEXT:    movss %xmm0, (%rdi)
 ; SSE2-NEXT:    addq $4, %rdi
 ; SSE2-NEXT:    testb $16, %al
@@ -1421,7 +1421,7 @@ define void @compressstore_v32f32_v32i32(float* %base, <32 x float> %V, <32 x i3
 ; SSE2-NEXT:    je LBB6_12
 ; SSE2-NEXT:  LBB6_11: ## %cond.store13
 ; SSE2-NEXT:    movaps %xmm1, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[1,1]
 ; SSE2-NEXT:    movss %xmm0, (%rdi)
 ; SSE2-NEXT:    addq $4, %rdi
 ; SSE2-NEXT:    testb $64, %al
@@ -1434,7 +1434,7 @@ define void @compressstore_v32f32_v32i32(float* %base, <32 x float> %V, <32 x i3
 ; SSE2-NEXT:    testb $-128, %al
 ; SSE2-NEXT:    je LBB6_16
 ; SSE2-NEXT:  LBB6_15: ## %cond.store19
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; SSE2-NEXT:    movss %xmm1, (%rdi)
 ; SSE2-NEXT:    addq $4, %rdi
 ; SSE2-NEXT:    testl $256, %eax ## imm = 0x100
@@ -1446,7 +1446,7 @@ define void @compressstore_v32f32_v32i32(float* %base, <32 x float> %V, <32 x i3
 ; SSE2-NEXT:    je LBB6_20
 ; SSE2-NEXT:  LBB6_19: ## %cond.store25
 ; SSE2-NEXT:    movaps %xmm2, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm2[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm2[1,1]
 ; SSE2-NEXT:    movss %xmm0, (%rdi)
 ; SSE2-NEXT:    addq $4, %rdi
 ; SSE2-NEXT:    testl $1024, %eax ## imm = 0x400
@@ -1459,7 +1459,7 @@ define void @compressstore_v32f32_v32i32(float* %base, <32 x float> %V, <32 x i3
 ; SSE2-NEXT:    testl $2048, %eax ## imm = 0x800
 ; SSE2-NEXT:    je LBB6_24
 ; SSE2-NEXT:  LBB6_23: ## %cond.store31
-; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,3,3,3]
 ; SSE2-NEXT:    movss %xmm2, (%rdi)
 ; SSE2-NEXT:    addq $4, %rdi
 ; SSE2-NEXT:    testl $4096, %eax ## imm = 0x1000
@@ -1471,7 +1471,7 @@ define void @compressstore_v32f32_v32i32(float* %base, <32 x float> %V, <32 x i3
 ; SSE2-NEXT:    je LBB6_28
 ; SSE2-NEXT:  LBB6_27: ## %cond.store37
 ; SSE2-NEXT:    movaps %xmm3, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm3[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm3[1,1]
 ; SSE2-NEXT:    movss %xmm0, (%rdi)
 ; SSE2-NEXT:    addq $4, %rdi
 ; SSE2-NEXT:    testl $16384, %eax ## imm = 0x4000
@@ -1484,7 +1484,7 @@ define void @compressstore_v32f32_v32i32(float* %base, <32 x float> %V, <32 x i3
 ; SSE2-NEXT:    testl $32768, %eax ## imm = 0x8000
 ; SSE2-NEXT:    je LBB6_32
 ; SSE2-NEXT:  LBB6_31: ## %cond.store43
-; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,3,3,3]
 ; SSE2-NEXT:    movss %xmm3, (%rdi)
 ; SSE2-NEXT:    addq $4, %rdi
 ; SSE2-NEXT:    testl $65536, %eax ## imm = 0x10000
@@ -1496,7 +1496,7 @@ define void @compressstore_v32f32_v32i32(float* %base, <32 x float> %V, <32 x i3
 ; SSE2-NEXT:    je LBB6_36
 ; SSE2-NEXT:  LBB6_35: ## %cond.store49
 ; SSE2-NEXT:    movaps %xmm4, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm4[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm4[1,1]
 ; SSE2-NEXT:    movss %xmm0, (%rdi)
 ; SSE2-NEXT:    addq $4, %rdi
 ; SSE2-NEXT:    testl $262144, %eax ## imm = 0x40000
@@ -1509,7 +1509,7 @@ define void @compressstore_v32f32_v32i32(float* %base, <32 x float> %V, <32 x i3
 ; SSE2-NEXT:    testl $524288, %eax ## imm = 0x80000
 ; SSE2-NEXT:    je LBB6_40
 ; SSE2-NEXT:  LBB6_39: ## %cond.store55
-; SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,3,3,3]
 ; SSE2-NEXT:    movss %xmm4, (%rdi)
 ; SSE2-NEXT:    addq $4, %rdi
 ; SSE2-NEXT:    testl $1048576, %eax ## imm = 0x100000
@@ -1521,7 +1521,7 @@ define void @compressstore_v32f32_v32i32(float* %base, <32 x float> %V, <32 x i3
 ; SSE2-NEXT:    je LBB6_44
 ; SSE2-NEXT:  LBB6_43: ## %cond.store61
 ; SSE2-NEXT:    movaps %xmm5, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm5[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm5[1,1]
 ; SSE2-NEXT:    movss %xmm0, (%rdi)
 ; SSE2-NEXT:    addq $4, %rdi
 ; SSE2-NEXT:    testl $4194304, %eax ## imm = 0x400000
@@ -1534,7 +1534,7 @@ define void @compressstore_v32f32_v32i32(float* %base, <32 x float> %V, <32 x i3
 ; SSE2-NEXT:    testl $8388608, %eax ## imm = 0x800000
 ; SSE2-NEXT:    je LBB6_48
 ; SSE2-NEXT:  LBB6_47: ## %cond.store67
-; SSE2-NEXT:    shufps {{.*#+}} xmm5 = xmm5[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm5 = xmm5[3,3,3,3]
 ; SSE2-NEXT:    movss %xmm5, (%rdi)
 ; SSE2-NEXT:    addq $4, %rdi
 ; SSE2-NEXT:    testl $16777216, %eax ## imm = 0x1000000
@@ -1546,7 +1546,7 @@ define void @compressstore_v32f32_v32i32(float* %base, <32 x float> %V, <32 x i3
 ; SSE2-NEXT:    je LBB6_52
 ; SSE2-NEXT:  LBB6_51: ## %cond.store73
 ; SSE2-NEXT:    movaps %xmm6, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm6[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm6[1,1]
 ; SSE2-NEXT:    movss %xmm0, (%rdi)
 ; SSE2-NEXT:    addq $4, %rdi
 ; SSE2-NEXT:    testl $67108864, %eax ## imm = 0x4000000
@@ -1559,7 +1559,7 @@ define void @compressstore_v32f32_v32i32(float* %base, <32 x float> %V, <32 x i3
 ; SSE2-NEXT:    testl $134217728, %eax ## imm = 0x8000000
 ; SSE2-NEXT:    je LBB6_56
 ; SSE2-NEXT:  LBB6_55: ## %cond.store79
-; SSE2-NEXT:    shufps {{.*#+}} xmm6 = xmm6[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm6 = xmm6[3,3,3,3]
 ; SSE2-NEXT:    movss %xmm6, (%rdi)
 ; SSE2-NEXT:    addq $4, %rdi
 ; SSE2-NEXT:    testl $268435456, %eax ## imm = 0x10000000
@@ -1571,7 +1571,7 @@ define void @compressstore_v32f32_v32i32(float* %base, <32 x float> %V, <32 x i3
 ; SSE2-NEXT:    je LBB6_60
 ; SSE2-NEXT:  LBB6_59: ## %cond.store85
 ; SSE2-NEXT:    movaps %xmm7, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm7[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm7[1,1]
 ; SSE2-NEXT:    movss %xmm0, (%rdi)
 ; SSE2-NEXT:    addq $4, %rdi
 ; SSE2-NEXT:    testl $1073741824, %eax ## imm = 0x40000000
@@ -1584,7 +1584,7 @@ define void @compressstore_v32f32_v32i32(float* %base, <32 x float> %V, <32 x i3
 ; SSE2-NEXT:    testl $-2147483648, %eax ## imm = 0x80000000
 ; SSE2-NEXT:    je LBB6_64
 ; SSE2-NEXT:  LBB6_63: ## %cond.store91
-; SSE2-NEXT:    shufps {{.*#+}} xmm7 = xmm7[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm7 = xmm7[3,3,3,3]
 ; SSE2-NEXT:    movss %xmm7, (%rdi)
 ; SSE2-NEXT:    retq
 ;
@@ -3062,7 +3062,7 @@ define void @compressstore_v4i32_v4i32(i32* %base, <4 x i32> %V, <4 x i32> %trig
 ; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je LBB10_4
 ; SSE2-NEXT:  LBB10_3: ## %cond.store1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    movd %xmm1, (%rdi)
 ; SSE2-NEXT:    addq $4, %rdi
 ; SSE2-NEXT:    testb $4, %al
@@ -3074,7 +3074,7 @@ define void @compressstore_v4i32_v4i32(i32* %base, <4 x i32> %V, <4 x i32> %trig
 ; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je LBB10_8
 ; SSE2-NEXT:  LBB10_7: ## %cond.store7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE2-NEXT:    movd %xmm0, (%rdi)
 ; SSE2-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/masked_expandload.ll b/llvm/test/CodeGen/X86/masked_expandload.ll
index 7d9e841c624f8..344f300b959c9 100644
--- a/llvm/test/CodeGen/X86/masked_expandload.ll
+++ b/llvm/test/CodeGen/X86/masked_expandload.ll
@@ -3499,7 +3499,7 @@ define <16 x i8> @expandload_v16i8_v16i8(i8* %base, <16 x i8> %src0, <16 x i8> %
 ; SSE2-NEXT:    pand %xmm1, %xmm0
 ; SSE2-NEXT:    movzbl (%rdi), %ecx
 ; SSE2-NEXT:    movd %ecx, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,1,2,0]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
 ; SSE2-NEXT:    pandn %xmm2, %xmm1
 ; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    incq %rdi
diff --git a/llvm/test/CodeGen/X86/masked_load.ll b/llvm/test/CodeGen/X86/masked_load.ll
index e8dc7412eef8b..4f5687109afc2 100644
--- a/llvm/test/CodeGen/X86/masked_load.ll
+++ b/llvm/test/CodeGen/X86/masked_load.ll
@@ -468,7 +468,7 @@ define <8 x double> @load_v8f64_v8i16(<8 x i16> %trigger, <8 x double>* %addr, <
 ; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm5, %ymm3
 ; AVX1-NEXT:    vpcmpeqw %xmm4, %xmm0, %xmm0
 ; AVX1-NEXT:    vpmovsxwq %xmm0, %xmm4
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmovsxwq %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm4, %ymm0
 ; AVX1-NEXT:    vmaskmovpd (%rdi), %ymm0, %ymm4
@@ -1788,7 +1788,7 @@ define <8 x i64> @load_v8i64_v8i16(<8 x i16> %trigger, <8 x i64>* %addr, <8 x i6
 ; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm5, %ymm3
 ; AVX1-NEXT:    vpcmpeqw %xmm4, %xmm0, %xmm0
 ; AVX1-NEXT:    vpmovsxwq %xmm0, %xmm4
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmovsxwq %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm4, %ymm0
 ; AVX1-NEXT:    vmaskmovpd (%rdi), %ymm0, %ymm4
@@ -3815,7 +3815,7 @@ define <16 x i8> @load_v16i8_v16i8(<16 x i8> %trigger, <16 x i8>* %addr, <16 x i
 ; SSE2-NEXT:    pand %xmm0, %xmm1
 ; SSE2-NEXT:    movzbl 12(%rdi), %ecx
 ; SSE2-NEXT:    movd %ecx, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,1,2,0]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
 ; SSE2-NEXT:    pandn %xmm2, %xmm0
 ; SSE2-NEXT:    por %xmm0, %xmm1
 ; SSE2-NEXT:    testl $8192, %eax ## imm = 0x2000
@@ -4567,7 +4567,7 @@ define <32 x i8> @load_v32i8_v32i8(<32 x i8> %trigger, <32 x i8>* %addr, <32 x i
 ; SSE2-NEXT:    pand %xmm0, %xmm2
 ; SSE2-NEXT:    movzbl 12(%rdi), %ecx
 ; SSE2-NEXT:    movd %ecx, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,1,2,0]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; SSE2-NEXT:    pandn %xmm1, %xmm0
 ; SSE2-NEXT:    por %xmm0, %xmm2
 ; SSE2-NEXT:    testl $8192, %eax ## imm = 0x2000
@@ -4724,7 +4724,7 @@ define <32 x i8> @load_v32i8_v32i8(<32 x i8> %trigger, <32 x i8>* %addr, <32 x i
 ; SSE2-NEXT:    pand %xmm0, %xmm3
 ; SSE2-NEXT:    movzbl 28(%rdi), %ecx
 ; SSE2-NEXT:    movd %ecx, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,1,2,0]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; SSE2-NEXT:    pandn %xmm1, %xmm0
 ; SSE2-NEXT:    por %xmm0, %xmm3
 ; SSE2-NEXT:    testl $536870912, %eax ## imm = 0x20000000
diff --git a/llvm/test/CodeGen/X86/masked_store.ll b/llvm/test/CodeGen/X86/masked_store.ll
index 389281726d273..380891847a5c2 100644
--- a/llvm/test/CodeGen/X86/masked_store.ll
+++ b/llvm/test/CodeGen/X86/masked_store.ll
@@ -181,7 +181,7 @@ define void @store_v2f32_v2i32(<2 x i32> %trigger, <2 x float>* %addr, <2 x floa
 ; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je LBB3_4
 ; SSE2-NEXT:  LBB3_3: ## %cond.store1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,1,1]
 ; SSE2-NEXT:    movss %xmm1, 4(%rdi)
 ; SSE2-NEXT:    retq
 ;
@@ -268,7 +268,7 @@ define void @store_v4f32_v4i32(<4 x float> %x, <4 x float>* %ptr, <4 x float> %y
 ; SSE2-NEXT:    je LBB4_4
 ; SSE2-NEXT:  LBB4_3: ## %cond.store1
 ; SSE2-NEXT:    movaps %xmm0, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[1,1]
 ; SSE2-NEXT:    movss %xmm1, 4(%rdi)
 ; SSE2-NEXT:    testb $4, %al
 ; SSE2-NEXT:    je LBB4_6
@@ -279,7 +279,7 @@ define void @store_v4f32_v4i32(<4 x float> %x, <4 x float>* %ptr, <4 x float> %y
 ; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je LBB4_8
 ; SSE2-NEXT:  LBB4_7: ## %cond.store5
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE2-NEXT:    movss %xmm0, 12(%rdi)
 ; SSE2-NEXT:    retq
 ;
@@ -386,7 +386,7 @@ define void @store_v8f32_v8i32(<8 x float> %x, <8 x float>* %ptr, <8 x float> %y
 ; SSE2-NEXT:    je LBB5_4
 ; SSE2-NEXT:  LBB5_3: ## %cond.store1
 ; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1],xmm0[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1],xmm0[1,1]
 ; SSE2-NEXT:    movss %xmm2, 4(%rdi)
 ; SSE2-NEXT:    testb $4, %al
 ; SSE2-NEXT:    je LBB5_6
@@ -397,7 +397,7 @@ define void @store_v8f32_v8i32(<8 x float> %x, <8 x float>* %ptr, <8 x float> %y
 ; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je LBB5_8
 ; SSE2-NEXT:  LBB5_7: ## %cond.store5
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE2-NEXT:    movss %xmm0, 12(%rdi)
 ; SSE2-NEXT:    testb $16, %al
 ; SSE2-NEXT:    je LBB5_10
@@ -407,7 +407,7 @@ define void @store_v8f32_v8i32(<8 x float> %x, <8 x float>* %ptr, <8 x float> %y
 ; SSE2-NEXT:    je LBB5_12
 ; SSE2-NEXT:  LBB5_11: ## %cond.store9
 ; SSE2-NEXT:    movaps %xmm1, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[1,1]
 ; SSE2-NEXT:    movss %xmm0, 20(%rdi)
 ; SSE2-NEXT:    testb $64, %al
 ; SSE2-NEXT:    je LBB5_14
@@ -418,7 +418,7 @@ define void @store_v8f32_v8i32(<8 x float> %x, <8 x float>* %ptr, <8 x float> %y
 ; SSE2-NEXT:    testb $-128, %al
 ; SSE2-NEXT:    je LBB5_16
 ; SSE2-NEXT:  LBB5_15: ## %cond.store13
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; SSE2-NEXT:    movss %xmm1, 28(%rdi)
 ; SSE2-NEXT:    retq
 ;
@@ -585,7 +585,7 @@ define void @store_v16f32_v16i32(<16 x float> %x, <16 x float>* %ptr, <16 x floa
 ; SSE2-NEXT:    je LBB6_4
 ; SSE2-NEXT:  LBB6_3: ## %cond.store1
 ; SSE2-NEXT:    movaps %xmm0, %xmm4
-; SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[1,1],xmm0[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[1,1],xmm0[1,1]
 ; SSE2-NEXT:    movss %xmm4, 4(%rdi)
 ; SSE2-NEXT:    testb $4, %al
 ; SSE2-NEXT:    je LBB6_6
@@ -596,7 +596,7 @@ define void @store_v16f32_v16i32(<16 x float> %x, <16 x float>* %ptr, <16 x floa
 ; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je LBB6_8
 ; SSE2-NEXT:  LBB6_7: ## %cond.store5
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE2-NEXT:    movss %xmm0, 12(%rdi)
 ; SSE2-NEXT:    testb $16, %al
 ; SSE2-NEXT:    je LBB6_10
@@ -606,7 +606,7 @@ define void @store_v16f32_v16i32(<16 x float> %x, <16 x float>* %ptr, <16 x floa
 ; SSE2-NEXT:    je LBB6_12
 ; SSE2-NEXT:  LBB6_11: ## %cond.store9
 ; SSE2-NEXT:    movaps %xmm1, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[1,1]
 ; SSE2-NEXT:    movss %xmm0, 20(%rdi)
 ; SSE2-NEXT:    testb $64, %al
 ; SSE2-NEXT:    je LBB6_14
@@ -617,7 +617,7 @@ define void @store_v16f32_v16i32(<16 x float> %x, <16 x float>* %ptr, <16 x floa
 ; SSE2-NEXT:    testb $-128, %al
 ; SSE2-NEXT:    je LBB6_16
 ; SSE2-NEXT:  LBB6_15: ## %cond.store13
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; SSE2-NEXT:    movss %xmm1, 28(%rdi)
 ; SSE2-NEXT:    testl $256, %eax ## imm = 0x100
 ; SSE2-NEXT:    je LBB6_18
@@ -627,7 +627,7 @@ define void @store_v16f32_v16i32(<16 x float> %x, <16 x float>* %ptr, <16 x floa
 ; SSE2-NEXT:    je LBB6_20
 ; SSE2-NEXT:  LBB6_19: ## %cond.store17
 ; SSE2-NEXT:    movaps %xmm2, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm2[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm2[1,1]
 ; SSE2-NEXT:    movss %xmm0, 36(%rdi)
 ; SSE2-NEXT:    testl $1024, %eax ## imm = 0x400
 ; SSE2-NEXT:    je LBB6_22
@@ -638,7 +638,7 @@ define void @store_v16f32_v16i32(<16 x float> %x, <16 x float>* %ptr, <16 x floa
 ; SSE2-NEXT:    testl $2048, %eax ## imm = 0x800
 ; SSE2-NEXT:    je LBB6_24
 ; SSE2-NEXT:  LBB6_23: ## %cond.store21
-; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,3,3,3]
 ; SSE2-NEXT:    movss %xmm2, 44(%rdi)
 ; SSE2-NEXT:    testl $4096, %eax ## imm = 0x1000
 ; SSE2-NEXT:    je LBB6_26
@@ -648,7 +648,7 @@ define void @store_v16f32_v16i32(<16 x float> %x, <16 x float>* %ptr, <16 x floa
 ; SSE2-NEXT:    je LBB6_28
 ; SSE2-NEXT:  LBB6_27: ## %cond.store25
 ; SSE2-NEXT:    movaps %xmm3, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm3[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm3[1,1]
 ; SSE2-NEXT:    movss %xmm0, 52(%rdi)
 ; SSE2-NEXT:    testl $16384, %eax ## imm = 0x4000
 ; SSE2-NEXT:    je LBB6_30
@@ -659,7 +659,7 @@ define void @store_v16f32_v16i32(<16 x float> %x, <16 x float>* %ptr, <16 x floa
 ; SSE2-NEXT:    testl $32768, %eax ## imm = 0x8000
 ; SSE2-NEXT:    je LBB6_32
 ; SSE2-NEXT:  LBB6_31: ## %cond.store29
-; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,3,3,3]
 ; SSE2-NEXT:    movss %xmm3, 60(%rdi)
 ; SSE2-NEXT:    retq
 ;
@@ -1057,7 +1057,7 @@ define void @store_v2i32_v2i32(<2 x i32> %trigger, <2 x i32>* %addr, <2 x i32> %
 ; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je LBB10_4
 ; SSE2-NEXT:  LBB10_3: ## %cond.store1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE2-NEXT:    movd %xmm0, 4(%rdi)
 ; SSE2-NEXT:    retq
 ;
@@ -1153,7 +1153,7 @@ define void @store_v4i32_v4i32(<4 x i32> %trigger, <4 x i32>* %addr, <4 x i32> %
 ; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je LBB11_4
 ; SSE2-NEXT:  LBB11_3: ## %cond.store1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE2-NEXT:    movd %xmm0, 4(%rdi)
 ; SSE2-NEXT:    testb $4, %al
 ; SSE2-NEXT:    je LBB11_6
@@ -1163,7 +1163,7 @@ define void @store_v4i32_v4i32(<4 x i32> %trigger, <4 x i32>* %addr, <4 x i32> %
 ; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je LBB11_8
 ; SSE2-NEXT:  LBB11_7: ## %cond.store5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[3,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[3,3,3,3]
 ; SSE2-NEXT:    movd %xmm0, 12(%rdi)
 ; SSE2-NEXT:    retq
 ;
@@ -1275,7 +1275,7 @@ define void @store_v8i32_v8i32(<8 x i32> %trigger, <8 x i32>* %addr, <8 x i32> %
 ; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je LBB12_4
 ; SSE2-NEXT:  LBB12_3: ## %cond.store1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
 ; SSE2-NEXT:    movd %xmm0, 4(%rdi)
 ; SSE2-NEXT:    testb $4, %al
 ; SSE2-NEXT:    je LBB12_6
@@ -1285,7 +1285,7 @@ define void @store_v8i32_v8i32(<8 x i32> %trigger, <8 x i32>* %addr, <8 x i32> %
 ; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je LBB12_8
 ; SSE2-NEXT:  LBB12_7: ## %cond.store5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[3,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[3,3,3,3]
 ; SSE2-NEXT:    movd %xmm0, 12(%rdi)
 ; SSE2-NEXT:    testb $16, %al
 ; SSE2-NEXT:    je LBB12_10
@@ -1294,7 +1294,7 @@ define void @store_v8i32_v8i32(<8 x i32> %trigger, <8 x i32>* %addr, <8 x i32> %
 ; SSE2-NEXT:    testb $32, %al
 ; SSE2-NEXT:    je LBB12_12
 ; SSE2-NEXT:  LBB12_11: ## %cond.store9
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,1,1]
 ; SSE2-NEXT:    movd %xmm0, 20(%rdi)
 ; SSE2-NEXT:    testb $64, %al
 ; SSE2-NEXT:    je LBB12_14
@@ -1304,7 +1304,7 @@ define void @store_v8i32_v8i32(<8 x i32> %trigger, <8 x i32>* %addr, <8 x i32> %
 ; SSE2-NEXT:    testb $-128, %al
 ; SSE2-NEXT:    je LBB12_16
 ; SSE2-NEXT:  LBB12_15: ## %cond.store13
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[3,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[3,3,3,3]
 ; SSE2-NEXT:    movd %xmm0, 28(%rdi)
 ; SSE2-NEXT:    retq
 ;
@@ -4746,7 +4746,7 @@ define void @one_mask_bit_set1_variable(<4 x float>* %addr, <4 x float> %val, <4
 ; SSE2-NEXT:    je LBB24_4
 ; SSE2-NEXT:  LBB24_3: ## %cond.store1
 ; SSE2-NEXT:    movaps %xmm0, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[1,1]
 ; SSE2-NEXT:    movss %xmm1, 4(%rdi)
 ; SSE2-NEXT:    testb $4, %al
 ; SSE2-NEXT:    je LBB24_6
@@ -4757,7 +4757,7 @@ define void @one_mask_bit_set1_variable(<4 x float>* %addr, <4 x float> %val, <4
 ; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je LBB24_8
 ; SSE2-NEXT:  LBB24_7: ## %cond.store5
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE2-NEXT:    movss %xmm0, 12(%rdi)
 ; SSE2-NEXT:    retq
 ;
@@ -4848,7 +4848,7 @@ define void @widen_masked_store(<3 x i32> %v, <3 x i32>* %p, <3 x i1> %mask) {
 ; SSE2-NEXT:    testb $2, %cl
 ; SSE2-NEXT:    je LBB25_4
 ; SSE2-NEXT:  LBB25_3: ## %cond.store1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    movd %xmm1, 4(%rdi)
 ; SSE2-NEXT:    testb $4, %cl
 ; SSE2-NEXT:    je LBB25_6
@@ -5028,7 +5028,7 @@ define void @PR11210(<4 x float> %x, <4 x float>* %ptr, <4 x float> %y, <2 x i64
 ; SSE2-NEXT:    je LBB27_4
 ; SSE2-NEXT:  LBB27_3: ## %cond.store1
 ; SSE2-NEXT:    movaps %xmm0, %xmm2
-; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1],xmm0[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1],xmm0[1,1]
 ; SSE2-NEXT:    movss %xmm2, 4(%rdi)
 ; SSE2-NEXT:    testb $4, %al
 ; SSE2-NEXT:    je LBB27_6
@@ -5039,7 +5039,7 @@ define void @PR11210(<4 x float> %x, <4 x float>* %ptr, <4 x float> %y, <2 x i64
 ; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je LBB27_8
 ; SSE2-NEXT:  LBB27_7: ## %cond.store5
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE2-NEXT:    movss %xmm0, 12(%rdi)
 ; SSE2-NEXT:    testb $1, %al
 ; SSE2-NEXT:    je LBB27_10
@@ -5049,7 +5049,7 @@ define void @PR11210(<4 x float> %x, <4 x float>* %ptr, <4 x float> %y, <2 x i64
 ; SSE2-NEXT:    je LBB27_12
 ; SSE2-NEXT:  LBB27_11: ## %cond.store10
 ; SSE2-NEXT:    movaps %xmm1, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[1,1]
 ; SSE2-NEXT:    movss %xmm0, 4(%rdi)
 ; SSE2-NEXT:    testb $4, %al
 ; SSE2-NEXT:    je LBB27_14
@@ -5060,7 +5060,7 @@ define void @PR11210(<4 x float> %x, <4 x float>* %ptr, <4 x float> %y, <2 x i64
 ; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je LBB27_16
 ; SSE2-NEXT:  LBB27_15: ## %cond.store14
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; SSE2-NEXT:    movss %xmm1, 12(%rdi)
 ; SSE2-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/masked_store_trunc.ll b/llvm/test/CodeGen/X86/masked_store_trunc.ll
index 640e145c20231..6cf5ff023f3cc 100644
--- a/llvm/test/CodeGen/X86/masked_store_trunc.ll
+++ b/llvm/test/CodeGen/X86/masked_store_trunc.ll
@@ -33,7 +33,7 @@ define void @truncstore_v8i64_v8i32(<8 x i64> %x, <8 x i32>* %p, <8 x i32> %mask
 ; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je .LBB0_8
 ; SSE2-NEXT:  .LBB0_7: # %cond.store5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE2-NEXT:    movd %xmm0, 12(%rdi)
 ; SSE2-NEXT:  .LBB0_8: # %else6
 ; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,2],xmm3[0,2]
@@ -55,7 +55,7 @@ define void @truncstore_v8i64_v8i32(<8 x i64> %x, <8 x i32>* %p, <8 x i32> %mask
 ; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je .LBB0_4
 ; SSE2-NEXT:  .LBB0_3: # %cond.store1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    movd %xmm1, 4(%rdi)
 ; SSE2-NEXT:    testb $4, %al
 ; SSE2-NEXT:    je .LBB0_6
@@ -70,7 +70,7 @@ define void @truncstore_v8i64_v8i32(<8 x i64> %x, <8 x i32>* %p, <8 x i32> %mask
 ; SSE2-NEXT:    testb $32, %al
 ; SSE2-NEXT:    je .LBB0_12
 ; SSE2-NEXT:  .LBB0_11: # %cond.store9
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
 ; SSE2-NEXT:    movd %xmm0, 20(%rdi)
 ; SSE2-NEXT:    testb $64, %al
 ; SSE2-NEXT:    je .LBB0_14
@@ -80,7 +80,7 @@ define void @truncstore_v8i64_v8i32(<8 x i64> %x, <8 x i32>* %p, <8 x i32> %mask
 ; SSE2-NEXT:    testb $-128, %al
 ; SSE2-NEXT:    je .LBB0_16
 ; SSE2-NEXT:  .LBB0_15: # %cond.store13
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[3,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[3,3,3,3]
 ; SSE2-NEXT:    movd %xmm0, 28(%rdi)
 ; SSE2-NEXT:    retq
 ;
@@ -1025,7 +1025,7 @@ define void @truncstore_v4i64_v4i32(<4 x i64> %x, <4 x i32>* %p, <4 x i32> %mask
 ; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je .LBB3_4
 ; SSE2-NEXT:  .LBB3_3: # %cond.store1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    movd %xmm1, 4(%rdi)
 ; SSE2-NEXT:    testb $4, %al
 ; SSE2-NEXT:    je .LBB3_6
@@ -1035,7 +1035,7 @@ define void @truncstore_v4i64_v4i32(<4 x i64> %x, <4 x i32>* %p, <4 x i32> %mask
 ; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je .LBB3_8
 ; SSE2-NEXT:  .LBB3_7: # %cond.store5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE2-NEXT:    movd %xmm0, 12(%rdi)
 ; SSE2-NEXT:    retq
 ;
@@ -1618,7 +1618,7 @@ define void @truncstore_v2i64_v2i32(<2 x i64> %x, <2 x i32>* %p, <2 x i64> %mask
 ; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je .LBB6_4
 ; SSE2-NEXT:  .LBB6_3: # %cond.store1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    movd %xmm0, 4(%rdi)
 ; SSE2-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/masked_store_trunc_ssat.ll b/llvm/test/CodeGen/X86/masked_store_trunc_ssat.ll
index 17c113f098eb4..026cb42f78d09 100644
--- a/llvm/test/CodeGen/X86/masked_store_trunc_ssat.ll
+++ b/llvm/test/CodeGen/X86/masked_store_trunc_ssat.ll
@@ -131,7 +131,7 @@ define void @truncstore_v8i64_v8i32(<8 x i64> %x, <8 x i32>* %p, <8 x i32> %mask
 ; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je .LBB0_4
 ; SSE2-NEXT:  # %bb.3: # %cond.store1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,1,1]
 ; SSE2-NEXT:    movd %xmm4, 4(%rdi)
 ; SSE2-NEXT:  .LBB0_4: # %else2
 ; SSE2-NEXT:    pand %xmm3, %xmm2
@@ -149,7 +149,7 @@ define void @truncstore_v8i64_v8i32(<8 x i64> %x, <8 x i32>* %p, <8 x i32> %mask
 ; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je .LBB0_8
 ; SSE2-NEXT:  # %bb.7: # %cond.store5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[3,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[3,3,3,3]
 ; SSE2-NEXT:    movd %xmm0, 12(%rdi)
 ; SSE2-NEXT:  .LBB0_8: # %else6
 ; SSE2-NEXT:    shufps {{.*#+}} xmm13 = xmm13[0,2],xmm2[0,2]
@@ -171,7 +171,7 @@ define void @truncstore_v8i64_v8i32(<8 x i64> %x, <8 x i32>* %p, <8 x i32> %mask
 ; SSE2-NEXT:    testb $32, %al
 ; SSE2-NEXT:    je .LBB0_12
 ; SSE2-NEXT:  .LBB0_11: # %cond.store9
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm13[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm13[1,1,1,1]
 ; SSE2-NEXT:    movd %xmm0, 20(%rdi)
 ; SSE2-NEXT:    testb $64, %al
 ; SSE2-NEXT:    je .LBB0_14
@@ -181,7 +181,7 @@ define void @truncstore_v8i64_v8i32(<8 x i64> %x, <8 x i32>* %p, <8 x i32> %mask
 ; SSE2-NEXT:    testb $-128, %al
 ; SSE2-NEXT:    je .LBB0_16
 ; SSE2-NEXT:  .LBB0_15: # %cond.store13
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm13[3,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm13[3,3,3,3]
 ; SSE2-NEXT:    movd %xmm0, 28(%rdi)
 ; SSE2-NEXT:    retq
 ;
@@ -1574,7 +1574,7 @@ define void @truncstore_v4i64_v4i32(<4 x i64> %x, <4 x i32>* %p, <4 x i32> %mask
 ; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je .LBB3_4
 ; SSE2-NEXT:  .LBB3_3: # %cond.store1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    movd %xmm1, 4(%rdi)
 ; SSE2-NEXT:    testb $4, %al
 ; SSE2-NEXT:    je .LBB3_6
@@ -1584,7 +1584,7 @@ define void @truncstore_v4i64_v4i32(<4 x i64> %x, <4 x i32>* %p, <4 x i32> %mask
 ; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je .LBB3_8
 ; SSE2-NEXT:  .LBB3_7: # %cond.store5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE2-NEXT:    movd %xmm0, 12(%rdi)
 ; SSE2-NEXT:    retq
 ;
@@ -2415,7 +2415,7 @@ define void @truncstore_v2i64_v2i32(<2 x i64> %x, <2 x i32>* %p, <2 x i64> %mask
 ; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je .LBB6_4
 ; SSE2-NEXT:  .LBB6_3: # %cond.store1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    movd %xmm0, 4(%rdi)
 ; SSE2-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/masked_store_trunc_usat.ll b/llvm/test/CodeGen/X86/masked_store_trunc_usat.ll
index 2ca9ebb0d5c6d..876476bd8c57c 100644
--- a/llvm/test/CodeGen/X86/masked_store_trunc_usat.ll
+++ b/llvm/test/CodeGen/X86/masked_store_trunc_usat.ll
@@ -77,7 +77,7 @@ define void @truncstore_v8i64_v8i32(<8 x i64> %x, <8 x i32>* %p, <8 x i32> %mask
 ; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je .LBB0_4
 ; SSE2-NEXT:  # %bb.3: # %cond.store1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm13[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm13[1,1,1,1]
 ; SSE2-NEXT:    movd %xmm1, 4(%rdi)
 ; SSE2-NEXT:  .LBB0_4: # %else2
 ; SSE2-NEXT:    pand %xmm0, %xmm3
@@ -95,7 +95,7 @@ define void @truncstore_v8i64_v8i32(<8 x i64> %x, <8 x i32>* %p, <8 x i32> %mask
 ; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je .LBB0_8
 ; SSE2-NEXT:  # %bb.7: # %cond.store5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm13[3,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm13[3,3,3,3]
 ; SSE2-NEXT:    movd %xmm0, 12(%rdi)
 ; SSE2-NEXT:  .LBB0_8: # %else6
 ; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,2],xmm3[0,2]
@@ -117,7 +117,7 @@ define void @truncstore_v8i64_v8i32(<8 x i64> %x, <8 x i32>* %p, <8 x i32> %mask
 ; SSE2-NEXT:    testb $32, %al
 ; SSE2-NEXT:    je .LBB0_12
 ; SSE2-NEXT:  .LBB0_11: # %cond.store9
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
 ; SSE2-NEXT:    movd %xmm0, 20(%rdi)
 ; SSE2-NEXT:    testb $64, %al
 ; SSE2-NEXT:    je .LBB0_14
@@ -127,7 +127,7 @@ define void @truncstore_v8i64_v8i32(<8 x i64> %x, <8 x i32>* %p, <8 x i32> %mask
 ; SSE2-NEXT:    testb $-128, %al
 ; SSE2-NEXT:    je .LBB0_16
 ; SSE2-NEXT:  .LBB0_15: # %cond.store13
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[3,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[3,3,3,3]
 ; SSE2-NEXT:    movd %xmm0, 28(%rdi)
 ; SSE2-NEXT:    retq
 ;
@@ -1346,7 +1346,7 @@ define void @truncstore_v4i64_v4i32(<4 x i64> %x, <4 x i32>* %p, <4 x i32> %mask
 ; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je .LBB3_4
 ; SSE2-NEXT:  .LBB3_3: # %cond.store1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE2-NEXT:    movd %xmm0, 4(%rdi)
 ; SSE2-NEXT:    testb $4, %al
 ; SSE2-NEXT:    je .LBB3_6
@@ -1356,7 +1356,7 @@ define void @truncstore_v4i64_v4i32(<4 x i64> %x, <4 x i32>* %p, <4 x i32> %mask
 ; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je .LBB3_8
 ; SSE2-NEXT:  .LBB3_7: # %cond.store5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[3,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[3,3,3,3]
 ; SSE2-NEXT:    movd %xmm0, 12(%rdi)
 ; SSE2-NEXT:    retq
 ;
@@ -2098,7 +2098,7 @@ define void @truncstore_v2i64_v2i32(<2 x i64> %x, <2 x i32>* %p, <2 x i64> %mask
 ; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je .LBB6_4
 ; SSE2-NEXT:  .LBB6_3: # %cond.store1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    movd %xmm0, 4(%rdi)
 ; SSE2-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/memset-nonzero.ll b/llvm/test/CodeGen/X86/memset-nonzero.ll
index 6e3dedbf83039..c499a1aa37945 100644
--- a/llvm/test/CodeGen/X86/memset-nonzero.ll
+++ b/llvm/test/CodeGen/X86/memset-nonzero.ll
@@ -296,7 +296,7 @@ define void @memset_16_nonconst_bytes(i8* %x, i8 %c) {
 ; SSE2FAST:       # %bb.0:
 ; SSE2FAST-NEXT:    movd %esi, %xmm0
 ; SSE2FAST-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2FAST-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; SSE2FAST-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; SSE2FAST-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; SSE2FAST-NEXT:    movdqu %xmm0, (%rdi)
 ; SSE2FAST-NEXT:    retq
@@ -343,7 +343,7 @@ define void @memset_32_nonconst_bytes(i8* %x, i8 %c) {
 ; SSE2FAST:       # %bb.0:
 ; SSE2FAST-NEXT:    movd %esi, %xmm0
 ; SSE2FAST-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2FAST-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; SSE2FAST-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; SSE2FAST-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; SSE2FAST-NEXT:    movdqu %xmm0, 16(%rdi)
 ; SSE2FAST-NEXT:    movdqu %xmm0, (%rdi)
@@ -398,7 +398,7 @@ define void @memset_64_nonconst_bytes(i8* %x, i8 %c) {
 ; SSE2FAST:       # %bb.0:
 ; SSE2FAST-NEXT:    movd %esi, %xmm0
 ; SSE2FAST-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2FAST-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; SSE2FAST-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; SSE2FAST-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; SSE2FAST-NEXT:    movdqu %xmm0, 48(%rdi)
 ; SSE2FAST-NEXT:    movdqu %xmm0, 32(%rdi)
@@ -474,7 +474,7 @@ define void @memset_128_nonconst_bytes(i8* %x, i8 %c) {
 ; SSE2FAST:       # %bb.0:
 ; SSE2FAST-NEXT:    movd %esi, %xmm0
 ; SSE2FAST-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2FAST-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; SSE2FAST-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; SSE2FAST-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; SSE2FAST-NEXT:    movdqu %xmm0, 112(%rdi)
 ; SSE2FAST-NEXT:    movdqu %xmm0, 96(%rdi)
@@ -541,7 +541,7 @@ define void @memset_256_nonconst_bytes(i8* %x, i8 %c) {
 ; SSE2FAST:       # %bb.0:
 ; SSE2FAST-NEXT:    movd %esi, %xmm0
 ; SSE2FAST-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2FAST-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; SSE2FAST-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; SSE2FAST-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; SSE2FAST-NEXT:    movdqu %xmm0, 240(%rdi)
 ; SSE2FAST-NEXT:    movdqu %xmm0, 224(%rdi)
diff --git a/llvm/test/CodeGen/X86/merge-consecutive-stores-nt.ll b/llvm/test/CodeGen/X86/merge-consecutive-stores-nt.ll
index 6dcc47b9a65c4..94bd6fa114c3e 100644
--- a/llvm/test/CodeGen/X86/merge-consecutive-stores-nt.ll
+++ b/llvm/test/CodeGen/X86/merge-consecutive-stores-nt.ll
@@ -306,24 +306,24 @@ define void @merge_2_v4f32_align1_ntstore(<4 x float>* %a0, <4 x float>* %a1) no
 ; X86-SSE2-NEXT:    movdqu 16(%ecx), %xmm1
 ; X86-SSE2-NEXT:    movd %xmm0, %ecx
 ; X86-SSE2-NEXT:    movntil %ecx, (%eax)
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
 ; X86-SSE2-NEXT:    movd %xmm2, %ecx
 ; X86-SSE2-NEXT:    movntil %ecx, 12(%eax)
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    movd %xmm2, %ecx
 ; X86-SSE2-NEXT:    movntil %ecx, 8(%eax)
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; X86-SSE2-NEXT:    movd %xmm0, %ecx
 ; X86-SSE2-NEXT:    movntil %ecx, 4(%eax)
 ; X86-SSE2-NEXT:    movd %xmm1, %ecx
 ; X86-SSE2-NEXT:    movntil %ecx, 16(%eax)
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[3,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[3,3,3,3]
 ; X86-SSE2-NEXT:    movd %xmm0, %ecx
 ; X86-SSE2-NEXT:    movntil %ecx, 28(%eax)
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; X86-SSE2-NEXT:    movd %xmm0, %ecx
 ; X86-SSE2-NEXT:    movntil %ecx, 24(%eax)
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X86-SSE2-NEXT:    movd %xmm0, %ecx
 ; X86-SSE2-NEXT:    movntil %ecx, 20(%eax)
 ; X86-SSE2-NEXT:    retl
@@ -419,24 +419,24 @@ define void @merge_2_v4f32_align1(<4 x float>* %a0, <4 x float>* %a1) nounwind {
 ; X86-SSE2-NEXT:    movdqu 16(%ecx), %xmm1
 ; X86-SSE2-NEXT:    movd %xmm0, %ecx
 ; X86-SSE2-NEXT:    movntil %ecx, (%eax)
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
 ; X86-SSE2-NEXT:    movd %xmm2, %ecx
 ; X86-SSE2-NEXT:    movntil %ecx, 12(%eax)
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; X86-SSE2-NEXT:    movd %xmm2, %ecx
 ; X86-SSE2-NEXT:    movntil %ecx, 8(%eax)
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; X86-SSE2-NEXT:    movd %xmm0, %ecx
 ; X86-SSE2-NEXT:    movntil %ecx, 4(%eax)
 ; X86-SSE2-NEXT:    movd %xmm1, %ecx
 ; X86-SSE2-NEXT:    movntil %ecx, 16(%eax)
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[3,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[3,3,3,3]
 ; X86-SSE2-NEXT:    movd %xmm0, %ecx
 ; X86-SSE2-NEXT:    movntil %ecx, 28(%eax)
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; X86-SSE2-NEXT:    movd %xmm0, %ecx
 ; X86-SSE2-NEXT:    movntil %ecx, 24(%eax)
-; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X86-SSE2-NEXT:    movd %xmm0, %ecx
 ; X86-SSE2-NEXT:    movntil %ecx, 20(%eax)
 ; X86-SSE2-NEXT:    retl
diff --git a/llvm/test/CodeGen/X86/min-legal-vector-width.ll b/llvm/test/CodeGen/X86/min-legal-vector-width.ll
index 666df4a1b960d..fe445d71fc7d6 100644
--- a/llvm/test/CodeGen/X86/min-legal-vector-width.ll
+++ b/llvm/test/CodeGen/X86/min-legal-vector-width.ll
@@ -199,7 +199,7 @@ define i32 @_Z9test_charPcS_i_256(i8* nocapture readonly, i8* nocapture readonly
 ; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    vmovd %xmm0, %eax
 ; CHECK-NEXT:    vzeroupper
@@ -265,7 +265,7 @@ define i32 @_Z9test_charPcS_i_512(i8* nocapture readonly, i8* nocapture readonly
 ; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    vmovd %xmm0, %eax
 ; CHECK-NEXT:    vzeroupper
@@ -329,7 +329,7 @@ define i32 @sad_16i8_256() "min-legal-vector-width"="256" {
 ; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    vmovd %xmm0, %eax
 ; CHECK-NEXT:    vzeroupper
@@ -390,7 +390,7 @@ define i32 @sad_16i8_512() "min-legal-vector-width"="512" {
 ; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    vmovd %xmm0, %eax
 ; CHECK-NEXT:    vzeroupper
diff --git a/llvm/test/CodeGen/X86/mmx-arith.ll b/llvm/test/CodeGen/X86/mmx-arith.ll
index 7b6186475a222..c5c78938c4442 100644
--- a/llvm/test/CodeGen/X86/mmx-arith.ll
+++ b/llvm/test/CodeGen/X86/mmx-arith.ll
@@ -151,7 +151,7 @@ define void @test1(x86_mmx* %A, x86_mmx* %B) {
 ; X32-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
 ; X32-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
 ; X32-NEXT:    pmuludq %xmm0, %xmm1
-; X32-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; X32-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; X32-NEXT:    pmuludq %xmm0, %xmm2
 ; X32-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
 ; X32-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
diff --git a/llvm/test/CodeGen/X86/nontemporal-2.ll b/llvm/test/CodeGen/X86/nontemporal-2.ll
index 4e83a7ce7231b..df832ee16e633 100644
--- a/llvm/test/CodeGen/X86/nontemporal-2.ll
+++ b/llvm/test/CodeGen/X86/nontemporal-2.ll
@@ -492,7 +492,7 @@ define void @test_arg_i64(i64 %arg, i64* %dst) {
 define void @test_extract_f32(<4 x float> %arg, float* %dst) {
 ; SSE2-LABEL: test_extract_f32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    movss %xmm0, (%rdi)
 ; SSE2-NEXT:    retq
 ;
@@ -527,14 +527,14 @@ define void @test_extract_f32(<4 x float> %arg, float* %dst) {
 define void @test_extract_i32(<4 x i32> %arg, i32* %dst) {
 ; SSE2-LABEL: test_extract_i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    movd %xmm0, %eax
 ; SSE2-NEXT:    movntil %eax, (%rdi)
 ; SSE2-NEXT:    retq
 ;
 ; SSE4A-LABEL: test_extract_i32:
 ; SSE4A:       # %bb.0:
-; SSE4A-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE4A-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE4A-NEXT:    movd %xmm0, %eax
 ; SSE4A-NEXT:    movntil %eax, (%rdi)
 ; SSE4A-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/oddshuffles.ll b/llvm/test/CodeGen/X86/oddshuffles.ll
index abec83fec594d..6b65fee913bfa 100644
--- a/llvm/test/CodeGen/X86/oddshuffles.ll
+++ b/llvm/test/CodeGen/X86/oddshuffles.ll
@@ -68,7 +68,7 @@ define void @v3f64(<2 x double> %a, <2 x double> %b, <3 x double>* %p) nounwind
 define void @v3i32(<2 x i32> %a, <2 x i32> %b, <3 x i32>* %p) nounwind {
 ; SSE2-LABEL: v3i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; SSE2-NEXT:    movd %xmm2, 8(%rdi)
 ; SSE2-NEXT:    movq %xmm0, (%rdi)
@@ -141,7 +141,7 @@ define void @v5i32(<4 x i32> %a, <4 x i32> %b, <5 x i32>* %p) nounwind {
 ; SSE2-LABEL: v5i32:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,2,2,3]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; SSE2-NEXT:    movd %xmm2, 16(%rdi)
 ; SSE2-NEXT:    movdqa %xmm0, (%rdi)
@@ -191,7 +191,7 @@ define void @v5f32(<4 x float> %a, <4 x float> %b, <5 x float>* %p) nounwind {
 ; SSE2-NEXT:    movaps %xmm0, %xmm2
 ; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,1],xmm1[1,2]
 ; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,2,1,3]
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE2-NEXT:    movss %xmm0, 16(%rdi)
 ; SSE2-NEXT:    movaps %xmm2, (%rdi)
 ; SSE2-NEXT:    retq
@@ -323,11 +323,11 @@ define void @v7i16(<4 x i16> %a, <4 x i16> %b, <7 x i16>* %p) nounwind {
 define void @v7i32(<4 x i32> %a, <4 x i32> %b, <7 x i32>* %p) nounwind {
 ; SSE2-LABEL: v7i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[0,1,2,2]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,2,2,2]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[0,1,0,3]
 ; SSE2-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,1,2,3]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
 ; SSE2-NEXT:    movd %xmm1, 24(%rdi)
 ; SSE2-NEXT:    movq %xmm0, 16(%rdi)
@@ -425,7 +425,7 @@ define void @v12i8(<8 x i8> %a, <8 x i8> %b, <12 x i8>* %p) nounwind {
 define void @v12i16(<8 x i16> %a, <8 x i16> %b, <12 x i16>* %p) nounwind {
 ; SSE2-LABEL: v12i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[0,0,0,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[0,0,0,0]
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [65535,65535,0,65535,65535,0,65535,65535]
 ; SSE2-NEXT:    pshufhw {{.*#+}} xmm4 = xmm0[0,1,2,3,6,5,4,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[0,3,2,1]
@@ -434,7 +434,7 @@ define void @v12i16(<8 x i16> %a, <8 x i16> %b, <12 x i16>* %p) nounwind {
 ; SSE2-NEXT:    pand %xmm3, %xmm4
 ; SSE2-NEXT:    pandn %xmm2, %xmm3
 ; SSE2-NEXT:    por %xmm4, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [0,65535,65535,0,65535,65535,65535,65535]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,3,1,3,4,5,6,7]
@@ -447,11 +447,11 @@ define void @v12i16(<8 x i16> %a, <8 x i16> %b, <12 x i16>* %p) nounwind {
 ;
 ; SSE42-LABEL: v12i16:
 ; SSE42:       # %bb.0:
-; SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,2,3]
+; SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
 ; SSE42-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[3,1,2,3]
 ; SSE42-NEXT:    pshuflw {{.*#+}} xmm3 = xmm3[0,3,1,3,4,5,6,7]
 ; SSE42-NEXT:    pblendw {{.*#+}} xmm3 = xmm2[0],xmm3[1,2],xmm2[3],xmm3[4,5,6,7]
-; SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,3]
+; SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; SSE42-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,1,8,9,u,u,2,3,10,11,u,u,4,5,12,13]
 ; SSE42-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2],xmm0[3,4],xmm1[5],xmm0[6,7]
 ; SSE42-NEXT:    movdqa %xmm0, (%rdi)
@@ -460,11 +460,11 @@ define void @v12i16(<8 x i16> %a, <8 x i16> %b, <12 x i16>* %p) nounwind {
 ;
 ; AVX1-LABEL: v12i16:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[3,1,2,3]
 ; AVX1-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm3[0,3,1,3,4,5,6,7]
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0],xmm3[1,2],xmm2[3],xmm3[4,5,6,7]
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,8,9,u,u,2,3,10,11,u,u,4,5,12,13]
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2],xmm0[3,4],xmm1[5],xmm0[6,7]
 ; AVX1-NEXT:    vmovdqa %xmm0, (%rdi)
@@ -473,7 +473,7 @@ define void @v12i16(<8 x i16> %a, <8 x i16> %b, <12 x i16>* %p) nounwind {
 ;
 ; AVX2-SLOW-LABEL: v12i16:
 ; AVX2-SLOW:       # %bb.0:
-; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,2,3]
+; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
 ; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[3,1,2,3]
 ; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm3[0,3,1,3,4,5,6,7]
 ; AVX2-SLOW-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0],xmm3[1,2],xmm2[3],xmm3[4,5,6,7]
@@ -489,7 +489,7 @@ define void @v12i16(<8 x i16> %a, <8 x i16> %b, <12 x i16>* %p) nounwind {
 ; AVX2-FAST-NEXT:    vpbroadcastd %xmm1, %xmm2
 ; AVX2-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm0[0,1,8,9,u,u,2,3,10,11,u,u,4,5,12,13]
 ; AVX2-FAST-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2],xmm3[3,4],xmm2[5],xmm3[6,7]
-; AVX2-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; AVX2-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
 ; AVX2-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,6,7,14,15,u,u,u,u,u,u,u,u,u,u]
 ; AVX2-FAST-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1,2],xmm1[3],xmm0[4,5,6,7]
 ; AVX2-FAST-NEXT:    vmovq %xmm0, 16(%rdi)
@@ -559,7 +559,7 @@ define void @v12i32(<8 x i32> %a, <8 x i32> %b, <12 x i32>* %p) nounwind {
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm3, %ymm3
 ; AVX1-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1],ymm3[2],ymm2[3,4],ymm3[5],ymm2[6,7]
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
-; AVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,3],xmm3[3,3]
+; AVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[3,3],xmm3[3,3]
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; AVX1-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2],xmm1[3]
 ; AVX1-NEXT:    vmovaps %xmm0, 32(%rdi)
@@ -574,7 +574,7 @@ define void @v12i32(<8 x i32> %a, <8 x i32> %b, <12 x i32>* %p) nounwind {
 ; AVX2-SLOW-NEXT:    vbroadcastsd %xmm1, %ymm3
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1],ymm3[2],ymm2[3,4],ymm3[5],ymm2[6,7]
 ; AVX2-SLOW-NEXT:    vextractf128 $1, %ymm0, %xmm3
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,3],xmm3[3,3]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[3,3],xmm3[3,3]
 ; AVX2-SLOW-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2],xmm1[3]
 ; AVX2-SLOW-NEXT:    vmovaps %xmm0, 32(%rdi)
@@ -605,7 +605,7 @@ define void @v12i32(<8 x i32> %a, <8 x i32> %b, <12 x i32>* %p) nounwind {
 ; XOP-NEXT:    vinsertf128 $1, %xmm1, %ymm3, %ymm3
 ; XOP-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1],ymm3[2],ymm2[3,4],ymm3[5],ymm2[6,7]
 ; XOP-NEXT:    vextractf128 $1, %ymm0, %xmm3
-; XOP-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,3],xmm3[3,3]
+; XOP-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[3,3],xmm3[3,3]
 ; XOP-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; XOP-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2],xmm1[3]
 ; XOP-NEXT:    vmovaps %xmm0, 32(%rdi)
@@ -806,8 +806,8 @@ define void @interleave_24i8_in(<24 x i8>* %p, <8 x i8>* %q1, <8 x i8>* %q2, <8
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [65535,65535,0,65535,65535,0,65535,65535]
 ; SSE2-NEXT:    pand %xmm5, %xmm4
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3],xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm2[0,1,3,3,4,5,6,7]
-; SSE2-NEXT:    pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,4,6,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm2[3,3,3,3,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,4,4,4]
 ; SSE2-NEXT:    pandn %xmm3, %xmm5
 ; SSE2-NEXT:    por %xmm4, %xmm5
 ; SSE2-NEXT:    movdqa %xmm1, %xmm3
@@ -819,7 +819,7 @@ define void @interleave_24i8_in(<24 x i8>* %p, <8 x i8>* %q1, <8 x i8>* %q2, <8
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [255,255,0,255,255,0,255,255,0,255,255,0,255,255,0,255]
 ; SSE2-NEXT:    pand %xmm4, %xmm3
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[0,1,0,1]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm5 = xmm5[0,0,0,3,4,5,6,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm5 = xmm5[0,0,0,0,4,5,6,7]
 ; SSE2-NEXT:    pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,5,5,6,6]
 ; SSE2-NEXT:    pandn %xmm5, %xmm4
 ; SSE2-NEXT:    por %xmm3, %xmm4
@@ -923,7 +923,7 @@ define void @interleave_24i16_out(<24 x i16>* %p, <8 x i16>* %q1, <8 x i16>* %q2
 ; SSE2-NEXT:    pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,5,4,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[0,3,2,3]
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm5 = xmm5[1,2,3,0,4,5,6,7]
-; SSE2-NEXT:    pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,5,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,5,5,5,5]
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [65535,65535,65535,65535,65535,0,0,0]
 ; SSE2-NEXT:    pand %xmm6, %xmm5
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm7 = xmm8[0,3,2,3,4,5,6,7]
@@ -1049,11 +1049,11 @@ define void @interleave_24i16_in(<24 x i16>* %p, <8 x i16>* %q1, <8 x i16>* %q2,
 ; SSE2-NEXT:    movdqu (%rsi), %xmm3
 ; SSE2-NEXT:    movdqu (%rdx), %xmm2
 ; SSE2-NEXT:    movdqu (%rcx), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[0,0,0,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[0,0,0,0]
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [65535,65535,0,65535,65535,0,65535,65535]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm5
 ; SSE2-NEXT:    pandn %xmm4, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,3,3,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[3,3,3,3]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[1,1,2,2]
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,1,2,1]
@@ -1063,7 +1063,7 @@ define void @interleave_24i16_in(<24 x i16>* %p, <8 x i16>* %q1, <8 x i16>* %q2,
 ; SSE2-NEXT:    por %xmm5, %xmm3
 ; SSE2-NEXT:    movdqa %xmm0, %xmm5
 ; SSE2-NEXT:    pandn %xmm4, %xmm5
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm4 = xmm2[0,1,3,3,4,5,6,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm4 = xmm2[3,3,3,3,4,5,6,7]
 ; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,1,2,3]
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[2,3,2,0,4,5,6,7]
@@ -1076,7 +1076,7 @@ define void @interleave_24i16_in(<24 x i16>* %p, <8 x i16>* %q1, <8 x i16>* %q2,
 ; SSE2-NEXT:    pandn %xmm6, %xmm5
 ; SSE2-NEXT:    por %xmm1, %xmm5
 ; SSE2-NEXT:    pand %xmm0, %xmm5
-; SSE2-NEXT:    pshufhw {{.*#+}} xmm1 = xmm4[0,1,2,3,4,4,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm1 = xmm4[0,1,2,3,4,4,4,4]
 ; SSE2-NEXT:    pandn %xmm1, %xmm0
 ; SSE2-NEXT:    por %xmm5, %xmm0
 ; SSE2-NEXT:    movdqu %xmm0, 16(%rdi)
@@ -1090,15 +1090,15 @@ define void @interleave_24i16_in(<24 x i16>* %p, <8 x i16>* %q1, <8 x i16>* %q2,
 ; SSE42-NEXT:    movdqu (%rdx), %xmm1
 ; SSE42-NEXT:    movdqu (%rcx), %xmm2
 ; SSE42-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,2,2]
-; SSE42-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[0,3,3,3]
+; SSE42-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[3,3,3,3]
 ; SSE42-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; SSE42-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,1,2,3,u,u,4,5,6,7,u,u,8,9,10,11]
-; SSE42-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[0,0,0,3]
+; SSE42-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[0,0,0,0]
 ; SSE42-NEXT:    pblendw {{.*#+}} xmm5 = xmm0[0,1],xmm5[2],xmm0[3,4],xmm5[5],xmm0[6,7]
 ; SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,2]
 ; SSE42-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0],xmm3[1],xmm0[2,3],xmm3[4],xmm0[5,6],xmm3[7]
-; SSE42-NEXT:    pshuflw {{.*#+}} xmm3 = xmm1[0,1,3,3,4,5,6,7]
-; SSE42-NEXT:    pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,4,6,7]
+; SSE42-NEXT:    pshuflw {{.*#+}} xmm3 = xmm1[3,3,3,3,4,5,6,7]
+; SSE42-NEXT:    pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,4,4,4]
 ; SSE42-NEXT:    pblendw {{.*#+}} xmm3 = xmm0[0,1],xmm3[2],xmm0[3,4],xmm3[5],xmm0[6,7]
 ; SSE42-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
 ; SSE42-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[4,5,6,7,u,u,8,9,10,11,u,u,12,13,14,15]
@@ -1113,8 +1113,8 @@ define void @interleave_24i16_in(<24 x i16>* %p, <8 x i16>* %q1, <8 x i16>* %q2,
 ; AVX1-NEXT:    vmovdqu (%rsi), %xmm0
 ; AVX1-NEXT:    vmovdqu (%rdx), %xmm1
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,2,2]
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm1[0,1,3,3,4,5,6,7]
-; AVX1-NEXT:    vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,4,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm1[3,3,3,3,4,5,6,7]
+; AVX1-NEXT:    vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,4,4,4]
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2],xmm2[3,4],xmm3[5],xmm2[6,7]
 ; AVX1-NEXT:    vmovdqu (%rcx), %xmm3
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm3[1,1,2,2]
@@ -1125,7 +1125,7 @@ define void @interleave_24i16_in(<24 x i16>* %p, <8 x i16>* %q1, <8 x i16>* %q2,
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0],xmm5[1],xmm4[2,3],xmm5[4],xmm4[5,6],xmm5[7]
 ; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,2,3,u,u,4,5,6,7,u,u,8,9,10,11]
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm3[0,0,0,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm3[0,0,0,0]
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2],xmm0[3,4],xmm1[5],xmm0[6,7]
 ; AVX1-NEXT:    vmovdqu %xmm0, (%rdi)
 ; AVX1-NEXT:    vmovdqu %xmm4, 32(%rdi)
@@ -1196,24 +1196,24 @@ define void @interleave_24i32_out(<24 x i32>* %p, <8 x i32>* %q1, <8 x i32>* %q2
 ; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,0],xmm6[0,0]
 ; SSE2-NEXT:    movaps %xmm0, %xmm1
 ; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,2],xmm6[3,3]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm6[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm6[1,1,1,1]
 ; SSE2-NEXT:    shufps {{.*#+}} xmm6 = xmm6[2,0],xmm10[1,0]
 ; SSE2-NEXT:    shufps {{.*#+}} xmm11 = xmm11[0,3],xmm6[0,2]
 ; SSE2-NEXT:    movaps %xmm12, %xmm6
 ; SSE2-NEXT:    shufps {{.*#+}} xmm6 = xmm6[1,0],xmm4[0,0]
 ; SSE2-NEXT:    movaps %xmm6, %xmm2
 ; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,2],xmm4[3,3]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,1,1]
 ; SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[2,0],xmm8[1,0]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm12[2,3,2,3]
 ; SSE2-NEXT:    shufps {{.*#+}} xmm12 = xmm12[0,3],xmm4[0,2]
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]
 ; SSE2-NEXT:    shufps {{.*#+}} xmm5 = xmm5[0,1],xmm8[0,3]
-; SSE2-NEXT:    shufps {{.*#+}} xmm8 = xmm8[2,0],xmm2[2,0]
+; SSE2-NEXT:    shufps {{.*#+}} xmm8 = xmm8[2,2],xmm2[2,0]
 ; SSE2-NEXT:    shufps {{.*#+}} xmm6 = xmm6[0,2],xmm8[2,0]
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm7 = xmm7[0],xmm9[0],xmm7[1],xmm9[1]
 ; SSE2-NEXT:    shufps {{.*#+}} xmm7 = xmm7[0,1],xmm10[0,3]
-; SSE2-NEXT:    shufps {{.*#+}} xmm10 = xmm10[2,0],xmm1[2,0]
+; SSE2-NEXT:    shufps {{.*#+}} xmm10 = xmm10[2,2],xmm1[2,0]
 ; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm10[2,0]
 ; SSE2-NEXT:    movups %xmm12, 16(%rsi)
 ; SSE2-NEXT:    movups %xmm11, (%rsi)
@@ -1241,11 +1241,11 @@ define void @interleave_24i32_out(<24 x i32>* %p, <8 x i32>* %q1, <8 x i32>* %q2
 ; SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[2,3,2,3]
 ; SSE42-NEXT:    shufps {{.*#+}} xmm5 = xmm5[0,3],xmm9[2,3]
 ; SSE42-NEXT:    insertps {{.*#+}} xmm5 = xmm5[0,1,2],xmm8[1]
-; SSE42-NEXT:    pshufd {{.*#+}} xmm3 = xmm10[0,1,2,2]
+; SSE42-NEXT:    pshufd {{.*#+}} xmm3 = xmm10[2,2,2,2]
 ; SSE42-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,0,3,3]
 ; SSE42-NEXT:    pblendw {{.*#+}} xmm6 = xmm6[0,1,2,3,4,5],xmm3[6,7]
 ; SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,0,3,3]
-; SSE42-NEXT:    pshufd {{.*#+}} xmm3 = xmm8[0,1,2,2]
+; SSE42-NEXT:    pshufd {{.*#+}} xmm3 = xmm8[2,2,2,2]
 ; SSE42-NEXT:    pblendw {{.*#+}} xmm3 = xmm1[0,1,2,3,4,5],xmm3[6,7]
 ; SSE42-NEXT:    pblendw {{.*#+}} xmm7 = xmm7[0,1],xmm2[2,3],xmm7[4,5,6,7]
 ; SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm10[0,1,0,3]
diff --git a/llvm/test/CodeGen/X86/phaddsub-extract.ll b/llvm/test/CodeGen/X86/phaddsub-extract.ll
index 7f3eb0898cc86..a80645f44d9d0 100644
--- a/llvm/test/CodeGen/X86/phaddsub-extract.ll
+++ b/llvm/test/CodeGen/X86/phaddsub-extract.ll
@@ -14,7 +14,7 @@ define i32 @extract_extract01_v4i32_add_i32(<4 x i32> %x) {
 ; SSE3-SLOW-LABEL: extract_extract01_v4i32_add_i32:
 ; SSE3-SLOW:       # %bb.0:
 ; SSE3-SLOW-NEXT:    movd %xmm0, %ecx
-; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE3-SLOW-NEXT:    movd %xmm0, %eax
 ; SSE3-SLOW-NEXT:    addl %ecx, %eax
 ; SSE3-SLOW-NEXT:    retq
@@ -48,7 +48,7 @@ define i32 @extract_extract23_v4i32_add_i32(<4 x i32> %x) {
 ; SSE3-SLOW:       # %bb.0:
 ; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE3-SLOW-NEXT:    movd %xmm1, %ecx
-; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE3-SLOW-NEXT:    movd %xmm0, %eax
 ; SSE3-SLOW-NEXT:    addl %ecx, %eax
 ; SSE3-SLOW-NEXT:    retq
@@ -56,7 +56,7 @@ define i32 @extract_extract23_v4i32_add_i32(<4 x i32> %x) {
 ; SSE3-FAST-LABEL: extract_extract23_v4i32_add_i32:
 ; SSE3-FAST:       # %bb.0:
 ; SSE3-FAST-NEXT:    phaddd %xmm0, %xmm0
-; SSE3-FAST-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE3-FAST-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE3-FAST-NEXT:    movd %xmm0, %eax
 ; SSE3-FAST-NEXT:    retq
 ;
@@ -82,7 +82,7 @@ define i32 @extract_extract01_v4i32_add_i32_commute(<4 x i32> %x) {
 ; SSE3-SLOW-LABEL: extract_extract01_v4i32_add_i32_commute:
 ; SSE3-SLOW:       # %bb.0:
 ; SSE3-SLOW-NEXT:    movd %xmm0, %ecx
-; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE3-SLOW-NEXT:    movd %xmm0, %eax
 ; SSE3-SLOW-NEXT:    addl %ecx, %eax
 ; SSE3-SLOW-NEXT:    retq
@@ -116,7 +116,7 @@ define i32 @extract_extract23_v4i32_add_i32_commute(<4 x i32> %x) {
 ; SSE3-SLOW:       # %bb.0:
 ; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE3-SLOW-NEXT:    movd %xmm1, %ecx
-; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE3-SLOW-NEXT:    movd %xmm0, %eax
 ; SSE3-SLOW-NEXT:    addl %ecx, %eax
 ; SSE3-SLOW-NEXT:    retq
@@ -124,7 +124,7 @@ define i32 @extract_extract23_v4i32_add_i32_commute(<4 x i32> %x) {
 ; SSE3-FAST-LABEL: extract_extract23_v4i32_add_i32_commute:
 ; SSE3-FAST:       # %bb.0:
 ; SSE3-FAST-NEXT:    phaddd %xmm0, %xmm0
-; SSE3-FAST-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE3-FAST-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE3-FAST-NEXT:    movd %xmm0, %eax
 ; SSE3-FAST-NEXT:    retq
 ;
@@ -294,7 +294,7 @@ define i32 @extract_extract01_v4i32_sub_i32(<4 x i32> %x) {
 ; SSE3-SLOW-LABEL: extract_extract01_v4i32_sub_i32:
 ; SSE3-SLOW:       # %bb.0:
 ; SSE3-SLOW-NEXT:    movd %xmm0, %eax
-; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE3-SLOW-NEXT:    movd %xmm0, %ecx
 ; SSE3-SLOW-NEXT:    subl %ecx, %eax
 ; SSE3-SLOW-NEXT:    retq
@@ -328,7 +328,7 @@ define i32 @extract_extract23_v4i32_sub_i32(<4 x i32> %x) {
 ; SSE3-SLOW:       # %bb.0:
 ; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE3-SLOW-NEXT:    movd %xmm1, %eax
-; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE3-SLOW-NEXT:    movd %xmm0, %ecx
 ; SSE3-SLOW-NEXT:    subl %ecx, %eax
 ; SSE3-SLOW-NEXT:    retq
@@ -336,7 +336,7 @@ define i32 @extract_extract23_v4i32_sub_i32(<4 x i32> %x) {
 ; SSE3-FAST-LABEL: extract_extract23_v4i32_sub_i32:
 ; SSE3-FAST:       # %bb.0:
 ; SSE3-FAST-NEXT:    phsubd %xmm0, %xmm0
-; SSE3-FAST-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE3-FAST-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE3-FAST-NEXT:    movd %xmm0, %eax
 ; SSE3-FAST-NEXT:    retq
 ;
@@ -362,7 +362,7 @@ define i32 @extract_extract01_v4i32_sub_i32_commute(<4 x i32> %x) {
 ; SSE3-LABEL: extract_extract01_v4i32_sub_i32_commute:
 ; SSE3:       # %bb.0:
 ; SSE3-NEXT:    movd %xmm0, %ecx
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE3-NEXT:    movd %xmm0, %eax
 ; SSE3-NEXT:    subl %ecx, %eax
 ; SSE3-NEXT:    retq
@@ -384,7 +384,7 @@ define i32 @extract_extract23_v4i32_sub_i32_commute(<4 x i32> %x) {
 ; SSE3:       # %bb.0:
 ; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE3-NEXT:    movd %xmm1, %ecx
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE3-NEXT:    movd %xmm0, %eax
 ; SSE3-NEXT:    subl %ecx, %eax
 ; SSE3-NEXT:    retq
@@ -523,7 +523,7 @@ define i32 @extract_extract01_v8i32_add_i32(<8 x i32> %x) {
 ; SSE3-SLOW-LABEL: extract_extract01_v8i32_add_i32:
 ; SSE3-SLOW:       # %bb.0:
 ; SSE3-SLOW-NEXT:    movd %xmm0, %ecx
-; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE3-SLOW-NEXT:    movd %xmm0, %eax
 ; SSE3-SLOW-NEXT:    addl %ecx, %eax
 ; SSE3-SLOW-NEXT:    retq
@@ -559,7 +559,7 @@ define i32 @extract_extract23_v8i32_add_i32(<8 x i32> %x) {
 ; SSE3-SLOW:       # %bb.0:
 ; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE3-SLOW-NEXT:    movd %xmm1, %ecx
-; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE3-SLOW-NEXT:    movd %xmm0, %eax
 ; SSE3-SLOW-NEXT:    addl %ecx, %eax
 ; SSE3-SLOW-NEXT:    retq
@@ -567,7 +567,7 @@ define i32 @extract_extract23_v8i32_add_i32(<8 x i32> %x) {
 ; SSE3-FAST-LABEL: extract_extract23_v8i32_add_i32:
 ; SSE3-FAST:       # %bb.0:
 ; SSE3-FAST-NEXT:    phaddd %xmm0, %xmm0
-; SSE3-FAST-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE3-FAST-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE3-FAST-NEXT:    movd %xmm0, %eax
 ; SSE3-FAST-NEXT:    retq
 ;
@@ -596,7 +596,7 @@ define i32 @extract_extract67_v8i32_add_i32(<8 x i32> %x) {
 ; SSE3-SLOW:       # %bb.0:
 ; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE3-SLOW-NEXT:    movd %xmm0, %ecx
-; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[3,1,2,3]
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[3,3,3,3]
 ; SSE3-SLOW-NEXT:    movd %xmm0, %eax
 ; SSE3-SLOW-NEXT:    addl %ecx, %eax
 ; SSE3-SLOW-NEXT:    retq
@@ -604,7 +604,7 @@ define i32 @extract_extract67_v8i32_add_i32(<8 x i32> %x) {
 ; SSE3-FAST-LABEL: extract_extract67_v8i32_add_i32:
 ; SSE3-FAST:       # %bb.0:
 ; SSE3-FAST-NEXT:    phaddd %xmm1, %xmm1
-; SSE3-FAST-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE3-FAST-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE3-FAST-NEXT:    movd %xmm0, %eax
 ; SSE3-FAST-NEXT:    retq
 ;
@@ -650,7 +650,7 @@ define i32 @extract_extract01_v8i32_add_i32_commute(<8 x i32> %x) {
 ; SSE3-SLOW-LABEL: extract_extract01_v8i32_add_i32_commute:
 ; SSE3-SLOW:       # %bb.0:
 ; SSE3-SLOW-NEXT:    movd %xmm0, %ecx
-; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE3-SLOW-NEXT:    movd %xmm0, %eax
 ; SSE3-SLOW-NEXT:    addl %ecx, %eax
 ; SSE3-SLOW-NEXT:    retq
@@ -686,7 +686,7 @@ define i32 @extract_extract23_v8i32_add_i32_commute(<8 x i32> %x) {
 ; SSE3-SLOW:       # %bb.0:
 ; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE3-SLOW-NEXT:    movd %xmm1, %ecx
-; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE3-SLOW-NEXT:    movd %xmm0, %eax
 ; SSE3-SLOW-NEXT:    addl %ecx, %eax
 ; SSE3-SLOW-NEXT:    retq
@@ -694,7 +694,7 @@ define i32 @extract_extract23_v8i32_add_i32_commute(<8 x i32> %x) {
 ; SSE3-FAST-LABEL: extract_extract23_v8i32_add_i32_commute:
 ; SSE3-FAST:       # %bb.0:
 ; SSE3-FAST-NEXT:    phaddd %xmm0, %xmm0
-; SSE3-FAST-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE3-FAST-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE3-FAST-NEXT:    movd %xmm0, %eax
 ; SSE3-FAST-NEXT:    retq
 ;
@@ -723,7 +723,7 @@ define i32 @extract_extract67_v8i32_add_i32_commute(<8 x i32> %x) {
 ; SSE3-SLOW:       # %bb.0:
 ; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE3-SLOW-NEXT:    movd %xmm0, %ecx
-; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[3,1,2,3]
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[3,3,3,3]
 ; SSE3-SLOW-NEXT:    movd %xmm0, %eax
 ; SSE3-SLOW-NEXT:    addl %ecx, %eax
 ; SSE3-SLOW-NEXT:    retq
@@ -731,7 +731,7 @@ define i32 @extract_extract67_v8i32_add_i32_commute(<8 x i32> %x) {
 ; SSE3-FAST-LABEL: extract_extract67_v8i32_add_i32_commute:
 ; SSE3-FAST:       # %bb.0:
 ; SSE3-FAST-NEXT:    phaddd %xmm1, %xmm1
-; SSE3-FAST-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE3-FAST-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE3-FAST-NEXT:    movd %xmm0, %eax
 ; SSE3-FAST-NEXT:    retq
 ;
@@ -1085,7 +1085,7 @@ define i32 @extract_extract01_v8i32_sub_i32(<8 x i32> %x) {
 ; SSE3-SLOW-LABEL: extract_extract01_v8i32_sub_i32:
 ; SSE3-SLOW:       # %bb.0:
 ; SSE3-SLOW-NEXT:    movd %xmm0, %eax
-; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE3-SLOW-NEXT:    movd %xmm0, %ecx
 ; SSE3-SLOW-NEXT:    subl %ecx, %eax
 ; SSE3-SLOW-NEXT:    retq
@@ -1121,7 +1121,7 @@ define i32 @extract_extract23_v8i32_sub_i32(<8 x i32> %x) {
 ; SSE3-SLOW:       # %bb.0:
 ; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE3-SLOW-NEXT:    movd %xmm1, %eax
-; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE3-SLOW-NEXT:    movd %xmm0, %ecx
 ; SSE3-SLOW-NEXT:    subl %ecx, %eax
 ; SSE3-SLOW-NEXT:    retq
@@ -1129,7 +1129,7 @@ define i32 @extract_extract23_v8i32_sub_i32(<8 x i32> %x) {
 ; SSE3-FAST-LABEL: extract_extract23_v8i32_sub_i32:
 ; SSE3-FAST:       # %bb.0:
 ; SSE3-FAST-NEXT:    phsubd %xmm0, %xmm0
-; SSE3-FAST-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE3-FAST-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE3-FAST-NEXT:    movd %xmm0, %eax
 ; SSE3-FAST-NEXT:    retq
 ;
@@ -1158,7 +1158,7 @@ define i32 @extract_extract67_v8i32_sub_i32(<8 x i32> %x) {
 ; SSE3-SLOW:       # %bb.0:
 ; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE3-SLOW-NEXT:    movd %xmm0, %eax
-; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[3,1,2,3]
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[3,3,3,3]
 ; SSE3-SLOW-NEXT:    movd %xmm0, %ecx
 ; SSE3-SLOW-NEXT:    subl %ecx, %eax
 ; SSE3-SLOW-NEXT:    retq
@@ -1166,7 +1166,7 @@ define i32 @extract_extract67_v8i32_sub_i32(<8 x i32> %x) {
 ; SSE3-FAST-LABEL: extract_extract67_v8i32_sub_i32:
 ; SSE3-FAST:       # %bb.0:
 ; SSE3-FAST-NEXT:    phsubd %xmm1, %xmm1
-; SSE3-FAST-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE3-FAST-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE3-FAST-NEXT:    movd %xmm0, %eax
 ; SSE3-FAST-NEXT:    retq
 ;
@@ -1214,7 +1214,7 @@ define i32 @extract_extract01_v8i32_sub_i32_commute(<8 x i32> %x) {
 ; SSE3-LABEL: extract_extract01_v8i32_sub_i32_commute:
 ; SSE3:       # %bb.0:
 ; SSE3-NEXT:    movd %xmm0, %ecx
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE3-NEXT:    movd %xmm0, %eax
 ; SSE3-NEXT:    subl %ecx, %eax
 ; SSE3-NEXT:    retq
@@ -1301,7 +1301,7 @@ define i32 @extract_extract01_v16i32_add_i32(<16 x i32> %x) {
 ; SSE3-SLOW-LABEL: extract_extract01_v16i32_add_i32:
 ; SSE3-SLOW:       # %bb.0:
 ; SSE3-SLOW-NEXT:    movd %xmm0, %ecx
-; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE3-SLOW-NEXT:    movd %xmm0, %eax
 ; SSE3-SLOW-NEXT:    addl %ecx, %eax
 ; SSE3-SLOW-NEXT:    retq
@@ -1336,7 +1336,7 @@ define i32 @extract_extract01_v16i32_add_i32_commute(<16 x i32> %x) {
 ; SSE3-SLOW-LABEL: extract_extract01_v16i32_add_i32_commute:
 ; SSE3-SLOW:       # %bb.0:
 ; SSE3-SLOW-NEXT:    movd %xmm0, %ecx
-; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE3-SLOW-NEXT:    movd %xmm0, %eax
 ; SSE3-SLOW-NEXT:    addl %ecx, %eax
 ; SSE3-SLOW-NEXT:    retq
@@ -1447,7 +1447,7 @@ define i32 @extract_extract01_v16i32_sub_i32(<16 x i32> %x) {
 ; SSE3-SLOW-LABEL: extract_extract01_v16i32_sub_i32:
 ; SSE3-SLOW:       # %bb.0:
 ; SSE3-SLOW-NEXT:    movd %xmm0, %eax
-; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE3-SLOW-NEXT:    movd %xmm0, %ecx
 ; SSE3-SLOW-NEXT:    subl %ecx, %eax
 ; SSE3-SLOW-NEXT:    retq
@@ -1482,7 +1482,7 @@ define i32 @extract_extract01_v16i32_sub_i32_commute(<16 x i32> %x) {
 ; SSE3-LABEL: extract_extract01_v16i32_sub_i32_commute:
 ; SSE3:       # %bb.0:
 ; SSE3-NEXT:    movd %xmm0, %ecx
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE3-NEXT:    movd %xmm0, %eax
 ; SSE3-NEXT:    subl %ecx, %eax
 ; SSE3-NEXT:    retq
@@ -1568,7 +1568,7 @@ define i32 @extract_extract01_v4i32_add_i32_uses1(<4 x i32> %x, i32* %p) {
 ; SSE3-SLOW:       # %bb.0:
 ; SSE3-SLOW-NEXT:    movd %xmm0, %ecx
 ; SSE3-SLOW-NEXT:    movd %xmm0, (%rdi)
-; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE3-SLOW-NEXT:    movd %xmm0, %eax
 ; SSE3-SLOW-NEXT:    addl %ecx, %eax
 ; SSE3-SLOW-NEXT:    retq
@@ -1605,7 +1605,7 @@ define i32 @extract_extract01_v4i32_add_i32_uses2(<4 x i32> %x, i32* %p) {
 ; SSE3-SLOW-LABEL: extract_extract01_v4i32_add_i32_uses2:
 ; SSE3-SLOW:       # %bb.0:
 ; SSE3-SLOW-NEXT:    movd %xmm0, %ecx
-; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE3-SLOW-NEXT:    movd %xmm0, %eax
 ; SSE3-SLOW-NEXT:    addl %ecx, %eax
 ; SSE3-SLOW-NEXT:    movd %xmm0, (%rdi)
@@ -1613,7 +1613,7 @@ define i32 @extract_extract01_v4i32_add_i32_uses2(<4 x i32> %x, i32* %p) {
 ;
 ; SSE3-FAST-LABEL: extract_extract01_v4i32_add_i32_uses2:
 ; SSE3-FAST:       # %bb.0:
-; SSE3-FAST-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE3-FAST-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE3-FAST-NEXT:    movd %xmm1, (%rdi)
 ; SSE3-FAST-NEXT:    phaddd %xmm0, %xmm0
 ; SSE3-FAST-NEXT:    movd %xmm0, %eax
@@ -1645,7 +1645,7 @@ define i32 @extract_extract01_v4i32_add_i32_uses3(<4 x i32> %x, i32* %p1, i32* %
 ; SSE3:       # %bb.0:
 ; SSE3-NEXT:    movd %xmm0, %ecx
 ; SSE3-NEXT:    movd %xmm0, (%rdi)
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE3-NEXT:    movd %xmm0, %eax
 ; SSE3-NEXT:    addl %ecx, %eax
 ; SSE3-NEXT:    movd %xmm0, (%rsi)
@@ -1674,7 +1674,7 @@ define i32 @partial_reduction_add_v8i32(<8 x i32> %x) {
 ; SSE3-SLOW:       # %bb.0:
 ; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE3-SLOW-NEXT:    paddd %xmm0, %xmm1
-; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE3-SLOW-NEXT:    paddd %xmm1, %xmm0
 ; SSE3-SLOW-NEXT:    movd %xmm0, %eax
 ; SSE3-SLOW-NEXT:    retq
@@ -1691,7 +1691,7 @@ define i32 @partial_reduction_add_v8i32(<8 x i32> %x) {
 ; AVX-SLOW:       # %bb.0:
 ; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX-SLOW-NEXT:    vmovd %xmm0, %eax
 ; AVX-SLOW-NEXT:    vzeroupper
@@ -1717,7 +1717,7 @@ define i32 @partial_reduction_add_v16i32(<16 x i32> %x) {
 ; SSE3-SLOW:       # %bb.0:
 ; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE3-SLOW-NEXT:    paddd %xmm0, %xmm1
-; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE3-SLOW-NEXT:    paddd %xmm1, %xmm0
 ; SSE3-SLOW-NEXT:    movd %xmm0, %eax
 ; SSE3-SLOW-NEXT:    retq
@@ -1734,7 +1734,7 @@ define i32 @partial_reduction_add_v16i32(<16 x i32> %x) {
 ; AVX-SLOW:       # %bb.0:
 ; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX-SLOW-NEXT:    vmovd %xmm0, %eax
 ; AVX-SLOW-NEXT:    vzeroupper
@@ -1760,7 +1760,7 @@ define i32 @partial_reduction_sub_v8i32(<8 x i32> %x) {
 ; SSE3-SLOW:       # %bb.0:
 ; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE3-SLOW-NEXT:    psubd %xmm1, %xmm0
-; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE3-SLOW-NEXT:    psubd %xmm1, %xmm0
 ; SSE3-SLOW-NEXT:    movd %xmm0, %eax
 ; SSE3-SLOW-NEXT:    retq
@@ -1777,7 +1777,7 @@ define i32 @partial_reduction_sub_v8i32(<8 x i32> %x) {
 ; AVX-SLOW:       # %bb.0:
 ; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-SLOW-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
-; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-SLOW-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
 ; AVX-SLOW-NEXT:    vmovd %xmm0, %eax
 ; AVX-SLOW-NEXT:    vzeroupper
@@ -1804,7 +1804,7 @@ define i32 @partial_reduction_sub_v16i32(<16 x i32> %x) {
 ; SSE3-SLOW:       # %bb.0:
 ; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE3-SLOW-NEXT:    psubd %xmm1, %xmm0
-; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE3-SLOW-NEXT:    psubd %xmm1, %xmm0
 ; SSE3-SLOW-NEXT:    movd %xmm0, %eax
 ; SSE3-SLOW-NEXT:    retq
@@ -1821,7 +1821,7 @@ define i32 @partial_reduction_sub_v16i32(<16 x i32> %x) {
 ; AVX-SLOW:       # %bb.0:
 ; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-SLOW-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
-; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-SLOW-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
 ; AVX-SLOW-NEXT:    vmovd %xmm0, %eax
 ; AVX-SLOW-NEXT:    vzeroupper
@@ -1840,7 +1840,7 @@ define i32 @partial_reduction_sub_v16i32(<16 x i32> %x) {
 ; AVX2-FAST:       # %bb.0:
 ; AVX2-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-FAST-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
-; AVX2-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-FAST-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
 ; AVX2-FAST-NEXT:    vmovd %xmm0, %eax
 ; AVX2-FAST-NEXT:    vzeroupper
@@ -1850,7 +1850,7 @@ define i32 @partial_reduction_sub_v16i32(<16 x i32> %x) {
 ; AVX512-FAST:       # %bb.0:
 ; AVX512-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-FAST-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
-; AVX512-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-FAST-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
 ; AVX512-FAST-NEXT:    vmovd %xmm0, %eax
 ; AVX512-FAST-NEXT:    vzeroupper
@@ -1870,7 +1870,7 @@ define i16 @hadd16_8(<8 x i16> %x223) {
 ; SSE3-SLOW:       # %bb.0:
 ; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE3-SLOW-NEXT:    paddw %xmm0, %xmm1
-; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE3-SLOW-NEXT:    paddw %xmm1, %xmm0
 ; SSE3-SLOW-NEXT:    movdqa %xmm0, %xmm1
 ; SSE3-SLOW-NEXT:    psrld $16, %xmm1
@@ -1892,7 +1892,7 @@ define i16 @hadd16_8(<8 x i16> %x223) {
 ; AVX-SLOW:       # %bb.0:
 ; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-SLOW-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
-; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-SLOW-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX-SLOW-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX-SLOW-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
@@ -1923,7 +1923,7 @@ define i32 @hadd32_4(<4 x i32> %x225) {
 ; SSE3-SLOW:       # %bb.0:
 ; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE3-SLOW-NEXT:    paddd %xmm0, %xmm1
-; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE3-SLOW-NEXT:    paddd %xmm1, %xmm0
 ; SSE3-SLOW-NEXT:    movd %xmm0, %eax
 ; SSE3-SLOW-NEXT:    retq
@@ -1939,7 +1939,7 @@ define i32 @hadd32_4(<4 x i32> %x225) {
 ; AVX-SLOW:       # %bb.0:
 ; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX-SLOW-NEXT:    vmovd %xmm0, %eax
 ; AVX-SLOW-NEXT:    retq
@@ -1963,7 +1963,7 @@ define i32 @hadd32_8(<8 x i32> %x225) {
 ; SSE3-SLOW:       # %bb.0:
 ; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE3-SLOW-NEXT:    paddd %xmm0, %xmm1
-; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE3-SLOW-NEXT:    paddd %xmm1, %xmm0
 ; SSE3-SLOW-NEXT:    movd %xmm0, %eax
 ; SSE3-SLOW-NEXT:    retq
@@ -1980,7 +1980,7 @@ define i32 @hadd32_8(<8 x i32> %x225) {
 ; AVX-SLOW:       # %bb.0:
 ; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX-SLOW-NEXT:    vmovd %xmm0, %eax
 ; AVX-SLOW-NEXT:    vzeroupper
@@ -2006,7 +2006,7 @@ define i32 @hadd32_16(<16 x i32> %x225) {
 ; SSE3-SLOW:       # %bb.0:
 ; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE3-SLOW-NEXT:    paddd %xmm0, %xmm1
-; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE3-SLOW-NEXT:    paddd %xmm1, %xmm0
 ; SSE3-SLOW-NEXT:    movd %xmm0, %eax
 ; SSE3-SLOW-NEXT:    retq
@@ -2023,7 +2023,7 @@ define i32 @hadd32_16(<16 x i32> %x225) {
 ; AVX-SLOW:       # %bb.0:
 ; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX-SLOW-NEXT:    vmovd %xmm0, %eax
 ; AVX-SLOW-NEXT:    vzeroupper
diff --git a/llvm/test/CodeGen/X86/phaddsub-undef.ll b/llvm/test/CodeGen/X86/phaddsub-undef.ll
index 24d2acabfb660..4068016ef5faf 100644
--- a/llvm/test/CodeGen/X86/phaddsub-undef.ll
+++ b/llvm/test/CodeGen/X86/phaddsub-undef.ll
@@ -36,11 +36,11 @@ define <8 x i32> @test15_undef(<8 x i32> %a, <8 x i32> %b) {
 ; SSE-SLOW-LABEL: test15_undef:
 ; SSE-SLOW:       # %bb.0:
 ; SSE-SLOW-NEXT:    movd %xmm0, %eax
-; SSE-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE-SLOW-NEXT:    movd %xmm0, %ecx
 ; SSE-SLOW-NEXT:    addl %eax, %ecx
 ; SSE-SLOW-NEXT:    movd %xmm3, %eax
-; SSE-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,2,3]
+; SSE-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,1,1]
 ; SSE-SLOW-NEXT:    movd %xmm0, %edx
 ; SSE-SLOW-NEXT:    addl %eax, %edx
 ; SSE-SLOW-NEXT:    movd %ecx, %xmm0
diff --git a/llvm/test/CodeGen/X86/phaddsub.ll b/llvm/test/CodeGen/X86/phaddsub.ll
index 501a26d06b831..ebf0951eb0d06 100644
--- a/llvm/test/CodeGen/X86/phaddsub.ll
+++ b/llvm/test/CodeGen/X86/phaddsub.ll
@@ -122,7 +122,7 @@ define <4 x i32> @phaddd5(<4 x i32> %x) {
 define <4 x i32> @phaddd6(<4 x i32> %x) {
 ; SSSE3-SLOW-LABEL: phaddd6:
 ; SSSE3-SLOW:       # %bb.0:
-; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSSE3-SLOW-NEXT:    paddd %xmm1, %xmm0
 ; SSSE3-SLOW-NEXT:    retq
 ;
@@ -133,7 +133,7 @@ define <4 x i32> @phaddd6(<4 x i32> %x) {
 ;
 ; AVX-SLOW-LABEL: phaddd6:
 ; AVX-SLOW:       # %bb.0:
-; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX-SLOW-NEXT:    retq
 ;
@@ -144,7 +144,7 @@ define <4 x i32> @phaddd6(<4 x i32> %x) {
 ;
 ; AVX2-SHUF-LABEL: phaddd6:
 ; AVX2-SHUF:       # %bb.0:
-; AVX2-SHUF-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-SHUF-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-SHUF-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-SHUF-NEXT:    retq
   %a = shufflevector <4 x i32> %x, <4 x i32> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
@@ -236,7 +236,7 @@ define <4 x i32> @phsubd3(<4 x i32> %x) {
 define <4 x i32> @phsubd4(<4 x i32> %x) {
 ; SSSE3-SLOW-LABEL: phsubd4:
 ; SSSE3-SLOW:       # %bb.0:
-; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSSE3-SLOW-NEXT:    psubd %xmm1, %xmm0
 ; SSSE3-SLOW-NEXT:    retq
 ;
@@ -247,7 +247,7 @@ define <4 x i32> @phsubd4(<4 x i32> %x) {
 ;
 ; AVX-SLOW-LABEL: phsubd4:
 ; AVX-SLOW:       # %bb.0:
-; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-SLOW-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
 ; AVX-SLOW-NEXT:    retq
 ;
@@ -258,7 +258,7 @@ define <4 x i32> @phsubd4(<4 x i32> %x) {
 ;
 ; AVX2-SHUF-LABEL: phsubd4:
 ; AVX2-SHUF:       # %bb.0:
-; AVX2-SHUF-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-SHUF-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-SHUF-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
 ; AVX2-SHUF-NEXT:    retq
   %a = shufflevector <4 x i32> %x, <4 x i32> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
@@ -378,7 +378,7 @@ define <4 x i32> @phaddd_single_source3(<4 x i32> %x) {
 define <4 x i32> @phaddd_single_source4(<4 x i32> %x) {
 ; SSSE3-SLOW-LABEL: phaddd_single_source4:
 ; SSSE3-SLOW:       # %bb.0:
-; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,1,2,2]
+; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,2,2,2]
 ; SSSE3-SLOW-NEXT:    paddd %xmm1, %xmm0
 ; SSSE3-SLOW-NEXT:    retq
 ;
@@ -389,7 +389,7 @@ define <4 x i32> @phaddd_single_source4(<4 x i32> %x) {
 ;
 ; AVX-SLOW-LABEL: phaddd_single_source4:
 ; AVX-SLOW:       # %bb.0:
-; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[0,1,2,2]
+; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,2,2,2]
 ; AVX-SLOW-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
 ; AVX-SLOW-NEXT:    retq
 ;
@@ -400,7 +400,7 @@ define <4 x i32> @phaddd_single_source4(<4 x i32> %x) {
 ;
 ; AVX2-SHUF-LABEL: phaddd_single_source4:
 ; AVX2-SHUF:       # %bb.0:
-; AVX2-SHUF-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[0,1,2,2]
+; AVX2-SHUF-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,2,2,2]
 ; AVX2-SHUF-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
 ; AVX2-SHUF-NEXT:    retq
   %l = shufflevector <4 x i32> %x, <4 x i32> undef, <4 x i32> <i32 undef, i32 undef, i32 undef, i32 2>
@@ -411,35 +411,35 @@ define <4 x i32> @phaddd_single_source4(<4 x i32> %x) {
 define <4 x i32> @phaddd_single_source5(<4 x i32> %x) {
 ; SSSE3-SLOW-LABEL: phaddd_single_source5:
 ; SSSE3-SLOW:       # %bb.0:
-; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,1,2,2]
+; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,2,2,2]
 ; SSSE3-SLOW-NEXT:    paddd %xmm0, %xmm1
-; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[3,1,2,3]
+; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[3,3,3,3]
 ; SSSE3-SLOW-NEXT:    retq
 ;
 ; SSSE3-FAST-LABEL: phaddd_single_source5:
 ; SSSE3-FAST:       # %bb.0:
 ; SSSE3-FAST-NEXT:    phaddd %xmm0, %xmm0
-; SSSE3-FAST-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSSE3-FAST-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSSE3-FAST-NEXT:    retq
 ;
 ; AVX-SLOW-LABEL: phaddd_single_source5:
 ; AVX-SLOW:       # %bb.0:
-; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[0,1,2,2]
+; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,2,2,2]
 ; AVX-SLOW-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
-; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX-SLOW-NEXT:    retq
 ;
 ; AVX-FAST-LABEL: phaddd_single_source5:
 ; AVX-FAST:       # %bb.0:
 ; AVX-FAST-NEXT:    vphaddd %xmm0, %xmm0, %xmm0
-; AVX-FAST-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX-FAST-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX-FAST-NEXT:    retq
 ;
 ; AVX2-SHUF-LABEL: phaddd_single_source5:
 ; AVX2-SHUF:       # %bb.0:
-; AVX2-SHUF-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[0,1,2,2]
+; AVX2-SHUF-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,2,2,2]
 ; AVX2-SHUF-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
-; AVX2-SHUF-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX2-SHUF-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX2-SHUF-NEXT:    retq
   %l = shufflevector <4 x i32> %x, <4 x i32> undef, <4 x i32> <i32 undef, i32 undef, i32 undef, i32 2>
   %add = add <4 x i32> %l, %x
@@ -487,27 +487,27 @@ define <8 x i16> @phaddw_single_source2(<8 x i16> %x) {
 ; SSSE3:       # %bb.0:
 ; SSSE3-NEXT:    phaddw %xmm0, %xmm0
 ; SSSE3-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,4,6,7]
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,1,2,3]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,2,2,2]
 ; SSSE3-NEXT:    retq
 ;
 ; AVX-SLOW-LABEL: phaddw_single_source2:
 ; AVX-SLOW:       # %bb.0:
 ; AVX-SLOW-NEXT:    vphaddw %xmm0, %xmm0, %xmm0
 ; AVX-SLOW-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,4,6,7]
-; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,1,2,3]
+; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,2,2]
 ; AVX-SLOW-NEXT:    retq
 ;
 ; AVX-FAST-LABEL: phaddw_single_source2:
 ; AVX-FAST:       # %bb.0:
 ; AVX-FAST-NEXT:    vphaddw %xmm0, %xmm0, %xmm0
 ; AVX-FAST-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,4,6,7]
-; AVX-FAST-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,1,2,3]
+; AVX-FAST-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,2,2]
 ; AVX-FAST-NEXT:    retq
 ;
 ; AVX2-SHUF-LABEL: phaddw_single_source2:
 ; AVX2-SHUF:       # %bb.0:
 ; AVX2-SHUF-NEXT:    vphaddw %xmm0, %xmm0, %xmm0
-; AVX2-SHUF-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[10,11,8,9,4,5,6,7,10,11,8,9,12,13,14,15]
+; AVX2-SHUF-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[10,11,8,9,10,11,8,9,10,11,8,9,10,11,8,9]
 ; AVX2-SHUF-NEXT:    retq
   %l = shufflevector <8 x i16> %x, <8 x i16> undef, <8 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 0, i32 2, i32 4, i32 6>
   %r = shufflevector <8 x i16> %x, <8 x i16> undef, <8 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 1, i32 3, i32 5, i32 7>
@@ -592,7 +592,7 @@ define i32 @PR39936_v8i32(<8 x i32>) {
 ; SSSE3-SLOW:       # %bb.0:
 ; SSSE3-SLOW-NEXT:    phaddd %xmm1, %xmm0
 ; SSSE3-SLOW-NEXT:    phaddd %xmm0, %xmm0
-; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSSE3-SLOW-NEXT:    paddd %xmm0, %xmm1
 ; SSSE3-SLOW-NEXT:    movd %xmm1, %eax
 ; SSSE3-SLOW-NEXT:    retq
@@ -610,7 +610,7 @@ define i32 @PR39936_v8i32(<8 x i32>) {
 ; AVX1-SLOW-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-SLOW-NEXT:    vphaddd %xmm1, %xmm0, %xmm0
 ; AVX1-SLOW-NEXT:    vphaddd %xmm0, %xmm0, %xmm0
-; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-SLOW-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
 ; AVX1-SLOW-NEXT:    vmovd %xmm0, %eax
 ; AVX1-SLOW-NEXT:    vzeroupper
@@ -631,7 +631,7 @@ define i32 @PR39936_v8i32(<8 x i32>) {
 ; AVX2-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-SLOW-NEXT:    vphaddd %xmm1, %xmm0, %xmm0
 ; AVX2-SLOW-NEXT:    vphaddd %xmm0, %xmm0, %xmm0
-; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-SLOW-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
 ; AVX2-SLOW-NEXT:    vmovd %xmm0, %eax
 ; AVX2-SLOW-NEXT:    vzeroupper
@@ -652,7 +652,7 @@ define i32 @PR39936_v8i32(<8 x i32>) {
 ; AVX2-SHUF-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-SHUF-NEXT:    vphaddd %xmm1, %xmm0, %xmm0
 ; AVX2-SHUF-NEXT:    vphaddd %xmm0, %xmm0, %xmm0
-; AVX2-SHUF-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-SHUF-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-SHUF-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
 ; AVX2-SHUF-NEXT:    vmovd %xmm0, %eax
 ; AVX2-SHUF-NEXT:    vzeroupper
diff --git a/llvm/test/CodeGen/X86/pmaddubsw.ll b/llvm/test/CodeGen/X86/pmaddubsw.ll
index e65762158f37e..2879dd16ac96e 100644
--- a/llvm/test/CodeGen/X86/pmaddubsw.ll
+++ b/llvm/test/CodeGen/X86/pmaddubsw.ll
@@ -331,20 +331,20 @@ define <8 x i16> @pmaddubsw_bad_extend(<16 x i8>* %Aptr, <16 x i8>* %Bptr) {
 ; AVX1-NEXT:    vpshufb %xmm2, %xmm1, %xmm2
 ; AVX1-NEXT:    vpshufb %xmm4, %xmm1, %xmm1
 ; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm4
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,1,1]
 ; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm5 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
 ; AVX1-NEXT:    vpmulld %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,1,1]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
 ; AVX1-NEXT:    vpmulld %xmm2, %xmm3, %xmm2
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm3 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm5
 ; AVX1-NEXT:    vpmulld %xmm5, %xmm3, %xmm3
 ; AVX1-NEXT:    vpaddd %xmm3, %xmm4, %xmm3
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
 ; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpaddd %xmm0, %xmm2, %xmm0
@@ -432,20 +432,20 @@ define <8 x i16> @pmaddubsw_bad_indices(<16 x i8>* %Aptr, <16 x i8>* %Bptr) {
 ; AVX1-NEXT:    vpshufb {{.*#+}} xmm3 = xmm1[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
 ; AVX1-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u]
 ; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm4
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,1,1]
 ; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm2
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm5 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
 ; AVX1-NEXT:    vpmulld %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,1,1]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
 ; AVX1-NEXT:    vpmulld %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm3
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm0
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm5 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; AVX1-NEXT:    vpmulld %xmm5, %xmm3, %xmm3
 ; AVX1-NEXT:    vpaddd %xmm3, %xmm4, %xmm3
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpaddd %xmm0, %xmm2, %xmm0
diff --git a/llvm/test/CodeGen/X86/pmul.ll b/llvm/test/CodeGen/X86/pmul.ll
index 4285e7b603f80..9aeb8292ac0ca 100644
--- a/llvm/test/CodeGen/X86/pmul.ll
+++ b/llvm/test/CodeGen/X86/pmul.ll
@@ -1304,11 +1304,11 @@ define <8 x i64> @mul_v8i64_sext(<8 x i16> %val1, <8 x i32> %val2) {
 ;
 ; SSE41-LABEL: mul_v8i64_sext:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[3,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[3,3,3,3]
 ; SSE41-NEXT:    pmovsxwq %xmm3, %xmm4
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovsxwq %xmm3, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,1,1]
 ; SSE41-NEXT:    pmovsxwq %xmm3, %xmm6
 ; SSE41-NEXT:    pmovsxwq %xmm0, %xmm7
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[2,2,3,3]
diff --git a/llvm/test/CodeGen/X86/pmulh.ll b/llvm/test/CodeGen/X86/pmulh.ll
index 31e113f9a003f..ba33fed60f3f6 100644
--- a/llvm/test/CodeGen/X86/pmulh.ll
+++ b/llvm/test/CodeGen/X86/pmulh.ll
@@ -1539,11 +1539,11 @@ define <8 x i64> @mulhuw_v8i16_lshr_i64(<8 x i16> %a, <8 x i16> %b) {
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pmulhuw %xmm1, %xmm0
 ; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm4 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm3 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
 ; SSE41-NEXT:    movdqa %xmm4, %xmm0
 ; SSE41-NEXT:    retq
@@ -1646,11 +1646,11 @@ define <8 x i64> @mulhsw_v8i16_lshr_i64(<8 x i16> %a, <8 x i16> %b) {
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pmulhw %xmm1, %xmm0
 ; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm4 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm3 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
 ; SSE41-NEXT:    movdqa %xmm4, %xmm0
 ; SSE41-NEXT:    retq
@@ -1773,11 +1773,11 @@ define <8 x i64> @mulhsw_v8i16_ashr_i64(<8 x i16> %a, <8 x i16> %b) {
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pmulhw %xmm1, %xmm0
 ; SSE41-NEXT:    pmovsxwq %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE41-NEXT:    pmovsxwq %xmm1, %xmm1
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovsxwq %xmm2, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE41-NEXT:    pmovsxwq %xmm0, %xmm3
 ; SSE41-NEXT:    movdqa %xmm4, %xmm0
 ; SSE41-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/pow.ll b/llvm/test/CodeGen/X86/pow.ll
index f3d713f5f2245..e382a2d32ccdd 100644
--- a/llvm/test/CodeGen/X86/pow.ll
+++ b/llvm/test/CodeGen/X86/pow.ll
@@ -152,7 +152,7 @@ define <4 x float> @pow_v4f32_one_fourth_not_enough_fmf(<4 x float> %x) nounwind
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    subq $56, %rsp
 ; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; CHECK-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; CHECK-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; CHECK-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; CHECK-NEXT:    callq powf
 ; CHECK-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
@@ -168,7 +168,7 @@ define <4 x float> @pow_v4f32_one_fourth_not_enough_fmf(<4 x float> %x) nounwind
 ; CHECK-NEXT:    callq powf
 ; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; CHECK-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; CHECK-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; CHECK-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; CHECK-NEXT:    callq powf
 ; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
diff --git a/llvm/test/CodeGen/X86/pr14161.ll b/llvm/test/CodeGen/X86/pr14161.ll
index 152f3d350e807..b78f064cfe5af 100644
--- a/llvm/test/CodeGen/X86/pr14161.ll
+++ b/llvm/test/CodeGen/X86/pr14161.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-linux-pc -mcpu=corei7 | FileCheck %s
 
 declare <4 x i32> @llvm.x86.sse41.pminud(<4 x i32>, <4 x i32>)
@@ -25,7 +26,7 @@ define <2 x i16> @bad(<4 x i32>*, <4 x i8>*) {
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    movdqa (%rdi), %xmm0
 ; CHECK-NEXT:    pminud {{.*}}(%rip), %xmm0
-; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; CHECK-NEXT:    retq
 entry:
   %2 = load <4 x i32>, <4 x i32>* %0, align 16
diff --git a/llvm/test/CodeGen/X86/pr29112.ll b/llvm/test/CodeGen/X86/pr29112.ll
index 0a2dfef43d577..d07829725cf67 100644
--- a/llvm/test/CodeGen/X86/pr29112.ll
+++ b/llvm/test/CodeGen/X86/pr29112.ll
@@ -32,7 +32,7 @@ define <4 x float> @bar(<4 x float>* %a1p, <4 x float>* %a2p, <4 x float> %a3, <
 ; CHECK-NEXT:    vmovaps %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; CHECK-NEXT:    vextractf32x4 $2, %zmm3, %xmm4
 ; CHECK-NEXT:    vblendps {{.*#+}} xmm4 = xmm1[0,1,2],xmm4[3]
-; CHECK-NEXT:    vpermilps {{.*#+}} xmm0 = xmm2[3,1,2,3]
+; CHECK-NEXT:    vpermilps {{.*#+}} xmm0 = xmm2[3,3,3,3]
 ; CHECK-NEXT:    vunpcklps {{.*#+}} xmm5 = xmm0[0],xmm5[0],xmm0[1],xmm5[1]
 ; CHECK-NEXT:    vinsertps {{.*#+}} xmm5 = xmm5[0,1],xmm2[1],xmm5[3]
 ; CHECK-NEXT:    vinsertps {{.*#+}} xmm5 = xmm5[0,1,2],xmm3[1]
diff --git a/llvm/test/CodeGen/X86/pr44976.ll b/llvm/test/CodeGen/X86/pr44976.ll
index 72ced2b1fa0f4..f16adfa6f0fc4 100644
--- a/llvm/test/CodeGen/X86/pr44976.ll
+++ b/llvm/test/CodeGen/X86/pr44976.ll
@@ -34,38 +34,35 @@ define <3 x i32> @f_29(<12 x i16> %a, <12 x i16> %b) {
 ; CHECK-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
 ; CHECK-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
 ; CHECK-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
-; CHECK-NEXT:    movd {{.*#+}} xmm3 = mem[0],zero,zero,zero
-; CHECK-NEXT:    pinsrw $1, {{[0-9]+}}(%rsp), %xmm3
-; CHECK-NEXT:    pinsrw $2, {{[0-9]+}}(%rsp), %xmm3
-; CHECK-NEXT:    pinsrw $3, {{[0-9]+}}(%rsp), %xmm3
 ; CHECK-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
 ; CHECK-NEXT:    pinsrw $1, {{[0-9]+}}(%rsp), %xmm2
 ; CHECK-NEXT:    pinsrw $2, {{[0-9]+}}(%rsp), %xmm2
 ; CHECK-NEXT:    pinsrw $3, {{[0-9]+}}(%rsp), %xmm2
+; CHECK-NEXT:    movd {{.*#+}} xmm3 = mem[0],zero,zero,zero
+; CHECK-NEXT:    pinsrw $1, {{[0-9]+}}(%rsp), %xmm3
+; CHECK-NEXT:    pinsrw $2, {{[0-9]+}}(%rsp), %xmm3
+; CHECK-NEXT:    pinsrw $3, {{[0-9]+}}(%rsp), %xmm3
 ; CHECK-NEXT:    movdqa %xmm0, %xmm4
 ; CHECK-NEXT:    pmulhuw %xmm1, %xmm4
 ; CHECK-NEXT:    pmullw %xmm1, %xmm0
 ; CHECK-NEXT:    movdqa %xmm0, %xmm1
 ; CHECK-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7]
-; CHECK-NEXT:    movdqa %xmm2, %xmm5
-; CHECK-NEXT:    pmulhuw %xmm3, %xmm5
-; CHECK-NEXT:    pmullw %xmm3, %xmm2
-; CHECK-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3]
 ; CHECK-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
-; CHECK-NEXT:    movdqa %xmm0, %xmm3
-; CHECK-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,3],xmm1[1,3]
+; CHECK-NEXT:    movdqa %xmm3, %xmm4
+; CHECK-NEXT:    pmulhuw %xmm2, %xmm4
+; CHECK-NEXT:    pmullw %xmm2, %xmm3
+; CHECK-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
+; CHECK-NEXT:    movdqa %xmm0, %xmm2
+; CHECK-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,3],xmm1[1,3]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[1,3,1,3]
 ; CHECK-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
-; CHECK-NEXT:    shufps {{.*#+}} xmm1 = xmm1[2,3],xmm2[0,1]
-; CHECK-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1,2,3]
-; CHECK-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[3,3,2,3]
-; CHECK-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
-; CHECK-NEXT:    paddd %xmm3, %xmm0
-; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,1,3]
-; CHECK-NEXT:    paddd %xmm1, %xmm2
-; CHECK-NEXT:    movdqa %xmm0, %xmm1
-; CHECK-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,3],xmm2[1,3]
-; CHECK-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm2[0,3]
-; CHECK-NEXT:    paddd %xmm1, %xmm0
+; CHECK-NEXT:    paddd %xmm2, %xmm0
+; CHECK-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[0,2,1,3]
+; CHECK-NEXT:    paddd %xmm4, %xmm1
+; CHECK-NEXT:    movdqa %xmm0, %xmm2
+; CHECK-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,3],xmm1[1,3]
+; CHECK-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,3]
+; CHECK-NEXT:    paddd %xmm2, %xmm0
 ; CHECK-NEXT:    retq
 entry:
   %a32 = zext <12 x i16> %a to <12 x i32>
diff --git a/llvm/test/CodeGen/X86/pr46455.ll b/llvm/test/CodeGen/X86/pr46455.ll
index c232d548faefd..ceb9275bd03b5 100644
--- a/llvm/test/CodeGen/X86/pr46455.ll
+++ b/llvm/test/CodeGen/X86/pr46455.ll
@@ -9,9 +9,9 @@ define void @EntryModule(i8** %buffer_table) {
 ; CHECK-NEXT:    movq 24(%rdi), %rcx
 ; CHECK-NEXT:    vcmpneqps (%rax), %ymm0, %ymm0
 ; CHECK-NEXT:    vpsrld $31, %xmm0, %xmm1
-; CHECK-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,2,3]
+; CHECK-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
 ; CHECK-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
-; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; CHECK-NEXT:    vpaddd %xmm1, %xmm3, %xmm1
 ; CHECK-NEXT:    vpsubd %xmm0, %xmm2, %xmm0
 ; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
diff --git a/llvm/test/CodeGen/X86/pr46527.ll b/llvm/test/CodeGen/X86/pr46527.ll
index 1d3f16f8c1aeb..48b1095a6329a 100644
--- a/llvm/test/CodeGen/X86/pr46527.ll
+++ b/llvm/test/CodeGen/X86/pr46527.ll
@@ -18,7 +18,7 @@ define void @f(<16 x i8>* %out, <16 x i8> %in, i1 %flag) {
 ; CHECK-NEXT:    movzbl %dl, %edx
 ; CHECK-NEXT:    movd %edx, %xmm1
 ; CHECK-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; CHECK-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,0,2,3,4,5,6,7]
+; CHECK-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,0,0,0,4,5,6,7]
 ; CHECK-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; CHECK-NEXT:    paddb %xmm1, %xmm1
 ; CHECK-NEXT:    pxor %xmm0, %xmm1
diff --git a/llvm/test/CodeGen/X86/prefer-avx256-mask-shuffle.ll b/llvm/test/CodeGen/X86/prefer-avx256-mask-shuffle.ll
index bdbc563816864..657b4525871f5 100644
--- a/llvm/test/CodeGen/X86/prefer-avx256-mask-shuffle.ll
+++ b/llvm/test/CodeGen/X86/prefer-avx256-mask-shuffle.ll
@@ -151,9 +151,7 @@ define <32 x i1> @shuf32i1_3_6_22_12_3_7_7_0_3_6_1_13_3_21_7_0_3_6_22_12_3_7_7_0
 ; AVX256VL-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[6,7,12,13],zero,zero,ymm1[8,9,6,7,14,15,14,15,0,1,22,23,28,29,18,19,26,27,22,23],zero,zero,ymm1[30,31,16,17]
 ; AVX256VL-NEXT:    vmovdqa32 %ymm0, %ymm2 {%k1} {z}
 ; AVX256VL-NEXT:    vpmovdw %ymm2, %xmm2
-; AVX256VL-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX256VL-NEXT:    vinserti128 $1, %xmm3, %ymm2, %ymm2
-; AVX256VL-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[1,1,2,1]
+; AVX256VL-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[1,1,1,1]
 ; AVX256VL-NEXT:    vpternlogq $220, {{.*}}(%rip), %ymm1, %ymm2
 ; AVX256VL-NEXT:    vpmovsxwd %xmm2, %ymm1
 ; AVX256VL-NEXT:    vpslld $31, %ymm1, %ymm1
@@ -196,7 +194,7 @@ define <32 x i1> @shuf32i1_3_6_22_12_3_7_7_0_3_6_1_13_3_21_7_0_3_6_22_12_3_7_7_0
 ; AVX256VLBW-NEXT:    vptestnmb %ymm0, %ymm0, %k0
 ; AVX256VLBW-NEXT:    vpmovm2b %k0, %ymm0
 ; AVX256VLBW-NEXT:    vpermq {{.*#+}} ymm1 = ymm0[2,3,2,3]
-; AVX256VLBW-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[1,1,2,1,5,5,6,5]
+; AVX256VLBW-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[1,1,1,1,5,5,5,5]
 ; AVX256VLBW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,0,1]
 ; AVX256VLBW-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[3,6,u,12,3,7,7,0,3,6,1,13,3,u,7,0,19,22,u,28,19,23,23,16,19,22,17,29,19,u,23,16]
 ; AVX256VLBW-NEXT:    movl $537141252, %eax # imm = 0x20042004
diff --git a/llvm/test/CodeGen/X86/psubus.ll b/llvm/test/CodeGen/X86/psubus.ll
index d785b10f9c327..53ff270ad340c 100644
--- a/llvm/test/CodeGen/X86/psubus.ll
+++ b/llvm/test/CodeGen/X86/psubus.ll
@@ -45,7 +45,7 @@ define <8 x i16> @test3(<8 x i16> %x, i16 zeroext %w) nounwind {
 ; SSE-LABEL: test3:
 ; SSE:       # %bb.0: # %vector.ph
 ; SSE-NEXT:    movd %edi, %xmm1
-; SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,0,2,3,4,5,6,7]
+; SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,0,0,0,4,5,6,7]
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; SSE-NEXT:    psubusw %xmm1, %xmm0
 ; SSE-NEXT:    retq
@@ -53,7 +53,7 @@ define <8 x i16> @test3(<8 x i16> %x, i16 zeroext %w) nounwind {
 ; AVX1-LABEL: test3:
 ; AVX1:       # %bb.0: # %vector.ph
 ; AVX1-NEXT:    vmovd %edi, %xmm1
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; AVX1-NEXT:    vpsubusw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    retq
@@ -118,7 +118,7 @@ define <16 x i8> @test6(<16 x i8> %x, i8 zeroext %w) nounwind {
 ; SSE2:       # %bb.0: # %vector.ph
 ; SSE2-NEXT:    movd %edi, %xmm1
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,0,2,3,4,5,6,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,0,0,0,4,5,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; SSE2-NEXT:    psubusb %xmm1, %xmm0
 ; SSE2-NEXT:    retq
@@ -269,7 +269,7 @@ define <16 x i16> @test9(<16 x i16> %x, i16 zeroext %w) nounwind {
 ; SSE-LABEL: test9:
 ; SSE:       # %bb.0: # %vector.ph
 ; SSE-NEXT:    movd %edi, %xmm2
-; SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,0,2,3,4,5,6,7]
+; SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,0,0,0,4,5,6,7]
 ; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
 ; SSE-NEXT:    psubusw %xmm2, %xmm0
 ; SSE-NEXT:    psubusw %xmm2, %xmm1
@@ -279,7 +279,7 @@ define <16 x i16> @test9(<16 x i16> %x, i16 zeroext %w) nounwind {
 ; AVX1:       # %bb.0: # %vector.ph
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vmovd %edi, %xmm2
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm2[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm2[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
 ; AVX1-NEXT:    vpsubusw %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vpsubusw %xmm2, %xmm0, %xmm0
@@ -409,7 +409,7 @@ define <32 x i8> @test12(<32 x i8> %x, i8 zeroext %w) nounwind {
 ; SSE2:       # %bb.0: # %vector.ph
 ; SSE2-NEXT:    movd %edi, %xmm2
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,0,2,3,4,5,6,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,0,0,0,4,5,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
 ; SSE2-NEXT:    psubusb %xmm2, %xmm0
 ; SSE2-NEXT:    psubusb %xmm2, %xmm1
@@ -694,12 +694,12 @@ define <16 x i8> @test14(<16 x i8> %x, <16 x i32> %y) nounwind {
 ;
 ; SSE41-LABEL: test14:
 ; SSE41:       # %bb.0: # %vector.ph
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,1,1,1]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm8 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm5 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm7 = xmm7[0],zero,zero,zero,xmm7[1],zero,zero,zero,xmm7[2],zero,zero,zero,xmm7[3],zero,zero,zero
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm0[3,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm0[3,3,3,3]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm6 = xmm6[0],zero,zero,zero,xmm6[1],zero,zero,zero,xmm6[2],zero,zero,zero,xmm6[3],zero,zero,zero
 ; SSE41-NEXT:    pmaxud %xmm4, %xmm6
 ; SSE41-NEXT:    pcmpeqd %xmm4, %xmm6
@@ -732,12 +732,12 @@ define <16 x i8> @test14(<16 x i8> %x, <16 x i32> %y) nounwind {
 ;
 ; AVX1-LABEL: test14:
 ; AVX1:       # %bb.0: # %vector.ph
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm4 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm5 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm0[3,3,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm0[3,3,3,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm6 = xmm6[0],zero,zero,zero,xmm6[1],zero,zero,zero,xmm6[2],zero,zero,zero,xmm6[3],zero,zero,zero
 ; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm7
 ; AVX1-NEXT:    vpmaxud %xmm6, %xmm7, %xmm6
@@ -1085,7 +1085,7 @@ define <64 x i8> @test17(<64 x i8> %x, i8 zeroext %w) nounwind {
 ; SSE2:       # %bb.0: # %vector.ph
 ; SSE2-NEXT:    movd %edi, %xmm4
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm4 = xmm4[0,0,2,3,4,5,6,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm4 = xmm4[0,0,0,0,4,5,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[0,0,0,0]
 ; SSE2-NEXT:    psubusb %xmm4, %xmm0
 ; SSE2-NEXT:    psubusb %xmm4, %xmm1
@@ -1156,7 +1156,7 @@ define <32 x i16> @test18(<32 x i16> %x, i16 zeroext %w) nounwind {
 ; SSE-LABEL: test18:
 ; SSE:       # %bb.0: # %vector.ph
 ; SSE-NEXT:    movd %edi, %xmm4
-; SSE-NEXT:    pshuflw {{.*#+}} xmm4 = xmm4[0,0,2,3,4,5,6,7]
+; SSE-NEXT:    pshuflw {{.*#+}} xmm4 = xmm4[0,0,0,0,4,5,6,7]
 ; SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[0,0,0,0]
 ; SSE-NEXT:    psubusw %xmm4, %xmm0
 ; SSE-NEXT:    psubusw %xmm4, %xmm1
@@ -1168,7 +1168,7 @@ define <32 x i16> @test18(<32 x i16> %x, i16 zeroext %w) nounwind {
 ; AVX1:       # %bb.0: # %vector.ph
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
 ; AVX1-NEXT:    vmovd %edi, %xmm3
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm3[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm3[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
 ; AVX1-NEXT:    vpsubusw %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vpsubusw %xmm3, %xmm0, %xmm0
diff --git a/llvm/test/CodeGen/X86/sad.ll b/llvm/test/CodeGen/X86/sad.ll
index a197e795754ad..ef26c84bdf5ed 100644
--- a/llvm/test/CodeGen/X86/sad.ll
+++ b/llvm/test/CodeGen/X86/sad.ll
@@ -29,7 +29,7 @@ define i32 @sad_16i8() nounwind {
 ; SSE2-NEXT:    paddd %xmm1, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE2-NEXT:    paddd %xmm1, %xmm0
 ; SSE2-NEXT:    movd %xmm0, %eax
 ; SSE2-NEXT:    retq
@@ -56,7 +56,7 @@ define i32 @sad_16i8() nounwind {
 ; AVX1-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
 ; AVX1-NEXT:    vzeroupper
@@ -81,7 +81,7 @@ define i32 @sad_16i8() nounwind {
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
 ; AVX2-NEXT:    vzeroupper
@@ -106,7 +106,7 @@ define i32 @sad_16i8() nounwind {
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
 ; AVX512-NEXT:    vzeroupper
@@ -174,7 +174,7 @@ define i32 @sad_32i8() nounwind {
 ; SSE2-NEXT:    paddd %xmm2, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    paddd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
 ; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    retq
@@ -209,7 +209,7 @@ define i32 @sad_32i8() nounwind {
 ; AVX1-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
 ; AVX1-NEXT:    vzeroupper
@@ -236,7 +236,7 @@ define i32 @sad_32i8() nounwind {
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
 ; AVX2-NEXT:    vzeroupper
@@ -263,7 +263,7 @@ define i32 @sad_32i8() nounwind {
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
 ; AVX512-NEXT:    vzeroupper
@@ -348,7 +348,7 @@ define i32 @sad_avx64i8() nounwind {
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    paddd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
 ; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    retq
@@ -399,7 +399,7 @@ define i32 @sad_avx64i8() nounwind {
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
 ; AVX1-NEXT:    vzeroupper
@@ -433,7 +433,7 @@ define i32 @sad_avx64i8() nounwind {
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
 ; AVX2-NEXT:    vzeroupper
@@ -465,7 +465,7 @@ define i32 @sad_avx64i8() nounwind {
 ; AVX512F-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512F-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512F-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX512F-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512F-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512F-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512F-NEXT:    vmovd %xmm0, %eax
 ; AVX512F-NEXT:    vzeroupper
@@ -494,7 +494,7 @@ define i32 @sad_avx64i8() nounwind {
 ; AVX512BW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512BW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512BW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vmovd %xmm0, %eax
 ; AVX512BW-NEXT:    vzeroupper
@@ -557,7 +557,7 @@ define i32 @sad_2i8() nounwind {
 ; SSE2-NEXT:    addq $2, %rax
 ; SSE2-NEXT:    jne .LBB3_1
 ; SSE2-NEXT:  # %bb.2: # %middle.block
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
 ; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    retq
@@ -579,7 +579,7 @@ define i32 @sad_2i8() nounwind {
 ; AVX-NEXT:    addq $2, %rax
 ; AVX-NEXT:    jne .LBB3_1
 ; AVX-NEXT:  # %bb.2: # %middle.block
-; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; AVX-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    vmovd %xmm0, %eax
 ; AVX-NEXT:    retq
@@ -630,7 +630,7 @@ define i32 @sad_4i8() nounwind {
 ; SSE2-NEXT:  # %bb.2: # %middle.block
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE2-NEXT:    paddd %xmm1, %xmm0
 ; SSE2-NEXT:    movd %xmm0, %eax
 ; SSE2-NEXT:    retq
@@ -651,7 +651,7 @@ define i32 @sad_4i8() nounwind {
 ; AVX-NEXT:  # %bb.2: # %middle.block
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovd %xmm0, %eax
 ; AVX-NEXT:    retq
@@ -989,7 +989,7 @@ define i32 @sad_unroll_nonzero_initial(<16 x i8>* %arg, <16 x i8>* %arg1, <16 x
 ; SSE2-NEXT:    paddd {{.*}}(%rip), %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE2-NEXT:    paddd %xmm2, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
 ; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    retq
@@ -1004,7 +1004,7 @@ define i32 @sad_unroll_nonzero_initial(<16 x i8>* %arg, <16 x i8>* %arg1, <16 x
 ; AVX-NEXT:    vpaddd {{.*}}(%rip), %xmm0, %xmm0
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovd %xmm0, %eax
 ; AVX-NEXT:    retq
@@ -1053,24 +1053,52 @@ define i32 @sad_double_reduction(<16 x i8>* %arg, <16 x i8>* %arg1, <16 x i8>* %
 ; SSE2-NEXT:    paddd %xmm1, %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE2-NEXT:    paddd %xmm2, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; SSE2-NEXT:    paddd %xmm0, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; SSE2-NEXT:    por %xmm0, %xmm1
 ; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    retq
 ;
-; AVX-LABEL: sad_double_reduction:
-; AVX:       # %bb.0: # %bb
-; AVX-NEXT:    vmovdqu (%rdi), %xmm0
-; AVX-NEXT:    vpsadbw (%rsi), %xmm0, %xmm0
-; AVX-NEXT:    vmovdqu (%rdx), %xmm1
-; AVX-NEXT:    vpsadbw (%rcx), %xmm1, %xmm1
-; AVX-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vmovd %xmm0, %eax
-; AVX-NEXT:    retq
+; AVX1-LABEL: sad_double_reduction:
+; AVX1:       # %bb.0: # %bb
+; AVX1-NEXT:    vmovdqu (%rdi), %xmm0
+; AVX1-NEXT:    vpsadbw (%rsi), %xmm0, %xmm0
+; AVX1-NEXT:    vmovdqu (%rdx), %xmm1
+; AVX1-NEXT:    vpsadbw (%rcx), %xmm1, %xmm1
+; AVX1-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vmovd %xmm0, %eax
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: sad_double_reduction:
+; AVX2:       # %bb.0: # %bb
+; AVX2-NEXT:    vmovdqu (%rdi), %xmm0
+; AVX2-NEXT:    vpsadbw (%rsi), %xmm0, %xmm0
+; AVX2-NEXT:    vmovdqu (%rdx), %xmm1
+; AVX2-NEXT:    vpsadbw (%rcx), %xmm1, %xmm1
+; AVX2-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vmovd %xmm0, %eax
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: sad_double_reduction:
+; AVX512:       # %bb.0: # %bb
+; AVX512-NEXT:    vmovdqu (%rdi), %xmm0
+; AVX512-NEXT:    vpsadbw (%rsi), %xmm0, %xmm0
+; AVX512-NEXT:    vmovdqu (%rdx), %xmm1
+; AVX512-NEXT:    vpsadbw (%rcx), %xmm1, %xmm1
+; AVX512-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vmovd %xmm0, %eax
+; AVX512-NEXT:    retq
 bb:
   %tmp = load <16 x i8>, <16 x i8>* %arg, align 1
   %tmp4 = load <16 x i8>, <16 x i8>* %arg1, align 1
diff --git a/llvm/test/CodeGen/X86/scalarize-fp.ll b/llvm/test/CodeGen/X86/scalarize-fp.ll
index 5b98272fdce5e..77b0e0120a621 100644
--- a/llvm/test/CodeGen/X86/scalarize-fp.ll
+++ b/llvm/test/CodeGen/X86/scalarize-fp.ll
@@ -779,14 +779,14 @@ define <4 x float> @multi_use_binop(<4 x float> %x, <4 x float> %y) {
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    mulps %xmm1, %xmm0
 ; SSE-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1,2,0]
+; SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; SSE-NEXT:    addps %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: multi_use_binop:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vmulps %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[0,1,2,0]
+; AVX-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[0,0,0,0]
 ; AVX-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0]
 ; AVX-NEXT:    vaddps %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/setcc-combine.ll b/llvm/test/CodeGen/X86/setcc-combine.ll
index 65e8a6d6ed256..600a123681f32 100644
--- a/llvm/test/CodeGen/X86/setcc-combine.ll
+++ b/llvm/test/CodeGen/X86/setcc-combine.ll
@@ -6,7 +6,7 @@ define i32 @test_eq_1(<4 x i32> %A, <4 x i32> %B) {
 ; SSE2-LABEL: test_eq_1:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE2-NEXT:    movd %xmm0, %eax
 ; SSE2-NEXT:    notl %eax
 ; SSE2-NEXT:    retq
@@ -29,7 +29,7 @@ define i32 @test_ne_1(<4 x i32> %A, <4 x i32> %B) {
 ; SSE2-LABEL: test_ne_1:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE2-NEXT:    movd %xmm0, %eax
 ; SSE2-NEXT:    retq
 ;
@@ -63,7 +63,7 @@ define i32 @test_ge_1(<4 x i32> %A, <4 x i32> %B) {
 ; SSE2-LABEL: test_ge_1:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE2-NEXT:    movd %xmm0, %eax
 ; SSE2-NEXT:    notl %eax
 ; SSE2-NEXT:    retq
@@ -86,7 +86,7 @@ define i32 @test_lt_1(<4 x i32> %A, <4 x i32> %B) {
 ; SSE2-LABEL: test_lt_1:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE2-NEXT:    movd %xmm0, %eax
 ; SSE2-NEXT:    retq
 ;
@@ -120,7 +120,7 @@ define i32 @test_eq_2(<4 x i32> %A, <4 x i32> %B) {
 ; SSE2-LABEL: test_eq_2:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    movd %xmm0, %eax
 ; SSE2-NEXT:    notl %eax
 ; SSE2-NEXT:    retq
@@ -143,7 +143,7 @@ define i32 @test_ne_2(<4 x i32> %A, <4 x i32> %B) {
 ; SSE2-LABEL: test_ne_2:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    movd %xmm0, %eax
 ; SSE2-NEXT:    retq
 ;
@@ -164,7 +164,7 @@ define i32 @test_le_2(<4 x i32> %A, <4 x i32> %B) {
 ; SSE2-LABEL: test_le_2:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    movd %xmm0, %eax
 ; SSE2-NEXT:    notl %eax
 ; SSE2-NEXT:    retq
@@ -200,7 +200,7 @@ define i32 @test_lt_2(<4 x i32> %A, <4 x i32> %B) {
 ; SSE2-LABEL: test_lt_2:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    movd %xmm0, %eax
 ; SSE2-NEXT:    retq
 ;
@@ -221,7 +221,7 @@ define i32 @test_gt_2(<4 x i32> %A, <4 x i32> %B) {
 ; SSE2-LABEL: test_gt_2:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    movd %xmm0, %eax
 ; SSE2-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/shrink_vmul.ll b/llvm/test/CodeGen/X86/shrink_vmul.ll
index 17cfc53266909..5a0f23f6a09ce 100644
--- a/llvm/test/CodeGen/X86/shrink_vmul.ll
+++ b/llvm/test/CodeGen/X86/shrink_vmul.ll
@@ -2089,7 +2089,7 @@ define void @PR34947(<9 x i16>* %p0, <9 x i32>* %p1) nounwind {
 ; X86-SSE-NEXT:    movdqa %xmm4, %xmm3
 ; X86-SSE-NEXT:    psrldq {{.*#+}} xmm3 = xmm3[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; X86-SSE-NEXT:    movd %xmm3, %eax
-; X86-SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm5[3,1,2,3]
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm5[3,3,3,3]
 ; X86-SSE-NEXT:    movd %xmm3, %esi
 ; X86-SSE-NEXT:    xorl %edx, %edx
 ; X86-SSE-NEXT:    divl %esi
@@ -2107,9 +2107,9 @@ define void @PR34947(<9 x i16>* %p0, <9 x i32>* %p1) nounwind {
 ; X86-SSE-NEXT:    xorl %edx, %edx
 ; X86-SSE-NEXT:    divl %esi
 ; X86-SSE-NEXT:    movd %edx, %xmm3
-; X86-SSE-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,2,3]
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,1,1]
 ; X86-SSE-NEXT:    movd %xmm6, %eax
-; X86-SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,2,3]
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,1,1]
 ; X86-SSE-NEXT:    movd %xmm5, %esi
 ; X86-SSE-NEXT:    xorl %edx, %edx
 ; X86-SSE-NEXT:    divl %esi
@@ -2119,7 +2119,7 @@ define void @PR34947(<9 x i16>* %p0, <9 x i32>* %p1) nounwind {
 ; X86-SSE-NEXT:    movdqa %xmm4, %xmm5
 ; X86-SSE-NEXT:    psrld $16, %xmm5
 ; X86-SSE-NEXT:    movd %xmm5, %eax
-; X86-SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[1,1,2,3]
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[1,1,1,1]
 ; X86-SSE-NEXT:    movd %xmm5, %esi
 ; X86-SSE-NEXT:    xorl %edx, %edx
 ; X86-SSE-NEXT:    divl %esi
@@ -2132,7 +2132,7 @@ define void @PR34947(<9 x i16>* %p0, <9 x i32>* %p1) nounwind {
 ; X86-SSE-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
 ; X86-SSE-NEXT:    psrlq $48, %xmm4
 ; X86-SSE-NEXT:    movd %xmm4, %eax
-; X86-SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[3,1,2,3]
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[3,3,3,3]
 ; X86-SSE-NEXT:    movd %xmm4, %esi
 ; X86-SSE-NEXT:    xorl %edx, %edx
 ; X86-SSE-NEXT:    divl %esi
@@ -2331,7 +2331,7 @@ define void @PR34947(<9 x i16>* %p0, <9 x i32>* %p1) nounwind {
 ; X64-SSE-NEXT:    movdqa %xmm4, %xmm3
 ; X64-SSE-NEXT:    psrldq {{.*#+}} xmm3 = xmm3[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; X64-SSE-NEXT:    movd %xmm3, %eax
-; X64-SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm5[3,1,2,3]
+; X64-SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm5[3,3,3,3]
 ; X64-SSE-NEXT:    movd %xmm3, %ecx
 ; X64-SSE-NEXT:    xorl %edx, %edx
 ; X64-SSE-NEXT:    divl %ecx
@@ -2349,9 +2349,9 @@ define void @PR34947(<9 x i16>* %p0, <9 x i32>* %p1) nounwind {
 ; X64-SSE-NEXT:    xorl %edx, %edx
 ; X64-SSE-NEXT:    divl %ecx
 ; X64-SSE-NEXT:    movd %edx, %xmm3
-; X64-SSE-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,2,3]
+; X64-SSE-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,1,1]
 ; X64-SSE-NEXT:    movd %xmm6, %eax
-; X64-SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,2,3]
+; X64-SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,1,1]
 ; X64-SSE-NEXT:    movd %xmm5, %ecx
 ; X64-SSE-NEXT:    xorl %edx, %edx
 ; X64-SSE-NEXT:    divl %ecx
@@ -2361,7 +2361,7 @@ define void @PR34947(<9 x i16>* %p0, <9 x i32>* %p1) nounwind {
 ; X64-SSE-NEXT:    movdqa %xmm4, %xmm5
 ; X64-SSE-NEXT:    psrld $16, %xmm5
 ; X64-SSE-NEXT:    movd %xmm5, %eax
-; X64-SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[1,1,2,3]
+; X64-SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[1,1,1,1]
 ; X64-SSE-NEXT:    movd %xmm5, %ecx
 ; X64-SSE-NEXT:    xorl %edx, %edx
 ; X64-SSE-NEXT:    divl %ecx
@@ -2374,7 +2374,7 @@ define void @PR34947(<9 x i16>* %p0, <9 x i32>* %p1) nounwind {
 ; X64-SSE-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
 ; X64-SSE-NEXT:    psrlq $48, %xmm4
 ; X64-SSE-NEXT:    movd %xmm4, %eax
-; X64-SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[3,1,2,3]
+; X64-SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[3,3,3,3]
 ; X64-SSE-NEXT:    movd %xmm4, %ecx
 ; X64-SSE-NEXT:    xorl %edx, %edx
 ; X64-SSE-NEXT:    divl %ecx
diff --git a/llvm/test/CodeGen/X86/shuffle-strided-with-offset-128.ll b/llvm/test/CodeGen/X86/shuffle-strided-with-offset-128.ll
index 3434d9b2f0832..e0bc3240726f6 100644
--- a/llvm/test/CodeGen/X86/shuffle-strided-with-offset-128.ll
+++ b/llvm/test/CodeGen/X86/shuffle-strided-with-offset-128.ll
@@ -436,7 +436,7 @@ define void @shuffle_v16i8_to_v2i8_1(<16 x i8>* %L, <2 x i8>* %S) nounwind {
 ; SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    packuswb %xmm0, %xmm0
 ; SSE2-NEXT:    movd %xmm0, %eax
 ; SSE2-NEXT:    movw %ax, (%rsi)
@@ -515,7 +515,7 @@ define void @shuffle_v16i8_to_v2i8_3(<16 x i8>* %L, <2 x i8>* %S) nounwind {
 ; SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE2-NEXT:    packuswb %xmm0, %xmm0
 ; SSE2-NEXT:    movd %xmm0, %eax
 ; SSE2-NEXT:    movw %ax, (%rsi)
@@ -594,7 +594,7 @@ define void @shuffle_v16i8_to_v2i8_5(<16 x i8>* %L, <2 x i8>* %S) nounwind {
 ; SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
 ; SSE2-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    packuswb %xmm0, %xmm0
 ; SSE2-NEXT:    movd %xmm0, %eax
 ; SSE2-NEXT:    movw %ax, (%rsi)
@@ -673,7 +673,7 @@ define void @shuffle_v16i8_to_v2i8_7(<16 x i8>* %L, <2 x i8>* %S) nounwind {
 ; SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
 ; SSE2-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE2-NEXT:    packuswb %xmm0, %xmm0
 ; SSE2-NEXT:    movd %xmm0, %eax
 ; SSE2-NEXT:    movw %ax, (%rsi)
diff --git a/llvm/test/CodeGen/X86/slow-pmulld.ll b/llvm/test/CodeGen/X86/slow-pmulld.ll
index 53deafc9a4b43..48f166bea82e4 100644
--- a/llvm/test/CodeGen/X86/slow-pmulld.ll
+++ b/llvm/test/CodeGen/X86/slow-pmulld.ll
@@ -147,7 +147,7 @@ define <8 x i32> @test_mul_v8i32_v8i8(<8 x i8> %A) {
 ;
 ; SSE4-32-LABEL: test_mul_v8i32_v8i8:
 ; SSE4-32:       # %bb.0:
-; SSE4-32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE4-32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE4-32-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; SSE4-32-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE4-32-NEXT:    movdqa {{.*#+}} xmm2 = [18778,18778,18778,18778]
@@ -157,7 +157,7 @@ define <8 x i32> @test_mul_v8i32_v8i8(<8 x i8> %A) {
 ;
 ; SSE4-64-LABEL: test_mul_v8i32_v8i8:
 ; SSE4-64:       # %bb.0:
-; SSE4-64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE4-64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE4-64-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; SSE4-64-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE4-64-NEXT:    movdqa {{.*#+}} xmm2 = [18778,18778,18778,18778]
@@ -304,11 +304,11 @@ define <16 x i32> @test_mul_v16i32_v16i8(<16 x i8> %A) {
 ;
 ; SSE4-32-LABEL: test_mul_v16i32_v16i8:
 ; SSE4-32:       # %bb.0:
-; SSE4-32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; SSE4-32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
 ; SSE4-32-NEXT:    pmovzxbd {{.*#+}} xmm3 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; SSE4-32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE4-32-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
-; SSE4-32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE4-32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE4-32-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; SSE4-32-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE4-32-NEXT:    movdqa {{.*#+}} xmm4 = [18778,18778,18778,18778]
@@ -320,11 +320,11 @@ define <16 x i32> @test_mul_v16i32_v16i8(<16 x i8> %A) {
 ;
 ; SSE4-64-LABEL: test_mul_v16i32_v16i8:
 ; SSE4-64:       # %bb.0:
-; SSE4-64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; SSE4-64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
 ; SSE4-64-NEXT:    pmovzxbd {{.*#+}} xmm3 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; SSE4-64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE4-64-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
-; SSE4-64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE4-64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE4-64-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; SSE4-64-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE4-64-NEXT:    movdqa {{.*#+}} xmm4 = [18778,18778,18778,18778]
@@ -762,7 +762,7 @@ define <8 x i32> @test_mul_v8i32_v8i8_minsize(<8 x i8> %A) minsize {
 ; SLM32-LABEL: test_mul_v8i32_v8i8_minsize:
 ; SLM32:       # %bb.0:
 ; SLM32-NEXT:    movdqa {{.*#+}} xmm2 = [18778,18778,18778,18778]
-; SLM32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SLM32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SLM32-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SLM32-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; SLM32-NEXT:    pmaddwd %xmm2, %xmm0
@@ -772,7 +772,7 @@ define <8 x i32> @test_mul_v8i32_v8i8_minsize(<8 x i8> %A) minsize {
 ; SLM64-LABEL: test_mul_v8i32_v8i8_minsize:
 ; SLM64:       # %bb.0:
 ; SLM64-NEXT:    movdqa {{.*#+}} xmm2 = [18778,18778,18778,18778]
-; SLM64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SLM64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SLM64-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SLM64-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; SLM64-NEXT:    pmaddwd %xmm2, %xmm0
@@ -781,7 +781,7 @@ define <8 x i32> @test_mul_v8i32_v8i8_minsize(<8 x i8> %A) minsize {
 ;
 ; SLOW32-LABEL: test_mul_v8i32_v8i8_minsize:
 ; SLOW32:       # %bb.0:
-; SLOW32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SLOW32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SLOW32-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; SLOW32-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SLOW32-NEXT:    movdqa {{.*#+}} xmm2 = [18778,18778,18778,18778]
@@ -791,7 +791,7 @@ define <8 x i32> @test_mul_v8i32_v8i8_minsize(<8 x i8> %A) minsize {
 ;
 ; SLOW64-LABEL: test_mul_v8i32_v8i8_minsize:
 ; SLOW64:       # %bb.0:
-; SLOW64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SLOW64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SLOW64-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; SLOW64-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SLOW64-NEXT:    movdqa {{.*#+}} xmm2 = [18778,18778,18778,18778]
@@ -801,7 +801,7 @@ define <8 x i32> @test_mul_v8i32_v8i8_minsize(<8 x i8> %A) minsize {
 ;
 ; SSE4-32-LABEL: test_mul_v8i32_v8i8_minsize:
 ; SSE4-32:       # %bb.0:
-; SSE4-32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE4-32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE4-32-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; SSE4-32-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE4-32-NEXT:    movdqa {{.*#+}} xmm2 = [18778,18778,18778,18778]
@@ -811,7 +811,7 @@ define <8 x i32> @test_mul_v8i32_v8i8_minsize(<8 x i8> %A) minsize {
 ;
 ; SSE4-64-LABEL: test_mul_v8i32_v8i8_minsize:
 ; SSE4-64:       # %bb.0:
-; SSE4-64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE4-64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE4-64-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; SSE4-64-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE4-64-NEXT:    movdqa {{.*#+}} xmm2 = [18778,18778,18778,18778]
@@ -876,9 +876,9 @@ define <8 x i32> @test_mul_v8i32_v8i8_minsize(<8 x i8> %A) minsize {
 define <16 x i32> @test_mul_v16i32_v16i8_minsize(<16 x i8> %A) minsize {
 ; SLM32-LABEL: test_mul_v16i32_v16i8_minsize:
 ; SLM32:       # %bb.0:
-; SLM32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; SLM32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
 ; SLM32-NEXT:    movdqa {{.*#+}} xmm5 = [18778,18778,18778,18778]
-; SLM32-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,2,3]
+; SLM32-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,1,1]
 ; SLM32-NEXT:    pmovzxbd {{.*#+}} xmm3 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; SLM32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SLM32-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
@@ -892,9 +892,9 @@ define <16 x i32> @test_mul_v16i32_v16i8_minsize(<16 x i8> %A) minsize {
 ;
 ; SLM64-LABEL: test_mul_v16i32_v16i8_minsize:
 ; SLM64:       # %bb.0:
-; SLM64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; SLM64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
 ; SLM64-NEXT:    movdqa {{.*#+}} xmm5 = [18778,18778,18778,18778]
-; SLM64-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,2,3]
+; SLM64-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,1,1]
 ; SLM64-NEXT:    pmovzxbd {{.*#+}} xmm3 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; SLM64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SLM64-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
@@ -908,11 +908,11 @@ define <16 x i32> @test_mul_v16i32_v16i8_minsize(<16 x i8> %A) minsize {
 ;
 ; SLOW32-LABEL: test_mul_v16i32_v16i8_minsize:
 ; SLOW32:       # %bb.0:
-; SLOW32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; SLOW32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
 ; SLOW32-NEXT:    pmovzxbd {{.*#+}} xmm3 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; SLOW32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SLOW32-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
-; SLOW32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SLOW32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SLOW32-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; SLOW32-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SLOW32-NEXT:    movdqa {{.*#+}} xmm4 = [18778,18778,18778,18778]
@@ -924,11 +924,11 @@ define <16 x i32> @test_mul_v16i32_v16i8_minsize(<16 x i8> %A) minsize {
 ;
 ; SLOW64-LABEL: test_mul_v16i32_v16i8_minsize:
 ; SLOW64:       # %bb.0:
-; SLOW64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; SLOW64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
 ; SLOW64-NEXT:    pmovzxbd {{.*#+}} xmm3 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; SLOW64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SLOW64-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
-; SLOW64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SLOW64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SLOW64-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; SLOW64-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SLOW64-NEXT:    movdqa {{.*#+}} xmm4 = [18778,18778,18778,18778]
@@ -940,11 +940,11 @@ define <16 x i32> @test_mul_v16i32_v16i8_minsize(<16 x i8> %A) minsize {
 ;
 ; SSE4-32-LABEL: test_mul_v16i32_v16i8_minsize:
 ; SSE4-32:       # %bb.0:
-; SSE4-32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; SSE4-32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
 ; SSE4-32-NEXT:    pmovzxbd {{.*#+}} xmm3 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; SSE4-32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE4-32-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
-; SSE4-32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE4-32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE4-32-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; SSE4-32-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE4-32-NEXT:    movdqa {{.*#+}} xmm4 = [18778,18778,18778,18778]
@@ -956,11 +956,11 @@ define <16 x i32> @test_mul_v16i32_v16i8_minsize(<16 x i8> %A) minsize {
 ;
 ; SSE4-64-LABEL: test_mul_v16i32_v16i8_minsize:
 ; SSE4-64:       # %bb.0:
-; SSE4-64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; SSE4-64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
 ; SSE4-64-NEXT:    pmovzxbd {{.*#+}} xmm3 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; SSE4-64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE4-64-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
-; SSE4-64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE4-64-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE4-64-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; SSE4-64-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE4-64-NEXT:    movdqa {{.*#+}} xmm4 = [18778,18778,18778,18778]
diff --git a/llvm/test/CodeGen/X86/smul_fix_sat.ll b/llvm/test/CodeGen/X86/smul_fix_sat.ll
index e0c1b762c1507..4d60536636596 100644
--- a/llvm/test/CodeGen/X86/smul_fix_sat.ll
+++ b/llvm/test/CodeGen/X86/smul_fix_sat.ll
@@ -189,10 +189,10 @@ define i4 @func3(i4 %x, i4 %y) nounwind {
 define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; X64-LABEL: vec:
 ; X64:       # %bb.0:
-; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,1,2,3]
+; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
 ; X64-NEXT:    movd %xmm2, %eax
 ; X64-NEXT:    cltq
-; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,1,2,3]
+; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
 ; X64-NEXT:    movd %xmm2, %ecx
 ; X64-NEXT:    movslq %ecx, %rdx
 ; X64-NEXT:    imulq %rax, %rdx
@@ -235,10 +235,10 @@ define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; X64-NEXT:    cmpl $-2, %edx
 ; X64-NEXT:    cmovll %ecx, %esi
 ; X64-NEXT:    movd %esi, %xmm2
-; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
 ; X64-NEXT:    movd %xmm1, %edx
 ; X64-NEXT:    movslq %edx, %rdx
-; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; X64-NEXT:    movd %xmm0, %esi
 ; X64-NEXT:    movslq %esi, %rsi
 ; X64-NEXT:    imulq %rdx, %rsi
@@ -454,9 +454,9 @@ define i4 @func6(i4 %x, i4 %y) nounwind {
 define <4 x i32> @vec2(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; X64-LABEL: vec2:
 ; X64:       # %bb.0:
-; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,2,3]
+; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
 ; X64-NEXT:    movd %xmm2, %ecx
-; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
+; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,1,1]
 ; X64-NEXT:    movd %xmm2, %r8d
 ; X64-NEXT:    movl %r8d, %edx
 ; X64-NEXT:    imull %ecx, %edx
@@ -488,9 +488,9 @@ define <4 x i32> @vec2(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; X64-NEXT:    addl $2147483647, %eax # imm = 0x7FFFFFFF
 ; X64-NEXT:    imull %edx, %esi
 ; X64-NEXT:    cmovol %eax, %esi
-; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; X64-NEXT:    movd %xmm1, %r9d
-; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; X64-NEXT:    movd %xmm0, %edx
 ; X64-NEXT:    movl %edx, %edi
 ; X64-NEXT:    imull %r9d, %edi
diff --git a/llvm/test/CodeGen/X86/split-vector-bitcast.ll b/llvm/test/CodeGen/X86/split-vector-bitcast.ll
index 267a910b2bc7b..c42159433ad61 100644
--- a/llvm/test/CodeGen/X86/split-vector-bitcast.ll
+++ b/llvm/test/CodeGen/X86/split-vector-bitcast.ll
@@ -14,7 +14,7 @@ define void @a(<2 x float>* %a, <2 x i32>* %b) {
 ; CHECK-NEXT:    movlps {{.*#+}} xmm0 = mem[0,1],xmm0[2,3]
 ; CHECK-NEXT:    addps %xmm0, %xmm0
 ; CHECK-NEXT:    movss %xmm0, {{[0-9]+}}(%esp)
-; CHECK-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; CHECK-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; CHECK-NEXT:    movss %xmm0, (%esp)
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; CHECK-NEXT:    movl (%esp), %edx
diff --git a/llvm/test/CodeGen/X86/split-vector-rem.ll b/llvm/test/CodeGen/X86/split-vector-rem.ll
index 959c9bd0ff769..ba8f57b2aaa01 100644
--- a/llvm/test/CodeGen/X86/split-vector-rem.ll
+++ b/llvm/test/CodeGen/X86/split-vector-rem.ll
@@ -5,9 +5,9 @@ define <8 x i32> @foo(<8 x i32> %t, <8 x i32> %u) {
 ; CHECK-LABEL: foo:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movdqa %xmm0, %xmm4
-; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; CHECK-NEXT:    movd %xmm0, %eax
-; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[3,1,2,3]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[3,3,3,3]
 ; CHECK-NEXT:    movd %xmm0, %ecx
 ; CHECK-NEXT:    cltd
 ; CHECK-NEXT:    idivl %ecx
@@ -25,18 +25,18 @@ define <8 x i32> @foo(<8 x i32> %t, <8 x i32> %u) {
 ; CHECK-NEXT:    cltd
 ; CHECK-NEXT:    idivl %ecx
 ; CHECK-NEXT:    movd %edx, %xmm0
-; CHECK-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,2,3]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,1,1]
 ; CHECK-NEXT:    movd %xmm4, %eax
-; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,2,3]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,1,1]
 ; CHECK-NEXT:    movd %xmm2, %ecx
 ; CHECK-NEXT:    cltd
 ; CHECK-NEXT:    idivl %ecx
 ; CHECK-NEXT:    movd %edx, %xmm2
 ; CHECK-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
 ; CHECK-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm5[0]
-; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,1,2,3]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
 ; CHECK-NEXT:    movd %xmm2, %eax
-; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[3,1,2,3]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[3,3,3,3]
 ; CHECK-NEXT:    movd %xmm2, %ecx
 ; CHECK-NEXT:    cltd
 ; CHECK-NEXT:    idivl %ecx
@@ -54,9 +54,9 @@ define <8 x i32> @foo(<8 x i32> %t, <8 x i32> %u) {
 ; CHECK-NEXT:    cltd
 ; CHECK-NEXT:    idivl %ecx
 ; CHECK-NEXT:    movd %edx, %xmm2
-; CHECK-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
 ; CHECK-NEXT:    movd %xmm1, %eax
-; CHECK-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,2,3]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,1,1]
 ; CHECK-NEXT:    movd %xmm1, %ecx
 ; CHECK-NEXT:    cltd
 ; CHECK-NEXT:    idivl %ecx
@@ -72,9 +72,9 @@ define <8 x i32> @bar(<8 x i32> %t, <8 x i32> %u) {
 ; CHECK-LABEL: bar:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movdqa %xmm0, %xmm4
-; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; CHECK-NEXT:    movd %xmm0, %eax
-; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[3,1,2,3]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[3,3,3,3]
 ; CHECK-NEXT:    movd %xmm0, %ecx
 ; CHECK-NEXT:    xorl %edx, %edx
 ; CHECK-NEXT:    divl %ecx
@@ -92,18 +92,18 @@ define <8 x i32> @bar(<8 x i32> %t, <8 x i32> %u) {
 ; CHECK-NEXT:    xorl %edx, %edx
 ; CHECK-NEXT:    divl %ecx
 ; CHECK-NEXT:    movd %edx, %xmm0
-; CHECK-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,2,3]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,1,1]
 ; CHECK-NEXT:    movd %xmm4, %eax
-; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,2,3]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,1,1]
 ; CHECK-NEXT:    movd %xmm2, %ecx
 ; CHECK-NEXT:    xorl %edx, %edx
 ; CHECK-NEXT:    divl %ecx
 ; CHECK-NEXT:    movd %edx, %xmm2
 ; CHECK-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
 ; CHECK-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm5[0]
-; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,1,2,3]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
 ; CHECK-NEXT:    movd %xmm2, %eax
-; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[3,1,2,3]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[3,3,3,3]
 ; CHECK-NEXT:    movd %xmm2, %ecx
 ; CHECK-NEXT:    xorl %edx, %edx
 ; CHECK-NEXT:    divl %ecx
@@ -121,9 +121,9 @@ define <8 x i32> @bar(<8 x i32> %t, <8 x i32> %u) {
 ; CHECK-NEXT:    xorl %edx, %edx
 ; CHECK-NEXT:    divl %ecx
 ; CHECK-NEXT:    movd %edx, %xmm2
-; CHECK-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
 ; CHECK-NEXT:    movd %xmm1, %eax
-; CHECK-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,2,3]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,1,1]
 ; CHECK-NEXT:    movd %xmm1, %ecx
 ; CHECK-NEXT:    xorl %edx, %edx
 ; CHECK-NEXT:    divl %ecx
@@ -144,9 +144,9 @@ define <8 x float> @qux(<8 x float> %t, <8 x float> %u) {
 ; CHECK-NEXT:    movaps %xmm2, (%rsp) # 16-byte Spill
 ; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; CHECK-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; CHECK-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; CHECK-NEXT:    movaps %xmm2, %xmm1
-; CHECK-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1],xmm2[2,3]
+; CHECK-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3],xmm2[3,3]
 ; CHECK-NEXT:    callq fmodf
 ; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
@@ -162,9 +162,9 @@ define <8 x float> @qux(<8 x float> %t, <8 x float> %u) {
 ; CHECK-NEXT:    callq fmodf
 ; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; CHECK-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; CHECK-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; CHECK-NEXT:    movaps (%rsp), %xmm1 # 16-byte Reload
-; CHECK-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; CHECK-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,1,1]
 ; CHECK-NEXT:    callq fmodf
 ; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
 ; CHECK-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
@@ -172,9 +172,9 @@ define <8 x float> @qux(<8 x float> %t, <8 x float> %u) {
 ; CHECK-NEXT:    # xmm1 = xmm1[0],mem[0]
 ; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; CHECK-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; CHECK-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; CHECK-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; CHECK-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; CHECK-NEXT:    callq fmodf
 ; CHECK-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
 ; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
@@ -190,9 +190,9 @@ define <8 x float> @qux(<8 x float> %t, <8 x float> %u) {
 ; CHECK-NEXT:    callq fmodf
 ; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; CHECK-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; CHECK-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; CHECK-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; CHECK-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,1,1]
 ; CHECK-NEXT:    callq fmodf
 ; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
 ; CHECK-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
diff --git a/llvm/test/CodeGen/X86/sse1.ll b/llvm/test/CodeGen/X86/sse1.ll
index 482062bc62520..e2b95eb7b93ab 100644
--- a/llvm/test/CodeGen/X86/sse1.ll
+++ b/llvm/test/CodeGen/X86/sse1.ll
@@ -17,9 +17,9 @@ define <2 x float> @test4(<2 x float> %A, <2 x float> %B) nounwind {
 ; CHECK-LABEL: test4:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    movaps %xmm0, %xmm2
-; CHECK-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1],xmm0[2,3]
+; CHECK-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1],xmm0[1,1]
 ; CHECK-NEXT:    addss %xmm1, %xmm0
-; CHECK-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; CHECK-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,1,1]
 ; CHECK-NEXT:    subss %xmm1, %xmm2
 ; CHECK-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
 ; CHECK-NEXT:    ret{{[l|q]}}
diff --git a/llvm/test/CodeGen/X86/sse2-intrinsics-fast-isel.ll b/llvm/test/CodeGen/X86/sse2-intrinsics-fast-isel.ll
index af591a6849f3e..e233bf5be8cfa 100644
--- a/llvm/test/CodeGen/X86/sse2-intrinsics-fast-isel.ll
+++ b/llvm/test/CodeGen/X86/sse2-intrinsics-fast-isel.ll
@@ -3758,8 +3758,8 @@ define <2 x i64> @test_mm_set1_epi8(i8 %a0) nounwind {
 ; X86-SSE-NEXT:    movd %eax, %xmm0 # encoding: [0x66,0x0f,0x6e,0xc0]
 ; X86-SSE-NEXT:    punpcklbw %xmm0, %xmm0 # encoding: [0x66,0x0f,0x60,0xc0]
 ; X86-SSE-NEXT:    # xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X86-SSE-NEXT:    pshuflw $224, %xmm0, %xmm0 # encoding: [0xf2,0x0f,0x70,0xc0,0xe0]
-; X86-SSE-NEXT:    # xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; X86-SSE-NEXT:    pshuflw $0, %xmm0, %xmm0 # encoding: [0xf2,0x0f,0x70,0xc0,0x00]
+; X86-SSE-NEXT:    # xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; X86-SSE-NEXT:    pshufd $0, %xmm0, %xmm0 # encoding: [0x66,0x0f,0x70,0xc0,0x00]
 ; X86-SSE-NEXT:    # xmm0 = xmm0[0,0,0,0]
 ; X86-SSE-NEXT:    retl # encoding: [0xc3]
@@ -3784,8 +3784,8 @@ define <2 x i64> @test_mm_set1_epi8(i8 %a0) nounwind {
 ; X64-SSE-NEXT:    movd %eax, %xmm0 # encoding: [0x66,0x0f,0x6e,0xc0]
 ; X64-SSE-NEXT:    punpcklbw %xmm0, %xmm0 # encoding: [0x66,0x0f,0x60,0xc0]
 ; X64-SSE-NEXT:    # xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X64-SSE-NEXT:    pshuflw $224, %xmm0, %xmm0 # encoding: [0xf2,0x0f,0x70,0xc0,0xe0]
-; X64-SSE-NEXT:    # xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; X64-SSE-NEXT:    pshuflw $0, %xmm0, %xmm0 # encoding: [0xf2,0x0f,0x70,0xc0,0x00]
+; X64-SSE-NEXT:    # xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; X64-SSE-NEXT:    pshufd $0, %xmm0, %xmm0 # encoding: [0x66,0x0f,0x70,0xc0,0x00]
 ; X64-SSE-NEXT:    # xmm0 = xmm0[0,0,0,0]
 ; X64-SSE-NEXT:    retq # encoding: [0xc3]
@@ -3827,8 +3827,8 @@ define <2 x i64> @test_mm_set1_epi16(i16 %a0) nounwind {
 ; X86-SSE:       # %bb.0:
 ; X86-SSE-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb7,0x44,0x24,0x04]
 ; X86-SSE-NEXT:    movd %eax, %xmm0 # encoding: [0x66,0x0f,0x6e,0xc0]
-; X86-SSE-NEXT:    pshuflw $224, %xmm0, %xmm0 # encoding: [0xf2,0x0f,0x70,0xc0,0xe0]
-; X86-SSE-NEXT:    # xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; X86-SSE-NEXT:    pshuflw $0, %xmm0, %xmm0 # encoding: [0xf2,0x0f,0x70,0xc0,0x00]
+; X86-SSE-NEXT:    # xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; X86-SSE-NEXT:    pshufd $0, %xmm0, %xmm0 # encoding: [0x66,0x0f,0x70,0xc0,0x00]
 ; X86-SSE-NEXT:    # xmm0 = xmm0[0,0,0,0]
 ; X86-SSE-NEXT:    retl # encoding: [0xc3]
@@ -3837,8 +3837,8 @@ define <2 x i64> @test_mm_set1_epi16(i16 %a0) nounwind {
 ; X86-AVX1:       # %bb.0:
 ; X86-AVX1-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb7,0x44,0x24,0x04]
 ; X86-AVX1-NEXT:    vmovd %eax, %xmm0 # encoding: [0xc5,0xf9,0x6e,0xc0]
-; X86-AVX1-NEXT:    vpshuflw $224, %xmm0, %xmm0 # encoding: [0xc5,0xfb,0x70,0xc0,0xe0]
-; X86-AVX1-NEXT:    # xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; X86-AVX1-NEXT:    vpshuflw $0, %xmm0, %xmm0 # encoding: [0xc5,0xfb,0x70,0xc0,0x00]
+; X86-AVX1-NEXT:    # xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; X86-AVX1-NEXT:    vpshufd $0, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0x70,0xc0,0x00]
 ; X86-AVX1-NEXT:    # xmm0 = xmm0[0,0,0,0]
 ; X86-AVX1-NEXT:    retl # encoding: [0xc3]
@@ -3852,8 +3852,8 @@ define <2 x i64> @test_mm_set1_epi16(i16 %a0) nounwind {
 ; X64-SSE-LABEL: test_mm_set1_epi16:
 ; X64-SSE:       # %bb.0:
 ; X64-SSE-NEXT:    movd %edi, %xmm0 # encoding: [0x66,0x0f,0x6e,0xc7]
-; X64-SSE-NEXT:    pshuflw $224, %xmm0, %xmm0 # encoding: [0xf2,0x0f,0x70,0xc0,0xe0]
-; X64-SSE-NEXT:    # xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; X64-SSE-NEXT:    pshuflw $0, %xmm0, %xmm0 # encoding: [0xf2,0x0f,0x70,0xc0,0x00]
+; X64-SSE-NEXT:    # xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; X64-SSE-NEXT:    pshufd $0, %xmm0, %xmm0 # encoding: [0x66,0x0f,0x70,0xc0,0x00]
 ; X64-SSE-NEXT:    # xmm0 = xmm0[0,0,0,0]
 ; X64-SSE-NEXT:    retq # encoding: [0xc3]
@@ -3861,8 +3861,8 @@ define <2 x i64> @test_mm_set1_epi16(i16 %a0) nounwind {
 ; X64-AVX1-LABEL: test_mm_set1_epi16:
 ; X64-AVX1:       # %bb.0:
 ; X64-AVX1-NEXT:    vmovd %edi, %xmm0 # encoding: [0xc5,0xf9,0x6e,0xc7]
-; X64-AVX1-NEXT:    vpshuflw $224, %xmm0, %xmm0 # encoding: [0xc5,0xfb,0x70,0xc0,0xe0]
-; X64-AVX1-NEXT:    # xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; X64-AVX1-NEXT:    vpshuflw $0, %xmm0, %xmm0 # encoding: [0xc5,0xfb,0x70,0xc0,0x00]
+; X64-AVX1-NEXT:    # xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; X64-AVX1-NEXT:    vpshufd $0, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0x70,0xc0,0x00]
 ; X64-AVX1-NEXT:    # xmm0 = xmm0[0,0,0,0]
 ; X64-AVX1-NEXT:    retq # encoding: [0xc3]
diff --git a/llvm/test/CodeGen/X86/sse3-avx-addsub-2.ll b/llvm/test/CodeGen/X86/sse3-avx-addsub-2.ll
index c43f293f5aadd..e26b26f1dfa5d 100644
--- a/llvm/test/CodeGen/X86/sse3-avx-addsub-2.ll
+++ b/llvm/test/CodeGen/X86/sse3-avx-addsub-2.ll
@@ -330,25 +330,25 @@ define <4 x float> @test12(<4 x float> %A, <4 x float> %B) {
 define <4 x float> @test13(<4 x float> %A, <4 x float> %B) {
 ; SSE-LABEL: test13:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
-; SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; SSE-NEXT:    addss %xmm0, %xmm1
-; SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,1,2,0]
+; SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; SSE-NEXT:    movaps %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: test13:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; AVX1-NEXT:    vaddss %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,1,2,0]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    retq
 ;
 ; AVX512-LABEL: test13:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
-; AVX512-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; AVX512-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; AVX512-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; AVX512-NEXT:    vaddss %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vbroadcastss %xmm0, %xmm0
 ; AVX512-NEXT:    retq
@@ -396,8 +396,8 @@ define <4 x float> @test15(<4 x float> %A, <4 x float> %B) {
 ; SSE-NEXT:    movshdup {{.*#+}} xmm3 = xmm0[1,1,3,3]
 ; SSE-NEXT:    movshdup {{.*#+}} xmm2 = xmm1[1,1,3,3]
 ; SSE-NEXT:    addss %xmm3, %xmm2
-; SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
-; SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; SSE-NEXT:    addss %xmm0, %xmm1
 ; SSE-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,0],xmm1[0,0]
 ; SSE-NEXT:    movaps %xmm2, %xmm0
@@ -408,8 +408,8 @@ define <4 x float> @test15(<4 x float> %A, <4 x float> %B) {
 ; AVX1-NEXT:    vmovshdup {{.*#+}} xmm2 = xmm0[1,1,3,3]
 ; AVX1-NEXT:    vmovshdup {{.*#+}} xmm3 = xmm1[1,1,3,3]
 ; AVX1-NEXT:    vaddss %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; AVX1-NEXT:    vaddss %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovsldup {{.*#+}} xmm1 = xmm2[0,0,2,2]
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
@@ -420,8 +420,8 @@ define <4 x float> @test15(<4 x float> %A, <4 x float> %B) {
 ; AVX512-NEXT:    vmovshdup {{.*#+}} xmm2 = xmm0[1,1,3,3]
 ; AVX512-NEXT:    vmovshdup {{.*#+}} xmm3 = xmm1[1,1,3,3]
 ; AVX512-NEXT:    vaddss %xmm3, %xmm2, %xmm2
-; AVX512-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
-; AVX512-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; AVX512-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; AVX512-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; AVX512-NEXT:    vaddss %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vbroadcastss %xmm2, %xmm1
 ; AVX512-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
@@ -451,8 +451,8 @@ define <4 x float> @test16(<4 x float> %A, <4 x float> %B) {
 ; SSE-NEXT:    movshdup {{.*#+}} xmm5 = xmm0[1,1,3,3]
 ; SSE-NEXT:    addss %xmm3, %xmm5
 ; SSE-NEXT:    unpcklps {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
-; SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
-; SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; SSE-NEXT:    addss %xmm0, %xmm1
 ; SSE-NEXT:    unpcklps {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
 ; SSE-NEXT:    movlhps {{.*#+}} xmm2 = xmm2[0],xmm4[0]
@@ -470,8 +470,8 @@ define <4 x float> @test16(<4 x float> %A, <4 x float> %B) {
 ; AVX-NEXT:    vaddss %xmm2, %xmm5, %xmm2
 ; AVX-NEXT:    vinsertps {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[2,3]
 ; AVX-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1],xmm4[0],xmm2[3]
-; AVX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
-; AVX-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; AVX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; AVX-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; AVX-NEXT:    vaddss %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[0]
 ; AVX-NEXT:    retq
@@ -541,8 +541,8 @@ define <16 x float> @test17(<16 x float> %A, <16 x float> %B) {
 ; AVX512-NEXT:    vaddss %xmm5, %xmm4, %xmm4
 ; AVX512-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[2,3]
 ; AVX512-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1],xmm3[0],xmm2[3]
-; AVX512-NEXT:    vpermilps {{.*#+}} xmm3 = xmm0[3,1,2,3]
-; AVX512-NEXT:    vpermilps {{.*#+}} xmm4 = xmm1[3,1,2,3]
+; AVX512-NEXT:    vpermilps {{.*#+}} xmm3 = xmm0[3,3,3,3]
+; AVX512-NEXT:    vpermilps {{.*#+}} xmm4 = xmm1[3,3,3,3]
 ; AVX512-NEXT:    vaddss %xmm4, %xmm3, %xmm3
 ; AVX512-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],xmm3[0]
 ; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm0
@@ -556,8 +556,8 @@ define <16 x float> @test17(<16 x float> %A, <16 x float> %B) {
 ; AVX512-NEXT:    vaddss %xmm6, %xmm5, %xmm5
 ; AVX512-NEXT:    vinsertps {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[2,3]
 ; AVX512-NEXT:    vinsertps {{.*#+}} xmm3 = xmm3[0,1],xmm4[0],xmm3[3]
-; AVX512-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
-; AVX512-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; AVX512-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; AVX512-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; AVX512-NEXT:    vaddss %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vinsertps {{.*#+}} xmm0 = xmm3[0,1,2],xmm0[0]
 ; AVX512-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
diff --git a/llvm/test/CodeGen/X86/sse3.ll b/llvm/test/CodeGen/X86/sse3.ll
index 09a0b865356a9..a1951442efad4 100644
--- a/llvm/test/CodeGen/X86/sse3.ll
+++ b/llvm/test/CodeGen/X86/sse3.ll
@@ -66,7 +66,7 @@ define <8 x i16> @t2(<8 x i16> %A, <8 x i16> %B) nounwind {
 ; X86:       # %bb.0:
 ; X86-NEXT:    movdqa {{.*#+}} xmm2 = [0,65535,65535,0,65535,65535,65535,65535]
 ; X86-NEXT:    pand %xmm2, %xmm0
-; X86-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[1,1,2,1,4,5,6,7]
+; X86-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[1,1,1,1,4,5,6,7]
 ; X86-NEXT:    pandn %xmm1, %xmm2
 ; X86-NEXT:    por %xmm2, %xmm0
 ; X86-NEXT:    retl
@@ -75,7 +75,7 @@ define <8 x i16> @t2(<8 x i16> %A, <8 x i16> %B) nounwind {
 ; X64:       # %bb.0:
 ; X64-NEXT:    movdqa {{.*#+}} xmm2 = [0,65535,65535,0,65535,65535,65535,65535]
 ; X64-NEXT:    pand %xmm2, %xmm0
-; X64-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[1,1,2,1,4,5,6,7]
+; X64-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[1,1,1,1,4,5,6,7]
 ; X64-NEXT:    pandn %xmm1, %xmm2
 ; X64-NEXT:    por %xmm2, %xmm0
 ; X64-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/sse41.ll b/llvm/test/CodeGen/X86/sse41.ll
index e2973ebbab89e..99cd686f50bd5 100644
--- a/llvm/test/CodeGen/X86/sse41.ll
+++ b/llvm/test/CodeGen/X86/sse41.ll
@@ -182,8 +182,8 @@ define float @ext_1(<4 x float> %v) nounwind {
 ; X86-SSE-LABEL: ext_1:
 ; X86-SSE:       ## %bb.0:
 ; X86-SSE-NEXT:    pushl %eax ## encoding: [0x50]
-; X86-SSE-NEXT:    shufps $231, %xmm0, %xmm0 ## encoding: [0x0f,0xc6,0xc0,0xe7]
-; X86-SSE-NEXT:    ## xmm0 = xmm0[3,1,2,3]
+; X86-SSE-NEXT:    shufps $255, %xmm0, %xmm0 ## encoding: [0x0f,0xc6,0xc0,0xff]
+; X86-SSE-NEXT:    ## xmm0 = xmm0[3,3,3,3]
 ; X86-SSE-NEXT:    addss LCPI5_0, %xmm0 ## encoding: [0xf3,0x0f,0x58,0x05,A,A,A,A]
 ; X86-SSE-NEXT:    ## fixup A - offset: 4, value: LCPI5_0, kind: FK_Data_4
 ; X86-SSE-NEXT:    movss %xmm0, (%esp) ## encoding: [0xf3,0x0f,0x11,0x04,0x24]
@@ -194,8 +194,8 @@ define float @ext_1(<4 x float> %v) nounwind {
 ; X86-AVX1-LABEL: ext_1:
 ; X86-AVX1:       ## %bb.0:
 ; X86-AVX1-NEXT:    pushl %eax ## encoding: [0x50]
-; X86-AVX1-NEXT:    vpermilps $231, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x04,0xc0,0xe7]
-; X86-AVX1-NEXT:    ## xmm0 = xmm0[3,1,2,3]
+; X86-AVX1-NEXT:    vpermilps $255, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x04,0xc0,0xff]
+; X86-AVX1-NEXT:    ## xmm0 = xmm0[3,3,3,3]
 ; X86-AVX1-NEXT:    vaddss LCPI5_0, %xmm0, %xmm0 ## encoding: [0xc5,0xfa,0x58,0x05,A,A,A,A]
 ; X86-AVX1-NEXT:    ## fixup A - offset: 4, value: LCPI5_0, kind: FK_Data_4
 ; X86-AVX1-NEXT:    vmovss %xmm0, (%esp) ## encoding: [0xc5,0xfa,0x11,0x04,0x24]
@@ -206,8 +206,8 @@ define float @ext_1(<4 x float> %v) nounwind {
 ; X86-AVX512-LABEL: ext_1:
 ; X86-AVX512:       ## %bb.0:
 ; X86-AVX512-NEXT:    pushl %eax ## encoding: [0x50]
-; X86-AVX512-NEXT:    vpermilps $231, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x04,0xc0,0xe7]
-; X86-AVX512-NEXT:    ## xmm0 = xmm0[3,1,2,3]
+; X86-AVX512-NEXT:    vpermilps $255, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x04,0xc0,0xff]
+; X86-AVX512-NEXT:    ## xmm0 = xmm0[3,3,3,3]
 ; X86-AVX512-NEXT:    vaddss LCPI5_0, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfa,0x58,0x05,A,A,A,A]
 ; X86-AVX512-NEXT:    ## fixup A - offset: 4, value: LCPI5_0, kind: FK_Data_4
 ; X86-AVX512-NEXT:    vmovss %xmm0, (%esp) ## EVEX TO VEX Compression encoding: [0xc5,0xfa,0x11,0x04,0x24]
@@ -217,24 +217,24 @@ define float @ext_1(<4 x float> %v) nounwind {
 ;
 ; X64-SSE-LABEL: ext_1:
 ; X64-SSE:       ## %bb.0:
-; X64-SSE-NEXT:    shufps $231, %xmm0, %xmm0 ## encoding: [0x0f,0xc6,0xc0,0xe7]
-; X64-SSE-NEXT:    ## xmm0 = xmm0[3,1,2,3]
+; X64-SSE-NEXT:    shufps $255, %xmm0, %xmm0 ## encoding: [0x0f,0xc6,0xc0,0xff]
+; X64-SSE-NEXT:    ## xmm0 = xmm0[3,3,3,3]
 ; X64-SSE-NEXT:    addss {{.*}}(%rip), %xmm0 ## encoding: [0xf3,0x0f,0x58,0x05,A,A,A,A]
 ; X64-SSE-NEXT:    ## fixup A - offset: 4, value: LCPI5_0-4, kind: reloc_riprel_4byte
 ; X64-SSE-NEXT:    retq ## encoding: [0xc3]
 ;
 ; X64-AVX1-LABEL: ext_1:
 ; X64-AVX1:       ## %bb.0:
-; X64-AVX1-NEXT:    vpermilps $231, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x04,0xc0,0xe7]
-; X64-AVX1-NEXT:    ## xmm0 = xmm0[3,1,2,3]
+; X64-AVX1-NEXT:    vpermilps $255, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x04,0xc0,0xff]
+; X64-AVX1-NEXT:    ## xmm0 = xmm0[3,3,3,3]
 ; X64-AVX1-NEXT:    vaddss {{.*}}(%rip), %xmm0, %xmm0 ## encoding: [0xc5,0xfa,0x58,0x05,A,A,A,A]
 ; X64-AVX1-NEXT:    ## fixup A - offset: 4, value: LCPI5_0-4, kind: reloc_riprel_4byte
 ; X64-AVX1-NEXT:    retq ## encoding: [0xc3]
 ;
 ; X64-AVX512-LABEL: ext_1:
 ; X64-AVX512:       ## %bb.0:
-; X64-AVX512-NEXT:    vpermilps $231, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x04,0xc0,0xe7]
-; X64-AVX512-NEXT:    ## xmm0 = xmm0[3,1,2,3]
+; X64-AVX512-NEXT:    vpermilps $255, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x04,0xc0,0xff]
+; X64-AVX512-NEXT:    ## xmm0 = xmm0[3,3,3,3]
 ; X64-AVX512-NEXT:    vaddss {{.*}}(%rip), %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfa,0x58,0x05,A,A,A,A]
 ; X64-AVX512-NEXT:    ## fixup A - offset: 4, value: LCPI5_0-4, kind: reloc_riprel_4byte
 ; X64-AVX512-NEXT:    retq ## encoding: [0xc3]
@@ -247,8 +247,8 @@ define float @ext_2(<4 x float> %v) nounwind {
 ; X86-SSE-LABEL: ext_2:
 ; X86-SSE:       ## %bb.0:
 ; X86-SSE-NEXT:    pushl %eax ## encoding: [0x50]
-; X86-SSE-NEXT:    shufps $231, %xmm0, %xmm0 ## encoding: [0x0f,0xc6,0xc0,0xe7]
-; X86-SSE-NEXT:    ## xmm0 = xmm0[3,1,2,3]
+; X86-SSE-NEXT:    shufps $255, %xmm0, %xmm0 ## encoding: [0x0f,0xc6,0xc0,0xff]
+; X86-SSE-NEXT:    ## xmm0 = xmm0[3,3,3,3]
 ; X86-SSE-NEXT:    movss %xmm0, (%esp) ## encoding: [0xf3,0x0f,0x11,0x04,0x24]
 ; X86-SSE-NEXT:    flds (%esp) ## encoding: [0xd9,0x04,0x24]
 ; X86-SSE-NEXT:    popl %eax ## encoding: [0x58]
@@ -257,8 +257,8 @@ define float @ext_2(<4 x float> %v) nounwind {
 ; X86-AVX1-LABEL: ext_2:
 ; X86-AVX1:       ## %bb.0:
 ; X86-AVX1-NEXT:    pushl %eax ## encoding: [0x50]
-; X86-AVX1-NEXT:    vpermilps $231, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x04,0xc0,0xe7]
-; X86-AVX1-NEXT:    ## xmm0 = xmm0[3,1,2,3]
+; X86-AVX1-NEXT:    vpermilps $255, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x04,0xc0,0xff]
+; X86-AVX1-NEXT:    ## xmm0 = xmm0[3,3,3,3]
 ; X86-AVX1-NEXT:    vmovss %xmm0, (%esp) ## encoding: [0xc5,0xfa,0x11,0x04,0x24]
 ; X86-AVX1-NEXT:    flds (%esp) ## encoding: [0xd9,0x04,0x24]
 ; X86-AVX1-NEXT:    popl %eax ## encoding: [0x58]
@@ -267,8 +267,8 @@ define float @ext_2(<4 x float> %v) nounwind {
 ; X86-AVX512-LABEL: ext_2:
 ; X86-AVX512:       ## %bb.0:
 ; X86-AVX512-NEXT:    pushl %eax ## encoding: [0x50]
-; X86-AVX512-NEXT:    vpermilps $231, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x04,0xc0,0xe7]
-; X86-AVX512-NEXT:    ## xmm0 = xmm0[3,1,2,3]
+; X86-AVX512-NEXT:    vpermilps $255, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x04,0xc0,0xff]
+; X86-AVX512-NEXT:    ## xmm0 = xmm0[3,3,3,3]
 ; X86-AVX512-NEXT:    vmovss %xmm0, (%esp) ## EVEX TO VEX Compression encoding: [0xc5,0xfa,0x11,0x04,0x24]
 ; X86-AVX512-NEXT:    flds (%esp) ## encoding: [0xd9,0x04,0x24]
 ; X86-AVX512-NEXT:    popl %eax ## encoding: [0x58]
@@ -276,20 +276,20 @@ define float @ext_2(<4 x float> %v) nounwind {
 ;
 ; X64-SSE-LABEL: ext_2:
 ; X64-SSE:       ## %bb.0:
-; X64-SSE-NEXT:    shufps $231, %xmm0, %xmm0 ## encoding: [0x0f,0xc6,0xc0,0xe7]
-; X64-SSE-NEXT:    ## xmm0 = xmm0[3,1,2,3]
+; X64-SSE-NEXT:    shufps $255, %xmm0, %xmm0 ## encoding: [0x0f,0xc6,0xc0,0xff]
+; X64-SSE-NEXT:    ## xmm0 = xmm0[3,3,3,3]
 ; X64-SSE-NEXT:    retq ## encoding: [0xc3]
 ;
 ; X64-AVX1-LABEL: ext_2:
 ; X64-AVX1:       ## %bb.0:
-; X64-AVX1-NEXT:    vpermilps $231, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x04,0xc0,0xe7]
-; X64-AVX1-NEXT:    ## xmm0 = xmm0[3,1,2,3]
+; X64-AVX1-NEXT:    vpermilps $255, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x04,0xc0,0xff]
+; X64-AVX1-NEXT:    ## xmm0 = xmm0[3,3,3,3]
 ; X64-AVX1-NEXT:    retq ## encoding: [0xc3]
 ;
 ; X64-AVX512-LABEL: ext_2:
 ; X64-AVX512:       ## %bb.0:
-; X64-AVX512-NEXT:    vpermilps $231, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x04,0xc0,0xe7]
-; X64-AVX512-NEXT:    ## xmm0 = xmm0[3,1,2,3]
+; X64-AVX512-NEXT:    vpermilps $255, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x04,0xc0,0xff]
+; X64-AVX512-NEXT:    ## xmm0 = xmm0[3,3,3,3]
 ; X64-AVX512-NEXT:    retq ## encoding: [0xc3]
   %s = extractelement <4 x float> %v, i32 3
   ret float %s
@@ -636,8 +636,8 @@ define <4 x i32> @pinsrd_from_shufflevector_i32(<4 x i32> %a, <4 x i32>* nocaptu
 ; X86-SSE-LABEL: pinsrd_from_shufflevector_i32:
 ; X86-SSE:       ## %bb.0: ## %entry
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x04]
-; X86-SSE-NEXT:    pshufd $36, (%eax), %xmm1 ## encoding: [0x66,0x0f,0x70,0x08,0x24]
-; X86-SSE-NEXT:    ## xmm1 = mem[0,1,2,0]
+; X86-SSE-NEXT:    pshufd $0, (%eax), %xmm1 ## encoding: [0x66,0x0f,0x70,0x08,0x00]
+; X86-SSE-NEXT:    ## xmm1 = mem[0,0,0,0]
 ; X86-SSE-NEXT:    pblendw $192, %xmm1, %xmm0 ## encoding: [0x66,0x0f,0x3a,0x0e,0xc1,0xc0]
 ; X86-SSE-NEXT:    ## xmm0 = xmm0[0,1,2,3,4,5],xmm1[6,7]
 ; X86-SSE-NEXT:    retl ## encoding: [0xc3]
@@ -645,8 +645,8 @@ define <4 x i32> @pinsrd_from_shufflevector_i32(<4 x i32> %a, <4 x i32>* nocaptu
 ; X86-AVX1-LABEL: pinsrd_from_shufflevector_i32:
 ; X86-AVX1:       ## %bb.0: ## %entry
 ; X86-AVX1-NEXT:    movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x04]
-; X86-AVX1-NEXT:    vpermilps $36, (%eax), %xmm1 ## encoding: [0xc4,0xe3,0x79,0x04,0x08,0x24]
-; X86-AVX1-NEXT:    ## xmm1 = mem[0,1,2,0]
+; X86-AVX1-NEXT:    vpermilps $0, (%eax), %xmm1 ## encoding: [0xc4,0xe3,0x79,0x04,0x08,0x00]
+; X86-AVX1-NEXT:    ## xmm1 = mem[0,0,0,0]
 ; X86-AVX1-NEXT:    vblendps $8, %xmm1, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x0c,0xc1,0x08]
 ; X86-AVX1-NEXT:    ## xmm0 = xmm0[0,1,2],xmm1[3]
 ; X86-AVX1-NEXT:    retl ## encoding: [0xc3]
@@ -654,32 +654,32 @@ define <4 x i32> @pinsrd_from_shufflevector_i32(<4 x i32> %a, <4 x i32>* nocaptu
 ; X86-AVX512-LABEL: pinsrd_from_shufflevector_i32:
 ; X86-AVX512:       ## %bb.0: ## %entry
 ; X86-AVX512-NEXT:    movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x04]
-; X86-AVX512-NEXT:    vpermilps $36, (%eax), %xmm1 ## EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x04,0x08,0x24]
-; X86-AVX512-NEXT:    ## xmm1 = mem[0,1,2,0]
+; X86-AVX512-NEXT:    vmovaps (%eax), %xmm1 ## EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0x08]
+; X86-AVX512-NEXT:    vbroadcastss %xmm1, %xmm1 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x18,0xc9]
 ; X86-AVX512-NEXT:    vblendps $8, %xmm1, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x0c,0xc1,0x08]
 ; X86-AVX512-NEXT:    ## xmm0 = xmm0[0,1,2],xmm1[3]
 ; X86-AVX512-NEXT:    retl ## encoding: [0xc3]
 ;
 ; X64-SSE-LABEL: pinsrd_from_shufflevector_i32:
 ; X64-SSE:       ## %bb.0: ## %entry
-; X64-SSE-NEXT:    pshufd $36, (%rdi), %xmm1 ## encoding: [0x66,0x0f,0x70,0x0f,0x24]
-; X64-SSE-NEXT:    ## xmm1 = mem[0,1,2,0]
+; X64-SSE-NEXT:    pshufd $0, (%rdi), %xmm1 ## encoding: [0x66,0x0f,0x70,0x0f,0x00]
+; X64-SSE-NEXT:    ## xmm1 = mem[0,0,0,0]
 ; X64-SSE-NEXT:    pblendw $192, %xmm1, %xmm0 ## encoding: [0x66,0x0f,0x3a,0x0e,0xc1,0xc0]
 ; X64-SSE-NEXT:    ## xmm0 = xmm0[0,1,2,3,4,5],xmm1[6,7]
 ; X64-SSE-NEXT:    retq ## encoding: [0xc3]
 ;
 ; X64-AVX1-LABEL: pinsrd_from_shufflevector_i32:
 ; X64-AVX1:       ## %bb.0: ## %entry
-; X64-AVX1-NEXT:    vpermilps $36, (%rdi), %xmm1 ## encoding: [0xc4,0xe3,0x79,0x04,0x0f,0x24]
-; X64-AVX1-NEXT:    ## xmm1 = mem[0,1,2,0]
+; X64-AVX1-NEXT:    vpermilps $0, (%rdi), %xmm1 ## encoding: [0xc4,0xe3,0x79,0x04,0x0f,0x00]
+; X64-AVX1-NEXT:    ## xmm1 = mem[0,0,0,0]
 ; X64-AVX1-NEXT:    vblendps $8, %xmm1, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x0c,0xc1,0x08]
 ; X64-AVX1-NEXT:    ## xmm0 = xmm0[0,1,2],xmm1[3]
 ; X64-AVX1-NEXT:    retq ## encoding: [0xc3]
 ;
 ; X64-AVX512-LABEL: pinsrd_from_shufflevector_i32:
 ; X64-AVX512:       ## %bb.0: ## %entry
-; X64-AVX512-NEXT:    vpermilps $36, (%rdi), %xmm1 ## EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x04,0x0f,0x24]
-; X64-AVX512-NEXT:    ## xmm1 = mem[0,1,2,0]
+; X64-AVX512-NEXT:    vmovaps (%rdi), %xmm1 ## EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0x0f]
+; X64-AVX512-NEXT:    vbroadcastss %xmm1, %xmm1 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x18,0xc9]
 ; X64-AVX512-NEXT:    vblendps $8, %xmm1, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x0c,0xc1,0x08]
 ; X64-AVX512-NEXT:    ## xmm0 = xmm0[0,1,2],xmm1[3]
 ; X64-AVX512-NEXT:    retq ## encoding: [0xc3]
@@ -1164,8 +1164,8 @@ define <4 x i32> @i32_shuf_XYW0(<4 x i32> %x, <4 x i32> %a) {
 define <4 x i32> @i32_shuf_W00W(<4 x i32> %x, <4 x i32> %a) {
 ; SSE-LABEL: i32_shuf_W00W:
 ; SSE:       ## %bb.0:
-; SSE-NEXT:    pshufd $231, %xmm0, %xmm1 ## encoding: [0x66,0x0f,0x70,0xc8,0xe7]
-; SSE-NEXT:    ## xmm1 = xmm0[3,1,2,3]
+; SSE-NEXT:    pshufd $255, %xmm0, %xmm1 ## encoding: [0x66,0x0f,0x70,0xc8,0xff]
+; SSE-NEXT:    ## xmm1 = xmm0[3,3,3,3]
 ; SSE-NEXT:    pxor %xmm0, %xmm0 ## encoding: [0x66,0x0f,0xef,0xc0]
 ; SSE-NEXT:    pblendw $195, %xmm1, %xmm0 ## encoding: [0x66,0x0f,0x3a,0x0e,0xc1,0xc3]
 ; SSE-NEXT:    ## xmm0 = xmm1[0,1],xmm0[2,3,4,5],xmm1[6,7]
@@ -1173,8 +1173,8 @@ define <4 x i32> @i32_shuf_W00W(<4 x i32> %x, <4 x i32> %a) {
 ;
 ; AVX1-LABEL: i32_shuf_W00W:
 ; AVX1:       ## %bb.0:
-; AVX1-NEXT:    vpermilps $231, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x04,0xc0,0xe7]
-; AVX1-NEXT:    ## xmm0 = xmm0[3,1,2,3]
+; AVX1-NEXT:    vpermilps $255, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x04,0xc0,0xff]
+; AVX1-NEXT:    ## xmm0 = xmm0[3,3,3,3]
 ; AVX1-NEXT:    vxorps %xmm1, %xmm1, %xmm1 ## encoding: [0xc5,0xf0,0x57,0xc9]
 ; AVX1-NEXT:    vblendps $6, %xmm1, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x0c,0xc1,0x06]
 ; AVX1-NEXT:    ## xmm0 = xmm0[0],xmm1[1,2],xmm0[3]
@@ -1182,8 +1182,8 @@ define <4 x i32> @i32_shuf_W00W(<4 x i32> %x, <4 x i32> %a) {
 ;
 ; AVX512-LABEL: i32_shuf_W00W:
 ; AVX512:       ## %bb.0:
-; AVX512-NEXT:    vpermilps $231, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x04,0xc0,0xe7]
-; AVX512-NEXT:    ## xmm0 = xmm0[3,1,2,3]
+; AVX512-NEXT:    vpermilps $255, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x04,0xc0,0xff]
+; AVX512-NEXT:    ## xmm0 = xmm0[3,3,3,3]
 ; AVX512-NEXT:    vxorps %xmm1, %xmm1, %xmm1 ## EVEX TO VEX Compression encoding: [0xc5,0xf0,0x57,0xc9]
 ; AVX512-NEXT:    vblendps $6, %xmm1, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x0c,0xc1,0x06]
 ; AVX512-NEXT:    ## xmm0 = xmm0[0],xmm1[1,2],xmm0[3]
@@ -1202,8 +1202,8 @@ define <4 x i32> @i32_shuf_X00A(<4 x i32> %x, <4 x i32> %a) {
 ; SSE-NEXT:    pxor %xmm2, %xmm2 ## encoding: [0x66,0x0f,0xef,0xd2]
 ; SSE-NEXT:    pblendw $252, %xmm2, %xmm0 ## encoding: [0x66,0x0f,0x3a,0x0e,0xc2,0xfc]
 ; SSE-NEXT:    ## xmm0 = xmm0[0,1],xmm2[2,3,4,5,6,7]
-; SSE-NEXT:    pshufd $36, %xmm1, %xmm1 ## encoding: [0x66,0x0f,0x70,0xc9,0x24]
-; SSE-NEXT:    ## xmm1 = xmm1[0,1,2,0]
+; SSE-NEXT:    pshufd $0, %xmm1, %xmm1 ## encoding: [0x66,0x0f,0x70,0xc9,0x00]
+; SSE-NEXT:    ## xmm1 = xmm1[0,0,0,0]
 ; SSE-NEXT:    pblendw $192, %xmm1, %xmm0 ## encoding: [0x66,0x0f,0x3a,0x0e,0xc1,0xc0]
 ; SSE-NEXT:    ## xmm0 = xmm0[0,1,2,3,4,5],xmm1[6,7]
 ; SSE-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
@@ -1213,8 +1213,8 @@ define <4 x i32> @i32_shuf_X00A(<4 x i32> %x, <4 x i32> %a) {
 ; AVX1-NEXT:    vxorps %xmm2, %xmm2, %xmm2 ## encoding: [0xc5,0xe8,0x57,0xd2]
 ; AVX1-NEXT:    vblendps $1, %xmm0, %xmm2, %xmm0 ## encoding: [0xc4,0xe3,0x69,0x0c,0xc0,0x01]
 ; AVX1-NEXT:    ## xmm0 = xmm0[0],xmm2[1,2,3]
-; AVX1-NEXT:    vpermilps $36, %xmm1, %xmm1 ## encoding: [0xc4,0xe3,0x79,0x04,0xc9,0x24]
-; AVX1-NEXT:    ## xmm1 = xmm1[0,1,2,0]
+; AVX1-NEXT:    vpermilps $0, %xmm1, %xmm1 ## encoding: [0xc4,0xe3,0x79,0x04,0xc9,0x00]
+; AVX1-NEXT:    ## xmm1 = xmm1[0,0,0,0]
 ; AVX1-NEXT:    vblendps $8, %xmm1, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x0c,0xc1,0x08]
 ; AVX1-NEXT:    ## xmm0 = xmm0[0,1,2],xmm1[3]
 ; AVX1-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
@@ -1224,8 +1224,7 @@ define <4 x i32> @i32_shuf_X00A(<4 x i32> %x, <4 x i32> %a) {
 ; AVX512-NEXT:    vxorps %xmm2, %xmm2, %xmm2 ## encoding: [0xc5,0xe8,0x57,0xd2]
 ; AVX512-NEXT:    vblendps $1, %xmm0, %xmm2, %xmm0 ## encoding: [0xc4,0xe3,0x69,0x0c,0xc0,0x01]
 ; AVX512-NEXT:    ## xmm0 = xmm0[0],xmm2[1,2,3]
-; AVX512-NEXT:    vpermilps $36, %xmm1, %xmm1 ## EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x04,0xc9,0x24]
-; AVX512-NEXT:    ## xmm1 = xmm1[0,1,2,0]
+; AVX512-NEXT:    vbroadcastss %xmm1, %xmm1 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x18,0xc9]
 ; AVX512-NEXT:    vblendps $8, %xmm1, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x0c,0xc1,0x08]
 ; AVX512-NEXT:    ## xmm0 = xmm0[0,1,2],xmm1[3]
 ; AVX512-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
@@ -1241,8 +1240,8 @@ define <4 x i32> @i32_shuf_X00X(<4 x i32> %x, <4 x i32> %a) {
 ; SSE-LABEL: i32_shuf_X00X:
 ; SSE:       ## %bb.0:
 ; SSE-NEXT:    pxor %xmm1, %xmm1 ## encoding: [0x66,0x0f,0xef,0xc9]
-; SSE-NEXT:    pshufd $36, %xmm0, %xmm0 ## encoding: [0x66,0x0f,0x70,0xc0,0x24]
-; SSE-NEXT:    ## xmm0 = xmm0[0,1,2,0]
+; SSE-NEXT:    pshufd $0, %xmm0, %xmm0 ## encoding: [0x66,0x0f,0x70,0xc0,0x00]
+; SSE-NEXT:    ## xmm0 = xmm0[0,0,0,0]
 ; SSE-NEXT:    pblendw $60, %xmm1, %xmm0 ## encoding: [0x66,0x0f,0x3a,0x0e,0xc1,0x3c]
 ; SSE-NEXT:    ## xmm0 = xmm0[0,1],xmm1[2,3,4,5],xmm0[6,7]
 ; SSE-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
@@ -1250,8 +1249,8 @@ define <4 x i32> @i32_shuf_X00X(<4 x i32> %x, <4 x i32> %a) {
 ; AVX1-LABEL: i32_shuf_X00X:
 ; AVX1:       ## %bb.0:
 ; AVX1-NEXT:    vxorps %xmm1, %xmm1, %xmm1 ## encoding: [0xc5,0xf0,0x57,0xc9]
-; AVX1-NEXT:    vpermilps $36, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x04,0xc0,0x24]
-; AVX1-NEXT:    ## xmm0 = xmm0[0,1,2,0]
+; AVX1-NEXT:    vpermilps $0, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x04,0xc0,0x00]
+; AVX1-NEXT:    ## xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    vblendps $6, %xmm1, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x0c,0xc1,0x06]
 ; AVX1-NEXT:    ## xmm0 = xmm0[0],xmm1[1,2],xmm0[3]
 ; AVX1-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
@@ -1276,8 +1275,8 @@ define <4 x i32> @i32_shuf_X0YC(<4 x i32> %x, <4 x i32> %a) {
 ; SSE:       ## %bb.0:
 ; SSE-NEXT:    pmovzxdq %xmm0, %xmm2 ## encoding: [0x66,0x0f,0x38,0x35,0xd0]
 ; SSE-NEXT:    ## xmm2 = xmm0[0],zero,xmm0[1],zero
-; SSE-NEXT:    pshufd $164, %xmm1, %xmm0 ## encoding: [0x66,0x0f,0x70,0xc1,0xa4]
-; SSE-NEXT:    ## xmm0 = xmm1[0,1,2,2]
+; SSE-NEXT:    pshufd $170, %xmm1, %xmm0 ## encoding: [0x66,0x0f,0x70,0xc1,0xaa]
+; SSE-NEXT:    ## xmm0 = xmm1[2,2,2,2]
 ; SSE-NEXT:    pblendw $63, %xmm2, %xmm0 ## encoding: [0x66,0x0f,0x3a,0x0e,0xc2,0x3f]
 ; SSE-NEXT:    ## xmm0 = xmm2[0,1,2,3,4,5],xmm0[6,7]
 ; SSE-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
@@ -1286,8 +1285,8 @@ define <4 x i32> @i32_shuf_X0YC(<4 x i32> %x, <4 x i32> %a) {
 ; AVX1:       ## %bb.0:
 ; AVX1-NEXT:    vpmovzxdq %xmm0, %xmm0 ## encoding: [0xc4,0xe2,0x79,0x35,0xc0]
 ; AVX1-NEXT:    ## xmm0 = xmm0[0],zero,xmm0[1],zero
-; AVX1-NEXT:    vpshufd $164, %xmm1, %xmm1 ## encoding: [0xc5,0xf9,0x70,0xc9,0xa4]
-; AVX1-NEXT:    ## xmm1 = xmm1[0,1,2,2]
+; AVX1-NEXT:    vpshufd $170, %xmm1, %xmm1 ## encoding: [0xc5,0xf9,0x70,0xc9,0xaa]
+; AVX1-NEXT:    ## xmm1 = xmm1[2,2,2,2]
 ; AVX1-NEXT:    vpblendw $192, %xmm1, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x0e,0xc1,0xc0]
 ; AVX1-NEXT:    ## xmm0 = xmm0[0,1,2,3,4,5],xmm1[6,7]
 ; AVX1-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
@@ -1296,8 +1295,8 @@ define <4 x i32> @i32_shuf_X0YC(<4 x i32> %x, <4 x i32> %a) {
 ; AVX512:       ## %bb.0:
 ; AVX512-NEXT:    vpmovzxdq %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x35,0xc0]
 ; AVX512-NEXT:    ## xmm0 = xmm0[0],zero,xmm0[1],zero
-; AVX512-NEXT:    vpshufd $164, %xmm1, %xmm1 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0x70,0xc9,0xa4]
-; AVX512-NEXT:    ## xmm1 = xmm1[0,1,2,2]
+; AVX512-NEXT:    vpshufd $170, %xmm1, %xmm1 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0x70,0xc9,0xaa]
+; AVX512-NEXT:    ## xmm1 = xmm1[2,2,2,2]
 ; AVX512-NEXT:    vpblendd $8, %xmm1, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x02,0xc1,0x08]
 ; AVX512-NEXT:    ## xmm0 = xmm0[0,1,2],xmm1[3]
 ; AVX512-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
diff --git a/llvm/test/CodeGen/X86/umul_fix_sat.ll b/llvm/test/CodeGen/X86/umul_fix_sat.ll
index ce744f93cdfe8..8a4331a998d96 100644
--- a/llvm/test/CodeGen/X86/umul_fix_sat.ll
+++ b/llvm/test/CodeGen/X86/umul_fix_sat.ll
@@ -139,9 +139,9 @@ define i4 @func3(i4 %x, i4 %y) nounwind {
 define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; X64-LABEL: vec:
 ; X64:       # %bb.0:
-; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,1,2,3]
+; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
 ; X64-NEXT:    movd %xmm2, %eax
-; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,1,2,3]
+; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
 ; X64-NEXT:    movd %xmm2, %ecx
 ; X64-NEXT:    imulq %rax, %rcx
 ; X64-NEXT:    movq %rcx, %rax
@@ -172,9 +172,9 @@ define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; X64-NEXT:    cmpl $3, %ecx
 ; X64-NEXT:    cmoval %eax, %edx
 ; X64-NEXT:    movd %edx, %xmm2
-; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
 ; X64-NEXT:    movd %xmm1, %ecx
-; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; X64-NEXT:    movd %xmm0, %edx
 ; X64-NEXT:    imulq %rcx, %rdx
 ; X64-NEXT:    movq %rdx, %rcx
@@ -353,9 +353,9 @@ define i4 @func6(i4 %x, i4 %y) nounwind {
 define <4 x i32> @vec2(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; X64-LABEL: vec2:
 ; X64:       # %bb.0:
-; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,1,2,3]
+; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
 ; X64-NEXT:    movd %xmm2, %eax
-; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,1,2,3]
+; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
 ; X64-NEXT:    movd %xmm2, %ecx
 ; X64-NEXT:    mull %ecx
 ; X64-NEXT:    movl $-1, %ecx
@@ -374,9 +374,9 @@ define <4 x i32> @vec2(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; X64-NEXT:    mull %edx
 ; X64-NEXT:    cmovol %ecx, %eax
 ; X64-NEXT:    movd %eax, %xmm2
-; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; X64-NEXT:    movd %xmm0, %eax
-; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X64-NEXT:    movd %xmm0, %edx
 ; X64-NEXT:    mull %edx
 ; X64-NEXT:    cmovol %ecx, %eax
diff --git a/llvm/test/CodeGen/X86/var-permute-128.ll b/llvm/test/CodeGen/X86/var-permute-128.ll
index 0a057852613ae..eb30af03d6c14 100644
--- a/llvm/test/CodeGen/X86/var-permute-128.ll
+++ b/llvm/test/CodeGen/X86/var-permute-128.ll
@@ -67,11 +67,11 @@ define <4 x i32> @var_shuffle_v4i32(<4 x i32> %v, <4 x i32> %indices) nounwind {
 ; SSE3-LABEL: var_shuffle_v4i32:
 ; SSE3:       # %bb.0:
 ; SSE3-NEXT:    movd %xmm1, %eax
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,2,3]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
 ; SSE3-NEXT:    movd %xmm2, %ecx
 ; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; SSE3-NEXT:    movd %xmm2, %edx
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; SSE3-NEXT:    movd %xmm1, %esi
 ; SSE3-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
 ; SSE3-NEXT:    andl $3, %eax
@@ -428,11 +428,11 @@ define <4 x float> @var_shuffle_v4f32(<4 x float> %v, <4 x i32> %indices) nounwi
 ; SSE3-LABEL: var_shuffle_v4f32:
 ; SSE3:       # %bb.0:
 ; SSE3-NEXT:    movd %xmm1, %eax
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,2,3]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
 ; SSE3-NEXT:    movd %xmm2, %ecx
 ; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; SSE3-NEXT:    movd %xmm2, %edx
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; SSE3-NEXT:    movd %xmm1, %esi
 ; SSE3-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
 ; SSE3-NEXT:    andl $3, %eax
diff --git a/llvm/test/CodeGen/X86/vec-libcalls.ll b/llvm/test/CodeGen/X86/vec-libcalls.ll
index 83663fc899d1c..506caf767d86f 100644
--- a/llvm/test/CodeGen/X86/vec-libcalls.ll
+++ b/llvm/test/CodeGen/X86/vec-libcalls.ll
@@ -108,8 +108,8 @@ define <4 x float> @sin_v4f32(<4 x float> %x) nounwind {
 ; CHECK-NEXT:    vmovaps (%rsp), %xmm1 # 16-byte Reload
 ; CHECK-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
 ; CHECK-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
-; CHECK-NEXT:    vpermilps $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
-; CHECK-NEXT:    # xmm0 = mem[3,1,2,3]
+; CHECK-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = mem[3,3,3,3]
 ; CHECK-NEXT:    callq sinf
 ; CHECK-NEXT:    vmovaps (%rsp), %xmm1 # 16-byte Reload
 ; CHECK-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
@@ -140,8 +140,8 @@ define <5 x float> @sin_v5f32(<5 x float> %x) nounwind {
 ; CHECK-NEXT:    vmovaps (%rsp), %xmm1 # 16-byte Reload
 ; CHECK-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
 ; CHECK-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
-; CHECK-NEXT:    vpermilps $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
-; CHECK-NEXT:    # xmm0 = mem[3,1,2,3]
+; CHECK-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = mem[3,3,3,3]
 ; CHECK-NEXT:    callq sinf
 ; CHECK-NEXT:    vmovaps (%rsp), %xmm1 # 16-byte Reload
 ; CHECK-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
@@ -191,8 +191,8 @@ define <6 x float> @sin_v6f32(<6 x float> %x) nounwind {
 ; CHECK-NEXT:    vmovaps (%rsp), %xmm1 # 16-byte Reload
 ; CHECK-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
 ; CHECK-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
-; CHECK-NEXT:    vpermilps $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
-; CHECK-NEXT:    # xmm0 = mem[3,1,2,3]
+; CHECK-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = mem[3,3,3,3]
 ; CHECK-NEXT:    callq sinf
 ; CHECK-NEXT:    vmovaps (%rsp), %xmm1 # 16-byte Reload
 ; CHECK-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
diff --git a/llvm/test/CodeGen/X86/vec-strict-128.ll b/llvm/test/CodeGen/X86/vec-strict-128.ll
index 4fecd2631dc01..d6d5dea887a25 100644
--- a/llvm/test/CodeGen/X86/vec-strict-128.ll
+++ b/llvm/test/CodeGen/X86/vec-strict-128.ll
@@ -248,23 +248,23 @@ define <4 x float> @f13(<4 x float> %a, <4 x float> %b, <4 x float> %c) #0 {
 ; SSE-X86-NEXT:    fstpt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Spill
 ; SSE-X86-NEXT:    wait
 ; SSE-X86-NEXT:    movups {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
-; SSE-X86-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE-X86-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE-X86-NEXT:    movss %xmm0, {{[0-9]+}}(%esp)
 ; SSE-X86-NEXT:    movups {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
-; SSE-X86-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE-X86-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE-X86-NEXT:    movss %xmm0, {{[0-9]+}}(%esp)
 ; SSE-X86-NEXT:    movups {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
-; SSE-X86-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE-X86-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE-X86-NEXT:    movss %xmm0, (%esp)
 ; SSE-X86-NEXT:    calll fmaf
 ; SSE-X86-NEXT:    movups {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
-; SSE-X86-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE-X86-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE-X86-NEXT:    movss %xmm0, {{[0-9]+}}(%esp)
 ; SSE-X86-NEXT:    movups {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
-; SSE-X86-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE-X86-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE-X86-NEXT:    movss %xmm0, {{[0-9]+}}(%esp)
 ; SSE-X86-NEXT:    movups {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
-; SSE-X86-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE-X86-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE-X86-NEXT:    movss %xmm0, (%esp)
 ; SSE-X86-NEXT:    fstps {{[0-9]+}}(%esp)
 ; SSE-X86-NEXT:    fldt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Reload
@@ -293,9 +293,9 @@ define <4 x float> @f13(<4 x float> %a, <4 x float> %b, <4 x float> %c) #0 {
 ; SSE-X64-NEXT:    movaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; SSE-X64-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; SSE-X64-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE-X64-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
-; SSE-X64-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1,2,3]
-; SSE-X64-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,1,2,3]
+; SSE-X64-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; SSE-X64-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
+; SSE-X64-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,3,3,3]
 ; SSE-X64-NEXT:    callq fmaf
 ; SSE-X64-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
 ; SSE-X64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
@@ -314,11 +314,11 @@ define <4 x float> @f13(<4 x float> %a, <4 x float> %b, <4 x float> %c) #0 {
 ; SSE-X64-NEXT:    callq fmaf
 ; SSE-X64-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; SSE-X64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; SSE-X64-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE-X64-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE-X64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; SSE-X64-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; SSE-X64-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,1,1]
 ; SSE-X64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
-; SSE-X64-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,2,3]
+; SSE-X64-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,1,1]
 ; SSE-X64-NEXT:    callq fmaf
 ; SSE-X64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
 ; SSE-X64-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
diff --git a/llvm/test/CodeGen/X86/vec-strict-cmp-128.ll b/llvm/test/CodeGen/X86/vec-strict-cmp-128.ll
index 72d9035e67385..ee0978ff15ceb 100644
--- a/llvm/test/CodeGen/X86/vec-strict-cmp-128.ll
+++ b/llvm/test/CodeGen/X86/vec-strict-cmp-128.ll
@@ -110,9 +110,9 @@ define <4 x i32> @test_v4f32_ogt_q(<4 x i32> %a, <4 x i32> %b, <4 x float> %f1,
 ; SSE-32-NEXT:    subl $16, %esp
 ; SSE-32-NEXT:    movaps 8(%ebp), %xmm3
 ; SSE-32-NEXT:    movaps %xmm3, %xmm4
-; SSE-32-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,1],xmm3[2,3]
+; SSE-32-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,3],xmm3[3,3]
 ; SSE-32-NEXT:    movaps %xmm2, %xmm5
-; SSE-32-NEXT:    shufps {{.*#+}} xmm5 = xmm5[3,1],xmm2[2,3]
+; SSE-32-NEXT:    shufps {{.*#+}} xmm5 = xmm5[3,3],xmm2[3,3]
 ; SSE-32-NEXT:    xorl %eax, %eax
 ; SSE-32-NEXT:    ucomiss %xmm4, %xmm5
 ; SSE-32-NEXT:    movl $-1, %ecx
@@ -132,8 +132,8 @@ define <4 x i32> @test_v4f32_ogt_q(<4 x i32> %a, <4 x i32> %b, <4 x float> %f1,
 ; SSE-32-NEXT:    movl $0, %edx
 ; SSE-32-NEXT:    cmoval %ecx, %edx
 ; SSE-32-NEXT:    movd %edx, %xmm4
-; SSE-32-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,2,3]
-; SSE-32-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,2,3]
+; SSE-32-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,1,1]
+; SSE-32-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,1,1]
 ; SSE-32-NEXT:    ucomiss %xmm3, %xmm2
 ; SSE-32-NEXT:    cmoval %ecx, %eax
 ; SSE-32-NEXT:    movd %eax, %xmm2
@@ -149,9 +149,9 @@ define <4 x i32> @test_v4f32_ogt_q(<4 x i32> %a, <4 x i32> %b, <4 x float> %f1,
 ; SSE-64-LABEL: test_v4f32_ogt_q:
 ; SSE-64:       # %bb.0:
 ; SSE-64-NEXT:    movaps %xmm3, %xmm4
-; SSE-64-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,1],xmm3[2,3]
+; SSE-64-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,3],xmm3[3,3]
 ; SSE-64-NEXT:    movaps %xmm2, %xmm5
-; SSE-64-NEXT:    shufps {{.*#+}} xmm5 = xmm5[3,1],xmm2[2,3]
+; SSE-64-NEXT:    shufps {{.*#+}} xmm5 = xmm5[3,3],xmm2[3,3]
 ; SSE-64-NEXT:    xorl %eax, %eax
 ; SSE-64-NEXT:    ucomiss %xmm4, %xmm5
 ; SSE-64-NEXT:    movl $-1, %ecx
@@ -171,8 +171,8 @@ define <4 x i32> @test_v4f32_ogt_q(<4 x i32> %a, <4 x i32> %b, <4 x float> %f1,
 ; SSE-64-NEXT:    movl $0, %edx
 ; SSE-64-NEXT:    cmoval %ecx, %edx
 ; SSE-64-NEXT:    movd %edx, %xmm4
-; SSE-64-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,2,3]
-; SSE-64-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,2,3]
+; SSE-64-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,1,1]
+; SSE-64-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,1,1]
 ; SSE-64-NEXT:    ucomiss %xmm3, %xmm2
 ; SSE-64-NEXT:    cmoval %ecx, %eax
 ; SSE-64-NEXT:    movd %eax, %xmm2
@@ -264,9 +264,9 @@ define <4 x i32> @test_v4f32_oge_q(<4 x i32> %a, <4 x i32> %b, <4 x float> %f1,
 ; SSE-32-NEXT:    subl $16, %esp
 ; SSE-32-NEXT:    movaps 8(%ebp), %xmm3
 ; SSE-32-NEXT:    movaps %xmm3, %xmm4
-; SSE-32-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,1],xmm3[2,3]
+; SSE-32-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,3],xmm3[3,3]
 ; SSE-32-NEXT:    movaps %xmm2, %xmm5
-; SSE-32-NEXT:    shufps {{.*#+}} xmm5 = xmm5[3,1],xmm2[2,3]
+; SSE-32-NEXT:    shufps {{.*#+}} xmm5 = xmm5[3,3],xmm2[3,3]
 ; SSE-32-NEXT:    xorl %eax, %eax
 ; SSE-32-NEXT:    ucomiss %xmm4, %xmm5
 ; SSE-32-NEXT:    movl $-1, %ecx
@@ -286,8 +286,8 @@ define <4 x i32> @test_v4f32_oge_q(<4 x i32> %a, <4 x i32> %b, <4 x float> %f1,
 ; SSE-32-NEXT:    movl $0, %edx
 ; SSE-32-NEXT:    cmovael %ecx, %edx
 ; SSE-32-NEXT:    movd %edx, %xmm4
-; SSE-32-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,2,3]
-; SSE-32-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,2,3]
+; SSE-32-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,1,1]
+; SSE-32-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,1,1]
 ; SSE-32-NEXT:    ucomiss %xmm3, %xmm2
 ; SSE-32-NEXT:    cmovael %ecx, %eax
 ; SSE-32-NEXT:    movd %eax, %xmm2
@@ -303,9 +303,9 @@ define <4 x i32> @test_v4f32_oge_q(<4 x i32> %a, <4 x i32> %b, <4 x float> %f1,
 ; SSE-64-LABEL: test_v4f32_oge_q:
 ; SSE-64:       # %bb.0:
 ; SSE-64-NEXT:    movaps %xmm3, %xmm4
-; SSE-64-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,1],xmm3[2,3]
+; SSE-64-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,3],xmm3[3,3]
 ; SSE-64-NEXT:    movaps %xmm2, %xmm5
-; SSE-64-NEXT:    shufps {{.*#+}} xmm5 = xmm5[3,1],xmm2[2,3]
+; SSE-64-NEXT:    shufps {{.*#+}} xmm5 = xmm5[3,3],xmm2[3,3]
 ; SSE-64-NEXT:    xorl %eax, %eax
 ; SSE-64-NEXT:    ucomiss %xmm4, %xmm5
 ; SSE-64-NEXT:    movl $-1, %ecx
@@ -325,8 +325,8 @@ define <4 x i32> @test_v4f32_oge_q(<4 x i32> %a, <4 x i32> %b, <4 x float> %f1,
 ; SSE-64-NEXT:    movl $0, %edx
 ; SSE-64-NEXT:    cmovael %ecx, %edx
 ; SSE-64-NEXT:    movd %edx, %xmm4
-; SSE-64-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,2,3]
-; SSE-64-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,2,3]
+; SSE-64-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,1,1]
+; SSE-64-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,1,1]
 ; SSE-64-NEXT:    ucomiss %xmm3, %xmm2
 ; SSE-64-NEXT:    cmovael %ecx, %eax
 ; SSE-64-NEXT:    movd %eax, %xmm2
@@ -418,9 +418,9 @@ define <4 x i32> @test_v4f32_olt_q(<4 x i32> %a, <4 x i32> %b, <4 x float> %f1,
 ; SSE-32-NEXT:    subl $16, %esp
 ; SSE-32-NEXT:    movaps 8(%ebp), %xmm3
 ; SSE-32-NEXT:    movaps %xmm2, %xmm4
-; SSE-32-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,1],xmm2[2,3]
+; SSE-32-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,3],xmm2[3,3]
 ; SSE-32-NEXT:    movaps %xmm3, %xmm5
-; SSE-32-NEXT:    shufps {{.*#+}} xmm5 = xmm5[3,1],xmm3[2,3]
+; SSE-32-NEXT:    shufps {{.*#+}} xmm5 = xmm5[3,3],xmm3[3,3]
 ; SSE-32-NEXT:    xorl %eax, %eax
 ; SSE-32-NEXT:    ucomiss %xmm4, %xmm5
 ; SSE-32-NEXT:    movl $-1, %ecx
@@ -440,8 +440,8 @@ define <4 x i32> @test_v4f32_olt_q(<4 x i32> %a, <4 x i32> %b, <4 x float> %f1,
 ; SSE-32-NEXT:    movl $0, %edx
 ; SSE-32-NEXT:    cmoval %ecx, %edx
 ; SSE-32-NEXT:    movd %edx, %xmm4
-; SSE-32-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,2,3]
-; SSE-32-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,2,3]
+; SSE-32-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,1,1]
+; SSE-32-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,1,1]
 ; SSE-32-NEXT:    ucomiss %xmm2, %xmm3
 ; SSE-32-NEXT:    cmoval %ecx, %eax
 ; SSE-32-NEXT:    movd %eax, %xmm2
@@ -457,9 +457,9 @@ define <4 x i32> @test_v4f32_olt_q(<4 x i32> %a, <4 x i32> %b, <4 x float> %f1,
 ; SSE-64-LABEL: test_v4f32_olt_q:
 ; SSE-64:       # %bb.0:
 ; SSE-64-NEXT:    movaps %xmm2, %xmm4
-; SSE-64-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,1],xmm2[2,3]
+; SSE-64-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,3],xmm2[3,3]
 ; SSE-64-NEXT:    movaps %xmm3, %xmm5
-; SSE-64-NEXT:    shufps {{.*#+}} xmm5 = xmm5[3,1],xmm3[2,3]
+; SSE-64-NEXT:    shufps {{.*#+}} xmm5 = xmm5[3,3],xmm3[3,3]
 ; SSE-64-NEXT:    xorl %eax, %eax
 ; SSE-64-NEXT:    ucomiss %xmm4, %xmm5
 ; SSE-64-NEXT:    movl $-1, %ecx
@@ -479,8 +479,8 @@ define <4 x i32> @test_v4f32_olt_q(<4 x i32> %a, <4 x i32> %b, <4 x float> %f1,
 ; SSE-64-NEXT:    movl $0, %edx
 ; SSE-64-NEXT:    cmoval %ecx, %edx
 ; SSE-64-NEXT:    movd %edx, %xmm4
-; SSE-64-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,2,3]
-; SSE-64-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,2,3]
+; SSE-64-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,1,1]
+; SSE-64-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,1,1]
 ; SSE-64-NEXT:    ucomiss %xmm2, %xmm3
 ; SSE-64-NEXT:    cmoval %ecx, %eax
 ; SSE-64-NEXT:    movd %eax, %xmm2
@@ -570,9 +570,9 @@ define <4 x i32> @test_v4f32_ole_q(<4 x i32> %a, <4 x i32> %b, <4 x float> %f1,
 ; SSE-32-NEXT:    subl $16, %esp
 ; SSE-32-NEXT:    movaps 8(%ebp), %xmm3
 ; SSE-32-NEXT:    movaps %xmm2, %xmm4
-; SSE-32-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,1],xmm2[2,3]
+; SSE-32-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,3],xmm2[3,3]
 ; SSE-32-NEXT:    movaps %xmm3, %xmm5
-; SSE-32-NEXT:    shufps {{.*#+}} xmm5 = xmm5[3,1],xmm3[2,3]
+; SSE-32-NEXT:    shufps {{.*#+}} xmm5 = xmm5[3,3],xmm3[3,3]
 ; SSE-32-NEXT:    xorl %eax, %eax
 ; SSE-32-NEXT:    ucomiss %xmm4, %xmm5
 ; SSE-32-NEXT:    movl $-1, %ecx
@@ -592,8 +592,8 @@ define <4 x i32> @test_v4f32_ole_q(<4 x i32> %a, <4 x i32> %b, <4 x float> %f1,
 ; SSE-32-NEXT:    movl $0, %edx
 ; SSE-32-NEXT:    cmovael %ecx, %edx
 ; SSE-32-NEXT:    movd %edx, %xmm4
-; SSE-32-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,2,3]
-; SSE-32-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,2,3]
+; SSE-32-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,1,1]
+; SSE-32-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,1,1]
 ; SSE-32-NEXT:    ucomiss %xmm2, %xmm3
 ; SSE-32-NEXT:    cmovael %ecx, %eax
 ; SSE-32-NEXT:    movd %eax, %xmm2
@@ -609,9 +609,9 @@ define <4 x i32> @test_v4f32_ole_q(<4 x i32> %a, <4 x i32> %b, <4 x float> %f1,
 ; SSE-64-LABEL: test_v4f32_ole_q:
 ; SSE-64:       # %bb.0:
 ; SSE-64-NEXT:    movaps %xmm2, %xmm4
-; SSE-64-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,1],xmm2[2,3]
+; SSE-64-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,3],xmm2[3,3]
 ; SSE-64-NEXT:    movaps %xmm3, %xmm5
-; SSE-64-NEXT:    shufps {{.*#+}} xmm5 = xmm5[3,1],xmm3[2,3]
+; SSE-64-NEXT:    shufps {{.*#+}} xmm5 = xmm5[3,3],xmm3[3,3]
 ; SSE-64-NEXT:    xorl %eax, %eax
 ; SSE-64-NEXT:    ucomiss %xmm4, %xmm5
 ; SSE-64-NEXT:    movl $-1, %ecx
@@ -631,8 +631,8 @@ define <4 x i32> @test_v4f32_ole_q(<4 x i32> %a, <4 x i32> %b, <4 x float> %f1,
 ; SSE-64-NEXT:    movl $0, %edx
 ; SSE-64-NEXT:    cmovael %ecx, %edx
 ; SSE-64-NEXT:    movd %edx, %xmm4
-; SSE-64-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,2,3]
-; SSE-64-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,2,3]
+; SSE-64-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,1,1]
+; SSE-64-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,1,1]
 ; SSE-64-NEXT:    ucomiss %xmm2, %xmm3
 ; SSE-64-NEXT:    cmovael %ecx, %eax
 ; SSE-64-NEXT:    movd %eax, %xmm2
@@ -1015,9 +1015,9 @@ define <4 x i32> @test_v4f32_ugt_q(<4 x i32> %a, <4 x i32> %b, <4 x float> %f1,
 ; SSE-32-NEXT:    subl $16, %esp
 ; SSE-32-NEXT:    movaps 8(%ebp), %xmm3
 ; SSE-32-NEXT:    movaps %xmm2, %xmm4
-; SSE-32-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,1],xmm2[2,3]
+; SSE-32-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,3],xmm2[3,3]
 ; SSE-32-NEXT:    movaps %xmm3, %xmm5
-; SSE-32-NEXT:    shufps {{.*#+}} xmm5 = xmm5[3,1],xmm3[2,3]
+; SSE-32-NEXT:    shufps {{.*#+}} xmm5 = xmm5[3,3],xmm3[3,3]
 ; SSE-32-NEXT:    xorl %eax, %eax
 ; SSE-32-NEXT:    ucomiss %xmm4, %xmm5
 ; SSE-32-NEXT:    movl $-1, %ecx
@@ -1037,8 +1037,8 @@ define <4 x i32> @test_v4f32_ugt_q(<4 x i32> %a, <4 x i32> %b, <4 x float> %f1,
 ; SSE-32-NEXT:    movl $0, %edx
 ; SSE-32-NEXT:    cmovbl %ecx, %edx
 ; SSE-32-NEXT:    movd %edx, %xmm4
-; SSE-32-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,2,3]
-; SSE-32-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,2,3]
+; SSE-32-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,1,1]
+; SSE-32-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,1,1]
 ; SSE-32-NEXT:    ucomiss %xmm2, %xmm3
 ; SSE-32-NEXT:    cmovbl %ecx, %eax
 ; SSE-32-NEXT:    movd %eax, %xmm2
@@ -1054,9 +1054,9 @@ define <4 x i32> @test_v4f32_ugt_q(<4 x i32> %a, <4 x i32> %b, <4 x float> %f1,
 ; SSE-64-LABEL: test_v4f32_ugt_q:
 ; SSE-64:       # %bb.0:
 ; SSE-64-NEXT:    movaps %xmm2, %xmm4
-; SSE-64-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,1],xmm2[2,3]
+; SSE-64-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,3],xmm2[3,3]
 ; SSE-64-NEXT:    movaps %xmm3, %xmm5
-; SSE-64-NEXT:    shufps {{.*#+}} xmm5 = xmm5[3,1],xmm3[2,3]
+; SSE-64-NEXT:    shufps {{.*#+}} xmm5 = xmm5[3,3],xmm3[3,3]
 ; SSE-64-NEXT:    xorl %eax, %eax
 ; SSE-64-NEXT:    ucomiss %xmm4, %xmm5
 ; SSE-64-NEXT:    movl $-1, %ecx
@@ -1076,8 +1076,8 @@ define <4 x i32> @test_v4f32_ugt_q(<4 x i32> %a, <4 x i32> %b, <4 x float> %f1,
 ; SSE-64-NEXT:    movl $0, %edx
 ; SSE-64-NEXT:    cmovbl %ecx, %edx
 ; SSE-64-NEXT:    movd %edx, %xmm4
-; SSE-64-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,2,3]
-; SSE-64-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,2,3]
+; SSE-64-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,1,1]
+; SSE-64-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,1,1]
 ; SSE-64-NEXT:    ucomiss %xmm2, %xmm3
 ; SSE-64-NEXT:    cmovbl %ecx, %eax
 ; SSE-64-NEXT:    movd %eax, %xmm2
@@ -1167,9 +1167,9 @@ define <4 x i32> @test_v4f32_uge_q(<4 x i32> %a, <4 x i32> %b, <4 x float> %f1,
 ; SSE-32-NEXT:    subl $16, %esp
 ; SSE-32-NEXT:    movaps 8(%ebp), %xmm3
 ; SSE-32-NEXT:    movaps %xmm2, %xmm4
-; SSE-32-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,1],xmm2[2,3]
+; SSE-32-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,3],xmm2[3,3]
 ; SSE-32-NEXT:    movaps %xmm3, %xmm5
-; SSE-32-NEXT:    shufps {{.*#+}} xmm5 = xmm5[3,1],xmm3[2,3]
+; SSE-32-NEXT:    shufps {{.*#+}} xmm5 = xmm5[3,3],xmm3[3,3]
 ; SSE-32-NEXT:    xorl %eax, %eax
 ; SSE-32-NEXT:    ucomiss %xmm4, %xmm5
 ; SSE-32-NEXT:    movl $-1, %ecx
@@ -1189,8 +1189,8 @@ define <4 x i32> @test_v4f32_uge_q(<4 x i32> %a, <4 x i32> %b, <4 x float> %f1,
 ; SSE-32-NEXT:    movl $0, %edx
 ; SSE-32-NEXT:    cmovbel %ecx, %edx
 ; SSE-32-NEXT:    movd %edx, %xmm4
-; SSE-32-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,2,3]
-; SSE-32-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,2,3]
+; SSE-32-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,1,1]
+; SSE-32-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,1,1]
 ; SSE-32-NEXT:    ucomiss %xmm2, %xmm3
 ; SSE-32-NEXT:    cmovbel %ecx, %eax
 ; SSE-32-NEXT:    movd %eax, %xmm2
@@ -1206,9 +1206,9 @@ define <4 x i32> @test_v4f32_uge_q(<4 x i32> %a, <4 x i32> %b, <4 x float> %f1,
 ; SSE-64-LABEL: test_v4f32_uge_q:
 ; SSE-64:       # %bb.0:
 ; SSE-64-NEXT:    movaps %xmm2, %xmm4
-; SSE-64-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,1],xmm2[2,3]
+; SSE-64-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,3],xmm2[3,3]
 ; SSE-64-NEXT:    movaps %xmm3, %xmm5
-; SSE-64-NEXT:    shufps {{.*#+}} xmm5 = xmm5[3,1],xmm3[2,3]
+; SSE-64-NEXT:    shufps {{.*#+}} xmm5 = xmm5[3,3],xmm3[3,3]
 ; SSE-64-NEXT:    xorl %eax, %eax
 ; SSE-64-NEXT:    ucomiss %xmm4, %xmm5
 ; SSE-64-NEXT:    movl $-1, %ecx
@@ -1228,8 +1228,8 @@ define <4 x i32> @test_v4f32_uge_q(<4 x i32> %a, <4 x i32> %b, <4 x float> %f1,
 ; SSE-64-NEXT:    movl $0, %edx
 ; SSE-64-NEXT:    cmovbel %ecx, %edx
 ; SSE-64-NEXT:    movd %edx, %xmm4
-; SSE-64-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,2,3]
-; SSE-64-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,2,3]
+; SSE-64-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,1,1]
+; SSE-64-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,1,1]
 ; SSE-64-NEXT:    ucomiss %xmm2, %xmm3
 ; SSE-64-NEXT:    cmovbel %ecx, %eax
 ; SSE-64-NEXT:    movd %eax, %xmm2
@@ -1319,9 +1319,9 @@ define <4 x i32> @test_v4f32_ult_q(<4 x i32> %a, <4 x i32> %b, <4 x float> %f1,
 ; SSE-32-NEXT:    subl $16, %esp
 ; SSE-32-NEXT:    movaps 8(%ebp), %xmm3
 ; SSE-32-NEXT:    movaps %xmm3, %xmm4
-; SSE-32-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,1],xmm3[2,3]
+; SSE-32-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,3],xmm3[3,3]
 ; SSE-32-NEXT:    movaps %xmm2, %xmm5
-; SSE-32-NEXT:    shufps {{.*#+}} xmm5 = xmm5[3,1],xmm2[2,3]
+; SSE-32-NEXT:    shufps {{.*#+}} xmm5 = xmm5[3,3],xmm2[3,3]
 ; SSE-32-NEXT:    xorl %eax, %eax
 ; SSE-32-NEXT:    ucomiss %xmm4, %xmm5
 ; SSE-32-NEXT:    movl $-1, %ecx
@@ -1341,8 +1341,8 @@ define <4 x i32> @test_v4f32_ult_q(<4 x i32> %a, <4 x i32> %b, <4 x float> %f1,
 ; SSE-32-NEXT:    movl $0, %edx
 ; SSE-32-NEXT:    cmovbl %ecx, %edx
 ; SSE-32-NEXT:    movd %edx, %xmm4
-; SSE-32-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,2,3]
-; SSE-32-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,2,3]
+; SSE-32-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,1,1]
+; SSE-32-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,1,1]
 ; SSE-32-NEXT:    ucomiss %xmm3, %xmm2
 ; SSE-32-NEXT:    cmovbl %ecx, %eax
 ; SSE-32-NEXT:    movd %eax, %xmm2
@@ -1358,9 +1358,9 @@ define <4 x i32> @test_v4f32_ult_q(<4 x i32> %a, <4 x i32> %b, <4 x float> %f1,
 ; SSE-64-LABEL: test_v4f32_ult_q:
 ; SSE-64:       # %bb.0:
 ; SSE-64-NEXT:    movaps %xmm3, %xmm4
-; SSE-64-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,1],xmm3[2,3]
+; SSE-64-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,3],xmm3[3,3]
 ; SSE-64-NEXT:    movaps %xmm2, %xmm5
-; SSE-64-NEXT:    shufps {{.*#+}} xmm5 = xmm5[3,1],xmm2[2,3]
+; SSE-64-NEXT:    shufps {{.*#+}} xmm5 = xmm5[3,3],xmm2[3,3]
 ; SSE-64-NEXT:    xorl %eax, %eax
 ; SSE-64-NEXT:    ucomiss %xmm4, %xmm5
 ; SSE-64-NEXT:    movl $-1, %ecx
@@ -1380,8 +1380,8 @@ define <4 x i32> @test_v4f32_ult_q(<4 x i32> %a, <4 x i32> %b, <4 x float> %f1,
 ; SSE-64-NEXT:    movl $0, %edx
 ; SSE-64-NEXT:    cmovbl %ecx, %edx
 ; SSE-64-NEXT:    movd %edx, %xmm4
-; SSE-64-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,2,3]
-; SSE-64-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,2,3]
+; SSE-64-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,1,1]
+; SSE-64-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,1,1]
 ; SSE-64-NEXT:    ucomiss %xmm3, %xmm2
 ; SSE-64-NEXT:    cmovbl %ecx, %eax
 ; SSE-64-NEXT:    movd %eax, %xmm2
@@ -1473,9 +1473,9 @@ define <4 x i32> @test_v4f32_ule_q(<4 x i32> %a, <4 x i32> %b, <4 x float> %f1,
 ; SSE-32-NEXT:    subl $16, %esp
 ; SSE-32-NEXT:    movaps 8(%ebp), %xmm3
 ; SSE-32-NEXT:    movaps %xmm3, %xmm4
-; SSE-32-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,1],xmm3[2,3]
+; SSE-32-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,3],xmm3[3,3]
 ; SSE-32-NEXT:    movaps %xmm2, %xmm5
-; SSE-32-NEXT:    shufps {{.*#+}} xmm5 = xmm5[3,1],xmm2[2,3]
+; SSE-32-NEXT:    shufps {{.*#+}} xmm5 = xmm5[3,3],xmm2[3,3]
 ; SSE-32-NEXT:    xorl %eax, %eax
 ; SSE-32-NEXT:    ucomiss %xmm4, %xmm5
 ; SSE-32-NEXT:    movl $-1, %ecx
@@ -1495,8 +1495,8 @@ define <4 x i32> @test_v4f32_ule_q(<4 x i32> %a, <4 x i32> %b, <4 x float> %f1,
 ; SSE-32-NEXT:    movl $0, %edx
 ; SSE-32-NEXT:    cmovbel %ecx, %edx
 ; SSE-32-NEXT:    movd %edx, %xmm4
-; SSE-32-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,2,3]
-; SSE-32-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,2,3]
+; SSE-32-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,1,1]
+; SSE-32-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,1,1]
 ; SSE-32-NEXT:    ucomiss %xmm3, %xmm2
 ; SSE-32-NEXT:    cmovbel %ecx, %eax
 ; SSE-32-NEXT:    movd %eax, %xmm2
@@ -1512,9 +1512,9 @@ define <4 x i32> @test_v4f32_ule_q(<4 x i32> %a, <4 x i32> %b, <4 x float> %f1,
 ; SSE-64-LABEL: test_v4f32_ule_q:
 ; SSE-64:       # %bb.0:
 ; SSE-64-NEXT:    movaps %xmm3, %xmm4
-; SSE-64-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,1],xmm3[2,3]
+; SSE-64-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,3],xmm3[3,3]
 ; SSE-64-NEXT:    movaps %xmm2, %xmm5
-; SSE-64-NEXT:    shufps {{.*#+}} xmm5 = xmm5[3,1],xmm2[2,3]
+; SSE-64-NEXT:    shufps {{.*#+}} xmm5 = xmm5[3,3],xmm2[3,3]
 ; SSE-64-NEXT:    xorl %eax, %eax
 ; SSE-64-NEXT:    ucomiss %xmm4, %xmm5
 ; SSE-64-NEXT:    movl $-1, %ecx
@@ -1534,8 +1534,8 @@ define <4 x i32> @test_v4f32_ule_q(<4 x i32> %a, <4 x i32> %b, <4 x float> %f1,
 ; SSE-64-NEXT:    movl $0, %edx
 ; SSE-64-NEXT:    cmovbel %ecx, %edx
 ; SSE-64-NEXT:    movd %edx, %xmm4
-; SSE-64-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,2,3]
-; SSE-64-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,2,3]
+; SSE-64-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,1,1]
+; SSE-64-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,1,1]
 ; SSE-64-NEXT:    ucomiss %xmm3, %xmm2
 ; SSE-64-NEXT:    cmovbel %ecx, %eax
 ; SSE-64-NEXT:    movd %eax, %xmm2
diff --git a/llvm/test/CodeGen/X86/vec-strict-cmp-sub128.ll b/llvm/test/CodeGen/X86/vec-strict-cmp-sub128.ll
index 204a6181bf115..97997018cff9d 100644
--- a/llvm/test/CodeGen/X86/vec-strict-cmp-sub128.ll
+++ b/llvm/test/CodeGen/X86/vec-strict-cmp-sub128.ll
@@ -22,8 +22,8 @@ define <2 x i32> @test_v2f32_ogt_s(<2 x i32> %a, <2 x i32> %b, <2 x float> %f1,
 ; SSE-32-NEXT:    movl $0, %edx
 ; SSE-32-NEXT:    cmoval %ecx, %edx
 ; SSE-32-NEXT:    movd %edx, %xmm4
-; SSE-32-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,2,3]
-; SSE-32-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,2,3]
+; SSE-32-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,1,1]
+; SSE-32-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,1,1]
 ; SSE-32-NEXT:    comiss %xmm3, %xmm2
 ; SSE-32-NEXT:    cmoval %ecx, %eax
 ; SSE-32-NEXT:    movd %eax, %xmm2
@@ -43,8 +43,8 @@ define <2 x i32> @test_v2f32_ogt_s(<2 x i32> %a, <2 x i32> %b, <2 x float> %f1,
 ; SSE-64-NEXT:    movl $0, %edx
 ; SSE-64-NEXT:    cmoval %ecx, %edx
 ; SSE-64-NEXT:    movd %edx, %xmm4
-; SSE-64-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,2,3]
-; SSE-64-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,2,3]
+; SSE-64-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,1,1]
+; SSE-64-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,1,1]
 ; SSE-64-NEXT:    comiss %xmm3, %xmm2
 ; SSE-64-NEXT:    cmoval %ecx, %eax
 ; SSE-64-NEXT:    movd %eax, %xmm2
@@ -210,8 +210,8 @@ define <2 x i32> @test_v2f32_oeq_q(<2 x i32> %a, <2 x i32> %b, <2 x float> %f1,
 ; SSE-32-NEXT:    cmovnel %eax, %edx
 ; SSE-32-NEXT:    cmovpl %eax, %edx
 ; SSE-32-NEXT:    movd %edx, %xmm4
-; SSE-32-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,2,3]
-; SSE-32-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,2,3]
+; SSE-32-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,1,1]
+; SSE-32-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,1,1]
 ; SSE-32-NEXT:    ucomiss %xmm3, %xmm2
 ; SSE-32-NEXT:    cmovnel %eax, %ecx
 ; SSE-32-NEXT:    cmovpl %eax, %ecx
@@ -233,8 +233,8 @@ define <2 x i32> @test_v2f32_oeq_q(<2 x i32> %a, <2 x i32> %b, <2 x float> %f1,
 ; SSE-64-NEXT:    cmovnel %eax, %edx
 ; SSE-64-NEXT:    cmovpl %eax, %edx
 ; SSE-64-NEXT:    movd %edx, %xmm4
-; SSE-64-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,2,3]
-; SSE-64-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,2,3]
+; SSE-64-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,1,1]
+; SSE-64-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,1,1]
 ; SSE-64-NEXT:    ucomiss %xmm3, %xmm2
 ; SSE-64-NEXT:    cmovnel %eax, %ecx
 ; SSE-64-NEXT:    cmovpl %eax, %ecx
diff --git a/llvm/test/CodeGen/X86/vec-strict-fptoint-128.ll b/llvm/test/CodeGen/X86/vec-strict-fptoint-128.ll
index 10303209f71c4..227c33cf62120 100644
--- a/llvm/test/CodeGen/X86/vec-strict-fptoint-128.ll
+++ b/llvm/test/CodeGen/X86/vec-strict-fptoint-128.ll
@@ -541,7 +541,7 @@ define <2 x i64> @strict_vector_fptosi_v2f32_to_v2i64(<2 x float> %a) #0 {
 ; SSE-32-NEXT:    andl $-8, %esp
 ; SSE-32-NEXT:    subl $24, %esp
 ; SSE-32-NEXT:    movss %xmm0, {{[0-9]+}}(%esp)
-; SSE-32-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE-32-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE-32-NEXT:    movss %xmm0, {{[0-9]+}}(%esp)
 ; SSE-32-NEXT:    flds {{[0-9]+}}(%esp)
 ; SSE-32-NEXT:    wait
@@ -573,7 +573,7 @@ define <2 x i64> @strict_vector_fptosi_v2f32_to_v2i64(<2 x float> %a) #0 {
 ; SSE-64:       # %bb.0:
 ; SSE-64-NEXT:    cvttss2si %xmm0, %rax
 ; SSE-64-NEXT:    movq %rax, %xmm1
-; SSE-64-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE-64-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE-64-NEXT:    cvttss2si %xmm0, %rax
 ; SSE-64-NEXT:    movq %rax, %xmm0
 ; SSE-64-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
@@ -715,7 +715,7 @@ define <2 x i64> @strict_vector_fptosi_v2f32_to_v2i64_load128(<4 x float>* %x) s
 ; SSE-32-NEXT:    movl 8(%ebp), %eax
 ; SSE-32-NEXT:    movaps (%eax), %xmm0
 ; SSE-32-NEXT:    movss %xmm0, {{[0-9]+}}(%esp)
-; SSE-32-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE-32-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE-32-NEXT:    movss %xmm0, {{[0-9]+}}(%esp)
 ; SSE-32-NEXT:    flds {{[0-9]+}}(%esp)
 ; SSE-32-NEXT:    wait
@@ -748,7 +748,7 @@ define <2 x i64> @strict_vector_fptosi_v2f32_to_v2i64_load128(<4 x float>* %x) s
 ; SSE-64-NEXT:    movaps (%rdi), %xmm1
 ; SSE-64-NEXT:    cvttss2si %xmm1, %rax
 ; SSE-64-NEXT:    movq %rax, %xmm0
-; SSE-64-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; SSE-64-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,1,1]
 ; SSE-64-NEXT:    cvttss2si %xmm1, %rax
 ; SSE-64-NEXT:    movq %rax, %xmm1
 ; SSE-64-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
@@ -926,7 +926,7 @@ define <2 x i64> @strict_vector_fptoui_v2f32_to_v2i64(<2 x float> %a) #0 {
 ; SSE-32-NEXT:    fldcw {{[0-9]+}}(%esp)
 ; SSE-32-NEXT:    fistpll {{[0-9]+}}(%esp)
 ; SSE-32-NEXT:    fldcw {{[0-9]+}}(%esp)
-; SSE-32-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE-32-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE-32-NEXT:    comiss %xmm2, %xmm0
 ; SSE-32-NEXT:    jb .LBB4_4
 ; SSE-32-NEXT:  # %bb.3:
@@ -980,7 +980,7 @@ define <2 x i64> @strict_vector_fptoui_v2f32_to_v2i64(<2 x float> %a) #0 {
 ; SSE-64-NEXT:    shlq $63, %rcx
 ; SSE-64-NEXT:    xorq %rax, %rcx
 ; SSE-64-NEXT:    movq %rcx, %xmm1
-; SSE-64-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE-64-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE-64-NEXT:    comiss %xmm3, %xmm0
 ; SSE-64-NEXT:    jb .LBB4_4
 ; SSE-64-NEXT:  # %bb.3:
@@ -1246,7 +1246,7 @@ define <2 x i64> @strict_vector_fptoui_v2f32_to_v2i64_load128(<4 x float>* %x) s
 ; SSE-32-NEXT:    fldcw {{[0-9]+}}(%esp)
 ; SSE-32-NEXT:    fistpll {{[0-9]+}}(%esp)
 ; SSE-32-NEXT:    fldcw {{[0-9]+}}(%esp)
-; SSE-32-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE-32-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE-32-NEXT:    comiss %xmm2, %xmm0
 ; SSE-32-NEXT:    jb .LBB5_4
 ; SSE-32-NEXT:  # %bb.3:
@@ -1301,7 +1301,7 @@ define <2 x i64> @strict_vector_fptoui_v2f32_to_v2i64_load128(<4 x float>* %x) s
 ; SSE-64-NEXT:    shlq $63, %rcx
 ; SSE-64-NEXT:    xorq %rax, %rcx
 ; SSE-64-NEXT:    movq %rcx, %xmm0
-; SSE-64-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; SSE-64-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,1,1]
 ; SSE-64-NEXT:    comiss %xmm3, %xmm1
 ; SSE-64-NEXT:    jb .LBB5_4
 ; SSE-64-NEXT:  # %bb.3:
@@ -1769,7 +1769,7 @@ define <2 x i32> @strict_vector_fptoui_v2f32_to_v2i32(<2 x float> %a) #0 {
 ; SSE-32-NEXT:    cvttss2si %xmm4, %ecx
 ; SSE-32-NEXT:    xorl %eax, %ecx
 ; SSE-32-NEXT:    movd %ecx, %xmm1
-; SSE-32-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE-32-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE-32-NEXT:    comiss %xmm3, %xmm0
 ; SSE-32-NEXT:    jb .LBB9_4
 ; SSE-32-NEXT:  # %bb.3:
@@ -1790,7 +1790,7 @@ define <2 x i32> @strict_vector_fptoui_v2f32_to_v2i32(<2 x float> %a) #0 {
 ; SSE-64:       # %bb.0:
 ; SSE-64-NEXT:    cvttss2si %xmm0, %rax
 ; SSE-64-NEXT:    movd %eax, %xmm1
-; SSE-64-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE-64-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE-64-NEXT:    cvttss2si %xmm0, %rax
 ; SSE-64-NEXT:    movd %eax, %xmm0
 ; SSE-64-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
@@ -2645,7 +2645,7 @@ define <2 x i1> @strict_vector_fptosi_v2f32_to_v2i1(<2 x float> %a) #0 {
 ; SSE-32-NEXT:    andl $-8, %esp
 ; SSE-32-NEXT:    subl $24, %esp
 ; SSE-32-NEXT:    movss %xmm0, {{[0-9]+}}(%esp)
-; SSE-32-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE-32-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE-32-NEXT:    movss %xmm0, {{[0-9]+}}(%esp)
 ; SSE-32-NEXT:    flds {{[0-9]+}}(%esp)
 ; SSE-32-NEXT:    wait
@@ -2677,7 +2677,7 @@ define <2 x i1> @strict_vector_fptosi_v2f32_to_v2i1(<2 x float> %a) #0 {
 ; SSE-64:       # %bb.0:
 ; SSE-64-NEXT:    cvttss2si %xmm0, %rax
 ; SSE-64-NEXT:    movq %rax, %xmm1
-; SSE-64-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE-64-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE-64-NEXT:    cvttss2si %xmm0, %rax
 ; SSE-64-NEXT:    movq %rax, %xmm0
 ; SSE-64-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
@@ -2813,7 +2813,7 @@ define <2 x i1> @strict_vector_fptoui_v2f32_to_v2i1(<2 x float> %a) #0 {
 ; SSE-32-NEXT:    fldcw {{[0-9]+}}(%esp)
 ; SSE-32-NEXT:    fistpll {{[0-9]+}}(%esp)
 ; SSE-32-NEXT:    fldcw {{[0-9]+}}(%esp)
-; SSE-32-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE-32-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE-32-NEXT:    comiss %xmm2, %xmm0
 ; SSE-32-NEXT:    jb .LBB21_4
 ; SSE-32-NEXT:  # %bb.3:
@@ -2867,7 +2867,7 @@ define <2 x i1> @strict_vector_fptoui_v2f32_to_v2i1(<2 x float> %a) #0 {
 ; SSE-64-NEXT:    shlq $63, %rcx
 ; SSE-64-NEXT:    xorq %rax, %rcx
 ; SSE-64-NEXT:    movq %rcx, %xmm1
-; SSE-64-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE-64-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE-64-NEXT:    comiss %xmm3, %xmm0
 ; SSE-64-NEXT:    jb .LBB21_4
 ; SSE-64-NEXT:  # %bb.3:
diff --git a/llvm/test/CodeGen/X86/vec-strict-fptoint-256.ll b/llvm/test/CodeGen/X86/vec-strict-fptoint-256.ll
index 52313174fed79..0344615364843 100644
--- a/llvm/test/CodeGen/X86/vec-strict-fptoint-256.ll
+++ b/llvm/test/CodeGen/X86/vec-strict-fptoint-256.ll
@@ -638,7 +638,7 @@ define <4 x i64> @strict_vector_fptosi_v4f32_to_v4i64(<4 x float> %a) #0 {
 ;
 ; AVX-64-LABEL: strict_vector_fptosi_v4f32_to_v4i64:
 ; AVX-64:       # %bb.0:
-; AVX-64-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; AVX-64-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,3,3,3]
 ; AVX-64-NEXT:    vcvttss2si %xmm1, %rax
 ; AVX-64-NEXT:    vmovq %rax, %xmm1
 ; AVX-64-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
@@ -692,7 +692,7 @@ define <4 x i64> @strict_vector_fptosi_v4f32_to_v4i64(<4 x float> %a) #0 {
 ;
 ; AVX512F-64-LABEL: strict_vector_fptosi_v4f32_to_v4i64:
 ; AVX512F-64:       # %bb.0:
-; AVX512F-64-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; AVX512F-64-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,3,3,3]
 ; AVX512F-64-NEXT:    vcvttss2si %xmm1, %rax
 ; AVX512F-64-NEXT:    vmovq %rax, %xmm1
 ; AVX512F-64-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
@@ -746,7 +746,7 @@ define <4 x i64> @strict_vector_fptosi_v4f32_to_v4i64(<4 x float> %a) #0 {
 ;
 ; AVX512VL-64-LABEL: strict_vector_fptosi_v4f32_to_v4i64:
 ; AVX512VL-64:       # %bb.0:
-; AVX512VL-64-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; AVX512VL-64-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,3,3,3]
 ; AVX512VL-64-NEXT:    vcvttss2si %xmm1, %rax
 ; AVX512VL-64-NEXT:    vmovq %rax, %xmm1
 ; AVX512VL-64-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
@@ -806,7 +806,7 @@ define <4 x i64> @strict_vector_fptoui_v4f32_to_v4i64(<4 x float> %a) #0 {
 ; AVX-32-NEXT:    movzbl %al, %eax
 ; AVX-32-NEXT:    shll $31, %eax
 ; AVX-32-NEXT:    xorl {{[0-9]+}}(%esp), %eax
-; AVX-32-NEXT:    vpermilps {{.*#+}} xmm3 = xmm0[3,1,2,3]
+; AVX-32-NEXT:    vpermilps {{.*#+}} xmm3 = xmm0[3,3,3,3]
 ; AVX-32-NEXT:    vcomiss %xmm1, %xmm3
 ; AVX-32-NEXT:    vxorps %xmm4, %xmm4, %xmm4
 ; AVX-32-NEXT:    jb .LBB3_4
@@ -868,7 +868,7 @@ define <4 x i64> @strict_vector_fptoui_v4f32_to_v4i64(<4 x float> %a) #0 {
 ;
 ; AVX-64-LABEL: strict_vector_fptoui_v4f32_to_v4i64:
 ; AVX-64:       # %bb.0:
-; AVX-64-NEXT:    vpermilps {{.*#+}} xmm3 = xmm0[3,1,2,3]
+; AVX-64-NEXT:    vpermilps {{.*#+}} xmm3 = xmm0[3,3,3,3]
 ; AVX-64-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; AVX-64-NEXT:    vcomiss %xmm1, %xmm3
 ; AVX-64-NEXT:    vxorps %xmm2, %xmm2, %xmm2
@@ -960,7 +960,7 @@ define <4 x i64> @strict_vector_fptoui_v4f32_to_v4i64(<4 x float> %a) #0 {
 ; AVX512F-32-NEXT:    shll $31, %eax
 ; AVX512F-32-NEXT:    xorl {{[0-9]+}}(%esp), %eax
 ; AVX512F-32-NEXT:    movl %eax, %esi
-; AVX512F-32-NEXT:    vpermilps {{.*#+}} xmm2 = xmm0[3,1,2,3]
+; AVX512F-32-NEXT:    vpermilps {{.*#+}} xmm2 = xmm0[3,3,3,3]
 ; AVX512F-32-NEXT:    xorl %ecx, %ecx
 ; AVX512F-32-NEXT:    vcomiss %xmm1, %xmm2
 ; AVX512F-32-NEXT:    setb %dl
@@ -1021,7 +1021,7 @@ define <4 x i64> @strict_vector_fptoui_v4f32_to_v4i64(<4 x float> %a) #0 {
 ;
 ; AVX512F-64-LABEL: strict_vector_fptoui_v4f32_to_v4i64:
 ; AVX512F-64:       # %bb.0:
-; AVX512F-64-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; AVX512F-64-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,3,3,3]
 ; AVX512F-64-NEXT:    vcvttss2usi %xmm1, %rax
 ; AVX512F-64-NEXT:    vmovq %rax, %xmm1
 ; AVX512F-64-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
@@ -1068,7 +1068,7 @@ define <4 x i64> @strict_vector_fptoui_v4f32_to_v4i64(<4 x float> %a) #0 {
 ; AVX512VL-32-NEXT:    shll $31, %eax
 ; AVX512VL-32-NEXT:    xorl {{[0-9]+}}(%esp), %eax
 ; AVX512VL-32-NEXT:    movl %eax, %esi
-; AVX512VL-32-NEXT:    vpermilps {{.*#+}} xmm2 = xmm0[3,1,2,3]
+; AVX512VL-32-NEXT:    vpermilps {{.*#+}} xmm2 = xmm0[3,3,3,3]
 ; AVX512VL-32-NEXT:    xorl %ecx, %ecx
 ; AVX512VL-32-NEXT:    vcomiss %xmm1, %xmm2
 ; AVX512VL-32-NEXT:    setb %dl
@@ -1129,7 +1129,7 @@ define <4 x i64> @strict_vector_fptoui_v4f32_to_v4i64(<4 x float> %a) #0 {
 ;
 ; AVX512VL-64-LABEL: strict_vector_fptoui_v4f32_to_v4i64:
 ; AVX512VL-64:       # %bb.0:
-; AVX512VL-64-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; AVX512VL-64-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,3,3,3]
 ; AVX512VL-64-NEXT:    vcvttss2usi %xmm1, %rax
 ; AVX512VL-64-NEXT:    vmovq %rax, %xmm1
 ; AVX512VL-64-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
diff --git a/llvm/test/CodeGen/X86/vec-strict-fptoint-512.ll b/llvm/test/CodeGen/X86/vec-strict-fptoint-512.ll
index 52ba8deec1c4b..2791a5582f5bc 100644
--- a/llvm/test/CodeGen/X86/vec-strict-fptoint-512.ll
+++ b/llvm/test/CodeGen/X86/vec-strict-fptoint-512.ll
@@ -407,7 +407,7 @@ define <8 x i64> @strict_vector_fptosi_v8f32_to_v8i64(<8 x float> %a) #0 {
 ; AVX512VL-64-LABEL: strict_vector_fptosi_v8f32_to_v8i64:
 ; AVX512VL-64:       # %bb.0:
 ; AVX512VL-64-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX512VL-64-NEXT:    vpermilps {{.*#+}} xmm2 = xmm1[3,1,2,3]
+; AVX512VL-64-NEXT:    vpermilps {{.*#+}} xmm2 = xmm1[3,3,3,3]
 ; AVX512VL-64-NEXT:    vcvttss2si %xmm2, %rax
 ; AVX512VL-64-NEXT:    vmovq %rax, %xmm2
 ; AVX512VL-64-NEXT:    vpermilpd {{.*#+}} xmm3 = xmm1[1,0]
@@ -421,7 +421,7 @@ define <8 x i64> @strict_vector_fptosi_v8f32_to_v8i64(<8 x float> %a) #0 {
 ; AVX512VL-64-NEXT:    vmovq %rax, %xmm1
 ; AVX512VL-64-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm3[0],xmm1[0]
 ; AVX512VL-64-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
-; AVX512VL-64-NEXT:    vpermilps {{.*#+}} xmm2 = xmm0[3,1,2,3]
+; AVX512VL-64-NEXT:    vpermilps {{.*#+}} xmm2 = xmm0[3,3,3,3]
 ; AVX512VL-64-NEXT:    vcvttss2si %xmm2, %rax
 ; AVX512VL-64-NEXT:    vmovq %rax, %xmm2
 ; AVX512VL-64-NEXT:    vpermilpd {{.*#+}} xmm3 = xmm0[1,0]
@@ -481,7 +481,7 @@ define <8 x i64> @strict_vector_fptoui_v8f32_to_v8i64(<8 x float> %a) #0 {
 ; AVX512VL-32-NEXT:    shll $31, %eax
 ; AVX512VL-32-NEXT:    xorl {{[0-9]+}}(%esp), %eax
 ; AVX512VL-32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; AVX512VL-32-NEXT:    vpermilps {{.*#+}} xmm3 = xmm0[3,1,2,3]
+; AVX512VL-32-NEXT:    vpermilps {{.*#+}} xmm3 = xmm0[3,3,3,3]
 ; AVX512VL-32-NEXT:    vcomiss %xmm1, %xmm3
 ; AVX512VL-32-NEXT:    setb %al
 ; AVX512VL-32-NEXT:    kmovw %eax, %k1
@@ -545,7 +545,7 @@ define <8 x i64> @strict_vector_fptoui_v8f32_to_v8i64(<8 x float> %a) #0 {
 ; AVX512VL-32-NEXT:    shll $31, %eax
 ; AVX512VL-32-NEXT:    xorl {{[0-9]+}}(%esp), %eax
 ; AVX512VL-32-NEXT:    movl %eax, %esi
-; AVX512VL-32-NEXT:    vpermilps {{.*#+}} xmm4 = xmm3[3,1,2,3]
+; AVX512VL-32-NEXT:    vpermilps {{.*#+}} xmm4 = xmm3[3,3,3,3]
 ; AVX512VL-32-NEXT:    xorl %edx, %edx
 ; AVX512VL-32-NEXT:    vcomiss %xmm1, %xmm4
 ; AVX512VL-32-NEXT:    setb %al
@@ -618,7 +618,7 @@ define <8 x i64> @strict_vector_fptoui_v8f32_to_v8i64(<8 x float> %a) #0 {
 ; AVX512VL-64-LABEL: strict_vector_fptoui_v8f32_to_v8i64:
 ; AVX512VL-64:       # %bb.0:
 ; AVX512VL-64-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX512VL-64-NEXT:    vpermilps {{.*#+}} xmm2 = xmm1[3,1,2,3]
+; AVX512VL-64-NEXT:    vpermilps {{.*#+}} xmm2 = xmm1[3,3,3,3]
 ; AVX512VL-64-NEXT:    vcvttss2usi %xmm2, %rax
 ; AVX512VL-64-NEXT:    vmovq %rax, %xmm2
 ; AVX512VL-64-NEXT:    vpermilpd {{.*#+}} xmm3 = xmm1[1,0]
@@ -632,7 +632,7 @@ define <8 x i64> @strict_vector_fptoui_v8f32_to_v8i64(<8 x float> %a) #0 {
 ; AVX512VL-64-NEXT:    vmovq %rax, %xmm1
 ; AVX512VL-64-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm3[0],xmm1[0]
 ; AVX512VL-64-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
-; AVX512VL-64-NEXT:    vpermilps {{.*#+}} xmm2 = xmm0[3,1,2,3]
+; AVX512VL-64-NEXT:    vpermilps {{.*#+}} xmm2 = xmm0[3,3,3,3]
 ; AVX512VL-64-NEXT:    vcvttss2usi %xmm2, %rax
 ; AVX512VL-64-NEXT:    vmovq %rax, %xmm2
 ; AVX512VL-64-NEXT:    vpermilpd {{.*#+}} xmm3 = xmm0[1,0]
diff --git a/llvm/test/CodeGen/X86/vec-strict-inttofp-128.ll b/llvm/test/CodeGen/X86/vec-strict-inttofp-128.ll
index 94f7d7eeaf39d..0f8881ad25c37 100644
--- a/llvm/test/CodeGen/X86/vec-strict-inttofp-128.ll
+++ b/llvm/test/CodeGen/X86/vec-strict-inttofp-128.ll
@@ -274,14 +274,14 @@ define <2 x float> @uitofp_v2i64_v2f32(<2 x i64> %x) #0 {
 ; SSE-32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-32-NEXT:    movq %xmm1, {{[0-9]+}}(%esp)
 ; SSE-32-NEXT:    movq %xmm0, {{[0-9]+}}(%esp)
-; SSE-32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; SSE-32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
 ; SSE-32-NEXT:    movd %xmm1, %eax
 ; SSE-32-NEXT:    shrl $31, %eax
 ; SSE-32-NEXT:    fildll {{[0-9]+}}(%esp)
 ; SSE-32-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
 ; SSE-32-NEXT:    fstps (%esp)
 ; SSE-32-NEXT:    wait
-; SSE-32-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE-32-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE-32-NEXT:    movd %xmm0, %eax
 ; SSE-32-NEXT:    shrl $31, %eax
 ; SSE-32-NEXT:    fildll {{[0-9]+}}(%esp)
@@ -343,14 +343,14 @@ define <2 x float> @uitofp_v2i64_v2f32(<2 x i64> %x) #0 {
 ; SSE41-32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE41-32-NEXT:    movq %xmm1, {{[0-9]+}}(%esp)
 ; SSE41-32-NEXT:    movq %xmm0, {{[0-9]+}}(%esp)
-; SSE41-32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; SSE41-32-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
 ; SSE41-32-NEXT:    movd %xmm1, %eax
 ; SSE41-32-NEXT:    shrl $31, %eax
 ; SSE41-32-NEXT:    fildll {{[0-9]+}}(%esp)
 ; SSE41-32-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
 ; SSE41-32-NEXT:    fstps (%esp)
 ; SSE41-32-NEXT:    wait
-; SSE41-32-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE41-32-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE41-32-NEXT:    movd %xmm0, %eax
 ; SSE41-32-NEXT:    shrl $31, %eax
 ; SSE41-32-NEXT:    fildll {{[0-9]+}}(%esp)
diff --git a/llvm/test/CodeGen/X86/vec-strict-inttofp-256.ll b/llvm/test/CodeGen/X86/vec-strict-inttofp-256.ll
index a5519e68f73a3..00be3baedac89 100644
--- a/llvm/test/CodeGen/X86/vec-strict-inttofp-256.ll
+++ b/llvm/test/CodeGen/X86/vec-strict-inttofp-256.ll
@@ -191,7 +191,7 @@ define <8 x float> @sitofp_v8i8_v8f32(<8 x i8> %x) #0 {
 ; AVX1-LABEL: sitofp_v8i8_v8f32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    vcvtdq2ps %ymm0, %ymm0
@@ -236,7 +236,7 @@ define <8 x float> @uitofp_v8i8_v8f32(<8 x i8> %x) #0 {
 ; AVX1-LABEL: uitofp_v8i8_v8f32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    vcvtdq2ps %ymm0, %ymm0
diff --git a/llvm/test/CodeGen/X86/vec3.ll b/llvm/test/CodeGen/X86/vec3.ll
index 6d3f71f47505d..b4069b5510681 100644
--- a/llvm/test/CodeGen/X86/vec3.ll
+++ b/llvm/test/CodeGen/X86/vec3.ll
@@ -4,7 +4,7 @@
 define <3 x float> @fadd(<3 x float> %v, float %d) {
 ; CHECK-LABEL: fadd:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,0,0,3]
+; CHECK-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; CHECK-NEXT:    addps %xmm1, %xmm0
 ; CHECK-NEXT:    retq
   %ins = insertelement <3 x float> undef, float %d, i32 0
@@ -18,7 +18,7 @@ define <3 x float> @fadd(<3 x float> %v, float %d) {
 define <3 x float> @fdiv(<3 x float> %v, float %d) {
 ; CHECK-LABEL: fdiv:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,0,0,3]
+; CHECK-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; CHECK-NEXT:    divps %xmm0, %xmm1
 ; CHECK-NEXT:    movaps %xmm1, %xmm0
 ; CHECK-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/vec_cast2.ll b/llvm/test/CodeGen/X86/vec_cast2.ll
index c979050fc1809..5a6f7e2814895 100644
--- a/llvm/test/CodeGen/X86/vec_cast2.ll
+++ b/llvm/test/CodeGen/X86/vec_cast2.ll
@@ -5,7 +5,7 @@ define <8 x float> @cvt_v8i8_v8f32(<8 x i8> %src) {
 ; CHECK-LABEL: cvt_v8i8_v8f32:
 ; CHECK:       ## %bb.0:
 ; CHECK-NEXT:    vpmovsxbd %xmm0, %xmm1
-; CHECK-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; CHECK-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; CHECK-NEXT:    vpmovsxbd %xmm0, %xmm0
 ; CHECK-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; CHECK-NEXT:    vcvtdq2ps %ymm0, %ymm0
@@ -51,7 +51,7 @@ define <8 x float> @cvt_v8u8_v8f32(<8 x i8> %src) {
 ; CHECK-LABEL: cvt_v8u8_v8f32:
 ; CHECK:       ## %bb.0:
 ; CHECK-NEXT:    vpmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; CHECK-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; CHECK-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; CHECK-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; CHECK-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; CHECK-NEXT:    vcvtdq2ps %ymm0, %ymm0
diff --git a/llvm/test/CodeGen/X86/vec_extract-mmx.ll b/llvm/test/CodeGen/X86/vec_extract-mmx.ll
index 5ce632d0aa149..af069b0838af0 100644
--- a/llvm/test/CodeGen/X86/vec_extract-mmx.ll
+++ b/llvm/test/CodeGen/X86/vec_extract-mmx.ll
@@ -114,14 +114,14 @@ define i32 @test4(x86_mmx %a) nounwind {
 ; X32-LABEL: test4:
 ; X32:       # %bb.0:
 ; X32-NEXT:    movq2dq %mm0, %xmm0
-; X32-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; X32-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; X32-NEXT:    movd %xmm0, %eax
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: test4:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movq2dq %mm0, %xmm0
-; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; X64-NEXT:    movd %xmm0, %eax
 ; X64-NEXT:    retq
   %tmp0 = bitcast x86_mmx %a to <2 x i32>
diff --git a/llvm/test/CodeGen/X86/vec_extract.ll b/llvm/test/CodeGen/X86/vec_extract.ll
index 9b347c12194c9..46cd15357c72c 100644
--- a/llvm/test/CodeGen/X86/vec_extract.ll
+++ b/llvm/test/CodeGen/X86/vec_extract.ll
@@ -58,14 +58,14 @@ define void @test3(float* %R, <4 x float>* %P1) nounwind {
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X32-NEXT:    movaps (%ecx), %xmm0
-; X32-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; X32-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; X32-NEXT:    movss %xmm0, (%eax)
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: test3:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    movaps (%rsi), %xmm0
-; X64-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; X64-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; X64-NEXT:    movss %xmm0, (%rdi)
 ; X64-NEXT:    retq
 entry:
diff --git a/llvm/test/CodeGen/X86/vec_fp_to_int.ll b/llvm/test/CodeGen/X86/vec_fp_to_int.ll
index 4722355cc89c4..0e0ba1f6b0205 100644
--- a/llvm/test/CodeGen/X86/vec_fp_to_int.ll
+++ b/llvm/test/CodeGen/X86/vec_fp_to_int.ll
@@ -845,7 +845,7 @@ define <2 x i64> @fptosi_2f32_to_2i64(<4 x float> %a) {
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    cvttss2si %xmm0, %rax
 ; SSE-NEXT:    movq %rax, %xmm1
-; SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE-NEXT:    cvttss2si %xmm0, %rax
 ; SSE-NEXT:    movq %rax, %xmm0
 ; SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
@@ -904,7 +904,7 @@ define <2 x i64> @fptosi_4f32_to_2i64(<4 x float> %a) {
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    cvttss2si %xmm0, %rax
 ; SSE-NEXT:    movq %rax, %xmm1
-; SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE-NEXT:    cvttss2si %xmm0, %rax
 ; SSE-NEXT:    movq %rax, %xmm0
 ; SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
@@ -981,12 +981,12 @@ define <4 x i64> @fptosi_4f32_to_4i64(<8 x float> %a) {
 ; SSE-NEXT:    cvttss2si %xmm0, %rax
 ; SSE-NEXT:    movq %rax, %xmm2
 ; SSE-NEXT:    movaps %xmm0, %xmm1
-; SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[2,3]
+; SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[1,1]
 ; SSE-NEXT:    cvttss2si %xmm1, %rax
 ; SSE-NEXT:    movq %rax, %xmm1
 ; SSE-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0]
 ; SSE-NEXT:    movaps %xmm0, %xmm1
-; SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1],xmm0[2,3]
+; SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3],xmm0[3,3]
 ; SSE-NEXT:    cvttss2si %xmm1, %rax
 ; SSE-NEXT:    movq %rax, %xmm3
 ; SSE-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
@@ -998,7 +998,7 @@ define <4 x i64> @fptosi_4f32_to_4i64(<8 x float> %a) {
 ;
 ; AVX1-LABEL: fptosi_4f32_to_4i64:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,3,3,3]
 ; AVX1-NEXT:    vcvttss2si %xmm1, %rax
 ; AVX1-NEXT:    vmovq %rax, %xmm1
 ; AVX1-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
@@ -1016,7 +1016,7 @@ define <4 x i64> @fptosi_4f32_to_4i64(<8 x float> %a) {
 ;
 ; AVX2-LABEL: fptosi_4f32_to_4i64:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,3,3,3]
 ; AVX2-NEXT:    vcvttss2si %xmm1, %rax
 ; AVX2-NEXT:    vmovq %rax, %xmm1
 ; AVX2-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
@@ -1034,7 +1034,7 @@ define <4 x i64> @fptosi_4f32_to_4i64(<8 x float> %a) {
 ;
 ; AVX512F-LABEL: fptosi_4f32_to_4i64:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; AVX512F-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,3,3,3]
 ; AVX512F-NEXT:    vcvttss2si %xmm1, %rax
 ; AVX512F-NEXT:    vmovq %rax, %xmm1
 ; AVX512F-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
@@ -1052,7 +1052,7 @@ define <4 x i64> @fptosi_4f32_to_4i64(<8 x float> %a) {
 ;
 ; AVX512VL-LABEL: fptosi_4f32_to_4i64:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; AVX512VL-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,3,3,3]
 ; AVX512VL-NEXT:    vcvttss2si %xmm1, %rax
 ; AVX512VL-NEXT:    vmovq %rax, %xmm1
 ; AVX512VL-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
@@ -1089,12 +1089,12 @@ define <4 x i64> @fptosi_8f32_to_4i64(<8 x float> %a) {
 ; SSE-NEXT:    cvttss2si %xmm0, %rax
 ; SSE-NEXT:    movq %rax, %xmm2
 ; SSE-NEXT:    movaps %xmm0, %xmm1
-; SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[2,3]
+; SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[1,1]
 ; SSE-NEXT:    cvttss2si %xmm1, %rax
 ; SSE-NEXT:    movq %rax, %xmm1
 ; SSE-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0]
 ; SSE-NEXT:    movaps %xmm0, %xmm1
-; SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1],xmm0[2,3]
+; SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3],xmm0[3,3]
 ; SSE-NEXT:    cvttss2si %xmm1, %rax
 ; SSE-NEXT:    movq %rax, %xmm3
 ; SSE-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
@@ -1106,7 +1106,7 @@ define <4 x i64> @fptosi_8f32_to_4i64(<8 x float> %a) {
 ;
 ; AVX1-LABEL: fptosi_8f32_to_4i64:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,3,3,3]
 ; AVX1-NEXT:    vcvttss2si %xmm1, %rax
 ; AVX1-NEXT:    vmovq %rax, %xmm1
 ; AVX1-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
@@ -1124,7 +1124,7 @@ define <4 x i64> @fptosi_8f32_to_4i64(<8 x float> %a) {
 ;
 ; AVX2-LABEL: fptosi_8f32_to_4i64:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,3,3,3]
 ; AVX2-NEXT:    vcvttss2si %xmm1, %rax
 ; AVX2-NEXT:    vmovq %rax, %xmm1
 ; AVX2-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
@@ -1147,7 +1147,7 @@ define <4 x i64> @fptosi_8f32_to_4i64(<8 x float> %a) {
 ; AVX512F-NEXT:    vcvttss2si %xmm0, %rcx
 ; AVX512F-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
 ; AVX512F-NEXT:    vcvttss2si %xmm1, %rdx
-; AVX512F-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX512F-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX512F-NEXT:    vcvttss2si %xmm0, %rsi
 ; AVX512F-NEXT:    vmovq %rsi, %xmm0
 ; AVX512F-NEXT:    vmovq %rdx, %xmm1
@@ -1165,7 +1165,7 @@ define <4 x i64> @fptosi_8f32_to_4i64(<8 x float> %a) {
 ; AVX512VL-NEXT:    vcvttss2si %xmm0, %rcx
 ; AVX512VL-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
 ; AVX512VL-NEXT:    vcvttss2si %xmm1, %rdx
-; AVX512VL-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX512VL-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX512VL-NEXT:    vcvttss2si %xmm0, %rsi
 ; AVX512VL-NEXT:    vmovq %rsi, %xmm0
 ; AVX512VL-NEXT:    vmovq %rdx, %xmm1
@@ -1345,7 +1345,7 @@ define <2 x i64> @fptoui_2f32_to_2i64(<4 x float> %a) {
 ; SSE-NEXT:    ucomiss %xmm2, %xmm0
 ; SSE-NEXT:    cmovaeq %rax, %rdx
 ; SSE-NEXT:    movq %rdx, %xmm1
-; SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE-NEXT:    movaps %xmm0, %xmm3
 ; SSE-NEXT:    subss %xmm2, %xmm3
 ; SSE-NEXT:    cvttss2si %xmm3, %rax
@@ -1430,7 +1430,7 @@ define <2 x i64> @fptoui_4f32_to_2i64(<4 x float> %a) {
 ; SSE-NEXT:    ucomiss %xmm2, %xmm0
 ; SSE-NEXT:    cmovaeq %rax, %rdx
 ; SSE-NEXT:    movq %rdx, %xmm1
-; SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE-NEXT:    movaps %xmm0, %xmm3
 ; SSE-NEXT:    subss %xmm2, %xmm3
 ; SSE-NEXT:    cvttss2si %xmm3, %rax
@@ -1595,7 +1595,7 @@ define <4 x i64> @fptoui_4f32_to_4i64(<8 x float> %a) {
 ; SSE-NEXT:    cmovaeq %rcx, %rdx
 ; SSE-NEXT:    movq %rdx, %xmm2
 ; SSE-NEXT:    movaps %xmm0, %xmm3
-; SSE-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm0[2,3]
+; SSE-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm0[1,1]
 ; SSE-NEXT:    movaps %xmm3, %xmm4
 ; SSE-NEXT:    subss %xmm1, %xmm4
 ; SSE-NEXT:    cvttss2si %xmm4, %rcx
@@ -1606,7 +1606,7 @@ define <4 x i64> @fptoui_4f32_to_4i64(<8 x float> %a) {
 ; SSE-NEXT:    movq %rdx, %xmm3
 ; SSE-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
 ; SSE-NEXT:    movaps %xmm0, %xmm3
-; SSE-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,1],xmm0[2,3]
+; SSE-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,3],xmm0[3,3]
 ; SSE-NEXT:    movaps %xmm3, %xmm4
 ; SSE-NEXT:    subss %xmm1, %xmm4
 ; SSE-NEXT:    cvttss2si %xmm4, %rcx
@@ -1630,7 +1630,7 @@ define <4 x i64> @fptoui_4f32_to_4i64(<8 x float> %a) {
 ;
 ; AVX1-LABEL: fptoui_4f32_to_4i64:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm2 = xmm0[3,1,2,3]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm2 = xmm0[3,3,3,3]
 ; AVX1-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; AVX1-NEXT:    vsubss %xmm1, %xmm2, %xmm3
 ; AVX1-NEXT:    vcvttss2si %xmm3, %rax
@@ -1670,7 +1670,7 @@ define <4 x i64> @fptoui_4f32_to_4i64(<8 x float> %a) {
 ;
 ; AVX2-LABEL: fptoui_4f32_to_4i64:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpermilps {{.*#+}} xmm2 = xmm0[3,1,2,3]
+; AVX2-NEXT:    vpermilps {{.*#+}} xmm2 = xmm0[3,3,3,3]
 ; AVX2-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; AVX2-NEXT:    vsubss %xmm1, %xmm2, %xmm3
 ; AVX2-NEXT:    vcvttss2si %xmm3, %rax
@@ -1710,7 +1710,7 @@ define <4 x i64> @fptoui_4f32_to_4i64(<8 x float> %a) {
 ;
 ; AVX512F-LABEL: fptoui_4f32_to_4i64:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; AVX512F-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,3,3,3]
 ; AVX512F-NEXT:    vcvttss2usi %xmm1, %rax
 ; AVX512F-NEXT:    vmovq %rax, %xmm1
 ; AVX512F-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
@@ -1728,7 +1728,7 @@ define <4 x i64> @fptoui_4f32_to_4i64(<8 x float> %a) {
 ;
 ; AVX512VL-LABEL: fptoui_4f32_to_4i64:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; AVX512VL-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,3,3,3]
 ; AVX512VL-NEXT:    vcvttss2usi %xmm1, %rax
 ; AVX512VL-NEXT:    vmovq %rax, %xmm1
 ; AVX512VL-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
@@ -1773,7 +1773,7 @@ define <4 x i64> @fptoui_8f32_to_4i64(<8 x float> %a) {
 ; SSE-NEXT:    cmovaeq %rcx, %rdx
 ; SSE-NEXT:    movq %rdx, %xmm2
 ; SSE-NEXT:    movaps %xmm0, %xmm3
-; SSE-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm0[2,3]
+; SSE-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm0[1,1]
 ; SSE-NEXT:    movaps %xmm3, %xmm4
 ; SSE-NEXT:    subss %xmm1, %xmm4
 ; SSE-NEXT:    cvttss2si %xmm4, %rcx
@@ -1784,7 +1784,7 @@ define <4 x i64> @fptoui_8f32_to_4i64(<8 x float> %a) {
 ; SSE-NEXT:    movq %rdx, %xmm3
 ; SSE-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
 ; SSE-NEXT:    movaps %xmm0, %xmm3
-; SSE-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,1],xmm0[2,3]
+; SSE-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,3],xmm0[3,3]
 ; SSE-NEXT:    movaps %xmm3, %xmm4
 ; SSE-NEXT:    subss %xmm1, %xmm4
 ; SSE-NEXT:    cvttss2si %xmm4, %rcx
@@ -1808,7 +1808,7 @@ define <4 x i64> @fptoui_8f32_to_4i64(<8 x float> %a) {
 ;
 ; AVX1-LABEL: fptoui_8f32_to_4i64:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm2 = xmm0[3,1,2,3]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm2 = xmm0[3,3,3,3]
 ; AVX1-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; AVX1-NEXT:    vsubss %xmm1, %xmm2, %xmm3
 ; AVX1-NEXT:    vcvttss2si %xmm3, %rax
@@ -1848,7 +1848,7 @@ define <4 x i64> @fptoui_8f32_to_4i64(<8 x float> %a) {
 ;
 ; AVX2-LABEL: fptoui_8f32_to_4i64:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpermilps {{.*#+}} xmm2 = xmm0[3,1,2,3]
+; AVX2-NEXT:    vpermilps {{.*#+}} xmm2 = xmm0[3,3,3,3]
 ; AVX2-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; AVX2-NEXT:    vsubss %xmm1, %xmm2, %xmm3
 ; AVX2-NEXT:    vcvttss2si %xmm3, %rax
@@ -1893,7 +1893,7 @@ define <4 x i64> @fptoui_8f32_to_4i64(<8 x float> %a) {
 ; AVX512F-NEXT:    vcvttss2usi %xmm0, %rcx
 ; AVX512F-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
 ; AVX512F-NEXT:    vcvttss2usi %xmm1, %rdx
-; AVX512F-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX512F-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX512F-NEXT:    vcvttss2usi %xmm0, %rsi
 ; AVX512F-NEXT:    vmovq %rsi, %xmm0
 ; AVX512F-NEXT:    vmovq %rdx, %xmm1
@@ -1911,7 +1911,7 @@ define <4 x i64> @fptoui_8f32_to_4i64(<8 x float> %a) {
 ; AVX512VL-NEXT:    vcvttss2usi %xmm0, %rcx
 ; AVX512VL-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
 ; AVX512VL-NEXT:    vcvttss2usi %xmm1, %rdx
-; AVX512VL-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX512VL-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX512VL-NEXT:    vcvttss2usi %xmm0, %rsi
 ; AVX512VL-NEXT:    vmovq %rsi, %xmm0
 ; AVX512VL-NEXT:    vmovq %rdx, %xmm1
@@ -2735,7 +2735,7 @@ define <2 x i64> @fptosi_2f32_to_2i64_load(<2 x float>* %x) {
 ; SSE-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
 ; SSE-NEXT:    cvttss2si %xmm1, %rax
 ; SSE-NEXT:    movq %rax, %xmm0
-; SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,1,1]
 ; SSE-NEXT:    cvttss2si %xmm1, %rax
 ; SSE-NEXT:    movq %rax, %xmm1
 ; SSE-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
@@ -2805,7 +2805,7 @@ define <2 x i64> @fptoui_2f32_to_2i64_load(<2 x float>* %x) {
 ; SSE-NEXT:    ucomiss %xmm2, %xmm1
 ; SSE-NEXT:    cmovaeq %rax, %rdx
 ; SSE-NEXT:    movq %rdx, %xmm0
-; SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,1,1]
 ; SSE-NEXT:    movaps %xmm1, %xmm3
 ; SSE-NEXT:    subss %xmm2, %xmm3
 ; SSE-NEXT:    cvttss2si %xmm3, %rax
diff --git a/llvm/test/CodeGen/X86/vec_int_to_fp.ll b/llvm/test/CodeGen/X86/vec_int_to_fp.ll
index a413752993b5a..d3d51a35d337e 100644
--- a/llvm/test/CodeGen/X86/vec_int_to_fp.ll
+++ b/llvm/test/CodeGen/X86/vec_int_to_fp.ll
@@ -1535,7 +1535,7 @@ define <4 x float> @sitofp_16i8_to_4f32(<16 x i8> %a) {
 ; AVX1-LABEL: sitofp_16i8_to_4f32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    vcvtdq2ps %ymm0, %ymm0
@@ -1765,7 +1765,7 @@ define <8 x float> @sitofp_8i8_to_8f32(<16 x i8> %a) {
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pmovsxbd %xmm0, %xmm1
 ; SSE41-NEXT:    cvtdq2ps %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE41-NEXT:    pmovsxbd %xmm0, %xmm0
 ; SSE41-NEXT:    cvtdq2ps %xmm0, %xmm1
 ; SSE41-NEXT:    movaps %xmm2, %xmm0
@@ -1774,7 +1774,7 @@ define <8 x float> @sitofp_8i8_to_8f32(<16 x i8> %a) {
 ; AVX1-LABEL: sitofp_8i8_to_8f32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    vcvtdq2ps %ymm0, %ymm0
@@ -1812,7 +1812,7 @@ define <8 x float> @sitofp_16i8_to_8f32(<16 x i8> %a) {
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pmovsxbd %xmm0, %xmm1
 ; SSE41-NEXT:    cvtdq2ps %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE41-NEXT:    pmovsxbd %xmm0, %xmm0
 ; SSE41-NEXT:    cvtdq2ps %xmm0, %xmm1
 ; SSE41-NEXT:    movaps %xmm2, %xmm0
@@ -1821,7 +1821,7 @@ define <8 x float> @sitofp_16i8_to_8f32(<16 x i8> %a) {
 ; AVX1-LABEL: sitofp_16i8_to_8f32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    vcvtdq2ps %ymm0, %ymm0
@@ -2417,7 +2417,7 @@ define <4 x float> @uitofp_16i8_to_4f32(<16 x i8> %a) {
 ; AVX1-LABEL: uitofp_16i8_to_4f32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    vcvtdq2ps %ymm0, %ymm0
@@ -2816,7 +2816,7 @@ define <8 x float> @uitofp_8i8_to_8f32(<16 x i8> %a) {
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE41-NEXT:    cvtdq2ps %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE41-NEXT:    cvtdq2ps %xmm0, %xmm1
 ; SSE41-NEXT:    movaps %xmm2, %xmm0
@@ -2825,7 +2825,7 @@ define <8 x float> @uitofp_8i8_to_8f32(<16 x i8> %a) {
 ; AVX1-LABEL: uitofp_8i8_to_8f32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    vcvtdq2ps %ymm0, %ymm0
@@ -2864,7 +2864,7 @@ define <8 x float> @uitofp_16i8_to_8f32(<16 x i8> %a) {
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE41-NEXT:    cvtdq2ps %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE41-NEXT:    cvtdq2ps %xmm0, %xmm1
 ; SSE41-NEXT:    movaps %xmm2, %xmm0
@@ -2873,7 +2873,7 @@ define <8 x float> @uitofp_16i8_to_8f32(<16 x i8> %a) {
 ; AVX1-LABEL: uitofp_16i8_to_8f32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    vcvtdq2ps %ymm0, %ymm0
@@ -5593,13 +5593,13 @@ define double @extract0_uitofp_v4i32_f64(<4 x i32> %x) nounwind {
 define float @extract3_sitofp_v4i32_f32(<4 x i32> %x) nounwind {
 ; SSE-LABEL: extract3_sitofp_v4i32_f32:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE-NEXT:    cvtdq2ps %xmm0, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: extract3_sitofp_v4i32_f32:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX-NEXT:    vcvtdq2ps %xmm0, %xmm0
 ; AVX-NEXT:    retq
   %e = extractelement <4 x i32> %x, i32 3
@@ -5610,7 +5610,7 @@ define float @extract3_sitofp_v4i32_f32(<4 x i32> %x) nounwind {
 define double @extract3_sitofp_v4i32_f64(<4 x i32> %x) nounwind {
 ; SSE2-LABEL: extract3_sitofp_v4i32_f64:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE2-NEXT:    movd %xmm0, %eax
 ; SSE2-NEXT:    xorps %xmm0, %xmm0
 ; SSE2-NEXT:    cvtsi2sd %eax, %xmm0
@@ -5625,7 +5625,7 @@ define double @extract3_sitofp_v4i32_f64(<4 x i32> %x) nounwind {
 ;
 ; AVX-LABEL: extract3_sitofp_v4i32_f64:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX-NEXT:    vcvtdq2pd %xmm0, %xmm0
 ; AVX-NEXT:    retq
   %e = extractelement <4 x i32> %x, i32 3
@@ -5636,7 +5636,7 @@ define double @extract3_sitofp_v4i32_f64(<4 x i32> %x) nounwind {
 define float @extract3_uitofp_v4i32_f32(<4 x i32> %x) nounwind {
 ; SSE2-LABEL: extract3_uitofp_v4i32_f32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE2-NEXT:    movd %xmm0, %eax
 ; SSE2-NEXT:    xorps %xmm0, %xmm0
 ; SSE2-NEXT:    cvtsi2ss %rax, %xmm0
@@ -5657,7 +5657,7 @@ define float @extract3_uitofp_v4i32_f32(<4 x i32> %x) nounwind {
 ;
 ; AVX512F-LABEL: extract3_uitofp_v4i32_f32:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX512F-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX512F-NEXT:    vcvtudq2ps %zmm0, %zmm0
 ; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512F-NEXT:    vzeroupper
@@ -5665,13 +5665,13 @@ define float @extract3_uitofp_v4i32_f32(<4 x i32> %x) nounwind {
 ;
 ; AVX512VL-LABEL: extract3_uitofp_v4i32_f32:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX512VL-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX512VL-NEXT:    vcvtudq2ps %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512DQ-LABEL: extract3_uitofp_v4i32_f32:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX512DQ-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX512DQ-NEXT:    vcvtudq2ps %zmm0, %zmm0
 ; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512DQ-NEXT:    vzeroupper
@@ -5679,7 +5679,7 @@ define float @extract3_uitofp_v4i32_f32(<4 x i32> %x) nounwind {
 ;
 ; AVX512VLDQ-LABEL: extract3_uitofp_v4i32_f32:
 ; AVX512VLDQ:       # %bb.0:
-; AVX512VLDQ-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX512VLDQ-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX512VLDQ-NEXT:    vcvtudq2ps %xmm0, %xmm0
 ; AVX512VLDQ-NEXT:    retq
   %e = extractelement <4 x i32> %x, i32 3
@@ -5690,7 +5690,7 @@ define float @extract3_uitofp_v4i32_f32(<4 x i32> %x) nounwind {
 define double @extract3_uitofp_v4i32_f64(<4 x i32> %x) nounwind {
 ; SSE2-LABEL: extract3_uitofp_v4i32_f64:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE2-NEXT:    movd %xmm0, %eax
 ; SSE2-NEXT:    xorps %xmm0, %xmm0
 ; SSE2-NEXT:    cvtsi2sd %rax, %xmm0
@@ -5711,7 +5711,7 @@ define double @extract3_uitofp_v4i32_f64(<4 x i32> %x) nounwind {
 ;
 ; AVX512F-LABEL: extract3_uitofp_v4i32_f64:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX512F-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX512F-NEXT:    vcvtudq2pd %ymm0, %zmm0
 ; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512F-NEXT:    vzeroupper
@@ -5719,13 +5719,13 @@ define double @extract3_uitofp_v4i32_f64(<4 x i32> %x) nounwind {
 ;
 ; AVX512VL-LABEL: extract3_uitofp_v4i32_f64:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX512VL-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX512VL-NEXT:    vcvtudq2pd %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512DQ-LABEL: extract3_uitofp_v4i32_f64:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX512DQ-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX512DQ-NEXT:    vcvtudq2pd %ymm0, %zmm0
 ; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512DQ-NEXT:    vzeroupper
@@ -5733,7 +5733,7 @@ define double @extract3_uitofp_v4i32_f64(<4 x i32> %x) nounwind {
 ;
 ; AVX512VLDQ-LABEL: extract3_uitofp_v4i32_f64:
 ; AVX512VLDQ:       # %bb.0:
-; AVX512VLDQ-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX512VLDQ-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX512VLDQ-NEXT:    vcvtudq2pd %xmm0, %xmm0
 ; AVX512VLDQ-NEXT:    retq
   %e = extractelement <4 x i32> %x, i32 3
diff --git a/llvm/test/CodeGen/X86/vec_saddo.ll b/llvm/test/CodeGen/X86/vec_saddo.ll
index 4f071c064e5c8..e1f780da4fce6 100644
--- a/llvm/test/CodeGen/X86/vec_saddo.ll
+++ b/llvm/test/CodeGen/X86/vec_saddo.ll
@@ -515,12 +515,12 @@ define <16 x i32> @saddo_v16i32(<16 x i32> %a0, <16 x i32> %a1, <16 x i32>* %p2)
 ; AVX1-NEXT:    vpackssdw %xmm4, %xmm0, %xmm0
 ; AVX1-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm4
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm4, %ymm0
 ; AVX1-NEXT:    vpacksswb %xmm1, %xmm1, %xmm1
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm4
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm4, %ymm1
 ; AVX1-NEXT:    vmovdqa %xmm8, 48(%rdi)
@@ -640,7 +640,7 @@ define <16 x i32> @saddo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nou
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm4 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm4
 ; SSE41-NEXT:    psrad $31, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,1,1]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm1
 ; SSE41-NEXT:    psrad $31, %xmm1
@@ -648,7 +648,7 @@ define <16 x i32> @saddo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nou
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm2
 ; SSE41-NEXT:    psrad $31, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[3,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm3
 ; SSE41-NEXT:    psrad $31, %xmm3
@@ -664,12 +664,12 @@ define <16 x i32> @saddo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nou
 ; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm1
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
 ; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm2
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
 ; AVX1-NEXT:    vmovdqa %xmm3, (%rdi)
@@ -886,13 +886,13 @@ define <4 x i32> @saddo_v4i24(<4 x i24> %a0, <4 x i24> %a1, <4 x i24>* %p2) noun
 ; SSE2-NEXT:    pxor %xmm1, %xmm0
 ; SSE2-NEXT:    movd %xmm2, %eax
 ; SSE2-NEXT:    movw %ax, (%rdi)
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[3,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[3,3,3,3]
 ; SSE2-NEXT:    movd %xmm1, %ecx
 ; SSE2-NEXT:    movw %cx, 9(%rdi)
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]
 ; SSE2-NEXT:    movd %xmm1, %edx
 ; SSE2-NEXT:    movw %dx, 6(%rdi)
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,1,1]
 ; SSE2-NEXT:    movd %xmm1, %esi
 ; SSE2-NEXT:    movw %si, 3(%rdi)
 ; SSE2-NEXT:    shrl $16, %eax
@@ -921,13 +921,13 @@ define <4 x i32> @saddo_v4i24(<4 x i24> %a0, <4 x i24> %a1, <4 x i24>* %p2) noun
 ; SSSE3-NEXT:    pxor %xmm1, %xmm0
 ; SSSE3-NEXT:    movd %xmm2, %eax
 ; SSSE3-NEXT:    movw %ax, (%rdi)
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[3,1,2,3]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[3,3,3,3]
 ; SSSE3-NEXT:    movd %xmm1, %ecx
 ; SSSE3-NEXT:    movw %cx, 9(%rdi)
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]
 ; SSSE3-NEXT:    movd %xmm1, %edx
 ; SSSE3-NEXT:    movw %dx, 6(%rdi)
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,2,3]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,1,1]
 ; SSSE3-NEXT:    movd %xmm1, %esi
 ; SSSE3-NEXT:    movw %si, 3(%rdi)
 ; SSSE3-NEXT:    shrl $16, %eax
diff --git a/llvm/test/CodeGen/X86/vec_set-H.ll b/llvm/test/CodeGen/X86/vec_set-H.ll
index d96c8bbc61718..071f9a162c387 100644
--- a/llvm/test/CodeGen/X86/vec_set-H.ll
+++ b/llvm/test/CodeGen/X86/vec_set-H.ll
@@ -5,7 +5,7 @@ define <2 x i64> @doload64(i16 signext  %x) nounwind  {
 ; CHECK-LABEL: doload64:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; CHECK-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; CHECK-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; CHECK-NEXT:    retl
   %tmp36 = insertelement <8 x i16> undef, i16 %x, i32 0
diff --git a/llvm/test/CodeGen/X86/vec_shift7.ll b/llvm/test/CodeGen/X86/vec_shift7.ll
index 2dfad54df5661..d8d6212b7b6c8 100644
--- a/llvm/test/CodeGen/X86/vec_shift7.ll
+++ b/llvm/test/CodeGen/X86/vec_shift7.ll
@@ -8,7 +8,7 @@ define i64 @test1(<2 x i64> %a) {
 ; X32-LABEL: test1:
 ; X32:       # %bb.0: # %entry
 ; X32-NEXT:    movd %xmm0, %eax
-; X32-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; X32-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; X32-NEXT:    movd %xmm0, %edx
 ; X32-NEXT:    retl
 ;
diff --git a/llvm/test/CodeGen/X86/vec_smulo.ll b/llvm/test/CodeGen/X86/vec_smulo.ll
index b5fefe296d77e..ad0a8f8ff12a0 100644
--- a/llvm/test/CodeGen/X86/vec_smulo.ll
+++ b/llvm/test/CodeGen/X86/vec_smulo.ll
@@ -1196,12 +1196,12 @@ define <16 x i32> @smulo_v16i32(<16 x i32> %a0, <16 x i32> %a1, <16 x i32>* %p2)
 ; AVX1-NEXT:    vpackssdw %xmm5, %xmm0, %xmm0
 ; AVX1-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm5
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm5, %ymm0
 ; AVX1-NEXT:    vpacksswb %xmm1, %xmm1, %xmm1
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm5
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm5, %ymm1
 ; AVX1-NEXT:    vmovdqa %xmm8, 48(%rdi)
@@ -1403,7 +1403,7 @@ define <16 x i32> @smulo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nou
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm0
 ; SSE41-NEXT:    psrad $31, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,1,1]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm1
 ; SSE41-NEXT:    psrad $31, %xmm1
@@ -1411,7 +1411,7 @@ define <16 x i32> @smulo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nou
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm2
 ; SSE41-NEXT:    psrad $31, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[3,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm3
 ; SSE41-NEXT:    psrad $31, %xmm3
@@ -1447,12 +1447,12 @@ define <16 x i32> @smulo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nou
 ; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm1
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,1,1]
 ; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
 ; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm3, %ymm1
 ; AVX1-NEXT:    vmovdqa %xmm2, (%rdi)
@@ -1779,7 +1779,7 @@ define <32 x i32> @smulo_v32i8(<32 x i8> %a0, <32 x i8> %a1, <32 x i8>* %p2) nou
 ; SSE41-NEXT:    packuswb %xmm3, %xmm2
 ; SSE41-NEXT:    pcmpeqb %xmm7, %xmm2
 ; SSE41-NEXT:    pxor %xmm8, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm8 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm8
 ; SSE41-NEXT:    psrad $31, %xmm8
@@ -1787,11 +1787,11 @@ define <32 x i32> @smulo_v32i8(<32 x i8> %a0, <32 x i8> %a1, <32 x i8>* %p2) nou
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm5 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm5
 ; SSE41-NEXT:    psrad $31, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[3,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[3,3,3,3]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm6 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm6
 ; SSE41-NEXT:    psrad $31, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,1,1]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm7 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm7
 ; SSE41-NEXT:    psrad $31, %xmm7
@@ -1799,7 +1799,7 @@ define <32 x i32> @smulo_v32i8(<32 x i8> %a0, <32 x i8> %a1, <32 x i8>* %p2) nou
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm3
 ; SSE41-NEXT:    psrad $31, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[3,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[3,3,3,3]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm0
 ; SSE41-NEXT:    psrad $31, %xmm0
@@ -1875,21 +1875,21 @@ define <32 x i32> @smulo_v32i8(<32 x i8> %a0, <32 x i8> %a1, <32 x i8>* %p2) nou
 ; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm1
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
 ; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm2
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm3[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm3[1,1,1,1]
 ; AVX1-NEXT:    vpmovsxbd %xmm4, %xmm4
 ; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm2, %ymm2
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm1[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm4, %xmm4
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm4, %ymm1
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm3[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm4, %xmm4
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[3,3,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
 ; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm4, %ymm3
 ; AVX1-NEXT:    vmovdqa %xmm9, 16(%rdi)
@@ -2528,12 +2528,12 @@ define <64 x i32> @smulo_v64i8(<64 x i8> %a0, <64 x i8> %a1, <64 x i8>* %p2) nou
 ; SSE41-NEXT:    pslld $31, %xmm0
 ; SSE41-NEXT:    psrad $31, %xmm0
 ; SSE41-NEXT:    movdqa %xmm0, 224(%rdi)
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm8[3,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm8[3,3,3,3]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm0
 ; SSE41-NEXT:    psrad $31, %xmm0
 ; SSE41-NEXT:    movdqa %xmm0, 240(%rdi)
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm8[1,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm8[1,1,1,1]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm0
 ; SSE41-NEXT:    psrad $31, %xmm0
@@ -2543,12 +2543,12 @@ define <64 x i32> @smulo_v64i8(<64 x i8> %a0, <64 x i8> %a1, <64 x i8>* %p2) nou
 ; SSE41-NEXT:    pslld $31, %xmm0
 ; SSE41-NEXT:    psrad $31, %xmm0
 ; SSE41-NEXT:    movdqa %xmm0, 160(%rdi)
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[3,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[3,3,3,3]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm0
 ; SSE41-NEXT:    psrad $31, %xmm0
 ; SSE41-NEXT:    movdqa %xmm0, 176(%rdi)
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,1,1]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm0
 ; SSE41-NEXT:    psrad $31, %xmm0
@@ -2558,12 +2558,12 @@ define <64 x i32> @smulo_v64i8(<64 x i8> %a0, <64 x i8> %a1, <64 x i8>* %p2) nou
 ; SSE41-NEXT:    pslld $31, %xmm0
 ; SSE41-NEXT:    psrad $31, %xmm0
 ; SSE41-NEXT:    movdqa %xmm0, 96(%rdi)
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[3,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[3,3,3,3]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm0
 ; SSE41-NEXT:    psrad $31, %xmm0
 ; SSE41-NEXT:    movdqa %xmm0, 112(%rdi)
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm0
 ; SSE41-NEXT:    psrad $31, %xmm0
@@ -2573,12 +2573,12 @@ define <64 x i32> @smulo_v64i8(<64 x i8> %a0, <64 x i8> %a1, <64 x i8>* %p2) nou
 ; SSE41-NEXT:    pslld $31, %xmm0
 ; SSE41-NEXT:    psrad $31, %xmm0
 ; SSE41-NEXT:    movdqa %xmm0, 32(%rdi)
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[3,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[3,3,3,3]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm0
 ; SSE41-NEXT:    psrad $31, %xmm0
 ; SSE41-NEXT:    movdqa %xmm0, 48(%rdi)
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm0
 ; SSE41-NEXT:    psrad $31, %xmm0
@@ -2702,37 +2702,37 @@ define <64 x i32> @smulo_v64i8(<64 x i8> %a0, <64 x i8> %a1, <64 x i8>* %p2) nou
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm6[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
 ; AVX1-NEXT:    vmovdqa %xmm1, 224(%rdi)
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm6[3,3,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm6[3,3,3,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
 ; AVX1-NEXT:    vmovdqa %xmm1, 240(%rdi)
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm6[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm6[1,1,1,1]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
 ; AVX1-NEXT:    vmovdqa %xmm1, 208(%rdi)
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm3[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
 ; AVX1-NEXT:    vmovdqa %xmm1, 160(%rdi)
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm3[3,3,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm3[3,3,3,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
 ; AVX1-NEXT:    vmovdqa %xmm1, 176(%rdi)
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm3[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm3[1,1,1,1]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
 ; AVX1-NEXT:    vmovdqa %xmm1, 144(%rdi)
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
 ; AVX1-NEXT:    vmovdqa %xmm1, 96(%rdi)
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm2[3,3,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm2[3,3,3,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
 ; AVX1-NEXT:    vmovdqa %xmm1, 112(%rdi)
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm2[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm2[1,1,1,1]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
 ; AVX1-NEXT:    vmovdqa %xmm1, 80(%rdi)
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
 ; AVX1-NEXT:    vmovdqa %xmm1, 32(%rdi)
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[3,3,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
 ; AVX1-NEXT:    vmovdqa %xmm1, 48(%rdi)
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm0
 ; AVX1-NEXT:    vmovdqa %xmm0, 16(%rdi)
 ; AVX1-NEXT:    vzeroupper
@@ -3165,7 +3165,7 @@ define <4 x i32> @smulo_v4i24(<4 x i24> %a0, <4 x i24> %a1, <4 x i24>* %p2) noun
 ; SSE2-NEXT:    movw %dx, 3(%rdi)
 ; SSE2-NEXT:    shrl $16, %eax
 ; SSE2-NEXT:    movb %al, 2(%rdi)
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,2,2,2]
 ; SSE2-NEXT:    movd %xmm0, %eax
 ; SSE2-NEXT:    movw %ax, 9(%rdi)
 ; SSE2-NEXT:    shrl $16, %ecx
@@ -3220,7 +3220,7 @@ define <4 x i32> @smulo_v4i24(<4 x i24> %a0, <4 x i24> %a1, <4 x i24>* %p2) noun
 ; SSSE3-NEXT:    movw %dx, 3(%rdi)
 ; SSSE3-NEXT:    shrl $16, %eax
 ; SSSE3-NEXT:    movb %al, 2(%rdi)
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,2,2,2]
 ; SSSE3-NEXT:    movd %xmm0, %eax
 ; SSSE3-NEXT:    movw %ax, 9(%rdi)
 ; SSSE3-NEXT:    shrl $16, %ecx
diff --git a/llvm/test/CodeGen/X86/vec_ssubo.ll b/llvm/test/CodeGen/X86/vec_ssubo.ll
index eb12f0dbcbff2..4e2c3a57831f5 100644
--- a/llvm/test/CodeGen/X86/vec_ssubo.ll
+++ b/llvm/test/CodeGen/X86/vec_ssubo.ll
@@ -520,12 +520,12 @@ define <16 x i32> @ssubo_v16i32(<16 x i32> %a0, <16 x i32> %a1, <16 x i32>* %p2)
 ; AVX1-NEXT:    vpackssdw %xmm4, %xmm0, %xmm0
 ; AVX1-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm4
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm4, %ymm0
 ; AVX1-NEXT:    vpacksswb %xmm1, %xmm1, %xmm1
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm4
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm4, %ymm1
 ; AVX1-NEXT:    vmovdqa %xmm8, 48(%rdi)
@@ -645,7 +645,7 @@ define <16 x i32> @ssubo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nou
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm4 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm4
 ; SSE41-NEXT:    psrad $31, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,1,1]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm1
 ; SSE41-NEXT:    psrad $31, %xmm1
@@ -653,7 +653,7 @@ define <16 x i32> @ssubo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nou
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm2
 ; SSE41-NEXT:    psrad $31, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[3,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm3
 ; SSE41-NEXT:    psrad $31, %xmm3
@@ -669,12 +669,12 @@ define <16 x i32> @ssubo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nou
 ; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm1
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
 ; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm2
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
 ; AVX1-NEXT:    vmovdqa %xmm3, (%rdi)
@@ -896,13 +896,13 @@ define <4 x i32> @ssubo_v4i24(<4 x i24> %a0, <4 x i24> %a1, <4 x i24>* %p2) noun
 ; SSE2-NEXT:    pxor %xmm1, %xmm0
 ; SSE2-NEXT:    movd %xmm2, %eax
 ; SSE2-NEXT:    movw %ax, (%rdi)
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[3,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[3,3,3,3]
 ; SSE2-NEXT:    movd %xmm1, %ecx
 ; SSE2-NEXT:    movw %cx, 9(%rdi)
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]
 ; SSE2-NEXT:    movd %xmm1, %edx
 ; SSE2-NEXT:    movw %dx, 6(%rdi)
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,1,1]
 ; SSE2-NEXT:    movd %xmm1, %esi
 ; SSE2-NEXT:    movw %si, 3(%rdi)
 ; SSE2-NEXT:    shrl $16, %eax
@@ -931,13 +931,13 @@ define <4 x i32> @ssubo_v4i24(<4 x i24> %a0, <4 x i24> %a1, <4 x i24>* %p2) noun
 ; SSSE3-NEXT:    pxor %xmm1, %xmm0
 ; SSSE3-NEXT:    movd %xmm2, %eax
 ; SSSE3-NEXT:    movw %ax, (%rdi)
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[3,1,2,3]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[3,3,3,3]
 ; SSSE3-NEXT:    movd %xmm1, %ecx
 ; SSSE3-NEXT:    movw %cx, 9(%rdi)
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]
 ; SSSE3-NEXT:    movd %xmm1, %edx
 ; SSSE3-NEXT:    movw %dx, 6(%rdi)
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,2,3]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,1,1]
 ; SSSE3-NEXT:    movd %xmm1, %esi
 ; SSSE3-NEXT:    movw %si, 3(%rdi)
 ; SSSE3-NEXT:    shrl $16, %eax
diff --git a/llvm/test/CodeGen/X86/vec_uaddo.ll b/llvm/test/CodeGen/X86/vec_uaddo.ll
index 9ff793b6b677f..9a153253a1695 100644
--- a/llvm/test/CodeGen/X86/vec_uaddo.ll
+++ b/llvm/test/CodeGen/X86/vec_uaddo.ll
@@ -604,12 +604,12 @@ define <16 x i32> @uaddo_v16i32(<16 x i32> %a0, <16 x i32> %a1, <16 x i32>* %p2)
 ; AVX1-NEXT:    vpackssdw %xmm7, %xmm0, %xmm0
 ; AVX1-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm6
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm6, %ymm0
 ; AVX1-NEXT:    vpacksswb %xmm1, %xmm1, %xmm1
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm6
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm6, %ymm1
 ; AVX1-NEXT:    vmovdqa %xmm4, 48(%rdi)
@@ -723,7 +723,7 @@ define <16 x i32> @uaddo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nou
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm0
 ; SSE41-NEXT:    psrad $31, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,1,1,1]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm4 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm4
 ; SSE41-NEXT:    psrad $31, %xmm4
@@ -731,7 +731,7 @@ define <16 x i32> @uaddo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nou
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm2
 ; SSE41-NEXT:    psrad $31, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[3,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm3
 ; SSE41-NEXT:    psrad $31, %xmm3
@@ -747,12 +747,12 @@ define <16 x i32> @uaddo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nou
 ; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm1
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,1,1]
 ; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
 ; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm3, %ymm1
 ; AVX1-NEXT:    vmovdqa %xmm2, (%rdi)
@@ -953,13 +953,13 @@ define <4 x i32> @uaddo_v4i24(<4 x i24> %a0, <4 x i24> %a1, <4 x i24>* %p2) noun
 ; SSE2-NEXT:    pxor %xmm3, %xmm0
 ; SSE2-NEXT:    movd %xmm2, %eax
 ; SSE2-NEXT:    movw %ax, (%rdi)
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[3,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[3,3,3,3]
 ; SSE2-NEXT:    movd %xmm1, %ecx
 ; SSE2-NEXT:    movw %cx, 9(%rdi)
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]
 ; SSE2-NEXT:    movd %xmm1, %edx
 ; SSE2-NEXT:    movw %dx, 6(%rdi)
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,1,1]
 ; SSE2-NEXT:    movd %xmm1, %esi
 ; SSE2-NEXT:    movw %si, 3(%rdi)
 ; SSE2-NEXT:    shrl $16, %eax
@@ -985,13 +985,13 @@ define <4 x i32> @uaddo_v4i24(<4 x i24> %a0, <4 x i24> %a1, <4 x i24>* %p2) noun
 ; SSSE3-NEXT:    pxor %xmm3, %xmm0
 ; SSSE3-NEXT:    movd %xmm2, %eax
 ; SSSE3-NEXT:    movw %ax, (%rdi)
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[3,1,2,3]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[3,3,3,3]
 ; SSSE3-NEXT:    movd %xmm1, %ecx
 ; SSSE3-NEXT:    movw %cx, 9(%rdi)
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]
 ; SSSE3-NEXT:    movd %xmm1, %edx
 ; SSSE3-NEXT:    movw %dx, 6(%rdi)
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,2,3]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,1,1]
 ; SSSE3-NEXT:    movd %xmm1, %esi
 ; SSSE3-NEXT:    movw %si, 3(%rdi)
 ; SSSE3-NEXT:    shrl $16, %eax
diff --git a/llvm/test/CodeGen/X86/vec_umulo.ll b/llvm/test/CodeGen/X86/vec_umulo.ll
index 87fe4922dfcb0..54bb86dc5a0f4 100644
--- a/llvm/test/CodeGen/X86/vec_umulo.ll
+++ b/llvm/test/CodeGen/X86/vec_umulo.ll
@@ -1014,12 +1014,12 @@ define <16 x i32> @umulo_v16i32(<16 x i32> %a0, <16 x i32> %a1, <16 x i32>* %p2)
 ; AVX1-NEXT:    vpmulld %xmm3, %xmm1, %xmm3
 ; AVX1-NEXT:    vpmulld %xmm10, %xmm12, %xmm6
 ; AVX1-NEXT:    vpmovsxbd %xmm5, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm5[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm5[1,1,1,1]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vpacksswb %xmm11, %xmm11, %xmm1
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm5
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm5, %ymm1
 ; AVX1-NEXT:    vmovdqa %xmm6, 48(%rdi)
@@ -1213,7 +1213,7 @@ define <16 x i32> @umulo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nou
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm0
 ; SSE41-NEXT:    psrad $31, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,1,1]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm1
 ; SSE41-NEXT:    psrad $31, %xmm1
@@ -1221,7 +1221,7 @@ define <16 x i32> @umulo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nou
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm2
 ; SSE41-NEXT:    psrad $31, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[3,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm3
 ; SSE41-NEXT:    psrad $31, %xmm3
@@ -1251,12 +1251,12 @@ define <16 x i32> @umulo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nou
 ; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm1
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,1,1]
 ; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
 ; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm3, %ymm1
 ; AVX1-NEXT:    vmovdqa %xmm2, (%rdi)
@@ -1556,7 +1556,7 @@ define <32 x i32> @umulo_v32i8(<32 x i8> %a0, <32 x i8> %a1, <32 x i8>* %p2) nou
 ; SSE41-NEXT:    packuswb %xmm0, %xmm6
 ; SSE41-NEXT:    pcmpeqb %xmm4, %xmm6
 ; SSE41-NEXT:    pxor %xmm1, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[1,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[1,1,1,1]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm0
 ; SSE41-NEXT:    psrad $31, %xmm0
@@ -1564,11 +1564,11 @@ define <32 x i32> @umulo_v32i8(<32 x i8> %a0, <32 x i8> %a1, <32 x i8>* %p2) nou
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm1
 ; SSE41-NEXT:    psrad $31, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[3,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[3,3,3,3]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm2
 ; SSE41-NEXT:    psrad $31, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm7[1,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm7[1,1,1,1]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm3
 ; SSE41-NEXT:    psrad $31, %xmm3
@@ -1576,7 +1576,7 @@ define <32 x i32> @umulo_v32i8(<32 x i8> %a0, <32 x i8> %a1, <32 x i8>* %p2) nou
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm4
 ; SSE41-NEXT:    psrad $31, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm7[3,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm7[3,3,3,3]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm5 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm5
 ; SSE41-NEXT:    psrad $31, %xmm5
@@ -1640,21 +1640,21 @@ define <32 x i32> @umulo_v32i8(<32 x i8> %a0, <32 x i8> %a1, <32 x i8>* %p2) nou
 ; AVX1-NEXT:    vpcmpeqb %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm1
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
 ; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm2
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; AVX1-NEXT:    vpmovsxbd %xmm6, %xmm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm6[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm6[1,1,1,1]
 ; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
 ; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm3, %ymm1
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm6[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm6[3,3,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm6[3,3,3,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm4, %xmm4
 ; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
 ; AVX1-NEXT:    vmovdqa %xmm5, 16(%rdi)
@@ -2235,12 +2235,12 @@ define <64 x i32> @umulo_v64i8(<64 x i8> %a0, <64 x i8> %a1, <64 x i8>* %p2) nou
 ; SSE41-NEXT:    pslld $31, %xmm0
 ; SSE41-NEXT:    psrad $31, %xmm0
 ; SSE41-NEXT:    movdqa %xmm0, 224(%rdi)
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm14[3,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm14[3,3,3,3]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm0
 ; SSE41-NEXT:    psrad $31, %xmm0
 ; SSE41-NEXT:    movdqa %xmm0, 240(%rdi)
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm14[1,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm14[1,1,1,1]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm0
 ; SSE41-NEXT:    psrad $31, %xmm0
@@ -2250,12 +2250,12 @@ define <64 x i32> @umulo_v64i8(<64 x i8> %a0, <64 x i8> %a1, <64 x i8>* %p2) nou
 ; SSE41-NEXT:    pslld $31, %xmm0
 ; SSE41-NEXT:    psrad $31, %xmm0
 ; SSE41-NEXT:    movdqa %xmm0, 160(%rdi)
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm12[3,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm12[3,3,3,3]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm0
 ; SSE41-NEXT:    psrad $31, %xmm0
 ; SSE41-NEXT:    movdqa %xmm0, 176(%rdi)
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm12[1,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm12[1,1,1,1]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm0
 ; SSE41-NEXT:    psrad $31, %xmm0
@@ -2265,12 +2265,12 @@ define <64 x i32> @umulo_v64i8(<64 x i8> %a0, <64 x i8> %a1, <64 x i8>* %p2) nou
 ; SSE41-NEXT:    pslld $31, %xmm0
 ; SSE41-NEXT:    psrad $31, %xmm0
 ; SSE41-NEXT:    movdqa %xmm0, 96(%rdi)
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm9[3,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm9[3,3,3,3]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm0
 ; SSE41-NEXT:    psrad $31, %xmm0
 ; SSE41-NEXT:    movdqa %xmm0, 112(%rdi)
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm9[1,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm9[1,1,1,1]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm0
 ; SSE41-NEXT:    psrad $31, %xmm0
@@ -2280,12 +2280,12 @@ define <64 x i32> @umulo_v64i8(<64 x i8> %a0, <64 x i8> %a1, <64 x i8>* %p2) nou
 ; SSE41-NEXT:    pslld $31, %xmm0
 ; SSE41-NEXT:    psrad $31, %xmm0
 ; SSE41-NEXT:    movdqa %xmm0, 32(%rdi)
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm8[3,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm8[3,3,3,3]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm0
 ; SSE41-NEXT:    psrad $31, %xmm0
 ; SSE41-NEXT:    movdqa %xmm0, 48(%rdi)
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm8[1,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm8[1,1,1,1]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm0
 ; SSE41-NEXT:    psrad $31, %xmm0
@@ -2393,37 +2393,37 @@ define <64 x i32> @umulo_v64i8(<64 x i8> %a0, <64 x i8> %a1, <64 x i8>* %p2) nou
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm3[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm4, %xmm4
 ; AVX1-NEXT:    vmovdqa %xmm4, 224(%rdi)
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm3[3,3,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm3[3,3,3,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm4, %xmm4
 ; AVX1-NEXT:    vmovdqa %xmm4, 240(%rdi)
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,1,1]
 ; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
 ; AVX1-NEXT:    vmovdqa %xmm3, 208(%rdi)
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
 ; AVX1-NEXT:    vmovdqa %xmm3, 160(%rdi)
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm2[3,3,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm2[3,3,3,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
 ; AVX1-NEXT:    vmovdqa %xmm3, 176(%rdi)
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,1,1]
 ; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm2
 ; AVX1-NEXT:    vmovdqa %xmm2, 144(%rdi)
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm2
 ; AVX1-NEXT:    vmovdqa %xmm2, 96(%rdi)
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[3,3,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm2
 ; AVX1-NEXT:    vmovdqa %xmm2, 112(%rdi)
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
 ; AVX1-NEXT:    vmovdqa %xmm1, 80(%rdi)
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
 ; AVX1-NEXT:    vmovdqa %xmm1, 32(%rdi)
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[3,3,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
 ; AVX1-NEXT:    vmovdqa %xmm1, 48(%rdi)
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm0
 ; AVX1-NEXT:    vmovdqa %xmm0, 16(%rdi)
 ; AVX1-NEXT:    vzeroupper
@@ -2836,7 +2836,7 @@ define <4 x i32> @umulo_v4i24(<4 x i24> %a0, <4 x i24> %a1, <4 x i24>* %p2) noun
 ; SSE2-NEXT:    movw %dx, 3(%rdi)
 ; SSE2-NEXT:    shrl $16, %eax
 ; SSE2-NEXT:    movb %al, 2(%rdi)
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,2,2,2]
 ; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    movw %ax, 9(%rdi)
 ; SSE2-NEXT:    shrl $16, %ecx
@@ -2880,7 +2880,7 @@ define <4 x i32> @umulo_v4i24(<4 x i24> %a0, <4 x i24> %a1, <4 x i24>* %p2) noun
 ; SSSE3-NEXT:    movw %dx, 3(%rdi)
 ; SSSE3-NEXT:    shrl $16, %eax
 ; SSSE3-NEXT:    movb %al, 2(%rdi)
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,2,2,2]
 ; SSSE3-NEXT:    movd %xmm1, %eax
 ; SSSE3-NEXT:    movw %ax, 9(%rdi)
 ; SSSE3-NEXT:    shrl $16, %ecx
diff --git a/llvm/test/CodeGen/X86/vec_usubo.ll b/llvm/test/CodeGen/X86/vec_usubo.ll
index 155c5591ce11a..0381394e74134 100644
--- a/llvm/test/CodeGen/X86/vec_usubo.ll
+++ b/llvm/test/CodeGen/X86/vec_usubo.ll
@@ -647,12 +647,12 @@ define <16 x i32> @usubo_v16i32(<16 x i32> %a0, <16 x i32> %a1, <16 x i32>* %p2)
 ; AVX1-NEXT:    vpackssdw %xmm7, %xmm0, %xmm0
 ; AVX1-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm6
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm6, %ymm0
 ; AVX1-NEXT:    vpacksswb %xmm1, %xmm1, %xmm1
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm6
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm6, %ymm1
 ; AVX1-NEXT:    vmovdqa %xmm4, 48(%rdi)
@@ -767,7 +767,7 @@ define <16 x i32> @usubo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nou
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm0
 ; SSE41-NEXT:    psrad $31, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,1,1]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm1
 ; SSE41-NEXT:    psrad $31, %xmm1
@@ -775,7 +775,7 @@ define <16 x i32> @usubo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nou
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm2
 ; SSE41-NEXT:    psrad $31, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[3,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm3
 ; SSE41-NEXT:    psrad $31, %xmm3
@@ -790,12 +790,12 @@ define <16 x i32> @usubo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nou
 ; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm1
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,1,1]
 ; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
 ; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm3, %ymm1
 ; AVX1-NEXT:    vmovdqa %xmm2, (%rdi)
@@ -1000,13 +1000,13 @@ define <4 x i32> @usubo_v4i24(<4 x i24> %a0, <4 x i24> %a1, <4 x i24>* %p2) noun
 ; SSE2-NEXT:    pxor %xmm3, %xmm0
 ; SSE2-NEXT:    movd %xmm2, %eax
 ; SSE2-NEXT:    movw %ax, (%rdi)
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[3,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[3,3,3,3]
 ; SSE2-NEXT:    movd %xmm1, %ecx
 ; SSE2-NEXT:    movw %cx, 9(%rdi)
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]
 ; SSE2-NEXT:    movd %xmm1, %edx
 ; SSE2-NEXT:    movw %dx, 6(%rdi)
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,1,1]
 ; SSE2-NEXT:    movd %xmm1, %esi
 ; SSE2-NEXT:    movw %si, 3(%rdi)
 ; SSE2-NEXT:    shrl $16, %eax
@@ -1032,13 +1032,13 @@ define <4 x i32> @usubo_v4i24(<4 x i24> %a0, <4 x i24> %a1, <4 x i24>* %p2) noun
 ; SSSE3-NEXT:    pxor %xmm3, %xmm0
 ; SSSE3-NEXT:    movd %xmm2, %eax
 ; SSSE3-NEXT:    movw %ax, (%rdi)
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[3,1,2,3]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[3,3,3,3]
 ; SSSE3-NEXT:    movd %xmm1, %ecx
 ; SSSE3-NEXT:    movw %cx, 9(%rdi)
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]
 ; SSSE3-NEXT:    movd %xmm1, %edx
 ; SSSE3-NEXT:    movw %dx, 6(%rdi)
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,2,3]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,1,1]
 ; SSSE3-NEXT:    movd %xmm1, %esi
 ; SSSE3-NEXT:    movw %si, 3(%rdi)
 ; SSSE3-NEXT:    shrl $16, %eax
diff --git a/llvm/test/CodeGen/X86/vector-constrained-fp-intrinsics.ll b/llvm/test/CodeGen/X86/vector-constrained-fp-intrinsics.ll
index 62bc377b9cec3..1e3bc0c61c913 100644
--- a/llvm/test/CodeGen/X86/vector-constrained-fp-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/vector-constrained-fp-intrinsics.ll
@@ -6371,7 +6371,7 @@ define <3 x double> @constrained_vector_sitofp_v3f64_v3i32(<3 x i32> %x) #0 {
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    movd %xmm0, %eax
 ; CHECK-NEXT:    cvtsi2sd %eax, %xmm2
-; CHECK-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; CHECK-NEXT:    movd %xmm1, %eax
 ; CHECK-NEXT:    xorps %xmm1, %xmm1
 ; CHECK-NEXT:    cvtsi2sd %eax, %xmm1
@@ -6409,7 +6409,7 @@ define <3 x float> @constrained_vector_sitofp_v3f32_v3i32(<3 x i32> %x) #0 {
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    movd %xmm0, %eax
 ; CHECK-NEXT:    cvtsi2ss %eax, %xmm1
-; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,1,1]
 ; CHECK-NEXT:    movd %xmm2, %eax
 ; CHECK-NEXT:    xorps %xmm2, %xmm2
 ; CHECK-NEXT:    cvtsi2ss %eax, %xmm2
@@ -7027,7 +7027,7 @@ define <3 x double> @constrained_vector_uitofp_v3f64_v3i32(<3 x i32> %x) #0 {
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    movd %xmm0, %eax
 ; CHECK-NEXT:    cvtsi2sd %rax, %xmm2
-; CHECK-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; CHECK-NEXT:    movd %xmm1, %eax
 ; CHECK-NEXT:    xorps %xmm1, %xmm1
 ; CHECK-NEXT:    cvtsi2sd %rax, %xmm1
@@ -7077,7 +7077,7 @@ define <3 x float> @constrained_vector_uitofp_v3f32_v3i32(<3 x i32> %x) #0 {
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    movd %xmm0, %eax
 ; CHECK-NEXT:    cvtsi2ss %rax, %xmm1
-; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,1,1]
 ; CHECK-NEXT:    movd %xmm2, %eax
 ; CHECK-NEXT:    xorps %xmm2, %xmm2
 ; CHECK-NEXT:    cvtsi2ss %rax, %xmm2
diff --git a/llvm/test/CodeGen/X86/vector-extend-inreg.ll b/llvm/test/CodeGen/X86/vector-extend-inreg.ll
index f6ab64975cd35..4b57e694ed60e 100644
--- a/llvm/test/CodeGen/X86/vector-extend-inreg.ll
+++ b/llvm/test/CodeGen/X86/vector-extend-inreg.ll
@@ -95,7 +95,7 @@ define i64 @extract_any_extend_vector_inreg_v16i64(<16 x i64> %a0, i32 %a1) noun
 ; X64-AVX-NEXT:    andq $-32, %rsp
 ; X64-AVX-NEXT:    subq $160, %rsp
 ; X64-AVX-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-AVX-NEXT:    vpermq {{.*#+}} ymm0 = ymm3[3,1,2,3]
+; X64-AVX-NEXT:    vpermq {{.*#+}} ymm0 = ymm3[3,3,3,3]
 ; X64-AVX-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; X64-AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
 ; X64-AVX-NEXT:    vmovaps %ymm1, {{[0-9]+}}(%rsp)
diff --git a/llvm/test/CodeGen/X86/vector-fshl-128.ll b/llvm/test/CodeGen/X86/vector-fshl-128.ll
index 81c291fa602b8..00c06564d2258 100644
--- a/llvm/test/CodeGen/X86/vector-fshl-128.ll
+++ b/llvm/test/CodeGen/X86/vector-fshl-128.ll
@@ -1614,7 +1614,7 @@ define <8 x i16> @splatvar_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %
 ; SSE2-NEXT:    pand {{.*}}(%rip), %xmm2
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
 ; SSE2-NEXT:    psubw %xmm2, %xmm3
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm4 = xmm2[0,0,2,3,4,5,6,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm4 = xmm2[0,0,0,0,4,5,6,7]
 ; SSE2-NEXT:    pslldq {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm2[0,1]
 ; SSE2-NEXT:    psrldq {{.*#+}} xmm2 = xmm2[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; SSE2-NEXT:    movdqa %xmm0, %xmm5
@@ -1643,7 +1643,7 @@ define <8 x i16> @splatvar_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %
 ; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
 ; SSE41-NEXT:    psrlw %xmm0, %xmm1
 ; SSE41-NEXT:    por %xmm1, %xmm4
-; SSE41-NEXT:    pshuflw {{.*#+}} xmm0 = xmm2[0,0,2,3,4,5,6,7]
+; SSE41-NEXT:    pshuflw {{.*#+}} xmm0 = xmm2[0,0,0,0,4,5,6,7]
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,0,0]
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
 ; SSE41-NEXT:    pcmpeqw %xmm1, %xmm0
@@ -1661,7 +1661,7 @@ define <8 x i16> @splatvar_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %
 ; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
 ; AVX1-NEXT:    vpsrlw %xmm4, %xmm1, %xmm1
 ; AVX1-NEXT:    vpor %xmm1, %xmm3, %xmm1
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm2[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm2[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
 ; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
 ; AVX1-NEXT:    vpcmpeqw %xmm3, %xmm2, %xmm2
@@ -1784,7 +1784,7 @@ define <8 x i16> @splatvar_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %
 ; XOPAVX1-NEXT:    vpmovzxwq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
 ; XOPAVX1-NEXT:    vpsrlw %xmm4, %xmm1, %xmm1
 ; XOPAVX1-NEXT:    vpor %xmm1, %xmm3, %xmm1
-; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm2[0,0,2,3,4,5,6,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm2[0,0,0,0,4,5,6,7]
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
 ; XOPAVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
 ; XOPAVX1-NEXT:    vpcomeqw %xmm3, %xmm2, %xmm2
@@ -1812,7 +1812,7 @@ define <8 x i16> @splatvar_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %
 ; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm2
 ; X32-SSE-NEXT:    movdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
 ; X32-SSE-NEXT:    psubw %xmm2, %xmm3
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm4 = xmm2[0,0,2,3,4,5,6,7]
+; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm4 = xmm2[0,0,0,0,4,5,6,7]
 ; X32-SSE-NEXT:    pslldq {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm2[0,1]
 ; X32-SSE-NEXT:    psrldq {{.*#+}} xmm2 = xmm2[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; X32-SSE-NEXT:    movdqa %xmm0, %xmm5
@@ -1846,7 +1846,7 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %
 ; SSE2-NEXT:    pcmpeqd %xmm6, %xmm6
 ; SSE2-NEXT:    psllw %xmm3, %xmm6
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm6[0,0,2,3,4,5,6,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm6[0,0,0,0,4,5,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
 ; SSE2-NEXT:    pand %xmm4, %xmm3
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
@@ -1857,12 +1857,12 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %
 ; SSE2-NEXT:    psrlw %xmm4, %xmm5
 ; SSE2-NEXT:    psrlw $8, %xmm5
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm4 = xmm5[0,0,2,3,4,5,6,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm4 = xmm5[0,0,0,0,4,5,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[0,0,0,0]
 ; SSE2-NEXT:    pand %xmm1, %xmm4
 ; SSE2-NEXT:    por %xmm3, %xmm4
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,2,3,4,5,6,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,0,0,4,5,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
 ; SSE2-NEXT:    pcmpeqb %xmm1, %xmm2
@@ -2105,7 +2105,7 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %
 ; X32-SSE-NEXT:    pcmpeqd %xmm6, %xmm6
 ; X32-SSE-NEXT:    psllw %xmm3, %xmm6
 ; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm3 = xmm6[0,0,2,3,4,5,6,7]
+; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm3 = xmm6[0,0,0,0,4,5,6,7]
 ; X32-SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
 ; X32-SSE-NEXT:    pand %xmm4, %xmm3
 ; X32-SSE-NEXT:    movdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
@@ -2116,12 +2116,12 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %
 ; X32-SSE-NEXT:    psrlw %xmm4, %xmm5
 ; X32-SSE-NEXT:    psrlw $8, %xmm5
 ; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm4 = xmm5[0,0,2,3,4,5,6,7]
+; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm4 = xmm5[0,0,0,0,4,5,6,7]
 ; X32-SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[0,0,0,0]
 ; X32-SSE-NEXT:    pand %xmm1, %xmm4
 ; X32-SSE-NEXT:    por %xmm3, %xmm4
 ; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,2,3,4,5,6,7]
+; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,0,0,4,5,6,7]
 ; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; X32-SSE-NEXT:    pxor %xmm2, %xmm2
 ; X32-SSE-NEXT:    pcmpeqb %xmm1, %xmm2
diff --git a/llvm/test/CodeGen/X86/vector-fshl-256.ll b/llvm/test/CodeGen/X86/vector-fshl-256.ll
index afe7716a36e49..5e701ff4a17cf 100644
--- a/llvm/test/CodeGen/X86/vector-fshl-256.ll
+++ b/llvm/test/CodeGen/X86/vector-fshl-256.ll
@@ -1248,7 +1248,7 @@ define <16 x i16> @splatvar_funnnel_v16i16(<16 x i16> %x, <16 x i16> %y, <16 x i
 ; AVX1-NEXT:    vpsllw %xmm4, %xmm3, %xmm3
 ; AVX1-NEXT:    vpsllw %xmm4, %xmm0, %xmm4
 ; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm4, %ymm3
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm2[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm2[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [16,16,16,16,16,16,16,16]
 ; AVX1-NEXT:    vpsubw %xmm2, %xmm4, %xmm4
@@ -1378,7 +1378,7 @@ define <16 x i16> @splatvar_funnnel_v16i16(<16 x i16> %x, <16 x i16> %y, <16 x i
 ; XOPAVX1-NEXT:    vpsllw %xmm4, %xmm3, %xmm3
 ; XOPAVX1-NEXT:    vpsllw %xmm4, %xmm0, %xmm4
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm4, %ymm3
-; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm2[0,0,2,3,4,5,6,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm2[0,0,0,0,4,5,6,7]
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
 ; XOPAVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [16,16,16,16,16,16,16,16]
 ; XOPAVX1-NEXT:    vpsubw %xmm2, %xmm4, %xmm4
@@ -1674,7 +1674,7 @@ define void @fancierRotate2(i32* %arr, i8* %control, i32 %rot0, i32 %rot1) {
 ; AVX1-NEXT:    vmovq {{.*#+}} xmm5 = mem[0],zero
 ; AVX1-NEXT:    vpcmpeqb %xmm5, %xmm8, %xmm5
 ; AVX1-NEXT:    vpmovsxbd %xmm5, %xmm6
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[1,1,1,1]
 ; AVX1-NEXT:    vpmovsxbd %xmm5, %xmm5
 ; AVX1-NEXT:    vmovdqu 4096(%rdi,%rax,4), %xmm7
 ; AVX1-NEXT:    vmovdqu 4112(%rdi,%rax,4), %xmm0
@@ -1876,7 +1876,7 @@ define void @fancierRotate2(i32* %arr, i8* %control, i32 %rot0, i32 %rot1) {
 ; XOPAVX1-NEXT:    vmovq {{.*#+}} xmm5 = mem[0],zero
 ; XOPAVX1-NEXT:    vpcomeqb %xmm2, %xmm5, %xmm5
 ; XOPAVX1-NEXT:    vpmovsxbd %xmm5, %xmm6
-; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[1,1,2,3]
+; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[1,1,1,1]
 ; XOPAVX1-NEXT:    vpmovsxbd %xmm5, %xmm5
 ; XOPAVX1-NEXT:    vblendvps %xmm5, %xmm3, %xmm4, %xmm5
 ; XOPAVX1-NEXT:    vprotd %xmm5, 4112(%rdi,%rax,4), %xmm5
diff --git a/llvm/test/CodeGen/X86/vector-fshl-rot-128.ll b/llvm/test/CodeGen/X86/vector-fshl-rot-128.ll
index b1eb2fdbe7af8..a8a03f5c28cc6 100644
--- a/llvm/test/CodeGen/X86/vector-fshl-rot-128.ll
+++ b/llvm/test/CodeGen/X86/vector-fshl-rot-128.ll
@@ -930,7 +930,7 @@ define <8 x i16> @splatvar_funnnel_v8i16(<8 x i16> %x, <8 x i16> %amt) nounwind
 ;
 ; XOPAVX1-LABEL: splatvar_funnnel_v8i16:
 ; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,0,2,3,4,5,6,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,0,0,0,4,5,6,7]
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; XOPAVX1-NEXT:    vprotw %xmm1, %xmm0, %xmm0
 ; XOPAVX1-NEXT:    retq
@@ -975,7 +975,7 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %amt) nounwind
 ; SSE2-NEXT:    pcmpeqd %xmm5, %xmm5
 ; SSE2-NEXT:    psllw %xmm3, %xmm5
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm5[0,0,2,3,4,5,6,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm5[0,0,0,0,4,5,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
 ; SSE2-NEXT:    pand %xmm3, %xmm1
 ; SSE2-NEXT:    pslldq {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm2[0]
@@ -984,7 +984,7 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %amt) nounwind
 ; SSE2-NEXT:    psrlw %xmm2, %xmm4
 ; SSE2-NEXT:    psrlw $8, %xmm4
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm4[0,0,2,3,4,5,6,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm4[0,0,0,0,4,5,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
 ; SSE2-NEXT:    pand %xmm0, %xmm2
 ; SSE2-NEXT:    por %xmm2, %xmm1
@@ -1150,7 +1150,7 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %amt) nounwind
 ; X32-SSE-NEXT:    pcmpeqd %xmm5, %xmm5
 ; X32-SSE-NEXT:    psllw %xmm3, %xmm5
 ; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm3 = xmm5[0,0,2,3,4,5,6,7]
+; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm3 = xmm5[0,0,0,0,4,5,6,7]
 ; X32-SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
 ; X32-SSE-NEXT:    pand %xmm3, %xmm1
 ; X32-SSE-NEXT:    pslldq {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm2[0]
@@ -1159,7 +1159,7 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %amt) nounwind
 ; X32-SSE-NEXT:    psrlw %xmm2, %xmm4
 ; X32-SSE-NEXT:    psrlw $8, %xmm4
 ; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm4[0,0,2,3,4,5,6,7]
+; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm4[0,0,0,0,4,5,6,7]
 ; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
 ; X32-SSE-NEXT:    pand %xmm0, %xmm2
 ; X32-SSE-NEXT:    por %xmm2, %xmm1
diff --git a/llvm/test/CodeGen/X86/vector-fshl-rot-256.ll b/llvm/test/CodeGen/X86/vector-fshl-rot-256.ll
index 7db474fc39da9..05bd0e01ec9bf 100644
--- a/llvm/test/CodeGen/X86/vector-fshl-rot-256.ll
+++ b/llvm/test/CodeGen/X86/vector-fshl-rot-256.ll
@@ -682,7 +682,7 @@ define <8 x i32> @splatvar_funnnel_v8i32(<8 x i32> %x, <8 x i32> %amt) nounwind
 define <16 x i16> @splatvar_funnnel_v16i16(<16 x i16> %x, <16 x i16> %amt) nounwind {
 ; AVX1-LABEL: splatvar_funnnel_v16i16:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
 ; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
@@ -727,7 +727,7 @@ define <16 x i16> @splatvar_funnnel_v16i16(<16 x i16> %x, <16 x i16> %amt) nounw
 ;
 ; XOPAVX1-LABEL: splatvar_funnnel_v16i16:
 ; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,0,2,3,4,5,6,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,0,0,0,4,5,6,7]
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
 ; XOPAVX1-NEXT:    vprotw %xmm1, %xmm2, %xmm2
diff --git a/llvm/test/CodeGen/X86/vector-fshr-128.ll b/llvm/test/CodeGen/X86/vector-fshr-128.ll
index db76d3eef282d..bad82a4247979 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-128.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-128.ll
@@ -1630,7 +1630,7 @@ define <8 x i16> @splatvar_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %
 ; SSE2-NEXT:    pand {{.*}}(%rip), %xmm2
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
 ; SSE2-NEXT:    psubw %xmm2, %xmm3
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm4 = xmm2[0,0,2,3,4,5,6,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm4 = xmm2[0,0,0,0,4,5,6,7]
 ; SSE2-NEXT:    pslldq {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm2[0,1]
 ; SSE2-NEXT:    psrldq {{.*#+}} xmm2 = xmm2[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; SSE2-NEXT:    movdqa %xmm1, %xmm5
@@ -1660,7 +1660,7 @@ define <8 x i16> @splatvar_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %
 ; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
 ; SSE41-NEXT:    psllw %xmm0, %xmm3
 ; SSE41-NEXT:    por %xmm4, %xmm3
-; SSE41-NEXT:    pshuflw {{.*#+}} xmm0 = xmm2[0,0,2,3,4,5,6,7]
+; SSE41-NEXT:    pshuflw {{.*#+}} xmm0 = xmm2[0,0,0,0,4,5,6,7]
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,0,0]
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
 ; SSE41-NEXT:    pcmpeqw %xmm2, %xmm0
@@ -1678,7 +1678,7 @@ define <8 x i16> @splatvar_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %
 ; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
 ; AVX1-NEXT:    vpsllw %xmm4, %xmm0, %xmm0
 ; AVX1-NEXT:    vpor %xmm3, %xmm0, %xmm0
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm2[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm2[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
 ; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
 ; AVX1-NEXT:    vpcmpeqw %xmm3, %xmm2, %xmm2
@@ -1801,7 +1801,7 @@ define <8 x i16> @splatvar_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %
 ; XOPAVX1-NEXT:    vpmovzxwq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
 ; XOPAVX1-NEXT:    vpsllw %xmm4, %xmm0, %xmm0
 ; XOPAVX1-NEXT:    vpor %xmm3, %xmm0, %xmm0
-; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm2[0,0,2,3,4,5,6,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm2[0,0,0,0,4,5,6,7]
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
 ; XOPAVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
 ; XOPAVX1-NEXT:    vpcomeqw %xmm3, %xmm2, %xmm2
@@ -1829,7 +1829,7 @@ define <8 x i16> @splatvar_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %
 ; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm2
 ; X32-SSE-NEXT:    movdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
 ; X32-SSE-NEXT:    psubw %xmm2, %xmm3
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm4 = xmm2[0,0,2,3,4,5,6,7]
+; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm4 = xmm2[0,0,0,0,4,5,6,7]
 ; X32-SSE-NEXT:    pslldq {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm2[0,1]
 ; X32-SSE-NEXT:    psrldq {{.*#+}} xmm2 = xmm2[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; X32-SSE-NEXT:    movdqa %xmm1, %xmm5
@@ -1865,7 +1865,7 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %
 ; SSE2-NEXT:    psrlw %xmm3, %xmm6
 ; SSE2-NEXT:    psrlw $8, %xmm6
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm6[0,0,2,3,4,5,6,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm6[0,0,0,0,4,5,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
 ; SSE2-NEXT:    pand %xmm4, %xmm3
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
@@ -1875,12 +1875,12 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %
 ; SSE2-NEXT:    psllw %xmm4, %xmm0
 ; SSE2-NEXT:    psllw %xmm4, %xmm5
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm4 = xmm5[0,0,2,3,4,5,6,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm4 = xmm5[0,0,0,0,4,5,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[0,0,0,0]
 ; SSE2-NEXT:    pand %xmm0, %xmm4
 ; SSE2-NEXT:    por %xmm3, %xmm4
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm2[0,0,2,3,4,5,6,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm2[0,0,0,0,4,5,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,0,0]
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
 ; SSE2-NEXT:    pcmpeqb %xmm2, %xmm0
@@ -2126,7 +2126,7 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %
 ; X32-SSE-NEXT:    psrlw %xmm3, %xmm6
 ; X32-SSE-NEXT:    psrlw $8, %xmm6
 ; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm3 = xmm6[0,0,2,3,4,5,6,7]
+; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm3 = xmm6[0,0,0,0,4,5,6,7]
 ; X32-SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
 ; X32-SSE-NEXT:    pand %xmm4, %xmm3
 ; X32-SSE-NEXT:    movdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
@@ -2136,12 +2136,12 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %
 ; X32-SSE-NEXT:    psllw %xmm4, %xmm0
 ; X32-SSE-NEXT:    psllw %xmm4, %xmm5
 ; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm4 = xmm5[0,0,2,3,4,5,6,7]
+; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm4 = xmm5[0,0,0,0,4,5,6,7]
 ; X32-SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[0,0,0,0]
 ; X32-SSE-NEXT:    pand %xmm0, %xmm4
 ; X32-SSE-NEXT:    por %xmm3, %xmm4
 ; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm2[0,0,2,3,4,5,6,7]
+; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm2[0,0,0,0,4,5,6,7]
 ; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,0,0]
 ; X32-SSE-NEXT:    pxor %xmm0, %xmm0
 ; X32-SSE-NEXT:    pcmpeqb %xmm2, %xmm0
diff --git a/llvm/test/CodeGen/X86/vector-fshr-256.ll b/llvm/test/CodeGen/X86/vector-fshr-256.ll
index 6449d58fa38c5..3f808acaeb265 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-256.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-256.ll
@@ -1250,7 +1250,7 @@ define <16 x i16> @splatvar_funnnel_v16i16(<16 x i16> %x, <16 x i16> %y, <16 x i
 ; AVX1-NEXT:    vpsrlw %xmm4, %xmm3, %xmm3
 ; AVX1-NEXT:    vpsrlw %xmm4, %xmm1, %xmm4
 ; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm4, %ymm3
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm2[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm2[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [16,16,16,16,16,16,16,16]
 ; AVX1-NEXT:    vpsubw %xmm2, %xmm4, %xmm4
@@ -1380,7 +1380,7 @@ define <16 x i16> @splatvar_funnnel_v16i16(<16 x i16> %x, <16 x i16> %y, <16 x i
 ; XOPAVX1-NEXT:    vpsrlw %xmm4, %xmm3, %xmm3
 ; XOPAVX1-NEXT:    vpsrlw %xmm4, %xmm1, %xmm4
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm4, %ymm3
-; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm2[0,0,2,3,4,5,6,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm2[0,0,0,0,4,5,6,7]
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
 ; XOPAVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [16,16,16,16,16,16,16,16]
 ; XOPAVX1-NEXT:    vpsubw %xmm2, %xmm4, %xmm4
diff --git a/llvm/test/CodeGen/X86/vector-fshr-rot-128.ll b/llvm/test/CodeGen/X86/vector-fshr-rot-128.ll
index 6cda97e1ebd39..da32eaaebfa15 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-rot-128.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-rot-128.ll
@@ -996,7 +996,7 @@ define <8 x i16> @splatvar_funnnel_v8i16(<8 x i16> %x, <8 x i16> %amt) nounwind
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; XOPAVX1-NEXT:    vpsubw %xmm1, %xmm2, %xmm1
-; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,0,2,3,4,5,6,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,0,0,0,4,5,6,7]
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; XOPAVX1-NEXT:    vprotw %xmm1, %xmm0, %xmm0
 ; XOPAVX1-NEXT:    retq
@@ -1046,7 +1046,7 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %amt) nounwind
 ; SSE2-NEXT:    pcmpeqd %xmm5, %xmm5
 ; SSE2-NEXT:    psllw %xmm2, %xmm5
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm5[0,0,2,3,4,5,6,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm5[0,0,0,0,4,5,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
 ; SSE2-NEXT:    pand %xmm2, %xmm1
 ; SSE2-NEXT:    pslldq {{.*#+}} xmm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm3[0]
@@ -1055,7 +1055,7 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %amt) nounwind
 ; SSE2-NEXT:    psrlw %xmm3, %xmm4
 ; SSE2-NEXT:    psrlw $8, %xmm4
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm4[0,0,2,3,4,5,6,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm4[0,0,0,0,4,5,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
 ; SSE2-NEXT:    pand %xmm0, %xmm2
 ; SSE2-NEXT:    por %xmm2, %xmm1
@@ -1230,7 +1230,7 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %amt) nounwind
 ; X32-SSE-NEXT:    pcmpeqd %xmm5, %xmm5
 ; X32-SSE-NEXT:    psllw %xmm2, %xmm5
 ; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm5[0,0,2,3,4,5,6,7]
+; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm5[0,0,0,0,4,5,6,7]
 ; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
 ; X32-SSE-NEXT:    pand %xmm2, %xmm1
 ; X32-SSE-NEXT:    pslldq {{.*#+}} xmm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm3[0]
@@ -1239,7 +1239,7 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %amt) nounwind
 ; X32-SSE-NEXT:    psrlw %xmm3, %xmm4
 ; X32-SSE-NEXT:    psrlw $8, %xmm4
 ; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm4[0,0,2,3,4,5,6,7]
+; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm4[0,0,0,0,4,5,6,7]
 ; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
 ; X32-SSE-NEXT:    pand %xmm0, %xmm2
 ; X32-SSE-NEXT:    por %xmm2, %xmm1
diff --git a/llvm/test/CodeGen/X86/vector-fshr-rot-256.ll b/llvm/test/CodeGen/X86/vector-fshr-rot-256.ll
index fffa286b79196..a9d70fbba5d03 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-rot-256.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-rot-256.ll
@@ -742,7 +742,7 @@ define <16 x i16> @splatvar_funnnel_v16i16(<16 x i16> %x, <16 x i16> %amt) nounw
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; AVX1-NEXT:    vpsubw %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
 ; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
@@ -793,7 +793,7 @@ define <16 x i16> @splatvar_funnnel_v16i16(<16 x i16> %x, <16 x i16> %amt) nounw
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; XOPAVX1-NEXT:    vpsubw %xmm1, %xmm2, %xmm1
-; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,0,2,3,4,5,6,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,0,0,0,4,5,6,7]
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
 ; XOPAVX1-NEXT:    vprotw %xmm1, %xmm2, %xmm2
diff --git a/llvm/test/CodeGen/X86/vector-idiv-v2i32.ll b/llvm/test/CodeGen/X86/vector-idiv-v2i32.ll
index 032c9e293b0a3..9e548bdb1db11 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-v2i32.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-v2i32.ll
@@ -31,7 +31,7 @@ define void @test_udiv7_v2i32(<2 x i32>* %x, <2 x i32>* %y) nounwind {
 ; X86-NEXT:    pmuludq %xmm1, %xmm2
 ; X86-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
 ; X86-NEXT:    movdqa %xmm0, %xmm3
-; X86-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,2,3]
+; X86-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,1,1]
 ; X86-NEXT:    pmuludq %xmm1, %xmm3
 ; X86-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,3,2,3]
 ; X86-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
@@ -81,7 +81,7 @@ define void @test_urem7_v2i32(<2 x i32>* %x, <2 x i32>* %y) nounwind {
 ; X86-NEXT:    pmuludq %xmm1, %xmm2
 ; X86-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
 ; X86-NEXT:    movdqa %xmm0, %xmm3
-; X86-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,2,3]
+; X86-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,1,1]
 ; X86-NEXT:    pmuludq %xmm1, %xmm3
 ; X86-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,3,2,3]
 ; X86-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
@@ -137,7 +137,7 @@ define void @test_sdiv7_v2i32(<2 x i32>* %x, <2 x i32>* %y) nounwind {
 ; X86-NEXT:    pmuludq %xmm1, %xmm2
 ; X86-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
 ; X86-NEXT:    movdqa %xmm0, %xmm3
-; X86-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,2,3]
+; X86-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,1,1]
 ; X86-NEXT:    pmuludq %xmm1, %xmm3
 ; X86-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,3,2,3]
 ; X86-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
@@ -198,7 +198,7 @@ define void @test_srem7_v2i32(<2 x i32>* %x, <2 x i32>* %y) nounwind {
 ; X86-NEXT:    pmuludq %xmm1, %xmm2
 ; X86-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
 ; X86-NEXT:    movdqa %xmm0, %xmm3
-; X86-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,2,3]
+; X86-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1,1,1]
 ; X86-NEXT:    pmuludq %xmm1, %xmm3
 ; X86-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,3,2,3]
 ; X86-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
@@ -332,9 +332,9 @@ define void @test_udiv_v2i32(<2 x i32>* %x, <2 x i32>* %y, <2 x i32>* %z) nounwi
 ; X64-NEXT:    xorl %edx, %edx
 ; X64-NEXT:    divl %esi
 ; X64-NEXT:    movd %eax, %xmm2
-; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; X64-NEXT:    movd %xmm0, %eax
-; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X64-NEXT:    movd %xmm0, %esi
 ; X64-NEXT:    xorl %edx, %edx
 ; X64-NEXT:    divl %esi
@@ -356,9 +356,9 @@ define void @test_udiv_v2i32(<2 x i32>* %x, <2 x i32>* %y, <2 x i32>* %z) nounwi
 ; X86-NEXT:    xorl %edx, %edx
 ; X86-NEXT:    divl %esi
 ; X86-NEXT:    movd %eax, %xmm2
-; X86-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; X86-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; X86-NEXT:    movd %xmm0, %eax
-; X86-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; X86-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,1,1]
 ; X86-NEXT:    movd %xmm1, %esi
 ; X86-NEXT:    xorl %edx, %edx
 ; X86-NEXT:    divl %esi
@@ -386,9 +386,9 @@ define void @test_urem_v2i32(<2 x i32>* %x, <2 x i32>* %y, <2 x i32>* %z) nounwi
 ; X64-NEXT:    xorl %edx, %edx
 ; X64-NEXT:    divl %esi
 ; X64-NEXT:    movd %edx, %xmm2
-; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; X64-NEXT:    movd %xmm0, %eax
-; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X64-NEXT:    movd %xmm0, %esi
 ; X64-NEXT:    xorl %edx, %edx
 ; X64-NEXT:    divl %esi
@@ -410,9 +410,9 @@ define void @test_urem_v2i32(<2 x i32>* %x, <2 x i32>* %y, <2 x i32>* %z) nounwi
 ; X86-NEXT:    xorl %edx, %edx
 ; X86-NEXT:    divl %esi
 ; X86-NEXT:    movd %edx, %xmm2
-; X86-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; X86-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; X86-NEXT:    movd %xmm0, %eax
-; X86-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; X86-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,1,1]
 ; X86-NEXT:    movd %xmm1, %esi
 ; X86-NEXT:    xorl %edx, %edx
 ; X86-NEXT:    divl %esi
@@ -440,9 +440,9 @@ define void @test_sdiv_v2i32(<2 x i32>* %x, <2 x i32>* %y, <2 x i32>* %z) nounwi
 ; X64-NEXT:    cltd
 ; X64-NEXT:    idivl %esi
 ; X64-NEXT:    movd %eax, %xmm2
-; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; X64-NEXT:    movd %xmm0, %eax
-; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X64-NEXT:    movd %xmm0, %esi
 ; X64-NEXT:    cltd
 ; X64-NEXT:    idivl %esi
@@ -453,32 +453,27 @@ define void @test_sdiv_v2i32(<2 x i32>* %x, <2 x i32>* %y, <2 x i32>* %z) nounwi
 ;
 ; X86-LABEL: test_sdiv_v2i32:
 ; X86:       # %bb.0:
-; X86-NEXT:    pushl %ebx
-; X86-NEXT:    pushl %edi
 ; X86-NEXT:    pushl %esi
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
 ; X86-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
-; X86-NEXT:    movd %xmm0, %ecx
-; X86-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
 ; X86-NEXT:    movd %xmm0, %eax
-; X86-NEXT:    movd %xmm1, %edi
-; X86-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,2,3]
-; X86-NEXT:    movd %xmm1, %ebx
+; X86-NEXT:    movd %xmm1, %esi
 ; X86-NEXT:    cltd
-; X86-NEXT:    idivl %ebx
-; X86-NEXT:    movd %eax, %xmm0
-; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    idivl %esi
+; X86-NEXT:    movd %eax, %xmm2
+; X86-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
+; X86-NEXT:    movd %xmm0, %eax
+; X86-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,1,1]
+; X86-NEXT:    movd %xmm1, %esi
 ; X86-NEXT:    cltd
-; X86-NEXT:    idivl %edi
-; X86-NEXT:    movd %eax, %xmm1
-; X86-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; X86-NEXT:    movq %xmm1, (%esi)
+; X86-NEXT:    idivl %esi
+; X86-NEXT:    movd %eax, %xmm0
+; X86-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; X86-NEXT:    movq %xmm2, (%ecx)
 ; X86-NEXT:    popl %esi
-; X86-NEXT:    popl %edi
-; X86-NEXT:    popl %ebx
 ; X86-NEXT:    retl
   %a = load <2 x i32>, <2 x i32>* %x
   %b = load <2 x i32>, <2 x i32>* %y
@@ -499,9 +494,9 @@ define void @test_srem_v2i32(<2 x i32>* %x, <2 x i32>* %y, <2 x i32>* %z) nounwi
 ; X64-NEXT:    cltd
 ; X64-NEXT:    idivl %esi
 ; X64-NEXT:    movd %eax, %xmm2
-; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; X64-NEXT:    movd %xmm0, %eax
-; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; X64-NEXT:    movd %xmm0, %esi
 ; X64-NEXT:    cltd
 ; X64-NEXT:    idivl %esi
@@ -512,32 +507,27 @@ define void @test_srem_v2i32(<2 x i32>* %x, <2 x i32>* %y, <2 x i32>* %z) nounwi
 ;
 ; X86-LABEL: test_srem_v2i32:
 ; X86:       # %bb.0:
-; X86-NEXT:    pushl %ebx
-; X86-NEXT:    pushl %edi
 ; X86-NEXT:    pushl %esi
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
 ; X86-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
-; X86-NEXT:    movd %xmm0, %ecx
-; X86-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
 ; X86-NEXT:    movd %xmm0, %eax
-; X86-NEXT:    movd %xmm1, %edi
-; X86-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,2,3]
-; X86-NEXT:    movd %xmm1, %ebx
+; X86-NEXT:    movd %xmm1, %esi
 ; X86-NEXT:    cltd
-; X86-NEXT:    idivl %ebx
-; X86-NEXT:    movd %eax, %xmm0
-; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    idivl %esi
+; X86-NEXT:    movd %eax, %xmm2
+; X86-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
+; X86-NEXT:    movd %xmm0, %eax
+; X86-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,1,1]
+; X86-NEXT:    movd %xmm1, %esi
 ; X86-NEXT:    cltd
-; X86-NEXT:    idivl %edi
-; X86-NEXT:    movd %eax, %xmm1
-; X86-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; X86-NEXT:    movq %xmm1, (%esi)
+; X86-NEXT:    idivl %esi
+; X86-NEXT:    movd %eax, %xmm0
+; X86-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; X86-NEXT:    movq %xmm2, (%ecx)
 ; X86-NEXT:    popl %esi
-; X86-NEXT:    popl %edi
-; X86-NEXT:    popl %ebx
 ; X86-NEXT:    retl
   %a = load <2 x i32>, <2 x i32>* %x
   %b = load <2 x i32>, <2 x i32>* %y
diff --git a/llvm/test/CodeGen/X86/vector-narrow-binop.ll b/llvm/test/CodeGen/X86/vector-narrow-binop.ll
index 383fde7038d04..e817da89862a7 100644
--- a/llvm/test/CodeGen/X86/vector-narrow-binop.ll
+++ b/llvm/test/CodeGen/X86/vector-narrow-binop.ll
@@ -108,7 +108,7 @@ define <2 x i8> @PR39893(<2 x i32> %x, <8 x i8> %y) {
 ; SSE-NEXT:    pxor %xmm2, %xmm2
 ; SSE-NEXT:    psubd %xmm0, %xmm2
 ; SSE-NEXT:    psrld $16, %xmm2
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
 ; SSE-NEXT:    movdqa %xmm2, %xmm0
 ; SSE-NEXT:    retq
@@ -118,7 +118,7 @@ define <2 x i8> @PR39893(<2 x i32> %x, <8 x i8> %y) {
 ; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; AVX-NEXT:    vpsubd %xmm0, %xmm2, %xmm0
 ; AVX-NEXT:    vpsrld $16, %xmm0, %xmm0
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
 ; AVX-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
 ; AVX-NEXT:    retq
   %sub = sub <2 x i32> <i32 0, i32 undef>, %x
diff --git a/llvm/test/CodeGen/X86/vector-reduce-add.ll b/llvm/test/CodeGen/X86/vector-reduce-add.ll
index a00e74fa1cacc..92f97d0469cd7 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-add.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-add.ll
@@ -198,14 +198,14 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 define i32 @test_v2i32(<2 x i32> %a0) {
 ; SSE-LABEL: test_v2i32:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE-NEXT:    paddd %xmm0, %xmm1
 ; SSE-NEXT:    movd %xmm1, %eax
 ; SSE-NEXT:    retq
 ;
 ; AVX1-SLOW-LABEL: test_v2i32:
 ; AVX1-SLOW:       # %bb.0:
-; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX1-SLOW-NEXT:    vmovd %xmm0, %eax
 ; AVX1-SLOW-NEXT:    retq
@@ -218,14 +218,14 @@ define i32 @test_v2i32(<2 x i32> %a0) {
 ;
 ; AVX2-LABEL: test_v2i32:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: test_v2i32:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
 ; AVX512-NEXT:    retq
@@ -238,7 +238,7 @@ define i32 @test_v4i32(<4 x i32> %a0) {
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    paddd %xmm0, %xmm1
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE-NEXT:    paddd %xmm1, %xmm0
 ; SSE-NEXT:    movd %xmm0, %eax
 ; SSE-NEXT:    retq
@@ -247,7 +247,7 @@ define i32 @test_v4i32(<4 x i32> %a0) {
 ; AVX1-SLOW:       # %bb.0:
 ; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX1-SLOW-NEXT:    vmovd %xmm0, %eax
 ; AVX1-SLOW-NEXT:    retq
@@ -263,7 +263,7 @@ define i32 @test_v4i32(<4 x i32> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
 ; AVX2-NEXT:    retq
@@ -272,7 +272,7 @@ define i32 @test_v4i32(<4 x i32> %a0) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
 ; AVX512-NEXT:    retq
@@ -286,7 +286,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; SSE-NEXT:    paddd %xmm1, %xmm0
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    paddd %xmm0, %xmm1
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE-NEXT:    paddd %xmm1, %xmm0
 ; SSE-NEXT:    movd %xmm0, %eax
 ; SSE-NEXT:    retq
@@ -297,7 +297,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX1-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX1-SLOW-NEXT:    vmovd %xmm0, %eax
 ; AVX1-SLOW-NEXT:    vzeroupper
@@ -319,7 +319,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
 ; AVX2-NEXT:    vzeroupper
@@ -331,7 +331,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
 ; AVX512-NEXT:    vzeroupper
@@ -348,7 +348,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; SSE-NEXT:    paddd %xmm0, %xmm1
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE-NEXT:    paddd %xmm1, %xmm0
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE-NEXT:    paddd %xmm0, %xmm1
 ; SSE-NEXT:    movd %xmm1, %eax
 ; SSE-NEXT:    retq
@@ -362,7 +362,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX1-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX1-SLOW-NEXT:    vmovd %xmm0, %eax
 ; AVX1-SLOW-NEXT:    vzeroupper
@@ -389,7 +389,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
 ; AVX2-NEXT:    vzeroupper
@@ -403,7 +403,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
 ; AVX512-NEXT:    vzeroupper
@@ -424,7 +424,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; SSE-NEXT:    paddd %xmm0, %xmm2
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE-NEXT:    paddd %xmm2, %xmm0
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE-NEXT:    paddd %xmm0, %xmm1
 ; SSE-NEXT:    movd %xmm1, %eax
 ; SSE-NEXT:    retq
@@ -444,7 +444,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX1-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX1-SLOW-NEXT:    vmovd %xmm0, %eax
 ; AVX1-SLOW-NEXT:    vzeroupper
@@ -479,7 +479,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
 ; AVX2-NEXT:    vzeroupper
@@ -494,7 +494,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
 ; AVX512-NEXT:    vzeroupper
@@ -554,7 +554,7 @@ define i16 @test_v2i16(<2 x i16> %a0) {
 define i16 @test_v4i16(<4 x i16> %a0) {
 ; SSE-LABEL: test_v4i16:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE-NEXT:    paddw %xmm0, %xmm1
 ; SSE-NEXT:    movdqa %xmm1, %xmm0
 ; SSE-NEXT:    psrld $16, %xmm0
@@ -565,7 +565,7 @@ define i16 @test_v4i16(<4 x i16> %a0) {
 ;
 ; AVX1-SLOW-LABEL: test_v4i16:
 ; AVX1-SLOW:       # %bb.0:
-; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-SLOW-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX1-SLOW-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX1-SLOW-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
@@ -575,7 +575,7 @@ define i16 @test_v4i16(<4 x i16> %a0) {
 ;
 ; AVX1-FAST-LABEL: test_v4i16:
 ; AVX1-FAST:       # %bb.0:
-; AVX1-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-FAST-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX1-FAST-NEXT:    vphaddw %xmm0, %xmm0, %xmm0
 ; AVX1-FAST-NEXT:    vmovd %xmm0, %eax
@@ -584,7 +584,7 @@ define i16 @test_v4i16(<4 x i16> %a0) {
 ;
 ; AVX2-LABEL: test_v4i16:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
@@ -594,7 +594,7 @@ define i16 @test_v4i16(<4 x i16> %a0) {
 ;
 ; AVX512-LABEL: test_v4i16:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
@@ -610,7 +610,7 @@ define i16 @test_v8i16(<8 x i16> %a0) {
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    paddw %xmm0, %xmm1
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE-NEXT:    paddw %xmm1, %xmm0
 ; SSE-NEXT:    movdqa %xmm0, %xmm1
 ; SSE-NEXT:    psrld $16, %xmm1
@@ -623,7 +623,7 @@ define i16 @test_v8i16(<8 x i16> %a0) {
 ; AVX1-SLOW:       # %bb.0:
 ; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-SLOW-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
-; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-SLOW-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX1-SLOW-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX1-SLOW-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
@@ -644,7 +644,7 @@ define i16 @test_v8i16(<8 x i16> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
@@ -656,7 +656,7 @@ define i16 @test_v8i16(<8 x i16> %a0) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
@@ -673,7 +673,7 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; SSE-NEXT:    paddw %xmm1, %xmm0
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    paddw %xmm0, %xmm1
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE-NEXT:    paddw %xmm1, %xmm0
 ; SSE-NEXT:    movdqa %xmm0, %xmm1
 ; SSE-NEXT:    psrld $16, %xmm1
@@ -688,7 +688,7 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX1-SLOW-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-SLOW-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
-; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-SLOW-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX1-SLOW-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX1-SLOW-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
@@ -715,7 +715,7 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
@@ -730,7 +730,7 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX512-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
@@ -750,7 +750,7 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; SSE-NEXT:    paddw %xmm0, %xmm1
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE-NEXT:    paddw %xmm1, %xmm0
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE-NEXT:    paddw %xmm0, %xmm1
 ; SSE-NEXT:    movdqa %xmm1, %xmm0
 ; SSE-NEXT:    psrld $16, %xmm0
@@ -768,7 +768,7 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX1-SLOW-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-SLOW-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
-; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-SLOW-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX1-SLOW-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX1-SLOW-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
@@ -786,7 +786,7 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX1-FAST-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX1-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-FAST-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
-; AVX1-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-FAST-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX1-FAST-NEXT:    vphaddw %xmm0, %xmm0, %xmm0
 ; AVX1-FAST-NEXT:    vmovd %xmm0, %eax
@@ -801,7 +801,7 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
@@ -818,7 +818,7 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX512-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
@@ -842,7 +842,7 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; SSE-NEXT:    paddw %xmm0, %xmm2
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE-NEXT:    paddw %xmm2, %xmm0
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE-NEXT:    paddw %xmm0, %xmm1
 ; SSE-NEXT:    movdqa %xmm1, %xmm0
 ; SSE-NEXT:    psrld $16, %xmm0
@@ -866,7 +866,7 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX1-SLOW-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-SLOW-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
-; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-SLOW-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX1-SLOW-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX1-SLOW-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
@@ -890,7 +890,7 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX1-FAST-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX1-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-FAST-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
-; AVX1-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-FAST-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX1-FAST-NEXT:    vphaddw %xmm0, %xmm0, %xmm0
 ; AVX1-FAST-NEXT:    vmovd %xmm0, %eax
@@ -907,7 +907,7 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
@@ -925,7 +925,7 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX512-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
diff --git a/llvm/test/CodeGen/X86/vector-reduce-and-cmp.ll b/llvm/test/CodeGen/X86/vector-reduce-and-cmp.ll
index 17a3d6f46e98c..91668148a9ac4 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-and-cmp.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-and-cmp.ll
@@ -214,7 +214,7 @@ define i1 @test_v16i64(<16 x i64> %a0) {
 define i1 @test_v2i32(<2 x i32> %a0) {
 ; SSE-LABEL: test_v2i32:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE-NEXT:    pand %xmm0, %xmm1
 ; SSE-NEXT:    movd %xmm1, %eax
 ; SSE-NEXT:    testl %eax, %eax
@@ -223,7 +223,7 @@ define i1 @test_v2i32(<2 x i32> %a0) {
 ;
 ; AVX-LABEL: test_v2i32:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovd %xmm0, %eax
 ; AVX-NEXT:    testl %eax, %eax
@@ -239,7 +239,7 @@ define i1 @test_v4i32(<4 x i32> %a0) {
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    pand %xmm0, %xmm1
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE-NEXT:    pand %xmm1, %xmm0
 ; SSE-NEXT:    movd %xmm0, %eax
 ; SSE-NEXT:    testl %eax, %eax
@@ -250,7 +250,7 @@ define i1 @test_v4i32(<4 x i32> %a0) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovd %xmm0, %eax
 ; AVX-NEXT:    testl %eax, %eax
@@ -267,7 +267,7 @@ define i1 @test_v8i32(<8 x i32> %a0) {
 ; SSE-NEXT:    pand %xmm1, %xmm0
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    pand %xmm0, %xmm1
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE-NEXT:    pand %xmm1, %xmm0
 ; SSE-NEXT:    movd %xmm0, %eax
 ; SSE-NEXT:    testl %eax, %eax
@@ -280,7 +280,7 @@ define i1 @test_v8i32(<8 x i32> %a0) {
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
 ; AVX1-NEXT:    testl %eax, %eax
@@ -294,7 +294,7 @@ define i1 @test_v8i32(<8 x i32> %a0) {
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
 ; AVX2-NEXT:    testl %eax, %eax
@@ -308,7 +308,7 @@ define i1 @test_v8i32(<8 x i32> %a0) {
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
 ; AVX512-NEXT:    testl %eax, %eax
@@ -328,7 +328,7 @@ define i1 @test_v16i32(<16 x i32> %a0) {
 ; SSE-NEXT:    pand %xmm0, %xmm1
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE-NEXT:    pand %xmm1, %xmm0
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE-NEXT:    pand %xmm0, %xmm1
 ; SSE-NEXT:    movd %xmm1, %eax
 ; SSE-NEXT:    testl %eax, %eax
@@ -342,7 +342,7 @@ define i1 @test_v16i32(<16 x i32> %a0) {
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
 ; AVX1-NEXT:    testl %eax, %eax
@@ -357,7 +357,7 @@ define i1 @test_v16i32(<16 x i32> %a0) {
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
 ; AVX2-NEXT:    testl %eax, %eax
@@ -373,7 +373,7 @@ define i1 @test_v16i32(<16 x i32> %a0) {
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
 ; AVX512-NEXT:    testl %eax, %eax
@@ -397,7 +397,7 @@ define i1 @test_v32i32(<32 x i32> %a0) {
 ; SSE-NEXT:    pand %xmm0, %xmm2
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE-NEXT:    pand %xmm2, %xmm0
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE-NEXT:    pand %xmm0, %xmm1
 ; SSE-NEXT:    movd %xmm1, %eax
 ; SSE-NEXT:    testl %eax, %eax
@@ -413,7 +413,7 @@ define i1 @test_v32i32(<32 x i32> %a0) {
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
 ; AVX1-NEXT:    testl %eax, %eax
@@ -430,7 +430,7 @@ define i1 @test_v32i32(<32 x i32> %a0) {
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
 ; AVX2-NEXT:    testl %eax, %eax
@@ -447,7 +447,7 @@ define i1 @test_v32i32(<32 x i32> %a0) {
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
 ; AVX512-NEXT:    testl %eax, %eax
@@ -490,7 +490,7 @@ define i1 @test_v2i16(<2 x i16> %a0) {
 define i1 @test_v4i16(<4 x i16> %a0) {
 ; SSE-LABEL: test_v4i16:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE-NEXT:    pand %xmm0, %xmm1
 ; SSE-NEXT:    movdqa %xmm1, %xmm0
 ; SSE-NEXT:    psrld $16, %xmm0
@@ -502,7 +502,7 @@ define i1 @test_v4i16(<4 x i16> %a0) {
 ;
 ; AVX-LABEL: test_v4i16:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -520,7 +520,7 @@ define i1 @test_v8i16(<8 x i16> %a0) {
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    pand %xmm0, %xmm1
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE-NEXT:    pand %xmm1, %xmm0
 ; SSE-NEXT:    movdqa %xmm0, %xmm1
 ; SSE-NEXT:    psrld $16, %xmm1
@@ -534,7 +534,7 @@ define i1 @test_v8i16(<8 x i16> %a0) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -553,7 +553,7 @@ define i1 @test_v16i16(<16 x i16> %a0) {
 ; SSE-NEXT:    pand %xmm1, %xmm0
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    pand %xmm0, %xmm1
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE-NEXT:    pand %xmm1, %xmm0
 ; SSE-NEXT:    movdqa %xmm0, %xmm1
 ; SSE-NEXT:    psrld $16, %xmm1
@@ -569,7 +569,7 @@ define i1 @test_v16i16(<16 x i16> %a0) {
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -585,7 +585,7 @@ define i1 @test_v16i16(<16 x i16> %a0) {
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -601,7 +601,7 @@ define i1 @test_v16i16(<16 x i16> %a0) {
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -623,7 +623,7 @@ define i1 @test_v32i16(<32 x i16> %a0) {
 ; SSE-NEXT:    pand %xmm0, %xmm1
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE-NEXT:    pand %xmm1, %xmm0
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE-NEXT:    pand %xmm0, %xmm1
 ; SSE-NEXT:    movdqa %xmm1, %xmm0
 ; SSE-NEXT:    psrld $16, %xmm0
@@ -640,7 +640,7 @@ define i1 @test_v32i16(<32 x i16> %a0) {
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -657,7 +657,7 @@ define i1 @test_v32i16(<32 x i16> %a0) {
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -675,7 +675,7 @@ define i1 @test_v32i16(<32 x i16> %a0) {
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -701,7 +701,7 @@ define i1 @test_v64i16(<64 x i16> %a0) {
 ; SSE-NEXT:    pand %xmm0, %xmm2
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE-NEXT:    pand %xmm2, %xmm0
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE-NEXT:    pand %xmm0, %xmm1
 ; SSE-NEXT:    movdqa %xmm1, %xmm0
 ; SSE-NEXT:    psrld $16, %xmm0
@@ -720,7 +720,7 @@ define i1 @test_v64i16(<64 x i16> %a0) {
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -739,7 +739,7 @@ define i1 @test_v64i16(<64 x i16> %a0) {
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -758,7 +758,7 @@ define i1 @test_v64i16(<64 x i16> %a0) {
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -832,7 +832,7 @@ define i1 @test_v4i8(<4 x i8> %a0) {
 define i1 @test_v8i8(<8 x i8> %a0) {
 ; SSE-LABEL: test_v8i8:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE-NEXT:    pand %xmm0, %xmm1
 ; SSE-NEXT:    movdqa %xmm1, %xmm0
 ; SSE-NEXT:    psrld $16, %xmm0
@@ -847,7 +847,7 @@ define i1 @test_v8i8(<8 x i8> %a0) {
 ;
 ; AVX-LABEL: test_v8i8:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -867,7 +867,7 @@ define i1 @test_v16i8(<16 x i8> %a0) {
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    pand %xmm0, %xmm1
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE-NEXT:    pand %xmm1, %xmm0
 ; SSE-NEXT:    movdqa %xmm0, %xmm1
 ; SSE-NEXT:    psrld $16, %xmm1
@@ -884,7 +884,7 @@ define i1 @test_v16i8(<16 x i8> %a0) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -905,7 +905,7 @@ define i1 @test_v32i8(<32 x i8> %a0) {
 ; SSE-NEXT:    pand %xmm1, %xmm0
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    pand %xmm0, %xmm1
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE-NEXT:    pand %xmm1, %xmm0
 ; SSE-NEXT:    movdqa %xmm0, %xmm1
 ; SSE-NEXT:    psrld $16, %xmm1
@@ -924,7 +924,7 @@ define i1 @test_v32i8(<32 x i8> %a0) {
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -942,7 +942,7 @@ define i1 @test_v32i8(<32 x i8> %a0) {
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -960,7 +960,7 @@ define i1 @test_v32i8(<32 x i8> %a0) {
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -984,7 +984,7 @@ define i1 @test_v64i8(<64 x i8> %a0) {
 ; SSE-NEXT:    pand %xmm0, %xmm1
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE-NEXT:    pand %xmm1, %xmm0
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE-NEXT:    pand %xmm0, %xmm1
 ; SSE-NEXT:    movdqa %xmm1, %xmm0
 ; SSE-NEXT:    psrld $16, %xmm0
@@ -1004,7 +1004,7 @@ define i1 @test_v64i8(<64 x i8> %a0) {
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -1023,7 +1023,7 @@ define i1 @test_v64i8(<64 x i8> %a0) {
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -1043,7 +1043,7 @@ define i1 @test_v64i8(<64 x i8> %a0) {
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -1071,7 +1071,7 @@ define i1 @test_v128i8(<128 x i8> %a0) {
 ; SSE-NEXT:    pand %xmm0, %xmm2
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE-NEXT:    pand %xmm2, %xmm0
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE-NEXT:    pand %xmm0, %xmm1
 ; SSE-NEXT:    movdqa %xmm1, %xmm0
 ; SSE-NEXT:    psrld $16, %xmm0
@@ -1093,7 +1093,7 @@ define i1 @test_v128i8(<128 x i8> %a0) {
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -1114,7 +1114,7 @@ define i1 @test_v128i8(<128 x i8> %a0) {
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -1135,7 +1135,7 @@ define i1 @test_v128i8(<128 x i8> %a0) {
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
diff --git a/llvm/test/CodeGen/X86/vector-reduce-and.ll b/llvm/test/CodeGen/X86/vector-reduce-and.ll
index 4a00c22a26701..088913ed69521 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-and.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-and.ll
@@ -182,14 +182,14 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 define i32 @test_v2i32(<2 x i32> %a0) {
 ; SSE-LABEL: test_v2i32:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE-NEXT:    pand %xmm0, %xmm1
 ; SSE-NEXT:    movd %xmm1, %eax
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: test_v2i32:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovd %xmm0, %eax
 ; AVX-NEXT:    retq
@@ -202,7 +202,7 @@ define i32 @test_v4i32(<4 x i32> %a0) {
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    pand %xmm0, %xmm1
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE-NEXT:    pand %xmm1, %xmm0
 ; SSE-NEXT:    movd %xmm0, %eax
 ; SSE-NEXT:    retq
@@ -211,7 +211,7 @@ define i32 @test_v4i32(<4 x i32> %a0) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovd %xmm0, %eax
 ; AVX-NEXT:    retq
@@ -225,7 +225,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; SSE-NEXT:    pand %xmm1, %xmm0
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    pand %xmm0, %xmm1
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE-NEXT:    pand %xmm1, %xmm0
 ; SSE-NEXT:    movd %xmm0, %eax
 ; SSE-NEXT:    retq
@@ -236,7 +236,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
 ; AVX1-NEXT:    vzeroupper
@@ -248,7 +248,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
 ; AVX2-NEXT:    vzeroupper
@@ -260,7 +260,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
 ; AVX512-NEXT:    vzeroupper
@@ -277,7 +277,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; SSE-NEXT:    pand %xmm0, %xmm1
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE-NEXT:    pand %xmm1, %xmm0
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE-NEXT:    pand %xmm0, %xmm1
 ; SSE-NEXT:    movd %xmm1, %eax
 ; SSE-NEXT:    retq
@@ -289,7 +289,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
 ; AVX1-NEXT:    vzeroupper
@@ -302,7 +302,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
 ; AVX2-NEXT:    vzeroupper
@@ -316,7 +316,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
 ; AVX512-NEXT:    vzeroupper
@@ -337,7 +337,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; SSE-NEXT:    pand %xmm0, %xmm2
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE-NEXT:    pand %xmm2, %xmm0
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE-NEXT:    pand %xmm0, %xmm1
 ; SSE-NEXT:    movd %xmm1, %eax
 ; SSE-NEXT:    retq
@@ -351,7 +351,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
 ; AVX1-NEXT:    vzeroupper
@@ -366,7 +366,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
 ; AVX2-NEXT:    vzeroupper
@@ -381,7 +381,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
 ; AVX512-NEXT:    vzeroupper
@@ -418,7 +418,7 @@ define i16 @test_v2i16(<2 x i16> %a0) {
 define i16 @test_v4i16(<4 x i16> %a0) {
 ; SSE-LABEL: test_v4i16:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE-NEXT:    pand %xmm0, %xmm1
 ; SSE-NEXT:    movdqa %xmm1, %xmm0
 ; SSE-NEXT:    psrld $16, %xmm0
@@ -429,7 +429,7 @@ define i16 @test_v4i16(<4 x i16> %a0) {
 ;
 ; AVX-LABEL: test_v4i16:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -445,7 +445,7 @@ define i16 @test_v8i16(<8 x i16> %a0) {
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    pand %xmm0, %xmm1
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE-NEXT:    pand %xmm1, %xmm0
 ; SSE-NEXT:    movdqa %xmm0, %xmm1
 ; SSE-NEXT:    psrld $16, %xmm1
@@ -458,7 +458,7 @@ define i16 @test_v8i16(<8 x i16> %a0) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -475,7 +475,7 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; SSE-NEXT:    pand %xmm1, %xmm0
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    pand %xmm0, %xmm1
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE-NEXT:    pand %xmm1, %xmm0
 ; SSE-NEXT:    movdqa %xmm0, %xmm1
 ; SSE-NEXT:    psrld $16, %xmm1
@@ -490,7 +490,7 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -505,7 +505,7 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -520,7 +520,7 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -540,7 +540,7 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; SSE-NEXT:    pand %xmm0, %xmm1
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE-NEXT:    pand %xmm1, %xmm0
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE-NEXT:    pand %xmm0, %xmm1
 ; SSE-NEXT:    movdqa %xmm1, %xmm0
 ; SSE-NEXT:    psrld $16, %xmm0
@@ -556,7 +556,7 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -572,7 +572,7 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -589,7 +589,7 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -613,7 +613,7 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; SSE-NEXT:    pand %xmm0, %xmm2
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE-NEXT:    pand %xmm2, %xmm0
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE-NEXT:    pand %xmm0, %xmm1
 ; SSE-NEXT:    movdqa %xmm1, %xmm0
 ; SSE-NEXT:    psrld $16, %xmm0
@@ -631,7 +631,7 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -649,7 +649,7 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -667,7 +667,7 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -733,7 +733,7 @@ define i8 @test_v4i8(<4 x i8> %a0) {
 define i8 @test_v8i8(<8 x i8> %a0) {
 ; SSE-LABEL: test_v8i8:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE-NEXT:    pand %xmm0, %xmm1
 ; SSE-NEXT:    movdqa %xmm1, %xmm0
 ; SSE-NEXT:    psrld $16, %xmm0
@@ -747,7 +747,7 @@ define i8 @test_v8i8(<8 x i8> %a0) {
 ;
 ; AVX-LABEL: test_v8i8:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -765,7 +765,7 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    pand %xmm0, %xmm1
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE-NEXT:    pand %xmm1, %xmm0
 ; SSE-NEXT:    movdqa %xmm0, %xmm1
 ; SSE-NEXT:    psrld $16, %xmm1
@@ -781,7 +781,7 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -800,7 +800,7 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; SSE-NEXT:    pand %xmm1, %xmm0
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    pand %xmm0, %xmm1
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE-NEXT:    pand %xmm1, %xmm0
 ; SSE-NEXT:    movdqa %xmm0, %xmm1
 ; SSE-NEXT:    psrld $16, %xmm1
@@ -818,7 +818,7 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -835,7 +835,7 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -852,7 +852,7 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -874,7 +874,7 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; SSE-NEXT:    pand %xmm0, %xmm1
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE-NEXT:    pand %xmm1, %xmm0
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE-NEXT:    pand %xmm0, %xmm1
 ; SSE-NEXT:    movdqa %xmm1, %xmm0
 ; SSE-NEXT:    psrld $16, %xmm0
@@ -893,7 +893,7 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -911,7 +911,7 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -930,7 +930,7 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -956,7 +956,7 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; SSE-NEXT:    pand %xmm0, %xmm2
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE-NEXT:    pand %xmm2, %xmm0
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE-NEXT:    pand %xmm0, %xmm1
 ; SSE-NEXT:    movdqa %xmm1, %xmm0
 ; SSE-NEXT:    psrld $16, %xmm0
@@ -977,7 +977,7 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -997,7 +997,7 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -1017,7 +1017,7 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
diff --git a/llvm/test/CodeGen/X86/vector-reduce-fadd-fast.ll b/llvm/test/CodeGen/X86/vector-reduce-fadd-fast.ll
index 7900669019056..e98833e3b4c21 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-fadd-fast.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-fadd-fast.ll
@@ -15,7 +15,7 @@ define float @test_v2f32(float %a0, <2 x float> %a1) {
 ; SSE2-LABEL: test_v2f32:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movaps %xmm1, %xmm2
-; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1],xmm1[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1],xmm1[1,1]
 ; SSE2-NEXT:    addss %xmm1, %xmm2
 ; SSE2-NEXT:    addss %xmm2, %xmm0
 ; SSE2-NEXT:    retq
@@ -64,7 +64,7 @@ define float @test_v4f32(float %a0, <4 x float> %a1) {
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm1[1]
 ; SSE2-NEXT:    addps %xmm1, %xmm2
 ; SSE2-NEXT:    movaps %xmm2, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm2[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm2[1,1]
 ; SSE2-NEXT:    addss %xmm2, %xmm1
 ; SSE2-NEXT:    addss %xmm1, %xmm0
 ; SSE2-NEXT:    retq
@@ -124,7 +124,7 @@ define float @test_v8f32(float %a0, <8 x float> %a1) {
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm1[1]
 ; SSE2-NEXT:    addps %xmm1, %xmm2
 ; SSE2-NEXT:    movaps %xmm2, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm2[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm2[1,1]
 ; SSE2-NEXT:    addss %xmm2, %xmm1
 ; SSE2-NEXT:    addss %xmm1, %xmm0
 ; SSE2-NEXT:    retq
@@ -199,7 +199,7 @@ define float @test_v16f32(float %a0, <16 x float> %a1) {
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm1[1]
 ; SSE2-NEXT:    addps %xmm1, %xmm2
 ; SSE2-NEXT:    movaps %xmm2, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm2[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm2[1,1]
 ; SSE2-NEXT:    addss %xmm2, %xmm1
 ; SSE2-NEXT:    addss %xmm1, %xmm0
 ; SSE2-NEXT:    retq
@@ -280,7 +280,7 @@ define float @test_v2f32_zero(<2 x float> %a0) {
 ; SSE2-LABEL: test_v2f32_zero:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movaps %xmm0, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[1,1]
 ; SSE2-NEXT:    addss %xmm0, %xmm1
 ; SSE2-NEXT:    movaps %xmm1, %xmm0
 ; SSE2-NEXT:    retq
@@ -324,7 +324,7 @@ define float @test_v4f32_zero(<4 x float> %a0) {
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
 ; SSE2-NEXT:    addps %xmm0, %xmm1
 ; SSE2-NEXT:    movaps %xmm1, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[1,1]
 ; SSE2-NEXT:    addss %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
@@ -379,7 +379,7 @@ define float @test_v8f32_zero(<8 x float> %a0) {
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
 ; SSE2-NEXT:    addps %xmm0, %xmm1
 ; SSE2-NEXT:    movaps %xmm1, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[1,1]
 ; SSE2-NEXT:    addss %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
@@ -449,7 +449,7 @@ define float @test_v16f32_zero(<16 x float> %a0) {
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
 ; SSE2-NEXT:    addps %xmm0, %xmm1
 ; SSE2-NEXT:    movaps %xmm1, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[1,1]
 ; SSE2-NEXT:    addss %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
@@ -525,7 +525,7 @@ define float @test_v2f32_undef(<2 x float> %a0) {
 ; SSE2-LABEL: test_v2f32_undef:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movaps %xmm0, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[1,1]
 ; SSE2-NEXT:    addss %xmm0, %xmm1
 ; SSE2-NEXT:    movaps %xmm1, %xmm0
 ; SSE2-NEXT:    retq
@@ -569,7 +569,7 @@ define float @test_v4f32_undef(<4 x float> %a0) {
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
 ; SSE2-NEXT:    addps %xmm0, %xmm1
 ; SSE2-NEXT:    movaps %xmm1, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[1,1]
 ; SSE2-NEXT:    addss %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
@@ -624,7 +624,7 @@ define float @test_v8f32_undef(<8 x float> %a0) {
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
 ; SSE2-NEXT:    addps %xmm0, %xmm1
 ; SSE2-NEXT:    movaps %xmm1, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[1,1]
 ; SSE2-NEXT:    addss %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
@@ -694,7 +694,7 @@ define float @test_v16f32_undef(<16 x float> %a0) {
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
 ; SSE2-NEXT:    addps %xmm0, %xmm1
 ; SSE2-NEXT:    movaps %xmm1, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[1,1]
 ; SSE2-NEXT:    addss %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/vector-reduce-fadd.ll b/llvm/test/CodeGen/X86/vector-reduce-fadd.ll
index 54fe527ceaeed..7de6a25029e7b 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-fadd.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-fadd.ll
@@ -15,7 +15,7 @@ define float @test_v2f32(float %a0, <2 x float> %a1) {
 ; SSE2-LABEL: test_v2f32:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    addss %xmm1, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,1,1]
 ; SSE2-NEXT:    addss %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
@@ -48,12 +48,12 @@ define float @test_v4f32(float %a0, <4 x float> %a1) {
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    addss %xmm1, %xmm0
 ; SSE2-NEXT:    movaps %xmm1, %xmm2
-; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1],xmm1[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1],xmm1[1,1]
 ; SSE2-NEXT:    addss %xmm2, %xmm0
 ; SSE2-NEXT:    movaps %xmm1, %xmm2
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm1[1]
 ; SSE2-NEXT:    addss %xmm2, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; SSE2-NEXT:    addss %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
@@ -65,7 +65,7 @@ define float @test_v4f32(float %a0, <4 x float> %a1) {
 ; SSE41-NEXT:    movaps %xmm1, %xmm2
 ; SSE41-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm1[1]
 ; SSE41-NEXT:    addss %xmm2, %xmm0
-; SSE41-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; SSE41-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; SSE41-NEXT:    addss %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
@@ -76,7 +76,7 @@ define float @test_v4f32(float %a0, <4 x float> %a1) {
 ; AVX-NEXT:    vaddss %xmm2, %xmm0, %xmm0
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm1[1,0]
 ; AVX-NEXT:    vaddss %xmm2, %xmm0, %xmm0
-; AVX-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; AVX-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; AVX-NEXT:    vaddss %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
@@ -87,7 +87,7 @@ define float @test_v4f32(float %a0, <4 x float> %a1) {
 ; AVX512-NEXT:    vaddss %xmm2, %xmm0, %xmm0
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm1[1,0]
 ; AVX512-NEXT:    vaddss %xmm2, %xmm0, %xmm0
-; AVX512-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; AVX512-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; AVX512-NEXT:    vaddss %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    retq
   %1 = call float @llvm.experimental.vector.reduce.v2.fadd.f32.v4f32(float %a0, <4 x float> %a1)
@@ -99,21 +99,21 @@ define float @test_v8f32(float %a0, <8 x float> %a1) {
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    addss %xmm1, %xmm0
 ; SSE2-NEXT:    movaps %xmm1, %xmm3
-; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm1[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm1[1,1]
 ; SSE2-NEXT:    addss %xmm3, %xmm0
 ; SSE2-NEXT:    movaps %xmm1, %xmm3
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm3 = xmm3[1],xmm1[1]
 ; SSE2-NEXT:    addss %xmm3, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; SSE2-NEXT:    addss %xmm1, %xmm0
 ; SSE2-NEXT:    addss %xmm2, %xmm0
 ; SSE2-NEXT:    movaps %xmm2, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm2[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm2[1,1]
 ; SSE2-NEXT:    addss %xmm1, %xmm0
 ; SSE2-NEXT:    movaps %xmm2, %xmm1
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm2[1]
 ; SSE2-NEXT:    addss %xmm1, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,3,3,3]
 ; SSE2-NEXT:    addss %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
@@ -125,7 +125,7 @@ define float @test_v8f32(float %a0, <8 x float> %a1) {
 ; SSE41-NEXT:    movaps %xmm1, %xmm3
 ; SSE41-NEXT:    unpckhpd {{.*#+}} xmm3 = xmm3[1],xmm1[1]
 ; SSE41-NEXT:    addss %xmm3, %xmm0
-; SSE41-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; SSE41-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; SSE41-NEXT:    addss %xmm1, %xmm0
 ; SSE41-NEXT:    addss %xmm2, %xmm0
 ; SSE41-NEXT:    movshdup {{.*#+}} xmm1 = xmm2[1,1,3,3]
@@ -133,7 +133,7 @@ define float @test_v8f32(float %a0, <8 x float> %a1) {
 ; SSE41-NEXT:    movaps %xmm2, %xmm1
 ; SSE41-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm2[1]
 ; SSE41-NEXT:    addss %xmm1, %xmm0
-; SSE41-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,1,2,3]
+; SSE41-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,3,3,3]
 ; SSE41-NEXT:    addss %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
@@ -144,7 +144,7 @@ define float @test_v8f32(float %a0, <8 x float> %a1) {
 ; AVX-NEXT:    vaddss %xmm2, %xmm0, %xmm0
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm1[1,0]
 ; AVX-NEXT:    vaddss %xmm2, %xmm0, %xmm0
-; AVX-NEXT:    vpermilps {{.*#+}} xmm2 = xmm1[3,1,2,3]
+; AVX-NEXT:    vpermilps {{.*#+}} xmm2 = xmm1[3,3,3,3]
 ; AVX-NEXT:    vaddss %xmm2, %xmm0, %xmm0
 ; AVX-NEXT:    vextractf128 $1, %ymm1, %xmm1
 ; AVX-NEXT:    vaddss %xmm1, %xmm0, %xmm0
@@ -152,7 +152,7 @@ define float @test_v8f32(float %a0, <8 x float> %a1) {
 ; AVX-NEXT:    vaddss %xmm2, %xmm0, %xmm0
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm1[1,0]
 ; AVX-NEXT:    vaddss %xmm2, %xmm0, %xmm0
-; AVX-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; AVX-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; AVX-NEXT:    vaddss %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vzeroupper
 ; AVX-NEXT:    retq
@@ -164,7 +164,7 @@ define float @test_v8f32(float %a0, <8 x float> %a1) {
 ; AVX512-NEXT:    vaddss %xmm2, %xmm0, %xmm0
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm1[1,0]
 ; AVX512-NEXT:    vaddss %xmm2, %xmm0, %xmm0
-; AVX512-NEXT:    vpermilps {{.*#+}} xmm2 = xmm1[3,1,2,3]
+; AVX512-NEXT:    vpermilps {{.*#+}} xmm2 = xmm1[3,3,3,3]
 ; AVX512-NEXT:    vaddss %xmm2, %xmm0, %xmm0
 ; AVX512-NEXT:    vextractf128 $1, %ymm1, %xmm1
 ; AVX512-NEXT:    vaddss %xmm1, %xmm0, %xmm0
@@ -172,7 +172,7 @@ define float @test_v8f32(float %a0, <8 x float> %a1) {
 ; AVX512-NEXT:    vaddss %xmm2, %xmm0, %xmm0
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm1[1,0]
 ; AVX512-NEXT:    vaddss %xmm2, %xmm0, %xmm0
-; AVX512-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; AVX512-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; AVX512-NEXT:    vaddss %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
@@ -185,39 +185,39 @@ define float @test_v16f32(float %a0, <16 x float> %a1) {
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    addss %xmm1, %xmm0
 ; SSE2-NEXT:    movaps %xmm1, %xmm5
-; SSE2-NEXT:    shufps {{.*#+}} xmm5 = xmm5[1,1],xmm1[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm5 = xmm5[1,1],xmm1[1,1]
 ; SSE2-NEXT:    addss %xmm5, %xmm0
 ; SSE2-NEXT:    movaps %xmm1, %xmm5
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm5 = xmm5[1],xmm1[1]
 ; SSE2-NEXT:    addss %xmm5, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; SSE2-NEXT:    addss %xmm1, %xmm0
 ; SSE2-NEXT:    addss %xmm2, %xmm0
 ; SSE2-NEXT:    movaps %xmm2, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm2[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm2[1,1]
 ; SSE2-NEXT:    addss %xmm1, %xmm0
 ; SSE2-NEXT:    movaps %xmm2, %xmm1
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm2[1]
 ; SSE2-NEXT:    addss %xmm1, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,3,3,3]
 ; SSE2-NEXT:    addss %xmm2, %xmm0
 ; SSE2-NEXT:    addss %xmm3, %xmm0
 ; SSE2-NEXT:    movaps %xmm3, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm3[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm3[1,1]
 ; SSE2-NEXT:    addss %xmm1, %xmm0
 ; SSE2-NEXT:    movaps %xmm3, %xmm1
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm3[1]
 ; SSE2-NEXT:    addss %xmm1, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,3,3,3]
 ; SSE2-NEXT:    addss %xmm3, %xmm0
 ; SSE2-NEXT:    addss %xmm4, %xmm0
 ; SSE2-NEXT:    movaps %xmm4, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm4[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm4[1,1]
 ; SSE2-NEXT:    addss %xmm1, %xmm0
 ; SSE2-NEXT:    movaps %xmm4, %xmm1
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm4[1]
 ; SSE2-NEXT:    addss %xmm1, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,3,3,3]
 ; SSE2-NEXT:    addss %xmm4, %xmm0
 ; SSE2-NEXT:    retq
 ;
@@ -229,7 +229,7 @@ define float @test_v16f32(float %a0, <16 x float> %a1) {
 ; SSE41-NEXT:    movaps %xmm1, %xmm5
 ; SSE41-NEXT:    unpckhpd {{.*#+}} xmm5 = xmm5[1],xmm1[1]
 ; SSE41-NEXT:    addss %xmm5, %xmm0
-; SSE41-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; SSE41-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; SSE41-NEXT:    addss %xmm1, %xmm0
 ; SSE41-NEXT:    addss %xmm2, %xmm0
 ; SSE41-NEXT:    movshdup {{.*#+}} xmm1 = xmm2[1,1,3,3]
@@ -237,7 +237,7 @@ define float @test_v16f32(float %a0, <16 x float> %a1) {
 ; SSE41-NEXT:    movaps %xmm2, %xmm1
 ; SSE41-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm2[1]
 ; SSE41-NEXT:    addss %xmm1, %xmm0
-; SSE41-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,1,2,3]
+; SSE41-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,3,3,3]
 ; SSE41-NEXT:    addss %xmm2, %xmm0
 ; SSE41-NEXT:    addss %xmm3, %xmm0
 ; SSE41-NEXT:    movshdup {{.*#+}} xmm1 = xmm3[1,1,3,3]
@@ -245,7 +245,7 @@ define float @test_v16f32(float %a0, <16 x float> %a1) {
 ; SSE41-NEXT:    movaps %xmm3, %xmm1
 ; SSE41-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm3[1]
 ; SSE41-NEXT:    addss %xmm1, %xmm0
-; SSE41-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,1,2,3]
+; SSE41-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,3,3,3]
 ; SSE41-NEXT:    addss %xmm3, %xmm0
 ; SSE41-NEXT:    addss %xmm4, %xmm0
 ; SSE41-NEXT:    movshdup {{.*#+}} xmm1 = xmm4[1,1,3,3]
@@ -253,7 +253,7 @@ define float @test_v16f32(float %a0, <16 x float> %a1) {
 ; SSE41-NEXT:    movaps %xmm4, %xmm1
 ; SSE41-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm4[1]
 ; SSE41-NEXT:    addss %xmm1, %xmm0
-; SSE41-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,1,2,3]
+; SSE41-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,3,3,3]
 ; SSE41-NEXT:    addss %xmm4, %xmm0
 ; SSE41-NEXT:    retq
 ;
@@ -264,7 +264,7 @@ define float @test_v16f32(float %a0, <16 x float> %a1) {
 ; AVX-NEXT:    vaddss %xmm3, %xmm0, %xmm0
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm3 = xmm1[1,0]
 ; AVX-NEXT:    vaddss %xmm3, %xmm0, %xmm0
-; AVX-NEXT:    vpermilps {{.*#+}} xmm3 = xmm1[3,1,2,3]
+; AVX-NEXT:    vpermilps {{.*#+}} xmm3 = xmm1[3,3,3,3]
 ; AVX-NEXT:    vaddss %xmm3, %xmm0, %xmm0
 ; AVX-NEXT:    vextractf128 $1, %ymm1, %xmm1
 ; AVX-NEXT:    vaddss %xmm1, %xmm0, %xmm0
@@ -272,14 +272,14 @@ define float @test_v16f32(float %a0, <16 x float> %a1) {
 ; AVX-NEXT:    vaddss %xmm3, %xmm0, %xmm0
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm3 = xmm1[1,0]
 ; AVX-NEXT:    vaddss %xmm3, %xmm0, %xmm0
-; AVX-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; AVX-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; AVX-NEXT:    vaddss %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vaddss %xmm2, %xmm0, %xmm0
 ; AVX-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm2[1,1,3,3]
 ; AVX-NEXT:    vaddss %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm2[1,0]
 ; AVX-NEXT:    vaddss %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpermilps {{.*#+}} xmm1 = xmm2[3,1,2,3]
+; AVX-NEXT:    vpermilps {{.*#+}} xmm1 = xmm2[3,3,3,3]
 ; AVX-NEXT:    vaddss %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vextractf128 $1, %ymm2, %xmm1
 ; AVX-NEXT:    vaddss %xmm1, %xmm0, %xmm0
@@ -287,7 +287,7 @@ define float @test_v16f32(float %a0, <16 x float> %a1) {
 ; AVX-NEXT:    vaddss %xmm2, %xmm0, %xmm0
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm1[1,0]
 ; AVX-NEXT:    vaddss %xmm2, %xmm0, %xmm0
-; AVX-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; AVX-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; AVX-NEXT:    vaddss %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vzeroupper
 ; AVX-NEXT:    retq
@@ -299,7 +299,7 @@ define float @test_v16f32(float %a0, <16 x float> %a1) {
 ; AVX512-NEXT:    vaddss %xmm2, %xmm0, %xmm0
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm1[1,0]
 ; AVX512-NEXT:    vaddss %xmm2, %xmm0, %xmm0
-; AVX512-NEXT:    vpermilps {{.*#+}} xmm2 = xmm1[3,1,2,3]
+; AVX512-NEXT:    vpermilps {{.*#+}} xmm2 = xmm1[3,3,3,3]
 ; AVX512-NEXT:    vaddss %xmm2, %xmm0, %xmm0
 ; AVX512-NEXT:    vextractf128 $1, %ymm1, %xmm2
 ; AVX512-NEXT:    vaddss %xmm2, %xmm0, %xmm0
@@ -307,7 +307,7 @@ define float @test_v16f32(float %a0, <16 x float> %a1) {
 ; AVX512-NEXT:    vaddss %xmm3, %xmm0, %xmm0
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm3 = xmm2[1,0]
 ; AVX512-NEXT:    vaddss %xmm3, %xmm0, %xmm0
-; AVX512-NEXT:    vpermilps {{.*#+}} xmm2 = xmm2[3,1,2,3]
+; AVX512-NEXT:    vpermilps {{.*#+}} xmm2 = xmm2[3,3,3,3]
 ; AVX512-NEXT:    vaddss %xmm2, %xmm0, %xmm0
 ; AVX512-NEXT:    vextractf32x4 $2, %zmm1, %xmm2
 ; AVX512-NEXT:    vaddss %xmm2, %xmm0, %xmm0
@@ -315,7 +315,7 @@ define float @test_v16f32(float %a0, <16 x float> %a1) {
 ; AVX512-NEXT:    vaddss %xmm3, %xmm0, %xmm0
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm3 = xmm2[1,0]
 ; AVX512-NEXT:    vaddss %xmm3, %xmm0, %xmm0
-; AVX512-NEXT:    vpermilps {{.*#+}} xmm2 = xmm2[3,1,2,3]
+; AVX512-NEXT:    vpermilps {{.*#+}} xmm2 = xmm2[3,3,3,3]
 ; AVX512-NEXT:    vaddss %xmm2, %xmm0, %xmm0
 ; AVX512-NEXT:    vextractf32x4 $3, %zmm1, %xmm1
 ; AVX512-NEXT:    vaddss %xmm1, %xmm0, %xmm0
@@ -323,7 +323,7 @@ define float @test_v16f32(float %a0, <16 x float> %a1) {
 ; AVX512-NEXT:    vaddss %xmm2, %xmm0, %xmm0
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm1[1,0]
 ; AVX512-NEXT:    vaddss %xmm2, %xmm0, %xmm0
-; AVX512-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; AVX512-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; AVX512-NEXT:    vaddss %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
@@ -340,7 +340,7 @@ define float @test_v2f32_zero(<2 x float> %a0) {
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    xorps %xmm1, %xmm1
 ; SSE2-NEXT:    addss %xmm0, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    addss %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
@@ -377,12 +377,12 @@ define float @test_v4f32_zero(<4 x float> %a0) {
 ; SSE2-NEXT:    xorps %xmm1, %xmm1
 ; SSE2-NEXT:    addss %xmm0, %xmm1
 ; SSE2-NEXT:    movaps %xmm0, %xmm2
-; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1],xmm0[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1],xmm0[1,1]
 ; SSE2-NEXT:    addss %xmm1, %xmm2
 ; SSE2-NEXT:    movaps %xmm0, %xmm1
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
 ; SSE2-NEXT:    addss %xmm2, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE2-NEXT:    addss %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
@@ -395,7 +395,7 @@ define float @test_v4f32_zero(<4 x float> %a0) {
 ; SSE41-NEXT:    movaps %xmm0, %xmm1
 ; SSE41-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
 ; SSE41-NEXT:    addss %xmm2, %xmm1
-; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE41-NEXT:    addss %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
@@ -407,7 +407,7 @@ define float @test_v4f32_zero(<4 x float> %a0) {
 ; AVX-NEXT:    vaddss %xmm2, %xmm1, %xmm1
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
 ; AVX-NEXT:    vaddss %xmm2, %xmm1, %xmm1
-; AVX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX-NEXT:    vaddss %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    retq
 ;
@@ -419,7 +419,7 @@ define float @test_v4f32_zero(<4 x float> %a0) {
 ; AVX512-NEXT:    vaddss %xmm2, %xmm1, %xmm1
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
 ; AVX512-NEXT:    vaddss %xmm2, %xmm1, %xmm1
-; AVX512-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX512-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX512-NEXT:    vaddss %xmm0, %xmm1, %xmm0
 ; AVX512-NEXT:    retq
   %1 = call float @llvm.experimental.vector.reduce.v2.fadd.f32.v4f32(float 0.0, <4 x float> %a0)
@@ -432,21 +432,21 @@ define float @test_v8f32_zero(<8 x float> %a0) {
 ; SSE2-NEXT:    xorps %xmm2, %xmm2
 ; SSE2-NEXT:    addss %xmm0, %xmm2
 ; SSE2-NEXT:    movaps %xmm0, %xmm3
-; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm0[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm0[1,1]
 ; SSE2-NEXT:    addss %xmm2, %xmm3
 ; SSE2-NEXT:    movaps %xmm0, %xmm2
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm0[1]
 ; SSE2-NEXT:    addss %xmm3, %xmm2
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE2-NEXT:    addss %xmm2, %xmm0
 ; SSE2-NEXT:    addss %xmm1, %xmm0
 ; SSE2-NEXT:    movaps %xmm1, %xmm2
-; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1],xmm1[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1],xmm1[1,1]
 ; SSE2-NEXT:    addss %xmm2, %xmm0
 ; SSE2-NEXT:    movaps %xmm1, %xmm2
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm1[1]
 ; SSE2-NEXT:    addss %xmm2, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; SSE2-NEXT:    addss %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
@@ -459,7 +459,7 @@ define float @test_v8f32_zero(<8 x float> %a0) {
 ; SSE41-NEXT:    movaps %xmm0, %xmm2
 ; SSE41-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm0[1]
 ; SSE41-NEXT:    addss %xmm3, %xmm2
-; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE41-NEXT:    addss %xmm2, %xmm0
 ; SSE41-NEXT:    addss %xmm1, %xmm0
 ; SSE41-NEXT:    movshdup {{.*#+}} xmm2 = xmm1[1,1,3,3]
@@ -467,7 +467,7 @@ define float @test_v8f32_zero(<8 x float> %a0) {
 ; SSE41-NEXT:    movaps %xmm1, %xmm2
 ; SSE41-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm1[1]
 ; SSE41-NEXT:    addss %xmm2, %xmm0
-; SSE41-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; SSE41-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; SSE41-NEXT:    addss %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
@@ -479,7 +479,7 @@ define float @test_v8f32_zero(<8 x float> %a0) {
 ; AVX-NEXT:    vaddss %xmm2, %xmm1, %xmm1
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
 ; AVX-NEXT:    vaddss %xmm2, %xmm1, %xmm1
-; AVX-NEXT:    vpermilps {{.*#+}} xmm2 = xmm0[3,1,2,3]
+; AVX-NEXT:    vpermilps {{.*#+}} xmm2 = xmm0[3,3,3,3]
 ; AVX-NEXT:    vaddss %xmm2, %xmm1, %xmm1
 ; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; AVX-NEXT:    vaddss %xmm0, %xmm1, %xmm1
@@ -487,7 +487,7 @@ define float @test_v8f32_zero(<8 x float> %a0) {
 ; AVX-NEXT:    vaddss %xmm2, %xmm1, %xmm1
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
 ; AVX-NEXT:    vaddss %xmm2, %xmm1, %xmm1
-; AVX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX-NEXT:    vaddss %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    vzeroupper
 ; AVX-NEXT:    retq
@@ -500,7 +500,7 @@ define float @test_v8f32_zero(<8 x float> %a0) {
 ; AVX512-NEXT:    vaddss %xmm2, %xmm1, %xmm1
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
 ; AVX512-NEXT:    vaddss %xmm2, %xmm1, %xmm1
-; AVX512-NEXT:    vpermilps {{.*#+}} xmm2 = xmm0[3,1,2,3]
+; AVX512-NEXT:    vpermilps {{.*#+}} xmm2 = xmm0[3,3,3,3]
 ; AVX512-NEXT:    vaddss %xmm2, %xmm1, %xmm1
 ; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; AVX512-NEXT:    vaddss %xmm0, %xmm1, %xmm1
@@ -508,7 +508,7 @@ define float @test_v8f32_zero(<8 x float> %a0) {
 ; AVX512-NEXT:    vaddss %xmm2, %xmm1, %xmm1
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
 ; AVX512-NEXT:    vaddss %xmm2, %xmm1, %xmm1
-; AVX512-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX512-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX512-NEXT:    vaddss %xmm0, %xmm1, %xmm0
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
@@ -522,39 +522,39 @@ define float @test_v16f32_zero(<16 x float> %a0) {
 ; SSE2-NEXT:    xorps %xmm4, %xmm4
 ; SSE2-NEXT:    addss %xmm0, %xmm4
 ; SSE2-NEXT:    movaps %xmm0, %xmm5
-; SSE2-NEXT:    shufps {{.*#+}} xmm5 = xmm5[1,1],xmm0[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm5 = xmm5[1,1],xmm0[1,1]
 ; SSE2-NEXT:    addss %xmm4, %xmm5
 ; SSE2-NEXT:    movaps %xmm0, %xmm4
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm0[1]
 ; SSE2-NEXT:    addss %xmm5, %xmm4
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE2-NEXT:    addss %xmm4, %xmm0
 ; SSE2-NEXT:    addss %xmm1, %xmm0
 ; SSE2-NEXT:    movaps %xmm1, %xmm4
-; SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[1,1],xmm1[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[1,1],xmm1[1,1]
 ; SSE2-NEXT:    addss %xmm4, %xmm0
 ; SSE2-NEXT:    movaps %xmm1, %xmm4
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm1[1]
 ; SSE2-NEXT:    addss %xmm4, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; SSE2-NEXT:    addss %xmm1, %xmm0
 ; SSE2-NEXT:    addss %xmm2, %xmm0
 ; SSE2-NEXT:    movaps %xmm2, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm2[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm2[1,1]
 ; SSE2-NEXT:    addss %xmm1, %xmm0
 ; SSE2-NEXT:    movaps %xmm2, %xmm1
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm2[1]
 ; SSE2-NEXT:    addss %xmm1, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,3,3,3]
 ; SSE2-NEXT:    addss %xmm2, %xmm0
 ; SSE2-NEXT:    addss %xmm3, %xmm0
 ; SSE2-NEXT:    movaps %xmm3, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm3[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm3[1,1]
 ; SSE2-NEXT:    addss %xmm1, %xmm0
 ; SSE2-NEXT:    movaps %xmm3, %xmm1
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm3[1]
 ; SSE2-NEXT:    addss %xmm1, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,3,3,3]
 ; SSE2-NEXT:    addss %xmm3, %xmm0
 ; SSE2-NEXT:    retq
 ;
@@ -567,7 +567,7 @@ define float @test_v16f32_zero(<16 x float> %a0) {
 ; SSE41-NEXT:    movaps %xmm0, %xmm4
 ; SSE41-NEXT:    unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm0[1]
 ; SSE41-NEXT:    addss %xmm5, %xmm4
-; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE41-NEXT:    addss %xmm4, %xmm0
 ; SSE41-NEXT:    addss %xmm1, %xmm0
 ; SSE41-NEXT:    movshdup {{.*#+}} xmm4 = xmm1[1,1,3,3]
@@ -575,7 +575,7 @@ define float @test_v16f32_zero(<16 x float> %a0) {
 ; SSE41-NEXT:    movaps %xmm1, %xmm4
 ; SSE41-NEXT:    unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm1[1]
 ; SSE41-NEXT:    addss %xmm4, %xmm0
-; SSE41-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; SSE41-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; SSE41-NEXT:    addss %xmm1, %xmm0
 ; SSE41-NEXT:    addss %xmm2, %xmm0
 ; SSE41-NEXT:    movshdup {{.*#+}} xmm1 = xmm2[1,1,3,3]
@@ -583,7 +583,7 @@ define float @test_v16f32_zero(<16 x float> %a0) {
 ; SSE41-NEXT:    movaps %xmm2, %xmm1
 ; SSE41-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm2[1]
 ; SSE41-NEXT:    addss %xmm1, %xmm0
-; SSE41-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,1,2,3]
+; SSE41-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,3,3,3]
 ; SSE41-NEXT:    addss %xmm2, %xmm0
 ; SSE41-NEXT:    addss %xmm3, %xmm0
 ; SSE41-NEXT:    movshdup {{.*#+}} xmm1 = xmm3[1,1,3,3]
@@ -591,7 +591,7 @@ define float @test_v16f32_zero(<16 x float> %a0) {
 ; SSE41-NEXT:    movaps %xmm3, %xmm1
 ; SSE41-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm3[1]
 ; SSE41-NEXT:    addss %xmm1, %xmm0
-; SSE41-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,1,2,3]
+; SSE41-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,3,3,3]
 ; SSE41-NEXT:    addss %xmm3, %xmm0
 ; SSE41-NEXT:    retq
 ;
@@ -603,7 +603,7 @@ define float @test_v16f32_zero(<16 x float> %a0) {
 ; AVX-NEXT:    vaddss %xmm3, %xmm2, %xmm2
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm3 = xmm0[1,0]
 ; AVX-NEXT:    vaddss %xmm3, %xmm2, %xmm2
-; AVX-NEXT:    vpermilps {{.*#+}} xmm3 = xmm0[3,1,2,3]
+; AVX-NEXT:    vpermilps {{.*#+}} xmm3 = xmm0[3,3,3,3]
 ; AVX-NEXT:    vaddss %xmm3, %xmm2, %xmm2
 ; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; AVX-NEXT:    vaddss %xmm0, %xmm2, %xmm2
@@ -611,14 +611,14 @@ define float @test_v16f32_zero(<16 x float> %a0) {
 ; AVX-NEXT:    vaddss %xmm3, %xmm2, %xmm2
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm3 = xmm0[1,0]
 ; AVX-NEXT:    vaddss %xmm3, %xmm2, %xmm2
-; AVX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX-NEXT:    vaddss %xmm0, %xmm2, %xmm0
 ; AVX-NEXT:    vaddss %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovshdup {{.*#+}} xmm2 = xmm1[1,1,3,3]
 ; AVX-NEXT:    vaddss %xmm2, %xmm0, %xmm0
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm1[1,0]
 ; AVX-NEXT:    vaddss %xmm2, %xmm0, %xmm0
-; AVX-NEXT:    vpermilps {{.*#+}} xmm2 = xmm1[3,1,2,3]
+; AVX-NEXT:    vpermilps {{.*#+}} xmm2 = xmm1[3,3,3,3]
 ; AVX-NEXT:    vaddss %xmm2, %xmm0, %xmm0
 ; AVX-NEXT:    vextractf128 $1, %ymm1, %xmm1
 ; AVX-NEXT:    vaddss %xmm1, %xmm0, %xmm0
@@ -626,7 +626,7 @@ define float @test_v16f32_zero(<16 x float> %a0) {
 ; AVX-NEXT:    vaddss %xmm2, %xmm0, %xmm0
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm1[1,0]
 ; AVX-NEXT:    vaddss %xmm2, %xmm0, %xmm0
-; AVX-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; AVX-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; AVX-NEXT:    vaddss %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vzeroupper
 ; AVX-NEXT:    retq
@@ -639,7 +639,7 @@ define float @test_v16f32_zero(<16 x float> %a0) {
 ; AVX512-NEXT:    vaddss %xmm2, %xmm1, %xmm1
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
 ; AVX512-NEXT:    vaddss %xmm2, %xmm1, %xmm1
-; AVX512-NEXT:    vpermilps {{.*#+}} xmm2 = xmm0[3,1,2,3]
+; AVX512-NEXT:    vpermilps {{.*#+}} xmm2 = xmm0[3,3,3,3]
 ; AVX512-NEXT:    vaddss %xmm2, %xmm1, %xmm1
 ; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm2
 ; AVX512-NEXT:    vaddss %xmm2, %xmm1, %xmm1
@@ -647,7 +647,7 @@ define float @test_v16f32_zero(<16 x float> %a0) {
 ; AVX512-NEXT:    vaddss %xmm3, %xmm1, %xmm1
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm3 = xmm2[1,0]
 ; AVX512-NEXT:    vaddss %xmm3, %xmm1, %xmm1
-; AVX512-NEXT:    vpermilps {{.*#+}} xmm2 = xmm2[3,1,2,3]
+; AVX512-NEXT:    vpermilps {{.*#+}} xmm2 = xmm2[3,3,3,3]
 ; AVX512-NEXT:    vaddss %xmm2, %xmm1, %xmm1
 ; AVX512-NEXT:    vextractf32x4 $2, %zmm0, %xmm2
 ; AVX512-NEXT:    vaddss %xmm2, %xmm1, %xmm1
@@ -655,7 +655,7 @@ define float @test_v16f32_zero(<16 x float> %a0) {
 ; AVX512-NEXT:    vaddss %xmm3, %xmm1, %xmm1
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm3 = xmm2[1,0]
 ; AVX512-NEXT:    vaddss %xmm3, %xmm1, %xmm1
-; AVX512-NEXT:    vpermilps {{.*#+}} xmm2 = xmm2[3,1,2,3]
+; AVX512-NEXT:    vpermilps {{.*#+}} xmm2 = xmm2[3,3,3,3]
 ; AVX512-NEXT:    vaddss %xmm2, %xmm1, %xmm1
 ; AVX512-NEXT:    vextractf32x4 $3, %zmm0, %xmm0
 ; AVX512-NEXT:    vaddss %xmm0, %xmm1, %xmm1
@@ -663,7 +663,7 @@ define float @test_v16f32_zero(<16 x float> %a0) {
 ; AVX512-NEXT:    vaddss %xmm2, %xmm1, %xmm1
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
 ; AVX512-NEXT:    vaddss %xmm2, %xmm1, %xmm1
-; AVX512-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX512-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX512-NEXT:    vaddss %xmm0, %xmm1, %xmm0
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
@@ -678,7 +678,7 @@ define float @test_v16f32_zero(<16 x float> %a0) {
 define float @test_v2f32_undef(<2 x float> %a0) {
 ; SSE2-LABEL: test_v2f32_undef:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    addss {{.*}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
@@ -707,12 +707,12 @@ define float @test_v4f32_undef(<4 x float> %a0) {
 ; SSE2-LABEL: test_v4f32_undef:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movaps %xmm0, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[1,1]
 ; SSE2-NEXT:    addss {{.*}}(%rip), %xmm1
 ; SSE2-NEXT:    movaps %xmm0, %xmm2
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm0[1]
 ; SSE2-NEXT:    addss %xmm1, %xmm2
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE2-NEXT:    addss %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
@@ -723,7 +723,7 @@ define float @test_v4f32_undef(<4 x float> %a0) {
 ; SSE41-NEXT:    movaps %xmm0, %xmm2
 ; SSE41-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm0[1]
 ; SSE41-NEXT:    addss %xmm1, %xmm2
-; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE41-NEXT:    addss %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
@@ -733,7 +733,7 @@ define float @test_v4f32_undef(<4 x float> %a0) {
 ; AVX-NEXT:    vaddss {{.*}}(%rip), %xmm1, %xmm1
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
 ; AVX-NEXT:    vaddss %xmm2, %xmm1, %xmm1
-; AVX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX-NEXT:    vaddss %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    retq
 ;
@@ -743,7 +743,7 @@ define float @test_v4f32_undef(<4 x float> %a0) {
 ; AVX512-NEXT:    vaddss {{.*}}(%rip), %xmm1, %xmm1
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
 ; AVX512-NEXT:    vaddss %xmm2, %xmm1, %xmm1
-; AVX512-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX512-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX512-NEXT:    vaddss %xmm0, %xmm1, %xmm0
 ; AVX512-NEXT:    retq
   %1 = call float @llvm.experimental.vector.reduce.v2.fadd.f32.v4f32(float undef, <4 x float> %a0)
@@ -754,21 +754,21 @@ define float @test_v8f32_undef(<8 x float> %a0) {
 ; SSE2-LABEL: test_v8f32_undef:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movaps %xmm0, %xmm2
-; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1],xmm0[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1],xmm0[1,1]
 ; SSE2-NEXT:    addss {{.*}}(%rip), %xmm2
 ; SSE2-NEXT:    movaps %xmm0, %xmm3
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm3 = xmm3[1],xmm0[1]
 ; SSE2-NEXT:    addss %xmm2, %xmm3
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE2-NEXT:    addss %xmm3, %xmm0
 ; SSE2-NEXT:    addss %xmm1, %xmm0
 ; SSE2-NEXT:    movaps %xmm1, %xmm2
-; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1],xmm1[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1],xmm1[1,1]
 ; SSE2-NEXT:    addss %xmm2, %xmm0
 ; SSE2-NEXT:    movaps %xmm1, %xmm2
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm1[1]
 ; SSE2-NEXT:    addss %xmm2, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; SSE2-NEXT:    addss %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
@@ -779,7 +779,7 @@ define float @test_v8f32_undef(<8 x float> %a0) {
 ; SSE41-NEXT:    movaps %xmm0, %xmm3
 ; SSE41-NEXT:    unpckhpd {{.*#+}} xmm3 = xmm3[1],xmm0[1]
 ; SSE41-NEXT:    addss %xmm2, %xmm3
-; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE41-NEXT:    addss %xmm3, %xmm0
 ; SSE41-NEXT:    addss %xmm1, %xmm0
 ; SSE41-NEXT:    movshdup {{.*#+}} xmm2 = xmm1[1,1,3,3]
@@ -787,7 +787,7 @@ define float @test_v8f32_undef(<8 x float> %a0) {
 ; SSE41-NEXT:    movaps %xmm1, %xmm2
 ; SSE41-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm1[1]
 ; SSE41-NEXT:    addss %xmm2, %xmm0
-; SSE41-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; SSE41-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; SSE41-NEXT:    addss %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
@@ -797,7 +797,7 @@ define float @test_v8f32_undef(<8 x float> %a0) {
 ; AVX-NEXT:    vaddss {{.*}}(%rip), %xmm1, %xmm1
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
 ; AVX-NEXT:    vaddss %xmm2, %xmm1, %xmm1
-; AVX-NEXT:    vpermilps {{.*#+}} xmm2 = xmm0[3,1,2,3]
+; AVX-NEXT:    vpermilps {{.*#+}} xmm2 = xmm0[3,3,3,3]
 ; AVX-NEXT:    vaddss %xmm2, %xmm1, %xmm1
 ; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; AVX-NEXT:    vaddss %xmm0, %xmm1, %xmm1
@@ -805,7 +805,7 @@ define float @test_v8f32_undef(<8 x float> %a0) {
 ; AVX-NEXT:    vaddss %xmm2, %xmm1, %xmm1
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
 ; AVX-NEXT:    vaddss %xmm2, %xmm1, %xmm1
-; AVX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX-NEXT:    vaddss %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    vzeroupper
 ; AVX-NEXT:    retq
@@ -816,7 +816,7 @@ define float @test_v8f32_undef(<8 x float> %a0) {
 ; AVX512-NEXT:    vaddss {{.*}}(%rip), %xmm1, %xmm1
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
 ; AVX512-NEXT:    vaddss %xmm2, %xmm1, %xmm1
-; AVX512-NEXT:    vpermilps {{.*#+}} xmm2 = xmm0[3,1,2,3]
+; AVX512-NEXT:    vpermilps {{.*#+}} xmm2 = xmm0[3,3,3,3]
 ; AVX512-NEXT:    vaddss %xmm2, %xmm1, %xmm1
 ; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; AVX512-NEXT:    vaddss %xmm0, %xmm1, %xmm1
@@ -824,7 +824,7 @@ define float @test_v8f32_undef(<8 x float> %a0) {
 ; AVX512-NEXT:    vaddss %xmm2, %xmm1, %xmm1
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
 ; AVX512-NEXT:    vaddss %xmm2, %xmm1, %xmm1
-; AVX512-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX512-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX512-NEXT:    vaddss %xmm0, %xmm1, %xmm0
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
@@ -836,39 +836,39 @@ define float @test_v16f32_undef(<16 x float> %a0) {
 ; SSE2-LABEL: test_v16f32_undef:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movaps %xmm0, %xmm4
-; SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[1,1],xmm0[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[1,1],xmm0[1,1]
 ; SSE2-NEXT:    addss {{.*}}(%rip), %xmm4
 ; SSE2-NEXT:    movaps %xmm0, %xmm5
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm5 = xmm5[1],xmm0[1]
 ; SSE2-NEXT:    addss %xmm4, %xmm5
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE2-NEXT:    addss %xmm5, %xmm0
 ; SSE2-NEXT:    addss %xmm1, %xmm0
 ; SSE2-NEXT:    movaps %xmm1, %xmm4
-; SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[1,1],xmm1[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[1,1],xmm1[1,1]
 ; SSE2-NEXT:    addss %xmm4, %xmm0
 ; SSE2-NEXT:    movaps %xmm1, %xmm4
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm1[1]
 ; SSE2-NEXT:    addss %xmm4, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; SSE2-NEXT:    addss %xmm1, %xmm0
 ; SSE2-NEXT:    addss %xmm2, %xmm0
 ; SSE2-NEXT:    movaps %xmm2, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm2[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm2[1,1]
 ; SSE2-NEXT:    addss %xmm1, %xmm0
 ; SSE2-NEXT:    movaps %xmm2, %xmm1
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm2[1]
 ; SSE2-NEXT:    addss %xmm1, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,3,3,3]
 ; SSE2-NEXT:    addss %xmm2, %xmm0
 ; SSE2-NEXT:    addss %xmm3, %xmm0
 ; SSE2-NEXT:    movaps %xmm3, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm3[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm3[1,1]
 ; SSE2-NEXT:    addss %xmm1, %xmm0
 ; SSE2-NEXT:    movaps %xmm3, %xmm1
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm3[1]
 ; SSE2-NEXT:    addss %xmm1, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,3,3,3]
 ; SSE2-NEXT:    addss %xmm3, %xmm0
 ; SSE2-NEXT:    retq
 ;
@@ -879,7 +879,7 @@ define float @test_v16f32_undef(<16 x float> %a0) {
 ; SSE41-NEXT:    movaps %xmm0, %xmm5
 ; SSE41-NEXT:    unpckhpd {{.*#+}} xmm5 = xmm5[1],xmm0[1]
 ; SSE41-NEXT:    addss %xmm4, %xmm5
-; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE41-NEXT:    addss %xmm5, %xmm0
 ; SSE41-NEXT:    addss %xmm1, %xmm0
 ; SSE41-NEXT:    movshdup {{.*#+}} xmm4 = xmm1[1,1,3,3]
@@ -887,7 +887,7 @@ define float @test_v16f32_undef(<16 x float> %a0) {
 ; SSE41-NEXT:    movaps %xmm1, %xmm4
 ; SSE41-NEXT:    unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm1[1]
 ; SSE41-NEXT:    addss %xmm4, %xmm0
-; SSE41-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; SSE41-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; SSE41-NEXT:    addss %xmm1, %xmm0
 ; SSE41-NEXT:    addss %xmm2, %xmm0
 ; SSE41-NEXT:    movshdup {{.*#+}} xmm1 = xmm2[1,1,3,3]
@@ -895,7 +895,7 @@ define float @test_v16f32_undef(<16 x float> %a0) {
 ; SSE41-NEXT:    movaps %xmm2, %xmm1
 ; SSE41-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm2[1]
 ; SSE41-NEXT:    addss %xmm1, %xmm0
-; SSE41-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,1,2,3]
+; SSE41-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,3,3,3]
 ; SSE41-NEXT:    addss %xmm2, %xmm0
 ; SSE41-NEXT:    addss %xmm3, %xmm0
 ; SSE41-NEXT:    movshdup {{.*#+}} xmm1 = xmm3[1,1,3,3]
@@ -903,7 +903,7 @@ define float @test_v16f32_undef(<16 x float> %a0) {
 ; SSE41-NEXT:    movaps %xmm3, %xmm1
 ; SSE41-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm3[1]
 ; SSE41-NEXT:    addss %xmm1, %xmm0
-; SSE41-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,1,2,3]
+; SSE41-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,3,3,3]
 ; SSE41-NEXT:    addss %xmm3, %xmm0
 ; SSE41-NEXT:    retq
 ;
@@ -913,7 +913,7 @@ define float @test_v16f32_undef(<16 x float> %a0) {
 ; AVX-NEXT:    vaddss {{.*}}(%rip), %xmm2, %xmm2
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm3 = xmm0[1,0]
 ; AVX-NEXT:    vaddss %xmm3, %xmm2, %xmm2
-; AVX-NEXT:    vpermilps {{.*#+}} xmm3 = xmm0[3,1,2,3]
+; AVX-NEXT:    vpermilps {{.*#+}} xmm3 = xmm0[3,3,3,3]
 ; AVX-NEXT:    vaddss %xmm3, %xmm2, %xmm2
 ; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; AVX-NEXT:    vaddss %xmm0, %xmm2, %xmm2
@@ -921,14 +921,14 @@ define float @test_v16f32_undef(<16 x float> %a0) {
 ; AVX-NEXT:    vaddss %xmm3, %xmm2, %xmm2
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm3 = xmm0[1,0]
 ; AVX-NEXT:    vaddss %xmm3, %xmm2, %xmm2
-; AVX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX-NEXT:    vaddss %xmm0, %xmm2, %xmm0
 ; AVX-NEXT:    vaddss %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovshdup {{.*#+}} xmm2 = xmm1[1,1,3,3]
 ; AVX-NEXT:    vaddss %xmm2, %xmm0, %xmm0
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm1[1,0]
 ; AVX-NEXT:    vaddss %xmm2, %xmm0, %xmm0
-; AVX-NEXT:    vpermilps {{.*#+}} xmm2 = xmm1[3,1,2,3]
+; AVX-NEXT:    vpermilps {{.*#+}} xmm2 = xmm1[3,3,3,3]
 ; AVX-NEXT:    vaddss %xmm2, %xmm0, %xmm0
 ; AVX-NEXT:    vextractf128 $1, %ymm1, %xmm1
 ; AVX-NEXT:    vaddss %xmm1, %xmm0, %xmm0
@@ -936,7 +936,7 @@ define float @test_v16f32_undef(<16 x float> %a0) {
 ; AVX-NEXT:    vaddss %xmm2, %xmm0, %xmm0
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm1[1,0]
 ; AVX-NEXT:    vaddss %xmm2, %xmm0, %xmm0
-; AVX-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; AVX-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; AVX-NEXT:    vaddss %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vzeroupper
 ; AVX-NEXT:    retq
@@ -947,7 +947,7 @@ define float @test_v16f32_undef(<16 x float> %a0) {
 ; AVX512-NEXT:    vaddss {{.*}}(%rip), %xmm1, %xmm1
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
 ; AVX512-NEXT:    vaddss %xmm2, %xmm1, %xmm1
-; AVX512-NEXT:    vpermilps {{.*#+}} xmm2 = xmm0[3,1,2,3]
+; AVX512-NEXT:    vpermilps {{.*#+}} xmm2 = xmm0[3,3,3,3]
 ; AVX512-NEXT:    vaddss %xmm2, %xmm1, %xmm1
 ; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm2
 ; AVX512-NEXT:    vaddss %xmm2, %xmm1, %xmm1
@@ -955,7 +955,7 @@ define float @test_v16f32_undef(<16 x float> %a0) {
 ; AVX512-NEXT:    vaddss %xmm3, %xmm1, %xmm1
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm3 = xmm2[1,0]
 ; AVX512-NEXT:    vaddss %xmm3, %xmm1, %xmm1
-; AVX512-NEXT:    vpermilps {{.*#+}} xmm2 = xmm2[3,1,2,3]
+; AVX512-NEXT:    vpermilps {{.*#+}} xmm2 = xmm2[3,3,3,3]
 ; AVX512-NEXT:    vaddss %xmm2, %xmm1, %xmm1
 ; AVX512-NEXT:    vextractf32x4 $2, %zmm0, %xmm2
 ; AVX512-NEXT:    vaddss %xmm2, %xmm1, %xmm1
@@ -963,7 +963,7 @@ define float @test_v16f32_undef(<16 x float> %a0) {
 ; AVX512-NEXT:    vaddss %xmm3, %xmm1, %xmm1
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm3 = xmm2[1,0]
 ; AVX512-NEXT:    vaddss %xmm3, %xmm1, %xmm1
-; AVX512-NEXT:    vpermilps {{.*#+}} xmm2 = xmm2[3,1,2,3]
+; AVX512-NEXT:    vpermilps {{.*#+}} xmm2 = xmm2[3,3,3,3]
 ; AVX512-NEXT:    vaddss %xmm2, %xmm1, %xmm1
 ; AVX512-NEXT:    vextractf32x4 $3, %zmm0, %xmm0
 ; AVX512-NEXT:    vaddss %xmm0, %xmm1, %xmm1
@@ -971,7 +971,7 @@ define float @test_v16f32_undef(<16 x float> %a0) {
 ; AVX512-NEXT:    vaddss %xmm2, %xmm1, %xmm1
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
 ; AVX512-NEXT:    vaddss %xmm2, %xmm1, %xmm1
-; AVX512-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX512-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX512-NEXT:    vaddss %xmm0, %xmm1, %xmm0
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/vector-reduce-fmax-nnan.ll b/llvm/test/CodeGen/X86/vector-reduce-fmax-nnan.ll
index 6a6333bcdda84..e2025be011343 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-fmax-nnan.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-fmax-nnan.ll
@@ -14,7 +14,7 @@ define float @test_v2f32(<2 x float> %a0) {
 ; SSE2-LABEL: test_v2f32:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movaps %xmm0, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[1,1]
 ; SSE2-NEXT:    maxss %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
@@ -46,7 +46,7 @@ define float @test_v4f32(<4 x float> %a0) {
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
 ; SSE2-NEXT:    maxps %xmm1, %xmm0
 ; SSE2-NEXT:    movaps %xmm0, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[1,1]
 ; SSE2-NEXT:    maxss %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
@@ -86,7 +86,7 @@ define float @test_v8f32(<8 x float> %a0) {
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
 ; SSE2-NEXT:    maxps %xmm1, %xmm0
 ; SSE2-NEXT:    movaps %xmm0, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[1,1]
 ; SSE2-NEXT:    maxss %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
@@ -135,7 +135,7 @@ define float @test_v16f32(<16 x float> %a0) {
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
 ; SSE2-NEXT:    maxps %xmm1, %xmm0
 ; SSE2-NEXT:    movaps %xmm0, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[1,1]
 ; SSE2-NEXT:    maxss %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/vector-reduce-fmax.ll b/llvm/test/CodeGen/X86/vector-reduce-fmax.ll
index f63d5dcdc0905..d3b17d25ef096 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-fmax.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-fmax.ll
@@ -14,7 +14,7 @@ define float @test_v2f32(<2 x float> %a0) {
 ; SSE2-LABEL: test_v2f32:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movaps %xmm0, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[1,1]
 ; SSE2-NEXT:    maxss %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
@@ -46,7 +46,7 @@ define float @test_v4f32(<4 x float> %a0) {
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
 ; SSE2-NEXT:    maxps %xmm1, %xmm0
 ; SSE2-NEXT:    movaps %xmm0, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[1,1]
 ; SSE2-NEXT:    maxss %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
@@ -86,7 +86,7 @@ define float @test_v8f32(<8 x float> %a0) {
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
 ; SSE2-NEXT:    maxps %xmm1, %xmm0
 ; SSE2-NEXT:    movaps %xmm0, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[1,1]
 ; SSE2-NEXT:    maxss %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
@@ -135,7 +135,7 @@ define float @test_v16f32(<16 x float> %a0) {
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
 ; SSE2-NEXT:    maxps %xmm1, %xmm0
 ; SSE2-NEXT:    movaps %xmm0, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[1,1]
 ; SSE2-NEXT:    maxss %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/vector-reduce-fmin-nnan.ll b/llvm/test/CodeGen/X86/vector-reduce-fmin-nnan.ll
index d3a799f18a5ce..f25852f0c6a85 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-fmin-nnan.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-fmin-nnan.ll
@@ -14,7 +14,7 @@ define float @test_v2f32(<2 x float> %a0) {
 ; SSE2-LABEL: test_v2f32:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movaps %xmm0, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[1,1]
 ; SSE2-NEXT:    minss %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
@@ -46,7 +46,7 @@ define float @test_v4f32(<4 x float> %a0) {
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
 ; SSE2-NEXT:    minps %xmm1, %xmm0
 ; SSE2-NEXT:    movaps %xmm0, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[1,1]
 ; SSE2-NEXT:    minss %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
@@ -86,7 +86,7 @@ define float @test_v8f32(<8 x float> %a0) {
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
 ; SSE2-NEXT:    minps %xmm1, %xmm0
 ; SSE2-NEXT:    movaps %xmm0, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[1,1]
 ; SSE2-NEXT:    minss %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
@@ -135,7 +135,7 @@ define float @test_v16f32(<16 x float> %a0) {
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
 ; SSE2-NEXT:    minps %xmm1, %xmm0
 ; SSE2-NEXT:    movaps %xmm0, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[1,1]
 ; SSE2-NEXT:    minss %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/vector-reduce-fmin.ll b/llvm/test/CodeGen/X86/vector-reduce-fmin.ll
index 74e5a722abda9..d6c681f507522 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-fmin.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-fmin.ll
@@ -14,7 +14,7 @@ define float @test_v2f32(<2 x float> %a0) {
 ; SSE2-LABEL: test_v2f32:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movaps %xmm0, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[1,1]
 ; SSE2-NEXT:    minss %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
@@ -46,7 +46,7 @@ define float @test_v4f32(<4 x float> %a0) {
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
 ; SSE2-NEXT:    minps %xmm1, %xmm0
 ; SSE2-NEXT:    movaps %xmm0, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[1,1]
 ; SSE2-NEXT:    minss %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
@@ -86,7 +86,7 @@ define float @test_v8f32(<8 x float> %a0) {
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
 ; SSE2-NEXT:    minps %xmm1, %xmm0
 ; SSE2-NEXT:    movaps %xmm0, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[1,1]
 ; SSE2-NEXT:    minss %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
@@ -135,7 +135,7 @@ define float @test_v16f32(<16 x float> %a0) {
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
 ; SSE2-NEXT:    minps %xmm1, %xmm0
 ; SSE2-NEXT:    movaps %xmm0, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[1,1]
 ; SSE2-NEXT:    minss %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/vector-reduce-fmul-fast.ll b/llvm/test/CodeGen/X86/vector-reduce-fmul-fast.ll
index f77ef5d7c2ed3..e99946edeee8a 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-fmul-fast.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-fmul-fast.ll
@@ -14,7 +14,7 @@ define float @test_v2f32(float %a0, <2 x float> %a1) {
 ; SSE2-LABEL: test_v2f32:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movaps %xmm1, %xmm2
-; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1],xmm1[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1],xmm1[1,1]
 ; SSE2-NEXT:    mulss %xmm1, %xmm2
 ; SSE2-NEXT:    mulss %xmm2, %xmm0
 ; SSE2-NEXT:    retq
@@ -50,7 +50,7 @@ define float @test_v4f32(float %a0, <4 x float> %a1) {
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm1[1]
 ; SSE2-NEXT:    mulps %xmm1, %xmm2
 ; SSE2-NEXT:    movaps %xmm2, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm2[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm2[1,1]
 ; SSE2-NEXT:    mulss %xmm2, %xmm1
 ; SSE2-NEXT:    mulss %xmm1, %xmm0
 ; SSE2-NEXT:    retq
@@ -94,7 +94,7 @@ define float @test_v8f32(float %a0, <8 x float> %a1) {
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm1[1]
 ; SSE2-NEXT:    mulps %xmm1, %xmm2
 ; SSE2-NEXT:    movaps %xmm2, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm2[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm2[1,1]
 ; SSE2-NEXT:    mulss %xmm2, %xmm1
 ; SSE2-NEXT:    mulss %xmm1, %xmm0
 ; SSE2-NEXT:    retq
@@ -147,7 +147,7 @@ define float @test_v16f32(float %a0, <16 x float> %a1) {
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm1[1]
 ; SSE2-NEXT:    mulps %xmm1, %xmm2
 ; SSE2-NEXT:    movaps %xmm2, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm2[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm2[1,1]
 ; SSE2-NEXT:    mulss %xmm2, %xmm1
 ; SSE2-NEXT:    mulss %xmm1, %xmm0
 ; SSE2-NEXT:    retq
@@ -203,7 +203,7 @@ define float @test_v2f32_zero(<2 x float> %a0) {
 ; SSE2-LABEL: test_v2f32_zero:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movaps %xmm0, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[1,1]
 ; SSE2-NEXT:    mulss %xmm0, %xmm1
 ; SSE2-NEXT:    movaps %xmm1, %xmm0
 ; SSE2-NEXT:    retq
@@ -236,7 +236,7 @@ define float @test_v4f32_zero(<4 x float> %a0) {
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
 ; SSE2-NEXT:    mulps %xmm0, %xmm1
 ; SSE2-NEXT:    movaps %xmm1, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[1,1]
 ; SSE2-NEXT:    mulss %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
@@ -277,7 +277,7 @@ define float @test_v8f32_zero(<8 x float> %a0) {
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
 ; SSE2-NEXT:    mulps %xmm0, %xmm1
 ; SSE2-NEXT:    movaps %xmm1, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[1,1]
 ; SSE2-NEXT:    mulss %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
@@ -327,7 +327,7 @@ define float @test_v16f32_zero(<16 x float> %a0) {
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
 ; SSE2-NEXT:    mulps %xmm0, %xmm1
 ; SSE2-NEXT:    movaps %xmm1, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[1,1]
 ; SSE2-NEXT:    mulss %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
@@ -380,7 +380,7 @@ define float @test_v2f32_undef(<2 x float> %a0) {
 ; SSE2-LABEL: test_v2f32_undef:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movaps %xmm0, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[1,1]
 ; SSE2-NEXT:    mulss %xmm0, %xmm1
 ; SSE2-NEXT:    movaps %xmm1, %xmm0
 ; SSE2-NEXT:    retq
@@ -413,7 +413,7 @@ define float @test_v4f32_undef(<4 x float> %a0) {
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
 ; SSE2-NEXT:    mulps %xmm0, %xmm1
 ; SSE2-NEXT:    movaps %xmm1, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[1,1]
 ; SSE2-NEXT:    mulss %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
@@ -454,7 +454,7 @@ define float @test_v8f32_undef(<8 x float> %a0) {
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
 ; SSE2-NEXT:    mulps %xmm0, %xmm1
 ; SSE2-NEXT:    movaps %xmm1, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[1,1]
 ; SSE2-NEXT:    mulss %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
@@ -504,7 +504,7 @@ define float @test_v16f32_undef(<16 x float> %a0) {
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
 ; SSE2-NEXT:    mulps %xmm0, %xmm1
 ; SSE2-NEXT:    movaps %xmm1, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[1,1]
 ; SSE2-NEXT:    mulss %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/vector-reduce-fmul.ll b/llvm/test/CodeGen/X86/vector-reduce-fmul.ll
index 6d3bf927ce8b1..7a1c044a0042d 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-fmul.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-fmul.ll
@@ -14,7 +14,7 @@ define float @test_v2f32(float %a0, <2 x float> %a1) {
 ; SSE2-LABEL: test_v2f32:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    mulss %xmm1, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,1,1]
 ; SSE2-NEXT:    mulss %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
@@ -47,12 +47,12 @@ define float @test_v4f32(float %a0, <4 x float> %a1) {
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    mulss %xmm1, %xmm0
 ; SSE2-NEXT:    movaps %xmm1, %xmm2
-; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1],xmm1[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1],xmm1[1,1]
 ; SSE2-NEXT:    mulss %xmm2, %xmm0
 ; SSE2-NEXT:    movaps %xmm1, %xmm2
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm1[1]
 ; SSE2-NEXT:    mulss %xmm2, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; SSE2-NEXT:    mulss %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
@@ -64,7 +64,7 @@ define float @test_v4f32(float %a0, <4 x float> %a1) {
 ; SSE41-NEXT:    movaps %xmm1, %xmm2
 ; SSE41-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm1[1]
 ; SSE41-NEXT:    mulss %xmm2, %xmm0
-; SSE41-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; SSE41-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; SSE41-NEXT:    mulss %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
@@ -75,7 +75,7 @@ define float @test_v4f32(float %a0, <4 x float> %a1) {
 ; AVX-NEXT:    vmulss %xmm2, %xmm0, %xmm0
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm1[1,0]
 ; AVX-NEXT:    vmulss %xmm2, %xmm0, %xmm0
-; AVX-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; AVX-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; AVX-NEXT:    vmulss %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
@@ -86,7 +86,7 @@ define float @test_v4f32(float %a0, <4 x float> %a1) {
 ; AVX512-NEXT:    vmulss %xmm2, %xmm0, %xmm0
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm1[1,0]
 ; AVX512-NEXT:    vmulss %xmm2, %xmm0, %xmm0
-; AVX512-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; AVX512-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; AVX512-NEXT:    vmulss %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    retq
   %1 = call float @llvm.experimental.vector.reduce.v2.fmul.f32.v4f32(float %a0, <4 x float> %a1)
@@ -98,21 +98,21 @@ define float @test_v8f32(float %a0, <8 x float> %a1) {
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    mulss %xmm1, %xmm0
 ; SSE2-NEXT:    movaps %xmm1, %xmm3
-; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm1[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm1[1,1]
 ; SSE2-NEXT:    mulss %xmm3, %xmm0
 ; SSE2-NEXT:    movaps %xmm1, %xmm3
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm3 = xmm3[1],xmm1[1]
 ; SSE2-NEXT:    mulss %xmm3, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; SSE2-NEXT:    mulss %xmm1, %xmm0
 ; SSE2-NEXT:    mulss %xmm2, %xmm0
 ; SSE2-NEXT:    movaps %xmm2, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm2[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm2[1,1]
 ; SSE2-NEXT:    mulss %xmm1, %xmm0
 ; SSE2-NEXT:    movaps %xmm2, %xmm1
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm2[1]
 ; SSE2-NEXT:    mulss %xmm1, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,3,3,3]
 ; SSE2-NEXT:    mulss %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
@@ -124,7 +124,7 @@ define float @test_v8f32(float %a0, <8 x float> %a1) {
 ; SSE41-NEXT:    movaps %xmm1, %xmm3
 ; SSE41-NEXT:    unpckhpd {{.*#+}} xmm3 = xmm3[1],xmm1[1]
 ; SSE41-NEXT:    mulss %xmm3, %xmm0
-; SSE41-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; SSE41-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; SSE41-NEXT:    mulss %xmm1, %xmm0
 ; SSE41-NEXT:    mulss %xmm2, %xmm0
 ; SSE41-NEXT:    movshdup {{.*#+}} xmm1 = xmm2[1,1,3,3]
@@ -132,7 +132,7 @@ define float @test_v8f32(float %a0, <8 x float> %a1) {
 ; SSE41-NEXT:    movaps %xmm2, %xmm1
 ; SSE41-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm2[1]
 ; SSE41-NEXT:    mulss %xmm1, %xmm0
-; SSE41-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,1,2,3]
+; SSE41-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,3,3,3]
 ; SSE41-NEXT:    mulss %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
@@ -143,7 +143,7 @@ define float @test_v8f32(float %a0, <8 x float> %a1) {
 ; AVX-NEXT:    vmulss %xmm2, %xmm0, %xmm0
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm1[1,0]
 ; AVX-NEXT:    vmulss %xmm2, %xmm0, %xmm0
-; AVX-NEXT:    vpermilps {{.*#+}} xmm2 = xmm1[3,1,2,3]
+; AVX-NEXT:    vpermilps {{.*#+}} xmm2 = xmm1[3,3,3,3]
 ; AVX-NEXT:    vmulss %xmm2, %xmm0, %xmm0
 ; AVX-NEXT:    vextractf128 $1, %ymm1, %xmm1
 ; AVX-NEXT:    vmulss %xmm1, %xmm0, %xmm0
@@ -151,7 +151,7 @@ define float @test_v8f32(float %a0, <8 x float> %a1) {
 ; AVX-NEXT:    vmulss %xmm2, %xmm0, %xmm0
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm1[1,0]
 ; AVX-NEXT:    vmulss %xmm2, %xmm0, %xmm0
-; AVX-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; AVX-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; AVX-NEXT:    vmulss %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vzeroupper
 ; AVX-NEXT:    retq
@@ -163,7 +163,7 @@ define float @test_v8f32(float %a0, <8 x float> %a1) {
 ; AVX512-NEXT:    vmulss %xmm2, %xmm0, %xmm0
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm1[1,0]
 ; AVX512-NEXT:    vmulss %xmm2, %xmm0, %xmm0
-; AVX512-NEXT:    vpermilps {{.*#+}} xmm2 = xmm1[3,1,2,3]
+; AVX512-NEXT:    vpermilps {{.*#+}} xmm2 = xmm1[3,3,3,3]
 ; AVX512-NEXT:    vmulss %xmm2, %xmm0, %xmm0
 ; AVX512-NEXT:    vextractf128 $1, %ymm1, %xmm1
 ; AVX512-NEXT:    vmulss %xmm1, %xmm0, %xmm0
@@ -171,7 +171,7 @@ define float @test_v8f32(float %a0, <8 x float> %a1) {
 ; AVX512-NEXT:    vmulss %xmm2, %xmm0, %xmm0
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm1[1,0]
 ; AVX512-NEXT:    vmulss %xmm2, %xmm0, %xmm0
-; AVX512-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; AVX512-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; AVX512-NEXT:    vmulss %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
@@ -184,39 +184,39 @@ define float @test_v16f32(float %a0, <16 x float> %a1) {
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    mulss %xmm1, %xmm0
 ; SSE2-NEXT:    movaps %xmm1, %xmm5
-; SSE2-NEXT:    shufps {{.*#+}} xmm5 = xmm5[1,1],xmm1[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm5 = xmm5[1,1],xmm1[1,1]
 ; SSE2-NEXT:    mulss %xmm5, %xmm0
 ; SSE2-NEXT:    movaps %xmm1, %xmm5
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm5 = xmm5[1],xmm1[1]
 ; SSE2-NEXT:    mulss %xmm5, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; SSE2-NEXT:    mulss %xmm1, %xmm0
 ; SSE2-NEXT:    mulss %xmm2, %xmm0
 ; SSE2-NEXT:    movaps %xmm2, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm2[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm2[1,1]
 ; SSE2-NEXT:    mulss %xmm1, %xmm0
 ; SSE2-NEXT:    movaps %xmm2, %xmm1
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm2[1]
 ; SSE2-NEXT:    mulss %xmm1, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,3,3,3]
 ; SSE2-NEXT:    mulss %xmm2, %xmm0
 ; SSE2-NEXT:    mulss %xmm3, %xmm0
 ; SSE2-NEXT:    movaps %xmm3, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm3[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm3[1,1]
 ; SSE2-NEXT:    mulss %xmm1, %xmm0
 ; SSE2-NEXT:    movaps %xmm3, %xmm1
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm3[1]
 ; SSE2-NEXT:    mulss %xmm1, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,3,3,3]
 ; SSE2-NEXT:    mulss %xmm3, %xmm0
 ; SSE2-NEXT:    mulss %xmm4, %xmm0
 ; SSE2-NEXT:    movaps %xmm4, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm4[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm4[1,1]
 ; SSE2-NEXT:    mulss %xmm1, %xmm0
 ; SSE2-NEXT:    movaps %xmm4, %xmm1
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm4[1]
 ; SSE2-NEXT:    mulss %xmm1, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,3,3,3]
 ; SSE2-NEXT:    mulss %xmm4, %xmm0
 ; SSE2-NEXT:    retq
 ;
@@ -228,7 +228,7 @@ define float @test_v16f32(float %a0, <16 x float> %a1) {
 ; SSE41-NEXT:    movaps %xmm1, %xmm5
 ; SSE41-NEXT:    unpckhpd {{.*#+}} xmm5 = xmm5[1],xmm1[1]
 ; SSE41-NEXT:    mulss %xmm5, %xmm0
-; SSE41-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; SSE41-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; SSE41-NEXT:    mulss %xmm1, %xmm0
 ; SSE41-NEXT:    mulss %xmm2, %xmm0
 ; SSE41-NEXT:    movshdup {{.*#+}} xmm1 = xmm2[1,1,3,3]
@@ -236,7 +236,7 @@ define float @test_v16f32(float %a0, <16 x float> %a1) {
 ; SSE41-NEXT:    movaps %xmm2, %xmm1
 ; SSE41-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm2[1]
 ; SSE41-NEXT:    mulss %xmm1, %xmm0
-; SSE41-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,1,2,3]
+; SSE41-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,3,3,3]
 ; SSE41-NEXT:    mulss %xmm2, %xmm0
 ; SSE41-NEXT:    mulss %xmm3, %xmm0
 ; SSE41-NEXT:    movshdup {{.*#+}} xmm1 = xmm3[1,1,3,3]
@@ -244,7 +244,7 @@ define float @test_v16f32(float %a0, <16 x float> %a1) {
 ; SSE41-NEXT:    movaps %xmm3, %xmm1
 ; SSE41-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm3[1]
 ; SSE41-NEXT:    mulss %xmm1, %xmm0
-; SSE41-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,1,2,3]
+; SSE41-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,3,3,3]
 ; SSE41-NEXT:    mulss %xmm3, %xmm0
 ; SSE41-NEXT:    mulss %xmm4, %xmm0
 ; SSE41-NEXT:    movshdup {{.*#+}} xmm1 = xmm4[1,1,3,3]
@@ -252,7 +252,7 @@ define float @test_v16f32(float %a0, <16 x float> %a1) {
 ; SSE41-NEXT:    movaps %xmm4, %xmm1
 ; SSE41-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm4[1]
 ; SSE41-NEXT:    mulss %xmm1, %xmm0
-; SSE41-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,1,2,3]
+; SSE41-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,3,3,3]
 ; SSE41-NEXT:    mulss %xmm4, %xmm0
 ; SSE41-NEXT:    retq
 ;
@@ -263,7 +263,7 @@ define float @test_v16f32(float %a0, <16 x float> %a1) {
 ; AVX-NEXT:    vmulss %xmm3, %xmm0, %xmm0
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm3 = xmm1[1,0]
 ; AVX-NEXT:    vmulss %xmm3, %xmm0, %xmm0
-; AVX-NEXT:    vpermilps {{.*#+}} xmm3 = xmm1[3,1,2,3]
+; AVX-NEXT:    vpermilps {{.*#+}} xmm3 = xmm1[3,3,3,3]
 ; AVX-NEXT:    vmulss %xmm3, %xmm0, %xmm0
 ; AVX-NEXT:    vextractf128 $1, %ymm1, %xmm1
 ; AVX-NEXT:    vmulss %xmm1, %xmm0, %xmm0
@@ -271,14 +271,14 @@ define float @test_v16f32(float %a0, <16 x float> %a1) {
 ; AVX-NEXT:    vmulss %xmm3, %xmm0, %xmm0
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm3 = xmm1[1,0]
 ; AVX-NEXT:    vmulss %xmm3, %xmm0, %xmm0
-; AVX-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; AVX-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; AVX-NEXT:    vmulss %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmulss %xmm2, %xmm0, %xmm0
 ; AVX-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm2[1,1,3,3]
 ; AVX-NEXT:    vmulss %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm2[1,0]
 ; AVX-NEXT:    vmulss %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpermilps {{.*#+}} xmm1 = xmm2[3,1,2,3]
+; AVX-NEXT:    vpermilps {{.*#+}} xmm1 = xmm2[3,3,3,3]
 ; AVX-NEXT:    vmulss %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vextractf128 $1, %ymm2, %xmm1
 ; AVX-NEXT:    vmulss %xmm1, %xmm0, %xmm0
@@ -286,7 +286,7 @@ define float @test_v16f32(float %a0, <16 x float> %a1) {
 ; AVX-NEXT:    vmulss %xmm2, %xmm0, %xmm0
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm1[1,0]
 ; AVX-NEXT:    vmulss %xmm2, %xmm0, %xmm0
-; AVX-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; AVX-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; AVX-NEXT:    vmulss %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vzeroupper
 ; AVX-NEXT:    retq
@@ -298,7 +298,7 @@ define float @test_v16f32(float %a0, <16 x float> %a1) {
 ; AVX512-NEXT:    vmulss %xmm2, %xmm0, %xmm0
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm1[1,0]
 ; AVX512-NEXT:    vmulss %xmm2, %xmm0, %xmm0
-; AVX512-NEXT:    vpermilps {{.*#+}} xmm2 = xmm1[3,1,2,3]
+; AVX512-NEXT:    vpermilps {{.*#+}} xmm2 = xmm1[3,3,3,3]
 ; AVX512-NEXT:    vmulss %xmm2, %xmm0, %xmm0
 ; AVX512-NEXT:    vextractf128 $1, %ymm1, %xmm2
 ; AVX512-NEXT:    vmulss %xmm2, %xmm0, %xmm0
@@ -306,7 +306,7 @@ define float @test_v16f32(float %a0, <16 x float> %a1) {
 ; AVX512-NEXT:    vmulss %xmm3, %xmm0, %xmm0
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm3 = xmm2[1,0]
 ; AVX512-NEXT:    vmulss %xmm3, %xmm0, %xmm0
-; AVX512-NEXT:    vpermilps {{.*#+}} xmm2 = xmm2[3,1,2,3]
+; AVX512-NEXT:    vpermilps {{.*#+}} xmm2 = xmm2[3,3,3,3]
 ; AVX512-NEXT:    vmulss %xmm2, %xmm0, %xmm0
 ; AVX512-NEXT:    vextractf32x4 $2, %zmm1, %xmm2
 ; AVX512-NEXT:    vmulss %xmm2, %xmm0, %xmm0
@@ -314,7 +314,7 @@ define float @test_v16f32(float %a0, <16 x float> %a1) {
 ; AVX512-NEXT:    vmulss %xmm3, %xmm0, %xmm0
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm3 = xmm2[1,0]
 ; AVX512-NEXT:    vmulss %xmm3, %xmm0, %xmm0
-; AVX512-NEXT:    vpermilps {{.*#+}} xmm2 = xmm2[3,1,2,3]
+; AVX512-NEXT:    vpermilps {{.*#+}} xmm2 = xmm2[3,3,3,3]
 ; AVX512-NEXT:    vmulss %xmm2, %xmm0, %xmm0
 ; AVX512-NEXT:    vextractf32x4 $3, %zmm1, %xmm1
 ; AVX512-NEXT:    vmulss %xmm1, %xmm0, %xmm0
@@ -322,7 +322,7 @@ define float @test_v16f32(float %a0, <16 x float> %a1) {
 ; AVX512-NEXT:    vmulss %xmm2, %xmm0, %xmm0
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm1[1,0]
 ; AVX512-NEXT:    vmulss %xmm2, %xmm0, %xmm0
-; AVX512-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; AVX512-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; AVX512-NEXT:    vmulss %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
@@ -338,7 +338,7 @@ define float @test_v2f32_one(<2 x float> %a0) {
 ; SSE2-LABEL: test_v2f32_one:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movaps %xmm0, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[1,1]
 ; SSE2-NEXT:    mulss %xmm0, %xmm1
 ; SSE2-NEXT:    movaps %xmm1, %xmm0
 ; SSE2-NEXT:    retq
@@ -368,12 +368,12 @@ define float @test_v4f32_one(<4 x float> %a0) {
 ; SSE2-LABEL: test_v4f32_one:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movaps %xmm0, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[1,1]
 ; SSE2-NEXT:    mulss %xmm0, %xmm1
 ; SSE2-NEXT:    movaps %xmm0, %xmm2
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm0[1]
 ; SSE2-NEXT:    mulss %xmm1, %xmm2
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE2-NEXT:    mulss %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
@@ -384,7 +384,7 @@ define float @test_v4f32_one(<4 x float> %a0) {
 ; SSE41-NEXT:    movaps %xmm0, %xmm2
 ; SSE41-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm0[1]
 ; SSE41-NEXT:    mulss %xmm1, %xmm2
-; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE41-NEXT:    mulss %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
@@ -394,7 +394,7 @@ define float @test_v4f32_one(<4 x float> %a0) {
 ; AVX-NEXT:    vmulss %xmm1, %xmm0, %xmm1
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
 ; AVX-NEXT:    vmulss %xmm2, %xmm1, %xmm1
-; AVX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX-NEXT:    vmulss %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    retq
 ;
@@ -404,7 +404,7 @@ define float @test_v4f32_one(<4 x float> %a0) {
 ; AVX512-NEXT:    vmulss %xmm1, %xmm0, %xmm1
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
 ; AVX512-NEXT:    vmulss %xmm2, %xmm1, %xmm1
-; AVX512-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX512-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX512-NEXT:    vmulss %xmm0, %xmm1, %xmm0
 ; AVX512-NEXT:    retq
   %1 = call float @llvm.experimental.vector.reduce.v2.fmul.f32.v4f32(float 1.0, <4 x float> %a0)
@@ -415,21 +415,21 @@ define float @test_v8f32_one(<8 x float> %a0) {
 ; SSE2-LABEL: test_v8f32_one:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movaps %xmm0, %xmm2
-; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1],xmm0[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1],xmm0[1,1]
 ; SSE2-NEXT:    mulss %xmm0, %xmm2
 ; SSE2-NEXT:    movaps %xmm0, %xmm3
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm3 = xmm3[1],xmm0[1]
 ; SSE2-NEXT:    mulss %xmm2, %xmm3
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE2-NEXT:    mulss %xmm3, %xmm0
 ; SSE2-NEXT:    mulss %xmm1, %xmm0
 ; SSE2-NEXT:    movaps %xmm1, %xmm2
-; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1],xmm1[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1],xmm1[1,1]
 ; SSE2-NEXT:    mulss %xmm2, %xmm0
 ; SSE2-NEXT:    movaps %xmm1, %xmm2
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm1[1]
 ; SSE2-NEXT:    mulss %xmm2, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; SSE2-NEXT:    mulss %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
@@ -440,7 +440,7 @@ define float @test_v8f32_one(<8 x float> %a0) {
 ; SSE41-NEXT:    movaps %xmm0, %xmm3
 ; SSE41-NEXT:    unpckhpd {{.*#+}} xmm3 = xmm3[1],xmm0[1]
 ; SSE41-NEXT:    mulss %xmm2, %xmm3
-; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE41-NEXT:    mulss %xmm3, %xmm0
 ; SSE41-NEXT:    mulss %xmm1, %xmm0
 ; SSE41-NEXT:    movshdup {{.*#+}} xmm2 = xmm1[1,1,3,3]
@@ -448,7 +448,7 @@ define float @test_v8f32_one(<8 x float> %a0) {
 ; SSE41-NEXT:    movaps %xmm1, %xmm2
 ; SSE41-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm1[1]
 ; SSE41-NEXT:    mulss %xmm2, %xmm0
-; SSE41-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; SSE41-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; SSE41-NEXT:    mulss %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
@@ -458,7 +458,7 @@ define float @test_v8f32_one(<8 x float> %a0) {
 ; AVX-NEXT:    vmulss %xmm1, %xmm0, %xmm1
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
 ; AVX-NEXT:    vmulss %xmm2, %xmm1, %xmm1
-; AVX-NEXT:    vpermilps {{.*#+}} xmm2 = xmm0[3,1,2,3]
+; AVX-NEXT:    vpermilps {{.*#+}} xmm2 = xmm0[3,3,3,3]
 ; AVX-NEXT:    vmulss %xmm2, %xmm1, %xmm1
 ; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; AVX-NEXT:    vmulss %xmm0, %xmm1, %xmm1
@@ -466,7 +466,7 @@ define float @test_v8f32_one(<8 x float> %a0) {
 ; AVX-NEXT:    vmulss %xmm2, %xmm1, %xmm1
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
 ; AVX-NEXT:    vmulss %xmm2, %xmm1, %xmm1
-; AVX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX-NEXT:    vmulss %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    vzeroupper
 ; AVX-NEXT:    retq
@@ -477,7 +477,7 @@ define float @test_v8f32_one(<8 x float> %a0) {
 ; AVX512-NEXT:    vmulss %xmm1, %xmm0, %xmm1
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
 ; AVX512-NEXT:    vmulss %xmm2, %xmm1, %xmm1
-; AVX512-NEXT:    vpermilps {{.*#+}} xmm2 = xmm0[3,1,2,3]
+; AVX512-NEXT:    vpermilps {{.*#+}} xmm2 = xmm0[3,3,3,3]
 ; AVX512-NEXT:    vmulss %xmm2, %xmm1, %xmm1
 ; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; AVX512-NEXT:    vmulss %xmm0, %xmm1, %xmm1
@@ -485,7 +485,7 @@ define float @test_v8f32_one(<8 x float> %a0) {
 ; AVX512-NEXT:    vmulss %xmm2, %xmm1, %xmm1
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
 ; AVX512-NEXT:    vmulss %xmm2, %xmm1, %xmm1
-; AVX512-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX512-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX512-NEXT:    vmulss %xmm0, %xmm1, %xmm0
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
@@ -497,39 +497,39 @@ define float @test_v16f32_one(<16 x float> %a0) {
 ; SSE2-LABEL: test_v16f32_one:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movaps %xmm0, %xmm4
-; SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[1,1],xmm0[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[1,1],xmm0[1,1]
 ; SSE2-NEXT:    mulss %xmm0, %xmm4
 ; SSE2-NEXT:    movaps %xmm0, %xmm5
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm5 = xmm5[1],xmm0[1]
 ; SSE2-NEXT:    mulss %xmm4, %xmm5
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE2-NEXT:    mulss %xmm5, %xmm0
 ; SSE2-NEXT:    mulss %xmm1, %xmm0
 ; SSE2-NEXT:    movaps %xmm1, %xmm4
-; SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[1,1],xmm1[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[1,1],xmm1[1,1]
 ; SSE2-NEXT:    mulss %xmm4, %xmm0
 ; SSE2-NEXT:    movaps %xmm1, %xmm4
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm1[1]
 ; SSE2-NEXT:    mulss %xmm4, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; SSE2-NEXT:    mulss %xmm1, %xmm0
 ; SSE2-NEXT:    mulss %xmm2, %xmm0
 ; SSE2-NEXT:    movaps %xmm2, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm2[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm2[1,1]
 ; SSE2-NEXT:    mulss %xmm1, %xmm0
 ; SSE2-NEXT:    movaps %xmm2, %xmm1
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm2[1]
 ; SSE2-NEXT:    mulss %xmm1, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,3,3,3]
 ; SSE2-NEXT:    mulss %xmm2, %xmm0
 ; SSE2-NEXT:    mulss %xmm3, %xmm0
 ; SSE2-NEXT:    movaps %xmm3, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm3[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm3[1,1]
 ; SSE2-NEXT:    mulss %xmm1, %xmm0
 ; SSE2-NEXT:    movaps %xmm3, %xmm1
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm3[1]
 ; SSE2-NEXT:    mulss %xmm1, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,3,3,3]
 ; SSE2-NEXT:    mulss %xmm3, %xmm0
 ; SSE2-NEXT:    retq
 ;
@@ -540,7 +540,7 @@ define float @test_v16f32_one(<16 x float> %a0) {
 ; SSE41-NEXT:    movaps %xmm0, %xmm5
 ; SSE41-NEXT:    unpckhpd {{.*#+}} xmm5 = xmm5[1],xmm0[1]
 ; SSE41-NEXT:    mulss %xmm4, %xmm5
-; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE41-NEXT:    mulss %xmm5, %xmm0
 ; SSE41-NEXT:    mulss %xmm1, %xmm0
 ; SSE41-NEXT:    movshdup {{.*#+}} xmm4 = xmm1[1,1,3,3]
@@ -548,7 +548,7 @@ define float @test_v16f32_one(<16 x float> %a0) {
 ; SSE41-NEXT:    movaps %xmm1, %xmm4
 ; SSE41-NEXT:    unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm1[1]
 ; SSE41-NEXT:    mulss %xmm4, %xmm0
-; SSE41-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; SSE41-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; SSE41-NEXT:    mulss %xmm1, %xmm0
 ; SSE41-NEXT:    mulss %xmm2, %xmm0
 ; SSE41-NEXT:    movshdup {{.*#+}} xmm1 = xmm2[1,1,3,3]
@@ -556,7 +556,7 @@ define float @test_v16f32_one(<16 x float> %a0) {
 ; SSE41-NEXT:    movaps %xmm2, %xmm1
 ; SSE41-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm2[1]
 ; SSE41-NEXT:    mulss %xmm1, %xmm0
-; SSE41-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,1,2,3]
+; SSE41-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,3,3,3]
 ; SSE41-NEXT:    mulss %xmm2, %xmm0
 ; SSE41-NEXT:    mulss %xmm3, %xmm0
 ; SSE41-NEXT:    movshdup {{.*#+}} xmm1 = xmm3[1,1,3,3]
@@ -564,7 +564,7 @@ define float @test_v16f32_one(<16 x float> %a0) {
 ; SSE41-NEXT:    movaps %xmm3, %xmm1
 ; SSE41-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm3[1]
 ; SSE41-NEXT:    mulss %xmm1, %xmm0
-; SSE41-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,1,2,3]
+; SSE41-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,3,3,3]
 ; SSE41-NEXT:    mulss %xmm3, %xmm0
 ; SSE41-NEXT:    retq
 ;
@@ -574,7 +574,7 @@ define float @test_v16f32_one(<16 x float> %a0) {
 ; AVX-NEXT:    vmulss %xmm2, %xmm0, %xmm2
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm3 = xmm0[1,0]
 ; AVX-NEXT:    vmulss %xmm3, %xmm2, %xmm2
-; AVX-NEXT:    vpermilps {{.*#+}} xmm3 = xmm0[3,1,2,3]
+; AVX-NEXT:    vpermilps {{.*#+}} xmm3 = xmm0[3,3,3,3]
 ; AVX-NEXT:    vmulss %xmm3, %xmm2, %xmm2
 ; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; AVX-NEXT:    vmulss %xmm0, %xmm2, %xmm2
@@ -582,14 +582,14 @@ define float @test_v16f32_one(<16 x float> %a0) {
 ; AVX-NEXT:    vmulss %xmm3, %xmm2, %xmm2
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm3 = xmm0[1,0]
 ; AVX-NEXT:    vmulss %xmm3, %xmm2, %xmm2
-; AVX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX-NEXT:    vmulss %xmm0, %xmm2, %xmm0
 ; AVX-NEXT:    vmulss %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovshdup {{.*#+}} xmm2 = xmm1[1,1,3,3]
 ; AVX-NEXT:    vmulss %xmm2, %xmm0, %xmm0
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm1[1,0]
 ; AVX-NEXT:    vmulss %xmm2, %xmm0, %xmm0
-; AVX-NEXT:    vpermilps {{.*#+}} xmm2 = xmm1[3,1,2,3]
+; AVX-NEXT:    vpermilps {{.*#+}} xmm2 = xmm1[3,3,3,3]
 ; AVX-NEXT:    vmulss %xmm2, %xmm0, %xmm0
 ; AVX-NEXT:    vextractf128 $1, %ymm1, %xmm1
 ; AVX-NEXT:    vmulss %xmm1, %xmm0, %xmm0
@@ -597,7 +597,7 @@ define float @test_v16f32_one(<16 x float> %a0) {
 ; AVX-NEXT:    vmulss %xmm2, %xmm0, %xmm0
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm1[1,0]
 ; AVX-NEXT:    vmulss %xmm2, %xmm0, %xmm0
-; AVX-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; AVX-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; AVX-NEXT:    vmulss %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vzeroupper
 ; AVX-NEXT:    retq
@@ -608,7 +608,7 @@ define float @test_v16f32_one(<16 x float> %a0) {
 ; AVX512-NEXT:    vmulss %xmm1, %xmm0, %xmm1
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
 ; AVX512-NEXT:    vmulss %xmm2, %xmm1, %xmm1
-; AVX512-NEXT:    vpermilps {{.*#+}} xmm2 = xmm0[3,1,2,3]
+; AVX512-NEXT:    vpermilps {{.*#+}} xmm2 = xmm0[3,3,3,3]
 ; AVX512-NEXT:    vmulss %xmm2, %xmm1, %xmm1
 ; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm2
 ; AVX512-NEXT:    vmulss %xmm2, %xmm1, %xmm1
@@ -616,7 +616,7 @@ define float @test_v16f32_one(<16 x float> %a0) {
 ; AVX512-NEXT:    vmulss %xmm3, %xmm1, %xmm1
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm3 = xmm2[1,0]
 ; AVX512-NEXT:    vmulss %xmm3, %xmm1, %xmm1
-; AVX512-NEXT:    vpermilps {{.*#+}} xmm2 = xmm2[3,1,2,3]
+; AVX512-NEXT:    vpermilps {{.*#+}} xmm2 = xmm2[3,3,3,3]
 ; AVX512-NEXT:    vmulss %xmm2, %xmm1, %xmm1
 ; AVX512-NEXT:    vextractf32x4 $2, %zmm0, %xmm2
 ; AVX512-NEXT:    vmulss %xmm2, %xmm1, %xmm1
@@ -624,7 +624,7 @@ define float @test_v16f32_one(<16 x float> %a0) {
 ; AVX512-NEXT:    vmulss %xmm3, %xmm1, %xmm1
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm3 = xmm2[1,0]
 ; AVX512-NEXT:    vmulss %xmm3, %xmm1, %xmm1
-; AVX512-NEXT:    vpermilps {{.*#+}} xmm2 = xmm2[3,1,2,3]
+; AVX512-NEXT:    vpermilps {{.*#+}} xmm2 = xmm2[3,3,3,3]
 ; AVX512-NEXT:    vmulss %xmm2, %xmm1, %xmm1
 ; AVX512-NEXT:    vextractf32x4 $3, %zmm0, %xmm0
 ; AVX512-NEXT:    vmulss %xmm0, %xmm1, %xmm1
@@ -632,7 +632,7 @@ define float @test_v16f32_one(<16 x float> %a0) {
 ; AVX512-NEXT:    vmulss %xmm2, %xmm1, %xmm1
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
 ; AVX512-NEXT:    vmulss %xmm2, %xmm1, %xmm1
-; AVX512-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX512-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX512-NEXT:    vmulss %xmm0, %xmm1, %xmm0
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
@@ -647,7 +647,7 @@ define float @test_v16f32_one(<16 x float> %a0) {
 define float @test_v2f32_undef(<2 x float> %a0) {
 ; SSE2-LABEL: test_v2f32_undef:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    mulss {{.*}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
@@ -676,12 +676,12 @@ define float @test_v4f32_undef(<4 x float> %a0) {
 ; SSE2-LABEL: test_v4f32_undef:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movaps %xmm0, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[1,1]
 ; SSE2-NEXT:    mulss {{.*}}(%rip), %xmm1
 ; SSE2-NEXT:    movaps %xmm0, %xmm2
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm0[1]
 ; SSE2-NEXT:    mulss %xmm1, %xmm2
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE2-NEXT:    mulss %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
@@ -692,7 +692,7 @@ define float @test_v4f32_undef(<4 x float> %a0) {
 ; SSE41-NEXT:    movaps %xmm0, %xmm2
 ; SSE41-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm0[1]
 ; SSE41-NEXT:    mulss %xmm1, %xmm2
-; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE41-NEXT:    mulss %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
@@ -702,7 +702,7 @@ define float @test_v4f32_undef(<4 x float> %a0) {
 ; AVX-NEXT:    vmulss {{.*}}(%rip), %xmm1, %xmm1
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
 ; AVX-NEXT:    vmulss %xmm2, %xmm1, %xmm1
-; AVX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX-NEXT:    vmulss %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    retq
 ;
@@ -712,7 +712,7 @@ define float @test_v4f32_undef(<4 x float> %a0) {
 ; AVX512-NEXT:    vmulss {{.*}}(%rip), %xmm1, %xmm1
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
 ; AVX512-NEXT:    vmulss %xmm2, %xmm1, %xmm1
-; AVX512-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX512-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX512-NEXT:    vmulss %xmm0, %xmm1, %xmm0
 ; AVX512-NEXT:    retq
   %1 = call float @llvm.experimental.vector.reduce.v2.fmul.f32.v4f32(float undef, <4 x float> %a0)
@@ -723,21 +723,21 @@ define float @test_v8f32_undef(<8 x float> %a0) {
 ; SSE2-LABEL: test_v8f32_undef:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movaps %xmm0, %xmm2
-; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1],xmm0[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1],xmm0[1,1]
 ; SSE2-NEXT:    mulss {{.*}}(%rip), %xmm2
 ; SSE2-NEXT:    movaps %xmm0, %xmm3
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm3 = xmm3[1],xmm0[1]
 ; SSE2-NEXT:    mulss %xmm2, %xmm3
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE2-NEXT:    mulss %xmm3, %xmm0
 ; SSE2-NEXT:    mulss %xmm1, %xmm0
 ; SSE2-NEXT:    movaps %xmm1, %xmm2
-; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1],xmm1[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1],xmm1[1,1]
 ; SSE2-NEXT:    mulss %xmm2, %xmm0
 ; SSE2-NEXT:    movaps %xmm1, %xmm2
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm1[1]
 ; SSE2-NEXT:    mulss %xmm2, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; SSE2-NEXT:    mulss %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
@@ -748,7 +748,7 @@ define float @test_v8f32_undef(<8 x float> %a0) {
 ; SSE41-NEXT:    movaps %xmm0, %xmm3
 ; SSE41-NEXT:    unpckhpd {{.*#+}} xmm3 = xmm3[1],xmm0[1]
 ; SSE41-NEXT:    mulss %xmm2, %xmm3
-; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE41-NEXT:    mulss %xmm3, %xmm0
 ; SSE41-NEXT:    mulss %xmm1, %xmm0
 ; SSE41-NEXT:    movshdup {{.*#+}} xmm2 = xmm1[1,1,3,3]
@@ -756,7 +756,7 @@ define float @test_v8f32_undef(<8 x float> %a0) {
 ; SSE41-NEXT:    movaps %xmm1, %xmm2
 ; SSE41-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm1[1]
 ; SSE41-NEXT:    mulss %xmm2, %xmm0
-; SSE41-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; SSE41-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; SSE41-NEXT:    mulss %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
@@ -766,7 +766,7 @@ define float @test_v8f32_undef(<8 x float> %a0) {
 ; AVX-NEXT:    vmulss {{.*}}(%rip), %xmm1, %xmm1
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
 ; AVX-NEXT:    vmulss %xmm2, %xmm1, %xmm1
-; AVX-NEXT:    vpermilps {{.*#+}} xmm2 = xmm0[3,1,2,3]
+; AVX-NEXT:    vpermilps {{.*#+}} xmm2 = xmm0[3,3,3,3]
 ; AVX-NEXT:    vmulss %xmm2, %xmm1, %xmm1
 ; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; AVX-NEXT:    vmulss %xmm0, %xmm1, %xmm1
@@ -774,7 +774,7 @@ define float @test_v8f32_undef(<8 x float> %a0) {
 ; AVX-NEXT:    vmulss %xmm2, %xmm1, %xmm1
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
 ; AVX-NEXT:    vmulss %xmm2, %xmm1, %xmm1
-; AVX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX-NEXT:    vmulss %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    vzeroupper
 ; AVX-NEXT:    retq
@@ -785,7 +785,7 @@ define float @test_v8f32_undef(<8 x float> %a0) {
 ; AVX512-NEXT:    vmulss {{.*}}(%rip), %xmm1, %xmm1
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
 ; AVX512-NEXT:    vmulss %xmm2, %xmm1, %xmm1
-; AVX512-NEXT:    vpermilps {{.*#+}} xmm2 = xmm0[3,1,2,3]
+; AVX512-NEXT:    vpermilps {{.*#+}} xmm2 = xmm0[3,3,3,3]
 ; AVX512-NEXT:    vmulss %xmm2, %xmm1, %xmm1
 ; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; AVX512-NEXT:    vmulss %xmm0, %xmm1, %xmm1
@@ -793,7 +793,7 @@ define float @test_v8f32_undef(<8 x float> %a0) {
 ; AVX512-NEXT:    vmulss %xmm2, %xmm1, %xmm1
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
 ; AVX512-NEXT:    vmulss %xmm2, %xmm1, %xmm1
-; AVX512-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX512-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX512-NEXT:    vmulss %xmm0, %xmm1, %xmm0
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
@@ -805,39 +805,39 @@ define float @test_v16f32_undef(<16 x float> %a0) {
 ; SSE2-LABEL: test_v16f32_undef:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movaps %xmm0, %xmm4
-; SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[1,1],xmm0[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[1,1],xmm0[1,1]
 ; SSE2-NEXT:    mulss {{.*}}(%rip), %xmm4
 ; SSE2-NEXT:    movaps %xmm0, %xmm5
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm5 = xmm5[1],xmm0[1]
 ; SSE2-NEXT:    mulss %xmm4, %xmm5
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE2-NEXT:    mulss %xmm5, %xmm0
 ; SSE2-NEXT:    mulss %xmm1, %xmm0
 ; SSE2-NEXT:    movaps %xmm1, %xmm4
-; SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[1,1],xmm1[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[1,1],xmm1[1,1]
 ; SSE2-NEXT:    mulss %xmm4, %xmm0
 ; SSE2-NEXT:    movaps %xmm1, %xmm4
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm1[1]
 ; SSE2-NEXT:    mulss %xmm4, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; SSE2-NEXT:    mulss %xmm1, %xmm0
 ; SSE2-NEXT:    mulss %xmm2, %xmm0
 ; SSE2-NEXT:    movaps %xmm2, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm2[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm2[1,1]
 ; SSE2-NEXT:    mulss %xmm1, %xmm0
 ; SSE2-NEXT:    movaps %xmm2, %xmm1
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm2[1]
 ; SSE2-NEXT:    mulss %xmm1, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,3,3,3]
 ; SSE2-NEXT:    mulss %xmm2, %xmm0
 ; SSE2-NEXT:    mulss %xmm3, %xmm0
 ; SSE2-NEXT:    movaps %xmm3, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm3[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm3[1,1]
 ; SSE2-NEXT:    mulss %xmm1, %xmm0
 ; SSE2-NEXT:    movaps %xmm3, %xmm1
 ; SSE2-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm3[1]
 ; SSE2-NEXT:    mulss %xmm1, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,1,2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,3,3,3]
 ; SSE2-NEXT:    mulss %xmm3, %xmm0
 ; SSE2-NEXT:    retq
 ;
@@ -848,7 +848,7 @@ define float @test_v16f32_undef(<16 x float> %a0) {
 ; SSE41-NEXT:    movaps %xmm0, %xmm5
 ; SSE41-NEXT:    unpckhpd {{.*#+}} xmm5 = xmm5[1],xmm0[1]
 ; SSE41-NEXT:    mulss %xmm4, %xmm5
-; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE41-NEXT:    mulss %xmm5, %xmm0
 ; SSE41-NEXT:    mulss %xmm1, %xmm0
 ; SSE41-NEXT:    movshdup {{.*#+}} xmm4 = xmm1[1,1,3,3]
@@ -856,7 +856,7 @@ define float @test_v16f32_undef(<16 x float> %a0) {
 ; SSE41-NEXT:    movaps %xmm1, %xmm4
 ; SSE41-NEXT:    unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm1[1]
 ; SSE41-NEXT:    mulss %xmm4, %xmm0
-; SSE41-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; SSE41-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; SSE41-NEXT:    mulss %xmm1, %xmm0
 ; SSE41-NEXT:    mulss %xmm2, %xmm0
 ; SSE41-NEXT:    movshdup {{.*#+}} xmm1 = xmm2[1,1,3,3]
@@ -864,7 +864,7 @@ define float @test_v16f32_undef(<16 x float> %a0) {
 ; SSE41-NEXT:    movaps %xmm2, %xmm1
 ; SSE41-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm2[1]
 ; SSE41-NEXT:    mulss %xmm1, %xmm0
-; SSE41-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,1,2,3]
+; SSE41-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,3,3,3]
 ; SSE41-NEXT:    mulss %xmm2, %xmm0
 ; SSE41-NEXT:    mulss %xmm3, %xmm0
 ; SSE41-NEXT:    movshdup {{.*#+}} xmm1 = xmm3[1,1,3,3]
@@ -872,7 +872,7 @@ define float @test_v16f32_undef(<16 x float> %a0) {
 ; SSE41-NEXT:    movaps %xmm3, %xmm1
 ; SSE41-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm3[1]
 ; SSE41-NEXT:    mulss %xmm1, %xmm0
-; SSE41-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,1,2,3]
+; SSE41-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,3,3,3]
 ; SSE41-NEXT:    mulss %xmm3, %xmm0
 ; SSE41-NEXT:    retq
 ;
@@ -882,7 +882,7 @@ define float @test_v16f32_undef(<16 x float> %a0) {
 ; AVX-NEXT:    vmulss {{.*}}(%rip), %xmm2, %xmm2
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm3 = xmm0[1,0]
 ; AVX-NEXT:    vmulss %xmm3, %xmm2, %xmm2
-; AVX-NEXT:    vpermilps {{.*#+}} xmm3 = xmm0[3,1,2,3]
+; AVX-NEXT:    vpermilps {{.*#+}} xmm3 = xmm0[3,3,3,3]
 ; AVX-NEXT:    vmulss %xmm3, %xmm2, %xmm2
 ; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; AVX-NEXT:    vmulss %xmm0, %xmm2, %xmm2
@@ -890,14 +890,14 @@ define float @test_v16f32_undef(<16 x float> %a0) {
 ; AVX-NEXT:    vmulss %xmm3, %xmm2, %xmm2
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm3 = xmm0[1,0]
 ; AVX-NEXT:    vmulss %xmm3, %xmm2, %xmm2
-; AVX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX-NEXT:    vmulss %xmm0, %xmm2, %xmm0
 ; AVX-NEXT:    vmulss %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovshdup {{.*#+}} xmm2 = xmm1[1,1,3,3]
 ; AVX-NEXT:    vmulss %xmm2, %xmm0, %xmm0
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm1[1,0]
 ; AVX-NEXT:    vmulss %xmm2, %xmm0, %xmm0
-; AVX-NEXT:    vpermilps {{.*#+}} xmm2 = xmm1[3,1,2,3]
+; AVX-NEXT:    vpermilps {{.*#+}} xmm2 = xmm1[3,3,3,3]
 ; AVX-NEXT:    vmulss %xmm2, %xmm0, %xmm0
 ; AVX-NEXT:    vextractf128 $1, %ymm1, %xmm1
 ; AVX-NEXT:    vmulss %xmm1, %xmm0, %xmm0
@@ -905,7 +905,7 @@ define float @test_v16f32_undef(<16 x float> %a0) {
 ; AVX-NEXT:    vmulss %xmm2, %xmm0, %xmm0
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm1[1,0]
 ; AVX-NEXT:    vmulss %xmm2, %xmm0, %xmm0
-; AVX-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; AVX-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; AVX-NEXT:    vmulss %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vzeroupper
 ; AVX-NEXT:    retq
@@ -916,7 +916,7 @@ define float @test_v16f32_undef(<16 x float> %a0) {
 ; AVX512-NEXT:    vmulss {{.*}}(%rip), %xmm1, %xmm1
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
 ; AVX512-NEXT:    vmulss %xmm2, %xmm1, %xmm1
-; AVX512-NEXT:    vpermilps {{.*#+}} xmm2 = xmm0[3,1,2,3]
+; AVX512-NEXT:    vpermilps {{.*#+}} xmm2 = xmm0[3,3,3,3]
 ; AVX512-NEXT:    vmulss %xmm2, %xmm1, %xmm1
 ; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm2
 ; AVX512-NEXT:    vmulss %xmm2, %xmm1, %xmm1
@@ -924,7 +924,7 @@ define float @test_v16f32_undef(<16 x float> %a0) {
 ; AVX512-NEXT:    vmulss %xmm3, %xmm1, %xmm1
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm3 = xmm2[1,0]
 ; AVX512-NEXT:    vmulss %xmm3, %xmm1, %xmm1
-; AVX512-NEXT:    vpermilps {{.*#+}} xmm2 = xmm2[3,1,2,3]
+; AVX512-NEXT:    vpermilps {{.*#+}} xmm2 = xmm2[3,3,3,3]
 ; AVX512-NEXT:    vmulss %xmm2, %xmm1, %xmm1
 ; AVX512-NEXT:    vextractf32x4 $2, %zmm0, %xmm2
 ; AVX512-NEXT:    vmulss %xmm2, %xmm1, %xmm1
@@ -932,7 +932,7 @@ define float @test_v16f32_undef(<16 x float> %a0) {
 ; AVX512-NEXT:    vmulss %xmm3, %xmm1, %xmm1
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm3 = xmm2[1,0]
 ; AVX512-NEXT:    vmulss %xmm3, %xmm1, %xmm1
-; AVX512-NEXT:    vpermilps {{.*#+}} xmm2 = xmm2[3,1,2,3]
+; AVX512-NEXT:    vpermilps {{.*#+}} xmm2 = xmm2[3,3,3,3]
 ; AVX512-NEXT:    vmulss %xmm2, %xmm1, %xmm1
 ; AVX512-NEXT:    vextractf32x4 $3, %zmm0, %xmm0
 ; AVX512-NEXT:    vmulss %xmm0, %xmm1, %xmm1
@@ -940,7 +940,7 @@ define float @test_v16f32_undef(<16 x float> %a0) {
 ; AVX512-NEXT:    vmulss %xmm2, %xmm1, %xmm1
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
 ; AVX512-NEXT:    vmulss %xmm2, %xmm1, %xmm1
-; AVX512-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX512-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX512-NEXT:    vmulss %xmm0, %xmm1, %xmm0
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/vector-reduce-mul.ll b/llvm/test/CodeGen/X86/vector-reduce-mul.ll
index 23bf09c7ec9c7..b0e3a79b6ff19 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-mul.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-mul.ll
@@ -778,28 +778,28 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 define i32 @test_v2i32(<2 x i32> %a0) {
 ; SSE2-LABEL: test_v2i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    pmuludq %xmm0, %xmm1
 ; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: test_v2i32:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE41-NEXT:    pmulld %xmm0, %xmm1
 ; SSE41-NEXT:    movd %xmm1, %eax
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: test_v2i32:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovd %xmm0, %eax
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: test_v2i32:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
 ; AVX512-NEXT:    retq
@@ -811,7 +811,7 @@ define i32 @test_v4i32(<4 x i32> %a0) {
 ; SSE2-LABEL: test_v4i32:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
 ; SSE2-NEXT:    pmuludq %xmm2, %xmm3
 ; SSE2-NEXT:    pmuludq %xmm0, %xmm1
@@ -823,7 +823,7 @@ define i32 @test_v4i32(<4 x i32> %a0) {
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmulld %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE41-NEXT:    pmulld %xmm1, %xmm0
 ; SSE41-NEXT:    movd %xmm0, %eax
 ; SSE41-NEXT:    retq
@@ -832,7 +832,7 @@ define i32 @test_v4i32(<4 x i32> %a0) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovd %xmm0, %eax
 ; AVX-NEXT:    retq
@@ -841,7 +841,7 @@ define i32 @test_v4i32(<4 x i32> %a0) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
 ; AVX512-NEXT:    retq
@@ -858,7 +858,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; SSE2-NEXT:    pmuludq %xmm1, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    pmuludq %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[2,0,2,2]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[2,2,2,2]
 ; SSE2-NEXT:    pmuludq %xmm3, %xmm0
 ; SSE2-NEXT:    pmuludq %xmm1, %xmm0
 ; SSE2-NEXT:    movd %xmm0, %eax
@@ -869,7 +869,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; SSE41-NEXT:    pmulld %xmm1, %xmm0
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmulld %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE41-NEXT:    pmulld %xmm1, %xmm0
 ; SSE41-NEXT:    movd %xmm0, %eax
 ; SSE41-NEXT:    retq
@@ -880,7 +880,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
 ; AVX1-NEXT:    vzeroupper
@@ -892,7 +892,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
 ; AVX2-NEXT:    vzeroupper
@@ -904,7 +904,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
 ; AVX512-NEXT:    vzeroupper
@@ -928,7 +928,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; SSE2-NEXT:    pmuludq %xmm1, %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    pmuludq %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,0,2,2]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,2,2,2]
 ; SSE2-NEXT:    pmuludq %xmm2, %xmm0
 ; SSE2-NEXT:    pmuludq %xmm1, %xmm0
 ; SSE2-NEXT:    movd %xmm0, %eax
@@ -941,7 +941,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; SSE41-NEXT:    pmulld %xmm0, %xmm1
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE41-NEXT:    pmulld %xmm1, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE41-NEXT:    pmulld %xmm0, %xmm1
 ; SSE41-NEXT:    movd %xmm1, %eax
 ; SSE41-NEXT:    retq
@@ -955,7 +955,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
 ; AVX1-NEXT:    vzeroupper
@@ -968,7 +968,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
 ; AVX2-NEXT:    vzeroupper
@@ -982,7 +982,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
 ; AVX512-NEXT:    vzeroupper
@@ -1018,7 +1018,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; SSE2-NEXT:    pmuludq %xmm0, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    pmuludq %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm11[2,0,2,2]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm11[2,2,2,2]
 ; SSE2-NEXT:    pmuludq %xmm11, %xmm1
 ; SSE2-NEXT:    pmuludq %xmm0, %xmm1
 ; SSE2-NEXT:    movd %xmm1, %eax
@@ -1035,7 +1035,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; SSE41-NEXT:    pmulld %xmm0, %xmm2
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE41-NEXT:    pmulld %xmm2, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE41-NEXT:    pmulld %xmm0, %xmm1
 ; SSE41-NEXT:    movd %xmm1, %eax
 ; SSE41-NEXT:    retq
@@ -1055,7 +1055,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
 ; AVX1-NEXT:    vzeroupper
@@ -1070,7 +1070,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
 ; AVX2-NEXT:    vzeroupper
@@ -1085,7 +1085,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
 ; AVX512-NEXT:    vzeroupper
@@ -1130,7 +1130,7 @@ define i16 @test_v2i16(<2 x i16> %a0) {
 define i16 @test_v4i16(<4 x i16> %a0) {
 ; SSE-LABEL: test_v4i16:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE-NEXT:    pmullw %xmm0, %xmm1
 ; SSE-NEXT:    movdqa %xmm1, %xmm0
 ; SSE-NEXT:    psrld $16, %xmm0
@@ -1141,7 +1141,7 @@ define i16 @test_v4i16(<4 x i16> %a0) {
 ;
 ; AVX-LABEL: test_v4i16:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -1151,7 +1151,7 @@ define i16 @test_v4i16(<4 x i16> %a0) {
 ;
 ; AVX512-LABEL: test_v4i16:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -1167,7 +1167,7 @@ define i16 @test_v8i16(<8 x i16> %a0) {
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    pmullw %xmm0, %xmm1
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE-NEXT:    pmullw %xmm1, %xmm0
 ; SSE-NEXT:    movdqa %xmm0, %xmm1
 ; SSE-NEXT:    psrld $16, %xmm1
@@ -1180,7 +1180,7 @@ define i16 @test_v8i16(<8 x i16> %a0) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -1192,7 +1192,7 @@ define i16 @test_v8i16(<8 x i16> %a0) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -1209,7 +1209,7 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; SSE-NEXT:    pmullw %xmm1, %xmm0
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    pmullw %xmm0, %xmm1
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE-NEXT:    pmullw %xmm1, %xmm0
 ; SSE-NEXT:    movdqa %xmm0, %xmm1
 ; SSE-NEXT:    psrld $16, %xmm1
@@ -1224,7 +1224,7 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -1239,7 +1239,7 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -1254,7 +1254,7 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -1274,7 +1274,7 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; SSE-NEXT:    pmullw %xmm0, %xmm1
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE-NEXT:    pmullw %xmm1, %xmm0
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE-NEXT:    pmullw %xmm0, %xmm1
 ; SSE-NEXT:    movdqa %xmm1, %xmm0
 ; SSE-NEXT:    psrld $16, %xmm0
@@ -1292,7 +1292,7 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -1308,7 +1308,7 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -1325,7 +1325,7 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -1342,7 +1342,7 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -1359,7 +1359,7 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512DQ-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -1376,7 +1376,7 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512DQVL-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -1400,7 +1400,7 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; SSE-NEXT:    pmullw %xmm0, %xmm2
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE-NEXT:    pmullw %xmm2, %xmm0
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE-NEXT:    pmullw %xmm0, %xmm1
 ; SSE-NEXT:    movdqa %xmm1, %xmm0
 ; SSE-NEXT:    psrld $16, %xmm0
@@ -1424,7 +1424,7 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -1442,7 +1442,7 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -1460,7 +1460,7 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -1478,7 +1478,7 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -1498,7 +1498,7 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512DQ-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -1518,7 +1518,7 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512DQVL-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -1581,7 +1581,7 @@ define i8 @test_v4i8(<4 x i8> %a0) {
 ; SSE41-LABEL: test_v4i8:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE41-NEXT:    pmullw %xmm0, %xmm1
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
 ; SSE41-NEXT:    psrld $16, %xmm0
@@ -1593,7 +1593,7 @@ define i8 @test_v4i8(<4 x i8> %a0) {
 ; AVX-LABEL: test_v4i8:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -1604,7 +1604,7 @@ define i8 @test_v4i8(<4 x i8> %a0) {
 ; AVX512-LABEL: test_v4i8:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -1636,7 +1636,7 @@ define i8 @test_v8i8(<8 x i8> %a0) {
 ; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmullw %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE41-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
 ; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; SSE41-NEXT:    pmullw %xmm0, %xmm1
@@ -1652,7 +1652,7 @@ define i8 @test_v8i8(<8 x i8> %a0) {
 ; AVX-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -1665,7 +1665,7 @@ define i8 @test_v8i8(<8 x i8> %a0) {
 ; AVX512-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -1719,7 +1719,7 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ; SSE41-NEXT:    pmullw %xmm2, %xmm0
 ; SSE41-NEXT:    pand %xmm0, %xmm1
 ; SSE41-NEXT:    packuswb %xmm3, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; SSE41-NEXT:    pmullw %xmm0, %xmm1
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
@@ -1737,7 +1737,7 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $8, %xmm0, %xmm1
 ; AVX1-NEXT:    vpsrlw $8, %xmm1, %xmm1
@@ -1754,9 +1754,9 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm0[1,1,2,3,4,5,6,7]
+; AVX2-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm0[1,1,1,1,4,5,6,7]
 ; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
 ; AVX2-NEXT:    # kill: def $al killed $al killed $eax
@@ -1769,7 +1769,7 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
 ; AVX512BW-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm1
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
 ; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
 ; AVX512BW-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm1
@@ -1791,7 +1791,7 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
 ; AVX512BWVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm1
-; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
 ; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
 ; AVX512BWVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm1
@@ -1814,7 +1814,7 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ; AVX512DQ-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
 ; AVX512DQ-NEXT:    vpmovdb %zmm1, %xmm1
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
 ; AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
 ; AVX512DQ-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
@@ -1839,7 +1839,7 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
 ; AVX512DQVL-NEXT:    vpmovdb %zmm1, %xmm1
-; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
 ; AVX512DQVL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
 ; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
@@ -1916,7 +1916,7 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; SSE41-NEXT:    pmullw %xmm0, %xmm2
 ; SSE41-NEXT:    pand %xmm2, %xmm1
 ; SSE41-NEXT:    packuswb %xmm3, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
 ; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; SSE41-NEXT:    pmullw %xmm0, %xmm1
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
@@ -1939,7 +1939,7 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $8, %xmm0, %xmm1
 ; AVX1-NEXT:    vpsrlw $8, %xmm1, %xmm1
@@ -1961,7 +1961,7 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX2-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; AVX2-NEXT:    vpmullw %xmm2, %xmm1, %xmm1
 ; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufb {{.*#+}} xmm1 = xmm0[8],zero,xmm0[10],zero,xmm0[12],zero,xmm0[14],zero,xmm0[8],zero,xmm0[10],zero,xmm0[12],zero,xmm0[14],zero
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrlq $32, %xmm0, %xmm1
 ; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -1986,7 +1986,7 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
 ; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm1
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
 ; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
 ; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm1
@@ -2014,7 +2014,7 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
 ; AVX512BWVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm1
-; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
 ; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
 ; AVX512BWVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm1
@@ -2043,7 +2043,7 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; AVX512DQ-NEXT:    vpmullw %xmm2, %xmm1, %xmm1
 ; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm1 = xmm0[8],zero,xmm0[10],zero,xmm0[12],zero,xmm0[14],zero,xmm0[8],zero,xmm0[10],zero,xmm0[12],zero,xmm0[14],zero
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512DQ-NEXT:    vpsrlq $32, %xmm0, %xmm1
 ; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -2069,7 +2069,7 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX512DQVL-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; AVX512DQVL-NEXT:    vpmullw %xmm2, %xmm1, %xmm1
 ; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512DQVL-NEXT:    vpshufb {{.*#+}} xmm1 = xmm0[8],zero,xmm0[10],zero,xmm0[12],zero,xmm0[14],zero,xmm0[8],zero,xmm0[10],zero,xmm0[12],zero,xmm0[14],zero
+; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512DQVL-NEXT:    vpsrlq $32, %xmm0, %xmm1
 ; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -2166,7 +2166,7 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; SSE41-NEXT:    pmullw %xmm0, %xmm1
 ; SSE41-NEXT:    pand %xmm1, %xmm2
 ; SSE41-NEXT:    packuswb %xmm3, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
 ; SSE41-NEXT:    pmullw %xmm0, %xmm1
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
@@ -2198,7 +2198,7 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $8, %xmm0, %xmm1
 ; AVX1-NEXT:    vpsrlw $8, %xmm1, %xmm1
@@ -2223,7 +2223,7 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm3
 ; AVX2-NEXT:    vpmullw %xmm0, %xmm3, %xmm0
 ; AVX2-NEXT:    vpmullw %xmm0, %xmm2, %xmm0
-; AVX2-NEXT:    vpshufb {{.*#+}} xmm2 = xmm0[8],zero,xmm0[10],zero,xmm0[12],zero,xmm0[14],zero,xmm0[8],zero,xmm0[10],zero,xmm0[12],zero,xmm0[14],zero
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrlq $32, %xmm0, %xmm2
 ; AVX2-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
@@ -2251,18 +2251,20 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX512BW-NEXT:    vpand %ymm3, %ymm0, %ymm1
 ; AVX512BW-NEXT:    vpackuswb %ymm4, %ymm1, %ymm1
 ; AVX512BW-NEXT:    vextracti128 $1, %ymm1, %xmm1
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX512BW-NEXT:    vpmullw %xmm3, %xmm2, %xmm2
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
-; AVX512BW-NEXT:    vpand %xmm3, %xmm2, %xmm2
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm3 = zmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX512BW-NEXT:    vpmullw %xmm1, %xmm2, %xmm1
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; AVX512BW-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpmullw %zmm1, %zmm3, %zmm1
+; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm1
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
 ; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; AVX512BW-NEXT:    vpmullw %xmm2, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm1 = xmm0[8],zero,xmm0[10],zero,xmm0[12],zero,xmm0[14],zero,xmm0[8],zero,xmm0[10],zero,xmm0[12],zero,xmm0[14],zero
 ; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vpsrlq $32, %xmm0, %xmm1
 ; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpand %xmm3, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpand %xmm2, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX512BW-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vpsrlw $8, %xmm0, %xmm1
@@ -2286,13 +2288,15 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX512BWVL-NEXT:    vpand %ymm3, %ymm0, %ymm1
 ; AVX512BWVL-NEXT:    vpackuswb %ymm4, %ymm1, %ymm1
 ; AVX512BWVL-NEXT:    vextracti128 $1, %ymm1, %xmm1
-; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX512BWVL-NEXT:    vpmullw %xmm3, %xmm2, %xmm2
-; AVX512BWVL-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
+; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm3 = zmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
+; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm2, %xmm1
+; AVX512BWVL-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm3, %zmm1
+; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
+; AVX512BWVL-NEXT:    vpmovwb %zmm0, %ymm1
+; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
 ; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; AVX512BWVL-NEXT:    vpmullw %xmm2, %xmm1, %xmm1
-; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512BWVL-NEXT:    vpshufb {{.*#+}} xmm1 = xmm0[8],zero,xmm0[10],zero,xmm0[12],zero,xmm0[14],zero,xmm0[8],zero,xmm0[10],zero,xmm0[12],zero,xmm0[14],zero
 ; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    vpshufb {{.*#+}} xmm1 = xmm0[4],zero,xmm0[6],zero,zero,zero,zero,zero,xmm0[12],zero,xmm0[14],zero,zero,zero,zero,zero
 ; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -2318,15 +2322,14 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX512DQ-NEXT:    vextracti128 $1, %ymm2, %xmm1
 ; AVX512DQ-NEXT:    vpmullw %xmm0, %xmm1, %xmm0
 ; AVX512DQ-NEXT:    vpmullw %xmm0, %xmm2, %xmm0
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm1 = xmm0[u,u,u,u,8,10,12,14,u,u,u,u,u,u,u,u]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
+; AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
 ; AVX512DQ-NEXT:    vpand %xmm1, %xmm0, %xmm2
 ; AVX512DQ-NEXT:    vpxor %xmm3, %xmm3, %xmm3
 ; AVX512DQ-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,2,3]
-; AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
-; AVX512DQ-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
-; AVX512DQ-NEXT:    vpand %xmm1, %xmm0, %xmm2
-; AVX512DQ-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
 ; AVX512DQ-NEXT:    vpsrld $16, %xmm2, %xmm2
 ; AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
 ; AVX512DQ-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
@@ -2353,15 +2356,14 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX512DQVL-NEXT:    vextracti128 $1, %ymm2, %xmm1
 ; AVX512DQVL-NEXT:    vpmullw %xmm0, %xmm1, %xmm0
 ; AVX512DQVL-NEXT:    vpmullw %xmm0, %xmm2, %xmm0
+; AVX512DQVL-NEXT:    vpshufb {{.*#+}} xmm1 = xmm0[u,u,u,u,8,10,12,14,u,u,u,u,u,u,u,u]
+; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
+; AVX512DQVL-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512DQVL-NEXT:    vmovdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
 ; AVX512DQVL-NEXT:    vpand %xmm1, %xmm0, %xmm2
 ; AVX512DQVL-NEXT:    vpxor %xmm3, %xmm3, %xmm3
 ; AVX512DQVL-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
-; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,2,3]
-; AVX512DQVL-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
-; AVX512DQVL-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
-; AVX512DQVL-NEXT:    vpand %xmm1, %xmm0, %xmm2
-; AVX512DQVL-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
 ; AVX512DQVL-NEXT:    vpsrld $16, %xmm2, %xmm2
 ; AVX512DQVL-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
 ; AVX512DQVL-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
@@ -2501,7 +2503,7 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; SSE41-NEXT:    pmullw %xmm1, %xmm0
 ; SSE41-NEXT:    pand %xmm0, %xmm4
 ; SSE41-NEXT:    packuswb %xmm2, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
 ; SSE41-NEXT:    pmullw %xmm0, %xmm1
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
@@ -2551,7 +2553,7 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $8, %xmm0, %xmm1
 ; AVX1-NEXT:    vpsrlw $8, %xmm1, %xmm1
@@ -2584,7 +2586,7 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm3
 ; AVX2-NEXT:    vpmullw %xmm1, %xmm3, %xmm1
 ; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufb {{.*#+}} xmm1 = xmm0[8],zero,xmm0[10],zero,xmm0[12],zero,xmm0[14],zero,xmm0[8],zero,xmm0[10],zero,xmm0[12],zero,xmm0[14],zero
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrlq $32, %xmm0, %xmm1
 ; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -2615,18 +2617,20 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX512BW-NEXT:    vpand %ymm1, %ymm2, %ymm1
 ; AVX512BW-NEXT:    vpackuswb %ymm3, %ymm1, %ymm1
 ; AVX512BW-NEXT:    vextracti128 $1, %ymm1, %xmm1
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX512BW-NEXT:    vpmullw %xmm3, %xmm0, %xmm0
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
-; AVX512BW-NEXT:    vpand %xmm3, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; AVX512BW-NEXT:    vpmullw %xmm0, %xmm1, %xmm0
-; AVX512BW-NEXT:    vpmullw %xmm0, %xmm2, %xmm0
-; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm1 = xmm0[8],zero,xmm0[10],zero,xmm0[12],zero,xmm0[14],zero,xmm0[8],zero,xmm0[10],zero,xmm0[12],zero,xmm0[14],zero
-; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpsrlq $32, %xmm0, %xmm1
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm3 = zmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpand %xmm3, %xmm0, %xmm0
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
+; AVX512BW-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpmullw %zmm0, %zmm3, %zmm0
+; AVX512BW-NEXT:    vpmullw %zmm0, %zmm2, %zmm0
+; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm2
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,1,1]
+; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
+; AVX512BW-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpsrlq $32, %xmm0, %xmm2
+; AVX512BW-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX512BW-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vpsrlw $8, %xmm0, %xmm1
@@ -2653,13 +2657,15 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX512BWVL-NEXT:    vpand %ymm1, %ymm2, %ymm1
 ; AVX512BWVL-NEXT:    vpackuswb %ymm3, %ymm1, %ymm1
 ; AVX512BWVL-NEXT:    vextracti128 $1, %ymm1, %xmm1
-; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX512BWVL-NEXT:    vpmullw %xmm3, %xmm0, %xmm0
+; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm3 = zmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
+; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX512BWVL-NEXT:    vpmullw %zmm0, %zmm3, %zmm0
+; AVX512BWVL-NEXT:    vpmullw %zmm0, %zmm2, %zmm0
+; AVX512BWVL-NEXT:    vpmovwb %zmm0, %ymm1
+; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
 ; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; AVX512BWVL-NEXT:    vpmullw %xmm0, %xmm1, %xmm0
-; AVX512BWVL-NEXT:    vpmullw %xmm0, %xmm2, %xmm0
-; AVX512BWVL-NEXT:    vpshufb {{.*#+}} xmm1 = xmm0[8],zero,xmm0[10],zero,xmm0[12],zero,xmm0[14],zero,xmm0[8],zero,xmm0[10],zero,xmm0[12],zero,xmm0[14],zero
 ; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    vpshufb {{.*#+}} xmm1 = xmm0[4],zero,xmm0[6],zero,zero,zero,zero,zero,xmm0[12],zero,xmm0[14],zero,zero,zero,zero,zero
 ; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
@@ -2694,15 +2700,14 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX512DQ-NEXT:    vextracti128 $1, %ymm0, %xmm2
 ; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm2, %xmm1
 ; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm1 = xmm0[u,u,u,u,8,10,12,14,u,u,u,u,u,u,u,u]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
+; AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
 ; AVX512DQ-NEXT:    vpand %xmm1, %xmm0, %xmm2
 ; AVX512DQ-NEXT:    vpxor %xmm3, %xmm3, %xmm3
 ; AVX512DQ-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,2,3]
-; AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
-; AVX512DQ-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
-; AVX512DQ-NEXT:    vpand %xmm1, %xmm0, %xmm2
-; AVX512DQ-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
 ; AVX512DQ-NEXT:    vpsrld $16, %xmm2, %xmm2
 ; AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
 ; AVX512DQ-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
@@ -2738,15 +2743,14 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX512DQVL-NEXT:    vextracti128 $1, %ymm0, %xmm2
 ; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm2, %xmm1
 ; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX512DQVL-NEXT:    vpshufb {{.*#+}} xmm1 = xmm0[u,u,u,u,8,10,12,14,u,u,u,u,u,u,u,u]
+; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
+; AVX512DQVL-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512DQVL-NEXT:    vmovdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
 ; AVX512DQVL-NEXT:    vpand %xmm1, %xmm0, %xmm2
 ; AVX512DQVL-NEXT:    vpxor %xmm3, %xmm3, %xmm3
 ; AVX512DQVL-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
-; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,2,3]
-; AVX512DQVL-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
-; AVX512DQVL-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
-; AVX512DQVL-NEXT:    vpand %xmm1, %xmm0, %xmm2
-; AVX512DQVL-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
 ; AVX512DQVL-NEXT:    vpsrld $16, %xmm2, %xmm2
 ; AVX512DQVL-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
 ; AVX512DQVL-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
diff --git a/llvm/test/CodeGen/X86/vector-reduce-or.ll b/llvm/test/CodeGen/X86/vector-reduce-or.ll
index 95bff8e03afae..622f503ef6cbf 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-or.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-or.ll
@@ -182,14 +182,14 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 define i32 @test_v2i32(<2 x i32> %a0) {
 ; SSE-LABEL: test_v2i32:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE-NEXT:    por %xmm0, %xmm1
 ; SSE-NEXT:    movd %xmm1, %eax
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: test_v2i32:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovd %xmm0, %eax
 ; AVX-NEXT:    retq
@@ -202,7 +202,7 @@ define i32 @test_v4i32(<4 x i32> %a0) {
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    por %xmm0, %xmm1
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE-NEXT:    por %xmm1, %xmm0
 ; SSE-NEXT:    movd %xmm0, %eax
 ; SSE-NEXT:    retq
@@ -211,7 +211,7 @@ define i32 @test_v4i32(<4 x i32> %a0) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovd %xmm0, %eax
 ; AVX-NEXT:    retq
@@ -225,7 +225,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; SSE-NEXT:    por %xmm1, %xmm0
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    por %xmm0, %xmm1
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE-NEXT:    por %xmm1, %xmm0
 ; SSE-NEXT:    movd %xmm0, %eax
 ; SSE-NEXT:    retq
@@ -236,7 +236,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
 ; AVX1-NEXT:    vzeroupper
@@ -248,7 +248,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
 ; AVX2-NEXT:    vzeroupper
@@ -260,7 +260,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
 ; AVX512-NEXT:    vzeroupper
@@ -277,7 +277,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; SSE-NEXT:    por %xmm0, %xmm1
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE-NEXT:    por %xmm1, %xmm0
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE-NEXT:    por %xmm0, %xmm1
 ; SSE-NEXT:    movd %xmm1, %eax
 ; SSE-NEXT:    retq
@@ -289,7 +289,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
 ; AVX1-NEXT:    vzeroupper
@@ -302,7 +302,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
 ; AVX2-NEXT:    vzeroupper
@@ -316,7 +316,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
 ; AVX512-NEXT:    vzeroupper
@@ -337,7 +337,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; SSE-NEXT:    por %xmm0, %xmm2
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE-NEXT:    por %xmm2, %xmm0
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE-NEXT:    por %xmm0, %xmm1
 ; SSE-NEXT:    movd %xmm1, %eax
 ; SSE-NEXT:    retq
@@ -351,7 +351,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
 ; AVX1-NEXT:    vzeroupper
@@ -366,7 +366,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
 ; AVX2-NEXT:    vzeroupper
@@ -381,7 +381,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
 ; AVX512-NEXT:    vzeroupper
@@ -418,7 +418,7 @@ define i16 @test_v2i16(<2 x i16> %a0) {
 define i16 @test_v4i16(<4 x i16> %a0) {
 ; SSE-LABEL: test_v4i16:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE-NEXT:    por %xmm0, %xmm1
 ; SSE-NEXT:    movdqa %xmm1, %xmm0
 ; SSE-NEXT:    psrld $16, %xmm0
@@ -429,7 +429,7 @@ define i16 @test_v4i16(<4 x i16> %a0) {
 ;
 ; AVX-LABEL: test_v4i16:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
@@ -445,7 +445,7 @@ define i16 @test_v8i16(<8 x i16> %a0) {
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    por %xmm0, %xmm1
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE-NEXT:    por %xmm1, %xmm0
 ; SSE-NEXT:    movdqa %xmm0, %xmm1
 ; SSE-NEXT:    psrld $16, %xmm1
@@ -458,7 +458,7 @@ define i16 @test_v8i16(<8 x i16> %a0) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
@@ -475,7 +475,7 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; SSE-NEXT:    por %xmm1, %xmm0
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    por %xmm0, %xmm1
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE-NEXT:    por %xmm1, %xmm0
 ; SSE-NEXT:    movdqa %xmm0, %xmm1
 ; SSE-NEXT:    psrld $16, %xmm1
@@ -490,7 +490,7 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
@@ -505,7 +505,7 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
@@ -520,7 +520,7 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
@@ -540,7 +540,7 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; SSE-NEXT:    por %xmm0, %xmm1
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE-NEXT:    por %xmm1, %xmm0
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE-NEXT:    por %xmm0, %xmm1
 ; SSE-NEXT:    movdqa %xmm1, %xmm0
 ; SSE-NEXT:    psrld $16, %xmm0
@@ -556,7 +556,7 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
@@ -572,7 +572,7 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
@@ -589,7 +589,7 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
@@ -613,7 +613,7 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; SSE-NEXT:    por %xmm0, %xmm2
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE-NEXT:    por %xmm2, %xmm0
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE-NEXT:    por %xmm0, %xmm1
 ; SSE-NEXT:    movdqa %xmm1, %xmm0
 ; SSE-NEXT:    psrld $16, %xmm0
@@ -631,7 +631,7 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
@@ -649,7 +649,7 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
@@ -667,7 +667,7 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
@@ -733,7 +733,7 @@ define i8 @test_v4i8(<4 x i8> %a0) {
 define i8 @test_v8i8(<8 x i8> %a0) {
 ; SSE-LABEL: test_v8i8:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE-NEXT:    por %xmm0, %xmm1
 ; SSE-NEXT:    movdqa %xmm1, %xmm0
 ; SSE-NEXT:    psrld $16, %xmm0
@@ -747,7 +747,7 @@ define i8 @test_v8i8(<8 x i8> %a0) {
 ;
 ; AVX-LABEL: test_v8i8:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
@@ -765,7 +765,7 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    por %xmm0, %xmm1
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE-NEXT:    por %xmm1, %xmm0
 ; SSE-NEXT:    movdqa %xmm0, %xmm1
 ; SSE-NEXT:    psrld $16, %xmm1
@@ -781,7 +781,7 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
@@ -800,7 +800,7 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; SSE-NEXT:    por %xmm1, %xmm0
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    por %xmm0, %xmm1
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE-NEXT:    por %xmm1, %xmm0
 ; SSE-NEXT:    movdqa %xmm0, %xmm1
 ; SSE-NEXT:    psrld $16, %xmm1
@@ -818,7 +818,7 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
@@ -835,7 +835,7 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
@@ -852,7 +852,7 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
@@ -874,7 +874,7 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; SSE-NEXT:    por %xmm0, %xmm1
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE-NEXT:    por %xmm1, %xmm0
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE-NEXT:    por %xmm0, %xmm1
 ; SSE-NEXT:    movdqa %xmm1, %xmm0
 ; SSE-NEXT:    psrld $16, %xmm0
@@ -893,7 +893,7 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
@@ -911,7 +911,7 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
@@ -930,7 +930,7 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
@@ -956,7 +956,7 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; SSE-NEXT:    por %xmm0, %xmm2
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE-NEXT:    por %xmm2, %xmm0
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE-NEXT:    por %xmm0, %xmm1
 ; SSE-NEXT:    movdqa %xmm1, %xmm0
 ; SSE-NEXT:    psrld $16, %xmm0
@@ -977,7 +977,7 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
@@ -997,7 +997,7 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
@@ -1017,7 +1017,7 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
diff --git a/llvm/test/CodeGen/X86/vector-reduce-smax.ll b/llvm/test/CodeGen/X86/vector-reduce-smax.ll
index 26bbfed521968..89354e6b2d9fe 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-smax.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-smax.ll
@@ -742,7 +742,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 define i32 @test_v2i32(<2 x i32> %a0) {
 ; SSE2-LABEL: test_v2i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
 ; SSE2-NEXT:    pand %xmm2, %xmm0
@@ -753,21 +753,21 @@ define i32 @test_v2i32(<2 x i32> %a0) {
 ;
 ; SSE4-LABEL: test_v2i32:
 ; SSE4:       # %bb.0:
-; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE4-NEXT:    pmaxsd %xmm0, %xmm1
 ; SSE4-NEXT:    movd %xmm1, %eax
 ; SSE4-NEXT:    retq
 ;
 ; AVX-LABEL: test_v2i32:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovd %xmm0, %eax
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: test_v2i32:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
 ; AVX512-NEXT:    retq
@@ -784,7 +784,7 @@ define i32 @test_v4i32(<4 x i32> %a0) {
 ; SSE2-NEXT:    pand %xmm2, %xmm0
 ; SSE2-NEXT:    pandn %xmm1, %xmm2
 ; SSE2-NEXT:    por %xmm0, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
 ; SSE2-NEXT:    movdqa %xmm2, %xmm1
 ; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
 ; SSE2-NEXT:    pand %xmm1, %xmm2
@@ -797,7 +797,7 @@ define i32 @test_v4i32(<4 x i32> %a0) {
 ; SSE4:       # %bb.0:
 ; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE4-NEXT:    pmaxsd %xmm0, %xmm1
-; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE4-NEXT:    pmaxsd %xmm1, %xmm0
 ; SSE4-NEXT:    movd %xmm0, %eax
 ; SSE4-NEXT:    retq
@@ -806,7 +806,7 @@ define i32 @test_v4i32(<4 x i32> %a0) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovd %xmm0, %eax
 ; AVX-NEXT:    retq
@@ -815,7 +815,7 @@ define i32 @test_v4i32(<4 x i32> %a0) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
 ; AVX512-NEXT:    retq
@@ -837,7 +837,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; SSE2-NEXT:    pand %xmm1, %xmm2
 ; SSE2-NEXT:    pandn %xmm0, %xmm1
 ; SSE2-NEXT:    por %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
 ; SSE2-NEXT:    pand %xmm2, %xmm1
@@ -851,7 +851,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; SSE4-NEXT:    pmaxsd %xmm1, %xmm0
 ; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE4-NEXT:    pmaxsd %xmm0, %xmm1
-; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE4-NEXT:    pmaxsd %xmm1, %xmm0
 ; SSE4-NEXT:    movd %xmm0, %eax
 ; SSE4-NEXT:    retq
@@ -862,7 +862,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX1-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
 ; AVX1-NEXT:    vzeroupper
@@ -874,7 +874,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX2-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
 ; AVX2-NEXT:    vzeroupper
@@ -886,7 +886,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX512-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
 ; AVX512-NEXT:    vzeroupper
@@ -919,7 +919,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; SSE2-NEXT:    pand %xmm2, %xmm0
 ; SSE2-NEXT:    pandn %xmm1, %xmm2
 ; SSE2-NEXT:    por %xmm0, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
 ; SSE2-NEXT:    movdqa %xmm2, %xmm1
 ; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
 ; SSE2-NEXT:    pand %xmm1, %xmm2
@@ -935,7 +935,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; SSE4-NEXT:    pmaxsd %xmm0, %xmm1
 ; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE4-NEXT:    pmaxsd %xmm1, %xmm0
-; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE4-NEXT:    pmaxsd %xmm0, %xmm1
 ; SSE4-NEXT:    movd %xmm1, %eax
 ; SSE4-NEXT:    retq
@@ -949,7 +949,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX1-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
 ; AVX1-NEXT:    vzeroupper
@@ -962,7 +962,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX2-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
 ; AVX2-NEXT:    vzeroupper
@@ -976,7 +976,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX512-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
 ; AVX512-NEXT:    vzeroupper
@@ -1029,7 +1029,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; SSE2-NEXT:    pand %xmm1, %xmm2
 ; SSE2-NEXT:    pandn %xmm0, %xmm1
 ; SSE2-NEXT:    por %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
 ; SSE2-NEXT:    pand %xmm2, %xmm1
@@ -1049,7 +1049,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; SSE4-NEXT:    pmaxsd %xmm0, %xmm2
 ; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE4-NEXT:    pmaxsd %xmm2, %xmm0
-; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE4-NEXT:    pmaxsd %xmm0, %xmm1
 ; SSE4-NEXT:    movd %xmm1, %eax
 ; SSE4-NEXT:    retq
@@ -1069,7 +1069,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX1-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
 ; AVX1-NEXT:    vzeroupper
@@ -1084,7 +1084,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX2-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
 ; AVX2-NEXT:    vzeroupper
@@ -1099,7 +1099,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX512-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
 ; AVX512-NEXT:    vzeroupper
@@ -1144,7 +1144,7 @@ define i16 @test_v2i16(<2 x i16> %a0) {
 define i16 @test_v4i16(<4 x i16> %a0) {
 ; SSE-LABEL: test_v4i16:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE-NEXT:    pmaxsw %xmm0, %xmm1
 ; SSE-NEXT:    movdqa %xmm1, %xmm0
 ; SSE-NEXT:    psrld $16, %xmm0
@@ -1155,7 +1155,7 @@ define i16 @test_v4i16(<4 x i16> %a0) {
 ;
 ; AVX-LABEL: test_v4i16:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-NEXT:    vpmaxsw %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX-NEXT:    vpmaxsw %xmm1, %xmm0, %xmm0
@@ -1165,7 +1165,7 @@ define i16 @test_v4i16(<4 x i16> %a0) {
 ;
 ; AVX512-LABEL: test_v4i16:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpmaxsw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpmaxsw %xmm1, %xmm0, %xmm0
@@ -1181,7 +1181,7 @@ define i16 @test_v8i16(<8 x i16> %a0) {
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    pmaxsw %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE2-NEXT:    pmaxsw %xmm1, %xmm0
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; SSE2-NEXT:    psrld $16, %xmm1
@@ -1226,7 +1226,7 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; SSE2-NEXT:    pmaxsw %xmm1, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    pmaxsw %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE2-NEXT:    pmaxsw %xmm1, %xmm0
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; SSE2-NEXT:    psrld $16, %xmm1
@@ -1292,7 +1292,7 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; SSE2-NEXT:    pmaxsw %xmm0, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    pmaxsw %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    pmaxsw %xmm0, %xmm1
 ; SSE2-NEXT:    movdqa %xmm1, %xmm0
 ; SSE2-NEXT:    psrld $16, %xmm0
@@ -1370,7 +1370,7 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; SSE2-NEXT:    pmaxsw %xmm0, %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE2-NEXT:    pmaxsw %xmm2, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    pmaxsw %xmm0, %xmm1
 ; SSE2-NEXT:    movdqa %xmm1, %xmm0
 ; SSE2-NEXT:    psrld $16, %xmm0
@@ -1554,7 +1554,7 @@ define i8 @test_v4i8(<4 x i8> %a0) {
 define i8 @test_v8i8(<8 x i8> %a0) {
 ; SSE2-LABEL: test_v8i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; SSE2-NEXT:    pcmpgtb %xmm1, %xmm2
 ; SSE2-NEXT:    pand %xmm2, %xmm0
@@ -1580,7 +1580,7 @@ define i8 @test_v8i8(<8 x i8> %a0) {
 ;
 ; SSE4-LABEL: test_v8i8:
 ; SSE4:       # %bb.0:
-; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE4-NEXT:    pmaxsb %xmm0, %xmm1
 ; SSE4-NEXT:    movdqa %xmm1, %xmm0
 ; SSE4-NEXT:    psrld $16, %xmm0
@@ -1594,7 +1594,7 @@ define i8 @test_v8i8(<8 x i8> %a0) {
 ;
 ; AVX-LABEL: test_v8i8:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-NEXT:    vpmaxsb %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX-NEXT:    vpmaxsb %xmm1, %xmm0, %xmm0
@@ -1606,7 +1606,7 @@ define i8 @test_v8i8(<8 x i8> %a0) {
 ;
 ; AVX512-LABEL: test_v8i8:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpmaxsb %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpmaxsb %xmm1, %xmm0, %xmm0
@@ -1628,7 +1628,7 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ; SSE2-NEXT:    pand %xmm2, %xmm0
 ; SSE2-NEXT:    pandn %xmm1, %xmm2
 ; SSE2-NEXT:    por %xmm0, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
 ; SSE2-NEXT:    movdqa %xmm2, %xmm1
 ; SSE2-NEXT:    pcmpgtb %xmm0, %xmm1
 ; SSE2-NEXT:    pand %xmm1, %xmm2
@@ -1703,7 +1703,7 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; SSE2-NEXT:    pand %xmm1, %xmm2
 ; SSE2-NEXT:    pandn %xmm0, %xmm1
 ; SSE2-NEXT:    por %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; SSE2-NEXT:    pcmpgtb %xmm0, %xmm2
 ; SSE2-NEXT:    pand %xmm2, %xmm1
@@ -1809,7 +1809,7 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; SSE2-NEXT:    pand %xmm2, %xmm0
 ; SSE2-NEXT:    pandn %xmm1, %xmm2
 ; SSE2-NEXT:    por %xmm0, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
 ; SSE2-NEXT:    movdqa %xmm2, %xmm1
 ; SSE2-NEXT:    pcmpgtb %xmm0, %xmm1
 ; SSE2-NEXT:    pand %xmm1, %xmm2
@@ -1943,7 +1943,7 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; SSE2-NEXT:    pand %xmm1, %xmm2
 ; SSE2-NEXT:    pandn %xmm0, %xmm1
 ; SSE2-NEXT:    por %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; SSE2-NEXT:    pcmpgtb %xmm0, %xmm2
 ; SSE2-NEXT:    pand %xmm2, %xmm1
diff --git a/llvm/test/CodeGen/X86/vector-reduce-smin.ll b/llvm/test/CodeGen/X86/vector-reduce-smin.ll
index 6f561c27d71dc..106888b99c28c 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-smin.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-smin.ll
@@ -742,7 +742,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 define i32 @test_v2i32(<2 x i32> %a0) {
 ; SSE2-LABEL: test_v2i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
 ; SSE2-NEXT:    pand %xmm2, %xmm0
@@ -753,21 +753,21 @@ define i32 @test_v2i32(<2 x i32> %a0) {
 ;
 ; SSE4-LABEL: test_v2i32:
 ; SSE4:       # %bb.0:
-; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE4-NEXT:    pminsd %xmm0, %xmm1
 ; SSE4-NEXT:    movd %xmm1, %eax
 ; SSE4-NEXT:    retq
 ;
 ; AVX-LABEL: test_v2i32:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovd %xmm0, %eax
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: test_v2i32:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
 ; AVX512-NEXT:    retq
@@ -784,7 +784,7 @@ define i32 @test_v4i32(<4 x i32> %a0) {
 ; SSE2-NEXT:    pand %xmm2, %xmm0
 ; SSE2-NEXT:    pandn %xmm1, %xmm2
 ; SSE2-NEXT:    por %xmm0, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; SSE2-NEXT:    pcmpgtd %xmm2, %xmm1
 ; SSE2-NEXT:    pand %xmm1, %xmm2
@@ -797,7 +797,7 @@ define i32 @test_v4i32(<4 x i32> %a0) {
 ; SSE4:       # %bb.0:
 ; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE4-NEXT:    pminsd %xmm0, %xmm1
-; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE4-NEXT:    pminsd %xmm1, %xmm0
 ; SSE4-NEXT:    movd %xmm0, %eax
 ; SSE4-NEXT:    retq
@@ -806,7 +806,7 @@ define i32 @test_v4i32(<4 x i32> %a0) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovd %xmm0, %eax
 ; AVX-NEXT:    retq
@@ -815,7 +815,7 @@ define i32 @test_v4i32(<4 x i32> %a0) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
 ; AVX512-NEXT:    retq
@@ -837,7 +837,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; SSE2-NEXT:    pand %xmm1, %xmm2
 ; SSE2-NEXT:    pandn %xmm0, %xmm1
 ; SSE2-NEXT:    por %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
 ; SSE2-NEXT:    pand %xmm2, %xmm1
@@ -851,7 +851,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; SSE4-NEXT:    pminsd %xmm1, %xmm0
 ; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE4-NEXT:    pminsd %xmm0, %xmm1
-; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE4-NEXT:    pminsd %xmm1, %xmm0
 ; SSE4-NEXT:    movd %xmm0, %eax
 ; SSE4-NEXT:    retq
@@ -862,7 +862,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX1-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
 ; AVX1-NEXT:    vzeroupper
@@ -874,7 +874,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX2-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
 ; AVX2-NEXT:    vzeroupper
@@ -886,7 +886,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX512-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
 ; AVX512-NEXT:    vzeroupper
@@ -919,7 +919,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; SSE2-NEXT:    pand %xmm2, %xmm1
 ; SSE2-NEXT:    pandn %xmm0, %xmm2
 ; SSE2-NEXT:    por %xmm1, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; SSE2-NEXT:    pcmpgtd %xmm2, %xmm1
 ; SSE2-NEXT:    pand %xmm1, %xmm2
@@ -935,7 +935,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; SSE4-NEXT:    pminsd %xmm0, %xmm1
 ; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE4-NEXT:    pminsd %xmm1, %xmm0
-; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE4-NEXT:    pminsd %xmm0, %xmm1
 ; SSE4-NEXT:    movd %xmm1, %eax
 ; SSE4-NEXT:    retq
@@ -949,7 +949,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX1-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
 ; AVX1-NEXT:    vzeroupper
@@ -962,7 +962,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX2-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
 ; AVX2-NEXT:    vzeroupper
@@ -976,7 +976,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX512-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
 ; AVX512-NEXT:    vzeroupper
@@ -1029,7 +1029,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; SSE2-NEXT:    pand %xmm2, %xmm1
 ; SSE2-NEXT:    pandn %xmm0, %xmm2
 ; SSE2-NEXT:    por %xmm1, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; SSE2-NEXT:    pcmpgtd %xmm2, %xmm1
 ; SSE2-NEXT:    pand %xmm1, %xmm2
@@ -1049,7 +1049,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; SSE4-NEXT:    pminsd %xmm0, %xmm2
 ; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE4-NEXT:    pminsd %xmm2, %xmm0
-; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE4-NEXT:    pminsd %xmm0, %xmm1
 ; SSE4-NEXT:    movd %xmm1, %eax
 ; SSE4-NEXT:    retq
@@ -1069,7 +1069,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX1-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
 ; AVX1-NEXT:    vzeroupper
@@ -1084,7 +1084,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX2-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
 ; AVX2-NEXT:    vzeroupper
@@ -1099,7 +1099,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX512-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
 ; AVX512-NEXT:    vzeroupper
@@ -1144,7 +1144,7 @@ define i16 @test_v2i16(<2 x i16> %a0) {
 define i16 @test_v4i16(<4 x i16> %a0) {
 ; SSE-LABEL: test_v4i16:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE-NEXT:    pminsw %xmm0, %xmm1
 ; SSE-NEXT:    movdqa %xmm1, %xmm0
 ; SSE-NEXT:    psrld $16, %xmm0
@@ -1155,7 +1155,7 @@ define i16 @test_v4i16(<4 x i16> %a0) {
 ;
 ; AVX-LABEL: test_v4i16:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-NEXT:    vpminsw %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX-NEXT:    vpminsw %xmm1, %xmm0, %xmm0
@@ -1165,7 +1165,7 @@ define i16 @test_v4i16(<4 x i16> %a0) {
 ;
 ; AVX512-LABEL: test_v4i16:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpminsw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpminsw %xmm1, %xmm0, %xmm0
@@ -1181,7 +1181,7 @@ define i16 @test_v8i16(<8 x i16> %a0) {
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    pminsw %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE2-NEXT:    pminsw %xmm1, %xmm0
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; SSE2-NEXT:    psrld $16, %xmm1
@@ -1226,7 +1226,7 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; SSE2-NEXT:    pminsw %xmm1, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    pminsw %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE2-NEXT:    pminsw %xmm1, %xmm0
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; SSE2-NEXT:    psrld $16, %xmm1
@@ -1292,7 +1292,7 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; SSE2-NEXT:    pminsw %xmm0, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    pminsw %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    pminsw %xmm0, %xmm1
 ; SSE2-NEXT:    movdqa %xmm1, %xmm0
 ; SSE2-NEXT:    psrld $16, %xmm0
@@ -1370,7 +1370,7 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; SSE2-NEXT:    pminsw %xmm0, %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE2-NEXT:    pminsw %xmm2, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    pminsw %xmm0, %xmm1
 ; SSE2-NEXT:    movdqa %xmm1, %xmm0
 ; SSE2-NEXT:    psrld $16, %xmm0
@@ -1554,7 +1554,7 @@ define i8 @test_v4i8(<4 x i8> %a0) {
 define i8 @test_v8i8(<8 x i8> %a0) {
 ; SSE2-LABEL: test_v8i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; SSE2-NEXT:    pcmpgtb %xmm0, %xmm2
 ; SSE2-NEXT:    pand %xmm2, %xmm0
@@ -1580,7 +1580,7 @@ define i8 @test_v8i8(<8 x i8> %a0) {
 ;
 ; SSE4-LABEL: test_v8i8:
 ; SSE4:       # %bb.0:
-; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE4-NEXT:    pminsb %xmm0, %xmm1
 ; SSE4-NEXT:    movdqa %xmm1, %xmm0
 ; SSE4-NEXT:    psrld $16, %xmm0
@@ -1594,7 +1594,7 @@ define i8 @test_v8i8(<8 x i8> %a0) {
 ;
 ; AVX-LABEL: test_v8i8:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-NEXT:    vpminsb %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX-NEXT:    vpminsb %xmm1, %xmm0, %xmm0
@@ -1606,7 +1606,7 @@ define i8 @test_v8i8(<8 x i8> %a0) {
 ;
 ; AVX512-LABEL: test_v8i8:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpminsb %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpminsb %xmm1, %xmm0, %xmm0
@@ -1628,7 +1628,7 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ; SSE2-NEXT:    pand %xmm2, %xmm0
 ; SSE2-NEXT:    pandn %xmm1, %xmm2
 ; SSE2-NEXT:    por %xmm0, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; SSE2-NEXT:    pcmpgtb %xmm2, %xmm1
 ; SSE2-NEXT:    pand %xmm1, %xmm2
@@ -1703,7 +1703,7 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; SSE2-NEXT:    pand %xmm1, %xmm2
 ; SSE2-NEXT:    pandn %xmm0, %xmm1
 ; SSE2-NEXT:    por %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; SSE2-NEXT:    pcmpgtb %xmm1, %xmm2
 ; SSE2-NEXT:    pand %xmm2, %xmm1
@@ -1809,7 +1809,7 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; SSE2-NEXT:    pand %xmm2, %xmm1
 ; SSE2-NEXT:    pandn %xmm0, %xmm2
 ; SSE2-NEXT:    por %xmm1, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; SSE2-NEXT:    pcmpgtb %xmm2, %xmm1
 ; SSE2-NEXT:    pand %xmm1, %xmm2
@@ -1943,7 +1943,7 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; SSE2-NEXT:    pand %xmm2, %xmm1
 ; SSE2-NEXT:    pandn %xmm0, %xmm2
 ; SSE2-NEXT:    por %xmm1, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; SSE2-NEXT:    pcmpgtb %xmm2, %xmm1
 ; SSE2-NEXT:    pand %xmm1, %xmm2
diff --git a/llvm/test/CodeGen/X86/vector-reduce-umax.ll b/llvm/test/CodeGen/X86/vector-reduce-umax.ll
index a11fff0a5b5c8..aedde4d11e060 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-umax.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-umax.ll
@@ -843,7 +843,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 define i32 @test_v2i32(<2 x i32> %a0) {
 ; SSE2-LABEL: test_v2i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm3
 ; SSE2-NEXT:    pxor %xmm2, %xmm3
@@ -857,21 +857,21 @@ define i32 @test_v2i32(<2 x i32> %a0) {
 ;
 ; SSE4-LABEL: test_v2i32:
 ; SSE4:       # %bb.0:
-; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE4-NEXT:    pmaxud %xmm0, %xmm1
 ; SSE4-NEXT:    movd %xmm1, %eax
 ; SSE4-NEXT:    retq
 ;
 ; AVX-LABEL: test_v2i32:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovd %xmm0, %eax
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: test_v2i32:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
 ; AVX512-NEXT:    retq
@@ -892,7 +892,7 @@ define i32 @test_v4i32(<4 x i32> %a0) {
 ; SSE2-NEXT:    pand %xmm3, %xmm0
 ; SSE2-NEXT:    pandn %xmm1, %xmm3
 ; SSE2-NEXT:    por %xmm0, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,1,1]
 ; SSE2-NEXT:    movdqa %xmm3, %xmm1
 ; SSE2-NEXT:    pxor %xmm2, %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm2
@@ -907,7 +907,7 @@ define i32 @test_v4i32(<4 x i32> %a0) {
 ; SSE4:       # %bb.0:
 ; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE4-NEXT:    pmaxud %xmm0, %xmm1
-; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE4-NEXT:    pmaxud %xmm1, %xmm0
 ; SSE4-NEXT:    movd %xmm0, %eax
 ; SSE4-NEXT:    retq
@@ -916,7 +916,7 @@ define i32 @test_v4i32(<4 x i32> %a0) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovd %xmm0, %eax
 ; AVX-NEXT:    retq
@@ -925,7 +925,7 @@ define i32 @test_v4i32(<4 x i32> %a0) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
 ; AVX512-NEXT:    retq
@@ -954,7 +954,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; SSE2-NEXT:    pand %xmm1, %xmm4
 ; SSE2-NEXT:    pandn %xmm0, %xmm1
 ; SSE2-NEXT:    por %xmm4, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE2-NEXT:    movdqa %xmm1, %xmm3
 ; SSE2-NEXT:    pxor %xmm2, %xmm3
 ; SSE2-NEXT:    pxor %xmm0, %xmm2
@@ -970,7 +970,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; SSE4-NEXT:    pmaxud %xmm1, %xmm0
 ; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE4-NEXT:    pmaxud %xmm0, %xmm1
-; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE4-NEXT:    pmaxud %xmm1, %xmm0
 ; SSE4-NEXT:    movd %xmm0, %eax
 ; SSE4-NEXT:    retq
@@ -981,7 +981,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX1-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
 ; AVX1-NEXT:    vzeroupper
@@ -993,7 +993,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX2-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
 ; AVX2-NEXT:    vzeroupper
@@ -1005,7 +1005,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX512-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
 ; AVX512-NEXT:    vzeroupper
@@ -1051,7 +1051,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; SSE2-NEXT:    pand %xmm2, %xmm1
 ; SSE2-NEXT:    pandn %xmm0, %xmm2
 ; SSE2-NEXT:    por %xmm1, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
 ; SSE2-NEXT:    movdqa %xmm2, %xmm1
 ; SSE2-NEXT:    pxor %xmm4, %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm4
@@ -1069,7 +1069,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; SSE4-NEXT:    pmaxud %xmm0, %xmm1
 ; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE4-NEXT:    pmaxud %xmm1, %xmm0
-; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE4-NEXT:    pmaxud %xmm0, %xmm1
 ; SSE4-NEXT:    movd %xmm1, %eax
 ; SSE4-NEXT:    retq
@@ -1083,7 +1083,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX1-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
 ; AVX1-NEXT:    vzeroupper
@@ -1096,7 +1096,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX2-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
 ; AVX2-NEXT:    vzeroupper
@@ -1110,7 +1110,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX512-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
 ; AVX512-NEXT:    vzeroupper
@@ -1188,7 +1188,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; SSE2-NEXT:    pand %xmm1, %xmm2
 ; SSE2-NEXT:    pandn %xmm0, %xmm1
 ; SSE2-NEXT:    por %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; SSE2-NEXT:    pxor %xmm8, %xmm2
 ; SSE2-NEXT:    pxor %xmm0, %xmm8
@@ -1210,7 +1210,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; SSE4-NEXT:    pmaxud %xmm0, %xmm2
 ; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE4-NEXT:    pmaxud %xmm2, %xmm0
-; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE4-NEXT:    pmaxud %xmm0, %xmm1
 ; SSE4-NEXT:    movd %xmm1, %eax
 ; SSE4-NEXT:    retq
@@ -1230,7 +1230,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX1-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
 ; AVX1-NEXT:    vzeroupper
@@ -1245,7 +1245,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX2-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
 ; AVX2-NEXT:    vzeroupper
@@ -1260,7 +1260,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX512-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
 ; AVX512-NEXT:    vzeroupper
@@ -1318,7 +1318,7 @@ define i16 @test_v2i16(<2 x i16> %a0) {
 define i16 @test_v4i16(<4 x i16> %a0) {
 ; SSE2-LABEL: test_v4i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [32768,32768,32768,32768,32768,32768,32768,32768]
 ; SSE2-NEXT:    pxor %xmm2, %xmm0
 ; SSE2-NEXT:    pxor %xmm2, %xmm1
@@ -1333,7 +1333,7 @@ define i16 @test_v4i16(<4 x i16> %a0) {
 ;
 ; SSE4-LABEL: test_v4i16:
 ; SSE4:       # %bb.0:
-; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE4-NEXT:    pmaxuw %xmm0, %xmm1
 ; SSE4-NEXT:    movdqa %xmm1, %xmm0
 ; SSE4-NEXT:    psrld $16, %xmm0
@@ -1344,7 +1344,7 @@ define i16 @test_v4i16(<4 x i16> %a0) {
 ;
 ; AVX-LABEL: test_v4i16:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-NEXT:    vpmaxuw %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX-NEXT:    vpmaxuw %xmm1, %xmm0, %xmm0
@@ -1354,7 +1354,7 @@ define i16 @test_v4i16(<4 x i16> %a0) {
 ;
 ; AVX512-LABEL: test_v4i16:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpmaxuw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpmaxuw %xmm1, %xmm0, %xmm0
@@ -1373,7 +1373,7 @@ define i16 @test_v8i16(<8 x i16> %a0) {
 ; SSE2-NEXT:    pxor %xmm2, %xmm0
 ; SSE2-NEXT:    pxor %xmm2, %xmm1
 ; SSE2-NEXT:    pmaxsw %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE2-NEXT:    pmaxsw %xmm1, %xmm0
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; SSE2-NEXT:    psrld $16, %xmm1
@@ -1435,7 +1435,7 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; SSE2-NEXT:    pmaxsw %xmm1, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    pmaxsw %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE2-NEXT:    pmaxsw %xmm1, %xmm0
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; SSE2-NEXT:    psrld $16, %xmm1
@@ -1522,7 +1522,7 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; SSE2-NEXT:    pmaxsw %xmm2, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    pmaxsw %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE2-NEXT:    pmaxsw %xmm1, %xmm0
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; SSE2-NEXT:    psrld $16, %xmm1
@@ -1627,7 +1627,7 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; SSE2-NEXT:    pmaxsw %xmm0, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    pmaxsw %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    pmaxsw %xmm0, %xmm1
 ; SSE2-NEXT:    movdqa %xmm1, %xmm0
 ; SSE2-NEXT:    psrld $16, %xmm0
@@ -1797,7 +1797,7 @@ define i8 @test_v4i8(<4 x i8> %a0) {
 define i8 @test_v8i8(<8 x i8> %a0) {
 ; SSE-LABEL: test_v8i8:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE-NEXT:    pmaxub %xmm0, %xmm1
 ; SSE-NEXT:    movdqa %xmm1, %xmm0
 ; SSE-NEXT:    psrld $16, %xmm0
@@ -1811,7 +1811,7 @@ define i8 @test_v8i8(<8 x i8> %a0) {
 ;
 ; AVX-LABEL: test_v8i8:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-NEXT:    vpmaxub %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX-NEXT:    vpmaxub %xmm1, %xmm0, %xmm0
@@ -1823,7 +1823,7 @@ define i8 @test_v8i8(<8 x i8> %a0) {
 ;
 ; AVX512-LABEL: test_v8i8:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpmaxub %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpmaxub %xmm1, %xmm0, %xmm0
@@ -1841,7 +1841,7 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    pmaxub %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE2-NEXT:    pmaxub %xmm1, %xmm0
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; SSE2-NEXT:    psrld $16, %xmm1
@@ -1911,7 +1911,7 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; SSE2-NEXT:    pmaxub %xmm1, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    pmaxub %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE2-NEXT:    pmaxub %xmm1, %xmm0
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; SSE2-NEXT:    psrld $16, %xmm1
@@ -2006,7 +2006,7 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; SSE2-NEXT:    pmaxub %xmm0, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    pmaxub %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    pmaxub %xmm0, %xmm1
 ; SSE2-NEXT:    movdqa %xmm1, %xmm0
 ; SSE2-NEXT:    psrld $16, %xmm0
@@ -2115,7 +2115,7 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; SSE2-NEXT:    pmaxub %xmm0, %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE2-NEXT:    pmaxub %xmm2, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    pmaxub %xmm0, %xmm1
 ; SSE2-NEXT:    movdqa %xmm1, %xmm0
 ; SSE2-NEXT:    psrld $16, %xmm0
diff --git a/llvm/test/CodeGen/X86/vector-reduce-umin.ll b/llvm/test/CodeGen/X86/vector-reduce-umin.ll
index 9da8d61223efd..c8195d2412944 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-umin.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-umin.ll
@@ -847,7 +847,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 define i32 @test_v2i32(<2 x i32> %a0) {
 ; SSE2-LABEL: test_v2i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm3
 ; SSE2-NEXT:    pxor %xmm2, %xmm3
@@ -861,21 +861,21 @@ define i32 @test_v2i32(<2 x i32> %a0) {
 ;
 ; SSE4-LABEL: test_v2i32:
 ; SSE4:       # %bb.0:
-; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE4-NEXT:    pminud %xmm0, %xmm1
 ; SSE4-NEXT:    movd %xmm1, %eax
 ; SSE4-NEXT:    retq
 ;
 ; AVX-LABEL: test_v2i32:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovd %xmm0, %eax
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: test_v2i32:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
 ; AVX512-NEXT:    retq
@@ -896,7 +896,7 @@ define i32 @test_v4i32(<4 x i32> %a0) {
 ; SSE2-NEXT:    pand %xmm4, %xmm0
 ; SSE2-NEXT:    pandn %xmm1, %xmm4
 ; SSE2-NEXT:    por %xmm0, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,1,1,1]
 ; SSE2-NEXT:    movdqa %xmm4, %xmm1
 ; SSE2-NEXT:    pxor %xmm2, %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm2
@@ -911,7 +911,7 @@ define i32 @test_v4i32(<4 x i32> %a0) {
 ; SSE4:       # %bb.0:
 ; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE4-NEXT:    pminud %xmm0, %xmm1
-; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE4-NEXT:    pminud %xmm1, %xmm0
 ; SSE4-NEXT:    movd %xmm0, %eax
 ; SSE4-NEXT:    retq
@@ -920,7 +920,7 @@ define i32 @test_v4i32(<4 x i32> %a0) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpminud %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovd %xmm0, %eax
 ; AVX-NEXT:    retq
@@ -929,7 +929,7 @@ define i32 @test_v4i32(<4 x i32> %a0) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpminud %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
 ; AVX512-NEXT:    retq
@@ -958,7 +958,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; SSE2-NEXT:    pand %xmm3, %xmm4
 ; SSE2-NEXT:    pandn %xmm0, %xmm3
 ; SSE2-NEXT:    por %xmm4, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,1,1]
 ; SSE2-NEXT:    movdqa %xmm3, %xmm1
 ; SSE2-NEXT:    pxor %xmm2, %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm2
@@ -974,7 +974,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; SSE4-NEXT:    pminud %xmm1, %xmm0
 ; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE4-NEXT:    pminud %xmm0, %xmm1
-; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE4-NEXT:    pminud %xmm1, %xmm0
 ; SSE4-NEXT:    movd %xmm0, %eax
 ; SSE4-NEXT:    retq
@@ -985,7 +985,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX1-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpminud %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
 ; AVX1-NEXT:    vzeroupper
@@ -997,7 +997,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX2-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpminud %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
 ; AVX2-NEXT:    vzeroupper
@@ -1009,7 +1009,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX512-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpminud %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
 ; AVX512-NEXT:    vzeroupper
@@ -1055,7 +1055,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; SSE2-NEXT:    pand %xmm3, %xmm1
 ; SSE2-NEXT:    pandn %xmm0, %xmm3
 ; SSE2-NEXT:    por %xmm1, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,1,1]
 ; SSE2-NEXT:    movdqa %xmm3, %xmm1
 ; SSE2-NEXT:    pxor %xmm4, %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm4
@@ -1073,7 +1073,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; SSE4-NEXT:    pminud %xmm0, %xmm1
 ; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE4-NEXT:    pminud %xmm1, %xmm0
-; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE4-NEXT:    pminud %xmm0, %xmm1
 ; SSE4-NEXT:    movd %xmm1, %eax
 ; SSE4-NEXT:    retq
@@ -1087,7 +1087,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX1-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpminud %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
 ; AVX1-NEXT:    vzeroupper
@@ -1100,7 +1100,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX2-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpminud %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
 ; AVX2-NEXT:    vzeroupper
@@ -1114,7 +1114,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX512-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpminud %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
 ; AVX512-NEXT:    vzeroupper
@@ -1192,7 +1192,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; SSE2-NEXT:    pand %xmm3, %xmm2
 ; SSE2-NEXT:    pandn %xmm0, %xmm3
 ; SSE2-NEXT:    por %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,1,1]
 ; SSE2-NEXT:    movdqa %xmm3, %xmm1
 ; SSE2-NEXT:    pxor %xmm8, %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm8
@@ -1214,7 +1214,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; SSE4-NEXT:    pminud %xmm0, %xmm2
 ; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE4-NEXT:    pminud %xmm2, %xmm0
-; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE4-NEXT:    pminud %xmm0, %xmm1
 ; SSE4-NEXT:    movd %xmm1, %eax
 ; SSE4-NEXT:    retq
@@ -1234,7 +1234,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX1-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpminud %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
 ; AVX1-NEXT:    vzeroupper
@@ -1249,7 +1249,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX2-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpminud %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
 ; AVX2-NEXT:    vzeroupper
@@ -1264,7 +1264,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX512-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpminud %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpminud %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
 ; AVX512-NEXT:    vzeroupper
@@ -1322,7 +1322,7 @@ define i16 @test_v2i16(<2 x i16> %a0) {
 define i16 @test_v4i16(<4 x i16> %a0) {
 ; SSE2-LABEL: test_v4i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [32768,32768,32768,32768,32768,32768,32768,32768]
 ; SSE2-NEXT:    pxor %xmm2, %xmm0
 ; SSE2-NEXT:    pxor %xmm2, %xmm1
@@ -1337,7 +1337,7 @@ define i16 @test_v4i16(<4 x i16> %a0) {
 ;
 ; SSE4-LABEL: test_v4i16:
 ; SSE4:       # %bb.0:
-; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE4-NEXT:    pminuw %xmm0, %xmm1
 ; SSE4-NEXT:    movdqa %xmm1, %xmm0
 ; SSE4-NEXT:    psrld $16, %xmm0
@@ -1348,7 +1348,7 @@ define i16 @test_v4i16(<4 x i16> %a0) {
 ;
 ; AVX-LABEL: test_v4i16:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-NEXT:    vpminuw %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX-NEXT:    vpminuw %xmm1, %xmm0, %xmm0
@@ -1358,7 +1358,7 @@ define i16 @test_v4i16(<4 x i16> %a0) {
 ;
 ; AVX512-LABEL: test_v4i16:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpminuw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpminuw %xmm1, %xmm0, %xmm0
@@ -1377,7 +1377,7 @@ define i16 @test_v8i16(<8 x i16> %a0) {
 ; SSE2-NEXT:    pxor %xmm2, %xmm0
 ; SSE2-NEXT:    pxor %xmm2, %xmm1
 ; SSE2-NEXT:    pminsw %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE2-NEXT:    pminsw %xmm1, %xmm0
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; SSE2-NEXT:    psrld $16, %xmm1
@@ -1420,7 +1420,7 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; SSE2-NEXT:    pminsw %xmm1, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    pminsw %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE2-NEXT:    pminsw %xmm1, %xmm0
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; SSE2-NEXT:    psrld $16, %xmm1
@@ -1484,7 +1484,7 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; SSE2-NEXT:    pminsw %xmm2, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    pminsw %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE2-NEXT:    pminsw %xmm1, %xmm0
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; SSE2-NEXT:    psrld $16, %xmm1
@@ -1564,7 +1564,7 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; SSE2-NEXT:    pminsw %xmm0, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    pminsw %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    pminsw %xmm0, %xmm1
 ; SSE2-NEXT:    movdqa %xmm1, %xmm0
 ; SSE2-NEXT:    psrld $16, %xmm0
@@ -1708,7 +1708,7 @@ define i8 @test_v4i8(<4 x i8> %a0) {
 define i8 @test_v8i8(<8 x i8> %a0) {
 ; SSE-LABEL: test_v8i8:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE-NEXT:    pminub %xmm0, %xmm1
 ; SSE-NEXT:    movdqa %xmm1, %xmm0
 ; SSE-NEXT:    psrld $16, %xmm0
@@ -1722,7 +1722,7 @@ define i8 @test_v8i8(<8 x i8> %a0) {
 ;
 ; AVX-LABEL: test_v8i8:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX-NEXT:    vpminub %xmm1, %xmm0, %xmm0
@@ -1734,7 +1734,7 @@ define i8 @test_v8i8(<8 x i8> %a0) {
 ;
 ; AVX512-LABEL: test_v8i8:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpminub %xmm1, %xmm0, %xmm0
@@ -1752,7 +1752,7 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    pminub %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE2-NEXT:    pminub %xmm1, %xmm0
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; SSE2-NEXT:    psrld $16, %xmm1
@@ -1801,7 +1801,7 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; SSE2-NEXT:    pminub %xmm1, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    pminub %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE2-NEXT:    pminub %xmm1, %xmm0
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; SSE2-NEXT:    psrld $16, %xmm1
@@ -1871,7 +1871,7 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; SSE2-NEXT:    pminub %xmm0, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE2-NEXT:    pminub %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    pminub %xmm0, %xmm1
 ; SSE2-NEXT:    movdqa %xmm1, %xmm0
 ; SSE2-NEXT:    psrld $16, %xmm0
@@ -1953,7 +1953,7 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; SSE2-NEXT:    pminub %xmm0, %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE2-NEXT:    pminub %xmm2, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    pminub %xmm0, %xmm1
 ; SSE2-NEXT:    movdqa %xmm1, %xmm0
 ; SSE2-NEXT:    psrld $16, %xmm0
diff --git a/llvm/test/CodeGen/X86/vector-reduce-xor.ll b/llvm/test/CodeGen/X86/vector-reduce-xor.ll
index 2d69190d9d18a..3c09e25341ad3 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-xor.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-xor.ll
@@ -182,14 +182,14 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 define i32 @test_v2i32(<2 x i32> %a0) {
 ; SSE-LABEL: test_v2i32:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE-NEXT:    pxor %xmm0, %xmm1
 ; SSE-NEXT:    movd %xmm1, %eax
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: test_v2i32:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovd %xmm0, %eax
 ; AVX-NEXT:    retq
@@ -202,7 +202,7 @@ define i32 @test_v4i32(<4 x i32> %a0) {
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    pxor %xmm0, %xmm1
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE-NEXT:    pxor %xmm1, %xmm0
 ; SSE-NEXT:    movd %xmm0, %eax
 ; SSE-NEXT:    retq
@@ -211,7 +211,7 @@ define i32 @test_v4i32(<4 x i32> %a0) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovd %xmm0, %eax
 ; AVX-NEXT:    retq
@@ -225,7 +225,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; SSE-NEXT:    pxor %xmm1, %xmm0
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    pxor %xmm0, %xmm1
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE-NEXT:    pxor %xmm1, %xmm0
 ; SSE-NEXT:    movd %xmm0, %eax
 ; SSE-NEXT:    retq
@@ -236,7 +236,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
 ; AVX1-NEXT:    vzeroupper
@@ -248,7 +248,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
 ; AVX2-NEXT:    vzeroupper
@@ -260,7 +260,7 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
 ; AVX512-NEXT:    vzeroupper
@@ -277,7 +277,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; SSE-NEXT:    pxor %xmm0, %xmm1
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE-NEXT:    pxor %xmm1, %xmm0
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE-NEXT:    pxor %xmm0, %xmm1
 ; SSE-NEXT:    movd %xmm1, %eax
 ; SSE-NEXT:    retq
@@ -289,7 +289,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
 ; AVX1-NEXT:    vzeroupper
@@ -302,7 +302,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
 ; AVX2-NEXT:    vzeroupper
@@ -316,7 +316,7 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
 ; AVX512-NEXT:    vzeroupper
@@ -337,7 +337,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; SSE-NEXT:    pxor %xmm0, %xmm2
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE-NEXT:    pxor %xmm2, %xmm0
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE-NEXT:    pxor %xmm0, %xmm1
 ; SSE-NEXT:    movd %xmm1, %eax
 ; SSE-NEXT:    retq
@@ -351,7 +351,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
 ; AVX1-NEXT:    vzeroupper
@@ -366,7 +366,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
 ; AVX2-NEXT:    vzeroupper
@@ -381,7 +381,7 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
 ; AVX512-NEXT:    vzeroupper
@@ -418,7 +418,7 @@ define i16 @test_v2i16(<2 x i16> %a0) {
 define i16 @test_v4i16(<4 x i16> %a0) {
 ; SSE-LABEL: test_v4i16:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE-NEXT:    pxor %xmm0, %xmm1
 ; SSE-NEXT:    movdqa %xmm1, %xmm0
 ; SSE-NEXT:    psrld $16, %xmm0
@@ -429,7 +429,7 @@ define i16 @test_v4i16(<4 x i16> %a0) {
 ;
 ; AVX-LABEL: test_v4i16:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
@@ -445,7 +445,7 @@ define i16 @test_v8i16(<8 x i16> %a0) {
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    pxor %xmm0, %xmm1
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE-NEXT:    pxor %xmm1, %xmm0
 ; SSE-NEXT:    movdqa %xmm0, %xmm1
 ; SSE-NEXT:    psrld $16, %xmm1
@@ -458,7 +458,7 @@ define i16 @test_v8i16(<8 x i16> %a0) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
@@ -475,7 +475,7 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; SSE-NEXT:    pxor %xmm1, %xmm0
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    pxor %xmm0, %xmm1
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE-NEXT:    pxor %xmm1, %xmm0
 ; SSE-NEXT:    movdqa %xmm0, %xmm1
 ; SSE-NEXT:    psrld $16, %xmm1
@@ -490,7 +490,7 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
@@ -505,7 +505,7 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
@@ -520,7 +520,7 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
@@ -540,7 +540,7 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; SSE-NEXT:    pxor %xmm0, %xmm1
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE-NEXT:    pxor %xmm1, %xmm0
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE-NEXT:    pxor %xmm0, %xmm1
 ; SSE-NEXT:    movdqa %xmm1, %xmm0
 ; SSE-NEXT:    psrld $16, %xmm0
@@ -556,7 +556,7 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
@@ -572,7 +572,7 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
@@ -589,7 +589,7 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
@@ -613,7 +613,7 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; SSE-NEXT:    pxor %xmm0, %xmm2
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE-NEXT:    pxor %xmm2, %xmm0
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE-NEXT:    pxor %xmm0, %xmm1
 ; SSE-NEXT:    movdqa %xmm1, %xmm0
 ; SSE-NEXT:    psrld $16, %xmm0
@@ -631,7 +631,7 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
@@ -649,7 +649,7 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
@@ -667,7 +667,7 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
@@ -733,7 +733,7 @@ define i8 @test_v4i8(<4 x i8> %a0) {
 define i8 @test_v8i8(<8 x i8> %a0) {
 ; SSE-LABEL: test_v8i8:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE-NEXT:    pxor %xmm0, %xmm1
 ; SSE-NEXT:    movdqa %xmm1, %xmm0
 ; SSE-NEXT:    psrld $16, %xmm0
@@ -747,7 +747,7 @@ define i8 @test_v8i8(<8 x i8> %a0) {
 ;
 ; AVX-LABEL: test_v8i8:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
@@ -765,7 +765,7 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    pxor %xmm0, %xmm1
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE-NEXT:    pxor %xmm1, %xmm0
 ; SSE-NEXT:    movdqa %xmm0, %xmm1
 ; SSE-NEXT:    psrld $16, %xmm1
@@ -781,7 +781,7 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
@@ -800,7 +800,7 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; SSE-NEXT:    pxor %xmm1, %xmm0
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    pxor %xmm0, %xmm1
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE-NEXT:    pxor %xmm1, %xmm0
 ; SSE-NEXT:    movdqa %xmm0, %xmm1
 ; SSE-NEXT:    psrld $16, %xmm1
@@ -818,7 +818,7 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
@@ -835,7 +835,7 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
@@ -852,7 +852,7 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
@@ -874,7 +874,7 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; SSE-NEXT:    pxor %xmm0, %xmm1
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
 ; SSE-NEXT:    pxor %xmm1, %xmm0
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE-NEXT:    pxor %xmm0, %xmm1
 ; SSE-NEXT:    movdqa %xmm1, %xmm0
 ; SSE-NEXT:    psrld $16, %xmm0
@@ -893,7 +893,7 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
@@ -911,7 +911,7 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
@@ -930,7 +930,7 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
@@ -956,7 +956,7 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; SSE-NEXT:    pxor %xmm0, %xmm2
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; SSE-NEXT:    pxor %xmm2, %xmm0
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE-NEXT:    pxor %xmm0, %xmm1
 ; SSE-NEXT:    movdqa %xmm1, %xmm0
 ; SSE-NEXT:    psrld $16, %xmm0
@@ -977,7 +977,7 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
@@ -997,7 +997,7 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
@@ -1017,7 +1017,7 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
diff --git a/llvm/test/CodeGen/X86/vector-rem.ll b/llvm/test/CodeGen/X86/vector-rem.ll
index deaab1c9161b9..22e060719c0ab 100644
--- a/llvm/test/CodeGen/X86/vector-rem.ll
+++ b/llvm/test/CodeGen/X86/vector-rem.ll
@@ -4,9 +4,9 @@
 define <4 x i32> @foo(<4 x i32> %t, <4 x i32> %u) nounwind {
 ; CHECK-LABEL: foo:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,1,2,3]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
 ; CHECK-NEXT:    movd %xmm2, %eax
-; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,1,2,3]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
 ; CHECK-NEXT:    movd %xmm2, %ecx
 ; CHECK-NEXT:    cltd
 ; CHECK-NEXT:    idivl %ecx
@@ -24,9 +24,9 @@ define <4 x i32> @foo(<4 x i32> %t, <4 x i32> %u) nounwind {
 ; CHECK-NEXT:    cltd
 ; CHECK-NEXT:    idivl %ecx
 ; CHECK-NEXT:    movd %edx, %xmm2
-; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; CHECK-NEXT:    movd %xmm0, %eax
-; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; CHECK-NEXT:    movd %xmm0, %ecx
 ; CHECK-NEXT:    cltd
 ; CHECK-NEXT:    idivl %ecx
@@ -42,9 +42,9 @@ define <4 x i32> @foo(<4 x i32> %t, <4 x i32> %u) nounwind {
 define <4 x i32> @bar(<4 x i32> %t, <4 x i32> %u) nounwind {
 ; CHECK-LABEL: bar:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,1,2,3]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
 ; CHECK-NEXT:    movd %xmm2, %eax
-; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,1,2,3]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
 ; CHECK-NEXT:    movd %xmm2, %ecx
 ; CHECK-NEXT:    xorl %edx, %edx
 ; CHECK-NEXT:    divl %ecx
@@ -62,9 +62,9 @@ define <4 x i32> @bar(<4 x i32> %t, <4 x i32> %u) nounwind {
 ; CHECK-NEXT:    xorl %edx, %edx
 ; CHECK-NEXT:    divl %ecx
 ; CHECK-NEXT:    movd %edx, %xmm2
-; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; CHECK-NEXT:    movd %xmm0, %eax
-; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; CHECK-NEXT:    movd %xmm0, %ecx
 ; CHECK-NEXT:    xorl %edx, %edx
 ; CHECK-NEXT:    divl %ecx
@@ -83,8 +83,8 @@ define <4 x float> @qux(<4 x float> %t, <4 x float> %u) nounwind {
 ; CHECK-NEXT:    subq $72, %rsp
 ; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; CHECK-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
-; CHECK-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; CHECK-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; CHECK-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; CHECK-NEXT:    callq fmodf
 ; CHECK-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
 ; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
@@ -100,9 +100,9 @@ define <4 x float> @qux(<4 x float> %t, <4 x float> %u) nounwind {
 ; CHECK-NEXT:    callq fmodf
 ; CHECK-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; CHECK-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; CHECK-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; CHECK-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; CHECK-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,1,1]
 ; CHECK-NEXT:    callq fmodf
 ; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
 ; CHECK-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
diff --git a/llvm/test/CodeGen/X86/vector-rotate-128.ll b/llvm/test/CodeGen/X86/vector-rotate-128.ll
index 6f11c5cbd7567..066a4db44e537 100644
--- a/llvm/test/CodeGen/X86/vector-rotate-128.ll
+++ b/llvm/test/CodeGen/X86/vector-rotate-128.ll
@@ -903,7 +903,7 @@ define <8 x i16> @splatvar_rotate_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
 ;
 ; XOPAVX1-LABEL: splatvar_rotate_v8i16:
 ; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,0,2,3,4,5,6,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,0,0,0,4,5,6,7]
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; XOPAVX1-NEXT:    vprotw %xmm1, %xmm0, %xmm0
 ; XOPAVX1-NEXT:    retq
@@ -951,7 +951,7 @@ define <16 x i8> @splatvar_rotate_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
 ; SSE2-NEXT:    pcmpeqd %xmm5, %xmm5
 ; SSE2-NEXT:    psllw %xmm3, %xmm5
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm5[0,0,2,3,4,5,6,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm5[0,0,0,0,4,5,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
 ; SSE2-NEXT:    pand %xmm3, %xmm1
 ; SSE2-NEXT:    pslldq {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm2[0]
@@ -960,7 +960,7 @@ define <16 x i8> @splatvar_rotate_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
 ; SSE2-NEXT:    psrlw %xmm2, %xmm4
 ; SSE2-NEXT:    psrlw $8, %xmm4
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm4[0,0,2,3,4,5,6,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm4[0,0,0,0,4,5,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
 ; SSE2-NEXT:    pand %xmm0, %xmm2
 ; SSE2-NEXT:    por %xmm2, %xmm1
@@ -1114,7 +1114,7 @@ define <16 x i8> @splatvar_rotate_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
 ; X32-SSE-NEXT:    pcmpeqd %xmm5, %xmm5
 ; X32-SSE-NEXT:    psllw %xmm3, %xmm5
 ; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm3 = xmm5[0,0,2,3,4,5,6,7]
+; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm3 = xmm5[0,0,0,0,4,5,6,7]
 ; X32-SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
 ; X32-SSE-NEXT:    pand %xmm3, %xmm1
 ; X32-SSE-NEXT:    pslldq {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm2[0]
@@ -1123,7 +1123,7 @@ define <16 x i8> @splatvar_rotate_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
 ; X32-SSE-NEXT:    psrlw %xmm2, %xmm4
 ; X32-SSE-NEXT:    psrlw $8, %xmm4
 ; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm4[0,0,2,3,4,5,6,7]
+; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm4[0,0,0,0,4,5,6,7]
 ; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
 ; X32-SSE-NEXT:    pand %xmm0, %xmm2
 ; X32-SSE-NEXT:    por %xmm2, %xmm1
diff --git a/llvm/test/CodeGen/X86/vector-rotate-256.ll b/llvm/test/CodeGen/X86/vector-rotate-256.ll
index f54aea8d86ddf..9dcbe43c0864b 100644
--- a/llvm/test/CodeGen/X86/vector-rotate-256.ll
+++ b/llvm/test/CodeGen/X86/vector-rotate-256.ll
@@ -669,7 +669,7 @@ define <8 x i32> @splatvar_rotate_v8i32(<8 x i32> %a, <8 x i32> %b) nounwind {
 define <16 x i16> @splatvar_rotate_v16i16(<16 x i16> %a, <16 x i16> %b) nounwind {
 ; AVX1-LABEL: splatvar_rotate_v16i16:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
 ; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
@@ -714,7 +714,7 @@ define <16 x i16> @splatvar_rotate_v16i16(<16 x i16> %a, <16 x i16> %b) nounwind
 ;
 ; XOPAVX1-LABEL: splatvar_rotate_v16i16:
 ; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,0,2,3,4,5,6,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,0,0,0,4,5,6,7]
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
 ; XOPAVX1-NEXT:    vprotw %xmm1, %xmm2, %xmm2
diff --git a/llvm/test/CodeGen/X86/vector-sext.ll b/llvm/test/CodeGen/X86/vector-sext.ll
index 2a4efccc07c7f..5087de2dc829a 100644
--- a/llvm/test/CodeGen/X86/vector-sext.ll
+++ b/llvm/test/CodeGen/X86/vector-sext.ll
@@ -286,7 +286,7 @@ define <8 x i32> @sext_16i8_to_8i32(<16 x i8> %A) nounwind uwtable readnone ssp
 ; SSE41-LABEL: sext_16i8_to_8i32:
 ; SSE41:       # %bb.0: # %entry
 ; SSE41-NEXT:    pmovsxbd %xmm0, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE41-NEXT:    pmovsxbd %xmm0, %xmm1
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; SSE41-NEXT:    retq
@@ -294,7 +294,7 @@ define <8 x i32> @sext_16i8_to_8i32(<16 x i8> %A) nounwind uwtable readnone ssp
 ; AVX1-LABEL: sext_16i8_to_8i32:
 ; AVX1:       # %bb.0: # %entry
 ; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
@@ -321,7 +321,7 @@ define <8 x i32> @sext_16i8_to_8i32(<16 x i8> %A) nounwind uwtable readnone ssp
 ; X32-SSE41-LABEL: sext_16i8_to_8i32:
 ; X32-SSE41:       # %bb.0: # %entry
 ; X32-SSE41-NEXT:    pmovsxbd %xmm0, %xmm2
-; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; X32-SSE41-NEXT:    pmovsxbd %xmm0, %xmm1
 ; X32-SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; X32-SSE41-NEXT:    retl
@@ -365,11 +365,11 @@ define <16 x i32> @sext_16i8_to_16i32(<16 x i8> %A) nounwind uwtable readnone ss
 ; SSE41-LABEL: sext_16i8_to_16i32:
 ; SSE41:       # %bb.0: # %entry
 ; SSE41-NEXT:    pmovsxbd %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE41-NEXT:    pmovsxbd %xmm1, %xmm1
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovsxbd %xmm2, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE41-NEXT:    pmovsxbd %xmm0, %xmm3
 ; SSE41-NEXT:    movdqa %xmm4, %xmm0
 ; SSE41-NEXT:    retq
@@ -377,12 +377,12 @@ define <16 x i32> @sext_16i8_to_16i32(<16 x i8> %A) nounwind uwtable readnone ss
 ; AVX1-LABEL: sext_16i8_to_16i32:
 ; AVX1:       # %bb.0: # %entry
 ; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm2
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm2
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,3,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
 ; AVX1-NEXT:    vmovaps %ymm2, %ymm0
@@ -419,11 +419,11 @@ define <16 x i32> @sext_16i8_to_16i32(<16 x i8> %A) nounwind uwtable readnone ss
 ; X32-SSE41-LABEL: sext_16i8_to_16i32:
 ; X32-SSE41:       # %bb.0: # %entry
 ; X32-SSE41-NEXT:    pmovsxbd %xmm0, %xmm4
-; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X32-SSE41-NEXT:    pmovsxbd %xmm1, %xmm1
 ; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; X32-SSE41-NEXT:    pmovsxbd %xmm2, %xmm2
-; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; X32-SSE41-NEXT:    pmovsxbd %xmm0, %xmm3
 ; X32-SSE41-NEXT:    movdqa %xmm4, %xmm0
 ; X32-SSE41-NEXT:    retl
@@ -604,7 +604,7 @@ define <8 x i64> @sext_16i8_to_8i64(<16 x i8> %A) nounwind uwtable readnone ssp
 ; SSE41-NEXT:    movdqa %xmm0, %xmm1
 ; SSE41-NEXT:    psrld $16, %xmm1
 ; SSE41-NEXT:    pmovsxbq %xmm1, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,1,1]
 ; SSE41-NEXT:    pmovsxbq %xmm2, %xmm2
 ; SSE41-NEXT:    psrlq $48, %xmm0
 ; SSE41-NEXT:    pmovsxbq %xmm0, %xmm3
@@ -617,7 +617,7 @@ define <8 x i64> @sext_16i8_to_8i64(<16 x i8> %A) nounwind uwtable readnone ssp
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm2
 ; AVX1-NEXT:    vpmovsxbq %xmm2, %xmm2
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmovsxbq %xmm0, %xmm1
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm0
 ; AVX1-NEXT:    vpmovsxbq %xmm0, %xmm0
@@ -628,7 +628,7 @@ define <8 x i64> @sext_16i8_to_8i64(<16 x i8> %A) nounwind uwtable readnone ssp
 ; AVX2-LABEL: sext_16i8_to_8i64:
 ; AVX2:       # %bb.0: # %entry
 ; AVX2-NEXT:    vpmovsxbq %xmm0, %ymm2
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpmovsxbq %xmm0, %ymm1
 ; AVX2-NEXT:    vmovdqa %ymm2, %ymm0
 ; AVX2-NEXT:    retq
@@ -663,7 +663,7 @@ define <8 x i64> @sext_16i8_to_8i64(<16 x i8> %A) nounwind uwtable readnone ssp
 ; X32-SSE41-NEXT:    movdqa %xmm0, %xmm1
 ; X32-SSE41-NEXT:    psrld $16, %xmm1
 ; X32-SSE41-NEXT:    pmovsxbq %xmm1, %xmm1
-; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
+; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,1,1]
 ; X32-SSE41-NEXT:    pmovsxbq %xmm2, %xmm2
 ; X32-SSE41-NEXT:    psrlq $48, %xmm0
 ; X32-SSE41-NEXT:    pmovsxbq %xmm0, %xmm3
@@ -952,7 +952,7 @@ define <4 x i64> @sext_8i16_to_4i64(<8 x i16> %A) nounwind uwtable readnone ssp
 ; SSE41-LABEL: sext_8i16_to_4i64:
 ; SSE41:       # %bb.0: # %entry
 ; SSE41-NEXT:    pmovsxwq %xmm0, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE41-NEXT:    pmovsxwq %xmm0, %xmm1
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; SSE41-NEXT:    retq
@@ -960,7 +960,7 @@ define <4 x i64> @sext_8i16_to_4i64(<8 x i16> %A) nounwind uwtable readnone ssp
 ; AVX1-LABEL: sext_8i16_to_4i64:
 ; AVX1:       # %bb.0: # %entry
 ; AVX1-NEXT:    vpmovsxwq %xmm0, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmovsxwq %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
@@ -989,7 +989,7 @@ define <4 x i64> @sext_8i16_to_4i64(<8 x i16> %A) nounwind uwtable readnone ssp
 ; X32-SSE41-LABEL: sext_8i16_to_4i64:
 ; X32-SSE41:       # %bb.0: # %entry
 ; X32-SSE41-NEXT:    pmovsxwq %xmm0, %xmm2
-; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; X32-SSE41-NEXT:    pmovsxwq %xmm0, %xmm1
 ; X32-SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; X32-SSE41-NEXT:    retl
@@ -1041,11 +1041,11 @@ define <8 x i64> @sext_8i16_to_8i64(<8 x i16> %A) nounwind uwtable readnone ssp
 ; SSE41-LABEL: sext_8i16_to_8i64:
 ; SSE41:       # %bb.0: # %entry
 ; SSE41-NEXT:    pmovsxwq %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE41-NEXT:    pmovsxwq %xmm1, %xmm1
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovsxwq %xmm2, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE41-NEXT:    pmovsxwq %xmm0, %xmm3
 ; SSE41-NEXT:    movdqa %xmm4, %xmm0
 ; SSE41-NEXT:    retq
@@ -1053,12 +1053,12 @@ define <8 x i64> @sext_8i16_to_8i64(<8 x i16> %A) nounwind uwtable readnone ssp
 ; AVX1-LABEL: sext_8i16_to_8i64:
 ; AVX1:       # %bb.0: # %entry
 ; AVX1-NEXT:    vpmovsxwq %xmm0, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmovsxwq %xmm2, %xmm2
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm2
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxwq %xmm1, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,3,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX1-NEXT:    vpmovsxwq %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
 ; AVX1-NEXT:    vmovaps %ymm2, %ymm0
@@ -1099,11 +1099,11 @@ define <8 x i64> @sext_8i16_to_8i64(<8 x i16> %A) nounwind uwtable readnone ssp
 ; X32-SSE41-LABEL: sext_8i16_to_8i64:
 ; X32-SSE41:       # %bb.0: # %entry
 ; X32-SSE41-NEXT:    pmovsxwq %xmm0, %xmm4
-; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; X32-SSE41-NEXT:    pmovsxwq %xmm1, %xmm1
 ; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; X32-SSE41-NEXT:    pmovsxwq %xmm2, %xmm2
-; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; X32-SSE41-NEXT:    pmovsxwq %xmm0, %xmm3
 ; X32-SSE41-NEXT:    movdqa %xmm4, %xmm0
 ; X32-SSE41-NEXT:    retl
@@ -2106,7 +2106,7 @@ define <8 x i16> @load_sext_8i1_to_8i16(<8 x i1> *%ptr) {
 ; SSE-LABEL: load_sext_8i1_to_8i16:
 ; SSE:       # %bb.0: # %entry
 ; SSE-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; SSE-NEXT:    movdqa {{.*#+}} xmm1 = [1,2,4,8,16,32,64,128]
 ; SSE-NEXT:    pand %xmm1, %xmm0
@@ -2116,7 +2116,7 @@ define <8 x i16> @load_sext_8i1_to_8i16(<8 x i1> *%ptr) {
 ; AVX1-LABEL: load_sext_8i1_to_8i16:
 ; AVX1:       # %bb.0: # %entry
 ; AVX1-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [1,2,4,8,16,32,64,128]
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
@@ -2152,7 +2152,7 @@ define <8 x i16> @load_sext_8i1_to_8i16(<8 x i1> *%ptr) {
 ; X32-SSE:       # %bb.0: # %entry
 ; X32-SSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-SSE-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; X32-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; X32-SSE-NEXT:    movdqa {{.*#+}} xmm1 = [1,2,4,8,16,32,64,128]
 ; X32-SSE-NEXT:    pand %xmm1, %xmm0
@@ -2540,7 +2540,7 @@ define <16 x i16> @load_sext_16i1_to_16i16(<16 x i1> *%ptr) {
 ; SSE-LABEL: load_sext_16i1_to_16i16:
 ; SSE:       # %bb.0: # %entry
 ; SSE-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,0,0]
 ; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [1,2,4,8,16,32,64,128]
 ; SSE-NEXT:    movdqa %xmm1, %xmm0
@@ -2554,7 +2554,7 @@ define <16 x i16> @load_sext_16i1_to_16i16(<16 x i1> *%ptr) {
 ; AVX1-LABEL: load_sext_16i1_to_16i16:
 ; AVX1:       # %bb.0: # %entry
 ; AVX1-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
@@ -2594,7 +2594,7 @@ define <16 x i16> @load_sext_16i1_to_16i16(<16 x i1> *%ptr) {
 ; X32-SSE:       # %bb.0: # %entry
 ; X32-SSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-SSE-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,0,0]
 ; X32-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [1,2,4,8,16,32,64,128]
 ; X32-SSE-NEXT:    movdqa %xmm1, %xmm0
@@ -3631,11 +3631,11 @@ define <8 x i64> @sext_8i6_to_8i64(i32 %x) nounwind uwtable readnone ssp {
 ; SSE2-LABEL: sext_8i6_to_8i64:
 ; SSE2:       # %bb.0: # %entry
 ; SSE2-NEXT:    movd %edi, %xmm0
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[0,0,0,0]
 ; SSE2-NEXT:    paddw {{.*}}(%rip), %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,1,0,3]
-; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,0,0]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,5,5]
 ; SSE2-NEXT:    psllq $58, %xmm0
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; SSE2-NEXT:    psrad $31, %xmm1
@@ -3643,8 +3643,8 @@ define <8 x i64> @sext_8i6_to_8i64(i32 %x) nounwind uwtable readnone ssp {
 ; SSE2-NEXT:    psrad $26, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,1,3]
-; SSE2-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,5,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,1,1]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,5,5,5]
 ; SSE2-NEXT:    psllq $58, %xmm1
 ; SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; SSE2-NEXT:    psrad $31, %xmm2
@@ -3652,8 +3652,8 @@ define <8 x i64> @sext_8i6_to_8i64(i32 %x) nounwind uwtable readnone ssp {
 ; SSE2-NEXT:    psrad $26, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,1,2,3]
-; SSE2-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,5,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,2,2,2]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,5,5,5]
 ; SSE2-NEXT:    psllq $58, %xmm2
 ; SSE2-NEXT:    movdqa %xmm2, %xmm4
 ; SSE2-NEXT:    psrad $31, %xmm4
@@ -3661,8 +3661,8 @@ define <8 x i64> @sext_8i6_to_8i64(i32 %x) nounwind uwtable readnone ssp {
 ; SSE2-NEXT:    psrad $26, %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[3,1,3,3]
-; SSE2-NEXT:    pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,5,5,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,5,5,5,5]
 ; SSE2-NEXT:    psllq $58, %xmm3
 ; SSE2-NEXT:    movdqa %xmm3, %xmm4
 ; SSE2-NEXT:    psrad $31, %xmm4
@@ -3675,11 +3675,11 @@ define <8 x i64> @sext_8i6_to_8i64(i32 %x) nounwind uwtable readnone ssp {
 ; SSSE3-LABEL: sext_8i6_to_8i64:
 ; SSSE3:       # %bb.0: # %entry
 ; SSSE3-NEXT:    movd %edi, %xmm0
-; SSSE3-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; SSSE3-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[0,0,0,0]
 ; SSSE3-NEXT:    paddw {{.*}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,1,0,3]
-; SSSE3-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,6,7]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,0,0]
+; SSSE3-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,5,5]
 ; SSSE3-NEXT:    psllq $58, %xmm0
 ; SSSE3-NEXT:    movdqa %xmm0, %xmm1
 ; SSSE3-NEXT:    psrad $31, %xmm1
@@ -3687,8 +3687,8 @@ define <8 x i64> @sext_8i6_to_8i64(i32 %x) nounwind uwtable readnone ssp {
 ; SSSE3-NEXT:    psrad $26, %xmm0
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
 ; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,1,3]
-; SSSE3-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,5,6,7]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,1,1]
+; SSSE3-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,5,5,5]
 ; SSSE3-NEXT:    psllq $58, %xmm1
 ; SSSE3-NEXT:    movdqa %xmm1, %xmm2
 ; SSSE3-NEXT:    psrad $31, %xmm2
@@ -3696,8 +3696,8 @@ define <8 x i64> @sext_8i6_to_8i64(i32 %x) nounwind uwtable readnone ssp {
 ; SSSE3-NEXT:    psrad $26, %xmm1
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
 ; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,1,2,3]
-; SSSE3-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,5,6,7]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,2,2,2]
+; SSSE3-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,5,5,5]
 ; SSSE3-NEXT:    psllq $58, %xmm2
 ; SSSE3-NEXT:    movdqa %xmm2, %xmm4
 ; SSSE3-NEXT:    psrad $31, %xmm4
@@ -3705,8 +3705,8 @@ define <8 x i64> @sext_8i6_to_8i64(i32 %x) nounwind uwtable readnone ssp {
 ; SSSE3-NEXT:    psrad $26, %xmm2
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
 ; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[3,1,3,3]
-; SSSE3-NEXT:    pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,5,5,6,7]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
+; SSSE3-NEXT:    pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,5,5,5,5]
 ; SSSE3-NEXT:    psllq $58, %xmm3
 ; SSSE3-NEXT:    movdqa %xmm3, %xmm4
 ; SSSE3-NEXT:    psrad $31, %xmm4
@@ -3719,7 +3719,7 @@ define <8 x i64> @sext_8i6_to_8i64(i32 %x) nounwind uwtable readnone ssp {
 ; SSE41-LABEL: sext_8i6_to_8i64:
 ; SSE41:       # %bb.0: # %entry
 ; SSE41-NEXT:    movd %edi, %xmm0
-; SSE41-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; SSE41-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[0,0,0,0]
 ; SSE41-NEXT:    paddw {{.*}}(%rip), %xmm3
 ; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm0 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero
@@ -3729,7 +3729,7 @@ define <8 x i64> @sext_8i6_to_8i64(i32 %x) nounwind uwtable readnone ssp {
 ; SSE41-NEXT:    psrad $26, %xmm0
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
 ; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,1,1]
 ; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
 ; SSE41-NEXT:    psllq $58, %xmm1
 ; SSE41-NEXT:    movdqa %xmm1, %xmm2
@@ -3745,7 +3745,7 @@ define <8 x i64> @sext_8i6_to_8i64(i32 %x) nounwind uwtable readnone ssp {
 ; SSE41-NEXT:    psrad $26, %xmm2
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
 ; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm4[2,3],xmm2[4,5],xmm4[6,7]
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[3,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
 ; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero
 ; SSE41-NEXT:    psllq $58, %xmm3
 ; SSE41-NEXT:    movdqa %xmm3, %xmm4
@@ -3758,18 +3758,18 @@ define <8 x i64> @sext_8i6_to_8i64(i32 %x) nounwind uwtable readnone ssp {
 ; AVX1-LABEL: sext_8i6_to_8i64:
 ; AVX1:       # %bb.0: # %entry
 ; AVX1-NEXT:    vmovd %edi, %xmm0
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    vpaddw {{.*}}(%rip), %xmm0, %xmm0
 ; AVX1-NEXT:    vpsllw $10, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsraw $10, %xmm0, %xmm1
 ; AVX1-NEXT:    vpmovsxwq %xmm1, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
 ; AVX1-NEXT:    vpmovsxwq %xmm2, %xmm2
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxwq %xmm2, %xmm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; AVX1-NEXT:    vpmovsxwq %xmm1, %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
 ; AVX1-NEXT:    retq
@@ -3799,11 +3799,11 @@ define <8 x i64> @sext_8i6_to_8i64(i32 %x) nounwind uwtable readnone ssp {
 ; X32-SSE2-LABEL: sext_8i6_to_8i64:
 ; X32-SSE2:       # %bb.0: # %entry
 ; X32-SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X32-SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; X32-SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[0,0,0,0]
 ; X32-SSE2-NEXT:    paddw {{\.LCPI.*}}, %xmm3
-; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,1,0,3]
-; X32-SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,6,7]
+; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,0,0]
+; X32-SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,5,5]
 ; X32-SSE2-NEXT:    psllq $58, %xmm0
 ; X32-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X32-SSE2-NEXT:    psrad $31, %xmm1
@@ -3811,8 +3811,8 @@ define <8 x i64> @sext_8i6_to_8i64(i32 %x) nounwind uwtable readnone ssp {
 ; X32-SSE2-NEXT:    psrad $26, %xmm0
 ; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
 ; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,1,3]
-; X32-SSE2-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,5,6,7]
+; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,1,1]
+; X32-SSE2-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,5,5,5]
 ; X32-SSE2-NEXT:    psllq $58, %xmm1
 ; X32-SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; X32-SSE2-NEXT:    psrad $31, %xmm2
@@ -3820,8 +3820,8 @@ define <8 x i64> @sext_8i6_to_8i64(i32 %x) nounwind uwtable readnone ssp {
 ; X32-SSE2-NEXT:    psrad $26, %xmm1
 ; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
 ; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,1,2,3]
-; X32-SSE2-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,5,6,7]
+; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,2,2,2]
+; X32-SSE2-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,5,5,5]
 ; X32-SSE2-NEXT:    psllq $58, %xmm2
 ; X32-SSE2-NEXT:    movdqa %xmm2, %xmm4
 ; X32-SSE2-NEXT:    psrad $31, %xmm4
@@ -3829,8 +3829,8 @@ define <8 x i64> @sext_8i6_to_8i64(i32 %x) nounwind uwtable readnone ssp {
 ; X32-SSE2-NEXT:    psrad $26, %xmm2
 ; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
 ; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
-; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[3,1,3,3]
-; X32-SSE2-NEXT:    pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,5,5,6,7]
+; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
+; X32-SSE2-NEXT:    pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,5,5,5,5]
 ; X32-SSE2-NEXT:    psllq $58, %xmm3
 ; X32-SSE2-NEXT:    movdqa %xmm3, %xmm4
 ; X32-SSE2-NEXT:    psrad $31, %xmm4
@@ -3843,7 +3843,7 @@ define <8 x i64> @sext_8i6_to_8i64(i32 %x) nounwind uwtable readnone ssp {
 ; X32-SSE41-LABEL: sext_8i6_to_8i64:
 ; X32-SSE41:       # %bb.0: # %entry
 ; X32-SSE41-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X32-SSE41-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; X32-SSE41-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[0,0,0,0]
 ; X32-SSE41-NEXT:    paddw {{\.LCPI.*}}, %xmm3
 ; X32-SSE41-NEXT:    pmovzxwq {{.*#+}} xmm0 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero
@@ -3853,7 +3853,7 @@ define <8 x i64> @sext_8i6_to_8i64(i32 %x) nounwind uwtable readnone ssp {
 ; X32-SSE41-NEXT:    psrad $26, %xmm0
 ; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
 ; X32-SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
-; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,2,3]
+; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,1,1]
 ; X32-SSE41-NEXT:    pmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
 ; X32-SSE41-NEXT:    psllq $58, %xmm1
 ; X32-SSE41-NEXT:    movdqa %xmm1, %xmm2
@@ -3869,7 +3869,7 @@ define <8 x i64> @sext_8i6_to_8i64(i32 %x) nounwind uwtable readnone ssp {
 ; X32-SSE41-NEXT:    psrad $26, %xmm2
 ; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
 ; X32-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm4[2,3],xmm2[4,5],xmm4[6,7]
-; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[3,1,2,3]
+; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
 ; X32-SSE41-NEXT:    pmovzxwq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero
 ; X32-SSE41-NEXT:    psllq $58, %xmm3
 ; X32-SSE41-NEXT:    movdqa %xmm3, %xmm4
@@ -3922,7 +3922,7 @@ define <8 x i32> @zext_negate_sext(<8 x i8> %x) {
 ;
 ; AVX1-LABEL: zext_negate_sext:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; AVX1-NEXT:    vpsubd %xmm1, %xmm2, %xmm1
@@ -4008,7 +4008,7 @@ define <8 x i32> @zext_decremenet_sext(<8 x i8> %x) {
 ;
 ; AVX1-LABEL: zext_decremenet_sext:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; AVX1-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
diff --git a/llvm/test/CodeGen/X86/vector-shift-ashr-128.ll b/llvm/test/CodeGen/X86/vector-shift-ashr-128.ll
index b4bcca54b1f73..eb7259c41feee 100644
--- a/llvm/test/CodeGen/X86/vector-shift-ashr-128.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-ashr-128.ll
@@ -793,7 +793,7 @@ define <16 x i8> @splatvar_shift_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
 ; SSE2-NEXT:    psrlw %xmm1, %xmm2
 ; SSE2-NEXT:    psrlw $8, %xmm2
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,0,2,3,4,5,6,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,0,0,0,4,5,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
 ; SSE2-NEXT:    pand %xmm2, %xmm0
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [32896,32896,32896,32896,32896,32896,32896,32896]
@@ -907,7 +907,7 @@ define <16 x i8> @splatvar_shift_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
 ; X32-SSE-NEXT:    psrlw %xmm1, %xmm2
 ; X32-SSE-NEXT:    psrlw $8, %xmm2
 ; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,0,2,3,4,5,6,7]
+; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,0,0,0,4,5,6,7]
 ; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
 ; X32-SSE-NEXT:    pand %xmm2, %xmm0
 ; X32-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [32896,32896,32896,32896,32896,32896,32896,32896]
diff --git a/llvm/test/CodeGen/X86/vector-shift-ashr-sub128.ll b/llvm/test/CodeGen/X86/vector-shift-ashr-sub128.ll
index bac8f615116e7..f75a75cdb50b4 100644
--- a/llvm/test/CodeGen/X86/vector-shift-ashr-sub128.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-ashr-sub128.ll
@@ -1314,7 +1314,7 @@ define <8 x i8> @splatvar_shift_v8i8(<8 x i8> %a, <8 x i8> %b) nounwind {
 ; SSE2-NEXT:    psrlw %xmm1, %xmm2
 ; SSE2-NEXT:    psrlw $8, %xmm2
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,0,2,3,4,5,6,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,0,0,0,4,5,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
 ; SSE2-NEXT:    pand %xmm2, %xmm0
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [32896,32896,32896,32896,32896,32896,32896,32896]
@@ -1429,7 +1429,7 @@ define <8 x i8> @splatvar_shift_v8i8(<8 x i8> %a, <8 x i8> %b) nounwind {
 ; X32-SSE-NEXT:    psrlw %xmm1, %xmm2
 ; X32-SSE-NEXT:    psrlw $8, %xmm2
 ; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,0,2,3,4,5,6,7]
+; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,0,0,0,4,5,6,7]
 ; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
 ; X32-SSE-NEXT:    pand %xmm2, %xmm0
 ; X32-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [32896,32896,32896,32896,32896,32896,32896,32896]
@@ -1452,7 +1452,7 @@ define <4 x i8> @splatvar_shift_v4i8(<4 x i8> %a, <4 x i8> %b) nounwind {
 ; SSE2-NEXT:    psrlw %xmm1, %xmm2
 ; SSE2-NEXT:    psrlw $8, %xmm2
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,0,2,3,4,5,6,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,0,0,0,4,5,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
 ; SSE2-NEXT:    pand %xmm2, %xmm0
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [32896,32896,32896,32896,32896,32896,32896,32896]
@@ -1507,7 +1507,7 @@ define <4 x i8> @splatvar_shift_v4i8(<4 x i8> %a, <4 x i8> %b) nounwind {
 ; XOPAVX1-LABEL: splatvar_shift_v4i8:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,0,2,3,4,5,6,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,0,0,0,4,5,6,7]
 ; XOPAVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; XOPAVX1-NEXT:    vpsubb %xmm1, %xmm2, %xmm1
 ; XOPAVX1-NEXT:    vpshab %xmm1, %xmm0, %xmm0
@@ -1567,7 +1567,7 @@ define <4 x i8> @splatvar_shift_v4i8(<4 x i8> %a, <4 x i8> %b) nounwind {
 ; X32-SSE-NEXT:    psrlw %xmm1, %xmm2
 ; X32-SSE-NEXT:    psrlw $8, %xmm2
 ; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,0,2,3,4,5,6,7]
+; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,0,0,0,4,5,6,7]
 ; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
 ; X32-SSE-NEXT:    pand %xmm2, %xmm0
 ; X32-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [32896,32896,32896,32896,32896,32896,32896,32896]
@@ -1590,7 +1590,7 @@ define <2 x i8> @splatvar_shift_v2i8(<2 x i8> %a, <2 x i8> %b) nounwind {
 ; SSE2-NEXT:    psrlw %xmm1, %xmm2
 ; SSE2-NEXT:    psrlw $8, %xmm2
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,0,2,3,4,5,6,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,0,0,0,4,5,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
 ; SSE2-NEXT:    pand %xmm2, %xmm0
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [32896,32896,32896,32896,32896,32896,32896,32896]
@@ -1696,7 +1696,7 @@ define <2 x i8> @splatvar_shift_v2i8(<2 x i8> %a, <2 x i8> %b) nounwind {
 ; X32-SSE-NEXT:    psrlw %xmm1, %xmm2
 ; X32-SSE-NEXT:    psrlw $8, %xmm2
 ; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,0,2,3,4,5,6,7]
+; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,0,0,0,4,5,6,7]
 ; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
 ; X32-SSE-NEXT:    pand %xmm2, %xmm0
 ; X32-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [32896,32896,32896,32896,32896,32896,32896,32896]
@@ -1719,7 +1719,7 @@ define <2 x i32> @constant_shift_v2i32(<2 x i32> %a) nounwind {
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; SSE2-NEXT:    psrad $4, %xmm1
 ; SSE2-NEXT:    psrad $5, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
 ; SSE2-NEXT:    movdqa %xmm1, %xmm0
 ; SSE2-NEXT:    retq
@@ -1769,7 +1769,7 @@ define <2 x i32> @constant_shift_v2i32(<2 x i32> %a) nounwind {
 ; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
 ; X32-SSE-NEXT:    psrad $4, %xmm1
 ; X32-SSE-NEXT:    psrad $5, %xmm0
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; X32-SSE-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
 ; X32-SSE-NEXT:    movdqa %xmm1, %xmm0
 ; X32-SSE-NEXT:    retl
diff --git a/llvm/test/CodeGen/X86/vector-shift-by-select-loop.ll b/llvm/test/CodeGen/X86/vector-shift-by-select-loop.ll
index 26f8a68f544e1..651b0f829120a 100644
--- a/llvm/test/CodeGen/X86/vector-shift-by-select-loop.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-by-select-loop.ll
@@ -42,19 +42,19 @@ define void @vector_variable_shift_left_loop(i32* nocapture %arr, i8* nocapture
 ; SSE-NEXT:    pxor %xmm1, %xmm1
 ; SSE-NEXT:    pcmpeqb %xmm1, %xmm0
 ; SSE-NEXT:    pmovsxbd %xmm0, %xmm7
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE-NEXT:    pmovsxbd %xmm0, %xmm0
 ; SSE-NEXT:    pcmpeqb %xmm1, %xmm3
 ; SSE-NEXT:    pmovsxbd %xmm3, %xmm13
-; SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,1,1]
 ; SSE-NEXT:    pmovsxbd %xmm3, %xmm6
 ; SSE-NEXT:    pcmpeqb %xmm1, %xmm4
 ; SSE-NEXT:    pmovsxbd %xmm4, %xmm11
-; SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,1,1]
 ; SSE-NEXT:    pmovsxbd %xmm3, %xmm2
 ; SSE-NEXT:    pcmpeqb %xmm1, %xmm5
 ; SSE-NEXT:    pmovsxbd %xmm5, %xmm9
-; SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm5[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm5[1,1,1,1]
 ; SSE-NEXT:    pmovsxbd %xmm3, %xmm10
 ; SSE-NEXT:    movdqu 16(%rdi,%rcx,4), %xmm3
 ; SSE-NEXT:    movdqa %xmm3, %xmm4
@@ -175,11 +175,11 @@ define void @vector_variable_shift_left_loop(i32* nocapture %arr, i8* nocapture
 ; AVX1-NEXT:    vpxor %xmm12, %xmm12, %xmm12
 ; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm12, %xmm1
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm5
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
 ; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm12, %xmm2
 ; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm6
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,1,1]
 ; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm2
 ; AVX1-NEXT:    vpcmpeqb %xmm3, %xmm12, %xmm3
 ; AVX1-NEXT:    vpmovzxdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Folded Reload
@@ -191,14 +191,14 @@ define void @vector_variable_shift_left_loop(i32* nocapture %arr, i8* nocapture
 ; AVX1-NEXT:    vpslld %xmm10, %xmm8, %xmm0
 ; AVX1-NEXT:    vblendvps %xmm5, %xmm9, %xmm0, %xmm8
 ; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm5
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,1,1]
 ; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
 ; AVX1-NEXT:    vpcmpeqb %xmm4, %xmm12, %xmm4
 ; AVX1-NEXT:    vmovdqu 16(%rdi,%rcx,4), %xmm0
 ; AVX1-NEXT:    vpslld %xmm7, %xmm0, %xmm7
 ; AVX1-NEXT:    vpslld %xmm10, %xmm0, %xmm0
 ; AVX1-NEXT:    vpmovsxbd %xmm4, %xmm9
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[1,1,1,1]
 ; AVX1-NEXT:    vpmovsxbd %xmm4, %xmm12
 ; AVX1-NEXT:    vblendvps %xmm1, %xmm7, %xmm0, %xmm10
 ; AVX1-NEXT:    vmovdqu 32(%rdi,%rcx,4), %xmm1
@@ -371,19 +371,19 @@ define void @vector_variable_shift_left_loop(i32* nocapture %arr, i8* nocapture
 ; XOP-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero
 ; XOP-NEXT:    vpcomeqb %xmm8, %xmm5, %xmm5
 ; XOP-NEXT:    vpmovsxbd %xmm5, %xmm0
-; XOP-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[1,1,2,3]
+; XOP-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[1,1,1,1]
 ; XOP-NEXT:    vpmovsxbd %xmm5, %xmm5
 ; XOP-NEXT:    vpcomeqb %xmm8, %xmm6, %xmm6
 ; XOP-NEXT:    vpmovsxbd %xmm6, %xmm10
-; XOP-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[1,1,2,3]
+; XOP-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[1,1,1,1]
 ; XOP-NEXT:    vpmovsxbd %xmm6, %xmm6
 ; XOP-NEXT:    vpcomeqb %xmm8, %xmm7, %xmm7
 ; XOP-NEXT:    vpmovsxbd %xmm7, %xmm11
-; XOP-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[1,1,2,3]
+; XOP-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[1,1,1,1]
 ; XOP-NEXT:    vpmovsxbd %xmm7, %xmm7
 ; XOP-NEXT:    vpcomeqb %xmm8, %xmm2, %xmm2
 ; XOP-NEXT:    vpmovsxbd %xmm2, %xmm12
-; XOP-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,2,3]
+; XOP-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,1,1]
 ; XOP-NEXT:    vpmovsxbd %xmm2, %xmm2
 ; XOP-NEXT:    vblendvps %xmm5, %xmm15, %xmm4, %xmm5
 ; XOP-NEXT:    vpshld %xmm5, 16(%rdi,%rcx,4), %xmm13
diff --git a/llvm/test/CodeGen/X86/vector-shift-lshr-128.ll b/llvm/test/CodeGen/X86/vector-shift-lshr-128.ll
index 7dab12f3f03e8..c8dfb3a062fd9 100644
--- a/llvm/test/CodeGen/X86/vector-shift-lshr-128.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-lshr-128.ll
@@ -659,7 +659,7 @@ define <16 x i8> @splatvar_shift_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
 ; SSE2-NEXT:    psrlw %xmm1, %xmm2
 ; SSE2-NEXT:    psrlw $8, %xmm2
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,2,3,4,5,6,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,0,0,4,5,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; SSE2-NEXT:    pand %xmm1, %xmm0
 ; SSE2-NEXT:    retq
@@ -757,7 +757,7 @@ define <16 x i8> @splatvar_shift_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
 ; X32-SSE-NEXT:    psrlw %xmm1, %xmm2
 ; X32-SSE-NEXT:    psrlw $8, %xmm2
 ; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,2,3,4,5,6,7]
+; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,0,0,4,5,6,7]
 ; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; X32-SSE-NEXT:    pand %xmm1, %xmm0
 ; X32-SSE-NEXT:    retl
diff --git a/llvm/test/CodeGen/X86/vector-shift-lshr-sub128.ll b/llvm/test/CodeGen/X86/vector-shift-lshr-sub128.ll
index 6b9ac09658e9e..e8ebfd0aa3584 100644
--- a/llvm/test/CodeGen/X86/vector-shift-lshr-sub128.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-lshr-sub128.ll
@@ -1080,7 +1080,7 @@ define <8 x i8> @splatvar_shift_v8i8(<8 x i8> %a, <8 x i8> %b) nounwind {
 ; SSE2-NEXT:    psrlw %xmm1, %xmm2
 ; SSE2-NEXT:    psrlw $8, %xmm2
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,2,3,4,5,6,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,0,0,4,5,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; SSE2-NEXT:    pand %xmm1, %xmm0
 ; SSE2-NEXT:    retq
@@ -1179,7 +1179,7 @@ define <8 x i8> @splatvar_shift_v8i8(<8 x i8> %a, <8 x i8> %b) nounwind {
 ; X32-SSE-NEXT:    psrlw %xmm1, %xmm2
 ; X32-SSE-NEXT:    psrlw $8, %xmm2
 ; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,2,3,4,5,6,7]
+; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,0,0,4,5,6,7]
 ; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; X32-SSE-NEXT:    pand %xmm1, %xmm0
 ; X32-SSE-NEXT:    retl
@@ -1198,7 +1198,7 @@ define <4 x i8> @splatvar_shift_v4i8(<4 x i8> %a, <4 x i8> %b) nounwind {
 ; SSE2-NEXT:    psrlw %xmm1, %xmm2
 ; SSE2-NEXT:    psrlw $8, %xmm2
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,2,3,4,5,6,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,0,0,4,5,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; SSE2-NEXT:    pand %xmm1, %xmm0
 ; SSE2-NEXT:    retq
@@ -1237,7 +1237,7 @@ define <4 x i8> @splatvar_shift_v4i8(<4 x i8> %a, <4 x i8> %b) nounwind {
 ; XOPAVX1-LABEL: splatvar_shift_v4i8:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,0,2,3,4,5,6,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,0,0,0,4,5,6,7]
 ; XOPAVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; XOPAVX1-NEXT:    vpsubb %xmm1, %xmm2, %xmm1
 ; XOPAVX1-NEXT:    vpshlb %xmm1, %xmm0, %xmm0
@@ -1297,7 +1297,7 @@ define <4 x i8> @splatvar_shift_v4i8(<4 x i8> %a, <4 x i8> %b) nounwind {
 ; X32-SSE-NEXT:    psrlw %xmm1, %xmm2
 ; X32-SSE-NEXT:    psrlw $8, %xmm2
 ; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,2,3,4,5,6,7]
+; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,0,0,4,5,6,7]
 ; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; X32-SSE-NEXT:    pand %xmm1, %xmm0
 ; X32-SSE-NEXT:    retl
@@ -1316,7 +1316,7 @@ define <2 x i8> @splatvar_shift_v2i8(<2 x i8> %a, <2 x i8> %b) nounwind {
 ; SSE2-NEXT:    psrlw %xmm1, %xmm2
 ; SSE2-NEXT:    psrlw $8, %xmm2
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,2,3,4,5,6,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,0,0,4,5,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; SSE2-NEXT:    pand %xmm1, %xmm0
 ; SSE2-NEXT:    retq
@@ -1406,7 +1406,7 @@ define <2 x i8> @splatvar_shift_v2i8(<2 x i8> %a, <2 x i8> %b) nounwind {
 ; X32-SSE-NEXT:    psrlw %xmm1, %xmm2
 ; X32-SSE-NEXT:    psrlw $8, %xmm2
 ; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,2,3,4,5,6,7]
+; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,0,0,4,5,6,7]
 ; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; X32-SSE-NEXT:    pand %xmm1, %xmm0
 ; X32-SSE-NEXT:    retl
@@ -1425,7 +1425,7 @@ define <2 x i32> @constant_shift_v2i32(<2 x i32> %a) nounwind {
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; SSE2-NEXT:    psrld $4, %xmm1
 ; SSE2-NEXT:    psrld $5, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
 ; SSE2-NEXT:    movdqa %xmm1, %xmm0
 ; SSE2-NEXT:    retq
@@ -1475,7 +1475,7 @@ define <2 x i32> @constant_shift_v2i32(<2 x i32> %a) nounwind {
 ; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
 ; X32-SSE-NEXT:    psrld $4, %xmm1
 ; X32-SSE-NEXT:    psrld $5, %xmm0
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; X32-SSE-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
 ; X32-SSE-NEXT:    movdqa %xmm1, %xmm0
 ; X32-SSE-NEXT:    retl
diff --git a/llvm/test/CodeGen/X86/vector-shift-shl-128.ll b/llvm/test/CodeGen/X86/vector-shift-shl-128.ll
index 4b83700e3a85f..321e4aad5f6c3 100644
--- a/llvm/test/CodeGen/X86/vector-shift-shl-128.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-shl-128.ll
@@ -571,7 +571,7 @@ define <16 x i8> @splatvar_shift_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
 ; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
 ; SSE2-NEXT:    psllw %xmm1, %xmm2
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,2,3,4,5,6,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,0,0,4,5,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; SSE2-NEXT:    pand %xmm1, %xmm0
 ; SSE2-NEXT:    retq
@@ -666,7 +666,7 @@ define <16 x i8> @splatvar_shift_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
 ; X32-SSE-NEXT:    pcmpeqd %xmm2, %xmm2
 ; X32-SSE-NEXT:    psllw %xmm1, %xmm2
 ; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,2,3,4,5,6,7]
+; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,0,0,4,5,6,7]
 ; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; X32-SSE-NEXT:    pand %xmm1, %xmm0
 ; X32-SSE-NEXT:    retl
diff --git a/llvm/test/CodeGen/X86/vector-shift-shl-sub128.ll b/llvm/test/CodeGen/X86/vector-shift-shl-sub128.ll
index 1c688089af494..4ed08f2846d1d 100644
--- a/llvm/test/CodeGen/X86/vector-shift-shl-sub128.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-shl-sub128.ll
@@ -936,7 +936,7 @@ define <8 x i8> @splatvar_shift_v8i8(<8 x i8> %a, <8 x i8> %b) nounwind {
 ; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
 ; SSE2-NEXT:    psllw %xmm1, %xmm2
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,2,3,4,5,6,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,0,0,4,5,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; SSE2-NEXT:    pand %xmm1, %xmm0
 ; SSE2-NEXT:    retq
@@ -1031,7 +1031,7 @@ define <8 x i8> @splatvar_shift_v8i8(<8 x i8> %a, <8 x i8> %b) nounwind {
 ; X32-SSE-NEXT:    pcmpeqd %xmm2, %xmm2
 ; X32-SSE-NEXT:    psllw %xmm1, %xmm2
 ; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,2,3,4,5,6,7]
+; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,0,0,4,5,6,7]
 ; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; X32-SSE-NEXT:    pand %xmm1, %xmm0
 ; X32-SSE-NEXT:    retl
@@ -1049,7 +1049,7 @@ define <4 x i8> @splatvar_shift_v4i8(<4 x i8> %a, <4 x i8> %b) nounwind {
 ; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
 ; SSE2-NEXT:    psllw %xmm1, %xmm2
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,2,3,4,5,6,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,0,0,4,5,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; SSE2-NEXT:    pand %xmm1, %xmm0
 ; SSE2-NEXT:    retq
@@ -1089,7 +1089,7 @@ define <4 x i8> @splatvar_shift_v4i8(<4 x i8> %a, <4 x i8> %b) nounwind {
 ; XOPAVX1-LABEL: splatvar_shift_v4i8:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,0,2,3,4,5,6,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,0,0,0,4,5,6,7]
 ; XOPAVX1-NEXT:    vpshlb %xmm1, %xmm0, %xmm0
 ; XOPAVX1-NEXT:    retq
 ;
@@ -1144,7 +1144,7 @@ define <4 x i8> @splatvar_shift_v4i8(<4 x i8> %a, <4 x i8> %b) nounwind {
 ; X32-SSE-NEXT:    pcmpeqd %xmm2, %xmm2
 ; X32-SSE-NEXT:    psllw %xmm1, %xmm2
 ; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,2,3,4,5,6,7]
+; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,0,0,4,5,6,7]
 ; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; X32-SSE-NEXT:    pand %xmm1, %xmm0
 ; X32-SSE-NEXT:    retl
@@ -1162,7 +1162,7 @@ define <2 x i8> @splatvar_shift_v2i8(<2 x i8> %a, <2 x i8> %b) nounwind {
 ; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
 ; SSE2-NEXT:    psllw %xmm1, %xmm2
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,2,3,4,5,6,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,0,0,4,5,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; SSE2-NEXT:    pand %xmm1, %xmm0
 ; SSE2-NEXT:    retq
@@ -1250,7 +1250,7 @@ define <2 x i8> @splatvar_shift_v2i8(<2 x i8> %a, <2 x i8> %b) nounwind {
 ; X32-SSE-NEXT:    pcmpeqd %xmm2, %xmm2
 ; X32-SSE-NEXT:    psllw %xmm1, %xmm2
 ; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,2,3,4,5,6,7]
+; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,0,0,4,5,6,7]
 ; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; X32-SSE-NEXT:    pand %xmm1, %xmm0
 ; X32-SSE-NEXT:    retl
@@ -1269,7 +1269,7 @@ define <2 x i32> @constant_shift_v2i32(<2 x i32> %a) nounwind {
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; SSE2-NEXT:    pslld $4, %xmm1
 ; SSE2-NEXT:    pslld $5, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
 ; SSE2-NEXT:    movdqa %xmm1, %xmm0
 ; SSE2-NEXT:    retq
@@ -1319,7 +1319,7 @@ define <2 x i32> @constant_shift_v2i32(<2 x i32> %a) nounwind {
 ; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
 ; X32-SSE-NEXT:    pslld $4, %xmm1
 ; X32-SSE-NEXT:    pslld $5, %xmm0
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; X32-SSE-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
 ; X32-SSE-NEXT:    movdqa %xmm1, %xmm0
 ; X32-SSE-NEXT:    retl
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-128-v16.ll b/llvm/test/CodeGen/X86/vector-shuffle-128-v16.ll
index 8e9b991c34bf5..aa5c713d527bf 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-128-v16.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-128-v16.ll
@@ -14,7 +14,7 @@ define <16 x i8> @shuffle_v16i8_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00(
 ; SSE2-LABEL: shuffle_v16i8_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; SSE2-NEXT:    retq
 ;
@@ -231,13 +231,13 @@ define <16 x i8> @shuffle_v16i8_00_00_01_01_02_02_03_03_04_04_05_05_06_06_07_07(
 define <16 x i8> @shuffle_v16i8_0101010101010101(<16 x i8> %a, <16 x i8> %b) {
 ; SSE-LABEL: shuffle_v16i8_0101010101010101:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: shuffle_v16i8_0101010101010101:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    retq
 ;
@@ -248,7 +248,7 @@ define <16 x i8> @shuffle_v16i8_0101010101010101(<16 x i8> %a, <16 x i8> %b) {
 ;
 ; XOPAVX1-LABEL: shuffle_v16i8_0101010101010101:
 ; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; XOPAVX1-NEXT:    retq
 ;
@@ -293,7 +293,7 @@ define <16 x i8> @shuffle_v16i8_16_00_16_01_16_02_16_03_16_04_16_05_16_06_16_07(
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm1[0,0,2,3,4,5,6,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm1[0,0,0,0,4,5,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
 ; SSE2-NEXT:    por %xmm2, %xmm0
@@ -1543,7 +1543,7 @@ define <16 x i8> @shuffle_v16i8_uu_10_02_07_22_14_07_02_18_03_01_14_18_09_11_00(
 ; SSE2-NEXT:    pand %xmm5, %xmm4
 ; SSE2-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,3,0,1]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,1,2,2,4,5,6,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[2,2,2,2,4,5,6,7]
 ; SSE2-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,7,7]
 ; SSE2-NEXT:    pandn %xmm2, %xmm5
 ; SSE2-NEXT:    por %xmm4, %xmm5
@@ -2060,7 +2060,7 @@ define <16 x i8> @insert_dup_mem_v16i8_i32(i32* %ptr) {
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; SSE2-NEXT:    retq
 ;
@@ -2114,7 +2114,7 @@ define <16 x i8> @insert_dup_mem_v16i8_sext_i8(i8* %ptr) {
 ; SSE2-NEXT:    movzbl (%rdi), %eax
 ; SSE2-NEXT:    movd %eax, %xmm0
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; SSE2-NEXT:    retq
 ;
@@ -2172,7 +2172,7 @@ define <16 x i8> @insert_dup_elt1_mem_v16i8_i32(i32* %ptr) {
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[1,1,2,3,4,5,6,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[1,1,1,1,4,5,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; SSE2-NEXT:    retq
 ;
@@ -2221,7 +2221,7 @@ define <16 x i8> @insert_dup_elt2_mem_v16i8_i32(i32* %ptr) {
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[2,2,2,3,4,5,6,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[2,2,2,2,4,5,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; SSE2-NEXT:    retq
 ;
@@ -2271,7 +2271,7 @@ define <16 x i8> @insert_dup_elt1_mem_v16i8_sext_i8(i8* %ptr) {
 ; SSE2-NEXT:    movsbl (%rdi), %eax
 ; SSE2-NEXT:    movd %eax, %xmm0
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[1,1,2,3,4,5,6,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[1,1,1,1,4,5,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; SSE2-NEXT:    retq
 ;
@@ -2339,7 +2339,7 @@ define <16 x i8> @insert_dup_elt2_mem_v16i8_sext_i8(i8* %ptr) {
 ; SSE2-NEXT:    movsbl (%rdi), %eax
 ; SSE2-NEXT:    movd %eax, %xmm0
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[2,2,2,3,4,5,6,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[2,2,2,2,4,5,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; SSE2-NEXT:    retq
 ;
@@ -2413,7 +2413,7 @@ define <16 x i8> @PR31364(i8* nocapture readonly %a, i8* nocapture readonly %b)
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm1[1,1,1,3,4,5,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,1]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,1,0,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[1,1,1,1,4,5,6,7]
 ; SSE2-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,4,4,4]
 ; SSE2-NEXT:    packuswb %xmm1, %xmm0
@@ -2458,12 +2458,12 @@ define <16 x i8> @PR31301(i8* nocapture readonly %x, i8* nocapture readonly %y)
 ; SSE2-NEXT:    movzbl (%rdi), %eax
 ; SSE2-NEXT:    movd %eax, %xmm0
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; SSE2-NEXT:    movzbl (%rsi), %eax
 ; SSE2-NEXT:    movd %eax, %xmm1
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,0,2,3,4,5,6,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,0,0,0,4,5,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
 ; SSE2-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-128-v4.ll b/llvm/test/CodeGen/X86/vector-shuffle-128-v4.ll
index 1f767645921db..b5fc242daf45b 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-128-v4.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-128-v4.ll
@@ -350,15 +350,21 @@ define <4 x i32> @shuffle_v4i32_0124(<4 x i32> %a, <4 x i32> %b) {
 ;
 ; SSE41-LABEL: shuffle_v4i32_0124:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,1,2,0]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5],xmm1[6,7]
 ; SSE41-NEXT:    retq
 ;
-; AVX1OR2-LABEL: shuffle_v4i32_0124:
-; AVX1OR2:       # %bb.0:
-; AVX1OR2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[0,1,2,0]
-; AVX1OR2-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3]
-; AVX1OR2-NEXT:    retq
+; AVX1-LABEL: shuffle_v4i32_0124:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[0,0,0,0]
+; AVX1-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3]
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: shuffle_v4i32_0124:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vbroadcastss %xmm1, %xmm1
+; AVX2-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3]
+; AVX2-NEXT:    retq
 ;
 ; AVX512VL-LABEL: shuffle_v4i32_0124:
 ; AVX512VL:       # %bb.0:
@@ -740,21 +746,21 @@ define <4 x float> @shuffle_v4f32_zuu4(<4 x float> %a) {
 ; SSE2-LABEL: shuffle_v4f32_zuu4:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    xorps %xmm1, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,1],xmm0[2,0]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,0],xmm0[0,0]
 ; SSE2-NEXT:    movaps %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: shuffle_v4f32_zuu4:
 ; SSE3:       # %bb.0:
 ; SSE3-NEXT:    xorps %xmm1, %xmm1
-; SSE3-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,1],xmm0[2,0]
+; SSE3-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,0],xmm0[0,0]
 ; SSE3-NEXT:    movaps %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: shuffle_v4f32_zuu4:
 ; SSSE3:       # %bb.0:
 ; SSSE3-NEXT:    xorps %xmm1, %xmm1
-; SSSE3-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,1],xmm0[2,0]
+; SSSE3-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,0],xmm0[0,0]
 ; SSSE3-NEXT:    movaps %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
@@ -1559,33 +1565,33 @@ define <4 x i32> @shuffle_v4i32_2345(<4 x i32> %a, <4 x i32> %b) {
 define <4 x i32> @shuffle_v4i32_2456(<4 x i32> %a, <4 x i32> %b) {
 ; SSE2-LABEL: shuffle_v4i32_2456:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[2,0],xmm1[0,0]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[2,2],xmm1[0,0]
 ; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[1,2]
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: shuffle_v4i32_2456:
 ; SSE3:       # %bb.0:
-; SSE3-NEXT:    shufps {{.*#+}} xmm0 = xmm0[2,0],xmm1[0,0]
+; SSE3-NEXT:    shufps {{.*#+}} xmm0 = xmm0[2,2],xmm1[0,0]
 ; SSE3-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[1,2]
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: shuffle_v4i32_2456:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,2,2]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,2,2,2]
 ; SSSE3-NEXT:    palignr {{.*#+}} xmm1 = xmm0[12,13,14,15],xmm1[0,1,2,3,4,5,6,7,8,9,10,11]
 ; SSSE3-NEXT:    movdqa %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: shuffle_v4i32_2456:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,2,2]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,2,2,2]
 ; SSE41-NEXT:    palignr {{.*#+}} xmm1 = xmm0[12,13,14,15],xmm1[0,1,2,3,4,5,6,7,8,9,10,11]
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1OR2-LABEL: shuffle_v4i32_2456:
 ; AVX1OR2:       # %bb.0:
-; AVX1OR2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,2,2]
+; AVX1OR2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,2,2]
 ; AVX1OR2-NEXT:    vpalignr {{.*#+}} xmm0 = xmm0[12,13,14,15],xmm1[0,1,2,3,4,5,6,7,8,9,10,11]
 ; AVX1OR2-NEXT:    retq
 ;
@@ -2053,7 +2059,7 @@ define <4 x float> @broadcast_v4f32_0101_from_v2f32(<2 x float>* %x) {
 define <4 x i32> @extract3_insert0_v4i32_7123(<4 x i32> %a0, <4 x i32> %a1) {
 ; SSE2-LABEL: extract3_insert0_v4i32_7123:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    movd %eax, %xmm1
 ; SSE2-NEXT:    movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
@@ -2061,7 +2067,7 @@ define <4 x i32> @extract3_insert0_v4i32_7123(<4 x i32> %a0, <4 x i32> %a1) {
 ;
 ; SSE3-LABEL: extract3_insert0_v4i32_7123:
 ; SSE3:       # %bb.0:
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; SSE3-NEXT:    movd %xmm1, %eax
 ; SSE3-NEXT:    movd %eax, %xmm1
 ; SSE3-NEXT:    movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
@@ -2069,7 +2075,7 @@ define <4 x i32> @extract3_insert0_v4i32_7123(<4 x i32> %a0, <4 x i32> %a1) {
 ;
 ; SSSE3-LABEL: extract3_insert0_v4i32_7123:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; SSSE3-NEXT:    movd %xmm1, %eax
 ; SSSE3-NEXT:    movd %eax, %xmm1
 ; SSSE3-NEXT:    movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-128-v8.ll b/llvm/test/CodeGen/X86/vector-shuffle-128-v8.ll
index a11fc5b7c8ed3..f448f41cf522e 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-128-v8.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-128-v8.ll
@@ -65,13 +65,13 @@ define <8 x i16> @shuffle_v8i16_456789AB(<8 x i16> %a, <8 x i16> %b) {
 define <8 x i16> @shuffle_v8i16_00000000(<8 x i16> %a, <8 x i16> %b) {
 ; SSE-LABEL: shuffle_v8i16_00000000:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: shuffle_v8i16_00000000:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    retq
 ;
@@ -82,7 +82,7 @@ define <8 x i16> @shuffle_v8i16_00000000(<8 x i16> %a, <8 x i16> %b) {
 ;
 ; XOPAVX1-LABEL: shuffle_v8i16_00000000:
 ; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; XOPAVX1-NEXT:    retq
 ;
@@ -1640,9 +1640,9 @@ define <8 x i16> @shuffle_v8i16_0923cde7(<8 x i16> %a, <8 x i16> %b) {
 define <8 x i16> @shuffle_v8i16_XXX1X579(<8 x i16> %a, <8 x i16> %b) {
 ; SSE2-LABEL: shuffle_v8i16_XXX1X579:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[0,1,2,0]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[0,0,0,0]
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [65535,65535,65535,65535,65535,65535,65535,0]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,1,2,1,4,5,6,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[1,1,1,1,4,5,6,7]
 ; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,7,7]
 ; SSE2-NEXT:    pand %xmm1, %xmm0
 ; SSE2-NEXT:    pandn %xmm2, %xmm1
@@ -1659,44 +1659,44 @@ define <8 x i16> @shuffle_v8i16_XXX1X579(<8 x i16> %a, <8 x i16> %b) {
 ;
 ; SSE41-LABEL: shuffle_v8i16_XXX1X579:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,1,2,0]
-; SSE41-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,1,2,1,4,5,6,7]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
+; SSE41-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[1,1,1,1,4,5,6,7]
 ; SSE41-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,7,7]
 ; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,6],xmm1[7]
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: shuffle_v8i16_XXX1X579:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,1,2,0]
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,1,2,1,4,5,6,7]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[1,1,1,1,4,5,6,7]
 ; AVX1-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,7,7]
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,6],xmm1[7]
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-SLOW-LABEL: shuffle_v8i16_XXX1X579:
 ; AVX2-SLOW:       # %bb.0:
-; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,1,2,0]
-; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,1,2,1,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpbroadcastd %xmm1, %xmm1
+; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[1,1,1,1,4,5,6,7]
 ; AVX2-SLOW-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,7,7]
 ; AVX2-SLOW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,6],xmm1[7]
 ; AVX2-SLOW-NEXT:    retq
 ;
 ; AVX2-FAST-LABEL: shuffle_v8i16_XXX1X579:
 ; AVX2-FAST:       # %bb.0:
-; AVX2-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,1,2,0]
-; AVX2-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,2,3,8,9,10,11,14,15,u,u]
+; AVX2-FAST-NEXT:    vpbroadcastd %xmm1, %xmm1
+; AVX2-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[2,3,2,3,2,3,2,3,8,9,10,11,14,15,u,u]
 ; AVX2-FAST-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,6],xmm1[7]
 ; AVX2-FAST-NEXT:    retq
 ;
 ; AVX512VL-LABEL: shuffle_v8i16_XXX1X579:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,1,2,1,4,5,7,9]
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm2 = [1,1,1,1,4,5,7,9]
 ; AVX512VL-NEXT:    vpermt2w %xmm1, %xmm2, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; XOP-LABEL: shuffle_v8i16_XXX1X579:
 ; XOP:       # %bb.0:
-; XOP-NEXT:    vpperm {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,2,3,8,9,10,11,14,15],xmm1[2,3]
+; XOP-NEXT:    vpperm {{.*#+}} xmm0 = xmm0[2,3,2,3,2,3,2,3,8,9,10,11,14,15],xmm1[2,3]
 ; XOP-NEXT:    retq
   %shuffle = shufflevector <8 x i16> %a, <8 x i16> %b, <8 x i32> <i32 undef, i32 undef, i32 undef, i32 1, i32 undef, i32 5, i32 7, i32 9>
   ret <8 x i16> %shuffle
@@ -2193,14 +2193,14 @@ define <8 x i16> @shuffle_v8i16_u6uu9abu(<8 x i16> %a, <8 x i16> %b) {
 define <8 x i16> @shuffle_v8i16_0uuu1uuu(<8 x i16> %a) {
 ; SSE2-LABEL: shuffle_v8i16_0uuu1uuu:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,0,3]
-; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,5,5]
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: shuffle_v8i16_0uuu1uuu:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,0,3]
-; SSSE3-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,6,7]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
+; SSSE3-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,5,5]
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: shuffle_v8i16_0uuu1uuu:
@@ -3115,14 +3115,14 @@ define <8 x i16> @insert_dup_mem_v8i16_i32(i32* %ptr) {
 ; SSE-LABEL: insert_dup_mem_v8i16_i32:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: insert_dup_mem_v8i16_i32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    retq
 ;
@@ -3134,7 +3134,7 @@ define <8 x i16> @insert_dup_mem_v8i16_i32(i32* %ptr) {
 ; XOPAVX1-LABEL: insert_dup_mem_v8i16_i32:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; XOPAVX1-NEXT:    retq
 ;
@@ -3154,7 +3154,7 @@ define <8 x i16> @insert_dup_mem_v8i16_sext_i16(i16* %ptr) {
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    movzwl (%rdi), %eax
 ; SSE-NEXT:    movd %eax, %xmm0
-; SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; SSE-NEXT:    retq
 ;
@@ -3162,7 +3162,7 @@ define <8 x i16> @insert_dup_mem_v8i16_sext_i16(i16* %ptr) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    movzwl (%rdi), %eax
 ; AVX1-NEXT:    vmovd %eax, %xmm0
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    retq
 ;
@@ -3175,7 +3175,7 @@ define <8 x i16> @insert_dup_mem_v8i16_sext_i16(i16* %ptr) {
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    movzwl (%rdi), %eax
 ; XOPAVX1-NEXT:    vmovd %eax, %xmm0
-; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; XOPAVX1-NEXT:    retq
 ;
@@ -3195,14 +3195,14 @@ define <8 x i16> @insert_dup_elt1_mem_v8i16_i32(i32* %ptr) {
 ; SSE-LABEL: insert_dup_elt1_mem_v8i16_i32:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[1,1,2,3,4,5,6,7]
+; SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[1,1,1,1,4,5,6,7]
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: insert_dup_elt1_mem_v8i16_i32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[1,1,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[1,1,1,1,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    retq
 ;
@@ -3214,7 +3214,7 @@ define <8 x i16> @insert_dup_elt1_mem_v8i16_i32(i32* %ptr) {
 ; XOPAVX1-LABEL: insert_dup_elt1_mem_v8i16_i32:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[1,1,2,3,4,5,6,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[1,1,1,1,4,5,6,7]
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; XOPAVX1-NEXT:    retq
 ;
@@ -3233,7 +3233,7 @@ define <8 x i16> @insert_dup_elt3_mem_v8i16_i32(i32* %ptr) {
 ; SSE2-LABEL: insert_dup_elt3_mem_v8i16_i32:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[1,1,0,1,4,5,6,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[1,1,1,1,4,5,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; SSE2-NEXT:    retq
 ;
@@ -3252,7 +3252,7 @@ define <8 x i16> @insert_dup_elt3_mem_v8i16_i32(i32* %ptr) {
 ; AVX1-LABEL: insert_dup_elt3_mem_v8i16_i32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vbroadcastss (%rdi), %xmm0
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[3,3,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[3,3,3,3,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    retq
 ;
@@ -3264,7 +3264,7 @@ define <8 x i16> @insert_dup_elt3_mem_v8i16_i32(i32* %ptr) {
 ; XOPAVX1-LABEL: insert_dup_elt3_mem_v8i16_i32:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vbroadcastss (%rdi), %xmm0
-; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[3,3,2,3,4,5,6,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[3,3,3,3,4,5,6,7]
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; XOPAVX1-NEXT:    retq
 ;
@@ -3284,7 +3284,7 @@ define <8 x i16> @insert_dup_elt1_mem_v8i16_sext_i16(i16* %ptr) {
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    movswl (%rdi), %eax
 ; SSE-NEXT:    movd %eax, %xmm0
-; SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[1,1,2,3,4,5,6,7]
+; SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[1,1,1,1,4,5,6,7]
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; SSE-NEXT:    retq
 ;
@@ -3292,7 +3292,7 @@ define <8 x i16> @insert_dup_elt1_mem_v8i16_sext_i16(i16* %ptr) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    movswl (%rdi), %eax
 ; AVX1-NEXT:    vmovd %eax, %xmm0
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[1,1,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[1,1,1,1,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    retq
 ;
@@ -3315,7 +3315,7 @@ define <8 x i16> @insert_dup_elt1_mem_v8i16_sext_i16(i16* %ptr) {
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    movswl (%rdi), %eax
 ; XOPAVX1-NEXT:    vmovd %eax, %xmm0
-; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[1,1,2,3,4,5,6,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[1,1,1,1,4,5,6,7]
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; XOPAVX1-NEXT:    retq
 ;
@@ -3339,7 +3339,7 @@ define <8 x i16> @insert_dup_elt3_mem_v8i16_sext_i16(i16* %ptr) {
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movswl (%rdi), %eax
 ; SSE2-NEXT:    movd %eax, %xmm0
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[1,1,0,1,4,5,6,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[1,1,1,1,4,5,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; SSE2-NEXT:    retq
 ;
@@ -3405,14 +3405,14 @@ define <8 x i16> @insert_dup_mem_v8i16_i64(i64* %ptr) {
 ; SSE-LABEL: insert_dup_mem_v8i16_i64:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: insert_dup_mem_v8i16_i64:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    retq
 ;
@@ -3424,7 +3424,7 @@ define <8 x i16> @insert_dup_mem_v8i16_i64(i64* %ptr) {
 ; XOPAVX1-LABEL: insert_dup_mem_v8i16_i64:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; XOPAVX1-NEXT:    retq
 ;
@@ -3443,14 +3443,14 @@ define <8 x i16> @insert_dup_elt1_mem_v8i16_i64(i64* %ptr) {
 ; SSE-LABEL: insert_dup_elt1_mem_v8i16_i64:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[1,1,2,3,4,5,6,7]
+; SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[1,1,1,1,4,5,6,7]
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: insert_dup_elt1_mem_v8i16_i64:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[1,1,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[1,1,1,1,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    retq
 ;
@@ -3462,7 +3462,7 @@ define <8 x i16> @insert_dup_elt1_mem_v8i16_i64(i64* %ptr) {
 ; XOPAVX1-LABEL: insert_dup_elt1_mem_v8i16_i64:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[1,1,2,3,4,5,6,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[1,1,1,1,4,5,6,7]
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; XOPAVX1-NEXT:    retq
 ;
@@ -3481,14 +3481,14 @@ define <8 x i16> @insert_dup_elt3_mem_v8i16_i64(i64* %ptr) {
 ; SSE-LABEL: insert_dup_elt3_mem_v8i16_i64:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[3,3,2,3,4,5,6,7]
+; SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[3,3,3,3,4,5,6,7]
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: insert_dup_elt3_mem_v8i16_i64:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[3,3,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[3,3,3,3,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    retq
 ;
@@ -3500,7 +3500,7 @@ define <8 x i16> @insert_dup_elt3_mem_v8i16_i64(i64* %ptr) {
 ; XOPAVX1-LABEL: insert_dup_elt3_mem_v8i16_i64:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[3,3,2,3,4,5,6,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[3,3,3,3,4,5,6,7]
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; XOPAVX1-NEXT:    retq
 ;
@@ -3520,7 +3520,7 @@ define <8 x i16> @insert_dup_elt7_mem_v8i16_i64(i64* %ptr) {
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
-; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,7,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,7,7,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,2,2,2]
 ; SSE2-NEXT:    retq
 ;
@@ -3539,7 +3539,7 @@ define <8 x i16> @insert_dup_elt7_mem_v8i16_i64(i64* %ptr) {
 ; AVX1-LABEL: insert_dup_elt7_mem_v8i16_i64:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vmovddup {{.*#+}} xmm0 = mem[0,0]
-; AVX1-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,7,6,7]
+; AVX1-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,7,7,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,2,2]
 ; AVX1-NEXT:    retq
 ;
@@ -3551,7 +3551,7 @@ define <8 x i16> @insert_dup_elt7_mem_v8i16_i64(i64* %ptr) {
 ; XOPAVX1-LABEL: insert_dup_elt7_mem_v8i16_i64:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vmovddup {{.*#+}} xmm0 = mem[0,0]
-; XOPAVX1-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,7,6,7]
+; XOPAVX1-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,7,7,7]
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,2,2]
 ; XOPAVX1-NEXT:    retq
 ;
@@ -3571,7 +3571,7 @@ define <8 x i16> @insert_dup_mem_v8i16_sext_i16_i64(i16* %ptr) {
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    movzwl (%rdi), %eax
 ; SSE-NEXT:    movd %eax, %xmm0
-; SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; SSE-NEXT:    retq
 ;
@@ -3579,7 +3579,7 @@ define <8 x i16> @insert_dup_mem_v8i16_sext_i16_i64(i16* %ptr) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    movzwl (%rdi), %eax
 ; AVX1-NEXT:    vmovd %eax, %xmm0
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    retq
 ;
@@ -3592,7 +3592,7 @@ define <8 x i16> @insert_dup_mem_v8i16_sext_i16_i64(i16* %ptr) {
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    movzwl (%rdi), %eax
 ; XOPAVX1-NEXT:    vmovd %eax, %xmm0
-; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; XOPAVX1-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-256-v16.ll b/llvm/test/CodeGen/X86/vector-shuffle-256-v16.ll
index 131c621162a50..729b1ad59429d 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-256-v16.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-256-v16.ll
@@ -10,7 +10,7 @@
 define <16 x i16> @shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00(<16 x i16> %a, <16 x i16> %b) {
 ; AVX1-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
@@ -22,7 +22,7 @@ define <16 x i16> @shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_0
 ;
 ; XOPAVX1-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00:
 ; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; XOPAVX1-NEXT:    retq
@@ -38,7 +38,7 @@ define <16 x i16> @shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_0
 define <16 x i16> @shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_00_00_00_01_00(<16 x i16> %a, <16 x i16> %b) {
 ; AVX1-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_00_00_00_01_00:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm0[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm0[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,1,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,1]
@@ -74,7 +74,7 @@ define <16 x i16> @shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_00_00_00_01_0
 ;
 ; XOPAVX1-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_00_00_00_01_00:
 ; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm0[0,0,2,3,4,5,6,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm0[0,0,0,0,4,5,6,7]
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,1,0,4,5,6,7]
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,1]
@@ -94,7 +94,7 @@ define <16 x i16> @shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_00_00_00_01_0
 define <16 x i16> @shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_00_00_02_00_00(<16 x i16> %a, <16 x i16> %b) {
 ; AVX1-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_00_00_02_00_00:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm0[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm0[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,2,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,1,0]
@@ -130,7 +130,7 @@ define <16 x i16> @shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_00_00_02_00_0
 ;
 ; XOPAVX1-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_00_00_02_00_00:
 ; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm0[0,0,2,3,4,5,6,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm0[0,0,0,0,4,5,6,7]
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,2,4,5,6,7]
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,1,0]
@@ -150,7 +150,7 @@ define <16 x i16> @shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_00_00_02_00_0
 define <16 x i16> @shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_00_03_00_00_00(<16 x i16> %a, <16 x i16> %b) {
 ; AVX1-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_00_03_00_00_00:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm0[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm0[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,3,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,1,0]
@@ -186,7 +186,7 @@ define <16 x i16> @shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_00_03_00_00_0
 ;
 ; XOPAVX1-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_00_03_00_00_00:
 ; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm0[0,0,2,3,4,5,6,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm0[0,0,0,0,4,5,6,7]
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,3,0,4,5,6,7]
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,1,0]
@@ -206,7 +206,7 @@ define <16 x i16> @shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_00_03_00_00_0
 define <16 x i16> @shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_04_00_00_00_00(<16 x i16> %a, <16 x i16> %b) {
 ; AVX1-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_04_00_00_00_00:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm0[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm0[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,0,1,0,1,8,9,0,1,0,1,0,1,0,1]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
@@ -232,7 +232,7 @@ define <16 x i16> @shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_04_00_00_00_0
 ;
 ; XOPAVX1-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_04_00_00_00_00:
 ; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm0[0,0,2,3,4,5,6,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm0[0,0,0,0,4,5,6,7]
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; XOPAVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,0,1,0,1,8,9,0,1,0,1,0,1,0,1]
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
@@ -250,7 +250,7 @@ define <16 x i16> @shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_04_00_00_00_0
 define <16 x i16> @shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_05_00_00_00_00_00(<16 x i16> %a, <16 x i16> %b) {
 ; AVX1-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_05_00_00_00_00_00:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm0[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm0[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,0,1,10,11,0,1,0,1,0,1,0,1,0,1]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
@@ -276,7 +276,7 @@ define <16 x i16> @shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_05_00_00_00_00_0
 ;
 ; XOPAVX1-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_05_00_00_00_00_00:
 ; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm0[0,0,2,3,4,5,6,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm0[0,0,0,0,4,5,6,7]
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; XOPAVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,0,1,10,11,0,1,0,1,0,1,0,1,0,1]
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
@@ -294,7 +294,7 @@ define <16 x i16> @shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_05_00_00_00_00_0
 define <16 x i16> @shuffle_v16i16_00_00_00_00_00_00_00_00_00_06_00_00_00_00_00_00(<16 x i16> %a, <16 x i16> %b) {
 ; AVX1-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_00_06_00_00_00_00_00_00:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm0[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm0[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,12,13,0,1,0,1,0,1,0,1,0,1,0,1]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
@@ -320,7 +320,7 @@ define <16 x i16> @shuffle_v16i16_00_00_00_00_00_00_00_00_00_06_00_00_00_00_00_0
 ;
 ; XOPAVX1-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_00_06_00_00_00_00_00_00:
 ; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm0[0,0,2,3,4,5,6,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm0[0,0,0,0,4,5,6,7]
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; XOPAVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,12,13,0,1,0,1,0,1,0,1,0,1,0,1]
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
@@ -338,7 +338,7 @@ define <16 x i16> @shuffle_v16i16_00_00_00_00_00_00_00_00_00_06_00_00_00_00_00_0
 define <16 x i16> @shuffle_v16i16_00_00_00_00_00_00_00_00_07_00_00_00_00_00_00_00(<16 x i16> %a, <16 x i16> %b) {
 ; AVX1-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_07_00_00_00_00_00_00_00:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm0[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm0[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[14,15,0,1,0,1,0,1,0,1,0,1,0,1,0,1]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
@@ -364,7 +364,7 @@ define <16 x i16> @shuffle_v16i16_00_00_00_00_00_00_00_00_07_00_00_00_00_00_00_0
 ;
 ; XOPAVX1-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_07_00_00_00_00_00_00_00:
 ; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm0[0,0,2,3,4,5,6,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm0[0,0,0,0,4,5,6,7]
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; XOPAVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[14,15,0,1,0,1,0,1,0,1,0,1,0,1,0,1]
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
@@ -386,7 +386,7 @@ define <16 x i16> @shuffle_v16i16_00_00_00_00_00_00_00_08_00_00_00_00_00_00_00_0
 ; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; AVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,0,0,1,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,1]
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
@@ -409,7 +409,7 @@ define <16 x i16> @shuffle_v16i16_00_00_00_00_00_00_00_08_00_00_00_00_00_00_00_0
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; XOPAVX1-NEXT:    vpperm {{.*#+}} xmm1 = xmm0[0,1,0,1,0,1,0,1,0,1,0,1,0,1],xmm1[0,1]
-; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; XOPAVX1-NEXT:    retq
@@ -432,7 +432,7 @@ define <16 x i16> @shuffle_v16i16_00_00_00_00_00_00_09_00_00_00_00_00_00_00_00_0
 ; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; AVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,0,3,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,1]
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
@@ -455,7 +455,7 @@ define <16 x i16> @shuffle_v16i16_00_00_00_00_00_00_09_00_00_00_00_00_00_00_00_0
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; XOPAVX1-NEXT:    vpperm {{.*#+}} xmm1 = xmm0[0,1,0,1,0,1,0,1,0,1,0,1],xmm1[2,3],xmm0[0,1]
-; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; XOPAVX1-NEXT:    retq
@@ -477,7 +477,7 @@ define <16 x i16> @shuffle_v16i16_00_00_00_00_00_10_00_00_00_00_00_00_00_00_00_0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; AVX1-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0,1,0,1,0,1,0,1,0,1,10,11,0,1,0,1]
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
@@ -506,7 +506,7 @@ define <16 x i16> @shuffle_v16i16_00_00_00_00_00_10_00_00_00_00_00_00_00_00_00_0
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; XOPAVX1-NEXT:    vpperm {{.*#+}} xmm1 = xmm0[0,1,0,1,0,1,0,1,0,1],xmm1[4,5],xmm0[0,1,0,1]
-; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; XOPAVX1-NEXT:    retq
@@ -527,7 +527,7 @@ define <16 x i16> @shuffle_v16i16_00_00_00_00_11_00_00_00_00_00_00_00_00_00_00_0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; AVX1-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0,1,0,1,0,1,0,1,14,15,0,1,0,1,0,1]
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
@@ -556,7 +556,7 @@ define <16 x i16> @shuffle_v16i16_00_00_00_00_11_00_00_00_00_00_00_00_00_00_00_0
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; XOPAVX1-NEXT:    vpperm {{.*#+}} xmm1 = xmm0[0,1,0,1,0,1,0,1],xmm1[6,7],xmm0[0,1,0,1,0,1]
-; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; XOPAVX1-NEXT:    retq
@@ -577,7 +577,7 @@ define <16 x i16> @shuffle_v16i16_00_00_00_12_00_00_00_00_00_00_00_00_00_00_00_0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0,1,2,3],xmm1[4,5,6,7]
 ; AVX1-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0,1,0,1,0,1,8,9,0,1,0,1,0,1,0,1]
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
@@ -598,7 +598,7 @@ define <16 x i16> @shuffle_v16i16_00_00_00_12_00_00_00_00_00_00_00_00_00_00_00_0
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; XOPAVX1-NEXT:    vpperm {{.*#+}} xmm1 = xmm0[0,1,0,1,0,1],xmm1[8,9],xmm0[0,1,0,1,0,1,0,1]
-; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; XOPAVX1-NEXT:    retq
@@ -618,7 +618,7 @@ define <16 x i16> @shuffle_v16i16_00_00_13_00_00_00_00_00_00_00_00_00_00_00_00_0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0,1,2,3],xmm1[4,5,6,7]
 ; AVX1-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0,1,0,1,10,11,0,1,0,1,0,1,0,1,0,1]
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
@@ -639,7 +639,7 @@ define <16 x i16> @shuffle_v16i16_00_00_13_00_00_00_00_00_00_00_00_00_00_00_00_0
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; XOPAVX1-NEXT:    vpperm {{.*#+}} xmm1 = xmm0[0,1,0,1],xmm1[10,11],xmm0[0,1,0,1,0,1,0,1,0,1]
-; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; XOPAVX1-NEXT:    retq
@@ -659,7 +659,7 @@ define <16 x i16> @shuffle_v16i16_00_14_00_00_00_00_00_00_00_00_00_00_00_00_00_0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0,1,2,3],xmm1[4,5,6,7]
 ; AVX1-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0,1,12,13,0,1,0,1,0,1,0,1,0,1,0,1]
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
@@ -680,7 +680,7 @@ define <16 x i16> @shuffle_v16i16_00_14_00_00_00_00_00_00_00_00_00_00_00_00_00_0
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; XOPAVX1-NEXT:    vpperm {{.*#+}} xmm1 = xmm0[0,1],xmm1[12,13],xmm0[0,1,0,1,0,1,0,1,0,1,0,1]
-; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; XOPAVX1-NEXT:    retq
@@ -700,7 +700,7 @@ define <16 x i16> @shuffle_v16i16_15_00_00_00_00_00_00_00_00_00_00_00_00_00_00_0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0,1,2,3],xmm1[4,5,6,7]
 ; AVX1-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[14,15,0,1,0,1,0,1,0,1,0,1,0,1,0,1]
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
@@ -721,7 +721,7 @@ define <16 x i16> @shuffle_v16i16_15_00_00_00_00_00_00_00_00_00_00_00_00_00_00_0
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; XOPAVX1-NEXT:    vpperm {{.*#+}} xmm1 = xmm1[14,15],xmm0[0,1,0,1,0,1,0,1,0,1,0,1,0,1]
-; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; XOPAVX1-NEXT:    retq
@@ -738,16 +738,16 @@ define <16 x i16> @shuffle_v16i16_15_00_00_00_00_00_00_00_00_00_00_00_00_00_00_0
 define <16 x i16> @shuffle_v16i16_00_00_00_00_00_00_00_00_08_08_08_08_08_08_08_08(<16 x i16> %a, <16 x i16> %b) {
 ; AVX1-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_08_08_08_08_08_08_08_08:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm0[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm0[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} ymm0 = ymm0[0,0,0,0,4,4,4,4]
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-SLOW-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_08_08_08_08_08_08_08_08:
 ; AVX2-SLOW:       # %bb.0:
-; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm0[0,0,2,3,4,5,6,7,8,8,10,11,12,13,14,15]
+; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm0[0,0,0,0,4,5,6,7,8,8,8,8,12,13,14,15]
 ; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[0,0,0,0,4,4,4,4]
 ; AVX2-SLOW-NEXT:    retq
 ;
@@ -758,7 +758,7 @@ define <16 x i16> @shuffle_v16i16_00_00_00_00_00_00_00_00_08_08_08_08_08_08_08_0
 ;
 ; AVX512VL-SLOW-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_08_08_08_08_08_08_08_08:
 ; AVX512VL-SLOW:       # %bb.0:
-; AVX512VL-SLOW-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm0[0,0,2,3,4,5,6,7,8,8,10,11,12,13,14,15]
+; AVX512VL-SLOW-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm0[0,0,0,0,4,5,6,7,8,8,8,8,12,13,14,15]
 ; AVX512VL-SLOW-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[0,0,0,0,4,4,4,4]
 ; AVX512VL-SLOW-NEXT:    retq
 ;
@@ -769,16 +769,16 @@ define <16 x i16> @shuffle_v16i16_00_00_00_00_00_00_00_00_08_08_08_08_08_08_08_0
 ;
 ; XOPAVX1-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_08_08_08_08_08_08_08_08:
 ; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm0[0,0,2,3,4,5,6,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm0[0,0,0,0,4,5,6,7]
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; XOPAVX1-NEXT:    vpermilps {{.*#+}} ymm0 = ymm0[0,0,0,0,4,4,4,4]
 ; XOPAVX1-NEXT:    retq
 ;
 ; XOPAVX2-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_08_08_08_08_08_08_08_08:
 ; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm0[0,0,2,3,4,5,6,7,8,8,10,11,12,13,14,15]
+; XOPAVX2-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm0[0,0,0,0,4,5,6,7,8,8,8,8,12,13,14,15]
 ; XOPAVX2-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[0,0,0,0,4,4,4,4]
 ; XOPAVX2-NEXT:    retq
   %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8>
@@ -788,16 +788,16 @@ define <16 x i16> @shuffle_v16i16_00_00_00_00_00_00_00_00_08_08_08_08_08_08_08_0
 define <16 x i16> @shuffle_v16i16_07_07_07_07_07_07_07_07_15_15_15_15_15_15_15_15(<16 x i16> %a, <16 x i16> %b) {
 ; AVX1-LABEL: shuffle_v16i16_07_07_07_07_07_07_07_07_15_15_15_15_15_15_15_15:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpshufhw {{.*#+}} xmm1 = xmm0[0,1,2,3,7,7,6,7]
+; AVX1-NEXT:    vpshufhw {{.*#+}} xmm1 = xmm0[0,1,2,3,7,7,7,7]
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,7,6,7]
+; AVX1-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,7,7,7]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} ymm0 = ymm0[2,2,2,2,6,6,6,6]
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-SLOW-LABEL: shuffle_v16i16_07_07_07_07_07_07_07_07_15_15_15_15_15_15_15_15:
 ; AVX2-SLOW:       # %bb.0:
-; AVX2-SLOW-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,7,7,6,7,8,9,10,11,15,15,14,15]
+; AVX2-SLOW-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,7,7,7,7,8,9,10,11,15,15,15,15]
 ; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[2,2,2,2,6,6,6,6]
 ; AVX2-SLOW-NEXT:    retq
 ;
@@ -808,7 +808,7 @@ define <16 x i16> @shuffle_v16i16_07_07_07_07_07_07_07_07_15_15_15_15_15_15_15_1
 ;
 ; AVX512VL-SLOW-LABEL: shuffle_v16i16_07_07_07_07_07_07_07_07_15_15_15_15_15_15_15_15:
 ; AVX512VL-SLOW:       # %bb.0:
-; AVX512VL-SLOW-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,7,7,6,7,8,9,10,11,15,15,14,15]
+; AVX512VL-SLOW-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,7,7,7,7,8,9,10,11,15,15,15,15]
 ; AVX512VL-SLOW-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[2,2,2,2,6,6,6,6]
 ; AVX512VL-SLOW-NEXT:    retq
 ;
@@ -819,16 +819,16 @@ define <16 x i16> @shuffle_v16i16_07_07_07_07_07_07_07_07_15_15_15_15_15_15_15_1
 ;
 ; XOPAVX1-LABEL: shuffle_v16i16_07_07_07_07_07_07_07_07_15_15_15_15_15_15_15_15:
 ; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vpshufhw {{.*#+}} xmm1 = xmm0[0,1,2,3,7,7,6,7]
+; XOPAVX1-NEXT:    vpshufhw {{.*#+}} xmm1 = xmm0[0,1,2,3,7,7,7,7]
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; XOPAVX1-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,7,6,7]
+; XOPAVX1-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,7,7,7]
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; XOPAVX1-NEXT:    vpermilps {{.*#+}} ymm0 = ymm0[2,2,2,2,6,6,6,6]
 ; XOPAVX1-NEXT:    retq
 ;
 ; XOPAVX2-LABEL: shuffle_v16i16_07_07_07_07_07_07_07_07_15_15_15_15_15_15_15_15:
 ; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,7,7,6,7,8,9,10,11,15,15,14,15]
+; XOPAVX2-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,7,7,7,7,8,9,10,11,15,15,15,15]
 ; XOPAVX2-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[2,2,2,2,6,6,6,6]
 ; XOPAVX2-NEXT:    retq
   %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
@@ -2526,10 +2526,10 @@ define <16 x i16> @shuffle_v16i16_04_04_04_04_00_00_00_00_08_08_08_08_12_12_12_1
 define <16 x i16> @shuffle_v16i16_00_uu_uu_00_00_00_00_00_08_08_uu_uu_08_08_14_08(<16 x i16> %a, <16 x i16> %b) {
 ; AVX1-LABEL: shuffle_v16i16_00_uu_uu_00_00_00_00_00_08_08_uu_uu_08_08_14_08:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm0[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm0[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,0,1,4,5,6,7,0,1,0,1,12,13,0,1]
+; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,0,1,0,1,0,1,0,1,0,1,12,13,0,1]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
 ;
@@ -2540,10 +2540,10 @@ define <16 x i16> @shuffle_v16i16_00_uu_uu_00_00_00_00_00_08_08_uu_uu_08_08_14_0
 ;
 ; XOPAVX1-LABEL: shuffle_v16i16_00_uu_uu_00_00_00_00_00_08_08_uu_uu_08_08_14_08:
 ; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm0[0,0,2,3,4,5,6,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm0[0,0,0,0,4,5,6,7]
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; XOPAVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,0,1,4,5,6,7,0,1,0,1,12,13,0,1]
+; XOPAVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,0,1,0,1,0,1,0,1,0,1,12,13,0,1]
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; XOPAVX1-NEXT:    retq
 ;
@@ -2560,7 +2560,7 @@ define <16 x i16> @shuffle_v16i16_07_uu_00_00_00_00_00_00_08_08_uu_uu_08_08_08_1
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpshufb {{.*#+}} xmm1 = xmm0[14,15,2,3,0,1,0,1,0,1,0,1,0,1,0,1]
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,0,1,4,5,6,7,0,1,0,1,0,1,14,15]
+; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,0,1,0,1,0,1,0,1,0,1,0,1,14,15]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
 ;
@@ -2573,7 +2573,7 @@ define <16 x i16> @shuffle_v16i16_07_uu_00_00_00_00_00_00_08_08_uu_uu_08_08_08_1
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpshufb {{.*#+}} xmm1 = xmm0[14,15,2,3,0,1,0,1,0,1,0,1,0,1,0,1]
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; XOPAVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,0,1,4,5,6,7,0,1,0,1,0,1,14,15]
+; XOPAVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,0,1,0,1,0,1,0,1,0,1,0,1,14,15]
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; XOPAVX1-NEXT:    retq
 ;
@@ -2620,10 +2620,10 @@ define <16 x i16> @shuffle_v16i16_00_uu_uu_02_04_04_uu_06_14_14_uu_12_10_10_08_0
 define <16 x i16> @shuffle_v16i16_04_04_04_04_uu_uu_uu_uu_08_08_08_uu_uu_12_12_12(<16 x i16> %a, <16 x i16> %b) {
 ; AVX1-LABEL: shuffle_v16i16_04_04_04_04_uu_uu_uu_uu_08_08_08_uu_uu_12_12_12:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpshufhw {{.*#+}} xmm1 = xmm0[0,1,2,3,4,4,6,7]
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,2,2,3]
+; AVX1-NEXT:    vpshufhw {{.*#+}} xmm1 = xmm0[0,1,2,3,4,4,4,4]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,2,2,2]
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,4,4,4]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
@@ -2635,10 +2635,10 @@ define <16 x i16> @shuffle_v16i16_04_04_04_04_uu_uu_uu_uu_08_08_08_uu_uu_12_12_1
 ;
 ; XOPAVX1-LABEL: shuffle_v16i16_04_04_04_04_uu_uu_uu_uu_08_08_08_uu_uu_12_12_12:
 ; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vpshufhw {{.*#+}} xmm1 = xmm0[0,1,2,3,4,4,6,7]
-; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,2,2,3]
+; XOPAVX1-NEXT:    vpshufhw {{.*#+}} xmm1 = xmm0[0,1,2,3,4,4,4,4]
+; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,2,2,2]
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,3,4,5,6,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; XOPAVX1-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,4,4,4]
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; XOPAVX1-NEXT:    retq
@@ -3107,7 +3107,7 @@ define <16 x i16> @shuffle_v16i16_16_zz_zz_zz_17_zz_zz_zz_18_zz_zz_zz_19_zz_zz_z
 ; AVX1-LABEL: shuffle_v16i16_16_zz_zz_zz_17_zz_zz_zz_18_zz_zz_zz_19_zz_zz_zz:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
@@ -3120,7 +3120,7 @@ define <16 x i16> @shuffle_v16i16_16_zz_zz_zz_17_zz_zz_zz_18_zz_zz_zz_19_zz_zz_z
 ; XOPAVX1-LABEL: shuffle_v16i16_16_zz_zz_zz_17_zz_zz_zz_18_zz_zz_zz_19_zz_zz_zz:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; XOPAVX1-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; XOPAVX1-NEXT:    retq
@@ -3169,7 +3169,7 @@ define <16 x i16> @shuffle_v16i16_28_zz_zz_zz_29_zz_zz_zz_30_zz_zz_zz_31_zz_zz_z
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
@@ -3193,7 +3193,7 @@ define <16 x i16> @shuffle_v16i16_28_zz_zz_zz_29_zz_zz_zz_30_zz_zz_zz_31_zz_zz_z
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; XOPAVX1-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
-; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; XOPAVX1-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; XOPAVX1-NEXT:    retq
@@ -3543,7 +3543,7 @@ define <16 x i16> @shuffle_v16i16_00_00_00_00_00_00_00_08_08_08_08_08_08_08_08_0
 ; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,1,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,1]
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
@@ -3552,7 +3552,7 @@ define <16 x i16> @shuffle_v16i16_00_00_00_00_00_00_00_08_08_08_08_08_08_08_08_0
 ; AVX2-SLOW:       # %bb.0:
 ; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm1 = ymm0[2,3,2,3]
 ; AVX2-SLOW-NEXT:    vpslldq {{.*#+}} ymm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm1[0,1],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm1[16,17]
-; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm0[0,0,2,3,4,5,6,7,8,8,10,11,12,13,14,15]
+; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm0[0,0,0,0,4,5,6,7,8,8,8,8,12,13,14,15]
 ; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[0,0,0,0,4,4,4,4]
 ; AVX2-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6],ymm1[7],ymm0[8,9,10,11,12,13,14],ymm1[15]
 ; AVX2-SLOW-NEXT:    retq
@@ -3575,7 +3575,7 @@ define <16 x i16> @shuffle_v16i16_00_00_00_00_00_00_00_08_08_08_08_08_08_08_08_0
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; XOPAVX1-NEXT:    vpperm {{.*#+}} xmm0 = xmm0[0,1,0,1,0,1,0,1,0,1,0,1,0,1],xmm1[0,1]
-; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,0,2,3,4,5,6,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,0,0,0,4,5,6,7]
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; XOPAVX1-NEXT:    retq
@@ -3584,7 +3584,7 @@ define <16 x i16> @shuffle_v16i16_00_00_00_00_00_00_00_08_08_08_08_08_08_08_08_0
 ; XOPAVX2:       # %bb.0:
 ; XOPAVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm0[2,3,2,3]
 ; XOPAVX2-NEXT:    vpslldq {{.*#+}} ymm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm1[0,1],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm1[16,17]
-; XOPAVX2-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm0[0,0,2,3,4,5,6,7,8,8,10,11,12,13,14,15]
+; XOPAVX2-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm0[0,0,0,0,4,5,6,7,8,8,8,8,12,13,14,15]
 ; XOPAVX2-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[0,0,0,0,4,4,4,4]
 ; XOPAVX2-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6],ymm1[7],ymm0[8,9,10,11,12,13,14],ymm1[15]
 ; XOPAVX2-NEXT:    retq
@@ -3598,7 +3598,7 @@ define <16 x i16> @shuffle_v16i16_00_00_00_00_04_04_04_12_08_08_08_08_12_12_12_1
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpsllq $48, %xmm1, %xmm2
 ; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
-; AVX1-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,4,4,7]
+; AVX1-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,4,4,4]
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,6],xmm2[7]
 ; AVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,4,4,4]
@@ -3610,7 +3610,7 @@ define <16 x i16> @shuffle_v16i16_00_00_00_00_04_04_04_12_08_08_08_08_12_12_12_1
 ; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm1 = ymm0[2,3,2,3]
 ; AVX2-SLOW-NEXT:    vpsllq $48, %ymm1, %ymm1
 ; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm0[0,0,0,0,4,5,6,7,8,8,8,8,12,13,14,15]
-; AVX2-SLOW-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,4,4,4,7,8,9,10,11,12,12,12,15]
+; AVX2-SLOW-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,4,4,4,4,8,9,10,11,12,12,12,12]
 ; AVX2-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6],ymm1[7],ymm0[8,9,10,11,12,13,14],ymm1[15]
 ; AVX2-SLOW-NEXT:    retq
 ;
@@ -3642,7 +3642,7 @@ define <16 x i16> @shuffle_v16i16_00_00_00_00_04_04_04_12_08_08_08_08_12_12_12_1
 ; XOPAVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm0[2,3,2,3]
 ; XOPAVX2-NEXT:    vpsllq $48, %ymm1, %ymm1
 ; XOPAVX2-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm0[0,0,0,0,4,5,6,7,8,8,8,8,12,13,14,15]
-; XOPAVX2-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,4,4,4,7,8,9,10,11,12,12,12,15]
+; XOPAVX2-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,4,4,4,4,8,9,10,11,12,12,12,12]
 ; XOPAVX2-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6],ymm1[7],ymm0[8,9,10,11,12,13,14],ymm1[15]
 ; XOPAVX2-NEXT:    retq
   %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 0, i32 0, i32 0, i32 0, i32 4, i32 4, i32 4, i32 12, i32 8, i32 8, i32 8, i32 8, i32 12, i32 12, i32 12, i32 12>
@@ -5214,7 +5214,7 @@ define <16 x i16> @shuffle_v16i16_00_16_01_17_06_22_07_31_08_24_09_25_14_30_15_3
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[0,3,2,3]
 ; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
 ; AVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,0,2,1,4,5,6,7]
-; AVX1-NEXT:    vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,6,6,7]
+; AVX1-NEXT:    vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,6,6,6]
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5],xmm2[6,7]
 ; AVX1-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[2,3,6,7,10,11,14,15,u,u,u,u,u,u,u,u]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,3,2,3]
@@ -5226,7 +5226,7 @@ define <16 x i16> @shuffle_v16i16_00_16_01_17_06_22_07_31_08_24_09_25_14_30_15_3
 ; AVX2-SLOW:       # %bb.0:
 ; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm2 = ymm1[2,3,2,3]
 ; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} ymm1 = ymm1[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]
-; AVX2-SLOW-NEXT:    vpshufhw {{.*#+}} ymm1 = ymm1[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]
+; AVX2-SLOW-NEXT:    vpshufhw {{.*#+}} ymm1 = ymm1[0,1,2,3,6,6,6,6,8,9,10,11,14,14,14,14]
 ; AVX2-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2],ymm2[3],ymm1[4,5,6],ymm2[7]
 ; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm0[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]
 ; AVX2-SLOW-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,6,5,7,7,8,9,10,11,14,13,15,15]
@@ -5236,7 +5236,7 @@ define <16 x i16> @shuffle_v16i16_00_16_01_17_06_22_07_31_08_24_09_25_14_30_15_3
 ; AVX2-FAST-LABEL: shuffle_v16i16_00_16_01_17_06_22_07_31_08_24_09_25_14_30_15_31:
 ; AVX2-FAST:       # %bb.0:
 ; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm2 = ymm1[2,3,2,3]
-; AVX2-FAST-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[0,1,0,1,4,5,2,3,8,9,12,13,u,u,u,u,16,17,16,17,20,21,18,19,24,25,28,29,u,u,u,u]
+; AVX2-FAST-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[0,1,0,1,4,5,2,3,12,13,12,13,u,u,u,u,16,17,16,17,20,21,18,19,28,29,28,29,u,u,u,u]
 ; AVX2-FAST-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2],ymm2[3],ymm1[4,5,6],ymm2[7]
 ; AVX2-FAST-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,u,u,2,3,u,u,12,13,u,u,14,15,u,u,16,17,u,u,18,19,u,u,28,29,u,u,30,31,u,u]
 ; AVX2-FAST-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4],ymm1[5],ymm0[6],ymm1[7],ymm0[8],ymm1[9],ymm0[10],ymm1[11],ymm0[12],ymm1[13],ymm0[14],ymm1[15]
@@ -5263,7 +5263,7 @@ define <16 x i16> @shuffle_v16i16_00_16_01_17_06_22_07_31_08_24_09_25_14_30_15_3
 ; XOPAVX2:       # %bb.0:
 ; XOPAVX2-NEXT:    vpermq {{.*#+}} ymm2 = ymm1[2,3,2,3]
 ; XOPAVX2-NEXT:    vpshuflw {{.*#+}} ymm1 = ymm1[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]
-; XOPAVX2-NEXT:    vpshufhw {{.*#+}} ymm1 = ymm1[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]
+; XOPAVX2-NEXT:    vpshufhw {{.*#+}} ymm1 = ymm1[0,1,2,3,6,6,6,6,8,9,10,11,14,14,14,14]
 ; XOPAVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2],ymm2[3],ymm1[4,5,6],ymm2[7]
 ; XOPAVX2-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm0[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]
 ; XOPAVX2-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,6,5,7,7,8,9,10,11,14,13,15,15]
@@ -5654,7 +5654,7 @@ define <16 x i16> @shuffle_v16i16_00_03_02_21_uu_uu_uu_uu_08_11_10_29_uu_uu_uu_u
 define <16 x i16> @shuffle_v16i16_uu_uu_uu_21_uu_uu_uu_uu_uu_uu_uu_29_uu_uu_uu_uu(<16 x i16> %a, <16 x i16> %b) {
 ; ALL-LABEL: shuffle_v16i16_uu_uu_uu_21_uu_uu_uu_uu_uu_uu_uu_29_uu_uu_uu_uu:
 ; ALL:       # %bb.0:
-; ALL-NEXT:    vpermilps {{.*#+}} ymm0 = ymm1[0,2,2,3,4,6,6,7]
+; ALL-NEXT:    vpermilps {{.*#+}} ymm0 = ymm1[2,2,2,2,6,6,6,6]
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 undef, i32 undef, i32 undef, i32 21, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 29, i32 undef, i32 undef, i32 undef, i32 undef>
   ret <16 x i16> %shuffle
@@ -5674,7 +5674,7 @@ define <16 x i16> @shuffle_v16i16_00_01_02_21_uu_uu_uu_uu_08_09_10_29_uu_uu_uu_u
 ;
 ; AVX2-LABEL: shuffle_v16i16_00_01_02_21_uu_uu_uu_uu_08_09_10_29_uu_uu_uu_uu:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[0,2,2,3,4,6,6,7]
+; AVX2-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[2,2,2,2,6,6,6,6]
 ; AVX2-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3],ymm0[4,5,6,7,8,9,10],ymm1[11],ymm0[12,13,14,15]
 ; AVX2-NEXT:    retq
 ;
@@ -5697,7 +5697,7 @@ define <16 x i16> @shuffle_v16i16_00_01_02_21_uu_uu_uu_uu_08_09_10_29_uu_uu_uu_u
 ;
 ; XOPAVX2-LABEL: shuffle_v16i16_00_01_02_21_uu_uu_uu_uu_08_09_10_29_uu_uu_uu_uu:
 ; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[0,2,2,3,4,6,6,7]
+; XOPAVX2-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[2,2,2,2,6,6,6,6]
 ; XOPAVX2-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3],ymm0[4,5,6,7,8,9,10],ymm1[11],ymm0[12,13,14,15]
 ; XOPAVX2-NEXT:    retq
   %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 0, i32 1, i32 2, i32 21, i32 undef, i32 undef, i32 undef, i32 undef, i32 8, i32 9, i32 10, i32 29, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -5887,12 +5887,12 @@ define <16 x i16> @shuffle_v16i16_uu_uu_uu_01_uu_05_07_25_uu_uu_uu_09_uu_13_15_2
 ; AVX1-LABEL: shuffle_v16i16_uu_uu_uu_01_uu_05_07_25_uu_uu_uu_09_uu_13_15_25:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,1,2,0]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm2[0,1,2,1,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm2[1,1,1,1,4,5,6,7]
 ; AVX1-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,7,7]
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,6],xmm1[7]
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,1,2,1,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[1,1,1,1,4,5,6,7]
 ; AVX1-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,7,7]
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,6],xmm1[7]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
@@ -5901,18 +5901,17 @@ define <16 x i16> @shuffle_v16i16_uu_uu_uu_01_uu_05_07_25_uu_uu_uu_09_uu_13_15_2
 ; AVX2-SLOW-LABEL: shuffle_v16i16_uu_uu_uu_01_uu_05_07_25_uu_uu_uu_09_uu_13_15_25:
 ; AVX2-SLOW:       # %bb.0:
 ; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,3,2,3]
-; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[0,1,2,0,4,5,6,4]
-; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm0[0,1,2,1,4,5,6,7,8,9,10,9,12,13,14,15]
+; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[0,0,0,0,4,4,4,4]
+; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm0[1,1,1,1,4,5,6,7,9,9,9,9,12,13,14,15]
 ; AVX2-SLOW-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,7,7,8,9,10,11,12,13,15,15]
 ; AVX2-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6],ymm1[7],ymm0[8,9,10,11,12,13,14],ymm1[15]
 ; AVX2-SLOW-NEXT:    retq
 ;
 ; AVX2-FAST-LABEL: shuffle_v16i16_uu_uu_uu_01_uu_05_07_25_uu_uu_uu_09_uu_13_15_25:
 ; AVX2-FAST:       # %bb.0:
-; AVX2-FAST-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [4,5,6,4,4,5,6,4]
-; AVX2-FAST-NEXT:    # ymm2 = mem[0,1,0,1]
+; AVX2-FAST-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [4,4,4,4,4,4,4,4]
 ; AVX2-FAST-NEXT:    vpermd %ymm1, %ymm2, %ymm1
-; AVX2-FAST-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,2,3,8,9,10,11,14,15,u,u,16,17,18,19,20,21,18,19,24,25,26,27,30,31,u,u]
+; AVX2-FAST-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[2,3,2,3,2,3,2,3,8,9,10,11,14,15,u,u,18,19,18,19,18,19,18,19,24,25,26,27,30,31,u,u]
 ; AVX2-FAST-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6],ymm1[7],ymm0[8,9,10,11,12,13,14],ymm1[15]
 ; AVX2-FAST-NEXT:    retq
 ;
@@ -5926,8 +5925,8 @@ define <16 x i16> @shuffle_v16i16_uu_uu_uu_01_uu_05_07_25_uu_uu_uu_09_uu_13_15_2
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
-; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,1,2,0]
-; XOPAVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,1,2,3,4,5,2,3,8,9,10,11,14,15,30,31]
+; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
+; XOPAVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [2,3,2,3,2,3,2,3,8,9,10,11,14,15,30,31]
 ; XOPAVX1-NEXT:    vpperm %xmm3, %xmm1, %xmm2, %xmm2
 ; XOPAVX1-NEXT:    vpperm %xmm3, %xmm1, %xmm0, %xmm0
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
@@ -5936,8 +5935,8 @@ define <16 x i16> @shuffle_v16i16_uu_uu_uu_01_uu_05_07_25_uu_uu_uu_09_uu_13_15_2
 ; XOPAVX2-LABEL: shuffle_v16i16_uu_uu_uu_01_uu_05_07_25_uu_uu_uu_09_uu_13_15_25:
 ; XOPAVX2:       # %bb.0:
 ; XOPAVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,3,2,3]
-; XOPAVX2-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[0,1,2,0,4,5,6,4]
-; XOPAVX2-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm0[0,1,2,1,4,5,6,7,8,9,10,9,12,13,14,15]
+; XOPAVX2-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[0,0,0,0,4,4,4,4]
+; XOPAVX2-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm0[1,1,1,1,4,5,6,7,9,9,9,9,12,13,14,15]
 ; XOPAVX2-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,7,7,8,9,10,11,12,13,15,15]
 ; XOPAVX2-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6],ymm1[7],ymm0[8,9,10,11,12,13,14],ymm1[15]
 ; XOPAVX2-NEXT:    retq
@@ -5963,7 +5962,7 @@ define <16 x i16> @shuffle_v16i16_uu_uu_04_uu_16_18_20_uu_uu_uu_12_uu_24_26_28_u
 ; AVX2-LABEL: shuffle_v16i16_uu_uu_04_uu_16_18_20_uu_uu_uu_12_uu_24_26_28_uu:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[u,u,u,u,u,u,u,u,0,1,4,5,8,9,4,5,u,u,u,u,u,u,u,u,16,17,20,21,24,25,20,21]
-; AVX2-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[0,2,2,3,4,6,6,7]
+; AVX2-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[2,2,2,2,6,6,6,6]
 ; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
 ; AVX2-NEXT:    retq
 ;
@@ -5987,7 +5986,7 @@ define <16 x i16> @shuffle_v16i16_uu_uu_04_uu_16_18_20_uu_uu_uu_12_uu_24_26_28_u
 ; XOPAVX2-LABEL: shuffle_v16i16_uu_uu_04_uu_16_18_20_uu_uu_uu_12_uu_24_26_28_uu:
 ; XOPAVX2:       # %bb.0:
 ; XOPAVX2-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[u,u,u,u,u,u,u,u,0,1,4,5,8,9,4,5,u,u,u,u,u,u,u,u,16,17,20,21,24,25,20,21]
-; XOPAVX2-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[0,2,2,3,4,6,6,7]
+; XOPAVX2-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[2,2,2,2,6,6,6,6]
 ; XOPAVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
 ; XOPAVX2-NEXT:    retq
   %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 undef, i32 undef, i32 4, i32 undef, i32 16, i32 18, i32 20, i32 undef, i32 undef, i32 undef, i32 12, i32 undef, i32 24, i32 26, i32 28, i32 undef>
@@ -6631,14 +6630,14 @@ define <16 x i16> @shuffle_v16i16_u_u_u_u_u_u_u_u_0_16_1_17_2_18_3_19(<16 x i16>
 define <16 x i16> @shuffle_v16i16_u_u_u_u_u_u_u_u_3_3_3_3_3_3_3_3(<16 x i16> %a, <16 x i16> %b) {
 ; AVX1-LABEL: shuffle_v16i16_u_u_u_u_u_u_u_u_3_3_3_3_3_3_3_3:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[3,3,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[3,3,3,3,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-SLOW-LABEL: shuffle_v16i16_u_u_u_u_u_u_u_u_3_3_3_3_3_3_3_3:
 ; AVX2-SLOW:       # %bb.0:
-; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[3,3,2,3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[3,3,3,3,4,5,6,7]
 ; AVX2-SLOW-NEXT:    vpbroadcastd %xmm0, %ymm0
 ; AVX2-SLOW-NEXT:    retq
 ;
@@ -6650,7 +6649,7 @@ define <16 x i16> @shuffle_v16i16_u_u_u_u_u_u_u_u_3_3_3_3_3_3_3_3(<16 x i16> %a,
 ;
 ; AVX512VL-SLOW-LABEL: shuffle_v16i16_u_u_u_u_u_u_u_u_3_3_3_3_3_3_3_3:
 ; AVX512VL-SLOW:       # %bb.0:
-; AVX512VL-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[3,3,2,3,4,5,6,7]
+; AVX512VL-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[3,3,3,3,4,5,6,7]
 ; AVX512VL-SLOW-NEXT:    vpbroadcastd %xmm0, %ymm0
 ; AVX512VL-SLOW-NEXT:    retq
 ;
@@ -6662,14 +6661,14 @@ define <16 x i16> @shuffle_v16i16_u_u_u_u_u_u_u_u_3_3_3_3_3_3_3_3(<16 x i16> %a,
 ;
 ; XOPAVX1-LABEL: shuffle_v16i16_u_u_u_u_u_u_u_u_3_3_3_3_3_3_3_3:
 ; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[3,3,2,3,4,5,6,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[3,3,3,3,4,5,6,7]
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; XOPAVX1-NEXT:    retq
 ;
 ; XOPAVX2-LABEL: shuffle_v16i16_u_u_u_u_u_u_u_u_3_3_3_3_3_3_3_3:
 ; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[3,3,2,3,4,5,6,7]
+; XOPAVX2-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[3,3,3,3,4,5,6,7]
 ; XOPAVX2-NEXT:    vpbroadcastd %xmm0, %ymm0
 ; XOPAVX2-NEXT:    retq
   %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
@@ -6680,7 +6679,7 @@ define <16 x i16> @shuffle_v16i16_8_8_8_8_8_8_8_8_8_8_8_8_8_8_8_8(<16 x i16> %a,
 ; AVX1-LABEL: shuffle_v16i16_8_8_8_8_8_8_8_8_8_8_8_8_8_8_8_8:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
@@ -6694,7 +6693,7 @@ define <16 x i16> @shuffle_v16i16_8_8_8_8_8_8_8_8_8_8_8_8_8_8_8_8(<16 x i16> %a,
 ; XOPAVX1-LABEL: shuffle_v16i16_8_8_8_8_8_8_8_8_8_8_8_8_8_8_8_8:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; XOPAVX1-NEXT:    retq
@@ -6720,13 +6719,13 @@ define <16 x i16> @shuffle_v16i16_4_20_5_21_6_22_7_23_u_u_u_u_u_u_u_u(<16 x i16>
 define <16 x i16> @shuffle_v16i16_3_3_3_3_3_3_3_3_u_u_u_u_u_u_u_u(<16 x i16> %a, <16 x i16> %b) {
 ; AVX1-LABEL: shuffle_v16i16_3_3_3_3_3_3_3_3_u_u_u_u_u_u_u_u:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[3,3,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[3,3,3,3,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-SLOW-LABEL: shuffle_v16i16_3_3_3_3_3_3_3_3_u_u_u_u_u_u_u_u:
 ; AVX2-SLOW:       # %bb.0:
-; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[3,3,2,3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[3,3,3,3,4,5,6,7]
 ; AVX2-SLOW-NEXT:    vpbroadcastd %xmm0, %xmm0
 ; AVX2-SLOW-NEXT:    retq
 ;
@@ -6737,7 +6736,7 @@ define <16 x i16> @shuffle_v16i16_3_3_3_3_3_3_3_3_u_u_u_u_u_u_u_u(<16 x i16> %a,
 ;
 ; AVX512VL-SLOW-LABEL: shuffle_v16i16_3_3_3_3_3_3_3_3_u_u_u_u_u_u_u_u:
 ; AVX512VL-SLOW:       # %bb.0:
-; AVX512VL-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[3,3,2,3,4,5,6,7]
+; AVX512VL-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[3,3,3,3,4,5,6,7]
 ; AVX512VL-SLOW-NEXT:    vpbroadcastd %xmm0, %xmm0
 ; AVX512VL-SLOW-NEXT:    retq
 ;
@@ -6748,13 +6747,13 @@ define <16 x i16> @shuffle_v16i16_3_3_3_3_3_3_3_3_u_u_u_u_u_u_u_u(<16 x i16> %a,
 ;
 ; XOPAVX1-LABEL: shuffle_v16i16_3_3_3_3_3_3_3_3_u_u_u_u_u_u_u_u:
 ; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[3,3,2,3,4,5,6,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[3,3,3,3,4,5,6,7]
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; XOPAVX1-NEXT:    retq
 ;
 ; XOPAVX2-LABEL: shuffle_v16i16_3_3_3_3_3_3_3_3_u_u_u_u_u_u_u_u:
 ; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[3,3,2,3,4,5,6,7]
+; XOPAVX2-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[3,3,3,3,4,5,6,7]
 ; XOPAVX2-NEXT:    vpbroadcastd %xmm0, %xmm0
 ; XOPAVX2-NEXT:    retq
   %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -6765,14 +6764,14 @@ define <16 x i16> @shuffle_v16i16_9_9_9_9_9_9_9_9_u_u_u_u_u_u_u_u(<16 x i16> %a,
 ; AVX1-LABEL: shuffle_v16i16_9_9_9_9_9_9_9_9_u_u_u_u_u_u_u_u:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[1,1,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[1,1,1,1,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-SLOW-LABEL: shuffle_v16i16_9_9_9_9_9_9_9_9_u_u_u_u_u_u_u_u:
 ; AVX2-SLOW:       # %bb.0:
 ; AVX2-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[1,1,2,3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[1,1,1,1,4,5,6,7]
 ; AVX2-SLOW-NEXT:    vpbroadcastd %xmm0, %xmm0
 ; AVX2-SLOW-NEXT:    retq
 ;
@@ -6785,7 +6784,7 @@ define <16 x i16> @shuffle_v16i16_9_9_9_9_9_9_9_9_u_u_u_u_u_u_u_u(<16 x i16> %a,
 ; AVX512VL-SLOW-LABEL: shuffle_v16i16_9_9_9_9_9_9_9_9_u_u_u_u_u_u_u_u:
 ; AVX512VL-SLOW:       # %bb.0:
 ; AVX512VL-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; AVX512VL-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[1,1,2,3,4,5,6,7]
+; AVX512VL-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[1,1,1,1,4,5,6,7]
 ; AVX512VL-SLOW-NEXT:    vpbroadcastd %xmm0, %xmm0
 ; AVX512VL-SLOW-NEXT:    retq
 ;
@@ -6798,14 +6797,14 @@ define <16 x i16> @shuffle_v16i16_9_9_9_9_9_9_9_9_u_u_u_u_u_u_u_u(<16 x i16> %a,
 ; XOPAVX1-LABEL: shuffle_v16i16_9_9_9_9_9_9_9_9_u_u_u_u_u_u_u_u:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[1,1,2,3,4,5,6,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[1,1,1,1,4,5,6,7]
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; XOPAVX1-NEXT:    retq
 ;
 ; XOPAVX2-LABEL: shuffle_v16i16_9_9_9_9_9_9_9_9_u_u_u_u_u_u_u_u:
 ; XOPAVX2:       # %bb.0:
 ; XOPAVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; XOPAVX2-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[1,1,2,3,4,5,6,7]
+; XOPAVX2-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[1,1,1,1,4,5,6,7]
 ; XOPAVX2-NEXT:    vpbroadcastd %xmm0, %xmm0
 ; XOPAVX2-NEXT:    retq
   %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 9, i32 9, i32 9, i32 9, i32 9, i32 9, i32 9, i32 9, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -7265,12 +7264,12 @@ define <16 x i16> @PR24935(<16 x i16> %a, <16 x i16> %b) {
 ; AVX1-NEXT:    vpalignr {{.*#+}} xmm4 = xmm3[2,3,4,5,6,7,8,9,10,11,12,13,14,15,0,1]
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm4[0,1],xmm2[2],xmm4[3,4,5,6,7]
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
-; AVX1-NEXT:    vpshufhw {{.*#+}} xmm5 = xmm4[0,1,2,3,5,5,6,7]
+; AVX1-NEXT:    vpshufhw {{.*#+}} xmm5 = xmm4[0,1,2,3,5,5,5,5]
 ; AVX1-NEXT:    vpshufb {{.*#+}} xmm6 = xmm0[2,3,u,u,u,u,u,u,u,u,8,9,0,1,u,u]
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm5 = xmm6[0],xmm5[1],xmm6[2,3],xmm5[4],xmm6[5,6,7]
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm5[0,1],xmm2[2,3],xmm5[4,5,6],xmm2[7]
 ; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
-; AVX1-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,7,7]
+; AVX1-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,7,7,7]
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3,4,5],xmm1[6,7]
 ; AVX1-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[6,7,4,5,u,u,10,11,4,5,14,15,u,u,0,1]
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2],xmm1[3,4,5],xmm0[6],xmm1[7]
@@ -7279,13 +7278,13 @@ define <16 x i16> @PR24935(<16 x i16> %a, <16 x i16> %b) {
 ;
 ; AVX2-SLOW-LABEL: PR24935:
 ; AVX2-SLOW:       # %bb.0:
-; AVX2-SLOW-NEXT:    vpshufb {{.*#+}} ymm2 = ymm1[8,9,10,11,4,5,8,9,0,1,14,15,12,13,0,1,24,25,26,27,20,21,24,25,16,17,30,31,28,29,16,17]
+; AVX2-SLOW-NEXT:    vpshufb {{.*#+}} ymm2 = ymm1[8,9,8,9,8,9,8,9,0,1,14,15,12,13,0,1,24,25,24,25,24,25,24,25,16,17,30,31,28,29,16,17]
 ; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,3,0,1]
-; AVX2-SLOW-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[6,7,4,5,0,1,10,11,4,5,10,11,4,5,6,7,22,23,20,21,16,17,26,27,20,21,26,27,20,21,22,23]
+; AVX2-SLOW-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[6,7,4,5,0,1,10,11,4,5,4,5,4,5,4,5,22,23,20,21,16,17,26,27,20,21,20,21,20,21,20,21]
 ; AVX2-SLOW-NEXT:    vmovdqa {{.*#+}} ymm3 = <255,255,255,255,u,u,255,255,255,255,0,0,u,u,0,0,u,u,u,u,255,255,0,0,u,u,u,u,u,u,0,0>
 ; AVX2-SLOW-NEXT:    vpblendvb %ymm3, %ymm1, %ymm2, %ymm1
-; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} ymm2 = ymm0[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]
-; AVX2-SLOW-NEXT:    vpshufhw {{.*#+}} ymm2 = ymm2[0,1,2,3,5,5,6,7,8,9,10,11,13,13,14,15]
+; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} ymm2 = ymm0[1,1,1,1,4,5,6,7,9,9,9,9,12,13,14,15]
+; AVX2-SLOW-NEXT:    vpshufhw {{.*#+}} ymm2 = ymm2[0,1,2,3,5,5,5,5,8,9,10,11,13,13,13,13]
 ; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,3,0,1]
 ; AVX2-SLOW-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,6,7,u,u,18,19,u,u,u,u,u,u,u,u,24,25,16,17,u,u]
 ; AVX2-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm2[1,2],ymm0[3],ymm2[4],ymm0[5,6,7,8],ymm2[9,10],ymm0[11],ymm2[12],ymm0[13,14,15]
@@ -7295,9 +7294,9 @@ define <16 x i16> @PR24935(<16 x i16> %a, <16 x i16> %b) {
 ;
 ; AVX2-FAST-LABEL: PR24935:
 ; AVX2-FAST:       # %bb.0:
-; AVX2-FAST-NEXT:    vpshufb {{.*#+}} ymm2 = ymm1[8,9,10,11,4,5,8,9,0,1,14,15,12,13,0,1,24,25,26,27,20,21,24,25,16,17,30,31,28,29,16,17]
+; AVX2-FAST-NEXT:    vpshufb {{.*#+}} ymm2 = ymm1[8,9,8,9,8,9,8,9,0,1,14,15,12,13,0,1,24,25,24,25,24,25,24,25,16,17,30,31,28,29,16,17]
 ; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,3,0,1]
-; AVX2-FAST-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[6,7,4,5,0,1,10,11,4,5,10,11,4,5,6,7,22,23,20,21,16,17,26,27,20,21,26,27,20,21,22,23]
+; AVX2-FAST-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[6,7,4,5,0,1,10,11,4,5,4,5,4,5,4,5,22,23,20,21,16,17,26,27,20,21,20,21,20,21,20,21]
 ; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} ymm3 = <255,255,255,255,u,u,255,255,255,255,0,0,u,u,0,0,u,u,u,u,255,255,0,0,u,u,u,u,u,u,0,0>
 ; AVX2-FAST-NEXT:    vpblendvb %ymm3, %ymm1, %ymm2, %ymm1
 ; AVX2-FAST-NEXT:    vpshufb {{.*#+}} ymm2 = ymm0[u,u,2,3,2,3,u,u,10,11,u,u,u,u,u,u,u,u,18,19,18,19,u,u,26,27,u,u,u,u,u,u]
@@ -7324,20 +7323,20 @@ define <16 x i16> @PR24935(<16 x i16> %a, <16 x i16> %b) {
 ; XOPAVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm5[0,1],xmm3[2,3],xmm5[4,5,6],xmm3[7]
 ; XOPAVX1-NEXT:    vpperm {{.*#+}} xmm1 = xmm2[6,7,4,5,u,u,10,11,4,5],xmm1[14,15],xmm2[u,u],xmm1[0,1]
 ; XOPAVX1-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
-; XOPAVX1-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,7,7]
+; XOPAVX1-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,7,7,7]
 ; XOPAVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2],xmm1[3,4,5],xmm0[6],xmm1[7]
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
 ; XOPAVX1-NEXT:    retq
 ;
 ; XOPAVX2-LABEL: PR24935:
 ; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vpshufb {{.*#+}} ymm2 = ymm1[8,9,10,11,4,5,8,9,0,1,14,15,12,13,0,1,24,25,26,27,20,21,24,25,16,17,30,31,28,29,16,17]
+; XOPAVX2-NEXT:    vpshufb {{.*#+}} ymm2 = ymm1[8,9,8,9,8,9,8,9,0,1,14,15,12,13,0,1,24,25,24,25,24,25,24,25,16,17,30,31,28,29,16,17]
 ; XOPAVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,3,0,1]
-; XOPAVX2-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[6,7,4,5,0,1,10,11,4,5,10,11,4,5,6,7,22,23,20,21,16,17,26,27,20,21,26,27,20,21,22,23]
+; XOPAVX2-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[6,7,4,5,0,1,10,11,4,5,4,5,4,5,4,5,22,23,20,21,16,17,26,27,20,21,20,21,20,21,20,21]
 ; XOPAVX2-NEXT:    vmovdqa {{.*#+}} ymm3 = <255,255,255,255,u,u,255,255,255,255,0,0,u,u,0,0,u,u,u,u,255,255,0,0,u,u,u,u,u,u,0,0>
 ; XOPAVX2-NEXT:    vpblendvb %ymm3, %ymm1, %ymm2, %ymm1
-; XOPAVX2-NEXT:    vpshuflw {{.*#+}} ymm2 = ymm0[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]
-; XOPAVX2-NEXT:    vpshufhw {{.*#+}} ymm2 = ymm2[0,1,2,3,5,5,6,7,8,9,10,11,13,13,14,15]
+; XOPAVX2-NEXT:    vpshuflw {{.*#+}} ymm2 = ymm0[1,1,1,1,4,5,6,7,9,9,9,9,12,13,14,15]
+; XOPAVX2-NEXT:    vpshufhw {{.*#+}} ymm2 = ymm2[0,1,2,3,5,5,5,5,8,9,10,11,13,13,13,13]
 ; XOPAVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,3,0,1]
 ; XOPAVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,6,7,u,u,18,19,u,u,u,u,u,u,u,u,24,25,16,17,u,u]
 ; XOPAVX2-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm2[1,2],ymm0[3],ymm2[4],ymm0[5,6,7,8],ymm2[9,10],ymm0[11],ymm2[12],ymm0[13,14,15]
@@ -7418,7 +7417,7 @@ define <16 x i16> @insert_dup_mem_v16i16_i32(i32* %ptr) {
 ; AVX1-LABEL: insert_dup_mem_v16i16_i32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
@@ -7431,7 +7430,7 @@ define <16 x i16> @insert_dup_mem_v16i16_i32(i32* %ptr) {
 ; XOPAVX1-LABEL: insert_dup_mem_v16i16_i32:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; XOPAVX1-NEXT:    retq
@@ -7452,7 +7451,7 @@ define <16 x i16> @insert_dup_mem_v16i16_sext_i16(i16* %ptr) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    movzwl (%rdi), %eax
 ; AVX1-NEXT:    vmovd %eax, %xmm0
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
@@ -7466,7 +7465,7 @@ define <16 x i16> @insert_dup_mem_v16i16_sext_i16(i16* %ptr) {
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    movzwl (%rdi), %eax
 ; XOPAVX1-NEXT:    vmovd %eax, %xmm0
-; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; XOPAVX1-NEXT:    retq
@@ -7487,7 +7486,7 @@ define <16 x i16> @insert_dup_elt1_mem_v16i16_i32(i32* %ptr) #0 {
 ; AVX1-LABEL: insert_dup_elt1_mem_v16i16_i32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[1,1,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[1,1,1,1,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
@@ -7500,7 +7499,7 @@ define <16 x i16> @insert_dup_elt1_mem_v16i16_i32(i32* %ptr) #0 {
 ; XOPAVX1-LABEL: insert_dup_elt1_mem_v16i16_i32:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[1,1,2,3,4,5,6,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[1,1,1,1,4,5,6,7]
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; XOPAVX1-NEXT:    retq
@@ -7520,7 +7519,7 @@ define <16 x i16> @insert_dup_elt3_mem_v16i16_i32(i32* %ptr) #0 {
 ; AVX1-LABEL: insert_dup_elt3_mem_v16i16_i32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vbroadcastss (%rdi), %xmm0
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[3,3,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[3,3,3,3,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
@@ -7533,7 +7532,7 @@ define <16 x i16> @insert_dup_elt3_mem_v16i16_i32(i32* %ptr) #0 {
 ; XOPAVX1-LABEL: insert_dup_elt3_mem_v16i16_i32:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vbroadcastss (%rdi), %xmm0
-; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[3,3,2,3,4,5,6,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[3,3,3,3,4,5,6,7]
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; XOPAVX1-NEXT:    retq
@@ -7553,7 +7552,7 @@ define <16 x i16> @insert_dup_mem_v16i16_i64(i64* %ptr) {
 ; AVX1-LABEL: insert_dup_mem_v16i16_i64:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
@@ -7566,7 +7565,7 @@ define <16 x i16> @insert_dup_mem_v16i16_i64(i64* %ptr) {
 ; XOPAVX1-LABEL: insert_dup_mem_v16i16_i64:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; XOPAVX1-NEXT:    retq
@@ -7586,7 +7585,7 @@ define <16 x i16> @insert_dup_elt1_mem_v16i16_i64(i64* %ptr) {
 ; AVX1-LABEL: insert_dup_elt1_mem_v16i16_i64:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[1,1,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[1,1,1,1,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
@@ -7599,7 +7598,7 @@ define <16 x i16> @insert_dup_elt1_mem_v16i16_i64(i64* %ptr) {
 ; XOPAVX1-LABEL: insert_dup_elt1_mem_v16i16_i64:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[1,1,2,3,4,5,6,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[1,1,1,1,4,5,6,7]
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; XOPAVX1-NEXT:    retq
@@ -7619,7 +7618,7 @@ define <16 x i16> @insert_dup_elt3_mem_v16i16_i64(i64* %ptr) {
 ; AVX1-LABEL: insert_dup_elt3_mem_v16i16_i64:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[3,3,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[3,3,3,3,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
@@ -7632,7 +7631,7 @@ define <16 x i16> @insert_dup_elt3_mem_v16i16_i64(i64* %ptr) {
 ; XOPAVX1-LABEL: insert_dup_elt3_mem_v16i16_i64:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[3,3,2,3,4,5,6,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[3,3,3,3,4,5,6,7]
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; XOPAVX1-NEXT:    retq
@@ -7684,7 +7683,7 @@ define <16 x i16> @insert_dup_mem_v16i16_sext_i16_i64(i16* %ptr) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    movzwl (%rdi), %eax
 ; AVX1-NEXT:    vmovd %eax, %xmm0
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
@@ -7698,7 +7697,7 @@ define <16 x i16> @insert_dup_mem_v16i16_sext_i16_i64(i16* %ptr) {
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    movzwl (%rdi), %eax
 ; XOPAVX1-NEXT:    vmovd %eax, %xmm0
-; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; XOPAVX1-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-256-v32.ll b/llvm/test/CodeGen/X86/vector-shuffle-256-v32.ll
index ff2654e800e23..82df05e5ae068 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-256-v32.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-256-v32.ll
@@ -2369,7 +2369,7 @@ define <32 x i8> @shuffle_v32i8_00_32_00_32_00_32_00_32_00_32_00_32_00_32_00_32_
 ; AVX1-LABEL: shuffle_v32i8_00_32_00_32_00_32_00_32_00_32_00_32_00_32_00_32_00_32_00_32_00_32_00_32_00_32_00_32_00_32_00_32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
@@ -2399,11 +2399,11 @@ define <32 x i8> @shuffle_v32i8_00_32_00_32_00_32_00_32_00_32_00_32_00_32_00_32_
 ; AVX1-LABEL: shuffle_v32i8_00_32_00_32_00_32_00_32_00_32_00_32_00_32_00_32_16_48_16_48_16_48_16_48_16_48_16_48_16_48_16_48:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm2[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm2[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} ymm0 = ymm0[0,0,0,0,4,4,4,4]
 ; AVX1-NEXT:    retq
@@ -2411,7 +2411,7 @@ define <32 x i8> @shuffle_v32i8_00_32_00_32_00_32_00_32_00_32_00_32_00_32_00_32_
 ; AVX2-SLOW-LABEL: shuffle_v32i8_00_32_00_32_00_32_00_32_00_32_00_32_00_32_00_32_16_48_16_48_16_48_16_48_16_48_16_48_16_48_16_48:
 ; AVX2-SLOW:       # %bb.0:
 ; AVX2-SLOW-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[16],ymm1[16],ymm0[17],ymm1[17],ymm0[18],ymm1[18],ymm0[19],ymm1[19],ymm0[20],ymm1[20],ymm0[21],ymm1[21],ymm0[22],ymm1[22],ymm0[23],ymm1[23]
-; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm0[0,0,2,3,4,5,6,7,8,8,10,11,12,13,14,15]
+; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm0[0,0,0,0,4,5,6,7,8,8,8,8,12,13,14,15]
 ; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[0,0,0,0,4,4,4,4]
 ; AVX2-SLOW-NEXT:    retq
 ;
@@ -2424,7 +2424,7 @@ define <32 x i8> @shuffle_v32i8_00_32_00_32_00_32_00_32_00_32_00_32_00_32_00_32_
 ; AVX512VLBW-SLOW-LABEL: shuffle_v32i8_00_32_00_32_00_32_00_32_00_32_00_32_00_32_00_32_16_48_16_48_16_48_16_48_16_48_16_48_16_48_16_48:
 ; AVX512VLBW-SLOW:       # %bb.0:
 ; AVX512VLBW-SLOW-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[16],ymm1[16],ymm0[17],ymm1[17],ymm0[18],ymm1[18],ymm0[19],ymm1[19],ymm0[20],ymm1[20],ymm0[21],ymm1[21],ymm0[22],ymm1[22],ymm0[23],ymm1[23]
-; AVX512VLBW-SLOW-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm0[0,0,2,3,4,5,6,7,8,8,10,11,12,13,14,15]
+; AVX512VLBW-SLOW-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm0[0,0,0,0,4,5,6,7,8,8,8,8,12,13,14,15]
 ; AVX512VLBW-SLOW-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[0,0,0,0,4,4,4,4]
 ; AVX512VLBW-SLOW-NEXT:    retq
 ;
@@ -2453,7 +2453,7 @@ define <32 x i8> @shuffle_v32i8_00_32_00_32_00_32_00_32_00_32_00_32_00_32_00_32_
 ; XOPAVX2-LABEL: shuffle_v32i8_00_32_00_32_00_32_00_32_00_32_00_32_00_32_00_32_16_48_16_48_16_48_16_48_16_48_16_48_16_48_16_48:
 ; XOPAVX2:       # %bb.0:
 ; XOPAVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[16],ymm1[16],ymm0[17],ymm1[17],ymm0[18],ymm1[18],ymm0[19],ymm1[19],ymm0[20],ymm1[20],ymm0[21],ymm1[21],ymm0[22],ymm1[22],ymm0[23],ymm1[23]
-; XOPAVX2-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm0[0,0,2,3,4,5,6,7,8,8,10,11,12,13,14,15]
+; XOPAVX2-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm0[0,0,0,0,4,5,6,7,8,8,8,8,12,13,14,15]
 ; XOPAVX2-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[0,0,0,0,4,4,4,4]
 ; XOPAVX2-NEXT:    retq
   %shuffle = shufflevector <32 x i8> %a, <32 x i8> %b, <32 x i32> <i32 0, i32 32, i32 0, i32 32, i32 0, i32 32, i32 0, i32 32, i32 0, i32 32, i32 0, i32 32, i32 0, i32 32, i32 0, i32 32, i32 16, i32 48, i32 16, i32 48, i32 16, i32 48, i32 16, i32 48, i32 16, i32 48, i32 16, i32 48, i32 16, i32 48, i32 16, i32 48>
@@ -3987,7 +3987,7 @@ define <32 x i8> @shuffle_v32i8_32_zz_zz_zz_33_zz_zz_zz_34_zz_zz_zz_35_zz_zz_zz_
 ; AVX1-LABEL: shuffle_v32i8_32_zz_zz_zz_33_zz_zz_zz_34_zz_zz_zz_35_zz_zz_zz_36_zz_zz_zz_37_zz_zz_zz_38_zz_zz_zz_39_zz_zz_zz:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
@@ -4000,7 +4000,7 @@ define <32 x i8> @shuffle_v32i8_32_zz_zz_zz_33_zz_zz_zz_34_zz_zz_zz_35_zz_zz_zz_
 ; XOPAVX1-LABEL: shuffle_v32i8_32_zz_zz_zz_33_zz_zz_zz_34_zz_zz_zz_35_zz_zz_zz_36_zz_zz_zz_37_zz_zz_zz_38_zz_zz_zz_39_zz_zz_zz:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; XOPAVX1-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; XOPAVX1-NEXT:    retq
@@ -4049,7 +4049,7 @@ define <32 x i8> @shuffle_v32i8_56_zz_zz_zz_57_zz_zz_zz_58_zz_zz_zz__zz_59_zz_zz
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
@@ -4079,7 +4079,7 @@ define <32 x i8> @shuffle_v32i8_56_zz_zz_zz_57_zz_zz_zz_58_zz_zz_zz__zz_59_zz_zz
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; XOPAVX1-NEXT:    vpmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
-; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; XOPAVX1-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; XOPAVX1-NEXT:    retq
@@ -5132,7 +5132,7 @@ define <32 x i8> @zeroable_src_to_zext(<32 x i8> %a0) {
 ; AVX1-LABEL: zeroable_src_to_zext:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
@@ -5145,7 +5145,7 @@ define <32 x i8> @zeroable_src_to_zext(<32 x i8> %a0) {
 ; XOPAVX1-LABEL: zeroable_src_to_zext:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; XOPAVX1-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; XOPAVX1-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-256-v4.ll b/llvm/test/CodeGen/X86/vector-shuffle-256-v4.ll
index 1fd61912ed4af..9c237f2a9b12d 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-256-v4.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-256-v4.ll
@@ -592,12 +592,12 @@ define <4 x double> @shuffle_v4f64_22uu(<4 x double> %a, <4 x double> %b) {
 ;
 ; AVX2-LABEL: shuffle_v4f64_22uu:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[2,2,2,3]
+; AVX2-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[2,2,2,2]
 ; AVX2-NEXT:    retq
 ;
 ; AVX512VL-LABEL: shuffle_v4f64_22uu:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[2,2,2,3]
+; AVX512VL-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[2,2,2,2]
 ; AVX512VL-NEXT:    retq
   %shuffle = shufflevector <4 x double> %a, <4 x double> %b, <4 x i32> <i32 2, i32 2, i32 undef, i32 undef>
   ret <4 x double> %shuffle
@@ -1143,7 +1143,7 @@ define <4 x i64> @shuffle_v4i64_1251(<4 x i64> %a, <4 x i64> %b) {
 ;
 ; AVX2-LABEL: shuffle_v4i64_1251:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,1,1,3]
+; AVX2-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[1,1,1,1]
 ; AVX2-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[1,2,2,1]
 ; AVX2-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5],ymm0[6,7]
 ; AVX2-NEXT:    retq
@@ -1369,12 +1369,12 @@ define <4 x i64> @shuffle_v4i64_22uu(<4 x i64> %a, <4 x i64> %b) {
 ;
 ; AVX2-LABEL: shuffle_v4i64_22uu:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[2,2,2,3]
+; AVX2-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[2,2,2,2]
 ; AVX2-NEXT:    retq
 ;
 ; AVX512VL-LABEL: shuffle_v4i64_22uu:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[2,2,2,3]
+; AVX512VL-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[2,2,2,2]
 ; AVX512VL-NEXT:    retq
   %shuffle = shufflevector <4 x i64> %a, <4 x i64> %b, <4 x i32> <i32 2, i32 2, i32 undef, i32 undef>
   ret <4 x i64> %shuffle
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-256-v8.ll b/llvm/test/CodeGen/X86/vector-shuffle-256-v8.ll
index 3077428386a26..ae402f507ac9e 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-256-v8.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-256-v8.ll
@@ -795,7 +795,7 @@ define <8 x float> @shuffle_v8f32_f511235a(<8 x float> %a, <8 x float> %b) {
 ; AVX1-NEXT:    vpermilps {{.*#+}} ymm1 = ymm1[3,1,2,2,7,5,6,6]
 ; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm1[2,3,0,1]
 ; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm0[2,3,0,1]
-; AVX1-NEXT:    vpermilps {{.*#+}} ymm0 = ymm0[0,1,1,1,4,5,5,5]
+; AVX1-NEXT:    vpermilps {{.*#+}} ymm0 = ymm0[1,1,1,1,5,5,5,5]
 ; AVX1-NEXT:    vshufpd {{.*#+}} ymm0 = ymm2[0],ymm0[1],ymm2[3],ymm0[3]
 ; AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0],ymm0[1,2,3,4,5,6],ymm1[7]
 ; AVX1-NEXT:    retq
@@ -2571,7 +2571,7 @@ define <8 x i32> @shuffle_v8i32_zuu8zuuc(<8 x i32> %a) {
 ; AVX1-LABEL: shuffle_v8i32_zuu8zuuc:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vshufps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,0],ymm1[4,5],ymm0[6,4]
+; AVX1-NEXT:    vshufps {{.*#+}} ymm0 = ymm1[0,0],ymm0[0,0],ymm1[4,4],ymm0[4,4]
 ; AVX1-NEXT:    retq
 ;
 ; AVX2OR512VL-LABEL: shuffle_v8i32_zuu8zuuc:
@@ -2718,7 +2718,7 @@ define <8 x i32> @shuffle_v8i32_uuuuuu7u(<8 x i32> %a, <8 x i32> %b) nounwind {
 ;
 ; AVX2OR512VL-LABEL: shuffle_v8i32_uuuuuu7u:
 ; AVX2OR512VL:       # %bb.0:
-; AVX2OR512VL-NEXT:    vpermilps {{.*#+}} ymm0 = ymm0[0,1,3,3,4,5,7,7]
+; AVX2OR512VL-NEXT:    vpermilps {{.*#+}} ymm0 = ymm0[3,3,3,3,7,7,7,7]
 ; AVX2OR512VL-NEXT:    retq
   %shuffle = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 7, i32 undef>
   ret <8 x i32> %shuffle
@@ -3134,7 +3134,7 @@ define <8 x float> @broadcast_concat_crash(<4 x float> %x, <4 x float> %y, float
 ; AVX2-LABEL: broadcast_concat_crash:
 ; AVX2:       # %bb.0: # %entry
 ; AVX2-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[3,1,2,3]
+; AVX2-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[3,3,3,3]
 ; AVX2-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[1,1,3,3]
 ; AVX2-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[2,3]
 ; AVX2-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
@@ -3143,7 +3143,7 @@ define <8 x float> @broadcast_concat_crash(<4 x float> %x, <4 x float> %y, float
 ; AVX512VL-SLOW-LABEL: broadcast_concat_crash:
 ; AVX512VL-SLOW:       # %bb.0: # %entry
 ; AVX512VL-SLOW-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX512VL-SLOW-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[3,1,2,3]
+; AVX512VL-SLOW-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[3,3,3,3]
 ; AVX512VL-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[1,1,3,3]
 ; AVX512VL-SLOW-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[2,3]
 ; AVX512VL-SLOW-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
@@ -3152,7 +3152,7 @@ define <8 x float> @broadcast_concat_crash(<4 x float> %x, <4 x float> %y, float
 ; AVX512VL-FAST-LABEL: broadcast_concat_crash:
 ; AVX512VL-FAST:       # %bb.0: # %entry
 ; AVX512VL-FAST-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX512VL-FAST-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[3,1,2,3]
+; AVX512VL-FAST-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[3,3,3,3]
 ; AVX512VL-FAST-NEXT:    vmovaps {{.*#+}} xmm1 = [1,4,3,3]
 ; AVX512VL-FAST-NEXT:    vpermi2ps %xmm2, %xmm0, %xmm1
 ; AVX512VL-FAST-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
@@ -3248,7 +3248,7 @@ define <8 x i32> @lowhalf_v8i32(<8 x i32> %x, <8 x i32> %y) {
 ; AVX1-LABEL: lowhalf_v8i32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[0,2,2,2]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[2,2,2,2]
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[2,2,3,3]
 ; AVX1-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
 ; AVX1-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-512-v16.ll b/llvm/test/CodeGen/X86/vector-shuffle-512-v16.ll
index bcd3a7b851500..e9f4aa99f148b 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-512-v16.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-512-v16.ll
@@ -341,7 +341,7 @@ define <4 x i32> @test_v16i32_0_1_2_12 (<16 x i32> %v) {
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vextractf64x4 $1, %zmm0, %ymm1
 ; ALL-NEXT:    vextractf128 $1, %ymm1, %xmm1
-; ALL-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[0,1,2,0]
+; ALL-NEXT:    vbroadcastss %xmm1, %xmm1
 ; ALL-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3]
 ; ALL-NEXT:    vzeroupper
 ; ALL-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-512-v64.ll b/llvm/test/CodeGen/X86/vector-shuffle-512-v64.ll
index 4237c4107d477..844f20132cf73 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-512-v64.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-512-v64.ll
@@ -367,7 +367,7 @@ define <64 x i8> @shuffle_v64i8_64_zz_zz_zz_zz_zz_zz_zz_65_zz_zz_zz_zz_zz_zz_zz_
 ; AVX512F-LABEL: shuffle_v64i8_64_zz_zz_zz_zz_zz_zz_zz_65_zz_zz_zz_zz_zz_zz_zz_66_zz_zz_zz_zz_zz_zz_zz_67_zz_zz_zz_zz_zz_zz_zz_68_zz_zz_zz_zz_zz_zz_zz_69_zz_zz_zz_zz_zz_zz_zz_70_zz_zz_zz_zz_zz_zz_zz_71_zz_zz_zz_zz_zz_zz_zz:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpmovzxbq {{.*#+}} ymm1 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero
-; AVX512F-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; AVX512F-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; AVX512F-NEXT:    vpmovzxbq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero
 ; AVX512F-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; AVX512F-NEXT:    retq
@@ -380,7 +380,7 @@ define <64 x i8> @shuffle_v64i8_64_zz_zz_zz_zz_zz_zz_zz_65_zz_zz_zz_zz_zz_zz_zz_
 ; AVX512DQ-LABEL: shuffle_v64i8_64_zz_zz_zz_zz_zz_zz_zz_65_zz_zz_zz_zz_zz_zz_zz_66_zz_zz_zz_zz_zz_zz_zz_67_zz_zz_zz_zz_zz_zz_zz_68_zz_zz_zz_zz_zz_zz_zz_69_zz_zz_zz_zz_zz_zz_zz_70_zz_zz_zz_zz_zz_zz_zz_71_zz_zz_zz_zz_zz_zz_zz:
 ; AVX512DQ:       # %bb.0:
 ; AVX512DQ-NEXT:    vpmovzxbq {{.*#+}} ymm1 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; AVX512DQ-NEXT:    vpmovzxbq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; AVX512DQ-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-combining-avx2.ll b/llvm/test/CodeGen/X86/vector-shuffle-combining-avx2.ll
index 03d130e7a0f6f..785202c88b6da 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-combining-avx2.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-combining-avx2.ll
@@ -819,7 +819,7 @@ define <32 x i8> @PR27320(<8 x i32> %a0) {
 define internal fastcc <8 x float> @PR34577(<8 x float> %inp0, <8 x float> %inp1, <8 x float> %inp2) {
 ; CHECK-LABEL: PR34577:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,1,1,3]
+; CHECK-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[1,1,1,1]
 ; CHECK-NEXT:    vxorps %xmm2, %xmm2, %xmm2
 ; CHECK-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
 ; CHECK-NEXT:    vmovaps {{.*#+}} ymm2 = <u,u,7,2,u,u,3,2>
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-combining.ll b/llvm/test/CodeGen/X86/vector-shuffle-combining.ll
index 622eb0881052e..f34a541150ba2 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-combining.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-combining.ll
@@ -762,12 +762,12 @@ define <4 x i32> @combine_nested_undef_test9(<4 x i32> %A, <4 x i32> %B) {
 define <4 x i32> @combine_nested_undef_test10(<4 x i32> %A, <4 x i32> %B) {
 ; SSE-LABEL: combine_nested_undef_test10:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: combine_nested_undef_test10:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[1,1,1,3]
+; AVX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; AVX-NEXT:    retq
   %1 = shufflevector <4 x i32> %A, <4 x i32> %B, <4 x i32> <i32 1, i32 1, i32 5, i32 5>
   %2 = shufflevector <4 x i32> %1, <4 x i32> undef, <4 x i32> <i32 0, i32 4, i32 1, i32 4>
@@ -1773,7 +1773,7 @@ define <4 x i8> @combine_test3c(<4 x i8>* %a, <4 x i8>* %b) {
 ; SSE-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; SSE-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; SSE-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: combine_test3c:
@@ -1781,7 +1781,7 @@ define <4 x i8> @combine_test3c(<4 x i8>* %a, <4 x i8>* %b) {
 ; AVX-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; AVX-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; AVX-NEXT:    retq
   %A = load <4 x i8>, <4 x i8>* %a
   %B = load <4 x i8>, <4 x i8>* %b
@@ -3061,7 +3061,7 @@ define void @PR43024() {
 ; SSE2-NEXT:    movaps {{.*#+}} xmm0 = [NaN,NaN,0.0E+0,0.0E+0]
 ; SSE2-NEXT:    movaps %xmm0, (%rax)
 ; SSE2-NEXT:    movaps %xmm0, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[1,1]
 ; SSE2-NEXT:    addss %xmm0, %xmm1
 ; SSE2-NEXT:    xorps %xmm0, %xmm0
 ; SSE2-NEXT:    addss %xmm0, %xmm1
@@ -3224,7 +3224,7 @@ define void @PR45604(<32 x i16>* %dst, <8 x i16>* %src) {
 ; AVX1-LABEL: PR45604:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vmovdqa (%rsi), %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[3,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
 ; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [11,11,11,0,11,11,11,0]
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
@@ -3232,7 +3232,7 @@ define void @PR45604(<32 x i16>* %dst, <8 x i16>* %src) {
 ; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm3, %ymm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
 ; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-mmx.ll b/llvm/test/CodeGen/X86/vector-shuffle-mmx.ll
index a00df1456b776..1fe283963a027 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-mmx.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-mmx.ll
@@ -9,14 +9,14 @@ define void @test0(<1 x i64>* %x) {
 ; X32:       ## %bb.0: ## %entry
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; X32-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; X32-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; X32-NEXT:    movlps %xmm0, (%eax)
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: test0:
 ; X64:       ## %bb.0: ## %entry
 ; X64-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; X64-NEXT:    movq %xmm0, (%rdi)
 ; X64-NEXT:    retq
 entry:
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-sse1.ll b/llvm/test/CodeGen/X86/vector-shuffle-sse1.ll
index 7a8c41519d654..bfb4a2da6d42b 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-sse1.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-sse1.ll
@@ -174,7 +174,7 @@ define <4 x float> @shuffle_v4f32_zuu4(<4 x float> %a) {
 ; SSE1-LABEL: shuffle_v4f32_zuu4:
 ; SSE1:       # %bb.0:
 ; SSE1-NEXT:    xorps %xmm1, %xmm1
-; SSE1-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,1],xmm0[2,0]
+; SSE1-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,0],xmm0[0,0]
 ; SSE1-NEXT:    movaps %xmm1, %xmm0
 ; SSE1-NEXT:    retq
   %shuffle = shufflevector <4 x float> zeroinitializer, <4 x float> %a, <4 x i32> <i32 0, i32 undef, i32 undef, i32 4>
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-v1.ll b/llvm/test/CodeGen/X86/vector-shuffle-v1.ll
index 2cc84c105125e..8c746a3425f51 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-v1.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-v1.ll
@@ -721,7 +721,7 @@ define i8 @shuf8i1__9_6_1_10_3_7_7_1(i8 %a) {
 ; AVX512VL-NEXT:    kmovw %edi, %k1
 ; AVX512VL-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
-; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; AVX512VL-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],mem[1,2,3,4,5,6,7]
 ; AVX512VL-NEXT:    vptestmd %ymm0, %ymm0, %k0
 ; AVX512VL-NEXT:    kmovw %k0, %eax
@@ -733,7 +733,7 @@ define i8 @shuf8i1__9_6_1_10_3_7_7_1(i8 %a) {
 ; VL_BW_DQ:       # %bb.0:
 ; VL_BW_DQ-NEXT:    kmovd %edi, %k0
 ; VL_BW_DQ-NEXT:    vpmovm2d %k0, %ymm0
-; VL_BW_DQ-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; VL_BW_DQ-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; VL_BW_DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],mem[1,2,3,4,5,6,7]
 ; VL_BW_DQ-NEXT:    vpmovd2m %ymm0, %k0
 ; VL_BW_DQ-NEXT:    kmovd %k0, %eax
diff --git a/llvm/test/CodeGen/X86/vector-trunc.ll b/llvm/test/CodeGen/X86/vector-trunc.ll
index 1d596f5db3aeb..f0398b15d04a4 100644
--- a/llvm/test/CodeGen/X86/vector-trunc.ll
+++ b/llvm/test/CodeGen/X86/vector-trunc.ll
@@ -1810,7 +1810,7 @@ entry:
 define <8 x i16> @PR32160(<8 x i32> %x) {
 ; SSE-LABEL: PR32160:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,4,6,7]
+; SSE-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,4,4,4]
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,2,2,2]
 ; SSE-NEXT:    retq
 ;
@@ -1822,8 +1822,8 @@ define <8 x i16> @PR32160(<8 x i32> %x) {
 ;
 ; AVX2-SLOW-LABEL: PR32160:
 ; AVX2-SLOW:       # %bb.0:
-; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[2,2,2,3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,2,2]
+; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[2,2,2,2,4,5,6,7]
 ; AVX2-SLOW-NEXT:    vpbroadcastd %xmm0, %xmm0
 ; AVX2-SLOW-NEXT:    vzeroupper
 ; AVX2-SLOW-NEXT:    retq
@@ -1838,7 +1838,7 @@ define <8 x i16> @PR32160(<8 x i32> %x) {
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
 ; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
-; AVX512F-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[2,2,2,3,4,5,6,7]
+; AVX512F-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[2,2,2,2,4,5,6,7]
 ; AVX512F-NEXT:    vpbroadcastd %xmm0, %xmm0
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/vector-zext.ll b/llvm/test/CodeGen/X86/vector-zext.ll
index 5d065213f925b..eb4f60dc14502 100644
--- a/llvm/test/CodeGen/X86/vector-zext.ll
+++ b/llvm/test/CodeGen/X86/vector-zext.ll
@@ -213,7 +213,7 @@ define <8 x i32> @zext_16i8_to_8i32(<16 x i8> %A) nounwind uwtable readnone ssp
 ; SSE41-LABEL: zext_16i8_to_8i32:
 ; SSE41:       # %bb.0: # %entry
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; SSE41-NEXT:    retq
@@ -221,7 +221,7 @@ define <8 x i32> @zext_16i8_to_8i32(<16 x i8> %A) nounwind uwtable readnone ssp
 ; AVX1-LABEL: zext_16i8_to_8i32:
 ; AVX1:       # %bb.0: # %entry
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
@@ -275,11 +275,11 @@ define <16 x i32> @zext_16i8_to_16i32(<16 x i8> %A) nounwind uwtable readnone ss
 ; SSE41-LABEL: zext_16i8_to_16i32:
 ; SSE41:       # %bb.0: # %entry
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm4 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm3 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE41-NEXT:    movdqa %xmm4, %xmm0
 ; SSE41-NEXT:    retq
@@ -287,12 +287,12 @@ define <16 x i32> @zext_16i8_to_16i32(<16 x i8> %A) nounwind uwtable readnone ss
 ; AVX1-LABEL: zext_16i8_to_16i32:
 ; AVX1:       # %bb.0: # %entry
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm2
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,3,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
 ; AVX1-NEXT:    vmovaps %ymm2, %ymm0
@@ -428,7 +428,7 @@ define <8 x i64> @zext_16i8_to_8i64(<16 x i8> %A) nounwind uwtable readnone ssp
 ; SSE41-NEXT:    movdqa %xmm0, %xmm1
 ; SSE41-NEXT:    psrld $16, %xmm1
 ; SSE41-NEXT:    pmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,1,1]
 ; SSE41-NEXT:    pmovzxbq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
 ; SSE41-NEXT:    psrlq $48, %xmm0
 ; SSE41-NEXT:    pmovzxbq {{.*#+}} xmm3 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
@@ -441,7 +441,7 @@ define <8 x i64> @zext_16i8_to_8i64(<16 x i8> %A) nounwind uwtable readnone ssp
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm2
 ; AVX1-NEXT:    vpmovzxbq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm0
 ; AVX1-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
@@ -452,7 +452,7 @@ define <8 x i64> @zext_16i8_to_8i64(<16 x i8> %A) nounwind uwtable readnone ssp
 ; AVX2-LABEL: zext_16i8_to_8i64:
 ; AVX2:       # %bb.0: # %entry
 ; AVX2-NEXT:    vpmovzxbq {{.*#+}} ymm2 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpmovzxbq {{.*#+}} ymm1 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero
 ; AVX2-NEXT:    vmovdqa %ymm2, %ymm0
 ; AVX2-NEXT:    retq
@@ -663,7 +663,7 @@ define <4 x i64> @zext_8i16_to_4i64(<8 x i16> %A) nounwind uwtable readnone ssp
 ; SSE41-LABEL: zext_8i16_to_4i64:
 ; SSE41:       # %bb.0: # %entry
 ; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm2 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; SSE41-NEXT:    retq
@@ -671,7 +671,7 @@ define <4 x i64> @zext_8i16_to_4i64(<8 x i16> %A) nounwind uwtable readnone ssp
 ; AVX1-LABEL: zext_8i16_to_4i64:
 ; AVX1:       # %bb.0: # %entry
 ; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
@@ -725,11 +725,11 @@ define <8 x i64> @zext_8i16_to_8i64(<8 x i16> %A) nounwind uwtable readnone ssp
 ; SSE41-LABEL: zext_8i16_to_8i64:
 ; SSE41:       # %bb.0: # %entry
 ; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm4 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm3 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
 ; SSE41-NEXT:    movdqa %xmm4, %xmm0
 ; SSE41-NEXT:    retq
@@ -737,12 +737,12 @@ define <8 x i64> @zext_8i16_to_8i64(<8 x i16> %A) nounwind uwtable readnone ssp
 ; AVX1-LABEL: zext_8i16_to_8i64:
 ; AVX1:       # %bb.0: # %entry
 ; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm2
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,3,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
 ; AVX1-NEXT:    vmovaps %ymm2, %ymm0
@@ -1124,7 +1124,7 @@ define <8 x i32> @load_zext_16i8_to_8i32(<16 x i8> *%ptr) {
 ; SSE41:       # %bb.0: # %entry
 ; SSE41-NEXT:    movdqa (%rdi), %xmm1
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; SSE41-NEXT:    retq
 ;
@@ -1132,7 +1132,7 @@ define <8 x i32> @load_zext_16i8_to_8i32(<16 x i8> *%ptr) {
 ; AVX1:       # %bb.0: # %entry
 ; AVX1-NEXT:    vmovdqa (%rdi), %xmm0
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
@@ -1522,7 +1522,7 @@ define <8 x i32> @zext_8i8_to_8i32(<8 x i8> %z) {
 ; SSE41-LABEL: zext_8i8_to_8i32:
 ; SSE41:       # %bb.0: # %entry
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; SSE41-NEXT:    retq
@@ -1530,7 +1530,7 @@ define <8 x i32> @zext_8i8_to_8i32(<8 x i8> %z) {
 ; AVX1-LABEL: zext_8i8_to_8i32:
 ; AVX1:       # %bb.0: # %entry
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
@@ -1669,7 +1669,7 @@ define <8 x i32> @shuf_zext_8i8_to_8i32(<8 x i8> %A) {
 ; SSE41-LABEL: shuf_zext_8i8_to_8i32:
 ; SSE41:       # %bb.0: # %entry
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; SSE41-NEXT:    retq
@@ -1677,7 +1677,7 @@ define <8 x i32> @shuf_zext_8i8_to_8i32(<8 x i8> %A) {
 ; AVX1-LABEL: shuf_zext_8i8_to_8i32:
 ; AVX1:       # %bb.0: # %entry
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
@@ -1883,7 +1883,7 @@ define <4 x i64> @shuf_zext_8i16_to_4i64_offset2(<8 x i16> %A) nounwind uwtable
 ;
 ; SSE41-LABEL: shuf_zext_8i16_to_4i64_offset2:
 ; SSE41:       # %bb.0: # %entry
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
@@ -1892,7 +1892,7 @@ define <4 x i64> @shuf_zext_8i16_to_4i64_offset2(<8 x i16> %A) nounwind uwtable
 ;
 ; AVX1-LABEL: shuf_zext_8i16_to_4i64_offset2:
 ; AVX1:       # %bb.0: # %entry
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
@@ -2211,18 +2211,18 @@ define <32 x i32> @zext_32i8_to_32i32(<32 x i8> %x) {
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movq %rdi, %rax
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,1,1]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm5 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[1,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[1,1,1,1]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm6 = xmm6[0],zero,zero,zero,xmm6[1],zero,zero,zero,xmm6[2],zero,zero,zero,xmm6[3],zero,zero,zero
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm1[2,3,2,3]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm7 = xmm7[0],zero,zero,zero,xmm7[1],zero,zero,zero,xmm7[2],zero,zero,zero,xmm7[3],zero,zero,zero
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; SSE41-NEXT:    movdqa %xmm1, 112(%rdi)
 ; SSE41-NEXT:    movdqa %xmm7, 96(%rdi)
@@ -2237,22 +2237,22 @@ define <32 x i32> @zext_32i8_to_32i32(<32 x i8> %x) {
 ; AVX1-LABEL: zext_32i8_to_32i32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm4
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm1 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm3[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm3[1,1,1,1]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm2
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,3,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[3,3,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
 ; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm3
 ; AVX1-NEXT:    vmovaps %ymm4, %ymm0
@@ -2448,61 +2448,61 @@ define <8 x i64> @zext_8i6_to_8i64(i32 %x) nounwind uwtable readnone ssp {
 ; SSE2-LABEL: zext_8i6_to_8i64:
 ; SSE2:       # %bb.0: # %entry
 ; SSE2-NEXT:    movd %edi, %xmm0
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[0,0,0,0]
 ; SSE2-NEXT:    paddw {{.*}}(%rip), %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,1,0,3]
-; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,0,0]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,5,5]
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [63,63]
 ; SSE2-NEXT:    pand %xmm4, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,1,3]
-; SSE2-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,5,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,1,1]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,5,5,5]
 ; SSE2-NEXT:    pand %xmm4, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,1,2,3]
-; SSE2-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,5,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,2,2,2]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,5,5,5]
 ; SSE2-NEXT:    pand %xmm4, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[3,1,3,3]
-; SSE2-NEXT:    pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,5,5,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,5,5,5,5]
 ; SSE2-NEXT:    pand %xmm4, %xmm3
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: zext_8i6_to_8i64:
 ; SSSE3:       # %bb.0: # %entry
 ; SSSE3-NEXT:    movd %edi, %xmm0
-; SSSE3-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; SSSE3-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[0,0,0,0]
 ; SSSE3-NEXT:    paddw {{.*}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,1,0,3]
-; SSSE3-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,6,7]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,0,0]
+; SSSE3-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,5,5]
 ; SSSE3-NEXT:    movdqa {{.*#+}} xmm4 = [63,63]
 ; SSSE3-NEXT:    pand %xmm4, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,1,3]
-; SSSE3-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,5,6,7]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,1,1]
+; SSSE3-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,5,5,5]
 ; SSSE3-NEXT:    pand %xmm4, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,1,2,3]
-; SSSE3-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,5,6,7]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,2,2,2]
+; SSSE3-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,5,5,5]
 ; SSSE3-NEXT:    pand %xmm4, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[3,1,3,3]
-; SSSE3-NEXT:    pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,5,5,6,7]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
+; SSSE3-NEXT:    pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,5,5,5,5]
 ; SSSE3-NEXT:    pand %xmm4, %xmm3
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: zext_8i6_to_8i64:
 ; SSE41:       # %bb.0: # %entry
 ; SSE41-NEXT:    movd %edi, %xmm0
-; SSE41-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; SSE41-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[0,0,0,0]
 ; SSE41-NEXT:    paddw {{.*}}(%rip), %xmm3
 ; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm0 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [63,63]
 ; SSE41-NEXT:    pand %xmm4, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,1,1]
 ; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
 ; SSE41-NEXT:    pand %xmm4, %xmm1
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]
 ; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
 ; SSE41-NEXT:    pand %xmm4, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[3,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
 ; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero
 ; SSE41-NEXT:    pand %xmm4, %xmm3
 ; SSE41-NEXT:    retq
@@ -2510,17 +2510,17 @@ define <8 x i64> @zext_8i6_to_8i64(i32 %x) nounwind uwtable readnone ssp {
 ; AVX1-LABEL: zext_8i6_to_8i64:
 ; AVX1:       # %bb.0: # %entry
 ; AVX1-NEXT:    vmovd %edi, %xmm0
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    vpaddw {{.*}}(%rip), %xmm0, %xmm0
 ; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm1
 ; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
 ; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
 ; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
 ; AVX1-NEXT:    retq
@@ -2698,8 +2698,8 @@ define <16 x i16> @splatshuf_zext_v16i16(<16 x i8> %x) {
 ; SSE2-LABEL: splatshuf_zext_v16i16:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,6,6,7]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,2,2,3]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,6,6,6]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,2,2,2]
 ; SSE2-NEXT:    pxor %xmm1, %xmm1
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
diff --git a/llvm/test/CodeGen/X86/vsel-cmp-load.ll b/llvm/test/CodeGen/X86/vsel-cmp-load.ll
index 89d1b549182ab..345a50450d74f 100644
--- a/llvm/test/CodeGen/X86/vsel-cmp-load.ll
+++ b/llvm/test/CodeGen/X86/vsel-cmp-load.ll
@@ -12,7 +12,7 @@ define <8 x i32> @eq_zero(<8 x i8>* %p, <8 x i32> %x, <8 x i32> %y) {
 ; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
 ; AVX1-NEXT:    vpcmpeqb %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm3
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,1,1]
 ; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm2
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
 ; AVX1-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
@@ -47,7 +47,7 @@ define <4 x i64> @ne_zero(<4 x i16>* %p, <4 x i64> %x, <4 x i64> %y) {
 ; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
 ; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vpmovsxwq %xmm2, %xmm3
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,1,1]
 ; AVX1-NEXT:    vpmovsxwq %xmm2, %xmm2
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
 ; AVX1-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
@@ -113,7 +113,7 @@ define <8 x i32> @slt_zero(<8 x i8>* %p, <8 x i32> %x, <8 x i32> %y) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero
 ; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm3
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,1,1]
 ; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm2
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
 ; AVX1-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
@@ -179,7 +179,7 @@ define <8 x float> @ne_zero_fp_select(<8 x i8>* %p, <8 x float> %x, <8 x float>
 ; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
 ; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm3
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,1,1]
 ; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm2
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
 ; AVX1-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
diff --git a/llvm/test/CodeGen/X86/vselect.ll b/llvm/test/CodeGen/X86/vselect.ll
index e5646ae126b4f..aba908f34696d 100644
--- a/llvm/test/CodeGen/X86/vselect.ll
+++ b/llvm/test/CodeGen/X86/vselect.ll
@@ -570,7 +570,7 @@ define <2 x i32> @simplify_select(i32 %x, <2 x i1> %z) {
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,0,1,1]
 ; SSE2-NEXT:    por %xmm1, %xmm2
 ; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,0],xmm2[1,1]
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[2,0],xmm2[2,3]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[2,0],xmm2[1,1]
 ; SSE2-NEXT:    pand %xmm0, %xmm2
 ; SSE2-NEXT:    pandn %xmm1, %xmm0
 ; SSE2-NEXT:    por %xmm2, %xmm0
@@ -583,7 +583,7 @@ define <2 x i32> @simplify_select(i32 %x, <2 x i1> %z) {
 ; SSE41-NEXT:    movd %edi, %xmm1
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,0,1,1]
 ; SSE41-NEXT:    por %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,1,1]
 ; SSE41-NEXT:    pinsrd $1, %edi, %xmm1
 ; SSE41-NEXT:    blendvps %xmm0, %xmm2, %xmm1
 ; SSE41-NEXT:    movaps %xmm1, %xmm0
@@ -596,7 +596,7 @@ define <2 x i32> @simplify_select(i32 %x, <2 x i1> %z) {
 ; AVX-NEXT:    vmovd %edi, %xmm1
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,0,1,1]
 ; AVX-NEXT:    vpor %xmm1, %xmm2, %xmm1
-; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,2,3]
+; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
 ; AVX-NEXT:    vpinsrd $1, %edi, %xmm2, %xmm2
 ; AVX-NEXT:    vblendvps %xmm0, %xmm1, %xmm2, %xmm0
 ; AVX-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/vshift-4.ll b/llvm/test/CodeGen/X86/vshift-4.ll
index 1131d07b15d35..7d6fe0688c84f 100644
--- a/llvm/test/CodeGen/X86/vshift-4.ll
+++ b/llvm/test/CodeGen/X86/vshift-4.ll
@@ -58,7 +58,7 @@ define void @shift2a(<4 x i32> %val, <4 x i32>* %dst, <2 x i32> %amt) nounwind {
 ; X32-LABEL: shift2a:
 ; X32:       # %bb.0: # %entry
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; X32-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
 ; X32-NEXT:    xorps %xmm2, %xmm2
 ; X32-NEXT:    movss {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
 ; X32-NEXT:    pslld %xmm2, %xmm0
@@ -67,7 +67,7 @@ define void @shift2a(<4 x i32> %val, <4 x i32>* %dst, <2 x i32> %amt) nounwind {
 ;
 ; X64-LABEL: shift2a:
 ; X64:       # %bb.0: # %entry
-; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
 ; X64-NEXT:    xorps %xmm2, %xmm2
 ; X64-NEXT:    movss {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
 ; X64-NEXT:    pslld %xmm2, %xmm0
@@ -84,7 +84,7 @@ define void @shift2b(<4 x i32> %val, <4 x i32>* %dst, <2 x i32> %amt) nounwind {
 ; X32-LABEL: shift2b:
 ; X32:       # %bb.0: # %entry
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; X32-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
 ; X32-NEXT:    xorps %xmm2, %xmm2
 ; X32-NEXT:    movss {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
 ; X32-NEXT:    pslld %xmm2, %xmm0
@@ -93,7 +93,7 @@ define void @shift2b(<4 x i32> %val, <4 x i32>* %dst, <2 x i32> %amt) nounwind {
 ;
 ; X64-LABEL: shift2b:
 ; X64:       # %bb.0: # %entry
-; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
 ; X64-NEXT:    xorps %xmm2, %xmm2
 ; X64-NEXT:    movss {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
 ; X64-NEXT:    pslld %xmm2, %xmm0
@@ -110,7 +110,7 @@ define void @shift2c(<4 x i32> %val, <4 x i32>* %dst, <2 x i32> %amt) nounwind {
 ; X32-LABEL: shift2c:
 ; X32:       # %bb.0: # %entry
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; X32-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
 ; X32-NEXT:    xorps %xmm2, %xmm2
 ; X32-NEXT:    movss {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
 ; X32-NEXT:    pslld %xmm2, %xmm0
@@ -119,7 +119,7 @@ define void @shift2c(<4 x i32> %val, <4 x i32>* %dst, <2 x i32> %amt) nounwind {
 ;
 ; X64-LABEL: shift2c:
 ; X64:       # %bb.0: # %entry
-; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
 ; X64-NEXT:    xorps %xmm2, %xmm2
 ; X64-NEXT:    movss {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
 ; X64-NEXT:    pslld %xmm2, %xmm0
@@ -136,7 +136,7 @@ define void @shift3a(<8 x i16> %val, <8 x i16>* %dst, <8 x i16> %amt) nounwind {
 ; X32-LABEL: shift3a:
 ; X32:       # %bb.0: # %entry
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; X32-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; X32-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1]
 ; X32-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; X32-NEXT:    psllw %xmm1, %xmm0
@@ -145,7 +145,7 @@ define void @shift3a(<8 x i16> %val, <8 x i16>* %dst, <8 x i16> %amt) nounwind {
 ;
 ; X64-LABEL: shift3a:
 ; X64:       # %bb.0: # %entry
-; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; X64-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1]
 ; X64-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; X64-NEXT:    psllw %xmm1, %xmm0
diff --git a/llvm/test/CodeGen/X86/widen_conv-3.ll b/llvm/test/CodeGen/X86/widen_conv-3.ll
index 21e80c99dcfa9..4ebfca6a3a009 100644
--- a/llvm/test/CodeGen/X86/widen_conv-3.ll
+++ b/llvm/test/CodeGen/X86/widen_conv-3.ll
@@ -68,7 +68,7 @@ define void @convert_v3i8_to_v3f32(<3 x float>* %dst.addr, <3 x i8>* %src.addr)
 ; X86-SSE2-NEXT:    movaps %xmm0, %xmm1
 ; X86-SSE2-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
 ; X86-SSE2-NEXT:    movss %xmm1, 8(%eax)
-; X86-SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; X86-SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; X86-SSE2-NEXT:    movss %xmm0, 4(%eax)
 ; X86-SSE2-NEXT:    retl
 ;
diff --git a/llvm/test/CodeGen/X86/widen_conv-4.ll b/llvm/test/CodeGen/X86/widen_conv-4.ll
index d5022b24eabe1..7f333d88eeb35 100644
--- a/llvm/test/CodeGen/X86/widen_conv-4.ll
+++ b/llvm/test/CodeGen/X86/widen_conv-4.ll
@@ -21,7 +21,7 @@ define void @convert_v7i16_v7f32(<7 x float>* %dst.addr, <7 x i16> %src) nounwin
 ; X86-SSE2-NEXT:    movaps %xmm2, %xmm0
 ; X86-SSE2-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm2[1]
 ; X86-SSE2-NEXT:    movss %xmm0, 24(%eax)
-; X86-SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,2,3]
+; X86-SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,1,1]
 ; X86-SSE2-NEXT:    movss %xmm2, 20(%eax)
 ; X86-SSE2-NEXT:    retl
 ;
@@ -94,7 +94,7 @@ define void @convert_v3i8_to_v3f32(<3 x float>* %dst.addr, <3 x i8>* %src.addr)
 ; X86-SSE2-NEXT:    movaps %xmm0, %xmm1
 ; X86-SSE2-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
 ; X86-SSE2-NEXT:    movss %xmm1, 8(%eax)
-; X86-SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; X86-SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; X86-SSE2-NEXT:    movss %xmm0, 4(%eax)
 ; X86-SSE2-NEXT:    retl
 ;
diff --git a/llvm/test/CodeGen/X86/widened-broadcast.ll b/llvm/test/CodeGen/X86/widened-broadcast.ll
index 2ffc413420f40..033f5b596f909 100644
--- a/llvm/test/CodeGen/X86/widened-broadcast.ll
+++ b/llvm/test/CodeGen/X86/widened-broadcast.ll
@@ -260,13 +260,13 @@ entry:
 define <16 x i8> @load_splat_16i8_16i8_0101010101010101(<16 x i8>* %ptr) nounwind uwtable readnone ssp {
 ; SSE-LABEL: load_splat_16i8_16i8_0101010101010101:
 ; SSE:       # %bb.0: # %entry
-; SSE-NEXT:    pshuflw {{.*#+}} xmm0 = mem[0,0,2,3,4,5,6,7]
+; SSE-NEXT:    pshuflw {{.*#+}} xmm0 = mem[0,0,0,0,4,5,6,7]
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: load_splat_16i8_16i8_0101010101010101:
 ; AVX1:       # %bb.0: # %entry
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = mem[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = mem[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    retq
 ;
@@ -340,14 +340,14 @@ entry:
 define <32 x i8> @load_splat_32i8_16i8_01010101010101010101010101010101(<16 x i8>* %ptr) nounwind uwtable readnone ssp {
 ; SSE-LABEL: load_splat_32i8_16i8_01010101010101010101010101010101:
 ; SSE:       # %bb.0: # %entry
-; SSE-NEXT:    pshuflw {{.*#+}} xmm0 = mem[0,0,2,3,4,5,6,7]
+; SSE-NEXT:    pshuflw {{.*#+}} xmm0 = mem[0,0,0,0,4,5,6,7]
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; SSE-NEXT:    movdqa %xmm0, %xmm1
 ; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: load_splat_32i8_16i8_01010101010101010101010101010101:
 ; AVX1:       # %bb.0: # %entry
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = mem[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = mem[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
@@ -404,14 +404,14 @@ entry:
 define <32 x i8> @load_splat_32i8_32i8_01010101010101010101010101010101(<32 x i8>* %ptr) nounwind uwtable readnone ssp {
 ; SSE-LABEL: load_splat_32i8_32i8_01010101010101010101010101010101:
 ; SSE:       # %bb.0: # %entry
-; SSE-NEXT:    pshuflw {{.*#+}} xmm0 = mem[0,0,2,3,4,5,6,7]
+; SSE-NEXT:    pshuflw {{.*#+}} xmm0 = mem[0,0,0,0,4,5,6,7]
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; SSE-NEXT:    movdqa %xmm0, %xmm1
 ; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: load_splat_32i8_32i8_01010101010101010101010101010101:
 ; AVX1:       # %bb.0: # %entry
-; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = mem[0,0,2,3,4,5,6,7]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = mem[0,0,0,0,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; AVX1-NEXT:    retq

From 5d9f33aaa00cc02143f137387adc8dd1e51b71d3 Mon Sep 17 00:00:00 2001
From: Stephan Herhut <herhut@google.com>
Date: Tue, 28 Jul 2020 19:08:40 +0200
Subject: [PATCH 0431/1035] [MLIR][Shape] Add conversion for missing ops to
 standard

This adds conversions for const_size and to_extent_tensor. Also, cast-like operations are now folded away if the source and target types are the same.

Differential Revision: https://reviews.llvm.org/D84745
---
 .../include/mlir/Dialect/Shape/IR/ShapeOps.td |  2 +-
 .../ShapeToStandard/ShapeToStandard.cpp       | 40 ++++++++++++++++++-
 mlir/lib/Dialect/Shape/IR/Shape.cpp           |  4 +-
 .../ShapeToStandard/shape-to-standard.mlir    | 25 ++++++++++++
 mlir/test/Dialect/Shape/canonicalize.mlir     | 19 +++++++++
 5 files changed, 86 insertions(+), 4 deletions(-)

diff --git a/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td b/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td
index 9ba838dbbb26f..6ea61376c34d4 100644
--- a/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td
+++ b/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td
@@ -440,7 +440,7 @@ def Shape_SizeToIndexOp : Shape_Op<"size_to_index", [NoSideEffect]> {
     arguments.
   }];
 
-  let arguments = (outs Shape_SizeOrIndexType:$arg);
+  let arguments = (ins Shape_SizeOrIndexType:$arg);
   let results = (outs Index:$result);
 
   let assemblyFormat = "$arg attr-dict `:` type($arg)";
diff --git a/mlir/lib/Conversion/ShapeToStandard/ShapeToStandard.cpp b/mlir/lib/Conversion/ShapeToStandard/ShapeToStandard.cpp
index b84b6ba3b5d67..efeaa18e17c11 100644
--- a/mlir/lib/Conversion/ShapeToStandard/ShapeToStandard.cpp
+++ b/mlir/lib/Conversion/ShapeToStandard/ShapeToStandard.cpp
@@ -56,6 +56,21 @@ class BinaryOpConversion : public OpConversionPattern<SrcOpTy> {
 };
 } // namespace
 
+namespace {
+class ConstSizeOpConversion : public OpConversionPattern<ConstSizeOp> {
+public:
+  using OpConversionPattern<ConstSizeOp>::OpConversionPattern;
+
+  LogicalResult
+  matchAndRewrite(ConstSizeOp op, ArrayRef<Value> operands,
+                  ConversionPatternRewriter &rewriter) const override {
+
+    rewriter.replaceOpWithNewOp<ConstantIndexOp>(op, op.value().getSExtValue());
+    return success();
+  }
+};
+} // namespace
+
 namespace {
 class ShapeOfOpConversion : public OpConversionPattern<ShapeOfOp> {
 public:
@@ -136,6 +151,27 @@ LogicalResult ConstShapeOpConverter::matchAndRewrite(
   return success();
 }
 
+namespace {
+class ToExtentTensorOpConversion
+    : public OpConversionPattern<ToExtentTensorOp> {
+public:
+  using OpConversionPattern<ToExtentTensorOp>::OpConversionPattern;
+
+  LogicalResult
+  matchAndRewrite(ToExtentTensorOp op, ArrayRef<Value> operands,
+                  ConversionPatternRewriter &rewriter) const override {
+    ToExtentTensorOpAdaptor adaptor(operands);
+
+    if (!adaptor.input().getType().isa<RankedTensorType>())
+      return rewriter.notifyMatchFailure(op, "input needs to be a tensor");
+
+    rewriter.replaceOpWithNewOp<TensorCastOp>(op, adaptor.input(),
+                                              op.getType());
+    return success();
+  }
+};
+} // namespace
+
 namespace {
 class GetExtentOpConverter : public OpConversionPattern<GetExtentOp> {
   using OpConversionPattern<GetExtentOp>::OpConversionPattern;
@@ -244,9 +280,11 @@ void mlir::populateShapeToStandardConversionPatterns(
       BinaryOpConversion<AddOp, AddIOp>,
       ConstShapeOpConverter,
       BinaryOpConversion<MulOp, MulIOp>,
+      ConstSizeOpConversion,
       GetExtentOpConverter,
       RankOpConverter,
-      ShapeOfOpConversion>(ctx);
+      ShapeOfOpConversion,
+      ToExtentTensorOpConversion>(ctx);
   // clang-format on
 }
 
diff --git a/mlir/lib/Dialect/Shape/IR/Shape.cpp b/mlir/lib/Dialect/Shape/IR/Shape.cpp
index 3c71e3409923e..02fe7b8129f7e 100644
--- a/mlir/lib/Dialect/Shape/IR/Shape.cpp
+++ b/mlir/lib/Dialect/Shape/IR/Shape.cpp
@@ -753,7 +753,7 @@ OpFoldResult SizeToIndexOp::fold(ArrayRef<Attribute> operands) {
   // `IntegerAttr`s which makes constant folding simple.
   if (Attribute arg = operands[0])
     return arg;
-  return {};
+  return impl::foldCastOp(*this);
 }
 
 void SizeToIndexOp::getCanonicalizationPatterns(
@@ -812,7 +812,7 @@ LogicalResult SplitAtOp::fold(ArrayRef<Attribute> operands,
 
 OpFoldResult ToExtentTensorOp::fold(ArrayRef<Attribute> operands) {
   if (!operands[0])
-    return nullptr;
+    return impl::foldCastOp(*this);
   Builder builder(getContext());
   auto shape = llvm::to_vector<6>(
       operands[0].cast<DenseIntElementsAttr>().getValues<int64_t>());
diff --git a/mlir/test/Conversion/ShapeToStandard/shape-to-standard.mlir b/mlir/test/Conversion/ShapeToStandard/shape-to-standard.mlir
index 7f875f3bb19f9..b94b24599351d 100644
--- a/mlir/test/Conversion/ShapeToStandard/shape-to-standard.mlir
+++ b/mlir/test/Conversion/ShapeToStandard/shape-to-standard.mlir
@@ -149,3 +149,28 @@ func @any_of_one(%a : tensor<?xindex>) -> tensor<?xindex> {
   return %result : tensor<?xindex>
 }
 
+// -----
+
+// Lower 'const_size` to `std.constant`
+// CHECK-LABEL: @const_size
+func @const_size() -> index {
+  // CHECK: %[[RES:.*]] = constant 42 : index
+  %size = shape.const_size 42
+  %result = shape.size_to_index %size : !shape.size
+  // CHECK: return %[[RES]]
+  return %result : index
+}
+
+// -----
+
+// Lower `to_extent_tensor` to `std.tensor_cast`
+// Fold to_extent_tensor when already on tensor.
+// CHECK-LABEL: @to_extent_tensor
+// CHECK-SAME: (%[[ARG:.*]]: tensor<?xindex>
+func @to_extent_tensor(%arg: tensor<?xindex>) -> tensor<3xindex> {
+  // CHECK-NOT: to_extent_tensor
+  // CHECK: %[[RES:.*]] = tensor_cast %[[ARG]] : tensor<?xindex> to tensor<3xindex
+  %casted = shape.to_extent_tensor %arg : tensor<?xindex> -> tensor<3xindex>
+  // CHECK: return %[[RES]]
+  return %casted : tensor<3xindex>
+}
diff --git a/mlir/test/Dialect/Shape/canonicalize.mlir b/mlir/test/Dialect/Shape/canonicalize.mlir
index 5fe2ac108a69d..ed5bf6999cd9a 100644
--- a/mlir/test/Dialect/Shape/canonicalize.mlir
+++ b/mlir/test/Dialect/Shape/canonicalize.mlir
@@ -774,3 +774,22 @@ func @fold_mul_mixed() -> !shape.size {
   return %result : !shape.size
 }
 
+// -----
+
+// Fold index_cast when already on index.
+// CHECK-LABEL: @fold_index_cast_on_index
+func @fold_index_cast_on_index(%arg: index) -> index {
+  // CHECK-NOT: size_to_index
+  %casted = shape.size_to_index %arg : index
+  return %casted : index
+}
+
+// -----
+
+// Fold to_extent_tensor when already on tensor.
+// CHECK-LABEL: @fold_to_extent_tensor_on_tensor
+func @fold_to_extent_tensor_on_tensor(%arg: tensor<?xindex>) -> tensor<?xindex> {
+  // CHECK-NOT: to_extent_tensor
+  %casted = shape.to_extent_tensor %arg : tensor<?xindex> -> tensor<?xindex>
+  return %casted : tensor<?xindex>
+}

From b6b9d3ea85cc158a5230b4b75147d299bfc372df Mon Sep 17 00:00:00 2001
From: Frederik Gossen <frgossen@google.com>
Date: Wed, 29 Jul 2020 10:45:07 +0000
Subject: [PATCH 0432/1035] [MLIR][Shape] Remove type conversion from lowering
 to standard

Operating on indices and extent tensors directly, the type conversion is no
longer needed for the supported cases.

Differential Revision: https://reviews.llvm.org/D84442
---
 .../ShapeToStandard/ShapeToStandard.cpp       | 33 ++-----------------
 .../ShapeToStandard/shape-to-standard.mlir    | 33 ++-----------------
 2 files changed, 5 insertions(+), 61 deletions(-)

diff --git a/mlir/lib/Conversion/ShapeToStandard/ShapeToStandard.cpp b/mlir/lib/Conversion/ShapeToStandard/ShapeToStandard.cpp
index efeaa18e17c11..4deaa8cd2df36 100644
--- a/mlir/lib/Conversion/ShapeToStandard/ShapeToStandard.cpp
+++ b/mlir/lib/Conversion/ShapeToStandard/ShapeToStandard.cpp
@@ -219,25 +219,6 @@ RankOpConverter::matchAndRewrite(shape::RankOp op, ArrayRef<Value> operands,
   return success();
 }
 
-namespace {
-/// Type conversions.
-class ShapeTypeConverter : public TypeConverter {
-public:
-  using TypeConverter::convertType;
-
-  ShapeTypeConverter(MLIRContext *ctx) {
-    // Add default pass-through conversion.
-    addConversion([&](Type type) { return type; });
-
-    addConversion([ctx](SizeType type) { return IndexType::get(ctx); });
-    addConversion([ctx](ShapeType type) {
-      return RankedTensorType::get({ShapedType::kDynamicSize},
-                                   IndexType::get(ctx));
-    });
-  }
-};
-} // namespace
-
 namespace {
 /// Conversion pass.
 class ConvertShapeToStandardPass
@@ -248,23 +229,15 @@ class ConvertShapeToStandardPass
 } // namespace
 
 void ConvertShapeToStandardPass::runOnOperation() {
-  // Setup type conversion.
-  MLIRContext &ctx = getContext();
-  ShapeTypeConverter typeConverter(&ctx);
-
   // Setup target legality.
+  MLIRContext &ctx = getContext();
   ConversionTarget target(ctx);
-  target.addLegalDialect<scf::SCFDialect, StandardOpsDialect>();
-  target.addLegalOp<ModuleOp, ModuleTerminatorOp, ReturnOp>();
-  target.addDynamicallyLegalOp<FuncOp>([&](FuncOp op) {
-    return typeConverter.isSignatureLegal(op.getType()) &&
-           typeConverter.isLegal(&op.getBody());
-  });
+  target.addLegalDialect<StandardOpsDialect>();
+  target.addLegalOp<FuncOp, ModuleOp, ModuleTerminatorOp>();
 
   // Setup conversion patterns.
   OwningRewritePatternList patterns;
   populateShapeToStandardConversionPatterns(patterns, &ctx);
-  populateFuncOpTypeConversionPattern(patterns, &ctx, typeConverter);
 
   // Apply conversion.
   auto module = getOperation();
diff --git a/mlir/test/Conversion/ShapeToStandard/shape-to-standard.mlir b/mlir/test/Conversion/ShapeToStandard/shape-to-standard.mlir
index b94b24599351d..0e30cc2bdf569 100644
--- a/mlir/test/Conversion/ShapeToStandard/shape-to-standard.mlir
+++ b/mlir/test/Conversion/ShapeToStandard/shape-to-standard.mlir
@@ -1,40 +1,11 @@
 // RUN: mlir-opt --split-input-file --convert-shape-to-std --verify-diagnostics %s | FileCheck %s
 
-// Convert `size` to `index` type.
-// CHECK-LABEL: @size_id
-// CHECK-SAME: (%[[SIZE:.*]]: index)
-func @size_id(%size : !shape.size) -> !shape.size {
-  // CHECK: return %[[SIZE]] : index
-  return %size : !shape.size
-}
-
-// -----
-
-// Convert `shape` to `tensor<?xindex>` type.
-// CHECK-LABEL: @shape_id
-// CHECK-SAME: (%[[SHAPE:.*]]: tensor<?xindex>)
-func @shape_id(%shape : !shape.shape) -> !shape.shape {
-  // CHECK: return %[[SHAPE]] : tensor<?xindex>
-  return %shape : !shape.shape
-}
-
-// -----
-
-// Lower binary ops.
-// CHECK-LABEL: @binary_ops
-// CHECK-SAME: (%[[LHS:.*]]: index, %[[RHS:.*]]: index)
-func @binary_ops(%lhs : !shape.size, %rhs : !shape.size) {
-  // CHECK: addi %[[LHS]], %[[RHS]] : index
-  %sum = "shape.add"(%lhs, %rhs) : (!shape.size, !shape.size) -> !shape.size
-  return
-}
-
-// -----
-
 // Lower binary ops.
 // CHECK-LABEL: @binary_ops
 // CHECK-SAME: (%[[LHS:.*]]: index, %[[RHS:.*]]: index)
 func @binary_ops(%lhs : index, %rhs : index) {
+  // CHECK: addi %[[LHS]], %[[RHS]] : index
+  %sum = shape.add %lhs, %rhs : index, index -> index
   // CHECK: muli %[[LHS]], %[[RHS]] : index
   %product = shape.mul %lhs, %rhs : index, index -> index
   return

From 380fee34d2794361f9e222fe3c8c065be3b9fff8 Mon Sep 17 00:00:00 2001
From: Hafiz Abid Qadeer <abidh@codesourcery.com>
Date: Wed, 29 Jul 2020 11:39:41 +0100
Subject: [PATCH 0433/1035] [libunwind] Provide a way to set
 '_LIBUNWIND_IS_BAREMETAL' through cmake.

Libunwind uses _LIBUNWIND_IS_BAREMETAL in a lot of places but there is no cmake variable to set it. This patch adds such a variable. It is quite like what LIBCXXABI_BAREMETAL does in libcxxabi.

Reviewed By: compnerd, #libunwind

Differential Revision: https://reviews.llvm.org/D84759
---
 libunwind/CMakeLists.txt | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/libunwind/CMakeLists.txt b/libunwind/CMakeLists.txt
index c5b536532f3c2..4606360f07ab7 100644
--- a/libunwind/CMakeLists.txt
+++ b/libunwind/CMakeLists.txt
@@ -137,6 +137,7 @@ option(LIBUNWIND_ENABLE_THREADS "Build libunwind with threading support." ON)
 option(LIBUNWIND_WEAK_PTHREAD_LIB "Use weak references to refer to pthread functions." OFF)
 option(LIBUNWIND_USE_COMPILER_RT "Use compiler-rt instead of libgcc" OFF)
 option(LIBUNWIND_INCLUDE_DOCS "Build the libunwind documentation." ${LLVM_INCLUDE_DOCS})
+option(LIBUNWIND_IS_BAREMETAL "Build libunwind for baremetal targets." OFF)
 
 set(LIBUNWIND_LIBDIR_SUFFIX "${LLVM_LIBDIR_SUFFIX}" CACHE STRING
     "Define suffix of library directory name (32/64)")
@@ -361,6 +362,10 @@ if (LIBUNWIND_ENABLE_ARM_WMMX)
   add_compile_flags(-D__ARM_WMMX)
 endif()
 
+if(LIBUNWIND_IS_BAREMETAL)
+  add_compile_definitions(_LIBUNWIND_IS_BAREMETAL)
+endif()
+
 # This is the _ONLY_ place where add_definitions is called.
 if (MSVC)
   add_definitions(-D_CRT_SECURE_NO_WARNINGS)

From 08a265435bc359841c8618081db6a698cef1f668 Mon Sep 17 00:00:00 2001
From: Georgii Rymar <grimar@accesssoftek.com>
Date: Wed, 15 Jul 2020 21:12:20 +0300
Subject: [PATCH 0434/1035] [llvm-readobj] - Move out the common code from
 printRelocations() methods.

This introduces the printRelocationsHelper() which now contains the common
code used by both GNU and LLVM output styles.

Differential revision: https://reviews.llvm.org/D83935
---
 llvm/tools/llvm-readobj/ELFDumper.cpp | 260 ++++++++++++++------------
 1 file changed, 139 insertions(+), 121 deletions(-)

diff --git a/llvm/tools/llvm-readobj/ELFDumper.cpp b/llvm/tools/llvm-readobj/ELFDumper.cpp
index 590108a86fa7e..0e9551edfb1b7 100644
--- a/llvm/tools/llvm-readobj/ELFDumper.cpp
+++ b/llvm/tools/llvm-readobj/ELFDumper.cpp
@@ -703,9 +703,7 @@ template <class ELFT> class MipsGOTParser;
 
 template <typename ELFT> class DumpStyle {
 public:
-  using Elf_Shdr = typename ELFT::Shdr;
-  using Elf_Sym = typename ELFT::Sym;
-  using Elf_Addr = typename ELFT::Addr;
+  TYPEDEF_ELF_TYPES(ELFT)
 
   DumpStyle(ELFDumper<ELFT> *Dumper) : Dumper(Dumper) {
     FileName = this->Dumper->getElfObject()->getFileName();
@@ -769,6 +767,15 @@ template <typename ELFT> class DumpStyle {
       function_ref<void(const Elf_Shdr &)> OnSectionStart,
       function_ref<void(StringRef, uint64_t)> OnSectionEntry);
 
+  virtual void printRelReloc(const ELFO *Obj, unsigned SecIndex,
+                             const Elf_Shdr *SymTab, const Elf_Rel &R,
+                             unsigned RelIndex) = 0;
+  virtual void printRelaReloc(const ELFO *Obj, unsigned SecIndex,
+                              const Elf_Shdr *SymTab, const Elf_Rela &R,
+                              unsigned RelIndex) = 0;
+  virtual void printRelrReloc(const Elf_Relr &R) = 0;
+  void printRelocationsHelper(const ELFFile<ELFT> *Obj, const Elf_Shdr &Sec);
+
   void reportUniqueWarning(Error Err) const;
   StringRef FileName;
 
@@ -880,13 +887,21 @@ template <typename ELFT> class GNUStyle : public DumpStyle<ELFT> {
   void printHashedSymbol(const ELFO *Obj, const Elf_Sym *FirstSym, uint32_t Sym,
                          StringRef StrTable, uint32_t Bucket);
   void printRelocHeader(unsigned SType);
+
+  void printRelReloc(const ELFO *Obj, unsigned SecIndex, const Elf_Shdr *SymTab,
+                     const Elf_Rel &R, unsigned RelIndex) override;
+  void printRelaReloc(const ELFO *Obj, unsigned SecIndex,
+                      const Elf_Shdr *SymTab, const Elf_Rela &R,
+                      unsigned RelIndex) override;
+  void printRelrReloc(const Elf_Relr &R) override;
+
   template <class RelTy>
-  void printRelocation(const ELFO *Obj, unsigned SecIndex,
-                       const Elf_Shdr *SymTab, const RelTy &R,
-                       unsigned RelIndex);
+  void printRelRelaReloc(const ELFO *Obj, unsigned SecIndex,
+                         const Elf_Shdr *SymTab, const RelTy &R,
+                         unsigned RelIndex);
   template <class RelTy>
-  void printRelocation(const ELFO *Obj, const Elf_Sym *Sym,
-                       StringRef SymbolName, const RelTy &R);
+  void printRelRelaReloc(const ELFO *Obj, const Elf_Sym *Sym,
+                         StringRef SymbolName, const RelTy &R);
   void printSymbol(const ELFO *Obj, const Elf_Sym *Symbol, const Elf_Sym *First,
                    Optional<StringRef> StrTable, bool IsDynamic,
                    bool NonVisibilityBitsUsed) override;
@@ -924,7 +939,6 @@ template <typename ELFT> class LLVMStyle : public DumpStyle<ELFT> {
   void printFileHeaders(const ELFO *Obj) override;
   void printGroupSections(const ELFFile<ELFT> *Obj) override;
   void printRelocations(const ELFO *Obj) override;
-  void printRelocations(const Elf_Shdr *Sec, const ELFO *Obj);
   void printSectionHeaders(const ELFO *Obj) override;
   void printSymbols(const ELFO *Obj, bool PrintSymbols,
                     bool PrintDynamicSymbols) override;
@@ -951,9 +965,15 @@ template <typename ELFT> class LLVMStyle : public DumpStyle<ELFT> {
   void printMipsABIFlags(const ELFObjectFile<ELFT> *Obj) override;
 
 private:
+  void printRelReloc(const ELFO *Obj, unsigned SecIndex, const Elf_Shdr *SymTab,
+                     const Elf_Rel &R, unsigned RelIndex) override;
+  void printRelaReloc(const ELFO *Obj, unsigned SecIndex,
+                      const Elf_Shdr *SymTab, const Elf_Rela &R,
+                      unsigned RelIndex) override;
+  void printRelrReloc(const Elf_Relr &R) override;
   template <class RelTy>
-  void printRelocation(const ELFO *Obj, unsigned SecIndex, const RelTy &Rel,
-                       unsigned RelIndex, const Elf_Shdr *SymTab);
+  void printRelRelaReloc(const ELFO *Obj, unsigned SecIndex, const RelTy &Rel,
+                         unsigned RelIndex, const Elf_Shdr *SymTab);
   template <class RelTy>
   void printDynamicRelocation(const ELFO *Obj, const RelTy& Rel);
 
@@ -3572,11 +3592,29 @@ template <class ELFT> void GNUStyle<ELFT>::printGroupSections(const ELFO *Obj) {
     OS << "There are no section groups in this file.\n";
 }
 
+template <class ELFT>
+void GNUStyle<ELFT>::printRelReloc(const ELFO *Obj, unsigned SecIndex,
+                                   const Elf_Shdr *SymTab, const Elf_Rel &R,
+                                   unsigned RelIndex) {
+  printRelRelaReloc(Obj, SecIndex, SymTab, R, RelIndex);
+}
+
+template <class ELFT>
+void GNUStyle<ELFT>::printRelaReloc(const ELFO *Obj, unsigned SecIndex,
+                                    const Elf_Shdr *SymTab, const Elf_Rela &R,
+                                    unsigned RelIndex) {
+  printRelRelaReloc(Obj, SecIndex, SymTab, R, RelIndex);
+}
+
+template <class ELFT> void GNUStyle<ELFT>::printRelrReloc(const Elf_Relr &R) {
+  OS << to_string(format_hex_no_prefix(R, ELFT::Is64Bits ? 16 : 8)) << "\n";
+}
+
 template <class ELFT>
 template <class RelTy>
-void GNUStyle<ELFT>::printRelocation(const ELFO *Obj, unsigned SecIndex,
-                                     const Elf_Shdr *SymTab, const RelTy &R,
-                                     unsigned RelIndex) {
+void GNUStyle<ELFT>::printRelRelaReloc(const ELFO *Obj, unsigned SecIndex,
+                                       const Elf_Shdr *SymTab, const RelTy &R,
+                                       unsigned RelIndex) {
   Expected<std::pair<const typename ELFT::Sym *, std::string>> Target =
       this->dumper()->getRelocationTarget(SymTab, R);
   if (!Target)
@@ -3584,7 +3622,7 @@ void GNUStyle<ELFT>::printRelocation(const ELFO *Obj, unsigned SecIndex,
         "unable to print relocation " + Twine(RelIndex) + " in section " +
         Twine(SecIndex) + ": " + toString(Target.takeError())));
   else
-    printRelocation(Obj, /*Sym=*/Target->first, /*Name=*/Target->second, R);
+    printRelRelaReloc(Obj, /*Sym=*/Target->first, /*Name=*/Target->second, R);
 }
 
 template <class ELFT>
@@ -3599,8 +3637,8 @@ static Optional<int64_t> getAddend(const typename ELFT::Rel &) {
 
 template <class ELFT>
 template <class RelTy>
-void GNUStyle<ELFT>::printRelocation(const ELFO *Obj, const Elf_Sym *Sym,
-                                     StringRef SymbolName, const RelTy &R) {
+void GNUStyle<ELFT>::printRelRelaReloc(const ELFO *Obj, const Elf_Sym *Sym,
+                                       StringRef SymbolName, const RelTy &R) {
   // First two fields are bit width dependent. The rest of them are fixed width.
   unsigned Bias = ELFT::Is64Bits ? 8 : 0;
   Field Fields[5] = {0, 10 + Bias, 19 + 2 * Bias, 42 + 2 * Bias, 53 + 2 * Bias};
@@ -3657,73 +3695,42 @@ template <class ELFT> void GNUStyle<ELFT>::printRelocHeader(unsigned SType) {
   OS << "\n";
 }
 
+template <class ELFT>
+static bool isRelocationSec(const typename ELFT::Shdr &Sec) {
+  return Sec.sh_type == ELF::SHT_REL || Sec.sh_type == ELF::SHT_RELA ||
+         Sec.sh_type == ELF::SHT_RELR || Sec.sh_type == ELF::SHT_ANDROID_REL ||
+         Sec.sh_type == ELF::SHT_ANDROID_RELA ||
+         Sec.sh_type == ELF::SHT_ANDROID_RELR;
+}
+
 template <class ELFT> void GNUStyle<ELFT>::printRelocations(const ELFO *Obj) {
   bool HasRelocSections = false;
   for (const Elf_Shdr &Sec : cantFail(Obj->sections())) {
-    if (Sec.sh_type != ELF::SHT_REL && Sec.sh_type != ELF::SHT_RELA &&
-        Sec.sh_type != ELF::SHT_RELR && Sec.sh_type != ELF::SHT_ANDROID_REL &&
-        Sec.sh_type != ELF::SHT_ANDROID_RELA &&
-        Sec.sh_type != ELF::SHT_ANDROID_RELR)
+    if (!isRelocationSec<ELFT>(Sec))
       continue;
     HasRelocSections = true;
+
     StringRef Name = unwrapOrError(this->FileName, Obj->getSectionName(&Sec));
-    unsigned Entries = Sec.getEntityCount();
-    std::vector<Elf_Rela> AndroidRelas;
+    unsigned Entries;
+    // Android's packed relocation section needs to be unpacked first
+    // to get the actual number of entries.
     if (Sec.sh_type == ELF::SHT_ANDROID_REL ||
         Sec.sh_type == ELF::SHT_ANDROID_RELA) {
-      // Android's packed relocation section needs to be unpacked first
-      // to get the actual number of entries.
-      AndroidRelas = unwrapOrError(this->FileName, Obj->android_relas(&Sec));
-      Entries = AndroidRelas.size();
-    }
-    std::vector<Elf_Rel> RelrRels;
-    if (!opts::RawRelr && (Sec.sh_type == ELF::SHT_RELR ||
-                           Sec.sh_type == ELF::SHT_ANDROID_RELR)) {
-      // .relr.dyn relative relocation section needs to be unpacked first
-      // to get the actual number of entries.
+      Entries = unwrapOrError(this->FileName, Obj->android_relas(&Sec)).size();
+    } else if (!opts::RawRelr && (Sec.sh_type == ELF::SHT_RELR ||
+                                  Sec.sh_type == ELF::SHT_ANDROID_RELR)) {
       Elf_Relr_Range Relrs = unwrapOrError(this->FileName, Obj->relrs(&Sec));
-      RelrRels = unwrapOrError(this->FileName, Obj->decode_relrs(Relrs));
-      Entries = RelrRels.size();
+      Entries = unwrapOrError(this->FileName, Obj->decode_relrs(Relrs)).size();
+    } else {
+      Entries = Sec.getEntityCount();
     }
+
     uintX_t Offset = Sec.sh_offset;
     OS << "\nRelocation section '" << Name << "' at offset 0x"
        << to_hexString(Offset, false) << " contains " << Entries
        << " entries:\n";
     printRelocHeader(Sec.sh_type);
-    const Elf_Shdr *SymTab =
-        unwrapOrError(this->FileName, Obj->getSection(Sec.sh_link));
-    unsigned SecNdx = &Sec - &cantFail(Obj->sections()).front();
-    unsigned RelNdx = 0;
-
-    switch (Sec.sh_type) {
-    case ELF::SHT_REL:
-      for (const Elf_Rel &R : unwrapOrError(this->FileName, Obj->rels(&Sec)))
-        printRelocation(Obj, SecNdx, SymTab, R, ++RelNdx);
-      break;
-    case ELF::SHT_RELA:
-      for (const Elf_Rela &R : unwrapOrError(this->FileName, Obj->relas(&Sec)))
-        printRelocation(Obj, SecNdx, SymTab, R, ++RelNdx);
-      break;
-    case ELF::SHT_RELR:
-    case ELF::SHT_ANDROID_RELR:
-      if (opts::RawRelr)
-        for (const Elf_Relr &R :
-             unwrapOrError(this->FileName, Obj->relrs(&Sec)))
-          OS << to_string(format_hex_no_prefix(R, ELFT::Is64Bits ? 16 : 8))
-             << "\n";
-      else
-        for (const Elf_Rel &R : RelrRels)
-          printRelocation(Obj, SecNdx, SymTab, R, ++RelNdx);
-      break;
-    case ELF::SHT_ANDROID_REL:
-      for (const Elf_Rela &R : AndroidRelas)
-        printRelocation(Obj, SecNdx, SymTab, (const Elf_Rel &)R, ++RelNdx);
-      break;
-    case ELF::SHT_ANDROID_RELA:
-      for (const Elf_Rela &R : AndroidRelas)
-        printRelocation(Obj, SecNdx, SymTab, R, ++RelNdx);
-      break;
-    }
+    this->printRelocationsHelper(Obj, Sec);
   }
   if (!HasRelocSections)
     OS << "\nThere are no relocations in this file.\n";
@@ -4370,7 +4377,7 @@ template <class ELFT>
 template <class RelTy>
 void GNUStyle<ELFT>::printDynamicRelocation(const ELFO *Obj, const RelTy &R) {
   RelSymbol<ELFT> S = getSymbolForReloc(Obj, this->FileName, this->dumper(), R);
-  printRelocation(Obj, S.Sym, S.Name, R);
+  printRelRelaReloc(Obj, S.Sym, S.Name, R);
 }
 
 template <class ELFT>
@@ -5450,6 +5457,46 @@ void DumpStyle<ELFT>::printDependentLibsHelper(
   }
 }
 
+template <class ELFT>
+void DumpStyle<ELFT>::printRelocationsHelper(const ELFFile<ELFT> *Obj,
+                                             const Elf_Shdr &Sec) {
+  const Elf_Shdr *SymTab =
+      unwrapOrError(this->FileName, Obj->getSection(Sec.sh_link));
+  unsigned SecNdx = &Sec - &cantFail(Obj->sections()).front();
+  unsigned RelNdx = 0;
+
+  switch (Sec.sh_type) {
+  case ELF::SHT_REL:
+    for (const Elf_Rel &R : unwrapOrError(this->FileName, Obj->rels(&Sec)))
+      printRelReloc(Obj, SecNdx, SymTab, R, ++RelNdx);
+    break;
+  case ELF::SHT_RELA:
+    for (const Elf_Rela &R : unwrapOrError(this->FileName, Obj->relas(&Sec)))
+      printRelaReloc(Obj, SecNdx, SymTab, R, ++RelNdx);
+    break;
+  case ELF::SHT_RELR:
+  case ELF::SHT_ANDROID_RELR: {
+    Elf_Relr_Range Relrs = unwrapOrError(this->FileName, Obj->relrs(&Sec));
+    if (opts::RawRelr) {
+      for (const Elf_Relr &R : Relrs)
+        printRelrReloc(R);
+      break;
+    }
+    std::vector<Elf_Rel> RelrRels =
+        unwrapOrError(this->FileName, Obj->decode_relrs(Relrs));
+    for (const Elf_Rel &R : RelrRels)
+      printRelReloc(Obj, SecNdx, SymTab, R, ++RelNdx);
+    break;
+  }
+  case ELF::SHT_ANDROID_REL:
+  case ELF::SHT_ANDROID_RELA:
+    for (const Elf_Rela &R :
+         unwrapOrError(this->FileName, Obj->android_relas(&Sec)))
+      printRelaReloc(Obj, SecNdx, SymTab, R, ++RelNdx);
+    break;
+  }
+}
+
 template <class ELFT>
 void GNUStyle<ELFT>::printDependentLibs(const ELFFile<ELFT> *Obj) {
   bool SectionStarted = false;
@@ -6059,72 +6106,43 @@ void LLVMStyle<ELFT>::printGroupSections(const ELFO *Obj) {
 template <class ELFT> void LLVMStyle<ELFT>::printRelocations(const ELFO *Obj) {
   ListScope D(W, "Relocations");
 
-  int SectionNumber = -1;
   for (const Elf_Shdr &Sec : cantFail(Obj->sections())) {
-    ++SectionNumber;
-
-    if (Sec.sh_type != ELF::SHT_REL && Sec.sh_type != ELF::SHT_RELA &&
-        Sec.sh_type != ELF::SHT_RELR && Sec.sh_type != ELF::SHT_ANDROID_REL &&
-        Sec.sh_type != ELF::SHT_ANDROID_RELA &&
-        Sec.sh_type != ELF::SHT_ANDROID_RELR)
+    if (!isRelocationSec<ELFT>(Sec))
       continue;
 
     StringRef Name = unwrapOrError(this->FileName, Obj->getSectionName(&Sec));
-
-    W.startLine() << "Section (" << SectionNumber << ") " << Name << " {\n";
+    unsigned SecNdx = &Sec - &cantFail(Obj->sections()).front();
+    W.startLine() << "Section (" << SecNdx << ") " << Name << " {\n";
     W.indent();
-
-    printRelocations(&Sec, Obj);
-
+    this->printRelocationsHelper(Obj, Sec);
     W.unindent();
     W.startLine() << "}\n";
   }
 }
 
 template <class ELFT>
-void LLVMStyle<ELFT>::printRelocations(const Elf_Shdr *Sec, const ELFO *Obj) {
-  const Elf_Shdr *SymTab =
-      unwrapOrError(this->FileName, Obj->getSection(Sec->sh_link));
-  unsigned SecNdx = Sec - &cantFail(Obj->sections()).front();
-  unsigned RelNdx = 0;
+void LLVMStyle<ELFT>::printRelReloc(const ELFO *Obj, unsigned SecIndex,
+                                    const Elf_Shdr *SymTab, const Elf_Rel &R,
+                                    unsigned RelIndex) {
+  printRelRelaReloc(Obj, SecIndex, R, RelIndex, SymTab);
+}
 
-  switch (Sec->sh_type) {
-  case ELF::SHT_REL:
-    for (const Elf_Rel &R : unwrapOrError(this->FileName, Obj->rels(Sec)))
-      printRelocation(Obj, SecNdx, R, ++RelNdx, SymTab);
-    break;
-  case ELF::SHT_RELA:
-    for (const Elf_Rela &R : unwrapOrError(this->FileName, Obj->relas(Sec)))
-      printRelocation(Obj, SecNdx, R, ++RelNdx, SymTab);
-    break;
-  case ELF::SHT_RELR:
-  case ELF::SHT_ANDROID_RELR: {
-    Elf_Relr_Range Relrs = unwrapOrError(this->FileName, Obj->relrs(Sec));
-    if (opts::RawRelr) {
-      for (const Elf_Relr &R : Relrs)
-        W.startLine() << W.hex(R) << "\n";
-    } else {
-      std::vector<Elf_Rel> RelrRels =
-          unwrapOrError(this->FileName, Obj->decode_relrs(Relrs));
-      for (const Elf_Rel &R : RelrRels)
-        printRelocation(Obj, SecNdx, R, ++RelNdx, SymTab);
-    }
-    break;
-  }
-  case ELF::SHT_ANDROID_REL:
-  case ELF::SHT_ANDROID_RELA:
-    for (const Elf_Rela &R :
-         unwrapOrError(this->FileName, Obj->android_relas(Sec)))
-      printRelocation(Obj, SecNdx, R, ++RelNdx, SymTab);
-    break;
-  }
+template <class ELFT>
+void LLVMStyle<ELFT>::printRelaReloc(const ELFO *Obj, unsigned SecIndex,
+                                     const Elf_Shdr *SymTab, const Elf_Rela &R,
+                                     unsigned RelIndex) {
+  printRelRelaReloc(Obj, SecIndex, R, RelIndex, SymTab);
+}
+
+template <class ELFT> void LLVMStyle<ELFT>::printRelrReloc(const Elf_Relr &R) {
+  W.startLine() << W.hex(R) << "\n";
 }
 
 template <class ELFT>
 template <class RelTy>
-void LLVMStyle<ELFT>::printRelocation(const ELFO *Obj, unsigned SecIndex,
-                                      const RelTy &Rel, unsigned RelIndex,
-                                      const Elf_Shdr *SymTab) {
+void LLVMStyle<ELFT>::printRelRelaReloc(const ELFO *Obj, unsigned SecIndex,
+                                        const RelTy &Rel, unsigned RelIndex,
+                                        const Elf_Shdr *SymTab) {
   Expected<std::pair<const typename ELFT::Sym *, std::string>> Target =
       this->dumper()->getRelocationTarget(SymTab, Rel);
   if (!Target) {
@@ -6187,7 +6205,7 @@ void LLVMStyle<ELFT>::printSectionHeaders(const ELFO *Obj) {
 
     if (opts::SectionRelocations) {
       ListScope D(W, "Relocations");
-      printRelocations(&Sec, Obj);
+      this->printRelocationsHelper(Obj, Sec);
     }
 
     if (opts::SectionSymbols) {

From c5ef1f1edd4d539df530b07daf7a0c8b94d0efc8 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Wed, 29 Jul 2020 11:40:46 +0100
Subject: [PATCH 0435/1035] [TTI] Add default cost expansion for
 abs/smax/smin/umax/umin intrinsics

---
 llvm/include/llvm/CodeGen/BasicTTIImpl.h      |  20 +
 llvm/test/Analysis/CostModel/X86/abs.ll       | 497 +++++++++++-------
 .../Analysis/CostModel/X86/arith-sminmax.ll   | 450 ++++++++++------
 .../Analysis/CostModel/X86/arith-uminmax.ll   | 450 ++++++++++------
 4 files changed, 902 insertions(+), 515 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index 9ee1d13a34d67..aabfb74b7cefc 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -1225,6 +1225,26 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
       IntrinsicCostAttributes Attrs(IID, RetTy, Args[0]->getType(), FMF, 1, I);
       return getIntrinsicInstrCost(Attrs, CostKind);
     }
+    case Intrinsic::abs:
+    case Intrinsic::smax:
+    case Intrinsic::smin:
+    case Intrinsic::umax:
+    case Intrinsic::umin: {
+      // abs(X) = select(icmp(X,0),X,sub(0,X))
+      // minmax(X,Y) = select(icmp(X,Y),X,Y)
+      Type *CondTy = RetTy->getWithNewBitWidth(1);
+      unsigned Cost = 0;
+      // TODO: Ideally getCmpSelInstrCost would accept an icmp condition code.
+      Cost += thisT()->getCmpSelInstrCost(BinaryOperator::ICmp, RetTy, CondTy,
+                                          CostKind);
+      Cost += thisT()->getCmpSelInstrCost(BinaryOperator::Select, RetTy, CondTy,
+                                          CostKind);
+      // TODO: Should we add an OperandValueProperties::OP_Zero property?
+      if (IID == Intrinsic::abs)
+        Cost += thisT()->getArithmeticInstrCost(
+            BinaryOperator::Sub, RetTy, CostKind, TTI::OK_UniformConstantValue);
+      return Cost;
+    }
     case Intrinsic::fshl:
     case Intrinsic::fshr: {
       const Value *X = Args[0];
diff --git a/llvm/test/Analysis/CostModel/X86/abs.ll b/llvm/test/Analysis/CostModel/X86/abs.ll
index ee61f5c07a393..9d9784ab1c29f 100644
--- a/llvm/test/Analysis/CostModel/X86/abs.ll
+++ b/llvm/test/Analysis/CostModel/X86/abs.ll
@@ -1,6 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+sse2 | FileCheck %s -check-prefixes=CHECK,SSE,SSE2
-; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+sse2 | FileCheck %s -check-prefixes=CHECK,SSE,SSE2
 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+ssse3 | FileCheck %s -check-prefixes=CHECK,SSE,SSSE3
 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+sse4.2 | FileCheck %s -check-prefixes=CHECK,SSE,SSE42
 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+avx | FileCheck %s -check-prefixes=CHECK,AVX,AVX1
@@ -15,38 +14,45 @@
 
 define void @cost_abs_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 x i64> %a512) {
 ; SSE2-LABEL: 'cost_abs_i64'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.abs.i64(i64 %a64, i1 false)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2I64 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %a128, i1 false)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %a256, i1 false)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V8I64 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> %a512, i1 false)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I64 = call i64 @llvm.abs.i64(i64 %a64, i1 false)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2I64 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %a128, i1 false)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %a256, i1 false)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V8I64 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> %a512, i1 false)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSSE3-LABEL: 'cost_abs_i64'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.abs.i64(i64 %a64, i1 false)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2I64 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %a128, i1 false)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %a256, i1 false)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V8I64 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> %a512, i1 false)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I64 = call i64 @llvm.abs.i64(i64 %a64, i1 false)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2I64 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %a128, i1 false)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %a256, i1 false)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V8I64 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> %a512, i1 false)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSE42-LABEL: 'cost_abs_i64'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.abs.i64(i64 %a64, i1 false)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %a128, i1 false)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %a256, i1 false)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I64 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> %a512, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I64 = call i64 @llvm.abs.i64(i64 %a64, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I64 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %a128, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %a256, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8I64 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> %a512, i1 false)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
-; AVX-LABEL: 'cost_abs_i64'
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.abs.i64(i64 %a64, i1 false)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %a128, i1 false)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %a256, i1 false)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V8I64 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> %a512, i1 false)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; AVX1-LABEL: 'cost_abs_i64'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I64 = call i64 @llvm.abs.i64(i64 %a64, i1 false)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I64 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %a128, i1 false)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %a256, i1 false)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V8I64 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> %a512, i1 false)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX2-LABEL: 'cost_abs_i64'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I64 = call i64 @llvm.abs.i64(i64 %a64, i1 false)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I64 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %a128, i1 false)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %a256, i1 false)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8I64 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> %a512, i1 false)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX512-LABEL: 'cost_abs_i64'
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.abs.i64(i64 %a64, i1 false)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %a128, i1 false)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %a256, i1 false)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V8I64 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> %a512, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I64 = call i64 @llvm.abs.i64(i64 %a64, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I64 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %a128, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %a256, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I64 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> %a512, i1 false)
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %I64    = call i64 @llvm.abs.i64(i64 %a64, i1 0)
@@ -58,38 +64,45 @@ define void @cost_abs_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 x i64>
 
 define void @cost_abs_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 x i32> %a512) {
 ; SSE2-LABEL: 'cost_abs_i32'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.abs.i32(i32 %a32, i1 false)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V2I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %a128, i1 false)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V4I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %a256, i1 false)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V8I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> %a512, i1 false)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I32 = call i32 @llvm.abs.i32(i32 %a32, i1 false)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %a128, i1 false)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %a256, i1 false)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V8I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> %a512, i1 false)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSSE3-LABEL: 'cost_abs_i32'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.abs.i32(i32 %a32, i1 false)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V2I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %a128, i1 false)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V4I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %a256, i1 false)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V8I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> %a512, i1 false)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I32 = call i32 @llvm.abs.i32(i32 %a32, i1 false)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %a128, i1 false)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %a256, i1 false)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V8I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> %a512, i1 false)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSE42-LABEL: 'cost_abs_i32'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.abs.i32(i32 %a32, i1 false)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %a128, i1 false)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V4I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %a256, i1 false)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V8I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> %a512, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I32 = call i32 @llvm.abs.i32(i32 %a32, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %a128, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %a256, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> %a512, i1 false)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
-; AVX-LABEL: 'cost_abs_i32'
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.abs.i32(i32 %a32, i1 false)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %a128, i1 false)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V4I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %a256, i1 false)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 58 for instruction: %V8I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> %a512, i1 false)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; AVX1-LABEL: 'cost_abs_i32'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I32 = call i32 @llvm.abs.i32(i32 %a32, i1 false)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %a128, i1 false)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %a256, i1 false)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V8I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> %a512, i1 false)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX2-LABEL: 'cost_abs_i32'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I32 = call i32 @llvm.abs.i32(i32 %a32, i1 false)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %a128, i1 false)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %a256, i1 false)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> %a512, i1 false)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX512-LABEL: 'cost_abs_i32'
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.abs.i32(i32 %a32, i1 false)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %a128, i1 false)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V4I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %a256, i1 false)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %V8I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> %a512, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I32 = call i32 @llvm.abs.i32(i32 %a32, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %a128, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %a256, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> %a512, i1 false)
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %I32   = call i32 @llvm.abs.i32(i32 %a32, i1 0)
@@ -100,26 +113,61 @@ define void @cost_abs_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 x i32>
 }
 
 define void @cost_abs_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <32 x i16> %a512) {
-; SSE-LABEL: 'cost_abs_i16'
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.abs.i16(i16 %a16, i1 false)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 false)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a256, i1 false)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %a512, i1 false)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; AVX-LABEL: 'cost_abs_i16'
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.abs.i16(i16 %a16, i1 false)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 false)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 57 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a256, i1 false)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 114 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %a512, i1 false)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; AVX512-LABEL: 'cost_abs_i16'
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.abs.i16(i16 %a16, i1 false)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 false)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 57 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a256, i1 false)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 123 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %a512, i1 false)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; SSE2-LABEL: 'cost_abs_i16'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I16 = call i16 @llvm.abs.i16(i16 %a16, i1 false)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 false)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a256, i1 false)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %a512, i1 false)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; SSSE3-LABEL: 'cost_abs_i16'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I16 = call i16 @llvm.abs.i16(i16 %a16, i1 false)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 false)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a256, i1 false)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %a512, i1 false)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; SSE42-LABEL: 'cost_abs_i16'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I16 = call i16 @llvm.abs.i16(i16 %a16, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a256, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %a512, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX1-LABEL: 'cost_abs_i16'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I16 = call i16 @llvm.abs.i16(i16 %a16, i1 false)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 false)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a256, i1 false)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %a512, i1 false)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX2-LABEL: 'cost_abs_i16'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I16 = call i16 @llvm.abs.i16(i16 %a16, i1 false)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 false)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a256, i1 false)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %a512, i1 false)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX512F-LABEL: 'cost_abs_i16'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I16 = call i16 @llvm.abs.i16(i16 %a16, i1 false)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 false)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a256, i1 false)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %a512, i1 false)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX512DQ-LABEL: 'cost_abs_i16'
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I16 = call i16 @llvm.abs.i16(i16 %a16, i1 false)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 false)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a256, i1 false)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %a512, i1 false)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX512BW-LABEL: 'cost_abs_i16'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I16 = call i16 @llvm.abs.i16(i16 %a16, i1 false)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 false)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a256, i1 false)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %a512, i1 false)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %I16    = call i16 @llvm.abs.i16(i16 %a16, i1 0)
   %V8I16  = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 0)
@@ -130,39 +178,60 @@ define void @cost_abs_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <32 x i16
 
 define void @cost_abs_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x i8> %a512) {
 ; SSE2-LABEL: 'cost_abs_i8'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.abs.i8(i8 %a8, i1 false)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 78 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 false)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 156 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 false)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 312 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 false)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I8 = call i8 @llvm.abs.i8(i8 %a8, i1 false)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 false)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 false)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 false)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSSE3-LABEL: 'cost_abs_i8'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.abs.i8(i8 %a8, i1 false)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 78 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 false)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 156 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 false)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 312 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 false)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I8 = call i8 @llvm.abs.i8(i8 %a8, i1 false)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 false)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 false)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 false)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSE42-LABEL: 'cost_abs_i8'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.abs.i8(i8 %a8, i1 false)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 false)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 false)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I8 = call i8 @llvm.abs.i8(i8 %a8, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 false)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
-; AVX-LABEL: 'cost_abs_i8'
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.abs.i8(i8 %a8, i1 false)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 false)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 113 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 false)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 226 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 false)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; AVX512-LABEL: 'cost_abs_i8'
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.abs.i8(i8 %a8, i1 false)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 false)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 113 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 false)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 243 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 false)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; AVX1-LABEL: 'cost_abs_i8'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I8 = call i8 @llvm.abs.i8(i8 %a8, i1 false)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 false)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 false)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 false)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX2-LABEL: 'cost_abs_i8'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I8 = call i8 @llvm.abs.i8(i8 %a8, i1 false)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 false)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 false)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 false)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX512F-LABEL: 'cost_abs_i8'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I8 = call i8 @llvm.abs.i8(i8 %a8, i1 false)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 false)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 false)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 false)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX512DQ-LABEL: 'cost_abs_i8'
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I8 = call i8 @llvm.abs.i8(i8 %a8, i1 false)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 false)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 false)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 false)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX512BW-LABEL: 'cost_abs_i8'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I8 = call i8 @llvm.abs.i8(i8 %a8, i1 false)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 false)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 false)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 false)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %I8    = call i8 @llvm.abs.i8(i8 %a8, i1 0)
   %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 0)
@@ -177,38 +246,45 @@ define void @cost_abs_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x i8> %a5
 
 define void @cost_abs_i64_poison(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 x i64> %a512) {
 ; SSE2-LABEL: 'cost_abs_i64_poison'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.abs.i64(i64 %a64, i1 true)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2I64 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %a128, i1 true)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %a256, i1 true)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V8I64 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> %a512, i1 true)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I64 = call i64 @llvm.abs.i64(i64 %a64, i1 true)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2I64 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %a128, i1 true)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %a256, i1 true)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V8I64 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> %a512, i1 true)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSSE3-LABEL: 'cost_abs_i64_poison'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.abs.i64(i64 %a64, i1 true)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2I64 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %a128, i1 true)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %a256, i1 true)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V8I64 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> %a512, i1 true)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I64 = call i64 @llvm.abs.i64(i64 %a64, i1 true)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2I64 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %a128, i1 true)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %a256, i1 true)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V8I64 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> %a512, i1 true)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSE42-LABEL: 'cost_abs_i64_poison'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.abs.i64(i64 %a64, i1 true)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %a128, i1 true)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %a256, i1 true)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I64 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> %a512, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I64 = call i64 @llvm.abs.i64(i64 %a64, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I64 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %a128, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %a256, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8I64 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> %a512, i1 true)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
-; AVX-LABEL: 'cost_abs_i64_poison'
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.abs.i64(i64 %a64, i1 true)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %a128, i1 true)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %a256, i1 true)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V8I64 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> %a512, i1 true)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; AVX1-LABEL: 'cost_abs_i64_poison'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I64 = call i64 @llvm.abs.i64(i64 %a64, i1 true)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I64 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %a128, i1 true)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %a256, i1 true)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V8I64 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> %a512, i1 true)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX2-LABEL: 'cost_abs_i64_poison'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I64 = call i64 @llvm.abs.i64(i64 %a64, i1 true)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I64 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %a128, i1 true)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %a256, i1 true)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8I64 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> %a512, i1 true)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX512-LABEL: 'cost_abs_i64_poison'
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.abs.i64(i64 %a64, i1 true)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %a128, i1 true)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %a256, i1 true)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V8I64 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> %a512, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I64 = call i64 @llvm.abs.i64(i64 %a64, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I64 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %a128, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %a256, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I64 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> %a512, i1 true)
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %I64    = call i64 @llvm.abs.i64(i64 %a64, i1 -1)
@@ -220,38 +296,45 @@ define void @cost_abs_i64_poison(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8
 
 define void @cost_abs_i32_poison(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 x i32> %a512) {
 ; SSE2-LABEL: 'cost_abs_i32_poison'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.abs.i32(i32 %a32, i1 true)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V2I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %a128, i1 true)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V4I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %a256, i1 true)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V8I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> %a512, i1 true)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I32 = call i32 @llvm.abs.i32(i32 %a32, i1 true)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %a128, i1 true)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %a256, i1 true)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V8I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> %a512, i1 true)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSSE3-LABEL: 'cost_abs_i32_poison'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.abs.i32(i32 %a32, i1 true)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V2I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %a128, i1 true)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V4I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %a256, i1 true)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V8I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> %a512, i1 true)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I32 = call i32 @llvm.abs.i32(i32 %a32, i1 true)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %a128, i1 true)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %a256, i1 true)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V8I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> %a512, i1 true)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSE42-LABEL: 'cost_abs_i32_poison'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.abs.i32(i32 %a32, i1 true)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %a128, i1 true)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V4I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %a256, i1 true)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V8I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> %a512, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I32 = call i32 @llvm.abs.i32(i32 %a32, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %a128, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %a256, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> %a512, i1 true)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
-; AVX-LABEL: 'cost_abs_i32_poison'
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.abs.i32(i32 %a32, i1 true)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %a128, i1 true)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V4I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %a256, i1 true)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 58 for instruction: %V8I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> %a512, i1 true)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; AVX1-LABEL: 'cost_abs_i32_poison'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I32 = call i32 @llvm.abs.i32(i32 %a32, i1 true)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %a128, i1 true)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %a256, i1 true)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V8I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> %a512, i1 true)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX2-LABEL: 'cost_abs_i32_poison'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I32 = call i32 @llvm.abs.i32(i32 %a32, i1 true)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %a128, i1 true)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %a256, i1 true)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> %a512, i1 true)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX512-LABEL: 'cost_abs_i32_poison'
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.abs.i32(i32 %a32, i1 true)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %a128, i1 true)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V4I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %a256, i1 true)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %V8I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> %a512, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I32 = call i32 @llvm.abs.i32(i32 %a32, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %a128, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %a256, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> %a512, i1 true)
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %I32   = call i32 @llvm.abs.i32(i32 %a32, i1 -1)
@@ -262,26 +345,61 @@ define void @cost_abs_i32_poison(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16
 }
 
 define void @cost_abs_i16_poison(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <32 x i16> %a512) {
-; SSE-LABEL: 'cost_abs_i16_poison'
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.abs.i16(i16 %a16, i1 true)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 true)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a256, i1 true)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %a512, i1 true)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; AVX-LABEL: 'cost_abs_i16_poison'
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.abs.i16(i16 %a16, i1 true)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 true)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 57 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a256, i1 true)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 114 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %a512, i1 true)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; AVX512-LABEL: 'cost_abs_i16_poison'
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.abs.i16(i16 %a16, i1 true)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 true)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 57 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a256, i1 true)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 123 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %a512, i1 true)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; SSE2-LABEL: 'cost_abs_i16_poison'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I16 = call i16 @llvm.abs.i16(i16 %a16, i1 true)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 true)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a256, i1 true)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %a512, i1 true)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; SSSE3-LABEL: 'cost_abs_i16_poison'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I16 = call i16 @llvm.abs.i16(i16 %a16, i1 true)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 true)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a256, i1 true)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %a512, i1 true)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; SSE42-LABEL: 'cost_abs_i16_poison'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I16 = call i16 @llvm.abs.i16(i16 %a16, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a256, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %a512, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX1-LABEL: 'cost_abs_i16_poison'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I16 = call i16 @llvm.abs.i16(i16 %a16, i1 true)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 true)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a256, i1 true)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %a512, i1 true)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX2-LABEL: 'cost_abs_i16_poison'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I16 = call i16 @llvm.abs.i16(i16 %a16, i1 true)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 true)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a256, i1 true)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %a512, i1 true)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX512F-LABEL: 'cost_abs_i16_poison'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I16 = call i16 @llvm.abs.i16(i16 %a16, i1 true)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 true)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a256, i1 true)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %a512, i1 true)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX512DQ-LABEL: 'cost_abs_i16_poison'
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I16 = call i16 @llvm.abs.i16(i16 %a16, i1 true)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 true)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a256, i1 true)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %a512, i1 true)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX512BW-LABEL: 'cost_abs_i16_poison'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I16 = call i16 @llvm.abs.i16(i16 %a16, i1 true)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 true)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a256, i1 true)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %a512, i1 true)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %I16    = call i16 @llvm.abs.i16(i16 %a16, i1 -1)
   %V8I16  = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 -1)
@@ -292,39 +410,60 @@ define void @cost_abs_i16_poison(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3
 
 define void @cost_abs_i8_poison(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x i8> %a512) {
 ; SSE2-LABEL: 'cost_abs_i8_poison'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.abs.i8(i8 %a8, i1 true)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 78 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 true)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 156 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 true)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 312 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 true)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I8 = call i8 @llvm.abs.i8(i8 %a8, i1 true)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 true)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 true)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 true)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSSE3-LABEL: 'cost_abs_i8_poison'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.abs.i8(i8 %a8, i1 true)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 78 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 true)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 156 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 true)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 312 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 true)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I8 = call i8 @llvm.abs.i8(i8 %a8, i1 true)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 true)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 true)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 true)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSE42-LABEL: 'cost_abs_i8_poison'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.abs.i8(i8 %a8, i1 true)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 true)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 true)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I8 = call i8 @llvm.abs.i8(i8 %a8, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 true)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
-; AVX-LABEL: 'cost_abs_i8_poison'
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.abs.i8(i8 %a8, i1 true)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 true)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 113 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 true)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 226 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 true)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; AVX512-LABEL: 'cost_abs_i8_poison'
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.abs.i8(i8 %a8, i1 true)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 true)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 113 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 true)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 243 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 true)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; AVX1-LABEL: 'cost_abs_i8_poison'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I8 = call i8 @llvm.abs.i8(i8 %a8, i1 true)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 true)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 true)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 true)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX2-LABEL: 'cost_abs_i8_poison'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I8 = call i8 @llvm.abs.i8(i8 %a8, i1 true)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 true)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 true)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 true)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX512F-LABEL: 'cost_abs_i8_poison'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I8 = call i8 @llvm.abs.i8(i8 %a8, i1 true)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 true)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 true)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 true)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX512DQ-LABEL: 'cost_abs_i8_poison'
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I8 = call i8 @llvm.abs.i8(i8 %a8, i1 true)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 true)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 true)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 true)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX512BW-LABEL: 'cost_abs_i8_poison'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I8 = call i8 @llvm.abs.i8(i8 %a8, i1 true)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 true)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 true)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 true)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %I8    = call i8 @llvm.abs.i8(i8 %a8, i1 -1)
   %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 -1)
diff --git a/llvm/test/Analysis/CostModel/X86/arith-sminmax.ll b/llvm/test/Analysis/CostModel/X86/arith-sminmax.ll
index 96180d01aae09..2d6f75b6cb92e 100644
--- a/llvm/test/Analysis/CostModel/X86/arith-sminmax.ll
+++ b/llvm/test/Analysis/CostModel/X86/arith-sminmax.ll
@@ -30,99 +30,156 @@ declare <64 x i8>  @llvm.smax.v64i8(<64 x i8>, <64 x i8>)
 
 define i32 @smax(i32 %arg) {
 ; SSE2-LABEL: 'smax'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.smax.i64(i64 undef, i64 undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I64 = call <2 x i64> @llvm.smax.v2i64(<2 x i64> undef, <2 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I64 = call <4 x i64> @llvm.smax.v4i64(<4 x i64> undef, <4 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V8I64 = call <8 x i64> @llvm.smax.v8i64(<8 x i64> undef, <8 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.smax.i32(i32 undef, i32 undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4I32 = call <4 x i32> @llvm.smax.v4i32(<4 x i32> undef, <4 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V8I32 = call <8 x i32> @llvm.smax.v8i32(<8 x i32> undef, <8 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V16I32 = call <16 x i32> @llvm.smax.v16i32(<16 x i32> undef, <16 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.smax.i16(i16 undef, i16 undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I16 = call <8 x i16> @llvm.smax.v8i16(<8 x i16> undef, <8 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I16 = call <16 x i16> @llvm.smax.v16i16(<16 x i16> undef, <16 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V32I16 = call <32 x i16> @llvm.smax.v32i16(<32 x i16> undef, <32 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.smax.i8(i8 undef, i8 undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V16I8 = call <16 x i8> @llvm.smax.v16i8(<16 x i8> undef, <16 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 94 for instruction: %V32I8 = call <32 x i8> @llvm.smax.v32i8(<32 x i8> undef, <32 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 188 for instruction: %V64I8 = call <64 x i8> @llvm.smax.v64i8(<64 x i8> undef, <64 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.smax.i64(i64 undef, i64 undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V2I64 = call <2 x i64> @llvm.smax.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V4I64 = call <4 x i64> @llvm.smax.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V8I64 = call <8 x i64> @llvm.smax.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.smax.i32(i32 undef, i32 undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I32 = call <4 x i32> @llvm.smax.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I32 = call <8 x i32> @llvm.smax.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I32 = call <16 x i32> @llvm.smax.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.smax.i16(i16 undef, i16 undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I16 = call <8 x i16> @llvm.smax.v8i16(<8 x i16> undef, <8 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I16 = call <16 x i16> @llvm.smax.v16i16(<16 x i16> undef, <16 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I16 = call <32 x i16> @llvm.smax.v32i16(<32 x i16> undef, <32 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.smax.i8(i8 undef, i8 undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I8 = call <16 x i8> @llvm.smax.v16i8(<16 x i8> undef, <16 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I8 = call <32 x i8> @llvm.smax.v32i8(<32 x i8> undef, <32 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I8 = call <64 x i8> @llvm.smax.v64i8(<64 x i8> undef, <64 x i8> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'smax'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.smax.i64(i64 undef, i64 undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I64 = call <2 x i64> @llvm.smax.v2i64(<2 x i64> undef, <2 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I64 = call <4 x i64> @llvm.smax.v4i64(<4 x i64> undef, <4 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V8I64 = call <8 x i64> @llvm.smax.v8i64(<8 x i64> undef, <8 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.smax.i32(i32 undef, i32 undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4I32 = call <4 x i32> @llvm.smax.v4i32(<4 x i32> undef, <4 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V8I32 = call <8 x i32> @llvm.smax.v8i32(<8 x i32> undef, <8 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V16I32 = call <16 x i32> @llvm.smax.v16i32(<16 x i32> undef, <16 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.smax.i16(i16 undef, i16 undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I16 = call <8 x i16> @llvm.smax.v8i16(<8 x i16> undef, <8 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I16 = call <16 x i16> @llvm.smax.v16i16(<16 x i16> undef, <16 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V32I16 = call <32 x i16> @llvm.smax.v32i16(<32 x i16> undef, <32 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.smax.i8(i8 undef, i8 undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V16I8 = call <16 x i8> @llvm.smax.v16i8(<16 x i8> undef, <16 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 94 for instruction: %V32I8 = call <32 x i8> @llvm.smax.v32i8(<32 x i8> undef, <32 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 188 for instruction: %V64I8 = call <64 x i8> @llvm.smax.v64i8(<64 x i8> undef, <64 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.smax.i64(i64 undef, i64 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V2I64 = call <2 x i64> @llvm.smax.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V4I64 = call <4 x i64> @llvm.smax.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V8I64 = call <8 x i64> @llvm.smax.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.smax.i32(i32 undef, i32 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I32 = call <4 x i32> @llvm.smax.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I32 = call <8 x i32> @llvm.smax.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I32 = call <16 x i32> @llvm.smax.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.smax.i16(i16 undef, i16 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I16 = call <8 x i16> @llvm.smax.v8i16(<8 x i16> undef, <8 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I16 = call <16 x i16> @llvm.smax.v16i16(<16 x i16> undef, <16 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I16 = call <32 x i16> @llvm.smax.v32i16(<32 x i16> undef, <32 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.smax.i8(i8 undef, i8 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I8 = call <16 x i8> @llvm.smax.v16i8(<16 x i8> undef, <16 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I8 = call <32 x i8> @llvm.smax.v32i8(<32 x i8> undef, <32 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I8 = call <64 x i8> @llvm.smax.v64i8(<64 x i8> undef, <64 x i8> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'smax'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.smax.i64(i64 undef, i64 undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I64 = call <2 x i64> @llvm.smax.v2i64(<2 x i64> undef, <2 x i64> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4I64 = call <4 x i64> @llvm.smax.v4i64(<4 x i64> undef, <4 x i64> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I64 = call <8 x i64> @llvm.smax.v8i64(<8 x i64> undef, <8 x i64> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.smax.i32(i32 undef, i32 undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4I32 = call <4 x i32> @llvm.smax.v4i32(<4 x i32> undef, <4 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I32 = call <8 x i32> @llvm.smax.v8i32(<8 x i32> undef, <8 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I32 = call <16 x i32> @llvm.smax.v16i32(<16 x i32> undef, <16 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.smax.i16(i16 undef, i16 undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I16 = call <8 x i16> @llvm.smax.v8i16(<8 x i16> undef, <8 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I16 = call <16 x i16> @llvm.smax.v16i16(<16 x i16> undef, <16 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V32I16 = call <32 x i16> @llvm.smax.v32i16(<32 x i16> undef, <32 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.smax.i8(i8 undef, i8 undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I8 = call <16 x i8> @llvm.smax.v16i8(<16 x i8> undef, <16 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V32I8 = call <32 x i8> @llvm.smax.v32i8(<32 x i8> undef, <32 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %V64I8 = call <64 x i8> @llvm.smax.v64i8(<64 x i8> undef, <64 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.smax.i64(i64 undef, i64 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = call <2 x i64> @llvm.smax.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I64 = call <4 x i64> @llvm.smax.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I64 = call <8 x i64> @llvm.smax.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.smax.i32(i32 undef, i32 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.smax.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = call <8 x i32> @llvm.smax.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = call <16 x i32> @llvm.smax.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.smax.i16(i16 undef, i16 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call <8 x i16> @llvm.smax.v8i16(<8 x i16> undef, <8 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = call <16 x i16> @llvm.smax.v16i16(<16 x i16> undef, <16 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call <32 x i16> @llvm.smax.v32i16(<32 x i16> undef, <32 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.smax.i8(i8 undef, i8 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call <16 x i8> @llvm.smax.v16i8(<16 x i8> undef, <16 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = call <32 x i8> @llvm.smax.v32i8(<32 x i8> undef, <32 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = call <64 x i8> @llvm.smax.v64i8(<64 x i8> undef, <64 x i8> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
-; AVX-LABEL: 'smax'
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.smax.i64(i64 undef, i64 undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I64 = call <2 x i64> @llvm.smax.v2i64(<2 x i64> undef, <2 x i64> undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I64 = call <4 x i64> @llvm.smax.v4i64(<4 x i64> undef, <4 x i64> undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V8I64 = call <8 x i64> @llvm.smax.v8i64(<8 x i64> undef, <8 x i64> undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.smax.i32(i32 undef, i32 undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4I32 = call <4 x i32> @llvm.smax.v4i32(<4 x i32> undef, <4 x i32> undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I32 = call <8 x i32> @llvm.smax.v8i32(<8 x i32> undef, <8 x i32> undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V16I32 = call <16 x i32> @llvm.smax.v16i32(<16 x i32> undef, <16 x i32> undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.smax.i16(i16 undef, i16 undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I16 = call <8 x i16> @llvm.smax.v8i16(<8 x i16> undef, <8 x i16> undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V16I16 = call <16 x i16> @llvm.smax.v16i16(<16 x i16> undef, <16 x i16> undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %V32I16 = call <32 x i16> @llvm.smax.v32i16(<32 x i16> undef, <32 x i16> undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.smax.i8(i8 undef, i8 undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I8 = call <16 x i8> @llvm.smax.v16i8(<16 x i8> undef, <16 x i8> undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: %V32I8 = call <32 x i8> @llvm.smax.v32i8(<32 x i8> undef, <32 x i8> undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %V64I8 = call <64 x i8> @llvm.smax.v64i8(<64 x i8> undef, <64 x i8> undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+; AVX1-LABEL: 'smax'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.smax.i64(i64 undef, i64 undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = call <2 x i64> @llvm.smax.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4I64 = call <4 x i64> @llvm.smax.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8I64 = call <8 x i64> @llvm.smax.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.smax.i32(i32 undef, i32 undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.smax.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8I32 = call <8 x i32> @llvm.smax.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V16I32 = call <16 x i32> @llvm.smax.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.smax.i16(i16 undef, i16 undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call <8 x i16> @llvm.smax.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16I16 = call <16 x i16> @llvm.smax.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V32I16 = call <32 x i16> @llvm.smax.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.smax.i8(i8 undef, i8 undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call <16 x i8> @llvm.smax.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V32I8 = call <32 x i8> @llvm.smax.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V64I8 = call <64 x i8> @llvm.smax.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
-; AVX512-LABEL: 'smax'
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.smax.i64(i64 undef, i64 undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I64 = call <2 x i64> @llvm.smax.v2i64(<2 x i64> undef, <2 x i64> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I64 = call <4 x i64> @llvm.smax.v4i64(<4 x i64> undef, <4 x i64> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V8I64 = call <8 x i64> @llvm.smax.v8i64(<8 x i64> undef, <8 x i64> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.smax.i32(i32 undef, i32 undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4I32 = call <4 x i32> @llvm.smax.v4i32(<4 x i32> undef, <4 x i32> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I32 = call <8 x i32> @llvm.smax.v8i32(<8 x i32> undef, <8 x i32> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V16I32 = call <16 x i32> @llvm.smax.v16i32(<16 x i32> undef, <16 x i32> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.smax.i16(i16 undef, i16 undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I16 = call <8 x i16> @llvm.smax.v8i16(<8 x i16> undef, <8 x i16> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V16I16 = call <16 x i16> @llvm.smax.v16i16(<16 x i16> undef, <16 x i16> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %V32I16 = call <32 x i16> @llvm.smax.v32i16(<32 x i16> undef, <32 x i16> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.smax.i8(i8 undef, i8 undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I8 = call <16 x i8> @llvm.smax.v16i8(<16 x i8> undef, <16 x i8> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: %V32I8 = call <32 x i8> @llvm.smax.v32i8(<32 x i8> undef, <32 x i8> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 131 for instruction: %V64I8 = call <64 x i8> @llvm.smax.v64i8(<64 x i8> undef, <64 x i8> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+; AVX2-LABEL: 'smax'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.smax.i64(i64 undef, i64 undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = call <2 x i64> @llvm.smax.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = call <4 x i64> @llvm.smax.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I64 = call <8 x i64> @llvm.smax.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.smax.i32(i32 undef, i32 undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.smax.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call <8 x i32> @llvm.smax.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = call <16 x i32> @llvm.smax.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.smax.i16(i16 undef, i16 undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call <8 x i16> @llvm.smax.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.smax.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.smax.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.smax.i8(i8 undef, i8 undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call <16 x i8> @llvm.smax.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.smax.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.smax.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512F-LABEL: 'smax'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.smax.i64(i64 undef, i64 undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = call <2 x i64> @llvm.smax.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = call <4 x i64> @llvm.smax.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = call <8 x i64> @llvm.smax.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.smax.i32(i32 undef, i32 undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.smax.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call <8 x i32> @llvm.smax.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = call <16 x i32> @llvm.smax.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.smax.i16(i16 undef, i16 undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call <8 x i16> @llvm.smax.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.smax.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.smax.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.smax.i8(i8 undef, i8 undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call <16 x i8> @llvm.smax.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.smax.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.smax.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512DQ-LABEL: 'smax'
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.smax.i64(i64 undef, i64 undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = call <2 x i64> @llvm.smax.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = call <4 x i64> @llvm.smax.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = call <8 x i64> @llvm.smax.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.smax.i32(i32 undef, i32 undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.smax.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call <8 x i32> @llvm.smax.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = call <16 x i32> @llvm.smax.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.smax.i16(i16 undef, i16 undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call <8 x i16> @llvm.smax.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.smax.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.smax.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.smax.i8(i8 undef, i8 undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call <16 x i8> @llvm.smax.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.smax.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.smax.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512BW-LABEL: 'smax'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.smax.i64(i64 undef, i64 undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = call <2 x i64> @llvm.smax.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = call <4 x i64> @llvm.smax.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = call <8 x i64> @llvm.smax.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.smax.i32(i32 undef, i32 undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.smax.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call <8 x i32> @llvm.smax.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = call <16 x i32> @llvm.smax.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.smax.i16(i16 undef, i16 undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call <8 x i16> @llvm.smax.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.smax.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = call <32 x i16> @llvm.smax.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.smax.i8(i8 undef, i8 undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call <16 x i8> @llvm.smax.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.smax.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = call <64 x i8> @llvm.smax.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %I64 = call i64 @llvm.smax.i64(i64 undef, i64 undef)
   %V2I64 = call <2 x i64> @llvm.smax.v2i64(<2 x i64> undef, <2 x i64> undef)
@@ -169,99 +226,156 @@ declare <64 x i8>  @llvm.smin.v64i8(<64 x i8>, <64 x i8>)
 
 define i32 @smin(i32 %arg) {
 ; SSE2-LABEL: 'smin'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.smin.i64(i64 undef, i64 undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I64 = call <2 x i64> @llvm.smin.v2i64(<2 x i64> undef, <2 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I64 = call <4 x i64> @llvm.smin.v4i64(<4 x i64> undef, <4 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V8I64 = call <8 x i64> @llvm.smin.v8i64(<8 x i64> undef, <8 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.smin.i32(i32 undef, i32 undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4I32 = call <4 x i32> @llvm.smin.v4i32(<4 x i32> undef, <4 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V8I32 = call <8 x i32> @llvm.smin.v8i32(<8 x i32> undef, <8 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V16I32 = call <16 x i32> @llvm.smin.v16i32(<16 x i32> undef, <16 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.smin.i16(i16 undef, i16 undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I16 = call <8 x i16> @llvm.smin.v8i16(<8 x i16> undef, <8 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I16 = call <16 x i16> @llvm.smin.v16i16(<16 x i16> undef, <16 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V32I16 = call <32 x i16> @llvm.smin.v32i16(<32 x i16> undef, <32 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.smin.i8(i8 undef, i8 undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V16I8 = call <16 x i8> @llvm.smin.v16i8(<16 x i8> undef, <16 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 94 for instruction: %V32I8 = call <32 x i8> @llvm.smin.v32i8(<32 x i8> undef, <32 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 188 for instruction: %V64I8 = call <64 x i8> @llvm.smin.v64i8(<64 x i8> undef, <64 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.smin.i64(i64 undef, i64 undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V2I64 = call <2 x i64> @llvm.smin.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V4I64 = call <4 x i64> @llvm.smin.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V8I64 = call <8 x i64> @llvm.smin.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.smin.i32(i32 undef, i32 undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I32 = call <4 x i32> @llvm.smin.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I32 = call <8 x i32> @llvm.smin.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I32 = call <16 x i32> @llvm.smin.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.smin.i16(i16 undef, i16 undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I16 = call <8 x i16> @llvm.smin.v8i16(<8 x i16> undef, <8 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I16 = call <16 x i16> @llvm.smin.v16i16(<16 x i16> undef, <16 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I16 = call <32 x i16> @llvm.smin.v32i16(<32 x i16> undef, <32 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.smin.i8(i8 undef, i8 undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I8 = call <16 x i8> @llvm.smin.v16i8(<16 x i8> undef, <16 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I8 = call <32 x i8> @llvm.smin.v32i8(<32 x i8> undef, <32 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I8 = call <64 x i8> @llvm.smin.v64i8(<64 x i8> undef, <64 x i8> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'smin'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.smin.i64(i64 undef, i64 undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I64 = call <2 x i64> @llvm.smin.v2i64(<2 x i64> undef, <2 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I64 = call <4 x i64> @llvm.smin.v4i64(<4 x i64> undef, <4 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V8I64 = call <8 x i64> @llvm.smin.v8i64(<8 x i64> undef, <8 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.smin.i32(i32 undef, i32 undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4I32 = call <4 x i32> @llvm.smin.v4i32(<4 x i32> undef, <4 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V8I32 = call <8 x i32> @llvm.smin.v8i32(<8 x i32> undef, <8 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V16I32 = call <16 x i32> @llvm.smin.v16i32(<16 x i32> undef, <16 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.smin.i16(i16 undef, i16 undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I16 = call <8 x i16> @llvm.smin.v8i16(<8 x i16> undef, <8 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I16 = call <16 x i16> @llvm.smin.v16i16(<16 x i16> undef, <16 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V32I16 = call <32 x i16> @llvm.smin.v32i16(<32 x i16> undef, <32 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.smin.i8(i8 undef, i8 undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V16I8 = call <16 x i8> @llvm.smin.v16i8(<16 x i8> undef, <16 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 94 for instruction: %V32I8 = call <32 x i8> @llvm.smin.v32i8(<32 x i8> undef, <32 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 188 for instruction: %V64I8 = call <64 x i8> @llvm.smin.v64i8(<64 x i8> undef, <64 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.smin.i64(i64 undef, i64 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V2I64 = call <2 x i64> @llvm.smin.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V4I64 = call <4 x i64> @llvm.smin.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V8I64 = call <8 x i64> @llvm.smin.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.smin.i32(i32 undef, i32 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I32 = call <4 x i32> @llvm.smin.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I32 = call <8 x i32> @llvm.smin.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I32 = call <16 x i32> @llvm.smin.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.smin.i16(i16 undef, i16 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I16 = call <8 x i16> @llvm.smin.v8i16(<8 x i16> undef, <8 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I16 = call <16 x i16> @llvm.smin.v16i16(<16 x i16> undef, <16 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I16 = call <32 x i16> @llvm.smin.v32i16(<32 x i16> undef, <32 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.smin.i8(i8 undef, i8 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I8 = call <16 x i8> @llvm.smin.v16i8(<16 x i8> undef, <16 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I8 = call <32 x i8> @llvm.smin.v32i8(<32 x i8> undef, <32 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I8 = call <64 x i8> @llvm.smin.v64i8(<64 x i8> undef, <64 x i8> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'smin'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.smin.i64(i64 undef, i64 undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I64 = call <2 x i64> @llvm.smin.v2i64(<2 x i64> undef, <2 x i64> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4I64 = call <4 x i64> @llvm.smin.v4i64(<4 x i64> undef, <4 x i64> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I64 = call <8 x i64> @llvm.smin.v8i64(<8 x i64> undef, <8 x i64> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.smin.i32(i32 undef, i32 undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4I32 = call <4 x i32> @llvm.smin.v4i32(<4 x i32> undef, <4 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I32 = call <8 x i32> @llvm.smin.v8i32(<8 x i32> undef, <8 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I32 = call <16 x i32> @llvm.smin.v16i32(<16 x i32> undef, <16 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.smin.i16(i16 undef, i16 undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I16 = call <8 x i16> @llvm.smin.v8i16(<8 x i16> undef, <8 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I16 = call <16 x i16> @llvm.smin.v16i16(<16 x i16> undef, <16 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V32I16 = call <32 x i16> @llvm.smin.v32i16(<32 x i16> undef, <32 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.smin.i8(i8 undef, i8 undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I8 = call <16 x i8> @llvm.smin.v16i8(<16 x i8> undef, <16 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V32I8 = call <32 x i8> @llvm.smin.v32i8(<32 x i8> undef, <32 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %V64I8 = call <64 x i8> @llvm.smin.v64i8(<64 x i8> undef, <64 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.smin.i64(i64 undef, i64 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = call <2 x i64> @llvm.smin.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I64 = call <4 x i64> @llvm.smin.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I64 = call <8 x i64> @llvm.smin.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.smin.i32(i32 undef, i32 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.smin.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = call <8 x i32> @llvm.smin.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = call <16 x i32> @llvm.smin.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.smin.i16(i16 undef, i16 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call <8 x i16> @llvm.smin.v8i16(<8 x i16> undef, <8 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = call <16 x i16> @llvm.smin.v16i16(<16 x i16> undef, <16 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call <32 x i16> @llvm.smin.v32i16(<32 x i16> undef, <32 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.smin.i8(i8 undef, i8 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call <16 x i8> @llvm.smin.v16i8(<16 x i8> undef, <16 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = call <32 x i8> @llvm.smin.v32i8(<32 x i8> undef, <32 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = call <64 x i8> @llvm.smin.v64i8(<64 x i8> undef, <64 x i8> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
-; AVX-LABEL: 'smin'
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.smin.i64(i64 undef, i64 undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I64 = call <2 x i64> @llvm.smin.v2i64(<2 x i64> undef, <2 x i64> undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I64 = call <4 x i64> @llvm.smin.v4i64(<4 x i64> undef, <4 x i64> undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V8I64 = call <8 x i64> @llvm.smin.v8i64(<8 x i64> undef, <8 x i64> undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.smin.i32(i32 undef, i32 undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4I32 = call <4 x i32> @llvm.smin.v4i32(<4 x i32> undef, <4 x i32> undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I32 = call <8 x i32> @llvm.smin.v8i32(<8 x i32> undef, <8 x i32> undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V16I32 = call <16 x i32> @llvm.smin.v16i32(<16 x i32> undef, <16 x i32> undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.smin.i16(i16 undef, i16 undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I16 = call <8 x i16> @llvm.smin.v8i16(<8 x i16> undef, <8 x i16> undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V16I16 = call <16 x i16> @llvm.smin.v16i16(<16 x i16> undef, <16 x i16> undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %V32I16 = call <32 x i16> @llvm.smin.v32i16(<32 x i16> undef, <32 x i16> undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.smin.i8(i8 undef, i8 undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I8 = call <16 x i8> @llvm.smin.v16i8(<16 x i8> undef, <16 x i8> undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: %V32I8 = call <32 x i8> @llvm.smin.v32i8(<32 x i8> undef, <32 x i8> undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %V64I8 = call <64 x i8> @llvm.smin.v64i8(<64 x i8> undef, <64 x i8> undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+; AVX1-LABEL: 'smin'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.smin.i64(i64 undef, i64 undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = call <2 x i64> @llvm.smin.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4I64 = call <4 x i64> @llvm.smin.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8I64 = call <8 x i64> @llvm.smin.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.smin.i32(i32 undef, i32 undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.smin.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8I32 = call <8 x i32> @llvm.smin.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V16I32 = call <16 x i32> @llvm.smin.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.smin.i16(i16 undef, i16 undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call <8 x i16> @llvm.smin.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16I16 = call <16 x i16> @llvm.smin.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V32I16 = call <32 x i16> @llvm.smin.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.smin.i8(i8 undef, i8 undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call <16 x i8> @llvm.smin.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V32I8 = call <32 x i8> @llvm.smin.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V64I8 = call <64 x i8> @llvm.smin.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX2-LABEL: 'smin'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.smin.i64(i64 undef, i64 undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = call <2 x i64> @llvm.smin.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = call <4 x i64> @llvm.smin.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I64 = call <8 x i64> @llvm.smin.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.smin.i32(i32 undef, i32 undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.smin.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call <8 x i32> @llvm.smin.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = call <16 x i32> @llvm.smin.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.smin.i16(i16 undef, i16 undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call <8 x i16> @llvm.smin.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.smin.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.smin.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.smin.i8(i8 undef, i8 undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call <16 x i8> @llvm.smin.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.smin.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.smin.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512F-LABEL: 'smin'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.smin.i64(i64 undef, i64 undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = call <2 x i64> @llvm.smin.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = call <4 x i64> @llvm.smin.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = call <8 x i64> @llvm.smin.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.smin.i32(i32 undef, i32 undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.smin.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call <8 x i32> @llvm.smin.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = call <16 x i32> @llvm.smin.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.smin.i16(i16 undef, i16 undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call <8 x i16> @llvm.smin.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.smin.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.smin.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.smin.i8(i8 undef, i8 undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call <16 x i8> @llvm.smin.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.smin.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.smin.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512DQ-LABEL: 'smin'
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.smin.i64(i64 undef, i64 undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = call <2 x i64> @llvm.smin.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = call <4 x i64> @llvm.smin.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = call <8 x i64> @llvm.smin.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.smin.i32(i32 undef, i32 undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.smin.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call <8 x i32> @llvm.smin.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = call <16 x i32> @llvm.smin.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.smin.i16(i16 undef, i16 undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call <8 x i16> @llvm.smin.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.smin.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.smin.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.smin.i8(i8 undef, i8 undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call <16 x i8> @llvm.smin.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.smin.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.smin.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
-; AVX512-LABEL: 'smin'
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.smin.i64(i64 undef, i64 undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I64 = call <2 x i64> @llvm.smin.v2i64(<2 x i64> undef, <2 x i64> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I64 = call <4 x i64> @llvm.smin.v4i64(<4 x i64> undef, <4 x i64> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V8I64 = call <8 x i64> @llvm.smin.v8i64(<8 x i64> undef, <8 x i64> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.smin.i32(i32 undef, i32 undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4I32 = call <4 x i32> @llvm.smin.v4i32(<4 x i32> undef, <4 x i32> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I32 = call <8 x i32> @llvm.smin.v8i32(<8 x i32> undef, <8 x i32> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V16I32 = call <16 x i32> @llvm.smin.v16i32(<16 x i32> undef, <16 x i32> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.smin.i16(i16 undef, i16 undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I16 = call <8 x i16> @llvm.smin.v8i16(<8 x i16> undef, <8 x i16> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V16I16 = call <16 x i16> @llvm.smin.v16i16(<16 x i16> undef, <16 x i16> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %V32I16 = call <32 x i16> @llvm.smin.v32i16(<32 x i16> undef, <32 x i16> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.smin.i8(i8 undef, i8 undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I8 = call <16 x i8> @llvm.smin.v16i8(<16 x i8> undef, <16 x i8> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: %V32I8 = call <32 x i8> @llvm.smin.v32i8(<32 x i8> undef, <32 x i8> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 131 for instruction: %V64I8 = call <64 x i8> @llvm.smin.v64i8(<64 x i8> undef, <64 x i8> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+; AVX512BW-LABEL: 'smin'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.smin.i64(i64 undef, i64 undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = call <2 x i64> @llvm.smin.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = call <4 x i64> @llvm.smin.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = call <8 x i64> @llvm.smin.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.smin.i32(i32 undef, i32 undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.smin.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call <8 x i32> @llvm.smin.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = call <16 x i32> @llvm.smin.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.smin.i16(i16 undef, i16 undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call <8 x i16> @llvm.smin.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.smin.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = call <32 x i16> @llvm.smin.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.smin.i8(i8 undef, i8 undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call <16 x i8> @llvm.smin.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.smin.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = call <64 x i8> @llvm.smin.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %I64 = call i64 @llvm.smin.i64(i64 undef, i64 undef)
   %V2I64 = call <2 x i64> @llvm.smin.v2i64(<2 x i64> undef, <2 x i64> undef)
diff --git a/llvm/test/Analysis/CostModel/X86/arith-uminmax.ll b/llvm/test/Analysis/CostModel/X86/arith-uminmax.ll
index 74aaa3b03a49e..09f52c227a46c 100644
--- a/llvm/test/Analysis/CostModel/X86/arith-uminmax.ll
+++ b/llvm/test/Analysis/CostModel/X86/arith-uminmax.ll
@@ -30,99 +30,156 @@ declare <64 x i8>  @llvm.umax.v64i8(<64 x i8>, <64 x i8>)
 
 define i32 @umax(i32 %arg) {
 ; SSE2-LABEL: 'umax'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.umax.i64(i64 undef, i64 undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I64 = call <2 x i64> @llvm.umax.v2i64(<2 x i64> undef, <2 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I64 = call <4 x i64> @llvm.umax.v4i64(<4 x i64> undef, <4 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V8I64 = call <8 x i64> @llvm.umax.v8i64(<8 x i64> undef, <8 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.umax.i32(i32 undef, i32 undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4I32 = call <4 x i32> @llvm.umax.v4i32(<4 x i32> undef, <4 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V8I32 = call <8 x i32> @llvm.umax.v8i32(<8 x i32> undef, <8 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V16I32 = call <16 x i32> @llvm.umax.v16i32(<16 x i32> undef, <16 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.umax.i16(i16 undef, i16 undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I16 = call <8 x i16> @llvm.umax.v8i16(<8 x i16> undef, <8 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I16 = call <16 x i16> @llvm.umax.v16i16(<16 x i16> undef, <16 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V32I16 = call <32 x i16> @llvm.umax.v32i16(<32 x i16> undef, <32 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.umax.i8(i8 undef, i8 undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V16I8 = call <16 x i8> @llvm.umax.v16i8(<16 x i8> undef, <16 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 94 for instruction: %V32I8 = call <32 x i8> @llvm.umax.v32i8(<32 x i8> undef, <32 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 188 for instruction: %V64I8 = call <64 x i8> @llvm.umax.v64i8(<64 x i8> undef, <64 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.umax.i64(i64 undef, i64 undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V2I64 = call <2 x i64> @llvm.umax.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V4I64 = call <4 x i64> @llvm.umax.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V8I64 = call <8 x i64> @llvm.umax.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.umax.i32(i32 undef, i32 undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I32 = call <4 x i32> @llvm.umax.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I32 = call <8 x i32> @llvm.umax.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I32 = call <16 x i32> @llvm.umax.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.umax.i16(i16 undef, i16 undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I16 = call <8 x i16> @llvm.umax.v8i16(<8 x i16> undef, <8 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I16 = call <16 x i16> @llvm.umax.v16i16(<16 x i16> undef, <16 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I16 = call <32 x i16> @llvm.umax.v32i16(<32 x i16> undef, <32 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.umax.i8(i8 undef, i8 undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I8 = call <16 x i8> @llvm.umax.v16i8(<16 x i8> undef, <16 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I8 = call <32 x i8> @llvm.umax.v32i8(<32 x i8> undef, <32 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I8 = call <64 x i8> @llvm.umax.v64i8(<64 x i8> undef, <64 x i8> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'umax'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.umax.i64(i64 undef, i64 undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I64 = call <2 x i64> @llvm.umax.v2i64(<2 x i64> undef, <2 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I64 = call <4 x i64> @llvm.umax.v4i64(<4 x i64> undef, <4 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V8I64 = call <8 x i64> @llvm.umax.v8i64(<8 x i64> undef, <8 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.umax.i32(i32 undef, i32 undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4I32 = call <4 x i32> @llvm.umax.v4i32(<4 x i32> undef, <4 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V8I32 = call <8 x i32> @llvm.umax.v8i32(<8 x i32> undef, <8 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V16I32 = call <16 x i32> @llvm.umax.v16i32(<16 x i32> undef, <16 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.umax.i16(i16 undef, i16 undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I16 = call <8 x i16> @llvm.umax.v8i16(<8 x i16> undef, <8 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I16 = call <16 x i16> @llvm.umax.v16i16(<16 x i16> undef, <16 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V32I16 = call <32 x i16> @llvm.umax.v32i16(<32 x i16> undef, <32 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.umax.i8(i8 undef, i8 undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V16I8 = call <16 x i8> @llvm.umax.v16i8(<16 x i8> undef, <16 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 94 for instruction: %V32I8 = call <32 x i8> @llvm.umax.v32i8(<32 x i8> undef, <32 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 188 for instruction: %V64I8 = call <64 x i8> @llvm.umax.v64i8(<64 x i8> undef, <64 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.umax.i64(i64 undef, i64 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V2I64 = call <2 x i64> @llvm.umax.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V4I64 = call <4 x i64> @llvm.umax.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V8I64 = call <8 x i64> @llvm.umax.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.umax.i32(i32 undef, i32 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I32 = call <4 x i32> @llvm.umax.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I32 = call <8 x i32> @llvm.umax.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I32 = call <16 x i32> @llvm.umax.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.umax.i16(i16 undef, i16 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I16 = call <8 x i16> @llvm.umax.v8i16(<8 x i16> undef, <8 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I16 = call <16 x i16> @llvm.umax.v16i16(<16 x i16> undef, <16 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I16 = call <32 x i16> @llvm.umax.v32i16(<32 x i16> undef, <32 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.umax.i8(i8 undef, i8 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I8 = call <16 x i8> @llvm.umax.v16i8(<16 x i8> undef, <16 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I8 = call <32 x i8> @llvm.umax.v32i8(<32 x i8> undef, <32 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I8 = call <64 x i8> @llvm.umax.v64i8(<64 x i8> undef, <64 x i8> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'umax'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.umax.i64(i64 undef, i64 undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I64 = call <2 x i64> @llvm.umax.v2i64(<2 x i64> undef, <2 x i64> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4I64 = call <4 x i64> @llvm.umax.v4i64(<4 x i64> undef, <4 x i64> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I64 = call <8 x i64> @llvm.umax.v8i64(<8 x i64> undef, <8 x i64> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.umax.i32(i32 undef, i32 undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4I32 = call <4 x i32> @llvm.umax.v4i32(<4 x i32> undef, <4 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I32 = call <8 x i32> @llvm.umax.v8i32(<8 x i32> undef, <8 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I32 = call <16 x i32> @llvm.umax.v16i32(<16 x i32> undef, <16 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.umax.i16(i16 undef, i16 undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I16 = call <8 x i16> @llvm.umax.v8i16(<8 x i16> undef, <8 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I16 = call <16 x i16> @llvm.umax.v16i16(<16 x i16> undef, <16 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V32I16 = call <32 x i16> @llvm.umax.v32i16(<32 x i16> undef, <32 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.umax.i8(i8 undef, i8 undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I8 = call <16 x i8> @llvm.umax.v16i8(<16 x i8> undef, <16 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V32I8 = call <32 x i8> @llvm.umax.v32i8(<32 x i8> undef, <32 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %V64I8 = call <64 x i8> @llvm.umax.v64i8(<64 x i8> undef, <64 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.umax.i64(i64 undef, i64 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = call <2 x i64> @llvm.umax.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I64 = call <4 x i64> @llvm.umax.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I64 = call <8 x i64> @llvm.umax.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.umax.i32(i32 undef, i32 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.umax.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = call <8 x i32> @llvm.umax.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = call <16 x i32> @llvm.umax.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.umax.i16(i16 undef, i16 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call <8 x i16> @llvm.umax.v8i16(<8 x i16> undef, <8 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = call <16 x i16> @llvm.umax.v16i16(<16 x i16> undef, <16 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call <32 x i16> @llvm.umax.v32i16(<32 x i16> undef, <32 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.umax.i8(i8 undef, i8 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call <16 x i8> @llvm.umax.v16i8(<16 x i8> undef, <16 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = call <32 x i8> @llvm.umax.v32i8(<32 x i8> undef, <32 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = call <64 x i8> @llvm.umax.v64i8(<64 x i8> undef, <64 x i8> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
-; AVX-LABEL: 'umax'
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.umax.i64(i64 undef, i64 undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I64 = call <2 x i64> @llvm.umax.v2i64(<2 x i64> undef, <2 x i64> undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I64 = call <4 x i64> @llvm.umax.v4i64(<4 x i64> undef, <4 x i64> undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V8I64 = call <8 x i64> @llvm.umax.v8i64(<8 x i64> undef, <8 x i64> undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.umax.i32(i32 undef, i32 undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4I32 = call <4 x i32> @llvm.umax.v4i32(<4 x i32> undef, <4 x i32> undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I32 = call <8 x i32> @llvm.umax.v8i32(<8 x i32> undef, <8 x i32> undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V16I32 = call <16 x i32> @llvm.umax.v16i32(<16 x i32> undef, <16 x i32> undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.umax.i16(i16 undef, i16 undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I16 = call <8 x i16> @llvm.umax.v8i16(<8 x i16> undef, <8 x i16> undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V16I16 = call <16 x i16> @llvm.umax.v16i16(<16 x i16> undef, <16 x i16> undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %V32I16 = call <32 x i16> @llvm.umax.v32i16(<32 x i16> undef, <32 x i16> undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.umax.i8(i8 undef, i8 undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I8 = call <16 x i8> @llvm.umax.v16i8(<16 x i8> undef, <16 x i8> undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: %V32I8 = call <32 x i8> @llvm.umax.v32i8(<32 x i8> undef, <32 x i8> undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %V64I8 = call <64 x i8> @llvm.umax.v64i8(<64 x i8> undef, <64 x i8> undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+; AVX1-LABEL: 'umax'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.umax.i64(i64 undef, i64 undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = call <2 x i64> @llvm.umax.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4I64 = call <4 x i64> @llvm.umax.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8I64 = call <8 x i64> @llvm.umax.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.umax.i32(i32 undef, i32 undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.umax.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8I32 = call <8 x i32> @llvm.umax.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V16I32 = call <16 x i32> @llvm.umax.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.umax.i16(i16 undef, i16 undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call <8 x i16> @llvm.umax.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16I16 = call <16 x i16> @llvm.umax.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V32I16 = call <32 x i16> @llvm.umax.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.umax.i8(i8 undef, i8 undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call <16 x i8> @llvm.umax.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V32I8 = call <32 x i8> @llvm.umax.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V64I8 = call <64 x i8> @llvm.umax.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
-; AVX512-LABEL: 'umax'
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.umax.i64(i64 undef, i64 undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I64 = call <2 x i64> @llvm.umax.v2i64(<2 x i64> undef, <2 x i64> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I64 = call <4 x i64> @llvm.umax.v4i64(<4 x i64> undef, <4 x i64> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V8I64 = call <8 x i64> @llvm.umax.v8i64(<8 x i64> undef, <8 x i64> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.umax.i32(i32 undef, i32 undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4I32 = call <4 x i32> @llvm.umax.v4i32(<4 x i32> undef, <4 x i32> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I32 = call <8 x i32> @llvm.umax.v8i32(<8 x i32> undef, <8 x i32> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V16I32 = call <16 x i32> @llvm.umax.v16i32(<16 x i32> undef, <16 x i32> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.umax.i16(i16 undef, i16 undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I16 = call <8 x i16> @llvm.umax.v8i16(<8 x i16> undef, <8 x i16> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V16I16 = call <16 x i16> @llvm.umax.v16i16(<16 x i16> undef, <16 x i16> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %V32I16 = call <32 x i16> @llvm.umax.v32i16(<32 x i16> undef, <32 x i16> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.umax.i8(i8 undef, i8 undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I8 = call <16 x i8> @llvm.umax.v16i8(<16 x i8> undef, <16 x i8> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: %V32I8 = call <32 x i8> @llvm.umax.v32i8(<32 x i8> undef, <32 x i8> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 131 for instruction: %V64I8 = call <64 x i8> @llvm.umax.v64i8(<64 x i8> undef, <64 x i8> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+; AVX2-LABEL: 'umax'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.umax.i64(i64 undef, i64 undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = call <2 x i64> @llvm.umax.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = call <4 x i64> @llvm.umax.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I64 = call <8 x i64> @llvm.umax.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.umax.i32(i32 undef, i32 undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.umax.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call <8 x i32> @llvm.umax.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = call <16 x i32> @llvm.umax.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.umax.i16(i16 undef, i16 undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call <8 x i16> @llvm.umax.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.umax.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.umax.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.umax.i8(i8 undef, i8 undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call <16 x i8> @llvm.umax.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.umax.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.umax.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512F-LABEL: 'umax'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.umax.i64(i64 undef, i64 undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = call <2 x i64> @llvm.umax.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = call <4 x i64> @llvm.umax.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = call <8 x i64> @llvm.umax.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.umax.i32(i32 undef, i32 undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.umax.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call <8 x i32> @llvm.umax.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = call <16 x i32> @llvm.umax.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.umax.i16(i16 undef, i16 undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call <8 x i16> @llvm.umax.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.umax.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.umax.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.umax.i8(i8 undef, i8 undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call <16 x i8> @llvm.umax.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.umax.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.umax.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512DQ-LABEL: 'umax'
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.umax.i64(i64 undef, i64 undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = call <2 x i64> @llvm.umax.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = call <4 x i64> @llvm.umax.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = call <8 x i64> @llvm.umax.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.umax.i32(i32 undef, i32 undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.umax.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call <8 x i32> @llvm.umax.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = call <16 x i32> @llvm.umax.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.umax.i16(i16 undef, i16 undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call <8 x i16> @llvm.umax.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.umax.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.umax.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.umax.i8(i8 undef, i8 undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call <16 x i8> @llvm.umax.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.umax.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.umax.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512BW-LABEL: 'umax'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.umax.i64(i64 undef, i64 undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = call <2 x i64> @llvm.umax.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = call <4 x i64> @llvm.umax.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = call <8 x i64> @llvm.umax.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.umax.i32(i32 undef, i32 undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.umax.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call <8 x i32> @llvm.umax.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = call <16 x i32> @llvm.umax.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.umax.i16(i16 undef, i16 undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call <8 x i16> @llvm.umax.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.umax.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = call <32 x i16> @llvm.umax.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.umax.i8(i8 undef, i8 undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call <16 x i8> @llvm.umax.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.umax.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = call <64 x i8> @llvm.umax.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %I64 = call i64 @llvm.umax.i64(i64 undef, i64 undef)
   %V2I64 = call <2 x i64> @llvm.umax.v2i64(<2 x i64> undef, <2 x i64> undef)
@@ -169,99 +226,156 @@ declare <64 x i8>  @llvm.umin.v64i8(<64 x i8>, <64 x i8>)
 
 define i32 @umin(i32 %arg) {
 ; SSE2-LABEL: 'umin'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.umin.i64(i64 undef, i64 undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I64 = call <2 x i64> @llvm.umin.v2i64(<2 x i64> undef, <2 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I64 = call <4 x i64> @llvm.umin.v4i64(<4 x i64> undef, <4 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V8I64 = call <8 x i64> @llvm.umin.v8i64(<8 x i64> undef, <8 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.umin.i32(i32 undef, i32 undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4I32 = call <4 x i32> @llvm.umin.v4i32(<4 x i32> undef, <4 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V8I32 = call <8 x i32> @llvm.umin.v8i32(<8 x i32> undef, <8 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V16I32 = call <16 x i32> @llvm.umin.v16i32(<16 x i32> undef, <16 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.umin.i16(i16 undef, i16 undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I16 = call <8 x i16> @llvm.umin.v8i16(<8 x i16> undef, <8 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I16 = call <16 x i16> @llvm.umin.v16i16(<16 x i16> undef, <16 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V32I16 = call <32 x i16> @llvm.umin.v32i16(<32 x i16> undef, <32 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.umin.i8(i8 undef, i8 undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V16I8 = call <16 x i8> @llvm.umin.v16i8(<16 x i8> undef, <16 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 94 for instruction: %V32I8 = call <32 x i8> @llvm.umin.v32i8(<32 x i8> undef, <32 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 188 for instruction: %V64I8 = call <64 x i8> @llvm.umin.v64i8(<64 x i8> undef, <64 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.umin.i64(i64 undef, i64 undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V2I64 = call <2 x i64> @llvm.umin.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V4I64 = call <4 x i64> @llvm.umin.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V8I64 = call <8 x i64> @llvm.umin.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.umin.i32(i32 undef, i32 undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I32 = call <4 x i32> @llvm.umin.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I32 = call <8 x i32> @llvm.umin.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I32 = call <16 x i32> @llvm.umin.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.umin.i16(i16 undef, i16 undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I16 = call <8 x i16> @llvm.umin.v8i16(<8 x i16> undef, <8 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I16 = call <16 x i16> @llvm.umin.v16i16(<16 x i16> undef, <16 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I16 = call <32 x i16> @llvm.umin.v32i16(<32 x i16> undef, <32 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.umin.i8(i8 undef, i8 undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I8 = call <16 x i8> @llvm.umin.v16i8(<16 x i8> undef, <16 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I8 = call <32 x i8> @llvm.umin.v32i8(<32 x i8> undef, <32 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I8 = call <64 x i8> @llvm.umin.v64i8(<64 x i8> undef, <64 x i8> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'umin'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.umin.i64(i64 undef, i64 undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I64 = call <2 x i64> @llvm.umin.v2i64(<2 x i64> undef, <2 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I64 = call <4 x i64> @llvm.umin.v4i64(<4 x i64> undef, <4 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V8I64 = call <8 x i64> @llvm.umin.v8i64(<8 x i64> undef, <8 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.umin.i32(i32 undef, i32 undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4I32 = call <4 x i32> @llvm.umin.v4i32(<4 x i32> undef, <4 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V8I32 = call <8 x i32> @llvm.umin.v8i32(<8 x i32> undef, <8 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V16I32 = call <16 x i32> @llvm.umin.v16i32(<16 x i32> undef, <16 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.umin.i16(i16 undef, i16 undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I16 = call <8 x i16> @llvm.umin.v8i16(<8 x i16> undef, <8 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I16 = call <16 x i16> @llvm.umin.v16i16(<16 x i16> undef, <16 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V32I16 = call <32 x i16> @llvm.umin.v32i16(<32 x i16> undef, <32 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.umin.i8(i8 undef, i8 undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V16I8 = call <16 x i8> @llvm.umin.v16i8(<16 x i8> undef, <16 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 94 for instruction: %V32I8 = call <32 x i8> @llvm.umin.v32i8(<32 x i8> undef, <32 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 188 for instruction: %V64I8 = call <64 x i8> @llvm.umin.v64i8(<64 x i8> undef, <64 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.umin.i64(i64 undef, i64 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V2I64 = call <2 x i64> @llvm.umin.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V4I64 = call <4 x i64> @llvm.umin.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V8I64 = call <8 x i64> @llvm.umin.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.umin.i32(i32 undef, i32 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I32 = call <4 x i32> @llvm.umin.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I32 = call <8 x i32> @llvm.umin.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I32 = call <16 x i32> @llvm.umin.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.umin.i16(i16 undef, i16 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I16 = call <8 x i16> @llvm.umin.v8i16(<8 x i16> undef, <8 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I16 = call <16 x i16> @llvm.umin.v16i16(<16 x i16> undef, <16 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I16 = call <32 x i16> @llvm.umin.v32i16(<32 x i16> undef, <32 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.umin.i8(i8 undef, i8 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I8 = call <16 x i8> @llvm.umin.v16i8(<16 x i8> undef, <16 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I8 = call <32 x i8> @llvm.umin.v32i8(<32 x i8> undef, <32 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I8 = call <64 x i8> @llvm.umin.v64i8(<64 x i8> undef, <64 x i8> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'umin'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.umin.i64(i64 undef, i64 undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I64 = call <2 x i64> @llvm.umin.v2i64(<2 x i64> undef, <2 x i64> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4I64 = call <4 x i64> @llvm.umin.v4i64(<4 x i64> undef, <4 x i64> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I64 = call <8 x i64> @llvm.umin.v8i64(<8 x i64> undef, <8 x i64> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.umin.i32(i32 undef, i32 undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4I32 = call <4 x i32> @llvm.umin.v4i32(<4 x i32> undef, <4 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I32 = call <8 x i32> @llvm.umin.v8i32(<8 x i32> undef, <8 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I32 = call <16 x i32> @llvm.umin.v16i32(<16 x i32> undef, <16 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.umin.i16(i16 undef, i16 undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I16 = call <8 x i16> @llvm.umin.v8i16(<8 x i16> undef, <8 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I16 = call <16 x i16> @llvm.umin.v16i16(<16 x i16> undef, <16 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V32I16 = call <32 x i16> @llvm.umin.v32i16(<32 x i16> undef, <32 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.umin.i8(i8 undef, i8 undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I8 = call <16 x i8> @llvm.umin.v16i8(<16 x i8> undef, <16 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V32I8 = call <32 x i8> @llvm.umin.v32i8(<32 x i8> undef, <32 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %V64I8 = call <64 x i8> @llvm.umin.v64i8(<64 x i8> undef, <64 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.umin.i64(i64 undef, i64 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = call <2 x i64> @llvm.umin.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I64 = call <4 x i64> @llvm.umin.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I64 = call <8 x i64> @llvm.umin.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.umin.i32(i32 undef, i32 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.umin.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = call <8 x i32> @llvm.umin.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = call <16 x i32> @llvm.umin.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.umin.i16(i16 undef, i16 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call <8 x i16> @llvm.umin.v8i16(<8 x i16> undef, <8 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = call <16 x i16> @llvm.umin.v16i16(<16 x i16> undef, <16 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call <32 x i16> @llvm.umin.v32i16(<32 x i16> undef, <32 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.umin.i8(i8 undef, i8 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call <16 x i8> @llvm.umin.v16i8(<16 x i8> undef, <16 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = call <32 x i8> @llvm.umin.v32i8(<32 x i8> undef, <32 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = call <64 x i8> @llvm.umin.v64i8(<64 x i8> undef, <64 x i8> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
-; AVX-LABEL: 'umin'
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.umin.i64(i64 undef, i64 undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I64 = call <2 x i64> @llvm.umin.v2i64(<2 x i64> undef, <2 x i64> undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I64 = call <4 x i64> @llvm.umin.v4i64(<4 x i64> undef, <4 x i64> undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V8I64 = call <8 x i64> @llvm.umin.v8i64(<8 x i64> undef, <8 x i64> undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.umin.i32(i32 undef, i32 undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4I32 = call <4 x i32> @llvm.umin.v4i32(<4 x i32> undef, <4 x i32> undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I32 = call <8 x i32> @llvm.umin.v8i32(<8 x i32> undef, <8 x i32> undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V16I32 = call <16 x i32> @llvm.umin.v16i32(<16 x i32> undef, <16 x i32> undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.umin.i16(i16 undef, i16 undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I16 = call <8 x i16> @llvm.umin.v8i16(<8 x i16> undef, <8 x i16> undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V16I16 = call <16 x i16> @llvm.umin.v16i16(<16 x i16> undef, <16 x i16> undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %V32I16 = call <32 x i16> @llvm.umin.v32i16(<32 x i16> undef, <32 x i16> undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.umin.i8(i8 undef, i8 undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I8 = call <16 x i8> @llvm.umin.v16i8(<16 x i8> undef, <16 x i8> undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: %V32I8 = call <32 x i8> @llvm.umin.v32i8(<32 x i8> undef, <32 x i8> undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %V64I8 = call <64 x i8> @llvm.umin.v64i8(<64 x i8> undef, <64 x i8> undef)
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+; AVX1-LABEL: 'umin'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.umin.i64(i64 undef, i64 undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = call <2 x i64> @llvm.umin.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4I64 = call <4 x i64> @llvm.umin.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8I64 = call <8 x i64> @llvm.umin.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.umin.i32(i32 undef, i32 undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.umin.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8I32 = call <8 x i32> @llvm.umin.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V16I32 = call <16 x i32> @llvm.umin.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.umin.i16(i16 undef, i16 undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call <8 x i16> @llvm.umin.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16I16 = call <16 x i16> @llvm.umin.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V32I16 = call <32 x i16> @llvm.umin.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.umin.i8(i8 undef, i8 undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call <16 x i8> @llvm.umin.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V32I8 = call <32 x i8> @llvm.umin.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V64I8 = call <64 x i8> @llvm.umin.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX2-LABEL: 'umin'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.umin.i64(i64 undef, i64 undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = call <2 x i64> @llvm.umin.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = call <4 x i64> @llvm.umin.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I64 = call <8 x i64> @llvm.umin.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.umin.i32(i32 undef, i32 undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.umin.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call <8 x i32> @llvm.umin.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = call <16 x i32> @llvm.umin.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.umin.i16(i16 undef, i16 undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call <8 x i16> @llvm.umin.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.umin.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.umin.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.umin.i8(i8 undef, i8 undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call <16 x i8> @llvm.umin.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.umin.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.umin.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512F-LABEL: 'umin'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.umin.i64(i64 undef, i64 undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = call <2 x i64> @llvm.umin.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = call <4 x i64> @llvm.umin.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = call <8 x i64> @llvm.umin.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.umin.i32(i32 undef, i32 undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.umin.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call <8 x i32> @llvm.umin.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = call <16 x i32> @llvm.umin.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.umin.i16(i16 undef, i16 undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call <8 x i16> @llvm.umin.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.umin.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.umin.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.umin.i8(i8 undef, i8 undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call <16 x i8> @llvm.umin.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.umin.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.umin.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512DQ-LABEL: 'umin'
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.umin.i64(i64 undef, i64 undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = call <2 x i64> @llvm.umin.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = call <4 x i64> @llvm.umin.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = call <8 x i64> @llvm.umin.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.umin.i32(i32 undef, i32 undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.umin.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call <8 x i32> @llvm.umin.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = call <16 x i32> @llvm.umin.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.umin.i16(i16 undef, i16 undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call <8 x i16> @llvm.umin.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.umin.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.umin.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.umin.i8(i8 undef, i8 undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call <16 x i8> @llvm.umin.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.umin.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.umin.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
-; AVX512-LABEL: 'umin'
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.umin.i64(i64 undef, i64 undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I64 = call <2 x i64> @llvm.umin.v2i64(<2 x i64> undef, <2 x i64> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I64 = call <4 x i64> @llvm.umin.v4i64(<4 x i64> undef, <4 x i64> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V8I64 = call <8 x i64> @llvm.umin.v8i64(<8 x i64> undef, <8 x i64> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.umin.i32(i32 undef, i32 undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4I32 = call <4 x i32> @llvm.umin.v4i32(<4 x i32> undef, <4 x i32> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I32 = call <8 x i32> @llvm.umin.v8i32(<8 x i32> undef, <8 x i32> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V16I32 = call <16 x i32> @llvm.umin.v16i32(<16 x i32> undef, <16 x i32> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.umin.i16(i16 undef, i16 undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I16 = call <8 x i16> @llvm.umin.v8i16(<8 x i16> undef, <8 x i16> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V16I16 = call <16 x i16> @llvm.umin.v16i16(<16 x i16> undef, <16 x i16> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %V32I16 = call <32 x i16> @llvm.umin.v32i16(<32 x i16> undef, <32 x i16> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.umin.i8(i8 undef, i8 undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I8 = call <16 x i8> @llvm.umin.v16i8(<16 x i8> undef, <16 x i8> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: %V32I8 = call <32 x i8> @llvm.umin.v32i8(<32 x i8> undef, <32 x i8> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 131 for instruction: %V64I8 = call <64 x i8> @llvm.umin.v64i8(<64 x i8> undef, <64 x i8> undef)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+; AVX512BW-LABEL: 'umin'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.umin.i64(i64 undef, i64 undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = call <2 x i64> @llvm.umin.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = call <4 x i64> @llvm.umin.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = call <8 x i64> @llvm.umin.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.umin.i32(i32 undef, i32 undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.umin.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call <8 x i32> @llvm.umin.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = call <16 x i32> @llvm.umin.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.umin.i16(i16 undef, i16 undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call <8 x i16> @llvm.umin.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.umin.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = call <32 x i16> @llvm.umin.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.umin.i8(i8 undef, i8 undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call <16 x i8> @llvm.umin.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.umin.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = call <64 x i8> @llvm.umin.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %I64 = call i64 @llvm.umin.i64(i64 undef, i64 undef)
   %V2I64 = call <2 x i64> @llvm.umin.v2i64(<2 x i64> undef, <2 x i64> undef)

From c970bb5b34c1999c9b55af88c35620c1930a581a Mon Sep 17 00:00:00 2001
From: Pushpinder Singh <Pushpinder.Singh@amd.com>
Date: Fri, 3 Jul 2020 01:32:45 -0400
Subject: [PATCH 0436/1035] [CMAKE] Fix 'clean' target not working

cmake was still considering the empty value of ${fake_version_inc}
even if it was not defined.

Reviewed By: vsapsai

Differential Revision: https://reviews.llvm.org/D82847
---
 llvm/include/llvm/Support/CMakeLists.txt | 28 ++++++++++++++++++------
 1 file changed, 21 insertions(+), 7 deletions(-)

diff --git a/llvm/include/llvm/Support/CMakeLists.txt b/llvm/include/llvm/Support/CMakeLists.txt
index da8a4da443edf..aa71b55721810 100644
--- a/llvm/include/llvm/Support/CMakeLists.txt
+++ b/llvm/include/llvm/Support/CMakeLists.txt
@@ -16,18 +16,32 @@ if(LLVM_APPEND_VC_REV)
   endif()
 endif()
 
+set(generated_files "${version_inc}")
+if (fake_version_inc)
+  list(APPEND generated_files "${fake_version_inc}")
+endif()
+
 # Create custom target to generate the VC revision include.
-add_custom_command(OUTPUT "${version_inc}" "${fake_version_inc}"
-  DEPENDS "${llvm_vc}" "${generate_vcs_version_script}"
-  COMMAND ${CMAKE_COMMAND} "-DNAMES=LLVM"
-                           "-DLLVM_SOURCE_DIR=${llvm_source_dir}"
-                           "-DHEADER_FILE=${version_inc}"
-                           -P "${generate_vcs_version_script}")
+if (fake_version_inc)
+  add_custom_command(OUTPUT "${version_inc}" "${fake_version_inc}"
+    DEPENDS "${llvm_vc}" "${generate_vcs_version_script}"
+    COMMAND ${CMAKE_COMMAND} "-DNAMES=LLVM"
+                             "-DLLVM_SOURCE_DIR=${llvm_source_dir}"
+                             "-DHEADER_FILE=${version_inc}"
+                             -P "${generate_vcs_version_script}")
+else()
+  add_custom_command(OUTPUT "${version_inc}"
+    DEPENDS "${llvm_vc}" "${generate_vcs_version_script}"
+    COMMAND ${CMAKE_COMMAND} "-DNAMES=LLVM"
+                             "-DLLVM_SOURCE_DIR=${llvm_source_dir}"
+                             "-DHEADER_FILE=${version_inc}"
+                             -P "${generate_vcs_version_script}")
+endif()
 
 # Mark the generated header as being generated.
 set_source_files_properties("${version_inc}"
   PROPERTIES GENERATED TRUE
              HEADER_FILE_ONLY TRUE)
 
-add_custom_target(llvm_vcsrevision_h ALL DEPENDS "${version_inc}" "${fake_version_inc}")
+add_custom_target(llvm_vcsrevision_h ALL DEPENDS "${generated_files}")
 set_target_properties(llvm_vcsrevision_h PROPERTIES FOLDER "Misc")

From 5d6cd61904aa03df6db4a5a33445d9af60cab0d5 Mon Sep 17 00:00:00 2001
From: Yevgeny Rouban <yrouban@azul.com>
Date: Wed, 29 Jul 2020 17:47:36 +0700
Subject: [PATCH 0437/1035] [LoopSimplifyCFG] Delete landing pads in dead exit
 blocks

In addition to removing phi nodes this patch removes any
landing pad that the dead exit block might have. Without
this fix Verifier complains about a new switch instruction
jumps to a block with a landing pad.

Differential Revision: https://reviews.llvm.org/D84320
---
 .../lib/Transforms/Scalar/LoopSimplifyCFG.cpp | 17 ++--
 .../LoopSimplifyCFG/handle_dead_exits.ll      | 88 +++++++++++++++++++
 2 files changed, 99 insertions(+), 6 deletions(-)
 create mode 100644 llvm/test/Transforms/LoopSimplifyCFG/handle_dead_exits.ll

diff --git a/llvm/lib/Transforms/Scalar/LoopSimplifyCFG.cpp b/llvm/lib/Transforms/Scalar/LoopSimplifyCFG.cpp
index 031e5b9c1d2c9..8ab3c2ca118d4 100644
--- a/llvm/lib/Transforms/Scalar/LoopSimplifyCFG.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopSimplifyCFG.cpp
@@ -366,15 +366,20 @@ class ConstantTerminatorFoldingImpl {
 
     unsigned DummyIdx = 1;
     for (BasicBlock *BB : DeadExitBlocks) {
-      SmallVector<Instruction *, 4> DeadPhis;
+      // Eliminate all Phis and LandingPads from dead exits.
+      // TODO: Consider removing all instructions in this dead block.
+      SmallVector<Instruction *, 4> DeadInstructions;
       for (auto &PN : BB->phis())
-        DeadPhis.push_back(&PN);
+        DeadInstructions.push_back(&PN);
 
-      // Eliminate all Phis from dead exits.
-      for (Instruction *PN : DeadPhis) {
-        PN->replaceAllUsesWith(UndefValue::get(PN->getType()));
-        PN->eraseFromParent();
+      if (auto *LandingPad = dyn_cast<LandingPadInst>(BB->getFirstNonPHI()))
+        DeadInstructions.emplace_back(LandingPad);
+
+      for (Instruction *I : DeadInstructions) {
+        I->replaceAllUsesWith(UndefValue::get(I->getType()));
+        I->eraseFromParent();
       }
+
       assert(DummyIdx != 0 && "Too many dead exits!");
       DummySwitch->addCase(Builder.getInt32(DummyIdx++), BB);
       DTUpdates.push_back({DominatorTree::Insert, Preheader, BB});
diff --git a/llvm/test/Transforms/LoopSimplifyCFG/handle_dead_exits.ll b/llvm/test/Transforms/LoopSimplifyCFG/handle_dead_exits.ll
new file mode 100644
index 0000000000000..1fb7fb4b13360
--- /dev/null
+++ b/llvm/test/Transforms/LoopSimplifyCFG/handle_dead_exits.ll
@@ -0,0 +1,88 @@
+; RUN: opt -S -enable-loop-simplifycfg-term-folding=true -loop-simplifycfg %s | FileCheck %s
+; RUN: opt -S -enable-loop-simplifycfg-term-folding=true -passes='require<domtree>,loop(simplify-cfg)' %s | FileCheck %s
+
+declare i32* @fake_personality_function()
+declare void @foo()
+
+define i32 @test_remove_lpad(i1 %exitcond) personality i32* ()* @fake_personality_function {
+; CHECK-LABEL: @test_remove_lpad(
+entry:
+  br label %for.body
+
+for.body:
+  br i1 0, label %never, label %next
+
+next:
+  br label %latch
+
+latch:
+  br i1 %exitcond, label %exit, label %for.body
+
+exit:
+  ret i32 0
+
+never:
+  invoke void @foo() to label %next unwind label %never-unwind
+
+never-unwind:
+; CHECK: never-unwind:
+; CHECK-NEXT: unreachable
+  %res = landingpad token cleanup
+  unreachable
+}
+
+define i32 @test_remove_phi_lpad(i1 %exitcond) personality i32* ()* @fake_personality_function {
+; CHECK-LABEL: @test_remove_phi_lpad(
+entry:
+  br label %for.body
+
+for.body:
+  br i1 0, label %never, label %next
+
+next:
+  br label %latch
+
+latch:
+  br i1 %exitcond, label %exit, label %for.body
+
+exit:
+  ret i32 0
+
+never:
+  invoke void @foo() to label %next unwind label %never-unwind
+
+never-unwind:
+; CHECK: never-unwind:
+; CHECK-NEXT: ret i32 undef
+  %p = phi i32 [1, %never]
+  %res = landingpad token cleanup
+  ret i32 %p
+}
+
+define i32 @test_split_remove_phi_lpad_(i1 %exitcond) personality i32* ()* @fake_personality_function {
+; CHECK-LABEL: @test_split_remove_phi_lpad_(
+entry:
+  invoke void @foo() to label %for.body unwind label %unwind-bb
+
+for.body:
+  br i1 0, label %never, label %next
+
+next:
+  br label %latch
+
+latch:
+  br i1 %exitcond, label %exit, label %for.body
+
+exit:
+  ret i32 0
+
+never:
+  invoke void @foo() to label %next unwind label %unwind-bb
+
+unwind-bb:
+; CHECK: unwind-bb.loopexit:
+; CHECK-NEXT: br label %unwind-bb
+  %p = phi i32 [1, %never], [2, %entry]
+  %res = landingpad token cleanup
+  ret i32 %p
+}

From 2aa2c40d23740331e01c4eeec849775ce9c33f95 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Wed, 29 Jul 2020 12:39:59 +0100
Subject: [PATCH 0438/1035] [NewGVN] Require asserts for crashing tests.

Without asserts, it might take a long time for the tests to crash.
Only run them with assert builds.
---
 llvm/test/Transforms/NewGVN/todo-pr35074-phi-of-ops.ll          | 2 ++
 .../Transforms/NewGVN/todo-pr37121-seens-this-value-a-lot.ll    | 2 ++
 llvm/test/Transforms/NewGVN/todo-pr42422-phi-of-ops.ll          | 2 ++
 3 files changed, 6 insertions(+)

diff --git a/llvm/test/Transforms/NewGVN/todo-pr35074-phi-of-ops.ll b/llvm/test/Transforms/NewGVN/todo-pr35074-phi-of-ops.ll
index e80292e94fd1c..0aa2407f9a790 100644
--- a/llvm/test/Transforms/NewGVN/todo-pr35074-phi-of-ops.ll
+++ b/llvm/test/Transforms/NewGVN/todo-pr35074-phi-of-ops.ll
@@ -1,3 +1,5 @@
+; REQUIRES: asserts
+
 ; RUN: opt -newgvn -S %s | FileCheck %s
 
 XFAIL: *
diff --git a/llvm/test/Transforms/NewGVN/todo-pr37121-seens-this-value-a-lot.ll b/llvm/test/Transforms/NewGVN/todo-pr37121-seens-this-value-a-lot.ll
index b4961fa32ef00..9da4245fe24d3 100644
--- a/llvm/test/Transforms/NewGVN/todo-pr37121-seens-this-value-a-lot.ll
+++ b/llvm/test/Transforms/NewGVN/todo-pr37121-seens-this-value-a-lot.ll
@@ -1,3 +1,5 @@
+; REQUIRES: asserts
+
 ; RUN: opt -newgvn -S %s | FileCheck %s
 
 ; XFAIL: *
diff --git a/llvm/test/Transforms/NewGVN/todo-pr42422-phi-of-ops.ll b/llvm/test/Transforms/NewGVN/todo-pr42422-phi-of-ops.ll
index 6c4cf5f82314b..ffe5d8eedf4d1 100644
--- a/llvm/test/Transforms/NewGVN/todo-pr42422-phi-of-ops.ll
+++ b/llvm/test/Transforms/NewGVN/todo-pr42422-phi-of-ops.ll
@@ -1,3 +1,5 @@
+; REQUIRES: asserts
+
 ; RUN: opt -newgvn -S %s | FileChecks %s
 
 ; XFAIL: *

From aec38c619dfa1c41b6b0f6c52a31d221ac108b6b Mon Sep 17 00:00:00 2001
From: Alex Zinenko <zinenko@google.com>
Date: Thu, 23 Jul 2020 10:32:12 +0200
Subject: [PATCH 0439/1035] [mlir] LLVMType: make getUnderlyingType private

The current modeling of LLVM IR types in MLIR is based on the LLVMType class
that wraps a raw `llvm::Type *` and delegates uniquing, printing and parsing to
LLVM itself. This is model makes thread-safe type manipulation hard and is
being progressively replaced with a cleaner MLIR model that replicates the type
system. In the new model, LLVMType will no longer have an underlying LLVM IR
type. Restrict access to this type in the current model in preparation for the
change.

Reviewed By: nicolasvasilache

Differential Revision: https://reviews.llvm.org/D84389
---
 .../include/mlir/Dialect/LLVMIR/LLVMDialect.h | 34 ++++++--
 .../include/mlir/Dialect/LLVMIR/LLVMOpBase.td |  6 +-
 mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td   | 11 ++-
 mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td  | 11 ++-
 .../mlir/Target/LLVMIR/ModuleTranslation.h    |  4 +
 .../SPIRVToLLVM/ConvertSPIRVToLLVM.cpp        |  6 +-
 .../StandardToLLVM/StandardToLLVM.cpp         | 10 +--
 .../VectorToLLVM/ConvertVectorToLLVM.cpp      |  2 +-
 mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp    | 79 ++++++++++---------
 mlir/lib/Target/LLVMIR/ConvertFromLLVMIR.cpp  | 13 ++-
 mlir/lib/Target/LLVMIR/ModuleTranslation.cpp  | 16 ++--
 .../tools/mlir-tblgen/LLVMIRConversionGen.cpp |  3 +-
 12 files changed, 111 insertions(+), 84 deletions(-)

diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.h b/mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.h
index 078cb1cfa4e57..52acfbfa8e507 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.h
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.h
@@ -47,6 +47,14 @@ struct LLVMTypeStorage;
 struct LLVMDialectImpl;
 } // namespace detail
 
+class LLVMType;
+
+/// Converts an MLIR LLVM dialect type to LLVM IR type. Note that this function
+/// exists exclusively for the purpose of gradual transition to the first-party
+/// modeling of LLVM types. It should not be used outside MLIR-to-LLVM
+/// translation.
+llvm::Type *convertLLVMType(LLVMType type);
+
 class LLVMType : public mlir::Type::TypeBase<LLVMType, mlir::Type,
                                              detail::LLVMTypeStorage> {
 public:
@@ -59,26 +67,32 @@ class LLVMType : public mlir::Type::TypeBase<LLVMType, mlir::Type,
   static bool kindof(unsigned kind) { return kind == LLVM_TYPE; }
 
   LLVMDialect &getDialect();
-  llvm::Type *getUnderlyingType() const;
 
   /// Utilities to identify types.
   bool isBFloatTy() { return getUnderlyingType()->isBFloatTy(); }
   bool isHalfTy() { return getUnderlyingType()->isHalfTy(); }
   bool isFloatTy() { return getUnderlyingType()->isFloatTy(); }
   bool isDoubleTy() { return getUnderlyingType()->isDoubleTy(); }
-  bool isIntegerTy() { return getUnderlyingType()->isIntegerTy(); }
-  bool isIntegerTy(unsigned bitwidth) {
-    return getUnderlyingType()->isIntegerTy(bitwidth);
-  }
+  bool isFloatingPointTy() { return getUnderlyingType()->isFloatingPointTy(); }
 
   /// Array type utilities.
   LLVMType getArrayElementType();
   unsigned getArrayNumElements();
   bool isArrayTy();
 
+  /// Integer type utilities.
+  unsigned getIntegerBitWidth() {
+    return getUnderlyingType()->getIntegerBitWidth();
+  }
+  bool isIntegerTy() { return getUnderlyingType()->isIntegerTy(); }
+  bool isIntegerTy(unsigned bitwidth) {
+    return getUnderlyingType()->isIntegerTy(bitwidth);
+  }
+
   /// Vector type utilities.
   LLVMType getVectorElementType();
   unsigned getVectorNumElements();
+  llvm::ElementCount getVectorElementCount();
   bool isVectorTy();
 
   /// Function type utilities.
@@ -86,11 +100,13 @@ class LLVMType : public mlir::Type::TypeBase<LLVMType, mlir::Type,
   unsigned getFunctionNumParams();
   LLVMType getFunctionResultType();
   bool isFunctionTy();
+  bool isFunctionVarArg();
 
   /// Pointer type utilities.
   LLVMType getPointerTo(unsigned addrSpace = 0);
   LLVMType getPointerElementTy();
   bool isPointerTy();
+  static bool isValidPointerElementType(LLVMType type);
 
   /// Struct type utilities.
   LLVMType getStructElementType(unsigned i);
@@ -194,6 +210,14 @@ class LLVMType : public mlir::Type::TypeBase<LLVMType, mlir::Type,
 
 private:
   friend LLVMDialect;
+  friend llvm::Type *convertLLVMType(LLVMType type);
+
+  /// Get the underlying LLVM IR type.
+  llvm::Type *getUnderlyingType() const;
+
+  /// Get the underlying LLVM IR types for the given array of types.
+  static void getUnderlyingTypes(ArrayRef<LLVMType> types,
+                                 SmallVectorImpl<llvm::Type *> &result);
 
   /// Get an LLVMType with a pre-existing llvm type.
   static LLVMType get(MLIRContext *context, llvm::Type *llvmType);
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td
index d88b372dbf43d..4d99bf265c65f 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td
@@ -134,11 +134,9 @@ class ListIntSubst<string pattern, list<int> values> {
 // or result in the operation.
 def LLVM_IntrPatterns {
   string operand =
-    [{opInst.getOperand($0).getType()
-      .cast<LLVM::LLVMType>().getUnderlyingType()}];
+    [{convertType(opInst.getOperand($0).getType().cast<LLVM::LLVMType>())}];
   string result =
-    [{opInst.getResult($0).getType()
-      .cast<LLVM::LLVMType>().getUnderlyingType()}];
+    [{convertType(opInst.getResult($0).getType().cast<LLVM::LLVMType>())}];
 }
 
 
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
index 29d7fd930030a..4da90575524b7 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
@@ -61,9 +61,8 @@ def LLVM_VoidResultTypeOpBuilder : OpBuilder<
   [{
     auto llvmType = resultType.dyn_cast<LLVM::LLVMType>(); (void)llvmType;
     assert(llvmType && "result must be an LLVM type");
-    assert(llvmType.getUnderlyingType() &&
-            llvmType.getUnderlyingType()->isVoidTy() &&
-            "for zero-result operands, only 'void' is accepted as result type");
+    assert(llvmType.isVoidTy() &&
+           "for zero-result operands, only 'void' is accepted as result type");
     build(builder, result, operands, attributes);
   }]>;
 
@@ -477,7 +476,7 @@ def LLVM_ShuffleVectorOp
   let verifier = [{
     auto wrappedVectorType1 = v1().getType().cast<LLVM::LLVMType>();
     auto wrappedVectorType2 = v2().getType().cast<LLVM::LLVMType>();
-    if (!wrappedVectorType2.getUnderlyingType()->isVectorTy())
+    if (!wrappedVectorType2.isVectorTy())
       return emitOpError("expected LLVM IR Dialect vector type for operand #2");
     if (wrappedVectorType1.getVectorElementType() !=
         wrappedVectorType2.getVectorElementType())
@@ -765,10 +764,10 @@ def LLVM_LLVMFuncOp
           .getValue().cast<LLVMType>();
     }
     bool isVarArg() {
-      return getType().getUnderlyingType()->isFunctionVarArg();
+      return getType().isFunctionVarArg();
     }
 
-    // Hook for OpTrait::FunctionLike, returns the number of function arguments.
+    // Hook for OpTrait::FunctionLike, returns the number of function arguments`.
     // Depends on the type attribute being correct as checked by verifyType.
     unsigned getNumFuncArguments();
 
diff --git a/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td b/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td
index 786b9ef217bd7..0cd11690daa8b 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td
@@ -139,7 +139,7 @@ def ROCDL_mfma_i32_16x16x16i8 : ROCDL_Mfma_IntrOp<"mfma.i32.16x16x16i8">;
 // Vector buffer load/store intrinsics
 
 def ROCDL_MubufLoadOp :
-  ROCDL_Op<"buffer.load">, 
+  ROCDL_Op<"buffer.load">,
   Results<(outs LLVM_Type:$res)>,
   Arguments<(ins LLVM_Type:$rsrc,
                  LLVM_Type:$vindex,
@@ -160,7 +160,7 @@ def ROCDL_MubufLoadOp :
 }
 
 def ROCDL_MubufStoreOp :
-  ROCDL_Op<"buffer.store">, 
+  ROCDL_Op<"buffer.store">,
   Arguments<(ins LLVM_Type:$vdata,
                  LLVM_Type:$rsrc,
                  LLVM_Type:$vindex,
@@ -168,14 +168,13 @@ def ROCDL_MubufStoreOp :
                  LLVM_Type:$glc,
                  LLVM_Type:$slc)>{
   string llvmBuilder = [{
-    auto vdataType = op.vdata().getType().cast<LLVM::LLVMType>()
-                       .getUnderlyingType();
+    auto vdataType = convertType(op.vdata().getType().cast<LLVM::LLVMType>());
     createIntrinsicCall(builder,
-          llvm::Intrinsic::amdgcn_buffer_store, {$vdata, $rsrc, $vindex, 
+          llvm::Intrinsic::amdgcn_buffer_store, {$vdata, $rsrc, $vindex,
           $offset, $glc, $slc}, {vdataType});
   }];
   let parser = [{ return parseROCDLMubufStoreOp(parser, result); }];
-  let printer = [{ 
+  let printer = [{
     Operation *op = this->getOperation();
     p << op->getName() << " " << op->getOperands()
       << " : " << vdata().getType();
diff --git a/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h b/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h
index e44ae976e0dd0..61f8f9fce64c0 100644
--- a/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h
+++ b/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h
@@ -89,6 +89,10 @@ class ModuleTranslation {
                                             llvm::IRBuilder<> &builder);
   virtual LogicalResult convertOmpParallel(Operation &op,
                                            llvm::IRBuilder<> &builder);
+
+  /// Converts the type from MLIR LLVM dialect to LLVM.
+  llvm::Type *convertType(LLVMType type);
+
   static std::unique_ptr<llvm::Module> prepareLLVMModule(Operation *m);
 
   /// A helper to look up remapped operands in the value remapping table.
diff --git a/mlir/lib/Conversion/SPIRVToLLVM/ConvertSPIRVToLLVM.cpp b/mlir/lib/Conversion/SPIRVToLLVM/ConvertSPIRVToLLVM.cpp
index 25a3ac07d5f4a..fd0e96b79d2b5 100644
--- a/mlir/lib/Conversion/SPIRVToLLVM/ConvertSPIRVToLLVM.cpp
+++ b/mlir/lib/Conversion/SPIRVToLLVM/ConvertSPIRVToLLVM.cpp
@@ -64,10 +64,8 @@ static unsigned getBitWidth(Type type) {
 
 /// Returns the bit width of LLVMType integer or vector.
 static unsigned getLLVMTypeBitWidth(LLVM::LLVMType type) {
-  return type.isVectorTy() ? type.getVectorElementType()
-                                 .getUnderlyingType()
-                                 ->getIntegerBitWidth()
-                           : type.getUnderlyingType()->getIntegerBitWidth();
+  return type.isVectorTy() ? type.getVectorElementType().getIntegerBitWidth()
+                           : type.getIntegerBitWidth();
 }
 
 /// Creates `IntegerAttribute` with all bits set for given type
diff --git a/mlir/lib/Conversion/StandardToLLVM/StandardToLLVM.cpp b/mlir/lib/Conversion/StandardToLLVM/StandardToLLVM.cpp
index 1e6fa6a8754ba..0d154796f049c 100644
--- a/mlir/lib/Conversion/StandardToLLVM/StandardToLLVM.cpp
+++ b/mlir/lib/Conversion/StandardToLLVM/StandardToLLVM.cpp
@@ -2248,10 +2248,8 @@ struct RsqrtOpLowering : public ConvertOpToLLVMPattern<RsqrtOp> {
         op, operands, typeConverter,
         [&](LLVM::LLVMType llvmVectorTy, ValueRange operands) {
           auto splatAttr = SplatElementsAttr::get(
-              mlir::VectorType::get(
-                  {cast<llvm::FixedVectorType>(llvmVectorTy.getUnderlyingType())
-                       ->getNumElements()},
-                  floatType),
+              mlir::VectorType::get({llvmVectorTy.getVectorNumElements()},
+                                    floatType),
               floatOne);
           auto one =
               rewriter.create<LLVM::ConstantOp>(loc, llvmVectorTy, splatAttr);
@@ -2511,8 +2509,8 @@ struct IndexCastOpLowering : public ConvertOpToLLVMPattern<IndexCastOp> {
         this->typeConverter.convertType(indexCastOp.getResult().getType())
             .cast<LLVM::LLVMType>();
     auto sourceType = transformed.in().getType().cast<LLVM::LLVMType>();
-    unsigned targetBits = targetType.getUnderlyingType()->getIntegerBitWidth();
-    unsigned sourceBits = sourceType.getUnderlyingType()->getIntegerBitWidth();
+    unsigned targetBits = targetType.getIntegerBitWidth();
+    unsigned sourceBits = sourceType.getIntegerBitWidth();
 
     if (targetBits == sourceBits)
       rewriter.replaceOp(op, transformed.in());
diff --git a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
index 5dbc8394b03a5..4fa7b573f84ef 100644
--- a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
+++ b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
@@ -127,7 +127,7 @@ LogicalResult getMemRefAlignment(LLVMTypeConverter &typeConverter, T op,
 
   auto dataLayout = typeConverter.getDialect()->getLLVMModule().getDataLayout();
   align = dataLayout.getPrefTypeAlignment(
-      elementTy.cast<LLVM::LLVMType>().getUnderlyingType());
+      LLVM::convertLLVMType(elementTy.cast<LLVM::LLVMType>()));
   return success();
 }
 
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
index cf7a5d926528f..17848c6bf3ee9 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
@@ -105,11 +105,9 @@ static ParseResult parseCmpOp(OpAsmParser &parser, OperationState &result) {
   auto argType = type.dyn_cast<LLVM::LLVMType>();
   if (!argType)
     return parser.emitError(trailingTypeLoc, "expected LLVM IR dialect type");
-  if (argType.getUnderlyingType()->isVectorTy())
-    resultType = LLVMType::getVectorTy(
-        resultType,
-        llvm::cast<llvm::FixedVectorType>(argType.getUnderlyingType())
-            ->getNumElements());
+  if (argType.isVectorTy())
+    resultType =
+        LLVMType::getVectorTy(resultType, argType.getVectorNumElements());
 
   result.addTypes({resultType});
   return success();
@@ -214,7 +212,7 @@ static Type getLoadStoreElementType(OpAsmParser &parser, Type type,
   if (!llvmTy)
     return parser.emitError(trailingTypeLoc, "expected LLVM IR dialect type"),
            nullptr;
-  if (!llvmTy.getUnderlyingType()->isPointerTy())
+  if (!llvmTy.isPointerTy())
     return parser.emitError(trailingTypeLoc, "expected LLVM pointer type"),
            nullptr;
   return llvmTy.getPointerElementTy();
@@ -683,8 +681,7 @@ static ParseResult parseExtractElementOp(OpAsmParser &parser,
       parser.resolveOperand(position, positionType, result.operands))
     return failure();
   auto wrappedVectorType = type.dyn_cast<LLVM::LLVMType>();
-  if (!wrappedVectorType ||
-      !wrappedVectorType.getUnderlyingType()->isVectorTy())
+  if (!wrappedVectorType || !wrappedVectorType.isVectorTy())
     return parser.emitError(
         loc, "expected LLVM IR dialect vector type for operand #1");
   result.addTypes(wrappedVectorType.getVectorElementType());
@@ -725,16 +722,15 @@ static LLVM::LLVMType getInsertExtractValueElementType(OpAsmParser &parser,
                               "expected an array of integer literals"),
              nullptr;
     int position = positionElementAttr.getInt();
-    auto *llvmContainerType = wrappedContainerType.getUnderlyingType();
-    if (llvmContainerType->isArrayTy()) {
+    if (wrappedContainerType.isArrayTy()) {
       if (position < 0 || static_cast<unsigned>(position) >=
-                              llvmContainerType->getArrayNumElements())
+                              wrappedContainerType.getArrayNumElements())
         return parser.emitError(attributeLoc, "position out of bounds"),
                nullptr;
       wrappedContainerType = wrappedContainerType.getArrayElementType();
-    } else if (llvmContainerType->isStructTy()) {
+    } else if (wrappedContainerType.isStructTy()) {
       if (position < 0 || static_cast<unsigned>(position) >=
-                              llvmContainerType->getStructNumElements())
+                              wrappedContainerType.getStructNumElements())
         return parser.emitError(attributeLoc, "position out of bounds"),
                nullptr;
       wrappedContainerType =
@@ -803,8 +799,7 @@ static ParseResult parseInsertElementOp(OpAsmParser &parser,
     return failure();
 
   auto wrappedVectorType = vectorType.dyn_cast<LLVM::LLVMType>();
-  if (!wrappedVectorType ||
-      !wrappedVectorType.getUnderlyingType()->isVectorTy())
+  if (!wrappedVectorType || !wrappedVectorType.isVectorTy())
     return parser.emitError(
         loc, "expected LLVM IR dialect vector type for operand #1");
   auto valueType = wrappedVectorType.getVectorElementType();
@@ -1125,7 +1120,7 @@ static ParseResult parseGlobalOp(OpAsmParser &parser, OperationState &result) {
 }
 
 static LogicalResult verify(GlobalOp op) {
-  if (!llvm::PointerType::isValidElementType(op.getType().getUnderlyingType()))
+  if (!LLVMType::isValidPointerElementType(op.getType()))
     return op.emitOpError(
         "expects type to be a valid element type for an LLVM pointer");
   if (op.getParentOp() && !satisfiesLLVMModule(op.getParentOp()))
@@ -1133,8 +1128,7 @@ static LogicalResult verify(GlobalOp op) {
 
   if (auto strAttr = op.getValueOrNull().dyn_cast_or_null<StringAttr>()) {
     auto type = op.getType();
-    if (!type.getUnderlyingType()->isArrayTy() ||
-        !type.getArrayElementType().getUnderlyingType()->isIntegerTy(8) ||
+    if (!type.isArrayTy() || !type.getArrayElementType().isIntegerTy(8) ||
         type.getArrayNumElements() != strAttr.getValue().size())
       return op.emitOpError(
           "requires an i8 array type of the length equal to that of the string "
@@ -1197,8 +1191,7 @@ static ParseResult parseShuffleVectorOp(OpAsmParser &parser,
       parser.resolveOperand(v2, typeV2, result.operands))
     return failure();
   auto wrappedContainerType1 = typeV1.dyn_cast<LLVM::LLVMType>();
-  if (!wrappedContainerType1 ||
-      !wrappedContainerType1.getUnderlyingType()->isVectorTy())
+  if (!wrappedContainerType1 || !wrappedContainerType1.isVectorTy())
     return parser.emitError(
         loc, "expected LLVM IR dialect vector type for operand #1");
   auto vType = LLVMType::getVectorTy(
@@ -1239,7 +1232,7 @@ void LLVMFuncOp::build(OpBuilder &builder, OperationState &result,
   if (argAttrs.empty())
     return;
 
-  unsigned numInputs = type.getUnderlyingType()->getFunctionNumParams();
+  unsigned numInputs = type.getFunctionNumParams();
   assert(numInputs == argAttrs.size() &&
          "expected as many argument attribute lists as arguments");
   SmallString<8> argAttrName;
@@ -1374,7 +1367,7 @@ static void printLLVMFuncOp(OpAsmPrinter &p, LLVMFuncOp op) {
 // getNumArguments hook not failing.
 LogicalResult LLVMFuncOp::verifyType() {
   auto llvmType = getTypeAttr().getValue().dyn_cast_or_null<LLVMType>();
-  if (!llvmType || !llvmType.getUnderlyingType()->isFunctionTy())
+  if (!llvmType || !llvmType.isFunctionTy())
     return emitOpError("requires '" + getTypeAttrName() +
                        "' attribute of wrapped LLVM function type");
 
@@ -1384,7 +1377,7 @@ LogicalResult LLVMFuncOp::verifyType() {
 // Hook for OpTrait::FunctionLike, returns the number of function arguments.
 // Depends on the type attribute being correct as checked by verifyType
 unsigned LLVMFuncOp::getNumFuncArguments() {
-  return getType().getUnderlyingType()->getFunctionNumParams();
+  return getType().getFunctionNumParams();
 }
 
 // Hook for OpTrait::FunctionLike, returns the number of function results.
@@ -1424,8 +1417,7 @@ static LogicalResult verify(LLVMFuncOp op) {
   if (op.isVarArg())
     return op.emitOpError("only external functions can be variadic");
 
-  auto *funcType = cast<llvm::FunctionType>(op.getType().getUnderlyingType());
-  unsigned numArguments = funcType->getNumParams();
+  unsigned numArguments = op.getType().getFunctionNumParams();
   Block &entryBlock = op.front();
   for (unsigned i = 0; i < numArguments; ++i) {
     Type argType = entryBlock.getArgument(i).getType();
@@ -1433,7 +1425,7 @@ static LogicalResult verify(LLVMFuncOp op) {
     if (!argLLVMType)
       return op.emitOpError("entry block argument #")
              << i << " is not of LLVM type";
-    if (funcType->getParamType(i) != argLLVMType.getUnderlyingType())
+    if (op.getType().getFunctionParamType(i) != argLLVMType)
       return op.emitOpError("the type of entry block argument #")
              << i << " does not match the function signature";
   }
@@ -1566,7 +1558,7 @@ static LogicalResult verify(AtomicRMWOp op) {
     return op.emitOpError(
         "expected LLVM IR result type to match type for operand #1");
   if (op.bin_op() == AtomicBinOp::fadd || op.bin_op() == AtomicBinOp::fsub) {
-    if (!valType.getUnderlyingType()->isFloatingPointTy())
+    if (!valType.isFloatingPointTy())
       return op.emitOpError("expected LLVM IR floating point type");
   } else if (op.bin_op() == AtomicBinOp::xchg) {
     if (!valType.isIntegerTy(8) && !valType.isIntegerTy(16) &&
@@ -1842,6 +1834,13 @@ llvm::Type *LLVMType::getUnderlyingType() const {
   return getImpl()->underlyingType;
 }
 
+void LLVMType::getUnderlyingTypes(ArrayRef<LLVMType> types,
+                                  SmallVectorImpl<llvm::Type *> &result) {
+  result.reserve(result.size() + types.size());
+  for (LLVMType ty : types)
+    result.push_back(ty.getUnderlyingType());
+}
+
 /// Array type utilities.
 LLVMType LLVMType::getArrayElementType() {
   return get(getContext(), getUnderlyingType()->getArrayElementType());
@@ -1861,6 +1860,9 @@ unsigned LLVMType::getVectorNumElements() {
   return llvm::cast<llvm::FixedVectorType>(getUnderlyingType())
       ->getNumElements();
 }
+llvm::ElementCount LLVMType::getVectorElementCount() {
+  return llvm::cast<llvm::VectorType>(getUnderlyingType())->getElementCount();
+}
 bool LLVMType::isVectorTy() { return getUnderlyingType()->isVectorTy(); }
 
 /// Function type utilities.
@@ -1876,6 +1878,9 @@ LLVMType LLVMType::getFunctionResultType() {
       llvm::cast<llvm::FunctionType>(getUnderlyingType())->getReturnType());
 }
 bool LLVMType::isFunctionTy() { return getUnderlyingType()->isFunctionTy(); }
+bool LLVMType::isFunctionVarArg() {
+  return getUnderlyingType()->isFunctionVarArg();
+}
 
 /// Pointer type utilities.
 LLVMType LLVMType::getPointerTo(unsigned addrSpace) {
@@ -1888,6 +1893,9 @@ LLVMType LLVMType::getPointerElementTy() {
   return get(getContext(), getUnderlyingType()->getPointerElementType());
 }
 bool LLVMType::isPointerTy() { return getUnderlyingType()->isPointerTy(); }
+bool LLVMType::isValidPointerElementType(LLVMType type) {
+  return llvm::PointerType::isValidElementType(type.getUnderlyingType());
+}
 
 /// Struct type utilities.
 LLVMType LLVMType::getStructElementType(unsigned i) {
@@ -1974,18 +1982,12 @@ LLVMType LLVMType::getStructTy(LLVMDialect *dialect,
                                  isPacked);
   });
 }
-inline static SmallVector<llvm::Type *, 8>
-toUnderlyingTypes(ArrayRef<LLVMType> elements) {
-  SmallVector<llvm::Type *, 8> llvmElements;
-  for (auto elt : elements)
-    llvmElements.push_back(elt.getUnderlyingType());
-  return llvmElements;
-}
 LLVMType LLVMType::createStructTy(LLVMDialect *dialect,
                                   ArrayRef<LLVMType> elements,
                                   Optional<StringRef> name, bool isPacked) {
   StringRef sr = name.hasValue() ? *name : "";
-  SmallVector<llvm::Type *, 8> llvmElements(toUnderlyingTypes(elements));
+  SmallVector<llvm::Type *, 8> llvmElements;
+  getUnderlyingTypes(elements, llvmElements);
   return getLocked(dialect, [=] {
     auto *rv = llvm::StructType::create(dialect->getLLVMContext(), sr);
     if (!llvmElements.empty())
@@ -1997,7 +1999,8 @@ LLVMType LLVMType::setStructTyBody(LLVMType structType,
                                    ArrayRef<LLVMType> elements, bool isPacked) {
   llvm::StructType *st =
       llvm::cast<llvm::StructType>(structType.getUnderlyingType());
-  SmallVector<llvm::Type *, 8> llvmElements(toUnderlyingTypes(elements));
+  SmallVector<llvm::Type *, 8> llvmElements;
+  getUnderlyingTypes(elements, llvmElements);
   return getLocked(&structType.getDialect(), [=] {
     st->setBody(llvmElements, isPacked);
     return st;
@@ -2017,6 +2020,10 @@ LLVMType LLVMType::getVoidTy(LLVMDialect *dialect) {
 
 bool LLVMType::isVoidTy() { return getUnderlyingType()->isVoidTy(); }
 
+llvm::Type *mlir::LLVM::convertLLVMType(LLVMType type) {
+  return type.getUnderlyingType();
+}
+
 //===----------------------------------------------------------------------===//
 // Utility functions.
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Target/LLVMIR/ConvertFromLLVMIR.cpp b/mlir/lib/Target/LLVMIR/ConvertFromLLVMIR.cpp
index 9754c614efdfb..77897d65e1a50 100644
--- a/mlir/lib/Target/LLVMIR/ConvertFromLLVMIR.cpp
+++ b/mlir/lib/Target/LLVMIR/ConvertFromLLVMIR.cpp
@@ -234,18 +234,17 @@ Type Importer::getStdTypeForAttr(LLVMType type) {
     return nullptr;
 
   if (type.isIntegerTy())
-    return b.getIntegerType(type.getUnderlyingType()->getIntegerBitWidth());
+    return b.getIntegerType(type.getIntegerBitWidth());
 
-  if (type.getUnderlyingType()->isFloatTy())
+  if (type.isFloatTy())
     return b.getF32Type();
 
-  if (type.getUnderlyingType()->isDoubleTy())
+  if (type.isDoubleTy())
     return b.getF64Type();
 
   // LLVM vectors can only contain scalars.
   if (type.isVectorTy()) {
-    auto numElements = llvm::cast<llvm::VectorType>(type.getUnderlyingType())
-                           ->getElementCount();
+    auto numElements = type.getVectorElementCount();
     if (numElements.Scalable) {
       emitError(unknownLoc) << "scalable vectors not supported";
       return nullptr;
@@ -270,9 +269,7 @@ Type Importer::getStdTypeForAttr(LLVMType type) {
     // attribute type.
     if (type.getArrayElementType().isVectorTy()) {
       LLVMType vectorType = type.getArrayElementType();
-      auto numElements =
-          llvm::cast<llvm::VectorType>(vectorType.getUnderlyingType())
-              ->getElementCount();
+      auto numElements = vectorType.getVectorElementCount();
       if (numElements.Scalable) {
         emitError(unknownLoc) << "scalable vectors not supported";
         return nullptr;
diff --git a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
index 3a70dd3932e92..a0aefc988a5da 100644
--- a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
@@ -574,7 +574,7 @@ LogicalResult ModuleTranslation::convertOperation(Operation &opInst,
   }
 
   if (auto lpOp = dyn_cast<LLVM::LandingpadOp>(opInst)) {
-    llvm::Type *ty = lpOp.getType().dyn_cast<LLVMType>().getUnderlyingType();
+    llvm::Type *ty = convertType(lpOp.getType().cast<LLVMType>());
     llvm::LandingPadInst *lpi =
         builder.CreateLandingPad(ty, lpOp.getNumOperands());
 
@@ -661,7 +661,7 @@ LogicalResult ModuleTranslation::convertBlock(Block &bb, bool ignoreArguments) {
       if (!wrappedType)
         return emitError(bb.front().getLoc(),
                          "block argument does not have an LLVM type");
-      llvm::Type *type = wrappedType.getUnderlyingType();
+      llvm::Type *type = convertType(wrappedType);
       llvm::PHINode *phi = builder.CreatePHI(type, numPredecessors);
       valueMapping[arg] = phi;
     }
@@ -687,7 +687,7 @@ LogicalResult ModuleTranslation::convertGlobals() {
   llvm::sys::SmartScopedLock<true> scopedLock(
       llvmDialect->getLLVMContextMutex());
   for (auto op : getModuleBody(mlirModule).getOps<LLVM::GlobalOp>()) {
-    llvm::Type *type = op.getType().getUnderlyingType();
+    llvm::Type *type = convertType(op.getType());
     llvm::Constant *cst = llvm::UndefValue::get(type);
     if (op.getValueOrNull()) {
       // String attributes are treated separately because they cannot appear as
@@ -826,7 +826,7 @@ LogicalResult ModuleTranslation::convertOneFunction(LLVMFuncOp func) {
       // NB: Attribute already verified to be boolean, so check if we can indeed
       // attach the attribute to this argument, based on its type.
       auto argTy = mlirArg.getType().dyn_cast<LLVM::LLVMType>();
-      if (!argTy.getUnderlyingType()->isPointerTy())
+      if (!argTy.isPointerTy())
         return func.emitError(
             "llvm.noalias attribute attached to LLVM non-pointer argument");
       if (attr.getValue())
@@ -837,7 +837,7 @@ LogicalResult ModuleTranslation::convertOneFunction(LLVMFuncOp func) {
       // NB: Attribute already verified to be int, so check if we can indeed
       // attach the attribute to this argument, based on its type.
       auto argTy = mlirArg.getType().dyn_cast<LLVM::LLVMType>();
-      if (!argTy.getUnderlyingType()->isPointerTy())
+      if (!argTy.isPointerTy())
         return func.emitError(
             "llvm.align attribute attached to LLVM non-pointer argument");
       llvmArg.addAttrs(
@@ -896,7 +896,7 @@ LogicalResult ModuleTranslation::convertFunctionSignatures() {
   for (auto function : getModuleBody(mlirModule).getOps<LLVMFuncOp>()) {
     llvm::FunctionCallee llvmFuncCst = llvmModule->getOrInsertFunction(
         function.getName(),
-        cast<llvm::FunctionType>(function.getType().getUnderlyingType()));
+        cast<llvm::FunctionType>(convertType(function.getType())));
     llvm::Function *llvmFunc = cast<llvm::Function>(llvmFuncCst.getCallee());
     llvmFunc->setLinkage(convertLinkageToLLVM(function.linkage()));
     functionMapping[function.getName()] = llvmFunc;
@@ -928,6 +928,10 @@ LogicalResult ModuleTranslation::convertFunctions() {
   return success();
 }
 
+llvm::Type *ModuleTranslation::convertType(LLVMType type) {
+  return LLVM::convertLLVMType(type);
+}
+
 /// A helper to look up remapped operands in the value remapping table.`
 SmallVector<llvm::Value *, 8>
 ModuleTranslation::lookupValues(ValueRange values) {
diff --git a/mlir/tools/mlir-tblgen/LLVMIRConversionGen.cpp b/mlir/tools/mlir-tblgen/LLVMIRConversionGen.cpp
index 9edbad8fdd540..f62e7aebe24a6 100644
--- a/mlir/tools/mlir-tblgen/LLVMIRConversionGen.cpp
+++ b/mlir/tools/mlir-tblgen/LLVMIRConversionGen.cpp
@@ -135,8 +135,7 @@ static bool emitOneBuilder(const Record &record, raw_ostream &os) {
     } else if (isResultName(op, name)) {
       bs << formatv("valueMapping[op.{0}()]", name);
     } else if (name == "_resultType") {
-      bs << "op.getResult().getType().cast<LLVM::LLVMType>()."
-            "getUnderlyingType()";
+      bs << "convertType(op.getResult().getType().cast<LLVM::LLVMType>())";
     } else if (name == "_hasResult") {
       bs << "opInst.getNumResults() == 1";
     } else if (name == "_location") {

From 5d84eafc6b86a42e261af8d753c3a823e0e7c67e Mon Sep 17 00:00:00 2001
From: David Sherwood <david.sherwood@arm.com>
Date: Fri, 3 Jul 2020 13:41:34 +0100
Subject: [PATCH 0440/1035] [CodeGen] Remove calls to getVectorNumElements in
 DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR

In DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR I have replaced
calls to getVectorNumElements with getVectorMinNumElements, since
this code path works for both fixed and scalable vector types. For
scalable vectors the index will be multiplied by VSCALE.

Fixes warnings in this test:

  sve-sext-zext.ll

Differential revision: https://reviews.llvm.org/D83198
---
 llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index 1394f084c6dc6..6963de2e50290 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -2197,13 +2197,19 @@ SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
   SDValue Idx = N->getOperand(1);
   SDLoc dl(N);
   SDValue Lo, Hi;
+
+  if (SubVT.isScalableVector() !=
+      N->getOperand(0).getValueType().isScalableVector())
+    report_fatal_error("Extracting a fixed-length vector from an illegal "
+                       "scalable vector is not yet supported");
+
   GetSplitVector(N->getOperand(0), Lo, Hi);
 
-  uint64_t LoElts = Lo.getValueType().getVectorNumElements();
+  uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
   uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
 
   if (IdxVal < LoElts) {
-    assert(IdxVal + SubVT.getVectorNumElements() <= LoElts &&
+    assert(IdxVal + SubVT.getVectorMinNumElements() <= LoElts &&
            "Extracted subvector crosses vector split!");
     return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Lo, Idx);
   } else {

From f43b5c7a76ab83dcc80e6769d41d5c4b761312b1 Mon Sep 17 00:00:00 2001
From: David Sherwood <david.sherwood@arm.com>
Date: Fri, 3 Jul 2020 14:20:59 +0100
Subject: [PATCH 0441/1035] [SVE] Add checks for no warnings in
 CodeGen/AArch64/sve-sext-zext.ll

Previous patches fixed up all the warnings in this test:

  llvm/test/CodeGen/AArch64/sve-sext-zext.ll

and this change simply checks that no new warnings are added in future.

Differential revision: https://reviews.llvm.org/D83205
---
 llvm/test/CodeGen/AArch64/sve-sext-zext.ll | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/llvm/test/CodeGen/AArch64/sve-sext-zext.ll b/llvm/test/CodeGen/AArch64/sve-sext-zext.ll
index 24cf433306bb5..1275811d175e1 100644
--- a/llvm/test/CodeGen/AArch64/sve-sext-zext.ll
+++ b/llvm/test/CodeGen/AArch64/sve-sext-zext.ll
@@ -1,5 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve < %s 2>%t | FileCheck %s
+; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t
+
+; WARN-NOT: warning
 
 define <vscale x 16 x i8> @sext_i1_i8(<vscale x 16 x i1> %a) {
 ; CHECK-LABEL: sext_i1_i8:

From 33abb7292e8f8f373f319a96a9456e05a1efce2d Mon Sep 17 00:00:00 2001
From: Chris Gyurgyik <37983775+cgyurgyik@users.noreply.github.com>
Date: Wed, 29 Jul 2020 08:18:07 -0400
Subject: [PATCH 0442/1035] [libc] [obvious] Fix typo in binary header.

---
 libc/utils/LibcTableGenUtil/APIIndexer.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libc/utils/LibcTableGenUtil/APIIndexer.cpp b/libc/utils/LibcTableGenUtil/APIIndexer.cpp
index 366dea9ec11c6..6ef9a6576b87f 100644
--- a/libc/utils/LibcTableGenUtil/APIIndexer.cpp
+++ b/libc/utils/LibcTableGenUtil/APIIndexer.cpp
@@ -1,4 +1,4 @@
-//===-- Implementation of APIIndexer clas ---------------------------------===//
+//===-- Implementation of APIIndexer class --------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.

From cdd45d5f9cf61e4a4c0bb01d5cdcdd3d12ffb10f Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Tue, 21 Jul 2020 22:15:18 -0400
Subject: [PATCH 0443/1035] AMDGPU/GlobalISel: Select
 llvm.amdgcn.global.atomic.csub

Remove the custom node boilerplate. Not sure why this tried to handle
the LDS atomic stuff.
---
 llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp |   3 +-
 llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp |   1 -
 llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h   |   1 -
 .../Target/AMDGPU/AMDGPURegisterBankInfo.cpp  |   1 +
 llvm/lib/Target/AMDGPU/BUFInstructions.td     |   2 +-
 llvm/lib/Target/AMDGPU/FLATInstructions.td    |   4 +-
 llvm/lib/Target/AMDGPU/SIISelLowering.cpp     |  13 --
 llvm/lib/Target/AMDGPU/SIInstrInfo.td         |   9 --
 .../llvm.amdgcn.global.atomic.csub.ll         | 115 ++++++++++++++++++
 9 files changed, 120 insertions(+), 29 deletions(-)
 create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.global.atomic.csub.ll

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index 5539f4e8699db..368af73b18d3c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -716,8 +716,7 @@ void AMDGPUDAGToDAGISel::Select(SDNode *N) {
       (Opc == AMDGPUISD::ATOMIC_INC || Opc == AMDGPUISD::ATOMIC_DEC ||
        Opc == ISD::ATOMIC_LOAD_FADD ||
        Opc == AMDGPUISD::ATOMIC_LOAD_FMIN ||
-       Opc == AMDGPUISD::ATOMIC_LOAD_FMAX ||
-       Opc == AMDGPUISD::ATOMIC_LOAD_CSUB)) {
+       Opc == AMDGPUISD::ATOMIC_LOAD_FMAX)) {
     N = glueCopyToM0LDSInit(N);
     SelectCode(N);
     return;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index f7cdc7461eacd..3ffd9b79864a2 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -4336,7 +4336,6 @@ const char* AMDGPUTargetLowering::getTargetNodeName(unsigned Opcode) const {
   NODE_NAME_CASE(ATOMIC_DEC)
   NODE_NAME_CASE(ATOMIC_LOAD_FMIN)
   NODE_NAME_CASE(ATOMIC_LOAD_FMAX)
-  NODE_NAME_CASE(ATOMIC_LOAD_CSUB)
   NODE_NAME_CASE(BUFFER_LOAD)
   NODE_NAME_CASE(BUFFER_LOAD_UBYTE)
   NODE_NAME_CASE(BUFFER_LOAD_USHORT)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
index 85f23c81db170..e995fdcd29ce4 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
@@ -508,7 +508,6 @@ enum NodeType : unsigned {
   ATOMIC_DEC,
   ATOMIC_LOAD_FMIN,
   ATOMIC_LOAD_FMAX,
-  ATOMIC_LOAD_CSUB,
   BUFFER_LOAD,
   BUFFER_LOAD_UBYTE,
   BUFFER_LOAD_USHORT,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
index 93006bb68ed37..d53246967f45e 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
@@ -4205,6 +4205,7 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
     case Intrinsic::amdgcn_ds_fadd:
     case Intrinsic::amdgcn_ds_fmin:
     case Intrinsic::amdgcn_ds_fmax:
+    case Intrinsic::amdgcn_global_atomic_csub:
       return getDefaultMappingAllVGPR(MI);
     case Intrinsic::amdgcn_ds_ordered_add:
     case Intrinsic::amdgcn_ds_ordered_swap: {
diff --git a/llvm/lib/Target/AMDGPU/BUFInstructions.td b/llvm/lib/Target/AMDGPU/BUFInstructions.td
index d175edd93c642..bb04ae06bc3b4 100644
--- a/llvm/lib/Target/AMDGPU/BUFInstructions.td
+++ b/llvm/lib/Target/AMDGPU/BUFInstructions.td
@@ -1007,7 +1007,7 @@ defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Pseudo_Atomics <
 
 let SubtargetPredicate = HasGFX10_BEncoding in
 defm BUFFER_ATOMIC_CSUB : MUBUF_Pseudo_Atomics_RTN <
-  "buffer_atomic_csub", VGPR_32, i32, atomic_csub_global_32
+  "buffer_atomic_csub", VGPR_32, i32, int_amdgcn_global_atomic_csub
 >;
 
 let SubtargetPredicate = isGFX8GFX9 in {
diff --git a/llvm/lib/Target/AMDGPU/FLATInstructions.td b/llvm/lib/Target/AMDGPU/FLATInstructions.td
index 69facada2e964..f99354b77272f 100644
--- a/llvm/lib/Target/AMDGPU/FLATInstructions.td
+++ b/llvm/lib/Target/AMDGPU/FLATInstructions.td
@@ -662,7 +662,7 @@ defm GLOBAL_ATOMIC_DEC_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_dec_x2",
 
 let SubtargetPredicate = HasGFX10_BEncoding in
 defm GLOBAL_ATOMIC_CSUB : FLAT_Global_Atomic_Pseudo_RTN <"global_atomic_csub",
-                              VGPR_32, i32, atomic_csub_global_32>;
+                              VGPR_32, i32, int_amdgcn_global_atomic_csub>;
 } // End is_flat_global = 1
 
 
@@ -959,7 +959,7 @@ def : FlatSignedAtomicPat <GLOBAL_ATOMIC_OR_RTN, atomic_load_or_global_32, i32>;
 def : FlatSignedAtomicPat <GLOBAL_ATOMIC_SWAP_RTN, atomic_swap_global_32, i32>;
 def : FlatSignedAtomicPat <GLOBAL_ATOMIC_CMPSWAP_RTN, AMDGPUatomic_cmp_swap_global_32, i32, v2i32>;
 def : FlatSignedAtomicPat <GLOBAL_ATOMIC_XOR_RTN, atomic_load_xor_global_32, i32>;
-def : FlatSignedAtomicPat <GLOBAL_ATOMIC_CSUB_RTN, atomic_csub_global_32, i32>;
+def : FlatSignedAtomicPat <GLOBAL_ATOMIC_CSUB_RTN, int_amdgcn_global_atomic_csub, i32>;
 
 def : FlatSignedAtomicPat <GLOBAL_ATOMIC_ADD_X2_RTN, atomic_load_add_global_64, i64>;
 def : FlatSignedAtomicPat <GLOBAL_ATOMIC_SUB_X2_RTN, atomic_load_sub_global_64, i64>;
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 4c9fce92f9a26..db280d9fc85b7 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -7196,19 +7196,6 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
     return DAG.getMemIntrinsicNode(AMDGPUISD::BUFFER_ATOMIC_CMPSWAP, DL,
                                    Op->getVTList(), Ops, VT, M->getMemOperand());
   }
-  case Intrinsic::amdgcn_global_atomic_csub: {
-    MemSDNode *M = cast<MemSDNode>(Op);
-    SDValue Ops[] = {
-      M->getOperand(0), // Chain
-      M->getOperand(2), // Ptr
-      M->getOperand(3)  // Value
-    };
-
-    return DAG.getMemIntrinsicNode(AMDGPUISD::ATOMIC_LOAD_CSUB, SDLoc(Op),
-                                   M->getVTList(), Ops, M->getMemoryVT(),
-                                   M->getMemOperand());
-  }
-
   default:
     if (const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr =
             AMDGPU::getImageDimIntrinsicInfo(IntrID))
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.td b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
index b6c2082b8d23d..d146114074a7e 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.td
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
@@ -55,10 +55,6 @@ def SIatomic_dec : SDNode<"AMDGPUISD::ATOMIC_DEC", SDTAtomic2,
   [SDNPMayLoad, SDNPMayStore, SDNPMemOperand, SDNPHasChain]
 >;
 
-def SIatomic_csub : SDNode<"AMDGPUISD::ATOMIC_LOAD_CSUB", SDTAtomic2,
-  [SDNPMayLoad, SDNPMayStore, SDNPMemOperand, SDNPHasChain]
->;
-
 def SDTAtomic2_f32 : SDTypeProfile<1, 2, [
   SDTCisSameAs<0,2>, SDTCisFP<0>, SDTCisPtrTy<1>
 ]>;
@@ -311,10 +307,6 @@ class isPackedType<ValueType SrcVT> {
 // PatFrags for global memory operations
 //===----------------------------------------------------------------------===//
 
-let AddressSpaces = !cast<AddressSpaceList>("LoadAddress_global").AddrSpaces in {
-defm atomic_csub_global : binary_atomic_op<SIatomic_csub>;
-}
-
 foreach as = [ "global", "flat", "constant", "local", "private", "region" ] in {
 let AddressSpaces = !cast<AddressSpaceList>("LoadAddress_"#as).AddrSpaces in {
 
@@ -668,7 +660,6 @@ multiclass SIAtomicM0Glue2 <string op_name, bit is_amdgpu = 0,
 
 defm atomic_load_add : SIAtomicM0Glue2 <"LOAD_ADD">;
 defm atomic_load_sub : SIAtomicM0Glue2 <"LOAD_SUB">;
-defm atomic_load_csub : SIAtomicM0Glue2 <"LOAD_CSUB", 1>;
 defm atomic_inc : SIAtomicM0Glue2 <"INC", 1>;
 defm atomic_dec : SIAtomicM0Glue2 <"DEC", 1>;
 defm atomic_load_and : SIAtomicM0Glue2 <"LOAD_AND">;
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.global.atomic.csub.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.global.atomic.csub.ll
new file mode 100644
index 0000000000000..9bc56d124a9cd
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.global.atomic.csub.ll
@@ -0,0 +1,115 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1030 -verify-machineinstrs < %s | FileCheck %s -check-prefix=GCN
+
+define i32 @global_atomic_csub(i32 addrspace(1)* %ptr, i32 %data) {
+; GCN-LABEL: global_atomic_csub:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    s_waitcnt_vscnt null, 0x0
+; GCN-NEXT:    global_atomic_csub v0, v[0:1], v2, off glc
+; GCN-NEXT:    ; implicit-def: $vcc_hi
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %ret = call i32 @llvm.amdgcn.global.atomic.csub.p1i32(i32 addrspace(1)* %ptr, i32 %data)
+  ret i32 %ret
+}
+
+define i32 @global_atomic_csub_offset(i32 addrspace(1)* %ptr, i32 %data) {
+; GCN-LABEL: global_atomic_csub_offset:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    s_waitcnt_vscnt null, 0x0
+; GCN-NEXT:    s_movk_i32 s4, 0x1000
+; GCN-NEXT:    s_mov_b32 s5, 0
+; GCN-NEXT:    v_mov_b32_e32 v3, s4
+; GCN-NEXT:    v_mov_b32_e32 v4, s5
+; GCN-NEXT:    ; implicit-def: $vcc_hi
+; GCN-NEXT:    v_add_co_u32_e64 v0, vcc_lo, v0, v3
+; GCN-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, v1, v4, vcc_lo
+; GCN-NEXT:    global_atomic_csub v0, v[0:1], v2, off glc
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i64 1024
+  %ret = call i32 @llvm.amdgcn.global.atomic.csub.p1i32(i32 addrspace(1)* %gep, i32 %data)
+  ret i32 %ret
+}
+
+define void @global_atomic_csub_nortn(i32 addrspace(1)* %ptr, i32 %data) {
+; GCN-LABEL: global_atomic_csub_nortn:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    s_waitcnt_vscnt null, 0x0
+; GCN-NEXT:    global_atomic_csub v0, v[0:1], v2, off glc
+; GCN-NEXT:    ; implicit-def: $vcc_hi
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %ret = call i32 @llvm.amdgcn.global.atomic.csub.p1i32(i32 addrspace(1)* %ptr, i32 %data)
+  ret void
+}
+
+define void @global_atomic_csub_offset_nortn(i32 addrspace(1)* %ptr, i32 %data) {
+; GCN-LABEL: global_atomic_csub_offset_nortn:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    s_waitcnt_vscnt null, 0x0
+; GCN-NEXT:    s_movk_i32 s4, 0x1000
+; GCN-NEXT:    s_mov_b32 s5, 0
+; GCN-NEXT:    v_mov_b32_e32 v3, s4
+; GCN-NEXT:    v_mov_b32_e32 v4, s5
+; GCN-NEXT:    ; implicit-def: $vcc_hi
+; GCN-NEXT:    v_add_co_u32_e64 v0, vcc_lo, v0, v3
+; GCN-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, v1, v4, vcc_lo
+; GCN-NEXT:    global_atomic_csub v0, v[0:1], v2, off glc
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i64 1024
+  %ret = call i32 @llvm.amdgcn.global.atomic.csub.p1i32(i32 addrspace(1)* %gep, i32 %data)
+  ret void
+}
+
+define amdgpu_kernel void @global_atomic_csub_sgpr_base_offset(i32 addrspace(1)* %ptr, i32 %data) {
+; GCN-LABEL: global_atomic_csub_sgpr_base_offset:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_clause 0x1
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GCN-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GCN-NEXT:    ; implicit-def: $vcc_hi
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_add_u32 s0, s0, 0x1000
+; GCN-NEXT:    s_addc_u32 s1, s1, 0
+; GCN-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-NEXT:    v_mov_b32_e32 v1, s1
+; GCN-NEXT:    v_mov_b32_e32 v2, s2
+; GCN-NEXT:    global_atomic_csub v0, v[0:1], v2, off glc
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    global_store_dword v[0:1], v0, off
+; GCN-NEXT:    s_endpgm
+  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i64 1024
+  %ret = call i32 @llvm.amdgcn.global.atomic.csub.p1i32(i32 addrspace(1)* %gep, i32 %data)
+  store i32 %ret, i32 addrspace(1)* undef
+  ret void
+}
+
+define amdgpu_kernel void @global_atomic_csub_sgpr_base_offset_nortn(i32 addrspace(1)* %ptr, i32 %data) {
+; GCN-LABEL: global_atomic_csub_sgpr_base_offset_nortn:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_clause 0x1
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GCN-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_add_u32 s0, s0, 0x1000
+; GCN-NEXT:    s_addc_u32 s1, s1, 0
+; GCN-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-NEXT:    v_mov_b32_e32 v1, s1
+; GCN-NEXT:    v_mov_b32_e32 v2, s2
+; GCN-NEXT:    global_atomic_csub v0, v[0:1], v2, off glc
+; GCN-NEXT:    s_endpgm
+  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i64 1024
+  %ret = call i32 @llvm.amdgcn.global.atomic.csub.p1i32(i32 addrspace(1)* %gep, i32 %data)
+  ret void
+}
+
+declare i32 @llvm.amdgcn.global.atomic.csub.p1i32(i32 addrspace(1)* nocapture, i32) #1
+
+attributes #0 = { nounwind willreturn }
+attributes #1 = { argmemonly nounwind }

From c230965ccf36af5c88c28d065488f76712333683 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Mon, 13 Jul 2020 20:57:31 -0400
Subject: [PATCH 0444/1035] AMDGPU: Make saturating add/sub legal for DAG path

---
 .../AMDGPU/AMDGPUTargetTransformInfo.cpp      |    8 +-
 llvm/lib/Target/AMDGPU/SIISelLowering.cpp     |   29 +
 llvm/test/CodeGen/AMDGPU/saddo.ll             |   44 +-
 llvm/test/CodeGen/AMDGPU/saddsat.ll           |  439 +++++++
 llvm/test/CodeGen/AMDGPU/ssubsat.ll           | 1004 +++++++++++++++++
 llvm/test/CodeGen/AMDGPU/uaddsat.ll           |  522 +++++++++
 llvm/test/CodeGen/AMDGPU/usubsat.ll           |  492 ++++++++
 .../SLPVectorizer/AMDGPU/add_sub_sat.ll       |  303 +++++
 8 files changed, 2815 insertions(+), 26 deletions(-)
 create mode 100644 llvm/test/CodeGen/AMDGPU/saddsat.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/ssubsat.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/uaddsat.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/usubsat.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/AMDGPU/add_sub_sat.ll

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index 7998b0cb9f6ac..4ae092ce55bff 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -583,13 +583,17 @@ int GCNTTIImpl::getArithmeticInstrCost(unsigned Opcode, Type *Ty,
                                        Opd1PropInfo, Opd2PropInfo, Args, CxtI);
 }
 
-// Return true if there's a potential benefit from using v2f16 instructions for
-// an intrinsic, even if it requires nontrivial legalization.
+// Return true if there's a potential benefit from using v2f16/v2i16
+// instructions for an intrinsic, even if it requires nontrivial legalization.
 static bool intrinsicHasPackedVectorBenefit(Intrinsic::ID ID) {
   switch (ID) {
   case Intrinsic::fma: // TODO: fmuladd
   // There's a small benefit to using vector ops in the legalized code.
   case Intrinsic::round:
+  case Intrinsic::uadd_sat:
+  case Intrinsic::usub_sat:
+  case Intrinsic::sadd_sat:
+  case Intrinsic::ssub_sat:
     return true;
   default:
     return false;
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index db280d9fc85b7..367e4fc974bb5 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -487,6 +487,19 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
   if (Subtarget->hasBFE())
     setHasExtractBitsInsn(true);
 
+  // Clamp modifier on add/sub
+  if (Subtarget->hasIntClamp()) {
+    setOperationAction(ISD::UADDSAT, MVT::i32, Legal);
+    setOperationAction(ISD::USUBSAT, MVT::i32, Legal);
+  }
+
+  if (Subtarget->hasAddNoCarry()) {
+    setOperationAction(ISD::SADDSAT, MVT::i16, Legal);
+    setOperationAction(ISD::SSUBSAT, MVT::i16, Legal);
+    setOperationAction(ISD::SADDSAT, MVT::i32, Legal);
+    setOperationAction(ISD::SSUBSAT, MVT::i32, Legal);
+  }
+
   setOperationAction(ISD::FMINNUM, MVT::f32, Custom);
   setOperationAction(ISD::FMAXNUM, MVT::f32, Custom);
   setOperationAction(ISD::FMINNUM, MVT::f64, Custom);
@@ -539,6 +552,8 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
     setOperationAction(ISD::UDIV, MVT::i16, Promote);
     setOperationAction(ISD::SREM, MVT::i16, Promote);
     setOperationAction(ISD::UREM, MVT::i16, Promote);
+    setOperationAction(ISD::UADDSAT, MVT::i16, Legal);
+    setOperationAction(ISD::USUBSAT, MVT::i16, Legal);
 
     setOperationAction(ISD::BITREVERSE, MVT::i16, Promote);
 
@@ -703,6 +718,11 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
     setOperationAction(ISD::SMAX, MVT::v2i16, Legal);
     setOperationAction(ISD::UMAX, MVT::v2i16, Legal);
 
+    setOperationAction(ISD::UADDSAT, MVT::v2i16, Legal);
+    setOperationAction(ISD::USUBSAT, MVT::v2i16, Legal);
+    setOperationAction(ISD::SADDSAT, MVT::v2i16, Legal);
+    setOperationAction(ISD::SSUBSAT, MVT::v2i16, Legal);
+
     setOperationAction(ISD::FADD, MVT::v2f16, Legal);
     setOperationAction(ISD::FMUL, MVT::v2f16, Legal);
     setOperationAction(ISD::FMA, MVT::v2f16, Legal);
@@ -730,6 +750,11 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
     setOperationAction(ISD::UMIN, MVT::v4i16, Custom);
     setOperationAction(ISD::UMAX, MVT::v4i16, Custom);
 
+    setOperationAction(ISD::UADDSAT, MVT::v4i16, Custom);
+    setOperationAction(ISD::SADDSAT, MVT::v4i16, Custom);
+    setOperationAction(ISD::USUBSAT, MVT::v4i16, Custom);
+    setOperationAction(ISD::SSUBSAT, MVT::v4i16, Custom);
+
     setOperationAction(ISD::FADD, MVT::v4f16, Custom);
     setOperationAction(ISD::FMUL, MVT::v4f16, Custom);
     setOperationAction(ISD::FMA, MVT::v4f16, Custom);
@@ -4473,6 +4498,10 @@ SDValue SITargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
   case ISD::FMUL:
   case ISD::FMINNUM_IEEE:
   case ISD::FMAXNUM_IEEE:
+  case ISD::UADDSAT:
+  case ISD::USUBSAT:
+  case ISD::SADDSAT:
+  case ISD::SSUBSAT:
     return splitBinaryVectorOp(Op, DAG);
   case ISD::SMULO:
   case ISD::UMULO:
diff --git a/llvm/test/CodeGen/AMDGPU/saddo.ll b/llvm/test/CodeGen/AMDGPU/saddo.ll
index fe9e6275e0d58..296e8e770827e 100644
--- a/llvm/test/CodeGen/AMDGPU/saddo.ll
+++ b/llvm/test/CodeGen/AMDGPU/saddo.ll
@@ -136,17 +136,16 @@ define amdgpu_kernel void @s_saddo_i32(i32 addrspace(1)* %out, i1 addrspace(1)*
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s4
-; GFX9-NEXT:    v_cmp_lt_i32_e64 s[2:3], s1, 0
-; GFX9-NEXT:    s_add_i32 s1, s0, s1
-; GFX9-NEXT:    v_mov_b32_e32 v4, s0
-; GFX9-NEXT:    v_cmp_lt_i32_e32 vcc, s1, v4
 ; GFX9-NEXT:    v_mov_b32_e32 v4, s1
+; GFX9-NEXT:    v_add_i32 v4, s0, v4 clamp
+; GFX9-NEXT:    s_add_i32 s0, s0, s1
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s5
-; GFX9-NEXT:    s_xor_b64 s[0:1], s[2:3], vcc
-; GFX9-NEXT:    global_store_dword v[0:1], v4, off
+; GFX9-NEXT:    v_mov_b32_e32 v5, s0
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, s0, v4
+; GFX9-NEXT:    global_store_dword v[0:1], v5, off
 ; GFX9-NEXT:    v_mov_b32_e32 v2, s6
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s7
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
 ; GFX9-NEXT:    global_store_byte v[2:3], v0, off
 ; GFX9-NEXT:    s_endpgm
   %sadd = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 %a, i32 %b) nounwind
@@ -227,12 +226,11 @@ define amdgpu_kernel void @v_saddo_i32(i32 addrspace(1)* %out, i1 addrspace(1)*
 ; GFX9-NEXT:    v_mov_b32_e32 v2, s2
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s3
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_add_u32_e32 v6, v4, v5
-; GFX9-NEXT:    v_cmp_gt_i32_e32 vcc, 0, v5
-; GFX9-NEXT:    v_cmp_lt_i32_e64 s[0:1], v6, v4
-; GFX9-NEXT:    s_xor_b64 s[0:1], vcc, s[0:1]
-; GFX9-NEXT:    global_store_dword v[0:1], v6, off
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; GFX9-NEXT:    v_add_i32 v6, v4, v5 clamp
+; GFX9-NEXT:    v_add_u32_e32 v4, v4, v5
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, v4, v6
+; GFX9-NEXT:    global_store_dword v[0:1], v4, off
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
 ; GFX9-NEXT:    global_store_byte v[2:3], v0, off
 ; GFX9-NEXT:    s_endpgm
   %a = load i32, i32 addrspace(1)* %aptr, align 4
@@ -495,17 +493,15 @@ define amdgpu_kernel void @v_saddo_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32>
 ; GFX9-NEXT:    v_mov_b32_e32 v6, s2
 ; GFX9-NEXT:    v_mov_b32_e32 v7, s3
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_add_u32_e32 v9, v1, v3
-; GFX9-NEXT:    v_add_u32_e32 v8, v0, v2
-; GFX9-NEXT:    v_cmp_gt_i32_e64 s[0:1], 0, v3
-; GFX9-NEXT:    v_cmp_lt_i32_e64 s[4:5], v9, v1
-; GFX9-NEXT:    s_xor_b64 s[0:1], s[0:1], s[4:5]
-; GFX9-NEXT:    v_cmp_gt_i32_e32 vcc, 0, v2
-; GFX9-NEXT:    v_cmp_lt_i32_e64 s[2:3], v8, v0
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s[0:1]
-; GFX9-NEXT:    s_xor_b64 s[0:1], vcc, s[2:3]
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
-; GFX9-NEXT:    global_store_dwordx2 v[4:5], v[8:9], off
+; GFX9-NEXT:    v_add_i32 v8, v0, v2 clamp
+; GFX9-NEXT:    v_add_u32_e32 v0, v0, v2
+; GFX9-NEXT:    v_add_i32 v2, v1, v3 clamp
+; GFX9-NEXT:    v_add_u32_e32 v1, v1, v3
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, v1, v2
+; GFX9-NEXT:    global_store_dwordx2 v[4:5], v[0:1], off
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, v0, v8
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
 ; GFX9-NEXT:    global_store_dwordx2 v[6:7], v[0:1], off
 ; GFX9-NEXT:    s_endpgm
   %a = load <2 x i32>, <2 x i32> addrspace(1)* %aptr, align 4
diff --git a/llvm/test/CodeGen/AMDGPU/saddsat.ll b/llvm/test/CodeGen/AMDGPU/saddsat.ll
new file mode 100644
index 0000000000000..73ceea2941868
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/saddsat.ll
@@ -0,0 +1,439 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=tahiti < %s | FileCheck -check-prefixes=GCN,GFX6 %s
+; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=fiji < %s | FileCheck -check-prefixes=GCN,GFX8 %s
+; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 < %s | FileCheck -check-prefixes=GCN,GFX9 %s
+
+define i8 @v_saddsat_i8(i8 %lhs, i8 %rhs) {
+; GFX6-LABEL: v_saddsat_i8:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_bfe_i32 v1, v1, 0, 8
+; GFX6-NEXT:    v_bfe_i32 v0, v0, 0, 8
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GFX6-NEXT:    v_min_i32_e32 v0, 0x7f, v0
+; GFX6-NEXT:    v_max_i32_e32 v0, 0xffffff80, v0
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_saddsat_i8:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_add_u16_sdwa v0, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-NEXT:    v_min_i16_e32 v0, 0x7f, v0
+; GFX8-NEXT:    v_max_i16_e32 v0, 0xff80, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_saddsat_i8:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
+; GFX9-NEXT:    v_add_i16 v0, v0, v1 clamp
+; GFX9-NEXT:    v_ashrrev_i16_e32 v0, 8, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i8 @llvm.sadd.sat.i8(i8 %lhs, i8 %rhs)
+  ret i8 %result
+}
+
+define i16 @v_saddsat_i16(i16 %lhs, i16 %rhs) {
+; GFX6-LABEL: v_saddsat_i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_bfe_i32 v1, v1, 0, 16
+; GFX6-NEXT:    v_bfe_i32 v0, v0, 0, 16
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GFX6-NEXT:    v_min_i32_e32 v0, 0x7fff, v0
+; GFX6-NEXT:    v_max_i32_e32 v0, 0xffff8000, v0
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_saddsat_i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_gt_i16_e32 vcc, 0, v1
+; GFX8-NEXT:    v_add_u16_e32 v1, v0, v1
+; GFX8-NEXT:    v_cmp_lt_i16_e64 s[4:5], v1, v0
+; GFX8-NEXT:    v_mov_b32_e32 v0, 0xffff8000
+; GFX8-NEXT:    v_mov_b32_e32 v2, 0x7fff
+; GFX8-NEXT:    v_cmp_gt_i16_e64 s[6:7], 0, v1
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[6:7]
+; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_saddsat_i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_add_i16 v0, v0, v1 clamp
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i16 @llvm.sadd.sat.i16(i16 %lhs, i16 %rhs)
+  ret i16 %result
+}
+
+define i32 @v_saddsat_i32(i32 %lhs, i32 %rhs) {
+; GFX6-LABEL: v_saddsat_i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_cmp_gt_i32_e32 vcc, 0, v1
+; GFX6-NEXT:    v_add_i32_e64 v1, s[4:5], v0, v1
+; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v1, v0
+; GFX6-NEXT:    v_bfrev_b32_e32 v0, 1
+; GFX6-NEXT:    v_bfrev_b32_e32 v2, -2
+; GFX6-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v1
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[6:7]
+; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_saddsat_i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_gt_i32_e32 vcc, 0, v1
+; GFX8-NEXT:    v_add_u32_e64 v1, s[4:5], v0, v1
+; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v1, v0
+; GFX8-NEXT:    v_bfrev_b32_e32 v0, 1
+; GFX8-NEXT:    v_bfrev_b32_e32 v2, -2
+; GFX8-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v1
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[6:7]
+; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_saddsat_i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_add_i32 v0, v0, v1 clamp
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i32 @llvm.sadd.sat.i32(i32 %lhs, i32 %rhs)
+  ret i32 %result
+}
+
+define <2 x i16> @v_saddsat_v2i16(<2 x i16> %lhs, <2 x i16> %rhs) {
+; GFX6-LABEL: v_saddsat_v2i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_bfe_i32 v2, v2, 0, 16
+; GFX6-NEXT:    v_bfe_i32 v0, v0, 0, 16
+; GFX6-NEXT:    v_bfe_i32 v3, v3, 0, 16
+; GFX6-NEXT:    v_bfe_i32 v1, v1, 0, 16
+; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
+; GFX6-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GFX6-NEXT:    v_min_i32_e32 v1, s4, v1
+; GFX6-NEXT:    s_movk_i32 s5, 0x8000
+; GFX6-NEXT:    v_min_i32_e32 v0, s4, v0
+; GFX6-NEXT:    v_max_i32_e32 v1, s5, v1
+; GFX6-NEXT:    v_max_i32_e32 v0, s5, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_saddsat_v2i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_lshrrev_b32_e32 v2, 16, v1
+; GFX8-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
+; GFX8-NEXT:    v_add_u16_e32 v4, v3, v2
+; GFX8-NEXT:    v_mov_b32_e32 v5, 0xffff8000
+; GFX8-NEXT:    v_mov_b32_e32 v6, 0x7fff
+; GFX8-NEXT:    v_cmp_gt_i16_e32 vcc, 0, v4
+; GFX8-NEXT:    v_cndmask_b32_e32 v7, v5, v6, vcc
+; GFX8-NEXT:    v_cmp_lt_i16_e32 vcc, v4, v3
+; GFX8-NEXT:    v_cmp_gt_i16_e64 s[4:5], 0, v2
+; GFX8-NEXT:    s_xor_b64 vcc, s[4:5], vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v4, v7, vcc
+; GFX8-NEXT:    v_cmp_gt_i16_e32 vcc, 0, v1
+; GFX8-NEXT:    v_add_u16_e32 v1, v0, v1
+; GFX8-NEXT:    v_cmp_lt_i16_e64 s[4:5], v1, v0
+; GFX8-NEXT:    v_cmp_gt_i16_e64 s[6:7], 0, v1
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v5, v6, s[6:7]
+; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_saddsat_v2i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_pk_add_i16 v0, v0, v1 clamp
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <2 x i16> @llvm.sadd.sat.v2i16(<2 x i16> %lhs, <2 x i16> %rhs)
+  ret <2 x i16> %result
+}
+
+define <3 x i16> @v_saddsat_v3i16(<3 x i16> %lhs, <3 x i16> %rhs) {
+; GFX6-LABEL: v_saddsat_v3i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_bfe_i32 v3, v3, 0, 16
+; GFX6-NEXT:    v_bfe_i32 v0, v0, 0, 16
+; GFX6-NEXT:    v_bfe_i32 v4, v4, 0, 16
+; GFX6-NEXT:    v_bfe_i32 v1, v1, 0, 16
+; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v1, v4
+; GFX6-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v3
+; GFX6-NEXT:    v_min_i32_e32 v1, s4, v1
+; GFX6-NEXT:    s_movk_i32 s5, 0x8000
+; GFX6-NEXT:    v_min_i32_e32 v0, s4, v0
+; GFX6-NEXT:    v_max_i32_e32 v1, s5, v1
+; GFX6-NEXT:    v_max_i32_e32 v0, s5, v0
+; GFX6-NEXT:    v_bfe_i32 v5, v5, 0, 16
+; GFX6-NEXT:    v_bfe_i32 v2, v2, 0, 16
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v2, v5
+; GFX6-NEXT:    v_min_i32_e32 v1, s4, v1
+; GFX6-NEXT:    v_max_i32_e32 v1, s5, v1
+; GFX6-NEXT:    v_or_b32_e32 v2, 0xffff0000, v1
+; GFX6-NEXT:    v_alignbit_b32 v1, v1, v0, 16
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_saddsat_v3i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_lshrrev_b32_e32 v4, 16, v2
+; GFX8-NEXT:    v_lshrrev_b32_e32 v5, 16, v0
+; GFX8-NEXT:    v_add_u16_e32 v6, v5, v4
+; GFX8-NEXT:    v_mov_b32_e32 v7, 0xffff8000
+; GFX8-NEXT:    v_mov_b32_e32 v8, 0x7fff
+; GFX8-NEXT:    v_cmp_gt_i16_e32 vcc, 0, v6
+; GFX8-NEXT:    v_cndmask_b32_e32 v9, v7, v8, vcc
+; GFX8-NEXT:    v_cmp_lt_i16_e32 vcc, v6, v5
+; GFX8-NEXT:    v_cmp_gt_i16_e64 s[4:5], 0, v4
+; GFX8-NEXT:    s_xor_b64 vcc, s[4:5], vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v6, v9, vcc
+; GFX8-NEXT:    v_cmp_gt_i16_e32 vcc, 0, v3
+; GFX8-NEXT:    v_add_u16_e32 v3, v1, v3
+; GFX8-NEXT:    v_cmp_lt_i16_e64 s[4:5], v3, v1
+; GFX8-NEXT:    v_cmp_gt_i16_e64 s[6:7], 0, v3
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, v7, v8, s[6:7]
+; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-NEXT:    v_cmp_gt_i16_e32 vcc, 0, v2
+; GFX8-NEXT:    v_add_u16_e32 v2, v0, v2
+; GFX8-NEXT:    v_cmp_lt_i16_e64 s[4:5], v2, v0
+; GFX8-NEXT:    v_cmp_gt_i16_e64 s[6:7], 0, v2
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v7, v8, s[6:7]
+; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v4
+; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_saddsat_v3i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_pk_add_i16 v1, v1, v3 clamp
+; GFX9-NEXT:    v_pk_add_i16 v0, v0, v2 clamp
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <3 x i16> @llvm.sadd.sat.v3i16(<3 x i16> %lhs, <3 x i16> %rhs)
+  ret <3 x i16> %result
+}
+
+define <2 x float> @v_saddsat_v4i16(<4 x i16> %lhs, <4 x i16> %rhs) {
+; GFX6-LABEL: v_saddsat_v4i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_bfe_i32 v4, v4, 0, 16
+; GFX6-NEXT:    v_bfe_i32 v0, v0, 0, 16
+; GFX6-NEXT:    v_bfe_i32 v5, v5, 0, 16
+; GFX6-NEXT:    v_bfe_i32 v1, v1, 0, 16
+; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v1, v5
+; GFX6-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
+; GFX6-NEXT:    v_min_i32_e32 v1, s4, v1
+; GFX6-NEXT:    s_movk_i32 s5, 0x8000
+; GFX6-NEXT:    v_min_i32_e32 v0, s4, v0
+; GFX6-NEXT:    v_max_i32_e32 v1, s5, v1
+; GFX6-NEXT:    v_max_i32_e32 v0, s5, v0
+; GFX6-NEXT:    s_mov_b32 s6, 0xffff
+; GFX6-NEXT:    v_bfe_i32 v6, v6, 0, 16
+; GFX6-NEXT:    v_bfe_i32 v2, v2, 0, 16
+; GFX6-NEXT:    v_bfe_i32 v7, v7, 0, 16
+; GFX6-NEXT:    v_bfe_i32 v3, v3, 0, 16
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_and_b32_e32 v0, s6, v0
+; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v6
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v3, v7
+; GFX6-NEXT:    v_min_i32_e32 v1, s4, v1
+; GFX6-NEXT:    v_min_i32_e32 v2, s4, v2
+; GFX6-NEXT:    v_max_i32_e32 v1, s5, v1
+; GFX6-NEXT:    v_max_i32_e32 v2, s5, v2
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_and_b32_e32 v2, s6, v2
+; GFX6-NEXT:    v_or_b32_e32 v1, v2, v1
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_saddsat_v4i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_lshrrev_b32_e32 v4, 16, v2
+; GFX8-NEXT:    v_lshrrev_b32_e32 v5, 16, v0
+; GFX8-NEXT:    v_add_u16_e32 v6, v5, v4
+; GFX8-NEXT:    v_mov_b32_e32 v7, 0xffff8000
+; GFX8-NEXT:    v_mov_b32_e32 v8, 0x7fff
+; GFX8-NEXT:    v_cmp_gt_i16_e32 vcc, 0, v6
+; GFX8-NEXT:    v_cndmask_b32_e32 v9, v7, v8, vcc
+; GFX8-NEXT:    v_cmp_lt_i16_e32 vcc, v6, v5
+; GFX8-NEXT:    v_cmp_gt_i16_e64 s[4:5], 0, v4
+; GFX8-NEXT:    s_xor_b64 vcc, s[4:5], vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v6, v9, vcc
+; GFX8-NEXT:    v_cmp_gt_i16_e32 vcc, 0, v2
+; GFX8-NEXT:    v_add_u16_e32 v2, v0, v2
+; GFX8-NEXT:    v_cmp_lt_i16_e64 s[4:5], v2, v0
+; GFX8-NEXT:    v_cmp_gt_i16_e64 s[6:7], 0, v2
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v7, v8, s[6:7]
+; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    v_lshrrev_b32_e32 v2, 16, v3
+; GFX8-NEXT:    v_lshrrev_b32_e32 v4, 16, v1
+; GFX8-NEXT:    v_add_u16_e32 v5, v4, v2
+; GFX8-NEXT:    v_cmp_gt_i16_e32 vcc, 0, v5
+; GFX8-NEXT:    v_cndmask_b32_e32 v6, v7, v8, vcc
+; GFX8-NEXT:    v_cmp_lt_i16_e32 vcc, v5, v4
+; GFX8-NEXT:    v_cmp_gt_i16_e64 s[4:5], 0, v2
+; GFX8-NEXT:    s_xor_b64 vcc, s[4:5], vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v5, v6, vcc
+; GFX8-NEXT:    v_cmp_gt_i16_e32 vcc, 0, v3
+; GFX8-NEXT:    v_add_u16_e32 v3, v1, v3
+; GFX8-NEXT:    v_cmp_lt_i16_e64 s[4:5], v3, v1
+; GFX8-NEXT:    v_cmp_gt_i16_e64 s[6:7], 0, v3
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, v7, v8, s[6:7]
+; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_saddsat_v4i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_pk_add_i16 v0, v0, v2 clamp
+; GFX9-NEXT:    v_pk_add_i16 v1, v1, v3 clamp
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <4 x i16> @llvm.sadd.sat.v4i16(<4 x i16> %lhs, <4 x i16> %rhs)
+  %cast = bitcast <4 x i16> %result to <2 x float>
+  ret <2 x float> %cast
+}
+
+define <2 x i32> @v_saddsat_v2i32(<2 x i32> %lhs, <2 x i32> %rhs) {
+; GFX6-LABEL: v_saddsat_v2i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_cmp_gt_i32_e32 vcc, 0, v2
+; GFX6-NEXT:    v_add_i32_e64 v2, s[4:5], v0, v2
+; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v2, v0
+; GFX6-NEXT:    v_bfrev_b32_e32 v4, 1
+; GFX6-NEXT:    v_bfrev_b32_e32 v5, -2
+; GFX6-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v2
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, v4, v5, s[6:7]
+; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX6-NEXT:    v_add_i32_e64 v2, s[4:5], v1, v3
+; GFX6-NEXT:    v_cmp_gt_i32_e32 vcc, 0, v3
+; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v2, v1
+; GFX6-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v2
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, v4, v5, s[6:7]
+; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_saddsat_v2i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_gt_i32_e32 vcc, 0, v2
+; GFX8-NEXT:    v_add_u32_e64 v2, s[4:5], v0, v2
+; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v2, v0
+; GFX8-NEXT:    v_bfrev_b32_e32 v4, 1
+; GFX8-NEXT:    v_bfrev_b32_e32 v5, -2
+; GFX8-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v2
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v4, v5, s[6:7]
+; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-NEXT:    v_add_u32_e64 v2, s[4:5], v1, v3
+; GFX8-NEXT:    v_cmp_gt_i32_e32 vcc, 0, v3
+; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v2, v1
+; GFX8-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v2
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, v4, v5, s[6:7]
+; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_saddsat_v2i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_add_i32 v0, v0, v2 clamp
+; GFX9-NEXT:    v_add_i32 v1, v1, v3 clamp
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <2 x i32> @llvm.sadd.sat.v2i32(<2 x i32> %lhs, <2 x i32> %rhs)
+  ret <2 x i32> %result
+}
+
+define i64 @v_saddsat_i64(i64 %lhs, i64 %rhs) {
+; GFX6-LABEL: v_saddsat_i64:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_add_i32_e32 v4, vcc, v0, v2
+; GFX6-NEXT:    v_addc_u32_e32 v5, vcc, v1, v3, vcc
+; GFX6-NEXT:    v_cmp_lt_i64_e32 vcc, v[4:5], v[0:1]
+; GFX6-NEXT:    v_cmp_gt_i64_e64 s[4:5], 0, v[2:3]
+; GFX6-NEXT:    v_bfrev_b32_e32 v1, 1
+; GFX6-NEXT:    s_xor_b64 vcc, s[4:5], vcc
+; GFX6-NEXT:    v_cmp_gt_i64_e64 s[4:5], 0, v[4:5]
+; GFX6-NEXT:    v_bfrev_b32_e32 v2, -2
+; GFX6-NEXT:    v_ashrrev_i32_e32 v0, 31, v5
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, v1, v2, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_saddsat_i64:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_add_u32_e32 v4, vcc, v0, v2
+; GFX8-NEXT:    v_addc_u32_e32 v5, vcc, v1, v3, vcc
+; GFX8-NEXT:    v_cmp_lt_i64_e32 vcc, v[4:5], v[0:1]
+; GFX8-NEXT:    v_cmp_gt_i64_e64 s[4:5], 0, v[2:3]
+; GFX8-NEXT:    v_bfrev_b32_e32 v1, 1
+; GFX8-NEXT:    s_xor_b64 vcc, s[4:5], vcc
+; GFX8-NEXT:    v_cmp_gt_i64_e64 s[4:5], 0, v[4:5]
+; GFX8-NEXT:    v_bfrev_b32_e32 v2, -2
+; GFX8-NEXT:    v_ashrrev_i32_e32 v0, 31, v5
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, v1, v2, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_saddsat_i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, v0, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, v1, v3, vcc
+; GFX9-NEXT:    v_cmp_lt_i64_e32 vcc, v[4:5], v[0:1]
+; GFX9-NEXT:    v_cmp_gt_i64_e64 s[4:5], 0, v[2:3]
+; GFX9-NEXT:    v_bfrev_b32_e32 v1, 1
+; GFX9-NEXT:    s_xor_b64 vcc, s[4:5], vcc
+; GFX9-NEXT:    v_cmp_gt_i64_e64 s[4:5], 0, v[4:5]
+; GFX9-NEXT:    v_bfrev_b32_e32 v2, -2
+; GFX9-NEXT:    v_ashrrev_i32_e32 v0, 31, v5
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, v2, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i64 @llvm.sadd.sat.i64(i64 %lhs, i64 %rhs)
+  ret i64 %result
+}
+
+declare i8 @llvm.sadd.sat.i8(i8, i8) #0
+declare i16 @llvm.sadd.sat.i16(i16, i16) #0
+declare <2 x i16> @llvm.sadd.sat.v2i16(<2 x i16>, <2 x i16>) #0
+declare <3 x i16> @llvm.sadd.sat.v3i16(<3 x i16>, <3 x i16>) #0
+declare <4 x i16> @llvm.sadd.sat.v4i16(<4 x i16>, <4 x i16>) #0
+declare i32 @llvm.sadd.sat.i32(i32, i32) #0
+declare <2 x i32> @llvm.sadd.sat.v2i32(<2 x i32>, <2 x i32>) #0
+declare i64 @llvm.sadd.sat.i64(i64, i64) #0
diff --git a/llvm/test/CodeGen/AMDGPU/ssubsat.ll b/llvm/test/CodeGen/AMDGPU/ssubsat.ll
new file mode 100644
index 0000000000000..f6048f6a357c4
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/ssubsat.ll
@@ -0,0 +1,1004 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=tahiti < %s | FileCheck -check-prefixes=GCN,GFX6 %s
+; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=fiji < %s | FileCheck -check-prefixes=GCN,GFX8 %s
+; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 < %s | FileCheck -check-prefixes=GCN,GFX9 %s
+
+define i8 @v_ssubsat_i8(i8 %lhs, i8 %rhs) {
+; GFX6-LABEL: v_ssubsat_i8:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_bfe_i32 v1, v1, 0, 8
+; GFX6-NEXT:    v_bfe_i32 v0, v0, 0, 8
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, v0, v1
+; GFX6-NEXT:    v_min_i32_e32 v0, 0x7f, v0
+; GFX6-NEXT:    v_max_i32_e32 v0, 0xffffff80, v0
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_ssubsat_i8:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_sub_u16_sdwa v0, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-NEXT:    v_min_i16_e32 v0, 0x7f, v0
+; GFX8-NEXT:    v_max_i16_e32 v0, 0xff80, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_ssubsat_i8:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
+; GFX9-NEXT:    v_sub_i16 v0, v0, v1 clamp
+; GFX9-NEXT:    v_ashrrev_i16_e32 v0, 8, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i8 @llvm.ssub.sat.i8(i8 %lhs, i8 %rhs)
+  ret i8 %result
+}
+
+define i16 @v_ssubsat_i16(i16 %lhs, i16 %rhs) {
+; GFX6-LABEL: v_ssubsat_i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_bfe_i32 v1, v1, 0, 16
+; GFX6-NEXT:    v_bfe_i32 v0, v0, 0, 16
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, v0, v1
+; GFX6-NEXT:    v_min_i32_e32 v0, 0x7fff, v0
+; GFX6-NEXT:    v_max_i32_e32 v0, 0xffff8000, v0
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_ssubsat_i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_lt_i16_e32 vcc, 0, v1
+; GFX8-NEXT:    v_sub_u16_e32 v1, v0, v1
+; GFX8-NEXT:    v_cmp_lt_i16_e64 s[4:5], v1, v0
+; GFX8-NEXT:    v_mov_b32_e32 v0, 0xffff8000
+; GFX8-NEXT:    v_mov_b32_e32 v2, 0x7fff
+; GFX8-NEXT:    v_cmp_gt_i16_e64 s[6:7], 0, v1
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[6:7]
+; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_ssubsat_i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_sub_i16 v0, v0, v1 clamp
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i16 @llvm.ssub.sat.i16(i16 %lhs, i16 %rhs)
+  ret i16 %result
+}
+
+define i32 @v_ssubsat_i32(i32 %lhs, i32 %rhs) {
+; GFX6-LABEL: v_ssubsat_i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v1
+; GFX6-NEXT:    v_sub_i32_e64 v1, s[4:5], v0, v1
+; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v1, v0
+; GFX6-NEXT:    v_bfrev_b32_e32 v0, 1
+; GFX6-NEXT:    v_bfrev_b32_e32 v2, -2
+; GFX6-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v1
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[6:7]
+; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_ssubsat_i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v1
+; GFX8-NEXT:    v_sub_u32_e64 v1, s[4:5], v0, v1
+; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v1, v0
+; GFX8-NEXT:    v_bfrev_b32_e32 v0, 1
+; GFX8-NEXT:    v_bfrev_b32_e32 v2, -2
+; GFX8-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v1
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[6:7]
+; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_ssubsat_i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_sub_i32 v0, v0, v1 clamp
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i32 @llvm.ssub.sat.i32(i32 %lhs, i32 %rhs)
+  ret i32 %result
+}
+
+define <2 x i16> @v_ssubsat_v2i16(<2 x i16> %lhs, <2 x i16> %rhs) {
+; GFX6-LABEL: v_ssubsat_v2i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_bfe_i32 v2, v2, 0, 16
+; GFX6-NEXT:    v_bfe_i32 v0, v0, 0, 16
+; GFX6-NEXT:    v_bfe_i32 v3, v3, 0, 16
+; GFX6-NEXT:    v_bfe_i32 v1, v1, 0, 16
+; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, v1, v3
+; GFX6-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, v0, v2
+; GFX6-NEXT:    v_min_i32_e32 v1, s4, v1
+; GFX6-NEXT:    s_movk_i32 s5, 0x8000
+; GFX6-NEXT:    v_min_i32_e32 v0, s4, v0
+; GFX6-NEXT:    v_max_i32_e32 v1, s5, v1
+; GFX6-NEXT:    v_max_i32_e32 v0, s5, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_ssubsat_v2i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_lshrrev_b32_e32 v2, 16, v1
+; GFX8-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
+; GFX8-NEXT:    v_sub_u16_e32 v4, v3, v2
+; GFX8-NEXT:    v_mov_b32_e32 v5, 0xffff8000
+; GFX8-NEXT:    v_mov_b32_e32 v6, 0x7fff
+; GFX8-NEXT:    v_cmp_gt_i16_e32 vcc, 0, v4
+; GFX8-NEXT:    v_cndmask_b32_e32 v7, v5, v6, vcc
+; GFX8-NEXT:    v_cmp_lt_i16_e32 vcc, v4, v3
+; GFX8-NEXT:    v_cmp_lt_i16_e64 s[4:5], 0, v2
+; GFX8-NEXT:    s_xor_b64 vcc, s[4:5], vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v4, v7, vcc
+; GFX8-NEXT:    v_cmp_lt_i16_e32 vcc, 0, v1
+; GFX8-NEXT:    v_sub_u16_e32 v1, v0, v1
+; GFX8-NEXT:    v_cmp_lt_i16_e64 s[4:5], v1, v0
+; GFX8-NEXT:    v_cmp_gt_i16_e64 s[6:7], 0, v1
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v5, v6, s[6:7]
+; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_ssubsat_v2i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_pk_sub_i16 v0, v0, v1 clamp
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <2 x i16> @llvm.ssub.sat.v2i16(<2 x i16> %lhs, <2 x i16> %rhs)
+  ret <2 x i16> %result
+}
+
+define <3 x i16> @v_ssubsat_v3i16(<3 x i16> %lhs, <3 x i16> %rhs) {
+; GFX6-LABEL: v_ssubsat_v3i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_bfe_i32 v3, v3, 0, 16
+; GFX6-NEXT:    v_bfe_i32 v0, v0, 0, 16
+; GFX6-NEXT:    v_bfe_i32 v4, v4, 0, 16
+; GFX6-NEXT:    v_bfe_i32 v1, v1, 0, 16
+; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, v1, v4
+; GFX6-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, v0, v3
+; GFX6-NEXT:    v_min_i32_e32 v1, s4, v1
+; GFX6-NEXT:    s_movk_i32 s5, 0x8000
+; GFX6-NEXT:    v_min_i32_e32 v0, s4, v0
+; GFX6-NEXT:    v_max_i32_e32 v1, s5, v1
+; GFX6-NEXT:    v_max_i32_e32 v0, s5, v0
+; GFX6-NEXT:    s_mov_b32 s6, 0xffff
+; GFX6-NEXT:    v_bfe_i32 v5, v5, 0, 16
+; GFX6-NEXT:    v_bfe_i32 v2, v2, 0, 16
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_and_b32_e32 v0, s6, v0
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, v2, v5
+; GFX6-NEXT:    v_min_i32_e32 v1, s4, v1
+; GFX6-NEXT:    v_max_i32_e32 v1, s5, v1
+; GFX6-NEXT:    v_and_b32_e32 v2, s6, v1
+; GFX6-NEXT:    v_alignbit_b32 v1, v1, v0, 16
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_ssubsat_v3i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_lshrrev_b32_e32 v4, 16, v2
+; GFX8-NEXT:    v_lshrrev_b32_e32 v5, 16, v0
+; GFX8-NEXT:    v_sub_u16_e32 v6, v5, v4
+; GFX8-NEXT:    v_mov_b32_e32 v7, 0xffff8000
+; GFX8-NEXT:    v_mov_b32_e32 v8, 0x7fff
+; GFX8-NEXT:    v_cmp_gt_i16_e32 vcc, 0, v6
+; GFX8-NEXT:    v_cndmask_b32_e32 v9, v7, v8, vcc
+; GFX8-NEXT:    v_cmp_lt_i16_e32 vcc, v6, v5
+; GFX8-NEXT:    v_cmp_lt_i16_e64 s[4:5], 0, v4
+; GFX8-NEXT:    s_xor_b64 vcc, s[4:5], vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v6, v9, vcc
+; GFX8-NEXT:    v_cmp_lt_i16_e32 vcc, 0, v3
+; GFX8-NEXT:    v_sub_u16_e32 v3, v1, v3
+; GFX8-NEXT:    v_cmp_lt_i16_e64 s[4:5], v3, v1
+; GFX8-NEXT:    v_cmp_gt_i16_e64 s[6:7], 0, v3
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, v7, v8, s[6:7]
+; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-NEXT:    v_cmp_lt_i16_e32 vcc, 0, v2
+; GFX8-NEXT:    v_sub_u16_e32 v2, v0, v2
+; GFX8-NEXT:    v_cmp_lt_i16_e64 s[4:5], v2, v0
+; GFX8-NEXT:    v_cmp_gt_i16_e64 s[6:7], 0, v2
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v7, v8, s[6:7]
+; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v4
+; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_ssubsat_v3i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_pk_sub_i16 v0, v0, v2 clamp
+; GFX9-NEXT:    v_pk_sub_i16 v1, v1, v3 clamp
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <3 x i16> @llvm.ssub.sat.v3i16(<3 x i16> %lhs, <3 x i16> %rhs)
+  ret <3 x i16> %result
+}
+
+define <2 x float> @v_ssubsat_v4i16(<4 x i16> %lhs, <4 x i16> %rhs) {
+; GFX6-LABEL: v_ssubsat_v4i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_bfe_i32 v4, v4, 0, 16
+; GFX6-NEXT:    v_bfe_i32 v0, v0, 0, 16
+; GFX6-NEXT:    v_bfe_i32 v5, v5, 0, 16
+; GFX6-NEXT:    v_bfe_i32 v1, v1, 0, 16
+; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, v1, v5
+; GFX6-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, v0, v4
+; GFX6-NEXT:    v_min_i32_e32 v1, s4, v1
+; GFX6-NEXT:    s_movk_i32 s5, 0x8000
+; GFX6-NEXT:    v_min_i32_e32 v0, s4, v0
+; GFX6-NEXT:    v_max_i32_e32 v1, s5, v1
+; GFX6-NEXT:    v_max_i32_e32 v0, s5, v0
+; GFX6-NEXT:    s_mov_b32 s6, 0xffff
+; GFX6-NEXT:    v_bfe_i32 v6, v6, 0, 16
+; GFX6-NEXT:    v_bfe_i32 v2, v2, 0, 16
+; GFX6-NEXT:    v_bfe_i32 v7, v7, 0, 16
+; GFX6-NEXT:    v_bfe_i32 v3, v3, 0, 16
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_and_b32_e32 v0, s6, v0
+; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, v2, v6
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, v3, v7
+; GFX6-NEXT:    v_min_i32_e32 v1, s4, v1
+; GFX6-NEXT:    v_min_i32_e32 v2, s4, v2
+; GFX6-NEXT:    v_max_i32_e32 v1, s5, v1
+; GFX6-NEXT:    v_max_i32_e32 v2, s5, v2
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_and_b32_e32 v2, s6, v2
+; GFX6-NEXT:    v_or_b32_e32 v1, v2, v1
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_ssubsat_v4i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_lshrrev_b32_e32 v4, 16, v2
+; GFX8-NEXT:    v_lshrrev_b32_e32 v5, 16, v0
+; GFX8-NEXT:    v_sub_u16_e32 v6, v5, v4
+; GFX8-NEXT:    v_mov_b32_e32 v7, 0xffff8000
+; GFX8-NEXT:    v_mov_b32_e32 v8, 0x7fff
+; GFX8-NEXT:    v_cmp_gt_i16_e32 vcc, 0, v6
+; GFX8-NEXT:    v_cndmask_b32_e32 v9, v7, v8, vcc
+; GFX8-NEXT:    v_cmp_lt_i16_e32 vcc, v6, v5
+; GFX8-NEXT:    v_cmp_lt_i16_e64 s[4:5], 0, v4
+; GFX8-NEXT:    s_xor_b64 vcc, s[4:5], vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v6, v9, vcc
+; GFX8-NEXT:    v_cmp_lt_i16_e32 vcc, 0, v2
+; GFX8-NEXT:    v_sub_u16_e32 v2, v0, v2
+; GFX8-NEXT:    v_cmp_lt_i16_e64 s[4:5], v2, v0
+; GFX8-NEXT:    v_cmp_gt_i16_e64 s[6:7], 0, v2
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v7, v8, s[6:7]
+; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    v_lshrrev_b32_e32 v2, 16, v3
+; GFX8-NEXT:    v_lshrrev_b32_e32 v4, 16, v1
+; GFX8-NEXT:    v_sub_u16_e32 v5, v4, v2
+; GFX8-NEXT:    v_cmp_gt_i16_e32 vcc, 0, v5
+; GFX8-NEXT:    v_cndmask_b32_e32 v6, v7, v8, vcc
+; GFX8-NEXT:    v_cmp_lt_i16_e32 vcc, v5, v4
+; GFX8-NEXT:    v_cmp_lt_i16_e64 s[4:5], 0, v2
+; GFX8-NEXT:    s_xor_b64 vcc, s[4:5], vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v5, v6, vcc
+; GFX8-NEXT:    v_cmp_lt_i16_e32 vcc, 0, v3
+; GFX8-NEXT:    v_sub_u16_e32 v3, v1, v3
+; GFX8-NEXT:    v_cmp_lt_i16_e64 s[4:5], v3, v1
+; GFX8-NEXT:    v_cmp_gt_i16_e64 s[6:7], 0, v3
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, v7, v8, s[6:7]
+; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_ssubsat_v4i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_pk_sub_i16 v0, v0, v2 clamp
+; GFX9-NEXT:    v_pk_sub_i16 v1, v1, v3 clamp
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <4 x i16> @llvm.ssub.sat.v4i16(<4 x i16> %lhs, <4 x i16> %rhs)
+  %cast = bitcast <4 x i16> %result to <2 x float>
+  ret <2 x float> %cast
+}
+
+define <2 x i32> @v_ssubsat_v2i32(<2 x i32> %lhs, <2 x i32> %rhs) {
+; GFX6-LABEL: v_ssubsat_v2i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v2
+; GFX6-NEXT:    v_sub_i32_e64 v2, s[4:5], v0, v2
+; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v2, v0
+; GFX6-NEXT:    v_bfrev_b32_e32 v4, 1
+; GFX6-NEXT:    v_bfrev_b32_e32 v5, -2
+; GFX6-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v2
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, v4, v5, s[6:7]
+; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX6-NEXT:    v_sub_i32_e64 v2, s[4:5], v1, v3
+; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v3
+; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v2, v1
+; GFX6-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v2
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, v4, v5, s[6:7]
+; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_ssubsat_v2i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v2
+; GFX8-NEXT:    v_sub_u32_e64 v2, s[4:5], v0, v2
+; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v2, v0
+; GFX8-NEXT:    v_bfrev_b32_e32 v4, 1
+; GFX8-NEXT:    v_bfrev_b32_e32 v5, -2
+; GFX8-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v2
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v4, v5, s[6:7]
+; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-NEXT:    v_sub_u32_e64 v2, s[4:5], v1, v3
+; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v3
+; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v2, v1
+; GFX8-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v2
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, v4, v5, s[6:7]
+; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_ssubsat_v2i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_sub_i32 v0, v0, v2 clamp
+; GFX9-NEXT:    v_sub_i32 v1, v1, v3 clamp
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <2 x i32> @llvm.ssub.sat.v2i32(<2 x i32> %lhs, <2 x i32> %rhs)
+  ret <2 x i32> %result
+}
+
+define <3 x i32> @v_ssubsat_v3i32(<3 x i32> %lhs, <3 x i32> %rhs) {
+; GFX6-LABEL: v_ssubsat_v3i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v3
+; GFX6-NEXT:    v_sub_i32_e64 v3, s[4:5], v0, v3
+; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v3, v0
+; GFX6-NEXT:    v_bfrev_b32_e32 v6, 1
+; GFX6-NEXT:    v_bfrev_b32_e32 v7, -2
+; GFX6-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v3
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, v6, v7, s[6:7]
+; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX6-NEXT:    v_sub_i32_e64 v3, s[4:5], v1, v4
+; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v4
+; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v3, v1
+; GFX6-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v3
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, v6, v7, s[6:7]
+; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX6-NEXT:    v_sub_i32_e64 v3, s[4:5], v2, v5
+; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v5
+; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v3, v2
+; GFX6-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v3
+; GFX6-NEXT:    v_cndmask_b32_e64 v2, v6, v7, s[6:7]
+; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_ssubsat_v3i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v3
+; GFX8-NEXT:    v_sub_u32_e64 v3, s[4:5], v0, v3
+; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v3, v0
+; GFX8-NEXT:    v_bfrev_b32_e32 v6, 1
+; GFX8-NEXT:    v_bfrev_b32_e32 v7, -2
+; GFX8-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v3
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v6, v7, s[6:7]
+; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX8-NEXT:    v_sub_u32_e64 v3, s[4:5], v1, v4
+; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v4
+; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v3, v1
+; GFX8-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v3
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, v6, v7, s[6:7]
+; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-NEXT:    v_sub_u32_e64 v3, s[4:5], v2, v5
+; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v5
+; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v3, v2
+; GFX8-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v3
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, v6, v7, s[6:7]
+; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_ssubsat_v3i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_sub_i32 v0, v0, v3 clamp
+; GFX9-NEXT:    v_sub_i32 v1, v1, v4 clamp
+; GFX9-NEXT:    v_sub_i32 v2, v2, v5 clamp
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <3 x i32> @llvm.ssub.sat.v3i32(<3 x i32> %lhs, <3 x i32> %rhs)
+  ret <3 x i32> %result
+}
+
+define <4 x i32> @v_ssubsat_v4i32(<4 x i32> %lhs, <4 x i32> %rhs) {
+; GFX6-LABEL: v_ssubsat_v4i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v4
+; GFX6-NEXT:    v_sub_i32_e64 v4, s[4:5], v0, v4
+; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v4, v0
+; GFX6-NEXT:    v_bfrev_b32_e32 v8, 1
+; GFX6-NEXT:    v_bfrev_b32_e32 v9, -2
+; GFX6-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v4
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, v8, v9, s[6:7]
+; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX6-NEXT:    v_sub_i32_e64 v4, s[4:5], v1, v5
+; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v5
+; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v4, v1
+; GFX6-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v4
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, v8, v9, s[6:7]
+; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
+; GFX6-NEXT:    v_sub_i32_e64 v4, s[4:5], v2, v6
+; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v6
+; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v4, v2
+; GFX6-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v4
+; GFX6-NEXT:    v_cndmask_b32_e64 v2, v8, v9, s[6:7]
+; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
+; GFX6-NEXT:    v_sub_i32_e64 v4, s[4:5], v3, v7
+; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v7
+; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v4, v3
+; GFX6-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v4
+; GFX6-NEXT:    v_cndmask_b32_e64 v3, v8, v9, s[6:7]
+; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_ssubsat_v4i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v4
+; GFX8-NEXT:    v_sub_u32_e64 v4, s[4:5], v0, v4
+; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v4, v0
+; GFX8-NEXT:    v_bfrev_b32_e32 v8, 1
+; GFX8-NEXT:    v_bfrev_b32_e32 v9, -2
+; GFX8-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v4
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v8, v9, s[6:7]
+; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX8-NEXT:    v_sub_u32_e64 v4, s[4:5], v1, v5
+; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v5
+; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v4, v1
+; GFX8-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v4
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, v8, v9, s[6:7]
+; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
+; GFX8-NEXT:    v_sub_u32_e64 v4, s[4:5], v2, v6
+; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v6
+; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v4, v2
+; GFX8-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v4
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, v8, v9, s[6:7]
+; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
+; GFX8-NEXT:    v_sub_u32_e64 v4, s[4:5], v3, v7
+; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v7
+; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v4, v3
+; GFX8-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v4
+; GFX8-NEXT:    v_cndmask_b32_e64 v3, v8, v9, s[6:7]
+; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_ssubsat_v4i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_sub_i32 v0, v0, v4 clamp
+; GFX9-NEXT:    v_sub_i32 v1, v1, v5 clamp
+; GFX9-NEXT:    v_sub_i32 v2, v2, v6 clamp
+; GFX9-NEXT:    v_sub_i32 v3, v3, v7 clamp
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> %lhs, <4 x i32> %rhs)
+  ret <4 x i32> %result
+}
+
+define <8 x i32> @v_ssubsat_v8i32(<8 x i32> %lhs, <8 x i32> %rhs) {
+; GFX6-LABEL: v_ssubsat_v8i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v8
+; GFX6-NEXT:    v_sub_i32_e64 v8, s[4:5], v0, v8
+; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v8, v0
+; GFX6-NEXT:    v_bfrev_b32_e32 v16, 1
+; GFX6-NEXT:    v_bfrev_b32_e32 v17, -2
+; GFX6-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v8
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, v16, v17, s[6:7]
+; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
+; GFX6-NEXT:    v_sub_i32_e64 v8, s[4:5], v1, v9
+; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v9
+; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v8, v1
+; GFX6-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v8
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, v16, v17, s[6:7]
+; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc
+; GFX6-NEXT:    v_sub_i32_e64 v8, s[4:5], v2, v10
+; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v10
+; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v8, v2
+; GFX6-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v8
+; GFX6-NEXT:    v_cndmask_b32_e64 v2, v16, v17, s[6:7]
+; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc
+; GFX6-NEXT:    v_sub_i32_e64 v8, s[4:5], v3, v11
+; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v11
+; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v8, v3
+; GFX6-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v8
+; GFX6-NEXT:    v_cndmask_b32_e64 v3, v16, v17, s[6:7]
+; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e32 v3, v8, v3, vcc
+; GFX6-NEXT:    v_sub_i32_e64 v8, s[4:5], v4, v12
+; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v12
+; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v8, v4
+; GFX6-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v8
+; GFX6-NEXT:    v_cndmask_b32_e64 v4, v16, v17, s[6:7]
+; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e32 v4, v8, v4, vcc
+; GFX6-NEXT:    v_sub_i32_e64 v8, s[4:5], v5, v13
+; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v13
+; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v8, v5
+; GFX6-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v8
+; GFX6-NEXT:    v_cndmask_b32_e64 v5, v16, v17, s[6:7]
+; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e32 v5, v8, v5, vcc
+; GFX6-NEXT:    v_sub_i32_e64 v8, s[4:5], v6, v14
+; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v14
+; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v8, v6
+; GFX6-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v8
+; GFX6-NEXT:    v_cndmask_b32_e64 v6, v16, v17, s[6:7]
+; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e32 v6, v8, v6, vcc
+; GFX6-NEXT:    v_sub_i32_e64 v8, s[4:5], v7, v15
+; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v15
+; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v8, v7
+; GFX6-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v8
+; GFX6-NEXT:    v_cndmask_b32_e64 v7, v16, v17, s[6:7]
+; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e32 v7, v8, v7, vcc
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_ssubsat_v8i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v8
+; GFX8-NEXT:    v_sub_u32_e64 v8, s[4:5], v0, v8
+; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v8, v0
+; GFX8-NEXT:    v_bfrev_b32_e32 v16, 1
+; GFX8-NEXT:    v_bfrev_b32_e32 v17, -2
+; GFX8-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v8
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v16, v17, s[6:7]
+; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
+; GFX8-NEXT:    v_sub_u32_e64 v8, s[4:5], v1, v9
+; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v9
+; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v8, v1
+; GFX8-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v8
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, v16, v17, s[6:7]
+; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc
+; GFX8-NEXT:    v_sub_u32_e64 v8, s[4:5], v2, v10
+; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v10
+; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v8, v2
+; GFX8-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v8
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, v16, v17, s[6:7]
+; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc
+; GFX8-NEXT:    v_sub_u32_e64 v8, s[4:5], v3, v11
+; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v11
+; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v8, v3
+; GFX8-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v8
+; GFX8-NEXT:    v_cndmask_b32_e64 v3, v16, v17, s[6:7]
+; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v8, v3, vcc
+; GFX8-NEXT:    v_sub_u32_e64 v8, s[4:5], v4, v12
+; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v12
+; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v8, v4
+; GFX8-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v8
+; GFX8-NEXT:    v_cndmask_b32_e64 v4, v16, v17, s[6:7]
+; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v8, v4, vcc
+; GFX8-NEXT:    v_sub_u32_e64 v8, s[4:5], v5, v13
+; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v13
+; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v8, v5
+; GFX8-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v8
+; GFX8-NEXT:    v_cndmask_b32_e64 v5, v16, v17, s[6:7]
+; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v5, v8, v5, vcc
+; GFX8-NEXT:    v_sub_u32_e64 v8, s[4:5], v6, v14
+; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v14
+; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v8, v6
+; GFX8-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v8
+; GFX8-NEXT:    v_cndmask_b32_e64 v6, v16, v17, s[6:7]
+; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v6, v8, v6, vcc
+; GFX8-NEXT:    v_sub_u32_e64 v8, s[4:5], v7, v15
+; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v15
+; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v8, v7
+; GFX8-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v8
+; GFX8-NEXT:    v_cndmask_b32_e64 v7, v16, v17, s[6:7]
+; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v7, v8, v7, vcc
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_ssubsat_v8i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_sub_i32 v0, v0, v8 clamp
+; GFX9-NEXT:    v_sub_i32 v1, v1, v9 clamp
+; GFX9-NEXT:    v_sub_i32 v2, v2, v10 clamp
+; GFX9-NEXT:    v_sub_i32 v3, v3, v11 clamp
+; GFX9-NEXT:    v_sub_i32 v4, v4, v12 clamp
+; GFX9-NEXT:    v_sub_i32 v5, v5, v13 clamp
+; GFX9-NEXT:    v_sub_i32 v6, v6, v14 clamp
+; GFX9-NEXT:    v_sub_i32 v7, v7, v15 clamp
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <8 x i32> @llvm.ssub.sat.v8i32(<8 x i32> %lhs, <8 x i32> %rhs)
+  ret <8 x i32> %result
+}
+
+define <16 x i32> @v_ssubsat_v16i32(<16 x i32> %lhs, <16 x i32> %rhs) {
+; GFX6-LABEL: v_ssubsat_v16i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v16
+; GFX6-NEXT:    v_sub_i32_e64 v16, s[4:5], v0, v16
+; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v16, v0
+; GFX6-NEXT:    v_bfrev_b32_e32 v32, 1
+; GFX6-NEXT:    v_bfrev_b32_e32 v33, -2
+; GFX6-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v16
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, v32, v33, s[6:7]
+; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v16, v0, vcc
+; GFX6-NEXT:    v_sub_i32_e64 v16, s[4:5], v1, v17
+; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v17
+; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v16, v1
+; GFX6-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v16
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, v32, v33, s[6:7]
+; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v16, v1, vcc
+; GFX6-NEXT:    v_sub_i32_e64 v16, s[4:5], v2, v18
+; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v18
+; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v16, v2
+; GFX6-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v16
+; GFX6-NEXT:    v_cndmask_b32_e64 v2, v32, v33, s[6:7]
+; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e32 v2, v16, v2, vcc
+; GFX6-NEXT:    v_sub_i32_e64 v16, s[4:5], v3, v19
+; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v19
+; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v16, v3
+; GFX6-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v16
+; GFX6-NEXT:    v_cndmask_b32_e64 v3, v32, v33, s[6:7]
+; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e32 v3, v16, v3, vcc
+; GFX6-NEXT:    v_sub_i32_e64 v16, s[4:5], v4, v20
+; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v20
+; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v16, v4
+; GFX6-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v16
+; GFX6-NEXT:    v_cndmask_b32_e64 v4, v32, v33, s[6:7]
+; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e32 v4, v16, v4, vcc
+; GFX6-NEXT:    v_sub_i32_e64 v16, s[4:5], v5, v21
+; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v21
+; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v16, v5
+; GFX6-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v16
+; GFX6-NEXT:    v_cndmask_b32_e64 v5, v32, v33, s[6:7]
+; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e32 v5, v16, v5, vcc
+; GFX6-NEXT:    v_sub_i32_e64 v16, s[4:5], v6, v22
+; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v22
+; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v16, v6
+; GFX6-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v16
+; GFX6-NEXT:    v_cndmask_b32_e64 v6, v32, v33, s[6:7]
+; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e32 v6, v16, v6, vcc
+; GFX6-NEXT:    v_sub_i32_e64 v16, s[4:5], v7, v23
+; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v23
+; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v16, v7
+; GFX6-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v16
+; GFX6-NEXT:    v_cndmask_b32_e64 v7, v32, v33, s[6:7]
+; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e32 v7, v16, v7, vcc
+; GFX6-NEXT:    v_sub_i32_e64 v16, s[4:5], v8, v24
+; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v24
+; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v16, v8
+; GFX6-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v16
+; GFX6-NEXT:    v_cndmask_b32_e64 v8, v32, v33, s[6:7]
+; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e32 v8, v16, v8, vcc
+; GFX6-NEXT:    v_sub_i32_e64 v16, s[4:5], v9, v25
+; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v25
+; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v16, v9
+; GFX6-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v16
+; GFX6-NEXT:    v_cndmask_b32_e64 v9, v32, v33, s[6:7]
+; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e32 v9, v16, v9, vcc
+; GFX6-NEXT:    v_sub_i32_e64 v16, s[4:5], v10, v26
+; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v26
+; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v16, v10
+; GFX6-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v16
+; GFX6-NEXT:    v_cndmask_b32_e64 v10, v32, v33, s[6:7]
+; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e32 v10, v16, v10, vcc
+; GFX6-NEXT:    v_sub_i32_e64 v16, s[4:5], v11, v27
+; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v27
+; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v16, v11
+; GFX6-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v16
+; GFX6-NEXT:    v_cndmask_b32_e64 v11, v32, v33, s[6:7]
+; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e32 v11, v16, v11, vcc
+; GFX6-NEXT:    v_sub_i32_e64 v16, s[4:5], v12, v28
+; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v28
+; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v16, v12
+; GFX6-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v16
+; GFX6-NEXT:    v_cndmask_b32_e64 v12, v32, v33, s[6:7]
+; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e32 v12, v16, v12, vcc
+; GFX6-NEXT:    v_sub_i32_e64 v16, s[4:5], v13, v29
+; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v29
+; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v16, v13
+; GFX6-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v16
+; GFX6-NEXT:    v_cndmask_b32_e64 v13, v32, v33, s[6:7]
+; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e32 v13, v16, v13, vcc
+; GFX6-NEXT:    v_sub_i32_e64 v16, s[4:5], v14, v30
+; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v30
+; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v16, v14
+; GFX6-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v16
+; GFX6-NEXT:    v_cndmask_b32_e64 v14, v32, v33, s[6:7]
+; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e32 v14, v16, v14, vcc
+; GFX6-NEXT:    v_sub_i32_e64 v16, s[4:5], v15, v31
+; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v31
+; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v16, v15
+; GFX6-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v16
+; GFX6-NEXT:    v_cndmask_b32_e64 v15, v32, v33, s[6:7]
+; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e32 v15, v16, v15, vcc
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_ssubsat_v16i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v16
+; GFX8-NEXT:    v_sub_u32_e64 v16, s[4:5], v0, v16
+; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v16, v0
+; GFX8-NEXT:    v_bfrev_b32_e32 v32, 1
+; GFX8-NEXT:    v_bfrev_b32_e32 v33, -2
+; GFX8-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v16
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v32, v33, s[6:7]
+; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v16, v0, vcc
+; GFX8-NEXT:    v_sub_u32_e64 v16, s[4:5], v1, v17
+; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v17
+; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v16, v1
+; GFX8-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v16
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, v32, v33, s[6:7]
+; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v16, v1, vcc
+; GFX8-NEXT:    v_sub_u32_e64 v16, s[4:5], v2, v18
+; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v18
+; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v16, v2
+; GFX8-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v16
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, v32, v33, s[6:7]
+; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v16, v2, vcc
+; GFX8-NEXT:    v_sub_u32_e64 v16, s[4:5], v3, v19
+; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v19
+; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v16, v3
+; GFX8-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v16
+; GFX8-NEXT:    v_cndmask_b32_e64 v3, v32, v33, s[6:7]
+; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v16, v3, vcc
+; GFX8-NEXT:    v_sub_u32_e64 v16, s[4:5], v4, v20
+; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v20
+; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v16, v4
+; GFX8-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v16
+; GFX8-NEXT:    v_cndmask_b32_e64 v4, v32, v33, s[6:7]
+; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v16, v4, vcc
+; GFX8-NEXT:    v_sub_u32_e64 v16, s[4:5], v5, v21
+; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v21
+; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v16, v5
+; GFX8-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v16
+; GFX8-NEXT:    v_cndmask_b32_e64 v5, v32, v33, s[6:7]
+; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v5, v16, v5, vcc
+; GFX8-NEXT:    v_sub_u32_e64 v16, s[4:5], v6, v22
+; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v22
+; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v16, v6
+; GFX8-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v16
+; GFX8-NEXT:    v_cndmask_b32_e64 v6, v32, v33, s[6:7]
+; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v6, v16, v6, vcc
+; GFX8-NEXT:    v_sub_u32_e64 v16, s[4:5], v7, v23
+; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v23
+; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v16, v7
+; GFX8-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v16
+; GFX8-NEXT:    v_cndmask_b32_e64 v7, v32, v33, s[6:7]
+; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v7, v16, v7, vcc
+; GFX8-NEXT:    v_sub_u32_e64 v16, s[4:5], v8, v24
+; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v24
+; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v16, v8
+; GFX8-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v16
+; GFX8-NEXT:    v_cndmask_b32_e64 v8, v32, v33, s[6:7]
+; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v8, v16, v8, vcc
+; GFX8-NEXT:    v_sub_u32_e64 v16, s[4:5], v9, v25
+; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v25
+; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v16, v9
+; GFX8-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v16
+; GFX8-NEXT:    v_cndmask_b32_e64 v9, v32, v33, s[6:7]
+; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v9, v16, v9, vcc
+; GFX8-NEXT:    v_sub_u32_e64 v16, s[4:5], v10, v26
+; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v26
+; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v16, v10
+; GFX8-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v16
+; GFX8-NEXT:    v_cndmask_b32_e64 v10, v32, v33, s[6:7]
+; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v10, v16, v10, vcc
+; GFX8-NEXT:    v_sub_u32_e64 v16, s[4:5], v11, v27
+; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v27
+; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v16, v11
+; GFX8-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v16
+; GFX8-NEXT:    v_cndmask_b32_e64 v11, v32, v33, s[6:7]
+; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v11, v16, v11, vcc
+; GFX8-NEXT:    v_sub_u32_e64 v16, s[4:5], v12, v28
+; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v28
+; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v16, v12
+; GFX8-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v16
+; GFX8-NEXT:    v_cndmask_b32_e64 v12, v32, v33, s[6:7]
+; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v12, v16, v12, vcc
+; GFX8-NEXT:    v_sub_u32_e64 v16, s[4:5], v13, v29
+; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v29
+; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v16, v13
+; GFX8-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v16
+; GFX8-NEXT:    v_cndmask_b32_e64 v13, v32, v33, s[6:7]
+; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v13, v16, v13, vcc
+; GFX8-NEXT:    v_sub_u32_e64 v16, s[4:5], v14, v30
+; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v30
+; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v16, v14
+; GFX8-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v16
+; GFX8-NEXT:    v_cndmask_b32_e64 v14, v32, v33, s[6:7]
+; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v14, v16, v14, vcc
+; GFX8-NEXT:    v_sub_u32_e64 v16, s[4:5], v15, v31
+; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v31
+; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v16, v15
+; GFX8-NEXT:    v_cmp_gt_i32_e64 s[6:7], 0, v16
+; GFX8-NEXT:    v_cndmask_b32_e64 v15, v32, v33, s[6:7]
+; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v15, v16, v15, vcc
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_ssubsat_v16i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_sub_i32 v0, v0, v16 clamp
+; GFX9-NEXT:    v_sub_i32 v1, v1, v17 clamp
+; GFX9-NEXT:    v_sub_i32 v2, v2, v18 clamp
+; GFX9-NEXT:    v_sub_i32 v3, v3, v19 clamp
+; GFX9-NEXT:    v_sub_i32 v4, v4, v20 clamp
+; GFX9-NEXT:    v_sub_i32 v5, v5, v21 clamp
+; GFX9-NEXT:    v_sub_i32 v6, v6, v22 clamp
+; GFX9-NEXT:    v_sub_i32 v7, v7, v23 clamp
+; GFX9-NEXT:    v_sub_i32 v8, v8, v24 clamp
+; GFX9-NEXT:    v_sub_i32 v9, v9, v25 clamp
+; GFX9-NEXT:    v_sub_i32 v10, v10, v26 clamp
+; GFX9-NEXT:    v_sub_i32 v11, v11, v27 clamp
+; GFX9-NEXT:    v_sub_i32 v12, v12, v28 clamp
+; GFX9-NEXT:    v_sub_i32 v13, v13, v29 clamp
+; GFX9-NEXT:    v_sub_i32 v14, v14, v30 clamp
+; GFX9-NEXT:    v_sub_i32 v15, v15, v31 clamp
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <16 x i32> @llvm.ssub.sat.v16i32(<16 x i32> %lhs, <16 x i32> %rhs)
+  ret <16 x i32> %result
+}
+
+
+define i64 @v_ssubsat_i64(i64 %lhs, i64 %rhs) {
+; GFX6-LABEL: v_ssubsat_i64:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_sub_i32_e32 v4, vcc, v0, v2
+; GFX6-NEXT:    v_subb_u32_e32 v5, vcc, v1, v3, vcc
+; GFX6-NEXT:    v_cmp_lt_i64_e32 vcc, v[4:5], v[0:1]
+; GFX6-NEXT:    v_cmp_lt_i64_e64 s[4:5], 0, v[2:3]
+; GFX6-NEXT:    v_bfrev_b32_e32 v1, 1
+; GFX6-NEXT:    s_xor_b64 vcc, s[4:5], vcc
+; GFX6-NEXT:    v_cmp_gt_i64_e64 s[4:5], 0, v[4:5]
+; GFX6-NEXT:    v_bfrev_b32_e32 v2, -2
+; GFX6-NEXT:    v_ashrrev_i32_e32 v0, 31, v5
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, v1, v2, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_ssubsat_i64:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_sub_u32_e32 v4, vcc, v0, v2
+; GFX8-NEXT:    v_subb_u32_e32 v5, vcc, v1, v3, vcc
+; GFX8-NEXT:    v_cmp_lt_i64_e32 vcc, v[4:5], v[0:1]
+; GFX8-NEXT:    v_cmp_lt_i64_e64 s[4:5], 0, v[2:3]
+; GFX8-NEXT:    v_bfrev_b32_e32 v1, 1
+; GFX8-NEXT:    s_xor_b64 vcc, s[4:5], vcc
+; GFX8-NEXT:    v_cmp_gt_i64_e64 s[4:5], 0, v[4:5]
+; GFX8-NEXT:    v_bfrev_b32_e32 v2, -2
+; GFX8-NEXT:    v_ashrrev_i32_e32 v0, 31, v5
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, v1, v2, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_ssubsat_i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_sub_co_u32_e32 v4, vcc, v0, v2
+; GFX9-NEXT:    v_subb_co_u32_e32 v5, vcc, v1, v3, vcc
+; GFX9-NEXT:    v_cmp_lt_i64_e32 vcc, v[4:5], v[0:1]
+; GFX9-NEXT:    v_cmp_lt_i64_e64 s[4:5], 0, v[2:3]
+; GFX9-NEXT:    v_bfrev_b32_e32 v1, 1
+; GFX9-NEXT:    s_xor_b64 vcc, s[4:5], vcc
+; GFX9-NEXT:    v_cmp_gt_i64_e64 s[4:5], 0, v[4:5]
+; GFX9-NEXT:    v_bfrev_b32_e32 v2, -2
+; GFX9-NEXT:    v_ashrrev_i32_e32 v0, 31, v5
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, v2, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i64 @llvm.ssub.sat.i64(i64 %lhs, i64 %rhs)
+  ret i64 %result
+}
+
+declare i8 @llvm.ssub.sat.i8(i8, i8) #0
+declare i16 @llvm.ssub.sat.i16(i16, i16) #0
+declare <2 x i16> @llvm.ssub.sat.v2i16(<2 x i16>, <2 x i16>) #0
+declare <3 x i16> @llvm.ssub.sat.v3i16(<3 x i16>, <3 x i16>) #0
+declare <4 x i16> @llvm.ssub.sat.v4i16(<4 x i16>, <4 x i16>) #0
+declare i32 @llvm.ssub.sat.i32(i32, i32) #0
+declare <2 x i32> @llvm.ssub.sat.v2i32(<2 x i32>, <2 x i32>) #0
+declare <3 x i32> @llvm.ssub.sat.v3i32(<3 x i32>, <3 x i32>) #0
+declare <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32>, <4 x i32>) #0
+declare <8 x i32> @llvm.ssub.sat.v8i32(<8 x i32>, <8 x i32>) #0
+declare <16 x i32> @llvm.ssub.sat.v16i32(<16 x i32>, <16 x i32>) #0
+declare i64 @llvm.ssub.sat.i64(i64, i64) #0
+
+attributes #0 = { nounwind readnone speculatable willreturn }
diff --git a/llvm/test/CodeGen/AMDGPU/uaddsat.ll b/llvm/test/CodeGen/AMDGPU/uaddsat.ll
new file mode 100644
index 0000000000000..48349cbed903c
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/uaddsat.ll
@@ -0,0 +1,522 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=tahiti < %s | FileCheck -check-prefixes=GCN,GFX6 %s
+; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=fiji < %s | FileCheck -check-prefixes=GCN,GFX8 %s
+; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 < %s | FileCheck -check-prefixes=GCN,GFX9 %s
+
+define i8 @v_uaddsat_i8(i8 %lhs, i8 %rhs) {
+; GFX6-LABEL: v_uaddsat_i8:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    s_movk_i32 s4, 0xff
+; GFX6-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX6-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GFX6-NEXT:    v_min_u32_e32 v0, s4, v0
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_uaddsat_i8:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; GFX8-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
+; GFX8-NEXT:    v_add_u16_e64 v0, v0, v1 clamp
+; GFX8-NEXT:    v_lshrrev_b16_e32 v0, 8, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_uaddsat_i8:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
+; GFX9-NEXT:    v_add_u16_e64 v0, v0, v1 clamp
+; GFX9-NEXT:    v_lshrrev_b16_e32 v0, 8, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i8 @llvm.uadd.sat.i8(i8 %lhs, i8 %rhs)
+  ret i8 %result
+}
+
+define i16 @v_uaddsat_i16(i16 %lhs, i16 %rhs) {
+; GFX6-LABEL: v_uaddsat_i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    s_mov_b32 s4, 0xffff
+; GFX6-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX6-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GFX6-NEXT:    v_min_u32_e32 v0, s4, v0
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_uaddsat_i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_add_u16_e64 v0, v0, v1 clamp
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_uaddsat_i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_add_u16_e64 v0, v0, v1 clamp
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i16 @llvm.uadd.sat.i16(i16 %lhs, i16 %rhs)
+  ret i16 %result
+}
+
+define i32 @v_uaddsat_i32(i32 %lhs, i32 %rhs) {
+; GFX6-LABEL: v_uaddsat_i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_not_b32_e32 v2, v1
+; GFX6-NEXT:    v_min_u32_e32 v0, v0, v2
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_uaddsat_i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_add_u32_e64 v0, s[4:5], v0, v1 clamp
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_uaddsat_i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_add_u32_e64 v0, v0, v1 clamp
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i32 @llvm.uadd.sat.i32(i32 %lhs, i32 %rhs)
+  ret i32 %result
+}
+
+define <2 x i16> @v_uaddsat_v2i16(<2 x i16> %lhs, <2 x i16> %rhs) {
+; GFX6-LABEL: v_uaddsat_v2i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    s_mov_b32 s4, 0xffff
+; GFX6-NEXT:    v_and_b32_e32 v3, s4, v3
+; GFX6-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
+; GFX6-NEXT:    v_and_b32_e32 v2, s4, v2
+; GFX6-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GFX6-NEXT:    v_min_u32_e32 v1, s4, v1
+; GFX6-NEXT:    v_min_u32_e32 v0, s4, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_uaddsat_v2i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_add_u16_sdwa v2, v0, v1 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_add_u16_e64 v0, v0, v1 clamp
+; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_uaddsat_v2i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, v1 clamp
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> %lhs, <2 x i16> %rhs)
+  ret <2 x i16> %result
+}
+
+define <3 x i16> @v_uaddsat_v3i16(<3 x i16> %lhs, <3 x i16> %rhs) {
+; GFX6-LABEL: v_uaddsat_v3i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    s_mov_b32 s4, 0xffff
+; GFX6-NEXT:    v_and_b32_e32 v4, s4, v4
+; GFX6-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v1, v4
+; GFX6-NEXT:    v_and_b32_e32 v3, s4, v3
+; GFX6-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v3
+; GFX6-NEXT:    v_min_u32_e32 v1, s4, v1
+; GFX6-NEXT:    v_and_b32_e32 v5, s4, v5
+; GFX6-NEXT:    v_and_b32_e32 v2, s4, v2
+; GFX6-NEXT:    v_min_u32_e32 v0, s4, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v2, v5
+; GFX6-NEXT:    v_min_u32_e32 v1, s4, v1
+; GFX6-NEXT:    v_or_b32_e32 v2, 0xffff0000, v1
+; GFX6-NEXT:    v_alignbit_b32 v1, v1, v0, 16
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_uaddsat_v3i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_add_u16_sdwa v4, v0, v2 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_add_u16_e64 v0, v0, v2 clamp
+; GFX8-NEXT:    v_add_u16_e64 v1, v1, v3 clamp
+; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_uaddsat_v3i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, v3 clamp
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, v2 clamp
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <3 x i16> @llvm.uadd.sat.v3i16(<3 x i16> %lhs, <3 x i16> %rhs)
+  ret <3 x i16> %result
+}
+
+define <2 x float> @v_uaddsat_v4i16(<4 x i16> %lhs, <4 x i16> %rhs) {
+; GFX6-LABEL: v_uaddsat_v4i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    s_mov_b32 s4, 0xffff
+; GFX6-NEXT:    v_and_b32_e32 v5, s4, v5
+; GFX6-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v1, v5
+; GFX6-NEXT:    v_and_b32_e32 v4, s4, v4
+; GFX6-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
+; GFX6-NEXT:    v_min_u32_e32 v1, s4, v1
+; GFX6-NEXT:    v_and_b32_e32 v7, s4, v7
+; GFX6-NEXT:    v_and_b32_e32 v3, s4, v3
+; GFX6-NEXT:    v_and_b32_e32 v6, s4, v6
+; GFX6-NEXT:    v_and_b32_e32 v2, s4, v2
+; GFX6-NEXT:    v_min_u32_e32 v0, s4, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v2, v6
+; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v3, v7
+; GFX6-NEXT:    v_min_u32_e32 v2, s4, v2
+; GFX6-NEXT:    v_min_u32_e32 v1, s4, v1
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT:    v_or_b32_e32 v1, v1, v2
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_uaddsat_v4i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_add_u16_sdwa v4, v0, v2 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_add_u16_e64 v0, v0, v2 clamp
+; GFX8-NEXT:    v_add_u16_sdwa v2, v1, v3 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_add_u16_e64 v1, v1, v3 clamp
+; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_uaddsat_v4i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, v2 clamp
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, v3 clamp
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <4 x i16> @llvm.uadd.sat.v4i16(<4 x i16> %lhs, <4 x i16> %rhs)
+  %cast = bitcast <4 x i16> %result to <2 x float>
+  ret <2 x float> %cast
+}
+
+define <2 x i32> @v_uaddsat_v2i32(<2 x i32> %lhs, <2 x i32> %rhs) {
+; GFX6-LABEL: v_uaddsat_v2i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_not_b32_e32 v4, v2
+; GFX6-NEXT:    v_min_u32_e32 v0, v0, v4
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GFX6-NEXT:    v_not_b32_e32 v2, v3
+; GFX6-NEXT:    v_min_u32_e32 v1, v1, v2
+; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_uaddsat_v2i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_add_u32_e64 v0, s[4:5], v0, v2 clamp
+; GFX8-NEXT:    v_add_u32_e64 v1, s[4:5], v1, v3 clamp
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_uaddsat_v2i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_add_u32_e64 v0, v0, v2 clamp
+; GFX9-NEXT:    v_add_u32_e64 v1, v1, v3 clamp
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <2 x i32> @llvm.uadd.sat.v2i32(<2 x i32> %lhs, <2 x i32> %rhs)
+  ret <2 x i32> %result
+}
+
+define <3 x i32> @v_uaddsat_v3i32(<3 x i32> %lhs, <3 x i32> %rhs) {
+; GFX6-LABEL: v_uaddsat_v3i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_not_b32_e32 v6, v3
+; GFX6-NEXT:    v_min_u32_e32 v0, v0, v6
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v3
+; GFX6-NEXT:    v_not_b32_e32 v3, v4
+; GFX6-NEXT:    v_min_u32_e32 v1, v1, v3
+; GFX6-NEXT:    v_not_b32_e32 v3, v5
+; GFX6-NEXT:    v_min_u32_e32 v2, v2, v3
+; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v1, v4
+; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v5
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_uaddsat_v3i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_add_u32_e64 v0, s[4:5], v0, v3 clamp
+; GFX8-NEXT:    v_add_u32_e64 v1, s[4:5], v1, v4 clamp
+; GFX8-NEXT:    v_add_u32_e64 v2, s[4:5], v2, v5 clamp
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_uaddsat_v3i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_add_u32_e64 v0, v0, v3 clamp
+; GFX9-NEXT:    v_add_u32_e64 v1, v1, v4 clamp
+; GFX9-NEXT:    v_add_u32_e64 v2, v2, v5 clamp
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <3 x i32> @llvm.uadd.sat.v3i32(<3 x i32> %lhs, <3 x i32> %rhs)
+  ret <3 x i32> %result
+}
+
+define <4 x i32> @v_uaddsat_v4i32(<4 x i32> %lhs, <4 x i32> %rhs) {
+; GFX6-LABEL: v_uaddsat_v4i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_not_b32_e32 v8, v4
+; GFX6-NEXT:    v_min_u32_e32 v0, v0, v8
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
+; GFX6-NEXT:    v_not_b32_e32 v4, v5
+; GFX6-NEXT:    v_min_u32_e32 v1, v1, v4
+; GFX6-NEXT:    v_not_b32_e32 v4, v6
+; GFX6-NEXT:    v_min_u32_e32 v2, v2, v4
+; GFX6-NEXT:    v_not_b32_e32 v4, v7
+; GFX6-NEXT:    v_min_u32_e32 v3, v3, v4
+; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v1, v5
+; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v6
+; GFX6-NEXT:    v_add_i32_e32 v3, vcc, v3, v7
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_uaddsat_v4i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_add_u32_e64 v0, s[4:5], v0, v4 clamp
+; GFX8-NEXT:    v_add_u32_e64 v1, s[4:5], v1, v5 clamp
+; GFX8-NEXT:    v_add_u32_e64 v2, s[4:5], v2, v6 clamp
+; GFX8-NEXT:    v_add_u32_e64 v3, s[4:5], v3, v7 clamp
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_uaddsat_v4i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_add_u32_e64 v0, v0, v4 clamp
+; GFX9-NEXT:    v_add_u32_e64 v1, v1, v5 clamp
+; GFX9-NEXT:    v_add_u32_e64 v2, v2, v6 clamp
+; GFX9-NEXT:    v_add_u32_e64 v3, v3, v7 clamp
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> %lhs, <4 x i32> %rhs)
+  ret <4 x i32> %result
+}
+
+define <8 x i32> @v_uaddsat_v8i32(<8 x i32> %lhs, <8 x i32> %rhs) {
+; GFX6-LABEL: v_uaddsat_v8i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_not_b32_e32 v16, v8
+; GFX6-NEXT:    v_min_u32_e32 v0, v0, v16
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v8
+; GFX6-NEXT:    v_not_b32_e32 v8, v9
+; GFX6-NEXT:    v_min_u32_e32 v1, v1, v8
+; GFX6-NEXT:    v_not_b32_e32 v8, v10
+; GFX6-NEXT:    v_min_u32_e32 v2, v2, v8
+; GFX6-NEXT:    v_not_b32_e32 v8, v11
+; GFX6-NEXT:    v_min_u32_e32 v3, v3, v8
+; GFX6-NEXT:    v_not_b32_e32 v8, v12
+; GFX6-NEXT:    v_min_u32_e32 v4, v4, v8
+; GFX6-NEXT:    v_not_b32_e32 v8, v13
+; GFX6-NEXT:    v_min_u32_e32 v5, v5, v8
+; GFX6-NEXT:    v_not_b32_e32 v8, v14
+; GFX6-NEXT:    v_min_u32_e32 v6, v6, v8
+; GFX6-NEXT:    v_not_b32_e32 v8, v15
+; GFX6-NEXT:    v_min_u32_e32 v7, v7, v8
+; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v1, v9
+; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v10
+; GFX6-NEXT:    v_add_i32_e32 v3, vcc, v3, v11
+; GFX6-NEXT:    v_add_i32_e32 v4, vcc, v4, v12
+; GFX6-NEXT:    v_add_i32_e32 v5, vcc, v5, v13
+; GFX6-NEXT:    v_add_i32_e32 v6, vcc, v6, v14
+; GFX6-NEXT:    v_add_i32_e32 v7, vcc, v7, v15
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_uaddsat_v8i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_add_u32_e64 v0, s[4:5], v0, v8 clamp
+; GFX8-NEXT:    v_add_u32_e64 v1, s[4:5], v1, v9 clamp
+; GFX8-NEXT:    v_add_u32_e64 v2, s[4:5], v2, v10 clamp
+; GFX8-NEXT:    v_add_u32_e64 v3, s[4:5], v3, v11 clamp
+; GFX8-NEXT:    v_add_u32_e64 v4, s[4:5], v4, v12 clamp
+; GFX8-NEXT:    v_add_u32_e64 v5, s[4:5], v5, v13 clamp
+; GFX8-NEXT:    v_add_u32_e64 v6, s[4:5], v6, v14 clamp
+; GFX8-NEXT:    v_add_u32_e64 v7, s[4:5], v7, v15 clamp
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_uaddsat_v8i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_add_u32_e64 v0, v0, v8 clamp
+; GFX9-NEXT:    v_add_u32_e64 v1, v1, v9 clamp
+; GFX9-NEXT:    v_add_u32_e64 v2, v2, v10 clamp
+; GFX9-NEXT:    v_add_u32_e64 v3, v3, v11 clamp
+; GFX9-NEXT:    v_add_u32_e64 v4, v4, v12 clamp
+; GFX9-NEXT:    v_add_u32_e64 v5, v5, v13 clamp
+; GFX9-NEXT:    v_add_u32_e64 v6, v6, v14 clamp
+; GFX9-NEXT:    v_add_u32_e64 v7, v7, v15 clamp
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <8 x i32> @llvm.uadd.sat.v8i32(<8 x i32> %lhs, <8 x i32> %rhs)
+  ret <8 x i32> %result
+}
+
+define <16 x i32> @v_uaddsat_v16i32(<16 x i32> %lhs, <16 x i32> %rhs) {
+; GFX6-LABEL: v_uaddsat_v16i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_not_b32_e32 v32, v16
+; GFX6-NEXT:    v_min_u32_e32 v0, v0, v32
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v16
+; GFX6-NEXT:    v_not_b32_e32 v16, v17
+; GFX6-NEXT:    v_min_u32_e32 v1, v1, v16
+; GFX6-NEXT:    v_not_b32_e32 v16, v18
+; GFX6-NEXT:    v_min_u32_e32 v2, v2, v16
+; GFX6-NEXT:    v_not_b32_e32 v16, v19
+; GFX6-NEXT:    v_min_u32_e32 v3, v3, v16
+; GFX6-NEXT:    v_not_b32_e32 v16, v20
+; GFX6-NEXT:    v_min_u32_e32 v4, v4, v16
+; GFX6-NEXT:    v_not_b32_e32 v16, v21
+; GFX6-NEXT:    v_min_u32_e32 v5, v5, v16
+; GFX6-NEXT:    v_not_b32_e32 v16, v22
+; GFX6-NEXT:    v_min_u32_e32 v6, v6, v16
+; GFX6-NEXT:    v_not_b32_e32 v16, v23
+; GFX6-NEXT:    v_min_u32_e32 v7, v7, v16
+; GFX6-NEXT:    v_not_b32_e32 v16, v24
+; GFX6-NEXT:    v_min_u32_e32 v8, v8, v16
+; GFX6-NEXT:    v_not_b32_e32 v16, v25
+; GFX6-NEXT:    v_min_u32_e32 v9, v9, v16
+; GFX6-NEXT:    v_not_b32_e32 v16, v26
+; GFX6-NEXT:    v_min_u32_e32 v10, v10, v16
+; GFX6-NEXT:    v_not_b32_e32 v16, v27
+; GFX6-NEXT:    v_min_u32_e32 v11, v11, v16
+; GFX6-NEXT:    v_not_b32_e32 v16, v28
+; GFX6-NEXT:    v_min_u32_e32 v12, v12, v16
+; GFX6-NEXT:    v_not_b32_e32 v16, v29
+; GFX6-NEXT:    v_min_u32_e32 v13, v13, v16
+; GFX6-NEXT:    v_not_b32_e32 v16, v30
+; GFX6-NEXT:    v_min_u32_e32 v14, v14, v16
+; GFX6-NEXT:    v_not_b32_e32 v16, v31
+; GFX6-NEXT:    v_min_u32_e32 v15, v15, v16
+; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v1, v17
+; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v18
+; GFX6-NEXT:    v_add_i32_e32 v3, vcc, v3, v19
+; GFX6-NEXT:    v_add_i32_e32 v4, vcc, v4, v20
+; GFX6-NEXT:    v_add_i32_e32 v5, vcc, v5, v21
+; GFX6-NEXT:    v_add_i32_e32 v6, vcc, v6, v22
+; GFX6-NEXT:    v_add_i32_e32 v7, vcc, v7, v23
+; GFX6-NEXT:    v_add_i32_e32 v8, vcc, v8, v24
+; GFX6-NEXT:    v_add_i32_e32 v9, vcc, v9, v25
+; GFX6-NEXT:    v_add_i32_e32 v10, vcc, v10, v26
+; GFX6-NEXT:    v_add_i32_e32 v11, vcc, v11, v27
+; GFX6-NEXT:    v_add_i32_e32 v12, vcc, v12, v28
+; GFX6-NEXT:    v_add_i32_e32 v13, vcc, v13, v29
+; GFX6-NEXT:    v_add_i32_e32 v14, vcc, v14, v30
+; GFX6-NEXT:    v_add_i32_e32 v15, vcc, v15, v31
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_uaddsat_v16i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_add_u32_e64 v0, s[4:5], v0, v16 clamp
+; GFX8-NEXT:    v_add_u32_e64 v1, s[4:5], v1, v17 clamp
+; GFX8-NEXT:    v_add_u32_e64 v2, s[4:5], v2, v18 clamp
+; GFX8-NEXT:    v_add_u32_e64 v3, s[4:5], v3, v19 clamp
+; GFX8-NEXT:    v_add_u32_e64 v4, s[4:5], v4, v20 clamp
+; GFX8-NEXT:    v_add_u32_e64 v5, s[4:5], v5, v21 clamp
+; GFX8-NEXT:    v_add_u32_e64 v6, s[4:5], v6, v22 clamp
+; GFX8-NEXT:    v_add_u32_e64 v7, s[4:5], v7, v23 clamp
+; GFX8-NEXT:    v_add_u32_e64 v8, s[4:5], v8, v24 clamp
+; GFX8-NEXT:    v_add_u32_e64 v9, s[4:5], v9, v25 clamp
+; GFX8-NEXT:    v_add_u32_e64 v10, s[4:5], v10, v26 clamp
+; GFX8-NEXT:    v_add_u32_e64 v11, s[4:5], v11, v27 clamp
+; GFX8-NEXT:    v_add_u32_e64 v12, s[4:5], v12, v28 clamp
+; GFX8-NEXT:    v_add_u32_e64 v13, s[4:5], v13, v29 clamp
+; GFX8-NEXT:    v_add_u32_e64 v14, s[4:5], v14, v30 clamp
+; GFX8-NEXT:    v_add_u32_e64 v15, s[4:5], v15, v31 clamp
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_uaddsat_v16i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_add_u32_e64 v0, v0, v16 clamp
+; GFX9-NEXT:    v_add_u32_e64 v1, v1, v17 clamp
+; GFX9-NEXT:    v_add_u32_e64 v2, v2, v18 clamp
+; GFX9-NEXT:    v_add_u32_e64 v3, v3, v19 clamp
+; GFX9-NEXT:    v_add_u32_e64 v4, v4, v20 clamp
+; GFX9-NEXT:    v_add_u32_e64 v5, v5, v21 clamp
+; GFX9-NEXT:    v_add_u32_e64 v6, v6, v22 clamp
+; GFX9-NEXT:    v_add_u32_e64 v7, v7, v23 clamp
+; GFX9-NEXT:    v_add_u32_e64 v8, v8, v24 clamp
+; GFX9-NEXT:    v_add_u32_e64 v9, v9, v25 clamp
+; GFX9-NEXT:    v_add_u32_e64 v10, v10, v26 clamp
+; GFX9-NEXT:    v_add_u32_e64 v11, v11, v27 clamp
+; GFX9-NEXT:    v_add_u32_e64 v12, v12, v28 clamp
+; GFX9-NEXT:    v_add_u32_e64 v13, v13, v29 clamp
+; GFX9-NEXT:    v_add_u32_e64 v14, v14, v30 clamp
+; GFX9-NEXT:    v_add_u32_e64 v15, v15, v31 clamp
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <16 x i32> @llvm.uadd.sat.v16i32(<16 x i32> %lhs, <16 x i32> %rhs)
+  ret <16 x i32> %result
+}
+
+
+define i64 @v_uaddsat_i64(i64 %lhs, i64 %rhs) {
+; GFX6-LABEL: v_uaddsat_i64:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v0, v2
+; GFX6-NEXT:    v_addc_u32_e32 v3, vcc, v1, v3, vcc
+; GFX6-NEXT:    v_cmp_lt_u64_e32 vcc, v[2:3], v[0:1]
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, v2, -1, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, v3, -1, vcc
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_uaddsat_i64:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v0, v2
+; GFX8-NEXT:    v_addc_u32_e32 v3, vcc, v1, v3, vcc
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, v[2:3], v[0:1]
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v2, -1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, v3, -1, vcc
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_uaddsat_i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v0, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, v1, v3, vcc
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, v[2:3], v[0:1]
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v2, -1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, v3, -1, vcc
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i64 @llvm.uadd.sat.i64(i64 %lhs, i64 %rhs)
+  ret i64 %result
+}
+
+declare i8 @llvm.uadd.sat.i8(i8, i8) #0
+declare i16 @llvm.uadd.sat.i16(i16, i16) #0
+declare <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16>, <2 x i16>) #0
+declare <3 x i16> @llvm.uadd.sat.v3i16(<3 x i16>, <3 x i16>) #0
+declare <4 x i16> @llvm.uadd.sat.v4i16(<4 x i16>, <4 x i16>) #0
+declare i32 @llvm.uadd.sat.i32(i32, i32) #0
+declare <2 x i32> @llvm.uadd.sat.v2i32(<2 x i32>, <2 x i32>) #0
+declare <3 x i32> @llvm.uadd.sat.v3i32(<3 x i32>, <3 x i32>) #0
+declare <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32>, <4 x i32>) #0
+declare <8 x i32> @llvm.uadd.sat.v8i32(<8 x i32>, <8 x i32>) #0
+declare <16 x i32> @llvm.uadd.sat.v16i32(<16 x i32>, <16 x i32>) #0
+declare i64 @llvm.uadd.sat.i64(i64, i64) #0
+
+attributes #0 = { nounwind readnone speculatable willreturn }
diff --git a/llvm/test/CodeGen/AMDGPU/usubsat.ll b/llvm/test/CodeGen/AMDGPU/usubsat.ll
new file mode 100644
index 0000000000000..55275450b1b4b
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/usubsat.ll
@@ -0,0 +1,492 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=tahiti < %s | FileCheck -check-prefixes=GCN,GFX6 %s
+; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=fiji < %s | FileCheck -check-prefixes=GCN,GFX8 %s
+; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 < %s | FileCheck -check-prefixes=GCN,GFX9 %s
+
+define i8 @v_usubsat_i8(i8 %lhs, i8 %rhs) {
+; GFX6-LABEL: v_usubsat_i8:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    s_movk_i32 s4, 0xff
+; GFX6-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX6-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX6-NEXT:    v_max_u32_e32 v0, v0, v1
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, v0, v1
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_usubsat_i8:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; GFX8-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
+; GFX8-NEXT:    v_sub_u16_e64 v0, v0, v1 clamp
+; GFX8-NEXT:    v_lshrrev_b16_e32 v0, 8, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_usubsat_i8:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
+; GFX9-NEXT:    v_sub_u16_e64 v0, v0, v1 clamp
+; GFX9-NEXT:    v_lshrrev_b16_e32 v0, 8, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i8 @llvm.usub.sat.i8(i8 %lhs, i8 %rhs)
+  ret i8 %result
+}
+
+define i16 @v_usubsat_i16(i16 %lhs, i16 %rhs) {
+; GFX6-LABEL: v_usubsat_i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    s_mov_b32 s4, 0xffff
+; GFX6-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX6-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX6-NEXT:    v_max_u32_e32 v0, v0, v1
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, v0, v1
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_usubsat_i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_sub_u16_e64 v0, v0, v1 clamp
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_usubsat_i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_sub_u16_e64 v0, v0, v1 clamp
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i16 @llvm.usub.sat.i16(i16 %lhs, i16 %rhs)
+  ret i16 %result
+}
+
+define i32 @v_usubsat_i32(i32 %lhs, i32 %rhs) {
+; GFX6-LABEL: v_usubsat_i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_max_u32_e32 v0, v0, v1
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, v0, v1
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_usubsat_i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_sub_u32_e64 v0, s[4:5], v0, v1 clamp
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_usubsat_i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_sub_u32_e64 v0, v0, v1 clamp
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i32 @llvm.usub.sat.i32(i32 %lhs, i32 %rhs)
+  ret i32 %result
+}
+
+define <2 x i16> @v_usubsat_v2i16(<2 x i16> %lhs, <2 x i16> %rhs) {
+; GFX6-LABEL: v_usubsat_v2i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    s_mov_b32 s4, 0xffff
+; GFX6-NEXT:    v_and_b32_e32 v4, s4, v2
+; GFX6-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX6-NEXT:    v_and_b32_e32 v5, s4, v3
+; GFX6-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX6-NEXT:    v_max_u32_e32 v1, v1, v5
+; GFX6-NEXT:    v_max_u32_e32 v0, v0, v4
+; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, v1, v3
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, v0, v2
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_usubsat_v2i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_sub_u16_sdwa v2, v0, v1 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_sub_u16_e64 v0, v0, v1 clamp
+; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_usubsat_v2i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_pk_sub_u16 v0, v0, v1 clamp
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <2 x i16> @llvm.usub.sat.v2i16(<2 x i16> %lhs, <2 x i16> %rhs)
+  ret <2 x i16> %result
+}
+
+define <3 x i16> @v_usubsat_v3i16(<3 x i16> %lhs, <3 x i16> %rhs) {
+; GFX6-LABEL: v_usubsat_v3i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    s_mov_b32 s4, 0xffff
+; GFX6-NEXT:    v_and_b32_e32 v7, s4, v3
+; GFX6-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX6-NEXT:    v_and_b32_e32 v8, s4, v4
+; GFX6-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX6-NEXT:    v_max_u32_e32 v1, v1, v8
+; GFX6-NEXT:    v_max_u32_e32 v0, v0, v7
+; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, v1, v4
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, v0, v3
+; GFX6-NEXT:    v_and_b32_e32 v6, s4, v5
+; GFX6-NEXT:    v_and_b32_e32 v2, s4, v2
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    v_max_u32_e32 v1, v2, v6
+; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, v1, v5
+; GFX6-NEXT:    v_and_b32_e32 v2, s4, v1
+; GFX6-NEXT:    v_alignbit_b32 v1, v1, v0, 16
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_usubsat_v3i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_sub_u16_sdwa v4, v0, v2 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_sub_u16_e64 v0, v0, v2 clamp
+; GFX8-NEXT:    v_sub_u16_e64 v1, v1, v3 clamp
+; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_usubsat_v3i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_pk_sub_u16 v0, v0, v2 clamp
+; GFX9-NEXT:    v_pk_sub_u16 v1, v1, v3 clamp
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <3 x i16> @llvm.usub.sat.v3i16(<3 x i16> %lhs, <3 x i16> %rhs)
+  ret <3 x i16> %result
+}
+
+define <2 x float> @v_usubsat_v4i16(<4 x i16> %lhs, <4 x i16> %rhs) {
+; GFX6-LABEL: v_usubsat_v4i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    s_mov_b32 s4, 0xffff
+; GFX6-NEXT:    v_and_b32_e32 v10, s4, v4
+; GFX6-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX6-NEXT:    v_and_b32_e32 v11, s4, v5
+; GFX6-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX6-NEXT:    v_max_u32_e32 v1, v1, v11
+; GFX6-NEXT:    v_max_u32_e32 v0, v0, v10
+; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, v1, v5
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, v0, v4
+; GFX6-NEXT:    v_and_b32_e32 v8, s4, v6
+; GFX6-NEXT:    v_and_b32_e32 v2, s4, v2
+; GFX6-NEXT:    v_and_b32_e32 v9, s4, v7
+; GFX6-NEXT:    v_and_b32_e32 v3, s4, v3
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX6-NEXT:    v_max_u32_e32 v2, v2, v8
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    v_max_u32_e32 v1, v3, v9
+; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, v1, v7
+; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, v2, v6
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GFX6-NEXT:    v_or_b32_e32 v1, v2, v1
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_usubsat_v4i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_sub_u16_sdwa v4, v0, v2 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_sub_u16_e64 v0, v0, v2 clamp
+; GFX8-NEXT:    v_sub_u16_sdwa v2, v1, v3 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_sub_u16_e64 v1, v1, v3 clamp
+; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_usubsat_v4i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_pk_sub_u16 v0, v0, v2 clamp
+; GFX9-NEXT:    v_pk_sub_u16 v1, v1, v3 clamp
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <4 x i16> @llvm.usub.sat.v4i16(<4 x i16> %lhs, <4 x i16> %rhs)
+  %cast = bitcast <4 x i16> %result to <2 x float>
+  ret <2 x float> %cast
+}
+
+define <2 x i32> @v_usubsat_v2i32(<2 x i32> %lhs, <2 x i32> %rhs) {
+; GFX6-LABEL: v_usubsat_v2i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_max_u32_e32 v0, v0, v2
+; GFX6-NEXT:    v_max_u32_e32 v1, v1, v3
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, v0, v2
+; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, v1, v3
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_usubsat_v2i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_sub_u32_e64 v0, s[4:5], v0, v2 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v1, s[4:5], v1, v3 clamp
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_usubsat_v2i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_sub_u32_e64 v0, v0, v2 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v1, v1, v3 clamp
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <2 x i32> @llvm.usub.sat.v2i32(<2 x i32> %lhs, <2 x i32> %rhs)
+  ret <2 x i32> %result
+}
+
+define <3 x i32> @v_usubsat_v3i32(<3 x i32> %lhs, <3 x i32> %rhs) {
+; GFX6-LABEL: v_usubsat_v3i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_max_u32_e32 v0, v0, v3
+; GFX6-NEXT:    v_max_u32_e32 v1, v1, v4
+; GFX6-NEXT:    v_max_u32_e32 v2, v2, v5
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, v0, v3
+; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, v1, v4
+; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, v2, v5
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_usubsat_v3i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_sub_u32_e64 v0, s[4:5], v0, v3 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v1, s[4:5], v1, v4 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v2, s[4:5], v2, v5 clamp
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_usubsat_v3i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_sub_u32_e64 v0, v0, v3 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v1, v1, v4 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v2, v2, v5 clamp
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <3 x i32> @llvm.usub.sat.v3i32(<3 x i32> %lhs, <3 x i32> %rhs)
+  ret <3 x i32> %result
+}
+
+define <4 x i32> @v_usubsat_v4i32(<4 x i32> %lhs, <4 x i32> %rhs) {
+; GFX6-LABEL: v_usubsat_v4i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_max_u32_e32 v0, v0, v4
+; GFX6-NEXT:    v_max_u32_e32 v1, v1, v5
+; GFX6-NEXT:    v_max_u32_e32 v2, v2, v6
+; GFX6-NEXT:    v_max_u32_e32 v3, v3, v7
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, v0, v4
+; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, v1, v5
+; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, v2, v6
+; GFX6-NEXT:    v_sub_i32_e32 v3, vcc, v3, v7
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_usubsat_v4i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_sub_u32_e64 v0, s[4:5], v0, v4 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v1, s[4:5], v1, v5 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v2, s[4:5], v2, v6 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v3, s[4:5], v3, v7 clamp
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_usubsat_v4i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_sub_u32_e64 v0, v0, v4 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v1, v1, v5 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v2, v2, v6 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v3, v3, v7 clamp
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> %lhs, <4 x i32> %rhs)
+  ret <4 x i32> %result
+}
+
+define <8 x i32> @v_usubsat_v8i32(<8 x i32> %lhs, <8 x i32> %rhs) {
+; GFX6-LABEL: v_usubsat_v8i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_max_u32_e32 v0, v0, v8
+; GFX6-NEXT:    v_max_u32_e32 v1, v1, v9
+; GFX6-NEXT:    v_max_u32_e32 v2, v2, v10
+; GFX6-NEXT:    v_max_u32_e32 v3, v3, v11
+; GFX6-NEXT:    v_max_u32_e32 v4, v4, v12
+; GFX6-NEXT:    v_max_u32_e32 v5, v5, v13
+; GFX6-NEXT:    v_max_u32_e32 v6, v6, v14
+; GFX6-NEXT:    v_max_u32_e32 v7, v7, v15
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, v0, v8
+; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, v1, v9
+; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, v2, v10
+; GFX6-NEXT:    v_sub_i32_e32 v3, vcc, v3, v11
+; GFX6-NEXT:    v_sub_i32_e32 v4, vcc, v4, v12
+; GFX6-NEXT:    v_sub_i32_e32 v5, vcc, v5, v13
+; GFX6-NEXT:    v_sub_i32_e32 v6, vcc, v6, v14
+; GFX6-NEXT:    v_sub_i32_e32 v7, vcc, v7, v15
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_usubsat_v8i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_sub_u32_e64 v0, s[4:5], v0, v8 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v1, s[4:5], v1, v9 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v2, s[4:5], v2, v10 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v3, s[4:5], v3, v11 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v4, s[4:5], v4, v12 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v5, s[4:5], v5, v13 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v6, s[4:5], v6, v14 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v7, s[4:5], v7, v15 clamp
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_usubsat_v8i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_sub_u32_e64 v0, v0, v8 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v1, v1, v9 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v2, v2, v10 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v3, v3, v11 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v4, v4, v12 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v5, v5, v13 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v6, v6, v14 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v7, v7, v15 clamp
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <8 x i32> @llvm.usub.sat.v8i32(<8 x i32> %lhs, <8 x i32> %rhs)
+  ret <8 x i32> %result
+}
+
+define <16 x i32> @v_usubsat_v16i32(<16 x i32> %lhs, <16 x i32> %rhs) {
+; GFX6-LABEL: v_usubsat_v16i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_max_u32_e32 v0, v0, v16
+; GFX6-NEXT:    v_max_u32_e32 v1, v1, v17
+; GFX6-NEXT:    v_max_u32_e32 v2, v2, v18
+; GFX6-NEXT:    v_max_u32_e32 v3, v3, v19
+; GFX6-NEXT:    v_max_u32_e32 v4, v4, v20
+; GFX6-NEXT:    v_max_u32_e32 v5, v5, v21
+; GFX6-NEXT:    v_max_u32_e32 v6, v6, v22
+; GFX6-NEXT:    v_max_u32_e32 v7, v7, v23
+; GFX6-NEXT:    v_max_u32_e32 v8, v8, v24
+; GFX6-NEXT:    v_max_u32_e32 v9, v9, v25
+; GFX6-NEXT:    v_max_u32_e32 v10, v10, v26
+; GFX6-NEXT:    v_max_u32_e32 v11, v11, v27
+; GFX6-NEXT:    v_max_u32_e32 v12, v12, v28
+; GFX6-NEXT:    v_max_u32_e32 v13, v13, v29
+; GFX6-NEXT:    v_max_u32_e32 v14, v14, v30
+; GFX6-NEXT:    v_max_u32_e32 v15, v15, v31
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, v0, v16
+; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, v1, v17
+; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, v2, v18
+; GFX6-NEXT:    v_sub_i32_e32 v3, vcc, v3, v19
+; GFX6-NEXT:    v_sub_i32_e32 v4, vcc, v4, v20
+; GFX6-NEXT:    v_sub_i32_e32 v5, vcc, v5, v21
+; GFX6-NEXT:    v_sub_i32_e32 v6, vcc, v6, v22
+; GFX6-NEXT:    v_sub_i32_e32 v7, vcc, v7, v23
+; GFX6-NEXT:    v_sub_i32_e32 v8, vcc, v8, v24
+; GFX6-NEXT:    v_sub_i32_e32 v9, vcc, v9, v25
+; GFX6-NEXT:    v_sub_i32_e32 v10, vcc, v10, v26
+; GFX6-NEXT:    v_sub_i32_e32 v11, vcc, v11, v27
+; GFX6-NEXT:    v_sub_i32_e32 v12, vcc, v12, v28
+; GFX6-NEXT:    v_sub_i32_e32 v13, vcc, v13, v29
+; GFX6-NEXT:    v_sub_i32_e32 v14, vcc, v14, v30
+; GFX6-NEXT:    v_sub_i32_e32 v15, vcc, v15, v31
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_usubsat_v16i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_sub_u32_e64 v0, s[4:5], v0, v16 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v1, s[4:5], v1, v17 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v2, s[4:5], v2, v18 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v3, s[4:5], v3, v19 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v4, s[4:5], v4, v20 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v5, s[4:5], v5, v21 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v6, s[4:5], v6, v22 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v7, s[4:5], v7, v23 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v8, s[4:5], v8, v24 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v9, s[4:5], v9, v25 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v10, s[4:5], v10, v26 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v11, s[4:5], v11, v27 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v12, s[4:5], v12, v28 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v13, s[4:5], v13, v29 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v14, s[4:5], v14, v30 clamp
+; GFX8-NEXT:    v_sub_u32_e64 v15, s[4:5], v15, v31 clamp
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_usubsat_v16i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_sub_u32_e64 v0, v0, v16 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v1, v1, v17 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v2, v2, v18 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v3, v3, v19 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v4, v4, v20 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v5, v5, v21 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v6, v6, v22 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v7, v7, v23 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v8, v8, v24 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v9, v9, v25 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v10, v10, v26 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v11, v11, v27 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v12, v12, v28 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v13, v13, v29 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v14, v14, v30 clamp
+; GFX9-NEXT:    v_sub_u32_e64 v15, v15, v31 clamp
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <16 x i32> @llvm.usub.sat.v16i32(<16 x i32> %lhs, <16 x i32> %rhs)
+  ret <16 x i32> %result
+}
+
+
+define i64 @v_usubsat_i64(i64 %lhs, i64 %rhs) {
+; GFX6-LABEL: v_usubsat_i64:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, v0, v2
+; GFX6-NEXT:    v_subb_u32_e32 v3, vcc, v1, v3, vcc
+; GFX6-NEXT:    v_cmp_gt_u64_e32 vcc, v[2:3], v[0:1]
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, v2, 0, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, v3, 0, vcc
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_usubsat_i64:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_sub_u32_e32 v2, vcc, v0, v2
+; GFX8-NEXT:    v_subb_u32_e32 v3, vcc, v1, v3, vcc
+; GFX8-NEXT:    v_cmp_gt_u64_e32 vcc, v[2:3], v[0:1]
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v2, 0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, v3, 0, vcc
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_usubsat_i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_sub_co_u32_e32 v2, vcc, v0, v2
+; GFX9-NEXT:    v_subb_co_u32_e32 v3, vcc, v1, v3, vcc
+; GFX9-NEXT:    v_cmp_gt_u64_e32 vcc, v[2:3], v[0:1]
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v2, 0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, v3, 0, vcc
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i64 @llvm.usub.sat.i64(i64 %lhs, i64 %rhs)
+  ret i64 %result
+}
+
+declare i8 @llvm.usub.sat.i8(i8, i8) #0
+declare i16 @llvm.usub.sat.i16(i16, i16) #0
+declare <2 x i16> @llvm.usub.sat.v2i16(<2 x i16>, <2 x i16>) #0
+declare <3 x i16> @llvm.usub.sat.v3i16(<3 x i16>, <3 x i16>) #0
+declare <4 x i16> @llvm.usub.sat.v4i16(<4 x i16>, <4 x i16>) #0
+declare i32 @llvm.usub.sat.i32(i32, i32) #0
+declare <2 x i32> @llvm.usub.sat.v2i32(<2 x i32>, <2 x i32>) #0
+declare <3 x i32> @llvm.usub.sat.v3i32(<3 x i32>, <3 x i32>) #0
+declare <4 x i32> @llvm.usub.sat.v4i32(<4 x i32>, <4 x i32>) #0
+declare <8 x i32> @llvm.usub.sat.v8i32(<8 x i32>, <8 x i32>) #0
+declare <16 x i32> @llvm.usub.sat.v16i32(<16 x i32>, <16 x i32>) #0
+declare i64 @llvm.usub.sat.i64(i64, i64) #0
+
+attributes #0 = { nounwind readnone speculatable willreturn }
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/add_sub_sat.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/add_sub_sat.ll
new file mode 100644
index 0000000000000..514cf07490860
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/add_sub_sat.ll
@@ -0,0 +1,303 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii -slp-vectorizer -instcombine %s | FileCheck -check-prefixes=GCN,GFX7 %s
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=fiji -slp-vectorizer -instcombine %s | FileCheck -check-prefixes=GCN,GFX8 %s
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -slp-vectorizer -instcombine %s | FileCheck -check-prefixes=GCN,GFX8 %s
+
+define <2 x i16> @uadd_sat_v2i16(<2 x i16> %arg0, <2 x i16> %arg1) {
+; GFX7-LABEL: @uadd_sat_v2i16(
+; GFX7-NEXT:  bb:
+; GFX7-NEXT:    [[ARG0_0:%.*]] = extractelement <2 x i16> [[ARG0:%.*]], i64 0
+; GFX7-NEXT:    [[ARG0_1:%.*]] = extractelement <2 x i16> [[ARG0]], i64 1
+; GFX7-NEXT:    [[ARG1_0:%.*]] = extractelement <2 x i16> [[ARG1:%.*]], i64 0
+; GFX7-NEXT:    [[ARG1_1:%.*]] = extractelement <2 x i16> [[ARG1]], i64 1
+; GFX7-NEXT:    [[ADD_0:%.*]] = call i16 @llvm.uadd.sat.i16(i16 [[ARG0_0]], i16 [[ARG1_0]])
+; GFX7-NEXT:    [[ADD_1:%.*]] = call i16 @llvm.uadd.sat.i16(i16 [[ARG0_1]], i16 [[ARG1_1]])
+; GFX7-NEXT:    [[INS_0:%.*]] = insertelement <2 x i16> undef, i16 [[ADD_0]], i64 0
+; GFX7-NEXT:    [[INS_1:%.*]] = insertelement <2 x i16> [[INS_0]], i16 [[ADD_1]], i64 1
+; GFX7-NEXT:    ret <2 x i16> [[INS_1]]
+;
+; GFX8-LABEL: @uadd_sat_v2i16(
+; GFX8-NEXT:  bb:
+; GFX8-NEXT:    [[TMP0:%.*]] = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> [[ARG0:%.*]], <2 x i16> [[ARG1:%.*]])
+; GFX8-NEXT:    ret <2 x i16> [[TMP0]]
+;
+bb:
+  %arg0.0 = extractelement <2 x i16> %arg0, i64 0
+  %arg0.1 = extractelement <2 x i16> %arg0, i64 1
+  %arg1.0 = extractelement <2 x i16> %arg1, i64 0
+  %arg1.1 = extractelement <2 x i16> %arg1, i64 1
+  %add.0 = call i16 @llvm.uadd.sat.i16(i16 %arg0.0, i16 %arg1.0)
+  %add.1 = call i16 @llvm.uadd.sat.i16(i16 %arg0.1, i16 %arg1.1)
+  %ins.0 = insertelement <2 x i16> undef, i16 %add.0, i64 0
+  %ins.1 = insertelement <2 x i16> %ins.0, i16 %add.1, i64 1
+  ret <2 x i16> %ins.1
+}
+
+define <2 x i16> @usub_sat_v2i16(<2 x i16> %arg0, <2 x i16> %arg1) {
+; GFX7-LABEL: @usub_sat_v2i16(
+; GFX7-NEXT:  bb:
+; GFX7-NEXT:    [[ARG0_0:%.*]] = extractelement <2 x i16> [[ARG0:%.*]], i64 0
+; GFX7-NEXT:    [[ARG0_1:%.*]] = extractelement <2 x i16> [[ARG0]], i64 1
+; GFX7-NEXT:    [[ARG1_0:%.*]] = extractelement <2 x i16> [[ARG1:%.*]], i64 0
+; GFX7-NEXT:    [[ARG1_1:%.*]] = extractelement <2 x i16> [[ARG1]], i64 1
+; GFX7-NEXT:    [[ADD_0:%.*]] = call i16 @llvm.usub.sat.i16(i16 [[ARG0_0]], i16 [[ARG1_0]])
+; GFX7-NEXT:    [[ADD_1:%.*]] = call i16 @llvm.usub.sat.i16(i16 [[ARG0_1]], i16 [[ARG1_1]])
+; GFX7-NEXT:    [[INS_0:%.*]] = insertelement <2 x i16> undef, i16 [[ADD_0]], i64 0
+; GFX7-NEXT:    [[INS_1:%.*]] = insertelement <2 x i16> [[INS_0]], i16 [[ADD_1]], i64 1
+; GFX7-NEXT:    ret <2 x i16> [[INS_1]]
+;
+; GFX8-LABEL: @usub_sat_v2i16(
+; GFX8-NEXT:  bb:
+; GFX8-NEXT:    [[TMP0:%.*]] = call <2 x i16> @llvm.usub.sat.v2i16(<2 x i16> [[ARG0:%.*]], <2 x i16> [[ARG1:%.*]])
+; GFX8-NEXT:    ret <2 x i16> [[TMP0]]
+;
+bb:
+  %arg0.0 = extractelement <2 x i16> %arg0, i64 0
+  %arg0.1 = extractelement <2 x i16> %arg0, i64 1
+  %arg1.0 = extractelement <2 x i16> %arg1, i64 0
+  %arg1.1 = extractelement <2 x i16> %arg1, i64 1
+  %add.0 = call i16 @llvm.usub.sat.i16(i16 %arg0.0, i16 %arg1.0)
+  %add.1 = call i16 @llvm.usub.sat.i16(i16 %arg0.1, i16 %arg1.1)
+  %ins.0 = insertelement <2 x i16> undef, i16 %add.0, i64 0
+  %ins.1 = insertelement <2 x i16> %ins.0, i16 %add.1, i64 1
+  ret <2 x i16> %ins.1
+}
+
+define <2 x i16> @sadd_sat_v2i16(<2 x i16> %arg0, <2 x i16> %arg1) {
+; GFX7-LABEL: @sadd_sat_v2i16(
+; GFX7-NEXT:  bb:
+; GFX7-NEXT:    [[ARG0_0:%.*]] = extractelement <2 x i16> [[ARG0:%.*]], i64 0
+; GFX7-NEXT:    [[ARG0_1:%.*]] = extractelement <2 x i16> [[ARG0]], i64 1
+; GFX7-NEXT:    [[ARG1_0:%.*]] = extractelement <2 x i16> [[ARG1:%.*]], i64 0
+; GFX7-NEXT:    [[ARG1_1:%.*]] = extractelement <2 x i16> [[ARG1]], i64 1
+; GFX7-NEXT:    [[ADD_0:%.*]] = call i16 @llvm.sadd.sat.i16(i16 [[ARG0_0]], i16 [[ARG1_0]])
+; GFX7-NEXT:    [[ADD_1:%.*]] = call i16 @llvm.sadd.sat.i16(i16 [[ARG0_1]], i16 [[ARG1_1]])
+; GFX7-NEXT:    [[INS_0:%.*]] = insertelement <2 x i16> undef, i16 [[ADD_0]], i64 0
+; GFX7-NEXT:    [[INS_1:%.*]] = insertelement <2 x i16> [[INS_0]], i16 [[ADD_1]], i64 1
+; GFX7-NEXT:    ret <2 x i16> [[INS_1]]
+;
+; GFX8-LABEL: @sadd_sat_v2i16(
+; GFX8-NEXT:  bb:
+; GFX8-NEXT:    [[TMP0:%.*]] = call <2 x i16> @llvm.sadd.sat.v2i16(<2 x i16> [[ARG0:%.*]], <2 x i16> [[ARG1:%.*]])
+; GFX8-NEXT:    ret <2 x i16> [[TMP0]]
+;
+bb:
+  %arg0.0 = extractelement <2 x i16> %arg0, i64 0
+  %arg0.1 = extractelement <2 x i16> %arg0, i64 1
+  %arg1.0 = extractelement <2 x i16> %arg1, i64 0
+  %arg1.1 = extractelement <2 x i16> %arg1, i64 1
+  %add.0 = call i16 @llvm.sadd.sat.i16(i16 %arg0.0, i16 %arg1.0)
+  %add.1 = call i16 @llvm.sadd.sat.i16(i16 %arg0.1, i16 %arg1.1)
+  %ins.0 = insertelement <2 x i16> undef, i16 %add.0, i64 0
+  %ins.1 = insertelement <2 x i16> %ins.0, i16 %add.1, i64 1
+  ret <2 x i16> %ins.1
+}
+
+define <2 x i16> @ssub_sat_v2i16(<2 x i16> %arg0, <2 x i16> %arg1) {
+; GFX7-LABEL: @ssub_sat_v2i16(
+; GFX7-NEXT:  bb:
+; GFX7-NEXT:    [[ARG0_0:%.*]] = extractelement <2 x i16> [[ARG0:%.*]], i64 0
+; GFX7-NEXT:    [[ARG0_1:%.*]] = extractelement <2 x i16> [[ARG0]], i64 1
+; GFX7-NEXT:    [[ARG1_0:%.*]] = extractelement <2 x i16> [[ARG1:%.*]], i64 0
+; GFX7-NEXT:    [[ARG1_1:%.*]] = extractelement <2 x i16> [[ARG1]], i64 1
+; GFX7-NEXT:    [[ADD_0:%.*]] = call i16 @llvm.ssub.sat.i16(i16 [[ARG0_0]], i16 [[ARG1_0]])
+; GFX7-NEXT:    [[ADD_1:%.*]] = call i16 @llvm.ssub.sat.i16(i16 [[ARG0_1]], i16 [[ARG1_1]])
+; GFX7-NEXT:    [[INS_0:%.*]] = insertelement <2 x i16> undef, i16 [[ADD_0]], i64 0
+; GFX7-NEXT:    [[INS_1:%.*]] = insertelement <2 x i16> [[INS_0]], i16 [[ADD_1]], i64 1
+; GFX7-NEXT:    ret <2 x i16> [[INS_1]]
+;
+; GFX8-LABEL: @ssub_sat_v2i16(
+; GFX8-NEXT:  bb:
+; GFX8-NEXT:    [[TMP0:%.*]] = call <2 x i16> @llvm.ssub.sat.v2i16(<2 x i16> [[ARG0:%.*]], <2 x i16> [[ARG1:%.*]])
+; GFX8-NEXT:    ret <2 x i16> [[TMP0]]
+;
+bb:
+  %arg0.0 = extractelement <2 x i16> %arg0, i64 0
+  %arg0.1 = extractelement <2 x i16> %arg0, i64 1
+  %arg1.0 = extractelement <2 x i16> %arg1, i64 0
+  %arg1.1 = extractelement <2 x i16> %arg1, i64 1
+  %add.0 = call i16 @llvm.ssub.sat.i16(i16 %arg0.0, i16 %arg1.0)
+  %add.1 = call i16 @llvm.ssub.sat.i16(i16 %arg0.1, i16 %arg1.1)
+  %ins.0 = insertelement <2 x i16> undef, i16 %add.0, i64 0
+  %ins.1 = insertelement <2 x i16> %ins.0, i16 %add.1, i64 1
+  ret <2 x i16> %ins.1
+}
+
+; FIXME: Should not vectorize
+define <2 x i32> @uadd_sat_v2i32(<2 x i32> %arg0, <2 x i32> %arg1) {
+; GCN-LABEL: @uadd_sat_v2i32(
+; GCN-NEXT:  bb:
+; GCN-NEXT:    [[TMP0:%.*]] = call <2 x i32> @llvm.uadd.sat.v2i32(<2 x i32> [[ARG0:%.*]], <2 x i32> [[ARG1:%.*]])
+; GCN-NEXT:    ret <2 x i32> [[TMP0]]
+;
+bb:
+  %arg0.0 = extractelement <2 x i32> %arg0, i64 0
+  %arg0.1 = extractelement <2 x i32> %arg0, i64 1
+  %arg1.0 = extractelement <2 x i32> %arg1, i64 0
+  %arg1.1 = extractelement <2 x i32> %arg1, i64 1
+  %add.0 = call i32 @llvm.uadd.sat.i32(i32 %arg0.0, i32 %arg1.0)
+  %add.1 = call i32 @llvm.uadd.sat.i32(i32 %arg0.1, i32 %arg1.1)
+  %ins.0 = insertelement <2 x i32> undef, i32 %add.0, i64 0
+  %ins.1 = insertelement <2 x i32> %ins.0, i32 %add.1, i64 1
+  ret <2 x i32> %ins.1
+}
+
+define <2 x i32> @usub_sat_v2i32(<2 x i32> %arg0, <2 x i32> %arg1) {
+; GCN-LABEL: @usub_sat_v2i32(
+; GCN-NEXT:  bb:
+; GCN-NEXT:    [[TMP0:%.*]] = call <2 x i32> @llvm.usub.sat.v2i32(<2 x i32> [[ARG0:%.*]], <2 x i32> [[ARG1:%.*]])
+; GCN-NEXT:    ret <2 x i32> [[TMP0]]
+;
+bb:
+  %arg0.0 = extractelement <2 x i32> %arg0, i64 0
+  %arg0.1 = extractelement <2 x i32> %arg0, i64 1
+  %arg1.0 = extractelement <2 x i32> %arg1, i64 0
+  %arg1.1 = extractelement <2 x i32> %arg1, i64 1
+  %add.0 = call i32 @llvm.usub.sat.i32(i32 %arg0.0, i32 %arg1.0)
+  %add.1 = call i32 @llvm.usub.sat.i32(i32 %arg0.1, i32 %arg1.1)
+  %ins.0 = insertelement <2 x i32> undef, i32 %add.0, i64 0
+  %ins.1 = insertelement <2 x i32> %ins.0, i32 %add.1, i64 1
+  ret <2 x i32> %ins.1
+}
+
+define <2 x i32> @sadd_sat_v2i32(<2 x i32> %arg0, <2 x i32> %arg1) {
+; GCN-LABEL: @sadd_sat_v2i32(
+; GCN-NEXT:  bb:
+; GCN-NEXT:    [[TMP0:%.*]] = call <2 x i32> @llvm.sadd.sat.v2i32(<2 x i32> [[ARG0:%.*]], <2 x i32> [[ARG1:%.*]])
+; GCN-NEXT:    ret <2 x i32> [[TMP0]]
+;
+bb:
+  %arg0.0 = extractelement <2 x i32> %arg0, i64 0
+  %arg0.1 = extractelement <2 x i32> %arg0, i64 1
+  %arg1.0 = extractelement <2 x i32> %arg1, i64 0
+  %arg1.1 = extractelement <2 x i32> %arg1, i64 1
+  %add.0 = call i32 @llvm.sadd.sat.i32(i32 %arg0.0, i32 %arg1.0)
+  %add.1 = call i32 @llvm.sadd.sat.i32(i32 %arg0.1, i32 %arg1.1)
+  %ins.0 = insertelement <2 x i32> undef, i32 %add.0, i64 0
+  %ins.1 = insertelement <2 x i32> %ins.0, i32 %add.1, i64 1
+  ret <2 x i32> %ins.1
+}
+
+define <2 x i32> @ssub_sat_v2i32(<2 x i32> %arg0, <2 x i32> %arg1) {
+; GCN-LABEL: @ssub_sat_v2i32(
+; GCN-NEXT:  bb:
+; GCN-NEXT:    [[TMP0:%.*]] = call <2 x i32> @llvm.ssub.sat.v2i32(<2 x i32> [[ARG0:%.*]], <2 x i32> [[ARG1:%.*]])
+; GCN-NEXT:    ret <2 x i32> [[TMP0]]
+;
+bb:
+  %arg0.0 = extractelement <2 x i32> %arg0, i64 0
+  %arg0.1 = extractelement <2 x i32> %arg0, i64 1
+  %arg1.0 = extractelement <2 x i32> %arg1, i64 0
+  %arg1.1 = extractelement <2 x i32> %arg1, i64 1
+  %add.0 = call i32 @llvm.ssub.sat.i32(i32 %arg0.0, i32 %arg1.0)
+  %add.1 = call i32 @llvm.ssub.sat.i32(i32 %arg0.1, i32 %arg1.1)
+  %ins.0 = insertelement <2 x i32> undef, i32 %add.0, i64 0
+  %ins.1 = insertelement <2 x i32> %ins.0, i32 %add.1, i64 1
+  ret <2 x i32> %ins.1
+}
+
+define <3 x i16> @uadd_sat_v3i16(<3 x i16> %arg0, <3 x i16> %arg1) {
+; GFX7-LABEL: @uadd_sat_v3i16(
+; GFX7-NEXT:  bb:
+; GFX7-NEXT:    [[ARG0_0:%.*]] = extractelement <3 x i16> [[ARG0:%.*]], i64 0
+; GFX7-NEXT:    [[ARG0_1:%.*]] = extractelement <3 x i16> [[ARG0]], i64 1
+; GFX7-NEXT:    [[ARG0_2:%.*]] = extractelement <3 x i16> [[ARG0]], i64 2
+; GFX7-NEXT:    [[ARG1_0:%.*]] = extractelement <3 x i16> [[ARG1:%.*]], i64 0
+; GFX7-NEXT:    [[ARG1_1:%.*]] = extractelement <3 x i16> [[ARG1]], i64 1
+; GFX7-NEXT:    [[ARG1_2:%.*]] = extractelement <3 x i16> [[ARG1]], i64 2
+; GFX7-NEXT:    [[ADD_0:%.*]] = call i16 @llvm.uadd.sat.i16(i16 [[ARG0_0]], i16 [[ARG1_0]])
+; GFX7-NEXT:    [[ADD_1:%.*]] = call i16 @llvm.uadd.sat.i16(i16 [[ARG0_1]], i16 [[ARG1_1]])
+; GFX7-NEXT:    [[ADD_2:%.*]] = call i16 @llvm.uadd.sat.i16(i16 [[ARG0_2]], i16 [[ARG1_2]])
+; GFX7-NEXT:    [[INS_0:%.*]] = insertelement <3 x i16> undef, i16 [[ADD_0]], i64 0
+; GFX7-NEXT:    [[INS_1:%.*]] = insertelement <3 x i16> [[INS_0]], i16 [[ADD_1]], i64 1
+; GFX7-NEXT:    [[INS_2:%.*]] = insertelement <3 x i16> [[INS_1]], i16 [[ADD_2]], i64 2
+; GFX7-NEXT:    ret <3 x i16> [[INS_2]]
+;
+; GFX8-LABEL: @uadd_sat_v3i16(
+; GFX8-NEXT:  bb:
+; GFX8-NEXT:    [[ARG0_2:%.*]] = extractelement <3 x i16> [[ARG0:%.*]], i64 2
+; GFX8-NEXT:    [[ARG1_2:%.*]] = extractelement <3 x i16> [[ARG1:%.*]], i64 2
+; GFX8-NEXT:    [[TMP0:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> undef, <2 x i32> <i32 0, i32 1>
+; GFX8-NEXT:    [[TMP1:%.*]] = shufflevector <3 x i16> [[ARG1]], <3 x i16> undef, <2 x i32> <i32 0, i32 1>
+; GFX8-NEXT:    [[TMP2:%.*]] = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> [[TMP0]], <2 x i16> [[TMP1]])
+; GFX8-NEXT:    [[ADD_2:%.*]] = call i16 @llvm.uadd.sat.i16(i16 [[ARG0_2]], i16 [[ARG1_2]])
+; GFX8-NEXT:    [[TMP3:%.*]] = extractelement <2 x i16> [[TMP2]], i32 0
+; GFX8-NEXT:    [[INS_0:%.*]] = insertelement <3 x i16> undef, i16 [[TMP3]], i64 0
+; GFX8-NEXT:    [[TMP4:%.*]] = extractelement <2 x i16> [[TMP2]], i32 1
+; GFX8-NEXT:    [[INS_1:%.*]] = insertelement <3 x i16> [[INS_0]], i16 [[TMP4]], i64 1
+; GFX8-NEXT:    [[INS_2:%.*]] = insertelement <3 x i16> [[INS_1]], i16 [[ADD_2]], i64 2
+; GFX8-NEXT:    ret <3 x i16> [[INS_2]]
+;
+bb:
+  %arg0.0 = extractelement <3 x i16> %arg0, i64 0
+  %arg0.1 = extractelement <3 x i16> %arg0, i64 1
+  %arg0.2 = extractelement <3 x i16> %arg0, i64 2
+  %arg1.0 = extractelement <3 x i16> %arg1, i64 0
+  %arg1.1 = extractelement <3 x i16> %arg1, i64 1
+  %arg1.2 = extractelement <3 x i16> %arg1, i64 2
+  %add.0 = call i16 @llvm.uadd.sat.i16(i16 %arg0.0, i16 %arg1.0)
+  %add.1 = call i16 @llvm.uadd.sat.i16(i16 %arg0.1, i16 %arg1.1)
+  %add.2 = call i16 @llvm.uadd.sat.i16(i16 %arg0.2, i16 %arg1.2)
+  %ins.0 = insertelement <3 x i16> undef, i16 %add.0, i64 0
+  %ins.1 = insertelement <3 x i16> %ins.0, i16 %add.1, i64 1
+  %ins.2 = insertelement <3 x i16> %ins.1, i16 %add.2, i64 2
+  ret <3 x i16> %ins.2
+}
+
+define <4 x i16> @uadd_sat_v4i16(<4 x i16> %arg0, <4 x i16> %arg1) {
+; GFX7-LABEL: @uadd_sat_v4i16(
+; GFX7-NEXT:  bb:
+; GFX7-NEXT:    [[ARG0_0:%.*]] = extractelement <4 x i16> [[ARG0:%.*]], i64 0
+; GFX7-NEXT:    [[ARG0_1:%.*]] = extractelement <4 x i16> [[ARG0]], i64 1
+; GFX7-NEXT:    [[ARG0_2:%.*]] = extractelement <4 x i16> [[ARG0]], i64 2
+; GFX7-NEXT:    [[ARG0_3:%.*]] = extractelement <4 x i16> [[ARG0]], i64 3
+; GFX7-NEXT:    [[ARG1_0:%.*]] = extractelement <4 x i16> [[ARG1:%.*]], i64 0
+; GFX7-NEXT:    [[ARG1_1:%.*]] = extractelement <4 x i16> [[ARG1]], i64 1
+; GFX7-NEXT:    [[ARG1_2:%.*]] = extractelement <4 x i16> [[ARG1]], i64 2
+; GFX7-NEXT:    [[ARG1_3:%.*]] = extractelement <4 x i16> [[ARG1]], i64 3
+; GFX7-NEXT:    [[ADD_0:%.*]] = call i16 @llvm.uadd.sat.i16(i16 [[ARG0_0]], i16 [[ARG1_0]])
+; GFX7-NEXT:    [[ADD_1:%.*]] = call i16 @llvm.uadd.sat.i16(i16 [[ARG0_1]], i16 [[ARG1_1]])
+; GFX7-NEXT:    [[ADD_2:%.*]] = call i16 @llvm.uadd.sat.i16(i16 [[ARG0_2]], i16 [[ARG1_2]])
+; GFX7-NEXT:    [[ADD_3:%.*]] = call i16 @llvm.uadd.sat.i16(i16 [[ARG0_3]], i16 [[ARG1_3]])
+; GFX7-NEXT:    [[INS_0:%.*]] = insertelement <4 x i16> undef, i16 [[ADD_0]], i64 0
+; GFX7-NEXT:    [[INS_1:%.*]] = insertelement <4 x i16> [[INS_0]], i16 [[ADD_1]], i64 1
+; GFX7-NEXT:    [[INS_2:%.*]] = insertelement <4 x i16> [[INS_1]], i16 [[ADD_2]], i64 2
+; GFX7-NEXT:    [[INS_3:%.*]] = insertelement <4 x i16> [[INS_2]], i16 [[ADD_3]], i64 3
+; GFX7-NEXT:    ret <4 x i16> [[INS_3]]
+;
+; GFX8-LABEL: @uadd_sat_v4i16(
+; GFX8-NEXT:  bb:
+; GFX8-NEXT:    [[TMP0:%.*]] = call <4 x i16> @llvm.uadd.sat.v4i16(<4 x i16> [[ARG0:%.*]], <4 x i16> [[ARG1:%.*]])
+; GFX8-NEXT:    ret <4 x i16> [[TMP0]]
+;
+bb:
+  %arg0.0 = extractelement <4 x i16> %arg0, i64 0
+  %arg0.1 = extractelement <4 x i16> %arg0, i64 1
+  %arg0.2 = extractelement <4 x i16> %arg0, i64 2
+  %arg0.3 = extractelement <4 x i16> %arg0, i64 3
+  %arg1.0 = extractelement <4 x i16> %arg1, i64 0
+  %arg1.1 = extractelement <4 x i16> %arg1, i64 1
+  %arg1.2 = extractelement <4 x i16> %arg1, i64 2
+  %arg1.3 = extractelement <4 x i16> %arg1, i64 3
+  %add.0 = call i16 @llvm.uadd.sat.i16(i16 %arg0.0, i16 %arg1.0)
+  %add.1 = call i16 @llvm.uadd.sat.i16(i16 %arg0.1, i16 %arg1.1)
+  %add.2 = call i16 @llvm.uadd.sat.i16(i16 %arg0.2, i16 %arg1.2)
+  %add.3 = call i16 @llvm.uadd.sat.i16(i16 %arg0.3, i16 %arg1.3)
+  %ins.0 = insertelement <4 x i16> undef, i16 %add.0, i64 0
+  %ins.1 = insertelement <4 x i16> %ins.0, i16 %add.1, i64 1
+  %ins.2 = insertelement <4 x i16> %ins.1, i16 %add.2, i64 2
+  %ins.3 = insertelement <4 x i16> %ins.2, i16 %add.3, i64 3
+  ret <4 x i16> %ins.3
+}
+
+declare i16 @llvm.uadd.sat.i16(i16, i16) #0
+declare i16 @llvm.usub.sat.i16(i16, i16) #0
+declare i16 @llvm.sadd.sat.i16(i16, i16) #0
+declare i16 @llvm.ssub.sat.i16(i16, i16) #0
+
+declare i32 @llvm.uadd.sat.i32(i32, i32) #0
+declare i32 @llvm.usub.sat.i32(i32, i32) #0
+declare i32 @llvm.sadd.sat.i32(i32, i32) #0
+declare i32 @llvm.ssub.sat.i32(i32, i32) #0
+
+attributes #0 = { nounwind readnone speculatable willreturn }

From 200bb5191a35333281ec241b0aa976bb17295043 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Mon, 20 Jul 2020 15:56:39 -0400
Subject: [PATCH 0445/1035] AMDGPU/GlobalISel: Refactor special argument
 management

---
 llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp | 29 +++++---
 .../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 69 +++++++------------
 llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h  |  8 +--
 3 files changed, 46 insertions(+), 60 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp
index a72d100b13075..9bf3653006006 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp
@@ -993,7 +993,7 @@ bool AMDGPUCallLowering::passSpecialInputs(MachineIRBuilder &MIRBuilder,
     Register InputReg = MRI.createGenericVirtualRegister(ArgTy);
 
     if (IncomingArg) {
-      LI->loadInputValue(InputReg, MIRBuilder, IncomingArg);
+      LI->loadInputValue(InputReg, MIRBuilder, IncomingArg, ArgRC, ArgTy);
     } else {
       assert(InputID == AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR);
       LI->getImplicitArgPtr(InputReg, MRI, MIRBuilder);
@@ -1026,13 +1026,16 @@ bool AMDGPUCallLowering::passSpecialInputs(MachineIRBuilder &MIRBuilder,
   if (!OutgoingArg)
     return false;
 
-  const ArgDescriptor *IncomingArgX = std::get<0>(
-      CallerArgInfo.getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_X));
-  const ArgDescriptor *IncomingArgY = std::get<0>(
-      CallerArgInfo.getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_Y));
-  const ArgDescriptor *IncomingArgZ = std::get<0>(
-      CallerArgInfo.getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_Z));
+  auto WorkitemIDX =
+      CallerArgInfo.getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_X);
+  auto WorkitemIDY =
+      CallerArgInfo.getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_Y);
+  auto WorkitemIDZ =
+      CallerArgInfo.getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_Z);
 
+  const ArgDescriptor *IncomingArgX = std::get<0>(WorkitemIDX);
+  const ArgDescriptor *IncomingArgY = std::get<0>(WorkitemIDY);
+  const ArgDescriptor *IncomingArgZ = std::get<0>(WorkitemIDZ);
   const LLT S32 = LLT::scalar(32);
 
   // If incoming ids are not packed we need to pack them.
@@ -1040,12 +1043,14 @@ bool AMDGPUCallLowering::passSpecialInputs(MachineIRBuilder &MIRBuilder,
   Register InputReg;
   if (IncomingArgX && !IncomingArgX->isMasked() && CalleeArgInfo->WorkItemIDX) {
     InputReg = MRI.createGenericVirtualRegister(S32);
-    LI->loadInputValue(InputReg, MIRBuilder, IncomingArgX);
+    LI->loadInputValue(InputReg, MIRBuilder, IncomingArgX,
+                       std::get<1>(WorkitemIDX), std::get<2>(WorkitemIDX));
   }
 
   if (IncomingArgY && !IncomingArgY->isMasked() && CalleeArgInfo->WorkItemIDY) {
     Register Y = MRI.createGenericVirtualRegister(S32);
-    LI->loadInputValue(Y, MIRBuilder, IncomingArgY);
+    LI->loadInputValue(Y, MIRBuilder, IncomingArgY, std::get<1>(WorkitemIDY),
+                       std::get<2>(WorkitemIDY));
 
     Y = MIRBuilder.buildShl(S32, Y, MIRBuilder.buildConstant(S32, 10)).getReg(0);
     InputReg = InputReg ? MIRBuilder.buildOr(S32, InputReg, Y).getReg(0) : Y;
@@ -1053,7 +1058,8 @@ bool AMDGPUCallLowering::passSpecialInputs(MachineIRBuilder &MIRBuilder,
 
   if (IncomingArgZ && !IncomingArgZ->isMasked() && CalleeArgInfo->WorkItemIDZ) {
     Register Z = MRI.createGenericVirtualRegister(S32);
-    LI->loadInputValue(Z, MIRBuilder, IncomingArgZ);
+    LI->loadInputValue(Z, MIRBuilder, IncomingArgZ, std::get<1>(WorkitemIDZ),
+                       std::get<2>(WorkitemIDZ));
 
     Z = MIRBuilder.buildShl(S32, Z, MIRBuilder.buildConstant(S32, 20)).getReg(0);
     InputReg = InputReg ? MIRBuilder.buildOr(S32, InputReg, Z).getReg(0) : Z;
@@ -1067,7 +1073,8 @@ bool AMDGPUCallLowering::passSpecialInputs(MachineIRBuilder &MIRBuilder,
     ArgDescriptor IncomingArg = ArgDescriptor::createArg(
       IncomingArgX ? *IncomingArgX :
         IncomingArgY ? *IncomingArgY : *IncomingArgZ, ~0u);
-    LI->loadInputValue(InputReg, MIRBuilder, &IncomingArg);
+    LI->loadInputValue(InputReg, MIRBuilder, &IncomingArg,
+                       &AMDGPU::VGPR_32RegClass, S32);
   }
 
   if (OutgoingArg->isRegister()) {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index aa437638cec59..21e3a168738ad 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -1630,8 +1630,7 @@ Register AMDGPULegalizerInfo::getSegmentAperture(
   Register QueuePtr = MRI.createGenericVirtualRegister(
     LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64));
 
-  const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
-  if (!loadInputValue(QueuePtr, B, &MFI->getArgInfo().QueuePtr))
+  if (!loadInputValue(QueuePtr, B, AMDGPUFunctionArgInfo::QUEUE_PTR))
     return Register();
 
   // Offset into amd_queue_t for group_segment_aperture_base_hi /
@@ -2496,33 +2495,16 @@ Register AMDGPULegalizerInfo::getLiveInRegister(MachineIRBuilder &B,
   return insertLiveInCopy(B, MRI, LiveIn, PhyReg);
 }
 
-const ArgDescriptor *AMDGPULegalizerInfo::getArgDescriptor(
-    MachineIRBuilder &B, AMDGPUFunctionArgInfo::PreloadedValue ArgType) const {
-  const SIMachineFunctionInfo *MFI = B.getMF().getInfo<SIMachineFunctionInfo>();
-  const ArgDescriptor *Arg;
-  const TargetRegisterClass *RC;
-  LLT ArgTy;
-  std::tie(Arg, RC, ArgTy) = MFI->getPreloadedValue(ArgType);
-  if (!Arg) {
-    LLVM_DEBUG(dbgs() << "Required arg register missing\n");
-    return nullptr;
-  }
-  return Arg;
-}
-
 bool AMDGPULegalizerInfo::loadInputValue(Register DstReg, MachineIRBuilder &B,
-                                         const ArgDescriptor *Arg) const {
-  if (!Arg->isRegister() || !Arg->getRegister().isValid())
-    return false; // TODO: Handle these
-
-  Register SrcReg = Arg->getRegister();
+                                         const ArgDescriptor *Arg,
+                                         const TargetRegisterClass *ArgRC,
+                                         LLT ArgTy) const {
+  MCRegister SrcReg = Arg->getRegister();
   assert(SrcReg.isPhysical() && "Physical register expected");
   assert(DstReg.isVirtual() && "Virtual register expected");
 
   MachineRegisterInfo &MRI = *B.getMRI();
-
-  LLT Ty = MRI.getType(DstReg);
-  Register LiveIn = getLiveInRegister(B, MRI, SrcReg, Ty);
+  Register LiveIn = getLiveInRegister(B, MRI, SrcReg, ArgTy);
 
   if (Arg->isMasked()) {
     // TODO: Should we try to emit this once in the entry block?
@@ -2545,15 +2527,24 @@ bool AMDGPULegalizerInfo::loadInputValue(Register DstReg, MachineIRBuilder &B,
   return true;
 }
 
-bool AMDGPULegalizerInfo::legalizePreloadedArgIntrin(
-    MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B,
+bool AMDGPULegalizerInfo::loadInputValue(
+    Register DstReg, MachineIRBuilder &B,
     AMDGPUFunctionArgInfo::PreloadedValue ArgType) const {
+  const SIMachineFunctionInfo *MFI = B.getMF().getInfo<SIMachineFunctionInfo>();
+  const ArgDescriptor *Arg;
+  const TargetRegisterClass *ArgRC;
+  LLT ArgTy;
+  std::tie(Arg, ArgRC, ArgTy) = MFI->getPreloadedValue(ArgType);
 
-  const ArgDescriptor *Arg = getArgDescriptor(B, ArgType);
-  if (!Arg)
-    return false;
+  if (!Arg->isRegister() || !Arg->getRegister().isValid())
+    return false; // TODO: Handle these
+  return loadInputValue(DstReg, B, Arg, ArgRC, ArgTy);
+}
 
-  if (!loadInputValue(MI.getOperand(0).getReg(), B, Arg))
+bool AMDGPULegalizerInfo::legalizePreloadedArgIntrin(
+    MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B,
+    AMDGPUFunctionArgInfo::PreloadedValue ArgType) const {
+  if (!loadInputValue(MI.getOperand(0).getReg(), B, ArgType))
     return false;
 
   MI.eraseFromParent();
@@ -3165,23 +3156,15 @@ bool AMDGPULegalizerInfo::legalizeFDIVFastIntrin(MachineInstr &MI,
 bool AMDGPULegalizerInfo::getImplicitArgPtr(Register DstReg,
                                             MachineRegisterInfo &MRI,
                                             MachineIRBuilder &B) const {
-  const SIMachineFunctionInfo *MFI = B.getMF().getInfo<SIMachineFunctionInfo>();
   uint64_t Offset =
     ST.getTargetLowering()->getImplicitParameterOffset(
       B.getMF(), AMDGPUTargetLowering::FIRST_IMPLICIT);
   LLT DstTy = MRI.getType(DstReg);
   LLT IdxTy = LLT::scalar(DstTy.getSizeInBits());
 
-  const ArgDescriptor *Arg;
-  const TargetRegisterClass *RC;
-  LLT ArgTy;
-  std::tie(Arg, RC, ArgTy) =
-      MFI->getPreloadedValue(AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR);
-  if (!Arg)
-    return false;
-
   Register KernargPtrReg = MRI.createGenericVirtualRegister(DstTy);
-  if (!loadInputValue(KernargPtrReg, B, Arg))
+  if (!loadInputValue(KernargPtrReg, B,
+                      AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR))
     return false;
 
   // FIXME: This should be nuw
@@ -4169,16 +4152,12 @@ bool AMDGPULegalizerInfo::legalizeTrapIntrinsic(MachineInstr &MI,
   } else {
     // Pass queue pointer to trap handler as input, and insert trap instruction
     // Reference: https://llvm.org/docs/AMDGPUUsage.html#trap-handler-abi
-    const ArgDescriptor *Arg =
-        getArgDescriptor(B, AMDGPUFunctionArgInfo::QUEUE_PTR);
-    if (!Arg)
-      return false;
     MachineRegisterInfo &MRI = *B.getMRI();
     Register SGPR01(AMDGPU::SGPR0_SGPR1);
     Register LiveIn = getLiveInRegister(
         B, MRI, SGPR01, LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64),
         /*InsertLiveInCopy=*/false);
-    if (!loadInputValue(LiveIn, B, Arg))
+    if (!loadInputValue(LiveIn, B, AMDGPUFunctionArgInfo::QUEUE_PTR))
       return false;
     B.buildCopy(SGPR01, LiveIn);
     B.buildInstr(AMDGPU::S_TRAP)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h
index 43be02c661ebe..fe4e17db48a6b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h
@@ -91,11 +91,11 @@ class AMDGPULegalizerInfo final : public LegalizerInfo {
                              bool InsertLiveInCopy = true) const;
   Register insertLiveInCopy(MachineIRBuilder &B, MachineRegisterInfo &MRI,
                             Register LiveIn, Register PhyReg) const;
-  const ArgDescriptor *
-  getArgDescriptor(MachineIRBuilder &B,
-                   AMDGPUFunctionArgInfo::PreloadedValue ArgType) const;
   bool loadInputValue(Register DstReg, MachineIRBuilder &B,
-                      const ArgDescriptor *Arg) const;
+                      const ArgDescriptor *Arg,
+                      const TargetRegisterClass *ArgRC, LLT ArgTy) const;
+  bool loadInputValue(Register DstReg, MachineIRBuilder &B,
+                      AMDGPUFunctionArgInfo::PreloadedValue ArgType) const;
   bool legalizePreloadedArgIntrin(
     MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B,
     AMDGPUFunctionArgInfo::PreloadedValue ArgType) const;

From 85342c27a303e68a4091797e5d7938aa7beb4da5 Mon Sep 17 00:00:00 2001
From: Sjoerd Meijer <sjoerd.meijer@arm.com>
Date: Wed, 29 Jul 2020 13:13:04 +0100
Subject: [PATCH 0446/1035] [ARM] Optimize immediate selection

Optimize some specific immediates selection by materializing them with sub/mvn
instructions as opposed to loading them from the constant pool.

Patch by Ben Shi, powerman1st@163.com.

Differential Revision: https://reviews.llvm.org/D83745
---
 llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp      |  2 +
 llvm/lib/Target/ARM/ARMExpandPseudoInsts.cpp  | 27 ++++---
 llvm/lib/Target/ARM/ARMInstrInfo.td           |  4 +-
 .../ARM/MCTargetDesc/ARMAddressingModes.h     | 14 ++++
 llvm/test/CodeGen/ARM/add-sub-imm.ll          | 70 +++++++++++++++++++
 llvm/test/CodeGen/ARM/select-imm.ll           |  3 +-
 6 files changed, 110 insertions(+), 10 deletions(-)
 create mode 100644 llvm/test/CodeGen/ARM/add-sub-imm.ll

diff --git a/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp b/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
index d340931f31d8d..b501dc06ca72f 100644
--- a/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
@@ -5513,6 +5513,8 @@ unsigned llvm::ConstantMaterializationCost(unsigned Val,
       return ForCodesize ? 4 : 1;
     if (ARM_AM::isSOImmTwoPartVal(Val)) // two instrs
       return ForCodesize ? 8 : 2;
+    if (ARM_AM::isSOImmTwoPartValNeg(Val)) // two instrs
+      return ForCodesize ? 8 : 2;
   }
   if (Subtarget->useMovt()) // MOVW + MOVT
     return ForCodesize ? 8 : 2;
diff --git a/llvm/lib/Target/ARM/ARMExpandPseudoInsts.cpp b/llvm/lib/Target/ARM/ARMExpandPseudoInsts.cpp
index ec72c2b5ac194..9be0b6fa33e93 100644
--- a/llvm/lib/Target/ARM/ARMExpandPseudoInsts.cpp
+++ b/llvm/lib/Target/ARM/ARMExpandPseudoInsts.cpp
@@ -873,16 +873,27 @@ void ARMExpandPseudo::ExpandMOV32BitImm(MachineBasicBlock &MBB,
     // FIXME Windows CE supports older ARM CPUs
     assert(!STI->isTargetWindows() && "Windows on ARM requires ARMv7+");
 
-    // Expand into a movi + orr.
-    LO16 = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::MOVi), DstReg);
-    HI16 = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::ORRri))
-      .addReg(DstReg, RegState::Define | getDeadRegState(DstIsDead))
-      .addReg(DstReg);
-
     assert (MO.isImm() && "MOVi32imm w/ non-immediate source operand!");
     unsigned ImmVal = (unsigned)MO.getImm();
-    unsigned SOImmValV1 = ARM_AM::getSOImmTwoPartFirst(ImmVal);
-    unsigned SOImmValV2 = ARM_AM::getSOImmTwoPartSecond(ImmVal);
+    unsigned SOImmValV1 = 0, SOImmValV2 = 0;
+
+    if (ARM_AM::isSOImmTwoPartVal(ImmVal)) { // Expand into a movi + orr.
+      LO16 = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::MOVi), DstReg);
+      HI16 = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::ORRri))
+          .addReg(DstReg, RegState::Define | getDeadRegState(DstIsDead))
+          .addReg(DstReg);
+      SOImmValV1 = ARM_AM::getSOImmTwoPartFirst(ImmVal);
+      SOImmValV2 = ARM_AM::getSOImmTwoPartSecond(ImmVal);
+    } else { // Expand into a mvn + sub.
+      LO16 = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::MVNi), DstReg);
+      HI16 = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::SUBri))
+          .addReg(DstReg, RegState::Define | getDeadRegState(DstIsDead))
+          .addReg(DstReg);
+      SOImmValV1 = ARM_AM::getSOImmTwoPartFirst(-ImmVal);
+      SOImmValV2 = ARM_AM::getSOImmTwoPartSecond(-ImmVal);
+      SOImmValV1 = ~(-SOImmValV1);
+    }
+
     unsigned MIFlags = MI.getFlags();
     LO16 = LO16.addImm(SOImmValV1);
     HI16 = HI16.addImm(SOImmValV2);
diff --git a/llvm/lib/Target/ARM/ARMInstrInfo.td b/llvm/lib/Target/ARM/ARMInstrInfo.td
index db2ab7a97aa66..d0c1e14989d32 100644
--- a/llvm/lib/Target/ARM/ARMInstrInfo.td
+++ b/llvm/lib/Target/ARM/ARMInstrInfo.td
@@ -824,7 +824,9 @@ def mod_imm_neg : Operand<i32>, PatLeaf<(imm), [{
 def arm_i32imm : IntImmLeaf<i32, [{
   if (Subtarget->useMovt())
     return true;
-  return ARM_AM::isSOImmTwoPartVal(Imm.getZExtValue());
+  if (ARM_AM::isSOImmTwoPartVal(Imm.getZExtValue()))
+    return true;
+  return ARM_AM::isSOImmTwoPartValNeg(Imm.getZExtValue());
 }]>;
 
 /// imm0_1 predicate - Immediate in the range [0,1].
diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMAddressingModes.h b/llvm/lib/Target/ARM/MCTargetDesc/ARMAddressingModes.h
index 24a9fabf09798..dca252a177f25 100644
--- a/llvm/lib/Target/ARM/MCTargetDesc/ARMAddressingModes.h
+++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMAddressingModes.h
@@ -205,6 +205,20 @@ namespace ARM_AM {
     return V;
   }
 
+  /// isSOImmTwoPartValNeg - Return true if the specified value can be obtained
+  /// by two SOImmVal, that -V = First + Second.
+  /// "R+V" can be optimized to (sub (sub R, First), Second).
+  /// "R=V" can be optimized to (sub (mvn R, ~(-First)), Second).
+  inline bool isSOImmTwoPartValNeg(unsigned V) {
+    unsigned First;
+    if (!isSOImmTwoPartVal(-V))
+      return false;
+    // Return false if ~(-First) is not a SoImmval.
+    First = getSOImmTwoPartFirst(-V);
+    First = ~(-First);
+    return !(rotr32(~255U, getSOImmValRotate(First)) & First);
+  }
+
   /// getThumbImmValShift - Try to handle Imm with a 8-bit immediate followed
   /// by a left shift. Returns the shift amount to use.
   inline unsigned getThumbImmValShift(unsigned Imm) {
diff --git a/llvm/test/CodeGen/ARM/add-sub-imm.ll b/llvm/test/CodeGen/ARM/add-sub-imm.ll
new file mode 100644
index 0000000000000..71a860537ba1b
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/add-sub-imm.ll
@@ -0,0 +1,70 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=arm-eabi %s -o - | FileCheck %s --check-prefix=CHECK
+
+;; Check how immediates are handled in add/sub.
+
+define i32 @sub0(i32 %0) {
+; CHECK-LABEL: sub0:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    sub r0, r0, #23
+; CHECK-NEXT:    mov pc, lr
+  %2 = sub i32 %0, 23
+  ret i32 %2
+}
+
+define i32 @sub1(i32 %0) {
+; CHECK-LABEL: sub1:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    ldr r1, .LCPI1_0
+; CHECK-NEXT:    add r0, r0, r1
+; CHECK-NEXT:    mov pc, lr
+; CHECK-NEXT:    .p2align 2
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI1_0:
+; CHECK-NEXT:    .long 4294836225 @ 0xfffe0001
+  %2 = sub i32 %0, 131071
+  ret i32 %2
+}
+
+define i32 @sub2(i32 %0) {
+; CHECK-LABEL: sub2:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    sub r0, r0, #35
+; CHECK-NEXT:    sub r0, r0, #8960
+; CHECK-NEXT:    mov pc, lr
+  %2 = sub i32 %0, 8995
+  ret i32 %2
+}
+
+define i32 @add0(i32 %0) {
+; CHECK-LABEL: add0:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    add r0, r0, #23
+; CHECK-NEXT:    mov pc, lr
+  %2 = add i32 %0, 23
+  ret i32 %2
+}
+
+define i32 @add1(i32 %0) {
+; CHECK-LABEL: add1:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    ldr r1, .LCPI4_0
+; CHECK-NEXT:    add r0, r0, r1
+; CHECK-NEXT:    mov pc, lr
+; CHECK-NEXT:    .p2align 2
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI4_0:
+; CHECK-NEXT:    .long 131071 @ 0x1ffff
+  %2 = add i32 %0, 131071
+  ret i32 %2
+}
+
+define i32 @add2(i32 %0) {
+; CHECK-LABEL: add2:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    add r0, r0, #8960
+; CHECK-NEXT:    add r0, r0, #2293760
+; CHECK-NEXT:    mov pc, lr
+  %2 = add i32 %0, 2302720
+  ret i32 %2
+}
diff --git a/llvm/test/CodeGen/ARM/select-imm.ll b/llvm/test/CodeGen/ARM/select-imm.ll
index 628bae6e1dc93..5cdfe3326e922 100644
--- a/llvm/test/CodeGen/ARM/select-imm.ll
+++ b/llvm/test/CodeGen/ARM/select-imm.ll
@@ -85,7 +85,8 @@ entry:
 define i32 @t4(i32 %a, i32 %b, i32 %x) nounwind {
 entry:
 ; ARM-LABEL: t4:
-; ARM: ldr
+; ARM: mvn [[R0:r[0-9]+]], #170
+; ARM: sub [[R0:r[0-9]+]], [[R0:r[0-9]+]], #11141120
 ; ARM: mov{{lt|ge}}
 
 ; ARMT2-LABEL: t4:

From 207877175944656bd9b52d36f391a092854572be Mon Sep 17 00:00:00 2001
From: David Sherwood <david.sherwood@arm.com>
Date: Thu, 16 Jul 2020 11:29:25 +0100
Subject: [PATCH 0447/1035] [SVE][CodeGen] Add simple integer add tests for SVE
 tuple types

I have added tests to:

  CodeGen/AArch64/sve-intrinsics-int-arith.ll

for doing simple integer add operations on tuple types. Since these
tests introduced new warnings due to incorrect use of
getVectorNumElements() I have also fixed up these warnings in the
same patch. These fixes are:

1. In narrowExtractedVectorBinOp I have changed the code to bail out
early for scalable vector types, since we've not yet hit a case that
proves the optimisations are profitable for scalable vectors.
2. In DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS I have replaced
calls to getVectorNumElements with getVectorMinNumElements in cases
that work with scalable vectors. For the other cases I have added
asserts that the vector is not scalable because we should not be
using shuffle vectors and build vectors in such cases.

Differential revision: https://reviews.llvm.org/D84016
---
 llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp |  5 ++-
 .../SelectionDAG/LegalizeVectorTypes.cpp      | 19 +++++++---
 .../AArch64/sve-intrinsics-int-arith.ll       | 37 +++++++++++++++++++
 3 files changed, 55 insertions(+), 6 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 8932503b9564b..8a5a7f098bd74 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -19194,7 +19194,10 @@ static SDValue narrowExtractedVectorBinOp(SDNode *Extract, SelectionDAG &DAG) {
 
   // The binop must be a vector type, so we can extract some fraction of it.
   EVT WideBVT = BinOp.getValueType();
-  if (!WideBVT.isVector())
+  // The optimisations below currently assume we are dealing with fixed length
+  // vectors. It is possible to add support for scalable vectors, but at the
+  // moment we've done no analysis to prove whether they are profitable or not.
+  if (!WideBVT.isFixedLengthVector())
     return SDValue();
 
   EVT VT = Extract->getValueType(0);
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index 6963de2e50290..c81d03cac81bc 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -3610,16 +3610,15 @@ SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) {
   EVT InVT = N->getOperand(0).getValueType();
   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
   SDLoc dl(N);
-  unsigned WidenNumElts = WidenVT.getVectorNumElements();
-  unsigned NumInElts = InVT.getVectorNumElements();
   unsigned NumOperands = N->getNumOperands();
 
   bool InputWidened = false; // Indicates we need to widen the input.
   if (getTypeAction(InVT) != TargetLowering::TypeWidenVector) {
-    if (WidenVT.getVectorNumElements() % InVT.getVectorNumElements() == 0) {
+    unsigned WidenNumElts = WidenVT.getVectorMinNumElements();
+    unsigned NumInElts = InVT.getVectorMinNumElements();
+    if (WidenNumElts % NumInElts == 0) {
       // Add undef vectors to widen to correct length.
-      unsigned NumConcat = WidenVT.getVectorNumElements() /
-                           InVT.getVectorNumElements();
+      unsigned NumConcat = WidenNumElts / NumInElts;
       SDValue UndefVal = DAG.getUNDEF(InVT);
       SmallVector<SDValue, 16> Ops(NumConcat);
       for (unsigned i=0; i < NumOperands; ++i)
@@ -3643,6 +3642,11 @@ SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) {
         return GetWidenedVector(N->getOperand(0));
 
       if (NumOperands == 2) {
+        assert(!WidenVT.isScalableVector() &&
+               "Cannot use vector shuffles to widen CONCAT_VECTOR result");
+        unsigned WidenNumElts = WidenVT.getVectorNumElements();
+        unsigned NumInElts = InVT.getVectorNumElements();
+
         // Replace concat of two operands with a shuffle.
         SmallVector<int, 16> MaskOps(WidenNumElts, -1);
         for (unsigned i = 0; i < NumInElts; ++i) {
@@ -3657,6 +3661,11 @@ SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) {
     }
   }
 
+  assert(!WidenVT.isScalableVector() &&
+         "Cannot use build vectors to widen CONCAT_VECTOR result");
+  unsigned WidenNumElts = WidenVT.getVectorNumElements();
+  unsigned NumInElts = InVT.getVectorNumElements();
+
   // Fall back to use extracts and build vector.
   EVT EltVT = WidenVT.getVectorElementType();
   SmallVector<SDValue, 16> Ops(WidenNumElts);
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-int-arith.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-int-arith.ll
index 8a5d669e4241e..fa67d92c2ae0e 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-int-arith.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-int-arith.ll
@@ -325,6 +325,39 @@ define <vscale x 2 x i64> @uqsub_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %
   ret <vscale x 2 x i64> %out
 }
 
+; ADD (tuples)
+
+define <vscale x 4 x i64> @add_i64_tuple2(<vscale x 4 x i64>* %out, <vscale x 2 x i64> %in1, <vscale x 2 x i64> %in2) {
+; CHECK-LABEL: add_i64_tuple2
+; CHECK: add z0.d, z0.d, z0.d
+; CHECK: add z1.d, z1.d, z1.d
+  %tuple = tail call <vscale x 4 x i64> @llvm.aarch64.sve.tuple.create2.nxv4i64.nxv2i64(<vscale x 2 x i64> %in1, <vscale x 2 x i64> %in2)
+  %res = add <vscale x 4 x i64> %tuple, %tuple
+  ret <vscale x 4 x i64> %res
+}
+
+define <vscale x 6 x i64> @add_i64_tuple3(<vscale x 6 x i64>* %out, <vscale x 2 x i64> %in1, <vscale x 2 x i64> %in2, <vscale x 2 x i64> %in3) {
+; CHECK-LABEL: add_i64_tuple3
+; CHECK: add z0.d, z0.d, z0.d
+; CHECK: add z1.d, z1.d, z1.d
+; CHECK: add z2.d, z2.d, z2.d
+  %tuple = tail call <vscale x 6 x i64> @llvm.aarch64.sve.tuple.create3.nxv6i64.nxv2i64(<vscale x 2 x i64> %in1, <vscale x 2 x i64> %in2, <vscale x 2 x i64> %in3)
+  %res = add <vscale x 6 x i64> %tuple, %tuple
+  ret <vscale x 6 x i64> %res
+}
+
+define <vscale x 8 x i64> @add_i64_tuple4(<vscale x 8 x i64>* %out, <vscale x 2 x i64> %in1, <vscale x 2 x i64> %in2, <vscale x 2 x i64> %in3, <vscale x 2 x i64> %in4) {
+; CHECK-LABEL: add_i64_tuple4
+; CHECK: add z0.d, z0.d, z0.d
+; CHECK: add z1.d, z1.d, z1.d
+; CHECK: add z2.d, z2.d, z2.d
+; CHECK: add z3.d, z3.d, z3.d
+  %tuple = tail call <vscale x 8 x i64> @llvm.aarch64.sve.tuple.create4.nxv8i64.nxv2i64(<vscale x 2 x i64> %in1, <vscale x 2 x i64> %in2, <vscale x 2 x i64> %in3, <vscale x 2 x i64> %in4)
+  %res = add <vscale x 8 x i64> %tuple, %tuple
+  ret <vscale x 8 x i64> %res
+}
+
+
 declare <vscale x 16 x i8> @llvm.aarch64.sve.abs.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i1>, <vscale x 16 x i8>)
 declare <vscale x 8 x i16> @llvm.aarch64.sve.abs.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i1>, <vscale x 8 x i16>)
 declare <vscale x 4 x i32> @llvm.aarch64.sve.abs.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i1>, <vscale x 4 x i32>)
@@ -366,3 +399,7 @@ declare <vscale x 16 x i8> @llvm.aarch64.sve.uqsub.x.nxv16i8(<vscale x 16 x i8>,
 declare <vscale x 8 x i16> @llvm.aarch64.sve.uqsub.x.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>)
 declare <vscale x 4 x i32> @llvm.aarch64.sve.uqsub.x.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>)
 declare <vscale x 2 x i64> @llvm.aarch64.sve.uqsub.x.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>)
+
+declare <vscale x 4 x i64> @llvm.aarch64.sve.tuple.create2.nxv4i64.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>)
+declare <vscale x 6 x i64> @llvm.aarch64.sve.tuple.create3.nxv6i64.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>)
+declare <vscale x 8 x i64> @llvm.aarch64.sve.tuple.create4.nxv8i64.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>)

From 60280e9818a6274d6e16556e4392821bbb65bd31 Mon Sep 17 00:00:00 2001
From: David Green <david.green@arm.com>
Date: Wed, 29 Jul 2020 13:32:53 +0100
Subject: [PATCH 0448/1035] [Analysis] TTI: Add CastContextHint for
 getCastInstrCost

Currently, getCastInstrCost has limited information about the cast it's
rating, often just the opcode and types.  Sometimes there is a context
instruction as well, but it isn't trustworthy: for instance, when the
vectorizer is rating a plan, it calls getCastInstrCost with the old
instructions when, in fact, it's trying to evaluate the cost of the
instruction post-vectorization.  Thus, the current system can get the
cost of certain casts incorrect as the correct cost can vary greatly
based on the context in which it's used.

For example, if the vectorizer queries getCastInstrCost to evaluate the
cost of a sext(load) with tail predication enabled, getCastInstrCost
will think it's free most of the time, but it's not always free. On ARM
MVE, a VLD2 group cannot be extended like a normal VLDR can. Similar
situations can come up with how masked loads can be extended when being
split.

To fix that, this path adds a new parameter to getCastInstrCost to give
it a hint about the context of the cast. It adds a CastContextHint enum
which contains the type of the load/store being created by the
vectorizer - one for each of the types it can produce.

Original patch by Pierre van Houtryve

Differential Revision: https://reviews.llvm.org/D79162
---
 .../llvm/Analysis/TargetTransformInfo.h       | 42 ++++++++++++++-
 .../llvm/Analysis/TargetTransformInfoImpl.h   |  4 +-
 llvm/include/llvm/CodeGen/BasicTTIImpl.h      | 26 +++++-----
 llvm/lib/Analysis/TargetTransformInfo.cpp     | 47 ++++++++++++++++-
 .../AArch64/AArch64TargetTransformInfo.cpp    | 16 ++++--
 .../AArch64/AArch64TargetTransformInfo.h      |  2 +-
 .../lib/Target/ARM/ARMTargetTransformInfo.cpp | 14 ++---
 llvm/lib/Target/ARM/ARMTargetTransformInfo.h  |  2 +-
 .../Hexagon/HexagonTargetTransformInfo.cpp    |  4 +-
 .../Hexagon/HexagonTargetTransformInfo.h      |  5 +-
 .../Target/PowerPC/PPCTargetTransformInfo.cpp |  3 +-
 .../Target/PowerPC/PPCTargetTransformInfo.h   |  2 +-
 .../SystemZ/SystemZTargetTransformInfo.cpp    |  9 ++--
 .../SystemZ/SystemZTargetTransformInfo.h      |  2 +-
 .../lib/Target/X86/X86TargetTransformInfo.cpp |  6 ++-
 llvm/lib/Target/X86/X86TargetTransformInfo.h  |  2 +-
 .../Scalar/RewriteStatepointsForGC.cpp        |  4 +-
 .../Utils/ScalarEvolutionExpander.cpp         |  5 +-
 .../Transforms/Vectorize/LoopVectorize.cpp    | 52 ++++++++++++++++---
 .../Transforms/Vectorize/SLPVectorizer.cpp    | 19 +++----
 20 files changed, 204 insertions(+), 62 deletions(-)

diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index e1426e3e5192a..092c9744a21bc 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -1021,10 +1021,47 @@ class TargetTransformInfo {
   int getShuffleCost(ShuffleKind Kind, VectorType *Tp, int Index = 0,
                      VectorType *SubTp = nullptr) const;
 
+  /// Represents a hint about the context in which a cast is used.
+  ///
+  /// For zext/sext, the context of the cast is the operand, which must be a
+  /// load of some kind. For trunc, the context is of the cast is the single
+  /// user of the instruction, which must be a store of some kind.
+  ///
+  /// This enum allows the vectorizer to give getCastInstrCost an idea of the
+  /// type of cast it's dealing with, as not every cast is equal. For instance,
+  /// the zext of a load may be free, but the zext of an interleaving load can
+  //// be (very) expensive!
+  ///
+  /// See \c getCastContextHint to compute a CastContextHint from a cast
+  /// Instruction*. Callers can use it if they don't need to override the
+  /// context and just want it to be calculated from the instruction.
+  ///
+  /// FIXME: This handles the types of load/store that the vectorizer can
+  /// produce, which are the cases where the context instruction is most
+  /// likely to be incorrect. There are other situations where that can happen
+  /// too, which might be handled here but in the long run a more general
+  /// solution of costing multiple instructions at the same times may be better.
+  enum class CastContextHint : uint8_t {
+    None,          ///< The cast is not used with a load/store of any kind.
+    Normal,        ///< The cast is used with a normal load/store.
+    Masked,        ///< The cast is used with a masked load/store.
+    GatherScatter, ///< The cast is used with a gather/scatter.
+    Interleave,    ///< The cast is used with an interleaved load/store.
+    Reversed,      ///< The cast is used with a reversed load/store.
+  };
+
+  /// Calculates a CastContextHint from \p I.
+  /// This should be used by callers of getCastInstrCost if they wish to
+  /// determine the context from some instruction.
+  /// \returns the CastContextHint for ZExt/SExt/Trunc, None if \p I is nullptr,
+  /// or if it's another type of cast.
+  static CastContextHint getCastContextHint(const Instruction *I);
+
   /// \return The expected cost of cast instructions, such as bitcast, trunc,
   /// zext, etc. If there is an existing instruction that holds Opcode, it
   /// may be passed in the 'I' parameter.
   int getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
+                       TTI::CastContextHint CCH,
                        TTI::TargetCostKind CostKind = TTI::TCK_SizeAndLatency,
                        const Instruction *I = nullptr) const;
 
@@ -1454,6 +1491,7 @@ class TargetTransformInfo::Concept {
   virtual int getShuffleCost(ShuffleKind Kind, VectorType *Tp, int Index,
                              VectorType *SubTp) = 0;
   virtual int getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
+                               CastContextHint CCH,
                                TTI::TargetCostKind CostKind,
                                const Instruction *I) = 0;
   virtual int getExtractWithExtendCost(unsigned Opcode, Type *Dst,
@@ -1882,9 +1920,9 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
     return Impl.getShuffleCost(Kind, Tp, Index, SubTp);
   }
   int getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
-                       TTI::TargetCostKind CostKind,
+                       CastContextHint CCH, TTI::TargetCostKind CostKind,
                        const Instruction *I) override {
-    return Impl.getCastInstrCost(Opcode, Dst, Src, CostKind, I);
+    return Impl.getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
   }
   int getExtractWithExtendCost(unsigned Opcode, Type *Dst, VectorType *VecTy,
                                unsigned Index) override {
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 73e5ff60a4be6..4dc0a90cc9db1 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -423,6 +423,7 @@ class TargetTransformInfoImplBase {
   }
 
   unsigned getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
+                            TTI::CastContextHint CCH,
                             TTI::TargetCostKind CostKind,
                             const Instruction *I) {
     switch (Opcode) {
@@ -915,7 +916,8 @@ class TargetTransformInfoImplCRTPBase : public TargetTransformInfoImplBase {
     case Instruction::SExt:
     case Instruction::ZExt:
     case Instruction::AddrSpaceCast:
-      return TargetTTI->getCastInstrCost(Opcode, Ty, OpTy, CostKind, I);
+      return TargetTTI->getCastInstrCost(
+          Opcode, Ty, OpTy, TTI::getCastContextHint(I), CostKind, I);
     case Instruction::Store: {
       auto *SI = cast<StoreInst>(U);
       Type *ValTy = U->getOperand(0)->getType();
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index aabfb74b7cefc..e9f6329797057 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -716,9 +716,10 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
   }
 
   unsigned getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
+                            TTI::CastContextHint CCH,
                             TTI::TargetCostKind CostKind,
                             const Instruction *I = nullptr) {
-    if (BaseT::getCastInstrCost(Opcode, Dst, Src, CostKind, I) == 0)
+    if (BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I) == 0)
       return 0;
 
     const TargetLoweringBase *TLI = getTLI();
@@ -756,15 +757,12 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
         return 0;
       LLVM_FALLTHROUGH;
     case Instruction::SExt:
-      if (!I)
-        break;
-
-      if (getTLI()->isExtFree(I))
+      if (I && getTLI()->isExtFree(I))
         return 0;
 
       // If this is a zext/sext of a load, return 0 if the corresponding
       // extending load exists on target.
-      if (I && isa<LoadInst>(I->getOperand(0))) {
+      if (CCH == TTI::CastContextHint::Normal) {
         EVT ExtVT = EVT::getEVT(Dst);
         EVT LoadVT = EVT::getEVT(Src);
         unsigned LType =
@@ -839,7 +837,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
         unsigned SplitCost =
             (!SplitSrc || !SplitDst) ? TTI->getVectorSplitCost() : 0;
         return SplitCost +
-               (2 * TTI->getCastInstrCost(Opcode, SplitDstTy, SplitSrcTy,
+               (2 * TTI->getCastInstrCost(Opcode, SplitDstTy, SplitSrcTy, CCH,
                                           CostKind, I));
       }
 
@@ -847,7 +845,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
       // the operation will get scalarized.
       unsigned Num = cast<FixedVectorType>(DstVTy)->getNumElements();
       unsigned Cost = thisT()->getCastInstrCost(
-          Opcode, Dst->getScalarType(), Src->getScalarType(), CostKind, I);
+          Opcode, Dst->getScalarType(), Src->getScalarType(), CCH, CostKind, I);
 
       // Return the cost of multiple scalar invocation plus the cost of
       // inserting and extracting the values.
@@ -872,7 +870,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
     return thisT()->getVectorInstrCost(Instruction::ExtractElement, VecTy,
                                        Index) +
            thisT()->getCastInstrCost(Opcode, Dst, VecTy->getElementType(),
-                                     TTI::TCK_RecipThroughput);
+                                     TTI::CastContextHint::None, TTI::TCK_RecipThroughput);
   }
 
   unsigned getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind) {
@@ -1522,13 +1520,14 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
 
       unsigned ExtOp =
           IID == Intrinsic::smul_fix ? Instruction::SExt : Instruction::ZExt;
+      TTI::CastContextHint CCH = TTI::CastContextHint::None;
 
       unsigned Cost = 0;
-      Cost += 2 * thisT()->getCastInstrCost(ExtOp, ExtTy, RetTy, CostKind);
+      Cost += 2 * thisT()->getCastInstrCost(ExtOp, ExtTy, RetTy, CCH, CostKind);
       Cost +=
           thisT()->getArithmeticInstrCost(Instruction::Mul, ExtTy, CostKind);
       Cost += 2 * thisT()->getCastInstrCost(Instruction::Trunc, RetTy, ExtTy,
-                                            CostKind);
+                                            CCH, CostKind);
       Cost += thisT()->getArithmeticInstrCost(Instruction::LShr, RetTy,
                                               CostKind, TTI::OK_AnyValue,
                                               TTI::OK_UniformConstantValue);
@@ -1587,13 +1586,14 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
 
       unsigned ExtOp =
           IID == Intrinsic::smul_fix ? Instruction::SExt : Instruction::ZExt;
+      TTI::CastContextHint CCH = TTI::CastContextHint::None;
 
       unsigned Cost = 0;
-      Cost += 2 * thisT()->getCastInstrCost(ExtOp, ExtTy, MulTy, CostKind);
+      Cost += 2 * thisT()->getCastInstrCost(ExtOp, ExtTy, MulTy, CCH, CostKind);
       Cost +=
           thisT()->getArithmeticInstrCost(Instruction::Mul, ExtTy, CostKind);
       Cost += 2 * thisT()->getCastInstrCost(Instruction::Trunc, MulTy, ExtTy,
-                                            CostKind);
+                                            CCH, CostKind);
       Cost += thisT()->getArithmeticInstrCost(Instruction::LShr, MulTy,
                                               CostKind, TTI::OK_AnyValue,
                                               TTI::OK_UniformConstantValue);
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index c9e702ce56b81..944e621fc0701 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -730,12 +730,57 @@ int TargetTransformInfo::getShuffleCost(ShuffleKind Kind, VectorType *Ty,
   return Cost;
 }
 
+TTI::CastContextHint
+TargetTransformInfo::getCastContextHint(const Instruction *I) {
+  if (!I)
+    return CastContextHint::None;
+
+  auto getLoadStoreKind = [](const Value *V, unsigned LdStOp, unsigned MaskedOp,
+                             unsigned GatScatOp) {
+    const Instruction *I = dyn_cast<Instruction>(V);
+    if (!I)
+      return CastContextHint::None;
+
+    if (I->getOpcode() == LdStOp)
+      return CastContextHint::Normal;
+
+    if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) {
+      if (II->getIntrinsicID() == MaskedOp)
+        return TTI::CastContextHint::Masked;
+      if (II->getIntrinsicID() == GatScatOp)
+        return TTI::CastContextHint::GatherScatter;
+    }
+
+    return TTI::CastContextHint::None;
+  };
+
+  switch (I->getOpcode()) {
+  case Instruction::ZExt:
+  case Instruction::SExt:
+  case Instruction::FPExt:
+    return getLoadStoreKind(I->getOperand(0), Instruction::Load,
+                            Intrinsic::masked_load, Intrinsic::masked_gather);
+  case Instruction::Trunc:
+  case Instruction::FPTrunc:
+    if (I->hasOneUse())
+      return getLoadStoreKind(*I->user_begin(), Instruction::Store,
+                              Intrinsic::masked_store,
+                              Intrinsic::masked_scatter);
+    break;
+  default:
+    return CastContextHint::None;
+  }
+
+  return TTI::CastContextHint::None;
+}
+
 int TargetTransformInfo::getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
+                                          CastContextHint CCH,
                                           TTI::TargetCostKind CostKind,
                                           const Instruction *I) const {
   assert((I == nullptr || I->getOpcode() == Opcode) &&
          "Opcode should reflect passed instruction.");
-  int Cost = TTIImpl->getCastInstrCost(Opcode, Dst, Src, CostKind, I);
+  int Cost = TTIImpl->getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
   assert(Cost >= 0 && "TTI should not produce negative costs!");
   return Cost;
 }
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index cf6de797727be..5f5da63b21b64 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -270,6 +270,7 @@ bool AArch64TTIImpl::isWideningInstruction(Type *DstTy, unsigned Opcode,
 }
 
 int AArch64TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
+                                     TTI::CastContextHint CCH,
                                      TTI::TargetCostKind CostKind,
                                      const Instruction *I) {
   int ISD = TLI->InstructionOpcodeToISD(Opcode);
@@ -306,7 +307,8 @@ int AArch64TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
   EVT DstTy = TLI->getValueType(DL, Dst);
 
   if (!SrcTy.isSimple() || !DstTy.isSimple())
-    return AdjustCost(BaseT::getCastInstrCost(Opcode, Dst, Src, CostKind, I));
+    return AdjustCost(
+        BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I));
 
   static const TypeConversionCostTblEntry
   ConversionTbl[] = {
@@ -410,7 +412,8 @@ int AArch64TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
                                                  SrcTy.getSimpleVT()))
     return AdjustCost(Entry->Cost);
 
-  return AdjustCost(BaseT::getCastInstrCost(Opcode, Dst, Src, CostKind, I));
+  return AdjustCost(
+      BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I));
 }
 
 int AArch64TTIImpl::getExtractWithExtendCost(unsigned Opcode, Type *Dst,
@@ -442,12 +445,14 @@ int AArch64TTIImpl::getExtractWithExtendCost(unsigned Opcode, Type *Dst,
   // we may get the extension for free. If not, get the default cost for the
   // extend.
   if (!VecLT.second.isVector() || !TLI->isTypeLegal(DstVT))
-    return Cost + getCastInstrCost(Opcode, Dst, Src, CostKind);
+    return Cost + getCastInstrCost(Opcode, Dst, Src, TTI::CastContextHint::None,
+                                   CostKind);
 
   // The destination type should be larger than the element type. If not, get
   // the default cost for the extend.
   if (DstVT.getSizeInBits() < SrcVT.getSizeInBits())
-    return Cost + getCastInstrCost(Opcode, Dst, Src, CostKind);
+    return Cost + getCastInstrCost(Opcode, Dst, Src, TTI::CastContextHint::None,
+                                   CostKind);
 
   switch (Opcode) {
   default:
@@ -466,7 +471,8 @@ int AArch64TTIImpl::getExtractWithExtendCost(unsigned Opcode, Type *Dst,
   }
 
   // If we are unable to perform the extend for free, get the default cost.
-  return Cost + getCastInstrCost(Opcode, Dst, Src, CostKind);
+  return Cost + getCastInstrCost(Opcode, Dst, Src, TTI::CastContextHint::None,
+                                 CostKind);
 }
 
 unsigned AArch64TTIImpl::getCFInstrCost(unsigned Opcode,
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index 1f029689a60e6..5d1371f13fb3e 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -114,7 +114,7 @@ class AArch64TTIImpl : public BasicTTIImplBase<AArch64TTIImpl> {
   unsigned getMaxInterleaveFactor(unsigned VF);
 
   int getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
-                       TTI::TargetCostKind CostKind,
+                       TTI::CastContextHint CCH, TTI::TargetCostKind CostKind,
                        const Instruction *I = nullptr);
 
   int getExtractWithExtendCost(unsigned Opcode, Type *Dst, VectorType *VecTy,
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
index 25d0a4c21fa6f..59f33e694442e 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
@@ -301,6 +301,7 @@ int ARMTTIImpl::getIntImmCostInst(unsigned Opcode, unsigned Idx, const APInt &Im
 }
 
 int ARMTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
+                                 TTI::CastContextHint CCH,
                                  TTI::TargetCostKind CostKind,
                                  const Instruction *I) {
   int ISD = TLI->InstructionOpcodeToISD(Opcode);
@@ -317,7 +318,8 @@ int ARMTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
   EVT DstTy = TLI->getValueType(DL, Dst);
 
   if (!SrcTy.isSimple() || !DstTy.isSimple())
-    return AdjustCost(BaseT::getCastInstrCost(Opcode, Dst, Src, CostKind, I));
+    return AdjustCost(
+        BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I));
 
   // The extend of a load is free
   if (I && isa<LoadInst>(I->getOperand(0))) {
@@ -388,8 +390,8 @@ int ARMTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
     };
     if (SrcTy.isVector() && ST->hasMVEIntegerOps()) {
       if (const auto *Entry =
-              ConvertCostTableLookup(MVELoadConversionTbl, ISD, SrcTy.getSimpleVT(),
-                                     DstTy.getSimpleVT()))
+              ConvertCostTableLookup(MVELoadConversionTbl, ISD,
+                                     SrcTy.getSimpleVT(), DstTy.getSimpleVT()))
         return AdjustCost(Entry->Cost * ST->getMVEVectorCostFactor());
     }
 
@@ -399,8 +401,8 @@ int ARMTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
     };
     if (SrcTy.isVector() && ST->hasMVEFloatOps()) {
       if (const auto *Entry =
-              ConvertCostTableLookup(MVEFLoadConversionTbl, ISD, SrcTy.getSimpleVT(),
-                                     DstTy.getSimpleVT()))
+              ConvertCostTableLookup(MVEFLoadConversionTbl, ISD,
+                                     SrcTy.getSimpleVT(), DstTy.getSimpleVT()))
         return AdjustCost(Entry->Cost * ST->getMVEVectorCostFactor());
     }
   }
@@ -672,7 +674,7 @@ int ARMTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
                      ? ST->getMVEVectorCostFactor()
                      : 1;
   return AdjustCost(
-    BaseCost * BaseT::getCastInstrCost(Opcode, Dst, Src, CostKind, I));
+      BaseCost * BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I));
 }
 
 int ARMTTIImpl::getVectorInstrCost(unsigned Opcode, Type *ValTy,
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
index 093dfbbf5f026..ac7d0378d90bd 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
@@ -210,7 +210,7 @@ class ARMTTIImpl : public BasicTTIImplBase<ARMTTIImpl> {
   }
 
   int getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
-                       TTI::TargetCostKind CostKind,
+                       TTI::CastContextHint CCH, TTI::TargetCostKind CostKind,
                        const Instruction *I = nullptr);
 
   int getCmpSelInstrCost(unsigned Opcode, Type *ValTy, Type *CondTy,
diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
index 80c8736cb74a0..68efaf767502c 100644
--- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
@@ -270,7 +270,9 @@ unsigned HexagonTTIImpl::getArithmeticInstrCost(
 }
 
 unsigned HexagonTTIImpl::getCastInstrCost(unsigned Opcode, Type *DstTy,
-      Type *SrcTy, TTI::TargetCostKind CostKind, const Instruction *I) {
+                                          Type *SrcTy, TTI::CastContextHint CCH,
+                                          TTI::TargetCostKind CostKind,
+                                          const Instruction *I) {
   if (SrcTy->isFPOrFPVectorTy() || DstTy->isFPOrFPVectorTy()) {
     unsigned SrcN = SrcTy->isFPOrFPVectorTy() ? getTypeNumElements(SrcTy) : 0;
     unsigned DstN = DstTy->isFPOrFPVectorTy() ? getTypeNumElements(DstTy) : 0;
diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
index 5fe397486402e..07e59fb5585e8 100644
--- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
+++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
@@ -146,8 +146,9 @@ class HexagonTTIImpl : public BasicTTIImplBase<HexagonTTIImpl> {
       ArrayRef<const Value *> Args = ArrayRef<const Value *>(),
       const Instruction *CxtI = nullptr);
   unsigned getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
-            TTI::TargetCostKind CostKind,
-            const Instruction *I = nullptr);
+                            TTI::CastContextHint CCH,
+                            TTI::TargetCostKind CostKind,
+                            const Instruction *I = nullptr);
   unsigned getVectorInstrCost(unsigned Opcode, Type *Val, unsigned Index);
 
   unsigned getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind) {
diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
index ee8842f4d8663..4171d4a35b25f 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
+++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
@@ -879,11 +879,12 @@ int PPCTTIImpl::getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind) {
 }
 
 int PPCTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
+                                 TTI::CastContextHint CCH,
                                  TTI::TargetCostKind CostKind,
                                  const Instruction *I) {
   assert(TLI->InstructionOpcodeToISD(Opcode) && "Invalid opcode");
 
-  int Cost = BaseT::getCastInstrCost(Opcode, Dst, Src, CostKind, I);
+  int Cost = BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
   Cost = vectorCostAdjustment(Cost, Opcode, Dst, Src);
   // TODO: Allow non-throughput costs that aren't binary.
   if (CostKind != TTI::TCK_RecipThroughput)
diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
index a3453b00a864a..d9aab298c7c16 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
+++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
@@ -106,7 +106,7 @@ class PPCTTIImpl : public BasicTTIImplBase<PPCTTIImpl> {
       const Instruction *CxtI = nullptr);
   int getShuffleCost(TTI::ShuffleKind Kind, Type *Tp, int Index, Type *SubTp);
   int getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
-                       TTI::TargetCostKind CostKind,
+                       TTI::CastContextHint CCH, TTI::TargetCostKind CostKind,
                        const Instruction *I = nullptr);
   int getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind);
   int getCmpSelInstrCost(unsigned Opcode, Type *ValTy, Type *CondTy,
diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
index 864200e5f71cc..8758ddee0aabf 100644
--- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
@@ -699,11 +699,12 @@ getBoolVecToIntConversionCost(unsigned Opcode, Type *Dst,
 }
 
 int SystemZTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
+                                     TTI::CastContextHint CCH,
                                      TTI::TargetCostKind CostKind,
                                      const Instruction *I) {
   // FIXME: Can the logic below also be used for these cost kinds?
   if (CostKind == TTI::TCK_CodeSize || CostKind == TTI::TCK_SizeAndLatency) {
-    int BaseCost = BaseT::getCastInstrCost(Opcode, Dst, Src, CostKind, I);
+    int BaseCost = BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
     return BaseCost == 0 ? BaseCost : 1;
   }
 
@@ -786,8 +787,8 @@ int SystemZTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
       // Return the cost of multiple scalar invocation plus the cost of
       // inserting and extracting the values. Base implementation does not
       // realize float->int gets scalarized.
-      unsigned ScalarCost = getCastInstrCost(Opcode, Dst->getScalarType(),
-                                             Src->getScalarType(), CostKind);
+      unsigned ScalarCost = getCastInstrCost(
+          Opcode, Dst->getScalarType(), Src->getScalarType(), CCH, CostKind);
       unsigned TotCost = VF * ScalarCost;
       bool NeedsInserts = true, NeedsExtracts = true;
       // FP128 registers do not get inserted or extracted.
@@ -828,7 +829,7 @@ int SystemZTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
     }
   }
 
-  return BaseT::getCastInstrCost(Opcode, Dst, Src, CostKind, I);
+  return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
 }
 
 // Scalar i8 / i16 operations will typically be made after first extending
diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
index 7f8f7f6f923ff..1aa31ff1690c6 100644
--- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
+++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
@@ -93,7 +93,7 @@ class SystemZTTIImpl : public BasicTTIImplBase<SystemZTTIImpl> {
   unsigned getBoolVecToIntConversionCost(unsigned Opcode, Type *Dst,
                                          const Instruction *I);
   int getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
-                       TTI::TargetCostKind CostKind,
+                       TTI::CastContextHint CCH, TTI::TargetCostKind CostKind,
                        const Instruction *I = nullptr);
   int getCmpSelInstrCost(unsigned Opcode, Type *ValTy, Type *CondTy,
                          TTI::TargetCostKind CostKind,
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index 491078fd0542e..a484114330cc7 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -1367,6 +1367,7 @@ int X86TTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *BaseTp,
 }
 
 int X86TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
+                                 TTI::CastContextHint CCH,
                                  TTI::TargetCostKind CostKind,
                                  const Instruction *I) {
   int ISD = TLI->InstructionOpcodeToISD(Opcode);
@@ -1988,7 +1989,7 @@ int X86TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
 
   // The function getSimpleVT only handles simple value types.
   if (!SrcTy.isSimple() || !DstTy.isSimple())
-    return AdjustCost(BaseT::getCastInstrCost(Opcode, Dst, Src, CostKind));
+    return AdjustCost(BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind));
 
   MVT SimpleSrcTy = SrcTy.getSimpleVT();
   MVT SimpleDstTy = DstTy.getSimpleVT();
@@ -2049,7 +2050,8 @@ int X86TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
       return AdjustCost(Entry->Cost);
   }
 
-  return AdjustCost(BaseT::getCastInstrCost(Opcode, Dst, Src, CostKind, I));
+  return AdjustCost(
+      BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I));
 }
 
 int X86TTIImpl::getCmpSelInstrCost(unsigned Opcode, Type *ValTy, Type *CondTy,
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.h b/llvm/lib/Target/X86/X86TargetTransformInfo.h
index fb4719064613e..8d2fa27ee7b00 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.h
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.h
@@ -130,7 +130,7 @@ class X86TTIImpl : public BasicTTIImplBase<X86TTIImpl> {
   int getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp, int Index,
                      VectorType *SubTp);
   int getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
-                       TTI::TargetCostKind CostKind,
+                       TTI::CastContextHint CCH, TTI::TargetCostKind CostKind,
                        const Instruction *I = nullptr);
   int getCmpSelInstrCost(unsigned Opcode, Type *ValTy, Type *CondTy,
                          TTI::TargetCostKind CostKind,
diff --git a/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp b/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
index dc2ad14ae61e5..c344c6c684776 100644
--- a/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
+++ b/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
@@ -2025,8 +2025,8 @@ chainToBasePointerCost(SmallVectorImpl<Instruction*> &Chain,
 
       Type *SrcTy = CI->getOperand(0)->getType();
       Cost += TTI.getCastInstrCost(CI->getOpcode(), CI->getType(), SrcTy,
-                                   TargetTransformInfo::TCK_SizeAndLatency,
-                                   CI);
+                                   TTI::getCastContextHint(CI),
+                                   TargetTransformInfo::TCK_SizeAndLatency, CI);
 
     } else if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Instr)) {
       // Cost of the address calculation
diff --git a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
index 3527ca4bbbbcc..c99d612a7768f 100644
--- a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
+++ b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
@@ -2150,8 +2150,9 @@ bool SCEVExpander::isHighCostExpansionHelper(
       llvm_unreachable("There are no other cast types.");
     }
     const SCEV *Op = CastExpr->getOperand();
-    BudgetRemaining -= TTI.getCastInstrCost(Opcode, /*Dst=*/S->getType(),
-                                            /*Src=*/Op->getType(), CostKind);
+    BudgetRemaining -= TTI.getCastInstrCost(
+        Opcode, /*Dst=*/S->getType(),
+        /*Src=*/Op->getType(), TTI::CastContextHint::None, CostKind);
     Worklist.emplace_back(Op);
     return false; // Will answer upon next entry into this function.
   }
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index d207ca250783f..4b519514d543c 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6458,13 +6458,54 @@ unsigned LoopVectorizationCostModel::getInstructionCost(Instruction *I,
   case Instruction::Trunc:
   case Instruction::FPTrunc:
   case Instruction::BitCast: {
+    // Computes the CastContextHint from a Load/Store instruction.
+    auto ComputeCCH = [&](Instruction *I) -> TTI::CastContextHint {
+      assert((isa<LoadInst>(I) || isa<StoreInst>(I)) &&
+             "Expected a load or a store!");
+
+      if (VF == 1)
+        return TTI::CastContextHint::Normal;
+
+      switch (getWideningDecision(I, VF)) {
+      case LoopVectorizationCostModel::CM_GatherScatter:
+        return TTI::CastContextHint::GatherScatter;
+      case LoopVectorizationCostModel::CM_Interleave:
+        return TTI::CastContextHint::Interleave;
+      case LoopVectorizationCostModel::CM_Scalarize:
+      case LoopVectorizationCostModel::CM_Widen:
+        return Legal->isMaskRequired(I) ? TTI::CastContextHint::Masked
+                                        : TTI::CastContextHint::Normal;
+      case LoopVectorizationCostModel::CM_Widen_Reverse:
+        return TTI::CastContextHint::Reversed;
+      case LoopVectorizationCostModel::CM_Unknown:
+        llvm_unreachable("Instr did not go through cost modelling?");
+      }
+
+      llvm_unreachable("Unhandled case!");
+    };
+
+    unsigned Opcode = I->getOpcode();
+    TTI::CastContextHint CCH = TTI::CastContextHint::None;
+    // For Trunc, the context is the only user, which must be a StoreInst.
+    if (Opcode == Instruction::Trunc || Opcode == Instruction::FPTrunc) {
+      if (I->hasOneUse())
+        if (StoreInst *Store = dyn_cast<StoreInst>(*I->user_begin()))
+          CCH = ComputeCCH(Store);
+    }
+    // For Z/Sext, the context is the operand, which must be a LoadInst.
+    else if (Opcode == Instruction::ZExt || Opcode == Instruction::SExt ||
+             Opcode == Instruction::FPExt) {
+      if (LoadInst *Load = dyn_cast<LoadInst>(I->getOperand(0)))
+        CCH = ComputeCCH(Load);
+    }
+
     // We optimize the truncation of induction variables having constant
     // integer steps. The cost of these truncations is the same as the scalar
     // operation.
     if (isOptimizableIVTruncate(I, VF)) {
       auto *Trunc = cast<TruncInst>(I);
       return TTI.getCastInstrCost(Instruction::Trunc, Trunc->getDestTy(),
-                                  Trunc->getSrcTy(), CostKind, Trunc);
+                                  Trunc->getSrcTy(), CCH, CostKind, Trunc);
     }
 
     Type *SrcScalarTy = I->getOperand(0)->getType();
@@ -6477,12 +6518,11 @@ unsigned LoopVectorizationCostModel::getInstructionCost(Instruction *I,
       //
       // Calculate the modified src and dest types.
       Type *MinVecTy = VectorTy;
-      if (I->getOpcode() == Instruction::Trunc) {
+      if (Opcode == Instruction::Trunc) {
         SrcVecTy = smallestIntegerVectorType(SrcVecTy, MinVecTy);
         VectorTy =
             largestIntegerVectorType(ToVectorTy(I->getType(), VF), MinVecTy);
-      } else if (I->getOpcode() == Instruction::ZExt ||
-                 I->getOpcode() == Instruction::SExt) {
+      } else if (Opcode == Instruction::ZExt || Opcode == Instruction::SExt) {
         SrcVecTy = largestIntegerVectorType(SrcVecTy, MinVecTy);
         VectorTy =
             smallestIntegerVectorType(ToVectorTy(I->getType(), VF), MinVecTy);
@@ -6490,8 +6530,8 @@ unsigned LoopVectorizationCostModel::getInstructionCost(Instruction *I,
     }
 
     unsigned N = isScalarAfterVectorization(I, VF) ? VF : 1;
-    return N * TTI.getCastInstrCost(I->getOpcode(), VectorTy, SrcVecTy,
-                                    CostKind, I);
+    return N *
+           TTI.getCastInstrCost(Opcode, VectorTy, SrcVecTy, CCH, CostKind, I);
   }
   case Instruction::Call: {
     bool NeedToScalarize;
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 5fb8ad56d8b3e..d575b3245c650 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -3399,8 +3399,8 @@ int BoUpSLP::getEntryCost(TreeEntry *E) {
                   Ext->getOpcode(), Ext->getType(), VecTy, i);
               // Add back the cost of s|zext which is subtracted separately.
               DeadCost += TTI->getCastInstrCost(
-                  Ext->getOpcode(), Ext->getType(), E->getType(), CostKind,
-                  Ext);
+                  Ext->getOpcode(), Ext->getType(), E->getType(),
+                  TTI::getCastContextHint(Ext), CostKind, Ext);
               continue;
             }
           }
@@ -3424,8 +3424,8 @@ int BoUpSLP::getEntryCost(TreeEntry *E) {
     case Instruction::BitCast: {
       Type *SrcTy = VL0->getOperand(0)->getType();
       int ScalarEltCost =
-          TTI->getCastInstrCost(E->getOpcode(), ScalarTy, SrcTy, CostKind,
-                                VL0);
+          TTI->getCastInstrCost(E->getOpcode(), ScalarTy, SrcTy,
+                                TTI::getCastContextHint(VL0), CostKind, VL0);
       if (NeedToShuffleReuses) {
         ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) * ScalarEltCost;
       }
@@ -3437,9 +3437,10 @@ int BoUpSLP::getEntryCost(TreeEntry *E) {
       int VecCost = 0;
       // Check if the values are candidates to demote.
       if (!MinBWs.count(VL0) || VecTy != SrcVecTy) {
-        VecCost = ReuseShuffleCost +
-                  TTI->getCastInstrCost(E->getOpcode(), VecTy, SrcVecTy,
-                                        CostKind, VL0);
+        VecCost =
+            ReuseShuffleCost +
+            TTI->getCastInstrCost(E->getOpcode(), VecTy, SrcVecTy,
+                                  TTI::getCastContextHint(VL0), CostKind, VL0);
       }
       return VecCost - ScalarCost;
     }
@@ -3644,9 +3645,9 @@ int BoUpSLP::getEntryCost(TreeEntry *E) {
         auto *Src0Ty = FixedVectorType::get(Src0SclTy, VL.size());
         auto *Src1Ty = FixedVectorType::get(Src1SclTy, VL.size());
         VecCost = TTI->getCastInstrCost(E->getOpcode(), VecTy, Src0Ty,
-                                        CostKind);
+                                        TTI::CastContextHint::None, CostKind);
         VecCost += TTI->getCastInstrCost(E->getAltOpcode(), VecTy, Src1Ty,
-                                         CostKind);
+                                         TTI::CastContextHint::None, CostKind);
       }
       VecCost += TTI->getShuffleCost(TargetTransformInfo::SK_Select, VecTy, 0);
       return ReuseShuffleCost + VecCost - ScalarCost;

From 9ddb28964c92f2a2185aff0db77eaa167ac48dcf Mon Sep 17 00:00:00 2001
From: David Green <david.green@arm.com>
Date: Wed, 29 Jul 2020 13:41:34 +0100
Subject: [PATCH 0449/1035] [ARM] Tune getCastInstrCost for extending masked
 loads and truncating masked stores

This patch uses the feature added in D79162 to fix the cost of a
sext/zext of a masked load, or a trunc for a masked store.
Previously, those were considered cheap or even free, but it's
not the case as we cannot split the load in the same way we would for
normal loads.

This updates the costs to better reflect reality, and adds a test for it
in test/Analysis/CostModel/ARM/cast.ll.

It also adds a vectorizer test that showcases the improvement: in some
cases, the vectorizer will now choose a smaller VF when
tail-predication is enabled, which results in better codegen. (Because
if it were to use a higher VF in those cases, the code we see above
would be generated, and the vmovs would block tail-predication later in
the process, resulting in very poor codegen overall)

Original Patch by Pierre van Houtryve

Differential Revision: https://reviews.llvm.org/D79163
---
 .../lib/Target/ARM/ARMTargetTransformInfo.cpp |  43 ++++--
 llvm/test/Analysis/CostModel/ARM/cast_ldst.ll | 138 +++++++++---------
 .../ARM/tail-folding-reduces-vf.ll            | 111 ++++++++++++++
 3 files changed, 207 insertions(+), 85 deletions(-)
 create mode 100644 llvm/test/Transforms/LoopVectorize/ARM/tail-folding-reduces-vf.ll

diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
index 59f33e694442e..9674a813a52af 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
@@ -313,6 +313,12 @@ int ARMTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
       return Cost == 0 ? 0 : 1;
     return Cost;
   };
+  auto IsLegalFPType = [this](EVT VT) {
+    EVT EltVT = VT.getScalarType();
+    return (EltVT == MVT::f32 && ST->hasVFP2Base()) ||
+            (EltVT == MVT::f64 && ST->hasFP64()) ||
+            (EltVT == MVT::f16 && ST->hasFullFP16());
+  };
 
   EVT SrcTy = TLI->getValueType(DL, Src);
   EVT DstTy = TLI->getValueType(DL, Dst);
@@ -321,8 +327,21 @@ int ARMTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
     return AdjustCost(
         BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I));
 
-  // The extend of a load is free
-  if (I && isa<LoadInst>(I->getOperand(0))) {
+  // Extending masked load/Truncating masked stores is expensive because we
+  // currently don't split them. This means that we'll likely end up
+  // loading/storing each element individually (hence the high cost).
+  if ((ST->hasMVEIntegerOps() &&
+       (Opcode == Instruction::Trunc || Opcode == Instruction::ZExt ||
+        Opcode == Instruction::SExt)) ||
+      (ST->hasMVEFloatOps() &&
+       (Opcode == Instruction::FPExt || Opcode == Instruction::FPTrunc) &&
+       IsLegalFPType(SrcTy) && IsLegalFPType(DstTy)))
+    if (CCH == TTI::CastContextHint::Masked && DstTy.getSizeInBits() > 128)
+      return 2 * DstTy.getVectorNumElements() * ST->getMVEVectorCostFactor();
+
+  // The extend of other kinds of load is free
+  if (CCH == TTI::CastContextHint::Normal ||
+      CCH == TTI::CastContextHint::Masked) {
     static const TypeConversionCostTblEntry LoadConversionTbl[] = {
         {ISD::SIGN_EXTEND, MVT::i32, MVT::i16, 0},
         {ISD::ZERO_EXTEND, MVT::i32, MVT::i16, 0},
@@ -376,11 +395,9 @@ int ARMTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
                                      DstTy.getSimpleVT(), SrcTy.getSimpleVT()))
         return AdjustCost(Entry->Cost * ST->getMVEVectorCostFactor());
     }
-  }
 
-  // The truncate of a store is free. This is the mirror of extends above.
-  if (I && I->hasOneUse() && isa<StoreInst>(*I->user_begin())) {
-    static const TypeConversionCostTblEntry MVELoadConversionTbl[] = {
+    // The truncate of a store is free. This is the mirror of extends above.
+    static const TypeConversionCostTblEntry MVEStoreConversionTbl[] = {
         {ISD::TRUNCATE, MVT::v4i32, MVT::v4i16, 0},
         {ISD::TRUNCATE, MVT::v4i32, MVT::v4i8, 0},
         {ISD::TRUNCATE, MVT::v8i16, MVT::v8i8, 0},
@@ -390,18 +407,18 @@ int ARMTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
     };
     if (SrcTy.isVector() && ST->hasMVEIntegerOps()) {
       if (const auto *Entry =
-              ConvertCostTableLookup(MVELoadConversionTbl, ISD,
+              ConvertCostTableLookup(MVEStoreConversionTbl, ISD,
                                      SrcTy.getSimpleVT(), DstTy.getSimpleVT()))
         return AdjustCost(Entry->Cost * ST->getMVEVectorCostFactor());
     }
 
-    static const TypeConversionCostTblEntry MVEFLoadConversionTbl[] = {
+    static const TypeConversionCostTblEntry MVEFStoreConversionTbl[] = {
         {ISD::FP_ROUND, MVT::v4f32, MVT::v4f16, 1},
         {ISD::FP_ROUND, MVT::v8f32, MVT::v8f16, 3},
     };
     if (SrcTy.isVector() && ST->hasMVEFloatOps()) {
       if (const auto *Entry =
-              ConvertCostTableLookup(MVEFLoadConversionTbl, ISD,
+              ConvertCostTableLookup(MVEFStoreConversionTbl, ISD,
                                      SrcTy.getSimpleVT(), DstTy.getSimpleVT()))
         return AdjustCost(Entry->Cost * ST->getMVEVectorCostFactor());
     }
@@ -638,14 +655,8 @@ int ARMTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
     int Lanes = 1;
     if (SrcTy.isFixedLengthVector())
       Lanes = SrcTy.getVectorNumElements();
-    auto IsLegal = [this](EVT VT) {
-      EVT EltVT = VT.getScalarType();
-      return (EltVT == MVT::f32 && ST->hasVFP2Base()) ||
-             (EltVT == MVT::f64 && ST->hasFP64()) ||
-             (EltVT == MVT::f16 && ST->hasFullFP16());
-    };
 
-    if (IsLegal(SrcTy) && IsLegal(DstTy))
+    if (IsLegalFPType(SrcTy) && IsLegalFPType(DstTy))
       return Lanes;
     else
       return Lanes * CallCost;
diff --git a/llvm/test/Analysis/CostModel/ARM/cast_ldst.ll b/llvm/test/Analysis/CostModel/ARM/cast_ldst.ll
index 491e0900e08a2..54029289661c0 100644
--- a/llvm/test/Analysis/CostModel/ARM/cast_ldst.ll
+++ b/llvm/test/Analysis/CostModel/ARM/cast_ldst.ll
@@ -1852,38 +1852,38 @@ define i32 @maskedload_extends() {
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2864u = zext <2 x i8> %loadv2i8 to <2 x i64>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4816s = sext <4 x i8> %loadv4i8 to <4 x i16>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4816u = zext <4 x i8> %loadv4i8 to <4 x i16>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4832s = sext <4 x i8> %loadv4i8 to <4 x i32>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4832u = zext <4 x i8> %loadv4i8 to <4 x i32>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 82 for instruction: %v4864s = sext <4 x i8> %loadv4i8 to <4 x i64>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %v4864u = zext <4 x i8> %loadv4i8 to <4 x i64>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8816s = sext <8 x i8> %loadv8i8 to <8 x i16>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8816u = zext <8 x i8> %loadv8i8 to <8 x i16>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %v8832s = sext <8 x i8> %loadv8i8 to <8 x i32>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %v8832u = zext <8 x i8> %loadv8i8 to <8 x i32>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 330 for instruction: %v8864s = sext <8 x i8> %loadv8i8 to <8 x i64>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 74 for instruction: %v8864u = zext <8 x i8> %loadv8i8 to <8 x i64>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v16816s = sext <16 x i8> %loadv16i8 to <16 x i16>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v16816u = zext <16 x i8> %loadv16i8 to <16 x i16>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 74 for instruction: %v16832s = sext <16 x i8> %loadv16i8 to <16 x i32>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 74 for instruction: %v16832u = zext <16 x i8> %loadv16i8 to <16 x i32>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1322 for instruction: %v16864s = sext <16 x i8> %loadv16i8 to <16 x i64>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 298 for instruction: %v16864u = zext <16 x i8> %loadv16i8 to <16 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4832s = sext <4 x i8> %loadv4i8 to <4 x i32>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4832u = zext <4 x i8> %loadv4i8 to <4 x i32>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v4864s = sext <4 x i8> %loadv4i8 to <4 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v4864u = zext <4 x i8> %loadv4i8 to <4 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8816s = sext <8 x i8> %loadv8i8 to <8 x i16>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8816u = zext <8 x i8> %loadv8i8 to <8 x i16>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v8832s = sext <8 x i8> %loadv8i8 to <8 x i32>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v8832u = zext <8 x i8> %loadv8i8 to <8 x i32>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v8864s = sext <8 x i8> %loadv8i8 to <8 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v8864u = zext <8 x i8> %loadv8i8 to <8 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v16816s = sext <16 x i8> %loadv16i8 to <16 x i16>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v16816u = zext <16 x i8> %loadv16i8 to <16 x i16>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v16832s = sext <16 x i8> %loadv16i8 to <16 x i32>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v16832u = zext <16 x i8> %loadv16i8 to <16 x i32>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v16864s = sext <16 x i8> %loadv16i8 to <16 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v16864u = zext <16 x i8> %loadv16i8 to <16 x i64>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v21632s = sext <2 x i16> %loadv2i16 to <2 x i32>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v21632u = zext <2 x i16> %loadv2i16 to <2 x i32>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v21664s = sext <2 x i16> %loadv2i16 to <2 x i64>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v21664u = zext <2 x i16> %loadv2i16 to <2 x i64>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v41632s = sext <4 x i16> %loadv4i16 to <4 x i32>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v41632u = zext <4 x i16> %loadv4i16 to <4 x i32>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 82 for instruction: %v41664s = sext <4 x i16> %loadv4i16 to <4 x i64>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %v41664u = zext <4 x i16> %loadv4i16 to <4 x i64>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v81632s = sext <8 x i16> %loadv8i16 to <8 x i32>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v81632u = zext <8 x i16> %loadv8i16 to <8 x i32>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 330 for instruction: %v81664s = sext <8 x i16> %loadv8i16 to <8 x i64>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 74 for instruction: %v81664u = zext <8 x i16> %loadv8i16 to <8 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v41632s = sext <4 x i16> %loadv4i16 to <4 x i32>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v41632u = zext <4 x i16> %loadv4i16 to <4 x i32>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v41664s = sext <4 x i16> %loadv4i16 to <4 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v41664u = zext <4 x i16> %loadv4i16 to <4 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v81632s = sext <8 x i16> %loadv8i16 to <8 x i32>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v81632u = zext <8 x i16> %loadv8i16 to <8 x i32>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v81664s = sext <8 x i16> %loadv8i16 to <8 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v81664u = zext <8 x i16> %loadv8i16 to <8 x i64>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v23264s = sext <2 x i32> %loadv2i32 to <2 x i64>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v23264u = zext <2 x i32> %loadv2i32 to <2 x i64>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %v43264s = sext <4 x i32> %loadv4i32 to <4 x i64>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %v43264u = zext <4 x i32> %loadv4i32 to <4 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v43264s = sext <4 x i32> %loadv4i32 to <4 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v43264u = zext <4 x i32> %loadv4i32 to <4 x i64>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
 ; CHECK-V8M-MAIN-RECIP-LABEL: 'maskedload_extends'
@@ -1922,15 +1922,15 @@ define i32 @maskedload_extends() {
 ; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v16864u = zext <16 x i8> %loadv16i8 to <16 x i64>
 ; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v21632s = sext <2 x i16> %loadv2i16 to <2 x i32>
 ; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v21632u = zext <2 x i16> %loadv2i16 to <2 x i32>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v21664s = sext <2 x i16> %loadv2i16 to <2 x i64>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v21664s = sext <2 x i16> %loadv2i16 to <2 x i64>
 ; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v21664u = zext <2 x i16> %loadv2i16 to <2 x i64>
 ; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v41632s = sext <4 x i16> %loadv4i16 to <4 x i32>
 ; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v41632u = zext <4 x i16> %loadv4i16 to <4 x i32>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %v41664s = sext <4 x i16> %loadv4i16 to <4 x i64>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v41664s = sext <4 x i16> %loadv4i16 to <4 x i64>
 ; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v41664u = zext <4 x i16> %loadv4i16 to <4 x i64>
 ; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v81632s = sext <8 x i16> %loadv8i16 to <8 x i32>
 ; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v81632u = zext <8 x i16> %loadv8i16 to <8 x i32>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %v81664s = sext <8 x i16> %loadv8i16 to <8 x i64>
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v81664s = sext <8 x i16> %loadv8i16 to <8 x i64>
 ; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v81664u = zext <8 x i16> %loadv8i16 to <8 x i64>
 ; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v23264s = sext <2 x i32> %loadv2i32 to <2 x i64>
 ; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v23264u = zext <2 x i32> %loadv2i32 to <2 x i64>
@@ -1974,15 +1974,15 @@ define i32 @maskedload_extends() {
 ; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v16864u = zext <16 x i8> %loadv16i8 to <16 x i64>
 ; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v21632s = sext <2 x i16> %loadv2i16 to <2 x i32>
 ; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v21632u = zext <2 x i16> %loadv2i16 to <2 x i32>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v21664s = sext <2 x i16> %loadv2i16 to <2 x i64>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v21664s = sext <2 x i16> %loadv2i16 to <2 x i64>
 ; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v21664u = zext <2 x i16> %loadv2i16 to <2 x i64>
 ; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v41632s = sext <4 x i16> %loadv4i16 to <4 x i32>
 ; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v41632u = zext <4 x i16> %loadv4i16 to <4 x i32>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %v41664s = sext <4 x i16> %loadv4i16 to <4 x i64>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v41664s = sext <4 x i16> %loadv4i16 to <4 x i64>
 ; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v41664u = zext <4 x i16> %loadv4i16 to <4 x i64>
 ; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v81632s = sext <8 x i16> %loadv8i16 to <8 x i32>
 ; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v81632u = zext <8 x i16> %loadv8i16 to <8 x i32>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %v81664s = sext <8 x i16> %loadv8i16 to <8 x i64>
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v81664s = sext <8 x i16> %loadv8i16 to <8 x i64>
 ; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v81664u = zext <8 x i16> %loadv8i16 to <8 x i64>
 ; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v23264s = sext <2 x i32> %loadv2i32 to <2 x i64>
 ; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v23264u = zext <2 x i32> %loadv2i32 to <2 x i64>
@@ -2060,38 +2060,38 @@ define i32 @maskedload_extends() {
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2864u = zext <2 x i8> %loadv2i8 to <2 x i64>
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4816s = sext <4 x i8> %loadv4i8 to <4 x i16>
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4816u = zext <4 x i8> %loadv4i8 to <4 x i16>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4832s = sext <4 x i8> %loadv4i8 to <4 x i32>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4832u = zext <4 x i8> %loadv4i8 to <4 x i32>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4864s = sext <4 x i8> %loadv4i8 to <4 x i64>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4864u = zext <4 x i8> %loadv4i8 to <4 x i64>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8816s = sext <8 x i8> %loadv8i8 to <8 x i16>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8816u = zext <8 x i8> %loadv8i8 to <8 x i16>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8832s = sext <8 x i8> %loadv8i8 to <8 x i32>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8832u = zext <8 x i8> %loadv8i8 to <8 x i32>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8864s = sext <8 x i8> %loadv8i8 to <8 x i64>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8864u = zext <8 x i8> %loadv8i8 to <8 x i64>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16816s = sext <16 x i8> %loadv16i8 to <16 x i16>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16816u = zext <16 x i8> %loadv16i8 to <16 x i16>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16832s = sext <16 x i8> %loadv16i8 to <16 x i32>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16832u = zext <16 x i8> %loadv16i8 to <16 x i32>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16864s = sext <16 x i8> %loadv16i8 to <16 x i64>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16864u = zext <16 x i8> %loadv16i8 to <16 x i64>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4832s = sext <4 x i8> %loadv4i8 to <4 x i32>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4832u = zext <4 x i8> %loadv4i8 to <4 x i32>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v4864s = sext <4 x i8> %loadv4i8 to <4 x i64>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v4864u = zext <4 x i8> %loadv4i8 to <4 x i64>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8816s = sext <8 x i8> %loadv8i8 to <8 x i16>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8816u = zext <8 x i8> %loadv8i8 to <8 x i16>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v8832s = sext <8 x i8> %loadv8i8 to <8 x i32>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v8832u = zext <8 x i8> %loadv8i8 to <8 x i32>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v8864s = sext <8 x i8> %loadv8i8 to <8 x i64>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v8864u = zext <8 x i8> %loadv8i8 to <8 x i64>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v16816s = sext <16 x i8> %loadv16i8 to <16 x i16>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v16816u = zext <16 x i8> %loadv16i8 to <16 x i16>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v16832s = sext <16 x i8> %loadv16i8 to <16 x i32>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v16832u = zext <16 x i8> %loadv16i8 to <16 x i32>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v16864s = sext <16 x i8> %loadv16i8 to <16 x i64>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v16864u = zext <16 x i8> %loadv16i8 to <16 x i64>
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21632s = sext <2 x i16> %loadv2i16 to <2 x i32>
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21632u = zext <2 x i16> %loadv2i16 to <2 x i32>
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21664s = sext <2 x i16> %loadv2i16 to <2 x i64>
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v21664u = zext <2 x i16> %loadv2i16 to <2 x i64>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41632s = sext <4 x i16> %loadv4i16 to <4 x i32>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41632u = zext <4 x i16> %loadv4i16 to <4 x i32>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41664s = sext <4 x i16> %loadv4i16 to <4 x i64>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41664u = zext <4 x i16> %loadv4i16 to <4 x i64>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81632s = sext <8 x i16> %loadv8i16 to <8 x i32>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81632u = zext <8 x i16> %loadv8i16 to <8 x i32>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81664s = sext <8 x i16> %loadv8i16 to <8 x i64>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81664u = zext <8 x i16> %loadv8i16 to <8 x i64>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v41632s = sext <4 x i16> %loadv4i16 to <4 x i32>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v41632u = zext <4 x i16> %loadv4i16 to <4 x i32>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v41664s = sext <4 x i16> %loadv4i16 to <4 x i64>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v41664u = zext <4 x i16> %loadv4i16 to <4 x i64>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v81632s = sext <8 x i16> %loadv8i16 to <8 x i32>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v81632u = zext <8 x i16> %loadv8i16 to <8 x i32>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v81664s = sext <8 x i16> %loadv8i16 to <8 x i64>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v81664u = zext <8 x i16> %loadv8i16 to <8 x i64>
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264s = sext <2 x i32> %loadv2i32 to <2 x i64>
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264u = zext <2 x i32> %loadv2i32 to <2 x i64>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v43264s = sext <4 x i32> %loadv4i32 to <4 x i64>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v43264u = zext <4 x i32> %loadv4i32 to <4 x i64>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v43264s = sext <4 x i32> %loadv4i32 to <4 x i64>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v43264u = zext <4 x i32> %loadv4i32 to <4 x i64>
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
 ; CHECK-V8M-MAIN-SIZE-LABEL: 'maskedload_extends'
@@ -2361,7 +2361,7 @@ define i32 @maskedstore_trunc() {
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8832 = trunc <8 x i32> undef to <8 x i8>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v8864 = trunc <8 x i64> undef to <8 x i8>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16816 = trunc <16 x i16> undef to <16 x i8>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v16832 = trunc <16 x i32> undef to <16 x i8>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v16832 = trunc <16 x i32> undef to <16 x i8>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %v16864 = trunc <16 x i64> undef to <16 x i8>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v21632 = trunc <2 x i32> undef to <2 x i16>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v21664 = trunc <2 x i64> undef to <2 x i16>
@@ -2774,9 +2774,9 @@ define i32 @maskedload_fpextends() {
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f32 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* undef, i32 4, <4 x i1> undef, <4 x float> undef)
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8f32 = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* undef, i32 4, <8 x i1> undef, <8 x float> undef)
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v1 = fpext <2 x half> %loadv2f16 to <2 x float>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2 = fpext <4 x half> %loadv4f16 to <4 x float>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v3 = fpext <8 x half> %loadv8f16 to <8 x float>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v4 = fpext <16 x half> %loadv16f16 to <16 x float>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2 = fpext <4 x half> %loadv4f16 to <4 x float>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v3 = fpext <8 x half> %loadv8f16 to <8 x float>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v4 = fpext <16 x half> %loadv16f16 to <16 x float>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v5 = fpext <2 x half> %loadv2f16 to <2 x double>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v6 = fpext <4 x half> %loadv4f16 to <4 x double>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v7 = fpext <8 x half> %loadv8f16 to <8 x double>
@@ -2866,9 +2866,9 @@ define i32 @maskedload_fpextends() {
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f32 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* undef, i32 4, <4 x i1> undef, <4 x float> undef)
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8f32 = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* undef, i32 4, <8 x i1> undef, <8 x float> undef)
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v1 = fpext <2 x half> %loadv2f16 to <2 x float>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2 = fpext <4 x half> %loadv4f16 to <4 x float>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v3 = fpext <8 x half> %loadv8f16 to <8 x float>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v4 = fpext <16 x half> %loadv16f16 to <16 x float>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2 = fpext <4 x half> %loadv4f16 to <4 x float>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v3 = fpext <8 x half> %loadv8f16 to <8 x float>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v4 = fpext <16 x half> %loadv16f16 to <16 x float>
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v5 = fpext <2 x half> %loadv2f16 to <2 x double>
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v6 = fpext <4 x half> %loadv4f16 to <4 x double>
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v7 = fpext <8 x half> %loadv8f16 to <8 x double>
@@ -2998,9 +2998,9 @@ define i32 @maskedload_fptrunc() {
 ; CHECK-MVE-RECIP-LABEL: 'maskedload_fptrunc'
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v21632 = fptrunc <2 x float> undef to <2 x half>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v21664 = fptrunc <2 x double> undef to <2 x half>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v41632 = fptrunc <4 x float> undef to <4 x half>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v41632 = fptrunc <4 x float> undef to <4 x half>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v41664 = fptrunc <4 x double> undef to <4 x half>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v81632 = fptrunc <8 x float> undef to <8 x half>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v81632 = fptrunc <8 x float> undef to <8 x half>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v81664 = fptrunc <8 x double> undef to <8 x half>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v23264 = fptrunc <2 x double> undef to <2 x float>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v43264 = fptrunc <4 x double> undef to <4 x float>
@@ -3074,9 +3074,9 @@ define i32 @maskedload_fptrunc() {
 ; CHECK-MVE-SIZE-LABEL: 'maskedload_fptrunc'
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v21632 = fptrunc <2 x float> undef to <2 x half>
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v21664 = fptrunc <2 x double> undef to <2 x half>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v41632 = fptrunc <4 x float> undef to <4 x half>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v41632 = fptrunc <4 x float> undef to <4 x half>
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v41664 = fptrunc <4 x double> undef to <4 x half>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v81632 = fptrunc <8 x float> undef to <8 x half>
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81632 = fptrunc <8 x float> undef to <8 x half>
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v81664 = fptrunc <8 x double> undef to <8 x half>
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v23264 = fptrunc <2 x double> undef to <2 x float>
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v43264 = fptrunc <4 x double> undef to <4 x float>
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-reduces-vf.ll b/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-reduces-vf.ll
new file mode 100644
index 0000000000000..58eca2c27847a
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-reduces-vf.ll
@@ -0,0 +1,111 @@
+; RUN: opt < %s -mattr=+mve,+mve.fp -loop-vectorize -S | FileCheck %s --check-prefixes=DEFAULT
+; RUN: opt < %s -mattr=+mve,+mve.fp -loop-vectorize -prefer-predicate-over-epilog -S | FileCheck %s --check-prefixes=TAILPRED
+
+target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
+target triple = "thumbv8.1m.main-arm-none-eabi"
+
+; When TP is disabled, this test can vectorize with a VF of 16.
+; When TP is enabled, this test should vectorize with a VF of 8.
+;
+; DEFAULT: load <16 x i8>, <16 x i8>*
+; DEFAULT: sext <16 x i8> %{{.*}} to <16 x i16>
+; DEFAULT: add <16 x i16>
+; DEFAULT-NOT: llvm.masked.load
+; DEFAULT-NOT: llvm.masked.store
+;
+; TAILPRED: llvm.masked.load.v8i8.p0v8i8
+; TAILPRED: sext <8 x i8> %{{.*}} to <8 x i16>
+; TAILPRED: add <8 x i16>
+; TAILPRED: call void @llvm.masked.store.v8i8.p0v8i8
+; TAILPRED-NOT: load <16 x i8>, <16 x i8>*
+
+define i32 @tp_reduces_vf(i8* nocapture %0, i32 %1, i8** %input) {
+  %3 = load i8*, i8** %input, align 8
+  %4 = sext i32 %1 to i64
+  %5 = icmp eq i32 %1, 0
+  br i1 %5, label %._crit_edge, label %.preheader47.preheader
+
+.preheader47.preheader:
+  br label %.preheader47
+
+.preheader47:
+  %.050 = phi i64 [ %54, %53 ], [ 0, %.preheader47.preheader ]
+  br label %.preheader
+
+._crit_edge.loopexit:
+  br label %._crit_edge
+
+._crit_edge:
+  ret i32 0
+
+.preheader:
+  %indvars.iv51 = phi i32 [ 1, %.preheader47 ], [ %indvars.iv.next52, %52 ]
+  %6 = mul nuw nsw i32 %indvars.iv51, 320
+  br label %7
+
+7:
+  %indvars.iv = phi i32 [ 1, %.preheader ], [ %indvars.iv.next, %7 ]
+  %8 = add nuw nsw i32 %6, %indvars.iv
+  %9 = add nsw i32 %8, -320
+  %10 = add nsw i32 %8, -321
+  %11 = getelementptr inbounds i8, i8* %3, i32 %10
+  %12 = load i8, i8* %11, align 1
+  %13 = sext i8 %12 to i32
+  %14 = getelementptr inbounds i8, i8* %3, i32 %9
+  %15 = load i8, i8* %14, align 1
+  %16 = sext i8 %15 to i32
+  %17 = add nsw i32 %8, -319
+  %18 = getelementptr inbounds i8, i8* %3, i32 %17
+  %19 = load i8, i8* %18, align 1
+  %20 = sext i8 %19 to i32
+  %21 = add nsw i32 %8, -1
+  %22 = getelementptr inbounds i8, i8* %3, i32 %21
+  %23 = load i8, i8* %22, align 1
+  %24 = sext i8 %23 to i32
+  %25 = getelementptr inbounds i8, i8* %3, i32 %8
+  %26 = load i8, i8* %25, align 1
+  %27 = sext i8 %26 to i32
+  %28 = mul nsw i32 %27, 255
+  %29 = add nuw nsw i32 %8, 1
+  %30 = getelementptr inbounds i8, i8* %3, i32 %29
+  %31 = load i8, i8* %30, align 1
+  %32 = sext i8 %31 to i32
+  %33 = add nuw nsw i32 %8, 320
+  %34 = add nuw nsw i32 %8, 319
+  %35 = getelementptr inbounds i8, i8* %3, i32 %34
+  %36 = load i8, i8* %35, align 1
+  %37 = sext i8 %36 to i32
+  %38 = getelementptr inbounds i8, i8* %3, i32 %33
+  %39 = load i8, i8* %38, align 1
+  %40 = sext i8 %39 to i32
+  %41 = add nuw nsw i32 %8, 321
+  %42 = getelementptr inbounds i8, i8* %3, i32 %41
+  %43 = load i8, i8* %42, align 1
+  %44 = sext i8 %43 to i32
+  %reass.add = add nsw i32 %16, %13
+  %reass.add44 = add nsw i32 %reass.add, %20
+  %reass.add45 = add nsw i32 %reass.add44, %24
+  %45 = add nsw i32 %reass.add45, %32
+  %46 = add nsw i32 %45, %37
+  %47 = add nsw i32 %46, %40
+  %reass.add46 = add nsw i32 %47, %44
+  %reass.mul = mul nsw i32 %reass.add46, -28
+  %48 = add nsw i32 %reass.mul, %28
+  %49 = lshr i32 %48, 8
+  %50 = trunc i32 %49 to i8
+  %51 = getelementptr inbounds i8, i8* %0, i32 %8
+  store i8 %50, i8* %51, align 1
+  %indvars.iv.next = add nuw nsw i32 %indvars.iv, 1
+  %exitcond = icmp eq i32 %indvars.iv.next, 319
+  br i1 %exitcond, label %52, label %7
+
+52:
+  %indvars.iv.next52 = add nuw nsw i32 %indvars.iv51, 1
+  %exitcond53 = icmp eq i32 %indvars.iv.next52, 239
+  br i1 %exitcond53, label %53, label %.preheader
+
+53:
+  %54 = add nuw i64 %.050, 1
+  %55 = icmp ult i64 %54, %4
+  br i1 %55, label %.preheader47, label %._crit_edge.loopexit
+}

From 823ffef009152ba1210740c44d472d1d6e56afa3 Mon Sep 17 00:00:00 2001
From: Stephan Herhut <herhut@google.com>
Date: Wed, 29 Jul 2020 12:50:05 +0200
Subject: [PATCH 0450/1035] [mlir][Standard] Allow unranked memrefs as operands
 to dim and rank

`std.dim` currently only accepts ranked memrefs and `std.rank` is limited to
tensors.

Differential Revision: https://reviews.llvm.org/D84790
---
 mlir/include/mlir/Dialect/StandardOps/IR/Ops.td | 12 +++++++-----
 mlir/lib/Dialect/StandardOps/IR/Ops.cpp         |  8 +++++---
 mlir/test/IR/invalid-ops.mlir                   |  2 +-
 mlir/test/Transforms/constant-fold.mlir         | 12 ++++++++++++
 4 files changed, 25 insertions(+), 9 deletions(-)

diff --git a/mlir/include/mlir/Dialect/StandardOps/IR/Ops.td b/mlir/include/mlir/Dialect/StandardOps/IR/Ops.td
index 78307b8974763..d9634fa2b9e69 100644
--- a/mlir/include/mlir/Dialect/StandardOps/IR/Ops.td
+++ b/mlir/include/mlir/Dialect/StandardOps/IR/Ops.td
@@ -1409,7 +1409,7 @@ def DimOp : Std_Op<"dim", [NoSideEffect]> {
     ```
   }];
 
-  let arguments = (ins AnyTypeOf<[AnyMemRef, AnyTensor],
+  let arguments = (ins AnyTypeOf<[AnyRankedOrUnrankedMemRef, AnyTensor],
                                  "any tensor or memref type">:$memrefOrTensor,
                        Index:$index);
   let results = (outs Index:$result);
@@ -2024,16 +2024,18 @@ def PrefetchOp : Std_Op<"prefetch"> {
 def RankOp : Std_Op<"rank", [NoSideEffect]> {
   let summary = "rank operation";
   let description = [{
-    The `rank` operation takes a tensor operand and returns its rank.
+    The `rank` operation takes a memref/tensor operand and returns its rank.
 
     Example:
 
     ```mlir
-    %1 = rank %0 : tensor<*xf32>
+    %1 = rank %arg0 : tensor<*xf32>
+    %2 = rank %arg1 : memref<*xf32>
     ```
   }];
 
-  let arguments = (ins AnyTensor);
+  let arguments = (ins AnyTypeOf<[AnyRankedOrUnrankedMemRef, AnyTensor],
+                                 "any tensor or memref type">:$memrefOrTensor);
   let results = (outs Index);
   let verifier = ?;
 
@@ -2044,7 +2046,7 @@ def RankOp : Std_Op<"rank", [NoSideEffect]> {
     }]>];
 
   let hasFolder = 1;
-  let assemblyFormat = "operands attr-dict `:` type(operands)";
+  let assemblyFormat = "$memrefOrTensor attr-dict `:` type($memrefOrTensor)";
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/StandardOps/IR/Ops.cpp b/mlir/lib/Dialect/StandardOps/IR/Ops.cpp
index 84c35c9fb7a56..a67e79ac4a7ce 100644
--- a/mlir/lib/Dialect/StandardOps/IR/Ops.cpp
+++ b/mlir/lib/Dialect/StandardOps/IR/Ops.cpp
@@ -2039,10 +2039,12 @@ LogicalResult PrefetchOp::fold(ArrayRef<Attribute> cstOperands,
 //===----------------------------------------------------------------------===//
 
 OpFoldResult RankOp::fold(ArrayRef<Attribute> operands) {
-  // Constant fold rank when the rank of the tensor is known.
+  // Constant fold rank when the rank of the operand is known.
   auto type = getOperand().getType();
-  if (auto tensorType = type.dyn_cast<RankedTensorType>())
-    return IntegerAttr::get(IndexType::get(getContext()), tensorType.getRank());
+  if (auto shapedType = type.dyn_cast<ShapedType>())
+    if (shapedType.hasRank())
+      return IntegerAttr::get(IndexType::get(getContext()),
+                              shapedType.getRank());
   return IntegerAttr();
 }
 
diff --git a/mlir/test/IR/invalid-ops.mlir b/mlir/test/IR/invalid-ops.mlir
index 3668c25253adc..6302a8a4acbf9 100644
--- a/mlir/test/IR/invalid-ops.mlir
+++ b/mlir/test/IR/invalid-ops.mlir
@@ -10,7 +10,7 @@ func @dim(%arg : tensor<1x?xf32>) {
 
 func @rank(f32) {
 ^bb(%0: f32):
-  "std.rank"(%0): (f32)->index // expected-error {{'std.rank' op operand #0 must be tensor of any type values}}
+  "std.rank"(%0): (f32)->index // expected-error {{'std.rank' op operand #0 must be any tensor or memref type}}
   return
 }
 
diff --git a/mlir/test/Transforms/constant-fold.mlir b/mlir/test/Transforms/constant-fold.mlir
index 0677b95723701..36fa234213ea1 100644
--- a/mlir/test/Transforms/constant-fold.mlir
+++ b/mlir/test/Transforms/constant-fold.mlir
@@ -686,6 +686,18 @@ func @fold_rank() -> (index) {
 
 // -----
 
+// CHECK-LABEL: func @fold_rank_memref
+func @fold_rank_memref(%arg0 : memref<?x?xf32>) -> (index) {
+  // Fold a rank into a constant
+  // CHECK-NEXT: [[C2:%.+]] = constant 2 : index
+  %rank_0 = rank %arg0 : memref<?x?xf32>
+
+  // CHECK-NEXT: return [[C2]]
+  return %rank_0 : index
+}
+
+// -----
+
 // CHECK-LABEL: func @nested_isolated_region
 func @nested_isolated_region() {
   // CHECK-NEXT: func @isolated_op

From 75182104f09b0df288a8c7eec3701e3246805457 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Wed, 29 Jul 2020 13:32:05 +0100
Subject: [PATCH 0451/1035] [TTI] Move abs/smax/smin/umax/umin cost expansion
 to ICA getIntrinsicInstrCost variant

This will simplify target overrides, and matches what we do for most integer intrinsic costs.
---
 llvm/include/llvm/CodeGen/BasicTTIImpl.h | 40 ++++++++++++------------
 1 file changed, 20 insertions(+), 20 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index e9f6329797057..d8af891a4b762 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -1223,26 +1223,6 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
       IntrinsicCostAttributes Attrs(IID, RetTy, Args[0]->getType(), FMF, 1, I);
       return getIntrinsicInstrCost(Attrs, CostKind);
     }
-    case Intrinsic::abs:
-    case Intrinsic::smax:
-    case Intrinsic::smin:
-    case Intrinsic::umax:
-    case Intrinsic::umin: {
-      // abs(X) = select(icmp(X,0),X,sub(0,X))
-      // minmax(X,Y) = select(icmp(X,Y),X,Y)
-      Type *CondTy = RetTy->getWithNewBitWidth(1);
-      unsigned Cost = 0;
-      // TODO: Ideally getCmpSelInstrCost would accept an icmp condition code.
-      Cost += thisT()->getCmpSelInstrCost(BinaryOperator::ICmp, RetTy, CondTy,
-                                          CostKind);
-      Cost += thisT()->getCmpSelInstrCost(BinaryOperator::Select, RetTy, CondTy,
-                                          CostKind);
-      // TODO: Should we add an OperandValueProperties::OP_Zero property?
-      if (IID == Intrinsic::abs)
-        Cost += thisT()->getArithmeticInstrCost(
-            BinaryOperator::Sub, RetTy, CostKind, TTI::OK_UniformConstantValue);
-      return Cost;
-    }
     case Intrinsic::fshl:
     case Intrinsic::fshr: {
       const Value *X = Args[0];
@@ -1475,6 +1455,26 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
           VecOpTy, cast<VectorType>(CmpInst::makeCmpResultType(VecOpTy)),
           /*IsPairwiseForm=*/false,
           /*IsUnsigned=*/true, CostKind);
+    case Intrinsic::abs:
+    case Intrinsic::smax:
+    case Intrinsic::smin:
+    case Intrinsic::umax:
+    case Intrinsic::umin: {
+      // abs(X) = select(icmp(X,0),X,sub(0,X))
+      // minmax(X,Y) = select(icmp(X,Y),X,Y)
+      Type *CondTy = RetTy->getWithNewBitWidth(1);
+      unsigned Cost = 0;
+      // TODO: Ideally getCmpSelInstrCost would accept an icmp condition code.
+      Cost += thisT()->getCmpSelInstrCost(BinaryOperator::ICmp, RetTy, CondTy,
+                                          CostKind);
+      Cost += thisT()->getCmpSelInstrCost(BinaryOperator::Select, RetTy, CondTy,
+                                          CostKind);
+      // TODO: Should we add an OperandValueProperties::OP_Zero property?
+      if (IID == Intrinsic::abs)
+        Cost += thisT()->getArithmeticInstrCost(
+            BinaryOperator::Sub, RetTy, CostKind, TTI::OK_UniformConstantValue);
+      return Cost;
+    }
     case Intrinsic::sadd_sat:
     case Intrinsic::ssub_sat: {
       Type *CondTy = RetTy->getWithNewBitWidth(1);

From 8725a49409c47c25118915c4b3b41bd0d9d32bd7 Mon Sep 17 00:00:00 2001
From: Andrew Ng <andrew.ng@sony.com>
Date: Tue, 28 Jul 2020 14:33:25 +0100
Subject: [PATCH 0452/1035] [ELF][test] Add test coverage of `__real_` to
 wrap-plt.s

Differential Revision: https://reviews.llvm.org/D84749
---
 lld/test/ELF/wrap-plt.s | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/lld/test/ELF/wrap-plt.s b/lld/test/ELF/wrap-plt.s
index 2584d6ac09bb1..57b417e7b2f0c 100644
--- a/lld/test/ELF/wrap-plt.s
+++ b/lld/test/ELF/wrap-plt.s
@@ -11,6 +11,7 @@
 // CHECK:      Relocations [
 // CHECK-NEXT:   Section ({{.*}}) .rela.plt {
 // CHECK-NEXT:     R_X86_64_JUMP_SLOT __wrap_foo 0x0
+// CHECK-NEXT:     R_X86_64_JUMP_SLOT foo 0x0
 // CHECK-NEXT:     R_X86_64_JUMP_SLOT _start 0x0
 // CHECK-NEXT:   }
 // CHECK-NEXT: ]
@@ -18,6 +19,7 @@
 // DISASM:      <_start>:
 // DISASM-NEXT:   jmp {{.*}} <__wrap_foo@plt>
 // DISASM-NEXT:   jmp {{.*}} <__wrap_foo@plt>
+// DISASM-NEXT:   jmp {{.*}} <foo@plt>
 // DISASM-NEXT:   jmp {{.*}} <_start@plt>
 
 .global foo
@@ -32,4 +34,5 @@ __wrap_foo:
 _start:
   jmp foo@plt
   jmp __wrap_foo@plt
+  jmp __real_foo@plt
   jmp _start@plt

From d1a3396bfbc6fd6df927f2864c18d86e742cabff Mon Sep 17 00:00:00 2001
From: Victor Campos <victor.campos@arm.com>
Date: Tue, 21 Jul 2020 17:18:20 +0100
Subject: [PATCH 0453/1035] [Driver][ARM] Disable unsupported features when
 nofp arch extension is used

A list of target features is disabled when there is no hardware
floating-point support. This is the case when one of the following
options is passed to clang:

 - -mfloat-abi=soft
 - -mfpu=none

This option list is missing, however, the extension "+nofp" that can be
specified in -march flags, such as "-march=armv8-a+nofp".

This patch also disables unsupported target features when nofp is passed
to -march.

Differential Revision: https://reviews.llvm.org/D82948
---
 clang/lib/Driver/ToolChains/Arch/ARM.cpp      | 57 +++++++++++--------
 clang/test/CodeGen/arm-bf16-softfloat.c       |  7 ++-
 .../test/Driver/arm-nofp-disabled-features.c  | 18 ++++++
 llvm/include/llvm/Support/ARMTargetParser.h   |  3 +-
 llvm/lib/Support/ARMTargetParser.cpp          |  8 ++-
 llvm/unittests/Support/TargetParserTest.cpp   |  3 +-
 6 files changed, 66 insertions(+), 30 deletions(-)
 create mode 100644 clang/test/Driver/arm-nofp-disabled-features.c

diff --git a/clang/lib/Driver/ToolChains/Arch/ARM.cpp b/clang/lib/Driver/ToolChains/Arch/ARM.cpp
index afe896b4a65bf..d74d5db0c083d 100644
--- a/clang/lib/Driver/ToolChains/Arch/ARM.cpp
+++ b/clang/lib/Driver/ToolChains/Arch/ARM.cpp
@@ -73,14 +73,15 @@ static unsigned getARMFPUFeatures(const Driver &D, const Arg *A,
 }
 
 // Decode ARM features from string like +[no]featureA+[no]featureB+...
-static bool DecodeARMFeatures(const Driver &D, StringRef text,
-                              StringRef CPU, llvm::ARM::ArchKind ArchKind,
-                              std::vector<StringRef> &Features) {
+static bool DecodeARMFeatures(const Driver &D, StringRef text, StringRef CPU,
+                              llvm::ARM::ArchKind ArchKind,
+                              std::vector<StringRef> &Features,
+                              unsigned &ArgFPUID) {
   SmallVector<StringRef, 8> Split;
   text.split(Split, StringRef("+"), -1, false);
 
   for (StringRef Feature : Split) {
-    if (!appendArchExtFeatures(CPU, ArchKind, Feature, Features))
+    if (!appendArchExtFeatures(CPU, ArchKind, Feature, Features, ArgFPUID))
       return false;
   }
   return true;
@@ -102,14 +103,14 @@ static void DecodeARMFeaturesFromCPU(const Driver &D, StringRef CPU,
 static void checkARMArchName(const Driver &D, const Arg *A, const ArgList &Args,
                              llvm::StringRef ArchName, llvm::StringRef CPUName,
                              std::vector<StringRef> &Features,
-                             const llvm::Triple &Triple) {
+                             const llvm::Triple &Triple, unsigned &ArgFPUID) {
   std::pair<StringRef, StringRef> Split = ArchName.split("+");
 
   std::string MArch = arm::getARMArch(ArchName, Triple);
   llvm::ARM::ArchKind ArchKind = llvm::ARM::parseArch(MArch);
   if (ArchKind == llvm::ARM::ArchKind::INVALID ||
-      (Split.second.size() && !DecodeARMFeatures(
-        D, Split.second, CPUName, ArchKind, Features)))
+      (Split.second.size() && !DecodeARMFeatures(D, Split.second, CPUName,
+                                                 ArchKind, Features, ArgFPUID)))
     D.Diag(clang::diag::err_drv_clang_unsupported) << A->getAsString(Args);
 }
 
@@ -117,15 +118,15 @@ static void checkARMArchName(const Driver &D, const Arg *A, const ArgList &Args,
 static void checkARMCPUName(const Driver &D, const Arg *A, const ArgList &Args,
                             llvm::StringRef CPUName, llvm::StringRef ArchName,
                             std::vector<StringRef> &Features,
-                            const llvm::Triple &Triple) {
+                            const llvm::Triple &Triple, unsigned &ArgFPUID) {
   std::pair<StringRef, StringRef> Split = CPUName.split("+");
 
   std::string CPU = arm::getARMTargetCPU(CPUName, ArchName, Triple);
   llvm::ARM::ArchKind ArchKind =
     arm::getLLVMArchKindForARM(CPU, ArchName, Triple);
   if (ArchKind == llvm::ARM::ArchKind::INVALID ||
-      (Split.second.size() && !DecodeARMFeatures(
-        D, Split.second, CPU, ArchKind, Features)))
+      (Split.second.size() &&
+       !DecodeARMFeatures(D, Split.second, CPU, ArchKind, Features, ArgFPUID)))
     D.Diag(clang::diag::err_drv_clang_unsupported) << A->getAsString(Args);
 }
 
@@ -347,6 +348,8 @@ void arm::getARMTargetFeatures(const Driver &D, const llvm::Triple &Triple,
   const Arg *CPUArg = Args.getLastArg(options::OPT_mcpu_EQ);
   StringRef ArchName;
   StringRef CPUName;
+  unsigned ArchArgFPUID = llvm::ARM::FK_INVALID;
+  unsigned CPUArgFPUID = llvm::ARM::FK_INVALID;
 
   // Check -mcpu. ClangAs gives preference to -Wa,-mcpu=.
   if (WaCPU) {
@@ -364,14 +367,14 @@ void arm::getARMTargetFeatures(const Driver &D, const llvm::Triple &Triple,
       D.Diag(clang::diag::warn_drv_unused_argument)
           << ArchArg->getAsString(Args);
     ArchName = StringRef(WaArch->getValue()).substr(7);
-    checkARMArchName(D, WaArch, Args, ArchName, CPUName,
-                     ExtensionFeatures, Triple);
+    checkARMArchName(D, WaArch, Args, ArchName, CPUName, ExtensionFeatures,
+                     Triple, ArchArgFPUID);
     // FIXME: Set Arch.
     D.Diag(clang::diag::warn_drv_unused_argument) << WaArch->getAsString(Args);
   } else if (ArchArg) {
     ArchName = ArchArg->getValue();
-    checkARMArchName(D, ArchArg, Args, ArchName, CPUName,
-                     ExtensionFeatures, Triple);
+    checkARMArchName(D, ArchArg, Args, ArchName, CPUName, ExtensionFeatures,
+                     Triple, ArchArgFPUID);
   }
 
   // Add CPU features for generic CPUs
@@ -390,8 +393,8 @@ void arm::getARMTargetFeatures(const Driver &D, const llvm::Triple &Triple,
   }
 
   if (CPUArg)
-    checkARMCPUName(D, CPUArg, Args, CPUName, ArchName,
-                    ExtensionFeatures, Triple);
+    checkARMCPUName(D, CPUArg, Args, CPUName, ArchName, ExtensionFeatures,
+                    Triple, CPUArgFPUID);
   // Honor -mfpu=. ClangAs gives preference to -Wa,-mfpu=.
   unsigned FPUID = llvm::ARM::FK_INVALID;
   const Arg *FPUArg = Args.getLastArg(options::OPT_mfpu_EQ);
@@ -455,20 +458,26 @@ void arm::getARMTargetFeatures(const Driver &D, const llvm::Triple &Triple,
       Features.push_back("+fullfp16");
   }
 
-  // Setting -msoft-float/-mfloat-abi=soft effectively disables the FPU (GCC
-  // ignores the -mfpu options in this case).
-  // Note that the ABI can also be set implicitly by the target selected.
+  // Setting -msoft-float/-mfloat-abi=soft, -mfpu=none, or adding +nofp to
+  // -march/-mcpu effectively disables the FPU (GCC ignores the -mfpu options in
+  // this case). Note that the ABI can also be set implicitly by the target
+  // selected.
   if (ABI == arm::FloatABI::Soft) {
     llvm::ARM::getFPUFeatures(llvm::ARM::FK_NONE, Features);
 
     // Disable all features relating to hardware FP, not already disabled by the
     // above call.
+    Features.insert(Features.end(), {"-dotprod", "-fp16fml", "-bf16", "-mve",
+                                     "-mve.fp", "-fpregs"});
+  } else if (FPUID == llvm::ARM::FK_NONE ||
+             ArchArgFPUID == llvm::ARM::FK_NONE ||
+             CPUArgFPUID == llvm::ARM::FK_NONE) {
+    // -mfpu=none, -march=armvX+nofp or -mcpu=X+nofp is *very* similar to
+    // -mfloat-abi=soft, only that it should not disable MVE-I. They disable the
+    // FPU, but not the FPU registers, thus MVE-I, which depends only on the
+    // latter, is still supported.
     Features.insert(Features.end(),
-                    {"-dotprod", "-fp16fml", "-mve", "-mve.fp", "-fpregs"});
-  } else if (FPUID == llvm::ARM::FK_NONE) {
-    // -mfpu=none is *very* similar to -mfloat-abi=soft, only that it should not
-    // disable MVE-I.
-    Features.insert(Features.end(), {"-dotprod", "-fp16fml", "-mve.fp"});
+                    {"-dotprod", "-fp16fml", "-bf16", "-mve.fp"});
     if (!hasIntegerMVE(Features))
       Features.emplace_back("-fpregs");
   }
diff --git a/clang/test/CodeGen/arm-bf16-softfloat.c b/clang/test/CodeGen/arm-bf16-softfloat.c
index 5ea7319ec50ba..3ff4f465223c8 100644
--- a/clang/test/CodeGen/arm-bf16-softfloat.c
+++ b/clang/test/CodeGen/arm-bf16-softfloat.c
@@ -1,4 +1,9 @@
-// RUN: not %clang -o %t.out -target arm-arm-eabi -march=armv8-a+bf16 -mfloat-abi=soft -c %s 2>&1 | FileCheck %s
+// RUN: not %clang -target arm-arm-none-eabi -march=armv8-a+bf16 -mfloat-abi=soft -c %s -o %t 2>&1 | FileCheck %s
+// RUN: not %clang -target arm-arm-none-eabi -march=armv8-a+bf16 -mfpu=none -c %s -o %t 2>&1 | FileCheck %s
+// RUN: not %clang -target arm-arm-none-eabi -march=armv8-a+bf16+nofp -c %s -o %t 2>&1 | FileCheck %s
+// RUN: not %clang -target arm-arm-none-eabi -march=armv8-a+bf16+fp+nofp -c %s -o %t 2>&1 | FileCheck %s
+// RUN: %clang -target arm-arm-none-eabi -march=armv8-a+bf16+fp -c %s -o %t
+// RUN: %clang -target arm-arm-none-eabi -march=armv8-a+bf16+nofp+fp -c %s -o %t
 
 // CHECK: error: __bf16 is not supported on this target
 extern __bf16 var;
diff --git a/clang/test/Driver/arm-nofp-disabled-features.c b/clang/test/Driver/arm-nofp-disabled-features.c
new file mode 100644
index 0000000000000..432e4a98cffc6
--- /dev/null
+++ b/clang/test/Driver/arm-nofp-disabled-features.c
@@ -0,0 +1,18 @@
+// RUN: %clang -target arm-arm-none-eabi -mfloat-abi=soft %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-MFLOAT-ABI-SOFT
+// CHECK-MFLOAT-ABI-SOFT: "-target-feature" "-dotprod"
+// CHECK-MFLOAT-ABI-SOFT: "-target-feature" "-fp16fml"
+// CHECK-MFLOAT-ABI-SOFT: "-target-feature" "-bf16"
+// CHECK-MFLOAT-ABI-SOFT: "-target-feature" "-mve"
+// CHECK-MFLOAT-ABI-SOFT: "-target-feature" "-mve.fp"
+// CHECK-MFLOAT-ABI-SOFT: "-target-feature" "-fpregs"
+
+// RUN: %clang -target arm-arm-none-eabi -mfpu=none %s -### 2>&1 | FileCheck %s
+// RUN: %clang -target arm-arm-none-eabi -march=armv8-a+nofp %s -### 2>&1 | FileCheck %s
+// RUN: %clang -target arm-arm-none-eabi -mcpu=cortex-a35+nofp %s -### 2>&1 | FileCheck %s
+// RUN: %clang -target arm-arm-none-eabi -march=armv8-a+nofp+nomve %s -### 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-NOMVE
+// RUN: %clang -target arm-arm-none-eabi -mcpu=cortex-a35+nofp+nomve %s -### 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-NOMVE
+// CHECK: "-target-feature" "-dotprod"
+// CHECK: "-target-feature" "-fp16fml"
+// CHECK: "-target-feature" "-bf16"
+// CHECK: "-target-feature" "-mve.fp"
+// CHECK-NOMVE: "-target-feature" "-fpregs"
diff --git a/llvm/include/llvm/Support/ARMTargetParser.h b/llvm/include/llvm/Support/ARMTargetParser.h
index 4e76b3c4b83e0..7dd2abd292128 100644
--- a/llvm/include/llvm/Support/ARMTargetParser.h
+++ b/llvm/include/llvm/Support/ARMTargetParser.h
@@ -250,7 +250,8 @@ StringRef getSubArch(ArchKind AK);
 StringRef getArchExtName(uint64_t ArchExtKind);
 StringRef getArchExtFeature(StringRef ArchExt);
 bool appendArchExtFeatures(StringRef CPU, ARM::ArchKind AK, StringRef ArchExt,
-                           std::vector<StringRef> &Features);
+                           std::vector<StringRef> &Features,
+                           unsigned &ArgFPUKind);
 StringRef getHWDivName(uint64_t HWDivKind);
 
 // Information by Name
diff --git a/llvm/lib/Support/ARMTargetParser.cpp b/llvm/lib/Support/ARMTargetParser.cpp
index 56a91f7dc7876..751f84475f42c 100644
--- a/llvm/lib/Support/ARMTargetParser.cpp
+++ b/llvm/lib/Support/ARMTargetParser.cpp
@@ -490,9 +490,10 @@ static unsigned findDoublePrecisionFPU(unsigned InputFPUKind) {
   return ARM::FK_INVALID;
 }
 
-bool ARM::appendArchExtFeatures(
-  StringRef CPU, ARM::ArchKind AK, StringRef ArchExt,
-  std::vector<StringRef> &Features) {
+bool ARM::appendArchExtFeatures(StringRef CPU, ARM::ArchKind AK,
+                                StringRef ArchExt,
+                                std::vector<StringRef> &Features,
+                                unsigned &ArgFPUID) {
 
   size_t StartingNumFeatures = Features.size();
   const bool Negated = stripNegationPrefix(ArchExt);
@@ -527,6 +528,7 @@ bool ARM::appendArchExtFeatures(
     } else {
       FPUKind = getDefaultFPU(CPU, AK);
     }
+    ArgFPUID = FPUKind;
     return ARM::getFPUFeatures(FPUKind, Features);
   }
   return StartingNumFeatures != Features.size();
diff --git a/llvm/unittests/Support/TargetParserTest.cpp b/llvm/unittests/Support/TargetParserTest.cpp
index 0127cb6ae0099..9f923e1358dde 100644
--- a/llvm/unittests/Support/TargetParserTest.cpp
+++ b/llvm/unittests/Support/TargetParserTest.cpp
@@ -668,9 +668,10 @@ static bool
 testArchExtDependency(const char *ArchExt,
                       const std::initializer_list<const char *> &Expected) {
   std::vector<StringRef> Features;
+  unsigned FPUID;
 
   if (!ARM::appendArchExtFeatures("", ARM::ArchKind::ARMV8_1MMainline, ArchExt,
-                                  Features))
+                                  Features, FPUID))
     return false;
 
   return llvm::all_of(Expected, [&](StringRef Ext) {

From 0a0f28254ab7ae5cc9e1868a0ef1ba03d945434d Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Wed, 29 Jul 2020 14:33:44 +0100
Subject: [PATCH 0454/1035] [CostModel][X86] Add SSE costs for ABS intrinsics

---
 .../lib/Target/X86/X86TargetTransformInfo.cpp |  27 ++
 llvm/test/Analysis/CostModel/X86/abs.ll       | 316 +++++++++---------
 2 files changed, 185 insertions(+), 158 deletions(-)

diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index a484114330cc7..4ab0c032603b5 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -2272,6 +2272,8 @@ int X86TTIImpl::getTypeBasedIntrinsicInstrCost(
     { ISD::CTLZ,       MVT::v16i8,   4 },
   };
   static const CostTblEntry AVX512BWCostTbl[] = {
+    { ISD::ABS,        MVT::v32i16,  1 },
+    { ISD::ABS,        MVT::v64i8,   1 },
     { ISD::BITREVERSE, MVT::v8i64,   5 },
     { ISD::BITREVERSE, MVT::v16i32,  5 },
     { ISD::BITREVERSE, MVT::v32i16,  5 },
@@ -2298,6 +2300,12 @@ int X86TTIImpl::getTypeBasedIntrinsicInstrCost(
     { ISD::USUBSAT,    MVT::v64i8,   1 },
   };
   static const CostTblEntry AVX512CostTbl[] = {
+    { ISD::ABS,        MVT::v8i64,   1 },
+    { ISD::ABS,        MVT::v16i32,  1 },
+    { ISD::ABS,        MVT::v32i16,  2 }, // FIXME: include split
+    { ISD::ABS,        MVT::v64i8,   2 }, // FIXME: include split
+    { ISD::ABS,        MVT::v4i64,   1 },
+    { ISD::ABS,        MVT::v2i64,   1 },
     { ISD::BITREVERSE, MVT::v8i64,  36 },
     { ISD::BITREVERSE, MVT::v16i32, 24 },
     { ISD::BITREVERSE, MVT::v32i16, 10 },
@@ -2354,6 +2362,10 @@ int X86TTIImpl::getTypeBasedIntrinsicInstrCost(
     { ISD::BITREVERSE, MVT::i8,      3 }
   };
   static const CostTblEntry AVX2CostTbl[] = {
+    { ISD::ABS,        MVT::v4i64,   2 }, // VBLENDVPD(X,VPSUBQ(0,X),X)
+    { ISD::ABS,        MVT::v8i32,   1 },
+    { ISD::ABS,        MVT::v16i16,  1 },
+    { ISD::ABS,        MVT::v32i8,   1 },
     { ISD::BITREVERSE, MVT::v4i64,   5 },
     { ISD::BITREVERSE, MVT::v8i32,   5 },
     { ISD::BITREVERSE, MVT::v16i16,  5 },
@@ -2391,6 +2403,10 @@ int X86TTIImpl::getTypeBasedIntrinsicInstrCost(
     { ISD::FSQRT,      MVT::v4f64,  28 }, // Haswell from http://www.agner.org/
   };
   static const CostTblEntry AVX1CostTbl[] = {
+    { ISD::ABS,        MVT::v4i64,   6 }, // VBLENDVPD(X,VPSUBQ(0,X),X)
+    { ISD::ABS,        MVT::v8i32,   3 },
+    { ISD::ABS,        MVT::v16i16,  3 },
+    { ISD::ABS,        MVT::v32i8,   3 },
     { ISD::BITREVERSE, MVT::v4i64,  12 }, // 2 x 128-bit Op + extract/insert
     { ISD::BITREVERSE, MVT::v8i32,  12 }, // 2 x 128-bit Op + extract/insert
     { ISD::BITREVERSE, MVT::v16i16, 12 }, // 2 x 128-bit Op + extract/insert
@@ -2446,12 +2462,16 @@ int X86TTIImpl::getTypeBasedIntrinsicInstrCost(
     { ISD::FSQRT, MVT::v2f64, 70 }, // sqrtpd
   };
   static const CostTblEntry SSE42CostTbl[] = {
+    { ISD::ABS,        MVT::v2i64,   3 }, // BLENDVPD(X,PSUBQ(0,X),X)
     { ISD::USUBSAT,    MVT::v4i32,   2 }, // pmaxud + psubd
     { ISD::UADDSAT,    MVT::v4i32,   3 }, // not + pminud + paddd
     { ISD::FSQRT,      MVT::f32,    18 }, // Nehalem from http://www.agner.org/
     { ISD::FSQRT,      MVT::v4f32,  18 }, // Nehalem from http://www.agner.org/
   };
   static const CostTblEntry SSSE3CostTbl[] = {
+    { ISD::ABS,        MVT::v4i32,   1 },
+    { ISD::ABS,        MVT::v8i16,   1 },
+    { ISD::ABS,        MVT::v16i8,   1 },
     { ISD::BITREVERSE, MVT::v2i64,   5 },
     { ISD::BITREVERSE, MVT::v4i32,   5 },
     { ISD::BITREVERSE, MVT::v8i16,   5 },
@@ -2473,6 +2493,10 @@ int X86TTIImpl::getTypeBasedIntrinsicInstrCost(
     { ISD::CTTZ,       MVT::v16i8,   9 }
   };
   static const CostTblEntry SSE2CostTbl[] = {
+    { ISD::ABS,        MVT::v2i64,   4 },
+    { ISD::ABS,        MVT::v4i32,   3 },
+    { ISD::ABS,        MVT::v8i16,   3 },
+    { ISD::ABS,        MVT::v16i8,   3 },
     { ISD::BITREVERSE, MVT::v2i64,  29 },
     { ISD::BITREVERSE, MVT::v4i32,  27 },
     { ISD::BITREVERSE, MVT::v8i16,  27 },
@@ -2571,6 +2595,9 @@ int X86TTIImpl::getTypeBasedIntrinsicInstrCost(
   switch (IID) {
   default:
     break;
+  case Intrinsic::abs:
+    ISD = ISD::ABS;
+    break;
   case Intrinsic::bitreverse:
     ISD = ISD::BITREVERSE;
     break;
diff --git a/llvm/test/Analysis/CostModel/X86/abs.ll b/llvm/test/Analysis/CostModel/X86/abs.ll
index 9d9784ab1c29f..20b8febc579ae 100644
--- a/llvm/test/Analysis/CostModel/X86/abs.ll
+++ b/llvm/test/Analysis/CostModel/X86/abs.ll
@@ -15,16 +15,16 @@
 define void @cost_abs_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 x i64> %a512) {
 ; SSE2-LABEL: 'cost_abs_i64'
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I64 = call i64 @llvm.abs.i64(i64 %a64, i1 false)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2I64 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %a128, i1 false)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %a256, i1 false)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V8I64 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> %a512, i1 false)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I64 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %a128, i1 false)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %a256, i1 false)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I64 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> %a512, i1 false)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSSE3-LABEL: 'cost_abs_i64'
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I64 = call i64 @llvm.abs.i64(i64 %a64, i1 false)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2I64 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %a128, i1 false)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %a256, i1 false)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V8I64 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> %a512, i1 false)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I64 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %a128, i1 false)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %a256, i1 false)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I64 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> %a512, i1 false)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSE42-LABEL: 'cost_abs_i64'
@@ -37,22 +37,22 @@ define void @cost_abs_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 x i64>
 ; AVX1-LABEL: 'cost_abs_i64'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I64 = call i64 @llvm.abs.i64(i64 %a64, i1 false)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I64 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %a128, i1 false)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %a256, i1 false)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V8I64 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> %a512, i1 false)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %a256, i1 false)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8I64 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> %a512, i1 false)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX2-LABEL: 'cost_abs_i64'
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I64 = call i64 @llvm.abs.i64(i64 %a64, i1 false)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I64 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %a128, i1 false)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %a256, i1 false)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8I64 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> %a512, i1 false)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %a256, i1 false)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I64 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> %a512, i1 false)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX512-LABEL: 'cost_abs_i64'
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I64 = call i64 @llvm.abs.i64(i64 %a64, i1 false)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I64 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %a128, i1 false)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %a256, i1 false)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I64 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> %a512, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %a128, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %a256, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> %a512, i1 false)
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %I64    = call i64 @llvm.abs.i64(i64 %a64, i1 0)
@@ -65,44 +65,44 @@ define void @cost_abs_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 x i64>
 define void @cost_abs_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 x i32> %a512) {
 ; SSE2-LABEL: 'cost_abs_i32'
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I32 = call i32 @llvm.abs.i32(i32 %a32, i1 false)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %a128, i1 false)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %a256, i1 false)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V8I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> %a512, i1 false)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %a128, i1 false)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %a256, i1 false)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> %a512, i1 false)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSSE3-LABEL: 'cost_abs_i32'
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I32 = call i32 @llvm.abs.i32(i32 %a32, i1 false)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %a128, i1 false)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %a256, i1 false)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V8I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> %a512, i1 false)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %a128, i1 false)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %a256, i1 false)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> %a512, i1 false)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSE42-LABEL: 'cost_abs_i32'
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I32 = call i32 @llvm.abs.i32(i32 %a32, i1 false)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %a128, i1 false)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %a256, i1 false)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> %a512, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %a128, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %a256, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> %a512, i1 false)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX1-LABEL: 'cost_abs_i32'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I32 = call i32 @llvm.abs.i32(i32 %a32, i1 false)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %a128, i1 false)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %a256, i1 false)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V8I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> %a512, i1 false)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %a128, i1 false)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %a256, i1 false)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> %a512, i1 false)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX2-LABEL: 'cost_abs_i32'
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I32 = call i32 @llvm.abs.i32(i32 %a32, i1 false)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %a128, i1 false)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %a256, i1 false)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> %a512, i1 false)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %a128, i1 false)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %a256, i1 false)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> %a512, i1 false)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX512-LABEL: 'cost_abs_i32'
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I32 = call i32 @llvm.abs.i32(i32 %a32, i1 false)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %a128, i1 false)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %a256, i1 false)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> %a512, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %a128, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %a256, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> %a512, i1 false)
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %I32   = call i32 @llvm.abs.i32(i32 %a32, i1 0)
@@ -115,58 +115,58 @@ define void @cost_abs_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 x i32>
 define void @cost_abs_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <32 x i16> %a512) {
 ; SSE2-LABEL: 'cost_abs_i16'
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I16 = call i16 @llvm.abs.i16(i16 %a16, i1 false)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 false)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a256, i1 false)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %a512, i1 false)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 false)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a256, i1 false)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %a512, i1 false)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSSE3-LABEL: 'cost_abs_i16'
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I16 = call i16 @llvm.abs.i16(i16 %a16, i1 false)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 false)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a256, i1 false)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %a512, i1 false)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 false)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a256, i1 false)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %a512, i1 false)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSE42-LABEL: 'cost_abs_i16'
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I16 = call i16 @llvm.abs.i16(i16 %a16, i1 false)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 false)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a256, i1 false)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %a512, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a256, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %a512, i1 false)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX1-LABEL: 'cost_abs_i16'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I16 = call i16 @llvm.abs.i16(i16 %a16, i1 false)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 false)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a256, i1 false)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %a512, i1 false)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 false)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a256, i1 false)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %a512, i1 false)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX2-LABEL: 'cost_abs_i16'
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I16 = call i16 @llvm.abs.i16(i16 %a16, i1 false)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 false)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a256, i1 false)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %a512, i1 false)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 false)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a256, i1 false)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %a512, i1 false)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX512F-LABEL: 'cost_abs_i16'
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I16 = call i16 @llvm.abs.i16(i16 %a16, i1 false)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 false)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a256, i1 false)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %a512, i1 false)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 false)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a256, i1 false)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %a512, i1 false)
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX512DQ-LABEL: 'cost_abs_i16'
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I16 = call i16 @llvm.abs.i16(i16 %a16, i1 false)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 false)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a256, i1 false)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %a512, i1 false)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 false)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a256, i1 false)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %a512, i1 false)
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX512BW-LABEL: 'cost_abs_i16'
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I16 = call i16 @llvm.abs.i16(i16 %a16, i1 false)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 false)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a256, i1 false)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %a512, i1 false)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 false)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a256, i1 false)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %a512, i1 false)
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %I16    = call i16 @llvm.abs.i16(i16 %a16, i1 0)
@@ -179,58 +179,58 @@ define void @cost_abs_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <32 x i16
 define void @cost_abs_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x i8> %a512) {
 ; SSE2-LABEL: 'cost_abs_i8'
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I8 = call i8 @llvm.abs.i8(i8 %a8, i1 false)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 false)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 false)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 false)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 false)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 false)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 false)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSSE3-LABEL: 'cost_abs_i8'
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I8 = call i8 @llvm.abs.i8(i8 %a8, i1 false)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 false)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 false)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 false)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 false)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 false)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 false)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSE42-LABEL: 'cost_abs_i8'
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I8 = call i8 @llvm.abs.i8(i8 %a8, i1 false)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 false)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 false)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 false)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX1-LABEL: 'cost_abs_i8'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I8 = call i8 @llvm.abs.i8(i8 %a8, i1 false)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 false)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 false)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 false)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 false)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 false)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 false)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX2-LABEL: 'cost_abs_i8'
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I8 = call i8 @llvm.abs.i8(i8 %a8, i1 false)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 false)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 false)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 false)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 false)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 false)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 false)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX512F-LABEL: 'cost_abs_i8'
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I8 = call i8 @llvm.abs.i8(i8 %a8, i1 false)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 false)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 false)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 false)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 false)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 false)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 false)
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX512DQ-LABEL: 'cost_abs_i8'
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I8 = call i8 @llvm.abs.i8(i8 %a8, i1 false)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 false)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 false)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 false)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 false)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 false)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 false)
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX512BW-LABEL: 'cost_abs_i8'
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I8 = call i8 @llvm.abs.i8(i8 %a8, i1 false)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 false)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 false)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 false)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 false)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 false)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 false)
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %I8    = call i8 @llvm.abs.i8(i8 %a8, i1 0)
@@ -247,16 +247,16 @@ define void @cost_abs_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x i8> %a5
 define void @cost_abs_i64_poison(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 x i64> %a512) {
 ; SSE2-LABEL: 'cost_abs_i64_poison'
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I64 = call i64 @llvm.abs.i64(i64 %a64, i1 true)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2I64 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %a128, i1 true)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %a256, i1 true)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V8I64 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> %a512, i1 true)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I64 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %a128, i1 true)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %a256, i1 true)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I64 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> %a512, i1 true)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSSE3-LABEL: 'cost_abs_i64_poison'
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I64 = call i64 @llvm.abs.i64(i64 %a64, i1 true)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2I64 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %a128, i1 true)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %a256, i1 true)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V8I64 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> %a512, i1 true)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I64 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %a128, i1 true)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %a256, i1 true)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I64 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> %a512, i1 true)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSE42-LABEL: 'cost_abs_i64_poison'
@@ -269,22 +269,22 @@ define void @cost_abs_i64_poison(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8
 ; AVX1-LABEL: 'cost_abs_i64_poison'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I64 = call i64 @llvm.abs.i64(i64 %a64, i1 true)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I64 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %a128, i1 true)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %a256, i1 true)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V8I64 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> %a512, i1 true)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %a256, i1 true)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8I64 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> %a512, i1 true)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX2-LABEL: 'cost_abs_i64_poison'
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I64 = call i64 @llvm.abs.i64(i64 %a64, i1 true)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I64 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %a128, i1 true)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %a256, i1 true)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8I64 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> %a512, i1 true)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %a256, i1 true)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I64 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> %a512, i1 true)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX512-LABEL: 'cost_abs_i64_poison'
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I64 = call i64 @llvm.abs.i64(i64 %a64, i1 true)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I64 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %a128, i1 true)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %a256, i1 true)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I64 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> %a512, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %a128, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %a256, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> %a512, i1 true)
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %I64    = call i64 @llvm.abs.i64(i64 %a64, i1 -1)
@@ -297,44 +297,44 @@ define void @cost_abs_i64_poison(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8
 define void @cost_abs_i32_poison(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 x i32> %a512) {
 ; SSE2-LABEL: 'cost_abs_i32_poison'
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I32 = call i32 @llvm.abs.i32(i32 %a32, i1 true)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %a128, i1 true)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %a256, i1 true)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V8I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> %a512, i1 true)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %a128, i1 true)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %a256, i1 true)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> %a512, i1 true)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSSE3-LABEL: 'cost_abs_i32_poison'
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I32 = call i32 @llvm.abs.i32(i32 %a32, i1 true)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %a128, i1 true)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %a256, i1 true)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V8I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> %a512, i1 true)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %a128, i1 true)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %a256, i1 true)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> %a512, i1 true)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSE42-LABEL: 'cost_abs_i32_poison'
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I32 = call i32 @llvm.abs.i32(i32 %a32, i1 true)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %a128, i1 true)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %a256, i1 true)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> %a512, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %a128, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %a256, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> %a512, i1 true)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX1-LABEL: 'cost_abs_i32_poison'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I32 = call i32 @llvm.abs.i32(i32 %a32, i1 true)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %a128, i1 true)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %a256, i1 true)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V8I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> %a512, i1 true)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %a128, i1 true)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %a256, i1 true)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> %a512, i1 true)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX2-LABEL: 'cost_abs_i32_poison'
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I32 = call i32 @llvm.abs.i32(i32 %a32, i1 true)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %a128, i1 true)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %a256, i1 true)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> %a512, i1 true)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %a128, i1 true)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %a256, i1 true)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> %a512, i1 true)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX512-LABEL: 'cost_abs_i32_poison'
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I32 = call i32 @llvm.abs.i32(i32 %a32, i1 true)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %a128, i1 true)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %a256, i1 true)
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> %a512, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %a128, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %a256, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> %a512, i1 true)
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %I32   = call i32 @llvm.abs.i32(i32 %a32, i1 -1)
@@ -347,58 +347,58 @@ define void @cost_abs_i32_poison(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16
 define void @cost_abs_i16_poison(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <32 x i16> %a512) {
 ; SSE2-LABEL: 'cost_abs_i16_poison'
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I16 = call i16 @llvm.abs.i16(i16 %a16, i1 true)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 true)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a256, i1 true)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %a512, i1 true)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 true)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a256, i1 true)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %a512, i1 true)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSSE3-LABEL: 'cost_abs_i16_poison'
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I16 = call i16 @llvm.abs.i16(i16 %a16, i1 true)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 true)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a256, i1 true)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %a512, i1 true)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 true)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a256, i1 true)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %a512, i1 true)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSE42-LABEL: 'cost_abs_i16_poison'
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I16 = call i16 @llvm.abs.i16(i16 %a16, i1 true)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 true)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a256, i1 true)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %a512, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a256, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %a512, i1 true)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX1-LABEL: 'cost_abs_i16_poison'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I16 = call i16 @llvm.abs.i16(i16 %a16, i1 true)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 true)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a256, i1 true)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %a512, i1 true)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 true)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a256, i1 true)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %a512, i1 true)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX2-LABEL: 'cost_abs_i16_poison'
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I16 = call i16 @llvm.abs.i16(i16 %a16, i1 true)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 true)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a256, i1 true)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %a512, i1 true)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 true)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a256, i1 true)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %a512, i1 true)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX512F-LABEL: 'cost_abs_i16_poison'
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I16 = call i16 @llvm.abs.i16(i16 %a16, i1 true)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 true)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a256, i1 true)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %a512, i1 true)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 true)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a256, i1 true)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %a512, i1 true)
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX512DQ-LABEL: 'cost_abs_i16_poison'
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I16 = call i16 @llvm.abs.i16(i16 %a16, i1 true)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 true)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a256, i1 true)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %a512, i1 true)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 true)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a256, i1 true)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %a512, i1 true)
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX512BW-LABEL: 'cost_abs_i16_poison'
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I16 = call i16 @llvm.abs.i16(i16 %a16, i1 true)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 true)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a256, i1 true)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %a512, i1 true)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a128, i1 true)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a256, i1 true)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %a512, i1 true)
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %I16    = call i16 @llvm.abs.i16(i16 %a16, i1 -1)
@@ -411,58 +411,58 @@ define void @cost_abs_i16_poison(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3
 define void @cost_abs_i8_poison(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x i8> %a512) {
 ; SSE2-LABEL: 'cost_abs_i8_poison'
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I8 = call i8 @llvm.abs.i8(i8 %a8, i1 true)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 true)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 true)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 true)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 true)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 true)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 true)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSSE3-LABEL: 'cost_abs_i8_poison'
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I8 = call i8 @llvm.abs.i8(i8 %a8, i1 true)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 true)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 true)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 true)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 true)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 true)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 true)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSE42-LABEL: 'cost_abs_i8_poison'
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I8 = call i8 @llvm.abs.i8(i8 %a8, i1 true)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 true)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 true)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 true)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX1-LABEL: 'cost_abs_i8_poison'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I8 = call i8 @llvm.abs.i8(i8 %a8, i1 true)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 true)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 true)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 true)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 true)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 true)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 true)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX2-LABEL: 'cost_abs_i8_poison'
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I8 = call i8 @llvm.abs.i8(i8 %a8, i1 true)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 true)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 true)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 true)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 true)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 true)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 true)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX512F-LABEL: 'cost_abs_i8_poison'
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I8 = call i8 @llvm.abs.i8(i8 %a8, i1 true)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 true)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 true)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 true)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 true)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 true)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 true)
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX512DQ-LABEL: 'cost_abs_i8_poison'
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I8 = call i8 @llvm.abs.i8(i8 %a8, i1 true)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 true)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 true)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 true)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 true)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 true)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 true)
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX512BW-LABEL: 'cost_abs_i8_poison'
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I8 = call i8 @llvm.abs.i8(i8 %a8, i1 true)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 true)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 true)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 true)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a128, i1 true)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a256, i1 true)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %a512, i1 true)
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %I8    = call i8 @llvm.abs.i8(i8 %a8, i1 -1)

From 71bf6dd682c03c8c29a365f602f5168d44757abe Mon Sep 17 00:00:00 2001
From: Victor Campos <victor.campos@arm.com>
Date: Wed, 29 Jul 2020 14:32:51 +0100
Subject: [PATCH 0455/1035] [Driver][ARM] Fix testcase that should only run on
 ARM

Fix testcase introduced in d1a3396bfbc6fd6df927f2864c18d86e742cabff.
---
 clang/test/CodeGen/arm-bf16-softfloat.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/clang/test/CodeGen/arm-bf16-softfloat.c b/clang/test/CodeGen/arm-bf16-softfloat.c
index 3ff4f465223c8..1792a1e3c9ceb 100644
--- a/clang/test/CodeGen/arm-bf16-softfloat.c
+++ b/clang/test/CodeGen/arm-bf16-softfloat.c
@@ -1,3 +1,4 @@
+// REQUIRES: arm-registered-target
 // RUN: not %clang -target arm-arm-none-eabi -march=armv8-a+bf16 -mfloat-abi=soft -c %s -o %t 2>&1 | FileCheck %s
 // RUN: not %clang -target arm-arm-none-eabi -march=armv8-a+bf16 -mfpu=none -c %s -o %t 2>&1 | FileCheck %s
 // RUN: not %clang -target arm-arm-none-eabi -march=armv8-a+bf16+nofp -c %s -o %t 2>&1 | FileCheck %s

From 8c3262a7b48017b5f5c8aef5b83b4cd1f3dcc324 Mon Sep 17 00:00:00 2001
From: Sanjay Patel <spatel@rotateright.com>
Date: Wed, 29 Jul 2020 09:35:26 -0400
Subject: [PATCH 0456/1035] [ConstantFolding] update test checks FP min/max
 intrinsics

There's a slight difference in functionality with the new CHECK lines:
before, we allowed either -0.0 or 0.0 for maxnum/minnum. That matches
the definition, but we should always get a deterministic result from
constant folding within the compiler, so now we assert that we got
the single expected result in all cases.
---
 llvm/test/Analysis/ConstantFolding/min-max.ll | 124 ++++++++----------
 1 file changed, 57 insertions(+), 67 deletions(-)

diff --git a/llvm/test/Analysis/ConstantFolding/min-max.ll b/llvm/test/Analysis/ConstantFolding/min-max.ll
index b872e4a1aca8c..5ba95c3aeb5ca 100644
--- a/llvm/test/Analysis/ConstantFolding/min-max.ll
+++ b/llvm/test/Analysis/ConstantFolding/min-max.ll
@@ -1,136 +1,126 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -instcombine -S -o - %s | FileCheck %s
-; Tests that constant folding of min and max operations works as expected.
 
 declare float @llvm.minnum.f32(float, float)
-declare float @llvm.maxnum.f32(float, float)
 declare <4 x float> @llvm.minnum.v4f32(<4 x float>, <4 x float>)
+
+declare float @llvm.maxnum.f32(float, float)
 declare <4 x float> @llvm.maxnum.v4f32(<4 x float>, <4 x float>)
 
 declare float @llvm.minimum.f32(float, float)
-declare float @llvm.maximum.f32(float, float)
 declare <4 x float> @llvm.minimum.v4f32(<4 x float>, <4 x float>)
+
+declare float @llvm.maximum.f32(float, float)
 declare <4 x float> @llvm.maximum.v4f32(<4 x float>, <4 x float>)
 
-; CHECK: define float @minnum_float() {
 define float @minnum_float() {
-  ; CHECK-NEXT: ret float 5.000000e+00
+; CHECK-LABEL: @minnum_float(
+; CHECK-NEXT:    ret float 5.000000e+00
+;
   %1 = call float @llvm.minnum.f32(float 5.0, float 42.0)
   ret float %1
 }
 
 ; Check that minnum constant folds to propagate non-NaN or smaller argument
-; CHECK: define <4 x float> @minnum_float_vec() {
+
 define <4 x float> @minnum_float_vec() {
-  ; CHECK-NEXT: ret <4 x float> <float 0x7FF8000000000000, float 5.000000e+00,
-  ; CHECK-SAME:                  float 4.200000e+01, float 5.000000e+00>
-  %1 = call <4 x float> @llvm.minnum.v4f32(
-    <4 x float> <float 0x7FF8000000000000, float 0x7FF8000000000000, float 42., float 42.>,
-    <4 x float> <float 0x7FF8000000000000, float 5., float 0x7FF8000000000000, float 5.>
-  )
+; CHECK-LABEL: @minnum_float_vec(
+; CHECK-NEXT:    ret <4 x float> <float 0x7FF8000000000000, float 5.000000e+00, float 4.200000e+01, float 5.000000e+00>
+;
+  %1 = call <4 x float> @llvm.minnum.v4f32(<4 x float> <float 0x7FF8000000000000, float 0x7FF8000000000000, float 42., float 42.>, <4 x float> <float 0x7FF8000000000000, float 5., float 0x7FF8000000000000, float 5.>)
   ret <4 x float> %1
 }
 
 ; Check that minnum constant folds to propagate one of its argument zeros
-; CHECK: define <4 x float> @minnum_float_zeros_vec() {
+
 define <4 x float> @minnum_float_zeros_vec() {
-  ; CHECK-NEXT: ret <4 x float> <float 0.000000e+00, float {{-?}}0.000000e+00,
-  ; CHECK-SAME:                  float {{-?}}0.000000e+00, float -0.000000e+00>
-  %1 = call <4 x float> @llvm.minnum.v4f32(
-    <4 x float> <float 0.0, float -0.0, float 0.0, float -0.0>,
-    <4 x float> <float 0.0, float 0.0, float -0.0, float -0.0>
-  )
+; CHECK-LABEL: @minnum_float_zeros_vec(
+; CHECK-NEXT:    ret <4 x float> <float 0.000000e+00, float -0.000000e+00, float 0.000000e+00, float -0.000000e+00>
+;
+  %1 = call <4 x float> @llvm.minnum.v4f32(<4 x float> <float 0.0, float -0.0, float 0.0, float -0.0>, <4 x float> <float 0.0, float 0.0, float -0.0, float -0.0>)
   ret <4 x float> %1
 }
 
-; CHECK: define float @maxnum_float() {
 define float @maxnum_float() {
-  ; CHECK-NEXT: ret float 4.200000e+01
+; CHECK-LABEL: @maxnum_float(
+; CHECK-NEXT:    ret float 4.200000e+01
+;
   %1 = call float @llvm.maxnum.f32(float 5.0, float 42.0)
   ret float %1
 }
 
 ; Check that maxnum constant folds to propagate non-NaN or greater argument
-; CHECK: define <4 x float> @maxnum_float_vec() {
+
 define <4 x float> @maxnum_float_vec() {
-  ; CHECK-NEXT: ret <4 x float> <float 0x7FF8000000000000, float 5.000000e+00,
-  ; CHECK-SAME:                  float 4.200000e+01, float 4.200000e+01>
-  %1 = call <4 x float> @llvm.maxnum.v4f32(
-    <4 x float> <float 0x7FF8000000000000, float 0x7FF8000000000000, float 42., float 42.>,
-    <4 x float> <float 0x7FF8000000000000, float 5., float 0x7FF8000000000000, float 5.>
-  )
+; CHECK-LABEL: @maxnum_float_vec(
+; CHECK-NEXT:    ret <4 x float> <float 0x7FF8000000000000, float 5.000000e+00, float 4.200000e+01, float 4.200000e+01>
+;
+  %1 = call <4 x float> @llvm.maxnum.v4f32(<4 x float> <float 0x7FF8000000000000, float 0x7FF8000000000000, float 42., float 42.>, <4 x float> <float 0x7FF8000000000000, float 5., float 0x7FF8000000000000, float 5.>)
   ret <4 x float> %1
 }
 
 ; Check that maxnum constant folds to propagate one of its argument zeros
-; CHECK: define <4 x float> @maxnum_float_zeros_vec() {
+
 define <4 x float> @maxnum_float_zeros_vec() {
-  ; CHECK-NEXT: ret <4 x float> <float 0.000000e+00, float {{-?}}0.000000e+00,
-  ; CHECK-SAME:                  float {{-?}}0.000000e+00, float -0.000000e+00>
-  %1 = call <4 x float> @llvm.maxnum.v4f32(
-    <4 x float> <float 0.0, float -0.0, float 0.0, float -0.0>,
-    <4 x float> <float 0.0, float 0.0, float -0.0, float -0.0>
-  )
+; CHECK-LABEL: @maxnum_float_zeros_vec(
+; CHECK-NEXT:    ret <4 x float> <float 0.000000e+00, float -0.000000e+00, float 0.000000e+00, float -0.000000e+00>
+;
+  %1 = call <4 x float> @llvm.maxnum.v4f32(<4 x float> <float 0.0, float -0.0, float 0.0, float -0.0>, <4 x float> <float 0.0, float 0.0, float -0.0, float -0.0>)
   ret <4 x float> %1
 }
 
-; CHECK: define float @minimum_float() {
 define float @minimum_float() {
-  ; CHECK-NEXT: ret float 5.000000e+00
+; CHECK-LABEL: @minimum_float(
+; CHECK-NEXT:    ret float 5.000000e+00
+;
   %1 = call float @llvm.minimum.f32(float 5.0, float 42.0)
   ret float %1
 }
 
 ; Check that minimum propagates its NaN or smaller argument
-; CHECK: define <4 x float> @minimum_float_vec() {
+
 define <4 x float> @minimum_float_vec() {
-  ; CHECK-NEXT: ret <4 x float> <float 0x7FF8000000000000, float 0x7FF8000000000000,
-  ; CHECK-SAME:                  float 0x7FF8000000000000, float 5.000000e+00>
-  %1 = call <4 x float> @llvm.minimum.v4f32(
-    <4 x float> <float 0x7FF8000000000000, float 0x7FF8000000000000, float 42., float 42.>,
-    <4 x float> <float 0x7FF8000000000000, float 5., float 0x7FF8000000000000, float 5.>
-  )
+; CHECK-LABEL: @minimum_float_vec(
+; CHECK-NEXT:    ret <4 x float> <float 0x7FF8000000000000, float 0x7FF8000000000000, float 0x7FF8000000000000, float 5.000000e+00>
+;
+  %1 = call <4 x float> @llvm.minimum.v4f32(<4 x float> <float 0x7FF8000000000000, float 0x7FF8000000000000, float 42., float 42.>, <4 x float> <float 0x7FF8000000000000, float 5., float 0x7FF8000000000000, float 5.>)
   ret <4 x float> %1
 }
 
 ; Check that minimum treats -0.0 as smaller than 0.0 while constant folding
-; CHECK: define <4 x float> @minimum_float_zeros_vec() {
+
 define <4 x float> @minimum_float_zeros_vec() {
-  ; CHECK-NEXT: ret <4 x float> <float 0.000000e+00, float -0.000000e+00,
-  ; CHECK-SAME:                  float -0.000000e+00, float -0.000000e+00>
-  %1 = call <4 x float> @llvm.minimum.v4f32(
-    <4 x float> <float 0.0, float -0.0, float 0.0, float -0.0>,
-    <4 x float> <float 0.0, float 0.0, float -0.0, float -0.0>
-  )
+; CHECK-LABEL: @minimum_float_zeros_vec(
+; CHECK-NEXT:    ret <4 x float> <float 0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>
+;
+  %1 = call <4 x float> @llvm.minimum.v4f32(<4 x float> <float 0.0, float -0.0, float 0.0, float -0.0>, <4 x float> <float 0.0, float 0.0, float -0.0, float -0.0>)
   ret <4 x float> %1
 }
 
-; CHECK: define float @maximum_float() {
 define float @maximum_float() {
-  ; CHECK-NEXT: ret float 4.200000e+01
+; CHECK-LABEL: @maximum_float(
+; CHECK-NEXT:    ret float 4.200000e+01
+;
   %1 = call float @llvm.maximum.f32(float 5.0, float 42.0)
   ret float %1
 }
 
 ; Check that maximum propagates its NaN or greater argument
-; CHECK: define <4 x float> @maximum_float_vec() {
+
 define <4 x float> @maximum_float_vec() {
-  ; CHECK-NEXT: ret <4 x float> <float 0x7FF8000000000000, float 0x7FF8000000000000,
-  ; CHECK-SAME:                  float 0x7FF8000000000000, float 4.200000e+01>
-  %1 = call <4 x float> @llvm.maximum.v4f32(
-    <4 x float> <float 0x7FF8000000000000, float 0x7FF8000000000000, float 42., float 42.>,
-    <4 x float> <float 0x7FF8000000000000, float 5., float 0x7FF8000000000000, float 5.>
-  )
+; CHECK-LABEL: @maximum_float_vec(
+; CHECK-NEXT:    ret <4 x float> <float 0x7FF8000000000000, float 0x7FF8000000000000, float 0x7FF8000000000000, float 4.200000e+01>
+;
+  %1 = call <4 x float> @llvm.maximum.v4f32(<4 x float> <float 0x7FF8000000000000, float 0x7FF8000000000000, float 42., float 42.>, <4 x float> <float 0x7FF8000000000000, float 5., float 0x7FF8000000000000, float 5.>)
   ret <4 x float> %1
 }
 
 ; Check that maximum treats -0.0 as smaller than 0.0 while constant folding
-; CHECK: define <4 x float> @maximum_float_zeros_vec() {
+
 define <4 x float> @maximum_float_zeros_vec() {
-  ; CHECK-NEXT: ret <4 x float> <float 0.000000e+00, float 0.000000e+00,
-  ; CHECK-SAME:                  float 0.000000e+00, float -0.000000e+00>
-  %1 = call <4 x float> @llvm.maximum.v4f32(
-    <4 x float> <float 0.0, float -0.0, float 0.0, float -0.0>,
-    <4 x float> <float 0.0, float 0.0, float -0.0, float -0.0>
-  )
+; CHECK-LABEL: @maximum_float_zeros_vec(
+; CHECK-NEXT:    ret <4 x float> <float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float -0.000000e+00>
+;
+  %1 = call <4 x float> @llvm.maximum.v4f32(<4 x float> <float 0.0, float -0.0, float 0.0, float -0.0>, <4 x float> <float 0.0, float 0.0, float -0.0, float -0.0>)
   ret <4 x float> %1
 }

From ad793ed90370f0e99fa7ae0cc4d4e97081b5561a Mon Sep 17 00:00:00 2001
From: Tres Popp <tpopp@google.com>
Date: Wed, 29 Jul 2020 12:24:17 +0200
Subject: [PATCH 0457/1035] Forward extent tensors through shape.broadcast.

Differential Revision: https://reviews.llvm.org/D84832
---
 mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td | 11 +++++++++--
 mlir/test/Dialect/Shape/canonicalize.mlir      | 10 +++++-----
 mlir/test/Dialect/Shape/invalid.mlir           | 16 ++++++++++++++++
 mlir/test/Dialect/Shape/ops.mlir               |  9 ++++++++-
 4 files changed, 38 insertions(+), 8 deletions(-)

diff --git a/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td b/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td
index 6ea61376c34d4..7b7da042834a0 100644
--- a/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td
+++ b/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td
@@ -89,16 +89,23 @@ def Shape_BroadcastOp : Shape_Op<"broadcast", [Commutative]> {
 
     Op has an optional string attribute for the error case where there is no
     broadcastable output shape possible for the given inputs.
+
+    Op may also return an ExtentTensor, but this should only be done when this
+    is statically guaranteed to never fail, either because of a dependency on a
+    cstr_broadcastable operation or other details of the construction of the
+    program.
   }];
 
   let arguments = (ins Shape_ShapeOrExtentTensorType:$lhs,
                        Shape_ShapeOrExtentTensorType:$rhs,
                        OptionalAttr<StrAttr>:$error);
-  let results = (outs Shape_ShapeType:$result);
+  let results = (outs Shape_ShapeOrExtentTensorType:$result);
 
-  let assemblyFormat = "$lhs `,` $rhs attr-dict `:` type($lhs) `,` type($rhs)";
+  let assemblyFormat = "$lhs `,` $rhs attr-dict `:` type($lhs) `,` type($rhs) `->` type($result)";
 
   let hasFolder = 1;
+
+  let verifier = [{ return ::verifyShapeOrExtentTensorOp(*this); }];
 }
 
 def Shape_ConstShapeOp : Shape_Op<"const_shape", [ConstantLike, NoSideEffect]> {
diff --git a/mlir/test/Dialect/Shape/canonicalize.mlir b/mlir/test/Dialect/Shape/canonicalize.mlir
index ed5bf6999cd9a..e18ff14df304a 100644
--- a/mlir/test/Dialect/Shape/canonicalize.mlir
+++ b/mlir/test/Dialect/Shape/canonicalize.mlir
@@ -54,7 +54,7 @@ func @f() -> !shape.shape {
   // CHECK: shape.const_shape [7, 2] : !shape.shape
   %0 = shape.const_shape [1, 2] : !shape.shape
   %1 = shape.const_shape [7, 1] : !shape.shape
-  %2 = shape.broadcast %0, %1 : !shape.shape, !shape.shape
+  %2 = shape.broadcast %0, %1 : !shape.shape, !shape.shape -> !shape.shape
   return %2 : !shape.shape
 }
 
@@ -65,7 +65,7 @@ func @f() -> !shape.shape {
 func @f(%arg0 : !shape.shape) -> !shape.shape {
   // CHECK: return %arg0
   %0 = shape.const_shape [] : !shape.shape
-  %1 = shape.broadcast %arg0, %0 : !shape.shape, !shape.shape
+  %1 = shape.broadcast %arg0, %0 : !shape.shape, !shape.shape -> !shape.shape
   return %1 : !shape.shape
 }
 
@@ -76,7 +76,7 @@ func @f(%arg0 : !shape.shape) -> !shape.shape {
 func @f(%arg0 : !shape.shape) -> !shape.shape {
   // CHECK: return %arg0
   %0 = shape.const_shape [] : !shape.shape
-  %1 = shape.broadcast %0, %arg0 : !shape.shape, !shape.shape
+  %1 = shape.broadcast %0, %arg0 : !shape.shape, !shape.shape -> !shape.shape
   return %1 : !shape.shape
 }
 
@@ -89,7 +89,7 @@ func @f() -> !shape.shape {
   // CHECK: return %[[CST]]
   %0 = shape.const_shape [] : !shape.shape
   %1 = shape.const_shape [1, 2, 3] : !shape.shape
-  %2 = shape.broadcast %0, %1 : !shape.shape, !shape.shape
+  %2 = shape.broadcast %0, %1 : !shape.shape, !shape.shape -> !shape.shape
   return %2 : !shape.shape
 }
 
@@ -101,7 +101,7 @@ func @f() -> !shape.shape {
   // CHECK: shape.broadcast
   %0 = shape.const_shape [2] : !shape.shape
   %1 = shape.const_shape [7] : !shape.shape
-  %2 = shape.broadcast %0, %1 : !shape.shape, !shape.shape
+  %2 = shape.broadcast %0, %1 : !shape.shape, !shape.shape -> !shape.shape
   return %2 : !shape.shape
 }
 
diff --git a/mlir/test/Dialect/Shape/invalid.mlir b/mlir/test/Dialect/Shape/invalid.mlir
index d804efbd9e4ec..448bd84e754ee 100644
--- a/mlir/test/Dialect/Shape/invalid.mlir
+++ b/mlir/test/Dialect/Shape/invalid.mlir
@@ -136,3 +136,19 @@ func @add(%lhs : !shape.size, %rhs : index) -> index {
   return %result : index
 }
 
+// -----
+
+func @broadcast_error_possible(%arg0 : !shape.shape, %arg1 : !shape.shape) -> tensor<?xindex> {
+  // expected-error@+1 {{if at least one of the operands can hold error values then the result must be of type `shape` to propagate them}}
+  %result = shape.broadcast %arg0, %arg1 : !shape.shape, !shape.shape -> tensor<?xindex>
+  return %result : tensor<?xindex>
+}
+
+
+// -----
+
+func @broadcast_error_possible(%arg0 : !shape.shape, %arg1 : tensor<?xindex>) -> tensor<?xindex> {
+  // expected-error@+1 {{if at least one of the operands can hold error values then the result must be of type `shape` to propagate them}}
+  %result = shape.broadcast %arg0, %arg1 : !shape.shape, tensor<?xindex> -> tensor<?xindex>
+  return %result : tensor<?xindex>
+}
diff --git a/mlir/test/Dialect/Shape/ops.mlir b/mlir/test/Dialect/Shape/ops.mlir
index 229f3948d31df..48b3805d0a3b7 100644
--- a/mlir/test/Dialect/Shape/ops.mlir
+++ b/mlir/test/Dialect/Shape/ops.mlir
@@ -49,11 +49,18 @@ func @test_shape_num_elements_fixed() {
 func @test_broadcast_fixed() {
   %0 = shape.const_shape [10, 1, 57, 92] : !shape.shape
   %1 = shape.const_shape [4, 57, 92] : !shape.shape
-  %2 = shape.broadcast %0, %1 : !shape.shape, !shape.shape
+  %2 = shape.broadcast %0, %1 : !shape.shape, !shape.shape -> !shape.shape
   %3 = "shape.print"(%2) : (!shape.shape) -> !shape.shape
   return
 }
 
+func @test_broadcast_extents() -> tensor<?xindex> {
+  %0 = shape.const_shape [10, 1, 57, 92] : tensor<?xindex>
+  %1 = shape.const_shape [4, 57, 92] : tensor<?xindex>
+  %2 = shape.broadcast %0, %1 : tensor<?xindex>, tensor<?xindex> -> tensor<?xindex>
+  return %2 : tensor<?xindex>
+}
+
 func @test_shape_any_fixed() {
   %0 = shape.const_shape [4, 57, 92] : !shape.shape
   %1 = shape.const_shape [4, 57, 92] : !shape.shape

From 1ae63b4179c222431cd6a4b2397abceaa5bc3d30 Mon Sep 17 00:00:00 2001
From: Bruno Ricci <riccibrun@gmail.com>
Date: Tue, 28 Jul 2020 15:49:05 +0100
Subject: [PATCH 0458/1035] [clang][NFC] Pass the ASTContext to
 CXXRecordDecl::setCaptures

In general Decl::getASTContext() is relatively expensive and here the changes
are non-invasive. NFC.
---
 clang/include/clang/AST/DeclCXX.h | 2 +-
 clang/lib/AST/ASTImporter.cpp     | 3 ++-
 clang/lib/AST/DeclCXX.cpp         | 4 ++--
 clang/lib/Sema/SemaLambda.cpp     | 7 ++++---
 4 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/clang/include/clang/AST/DeclCXX.h b/clang/include/clang/AST/DeclCXX.h
index 2b8d7e879a0aa..20f058b87e7f3 100644
--- a/clang/include/clang/AST/DeclCXX.h
+++ b/clang/include/clang/AST/DeclCXX.h
@@ -1025,7 +1025,7 @@ class CXXRecordDecl : public RecordDecl {
   }
 
   /// Set the captures for this lambda closure type.
-  void setCaptures(ArrayRef<LambdaCapture> Captures);
+  void setCaptures(ASTContext &Context, ArrayRef<LambdaCapture> Captures);
 
   /// For a closure type, retrieve the mapping from captured
   /// variables and \c this to the non-static data members that store the
diff --git a/clang/lib/AST/ASTImporter.cpp b/clang/lib/AST/ASTImporter.cpp
index e0bca8f08bb41..12dcd14c06bfa 100644
--- a/clang/lib/AST/ASTImporter.cpp
+++ b/clang/lib/AST/ASTImporter.cpp
@@ -1900,7 +1900,8 @@ Error ASTNodeImporter::ImportDefinition(
           else
             return ToCaptureOrErr.takeError();
         }
-        cast<CXXRecordDecl>(To)->setCaptures(ToCaptures);
+        cast<CXXRecordDecl>(To)->setCaptures(Importer.getToContext(),
+                                             ToCaptures);
       }
 
       Error Result = ImportDeclContext(From, /*ForceImport=*/true);
diff --git a/clang/lib/AST/DeclCXX.cpp b/clang/lib/AST/DeclCXX.cpp
index 6f1fd2f14edeb..59ae5cb300f72 100644
--- a/clang/lib/AST/DeclCXX.cpp
+++ b/clang/lib/AST/DeclCXX.cpp
@@ -1383,8 +1383,8 @@ void CXXRecordDecl::finishedDefaultedOrDeletedMember(CXXMethodDecl *D) {
     data().DeclaredNonTrivialSpecialMembers |= SMKind;
 }
 
-void CXXRecordDecl::setCaptures(ArrayRef<LambdaCapture> Captures) {
-  ASTContext &Context = getASTContext();
+void CXXRecordDecl::setCaptures(ASTContext &Context,
+                                ArrayRef<LambdaCapture> Captures) {
   CXXRecordDecl::LambdaDefinitionData &Data = getLambdaData();
 
   // Copy captures.
diff --git a/clang/lib/Sema/SemaLambda.cpp b/clang/lib/Sema/SemaLambda.cpp
index dc74f6e2f7dc5..c9f2854f7accf 100644
--- a/clang/lib/Sema/SemaLambda.cpp
+++ b/clang/lib/Sema/SemaLambda.cpp
@@ -1624,8 +1624,9 @@ FieldDecl *Sema::BuildCaptureField(RecordDecl *RD,
 
   // Build the non-static data member.
   FieldDecl *Field =
-      FieldDecl::Create(Context, RD, Loc, Loc, nullptr, FieldType, TSI, nullptr,
-                        false, ICIS_NoInit);
+      FieldDecl::Create(Context, RD, /*StartLoc=*/Loc, /*IdLoc=*/Loc,
+                        /*Id=*/nullptr, FieldType, TSI, /*BW=*/nullptr,
+                        /*Mutable=*/false, ICIS_NoInit);
   // If the variable being captured has an invalid type, mark the class as
   // invalid as well.
   if (!FieldType->isDependentType()) {
@@ -1785,7 +1786,7 @@ ExprResult Sema::BuildLambdaExpr(SourceLocation StartLoc, SourceLocation EndLoc,
         CUDACheckLambdaCapture(CallOperator, From);
     }
 
-    Class->setCaptures(Captures);
+    Class->setCaptures(Context, Captures);
 
     // C++11 [expr.prim.lambda]p6:
     //   The closure type for a lambda-expression with no lambda-capture

From 517fe058d42a1f937e14de4b61a5ac2ad326c850 Mon Sep 17 00:00:00 2001
From: Bruno Ricci <riccibrun@gmail.com>
Date: Wed, 29 Jul 2020 14:49:59 +0100
Subject: [PATCH 0459/1035] [clang][NFC] clang-format fix after
 eb10b065f2a870b425dcc2040b9955e0eee464b4

---
 clang/lib/Sema/SemaDeclAttr.cpp | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp
index 58602a4c58d4e..d46c791b0e3a8 100644
--- a/clang/lib/Sema/SemaDeclAttr.cpp
+++ b/clang/lib/Sema/SemaDeclAttr.cpp
@@ -3640,15 +3640,15 @@ static void handleTransparentUnionAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
         S.Context.getTypeAlign(FieldType) > FirstAlign) {
       // Warn if we drop the attribute.
       bool isSize = S.Context.getTypeSize(FieldType) != FirstSize;
-      unsigned FieldBits = isSize? S.Context.getTypeSize(FieldType)
-                                 : S.Context.getTypeAlign(FieldType);
+      unsigned FieldBits = isSize ? S.Context.getTypeSize(FieldType)
+                                  : S.Context.getTypeAlign(FieldType);
       S.Diag(Field->getLocation(),
              diag::warn_transparent_union_attribute_field_size_align)
           << isSize << *Field << FieldBits;
-      unsigned FirstBits = isSize? FirstSize : FirstAlign;
+      unsigned FirstBits = isSize ? FirstSize : FirstAlign;
       S.Diag(FirstField->getLocation(),
              diag::note_transparent_union_first_field_size_align)
-        << isSize << FirstBits;
+          << isSize << FirstBits;
       return;
     }
   }

From 6673c6cd82f79b76c1676ab1ab30a288286acb71 Mon Sep 17 00:00:00 2001
From: Frederik Gossen <frgossen@google.com>
Date: Wed, 29 Jul 2020 13:53:41 +0000
Subject: [PATCH 0460/1035] [MLIR][Shape] Limit shape to standard lowerings to
 their supported types

The lowering does not support all types for its source operations. This change
makes the patterns fail in a well-defined manner.

Differential Revision: https://reviews.llvm.org/D84443
---
 .../ShapeToStandard/ShapeToStandard.cpp       | 45 +++++++++++++------
 .../ShapeToStandard/shape-to-standard.mlir    | 35 +++++++++++++++
 2 files changed, 66 insertions(+), 14 deletions(-)

diff --git a/mlir/lib/Conversion/ShapeToStandard/ShapeToStandard.cpp b/mlir/lib/Conversion/ShapeToStandard/ShapeToStandard.cpp
index 4deaa8cd2df36..41d4d90b33d3b 100644
--- a/mlir/lib/Conversion/ShapeToStandard/ShapeToStandard.cpp
+++ b/mlir/lib/Conversion/ShapeToStandard/ShapeToStandard.cpp
@@ -49,8 +49,14 @@ class BinaryOpConversion : public OpConversionPattern<SrcOpTy> {
   LogicalResult
   matchAndRewrite(SrcOpTy op, ArrayRef<Value> operands,
                   ConversionPatternRewriter &rewriter) const override {
-    typename SrcOpTy::Adaptor adaptor(operands);
-    rewriter.replaceOpWithNewOp<DstOpTy>(op, adaptor.lhs(), adaptor.rhs());
+    typename SrcOpTy::Adaptor transformed(operands);
+
+    // For now, only error-free types are supported by this lowering.
+    if (op.getType().template isa<SizeType>())
+      return failure();
+
+    rewriter.replaceOpWithNewOp<DstOpTy>(op, transformed.lhs(),
+                                         transformed.rhs());
     return success();
   }
 };
@@ -85,27 +91,31 @@ class ShapeOfOpConversion : public OpConversionPattern<ShapeOfOp> {
 LogicalResult ShapeOfOpConversion::matchAndRewrite(
     ShapeOfOp op, ArrayRef<Value> operands,
     ConversionPatternRewriter &rewriter) const {
-  ShapeOfOp::Adaptor transformed(operands);
-  auto loc = op.getLoc();
-  auto tensorVal = transformed.arg();
-  auto tensorTy = tensorVal.getType();
+
+  // For now, only error-free types are supported by this lowering.
+  if (op.getType().isa<ShapeType>())
+    return failure();
 
   // For unranked tensors `shape_of` lowers to `scf` and the pattern can be
   // found in the corresponding pass.
+  ShapeOfOp::Adaptor transformed(operands);
+  Value tensorVal = transformed.arg();
+  Type tensorTy = tensorVal.getType();
   if (tensorTy.isa<UnrankedTensorType>())
     return failure();
 
   // Build values for individual dimensions.
   SmallVector<Value, 8> dimValues;
-  auto rankedTensorTy = tensorTy.cast<RankedTensorType>();
+  RankedTensorType rankedTensorTy = tensorTy.cast<RankedTensorType>();
   int64_t rank = rankedTensorTy.getRank();
+  auto loc = op.getLoc();
   for (int64_t i = 0; i < rank; i++) {
     if (rankedTensorTy.isDynamicDim(i)) {
-      auto dimVal = rewriter.create<DimOp>(loc, tensorVal, i);
+      Value dimVal = rewriter.create<DimOp>(loc, tensorVal, i);
       dimValues.push_back(dimVal);
     } else {
       int64_t dim = rankedTensorTy.getDimSize(i);
-      auto dimVal = rewriter.create<ConstantIndexOp>(loc, dim);
+      Value dimVal = rewriter.create<ConstantIndexOp>(loc, dim);
       dimValues.push_back(dimVal);
     }
   }
@@ -187,11 +197,18 @@ LogicalResult GetExtentOpConverter::matchAndRewrite(
     ConversionPatternRewriter &rewriter) const {
   GetExtentOp::Adaptor transformed(operands);
 
-  // Derive shape extent directly from shape origin if possible.
-  // This circumvents the necessity to materialize the shape in memory.
+  // For now, only error-free types are supported by this lowering.
+  if (op.getType().isa<SizeType>())
+    return failure();
+
+  // Derive shape extent directly from shape origin if possible. This
+  // circumvents the necessity to materialize the shape in memory.
   if (auto shapeOfOp = op.shape().getDefiningOp<ShapeOfOp>()) {
-    rewriter.replaceOpWithNewOp<DimOp>(op, shapeOfOp.arg(), transformed.dim());
-    return success();
+    if (shapeOfOp.arg().getType().isa<ShapedType>()) {
+      rewriter.replaceOpWithNewOp<DimOp>(op, shapeOfOp.arg(),
+                                         transformed.dim());
+      return success();
+    }
   }
 
   rewriter.replaceOpWithNewOp<ExtractElementOp>(op, rewriter.getIndexType(),
@@ -241,7 +258,7 @@ void ConvertShapeToStandardPass::runOnOperation() {
 
   // Apply conversion.
   auto module = getOperation();
-  if (failed(applyFullConversion(module, target, patterns)))
+  if (failed(applyPartialConversion(module, target, patterns)))
     signalPassFailure();
 }
 
diff --git a/mlir/test/Conversion/ShapeToStandard/shape-to-standard.mlir b/mlir/test/Conversion/ShapeToStandard/shape-to-standard.mlir
index 0e30cc2bdf569..3f19de9c52f00 100644
--- a/mlir/test/Conversion/ShapeToStandard/shape-to-standard.mlir
+++ b/mlir/test/Conversion/ShapeToStandard/shape-to-standard.mlir
@@ -13,6 +13,30 @@ func @binary_ops(%lhs : index, %rhs : index) {
 
 // -----
 
+// Don't lower binary ops when they operate on `shape.size`.
+// CHECK-LABEL: @binary_ops_on_size
+// CHECK-SAME: (%[[LHS:.*]]: !shape.size, %[[RHS:.*]]: !shape.size)
+func @binary_ops_on_size(%lhs : !shape.size, %rhs : !shape.size) {
+  // CHECK: shape.add %[[LHS]], %[[RHS]] : !shape.size, !shape.size -> !shape.size
+  // CHECK: shape.mul %[[LHS]], %[[RHS]] : !shape.size, !shape.size -> !shape.size
+  %sum = shape.add %lhs, %rhs : !shape.size, !shape.size -> !shape.size
+  %prod = shape.mul %lhs, %rhs : !shape.size, !shape.size -> !shape.size
+  return
+}
+
+// -----
+
+// Don't lower `shape_of` with `shape.shape` type.
+// CHECK-LABEL: @shape_of
+// CHECK-SAME: (%[[ARG:.*]]: tensor<1x2x3xf32>)
+func @shape_of_stat(%arg : tensor<1x2x3xf32>) {
+  // CHECK: shape.shape_of %[[ARG]] : tensor<1x2x3xf32> -> !shape.shape
+  %shape = shape.shape_of %arg : tensor<1x2x3xf32> -> !shape.shape
+  return
+}
+
+// -----
+
 // Lower `shape_of` for statically shaped tensor.
 // CHECK-LABEL: @shape_of_stat
 // CHECK-SAME: (%[[ARG:.*]]: tensor<1x2x3xf32>)
@@ -55,6 +79,17 @@ func @rank(%shape : tensor<?xindex>) -> index {
 
 // -----
 
+// Don't lower `get_extent` if it is of type `shape.size`.
+// CHECK-LABEL: @get_extent
+func @get_extent(%shape : tensor<?xindex>, %idx : !shape.size) -> !shape.size {
+  // CHECK: shape.get_extent
+  %result = shape.get_extent %shape, %idx
+      : tensor<?xindex>, !shape.size -> !shape.size
+  return %result : !shape.size
+}
+
+// -----
+
 // Express `get_extent` as `std.dim` when it relies directly on the outcome of a
 // `shape_of` operation.
 // CHECK-LABEL: @get_extent_shape_of

From 99166fd4fb422351f131fb1265cb85d5f6c5b8da Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Wed, 29 Jul 2020 14:54:03 +0100
Subject: [PATCH 0461/1035] [SCEVExpander] Add option to preserve LCSSA
 directly.

This patch teaches SCEVExpander to directly preserve LCSSA.

As it is currently, SCEV does not look through PHI nodes in loops,
as it might break LCSSA form. Once SCEVExpander can preserve
LCSSA form, it should be safe for SCEV to look through PHIs.

To preserve LCSSA form, this patch uses formLCSSAForInstructions
on operands of newly created instructions, if the definition is inside
a different loop than the new instruction.

The final value we return from expandCodeFor may also need LCSSA
phis, depending on the insert point. As no user for it exists there yet,
create a temporary instruction at the insert point, which can be passed
to formLCSSAForInstructions. This temporary instruction is removed
after LCSSA construction.

Reviewed By: mkazantsev

Differential Revision: https://reviews.llvm.org/D71538
---
 .../Utils/ScalarEvolutionExpander.h           |  41 ++++-
 .../Transforms/Scalar/LoopStrengthReduce.cpp  |   6 +-
 .../Utils/ScalarEvolutionExpander.cpp         | 173 +++++++++++++-----
 .../Utils/ScalarEvolutionExpanderTest.cpp     |   7 +-
 4 files changed, 166 insertions(+), 61 deletions(-)

diff --git a/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h b/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h
index 6e53d15d25fc7..cb212b2705eed 100644
--- a/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h
+++ b/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h
@@ -52,6 +52,9 @@ class SCEVExpander : public SCEVVisitor<SCEVExpander, Value *> {
   // New instructions receive a name to identify them with the current pass.
   const char *IVName;
 
+  /// Indicates whether LCSSA phis should be created for inserted values.
+  bool PreserveLCSSA;
+
   // InsertedExpressions caches Values for reuse, so must track RAUW.
   DenseMap<std::pair<const SCEV *, Instruction *>, TrackingVH<Value>>
       InsertedExpressions;
@@ -146,9 +149,10 @@ class SCEVExpander : public SCEVVisitor<SCEVExpander, Value *> {
 public:
   /// Construct a SCEVExpander in "canonical" mode.
   explicit SCEVExpander(ScalarEvolution &se, const DataLayout &DL,
-                        const char *name)
-      : SE(se), DL(DL), IVName(name), IVIncInsertLoop(nullptr),
-        IVIncInsertPos(nullptr), CanonicalMode(true), LSRMode(false),
+                        const char *name, bool PreserveLCSSA = true)
+      : SE(se), DL(DL), IVName(name), PreserveLCSSA(PreserveLCSSA),
+        IVIncInsertLoop(nullptr), IVIncInsertPos(nullptr), CanonicalMode(true),
+        LSRMode(false),
         Builder(se.getContext(), TargetFolder(DL),
                 IRBuilderCallbackInserter(
                     [this](Instruction *I) { rememberInstruction(I); })) {
@@ -223,14 +227,18 @@ class SCEVExpander : public SCEVVisitor<SCEVExpander, Value *> {
                                const TargetTransformInfo *TTI = nullptr);
 
   /// Insert code to directly compute the specified SCEV expression into the
-  /// program.  The inserted code is inserted into the specified block.
-  Value *expandCodeFor(const SCEV *SH, Type *Ty, Instruction *I);
+  /// program.  The code is inserted into the specified block.
+  Value *expandCodeFor(const SCEV *SH, Type *Ty, Instruction *I) {
+    return expandCodeForImpl(SH, Ty, I, true);
+  }
 
   /// Insert code to directly compute the specified SCEV expression into the
-  /// program.  The inserted code is inserted into the SCEVExpander's current
+  /// program.  The code is inserted into the SCEVExpander's current
   /// insertion point. If a type is specified, the result will be expanded to
   /// have that type, with a cast if necessary.
-  Value *expandCodeFor(const SCEV *SH, Type *Ty = nullptr);
+  Value *expandCodeFor(const SCEV *SH, Type *Ty = nullptr) {
+    return expandCodeForImpl(SH, Ty, true);
+  }
 
   /// Generates a code sequence that evaluates this predicate.  The inserted
   /// instructions will be at position \p Loc.  The result will be of type i1
@@ -338,6 +346,20 @@ class SCEVExpander : public SCEVVisitor<SCEVExpander, Value *> {
 private:
   LLVMContext &getContext() const { return SE.getContext(); }
 
+  /// Insert code to directly compute the specified SCEV expression into the
+  /// program. The code is inserted into the SCEVExpander's current
+  /// insertion point. If a type is specified, the result will be expanded to
+  /// have that type, with a cast if necessary. If \p Root is true, this
+  /// indicates that \p SH is the top-level expression to expand passed from
+  /// an external client call.
+  Value *expandCodeForImpl(const SCEV *SH, Type *Ty, bool Root);
+
+  /// Insert code to directly compute the specified SCEV expression into the
+  /// program. The code is inserted into the specified block. If \p
+  /// Root is true, this indicates that \p SH is the top-level expression to
+  /// expand passed from an external client call.
+  Value *expandCodeForImpl(const SCEV *SH, Type *Ty, Instruction *I, bool Root);
+
   /// Recursive helper function for isHighCostExpansion.
   bool isHighCostExpansionHelper(const SCEV *S, Loop *L, const Instruction &At,
                                  int &BudgetRemaining,
@@ -419,6 +441,11 @@ class SCEVExpander : public SCEVVisitor<SCEVExpander, Value *> {
                       Instruction *Pos, PHINode *LoopPhi);
 
   void fixupInsertPoints(Instruction *I);
+
+  /// If required, create LCSSA PHIs for \p Users' operand \p OpIdx. If new
+  /// LCSSA PHIs have been created, return the LCSSA PHI available at \p User.
+  /// If no PHIs have been created, return the unchanged operand \p OpIdx.
+  Value *fixupLCSSAFormFor(Instruction *User, unsigned OpIdx);
 };
 } // namespace llvm
 
diff --git a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
index cf02ef1e83f3f..c3e46c1fadef3 100644
--- a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
@@ -5514,8 +5514,8 @@ void LSRInstance::ImplementSolution(
   // we can remove them after we are done working.
   SmallVector<WeakTrackingVH, 16> DeadInsts;
 
-  SCEVExpander Rewriter(SE, L->getHeader()->getModule()->getDataLayout(),
-                        "lsr");
+  SCEVExpander Rewriter(SE, L->getHeader()->getModule()->getDataLayout(), "lsr",
+                        false);
 #ifndef NDEBUG
   Rewriter.setDebugType(DEBUG_TYPE);
 #endif
@@ -5780,7 +5780,7 @@ static bool ReduceLoopStrength(Loop *L, IVUsers &IU, ScalarEvolution &SE,
   if (EnablePhiElim && L->isLoopSimplifyForm()) {
     SmallVector<WeakTrackingVH, 16> DeadInsts;
     const DataLayout &DL = L->getHeader()->getModule()->getDataLayout();
-    SCEVExpander Rewriter(SE, DL, "lsr");
+    SCEVExpander Rewriter(SE, DL, "lsr", false);
 #ifndef NDEBUG
     Rewriter.setDebugType(DEBUG_TYPE);
 #endif
diff --git a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
index c99d612a7768f..1a10e580c68c0 100644
--- a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
+++ b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
@@ -27,6 +27,7 @@
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
+#include "llvm/Transforms/Utils/LoopUtils.h"
 
 using namespace llvm;
 
@@ -461,9 +462,10 @@ Value *SCEVExpander::expandAddToGEP(const SCEV *const *op_begin,
     // we didn't find any operands that could be factored, tentatively
     // assume that element zero was selected (since the zero offset
     // would obviously be folded away).
-    Value *Scaled = ScaledOps.empty() ?
-                    Constant::getNullValue(Ty) :
-                    expandCodeFor(SE.getAddExpr(ScaledOps), Ty);
+    Value *Scaled =
+        ScaledOps.empty()
+            ? Constant::getNullValue(Ty)
+            : expandCodeForImpl(SE.getAddExpr(ScaledOps), Ty, false);
     GepIndices.push_back(Scaled);
 
     // Collect struct field index operands.
@@ -522,7 +524,7 @@ Value *SCEVExpander::expandAddToGEP(const SCEV *const *op_begin,
            SE.DT.dominates(cast<Instruction>(V), &*Builder.GetInsertPoint()));
 
     // Expand the operands for a plain byte offset.
-    Value *Idx = expandCodeFor(SE.getAddExpr(Ops), Ty);
+    Value *Idx = expandCodeForImpl(SE.getAddExpr(Ops), Ty, false);
 
     // Fold a GEP with constant operands.
     if (Constant *CLHS = dyn_cast<Constant>(V))
@@ -743,14 +745,14 @@ Value *SCEVExpander::visitAddExpr(const SCEVAddExpr *S) {
       Sum = expandAddToGEP(NewOps.begin(), NewOps.end(), PTy, Ty, expand(Op));
     } else if (Op->isNonConstantNegative()) {
       // Instead of doing a negate and add, just do a subtract.
-      Value *W = expandCodeFor(SE.getNegativeSCEV(Op), Ty);
+      Value *W = expandCodeForImpl(SE.getNegativeSCEV(Op), Ty, false);
       Sum = InsertNoopCastOfTo(Sum, Ty);
       Sum = InsertBinop(Instruction::Sub, Sum, W, SCEV::FlagAnyWrap,
                         /*IsSafeToHoist*/ true);
       ++I;
     } else {
       // A simple add.
-      Value *W = expandCodeFor(Op, Ty);
+      Value *W = expandCodeForImpl(Op, Ty, false);
       Sum = InsertNoopCastOfTo(Sum, Ty);
       // Canonicalize a constant to the RHS.
       if (isa<Constant>(Sum)) std::swap(Sum, W);
@@ -802,7 +804,7 @@ Value *SCEVExpander::visitMulExpr(const SCEVMulExpr *S) {
 
     // Calculate powers with exponents 1, 2, 4, 8 etc. and include those of them
     // that are needed into the result.
-    Value *P = expandCodeFor(I->second, Ty);
+    Value *P = expandCodeForImpl(I->second, Ty, false);
     Value *Result = nullptr;
     if (Exponent & 1)
       Result = P;
@@ -861,7 +863,7 @@ Value *SCEVExpander::visitMulExpr(const SCEVMulExpr *S) {
 Value *SCEVExpander::visitUDivExpr(const SCEVUDivExpr *S) {
   Type *Ty = SE.getEffectiveSCEVType(S->getType());
 
-  Value *LHS = expandCodeFor(S->getLHS(), Ty);
+  Value *LHS = expandCodeForImpl(S->getLHS(), Ty, false);
   if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(S->getRHS())) {
     const APInt &RHS = SC->getAPInt();
     if (RHS.isPowerOf2())
@@ -870,7 +872,7 @@ Value *SCEVExpander::visitUDivExpr(const SCEVUDivExpr *S) {
                          SCEV::FlagAnyWrap, /*IsSafeToHoist*/ true);
   }
 
-  Value *RHS = expandCodeFor(S->getRHS(), Ty);
+  Value *RHS = expandCodeForImpl(S->getRHS(), Ty, false);
   return InsertBinop(Instruction::UDiv, LHS, RHS, SCEV::FlagAnyWrap,
                      /*IsSafeToHoist*/ SE.isKnownNonZero(S->getRHS()));
 }
@@ -1265,8 +1267,9 @@ SCEVExpander::getAddRecExprPHILiterally(const SCEVAddRecExpr *Normalized,
   // Expand code for the start value into the loop preheader.
   assert(L->getLoopPreheader() &&
          "Can't expand add recurrences without a loop preheader!");
-  Value *StartV = expandCodeFor(Normalized->getStart(), ExpandTy,
-                                L->getLoopPreheader()->getTerminator());
+  Value *StartV =
+      expandCodeForImpl(Normalized->getStart(), ExpandTy,
+                        L->getLoopPreheader()->getTerminator(), false);
 
   // StartV must have been be inserted into L's preheader to dominate the new
   // phi.
@@ -1284,8 +1287,8 @@ SCEVExpander::getAddRecExprPHILiterally(const SCEVAddRecExpr *Normalized,
   if (useSubtract)
     Step = SE.getNegativeSCEV(Step);
   // Expand the step somewhere that dominates the loop header.
-  Value *StepV = expandCodeFor(Step, IntTy,
-                               &*L->getHeader()->getFirstInsertionPt());
+  Value *StepV = expandCodeForImpl(
+      Step, IntTy, &*L->getHeader()->getFirstInsertionPt(), false);
 
   // The no-wrap behavior proved by IsIncrement(NUW|NSW) is only applicable if
   // we actually do emit an addition.  It does not apply if we emit a
@@ -1430,8 +1433,8 @@ Value *SCEVExpander::expandAddRecExprLiterally(const SCEVAddRecExpr *S) {
       {
         // Expand the step somewhere that dominates the loop header.
         SCEVInsertPointGuard Guard(Builder, this);
-        StepV = expandCodeFor(Step, IntTy,
-                              &*L->getHeader()->getFirstInsertionPt());
+        StepV = expandCodeForImpl(
+            Step, IntTy, &*L->getHeader()->getFirstInsertionPt(), false);
       }
       Result = expandIVInc(PN, StepV, L, ExpandTy, IntTy, useSubtract);
     }
@@ -1450,8 +1453,8 @@ Value *SCEVExpander::expandAddRecExprLiterally(const SCEVAddRecExpr *S) {
 
     // Invert the result.
     if (InvertStep)
-      Result = Builder.CreateSub(expandCodeFor(Normalized->getStart(), TruncTy),
-                                 Result);
+      Result = Builder.CreateSub(
+          expandCodeForImpl(Normalized->getStart(), TruncTy, false), Result);
   }
 
   // Re-apply any non-loop-dominating scale.
@@ -1459,22 +1462,22 @@ Value *SCEVExpander::expandAddRecExprLiterally(const SCEVAddRecExpr *S) {
     assert(S->isAffine() && "Can't linearly scale non-affine recurrences.");
     Result = InsertNoopCastOfTo(Result, IntTy);
     Result = Builder.CreateMul(Result,
-                               expandCodeFor(PostLoopScale, IntTy));
+                               expandCodeForImpl(PostLoopScale, IntTy, false));
   }
 
   // Re-apply any non-loop-dominating offset.
   if (PostLoopOffset) {
     if (PointerType *PTy = dyn_cast<PointerType>(ExpandTy)) {
       if (Result->getType()->isIntegerTy()) {
-        Value *Base = expandCodeFor(PostLoopOffset, ExpandTy);
+        Value *Base = expandCodeForImpl(PostLoopOffset, ExpandTy, false);
         Result = expandAddToGEP(SE.getUnknown(Result), PTy, IntTy, Base);
       } else {
         Result = expandAddToGEP(PostLoopOffset, PTy, IntTy, Result);
       }
     } else {
       Result = InsertNoopCastOfTo(Result, IntTy);
-      Result = Builder.CreateAdd(Result,
-                                 expandCodeFor(PostLoopOffset, IntTy));
+      Result = Builder.CreateAdd(
+          Result, expandCodeForImpl(PostLoopOffset, IntTy, false));
     }
   }
 
@@ -1516,8 +1519,8 @@ Value *SCEVExpander::visitAddRecExpr(const SCEVAddRecExpr *S) {
                                        S->getNoWrapFlags(SCEV::FlagNW)));
     BasicBlock::iterator NewInsertPt =
         findInsertPointAfter(cast<Instruction>(V), Builder.GetInsertBlock());
-    V = expandCodeFor(SE.getTruncateExpr(SE.getUnknown(V), Ty), nullptr,
-                      &*NewInsertPt);
+    V = expandCodeForImpl(SE.getTruncateExpr(SE.getUnknown(V), Ty), nullptr,
+                          &*NewInsertPt, false);
     return V;
   }
 
@@ -1632,22 +1635,25 @@ Value *SCEVExpander::visitAddRecExpr(const SCEVAddRecExpr *S) {
 
 Value *SCEVExpander::visitTruncateExpr(const SCEVTruncateExpr *S) {
   Type *Ty = SE.getEffectiveSCEVType(S->getType());
-  Value *V = expandCodeFor(S->getOperand(),
-                           SE.getEffectiveSCEVType(S->getOperand()->getType()));
+  Value *V = expandCodeForImpl(
+      S->getOperand(), SE.getEffectiveSCEVType(S->getOperand()->getType()),
+      false);
   return Builder.CreateTrunc(V, Ty);
 }
 
 Value *SCEVExpander::visitZeroExtendExpr(const SCEVZeroExtendExpr *S) {
   Type *Ty = SE.getEffectiveSCEVType(S->getType());
-  Value *V = expandCodeFor(S->getOperand(),
-                           SE.getEffectiveSCEVType(S->getOperand()->getType()));
+  Value *V = expandCodeForImpl(
+      S->getOperand(), SE.getEffectiveSCEVType(S->getOperand()->getType()),
+      false);
   return Builder.CreateZExt(V, Ty);
 }
 
 Value *SCEVExpander::visitSignExtendExpr(const SCEVSignExtendExpr *S) {
   Type *Ty = SE.getEffectiveSCEVType(S->getType());
-  Value *V = expandCodeFor(S->getOperand(),
-                           SE.getEffectiveSCEVType(S->getOperand()->getType()));
+  Value *V = expandCodeForImpl(
+      S->getOperand(), SE.getEffectiveSCEVType(S->getOperand()->getType()),
+      false);
   return Builder.CreateSExt(V, Ty);
 }
 
@@ -1662,7 +1668,7 @@ Value *SCEVExpander::visitSMaxExpr(const SCEVSMaxExpr *S) {
       Ty = SE.getEffectiveSCEVType(Ty);
       LHS = InsertNoopCastOfTo(LHS, Ty);
     }
-    Value *RHS = expandCodeFor(S->getOperand(i), Ty);
+    Value *RHS = expandCodeForImpl(S->getOperand(i), Ty, false);
     Value *ICmp = Builder.CreateICmpSGT(LHS, RHS);
     Value *Sel = Builder.CreateSelect(ICmp, LHS, RHS, "smax");
     LHS = Sel;
@@ -1685,7 +1691,7 @@ Value *SCEVExpander::visitUMaxExpr(const SCEVUMaxExpr *S) {
       Ty = SE.getEffectiveSCEVType(Ty);
       LHS = InsertNoopCastOfTo(LHS, Ty);
     }
-    Value *RHS = expandCodeFor(S->getOperand(i), Ty);
+    Value *RHS = expandCodeForImpl(S->getOperand(i), Ty, false);
     Value *ICmp = Builder.CreateICmpUGT(LHS, RHS);
     Value *Sel = Builder.CreateSelect(ICmp, LHS, RHS, "umax");
     LHS = Sel;
@@ -1708,7 +1714,7 @@ Value *SCEVExpander::visitSMinExpr(const SCEVSMinExpr *S) {
       Ty = SE.getEffectiveSCEVType(Ty);
       LHS = InsertNoopCastOfTo(LHS, Ty);
     }
-    Value *RHS = expandCodeFor(S->getOperand(i), Ty);
+    Value *RHS = expandCodeForImpl(S->getOperand(i), Ty, false);
     Value *ICmp = Builder.CreateICmpSLT(LHS, RHS);
     Value *Sel = Builder.CreateSelect(ICmp, LHS, RHS, "smin");
     LHS = Sel;
@@ -1731,7 +1737,7 @@ Value *SCEVExpander::visitUMinExpr(const SCEVUMinExpr *S) {
       Ty = SE.getEffectiveSCEVType(Ty);
       LHS = InsertNoopCastOfTo(LHS, Ty);
     }
-    Value *RHS = expandCodeFor(S->getOperand(i), Ty);
+    Value *RHS = expandCodeForImpl(S->getOperand(i), Ty, false);
     Value *ICmp = Builder.CreateICmpULT(LHS, RHS);
     Value *Sel = Builder.CreateSelect(ICmp, LHS, RHS, "umin");
     LHS = Sel;
@@ -1743,15 +1749,43 @@ Value *SCEVExpander::visitUMinExpr(const SCEVUMinExpr *S) {
   return LHS;
 }
 
-Value *SCEVExpander::expandCodeFor(const SCEV *SH, Type *Ty,
-                                   Instruction *IP) {
+Value *SCEVExpander::expandCodeForImpl(const SCEV *SH, Type *Ty,
+                                       Instruction *IP, bool Root) {
   setInsertPoint(IP);
-  return expandCodeFor(SH, Ty);
+  Value *V = expandCodeForImpl(SH, Ty, Root);
+  return V;
 }
 
-Value *SCEVExpander::expandCodeFor(const SCEV *SH, Type *Ty) {
+Value *SCEVExpander::expandCodeForImpl(const SCEV *SH, Type *Ty, bool Root) {
   // Expand the code for this SCEV.
   Value *V = expand(SH);
+
+  if (PreserveLCSSA) {
+    if (auto *Inst = dyn_cast<Instruction>(V)) {
+      // Create a temporary instruction to at the current insertion point, so we
+      // can hand it off to the helper to create LCSSA PHIs if required for the
+      // new use.
+      // FIXME: Ideally formLCSSAForInstructions (used in fixupLCSSAFormFor)
+      // would accept a insertion point and return an LCSSA phi for that
+      // insertion point, so there is no need to insert & remove the temporary
+      // instruction.
+      Instruction *Tmp;
+      if (Inst->getType()->isIntegerTy())
+        Tmp = cast<Instruction>(Builder.CreateAdd(Inst, Inst));
+      else {
+        assert(Inst->getType()->isPointerTy());
+        Tmp = cast<Instruction>(Builder.CreateGEP(Inst, Builder.getInt32(1)));
+      }
+      V = fixupLCSSAFormFor(Tmp, 0);
+
+      // Clean up temporary instruction.
+      InsertedValues.erase(Tmp);
+      InsertedPostIncValues.erase(Tmp);
+      Tmp->eraseFromParent();
+    }
+  }
+
+  InsertedExpressions[std::make_pair(SH, &*Builder.GetInsertPoint())] = V;
   if (Ty) {
     assert(SE.getTypeSizeInBits(Ty) == SE.getTypeSizeInBits(SH->getType()) &&
            "non-trivial casts should be done with the SCEVs directly!");
@@ -1891,10 +1925,28 @@ Value *SCEVExpander::expand(const SCEV *S) {
 }
 
 void SCEVExpander::rememberInstruction(Value *I) {
-  if (!PostIncLoops.empty())
-    InsertedPostIncValues.insert(I);
-  else
-    InsertedValues.insert(I);
+  auto DoInsert = [this](Value *V) {
+    if (!PostIncLoops.empty())
+      InsertedPostIncValues.insert(V);
+    else
+      InsertedValues.insert(V);
+  };
+  DoInsert(I);
+
+  if (!PreserveLCSSA)
+    return;
+
+  if (auto *Inst = dyn_cast<Instruction>(I)) {
+    // A new instruction has been added, which might introduce new uses outside
+    // a defining loop. Fix LCSSA from for each operand of the new instruction,
+    // if required.
+    for (unsigned OpIdx = 0, OpEnd = Inst->getNumOperands(); OpIdx != OpEnd;
+         OpIdx++) {
+      auto *V = fixupLCSSAFormFor(Inst, OpIdx);
+      if (V != I)
+        DoInsert(V);
+    }
+  }
 }
 
 /// getOrInsertCanonicalInductionVariable - This method returns the
@@ -1913,9 +1965,8 @@ SCEVExpander::getOrInsertCanonicalInductionVariable(const Loop *L,
 
   // Emit code for it.
   SCEVInsertPointGuard Guard(Builder, this);
-  PHINode *V =
-      cast<PHINode>(expandCodeFor(H, nullptr,
-                                  &*L->getHeader()->getFirstInsertionPt()));
+  PHINode *V = cast<PHINode>(expandCodeForImpl(
+      H, nullptr, &*L->getHeader()->getFirstInsertionPt(), false));
 
   return V;
 }
@@ -2315,8 +2366,10 @@ Value *SCEVExpander::expandCodeForPredicate(const SCEVPredicate *Pred,
 
 Value *SCEVExpander::expandEqualPredicate(const SCEVEqualPredicate *Pred,
                                           Instruction *IP) {
-  Value *Expr0 = expandCodeFor(Pred->getLHS(), Pred->getLHS()->getType(), IP);
-  Value *Expr1 = expandCodeFor(Pred->getRHS(), Pred->getRHS()->getType(), IP);
+  Value *Expr0 =
+      expandCodeForImpl(Pred->getLHS(), Pred->getLHS()->getType(), IP, false);
+  Value *Expr1 =
+      expandCodeForImpl(Pred->getRHS(), Pred->getRHS()->getType(), IP, false);
 
   Builder.SetInsertPoint(IP);
   auto *I = Builder.CreateICmpNE(Expr0, Expr1, "ident.check");
@@ -2348,15 +2401,16 @@ Value *SCEVExpander::generateOverflowCheck(const SCEVAddRecExpr *AR,
 
   IntegerType *CountTy = IntegerType::get(Loc->getContext(), SrcBits);
   Builder.SetInsertPoint(Loc);
-  Value *TripCountVal = expandCodeFor(ExitCount, CountTy, Loc);
+  Value *TripCountVal = expandCodeForImpl(ExitCount, CountTy, Loc, false);
 
   IntegerType *Ty =
       IntegerType::get(Loc->getContext(), SE.getTypeSizeInBits(ARTy));
   Type *ARExpandTy = DL.isNonIntegralPointerType(ARTy) ? ARTy : Ty;
 
-  Value *StepValue = expandCodeFor(Step, Ty, Loc);
-  Value *NegStepValue = expandCodeFor(SE.getNegativeSCEV(Step), Ty, Loc);
-  Value *StartValue = expandCodeFor(Start, ARExpandTy, Loc);
+  Value *StepValue = expandCodeForImpl(Step, Ty, Loc, false);
+  Value *NegStepValue =
+      expandCodeForImpl(SE.getNegativeSCEV(Step), Ty, Loc, false);
+  Value *StartValue = expandCodeForImpl(Start, ARExpandTy, Loc, false);
 
   ConstantInt *Zero =
       ConstantInt::get(Loc->getContext(), APInt::getNullValue(DstBits));
@@ -2459,6 +2513,25 @@ Value *SCEVExpander::expandUnionPredicate(const SCEVUnionPredicate *Union,
   return Check;
 }
 
+Value *SCEVExpander::fixupLCSSAFormFor(Instruction *User, unsigned OpIdx) {
+  assert(PreserveLCSSA);
+  SmallVector<Instruction *, 1> ToUpdate;
+
+  auto *OpV = User->getOperand(OpIdx);
+  auto *OpI = dyn_cast<Instruction>(OpV);
+  if (!OpI)
+    return OpV;
+
+  Loop *DefLoop = SE.LI.getLoopFor(OpI->getParent());
+  Loop *UseLoop = SE.LI.getLoopFor(User->getParent());
+  if (!DefLoop || UseLoop == DefLoop || DefLoop->contains(UseLoop))
+    return OpV;
+
+  ToUpdate.push_back(OpI);
+  formLCSSAForInstructions(ToUpdate, SE.DT, SE.LI, &SE);
+  return User->getOperand(OpIdx);
+}
+
 namespace {
 // Search for a SCEV subexpression that is not safe to expand.  Any expression
 // that may expand to a !isSafeToSpeculativelyExecute value is unsafe, namely
diff --git a/llvm/unittests/Transforms/Utils/ScalarEvolutionExpanderTest.cpp b/llvm/unittests/Transforms/Utils/ScalarEvolutionExpanderTest.cpp
index c146d155ffbea..6e83370dfeb95 100644
--- a/llvm/unittests/Transforms/Utils/ScalarEvolutionExpanderTest.cpp
+++ b/llvm/unittests/Transforms/Utils/ScalarEvolutionExpanderTest.cpp
@@ -265,7 +265,7 @@ TEST_F(ScalarEvolutionExpanderTest, SCEVExpanderIsSafeToExpandAt) {
   Phi->addIncoming(Add, L);
 
   Builder.SetInsertPoint(Post);
-  Builder.CreateRetVoid();
+  Instruction *Ret = Builder.CreateRetVoid();
 
   ScalarEvolution SE = buildSE(*F);
   const SCEV *S = SE.getSCEV(Phi);
@@ -276,6 +276,11 @@ TEST_F(ScalarEvolutionExpanderTest, SCEVExpanderIsSafeToExpandAt) {
   EXPECT_FALSE(isSafeToExpandAt(AR, LPh->getTerminator(), SE));
   EXPECT_TRUE(isSafeToExpandAt(AR, L->getTerminator(), SE));
   EXPECT_TRUE(isSafeToExpandAt(AR, Post->getTerminator(), SE));
+
+  EXPECT_TRUE(LI->getLoopFor(L)->isLCSSAForm(*DT));
+  SCEVExpander Exp(SE, M.getDataLayout(), "expander");
+  Exp.expandCodeFor(SE.getSCEV(Add), nullptr, Ret);
+  EXPECT_TRUE(LI->getLoopFor(L)->isLCSSAForm(*DT));
 }
 
 // Check that SCEV expander does not use the nuw instruction

From 672df0fc670e79ce7ac710ce4b5f47804f54a18a Mon Sep 17 00:00:00 2001
From: Juneyoung Lee <aqjune@gmail.com>
Date: Wed, 29 Jul 2020 23:21:39 +0900
Subject: [PATCH 0462/1035] [InstSimplify] add tests for
 expandCommutativeBinOp; NFC

---
 .../Transforms/InstSimplify/distribute.ll     | 32 +++++++++++++++++++
 1 file changed, 32 insertions(+)
 create mode 100644 llvm/test/Transforms/InstSimplify/distribute.ll

diff --git a/llvm/test/Transforms/InstSimplify/distribute.ll b/llvm/test/Transforms/InstSimplify/distribute.ll
new file mode 100644
index 0000000000000..9134938406d70
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/distribute.ll
@@ -0,0 +1,32 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+; %result = x & (1 ^ sel) = (x & 1) ^ (x & sel)
+;         = x ^ (select cond, false, false)
+;         = x ^ false = x
+;
+define i1 @f(i1 %cond, i1 noundef %x, i1 %y, i1 %z) {
+; CHECK-LABEL: @f(
+; CHECK-NEXT:    ret i1 [[X:%.*]]
+;
+  %notx = xor i1 %x, 1
+  %lhs = and i1 %notx, %y
+  %rhs = and i1 %notx, %z
+  %sel = select i1 %cond, i1 %lhs, i1 %rhs
+  %op1 = xor i1 1, %sel
+  %result = and i1 %x, %op1
+  ret i1 %result
+}
+
+define i1 @f_dontfold(i1 %cond, i1 %x, i1 %y, i1 %z) {
+; CHECK-LABEL: @f_dontfold(
+; CHECK-NEXT:    ret i1 [[X:%.*]]
+;
+  %notx = xor i1 %x, 1
+  %lhs = and i1 %notx, %y
+  %rhs = and i1 %notx, %z
+  %sel = select i1 %cond, i1 %lhs, i1 %rhs
+  %op1 = xor i1 1, %sel
+  %result = and i1 %x, %op1
+  ret i1 %result
+}

From 2c662f3d3d957365ad86f35eee0bea05e4cf0188 Mon Sep 17 00:00:00 2001
From: Tim Keith <tkeith@nvidia.com>
Date: Wed, 29 Jul 2020 07:23:28 -0700
Subject: [PATCH 0463/1035] [flang] Fix bug with intrinsic in type declaration
 stmt

When an instrinsic function is declared in a type declaration statement
we need to set the INTRINSIC attribute and (per 8.2(3)) ignore the
specified type.

To simplify the check, add IsIntrinsic utility to BaseVisitor.

Also, intrinsics and external procedures were getting assigned a size
and offset and they shouldn't be.

Differential Revision: https://reviews.llvm.org/D84702
---
 flang/include/flang/Semantics/symbol.h  |  5 +----
 flang/lib/Semantics/compute-offsets.cpp |  5 ++++-
 flang/lib/Semantics/resolve-names.cpp   | 19 +++++++++++++------
 flang/test/Semantics/symbol18.f90       | 21 +++++++++++++++++++++
 4 files changed, 39 insertions(+), 11 deletions(-)
 create mode 100644 flang/test/Semantics/symbol18.f90

diff --git a/flang/include/flang/Semantics/symbol.h b/flang/include/flang/Semantics/symbol.h
index 3000a39c3b589..c0a50364b63db 100644
--- a/flang/include/flang/Semantics/symbol.h
+++ b/flang/include/flang/Semantics/symbol.h
@@ -219,10 +219,7 @@ class ProcEntityDetails : public EntityDetails, public WithPassArg {
 
   const ProcInterface &interface() const { return interface_; }
   ProcInterface &interface() { return interface_; }
-  void set_interface(const ProcInterface &interface) {
-    CHECK(!IsInterfaceSet());
-    interface_ = interface;
-  }
+  void set_interface(const ProcInterface &interface) { interface_ = interface; }
   bool IsInterfaceSet() {
     return interface_.symbol() != nullptr || interface_.type() != nullptr;
   }
diff --git a/flang/lib/Semantics/compute-offsets.cpp b/flang/lib/Semantics/compute-offsets.cpp
index bcc4c0caf46a5..8b24f6a1f3eae 100644
--- a/flang/lib/Semantics/compute-offsets.cpp
+++ b/flang/lib/Semantics/compute-offsets.cpp
@@ -257,7 +257,7 @@ auto ComputeOffsetsHelper::GetElementSize(const Symbol &symbol)
   // TODO: The size of procedure pointers is not yet known
   // and is independent of rank (and probably also the number
   // of length type parameters).
-  if (IsDescriptor(symbol) || IsProcedure(symbol)) {
+  if (IsDescriptor(symbol) || IsProcedurePointer(symbol)) {
     int lenParams{0};
     if (const DerivedTypeSpec * derived{type->AsDerived()}) {
       lenParams = CountLenParameters(*derived);
@@ -266,6 +266,9 @@ auto ComputeOffsetsHelper::GetElementSize(const Symbol &symbol)
         runtime::Descriptor::SizeInBytes(symbol.Rank(), false, lenParams)};
     return {size, maxAlignment};
   }
+  if (IsProcedure(symbol)) {
+    return {};
+  }
   SizeAndAlignment result;
   if (const IntrinsicTypeSpec * intrinsic{type->AsIntrinsic()}) {
     if (auto kind{ToInt64(intrinsic->kind())}) {
diff --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp
index c1aef4b9b34c3..50ea735d81d5e 100644
--- a/flang/lib/Semantics/resolve-names.cpp
+++ b/flang/lib/Semantics/resolve-names.cpp
@@ -156,6 +156,9 @@ class BaseVisitor {
   evaluate::FoldingContext &GetFoldingContext() const {
     return context_->foldingContext();
   }
+  bool IsIntrinsic(const SourceName &name) const {
+    return context_->intrinsics().IsIntrinsic(name.ToString());
+  }
 
   // Make a placeholder symbol for a Name that otherwise wouldn't have one.
   // It is not in any scope and always has MiscDetails.
@@ -2046,14 +2049,14 @@ static bool NeedsType(const Symbol &symbol) {
                  },
           symbol.details());
 }
+
 void ScopeHandler::ApplyImplicitRules(Symbol &symbol) {
   if (NeedsType(symbol)) {
     if (const DeclTypeSpec * type{GetImplicitType(symbol)}) {
       symbol.set(Symbol::Flag::Implicit);
       symbol.SetType(*type);
     } else if (symbol.has<ProcEntityDetails>() &&
-        !symbol.attrs().test(Attr::EXTERNAL) &&
-        context().intrinsics().IsIntrinsic(symbol.name().ToString())) {
+        !symbol.attrs().test(Attr::EXTERNAL) && IsIntrinsic(symbol.name())) {
       // type will be determined in expression semantics
       symbol.attrs().set(Attr::INTRINSIC);
     } else if (!context().HasError(symbol)) {
@@ -2062,6 +2065,7 @@ void ScopeHandler::ApplyImplicitRules(Symbol &symbol) {
     }
   }
 }
+
 const DeclTypeSpec *ScopeHandler::GetImplicitType(Symbol &symbol) {
   const DeclTypeSpec *type{implicitRules().GetType(symbol.name().begin()[0])};
   if (type) {
@@ -3284,8 +3288,7 @@ bool DeclarationVisitor::HandleAttributeStmt(
 }
 Symbol &DeclarationVisitor::HandleAttributeStmt(
     Attr attr, const parser::Name &name) {
-  if (attr == Attr::INTRINSIC &&
-      !context().intrinsics().IsIntrinsic(name.source.ToString())) {
+  if (attr == Attr::INTRINSIC && !IsIntrinsic(name.source)) {
     Say(name.source, "'%s' is not a known intrinsic procedure"_err_en_US);
   }
   auto *symbol{FindInScope(currScope(), name)};
@@ -5700,7 +5703,7 @@ void ResolveNamesVisitor::HandleProcedureName(
   CHECK(flag == Symbol::Flag::Function || flag == Symbol::Flag::Subroutine);
   auto *symbol{FindSymbol(NonDerivedTypeScope(), name)};
   if (!symbol) {
-    if (context().intrinsics().IsIntrinsic(name.source.ToString())) {
+    if (IsIntrinsic(name.source)) {
       symbol =
           &MakeSymbol(InclusiveScope(), name.source, Attrs{Attr::INTRINSIC});
     } else {
@@ -5729,7 +5732,11 @@ void ResolveNamesVisitor::HandleProcedureName(
     // error was reported
   } else {
     symbol = &Resolve(name, symbol)->GetUltimate();
-    ConvertToProcEntity(*symbol);
+    if (ConvertToProcEntity(*symbol) && IsIntrinsic(symbol->name())) {
+      symbol->attrs().set(Attr::INTRINSIC);
+      // 8.2(3): ignore type from intrinsic in type-declaration-stmt
+      symbol->get<ProcEntityDetails>().set_interface(ProcInterface{});
+    }
     if (!SetProcFlag(name, *symbol, flag)) {
       return; // reported error
     }
diff --git a/flang/test/Semantics/symbol18.f90 b/flang/test/Semantics/symbol18.f90
new file mode 100644
index 0000000000000..b7269b70be0a9
--- /dev/null
+++ b/flang/test/Semantics/symbol18.f90
@@ -0,0 +1,21 @@
+! RUN: %S/test_symbols.sh %s %t %f18
+
+! Intrinsic function in type declaration statement: type is ignored
+
+!DEF: /p1 MainProgram
+program p1
+ !DEF: /p1/cos INTRINSIC (Function) ProcEntity
+ integer cos
+ !DEF: /p1/y (Implicit) ObjectEntity REAL(4)
+ !REF: /p1/cos
+ !DEF: /p1/x (Implicit) ObjectEntity REAL(4)
+ y = cos(x)
+ !REF: /p1/y
+ !DEF: /p1/sin INTRINSIC (Function) ProcEntity
+ !REF: /p1/x
+ y = sin(x)
+ !REF: /p1/y
+ !DEF: /f EXTERNAL (Function, Implicit) ProcEntity REAL(4)
+ !REF: /p1/x
+ y = f(x)
+end program

From 8d27be8dbaffce0519ac41173d51923fc2524b1b Mon Sep 17 00:00:00 2001
From: Alexey Bader <alexey.bader@intel.com>
Date: Wed, 29 Jul 2020 15:07:06 +0300
Subject: [PATCH 0464/1035] [OpenCL] Add global_device and global_host address
 spaces

This patch introduces 2 new address spaces in OpenCL: global_device and global_host
which are a subset of a global address space, so the address space scheme will be
looking like:

```
generic->global->host
                          ->device
             ->private
             ->local
constant
```

Justification: USM allocations may be associated with both host and device memory. We
want to give users a way to tell the compiler the allocation type of a USM pointer for
optimization purposes. (Link to the Unified Shared Memory extension:
https://github.com/intel/llvm/blob/sycl/sycl/doc/extensions/USM/cl_intel_unified_shared_memory.asciidoc)

Before this patch USM pointer could be only in opencl_global
address space, hence a device backend can't tell if a particular pointer
points to host or device memory. On FPGAs at least we can generate more
efficient hardware code if the user tells us where the pointer can point -
being able to distinguish between these types of pointers at compile time
allows us to instantiate simpler load-store units to perform memory
transactions.

Patch by Dmitry Sidorov.

Reviewed By: Anastasia

Differential Revision: https://reviews.llvm.org/D82174
---
 clang/include/clang/AST/Type.h                |  5 ++
 clang/include/clang/Basic/AddressSpaces.h     |  2 +
 clang/include/clang/Basic/Attr.td             | 10 +++
 clang/include/clang/Basic/AttrDocs.td         | 24 +++++++
 clang/include/clang/Sema/ParsedAttr.h         |  4 ++
 clang/lib/AST/ASTContext.cpp                  | 26 +++----
 clang/lib/AST/ItaniumMangle.cpp               | 41 +++++++++---
 clang/lib/AST/MicrosoftMangle.cpp             |  8 ++-
 clang/lib/AST/TypePrinter.cpp                 |  6 ++
 clang/lib/Basic/Targets/AMDGPU.cpp            |  4 ++
 clang/lib/Basic/Targets/NVPTX.h               |  2 +
 clang/lib/Basic/Targets/SPIR.h                |  2 +
 clang/lib/Basic/Targets/TCE.h                 |  2 +
 clang/lib/Basic/Targets/X86.h                 |  2 +
 clang/lib/CodeGen/CodeGenModule.cpp           | 18 +++--
 clang/lib/Sema/SemaType.cpp                   |  4 ++
 .../AST/language_address_space_attribute.cpp  | 12 ++++
 .../test/CodeGenCXX/mangle-address-space.cpp  | 12 ++++
 .../address-spaces-conversions.cl             | 28 +++++++-
 clang/test/CodeGenOpenCL/address-spaces.cl    |  8 +++
 .../usm-address-spaces-conversions.cl         | 67 +++++++++++++++++++
 .../SemaTemplate/address_space-dependent.cpp  |  4 +-
 22 files changed, 262 insertions(+), 29 deletions(-)
 create mode 100644 clang/test/SemaOpenCL/usm-address-spaces-conversions.cl

diff --git a/clang/include/clang/AST/Type.h b/clang/include/clang/AST/Type.h
index 7fe652492b0e0..7d943ebc78c01 100644
--- a/clang/include/clang/AST/Type.h
+++ b/clang/include/clang/AST/Type.h
@@ -480,6 +480,11 @@ class Qualifiers {
            // Otherwise in OpenCLC v2.0 s6.5.5: every address space except
            // for __constant can be used as __generic.
            (A == LangAS::opencl_generic && B != LangAS::opencl_constant) ||
+           // We also define global_device and global_host address spaces,
+           // to distinguish global pointers allocated on host from pointers
+           // allocated on device, which are a subset of __global.
+           (A == LangAS::opencl_global && (B == LangAS::opencl_global_device ||
+                                           B == LangAS::opencl_global_host)) ||
            // Consider pointer size address spaces to be equivalent to default.
            ((isPtrSizeAddressSpace(A) || A == LangAS::Default) &&
             (isPtrSizeAddressSpace(B) || B == LangAS::Default));
diff --git a/clang/include/clang/Basic/AddressSpaces.h b/clang/include/clang/Basic/AddressSpaces.h
index faf7f303aa2d6..a9db52dfcc9c8 100644
--- a/clang/include/clang/Basic/AddressSpaces.h
+++ b/clang/include/clang/Basic/AddressSpaces.h
@@ -36,6 +36,8 @@ enum class LangAS : unsigned {
   opencl_constant,
   opencl_private,
   opencl_generic,
+  opencl_global_device,
+  opencl_global_host,
 
   // CUDA specific address spaces.
   cuda_device,
diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td
index 0ee3c5188563f..f9bf3f0acd553 100644
--- a/clang/include/clang/Basic/Attr.td
+++ b/clang/include/clang/Basic/Attr.td
@@ -1178,6 +1178,16 @@ def OpenCLGlobalAddressSpace : TypeAttr {
   let Documentation = [OpenCLAddressSpaceGlobalDocs];
 }
 
+def OpenCLGlobalDeviceAddressSpace : TypeAttr {
+  let Spellings = [Clang<"opencl_global_device">];
+  let Documentation = [OpenCLAddressSpaceGlobalExtDocs];
+}
+
+def OpenCLGlobalHostAddressSpace : TypeAttr {
+  let Spellings = [Clang<"opencl_global_host">];
+  let Documentation = [OpenCLAddressSpaceGlobalExtDocs];
+}
+
 def OpenCLLocalAddressSpace : TypeAttr {
   let Spellings = [Keyword<"__local">, Keyword<"local">, Clang<"opencl_local">];
   let Documentation = [OpenCLAddressSpaceLocalDocs];
diff --git a/clang/include/clang/Basic/AttrDocs.td b/clang/include/clang/Basic/AttrDocs.td
index c835e6da04d59..b9fcf9af323b7 100644
--- a/clang/include/clang/Basic/AttrDocs.td
+++ b/clang/include/clang/Basic/AttrDocs.td
@@ -3123,6 +3123,30 @@ scope) variables and static local variable as well.
   }];
 }
 
+def OpenCLAddressSpaceGlobalExtDocs : Documentation {
+  let Category = DocOpenCLAddressSpaces;
+  let Heading = "[[clang::opencl_global_device]], [[clang::opencl_global_host]]";
+  let Content = [{
+The ``global_device`` and ``global_host`` address space attributes specify that
+an object is allocated in global memory on the device/host. It helps to
+distinguish USM (Unified Shared Memory) pointers that access global device
+memory from those that access global host memory. These new address spaces are
+a subset of the ``__global/opencl_global`` address space, the full address space
+set model for OpenCL 2.0 with the extension looks as follows:
+  generic->global->host
+                 ->device
+         ->private
+         ->local
+  constant
+
+As ``global_device`` and ``global_host`` are a subset of
+``__global/opencl_global`` address spaces it is allowed to convert
+``global_device`` and ``global_host`` address spaces to
+``__global/opencl_global`` address spaces (following ISO/IEC TR 18037 5.1.3
+"Address space nesting and rules for pointers).
+  }];
+}
+
 def OpenCLAddressSpaceLocalDocs : Documentation {
   let Category = DocOpenCLAddressSpaces;
   let Heading = "__local, local, [[clang::opencl_local]]";
diff --git a/clang/include/clang/Sema/ParsedAttr.h b/clang/include/clang/Sema/ParsedAttr.h
index 21e030fe51346..8946b12ee03fc 100644
--- a/clang/include/clang/Sema/ParsedAttr.h
+++ b/clang/include/clang/Sema/ParsedAttr.h
@@ -606,6 +606,10 @@ class ParsedAttr final
       return LangAS::opencl_constant;
     case ParsedAttr::AT_OpenCLGlobalAddressSpace:
       return LangAS::opencl_global;
+    case ParsedAttr::AT_OpenCLGlobalDeviceAddressSpace:
+      return LangAS::opencl_global_device;
+    case ParsedAttr::AT_OpenCLGlobalHostAddressSpace:
+      return LangAS::opencl_global_host;
     case ParsedAttr::AT_OpenCLLocalAddressSpace:
       return LangAS::opencl_local;
     case ParsedAttr::AT_OpenCLPrivateAddressSpace:
diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index 25bf71519d1c3..43bbe41fb6112 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -919,18 +919,20 @@ static const LangASMap *getAddressSpaceMap(const TargetInfo &T,
     // The fake address space map must have a distinct entry for each
     // language-specific address space.
     static const unsigned FakeAddrSpaceMap[] = {
-        0, // Default
-        1, // opencl_global
-        3, // opencl_local
-        2, // opencl_constant
-        0, // opencl_private
-        4, // opencl_generic
-        5, // cuda_device
-        6, // cuda_constant
-        7, // cuda_shared
-        8, // ptr32_sptr
-        9, // ptr32_uptr
-        10 // ptr64
+        0,  // Default
+        1,  // opencl_global
+        3,  // opencl_local
+        2,  // opencl_constant
+        0,  // opencl_private
+        4,  // opencl_generic
+        5,  // opencl_global_device
+        6,  // opencl_global_host
+        7,  // cuda_device
+        8,  // cuda_constant
+        9,  // cuda_shared
+        10, // ptr32_sptr
+        11, // ptr32_uptr
+        12  // ptr64
     };
     return &FakeAddrSpaceMap;
   } else {
diff --git a/clang/lib/AST/ItaniumMangle.cpp b/clang/lib/AST/ItaniumMangle.cpp
index ddfbe9f864991..2b207c17d9393 100644
--- a/clang/lib/AST/ItaniumMangle.cpp
+++ b/clang/lib/AST/ItaniumMangle.cpp
@@ -2388,16 +2388,39 @@ void CXXNameMangler::mangleQualifiers(Qualifiers Quals, const DependentAddressSp
       switch (AS) {
       default: llvm_unreachable("Not a language specific address space");
       //  <OpenCL-addrspace> ::= "CL" [ "global" | "local" | "constant" |
-      //                                "private"| "generic" ]
-      case LangAS::opencl_global:   ASString = "CLglobal";   break;
-      case LangAS::opencl_local:    ASString = "CLlocal";    break;
-      case LangAS::opencl_constant: ASString = "CLconstant"; break;
-      case LangAS::opencl_private:  ASString = "CLprivate";  break;
-      case LangAS::opencl_generic:  ASString = "CLgeneric";  break;
+      //                                "private"| "generic" | "device" |
+      //                                "host" ]
+      case LangAS::opencl_global:
+        ASString = "CLglobal";
+        break;
+      case LangAS::opencl_global_device:
+        ASString = "CLdevice";
+        break;
+      case LangAS::opencl_global_host:
+        ASString = "CLhost";
+        break;
+      case LangAS::opencl_local:
+        ASString = "CLlocal";
+        break;
+      case LangAS::opencl_constant:
+        ASString = "CLconstant";
+        break;
+      case LangAS::opencl_private:
+        ASString = "CLprivate";
+        break;
+      case LangAS::opencl_generic:
+        ASString = "CLgeneric";
+        break;
       //  <CUDA-addrspace> ::= "CU" [ "device" | "constant" | "shared" ]
-      case LangAS::cuda_device:     ASString = "CUdevice";   break;
-      case LangAS::cuda_constant:   ASString = "CUconstant"; break;
-      case LangAS::cuda_shared:     ASString = "CUshared";   break;
+      case LangAS::cuda_device:
+        ASString = "CUdevice";
+        break;
+      case LangAS::cuda_constant:
+        ASString = "CUconstant";
+        break;
+      case LangAS::cuda_shared:
+        ASString = "CUshared";
+        break;
       //  <ptrsize-addrspace> ::= [ "ptr32_sptr" | "ptr32_uptr" | "ptr64" ]
       case LangAS::ptr32_sptr:
         ASString = "ptr32_sptr";
diff --git a/clang/lib/AST/MicrosoftMangle.cpp b/clang/lib/AST/MicrosoftMangle.cpp
index 09579c28061ae..55ac7629a54c3 100644
--- a/clang/lib/AST/MicrosoftMangle.cpp
+++ b/clang/lib/AST/MicrosoftMangle.cpp
@@ -1798,7 +1798,7 @@ void MicrosoftCXXNameMangler::mangleAddressSpaceType(QualType T,
   // where:
   //  <language_addr_space> ::= <OpenCL-addrspace> | <CUDA-addrspace>
   //    <OpenCL-addrspace> ::= "CL" [ "global" | "local" | "constant" |
-  //                                "private"| "generic" ]
+  //                                "private"| "generic" | "device" | "host" ]
   //    <CUDA-addrspace> ::= "CU" [ "device" | "constant" | "shared" ]
   //    Note that the above were chosen to match the Itanium mangling for this.
   //
@@ -1823,6 +1823,12 @@ void MicrosoftCXXNameMangler::mangleAddressSpaceType(QualType T,
     case LangAS::opencl_global:
       Extra.mangleSourceName("_ASCLglobal");
       break;
+    case LangAS::opencl_global_device:
+      Extra.mangleSourceName("_ASCLdevice");
+      break;
+    case LangAS::opencl_global_host:
+      Extra.mangleSourceName("_ASCLhost");
+      break;
     case LangAS::opencl_local:
       Extra.mangleSourceName("_ASCLlocal");
       break;
diff --git a/clang/lib/AST/TypePrinter.cpp b/clang/lib/AST/TypePrinter.cpp
index eff8e99764350..7286a88f3fc41 100644
--- a/clang/lib/AST/TypePrinter.cpp
+++ b/clang/lib/AST/TypePrinter.cpp
@@ -1564,6 +1564,8 @@ void TypePrinter::printAttributedAfter(const AttributedType *T,
 
   case attr::OpenCLPrivateAddressSpace:
   case attr::OpenCLGlobalAddressSpace:
+  case attr::OpenCLGlobalDeviceAddressSpace:
+  case attr::OpenCLGlobalHostAddressSpace:
   case attr::OpenCLLocalAddressSpace:
   case attr::OpenCLConstantAddressSpace:
   case attr::OpenCLGenericAddressSpace:
@@ -1866,6 +1868,10 @@ std::string Qualifiers::getAddrSpaceAsString(LangAS AS) {
     return "__constant";
   case LangAS::opencl_generic:
     return "__generic";
+  case LangAS::opencl_global_device:
+    return "__global_device";
+  case LangAS::opencl_global_host:
+    return "__global_host";
   case LangAS::cuda_device:
     return "__device__";
   case LangAS::cuda_constant:
diff --git a/clang/lib/Basic/Targets/AMDGPU.cpp b/clang/lib/Basic/Targets/AMDGPU.cpp
index db7db8d36d037..e147045110a99 100644
--- a/clang/lib/Basic/Targets/AMDGPU.cpp
+++ b/clang/lib/Basic/Targets/AMDGPU.cpp
@@ -46,6 +46,8 @@ const LangASMap AMDGPUTargetInfo::AMDGPUDefIsGenMap = {
     Constant, // opencl_constant
     Private,  // opencl_private
     Generic,  // opencl_generic
+    Global,   // opencl_global_device
+    Global,   // opencl_global_host
     Global,   // cuda_device
     Constant, // cuda_constant
     Local,    // cuda_shared
@@ -61,6 +63,8 @@ const LangASMap AMDGPUTargetInfo::AMDGPUDefIsPrivMap = {
     Constant, // opencl_constant
     Private,  // opencl_private
     Generic,  // opencl_generic
+    Global,   // opencl_global_device
+    Global,   // opencl_global_host
     Global,   // cuda_device
     Constant, // cuda_constant
     Local,    // cuda_shared
diff --git a/clang/lib/Basic/Targets/NVPTX.h b/clang/lib/Basic/Targets/NVPTX.h
index f57a0f18efa3f..84828f0396c39 100644
--- a/clang/lib/Basic/Targets/NVPTX.h
+++ b/clang/lib/Basic/Targets/NVPTX.h
@@ -30,6 +30,8 @@ static const unsigned NVPTXAddrSpaceMap[] = {
     0, // opencl_private
     // FIXME: generic has to be added to the target
     0, // opencl_generic
+    1, // opencl_global_device
+    1, // opencl_global_host
     1, // cuda_device
     4, // cuda_constant
     3, // cuda_shared
diff --git a/clang/lib/Basic/Targets/SPIR.h b/clang/lib/Basic/Targets/SPIR.h
index f625d4980e29b..6473138982c1f 100644
--- a/clang/lib/Basic/Targets/SPIR.h
+++ b/clang/lib/Basic/Targets/SPIR.h
@@ -28,6 +28,8 @@ static const unsigned SPIRAddrSpaceMap[] = {
     2, // opencl_constant
     0, // opencl_private
     4, // opencl_generic
+    5, // opencl_global_device
+    6, // opencl_global_host
     0, // cuda_device
     0, // cuda_constant
     0, // cuda_shared
diff --git a/clang/lib/Basic/Targets/TCE.h b/clang/lib/Basic/Targets/TCE.h
index 9cbf2a3688a2e..445fe4fe72936 100644
--- a/clang/lib/Basic/Targets/TCE.h
+++ b/clang/lib/Basic/Targets/TCE.h
@@ -35,6 +35,8 @@ static const unsigned TCEOpenCLAddrSpaceMap[] = {
     4, // opencl_local
     5, // opencl_constant
     0, // opencl_private
+    1, // opencl_global_device
+    1, // opencl_global_host
     // FIXME: generic has to be added to the target
     0, // opencl_generic
     0, // cuda_device
diff --git a/clang/lib/Basic/Targets/X86.h b/clang/lib/Basic/Targets/X86.h
index 50e77937febd5..d99358d6f8066 100644
--- a/clang/lib/Basic/Targets/X86.h
+++ b/clang/lib/Basic/Targets/X86.h
@@ -30,6 +30,8 @@ static const unsigned X86AddrSpaceMap[] = {
     0,   // opencl_constant
     0,   // opencl_private
     0,   // opencl_generic
+    0,   // opencl_global_device
+    0,   // opencl_global_host
     0,   // cuda_device
     0,   // cuda_constant
     0,   // cuda_shared
diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp
index 4c792520b5f37..48a1dddfb3315 100644
--- a/clang/lib/CodeGen/CodeGenModule.cpp
+++ b/clang/lib/CodeGen/CodeGenModule.cpp
@@ -1324,10 +1324,18 @@ static void removeImageAccessQualifier(std::string& TyName) {
 // (basically all single AS CPUs).
 static unsigned ArgInfoAddressSpace(LangAS AS) {
   switch (AS) {
-  case LangAS::opencl_global:   return 1;
-  case LangAS::opencl_constant: return 2;
-  case LangAS::opencl_local:    return 3;
-  case LangAS::opencl_generic:  return 4; // Not in SPIR 2.0 specs.
+  case LangAS::opencl_global:
+    return 1;
+  case LangAS::opencl_constant:
+    return 2;
+  case LangAS::opencl_local:
+    return 3;
+  case LangAS::opencl_generic:
+    return 4; // Not in SPIR 2.0 specs.
+  case LangAS::opencl_global_device:
+    return 5;
+  case LangAS::opencl_global_host:
+    return 6;
   default:
     return 0; // Assume private.
   }
@@ -3792,6 +3800,8 @@ LangAS CodeGenModule::GetGlobalVarAddressSpace(const VarDecl *D) {
   if (LangOpts.OpenCL) {
     AddrSpace = D ? D->getType().getAddressSpace() : LangAS::opencl_global;
     assert(AddrSpace == LangAS::opencl_global ||
+           AddrSpace == LangAS::opencl_global_device ||
+           AddrSpace == LangAS::opencl_global_host ||
            AddrSpace == LangAS::opencl_constant ||
            AddrSpace == LangAS::opencl_local ||
            AddrSpace >= LangAS::FirstTargetAddressSpace);
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index 78c65ad01e475..abf1d6450036f 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -7968,6 +7968,8 @@ static bool isAddressSpaceKind(const ParsedAttr &attr) {
   return attrKind == ParsedAttr::AT_AddressSpace ||
          attrKind == ParsedAttr::AT_OpenCLPrivateAddressSpace ||
          attrKind == ParsedAttr::AT_OpenCLGlobalAddressSpace ||
+         attrKind == ParsedAttr::AT_OpenCLGlobalDeviceAddressSpace ||
+         attrKind == ParsedAttr::AT_OpenCLGlobalHostAddressSpace ||
          attrKind == ParsedAttr::AT_OpenCLLocalAddressSpace ||
          attrKind == ParsedAttr::AT_OpenCLConstantAddressSpace ||
          attrKind == ParsedAttr::AT_OpenCLGenericAddressSpace;
@@ -8048,6 +8050,8 @@ static void processTypeAttrs(TypeProcessingState &state, QualType &type,
       break;
     case ParsedAttr::AT_OpenCLPrivateAddressSpace:
     case ParsedAttr::AT_OpenCLGlobalAddressSpace:
+    case ParsedAttr::AT_OpenCLGlobalDeviceAddressSpace:
+    case ParsedAttr::AT_OpenCLGlobalHostAddressSpace:
     case ParsedAttr::AT_OpenCLLocalAddressSpace:
     case ParsedAttr::AT_OpenCLConstantAddressSpace:
     case ParsedAttr::AT_OpenCLGenericAddressSpace:
diff --git a/clang/test/AST/language_address_space_attribute.cpp b/clang/test/AST/language_address_space_attribute.cpp
index fcf7503302f98..93a23304d9a63 100644
--- a/clang/test/AST/language_address_space_attribute.cpp
+++ b/clang/test/AST/language_address_space_attribute.cpp
@@ -17,6 +17,18 @@ void langas() {
   // CHECK: VarDecl {{.*}} z_global '__global int *'
   [[clang::opencl_global]] int *z_global;
 
+  // CHECK: VarDecl {{.*}} x_global_device '__global_device int *'
+  __attribute__((opencl_global_device)) int *x_global_device;
+
+  // CHECK: VarDecl {{.*}} z_global_device '__global_device int *'
+  [[clang::opencl_global_device]] int *z_global_device;
+
+  // CHECK: VarDecl {{.*}} x_global_host '__global_host int *'
+  __attribute__((opencl_global_host)) int *x_global_host;
+
+  // CHECK: VarDecl {{.*}} z_global_host '__global_host int *'
+  [[clang::opencl_global_host]] int *z_global_host;
+
   // CHECK: VarDecl {{.*}} x_local '__local int *'
   __attribute__((opencl_local)) int *x_local;
 
diff --git a/clang/test/CodeGenCXX/mangle-address-space.cpp b/clang/test/CodeGenCXX/mangle-address-space.cpp
index 2c5cdfc3dadbe..600e64ef7e673 100644
--- a/clang/test/CodeGenCXX/mangle-address-space.cpp
+++ b/clang/test/CodeGenCXX/mangle-address-space.cpp
@@ -43,6 +43,10 @@ void ocl_f0(char __private *p) { }
 
 struct ocl_OpaqueType;
 typedef ocl_OpaqueType __global * ocl_OpaqueTypePtr;
+typedef ocl_OpaqueType __attribute__((opencl_global_host)) * ocl_OpaqueTypePtrH;
+typedef ocl_OpaqueType
+    __attribute__((opencl_global_device)) *
+    ocl_OpaqueTypePtrD;
 
 // CHECKOCL-LABEL: define {{.*}}void @_Z6ocl_f0PU8CLglobal14ocl_OpaqueType
 // WINOCL-LABEL: define {{.*}}void @"?ocl_f0@@YAXPEAU?$_ASCLglobal@$$CAUocl_OpaqueType@@@__clang@@@Z"
@@ -61,4 +65,12 @@ __constant float *ocl_f1(char __generic const *p) { return 0;}
 // CHECKOCL-LABEL: define {{.*}}float* @_Z6ocl_f2PU9CLgenericKc
 // WINOCL-LABEL: define {{.*}}float* @"?ocl_f2@@YAPEAU?$_ASCLgeneric@$$CAM@__clang@@QEAU?$_ASCLgeneric@$$CBD@2@@Z"
 __generic float *ocl_f2(__generic char const * const p) { return 0;}
+
+// CHECKOCL-LABEL: define {{.*}}void @_Z6ocl_f3PU6CLhost14ocl_OpaqueType
+// WINOCL-LABEL: define {{.*}}void @"?ocl_f3@@YAXPEAU?$_ASCLhost@$$CAUocl_OpaqueType@@@__clang@@@Z"
+void ocl_f3(ocl_OpaqueTypePtrH) {}
+
+// CHECKOCL-LABEL: define {{.*}}void @_Z6ocl_f4PU8CLdevice14ocl_OpaqueType
+// WINOCL-LABEL: define {{.*}}void @"?ocl_f4@@YAXPEAU?$_ASCLdevice@$$CAUocl_OpaqueType@@@__clang@@@Z"
+void ocl_f4(ocl_OpaqueTypePtrD) {}
 #endif
diff --git a/clang/test/CodeGenOpenCL/address-spaces-conversions.cl b/clang/test/CodeGenOpenCL/address-spaces-conversions.cl
index 52feccc540bb8..39cc86fef7509 100644
--- a/clang/test/CodeGenOpenCL/address-spaces-conversions.cl
+++ b/clang/test/CodeGenOpenCL/address-spaces-conversions.cl
@@ -6,7 +6,9 @@
 // pointers to different address spaces
 
 // CHECK: define void @test
-void test(global int *arg_glob, generic int *arg_gen) {
+void test(global int *arg_glob, generic int *arg_gen,
+          __attribute__((opencl_global_device)) int *arg_device,
+          __attribute__((opencl_global_host)) int *arg_host) {
   int var_priv;
   arg_gen = arg_glob; // implicit cast global -> generic
   // CHECK: %{{[0-9]+}} = addrspacecast i32 addrspace(1)* %{{[0-9]+}} to i32 addrspace(4)*
@@ -39,6 +41,30 @@ void test(global int *arg_glob, generic int *arg_gen) {
   // CHECK-NOT: bitcast
   // CHECK-NOFAKE: bitcast
   // CHECK-NOFAKE-NOT: addrspacecast
+
+  arg_glob = arg_device; // implicit cast
+  // CHECK: addrspacecast
+  // CHECK-NOFAKE-NOT: addrspacecast
+
+  arg_glob = arg_host; // implicit cast
+  // CHECK: addrspacecast
+  // CHECK-NOFAKE-NOT: addrspacecast
+
+  arg_glob = (global int *)arg_device; // explicit cast
+  // CHECK: addrspacecast
+  // CHECK-NOFAKE-NOT: addrspacecast
+
+  arg_glob = (global int *)arg_host; // explicit cast
+  // CHECK: addrspacecast
+  // CHECK-NOFAKE-NOT: addrspacecast
+
+  arg_device = (__attribute((opencl_global_device)) int *)arg_glob; // explicit cast
+  // CHECK: addrspacecast
+  // CHECK-NOFAKE-NOT: addrspacecast
+
+  arg_host = (__attribute((opencl_global_host)) int *)arg_glob; // explicit cast
+  // CHECK: addrspacecast
+  // CHECK-NOFAKE-NOT: addrspacecast
 }
 
 // Test ternary operator.
diff --git a/clang/test/CodeGenOpenCL/address-spaces.cl b/clang/test/CodeGenOpenCL/address-spaces.cl
index 144466a690b3f..ebd227a6a9c7d 100644
--- a/clang/test/CodeGenOpenCL/address-spaces.cl
+++ b/clang/test/CodeGenOpenCL/address-spaces.cl
@@ -51,6 +51,14 @@ void fl(local int *arg) {}
 // AMDGCN: i32 addrspace(4)* %arg
 void fc(constant int *arg) {}
 
+// SPIR: i32 addrspace(5)* %arg
+// AMDGCN: i32 addrspace(1)* %arg
+void fd(__attribute__((opencl_global_device)) int *arg) {}
+
+// SPIR: i32 addrspace(6)* %arg
+// AMDGCN: i32 addrspace(1)* %arg
+void fh(__attribute__((opencl_global_host)) int *arg) {}
+
 #ifdef CL20
 int i;
 // CL20-DAG: @i = {{(dso_local )?}}addrspace(1) global i32 0
diff --git a/clang/test/SemaOpenCL/usm-address-spaces-conversions.cl b/clang/test/SemaOpenCL/usm-address-spaces-conversions.cl
new file mode 100644
index 0000000000000..35c333630da8e
--- /dev/null
+++ b/clang/test/SemaOpenCL/usm-address-spaces-conversions.cl
@@ -0,0 +1,67 @@
+// RUN: %clang_cc1 %s -ffake-address-space-map -verify -pedantic -fsyntax-only -cl-std=CL2.0
+// RUN: %clang_cc1 %s -ffake-address-space-map -verify -pedantic -fsyntax-only -cl-std=CL2.0 -DGENERIC
+// RUN: %clang_cc1 %s -ffake-address-space-map -verify -pedantic -fsyntax-only -cl-std=CL2.0 -DCONSTANT
+// RUN: %clang_cc1 %s -ffake-address-space-map -verify -pedantic -fsyntax-only -cl-std=CL2.0 -DLOCAL
+
+/* USM (unified shared memory) extension for OpenCLC 2.0 adds two new address
+ * spaces: global_device and global_host that are a subset of __global address
+ * space. As ISO/IEC TR 18037 5.1.3 declares - it's possible to implicitly
+ * convert a subset address space to a superset address space, while conversion
+ * in a reversed direction could be achived only with an explicit cast */
+
+#ifdef GENERIC
+#define AS_COMP __generic
+#else
+#define AS_COMP __global
+#endif // GENERIC
+
+#ifdef CONSTANT
+#define AS_INCOMP __constant
+#elif LOCAL
+#define AS_INCOMP __local
+#else // PRIVATE
+#define AS_INCOMP __private
+#endif // CONSTANT
+
+void test(AS_COMP int *arg_comp,
+          __attribute__((opencl_global_device)) int *arg_device,
+          __attribute__((opencl_global_host)) int *arg_host) {
+  AS_COMP int *var_glob1 = arg_device;
+  AS_COMP int *var_glob2 = arg_host;
+  AS_COMP int *var_glob3 = (AS_COMP int *)arg_device;
+  AS_COMP int *var_glob4 = (AS_COMP int *)arg_host;
+  arg_device = (__attribute__((opencl_global_device)) int *)arg_comp;
+  arg_host = (__attribute__((opencl_global_host)) int *)arg_comp;
+#ifdef GENERIC
+  // expected-error@+6{{assigning '__generic int *__private' to '__global_device int *__private' changes address space of pointer}}
+  // expected-error@+6{{assigning '__generic int *__private' to '__global_host int *__private' changes address space of pointer}}
+#else
+  // expected-error@+3{{assigning '__global int *__private' to '__global_device int *__private' changes address space of pointer}}
+  // expected-error@+3{{assigning '__global int *__private' to '__global_host int *__private' changes address space of pointer}}
+#endif // GENERIC
+  arg_device = arg_comp;
+  arg_host = arg_comp;
+
+#ifdef CONSTANT
+  // expected-error@+15{{initializing '__constant int *__private' with an expression of type '__global_device int *__private' changes address space of pointer}}
+  // expected-error@+15{{initializing '__constant int *__private' with an expression of type '__global_host int *__private' changes address space of pointer}}
+  // expected-error@+15{{initializing '__constant int *__private' with an expression of type '__global_device int *' changes address space of pointer}}
+  // expected-error@+16{{initializing '__constant int *__private' with an expression of type '__global_host int *' changes address space of pointer}}
+#elif LOCAL
+  // expected-error@+10{{initializing '__local int *__private' with an expression of type '__global_device int *__private' changes address space of pointer}}
+  // expected-error@+10{{initializing '__local int *__private' with an expression of type '__global_host int *__private' changes address space of pointer}}
+  // expected-error@+10{{initializing '__local int *__private' with an expression of type '__global_device int *' changes address space of pointer}}
+  // expected-error@+11{{initializing '__local int *__private' with an expression of type '__global_host int *' changes address space of pointer}}
+#else // PRIVATE
+  // expected-error@+5{{initializing '__private int *__private' with an expression of type '__global_device int *__private' changes address space of pointer}}
+  // expected-error@+5{{initializing '__private int *__private' with an expression of type '__global_host int *__private' changes address space of pointer}}
+  // expected-error@+5{{initializing '__private int *__private' with an expression of type '__global_device int *' changes address space of pointer}}
+  // expected-error@+6{{initializing '__private int *__private' with an expression of type '__global_host int *' changes address space of pointer}}
+#endif // CONSTANT
+  AS_INCOMP int *var_incomp1 = arg_device;
+  AS_INCOMP int *var_incomp2 = arg_host;
+  AS_INCOMP int *var_incomp3 =
+      (__attribute__((opencl_global_device)) int *)arg_device;
+  AS_INCOMP int *var_incomp4 =
+      (__attribute__((opencl_global_host)) int *)arg_host;
+}
diff --git a/clang/test/SemaTemplate/address_space-dependent.cpp b/clang/test/SemaTemplate/address_space-dependent.cpp
index bab6ca5d7b3d9..09ae0a620d9f3 100644
--- a/clang/test/SemaTemplate/address_space-dependent.cpp
+++ b/clang/test/SemaTemplate/address_space-dependent.cpp
@@ -43,7 +43,7 @@ void neg() {
 
 template <long int I>
 void tooBig() {
-  __attribute__((address_space(I))) int *bounds; // expected-error {{address space is larger than the maximum supported (8388595)}}
+  __attribute__((address_space(I))) int *bounds; // expected-error {{address space is larger than the maximum supported (8388593)}}
 }
 
 template <long int I>
@@ -101,7 +101,7 @@ int main() {
   car<1, 2, 3>(); // expected-note {{in instantiation of function template specialization 'car<1, 2, 3>' requested here}}
   HasASTemplateFields<1> HASTF;
   neg<-1>(); // expected-note {{in instantiation of function template specialization 'neg<-1>' requested here}}
-  correct<0x7FFFF3>();
+  correct<0x7FFFF1>();
   tooBig<8388650>(); // expected-note {{in instantiation of function template specialization 'tooBig<8388650>' requested here}}
 
   __attribute__((address_space(1))) char *x;

From b99630e432614d06b380afb15c466665065eaa0a Mon Sep 17 00:00:00 2001
From: Nathan James <n.james93@hotmail.co.uk>
Date: Wed, 29 Jul 2020 15:35:28 +0100
Subject: [PATCH 0465/1035] [clang-tidy] Fix RedundantStringCStrCheck with r
 values

The previous fix for this, https://reviews.llvm.org/D76761, Passed test cases but failed in the real world as std::string has a non trivial destructor so creates a CXXBindTemporaryExpr.

This handles that shortfall and updates the test case std::basic_string implementation to use a non trivial destructor to reflect real world behaviour.

Reviewed By: gribozavr2

Differential Revision: https://reviews.llvm.org/D84831
---
 .../readability/RedundantStringCStrCheck.cpp       | 14 ++++++++------
 .../checkers/readability-redundant-string-cstr.cpp |  2 ++
 2 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/clang-tools-extra/clang-tidy/readability/RedundantStringCStrCheck.cpp b/clang-tools-extra/clang-tidy/readability/RedundantStringCStrCheck.cpp
index bea02a6ba111b..1f371eed2db80 100644
--- a/clang-tools-extra/clang-tidy/readability/RedundantStringCStrCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/RedundantStringCStrCheck.cpp
@@ -92,16 +92,18 @@ void RedundantStringCStrCheck::registerMatchers(
                         callee(memberExpr().bind("member")),
                         callee(cxxMethodDecl(hasAnyName("c_str", "data"))))
           .bind("call");
-
+  const auto HasRValueTempParent =
+      hasParent(materializeTemporaryExpr(unless(isBoundToLValue())));
   // Detect redundant 'c_str()' calls through a string constructor.
   // If CxxConstructExpr is the part of some CallExpr we need to
   // check that matched ParamDecl of the ancestor CallExpr is not rvalue.
   Finder->addMatcher(
-      traverse(ast_type_traits::TK_AsIs,
-               cxxConstructExpr(StringConstructorExpr,
-                                hasArgument(0, StringCStrCallExpr),
-                                unless(hasParent(materializeTemporaryExpr(
-                                    unless(isBoundToLValue())))))),
+      traverse(
+          ast_type_traits::TK_AsIs,
+          cxxConstructExpr(
+              StringConstructorExpr, hasArgument(0, StringCStrCallExpr),
+              unless(anyOf(HasRValueTempParent, hasParent(cxxBindTemporaryExpr(
+                                                    HasRValueTempParent)))))),
       this);
 
   // Detect: 's == str.c_str()'  ->  's == str'
diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability-redundant-string-cstr.cpp b/clang-tools-extra/test/clang-tidy/checkers/readability-redundant-string-cstr.cpp
index 2561b81805bda..e1df8cccc10b3 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/readability-redundant-string-cstr.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/readability-redundant-string-cstr.cpp
@@ -15,6 +15,8 @@ struct basic_string {
   basic_string();
   basic_string(const C *p, const A &a = A());
 
+  ~basic_string();
+
   const C *c_str() const;
   const C *data() const;
 

From 1aaf8aa53d694309087b322861038130490bdd5e Mon Sep 17 00:00:00 2001
From: Jakub Lichman <limo@google.com>
Date: Wed, 29 Jul 2020 16:24:48 +0200
Subject: [PATCH 0466/1035] [mlir][Linalg] Conv1D, Conv2D and Conv3D added as
 named ops

This commit is part of a greater project which aims to add
full end-to-end support for convolutions inside mlir. The
reason behind having conv ops for each rank rather than
having one generic ConvOp is to enable better optimizations
for every N-D case which reflects memory layout of input/kernel
buffers better and simplifies code as well. We expect plain linalg.conv
to be progressively retired.

Reviewed By: ftynse

Differential Revision: https://reviews.llvm.org/D83879
---
 .../mlir/Dialect/Linalg/IR/LinalgOps.h        |   8 +
 .../Dialect/Linalg/IR/LinalgStructuredOps.td  | 125 ++++++++++++++
 .../LinalgToStandard/LinalgToStandard.cpp     |   5 +-
 mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp      |  44 +++++
 mlir/lib/Dialect/Linalg/Transforms/Loops.cpp  |  55 +++++++
 mlir/test/Dialect/Linalg/invalid.mlir         |   8 +
 mlir/test/Dialect/Linalg/loops.mlir           | 153 ++++++++++++++++++
 7 files changed, 397 insertions(+), 1 deletion(-)

diff --git a/mlir/include/mlir/Dialect/Linalg/IR/LinalgOps.h b/mlir/include/mlir/Dialect/Linalg/IR/LinalgOps.h
index 21bff4185abf8..75e6599bf9fee 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/LinalgOps.h
+++ b/mlir/include/mlir/Dialect/Linalg/IR/LinalgOps.h
@@ -85,6 +85,14 @@ AffineMap extractOrIdentityMap(Optional<AffineMap> maybeMap, unsigned rank,
 SmallVector<AffineExpr, 4> concat(ArrayRef<AffineExpr> a,
                                   ArrayRef<AffineExpr> b);
 
+/// Generates indexing maps for convolution with the following structure:
+/// input:   (m_1, ..., m_r, n_1, ..., n_r) -> (m_1 + n_1, ..., m_r + n_r)
+/// kernel:  (m_1, ..., m_r, n_1, ..., n_r) -> (n_1, ..., n_r)
+/// output:  (m_1, ..., m_r, n_1, ..., n_r) -> (m_1, ..., m_r)
+/// where r is the rank of the input, kernel and output
+llvm::Optional<SmallVector<AffineMap, 8>>
+createConvNDIndexingMaps(MLIRContext *context, unsigned rank);
+
 #include "mlir/Dialect/Linalg/IR/LinalgStructuredOpsInterfaces.h.inc"
 
 #define GET_OP_CLASSES
diff --git a/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOps.td b/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOps.td
index 1e3321af981e6..84ae8e440bee6 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOps.td
+++ b/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOps.td
@@ -180,6 +180,131 @@ def FillOp : LinalgStructured_Op<"fill", [NInputs<0>, NOutputs<1>]> {
   let hasFolder = 1;
 }
 
+class ConvOpBase<string mnemonic, int N>
+  : LinalgStructured_Op<mnemonic, [NInputs<2>, NOutputs<1>]> {
+  let description = [{
+    Base operation for any N-D Convolution implemented as a linalg.generic op.
+
+    Usage:
+
+    ```mlir
+    linalg.conv<N>D(%in, %filter, %out) : memref<(?x)+f32>,
+                                          memref<(?x)+f32>,
+                                          memref<(?x)+f32>
+    ```
+
+    where    %in:     input array
+             %filter: kernel or filter that will be applied on the input array
+             %out:    output array
+
+    and rank of the operands is *N*.
+
+    Every child convolution is expressed as:
+
+    ```mlir
+    #conv_trait = {
+      args_in = 2,
+      args_out = 1,
+      indexing_maps = #conv_accesses,
+      library_call  = "linalg_conv",
+      iterator_types = [("parallel", "parallel")+], // `2 * rank` iterators
+    }
+
+    linalg.generic #conv_trait %in, %filter, %out {
+      ^bb0(%a: f32, %b: f32, %c: f32) :
+        %d = mulf %a, %b : f32
+        %e = addf %c, %d : f32
+        linalg.yield %e : f32
+    } : memref<(?x)+f32>,
+        memref<(?x)+f32>,
+        memref<(?x)+f32>
+    ```
+
+    where #conv_accesses depend on the rank of the operands and thus
+    can be found in the documentation of each N-D case.
+    Please note that the input array is expected to be right-padded i.e.
+    the size of the input is greater than or equal to the size of the output
+    + size of the kernel - 1. If it is not padded the behavior of the op
+    is undefined.
+  }];
+
+  let arguments = (ins AnyStridedMemRefOfRank<N>,
+                       AnyStridedMemRefOfRank<N>,
+                       AnyStridedMemRefOfRank<N>);
+
+  let extraClassDeclaration = libraryCallName # [{
+    llvm::Optional<SmallVector<StringRef, 8>> referenceIterators() {
+      // There are always 2 loops for each dimension of the convolution. First
+      // iterates output and second kernel. Since ranks of all 3 operands must
+      // be the same it does not matter which operand is picked to get the rank.
+      // Loops iterating the output can be parallelized and thus are marked as
+      // "parallel" while loops iterating the kernel are accumulating the
+      // products and therefore are marked as "reduction".
+      unsigned rank = getInputShapedType(0).getRank();
+      SmallVector<StringRef, 8> parallel(rank, getParallelIteratorTypeName());
+      SmallVector<StringRef, 8> reduction(rank, getReductionIteratorTypeName());
+      parallel.insert(parallel.end(), reduction.begin(), reduction.end());
+      return parallel;
+    }
+
+    // Generates indexing maps with the following structure:
+    // input:   (m_1, ..., m_r, n_1, ..., n_r) -> (m_1 + n_1, ..., m_r + n_r)
+    // kernel:  (m_1, ..., m_r, n_1, ..., n_r) -> (n_1, ..., n_r)
+    // output:  (m_1, ..., m_r, n_1, ..., n_r) -> (m_1, ..., m_r)
+    // where r is the rank of the input, kernel and output
+    llvm::Optional<SmallVector<AffineMap, 8>> referenceIndexingMaps() {
+      MLIRContext *context = getContext();
+      unsigned rank = getInputShapedType(0).getRank();
+      return createConvNDIndexingMaps(context, rank);
+    }
+  }];
+
+  let hasFolder = 1;
+  let verifier = [{ return ::verify(*this); }];
+}
+
+def Conv1DOp : ConvOpBase<"conv1D", 1> {
+  let description = [{
+    *1D* convolution which uses following affine maps to access operands:
+
+    ```mlir
+    #conv_accesses = [
+      affine_map<(m, n) -> (m + n)>, // in
+      affine_map<(m, n) -> (n)>, // kernel
+      affine_map<(m, n) -> (m)> // out
+    ]
+    ```
+  }];
+}
+
+def Conv2DOp : ConvOpBase<"conv2D", 2> {
+  let description = [{
+    *2D* convolution which uses following affine maps to access operands:
+
+    ```mlir
+    #conv_accesses = [
+      affine_map<(m1, m2, n1, n2) -> (m1 + n1, m2 + n2)>, // in
+      affine_map<(m1, m2, n1, n2) -> (n1, n2)>, // kernel
+      affine_map<(m1, m2, n1, n2) -> (m1, m2) // out
+    ]
+    ```
+  }];
+}
+
+def Conv3DOp : ConvOpBase<"conv3D", 3> {
+  let description = [{
+    *3D* convolution which uses following affine maps to access operands:
+
+    ```mlir
+    #conv_accesses = [
+      affine_map<(m1, m2, m3, n1, n2, n3) -> (m1 + n1, m2 + n2, m3 + n3)>, // in
+      affine_map<(m1, m2, m3, n1, n2, n3) -> (n1, n2, n3)>, // kernel
+      affine_map<(m1, m2, m3, n1, n2, n3) -> (m1, m2, m3)> // out
+    ]
+    ```
+  }];
+}
+
 /// A base class for pooling operation such as conv. The arguments must contain
 /// optional arguments `strides`, `dilations` and `padding` with following type:
 ///   OptionalAttr<I64ArrayAttr>:$strides
diff --git a/mlir/lib/Conversion/LinalgToStandard/LinalgToStandard.cpp b/mlir/lib/Conversion/LinalgToStandard/LinalgToStandard.cpp
index 8a54c93d7685e..921445bd03b1f 100644
--- a/mlir/lib/Conversion/LinalgToStandard/LinalgToStandard.cpp
+++ b/mlir/lib/Conversion/LinalgToStandard/LinalgToStandard.cpp
@@ -235,7 +235,10 @@ void mlir::populateLinalgToStandardConversionPatterns(
       LinalgOpConversion<PoolingMaxOp>,
       LinalgOpConversion<PoolingMinOp>,
       LinalgOpConversion<PoolingSumOp>,
-      LinalgOpConversion<CopyOp>, 
+      LinalgOpConversion<CopyOp>,
+      LinalgOpConversion<Conv1DOp>,
+      LinalgOpConversion<Conv2DOp>,
+      LinalgOpConversion<Conv3DOp>,
       LinalgOpConversion<FillOp>,
       LinalgOpConversion<GenericOp>,
       LinalgOpConversion<IndexedGenericOp>>(ctx);
diff --git a/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp b/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
index 192179b3ff506..e67adf8c20420 100644
--- a/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
+++ b/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
@@ -986,6 +986,17 @@ static LogicalResult verifyStrideOrDilation(LinalgPoolingOp op,
   return success();
 }
 
+template <typename ConvNDOp>
+static LogicalResult verify(ConvNDOp op) {
+  auto outputType = op.getOutputShapedType(0).getElementType();
+  auto inputType = op.getInputShapedType(0).getElementType();
+  auto kernelType = op.getInputShapedType(1).getElementType();
+  if (outputType != inputType || inputType != kernelType)
+    return op.emitOpError("expected all element types of operands to match");
+
+  return success();
+}
+
 static LogicalResult verify(ConvOp op) {
   auto oType = op.output().getType().cast<MemRefType>();
   auto fType = op.filter().getType().cast<MemRefType>();
@@ -1096,6 +1107,27 @@ mlir::linalg::weightedPoolingInputIndex(PoolingOp op,
   return res;
 }
 
+llvm::Optional<SmallVector<AffineMap, 8>>
+mlir::linalg::createConvNDIndexingMaps(MLIRContext *context, unsigned rank) {
+  unsigned numDims = rank * 2, idx = 0;
+
+  SmallVector<AffineExpr, 8> dims, in, kernel, out;
+  dims = makeAffineDimExprs(numDims, idx, context);
+  in.reserve(rank);
+  kernel.reserve(rank);
+  out.reserve(rank);
+
+  for (unsigned i = 0; i < rank; i++) {
+    in.push_back(dims[i] + dims[rank + i]);
+    kernel.push_back(dims[rank + i]);
+    out.push_back(dims[i]);
+  }
+
+  return SmallVector<AffineMap, 8>{AffineMap::get(numDims, 0, in, context),
+                                   AffineMap::get(numDims, 0, kernel, context),
+                                   AffineMap::get(numDims, 0, out, context)};
+}
+
 #define INSTANTIATE_WEIGHTED_POOLING_INPUT_INDEX(OP_TYPE)                      \
   template SmallVector<AffineExpr, 4>                                          \
   mlir::linalg::weightedPoolingInputIndex<OP_TYPE>(                            \
@@ -1177,6 +1209,18 @@ LogicalResult FillOp::fold(ArrayRef<Attribute>,
                            SmallVectorImpl<OpFoldResult> &) {
   return foldMemRefCast(*this);
 }
+LogicalResult Conv1DOp::fold(ArrayRef<Attribute>,
+                             SmallVectorImpl<OpFoldResult> &) {
+  return foldMemRefCast(*this);
+}
+LogicalResult Conv2DOp::fold(ArrayRef<Attribute>,
+                             SmallVectorImpl<OpFoldResult> &) {
+  return foldMemRefCast(*this);
+}
+LogicalResult Conv3DOp::fold(ArrayRef<Attribute>,
+                             SmallVectorImpl<OpFoldResult> &) {
+  return foldMemRefCast(*this);
+}
 LogicalResult GenericOp::fold(ArrayRef<Attribute>,
                               SmallVectorImpl<OpFoldResult> &) {
   return foldMemRefCast(*this);
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp b/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp
index 32e50cb597d7a..db29835e2caa7 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp
@@ -295,6 +295,61 @@ void emitScalarImplementation(ArrayRef<Value> allIvs, FillOp fillOp) {
   nPar > 0 ? O(ivs) = fillOp.value() : O() = fillOp.value();
 }
 
+/// Following functions emit scalar part of the N-D convolution op.
+/// N-D convolution has 2N loops:
+///   1-N: Iterate over the output array *O* with iterators *m1, ..., mN*.
+///   N-2N:. Iterate over the kernel *K* with iterators *n1, ..., nN*.
+///
+/// The scalar part accumulates products of input array *I* values with kernel
+/// ones. The accumulation expression therefore looks like:
+///   O[m1, ..., mN] += I[m1 + n1, ..., mN + nN] * K[n1, ..., nN].
+/// Note that the input array has to be padded in order to prevent
+/// out of bounds accesses.
+template <typename IndexedValueType>
+void emitScalarImplementation(ArrayRef<Value> allIvs, Conv1DOp convOp) {
+  assert(convOp.hasBufferSemantics() &&
+         "expected linalg op with buffer semantics");
+  assert(allIvs.size() == 2);
+  Value m1(allIvs[0]);
+  Value n1(allIvs[1]);
+  IndexedValueType I(convOp.getInput(0)), K(convOp.getInput(1)),
+      O(convOp.getOutputBuffer(0));
+  // Emit scalar form for the 1D conv case.
+  Value i1 = m1 + n1;
+  O(m1) = O(m1) + I(i1) * K(n1);
+}
+
+template <typename IndexedValueType>
+void emitScalarImplementation(ArrayRef<Value> allIvs, Conv2DOp convOp) {
+  assert(convOp.hasBufferSemantics() &&
+         "expected linalg op with buffer semantics");
+  assert(allIvs.size() == 4);
+  Value m1(allIvs[0]), m2(allIvs[1]);
+  Value n1(allIvs[2]), n2(allIvs[3]);
+  IndexedValueType I(convOp.getInput(0)), K(convOp.getInput(1)),
+      O(convOp.getOutputBuffer(0));
+  // Emit scalar form for the 2D conv case.
+  Value i1 = m1 + n1;
+  Value i2 = m2 + n2;
+  O(m1, m2) = O(m1, m2) + I(i1, i2) * K(n1, n2);
+}
+
+template <typename IndexedValueType>
+void emitScalarImplementation(ArrayRef<Value> allIvs, Conv3DOp convOp) {
+  assert(convOp.hasBufferSemantics() &&
+         "expected linalg op with buffer semantics");
+  assert(allIvs.size() == 6);
+  Value m1(allIvs[0]), m2(allIvs[1]), m3(allIvs[2]);
+  Value n1(allIvs[3]), n2(allIvs[4]), n3(allIvs[5]);
+  IndexedValueType I(convOp.getInput(0)), K(convOp.getInput(1)),
+      O(convOp.getOutputBuffer(0));
+  // Emit scalar form for the 3D conv case.
+  Value i1 = m1 + n1;
+  Value i2 = m2 + n2;
+  Value i3 = m3 + n3;
+  O(m1, m2, m3) = O(m1, m2, m3) + I(i1, i2, i3) * K(n1, n2, n3);
+}
+
 template <typename IndexedValueType>
 Value getConvOpInput(ConvOp convOp, StdIndexedValue im,
                      MutableArrayRef<Value> imIdx) {
diff --git a/mlir/test/Dialect/Linalg/invalid.mlir b/mlir/test/Dialect/Linalg/invalid.mlir
index ca59ecd387ec3..a5a6e9bee34f8 100644
--- a/mlir/test/Dialect/Linalg/invalid.mlir
+++ b/mlir/test/Dialect/Linalg/invalid.mlir
@@ -507,3 +507,11 @@ func @named_ops(%a3: memref<?x?x?xf32>, %b3: memref<?x?xf32>, %c3: memref<?x?x?x
   linalg.batch_matmul %a3, %b3, %c3 : (memref<?x?x?xf32>, memref<?x?xf32>, memref<?x?x?xf32>) -> ()
   return
 }
+
+// -----
+
+func @conv_type_mismatch(%in: memref<?xi32>, %filter: memref<?xf32>, %out: memref<?xf32>) {
+  // expected-error @+1 {{expected all element types of operands to match}}
+  linalg.conv1D(%in, %filter, %out) : memref<?xi32>, memref<?xf32>, memref<?xf32>
+  return
+}
diff --git a/mlir/test/Dialect/Linalg/loops.mlir b/mlir/test/Dialect/Linalg/loops.mlir
index b01beb7e8f175..ee63d59ca8c46 100644
--- a/mlir/test/Dialect/Linalg/loops.mlir
+++ b/mlir/test/Dialect/Linalg/loops.mlir
@@ -1286,3 +1286,156 @@ func @conv4d(%in : memref<?x?x?x?xf32>, %filter : memref<?x?x?x?xf32>, %out :  m
 //       CHECKPARALLEL:   %[[inc:.*]] = mulf %[[va]], %[[vb]] : f32
 //       CHECKPARALLEL:   %[[res:.*]] = addf %[[vc]], %[[inc]] : f32
 //       CHECKPARALLEL:   store %[[res]], %[[arg2]][%[[i0]], %[[i1]], %[[i2]], %[[i3]]] : memref<?x?x?x?xf32>
+
+func @conv1d_no_symbols(%in : memref<?xf32>, %filter : memref<?xf32>, %out : memref<?xf32>) -> () {
+  linalg.conv1D(%in, %filter, %out) : memref<?xf32>, memref<?xf32>, memref<?xf32>
+  return
+}
+
+// CHECKLOOP-LABEL: @conv1d_no_symbols
+//  CHECKLOOP-SAME: %[[arg0:[a-zA-Z0-9]+]]: memref<?xf32>
+//  CHECKLOOP-SAME: %[[arg1:[a-zA-Z0-9]+]]: memref<?xf32>
+//  CHECKLOOP-SAME: %[[arg2:[a-zA-Z0-9]+]]: memref<?xf32>
+//       CHECKLOOP: %[[c0:.*]] = constant 0 : index
+//       CHECKLOOP: %[[c1:.*]] = constant 1 : index
+//       CHECKLOOP: %[[dim0:.*]] = dim %[[arg1]], %[[c0]] : memref<?xf32>
+//       CHECKLOOP: %[[dim1:.*]] = dim %[[arg2]], %[[c0]] : memref<?xf32>
+//       CHECKLOOP: scf.for %[[b:.*]] = %[[c0]] to %[[dim1]] step %[[c1]] {
+//       CHECKLOOP:   scf.for %[[m:.*]] = %[[c0]] to %[[dim0]] step %[[c1]] {
+//       CHECKLOOP:     %[[aff:.*]] = affine.apply #[[$stride1Dilation1]](%[[b]], %[[m]])
+//       CHECKLOOP:     %[[va:.*]] = load %[[arg1]][%[[m]]] : memref<?xf32>
+//       CHECKLOOP:     %[[vb:.*]] = load %[[arg0]][%[[aff]]] : memref<?xf32>
+//       CHECKLOOP:     %[[inc:.*]] = mulf %[[vb]], %[[va]] : f32
+//       CHECKLOOP:     %[[vc:.*]] = load %[[arg2]][%[[b]]] : memref<?xf32>
+//       CHECKLOOP:     %[[res:.*]] = addf %[[vc]], %[[inc]] : f32
+//       CHECKLOOP:     store %[[res]], %[[arg2]][%[[b]]] : memref<?xf32>
+
+// CHECKPARALLEL-LABEL: @conv1d_no_symbols
+//  CHECKPARALLEL-SAME: %[[arg0:[a-zA-Z0-9]+]]: memref<?xf32>
+//  CHECKPARALLEL-SAME: %[[arg1:[a-zA-Z0-9]+]]: memref<?xf32>
+//  CHECKPARALLEL-SAME: %[[arg2:[a-zA-Z0-9]+]]: memref<?xf32>
+//       CHECKPARALLEL: %[[c0:.*]] = constant 0 : index
+//       CHECKPARALLEL: %[[c1:.*]] = constant 1 : index
+//       CHECKPARALLEL: %[[dim0:.*]] = dim %[[arg1]], %[[c0]] : memref<?xf32>
+//       CHECKPARALLEL: %[[dim1:.*]] = dim %[[arg2]], %[[c0]] : memref<?xf32>
+//       CHECKPARALLEL: scf.parallel (%[[b:.*]]) = (%[[c0]]) to (%[[dim1]]) step (%[[c1]]) {
+//       CHECKPARALLEL:   scf.for %[[m:.*]] = %[[c0]] to %[[dim0]] step %[[c1]] {
+//       CHECKPARALLEL:     %[[aff:.*]] = affine.apply #[[$stride1Dilation1]](%[[b]], %[[m]])
+//       CHECKPARALLEL:     %[[va:.*]] = load %[[arg1]][%[[m]]] : memref<?xf32>
+//       CHECKPARALLEL:     %[[vb:.*]] = load %[[arg0]][%[[aff]]] : memref<?xf32>
+//       CHECKPARALLEL:     %[[inc:.*]] = mulf %[[vb]], %[[va]] : f32
+//       CHECKPARALLEL:     %[[vc:.*]] = load %[[arg2]][%[[b]]] : memref<?xf32>
+//       CHECKPARALLEL:     %[[res:.*]] = addf %[[vc]], %[[inc]] : f32
+//       CHECKPARALLEL:     store %[[res]], %[[arg2]][%[[b]]] : memref<?xf32>
+
+
+func @conv2d_no_symbols(%in : memref<?x?xf32>, %filter : memref<?x?xf32>, %out : memref<?x?xf32>) -> () {
+  linalg.conv2D(%in, %filter, %out) : memref<?x?xf32>, memref<?x?xf32>, memref<?x?xf32>
+  return
+}
+// CHECKLOOP-LABEL: @conv2d_no_symbols
+//  CHECKLOOP-SAME: %[[arg0:[a-zA-Z0-9]+]]: memref<?x?xf32>
+//  CHECKLOOP-SAME: %[[arg1:[a-zA-Z0-9]+]]: memref<?x?xf32>
+//  CHECKLOOP-SAME: %[[arg2:[a-zA-Z0-9]+]]: memref<?x?xf32>
+//       CHECKLOOP: %[[c0:.*]] = constant 0 : index
+//       CHECKLOOP: %[[c1:.*]] = constant 1 : index
+//       CHECKLOOP: %[[dim0:.*]] = dim %[[arg1]], %[[c0]] : memref<?x?xf32>
+//       CHECKLOOP: %[[dim1:.*]] = dim %[[arg1]], %[[c1]] : memref<?x?xf32>
+//       CHECKLOOP: %[[dim2:.*]] = dim %[[arg2]], %[[c0]] : memref<?x?xf32>
+//       CHECKLOOP: %[[dim3:.*]] = dim %[[arg2]], %[[c1]] : memref<?x?xf32>
+//       CHECKLOOP: scf.for %[[arg3:.*]] = %[[c0]] to %[[dim2]] step %[[c1]] {
+//       CHECKLOOP:   scf.for %[[arg4:.*]] = %[[c0]] to %[[dim3]] step %[[c1]] {
+//       CHECKLOOP:     scf.for %[[arg5:.*]] = %[[c0]] to %[[dim0]] step %[[c1]] {
+//       CHECKLOOP:       scf.for %[[arg6:.*]] = %[[c0]] to %[[dim1]] step %[[c1]] {
+//       CHECKLOOP:         %[[aff:.*]] = affine.apply #[[$stride1Dilation1]](%[[arg3]], %[[arg5]])
+//       CHECKLOOP:         %[[aff2:.*]] = affine.apply #[[$stride1Dilation1]](%[[arg4]], %[[arg6]])
+//       CHECKLOOP:         %[[va:.*]] = load %[[arg1]][%[[arg5]], %[[arg6]]] : memref<?x?xf32>
+//       CHECKLOOP:         %[[vb:.*]] = load %[[arg0]][%[[aff]], %[[aff2]]] : memref<?x?xf32>
+//       CHECKLOOP:         %[[inc:.*]] = mulf %[[vb]], %[[va]] : f32
+//       CHECKLOOP:         %[[vc:.*]] = load %[[arg2]][%[[arg3]], %[[arg4]]] : memref<?x?xf32>
+//       CHECKLOOP:         %[[res:.*]] = addf %[[vc]], %[[inc]] : f32
+//       CHECKLOOP:         store %[[res]], %[[arg2]][%[[arg3]], %[[arg4]]] : memref<?x?xf32>
+
+// CHECKPARALLEL-LABEL: @conv2d_no_symbols
+//  CHECKPARALLEL-SAME: %[[arg0:[a-zA-Z0-9]+]]: memref<?x?xf32>
+//  CHECKPARALLEL-SAME: %[[arg1:[a-zA-Z0-9]+]]: memref<?x?xf32>
+//  CHECKPARALLEL-SAME: %[[arg2:[a-zA-Z0-9]+]]: memref<?x?xf32>
+//       CHECKPARALLEL: %[[c0:.*]] = constant 0 : index
+//       CHECKPARALLEL: %[[c1:.*]] = constant 1 : index
+//       CHECKPARALLEL: %[[dim0:.*]] = dim %[[arg1]], %[[c0]] : memref<?x?xf32>
+//       CHECKPARALLEL: %[[dim1:.*]] = dim %[[arg1]], %[[c1]] : memref<?x?xf32>
+//       CHECKPARALLEL: %[[dim2:.*]] = dim %[[arg2]], %[[c0]] : memref<?x?xf32>
+//       CHECKPARALLEL: %[[dim3:.*]] = dim %[[arg2]], %[[c1]] : memref<?x?xf32>
+//       CHECKPARALLEL: scf.parallel (%[[arg3:.*]], %[[arg4:.*]]) = (%[[c0]], %[[c0]]) to (%[[dim2]], %[[dim3]]) step (%[[c1]], %[[c1]]) {
+//       CHECKPARALLEL:   scf.for %[[arg5:.*]] = %[[c0]] to %[[dim0]] step %[[c1]] {
+//       CHECKPARALLEL:     scf.for %[[arg6:.*]] = %[[c0]] to %[[dim1]] step %[[c1]] {
+//       CHECKPARALLEL:       %[[aff:.*]] = affine.apply #[[$stride1Dilation1]](%[[arg3]], %[[arg5]])
+//       CHECKPARALLEL:       %[[aff2:.*]] = affine.apply #[[$stride1Dilation1]](%[[arg4]], %[[arg6]])
+//       CHECKPARALLEL:       %[[va:.*]] = load %[[arg1]][%[[arg5]], %[[arg6]]] : memref<?x?xf32>
+//       CHECKPARALLEL:       %[[vb:.*]] = load %[[arg0]][%[[aff]], %[[aff2]]] : memref<?x?xf32>
+//       CHECKPARALLEL:       %[[inc:.*]] = mulf %[[vb]], %[[va]] : f32
+//       CHECKPARALLEL:       %[[vc:.*]] = load %[[arg2]][%[[arg3]], %[[arg4]]] : memref<?x?xf32>
+//       CHECKPARALLEL:       %[[res:.*]] = addf %[[vc]], %[[inc]] : f32
+//       CHECKPARALLEL:       store %[[res]], %[[arg2]][%[[arg3]], %[[arg4]]] : memref<?x?xf32>
+
+
+func @conv3d_no_symbols(%in : memref<?x?x?xf32>, %filter : memref<?x?x?xf32>, %out : memref<?x?x?xf32>) -> () {
+  linalg.conv3D(%in, %filter, %out) : memref<?x?x?xf32>, memref<?x?x?xf32>, memref<?x?x?xf32>
+  return
+}
+
+// CHECKLOOP-LABEL: @conv3d_no_symbols
+//  CHECKLOOP-SAME: %[[arg0:[a-zA-Z0-9]+]]: memref<?x?x?xf32>
+//  CHECKLOOP-SAME: %[[arg1:[a-zA-Z0-9]+]]: memref<?x?x?xf32>
+//  CHECKLOOP-SAME: %[[arg2:[a-zA-Z0-9]+]]: memref<?x?x?xf32>
+//       CHECKLOOP: %[[c2:.*]] = constant 2 : index
+//       CHECKLOOP: %[[c0:.*]] = constant 0 : index
+//       CHECKLOOP: %[[c1:.*]] = constant 1 : index
+//       CHECKLOOP: %[[dim0:.*]] = dim %[[arg1]], %[[c0]] : memref<?x?x?xf32>
+//       CHECKLOOP: %[[dim1:.*]] = dim %[[arg1]], %[[c1]] : memref<?x?x?xf32>
+//       CHECKLOOP: %[[dim2:.*]] = dim %[[arg1]], %[[c2]] : memref<?x?x?xf32>
+//       CHECKLOOP: %[[dim3:.*]] = dim %[[arg2]], %[[c0]] : memref<?x?x?xf32>
+//       CHECKLOOP: %[[dim4:.*]] = dim %[[arg2]], %[[c1]] : memref<?x?x?xf32>
+//       CHECKLOOP: %[[dim5:.*]] = dim %[[arg2]], %[[c2]] : memref<?x?x?xf32>
+//       CHECKLOOP: scf.for %[[arg3:.*]] = %[[c0]] to %[[dim3]] step %[[c1]] {
+//       CHECKLOOP:   scf.for %[[arg4:.*]] = %[[c0]] to %[[dim4]] step %[[c1]] {
+//       CHECKLOOP:     scf.for %[[arg5:.*]] = %[[c0]] to %[[dim5]] step %[[c1]] {
+//       CHECKLOOP:       scf.for %[[arg6:.*]] = %[[c0]] to %[[dim0]] step %[[c1]] {
+//       CHECKLOOP:         scf.for %[[arg7:.*]] = %[[c0]] to %[[dim1]] step %[[c1]] {
+//       CHECKLOOP:           scf.for %[[arg8:.*]] = %[[c0]] to %[[dim2]] step %[[c1]] {
+//       CHECKLOOP:             %[[aff:.*]] = affine.apply #[[$stride1Dilation1]](%[[arg3]], %[[arg6]])
+//       CHECKLOOP:             %[[aff2:.*]] = affine.apply #[[$stride1Dilation1]](%[[arg4]], %[[arg7]])
+//       CHECKLOOP:             %[[aff3:.*]] = affine.apply #[[$stride1Dilation1]](%[[arg5]], %[[arg8]])
+//       CHECKLOOP:             %[[va:.*]] = load %[[arg1]][%[[arg6]], %[[arg7]], %[[arg8]]] : memref<?x?x?xf32>
+//       CHECKLOOP:             %[[vb:.*]] = load %[[arg0]][%[[aff]], %[[aff2]], %[[aff3]]] : memref<?x?x?xf32>
+//       CHECKLOOP:             %[[inc:.*]] = mulf %[[vb]], %[[va]] : f32
+//       CHECKLOOP:             %[[vc:.*]] = load %[[arg2]][%[[arg3]], %[[arg4]], %[[arg5]]] : memref<?x?x?xf32>
+//       CHECKLOOP:             %[[res:.*]] = addf %[[vc]], %[[inc]] : f32
+//       CHECKLOOP:             store %[[res]], %[[arg2]][%[[arg3]], %[[arg4]], %[[arg5]]] : memref<?x?x?xf32>
+
+// CHECKPARALLEL-LABEL: @conv3d_no_symbols
+//  CHECKPARALLEL-SAME: %[[arg0:[a-zA-Z0-9]+]]: memref<?x?x?xf32>
+//  CHECKPARALLEL-SAME: %[[arg1:[a-zA-Z0-9]+]]: memref<?x?x?xf32>
+//  CHECKPARALLEL-SAME: %[[arg2:[a-zA-Z0-9]+]]: memref<?x?x?xf32>
+//       CHECKPARALLEL: %[[c2:.*]] = constant 2 : index
+//       CHECKPARALLEL: %[[c0:.*]] = constant 0 : index
+//       CHECKPARALLEL: %[[c1:.*]] = constant 1 : index
+//       CHECKPARALLEL: %[[dim0:.*]] = dim %[[arg1]], %[[c0]] : memref<?x?x?xf32>
+//       CHECKPARALLEL: %[[dim1:.*]] = dim %[[arg1]], %[[c1]] : memref<?x?x?xf32>
+//       CHECKPARALLEL: %[[dim2:.*]] = dim %[[arg1]], %[[c2]] : memref<?x?x?xf32>
+//       CHECKPARALLEL: %[[dim3:.*]] = dim %[[arg2]], %[[c0]] : memref<?x?x?xf32>
+//       CHECKPARALLEL: %[[dim4:.*]] = dim %[[arg2]], %[[c1]] : memref<?x?x?xf32>
+//       CHECKPARALLEL: %[[dim5:.*]] = dim %[[arg2]], %[[c2]] : memref<?x?x?xf32>
+//       CHECKPARALLEL: scf.parallel (%[[arg3:.*]], %[[arg4:.*]], %[[arg5:.*]]) = (%[[c0]], %[[c0]], %[[c0]]) to (%[[dim3]], %[[dim4]], %[[dim5]]) step (%[[c1]], %[[c1]], %[[c1]]) {
+//       CHECKPARALLEL:   scf.for %[[arg6:.*]] = %[[c0]] to %[[dim0]] step %[[c1]] {
+//       CHECKPARALLEL:     scf.for %[[arg7:.*]] = %[[c0]] to %[[dim1]] step %[[c1]] {
+//       CHECKPARALLEL:       scf.for %[[arg8:.*]] = %[[c0]] to %[[dim2]] step %[[c1]] {
+//       CHECKPARALLEL:         %[[aff:.*]] = affine.apply #[[$stride1Dilation1]](%[[arg3]], %[[arg6]])
+//       CHECKPARALLEL:         %[[aff2:.*]] = affine.apply #[[$stride1Dilation1]](%[[arg4]], %[[arg7]])
+//       CHECKPARALLEL:         %[[aff3:.*]] = affine.apply #[[$stride1Dilation1]](%[[arg5]], %[[arg8]])
+//       CHECKPARALLEL:         %[[va:.*]] = load %[[arg1]][%[[arg6]], %[[arg7]], %[[arg8]]] : memref<?x?x?xf32>
+//       CHECKPARALLEL:         %[[vb:.*]] = load %[[arg0]][%[[aff]], %[[aff2]], %[[aff3]]] : memref<?x?x?xf32>
+//       CHECKPARALLEL:         %[[inc:.*]] = mulf %[[vb]], %[[va]] : f32
+//       CHECKPARALLEL:         %[[vc:.*]] = load %[[arg2]][%[[arg3]], %[[arg4]], %[[arg5]]] : memref<?x?x?xf32>
+//       CHECKPARALLEL:         %[[res:.*]] = addf %[[vc]], %[[inc]] : f32
+//       CHECKPARALLEL:         store %[[res]], %[[arg2]][%[[arg3]], %[[arg4]], %[[arg5]]] : memref<?x?x?xf32>

From 0c1dbf00b20a04bee6e0721a2b39b1574f77b849 Mon Sep 17 00:00:00 2001
From: Soumi Manna <soumi.manna@intel.com>
Date: Wed, 29 Jul 2020 07:44:26 -0700
Subject: [PATCH 0467/1035] [WIP]{SYCL] Add support for union types kernel
 arguments

Signed-off-by: Soumi Manna <soumi.manna@intel.com>
---
 clang/lib/Sema/SemaSYCL.cpp                | 22 ++++++++++++++--
 clang/test/SemaSYCL/union-kernel-param.cpp | 30 ++++++++++++++++++++++
 2 files changed, 50 insertions(+), 2 deletions(-)
 create mode 100644 clang/test/SemaSYCL/union-kernel-param.cpp

diff --git a/clang/lib/Sema/SemaSYCL.cpp b/clang/lib/Sema/SemaSYCL.cpp
index ad5a3768846c7..e8ab10da21cce 100644
--- a/clang/lib/Sema/SemaSYCL.cpp
+++ b/clang/lib/Sema/SemaSYCL.cpp
@@ -826,6 +826,8 @@ class KernelObjVisitor {
       VisitArrayElements(ArrayField, ElementTy, handlers...);
     else if (ElementTy->isScalarType())
       KF_FOR_EACH(handleScalarType, ArrayField, ElementTy);
+    else if (ElementTy->isUnionType())
+      KF_FOR_EACH(handleUnionType, ArrayField, ElementTy);
   }
 
   template <typename... Handlers>
@@ -941,9 +943,11 @@ class KernelObjVisitor {
       else if (FieldTy->isArrayType()) {
         if (KF_FOR_EACH(handleArrayType, Field, FieldTy))
           VisitArrayElements(Field, FieldTy, handlers...);
-      } else if (FieldTy->isScalarType() || FieldTy->isVectorType())
+      } else if (FieldTy->isScalarType() || FieldTy->isVectorType()) {
         KF_FOR_EACH(handleScalarType, Field, FieldTy);
-      else
+      } else if (FieldTy->isUnionType()) {
+        KF_FOR_EACH(handleUnionType, Field, FieldTy);
+      } else
         KF_FOR_EACH(handleOtherType, Field, FieldTy);
       (void)std::initializer_list<int>{
           (handlers.leaveField(Owner, Field), 0)...};
@@ -1001,6 +1005,7 @@ class SyclKernelFieldHandler {
   virtual bool handlePointerType(FieldDecl *, QualType) { return true; }
   virtual bool handleArrayType(FieldDecl *, QualType) { return true; }
   virtual bool handleScalarType(FieldDecl *, QualType) { return true; }
+  virtual bool handleUnionType(FieldDecl *, QualType) { return true; }
   // Most handlers shouldn't be handling this, just the field checker.
   virtual bool handleOtherType(FieldDecl *, QualType) { return true; }
 
@@ -1301,6 +1306,11 @@ class SyclKernelDeclCreator : public SyclKernelFieldHandler {
     return true;
   }
 
+  bool handleUnionType(FieldDecl *FD, QualType FieldTy) final {
+    addParam(FD, FieldTy);
+    return true;
+  }
+
   bool handleSyclStreamType(const CXXBaseSpecifier &, QualType FieldTy) final {
     // FIXME SYCL stream should be usable as a base type
     // See https://github.com/intel/llvm/issues/1552
@@ -1613,6 +1623,14 @@ class SyclKernelBodyCreator : public SyclKernelFieldHandler {
     return true;
   }
 
+  bool handleUnionType(FieldDecl *FD, QualType FieldTy) final {
+    if (dyn_cast<ArraySubscriptExpr>(MemberExprBases.back()))
+      createExprForScalarElement(FD);
+    else
+      createExprForStructOrScalar(FD);
+    return true;
+  }
+
   bool enterStruct(const CXXRecordDecl *RD, const CXXBaseSpecifier &BS) final {
     CXXCastPath BasePath;
     QualType DerivedTy(RD->getTypeForDecl(), 0);
diff --git a/clang/test/SemaSYCL/union-kernel-param.cpp b/clang/test/SemaSYCL/union-kernel-param.cpp
new file mode 100644
index 0000000000000..801be04b816e9
--- /dev/null
+++ b/clang/test/SemaSYCL/union-kernel-param.cpp
@@ -0,0 +1,30 @@
+// RUN: %clang_cc1 -I %S/Inputs -fsycl -fsyntax-only -verify %s
+// expected-no-diagnostics
+
+#include <sycl.hpp>
+
+using namespace cl::sycl;
+
+namespace s = sycl;
+
+struct P {
+    union S {
+        int32_t p;
+    } q;
+};
+
+static_assert(std::is_trivially_copyable<P>::value, "");
+static_assert(std::is_trivially_copyable<decltype(P{}.q)>::value, "");
+
+int main() {
+    s::queue q;
+
+    P p = {};
+
+    q.submit([&](s::handler& h) {
+        h.single_task<class foo>([=](){
+            (void) p;
+        });
+    });
+}
+

From 5fc34fafa72e311763b43f3041bb7113c8ae5b6f Mon Sep 17 00:00:00 2001
From: Frederik Gossen <frgossen@google.com>
Date: Wed, 29 Jul 2020 14:52:27 +0000
Subject: [PATCH 0468/1035] [MLIR][Shape] Limit shape to SCF lowering patterns
 to their supported types

Differential Revision: https://reviews.llvm.org/D84444
---
 mlir/lib/Conversion/ShapeToSCF/ShapeToSCF.cpp | 11 ++++---
 .../Conversion/ShapeToSCF/shape-to-scf.mlir   | 33 ++++++++++++-------
 2 files changed, 29 insertions(+), 15 deletions(-)

diff --git a/mlir/lib/Conversion/ShapeToSCF/ShapeToSCF.cpp b/mlir/lib/Conversion/ShapeToSCF/ShapeToSCF.cpp
index 824db2685a777..0101c9e7fdc01 100644
--- a/mlir/lib/Conversion/ShapeToSCF/ShapeToSCF.cpp
+++ b/mlir/lib/Conversion/ShapeToSCF/ShapeToSCF.cpp
@@ -121,7 +121,7 @@ LogicalResult
 ReduceOpConverter::matchAndRewrite(shape::ReduceOp op, ArrayRef<Value> operands,
                                    ConversionPatternRewriter &rewriter) const {
   // For now, this lowering is only defined on `tensor<?xindex>` operands.
-  if (!op.shape().getType().isa<RankedTensorType>())
+  if (op.shape().getType().isa<ShapeType>())
     return failure();
 
   auto loc = op.getLoc();
@@ -171,12 +171,15 @@ class ShapeOfOpConverter : public OpConversionPattern<ShapeOfOp> {
 LogicalResult
 ShapeOfOpConverter::matchAndRewrite(ShapeOfOp op, ArrayRef<Value> operands,
                                     ConversionPatternRewriter &rewriter) const {
-  ShapeOfOp::Adaptor transformed(operands);
-  Value arg = transformed.arg();
-  Type argTy = arg.getType();
+  // For now, this lowering supports only error-free arguments.
+  if (op.getType().isa<ShapeType>())
+    return failure();
 
   // For ranked tensors `shape_of` lowers to `std` and the pattern can be
   // found in the corresponding pass.
+  ShapeOfOp::Adaptor transformed(operands);
+  Value arg = transformed.arg();
+  Type argTy = arg.getType();
   if (argTy.isa<RankedTensorType>())
     return failure();
 
diff --git a/mlir/test/Conversion/ShapeToSCF/shape-to-scf.mlir b/mlir/test/Conversion/ShapeToSCF/shape-to-scf.mlir
index 441b2e92cc3d9..97d2bce5a0948 100644
--- a/mlir/test/Conversion/ShapeToSCF/shape-to-scf.mlir
+++ b/mlir/test/Conversion/ShapeToSCF/shape-to-scf.mlir
@@ -24,21 +24,32 @@ func @shape_reduce(%shape : tensor<?xindex>) -> index {
 
 // -----
 
+// Don't lower `shape_of` for result type of `shape.shape`.
+// CHECK-LABEL: @shape_of
+// CHECK-SAME: (%[[ARG:.*]]: tensor<*xf32>)
+func @shape_of(%arg : tensor<*xf32>) {
+  // CHECK: shape.shape
+  %shape = shape.shape_of %arg : tensor<*xf32> -> !shape.shape
+  return
+}
+
+// -----
+
 // Lower `shape_of` for unranked tensors.
 // CHECK-LABEL: @shape_of_unranked
 // CHECK-SAME: (%[[ARG:.*]]: tensor<*xf32>)
 func @shape_of_unranked(%arg : tensor<*xf32>) {
-  // CHECK-DAG: %[[RANK:.*]] = rank %[[ARG]] : tensor<*xf32>
-  // CHECK-DAG: %[[SHAPE_MEM:.*]] = alloca(%[[RANK]]) : memref<?xi64>
-  // CHECK-DAG: %[[C0:.*]] = constant 0 : index
-  // CHECK-DAG: %[[C1:.*]] = constant 1 : index
-  // CHECK:     scf.for %[[I:.*]] = %[[C0]] to %[[RANK]] step %[[C1]] {
-  // CHECK-DAG:   %[[DIM:.]] = dim %[[ARG]], %[[I]] : tensor<*xf32>
-  // CHECK-DAG:   %[[DIM_INT:.*]] = index_cast %[[DIM]] : index to i64
-  // CHECK-DAG:   store %[[DIM_INT]], %[[SHAPE_MEM]][%[[I]]] : memref<?xi64>
-  // CHECK:     }
-  // CHECK-DAG: %[[SHAPE_INT:.*]] = tensor_load %[[SHAPE_MEM]] : memref<?xi64>
-  // CHECK-DAG: %[[SHAPE:.*]] = index_cast %[[SHAPE_INT]] : tensor<?xi64> to tensor<?xindex>
+  // CHECK: %[[RANK:.*]] = rank %[[ARG]] : tensor<*xf32>
+  // CHECK: %[[SHAPE_MEM:.*]] = alloca(%[[RANK]]) : memref<?xi64>
+  // CHECK: %[[C0:.*]] = constant 0 : index
+  // CHECK: %[[C1:.*]] = constant 1 : index
+  // CHECK: scf.for %[[I:.*]] = %[[C0]] to %[[RANK]] step %[[C1]] {
+  // CHECK:   %[[DIM:.]] = dim %[[ARG]], %[[I]] : tensor<*xf32>
+  // CHECK:   %[[DIM_INT:.*]] = index_cast %[[DIM]] : index to i64
+  // CHECK:   store %[[DIM_INT]], %[[SHAPE_MEM]][%[[I]]] : memref<?xi64>
+  // CHECK: }
+  // CHECK: %[[SHAPE_INT:.*]] = tensor_load %[[SHAPE_MEM]] : memref<?xi64>
+  // CHECK: %[[SHAPE:.*]] = index_cast %[[SHAPE_INT]] : tensor<?xi64> to tensor<?xindex>
   %shape = shape.shape_of %arg : tensor<*xf32> -> tensor<?xindex>
   return
 }

From d1abca187d9c77d1e238b90874cada81ec4f05cc Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Wed, 29 Jul 2020 15:47:28 +0100
Subject: [PATCH 0469/1035] [CostModel][X86] Add SSE costs for
 SMAX/SMIN/UMAX/UMIN intrinsics

---
 .../lib/Target/X86/X86TargetTransformInfo.cpp |  86 ++++++
 .../Analysis/CostModel/X86/arith-sminmax.ll   | 276 +++++++++---------
 .../Analysis/CostModel/X86/arith-uminmax.ll   | 276 +++++++++---------
 3 files changed, 362 insertions(+), 276 deletions(-)

diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index 4ab0c032603b5..c9179742bcb9c 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -2292,10 +2292,18 @@ int X86TTIImpl::getTypeBasedIntrinsicInstrCost(
     { ISD::CTTZ,       MVT::v64i8,   9 },
     { ISD::SADDSAT,    MVT::v32i16,  1 },
     { ISD::SADDSAT,    MVT::v64i8,   1 },
+    { ISD::SMAX,       MVT::v32i16,  1 },
+    { ISD::SMAX,       MVT::v64i8,   1 },
+    { ISD::SMIN,       MVT::v32i16,  1 },
+    { ISD::SMIN,       MVT::v64i8,   1 },
     { ISD::SSUBSAT,    MVT::v32i16,  1 },
     { ISD::SSUBSAT,    MVT::v64i8,   1 },
     { ISD::UADDSAT,    MVT::v32i16,  1 },
     { ISD::UADDSAT,    MVT::v64i8,   1 },
+    { ISD::UMAX,       MVT::v32i16,  1 },
+    { ISD::UMAX,       MVT::v64i8,   1 },
+    { ISD::UMIN,       MVT::v32i16,  1 },
+    { ISD::UMIN,       MVT::v64i8,   1 },
     { ISD::USUBSAT,    MVT::v32i16,  1 },
     { ISD::USUBSAT,    MVT::v64i8,   1 },
   };
@@ -2322,6 +2330,30 @@ int X86TTIImpl::getTypeBasedIntrinsicInstrCost(
     { ISD::CTTZ,       MVT::v16i32, 28 },
     { ISD::CTTZ,       MVT::v32i16, 24 },
     { ISD::CTTZ,       MVT::v64i8,  18 },
+    { ISD::SMAX,       MVT::v8i64,   1 },
+    { ISD::SMAX,       MVT::v16i32,  1 },
+    { ISD::SMAX,       MVT::v32i16,  2 }, // FIXME: include split
+    { ISD::SMAX,       MVT::v64i8,   2 }, // FIXME: include split
+    { ISD::SMAX,       MVT::v4i64,   1 },
+    { ISD::SMAX,       MVT::v2i64,   1 },
+    { ISD::SMIN,       MVT::v8i64,   1 },
+    { ISD::SMIN,       MVT::v16i32,  1 },
+    { ISD::SMIN,       MVT::v32i16,  2 }, // FIXME: include split
+    { ISD::SMIN,       MVT::v64i8,   2 }, // FIXME: include split
+    { ISD::SMIN,       MVT::v4i64,   1 },
+    { ISD::SMIN,       MVT::v2i64,   1 },
+    { ISD::UMAX,       MVT::v8i64,   1 },
+    { ISD::UMAX,       MVT::v16i32,  1 },
+    { ISD::UMAX,       MVT::v32i16,  2 }, // FIXME: include split
+    { ISD::UMAX,       MVT::v64i8,   2 }, // FIXME: include split
+    { ISD::UMAX,       MVT::v4i64,   1 },
+    { ISD::UMAX,       MVT::v2i64,   1 },
+    { ISD::UMIN,       MVT::v8i64,   1 },
+    { ISD::UMIN,       MVT::v16i32,  1 },
+    { ISD::UMIN,       MVT::v32i16,  2 }, // FIXME: include split
+    { ISD::UMIN,       MVT::v64i8,   2 }, // FIXME: include split
+    { ISD::UMIN,       MVT::v4i64,   1 },
+    { ISD::UMIN,       MVT::v2i64,   1 },
     { ISD::USUBSAT,    MVT::v16i32,  2 }, // pmaxud + psubd
     { ISD::USUBSAT,    MVT::v2i64,   2 }, // pmaxuq + psubq
     { ISD::USUBSAT,    MVT::v4i64,   2 }, // pmaxuq + psubq
@@ -2387,11 +2419,23 @@ int X86TTIImpl::getTypeBasedIntrinsicInstrCost(
     { ISD::CTTZ,       MVT::v32i8,   9 },
     { ISD::SADDSAT,    MVT::v16i16,  1 },
     { ISD::SADDSAT,    MVT::v32i8,   1 },
+    { ISD::SMAX,       MVT::v8i32,   1 },
+    { ISD::SMAX,       MVT::v16i16,  1 },
+    { ISD::SMAX,       MVT::v32i8,   1 },
+    { ISD::SMIN,       MVT::v8i32,   1 },
+    { ISD::SMIN,       MVT::v16i16,  1 },
+    { ISD::SMIN,       MVT::v32i8,   1 },
     { ISD::SSUBSAT,    MVT::v16i16,  1 },
     { ISD::SSUBSAT,    MVT::v32i8,   1 },
     { ISD::UADDSAT,    MVT::v16i16,  1 },
     { ISD::UADDSAT,    MVT::v32i8,   1 },
     { ISD::UADDSAT,    MVT::v8i32,   3 }, // not + pminud + paddd
+    { ISD::UMAX,       MVT::v8i32,   1 },
+    { ISD::UMAX,       MVT::v16i16,  1 },
+    { ISD::UMAX,       MVT::v32i8,   1 },
+    { ISD::UMIN,       MVT::v8i32,   1 },
+    { ISD::UMIN,       MVT::v16i16,  1 },
+    { ISD::UMIN,       MVT::v32i8,   1 },
     { ISD::USUBSAT,    MVT::v16i16,  1 },
     { ISD::USUBSAT,    MVT::v32i8,   1 },
     { ISD::USUBSAT,    MVT::v8i32,   2 }, // pmaxud + psubd
@@ -2428,11 +2472,23 @@ int X86TTIImpl::getTypeBasedIntrinsicInstrCost(
     { ISD::CTTZ,       MVT::v32i8,  20 }, // 2 x 128-bit Op + extract/insert
     { ISD::SADDSAT,    MVT::v16i16,  4 }, // 2 x 128-bit Op + extract/insert
     { ISD::SADDSAT,    MVT::v32i8,   4 }, // 2 x 128-bit Op + extract/insert
+    { ISD::SMAX,       MVT::v8i32,   4 }, // 2 x 128-bit Op + extract/insert
+    { ISD::SMAX,       MVT::v16i16,  4 }, // 2 x 128-bit Op + extract/insert
+    { ISD::SMAX,       MVT::v32i8,   4 }, // 2 x 128-bit Op + extract/insert
+    { ISD::SMIN,       MVT::v8i32,   4 }, // 2 x 128-bit Op + extract/insert
+    { ISD::SMIN,       MVT::v16i16,  4 }, // 2 x 128-bit Op + extract/insert
+    { ISD::SMIN,       MVT::v32i8,   4 }, // 2 x 128-bit Op + extract/insert
     { ISD::SSUBSAT,    MVT::v16i16,  4 }, // 2 x 128-bit Op + extract/insert
     { ISD::SSUBSAT,    MVT::v32i8,   4 }, // 2 x 128-bit Op + extract/insert
     { ISD::UADDSAT,    MVT::v16i16,  4 }, // 2 x 128-bit Op + extract/insert
     { ISD::UADDSAT,    MVT::v32i8,   4 }, // 2 x 128-bit Op + extract/insert
     { ISD::UADDSAT,    MVT::v8i32,   8 }, // 2 x 128-bit Op + extract/insert
+    { ISD::UMAX,       MVT::v8i32,   4 }, // 2 x 128-bit Op + extract/insert
+    { ISD::UMAX,       MVT::v16i16,  4 }, // 2 x 128-bit Op + extract/insert
+    { ISD::UMAX,       MVT::v32i8,   4 }, // 2 x 128-bit Op + extract/insert
+    { ISD::UMIN,       MVT::v8i32,   4 }, // 2 x 128-bit Op + extract/insert
+    { ISD::UMIN,       MVT::v16i16,  4 }, // 2 x 128-bit Op + extract/insert
+    { ISD::UMIN,       MVT::v32i8,   4 }, // 2 x 128-bit Op + extract/insert
     { ISD::USUBSAT,    MVT::v16i16,  4 }, // 2 x 128-bit Op + extract/insert
     { ISD::USUBSAT,    MVT::v32i8,   4 }, // 2 x 128-bit Op + extract/insert
     { ISD::USUBSAT,    MVT::v8i32,   6 }, // 2 x 128-bit Op + extract/insert
@@ -2468,6 +2524,16 @@ int X86TTIImpl::getTypeBasedIntrinsicInstrCost(
     { ISD::FSQRT,      MVT::f32,    18 }, // Nehalem from http://www.agner.org/
     { ISD::FSQRT,      MVT::v4f32,  18 }, // Nehalem from http://www.agner.org/
   };
+  static const CostTblEntry SSE41CostTbl[] = {
+    { ISD::SMAX,       MVT::v4i32,   1 },
+    { ISD::SMAX,       MVT::v16i8,   1 },
+    { ISD::SMIN,       MVT::v4i32,   1 },
+    { ISD::SMIN,       MVT::v16i8,   1 },
+    { ISD::UMAX,       MVT::v4i32,   1 },
+    { ISD::UMAX,       MVT::v8i16,   1 },
+    { ISD::UMIN,       MVT::v4i32,   1 },
+    { ISD::UMIN,       MVT::v8i16,   1 },
+  };
   static const CostTblEntry SSSE3CostTbl[] = {
     { ISD::ABS,        MVT::v4i32,   1 },
     { ISD::ABS,        MVT::v8i16,   1 },
@@ -2518,10 +2584,14 @@ int X86TTIImpl::getTypeBasedIntrinsicInstrCost(
     { ISD::CTTZ,       MVT::v16i8,  13 },
     { ISD::SADDSAT,    MVT::v8i16,   1 },
     { ISD::SADDSAT,    MVT::v16i8,   1 },
+    { ISD::SMAX,       MVT::v8i16,   1 },
+    { ISD::SMIN,       MVT::v8i16,   1 },
     { ISD::SSUBSAT,    MVT::v8i16,   1 },
     { ISD::SSUBSAT,    MVT::v16i8,   1 },
     { ISD::UADDSAT,    MVT::v8i16,   1 },
     { ISD::UADDSAT,    MVT::v16i8,   1 },
+    { ISD::UMAX,       MVT::v16i8,   1 },
+    { ISD::UMIN,       MVT::v16i8,   1 },
     { ISD::USUBSAT,    MVT::v8i16,   1 },
     { ISD::USUBSAT,    MVT::v16i8,   1 },
     { ISD::FMAXNUM,    MVT::f64,     4 },
@@ -2621,12 +2691,24 @@ int X86TTIImpl::getTypeBasedIntrinsicInstrCost(
   case Intrinsic::sadd_sat:
     ISD = ISD::SADDSAT;
     break;
+  case Intrinsic::smax:
+    ISD = ISD::SMAX;
+    break;
+  case Intrinsic::smin:
+    ISD = ISD::SMIN;
+    break;
   case Intrinsic::ssub_sat:
     ISD = ISD::SSUBSAT;
     break;
   case Intrinsic::uadd_sat:
     ISD = ISD::UADDSAT;
     break;
+  case Intrinsic::umax:
+    ISD = ISD::UMAX;
+    break;
+  case Intrinsic::umin:
+    ISD = ISD::UMIN;
+    break;
   case Intrinsic::usub_sat:
     ISD = ISD::USUBSAT;
     break;
@@ -2689,6 +2771,10 @@ int X86TTIImpl::getTypeBasedIntrinsicInstrCost(
       if (const auto *Entry = CostTableLookup(SSE42CostTbl, ISD, MTy))
         return LT.first * Entry->Cost;
 
+    if (ST->hasSSE41())
+      if (const auto *Entry = CostTableLookup(SSE41CostTbl, ISD, MTy))
+        return LT.first * Entry->Cost;
+
     if (ST->hasSSSE3())
       if (const auto *Entry = CostTableLookup(SSSE3CostTbl, ISD, MTy))
         return LT.first * Entry->Cost;
diff --git a/llvm/test/Analysis/CostModel/X86/arith-sminmax.ll b/llvm/test/Analysis/CostModel/X86/arith-sminmax.ll
index 2d6f75b6cb92e..7270a0eced4af 100644
--- a/llvm/test/Analysis/CostModel/X86/arith-sminmax.ll
+++ b/llvm/test/Analysis/CostModel/X86/arith-sminmax.ll
@@ -39,9 +39,9 @@ define i32 @smax(i32 %arg) {
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I32 = call <8 x i32> @llvm.smax.v8i32(<8 x i32> undef, <8 x i32> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I32 = call <16 x i32> @llvm.smax.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.smax.i16(i16 undef, i16 undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I16 = call <8 x i16> @llvm.smax.v8i16(<8 x i16> undef, <8 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I16 = call <16 x i16> @llvm.smax.v16i16(<16 x i16> undef, <16 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I16 = call <32 x i16> @llvm.smax.v32i16(<32 x i16> undef, <32 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.smax.v8i16(<8 x i16> undef, <8 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.smax.v16i16(<16 x i16> undef, <16 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.smax.v32i16(<32 x i16> undef, <32 x i16> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.smax.i8(i8 undef, i8 undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I8 = call <16 x i8> @llvm.smax.v16i8(<16 x i8> undef, <16 x i8> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I8 = call <32 x i8> @llvm.smax.v32i8(<32 x i8> undef, <32 x i8> undef)
@@ -58,9 +58,9 @@ define i32 @smax(i32 %arg) {
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I32 = call <8 x i32> @llvm.smax.v8i32(<8 x i32> undef, <8 x i32> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I32 = call <16 x i32> @llvm.smax.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.smax.i16(i16 undef, i16 undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I16 = call <8 x i16> @llvm.smax.v8i16(<8 x i16> undef, <8 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I16 = call <16 x i16> @llvm.smax.v16i16(<16 x i16> undef, <16 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I16 = call <32 x i16> @llvm.smax.v32i16(<32 x i16> undef, <32 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.smax.v8i16(<8 x i16> undef, <8 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.smax.v16i16(<16 x i16> undef, <16 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.smax.v32i16(<32 x i16> undef, <32 x i16> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.smax.i8(i8 undef, i8 undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I8 = call <16 x i8> @llvm.smax.v16i8(<16 x i8> undef, <16 x i8> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I8 = call <32 x i8> @llvm.smax.v32i8(<32 x i8> undef, <32 x i8> undef)
@@ -73,17 +73,17 @@ define i32 @smax(i32 %arg) {
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I64 = call <4 x i64> @llvm.smax.v4i64(<4 x i64> undef, <4 x i64> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I64 = call <8 x i64> @llvm.smax.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.smax.i32(i32 undef, i32 undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.smax.v4i32(<4 x i32> undef, <4 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = call <8 x i32> @llvm.smax.v8i32(<8 x i32> undef, <8 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = call <16 x i32> @llvm.smax.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <4 x i32> @llvm.smax.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call <8 x i32> @llvm.smax.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = call <16 x i32> @llvm.smax.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.smax.i16(i16 undef, i16 undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call <8 x i16> @llvm.smax.v8i16(<8 x i16> undef, <8 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = call <16 x i16> @llvm.smax.v16i16(<16 x i16> undef, <16 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call <32 x i16> @llvm.smax.v32i16(<32 x i16> undef, <32 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.smax.v8i16(<8 x i16> undef, <8 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.smax.v16i16(<16 x i16> undef, <16 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.smax.v32i16(<32 x i16> undef, <32 x i16> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.smax.i8(i8 undef, i8 undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call <16 x i8> @llvm.smax.v16i8(<16 x i8> undef, <16 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = call <32 x i8> @llvm.smax.v32i8(<32 x i8> undef, <32 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = call <64 x i8> @llvm.smax.v64i8(<64 x i8> undef, <64 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.smax.v16i8(<16 x i8> undef, <16 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.smax.v32i8(<32 x i8> undef, <32 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.smax.v64i8(<64 x i8> undef, <64 x i8> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX1-LABEL: 'smax'
@@ -92,17 +92,17 @@ define i32 @smax(i32 %arg) {
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4I64 = call <4 x i64> @llvm.smax.v4i64(<4 x i64> undef, <4 x i64> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8I64 = call <8 x i64> @llvm.smax.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.smax.i32(i32 undef, i32 undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.smax.v4i32(<4 x i32> undef, <4 x i32> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8I32 = call <8 x i32> @llvm.smax.v8i32(<8 x i32> undef, <8 x i32> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V16I32 = call <16 x i32> @llvm.smax.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <4 x i32> @llvm.smax.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = call <8 x i32> @llvm.smax.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = call <16 x i32> @llvm.smax.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.smax.i16(i16 undef, i16 undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call <8 x i16> @llvm.smax.v8i16(<8 x i16> undef, <8 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16I16 = call <16 x i16> @llvm.smax.v16i16(<16 x i16> undef, <16 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V32I16 = call <32 x i16> @llvm.smax.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.smax.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = call <16 x i16> @llvm.smax.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call <32 x i16> @llvm.smax.v32i16(<32 x i16> undef, <32 x i16> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.smax.i8(i8 undef, i8 undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call <16 x i8> @llvm.smax.v16i8(<16 x i8> undef, <16 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V32I8 = call <32 x i8> @llvm.smax.v32i8(<32 x i8> undef, <32 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V64I8 = call <64 x i8> @llvm.smax.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.smax.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = call <32 x i8> @llvm.smax.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = call <64 x i8> @llvm.smax.v64i8(<64 x i8> undef, <64 x i8> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX2-LABEL: 'smax'
@@ -111,74 +111,74 @@ define i32 @smax(i32 %arg) {
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = call <4 x i64> @llvm.smax.v4i64(<4 x i64> undef, <4 x i64> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I64 = call <8 x i64> @llvm.smax.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.smax.i32(i32 undef, i32 undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.smax.v4i32(<4 x i32> undef, <4 x i32> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call <8 x i32> @llvm.smax.v8i32(<8 x i32> undef, <8 x i32> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = call <16 x i32> @llvm.smax.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <4 x i32> @llvm.smax.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <8 x i32> @llvm.smax.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = call <16 x i32> @llvm.smax.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.smax.i16(i16 undef, i16 undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call <8 x i16> @llvm.smax.v8i16(<8 x i16> undef, <8 x i16> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.smax.v16i16(<16 x i16> undef, <16 x i16> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.smax.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.smax.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.smax.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = call <32 x i16> @llvm.smax.v32i16(<32 x i16> undef, <32 x i16> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.smax.i8(i8 undef, i8 undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call <16 x i8> @llvm.smax.v16i8(<16 x i8> undef, <16 x i8> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.smax.v32i8(<32 x i8> undef, <32 x i8> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.smax.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.smax.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = call <32 x i8> @llvm.smax.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = call <64 x i8> @llvm.smax.v64i8(<64 x i8> undef, <64 x i8> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512F-LABEL: 'smax'
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.smax.i64(i64 undef, i64 undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = call <2 x i64> @llvm.smax.v2i64(<2 x i64> undef, <2 x i64> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = call <4 x i64> @llvm.smax.v4i64(<4 x i64> undef, <4 x i64> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = call <8 x i64> @llvm.smax.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.smax.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.smax.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.smax.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.smax.i32(i32 undef, i32 undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.smax.v4i32(<4 x i32> undef, <4 x i32> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call <8 x i32> @llvm.smax.v8i32(<8 x i32> undef, <8 x i32> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = call <16 x i32> @llvm.smax.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <4 x i32> @llvm.smax.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <8 x i32> @llvm.smax.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I32 = call <16 x i32> @llvm.smax.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.smax.i16(i16 undef, i16 undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call <8 x i16> @llvm.smax.v8i16(<8 x i16> undef, <8 x i16> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.smax.v16i16(<16 x i16> undef, <16 x i16> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.smax.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.smax.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.smax.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = call <32 x i16> @llvm.smax.v32i16(<32 x i16> undef, <32 x i16> undef)
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.smax.i8(i8 undef, i8 undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call <16 x i8> @llvm.smax.v16i8(<16 x i8> undef, <16 x i8> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.smax.v32i8(<32 x i8> undef, <32 x i8> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.smax.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.smax.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = call <32 x i8> @llvm.smax.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = call <64 x i8> @llvm.smax.v64i8(<64 x i8> undef, <64 x i8> undef)
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512DQ-LABEL: 'smax'
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.smax.i64(i64 undef, i64 undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = call <2 x i64> @llvm.smax.v2i64(<2 x i64> undef, <2 x i64> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = call <4 x i64> @llvm.smax.v4i64(<4 x i64> undef, <4 x i64> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = call <8 x i64> @llvm.smax.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.smax.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.smax.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.smax.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.smax.i32(i32 undef, i32 undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.smax.v4i32(<4 x i32> undef, <4 x i32> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call <8 x i32> @llvm.smax.v8i32(<8 x i32> undef, <8 x i32> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = call <16 x i32> @llvm.smax.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <4 x i32> @llvm.smax.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <8 x i32> @llvm.smax.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I32 = call <16 x i32> @llvm.smax.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.smax.i16(i16 undef, i16 undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call <8 x i16> @llvm.smax.v8i16(<8 x i16> undef, <8 x i16> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.smax.v16i16(<16 x i16> undef, <16 x i16> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.smax.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.smax.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.smax.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = call <32 x i16> @llvm.smax.v32i16(<32 x i16> undef, <32 x i16> undef)
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.smax.i8(i8 undef, i8 undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call <16 x i8> @llvm.smax.v16i8(<16 x i8> undef, <16 x i8> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.smax.v32i8(<32 x i8> undef, <32 x i8> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.smax.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.smax.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = call <32 x i8> @llvm.smax.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = call <64 x i8> @llvm.smax.v64i8(<64 x i8> undef, <64 x i8> undef)
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512BW-LABEL: 'smax'
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.smax.i64(i64 undef, i64 undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = call <2 x i64> @llvm.smax.v2i64(<2 x i64> undef, <2 x i64> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = call <4 x i64> @llvm.smax.v4i64(<4 x i64> undef, <4 x i64> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = call <8 x i64> @llvm.smax.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.smax.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.smax.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.smax.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.smax.i32(i32 undef, i32 undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.smax.v4i32(<4 x i32> undef, <4 x i32> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call <8 x i32> @llvm.smax.v8i32(<8 x i32> undef, <8 x i32> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = call <16 x i32> @llvm.smax.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <4 x i32> @llvm.smax.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <8 x i32> @llvm.smax.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I32 = call <16 x i32> @llvm.smax.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.smax.i16(i16 undef, i16 undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call <8 x i16> @llvm.smax.v8i16(<8 x i16> undef, <8 x i16> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.smax.v16i16(<16 x i16> undef, <16 x i16> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = call <32 x i16> @llvm.smax.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.smax.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.smax.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.smax.v32i16(<32 x i16> undef, <32 x i16> undef)
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.smax.i8(i8 undef, i8 undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call <16 x i8> @llvm.smax.v16i8(<16 x i8> undef, <16 x i8> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.smax.v32i8(<32 x i8> undef, <32 x i8> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = call <64 x i8> @llvm.smax.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.smax.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = call <32 x i8> @llvm.smax.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64I8 = call <64 x i8> @llvm.smax.v64i8(<64 x i8> undef, <64 x i8> undef)
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %I64 = call i64 @llvm.smax.i64(i64 undef, i64 undef)
@@ -235,9 +235,9 @@ define i32 @smin(i32 %arg) {
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I32 = call <8 x i32> @llvm.smin.v8i32(<8 x i32> undef, <8 x i32> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I32 = call <16 x i32> @llvm.smin.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.smin.i16(i16 undef, i16 undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I16 = call <8 x i16> @llvm.smin.v8i16(<8 x i16> undef, <8 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I16 = call <16 x i16> @llvm.smin.v16i16(<16 x i16> undef, <16 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I16 = call <32 x i16> @llvm.smin.v32i16(<32 x i16> undef, <32 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.smin.v8i16(<8 x i16> undef, <8 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.smin.v16i16(<16 x i16> undef, <16 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.smin.v32i16(<32 x i16> undef, <32 x i16> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.smin.i8(i8 undef, i8 undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I8 = call <16 x i8> @llvm.smin.v16i8(<16 x i8> undef, <16 x i8> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I8 = call <32 x i8> @llvm.smin.v32i8(<32 x i8> undef, <32 x i8> undef)
@@ -254,9 +254,9 @@ define i32 @smin(i32 %arg) {
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I32 = call <8 x i32> @llvm.smin.v8i32(<8 x i32> undef, <8 x i32> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I32 = call <16 x i32> @llvm.smin.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.smin.i16(i16 undef, i16 undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I16 = call <8 x i16> @llvm.smin.v8i16(<8 x i16> undef, <8 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I16 = call <16 x i16> @llvm.smin.v16i16(<16 x i16> undef, <16 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I16 = call <32 x i16> @llvm.smin.v32i16(<32 x i16> undef, <32 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.smin.v8i16(<8 x i16> undef, <8 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.smin.v16i16(<16 x i16> undef, <16 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.smin.v32i16(<32 x i16> undef, <32 x i16> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.smin.i8(i8 undef, i8 undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I8 = call <16 x i8> @llvm.smin.v16i8(<16 x i8> undef, <16 x i8> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I8 = call <32 x i8> @llvm.smin.v32i8(<32 x i8> undef, <32 x i8> undef)
@@ -269,17 +269,17 @@ define i32 @smin(i32 %arg) {
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I64 = call <4 x i64> @llvm.smin.v4i64(<4 x i64> undef, <4 x i64> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I64 = call <8 x i64> @llvm.smin.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.smin.i32(i32 undef, i32 undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.smin.v4i32(<4 x i32> undef, <4 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = call <8 x i32> @llvm.smin.v8i32(<8 x i32> undef, <8 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = call <16 x i32> @llvm.smin.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <4 x i32> @llvm.smin.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call <8 x i32> @llvm.smin.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = call <16 x i32> @llvm.smin.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.smin.i16(i16 undef, i16 undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call <8 x i16> @llvm.smin.v8i16(<8 x i16> undef, <8 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = call <16 x i16> @llvm.smin.v16i16(<16 x i16> undef, <16 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call <32 x i16> @llvm.smin.v32i16(<32 x i16> undef, <32 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.smin.v8i16(<8 x i16> undef, <8 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.smin.v16i16(<16 x i16> undef, <16 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.smin.v32i16(<32 x i16> undef, <32 x i16> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.smin.i8(i8 undef, i8 undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call <16 x i8> @llvm.smin.v16i8(<16 x i8> undef, <16 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = call <32 x i8> @llvm.smin.v32i8(<32 x i8> undef, <32 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = call <64 x i8> @llvm.smin.v64i8(<64 x i8> undef, <64 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.smin.v16i8(<16 x i8> undef, <16 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.smin.v32i8(<32 x i8> undef, <32 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.smin.v64i8(<64 x i8> undef, <64 x i8> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX1-LABEL: 'smin'
@@ -288,17 +288,17 @@ define i32 @smin(i32 %arg) {
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4I64 = call <4 x i64> @llvm.smin.v4i64(<4 x i64> undef, <4 x i64> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8I64 = call <8 x i64> @llvm.smin.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.smin.i32(i32 undef, i32 undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.smin.v4i32(<4 x i32> undef, <4 x i32> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8I32 = call <8 x i32> @llvm.smin.v8i32(<8 x i32> undef, <8 x i32> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V16I32 = call <16 x i32> @llvm.smin.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <4 x i32> @llvm.smin.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = call <8 x i32> @llvm.smin.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = call <16 x i32> @llvm.smin.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.smin.i16(i16 undef, i16 undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call <8 x i16> @llvm.smin.v8i16(<8 x i16> undef, <8 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16I16 = call <16 x i16> @llvm.smin.v16i16(<16 x i16> undef, <16 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V32I16 = call <32 x i16> @llvm.smin.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.smin.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = call <16 x i16> @llvm.smin.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call <32 x i16> @llvm.smin.v32i16(<32 x i16> undef, <32 x i16> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.smin.i8(i8 undef, i8 undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call <16 x i8> @llvm.smin.v16i8(<16 x i8> undef, <16 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V32I8 = call <32 x i8> @llvm.smin.v32i8(<32 x i8> undef, <32 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V64I8 = call <64 x i8> @llvm.smin.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.smin.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = call <32 x i8> @llvm.smin.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = call <64 x i8> @llvm.smin.v64i8(<64 x i8> undef, <64 x i8> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX2-LABEL: 'smin'
@@ -307,74 +307,74 @@ define i32 @smin(i32 %arg) {
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = call <4 x i64> @llvm.smin.v4i64(<4 x i64> undef, <4 x i64> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I64 = call <8 x i64> @llvm.smin.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.smin.i32(i32 undef, i32 undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.smin.v4i32(<4 x i32> undef, <4 x i32> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call <8 x i32> @llvm.smin.v8i32(<8 x i32> undef, <8 x i32> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = call <16 x i32> @llvm.smin.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <4 x i32> @llvm.smin.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <8 x i32> @llvm.smin.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = call <16 x i32> @llvm.smin.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.smin.i16(i16 undef, i16 undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call <8 x i16> @llvm.smin.v8i16(<8 x i16> undef, <8 x i16> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.smin.v16i16(<16 x i16> undef, <16 x i16> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.smin.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.smin.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.smin.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = call <32 x i16> @llvm.smin.v32i16(<32 x i16> undef, <32 x i16> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.smin.i8(i8 undef, i8 undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call <16 x i8> @llvm.smin.v16i8(<16 x i8> undef, <16 x i8> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.smin.v32i8(<32 x i8> undef, <32 x i8> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.smin.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.smin.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = call <32 x i8> @llvm.smin.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = call <64 x i8> @llvm.smin.v64i8(<64 x i8> undef, <64 x i8> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512F-LABEL: 'smin'
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.smin.i64(i64 undef, i64 undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = call <2 x i64> @llvm.smin.v2i64(<2 x i64> undef, <2 x i64> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = call <4 x i64> @llvm.smin.v4i64(<4 x i64> undef, <4 x i64> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = call <8 x i64> @llvm.smin.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.smin.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.smin.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.smin.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.smin.i32(i32 undef, i32 undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.smin.v4i32(<4 x i32> undef, <4 x i32> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call <8 x i32> @llvm.smin.v8i32(<8 x i32> undef, <8 x i32> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = call <16 x i32> @llvm.smin.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <4 x i32> @llvm.smin.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <8 x i32> @llvm.smin.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I32 = call <16 x i32> @llvm.smin.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.smin.i16(i16 undef, i16 undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call <8 x i16> @llvm.smin.v8i16(<8 x i16> undef, <8 x i16> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.smin.v16i16(<16 x i16> undef, <16 x i16> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.smin.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.smin.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.smin.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = call <32 x i16> @llvm.smin.v32i16(<32 x i16> undef, <32 x i16> undef)
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.smin.i8(i8 undef, i8 undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call <16 x i8> @llvm.smin.v16i8(<16 x i8> undef, <16 x i8> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.smin.v32i8(<32 x i8> undef, <32 x i8> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.smin.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.smin.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = call <32 x i8> @llvm.smin.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = call <64 x i8> @llvm.smin.v64i8(<64 x i8> undef, <64 x i8> undef)
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512DQ-LABEL: 'smin'
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.smin.i64(i64 undef, i64 undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = call <2 x i64> @llvm.smin.v2i64(<2 x i64> undef, <2 x i64> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = call <4 x i64> @llvm.smin.v4i64(<4 x i64> undef, <4 x i64> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = call <8 x i64> @llvm.smin.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.smin.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.smin.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.smin.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.smin.i32(i32 undef, i32 undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.smin.v4i32(<4 x i32> undef, <4 x i32> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call <8 x i32> @llvm.smin.v8i32(<8 x i32> undef, <8 x i32> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = call <16 x i32> @llvm.smin.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <4 x i32> @llvm.smin.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <8 x i32> @llvm.smin.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I32 = call <16 x i32> @llvm.smin.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.smin.i16(i16 undef, i16 undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call <8 x i16> @llvm.smin.v8i16(<8 x i16> undef, <8 x i16> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.smin.v16i16(<16 x i16> undef, <16 x i16> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.smin.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.smin.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.smin.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = call <32 x i16> @llvm.smin.v32i16(<32 x i16> undef, <32 x i16> undef)
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.smin.i8(i8 undef, i8 undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call <16 x i8> @llvm.smin.v16i8(<16 x i8> undef, <16 x i8> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.smin.v32i8(<32 x i8> undef, <32 x i8> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.smin.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.smin.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = call <32 x i8> @llvm.smin.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = call <64 x i8> @llvm.smin.v64i8(<64 x i8> undef, <64 x i8> undef)
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512BW-LABEL: 'smin'
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.smin.i64(i64 undef, i64 undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = call <2 x i64> @llvm.smin.v2i64(<2 x i64> undef, <2 x i64> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = call <4 x i64> @llvm.smin.v4i64(<4 x i64> undef, <4 x i64> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = call <8 x i64> @llvm.smin.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.smin.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.smin.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.smin.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.smin.i32(i32 undef, i32 undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.smin.v4i32(<4 x i32> undef, <4 x i32> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call <8 x i32> @llvm.smin.v8i32(<8 x i32> undef, <8 x i32> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = call <16 x i32> @llvm.smin.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <4 x i32> @llvm.smin.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <8 x i32> @llvm.smin.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I32 = call <16 x i32> @llvm.smin.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.smin.i16(i16 undef, i16 undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call <8 x i16> @llvm.smin.v8i16(<8 x i16> undef, <8 x i16> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.smin.v16i16(<16 x i16> undef, <16 x i16> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = call <32 x i16> @llvm.smin.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.smin.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.smin.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.smin.v32i16(<32 x i16> undef, <32 x i16> undef)
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.smin.i8(i8 undef, i8 undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call <16 x i8> @llvm.smin.v16i8(<16 x i8> undef, <16 x i8> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.smin.v32i8(<32 x i8> undef, <32 x i8> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = call <64 x i8> @llvm.smin.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.smin.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = call <32 x i8> @llvm.smin.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64I8 = call <64 x i8> @llvm.smin.v64i8(<64 x i8> undef, <64 x i8> undef)
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %I64 = call i64 @llvm.smin.i64(i64 undef, i64 undef)
diff --git a/llvm/test/Analysis/CostModel/X86/arith-uminmax.ll b/llvm/test/Analysis/CostModel/X86/arith-uminmax.ll
index 09f52c227a46c..8b6f6f20e1b40 100644
--- a/llvm/test/Analysis/CostModel/X86/arith-uminmax.ll
+++ b/llvm/test/Analysis/CostModel/X86/arith-uminmax.ll
@@ -43,9 +43,9 @@ define i32 @umax(i32 %arg) {
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I16 = call <16 x i16> @llvm.umax.v16i16(<16 x i16> undef, <16 x i16> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I16 = call <32 x i16> @llvm.umax.v32i16(<32 x i16> undef, <32 x i16> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.umax.i8(i8 undef, i8 undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I8 = call <16 x i8> @llvm.umax.v16i8(<16 x i8> undef, <16 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I8 = call <32 x i8> @llvm.umax.v32i8(<32 x i8> undef, <32 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I8 = call <64 x i8> @llvm.umax.v64i8(<64 x i8> undef, <64 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.umax.v16i8(<16 x i8> undef, <16 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.umax.v32i8(<32 x i8> undef, <32 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.umax.v64i8(<64 x i8> undef, <64 x i8> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'umax'
@@ -62,9 +62,9 @@ define i32 @umax(i32 %arg) {
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I16 = call <16 x i16> @llvm.umax.v16i16(<16 x i16> undef, <16 x i16> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I16 = call <32 x i16> @llvm.umax.v32i16(<32 x i16> undef, <32 x i16> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.umax.i8(i8 undef, i8 undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I8 = call <16 x i8> @llvm.umax.v16i8(<16 x i8> undef, <16 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I8 = call <32 x i8> @llvm.umax.v32i8(<32 x i8> undef, <32 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I8 = call <64 x i8> @llvm.umax.v64i8(<64 x i8> undef, <64 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.umax.v16i8(<16 x i8> undef, <16 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.umax.v32i8(<32 x i8> undef, <32 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.umax.v64i8(<64 x i8> undef, <64 x i8> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'umax'
@@ -73,17 +73,17 @@ define i32 @umax(i32 %arg) {
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I64 = call <4 x i64> @llvm.umax.v4i64(<4 x i64> undef, <4 x i64> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I64 = call <8 x i64> @llvm.umax.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.umax.i32(i32 undef, i32 undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.umax.v4i32(<4 x i32> undef, <4 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = call <8 x i32> @llvm.umax.v8i32(<8 x i32> undef, <8 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = call <16 x i32> @llvm.umax.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <4 x i32> @llvm.umax.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call <8 x i32> @llvm.umax.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = call <16 x i32> @llvm.umax.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.umax.i16(i16 undef, i16 undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call <8 x i16> @llvm.umax.v8i16(<8 x i16> undef, <8 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = call <16 x i16> @llvm.umax.v16i16(<16 x i16> undef, <16 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call <32 x i16> @llvm.umax.v32i16(<32 x i16> undef, <32 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.umax.v8i16(<8 x i16> undef, <8 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.umax.v16i16(<16 x i16> undef, <16 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.umax.v32i16(<32 x i16> undef, <32 x i16> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.umax.i8(i8 undef, i8 undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call <16 x i8> @llvm.umax.v16i8(<16 x i8> undef, <16 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = call <32 x i8> @llvm.umax.v32i8(<32 x i8> undef, <32 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = call <64 x i8> @llvm.umax.v64i8(<64 x i8> undef, <64 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.umax.v16i8(<16 x i8> undef, <16 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.umax.v32i8(<32 x i8> undef, <32 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.umax.v64i8(<64 x i8> undef, <64 x i8> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX1-LABEL: 'umax'
@@ -92,17 +92,17 @@ define i32 @umax(i32 %arg) {
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4I64 = call <4 x i64> @llvm.umax.v4i64(<4 x i64> undef, <4 x i64> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8I64 = call <8 x i64> @llvm.umax.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.umax.i32(i32 undef, i32 undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.umax.v4i32(<4 x i32> undef, <4 x i32> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8I32 = call <8 x i32> @llvm.umax.v8i32(<8 x i32> undef, <8 x i32> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V16I32 = call <16 x i32> @llvm.umax.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <4 x i32> @llvm.umax.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = call <8 x i32> @llvm.umax.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = call <16 x i32> @llvm.umax.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.umax.i16(i16 undef, i16 undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call <8 x i16> @llvm.umax.v8i16(<8 x i16> undef, <8 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16I16 = call <16 x i16> @llvm.umax.v16i16(<16 x i16> undef, <16 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V32I16 = call <32 x i16> @llvm.umax.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.umax.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = call <16 x i16> @llvm.umax.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call <32 x i16> @llvm.umax.v32i16(<32 x i16> undef, <32 x i16> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.umax.i8(i8 undef, i8 undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call <16 x i8> @llvm.umax.v16i8(<16 x i8> undef, <16 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V32I8 = call <32 x i8> @llvm.umax.v32i8(<32 x i8> undef, <32 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V64I8 = call <64 x i8> @llvm.umax.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.umax.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = call <32 x i8> @llvm.umax.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = call <64 x i8> @llvm.umax.v64i8(<64 x i8> undef, <64 x i8> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX2-LABEL: 'umax'
@@ -111,74 +111,74 @@ define i32 @umax(i32 %arg) {
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = call <4 x i64> @llvm.umax.v4i64(<4 x i64> undef, <4 x i64> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I64 = call <8 x i64> @llvm.umax.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.umax.i32(i32 undef, i32 undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.umax.v4i32(<4 x i32> undef, <4 x i32> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call <8 x i32> @llvm.umax.v8i32(<8 x i32> undef, <8 x i32> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = call <16 x i32> @llvm.umax.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <4 x i32> @llvm.umax.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <8 x i32> @llvm.umax.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = call <16 x i32> @llvm.umax.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.umax.i16(i16 undef, i16 undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call <8 x i16> @llvm.umax.v8i16(<8 x i16> undef, <8 x i16> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.umax.v16i16(<16 x i16> undef, <16 x i16> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.umax.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.umax.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.umax.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = call <32 x i16> @llvm.umax.v32i16(<32 x i16> undef, <32 x i16> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.umax.i8(i8 undef, i8 undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call <16 x i8> @llvm.umax.v16i8(<16 x i8> undef, <16 x i8> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.umax.v32i8(<32 x i8> undef, <32 x i8> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.umax.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.umax.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = call <32 x i8> @llvm.umax.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = call <64 x i8> @llvm.umax.v64i8(<64 x i8> undef, <64 x i8> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512F-LABEL: 'umax'
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.umax.i64(i64 undef, i64 undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = call <2 x i64> @llvm.umax.v2i64(<2 x i64> undef, <2 x i64> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = call <4 x i64> @llvm.umax.v4i64(<4 x i64> undef, <4 x i64> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = call <8 x i64> @llvm.umax.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.umax.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.umax.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.umax.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.umax.i32(i32 undef, i32 undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.umax.v4i32(<4 x i32> undef, <4 x i32> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call <8 x i32> @llvm.umax.v8i32(<8 x i32> undef, <8 x i32> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = call <16 x i32> @llvm.umax.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <4 x i32> @llvm.umax.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <8 x i32> @llvm.umax.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I32 = call <16 x i32> @llvm.umax.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.umax.i16(i16 undef, i16 undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call <8 x i16> @llvm.umax.v8i16(<8 x i16> undef, <8 x i16> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.umax.v16i16(<16 x i16> undef, <16 x i16> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.umax.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.umax.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.umax.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = call <32 x i16> @llvm.umax.v32i16(<32 x i16> undef, <32 x i16> undef)
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.umax.i8(i8 undef, i8 undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call <16 x i8> @llvm.umax.v16i8(<16 x i8> undef, <16 x i8> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.umax.v32i8(<32 x i8> undef, <32 x i8> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.umax.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.umax.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = call <32 x i8> @llvm.umax.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = call <64 x i8> @llvm.umax.v64i8(<64 x i8> undef, <64 x i8> undef)
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512DQ-LABEL: 'umax'
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.umax.i64(i64 undef, i64 undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = call <2 x i64> @llvm.umax.v2i64(<2 x i64> undef, <2 x i64> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = call <4 x i64> @llvm.umax.v4i64(<4 x i64> undef, <4 x i64> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = call <8 x i64> @llvm.umax.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.umax.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.umax.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.umax.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.umax.i32(i32 undef, i32 undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.umax.v4i32(<4 x i32> undef, <4 x i32> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call <8 x i32> @llvm.umax.v8i32(<8 x i32> undef, <8 x i32> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = call <16 x i32> @llvm.umax.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <4 x i32> @llvm.umax.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <8 x i32> @llvm.umax.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I32 = call <16 x i32> @llvm.umax.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.umax.i16(i16 undef, i16 undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call <8 x i16> @llvm.umax.v8i16(<8 x i16> undef, <8 x i16> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.umax.v16i16(<16 x i16> undef, <16 x i16> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.umax.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.umax.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.umax.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = call <32 x i16> @llvm.umax.v32i16(<32 x i16> undef, <32 x i16> undef)
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.umax.i8(i8 undef, i8 undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call <16 x i8> @llvm.umax.v16i8(<16 x i8> undef, <16 x i8> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.umax.v32i8(<32 x i8> undef, <32 x i8> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.umax.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.umax.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = call <32 x i8> @llvm.umax.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = call <64 x i8> @llvm.umax.v64i8(<64 x i8> undef, <64 x i8> undef)
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512BW-LABEL: 'umax'
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.umax.i64(i64 undef, i64 undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = call <2 x i64> @llvm.umax.v2i64(<2 x i64> undef, <2 x i64> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = call <4 x i64> @llvm.umax.v4i64(<4 x i64> undef, <4 x i64> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = call <8 x i64> @llvm.umax.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.umax.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.umax.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.umax.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.umax.i32(i32 undef, i32 undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.umax.v4i32(<4 x i32> undef, <4 x i32> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call <8 x i32> @llvm.umax.v8i32(<8 x i32> undef, <8 x i32> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = call <16 x i32> @llvm.umax.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <4 x i32> @llvm.umax.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <8 x i32> @llvm.umax.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I32 = call <16 x i32> @llvm.umax.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.umax.i16(i16 undef, i16 undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call <8 x i16> @llvm.umax.v8i16(<8 x i16> undef, <8 x i16> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.umax.v16i16(<16 x i16> undef, <16 x i16> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = call <32 x i16> @llvm.umax.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.umax.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.umax.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.umax.v32i16(<32 x i16> undef, <32 x i16> undef)
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.umax.i8(i8 undef, i8 undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call <16 x i8> @llvm.umax.v16i8(<16 x i8> undef, <16 x i8> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.umax.v32i8(<32 x i8> undef, <32 x i8> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = call <64 x i8> @llvm.umax.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.umax.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = call <32 x i8> @llvm.umax.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64I8 = call <64 x i8> @llvm.umax.v64i8(<64 x i8> undef, <64 x i8> undef)
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %I64 = call i64 @llvm.umax.i64(i64 undef, i64 undef)
@@ -239,9 +239,9 @@ define i32 @umin(i32 %arg) {
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I16 = call <16 x i16> @llvm.umin.v16i16(<16 x i16> undef, <16 x i16> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I16 = call <32 x i16> @llvm.umin.v32i16(<32 x i16> undef, <32 x i16> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.umin.i8(i8 undef, i8 undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I8 = call <16 x i8> @llvm.umin.v16i8(<16 x i8> undef, <16 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I8 = call <32 x i8> @llvm.umin.v32i8(<32 x i8> undef, <32 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I8 = call <64 x i8> @llvm.umin.v64i8(<64 x i8> undef, <64 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.umin.v16i8(<16 x i8> undef, <16 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.umin.v32i8(<32 x i8> undef, <32 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.umin.v64i8(<64 x i8> undef, <64 x i8> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'umin'
@@ -258,9 +258,9 @@ define i32 @umin(i32 %arg) {
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I16 = call <16 x i16> @llvm.umin.v16i16(<16 x i16> undef, <16 x i16> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I16 = call <32 x i16> @llvm.umin.v32i16(<32 x i16> undef, <32 x i16> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.umin.i8(i8 undef, i8 undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I8 = call <16 x i8> @llvm.umin.v16i8(<16 x i8> undef, <16 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I8 = call <32 x i8> @llvm.umin.v32i8(<32 x i8> undef, <32 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I8 = call <64 x i8> @llvm.umin.v64i8(<64 x i8> undef, <64 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.umin.v16i8(<16 x i8> undef, <16 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.umin.v32i8(<32 x i8> undef, <32 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.umin.v64i8(<64 x i8> undef, <64 x i8> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'umin'
@@ -269,17 +269,17 @@ define i32 @umin(i32 %arg) {
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I64 = call <4 x i64> @llvm.umin.v4i64(<4 x i64> undef, <4 x i64> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I64 = call <8 x i64> @llvm.umin.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.umin.i32(i32 undef, i32 undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.umin.v4i32(<4 x i32> undef, <4 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = call <8 x i32> @llvm.umin.v8i32(<8 x i32> undef, <8 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = call <16 x i32> @llvm.umin.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <4 x i32> @llvm.umin.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call <8 x i32> @llvm.umin.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = call <16 x i32> @llvm.umin.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.umin.i16(i16 undef, i16 undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call <8 x i16> @llvm.umin.v8i16(<8 x i16> undef, <8 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = call <16 x i16> @llvm.umin.v16i16(<16 x i16> undef, <16 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call <32 x i16> @llvm.umin.v32i16(<32 x i16> undef, <32 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.umin.v8i16(<8 x i16> undef, <8 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.umin.v16i16(<16 x i16> undef, <16 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.umin.v32i16(<32 x i16> undef, <32 x i16> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.umin.i8(i8 undef, i8 undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call <16 x i8> @llvm.umin.v16i8(<16 x i8> undef, <16 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = call <32 x i8> @llvm.umin.v32i8(<32 x i8> undef, <32 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = call <64 x i8> @llvm.umin.v64i8(<64 x i8> undef, <64 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.umin.v16i8(<16 x i8> undef, <16 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.umin.v32i8(<32 x i8> undef, <32 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.umin.v64i8(<64 x i8> undef, <64 x i8> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX1-LABEL: 'umin'
@@ -288,17 +288,17 @@ define i32 @umin(i32 %arg) {
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4I64 = call <4 x i64> @llvm.umin.v4i64(<4 x i64> undef, <4 x i64> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8I64 = call <8 x i64> @llvm.umin.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.umin.i32(i32 undef, i32 undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.umin.v4i32(<4 x i32> undef, <4 x i32> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8I32 = call <8 x i32> @llvm.umin.v8i32(<8 x i32> undef, <8 x i32> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V16I32 = call <16 x i32> @llvm.umin.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <4 x i32> @llvm.umin.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = call <8 x i32> @llvm.umin.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = call <16 x i32> @llvm.umin.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.umin.i16(i16 undef, i16 undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call <8 x i16> @llvm.umin.v8i16(<8 x i16> undef, <8 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16I16 = call <16 x i16> @llvm.umin.v16i16(<16 x i16> undef, <16 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V32I16 = call <32 x i16> @llvm.umin.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.umin.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = call <16 x i16> @llvm.umin.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call <32 x i16> @llvm.umin.v32i16(<32 x i16> undef, <32 x i16> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.umin.i8(i8 undef, i8 undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call <16 x i8> @llvm.umin.v16i8(<16 x i8> undef, <16 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V32I8 = call <32 x i8> @llvm.umin.v32i8(<32 x i8> undef, <32 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V64I8 = call <64 x i8> @llvm.umin.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.umin.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = call <32 x i8> @llvm.umin.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = call <64 x i8> @llvm.umin.v64i8(<64 x i8> undef, <64 x i8> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX2-LABEL: 'umin'
@@ -307,74 +307,74 @@ define i32 @umin(i32 %arg) {
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = call <4 x i64> @llvm.umin.v4i64(<4 x i64> undef, <4 x i64> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I64 = call <8 x i64> @llvm.umin.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.umin.i32(i32 undef, i32 undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.umin.v4i32(<4 x i32> undef, <4 x i32> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call <8 x i32> @llvm.umin.v8i32(<8 x i32> undef, <8 x i32> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = call <16 x i32> @llvm.umin.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <4 x i32> @llvm.umin.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <8 x i32> @llvm.umin.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = call <16 x i32> @llvm.umin.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.umin.i16(i16 undef, i16 undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call <8 x i16> @llvm.umin.v8i16(<8 x i16> undef, <8 x i16> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.umin.v16i16(<16 x i16> undef, <16 x i16> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.umin.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.umin.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.umin.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = call <32 x i16> @llvm.umin.v32i16(<32 x i16> undef, <32 x i16> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.umin.i8(i8 undef, i8 undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call <16 x i8> @llvm.umin.v16i8(<16 x i8> undef, <16 x i8> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.umin.v32i8(<32 x i8> undef, <32 x i8> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.umin.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.umin.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = call <32 x i8> @llvm.umin.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = call <64 x i8> @llvm.umin.v64i8(<64 x i8> undef, <64 x i8> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512F-LABEL: 'umin'
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.umin.i64(i64 undef, i64 undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = call <2 x i64> @llvm.umin.v2i64(<2 x i64> undef, <2 x i64> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = call <4 x i64> @llvm.umin.v4i64(<4 x i64> undef, <4 x i64> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = call <8 x i64> @llvm.umin.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.umin.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.umin.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.umin.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.umin.i32(i32 undef, i32 undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.umin.v4i32(<4 x i32> undef, <4 x i32> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call <8 x i32> @llvm.umin.v8i32(<8 x i32> undef, <8 x i32> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = call <16 x i32> @llvm.umin.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <4 x i32> @llvm.umin.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <8 x i32> @llvm.umin.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I32 = call <16 x i32> @llvm.umin.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.umin.i16(i16 undef, i16 undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call <8 x i16> @llvm.umin.v8i16(<8 x i16> undef, <8 x i16> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.umin.v16i16(<16 x i16> undef, <16 x i16> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.umin.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.umin.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.umin.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = call <32 x i16> @llvm.umin.v32i16(<32 x i16> undef, <32 x i16> undef)
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.umin.i8(i8 undef, i8 undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call <16 x i8> @llvm.umin.v16i8(<16 x i8> undef, <16 x i8> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.umin.v32i8(<32 x i8> undef, <32 x i8> undef)
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.umin.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.umin.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = call <32 x i8> @llvm.umin.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = call <64 x i8> @llvm.umin.v64i8(<64 x i8> undef, <64 x i8> undef)
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512DQ-LABEL: 'umin'
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.umin.i64(i64 undef, i64 undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = call <2 x i64> @llvm.umin.v2i64(<2 x i64> undef, <2 x i64> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = call <4 x i64> @llvm.umin.v4i64(<4 x i64> undef, <4 x i64> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = call <8 x i64> @llvm.umin.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.umin.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.umin.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.umin.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.umin.i32(i32 undef, i32 undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.umin.v4i32(<4 x i32> undef, <4 x i32> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call <8 x i32> @llvm.umin.v8i32(<8 x i32> undef, <8 x i32> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = call <16 x i32> @llvm.umin.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <4 x i32> @llvm.umin.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <8 x i32> @llvm.umin.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I32 = call <16 x i32> @llvm.umin.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.umin.i16(i16 undef, i16 undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call <8 x i16> @llvm.umin.v8i16(<8 x i16> undef, <8 x i16> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.umin.v16i16(<16 x i16> undef, <16 x i16> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.umin.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.umin.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.umin.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = call <32 x i16> @llvm.umin.v32i16(<32 x i16> undef, <32 x i16> undef)
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.umin.i8(i8 undef, i8 undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call <16 x i8> @llvm.umin.v16i8(<16 x i8> undef, <16 x i8> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.umin.v32i8(<32 x i8> undef, <32 x i8> undef)
-; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.umin.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.umin.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = call <32 x i8> @llvm.umin.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = call <64 x i8> @llvm.umin.v64i8(<64 x i8> undef, <64 x i8> undef)
 ; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512BW-LABEL: 'umin'
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.umin.i64(i64 undef, i64 undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = call <2 x i64> @llvm.umin.v2i64(<2 x i64> undef, <2 x i64> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = call <4 x i64> @llvm.umin.v4i64(<4 x i64> undef, <4 x i64> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = call <8 x i64> @llvm.umin.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.umin.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.umin.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.umin.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.umin.i32(i32 undef, i32 undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.umin.v4i32(<4 x i32> undef, <4 x i32> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call <8 x i32> @llvm.umin.v8i32(<8 x i32> undef, <8 x i32> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = call <16 x i32> @llvm.umin.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <4 x i32> @llvm.umin.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <8 x i32> @llvm.umin.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I32 = call <16 x i32> @llvm.umin.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.umin.i16(i16 undef, i16 undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call <8 x i16> @llvm.umin.v8i16(<8 x i16> undef, <8 x i16> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.umin.v16i16(<16 x i16> undef, <16 x i16> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = call <32 x i16> @llvm.umin.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.umin.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.umin.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.umin.v32i16(<32 x i16> undef, <32 x i16> undef)
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.umin.i8(i8 undef, i8 undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call <16 x i8> @llvm.umin.v16i8(<16 x i8> undef, <16 x i8> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.umin.v32i8(<32 x i8> undef, <32 x i8> undef)
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = call <64 x i8> @llvm.umin.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.umin.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = call <32 x i8> @llvm.umin.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64I8 = call <64 x i8> @llvm.umin.v64i8(<64 x i8> undef, <64 x i8> undef)
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %I64 = call i64 @llvm.umin.i64(i64 undef, i64 undef)

From d5776f250fbc80972b98bf1046c4a5d9b6d84e67 Mon Sep 17 00:00:00 2001
From: Chris Bowler <cebowleratibm@gmail.com>
Date: Wed, 29 Jul 2020 09:52:34 -0400
Subject: [PATCH 0470/1035] [NFC][PPC][AIX] Add test coverage for _Complex
 return values

Differential Revision: https://reviews.llvm.org/D84069
---
 test/CodeGen/PowerPC/aix-complex.ll | 125 ++++++++++++++++++++++++++++
 1 file changed, 125 insertions(+)
 create mode 100644 test/CodeGen/PowerPC/aix-complex.ll

diff --git a/test/CodeGen/PowerPC/aix-complex.ll b/test/CodeGen/PowerPC/aix-complex.ll
new file mode 100644
index 0000000000000..9723614f68994
--- /dev/null
+++ b/test/CodeGen/PowerPC/aix-complex.ll
@@ -0,0 +1,125 @@
+; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec -mtriple powerpc-ibm-aix-xcoff < %s | \
+; RUN:   FileCheck --check-prefixes=CHECK,32BIT %s
+; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec -mtriple powerpc64-ibm-aix-xcoff < %s | \
+; RUN:   FileCheck --check-prefixes=CHECK,64BIT %s
+
+@gcd = external global { double, double }, align 8
+@gcf = external global { float, float }, align 4
+@gcfp128 = external global { ppc_fp128, ppc_fp128 }, align 16
+
+declare void @anchor(...)
+
+define dso_local { double, double } @dblCmplxRetCallee()  {
+entry:
+  %retval = alloca { double, double }, align 8
+  %retval.realp = getelementptr inbounds { double, double }, { double, double }* %retval, i32 0, i32 0
+  store double 1.000000e+00, double* %retval.realp, align 8
+  %retval.imagp = getelementptr inbounds { double, double }, { double, double }* %retval, i32 0, i32 1
+  store double 0.000000e+00, double* %retval.imagp, align 8
+  %0 = load { double, double }, { double, double }* %retval, align 8
+  ret { double, double } %0
+}
+
+; CHECK-LABEL: .dblCmplxRetCallee:
+
+; CHECK-DAG:   lfs 1,
+; CHECK-DAG:   lfs 2,
+; CHECK:       blr
+
+define dso_local void @dblCmplxRetCaller()  {
+entry:
+  %call = call { double, double } @dblCmplxRetCallee()
+  %0 = extractvalue { double, double } %call, 0
+  %1 = extractvalue { double, double } %call, 1
+  store double %0, double* getelementptr inbounds ({ double, double }, { double, double }* @gcd, i32 0, i32 0), align 8
+  store double %1, double* getelementptr inbounds ({ double, double }, { double, double }* @gcd, i32 0, i32 1), align 8
+  call void bitcast (void (...)* @anchor to void ()*)()
+  ret void
+}
+
+; CHECK-LABEL: .dblCmplxRetCaller:
+
+; CHECK:        bl .dblCmplxRetCallee
+; 32BIT-NEXT:   lwz [[REG:[0-9]+]], L..C{{[0-9]+}}(2)
+; 64BIT-NEXT:   ld [[REG:[0-9]+]], L..C{{[0-9]+}}(2)
+; CHECK-DAG:    stfd 1, 0([[REG]])
+; CHECK-DAG:    stfd 2, 8([[REG]])
+; CHECK-NEXT:   bl .anchor
+
+define dso_local { float, float } @fltCmplxRetCallee()  {
+entry:
+  %retval = alloca { float, float }, align 4
+  %retval.realp = getelementptr inbounds { float, float }, { float, float }* %retval, i32 0, i32 0
+  %retval.imagp = getelementptr inbounds { float, float }, { float, float }* %retval, i32 0, i32 1
+  store float 1.000000e+00, float* %retval.realp, align 4
+  store float 0.000000e+00, float* %retval.imagp, align 4
+  %0 = load { float, float }, { float, float }* %retval, align 4
+  ret { float, float } %0
+}
+
+; CHECK-LABEL: .fltCmplxRetCallee:
+
+; CHECK-DAG:   lfs 1,
+; CHECK-DAG:   lfs 2,
+; CHECK:       blr
+
+define dso_local void @fltCmplxRetCaller()  {
+entry:
+  %call = call { float, float } @fltCmplxRetCallee()
+  %0 = extractvalue { float, float } %call, 0
+  %1 = extractvalue { float, float } %call, 1
+  store float %0, float* getelementptr inbounds ({ float, float }, { float, float }* @gcf, i32 0, i32 0), align 4
+  store float %1, float* getelementptr inbounds ({ float, float }, { float, float }* @gcf, i32 0, i32 1), align 4
+  call void bitcast (void (...)* @anchor to void ()*)()
+  ret void
+}
+
+; CHECK-LABEL: .fltCmplxRetCaller:
+
+; CHECK:        bl .fltCmplxRetCallee
+; 32BIT-NEXT:   lwz [[REG:[0-9]+]], L..C{{[0-9]+}}(2)
+; 64BIT-NEXT:   ld [[REG:[0-9]+]], L..C{{[0-9]+}}(2)
+; CHECK-DAG:    stfs 1, 0([[REG]])
+; CHECK-DAG:    stfs 2, 4([[REG]])
+; CHECK-NEXT:   bl .anchor
+
+define dso_local { ppc_fp128, ppc_fp128 } @fp128CmplxRetCallee()  {
+entry:
+  %retval = alloca { ppc_fp128, ppc_fp128 }, align 16
+  %retval.realp = getelementptr inbounds { ppc_fp128, ppc_fp128 }, { ppc_fp128, ppc_fp128 }* %retval, i32 0, i32 0
+  %retval.imagp = getelementptr inbounds { ppc_fp128, ppc_fp128 }, { ppc_fp128, ppc_fp128 }* %retval, i32 0, i32 1
+  store ppc_fp128 0xM7ffeffffffffffffffffffffffffffff, ppc_fp128* %retval.realp, align 16
+  store ppc_fp128 0xM3ffefffffffffffffffffffffffffffe, ppc_fp128* %retval.imagp, align 16
+  %0 = load { ppc_fp128, ppc_fp128 }, { ppc_fp128, ppc_fp128 }* %retval, align 16
+  ret { ppc_fp128, ppc_fp128 } %0
+}
+
+; CHECK-LABEL: .fp128CmplxRetCallee:
+
+; CHECK-DAG:  lfd 1,
+; CHECK-DAG:  lfd 2,
+; CHECK-DAG:  lfd 3,
+; CHECK-DAG:  lfd 4,
+; CHECK:      blr
+
+define dso_local void @fp128CmplxRetCaller()  {
+entry:
+  %call = call { ppc_fp128, ppc_fp128 } @fp128CmplxRetCallee()
+  %0 = extractvalue { ppc_fp128, ppc_fp128 } %call, 0
+  %1 = extractvalue { ppc_fp128, ppc_fp128 } %call, 1
+  store ppc_fp128 %0, ppc_fp128* getelementptr inbounds ({ ppc_fp128, ppc_fp128 }, { ppc_fp128, ppc_fp128 }* @gcfp128, i32 0, i32 0), align 16
+  store ppc_fp128 %1, ppc_fp128* getelementptr inbounds ({ ppc_fp128, ppc_fp128 }, { ppc_fp128, ppc_fp128 }* @gcfp128, i32 0, i32 1), align 16
+  call void bitcast (void (...)* @anchor to void ()*)()
+  ret void
+}
+
+; CHECK-LABEL: .fp128CmplxRetCaller:
+
+; CHECK:        bl .fp128CmplxRetCallee
+; 32BIT-NEXT:   lwz [[REG:[0-9]+]], L..C{{[0-9]+}}(2)
+; 64BIT-NEXT:   ld [[REG:[0-9]+]], L..C{{[0-9]+}}(2)
+; CHECK-DAG:    stfd 1, 0([[REG]])
+; CHECK-DAG:    stfd 2, 8([[REG]])
+; CHECK-DAG:    stfd 3, 16([[REG]])
+; CHECK-DAG:    stfd 4, 24([[REG]])
+; CHECK-NEXT:   bl .anchor

From 9f958958334f380c675b832e8207304dfe0a874f Mon Sep 17 00:00:00 2001
From: Sanjay Patel <spatel@rotateright.com>
Date: Wed, 29 Jul 2020 09:44:51 -0400
Subject: [PATCH 0471/1035] [ConstantFolding] add tests for integer min/max
 intrinsics; NFC

---
 llvm/test/Analysis/ConstantFolding/min-max.ll | 86 ++++++++++++++++++-
 1 file changed, 85 insertions(+), 1 deletion(-)

diff --git a/llvm/test/Analysis/ConstantFolding/min-max.ll b/llvm/test/Analysis/ConstantFolding/min-max.ll
index 5ba95c3aeb5ca..26952926ccc96 100644
--- a/llvm/test/Analysis/ConstantFolding/min-max.ll
+++ b/llvm/test/Analysis/ConstantFolding/min-max.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -instcombine -S -o - %s | FileCheck %s
+; RUN: opt -constprop -S < %s | FileCheck %s
 
 declare float @llvm.minnum.f32(float, float)
 declare <4 x float> @llvm.minnum.v4f32(<4 x float>, <4 x float>)
@@ -13,6 +13,18 @@ declare <4 x float> @llvm.minimum.v4f32(<4 x float>, <4 x float>)
 declare float @llvm.maximum.f32(float, float)
 declare <4 x float> @llvm.maximum.v4f32(<4 x float>, <4 x float>)
 
+declare i8 @llvm.smax.i8(i8, i8)
+declare <5 x i8> @llvm.smax.v5i8(<5 x i8>, <5 x i8>)
+
+declare i8 @llvm.smin.i8(i8, i8)
+declare <5 x i8> @llvm.smin.v5i8(<5 x i8>, <5 x i8>)
+
+declare i8 @llvm.umax.i8(i8, i8)
+declare <5 x i8> @llvm.umax.v5i8(<5 x i8>, <5 x i8>)
+
+declare i8 @llvm.umin.i8(i8, i8)
+declare <5 x i8> @llvm.umin.v5i8(<5 x i8>, <5 x i8>)
+
 define float @minnum_float() {
 ; CHECK-LABEL: @minnum_float(
 ; CHECK-NEXT:    ret float 5.000000e+00
@@ -124,3 +136,75 @@ define <4 x float> @maximum_float_zeros_vec() {
   %1 = call <4 x float> @llvm.maximum.v4f32(<4 x float> <float 0.0, float -0.0, float 0.0, float -0.0>, <4 x float> <float 0.0, float 0.0, float -0.0, float -0.0>)
   ret <4 x float> %1
 }
+
+define i8 @smax() {
+; CHECK-LABEL: @smax(
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.smax.i8(i8 -128, i8 -127)
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %r = call i8 @llvm.smax.i8(i8 128, i8 129)
+  ret i8 %r
+}
+
+define <5 x i8> @smax_vec() {
+; CHECK-LABEL: @smax_vec(
+; CHECK-NEXT:    [[R:%.*]] = call <5 x i8> @llvm.smax.v5i8(<5 x i8> <i8 undef, i8 undef, i8 1, i8 42, i8 42>, <5 x i8> <i8 undef, i8 1, i8 undef, i8 42, i8 127>)
+; CHECK-NEXT:    ret <5 x i8> [[R]]
+;
+  %r = call <5 x i8> @llvm.smax.v5i8(<5 x i8> <i8 undef, i8 undef, i8 1, i8 42, i8 42>, <5 x i8> <i8 undef, i8 1, i8 undef, i8 42, i8 127>)
+  ret <5 x i8> %r
+}
+
+define i8 @smin() {
+; CHECK-LABEL: @smin(
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.smin.i8(i8 -128, i8 127)
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %r = call i8 @llvm.smin.i8(i8 128, i8 127)
+  ret i8 %r
+}
+
+define <5 x i8> @smin_vec() {
+; CHECK-LABEL: @smin_vec(
+; CHECK-NEXT:    [[R:%.*]] = call <5 x i8> @llvm.smin.v5i8(<5 x i8> <i8 undef, i8 undef, i8 1, i8 42, i8 42>, <5 x i8> <i8 undef, i8 1, i8 undef, i8 42, i8 -127>)
+; CHECK-NEXT:    ret <5 x i8> [[R]]
+;
+  %r = call <5 x i8> @llvm.smin.v5i8(<5 x i8> <i8 undef, i8 undef, i8 1, i8 42, i8 42>, <5 x i8> <i8 undef, i8 1, i8 undef, i8 42, i8 129>)
+  ret <5 x i8> %r
+}
+
+define i8 @umax() {
+; CHECK-LABEL: @umax(
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.umax.i8(i8 -128, i8 127)
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %r = call i8 @llvm.umax.i8(i8 128, i8 127)
+  ret i8 %r
+}
+
+define <5 x i8> @umax_vec() {
+; CHECK-LABEL: @umax_vec(
+; CHECK-NEXT:    [[R:%.*]] = call <5 x i8> @llvm.umax.v5i8(<5 x i8> <i8 undef, i8 undef, i8 1, i8 42, i8 42>, <5 x i8> <i8 undef, i8 1, i8 undef, i8 42, i8 -128>)
+; CHECK-NEXT:    ret <5 x i8> [[R]]
+;
+  %r = call <5 x i8> @llvm.umax.v5i8(<5 x i8> <i8 undef, i8 undef, i8 1, i8 42, i8 42>, <5 x i8> <i8 undef, i8 1, i8 undef, i8 42, i8 128>)
+  ret <5 x i8> %r
+}
+
+define i8 @umin() {
+; CHECK-LABEL: @umin(
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.umin.i8(i8 -128, i8 127)
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %r = call i8 @llvm.umin.i8(i8 128, i8 127)
+  ret i8 %r
+}
+
+define <5 x i8> @umin_vec() {
+; CHECK-LABEL: @umin_vec(
+; CHECK-NEXT:    [[R:%.*]] = call <5 x i8> @llvm.umin.v5i8(<5 x i8> <i8 undef, i8 undef, i8 1, i8 42, i8 42>, <5 x i8> <i8 undef, i8 1, i8 undef, i8 42, i8 -128>)
+; CHECK-NEXT:    ret <5 x i8> [[R]]
+;
+  %r = call <5 x i8> @llvm.umin.v5i8(<5 x i8> <i8 undef, i8 undef, i8 1, i8 42, i8 42>, <5 x i8> <i8 undef, i8 1, i8 undef, i8 42, i8 128>)
+  ret <5 x i8> %r
+}

From 9ee7d7122c067d929667e65cc11f600e4592cd8a Mon Sep 17 00:00:00 2001
From: Sanjay Patel <spatel@rotateright.com>
Date: Wed, 29 Jul 2020 10:54:47 -0400
Subject: [PATCH 0472/1035] [ConstantFolding] fold integer min/max intrinsics

If both operands are undef, return undef.
If one operand is undef, clamp to limit constant.
---
 llvm/lib/Analysis/ConstantFolding.cpp         | 33 +++++++++++++++++++
 llvm/test/Analysis/ConstantFolding/min-max.ll | 24 +++++---------
 2 files changed, 41 insertions(+), 16 deletions(-)

diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp
index 7e4293d72a0e5..a7458ffe4572c 100644
--- a/llvm/lib/Analysis/ConstantFolding.cpp
+++ b/llvm/lib/Analysis/ConstantFolding.cpp
@@ -1436,6 +1436,10 @@ bool llvm::canConstantFoldCallTo(const CallBase *Call, const Function *F) {
   case Intrinsic::launder_invariant_group:
   case Intrinsic::strip_invariant_group:
   case Intrinsic::masked_load:
+  case Intrinsic::smax:
+  case Intrinsic::smin:
+  case Intrinsic::umax:
+  case Intrinsic::umin:
   case Intrinsic::sadd_with_overflow:
   case Intrinsic::uadd_with_overflow:
   case Intrinsic::ssub_with_overflow:
@@ -2386,8 +2390,37 @@ static Constant *ConstantFoldScalarCall2(StringRef Name,
         !getConstIntOrUndef(Operands[1], C1))
       return nullptr;
 
+    unsigned BitWidth = Ty->getScalarSizeInBits();
     switch (IntrinsicID) {
     default: break;
+    case Intrinsic::smax:
+      if (!C0 && !C1)
+        return UndefValue::get(Ty);
+      if (!C0 || !C1)
+        return ConstantInt::get(Ty, APInt::getSignedMaxValue(BitWidth));
+      return ConstantInt::get(Ty, C0->sgt(*C1) ? *C0 : *C1);
+
+    case Intrinsic::smin:
+      if (!C0 && !C1)
+        return UndefValue::get(Ty);
+      if (!C0 || !C1)
+        return ConstantInt::get(Ty, APInt::getSignedMinValue(BitWidth));
+      return ConstantInt::get(Ty, C0->slt(*C1) ? *C0 : *C1);
+
+    case Intrinsic::umax:
+      if (!C0 && !C1)
+        return UndefValue::get(Ty);
+      if (!C0 || !C1)
+        return ConstantInt::get(Ty, APInt::getMaxValue(BitWidth));
+      return ConstantInt::get(Ty, C0->ugt(*C1) ? *C0 : *C1);
+
+    case Intrinsic::umin:
+      if (!C0 && !C1)
+        return UndefValue::get(Ty);
+      if (!C0 || !C1)
+        return ConstantInt::get(Ty, APInt::getMinValue(BitWidth));
+      return ConstantInt::get(Ty, C0->ult(*C1) ? *C0 : *C1);
+
     case Intrinsic::usub_with_overflow:
     case Intrinsic::ssub_with_overflow:
     case Intrinsic::uadd_with_overflow:
diff --git a/llvm/test/Analysis/ConstantFolding/min-max.ll b/llvm/test/Analysis/ConstantFolding/min-max.ll
index 26952926ccc96..c01fd23683d1b 100644
--- a/llvm/test/Analysis/ConstantFolding/min-max.ll
+++ b/llvm/test/Analysis/ConstantFolding/min-max.ll
@@ -139,8 +139,7 @@ define <4 x float> @maximum_float_zeros_vec() {
 
 define i8 @smax() {
 ; CHECK-LABEL: @smax(
-; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.smax.i8(i8 -128, i8 -127)
-; CHECK-NEXT:    ret i8 [[R]]
+; CHECK-NEXT:    ret i8 -127
 ;
   %r = call i8 @llvm.smax.i8(i8 128, i8 129)
   ret i8 %r
@@ -148,8 +147,7 @@ define i8 @smax() {
 
 define <5 x i8> @smax_vec() {
 ; CHECK-LABEL: @smax_vec(
-; CHECK-NEXT:    [[R:%.*]] = call <5 x i8> @llvm.smax.v5i8(<5 x i8> <i8 undef, i8 undef, i8 1, i8 42, i8 42>, <5 x i8> <i8 undef, i8 1, i8 undef, i8 42, i8 127>)
-; CHECK-NEXT:    ret <5 x i8> [[R]]
+; CHECK-NEXT:    ret <5 x i8> <i8 undef, i8 127, i8 127, i8 42, i8 127>
 ;
   %r = call <5 x i8> @llvm.smax.v5i8(<5 x i8> <i8 undef, i8 undef, i8 1, i8 42, i8 42>, <5 x i8> <i8 undef, i8 1, i8 undef, i8 42, i8 127>)
   ret <5 x i8> %r
@@ -157,8 +155,7 @@ define <5 x i8> @smax_vec() {
 
 define i8 @smin() {
 ; CHECK-LABEL: @smin(
-; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.smin.i8(i8 -128, i8 127)
-; CHECK-NEXT:    ret i8 [[R]]
+; CHECK-NEXT:    ret i8 -128
 ;
   %r = call i8 @llvm.smin.i8(i8 128, i8 127)
   ret i8 %r
@@ -166,8 +163,7 @@ define i8 @smin() {
 
 define <5 x i8> @smin_vec() {
 ; CHECK-LABEL: @smin_vec(
-; CHECK-NEXT:    [[R:%.*]] = call <5 x i8> @llvm.smin.v5i8(<5 x i8> <i8 undef, i8 undef, i8 1, i8 42, i8 42>, <5 x i8> <i8 undef, i8 1, i8 undef, i8 42, i8 -127>)
-; CHECK-NEXT:    ret <5 x i8> [[R]]
+; CHECK-NEXT:    ret <5 x i8> <i8 undef, i8 -128, i8 -128, i8 42, i8 -127>
 ;
   %r = call <5 x i8> @llvm.smin.v5i8(<5 x i8> <i8 undef, i8 undef, i8 1, i8 42, i8 42>, <5 x i8> <i8 undef, i8 1, i8 undef, i8 42, i8 129>)
   ret <5 x i8> %r
@@ -175,8 +171,7 @@ define <5 x i8> @smin_vec() {
 
 define i8 @umax() {
 ; CHECK-LABEL: @umax(
-; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.umax.i8(i8 -128, i8 127)
-; CHECK-NEXT:    ret i8 [[R]]
+; CHECK-NEXT:    ret i8 -128
 ;
   %r = call i8 @llvm.umax.i8(i8 128, i8 127)
   ret i8 %r
@@ -184,8 +179,7 @@ define i8 @umax() {
 
 define <5 x i8> @umax_vec() {
 ; CHECK-LABEL: @umax_vec(
-; CHECK-NEXT:    [[R:%.*]] = call <5 x i8> @llvm.umax.v5i8(<5 x i8> <i8 undef, i8 undef, i8 1, i8 42, i8 42>, <5 x i8> <i8 undef, i8 1, i8 undef, i8 42, i8 -128>)
-; CHECK-NEXT:    ret <5 x i8> [[R]]
+; CHECK-NEXT:    ret <5 x i8> <i8 undef, i8 -1, i8 -1, i8 42, i8 -128>
 ;
   %r = call <5 x i8> @llvm.umax.v5i8(<5 x i8> <i8 undef, i8 undef, i8 1, i8 42, i8 42>, <5 x i8> <i8 undef, i8 1, i8 undef, i8 42, i8 128>)
   ret <5 x i8> %r
@@ -193,8 +187,7 @@ define <5 x i8> @umax_vec() {
 
 define i8 @umin() {
 ; CHECK-LABEL: @umin(
-; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.umin.i8(i8 -128, i8 127)
-; CHECK-NEXT:    ret i8 [[R]]
+; CHECK-NEXT:    ret i8 127
 ;
   %r = call i8 @llvm.umin.i8(i8 128, i8 127)
   ret i8 %r
@@ -202,8 +195,7 @@ define i8 @umin() {
 
 define <5 x i8> @umin_vec() {
 ; CHECK-LABEL: @umin_vec(
-; CHECK-NEXT:    [[R:%.*]] = call <5 x i8> @llvm.umin.v5i8(<5 x i8> <i8 undef, i8 undef, i8 1, i8 42, i8 42>, <5 x i8> <i8 undef, i8 1, i8 undef, i8 42, i8 -128>)
-; CHECK-NEXT:    ret <5 x i8> [[R]]
+; CHECK-NEXT:    ret <5 x i8> <i8 undef, i8 0, i8 0, i8 42, i8 42>
 ;
   %r = call <5 x i8> @llvm.umin.v5i8(<5 x i8> <i8 undef, i8 undef, i8 1, i8 42, i8 42>, <5 x i8> <i8 undef, i8 1, i8 undef, i8 42, i8 128>)
   ret <5 x i8> %r

From bfa140376df89eea4e984ad983b1809f60187b7e Mon Sep 17 00:00:00 2001
From: Xing GUO <higuoxing@gmail.com>
Date: Wed, 29 Jul 2020 23:05:47 +0800
Subject: [PATCH 0473/1035] [DWARFYAML] Make the field names consistent with
 the DWARF spec. NFC.

This patch replaces 'AddrSize'/'SegSize' with
'AddressSize'/'SegmentSelectorSize'. NFC.
---
 llvm/lib/ObjectYAML/DWARFYAML.cpp             |  4 +-
 .../ObjectYAML/MachO/DWARF-debug_aranges.yaml | 20 ++---
 .../ObjectYAML/MachO/DWARF-debug_info.yaml    | 10 +--
 .../ObjectYAML/MachO/DWARF-debug_line.yaml    | 10 +--
 .../ObjectYAML/MachO/DWARF5-debug_info.yaml   | 10 +--
 .../ARM_AArch64/fat-macho-dwarf.yaml          | 20 ++---
 .../tools/llvm-gsymutil/X86/mach-dwarf.yaml   | 10 +--
 .../MachO/DWARF-debug_aranges-error.yaml      | 10 +--
 .../yaml2obj/ELF/DWARF/debug-aranges.yaml     | 76 +++++++++----------
 9 files changed, 85 insertions(+), 85 deletions(-)

diff --git a/llvm/lib/ObjectYAML/DWARFYAML.cpp b/llvm/lib/ObjectYAML/DWARFYAML.cpp
index adc167249226b..fda1ba797a916 100644
--- a/llvm/lib/ObjectYAML/DWARFYAML.cpp
+++ b/llvm/lib/ObjectYAML/DWARFYAML.cpp
@@ -106,8 +106,8 @@ void MappingTraits<DWARFYAML::ARange>::mapping(IO &IO,
   IO.mapRequired("Length", ARange.Length);
   IO.mapRequired("Version", ARange.Version);
   IO.mapRequired("CuOffset", ARange.CuOffset);
-  IO.mapRequired("AddrSize", ARange.AddrSize);
-  IO.mapRequired("SegSize", ARange.SegSize);
+  IO.mapRequired("AddressSize", ARange.AddrSize);
+  IO.mapRequired("SegmentSelectorSize", ARange.SegSize);
   IO.mapRequired("Descriptors", ARange.Descriptors);
 }
 
diff --git a/llvm/test/ObjectYAML/MachO/DWARF-debug_aranges.yaml b/llvm/test/ObjectYAML/MachO/DWARF-debug_aranges.yaml
index 2953317518795..868054b08a991 100644
--- a/llvm/test/ObjectYAML/MachO/DWARF-debug_aranges.yaml
+++ b/llvm/test/ObjectYAML/MachO/DWARF-debug_aranges.yaml
@@ -313,22 +313,22 @@ LinkEditData:
     - _main
 DWARF:           
   debug_aranges:   
-    - Length:          44
-      Version:         2
-      CuOffset:        0
-      AddrSize:        8
-      SegSize:         0
+    - Length:              44
+      Version:             2
+      CuOffset:            0
+      AddressSize:         8
+      SegmentSelectorSize: 0
       Descriptors:     
         - Address:         0x0000000100000F50
           Length:          52
 ...
 
 #      CHECK:   debug_aranges:
-# CHECK-NEXT:     - Length:   0x000000000000002C
-# CHECK-NEXT:       Version:  2
-# CHECK-NEXT:       CuOffset: 0x0000000000000000
-# CHECK-NEXT:       AddrSize: 0x08
-# CHECK-NEXT:       SegSize:  0x00
+# CHECK-NEXT:     - Length:              0x000000000000002C
+# CHECK-NEXT:       Version:             2
+# CHECK-NEXT:       CuOffset:            0x0000000000000000
+# CHECK-NEXT:       AddressSize:         0x08
+# CHECK-NEXT:       SegmentSelectorSize: 0x00
 # CHECK-NEXT:       Descriptors:
 # CHECK-NEXT:         - Address: 0x0000000100000F50
 # CHECK-NEXT:           Length:  52
diff --git a/llvm/test/ObjectYAML/MachO/DWARF-debug_info.yaml b/llvm/test/ObjectYAML/MachO/DWARF-debug_info.yaml
index 1ca45f5dadfed..937c84b863a67 100644
--- a/llvm/test/ObjectYAML/MachO/DWARF-debug_info.yaml
+++ b/llvm/test/ObjectYAML/MachO/DWARF-debug_info.yaml
@@ -381,11 +381,11 @@ DWARF:
         - Attribute:       DW_AT_type
           Form:            DW_FORM_ref4
   debug_aranges:   
-    - Length:          44
-      Version:         2
-      CuOffset:        0
-      AddrSize:        8
-      SegSize:         0
+    - Length:              44
+      Version:             2
+      CuOffset:            0
+      AddressSize:         8
+      SegmentSelectorSize: 0
       Descriptors:     
         - Address:         0x0000000100000F50
           Length:          52
diff --git a/llvm/test/ObjectYAML/MachO/DWARF-debug_line.yaml b/llvm/test/ObjectYAML/MachO/DWARF-debug_line.yaml
index 6575e95ed2126..5acca5ccb7fc3 100644
--- a/llvm/test/ObjectYAML/MachO/DWARF-debug_line.yaml
+++ b/llvm/test/ObjectYAML/MachO/DWARF-debug_line.yaml
@@ -394,11 +394,11 @@ DWARF:
         - Attribute:       DW_AT_type
           Form:            DW_FORM_ref4
   debug_aranges:   
-    - Length:          44
-      Version:         2
-      CuOffset:        0
-      AddrSize:        8
-      SegSize:         0
+    - Length:              44
+      Version:             2
+      CuOffset:            0
+      AddressSize:         8
+      SegmentSelectorSize: 0
       Descriptors:     
         - Address:         0x0000000100000F50
           Length:          52
diff --git a/llvm/test/ObjectYAML/MachO/DWARF5-debug_info.yaml b/llvm/test/ObjectYAML/MachO/DWARF5-debug_info.yaml
index fc2280066c0c0..c41348477a965 100644
--- a/llvm/test/ObjectYAML/MachO/DWARF5-debug_info.yaml
+++ b/llvm/test/ObjectYAML/MachO/DWARF5-debug_info.yaml
@@ -381,11 +381,11 @@ DWARF:
         - Attribute:       DW_AT_type
           Form:            DW_FORM_ref4
   debug_aranges:   
-    - Length:          44
-      Version:         2
-      CuOffset:        0
-      AddrSize:        8
-      SegSize:         0
+    - Length:              44
+      Version:             2
+      CuOffset:            0
+      AddressSize:         8
+      SegmentSelectorSize: 0
       Descriptors:     
         - Address:         0x0000000100000F50
           Length:          52
diff --git a/llvm/test/tools/llvm-gsymutil/ARM_AArch64/fat-macho-dwarf.yaml b/llvm/test/tools/llvm-gsymutil/ARM_AArch64/fat-macho-dwarf.yaml
index aef062428e8f4..ebb82b5492e6a 100644
--- a/llvm/test/tools/llvm-gsymutil/ARM_AArch64/fat-macho-dwarf.yaml
+++ b/llvm/test/tools/llvm-gsymutil/ARM_AArch64/fat-macho-dwarf.yaml
@@ -378,11 +378,11 @@ Slices:
             - Attribute:       DW_AT_type
               Form:            DW_FORM_ref_addr
       debug_aranges:
-        - Length:          28
-          Version:         2
-          CuOffset:        0
-          AddrSize:        4
-          SegSize:         0
+        - Length:              28
+          Version:             2
+          CuOffset:            0
+          AddressSize:         4
+          SegmentSelectorSize: 0
           Descriptors:
             - Address:         0x000000000000BFF0
               Length:          16
@@ -856,11 +856,11 @@ Slices:
             - Attribute:       DW_AT_type
               Form:            DW_FORM_ref_addr
       debug_aranges:
-        - Length:          44
-          Version:         2
-          CuOffset:        0
-          AddrSize:        8
-          SegSize:         0
+        - Length:              44
+          Version:             2
+          CuOffset:            0
+          AddressSize:         8
+          SegmentSelectorSize: 0
           Descriptors:
             - Address:         0x0000000100007F9C
               Length:          28
diff --git a/llvm/test/tools/llvm-gsymutil/X86/mach-dwarf.yaml b/llvm/test/tools/llvm-gsymutil/X86/mach-dwarf.yaml
index 1368257964166..fa642d1120679 100644
--- a/llvm/test/tools/llvm-gsymutil/X86/mach-dwarf.yaml
+++ b/llvm/test/tools/llvm-gsymutil/X86/mach-dwarf.yaml
@@ -539,11 +539,11 @@ DWARF:
         - Attribute:       DW_AT_type
           Form:            DW_FORM_ref_addr
   debug_aranges:
-    - Length:          60
-      Version:         2
-      CuOffset:        0
-      AddrSize:        8
-      SegSize:         0
+    - Length:              60
+      Version:             2
+      CuOffset:            0
+      AddressSize:         8
+      SegmentSelectorSize: 0
       Descriptors:
         - Address:         0x0000000100000F70
           Length:          17
diff --git a/llvm/test/tools/obj2yaml/MachO/DWARF-debug_aranges-error.yaml b/llvm/test/tools/obj2yaml/MachO/DWARF-debug_aranges-error.yaml
index 8f392bc6b7003..bde5499cf1a75 100644
--- a/llvm/test/tools/obj2yaml/MachO/DWARF-debug_aranges-error.yaml
+++ b/llvm/test/tools/obj2yaml/MachO/DWARF-debug_aranges-error.yaml
@@ -287,11 +287,11 @@ LoadCommands:
         reserved3:       0x00000000
 DWARF:
   debug_aranges:
-    - Length:          45
-      Version:         2
-      CuOffset:        0
-      AddrSize:        8
-      SegSize:         0
+    - Length:              45
+      Version:             2
+      CuOffset:            0
+      AddressSize:         8
+      SegmentSelectorSize: 0
       Descriptors:
         - Address:         0x0000000100000F50
           Length:          52
diff --git a/llvm/test/tools/yaml2obj/ELF/DWARF/debug-aranges.yaml b/llvm/test/tools/yaml2obj/ELF/DWARF/debug-aranges.yaml
index 78d0b15c26a07..b1064a9ae2153 100644
--- a/llvm/test/tools/yaml2obj/ELF/DWARF/debug-aranges.yaml
+++ b/llvm/test/tools/yaml2obj/ELF/DWARF/debug-aranges.yaml
@@ -69,21 +69,21 @@ FileHeader:
   Machine: EM_X86_64
 DWARF:
   debug_aranges:
-    - Format:   [[FORMAT]]
-      Length:   0x2c
-      Version:  2
-      CuOffset: 0
-      AddrSize: 0x04
-      SegSize:  0x00
+    - Format:               [[FORMAT]]
+      Length:               0x2c
+      Version:              2
+      CuOffset:             0
+      AddressSize:          0x04
+      SegmentSelectorSize:  0x00
       Descriptors:
         - Address: 0x00001234
           Length:  0x20
-    - Format:   [[FORMAT]]
-      Length:   0x2c
-      Version:  2
-      CuOffset: 0x65
-      AddrSize: 0x08
-      SegSize:  0x00
+    - Format:              [[FORMAT]]
+      Length:              0x2c
+      Version:             2
+      CuOffset:            0x65
+      AddressSize:         0x08
+      SegmentSelectorSize: 0x00
       Descriptors:
         - Address: 0x0000000000005678
           Length:  0x20
@@ -307,11 +307,11 @@ Sections:
     Size: 0x10
 DWARF:
   debug_aranges:
-    - Length:   0x2c
-      Version:  2
-      CuOffset: 0
-      AddrSize: 0x08
-      SegSize:  0x00
+    - Length:              0x2c
+      Version:             2
+      CuOffset:            0
+      AddressSize:         0x08
+      SegmentSelectorSize: 0x00
       Descriptors:
         - Address: 0x0000000000001234
           Length:  0x20
@@ -333,11 +333,11 @@ Sections:
     Content: "00"
 DWARF:
   debug_aranges:
-    - Length:   0x2c
-      Version:  2
-      CuOffset: 0
-      AddrSize: 0x08
-      SegSize:  0x00
+    - Length:              0x2c
+      Version:             2
+      CuOffset:            0
+      AddressSize:         0x08
+      SegmentSelectorSize: 0x00
       Descriptors:
         - Address: 0x0000000000001234
           Length:  0x20
@@ -398,11 +398,11 @@ Sections:
     Type:         SHT_STRTAB
 DWARF:
   debug_aranges:
-    - Length:   0x2c
-      Version:  2
-      CuOffset: 0
-      AddrSize: 0x08
-      SegSize:  0x00
+    - Length:              0x2c
+      Version:             2
+      CuOffset:            0
+      AddressSize:         0x08
+      SegmentSelectorSize: 0x00
       Descriptors:
         - Address: 0x0000000000001234
           Length:  0x20
@@ -427,12 +427,12 @@ FileHeader:
   Machine: EM_X86_64
 DWARF:
   debug_aranges:
-    - Length:      0x2c
-      Version:     2
-      CuOffset:    0
-      AddrSize:    0x07 ## Invalid address_size.
-      SegSize:     0
-      Descriptors: []
+    - Length:              0x2c
+      Version:             2
+      CuOffset:            0
+      AddressSize:         0x07 ## Invalid address_size.
+      SegmentSelectorSize: 0
+      Descriptors:         []
 
 ## i) Test that yaml2obj emits an error message if we try to assign an invalid value to
 ## 'AddrSize' when the 'Descriptors' list isn't empty.
@@ -450,11 +450,11 @@ FileHeader:
   Machine: EM_X86_64
 DWARF:
   debug_aranges:
-    - Length:      0x2c
-      Version:     2
-      CuOffset:    0
-      AddrSize:    0x07 ## Invalid address_size.
-      SegSize:     0
+    - Length:              0x2c
+      Version:             2
+      CuOffset:            0
+      AddressSize:         0x07 ## Invalid address_size.
+      SegmentSelectorSize: 0
       Descriptors:
         - Address: 0x1234
           Length:  0x1000

From 62beb7c6f4f2288793751740f06edc0dc25d01f6 Mon Sep 17 00:00:00 2001
From: Nathan James <n.james93@hotmail.co.uk>
Date: Wed, 29 Jul 2020 16:19:06 +0100
Subject: [PATCH 0474/1035] [clang-tidy] Fix module options being registered
 with different priorities

Not a bug that is ever likely to materialise, but still worth fixing

Reviewed By: DmitryPolukhin

Differential Revision: https://reviews.llvm.org/D84850
---
 clang-tools-extra/clang-tidy/ClangTidyOptions.cpp | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/clang-tools-extra/clang-tidy/ClangTidyOptions.cpp b/clang-tools-extra/clang-tidy/ClangTidyOptions.cpp
index 4c1f286737817..bb5a4b513967d 100644
--- a/clang-tools-extra/clang-tidy/ClangTidyOptions.cpp
+++ b/clang-tools-extra/clang-tidy/ClangTidyOptions.cpp
@@ -114,11 +114,9 @@ ClangTidyOptions ClangTidyOptions::getDefaults() {
   Options.SystemHeaders = false;
   Options.FormatStyle = "none";
   Options.User = llvm::None;
-  unsigned Priority = 0;
   for (const ClangTidyModuleRegistry::entry &Module :
        ClangTidyModuleRegistry::entries())
-    Options =
-        Options.mergeWith(Module.instantiate()->getModuleOptions(), ++Priority);
+    Options = Options.mergeWith(Module.instantiate()->getModuleOptions(), 0);
   return Options;
 }
 

From 6a05c6bfc8d363a6d3d6cd7015d0e6fc01a91ce2 Mon Sep 17 00:00:00 2001
From: Simon Wallis <simon.wallis2@arm.com>
Date: Wed, 29 Jul 2020 16:17:47 +0100
Subject: [PATCH 0475/1035] [MachineCopyPropagation] BackwardPropagatableCopy:
 add check for hasOverlappingMultipleDef

In MachineCopyPropagation::BackwardPropagatableCopy(),
a check is added for multiple destination registers.

The copy propagation is avoided if the copied destination register
is the same register as another destination on the same instruction.

A new test is added.  This used to fail on ARM like this:
error: unpredictable instruction, RdHi and RdLo must be different
        umull   r9, r9, lr, r0

Reviewed By: lkail

Differential Revision: https://reviews.llvm.org/D82638
---
 llvm/lib/CodeGen/MachineCopyPropagation.cpp   | 20 +++++++++++++++++++
 .../test/CodeGen/ARM/mcp-dest-regs-no-dup.mir | 13 ++++++++++++
 2 files changed, 33 insertions(+)
 create mode 100644 llvm/test/CodeGen/ARM/mcp-dest-regs-no-dup.mir

diff --git a/llvm/lib/CodeGen/MachineCopyPropagation.cpp b/llvm/lib/CodeGen/MachineCopyPropagation.cpp
index 70d6dcc2e3e29..67d15129b9045 100644
--- a/llvm/lib/CodeGen/MachineCopyPropagation.cpp
+++ b/llvm/lib/CodeGen/MachineCopyPropagation.cpp
@@ -288,6 +288,8 @@ class MachineCopyPropagation : public MachineFunctionPass {
                                           const MachineInstr &UseI,
                                           unsigned UseIdx);
   bool hasImplicitOverlap(const MachineInstr &MI, const MachineOperand &Use);
+  bool hasOverlappingMultipleDef(const MachineInstr &MI,
+                                 const MachineOperand &MODef, Register Def);
 
   /// Candidates for deletion.
   SmallSetVector<MachineInstr *, 8> MaybeDeadCopies;
@@ -461,6 +463,21 @@ bool MachineCopyPropagation::hasImplicitOverlap(const MachineInstr &MI,
   return false;
 }
 
+/// For an MI that has multiple definitions, check whether \p MI has
+/// a definition that overlaps with another of its definitions.
+/// For example, on ARM: umull   r9, r9, lr, r0
+/// The umull instruction is unpredictable unless RdHi and RdLo are different.
+bool MachineCopyPropagation::hasOverlappingMultipleDef(
+    const MachineInstr &MI, const MachineOperand &MODef, Register Def) {
+  for (const MachineOperand &MIDef : MI.defs()) {
+    if ((&MIDef != &MODef) && MIDef.isReg() &&
+        TRI->regsOverlap(Def, MIDef.getReg()))
+      return true;
+  }
+
+  return false;
+}
+
 /// Look for available copies whose destination register is used by \p MI and
 /// replace the use in \p MI with the copy's source register.
 void MachineCopyPropagation::forwardUses(MachineInstr &MI) {
@@ -786,6 +803,9 @@ void MachineCopyPropagation::propagateDefs(MachineInstr &MI) {
     if (hasImplicitOverlap(MI, MODef))
       continue;
 
+    if (hasOverlappingMultipleDef(MI, MODef, Def))
+      continue;
+
     LLVM_DEBUG(dbgs() << "MCP: Replacing " << printReg(MODef.getReg(), TRI)
                       << "\n     with " << printReg(Def, TRI) << "\n     in "
                       << MI << "     from " << *Copy);
diff --git a/llvm/test/CodeGen/ARM/mcp-dest-regs-no-dup.mir b/llvm/test/CodeGen/ARM/mcp-dest-regs-no-dup.mir
new file mode 100644
index 0000000000000..c5a8fabfdc79b
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/mcp-dest-regs-no-dup.mir
@@ -0,0 +1,13 @@
+# RUN: llc -mtriple=arm-eabi -O1 -run-pass=machine-cp %s -o - \
+# RUN: -verify-machineinstrs -simplify-mir | FileCheck %s
+
+name:            h
+body:             |
+  bb.0:
+
+    dead renamable $r9, renamable $r0 = UMULL renamable $lr, killed renamable $r0, 14 /* CC::al */, $noreg, $noreg
+
+  ; CHECK: dead renamable $r9, renamable $r0 = UMULL renamable $lr, killed renamable $r0, 14 /* CC::al */, $noreg, $noreg
+
+    renamable $r9 = COPY killed renamable $r0
+...

From 7db017bf3405c7fa43786fe27380d88702e19584 Mon Sep 17 00:00:00 2001
From: Johannes Doerfert <johannes@jdoerfert.de>
Date: Wed, 15 Jul 2020 09:50:46 -0500
Subject: [PATCH 0476/1035] [OpenMP][Docs] Update Clang Support docs after
 D75591

---
 clang/docs/OpenMPSupport.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/docs/OpenMPSupport.rst b/clang/docs/OpenMPSupport.rst
index a1d1b120bcecc..bda52e934c266 100644
--- a/clang/docs/OpenMPSupport.rst
+++ b/clang/docs/OpenMPSupport.rst
@@ -264,7 +264,7 @@ want to help with the implementation.
 +==============================+==============================================================+==========================+=======================================================================+
 | misc extension               | user-defined function variants with #ifdef protection        | :part:`worked on`        | D71179                                                                |
 +------------------------------+--------------------------------------------------------------+--------------------------+-----------------------------------------------------------------------+
-| misc extension               | default(firstprivate) & default(private)                     | :part:`worked on`        |                                                                       |
+| misc extension               | default(firstprivate) & default(private)                     | :part:`partial`          | firstprivate done: D75591                                             |
 +------------------------------+--------------------------------------------------------------+--------------------------+-----------------------------------------------------------------------+
 | loop extension               | Loop tiling transformation                                   | :part:`claimed`          |                                                                       |
 +------------------------------+--------------------------------------------------------------+--------------------------+-----------------------------------------------------------------------+

From ee05167cc42b95f70bc2ff1bd4402969f356f53b Mon Sep 17 00:00:00 2001
From: Johannes Doerfert <johannes@jdoerfert.de>
Date: Tue, 7 Jul 2020 01:08:03 -0500
Subject: [PATCH 0477/1035] [OpenMP] Allow traits for the OpenMP context
 selector `isa`

It was unclear what `isa` was supposed to mean so we did not provide any
traits for this context selector. With this patch we will allow *any*
string or identifier. We use the target attribute and target info to
determine if the trait matches. In other words, we will check if the
provided value is a target feature that is available (at the call site).

Fixes PR46338

Reviewed By: ABataev

Differential Revision: https://reviews.llvm.org/D83281
---
 clang/include/clang/AST/OpenMPClause.h        | 21 +++++
 .../clang/Basic/DiagnosticParseKinds.td       |  5 ++
 .../clang/Basic/DiagnosticSemaKinds.td        |  5 ++
 clang/lib/AST/OpenMPClause.cpp                | 44 ++++++++--
 clang/lib/Parse/ParseOpenMP.cpp               | 40 ++++++----
 clang/lib/Sema/SemaOpenMP.cpp                 | 14 +++-
 .../declare_variant_device_isa_codegen_1.c    | 49 ++++++++++++
 clang/test/OpenMP/declare_variant_messages.c  | 12 +++
 .../include/llvm/Frontend/OpenMP/OMPContext.h | 49 +++++++++---
 .../include/llvm/Frontend/OpenMP/OMPKinds.def |  6 +-
 llvm/lib/Frontend/OpenMP/OMPContext.cpp       | 27 +++++--
 llvm/unittests/Frontend/OpenMPContextTest.cpp | 80 ++++++++++---------
 12 files changed, 270 insertions(+), 82 deletions(-)
 create mode 100644 clang/test/OpenMP/declare_variant_device_isa_codegen_1.c

diff --git a/clang/include/clang/AST/OpenMPClause.h b/clang/include/clang/AST/OpenMPClause.h
index c649502f765b4..4f94aa7074ee7 100644
--- a/clang/include/clang/AST/OpenMPClause.h
+++ b/clang/include/clang/AST/OpenMPClause.h
@@ -7635,6 +7635,10 @@ class OMPClausePrinter final : public OMPClauseVisitor<OMPClausePrinter> {
 
 struct OMPTraitProperty {
   llvm::omp::TraitProperty Kind = llvm::omp::TraitProperty::invalid;
+
+  /// The raw string as we parsed it. This is needed for the `isa` trait set
+  /// (which accepts anything) and (later) extensions.
+  StringRef RawString;
 };
 struct OMPTraitSelector {
   Expr *ScoreOrCondition = nullptr;
@@ -7692,6 +7696,23 @@ class OMPTraitInfo {
 llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const OMPTraitInfo &TI);
 llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const OMPTraitInfo *TI);
 
+/// Clang specific specialization of the OMPContext to lookup target features.
+struct TargetOMPContext final : public llvm::omp::OMPContext {
+
+  TargetOMPContext(ASTContext &ASTCtx,
+                   std::function<void(StringRef)> &&DiagUnknownTrait,
+                   const FunctionDecl *CurrentFunctionDecl);
+  virtual ~TargetOMPContext() = default;
+
+  /// See llvm::omp::OMPContext::matchesISATrait
+  bool matchesISATrait(StringRef RawString) const override;
+
+private:
+  std::function<bool(StringRef)> FeatureValidityCheck;
+  std::function<void(StringRef)> DiagUnknownTrait;
+  llvm::StringMap<bool> FeatureMap;
+};
+
 } // namespace clang
 
 #endif // LLVM_CLANG_AST_OPENMPCLAUSE_H
diff --git a/clang/include/clang/Basic/DiagnosticParseKinds.td b/clang/include/clang/Basic/DiagnosticParseKinds.td
index 6138b27fb87f9..08b91de31993f 100644
--- a/clang/include/clang/Basic/DiagnosticParseKinds.td
+++ b/clang/include/clang/Basic/DiagnosticParseKinds.td
@@ -1278,6 +1278,11 @@ def warn_omp_declare_variant_string_literal_or_identifier
               "%select{set|selector|property}0; "
               "%select{set|selector|property}0 skipped">,
       InGroup<OpenMPClauses>;
+def warn_unknown_begin_declare_variant_isa_trait
+    : Warning<"isa trait '%0' is not known to the current target; verify the "
+              "spelling or consider restricting the context selector with the "
+              "'arch' selector further">,
+      InGroup<SourceUsesOpenMP>;
 def note_omp_declare_variant_ctx_options
     : Note<"context %select{set|selector|property}0 options are: %1">;
 def warn_omp_declare_variant_expected
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 8093e7ed3fbe1..ae693a08108c1 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -10320,6 +10320,11 @@ def warn_nested_declare_variant
     : Warning<"nesting `omp begin/end declare variant` is not supported yet; "
               "nested context ignored">,
       InGroup<SourceUsesOpenMP>;
+def warn_unknown_declare_variant_isa_trait
+    : Warning<"isa trait '%0' is not known to the current target; verify the "
+              "spelling or consider restricting the context selector with the "
+              "'arch' selector further">,
+      InGroup<SourceUsesOpenMP>;
 def err_omp_non_pointer_type_array_shaping_base : Error<
   "expected expression with a pointer to a complete type as a base of an array "
   "shaping operation">;
diff --git a/clang/lib/AST/OpenMPClause.cpp b/clang/lib/AST/OpenMPClause.cpp
index 6933c57425520..9caa691188fdb 100644
--- a/clang/lib/AST/OpenMPClause.cpp
+++ b/clang/lib/AST/OpenMPClause.cpp
@@ -17,6 +17,7 @@
 #include "clang/AST/DeclOpenMP.h"
 #include "clang/Basic/LLVM.h"
 #include "clang/Basic/OpenMPKinds.h"
+#include "clang/Basic/TargetInfo.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -2131,9 +2132,10 @@ void OMPTraitInfo::getAsVariantMatchInfo(ASTContext &ASTCtx,
                 Selector.ScoreOrCondition->getIntegerConstantExpr(ASTCtx))
           VMI.addTrait(CondVal->isNullValue()
                            ? TraitProperty::user_condition_false
-                           : TraitProperty::user_condition_true);
+                           : TraitProperty::user_condition_true,
+                       "<condition>");
         else
-          VMI.addTrait(TraitProperty::user_condition_false);
+          VMI.addTrait(TraitProperty::user_condition_false, "<condition>");
         continue;
       }
 
@@ -2143,11 +2145,12 @@ void OMPTraitInfo::getAsVariantMatchInfo(ASTContext &ASTCtx,
         if ((Score = Selector.ScoreOrCondition->getIntegerConstantExpr(ASTCtx)))
           ScorePtr = &*Score;
         else
-          VMI.addTrait(TraitProperty::user_condition_false);
+          VMI.addTrait(TraitProperty::user_condition_false,
+                       "<non-constant-score>");
       }
 
       for (const OMPTraitProperty &Property : Selector.Properties)
-        VMI.addTrait(Set.Kind, Property.Kind, ScorePtr);
+        VMI.addTrait(Set.Kind, Property.Kind, Property.RawString, ScorePtr);
 
       if (Set.Kind != TraitSet::construct)
         continue;
@@ -2204,7 +2207,8 @@ void OMPTraitInfo::print(llvm::raw_ostream &OS,
           if (!FirstProperty)
             OS << ", ";
           FirstProperty = false;
-          OS << getOpenMPContextTraitPropertyName(Property.Kind);
+          OS << getOpenMPContextTraitPropertyName(Property.Kind,
+                                                  Property.RawString);
         }
       }
       OS << ")";
@@ -2231,7 +2235,9 @@ std::string OMPTraitInfo::getMangledName() const {
         continue;
 
       for (const OMPTraitProperty &Property : Selector.Properties)
-        OS << '$' << 'P' << getOpenMPContextTraitPropertyName(Property.Kind);
+        OS << '$' << 'P'
+           << getOpenMPContextTraitPropertyName(Property.Kind,
+                                                Property.RawString);
     }
   }
   return OS.str();
@@ -2261,8 +2267,9 @@ OMPTraitInfo::OMPTraitInfo(StringRef MangledName) {
         Selector.Properties.push_back(OMPTraitProperty());
         OMPTraitProperty &Property = Selector.Properties.back();
         std::pair<StringRef, StringRef> PropRestPair = MangledName.split('$');
-        Property.Kind =
-            getOpenMPContextTraitPropertyKind(Set.Kind, PropRestPair.first);
+        Property.RawString = PropRestPair.first;
+        Property.Kind = getOpenMPContextTraitPropertyKind(
+            Set.Kind, Selector.Kind, PropRestPair.first);
         MangledName = PropRestPair.second;
       } while (true);
     } while (true);
@@ -2280,3 +2287,24 @@ llvm::raw_ostream &clang::operator<<(llvm::raw_ostream &OS,
                                      const OMPTraitInfo *TI) {
   return TI ? OS << *TI : OS;
 }
+
+TargetOMPContext::TargetOMPContext(
+    ASTContext &ASTCtx, std::function<void(StringRef)> &&DiagUnknownTrait,
+    const FunctionDecl *CurrentFunctionDecl)
+    : OMPContext(ASTCtx.getLangOpts().OpenMPIsDevice,
+                 ASTCtx.getTargetInfo().getTriple()),
+      FeatureValidityCheck([&](StringRef FeatureName) {
+        return ASTCtx.getTargetInfo().isValidFeatureName(FeatureName);
+      }),
+      DiagUnknownTrait(std::move(DiagUnknownTrait)) {
+  ASTCtx.getFunctionFeatureMap(FeatureMap, CurrentFunctionDecl);
+}
+
+bool TargetOMPContext::matchesISATrait(StringRef RawString) const {
+  auto It = FeatureMap.find(RawString);
+  if (It != FeatureMap.end())
+    return It->second;
+  if (!FeatureValidityCheck(RawString))
+    DiagUnknownTrait(RawString);
+  return false;
+}
diff --git a/clang/lib/Parse/ParseOpenMP.cpp b/clang/lib/Parse/ParseOpenMP.cpp
index 6a5d7c604c6bb..ac1366e6b0ef6 100644
--- a/clang/lib/Parse/ParseOpenMP.cpp
+++ b/clang/lib/Parse/ParseOpenMP.cpp
@@ -869,7 +869,8 @@ void Parser::parseOMPTraitPropertyKind(
     return;
   }
 
-  TIProperty.Kind = getOpenMPContextTraitPropertyKind(Set, Name);
+  TIProperty.RawString = Name;
+  TIProperty.Kind = getOpenMPContextTraitPropertyKind(Set, Selector, Name);
   if (TIProperty.Kind != TraitProperty::invalid) {
     if (checkForDuplicates(*this, Name, NameLoc, Seen, CONTEXT_TRAIT_LVL))
       TIProperty.Kind = TraitProperty::invalid;
@@ -910,7 +911,7 @@ void Parser::parseOMPTraitPropertyKind(
        {TraitSet::construct, TraitSet::user, TraitSet::implementation,
         TraitSet::device}) {
     TraitProperty PropertyForName =
-        getOpenMPContextTraitPropertyKind(PotentialSet, Name);
+        getOpenMPContextTraitPropertyKind(PotentialSet, Selector, Name);
     if (PropertyForName == TraitProperty::invalid)
       continue;
     Diag(NameLoc, diag::note_omp_declare_variant_ctx_try)
@@ -949,8 +950,8 @@ static bool checkExtensionProperty(Parser &P, SourceLocation Loc,
     for (OMPTraitProperty &SeenProp : TISelector.Properties)
       if (IsMatchExtension(SeenProp)) {
         P.Diag(Loc, diag::err_omp_variant_ctx_second_match_extension);
-        StringRef SeenName =
-            llvm::omp::getOpenMPContextTraitPropertyName(SeenProp.Kind);
+        StringRef SeenName = llvm::omp::getOpenMPContextTraitPropertyName(
+            SeenProp.Kind, SeenProp.RawString);
         SourceLocation SeenLoc = Seen[SeenName];
         P.Diag(SeenLoc, diag::note_omp_declare_variant_ctx_used_here)
             << CONTEXT_TRAIT_LVL << SeenName;
@@ -995,11 +996,13 @@ void Parser::parseOMPContextProperty(OMPTraitSelector &TISelector,
   }
 
   Diag(PropertyLoc, diag::warn_omp_ctx_incompatible_property_for_selector)
-      << getOpenMPContextTraitPropertyName(TIProperty.Kind)
+      << getOpenMPContextTraitPropertyName(TIProperty.Kind,
+                                           TIProperty.RawString)
       << getOpenMPContextTraitSelectorName(TISelector.Kind)
       << getOpenMPContextTraitSetName(Set);
   Diag(PropertyLoc, diag::note_omp_ctx_compatible_set_and_selector_for_property)
-      << getOpenMPContextTraitPropertyName(TIProperty.Kind)
+      << getOpenMPContextTraitPropertyName(TIProperty.Kind,
+                                           TIProperty.RawString)
       << getOpenMPContextTraitSelectorName(
              getOpenMPContextTraitSelectorForProperty(TIProperty.Kind))
       << getOpenMPContextTraitSetName(
@@ -1045,8 +1048,8 @@ void Parser::parseOMPTraitSelectorKind(
   for (const auto &PotentialSet :
        {TraitSet::construct, TraitSet::user, TraitSet::implementation,
         TraitSet::device}) {
-    TraitProperty PropertyForName =
-        getOpenMPContextTraitPropertyKind(PotentialSet, Name);
+    TraitProperty PropertyForName = getOpenMPContextTraitPropertyKind(
+        PotentialSet, TraitSelector::invalid, Name);
     if (PropertyForName == TraitProperty::invalid)
       continue;
     Diag(NameLoc, diag::note_omp_declare_variant_ctx_is_a)
@@ -1140,7 +1143,8 @@ void Parser::parseOMPContextSelector(
 
   if (!RequiresProperty) {
     TISelector.Properties.push_back(
-        {getOpenMPContextTraitPropertyForSelector(TISelector.Kind)});
+        {getOpenMPContextTraitPropertyForSelector(TISelector.Kind),
+         getOpenMPContextTraitSelectorName(TISelector.Kind)});
     return;
   }
 
@@ -1157,7 +1161,8 @@ void Parser::parseOMPContextSelector(
     if (!Condition.isUsable())
       return FinishSelector();
     TISelector.ScoreOrCondition = Condition.get();
-    TISelector.Properties.push_back({TraitProperty::user_condition_unknown});
+    TISelector.Properties.push_back(
+        {TraitProperty::user_condition_unknown, "<condition>"});
     return;
   }
 
@@ -1236,8 +1241,8 @@ void Parser::parseOMPTraitSetKind(OMPTraitSet &TISet,
   for (const auto &PotentialSet :
        {TraitSet::construct, TraitSet::user, TraitSet::implementation,
         TraitSet::device}) {
-    TraitProperty PropertyForName =
-        getOpenMPContextTraitPropertyKind(PotentialSet, Name);
+    TraitProperty PropertyForName = getOpenMPContextTraitPropertyKind(
+        PotentialSet, TraitSelector::invalid, Name);
     if (PropertyForName == TraitProperty::invalid)
       continue;
     Diag(NameLoc, diag::note_omp_declare_variant_ctx_is_a)
@@ -1820,8 +1825,15 @@ Parser::DeclGroupPtrTy Parser::ParseOpenMPDeclarativeDirectiveWithExtDecl(
     VariantMatchInfo VMI;
     ASTContext &ASTCtx = Actions.getASTContext();
     TI.getAsVariantMatchInfo(ASTCtx, VMI);
-    OMPContext OMPCtx(ASTCtx.getLangOpts().OpenMPIsDevice,
-                      ASTCtx.getTargetInfo().getTriple());
+
+    std::function<void(StringRef)> DiagUnknownTrait = [this, Loc](
+                                                          StringRef ISATrait) {
+      // TODO Track the selector locations in a way that is accessible here to
+      // improve the diagnostic location.
+      Diag(Loc, diag::warn_unknown_begin_declare_variant_isa_trait) << ISATrait;
+    };
+    TargetOMPContext OMPCtx(ASTCtx, std::move(DiagUnknownTrait),
+                            /* CurrentFunctionDecl */ nullptr);
 
     if (isVariantApplicableInContext(VMI, OMPCtx, /* DeviceSetOnly */ true)) {
       Actions.ActOnOpenMPBeginDeclareVariant(Loc, TI);
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index 0192df3bd170d..4f69975c54edd 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -5895,8 +5895,15 @@ ExprResult Sema::ActOnOpenMPCall(ExprResult Call, Scope *Scope,
     return Call;
 
   ASTContext &Context = getASTContext();
-  OMPContext OMPCtx(getLangOpts().OpenMPIsDevice,
-                    Context.getTargetInfo().getTriple());
+  std::function<void(StringRef)> DiagUnknownTrait = [this,
+                                                     CE](StringRef ISATrait) {
+    // TODO Track the selector locations in a way that is accessible here to
+    // improve the diagnostic location.
+    Diag(CE->getBeginLoc(), diag::warn_unknown_declare_variant_isa_trait)
+        << ISATrait;
+  };
+  TargetOMPContext OMPCtx(Context, std::move(DiagUnknownTrait),
+                          getCurFunctionDecl());
 
   SmallVector<Expr *, 4> Exprs;
   SmallVector<VariantMatchInfo, 4> VMIs;
@@ -5908,7 +5915,8 @@ ExprResult Sema::ActOnOpenMPCall(ExprResult Call, Scope *Scope,
       VariantMatchInfo VMI;
       OMPTraitInfo &TI = A->getTraitInfo();
       TI.getAsVariantMatchInfo(Context, VMI);
-      if (!isVariantApplicableInContext(VMI, OMPCtx, /* DeviceSetOnly */ false))
+      if (!isVariantApplicableInContext(VMI, OMPCtx,
+                                        /* DeviceSetOnly */ false))
         continue;
 
       VMIs.push_back(VMI);
diff --git a/clang/test/OpenMP/declare_variant_device_isa_codegen_1.c b/clang/test/OpenMP/declare_variant_device_isa_codegen_1.c
new file mode 100644
index 0000000000000..baa5eb8f88307
--- /dev/null
+++ b/clang/test/OpenMP/declare_variant_device_isa_codegen_1.c
@@ -0,0 +1,49 @@
+// RUN: %clang_cc1 -verify -fopenmp -x c -triple %itanium_abi_triple -emit-llvm %s -o - -fopenmp-version=50 | FileCheck %s --check-prefix=GENERIC
+// RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple %itanium_abi_triple -fexceptions -fcxx-exceptions -emit-pch -o %t -fopenmp-version=50 %s
+// RUN: %clang_cc1 -fopenmp -x c++ -triple %itanium_abi_triple -fexceptions -fcxx-exceptions -std=c++11 -include-pch %t -verify %s -emit-llvm -o - -fopenmp-version=50 | FileCheck %s --check-prefix=GENERIC
+
+// RUN: %clang_cc1 -target-feature +avx512f -verify -fopenmp -x c -triple %itanium_abi_triple -emit-llvm %s -o - -fopenmp-version=50 | FileCheck %s --check-prefix=WITHFEATURE
+// RUN: %clang_cc1 -target-feature +avx512f -fopenmp -x c++ -std=c++11 -triple %itanium_abi_triple -fexceptions -fcxx-exceptions -emit-pch -o %t -fopenmp-version=50 %s
+// RUN: %clang_cc1 -target-feature +avx512f -fopenmp -x c++ -triple %itanium_abi_triple -fexceptions -fcxx-exceptions -std=c++11 -include-pch %t -verify %s -emit-llvm -o - -fopenmp-version=50 | FileCheck %s --check-prefix=WITHFEATURE
+
+// expected-no-diagnostics
+
+// Test taken from PR46338 (by linna su)
+
+#ifndef HEADER
+#define HEADER
+
+void base_saxpy(int, float, float *, float *);
+void avx512_saxpy(int, float, float *, float *);
+
+#pragma omp declare variant(avx512_saxpy) \
+    match(device = {isa(avx512f)})
+void base_saxpy(int n, float s, float *x, float *y) {
+#pragma omp parallel for
+  for (int i = 0; i < n; i++)
+    y[i] = s * x[i] + y[i];
+}
+
+void avx512_saxpy(int n, float s, float *x, float *y) {
+#pragma omp parallel for simd simdlen(16) aligned(x, y : 64)
+  for (int i = 0; i < n; i++)
+    y[i] = s * x[i] + y[i];
+}
+
+void caller(int n, float s, float *x, float *y) {
+  // GENERIC:     define void @{{.*}}caller
+  // GENERIC:      call void @{{.*}}base_saxpy
+  // WITHFEATURE: define void @{{.*}}caller
+  // WITHFEATURE:  call void @{{.*}}avx512_saxpy
+  base_saxpy(n, s, x, y);
+}
+
+__attribute__((target("avx512f"))) void variant_caller(int n, float s, float *x, float *y) {
+  // GENERIC:     define void @{{.*}}variant_caller
+  // GENERIC:      call void @{{.*}}avx512_saxpy
+  // WITHFEATURE: define void @{{.*}}variant_caller
+  // WITHFEATURE:  call void @{{.*}}avx512_saxpy
+  base_saxpy(n, s, x, y);
+}
+
+#endif
diff --git a/clang/test/OpenMP/declare_variant_messages.c b/clang/test/OpenMP/declare_variant_messages.c
index ecbf022351a75..84a56c5fd4094 100644
--- a/clang/test/OpenMP/declare_variant_messages.c
+++ b/clang/test/OpenMP/declare_variant_messages.c
@@ -137,6 +137,18 @@ void marked_variant(void);
 #pragma omp declare variant(marked_variant) match(xxx={}) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-warning {{variant function in '#pragma omp declare variant' is itself marked as '#pragma omp declare variant'}} expected-note {{context set options are: 'construct' 'device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
 void marked(void);
 
+#pragma omp declare variant(foo) match(device = {isa("foo")})
+int unknown_isa_trait(void);
+#pragma omp declare variant(foo) match(device = {isa(foo)})
+int unknown_isa_trait2(void);
+#pragma omp declare variant(foo) match(device = {kind(fpga), isa(bar)})
+int ignored_isa_trait(void);
+
+void caller() {
+  unknown_isa_trait();  // expected-warning {{isa trait 'foo' is not known to the current target; verify the spelling or consider restricting the context selector with the 'arch' selector further}}
+  unknown_isa_trait2(); // expected-warning {{isa trait 'foo' is not known to the current target; verify the spelling or consider restricting the context selector with the 'arch' selector further}}
+  ignored_isa_trait();
+}
 
 #pragma omp declare variant // expected-error {{function declaration is expected after 'declare variant' directive}}
 
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPContext.h b/llvm/include/llvm/Frontend/OpenMP/OMPContext.h
index 1a42d189db44a..8a4179167c894 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPContext.h
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPContext.h
@@ -70,15 +70,20 @@ TraitSelector getOpenMPContextTraitSelectorForProperty(TraitProperty Property);
 /// Return a textual representation of the trait selector \p Kind.
 StringRef getOpenMPContextTraitSelectorName(TraitSelector Kind);
 
-/// Parse \p Str and return the trait set it matches or
-/// TraitProperty::invalid.
-TraitProperty getOpenMPContextTraitPropertyKind(TraitSet Set, StringRef Str);
+/// Parse \p Str and return the trait property it matches in the set \p Set and
+/// selector \p Selector or TraitProperty::invalid.
+TraitProperty getOpenMPContextTraitPropertyKind(TraitSet Set,
+                                                TraitSelector Selector,
+                                                StringRef Str);
 
 /// Return the trait property for a singleton selector \p Selector.
 TraitProperty getOpenMPContextTraitPropertyForSelector(TraitSelector Selector);
 
-/// Return a textual representation of the trait property \p Kind.
-StringRef getOpenMPContextTraitPropertyName(TraitProperty Kind);
+/// Return a textual representation of the trait property \p Kind, which might
+/// be the raw string we parsed (\p RawString) if we do not translate the
+/// property into a (distinct) enum.
+StringRef getOpenMPContextTraitPropertyName(TraitProperty Kind,
+                                            StringRef RawString);
 
 /// Return a textual representation of the trait property \p Kind with selector
 /// and set name included.
@@ -112,24 +117,36 @@ bool isValidTraitPropertyForTraitSetAndSelector(TraitProperty Property,
 /// scored (via the ScoresMap). In addition, the required consturct nesting is
 /// decribed as well.
 struct VariantMatchInfo {
-  /// Add the trait \p Property to the required trait set. If \p Score is not
-  /// null, it recorded as well. If \p Property is in the `construct` set it
-  /// is recorded in-order in the ConstructTraits as well.
-  void addTrait(TraitProperty Property, APInt *Score = nullptr) {
-    addTrait(getOpenMPContextTraitSetForProperty(Property), Property, Score);
+  /// Add the trait \p Property to the required trait set. \p RawString is the
+  /// string we parsed and derived \p Property from. If \p Score is not null, it
+  /// recorded as well. If \p Property is in the `construct` set it is recorded
+  /// in-order in the ConstructTraits as well.
+  void addTrait(TraitProperty Property, StringRef RawString,
+                APInt *Score = nullptr) {
+    addTrait(getOpenMPContextTraitSetForProperty(Property), Property, RawString,
+             Score);
   }
   /// Add the trait \p Property which is in set \p Set to the required trait
-  /// set. If \p Score is not null, it recorded as well. If \p Set is the
-  /// `construct` set it is recorded in-order in the ConstructTraits as well.
-  void addTrait(TraitSet Set, TraitProperty Property, APInt *Score = nullptr) {
+  /// set. \p RawString is the string we parsed and derived \p Property from. If
+  /// \p Score is not null, it recorded as well. If \p Set is the `construct`
+  /// set it is recorded in-order in the ConstructTraits as well.
+  void addTrait(TraitSet Set, TraitProperty Property, StringRef RawString,
+                APInt *Score = nullptr) {
     if (Score)
       ScoreMap[Property] = *Score;
+
+    // Special handling for `device={isa(...)}` as we do not match the enum but
+    // the raw string.
+    if (Property == TraitProperty::device_isa___ANY)
+      ISATraits.push_back(RawString);
+
     RequiredTraits.set(unsigned(Property));
     if (Set == TraitSet::construct)
       ConstructTraits.push_back(Property);
   }
 
   BitVector RequiredTraits = BitVector(unsigned(TraitProperty::Last) + 1);
+  SmallVector<StringRef, 8> ISATraits;
   SmallVector<TraitProperty, 8> ConstructTraits;
   SmallDenseMap<TraitProperty, APInt> ScoreMap;
 };
@@ -139,6 +156,7 @@ struct VariantMatchInfo {
 /// in OpenMP constructs at the location.
 struct OMPContext {
   OMPContext(bool IsDeviceCompilation, Triple TargetTriple);
+  virtual ~OMPContext() = default;
 
   void addTrait(TraitProperty Property) {
     addTrait(getOpenMPContextTraitSetForProperty(Property), Property);
@@ -149,6 +167,11 @@ struct OMPContext {
       ConstructTraits.push_back(Property);
   }
 
+  /// Hook for users to check if an ISA trait matches. The trait is described as
+  /// the string that got parsed and it depends on the target and context if
+  /// this matches or not.
+  virtual bool matchesISATrait(StringRef) const { return false; }
+
   BitVector ActiveTraits = BitVector(unsigned(TraitProperty::Last) + 1);
   SmallVector<TraitProperty, 8> ConstructTraits;
 };
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def b/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
index 7771dcd72d6a9..3fc87dc34cd34 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
@@ -1071,7 +1071,11 @@ __OMP_TRAIT_PROPERTY(device, kind, any)
 
 __OMP_TRAIT_SELECTOR(device, isa, true)
 
-// TODO: What do we want for ISA?
+// We use "__ANY" as a placeholder in the isa property to denote the
+// conceptual "any", not the literal `any` used in kind. The string we
+// we use is not important except that it will show up in diagnostics.
+OMP_TRAIT_PROPERTY(device_isa___ANY, device, device_isa,
+                   "<any, entirely target dependent>")
 
 __OMP_TRAIT_SELECTOR(device, arch, true)
 
diff --git a/llvm/lib/Frontend/OpenMP/OMPContext.cpp b/llvm/lib/Frontend/OpenMP/OMPContext.cpp
index c44e858ab5ed5..56a6e2b08bd90 100644
--- a/llvm/lib/Frontend/OpenMP/OMPContext.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPContext.cpp
@@ -175,11 +175,11 @@ static int isVariantApplicableInContextHelper(
     LLVM_DEBUG({
       if (MK == MK_ALL)
         dbgs() << "[" << DEBUG_TYPE << "] Property "
-               << getOpenMPContextTraitPropertyName(Property)
+               << getOpenMPContextTraitPropertyName(Property, "")
                << " was not in the OpenMP context but match kind is all.\n";
       if (MK == MK_NONE)
         dbgs() << "[" << DEBUG_TYPE << "] Property "
-               << getOpenMPContextTraitPropertyName(Property)
+               << getOpenMPContextTraitPropertyName(Property, "")
                << " was in the OpenMP context but match kind is none.\n";
     });
     return false;
@@ -198,6 +198,14 @@ static int isVariantApplicableInContextHelper(
       continue;
 
     bool IsActiveTrait = Ctx.ActiveTraits.test(unsigned(Property));
+
+    // We overwrite the isa trait as it is actually up to the OMPContext hook to
+    // check the raw string(s).
+    if (Property == TraitProperty::device_isa___ANY)
+      IsActiveTrait = llvm::all_of(VMI.ISATraits, [&](StringRef RawString) {
+        return Ctx.matchesISATrait(RawString);
+      });
+
     Optional<bool> Result = HandleTrait(Property, IsActiveTrait);
     if (Result.hasValue())
       return Result.getValue();
@@ -225,7 +233,7 @@ static int isVariantApplicableInContextHelper(
 
       if (!FoundInOrder) {
         LLVM_DEBUG(dbgs() << "[" << DEBUG_TYPE << "] Construct property "
-                          << getOpenMPContextTraitPropertyName(Property)
+                          << getOpenMPContextTraitPropertyName(Property, "")
                           << " was not nested properly.\n");
         return false;
       }
@@ -425,8 +433,12 @@ StringRef llvm::omp::getOpenMPContextTraitSelectorName(TraitSelector Kind) {
   llvm_unreachable("Unknown trait selector!");
 }
 
-TraitProperty llvm::omp::getOpenMPContextTraitPropertyKind(TraitSet Set,
-                                                           StringRef S) {
+TraitProperty llvm::omp::getOpenMPContextTraitPropertyKind(
+    TraitSet Set, TraitSelector Selector, StringRef S) {
+  // Special handling for `device={isa(...)}` as we accept anything here. It is
+  // up to the target to decide if the feature is available.
+  if (Set == TraitSet::device && Selector == TraitSelector::device_isa)
+    return TraitProperty::device_isa___ANY;
 #define OMP_TRAIT_PROPERTY(Enum, TraitSetEnum, TraitSelectorEnum, Str)         \
   if (Set == TraitSet::TraitSetEnum && Str == S)                               \
     return TraitProperty::Enum;
@@ -444,7 +456,10 @@ llvm::omp::getOpenMPContextTraitPropertyForSelector(TraitSelector Selector) {
 #include "llvm/Frontend/OpenMP/OMPKinds.def"
       .Default(TraitProperty::invalid);
 }
-StringRef llvm::omp::getOpenMPContextTraitPropertyName(TraitProperty Kind) {
+StringRef llvm::omp::getOpenMPContextTraitPropertyName(TraitProperty Kind,
+                                                       StringRef RawString) {
+  if (Kind == TraitProperty::device_isa___ANY)
+    return RawString;
   switch (Kind) {
 #define OMP_TRAIT_PROPERTY(Enum, TraitSetEnum, TraitSelectorEnum, Str)         \
   case TraitProperty::Enum:                                                    \
diff --git a/llvm/unittests/Frontend/OpenMPContextTest.cpp b/llvm/unittests/Frontend/OpenMPContextTest.cpp
index eb505be042cb3..5938559aacbc9 100644
--- a/llvm/unittests/Frontend/OpenMPContextTest.cpp
+++ b/llvm/unittests/Frontend/OpenMPContextTest.cpp
@@ -38,11 +38,13 @@ TEST_F(OpenMPContextTest, RoundTripAndAssociation) {
 #define OMP_TRAIT_PROPERTY(Enum, TraitSetEnum, TraitSelectorEnum, Str)         \
   EXPECT_EQ(TraitProperty::Enum,                                               \
             getOpenMPContextTraitPropertyKind(                                 \
-                TraitSet::TraitSetEnum,                                        \
-                getOpenMPContextTraitPropertyName(TraitProperty::Enum)));      \
+                TraitSet::TraitSetEnum, TraitSelector::TraitSelectorEnum,      \
+                getOpenMPContextTraitPropertyName(TraitProperty::Enum, Str))); \
   EXPECT_EQ(Str, getOpenMPContextTraitPropertyName(                            \
-                     getOpenMPContextTraitPropertyKind(TraitSet::TraitSetEnum, \
-                                                       Str)));                 \
+                     getOpenMPContextTraitPropertyKind(                        \
+                         TraitSet::TraitSetEnum,                               \
+                         TraitSelector::TraitSelectorEnum, Str),               \
+                     Str));                                                    \
   EXPECT_EQ(TraitSet::TraitSetEnum,                                            \
             getOpenMPContextTraitSetForProperty(TraitProperty::Enum));         \
   EXPECT_EQ(TraitSelector::TraitSelectorEnum,                                  \
@@ -77,31 +79,31 @@ TEST_F(OpenMPContextTest, ApplicabilityNonConstruct) {
   EXPECT_TRUE(isVariantApplicableInContext(Empty, DeviceNVPTX));
 
   VariantMatchInfo UserCondFalse;
-  UserCondFalse.addTrait(TraitProperty::user_condition_false);
+  UserCondFalse.addTrait(TraitProperty::user_condition_false, "");
   EXPECT_FALSE(isVariantApplicableInContext(UserCondFalse, HostLinux));
   EXPECT_FALSE(isVariantApplicableInContext(UserCondFalse, DeviceLinux));
   EXPECT_FALSE(isVariantApplicableInContext(UserCondFalse, HostNVPTX));
   EXPECT_FALSE(isVariantApplicableInContext(UserCondFalse, DeviceNVPTX));
 
   VariantMatchInfo DeviceArchArm;
-  DeviceArchArm.addTrait(TraitProperty::device_arch_arm);
+  DeviceArchArm.addTrait(TraitProperty::device_arch_arm, "");
   EXPECT_FALSE(isVariantApplicableInContext(DeviceArchArm, HostLinux));
   EXPECT_FALSE(isVariantApplicableInContext(DeviceArchArm, DeviceLinux));
   EXPECT_FALSE(isVariantApplicableInContext(DeviceArchArm, HostNVPTX));
   EXPECT_FALSE(isVariantApplicableInContext(DeviceArchArm, DeviceNVPTX));
 
   VariantMatchInfo LLVMHostUserCondTrue;
-  LLVMHostUserCondTrue.addTrait(TraitProperty::implementation_vendor_llvm);
-  LLVMHostUserCondTrue.addTrait(TraitProperty::device_kind_host);
-  LLVMHostUserCondTrue.addTrait(TraitProperty::device_kind_any);
-  LLVMHostUserCondTrue.addTrait(TraitProperty::user_condition_true);
+  LLVMHostUserCondTrue.addTrait(TraitProperty::implementation_vendor_llvm, "");
+  LLVMHostUserCondTrue.addTrait(TraitProperty::device_kind_host, "");
+  LLVMHostUserCondTrue.addTrait(TraitProperty::device_kind_any, "");
+  LLVMHostUserCondTrue.addTrait(TraitProperty::user_condition_true, "");
   EXPECT_TRUE(isVariantApplicableInContext(LLVMHostUserCondTrue, HostLinux));
   EXPECT_FALSE(isVariantApplicableInContext(LLVMHostUserCondTrue, DeviceLinux));
   EXPECT_TRUE(isVariantApplicableInContext(LLVMHostUserCondTrue, HostNVPTX));
   EXPECT_FALSE(isVariantApplicableInContext(LLVMHostUserCondTrue, DeviceNVPTX));
 
   VariantMatchInfo LLVMHostUserCondTrueCPU = LLVMHostUserCondTrue;
-  LLVMHostUserCondTrueCPU.addTrait(TraitProperty::device_kind_cpu);
+  LLVMHostUserCondTrueCPU.addTrait(TraitProperty::device_kind_cpu, "");
   EXPECT_TRUE(isVariantApplicableInContext(LLVMHostUserCondTrueCPU, HostLinux));
   EXPECT_FALSE(
       isVariantApplicableInContext(LLVMHostUserCondTrueCPU, DeviceLinux));
@@ -111,14 +113,14 @@ TEST_F(OpenMPContextTest, ApplicabilityNonConstruct) {
       isVariantApplicableInContext(LLVMHostUserCondTrueCPU, DeviceNVPTX));
 
   VariantMatchInfo GPU;
-  GPU.addTrait(TraitProperty::device_kind_gpu);
+  GPU.addTrait(TraitProperty::device_kind_gpu, "");
   EXPECT_FALSE(isVariantApplicableInContext(GPU, HostLinux));
   EXPECT_FALSE(isVariantApplicableInContext(GPU, DeviceLinux));
   EXPECT_TRUE(isVariantApplicableInContext(GPU, HostNVPTX));
   EXPECT_TRUE(isVariantApplicableInContext(GPU, DeviceNVPTX));
 
   VariantMatchInfo NoHost;
-  NoHost.addTrait(TraitProperty::device_kind_nohost);
+  NoHost.addTrait(TraitProperty::device_kind_nohost, "");
   EXPECT_FALSE(isVariantApplicableInContext(NoHost, HostLinux));
   EXPECT_TRUE(isVariantApplicableInContext(NoHost, DeviceLinux));
   EXPECT_FALSE(isVariantApplicableInContext(NoHost, HostNVPTX));
@@ -154,7 +156,7 @@ TEST_F(OpenMPContextTest, ApplicabilityAllTraits) {
         isVariantApplicableInContext(Empty, DeviceNVPTXTargetTeamsParallel));
 
     VariantMatchInfo UserCondFalse;
-    UserCondFalse.addTrait(TraitProperty::user_condition_false);
+    UserCondFalse.addTrait(TraitProperty::user_condition_false, "");
     EXPECT_FALSE(
         isVariantApplicableInContext(UserCondFalse, HostLinuxParallelParallel));
     EXPECT_FALSE(
@@ -164,7 +166,7 @@ TEST_F(OpenMPContextTest, ApplicabilityAllTraits) {
                                               DeviceNVPTXTargetTeamsParallel));
 
     VariantMatchInfo DeviceArchArm;
-    DeviceArchArm.addTrait(TraitProperty::device_arch_arm);
+    DeviceArchArm.addTrait(TraitProperty::device_arch_arm, "");
     EXPECT_FALSE(
         isVariantApplicableInContext(DeviceArchArm, HostLinuxParallelParallel));
     EXPECT_FALSE(
@@ -175,10 +177,12 @@ TEST_F(OpenMPContextTest, ApplicabilityAllTraits) {
 
     APInt Score(32, 1000);
     VariantMatchInfo LLVMHostUserCondTrue;
-    LLVMHostUserCondTrue.addTrait(TraitProperty::implementation_vendor_llvm);
-    LLVMHostUserCondTrue.addTrait(TraitProperty::device_kind_host);
-    LLVMHostUserCondTrue.addTrait(TraitProperty::device_kind_any);
-    LLVMHostUserCondTrue.addTrait(TraitProperty::user_condition_true, &Score);
+    LLVMHostUserCondTrue.addTrait(TraitProperty::implementation_vendor_llvm,
+                                  "");
+    LLVMHostUserCondTrue.addTrait(TraitProperty::device_kind_host, "");
+    LLVMHostUserCondTrue.addTrait(TraitProperty::device_kind_any, "");
+    LLVMHostUserCondTrue.addTrait(TraitProperty::user_condition_true, "",
+                                  &Score);
     EXPECT_TRUE(isVariantApplicableInContext(LLVMHostUserCondTrue,
                                              HostLinuxParallelParallel));
     EXPECT_FALSE(isVariantApplicableInContext(LLVMHostUserCondTrue,
@@ -189,7 +193,7 @@ TEST_F(OpenMPContextTest, ApplicabilityAllTraits) {
                                               DeviceNVPTXTargetTeamsParallel));
 
     VariantMatchInfo LLVMHostUserCondTrueCPU = LLVMHostUserCondTrue;
-    LLVMHostUserCondTrueCPU.addTrait(TraitProperty::device_kind_cpu);
+    LLVMHostUserCondTrueCPU.addTrait(TraitProperty::device_kind_cpu, "");
     EXPECT_TRUE(isVariantApplicableInContext(LLVMHostUserCondTrueCPU,
                                              HostLinuxParallelParallel));
     EXPECT_FALSE(isVariantApplicableInContext(LLVMHostUserCondTrueCPU,
@@ -200,7 +204,7 @@ TEST_F(OpenMPContextTest, ApplicabilityAllTraits) {
                                               DeviceNVPTXTargetTeamsParallel));
 
     VariantMatchInfo GPU;
-    GPU.addTrait(TraitProperty::device_kind_gpu);
+    GPU.addTrait(TraitProperty::device_kind_gpu, "");
     EXPECT_FALSE(isVariantApplicableInContext(GPU, HostLinuxParallelParallel));
     EXPECT_FALSE(isVariantApplicableInContext(GPU, DeviceLinuxTargetParallel));
     EXPECT_TRUE(isVariantApplicableInContext(GPU, HostNVPTXFor));
@@ -208,7 +212,7 @@ TEST_F(OpenMPContextTest, ApplicabilityAllTraits) {
         isVariantApplicableInContext(GPU, DeviceNVPTXTargetTeamsParallel));
 
     VariantMatchInfo NoHost;
-    NoHost.addTrait(TraitProperty::device_kind_nohost);
+    NoHost.addTrait(TraitProperty::device_kind_nohost, "");
     EXPECT_FALSE(
         isVariantApplicableInContext(NoHost, HostLinuxParallelParallel));
     EXPECT_TRUE(
@@ -219,8 +223,9 @@ TEST_F(OpenMPContextTest, ApplicabilityAllTraits) {
   }
   { // variants with all sets
     VariantMatchInfo DeviceArchArmParallel;
-    DeviceArchArmParallel.addTrait(TraitProperty::construct_parallel_parallel);
-    DeviceArchArmParallel.addTrait(TraitProperty::device_arch_arm);
+    DeviceArchArmParallel.addTrait(TraitProperty::construct_parallel_parallel,
+                                   "");
+    DeviceArchArmParallel.addTrait(TraitProperty::device_arch_arm, "");
     EXPECT_FALSE(isVariantApplicableInContext(DeviceArchArmParallel,
                                               HostLinuxParallelParallel));
     EXPECT_FALSE(isVariantApplicableInContext(DeviceArchArmParallel,
@@ -232,12 +237,13 @@ TEST_F(OpenMPContextTest, ApplicabilityAllTraits) {
 
     VariantMatchInfo LLVMHostUserCondTrueParallel;
     LLVMHostUserCondTrueParallel.addTrait(
-        TraitProperty::implementation_vendor_llvm);
-    LLVMHostUserCondTrueParallel.addTrait(TraitProperty::device_kind_host);
-    LLVMHostUserCondTrueParallel.addTrait(TraitProperty::device_kind_any);
-    LLVMHostUserCondTrueParallel.addTrait(TraitProperty::user_condition_true);
+        TraitProperty::implementation_vendor_llvm, "");
+    LLVMHostUserCondTrueParallel.addTrait(TraitProperty::device_kind_host, "");
+    LLVMHostUserCondTrueParallel.addTrait(TraitProperty::device_kind_any, "");
+    LLVMHostUserCondTrueParallel.addTrait(TraitProperty::user_condition_true,
+                                          "");
     LLVMHostUserCondTrueParallel.addTrait(
-        TraitProperty::construct_parallel_parallel);
+        TraitProperty::construct_parallel_parallel, "");
     EXPECT_TRUE(isVariantApplicableInContext(LLVMHostUserCondTrueParallel,
                                              HostLinuxParallelParallel));
     EXPECT_FALSE(isVariantApplicableInContext(LLVMHostUserCondTrueParallel,
@@ -250,7 +256,7 @@ TEST_F(OpenMPContextTest, ApplicabilityAllTraits) {
     VariantMatchInfo LLVMHostUserCondTrueParallelParallel =
         LLVMHostUserCondTrueParallel;
     LLVMHostUserCondTrueParallelParallel.addTrait(
-        TraitProperty::construct_parallel_parallel);
+        TraitProperty::construct_parallel_parallel, "");
     EXPECT_TRUE(isVariantApplicableInContext(
         LLVMHostUserCondTrueParallelParallel, HostLinuxParallelParallel));
     EXPECT_FALSE(isVariantApplicableInContext(
@@ -263,7 +269,7 @@ TEST_F(OpenMPContextTest, ApplicabilityAllTraits) {
     VariantMatchInfo LLVMHostUserCondTrueParallelParallelParallel =
         LLVMHostUserCondTrueParallelParallel;
     LLVMHostUserCondTrueParallelParallelParallel.addTrait(
-        TraitProperty::construct_parallel_parallel);
+        TraitProperty::construct_parallel_parallel, "");
     EXPECT_FALSE(isVariantApplicableInContext(
         LLVMHostUserCondTrueParallelParallelParallel,
         HostLinuxParallelParallel));
@@ -277,9 +283,9 @@ TEST_F(OpenMPContextTest, ApplicabilityAllTraits) {
         DeviceNVPTXTargetTeamsParallel));
 
     VariantMatchInfo GPUTargetTeams;
-    GPUTargetTeams.addTrait(TraitProperty::construct_target_target);
-    GPUTargetTeams.addTrait(TraitProperty::construct_teams_teams);
-    GPUTargetTeams.addTrait(TraitProperty::device_kind_gpu);
+    GPUTargetTeams.addTrait(TraitProperty::construct_target_target, "");
+    GPUTargetTeams.addTrait(TraitProperty::construct_teams_teams, "");
+    GPUTargetTeams.addTrait(TraitProperty::device_kind_gpu, "");
     EXPECT_FALSE(isVariantApplicableInContext(GPUTargetTeams,
                                               HostLinuxParallelParallel));
     EXPECT_FALSE(isVariantApplicableInContext(GPUTargetTeams,
@@ -289,9 +295,9 @@ TEST_F(OpenMPContextTest, ApplicabilityAllTraits) {
                                              DeviceNVPTXTargetTeamsParallel));
 
     VariantMatchInfo GPUTargetParallel;
-    GPUTargetParallel.addTrait(TraitProperty::construct_target_target);
-    GPUTargetParallel.addTrait(TraitProperty::construct_parallel_parallel);
-    GPUTargetParallel.addTrait(TraitProperty::device_kind_gpu);
+    GPUTargetParallel.addTrait(TraitProperty::construct_target_target, "");
+    GPUTargetParallel.addTrait(TraitProperty::construct_parallel_parallel, "");
+    GPUTargetParallel.addTrait(TraitProperty::device_kind_gpu, "");
     EXPECT_FALSE(isVariantApplicableInContext(GPUTargetParallel,
                                               HostLinuxParallelParallel));
     EXPECT_FALSE(isVariantApplicableInContext(GPUTargetParallel,

From 5cf0c2e67b802b71ba1d8e02d985c9133afb7fc8 Mon Sep 17 00:00:00 2001
From: Yuanfang Chen <yuanfang.chen@sony.com>
Date: Tue, 28 Jul 2020 16:31:29 -0700
Subject: [PATCH 0478/1035] [NewPM][PassInstrument] Add a new kind of
 before-pass callback that only get called if the pass is not skipped

TODO
 * PrintIRInstrumentation and TimePassesHandler would be using this new callback.
 * "Running pass" logging will also be moved to use this callback.

Reviewed By: aeubanks

Differential Revision: https://reviews.llvm.org/D84772
---
 llvm/include/llvm/IR/PassInstrumentation.h    | 22 +++++-
 .../unittests/IR/PassBuilderCallbacksTest.cpp | 68 +++++++++++++++++--
 2 files changed, 82 insertions(+), 8 deletions(-)

diff --git a/llvm/include/llvm/IR/PassInstrumentation.h b/llvm/include/llvm/IR/PassInstrumentation.h
index 94833bb0d2c63..df5a0ceb7d016 100644
--- a/llvm/include/llvm/IR/PassInstrumentation.h
+++ b/llvm/include/llvm/IR/PassInstrumentation.h
@@ -67,11 +67,14 @@ class PassInstrumentationCallbacks {
   // to take them as constant pointers, wrapped with llvm::Any.
   // For the case when IRUnit has been invalidated there is a different
   // callback to use - AfterPassInvalidated.
+  // We call all BeforePassFuncs to determine if a pass should run or not.
+  // BeforeNonSkippedPassFuncs are called only if the pass should run.
   // TODO: currently AfterPassInvalidated does not accept IRUnit, since passing
-  // already invalidated IRUnit is unsafe. There are ways to handle invalidated IRUnits
-  // in a safe way, and we might pursue that as soon as there is a useful instrumentation
-  // that needs it.
+  // already invalidated IRUnit is unsafe. There are ways to handle invalidated
+  // IRUnits in a safe way, and we might pursue that as soon as there is a
+  // useful instrumentation that needs it.
   using BeforePassFunc = bool(StringRef, Any);
+  using BeforeNonSkippedPassFunc = void(StringRef, Any);
   using AfterPassFunc = void(StringRef, Any);
   using AfterPassInvalidatedFunc = void(StringRef);
   using BeforeAnalysisFunc = void(StringRef, Any);
@@ -88,6 +91,11 @@ class PassInstrumentationCallbacks {
     BeforePassCallbacks.emplace_back(std::move(C));
   }
 
+  template <typename CallableT>
+  void registerBeforeNonSkippedPassCallback(CallableT C) {
+    BeforeNonSkippedPassCallbacks.emplace_back(std::move(C));
+  }
+
   template <typename CallableT> void registerAfterPassCallback(CallableT C) {
     AfterPassCallbacks.emplace_back(std::move(C));
   }
@@ -111,6 +119,8 @@ class PassInstrumentationCallbacks {
   friend class PassInstrumentation;
 
   SmallVector<llvm::unique_function<BeforePassFunc>, 4> BeforePassCallbacks;
+  SmallVector<llvm::unique_function<BeforeNonSkippedPassFunc>, 4>
+      BeforeNonSkippedPassCallbacks;
   SmallVector<llvm::unique_function<AfterPassFunc>, 4> AfterPassCallbacks;
   SmallVector<llvm::unique_function<AfterPassInvalidatedFunc>, 4>
       AfterPassInvalidatedCallbacks;
@@ -165,6 +175,12 @@ class PassInstrumentation {
     for (auto &C : Callbacks->BeforePassCallbacks)
       ShouldRun &= C(Pass.name(), llvm::Any(&IR));
     ShouldRun = ShouldRun || isRequired(Pass);
+
+    if (ShouldRun) {
+      for (auto &C : Callbacks->BeforeNonSkippedPassCallbacks)
+        C(Pass.name(), llvm::Any(&IR));
+    }
+
     return ShouldRun;
   }
 
diff --git a/llvm/unittests/IR/PassBuilderCallbacksTest.cpp b/llvm/unittests/IR/PassBuilderCallbacksTest.cpp
index c2c15069f4132..e197f73983039 100644
--- a/llvm/unittests/IR/PassBuilderCallbacksTest.cpp
+++ b/llvm/unittests/IR/PassBuilderCallbacksTest.cpp
@@ -320,6 +320,7 @@ struct MockPassInstrumentationCallbacks {
     ON_CALL(*this, runBeforePass(_, _)).WillByDefault(Return(true));
   }
   MOCK_METHOD2(runBeforePass, bool(StringRef PassID, llvm::Any));
+  MOCK_METHOD2(runBeforeNonSkippedPass, void(StringRef PassID, llvm::Any));
   MOCK_METHOD2(runAfterPass, void(StringRef PassID, llvm::Any));
   MOCK_METHOD1(runAfterPassInvalidated, void(StringRef PassID));
   MOCK_METHOD2(runBeforeAnalysis, void(StringRef PassID, llvm::Any));
@@ -329,6 +330,10 @@ struct MockPassInstrumentationCallbacks {
     Callbacks.registerBeforePassCallback([this](StringRef P, llvm::Any IR) {
       return this->runBeforePass(P, IR);
     });
+    Callbacks.registerBeforeNonSkippedPassCallback(
+        [this](StringRef P, llvm::Any IR) {
+          this->runBeforeNonSkippedPass(P, IR);
+        });
     Callbacks.registerAfterPassCallback(
         [this](StringRef P, llvm::Any IR) { this->runAfterPass(P, IR); });
     Callbacks.registerAfterPassInvalidatedCallback(
@@ -349,6 +354,9 @@ struct MockPassInstrumentationCallbacks {
     EXPECT_CALL(*this,
                 runBeforePass(Not(HasNameRegex("Mock")), HasName(IRName)))
         .Times(AnyNumber());
+    EXPECT_CALL(*this, runBeforeNonSkippedPass(Not(HasNameRegex("Mock")),
+                                               HasName(IRName)))
+        .Times(AnyNumber());
     EXPECT_CALL(*this, runAfterPass(Not(HasNameRegex("Mock")), HasName(IRName)))
         .Times(AnyNumber());
     EXPECT_CALL(*this,
@@ -500,6 +508,10 @@ TEST_F(ModuleCallbacksTest, InstrumentedPasses) {
   EXPECT_CALL(CallbacksHandle, runBeforePass(HasNameRegex("MockPassHandle"),
                                              HasName("<string>")))
       .InSequence(PISequence);
+  EXPECT_CALL(CallbacksHandle,
+              runBeforeNonSkippedPass(HasNameRegex("MockPassHandle"),
+                                      HasName("<string>")))
+      .InSequence(PISequence);
   EXPECT_CALL(CallbacksHandle,
               runBeforeAnalysis(HasNameRegex("MockAnalysisHandle"),
                                 HasName("<string>")))
@@ -532,8 +544,11 @@ TEST_F(ModuleCallbacksTest, InstrumentedSkippedPasses) {
   EXPECT_CALL(AnalysisHandle, run(HasName("<string>"), _)).Times(0);
   EXPECT_CALL(PassHandle, run(HasName("<string>"), _)).Times(0);
 
-  // As the pass is skipped there is no afterPass, beforeAnalysis/afterAnalysis
-  // as well.
+  // As the pass is skipped there is no nonskippedpass/afterPass,
+  // beforeAnalysis/afterAnalysis as well.
+  EXPECT_CALL(CallbacksHandle,
+              runBeforeNonSkippedPass(HasNameRegex("MockPassHandle"), _))
+      .Times(0);
   EXPECT_CALL(CallbacksHandle, runAfterPass(HasNameRegex("MockPassHandle"), _))
       .Times(0);
   EXPECT_CALL(CallbacksHandle,
@@ -545,12 +560,35 @@ TEST_F(ModuleCallbacksTest, InstrumentedSkippedPasses) {
 
   // Order is important here. `Adaptor` expectations should be checked first
   // because the its argument contains 'PassManager' (for example:
-  // ModuleToFunctionPassAdaptor{{.*}}PassManager{{.*}}). Here only check
-  // `runAfterPass` to show that they are not skipped.
-
+  // ModuleToFunctionPassAdaptor{{.*}}PassManager{{.*}}). Check
+  // `runBeforeNonSkippedPass` and `runAfterPass` to show that they are not
+  // skipped.
+  //
   // Pass managers are not ignored.
   // 5 = (1) ModulePassManager + (2) FunctionPassMangers + (1) LoopPassManager +
   //     (1) CGSCCPassManager
+  EXPECT_CALL(CallbacksHandle,
+              runBeforeNonSkippedPass(HasNameRegex("PassManager"), _))
+      .Times(5);
+  EXPECT_CALL(
+      CallbacksHandle,
+      runBeforeNonSkippedPass(HasNameRegex("ModuleToFunctionPassAdaptor"), _))
+      .Times(1);
+  EXPECT_CALL(CallbacksHandle,
+              runBeforeNonSkippedPass(
+                  HasNameRegex("ModuleToPostOrderCGSCCPassAdaptor"), _))
+      .Times(1);
+  EXPECT_CALL(
+      CallbacksHandle,
+      runBeforeNonSkippedPass(HasNameRegex("CGSCCToFunctionPassAdaptor"), _))
+      .Times(1);
+  EXPECT_CALL(
+      CallbacksHandle,
+      runBeforeNonSkippedPass(HasNameRegex("FunctionToLoopPassAdaptor"), _))
+      .Times(1);
+
+  // The `runAfterPass` checks are the same as these of
+  // `runBeforeNonSkippedPass`.
   EXPECT_CALL(CallbacksHandle, runAfterPass(HasNameRegex("PassManager"), _))
       .Times(5);
   EXPECT_CALL(CallbacksHandle,
@@ -630,6 +668,10 @@ TEST_F(FunctionCallbacksTest, InstrumentedPasses) {
   EXPECT_CALL(CallbacksHandle,
               runBeforePass(HasNameRegex("MockPassHandle"), HasName("foo")))
       .InSequence(PISequence);
+  EXPECT_CALL(
+      CallbacksHandle,
+      runBeforeNonSkippedPass(HasNameRegex("MockPassHandle"), HasName("foo")))
+      .InSequence(PISequence);
   EXPECT_CALL(
       CallbacksHandle,
       runBeforeAnalysis(HasNameRegex("MockAnalysisHandle"), HasName("foo")))
@@ -717,6 +759,10 @@ TEST_F(LoopCallbacksTest, InstrumentedPasses) {
   EXPECT_CALL(CallbacksHandle,
               runBeforePass(HasNameRegex("MockPassHandle"), HasName("loop")))
       .InSequence(PISequence);
+  EXPECT_CALL(
+      CallbacksHandle,
+      runBeforeNonSkippedPass(HasNameRegex("MockPassHandle"), HasName("loop")))
+      .InSequence(PISequence);
   EXPECT_CALL(
       CallbacksHandle,
       runBeforeAnalysis(HasNameRegex("MockAnalysisHandle"), HasName("loop")))
@@ -758,6 +804,10 @@ TEST_F(LoopCallbacksTest, InstrumentedInvalidatingPasses) {
   EXPECT_CALL(CallbacksHandle,
               runBeforePass(HasNameRegex("MockPassHandle"), HasName("loop")))
       .InSequence(PISequence);
+  EXPECT_CALL(
+      CallbacksHandle,
+      runBeforeNonSkippedPass(HasNameRegex("MockPassHandle"), HasName("loop")))
+      .InSequence(PISequence);
   EXPECT_CALL(
       CallbacksHandle,
       runBeforeAnalysis(HasNameRegex("MockAnalysisHandle"), HasName("loop")))
@@ -847,6 +897,10 @@ TEST_F(CGSCCCallbacksTest, InstrumentedPasses) {
   EXPECT_CALL(CallbacksHandle,
               runBeforePass(HasNameRegex("MockPassHandle"), HasName("(foo)")))
       .InSequence(PISequence);
+  EXPECT_CALL(
+      CallbacksHandle,
+      runBeforeNonSkippedPass(HasNameRegex("MockPassHandle"), HasName("(foo)")))
+      .InSequence(PISequence);
   EXPECT_CALL(
       CallbacksHandle,
       runBeforeAnalysis(HasNameRegex("MockAnalysisHandle"), HasName("(foo)")))
@@ -888,6 +942,10 @@ TEST_F(CGSCCCallbacksTest, InstrumentedInvalidatingPasses) {
   EXPECT_CALL(CallbacksHandle,
               runBeforePass(HasNameRegex("MockPassHandle"), HasName("(foo)")))
       .InSequence(PISequence);
+  EXPECT_CALL(
+      CallbacksHandle,
+      runBeforeNonSkippedPass(HasNameRegex("MockPassHandle"), HasName("(foo)")))
+      .InSequence(PISequence);
   EXPECT_CALL(
       CallbacksHandle,
       runBeforeAnalysis(HasNameRegex("MockAnalysisHandle"), HasName("(foo)")))

From 7a2e1122ae4611916c920c815a4310085ed04830 Mon Sep 17 00:00:00 2001
From: Yuanfang Chen <yuanfang.chen@sony.com>
Date: Tue, 28 Jul 2020 16:31:46 -0700
Subject: [PATCH 0479/1035] [NewPM][PassInstrument] Make PrintIR and TimePasses
 to use before-pass-run callback

Reviewed By: asbirlea, aeubanks

Differential Revision: https://reviews.llvm.org/D84773
---
 llvm/include/llvm/IR/PassTimingInfo.h               |  2 +-
 llvm/include/llvm/Passes/StandardInstrumentations.h |  2 +-
 llvm/lib/IR/PassTimingInfo.cpp                      | 11 ++++-------
 llvm/lib/Passes/StandardInstrumentations.cpp        | 12 ++++++------
 4 files changed, 12 insertions(+), 15 deletions(-)

diff --git a/llvm/include/llvm/IR/PassTimingInfo.h b/llvm/include/llvm/IR/PassTimingInfo.h
index a42d6aa3bede6..a3608971217bd 100644
--- a/llvm/include/llvm/IR/PassTimingInfo.h
+++ b/llvm/include/llvm/IR/PassTimingInfo.h
@@ -100,7 +100,7 @@ class TimePassesHandler {
   void stopTimer(StringRef PassID);
 
   // Implementation of pass instrumentation callbacks.
-  bool runBeforePass(StringRef PassID);
+  void runBeforePass(StringRef PassID);
   void runAfterPass(StringRef PassID);
 };
 
diff --git a/llvm/include/llvm/Passes/StandardInstrumentations.h b/llvm/include/llvm/Passes/StandardInstrumentations.h
index bd8b886a2bb3c..9c29bfc35941b 100644
--- a/llvm/include/llvm/Passes/StandardInstrumentations.h
+++ b/llvm/include/llvm/Passes/StandardInstrumentations.h
@@ -39,7 +39,7 @@ class PrintIRInstrumentation {
   void registerCallbacks(PassInstrumentationCallbacks &PIC);
 
 private:
-  bool printBeforePass(StringRef PassID, Any IR);
+  void printBeforePass(StringRef PassID, Any IR);
   void printAfterPass(StringRef PassID, Any IR);
   void printAfterPassInvalidated(StringRef PassID);
 
diff --git a/llvm/lib/IR/PassTimingInfo.cpp b/llvm/lib/IR/PassTimingInfo.cpp
index 829c3e8ea9ad4..b3cb4ae601647 100644
--- a/llvm/lib/IR/PassTimingInfo.cpp
+++ b/llvm/lib/IR/PassTimingInfo.cpp
@@ -240,17 +240,14 @@ static bool matchPassManager(StringRef PassID) {
          Prefix.endswith("AnalysisManagerProxy");
 }
 
-bool TimePassesHandler::runBeforePass(StringRef PassID) {
+void TimePassesHandler::runBeforePass(StringRef PassID) {
   if (matchPassManager(PassID))
-    return true;
+    return;
 
   startTimer(PassID);
 
   LLVM_DEBUG(dbgs() << "after runBeforePass(" << PassID << ")\n");
   LLVM_DEBUG(dump());
-
-  // we are not going to skip this pass, thus return true.
-  return true;
 }
 
 void TimePassesHandler::runAfterPass(StringRef PassID) {
@@ -267,8 +264,8 @@ void TimePassesHandler::registerCallbacks(PassInstrumentationCallbacks &PIC) {
   if (!Enabled)
     return;
 
-  PIC.registerBeforePassCallback(
-      [this](StringRef P, Any) { return this->runBeforePass(P); });
+  PIC.registerBeforeNonSkippedPassCallback(
+      [this](StringRef P, Any) { this->runBeforePass(P); });
   PIC.registerAfterPassCallback(
       [this](StringRef P, Any) { this->runAfterPass(P); });
   PIC.registerAfterPassInvalidatedCallback(
diff --git a/llvm/lib/Passes/StandardInstrumentations.cpp b/llvm/lib/Passes/StandardInstrumentations.cpp
index 22a7103715d56..2f802c9eee8a5 100644
--- a/llvm/lib/Passes/StandardInstrumentations.cpp
+++ b/llvm/lib/Passes/StandardInstrumentations.cpp
@@ -180,9 +180,9 @@ PrintIRInstrumentation::popModuleDesc(StringRef PassID) {
   return ModuleDesc;
 }
 
-bool PrintIRInstrumentation::printBeforePass(StringRef PassID, Any IR) {
+void PrintIRInstrumentation::printBeforePass(StringRef PassID, Any IR) {
   if (PassID.startswith("PassManager<") || PassID.contains("PassAdaptor<"))
-    return true;
+    return;
 
   // Saving Module for AfterPassInvalidated operations.
   // Note: here we rely on a fact that we do not change modules while
@@ -192,11 +192,11 @@ bool PrintIRInstrumentation::printBeforePass(StringRef PassID, Any IR) {
     pushModuleDesc(PassID, IR);
 
   if (!llvm::shouldPrintBeforePass(PassID))
-    return true;
+    return;
 
   SmallString<20> Banner = formatv("*** IR Dump Before {0} ***", PassID);
   unwrapAndPrint(IR, Banner, llvm::forcePrintModuleIR());
-  return true;
+  return;
 }
 
 void PrintIRInstrumentation::printAfterPass(StringRef PassID, Any IR) {
@@ -240,8 +240,8 @@ void PrintIRInstrumentation::registerCallbacks(
   // for later use in AfterPassInvalidated.
   StoreModuleDesc = llvm::forcePrintModuleIR() && llvm::shouldPrintAfterPass();
   if (llvm::shouldPrintBeforePass() || StoreModuleDesc)
-    PIC.registerBeforePassCallback(
-        [this](StringRef P, Any IR) { return this->printBeforePass(P, IR); });
+    PIC.registerBeforeNonSkippedPassCallback(
+        [this](StringRef P, Any IR) { this->printBeforePass(P, IR); });
 
   if (llvm::shouldPrintAfterPass()) {
     PIC.registerAfterPassCallback(

From 9ad7c980bb47edd7db8f8db828b487cc7dfc9921 Mon Sep 17 00:00:00 2001
From: David Sherwood <david.sherwood@arm.com>
Date: Fri, 10 Jul 2020 10:26:33 +0100
Subject: [PATCH 0480/1035] [SVE] Don't consider scalable vector types in
 SLPVectorizerPass::vectorizeChainsInBlock

In vectorizeChainsInBlock we try to collect chains of PHI nodes
that have the same element type, but the code is relying upon
the implicit conversion from TypeSize -> uint64_t. For now, I have
modified the code to ignore PHI nodes with scalable types.

Differential Revision: https://reviews.llvm.org/D83542
---
 .../Transforms/Vectorize/SLPVectorizer.cpp    | 13 +++++++--
 .../SLPVectorizer/AArch64/scalable-vector.ll  | 28 ++++++++++++++++++-
 2 files changed, 38 insertions(+), 3 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index d575b3245c650..dd6ff20b1defa 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -7398,8 +7398,17 @@ bool SLPVectorizerPass::vectorizeChainsInBlock(BasicBlock *BB, BoUpSLP &R) {
       // Look for the next elements with the same type.
       SmallVector<Value *, 4>::iterator SameTypeIt = IncIt;
       Type *EltTy = (*IncIt)->getType();
-      unsigned EltSize = EltTy->isSized() ? DL->getTypeSizeInBits(EltTy)
-                                          : MaxVecRegSize;
+
+      assert(EltTy->isSized() &&
+             "Instructions should all be sized at this point");
+      TypeSize EltTS = DL->getTypeSizeInBits(EltTy);
+      if (EltTS.isScalable()) {
+        // For now, just ignore vectorizing scalable types.
+        ++IncIt;
+        continue;
+      }
+
+      unsigned EltSize = EltTS.getFixedSize();
       unsigned MaxNumElts = MaxVecRegSize / EltSize;
       if (MaxNumElts < 2) {
         ++IncIt;
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/scalable-vector.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/scalable-vector.ll
index 70ce0dc4d7ba6..99c60912f9dba 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/scalable-vector.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/scalable-vector.ll
@@ -1,5 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -slp-vectorizer -S | FileCheck %s
+; RUN: opt < %s -slp-vectorizer -S 2>%t | FileCheck %s
+; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t
+
+; WARN-NOT: warning
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64-unknown-linux-gnu"
@@ -21,5 +24,28 @@ define void @test() {
   ret void
 }
 
+define <vscale x 4 x i32> @scalable_phi(<vscale x 4 x i32> %a, i32 %b) {
+; CHECK-LABEL: @scalable_phi(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[B:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    [[RETVAL:%.*]] = phi <vscale x 4 x i32> [ [[A:%.*]], [[ENTRY:%.*]] ], [ zeroinitializer, [[IF_THEN]] ]
+; CHECK-NEXT:    ret <vscale x 4 x i32> [[RETVAL]]
+;
+entry:
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %if.then, label %end
+
+if.then:
+  br label %end
+
+end:
+  %retval = phi <vscale x 4 x i32> [ %a, %entry ], [ zeroinitializer, %if.then ]
+  ret <vscale x 4 x i32> %retval
+}
+
 declare <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0nxv16i8(<vscale x 16 x i8>*, i32 immarg, <vscale x 16 x i1>, <vscale x 16 x i8>)
 declare void @llvm.masked.store.nxv16i8.p0nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i8>*, i32 immarg, <vscale x 16 x i1>)

From bbc2ddecbd342d4502fe43466bd3658b89ddad7d Mon Sep 17 00:00:00 2001
From: Nathan James <n.james93@hotmail.co.uk>
Date: Wed, 29 Jul 2020 16:34:06 +0100
Subject: [PATCH 0481/1035] [clang-tidy] Handled insertion only fixits when
 determining conflicts.

Handle insertion fix-its when removing incompatible errors by introducting a new EventType `ET_Insert`
This has lower prioirty than End events, but higher than begin.
Idea being If an insert is at the same place as a begin event, the insert should be processed first to reduce unnecessary conflicts.
Likewise if its at the same place as an end event, process the end event first for the same reason.

This also fixes https://bugs.llvm.org/show_bug.cgi?id=46511.

Reviewed By: aaron.ballman

Differential Revision: https://reviews.llvm.org/D82898
---
 .../ClangTidyDiagnosticConsumer.cpp           | 51 ++++++++++++-------
 ...coreguidelines-init-variables-conflict.cpp | 15 ++++++
 2 files changed, 48 insertions(+), 18 deletions(-)
 create mode 100644 clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines-init-variables-conflict.cpp

diff --git a/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.cpp b/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.cpp
index 521e6ef549b96..1471301a34317 100644
--- a/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.cpp
+++ b/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.cpp
@@ -31,6 +31,7 @@
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringMap.h"
+#include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/Regex.h"
 #include <tuple>
@@ -590,6 +591,7 @@ void ClangTidyDiagnosticConsumer::removeIncompatibleErrors() {
     // An event can be either the begin or the end of an interval.
     enum EventType {
       ET_Begin = 1,
+      ET_Insert = 0,
       ET_End = -1,
     };
 
@@ -621,10 +623,17 @@ void ClangTidyDiagnosticConsumer::removeIncompatibleErrors() {
       //   one will be processed before, disallowing the second one, and the
       //   end point of the first one will also be processed before,
       //   disallowing the first one.
-      if (Type == ET_Begin)
+      switch (Type) {
+      case ET_Begin:
         Priority = std::make_tuple(Begin, Type, -End, -ErrorSize, ErrorId);
-      else
+        break;
+      case ET_Insert:
+        Priority = std::make_tuple(Begin, Type, -End, ErrorSize, ErrorId);
+        break;
+      case ET_End:
         Priority = std::make_tuple(End, Type, -Begin, ErrorSize, ErrorId);
+        break;
+      }
     }
 
     bool operator<(const Event &Other) const {
@@ -662,19 +671,19 @@ void ClangTidyDiagnosticConsumer::removeIncompatibleErrors() {
   }
 
   // Build events from error intervals.
-  std::map<std::string, std::vector<Event>> FileEvents;
+  llvm::StringMap<std::vector<Event>> FileEvents;
   for (unsigned I = 0; I < ErrorFixes.size(); ++I) {
     for (const auto &FileAndReplace : *ErrorFixes[I].second) {
       for (const auto &Replace : FileAndReplace.second) {
         unsigned Begin = Replace.getOffset();
         unsigned End = Begin + Replace.getLength();
-        const std::string &FilePath = std::string(Replace.getFilePath());
-        // FIXME: Handle empty intervals, such as those from insertions.
-        if (Begin == End)
-          continue;
-        auto &Events = FileEvents[FilePath];
-        Events.emplace_back(Begin, End, Event::ET_Begin, I, Sizes[I]);
-        Events.emplace_back(Begin, End, Event::ET_End, I, Sizes[I]);
+        auto &Events = FileEvents[Replace.getFilePath()];
+        if (Begin == End) {
+          Events.emplace_back(Begin, End, Event::ET_Insert, I, Sizes[I]);
+        } else {
+          Events.emplace_back(Begin, End, Event::ET_Begin, I, Sizes[I]);
+          Events.emplace_back(Begin, End, Event::ET_End, I, Sizes[I]);
+        }
       }
     }
   }
@@ -686,14 +695,20 @@ void ClangTidyDiagnosticConsumer::removeIncompatibleErrors() {
     llvm::sort(Events);
     int OpenIntervals = 0;
     for (const auto &Event : Events) {
-      if (Event.Type == Event::ET_End)
-        --OpenIntervals;
-      // This has to be checked after removing the interval from the count if it
-      // is an end event, or before adding it if it is a begin event.
-      if (OpenIntervals != 0)
-        Apply[Event.ErrorId] = false;
-      if (Event.Type == Event::ET_Begin)
-        ++OpenIntervals;
+      switch (Event.Type) {
+      case Event::ET_Begin:
+        if (OpenIntervals++ != 0)
+          Apply[Event.ErrorId] = false;
+        break;
+      case Event::ET_Insert:
+        if (OpenIntervals != 0)
+          Apply[Event.ErrorId] = false;
+        break;
+      case Event::ET_End:
+        if (--OpenIntervals != 0)
+          Apply[Event.ErrorId] = false;
+        break;
+      }
     }
     assert(OpenIntervals == 0 && "Amount of begin/end points doesn't match");
   }
diff --git a/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines-init-variables-conflict.cpp b/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines-init-variables-conflict.cpp
new file mode 100644
index 0000000000000..4d67feb4161f8
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines-init-variables-conflict.cpp
@@ -0,0 +1,15 @@
+// RUN: %check_clang_tidy %s cppcoreguidelines-init-variables,readability-isolate-declaration %t
+
+void foo() {
+  int A, B, C;
+  // CHECK-MESSAGES-DAG: :[[@LINE-1]]:7: warning: variable 'A' is not initialized
+  // CHECK-MESSAGES-DAG: :[[@LINE-2]]:10: warning: variable 'B' is not initialized
+  // CHECK-MESSAGES-DAG: :[[@LINE-3]]:13: warning: variable 'C' is not initialized
+  // CHECK-MESSAGES-DAG: :[[@LINE-4]]:3: warning: multiple declarations in a single statement reduces readability
+
+  // Only the isolate declarations fix-it should be applied
+
+  //      CHECK-FIXES: int A;
+  // CHECK-FIXES-NEXT: int B;
+  // CHECK-FIXES-NEXT: int C;
+}

From d42c7b22118c763090e3bec180cb8f01b5c4cfeb Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Fri, 17 Jul 2020 21:32:24 -0400
Subject: [PATCH 0482/1035] AMDGPU: Account for the size of LDS globals used
 through constant expressions.

Also "fix" the longstanding bug where the computed size depends on the
order of the visitation. We could try to predict the allocation order
used by legalization, but it would never be 100% perfect. Until we
start fixing the addresses somehow (or have a more reliable allocation
scheme later), just try to compute the size based on the worst case
padding.
---
 .../lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp |  86 ++++++++---
 .../promote-alloca-padding-size-estimate.ll   |  14 +-
 .../promote-alloca-to-lds-constantexpr-use.ll | 140 +++++++++++++++++-
 3 files changed, 205 insertions(+), 35 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
index de05c2119d985..c9e2bd768ccec 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
@@ -749,41 +749,79 @@ bool AMDGPUPromoteAlloca::hasSufficientLocalMem(const Function &F) {
   if (LocalMemLimit == 0)
     return false;
 
-  const DataLayout &DL = Mod->getDataLayout();
+  SmallVector<const Constant *, 16> Stack;
+  SmallPtrSet<const Constant *, 8> VisitedConstants;
+  SmallPtrSet<const GlobalVariable *, 8> UsedLDS;
+
+  auto visitUsers = [&](const GlobalVariable *GV, const Constant *Val) -> bool {
+    for (const User *U : Val->users()) {
+      if (const Instruction *Use = dyn_cast<Instruction>(U)) {
+        if (Use->getParent()->getParent() == &F)
+          return true;
+      } else {
+        const Constant *C = cast<Constant>(U);
+        if (VisitedConstants.insert(C).second)
+          Stack.push_back(C);
+      }
+    }
+
+    return false;
+  };
 
-  // Check how much local memory is being used by global objects
-  CurrentLocalMemUsage = 0;
   for (GlobalVariable &GV : Mod->globals()) {
     if (GV.getAddressSpace() != AMDGPUAS::LOCAL_ADDRESS)
       continue;
 
-    for (const User *U : GV.users()) {
-      const Instruction *Use = dyn_cast<Instruction>(U);
-      if (!Use) {
-        // FIXME: This is probably a constant expression use. We should
-        // recursively search the users of it for the parent function instead of
-        // bailing.
-        LLVM_DEBUG(dbgs() << "Giving up on LDS size estimate "
-                             "due to constant expression\n");
-        return false;
-      }
+    if (visitUsers(&GV, &GV)) {
+      UsedLDS.insert(&GV);
+      Stack.clear();
+      continue;
+    }
 
-      if (Use->getParent()->getParent() == &F) {
-        Align Alignment =
-            DL.getValueOrABITypeAlignment(GV.getAlign(), GV.getValueType());
-
-        // FIXME: Try to account for padding here. The padding is currently
-        // determined from the inverse order of uses in the function. I'm not
-        // sure if the use list order is in any way connected to this, so the
-        // total reported size is likely incorrect.
-        uint64_t AllocSize = DL.getTypeAllocSize(GV.getValueType());
-        CurrentLocalMemUsage = alignTo(CurrentLocalMemUsage, Alignment);
-        CurrentLocalMemUsage += AllocSize;
+    // For any ConstantExpr uses, we need to recursively search the users until
+    // we see a function.
+    while (!Stack.empty()) {
+      const Constant *C = Stack.pop_back_val();
+      if (visitUsers(&GV, C)) {
+        UsedLDS.insert(&GV);
+        Stack.clear();
         break;
       }
     }
   }
 
+  const DataLayout &DL = Mod->getDataLayout();
+  SmallVector<std::pair<uint64_t, Align>, 16> AllocatedSizes;
+  AllocatedSizes.reserve(UsedLDS.size());
+
+  for (const GlobalVariable *GV : UsedLDS) {
+    Align Alignment =
+        DL.getValueOrABITypeAlignment(GV->getAlign(), GV->getValueType());
+    uint64_t AllocSize = DL.getTypeAllocSize(GV->getValueType());
+    AllocatedSizes.emplace_back(AllocSize, Alignment);
+  }
+
+  // Sort to try to estimate the worst case alignment padding
+  //
+  // FIXME: We should really do something to fix the addresses to a more optimal
+  // value instead
+  llvm::sort(AllocatedSizes.begin(), AllocatedSizes.end(),
+             [](std::pair<uint64_t, Align> LHS, std::pair<uint64_t, Align> RHS) {
+               return LHS.second < RHS.second;
+             });
+
+  // Check how much local memory is being used by global objects
+  CurrentLocalMemUsage = 0;
+
+  // FIXME: Try to account for padding here. The real padding and address is
+  // currently determined from the inverse order of uses in the function when
+  // legalizing, which could also potentially change. We try to estimate the
+  // worst case here, but we probably should fix the addresses earlier.
+  for (auto Alloc : AllocatedSizes) {
+    CurrentLocalMemUsage = alignTo(CurrentLocalMemUsage, Alloc.second);
+    CurrentLocalMemUsage += Alloc.first;
+  }
+
   unsigned MaxOccupancy = ST.getOccupancyWithLocalMemSize(CurrentLocalMemUsage,
                                                           F);
 
diff --git a/llvm/test/CodeGen/AMDGPU/promote-alloca-padding-size-estimate.ll b/llvm/test/CodeGen/AMDGPU/promote-alloca-padding-size-estimate.ll
index 83a608ad5f31c..69fbac7b44a33 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-alloca-padding-size-estimate.ll
+++ b/llvm/test/CodeGen/AMDGPU/promote-alloca-padding-size-estimate.ll
@@ -1,10 +1,12 @@
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=kaveri -mattr=-code-object-v3 -disable-promote-alloca-to-vector < %s | FileCheck -check-prefix=GCN %s
 
-; This shows that the amount of LDS estimate is sensitive to the order
-; of the LDS globals.
+; This shows that the amount LDS size estimate should try to not be
+; sensitive to the order of the LDS globals. This should try to
+; estimate the worst case padding behavior to avoid overallocating
+; LDS.
 
-; Both of these functions use the same amount of LDS, but the total
-; changes depending on the visit order of first use.
+; These functions use the same amount of LDS, but the total, final
+; size changes depending on the visit order of first use.
 
 ; The one with the suboptimal order resulting in extra padding exceeds
 ; the desired limit
@@ -29,7 +31,7 @@
 
 
 ; GCN-LABEL: {{^}}promote_alloca_size_order_0:
-; GCN: workgroup_group_segment_byte_size = 2340
+; GCN: workgroup_group_segment_byte_size = 1060
 define amdgpu_kernel void @promote_alloca_size_order_0(i32 addrspace(1)* nocapture %out, i32 addrspace(1)* nocapture %in, i32 %idx) #0 {
 entry:
   %stack = alloca [5 x i32], align 4, addrspace(5)
@@ -61,7 +63,7 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}promote_alloca_size_order_1:
-; GCN: workgroup_group_segment_byte_size = 2352
+; GCN: workgroup_group_segment_byte_size = 1072
 define amdgpu_kernel void @promote_alloca_size_order_1(i32 addrspace(1)* nocapture %out, i32 addrspace(1)* nocapture %in, i32 %idx) #0 {
 entry:
   %stack = alloca [5 x i32], align 4, addrspace(5)
diff --git a/llvm/test/CodeGen/AMDGPU/promote-alloca-to-lds-constantexpr-use.ll b/llvm/test/CodeGen/AMDGPU/promote-alloca-to-lds-constantexpr-use.ll
index a6511910603e1..4e23875502e96 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-alloca-to-lds-constantexpr-use.ll
+++ b/llvm/test/CodeGen/AMDGPU/promote-alloca-to-lds-constantexpr-use.ll
@@ -4,16 +4,20 @@
 target datalayout = "A5"
 
 @all_lds = internal unnamed_addr addrspace(3) global [16384 x i32] undef, align 4
+@some_lds = internal unnamed_addr addrspace(3) global [32 x i32] undef, align 4
+
+@initializer_user_some = addrspace(1) global i32 ptrtoint ([32 x i32] addrspace(3)* @some_lds to i32), align 4
+@initializer_user_all = addrspace(1) global i32 ptrtoint ([16384 x i32] addrspace(3)* @all_lds to i32), align 4
 
 ; This function cannot promote to using LDS because of the size of the
 ; constant expression use in the function, which was previously not
 ; detected.
-; IR-LABEL: @constant_expression_uses_lds(
+; IR-LABEL: @constant_expression_uses_all_lds(
 ; IR: alloca
 
-; ASM-LABEL: constant_expression_uses_lds:
-; ASM: .group_segment_fixed_size: 65536
-define amdgpu_kernel void @constant_expression_uses_lds(i32 addrspace(1)* nocapture %out, i32 %idx) #0 {
+; ASM-LABEL: constant_expression_uses_all_lds:
+; ASM: .amdhsa_group_segment_fixed_size 65536
+define amdgpu_kernel void @constant_expression_uses_all_lds(i32 addrspace(1)* nocapture %out, i32 %idx) #0 {
 entry:
   %stack = alloca [4 x i32], align 4, addrspace(5)
   %gep0 = getelementptr inbounds [4 x i32], [4 x i32] addrspace(5)* %stack, i32 0, i32 0
@@ -32,4 +36,130 @@ entry:
   ret void
 }
 
-attributes #0 = { "amdgpu-waves-per-eu"="1,5" }
+; Has a constant expression use through a single level of constant
+; expression, but not enough LDS to block promotion
+
+; IR-LABEL: @constant_expression_uses_some_lds(
+; IR-NOT: alloca
+
+; ASM-LABEL: {{^}}constant_expression_uses_some_lds:
+; ASM: .amdhsa_group_segment_fixed_size 4224{{$}}
+define amdgpu_kernel void @constant_expression_uses_some_lds(i32 addrspace(1)* nocapture %out, i32 %idx) #0 {
+entry:
+  %stack = alloca [4 x i32], align 4, addrspace(5)
+  %gep0 = getelementptr inbounds [4 x i32], [4 x i32] addrspace(5)* %stack, i32 0, i32 0
+  %gep1 = getelementptr inbounds [4 x i32], [4 x i32] addrspace(5)* %stack, i32 0, i32 1
+  %gep2 = getelementptr inbounds [4 x i32], [4 x i32] addrspace(5)* %stack, i32 0, i32 2
+  %gep3 = getelementptr inbounds [4 x i32], [4 x i32] addrspace(5)* %stack, i32 0, i32 3
+  store i32 9, i32 addrspace(5)* %gep0
+  store i32 10, i32 addrspace(5)* %gep1
+  store i32 99, i32 addrspace(5)* %gep2
+  store i32 43, i32 addrspace(5)* %gep3
+  %arrayidx = getelementptr inbounds [4 x i32], [4 x i32] addrspace(5)* %stack, i32 0, i32 %idx
+  %load = load i32, i32 addrspace(5)* %arrayidx, align 4
+  store i32 %load, i32 addrspace(1)* %out
+  store volatile i32 ptrtoint ([32 x i32] addrspace(3)* @some_lds to i32), i32 addrspace(1)* undef
+  ret void
+}
+
+declare void @callee(i8*)
+
+; IR-LABEL: @constant_expression_uses_all_lds_multi_level(
+; IR: alloca
+
+; ASM-LABEL: {{^}}constant_expression_uses_all_lds_multi_level:
+; ASM: .amdhsa_group_segment_fixed_size 65536{{$}}
+define amdgpu_kernel void @constant_expression_uses_all_lds_multi_level(i32 addrspace(1)* nocapture %out, i32 %idx) #0 {
+entry:
+  %stack = alloca [4 x i32], align 4, addrspace(5)
+  %gep0 = getelementptr inbounds [4 x i32], [4 x i32] addrspace(5)* %stack, i32 0, i32 0
+  %gep1 = getelementptr inbounds [4 x i32], [4 x i32] addrspace(5)* %stack, i32 0, i32 1
+  %gep2 = getelementptr inbounds [4 x i32], [4 x i32] addrspace(5)* %stack, i32 0, i32 2
+  %gep3 = getelementptr inbounds [4 x i32], [4 x i32] addrspace(5)* %stack, i32 0, i32 3
+  store i32 9, i32 addrspace(5)* %gep0
+  store i32 10, i32 addrspace(5)* %gep1
+  store i32 99, i32 addrspace(5)* %gep2
+  store i32 43, i32 addrspace(5)* %gep3
+  %arrayidx = getelementptr inbounds [4 x i32], [4 x i32] addrspace(5)* %stack, i32 0, i32 %idx
+  %load = load i32, i32 addrspace(5)* %arrayidx, align 4
+  store i32 %load, i32 addrspace(1)* %out
+  call void @callee(i8* addrspacecast (i8 addrspace(3)* bitcast (i32 addrspace(3)* getelementptr inbounds ([16384 x i32], [16384 x i32] addrspace(3)* @all_lds, i32 0, i32 8) to i8 addrspace(3)*) to i8*))
+  ret void
+}
+
+; IR-LABEL: @constant_expression_uses_some_lds_multi_level(
+; IR-NOT: alloca
+; IR: llvm.amdgcn.workitem.id
+
+; ASM-LABEL: {{^}}constant_expression_uses_some_lds_multi_level:
+; ASM: .amdhsa_group_segment_fixed_size 4224{{$}}
+define amdgpu_kernel void @constant_expression_uses_some_lds_multi_level(i32 addrspace(1)* nocapture %out, i32 %idx) #0 {
+entry:
+  %stack = alloca [4 x i32], align 4, addrspace(5)
+  %gep0 = getelementptr inbounds [4 x i32], [4 x i32] addrspace(5)* %stack, i32 0, i32 0
+  %gep1 = getelementptr inbounds [4 x i32], [4 x i32] addrspace(5)* %stack, i32 0, i32 1
+  %gep2 = getelementptr inbounds [4 x i32], [4 x i32] addrspace(5)* %stack, i32 0, i32 2
+  %gep3 = getelementptr inbounds [4 x i32], [4 x i32] addrspace(5)* %stack, i32 0, i32 3
+  store i32 9, i32 addrspace(5)* %gep0
+  store i32 10, i32 addrspace(5)* %gep1
+  store i32 99, i32 addrspace(5)* %gep2
+  store i32 43, i32 addrspace(5)* %gep3
+  %arrayidx = getelementptr inbounds [4 x i32], [4 x i32] addrspace(5)* %stack, i32 0, i32 %idx
+  %load = load i32, i32 addrspace(5)* %arrayidx, align 4
+  store i32 %load, i32 addrspace(1)* %out
+  call void @callee(i8* addrspacecast (i8 addrspace(3)* bitcast (i32 addrspace(3)* getelementptr inbounds ([32 x i32], [32 x i32] addrspace(3)* @some_lds, i32 0, i32 8) to i8 addrspace(3)*) to i8*))
+  ret void
+}
+
+; IR-LABEL: @constant_expression_uses_some_lds_global_initializer(
+; IR-NOT: alloca
+; IR: llvm.amdgcn.workitem.id
+
+; ASM-LABEL: {{^}}constant_expression_uses_some_lds_global_initializer:
+; ASM: .amdhsa_group_segment_fixed_size 4096{{$}}
+define amdgpu_kernel void @constant_expression_uses_some_lds_global_initializer(i32 addrspace(1)* nocapture %out, i32 %idx) #0 {
+entry:
+  %stack = alloca [4 x i32], align 4, addrspace(5)
+  %gep0 = getelementptr inbounds [4 x i32], [4 x i32] addrspace(5)* %stack, i32 0, i32 0
+  %gep1 = getelementptr inbounds [4 x i32], [4 x i32] addrspace(5)* %stack, i32 0, i32 1
+  %gep2 = getelementptr inbounds [4 x i32], [4 x i32] addrspace(5)* %stack, i32 0, i32 2
+  %gep3 = getelementptr inbounds [4 x i32], [4 x i32] addrspace(5)* %stack, i32 0, i32 3
+  store i32 9, i32 addrspace(5)* %gep0
+  store i32 10, i32 addrspace(5)* %gep1
+  store i32 99, i32 addrspace(5)* %gep2
+  store i32 43, i32 addrspace(5)* %gep3
+  %arrayidx = getelementptr inbounds [4 x i32], [4 x i32] addrspace(5)* %stack, i32 0, i32 %idx
+  %load = load i32, i32 addrspace(5)* %arrayidx, align 4
+  store i32 %load, i32 addrspace(1)* %out
+
+  store volatile i32 ptrtoint (i32 addrspace(1)* @initializer_user_some to i32), i32 addrspace(1)* undef
+  ret void
+}
+
+; We can't actually handle LDS initializers in global initializers,
+; but this should count as usage.
+
+; IR-LABEL: @constant_expression_uses_all_lds_global_initializer(
+; IR: alloca
+
+; ASM-LABEL: {{^}}constant_expression_uses_all_lds_global_initializer:
+; ASM: .group_segment_fixed_size: 65536
+define amdgpu_kernel void @constant_expression_uses_all_lds_global_initializer(i32 addrspace(1)* nocapture %out, i32 %idx) #0 {
+entry:
+  %stack = alloca [4 x i32], align 4, addrspace(5)
+  %gep0 = getelementptr inbounds [4 x i32], [4 x i32] addrspace(5)* %stack, i32 0, i32 0
+  %gep1 = getelementptr inbounds [4 x i32], [4 x i32] addrspace(5)* %stack, i32 0, i32 1
+  %gep2 = getelementptr inbounds [4 x i32], [4 x i32] addrspace(5)* %stack, i32 0, i32 2
+  %gep3 = getelementptr inbounds [4 x i32], [4 x i32] addrspace(5)* %stack, i32 0, i32 3
+  store i32 9, i32 addrspace(5)* %gep0
+  store i32 10, i32 addrspace(5)* %gep1
+  store i32 99, i32 addrspace(5)* %gep2
+  store i32 43, i32 addrspace(5)* %gep3
+  %arrayidx = getelementptr inbounds [4 x i32], [4 x i32] addrspace(5)* %stack, i32 0, i32 %idx
+  %load = load i32, i32 addrspace(5)* %arrayidx, align 4
+  store i32 %load, i32 addrspace(1)* %out
+  store volatile i32 ptrtoint (i32 addrspace(1)* @initializer_user_all to i32), i32 addrspace(1)* undef
+  ret void
+}
+
+attributes #0 = { "amdgpu-waves-per-eu"="1,5" "amdgpu-flat-work-group-size"="256,256" }

From a4ade9ed21e55946c52961a549ca4d23f9058b08 Mon Sep 17 00:00:00 2001
From: Kang Zhang <shkzhang@cn.ibm.com>
Date: Wed, 29 Jul 2020 15:43:47 +0000
Subject: [PATCH 0483/1035] [MachineVerifier] Handle the PHI node for
 verifyLiveVariables()

Summary:
When doing MachineVerifier for LiveVariables, the MachineVerifier pass
will calculate the LiveVariables, and compares the result with the
result livevars pass gave. If they are different, verifyLiveVariables()
will give error.

But when we calculate the LiveVariables in MachineVerifier, we don't
consider the PHI node, while livevars considers.

This patch is to fix above bug.

Reviewed By: bjope

Differential Revision: https://reviews.llvm.org/D80274
---
 llvm/lib/CodeGen/MachineVerifier.cpp          | 20 ++++++-
 llvm/test/CodeGen/PowerPC/livevars-crash1.mir | 53 ++++++++++++++---
 llvm/test/CodeGen/PowerPC/livevars-crash2.mir | 58 +++++++++++++------
 3 files changed, 105 insertions(+), 26 deletions(-)

diff --git a/llvm/lib/CodeGen/MachineVerifier.cpp b/llvm/lib/CodeGen/MachineVerifier.cpp
index c1a2c4e0bc6e6..e45e6963edb11 100644
--- a/llvm/lib/CodeGen/MachineVerifier.cpp
+++ b/llvm/lib/CodeGen/MachineVerifier.cpp
@@ -132,7 +132,8 @@ namespace {
       bool reachable = false;
 
       // Vregs that must be live in because they are used without being
-      // defined. Map value is the user.
+      // defined. Map value is the user. vregsLiveIn doesn't include regs
+      // that only are used by PHI nodes.
       RegMap vregsLiveIn;
 
       // Regs killed in MBB. They may be defined again, and will then be in both
@@ -2302,6 +2303,23 @@ void MachineVerifier::calcRegsRequired() {
       if (PInfo.addRequired(MInfo.vregsLiveIn))
         todo.insert(Pred);
     }
+
+    // Handle the PHI node.
+    for (const MachineInstr &MI : MBB.phis()) {
+      for (unsigned i = 1, e = MI.getNumOperands(); i != e; i += 2) {
+        // Skip those Operands which are undef regs or not regs.
+        if (!MI.getOperand(i).isReg() || !MI.getOperand(i).readsReg())
+          continue;
+
+        // Get register and predecessor for one PHI edge.
+        Register Reg = MI.getOperand(i).getReg();
+        const MachineBasicBlock *Pred = MI.getOperand(i + 1).getMBB();
+
+        BBInfo &PInfo = MBBInfoMap[Pred];
+        if (PInfo.addRequired(Reg))
+          todo.insert(Pred);
+      }
+    }
   }
 
   // Iteratively push vregsRequired to predecessors. This will converge to the
diff --git a/llvm/test/CodeGen/PowerPC/livevars-crash1.mir b/llvm/test/CodeGen/PowerPC/livevars-crash1.mir
index 53337040e70a8..6457a3d242362 100644
--- a/llvm/test/CodeGen/PowerPC/livevars-crash1.mir
+++ b/llvm/test/CodeGen/PowerPC/livevars-crash1.mir
@@ -1,6 +1,6 @@
-# RUN: not --crash llc -mtriple powerpc64le-unknown-linux-gnu %s -o - 2>&1 \
-# RUN:   -run-pass=livevars,phi-node-elimination -verify-machineinstrs \
-# RUN:   | FileCheck %s
+# RUN: llc -mtriple powerpc64le-unknown-linux-gnu %s -o - 2>&1 \
+# RUN:   -run-pass=livevars,phi-node-elimination -verify-machineinstrs | \
+# RUN:  FileCheck %s
 
 --- |
   ; Function Attrs: noreturn nounwind
@@ -82,9 +82,46 @@ body:             |
     STD %3, 0, %4 :: (store 8 into %ir.p)
     B %bb.1
 
+  ; CHECK-LABEL: name: zext_free
+  ; CHECK: bb.0.entry:
+  ; CHECK:   successors: %bb.1(0x80000000)
+  ; CHECK:   liveins: $x3
+
+  ; CHECK:   %4:g8rc_and_g8rc_nox0 = COPY killed $x3
+  ; CHECK:   %0:g8rc = LD 0, %4 :: (dereferenceable load 8 from %ir.p)
+  ; CHECK:    %12:g8rc_and_g8rc_nox0 = COPY killed %0
+
+  ; CHECK: bb.1.loop:
+  ; CHECK:   successors: %bb.1(0x20000000), %bb.2(0x60000000)
+
+  ; CHECK:   %1:g8rc_and_g8rc_nox0 = COPY killed %12
+  ; CHECK:   %5:gprc = LBZ 0, %1 :: (load 1 from %ir.0)
+  ; CHECK:   %6:crrc = CMPWI killed %5, 0
+  ; CHEXK:   %7:crbitrc = COPY killed %6.sub_eq
+  ; CHECK:   %2:g8rc = nuw ADDI8 %1, 1
+  ; CHECK:   STD %2, 0, %4 :: (store 8 into %ir.p)
+  ; CHECK:   %8:gprc = LBZ 1, %1 :: (load 1 from %ir.incdec.ptr)
+  ; CHECK:   %12:g8rc_and_g8rc_nox0 = COPY %2
+  ; CHECK:   BCn killed %7, %bb.1
+  ; CHECK:   B %bb.2
+
+  ; CHECK: bb.2.loop:
+  ; CHECK:   successors: %bb.3(0x55555555), %bb.1(0x2aaaaaab)
+
+  ; CHECK:   %9:crrc = CMPWI killed %8, 0
+  ; CHECK:   %10:crbitrc = COPY killed %9.sub_eq
+  ; CHECK:   %12:g8rc_and_g8rc_nox0 = COPY killed %2
+  ; CHECK:   BC killed %10, %bb.1
+  ; CHECK:   B %bb.3
+
+  ; CHECK: bb.3.if.then3:
+  ; CHECK:   successors: %bb.1(0x80000000)
+
+  ; CHECK:   %3:g8rc = nuw ADDI8 killed %1, 2
+  ; CHECK:   STD %3, 0, %4 :: (store 8 into %ir.p)
+  ; CHECK:   %12:g8rc_and_g8rc_nox0 = COPY killed %3
+  ; CHECK:   B %bb.1
+
+
 ...
-# CHECK-LABEL: Bad machine code: LiveVariables: Block should not be in AliveBlocks
-# CHECK-NEXT:  - function:    zext_free
-# CHECK-NEXT:  - basic block: %bb.2 loop
-# CHECK-NEXT:  Virtual register %2 is not needed live through the block.
-# CHECK-NEXT:  LLVM ERROR: Found 1 machine code errors.
+
diff --git a/llvm/test/CodeGen/PowerPC/livevars-crash2.mir b/llvm/test/CodeGen/PowerPC/livevars-crash2.mir
index e62db1ed4cb27..1bdf0419a8f59 100644
--- a/llvm/test/CodeGen/PowerPC/livevars-crash2.mir
+++ b/llvm/test/CodeGen/PowerPC/livevars-crash2.mir
@@ -1,6 +1,6 @@
-# RUN: not --crash llc -mtriple powerpc64le-unknown-linux-gnu %s -o - 2>&1 \
-# RUN:   -run-pass=livevars,phi-node-elimination -verify-machineinstrs \
-# RUN:   | FileCheck %s
+# RUN: llc -mtriple powerpc64le-unknown-linux-gnu %s -o - 2>&1 \
+# RUN:   -run-pass=livevars,phi-node-elimination -verify-machineinstrs | \
+# RUN:   FileCheck %s
 
 --- |
   define float @testfloatslt(float %c1, float %c2, float %c3, float %c4, float %a1, float %a2) {
@@ -176,20 +176,44 @@ body:             |
     STD %3, 0, %4 :: (store 8 into %ir.p)
     B %bb.1
 
-...
+  ; CHECK-LABEL: name: testfloatslt
+  ; CHECK: bb.0.entry:
+  ; CHECK:   successors: %bb.1(0x80000000)
+  ; CHECK:   liveins: $x3
+
+  ; CHECK:   %4:g8rc_and_g8rc_nox0 = COPY killed $x3
+  ; CHECK:   %0:g8rc = LD 0, %4 :: (dereferenceable load 8 from %ir.p)
+  ; CHECK:   %12:g8rc_and_g8rc_nox0 = COPY killed %0
+
+  ; CHECK: bb.1.loop:
+  ; CHECK:   successors: %bb.1(0x20000000), %bb.2(0x60000000)
+
+  ; CHECK:   %1:g8rc_and_g8rc_nox0 = COPY killed %12
+  ; CHECK:   %5:gprc = LBZ 0, %1 :: (load 1 from %ir.0)
+  ; CHECK:   %6:crrc = CMPWI killed %5, 0
+  ; CEHCK:   %7:crbitrc = COPY killed %6.sub_eq
+  ; CHECK:   %2:g8rc = nuw ADDI8 %1, 1
+  ; CHECK:   STD %2, 0, %4 :: (store 8 into %ir.p)
+  ; CHECK:   %8:gprc = LBZ 1, %1 :: (load 1 from %ir.incdec.ptr)
+  ; CHECK:   %12:g8rc_and_g8rc_nox0 = COPY %2
+  ; CHECK:   BCn killed %7, %bb.1
+  ; CHECK:   B %bb.2
 
-# CHECK-LABEL: Bad machine code: LiveVariables: Block should not be in AliveBlocks
-# CHECK-NEXT:  - function:    testfloatslt
-# CHECK-NEXT:  - basic block: %bb.1 entry
-# CHECK-NEXT:  Virtual register %4 is not needed live through the block.
+  ; CHECK: bb.2.loop:
+  ; CHECK:   successors: %bb.3(0x55555555), %bb.1(0x2aaaaaab)
 
-# CHECK-LABEL: Bad machine code: LiveVariables: Block should not be in AliveBlocks
-# CHECK-NEXT:  - function:    testfloatslt
-# CHECK-NEXT:  - basic block: %bb.1 entry
-# CHECK-NEXT:  Virtual register %5 is not needed live through the block.
+  ; CHECK:   %9:crrc = CMPWI killed %8, 0
+  ; CHECK:   %10:crbitrc = COPY killed %9.sub_eq
+  ; CHECK:   %12:g8rc_and_g8rc_nox0 = COPY killed %2
+  ; CHECK:   BC killed %10, %bb.1
+  ; CHECK:   B %bb.3
 
-# CHECK-LABEL: Bad machine code: LiveVariables: Block should not be in AliveBlocks
-# CHECK-NEXT:  - function:    testfloatslt
-# CHECK-NEXT:  - basic block: %bb.2 entry
-# CHECK-NEXT:  Virtual register %5 is not needed live through the block.
-# CHECK-NEXT:  LLVM ERROR: Found 3 machine code errors.
+  ; CHECK: bb.3.if.then3:
+  ; CHECK:   successors: %bb.1(0x80000000)
+
+  ; CHECK:   %3:g8rc = nuw ADDI8 killed %1, 2
+  ; CHECK:   STD %3, 0, %4 :: (store 8 into %ir.p)
+  ; CHECK:   %12:g8rc_and_g8rc_nox0 = COPY killed %3
+  ; CHECK:   B %bb.1
+
+...

From 3c20ede18b85ba17557d7060d7d5ef2050ce77c0 Mon Sep 17 00:00:00 2001
From: Sanjay Patel <spatel@rotateright.com>
Date: Wed, 29 Jul 2020 11:17:11 -0400
Subject: [PATCH 0484/1035] [InstSimplify] fold integer min/max intrinsic with
 same args

---
 llvm/lib/Analysis/InstructionSimplify.cpp            |  4 ++++
 .../Transforms/InstSimplify/maxmin_intrinsics.ll     | 12 ++++--------
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp
index 01b2eb14a5b49..0f45182788e46 100644
--- a/llvm/lib/Analysis/InstructionSimplify.cpp
+++ b/llvm/lib/Analysis/InstructionSimplify.cpp
@@ -5258,6 +5258,10 @@ static Value *simplifyBinaryIntrinsic(Function *F, Value *Op0, Value *Op1,
   case Intrinsic::smin:
   case Intrinsic::umax:
   case Intrinsic::umin: {
+    // If the arguments are the same, this is a no-op.
+    if (Op0 == Op1)
+      return Op0;
+
     // Canonicalize constant operand as Op1.
     if (isa<Constant>(Op0))
       std::swap(Op0, Op1);
diff --git a/llvm/test/Transforms/InstSimplify/maxmin_intrinsics.ll b/llvm/test/Transforms/InstSimplify/maxmin_intrinsics.ll
index 13c9f0ebd4abb..fee630f9f230a 100644
--- a/llvm/test/Transforms/InstSimplify/maxmin_intrinsics.ll
+++ b/llvm/test/Transforms/InstSimplify/maxmin_intrinsics.ll
@@ -14,8 +14,7 @@ declare <2 x i8> @llvm.umin.v2i8(<2 x i8>, <2 x i8>)
 
 define i81 @smax_sameval(i81 %x) {
 ; CHECK-LABEL: @smax_sameval(
-; CHECK-NEXT:    [[R:%.*]] = call i81 @llvm.smax.i81(i81 [[X:%.*]], i81 [[X]])
-; CHECK-NEXT:    ret i81 [[R]]
+; CHECK-NEXT:    ret i81 [[X:%.*]]
 ;
   %r = call i81 @llvm.smax.i81(i81 %x, i81 %x)
   ret i81 %r
@@ -23,8 +22,7 @@ define i81 @smax_sameval(i81 %x) {
 
 define i3 @smin_sameval(i3 %x) {
 ; CHECK-LABEL: @smin_sameval(
-; CHECK-NEXT:    [[R:%.*]] = call i3 @llvm.smin.i3(i3 [[X:%.*]], i3 [[X]])
-; CHECK-NEXT:    ret i3 [[R]]
+; CHECK-NEXT:    ret i3 [[X:%.*]]
 ;
   %r = call i3 @llvm.smin.i3(i3 %x, i3 %x)
   ret i3 %r
@@ -32,8 +30,7 @@ define i3 @smin_sameval(i3 %x) {
 
 define <2 x i8> @umax_sameval(<2 x i8> %x) {
 ; CHECK-LABEL: @umax_sameval(
-; CHECK-NEXT:    [[R:%.*]] = call <2 x i8> @llvm.umax.v2i8(<2 x i8> [[X:%.*]], <2 x i8> [[X]])
-; CHECK-NEXT:    ret <2 x i8> [[R]]
+; CHECK-NEXT:    ret <2 x i8> [[X:%.*]]
 ;
   %r = call <2 x i8> @llvm.umax.v2i8(<2 x i8> %x, <2 x i8> %x)
   ret <2 x i8> %r
@@ -41,8 +38,7 @@ define <2 x i8> @umax_sameval(<2 x i8> %x) {
 
 define <2 x i8> @umin_sameval(<2 x i8> %x) {
 ; CHECK-LABEL: @umin_sameval(
-; CHECK-NEXT:    [[R:%.*]] = call <2 x i8> @llvm.umin.v2i8(<2 x i8> [[X:%.*]], <2 x i8> [[X]])
-; CHECK-NEXT:    ret <2 x i8> [[R]]
+; CHECK-NEXT:    ret <2 x i8> [[X:%.*]]
 ;
   %r = call <2 x i8> @llvm.umin.v2i8(<2 x i8> %x, <2 x i8> %x)
   ret <2 x i8> %r

From 3e8534fbc62c87bd72c4145598a61eb805c82393 Mon Sep 17 00:00:00 2001
From: Sanjay Patel <spatel@rotateright.com>
Date: Wed, 29 Jul 2020 11:24:36 -0400
Subject: [PATCH 0485/1035] [InstSimplify] allow partial undef constants for
 vector min/max folds

---
 llvm/lib/Analysis/InstructionSimplify.cpp            |  5 ++---
 .../Transforms/InstSimplify/maxmin_intrinsics.ll     | 12 ++++--------
 2 files changed, 6 insertions(+), 11 deletions(-)

diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp
index 0f45182788e46..af487f99e8e24 100644
--- a/llvm/lib/Analysis/InstructionSimplify.cpp
+++ b/llvm/lib/Analysis/InstructionSimplify.cpp
@@ -5266,16 +5266,15 @@ static Value *simplifyBinaryIntrinsic(Function *F, Value *Op0, Value *Op1,
     if (isa<Constant>(Op0))
       std::swap(Op0, Op1);
 
-    // TODO: Allow partial undef vector constants.
     const APInt *C;
-    if (!match(Op1, m_APInt(C)))
+    if (!match(Op1, m_APIntAllowUndef(C)))
       break;
 
     if ((IID == Intrinsic::smax && C->isMaxSignedValue()) ||
         (IID == Intrinsic::smin && C->isMinSignedValue()) ||
         (IID == Intrinsic::umax && C->isMaxValue()) ||
         (IID == Intrinsic::umin && C->isMinValue()))
-      return Op1;
+      return ConstantInt::get(ReturnType, *C);
 
     break;
   }
diff --git a/llvm/test/Transforms/InstSimplify/maxmin_intrinsics.ll b/llvm/test/Transforms/InstSimplify/maxmin_intrinsics.ll
index fee630f9f230a..312056d745a90 100644
--- a/llvm/test/Transforms/InstSimplify/maxmin_intrinsics.ll
+++ b/llvm/test/Transforms/InstSimplify/maxmin_intrinsics.ll
@@ -218,8 +218,7 @@ define <2 x i8> @umin_maxval_commute(<2 x i8> %x) {
 
 define <2 x i8> @smax_maxval_partial_undef(<2 x i8> %x) {
 ; CHECK-LABEL: @smax_maxval_partial_undef(
-; CHECK-NEXT:    [[R:%.*]] = call <2 x i8> @llvm.smax.v2i8(<2 x i8> <i8 undef, i8 127>, <2 x i8> [[X:%.*]])
-; CHECK-NEXT:    ret <2 x i8> [[R]]
+; CHECK-NEXT:    ret <2 x i8> <i8 127, i8 127>
 ;
   %r = call <2 x i8> @llvm.smax.v2i8(<2 x i8> <i8 undef, i8 127>, <2 x i8> %x)
   ret <2 x i8> %r
@@ -227,8 +226,7 @@ define <2 x i8> @smax_maxval_partial_undef(<2 x i8> %x) {
 
 define <2 x i8> @smin_minval_partial_undef(<2 x i8> %x) {
 ; CHECK-LABEL: @smin_minval_partial_undef(
-; CHECK-NEXT:    [[R:%.*]] = call <2 x i8> @llvm.smin.v2i8(<2 x i8> [[X:%.*]], <2 x i8> <i8 -128, i8 undef>)
-; CHECK-NEXT:    ret <2 x i8> [[R]]
+; CHECK-NEXT:    ret <2 x i8> <i8 -128, i8 -128>
 ;
   %r = call <2 x i8> @llvm.smin.v2i8(<2 x i8> %x, <2 x i8> <i8 -128, i8 undef>)
   ret <2 x i8> %r
@@ -236,8 +234,7 @@ define <2 x i8> @smin_minval_partial_undef(<2 x i8> %x) {
 
 define <2 x i8> @umax_maxval_partial_undef(<2 x i8> %x) {
 ; CHECK-LABEL: @umax_maxval_partial_undef(
-; CHECK-NEXT:    [[R:%.*]] = call <2 x i8> @llvm.umax.v2i8(<2 x i8> <i8 -1, i8 undef>, <2 x i8> [[X:%.*]])
-; CHECK-NEXT:    ret <2 x i8> [[R]]
+; CHECK-NEXT:    ret <2 x i8> <i8 -1, i8 -1>
 ;
   %r = call <2 x i8> @llvm.umax.v2i8(<2 x i8> <i8 255, i8 undef>, <2 x i8> %x)
   ret <2 x i8> %r
@@ -245,8 +242,7 @@ define <2 x i8> @umax_maxval_partial_undef(<2 x i8> %x) {
 
 define <2 x i8> @umin_minval_partial_undef(<2 x i8> %x) {
 ; CHECK-LABEL: @umin_minval_partial_undef(
-; CHECK-NEXT:    [[R:%.*]] = call <2 x i8> @llvm.umin.v2i8(<2 x i8> [[X:%.*]], <2 x i8> <i8 undef, i8 0>)
-; CHECK-NEXT:    ret <2 x i8> [[R]]
+; CHECK-NEXT:    ret <2 x i8> zeroinitializer
 ;
   %r = call <2 x i8> @llvm.umin.v2i8(<2 x i8> %x, <2 x i8> <i8 undef, i8 0>)
   ret <2 x i8> %r

From 4a10029d7ed910a74c589de1dd26bca9176e0e62 Mon Sep 17 00:00:00 2001
From: Arthur Eubanks <aeubanks@google.com>
Date: Tue, 28 Jul 2020 17:57:21 -0700
Subject: [PATCH 0486/1035] [NewPM][Attributor] Pin tests with -attributor to
 legacy PM

All these tests already explicitly test against both legacy PM and NPM.

$ sed -i 's/ -attributor / -attributor -enable-new-pm=0 /g' $(rg --path-separator // -l -- -passes=)
$ sed -i 's/ -attributor-cgscc / -attributor-cgscc -enable-new-pm=0 /g' $(rg --path-separator // -l -- -passes=)

Now all tests in Transforms/Attributor/ pass under NPM.

Reviewed By: jdoerfert

Differential Revision: https://reviews.llvm.org/D84813
---
 .../Attributor/ArgumentPromotion/2008-02-01-ReturnAttrs.ll    | 4 ++--
 .../Attributor/ArgumentPromotion/2008-07-02-array-indexing.ll | 4 ++--
 .../Attributor/ArgumentPromotion/2008-09-07-CGUpdate.ll       | 4 ++--
 .../ArgumentPromotion/2008-09-08-CGUpdateSelfEdge.ll          | 4 ++--
 .../Transforms/Attributor/ArgumentPromotion/X86/attributes.ll | 4 ++--
 .../ArgumentPromotion/X86/min-legal-vector-width.ll           | 4 ++--
 .../Transforms/Attributor/ArgumentPromotion/X86/thiscall.ll   | 4 ++--
 .../Attributor/ArgumentPromotion/aggregate-promote.ll         | 4 ++--
 .../test/Transforms/Attributor/ArgumentPromotion/alignment.ll | 4 ++--
 llvm/test/Transforms/Attributor/ArgumentPromotion/attrs.ll    | 4 ++--
 .../test/Transforms/Attributor/ArgumentPromotion/basictest.ll | 4 ++--
 llvm/test/Transforms/Attributor/ArgumentPromotion/byval-2.ll  | 4 ++--
 llvm/test/Transforms/Attributor/ArgumentPromotion/byval.ll    | 4 ++--
 llvm/test/Transforms/Attributor/ArgumentPromotion/chained.ll  | 4 ++--
 .../Transforms/Attributor/ArgumentPromotion/control-flow.ll   | 4 ++--
 .../Transforms/Attributor/ArgumentPromotion/control-flow2.ll  | 4 ++--
 llvm/test/Transforms/Attributor/ArgumentPromotion/crash.ll    | 4 ++--
 llvm/test/Transforms/Attributor/ArgumentPromotion/dbg.ll      | 4 ++--
 llvm/test/Transforms/Attributor/ArgumentPromotion/fp80.ll     | 4 ++--
 llvm/test/Transforms/Attributor/ArgumentPromotion/inalloca.ll | 4 ++--
 .../Attributor/ArgumentPromotion/live_called_from_dead.ll     | 4 ++--
 .../Attributor/ArgumentPromotion/live_called_from_dead_2.ll   | 4 ++--
 llvm/test/Transforms/Attributor/ArgumentPromotion/musttail.ll | 4 ++--
 .../Attributor/ArgumentPromotion/naked_functions.ll           | 4 ++--
 .../Attributor/ArgumentPromotion/nonzero-address-spaces.ll    | 4 ++--
 llvm/test/Transforms/Attributor/ArgumentPromotion/pr27568.ll  | 4 ++--
 llvm/test/Transforms/Attributor/ArgumentPromotion/pr32917.ll  | 4 ++--
 .../ArgumentPromotion/pr33641_remove_arg_dbgvalue.ll          | 4 ++--
 llvm/test/Transforms/Attributor/ArgumentPromotion/profile.ll  | 4 ++--
 .../Transforms/Attributor/ArgumentPromotion/reserve-tbaa.ll   | 4 ++--
 llvm/test/Transforms/Attributor/ArgumentPromotion/sret.ll     | 4 ++--
 llvm/test/Transforms/Attributor/ArgumentPromotion/tail.ll     | 4 ++--
 llvm/test/Transforms/Attributor/ArgumentPromotion/variadic.ll | 4 ++--
 .../Attributor/IPConstantProp/2008-06-09-WeakProp.ll          | 4 ++--
 .../Attributor/IPConstantProp/2009-09-24-byval-ptr.ll         | 4 ++--
 llvm/test/Transforms/Attributor/IPConstantProp/PR16052.ll     | 4 ++--
 llvm/test/Transforms/Attributor/IPConstantProp/PR26044.ll     | 4 ++--
 llvm/test/Transforms/Attributor/IPConstantProp/PR43857.ll     | 4 ++--
 .../Attributor/IPConstantProp/arg-count-mismatch.ll           | 4 ++--
 .../Transforms/Attributor/IPConstantProp/arg-type-mismatch.ll | 4 ++--
 llvm/test/Transforms/Attributor/IPConstantProp/comdat-ipo.ll  | 4 ++--
 .../Attributor/IPConstantProp/dangling-block-address.ll       | 4 ++--
 llvm/test/Transforms/Attributor/IPConstantProp/deadarg.ll     | 4 ++--
 .../Attributor/IPConstantProp/fp-bc-icmp-const-fold.ll        | 4 ++--
 llvm/test/Transforms/Attributor/IPConstantProp/global.ll      | 4 ++--
 .../Attributor/IPConstantProp/multiple_callbacks.ll           | 4 ++--
 .../Transforms/Attributor/IPConstantProp/musttail-call.ll     | 4 ++--
 .../test/Transforms/Attributor/IPConstantProp/naked-return.ll | 4 ++--
 .../Attributor/IPConstantProp/openmp_parallel_for.ll          | 4 ++--
 llvm/test/Transforms/Attributor/IPConstantProp/pthreads.ll    | 4 ++--
 llvm/test/Transforms/Attributor/IPConstantProp/recursion.ll   | 4 ++--
 .../Transforms/Attributor/IPConstantProp/remove-call-inst.ll  | 4 ++--
 .../Transforms/Attributor/IPConstantProp/return-argument.ll   | 4 ++--
 .../Transforms/Attributor/IPConstantProp/return-constant.ll   | 4 ++--
 .../Transforms/Attributor/IPConstantProp/return-constants.ll  | 4 ++--
 .../solve-after-each-resolving-undefs-for-function.ll         | 4 ++--
 .../Transforms/Attributor/IPConstantProp/thread_local_acs.ll  | 4 ++--
 llvm/test/Transforms/Attributor/align.ll                      | 4 ++--
 llvm/test/Transforms/Attributor/alwaysinline.ll               | 4 ++--
 llvm/test/Transforms/Attributor/callbacks.ll                  | 4 ++--
 llvm/test/Transforms/Attributor/dereferenceable-1.ll          | 4 ++--
 llvm/test/Transforms/Attributor/dereferenceable-2.ll          | 4 ++--
 llvm/test/Transforms/Attributor/heap_to_stack.ll              | 4 ++--
 llvm/test/Transforms/Attributor/internal-noalias.ll           | 4 ++--
 llvm/test/Transforms/Attributor/liveness.ll                   | 4 ++--
 llvm/test/Transforms/Attributor/liveness_chains.ll            | 4 ++--
 llvm/test/Transforms/Attributor/lvi-after-jumpthreading.ll    | 4 ++--
 llvm/test/Transforms/Attributor/lvi-for-ashr.ll               | 4 ++--
 llvm/test/Transforms/Attributor/memory_locations.ll           | 4 ++--
 llvm/test/Transforms/Attributor/misc.ll                       | 4 ++--
 llvm/test/Transforms/Attributor/misc_crash.ll                 | 2 +-
 llvm/test/Transforms/Attributor/noalias.ll                    | 4 ++--
 llvm/test/Transforms/Attributor/nocapture-1.ll                | 4 ++--
 llvm/test/Transforms/Attributor/nocapture-2.ll                | 4 ++--
 llvm/test/Transforms/Attributor/nofree.ll                     | 4 ++--
 llvm/test/Transforms/Attributor/nonnull.ll                    | 4 ++--
 llvm/test/Transforms/Attributor/norecurse.ll                  | 4 ++--
 llvm/test/Transforms/Attributor/noreturn.ll                   | 4 ++--
 llvm/test/Transforms/Attributor/nosync.ll                     | 4 ++--
 llvm/test/Transforms/Attributor/nounwind.ll                   | 4 ++--
 llvm/test/Transforms/Attributor/range.ll                      | 4 ++--
 .../Attributor/read_write_returned_arguments_scc.ll           | 4 ++--
 llvm/test/Transforms/Attributor/readattrs.ll                  | 4 ++--
 .../Transforms/Attributor/reduced/register_benchmark_test.ll  | 4 ++--
 llvm/test/Transforms/Attributor/returned.ll                   | 4 ++--
 llvm/test/Transforms/Attributor/undefined_behavior.ll         | 4 ++--
 llvm/test/Transforms/Attributor/value-simplify.ll             | 4 ++--
 llvm/test/Transforms/Attributor/willreturn.ll                 | 4 ++--
 88 files changed, 175 insertions(+), 175 deletions(-)

diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-02-01-ReturnAttrs.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-02-01-ReturnAttrs.ll
index 58135b8d22707..639772d553f6a 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-02-01-ReturnAttrs.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-02-01-ReturnAttrs.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=6 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=6 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=6 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 define internal i32 @deref(i32* %x) nounwind {
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-07-02-array-indexing.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-07-02-array-indexing.ll
index d0817a05a5fdb..822ffc93f612d 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-07-02-array-indexing.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-07-02-array-indexing.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=2 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=2 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=2 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 ; PR2498
 
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-09-07-CGUpdate.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-09-07-CGUpdate.ll
index 68fc3d62e5d2e..a54d2ae57a7cb 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-09-07-CGUpdate.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-09-07-CGUpdate.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 define internal fastcc i32 @hash(i32* %ts, i32 %mod) nounwind {
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-09-08-CGUpdateSelfEdge.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-09-08-CGUpdateSelfEdge.ll
index 5f56e670aab91..42901ddaa7162 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-09-08-CGUpdateSelfEdge.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-09-08-CGUpdateSelfEdge.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 define internal fastcc i32 @term_SharingList(i32* %Term, i32* %List) nounwind {
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/X86/attributes.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/X86/attributes.ll
index 49342ae4f04c0..db349295a54d4 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/X86/attributes.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/X86/attributes.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=4 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=4 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=3 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 ; Test that we only promote arguments when the caller/callee have compatible
 ; function attrubtes.
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/X86/min-legal-vector-width.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/X86/min-legal-vector-width.ll
index 361dfa0cd5923..4274e3c89111c 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/X86/min-legal-vector-width.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/X86/min-legal-vector-width.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=4 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=4 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=3 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 ; Test that we only promote arguments when the caller/callee have compatible
 ; function attrubtes.
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/X86/thiscall.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/X86/thiscall.ll
index 20d340fcb569d..fa289c15cacd5 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/X86/thiscall.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/X86/thiscall.ll
@@ -4,9 +4,9 @@
 ; we don't do that anymore. It also verifies that the combination of
 ; globalopt and argpromotion is able to optimize the call safely.
 ;
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=7 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=7 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=7 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 target datalayout = "e-m:x-p:32:32-i64:64-f80:32-n8:16:32-a:0:32-S32"
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/aggregate-promote.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/aggregate-promote.ll
index 7728b12af97a9..3f5eaac99a6b3 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/aggregate-promote.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/aggregate-promote.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 %T = type { i32, i32, i32, i32 }
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/alignment.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/alignment.ll
index 98f3a5943e37b..33cc4975d5960 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/alignment.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/alignment.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=11 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=11 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=11 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 define void @f() {
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/attrs.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/attrs.ll
index 8ecc12b171af5..e7e67c71cf509 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/attrs.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/attrs.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=4 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=4 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=4 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 %struct.ss = type { i32, i64 }
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/basictest.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/basictest.ll
index 13a4187830c1a..1522dfe907f00 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/basictest.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/basictest.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=8 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=8 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=8 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
 
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/byval-2.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/byval-2.ll
index bd1dfb5a11b7d..3d410cf51bcc1 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/byval-2.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/byval-2.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=3 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=3 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=3 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 %struct.ss = type { i32, i64 }
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/byval.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/byval.ll
index f05d8883be5b9..f87bd4b802eb8 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/byval.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/byval.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=4 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=4 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=4 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/chained.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/chained.ll
index 75e44569407aa..30ed9a5b9fa57 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/chained.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/chained.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 @G1 = constant i32 0
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/control-flow.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/control-flow.ll
index cc66b88e391f2..cd6b510b201bc 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/control-flow.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/control-flow.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 ; Don't promote around control flow.
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/control-flow2.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/control-flow2.ll
index bba3e92959d94..ce997ba494931 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/control-flow2.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/control-flow2.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=8 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=8 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=8 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/crash.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/crash.ll
index dfd40d0b32a0a..f1bd8724afe79 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/crash.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/crash.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=3 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=3 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=3 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 %S = type { %S* }
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/dbg.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/dbg.ll
index 051e486ea70b1..18b816caf881b 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/dbg.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/dbg.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=4 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=4 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=4 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 declare void @sink(i32)
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/fp80.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/fp80.ll
index 295907cbca0b2..3d781e18c4476 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/fp80.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/fp80.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=4 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=4 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=4 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; TODO: The old pass manager cgscc run is disabled as it causes a crash on windows which is under investigation: http://lab.llvm.org:8011/builders/llvm-clang-x86_64-expensive-checks-win/builds/23151
-; opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/inalloca.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/inalloca.ll
index ea13554f587b8..b7ff607c27038 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/inalloca.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/inalloca.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=6 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=6 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=6 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/live_called_from_dead.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/live_called_from_dead.ll
index a53ab08d60ec3..bb14e16820c76 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/live_called_from_dead.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/live_called_from_dead.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=5 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=5 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=5 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/live_called_from_dead_2.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/live_called_from_dead_2.ll
index 6b2b7def5b956..fc1e658949903 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/live_called_from_dead_2.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/live_called_from_dead_2.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=8 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=8 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=9 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/musttail.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/musttail.ll
index 4b266d8a5d17c..df762638250d5 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/musttail.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/musttail.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=5 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=5 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=5 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 ; PR36543
 
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/naked_functions.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/naked_functions.ll
index f566358cef61e..0d3464c062fa2 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/naked_functions.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/naked_functions.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 ; Don't promote paramaters of/arguments to naked functions
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/nonzero-address-spaces.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/nonzero-address-spaces.ll
index 96acd981a162a..a6df2aeece164 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/nonzero-address-spaces.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/nonzero-address-spaces.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 ; ArgumentPromotion should preserve the default function address space
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/pr27568.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/pr27568.ll
index 546bd149a4f0a..587688c5135a9 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/pr27568.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/pr27568.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 target triple = "x86_64-pc-windows-msvc"
 
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/pr32917.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/pr32917.ll
index 40edeef6db927..b390eea528521 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/pr32917.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/pr32917.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=5 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=5 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=5 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 ; PR 32917
 
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/pr33641_remove_arg_dbgvalue.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/pr33641_remove_arg_dbgvalue.ll
index 302f0e2b3b3fa..66217646002c6 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/pr33641_remove_arg_dbgvalue.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/pr33641_remove_arg_dbgvalue.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 ; Fix for PR33641. ArgumentPromotion removed the argument to bar but left the call to
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/profile.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/profile.ll
index bf18851fb7c2c..1c847b88b5286 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/profile.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/profile.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=4 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=4 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=4 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
 
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/reserve-tbaa.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/reserve-tbaa.ll
index a649971612297..8a5cb81c08234 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/reserve-tbaa.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/reserve-tbaa.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=3 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=3 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=3 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 ; PR17906
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/sret.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/sret.ll
index 2c4312c6159c0..a16d6fc49ee3d 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/sret.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/sret.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=4 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=4 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=4 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/tail.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/tail.ll
index 9d5dab8f3be09..ce4f78065d116 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/tail.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/tail.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=2 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=2 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=2 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 ; PR14710
 
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/variadic.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/variadic.ll
index df9b420d11a27..6120b725cc74f 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/variadic.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/variadic.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 ; Unused arguments from variadic functions cannot be eliminated as that changes
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/2008-06-09-WeakProp.ll b/llvm/test/Transforms/Attributor/IPConstantProp/2008-06-09-WeakProp.ll
index ebd06c15bb63d..685f356a6fb04 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/2008-06-09-WeakProp.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/2008-06-09-WeakProp.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 ; Should not propagate the result of a weak function.
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/2009-09-24-byval-ptr.ll b/llvm/test/Transforms/Attributor/IPConstantProp/2009-09-24-byval-ptr.ll
index 2801725d3662c..98051fc678ad3 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/2009-09-24-byval-ptr.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/2009-09-24-byval-ptr.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=6 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=6 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=6 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 ; Don't constant-propagate byval pointers, since they are not pointers!
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/PR16052.ll b/llvm/test/Transforms/Attributor/IPConstantProp/PR16052.ll
index 8e66dcf04c183..ff116da94c932 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/PR16052.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/PR16052.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=2 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=2 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=2 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/PR26044.ll b/llvm/test/Transforms/Attributor/IPConstantProp/PR26044.ll
index ae1071032e984..bb6724253e14f 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/PR26044.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/PR26044.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/PR43857.ll b/llvm/test/Transforms/Attributor/IPConstantProp/PR43857.ll
index 08e664239f318..c993db196ddbe 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/PR43857.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/PR43857.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 %struct.wobble = type { i32 }
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/arg-count-mismatch.ll b/llvm/test/Transforms/Attributor/IPConstantProp/arg-count-mismatch.ll
index 1124b5d012800..4f75f7f3a72b7 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/arg-count-mismatch.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/arg-count-mismatch.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 ; The original C source looked like this:
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/arg-type-mismatch.ll b/llvm/test/Transforms/Attributor/IPConstantProp/arg-type-mismatch.ll
index adc52ab707d17..fd6118d1464cc 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/arg-type-mismatch.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/arg-type-mismatch.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 ; This test is just to verify that we do not crash/assert due to mismatch in
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/comdat-ipo.ll b/llvm/test/Transforms/Attributor/IPConstantProp/comdat-ipo.ll
index 0a57b47508ed0..da20e0ec9f4cb 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/comdat-ipo.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/comdat-ipo.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 ; See PR26774
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/dangling-block-address.ll b/llvm/test/Transforms/Attributor/IPConstantProp/dangling-block-address.ll
index 381e2d58bc796..d1412dd5ad817 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/dangling-block-address.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/dangling-block-address.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 ; PR5569
 
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/deadarg.ll b/llvm/test/Transforms/Attributor/IPConstantProp/deadarg.ll
index 4d78ab0906d3f..0b4c472aac6f4 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/deadarg.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/deadarg.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 define internal void @foo(i32 %X) {
 ; CHECK-LABEL: define {{[^@]+}}@foo
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/fp-bc-icmp-const-fold.ll b/llvm/test/Transforms/Attributor/IPConstantProp/fp-bc-icmp-const-fold.ll
index 67c22f9470779..e0942a7a4cc7e 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/fp-bc-icmp-const-fold.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/fp-bc-icmp-const-fold.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 target datalayout = "E-m:e-i64:64-n32:64"
 target triple = "powerpc64le-unknown-linux"
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/global.ll b/llvm/test/Transforms/Attributor/IPConstantProp/global.ll
index d30c764e5258b..b60153b517714 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/global.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/global.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=4 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=4 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=4 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 @_ZL6test1g = internal global i32 42, align 4
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/multiple_callbacks.ll b/llvm/test/Transforms/Attributor/IPConstantProp/multiple_callbacks.ll
index c1a50c55e8477..e6b8e8317a42a 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/multiple_callbacks.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/multiple_callbacks.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=6 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=6 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=6 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 ;
 ;
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/musttail-call.ll b/llvm/test/Transforms/Attributor/IPConstantProp/musttail-call.ll
index 467d641bac07a..70c89cfff75bf 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/musttail-call.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/musttail-call.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=3 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=3 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=3 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 ; PR36485
 ; musttail call result can't be replaced with a constant, unless the call can be removed
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/naked-return.ll b/llvm/test/Transforms/Attributor/IPConstantProp/naked-return.ll
index e954d34652425..9052db23b7175 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/naked-return.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/naked-return.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=2 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=2 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=2 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 target datalayout = "e-m:x-p:32:32-i64:64-f80:32-n8:16:32-a:0:32-S32"
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/openmp_parallel_for.ll b/llvm/test/Transforms/Attributor/IPConstantProp/openmp_parallel_for.ll
index 00ca0cc92e8b3..fc6fd6fcafc0d 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/openmp_parallel_for.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/openmp_parallel_for.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=7 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=7 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=6 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 ;
 ;    void bar(int, float, double);
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/pthreads.ll b/llvm/test/Transforms/Attributor/IPConstantProp/pthreads.ll
index 90021fdd57560..7ac5b42d74172 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/pthreads.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/pthreads.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=7 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=7 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=7 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 ;
 ;    #include <pthread.h>
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/recursion.ll b/llvm/test/Transforms/Attributor/IPConstantProp/recursion.ll
index 4d9a6ce4600f6..aeb168aa5d664 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/recursion.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/recursion.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 ; CHECK-NOT: %X
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/remove-call-inst.ll b/llvm/test/Transforms/Attributor/IPConstantProp/remove-call-inst.ll
index 63115d831883c..5e92f6326eb61 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/remove-call-inst.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/remove-call-inst.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 ; PR5596
 
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/return-argument.ll b/llvm/test/Transforms/Attributor/IPConstantProp/return-argument.ll
index 933887fb3cc04..f9302dcbaa76f 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/return-argument.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/return-argument.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=9 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=9 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=9 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 ;; This function returns its second argument on all return statements
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/return-constant.ll b/llvm/test/Transforms/Attributor/IPConstantProp/return-constant.ll
index e598d759dc4e9..c928a25aac9ba 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/return-constant.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/return-constant.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 ; FIXME: icmp folding is missing
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/return-constants.ll b/llvm/test/Transforms/Attributor/IPConstantProp/return-constants.ll
index 0934412e9112a..5723189178678 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/return-constants.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/return-constants.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=8 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=8 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=8 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 ;; FIXME: support for extractvalue and insertvalue missing.
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/solve-after-each-resolving-undefs-for-function.ll b/llvm/test/Transforms/Attributor/IPConstantProp/solve-after-each-resolving-undefs-for-function.ll
index 11b65dfe51dc8..21a44c142bbda 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/solve-after-each-resolving-undefs-for-function.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/solve-after-each-resolving-undefs-for-function.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 define internal i32 @testf(i1 %c) {
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/thread_local_acs.ll b/llvm/test/Transforms/Attributor/IPConstantProp/thread_local_acs.ll
index 58dfbaefaa98b..904860e4921c7 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/thread_local_acs.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/thread_local_acs.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 ;
 ;    #include <threads.h>
diff --git a/llvm/test/Transforms/Attributor/align.ll b/llvm/test/Transforms/Attributor/align.ll
index 0ee08d76f5416..b31327b6a8523 100644
--- a/llvm/test/Transforms/Attributor/align.ll
+++ b/llvm/test/Transforms/Attributor/align.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=11 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=11 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=11 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/Attributor/alwaysinline.ll b/llvm/test/Transforms/Attributor/alwaysinline.ll
index a3aaa3782645f..8dd6836a67a58 100644
--- a/llvm/test/Transforms/Attributor/alwaysinline.ll
+++ b/llvm/test/Transforms/Attributor/alwaysinline.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=2 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=2 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=2 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 ;
 ; When a function is marked `alwaysinline` and is able to be inlined,
diff --git a/llvm/test/Transforms/Attributor/callbacks.ll b/llvm/test/Transforms/Attributor/callbacks.ll
index d7fe9dd842614..951f9830ae59e 100644
--- a/llvm/test/Transforms/Attributor/callbacks.ll
+++ b/llvm/test/Transforms/Attributor/callbacks.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=3 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=3 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=3 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/Attributor/dereferenceable-1.ll b/llvm/test/Transforms/Attributor/dereferenceable-1.ll
index 402ed41f91fa6..0e9333fd7fced 100644
--- a/llvm/test/Transforms/Attributor/dereferenceable-1.ll
+++ b/llvm/test/Transforms/Attributor/dereferenceable-1.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=16 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=16 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=16 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 ; FIXME: Figure out why we need 16 iterations here.
 
diff --git a/llvm/test/Transforms/Attributor/dereferenceable-2.ll b/llvm/test/Transforms/Attributor/dereferenceable-2.ll
index af38f5dfd9bcc..8ba77e0f3ca16 100644
--- a/llvm/test/Transforms/Attributor/dereferenceable-2.ll
+++ b/llvm/test/Transforms/Attributor/dereferenceable-2.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=2 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=2 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=2 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 ; Determine dereference-ability before unused loads get deleted:
diff --git a/llvm/test/Transforms/Attributor/heap_to_stack.ll b/llvm/test/Transforms/Attributor/heap_to_stack.ll
index f46da51e2dfdb..54e293e73179f 100644
--- a/llvm/test/Transforms/Attributor/heap_to_stack.ll
+++ b/llvm/test/Transforms/Attributor/heap_to_stack.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=5 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=5 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=5 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 declare noalias i8* @malloc(i64)
diff --git a/llvm/test/Transforms/Attributor/internal-noalias.ll b/llvm/test/Transforms/Attributor/internal-noalias.ll
index 67aa1ec985e95..d4e27d24bc553 100644
--- a/llvm/test/Transforms/Attributor/internal-noalias.ll
+++ b/llvm/test/Transforms/Attributor/internal-noalias.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=9 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=9 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=9 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 define dso_local i32 @visible(i32* noalias %A, i32* noalias %B) #0 {
diff --git a/llvm/test/Transforms/Attributor/liveness.ll b/llvm/test/Transforms/Attributor/liveness.ll
index 04899eb1f7121..c4093489cf87f 100644
--- a/llvm/test/Transforms/Attributor/liveness.ll
+++ b/llvm/test/Transforms/Attributor/liveness.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=17 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=17 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=17 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; TODO: The old pass manager cgscc run is disabled as it causes a crash on windows which is under investigation: http://lab.llvm.org:8011/builders/llvm-clang-x86_64-expensive-checks-win/builds/23151
-; opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 ; NOT_CGSCC_OPM: @dead_with_blockaddress_users.l = constant [2 x i8*] [i8* inttoptr (i32 1 to i8*), i8* inttoptr (i32 1 to i8*)]
diff --git a/llvm/test/Transforms/Attributor/liveness_chains.ll b/llvm/test/Transforms/Attributor/liveness_chains.ll
index 5339ea839e337..11bbfed67d139 100644
--- a/llvm/test/Transforms/Attributor/liveness_chains.ll
+++ b/llvm/test/Transforms/Attributor/liveness_chains.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 ; Make sure we need a single iteration to determine the chains are dead/alive.
diff --git a/llvm/test/Transforms/Attributor/lvi-after-jumpthreading.ll b/llvm/test/Transforms/Attributor/lvi-after-jumpthreading.ll
index 6e74cb2bed613..41f77c729c46c 100644
--- a/llvm/test/Transforms/Attributor/lvi-after-jumpthreading.ll
+++ b/llvm/test/Transforms/Attributor/lvi-after-jumpthreading.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=5 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=5 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=6 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 define i8 @test1(i32 %a, i32 %length) {
diff --git a/llvm/test/Transforms/Attributor/lvi-for-ashr.ll b/llvm/test/Transforms/Attributor/lvi-for-ashr.ll
index a6be5bcc9740f..aae5c1210313e 100644
--- a/llvm/test/Transforms/Attributor/lvi-for-ashr.ll
+++ b/llvm/test/Transforms/Attributor/lvi-for-ashr.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 ; FIXME: DOT should be replaced with 3
diff --git a/llvm/test/Transforms/Attributor/memory_locations.ll b/llvm/test/Transforms/Attributor/memory_locations.ll
index be3f9a0f829bd..23bff12910180 100644
--- a/llvm/test/Transforms/Attributor/memory_locations.ll
+++ b/llvm/test/Transforms/Attributor/memory_locations.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=11 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=11 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=11 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/llvm/test/Transforms/Attributor/misc.ll b/llvm/test/Transforms/Attributor/misc.ll
index 6101380c4bae5..46a0449e5be6e 100644
--- a/llvm/test/Transforms/Attributor/misc.ll
+++ b/llvm/test/Transforms/Attributor/misc.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=6 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=6 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=6 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 ;
 ; Mostly check we do not crash on these uses
diff --git a/llvm/test/Transforms/Attributor/misc_crash.ll b/llvm/test/Transforms/Attributor/misc_crash.ll
index afb3093aeeb77..b49cbd94322d1 100644
--- a/llvm/test/Transforms/Attributor/misc_crash.ll
+++ b/llvm/test/Transforms/Attributor/misc_crash.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -S %s | FileCheck %s
+; RUN: opt -attributor -enable-new-pm=0 -S %s | FileCheck %s
 ; RUN: opt -passes=attributor -S %s | FileCheck %s
 
 @var1 = internal global [1 x i32] undef
diff --git a/llvm/test/Transforms/Attributor/noalias.ll b/llvm/test/Transforms/Attributor/noalias.ll
index d17a98685d72c..5314ea53f5a94 100644
--- a/llvm/test/Transforms/Attributor/noalias.ll
+++ b/llvm/test/Transforms/Attributor/noalias.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=9 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=9 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=9 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; TODO: The old pass manager cgscc run is disabled as it causes a crash on windows which is under investigation: http://lab.llvm.org:8011/builders/llvm-clang-x86_64-expensive-checks-win/builds/23151
-; opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 ; TEST 1 - negative.
diff --git a/llvm/test/Transforms/Attributor/nocapture-1.ll b/llvm/test/Transforms/Attributor/nocapture-1.ll
index 45606b09bec84..1e76019fba3df 100644
--- a/llvm/test/Transforms/Attributor/nocapture-1.ll
+++ b/llvm/test/Transforms/Attributor/nocapture-1.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=6 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=6 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=6 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 @g = global i32* null		; <i32**> [#uses=1]
diff --git a/llvm/test/Transforms/Attributor/nocapture-2.ll b/llvm/test/Transforms/Attributor/nocapture-2.ll
index c16047de065d4..5ed73b74c8693 100644
--- a/llvm/test/Transforms/Attributor/nocapture-2.ll
+++ b/llvm/test/Transforms/Attributor/nocapture-2.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=10 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=10 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=10 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 ;
 ; Test cases specifically designed for the "no-capture" argument attribute.
diff --git a/llvm/test/Transforms/Attributor/nofree.ll b/llvm/test/Transforms/Attributor/nofree.ll
index d19e174797650..953b7fae3dc2f 100644
--- a/llvm/test/Transforms/Attributor/nofree.ll
+++ b/llvm/test/Transforms/Attributor/nofree.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=11 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=11 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=11 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/Attributor/nonnull.ll b/llvm/test/Transforms/Attributor/nonnull.ll
index c67bc3e4f17b7..238efb79e8d77 100644
--- a/llvm/test/Transforms/Attributor/nonnull.ll
+++ b/llvm/test/Transforms/Attributor/nonnull.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=9 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=9 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=9 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 
diff --git a/llvm/test/Transforms/Attributor/norecurse.ll b/llvm/test/Transforms/Attributor/norecurse.ll
index da40ea6c23215..66db334224be5 100644
--- a/llvm/test/Transforms/Attributor/norecurse.ll
+++ b/llvm/test/Transforms/Attributor/norecurse.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=9 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=9 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=9 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 define i32 @leaf() {
diff --git a/llvm/test/Transforms/Attributor/noreturn.ll b/llvm/test/Transforms/Attributor/noreturn.ll
index ea0e2bd6103f7..4d39523cfc3ec 100644
--- a/llvm/test/Transforms/Attributor/noreturn.ll
+++ b/llvm/test/Transforms/Attributor/noreturn.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 ;
 ; Test cases specifically designed for the "no-return" function attribute.
diff --git a/llvm/test/Transforms/Attributor/nosync.ll b/llvm/test/Transforms/Attributor/nosync.ll
index 33614b636f53e..1404cc4b34c23 100644
--- a/llvm/test/Transforms/Attributor/nosync.ll
+++ b/llvm/test/Transforms/Attributor/nosync.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=2 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=2 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=2 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/llvm/test/Transforms/Attributor/nounwind.ll b/llvm/test/Transforms/Attributor/nounwind.ll
index 3eb525a015e7f..30009398790a0 100644
--- a/llvm/test/Transforms/Attributor/nounwind.ll
+++ b/llvm/test/Transforms/Attributor/nounwind.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=3 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=3 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=3 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 ; TEST 1
diff --git a/llvm/test/Transforms/Attributor/range.ll b/llvm/test/Transforms/Attributor/range.ll
index 8f6ccf4c1e5d6..c20e4efc5ecea 100644
--- a/llvm/test/Transforms/Attributor/range.ll
+++ b/llvm/test/Transforms/Attributor/range.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=17 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=17 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=17 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 ; FIXME: CGSCC is not looking at callees and calleers even though it could be allowed.
diff --git a/llvm/test/Transforms/Attributor/read_write_returned_arguments_scc.ll b/llvm/test/Transforms/Attributor/read_write_returned_arguments_scc.ll
index 3c7e8c5ff4d79..7c11450032800 100644
--- a/llvm/test/Transforms/Attributor/read_write_returned_arguments_scc.ll
+++ b/llvm/test/Transforms/Attributor/read_write_returned_arguments_scc.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=16 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=16 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=16 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 ;
diff --git a/llvm/test/Transforms/Attributor/readattrs.ll b/llvm/test/Transforms/Attributor/readattrs.ll
index 058fa0d83d001..2f2c18d293ba0 100644
--- a/llvm/test/Transforms/Attributor/readattrs.ll
+++ b/llvm/test/Transforms/Attributor/readattrs.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=5 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=5 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=5 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 
diff --git a/llvm/test/Transforms/Attributor/reduced/register_benchmark_test.ll b/llvm/test/Transforms/Attributor/reduced/register_benchmark_test.ll
index 462d333911bae..f7a203b1ded0c 100644
--- a/llvm/test/Transforms/Attributor/reduced/register_benchmark_test.ll
+++ b/llvm/test/Transforms/Attributor/reduced/register_benchmark_test.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s
-; RUN: opt -attributor-cgscc  -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s
+; RUN: opt -attributor -enable-new-pm=0  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s
+; RUN: opt -attributor-cgscc -enable-new-pm=0  -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s
 ; RUN: opt -passes=attributor  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s
 ; RUN: opt -passes='attributor-cgscc'  -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s
 
diff --git a/llvm/test/Transforms/Attributor/returned.ll b/llvm/test/Transforms/Attributor/returned.ll
index ab6f73acdcdb0..b0007746592de 100644
--- a/llvm/test/Transforms/Attributor/returned.ll
+++ b/llvm/test/Transforms/Attributor/returned.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=14 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=14 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=14 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 ;
 ; Test cases specifically designed for the "returned" argument attribute.
diff --git a/llvm/test/Transforms/Attributor/undefined_behavior.ll b/llvm/test/Transforms/Attributor/undefined_behavior.ll
index 6894037681749..52761da4b869d 100644
--- a/llvm/test/Transforms/Attributor/undefined_behavior.ll
+++ b/llvm/test/Transforms/Attributor/undefined_behavior.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=5 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=5 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=5 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/Attributor/value-simplify.ll b/llvm/test/Transforms/Attributor/value-simplify.ll
index 24f3954b81652..a3401d2ccb91a 100644
--- a/llvm/test/Transforms/Attributor/value-simplify.ll
+++ b/llvm/test/Transforms/Attributor/value-simplify.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=4 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=4 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=4 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/Attributor/willreturn.ll b/llvm/test/Transforms/Attributor/willreturn.ll
index c568b2a956d5c..455f0bd0a192a 100644
--- a/llvm/test/Transforms/Attributor/willreturn.ll
+++ b/llvm/test/Transforms/Attributor/willreturn.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=8 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=8 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=8 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 target datalayout = "e-m:e-i54:64-f80:128-n8:16:32:64-S128"

From 9f2f3b9de6314a009322b6081c792ebf9a469460 Mon Sep 17 00:00:00 2001
From: "Joel E. Denny" <jdenny.ornl@gmail.com>
Date: Wed, 29 Jul 2020 12:18:45 -0400
Subject: [PATCH 0487/1035] [OpenMP] Implement TR8 `present` motion modifier in
 Clang (1/2)

This patch implements Clang front end support for the OpenMP TR8
`present` motion modifier for `omp target update` directives.  The
next patch in this series implements OpenMP runtime support.

Reviewed By: ABataev

Differential Revision: https://reviews.llvm.org/D84711
---
 clang/include/clang/AST/OpenMPClause.h        | 187 +++++++++++++-
 .../clang/Basic/DiagnosticSemaKinds.td        |   2 +
 clang/include/clang/Basic/OpenMPKinds.def     |   1 +
 clang/include/clang/Basic/OpenMPKinds.h       |   4 +
 clang/include/clang/Parse/Parser.h            |   3 +
 clang/include/clang/Sema/Sema.h               |  23 +-
 clang/lib/AST/OpenMPClause.cpp                |  94 +++----
 clang/lib/Basic/OpenMPKinds.cpp               |   8 +-
 clang/lib/CodeGen/CGOpenMPRuntime.cpp         |  85 ++++---
 clang/lib/Parse/ParseOpenMP.cpp               |  45 +++-
 clang/lib/Sema/SemaOpenMP.cpp                 |  78 ++++--
 clang/lib/Sema/TreeTransform.h                |  42 ++--
 clang/lib/Serialization/ASTReader.cpp         |  12 +
 clang/lib/Serialization/ASTWriter.cpp         |  10 +
 clang/test/OpenMP/declare_mapper_ast_print.c  |  14 ++
 clang/test/OpenMP/declare_mapper_codegen.cpp  | 238 ++++++++++++++++++
 clang/test/OpenMP/target_update_ast_print.cpp |  24 ++
 clang/test/OpenMP/target_update_codegen.cpp   |  63 +++++
 clang/test/OpenMP/target_update_messages.cpp  | 117 ++++++++-
 19 files changed, 898 insertions(+), 152 deletions(-)

diff --git a/clang/include/clang/AST/OpenMPClause.h b/clang/include/clang/AST/OpenMPClause.h
index 4f94aa7074ee7..5b588f4b5740d 100644
--- a/clang/include/clang/AST/OpenMPClause.h
+++ b/clang/include/clang/AST/OpenMPClause.h
@@ -6329,8 +6329,20 @@ class OMPToClause final : public OMPMappableExprListClause<OMPToClause>,
   friend OMPVarListClause;
   friend TrailingObjects;
 
+  /// Motion-modifiers for the 'to' clause.
+  OpenMPMotionModifierKind MotionModifiers[NumberOfOMPMotionModifiers] = {
+      OMPC_MOTION_MODIFIER_unknown, OMPC_MOTION_MODIFIER_unknown};
+
+  /// Location of motion-modifiers for the 'to' clause.
+  SourceLocation MotionModifiersLoc[NumberOfOMPMotionModifiers];
+
+  /// Colon location.
+  SourceLocation ColonLoc;
+
   /// Build clause with number of variables \a NumVars.
   ///
+  /// \param TheMotionModifiers Motion-modifiers.
+  /// \param TheMotionModifiersLoc Locations of motion-modifiers.
   /// \param MapperQualifierLoc C++ nested name specifier for the associated
   /// user-defined mapper.
   /// \param MapperIdInfo The identifier of associated user-defined mapper.
@@ -6342,13 +6354,24 @@ class OMPToClause final : public OMPMappableExprListClause<OMPToClause>,
   /// NumUniqueDeclarations: number of unique base declarations in this clause;
   /// 3) NumComponentLists: number of component lists in this clause; and 4)
   /// NumComponents: total number of expression components in the clause.
-  explicit OMPToClause(NestedNameSpecifierLoc MapperQualifierLoc,
+  explicit OMPToClause(ArrayRef<OpenMPMotionModifierKind> TheMotionModifiers,
+                       ArrayRef<SourceLocation> TheMotionModifiersLoc,
+                       NestedNameSpecifierLoc MapperQualifierLoc,
                        DeclarationNameInfo MapperIdInfo,
                        const OMPVarListLocTy &Locs,
                        const OMPMappableExprListSizeTy &Sizes)
       : OMPMappableExprListClause(llvm::omp::OMPC_to, Locs, Sizes,
                                   /*SupportsMapper=*/true, &MapperQualifierLoc,
-                                  &MapperIdInfo) {}
+                                  &MapperIdInfo) {
+    assert(llvm::array_lengthof(MotionModifiers) == TheMotionModifiers.size() &&
+           "Unexpected number of motion modifiers.");
+    llvm::copy(TheMotionModifiers, std::begin(MotionModifiers));
+
+    assert(llvm::array_lengthof(MotionModifiersLoc) ==
+               TheMotionModifiersLoc.size() &&
+           "Unexpected number of motion modifier locations.");
+    llvm::copy(TheMotionModifiersLoc, std::begin(MotionModifiersLoc));
+  }
 
   /// Build an empty clause.
   ///
@@ -6361,6 +6384,29 @@ class OMPToClause final : public OMPMappableExprListClause<OMPToClause>,
       : OMPMappableExprListClause(llvm::omp::OMPC_to, OMPVarListLocTy(), Sizes,
                                   /*SupportsMapper=*/true) {}
 
+  /// Set motion-modifier for the clause.
+  ///
+  /// \param I index for motion-modifier.
+  /// \param T motion-modifier for the clause.
+  void setMotionModifier(unsigned I, OpenMPMotionModifierKind T) {
+    assert(I < NumberOfOMPMotionModifiers &&
+           "Unexpected index to store motion modifier, exceeds array size.");
+    MotionModifiers[I] = T;
+  }
+
+  /// Set location for the motion-modifier.
+  ///
+  /// \param I index for motion-modifier location.
+  /// \param TLoc motion-modifier location.
+  void setMotionModifierLoc(unsigned I, SourceLocation TLoc) {
+    assert(I < NumberOfOMPMotionModifiers &&
+           "Index to store motion modifier location exceeds array size.");
+    MotionModifiersLoc[I] = TLoc;
+  }
+
+  /// Set colon location.
+  void setColonLoc(SourceLocation Loc) { ColonLoc = Loc; }
+
   /// Define the sizes of each trailing object array except the last one. This
   /// is required for TrailingObjects to work properly.
   size_t numTrailingObjects(OverloadToken<Expr *>) const {
@@ -6385,6 +6431,8 @@ class OMPToClause final : public OMPMappableExprListClause<OMPToClause>,
   /// \param Vars The original expression used in the clause.
   /// \param Declarations Declarations used in the clause.
   /// \param ComponentLists Component lists used in the clause.
+  /// \param MotionModifiers Motion-modifiers.
+  /// \param MotionModifiersLoc Location of motion-modifiers.
   /// \param UDMapperRefs References to user-defined mappers associated with
   /// expressions used in the clause.
   /// \param UDMQualifierLoc C++ nested name specifier for the associated
@@ -6395,6 +6443,8 @@ class OMPToClause final : public OMPMappableExprListClause<OMPToClause>,
                              ArrayRef<ValueDecl *> Declarations,
                              MappableExprComponentListsRef ComponentLists,
                              ArrayRef<Expr *> UDMapperRefs,
+                             ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
+                             ArrayRef<SourceLocation> MotionModifiersLoc,
                              NestedNameSpecifierLoc UDMQualifierLoc,
                              DeclarationNameInfo MapperId);
 
@@ -6409,6 +6459,38 @@ class OMPToClause final : public OMPMappableExprListClause<OMPToClause>,
   static OMPToClause *CreateEmpty(const ASTContext &C,
                                   const OMPMappableExprListSizeTy &Sizes);
 
+  /// Fetches the motion-modifier at 'Cnt' index of array of modifiers.
+  ///
+  /// \param Cnt index for motion-modifier.
+  OpenMPMotionModifierKind getMotionModifier(unsigned Cnt) const LLVM_READONLY {
+    assert(Cnt < NumberOfOMPMotionModifiers &&
+           "Requested modifier exceeds the total number of modifiers.");
+    return MotionModifiers[Cnt];
+  }
+
+  /// Fetches the motion-modifier location at 'Cnt' index of array of modifiers'
+  /// locations.
+  ///
+  /// \param Cnt index for motion-modifier location.
+  SourceLocation getMotionModifierLoc(unsigned Cnt) const LLVM_READONLY {
+    assert(Cnt < NumberOfOMPMotionModifiers &&
+           "Requested modifier location exceeds total number of modifiers.");
+    return MotionModifiersLoc[Cnt];
+  }
+
+  /// Fetches ArrayRef of motion-modifiers.
+  ArrayRef<OpenMPMotionModifierKind> getMotionModifiers() const LLVM_READONLY {
+    return llvm::makeArrayRef(MotionModifiers);
+  }
+
+  /// Fetches ArrayRef of location of motion-modifiers.
+  ArrayRef<SourceLocation> getMotionModifiersLoc() const LLVM_READONLY {
+    return llvm::makeArrayRef(MotionModifiersLoc);
+  }
+
+  /// Get colon location.
+  SourceLocation getColonLoc() const { return ColonLoc; }
+
   child_range children() {
     return child_range(reinterpret_cast<Stmt **>(varlist_begin()),
                        reinterpret_cast<Stmt **>(varlist_end()));
@@ -6449,8 +6531,20 @@ class OMPFromClause final
   friend OMPVarListClause;
   friend TrailingObjects;
 
+  /// Motion-modifiers for the 'from' clause.
+  OpenMPMotionModifierKind MotionModifiers[NumberOfOMPMotionModifiers] = {
+      OMPC_MOTION_MODIFIER_unknown, OMPC_MOTION_MODIFIER_unknown};
+
+  /// Location of motion-modifiers for the 'from' clause.
+  SourceLocation MotionModifiersLoc[NumberOfOMPMotionModifiers];
+
+  /// Colon location.
+  SourceLocation ColonLoc;
+
   /// Build clause with number of variables \a NumVars.
   ///
+  /// \param TheMotionModifiers Motion-modifiers.
+  /// \param TheMotionModifiersLoc Locations of motion-modifiers.
   /// \param MapperQualifierLoc C++ nested name specifier for the associated
   /// user-defined mapper.
   /// \param MapperIdInfo The identifier of associated user-defined mapper.
@@ -6462,13 +6556,24 @@ class OMPFromClause final
   /// NumUniqueDeclarations: number of unique base declarations in this clause;
   /// 3) NumComponentLists: number of component lists in this clause; and 4)
   /// NumComponents: total number of expression components in the clause.
-  explicit OMPFromClause(NestedNameSpecifierLoc MapperQualifierLoc,
+  explicit OMPFromClause(ArrayRef<OpenMPMotionModifierKind> TheMotionModifiers,
+                         ArrayRef<SourceLocation> TheMotionModifiersLoc,
+                         NestedNameSpecifierLoc MapperQualifierLoc,
                          DeclarationNameInfo MapperIdInfo,
                          const OMPVarListLocTy &Locs,
                          const OMPMappableExprListSizeTy &Sizes)
       : OMPMappableExprListClause(llvm::omp::OMPC_from, Locs, Sizes,
                                   /*SupportsMapper=*/true, &MapperQualifierLoc,
-                                  &MapperIdInfo) {}
+                                  &MapperIdInfo) {
+    assert(llvm::array_lengthof(MotionModifiers) == TheMotionModifiers.size() &&
+           "Unexpected number of motion modifiers.");
+    llvm::copy(TheMotionModifiers, std::begin(MotionModifiers));
+
+    assert(llvm::array_lengthof(MotionModifiersLoc) ==
+               TheMotionModifiersLoc.size() &&
+           "Unexpected number of motion modifier locations.");
+    llvm::copy(TheMotionModifiersLoc, std::begin(MotionModifiersLoc));
+  }
 
   /// Build an empty clause.
   ///
@@ -6481,6 +6586,29 @@ class OMPFromClause final
       : OMPMappableExprListClause(llvm::omp::OMPC_from, OMPVarListLocTy(),
                                   Sizes, /*SupportsMapper=*/true) {}
 
+  /// Set motion-modifier for the clause.
+  ///
+  /// \param I index for motion-modifier.
+  /// \param T motion-modifier for the clause.
+  void setMotionModifier(unsigned I, OpenMPMotionModifierKind T) {
+    assert(I < NumberOfOMPMotionModifiers &&
+           "Unexpected index to store motion modifier, exceeds array size.");
+    MotionModifiers[I] = T;
+  }
+
+  /// Set location for the motion-modifier.
+  ///
+  /// \param I index for motion-modifier location.
+  /// \param TLoc motion-modifier location.
+  void setMotionModifierLoc(unsigned I, SourceLocation TLoc) {
+    assert(I < NumberOfOMPMotionModifiers &&
+           "Index to store motion modifier location exceeds array size.");
+    MotionModifiersLoc[I] = TLoc;
+  }
+
+  /// Set colon location.
+  void setColonLoc(SourceLocation Loc) { ColonLoc = Loc; }
+
   /// Define the sizes of each trailing object array except the last one. This
   /// is required for TrailingObjects to work properly.
   size_t numTrailingObjects(OverloadToken<Expr *>) const {
@@ -6505,18 +6633,21 @@ class OMPFromClause final
   /// \param Vars The original expression used in the clause.
   /// \param Declarations Declarations used in the clause.
   /// \param ComponentLists Component lists used in the clause.
+  /// \param MotionModifiers Motion-modifiers.
+  /// \param MotionModifiersLoc Location of motion-modifiers.
   /// \param UDMapperRefs References to user-defined mappers associated with
   /// expressions used in the clause.
   /// \param UDMQualifierLoc C++ nested name specifier for the associated
   /// user-defined mapper.
   /// \param MapperId The identifier of associated user-defined mapper.
-  static OMPFromClause *Create(const ASTContext &C, const OMPVarListLocTy &Locs,
-                               ArrayRef<Expr *> Vars,
-                               ArrayRef<ValueDecl *> Declarations,
-                               MappableExprComponentListsRef ComponentLists,
-                               ArrayRef<Expr *> UDMapperRefs,
-                               NestedNameSpecifierLoc UDMQualifierLoc,
-                               DeclarationNameInfo MapperId);
+  static OMPFromClause *
+  Create(const ASTContext &C, const OMPVarListLocTy &Locs,
+         ArrayRef<Expr *> Vars, ArrayRef<ValueDecl *> Declarations,
+         MappableExprComponentListsRef ComponentLists,
+         ArrayRef<Expr *> UDMapperRefs,
+         ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
+         ArrayRef<SourceLocation> MotionModifiersLoc,
+         NestedNameSpecifierLoc UDMQualifierLoc, DeclarationNameInfo MapperId);
 
   /// Creates an empty clause with the place for \a NumVars variables.
   ///
@@ -6529,6 +6660,38 @@ class OMPFromClause final
   static OMPFromClause *CreateEmpty(const ASTContext &C,
                                     const OMPMappableExprListSizeTy &Sizes);
 
+  /// Fetches the motion-modifier at 'Cnt' index of array of modifiers.
+  ///
+  /// \param Cnt index for motion-modifier.
+  OpenMPMotionModifierKind getMotionModifier(unsigned Cnt) const LLVM_READONLY {
+    assert(Cnt < NumberOfOMPMotionModifiers &&
+           "Requested modifier exceeds the total number of modifiers.");
+    return MotionModifiers[Cnt];
+  }
+
+  /// Fetches the motion-modifier location at 'Cnt' index of array of modifiers'
+  /// locations.
+  ///
+  /// \param Cnt index for motion-modifier location.
+  SourceLocation getMotionModifierLoc(unsigned Cnt) const LLVM_READONLY {
+    assert(Cnt < NumberOfOMPMotionModifiers &&
+           "Requested modifier location exceeds total number of modifiers.");
+    return MotionModifiersLoc[Cnt];
+  }
+
+  /// Fetches ArrayRef of motion-modifiers.
+  ArrayRef<OpenMPMotionModifierKind> getMotionModifiers() const LLVM_READONLY {
+    return llvm::makeArrayRef(MotionModifiers);
+  }
+
+  /// Fetches ArrayRef of location of motion-modifiers.
+  ArrayRef<SourceLocation> getMotionModifiersLoc() const LLVM_READONLY {
+    return llvm::makeArrayRef(MotionModifiersLoc);
+  }
+
+  /// Get colon location.
+  SourceLocation getColonLoc() const { return ColonLoc; }
+
   child_range children() {
     return child_range(reinterpret_cast<Stmt **>(varlist_begin()),
                        reinterpret_cast<Stmt **>(varlist_end()));
@@ -7623,6 +7786,8 @@ class OMPClausePrinter final : public OMPClauseVisitor<OMPClausePrinter> {
 
   /// Process clauses with list of variables.
   template <typename T> void VisitOMPClauseList(T *Node, char StartSym);
+  /// Process motion clauses.
+  template <typename T> void VisitOMPMotionClause(T *Node);
 
 public:
   OMPClausePrinter(raw_ostream &OS, const PrintingPolicy &Policy)
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index ae693a08108c1..794ef23f07619 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -9805,6 +9805,8 @@ def err_omp_multiple_array_items_in_map_clause : Error<
   "multiple array elements associated with the same variable are not allowed in map clauses of the same construct">;
 def err_omp_duplicate_map_type_modifier : Error<
   "same map type modifier has been specified more than once">;
+def err_omp_duplicate_motion_modifier : Error<
+  "same motion modifier has been specified more than once">;
 def err_omp_pointer_mapped_along_with_derived_section : Error<
   "pointer cannot be mapped along with a section derived from itself">;
 def err_omp_original_storage_is_shared_and_does_not_contain : Error<
diff --git a/clang/include/clang/Basic/OpenMPKinds.def b/clang/include/clang/Basic/OpenMPKinds.def
index 04ecbeaaa03e2..59bb331a7a8a6 100644
--- a/clang/include/clang/Basic/OpenMPKinds.def
+++ b/clang/include/clang/Basic/OpenMPKinds.def
@@ -125,6 +125,7 @@ OPENMP_MAP_MODIFIER_KIND(present)
 
 // Modifiers for 'to' or 'from' clause.
 OPENMP_MOTION_MODIFIER_KIND(mapper)
+OPENMP_MOTION_MODIFIER_KIND(present)
 
 // Static attributes for 'dist_schedule' clause.
 OPENMP_DIST_SCHEDULE_KIND(static)
diff --git a/clang/include/clang/Basic/OpenMPKinds.h b/clang/include/clang/Basic/OpenMPKinds.h
index 3e9b5c4a8b14a..0f37dc9ad997d 100644
--- a/clang/include/clang/Basic/OpenMPKinds.h
+++ b/clang/include/clang/Basic/OpenMPKinds.h
@@ -94,6 +94,10 @@ enum OpenMPMotionModifierKind {
   OMPC_MOTION_MODIFIER_unknown
 };
 
+/// Number of allowed motion-modifiers.
+static constexpr unsigned NumberOfOMPMotionModifiers =
+    OMPC_MOTION_MODIFIER_unknown;
+
 /// OpenMP attributes for 'dist_schedule' clause.
 enum OpenMPDistScheduleClauseKind {
 #define OPENMP_DIST_SCHEDULE_KIND(Name) OMPC_DIST_SCHEDULE_##Name,
diff --git a/clang/include/clang/Parse/Parser.h b/clang/include/clang/Parse/Parser.h
index e809d87b59a0f..4068e6a444c9c 100644
--- a/clang/include/clang/Parse/Parser.h
+++ b/clang/include/clang/Parse/Parser.h
@@ -3221,6 +3221,9 @@ class Parser : public CodeCompletionHandler {
     MapTypeModifiers;
     SmallVector<SourceLocation, NumberOfOMPMapClauseModifiers>
     MapTypeModifiersLoc;
+    SmallVector<OpenMPMotionModifierKind, NumberOfOMPMotionModifiers>
+        MotionModifiers;
+    SmallVector<SourceLocation, NumberOfOMPMotionModifiers> MotionModifiersLoc;
     bool IsMapTypeImplicit = false;
     SourceLocation ExtraModifierLoc;
   };
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index fd4300f563a98..4721ccdf14590 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -10725,7 +10725,9 @@ class Sema final {
       DeclarationNameInfo &ReductionOrMapperId, int ExtraModifier,
       ArrayRef<OpenMPMapModifierKind> MapTypeModifiers,
       ArrayRef<SourceLocation> MapTypeModifiersLoc, bool IsMapTypeImplicit,
-      SourceLocation ExtraModifierLoc);
+      SourceLocation ExtraModifierLoc,
+      ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
+      ArrayRef<SourceLocation> MotionModifiersLoc);
   /// Called on well-formed 'inclusive' clause.
   OMPClause *ActOnOpenMPInclusiveClause(ArrayRef<Expr *> VarList,
                                         SourceLocation StartLoc,
@@ -10862,15 +10864,20 @@ class Sema final {
       SourceLocation KindLoc, SourceLocation EndLoc);
   /// Called on well-formed 'to' clause.
   OMPClause *
-  ActOnOpenMPToClause(ArrayRef<Expr *> VarList, CXXScopeSpec &MapperIdScopeSpec,
-                      DeclarationNameInfo &MapperId,
-                      const OMPVarListLocTy &Locs,
+  ActOnOpenMPToClause(ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
+                      ArrayRef<SourceLocation> MotionModifiersLoc,
+                      CXXScopeSpec &MapperIdScopeSpec,
+                      DeclarationNameInfo &MapperId, SourceLocation ColonLoc,
+                      ArrayRef<Expr *> VarList, const OMPVarListLocTy &Locs,
                       ArrayRef<Expr *> UnresolvedMappers = llvm::None);
   /// Called on well-formed 'from' clause.
-  OMPClause *ActOnOpenMPFromClause(
-      ArrayRef<Expr *> VarList, CXXScopeSpec &MapperIdScopeSpec,
-      DeclarationNameInfo &MapperId, const OMPVarListLocTy &Locs,
-      ArrayRef<Expr *> UnresolvedMappers = llvm::None);
+  OMPClause *
+  ActOnOpenMPFromClause(ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
+                        ArrayRef<SourceLocation> MotionModifiersLoc,
+                        CXXScopeSpec &MapperIdScopeSpec,
+                        DeclarationNameInfo &MapperId, SourceLocation ColonLoc,
+                        ArrayRef<Expr *> VarList, const OMPVarListLocTy &Locs,
+                        ArrayRef<Expr *> UnresolvedMappers = llvm::None);
   /// Called on well-formed 'use_device_ptr' clause.
   OMPClause *ActOnOpenMPUseDevicePtrClause(ArrayRef<Expr *> VarList,
                                            const OMPVarListLocTy &Locs);
diff --git a/clang/lib/AST/OpenMPClause.cpp b/clang/lib/AST/OpenMPClause.cpp
index 9caa691188fdb..e846d325560d0 100644
--- a/clang/lib/AST/OpenMPClause.cpp
+++ b/clang/lib/AST/OpenMPClause.cpp
@@ -1098,6 +1098,8 @@ OMPToClause *OMPToClause::Create(
     const ASTContext &C, const OMPVarListLocTy &Locs, ArrayRef<Expr *> Vars,
     ArrayRef<ValueDecl *> Declarations,
     MappableExprComponentListsRef ComponentLists, ArrayRef<Expr *> UDMapperRefs,
+    ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
+    ArrayRef<SourceLocation> MotionModifiersLoc,
     NestedNameSpecifierLoc UDMQualifierLoc, DeclarationNameInfo MapperId) {
   OMPMappableExprListSizeTy Sizes;
   Sizes.NumVars = Vars.size();
@@ -1122,7 +1124,8 @@ OMPToClause *OMPToClause::Create(
           Sizes.NumUniqueDeclarations + Sizes.NumComponentLists,
           Sizes.NumComponents));
 
-  auto *Clause = new (Mem) OMPToClause(UDMQualifierLoc, MapperId, Locs, Sizes);
+  auto *Clause = new (Mem) OMPToClause(MotionModifiers, MotionModifiersLoc,
+                                       UDMQualifierLoc, MapperId, Locs, Sizes);
 
   Clause->setVarRefs(Vars);
   Clause->setUDMapperRefs(UDMapperRefs);
@@ -1145,6 +1148,8 @@ OMPFromClause *OMPFromClause::Create(
     const ASTContext &C, const OMPVarListLocTy &Locs, ArrayRef<Expr *> Vars,
     ArrayRef<ValueDecl *> Declarations,
     MappableExprComponentListsRef ComponentLists, ArrayRef<Expr *> UDMapperRefs,
+    ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
+    ArrayRef<SourceLocation> MotionModifiersLoc,
     NestedNameSpecifierLoc UDMQualifierLoc, DeclarationNameInfo MapperId) {
   OMPMappableExprListSizeTy Sizes;
   Sizes.NumVars = Vars.size();
@@ -1170,7 +1175,8 @@ OMPFromClause *OMPFromClause::Create(
           Sizes.NumComponents));
 
   auto *Clause =
-      new (Mem) OMPFromClause(UDMQualifierLoc, MapperId, Locs, Sizes);
+      new (Mem) OMPFromClause(MotionModifiers, MotionModifiersLoc,
+                              UDMQualifierLoc, MapperId, Locs, Sizes);
 
   Clause->setVarRefs(Vars);
   Clause->setUDMapperRefs(UDMapperRefs);
@@ -1937,6 +1943,17 @@ void OMPClausePrinter::VisitOMPDependClause(OMPDependClause *Node) {
   OS << ")";
 }
 
+template <typename T>
+static void PrintMapper(raw_ostream &OS, T *Node,
+                        const PrintingPolicy &Policy) {
+  OS << '(';
+  NestedNameSpecifier *MapperNNS =
+      Node->getMapperQualifierLoc().getNestedNameSpecifier();
+  if (MapperNNS)
+    MapperNNS->print(OS, Policy);
+  OS << Node->getMapperIdInfo() << ')';
+}
+
 void OMPClausePrinter::VisitOMPMapClause(OMPMapClause *Node) {
   if (!Node->varlist_empty()) {
     OS << "map(";
@@ -1945,14 +1962,8 @@ void OMPClausePrinter::VisitOMPMapClause(OMPMapClause *Node) {
         if (Node->getMapTypeModifier(I) != OMPC_MAP_MODIFIER_unknown) {
           OS << getOpenMPSimpleClauseTypeName(OMPC_map,
                                               Node->getMapTypeModifier(I));
-          if (Node->getMapTypeModifier(I) == OMPC_MAP_MODIFIER_mapper) {
-            OS << '(';
-            NestedNameSpecifier *MapperNNS =
-                Node->getMapperQualifierLoc().getNestedNameSpecifier();
-            if (MapperNNS)
-              MapperNNS->print(OS, Policy);
-            OS << Node->getMapperIdInfo() << ')';
-          }
+          if (Node->getMapTypeModifier(I) == OMPC_MAP_MODIFIER_mapper)
+            PrintMapper(OS, Node, Policy);
           OS << ',';
         }
       }
@@ -1964,44 +1975,41 @@ void OMPClausePrinter::VisitOMPMapClause(OMPMapClause *Node) {
   }
 }
 
-void OMPClausePrinter::VisitOMPToClause(OMPToClause *Node) {
-  if (!Node->varlist_empty()) {
-    OS << "to";
-    DeclarationNameInfo MapperId = Node->getMapperIdInfo();
-    if (MapperId.getName() && !MapperId.getName().isEmpty()) {
-      OS << '(';
-      OS << "mapper(";
-      NestedNameSpecifier *MapperNNS =
-          Node->getMapperQualifierLoc().getNestedNameSpecifier();
-      if (MapperNNS)
-        MapperNNS->print(OS, Policy);
-      OS << MapperId << "):";
-      VisitOMPClauseList(Node, ' ');
-    } else {
-      VisitOMPClauseList(Node, '(');
+template <typename T> void OMPClausePrinter::VisitOMPMotionClause(T *Node) {
+  if (Node->varlist_empty())
+    return;
+  OS << getOpenMPClauseName(Node->getClauseKind());
+  unsigned ModifierCount = 0;
+  for (unsigned I = 0; I < NumberOfOMPMotionModifiers; ++I) {
+    if (Node->getMotionModifier(I) != OMPC_MOTION_MODIFIER_unknown)
+      ++ModifierCount;
+  }
+  if (ModifierCount) {
+    OS << '(';
+    for (unsigned I = 0; I < NumberOfOMPMotionModifiers; ++I) {
+      if (Node->getMotionModifier(I) != OMPC_MOTION_MODIFIER_unknown) {
+        OS << getOpenMPSimpleClauseTypeName(Node->getClauseKind(),
+                                            Node->getMotionModifier(I));
+        if (Node->getMotionModifier(I) == OMPC_MOTION_MODIFIER_mapper)
+          PrintMapper(OS, Node, Policy);
+        if (I < ModifierCount - 1)
+          OS << ", ";
+      }
     }
-    OS << ")";
+    OS << ':';
+    VisitOMPClauseList(Node, ' ');
+  } else {
+    VisitOMPClauseList(Node, '(');
   }
+  OS << ")";
+}
+
+void OMPClausePrinter::VisitOMPToClause(OMPToClause *Node) {
+  VisitOMPMotionClause(Node);
 }
 
 void OMPClausePrinter::VisitOMPFromClause(OMPFromClause *Node) {
-  if (!Node->varlist_empty()) {
-    OS << "from";
-    DeclarationNameInfo MapperId = Node->getMapperIdInfo();
-    if (MapperId.getName() && !MapperId.getName().isEmpty()) {
-      OS << '(';
-      OS << "mapper(";
-      NestedNameSpecifier *MapperNNS =
-          Node->getMapperQualifierLoc().getNestedNameSpecifier();
-      if (MapperNNS)
-        MapperNNS->print(OS, Policy);
-      OS << MapperId << "):";
-      VisitOMPClauseList(Node, ' ');
-    } else {
-      VisitOMPClauseList(Node, '(');
-    }
-    OS << ")";
-  }
+  VisitOMPMotionClause(Node);
 }
 
 void OMPClausePrinter::VisitOMPDistScheduleClause(OMPDistScheduleClause *Node) {
diff --git a/clang/lib/Basic/OpenMPKinds.cpp b/clang/lib/Basic/OpenMPKinds.cpp
index da362f99ed297..5c19d60cbd6e3 100644
--- a/clang/lib/Basic/OpenMPKinds.cpp
+++ b/clang/lib/Basic/OpenMPKinds.cpp
@@ -64,12 +64,16 @@ unsigned clang::getOpenMPSimpleClauseType(OpenMPClauseKind Kind, StringRef Str,
     return Type;
   }
   case OMPC_to:
-  case OMPC_from:
-    return llvm::StringSwitch<unsigned>(Str)
+  case OMPC_from: {
+    unsigned Type = llvm::StringSwitch<unsigned>(Str)
 #define OPENMP_MOTION_MODIFIER_KIND(Name)                                      \
   .Case(#Name, static_cast<unsigned>(OMPC_MOTION_MODIFIER_##Name))
 #include "clang/Basic/OpenMPKinds.def"
         .Default(OMPC_MOTION_MODIFIER_unknown);
+    if (OpenMPVersion < 51 && Type == OMPC_MOTION_MODIFIER_present)
+      return OMPC_MOTION_MODIFIER_unknown;
+    return Type;
+  }
   case OMPC_dist_schedule:
     return llvm::StringSwitch<OpenMPDistScheduleClauseKind>(Str)
 #define OPENMP_DIST_SCHEDULE_KIND(Name) .Case(#Name, OMPC_DIST_SCHEDULE_##Name)
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
index f16f2ed89de89..b8e33948c21ca 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -7122,6 +7122,7 @@ class MappableExprsHandler {
     OMPClauseMappableExprCommon::MappableExprComponentListRef Components;
     OpenMPMapClauseKind MapType = OMPC_MAP_unknown;
     ArrayRef<OpenMPMapModifierKind> MapModifiers;
+    ArrayRef<OpenMPMotionModifierKind> MotionModifiers;
     bool ReturnDevicePointer = false;
     bool IsImplicit = false;
     const ValueDecl *Mapper = nullptr;
@@ -7131,10 +7132,12 @@ class MappableExprsHandler {
     MapInfo(
         OMPClauseMappableExprCommon::MappableExprComponentListRef Components,
         OpenMPMapClauseKind MapType,
-        ArrayRef<OpenMPMapModifierKind> MapModifiers, bool ReturnDevicePointer,
-        bool IsImplicit, const ValueDecl *Mapper = nullptr,
-        bool ForDeviceAddr = false)
+        ArrayRef<OpenMPMapModifierKind> MapModifiers,
+        ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
+        bool ReturnDevicePointer, bool IsImplicit,
+        const ValueDecl *Mapper = nullptr, bool ForDeviceAddr = false)
         : Components(Components), MapType(MapType), MapModifiers(MapModifiers),
+          MotionModifiers(MotionModifiers),
           ReturnDevicePointer(ReturnDevicePointer), IsImplicit(IsImplicit),
           Mapper(Mapper), ForDeviceAddr(ForDeviceAddr) {}
   };
@@ -7254,7 +7257,8 @@ class MappableExprsHandler {
   /// expression.
   OpenMPOffloadMappingFlags getMapTypeBits(
       OpenMPMapClauseKind MapType, ArrayRef<OpenMPMapModifierKind> MapModifiers,
-      bool IsImplicit, bool AddPtrFlag, bool AddIsTargetParamFlag) const {
+      ArrayRef<OpenMPMotionModifierKind> MotionModifiers, bool IsImplicit,
+      bool AddPtrFlag, bool AddIsTargetParamFlag) const {
     OpenMPOffloadMappingFlags Bits =
         IsImplicit ? OMP_MAP_IMPLICIT : OMP_MAP_NONE;
     switch (MapType) {
@@ -7293,6 +7297,9 @@ class MappableExprsHandler {
     if (llvm::find(MapModifiers, OMPC_MAP_MODIFIER_present)
         != MapModifiers.end())
       Bits |= OMP_MAP_PRESENT;
+    if (llvm::find(MotionModifiers, OMPC_MOTION_MODIFIER_present)
+        != MotionModifiers.end())
+      Bits |= OMP_MAP_PRESENT;
     return Bits;
   }
 
@@ -7337,11 +7344,12 @@ class MappableExprsHandler {
 
   /// Generate the base pointers, section pointers, sizes, map type bits, and
   /// user-defined mappers (all included in \a CombinedInfo) for the provided
-  /// map type, map modifier, and expression components. \a IsFirstComponent
-  /// should be set to true if the provided set of components is the first
-  /// associated with a capture.
+  /// map type, map or motion modifiers, and expression components.
+  /// \a IsFirstComponent should be set to true if the provided set of
+  /// components is the first associated with a capture.
   void generateInfoForComponentList(
       OpenMPMapClauseKind MapType, ArrayRef<OpenMPMapModifierKind> MapModifiers,
+      ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
       OMPClauseMappableExprCommon::MappableExprComponentListRef Components,
       MapCombinedInfoTy &CombinedInfo, StructRangeInfoTy &PartialStruct,
       bool IsFirstComponentList, bool IsImplicit,
@@ -7694,7 +7702,7 @@ class MappableExprsHandler {
           // Emit data for non-overlapped data.
           OpenMPOffloadMappingFlags Flags =
               OMP_MAP_MEMBER_OF |
-              getMapTypeBits(MapType, MapModifiers, IsImplicit,
+              getMapTypeBits(MapType, MapModifiers, MotionModifiers, IsImplicit,
                              /*AddPtrFlag=*/false,
                              /*AddIsTargetParamFlag=*/false);
           LB = BP;
@@ -7750,10 +7758,10 @@ class MappableExprsHandler {
           // same expression except for the first one. We also need to signal
           // this map is the first one that relates with the current capture
           // (there is a set of entries for each capture).
-          OpenMPOffloadMappingFlags Flags = getMapTypeBits(
-              MapType, MapModifiers, IsImplicit,
-              !IsExpressionFirstInfo || RequiresReference,
-              IsCaptureFirstInfo && !RequiresReference);
+          OpenMPOffloadMappingFlags Flags =
+              getMapTypeBits(MapType, MapModifiers, MotionModifiers, IsImplicit,
+                             !IsExpressionFirstInfo || RequiresReference,
+                             IsCaptureFirstInfo && !RequiresReference);
 
           if (!IsExpressionFirstInfo) {
             // If we have a PTR_AND_OBJ pair where the OBJ is a pointer as well,
@@ -8017,14 +8025,16 @@ class MappableExprsHandler {
             OMPClauseMappableExprCommon::MappableExprComponentListRef L,
             OpenMPMapClauseKind MapType,
             ArrayRef<OpenMPMapModifierKind> MapModifiers,
+            ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
             bool ReturnDevicePointer, bool IsImplicit, const ValueDecl *Mapper,
             bool ForDeviceAddr = false) {
           const ValueDecl *VD =
               D ? cast<ValueDecl>(D->getCanonicalDecl()) : nullptr;
           if (SkipVarSet.count(VD))
             return;
-          Info[VD].emplace_back(L, MapType, MapModifiers, ReturnDevicePointer,
-                                IsImplicit, Mapper, ForDeviceAddr);
+          Info[VD].emplace_back(L, MapType, MapModifiers, MotionModifiers,
+                                ReturnDevicePointer, IsImplicit, Mapper,
+                                ForDeviceAddr);
         };
 
     assert(CurDir.is<const OMPExecutableDirective *>() &&
@@ -8033,18 +8043,20 @@ class MappableExprsHandler {
     for (const auto *C : CurExecDir->getClausesOfKind<OMPMapClause>())
       for (const auto L : C->component_lists()) {
         InfoGen(std::get<0>(L), std::get<1>(L), C->getMapType(),
-                C->getMapTypeModifiers(), /*ReturnDevicePointer=*/false,
-                C->isImplicit(), std::get<2>(L));
+                C->getMapTypeModifiers(), llvm::None,
+                /*ReturnDevicePointer=*/false, C->isImplicit(), std::get<2>(L));
       }
     for (const auto *C : CurExecDir->getClausesOfKind<OMPToClause>())
       for (const auto L : C->component_lists()) {
         InfoGen(std::get<0>(L), std::get<1>(L), OMPC_MAP_to, llvm::None,
-                /*ReturnDevicePointer=*/false, C->isImplicit(), std::get<2>(L));
+                C->getMotionModifiers(), /*ReturnDevicePointer=*/false,
+                C->isImplicit(), std::get<2>(L));
       }
     for (const auto *C : CurExecDir->getClausesOfKind<OMPFromClause>())
       for (const auto L : C->component_lists()) {
         InfoGen(std::get<0>(L), std::get<1>(L), OMPC_MAP_from, llvm::None,
-                /*ReturnDevicePointer=*/false, C->isImplicit(), std::get<2>(L));
+                C->getMotionModifiers(), /*ReturnDevicePointer=*/false,
+                C->isImplicit(), std::get<2>(L));
       }
 
     // Look at the use_device_ptr clause information and mark the existing map
@@ -8097,7 +8109,7 @@ class MappableExprsHandler {
           // Nonetheless, generateInfoForComponentList must be called to take
           // the pointer into account for the calculation of the range of the
           // partial struct.
-          InfoGen(nullptr, Components, OMPC_MAP_unknown, llvm::None,
+          InfoGen(nullptr, Components, OMPC_MAP_unknown, llvm::None, llvm::None,
                   /*ReturnDevicePointer=*/false, C->isImplicit(), nullptr);
           DeferredInfo[nullptr].emplace_back(IE, VD, /*ForDeviceAddr=*/false);
         } else {
@@ -8162,8 +8174,8 @@ class MappableExprsHandler {
           // the pointer into account for the calculation of the range of the
           // partial struct.
           InfoGen(nullptr, std::get<1>(L), OMPC_MAP_unknown, llvm::None,
-                  /*ReturnDevicePointer=*/false, C->isImplicit(), nullptr,
-                  /*ForDeviceAddr=*/true);
+                  llvm::None, /*ReturnDevicePointer=*/false, C->isImplicit(),
+                  nullptr, /*ForDeviceAddr=*/true);
           DeferredInfo[nullptr].emplace_back(IE, VD, /*ForDeviceAddr=*/true);
         } else {
           llvm::Value *Ptr;
@@ -8195,9 +8207,10 @@ class MappableExprsHandler {
 
         // Remember the current base pointer index.
         unsigned CurrentBasePointersIdx = CurInfo.BasePointers.size();
-        generateInfoForComponentList(
-            L.MapType, L.MapModifiers, L.Components, CurInfo, PartialStruct,
-            IsFirstComponentList, L.IsImplicit, L.Mapper, L.ForDeviceAddr);
+        generateInfoForComponentList(L.MapType, L.MapModifiers,
+                                     L.MotionModifiers, L.Components, CurInfo,
+                                     PartialStruct, IsFirstComponentList,
+                                     L.IsImplicit, L.Mapper, L.ForDeviceAddr);
 
         // If this entry relates with a device pointer, set the relevant
         // declaration and add the 'return pointer' flag.
@@ -8282,9 +8295,10 @@ class MappableExprsHandler {
             std::get<0>(L) ? cast<ValueDecl>(std::get<0>(L)->getCanonicalDecl())
                            : nullptr;
         // Get the corresponding user-defined mapper.
-        Info[VD].emplace_back(
-            std::get<1>(L), MC->getMapType(), MC->getMapTypeModifiers(),
-            /*ReturnDevicePointer=*/false, MC->isImplicit(), std::get<2>(L));
+        Info[VD].emplace_back(std::get<1>(L), MC->getMapType(),
+                              MC->getMapTypeModifiers(), llvm::None,
+                              /*ReturnDevicePointer=*/false, MC->isImplicit(),
+                              std::get<2>(L));
       }
     }
 
@@ -8300,9 +8314,10 @@ class MappableExprsHandler {
       for (const MapInfo &L : M.second) {
         assert(!L.Components.empty() &&
                "Not expecting declaration with no component lists.");
-        generateInfoForComponentList(
-            L.MapType, L.MapModifiers, L.Components, CurInfo, PartialStruct,
-            IsFirstComponentList, L.IsImplicit, L.Mapper, L.ForDeviceAddr);
+        generateInfoForComponentList(L.MapType, L.MapModifiers,
+                                     L.MotionModifiers, L.Components, CurInfo,
+                                     PartialStruct, IsFirstComponentList,
+                                     L.IsImplicit, L.Mapper, L.ForDeviceAddr);
         IsFirstComponentList = false;
       }
 
@@ -8568,9 +8583,9 @@ class MappableExprsHandler {
           OverlappedComponents = Pair.getSecond();
       bool IsFirstComponentList = true;
       generateInfoForComponentList(
-          MapType, MapModifiers, Components, CombinedInfo, PartialStruct,
-          IsFirstComponentList, IsImplicit, Mapper, /*ForDeviceAddr=*/false,
-          OverlappedComponents);
+          MapType, MapModifiers, llvm::None, Components, CombinedInfo,
+          PartialStruct, IsFirstComponentList, IsImplicit, Mapper,
+          /*ForDeviceAddr=*/false, OverlappedComponents);
     }
     // Go through other elements without overlapped elements.
     bool IsFirstComponentList = OverlappedData.empty();
@@ -8583,8 +8598,8 @@ class MappableExprsHandler {
       std::tie(Components, MapType, MapModifiers, IsImplicit, Mapper) = L;
       auto It = OverlappedData.find(&L);
       if (It == OverlappedData.end())
-        generateInfoForComponentList(MapType, MapModifiers, Components,
-                                     CombinedInfo, PartialStruct,
+        generateInfoForComponentList(MapType, MapModifiers, llvm::None,
+                                     Components, CombinedInfo, PartialStruct,
                                      IsFirstComponentList, IsImplicit, Mapper);
       IsFirstComponentList = false;
     }
diff --git a/clang/lib/Parse/ParseOpenMP.cpp b/clang/lib/Parse/ParseOpenMP.cpp
index ac1366e6b0ef6..8a92a8259c444 100644
--- a/clang/lib/Parse/ParseOpenMP.cpp
+++ b/clang/lib/Parse/ParseOpenMP.cpp
@@ -3453,25 +3453,43 @@ bool Parser::ParseOpenMPVarList(OpenMPDirectiveKind DKind,
     if (Tok.is(tok::colon))
       Data.ColonLoc = ConsumeToken();
   } else if (Kind == OMPC_to || Kind == OMPC_from) {
-    if (Tok.is(tok::identifier)) {
+    while (Tok.is(tok::identifier)) {
       auto Modifier =
           static_cast<OpenMPMotionModifierKind>(getOpenMPSimpleClauseType(
               Kind, PP.getSpelling(Tok), getLangOpts().OpenMP));
+      if (Modifier == OMPC_MOTION_MODIFIER_unknown)
+        break;
+      Data.MotionModifiers.push_back(Modifier);
+      Data.MotionModifiersLoc.push_back(Tok.getLocation());
+      ConsumeToken();
       if (Modifier == OMPC_MOTION_MODIFIER_mapper) {
-        // Parse the mapper modifier.
-        ConsumeToken();
         IsInvalidMapperModifier = parseMapperModifier(Data);
-        if (Tok.isNot(tok::colon)) {
-          if (!IsInvalidMapperModifier)
-            Diag(Tok, diag::warn_pragma_expected_colon) << ")";
-          SkipUntil(tok::colon, tok::r_paren, tok::annot_pragma_openmp_end,
-                    StopBeforeMatch);
-        }
-        // Consume ':'.
-        if (Tok.is(tok::colon))
-          ConsumeToken();
+        if (IsInvalidMapperModifier)
+          break;
       }
+      // OpenMP < 5.1 doesn't permit a ',' or additional modifiers.
+      if (getLangOpts().OpenMP < 51)
+        break;
+      // OpenMP 5.1 accepts an optional ',' even if the next character is ':'.
+      // TODO: Is that intentional?
+      if (Tok.is(tok::comma))
+        ConsumeToken();
+    }
+    if (!Data.MotionModifiers.empty() && Tok.isNot(tok::colon)) {
+      if (!IsInvalidMapperModifier) {
+        if (getLangOpts().OpenMP < 51)
+          Diag(Tok, diag::warn_pragma_expected_colon) << ")";
+        else
+          Diag(Tok, diag::warn_pragma_expected_colon) << "motion modifier";
+      }
+      SkipUntil(tok::colon, tok::r_paren, tok::annot_pragma_openmp_end,
+                StopBeforeMatch);
     }
+    // OpenMP 5.1 permits a ':' even without a preceding modifier.  TODO: Is
+    // that intentional?
+    if ((!Data.MotionModifiers.empty() || getLangOpts().OpenMP >= 51) &&
+        Tok.is(tok::colon))
+      Data.ColonLoc = ConsumeToken();
   } else if (Kind == OMPC_allocate ||
              (Kind == OMPC_affinity && Tok.is(tok::identifier) &&
               PP.getSpelling(Tok) == "iterator")) {
@@ -3647,6 +3665,7 @@ OMPClause *Parser::ParseOpenMPVarListClause(OpenMPDirectiveKind DKind,
       Kind, Vars, Data.DepModOrTailExpr, Locs, Data.ColonLoc,
       Data.ReductionOrMapperIdScopeSpec, Data.ReductionOrMapperId,
       Data.ExtraModifier, Data.MapTypeModifiers, Data.MapTypeModifiersLoc,
-      Data.IsMapTypeImplicit, Data.ExtraModifierLoc);
+      Data.IsMapTypeImplicit, Data.ExtraModifierLoc, Data.MotionModifiers,
+      Data.MotionModifiersLoc);
 }
 
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index 4f69975c54edd..0367d5c8c6016 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -13544,7 +13544,9 @@ OMPClause *Sema::ActOnOpenMPVarListClause(
     DeclarationNameInfo &ReductionOrMapperId, int ExtraModifier,
     ArrayRef<OpenMPMapModifierKind> MapTypeModifiers,
     ArrayRef<SourceLocation> MapTypeModifiersLoc, bool IsMapTypeImplicit,
-    SourceLocation ExtraModifierLoc) {
+    SourceLocation ExtraModifierLoc,
+    ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
+    ArrayRef<SourceLocation> MotionModifiersLoc) {
   SourceLocation StartLoc = Locs.StartLoc;
   SourceLocation LParenLoc = Locs.LParenLoc;
   SourceLocation EndLoc = Locs.EndLoc;
@@ -13621,12 +13623,14 @@ OMPClause *Sema::ActOnOpenMPVarListClause(
         IsMapTypeImplicit, ExtraModifierLoc, ColonLoc, VarList, Locs);
     break;
   case OMPC_to:
-    Res = ActOnOpenMPToClause(VarList, ReductionOrMapperIdScopeSpec,
-                              ReductionOrMapperId, Locs);
+    Res = ActOnOpenMPToClause(MotionModifiers, MotionModifiersLoc,
+                              ReductionOrMapperIdScopeSpec, ReductionOrMapperId,
+                              ColonLoc, VarList, Locs);
     break;
   case OMPC_from:
-    Res = ActOnOpenMPFromClause(VarList, ReductionOrMapperIdScopeSpec,
-                                ReductionOrMapperId, Locs);
+    Res = ActOnOpenMPFromClause(MotionModifiers, MotionModifiersLoc,
+                                ReductionOrMapperIdScopeSpec,
+                                ReductionOrMapperId, ColonLoc, VarList, Locs);
     break;
   case OMPC_use_device_ptr:
     Res = ActOnOpenMPUseDevicePtrClause(VarList, Locs);
@@ -18584,11 +18588,31 @@ void Sema::checkDeclIsAllowedInOpenMPTarget(Expr *E, Decl *D,
   checkDeclInTargetContext(E->getExprLoc(), E->getSourceRange(), *this, D);
 }
 
-OMPClause *Sema::ActOnOpenMPToClause(ArrayRef<Expr *> VarList,
-                                     CXXScopeSpec &MapperIdScopeSpec,
-                                     DeclarationNameInfo &MapperId,
-                                     const OMPVarListLocTy &Locs,
-                                     ArrayRef<Expr *> UnresolvedMappers) {
+OMPClause *Sema::ActOnOpenMPToClause(
+    ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
+    ArrayRef<SourceLocation> MotionModifiersLoc,
+    CXXScopeSpec &MapperIdScopeSpec, DeclarationNameInfo &MapperId,
+    SourceLocation ColonLoc, ArrayRef<Expr *> VarList,
+    const OMPVarListLocTy &Locs, ArrayRef<Expr *> UnresolvedMappers) {
+  OpenMPMotionModifierKind Modifiers[] = {OMPC_MOTION_MODIFIER_unknown,
+                                          OMPC_MOTION_MODIFIER_unknown};
+  SourceLocation ModifiersLoc[NumberOfOMPMotionModifiers];
+
+  // Process motion-modifiers, flag errors for duplicate modifiers.
+  unsigned Count = 0;
+  for (unsigned I = 0, E = MotionModifiers.size(); I < E; ++I) {
+    if (MotionModifiers[I] != OMPC_MOTION_MODIFIER_unknown &&
+        llvm::find(Modifiers, MotionModifiers[I]) != std::end(Modifiers)) {
+      Diag(MotionModifiersLoc[I], diag::err_omp_duplicate_motion_modifier);
+      continue;
+    }
+    assert(Count < NumberOfOMPMotionModifiers &&
+           "Modifiers exceed the allowed number of motion modifiers");
+    Modifiers[Count] = MotionModifiers[I];
+    ModifiersLoc[Count] = MotionModifiersLoc[I];
+    ++Count;
+  }
+
   MappableVarListInfo MVLI(VarList);
   checkMappableExpressionList(*this, DSAStack, OMPC_to, MVLI, Locs.StartLoc,
                               MapperIdScopeSpec, MapperId, UnresolvedMappers);
@@ -18597,15 +18621,35 @@ OMPClause *Sema::ActOnOpenMPToClause(ArrayRef<Expr *> VarList,
 
   return OMPToClause::Create(
       Context, Locs, MVLI.ProcessedVarList, MVLI.VarBaseDeclarations,
-      MVLI.VarComponents, MVLI.UDMapperList,
+      MVLI.VarComponents, MVLI.UDMapperList, Modifiers, ModifiersLoc,
       MapperIdScopeSpec.getWithLocInContext(Context), MapperId);
 }
 
-OMPClause *Sema::ActOnOpenMPFromClause(ArrayRef<Expr *> VarList,
-                                       CXXScopeSpec &MapperIdScopeSpec,
-                                       DeclarationNameInfo &MapperId,
-                                       const OMPVarListLocTy &Locs,
-                                       ArrayRef<Expr *> UnresolvedMappers) {
+OMPClause *Sema::ActOnOpenMPFromClause(
+    ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
+    ArrayRef<SourceLocation> MotionModifiersLoc,
+    CXXScopeSpec &MapperIdScopeSpec, DeclarationNameInfo &MapperId,
+    SourceLocation ColonLoc, ArrayRef<Expr *> VarList,
+    const OMPVarListLocTy &Locs, ArrayRef<Expr *> UnresolvedMappers) {
+  OpenMPMotionModifierKind Modifiers[] = {OMPC_MOTION_MODIFIER_unknown,
+                                          OMPC_MOTION_MODIFIER_unknown};
+  SourceLocation ModifiersLoc[NumberOfOMPMotionModifiers];
+
+  // Process motion-modifiers, flag errors for duplicate modifiers.
+  unsigned Count = 0;
+  for (unsigned I = 0, E = MotionModifiers.size(); I < E; ++I) {
+    if (MotionModifiers[I] != OMPC_MOTION_MODIFIER_unknown &&
+        llvm::find(Modifiers, MotionModifiers[I]) != std::end(Modifiers)) {
+      Diag(MotionModifiersLoc[I], diag::err_omp_duplicate_motion_modifier);
+      continue;
+    }
+    assert(Count < NumberOfOMPMotionModifiers &&
+           "Modifiers exceed the allowed number of motion modifiers");
+    Modifiers[Count] = MotionModifiers[I];
+    ModifiersLoc[Count] = MotionModifiersLoc[I];
+    ++Count;
+  }
+
   MappableVarListInfo MVLI(VarList);
   checkMappableExpressionList(*this, DSAStack, OMPC_from, MVLI, Locs.StartLoc,
                               MapperIdScopeSpec, MapperId, UnresolvedMappers);
@@ -18614,7 +18658,7 @@ OMPClause *Sema::ActOnOpenMPFromClause(ArrayRef<Expr *> VarList,
 
   return OMPFromClause::Create(
       Context, Locs, MVLI.ProcessedVarList, MVLI.VarBaseDeclarations,
-      MVLI.VarComponents, MVLI.UDMapperList,
+      MVLI.VarComponents, MVLI.UDMapperList, Modifiers, ModifiersLoc,
       MapperIdScopeSpec.getWithLocInContext(Context), MapperId);
 }
 
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index 0f83431c94cf7..53064d28dd991 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -2006,26 +2006,32 @@ class TreeTransform {
   ///
   /// By default, performs semantic analysis to build the new statement.
   /// Subclasses may override this routine to provide different behavior.
-  OMPClause *RebuildOMPToClause(ArrayRef<Expr *> VarList,
-                                CXXScopeSpec &MapperIdScopeSpec,
-                                DeclarationNameInfo &MapperId,
-                                const OMPVarListLocTy &Locs,
-                                ArrayRef<Expr *> UnresolvedMappers) {
-    return getSema().ActOnOpenMPToClause(VarList, MapperIdScopeSpec, MapperId,
-                                         Locs, UnresolvedMappers);
+  OMPClause *
+  RebuildOMPToClause(ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
+                     ArrayRef<SourceLocation> MotionModifiersLoc,
+                     CXXScopeSpec &MapperIdScopeSpec,
+                     DeclarationNameInfo &MapperId, SourceLocation ColonLoc,
+                     ArrayRef<Expr *> VarList, const OMPVarListLocTy &Locs,
+                     ArrayRef<Expr *> UnresolvedMappers) {
+    return getSema().ActOnOpenMPToClause(MotionModifiers, MotionModifiersLoc,
+                                         MapperIdScopeSpec, MapperId, ColonLoc,
+                                         VarList, Locs, UnresolvedMappers);
   }
 
   /// Build a new OpenMP 'from' clause.
   ///
   /// By default, performs semantic analysis to build the new statement.
   /// Subclasses may override this routine to provide different behavior.
-  OMPClause *RebuildOMPFromClause(ArrayRef<Expr *> VarList,
-                                  CXXScopeSpec &MapperIdScopeSpec,
-                                  DeclarationNameInfo &MapperId,
-                                  const OMPVarListLocTy &Locs,
-                                  ArrayRef<Expr *> UnresolvedMappers) {
-    return getSema().ActOnOpenMPFromClause(VarList, MapperIdScopeSpec, MapperId,
-                                           Locs, UnresolvedMappers);
+  OMPClause *
+  RebuildOMPFromClause(ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
+                       ArrayRef<SourceLocation> MotionModifiersLoc,
+                       CXXScopeSpec &MapperIdScopeSpec,
+                       DeclarationNameInfo &MapperId, SourceLocation ColonLoc,
+                       ArrayRef<Expr *> VarList, const OMPVarListLocTy &Locs,
+                       ArrayRef<Expr *> UnresolvedMappers) {
+    return getSema().ActOnOpenMPFromClause(
+        MotionModifiers, MotionModifiersLoc, MapperIdScopeSpec, MapperId,
+        ColonLoc, VarList, Locs, UnresolvedMappers);
   }
 
   /// Build a new OpenMP 'use_device_ptr' clause.
@@ -9738,8 +9744,9 @@ OMPClause *TreeTransform<Derived>::TransformOMPToClause(OMPToClause *C) {
   if (transformOMPMappableExprListClause<Derived, OMPToClause>(
           *this, C, Vars, MapperIdScopeSpec, MapperIdInfo, UnresolvedMappers))
     return nullptr;
-  return getDerived().RebuildOMPToClause(Vars, MapperIdScopeSpec, MapperIdInfo,
-                                         Locs, UnresolvedMappers);
+  return getDerived().RebuildOMPToClause(
+      C->getMotionModifiers(), C->getMotionModifiersLoc(), MapperIdScopeSpec,
+      MapperIdInfo, C->getColonLoc(), Vars, Locs, UnresolvedMappers);
 }
 
 template <typename Derived>
@@ -9753,7 +9760,8 @@ OMPClause *TreeTransform<Derived>::TransformOMPFromClause(OMPFromClause *C) {
           *this, C, Vars, MapperIdScopeSpec, MapperIdInfo, UnresolvedMappers))
     return nullptr;
   return getDerived().RebuildOMPFromClause(
-      Vars, MapperIdScopeSpec, MapperIdInfo, Locs, UnresolvedMappers);
+      C->getMotionModifiers(), C->getMotionModifiersLoc(), MapperIdScopeSpec,
+      MapperIdInfo, C->getColonLoc(), Vars, Locs, UnresolvedMappers);
 }
 
 template <typename Derived>
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 6b96b4ff59b8e..a27c36420fdbc 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -12601,8 +12601,14 @@ void OMPClauseReader::VisitOMPDefaultmapClause(OMPDefaultmapClause *C) {
 
 void OMPClauseReader::VisitOMPToClause(OMPToClause *C) {
   C->setLParenLoc(Record.readSourceLocation());
+  for (unsigned I = 0; I < NumberOfOMPMotionModifiers; ++I) {
+    C->setMotionModifier(
+        I, static_cast<OpenMPMotionModifierKind>(Record.readInt()));
+    C->setMotionModifierLoc(I, Record.readSourceLocation());
+  }
   C->setMapperQualifierLoc(Record.readNestedNameSpecifierLoc());
   C->setMapperIdInfo(Record.readDeclarationNameInfo());
+  C->setColonLoc(Record.readSourceLocation());
   auto NumVars = C->varlist_size();
   auto UniqueDecls = C->getUniqueDeclarationsNum();
   auto TotalLists = C->getTotalComponentListNum();
@@ -12651,8 +12657,14 @@ void OMPClauseReader::VisitOMPToClause(OMPToClause *C) {
 
 void OMPClauseReader::VisitOMPFromClause(OMPFromClause *C) {
   C->setLParenLoc(Record.readSourceLocation());
+  for (unsigned I = 0; I < NumberOfOMPMotionModifiers; ++I) {
+    C->setMotionModifier(
+        I, static_cast<OpenMPMotionModifierKind>(Record.readInt()));
+    C->setMotionModifierLoc(I, Record.readSourceLocation());
+  }
   C->setMapperQualifierLoc(Record.readNestedNameSpecifierLoc());
   C->setMapperIdInfo(Record.readDeclarationNameInfo());
+  C->setColonLoc(Record.readSourceLocation());
   auto NumVars = C->varlist_size();
   auto UniqueDecls = C->getUniqueDeclarationsNum();
   auto TotalLists = C->getTotalComponentListNum();
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index 9ea90b2a0212a..8c693b7d34fdd 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -6581,8 +6581,13 @@ void OMPClauseWriter::VisitOMPToClause(OMPToClause *C) {
   Record.push_back(C->getTotalComponentListNum());
   Record.push_back(C->getTotalComponentsNum());
   Record.AddSourceLocation(C->getLParenLoc());
+  for (unsigned I = 0; I < NumberOfOMPMotionModifiers; ++I) {
+    Record.push_back(C->getMotionModifier(I));
+    Record.AddSourceLocation(C->getMotionModifierLoc(I));
+  }
   Record.AddNestedNameSpecifierLoc(C->getMapperQualifierLoc());
   Record.AddDeclarationNameInfo(C->getMapperIdInfo());
+  Record.AddSourceLocation(C->getColonLoc());
   for (auto *E : C->varlists())
     Record.AddStmt(E);
   for (auto *E : C->mapperlists())
@@ -6605,8 +6610,13 @@ void OMPClauseWriter::VisitOMPFromClause(OMPFromClause *C) {
   Record.push_back(C->getTotalComponentListNum());
   Record.push_back(C->getTotalComponentsNum());
   Record.AddSourceLocation(C->getLParenLoc());
+  for (unsigned I = 0; I < NumberOfOMPMotionModifiers; ++I) {
+    Record.push_back(C->getMotionModifier(I));
+    Record.AddSourceLocation(C->getMotionModifierLoc(I));
+  }
   Record.AddNestedNameSpecifierLoc(C->getMapperQualifierLoc());
   Record.AddDeclarationNameInfo(C->getMapperIdInfo());
+  Record.AddSourceLocation(C->getColonLoc());
   for (auto *E : C->varlists())
     Record.AddStmt(E);
   for (auto *E : C->mapperlists())
diff --git a/clang/test/OpenMP/declare_mapper_ast_print.c b/clang/test/OpenMP/declare_mapper_ast_print.c
index e82bc4c2d3109..b7c9976c122af 100644
--- a/clang/test/OpenMP/declare_mapper_ast_print.c
+++ b/clang/test/OpenMP/declare_mapper_ast_print.c
@@ -5,6 +5,14 @@
 // RUN: %clang_cc1 -verify -fopenmp-simd -ast-print %s | FileCheck %s
 // RUN: %clang_cc1 -fopenmp-simd -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp-simd -include-pch %t -fsyntax-only -verify %s -ast-print | FileCheck %s
+
+// RUN: %clang_cc1 -DOMP51 -verify -fopenmp -fopenmp-version=51 -ast-print %s | FileCheck -check-prefixes=CHECK,OMP51 %s
+// RUN: %clang_cc1 -DOMP51 -fopenmp -fopenmp-version=51 -emit-pch -o %t %s
+// RUN: %clang_cc1 -DOMP51 -fopenmp -fopenmp-version=51 -include-pch %t -fsyntax-only -verify %s -ast-print | FileCheck -check-prefixes=CHECK,OMP51 %s
+
+// RUN: %clang_cc1 -DOMP51 -verify -fopenmp-simd -fopenmp-version=51 -ast-print %s | FileCheck -check-prefixes=CHECK,OMP51 %s
+// RUN: %clang_cc1 -DOMP51 -fopenmp-simd -fopenmp-version=51 -emit-pch -o %t %s
+// RUN: %clang_cc1 -DOMP51 -fopenmp-simd -fopenmp-version=51 -include-pch %t -fsyntax-only -verify %s -ast-print | FileCheck -check-prefixes=CHECK,OMP51 %s
 // expected-no-diagnostics
 
 #ifndef HEADER
@@ -50,6 +58,12 @@ int main() {
     { dd[0].i++; }
 #pragma omp target update to(mapper(id): vv) from(mapper(default): dd[0:10])
 // CHECK: #pragma omp target update to(mapper(id): vv) from(mapper(default): dd[0:10])
+#ifdef OMP51
+#pragma omp target update to(mapper(id) present: vv) from(mapper(default), present: dd[0:10])
+// OMP51: #pragma omp target update to(mapper(id), present: vv) from(mapper(default), present: dd[0:10])
+#pragma omp target update to(present mapper(id): vv) from(present, mapper(default): dd[0:10])
+// OMP51: #pragma omp target update to(present, mapper(id): vv) from(present, mapper(default): dd[0:10])
+#endif
   }
   return 0;
 }
diff --git a/clang/test/OpenMP/declare_mapper_codegen.cpp b/clang/test/OpenMP/declare_mapper_codegen.cpp
index ee64fe2099007..2fd4b3cb7ed07 100644
--- a/clang/test/OpenMP/declare_mapper_codegen.cpp
+++ b/clang/test/OpenMP/declare_mapper_codegen.cpp
@@ -839,4 +839,242 @@ void foo(int a){
 
 #endif // CK3
 
+///==========================================================================///
+// RUN: %clang_cc1 -DCK4 -verify -fopenmp -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm -femit-all-decls -disable-llvm-passes %s -o - | FileCheck --check-prefix CK4 --check-prefix CK4-64 %s
+// RUN: %clang_cc1 -DCK4 -fopenmp -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -femit-all-decls -disable-llvm-passes -o %t %s
+// RUN: %clang_cc1 -DCK4 -fopenmp -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -femit-all-decls -disable-llvm-passes -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix CK4 --check-prefix CK4-64 %s
+// RUN: %clang_cc1 -DCK4 -verify -fopenmp -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm -femit-all-decls -disable-llvm-passes %s -o - | FileCheck --check-prefix CK4 --check-prefix CK4-32 %s
+// RUN: %clang_cc1 -DCK4 -fopenmp -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -femit-all-decls -disable-llvm-passes -o %t %s
+// RUN: %clang_cc1 -DCK4 -fopenmp -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -femit-all-decls -disable-llvm-passes -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix CK4 --check-prefix CK4-32 %s
+
+// RUN: %clang_cc1 -DCK4 -verify -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm -femit-all-decls -disable-llvm-passes %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK4 -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -femit-all-decls -disable-llvm-passes -o %t %s
+// RUN: %clang_cc1 -DCK4 -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -femit-all-decls -disable-llvm-passes -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK4 -verify -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm -femit-all-decls -disable-llvm-passes %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK4 -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -femit-all-decls -disable-llvm-passes -o %t %s
+// RUN: %clang_cc1 -DCK4 -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -femit-all-decls -disable-llvm-passes -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+
+#ifdef CK4
+// Mapper function code generation and runtime interface.
+
+// CK4-64: [[TSIZES:@.+]] = {{.+}}constant [1 x i64] [i64 16]
+// CK4-32: [[TSIZES:@.+]] = {{.+}}constant [1 x i64] [i64 8]
+// PRESENT=0x1000 | TARGET_PARAM=0x20 | TO=0x1 = 0x1021
+// CK4: [[TTYPES:@.+]] = {{.+}}constant [1 x i64] [i64 [[#0x1021]]]
+
+// CK4-64: [[FSIZES:@.+]] = {{.+}}constant [1 x i64] [i64 16]
+// CK4-32: [[FSIZES:@.+]] = {{.+}}constant [1 x i64] [i64 8]
+// PRESENT=0x1000 | TARGET_PARAM=0x20 | FROM=0x2 = 0x1022
+// CK4: [[FTYPES:@.+]] = {{.+}}constant [1 x i64] [i64 [[#0x1022]]]
+
+class C {
+public:
+  int a;
+  double *b;
+};
+
+#pragma omp declare mapper(id: C s) map(s.a, s.b[0:2])
+
+// CK4: define {{.*}}void [[MPRFUNC:@[.]omp_mapper[.].*C[.]id]](i8*{{.*}}, i8*{{.*}}, i8*{{.*}}, i64{{.*}}, i64{{.*}})
+// CK4: store i8* %{{[^,]+}}, i8** [[HANDLEADDR:%[^,]+]]
+// CK4: store i8* %{{[^,]+}}, i8** [[BPTRADDR:%[^,]+]]
+// CK4: store i8* %{{[^,]+}}, i8** [[VPTRADDR:%[^,]+]]
+// CK4: store i64 %{{[^,]+}}, i{{64|32}}* [[SIZEADDR:%[^,]+]]
+// CK4: store i64 %{{[^,]+}}, i64* [[TYPEADDR:%[^,]+]]
+// CK4-DAG: [[BYTESIZE:%.+]] = load i64, i64* [[SIZEADDR]]
+// CK4-64-DAG: [[SIZE:%.+]] = udiv exact i64 [[BYTESIZE]], 16
+// CK4-32-DAG: [[SIZE:%.+]] = udiv exact i64 [[BYTESIZE]], 8
+// CK4-DAG: [[TYPE:%.+]] = load i64, i64* [[TYPEADDR]]
+// CK4-DAG: [[HANDLE:%.+]] = load i8*, i8** [[HANDLEADDR]]
+// CK4-DAG: [[PTRBEGIN:%.+]] = bitcast i8** [[VPTRADDR]] to %class.C**
+// CK4-DAG: [[PTREND:%.+]] = getelementptr %class.C*, %class.C** [[PTRBEGIN]], i64 [[SIZE]]
+// CK4-DAG: [[BPTR:%.+]] = load i8*, i8** [[BPTRADDR]]
+// CK4-DAG: [[BEGIN:%.+]] = load i8*, i8** [[VPTRADDR]]
+// CK4: [[ISARRAY:%.+]] = icmp sge i64 [[SIZE]], 1
+// CK4: br i1 [[ISARRAY]], label %[[INITEVALDEL:[^,]+]], label %[[LHEAD:[^,]+]]
+
+// CK4: [[INITEVALDEL]]
+// CK4: [[TYPEDEL:%.+]] = and i64 [[TYPE]], 8
+// CK4: [[ISNOTDEL:%.+]] = icmp eq i64 [[TYPEDEL]], 0
+// CK4: br i1 [[ISNOTDEL]], label %[[INIT:[^,]+]], label %[[LHEAD:[^,]+]]
+// CK4: [[INIT]]
+// CK4-64-DAG: [[ARRSIZE:%.+]] = mul nuw i64 [[SIZE]], 16
+// CK4-32-DAG: [[ARRSIZE:%.+]] = mul nuw i64 [[SIZE]], 8
+// CK4-DAG: [[ITYPE:%.+]] = and i64 [[TYPE]], -4
+// CK4: call void @__tgt_push_mapper_component(i8* [[HANDLE]], i8* [[BPTR]], i8* [[BEGIN]], i64 [[ARRSIZE]], i64 [[ITYPE]])
+// CK4: br label %[[LHEAD:[^,]+]]
+
+// CK4: [[LHEAD]]
+// CK4: [[ISEMPTY:%.+]] = icmp eq %class.C** [[PTRBEGIN]], [[PTREND]]
+// CK4: br i1 [[ISEMPTY]], label %[[DONE:[^,]+]], label %[[LBODY:[^,]+]]
+// CK4: [[LBODY]]
+// CK4: [[PTR:%.+]] = phi %class.C** [ [[PTRBEGIN]], %[[LHEAD]] ], [ [[PTRNEXT:%.+]], %[[LCORRECT:[^,]+]] ]
+// CK4: [[OBJ:%.+]] = load %class.C*, %class.C** [[PTR]]
+// CK4-DAG: [[ABEGIN:%.+]] = getelementptr inbounds %class.C, %class.C* [[OBJ]], i32 0, i32 0
+// CK4-DAG: [[BBEGIN:%.+]] = getelementptr inbounds %class.C, %class.C* [[OBJ]], i32 0, i32 1
+// CK4-DAG: [[BBEGIN2:%.+]] = getelementptr inbounds %class.C, %class.C* [[OBJ]], i32 0, i32 1
+// CK4-DAG: [[BARRBEGIN:%.+]] = load double*, double** [[BBEGIN2]]
+// CK4-DAG: [[BARRBEGINGEP:%.+]] = getelementptr inbounds double, double* [[BARRBEGIN]], i[[sz:64|32]] 0
+// CK4-DAG: [[BEND:%.+]] = getelementptr double*, double** [[BBEGIN]], i32 1
+// CK4-DAG: [[ABEGINV:%.+]] = bitcast i32* [[ABEGIN]] to i8*
+// CK4-DAG: [[BENDV:%.+]] = bitcast double** [[BEND]] to i8*
+// CK4-DAG: [[ABEGINI:%.+]] = ptrtoint i8* [[ABEGINV]] to i64
+// CK4-DAG: [[BENDI:%.+]] = ptrtoint i8* [[BENDV]] to i64
+// CK4-DAG: [[CSIZE:%.+]] = sub i64 [[BENDI]], [[ABEGINI]]
+// CK4-DAG: [[CUSIZE:%.+]] = sdiv exact i64 [[CSIZE]], ptrtoint (i8* getelementptr (i8, i8* null, i32 1) to i64)
+// CK4-DAG: [[BPTRADDR0BC:%.+]] = bitcast %class.C* [[OBJ]] to i8*
+// CK4-DAG: [[PTRADDR0BC:%.+]] = bitcast i32* [[ABEGIN]] to i8*
+// CK4-DAG: [[PRESIZE:%.+]] = call i64 @__tgt_mapper_num_components(i8* [[HANDLE]])
+// CK4-DAG: [[SHIPRESIZE:%.+]] = shl i64 [[PRESIZE]], 48
+// CK4-DAG: br label %[[MEMBER:[^,]+]]
+// CK4-DAG: [[MEMBER]]
+// CK4-DAG: br i1 true, label %[[LTYPE:[^,]+]], label %[[MEMBERCOM:[^,]+]]
+// CK4-DAG: [[MEMBERCOM]]
+// CK4-DAG: [[MEMBERCOMTYPE:%.+]] = add nuw i64 32, [[SHIPRESIZE]]
+// CK4-DAG: br label %[[LTYPE]]
+// CK4-DAG: [[LTYPE]]
+// CK4-DAG: [[MEMBERTYPE:%.+]] = phi i64 [ 32, %[[MEMBER]] ], [ [[MEMBERCOMTYPE]], %[[MEMBERCOM]] ]
+// CK4-DAG: [[TYPETF:%.+]] = and i64 [[TYPE]], 3
+// CK4-DAG: [[ISALLOC:%.+]] = icmp eq i64 [[TYPETF]], 0
+// CK4-DAG: br i1 [[ISALLOC]], label %[[ALLOC:[^,]+]], label %[[ALLOCELSE:[^,]+]]
+// CK4-DAG: [[ALLOC]]
+// CK4-DAG: [[ALLOCTYPE:%.+]] = and i64 [[MEMBERTYPE]], -4
+// CK4-DAG: br label %[[TYEND:[^,]+]]
+// CK4-DAG: [[ALLOCELSE]]
+// CK4-DAG: [[ISTO:%.+]] = icmp eq i64 [[TYPETF]], 1
+// CK4-DAG: br i1 [[ISTO]], label %[[TO:[^,]+]], label %[[TOELSE:[^,]+]]
+// CK4-DAG: [[TO]]
+// CK4-DAG: [[TOTYPE:%.+]] = and i64 [[MEMBERTYPE]], -3
+// CK4-DAG: br label %[[TYEND]]
+// CK4-DAG: [[TOELSE]]
+// CK4-DAG: [[ISFROM:%.+]] = icmp eq i64 [[TYPETF]], 2
+// CK4-DAG: br i1 [[ISFROM]], label %[[FROM:[^,]+]], label %[[TYEND]]
+// CK4-DAG: [[FROM]]
+// CK4-DAG: [[FROMTYPE:%.+]] = and i64 [[MEMBERTYPE]], -2
+// CK4-DAG: br label %[[TYEND]]
+// CK4-DAG: [[TYEND]]
+// CK4-DAG: [[PHITYPE0:%.+]] = phi i64 [ [[ALLOCTYPE]], %[[ALLOC]] ], [ [[TOTYPE]], %[[TO]] ], [ [[FROMTYPE]], %[[FROM]] ], [ [[MEMBERTYPE]], %[[TOELSE]] ]
+// CK4: call void @__tgt_push_mapper_component(i8* [[HANDLE]], i8* [[BPTRADDR0BC]], i8* [[PTRADDR0BC]], i64 [[CUSIZE]], i64 [[PHITYPE0]])
+// CK4-DAG: [[BPTRADDR1BC:%.+]] = bitcast %class.C* [[OBJ]] to i8*
+// CK4-DAG: [[PTRADDR1BC:%.+]] = bitcast i32* [[ABEGIN]] to i8*
+// CK4-DAG: br label %[[MEMBER:[^,]+]]
+// CK4-DAG: [[MEMBER]]
+// CK4-DAG: br i1 false, label %[[LTYPE:[^,]+]], label %[[MEMBERCOM:[^,]+]]
+// CK4-DAG: [[MEMBERCOM]]
+// 281474976710659 == 0x1,000,000,003
+// CK4-DAG: [[MEMBERCOMTYPE:%.+]] = add nuw i64 281474976710659, [[SHIPRESIZE]]
+// CK4-DAG: br label %[[LTYPE]]
+// CK4-DAG: [[LTYPE]]
+// CK4-DAG: [[MEMBERTYPE:%.+]] = phi i64 [ 281474976710659, %[[MEMBER]] ], [ [[MEMBERCOMTYPE]], %[[MEMBERCOM]] ]
+// CK4-DAG: [[TYPETF:%.+]] = and i64 [[TYPE]], 3
+// CK4-DAG: [[ISALLOC:%.+]] = icmp eq i64 [[TYPETF]], 0
+// CK4-DAG: br i1 [[ISALLOC]], label %[[ALLOC:[^,]+]], label %[[ALLOCELSE:[^,]+]]
+// CK4-DAG: [[ALLOC]]
+// CK4-DAG: [[ALLOCTYPE:%.+]] = and i64 [[MEMBERTYPE]], -4
+// CK4-DAG: br label %[[TYEND:[^,]+]]
+// CK4-DAG: [[ALLOCELSE]]
+// CK4-DAG: [[ISTO:%.+]] = icmp eq i64 [[TYPETF]], 1
+// CK4-DAG: br i1 [[ISTO]], label %[[TO:[^,]+]], label %[[TOELSE:[^,]+]]
+// CK4-DAG: [[TO]]
+// CK4-DAG: [[TOTYPE:%.+]] = and i64 [[MEMBERTYPE]], -3
+// CK4-DAG: br label %[[TYEND]]
+// CK4-DAG: [[TOELSE]]
+// CK4-DAG: [[ISFROM:%.+]] = icmp eq i64 [[TYPETF]], 2
+// CK4-DAG: br i1 [[ISFROM]], label %[[FROM:[^,]+]], label %[[TYEND]]
+// CK4-DAG: [[FROM]]
+// CK4-DAG: [[FROMTYPE:%.+]] = and i64 [[MEMBERTYPE]], -2
+// CK4-DAG: br label %[[TYEND]]
+// CK4-DAG: [[TYEND]]
+// CK4-DAG: [[TYPE1:%.+]] = phi i64 [ [[ALLOCTYPE]], %[[ALLOC]] ], [ [[TOTYPE]], %[[TO]] ], [ [[FROMTYPE]], %[[FROM]] ], [ [[MEMBERTYPE]], %[[TOELSE]] ]
+// CK4: call void @__tgt_push_mapper_component(i8* [[HANDLE]], i8* [[BPTRADDR1BC]], i8* [[PTRADDR1BC]], i64 4, i64 [[TYPE1]])
+// CK4-DAG: [[BPTRADDR2BC:%.+]] = bitcast double** [[BBEGIN]] to i8*
+// CK4-DAG: [[PTRADDR2BC:%.+]] = bitcast double* [[BARRBEGINGEP]] to i8*
+// CK4-DAG: br label %[[MEMBER:[^,]+]]
+// CK4-DAG: [[MEMBER]]
+// CK4-DAG: br i1 false, label %[[LTYPE:[^,]+]], label %[[MEMBERCOM:[^,]+]]
+// CK4-DAG: [[MEMBERCOM]]
+// 281474976710675 == 0x1,000,000,013
+// CK4-DAG: [[MEMBERCOMTYPE:%.+]] = add nuw i64 281474976710675, [[SHIPRESIZE]]
+// CK4-DAG: br label %[[LTYPE]]
+// CK4-DAG: [[LTYPE]]
+// CK4-DAG: [[MEMBERTYPE:%.+]] = phi i64 [ 281474976710675, %[[MEMBER]] ], [ [[MEMBERCOMTYPE]], %[[MEMBERCOM]] ]
+// CK4-DAG: [[TYPETF:%.+]] = and i64 [[TYPE]], 3
+// CK4-DAG: [[ISALLOC:%.+]] = icmp eq i64 [[TYPETF]], 0
+// CK4-DAG: br i1 [[ISALLOC]], label %[[ALLOC:[^,]+]], label %[[ALLOCELSE:[^,]+]]
+// CK4-DAG: [[ALLOC]]
+// CK4-DAG: [[ALLOCTYPE:%.+]] = and i64 [[MEMBERTYPE]], -4
+// CK4-DAG: br label %[[TYEND:[^,]+]]
+// CK4-DAG: [[ALLOCELSE]]
+// CK4-DAG: [[ISTO:%.+]] = icmp eq i64 [[TYPETF]], 1
+// CK4-DAG: br i1 [[ISTO]], label %[[TO:[^,]+]], label %[[TOELSE:[^,]+]]
+// CK4-DAG: [[TO]]
+// CK4-DAG: [[TOTYPE:%.+]] = and i64 [[MEMBERTYPE]], -3
+// CK4-DAG: br label %[[TYEND]]
+// CK4-DAG: [[TOELSE]]
+// CK4-DAG: [[ISFROM:%.+]] = icmp eq i64 [[TYPETF]], 2
+// CK4-DAG: br i1 [[ISFROM]], label %[[FROM:[^,]+]], label %[[TYEND]]
+// CK4-DAG: [[FROM]]
+// CK4-DAG: [[FROMTYPE:%.+]] = and i64 [[MEMBERTYPE]], -2
+// CK4-DAG: br label %[[TYEND]]
+// CK4-DAG: [[TYEND]]
+// CK4-DAG: [[TYPE2:%.+]] = phi i64 [ [[ALLOCTYPE]], %[[ALLOC]] ], [ [[TOTYPE]], %[[TO]] ], [ [[FROMTYPE]], %[[FROM]] ], [ [[MEMBERTYPE]], %[[TOELSE]] ]
+// CK4: call void @__tgt_push_mapper_component(i8* [[HANDLE]], i8* [[BPTRADDR2BC]], i8* [[PTRADDR2BC]], i64 16, i64 [[TYPE2]])
+// CK4: [[PTRNEXT]] = getelementptr %class.C*, %class.C** [[PTR]], i32 1
+// CK4: [[ISDONE:%.+]] = icmp eq %class.C** [[PTRNEXT]], [[PTREND]]
+// CK4: br i1 [[ISDONE]], label %[[LEXIT:[^,]+]], label %[[LBODY]]
+
+// CK4: [[LEXIT]]
+// CK4: [[ISARRAY:%.+]] = icmp sge i64 [[SIZE]], 1
+// CK4: br i1 [[ISARRAY]], label %[[EVALDEL:[^,]+]], label %[[DONE]]
+// CK4: [[EVALDEL]]
+// CK4: [[TYPEDEL:%.+]] = and i64 [[TYPE]], 8
+// CK4: [[ISDEL:%.+]] = icmp ne i64 [[TYPEDEL]], 0
+// CK4: br i1 [[ISDEL]], label %[[DEL:[^,]+]], label %[[DONE]]
+// CK4: [[DEL]]
+// CK4-64-DAG: [[ARRSIZE:%.+]] = mul nuw i64 [[SIZE]], 16
+// CK4-32-DAG: [[ARRSIZE:%.+]] = mul nuw i64 [[SIZE]], 8
+// CK4-DAG: [[DTYPE:%.+]] = and i64 [[TYPE]], -4
+// CK4: call void @__tgt_push_mapper_component(i8* [[HANDLE]], i8* [[BPTR]], i8* [[BEGIN]], i64 [[ARRSIZE]], i64 [[DTYPE]])
+// CK4: br label %[[DONE]]
+// CK4: [[DONE]]
+// CK4: ret void
+
+
+// CK4-LABEL: define {{.*}}void @{{.*}}foo{{.*}}
+void foo(int a){
+  int i = a;
+  C c;
+  c.a = a;
+
+  // CK4-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 1, i8** [[TGEPBP:%.+]], i8** [[TGEPP:%.+]], i64* getelementptr {{.+}}[1 x i64]* [[TSIZES]], i32 0, i32 0), {{.+}}getelementptr {{.+}}[1 x i64]* [[TTYPES]]{{.+}}, i8** [[TMPRGEP:%.+]])
+  // CK4-DAG: [[TGEPBP]] = getelementptr inbounds {{.+}}[[TBP:%[^,]+]], i{{.+}} 0, i{{.+}} 0
+  // CK4-DAG: [[TGEPP]] = getelementptr inbounds {{.+}}[[TP:%[^,]+]], i{{.+}} 0, i{{.+}} 0
+  // CK4-DAG: [[TMPRGEP]] = bitcast [1 x i8*]* [[TMPR:%[^,]+]] to i8**
+  // CK4-DAG: [[TBP0:%.+]] = getelementptr inbounds {{.+}}[[TBP]], i{{.+}} 0, i{{.+}} 0
+  // CK4-DAG: [[TP0:%.+]] = getelementptr inbounds {{.+}}[[TP]], i{{.+}} 0, i{{.+}} 0
+  // CK4-DAG: [[TMPR1:%.+]] = getelementptr inbounds {{.+}}[[TMPR]], i[[sz]] 0, i[[sz]] 0
+  // CK4-DAG: [[TCBP0:%.+]] = bitcast i8** [[TBP0]] to %class.C**
+  // CK4-DAG: [[TCP0:%.+]] = bitcast i8** [[TP0]] to %class.C**
+  // CK4-DAG: store %class.C* [[VAL:%[^,]+]], %class.C** [[TCBP0]]
+  // CK4-DAG: store %class.C* [[VAL]], %class.C** [[TCP0]]
+  // CK4-DAG: store i8* bitcast (void (i8*, i8*, i8*, i64, i64)* [[MPRFUNC]] to i8*), i8** [[TMPR1]]
+  #pragma omp target update to(present, mapper(id): c)
+
+  // CK4-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 1, i8** [[FGEPBP:%.+]], i8** [[FGEPP:%.+]], i64* getelementptr {{.+}}[1 x i64]* [[FSIZES]], i32 0, i32 0), {{.+}}getelementptr {{.+}}[1 x i64]* [[FTYPES]]{{.+}}, i8** [[FMPRGEP:%.+]])
+  // CK4-DAG: [[FGEPBP]] = getelementptr inbounds {{.+}}[[FBP:%[^,]+]], i{{.+}} 0, i{{.+}} 0
+  // CK4-DAG: [[FGEPP]] = getelementptr inbounds {{.+}}[[FP:%[^,]+]], i{{.+}} 0, i{{.+}} 0
+  // CK4-DAG: [[FMPRGEP]] = bitcast [1 x i8*]* [[FMPR:%[^,]+]] to i8**
+  // CK4-DAG: [[FBP0:%.+]] = getelementptr inbounds {{.+}}[[FBP]], i{{.+}} 0, i{{.+}} 0
+  // CK4-DAG: [[FP0:%.+]] = getelementptr inbounds {{.+}}[[FP]], i{{.+}} 0, i{{.+}} 0
+  // CK4-DAG: [[FMPR1:%.+]] = getelementptr inbounds {{.+}}[[FMPR]], i[[sz]] 0, i[[sz]] 0
+  // CK4-DAG: [[FCBP0:%.+]] = bitcast i8** [[FBP0]] to %class.C**
+  // CK4-DAG: [[FCP0:%.+]] = bitcast i8** [[FP0]] to %class.C**
+  // CK4-DAG: store %class.C* [[VAL]], %class.C** [[FCBP0]]
+  // CK4-DAG: store %class.C* [[VAL]], %class.C** [[FCP0]]
+  // CK4-DAG: store i8* bitcast (void (i8*, i8*, i8*, i64, i64)* [[MPRFUNC]] to i8*), i8** [[FMPR1]]
+  #pragma omp target update from(mapper(id), present: c)
+}
+
+#endif // CK4
+
 #endif // HEADER
diff --git a/clang/test/OpenMP/target_update_ast_print.cpp b/clang/test/OpenMP/target_update_ast_print.cpp
index 0111432fde8f6..745233c430eb1 100644
--- a/clang/test/OpenMP/target_update_ast_print.cpp
+++ b/clang/test/OpenMP/target_update_ast_print.cpp
@@ -5,6 +5,14 @@
 // RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=50 -ast-print %s | FileCheck %s
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=50 -x c++ -std=c++11 -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=50 -std=c++11 -include-pch %t -fsyntax-only -verify %s -ast-print | FileCheck %s
+
+// RUN: %clang_cc1 -DOMP51 -verify -fopenmp -fopenmp-version=51 -ast-print %s | FileCheck -check-prefixes=CHECK,OMP51 %s
+// RUN: %clang_cc1 -DOMP51 -fopenmp -fopenmp-version=51 -x c++ -std=c++11 -emit-pch -o %t %s
+// RUN: %clang_cc1 -DOMP51 -fopenmp -fopenmp-version=51 -std=c++11 -include-pch %t -fsyntax-only -verify %s -ast-print | FileCheck -check-prefixes=CHECK,OMP51 %s
+
+// RUN: %clang_cc1 -DOMP51 -verify -fopenmp-simd -fopenmp-version=51 -ast-print %s | FileCheck -check-prefixes=CHECK,OMP51 %s
+// RUN: %clang_cc1 -DOMP51 -fopenmp-simd -fopenmp-version=51 -x c++ -std=c++11 -emit-pch -o %t %s
+// RUN: %clang_cc1 -DOMP51 -fopenmp-simd -fopenmp-version=51 -std=c++11 -include-pch %t -fsyntax-only -verify %s -ast-print | FileCheck -check-prefixes=CHECK,OMP51 %s
 // expected-no-diagnostics
 
 #ifndef HEADER
@@ -25,6 +33,13 @@ T foo(T targ, U uarg) {
 #pragma omp target update to(arr[2][0:1:2])
 
 #pragma omp target update from(arr[2][0:1:2])
+
+#ifdef OMP51
+#pragma omp target update to(present: arr[2][0:1:2])
+
+#pragma omp target update from(present: arr[2][0:1:2], a)
+#endif
+
   return a + targ + (T)b;
 }
 // CHECK:      static T a, *p;
@@ -45,6 +60,8 @@ T foo(T targ, U uarg) {
 // CHECK:      int arr[100][100];
 // CHECK-NEXT: #pragma omp target update to(arr[2][0:1:2])
 // CHECK-NEXT: #pragma omp target update from(arr[2][0:1:2])
+// OMP5-NEXT: #pragma omp target update to(present: arr[2][0:1:2])
+// OMP5-NEXT: #pragma omp target update from(present: arr[2][0:1:2], a)
 
 int main(int argc, char **argv) {
   static int a;
@@ -62,6 +79,13 @@ int main(int argc, char **argv) {
 // CHECK-NEXT: #pragma omp target update to(argv[2][0:1:2])
 #pragma omp target update from(argv[2][0:1:2])
 // CHECK-NEXT: #pragma omp target update from(argv[2][0:1:2])
+#ifdef OMP51
+#pragma omp target update to(present: argv[2][0:1:2])
+// OMP5-NEXT: #pragma omp target update to(present: arr[2][0:1:2])
+#pragma omp target update from(argv[2][0:1:2], a)
+// OMP5-NEXT: #pragma omp target update from(present: arr[2][0:1:2], a)
+#endif
+
 
   return foo(argc, f) + foo(argv[0][0], f) + a;
 }
diff --git a/clang/test/OpenMP/target_update_codegen.cpp b/clang/test/OpenMP/target_update_codegen.cpp
index 637b580d754f2..feea1129be4cc 100644
--- a/clang/test/OpenMP/target_update_codegen.cpp
+++ b/clang/test/OpenMP/target_update_codegen.cpp
@@ -1084,5 +1084,68 @@ void array_shaping(float *f, int sa) {
   #pragma omp target update from(([sa][5])f)
 }
 
+#endif
+///==========================================================================///
+// RUN: %clang_cc1 -DCK19 -verify -fopenmp -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK19 --check-prefix CK19-64
+// RUN: %clang_cc1 -DCK19 -fopenmp -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK19 --check-prefix CK19-64
+// RUN: %clang_cc1 -DCK19 -verify -fopenmp -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK19 --check-prefix CK19-32
+// RUN: %clang_cc1 -DCK19 -fopenmp -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK19 --check-prefix CK19-32
+
+// RUN: %clang_cc1 -DCK19 -verify -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK19 -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK19 -verify -fopenmp-version=51 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK19 -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
+#ifdef CK19
+
+// PRESENT=0x1000 | TARGET_PARAM=0x20 | TO=0x1 = 0x1021
+// CK19: [[MTYPE00:@.+]] = {{.+}}constant [1 x i64] [i64 [[#0x1021]]]
+
+// PRESENT=0x1000 | TARGET_PARAM=0x20 | FROM=0x2 = 0x1022
+// CK19: [[MTYPE01:@.+]] = {{.+}}constant [1 x i64] [i64 [[#0x1022]]]
+
+// CK19-LABEL: _Z13check_presenti
+void check_present(int arg) {
+  int la;
+  float lb[arg];
+
+  // Region 00
+  // CK19-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 1, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i64* [[GEPS:%.+]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
+  // CK19-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
+  // CK19-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
+  // CK19-DAG: [[GEPS]] = getelementptr inbounds {{.+}}[[S:%[^,]+]]
+
+  // CK19-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 0
+  // CK19-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 0
+  // CK19-DAG: [[S0:%.+]] = getelementptr inbounds {{.+}}[[S]], i{{.+}} 0, i{{.+}} 0
+  // CK19-DAG: [[CBP0:%.+]] = bitcast i8** [[BP0]] to float**
+  // CK19-DAG: [[CP0:%.+]] = bitcast i8** [[P0]] to float**
+  // CK19-DAG: store float* [[VAL0:%[^,]+]], float** [[CBP0]]
+  // CK19-DAG: store float* [[VAL0]], float** [[CP0]]
+  // CK19-DAG: store i64 [[CSVAL0:%[^,]+]], i64* [[S0]]
+  #pragma omp target update to(present: lb)
+  ;
+
+  // Region 01
+  // CK19-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 1, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i64* [[GEPS:%.+]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[MTYPE01]]{{.+}}, i8** null)
+  // CK19-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
+  // CK19-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
+  // CK19-DAG: [[GEPS]] = getelementptr inbounds {{.+}}[[S:%[^,]+]]
+
+  // CK19-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 0
+  // CK19-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 0
+  // CK19-DAG: [[S0:%.+]] = getelementptr inbounds {{.+}}[[S]], i{{.+}} 0, i{{.+}} 0
+  // CK19-DAG: [[CBP0:%.+]] = bitcast i8** [[BP0]] to float**
+  // CK19-DAG: [[CP0:%.+]] = bitcast i8** [[P0]] to float**
+  // CK19-DAG: store float* [[VAL0:%[^,]+]], float** [[CBP0]]
+  // CK19-DAG: store float* [[VAL0]], float** [[CP0]]
+  // CK19-DAG: store i64 [[CSVAL0:%[^,]+]], i64* [[S0]]
+  #pragma omp target update from(present: lb)
+  ;
+}
 #endif
 #endif
diff --git a/clang/test/OpenMP/target_update_messages.cpp b/clang/test/OpenMP/target_update_messages.cpp
index 4092b623bdc97..401941b63028c 100644
--- a/clang/test/OpenMP/target_update_messages.cpp
+++ b/clang/test/OpenMP/target_update_messages.cpp
@@ -1,8 +1,10 @@
-// RUN: %clang_cc1 -verify=expected,omp45 -fopenmp -fopenmp-version=45 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
-// RUN: %clang_cc1 -verify=expected,omp50 -fopenmp -fopenmp-version=50 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,lt50,lt51 -fopenmp -fopenmp-version=45 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,ge50,lt51 -fopenmp -fopenmp-version=50 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,ge50,ge51 -fopenmp -fopenmp-version=51 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
 
-// RUN: %clang_cc1 -verify=expected,omp45 -fopenmp-simd -fopenmp-version=45 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
-// RUN: %clang_cc1 -verify=expected,omp50 -fopenmp-simd -fopenmp-version=50 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,lt50,lt51 -fopenmp-simd -fopenmp-version=45 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,ge50,lt51 -fopenmp-simd -fopenmp-version=50 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,ge50,ge51 -fopenmp-simd -fopenmp-version=51 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
 
 void xxx(int argc) {
   int x; // expected-note {{initialize the variable 'x' to silence this warning}}
@@ -25,6 +27,10 @@ int tmain(T argc, S **argv) {
   return 0;
 }
 
+struct S {
+  int i;
+};
+
 int main(int argc, char **argv) {
   int m;
   #pragma omp target update // expected-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
@@ -34,15 +40,114 @@ int main(int argc, char **argv) {
   #pragma omp target update to(m) ] // expected-warning {{extra tokens at the end of '#pragma omp target update' are ignored}}
   #pragma omp target update to(m) ) // expected-warning {{extra tokens at the end of '#pragma omp target update' are ignored}}
 
+  #pragma omp declare mapper(id: S s) map(s.i)
+  S s;
+
+  // Check parsing with no modifiers.
+  // lt51-error@+2 {{expected expression}}
+  // lt51-error@+1 {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
+  #pragma omp target update to(: s)
+  // expected-error@+2 {{expected expression}}
+  // expected-error@+1 {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
+  #pragma omp target update to(:)
+  // expected-error@+2 2 {{expected expression}}
+  // expected-error@+1 {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
+  #pragma omp target update to(,:)
+
+  // Check parsing with one modifier.
+  // expected-error@+2 {{use of undeclared identifier 'foobar'}}
+  // expected-error@+1 {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
+  #pragma omp target update to(foobar: s)
+  // expected-error@+3 {{expected ',' or ')' in 'to' clause}}
+  // expected-error@+2 {{expected ')'}}
+  // expected-note@+1 {{to match this '('}}
+  #pragma omp target update to(m: s)
+  #pragma omp target update to(mapper(id): s)
+  // lt51-error@+2 {{use of undeclared identifier 'present'}}
+  // lt51-error@+1 {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
+  #pragma omp target update to(present: s)
+  // ge51-warning@+4 {{missing ':' after motion modifier - ignoring}}
+  // lt51-warning@+3 {{missing ':' after ) - ignoring}}
+  // expected-error@+2 {{expected expression}}
+  // expected-error@+1 {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
+  #pragma omp target update to(mapper(id) s)
+  // ge51-warning@+4 {{missing ':' after motion modifier - ignoring}}
+  // ge51-error@+3 {{expected expression}}
+  // lt51-error@+2 {{use of undeclared identifier 'present'}}
+  // expected-error@+1 {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
+  #pragma omp target update to(present s)
+  // ge51-warning@+4 {{missing ':' after motion modifier - ignoring}}
+  // lt51-warning@+3 {{missing ':' after ) - ignoring}}
+  // expected-error@+2 {{expected expression}}
+  // expected-error@+1 {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
+  #pragma omp target update to(mapper(id))
+  // ge51-warning@+4 {{missing ':' after motion modifier - ignoring}}
+  // ge51-error@+3 {{expected expression}}
+  // lt51-error@+2 {{use of undeclared identifier 'present'}}
+  // expected-error@+1 {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
+  #pragma omp target update to(present)
+  // expected-error@+2 {{expected expression}}
+  // expected-error@+1 {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
+  #pragma omp target update to(mapper(id):)
+  // ge51-error@+3 {{expected expression}}
+  // lt51-error@+2 {{use of undeclared identifier 'present'}}
+  // expected-error@+1 {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
+  #pragma omp target update to(present:)
+
+  // Check parsing with two modifiers.
+  // lt51-warning@+1 {{missing ':' after ) - ignoring}}
+  #pragma omp target update to(mapper(id), present: s)
+  // lt51-error@+3 {{use of undeclared identifier 'present'}}
+  // lt51-error@+2 {{use of undeclared identifier 'id'}}
+  // lt51-error@+1 {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
+  #pragma omp target update to(present, mapper(id): s)
+  // lt51-warning@+1 {{missing ':' after ) - ignoring}}
+  #pragma omp target update to(mapper(id) present: s)
+  // lt51-error@+2 {{use of undeclared identifier 'present'}}
+  // lt51-error@+1 {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
+  #pragma omp target update to(present mapper(id): s)
+
+  // Check parsing with unnecessary commas.
+  // lt51-warning@+1 {{missing ':' after ) - ignoring}}
+  #pragma omp target update to(mapper(id),: s)
+  // lt51-error@+3 {{use of undeclared identifier 'present'}}
+  // lt51-error@+2 {{expected expression}}
+  // lt51-error@+1 {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
+  #pragma omp target update to(present , : s)
+  // ge51-warning@+2 {{missing ':' after motion modifier - ignoring}}
+  // lt51-warning@+1 {{missing ':' after ) - ignoring}}
+  #pragma omp target update to(mapper(id),,: s)
+  // ge51-warning@+5 {{missing ':' after motion modifier - ignoring}}
+  // lt51-error@+4 {{use of undeclared identifier 'present'}}
+  // lt51-error@+3 {{expected expression}}
+  // lt51-error@+2 {{expected expression}}
+  // lt51-error@+1 {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
+  #pragma omp target update to(present,,: s)
+  // lt51-warning@+1 {{missing ':' after ) - ignoring}}
+  #pragma omp target update to(mapper(id), present,: s)
+  // lt51-error@+4 {{use of undeclared identifier 'present'}}
+  // lt51-error@+3 {{use of undeclared identifier 'id'}}
+  // lt51-error@+2 {{expected expression}}
+  // lt51-error@+1 {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
+  #pragma omp target update to(present, mapper(id),: s)
+
   #pragma omp target update from(m) allocate(m) // expected-error {{unexpected OpenMP clause 'allocate' in directive '#pragma omp target update'}}
   {
     foo();
   }
 
   int iarr[5][5];
-#pragma omp target update to(iarr[0:][1:2:-1]) // omp50-error {{section stride is evaluated to a non-positive value -1}} omp45-error {{expected ']'}} omp45-note {{to match this '['}} expected-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
+// ge50-error@+4 {{section stride is evaluated to a non-positive value -1}}
+// lt50-error@+3 {{expected ']'}}
+// lt50-note@+2 {{to match this '['}}
+// expected-error@+1 {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
+#pragma omp target update to(iarr[0:][1:2:-1])
   {}
-#pragma omp target update from(iarr[0:][1:2:-1]) // omp50-error {{section stride is evaluated to a non-positive value -1}} omp45-error {{expected ']'}} omp45-note {{to match this '['}} expected-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
+// ge50-error@+4 {{section stride is evaluated to a non-positive value -1}}
+// lt50-error@+3 {{expected ']'}}
+// lt50-note@+2 {{to match this '['}}
+// expected-error@+1 {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
+#pragma omp target update from(iarr[0:][1:2:-1])
 
   return tmain(argc, argv);
 }

From cee52dd02672ecff33d9bd8300aa09a91853a149 Mon Sep 17 00:00:00 2001
From: "Joel E. Denny" <jdenny.ornl@gmail.com>
Date: Wed, 29 Jul 2020 12:18:50 -0400
Subject: [PATCH 0488/1035] [OpenMP] Implement TR8 `present` motion modifier in
 runtime (2/2)

This patch implements OpenMP runtime support for the OpenMP TR8
`present` motion modifier for `omp target update` directives.  The
previous patch in this series implements Clang front end support.

Reviewed By: grokos

Differential Revision: https://reviews.llvm.org/D84712
---
 openmp/libomptarget/src/omptarget.cpp         |  6 ++
 .../test/mapping/present/target_update.c      | 73 +++++++++++++++++++
 2 files changed, 79 insertions(+)
 create mode 100644 openmp/libomptarget/test/mapping/present/target_update.c

diff --git a/openmp/libomptarget/src/omptarget.cpp b/openmp/libomptarget/src/omptarget.cpp
index d391e245fdd66..409089500015e 100644
--- a/openmp/libomptarget/src/omptarget.cpp
+++ b/openmp/libomptarget/src/omptarget.cpp
@@ -628,6 +628,12 @@ int target_data_update(DeviceTy &Device, int32_t arg_num,
         false, IsHostPtr);
     if (!TgtPtrBegin) {
       DP("hst data:" DPxMOD " not found, becomes a noop\n", DPxPTR(HstPtrBegin));
+      if (arg_types[i] & OMP_TGT_MAPTYPE_PRESENT) {
+        MESSAGE("device mapping required by 'present' motion modifier does not "
+                "exist for host address " DPxMOD " (%ld bytes)",
+                DPxPTR(HstPtrBegin), MapSize);
+        return OFFLOAD_FAIL;
+      }
       continue;
     }
 
diff --git a/openmp/libomptarget/test/mapping/present/target_update.c b/openmp/libomptarget/test/mapping/present/target_update.c
new file mode 100644
index 0000000000000..3a3575a27d015
--- /dev/null
+++ b/openmp/libomptarget/test/mapping/present/target_update.c
@@ -0,0 +1,73 @@
+// --------------------------------------------------
+// Check 'to'
+// --------------------------------------------------
+
+// RUN: %libomptarget-compile-aarch64-unknown-linux-gnu \
+// RUN:   -fopenmp-version=51 -DCLAUSE=to
+// RUN: %libomptarget-run-fail-aarch64-unknown-linux-gnu 2>&1 \
+// RUN: | %fcheck-aarch64-unknown-linux-gnu
+
+// RUN: %libomptarget-compile-powerpc64-ibm-linux-gnu \
+// RUN:   -fopenmp-version=51 -DCLAUSE=to
+// RUN: %libomptarget-run-fail-powerpc64-ibm-linux-gnu 2>&1 \
+// RUN: | %fcheck-powerpc64-ibm-linux-gnu
+
+// RUN: %libomptarget-compile-powerpc64le-ibm-linux-gnu \
+// RUN:   -fopenmp-version=51 -DCLAUSE=to
+// RUN: %libomptarget-run-fail-powerpc64le-ibm-linux-gnu 2>&1 \
+// RUN: | %fcheck-powerpc64le-ibm-linux-gnu
+
+// RUN: %libomptarget-compile-x86_64-pc-linux-gnu \
+// RUN:   -fopenmp-version=51 -DCLAUSE=to
+// RUN: %libomptarget-run-fail-x86_64-pc-linux-gnu 2>&1 \
+// RUN: | %fcheck-x86_64-pc-linux-gnu
+
+// --------------------------------------------------
+// Check 'from'
+// --------------------------------------------------
+
+// RUN: %libomptarget-compile-aarch64-unknown-linux-gnu \
+// RUN:   -fopenmp-version=51 -DCLAUSE=from
+// RUN: %libomptarget-run-fail-aarch64-unknown-linux-gnu 2>&1 \
+// RUN: | %fcheck-aarch64-unknown-linux-gnu
+
+// RUN: %libomptarget-compile-powerpc64-ibm-linux-gnu \
+// RUN:   -fopenmp-version=51 -DCLAUSE=from
+// RUN: %libomptarget-run-fail-powerpc64-ibm-linux-gnu 2>&1 \
+// RUN: | %fcheck-powerpc64-ibm-linux-gnu
+
+// RUN: %libomptarget-compile-powerpc64le-ibm-linux-gnu \
+// RUN:   -fopenmp-version=51 -DCLAUSE=from
+// RUN: %libomptarget-run-fail-powerpc64le-ibm-linux-gnu 2>&1 \
+// RUN: | %fcheck-powerpc64le-ibm-linux-gnu
+
+// RUN: %libomptarget-compile-x86_64-pc-linux-gnu \
+// RUN:   -fopenmp-version=51 -DCLAUSE=from
+// RUN: %libomptarget-run-fail-x86_64-pc-linux-gnu 2>&1 \
+// RUN: | %fcheck-x86_64-pc-linux-gnu
+
+#include <stdio.h>
+
+int main() {
+  int i;
+
+  // CHECK: addr=0x[[#%x,HOST_ADDR:]], size=[[#%u,SIZE:]]
+  fprintf(stderr, "addr=%p, size=%ld\n", &i, sizeof i);
+
+  // CHECK-NOT: Libomptarget
+#pragma omp target enter data map(alloc: i)
+#pragma omp target update CLAUSE(present: i)
+#pragma omp target exit data map(delete: i)
+
+  // CHECK: i is present
+  fprintf(stderr, "i is present\n");
+
+  // CHECK: Libomptarget message: device mapping required by 'present' motion modifier does not exist for host address 0x{{0*}}[[#HOST_ADDR]] ([[#SIZE]] bytes)
+  // CHECK: Libomptarget fatal error 1: failure of target construct while offloading is mandatory
+#pragma omp target update CLAUSE(present: i)
+
+  // CHECK-NOT: i is present
+  fprintf(stderr, "i is present\n");
+
+  return 0;
+}

From 31342eb63e93ae1f453fe61924bd2c1f134dd1c0 Mon Sep 17 00:00:00 2001
From: Philip Reames <listmail@philipreames.com>
Date: Wed, 29 Jul 2020 09:13:15 -0700
Subject: [PATCH 0489/1035] [Statepoint] When using the tied def lowering,
 unconditionally use vregs [almost NFC]

This builds on 3da1a96 on the path towards supporting invokes and cross block relocations. The actual change attempts to be NFC, but does fail in one corner-case explained below.

The change itself is fairly mechanical. Rather than remember SDValues - which are inherently block local - immediately produce a virtual register copy and remember that.

Once this lands, we'll update the FunctionLoweringInfo::StatepointSpillMap map to allow register based lowerings, delete VirtRegs from StatepointLowering, and drop the restriction against cross block relocations. I deliberately separate the semantic part into it's own change for easy of understanding and fault isolation.

The corner-case which isn't quite NFC is that the old implementation implicitly CSEd gc.relocates of the same SDValue regardless of type. The new implementation still only relocates once, but it produces distinct vregs for the bitcast and it's source, whereas SelectionDAG's generic CSE was able to remove the bitcast in the old implementation. Note that the final assembly doesn't change (at least in the test), as our MI level optimizations catch the duplication.

I assert that this is an uninteresting corner-case. It's functionally correct, and if we find a case where this influences performance, we should really be canonicalizing types to i8* at the IR level.

Differential Revision: https://reviews.llvm.org/D84692
---
 .../SelectionDAG/StatepointLowering.cpp       | 60 +++++++++++++------
 .../CodeGen/SelectionDAG/StatepointLowering.h |  6 +-
 llvm/test/CodeGen/X86/statepoint-vreg.ll      |  5 +-
 3 files changed, 47 insertions(+), 24 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
index 4d93dec9e1fad..5fd1ab6f20912 100644
--- a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
@@ -90,13 +90,13 @@ void StatepointLoweringState::startNewStatepoint(SelectionDAGBuilder &Builder) {
   // FunctionLoweringInfo.  Also need to ensure used bits get cleared.
   AllocatedStackSlots.clear();
   AllocatedStackSlots.resize(Builder.FuncInfo.StatepointStackSlots.size());
-  DerivedPtrMap.clear();
+  VirtRegs.clear();
 }
 
 void StatepointLoweringState::clear() {
   Locations.clear();
   AllocatedStackSlots.clear();
-  DerivedPtrMap.clear();
+  VirtRegs.clear();
   assert(PendingGCRelocateCalls.empty() &&
          "cleared before statepoint sequence completed");
 }
@@ -691,8 +691,9 @@ lowerStatepointMetaArgs(SmallVectorImpl<SDValue> &Ops,
 
       // Conservatively export all values used by gc.relocates outside this
       // block.  This is currently only needed for expressions which don't need
-      // relocation, but will likely be extended for vreg case shortly.
-      if (Relocate->getParent() != StatepointInstr->getParent()) {
+      // relocation (such as constants and allocas).
+      if (!LowerAsVReg.count(SDV) &&
+          Relocate->getParent() != StatepointInstr->getParent()) {
         Builder.ExportFromCurrentBlock(V);
         assert(!LowerAsVReg.count(SDV));
       }
@@ -857,15 +858,31 @@ SDValue SelectionDAGBuilder::LowerAsSTATEPOINT(
 
   SDNode *SinkNode = StatepointMCNode;
 
-  // Fill mapping from derived pointer to statepoint result denoting its
-  // relocated value.
-  auto &DPtrMap = StatepointLowering.DerivedPtrMap;
+  // Remember the mapping between values relocated in registers and the virtual
+  // register holding the relocation.  Note that for simplicity, we *always*
+  // create a vreg even within a single block.
+  auto &VirtRegs = StatepointLowering.VirtRegs;
   for (const auto *Relocate : SI.GCRelocates) {
     Value *Derived = Relocate->getDerivedPtr();
     SDValue SD = getValue(Derived);
     if (!LowerAsVReg.count(SD))
       continue;
-    DPtrMap[Derived] = SDValue(StatepointMCNode, LowerAsVReg[SD]);
+
+    // Handle multiple gc.relocates of the same input efficiently.
+    if (VirtRegs.count(Derived))
+      continue;
+
+    SDValue Relocated = SDValue(StatepointMCNode, LowerAsVReg[SD]);
+
+    auto *RetTy = Relocate->getType();
+    Register Reg = FuncInfo.CreateRegs(RetTy);
+    RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(),
+                     DAG.getDataLayout(), Reg, RetTy, None);
+    SDValue Chain = DAG.getEntryNode();
+    RFV.getCopyToRegs(Relocated, DAG, getCurSDLoc(), Chain, nullptr);
+    PendingExports.push_back(Chain);
+    
+    VirtRegs[Derived] = Reg; 
   }
 
   // Build the GC_TRANSITION_END node if necessary.
@@ -1101,6 +1118,22 @@ void SelectionDAGBuilder::visitGCRelocate(const GCRelocateInst &Relocate) {
 #endif
 
   const Value *DerivedPtr = Relocate.getDerivedPtr();
+
+  // If relocation was done via virtual register..
+  auto &VirtRegs = StatepointLowering.VirtRegs;
+  auto It = VirtRegs.find(DerivedPtr);
+  if (It != VirtRegs.end()) {
+    Register InReg = It->second;
+    RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(),
+                     DAG.getDataLayout(), InReg, Relocate.getType(),
+                     None); // This is not an ABI copy.
+    SDValue Chain = DAG.getEntryNode();
+    SDValue Relocation = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(),
+                                             Chain, nullptr, nullptr);
+    setValue(&Relocate, Relocation);
+    return;
+  }
+  
   SDValue SD = getValue(DerivedPtr);
 
   if (SD.isUndef() && SD.getValueType().getSizeInBits() <= 64) {
@@ -1110,17 +1143,6 @@ void SelectionDAGBuilder::visitGCRelocate(const GCRelocateInst &Relocate) {
     return;
   }
 
-  // Relocate is local to statepoint block and its pointer was assigned
-  // to VReg. Use corresponding statepoint result.
-  auto &DPtrMap = StatepointLowering.DerivedPtrMap;
-  auto It = DPtrMap.find(DerivedPtr);
-  if (It != DPtrMap.end()) {
-    setValue(&Relocate, It->second);
-    assert(Relocate.getParent() == Relocate.getStatepoint()->getParent() &&
-           "unexpected DPtrMap entry");
-    return;
-  }
-
   auto &SpillMap = FuncInfo.StatepointSpillMaps[Relocate.getStatepoint()];
   auto SlotIt = SpillMap.find(DerivedPtr);
   assert(SlotIt != SpillMap.end() && "Relocating not lowered gc value");
diff --git a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.h b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.h
index d6c18379c5add..521a5c60053da 100644
--- a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.h
+++ b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.h
@@ -103,9 +103,9 @@ class StatepointLoweringState {
     return AllocatedStackSlots.test(Offset);
   }
 
-  /// For each statepoint keep mapping from original derived pointer to
-  /// the statepoint node result defining its new value.
-  DenseMap<const Value *, SDValue> DerivedPtrMap;
+  /// Maps pre-relocated value to virtual register holding it's relocation if
+  /// vreg lowering was used.
+  DenseMap<const Value *, Register> VirtRegs;
 
 private:
   /// Maps pre-relocation value (gc pointer directly incoming into statepoint)
diff --git a/llvm/test/CodeGen/X86/statepoint-vreg.ll b/llvm/test/CodeGen/X86/statepoint-vreg.ll
index bb86e9e1f1cfb..2d8dea4179a07 100644
--- a/llvm/test/CodeGen/X86/statepoint-vreg.ll
+++ b/llvm/test/CodeGen/X86/statepoint-vreg.ll
@@ -310,10 +310,11 @@ define void @test_gcptr_uniqueing(i32 addrspace(1)* %ptr) gc "statepoint-example
 ; CHECK-VREG-LABEL: name:            test_gcptr_uniqueing
 ; CHECK-VREG:    %0:gr64 = COPY $rdi
 ; CHECK-VREG:    ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
-; CHECK-VREG:    %1:gr64 = STATEPOINT 0, 0, 0, @func, 2, 0, 2, 0, 2, 2, %0, 2, 4278124286, %0, %0(tied-def 0), csr_64, implicit-def $rsp, implicit-def $ssp
+; CHECK-VREG:    %2:gr64 = STATEPOINT 0, 0, 0, @func, 2, 0, 2, 0, 2, 2, %0, 2, 4278124286, %0, %0(tied-def 0), csr_64, implicit-def $rsp, implicit-def $ssp
 ; CHECK-VREG:    ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+; CHECK-VREG:    %1:gr64 = COPY %2
 ; CHECK-VREG:    ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
-; CHECK-VREG:    $rdi = COPY %1
+; CHECK-VREG:    $rdi = COPY %2
 ; CHECK-VREG:    $rsi = COPY %1
 ; CHECK-VREG:    CALL64pcrel32 @use1, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit $rsi, implicit-def $rsp, implicit-def $ssp
 

From 802c043078ad653aca131648a130b59f041df0b5 Mon Sep 17 00:00:00 2001
From: Kang Zhang <shkzhang@cn.ibm.com>
Date: Wed, 29 Jul 2020 16:39:27 +0000
Subject: [PATCH 0490/1035] [PowerPC] Set v1i128 to expand for SETCC to avoid
 crash

Summary:
PPC only supports the instruction selection for v16i8, v8i16, v4i32,
v2i64, v4f32 and v2f64 for ISD::SETCC, don't support the v1i128, so
v1i128 for ISD::SETCC will crash.

This patch is to set v1i128 to expand to avoid crash.

Reviewed By: steven.zhang

Differential Revision: https://reviews.llvm.org/D84238
---
 llvm/lib/Target/PowerPC/PPCISelLowering.cpp |  2 +
 llvm/test/CodeGen/PowerPC/setcc-vector.ll   | 49 +++++++++++++++++++++
 2 files changed, 51 insertions(+)
 create mode 100644 llvm/test/CodeGen/PowerPC/setcc-vector.ll

diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index 29a6d54d3296c..7f93831f44f65 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -937,6 +937,8 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM,
         setOperationAction(ISD::SUB, MVT::v2i64, Expand);
       }
 
+      setOperationAction(ISD::SETCC, MVT::v1i128, Expand);
+
       setOperationAction(ISD::LOAD, MVT::v2i64, Promote);
       AddPromotedToType (ISD::LOAD, MVT::v2i64, MVT::v2f64);
       setOperationAction(ISD::STORE, MVT::v2i64, Promote);
diff --git a/llvm/test/CodeGen/PowerPC/setcc-vector.ll b/llvm/test/CodeGen/PowerPC/setcc-vector.ll
new file mode 100644
index 0000000000000..5917ccabf84ed
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/setcc-vector.ll
@@ -0,0 +1,49 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -verify-machineinstrs -mcpu=pwr9 -mtriple=powerpc64le-unknown-unknown \
+; RUN:   -ppc-asm-full-reg-names < %s | FileCheck -check-prefixes=CHECK-PWR9 %s
+; RUN: llc -verify-machineinstrs -mcpu=pwr8 -mtriple=powerpc64le-unknown-unknown \
+; RUN:   -ppc-asm-full-reg-names < %s | FileCheck -check-prefixes=CHECK-PWR8  %s
+; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mtriple=powerpc64-unknown-unknown \
+; RUN:   -ppc-asm-full-reg-names < %s | FileCheck -check-prefixes=CHECK-PWR7 %s
+
+define <1 x i64> @setcc_v1i128(<1 x i128> %a) {
+; CHECK-PWR9-LABEL: setcc_v1i128:
+; CHECK-PWR9:       # %bb.0: # %entry
+; CHECK-PWR9-NEXT:    mfvsrld r3, vs34
+; CHECK-PWR9-NEXT:    cmpldi r3, 35708
+; CHECK-PWR9-NEXT:    mfvsrd r3, vs34
+; CHECK-PWR9-NEXT:    cmpdi cr1, r3, 0
+; CHECK-PWR9-NEXT:    li r3, 1
+; CHECK-PWR9-NEXT:    crnand 4*cr5+lt, 4*cr1+eq, lt
+; CHECK-PWR9-NEXT:    isel r3, 0, r3, 4*cr5+lt
+; CHECK-PWR9-NEXT:    blr
+;
+; CHECK-PWR8-LABEL: setcc_v1i128:
+; CHECK-PWR8:       # %bb.0: # %entry
+; CHECK-PWR8-NEXT:    xxswapd vs0, vs34
+; CHECK-PWR8-NEXT:    mfvsrd r3, vs34
+; CHECK-PWR8-NEXT:    cmpdi r3, 0
+; CHECK-PWR8-NEXT:    li r3, 1
+; CHECK-PWR8-NEXT:    mffprd r4, f0
+; CHECK-PWR8-NEXT:    cmpldi cr1, r4, 35708
+; CHECK-PWR8-NEXT:    crnand 4*cr5+lt, eq, 4*cr1+lt
+; CHECK-PWR8-NEXT:    isel r3, 0, r3, 4*cr5+lt
+; CHECK-PWR8-NEXT:    blr
+;
+; CHECK-PWR7-LABEL: setcc_v1i128:
+; CHECK-PWR7:       # %bb.0: # %entry
+; CHECK-PWR7-NEXT:    li r5, 0
+; CHECK-PWR7-NEXT:    cntlzd r3, r3
+; CHECK-PWR7-NEXT:    ori r5, r5, 35708
+; CHECK-PWR7-NEXT:    rldicl r3, r3, 58, 63
+; CHECK-PWR7-NEXT:    subc r5, r4, r5
+; CHECK-PWR7-NEXT:    subfe r4, r4, r4
+; CHECK-PWR7-NEXT:    neg r4, r4
+; CHECK-PWR7-NEXT:    and r3, r3, r4
+; CHECK-PWR7-NEXT:    blr
+entry:
+  %0 = icmp ult <1 x i128> %a, <i128 35708>
+  %1 = zext <1 x i1> %0 to <1 x i64>
+  ret <1 x i64> %1
+}
+

From 1d51dc38d89bd33fb8874e242ab87b265b4dec1c Mon Sep 17 00:00:00 2001
From: Roman Lebedev <lebedev.ri@gmail.com>
Date: Wed, 29 Jul 2020 19:54:33 +0300
Subject: [PATCH 0491/1035] [SimplifyCFG][LoopRotate] SimplifyCFG: disable
 common instruction hoisting by default, enable late in pipeline

I've been looking at missed vectorizations in one codebase.
One particular thing that stands out is that some of the loops
reach vectorizer in a rather mangled form, with weird PHI's,
and some of the loops aren't even in a rotated form.

After taking a more detailed look, that happened because
the loop's headers were too big by then. It is evident that
SimplifyCFG's common code hoisting transform is at fault there,
because the pattern it handles is precisely the unrotated
loop basic block structure.

Surprizingly, `SimplifyCFGOpt::HoistThenElseCodeToIf()` is enabled
by default, and is always run, unlike it's friend, common code sinking
transform, `SinkCommonCodeFromPredecessors()`, which is not enabled
by default and is only run once very late in the pipeline.

I'm proposing to harmonize this, and disable common code hoisting
until //late// in pipeline. Definition of //late// may vary,
here currently i've picked the same one as for code sinking,
but i suppose we could enable it as soon as right after
loop rotation happens.

Experimentation shows that this does indeed unsurprizingly help,
more loops got rotated, although other issues remain elsewhere.

Now, this undoubtedly seriously shakes phase ordering.
This will undoubtedly be a mixed bag in terms of both compile- and
run- time performance, codesize. Since we no longer aggressively
hoist+deduplicate common code, we don't pay the price of said hoisting
(which wasn't big). That may allow more loops to be rotated,
so we pay that price. That, in turn, that may enable all the transforms
that require canonical (rotated) loop form, including but not limited to
vectorization, so we pay that too. And in general, no deduplication means
more [duplicate] instructions going through the optimizations. But there's still
late hoisting, some of them will be caught late.

As per benchmarks i've run {F12360204}, this is mostly within the noise,
there are some small improvements, some small regressions.
One big regression i saw i fixed in rG8d487668d09fb0e4e54f36207f07c1480ffabbfd, but i'm sure
this will expose many more pre-existing missed optimizations, as usual :S

llvm-compile-time-tracker.com thoughts on this:
http://llvm-compile-time-tracker.com/compare.php?from=e40315d2b4ed1e38962a8f33ff151693ed4ada63&to=c8289c0ecbf235da9fb0e3bc052e3c0d6bff5cf9&stat=instructions
* this does regress compile-time by +0.5% geomean (unsurprizingly)
* size impact varies; for ThinLTO it's actually an improvement

The largest fallout appears to be in GVN's load partial redundancy
elimination, it spends *much* more time in
`MemoryDependenceResults::getNonLocalPointerDependency()`.
Non-local `MemoryDependenceResults` is widely-known to be, uh, costly.
There does not appear to be a proper solution to this issue,
other than silencing the compile-time performance regression
by tuning cut-off thresholds in `MemoryDependenceResults`,
at the cost of potentially regressing run-time performance.
D84609 attempts to move in that direction, but the path is unclear
and is going to take some time.

If we look at stats before/after diffs, some excerpts:
* RawSpeed (the target) {F12360200}
  * -14 (-73.68%) loops not rotated due to the header size (yay)
  * -272 (-0.67%) `"Number of live out of a loop variables"` - good for vectorizer
  * -3937 (-64.19%) common instructions hoisted
  * +561 (+0.06%) x86 asm instructions
  * -2 basic blocks
  * +2418 (+0.11%) IR instructions
* vanilla test-suite + RawSpeed + darktable  {F12360201}
  * -36396 (-65.29%) common instructions hoisted
  * +1676 (+0.02%) x86 asm instructions
  * +662 (+0.06%) basic blocks
  * +4395 (+0.04%) IR instructions

It is likely to be sub-optimal for when optimizing for code size,
so one might want to change tune pipeline by enabling sinking/hoisting
when optimizing for size.

Reviewed By: mkazantsev

Differential Revision: https://reviews.llvm.org/D84108
---
 .../Transforms/Utils/SimplifyCFGOptions.h     |  2 +-
 llvm/lib/Passes/PassBuilder.cpp               | 13 +++++----
 .../Target/AArch64/AArch64TargetMachine.cpp   |  1 +
 llvm/lib/Target/ARM/ARMTargetMachine.cpp      |  3 +-
 .../Target/Hexagon/HexagonTargetMachine.cpp   |  1 +
 .../lib/Transforms/IPO/PassManagerBuilder.cpp |  3 ++
 .../lib/Transforms/Scalar/SimplifyCFGPass.cpp |  4 +--
 llvm/test/Transforms/PGOProfile/chr.ll        |  7 +++++
 .../loop-rotation-vs-common-code-hoisting.ll  | 29 +++++++++----------
 .../SimplifyCFG/common-code-hoisting.ll       |  2 +-
 10 files changed, 39 insertions(+), 26 deletions(-)

diff --git a/llvm/include/llvm/Transforms/Utils/SimplifyCFGOptions.h b/llvm/include/llvm/Transforms/Utils/SimplifyCFGOptions.h
index 46f6ca0462f8b..fb3a7490346f4 100644
--- a/llvm/include/llvm/Transforms/Utils/SimplifyCFGOptions.h
+++ b/llvm/include/llvm/Transforms/Utils/SimplifyCFGOptions.h
@@ -25,7 +25,7 @@ struct SimplifyCFGOptions {
   bool ForwardSwitchCondToPhi = false;
   bool ConvertSwitchToLookupTable = false;
   bool NeedCanonicalLoop = true;
-  bool HoistCommonInsts = true;
+  bool HoistCommonInsts = false;
   bool SinkCommonInsts = false;
   bool SimplifyCondBranch = true;
   bool FoldTwoEntryPHINode = true;
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index 771675d35fdf2..e306833d1d30d 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -1144,11 +1144,14 @@ ModulePassManager PassBuilder::buildModuleOptimizationPipeline(
   // convert to more optimized IR using more aggressive simplify CFG options.
   // The extra sinking transform can create larger basic blocks, so do this
   // before SLP vectorization.
-  OptimizePM.addPass(SimplifyCFGPass(SimplifyCFGOptions().
-                                     forwardSwitchCondToPhi(true).
-                                     convertSwitchToLookupTable(true).
-                                     needCanonicalLoops(false).
-                                     sinkCommonInsts(true)));
+  // FIXME: study whether hoisting and/or sinking of common instructions should
+  //        be delayed until after SLP vectorizer.
+  OptimizePM.addPass(SimplifyCFGPass(SimplifyCFGOptions()
+                                         .forwardSwitchCondToPhi(true)
+                                         .convertSwitchToLookupTable(true)
+                                         .needCanonicalLoops(false)
+                                         .hoistCommonInsts(true)
+                                         .sinkCommonInsts(true)));
 
   // Optimize parallel scalar instruction chains into SIMD instructions.
   if (PTO.SLPVectorization)
diff --git a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
index 40d71def6d09a..819f7b614106e 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
@@ -457,6 +457,7 @@ void AArch64PassConfig::addIRPasses() {
                                             .forwardSwitchCondToPhi(true)
                                             .convertSwitchToLookupTable(true)
                                             .needCanonicalLoops(false)
+                                            .hoistCommonInsts(true)
                                             .sinkCommonInsts(true)));
 
   // Run LoopDataPrefetch
diff --git a/llvm/lib/Target/ARM/ARMTargetMachine.cpp b/llvm/lib/Target/ARM/ARMTargetMachine.cpp
index b316b1041f2c5..87a106474c5ec 100644
--- a/llvm/lib/Target/ARM/ARMTargetMachine.cpp
+++ b/llvm/lib/Target/ARM/ARMTargetMachine.cpp
@@ -409,7 +409,8 @@ void ARMPassConfig::addIRPasses() {
   // ldrex/strex loops to simplify this, but it needs tidying up.
   if (TM->getOptLevel() != CodeGenOpt::None && EnableAtomicTidy)
     addPass(createCFGSimplificationPass(
-        SimplifyCFGOptions().sinkCommonInsts(true), [this](const Function &F) {
+        SimplifyCFGOptions().hoistCommonInsts(true).sinkCommonInsts(true),
+        [this](const Function &F) {
           const auto &ST = this->TM->getSubtarget<ARMSubtarget>(F);
           return ST.hasAnyDataBarrier() && !ST.isThumb1Only();
         }));
diff --git a/llvm/lib/Target/Hexagon/HexagonTargetMachine.cpp b/llvm/lib/Target/Hexagon/HexagonTargetMachine.cpp
index 49d98622d946c..03cc7d240aa66 100644
--- a/llvm/lib/Target/Hexagon/HexagonTargetMachine.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonTargetMachine.cpp
@@ -324,6 +324,7 @@ void HexagonPassConfig::addIRPasses() {
                                               .forwardSwitchCondToPhi(true)
                                               .convertSwitchToLookupTable(true)
                                               .needCanonicalLoops(false)
+                                              .hoistCommonInsts(true)
                                               .sinkCommonInsts(true)));
     if (EnableLoopPrefetch)
       addPass(createLoopDataPrefetchPass());
diff --git a/llvm/lib/Transforms/IPO/PassManagerBuilder.cpp b/llvm/lib/Transforms/IPO/PassManagerBuilder.cpp
index c045c277706b3..22daa8e812f69 100644
--- a/llvm/lib/Transforms/IPO/PassManagerBuilder.cpp
+++ b/llvm/lib/Transforms/IPO/PassManagerBuilder.cpp
@@ -784,10 +784,13 @@ void PassManagerBuilder::populateModulePassManager(
   // convert to more optimized IR using more aggressive simplify CFG options.
   // The extra sinking transform can create larger basic blocks, so do this
   // before SLP vectorization.
+  // FIXME: study whether hoisting and/or sinking of common instructions should
+  //        be delayed until after SLP vectorizer.
   MPM.add(createCFGSimplificationPass(SimplifyCFGOptions()
                                           .forwardSwitchCondToPhi(true)
                                           .convertSwitchToLookupTable(true)
                                           .needCanonicalLoops(false)
+                                          .hoistCommonInsts(true)
                                           .sinkCommonInsts(true)));
 
   if (SLPVectorize) {
diff --git a/llvm/lib/Transforms/Scalar/SimplifyCFGPass.cpp b/llvm/lib/Transforms/Scalar/SimplifyCFGPass.cpp
index db5211df397a8..b0435bf6e4eac 100644
--- a/llvm/lib/Transforms/Scalar/SimplifyCFGPass.cpp
+++ b/llvm/lib/Transforms/Scalar/SimplifyCFGPass.cpp
@@ -63,8 +63,8 @@ static cl::opt<bool> UserForwardSwitchCond(
     cl::desc("Forward switch condition to phi ops (default = false)"));
 
 static cl::opt<bool> UserHoistCommonInsts(
-    "hoist-common-insts", cl::Hidden, cl::init(true),
-    cl::desc("hoist common instructions (default = true)"));
+    "hoist-common-insts", cl::Hidden, cl::init(false),
+    cl::desc("hoist common instructions (default = false)"));
 
 static cl::opt<bool> UserSinkCommonInsts(
     "sink-common-insts", cl::Hidden, cl::init(false),
diff --git a/llvm/test/Transforms/PGOProfile/chr.ll b/llvm/test/Transforms/PGOProfile/chr.ll
index e2161d617022f..776501402b195 100644
--- a/llvm/test/Transforms/PGOProfile/chr.ll
+++ b/llvm/test/Transforms/PGOProfile/chr.ll
@@ -2008,9 +2008,16 @@ define i64 @test_chr_22(i1 %i, i64* %j, i64 %v0) !prof !14 {
 ; CHECK-NEXT:  bb0:
 ; CHECK-NEXT:    [[REASS_ADD:%.*]] = shl i64 [[V0:%.*]], 1
 ; CHECK-NEXT:    [[V2:%.*]] = add i64 [[REASS_ADD]], 3
+; CHECK-NEXT:    [[C1:%.*]] = icmp slt i64 [[V2]], 100
+; CHECK-NEXT:    br i1 [[C1]], label [[BB0_SPLIT:%.*]], label [[BB0_SPLIT_NONCHR:%.*]], !prof !15
+; CHECK:       bb0.split:
 ; CHECK-NEXT:    [[V299:%.*]] = mul i64 [[V2]], 7860086430977039991
 ; CHECK-NEXT:    store i64 [[V299]], i64* [[J:%.*]], align 4
 ; CHECK-NEXT:    ret i64 99
+; CHECK:       bb0.split.nonchr:
+; CHECK-NEXT:    [[V299_NONCHR:%.*]] = mul i64 [[V2]], 7860086430977039991
+; CHECK-NEXT:    store i64 [[V299_NONCHR]], i64* [[J]], align 4
+; CHECK-NEXT:    ret i64 99
 ;
 bb0:
   %v1 = add i64 %v0, 3
diff --git a/llvm/test/Transforms/PhaseOrdering/loop-rotation-vs-common-code-hoisting.ll b/llvm/test/Transforms/PhaseOrdering/loop-rotation-vs-common-code-hoisting.ll
index 1d8cce6879e9d..314af1c141454 100644
--- a/llvm/test/Transforms/PhaseOrdering/loop-rotation-vs-common-code-hoisting.ll
+++ b/llvm/test/Transforms/PhaseOrdering/loop-rotation-vs-common-code-hoisting.ll
@@ -5,14 +5,11 @@
 ; RUN: opt -O3 -rotation-max-header-size=1 -S < %s                    | FileCheck %s --check-prefixes=HOIST,THR1,FALLBACK2
 ; RUN: opt -passes='default<O3>' -rotation-max-header-size=1 -S < %s  | FileCheck %s --check-prefixes=HOIST,THR1,FALLBACK3
 
-; RUN: opt -O3 -rotation-max-header-size=2 -S < %s                    | FileCheck %s --check-prefixes=HOIST,THR2,FALLBACK4
-; RUN: opt -passes='default<O3>' -rotation-max-header-size=2 -S < %s  | FileCheck %s --check-prefixes=HOIST,THR2,FALLBACK5
+; RUN: opt -O3 -rotation-max-header-size=2 -S < %s                    | FileCheck %s --check-prefixes=ROTATED_LATER,ROTATED_LATER_OLDPM,FALLBACK4
+; RUN: opt -passes='default<O3>' -rotation-max-header-size=2 -S < %s  | FileCheck %s --check-prefixes=ROTATED_LATER,ROTATED_LATER_NEWPM,FALLBACK5
 
-; RUN: opt -O3 -rotation-max-header-size=3 -S < %s                    | FileCheck %s --check-prefixes=ROTATED_LATER,ROTATED_LATER_OLDPM,FALLBACK6
-; RUN: opt -passes='default<O3>' -rotation-max-header-size=3 -S < %s  | FileCheck %s --check-prefixes=ROTATED_LATER,ROTATED_LATER_NEWPM,FALLBACK7
-
-; RUN: opt -O3 -rotation-max-header-size=4 -S < %s                    | FileCheck %s --check-prefixes=ROTATE,ROTATE_OLDPM,FALLBACK8
-; RUN: opt -passes='default<O3>' -rotation-max-header-size=4 -S < %s  | FileCheck %s --check-prefixes=ROTATE,ROTATE_NEWPM,FALLBACK9
+; RUN: opt -O3 -rotation-max-header-size=3 -S < %s                    | FileCheck %s --check-prefixes=ROTATE,ROTATE_OLDPM,FALLBACK6
+; RUN: opt -passes='default<O3>' -rotation-max-header-size=3 -S < %s  | FileCheck %s --check-prefixes=ROTATE,ROTATE_NEWPM,FALLBACK7
 
 ; This example is produced from a very basic C code:
 ;
@@ -61,8 +58,8 @@ define void @_Z4loopi(i32 %width) {
 ; HOIST-NEXT:    br label [[FOR_COND:%.*]]
 ; HOIST:       for.cond:
 ; HOIST-NEXT:    [[I_0:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY:%.*]] ], [ 0, [[FOR_COND_PREHEADER]] ]
-; HOIST-NEXT:    tail call void @f0()
 ; HOIST-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[I_0]], [[TMP0]]
+; HOIST-NEXT:    tail call void @f0()
 ; HOIST-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY]]
 ; HOIST:       for.cond.cleanup:
 ; HOIST-NEXT:    tail call void @f2()
@@ -80,17 +77,17 @@ define void @_Z4loopi(i32 %width) {
 ; ROTATED_LATER_OLDPM-NEXT:    br i1 [[CMP]], label [[RETURN:%.*]], label [[FOR_COND_PREHEADER:%.*]]
 ; ROTATED_LATER_OLDPM:       for.cond.preheader:
 ; ROTATED_LATER_OLDPM-NEXT:    [[TMP0:%.*]] = add nsw i32 [[WIDTH]], -1
-; ROTATED_LATER_OLDPM-NEXT:    tail call void @f0()
 ; ROTATED_LATER_OLDPM-NEXT:    [[EXITCOND_NOT3:%.*]] = icmp eq i32 [[TMP0]], 0
 ; ROTATED_LATER_OLDPM-NEXT:    br i1 [[EXITCOND_NOT3]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY:%.*]]
 ; ROTATED_LATER_OLDPM:       for.cond.cleanup:
+; ROTATED_LATER_OLDPM-NEXT:    tail call void @f0()
 ; ROTATED_LATER_OLDPM-NEXT:    tail call void @f2()
 ; ROTATED_LATER_OLDPM-NEXT:    br label [[RETURN]]
 ; ROTATED_LATER_OLDPM:       for.body:
 ; ROTATED_LATER_OLDPM-NEXT:    [[I_04:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[FOR_COND_PREHEADER]] ]
+; ROTATED_LATER_OLDPM-NEXT:    tail call void @f0()
 ; ROTATED_LATER_OLDPM-NEXT:    tail call void @f1()
 ; ROTATED_LATER_OLDPM-NEXT:    [[INC]] = add nuw i32 [[I_04]], 1
-; ROTATED_LATER_OLDPM-NEXT:    tail call void @f0()
 ; ROTATED_LATER_OLDPM-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[TMP0]]
 ; ROTATED_LATER_OLDPM-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]]
 ; ROTATED_LATER_OLDPM:       return:
@@ -102,19 +99,19 @@ define void @_Z4loopi(i32 %width) {
 ; ROTATED_LATER_NEWPM-NEXT:    br i1 [[CMP]], label [[RETURN:%.*]], label [[FOR_COND_PREHEADER:%.*]]
 ; ROTATED_LATER_NEWPM:       for.cond.preheader:
 ; ROTATED_LATER_NEWPM-NEXT:    [[TMP0:%.*]] = add nsw i32 [[WIDTH]], -1
-; ROTATED_LATER_NEWPM-NEXT:    tail call void @f0()
 ; ROTATED_LATER_NEWPM-NEXT:    [[EXITCOND_NOT3:%.*]] = icmp eq i32 [[TMP0]], 0
 ; ROTATED_LATER_NEWPM-NEXT:    br i1 [[EXITCOND_NOT3]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_COND_PREHEADER_FOR_BODY_CRIT_EDGE:%.*]]
 ; ROTATED_LATER_NEWPM:       for.cond.preheader.for.body_crit_edge:
 ; ROTATED_LATER_NEWPM-NEXT:    [[INC_1:%.*]] = add nuw i32 0, 1
 ; ROTATED_LATER_NEWPM-NEXT:    br label [[FOR_BODY:%.*]]
 ; ROTATED_LATER_NEWPM:       for.cond.cleanup:
+; ROTATED_LATER_NEWPM-NEXT:    tail call void @f0()
 ; ROTATED_LATER_NEWPM-NEXT:    tail call void @f2()
 ; ROTATED_LATER_NEWPM-NEXT:    br label [[RETURN]]
 ; ROTATED_LATER_NEWPM:       for.body:
 ; ROTATED_LATER_NEWPM-NEXT:    [[INC_PHI:%.*]] = phi i32 [ [[INC_0:%.*]], [[FOR_BODY_FOR_BODY_CRIT_EDGE:%.*]] ], [ [[INC_1]], [[FOR_COND_PREHEADER_FOR_BODY_CRIT_EDGE]] ]
-; ROTATED_LATER_NEWPM-NEXT:    tail call void @f1()
 ; ROTATED_LATER_NEWPM-NEXT:    tail call void @f0()
+; ROTATED_LATER_NEWPM-NEXT:    tail call void @f1()
 ; ROTATED_LATER_NEWPM-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC_PHI]], [[TMP0]]
 ; ROTATED_LATER_NEWPM-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY_FOR_BODY_CRIT_EDGE]]
 ; ROTATED_LATER_NEWPM:       for.body.for.body_crit_edge:
@@ -129,19 +126,19 @@ define void @_Z4loopi(i32 %width) {
 ; ROTATE_OLDPM-NEXT:    br i1 [[CMP]], label [[RETURN:%.*]], label [[FOR_COND_PREHEADER:%.*]]
 ; ROTATE_OLDPM:       for.cond.preheader:
 ; ROTATE_OLDPM-NEXT:    [[CMP13_NOT:%.*]] = icmp eq i32 [[WIDTH]], 1
-; ROTATE_OLDPM-NEXT:    tail call void @f0()
 ; ROTATE_OLDPM-NEXT:    br i1 [[CMP13_NOT]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY_PREHEADER:%.*]]
 ; ROTATE_OLDPM:       for.body.preheader:
 ; ROTATE_OLDPM-NEXT:    [[TMP0:%.*]] = add nsw i32 [[WIDTH]], -1
 ; ROTATE_OLDPM-NEXT:    br label [[FOR_BODY:%.*]]
 ; ROTATE_OLDPM:       for.cond.cleanup:
+; ROTATE_OLDPM-NEXT:    tail call void @f0()
 ; ROTATE_OLDPM-NEXT:    tail call void @f2()
 ; ROTATE_OLDPM-NEXT:    br label [[RETURN]]
 ; ROTATE_OLDPM:       for.body:
 ; ROTATE_OLDPM-NEXT:    [[I_04:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; ROTATE_OLDPM-NEXT:    tail call void @f0()
 ; ROTATE_OLDPM-NEXT:    tail call void @f1()
 ; ROTATE_OLDPM-NEXT:    [[INC]] = add nuw nsw i32 [[I_04]], 1
-; ROTATE_OLDPM-NEXT:    tail call void @f0()
 ; ROTATE_OLDPM-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[TMP0]]
 ; ROTATE_OLDPM-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]]
 ; ROTATE_OLDPM:       return:
@@ -153,19 +150,19 @@ define void @_Z4loopi(i32 %width) {
 ; ROTATE_NEWPM-NEXT:    br i1 [[CMP]], label [[RETURN:%.*]], label [[FOR_COND_PREHEADER:%.*]]
 ; ROTATE_NEWPM:       for.cond.preheader:
 ; ROTATE_NEWPM-NEXT:    [[CMP13_NOT:%.*]] = icmp eq i32 [[WIDTH]], 1
-; ROTATE_NEWPM-NEXT:    tail call void @f0()
 ; ROTATE_NEWPM-NEXT:    br i1 [[CMP13_NOT]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY_PREHEADER:%.*]]
 ; ROTATE_NEWPM:       for.body.preheader:
 ; ROTATE_NEWPM-NEXT:    [[TMP0:%.*]] = add nsw i32 [[WIDTH]], -1
 ; ROTATE_NEWPM-NEXT:    [[INC_1:%.*]] = add nuw nsw i32 0, 1
 ; ROTATE_NEWPM-NEXT:    br label [[FOR_BODY:%.*]]
 ; ROTATE_NEWPM:       for.cond.cleanup:
+; ROTATE_NEWPM-NEXT:    tail call void @f0()
 ; ROTATE_NEWPM-NEXT:    tail call void @f2()
 ; ROTATE_NEWPM-NEXT:    br label [[RETURN]]
 ; ROTATE_NEWPM:       for.body:
 ; ROTATE_NEWPM-NEXT:    [[INC_PHI:%.*]] = phi i32 [ [[INC_0:%.*]], [[FOR_BODY_FOR_BODY_CRIT_EDGE:%.*]] ], [ [[INC_1]], [[FOR_BODY_PREHEADER]] ]
-; ROTATE_NEWPM-NEXT:    tail call void @f1()
 ; ROTATE_NEWPM-NEXT:    tail call void @f0()
+; ROTATE_NEWPM-NEXT:    tail call void @f1()
 ; ROTATE_NEWPM-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC_PHI]], [[TMP0]]
 ; ROTATE_NEWPM-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY_FOR_BODY_CRIT_EDGE]]
 ; ROTATE_NEWPM:       for.body.for.body_crit_edge:
diff --git a/llvm/test/Transforms/SimplifyCFG/common-code-hoisting.ll b/llvm/test/Transforms/SimplifyCFG/common-code-hoisting.ll
index b58017ba7ef0b..37cbc4640e415 100644
--- a/llvm/test/Transforms/SimplifyCFG/common-code-hoisting.ll
+++ b/llvm/test/Transforms/SimplifyCFG/common-code-hoisting.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -simplifycfg -hoist-common-insts=1 -S < %s                    | FileCheck %s --check-prefixes=HOIST
 ; RUN: opt -simplifycfg -hoist-common-insts=0 -S < %s                    | FileCheck %s --check-prefixes=NOHOIST
-; RUN: opt -simplifycfg                       -S < %s                    | FileCheck %s --check-prefixes=HOIST,DEFAULT
+; RUN: opt -simplifycfg                       -S < %s                    | FileCheck %s --check-prefixes=NOHOIST,DEFAULT
 
 ; This example is produced from a very basic C code:
 ;

From fdc902774e7ae0dae0833216500da047c7c4bec6 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Wed, 29 Jul 2020 17:33:56 +0100
Subject: [PATCH 0492/1035] [DAG][AMDGPU][X86] Add
 SimplifyMultipleUseDemandedBits handling for SIGN/ZERO_EXTEND +
 SIGN/ZERO_EXTEND_VECTOR_INREG

Peek through multiple use ops like we already do for ANY_EXTEND/ANY_EXTEND_VECTOR_INREG

Differential Revision: https://reviews.llvm.org/D84863
---
 .../CodeGen/SelectionDAG/TargetLowering.cpp   |  10 ++
 llvm/test/CodeGen/AMDGPU/bswap.ll             |  36 +++----
 llvm/test/CodeGen/AMDGPU/fshr.ll              |  84 +++++++--------
 llvm/test/CodeGen/AMDGPU/idot8s.ll            |  26 ++---
 llvm/test/CodeGen/AMDGPU/idot8u.ll            |  26 ++---
 llvm/test/CodeGen/AMDGPU/saddsat.ll           |  14 +--
 llvm/test/CodeGen/AMDGPU/ssubsat.ll           |  14 +--
 llvm/test/CodeGen/AMDGPU/uaddsat.ll           |  12 +--
 llvm/test/CodeGen/AMDGPU/usubsat.ll           |  12 +--
 llvm/test/CodeGen/X86/vector-fshl-128.ll      |  84 +++++++--------
 llvm/test/CodeGen/X86/vector-fshr-128.ll      | 100 +++++++++---------
 11 files changed, 214 insertions(+), 204 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index d140b15067a6e..4c7c46218c785 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -1858,6 +1858,11 @@ bool TargetLowering::SimplifyDemandedBits(
     assert(!Known.hasConflict() && "Bits known to be one AND zero?");
     assert(Known.getBitWidth() == InBits && "Src width has changed?");
     Known = Known.zext(BitWidth);
+
+    // Attempt to avoid multi-use ops if we don't need anything from them.
+    if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
+            Src, InDemandedBits, InDemandedElts, TLO.DAG, Depth + 1))
+      return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc));
     break;
   }
   case ISD::SIGN_EXTEND:
@@ -1906,6 +1911,11 @@ bool TargetLowering::SimplifyDemandedBits(
       if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))
         return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));
     }
+
+    // Attempt to avoid multi-use ops if we don't need anything from them.
+    if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
+            Src, InDemandedBits, InDemandedElts, TLO.DAG, Depth + 1))
+      return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc));
     break;
   }
   case ISD::ANY_EXTEND:
diff --git a/llvm/test/CodeGen/AMDGPU/bswap.ll b/llvm/test/CodeGen/AMDGPU/bswap.ll
index 1cdd6f4e37105..884457b319f4b 100644
--- a/llvm/test/CodeGen/AMDGPU/bswap.ll
+++ b/llvm/test/CodeGen/AMDGPU/bswap.ll
@@ -493,11 +493,11 @@ define <3 x i16> @v_bswap_v3i16(<3 x i16> %src) {
 ; SI-NEXT:    v_bfi_b32 v1, s4, v1, v3
 ; SI-NEXT:    v_bfi_b32 v0, s4, v0, v4
 ; SI-NEXT:    v_bfi_b32 v2, s4, v2, v5
-; SI-NEXT:    v_and_b32_e32 v1, 0xffff0000, v1
+; SI-NEXT:    v_and_b32_e32 v3, 0xffff0000, v1
 ; SI-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; SI-NEXT:    v_or_b32_e32 v0, v0, v1
 ; SI-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
-; SI-NEXT:    v_alignbit_b32 v1, v2, v0, 16
+; SI-NEXT:    v_or_b32_e32 v0, v0, v3
+; SI-NEXT:    v_alignbit_b32 v1, v2, v1, 16
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: v_bswap_v3i16:
@@ -515,27 +515,27 @@ define <4 x i16> @v_bswap_v4i16(<4 x i16> %src) {
 ; SI-LABEL: v_bswap_v4i16:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT:    v_alignbit_b32 v4, v3, v3, 8
-; SI-NEXT:    v_alignbit_b32 v3, v3, v3, 24
+; SI-NEXT:    v_alignbit_b32 v4, v1, v1, 8
+; SI-NEXT:    v_alignbit_b32 v1, v1, v1, 24
 ; SI-NEXT:    s_mov_b32 s4, 0xff00ff
 ; SI-NEXT:    s_mov_b32 s5, 0xffff0000
-; SI-NEXT:    v_alignbit_b32 v5, v2, v2, 8
-; SI-NEXT:    v_alignbit_b32 v2, v2, v2, 24
-; SI-NEXT:    v_alignbit_b32 v6, v1, v1, 8
-; SI-NEXT:    v_alignbit_b32 v1, v1, v1, 24
-; SI-NEXT:    v_alignbit_b32 v7, v0, v0, 8
+; SI-NEXT:    v_alignbit_b32 v5, v0, v0, 8
 ; SI-NEXT:    v_alignbit_b32 v0, v0, v0, 24
-; SI-NEXT:    v_bfi_b32 v3, s4, v3, v4
-; SI-NEXT:    v_bfi_b32 v2, s4, v2, v5
-; SI-NEXT:    v_bfi_b32 v1, s4, v1, v6
-; SI-NEXT:    v_bfi_b32 v0, s4, v0, v7
+; SI-NEXT:    v_alignbit_b32 v6, v3, v3, 8
+; SI-NEXT:    v_alignbit_b32 v3, v3, v3, 24
+; SI-NEXT:    v_alignbit_b32 v7, v2, v2, 8
+; SI-NEXT:    v_alignbit_b32 v2, v2, v2, 24
+; SI-NEXT:    v_bfi_b32 v1, s4, v1, v4
+; SI-NEXT:    v_bfi_b32 v0, s4, v0, v5
+; SI-NEXT:    v_bfi_b32 v3, s4, v3, v6
+; SI-NEXT:    v_bfi_b32 v2, s4, v2, v7
+; SI-NEXT:    v_and_b32_e32 v4, s5, v1
+; SI-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
 ; SI-NEXT:    v_and_b32_e32 v3, s5, v3
 ; SI-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
-; SI-NEXT:    v_and_b32_e32 v1, s5, v1
-; SI-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; SI-NEXT:    v_or_b32_e32 v0, v0, v4
 ; SI-NEXT:    v_or_b32_e32 v2, v2, v3
-; SI-NEXT:    v_or_b32_e32 v0, v0, v1
-; SI-NEXT:    v_alignbit_b32 v1, v2, v0, 16
+; SI-NEXT:    v_alignbit_b32 v1, v2, v1, 16
 ; SI-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
diff --git a/llvm/test/CodeGen/AMDGPU/fshr.ll b/llvm/test/CodeGen/AMDGPU/fshr.ll
index 2dadb5e80dfd7..83b3b56172c92 100644
--- a/llvm/test/CodeGen/AMDGPU/fshr.ll
+++ b/llvm/test/CodeGen/AMDGPU/fshr.ll
@@ -767,21 +767,21 @@ define <3 x i16> @v_fshr_v3i16(<3 x i16> %src0, <3 x i16> %src1, <3 x i16> %src2
 ; SI-NEXT:    v_lshl_b32_e32 v0, v0, v7
 ; SI-NEXT:    v_or_b32_e32 v0, v0, v6
 ; SI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
-; SI-NEXT:    v_mov_b32_e32 v9, 0xffff
 ; SI-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; SI-NEXT:    v_and_b32_e32 v3, 15, v8
+; SI-NEXT:    v_sub_i32_e32 v6, vcc, 16, v3
+; SI-NEXT:    v_and_b32_e32 v10, s4, v5
+; SI-NEXT:    v_lshr_b32_e32 v4, v10, v3
+; SI-NEXT:    v_lshl_b32_e32 v2, v2, v6
+; SI-NEXT:    v_mov_b32_e32 v9, 0xffff
+; SI-NEXT:    v_or_b32_e32 v2, v2, v4
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; SI-NEXT:    v_cndmask_b32_e32 v3, v2, v5, vcc
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; SI-NEXT:    v_and_b32_e32 v0, v9, v0
 ; SI-NEXT:    v_or_b32_e32 v0, v0, v1
-; SI-NEXT:    v_and_b32_e32 v1, 15, v8
-; SI-NEXT:    v_sub_i32_e32 v4, vcc, 16, v1
-; SI-NEXT:    v_and_b32_e32 v10, s4, v5
-; SI-NEXT:    v_lshr_b32_e32 v3, v10, v1
-; SI-NEXT:    v_lshl_b32_e32 v2, v2, v4
-; SI-NEXT:    v_or_b32_e32 v2, v2, v3
-; SI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
-; SI-NEXT:    v_cndmask_b32_e32 v1, v2, v5, vcc
-; SI-NEXT:    v_and_b32_e32 v2, v9, v1
-; SI-NEXT:    v_alignbit_b32 v1, v1, v0, 16
+; SI-NEXT:    v_and_b32_e32 v2, v9, v3
+; SI-NEXT:    v_alignbit_b32 v1, v3, v1, 16
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: v_fshr_v3i16:
@@ -865,46 +865,46 @@ define <4 x i16> @v_fshr_v4i16(<4 x i16> %src0, <4 x i16> %src1, <4 x i16> %src2
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-NEXT:    s_mov_b32 s4, 0xffff
-; SI-NEXT:    v_and_b32_e32 v11, 15, v11
-; SI-NEXT:    v_and_b32_e32 v16, s4, v7
-; SI-NEXT:    v_sub_i32_e32 v17, vcc, 16, v11
-; SI-NEXT:    v_lshr_b32_e32 v16, v16, v11
-; SI-NEXT:    v_lshl_b32_e32 v3, v3, v17
-; SI-NEXT:    v_or_b32_e32 v3, v3, v16
-; SI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v11
+; SI-NEXT:    v_and_b32_e32 v9, 15, v9
+; SI-NEXT:    v_and_b32_e32 v16, s4, v5
+; SI-NEXT:    v_sub_i32_e32 v17, vcc, 16, v9
+; SI-NEXT:    v_lshr_b32_e32 v16, v16, v9
+; SI-NEXT:    v_lshl_b32_e32 v1, v1, v17
+; SI-NEXT:    v_or_b32_e32 v1, v1, v16
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v9
+; SI-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
+; SI-NEXT:    v_and_b32_e32 v5, 15, v8
+; SI-NEXT:    v_sub_i32_e32 v9, vcc, 16, v5
+; SI-NEXT:    v_and_b32_e32 v15, s4, v4
+; SI-NEXT:    v_lshr_b32_e32 v8, v15, v5
+; SI-NEXT:    v_lshl_b32_e32 v0, v0, v9
+; SI-NEXT:    v_or_b32_e32 v0, v0, v8
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
+; SI-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; SI-NEXT:    v_and_b32_e32 v4, 15, v11
+; SI-NEXT:    v_sub_i32_e32 v8, vcc, 16, v4
+; SI-NEXT:    v_and_b32_e32 v14, s4, v7
+; SI-NEXT:    v_lshr_b32_e32 v5, v14, v4
+; SI-NEXT:    v_lshl_b32_e32 v3, v3, v8
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
+; SI-NEXT:    v_or_b32_e32 v3, v3, v5
+; SI-NEXT:    v_and_b32_e32 v4, 15, v10
 ; SI-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc
-; SI-NEXT:    v_and_b32_e32 v7, 15, v10
-; SI-NEXT:    v_sub_i32_e32 v11, vcc, 16, v7
-; SI-NEXT:    v_and_b32_e32 v15, s4, v6
-; SI-NEXT:    v_lshr_b32_e32 v10, v15, v7
-; SI-NEXT:    v_lshl_b32_e32 v2, v2, v11
-; SI-NEXT:    v_or_b32_e32 v2, v2, v10
-; SI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v7
+; SI-NEXT:    v_sub_i32_e32 v7, vcc, 16, v4
+; SI-NEXT:    v_and_b32_e32 v13, s4, v6
+; SI-NEXT:    v_lshr_b32_e32 v5, v13, v4
+; SI-NEXT:    v_lshl_b32_e32 v2, v2, v7
+; SI-NEXT:    v_or_b32_e32 v2, v2, v5
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
 ; SI-NEXT:    v_mov_b32_e32 v12, 0xffff
 ; SI-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc
 ; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; SI-NEXT:    v_and_b32_e32 v2, v12, v2
 ; SI-NEXT:    v_or_b32_e32 v2, v2, v3
-; SI-NEXT:    v_and_b32_e32 v3, 15, v9
-; SI-NEXT:    v_sub_i32_e32 v7, vcc, 16, v3
-; SI-NEXT:    v_and_b32_e32 v14, s4, v5
-; SI-NEXT:    v_lshr_b32_e32 v6, v14, v3
-; SI-NEXT:    v_lshl_b32_e32 v1, v1, v7
-; SI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; SI-NEXT:    v_or_b32_e32 v1, v1, v6
-; SI-NEXT:    v_and_b32_e32 v3, 15, v8
-; SI-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
-; SI-NEXT:    v_sub_i32_e32 v6, vcc, 16, v3
-; SI-NEXT:    v_and_b32_e32 v13, s4, v4
-; SI-NEXT:    v_lshr_b32_e32 v5, v13, v3
-; SI-NEXT:    v_lshl_b32_e32 v0, v0, v6
-; SI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; SI-NEXT:    v_or_b32_e32 v0, v0, v5
-; SI-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; SI-NEXT:    v_and_b32_e32 v0, v12, v0
 ; SI-NEXT:    v_or_b32_e32 v0, v0, v1
-; SI-NEXT:    v_alignbit_b32 v1, v2, v0, 16
+; SI-NEXT:    v_alignbit_b32 v1, v2, v1, 16
 ; SI-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
diff --git a/llvm/test/CodeGen/AMDGPU/idot8s.ll b/llvm/test/CodeGen/AMDGPU/idot8s.ll
index 9a88c82b5a85e..e7d48de6ed606 100644
--- a/llvm/test/CodeGen/AMDGPU/idot8s.ll
+++ b/llvm/test/CodeGen/AMDGPU/idot8s.ll
@@ -1470,21 +1470,21 @@ define amdgpu_kernel void @idot8_acc16_vecMul(<8 x i4> addrspace(1)* %src1,
 ; GFX7-NEXT:    s_bfe_i32 s15, s5, 0x40018
 ; GFX7-NEXT:    s_bfe_i32 s16, s5, 0x40014
 ; GFX7-NEXT:    s_bfe_i32 s17, s5, 0x40010
-; GFX7-NEXT:    s_bfe_i32 s18, s5, 0x40000
-; GFX7-NEXT:    s_bfe_i32 s19, s5, 0x40004
-; GFX7-NEXT:    s_bfe_i32 s20, s5, 0x40008
+; GFX7-NEXT:    s_bfe_i32 s18, s5, 0x40008
+; GFX7-NEXT:    s_bfe_i32 s19, s5, 0x4000c
+; GFX7-NEXT:    s_bfe_i32 s20, s5, 0x40000
 ; GFX7-NEXT:    s_ashr_i32 s14, s5, 28
-; GFX7-NEXT:    s_bfe_i32 s5, s5, 0x4000c
+; GFX7-NEXT:    s_bfe_i32 s5, s5, 0x40004
 ; GFX7-NEXT:    s_bfe_i32 s7, s4, 0x40018
 ; GFX7-NEXT:    s_bfe_i32 s9, s4, 0x40014
 ; GFX7-NEXT:    s_bfe_i32 s10, s4, 0x40010
-; GFX7-NEXT:    s_bfe_i32 s11, s4, 0x40000
+; GFX7-NEXT:    s_bfe_i32 s11, s4, 0x40008
 ; GFX7-NEXT:    v_mov_b32_e32 v4, s18
-; GFX7-NEXT:    s_bfe_i32 s12, s4, 0x40004
+; GFX7-NEXT:    s_bfe_i32 s12, s4, 0x4000c
 ; GFX7-NEXT:    v_mov_b32_e32 v3, s19
-; GFX7-NEXT:    s_bfe_i32 s13, s4, 0x40008
+; GFX7-NEXT:    s_bfe_i32 s13, s4, 0x40000
 ; GFX7-NEXT:    v_mov_b32_e32 v2, s20
-; GFX7-NEXT:    s_bfe_i32 s4, s4, 0x4000c
+; GFX7-NEXT:    s_bfe_i32 s4, s4, 0x40004
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s5
 ; GFX7-NEXT:    v_mul_i32_i24_e32 v1, s4, v1
 ; GFX7-NEXT:    v_mul_i32_i24_e32 v2, s13, v2
@@ -1494,17 +1494,17 @@ define amdgpu_kernel void @idot8_acc16_vecMul(<8 x i4> addrspace(1)* %src1,
 ; GFX7-NEXT:    v_and_b32_e32 v2, s8, v2
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; GFX7-NEXT:    v_and_b32_e32 v4, s8, v4
-; GFX7-NEXT:    v_or_b32_e32 v1, v2, v1
-; GFX7-NEXT:    v_or_b32_e32 v2, v4, v3
-; GFX7-NEXT:    v_alignbit_b32 v3, v1, v2, 16
-; GFX7-NEXT:    v_lshrrev_b32_e32 v4, 16, v1
+; GFX7-NEXT:    v_or_b32_e32 v3, v4, v3
+; GFX7-NEXT:    v_or_b32_e32 v2, v2, v1
+; GFX7-NEXT:    v_alignbit_b32 v1, v3, v1, 16
+; GFX7-NEXT:    v_lshrrev_b32_e32 v4, 16, v3
 ; GFX7-NEXT:    v_mov_b32_e32 v5, s17
 ; GFX7-NEXT:    v_mov_b32_e32 v6, s16
 ; GFX7-NEXT:    v_mov_b32_e32 v7, s15
 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v1, v0
 ; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v3, v0
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
 ; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v4, v0
 ; GFX7-NEXT:    v_mad_i32_i24 v0, s10, v5, v0
 ; GFX7-NEXT:    v_mad_i32_i24 v0, s9, v6, v0
diff --git a/llvm/test/CodeGen/AMDGPU/idot8u.ll b/llvm/test/CodeGen/AMDGPU/idot8u.ll
index 171cb6ac6ea77..ef0825120f20d 100644
--- a/llvm/test/CodeGen/AMDGPU/idot8u.ll
+++ b/llvm/test/CodeGen/AMDGPU/idot8u.ll
@@ -1988,17 +1988,17 @@ define amdgpu_kernel void @udot8_acc16_vecMul(<8 x i4> addrspace(1)* %src1,
 ; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
 ; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_bfe_u32 s10, s4, 0x40004
-; GFX7-NEXT:    s_bfe_u32 s17, s5, 0x40004
-; GFX7-NEXT:    s_bfe_u32 s19, s5, 0x4000c
+; GFX7-NEXT:    s_bfe_u32 s10, s4, 0x4000c
+; GFX7-NEXT:    s_bfe_u32 s17, s5, 0x4000c
+; GFX7-NEXT:    s_bfe_u32 s19, s5, 0x40004
 ; GFX7-NEXT:    v_mov_b32_e32 v4, s17
 ; GFX7-NEXT:    s_bfe_u32 s14, s5, 0x40018
 ; GFX7-NEXT:    s_bfe_u32 s15, s5, 0x40014
 ; GFX7-NEXT:    s_bfe_u32 s16, s5, 0x40010
-; GFX7-NEXT:    s_and_b32 s18, s5, 15
+; GFX7-NEXT:    s_bfe_u32 s18, s5, 0x40008
 ; GFX7-NEXT:    s_lshr_b32 s13, s5, 28
-; GFX7-NEXT:    s_bfe_u32 s5, s5, 0x40008
-; GFX7-NEXT:    s_bfe_u32 s12, s4, 0x4000c
+; GFX7-NEXT:    s_and_b32 s5, s5, 15
+; GFX7-NEXT:    s_bfe_u32 s12, s4, 0x40004
 ; GFX7-NEXT:    v_mov_b32_e32 v2, s19
 ; GFX7-NEXT:    v_mul_u32_u24_e32 v2, s12, v2
 ; GFX7-NEXT:    v_mul_u32_u24_e32 v4, s10, v4
@@ -2006,25 +2006,25 @@ define amdgpu_kernel void @udot8_acc16_vecMul(<8 x i4> addrspace(1)* %src1,
 ; GFX7-NEXT:    s_bfe_u32 s7, s4, 0x40018
 ; GFX7-NEXT:    s_bfe_u32 s8, s4, 0x40014
 ; GFX7-NEXT:    s_bfe_u32 s9, s4, 0x40010
-; GFX7-NEXT:    s_and_b32 s11, s4, 15
+; GFX7-NEXT:    s_bfe_u32 s11, s4, 0x40008
 ; GFX7-NEXT:    v_mov_b32_e32 v3, s18
-; GFX7-NEXT:    s_bfe_u32 s4, s4, 0x40008
+; GFX7-NEXT:    s_and_b32 s4, s4, 15
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s5
 ; GFX7-NEXT:    v_mul_u32_u24_e32 v1, s4, v1
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX7-NEXT:    v_mul_u32_u24_e32 v3, s11, v3
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT:    v_or_b32_e32 v3, v3, v4
 ; GFX7-NEXT:    v_or_b32_e32 v1, v1, v2
-; GFX7-NEXT:    v_or_b32_e32 v2, v3, v4
-; GFX7-NEXT:    v_alignbit_b32 v3, v1, v2, 16
-; GFX7-NEXT:    v_lshrrev_b32_e32 v4, 16, v1
+; GFX7-NEXT:    v_alignbit_b32 v2, v3, v2, 16
+; GFX7-NEXT:    v_lshrrev_b32_e32 v4, 16, v3
 ; GFX7-NEXT:    v_mov_b32_e32 v5, s16
 ; GFX7-NEXT:    v_mov_b32_e32 v6, s15
 ; GFX7-NEXT:    v_mov_b32_e32 v7, s14
 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v3, v0
 ; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v2, v0
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v3, v0
 ; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v4, v0
 ; GFX7-NEXT:    v_mad_u32_u24 v0, s9, v5, v0
 ; GFX7-NEXT:    v_mad_u32_u24 v0, s8, v6, v0
diff --git a/llvm/test/CodeGen/AMDGPU/saddsat.ll b/llvm/test/CodeGen/AMDGPU/saddsat.ll
index 73ceea2941868..e78889e349dc8 100644
--- a/llvm/test/CodeGen/AMDGPU/saddsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/saddsat.ll
@@ -170,24 +170,24 @@ define <3 x i16> @v_saddsat_v3i16(<3 x i16> %lhs, <3 x i16> %rhs) {
 ; GFX6-NEXT:    v_bfe_i32 v0, v0, 0, 16
 ; GFX6-NEXT:    v_bfe_i32 v4, v4, 0, 16
 ; GFX6-NEXT:    v_bfe_i32 v1, v1, 0, 16
+; GFX6-NEXT:    v_bfe_i32 v5, v5, 0, 16
+; GFX6-NEXT:    v_bfe_i32 v2, v2, 0, 16
 ; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v1, v4
 ; GFX6-NEXT:    s_movk_i32 s4, 0x7fff
 ; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v3
+; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v5
 ; GFX6-NEXT:    v_min_i32_e32 v1, s4, v1
 ; GFX6-NEXT:    s_movk_i32 s5, 0x8000
 ; GFX6-NEXT:    v_min_i32_e32 v0, s4, v0
 ; GFX6-NEXT:    v_max_i32_e32 v1, s5, v1
 ; GFX6-NEXT:    v_max_i32_e32 v0, s5, v0
-; GFX6-NEXT:    v_bfe_i32 v5, v5, 0, 16
-; GFX6-NEXT:    v_bfe_i32 v2, v2, 0, 16
+; GFX6-NEXT:    v_min_i32_e32 v2, s4, v2
+; GFX6-NEXT:    v_max_i32_e32 v3, s5, v2
 ; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX6-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v2, v5
-; GFX6-NEXT:    v_min_i32_e32 v1, s4, v1
-; GFX6-NEXT:    v_max_i32_e32 v1, s5, v1
-; GFX6-NEXT:    v_or_b32_e32 v2, 0xffff0000, v1
-; GFX6-NEXT:    v_alignbit_b32 v1, v1, v0, 16
+; GFX6-NEXT:    v_or_b32_e32 v2, 0xffff0000, v3
+; GFX6-NEXT:    v_alignbit_b32 v1, v3, v1, 16
 ; GFX6-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_saddsat_v3i16:
diff --git a/llvm/test/CodeGen/AMDGPU/ssubsat.ll b/llvm/test/CodeGen/AMDGPU/ssubsat.ll
index f6048f6a357c4..138488ece9979 100644
--- a/llvm/test/CodeGen/AMDGPU/ssubsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/ssubsat.ll
@@ -170,25 +170,25 @@ define <3 x i16> @v_ssubsat_v3i16(<3 x i16> %lhs, <3 x i16> %rhs) {
 ; GFX6-NEXT:    v_bfe_i32 v0, v0, 0, 16
 ; GFX6-NEXT:    v_bfe_i32 v4, v4, 0, 16
 ; GFX6-NEXT:    v_bfe_i32 v1, v1, 0, 16
+; GFX6-NEXT:    v_bfe_i32 v5, v5, 0, 16
+; GFX6-NEXT:    v_bfe_i32 v2, v2, 0, 16
 ; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, v1, v4
 ; GFX6-NEXT:    s_movk_i32 s4, 0x7fff
 ; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, v0, v3
+; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, v2, v5
 ; GFX6-NEXT:    v_min_i32_e32 v1, s4, v1
 ; GFX6-NEXT:    s_movk_i32 s5, 0x8000
 ; GFX6-NEXT:    v_min_i32_e32 v0, s4, v0
 ; GFX6-NEXT:    v_max_i32_e32 v1, s5, v1
+; GFX6-NEXT:    v_min_i32_e32 v2, s4, v2
 ; GFX6-NEXT:    v_max_i32_e32 v0, s5, v0
 ; GFX6-NEXT:    s_mov_b32 s6, 0xffff
-; GFX6-NEXT:    v_bfe_i32 v5, v5, 0, 16
-; GFX6-NEXT:    v_bfe_i32 v2, v2, 0, 16
+; GFX6-NEXT:    v_max_i32_e32 v3, s5, v2
 ; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX6-NEXT:    v_and_b32_e32 v0, s6, v0
 ; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, v2, v5
-; GFX6-NEXT:    v_min_i32_e32 v1, s4, v1
-; GFX6-NEXT:    v_max_i32_e32 v1, s5, v1
-; GFX6-NEXT:    v_and_b32_e32 v2, s6, v1
-; GFX6-NEXT:    v_alignbit_b32 v1, v1, v0, 16
+; GFX6-NEXT:    v_and_b32_e32 v2, s6, v3
+; GFX6-NEXT:    v_alignbit_b32 v1, v3, v1, 16
 ; GFX6-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_ssubsat_v3i16:
diff --git a/llvm/test/CodeGen/AMDGPU/uaddsat.ll b/llvm/test/CodeGen/AMDGPU/uaddsat.ll
index 48349cbed903c..56e4123e182a0 100644
--- a/llvm/test/CodeGen/AMDGPU/uaddsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/uaddsat.ll
@@ -128,19 +128,19 @@ define <3 x i16> @v_uaddsat_v3i16(<3 x i16> %lhs, <3 x i16> %rhs) {
 ; GFX6-NEXT:    v_and_b32_e32 v4, s4, v4
 ; GFX6-NEXT:    v_and_b32_e32 v1, s4, v1
 ; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v1, v4
+; GFX6-NEXT:    v_and_b32_e32 v5, s4, v5
+; GFX6-NEXT:    v_and_b32_e32 v2, s4, v2
 ; GFX6-NEXT:    v_and_b32_e32 v3, s4, v3
 ; GFX6-NEXT:    v_and_b32_e32 v0, s4, v0
 ; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v3
 ; GFX6-NEXT:    v_min_u32_e32 v1, s4, v1
-; GFX6-NEXT:    v_and_b32_e32 v5, s4, v5
-; GFX6-NEXT:    v_and_b32_e32 v2, s4, v2
+; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v5
+; GFX6-NEXT:    v_min_u32_e32 v3, s4, v2
 ; GFX6-NEXT:    v_min_u32_e32 v0, s4, v0
 ; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v2, v5
-; GFX6-NEXT:    v_min_u32_e32 v1, s4, v1
-; GFX6-NEXT:    v_or_b32_e32 v2, 0xffff0000, v1
-; GFX6-NEXT:    v_alignbit_b32 v1, v1, v0, 16
+; GFX6-NEXT:    v_or_b32_e32 v2, 0xffff0000, v3
+; GFX6-NEXT:    v_alignbit_b32 v1, v3, v1, 16
 ; GFX6-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_uaddsat_v3i16:
diff --git a/llvm/test/CodeGen/AMDGPU/usubsat.ll b/llvm/test/CodeGen/AMDGPU/usubsat.ll
index 55275450b1b4b..97bfdc479e090 100644
--- a/llvm/test/CodeGen/AMDGPU/usubsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/usubsat.ll
@@ -131,17 +131,17 @@ define <3 x i16> @v_usubsat_v3i16(<3 x i16> %lhs, <3 x i16> %rhs) {
 ; GFX6-NEXT:    v_and_b32_e32 v1, s4, v1
 ; GFX6-NEXT:    v_max_u32_e32 v1, v1, v8
 ; GFX6-NEXT:    v_max_u32_e32 v0, v0, v7
-; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, v1, v4
-; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, v0, v3
 ; GFX6-NEXT:    v_and_b32_e32 v6, s4, v5
 ; GFX6-NEXT:    v_and_b32_e32 v2, s4, v2
+; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, v1, v4
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, v0, v3
+; GFX6-NEXT:    v_max_u32_e32 v2, v2, v6
+; GFX6-NEXT:    v_sub_i32_e32 v3, vcc, v2, v5
 ; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX6-NEXT:    v_and_b32_e32 v0, s4, v0
 ; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX6-NEXT:    v_max_u32_e32 v1, v2, v6
-; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, v1, v5
-; GFX6-NEXT:    v_and_b32_e32 v2, s4, v1
-; GFX6-NEXT:    v_alignbit_b32 v1, v1, v0, 16
+; GFX6-NEXT:    v_and_b32_e32 v2, s4, v3
+; GFX6-NEXT:    v_alignbit_b32 v1, v3, v1, 16
 ; GFX6-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_usubsat_v3i16:
diff --git a/llvm/test/CodeGen/X86/vector-fshl-128.ll b/llvm/test/CodeGen/X86/vector-fshl-128.ll
index 00c06564d2258..b8755a23e9760 100644
--- a/llvm/test/CodeGen/X86/vector-fshl-128.ll
+++ b/llvm/test/CodeGen/X86/vector-fshl-128.ll
@@ -1947,18 +1947,18 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %
 ;
 ; AVX512F-LABEL: splatvar_funnnel_v16i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512F-NEXT:    vpsubb %xmm2, %xmm3, %xmm3
-; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512F-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
-; AVX512F-NEXT:    vpsrld %xmm3, %zmm1, %zmm1
 ; AVX512F-NEXT:    vpmovzxbd {{.*#+}} zmm3 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
-; AVX512F-NEXT:    vpbroadcastb %xmm2, %xmm2
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
 ; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512F-NEXT:    vpslld %xmm4, %zmm3, %zmm3
+; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512F-NEXT:    vpsubb %xmm2, %xmm4, %xmm4
+; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512F-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; AVX512F-NEXT:    vpsrld %xmm4, %zmm1, %zmm1
 ; AVX512F-NEXT:    vpord %zmm1, %zmm3, %zmm1
 ; AVX512F-NEXT:    vpmovdb %zmm1, %xmm1
+; AVX512F-NEXT:    vpbroadcastb %xmm2, %xmm2
 ; AVX512F-NEXT:    vpxor %xmm3, %xmm3, %xmm3
 ; AVX512F-NEXT:    vpcmpeqb %xmm3, %xmm2, %xmm2
 ; AVX512F-NEXT:    vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
@@ -1967,18 +1967,18 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %
 ;
 ; AVX512VL-LABEL: splatvar_funnnel_v16i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512VL-NEXT:    vpsubb %xmm2, %xmm3, %xmm3
-; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512VL-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
-; AVX512VL-NEXT:    vpsrld %xmm3, %zmm1, %zmm1
 ; AVX512VL-NEXT:    vpmovzxbd {{.*#+}} zmm3 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
-; AVX512VL-NEXT:    vpbroadcastb %xmm2, %xmm2
+; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
 ; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512VL-NEXT:    vpslld %xmm4, %zmm3, %zmm3
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VL-NEXT:    vpsubb %xmm2, %xmm4, %xmm4
+; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VL-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; AVX512VL-NEXT:    vpsrld %xmm4, %zmm1, %zmm1
 ; AVX512VL-NEXT:    vpord %zmm1, %zmm3, %zmm1
 ; AVX512VL-NEXT:    vpmovdb %zmm1, %xmm1
+; AVX512VL-NEXT:    vpbroadcastb %xmm2, %xmm2
 ; AVX512VL-NEXT:    vpxor %xmm3, %xmm3, %xmm3
 ; AVX512VL-NEXT:    vpcmpeqb %xmm3, %xmm2, %xmm2
 ; AVX512VL-NEXT:    vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
@@ -1988,18 +1988,18 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %
 ; AVX512BW-LABEL: splatvar_funnnel_v16i8:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; AVX512BW-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512BW-NEXT:    vpsubb %xmm2, %xmm3, %xmm3
-; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512BW-NEXT:    vpsrlw %xmm3, %ymm1, %ymm1
 ; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
-; AVX512BW-NEXT:    vpbroadcastb %xmm2, %xmm2
+; AVX512BW-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
 ; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512BW-NEXT:    vpsllw %xmm4, %ymm3, %ymm3
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512BW-NEXT:    vpsubb %xmm2, %xmm4, %xmm4
+; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512BW-NEXT:    vpsrlw %xmm4, %ymm1, %ymm1
 ; AVX512BW-NEXT:    vpor %ymm1, %ymm3, %ymm1
 ; AVX512BW-NEXT:    vpmovwb %zmm1, %ymm1
+; AVX512BW-NEXT:    vpbroadcastb %xmm2, %xmm2
 ; AVX512BW-NEXT:    vptestnmb %zmm2, %zmm2, %k1
 ; AVX512BW-NEXT:    vmovdqu8 %zmm0, %zmm1 {%k1}
 ; AVX512BW-NEXT:    vmovdqa %xmm1, %xmm0
@@ -2009,18 +2009,18 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %
 ; AVX512VBMI2-LABEL: splatvar_funnnel_v16i8:
 ; AVX512VBMI2:       # %bb.0:
 ; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; AVX512VBMI2-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
-; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512VBMI2-NEXT:    vpsubb %xmm2, %xmm3, %xmm3
-; AVX512VBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512VBMI2-NEXT:    vpsrlw %xmm3, %ymm1, %ymm1
 ; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
-; AVX512VBMI2-NEXT:    vpbroadcastb %xmm2, %xmm2
+; AVX512VBMI2-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
 ; AVX512VBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512VBMI2-NEXT:    vpsllw %xmm4, %ymm3, %ymm3
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VBMI2-NEXT:    vpsubb %xmm2, %xmm4, %xmm4
+; AVX512VBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512VBMI2-NEXT:    vpsrlw %xmm4, %ymm1, %ymm1
 ; AVX512VBMI2-NEXT:    vpor %ymm1, %ymm3, %ymm1
 ; AVX512VBMI2-NEXT:    vpmovwb %zmm1, %ymm1
+; AVX512VBMI2-NEXT:    vpbroadcastb %xmm2, %xmm2
 ; AVX512VBMI2-NEXT:    vptestnmb %zmm2, %zmm2, %k1
 ; AVX512VBMI2-NEXT:    vmovdqu8 %zmm0, %zmm1 {%k1}
 ; AVX512VBMI2-NEXT:    vmovdqa %xmm1, %xmm0
@@ -2029,18 +2029,18 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %
 ;
 ; AVX512VLBW-LABEL: splatvar_funnnel_v16i8:
 ; AVX512VLBW:       # %bb.0:
-; AVX512VLBW-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
-; AVX512VLBW-NEXT:    vmovdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512VLBW-NEXT:    vpsubb %xmm2, %xmm3, %xmm3
-; AVX512VLBW-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512VLBW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512VLBW-NEXT:    vpsrlw %xmm3, %ymm1, %ymm1
 ; AVX512VLBW-NEXT:    vpmovzxbw {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
-; AVX512VLBW-NEXT:    vpbroadcastb %xmm2, %xmm2
+; AVX512VLBW-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
 ; AVX512VLBW-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512VLBW-NEXT:    vpsllw %xmm4, %ymm3, %ymm3
+; AVX512VLBW-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VLBW-NEXT:    vpsubb %xmm2, %xmm4, %xmm4
+; AVX512VLBW-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VLBW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512VLBW-NEXT:    vpsrlw %xmm4, %ymm1, %ymm1
 ; AVX512VLBW-NEXT:    vpor %ymm1, %ymm3, %ymm1
 ; AVX512VLBW-NEXT:    vpmovwb %ymm1, %xmm1
+; AVX512VLBW-NEXT:    vpbroadcastb %xmm2, %xmm2
 ; AVX512VLBW-NEXT:    vptestnmb %xmm2, %xmm2, %k1
 ; AVX512VLBW-NEXT:    vmovdqu8 %xmm0, %xmm1 {%k1}
 ; AVX512VLBW-NEXT:    vmovdqa %xmm1, %xmm0
@@ -2049,18 +2049,18 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %
 ;
 ; AVX512VLVBMI2-LABEL: splatvar_funnnel_v16i8:
 ; AVX512VLVBMI2:       # %bb.0:
-; AVX512VLVBMI2-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
-; AVX512VLVBMI2-NEXT:    vmovdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512VLVBMI2-NEXT:    vpsubb %xmm2, %xmm3, %xmm3
-; AVX512VLVBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512VLVBMI2-NEXT:    vpsrlw %xmm3, %ymm1, %ymm1
 ; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
-; AVX512VLVBMI2-NEXT:    vpbroadcastb %xmm2, %xmm2
+; AVX512VLVBMI2-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
 ; AVX512VLVBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512VLVBMI2-NEXT:    vpsllw %xmm4, %ymm3, %ymm3
+; AVX512VLVBMI2-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VLVBMI2-NEXT:    vpsubb %xmm2, %xmm4, %xmm4
+; AVX512VLVBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512VLVBMI2-NEXT:    vpsrlw %xmm4, %ymm1, %ymm1
 ; AVX512VLVBMI2-NEXT:    vpor %ymm1, %ymm3, %ymm1
 ; AVX512VLVBMI2-NEXT:    vpmovwb %ymm1, %xmm1
+; AVX512VLVBMI2-NEXT:    vpbroadcastb %xmm2, %xmm2
 ; AVX512VLVBMI2-NEXT:    vptestnmb %xmm2, %xmm2, %k1
 ; AVX512VLVBMI2-NEXT:    vmovdqu8 %xmm0, %xmm1 {%k1}
 ; AVX512VLVBMI2-NEXT:    vmovdqa %xmm1, %xmm0
diff --git a/llvm/test/CodeGen/X86/vector-fshr-128.ll b/llvm/test/CodeGen/X86/vector-fshr-128.ll
index bad82a4247979..43b8a49d1776a 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-128.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-128.ll
@@ -1967,18 +1967,18 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpmovzxbd {{.*#+}} zmm3 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
 ; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
-; AVX512F-NEXT:    vpbroadcastb %xmm2, %xmm4
-; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm5 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512F-NEXT:    vpsrld %xmm5, %zmm3, %zmm3
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm5 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512F-NEXT:    vpsubb %xmm2, %xmm5, %xmm2
-; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512F-NEXT:    vpsrld %xmm4, %zmm3, %zmm3
+; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512F-NEXT:    vpsubb %xmm2, %xmm4, %xmm4
+; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512F-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
-; AVX512F-NEXT:    vpslld %xmm2, %zmm0, %zmm0
+; AVX512F-NEXT:    vpslld %xmm4, %zmm0, %zmm0
 ; AVX512F-NEXT:    vpord %zmm3, %zmm0, %zmm0
 ; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm4, %xmm2
+; AVX512F-NEXT:    vpbroadcastb %xmm2, %xmm2
+; AVX512F-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX512F-NEXT:    vpcmpeqb %xmm3, %xmm2, %xmm2
 ; AVX512F-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
@@ -1987,18 +1987,18 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpmovzxbd {{.*#+}} zmm3 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
 ; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
-; AVX512VL-NEXT:    vpbroadcastb %xmm2, %xmm4
-; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm5 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512VL-NEXT:    vpsrld %xmm5, %zmm3, %zmm3
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm5 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512VL-NEXT:    vpsubb %xmm2, %xmm5, %xmm2
-; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VL-NEXT:    vpsrld %xmm4, %zmm3, %zmm3
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VL-NEXT:    vpsubb %xmm2, %xmm4, %xmm4
+; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512VL-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
-; AVX512VL-NEXT:    vpslld %xmm2, %zmm0, %zmm0
+; AVX512VL-NEXT:    vpslld %xmm4, %zmm0, %zmm0
 ; AVX512VL-NEXT:    vpord %zmm3, %zmm0, %zmm0
 ; AVX512VL-NEXT:    vpmovdb %zmm0, %xmm0
-; AVX512VL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VL-NEXT:    vpcmpeqb %xmm2, %xmm4, %xmm2
+; AVX512VL-NEXT:    vpbroadcastb %xmm2, %xmm2
+; AVX512VL-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX512VL-NEXT:    vpcmpeqb %xmm3, %xmm2, %xmm2
 ; AVX512VL-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
@@ -2008,17 +2008,17 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %
 ; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm3 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
 ; AVX512BW-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
-; AVX512BW-NEXT:    vpbroadcastb %xmm2, %xmm4
-; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm5 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512BW-NEXT:    vpsrlw %xmm5, %ymm3, %ymm3
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm5 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512BW-NEXT:    vpsubb %xmm2, %xmm5, %xmm2
-; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512BW-NEXT:    vpsrlw %xmm4, %ymm3, %ymm3
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512BW-NEXT:    vpsubb %xmm2, %xmm4, %xmm4
+; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
-; AVX512BW-NEXT:    vpsllw %xmm2, %ymm0, %ymm0
+; AVX512BW-NEXT:    vpsllw %xmm4, %ymm0, %ymm0
 ; AVX512BW-NEXT:    vpor %ymm3, %ymm0, %ymm0
 ; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
-; AVX512BW-NEXT:    vptestnmb %zmm4, %zmm4, %k1
+; AVX512BW-NEXT:    vpbroadcastb %xmm2, %xmm2
+; AVX512BW-NEXT:    vptestnmb %zmm2, %zmm2, %k1
 ; AVX512BW-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k1}
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512BW-NEXT:    vzeroupper
@@ -2029,17 +2029,17 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %
 ; AVX512VBMI2-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm3 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
 ; AVX512VBMI2-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
-; AVX512VBMI2-NEXT:    vpbroadcastb %xmm2, %xmm4
-; AVX512VBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm5 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512VBMI2-NEXT:    vpsrlw %xmm5, %ymm3, %ymm3
-; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm5 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512VBMI2-NEXT:    vpsubb %xmm2, %xmm5, %xmm2
-; AVX512VBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VBMI2-NEXT:    vpsrlw %xmm4, %ymm3, %ymm3
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VBMI2-NEXT:    vpsubb %xmm2, %xmm4, %xmm4
+; AVX512VBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
-; AVX512VBMI2-NEXT:    vpsllw %xmm2, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    vpsllw %xmm4, %ymm0, %ymm0
 ; AVX512VBMI2-NEXT:    vpor %ymm3, %ymm0, %ymm0
 ; AVX512VBMI2-NEXT:    vpmovwb %zmm0, %ymm0
-; AVX512VBMI2-NEXT:    vptestnmb %zmm4, %zmm4, %k1
+; AVX512VBMI2-NEXT:    vpbroadcastb %xmm2, %xmm2
+; AVX512VBMI2-NEXT:    vptestnmb %zmm2, %zmm2, %k1
 ; AVX512VBMI2-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k1}
 ; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512VBMI2-NEXT:    vzeroupper
@@ -2049,17 +2049,17 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpmovzxbw {{.*#+}} ymm3 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
 ; AVX512VLBW-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
-; AVX512VLBW-NEXT:    vpbroadcastb %xmm2, %xmm4
-; AVX512VLBW-NEXT:    vpmovzxbq {{.*#+}} xmm5 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512VLBW-NEXT:    vpsrlw %xmm5, %ymm3, %ymm3
-; AVX512VLBW-NEXT:    vmovdqa {{.*#+}} xmm5 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512VLBW-NEXT:    vpsubb %xmm2, %xmm5, %xmm2
-; AVX512VLBW-NEXT:    vpmovzxbq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VLBW-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VLBW-NEXT:    vpsrlw %xmm4, %ymm3, %ymm3
+; AVX512VLBW-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VLBW-NEXT:    vpsubb %xmm2, %xmm4, %xmm4
+; AVX512VLBW-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512VLBW-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
-; AVX512VLBW-NEXT:    vpsllw %xmm2, %ymm0, %ymm0
+; AVX512VLBW-NEXT:    vpsllw %xmm4, %ymm0, %ymm0
 ; AVX512VLBW-NEXT:    vpor %ymm3, %ymm0, %ymm0
 ; AVX512VLBW-NEXT:    vpmovwb %ymm0, %xmm0
-; AVX512VLBW-NEXT:    vptestnmb %xmm4, %xmm4, %k1
+; AVX512VLBW-NEXT:    vpbroadcastb %xmm2, %xmm2
+; AVX512VLBW-NEXT:    vptestnmb %xmm2, %xmm2, %k1
 ; AVX512VLBW-NEXT:    vmovdqu8 %xmm1, %xmm0 {%k1}
 ; AVX512VLBW-NEXT:    vzeroupper
 ; AVX512VLBW-NEXT:    retq
@@ -2068,17 +2068,17 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %
 ; AVX512VLVBMI2:       # %bb.0:
 ; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm3 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
 ; AVX512VLVBMI2-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
-; AVX512VLVBMI2-NEXT:    vpbroadcastb %xmm2, %xmm4
-; AVX512VLVBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm5 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512VLVBMI2-NEXT:    vpsrlw %xmm5, %ymm3, %ymm3
-; AVX512VLVBMI2-NEXT:    vmovdqa {{.*#+}} xmm5 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512VLVBMI2-NEXT:    vpsubb %xmm2, %xmm5, %xmm2
-; AVX512VLVBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VLVBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VLVBMI2-NEXT:    vpsrlw %xmm4, %ymm3, %ymm3
+; AVX512VLVBMI2-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VLVBMI2-NEXT:    vpsubb %xmm2, %xmm4, %xmm4
+; AVX512VLVBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
-; AVX512VLVBMI2-NEXT:    vpsllw %xmm2, %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    vpsllw %xmm4, %ymm0, %ymm0
 ; AVX512VLVBMI2-NEXT:    vpor %ymm3, %ymm0, %ymm0
 ; AVX512VLVBMI2-NEXT:    vpmovwb %ymm0, %xmm0
-; AVX512VLVBMI2-NEXT:    vptestnmb %xmm4, %xmm4, %k1
+; AVX512VLVBMI2-NEXT:    vpbroadcastb %xmm2, %xmm2
+; AVX512VLVBMI2-NEXT:    vptestnmb %xmm2, %xmm2, %k1
 ; AVX512VLVBMI2-NEXT:    vmovdqu8 %xmm1, %xmm0 {%k1}
 ; AVX512VLVBMI2-NEXT:    vzeroupper
 ; AVX512VLVBMI2-NEXT:    retq

From ee9617e96b05de04b2f189596b86c6149c807764 Mon Sep 17 00:00:00 2001
From: Sanjay Patel <spatel@rotateright.com>
Date: Wed, 29 Jul 2020 13:12:51 -0400
Subject: [PATCH 0493/1035] [InstSimplify] try constant folding intrinsics
 before general simplifications

This matches the behavior of simplify calls for regular opcodes -
rely on ConstantFolding before spending time on folds with variables.

I am not aware of any diffs from this re-ordering currently, but there was
potential for unintended behavior from the min/max intrinsics because that
code is implicitly assuming that only 1 of the input operands is constant.
---
 llvm/lib/Analysis/InstructionSimplify.cpp | 48 ++++++++++++-----------
 1 file changed, 26 insertions(+), 22 deletions(-)

diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp
index af487f99e8e24..de4e23aac9826 100644
--- a/llvm/lib/Analysis/InstructionSimplify.cpp
+++ b/llvm/lib/Analysis/InstructionSimplify.cpp
@@ -5484,28 +5484,9 @@ static Value *simplifyIntrinsic(CallBase *Call, const SimplifyQuery &Q) {
   }
 }
 
-Value *llvm::SimplifyCall(CallBase *Call, const SimplifyQuery &Q) {
-  Value *Callee = Call->getCalledOperand();
-
-  // musttail calls can only be simplified if they are also DCEd.
-  // As we can't guarantee this here, don't simplify them.
-  if (Call->isMustTailCall())
-    return nullptr;
-
-  // call undef -> undef
-  // call null -> undef
-  if (isa<UndefValue>(Callee) || isa<ConstantPointerNull>(Callee))
-    return UndefValue::get(Call->getType());
-
-  Function *F = dyn_cast<Function>(Callee);
-  if (!F)
-    return nullptr;
-
-  if (F->isIntrinsic())
-    if (Value *Ret = simplifyIntrinsic(Call, Q))
-      return Ret;
-
-  if (!canConstantFoldCallTo(Call, F))
+static Value *tryConstantFoldCall(CallBase *Call, const SimplifyQuery &Q) {
+  auto *F = dyn_cast<Function>(Call->getCalledOperand());
+  if (!F || !canConstantFoldCallTo(Call, F))
     return nullptr;
 
   SmallVector<Constant *, 4> ConstantArgs;
@@ -5524,6 +5505,29 @@ Value *llvm::SimplifyCall(CallBase *Call, const SimplifyQuery &Q) {
   return ConstantFoldCall(Call, F, ConstantArgs, Q.TLI);
 }
 
+Value *llvm::SimplifyCall(CallBase *Call, const SimplifyQuery &Q) {
+  // musttail calls can only be simplified if they are also DCEd.
+  // As we can't guarantee this here, don't simplify them.
+  if (Call->isMustTailCall())
+    return nullptr;
+
+  // call undef -> undef
+  // call null -> undef
+  Value *Callee = Call->getCalledOperand();
+  if (isa<UndefValue>(Callee) || isa<ConstantPointerNull>(Callee))
+    return UndefValue::get(Call->getType());
+
+  if (Value *V = tryConstantFoldCall(Call, Q))
+    return V;
+
+  auto *F = dyn_cast<Function>(Callee);
+  if (F && F->isIntrinsic())
+    if (Value *Ret = simplifyIntrinsic(Call, Q))
+      return Ret;
+
+  return nullptr;
+}
+
 /// Given operands for a Freeze, see if we can fold the result.
 static Value *SimplifyFreezeInst(Value *Op0, const SimplifyQuery &Q) {
   // Use a utility function defined in ValueTracking.

From 71d0a2b8a31344ab29d1afd0c54d89873fb3cc9e Mon Sep 17 00:00:00 2001
From: Arthur Eubanks <aeubanks@google.com>
Date: Mon, 27 Jul 2020 14:19:12 -0700
Subject: [PATCH 0494/1035] [DFSan][NewPM] Port DataFlowSanitizer to NewPM

Reviewed By: ychen, morehouse

Differential Revision: https://reviews.llvm.org/D84707
---
 clang/lib/CodeGen/BackendUtil.cpp             |  3 +-
 .../go/llvm/InstrumentationBindings.cpp       |  2 +-
 llvm/include/llvm/InitializePasses.h          |  2 +-
 .../include/llvm/Transforms/Instrumentation.h |  2 +-
 .../Instrumentation/DataFlowSanitizer.h       | 31 +++++++++
 llvm/lib/Passes/PassBuilder.cpp               |  1 +
 llvm/lib/Passes/PassRegistry.def              |  1 +
 .../Instrumentation/DataFlowSanitizer.cpp     | 67 +++++++++++++------
 .../Instrumentation/Instrumentation.cpp       |  2 +-
 .../Instrumentation/DataFlowSanitizer/call.ll |  1 +
 10 files changed, 86 insertions(+), 26 deletions(-)
 create mode 100644 llvm/include/llvm/Transforms/Instrumentation/DataFlowSanitizer.h

diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp
index 111ca1981fe3a..f0058690e43dc 100644
--- a/clang/lib/CodeGen/BackendUtil.cpp
+++ b/clang/lib/CodeGen/BackendUtil.cpp
@@ -350,7 +350,8 @@ static void addDataFlowSanitizerPass(const PassManagerBuilder &Builder,
   const PassManagerBuilderWrapper &BuilderWrapper =
       static_cast<const PassManagerBuilderWrapper&>(Builder);
   const LangOptions &LangOpts = BuilderWrapper.getLangOpts();
-  PM.add(createDataFlowSanitizerPass(LangOpts.SanitizerBlacklistFiles));
+  PM.add(
+      createDataFlowSanitizerLegacyPassPass(LangOpts.SanitizerBlacklistFiles));
 }
 
 static TargetLibraryInfoImpl *createTLII(llvm::Triple &TargetTriple,
diff --git a/llvm/bindings/go/llvm/InstrumentationBindings.cpp b/llvm/bindings/go/llvm/InstrumentationBindings.cpp
index 794657fbce381..71d43de254a37 100644
--- a/llvm/bindings/go/llvm/InstrumentationBindings.cpp
+++ b/llvm/bindings/go/llvm/InstrumentationBindings.cpp
@@ -44,5 +44,5 @@ void LLVMAddDataFlowSanitizerPass(LLVMPassManagerRef PM,
   for (int i = 0; i != ABIListFilesNum; ++i) {
     ABIListFilesVec.push_back(ABIListFiles[i]);
   }
-  unwrap(PM)->add(createDataFlowSanitizerPass(ABIListFilesVec));
+  unwrap(PM)->add(createDataFlowSanitizerLegacyPassPass(ABIListFilesVec));
 }
diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h
index 7a85a961aac6d..cce6a43504021 100644
--- a/llvm/include/llvm/InitializePasses.h
+++ b/llvm/include/llvm/InitializePasses.h
@@ -122,7 +122,7 @@ void initializeDAEPass(PassRegistry&);
 void initializeDAHPass(PassRegistry&);
 void initializeDCELegacyPassPass(PassRegistry&);
 void initializeDSELegacyPassPass(PassRegistry&);
-void initializeDataFlowSanitizerPass(PassRegistry&);
+void initializeDataFlowSanitizerLegacyPassPass(PassRegistry &);
 void initializeDeadInstEliminationPass(PassRegistry&);
 void initializeDeadMachineInstructionElimPass(PassRegistry&);
 void initializeDebugifyMachineModulePass(PassRegistry &);
diff --git a/llvm/include/llvm/Transforms/Instrumentation.h b/llvm/include/llvm/Transforms/Instrumentation.h
index f2e084041dd67..0453cb428bc07 100644
--- a/llvm/include/llvm/Transforms/Instrumentation.h
+++ b/llvm/include/llvm/Transforms/Instrumentation.h
@@ -143,7 +143,7 @@ ModulePass *createInstrProfilingLegacyPass(
 ModulePass *createInstrOrderFilePass();
 
 // Insert DataFlowSanitizer (dynamic data flow analysis) instrumentation
-ModulePass *createDataFlowSanitizerPass(
+ModulePass *createDataFlowSanitizerLegacyPassPass(
     const std::vector<std::string> &ABIListFiles = std::vector<std::string>());
 
 // Options for sanitizer coverage instrumentation.
diff --git a/llvm/include/llvm/Transforms/Instrumentation/DataFlowSanitizer.h b/llvm/include/llvm/Transforms/Instrumentation/DataFlowSanitizer.h
new file mode 100644
index 0000000000000..c43de78df592e
--- /dev/null
+++ b/llvm/include/llvm/Transforms/Instrumentation/DataFlowSanitizer.h
@@ -0,0 +1,31 @@
+//===- DataFlowSanitizer.h - dynamic data flow analysis -------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_TRANSFORMS_INSTRUMENTATION_DATAFLOWSANITIZER_H
+#define LLVM_TRANSFORMS_INSTRUMENTATION_DATAFLOWSANITIZER_H
+
+#include "llvm/IR/Module.h"
+#include "llvm/IR/PassManager.h"
+#include <string>
+#include <vector>
+
+namespace llvm {
+
+class DataFlowSanitizerPass : public PassInfoMixin<DataFlowSanitizerPass> {
+private:
+  std::vector<std::string> ABIListFiles;
+
+public:
+  DataFlowSanitizerPass(
+      const std::vector<std::string> &ABIListFiles = std::vector<std::string>())
+      : ABIListFiles(ABIListFiles) {}
+  PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+};
+
+} // namespace llvm
+
+#endif
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index e306833d1d30d..eb9041311e1bf 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -106,6 +106,7 @@
 #include "llvm/Transforms/Instrumentation/BoundsChecking.h"
 #include "llvm/Transforms/Instrumentation/CGProfile.h"
 #include "llvm/Transforms/Instrumentation/ControlHeightReduction.h"
+#include "llvm/Transforms/Instrumentation/DataFlowSanitizer.h"
 #include "llvm/Transforms/Instrumentation/GCOVProfiler.h"
 #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
 #include "llvm/Transforms/Instrumentation/InstrOrderFile.h"
diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def
index 11154686fa77a..38a6575b831a5 100644
--- a/llvm/lib/Passes/PassRegistry.def
+++ b/llvm/lib/Passes/PassRegistry.def
@@ -89,6 +89,7 @@ MODULE_PASS("strip-dead-prototypes", StripDeadPrototypesPass())
 MODULE_PASS("synthetic-counts-propagation", SyntheticCountsPropagation())
 MODULE_PASS("wholeprogramdevirt", WholeProgramDevirtPass(nullptr, nullptr))
 MODULE_PASS("verify", VerifierPass())
+MODULE_PASS("dfsan", DataFlowSanitizerPass())
 MODULE_PASS("asan-module", ModuleAddressSanitizerPass(/*CompileKernel=*/false, false, true, false))
 MODULE_PASS("msan-module", MemorySanitizerPass({}))
 MODULE_PASS("tsan-module", ThreadSanitizerPass())
diff --git a/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
index a5a785cb55f78..317b0904876ae 100644
--- a/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
@@ -46,6 +46,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "llvm/Transforms/Instrumentation/DataFlowSanitizer.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/DepthFirstIterator.h"
@@ -78,6 +79,7 @@
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/MDBuilder.h"
 #include "llvm/IR/Module.h"
+#include "llvm/IR/PassManager.h"
 #include "llvm/IR/Type.h"
 #include "llvm/IR/User.h"
 #include "llvm/IR/Value.h"
@@ -292,7 +294,7 @@ AttributeList TransformFunctionAttributes(
       llvm::makeArrayRef(ArgumentAttributes));
 }
 
-class DataFlowSanitizer : public ModulePass {
+class DataFlowSanitizer {
   friend struct DFSanFunction;
   friend class DFSanVisitor;
 
@@ -390,14 +392,12 @@ class DataFlowSanitizer : public ModulePass {
   void initializeCallbackFunctions(Module &M);
   void initializeRuntimeFunctions(Module &M);
 
-public:
-  static char ID;
+  bool init(Module &M);
 
-  DataFlowSanitizer(const std::vector<std::string> &ABIListFiles =
-                        std::vector<std::string>());
+public:
+  DataFlowSanitizer(const std::vector<std::string> &ABIListFiles);
 
-  bool doInitialization(Module &M) override;
-  bool runOnModule(Module &M) override;
+  bool runImpl(Module &M);
 };
 
 struct DFSanFunction {
@@ -482,19 +482,8 @@ class DFSanVisitor : public InstVisitor<DFSanVisitor> {
 
 } // end anonymous namespace
 
-char DataFlowSanitizer::ID;
-
-INITIALIZE_PASS(DataFlowSanitizer, "dfsan",
-                "DataFlowSanitizer: dynamic data flow analysis.", false, false)
-
-ModulePass *llvm::createDataFlowSanitizerPass(
-    const std::vector<std::string> &ABIListFiles) {
-  return new DataFlowSanitizer(ABIListFiles);
-}
-
 DataFlowSanitizer::DataFlowSanitizer(
-    const std::vector<std::string> &ABIListFiles)
-    : ModulePass(ID) {
+    const std::vector<std::string> &ABIListFiles) {
   std::vector<std::string> AllABIListFiles(std::move(ABIListFiles));
   AllABIListFiles.insert(AllABIListFiles.end(), ClABIListFiles.begin(),
                          ClABIListFiles.end());
@@ -559,7 +548,7 @@ TransformedFunction DataFlowSanitizer::getCustomFunctionType(FunctionType *T) {
       ArgumentIndexMapping);
 }
 
-bool DataFlowSanitizer::doInitialization(Module &M) {
+bool DataFlowSanitizer::init(Module &M) {
   Triple TargetTriple(M.getTargetTriple());
   bool IsX86_64 = TargetTriple.getArch() == Triple::x86_64;
   bool IsMIPS64 = TargetTriple.isMIPS64();
@@ -785,7 +774,9 @@ void DataFlowSanitizer::initializeCallbackFunctions(Module &M) {
                                                 DFSanLoadStoreCmpCallbackFnTy);
 }
 
-bool DataFlowSanitizer::runOnModule(Module &M) {
+bool DataFlowSanitizer::runImpl(Module &M) {
+  init(M);
+
   if (ABIList.isIn(M, "skip"))
     return false;
 
@@ -1817,3 +1808,37 @@ void DFSanVisitor::visitPHINode(PHINode &PN) {
   DFSF.PHIFixups.push_back(std::make_pair(&PN, ShadowPN));
   DFSF.setShadow(&PN, ShadowPN);
 }
+
+class DataFlowSanitizerLegacyPass : public ModulePass {
+private:
+  std::vector<std::string> ABIListFiles;
+
+public:
+  static char ID;
+
+  DataFlowSanitizerLegacyPass(
+      const std::vector<std::string> &ABIListFiles = std::vector<std::string>())
+      : ModulePass(ID), ABIListFiles(ABIListFiles) {}
+
+  bool runOnModule(Module &M) override {
+    return DataFlowSanitizer(ABIListFiles).runImpl(M);
+  }
+};
+
+char DataFlowSanitizerLegacyPass::ID;
+
+INITIALIZE_PASS(DataFlowSanitizerLegacyPass, "dfsan",
+                "DataFlowSanitizer: dynamic data flow analysis.", false, false)
+
+ModulePass *llvm::createDataFlowSanitizerLegacyPassPass(
+    const std::vector<std::string> &ABIListFiles) {
+  return new DataFlowSanitizerLegacyPass(ABIListFiles);
+}
+
+PreservedAnalyses DataFlowSanitizerPass::run(Module &M,
+                                             ModuleAnalysisManager &AM) {
+  if (DataFlowSanitizer(ABIListFiles).runImpl(M)) {
+    return PreservedAnalyses::none();
+  }
+  return PreservedAnalyses::all();
+}
diff --git a/llvm/lib/Transforms/Instrumentation/Instrumentation.cpp b/llvm/lib/Transforms/Instrumentation/Instrumentation.cpp
index ad238f1357c69..eda38e7da4f4d 100644
--- a/llvm/lib/Transforms/Instrumentation/Instrumentation.cpp
+++ b/llvm/lib/Transforms/Instrumentation/Instrumentation.cpp
@@ -119,7 +119,7 @@ void llvm::initializeInstrumentation(PassRegistry &Registry) {
   initializeHWAddressSanitizerLegacyPassPass(Registry);
   initializeThreadSanitizerLegacyPassPass(Registry);
   initializeModuleSanitizerCoverageLegacyPassPass(Registry);
-  initializeDataFlowSanitizerPass(Registry);
+  initializeDataFlowSanitizerLegacyPassPass(Registry);
 }
 
 /// LLVMInitializeInstrumentation - C binding for
diff --git a/llvm/test/Instrumentation/DataFlowSanitizer/call.ll b/llvm/test/Instrumentation/DataFlowSanitizer/call.ll
index f196d1be4eacf..4d9c8e999cf6f 100644
--- a/llvm/test/Instrumentation/DataFlowSanitizer/call.ll
+++ b/llvm/test/Instrumentation/DataFlowSanitizer/call.ll
@@ -1,4 +1,5 @@
 ; RUN: opt < %s -dfsan -S | FileCheck %s
+; RUN: opt < %s -passes=dfsan -S | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 

From 3efc978baede3ffb4616774e3d1c722fe4128ef5 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@intel.com>
Date: Wed, 29 Jul 2020 10:05:25 -0700
Subject: [PATCH 0495/1035] [LV] Add abs/smin/smax/umin/umax intrinsics to
 isTriviallyVectorizable

This patch adds support for vectorizing these intrinsics.

Differential Revision: https://reviews.llvm.org/D84796
---
 llvm/lib/Analysis/VectorUtils.cpp             |   8 +-
 .../Transforms/LoopVectorize/intrinsic.ll     | 130 ++++++++++++++++++
 2 files changed, 137 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/Analysis/VectorUtils.cpp b/llvm/lib/Analysis/VectorUtils.cpp
index fbd5c80350627..cdcbd1579e28d 100644
--- a/llvm/lib/Analysis/VectorUtils.cpp
+++ b/llvm/lib/Analysis/VectorUtils.cpp
@@ -43,13 +43,18 @@ static cl::opt<unsigned> MaxInterleaveGroupFactor(
 /// hasVectorInstrinsicScalarOpd).
 bool llvm::isTriviallyVectorizable(Intrinsic::ID ID) {
   switch (ID) {
-  case Intrinsic::bswap: // Begin integer bit-manipulation.
+  case Intrinsic::abs:   // Begin integer bit-manipulation.
+  case Intrinsic::bswap:
   case Intrinsic::bitreverse:
   case Intrinsic::ctpop:
   case Intrinsic::ctlz:
   case Intrinsic::cttz:
   case Intrinsic::fshl:
   case Intrinsic::fshr:
+  case Intrinsic::smax:
+  case Intrinsic::smin:
+  case Intrinsic::umax:
+  case Intrinsic::umin:
   case Intrinsic::sadd_sat:
   case Intrinsic::ssub_sat:
   case Intrinsic::uadd_sat:
@@ -94,6 +99,7 @@ bool llvm::isTriviallyVectorizable(Intrinsic::ID ID) {
 bool llvm::hasVectorInstrinsicScalarOpd(Intrinsic::ID ID,
                                         unsigned ScalarOpdIdx) {
   switch (ID) {
+  case Intrinsic::abs:
   case Intrinsic::ctlz:
   case Intrinsic::cttz:
   case Intrinsic::powi:
diff --git a/llvm/test/Transforms/LoopVectorize/intrinsic.ll b/llvm/test/Transforms/LoopVectorize/intrinsic.ll
index c2036c611334d..4639579d92017 100644
--- a/llvm/test/Transforms/LoopVectorize/intrinsic.ll
+++ b/llvm/test/Transforms/LoopVectorize/intrinsic.ll
@@ -1244,6 +1244,136 @@ for.end:                                          ; preds = %for.body, %entry
   ret void
 }
 
+declare i64 @llvm.abs.i64 (i64, i1) nounwind readnone
+
+define void @abs_i64(i32 %n, i64* noalias %y, i64* noalias %x) nounwind uwtable {
+;CHECK-LABEL: @abs_i64(
+;CHECK: llvm.abs.v4i64(<4 x i64> [[WIDE_LOADX:%.*]], i1 true)
+;CHECK: ret void
+entry:
+  %cmp9 = icmp sgt i32 %n, 0
+  br i1 %cmp9, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i64, i64* %y, i64 %indvars.iv
+  %0 = load i64, i64* %arrayidx, align 8
+  %call = tail call i64 @llvm.abs.i64(i64 %0, i1 true) nounwind readnone
+  %arrayidx4 = getelementptr inbounds i64, i64* %x, i64 %indvars.iv
+  store i64 %call, i64* %arrayidx4, align 8
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+declare i32 @llvm.smin.i32 (i32, i32)
+
+define void @smin_i32(i32 %n, i32* noalias %x, i32* noalias %y) {
+; CHECK-LABEL: @smin_i32(
+; CHECK:         call <4 x i32> @llvm.smin.v4i32(<4 x i32> [[WIDE_LOADX:%.*]], <4 x i32> [[WIDE_LOADY:%.*]])
+; CHECK:         ret void
+entry:
+  %cmp = icmp sgt i32 %n, 0
+  br i1 %cmp, label %loop, label %end
+
+loop:
+  %iv = phi i32 [ %iv.next, %loop ], [ 0, %entry ]
+  %xi = getelementptr inbounds i32, i32* %x, i32 %iv
+  %yi = getelementptr inbounds i32, i32* %y, i32 %iv
+  %xld = load i32, i32* %xi, align 4
+  %yld = load i32, i32* %yi, align 4
+  %call = tail call i32 @llvm.smin.i32(i32 %xld, i32 %yld)
+  store i32 %call, i32* %xi, align 4
+  %iv.next = add i32 %iv, 1
+  %exitcond = icmp eq i32 %iv.next, %n
+  br i1 %exitcond, label %end, label %loop
+
+end:
+  ret void
+}
+
+declare i32 @llvm.smax.i32 (i32, i32)
+
+define void @smax_i32(i32 %n, i32* noalias %x, i32* noalias %y) {
+; CHECK-LABEL: @smax_i32(
+; CHECK:         call <4 x i32> @llvm.smax.v4i32(<4 x i32> [[WIDE_LOADX:%.*]], <4 x i32> [[WIDE_LOADY:%.*]])
+; CHECK:         ret void
+entry:
+  %cmp = icmp sgt i32 %n, 0
+  br i1 %cmp, label %loop, label %end
+
+loop:
+  %iv = phi i32 [ %iv.next, %loop ], [ 0, %entry ]
+  %xi = getelementptr inbounds i32, i32* %x, i32 %iv
+  %yi = getelementptr inbounds i32, i32* %y, i32 %iv
+  %xld = load i32, i32* %xi, align 4
+  %yld = load i32, i32* %yi, align 4
+  %call = tail call i32 @llvm.smax.i32(i32 %xld, i32 %yld)
+  store i32 %call, i32* %xi, align 4
+  %iv.next = add i32 %iv, 1
+  %exitcond = icmp eq i32 %iv.next, %n
+  br i1 %exitcond, label %end, label %loop
+
+end:
+  ret void
+}
+
+declare i32 @llvm.umin.i32 (i32, i32)
+
+define void @umin_i32(i32 %n, i32* noalias %x, i32* noalias %y) {
+; CHECK-LABEL: @umin_i32(
+; CHECK:         call <4 x i32> @llvm.umin.v4i32(<4 x i32> [[WIDE_LOADX:%.*]], <4 x i32> [[WIDE_LOADY:%.*]])
+; CHECK:         ret void
+entry:
+  %cmp = icmp sgt i32 %n, 0
+  br i1 %cmp, label %loop, label %end
+
+loop:
+  %iv = phi i32 [ %iv.next, %loop ], [ 0, %entry ]
+  %xi = getelementptr inbounds i32, i32* %x, i32 %iv
+  %yi = getelementptr inbounds i32, i32* %y, i32 %iv
+  %xld = load i32, i32* %xi, align 4
+  %yld = load i32, i32* %yi, align 4
+  %call = tail call i32 @llvm.umin.i32(i32 %xld, i32 %yld)
+  store i32 %call, i32* %xi, align 4
+  %iv.next = add i32 %iv, 1
+  %exitcond = icmp eq i32 %iv.next, %n
+  br i1 %exitcond, label %end, label %loop
+
+end:
+  ret void
+}
+
+declare i32 @llvm.umax.i32 (i32, i32)
+
+define void @umax_i32(i32 %n, i32* noalias %x, i32* noalias %y) {
+; CHECK-LABEL: @umax_i32(
+; CHECK:         call <4 x i32> @llvm.umax.v4i32(<4 x i32> [[WIDE_LOADX:%.*]], <4 x i32> [[WIDE_LOADY:%.*]])
+; CHECK:         ret void
+entry:
+  %cmp = icmp sgt i32 %n, 0
+  br i1 %cmp, label %loop, label %end
+
+loop:
+  %iv = phi i32 [ %iv.next, %loop ], [ 0, %entry ]
+  %xi = getelementptr inbounds i32, i32* %x, i32 %iv
+  %yi = getelementptr inbounds i32, i32* %y, i32 %iv
+  %xld = load i32, i32* %xi, align 4
+  %yld = load i32, i32* %yi, align 4
+  %call = tail call i32 @llvm.umax.i32(i32 %xld, i32 %yld)
+  store i32 %call, i32* %xi, align 4
+  %iv.next = add i32 %iv, 1
+  %exitcond = icmp eq i32 %iv.next, %n
+  br i1 %exitcond, label %end, label %loop
+
+end:
+  ret void
+}
+
 declare i32 @llvm.fshl.i32 (i32, i32, i32)
 
 define void @fshl_i32(i32 %n, i32* noalias %x, i32* noalias %y, i32 %shAmt) {

From c4823b24a41b9e3552af7781e05ed1784fb84f46 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@intel.com>
Date: Wed, 29 Jul 2020 10:08:12 -0700
Subject: [PATCH 0496/1035] [X86] Add custom lowering for llvm.roundeven with
 sse4.1.

We can use the roundss/sd/ps/pd instructions like we do for
ceil/floor/trunc/rint/nearbyint.

Differential Revision: https://reviews.llvm.org/D84592
---
 llvm/lib/Target/X86/X86ISelDAGToDAG.cpp       |   4 +
 llvm/lib/Target/X86/X86ISelLowering.cpp       |   6 +
 llvm/test/CodeGen/X86/fp-roundeven.ll         | 690 +-----------------
 .../CodeGen/X86/fp-strict-scalar-round.ll     |  64 +-
 4 files changed, 76 insertions(+), 688 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
index 4098911dee3b1..697c160f5bffd 100644
--- a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
@@ -1005,6 +1005,8 @@ void X86DAGToDAGISel::PreprocessISelDAG() {
     case ISD::STRICT_FFLOOR:
     case ISD::FTRUNC:
     case ISD::STRICT_FTRUNC:
+    case ISD::FROUNDEVEN:
+    case ISD::STRICT_FROUNDEVEN:
     case ISD::FNEARBYINT:
     case ISD::STRICT_FNEARBYINT:
     case ISD::FRINT:
@@ -1020,6 +1022,8 @@ void X86DAGToDAGISel::PreprocessISelDAG() {
       case ISD::FFLOOR:     Imm = 0x9; break;
       case ISD::STRICT_FTRUNC:
       case ISD::FTRUNC:     Imm = 0xB; break;
+      case ISD::STRICT_FROUNDEVEN:
+      case ISD::FROUNDEVEN: Imm = 0x8; break;
       case ISD::STRICT_FNEARBYINT:
       case ISD::FNEARBYINT: Imm = 0xC; break;
       case ISD::STRICT_FRINT:
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 94e2b1708b9db..556bb58468817 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -1081,6 +1081,8 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
       setOperationAction(ISD::STRICT_FRINT,      RoundedTy,  Legal);
       setOperationAction(ISD::FNEARBYINT,        RoundedTy,  Legal);
       setOperationAction(ISD::STRICT_FNEARBYINT, RoundedTy,  Legal);
+      setOperationAction(ISD::FROUNDEVEN,        RoundedTy,  Legal);
+      setOperationAction(ISD::STRICT_FROUNDEVEN, RoundedTy,  Legal);
 
       setOperationAction(ISD::FROUND,            RoundedTy,  Custom);
     }
@@ -1175,6 +1177,8 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
       setOperationAction(ISD::STRICT_FRINT,      VT, Legal);
       setOperationAction(ISD::FNEARBYINT,        VT, Legal);
       setOperationAction(ISD::STRICT_FNEARBYINT, VT, Legal);
+      setOperationAction(ISD::FROUNDEVEN,        VT, Legal);
+      setOperationAction(ISD::STRICT_FROUNDEVEN, VT, Legal);
 
       setOperationAction(ISD::FROUND,            VT, Custom);
 
@@ -1560,6 +1564,8 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
       setOperationAction(ISD::STRICT_FRINT,      VT, Legal);
       setOperationAction(ISD::FNEARBYINT,        VT, Legal);
       setOperationAction(ISD::STRICT_FNEARBYINT, VT, Legal);
+      setOperationAction(ISD::FROUNDEVEN,        VT, Legal);
+      setOperationAction(ISD::STRICT_FROUNDEVEN, VT, Legal);
 
       setOperationAction(ISD::FROUND,            VT, Custom);
     }
diff --git a/llvm/test/CodeGen/X86/fp-roundeven.ll b/llvm/test/CodeGen/X86/fp-roundeven.ll
index cc6cc35ee2e02..e780469682c98 100644
--- a/llvm/test/CodeGen/X86/fp-roundeven.ll
+++ b/llvm/test/CodeGen/X86/fp-roundeven.ll
@@ -11,11 +11,13 @@ define float @roundeven_f32(float %x) {
 ;
 ; SSE41-LABEL: roundeven_f32:
 ; SSE41:       ## %bb.0:
-; SSE41-NEXT:    jmp _roundevenf ## TAILCALL
+; SSE41-NEXT:    roundss $8, %xmm0, %xmm0
+; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: roundeven_f32:
 ; AVX:       ## %bb.0:
-; AVX-NEXT:    jmp _roundevenf ## TAILCALL
+; AVX-NEXT:    vroundss $8, %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    retq
   %a = call float @llvm.roundeven.f32(float %x)
   ret float %a
 }
@@ -27,11 +29,13 @@ define double @roundeven_f64(double %x) {
 ;
 ; SSE41-LABEL: roundeven_f64:
 ; SSE41:       ## %bb.0:
-; SSE41-NEXT:    jmp _roundeven ## TAILCALL
+; SSE41-NEXT:    roundsd $8, %xmm0, %xmm0
+; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: roundeven_f64:
 ; AVX:       ## %bb.0:
-; AVX-NEXT:    jmp _roundeven ## TAILCALL
+; AVX-NEXT:    vroundsd $8, %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    retq
   %a = call double @llvm.roundeven.f64(double %x)
   ret double %a
 }
@@ -67,57 +71,12 @@ define <4 x float> @roundeven_v4f32(<4 x float> %x) {
 ;
 ; SSE41-LABEL: roundeven_v4f32:
 ; SSE41:       ## %bb.0:
-; SSE41-NEXT:    subq $40, %rsp
-; SSE41-NEXT:    .cfi_def_cfa_offset 48
-; SSE41-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; SSE41-NEXT:    callq _roundevenf
-; SSE41-NEXT:    movaps %xmm0, (%rsp) ## 16-byte Spill
-; SSE41-NEXT:    movshdup {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
-; SSE41-NEXT:    ## xmm0 = mem[1,1,3,3]
-; SSE41-NEXT:    callq _roundevenf
-; SSE41-NEXT:    movaps (%rsp), %xmm1 ## 16-byte Reload
-; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[2,3]
-; SSE41-NEXT:    movaps %xmm1, (%rsp) ## 16-byte Spill
-; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
-; SSE41-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
-; SSE41-NEXT:    callq _roundevenf
-; SSE41-NEXT:    movaps (%rsp), %xmm1 ## 16-byte Reload
-; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0,1],xmm0[0],xmm1[3]
-; SSE41-NEXT:    movaps %xmm1, (%rsp) ## 16-byte Spill
-; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
-; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; SSE41-NEXT:    callq _roundevenf
-; SSE41-NEXT:    movaps (%rsp), %xmm1 ## 16-byte Reload
-; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm0[0]
-; SSE41-NEXT:    movaps %xmm1, %xmm0
-; SSE41-NEXT:    addq $40, %rsp
+; SSE41-NEXT:    roundps $8, %xmm0, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: roundeven_v4f32:
 ; AVX:       ## %bb.0:
-; AVX-NEXT:    subq $40, %rsp
-; AVX-NEXT:    .cfi_def_cfa_offset 48
-; AVX-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; AVX-NEXT:    callq _roundevenf
-; AVX-NEXT:    vmovaps %xmm0, (%rsp) ## 16-byte Spill
-; AVX-NEXT:    vmovshdup {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
-; AVX-NEXT:    ## xmm0 = mem[1,1,3,3]
-; AVX-NEXT:    callq _roundevenf
-; AVX-NEXT:    vmovaps (%rsp), %xmm1 ## 16-byte Reload
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3]
-; AVX-NEXT:    vmovaps %xmm0, (%rsp) ## 16-byte Spill
-; AVX-NEXT:    vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
-; AVX-NEXT:    ## xmm0 = mem[1,0]
-; AVX-NEXT:    callq _roundevenf
-; AVX-NEXT:    vmovaps (%rsp), %xmm1 ## 16-byte Reload
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
-; AVX-NEXT:    vmovaps %xmm0, (%rsp) ## 16-byte Spill
-; AVX-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
-; AVX-NEXT:    ## xmm0 = mem[3,3,3,3]
-; AVX-NEXT:    callq _roundevenf
-; AVX-NEXT:    vmovaps (%rsp), %xmm1 ## 16-byte Reload
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
-; AVX-NEXT:    addq $40, %rsp
+; AVX-NEXT:    vroundps $8, %xmm0, %xmm0
 ; AVX-NEXT:    retq
   %a = call <4 x float> @llvm.roundeven.v4f32(<4 x float> %x)
   ret <4 x float> %a
@@ -142,33 +101,12 @@ define <2 x double> @roundeven_v2f64(<2 x double> %x) {
 ;
 ; SSE41-LABEL: roundeven_v2f64:
 ; SSE41:       ## %bb.0:
-; SSE41-NEXT:    subq $40, %rsp
-; SSE41-NEXT:    .cfi_def_cfa_offset 48
-; SSE41-NEXT:    movaps %xmm0, (%rsp) ## 16-byte Spill
-; SSE41-NEXT:    callq _roundeven
-; SSE41-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; SSE41-NEXT:    movaps (%rsp), %xmm0 ## 16-byte Reload
-; SSE41-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
-; SSE41-NEXT:    callq _roundeven
-; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
-; SSE41-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; SSE41-NEXT:    movaps %xmm1, %xmm0
-; SSE41-NEXT:    addq $40, %rsp
+; SSE41-NEXT:    roundpd $8, %xmm0, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: roundeven_v2f64:
 ; AVX:       ## %bb.0:
-; AVX-NEXT:    subq $40, %rsp
-; AVX-NEXT:    .cfi_def_cfa_offset 48
-; AVX-NEXT:    vmovaps %xmm0, (%rsp) ## 16-byte Spill
-; AVX-NEXT:    callq _roundeven
-; AVX-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; AVX-NEXT:    vpermilpd $1, (%rsp), %xmm0 ## 16-byte Folded Reload
-; AVX-NEXT:    ## xmm0 = mem[1,0]
-; AVX-NEXT:    callq _roundeven
-; AVX-NEXT:    vmovapd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
-; AVX-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX-NEXT:    addq $40, %rsp
+; AVX-NEXT:    vroundpd $8, %xmm0, %xmm0
 ; AVX-NEXT:    retq
   %a = call <2 x double> @llvm.roundeven.v2f64(<2 x double> %x)
   ret <2 x double> %a
@@ -227,106 +165,13 @@ define <8 x float> @roundeven_v8f32(<8 x float> %x) {
 ;
 ; SSE41-LABEL: roundeven_v8f32:
 ; SSE41:       ## %bb.0:
-; SSE41-NEXT:    subq $56, %rsp
-; SSE41-NEXT:    .cfi_def_cfa_offset 64
-; SSE41-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; SSE41-NEXT:    movaps %xmm0, (%rsp) ## 16-byte Spill
-; SSE41-NEXT:    callq _roundevenf
-; SSE41-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; SSE41-NEXT:    movshdup (%rsp), %xmm0 ## 16-byte Folded Reload
-; SSE41-NEXT:    ## xmm0 = mem[1,1,3,3]
-; SSE41-NEXT:    callq _roundevenf
-; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
-; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[2,3]
-; SSE41-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; SSE41-NEXT:    movaps (%rsp), %xmm0 ## 16-byte Reload
-; SSE41-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
-; SSE41-NEXT:    callq _roundevenf
-; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
-; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0,1],xmm0[0],xmm1[3]
-; SSE41-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; SSE41-NEXT:    movaps (%rsp), %xmm0 ## 16-byte Reload
-; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; SSE41-NEXT:    callq _roundevenf
-; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
-; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm0[0]
-; SSE41-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
-; SSE41-NEXT:    callq _roundevenf
-; SSE41-NEXT:    movaps %xmm0, (%rsp) ## 16-byte Spill
-; SSE41-NEXT:    movshdup {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
-; SSE41-NEXT:    ## xmm0 = mem[1,1,3,3]
-; SSE41-NEXT:    callq _roundevenf
-; SSE41-NEXT:    movaps (%rsp), %xmm1 ## 16-byte Reload
-; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[2,3]
-; SSE41-NEXT:    movaps %xmm1, (%rsp) ## 16-byte Spill
-; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
-; SSE41-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
-; SSE41-NEXT:    callq _roundevenf
-; SSE41-NEXT:    movaps (%rsp), %xmm1 ## 16-byte Reload
-; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0,1],xmm0[0],xmm1[3]
-; SSE41-NEXT:    movaps %xmm1, (%rsp) ## 16-byte Spill
-; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
-; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; SSE41-NEXT:    callq _roundevenf
-; SSE41-NEXT:    movaps (%rsp), %xmm1 ## 16-byte Reload
-; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm0[0]
-; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
-; SSE41-NEXT:    addq $56, %rsp
+; SSE41-NEXT:    roundps $8, %xmm0, %xmm0
+; SSE41-NEXT:    roundps $8, %xmm1, %xmm1
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: roundeven_v8f32:
 ; AVX:       ## %bb.0:
-; AVX-NEXT:    subq $88, %rsp
-; AVX-NEXT:    .cfi_def_cfa_offset 96
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 32-byte Spill
-; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; AVX-NEXT:    vzeroupper
-; AVX-NEXT:    callq _roundevenf
-; AVX-NEXT:    vmovaps %xmm0, (%rsp) ## 16-byte Spill
-; AVX-NEXT:    vmovshdup {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
-; AVX-NEXT:    ## xmm0 = mem[1,1,3,3]
-; AVX-NEXT:    callq _roundevenf
-; AVX-NEXT:    vmovaps (%rsp), %xmm1 ## 16-byte Reload
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3]
-; AVX-NEXT:    vmovaps %xmm0, (%rsp) ## 16-byte Spill
-; AVX-NEXT:    vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
-; AVX-NEXT:    ## xmm0 = mem[1,0]
-; AVX-NEXT:    callq _roundevenf
-; AVX-NEXT:    vmovaps (%rsp), %xmm1 ## 16-byte Reload
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
-; AVX-NEXT:    vmovaps %xmm0, (%rsp) ## 16-byte Spill
-; AVX-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
-; AVX-NEXT:    ## xmm0 = mem[3,3,3,3]
-; AVX-NEXT:    callq _roundevenf
-; AVX-NEXT:    vmovaps (%rsp), %xmm1 ## 16-byte Reload
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
-; AVX-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 ## 32-byte Reload
-; AVX-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $ymm0
-; AVX-NEXT:    vzeroupper
-; AVX-NEXT:    callq _roundevenf
-; AVX-NEXT:    vmovaps %xmm0, (%rsp) ## 16-byte Spill
-; AVX-NEXT:    vmovshdup {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
-; AVX-NEXT:    ## xmm0 = mem[1,1,3,3]
-; AVX-NEXT:    callq _roundevenf
-; AVX-NEXT:    vmovaps (%rsp), %xmm1 ## 16-byte Reload
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3]
-; AVX-NEXT:    vmovaps %xmm0, (%rsp) ## 16-byte Spill
-; AVX-NEXT:    vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
-; AVX-NEXT:    ## xmm0 = mem[1,0]
-; AVX-NEXT:    callq _roundevenf
-; AVX-NEXT:    vmovaps (%rsp), %xmm1 ## 16-byte Reload
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
-; AVX-NEXT:    vmovaps %xmm0, (%rsp) ## 16-byte Spill
-; AVX-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
-; AVX-NEXT:    ## xmm0 = mem[3,3,3,3]
-; AVX-NEXT:    callq _roundevenf
-; AVX-NEXT:    vmovaps (%rsp), %xmm1 ## 16-byte Reload
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
-; AVX-NEXT:    vinsertf128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 ## 16-byte Folded Reload
-; AVX-NEXT:    addq $88, %rsp
+; AVX-NEXT:    vroundps $8, %ymm0, %ymm0
 ; AVX-NEXT:    retq
   %a = call <8 x float> @llvm.roundeven.v8f32(<8 x float> %x)
   ret <8 x float> %a
@@ -361,58 +206,13 @@ define <4 x double> @roundeven_v4f64(<4 x double> %x) {
 ;
 ; SSE41-LABEL: roundeven_v4f64:
 ; SSE41:       ## %bb.0:
-; SSE41-NEXT:    subq $56, %rsp
-; SSE41-NEXT:    .cfi_def_cfa_offset 64
-; SSE41-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; SSE41-NEXT:    movaps %xmm0, (%rsp) ## 16-byte Spill
-; SSE41-NEXT:    callq _roundeven
-; SSE41-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; SSE41-NEXT:    movaps (%rsp), %xmm0 ## 16-byte Reload
-; SSE41-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
-; SSE41-NEXT:    callq _roundeven
-; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
-; SSE41-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; SSE41-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
-; SSE41-NEXT:    callq _roundeven
-; SSE41-NEXT:    movaps %xmm0, (%rsp) ## 16-byte Spill
-; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
-; SSE41-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
-; SSE41-NEXT:    callq _roundeven
-; SSE41-NEXT:    movaps (%rsp), %xmm1 ## 16-byte Reload
-; SSE41-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
-; SSE41-NEXT:    addq $56, %rsp
+; SSE41-NEXT:    roundpd $8, %xmm0, %xmm0
+; SSE41-NEXT:    roundpd $8, %xmm1, %xmm1
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: roundeven_v4f64:
 ; AVX:       ## %bb.0:
-; AVX-NEXT:    subq $88, %rsp
-; AVX-NEXT:    .cfi_def_cfa_offset 96
-; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 32-byte Spill
-; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX-NEXT:    vmovaps %xmm0, (%rsp) ## 16-byte Spill
-; AVX-NEXT:    vzeroupper
-; AVX-NEXT:    callq _roundeven
-; AVX-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; AVX-NEXT:    vpermilpd $1, (%rsp), %xmm0 ## 16-byte Folded Reload
-; AVX-NEXT:    ## xmm0 = mem[1,0]
-; AVX-NEXT:    callq _roundeven
-; AVX-NEXT:    vmovapd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
-; AVX-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX-NEXT:    vmovapd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 ## 32-byte Reload
-; AVX-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $ymm0
-; AVX-NEXT:    vzeroupper
-; AVX-NEXT:    callq _roundeven
-; AVX-NEXT:    vmovaps %xmm0, (%rsp) ## 16-byte Spill
-; AVX-NEXT:    vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
-; AVX-NEXT:    ## xmm0 = mem[1,0]
-; AVX-NEXT:    callq _roundeven
-; AVX-NEXT:    vmovapd (%rsp), %xmm1 ## 16-byte Reload
-; AVX-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX-NEXT:    vinsertf128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 ## 16-byte Folded Reload
-; AVX-NEXT:    addq $88, %rsp
+; AVX-NEXT:    vroundpd $8, %ymm0, %ymm0
 ; AVX-NEXT:    retq
   %a = call <4 x double> @llvm.roundeven.v4f64(<4 x double> %x)
   ret <4 x double> %a
@@ -517,306 +317,21 @@ define <16 x float> @roundeven_v16f32(<16 x float> %x) {
 ;
 ; SSE41-LABEL: roundeven_v16f32:
 ; SSE41:       ## %bb.0:
-; SSE41-NEXT:    subq $88, %rsp
-; SSE41-NEXT:    .cfi_def_cfa_offset 96
-; SSE41-NEXT:    movaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; SSE41-NEXT:    movaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; SSE41-NEXT:    movaps %xmm1, (%rsp) ## 16-byte Spill
-; SSE41-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; SSE41-NEXT:    callq _roundevenf
-; SSE41-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; SSE41-NEXT:    movshdup {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
-; SSE41-NEXT:    ## xmm0 = mem[1,1,3,3]
-; SSE41-NEXT:    callq _roundevenf
-; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
-; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[2,3]
-; SSE41-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
-; SSE41-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
-; SSE41-NEXT:    callq _roundevenf
-; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
-; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0,1],xmm0[0],xmm1[3]
-; SSE41-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
-; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; SSE41-NEXT:    callq _roundevenf
-; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
-; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm0[0]
-; SSE41-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; SSE41-NEXT:    movaps (%rsp), %xmm0 ## 16-byte Reload
-; SSE41-NEXT:    callq _roundevenf
-; SSE41-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; SSE41-NEXT:    movshdup (%rsp), %xmm0 ## 16-byte Folded Reload
-; SSE41-NEXT:    ## xmm0 = mem[1,1,3,3]
-; SSE41-NEXT:    callq _roundevenf
-; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
-; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[2,3]
-; SSE41-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; SSE41-NEXT:    movaps (%rsp), %xmm0 ## 16-byte Reload
-; SSE41-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
-; SSE41-NEXT:    callq _roundevenf
-; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
-; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0,1],xmm0[0],xmm1[3]
-; SSE41-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; SSE41-NEXT:    movaps (%rsp), %xmm0 ## 16-byte Reload
-; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; SSE41-NEXT:    callq _roundevenf
-; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
-; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm0[0]
-; SSE41-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
-; SSE41-NEXT:    callq _roundevenf
-; SSE41-NEXT:    movaps %xmm0, (%rsp) ## 16-byte Spill
-; SSE41-NEXT:    movshdup {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
-; SSE41-NEXT:    ## xmm0 = mem[1,1,3,3]
-; SSE41-NEXT:    callq _roundevenf
-; SSE41-NEXT:    movaps (%rsp), %xmm1 ## 16-byte Reload
-; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[2,3]
-; SSE41-NEXT:    movaps %xmm1, (%rsp) ## 16-byte Spill
-; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
-; SSE41-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
-; SSE41-NEXT:    callq _roundevenf
-; SSE41-NEXT:    movaps (%rsp), %xmm1 ## 16-byte Reload
-; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0,1],xmm0[0],xmm1[3]
-; SSE41-NEXT:    movaps %xmm1, (%rsp) ## 16-byte Spill
-; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
-; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; SSE41-NEXT:    callq _roundevenf
-; SSE41-NEXT:    movaps (%rsp), %xmm1 ## 16-byte Reload
-; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm0[0]
-; SSE41-NEXT:    movaps %xmm1, (%rsp) ## 16-byte Spill
-; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
-; SSE41-NEXT:    callq _roundevenf
-; SSE41-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; SSE41-NEXT:    movshdup {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
-; SSE41-NEXT:    ## xmm0 = mem[1,1,3,3]
-; SSE41-NEXT:    callq _roundevenf
-; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
-; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[2,3]
-; SSE41-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
-; SSE41-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
-; SSE41-NEXT:    callq _roundevenf
-; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
-; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0,1],xmm0[0],xmm1[3]
-; SSE41-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
-; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; SSE41-NEXT:    callq _roundevenf
-; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 ## 16-byte Reload
-; SSE41-NEXT:    insertps {{.*#+}} xmm3 = xmm3[0,1,2],xmm0[0]
-; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
-; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
-; SSE41-NEXT:    movaps (%rsp), %xmm2 ## 16-byte Reload
-; SSE41-NEXT:    addq $88, %rsp
+; SSE41-NEXT:    roundps $8, %xmm0, %xmm0
+; SSE41-NEXT:    roundps $8, %xmm1, %xmm1
+; SSE41-NEXT:    roundps $8, %xmm2, %xmm2
+; SSE41-NEXT:    roundps $8, %xmm3, %xmm3
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: roundeven_v16f32:
 ; AVX1:       ## %bb.0:
-; AVX1-NEXT:    subq $152, %rsp
-; AVX1-NEXT:    .cfi_def_cfa_offset 160
-; AVX1-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) ## 32-byte Spill
-; AVX1-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 32-byte Spill
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; AVX1-NEXT:    vzeroupper
-; AVX1-NEXT:    callq _roundevenf
-; AVX1-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; AVX1-NEXT:    vmovshdup {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
-; AVX1-NEXT:    ## xmm0 = mem[1,1,3,3]
-; AVX1-NEXT:    callq _roundevenf
-; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
-; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3]
-; AVX1-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; AVX1-NEXT:    vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
-; AVX1-NEXT:    ## xmm0 = mem[1,0]
-; AVX1-NEXT:    callq _roundevenf
-; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
-; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
-; AVX1-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; AVX1-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
-; AVX1-NEXT:    ## xmm0 = mem[3,3,3,3]
-; AVX1-NEXT:    callq _roundevenf
-; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
-; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
-; AVX1-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; AVX1-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 ## 32-byte Reload
-; AVX1-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $ymm0
-; AVX1-NEXT:    vzeroupper
-; AVX1-NEXT:    callq _roundevenf
-; AVX1-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; AVX1-NEXT:    vmovshdup {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
-; AVX1-NEXT:    ## xmm0 = mem[1,1,3,3]
-; AVX1-NEXT:    callq _roundevenf
-; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
-; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3]
-; AVX1-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; AVX1-NEXT:    vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
-; AVX1-NEXT:    ## xmm0 = mem[1,0]
-; AVX1-NEXT:    callq _roundevenf
-; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
-; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
-; AVX1-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; AVX1-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
-; AVX1-NEXT:    ## xmm0 = mem[3,3,3,3]
-; AVX1-NEXT:    callq _roundevenf
-; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
-; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
-; AVX1-NEXT:    vinsertf128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 ## 16-byte Folded Reload
-; AVX1-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 32-byte Spill
-; AVX1-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 ## 32-byte Reload
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; AVX1-NEXT:    vzeroupper
-; AVX1-NEXT:    callq _roundevenf
-; AVX1-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; AVX1-NEXT:    vmovshdup {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
-; AVX1-NEXT:    ## xmm0 = mem[1,1,3,3]
-; AVX1-NEXT:    callq _roundevenf
-; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
-; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3]
-; AVX1-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; AVX1-NEXT:    vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
-; AVX1-NEXT:    ## xmm0 = mem[1,0]
-; AVX1-NEXT:    callq _roundevenf
-; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
-; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
-; AVX1-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; AVX1-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
-; AVX1-NEXT:    ## xmm0 = mem[3,3,3,3]
-; AVX1-NEXT:    callq _roundevenf
-; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
-; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
-; AVX1-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; AVX1-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 ## 32-byte Reload
-; AVX1-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $ymm0
-; AVX1-NEXT:    vzeroupper
-; AVX1-NEXT:    callq _roundevenf
-; AVX1-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; AVX1-NEXT:    vmovshdup {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
-; AVX1-NEXT:    ## xmm0 = mem[1,1,3,3]
-; AVX1-NEXT:    callq _roundevenf
-; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
-; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3]
-; AVX1-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; AVX1-NEXT:    vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
-; AVX1-NEXT:    ## xmm0 = mem[1,0]
-; AVX1-NEXT:    callq _roundevenf
-; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
-; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
-; AVX1-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; AVX1-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
-; AVX1-NEXT:    ## xmm0 = mem[3,3,3,3]
-; AVX1-NEXT:    callq _roundevenf
-; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
-; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
-; AVX1-NEXT:    vinsertf128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 ## 16-byte Folded Reload
-; AVX1-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 ## 32-byte Reload
-; AVX1-NEXT:    addq $152, %rsp
+; AVX1-NEXT:    vroundps $8, %ymm0, %ymm0
+; AVX1-NEXT:    vroundps $8, %ymm1, %ymm1
 ; AVX1-NEXT:    retq
 ;
 ; AVX512-LABEL: roundeven_v16f32:
 ; AVX512:       ## %bb.0:
-; AVX512-NEXT:    subq $184, %rsp
-; AVX512-NEXT:    .cfi_def_cfa_offset 192
-; AVX512-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 64-byte Spill
-; AVX512-NEXT:    vextractf32x4 $3, %zmm0, %xmm0
-; AVX512-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; AVX512-NEXT:    vzeroupper
-; AVX512-NEXT:    callq _roundevenf
-; AVX512-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; AVX512-NEXT:    vmovshdup {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
-; AVX512-NEXT:    ## xmm0 = mem[1,1,3,3]
-; AVX512-NEXT:    callq _roundevenf
-; AVX512-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
-; AVX512-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3]
-; AVX512-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; AVX512-NEXT:    vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
-; AVX512-NEXT:    ## xmm0 = mem[1,0]
-; AVX512-NEXT:    callq _roundevenf
-; AVX512-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
-; AVX512-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
-; AVX512-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; AVX512-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
-; AVX512-NEXT:    ## xmm0 = mem[3,3,3,3]
-; AVX512-NEXT:    callq _roundevenf
-; AVX512-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
-; AVX512-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
-; AVX512-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; AVX512-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 ## 64-byte Reload
-; AVX512-NEXT:    vextractf32x4 $2, %zmm0, %xmm0
-; AVX512-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; AVX512-NEXT:    vzeroupper
-; AVX512-NEXT:    callq _roundevenf
-; AVX512-NEXT:    vmovaps %xmm0, (%rsp) ## 16-byte Spill
-; AVX512-NEXT:    vmovshdup {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
-; AVX512-NEXT:    ## xmm0 = mem[1,1,3,3]
-; AVX512-NEXT:    callq _roundevenf
-; AVX512-NEXT:    vmovaps (%rsp), %xmm1 ## 16-byte Reload
-; AVX512-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3]
-; AVX512-NEXT:    vmovaps %xmm0, (%rsp) ## 16-byte Spill
-; AVX512-NEXT:    vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
-; AVX512-NEXT:    ## xmm0 = mem[1,0]
-; AVX512-NEXT:    callq _roundevenf
-; AVX512-NEXT:    vmovaps (%rsp), %xmm1 ## 16-byte Reload
-; AVX512-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
-; AVX512-NEXT:    vmovaps %xmm0, (%rsp) ## 16-byte Spill
-; AVX512-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
-; AVX512-NEXT:    ## xmm0 = mem[3,3,3,3]
-; AVX512-NEXT:    callq _roundevenf
-; AVX512-NEXT:    vmovaps (%rsp), %xmm1 ## 16-byte Reload
-; AVX512-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
-; AVX512-NEXT:    vinsertf128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 ## 16-byte Folded Reload
-; AVX512-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 32-byte Spill
-; AVX512-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 ## 64-byte Reload
-; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX512-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; AVX512-NEXT:    vzeroupper
-; AVX512-NEXT:    callq _roundevenf
-; AVX512-NEXT:    vmovaps %xmm0, (%rsp) ## 16-byte Spill
-; AVX512-NEXT:    vmovshdup {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
-; AVX512-NEXT:    ## xmm0 = mem[1,1,3,3]
-; AVX512-NEXT:    callq _roundevenf
-; AVX512-NEXT:    vmovaps (%rsp), %xmm1 ## 16-byte Reload
-; AVX512-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3]
-; AVX512-NEXT:    vmovaps %xmm0, (%rsp) ## 16-byte Spill
-; AVX512-NEXT:    vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
-; AVX512-NEXT:    ## xmm0 = mem[1,0]
-; AVX512-NEXT:    callq _roundevenf
-; AVX512-NEXT:    vmovaps (%rsp), %xmm1 ## 16-byte Reload
-; AVX512-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
-; AVX512-NEXT:    vmovaps %xmm0, (%rsp) ## 16-byte Spill
-; AVX512-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
-; AVX512-NEXT:    ## xmm0 = mem[3,3,3,3]
-; AVX512-NEXT:    callq _roundevenf
-; AVX512-NEXT:    vmovaps (%rsp), %xmm1 ## 16-byte Reload
-; AVX512-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
-; AVX512-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; AVX512-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 ## 64-byte Reload
-; AVX512-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $zmm0
-; AVX512-NEXT:    vzeroupper
-; AVX512-NEXT:    callq _roundevenf
-; AVX512-NEXT:    vmovaps %xmm0, (%rsp) ## 16-byte Spill
-; AVX512-NEXT:    vmovshdup {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
-; AVX512-NEXT:    ## xmm0 = mem[1,1,3,3]
-; AVX512-NEXT:    callq _roundevenf
-; AVX512-NEXT:    vmovaps (%rsp), %xmm1 ## 16-byte Reload
-; AVX512-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3]
-; AVX512-NEXT:    vmovaps %xmm0, (%rsp) ## 16-byte Spill
-; AVX512-NEXT:    vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
-; AVX512-NEXT:    ## xmm0 = mem[1,0]
-; AVX512-NEXT:    callq _roundevenf
-; AVX512-NEXT:    vmovaps (%rsp), %xmm1 ## 16-byte Reload
-; AVX512-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
-; AVX512-NEXT:    vmovaps %xmm0, (%rsp) ## 16-byte Spill
-; AVX512-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
-; AVX512-NEXT:    ## xmm0 = mem[3,3,3,3]
-; AVX512-NEXT:    callq _roundevenf
-; AVX512-NEXT:    vmovaps (%rsp), %xmm1 ## 16-byte Reload
-; AVX512-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
-; AVX512-NEXT:    vinsertf128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 ## 16-byte Folded Reload
-; AVX512-NEXT:    vinsertf64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 ## 32-byte Folded Reload
-; AVX512-NEXT:    addq $184, %rsp
+; AVX512-NEXT:    vrndscaleps $8, %zmm0, %zmm0
 ; AVX512-NEXT:    retq
   %a = call <16 x float> @llvm.roundeven.v16f32(<16 x float> %x)
   ret <16 x float> %a
@@ -873,162 +388,21 @@ define <8 x double> @roundeven_v8f64(<8 x double> %x) {
 ;
 ; SSE41-LABEL: roundeven_v8f64:
 ; SSE41:       ## %bb.0:
-; SSE41-NEXT:    subq $88, %rsp
-; SSE41-NEXT:    .cfi_def_cfa_offset 96
-; SSE41-NEXT:    movaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; SSE41-NEXT:    movaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; SSE41-NEXT:    movaps %xmm1, (%rsp) ## 16-byte Spill
-; SSE41-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; SSE41-NEXT:    callq _roundeven
-; SSE41-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
-; SSE41-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
-; SSE41-NEXT:    callq _roundeven
-; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
-; SSE41-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; SSE41-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; SSE41-NEXT:    movaps (%rsp), %xmm0 ## 16-byte Reload
-; SSE41-NEXT:    callq _roundeven
-; SSE41-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; SSE41-NEXT:    movaps (%rsp), %xmm0 ## 16-byte Reload
-; SSE41-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
-; SSE41-NEXT:    callq _roundeven
-; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
-; SSE41-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; SSE41-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
-; SSE41-NEXT:    callq _roundeven
-; SSE41-NEXT:    movaps %xmm0, (%rsp) ## 16-byte Spill
-; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
-; SSE41-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
-; SSE41-NEXT:    callq _roundeven
-; SSE41-NEXT:    movaps (%rsp), %xmm1 ## 16-byte Reload
-; SSE41-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; SSE41-NEXT:    movaps %xmm1, (%rsp) ## 16-byte Spill
-; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
-; SSE41-NEXT:    callq _roundeven
-; SSE41-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
-; SSE41-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
-; SSE41-NEXT:    callq _roundeven
-; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 ## 16-byte Reload
-; SSE41-NEXT:    movlhps {{.*#+}} xmm3 = xmm3[0],xmm0[0]
-; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
-; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
-; SSE41-NEXT:    movaps (%rsp), %xmm2 ## 16-byte Reload
-; SSE41-NEXT:    addq $88, %rsp
+; SSE41-NEXT:    roundpd $8, %xmm0, %xmm0
+; SSE41-NEXT:    roundpd $8, %xmm1, %xmm1
+; SSE41-NEXT:    roundpd $8, %xmm2, %xmm2
+; SSE41-NEXT:    roundpd $8, %xmm3, %xmm3
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: roundeven_v8f64:
 ; AVX1:       ## %bb.0:
-; AVX1-NEXT:    subq $120, %rsp
-; AVX1-NEXT:    .cfi_def_cfa_offset 128
-; AVX1-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) ## 32-byte Spill
-; AVX1-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 32-byte Spill
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT:    vmovaps %xmm0, (%rsp) ## 16-byte Spill
-; AVX1-NEXT:    vzeroupper
-; AVX1-NEXT:    callq _roundeven
-; AVX1-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; AVX1-NEXT:    vpermilpd $1, (%rsp), %xmm0 ## 16-byte Folded Reload
-; AVX1-NEXT:    ## xmm0 = mem[1,0]
-; AVX1-NEXT:    callq _roundeven
-; AVX1-NEXT:    vmovapd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
-; AVX1-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX1-NEXT:    vmovapd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; AVX1-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 ## 32-byte Reload
-; AVX1-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $ymm0
-; AVX1-NEXT:    vzeroupper
-; AVX1-NEXT:    callq _roundeven
-; AVX1-NEXT:    vmovaps %xmm0, (%rsp) ## 16-byte Spill
-; AVX1-NEXT:    vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
-; AVX1-NEXT:    ## xmm0 = mem[1,0]
-; AVX1-NEXT:    callq _roundeven
-; AVX1-NEXT:    vmovapd (%rsp), %xmm1 ## 16-byte Reload
-; AVX1-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX1-NEXT:    vinsertf128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 ## 16-byte Folded Reload
-; AVX1-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 32-byte Spill
-; AVX1-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 ## 32-byte Reload
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT:    vmovaps %xmm0, (%rsp) ## 16-byte Spill
-; AVX1-NEXT:    vzeroupper
-; AVX1-NEXT:    callq _roundeven
-; AVX1-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; AVX1-NEXT:    vpermilpd $1, (%rsp), %xmm0 ## 16-byte Folded Reload
-; AVX1-NEXT:    ## xmm0 = mem[1,0]
-; AVX1-NEXT:    callq _roundeven
-; AVX1-NEXT:    vmovapd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
-; AVX1-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX1-NEXT:    vmovapd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; AVX1-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 ## 32-byte Reload
-; AVX1-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $ymm0
-; AVX1-NEXT:    vzeroupper
-; AVX1-NEXT:    callq _roundeven
-; AVX1-NEXT:    vmovaps %xmm0, (%rsp) ## 16-byte Spill
-; AVX1-NEXT:    vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
-; AVX1-NEXT:    ## xmm0 = mem[1,0]
-; AVX1-NEXT:    callq _roundeven
-; AVX1-NEXT:    vmovapd (%rsp), %xmm1 ## 16-byte Reload
-; AVX1-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX1-NEXT:    vinsertf128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 ## 16-byte Folded Reload
-; AVX1-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 ## 32-byte Reload
-; AVX1-NEXT:    addq $120, %rsp
+; AVX1-NEXT:    vroundpd $8, %ymm0, %ymm0
+; AVX1-NEXT:    vroundpd $8, %ymm1, %ymm1
 ; AVX1-NEXT:    retq
 ;
 ; AVX512-LABEL: roundeven_v8f64:
 ; AVX512:       ## %bb.0:
-; AVX512-NEXT:    subq $184, %rsp
-; AVX512-NEXT:    .cfi_def_cfa_offset 192
-; AVX512-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 64-byte Spill
-; AVX512-NEXT:    vextractf32x4 $3, %zmm0, %xmm0
-; AVX512-NEXT:    vmovaps %xmm0, (%rsp) ## 16-byte Spill
-; AVX512-NEXT:    vzeroupper
-; AVX512-NEXT:    callq _roundeven
-; AVX512-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; AVX512-NEXT:    vpermilpd $1, (%rsp), %xmm0 ## 16-byte Folded Reload
-; AVX512-NEXT:    ## xmm0 = mem[1,0]
-; AVX512-NEXT:    callq _roundeven
-; AVX512-NEXT:    vmovapd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
-; AVX512-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX512-NEXT:    vmovapd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; AVX512-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 ## 64-byte Reload
-; AVX512-NEXT:    vextractf32x4 $2, %zmm0, %xmm0
-; AVX512-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; AVX512-NEXT:    vzeroupper
-; AVX512-NEXT:    callq _roundeven
-; AVX512-NEXT:    vmovaps %xmm0, (%rsp) ## 16-byte Spill
-; AVX512-NEXT:    vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
-; AVX512-NEXT:    ## xmm0 = mem[1,0]
-; AVX512-NEXT:    callq _roundeven
-; AVX512-NEXT:    vmovapd (%rsp), %xmm1 ## 16-byte Reload
-; AVX512-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX512-NEXT:    vinsertf128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 ## 16-byte Folded Reload
-; AVX512-NEXT:    vmovupd %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 32-byte Spill
-; AVX512-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 ## 64-byte Reload
-; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX512-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; AVX512-NEXT:    vzeroupper
-; AVX512-NEXT:    callq _roundeven
-; AVX512-NEXT:    vmovaps %xmm0, (%rsp) ## 16-byte Spill
-; AVX512-NEXT:    vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
-; AVX512-NEXT:    ## xmm0 = mem[1,0]
-; AVX512-NEXT:    callq _roundeven
-; AVX512-NEXT:    vmovapd (%rsp), %xmm1 ## 16-byte Reload
-; AVX512-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX512-NEXT:    vmovapd %xmm0, (%rsp) ## 16-byte Spill
-; AVX512-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 ## 64-byte Reload
-; AVX512-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $zmm0
-; AVX512-NEXT:    vzeroupper
-; AVX512-NEXT:    callq _roundeven
-; AVX512-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
-; AVX512-NEXT:    vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
-; AVX512-NEXT:    ## xmm0 = mem[1,0]
-; AVX512-NEXT:    callq _roundeven
-; AVX512-NEXT:    vmovapd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
-; AVX512-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX512-NEXT:    vinsertf128 $1, (%rsp), %ymm0, %ymm0 ## 16-byte Folded Reload
-; AVX512-NEXT:    vinsertf64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 ## 32-byte Folded Reload
-; AVX512-NEXT:    addq $184, %rsp
+; AVX512-NEXT:    vrndscalepd $8, %zmm0, %zmm0
 ; AVX512-NEXT:    retq
   %a = call <8 x double> @llvm.roundeven.v8f64(<8 x double> %x)
   ret <8 x double> %a
diff --git a/llvm/test/CodeGen/X86/fp-strict-scalar-round.ll b/llvm/test/CodeGen/X86/fp-strict-scalar-round.ll
index f5a6af9c4d657..9ca76295fa0a3 100644
--- a/llvm/test/CodeGen/X86/fp-strict-scalar-round.ll
+++ b/llvm/test/CodeGen/X86/fp-strict-scalar-round.ll
@@ -591,19 +591,17 @@ define float @froundeven32(float %f) #0 {
 ; SSE41-X86-NEXT:    pushl %eax
 ; SSE41-X86-NEXT:    .cfi_def_cfa_offset 8
 ; SSE41-X86-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE41-X86-NEXT:    roundss $8, %xmm0, %xmm0
 ; SSE41-X86-NEXT:    movss %xmm0, (%esp)
-; SSE41-X86-NEXT:    calll roundevenf
+; SSE41-X86-NEXT:    flds (%esp)
+; SSE41-X86-NEXT:    wait
 ; SSE41-X86-NEXT:    popl %eax
 ; SSE41-X86-NEXT:    .cfi_def_cfa_offset 4
 ; SSE41-X86-NEXT:    retl
 ;
 ; SSE41-X64-LABEL: froundeven32:
 ; SSE41-X64:       # %bb.0:
-; SSE41-X64-NEXT:    pushq %rax
-; SSE41-X64-NEXT:    .cfi_def_cfa_offset 16
-; SSE41-X64-NEXT:    callq roundevenf
-; SSE41-X64-NEXT:    popq %rax
-; SSE41-X64-NEXT:    .cfi_def_cfa_offset 8
+; SSE41-X64-NEXT:    roundss $8, %xmm0, %xmm0
 ; SSE41-X64-NEXT:    retq
 ;
 ; AVX-X86-LABEL: froundeven32:
@@ -611,19 +609,17 @@ define float @froundeven32(float %f) #0 {
 ; AVX-X86-NEXT:    pushl %eax
 ; AVX-X86-NEXT:    .cfi_def_cfa_offset 8
 ; AVX-X86-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-X86-NEXT:    vroundss $8, %xmm0, %xmm0, %xmm0
 ; AVX-X86-NEXT:    vmovss %xmm0, (%esp)
-; AVX-X86-NEXT:    calll roundevenf
+; AVX-X86-NEXT:    flds (%esp)
+; AVX-X86-NEXT:    wait
 ; AVX-X86-NEXT:    popl %eax
 ; AVX-X86-NEXT:    .cfi_def_cfa_offset 4
 ; AVX-X86-NEXT:    retl
 ;
 ; AVX-X64-LABEL: froundeven32:
 ; AVX-X64:       # %bb.0:
-; AVX-X64-NEXT:    pushq %rax
-; AVX-X64-NEXT:    .cfi_def_cfa_offset 16
-; AVX-X64-NEXT:    callq roundevenf
-; AVX-X64-NEXT:    popq %rax
-; AVX-X64-NEXT:    .cfi_def_cfa_offset 8
+; AVX-X64-NEXT:    vroundss $8, %xmm0, %xmm0, %xmm0
 ; AVX-X64-NEXT:    retq
   %res = call float @llvm.experimental.constrained.roundeven.f32(
                         float %f, metadata !"fpexcept.strict") #0
@@ -633,42 +629,50 @@ define float @froundeven32(float %f) #0 {
 define double @froundevenf64(double %f) #0 {
 ; SSE41-X86-LABEL: froundevenf64:
 ; SSE41-X86:       # %bb.0:
+; SSE41-X86-NEXT:    pushl %ebp
+; SSE41-X86-NEXT:    .cfi_def_cfa_offset 8
+; SSE41-X86-NEXT:    .cfi_offset %ebp, -8
+; SSE41-X86-NEXT:    movl %esp, %ebp
+; SSE41-X86-NEXT:    .cfi_def_cfa_register %ebp
+; SSE41-X86-NEXT:    andl $-8, %esp
 ; SSE41-X86-NEXT:    subl $8, %esp
-; SSE41-X86-NEXT:    .cfi_def_cfa_offset 12
 ; SSE41-X86-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE41-X86-NEXT:    roundsd $8, %xmm0, %xmm0
 ; SSE41-X86-NEXT:    movsd %xmm0, (%esp)
-; SSE41-X86-NEXT:    calll roundeven
-; SSE41-X86-NEXT:    addl $8, %esp
-; SSE41-X86-NEXT:    .cfi_def_cfa_offset 4
+; SSE41-X86-NEXT:    fldl (%esp)
+; SSE41-X86-NEXT:    wait
+; SSE41-X86-NEXT:    movl %ebp, %esp
+; SSE41-X86-NEXT:    popl %ebp
+; SSE41-X86-NEXT:    .cfi_def_cfa %esp, 4
 ; SSE41-X86-NEXT:    retl
 ;
 ; SSE41-X64-LABEL: froundevenf64:
 ; SSE41-X64:       # %bb.0:
-; SSE41-X64-NEXT:    pushq %rax
-; SSE41-X64-NEXT:    .cfi_def_cfa_offset 16
-; SSE41-X64-NEXT:    callq roundeven
-; SSE41-X64-NEXT:    popq %rax
-; SSE41-X64-NEXT:    .cfi_def_cfa_offset 8
+; SSE41-X64-NEXT:    roundsd $8, %xmm0, %xmm0
 ; SSE41-X64-NEXT:    retq
 ;
 ; AVX-X86-LABEL: froundevenf64:
 ; AVX-X86:       # %bb.0:
+; AVX-X86-NEXT:    pushl %ebp
+; AVX-X86-NEXT:    .cfi_def_cfa_offset 8
+; AVX-X86-NEXT:    .cfi_offset %ebp, -8
+; AVX-X86-NEXT:    movl %esp, %ebp
+; AVX-X86-NEXT:    .cfi_def_cfa_register %ebp
+; AVX-X86-NEXT:    andl $-8, %esp
 ; AVX-X86-NEXT:    subl $8, %esp
-; AVX-X86-NEXT:    .cfi_def_cfa_offset 12
 ; AVX-X86-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-X86-NEXT:    vroundsd $8, %xmm0, %xmm0, %xmm0
 ; AVX-X86-NEXT:    vmovsd %xmm0, (%esp)
-; AVX-X86-NEXT:    calll roundeven
-; AVX-X86-NEXT:    addl $8, %esp
-; AVX-X86-NEXT:    .cfi_def_cfa_offset 4
+; AVX-X86-NEXT:    fldl (%esp)
+; AVX-X86-NEXT:    wait
+; AVX-X86-NEXT:    movl %ebp, %esp
+; AVX-X86-NEXT:    popl %ebp
+; AVX-X86-NEXT:    .cfi_def_cfa %esp, 4
 ; AVX-X86-NEXT:    retl
 ;
 ; AVX-X64-LABEL: froundevenf64:
 ; AVX-X64:       # %bb.0:
-; AVX-X64-NEXT:    pushq %rax
-; AVX-X64-NEXT:    .cfi_def_cfa_offset 16
-; AVX-X64-NEXT:    callq roundeven
-; AVX-X64-NEXT:    popq %rax
-; AVX-X64-NEXT:    .cfi_def_cfa_offset 8
+; AVX-X64-NEXT:    vroundsd $8, %xmm0, %xmm0, %xmm0
 ; AVX-X64-NEXT:    retq
   %res = call double @llvm.experimental.constrained.roundeven.f64(
                         double %f, metadata !"fpexcept.strict") #0

From cd890944ad344b1b8cac58332ab11c9eec6b61e9 Mon Sep 17 00:00:00 2001
From: Hiroshi Yamauchi <yamauchi@google.com>
Date: Wed, 29 Jul 2020 09:13:37 -0700
Subject: [PATCH 0497/1035] [PGO] Remove insignificant function hash values
 from some tests.

This is to avoid the need to update a bunch of test files when the PGO
instrumentation function hashing changes.

Split off of D84782.

Differential Revision: https://reviews.llvm.org/D84865
---
 llvm/test/Transforms/PGOProfile/PR41279.ll    |  8 +++----
 llvm/test/Transforms/PGOProfile/PR41279_2.ll  |  4 ++--
 llvm/test/Transforms/PGOProfile/branch1.ll    |  4 ++--
 llvm/test/Transforms/PGOProfile/branch2.ll    |  6 ++---
 .../Transforms/PGOProfile/criticaledge.ll     | 24 +++++++++----------
 .../Transforms/PGOProfile/instr_entry_bb.ll   |  4 ++--
 llvm/test/Transforms/PGOProfile/landingpad.ll | 14 +++++------
 llvm/test/Transforms/PGOProfile/loop1.ll      |  8 +++----
 llvm/test/Transforms/PGOProfile/loop2.ll      | 12 +++++-----
 .../PGOProfile/memop_size_from_strlen.ll      |  2 +-
 llvm/test/Transforms/PGOProfile/single_bb.ll  |  2 +-
 llvm/test/Transforms/PGOProfile/switch.ll     | 10 ++++----
 12 files changed, 49 insertions(+), 49 deletions(-)

diff --git a/llvm/test/Transforms/PGOProfile/PR41279.ll b/llvm/test/Transforms/PGOProfile/PR41279.ll
index b150353dbaa23..6b5b21fdbc360 100644
--- a/llvm/test/Transforms/PGOProfile/PR41279.ll
+++ b/llvm/test/Transforms/PGOProfile/PR41279.ll
@@ -27,7 +27,7 @@ define void @foo({ i8*, i64 }*, { i8*, i64 }*) personality i32 (...)* @__CxxFram
 6:
   cleanupret from %17 unwind to caller
 ; GEN: 6:
-; GEN:  call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_foo, i32 0, i32 0), i64 60927483247, i32 4, i32 2)
+; GEN:  call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_foo, i32 0, i32 0), i64 {{[0-9]+}}, i32 4, i32 2)
 
 7:
   store i8 1, i8* %3, align 1
@@ -50,13 +50,13 @@ define void @foo({ i8*, i64 }*, { i8*, i64 }*) personality i32 (...)* @__CxxFram
   store i8 0, i8* %3, align 1
   br label %14
 ; GEN: 12:
-; GEN:  call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_foo, i32 0, i32 0), i64 60927483247, i32 4, i32 1)
+; GEN:  call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_foo, i32 0, i32 0), i64 {{[0-9]+}}, i32 4, i32 1)
 
 13:
   call void @f3({ i8*, i64 }* %0, { i8*, i64 }* %1, i64 1)
   br label %14
 ; GEN: 13:
-; GEN:  call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_foo, i32 0, i32 0), i64 60927483247, i32 4, i32 0)
+; GEN:  call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_foo, i32 0, i32 0), i64 {{[0-9]+}}, i32 4, i32 0)
 
 14:
   ret void
@@ -65,7 +65,7 @@ define void @foo({ i8*, i64 }*, { i8*, i64 }*) personality i32 (...)* @__CxxFram
   store i8 0, i8* %3, align 1
   br label %6
 ; GEN: 15:
-; GEN:  call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_foo, i32 0, i32 0), i64 60927483247, i32 4, i32 3)
+; GEN:  call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_foo, i32 0, i32 0), i64 {{[0-9]+}}, i32 4, i32 3)
 
 16:
   %17 = cleanuppad within none []
diff --git a/llvm/test/Transforms/PGOProfile/PR41279_2.ll b/llvm/test/Transforms/PGOProfile/PR41279_2.ll
index 8d5813fac4212..cd20d569ff01d 100644
--- a/llvm/test/Transforms/PGOProfile/PR41279_2.ll
+++ b/llvm/test/Transforms/PGOProfile/PR41279_2.ll
@@ -30,7 +30,7 @@ __except1:
   store i32 %2, i32* %__exception_code, align 4
   br label %__try.cont7
 ;GEN:  _except1:
-;GEN:    call void @llvm.instrprof.increment(i8* getelementptr inbounds ([1 x i8], [1 x i8]* @__profn_f, i32 0, i32 0), i64 62077759478, i32 2, i32 1)
+;GEN:    call void @llvm.instrprof.increment(i8* getelementptr inbounds ([1 x i8], [1 x i8]* @__profn_f, i32 0, i32 0), i64 {{[0-9]+}}, i32 2, i32 1)
 
 invoke.cont:
   br label %__try.cont
@@ -59,7 +59,7 @@ __try.cont7:
 invoke.cont3:
   br label %__try.cont7
 ;GEN: invoke.cont3:
-;GEN:  call void @llvm.instrprof.increment(i8* getelementptr inbounds ([1 x i8], [1 x i8]* @__profn_f, i32 0, i32 0), i64 62077759478, i32 2, i32 0)
+;GEN:  call void @llvm.instrprof.increment(i8* getelementptr inbounds ([1 x i8], [1 x i8]* @__profn_f, i32 0, i32 0), i64 {{[0-9]+}}, i32 2, i32 0)
 
 }
 
diff --git a/llvm/test/Transforms/PGOProfile/branch1.ll b/llvm/test/Transforms/PGOProfile/branch1.ll
index e3eb992286c05..0b6f6862ccd31 100644
--- a/llvm/test/Transforms/PGOProfile/branch1.ll
+++ b/llvm/test/Transforms/PGOProfile/branch1.ll
@@ -32,7 +32,7 @@ define i32 @test_br_1(i32 %i) {
 ; USE-SAME: !prof ![[FUNC_ENTRY_COUNT:[0-9]+]]
 entry:
 ; GEN: entry:
-; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([9 x i8], [9 x i8]* @__profn_test_br_1, i32 0, i32 0), i64 25571299074, i32 2, i32 0)
+; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([9 x i8], [9 x i8]* @__profn_test_br_1, i32 0, i32 0), i64 {{[0-9]+}}, i32 2, i32 0)
   %cmp = icmp sgt i32 %i, 0
   br i1 %cmp, label %if.then, label %if.end
 ; USE: br i1 %cmp, label %if.then, label %if.end
@@ -44,7 +44,7 @@ entry:
 
 if.then:
 ; GEN: if.then:
-; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([9 x i8], [9 x i8]* @__profn_test_br_1, i32 0, i32 0), i64 25571299074, i32 2, i32 1)
+; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([9 x i8], [9 x i8]* @__profn_test_br_1, i32 0, i32 0), i64 {{[0-9]+}}, i32 2, i32 1)
   %add = add nsw i32 %i, 2
   br label %if.end
 
diff --git a/llvm/test/Transforms/PGOProfile/branch2.ll b/llvm/test/Transforms/PGOProfile/branch2.ll
index 154f86236398e..7523c1f1b7d5e 100644
--- a/llvm/test/Transforms/PGOProfile/branch2.ll
+++ b/llvm/test/Transforms/PGOProfile/branch2.ll
@@ -20,7 +20,7 @@ define i32 @test_br_2(i32 %i) {
 entry:
 ; GEN: entry:
 ; NOTENTRY-NOT: llvm.instrprof.increment
-; ENTRY: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([9 x i8], [9 x i8]* @__profn_test_br_2, i32 0, i32 0), i64 29667547796, i32 2, i32 0)
+; ENTRY: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([9 x i8], [9 x i8]* @__profn_test_br_2, i32 0, i32 0), i64 {{[0-9]+}}, i32 2, i32 0)
   %cmp = icmp sgt i32 %i, 0
   br i1 %cmp, label %if.then, label %if.else
 ; USE: br i1 %cmp, label %if.then, label %if.else
@@ -29,14 +29,14 @@ entry:
 
 if.then:
 ; GEN: if.then:
-; NOTENTRY: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([9 x i8], [9 x i8]* @__profn_test_br_2, i32 0, i32 0), i64 29667547796, i32 2, i32 0)
+; NOTENTRY: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([9 x i8], [9 x i8]* @__profn_test_br_2, i32 0, i32 0), i64 {{[0-9]+}}, i32 2, i32 0)
 ; ENTRY-NOT: llvm.instrprof.increment
   %add = add nsw i32 %i, 2
   br label %if.end
 
 if.else:
 ; GEN: if.else:
-; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([9 x i8], [9 x i8]* @__profn_test_br_2, i32 0, i32 0), i64 29667547796, i32 2, i32 1)
+; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([9 x i8], [9 x i8]* @__profn_test_br_2, i32 0, i32 0), i64 {{[0-9]+}}, i32 2, i32 1)
   %sub = sub nsw i32 %i, 2
   br label %if.end
 
diff --git a/llvm/test/Transforms/PGOProfile/criticaledge.ll b/llvm/test/Transforms/PGOProfile/criticaledge.ll
index 49ef155ada38b..a2ed572550750 100644
--- a/llvm/test/Transforms/PGOProfile/criticaledge.ll
+++ b/llvm/test/Transforms/PGOProfile/criticaledge.ll
@@ -21,7 +21,7 @@ define i32 @test_criticalEdge(i32 %i, i32 %j) {
 entry:
 ; CHECK: entry:
 ; NOTENTRY-NOT: call void @llvm.instrprof.increment
-; ENTRY:   call void @llvm.instrprof.increment(i8* getelementptr inbounds ([17 x i8], [17 x i8]* @__profn_test_criticalEdge, i32 0, i32 0), i64 82323253069, i32 8, i32 0)
+; ENTRY:   call void @llvm.instrprof.increment(i8* getelementptr inbounds ([17 x i8], [17 x i8]* @__profn_test_criticalEdge, i32 0, i32 0), i64 {{[0-9]+}}, i32 8, i32 0)
   switch i32 %i, label %sw.default [
     i32 1, label %sw.bb
     i32 2, label %sw.bb1
@@ -35,24 +35,24 @@ entry:
 ; USE-SAME: !prof ![[BW_SWITCH:[0-9]+]]
 
 ; CHECK: entry.sw.bb2_crit_edge1:
-; NOTENTRY:   call void @llvm.instrprof.increment(i8* getelementptr inbounds ([17 x i8], [17 x i8]* @__profn_test_criticalEdge, i32 0, i32 0), i64 82323253069, i32 8, i32 1)
-; ENTRY: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([17 x i8], [17 x i8]* @__profn_test_criticalEdge, i32 0, i32 0), i64 82323253069, i32 8, i32 2)
+; NOTENTRY:   call void @llvm.instrprof.increment(i8* getelementptr inbounds ([17 x i8], [17 x i8]* @__profn_test_criticalEdge, i32 0, i32 0), i64 {{[0-9]+}}, i32 8, i32 1)
+; ENTRY: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([17 x i8], [17 x i8]* @__profn_test_criticalEdge, i32 0, i32 0), i64 {{[0-9]+}}, i32 8, i32 2)
 ; CHECK:   br label %sw.bb2
 
 ; CHECK: entry.sw.bb2_crit_edge:
-; NOTENTRY:   call void @llvm.instrprof.increment(i8* getelementptr inbounds ([17 x i8], [17 x i8]* @__profn_test_criticalEdge, i32 0, i32 0), i64 82323253069, i32 8, i32 0)
-; TENTRY:   call void @llvm.instrprof.increment(i8* getelementptr inbounds ([17 x i8], [17 x i8]* @__profn_test_criticalEdge, i32 0, i32 0), i64 82323253069, i32 8, i32 1)
+; NOTENTRY:   call void @llvm.instrprof.increment(i8* getelementptr inbounds ([17 x i8], [17 x i8]* @__profn_test_criticalEdge, i32 0, i32 0), i64 {{[0-9]+}}, i32 8, i32 0)
+; TENTRY:   call void @llvm.instrprof.increment(i8* getelementptr inbounds ([17 x i8], [17 x i8]* @__profn_test_criticalEdge, i32 0, i32 0), i64 {{[0-9]+}}, i32 8, i32 1)
 ; CHECK:   br label %sw.bb2
 
 sw.bb:
 ; GEN: sw.bb:
-; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([17 x i8], [17 x i8]* @__profn_test_criticalEdge, i32 0, i32 0), i64 82323253069, i32 8, i32 5)
+; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([17 x i8], [17 x i8]* @__profn_test_criticalEdge, i32 0, i32 0), i64 {{[0-9]+}}, i32 8, i32 5)
   %call = call i32 @bar(i32 2)
   br label %sw.epilog
 
 sw.bb1:
 ; GEN: sw.bb1:
-; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([17 x i8], [17 x i8]* @__profn_test_criticalEdge, i32 0, i32 0), i64 82323253069, i32 8, i32 4)
+; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([17 x i8], [17 x i8]* @__profn_test_criticalEdge, i32 0, i32 0), i64 {{[0-9]+}}, i32 8, i32 4)
   %call2 = call i32 @bar(i32 1024)
   br label %sw.epilog
 
@@ -66,14 +66,14 @@ sw.bb2:
 
 if.then:
 ; GEN: if.then:
-; NOTENTRY: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([17 x i8], [17 x i8]* @__profn_test_criticalEdge, i32 0, i32 0), i64 82323253069, i32 8, i32 2)
+; NOTENTRY: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([17 x i8], [17 x i8]* @__profn_test_criticalEdge, i32 0, i32 0), i64 {{[0-9]+}}, i32 8, i32 2)
 ; ENTRY-NOT: call void @llvm.instrprof.increment
   %call4 = call i32 @bar(i32 4)
   br label %return
 
 if.end:
 ; GEN: if.end:
-; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([17 x i8], [17 x i8]* @__profn_test_criticalEdge, i32 0, i32 0), i64 82323253069, i32 8, i32 3)
+; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([17 x i8], [17 x i8]* @__profn_test_criticalEdge, i32 0, i32 0), i64 {{[0-9]+}}, i32 8, i32 3)
   %call5 = call i32 @bar(i32 8)
   br label %sw.epilog
 
@@ -88,13 +88,13 @@ sw.default:
 
 if.then8:
 ; GEN: if.then8:
-; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([17 x i8], [17 x i8]* @__profn_test_criticalEdge, i32 0, i32 0), i64 82323253069, i32 8, i32 7)
+; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([17 x i8], [17 x i8]* @__profn_test_criticalEdge, i32 0, i32 0), i64 {{[0-9]+}}, i32 8, i32 7)
   %add = add nsw i32 %call6, 10
   br label %if.end9
 
 if.end9:
 ; GEN: if.end9:
-; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([17 x i8], [17 x i8]* @__profn_test_criticalEdge, i32 0, i32 0), i64 82323253069, i32 8, i32 6)
+; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([17 x i8], [17 x i8]* @__profn_test_criticalEdge, i32 0, i32 0), i64 {{[0-9]+}}, i32 8, i32 6)
   %res.0 = phi i32 [ %add, %if.then8 ], [ %call6, %sw.default ]
   br label %sw.epilog
 
@@ -113,7 +113,7 @@ return:
 
 define internal i32 @bar(i32 %i) {
 entry:
-; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @__profn__stdin__bar, i32 0, i32 0), i64 12884901887, i32 1, i32 0)
+; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @__profn__stdin__bar, i32 0, i32 0), i64 {{[0-9]+}}, i32 1, i32 0)
   ret i32 %i
 }
 
diff --git a/llvm/test/Transforms/PGOProfile/instr_entry_bb.ll b/llvm/test/Transforms/PGOProfile/instr_entry_bb.ll
index 4ce701a767ccf..80e30a3c576a6 100644
--- a/llvm/test/Transforms/PGOProfile/instr_entry_bb.ll
+++ b/llvm/test/Transforms/PGOProfile/instr_entry_bb.ll
@@ -16,7 +16,7 @@ target triple = "x86_64-unknown-linux-gnu"
 define i32 @test_br_2(i32 %i) {
 entry:
 ; GEN: entry:
-; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([9 x i8], [9 x i8]* @__profn_test_br_2, i32 0, i32 0), i64 29667547796, i32 2, i32 0)
+; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([9 x i8], [9 x i8]* @__profn_test_br_2, i32 0, i32 0), i64 {{[0-9]+}}, i32 2, i32 0)
 ; GENA: entry:
 ; GENA: %{{[0-9+]}} = atomicrmw add i64* getelementptr inbounds ([2 x i64], [2 x i64]* @__profc_test_br_2, i64 0, i64 0), i64 1 monotonic
 ; USE: br i1 %cmp, label %if.then, label %if.else
@@ -33,7 +33,7 @@ if.then:
 
 if.else:
 ; GEN: if.else:
-; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([9 x i8], [9 x i8]* @__profn_test_br_2, i32 0, i32 0), i64 29667547796, i32 2, i32 1)
+; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([9 x i8], [9 x i8]* @__profn_test_br_2, i32 0, i32 0), i64 {{[0-9]+}}, i32 2, i32 1)
 ; GENA: if.else:
 ; GENA:  %pgocount = load i64, i64* getelementptr inbounds ([2 x i64], [2 x i64]* @__profc_test_br_2, i64 0, i64 1), align 8
 ; GENA:  [[V:%[0-9]*]] = add i64 %pgocount, 1
diff --git a/llvm/test/Transforms/PGOProfile/landingpad.ll b/llvm/test/Transforms/PGOProfile/landingpad.ll
index 998cda2488aaa..a0ca799fa8a43 100644
--- a/llvm/test/Transforms/PGOProfile/landingpad.ll
+++ b/llvm/test/Transforms/PGOProfile/landingpad.ll
@@ -22,7 +22,7 @@ target triple = "x86_64-unknown-linux-gnu"
 define i32 @bar(i32 %i) {
 entry:
 ; GEN: entry:
-; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_bar, i32 0, i32 0), i64 24868915205, i32 2, i32 0)
+; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_bar, i32 0, i32 0), i64 {{[0-9]+}}, i32 2, i32 0)
   %rem = srem i32 %i, 3
   %tobool = icmp ne i32 %rem, 0
   br i1 %tobool, label %if.then, label %if.end
@@ -31,7 +31,7 @@ entry:
 
 if.then:
 ; GEN: if.then:
-; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_bar, i32 0, i32 0), i64 24868915205, i32 2, i32 1)
+; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_bar, i32 0, i32 0), i64 {{[0-9]+}}, i32 2, i32 1)
   %exception = call i8* @__cxa_allocate_exception(i64 4)
   %tmp = bitcast i8* %exception to i32*
   store i32 %i, i32* %tmp, align 16
@@ -53,7 +53,7 @@ define i32 @foo(i32 %i) personality i8* bitcast (i32 (...)* @__gxx_personality_v
 entry:
 ; GEN: entry:
 ; NOTENTRY-NOT: call void @llvm.instrprof.increment
-; ENTRY: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_foo, i32 0, i32 0), i64 59130013419, i32 4, i32 0)
+; ENTRY: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_foo, i32 0, i32 0), i64 {{[0-9]+}}, i32 4, i32 0)
   %rem = srem i32 %i, 2
   %tobool = icmp ne i32 %rem, 0
   br i1 %tobool, label %if.then, label %if.end
@@ -69,7 +69,7 @@ if.then:
 
 invoke.cont:
 ; GEN: invoke.cont:
-; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_foo, i32 0, i32 0), i64 59130013419, i32 4, i32 1)
+; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_foo, i32 0, i32 0), i64 {{[0-9]+}}, i32 4, i32 1)
   br label %if.end
 
 lpad:
@@ -92,7 +92,7 @@ catch.dispatch:
 
 catch:
 ; GEN: catch:
-; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_foo, i32 0, i32 0), i64 59130013419, i32 4, i32 2)
+; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_foo, i32 0, i32 0), i64 {{[0-9]+}}, i32 4, i32 2)
   %tmp4 = call i8* @__cxa_begin_catch(i8* %tmp1)
   %tmp5 = bitcast i8* %tmp4 to i32*
   %tmp6 = load i32, i32* %tmp5, align 4
@@ -109,7 +109,7 @@ try.cont:
 
 if.end:
 ; GEN: if.end:
-; NOTENTRY: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_foo, i32 0, i32 0), i64 59130013419, i32 4, i32 0)
+; NOTENTRY: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_foo, i32 0, i32 0), i64 {{[0-9]+}}, i32 4, i32 0)
 ; ENTRY-NOT: call void @llvm.instrprof.increment
   %tmp8 = load i32, i32* @val, align 4
   %add = add nsw i32 %tmp8, %i
@@ -118,7 +118,7 @@ if.end:
 
 eh.resume:
 ; GEN: eh.resume:
-; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_foo, i32 0, i32 0), i64 59130013419, i32 4, i32 3)
+; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_foo, i32 0, i32 0), i64 {{[0-9]+}}, i32 4, i32 3)
   %lpad.val = insertvalue { i8*, i32 } undef, i8* %tmp1, 0
   %lpad.val3 = insertvalue { i8*, i32 } %lpad.val, i32 %tmp2, 1
   resume { i8*, i32 } %lpad.val3
diff --git a/llvm/test/Transforms/PGOProfile/loop1.ll b/llvm/test/Transforms/PGOProfile/loop1.ll
index 94c867445fbb3..34855750813cc 100644
--- a/llvm/test/Transforms/PGOProfile/loop1.ll
+++ b/llvm/test/Transforms/PGOProfile/loop1.ll
@@ -19,8 +19,8 @@ target triple = "x86_64-unknown-linux-gnu"
 define i32 @test_simple_for(i32 %n) {
 entry:
 ; GEN: entry:
-; NOTENTRY: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([15 x i8], [15 x i8]* @__profn_test_simple_for, i32 0, i32 0), i64 34137660316, i32 2, i32 1)
-; ENTRY: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([15 x i8], [15 x i8]* @__profn_test_simple_for, i32 0, i32 0), i64 34137660316, i32 2, i32 0)
+; NOTENTRY: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([15 x i8], [15 x i8]* @__profn_test_simple_for, i32 0, i32 0), i64 {{[0-9]+}}, i32 2, i32 1)
+; ENTRY: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([15 x i8], [15 x i8]* @__profn_test_simple_for, i32 0, i32 0), i64 {{[0-9]+}}, i32 2, i32 0)
   br label %for.cond
 
 for.cond:
@@ -42,8 +42,8 @@ for.body:
 
 for.inc:
 ; GEN: for.inc:
-; NOTENTRY: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([15 x i8], [15 x i8]* @__profn_test_simple_for, i32 0, i32 0), i64 34137660316, i32 2, i32 0)
-; ENTRY: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([15 x i8], [15 x i8]* @__profn_test_simple_for, i32 0, i32 0), i64 34137660316, i32 2, i32 1)
+; NOTENTRY: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([15 x i8], [15 x i8]* @__profn_test_simple_for, i32 0, i32 0), i64 {{[0-9]+}}, i32 2, i32 0)
+; ENTRY: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([15 x i8], [15 x i8]* @__profn_test_simple_for, i32 0, i32 0), i64 {{[0-9]+}}, i32 2, i32 1)
   %inc1 = add nsw i32 %i, 1
   br label %for.cond
 
diff --git a/llvm/test/Transforms/PGOProfile/loop2.ll b/llvm/test/Transforms/PGOProfile/loop2.ll
index 90cb2e3e49d98..7e993db54ef7b 100644
--- a/llvm/test/Transforms/PGOProfile/loop2.ll
+++ b/llvm/test/Transforms/PGOProfile/loop2.ll
@@ -18,8 +18,8 @@ target triple = "x86_64-unknown-linux-gnu"
 define i32 @test_nested_for(i32 %r, i32 %s) {
 entry:
 ; GEN: entry:
-; NOTENTRY: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([15 x i8], [15 x i8]* @__profn_test_nested_for, i32 0, i32 0), i64 53929068288, i32 3, i32 2)
-; ENTRY: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([15 x i8], [15 x i8]* @__profn_test_nested_for, i32 0, i32 0), i64 53929068288, i32 3, i32 0)
+; NOTENTRY: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([15 x i8], [15 x i8]* @__profn_test_nested_for, i32 0, i32 0), i64 {{[0-9]+}}, i32 3, i32 2)
+; ENTRY: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([15 x i8], [15 x i8]* @__profn_test_nested_for, i32 0, i32 0), i64 {{[0-9]+}}, i32 3, i32 0)
   br label %for.cond.outer
 
 for.cond.outer:
@@ -55,8 +55,8 @@ for.body.inner:
 
 for.inc.inner:
 ; GEN: for.inc.inner:
-; NOTENTRY: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([15 x i8], [15 x i8]* @__profn_test_nested_for, i32 0, i32 0), i64 53929068288, i32 3, i32 0)
-; ENTRY: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([15 x i8], [15 x i8]* @__profn_test_nested_for, i32 0, i32 0), i64 53929068288, i32 3, i32 1)
+; NOTENTRY: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([15 x i8], [15 x i8]* @__profn_test_nested_for, i32 0, i32 0), i64 {{[0-9]+}}, i32 3, i32 0)
+; ENTRY: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([15 x i8], [15 x i8]* @__profn_test_nested_for, i32 0, i32 0), i64 {{[0-9]+}}, i32 3, i32 1)
   %inc.1 = add nsw i32 %j.0, 1
   br label %for.cond.inner
 
@@ -66,8 +66,8 @@ for.end.inner:
 
 for.inc.outer:
 ; GEN: for.inc.outer:
-; NOTENTRY: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([15 x i8], [15 x i8]* @__profn_test_nested_for, i32 0, i32 0), i64 53929068288, i32 3, i32 1)
-; ENTRY: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([15 x i8], [15 x i8]* @__profn_test_nested_for, i32 0, i32 0), i64 53929068288, i32 3, i32 2)
+; NOTENTRY: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([15 x i8], [15 x i8]* @__profn_test_nested_for, i32 0, i32 0), i64 {{[0-9]+}}, i32 3, i32 1)
+; ENTRY: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([15 x i8], [15 x i8]* @__profn_test_nested_for, i32 0, i32 0), i64 {{[0-9]+}}, i32 3, i32 2)
   %inc.2 = add nsw i32 %i.0, 1
   br label %for.cond.outer
 
diff --git a/llvm/test/Transforms/PGOProfile/memop_size_from_strlen.ll b/llvm/test/Transforms/PGOProfile/memop_size_from_strlen.ll
index 1d8509a0b0ba7..601aeaf899860 100644
--- a/llvm/test/Transforms/PGOProfile/memop_size_from_strlen.ll
+++ b/llvm/test/Transforms/PGOProfile/memop_size_from_strlen.ll
@@ -5,7 +5,7 @@ declare i32 @strlen(i8* nocapture)
 
 ; CHECK_LABEL: test
 ; CHECK: %1 = zext i32 %c to i64
-; CHECK:  call void @llvm.instrprof.value.profile(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @__profn_test, i32 0, i32 0), i64 12884901887, i64 %1, i32 1, i32 0)
+; CHECK:  call void @llvm.instrprof.value.profile(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @__profn_test, i32 0, i32 0), i64 {{[0-9]+}}, i64 %1, i32 1, i32 0)
 
 define void @test(i8* %a, i8* %p) {
   %c = call i32 @strlen(i8* %p)
diff --git a/llvm/test/Transforms/PGOProfile/single_bb.ll b/llvm/test/Transforms/PGOProfile/single_bb.ll
index 646f798a6ff89..ff296fce54086 100644
--- a/llvm/test/Transforms/PGOProfile/single_bb.ll
+++ b/llvm/test/Transforms/PGOProfile/single_bb.ll
@@ -10,6 +10,6 @@ target triple = "x86_64-unknown-linux-gnu"
 define i32 @single_bb() {
 entry:
 ; GEN: entry:
-; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([9 x i8], [9 x i8]* @__profn_single_bb, i32 0, i32 0), i64 12884901887, i32 1, i32 0)
+; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([9 x i8], [9 x i8]* @__profn_single_bb, i32 0, i32 0), i64 {{[0-9]+}}, i32 1, i32 0)
   ret i32 0
 }
diff --git a/llvm/test/Transforms/PGOProfile/switch.ll b/llvm/test/Transforms/PGOProfile/switch.ll
index 983470408a48d..ca7c5ba6df744 100644
--- a/llvm/test/Transforms/PGOProfile/switch.ll
+++ b/llvm/test/Transforms/PGOProfile/switch.ll
@@ -19,7 +19,7 @@ define void @test_switch(i32 %i) {
 entry:
 ; GEN: entry:
 ; NOTENTRY-NOT: call void @llvm.instrprof.increment
-; ENTRY: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @__profn_test_switch, i32 0, i32 0), i64 46200943743, i32 4, i32 0)
+; ENTRY: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @__profn_test_switch, i32 0, i32 0), i64 {{[0-9]+}}, i32 4, i32 0)
   switch i32 %i, label %sw.default [
     i32 1, label %sw.bb
     i32 2, label %sw.bb1
@@ -31,23 +31,23 @@ entry:
 
 sw.bb:
 ; GEN: sw.bb:
-; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @__profn_test_switch, i32 0, i32 0), i64 46200943743, i32 4, i32 2)
+; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @__profn_test_switch, i32 0, i32 0), i64 {{[0-9]+}}, i32 4, i32 2)
   br label %sw.epilog
 
 sw.bb1:
 ; GEN: sw.bb1:
-; NOTENTRY: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @__profn_test_switch, i32 0, i32 0), i64 46200943743, i32 4, i32 0)
+; NOTENTRY: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @__profn_test_switch, i32 0, i32 0), i64 {{[0-9]+}}, i32 4, i32 0)
 ; ENTRY-NOT: call void @llvm.instrprof.increment
   br label %sw.epilog
 
 sw.bb2:
 ; GEN: sw.bb2:
-; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @__profn_test_switch, i32 0, i32 0), i64 46200943743, i32 4, i32 1)
+; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @__profn_test_switch, i32 0, i32 0), i64 {{[0-9]+}}, i32 4, i32 1)
   br label %sw.epilog
 
 sw.default:
 ; GEN: sw.default:
-; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @__profn_test_switch, i32 0, i32 0), i64 46200943743, i32 4, i32 3)
+; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @__profn_test_switch, i32 0, i32 0), i64 {{[0-9]+}}, i32 4, i32 3)
   br label %sw.epilog
 
 sw.epilog:

From e7196bdf81251a4df97746fd54b7db749ca99097 Mon Sep 17 00:00:00 2001
From: Arthur Eubanks <aeubanks@google.com>
Date: Tue, 28 Jul 2020 14:29:13 -0700
Subject: [PATCH 0498/1035] [Scudo][CMake] Add -fno-lto to Scudo libraries

-fno-lto is in SANITIZER_COMMON_CFLAGS but not here.
Don't use SANITIZER_COMMON_CFLAGS because of performance issues.
See https://bugs.llvm.org/show_bug.cgi?id=46838.

Fixes
$ ninja TScudoCUnitTest-i386-Test
on an LLVM build with -DLLVM_ENABLE_LTO=Thin.
check-scudo now passes.

Reviewed By: cryptoad

Differential Revision: https://reviews.llvm.org/D84805
---
 compiler-rt/lib/scudo/standalone/CMakeLists.txt | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/compiler-rt/lib/scudo/standalone/CMakeLists.txt b/compiler-rt/lib/scudo/standalone/CMakeLists.txt
index bdaeb569efdd1..dfae6dde5e4d0 100644
--- a/compiler-rt/lib/scudo/standalone/CMakeLists.txt
+++ b/compiler-rt/lib/scudo/standalone/CMakeLists.txt
@@ -19,6 +19,9 @@ append_list_if(COMPILER_RT_HAS_FFREESTANDING_FLAG -ffreestanding SCUDO_CFLAGS)
 
 append_list_if(COMPILER_RT_HAS_FVISIBILITY_HIDDEN_FLAG -fvisibility=hidden SCUDO_CFLAGS)
 
+# FIXME: find cleaner way to agree with GWPAsan flags
+append_list_if(COMPILER_RT_HAS_FNO_LTO_FLAG -fno-lto SCUDO_CFLAGS)
+
 if (COMPILER_RT_HAS_GWP_ASAN)
   append_list_if(COMPILER_RT_HAS_OMIT_FRAME_POINTER_FLAG -fno-omit-frame-pointer
                 SCUDO_CFLAGS)

From 4add853647b358d9bb7f162682e806725888848b Mon Sep 17 00:00:00 2001
From: Jonas Devlieghere <jonas@devlieghere.com>
Date: Wed, 29 Jul 2020 08:59:57 -0700
Subject: [PATCH 0499/1035] [lldb] Improve platform handling in
 CreateTargetInternal

Currently, `target create` has no --platform option. However,
TargetList::CreateTargetInternal which is called under the hood, will
return an error when either no platform or multiple matching platforms
are found, saying that a platform should be specified with --platform.

This patch adds the platform option, but that doesn't solve either of
these errors.

 - If more than one platform matches, specifying the platform isn't
   going to fix that. The current code will only look at the
   architecture instead. I've updated the error message to ask the user
   to specify an architecture.

 - If no architecture is found, specifying a new one via platform isn't
   going to change that either because we already try to find one that
   matches the given architecture.

Differential revision: https://reviews.llvm.org/D84809
---
 lldb/source/Commands/CommandObjectTarget.cpp  |   7 +-
 lldb/source/Target/TargetList.cpp             | 229 +++++++++---------
 .../target/basic/TestTargetCommand.py         |  27 +++
 .../test/API/commands/target/basic/bogus.yaml | 194 +++++++++++++++
 4 files changed, 335 insertions(+), 122 deletions(-)
 create mode 100644 lldb/test/API/commands/target/basic/bogus.yaml

diff --git a/lldb/source/Commands/CommandObjectTarget.cpp b/lldb/source/Commands/CommandObjectTarget.cpp
index e975d99ee0246..84b7c988353a0 100644
--- a/lldb/source/Commands/CommandObjectTarget.cpp
+++ b/lldb/source/Commands/CommandObjectTarget.cpp
@@ -23,6 +23,7 @@
 #include "lldb/Interpreter/OptionGroupBoolean.h"
 #include "lldb/Interpreter/OptionGroupFile.h"
 #include "lldb/Interpreter/OptionGroupFormat.h"
+#include "lldb/Interpreter/OptionGroupPlatform.h"
 #include "lldb/Interpreter/OptionGroupString.h"
 #include "lldb/Interpreter/OptionGroupUInt64.h"
 #include "lldb/Interpreter/OptionGroupUUID.h"
@@ -214,6 +215,7 @@ class CommandObjectTargetCreate : public CommandObjectParsed {
             "Create a target using the argument as the main executable.",
             nullptr),
         m_option_group(), m_arch_option(),
+        m_platform_options(true), // Include the --platform option.
         m_core_file(LLDB_OPT_SET_1, false, "core", 'c', 0, eArgTypeFilename,
                     "Fullpath to a core file to use for this target."),
         m_symbol_file(LLDB_OPT_SET_1, false, "symfile", 's', 0,
@@ -240,6 +242,7 @@ class CommandObjectTargetCreate : public CommandObjectParsed {
     m_arguments.push_back(arg);
 
     m_option_group.Append(&m_arch_option, LLDB_OPT_SET_ALL, LLDB_OPT_SET_1);
+    m_option_group.Append(&m_platform_options, LLDB_OPT_SET_ALL, 1);
     m_option_group.Append(&m_core_file, LLDB_OPT_SET_ALL, LLDB_OPT_SET_1);
     m_option_group.Append(&m_symbol_file, LLDB_OPT_SET_ALL, LLDB_OPT_SET_1);
     m_option_group.Append(&m_remote_file, LLDB_OPT_SET_ALL, LLDB_OPT_SET_1);
@@ -311,7 +314,8 @@ class CommandObjectTargetCreate : public CommandObjectParsed {
       llvm::StringRef arch_cstr = m_arch_option.GetArchitectureName();
       Status error(debugger.GetTargetList().CreateTarget(
           debugger, file_path, arch_cstr,
-          m_add_dependents.m_load_dependent_files, nullptr, target_sp));
+          m_add_dependents.m_load_dependent_files, &m_platform_options,
+          target_sp));
 
       if (target_sp) {
         // Only get the platform after we create the target because we might
@@ -442,6 +446,7 @@ class CommandObjectTargetCreate : public CommandObjectParsed {
 private:
   OptionGroupOptions m_option_group;
   OptionGroupArchitecture m_arch_option;
+  OptionGroupPlatform m_platform_options;
   OptionGroupFile m_core_file;
   OptionGroupFile m_symbol_file;
   OptionGroupFile m_remote_file;
diff --git a/lldb/source/Target/TargetList.cpp b/lldb/source/Target/TargetList.cpp
index 3974cb5de4198..75a022d2c4636 100644
--- a/lldb/source/Target/TargetList.cpp
+++ b/lldb/source/Target/TargetList.cpp
@@ -75,55 +75,49 @@ Status TargetList::CreateTargetInternal(
     const OptionGroupPlatform *platform_options, TargetSP &target_sp,
     bool is_dummy_target) {
   Status error;
-  PlatformSP platform_sp;
 
-  // This is purposely left empty unless it is specified by triple_cstr. If not
-  // initialized via triple_cstr, then the currently selected platform will set
-  // the architecture correctly.
+  // Let's start by looking at the selected platform.
+  PlatformSP platform_sp = debugger.GetPlatformList().GetSelectedPlatform();
+
+  // This variable corresponds to the architecture specified by the triple
+  // string. If that string was empty the currently selected platform will
+  // determine the architecture.
   const ArchSpec arch(triple_str);
-  if (!triple_str.empty()) {
-    if (!arch.IsValid()) {
-      error.SetErrorStringWithFormat("invalid triple '%s'",
-                                     triple_str.str().c_str());
-      return error;
-    }
+  if (!triple_str.empty() && !arch.IsValid()) {
+    error.SetErrorStringWithFormat("invalid triple '%s'",
+                                   triple_str.str().c_str());
+    return error;
   }
 
   ArchSpec platform_arch(arch);
 
-  bool prefer_platform_arch = false;
-
-  CommandInterpreter &interpreter = debugger.GetCommandInterpreter();
-
-  // let's see if there is already an existing platform before we go creating
-  // another...
-  platform_sp = debugger.GetPlatformList().GetSelectedPlatform();
-
-  if (platform_options && platform_options->PlatformWasSpecified()) {
-    // Create a new platform if it doesn't match the selected platform
-    if (!platform_options->PlatformMatches(platform_sp)) {
-      const bool select_platform = true;
-      platform_sp = platform_options->CreatePlatformWithOptions(
-          interpreter, arch, select_platform, error, platform_arch);
-      if (!platform_sp)
-        return error;
-    }
+  // Create a new platform if a platform was specified in the platform options
+  // and doesn't match the selected platform.
+  if (platform_options && platform_options->PlatformWasSpecified() &&
+      !platform_options->PlatformMatches(platform_sp)) {
+    const bool select_platform = true;
+    platform_sp = platform_options->CreatePlatformWithOptions(
+        debugger.GetCommandInterpreter(), arch, select_platform, error,
+        platform_arch);
+    if (!platform_sp)
+      return error;
   }
 
+  bool prefer_platform_arch = false;
+
   if (!user_exe_path.empty()) {
-    ModuleSpecList module_specs;
-    ModuleSpec module_spec;
-    module_spec.GetFileSpec().SetFile(user_exe_path, FileSpec::Style::native);
+    ModuleSpec module_spec(FileSpec(user_exe_path, FileSpec::Style::native));
     FileSystem::Instance().Resolve(module_spec.GetFileSpec());
-
     // Resolve the executable in case we are given a path to a application
-    // bundle like a .app bundle on MacOSX
+    // bundle like a .app bundle on MacOSX.
     Host::ResolveExecutableInBundle(module_spec.GetFileSpec());
 
     lldb::offset_t file_offset = 0;
     lldb::offset_t file_size = 0;
+    ModuleSpecList module_specs;
     const size_t num_specs = ObjectFile::GetModuleSpecifications(
         module_spec.GetFileSpec(), file_offset, file_size, module_specs);
+
     if (num_specs > 0) {
       ModuleSpec matching_module_spec;
 
@@ -134,7 +128,7 @@ Status TargetList::CreateTargetInternal(
                     matching_module_spec.GetArchitecture())) {
               // If the OS or vendor weren't specified, then adopt the module's
               // architecture so that the platform matching can be more
-              // accurate
+              // accurate.
               if (!platform_arch.TripleOSWasSpecified() ||
                   !platform_arch.TripleVendorWasSpecified()) {
                 prefer_platform_arch = true;
@@ -155,113 +149,107 @@ Status TargetList::CreateTargetInternal(
               return error;
             }
           } else {
-            // Only one arch and none was specified
+            // Only one arch and none was specified.
             prefer_platform_arch = true;
             platform_arch = matching_module_spec.GetArchitecture();
           }
         }
+      } else if (arch.IsValid()) {
+        // A (valid) architecture was specified.
+        module_spec.GetArchitecture() = arch;
+        if (module_specs.FindMatchingModuleSpec(module_spec,
+                                                matching_module_spec)) {
+          prefer_platform_arch = true;
+          platform_arch = matching_module_spec.GetArchitecture();
+        }
       } else {
-        if (arch.IsValid()) {
-          module_spec.GetArchitecture() = arch;
-          if (module_specs.FindMatchingModuleSpec(module_spec,
-                                                  matching_module_spec)) {
-            prefer_platform_arch = true;
-            platform_arch = matching_module_spec.GetArchitecture();
-          }
-        } else {
-          // No architecture specified, check if there is only one platform for
-          // all of the architectures.
-
-          typedef std::vector<PlatformSP> PlatformList;
-          PlatformList platforms;
-          PlatformSP host_platform_sp = Platform::GetHostPlatform();
-          for (size_t i = 0; i < num_specs; ++i) {
-            ModuleSpec module_spec;
-            if (module_specs.GetModuleSpecAtIndex(i, module_spec)) {
-              // See if there was a selected platform and check that first
-              // since the user may have specified it.
-              if (platform_sp) {
-                if (platform_sp->IsCompatibleArchitecture(
-                        module_spec.GetArchitecture(), false, nullptr)) {
-                  platforms.push_back(platform_sp);
-                  continue;
-                }
-              }
-
-              // Next check the host platform it if wasn't already checked
-              // above
-              if (host_platform_sp &&
-                  (!platform_sp ||
-                   host_platform_sp->GetName() != platform_sp->GetName())) {
-                if (host_platform_sp->IsCompatibleArchitecture(
-                        module_spec.GetArchitecture(), false, nullptr)) {
-                  platforms.push_back(host_platform_sp);
-                  continue;
-                }
+        // No architecture specified, check if there is only one platform for
+        // all of the architectures.
+        PlatformSP host_platform_sp = Platform::GetHostPlatform();
+        std::vector<PlatformSP> platforms;
+        for (size_t i = 0; i < num_specs; ++i) {
+          ModuleSpec module_spec;
+          if (module_specs.GetModuleSpecAtIndex(i, module_spec)) {
+            // First consider the platform specified by the user, if any, and
+            // the selected platform otherwise.
+            if (platform_sp) {
+              if (platform_sp->IsCompatibleArchitecture(
+                      module_spec.GetArchitecture(), false, nullptr)) {
+                platforms.push_back(platform_sp);
+                continue;
               }
+            }
 
-              // Just find a platform that matches the architecture in the
-              // executable file
-              PlatformSP fallback_platform_sp(
-                  Platform::GetPlatformForArchitecture(
-                      module_spec.GetArchitecture(), nullptr));
-              if (fallback_platform_sp) {
-                platforms.push_back(fallback_platform_sp);
+            // Now consider the host platform if it is different from the
+            // specified/selected platform.
+            if (host_platform_sp &&
+                (!platform_sp ||
+                 host_platform_sp->GetName() != platform_sp->GetName())) {
+              if (host_platform_sp->IsCompatibleArchitecture(
+                      module_spec.GetArchitecture(), false, nullptr)) {
+                platforms.push_back(host_platform_sp);
+                continue;
               }
             }
-          }
 
-          Platform *platform_ptr = nullptr;
-          bool more_than_one_platforms = false;
-          for (const auto &the_platform_sp : platforms) {
-            if (platform_ptr) {
-              if (platform_ptr->GetName() != the_platform_sp->GetName()) {
-                more_than_one_platforms = true;
-                platform_ptr = nullptr;
-                break;
-              }
-            } else {
-              platform_ptr = the_platform_sp.get();
+            // Finally find a platform that matches the architecture in the
+            // executable file.
+            PlatformSP fallback_platform_sp(
+                Platform::GetPlatformForArchitecture(
+                    module_spec.GetArchitecture(), nullptr));
+            if (fallback_platform_sp) {
+              platforms.push_back(fallback_platform_sp);
             }
           }
+        }
 
+        Platform *platform_ptr = nullptr;
+        bool more_than_one_platforms = false;
+        for (const auto &the_platform_sp : platforms) {
           if (platform_ptr) {
-            // All platforms for all modules in the executable match, so we can
-            // select this platform
-            platform_sp = platforms.front();
-          } else if (!more_than_one_platforms) {
-            // No platforms claim to support this file
-            error.SetErrorString("No matching platforms found for this file, "
-                                 "specify one with the --platform option");
-            return error;
+            if (platform_ptr->GetName() != the_platform_sp->GetName()) {
+              more_than_one_platforms = true;
+              platform_ptr = nullptr;
+              break;
+            }
           } else {
-            // More than one platform claims to support this file, so the
-            // --platform option must be specified
-            StreamString error_strm;
-            std::set<Platform *> platform_set;
-            error_strm.Printf(
-                "more than one platform supports this executable (");
-            for (const auto &the_platform_sp : platforms) {
-              if (platform_set.find(the_platform_sp.get()) ==
-                  platform_set.end()) {
-                if (!platform_set.empty())
-                  error_strm.PutCString(", ");
-                error_strm.PutCString(the_platform_sp->GetName().GetCString());
-                platform_set.insert(the_platform_sp.get());
-              }
+            platform_ptr = the_platform_sp.get();
+          }
+        }
+
+        if (platform_ptr) {
+          // All platforms for all modules in the executable match, so we can
+          // select this platform.
+          platform_sp = platforms.front();
+        } else if (!more_than_one_platforms) {
+          // No platforms claim to support this file.
+          error.SetErrorString("no matching platforms found for this file");
+          return error;
+        } else {
+          // More than one platform claims to support this file.
+          StreamString error_strm;
+          std::set<Platform *> platform_set;
+          error_strm.Printf(
+              "more than one platform supports this executable (");
+          for (const auto &the_platform_sp : platforms) {
+            if (platform_set.find(the_platform_sp.get()) ==
+                platform_set.end()) {
+              if (!platform_set.empty())
+                error_strm.PutCString(", ");
+              error_strm.PutCString(the_platform_sp->GetName().GetCString());
+              platform_set.insert(the_platform_sp.get());
             }
-            error_strm.Printf(
-                "), use the --platform option to specify a platform");
-            error.SetErrorString(error_strm.GetString());
-            return error;
           }
+          error_strm.Printf("), specify an architecture to disambiguate");
+          error.SetErrorString(error_strm.GetString());
+          return error;
         }
       }
     }
   }
 
   // If we have a valid architecture, make sure the current platform is
-  // compatible with that architecture
+  // compatible with that architecture.
   if (!prefer_platform_arch && arch.IsValid()) {
     if (!platform_sp->IsCompatibleArchitecture(arch, false, &platform_arch)) {
       platform_sp = Platform::GetPlatformForArchitecture(arch, &platform_arch);
@@ -269,8 +257,8 @@ Status TargetList::CreateTargetInternal(
         debugger.GetPlatformList().SetSelectedPlatform(platform_sp);
     }
   } else if (platform_arch.IsValid()) {
-    // if "arch" isn't valid, yet "platform_arch" is, it means we have an
-    // executable file with a single architecture which should be used
+    // If "arch" isn't valid, yet "platform_arch" is, it means we have an
+    // executable file with a single architecture which should be used.
     ArchSpec fixed_platform_arch;
     if (!platform_sp->IsCompatibleArchitecture(platform_arch, false,
                                                &fixed_platform_arch)) {
@@ -284,10 +272,9 @@ Status TargetList::CreateTargetInternal(
   if (!platform_arch.IsValid())
     platform_arch = arch;
 
-  error = TargetList::CreateTargetInternal(
+  return TargetList::CreateTargetInternal(
       debugger, user_exe_path, platform_arch, load_dependent_files, platform_sp,
       target_sp, is_dummy_target);
-  return error;
 }
 
 lldb::TargetSP TargetList::GetDummyTarget(lldb_private::Debugger &debugger) {
diff --git a/lldb/test/API/commands/target/basic/TestTargetCommand.py b/lldb/test/API/commands/target/basic/TestTargetCommand.py
index 83e27e2724642..be6eeb938ab8b 100644
--- a/lldb/test/API/commands/target/basic/TestTargetCommand.py
+++ b/lldb/test/API/commands/target/basic/TestTargetCommand.py
@@ -115,6 +115,33 @@ def do_target_command(self):
 
         self.runCmd("target list")
 
+    @no_debug_info_test
+    def test_target_create_invalid_arch(self):
+        exe = self.getBuildArtifact("a.out")
+        self.expect("target create {} --arch doesntexist".format(exe), error=True,
+                    patterns=["error: invalid triple 'doesntexist'"])
+
+    @no_debug_info_test
+    def test_target_create_platform(self):
+        self.buildB()
+        exe = self.getBuildArtifact("b.out")
+        self.expect("target create {} --platform host".format(exe))
+
+    @no_debug_info_test
+    def test_target_create_unsupported_platform(self):
+        yaml = os.path.join(self.getSourceDir(), "bogus.yaml")
+        exe = self.getBuildArtifact("bogus")
+        self.yaml2obj(yaml, exe)
+        self.expect("target create {}".format(exe), error=True,
+                    patterns=['error: no matching platforms found for this file'])
+
+    @no_debug_info_test
+    def test_target_create_invalid_platform(self):
+        self.buildB()
+        exe = self.getBuildArtifact("b.out")
+        self.expect("target create {} --platform doesntexist".format(exe), error=True,
+                    patterns=['error: unable to find a plug-in for the platform named "doesntexist"'])
+
     def do_target_variable_command(self, exe_name):
         """Exercise 'target variable' command before and after starting the inferior."""
         self.runCmd("file " + self.getBuildArtifact(exe_name),
diff --git a/lldb/test/API/commands/target/basic/bogus.yaml b/lldb/test/API/commands/target/basic/bogus.yaml
new file mode 100644
index 0000000000000..d36695cdedd6a
--- /dev/null
+++ b/lldb/test/API/commands/target/basic/bogus.yaml
@@ -0,0 +1,194 @@
+--- !fat-mach-o
+FatHeader:
+  magic:           0xCAFEBABE
+  nfat_arch:       3
+FatArchs:
+  - cputype:         0x0000000C
+    cpusubtype:      0x00000009
+    offset:          0x0000000000004000
+    size:            200
+    align:           14
+  - cputype:         0x0000000C
+    cpusubtype:      0x0000000B
+    offset:          0x0000000000008000
+    size:            200
+    align:           14
+  - cputype:         0x0100000C
+    cpusubtype:      0x00000000
+    offset:          0x000000000000C000
+    size:            332
+    align:           14
+Slices:
+  - !mach-o
+    FileHeader:
+      magic:           0xFEEDFACE
+      # Bogus
+      cputype:         0x00000003
+      cpusubtype:      0x00000009
+      filetype:        0x00000001
+      ncmds:           4
+      sizeofcmds:      112
+      flags:           0x00002000
+    LoadCommands:
+      - cmd:             LC_SEGMENT
+        cmdsize:         56
+        segname:         ''
+        vmaddr:          0
+        vmsize:          0
+        fileoff:         0
+        filesize:        0
+        maxprot:         7
+        initprot:        7
+        nsects:          0
+        flags:           0
+      - cmd:             LC_SYMTAB
+        cmdsize:         24
+        symoff:          172
+        nsyms:           1
+        stroff:          184
+        strsize:         16
+      - cmd:             LC_VERSION_MIN_IPHONEOS
+        cmdsize:         16
+        version:         327680
+        sdk:             0
+      - cmd:             LC_DATA_IN_CODE
+        cmdsize:         16
+        dataoff:         172
+        datasize:        0
+    LinkEditData:
+      NameList:
+        - n_strx:          4
+          n_type:          0x01
+          n_sect:          0
+          n_desc:          512
+          n_value:         4
+      StringTable:
+        - ''
+        - ''
+        - ''
+        - ''
+        - _armv7_var
+        - ''
+  - !mach-o
+    FileHeader:
+      magic:           0xFEEDFACE
+      # Bogus
+      cputype:         0x00000002
+      cpusubtype:      0x0000000B
+      filetype:        0x00000001
+      ncmds:           4
+      sizeofcmds:      112
+      flags:           0x00002000
+    LoadCommands:
+      - cmd:             LC_SEGMENT
+        cmdsize:         56
+        segname:         ''
+        vmaddr:          0
+        vmsize:          0
+        fileoff:         0
+        filesize:        0
+        maxprot:         7
+        initprot:        7
+        nsects:          0
+        flags:           0
+      - cmd:             LC_SYMTAB
+        cmdsize:         24
+        symoff:          172
+        nsyms:           1
+        stroff:          184
+        strsize:         16
+      - cmd:             LC_VERSION_MIN_IPHONEOS
+        cmdsize:         16
+        version:         327680
+        sdk:             0
+      - cmd:             LC_DATA_IN_CODE
+        cmdsize:         16
+        dataoff:         172
+        datasize:        0
+    LinkEditData:
+      NameList:
+        - n_strx:          4
+          n_type:          0x01
+          n_sect:          0
+          n_desc:          512
+          n_value:         4
+      StringTable:
+        - ''
+        - ''
+        - ''
+        - ''
+        - _armv7s_var
+  - !mach-o
+    FileHeader:
+      magic:           0xFEEDFACF
+      # Bogus
+      cputype:         0x00000001
+      cpusubtype:      0x00000000
+      filetype:        0x00000001
+      ncmds:           4
+      sizeofcmds:      208
+      flags:           0x00002000
+      reserved:        0x00000000
+    LoadCommands:
+      - cmd:             LC_SEGMENT_64
+        cmdsize:         152
+        segname:         ''
+        vmaddr:          0
+        vmsize:          0
+        fileoff:         272
+        filesize:        0
+        maxprot:         7
+        initprot:        7
+        nsects:          1
+        flags:           0
+        Sections:
+          - sectname:        __text
+            segname:         __TEXT
+            addr:            0x0000000000000000
+            size:            0
+            offset:          0x00000110
+            align:           0
+            reloff:          0x00000000
+            nreloc:          0
+            flags:           0x80000400
+            reserved1:       0x00000000
+            reserved2:       0x00000000
+            reserved3:       0x00000000
+            content:         ''
+      - cmd:             LC_SYMTAB
+        cmdsize:         24
+        symoff:          276
+        nsyms:           2
+        stroff:          308
+        strsize:         24
+      - cmd:             LC_VERSION_MIN_IPHONEOS
+        cmdsize:         16
+        version:         327680
+        sdk:             0
+      - cmd:             LC_DATA_IN_CODE
+        cmdsize:         16
+        dataoff:         276
+        datasize:        0
+    LinkEditData:
+      NameList:
+        - n_strx:          15
+          n_type:          0x0E
+          n_sect:          1
+          n_desc:          0
+          n_value:         0
+        - n_strx:          4
+          n_type:          0x01
+          n_sect:          0
+          n_desc:          512
+          n_value:         4
+      StringTable:
+        - ''
+        - ''
+        - ''
+        - ''
+        - _arm64_var
+        - ltmp0
+        - ''
+        - ''
+        - ''
+...

From 276f9e8cfaf306d65ac7246e91422004d4bdf54a Mon Sep 17 00:00:00 2001
From: Heejin Ahn <aheejin@gmail.com>
Date: Wed, 29 Jul 2020 01:19:36 -0700
Subject: [PATCH 0500/1035] [WebAssembly] Fix getBottom for loops

When it was first created, CFGSort only made sure BBs in each
`MachineLoop` are sorted together. After we added exception support,
CFGSort now also sorts BBs in each `WebAssemblyException`, which
represents a `catch` block, together, and
`Region` class was introduced to be a thin wrapper for both
`MachineLoop` and `WebAssemblyException`.

But how we compute those loops and exceptions is different.
`MachineLoopInfo` is constructed using the standard loop computation
algorithm in LLVM; the definition of loop is "a set of BBs that are
dominated by a loop header and have a path back to the loop header". So
even if some BBs are semantically contained by a loop in the original
program, or in other words dominated by a loop header, if they don't
have a path back to the loop header, they are not considered a part of
the loop. For example, if a BB is dominated by a loop header but
contains `call abort()` or `rethrow`, it wouldn't have a path back to
the header, so it is not included in the loop.

But `WebAssemblyException` is wasm-specific data structure, and its
algorithm is simple: a `WebAssemblyException` consists of an EH pad and
all BBs dominated by the EH pad. So this scenario is possible: (This is
also the situation in the newly added test in cfg-stackify-eh.ll)

```
Loop L: header, A, ehpad, latch
Exception E: ehpad, latch, B
```
(B contains `abort()`, so it does not have a path back to the loop
header, so it is not included in L.)

And it is sorted in this order:
```
header
A
ehpad
latch
B
```

And when CFGStackify places `end_loop` or `end_try` markers, it
previously used `WebAssembly::getBottom()`, which returns the latest BB
in the sorted order, and placed the marker there. So in this case the
marker placements will be like this:
```
loop
  header
  try
    A
  catch
    ehpad
    latch
end_loop         <-- misplaced!
    B
  end_try
```
in which nesting between the loop and the exception is not correct.
`end_loop` marker has to be placed after `B`, and also after `end_try`.

Maybe the fundamental way to solve this problem is to come up with our
own algorithm for computing loop region too, in which we include all BBs
dominated by a loop header in a loop. But this takes a lot more effort.
The only thing we need to fix is actually, `getBottom()`. If we make it
return the right BB, which means in case of a loop, the latest BB of the
loop itself and all exceptions contained in there, we are good.

This renames `Region` and `RegionInfo` to `SortRegion` and
`SortRegionInfo` and extracts them into their own file. And add
`getBottom` to `SortRegionInfo` class, from which it can access
`WebAssemblyExceptionInfo`, so that it can compute a correct bottom
block for loops.

Reviewed By: dschuff

Differential Revision: https://reviews.llvm.org/D84724
---
 llvm/lib/Target/WebAssembly/CMakeLists.txt    |  1 +
 .../Target/WebAssembly/WebAssemblyCFGSort.cpp | 91 +++----------------
 .../WebAssembly/WebAssemblyCFGStackify.cpp    | 10 +-
 .../WebAssembly/WebAssemblySortRegion.cpp     | 73 +++++++++++++++
 .../WebAssembly/WebAssemblySortRegion.h       | 91 +++++++++++++++++++
 .../Target/WebAssembly/WebAssemblyUtilities.h | 12 +--
 .../CodeGen/WebAssembly/cfg-stackify-eh.ll    | 48 ++++++++++
 7 files changed, 233 insertions(+), 93 deletions(-)
 create mode 100644 llvm/lib/Target/WebAssembly/WebAssemblySortRegion.cpp
 create mode 100644 llvm/lib/Target/WebAssembly/WebAssemblySortRegion.h

diff --git a/llvm/lib/Target/WebAssembly/CMakeLists.txt b/llvm/lib/Target/WebAssembly/CMakeLists.txt
index b730172715d93..3cbc21c4cbe2d 100644
--- a/llvm/lib/Target/WebAssembly/CMakeLists.txt
+++ b/llvm/lib/Target/WebAssembly/CMakeLists.txt
@@ -48,6 +48,7 @@ add_llvm_target(WebAssemblyCodeGen
   WebAssemblyRuntimeLibcallSignatures.cpp
   WebAssemblySelectionDAGInfo.cpp
   WebAssemblySetP2AlignOperands.cpp
+  WebAssemblySortRegion.cpp
   WebAssemblyMemIntrinsicResults.cpp
   WebAssemblySubtarget.cpp
   WebAssemblyTargetMachine.cpp
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyCFGSort.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyCFGSort.cpp
index 8442b49e25f45..eb3e9b91d40df 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyCFGSort.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyCFGSort.cpp
@@ -19,6 +19,7 @@
 #include "MCTargetDesc/WebAssemblyMCTargetDesc.h"
 #include "WebAssembly.h"
 #include "WebAssemblyExceptionInfo.h"
+#include "WebAssemblySortRegion.h"
 #include "WebAssemblySubtarget.h"
 #include "WebAssemblyUtilities.h"
 #include "llvm/ADT/PriorityQueue.h"
@@ -31,6 +32,8 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
 using namespace llvm;
+using WebAssembly::SortRegion;
+using WebAssembly::SortRegionInfo;
 
 #define DEBUG_TYPE "wasm-cfg-sort"
 
@@ -44,78 +47,6 @@ static cl::opt<bool> WasmDisableEHPadSort(
 
 namespace {
 
-// Wrapper for loops and exceptions
-class Region {
-public:
-  virtual ~Region() = default;
-  virtual MachineBasicBlock *getHeader() const = 0;
-  virtual bool contains(const MachineBasicBlock *MBB) const = 0;
-  virtual unsigned getNumBlocks() const = 0;
-  using block_iterator = typename ArrayRef<MachineBasicBlock *>::const_iterator;
-  virtual iterator_range<block_iterator> blocks() const = 0;
-  virtual bool isLoop() const = 0;
-};
-
-template <typename T> class ConcreteRegion : public Region {
-  const T *Region;
-
-public:
-  ConcreteRegion(const T *Region) : Region(Region) {}
-  MachineBasicBlock *getHeader() const override { return Region->getHeader(); }
-  bool contains(const MachineBasicBlock *MBB) const override {
-    return Region->contains(MBB);
-  }
-  unsigned getNumBlocks() const override { return Region->getNumBlocks(); }
-  iterator_range<block_iterator> blocks() const override {
-    return Region->blocks();
-  }
-  bool isLoop() const override { return false; }
-};
-
-template <> bool ConcreteRegion<MachineLoop>::isLoop() const { return true; }
-
-// This class has information of nested Regions; this is analogous to what
-// LoopInfo is for loops.
-class RegionInfo {
-  const MachineLoopInfo &MLI;
-  const WebAssemblyExceptionInfo &WEI;
-  DenseMap<const MachineLoop *, std::unique_ptr<Region>> LoopMap;
-  DenseMap<const WebAssemblyException *, std::unique_ptr<Region>> ExceptionMap;
-
-public:
-  RegionInfo(const MachineLoopInfo &MLI, const WebAssemblyExceptionInfo &WEI)
-      : MLI(MLI), WEI(WEI) {}
-
-  // Returns a smallest loop or exception that contains MBB
-  const Region *getRegionFor(const MachineBasicBlock *MBB) {
-    const auto *ML = MLI.getLoopFor(MBB);
-    const auto *WE = WEI.getExceptionFor(MBB);
-    if (!ML && !WE)
-      return nullptr;
-    // We determine subregion relationship by domination of their headers, i.e.,
-    // if region A's header dominates region B's header, B is a subregion of A.
-    // WebAssemblyException contains BBs in all its subregions (loops or
-    // exceptions), but MachineLoop may not, because MachineLoop does not contain
-    // BBs that don't have a path to its header even if they are dominated by
-    // its header. So here we should use WE->contains(ML->getHeader()), but not
-    // ML->contains(WE->getHeader()).
-    if ((ML && !WE) || (ML && WE && WE->contains(ML->getHeader()))) {
-      // If the smallest region containing MBB is a loop
-      if (LoopMap.count(ML))
-        return LoopMap[ML].get();
-      LoopMap[ML] = std::make_unique<ConcreteRegion<MachineLoop>>(ML);
-      return LoopMap[ML].get();
-    } else {
-      // If the smallest region containing MBB is an exception
-      if (ExceptionMap.count(WE))
-        return ExceptionMap[WE].get();
-      ExceptionMap[WE] =
-          std::make_unique<ConcreteRegion<WebAssemblyException>>(WE);
-      return ExceptionMap[WE].get();
-    }
-  }
-};
-
 class WebAssemblyCFGSort final : public MachineFunctionPass {
   StringRef getPassName() const override { return "WebAssembly CFG Sort"; }
 
@@ -236,14 +167,14 @@ struct CompareBlockNumbersBackwards {
 /// Bookkeeping for a region to help ensure that we don't mix blocks not
 /// dominated by the its header among its blocks.
 struct Entry {
-  const Region *TheRegion;
+  const SortRegion *TheRegion;
   unsigned NumBlocksLeft;
 
   /// List of blocks not dominated by Loop's header that are deferred until
   /// after all of Loop's blocks have been seen.
   std::vector<MachineBasicBlock *> Deferred;
 
-  explicit Entry(const class Region *R)
+  explicit Entry(const SortRegion *R)
       : TheRegion(R), NumBlocksLeft(R->getNumBlocks()) {}
 };
 } // end anonymous namespace
@@ -287,10 +218,10 @@ static void sortBlocks(MachineFunction &MF, const MachineLoopInfo &MLI,
                 CompareBlockNumbersBackwards>
       Ready;
 
-  RegionInfo RI(MLI, WEI);
+  SortRegionInfo SRI(MLI, WEI);
   SmallVector<Entry, 4> Entries;
   for (MachineBasicBlock *MBB = &MF.front();;) {
-    const Region *R = RI.getRegionFor(MBB);
+    const SortRegion *R = SRI.getRegionFor(MBB);
     if (R) {
       // If MBB is a region header, add it to the active region list. We can't
       // put any blocks that it doesn't dominate until we see the end of the
@@ -373,7 +304,7 @@ static void sortBlocks(MachineFunction &MF, const MachineLoopInfo &MLI,
   MF.RenumberBlocks();
 
 #ifndef NDEBUG
-  SmallSetVector<const Region *, 8> OnStack;
+  SmallSetVector<const SortRegion *, 8> OnStack;
 
   // Insert a sentinel representing the degenerate loop that starts at the
   // function entry block and includes the entire function as a "loop" that
@@ -382,7 +313,7 @@ static void sortBlocks(MachineFunction &MF, const MachineLoopInfo &MLI,
 
   for (auto &MBB : MF) {
     assert(MBB.getNumber() >= 0 && "Renumbered blocks should be non-negative.");
-    const Region *Region = RI.getRegionFor(&MBB);
+    const SortRegion *Region = SRI.getRegionFor(&MBB);
 
     if (Region && &MBB == Region->getHeader()) {
       // Region header.
@@ -408,10 +339,10 @@ static void sortBlocks(MachineFunction &MF, const MachineLoopInfo &MLI,
       for (auto Pred : MBB.predecessors())
         assert(Pred->getNumber() < MBB.getNumber() &&
                "Non-loop-header predecessors should be topologically sorted");
-      assert(OnStack.count(RI.getRegionFor(&MBB)) &&
+      assert(OnStack.count(SRI.getRegionFor(&MBB)) &&
              "Blocks must be nested in their regions");
     }
-    while (OnStack.size() > 1 && &MBB == WebAssembly::getBottom(OnStack.back()))
+    while (OnStack.size() > 1 && &MBB == SRI.getBottom(OnStack.back()))
       OnStack.pop_back();
   }
   assert(OnStack.pop_back_val() == nullptr &&
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyCFGStackify.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyCFGStackify.cpp
index 8cbfc98e81978..02330a2dd4afa 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyCFGStackify.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyCFGStackify.cpp
@@ -24,6 +24,7 @@
 #include "WebAssembly.h"
 #include "WebAssemblyExceptionInfo.h"
 #include "WebAssemblyMachineFunctionInfo.h"
+#include "WebAssemblySortRegion.h"
 #include "WebAssemblySubtarget.h"
 #include "WebAssemblyUtilities.h"
 #include "llvm/ADT/Statistic.h"
@@ -33,6 +34,7 @@
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/Target/TargetMachine.h"
 using namespace llvm;
+using WebAssembly::SortRegionInfo;
 
 #define DEBUG_TYPE "wasm-cfg-stackify"
 
@@ -382,6 +384,8 @@ void WebAssemblyCFGStackify::placeBlockMarker(MachineBasicBlock &MBB) {
 void WebAssemblyCFGStackify::placeLoopMarker(MachineBasicBlock &MBB) {
   MachineFunction &MF = *MBB.getParent();
   const auto &MLI = getAnalysis<MachineLoopInfo>();
+  const auto &WEI = getAnalysis<WebAssemblyExceptionInfo>();
+  SortRegionInfo SRI(MLI, WEI);
   const auto &TII = *MF.getSubtarget<WebAssemblySubtarget>().getInstrInfo();
 
   MachineLoop *Loop = MLI.getLoopFor(&MBB);
@@ -390,7 +394,7 @@ void WebAssemblyCFGStackify::placeLoopMarker(MachineBasicBlock &MBB) {
 
   // The operand of a LOOP is the first block after the loop. If the loop is the
   // bottom of the function, insert a dummy block at the end.
-  MachineBasicBlock *Bottom = WebAssembly::getBottom(Loop);
+  MachineBasicBlock *Bottom = SRI.getBottom(Loop);
   auto Iter = std::next(Bottom->getIterator());
   if (Iter == MF.end()) {
     getAppendixBlock(MF);
@@ -450,7 +454,9 @@ void WebAssemblyCFGStackify::placeTryMarker(MachineBasicBlock &MBB) {
   MachineFunction &MF = *MBB.getParent();
   auto &MDT = getAnalysis<MachineDominatorTree>();
   const auto &TII = *MF.getSubtarget<WebAssemblySubtarget>().getInstrInfo();
+  const auto &MLI = getAnalysis<MachineLoopInfo>();
   const auto &WEI = getAnalysis<WebAssemblyExceptionInfo>();
+  SortRegionInfo SRI(MLI, WEI);
   const auto &MFI = *MF.getInfo<WebAssemblyFunctionInfo>();
 
   // Compute the nearest common dominator of all unwind predecessors
@@ -470,7 +476,7 @@ void WebAssemblyCFGStackify::placeTryMarker(MachineBasicBlock &MBB) {
   // end.
   WebAssemblyException *WE = WEI.getExceptionFor(&MBB);
   assert(WE);
-  MachineBasicBlock *Bottom = WebAssembly::getBottom(WE);
+  MachineBasicBlock *Bottom = SRI.getBottom(WE);
 
   auto Iter = std::next(Bottom->getIterator());
   if (Iter == MF.end()) {
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblySortRegion.cpp b/llvm/lib/Target/WebAssembly/WebAssemblySortRegion.cpp
new file mode 100644
index 0000000000000..475f9120f7b27
--- /dev/null
+++ b/llvm/lib/Target/WebAssembly/WebAssemblySortRegion.cpp
@@ -0,0 +1,73 @@
+#include "WebAssemblySortRegion.h"
+#include "WebAssemblyExceptionInfo.h"
+#include "llvm/CodeGen/MachineLoopInfo.h"
+
+using namespace llvm;
+using namespace WebAssembly;
+
+template <> bool ConcreteSortRegion<MachineLoop>::isLoop() const {
+  return true;
+}
+
+const SortRegion *SortRegionInfo::getRegionFor(const MachineBasicBlock *MBB) {
+  const auto *ML = MLI.getLoopFor(MBB);
+  const auto *WE = WEI.getExceptionFor(MBB);
+  if (!ML && !WE)
+    return nullptr;
+  // We determine subregion relationship by domination of their headers, i.e.,
+  // if region A's header dominates region B's header, B is a subregion of A.
+  // WebAssemblyException contains BBs in all its subregions (loops or
+  // exceptions), but MachineLoop may not, because MachineLoop does not
+  // contain BBs that don't have a path to its header even if they are
+  // dominated by its header. So here we should use
+  // WE->contains(ML->getHeader()), but not ML->contains(WE->getHeader()).
+  if ((ML && !WE) || (ML && WE && WE->contains(ML->getHeader()))) {
+    // If the smallest region containing MBB is a loop
+    if (LoopMap.count(ML))
+      return LoopMap[ML].get();
+    LoopMap[ML] = std::make_unique<ConcreteSortRegion<MachineLoop>>(ML);
+    return LoopMap[ML].get();
+  } else {
+    // If the smallest region containing MBB is an exception
+    if (ExceptionMap.count(WE))
+      return ExceptionMap[WE].get();
+    ExceptionMap[WE] =
+        std::make_unique<ConcreteSortRegion<WebAssemblyException>>(WE);
+    return ExceptionMap[WE].get();
+  }
+}
+
+MachineBasicBlock *SortRegionInfo::getBottom(const SortRegion *R) {
+  if (R->isLoop())
+    return getBottom(MLI.getLoopFor(R->getHeader()));
+  else
+    return getBottom(WEI.getExceptionFor(R->getHeader()));
+}
+
+MachineBasicBlock *SortRegionInfo::getBottom(const MachineLoop *ML) {
+  MachineBasicBlock *Bottom = ML->getHeader();
+  for (MachineBasicBlock *MBB : ML->blocks()) {
+    if (MBB->getNumber() > Bottom->getNumber())
+      Bottom = MBB;
+    // MachineLoop does not contain all BBs dominated by its header. BBs that
+    // don't have a path back to the loop header aren't included. But for the
+    // purpose of CFG sorting and stackification, we need a bottom BB among all
+    // BBs that are dominated by the loop header. So we check if there is any
+    // WebAssemblyException contained in this loop, and computes the most bottom
+    // BB of them all.
+    if (MBB->isEHPad()) {
+      MachineBasicBlock *ExBottom = getBottom(WEI.getExceptionFor(MBB));
+      if (ExBottom->getNumber() > Bottom->getNumber())
+        Bottom = ExBottom;
+    }
+  }
+  return Bottom;
+}
+
+MachineBasicBlock *SortRegionInfo::getBottom(const WebAssemblyException *WE) {
+  MachineBasicBlock *Bottom = WE->getHeader();
+  for (MachineBasicBlock *MBB : WE->blocks())
+    if (MBB->getNumber() > Bottom->getNumber())
+      Bottom = MBB;
+  return Bottom;
+}
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblySortRegion.h b/llvm/lib/Target/WebAssembly/WebAssemblySortRegion.h
new file mode 100644
index 0000000000000..e92bf17641854
--- /dev/null
+++ b/llvm/lib/Target/WebAssembly/WebAssemblySortRegion.h
@@ -0,0 +1,91 @@
+//===-- WebAssemblySortRegion.h - WebAssembly Sort SortRegion ----*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// \brief This file implements regions used in CFGSort and CFGStackify.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_WEBASSEMBLY_WEBASSEMBLYSORTREGION_H
+#define LLVM_LIB_TARGET_WEBASSEMBLY_WEBASSEMBLYSORTREGION_H
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/iterator_range.h"
+
+namespace llvm {
+
+class MachineBasicBlock;
+class MachineLoop;
+class MachineLoopInfo;
+class WebAssemblyException;
+class WebAssemblyExceptionInfo;
+
+namespace WebAssembly {
+
+// Wrapper for loops and exceptions
+class SortRegion {
+public:
+  virtual ~SortRegion() = default;
+  virtual MachineBasicBlock *getHeader() const = 0;
+  virtual bool contains(const MachineBasicBlock *MBB) const = 0;
+  virtual unsigned getNumBlocks() const = 0;
+  using block_iterator = typename ArrayRef<MachineBasicBlock *>::const_iterator;
+  virtual iterator_range<block_iterator> blocks() const = 0;
+  virtual bool isLoop() const = 0;
+};
+
+template <typename T> class ConcreteSortRegion : public SortRegion {
+  const T *Unit;
+
+public:
+  ConcreteSortRegion(const T *Unit) : Unit(Unit) {}
+  MachineBasicBlock *getHeader() const override { return Unit->getHeader(); }
+  bool contains(const MachineBasicBlock *MBB) const override {
+    return Unit->contains(MBB);
+  }
+  unsigned getNumBlocks() const override { return Unit->getNumBlocks(); }
+  iterator_range<block_iterator> blocks() const override {
+    return Unit->blocks();
+  }
+  bool isLoop() const override { return false; }
+};
+
+// This class has information of nested SortRegions; this is analogous to what
+// LoopInfo is for loops.
+class SortRegionInfo {
+  friend class ConcreteSortRegion<MachineLoopInfo>;
+  friend class ConcreteSortRegion<WebAssemblyException>;
+
+  const MachineLoopInfo &MLI;
+  const WebAssemblyExceptionInfo &WEI;
+  DenseMap<const MachineLoop *, std::unique_ptr<SortRegion>> LoopMap;
+  DenseMap<const WebAssemblyException *, std::unique_ptr<SortRegion>>
+      ExceptionMap;
+
+public:
+  SortRegionInfo(const MachineLoopInfo &MLI,
+                 const WebAssemblyExceptionInfo &WEI)
+      : MLI(MLI), WEI(WEI) {}
+
+  // Returns a smallest loop or exception that contains MBB
+  const SortRegion *getRegionFor(const MachineBasicBlock *MBB);
+
+  // Return the "bottom" block among all blocks dominated by the region
+  // (MachineLoop or WebAssemblyException) header. This works when the entity is
+  // discontiguous.
+  MachineBasicBlock *getBottom(const SortRegion *R);
+  MachineBasicBlock *getBottom(const MachineLoop *ML);
+  MachineBasicBlock *getBottom(const WebAssemblyException *WE);
+};
+
+} // end namespace WebAssembly
+
+} // end namespace llvm
+
+#endif
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyUtilities.h b/llvm/lib/Target/WebAssembly/WebAssemblyUtilities.h
index 4f0ed43a24816..337077178557d 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyUtilities.h
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyUtilities.h
@@ -16,6 +16,7 @@
 #define LLVM_LIB_TARGET_WEBASSEMBLY_WEBASSEMBLYUTILITIES_H
 
 #include "llvm/CodeGen/MachineBasicBlock.h"
+#include "llvm/CodeGen/MachineFunction.h"
 
 namespace llvm {
 
@@ -33,17 +34,6 @@ extern const char *const CxaRethrowFn;
 extern const char *const StdTerminateFn;
 extern const char *const PersonalityWrapperFn;
 
-/// Return the "bottom" block of an entity, which can be either a MachineLoop or
-/// WebAssemblyException. This differs from MachineLoop::getBottomBlock in that
-/// it works even if the entity is discontiguous.
-template <typename T> MachineBasicBlock *getBottom(const T *Unit) {
-  MachineBasicBlock *Bottom = Unit->getHeader();
-  for (MachineBasicBlock *MBB : Unit->blocks())
-    if (MBB->getNumber() > Bottom->getNumber())
-      Bottom = MBB;
-  return Bottom;
-}
-
 /// Returns the operand number of a callee, assuming the argument is a call
 /// instruction.
 const MachineOperand &getCalleeOp(const MachineInstr &MI);
diff --git a/llvm/test/CodeGen/WebAssembly/cfg-stackify-eh.ll b/llvm/test/CodeGen/WebAssembly/cfg-stackify-eh.ll
index 6c7b8f9cdf8af..887dc470b3bc8 100644
--- a/llvm/test/CodeGen/WebAssembly/cfg-stackify-eh.ll
+++ b/llvm/test/CodeGen/WebAssembly/cfg-stackify-eh.ll
@@ -975,6 +975,54 @@ try.cont:                                         ; preds = %catch.start, %for.e
   ret void
 }
 
+; Here an exception is semantically contained in a loop. 'ehcleanup' BB belongs
+; to the exception, but does not belong to the loop (because it does not have a
+; path back to the loop header), and is placed after the loop latch block
+; 'invoke.cont' intentionally. This tests if 'end_loop' marker is placed
+; correctly not right after 'invoke.cont' part but after 'ehcleanup' part,
+; NOSORT-LABEL: test18
+; NOSORT: loop
+; NOSORT: try
+; NOSORT: end_try
+; NOSORT: end_loop
+define void @test18(i32 %n) personality i8* bitcast (i32 (...)* @__gxx_wasm_personality_v0 to i8*) {
+entry:
+  br label %while.cond
+
+while.cond:                                       ; preds = %invoke.cont, %entry
+  %n.addr.0 = phi i32 [ %n, %entry ], [ %dec, %invoke.cont ]
+  %tobool = icmp ne i32 %n.addr.0, 0
+  br i1 %tobool, label %while.body, label %while.end
+
+while.body:                                       ; preds = %while.cond
+  %dec = add nsw i32 %n.addr.0, -1
+  invoke void @foo()
+          to label %while.end unwind label %catch.dispatch
+
+catch.dispatch:                                   ; preds = %while.body
+  %0 = catchswitch within none [label %catch.start] unwind to caller
+
+catch.start:                                      ; preds = %catch.dispatch
+  %1 = catchpad within %0 [i8* null]
+  %2 = call i8* @llvm.wasm.get.exception(token %1)
+  %3 = call i32 @llvm.wasm.get.ehselector(token %1)
+  %4 = call i8* @__cxa_begin_catch(i8* %2) [ "funclet"(token %1) ]
+  invoke void @__cxa_end_catch() [ "funclet"(token %1) ]
+          to label %invoke.cont unwind label %ehcleanup
+
+invoke.cont:                                      ; preds = %catch.start
+  catchret from %1 to label %while.cond
+
+ehcleanup:                                        ; preds = %catch.start
+  %5 = cleanuppad within %1 []
+  %6 = call i8* @llvm.wasm.get.exception(token %5)
+  call void @__clang_call_terminate(i8* %6) [ "funclet"(token %5) ]
+  unreachable
+
+while.end:                                        ; preds = %while.body, %while.cond
+  ret void
+}
+
 ; Check if the unwind destination mismatch stats are correct
 ; NOSORT-STAT: 17 wasm-cfg-stackify    - Number of EH pad unwind mismatches found
 

From 27254ba44296d956639114487c5f5f3e7779369c Mon Sep 17 00:00:00 2001
From: Soumi Manna <soumi.manna@intel.com>
Date: Wed, 29 Jul 2020 10:36:58 -0700
Subject: [PATCH 0501/1035] updtae patch based on reviews

Signed-off-by: Soumi Manna <soumi.manna@intel.com>
---
 clang/lib/Sema/SemaSYCL.cpp | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/clang/lib/Sema/SemaSYCL.cpp b/clang/lib/Sema/SemaSYCL.cpp
index e8ab10da21cce..444b63487e1a1 100644
--- a/clang/lib/Sema/SemaSYCL.cpp
+++ b/clang/lib/Sema/SemaSYCL.cpp
@@ -943,7 +943,8 @@ class KernelObjVisitor {
       else if (FieldTy->isArrayType()) {
         if (KF_FOR_EACH(handleArrayType, Field, FieldTy))
           VisitArrayElements(Field, FieldTy, handlers...);
-      } else if (FieldTy->isScalarType() || FieldTy->isVectorType()) {
+      } else if (FieldTy->isScalarType() || FieldTy->isVectorType()
+                 || FieldTy->isUnionType()) {
         KF_FOR_EACH(handleScalarType, Field, FieldTy);
       } else if (FieldTy->isUnionType()) {
         KF_FOR_EACH(handleUnionType, Field, FieldTy);
@@ -1840,6 +1841,11 @@ class SyclKernelIntHeaderCreator : public SyclKernelFieldHandler {
     return true;
   }
 
+  bool handleUnionType(FieldDecl *FD, QualType FieldTy) final {
+    addParam(FD, FieldTy, SYCLIntegrationHeader::kind_std_layout);
+    return true;
+  }
+
   bool handleSyclStreamType(const CXXBaseSpecifier &BC,
                             QualType FieldTy) final {
     // FIXME SYCL stream should be usable as a base type

From 0c4ea2751051493bd71ffdb8dfe5af89a2324c24 Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
Date: Wed, 29 Jul 2020 17:37:10 +0000
Subject: [PATCH 0502/1035] [gn build] Port 276f9e8cfaf

---
 llvm/utils/gn/secondary/llvm/lib/Target/WebAssembly/BUILD.gn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/WebAssembly/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/WebAssembly/BUILD.gn
index f3f1019ba4609..c9947f705a323 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/WebAssembly/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/WebAssembly/BUILD.gn
@@ -95,6 +95,7 @@ static_library("LLVMWebAssemblyCodeGen") {
     "WebAssemblyRuntimeLibcallSignatures.cpp",
     "WebAssemblySelectionDAGInfo.cpp",
     "WebAssemblySetP2AlignOperands.cpp",
+    "WebAssemblySortRegion.cpp",
     "WebAssemblySubtarget.cpp",
     "WebAssemblyTargetMachine.cpp",
     "WebAssemblyTargetObjectFile.cpp",

From 90b76dac57861e438880a260834afb193e603c5a Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Wed, 29 Jul 2020 13:31:59 -0400
Subject: [PATCH 0503/1035] GloblaISel: Remove unreachable condition

Fixes bug 46882
---
 llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index 441e5ca6d2098..15e1ac8a3974e 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -1614,8 +1614,7 @@ LegalizerHelper::widenScalarExtract(MachineInstr &MI, unsigned TypeIdx,
     if (WideTy.getSizeInBits() > SrcTy.getSizeInBits()) {
       Src = MIRBuilder.buildAnyExt(WideTy, Src);
       ShiftTy = WideTy;
-    } else if (WideTy.getSizeInBits() > SrcTy.getSizeInBits())
-      return UnableToLegalize;
+    }
 
     auto LShr = MIRBuilder.buildLShr(
       ShiftTy, Src, MIRBuilder.buildConstant(ShiftTy, Offset));

From f05308a277b758462c91da7799f23eb4ede30c0c Mon Sep 17 00:00:00 2001
From: Tres Popp <tpopp@google.com>
Date: Wed, 29 Jul 2020 16:03:53 +0200
Subject: [PATCH 0504/1035] [MLIR][NFC] Move Shape::WitnessType Declaration.

This moves it from ShapeOps.td to ShapeBase.td

Differential Revision: https://reviews.llvm.org/D84845
---
 .../mlir/Dialect/Shape/IR/ShapeBase.td        | 26 +++++++++++++++++++
 .../include/mlir/Dialect/Shape/IR/ShapeOps.td | 26 -------------------
 2 files changed, 26 insertions(+), 26 deletions(-)

diff --git a/mlir/include/mlir/Dialect/Shape/IR/ShapeBase.td b/mlir/include/mlir/Dialect/Shape/IR/ShapeBase.td
index 4c00c1f3c8f7c..8f64e3c081e6b 100644
--- a/mlir/include/mlir/Dialect/Shape/IR/ShapeBase.td
+++ b/mlir/include/mlir/Dialect/Shape/IR/ShapeBase.td
@@ -124,4 +124,30 @@ def Shape_ShapeOrExtentTensorType : AnyTypeOf<[Shape_ShapeType,
 
 def Shape_SizeOrIndexType : AnyTypeOf<[Shape_SizeType, Index], "size or index">;
 
+def Shape_WitnessType : DialectType<ShapeDialect,
+    CPred<"$_self.isa<::mlir::shape::WitnessType>()">, "witness">,
+    BuildableType<"$_builder.getType<::mlir::shape::WitnessType>()"> {
+  let typeDescription = [{
+    A witness is a structural device in the compiler to maintain ordering of
+    code relying on information obtained from passing assertions. Witnesses do
+    not represent any physical data.
+
+    "cstr_" operations will return witnesses and be lowered into assertion logic
+    when not resolvable at compile time.
+
+    "assuming_" operations will take witnesses as input and represent only
+    information to the compiler, so they do not exist in executing code. Code
+    that is dependent on "assuming_" operations can assume all cstr operations
+    transitively before are honored as true.
+
+    These abstractions are intended to allow the compiler more freedom with
+    assertions by merely showing the assertion through dataflow at this time
+    rather than a side effecting operation that acts as a barrier. This can be
+    viewed similarly to a compiler representation of promises from asynchronous,
+    possibly crashing assertions. Reliant code will not be reordered to before
+    the code and non-reliant code can be reordered freely, and there are no
+    guarantees on the final ordering of the assertions or their related code.
+  }];
+}
+
 #endif // SHAPE_BASE_TD
diff --git a/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td b/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td
index 7b7da042834a0..72e392b256db3 100644
--- a/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td
+++ b/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td
@@ -19,32 +19,6 @@ include "mlir/Interfaces/InferTypeOpInterface.td"
 include "mlir/Interfaces/SideEffectInterfaces.td"
 include "mlir/IR/OpAsmInterface.td"
 
-def Shape_WitnessType : DialectType<ShapeDialect,
-    CPred<"$_self.isa<::mlir::shape::WitnessType>()">, "witness">,
-    BuildableType<"$_builder.getType<::mlir::shape::WitnessType>()"> {
-  let typeDescription = [{
-    A witness is a structural device in the compiler to maintain ordering of
-    code relying on information obtained from passing assertions. Witnesses do
-    not represent any physical data.
-
-    "cstr_" operations will return witnesses and be lowered into assertion logic
-    when not resolvable at compile time.
-
-    "assuming_" operations will take witnesses as input and represent only
-    information to the compiler, so they do not exist in executing code. Code
-    that is dependent on "assuming_" operations can assume all cstr operations
-    transitively before are honored as true.
-
-    These abstractions are intended to allow the compiler more freedom with
-    assertions by merely showing the assertion through dataflow at this time
-    rather than a side effecting operation that acts as a barrier. This can be
-    viewed similarly to a compiler representation of promises from asynchronous,
-    possibly crashing assertions. Reliant code will not be reordered to before
-    the code and non-reliant code can be reordered freely, and there are no
-    guarantees on the final ordering of the assertions or their related code.
-  }];
-}
-
 //===----------------------------------------------------------------------===//
 // Shape op definitions
 //===----------------------------------------------------------------------===//

From 766cb615a3b96025192707f4670cdf171da84034 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Sun, 19 Jul 2020 22:57:24 -0400
Subject: [PATCH 0505/1035] AMDGPU: Relax restriction on folding immediates
 into physregs

I never completed the work on the patches referenced by
f8bf7d7f42f28fa18144091022236208e199f331, but this was intended to
avoid folding immediate writes into m0 which the coalescer doesn't
understand very well. Relax this to allow simple SGPR immediates to
fold directly into VGPR copies. This pattern shows up routinely in
current GlobalISel code since nothing is smart enough to emit VGPR
constants yet.
---
 llvm/lib/Target/AMDGPU/SIFoldOperands.cpp     | 53 +++++++++----------
 .../llvm.amdgcn.kernarg.segment.ptr.ll        |  6 +--
 .../CodeGen/AMDGPU/GlobalISel/zextload.ll     | 27 ++++------
 llvm/test/CodeGen/AMDGPU/fold-imm-copy.mir    | 28 ++++++++++
 4 files changed, 64 insertions(+), 50 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
index ea1d20f7387df..88d7304550f60 100644
--- a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
@@ -643,38 +643,35 @@ void SIFoldOperands::foldOperand(
 
   if (FoldingImmLike && UseMI->isCopy()) {
     Register DestReg = UseMI->getOperand(0).getReg();
+    Register SrcReg = UseMI->getOperand(1).getReg();
+    assert(SrcReg.isVirtual());
+
+    const TargetRegisterClass *SrcRC = MRI->getRegClass(SrcReg);
 
-    // Don't fold into a copy to a physical register. Doing so would interfere
-    // with the register coalescer's logic which would avoid redundant
-    // initalizations.
-    if (DestReg.isPhysical())
+    // Don't fold into a copy to a physical register with the same class. Doing
+    // so would interfere with the register coalescer's logic which would avoid
+    // redundant initalizations.
+    if (DestReg.isPhysical() && SrcRC->contains(DestReg))
       return;
 
-    const TargetRegisterClass *DestRC =  MRI->getRegClass(DestReg);
+    const TargetRegisterClass *DestRC = TRI->getRegClassForReg(*MRI, DestReg);
+    if (TRI->isSGPRClass(SrcRC) && TRI->hasVectorRegisters(DestRC)) {
+      MachineRegisterInfo::use_iterator NextUse;
+      SmallVector<FoldCandidate, 4> CopyUses;
+      for (MachineRegisterInfo::use_iterator Use = MRI->use_begin(DestReg),
+                                             E = MRI->use_end();
+           Use != E; Use = NextUse) {
+        NextUse = std::next(Use);
+        // There's no point trying to fold into an implicit operand.
+        if (Use->isImplicit())
+          continue;
 
-    Register SrcReg = UseMI->getOperand(1).getReg();
-    if (SrcReg.isVirtual()) { // XXX - This can be an assert?
-      const TargetRegisterClass * SrcRC = MRI->getRegClass(SrcReg);
-      if (TRI->isSGPRClass(SrcRC) && TRI->hasVectorRegisters(DestRC)) {
-        MachineRegisterInfo::use_iterator NextUse;
-        SmallVector<FoldCandidate, 4> CopyUses;
-        for (MachineRegisterInfo::use_iterator
-          Use = MRI->use_begin(DestReg), E = MRI->use_end();
-          Use != E; Use = NextUse) {
-          NextUse = std::next(Use);
-
-          // There's no point trying to fold into an implicit operand.
-          if (Use->isImplicit())
-            continue;
-
-          FoldCandidate FC = FoldCandidate(Use->getParent(),
-           Use.getOperandNo(), &UseMI->getOperand(1));
-          CopyUses.push_back(FC);
-       }
-        for (auto & F : CopyUses) {
-          foldOperand(*F.OpToFold, F.UseMI, F.UseOpNo,
-           FoldList, CopiesToReplace);
-        }
+        FoldCandidate FC = FoldCandidate(Use->getParent(), Use.getOperandNo(),
+                                         &UseMI->getOperand(1));
+        CopyUses.push_back(FC);
+      }
+      for (auto &F : CopyUses) {
+        foldOperand(*F.OpToFold, F.UseMI, F.UseOpNo, FoldList, CopiesToReplace);
       }
     }
 
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.kernarg.segment.ptr.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.kernarg.segment.ptr.ll
index e3fd488af5e0d..f44c91ff6710d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.kernarg.segment.ptr.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.kernarg.segment.ptr.ll
@@ -115,10 +115,8 @@ define amdgpu_kernel void @opencl_test_implicit_alignment_no_explicit_kernargs_r
 }
 
 ; ALL-LABEL: {{^}}func_kernarg_segment_ptr:
-; ALL: s_mov_b32 [[S_LO:s[0-9]+]], 0{{$}}
-; ALL: s_mov_b32 [[S_HI:s[0-9]+]], 0{{$}}
-; ALL: v_mov_b32_e32 v0, [[S_LO]]{{$}}
-; ALL: v_mov_b32_e32 v1, [[S_HI]]{{$}}
+; ALL: v_mov_b32_e32 v0, 0{{$}}
+; ALL: v_mov_b32_e32 v1, 0{{$}}
 define i8 addrspace(4)* @func_kernarg_segment_ptr() {
   %ptr = call i8 addrspace(4)* @llvm.amdgcn.kernarg.segment.ptr()
   ret i8 addrspace(4)* %ptr
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/zextload.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/zextload.ll
index dc899ed8ba98a..1ebca23e3eda7 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/zextload.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/zextload.ll
@@ -139,9 +139,8 @@ define i96 @zextload_global_i32_to_i96(i32 addrspace(1)* %ptr) {
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    global_load_dword v0, v[0:1], off
-; GFX9-NEXT:    s_mov_b32 s4, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-NEXT:    v_mov_b32_e32 v2, s4
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -149,9 +148,8 @@ define i96 @zextload_global_i32_to_i96(i32 addrspace(1)* %ptr) {
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    flat_load_dword v0, v[0:1]
-; GFX8-NEXT:    s_mov_b32 s4, 0
 ; GFX8-NEXT:    v_mov_b32_e32 v1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v2, s4
+; GFX8-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -162,9 +160,8 @@ define i96 @zextload_global_i32_to_i96(i32 addrspace(1)* %ptr) {
 ; GFX6-NEXT:    s_mov_b32 s7, 0xf000
 ; GFX6-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX6-NEXT:    buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
-; GFX6-NEXT:    s_mov_b32 s4, 0
 ; GFX6-NEXT:    v_mov_b32_e32 v1, 0
-; GFX6-NEXT:    v_mov_b32_e32 v2, s4
+; GFX6-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    s_setpc_b64 s[30:31]
   %load = load i32, i32 addrspace(1)* %ptr
@@ -177,11 +174,9 @@ define i128 @zextload_global_i32_to_i128(i32 addrspace(1)* %ptr) {
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    global_load_dword v0, v[0:1], off
-; GFX9-NEXT:    s_mov_b32 s4, 0
-; GFX9-NEXT:    s_mov_b32 s5, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-NEXT:    v_mov_b32_e32 v2, s4
-; GFX9-NEXT:    v_mov_b32_e32 v3, s5
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-NEXT:    v_mov_b32_e32 v3, 0
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -189,11 +184,9 @@ define i128 @zextload_global_i32_to_i128(i32 addrspace(1)* %ptr) {
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    flat_load_dword v0, v[0:1]
-; GFX8-NEXT:    s_mov_b32 s4, 0
-; GFX8-NEXT:    s_mov_b32 s5, 0
 ; GFX8-NEXT:    v_mov_b32_e32 v1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v2, s4
-; GFX8-NEXT:    v_mov_b32_e32 v3, s5
+; GFX8-NEXT:    v_mov_b32_e32 v2, 0
+; GFX8-NEXT:    v_mov_b32_e32 v3, 0
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -204,11 +197,9 @@ define i128 @zextload_global_i32_to_i128(i32 addrspace(1)* %ptr) {
 ; GFX6-NEXT:    s_mov_b32 s7, 0xf000
 ; GFX6-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX6-NEXT:    buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
-; GFX6-NEXT:    s_mov_b32 s4, 0
-; GFX6-NEXT:    s_mov_b32 s5, 0
 ; GFX6-NEXT:    v_mov_b32_e32 v1, 0
-; GFX6-NEXT:    v_mov_b32_e32 v2, s4
-; GFX6-NEXT:    v_mov_b32_e32 v3, s5
+; GFX6-NEXT:    v_mov_b32_e32 v2, 0
+; GFX6-NEXT:    v_mov_b32_e32 v3, 0
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    s_setpc_b64 s[30:31]
   %load = load i32, i32 addrspace(1)* %ptr
diff --git a/llvm/test/CodeGen/AMDGPU/fold-imm-copy.mir b/llvm/test/CodeGen/AMDGPU/fold-imm-copy.mir
index c53b817344c50..9164e5e267914 100644
--- a/llvm/test/CodeGen/AMDGPU/fold-imm-copy.mir
+++ b/llvm/test/CodeGen/AMDGPU/fold-imm-copy.mir
@@ -59,3 +59,31 @@ body:             |
     S_ENDPGM 0, implicit %1, implicit %2
 
 ...
+
+# GCN-LABEL: name: no_fold_imm_into_m0{{$}}
+# GCN: %0:sreg_32 = S_MOV_B32 -8
+# GCN-NEXT: $m0 = COPY %0
+
+---
+name: no_fold_imm_into_m0
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    %0:sreg_32 = S_MOV_B32 -8
+    $m0 = COPY %0
+    S_ENDPGM 0, implicit $m0
+
+...
+
+# GCN-LABEL: name: fold_sgpr_imm_to_vgpr_copy{{$}}
+# GCN: $vgpr0 = V_MOV_B32_e32 -8, implicit $exec
+---
+name: fold_sgpr_imm_to_vgpr_copy
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    %0:sreg_32 = S_MOV_B32 -8
+    $vgpr0 = COPY %0
+    S_ENDPGM 0, implicit $vgpr0
+
+...

From 7ff9575594d001f0c514f35add36eaf7b6c5e12f Mon Sep 17 00:00:00 2001
From: Jessica Paquette <jpaquette@apple.com>
Date: Tue, 28 Jul 2020 11:33:39 -0700
Subject: [PATCH 0506/1035] [AArch64][GlobalISel] Select XRO addressing mode
 with wide immediates

Port the wide immediate case from AArch64DAGToDAGISel::SelectAddrModeXRO.

If we have a wide immediate which can't be represented in an add, we can end up
with code like this:

```
mov  x0, imm
add x1, base, x0
ldr  x2, [x1, 0]
```

If we use the [base, xN] addressing mode instead, we can produce this:

```
mov  x0, imm
ldr  x2, [base, x0]
```

This saves 0.4% code size on 7zip at -O3, and gives a geomean code size
improvement of 0.1% on CTMark.

Differential Revision: https://reviews.llvm.org/D84784
---
 .../GISel/AArch64InstructionSelector.cpp      |  56 ++++-
 .../xro-addressing-mode-constant.mir          | 211 ++++++++++++++++++
 2 files changed, 263 insertions(+), 4 deletions(-)
 create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/xro-addressing-mode-constant.mir

diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
index 6de5e5ea5b955..b96451e376c44 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
@@ -5083,12 +5083,60 @@ InstructionSelector::ComplexRendererFns
 AArch64InstructionSelector::selectAddrModeXRO(MachineOperand &Root,
                                               unsigned SizeInBytes) const {
   MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
-
-  // If we have a constant offset, then we probably don't want to match a
-  // register offset.
-  if (isBaseWithConstantOffset(Root, MRI))
+  if (!Root.isReg())
+    return None;
+  MachineInstr *PtrAdd =
+      getOpcodeDef(TargetOpcode::G_PTR_ADD, Root.getReg(), MRI);
+  if (!PtrAdd)
     return None;
 
+  // Check for an immediates which cannot be encoded in the [base + imm]
+  // addressing mode, and can't be encoded in an add/sub. If this happens, we'll
+  // end up with code like:
+  //
+  // mov x0, wide
+  // add x1 base, x0
+  // ldr x2, [x1, x0]
+  //
+  // In this situation, we can use the [base, xreg] addressing mode to save an
+  // add/sub:
+  //
+  // mov x0, wide
+  // ldr x2, [base, x0]
+  auto ValAndVReg =
+      getConstantVRegValWithLookThrough(PtrAdd->getOperand(2).getReg(), MRI);
+  if (ValAndVReg) {
+    unsigned Scale = Log2_32(SizeInBytes);
+    int64_t ImmOff = ValAndVReg->Value;
+
+    // Skip immediates that can be selected in the load/store addresing
+    // mode.
+    if (ImmOff % SizeInBytes == 0 && ImmOff >= 0 &&
+        ImmOff < (0x1000 << Scale))
+      return None;
+
+    // Helper lambda to decide whether or not it is preferable to emit an add.
+    auto isPreferredADD = [](int64_t ImmOff) {
+      // Constants in [0x0, 0xfff] can be encoded in an add.
+      if ((ImmOff & 0xfffffffffffff000LL) == 0x0LL)
+        return true;
+
+      // Can it be encoded in an add lsl #12?
+      if ((ImmOff & 0xffffffffff000fffLL) != 0x0LL)
+        return false;
+
+      // It can be encoded in an add lsl #12, but we may not want to. If it is
+      // possible to select this as a single movz, then prefer that. A single
+      // movz is faster than an add with a shift.
+      return (ImmOff & 0xffffffffff00ffffLL) != 0x0LL &&
+             (ImmOff & 0xffffffffffff0fffLL) != 0x0LL;
+    };
+
+    // If the immediate can be encoded in a single add/sub, then bail out.
+    if (isPreferredADD(ImmOff) || isPreferredADD(-ImmOff))
+      return None;
+  }
+
   // Try to fold shifts into the addressing mode.
   auto AddrModeFns = selectAddrModeShiftedExtendXReg(Root, SizeInBytes);
   if (AddrModeFns)
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/xro-addressing-mode-constant.mir b/llvm/test/CodeGen/AArch64/GlobalISel/xro-addressing-mode-constant.mir
new file mode 100644
index 0000000000000..7b2dae2f7e505
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/xro-addressing-mode-constant.mir
@@ -0,0 +1,211 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=aarch64-unknown-unknown -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s
+#
+# Test using the xro addressing mode with immediates. This should be done for
+# wide constants which are preferably selected using a mov rather than an add.
+
+...
+---
+name:            use_xro_cannot_encode_add_lsl
+alignment:       4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $x0
+    ; Check that we use the XRO addressing mode when the constant cannot be
+    ; represented using an add + lsl.
+    ;
+    ; cst = 0000000111000000
+    ; cst & 000fffffff000000 != 0
+    ;
+    ; CHECK-LABEL: name: use_xro_cannot_encode_add_lsl
+    ; CHECK: liveins: $x0
+    ; CHECK: %copy:gpr64sp = COPY $x0
+    ; CHECK: %cst:gpr64 = MOVi64imm 4580179968
+    ; CHECK: %load:gpr64 = LDRXroX %copy, %cst, 0, 0 :: (volatile load 8)
+    ; CHECK: RET_ReallyLR
+    %copy:gpr(p0) = COPY $x0
+    %cst:gpr(s64) = G_CONSTANT i64 4580179968
+    %addr:gpr(p0) = G_PTR_ADD %copy, %cst(s64)
+    %load:gpr(s64) = G_LOAD %addr(p0) :: (volatile load 8)
+    RET_ReallyLR
+
+...
+---
+name:            use_xro_preferred_mov
+alignment:       4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $x0
+    ; Check that we use the XRO addressing mode when the constant can be
+    ; represented using a single movk.
+    ;
+    ; cst = 000000000000f000
+    ; cst & 000fffffff000000 == 0
+    ; cst & ffffffffffff0fff != 0
+    ;
+    ; CHECK-LABEL: name: use_xro_preferred_mov
+    ; CHECK: liveins: $x0
+    ; CHECK: %copy:gpr64sp = COPY $x0
+    ; CHECK: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 61440
+    ; CHECK: %cst:gpr64 = SUBREG_TO_REG 0, [[MOVi32imm]], %subreg.sub_32
+    ; CHECK: %load:gpr64 = LDRXroX %copy, %cst, 0, 0 :: (volatile load 8)
+    ; CHECK: RET_ReallyLR
+    %copy:gpr(p0) = COPY $x0
+    %cst:gpr(s64) = G_CONSTANT i64 61440
+    %addr:gpr(p0) = G_PTR_ADD %copy, %cst(s64)
+    %load:gpr(s64) = G_LOAD %addr(p0) :: (volatile load 8)
+    RET_ReallyLR
+
+...
+---
+name:            use_xro_negative_imm
+alignment:       4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $x0
+    ; Check that this works even if we have a negative immediate.
+    ;
+    ; CHECK-LABEL: name: use_xro_negative_imm
+    ; CHECK: liveins: $x0
+    ; CHECK: %copy:gpr64sp = COPY $x0
+    ; CHECK: %cst:gpr64 = MOVi64imm -61440
+    ; CHECK: %load:gpr64 = LDRXroX %copy, %cst, 0, 0 :: (volatile load 8)
+    ; CHECK: RET_ReallyLR
+    %copy:gpr(p0) = COPY $x0
+    %cst:gpr(s64) = G_CONSTANT i64 -61440
+    %addr:gpr(p0) = G_PTR_ADD %copy, %cst(s64)
+    %load:gpr(s64) = G_LOAD %addr(p0) :: (volatile load 8)
+    RET_ReallyLR
+
+...
+---
+name:            dont_use_xro_selectable_imm
+alignment:       4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $x0
+    ; Immediates that can be encoded in a LDRXui should be skipped.
+    ;
+    ; CHECK-LABEL: name: dont_use_xro_selectable_imm
+    ; CHECK: liveins: $x0
+    ; CHECK: %copy:gpr64sp = COPY $x0
+    ; CHECK: %load:gpr64 = LDRXui %copy, 2 :: (volatile load 8)
+    ; CHECK: RET_ReallyLR
+    %copy:gpr(p0) = COPY $x0
+    %cst:gpr(s64) = G_CONSTANT i64 16
+    %addr:gpr(p0) = G_PTR_ADD %copy, %cst(s64)
+    %load:gpr(s64) = G_LOAD %addr(p0) :: (volatile load 8)
+    RET_ReallyLR
+
+...
+---
+name:            dont_use_xro_selectable_negative_imm
+alignment:       4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $x0
+    ; Immediates that can be encoded in a LDRXui should be skipped.
+    ;
+    ; CHECK-LABEL: name: dont_use_xro_selectable_negative_imm
+    ; CHECK: liveins: $x0
+    ; CHECK: %copy:gpr64sp = COPY $x0
+    ; CHECK: %load:gpr64 = LDURXi %copy, -16 :: (volatile load 8)
+    ; CHECK: RET_ReallyLR
+    %copy:gpr(p0) = COPY $x0
+    %cst:gpr(s64) = G_CONSTANT i64 -16
+    %addr:gpr(p0) = G_PTR_ADD %copy, %cst(s64)
+    %load:gpr(s64) = G_LOAD %addr(p0) :: (volatile load 8)
+    RET_ReallyLR
+
+...
+---
+name:            dont_use_xro_zero
+alignment:       4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $x0
+    ; Immediates that can be encoded in a LDRXui should be skipped.
+    ;
+    ; CHECK-LABEL: name: dont_use_xro_zero
+    ; CHECK: liveins: $x0
+    ; CHECK: %copy:gpr64sp = COPY $x0
+    ; CHECK: %load:gpr64 = LDRXui %copy, 0 :: (volatile load 8)
+    ; CHECK: RET_ReallyLR
+    %copy:gpr(p0) = COPY $x0
+    %cst:gpr(s64) = G_CONSTANT i64 0
+    %addr:gpr(p0) = G_PTR_ADD %copy, %cst(s64)
+    %load:gpr(s64) = G_LOAD %addr(p0) :: (volatile load 8)
+    RET_ReallyLR
+
+...
+---
+name:            dont_use_xro_in_range
+alignment:       4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $x0
+    ; Check that we skip constants which can be encoded in an add.
+    ; 17 is in [0x0, 0xfff]
+    ;
+    ; CHECK-LABEL: name: dont_use_xro_in_range
+    ; CHECK: liveins: $x0
+    ; CHECK: %copy:gpr64sp = COPY $x0
+    ; CHECK: %load:gpr64 = LDURXi %copy, 17 :: (volatile load 8)
+    ; CHECK: RET_ReallyLR
+    %copy:gpr(p0) = COPY $x0
+    %cst:gpr(s64) = G_CONSTANT i64 17
+    %addr:gpr(p0) = G_PTR_ADD %copy, %cst(s64)
+    %load:gpr(s64) = G_LOAD %addr(p0) :: (volatile load 8)
+    RET_ReallyLR
+
+...
+---
+name:            dont_use_xro_add_lsl
+alignment:       4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $x0
+    ; Check that we skip when we have an add with an lsl which cannot be
+    ; represented as a movk.
+    ;
+    ; cst = 0x0000000000111000
+    ; cst & 000fffffff000000 = 0
+    ; cst & ffffffffff00ffff != 0
+    ; cst & ffffffffffff0fff != 0
+    ;
+    ; CHECK-LABEL: name: dont_use_xro_add_lsl
+    ; CHECK: liveins: $x0
+    ; CHECK: %copy:gpr64 = COPY $x0
+    ; CHECK: [[COPY:%[0-9]+]]:gpr64common = COPY %copy
+    ; CHECK: %addr:gpr64sp = ADDXri [[COPY]], 273, 12
+    ; CHECK: %load:gpr64 = LDRXui %addr, 0 :: (volatile load 8)
+    ; CHECK: RET_ReallyLR
+    %copy:gpr(p0) = COPY $x0
+    %cst:gpr(s64) = G_CONSTANT i64 1118208
+    %addr:gpr(p0) = G_PTR_ADD %copy, %cst(s64)
+    %load:gpr(s64) = G_LOAD %addr(p0) :: (volatile load 8)
+    RET_ReallyLR

From 58743040ff50afe18253b51c5ee4a2d0e8324bbf Mon Sep 17 00:00:00 2001
From: Kostya Serebryany <kcc@google.com>
Date: Wed, 29 Jul 2020 10:34:07 -0700
Subject: [PATCH 0507/1035] Add more debug code for
 https://github.com/google/sanitizers/issues/1193 (getting desperate, not
 being able to reproduce it for a few months, but the users are seeing it)

mode debug code

Reviewed By: vitalybuka

Differential Revision: https://reviews.llvm.org/D84819
---
 compiler-rt/lib/asan/asan_allocator.cpp       | 37 ++++++++++++++++++-
 .../sanitizer_allocator_combined.h            |  7 ++++
 .../sanitizer_allocator_primary32.h           |  1 +
 .../sanitizer_allocator_primary64.h           | 23 ++++++++++++
 4 files changed, 67 insertions(+), 1 deletion(-)

diff --git a/compiler-rt/lib/asan/asan_allocator.cpp b/compiler-rt/lib/asan/asan_allocator.cpp
index 126d26d0823b3..c2f2804769d10 100644
--- a/compiler-rt/lib/asan/asan_allocator.cpp
+++ b/compiler-rt/lib/asan/asan_allocator.cpp
@@ -732,6 +732,25 @@ struct Allocator {
     return reinterpret_cast<AsanChunk *>(alloc_beg);
   }
 
+  AsanChunk *GetAsanChunkDebug(void *alloc_beg) {
+    if (!alloc_beg) return nullptr;
+    if (!allocator.FromPrimary(alloc_beg)) {
+      uptr *meta = reinterpret_cast<uptr *>(allocator.GetMetaData(alloc_beg));
+      AsanChunk *m = reinterpret_cast<AsanChunk *>(meta[1]);
+      Printf("GetAsanChunkDebug1 alloc_beg %p meta %p m %p\n", alloc_beg, meta, m);
+      return m;
+    }
+    uptr *alloc_magic = reinterpret_cast<uptr *>(alloc_beg);
+    Printf(
+        "GetAsanChunkDebug2 alloc_beg %p  alloc_magic %p alloc_magic[0] %p "
+        "alloc_magic[1] %p\n",
+        alloc_beg, alloc_magic, alloc_magic[0], alloc_magic[1]);
+    if (alloc_magic[0] == kAllocBegMagic)
+      return reinterpret_cast<AsanChunk *>(alloc_magic[1]);
+    return reinterpret_cast<AsanChunk *>(alloc_beg);
+  }
+
+
   AsanChunk *GetAsanChunkByAddr(uptr p) {
     void *alloc_beg = allocator.GetBlockBegin(reinterpret_cast<void *>(p));
     return GetAsanChunk(alloc_beg);
@@ -744,6 +763,13 @@ struct Allocator {
     return GetAsanChunk(alloc_beg);
   }
 
+  AsanChunk *GetAsanChunkByAddrFastLockedDebug(uptr p) {
+    void *alloc_beg =
+        allocator.GetBlockBeginFastLockedDebug(reinterpret_cast<void *>(p));
+    Printf("GetAsanChunkByAddrFastLockedDebug p %p alloc_beg %p\n", p, alloc_beg);
+    return GetAsanChunkDebug(alloc_beg);
+  }
+
   uptr AllocationSize(uptr p) {
     AsanChunk *m = GetAsanChunkByAddr(p);
     if (!m) return 0;
@@ -1040,16 +1066,25 @@ uptr PointsIntoChunk(void* p) {
 // Debug code. Delete once issue #1193 is chased down.
 extern "C" SANITIZER_WEAK_ATTRIBUTE const char *__lsan_current_stage;
 
+void GetUserBeginDebug(uptr chunk) {
+  Printf("GetUserBeginDebug1 chunk %p\n", chunk);
+  __asan::AsanChunk *m = __asan::instance.GetAsanChunkByAddrFastLockedDebug(chunk);
+  Printf("GetUserBeginDebug2 m     %p\n", m);
+}
+
 uptr GetUserBegin(uptr chunk) {
   __asan::AsanChunk *m = __asan::instance.GetAsanChunkByAddrFastLocked(chunk);
-  if (!m)
+  if (!m) {
     Printf(
         "ASAN is about to crash with a CHECK failure.\n"
         "The ASAN developers are trying to chase down this bug,\n"
         "so if you've encountered this bug please let us know.\n"
         "See also: https://github.com/google/sanitizers/issues/1193\n"
+        "Internal ref b/149237057\n"
         "chunk: %p caller %p __lsan_current_stage %s\n",
         chunk, GET_CALLER_PC(), __lsan_current_stage);
+    GetUserBeginDebug(chunk);
+  }
   CHECK(m);
   return m->Beg();
 }
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_allocator_combined.h b/compiler-rt/lib/sanitizer_common/sanitizer_allocator_combined.h
index 33f89d6d49928..6d73784d77d09 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_allocator_combined.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_allocator_combined.h
@@ -142,6 +142,13 @@ class CombinedAllocator {
     return secondary_.GetBlockBeginFastLocked(p);
   }
 
+  void *GetBlockBeginFastLockedDebug(void *p) {
+    if (primary_.PointerIsMine(p))
+      return primary_.GetBlockBeginDebug(p);
+    return secondary_.GetBlockBeginFastLocked(p);
+  }
+
+
   uptr GetActuallyAllocatedSize(void *p) {
     if (primary_.PointerIsMine(p))
       return primary_.GetActuallyAllocatedSize(p);
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary32.h b/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary32.h
index 3b1838b3985ac..47cc42cb411c2 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary32.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary32.h
@@ -210,6 +210,7 @@ class SizeClassAllocator32 {
     uptr res = beg + (n * (u32)size);
     return reinterpret_cast<void*>(res);
   }
+  void *GetBlockBeginDebug(const void *p) { return GetBlockBegin(p); }
 
   uptr GetActuallyAllocatedSize(void *p) {
     CHECK(PointerIsMine(p));
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary64.h b/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary64.h
index 1d9a29c70f308..acc61cc6ba8dd 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary64.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary64.h
@@ -199,6 +199,29 @@ class SizeClassAllocator64 {
     return nullptr;
   }
 
+  void *GetBlockBeginDebug(const void *p) {
+    uptr class_id = GetSizeClass(p);
+    uptr size = ClassIdToSize(class_id);
+    Printf("GetBlockBeginDebug1 p %p class_id %p size %p\n", p, class_id, size);
+    if (!size) return nullptr;
+    uptr chunk_idx = GetChunkIdx((uptr)p, size);
+    uptr reg_beg = GetRegionBegin(p);
+    uptr beg = chunk_idx * size;
+    uptr next_beg = beg + size;
+    Printf(
+        "GetBlockBeginDebug2 chunk_idx %p reg_beg %p beg %p next_beg %p "
+        "kNumClasses %p\n",
+        chunk_idx, reg_beg, beg, next_beg, kNumClasses);
+    if (class_id >= kNumClasses) return nullptr;
+    const RegionInfo *region = AddressSpaceView::Load(GetRegionInfo(class_id));
+    Printf("GetBlockBeginDebug3 region %p region->mapped_user %p\n", region,
+           region->mapped_user);
+    if (region->mapped_user >= next_beg)
+      return reinterpret_cast<void*>(reg_beg + beg);
+    return nullptr;
+  }
+
+
   uptr GetActuallyAllocatedSize(void *p) {
     CHECK(PointerIsMine(p));
     return ClassIdToSize(GetSizeClass(p));

From 0b7de7966ff0015d4ffb522e994b3c67307f284a Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Sun, 26 Jul 2020 21:25:10 -0400
Subject: [PATCH 0508/1035] GlobalISel: Implement lower for
 G_EXTRACT_VECTOR_ELT

Use the basic store to stack and reload.
---
 .../llvm/CodeGen/GlobalISel/LegalizerHelper.h |  15 +
 .../CodeGen/GlobalISel/MachineIRBuilder.h     |  20 +-
 .../CodeGen/GlobalISel/LegalizerHelper.cpp    | 115 +++++
 .../CodeGen/GlobalISel/MachineIRBuilder.cpp   |  34 +-
 .../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp |   6 +-
 .../legalize-extract-vector-elt.mir           | 452 ++++++++++++++++++
 6 files changed, 635 insertions(+), 7 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
index 0a490249f1568..d5f5ebe2a081e 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
@@ -229,6 +229,20 @@ class LegalizerHelper {
                          ArrayRef<Register> Src2Regs, LLT NarrowTy);
 
 public:
+  /// Return the alignment to use for a stack temporary object with the given
+  /// type.
+  Align getStackTemporaryAlignment(LLT Type, Align MinAlign = Align()) const;
+
+  /// Create a stack temporary based on the size in bytes and the alignment
+  MachineInstrBuilder createStackTemporary(TypeSize Bytes, Align Alignment,
+                                           MachinePointerInfo &PtrInfo);
+
+  /// Get a pointer to vector element \p Index located in memory for a vector of
+  /// type \p VecTy starting at a base address of \p VecPtr. If \p Index is out
+  /// of bounds the returned pointer is unspecified, but will be within the
+  /// vector bounds.
+  Register getVectorElementPointer(Register VecPtr, LLT VecTy, Register Index);
+
   LegalizeResult fewerElementsVectorImplicitDef(MachineInstr &MI,
                                                 unsigned TypeIdx, LLT NarrowTy);
 
@@ -312,6 +326,7 @@ class LegalizerHelper {
   LegalizeResult lowerFFloor(MachineInstr &MI);
   LegalizeResult lowerMergeValues(MachineInstr &MI);
   LegalizeResult lowerUnmergeValues(MachineInstr &MI);
+  LegalizeResult lowerExtractVectorElt(MachineInstr &MI);
   LegalizeResult lowerShuffleVector(MachineInstr &MI);
   LegalizeResult lowerDynStackAlloc(MachineInstr &MI);
   LegalizeResult lowerExtract(MachineInstr &MI);
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
index 3ad2effea1b61..cd0dcefb49c7e 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
@@ -813,7 +813,17 @@ class MachineIRBuilder {
   ///
   /// \return a MachineInstrBuilder for the newly created instruction.
   MachineInstrBuilder buildLoad(const DstOp &Res, const SrcOp &Addr,
-                                MachineMemOperand &MMO);
+                                MachineMemOperand &MMO) {
+    return buildLoadInstr(TargetOpcode::G_LOAD, Res, Addr, MMO);
+  }
+
+  /// Build and insert a G_LOAD instruction, while constructing the
+  /// MachineMemOperand.
+  MachineInstrBuilder
+  buildLoad(const DstOp &Res, const SrcOp &Addr, MachinePointerInfo PtrInfo,
+            Align Alignment,
+            MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
+            const AAMDNodes &AAInfo = AAMDNodes());
 
   /// Build and insert `Res = <opcode> Addr, MMO`.
   ///
@@ -847,6 +857,14 @@ class MachineIRBuilder {
   MachineInstrBuilder buildStore(const SrcOp &Val, const SrcOp &Addr,
                                  MachineMemOperand &MMO);
 
+  /// Build and insert a G_STORE instruction, while constructing the
+  /// MachineMemOperand.
+  MachineInstrBuilder
+  buildStore(const SrcOp &Val, const SrcOp &Addr, MachinePointerInfo PtrInfo,
+             Align Alignment,
+             MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
+             const AAMDNodes &AAInfo = AAMDNodes());
+
   /// Build and insert `Res0, ... = G_EXTRACT Src, Idx0`.
   ///
   /// \pre setBasicBlock or setMI must have been called.
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index 15e1ac8a3974e..7609adfc7c655 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -16,6 +16,7 @@
 #include "llvm/CodeGen/GlobalISel/CallLowering.h"
 #include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
 #include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
+#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/TargetFrameLowering.h"
 #include "llvm/CodeGen/TargetInstrInfo.h"
@@ -29,6 +30,7 @@
 
 using namespace llvm;
 using namespace LegalizeActions;
+using namespace MIPatternMatch;
 
 /// Try to break down \p OrigTy into \p NarrowTy sized pieces.
 ///
@@ -2713,6 +2715,8 @@ LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
     MI.eraseFromParent();
     return Legalized;
   }
+  case G_EXTRACT_VECTOR_ELT:
+    return lowerExtractVectorElt(MI);
   case G_SHUFFLE_VECTOR:
     return lowerShuffleVector(MI);
   case G_DYN_STACKALLOC:
@@ -2752,6 +2756,66 @@ LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
   }
 }
 
+Align LegalizerHelper::getStackTemporaryAlignment(LLT Ty,
+                                                  Align MinAlign) const {
+  // FIXME: We're missing a way to go back from LLT to llvm::Type to query the
+  // datalayout for the preferred alignment. Also there should be a target hook
+  // for this to allow targets to reduce the alignment and ignore the
+  // datalayout. e.g. AMDGPU should always use a 4-byte alignment, regardless of
+  // the type.
+  return std::max(Align(PowerOf2Ceil(Ty.getSizeInBytes())), MinAlign);
+}
+
+MachineInstrBuilder
+LegalizerHelper::createStackTemporary(TypeSize Bytes, Align Alignment,
+                                      MachinePointerInfo &PtrInfo) {
+  MachineFunction &MF = MIRBuilder.getMF();
+  const DataLayout &DL = MIRBuilder.getDataLayout();
+  int FrameIdx = MF.getFrameInfo().CreateStackObject(Bytes, Alignment, false);
+
+  unsigned AddrSpace = DL.getAllocaAddrSpace();
+  LLT FramePtrTy = LLT::pointer(AddrSpace, DL.getPointerSizeInBits(AddrSpace));
+
+  PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIdx);
+  return MIRBuilder.buildFrameIndex(FramePtrTy, FrameIdx);
+}
+
+static Register clampDynamicVectorIndex(MachineIRBuilder &B, Register IdxReg,
+                                        LLT VecTy) {
+  int64_t IdxVal;
+  if (mi_match(IdxReg, *B.getMRI(), m_ICst(IdxVal)))
+    return IdxReg;
+
+  LLT IdxTy = B.getMRI()->getType(IdxReg);
+  unsigned NElts = VecTy.getNumElements();
+  if (isPowerOf2_32(NElts)) {
+    APInt Imm = APInt::getLowBitsSet(IdxTy.getSizeInBits(), Log2_32(NElts));
+    return B.buildAnd(IdxTy, IdxReg, B.buildConstant(IdxTy, Imm)).getReg(0);
+  }
+
+  return B.buildUMin(IdxTy, IdxReg, B.buildConstant(IdxTy, NElts - 1))
+      .getReg(0);
+}
+
+Register LegalizerHelper::getVectorElementPointer(Register VecPtr, LLT VecTy,
+                                                  Register Index) {
+  LLT EltTy = VecTy.getElementType();
+
+  // Calculate the element offset and add it to the pointer.
+  unsigned EltSize = EltTy.getSizeInBits() / 8; // FIXME: should be ABI size.
+  assert(EltSize * 8 == EltTy.getSizeInBits() &&
+         "Converting bits to bytes lost precision");
+
+  Index = clampDynamicVectorIndex(MIRBuilder, Index, VecTy);
+
+  LLT IdxTy = MRI.getType(Index);
+  auto Mul = MIRBuilder.buildMul(IdxTy, Index,
+                                 MIRBuilder.buildConstant(IdxTy, EltSize));
+
+  LLT PtrTy = MRI.getType(VecPtr);
+  return MIRBuilder.buildPtrAdd(PtrTy, VecPtr, Mul).getReg(0);
+}
+
 LegalizerHelper::LegalizeResult LegalizerHelper::fewerElementsVectorImplicitDef(
     MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy) {
   SmallVector<Register, 2> DstRegs;
@@ -5122,6 +5186,57 @@ LegalizerHelper::lowerUnmergeValues(MachineInstr &MI) {
   return Legalized;
 }
 
+/// Lower a vector extract by writing the vector to a stack temporary and
+/// reloading the element.
+///
+/// %dst = G_EXTRACT_VECTOR_ELT %vec, %idx
+///  =>
+///  %stack_temp = G_FRAME_INDEX
+///  G_STORE %vec, %stack_temp
+///  %idx = clamp(%idx, %vec.getNumElements())
+///  %element_ptr = G_PTR_ADD %stack_temp, %idx
+///  %dst = G_LOAD %element_ptr
+LegalizerHelper::LegalizeResult
+LegalizerHelper::lowerExtractVectorElt(MachineInstr &MI) {
+  Register DstReg = MI.getOperand(0).getReg();
+  Register SrcVec = MI.getOperand(1).getReg();
+  Register Idx = MI.getOperand(2).getReg();
+  LLT VecTy = MRI.getType(SrcVec);
+  LLT EltTy = VecTy.getElementType();
+  if (!EltTy.isByteSized()) { // Not implemented.
+    LLVM_DEBUG(dbgs() << "Can't handle non-byte element vectors yet\n");
+    return UnableToLegalize;
+  }
+
+  unsigned EltBytes = EltTy.getSizeInBytes();
+  Align StoreAlign = getStackTemporaryAlignment(VecTy);
+  Align LoadAlign;
+
+  MachinePointerInfo PtrInfo;
+  auto StackTemp = createStackTemporary(TypeSize::Fixed(VecTy.getSizeInBytes()),
+                                        StoreAlign, PtrInfo);
+  MIRBuilder.buildStore(SrcVec, StackTemp, PtrInfo, StoreAlign);
+
+  // Get the pointer to the element, and be sure not to hit undefined behavior
+  // if the index is out of bounds.
+  Register LoadPtr = getVectorElementPointer(StackTemp.getReg(0), VecTy, Idx);
+
+  int64_t IdxVal;
+  if (mi_match(Idx, MRI, m_ICst(IdxVal))) {
+    int64_t Offset = IdxVal * EltBytes;
+    PtrInfo = PtrInfo.getWithOffset(Offset);
+    LoadAlign = commonAlignment(StoreAlign, Offset);
+  } else {
+    // We lose information with a variable offset.
+    LoadAlign = getStackTemporaryAlignment(EltTy);
+    PtrInfo = MachinePointerInfo(MRI.getType(LoadPtr).getAddressSpace());
+  }
+
+  MIRBuilder.buildLoad(DstReg, LoadPtr, PtrInfo, LoadAlign);
+  MI.eraseFromParent();
+  return Legalized;
+}
+
 LegalizerHelper::LegalizeResult
 LegalizerHelper::lowerShuffleVector(MachineInstr &MI) {
   Register DstReg = MI.getOperand(0).getReg();
diff --git a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
index 6a6137d466766..e5a1069ce5c1e 100644
--- a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
@@ -9,8 +9,8 @@
 /// This file implements the MachineIRBuidler class.
 //===----------------------------------------------------------------------===//
 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
+#include "llvm/Analysis/MemoryLocation.h"
 #include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
-
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineInstr.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
@@ -319,10 +319,19 @@ MachineInstrBuilder MachineIRBuilder::buildBrCond(Register Tst,
   return buildInstr(TargetOpcode::G_BRCOND).addUse(Tst).addMBB(&Dest);
 }
 
-MachineInstrBuilder MachineIRBuilder::buildLoad(const DstOp &Res,
-                                                const SrcOp &Addr,
-                                                MachineMemOperand &MMO) {
-  return buildLoadInstr(TargetOpcode::G_LOAD, Res, Addr, MMO);
+MachineInstrBuilder
+MachineIRBuilder::buildLoad(const DstOp &Dst, const SrcOp &Addr,
+                            MachinePointerInfo PtrInfo, Align Alignment,
+                            MachineMemOperand::Flags MMOFlags,
+                            const AAMDNodes &AAInfo) {
+  MMOFlags |= MachineMemOperand::MOLoad;
+  assert((MMOFlags & MachineMemOperand::MOStore) == 0);
+
+  uint64_t Size = MemoryLocation::getSizeOrUnknown(
+      TypeSize::Fixed(Dst.getLLTTy(*getMRI()).getSizeInBytes()));
+  MachineMemOperand *MMO =
+      getMF().getMachineMemOperand(PtrInfo, MMOFlags, Size, Alignment, AAInfo);
+  return buildLoad(Dst, Addr, *MMO);
 }
 
 MachineInstrBuilder MachineIRBuilder::buildLoadInstr(unsigned Opcode,
@@ -369,6 +378,21 @@ MachineInstrBuilder MachineIRBuilder::buildStore(const SrcOp &Val,
   return MIB;
 }
 
+MachineInstrBuilder
+MachineIRBuilder::buildStore(const SrcOp &Val, const SrcOp &Addr,
+                             MachinePointerInfo PtrInfo, Align Alignment,
+                             MachineMemOperand::Flags MMOFlags,
+                             const AAMDNodes &AAInfo) {
+  MMOFlags |= MachineMemOperand::MOStore;
+  assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
+
+  uint64_t Size = MemoryLocation::getSizeOrUnknown(
+      TypeSize::Fixed(Val.getLLTTy(*getMRI()).getSizeInBytes()));
+  MachineMemOperand *MMO =
+      getMF().getMachineMemOperand(PtrInfo, MMOFlags, Size, Alignment, AAInfo);
+  return buildStore(Val, Addr, *MMO);
+}
+
 MachineInstrBuilder MachineIRBuilder::buildAnyExt(const DstOp &Res,
                                                   const SrcOp &Op) {
   return buildInstr(TargetOpcode::G_ANYEXT, Res, Op);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 21e3a168738ad..bac4f9ab5c25b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -1286,7 +1286,11 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
         })
       .clampScalar(EltTypeIdx, S32, S64)
       .clampScalar(VecTypeIdx, S32, S64)
-      .clampScalar(IdxTypeIdx, S32, S32);
+      .clampScalar(IdxTypeIdx, S32, S32)
+      // TODO: Clamp the number of elements before resorting to stack lowering.
+      // It should only be necessary with variable indexes.
+      // As a last resort, lower to the stack
+      .lower();
   }
 
   getActionDefinitionsBuilder(G_EXTRACT_VECTOR_ELT)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-extract-vector-elt.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-extract-vector-elt.mir
index 6490e9a603b79..1d3311673cf6c 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-extract-vector-elt.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-extract-vector-elt.mir
@@ -694,3 +694,455 @@ body: |
     %3:_(s32) = G_EXTRACT_VECTOR_ELT %0, %2
     $vgpr0 = COPY %3
 ...
+
+---
+name: extract_vector_elt_7_v64s32
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1
+
+    ; CHECK-LABEL: name: extract_vector_elt_7_v64s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $sgpr0_sgpr1
+    ; CHECK: [[LOAD:%[0-9]+]]:_(<16 x s32>) = G_LOAD [[COPY]](p1) :: (load 64, align 4, addrspace 4)
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 64
+    ; CHECK: [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C]](s64)
+    ; CHECK: [[LOAD1:%[0-9]+]]:_(<16 x s32>) = G_LOAD [[PTR_ADD]](p1) :: (load 64 + 64, align 4, addrspace 4)
+    ; CHECK: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 128
+    ; CHECK: [[PTR_ADD1:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C1]](s64)
+    ; CHECK: [[LOAD2:%[0-9]+]]:_(<16 x s32>) = G_LOAD [[PTR_ADD1]](p1) :: (load 64 + 128, align 4, addrspace 4)
+    ; CHECK: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 192
+    ; CHECK: [[PTR_ADD2:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C2]](s64)
+    ; CHECK: [[LOAD3:%[0-9]+]]:_(<16 x s32>) = G_LOAD [[PTR_ADD2]](p1) :: (load 64 + 192, align 4, addrspace 4)
+    ; CHECK: [[FRAME_INDEX:%[0-9]+]]:_(p5) = G_FRAME_INDEX %stack.0
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32), [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32), [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32), [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32), [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32), [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[LOAD]](<16 x s32>)
+    ; CHECK: [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32), [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32), [[UV20:%[0-9]+]]:_(s32), [[UV21:%[0-9]+]]:_(s32), [[UV22:%[0-9]+]]:_(s32), [[UV23:%[0-9]+]]:_(s32), [[UV24:%[0-9]+]]:_(s32), [[UV25:%[0-9]+]]:_(s32), [[UV26:%[0-9]+]]:_(s32), [[UV27:%[0-9]+]]:_(s32), [[UV28:%[0-9]+]]:_(s32), [[UV29:%[0-9]+]]:_(s32), [[UV30:%[0-9]+]]:_(s32), [[UV31:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[LOAD1]](<16 x s32>)
+    ; CHECK: [[UV32:%[0-9]+]]:_(s32), [[UV33:%[0-9]+]]:_(s32), [[UV34:%[0-9]+]]:_(s32), [[UV35:%[0-9]+]]:_(s32), [[UV36:%[0-9]+]]:_(s32), [[UV37:%[0-9]+]]:_(s32), [[UV38:%[0-9]+]]:_(s32), [[UV39:%[0-9]+]]:_(s32), [[UV40:%[0-9]+]]:_(s32), [[UV41:%[0-9]+]]:_(s32), [[UV42:%[0-9]+]]:_(s32), [[UV43:%[0-9]+]]:_(s32), [[UV44:%[0-9]+]]:_(s32), [[UV45:%[0-9]+]]:_(s32), [[UV46:%[0-9]+]]:_(s32), [[UV47:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[LOAD2]](<16 x s32>)
+    ; CHECK: [[UV48:%[0-9]+]]:_(s32), [[UV49:%[0-9]+]]:_(s32), [[UV50:%[0-9]+]]:_(s32), [[UV51:%[0-9]+]]:_(s32), [[UV52:%[0-9]+]]:_(s32), [[UV53:%[0-9]+]]:_(s32), [[UV54:%[0-9]+]]:_(s32), [[UV55:%[0-9]+]]:_(s32), [[UV56:%[0-9]+]]:_(s32), [[UV57:%[0-9]+]]:_(s32), [[UV58:%[0-9]+]]:_(s32), [[UV59:%[0-9]+]]:_(s32), [[UV60:%[0-9]+]]:_(s32), [[UV61:%[0-9]+]]:_(s32), [[UV62:%[0-9]+]]:_(s32), [[UV63:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[LOAD3]](<16 x s32>)
+    ; CHECK: G_STORE [[UV]](s32), [[FRAME_INDEX]](p5) :: (store 4 into %stack.0, align 256, addrspace 5)
+    ; CHECK: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; CHECK: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C3]](s32)
+    ; CHECK: G_STORE [[UV1]](s32), [[PTR_ADD3]](p5) :: (store 4 into %stack.0 + 4, align 256, addrspace 5)
+    ; CHECK: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; CHECK: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C4]](s32)
+    ; CHECK: G_STORE [[UV2]](s32), [[PTR_ADD4]](p5) :: (store 4 into %stack.0 + 8, align 256, addrspace 5)
+    ; CHECK: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+    ; CHECK: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C5]](s32)
+    ; CHECK: G_STORE [[UV3]](s32), [[PTR_ADD5]](p5) :: (store 4 into %stack.0 + 12, align 256, addrspace 5)
+    ; CHECK: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CHECK: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C6]](s32)
+    ; CHECK: G_STORE [[UV4]](s32), [[PTR_ADD6]](p5) :: (store 4 into %stack.0 + 16, align 256, addrspace 5)
+    ; CHECK: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+    ; CHECK: [[PTR_ADD7:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C7]](s32)
+    ; CHECK: G_STORE [[UV5]](s32), [[PTR_ADD7]](p5) :: (store 4 into %stack.0 + 20, align 256, addrspace 5)
+    ; CHECK: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; CHECK: [[PTR_ADD8:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C8]](s32)
+    ; CHECK: G_STORE [[UV6]](s32), [[PTR_ADD8]](p5) :: (store 4 into %stack.0 + 24, align 256, addrspace 5)
+    ; CHECK: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 28
+    ; CHECK: [[PTR_ADD9:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C9]](s32)
+    ; CHECK: [[COPY1:%[0-9]+]]:_(p5) = COPY [[PTR_ADD9]](p5)
+    ; CHECK: G_STORE [[UV7]](s32), [[COPY1]](p5) :: (store 4 into %stack.0 + 28, align 256, addrspace 5)
+    ; CHECK: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
+    ; CHECK: [[PTR_ADD10:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C10]](s32)
+    ; CHECK: G_STORE [[UV8]](s32), [[PTR_ADD10]](p5) :: (store 4 into %stack.0 + 32, align 256, addrspace 5)
+    ; CHECK: [[C11:%[0-9]+]]:_(s32) = G_CONSTANT i32 36
+    ; CHECK: [[PTR_ADD11:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C11]](s32)
+    ; CHECK: G_STORE [[UV9]](s32), [[PTR_ADD11]](p5) :: (store 4 into %stack.0 + 36, align 256, addrspace 5)
+    ; CHECK: [[C12:%[0-9]+]]:_(s32) = G_CONSTANT i32 40
+    ; CHECK: [[PTR_ADD12:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C12]](s32)
+    ; CHECK: G_STORE [[UV10]](s32), [[PTR_ADD12]](p5) :: (store 4 into %stack.0 + 40, align 256, addrspace 5)
+    ; CHECK: [[C13:%[0-9]+]]:_(s32) = G_CONSTANT i32 44
+    ; CHECK: [[PTR_ADD13:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C13]](s32)
+    ; CHECK: G_STORE [[UV11]](s32), [[PTR_ADD13]](p5) :: (store 4 into %stack.0 + 44, align 256, addrspace 5)
+    ; CHECK: [[C14:%[0-9]+]]:_(s32) = G_CONSTANT i32 48
+    ; CHECK: [[PTR_ADD14:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C14]](s32)
+    ; CHECK: G_STORE [[UV12]](s32), [[PTR_ADD14]](p5) :: (store 4 into %stack.0 + 48, align 256, addrspace 5)
+    ; CHECK: [[C15:%[0-9]+]]:_(s32) = G_CONSTANT i32 52
+    ; CHECK: [[PTR_ADD15:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C15]](s32)
+    ; CHECK: G_STORE [[UV13]](s32), [[PTR_ADD15]](p5) :: (store 4 into %stack.0 + 52, align 256, addrspace 5)
+    ; CHECK: [[C16:%[0-9]+]]:_(s32) = G_CONSTANT i32 56
+    ; CHECK: [[PTR_ADD16:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C16]](s32)
+    ; CHECK: G_STORE [[UV14]](s32), [[PTR_ADD16]](p5) :: (store 4 into %stack.0 + 56, align 256, addrspace 5)
+    ; CHECK: [[C17:%[0-9]+]]:_(s32) = G_CONSTANT i32 60
+    ; CHECK: [[PTR_ADD17:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C17]](s32)
+    ; CHECK: G_STORE [[UV15]](s32), [[PTR_ADD17]](p5) :: (store 4 into %stack.0 + 60, align 256, addrspace 5)
+    ; CHECK: [[C18:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; CHECK: [[PTR_ADD18:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C18]](s32)
+    ; CHECK: G_STORE [[UV16]](s32), [[PTR_ADD18]](p5) :: (store 4 into %stack.0 + 64, align 256, addrspace 5)
+    ; CHECK: [[C19:%[0-9]+]]:_(s32) = G_CONSTANT i32 68
+    ; CHECK: [[PTR_ADD19:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C19]](s32)
+    ; CHECK: G_STORE [[UV17]](s32), [[PTR_ADD19]](p5) :: (store 4 into %stack.0 + 68, align 256, addrspace 5)
+    ; CHECK: [[C20:%[0-9]+]]:_(s32) = G_CONSTANT i32 72
+    ; CHECK: [[PTR_ADD20:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C20]](s32)
+    ; CHECK: G_STORE [[UV18]](s32), [[PTR_ADD20]](p5) :: (store 4 into %stack.0 + 72, align 256, addrspace 5)
+    ; CHECK: [[C21:%[0-9]+]]:_(s32) = G_CONSTANT i32 76
+    ; CHECK: [[PTR_ADD21:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C21]](s32)
+    ; CHECK: G_STORE [[UV19]](s32), [[PTR_ADD21]](p5) :: (store 4 into %stack.0 + 76, align 256, addrspace 5)
+    ; CHECK: [[C22:%[0-9]+]]:_(s32) = G_CONSTANT i32 80
+    ; CHECK: [[PTR_ADD22:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C22]](s32)
+    ; CHECK: G_STORE [[UV20]](s32), [[PTR_ADD22]](p5) :: (store 4 into %stack.0 + 80, align 256, addrspace 5)
+    ; CHECK: [[C23:%[0-9]+]]:_(s32) = G_CONSTANT i32 84
+    ; CHECK: [[PTR_ADD23:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C23]](s32)
+    ; CHECK: G_STORE [[UV21]](s32), [[PTR_ADD23]](p5) :: (store 4 into %stack.0 + 84, align 256, addrspace 5)
+    ; CHECK: [[C24:%[0-9]+]]:_(s32) = G_CONSTANT i32 88
+    ; CHECK: [[PTR_ADD24:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C24]](s32)
+    ; CHECK: G_STORE [[UV22]](s32), [[PTR_ADD24]](p5) :: (store 4 into %stack.0 + 88, align 256, addrspace 5)
+    ; CHECK: [[C25:%[0-9]+]]:_(s32) = G_CONSTANT i32 92
+    ; CHECK: [[PTR_ADD25:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C25]](s32)
+    ; CHECK: G_STORE [[UV23]](s32), [[PTR_ADD25]](p5) :: (store 4 into %stack.0 + 92, align 256, addrspace 5)
+    ; CHECK: [[C26:%[0-9]+]]:_(s32) = G_CONSTANT i32 96
+    ; CHECK: [[PTR_ADD26:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C26]](s32)
+    ; CHECK: G_STORE [[UV24]](s32), [[PTR_ADD26]](p5) :: (store 4 into %stack.0 + 96, align 256, addrspace 5)
+    ; CHECK: [[C27:%[0-9]+]]:_(s32) = G_CONSTANT i32 100
+    ; CHECK: [[PTR_ADD27:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C27]](s32)
+    ; CHECK: G_STORE [[UV25]](s32), [[PTR_ADD27]](p5) :: (store 4 into %stack.0 + 100, align 256, addrspace 5)
+    ; CHECK: [[C28:%[0-9]+]]:_(s32) = G_CONSTANT i32 104
+    ; CHECK: [[PTR_ADD28:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C28]](s32)
+    ; CHECK: G_STORE [[UV26]](s32), [[PTR_ADD28]](p5) :: (store 4 into %stack.0 + 104, align 256, addrspace 5)
+    ; CHECK: [[C29:%[0-9]+]]:_(s32) = G_CONSTANT i32 108
+    ; CHECK: [[PTR_ADD29:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C29]](s32)
+    ; CHECK: G_STORE [[UV27]](s32), [[PTR_ADD29]](p5) :: (store 4 into %stack.0 + 108, align 256, addrspace 5)
+    ; CHECK: [[C30:%[0-9]+]]:_(s32) = G_CONSTANT i32 112
+    ; CHECK: [[PTR_ADD30:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C30]](s32)
+    ; CHECK: G_STORE [[UV28]](s32), [[PTR_ADD30]](p5) :: (store 4 into %stack.0 + 112, align 256, addrspace 5)
+    ; CHECK: [[C31:%[0-9]+]]:_(s32) = G_CONSTANT i32 116
+    ; CHECK: [[PTR_ADD31:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C31]](s32)
+    ; CHECK: G_STORE [[UV29]](s32), [[PTR_ADD31]](p5) :: (store 4 into %stack.0 + 116, align 256, addrspace 5)
+    ; CHECK: [[C32:%[0-9]+]]:_(s32) = G_CONSTANT i32 120
+    ; CHECK: [[PTR_ADD32:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C32]](s32)
+    ; CHECK: G_STORE [[UV30]](s32), [[PTR_ADD32]](p5) :: (store 4 into %stack.0 + 120, align 256, addrspace 5)
+    ; CHECK: [[C33:%[0-9]+]]:_(s32) = G_CONSTANT i32 124
+    ; CHECK: [[PTR_ADD33:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C33]](s32)
+    ; CHECK: G_STORE [[UV31]](s32), [[PTR_ADD33]](p5) :: (store 4 into %stack.0 + 124, align 256, addrspace 5)
+    ; CHECK: [[C34:%[0-9]+]]:_(s32) = G_CONSTANT i32 128
+    ; CHECK: [[PTR_ADD34:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C34]](s32)
+    ; CHECK: G_STORE [[UV32]](s32), [[PTR_ADD34]](p5) :: (store 4 into %stack.0 + 128, align 256, addrspace 5)
+    ; CHECK: [[C35:%[0-9]+]]:_(s32) = G_CONSTANT i32 132
+    ; CHECK: [[PTR_ADD35:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C35]](s32)
+    ; CHECK: G_STORE [[UV33]](s32), [[PTR_ADD35]](p5) :: (store 4 into %stack.0 + 132, align 256, addrspace 5)
+    ; CHECK: [[C36:%[0-9]+]]:_(s32) = G_CONSTANT i32 136
+    ; CHECK: [[PTR_ADD36:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C36]](s32)
+    ; CHECK: G_STORE [[UV34]](s32), [[PTR_ADD36]](p5) :: (store 4 into %stack.0 + 136, align 256, addrspace 5)
+    ; CHECK: [[C37:%[0-9]+]]:_(s32) = G_CONSTANT i32 140
+    ; CHECK: [[PTR_ADD37:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C37]](s32)
+    ; CHECK: G_STORE [[UV35]](s32), [[PTR_ADD37]](p5) :: (store 4 into %stack.0 + 140, align 256, addrspace 5)
+    ; CHECK: [[C38:%[0-9]+]]:_(s32) = G_CONSTANT i32 144
+    ; CHECK: [[PTR_ADD38:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C38]](s32)
+    ; CHECK: G_STORE [[UV36]](s32), [[PTR_ADD38]](p5) :: (store 4 into %stack.0 + 144, align 256, addrspace 5)
+    ; CHECK: [[C39:%[0-9]+]]:_(s32) = G_CONSTANT i32 148
+    ; CHECK: [[PTR_ADD39:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C39]](s32)
+    ; CHECK: G_STORE [[UV37]](s32), [[PTR_ADD39]](p5) :: (store 4 into %stack.0 + 148, align 256, addrspace 5)
+    ; CHECK: [[C40:%[0-9]+]]:_(s32) = G_CONSTANT i32 152
+    ; CHECK: [[PTR_ADD40:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C40]](s32)
+    ; CHECK: G_STORE [[UV38]](s32), [[PTR_ADD40]](p5) :: (store 4 into %stack.0 + 152, align 256, addrspace 5)
+    ; CHECK: [[C41:%[0-9]+]]:_(s32) = G_CONSTANT i32 156
+    ; CHECK: [[PTR_ADD41:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C41]](s32)
+    ; CHECK: G_STORE [[UV39]](s32), [[PTR_ADD41]](p5) :: (store 4 into %stack.0 + 156, align 256, addrspace 5)
+    ; CHECK: [[C42:%[0-9]+]]:_(s32) = G_CONSTANT i32 160
+    ; CHECK: [[PTR_ADD42:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C42]](s32)
+    ; CHECK: G_STORE [[UV40]](s32), [[PTR_ADD42]](p5) :: (store 4 into %stack.0 + 160, align 256, addrspace 5)
+    ; CHECK: [[C43:%[0-9]+]]:_(s32) = G_CONSTANT i32 164
+    ; CHECK: [[PTR_ADD43:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C43]](s32)
+    ; CHECK: G_STORE [[UV41]](s32), [[PTR_ADD43]](p5) :: (store 4 into %stack.0 + 164, align 256, addrspace 5)
+    ; CHECK: [[C44:%[0-9]+]]:_(s32) = G_CONSTANT i32 168
+    ; CHECK: [[PTR_ADD44:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C44]](s32)
+    ; CHECK: G_STORE [[UV42]](s32), [[PTR_ADD44]](p5) :: (store 4 into %stack.0 + 168, align 256, addrspace 5)
+    ; CHECK: [[C45:%[0-9]+]]:_(s32) = G_CONSTANT i32 172
+    ; CHECK: [[PTR_ADD45:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C45]](s32)
+    ; CHECK: G_STORE [[UV43]](s32), [[PTR_ADD45]](p5) :: (store 4 into %stack.0 + 172, align 256, addrspace 5)
+    ; CHECK: [[C46:%[0-9]+]]:_(s32) = G_CONSTANT i32 176
+    ; CHECK: [[PTR_ADD46:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C46]](s32)
+    ; CHECK: G_STORE [[UV44]](s32), [[PTR_ADD46]](p5) :: (store 4 into %stack.0 + 176, align 256, addrspace 5)
+    ; CHECK: [[C47:%[0-9]+]]:_(s32) = G_CONSTANT i32 180
+    ; CHECK: [[PTR_ADD47:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C47]](s32)
+    ; CHECK: G_STORE [[UV45]](s32), [[PTR_ADD47]](p5) :: (store 4 into %stack.0 + 180, align 256, addrspace 5)
+    ; CHECK: [[C48:%[0-9]+]]:_(s32) = G_CONSTANT i32 184
+    ; CHECK: [[PTR_ADD48:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C48]](s32)
+    ; CHECK: G_STORE [[UV46]](s32), [[PTR_ADD48]](p5) :: (store 4 into %stack.0 + 184, align 256, addrspace 5)
+    ; CHECK: [[C49:%[0-9]+]]:_(s32) = G_CONSTANT i32 188
+    ; CHECK: [[PTR_ADD49:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C49]](s32)
+    ; CHECK: G_STORE [[UV47]](s32), [[PTR_ADD49]](p5) :: (store 4 into %stack.0 + 188, align 256, addrspace 5)
+    ; CHECK: [[C50:%[0-9]+]]:_(s32) = G_CONSTANT i32 192
+    ; CHECK: [[PTR_ADD50:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C50]](s32)
+    ; CHECK: G_STORE [[UV48]](s32), [[PTR_ADD50]](p5) :: (store 4 into %stack.0 + 192, align 256, addrspace 5)
+    ; CHECK: [[C51:%[0-9]+]]:_(s32) = G_CONSTANT i32 196
+    ; CHECK: [[PTR_ADD51:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C51]](s32)
+    ; CHECK: G_STORE [[UV49]](s32), [[PTR_ADD51]](p5) :: (store 4 into %stack.0 + 196, align 256, addrspace 5)
+    ; CHECK: [[C52:%[0-9]+]]:_(s32) = G_CONSTANT i32 200
+    ; CHECK: [[PTR_ADD52:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C52]](s32)
+    ; CHECK: G_STORE [[UV50]](s32), [[PTR_ADD52]](p5) :: (store 4 into %stack.0 + 200, align 256, addrspace 5)
+    ; CHECK: [[C53:%[0-9]+]]:_(s32) = G_CONSTANT i32 204
+    ; CHECK: [[PTR_ADD53:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C53]](s32)
+    ; CHECK: G_STORE [[UV51]](s32), [[PTR_ADD53]](p5) :: (store 4 into %stack.0 + 204, align 256, addrspace 5)
+    ; CHECK: [[C54:%[0-9]+]]:_(s32) = G_CONSTANT i32 208
+    ; CHECK: [[PTR_ADD54:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C54]](s32)
+    ; CHECK: G_STORE [[UV52]](s32), [[PTR_ADD54]](p5) :: (store 4 into %stack.0 + 208, align 256, addrspace 5)
+    ; CHECK: [[C55:%[0-9]+]]:_(s32) = G_CONSTANT i32 212
+    ; CHECK: [[PTR_ADD55:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C55]](s32)
+    ; CHECK: G_STORE [[UV53]](s32), [[PTR_ADD55]](p5) :: (store 4 into %stack.0 + 212, align 256, addrspace 5)
+    ; CHECK: [[C56:%[0-9]+]]:_(s32) = G_CONSTANT i32 216
+    ; CHECK: [[PTR_ADD56:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C56]](s32)
+    ; CHECK: G_STORE [[UV54]](s32), [[PTR_ADD56]](p5) :: (store 4 into %stack.0 + 216, align 256, addrspace 5)
+    ; CHECK: [[C57:%[0-9]+]]:_(s32) = G_CONSTANT i32 220
+    ; CHECK: [[PTR_ADD57:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C57]](s32)
+    ; CHECK: G_STORE [[UV55]](s32), [[PTR_ADD57]](p5) :: (store 4 into %stack.0 + 220, align 256, addrspace 5)
+    ; CHECK: [[C58:%[0-9]+]]:_(s32) = G_CONSTANT i32 224
+    ; CHECK: [[PTR_ADD58:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C58]](s32)
+    ; CHECK: G_STORE [[UV56]](s32), [[PTR_ADD58]](p5) :: (store 4 into %stack.0 + 224, align 256, addrspace 5)
+    ; CHECK: [[C59:%[0-9]+]]:_(s32) = G_CONSTANT i32 228
+    ; CHECK: [[PTR_ADD59:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C59]](s32)
+    ; CHECK: G_STORE [[UV57]](s32), [[PTR_ADD59]](p5) :: (store 4 into %stack.0 + 228, align 256, addrspace 5)
+    ; CHECK: [[C60:%[0-9]+]]:_(s32) = G_CONSTANT i32 232
+    ; CHECK: [[PTR_ADD60:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C60]](s32)
+    ; CHECK: G_STORE [[UV58]](s32), [[PTR_ADD60]](p5) :: (store 4 into %stack.0 + 232, align 256, addrspace 5)
+    ; CHECK: [[C61:%[0-9]+]]:_(s32) = G_CONSTANT i32 236
+    ; CHECK: [[PTR_ADD61:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C61]](s32)
+    ; CHECK: G_STORE [[UV59]](s32), [[PTR_ADD61]](p5) :: (store 4 into %stack.0 + 236, align 256, addrspace 5)
+    ; CHECK: [[C62:%[0-9]+]]:_(s32) = G_CONSTANT i32 240
+    ; CHECK: [[PTR_ADD62:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C62]](s32)
+    ; CHECK: G_STORE [[UV60]](s32), [[PTR_ADD62]](p5) :: (store 4 into %stack.0 + 240, align 256, addrspace 5)
+    ; CHECK: [[C63:%[0-9]+]]:_(s32) = G_CONSTANT i32 244
+    ; CHECK: [[PTR_ADD63:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C63]](s32)
+    ; CHECK: G_STORE [[UV61]](s32), [[PTR_ADD63]](p5) :: (store 4 into %stack.0 + 244, align 256, addrspace 5)
+    ; CHECK: [[C64:%[0-9]+]]:_(s32) = G_CONSTANT i32 248
+    ; CHECK: [[PTR_ADD64:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C64]](s32)
+    ; CHECK: G_STORE [[UV62]](s32), [[PTR_ADD64]](p5) :: (store 4 into %stack.0 + 248, align 256, addrspace 5)
+    ; CHECK: [[C65:%[0-9]+]]:_(s32) = G_CONSTANT i32 252
+    ; CHECK: [[PTR_ADD65:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C65]](s32)
+    ; CHECK: G_STORE [[UV63]](s32), [[PTR_ADD65]](p5) :: (store 4 into %stack.0 + 252, align 256, addrspace 5)
+    ; CHECK: [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD9]](p5) :: (load 4 from %stack.0 + 28, addrspace 5)
+    ; CHECK: S_ENDPGM 0, implicit [[LOAD4]](s32)
+    %0:_(p1) = COPY $sgpr0_sgpr1
+    %1:_(s32) = G_CONSTANT i32 7
+    %2:_(<64 x s32>) = G_LOAD %0 :: (load 256, align 4, addrspace 4)
+    %3:_(s32) = G_EXTRACT_VECTOR_ELT %2, %1
+    S_ENDPGM 0, implicit %3
+...
+
+---
+name: extract_vector_elt_varidx_v64s32
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2
+
+    ; CHECK-LABEL: name: extract_vector_elt_varidx_v64s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $sgpr0_sgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr2
+    ; CHECK: [[LOAD:%[0-9]+]]:_(<16 x s32>) = G_LOAD [[COPY]](p1) :: (load 64, align 4, addrspace 4)
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 64
+    ; CHECK: [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C]](s64)
+    ; CHECK: [[LOAD1:%[0-9]+]]:_(<16 x s32>) = G_LOAD [[PTR_ADD]](p1) :: (load 64 + 64, align 4, addrspace 4)
+    ; CHECK: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 128
+    ; CHECK: [[PTR_ADD1:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C1]](s64)
+    ; CHECK: [[LOAD2:%[0-9]+]]:_(<16 x s32>) = G_LOAD [[PTR_ADD1]](p1) :: (load 64 + 128, align 4, addrspace 4)
+    ; CHECK: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 192
+    ; CHECK: [[PTR_ADD2:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C2]](s64)
+    ; CHECK: [[LOAD3:%[0-9]+]]:_(<16 x s32>) = G_LOAD [[PTR_ADD2]](p1) :: (load 64 + 192, align 4, addrspace 4)
+    ; CHECK: [[FRAME_INDEX:%[0-9]+]]:_(p5) = G_FRAME_INDEX %stack.0
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32), [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32), [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32), [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32), [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32), [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[LOAD]](<16 x s32>)
+    ; CHECK: [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32), [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32), [[UV20:%[0-9]+]]:_(s32), [[UV21:%[0-9]+]]:_(s32), [[UV22:%[0-9]+]]:_(s32), [[UV23:%[0-9]+]]:_(s32), [[UV24:%[0-9]+]]:_(s32), [[UV25:%[0-9]+]]:_(s32), [[UV26:%[0-9]+]]:_(s32), [[UV27:%[0-9]+]]:_(s32), [[UV28:%[0-9]+]]:_(s32), [[UV29:%[0-9]+]]:_(s32), [[UV30:%[0-9]+]]:_(s32), [[UV31:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[LOAD1]](<16 x s32>)
+    ; CHECK: [[UV32:%[0-9]+]]:_(s32), [[UV33:%[0-9]+]]:_(s32), [[UV34:%[0-9]+]]:_(s32), [[UV35:%[0-9]+]]:_(s32), [[UV36:%[0-9]+]]:_(s32), [[UV37:%[0-9]+]]:_(s32), [[UV38:%[0-9]+]]:_(s32), [[UV39:%[0-9]+]]:_(s32), [[UV40:%[0-9]+]]:_(s32), [[UV41:%[0-9]+]]:_(s32), [[UV42:%[0-9]+]]:_(s32), [[UV43:%[0-9]+]]:_(s32), [[UV44:%[0-9]+]]:_(s32), [[UV45:%[0-9]+]]:_(s32), [[UV46:%[0-9]+]]:_(s32), [[UV47:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[LOAD2]](<16 x s32>)
+    ; CHECK: [[UV48:%[0-9]+]]:_(s32), [[UV49:%[0-9]+]]:_(s32), [[UV50:%[0-9]+]]:_(s32), [[UV51:%[0-9]+]]:_(s32), [[UV52:%[0-9]+]]:_(s32), [[UV53:%[0-9]+]]:_(s32), [[UV54:%[0-9]+]]:_(s32), [[UV55:%[0-9]+]]:_(s32), [[UV56:%[0-9]+]]:_(s32), [[UV57:%[0-9]+]]:_(s32), [[UV58:%[0-9]+]]:_(s32), [[UV59:%[0-9]+]]:_(s32), [[UV60:%[0-9]+]]:_(s32), [[UV61:%[0-9]+]]:_(s32), [[UV62:%[0-9]+]]:_(s32), [[UV63:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[LOAD3]](<16 x s32>)
+    ; CHECK: G_STORE [[UV]](s32), [[FRAME_INDEX]](p5) :: (store 4 into %stack.0, align 256, addrspace 5)
+    ; CHECK: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; CHECK: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C3]](s32)
+    ; CHECK: G_STORE [[UV1]](s32), [[PTR_ADD3]](p5) :: (store 4 into %stack.0 + 4, align 256, addrspace 5)
+    ; CHECK: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; CHECK: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C4]](s32)
+    ; CHECK: G_STORE [[UV2]](s32), [[PTR_ADD4]](p5) :: (store 4 into %stack.0 + 8, align 256, addrspace 5)
+    ; CHECK: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+    ; CHECK: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C5]](s32)
+    ; CHECK: G_STORE [[UV3]](s32), [[PTR_ADD5]](p5) :: (store 4 into %stack.0 + 12, align 256, addrspace 5)
+    ; CHECK: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CHECK: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C6]](s32)
+    ; CHECK: G_STORE [[UV4]](s32), [[PTR_ADD6]](p5) :: (store 4 into %stack.0 + 16, align 256, addrspace 5)
+    ; CHECK: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+    ; CHECK: [[PTR_ADD7:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C7]](s32)
+    ; CHECK: G_STORE [[UV5]](s32), [[PTR_ADD7]](p5) :: (store 4 into %stack.0 + 20, align 256, addrspace 5)
+    ; CHECK: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; CHECK: [[PTR_ADD8:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C8]](s32)
+    ; CHECK: G_STORE [[UV6]](s32), [[PTR_ADD8]](p5) :: (store 4 into %stack.0 + 24, align 256, addrspace 5)
+    ; CHECK: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 28
+    ; CHECK: [[PTR_ADD9:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C9]](s32)
+    ; CHECK: G_STORE [[UV7]](s32), [[PTR_ADD9]](p5) :: (store 4 into %stack.0 + 28, align 256, addrspace 5)
+    ; CHECK: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
+    ; CHECK: [[PTR_ADD10:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C10]](s32)
+    ; CHECK: G_STORE [[UV8]](s32), [[PTR_ADD10]](p5) :: (store 4 into %stack.0 + 32, align 256, addrspace 5)
+    ; CHECK: [[C11:%[0-9]+]]:_(s32) = G_CONSTANT i32 36
+    ; CHECK: [[PTR_ADD11:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C11]](s32)
+    ; CHECK: G_STORE [[UV9]](s32), [[PTR_ADD11]](p5) :: (store 4 into %stack.0 + 36, align 256, addrspace 5)
+    ; CHECK: [[C12:%[0-9]+]]:_(s32) = G_CONSTANT i32 40
+    ; CHECK: [[PTR_ADD12:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C12]](s32)
+    ; CHECK: G_STORE [[UV10]](s32), [[PTR_ADD12]](p5) :: (store 4 into %stack.0 + 40, align 256, addrspace 5)
+    ; CHECK: [[C13:%[0-9]+]]:_(s32) = G_CONSTANT i32 44
+    ; CHECK: [[PTR_ADD13:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C13]](s32)
+    ; CHECK: G_STORE [[UV11]](s32), [[PTR_ADD13]](p5) :: (store 4 into %stack.0 + 44, align 256, addrspace 5)
+    ; CHECK: [[C14:%[0-9]+]]:_(s32) = G_CONSTANT i32 48
+    ; CHECK: [[PTR_ADD14:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C14]](s32)
+    ; CHECK: G_STORE [[UV12]](s32), [[PTR_ADD14]](p5) :: (store 4 into %stack.0 + 48, align 256, addrspace 5)
+    ; CHECK: [[C15:%[0-9]+]]:_(s32) = G_CONSTANT i32 52
+    ; CHECK: [[PTR_ADD15:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C15]](s32)
+    ; CHECK: G_STORE [[UV13]](s32), [[PTR_ADD15]](p5) :: (store 4 into %stack.0 + 52, align 256, addrspace 5)
+    ; CHECK: [[C16:%[0-9]+]]:_(s32) = G_CONSTANT i32 56
+    ; CHECK: [[PTR_ADD16:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C16]](s32)
+    ; CHECK: G_STORE [[UV14]](s32), [[PTR_ADD16]](p5) :: (store 4 into %stack.0 + 56, align 256, addrspace 5)
+    ; CHECK: [[C17:%[0-9]+]]:_(s32) = G_CONSTANT i32 60
+    ; CHECK: [[PTR_ADD17:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C17]](s32)
+    ; CHECK: G_STORE [[UV15]](s32), [[PTR_ADD17]](p5) :: (store 4 into %stack.0 + 60, align 256, addrspace 5)
+    ; CHECK: [[C18:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; CHECK: [[PTR_ADD18:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C18]](s32)
+    ; CHECK: G_STORE [[UV16]](s32), [[PTR_ADD18]](p5) :: (store 4 into %stack.0 + 64, align 256, addrspace 5)
+    ; CHECK: [[C19:%[0-9]+]]:_(s32) = G_CONSTANT i32 68
+    ; CHECK: [[PTR_ADD19:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C19]](s32)
+    ; CHECK: G_STORE [[UV17]](s32), [[PTR_ADD19]](p5) :: (store 4 into %stack.0 + 68, align 256, addrspace 5)
+    ; CHECK: [[C20:%[0-9]+]]:_(s32) = G_CONSTANT i32 72
+    ; CHECK: [[PTR_ADD20:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C20]](s32)
+    ; CHECK: G_STORE [[UV18]](s32), [[PTR_ADD20]](p5) :: (store 4 into %stack.0 + 72, align 256, addrspace 5)
+    ; CHECK: [[C21:%[0-9]+]]:_(s32) = G_CONSTANT i32 76
+    ; CHECK: [[PTR_ADD21:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C21]](s32)
+    ; CHECK: G_STORE [[UV19]](s32), [[PTR_ADD21]](p5) :: (store 4 into %stack.0 + 76, align 256, addrspace 5)
+    ; CHECK: [[C22:%[0-9]+]]:_(s32) = G_CONSTANT i32 80
+    ; CHECK: [[PTR_ADD22:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C22]](s32)
+    ; CHECK: G_STORE [[UV20]](s32), [[PTR_ADD22]](p5) :: (store 4 into %stack.0 + 80, align 256, addrspace 5)
+    ; CHECK: [[C23:%[0-9]+]]:_(s32) = G_CONSTANT i32 84
+    ; CHECK: [[PTR_ADD23:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C23]](s32)
+    ; CHECK: G_STORE [[UV21]](s32), [[PTR_ADD23]](p5) :: (store 4 into %stack.0 + 84, align 256, addrspace 5)
+    ; CHECK: [[C24:%[0-9]+]]:_(s32) = G_CONSTANT i32 88
+    ; CHECK: [[PTR_ADD24:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C24]](s32)
+    ; CHECK: G_STORE [[UV22]](s32), [[PTR_ADD24]](p5) :: (store 4 into %stack.0 + 88, align 256, addrspace 5)
+    ; CHECK: [[C25:%[0-9]+]]:_(s32) = G_CONSTANT i32 92
+    ; CHECK: [[PTR_ADD25:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C25]](s32)
+    ; CHECK: G_STORE [[UV23]](s32), [[PTR_ADD25]](p5) :: (store 4 into %stack.0 + 92, align 256, addrspace 5)
+    ; CHECK: [[C26:%[0-9]+]]:_(s32) = G_CONSTANT i32 96
+    ; CHECK: [[PTR_ADD26:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C26]](s32)
+    ; CHECK: G_STORE [[UV24]](s32), [[PTR_ADD26]](p5) :: (store 4 into %stack.0 + 96, align 256, addrspace 5)
+    ; CHECK: [[C27:%[0-9]+]]:_(s32) = G_CONSTANT i32 100
+    ; CHECK: [[PTR_ADD27:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C27]](s32)
+    ; CHECK: G_STORE [[UV25]](s32), [[PTR_ADD27]](p5) :: (store 4 into %stack.0 + 100, align 256, addrspace 5)
+    ; CHECK: [[C28:%[0-9]+]]:_(s32) = G_CONSTANT i32 104
+    ; CHECK: [[PTR_ADD28:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C28]](s32)
+    ; CHECK: G_STORE [[UV26]](s32), [[PTR_ADD28]](p5) :: (store 4 into %stack.0 + 104, align 256, addrspace 5)
+    ; CHECK: [[C29:%[0-9]+]]:_(s32) = G_CONSTANT i32 108
+    ; CHECK: [[PTR_ADD29:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C29]](s32)
+    ; CHECK: G_STORE [[UV27]](s32), [[PTR_ADD29]](p5) :: (store 4 into %stack.0 + 108, align 256, addrspace 5)
+    ; CHECK: [[C30:%[0-9]+]]:_(s32) = G_CONSTANT i32 112
+    ; CHECK: [[PTR_ADD30:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C30]](s32)
+    ; CHECK: G_STORE [[UV28]](s32), [[PTR_ADD30]](p5) :: (store 4 into %stack.0 + 112, align 256, addrspace 5)
+    ; CHECK: [[C31:%[0-9]+]]:_(s32) = G_CONSTANT i32 116
+    ; CHECK: [[PTR_ADD31:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C31]](s32)
+    ; CHECK: G_STORE [[UV29]](s32), [[PTR_ADD31]](p5) :: (store 4 into %stack.0 + 116, align 256, addrspace 5)
+    ; CHECK: [[C32:%[0-9]+]]:_(s32) = G_CONSTANT i32 120
+    ; CHECK: [[PTR_ADD32:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C32]](s32)
+    ; CHECK: G_STORE [[UV30]](s32), [[PTR_ADD32]](p5) :: (store 4 into %stack.0 + 120, align 256, addrspace 5)
+    ; CHECK: [[C33:%[0-9]+]]:_(s32) = G_CONSTANT i32 124
+    ; CHECK: [[PTR_ADD33:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C33]](s32)
+    ; CHECK: G_STORE [[UV31]](s32), [[PTR_ADD33]](p5) :: (store 4 into %stack.0 + 124, align 256, addrspace 5)
+    ; CHECK: [[C34:%[0-9]+]]:_(s32) = G_CONSTANT i32 128
+    ; CHECK: [[PTR_ADD34:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C34]](s32)
+    ; CHECK: G_STORE [[UV32]](s32), [[PTR_ADD34]](p5) :: (store 4 into %stack.0 + 128, align 256, addrspace 5)
+    ; CHECK: [[C35:%[0-9]+]]:_(s32) = G_CONSTANT i32 132
+    ; CHECK: [[PTR_ADD35:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C35]](s32)
+    ; CHECK: G_STORE [[UV33]](s32), [[PTR_ADD35]](p5) :: (store 4 into %stack.0 + 132, align 256, addrspace 5)
+    ; CHECK: [[C36:%[0-9]+]]:_(s32) = G_CONSTANT i32 136
+    ; CHECK: [[PTR_ADD36:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C36]](s32)
+    ; CHECK: G_STORE [[UV34]](s32), [[PTR_ADD36]](p5) :: (store 4 into %stack.0 + 136, align 256, addrspace 5)
+    ; CHECK: [[C37:%[0-9]+]]:_(s32) = G_CONSTANT i32 140
+    ; CHECK: [[PTR_ADD37:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C37]](s32)
+    ; CHECK: G_STORE [[UV35]](s32), [[PTR_ADD37]](p5) :: (store 4 into %stack.0 + 140, align 256, addrspace 5)
+    ; CHECK: [[C38:%[0-9]+]]:_(s32) = G_CONSTANT i32 144
+    ; CHECK: [[PTR_ADD38:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C38]](s32)
+    ; CHECK: G_STORE [[UV36]](s32), [[PTR_ADD38]](p5) :: (store 4 into %stack.0 + 144, align 256, addrspace 5)
+    ; CHECK: [[C39:%[0-9]+]]:_(s32) = G_CONSTANT i32 148
+    ; CHECK: [[PTR_ADD39:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C39]](s32)
+    ; CHECK: G_STORE [[UV37]](s32), [[PTR_ADD39]](p5) :: (store 4 into %stack.0 + 148, align 256, addrspace 5)
+    ; CHECK: [[C40:%[0-9]+]]:_(s32) = G_CONSTANT i32 152
+    ; CHECK: [[PTR_ADD40:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C40]](s32)
+    ; CHECK: G_STORE [[UV38]](s32), [[PTR_ADD40]](p5) :: (store 4 into %stack.0 + 152, align 256, addrspace 5)
+    ; CHECK: [[C41:%[0-9]+]]:_(s32) = G_CONSTANT i32 156
+    ; CHECK: [[PTR_ADD41:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C41]](s32)
+    ; CHECK: G_STORE [[UV39]](s32), [[PTR_ADD41]](p5) :: (store 4 into %stack.0 + 156, align 256, addrspace 5)
+    ; CHECK: [[C42:%[0-9]+]]:_(s32) = G_CONSTANT i32 160
+    ; CHECK: [[PTR_ADD42:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C42]](s32)
+    ; CHECK: G_STORE [[UV40]](s32), [[PTR_ADD42]](p5) :: (store 4 into %stack.0 + 160, align 256, addrspace 5)
+    ; CHECK: [[C43:%[0-9]+]]:_(s32) = G_CONSTANT i32 164
+    ; CHECK: [[PTR_ADD43:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C43]](s32)
+    ; CHECK: G_STORE [[UV41]](s32), [[PTR_ADD43]](p5) :: (store 4 into %stack.0 + 164, align 256, addrspace 5)
+    ; CHECK: [[C44:%[0-9]+]]:_(s32) = G_CONSTANT i32 168
+    ; CHECK: [[PTR_ADD44:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C44]](s32)
+    ; CHECK: G_STORE [[UV42]](s32), [[PTR_ADD44]](p5) :: (store 4 into %stack.0 + 168, align 256, addrspace 5)
+    ; CHECK: [[C45:%[0-9]+]]:_(s32) = G_CONSTANT i32 172
+    ; CHECK: [[PTR_ADD45:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C45]](s32)
+    ; CHECK: G_STORE [[UV43]](s32), [[PTR_ADD45]](p5) :: (store 4 into %stack.0 + 172, align 256, addrspace 5)
+    ; CHECK: [[C46:%[0-9]+]]:_(s32) = G_CONSTANT i32 176
+    ; CHECK: [[PTR_ADD46:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C46]](s32)
+    ; CHECK: G_STORE [[UV44]](s32), [[PTR_ADD46]](p5) :: (store 4 into %stack.0 + 176, align 256, addrspace 5)
+    ; CHECK: [[C47:%[0-9]+]]:_(s32) = G_CONSTANT i32 180
+    ; CHECK: [[PTR_ADD47:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C47]](s32)
+    ; CHECK: G_STORE [[UV45]](s32), [[PTR_ADD47]](p5) :: (store 4 into %stack.0 + 180, align 256, addrspace 5)
+    ; CHECK: [[C48:%[0-9]+]]:_(s32) = G_CONSTANT i32 184
+    ; CHECK: [[PTR_ADD48:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C48]](s32)
+    ; CHECK: G_STORE [[UV46]](s32), [[PTR_ADD48]](p5) :: (store 4 into %stack.0 + 184, align 256, addrspace 5)
+    ; CHECK: [[C49:%[0-9]+]]:_(s32) = G_CONSTANT i32 188
+    ; CHECK: [[PTR_ADD49:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C49]](s32)
+    ; CHECK: G_STORE [[UV47]](s32), [[PTR_ADD49]](p5) :: (store 4 into %stack.0 + 188, align 256, addrspace 5)
+    ; CHECK: [[C50:%[0-9]+]]:_(s32) = G_CONSTANT i32 192
+    ; CHECK: [[PTR_ADD50:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C50]](s32)
+    ; CHECK: G_STORE [[UV48]](s32), [[PTR_ADD50]](p5) :: (store 4 into %stack.0 + 192, align 256, addrspace 5)
+    ; CHECK: [[C51:%[0-9]+]]:_(s32) = G_CONSTANT i32 196
+    ; CHECK: [[PTR_ADD51:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C51]](s32)
+    ; CHECK: G_STORE [[UV49]](s32), [[PTR_ADD51]](p5) :: (store 4 into %stack.0 + 196, align 256, addrspace 5)
+    ; CHECK: [[C52:%[0-9]+]]:_(s32) = G_CONSTANT i32 200
+    ; CHECK: [[PTR_ADD52:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C52]](s32)
+    ; CHECK: G_STORE [[UV50]](s32), [[PTR_ADD52]](p5) :: (store 4 into %stack.0 + 200, align 256, addrspace 5)
+    ; CHECK: [[C53:%[0-9]+]]:_(s32) = G_CONSTANT i32 204
+    ; CHECK: [[PTR_ADD53:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C53]](s32)
+    ; CHECK: G_STORE [[UV51]](s32), [[PTR_ADD53]](p5) :: (store 4 into %stack.0 + 204, align 256, addrspace 5)
+    ; CHECK: [[C54:%[0-9]+]]:_(s32) = G_CONSTANT i32 208
+    ; CHECK: [[PTR_ADD54:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C54]](s32)
+    ; CHECK: G_STORE [[UV52]](s32), [[PTR_ADD54]](p5) :: (store 4 into %stack.0 + 208, align 256, addrspace 5)
+    ; CHECK: [[C55:%[0-9]+]]:_(s32) = G_CONSTANT i32 212
+    ; CHECK: [[PTR_ADD55:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C55]](s32)
+    ; CHECK: G_STORE [[UV53]](s32), [[PTR_ADD55]](p5) :: (store 4 into %stack.0 + 212, align 256, addrspace 5)
+    ; CHECK: [[C56:%[0-9]+]]:_(s32) = G_CONSTANT i32 216
+    ; CHECK: [[PTR_ADD56:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C56]](s32)
+    ; CHECK: G_STORE [[UV54]](s32), [[PTR_ADD56]](p5) :: (store 4 into %stack.0 + 216, align 256, addrspace 5)
+    ; CHECK: [[C57:%[0-9]+]]:_(s32) = G_CONSTANT i32 220
+    ; CHECK: [[PTR_ADD57:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C57]](s32)
+    ; CHECK: G_STORE [[UV55]](s32), [[PTR_ADD57]](p5) :: (store 4 into %stack.0 + 220, align 256, addrspace 5)
+    ; CHECK: [[C58:%[0-9]+]]:_(s32) = G_CONSTANT i32 224
+    ; CHECK: [[PTR_ADD58:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C58]](s32)
+    ; CHECK: G_STORE [[UV56]](s32), [[PTR_ADD58]](p5) :: (store 4 into %stack.0 + 224, align 256, addrspace 5)
+    ; CHECK: [[C59:%[0-9]+]]:_(s32) = G_CONSTANT i32 228
+    ; CHECK: [[PTR_ADD59:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C59]](s32)
+    ; CHECK: G_STORE [[UV57]](s32), [[PTR_ADD59]](p5) :: (store 4 into %stack.0 + 228, align 256, addrspace 5)
+    ; CHECK: [[C60:%[0-9]+]]:_(s32) = G_CONSTANT i32 232
+    ; CHECK: [[PTR_ADD60:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C60]](s32)
+    ; CHECK: G_STORE [[UV58]](s32), [[PTR_ADD60]](p5) :: (store 4 into %stack.0 + 232, align 256, addrspace 5)
+    ; CHECK: [[C61:%[0-9]+]]:_(s32) = G_CONSTANT i32 236
+    ; CHECK: [[PTR_ADD61:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C61]](s32)
+    ; CHECK: G_STORE [[UV59]](s32), [[PTR_ADD61]](p5) :: (store 4 into %stack.0 + 236, align 256, addrspace 5)
+    ; CHECK: [[C62:%[0-9]+]]:_(s32) = G_CONSTANT i32 240
+    ; CHECK: [[PTR_ADD62:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C62]](s32)
+    ; CHECK: G_STORE [[UV60]](s32), [[PTR_ADD62]](p5) :: (store 4 into %stack.0 + 240, align 256, addrspace 5)
+    ; CHECK: [[C63:%[0-9]+]]:_(s32) = G_CONSTANT i32 244
+    ; CHECK: [[PTR_ADD63:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C63]](s32)
+    ; CHECK: G_STORE [[UV61]](s32), [[PTR_ADD63]](p5) :: (store 4 into %stack.0 + 244, align 256, addrspace 5)
+    ; CHECK: [[C64:%[0-9]+]]:_(s32) = G_CONSTANT i32 248
+    ; CHECK: [[PTR_ADD64:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C64]](s32)
+    ; CHECK: G_STORE [[UV62]](s32), [[PTR_ADD64]](p5) :: (store 4 into %stack.0 + 248, align 256, addrspace 5)
+    ; CHECK: [[C65:%[0-9]+]]:_(s32) = G_CONSTANT i32 252
+    ; CHECK: [[PTR_ADD65:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C65]](s32)
+    ; CHECK: G_STORE [[UV63]](s32), [[PTR_ADD65]](p5) :: (store 4 into %stack.0 + 252, align 256, addrspace 5)
+    ; CHECK: [[C66:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C66]]
+    ; CHECK: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND]], [[C3]]
+    ; CHECK: [[PTR_ADD66:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[MUL]](s32)
+    ; CHECK: [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD66]](p5) :: (load 4, addrspace 5)
+    ; CHECK: S_ENDPGM 0, implicit [[LOAD4]](s32)
+    %0:_(p1) = COPY $sgpr0_sgpr1
+    %1:_(s32) = COPY $sgpr2
+    %2:_(<64 x s32>) = G_LOAD %0 :: (load 256, align 4, addrspace 4)
+    %3:_(s32) = G_EXTRACT_VECTOR_ELT %2, %1
+    S_ENDPGM 0, implicit %3
+...

From b5e896c0493d44c2d9819d584fe2814af7cfd476 Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne@apple.com>
Date: Wed, 29 Jul 2020 14:16:51 -0400
Subject: [PATCH 0509/1035] [libc++][pstl] Remove c++98 from UNSUPPORTED
 annotations

c++98 isn't used by the test suite anymore, only c++03 is.
---
 libcxx/test/std/numerics/numbers/defined.pass.cpp             | 2 +-
 libcxx/test/std/numerics/numbers/illformed.verify.cpp         | 2 +-
 libcxx/test/std/numerics/numbers/specialize.pass.cpp          | 2 +-
 libcxx/test/std/numerics/numbers/user_type.pass.cpp           | 2 +-
 libcxx/test/std/numerics/numbers/value.pass.cpp               | 2 +-
 pstl/test/pstl/header_inclusion_order_algorithm_0.pass.cpp    | 2 +-
 pstl/test/pstl/header_inclusion_order_algorithm_1.pass.cpp    | 2 +-
 pstl/test/pstl/header_inclusion_order_memory_0.pass.cpp       | 2 +-
 pstl/test/pstl/header_inclusion_order_memory_1.pass.cpp       | 2 +-
 pstl/test/pstl/header_inclusion_order_numeric_0.pass.cpp      | 2 +-
 pstl/test/pstl/header_inclusion_order_numeric_1.pass.cpp      | 2 +-
 pstl/test/std/algorithms/alg.merge/inplace_merge.pass.cpp     | 4 ++--
 pstl/test/std/algorithms/alg.merge/merge.pass.cpp             | 2 +-
 .../alg.modifying.operations/alg.copy/copy_if.pass.cpp        | 2 +-
 .../alg.partitions/is_partitioned.pass.cpp                    | 2 +-
 .../alg.partitions/partition.pass.cpp                         | 2 +-
 .../alg.partitions/partition_copy.pass.cpp                    | 2 +-
 .../alg.modifying.operations/alg.reverse/reverse.pass.cpp     | 2 +-
 .../alg.reverse/reverse_copy.pass.cpp                         | 2 +-
 .../algorithms/alg.modifying.operations/copy_move.pass.cpp    | 2 +-
 .../std/algorithms/alg.modifying.operations/fill.pass.cpp     | 2 +-
 .../std/algorithms/alg.modifying.operations/generate.pass.cpp | 2 +-
 .../std/algorithms/alg.modifying.operations/remove.pass.cpp   | 4 ++--
 .../algorithms/alg.modifying.operations/remove_copy.pass.cpp  | 2 +-
 .../std/algorithms/alg.modifying.operations/replace.pass.cpp  | 2 +-
 .../algorithms/alg.modifying.operations/replace_copy.pass.cpp | 2 +-
 .../std/algorithms/alg.modifying.operations/rotate.pass.cpp   | 2 +-
 .../algorithms/alg.modifying.operations/rotate_copy.pass.cpp  | 2 +-
 .../algorithms/alg.modifying.operations/swap_ranges.pass.cpp  | 2 +-
 .../alg.modifying.operations/transform_binary.pass.cpp        | 2 +-
 .../alg.modifying.operations/transform_unary.pass.cpp         | 2 +-
 .../std/algorithms/alg.modifying.operations/unique.pass.cpp   | 4 ++--
 .../alg.modifying.operations/unique_copy_equal.pass.cpp       | 2 +-
 .../std/algorithms/alg.nonmodifying/adjacent_find.pass.cpp    | 2 +-
 pstl/test/std/algorithms/alg.nonmodifying/all_of.pass.cpp     | 2 +-
 pstl/test/std/algorithms/alg.nonmodifying/any_of.pass.cpp     | 2 +-
 pstl/test/std/algorithms/alg.nonmodifying/count.pass.cpp      | 2 +-
 pstl/test/std/algorithms/alg.nonmodifying/equal.pass.cpp      | 2 +-
 pstl/test/std/algorithms/alg.nonmodifying/find.pass.cpp       | 2 +-
 pstl/test/std/algorithms/alg.nonmodifying/find_end.pass.cpp   | 2 +-
 .../std/algorithms/alg.nonmodifying/find_first_of.pass.cpp    | 2 +-
 pstl/test/std/algorithms/alg.nonmodifying/find_if.pass.cpp    | 2 +-
 pstl/test/std/algorithms/alg.nonmodifying/for_each.pass.cpp   | 2 +-
 pstl/test/std/algorithms/alg.nonmodifying/mismatch.pass.cpp   | 2 +-
 pstl/test/std/algorithms/alg.nonmodifying/none_of.pass.cpp    | 2 +-
 .../test/std/algorithms/alg.nonmodifying/nth_element.pass.cpp | 2 +-
 pstl/test/std/algorithms/alg.nonmodifying/search_n.pass.cpp   | 2 +-
 .../alg.sorting/alg.heap.operations/is_heap.pass.cpp          | 2 +-
 .../alg.lex.comparison/lexicographical_compare.pass.cpp       | 2 +-
 .../alg.sorting/alg.min.max/minmax_element.pass.cpp           | 2 +-
 .../alg.sorting/alg.set.operations/includes.pass.cpp          | 2 +-
 .../algorithms/alg.sorting/alg.set.operations/set.pass.cpp    | 2 +-
 pstl/test/std/algorithms/alg.sorting/is_sorted.pass.cpp       | 2 +-
 pstl/test/std/algorithms/alg.sorting/partial_sort.pass.cpp    | 2 +-
 .../std/algorithms/alg.sorting/partial_sort_copy.pass.cpp     | 4 ++--
 pstl/test/std/algorithms/alg.sorting/sort.pass.cpp            | 2 +-
 .../std/numerics/numeric.ops/adjacent_difference.pass.cpp     | 2 +-
 pstl/test/std/numerics/numeric.ops/reduce.pass.cpp            | 2 +-
 pstl/test/std/numerics/numeric.ops/scan.fail.cpp              | 2 +-
 pstl/test/std/numerics/numeric.ops/scan.pass.cpp              | 2 +-
 pstl/test/std/numerics/numeric.ops/transform_reduce.pass.cpp  | 2 +-
 pstl/test/std/numerics/numeric.ops/transform_scan.pass.cpp    | 2 +-
 .../specialized.algorithms/uninitialized_construct.pass.cpp   | 2 +-
 .../specialized.algorithms/uninitialized_copy_move.pass.cpp   | 2 +-
 .../uninitialized_fill_destroy.pass.cpp                       | 2 +-
 65 files changed, 69 insertions(+), 69 deletions(-)

diff --git a/libcxx/test/std/numerics/numbers/defined.pass.cpp b/libcxx/test/std/numerics/numbers/defined.pass.cpp
index cc8bf179a74e3..ef432b8585e40 100644
--- a/libcxx/test/std/numerics/numbers/defined.pass.cpp
+++ b/libcxx/test/std/numerics/numbers/defined.pass.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14, c++17
+// UNSUPPORTED: c++03, c++11, c++14, c++17
 // UNSUPPORTED: libcpp-no-concepts
 
 #include <numbers>
diff --git a/libcxx/test/std/numerics/numbers/illformed.verify.cpp b/libcxx/test/std/numerics/numbers/illformed.verify.cpp
index 5e35181f8fd94..d0b21bfa79a59 100644
--- a/libcxx/test/std/numerics/numbers/illformed.verify.cpp
+++ b/libcxx/test/std/numerics/numbers/illformed.verify.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14, c++17
+// UNSUPPORTED: c++03, c++11, c++14, c++17
 // UNSUPPORTED: libcpp-no-concepts
 
 #include <numbers>
diff --git a/libcxx/test/std/numerics/numbers/specialize.pass.cpp b/libcxx/test/std/numerics/numbers/specialize.pass.cpp
index 380892cca4eae..df31c8afb4baf 100644
--- a/libcxx/test/std/numerics/numbers/specialize.pass.cpp
+++ b/libcxx/test/std/numerics/numbers/specialize.pass.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14, c++17
+// UNSUPPORTED: c++03, c++11, c++14, c++17
 // UNSUPPORTED: libcpp-no-concepts
 
 #include <cassert>
diff --git a/libcxx/test/std/numerics/numbers/user_type.pass.cpp b/libcxx/test/std/numerics/numbers/user_type.pass.cpp
index d4152c9ff74f6..fa547cf05957e 100644
--- a/libcxx/test/std/numerics/numbers/user_type.pass.cpp
+++ b/libcxx/test/std/numerics/numbers/user_type.pass.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14, c++17
+// UNSUPPORTED: c++03, c++11, c++14, c++17
 // UNSUPPORTED: libcpp-no-concepts
 
 #include <numbers>
diff --git a/libcxx/test/std/numerics/numbers/value.pass.cpp b/libcxx/test/std/numerics/numbers/value.pass.cpp
index ff6a05ae2de30..9f623e55abaa8 100644
--- a/libcxx/test/std/numerics/numbers/value.pass.cpp
+++ b/libcxx/test/std/numerics/numbers/value.pass.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14, c++17
+// UNSUPPORTED: c++03, c++11, c++14, c++17
 // UNSUPPORTED: libcpp-no-concepts
 
 #include <cassert>
diff --git a/pstl/test/pstl/header_inclusion_order_algorithm_0.pass.cpp b/pstl/test/pstl/header_inclusion_order_algorithm_0.pass.cpp
index df8ebe58390f5..8cdeb18d6d758 100644
--- a/pstl/test/pstl/header_inclusion_order_algorithm_0.pass.cpp
+++ b/pstl/test/pstl/header_inclusion_order_algorithm_0.pass.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14
 
 #include "support/pstl_test_config.h"
 
diff --git a/pstl/test/pstl/header_inclusion_order_algorithm_1.pass.cpp b/pstl/test/pstl/header_inclusion_order_algorithm_1.pass.cpp
index 818ae4ba74576..ea6c4f49a0ae1 100644
--- a/pstl/test/pstl/header_inclusion_order_algorithm_1.pass.cpp
+++ b/pstl/test/pstl/header_inclusion_order_algorithm_1.pass.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14
 
 #include "support/pstl_test_config.h"
 
diff --git a/pstl/test/pstl/header_inclusion_order_memory_0.pass.cpp b/pstl/test/pstl/header_inclusion_order_memory_0.pass.cpp
index 104610a8d8968..dbb70c5566bd5 100644
--- a/pstl/test/pstl/header_inclusion_order_memory_0.pass.cpp
+++ b/pstl/test/pstl/header_inclusion_order_memory_0.pass.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14
 
 #include "support/pstl_test_config.h"
 
diff --git a/pstl/test/pstl/header_inclusion_order_memory_1.pass.cpp b/pstl/test/pstl/header_inclusion_order_memory_1.pass.cpp
index aece3eeef4977..99f059c2e736d 100644
--- a/pstl/test/pstl/header_inclusion_order_memory_1.pass.cpp
+++ b/pstl/test/pstl/header_inclusion_order_memory_1.pass.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14
 
 #include "support/pstl_test_config.h"
 
diff --git a/pstl/test/pstl/header_inclusion_order_numeric_0.pass.cpp b/pstl/test/pstl/header_inclusion_order_numeric_0.pass.cpp
index 79086e70ed3eb..4b1865729bffc 100644
--- a/pstl/test/pstl/header_inclusion_order_numeric_0.pass.cpp
+++ b/pstl/test/pstl/header_inclusion_order_numeric_0.pass.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14
 
 #include "support/pstl_test_config.h"
 
diff --git a/pstl/test/pstl/header_inclusion_order_numeric_1.pass.cpp b/pstl/test/pstl/header_inclusion_order_numeric_1.pass.cpp
index a9b32c8225c2d..a0bb24e554aed 100644
--- a/pstl/test/pstl/header_inclusion_order_numeric_1.pass.cpp
+++ b/pstl/test/pstl/header_inclusion_order_numeric_1.pass.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14
 
 #include "support/pstl_test_config.h"
 
diff --git a/pstl/test/std/algorithms/alg.merge/inplace_merge.pass.cpp b/pstl/test/std/algorithms/alg.merge/inplace_merge.pass.cpp
index f2cc4040bb1e9..8e49bf4b719a9 100644
--- a/pstl/test/std/algorithms/alg.merge/inplace_merge.pass.cpp
+++ b/pstl/test/std/algorithms/alg.merge/inplace_merge.pass.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14
 
 #include "support/pstl_test_config.h"
 
@@ -153,7 +153,7 @@ main()
         [](const MemoryChecker& val1, const MemoryChecker& val2){ return val1.value() == val2.value(); });
     EXPECT_FALSE(MemoryChecker::alive_objects() < 0, "wrong effect from inplace_merge: number of ctors calls < num of dtors calls");
     EXPECT_FALSE(MemoryChecker::alive_objects() > 0, "wrong effect from inplace_merge: number of ctors calls > num of dtors calls");
-    
+
     std::cout << done() << std::endl;
     return 0;
 }
diff --git a/pstl/test/std/algorithms/alg.merge/merge.pass.cpp b/pstl/test/std/algorithms/alg.merge/merge.pass.cpp
index d2381036cda07..c8970e0fb5c39 100644
--- a/pstl/test/std/algorithms/alg.merge/merge.pass.cpp
+++ b/pstl/test/std/algorithms/alg.merge/merge.pass.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14
 
 #include "support/pstl_test_config.h"
 
diff --git a/pstl/test/std/algorithms/alg.modifying.operations/alg.copy/copy_if.pass.cpp b/pstl/test/std/algorithms/alg.modifying.operations/alg.copy/copy_if.pass.cpp
index aeafb020f65fa..a21c2231c6663 100644
--- a/pstl/test/std/algorithms/alg.modifying.operations/alg.copy/copy_if.pass.cpp
+++ b/pstl/test/std/algorithms/alg.modifying.operations/alg.copy/copy_if.pass.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14
 
 // Tests for copy_if and remove_copy_if
 #include "support/pstl_test_config.h"
diff --git a/pstl/test/std/algorithms/alg.modifying.operations/alg.partitions/is_partitioned.pass.cpp b/pstl/test/std/algorithms/alg.modifying.operations/alg.partitions/is_partitioned.pass.cpp
index e5856a13b07b0..844606f119f37 100644
--- a/pstl/test/std/algorithms/alg.modifying.operations/alg.partitions/is_partitioned.pass.cpp
+++ b/pstl/test/std/algorithms/alg.modifying.operations/alg.partitions/is_partitioned.pass.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14
 
 #include "support/pstl_test_config.h"
 
diff --git a/pstl/test/std/algorithms/alg.modifying.operations/alg.partitions/partition.pass.cpp b/pstl/test/std/algorithms/alg.modifying.operations/alg.partitions/partition.pass.cpp
index c376f895664b9..159c9bf5bdab8 100644
--- a/pstl/test/std/algorithms/alg.modifying.operations/alg.partitions/partition.pass.cpp
+++ b/pstl/test/std/algorithms/alg.modifying.operations/alg.partitions/partition.pass.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14
 
 // Tests for stable_partition and partition
 #include "support/pstl_test_config.h"
diff --git a/pstl/test/std/algorithms/alg.modifying.operations/alg.partitions/partition_copy.pass.cpp b/pstl/test/std/algorithms/alg.modifying.operations/alg.partitions/partition_copy.pass.cpp
index 1a06c55d3566f..29e7aa578a2ed 100644
--- a/pstl/test/std/algorithms/alg.modifying.operations/alg.partitions/partition_copy.pass.cpp
+++ b/pstl/test/std/algorithms/alg.modifying.operations/alg.partitions/partition_copy.pass.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14
 
 // Tests for stable_partition and partition_copy
 #include "support/pstl_test_config.h"
diff --git a/pstl/test/std/algorithms/alg.modifying.operations/alg.reverse/reverse.pass.cpp b/pstl/test/std/algorithms/alg.modifying.operations/alg.reverse/reverse.pass.cpp
index ab51504dd3c99..d2a6079f4407c 100644
--- a/pstl/test/std/algorithms/alg.modifying.operations/alg.reverse/reverse.pass.cpp
+++ b/pstl/test/std/algorithms/alg.modifying.operations/alg.reverse/reverse.pass.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14
 
 #include "support/pstl_test_config.h"
 
diff --git a/pstl/test/std/algorithms/alg.modifying.operations/alg.reverse/reverse_copy.pass.cpp b/pstl/test/std/algorithms/alg.modifying.operations/alg.reverse/reverse_copy.pass.cpp
index 8c79de7824410..b7358ce7f7b95 100644
--- a/pstl/test/std/algorithms/alg.modifying.operations/alg.reverse/reverse_copy.pass.cpp
+++ b/pstl/test/std/algorithms/alg.modifying.operations/alg.reverse/reverse_copy.pass.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14
 
 #include "support/pstl_test_config.h"
 
diff --git a/pstl/test/std/algorithms/alg.modifying.operations/copy_move.pass.cpp b/pstl/test/std/algorithms/alg.modifying.operations/copy_move.pass.cpp
index 5bef48e625bb9..5fe195e575082 100644
--- a/pstl/test/std/algorithms/alg.modifying.operations/copy_move.pass.cpp
+++ b/pstl/test/std/algorithms/alg.modifying.operations/copy_move.pass.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14
 
 // Tests for copy, move and copy_n
 #include "support/pstl_test_config.h"
diff --git a/pstl/test/std/algorithms/alg.modifying.operations/fill.pass.cpp b/pstl/test/std/algorithms/alg.modifying.operations/fill.pass.cpp
index 5d1715705c604..d44a1a483c197 100644
--- a/pstl/test/std/algorithms/alg.modifying.operations/fill.pass.cpp
+++ b/pstl/test/std/algorithms/alg.modifying.operations/fill.pass.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14
 
 #include "support/pstl_test_config.h"
 
diff --git a/pstl/test/std/algorithms/alg.modifying.operations/generate.pass.cpp b/pstl/test/std/algorithms/alg.modifying.operations/generate.pass.cpp
index 4561f3c6dcefd..92d87ea26e541 100644
--- a/pstl/test/std/algorithms/alg.modifying.operations/generate.pass.cpp
+++ b/pstl/test/std/algorithms/alg.modifying.operations/generate.pass.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14
 
 #include "support/pstl_test_config.h"
 
diff --git a/pstl/test/std/algorithms/alg.modifying.operations/remove.pass.cpp b/pstl/test/std/algorithms/alg.modifying.operations/remove.pass.cpp
index c41a31d7310d8..5bb3cf2ca7597 100644
--- a/pstl/test/std/algorithms/alg.modifying.operations/remove.pass.cpp
+++ b/pstl/test/std/algorithms/alg.modifying.operations/remove.pass.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14
 
 // Test for remove, remove_if
 #include "support/pstl_test_config.h"
@@ -155,7 +155,7 @@ main()
     );
     EXPECT_FALSE(MemoryChecker::alive_objects() < 0, "wrong effect from remove,remove_if: number of ctors calls < num of dtors calls");
     EXPECT_FALSE(MemoryChecker::alive_objects() > 0, "wrong effect from remove,remove_if: number of ctors calls > num of dtors calls");
-    
+
     std::cout << done() << std::endl;
     return 0;
 }
diff --git a/pstl/test/std/algorithms/alg.modifying.operations/remove_copy.pass.cpp b/pstl/test/std/algorithms/alg.modifying.operations/remove_copy.pass.cpp
index 969343b3764cd..9cdd7d62a4f24 100644
--- a/pstl/test/std/algorithms/alg.modifying.operations/remove_copy.pass.cpp
+++ b/pstl/test/std/algorithms/alg.modifying.operations/remove_copy.pass.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14
 
 #include "support/pstl_test_config.h"
 
diff --git a/pstl/test/std/algorithms/alg.modifying.operations/replace.pass.cpp b/pstl/test/std/algorithms/alg.modifying.operations/replace.pass.cpp
index d42dae76ebd1b..7b9bb1221ee13 100644
--- a/pstl/test/std/algorithms/alg.modifying.operations/replace.pass.cpp
+++ b/pstl/test/std/algorithms/alg.modifying.operations/replace.pass.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14
 
 #include "support/pstl_test_config.h"
 
diff --git a/pstl/test/std/algorithms/alg.modifying.operations/replace_copy.pass.cpp b/pstl/test/std/algorithms/alg.modifying.operations/replace_copy.pass.cpp
index 4414d1cbd3532..0ebff672b0979 100644
--- a/pstl/test/std/algorithms/alg.modifying.operations/replace_copy.pass.cpp
+++ b/pstl/test/std/algorithms/alg.modifying.operations/replace_copy.pass.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14
 
 // Tests for replace_copy and replace_copy_if
 
diff --git a/pstl/test/std/algorithms/alg.modifying.operations/rotate.pass.cpp b/pstl/test/std/algorithms/alg.modifying.operations/rotate.pass.cpp
index 03f2b3cb2e872..5402f792dfb2c 100644
--- a/pstl/test/std/algorithms/alg.modifying.operations/rotate.pass.cpp
+++ b/pstl/test/std/algorithms/alg.modifying.operations/rotate.pass.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14
 
 #include "support/pstl_test_config.h"
 
diff --git a/pstl/test/std/algorithms/alg.modifying.operations/rotate_copy.pass.cpp b/pstl/test/std/algorithms/alg.modifying.operations/rotate_copy.pass.cpp
index 3749492e2a28a..b18fd0ea60c55 100644
--- a/pstl/test/std/algorithms/alg.modifying.operations/rotate_copy.pass.cpp
+++ b/pstl/test/std/algorithms/alg.modifying.operations/rotate_copy.pass.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14
 
 #include "support/pstl_test_config.h"
 
diff --git a/pstl/test/std/algorithms/alg.modifying.operations/swap_ranges.pass.cpp b/pstl/test/std/algorithms/alg.modifying.operations/swap_ranges.pass.cpp
index 2c8f652fdefd9..c3c3693af5942 100644
--- a/pstl/test/std/algorithms/alg.modifying.operations/swap_ranges.pass.cpp
+++ b/pstl/test/std/algorithms/alg.modifying.operations/swap_ranges.pass.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14
 
 #include "support/pstl_test_config.h"
 
diff --git a/pstl/test/std/algorithms/alg.modifying.operations/transform_binary.pass.cpp b/pstl/test/std/algorithms/alg.modifying.operations/transform_binary.pass.cpp
index d0f8c68afa726..949e6f54ffb41 100644
--- a/pstl/test/std/algorithms/alg.modifying.operations/transform_binary.pass.cpp
+++ b/pstl/test/std/algorithms/alg.modifying.operations/transform_binary.pass.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14
 
 #include "support/pstl_test_config.h"
 
diff --git a/pstl/test/std/algorithms/alg.modifying.operations/transform_unary.pass.cpp b/pstl/test/std/algorithms/alg.modifying.operations/transform_unary.pass.cpp
index 55726db0c2764..69a410c4cd6eb 100644
--- a/pstl/test/std/algorithms/alg.modifying.operations/transform_unary.pass.cpp
+++ b/pstl/test/std/algorithms/alg.modifying.operations/transform_unary.pass.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14
 
 #include "support/pstl_test_config.h"
 
diff --git a/pstl/test/std/algorithms/alg.modifying.operations/unique.pass.cpp b/pstl/test/std/algorithms/alg.modifying.operations/unique.pass.cpp
index 2f68c00401fee..b33bb80c4304a 100644
--- a/pstl/test/std/algorithms/alg.modifying.operations/unique.pass.cpp
+++ b/pstl/test/std/algorithms/alg.modifying.operations/unique.pass.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14
 
 // Test for unique
 #include "support/pstl_test_config.h"
@@ -157,7 +157,7 @@ main()
         [](const MemoryChecker& val1, const MemoryChecker& val2){ return val1.value() == val2.value(); });
     EXPECT_FALSE(MemoryChecker::alive_objects() < 0, "wrong effect from unique: number of ctors calls < num of dtors calls");
     EXPECT_FALSE(MemoryChecker::alive_objects() > 0, "wrong effect from unique: number of ctors calls > num of dtors calls");
-    
+
     std::cout << done() << std::endl;
     return 0;
 }
diff --git a/pstl/test/std/algorithms/alg.modifying.operations/unique_copy_equal.pass.cpp b/pstl/test/std/algorithms/alg.modifying.operations/unique_copy_equal.pass.cpp
index b1590ef0b14ce..e61a448be41d2 100644
--- a/pstl/test/std/algorithms/alg.modifying.operations/unique_copy_equal.pass.cpp
+++ b/pstl/test/std/algorithms/alg.modifying.operations/unique_copy_equal.pass.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14
 
 // Tests for unique_copy
 #include "support/pstl_test_config.h"
diff --git a/pstl/test/std/algorithms/alg.nonmodifying/adjacent_find.pass.cpp b/pstl/test/std/algorithms/alg.nonmodifying/adjacent_find.pass.cpp
index 8f00351dcd077..36a623bac42a4 100644
--- a/pstl/test/std/algorithms/alg.nonmodifying/adjacent_find.pass.cpp
+++ b/pstl/test/std/algorithms/alg.nonmodifying/adjacent_find.pass.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14
 
 #include "support/pstl_test_config.h"
 
diff --git a/pstl/test/std/algorithms/alg.nonmodifying/all_of.pass.cpp b/pstl/test/std/algorithms/alg.nonmodifying/all_of.pass.cpp
index 97e41441ac4f7..fb5e78c266e1f 100644
--- a/pstl/test/std/algorithms/alg.nonmodifying/all_of.pass.cpp
+++ b/pstl/test/std/algorithms/alg.nonmodifying/all_of.pass.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14
 
 #include "support/pstl_test_config.h"
 
diff --git a/pstl/test/std/algorithms/alg.nonmodifying/any_of.pass.cpp b/pstl/test/std/algorithms/alg.nonmodifying/any_of.pass.cpp
index 1430de9b63db3..82e8975c344fe 100644
--- a/pstl/test/std/algorithms/alg.nonmodifying/any_of.pass.cpp
+++ b/pstl/test/std/algorithms/alg.nonmodifying/any_of.pass.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14
 
 #include "support/pstl_test_config.h"
 
diff --git a/pstl/test/std/algorithms/alg.nonmodifying/count.pass.cpp b/pstl/test/std/algorithms/alg.nonmodifying/count.pass.cpp
index 7ec77f3760485..68f557d673347 100644
--- a/pstl/test/std/algorithms/alg.nonmodifying/count.pass.cpp
+++ b/pstl/test/std/algorithms/alg.nonmodifying/count.pass.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14
 
 // Tests for count and count_if
 #include "support/pstl_test_config.h"
diff --git a/pstl/test/std/algorithms/alg.nonmodifying/equal.pass.cpp b/pstl/test/std/algorithms/alg.nonmodifying/equal.pass.cpp
index 45ec237a4b2cd..a7ac2397e63b9 100644
--- a/pstl/test/std/algorithms/alg.nonmodifying/equal.pass.cpp
+++ b/pstl/test/std/algorithms/alg.nonmodifying/equal.pass.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14
 
 #include "support/pstl_test_config.h"
 
diff --git a/pstl/test/std/algorithms/alg.nonmodifying/find.pass.cpp b/pstl/test/std/algorithms/alg.nonmodifying/find.pass.cpp
index f5a5d94f5e411..f09b240603987 100644
--- a/pstl/test/std/algorithms/alg.nonmodifying/find.pass.cpp
+++ b/pstl/test/std/algorithms/alg.nonmodifying/find.pass.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14
 
 // Tests for find
 #include "support/pstl_test_config.h"
diff --git a/pstl/test/std/algorithms/alg.nonmodifying/find_end.pass.cpp b/pstl/test/std/algorithms/alg.nonmodifying/find_end.pass.cpp
index 20826ae064e27..c72d6239273ec 100644
--- a/pstl/test/std/algorithms/alg.nonmodifying/find_end.pass.cpp
+++ b/pstl/test/std/algorithms/alg.nonmodifying/find_end.pass.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14
 
 #include "support/pstl_test_config.h"
 
diff --git a/pstl/test/std/algorithms/alg.nonmodifying/find_first_of.pass.cpp b/pstl/test/std/algorithms/alg.nonmodifying/find_first_of.pass.cpp
index 84c35e95cfc54..d2f47d046278e 100644
--- a/pstl/test/std/algorithms/alg.nonmodifying/find_first_of.pass.cpp
+++ b/pstl/test/std/algorithms/alg.nonmodifying/find_first_of.pass.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14
 
 #include "support/pstl_test_config.h"
 
diff --git a/pstl/test/std/algorithms/alg.nonmodifying/find_if.pass.cpp b/pstl/test/std/algorithms/alg.nonmodifying/find_if.pass.cpp
index 3dd41c07288ea..0a9e3aacc704f 100644
--- a/pstl/test/std/algorithms/alg.nonmodifying/find_if.pass.cpp
+++ b/pstl/test/std/algorithms/alg.nonmodifying/find_if.pass.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14
 
 // Tests for find_if and find_if_not
 #include "support/pstl_test_config.h"
diff --git a/pstl/test/std/algorithms/alg.nonmodifying/for_each.pass.cpp b/pstl/test/std/algorithms/alg.nonmodifying/for_each.pass.cpp
index 3aae4a4aaefc6..14a0ad4f32b12 100644
--- a/pstl/test/std/algorithms/alg.nonmodifying/for_each.pass.cpp
+++ b/pstl/test/std/algorithms/alg.nonmodifying/for_each.pass.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14
 
 #include "support/pstl_test_config.h"
 
diff --git a/pstl/test/std/algorithms/alg.nonmodifying/mismatch.pass.cpp b/pstl/test/std/algorithms/alg.nonmodifying/mismatch.pass.cpp
index 109ac4b825670..4d83ad6de7824 100644
--- a/pstl/test/std/algorithms/alg.nonmodifying/mismatch.pass.cpp
+++ b/pstl/test/std/algorithms/alg.nonmodifying/mismatch.pass.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14
 
 #include "support/pstl_test_config.h"
 
diff --git a/pstl/test/std/algorithms/alg.nonmodifying/none_of.pass.cpp b/pstl/test/std/algorithms/alg.nonmodifying/none_of.pass.cpp
index e84de3cbaf08f..772c42e536f9e 100644
--- a/pstl/test/std/algorithms/alg.nonmodifying/none_of.pass.cpp
+++ b/pstl/test/std/algorithms/alg.nonmodifying/none_of.pass.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14
 
 #include "support/pstl_test_config.h"
 
diff --git a/pstl/test/std/algorithms/alg.nonmodifying/nth_element.pass.cpp b/pstl/test/std/algorithms/alg.nonmodifying/nth_element.pass.cpp
index 71009ac285ad8..cf87085ef2139 100644
--- a/pstl/test/std/algorithms/alg.nonmodifying/nth_element.pass.cpp
+++ b/pstl/test/std/algorithms/alg.nonmodifying/nth_element.pass.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14
 
 #include "support/pstl_test_config.h"
 
diff --git a/pstl/test/std/algorithms/alg.nonmodifying/search_n.pass.cpp b/pstl/test/std/algorithms/alg.nonmodifying/search_n.pass.cpp
index 9bcea249a2ea8..3e7868a11e5dc 100644
--- a/pstl/test/std/algorithms/alg.nonmodifying/search_n.pass.cpp
+++ b/pstl/test/std/algorithms/alg.nonmodifying/search_n.pass.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14
 
 #include "support/pstl_test_config.h"
 
diff --git a/pstl/test/std/algorithms/alg.sorting/alg.heap.operations/is_heap.pass.cpp b/pstl/test/std/algorithms/alg.sorting/alg.heap.operations/is_heap.pass.cpp
index 78e7a3079dc49..1fe1823136958 100644
--- a/pstl/test/std/algorithms/alg.sorting/alg.heap.operations/is_heap.pass.cpp
+++ b/pstl/test/std/algorithms/alg.sorting/alg.heap.operations/is_heap.pass.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14
 
 // Tests for is_heap, is_heap_until
 #include "support/pstl_test_config.h"
diff --git a/pstl/test/std/algorithms/alg.sorting/alg.lex.comparison/lexicographical_compare.pass.cpp b/pstl/test/std/algorithms/alg.sorting/alg.lex.comparison/lexicographical_compare.pass.cpp
index abd10667cad44..595c53276f7b5 100644
--- a/pstl/test/std/algorithms/alg.sorting/alg.lex.comparison/lexicographical_compare.pass.cpp
+++ b/pstl/test/std/algorithms/alg.sorting/alg.lex.comparison/lexicographical_compare.pass.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14
 
 #include "support/pstl_test_config.h"
 
diff --git a/pstl/test/std/algorithms/alg.sorting/alg.min.max/minmax_element.pass.cpp b/pstl/test/std/algorithms/alg.sorting/alg.min.max/minmax_element.pass.cpp
index a1a29b7b646a7..e1f5051a0e6fa 100644
--- a/pstl/test/std/algorithms/alg.sorting/alg.min.max/minmax_element.pass.cpp
+++ b/pstl/test/std/algorithms/alg.sorting/alg.min.max/minmax_element.pass.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14
 
 #include "support/pstl_test_config.h"
 
diff --git a/pstl/test/std/algorithms/alg.sorting/alg.set.operations/includes.pass.cpp b/pstl/test/std/algorithms/alg.sorting/alg.set.operations/includes.pass.cpp
index 8712f173f10da..778d265f9ca7f 100644
--- a/pstl/test/std/algorithms/alg.sorting/alg.set.operations/includes.pass.cpp
+++ b/pstl/test/std/algorithms/alg.sorting/alg.set.operations/includes.pass.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14
 
 #include "support/pstl_test_config.h"
 
diff --git a/pstl/test/std/algorithms/alg.sorting/alg.set.operations/set.pass.cpp b/pstl/test/std/algorithms/alg.sorting/alg.set.operations/set.pass.cpp
index 4b2b93dc8e30c..40ae3d92c8a03 100644
--- a/pstl/test/std/algorithms/alg.sorting/alg.set.operations/set.pass.cpp
+++ b/pstl/test/std/algorithms/alg.sorting/alg.set.operations/set.pass.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14
 
 #include "support/pstl_test_config.h"
 
diff --git a/pstl/test/std/algorithms/alg.sorting/is_sorted.pass.cpp b/pstl/test/std/algorithms/alg.sorting/is_sorted.pass.cpp
index 3a48160fc17bc..d4c6fe291ed84 100644
--- a/pstl/test/std/algorithms/alg.sorting/is_sorted.pass.cpp
+++ b/pstl/test/std/algorithms/alg.sorting/is_sorted.pass.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14
 
 #include "support/pstl_test_config.h"
 
diff --git a/pstl/test/std/algorithms/alg.sorting/partial_sort.pass.cpp b/pstl/test/std/algorithms/alg.sorting/partial_sort.pass.cpp
index e378cb491ab6b..bf6e68be8c079 100644
--- a/pstl/test/std/algorithms/alg.sorting/partial_sort.pass.cpp
+++ b/pstl/test/std/algorithms/alg.sorting/partial_sort.pass.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14
 
 #include "support/pstl_test_config.h"
 
diff --git a/pstl/test/std/algorithms/alg.sorting/partial_sort_copy.pass.cpp b/pstl/test/std/algorithms/alg.sorting/partial_sort_copy.pass.cpp
index ad0f0d6ef9b05..43afee0bad300 100644
--- a/pstl/test/std/algorithms/alg.sorting/partial_sort_copy.pass.cpp
+++ b/pstl/test/std/algorithms/alg.sorting/partial_sort_copy.pass.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14
 
 // Tests for partial_sort_copy
 #include "support/pstl_test_config.h"
@@ -190,7 +190,7 @@ main()
         [](const MemoryChecker& val1, const MemoryChecker& val2){ return val1.value() < val2.value(); });
     EXPECT_FALSE(MemoryChecker::alive_objects() < 0, "wrong effect from partial_sort_copy: number of ctors calls < num of dtors calls");
     EXPECT_FALSE(MemoryChecker::alive_objects() > 0, "wrong effect from partial_sort_copy: number of ctors calls > num of dtors calls");
-    
+
     std::cout << done() << std::endl;
     return 0;
 }
diff --git a/pstl/test/std/algorithms/alg.sorting/sort.pass.cpp b/pstl/test/std/algorithms/alg.sorting/sort.pass.cpp
index add69d66bc2d2..5617a6019f801 100644
--- a/pstl/test/std/algorithms/alg.sorting/sort.pass.cpp
+++ b/pstl/test/std/algorithms/alg.sorting/sort.pass.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14
 
 #include "support/pstl_test_config.h"
 
diff --git a/pstl/test/std/numerics/numeric.ops/adjacent_difference.pass.cpp b/pstl/test/std/numerics/numeric.ops/adjacent_difference.pass.cpp
index 65251550381f8..d96da21f0cce9 100644
--- a/pstl/test/std/numerics/numeric.ops/adjacent_difference.pass.cpp
+++ b/pstl/test/std/numerics/numeric.ops/adjacent_difference.pass.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14
 
 #include "support/pstl_test_config.h"
 
diff --git a/pstl/test/std/numerics/numeric.ops/reduce.pass.cpp b/pstl/test/std/numerics/numeric.ops/reduce.pass.cpp
index 743208fd19c44..b8c4598f9a278 100644
--- a/pstl/test/std/numerics/numeric.ops/reduce.pass.cpp
+++ b/pstl/test/std/numerics/numeric.ops/reduce.pass.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14
 
 #include "support/pstl_test_config.h"
 
diff --git a/pstl/test/std/numerics/numeric.ops/scan.fail.cpp b/pstl/test/std/numerics/numeric.ops/scan.fail.cpp
index 13b2c9b751fec..8202dc2207c14 100644
--- a/pstl/test/std/numerics/numeric.ops/scan.fail.cpp
+++ b/pstl/test/std/numerics/numeric.ops/scan.fail.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14
 
 #include <execution>
 #include <numeric>
diff --git a/pstl/test/std/numerics/numeric.ops/scan.pass.cpp b/pstl/test/std/numerics/numeric.ops/scan.pass.cpp
index 64156f001a5ca..b9eef530f5f59 100644
--- a/pstl/test/std/numerics/numeric.ops/scan.pass.cpp
+++ b/pstl/test/std/numerics/numeric.ops/scan.pass.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14
 
 #include "support/pstl_test_config.h"
 
diff --git a/pstl/test/std/numerics/numeric.ops/transform_reduce.pass.cpp b/pstl/test/std/numerics/numeric.ops/transform_reduce.pass.cpp
index dd8bb875e8e30..9aec75a9a8e05 100644
--- a/pstl/test/std/numerics/numeric.ops/transform_reduce.pass.cpp
+++ b/pstl/test/std/numerics/numeric.ops/transform_reduce.pass.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14
 
 #include "support/pstl_test_config.h"
 
diff --git a/pstl/test/std/numerics/numeric.ops/transform_scan.pass.cpp b/pstl/test/std/numerics/numeric.ops/transform_scan.pass.cpp
index a7dd44a8be2f9..1c9b470aa052d 100644
--- a/pstl/test/std/numerics/numeric.ops/transform_scan.pass.cpp
+++ b/pstl/test/std/numerics/numeric.ops/transform_scan.pass.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14
 
 #include "support/pstl_test_config.h"
 
diff --git a/pstl/test/std/utilities/memory/specialized.algorithms/uninitialized_construct.pass.cpp b/pstl/test/std/utilities/memory/specialized.algorithms/uninitialized_construct.pass.cpp
index c9c1e79fb40d0..23c76130bd624 100644
--- a/pstl/test/std/utilities/memory/specialized.algorithms/uninitialized_construct.pass.cpp
+++ b/pstl/test/std/utilities/memory/specialized.algorithms/uninitialized_construct.pass.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14
 
 // Tests for uninitialized_default_construct, uninitialized_default_construct_n,
 //           uninitialized_value_construct,   uninitialized_value_construct_n
diff --git a/pstl/test/std/utilities/memory/specialized.algorithms/uninitialized_copy_move.pass.cpp b/pstl/test/std/utilities/memory/specialized.algorithms/uninitialized_copy_move.pass.cpp
index d1c9e3a7a0716..718be4e1e772c 100644
--- a/pstl/test/std/utilities/memory/specialized.algorithms/uninitialized_copy_move.pass.cpp
+++ b/pstl/test/std/utilities/memory/specialized.algorithms/uninitialized_copy_move.pass.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14
 
 // Tests for uninitialized_copy, uninitialized_copy_n, uninitialized_move, uninitialized_move_n
 
diff --git a/pstl/test/std/utilities/memory/specialized.algorithms/uninitialized_fill_destroy.pass.cpp b/pstl/test/std/utilities/memory/specialized.algorithms/uninitialized_fill_destroy.pass.cpp
index fe85f32a1f7b3..d579ea558839d 100644
--- a/pstl/test/std/utilities/memory/specialized.algorithms/uninitialized_fill_destroy.pass.cpp
+++ b/pstl/test/std/utilities/memory/specialized.algorithms/uninitialized_fill_destroy.pass.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03, c++11, c++14
+// UNSUPPORTED: c++03, c++11, c++14
 
 #include "support/pstl_test_config.h"
 

From 82b6dec0ed1dd5a4fa5484009c42fa25e603a875 Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne@apple.com>
Date: Wed, 29 Jul 2020 14:17:57 -0400
Subject: [PATCH 0510/1035] [libc++] Remove c++98 from the possible Standards
 of the test suite

Clang treats C++98 and C++03 as the same anyway, so it's no use having
two different settings for the same standard.
---
 libcxx/utils/libcxx/test/params.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcxx/utils/libcxx/test/params.py b/libcxx/utils/libcxx/test/params.py
index a9431ec073f8d..773890b015097 100644
--- a/libcxx/utils/libcxx/test/params.py
+++ b/libcxx/utils/libcxx/test/params.py
@@ -8,7 +8,7 @@
 
 from libcxx.test.dsl import *
 
-_allStandards = ['c++98', 'c++03', 'c++11', 'c++14', 'c++17', 'c++2a']
+_allStandards = ['c++03', 'c++11', 'c++14', 'c++17', 'c++2a']
 
 DEFAULT_PARAMETERS = [
   # Core parameters of the test suite

From dc2867576886247cbe351e7c63618c09ab6af808 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Wed, 29 Jul 2020 19:17:24 +0100
Subject: [PATCH 0511/1035] Revert "[SCEVExpander] Add option to preserve LCSSA
 directly."

This reverts commit 99166fd4fb422351f131fb1265cb85d5f6c5b8da, because it
breaks the polly builders.

polly/test/Isl/CodeGen/invariant_load_escaping_second_scop.ll fails
because a apparently unnecessary LCSSA phi node is introduced.

Make the bots green again, while I take a closer look.
---
 .../Utils/ScalarEvolutionExpander.h           |  41 +----
 .../Transforms/Scalar/LoopStrengthReduce.cpp  |   6 +-
 .../Utils/ScalarEvolutionExpander.cpp         | 173 +++++-------------
 .../Utils/ScalarEvolutionExpanderTest.cpp     |   7 +-
 4 files changed, 61 insertions(+), 166 deletions(-)

diff --git a/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h b/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h
index cb212b2705eed..6e53d15d25fc7 100644
--- a/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h
+++ b/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h
@@ -52,9 +52,6 @@ class SCEVExpander : public SCEVVisitor<SCEVExpander, Value *> {
   // New instructions receive a name to identify them with the current pass.
   const char *IVName;
 
-  /// Indicates whether LCSSA phis should be created for inserted values.
-  bool PreserveLCSSA;
-
   // InsertedExpressions caches Values for reuse, so must track RAUW.
   DenseMap<std::pair<const SCEV *, Instruction *>, TrackingVH<Value>>
       InsertedExpressions;
@@ -149,10 +146,9 @@ class SCEVExpander : public SCEVVisitor<SCEVExpander, Value *> {
 public:
   /// Construct a SCEVExpander in "canonical" mode.
   explicit SCEVExpander(ScalarEvolution &se, const DataLayout &DL,
-                        const char *name, bool PreserveLCSSA = true)
-      : SE(se), DL(DL), IVName(name), PreserveLCSSA(PreserveLCSSA),
-        IVIncInsertLoop(nullptr), IVIncInsertPos(nullptr), CanonicalMode(true),
-        LSRMode(false),
+                        const char *name)
+      : SE(se), DL(DL), IVName(name), IVIncInsertLoop(nullptr),
+        IVIncInsertPos(nullptr), CanonicalMode(true), LSRMode(false),
         Builder(se.getContext(), TargetFolder(DL),
                 IRBuilderCallbackInserter(
                     [this](Instruction *I) { rememberInstruction(I); })) {
@@ -227,18 +223,14 @@ class SCEVExpander : public SCEVVisitor<SCEVExpander, Value *> {
                                const TargetTransformInfo *TTI = nullptr);
 
   /// Insert code to directly compute the specified SCEV expression into the
-  /// program.  The code is inserted into the specified block.
-  Value *expandCodeFor(const SCEV *SH, Type *Ty, Instruction *I) {
-    return expandCodeForImpl(SH, Ty, I, true);
-  }
+  /// program.  The inserted code is inserted into the specified block.
+  Value *expandCodeFor(const SCEV *SH, Type *Ty, Instruction *I);
 
   /// Insert code to directly compute the specified SCEV expression into the
-  /// program.  The code is inserted into the SCEVExpander's current
+  /// program.  The inserted code is inserted into the SCEVExpander's current
   /// insertion point. If a type is specified, the result will be expanded to
   /// have that type, with a cast if necessary.
-  Value *expandCodeFor(const SCEV *SH, Type *Ty = nullptr) {
-    return expandCodeForImpl(SH, Ty, true);
-  }
+  Value *expandCodeFor(const SCEV *SH, Type *Ty = nullptr);
 
   /// Generates a code sequence that evaluates this predicate.  The inserted
   /// instructions will be at position \p Loc.  The result will be of type i1
@@ -346,20 +338,6 @@ class SCEVExpander : public SCEVVisitor<SCEVExpander, Value *> {
 private:
   LLVMContext &getContext() const { return SE.getContext(); }
 
-  /// Insert code to directly compute the specified SCEV expression into the
-  /// program. The code is inserted into the SCEVExpander's current
-  /// insertion point. If a type is specified, the result will be expanded to
-  /// have that type, with a cast if necessary. If \p Root is true, this
-  /// indicates that \p SH is the top-level expression to expand passed from
-  /// an external client call.
-  Value *expandCodeForImpl(const SCEV *SH, Type *Ty, bool Root);
-
-  /// Insert code to directly compute the specified SCEV expression into the
-  /// program. The code is inserted into the specified block. If \p
-  /// Root is true, this indicates that \p SH is the top-level expression to
-  /// expand passed from an external client call.
-  Value *expandCodeForImpl(const SCEV *SH, Type *Ty, Instruction *I, bool Root);
-
   /// Recursive helper function for isHighCostExpansion.
   bool isHighCostExpansionHelper(const SCEV *S, Loop *L, const Instruction &At,
                                  int &BudgetRemaining,
@@ -441,11 +419,6 @@ class SCEVExpander : public SCEVVisitor<SCEVExpander, Value *> {
                       Instruction *Pos, PHINode *LoopPhi);
 
   void fixupInsertPoints(Instruction *I);
-
-  /// If required, create LCSSA PHIs for \p Users' operand \p OpIdx. If new
-  /// LCSSA PHIs have been created, return the LCSSA PHI available at \p User.
-  /// If no PHIs have been created, return the unchanged operand \p OpIdx.
-  Value *fixupLCSSAFormFor(Instruction *User, unsigned OpIdx);
 };
 } // namespace llvm
 
diff --git a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
index c3e46c1fadef3..cf02ef1e83f3f 100644
--- a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
@@ -5514,8 +5514,8 @@ void LSRInstance::ImplementSolution(
   // we can remove them after we are done working.
   SmallVector<WeakTrackingVH, 16> DeadInsts;
 
-  SCEVExpander Rewriter(SE, L->getHeader()->getModule()->getDataLayout(), "lsr",
-                        false);
+  SCEVExpander Rewriter(SE, L->getHeader()->getModule()->getDataLayout(),
+                        "lsr");
 #ifndef NDEBUG
   Rewriter.setDebugType(DEBUG_TYPE);
 #endif
@@ -5780,7 +5780,7 @@ static bool ReduceLoopStrength(Loop *L, IVUsers &IU, ScalarEvolution &SE,
   if (EnablePhiElim && L->isLoopSimplifyForm()) {
     SmallVector<WeakTrackingVH, 16> DeadInsts;
     const DataLayout &DL = L->getHeader()->getModule()->getDataLayout();
-    SCEVExpander Rewriter(SE, DL, "lsr", false);
+    SCEVExpander Rewriter(SE, DL, "lsr");
 #ifndef NDEBUG
     Rewriter.setDebugType(DEBUG_TYPE);
 #endif
diff --git a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
index 1a10e580c68c0..c99d612a7768f 100644
--- a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
+++ b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
@@ -27,7 +27,6 @@
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
-#include "llvm/Transforms/Utils/LoopUtils.h"
 
 using namespace llvm;
 
@@ -462,10 +461,9 @@ Value *SCEVExpander::expandAddToGEP(const SCEV *const *op_begin,
     // we didn't find any operands that could be factored, tentatively
     // assume that element zero was selected (since the zero offset
     // would obviously be folded away).
-    Value *Scaled =
-        ScaledOps.empty()
-            ? Constant::getNullValue(Ty)
-            : expandCodeForImpl(SE.getAddExpr(ScaledOps), Ty, false);
+    Value *Scaled = ScaledOps.empty() ?
+                    Constant::getNullValue(Ty) :
+                    expandCodeFor(SE.getAddExpr(ScaledOps), Ty);
     GepIndices.push_back(Scaled);
 
     // Collect struct field index operands.
@@ -524,7 +522,7 @@ Value *SCEVExpander::expandAddToGEP(const SCEV *const *op_begin,
            SE.DT.dominates(cast<Instruction>(V), &*Builder.GetInsertPoint()));
 
     // Expand the operands for a plain byte offset.
-    Value *Idx = expandCodeForImpl(SE.getAddExpr(Ops), Ty, false);
+    Value *Idx = expandCodeFor(SE.getAddExpr(Ops), Ty);
 
     // Fold a GEP with constant operands.
     if (Constant *CLHS = dyn_cast<Constant>(V))
@@ -745,14 +743,14 @@ Value *SCEVExpander::visitAddExpr(const SCEVAddExpr *S) {
       Sum = expandAddToGEP(NewOps.begin(), NewOps.end(), PTy, Ty, expand(Op));
     } else if (Op->isNonConstantNegative()) {
       // Instead of doing a negate and add, just do a subtract.
-      Value *W = expandCodeForImpl(SE.getNegativeSCEV(Op), Ty, false);
+      Value *W = expandCodeFor(SE.getNegativeSCEV(Op), Ty);
       Sum = InsertNoopCastOfTo(Sum, Ty);
       Sum = InsertBinop(Instruction::Sub, Sum, W, SCEV::FlagAnyWrap,
                         /*IsSafeToHoist*/ true);
       ++I;
     } else {
       // A simple add.
-      Value *W = expandCodeForImpl(Op, Ty, false);
+      Value *W = expandCodeFor(Op, Ty);
       Sum = InsertNoopCastOfTo(Sum, Ty);
       // Canonicalize a constant to the RHS.
       if (isa<Constant>(Sum)) std::swap(Sum, W);
@@ -804,7 +802,7 @@ Value *SCEVExpander::visitMulExpr(const SCEVMulExpr *S) {
 
     // Calculate powers with exponents 1, 2, 4, 8 etc. and include those of them
     // that are needed into the result.
-    Value *P = expandCodeForImpl(I->second, Ty, false);
+    Value *P = expandCodeFor(I->second, Ty);
     Value *Result = nullptr;
     if (Exponent & 1)
       Result = P;
@@ -863,7 +861,7 @@ Value *SCEVExpander::visitMulExpr(const SCEVMulExpr *S) {
 Value *SCEVExpander::visitUDivExpr(const SCEVUDivExpr *S) {
   Type *Ty = SE.getEffectiveSCEVType(S->getType());
 
-  Value *LHS = expandCodeForImpl(S->getLHS(), Ty, false);
+  Value *LHS = expandCodeFor(S->getLHS(), Ty);
   if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(S->getRHS())) {
     const APInt &RHS = SC->getAPInt();
     if (RHS.isPowerOf2())
@@ -872,7 +870,7 @@ Value *SCEVExpander::visitUDivExpr(const SCEVUDivExpr *S) {
                          SCEV::FlagAnyWrap, /*IsSafeToHoist*/ true);
   }
 
-  Value *RHS = expandCodeForImpl(S->getRHS(), Ty, false);
+  Value *RHS = expandCodeFor(S->getRHS(), Ty);
   return InsertBinop(Instruction::UDiv, LHS, RHS, SCEV::FlagAnyWrap,
                      /*IsSafeToHoist*/ SE.isKnownNonZero(S->getRHS()));
 }
@@ -1267,9 +1265,8 @@ SCEVExpander::getAddRecExprPHILiterally(const SCEVAddRecExpr *Normalized,
   // Expand code for the start value into the loop preheader.
   assert(L->getLoopPreheader() &&
          "Can't expand add recurrences without a loop preheader!");
-  Value *StartV =
-      expandCodeForImpl(Normalized->getStart(), ExpandTy,
-                        L->getLoopPreheader()->getTerminator(), false);
+  Value *StartV = expandCodeFor(Normalized->getStart(), ExpandTy,
+                                L->getLoopPreheader()->getTerminator());
 
   // StartV must have been be inserted into L's preheader to dominate the new
   // phi.
@@ -1287,8 +1284,8 @@ SCEVExpander::getAddRecExprPHILiterally(const SCEVAddRecExpr *Normalized,
   if (useSubtract)
     Step = SE.getNegativeSCEV(Step);
   // Expand the step somewhere that dominates the loop header.
-  Value *StepV = expandCodeForImpl(
-      Step, IntTy, &*L->getHeader()->getFirstInsertionPt(), false);
+  Value *StepV = expandCodeFor(Step, IntTy,
+                               &*L->getHeader()->getFirstInsertionPt());
 
   // The no-wrap behavior proved by IsIncrement(NUW|NSW) is only applicable if
   // we actually do emit an addition.  It does not apply if we emit a
@@ -1433,8 +1430,8 @@ Value *SCEVExpander::expandAddRecExprLiterally(const SCEVAddRecExpr *S) {
       {
         // Expand the step somewhere that dominates the loop header.
         SCEVInsertPointGuard Guard(Builder, this);
-        StepV = expandCodeForImpl(
-            Step, IntTy, &*L->getHeader()->getFirstInsertionPt(), false);
+        StepV = expandCodeFor(Step, IntTy,
+                              &*L->getHeader()->getFirstInsertionPt());
       }
       Result = expandIVInc(PN, StepV, L, ExpandTy, IntTy, useSubtract);
     }
@@ -1453,8 +1450,8 @@ Value *SCEVExpander::expandAddRecExprLiterally(const SCEVAddRecExpr *S) {
 
     // Invert the result.
     if (InvertStep)
-      Result = Builder.CreateSub(
-          expandCodeForImpl(Normalized->getStart(), TruncTy, false), Result);
+      Result = Builder.CreateSub(expandCodeFor(Normalized->getStart(), TruncTy),
+                                 Result);
   }
 
   // Re-apply any non-loop-dominating scale.
@@ -1462,22 +1459,22 @@ Value *SCEVExpander::expandAddRecExprLiterally(const SCEVAddRecExpr *S) {
     assert(S->isAffine() && "Can't linearly scale non-affine recurrences.");
     Result = InsertNoopCastOfTo(Result, IntTy);
     Result = Builder.CreateMul(Result,
-                               expandCodeForImpl(PostLoopScale, IntTy, false));
+                               expandCodeFor(PostLoopScale, IntTy));
   }
 
   // Re-apply any non-loop-dominating offset.
   if (PostLoopOffset) {
     if (PointerType *PTy = dyn_cast<PointerType>(ExpandTy)) {
       if (Result->getType()->isIntegerTy()) {
-        Value *Base = expandCodeForImpl(PostLoopOffset, ExpandTy, false);
+        Value *Base = expandCodeFor(PostLoopOffset, ExpandTy);
         Result = expandAddToGEP(SE.getUnknown(Result), PTy, IntTy, Base);
       } else {
         Result = expandAddToGEP(PostLoopOffset, PTy, IntTy, Result);
       }
     } else {
       Result = InsertNoopCastOfTo(Result, IntTy);
-      Result = Builder.CreateAdd(
-          Result, expandCodeForImpl(PostLoopOffset, IntTy, false));
+      Result = Builder.CreateAdd(Result,
+                                 expandCodeFor(PostLoopOffset, IntTy));
     }
   }
 
@@ -1519,8 +1516,8 @@ Value *SCEVExpander::visitAddRecExpr(const SCEVAddRecExpr *S) {
                                        S->getNoWrapFlags(SCEV::FlagNW)));
     BasicBlock::iterator NewInsertPt =
         findInsertPointAfter(cast<Instruction>(V), Builder.GetInsertBlock());
-    V = expandCodeForImpl(SE.getTruncateExpr(SE.getUnknown(V), Ty), nullptr,
-                          &*NewInsertPt, false);
+    V = expandCodeFor(SE.getTruncateExpr(SE.getUnknown(V), Ty), nullptr,
+                      &*NewInsertPt);
     return V;
   }
 
@@ -1635,25 +1632,22 @@ Value *SCEVExpander::visitAddRecExpr(const SCEVAddRecExpr *S) {
 
 Value *SCEVExpander::visitTruncateExpr(const SCEVTruncateExpr *S) {
   Type *Ty = SE.getEffectiveSCEVType(S->getType());
-  Value *V = expandCodeForImpl(
-      S->getOperand(), SE.getEffectiveSCEVType(S->getOperand()->getType()),
-      false);
+  Value *V = expandCodeFor(S->getOperand(),
+                           SE.getEffectiveSCEVType(S->getOperand()->getType()));
   return Builder.CreateTrunc(V, Ty);
 }
 
 Value *SCEVExpander::visitZeroExtendExpr(const SCEVZeroExtendExpr *S) {
   Type *Ty = SE.getEffectiveSCEVType(S->getType());
-  Value *V = expandCodeForImpl(
-      S->getOperand(), SE.getEffectiveSCEVType(S->getOperand()->getType()),
-      false);
+  Value *V = expandCodeFor(S->getOperand(),
+                           SE.getEffectiveSCEVType(S->getOperand()->getType()));
   return Builder.CreateZExt(V, Ty);
 }
 
 Value *SCEVExpander::visitSignExtendExpr(const SCEVSignExtendExpr *S) {
   Type *Ty = SE.getEffectiveSCEVType(S->getType());
-  Value *V = expandCodeForImpl(
-      S->getOperand(), SE.getEffectiveSCEVType(S->getOperand()->getType()),
-      false);
+  Value *V = expandCodeFor(S->getOperand(),
+                           SE.getEffectiveSCEVType(S->getOperand()->getType()));
   return Builder.CreateSExt(V, Ty);
 }
 
@@ -1668,7 +1662,7 @@ Value *SCEVExpander::visitSMaxExpr(const SCEVSMaxExpr *S) {
       Ty = SE.getEffectiveSCEVType(Ty);
       LHS = InsertNoopCastOfTo(LHS, Ty);
     }
-    Value *RHS = expandCodeForImpl(S->getOperand(i), Ty, false);
+    Value *RHS = expandCodeFor(S->getOperand(i), Ty);
     Value *ICmp = Builder.CreateICmpSGT(LHS, RHS);
     Value *Sel = Builder.CreateSelect(ICmp, LHS, RHS, "smax");
     LHS = Sel;
@@ -1691,7 +1685,7 @@ Value *SCEVExpander::visitUMaxExpr(const SCEVUMaxExpr *S) {
       Ty = SE.getEffectiveSCEVType(Ty);
       LHS = InsertNoopCastOfTo(LHS, Ty);
     }
-    Value *RHS = expandCodeForImpl(S->getOperand(i), Ty, false);
+    Value *RHS = expandCodeFor(S->getOperand(i), Ty);
     Value *ICmp = Builder.CreateICmpUGT(LHS, RHS);
     Value *Sel = Builder.CreateSelect(ICmp, LHS, RHS, "umax");
     LHS = Sel;
@@ -1714,7 +1708,7 @@ Value *SCEVExpander::visitSMinExpr(const SCEVSMinExpr *S) {
       Ty = SE.getEffectiveSCEVType(Ty);
       LHS = InsertNoopCastOfTo(LHS, Ty);
     }
-    Value *RHS = expandCodeForImpl(S->getOperand(i), Ty, false);
+    Value *RHS = expandCodeFor(S->getOperand(i), Ty);
     Value *ICmp = Builder.CreateICmpSLT(LHS, RHS);
     Value *Sel = Builder.CreateSelect(ICmp, LHS, RHS, "smin");
     LHS = Sel;
@@ -1737,7 +1731,7 @@ Value *SCEVExpander::visitUMinExpr(const SCEVUMinExpr *S) {
       Ty = SE.getEffectiveSCEVType(Ty);
       LHS = InsertNoopCastOfTo(LHS, Ty);
     }
-    Value *RHS = expandCodeForImpl(S->getOperand(i), Ty, false);
+    Value *RHS = expandCodeFor(S->getOperand(i), Ty);
     Value *ICmp = Builder.CreateICmpULT(LHS, RHS);
     Value *Sel = Builder.CreateSelect(ICmp, LHS, RHS, "umin");
     LHS = Sel;
@@ -1749,43 +1743,15 @@ Value *SCEVExpander::visitUMinExpr(const SCEVUMinExpr *S) {
   return LHS;
 }
 
-Value *SCEVExpander::expandCodeForImpl(const SCEV *SH, Type *Ty,
-                                       Instruction *IP, bool Root) {
+Value *SCEVExpander::expandCodeFor(const SCEV *SH, Type *Ty,
+                                   Instruction *IP) {
   setInsertPoint(IP);
-  Value *V = expandCodeForImpl(SH, Ty, Root);
-  return V;
+  return expandCodeFor(SH, Ty);
 }
 
-Value *SCEVExpander::expandCodeForImpl(const SCEV *SH, Type *Ty, bool Root) {
+Value *SCEVExpander::expandCodeFor(const SCEV *SH, Type *Ty) {
   // Expand the code for this SCEV.
   Value *V = expand(SH);
-
-  if (PreserveLCSSA) {
-    if (auto *Inst = dyn_cast<Instruction>(V)) {
-      // Create a temporary instruction to at the current insertion point, so we
-      // can hand it off to the helper to create LCSSA PHIs if required for the
-      // new use.
-      // FIXME: Ideally formLCSSAForInstructions (used in fixupLCSSAFormFor)
-      // would accept a insertion point and return an LCSSA phi for that
-      // insertion point, so there is no need to insert & remove the temporary
-      // instruction.
-      Instruction *Tmp;
-      if (Inst->getType()->isIntegerTy())
-        Tmp = cast<Instruction>(Builder.CreateAdd(Inst, Inst));
-      else {
-        assert(Inst->getType()->isPointerTy());
-        Tmp = cast<Instruction>(Builder.CreateGEP(Inst, Builder.getInt32(1)));
-      }
-      V = fixupLCSSAFormFor(Tmp, 0);
-
-      // Clean up temporary instruction.
-      InsertedValues.erase(Tmp);
-      InsertedPostIncValues.erase(Tmp);
-      Tmp->eraseFromParent();
-    }
-  }
-
-  InsertedExpressions[std::make_pair(SH, &*Builder.GetInsertPoint())] = V;
   if (Ty) {
     assert(SE.getTypeSizeInBits(Ty) == SE.getTypeSizeInBits(SH->getType()) &&
            "non-trivial casts should be done with the SCEVs directly!");
@@ -1925,28 +1891,10 @@ Value *SCEVExpander::expand(const SCEV *S) {
 }
 
 void SCEVExpander::rememberInstruction(Value *I) {
-  auto DoInsert = [this](Value *V) {
-    if (!PostIncLoops.empty())
-      InsertedPostIncValues.insert(V);
-    else
-      InsertedValues.insert(V);
-  };
-  DoInsert(I);
-
-  if (!PreserveLCSSA)
-    return;
-
-  if (auto *Inst = dyn_cast<Instruction>(I)) {
-    // A new instruction has been added, which might introduce new uses outside
-    // a defining loop. Fix LCSSA from for each operand of the new instruction,
-    // if required.
-    for (unsigned OpIdx = 0, OpEnd = Inst->getNumOperands(); OpIdx != OpEnd;
-         OpIdx++) {
-      auto *V = fixupLCSSAFormFor(Inst, OpIdx);
-      if (V != I)
-        DoInsert(V);
-    }
-  }
+  if (!PostIncLoops.empty())
+    InsertedPostIncValues.insert(I);
+  else
+    InsertedValues.insert(I);
 }
 
 /// getOrInsertCanonicalInductionVariable - This method returns the
@@ -1965,8 +1913,9 @@ SCEVExpander::getOrInsertCanonicalInductionVariable(const Loop *L,
 
   // Emit code for it.
   SCEVInsertPointGuard Guard(Builder, this);
-  PHINode *V = cast<PHINode>(expandCodeForImpl(
-      H, nullptr, &*L->getHeader()->getFirstInsertionPt(), false));
+  PHINode *V =
+      cast<PHINode>(expandCodeFor(H, nullptr,
+                                  &*L->getHeader()->getFirstInsertionPt()));
 
   return V;
 }
@@ -2366,10 +2315,8 @@ Value *SCEVExpander::expandCodeForPredicate(const SCEVPredicate *Pred,
 
 Value *SCEVExpander::expandEqualPredicate(const SCEVEqualPredicate *Pred,
                                           Instruction *IP) {
-  Value *Expr0 =
-      expandCodeForImpl(Pred->getLHS(), Pred->getLHS()->getType(), IP, false);
-  Value *Expr1 =
-      expandCodeForImpl(Pred->getRHS(), Pred->getRHS()->getType(), IP, false);
+  Value *Expr0 = expandCodeFor(Pred->getLHS(), Pred->getLHS()->getType(), IP);
+  Value *Expr1 = expandCodeFor(Pred->getRHS(), Pred->getRHS()->getType(), IP);
 
   Builder.SetInsertPoint(IP);
   auto *I = Builder.CreateICmpNE(Expr0, Expr1, "ident.check");
@@ -2401,16 +2348,15 @@ Value *SCEVExpander::generateOverflowCheck(const SCEVAddRecExpr *AR,
 
   IntegerType *CountTy = IntegerType::get(Loc->getContext(), SrcBits);
   Builder.SetInsertPoint(Loc);
-  Value *TripCountVal = expandCodeForImpl(ExitCount, CountTy, Loc, false);
+  Value *TripCountVal = expandCodeFor(ExitCount, CountTy, Loc);
 
   IntegerType *Ty =
       IntegerType::get(Loc->getContext(), SE.getTypeSizeInBits(ARTy));
   Type *ARExpandTy = DL.isNonIntegralPointerType(ARTy) ? ARTy : Ty;
 
-  Value *StepValue = expandCodeForImpl(Step, Ty, Loc, false);
-  Value *NegStepValue =
-      expandCodeForImpl(SE.getNegativeSCEV(Step), Ty, Loc, false);
-  Value *StartValue = expandCodeForImpl(Start, ARExpandTy, Loc, false);
+  Value *StepValue = expandCodeFor(Step, Ty, Loc);
+  Value *NegStepValue = expandCodeFor(SE.getNegativeSCEV(Step), Ty, Loc);
+  Value *StartValue = expandCodeFor(Start, ARExpandTy, Loc);
 
   ConstantInt *Zero =
       ConstantInt::get(Loc->getContext(), APInt::getNullValue(DstBits));
@@ -2513,25 +2459,6 @@ Value *SCEVExpander::expandUnionPredicate(const SCEVUnionPredicate *Union,
   return Check;
 }
 
-Value *SCEVExpander::fixupLCSSAFormFor(Instruction *User, unsigned OpIdx) {
-  assert(PreserveLCSSA);
-  SmallVector<Instruction *, 1> ToUpdate;
-
-  auto *OpV = User->getOperand(OpIdx);
-  auto *OpI = dyn_cast<Instruction>(OpV);
-  if (!OpI)
-    return OpV;
-
-  Loop *DefLoop = SE.LI.getLoopFor(OpI->getParent());
-  Loop *UseLoop = SE.LI.getLoopFor(User->getParent());
-  if (!DefLoop || UseLoop == DefLoop || DefLoop->contains(UseLoop))
-    return OpV;
-
-  ToUpdate.push_back(OpI);
-  formLCSSAForInstructions(ToUpdate, SE.DT, SE.LI, &SE);
-  return User->getOperand(OpIdx);
-}
-
 namespace {
 // Search for a SCEV subexpression that is not safe to expand.  Any expression
 // that may expand to a !isSafeToSpeculativelyExecute value is unsafe, namely
diff --git a/llvm/unittests/Transforms/Utils/ScalarEvolutionExpanderTest.cpp b/llvm/unittests/Transforms/Utils/ScalarEvolutionExpanderTest.cpp
index 6e83370dfeb95..c146d155ffbea 100644
--- a/llvm/unittests/Transforms/Utils/ScalarEvolutionExpanderTest.cpp
+++ b/llvm/unittests/Transforms/Utils/ScalarEvolutionExpanderTest.cpp
@@ -265,7 +265,7 @@ TEST_F(ScalarEvolutionExpanderTest, SCEVExpanderIsSafeToExpandAt) {
   Phi->addIncoming(Add, L);
 
   Builder.SetInsertPoint(Post);
-  Instruction *Ret = Builder.CreateRetVoid();
+  Builder.CreateRetVoid();
 
   ScalarEvolution SE = buildSE(*F);
   const SCEV *S = SE.getSCEV(Phi);
@@ -276,11 +276,6 @@ TEST_F(ScalarEvolutionExpanderTest, SCEVExpanderIsSafeToExpandAt) {
   EXPECT_FALSE(isSafeToExpandAt(AR, LPh->getTerminator(), SE));
   EXPECT_TRUE(isSafeToExpandAt(AR, L->getTerminator(), SE));
   EXPECT_TRUE(isSafeToExpandAt(AR, Post->getTerminator(), SE));
-
-  EXPECT_TRUE(LI->getLoopFor(L)->isLCSSAForm(*DT));
-  SCEVExpander Exp(SE, M.getDataLayout(), "expander");
-  Exp.expandCodeFor(SE.getSCEV(Add), nullptr, Ret);
-  EXPECT_TRUE(LI->getLoopFor(L)->isLCSSAForm(*DT));
 }
 
 // Check that SCEV expander does not use the nuw instruction

From c114352edfe0de69050bbf1cbbc478bcfb524622 Mon Sep 17 00:00:00 2001
From: Tatyana Krasnukha <tatyana@synopsys.com>
Date: Wed, 22 Jul 2020 14:41:15 +0300
Subject: [PATCH 0512/1035] [lldb/test] Put hardware breakpoint tests together,
 NFC

Create a common base class for them to re-use supports_hw_breakpoints function in decorators.

Differential Revision: https://reviews.llvm.org/D84311
---
 .../breakpoint/hardware_breakpoints/base.py   | 19 +++++++++++++++++++
 .../TestHWBreakMultiThread.py                 | 15 +++++++++------
 .../require_hw_breakpoints/Makefile           |  0
 .../TestRequireHWBreakpoints.py               | 13 +++----------
 .../require_hw_breakpoints/main.c             |  0
 5 files changed, 31 insertions(+), 16 deletions(-)
 create mode 100644 lldb/test/API/functionalities/breakpoint/hardware_breakpoints/base.py
 rename lldb/test/API/functionalities/breakpoint/{ => hardware_breakpoints}/require_hw_breakpoints/Makefile (100%)
 rename lldb/test/API/functionalities/breakpoint/{ => hardware_breakpoints}/require_hw_breakpoints/TestRequireHWBreakpoints.py (89%)
 rename lldb/test/API/functionalities/breakpoint/{ => hardware_breakpoints}/require_hw_breakpoints/main.c (100%)

diff --git a/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/base.py b/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/base.py
new file mode 100644
index 0000000000000..9593a72b3d403
--- /dev/null
+++ b/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/base.py
@@ -0,0 +1,19 @@
+"""
+Base class for hardware breakpoints tests.
+"""
+
+from lldbsuite.test.lldbtest import *
+
+class HardwareBreakpointTestBase(TestBase):
+    NO_DEBUG_INFO_TESTCASE = True
+
+
+    def supports_hw_breakpoints(self):
+        self.build()
+        self.runCmd("file " + self.getBuildArtifact("a.out"),
+                    CURRENT_EXECUTABLE_SET)
+        self.runCmd("breakpoint set -b main --hardware")
+        self.runCmd("run")
+        if 'stopped' in self.res.GetOutput():
+            return 'Hardware breakpoints are supported'
+        return None
diff --git a/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/hardware_breakpoint_on_multiple_threads/TestHWBreakMultiThread.py b/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/hardware_breakpoint_on_multiple_threads/TestHWBreakMultiThread.py
index bca7b278631fb..09db3ffd3ecd8 100644
--- a/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/hardware_breakpoint_on_multiple_threads/TestHWBreakMultiThread.py
+++ b/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/hardware_breakpoint_on_multiple_threads/TestHWBreakMultiThread.py
@@ -9,15 +9,18 @@
 from lldbsuite.test.lldbtest import *
 from lldbsuite.test import lldbutil
 
-class HardwareBreakpointMultiThreadTestCase(TestBase):
-    NO_DEBUG_INFO_TESTCASE = True
+from functionalities.breakpoint.hardware_breakpoints.base import *
 
+class HardwareBreakpointMultiThreadTestCase(HardwareBreakpointTestBase):
     mydir = TestBase.compute_mydir(__file__)
 
+    def does_not_support_hw_breakpoints(self):
+        return not super().supports_hw_breakpoints()
+
     # LLDB on linux supports hardware breakpoints for arm and aarch64
     # architectures.
     @skipUnlessPlatform(oslist=['linux'])
-    @skipIf(archs=no_match(['arm', 'aarch64']))
+    @skipTestIfFn(does_not_support_hw_breakpoints)
     def test_hw_break_set_delete_multi_thread_linux(self):
         self.build()
         self.setTearDownCleanup()
@@ -26,7 +29,7 @@ def test_hw_break_set_delete_multi_thread_linux(self):
     # LLDB on linux supports hardware breakpoints for arm and aarch64
     # architectures.
     @skipUnlessPlatform(oslist=['linux'])
-    @skipIf(archs=no_match(['arm', 'aarch64']))
+    @skipTestIfFn(does_not_support_hw_breakpoints)
     def test_hw_break_set_disable_multi_thread_linux(self):
         self.build()
         self.setTearDownCleanup()
@@ -36,7 +39,7 @@ def test_hw_break_set_disable_multi_thread_linux(self):
     # architectures.
     @skipUnlessDarwin
     @skipIfOutOfTreeDebugserver
-    @expectedFailureAll(archs=["arm64"])
+    @skipTestIfFn(does_not_support_hw_breakpoints)
     def test_hw_break_set_delete_multi_thread_macos(self):
         self.build()
         self.setTearDownCleanup()
@@ -46,7 +49,7 @@ def test_hw_break_set_delete_multi_thread_macos(self):
     # architectures.
     @skipUnlessDarwin
     @skipIfOutOfTreeDebugserver
-    @expectedFailureAll(archs=["arm64"])
+    @skipTestIfFn(does_not_support_hw_breakpoints)
     def test_hw_break_set_disable_multi_thread_macos(self):
         self.build()
         self.setTearDownCleanup()
diff --git a/lldb/test/API/functionalities/breakpoint/require_hw_breakpoints/Makefile b/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/require_hw_breakpoints/Makefile
similarity index 100%
rename from lldb/test/API/functionalities/breakpoint/require_hw_breakpoints/Makefile
rename to lldb/test/API/functionalities/breakpoint/hardware_breakpoints/require_hw_breakpoints/Makefile
diff --git a/lldb/test/API/functionalities/breakpoint/require_hw_breakpoints/TestRequireHWBreakpoints.py b/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/require_hw_breakpoints/TestRequireHWBreakpoints.py
similarity index 89%
rename from lldb/test/API/functionalities/breakpoint/require_hw_breakpoints/TestRequireHWBreakpoints.py
rename to lldb/test/API/functionalities/breakpoint/hardware_breakpoints/require_hw_breakpoints/TestRequireHWBreakpoints.py
index 61e4171131013..c97795f7ae67c 100644
--- a/lldb/test/API/functionalities/breakpoint/require_hw_breakpoints/TestRequireHWBreakpoints.py
+++ b/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/require_hw_breakpoints/TestRequireHWBreakpoints.py
@@ -8,20 +8,13 @@
 from lldbsuite.test.lldbtest import *
 from lldbsuite.test import lldbutil
 
+from functionalities.breakpoint.hardware_breakpoints.base import *
 
-class BreakpointLocationsTestCase(TestBase):
-    NO_DEBUG_INFO_TESTCASE = True
+class BreakpointLocationsTestCase(HardwareBreakpointTestBase):
     mydir = TestBase.compute_mydir(__file__)
 
     def supports_hw_breakpoints(self):
-        self.build()
-        self.runCmd("file " + self.getBuildArtifact("a.out"),
-                    CURRENT_EXECUTABLE_SET)
-        self.runCmd("breakpoint set -b main --hardware")
-        self.runCmd("run")
-        if 'stopped' in self.res.GetOutput():
-            return 'Hardware breakpoints are supported'
-        return None
+        return super().supports_hw_breakpoints()
 
     def test_breakpoint(self):
         """Test regular breakpoints when hardware breakpoints are required."""
diff --git a/lldb/test/API/functionalities/breakpoint/require_hw_breakpoints/main.c b/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/require_hw_breakpoints/main.c
similarity index 100%
rename from lldb/test/API/functionalities/breakpoint/require_hw_breakpoints/main.c
rename to lldb/test/API/functionalities/breakpoint/hardware_breakpoints/require_hw_breakpoints/main.c

From 7e946d0c8289ff8cafcae88595b805ec511db6b4 Mon Sep 17 00:00:00 2001
From: Julian Lettner <julian.lettner@apple.com>
Date: Wed, 29 Jul 2020 09:55:52 -0700
Subject: [PATCH 0513/1035] [compiler-rt][Darwin] Disable EXC_GUARD exceptions
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

ASan/TSan use mmap in a way that creates “deallocation gaps” which
triggers EXC_GUARD exceptions on macOS 10.15+ (XNU 19.0+).  Let's
suppress those.
---
 .../lib/sanitizer_common/sanitizer_mac.cpp        | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp
index f96f187131972..5b06fee62c13f 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp
@@ -831,6 +831,19 @@ void SignalContext::InitPcSpBp() {
   GetPcSpBp(context, &pc, &sp, &bp);
 }
 
+// ASan/TSan use mmap in a way that creates “deallocation gaps” which triggers
+// EXC_GUARD exceptions on macOS 10.15+ (XNU 19.0+).
+static void DisableMmapExcGuardExceptions() {
+  using task_exc_guard_behavior_t = uint32_t;
+  using task_set_exc_guard_behavior_t =
+      kern_return_t(task_t task, task_exc_guard_behavior_t behavior);
+  auto *set_behavior = (task_set_exc_guard_behavior_t *)dlsym(
+      RTLD_DEFAULT, "task_set_exc_guard_behavior");
+  if (set_behavior == nullptr) return;
+  const task_exc_guard_behavior_t task_exc_guard_none = 0;
+  set_behavior(mach_task_self(), task_exc_guard_none);
+}
+
 void InitializePlatformEarly() {
   // Only use xnu_fast_mmap when on x86_64 and the kernel supports it.
   use_xnu_fast_mmap =
@@ -839,6 +852,8 @@ void InitializePlatformEarly() {
 #else
       false;
 #endif
+  if (GetDarwinKernelVersion() >= DarwinKernelVersion(19, 0))
+    DisableMmapExcGuardExceptions();
 }
 
 #if !SANITIZER_GO

From 59fac51ff24e198ff4896be03dea0096fa3ac188 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Tue, 21 Jul 2020 19:41:24 -0400
Subject: [PATCH 0514/1035] AMDGPU/GlobalISel: Handle
 llvm.amdgcn.reloc.constant

---
 .../AMDGPU/AMDGPUInstructionSelector.cpp      | 27 ++++++++++
 .../Target/AMDGPU/AMDGPUInstructionSelector.h |  1 +
 .../Target/AMDGPU/AMDGPURegisterBankInfo.cpp  |  3 +-
 .../inst-select-amdgcn.reloc.constant.mir     | 50 +++++++++++++++++++
 .../test/CodeGen/AMDGPU/amdgpu-reloc-const.ll |  4 ++
 5 files changed, 84 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgcn.reloc.constant.mir

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
index ff83e1ef404cf..23899a7e4dd71 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
@@ -904,6 +904,8 @@ bool AMDGPUInstructionSelector::selectG_INTRINSIC(MachineInstr &I) const {
     return selectIntrinsicIcmp(I);
   case Intrinsic::amdgcn_ballot:
     return selectBallot(I);
+  case Intrinsic::amdgcn_reloc_constant:
+    return selectRelocConstant(I);
   default:
     return selectImpl(I, *CoverageInfo);
   }
@@ -1084,6 +1086,31 @@ bool AMDGPUInstructionSelector::selectBallot(MachineInstr &I) const {
   return true;
 }
 
+bool AMDGPUInstructionSelector::selectRelocConstant(MachineInstr &I) const {
+  Register DstReg = I.getOperand(0).getReg();
+  const RegisterBank *DstBank = RBI.getRegBank(DstReg, *MRI, TRI);
+  const TargetRegisterClass *DstRC =
+    TRI.getRegClassForSizeOnBank(32, *DstBank, *MRI);
+  if (!DstRC || !RBI.constrainGenericRegister(DstReg, *DstRC, *MRI))
+    return false;
+
+  const bool IsVALU = DstBank->getID() == AMDGPU::VGPRRegBankID;
+
+  Module *M = MF->getFunction().getParent();
+  const MDNode *Metadata = I.getOperand(2).getMetadata();
+  auto SymbolName = cast<MDString>(Metadata->getOperand(0))->getString();
+  auto RelocSymbol = cast<GlobalVariable>(
+    M->getOrInsertGlobal(SymbolName, Type::getInt32Ty(M->getContext())));
+
+  MachineBasicBlock *BB = I.getParent();
+  BuildMI(*BB, &I, I.getDebugLoc(),
+          TII.get(IsVALU ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32), DstReg)
+    .addGlobalAddress(RelocSymbol, 0, SIInstrInfo::MO_ABS32_LO);
+
+  I.eraseFromParent();
+  return true;
+}
+
 bool AMDGPUInstructionSelector::selectEndCfIntrinsic(MachineInstr &MI) const {
   // FIXME: Manually selecting to avoid dealiing with the SReg_1 trick
   // SelectionDAG uses for wave32 vs wave64.
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
index 1fb64442347c9..1a0a9cefacf98 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
@@ -108,6 +108,7 @@ class AMDGPUInstructionSelector final : public InstructionSelector {
   bool selectDivScale(MachineInstr &MI) const;
   bool selectIntrinsicIcmp(MachineInstr &MI) const;
   bool selectBallot(MachineInstr &I) const;
+  bool selectRelocConstant(MachineInstr &I) const;
   bool selectG_INTRINSIC(MachineInstr &I) const;
 
   bool selectEndCfIntrinsic(MachineInstr &MI) const;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
index d53246967f45e..3fed45e84b8f7 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
@@ -4023,7 +4023,8 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
       return getDefaultMappingAllVGPR(MI);
     case Intrinsic::amdgcn_kernarg_segment_ptr:
     case Intrinsic::amdgcn_s_getpc:
-    case Intrinsic::amdgcn_groupstaticsize: {
+    case Intrinsic::amdgcn_groupstaticsize:
+    case Intrinsic::amdgcn_reloc_constant: {
       unsigned Size = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
       OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size);
       break;
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgcn.reloc.constant.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgcn.reloc.constant.mir
new file mode 100644
index 0000000000000..76582ad083f69
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgcn.reloc.constant.mir
@@ -0,0 +1,50 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=tahiti -run-pass=instruction-select -verify-machineinstrs -o - %s | FileCheck -check-prefix=GCN %s
+
+--- |
+
+  define void @reloc_constant_sgpr32() { ret void }
+  define void @reloc_constant_vgpr32() { ret void }
+
+  declare i32 @llvm.amdgcn.reloc.constant(metadata)
+
+  !0 = !{!"arst"}
+
+...
+
+---
+name:            reloc_constant_sgpr32
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0:
+
+    ; GCN-LABEL: name: reloc_constant_sgpr32
+    ; GCN: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-lo) @arst
+    ; GCN: $sgpr0 = COPY [[S_MOV_B32_]]
+    ; GCN: S_ENDPGM 0, implicit $sgpr0
+    %0:sgpr(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.reloc.constant), !0
+    $sgpr0 = COPY %0
+    S_ENDPGM 0, implicit $sgpr0
+
+...
+
+---
+name:            reloc_constant_vgpr32
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0:
+
+    ; GCN-LABEL: name: reloc_constant_vgpr32
+    ; GCN: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 target-flags(amdgpu-abs32-lo) @arst, implicit $exec
+    ; GCN: $vgpr0 = COPY [[V_MOV_B32_e32_]]
+    ; GCN: S_ENDPGM 0, implicit $vgpr0
+    %0:vgpr(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.reloc.constant), !0
+    $vgpr0 = COPY %0
+    S_ENDPGM 0, implicit $vgpr0
+
+...
+
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-reloc-const.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-reloc-const.ll
index 569f0101e3bee..d6e977bd20e10 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-reloc-const.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-reloc-const.ll
@@ -1,5 +1,9 @@
 ; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
 ; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx900 -filetype=obj -o %t.o < %s && llvm-readobj -relocations %t.o | FileCheck --check-prefix=ELF %s
+
+; RUN: llc -global-isel -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -global-isel -mtriple=amdgcn--amdpal -mcpu=gfx900 -filetype=obj -o %t.o < %s && llvm-readobj -relocations %t.o | FileCheck --check-prefix=ELF %s
+
 ; GCN-LABEL: {{^}}ps_main:
 ; GCN: v_mov_b32_{{.*}} v[[relocreg:[0-9]+]], doff_0_0_b@abs32@lo
 ; GCN-NEXT: exp {{.*}} v[[relocreg]], {{.*}}

From f7ec3e3be70d10a630ac22c64073a845b168c829 Mon Sep 17 00:00:00 2001
From: Tatyana Krasnukha <tatyana@synopsys.com>
Date: Wed, 22 Jul 2020 14:45:10 +0300
Subject: [PATCH 0515/1035] [lldb] Skip overlapping hardware and external
 breakpoints when writing memory

This fixes the assertion `assert(intersects);` in the Process::WriteMemory function.

Differential Revision: https://reviews.llvm.org/D84254
---
 lldb/source/Breakpoint/BreakpointSite.cpp     | 61 +++++++++----------
 lldb/source/Target/Process.cpp                |  3 +
 .../write_memory_with_hw_breakpoint/Makefile  |  3 +
 .../TestWriteMemoryWithHWBreakpoint.py        | 51 ++++++++++++++++
 .../write_memory_with_hw_breakpoint/main.cpp  |  9 +++
 5 files changed, 96 insertions(+), 31 deletions(-)
 create mode 100644 lldb/test/API/functionalities/breakpoint/hardware_breakpoints/write_memory_with_hw_breakpoint/Makefile
 create mode 100644 lldb/test/API/functionalities/breakpoint/hardware_breakpoints/write_memory_with_hw_breakpoint/TestWriteMemoryWithHWBreakpoint.py
 create mode 100644 lldb/test/API/functionalities/breakpoint/hardware_breakpoints/write_memory_with_hw_breakpoint/main.cpp

diff --git a/lldb/source/Breakpoint/BreakpointSite.cpp b/lldb/source/Breakpoint/BreakpointSite.cpp
index a33fd0a1c462c..bdcabd7cce5ee 100644
--- a/lldb/source/Breakpoint/BreakpointSite.cpp
+++ b/lldb/source/Breakpoint/BreakpointSite.cpp
@@ -167,40 +167,39 @@ bool BreakpointSite::IntersectsRange(lldb::addr_t addr, size_t size,
                                      lldb::addr_t *intersect_addr,
                                      size_t *intersect_size,
                                      size_t *opcode_offset) const {
-  // We only use software traps for software breakpoints
-  if (!IsHardware()) {
-    if (m_byte_size > 0) {
-      const lldb::addr_t bp_end_addr = m_addr + m_byte_size;
-      const lldb::addr_t end_addr = addr + size;
-      // Is the breakpoint end address before the passed in start address?
-      if (bp_end_addr <= addr)
-        return false;
-      // Is the breakpoint start address after passed in end address?
-      if (end_addr <= m_addr)
-        return false;
-      if (intersect_addr || intersect_size || opcode_offset) {
-        if (m_addr < addr) {
-          if (intersect_addr)
-            *intersect_addr = addr;
-          if (intersect_size)
-            *intersect_size =
-                std::min<lldb::addr_t>(bp_end_addr, end_addr) - addr;
-          if (opcode_offset)
-            *opcode_offset = addr - m_addr;
-        } else {
-          if (intersect_addr)
-            *intersect_addr = m_addr;
-          if (intersect_size)
-            *intersect_size =
-                std::min<lldb::addr_t>(bp_end_addr, end_addr) - m_addr;
-          if (opcode_offset)
-            *opcode_offset = 0;
-        }
+  // The function should be called only for software breakpoints.
+  lldbassert(GetType() == Type::eSoftware);
+
+  if (m_byte_size > 0) {
+    const lldb::addr_t bp_end_addr = m_addr + m_byte_size;
+    const lldb::addr_t end_addr = addr + size;
+    // Is the breakpoint end address before the passed in start address?
+    if (bp_end_addr <= addr)
+      return false;
+    // Is the breakpoint start address after passed in end address?
+    if (end_addr <= m_addr)
+      return false;
+    if (intersect_addr || intersect_size || opcode_offset) {
+      if (m_addr < addr) {
+        if (intersect_addr)
+          *intersect_addr = addr;
+        if (intersect_size)
+          *intersect_size =
+              std::min<lldb::addr_t>(bp_end_addr, end_addr) - addr;
+        if (opcode_offset)
+          *opcode_offset = addr - m_addr;
+      } else {
+        if (intersect_addr)
+          *intersect_addr = m_addr;
+        if (intersect_size)
+          *intersect_size =
+              std::min<lldb::addr_t>(bp_end_addr, end_addr) - m_addr;
+        if (opcode_offset)
+          *opcode_offset = 0;
       }
-      return true;
     }
+    return true;
   }
-  return false;
 }
 
 size_t
diff --git a/lldb/source/Target/Process.cpp b/lldb/source/Target/Process.cpp
index 3776a90e546ae..36a2930f79150 100644
--- a/lldb/source/Target/Process.cpp
+++ b/lldb/source/Target/Process.cpp
@@ -2291,6 +2291,9 @@ size_t Process::WriteMemory(addr_t addr, const void *buf, size_t size,
     if (error.Fail())
       return;
 
+    if (bp->GetType() != BreakpointSite::eSoftware)
+      return;
+
     addr_t intersect_addr;
     size_t intersect_size;
     size_t opcode_offset;
diff --git a/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/write_memory_with_hw_breakpoint/Makefile b/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/write_memory_with_hw_breakpoint/Makefile
new file mode 100644
index 0000000000000..99998b20bcb05
--- /dev/null
+++ b/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/write_memory_with_hw_breakpoint/Makefile
@@ -0,0 +1,3 @@
+CXX_SOURCES := main.cpp
+
+include Makefile.rules
diff --git a/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/write_memory_with_hw_breakpoint/TestWriteMemoryWithHWBreakpoint.py b/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/write_memory_with_hw_breakpoint/TestWriteMemoryWithHWBreakpoint.py
new file mode 100644
index 0000000000000..842519be6f897
--- /dev/null
+++ b/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/write_memory_with_hw_breakpoint/TestWriteMemoryWithHWBreakpoint.py
@@ -0,0 +1,51 @@
+"""
+Test that writing memory does't affect hardware breakpoints.
+"""
+
+
+
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+from functionalities.breakpoint.hardware_breakpoints.base import *
+
+class WriteMemoryWithHWBreakpoint(HardwareBreakpointTestBase):
+    mydir = TestBase.compute_mydir(__file__)
+
+    def does_not_support_hw_breakpoints(self):
+        return not super().supports_hw_breakpoints()
+
+    @skipTestIfFn(does_not_support_hw_breakpoints)
+    def test_copy_memory_with_hw_break(self):
+        self.build()
+        exe = self.getBuildArtifact("a.out")
+
+        # Create a target by the debugger.
+        target = self.dbg.CreateTarget(exe)
+        self.assertTrue(target, VALID_TARGET)
+
+        # Run the program and stop at entry.
+        self.expect("process launch --stop-at-entry",
+            patterns=["Process .* launched: .*a.out"])
+
+        process = target.GetProcess()
+        self.assertTrue(process, PROCESS_IS_VALID)
+
+        # Set a hardware breakpoint.
+        bp_id = lldbutil.run_break_set_by_symbol(self, "hw_break_function",
+                                                 extra_options="--hardware")
+
+        # Get breakpoint location from the breakpoint.
+        location = target.FindBreakpointByID(bp_id).GetLocationAtIndex(0)
+        self.assertTrue(location and location.IsResolved(),
+                        VALID_BREAKPOINT_LOCATION)
+
+        # Check that writing overlapping memory doesn't crash.
+        address = location.GetLoadAddress()
+        data = str("\x01\x02\x03\x04")
+        error = lldb.SBError()
+
+        result = process.WriteMemory(address, data, error)
+        self.assertTrue(error.Success() and result == len(bytes))
diff --git a/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/write_memory_with_hw_breakpoint/main.cpp b/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/write_memory_with_hw_breakpoint/main.cpp
new file mode 100644
index 0000000000000..bad219082cafe
--- /dev/null
+++ b/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/write_memory_with_hw_breakpoint/main.cpp
@@ -0,0 +1,9 @@
+static volatile int num = 1;
+
+bool hw_break_function (int i) {
+  return num == i;
+}
+
+int main (int argc, char const *argv[]) {
+  return hw_break_function(argc) ? 0 : 1;
+}

From b352e62feadd0aabaa7373b6fb40701f00a6aa91 Mon Sep 17 00:00:00 2001
From: Tatyana Krasnukha <tatyana@synopsys.com>
Date: Wed, 22 Jul 2020 14:52:52 +0300
Subject: [PATCH 0516/1035] [lldb] Make process plugins check whether a
 hardware breakpoint is required

Remove @skipIfWindows as process should report the error correctly on Windows now.

Differential Revision: https://reviews.llvm.org/D84255
---
 lldb/source/Plugins/Process/FreeBSD/ProcessFreeBSD.cpp        | 3 +++
 lldb/source/Plugins/Process/MacOSX-Kernel/ProcessKDP.cpp      | 3 +++
 lldb/source/Plugins/Process/Windows/Common/ProcessWindows.cpp | 3 +++
 .../require_hw_breakpoints/TestRequireHWBreakpoints.py        | 4 ----
 4 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/lldb/source/Plugins/Process/FreeBSD/ProcessFreeBSD.cpp b/lldb/source/Plugins/Process/FreeBSD/ProcessFreeBSD.cpp
index a44080640f6c4..f1a424ccbca57 100644
--- a/lldb/source/Plugins/Process/FreeBSD/ProcessFreeBSD.cpp
+++ b/lldb/source/Plugins/Process/FreeBSD/ProcessFreeBSD.cpp
@@ -683,6 +683,9 @@ ProcessFreeBSD::GetSoftwareBreakpointTrapOpcode(BreakpointSite *bp_site) {
 }
 
 Status ProcessFreeBSD::EnableBreakpointSite(BreakpointSite *bp_site) {
+  if (bp_site->HardwareRequired())
+    return Status("Hardware breakpoints are not supported.");
+
   return EnableSoftwareBreakpoint(bp_site);
 }
 
diff --git a/lldb/source/Plugins/Process/MacOSX-Kernel/ProcessKDP.cpp b/lldb/source/Plugins/Process/MacOSX-Kernel/ProcessKDP.cpp
index dde25184a8c59..6e394eac6f9e6 100644
--- a/lldb/source/Plugins/Process/MacOSX-Kernel/ProcessKDP.cpp
+++ b/lldb/source/Plugins/Process/MacOSX-Kernel/ProcessKDP.cpp
@@ -644,6 +644,9 @@ Status ProcessKDP::DoDeallocateMemory(lldb::addr_t addr) {
 }
 
 Status ProcessKDP::EnableBreakpointSite(BreakpointSite *bp_site) {
+  if (bp_site->HardwareRequired())
+    return Status("Hardware breakpoints are not supported.");
+
   if (m_comm.LocalBreakpointsAreSupported()) {
     Status error;
     if (!bp_site->IsEnabled()) {
diff --git a/lldb/source/Plugins/Process/Windows/Common/ProcessWindows.cpp b/lldb/source/Plugins/Process/Windows/Common/ProcessWindows.cpp
index 7b020f55e9938..96e2603b993ea 100644
--- a/lldb/source/Plugins/Process/Windows/Common/ProcessWindows.cpp
+++ b/lldb/source/Plugins/Process/Windows/Common/ProcessWindows.cpp
@@ -150,6 +150,9 @@ lldb_private::ConstString ProcessWindows::GetPluginName() {
 uint32_t ProcessWindows::GetPluginVersion() { return 1; }
 
 Status ProcessWindows::EnableBreakpointSite(BreakpointSite *bp_site) {
+  if (bp_site->HardwareRequired())
+    return Status("Hardware breakpoints are not supported.");
+
   Log *log = ProcessWindowsLog::GetLogIfAny(WINDOWS_LOG_BREAKPOINTS);
   LLDB_LOG(log, "bp_site = {0:x}, id={1}, addr={2:x}", bp_site,
            bp_site->GetID(), bp_site->GetLoadAddress());
diff --git a/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/require_hw_breakpoints/TestRequireHWBreakpoints.py b/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/require_hw_breakpoints/TestRequireHWBreakpoints.py
index c97795f7ae67c..dfb946036aa25 100644
--- a/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/require_hw_breakpoints/TestRequireHWBreakpoints.py
+++ b/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/require_hw_breakpoints/TestRequireHWBreakpoints.py
@@ -27,7 +27,6 @@ def test_breakpoint(self):
         breakpoint = target.BreakpointCreateByLocation("main.c", 1)
         self.assertTrue(breakpoint.IsHardware())
 
-    @skipIfWindows
     @expectedFailure(supports_hw_breakpoints)
     def test_step_range(self):
         """Test stepping when hardware breakpoints are required."""
@@ -49,7 +48,6 @@ def test_step_range(self):
         self.assertTrue("Could not create hardware breakpoint for thread plan"
                         in error.GetCString())
 
-    @skipIfWindows
     @expectedFailure(supports_hw_breakpoints)
     def test_step_out(self):
         """Test stepping out when hardware breakpoints are required."""
@@ -70,7 +68,6 @@ def test_step_out(self):
         self.assertTrue("Could not create hardware breakpoint for thread plan"
                         in error.GetCString())
 
-    @skipIfWindows
     @expectedFailure(supports_hw_breakpoints)
     def test_step_over(self):
         """Test stepping over when hardware breakpoints are required."""
@@ -89,7 +86,6 @@ def test_step_over(self):
                 'error: Could not create hardware breakpoint for thread plan.'
             ])
 
-    @skipIfWindows
     @expectedFailure(supports_hw_breakpoints)
     def test_step_until(self):
         """Test stepping until when hardware breakpoints are required."""

From ebaa8b1c60749883c6449a7c16096f1c40ccf4bc Mon Sep 17 00:00:00 2001
From: Tatyana Krasnukha <tatyana@synopsys.com>
Date: Tue, 21 Jul 2020 21:04:36 +0300
Subject: [PATCH 0517/1035] [lldb] Don't use hardware index to determine
 whether a breakpoint site is hardware

Most process plugins (if not all) don't set hardware index for breakpoints. They even
are not able to determine this index.

This patch makes StoppointLocation::IsHardware pure virtual and lets BreakpointSite
override it using more accurate BreakpointSite::Type.

It also adds assertions to be sure that a breakpoint site is hardware when this is required.

Differential Revision: https://reviews.llvm.org/D84257
---
 .../lldb/Breakpoint/BreakpointLocation.h      |  6 ++++
 lldb/include/lldb/Breakpoint/BreakpointSite.h |  7 ++++
 .../lldb/Breakpoint/StoppointLocation.h       |  4 +--
 lldb/source/Breakpoint/BreakpointLocation.cpp |  8 +++++
 lldb/source/Breakpoint/Watchpoint.cpp         |  5 ++-
 .../Process/gdb-remote/ProcessGDBRemote.cpp   | 10 ++----
 .../TestHWBreakMultiThread.py                 | 34 +++----------------
 7 files changed, 32 insertions(+), 42 deletions(-)

diff --git a/lldb/include/lldb/Breakpoint/BreakpointLocation.h b/lldb/include/lldb/Breakpoint/BreakpointLocation.h
index 3fc571eaa292b..04b2b89068998 100644
--- a/lldb/include/lldb/Breakpoint/BreakpointLocation.h
+++ b/lldb/include/lldb/Breakpoint/BreakpointLocation.h
@@ -76,6 +76,12 @@ class BreakpointLocation
   ///     \b true if the breakpoint is enabled, \b false if disabled.
   bool IsEnabled() const;
 
+  /// Check if it is a hardware breakpoint.
+  ///
+  /// \return
+  ///     \b true if the breakpoint is a harware breakpoint.
+  bool IsHardware() const override;
+
   /// If \a auto_continue is \b true, set the breakpoint to continue when hit.
   void SetAutoContinue(bool auto_continue);
 
diff --git a/lldb/include/lldb/Breakpoint/BreakpointSite.h b/lldb/include/lldb/Breakpoint/BreakpointSite.h
index 5ce17f511db44..6ddab5dc14606 100644
--- a/lldb/include/lldb/Breakpoint/BreakpointSite.h
+++ b/lldb/include/lldb/Breakpoint/BreakpointSite.h
@@ -15,6 +15,7 @@
 
 #include "lldb/Breakpoint/BreakpointLocationCollection.h"
 #include "lldb/Breakpoint/StoppointLocation.h"
+#include "lldb/Utility/LLDBAssert.h"
 #include "lldb/Utility/UserID.h"
 #include "lldb/lldb-forward.h"
 
@@ -184,6 +185,12 @@ class BreakpointSite : public std::enable_shared_from_this<BreakpointSite>,
   ///     \b false otherwise.
   bool IsInternal() const;
 
+  bool IsHardware() const override {
+    lldbassert(BreakpointSite::Type::eHardware == GetType() ||
+               !HardwareRequired());
+    return BreakpointSite::Type::eHardware == GetType();
+  }
+
   BreakpointSite::Type GetType() const { return m_type; }
 
   void SetType(BreakpointSite::Type type) { m_type = type; }
diff --git a/lldb/include/lldb/Breakpoint/StoppointLocation.h b/lldb/include/lldb/Breakpoint/StoppointLocation.h
index 4d6ca044ccc43..4392c7b84f9e8 100644
--- a/lldb/include/lldb/Breakpoint/StoppointLocation.h
+++ b/lldb/include/lldb/Breakpoint/StoppointLocation.h
@@ -40,9 +40,7 @@ class StoppointLocation {
 
   bool HardwareRequired() const { return m_hardware; }
 
-  virtual bool IsHardware() const {
-    return m_hardware_index != LLDB_INVALID_INDEX32;
-  }
+  virtual bool IsHardware() const = 0;
 
   virtual bool ShouldStop(StoppointCallbackContext *context) { return true; }
 
diff --git a/lldb/source/Breakpoint/BreakpointLocation.cpp b/lldb/source/Breakpoint/BreakpointLocation.cpp
index 93d54c051ee5f..eae1c1e033ad9 100644
--- a/lldb/source/Breakpoint/BreakpointLocation.cpp
+++ b/lldb/source/Breakpoint/BreakpointLocation.cpp
@@ -68,6 +68,14 @@ Breakpoint &BreakpointLocation::GetBreakpoint() { return m_owner; }
 
 Target &BreakpointLocation::GetTarget() { return m_owner.GetTarget(); }
 
+bool BreakpointLocation::IsHardware() const {
+  if (m_bp_site_sp)
+    return m_bp_site_sp->IsHardware();
+
+  // If breakpoint location is not resolved yet, it cannot be hardware.
+  return false;
+}
+
 bool BreakpointLocation::IsEnabled() const {
   if (!m_owner.IsEnabled())
     return false;
diff --git a/lldb/source/Breakpoint/Watchpoint.cpp b/lldb/source/Breakpoint/Watchpoint.cpp
index df73c6a172303..481c7b7eba8c0 100644
--- a/lldb/source/Breakpoint/Watchpoint.cpp
+++ b/lldb/source/Breakpoint/Watchpoint.cpp
@@ -95,7 +95,10 @@ void Watchpoint::SetWatchSpec(const std::string &str) {
 
 // Override default impl of StoppointLocation::IsHardware() since m_is_hardware
 // member field is more accurate.
-bool Watchpoint::IsHardware() const { return m_is_hardware; }
+bool Watchpoint::IsHardware() const {
+  lldbassert(m_is_hardware || !HardwareRequired());
+  return m_is_hardware;
+}
 
 bool Watchpoint::IsWatchVariable() const { return m_is_watch_variable; }
 
diff --git a/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp b/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp
index 1fed8e0642670..8dea8b9809854 100644
--- a/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp
+++ b/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp
@@ -3204,14 +3204,8 @@ Status ProcessGDBRemote::DisableBreakpointSite(BreakpointSite *bp_site) {
       break;
 
     case BreakpointSite::eExternal: {
-      GDBStoppointType stoppoint_type;
-      if (bp_site->IsHardware())
-        stoppoint_type = eBreakpointHardware;
-      else
-        stoppoint_type = eBreakpointSoftware;
-
-      if (m_gdb_comm.SendGDBStoppointTypePacket(stoppoint_type, false, addr,
-                                                bp_op_size))
+      if (m_gdb_comm.SendGDBStoppointTypePacket(eBreakpointSoftware, false,
+                                                addr, bp_op_size))
         error.SetErrorToGenericError();
     } break;
     }
diff --git a/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/hardware_breakpoint_on_multiple_threads/TestHWBreakMultiThread.py b/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/hardware_breakpoint_on_multiple_threads/TestHWBreakMultiThread.py
index 09db3ffd3ecd8..01bf33693a23b 100644
--- a/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/hardware_breakpoint_on_multiple_threads/TestHWBreakMultiThread.py
+++ b/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/hardware_breakpoint_on_multiple_threads/TestHWBreakMultiThread.py
@@ -17,27 +17,6 @@ class HardwareBreakpointMultiThreadTestCase(HardwareBreakpointTestBase):
     def does_not_support_hw_breakpoints(self):
         return not super().supports_hw_breakpoints()
 
-    # LLDB on linux supports hardware breakpoints for arm and aarch64
-    # architectures.
-    @skipUnlessPlatform(oslist=['linux'])
-    @skipTestIfFn(does_not_support_hw_breakpoints)
-    def test_hw_break_set_delete_multi_thread_linux(self):
-        self.build()
-        self.setTearDownCleanup()
-        self.break_multi_thread('delete', False) # llvm.org/PR44659
-
-    # LLDB on linux supports hardware breakpoints for arm and aarch64
-    # architectures.
-    @skipUnlessPlatform(oslist=['linux'])
-    @skipTestIfFn(does_not_support_hw_breakpoints)
-    def test_hw_break_set_disable_multi_thread_linux(self):
-        self.build()
-        self.setTearDownCleanup()
-        self.break_multi_thread('disable', False) # llvm.org/PR44659
-
-    # LLDB on darwin supports hardware breakpoints for x86_64 and i386
-    # architectures.
-    @skipUnlessDarwin
     @skipIfOutOfTreeDebugserver
     @skipTestIfFn(does_not_support_hw_breakpoints)
     def test_hw_break_set_delete_multi_thread_macos(self):
@@ -45,9 +24,6 @@ def test_hw_break_set_delete_multi_thread_macos(self):
         self.setTearDownCleanup()
         self.break_multi_thread('delete')
 
-    # LLDB on darwin supports hardware breakpoints for x86_64 and i386
-    # architectures.
-    @skipUnlessDarwin
     @skipIfOutOfTreeDebugserver
     @skipTestIfFn(does_not_support_hw_breakpoints)
     def test_hw_break_set_disable_multi_thread_macos(self):
@@ -55,7 +31,6 @@ def test_hw_break_set_disable_multi_thread_macos(self):
         self.setTearDownCleanup()
         self.break_multi_thread('disable')
 
-
     def setUp(self):
         # Call super's setUp().
         TestBase.setUp(self)
@@ -65,7 +40,7 @@ def setUp(self):
         self.first_stop = line_number(
             self.source, 'Starting thread creation with hardware breakpoint set')
 
-    def break_multi_thread(self, removal_type, check_hw_bp=True):
+    def break_multi_thread(self, removal_type):
         """Test that lldb hardware breakpoints work for multiple threads."""
         self.runCmd("file " + self.getBuildArtifact("a.out"),
                     CURRENT_EXECUTABLE_SET)
@@ -109,10 +84,9 @@ def break_multi_thread(self, removal_type, check_hw_bp=True):
             # Continue the loop and test that we are stopped 4 times.
             count += 1
 
-        if check_hw_bp:
-            # Check the breakpoint list.
-            self.expect("breakpoint list", substrs=['hw_break_function', 'hardware'])
-            self.expect("breakpoint list -v", substrs=['function = hw_break_function', 'hardware = true'])
+        # Check the breakpoint list.
+        self.expect("breakpoint list", substrs=['hw_break_function', 'hardware'])
+        self.expect("breakpoint list -v", substrs=['function = hw_break_function', 'hardware = true'])
 
         if removal_type == 'delete':
             self.runCmd("settings set auto-confirm true")

From d8ba6222097a585bd22e93b2dd343226ae2b9867 Mon Sep 17 00:00:00 2001
From: Amara Emerson <amara@apple.com>
Date: Wed, 29 Jul 2020 00:21:15 -0700
Subject: [PATCH 0518/1035] [AArch64][GlobalISel] Selection support for vector
 DUP[X]lane instructions.

In future, we'd like to use the perfect-shuffle mechanism to deal with these
shuffle permutations. For now, this improves performance by avoiding the
super-expensive const-pool load + tbl instruction.

Differential Revision: https://reviews.llvm.org/D84866
---
 llvm/include/llvm/CodeGen/GlobalISel/Utils.h  |   4 +
 llvm/lib/CodeGen/GlobalISel/Utils.cpp         |  21 ++++
 .../GISel/AArch64InstructionSelector.cpp      |  66 +++++++++++
 .../GISel/AArch64PostLegalizerCombiner.cpp    |  24 +---
 .../GlobalISel/select-shuffle-to-duplane.mir  | 103 ++++++++++++++++++
 .../GlobalISel/select-shuffle-vector.mir      |   4 +-
 6 files changed, 197 insertions(+), 25 deletions(-)
 create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/select-shuffle-to-duplane.mir

diff --git a/llvm/include/llvm/CodeGen/GlobalISel/Utils.h b/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
index 8eb95a8899913..35add316b5b67 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
@@ -213,5 +213,9 @@ LLT getLCMType(LLT OrigTy, LLT TargetTy);
 LLVM_READNONE
 LLT getGCDType(LLT OrigTy, LLT TargetTy);
 
+/// \returns The splat index of a G_SHUFFLE_VECTOR \p MI when \p MI is a splat.
+/// If \p MI is not a splat, returns None.
+Optional<int> getSplatIndex(MachineInstr &MI);
+
 } // End namespace llvm.
 #endif
diff --git a/llvm/lib/CodeGen/GlobalISel/Utils.cpp b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
index 584a691efb238..4a7513f23c6b9 100644
--- a/llvm/lib/CodeGen/GlobalISel/Utils.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
@@ -604,3 +604,24 @@ LLT llvm::getGCDType(LLT OrigTy, LLT TargetTy) {
   unsigned GCD = greatestCommonDivisor(OrigSize, TargetSize);
   return LLT::scalar(GCD);
 }
+
+Optional<int> llvm::getSplatIndex(MachineInstr &MI) {
+  assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR &&
+         "Only G_SHUFFLE_VECTOR can have a splat index!");
+  ArrayRef<int> Mask = MI.getOperand(3).getShuffleMask();
+  auto FirstDefinedIdx = find_if(Mask, [](int Elt) { return Elt >= 0; });
+
+  // If all elements are undefined, this shuffle can be considered a splat.
+  // Return 0 for better potential for callers to simplify.
+  if (FirstDefinedIdx == Mask.end())
+    return 0;
+
+  // Make sure all remaining elements are either undef or the same
+  // as the first non-undef value.
+  int SplatValue = *FirstDefinedIdx;
+  if (any_of(make_range(std::next(FirstDefinedIdx), Mask.end()),
+             [&SplatValue](int Elt) { return Elt >= 0 && Elt != SplatValue; }))
+    return None;
+
+  return SplatValue;
+}
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
index b96451e376c44..9f7950851f65c 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
@@ -133,6 +133,8 @@ class AArch64InstructionSelector : public InstructionSelector {
   bool selectMergeValues(MachineInstr &I, MachineRegisterInfo &MRI) const;
   bool selectUnmergeValues(MachineInstr &I, MachineRegisterInfo &MRI) const;
 
+  bool tryOptShuffleDupLane(MachineInstr &I, LLT DstTy, LLT SrcTy,
+                            ArrayRef<int> Mask, MachineRegisterInfo &MRI) const;
   bool selectShuffleVector(MachineInstr &I, MachineRegisterInfo &MRI) const;
   bool selectExtractElt(MachineInstr &I, MachineRegisterInfo &MRI) const;
   bool selectConcatVectors(MachineInstr &I, MachineRegisterInfo &MRI) const;
@@ -4306,6 +4308,67 @@ MachineInstr *AArch64InstructionSelector::tryOptArithShiftedCompare(
   return &*CmpMI;
 }
 
+bool AArch64InstructionSelector::tryOptShuffleDupLane(
+    MachineInstr &I, LLT DstTy, LLT SrcTy, ArrayRef<int> Mask,
+    MachineRegisterInfo &MRI) const {
+  assert(I.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
+
+  // We assume that scalar->vector splats have been been handled in the
+  // post-legalizer combiner to G_DUP. However splats of a source vector's
+  // lane don't fit that pattern, detect it here:
+  //  %res = G_SHUFFLE_VECTOR %src:<n x ty>, undef, <n x i32> splat(lane-idx)
+  //    =>
+  //  %res = DUPv[N][Ty]lane %src, lane-idx
+  // FIXME: this case should be covered by re-implementing the perfect shuffle
+  // codegen mechanism.
+
+  auto LaneIdx = getSplatIndex(I);
+  if (!LaneIdx)
+    return false;
+
+  // The lane idx should be within the first source vector.
+  if (*LaneIdx >= SrcTy.getNumElements())
+    return false;
+
+  if (DstTy != SrcTy)
+    return false;
+
+  LLT ScalarTy = SrcTy.getElementType();
+  unsigned ScalarSize = ScalarTy.getSizeInBits();
+
+  unsigned Opc = 0;
+  switch (SrcTy.getNumElements()) {
+  case 2:
+    if (ScalarSize == 64)
+      Opc = AArch64::DUPv2i64lane;
+    break;
+  case 4:
+    if (ScalarSize == 32)
+      Opc = AArch64::DUPv4i32lane;
+    break;
+  case 8:
+    if (ScalarSize == 16)
+      Opc = AArch64::DUPv8i16lane;
+    break;
+  case 16:
+    if (ScalarSize == 8)
+      Opc = AArch64::DUPv16i8lane;
+    break;
+  default:
+    break;
+  }
+  if (!Opc)
+    return false;
+
+  MachineIRBuilder MIB(I);
+  auto Dup = MIB.buildInstr(Opc, {I.getOperand(0).getReg()},
+                            {I.getOperand(1).getReg()})
+                 .addImm(*LaneIdx);
+  constrainSelectedInstRegOperands(*Dup, TII, TRI, RBI);
+  I.eraseFromParent();
+  return true;
+}
+
 bool AArch64InstructionSelector::selectShuffleVector(
     MachineInstr &I, MachineRegisterInfo &MRI) const {
   const LLT DstTy = MRI.getType(I.getOperand(0).getReg());
@@ -4327,6 +4390,9 @@ bool AArch64InstructionSelector::selectShuffleVector(
     return false;
   }
 
+  if (tryOptShuffleDupLane(I, DstTy, Src1Ty, Mask, MRI))
+    return true;
+
   unsigned BytesPerElt = DstTy.getElementType().getSizeInBits() / 8;
 
   SmallVector<Constant *, 64> CstIdxs;
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp b/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp
index baa8515baf3ea..15fa5cae2c424 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp
@@ -20,6 +20,7 @@
 #include "llvm/CodeGen/GlobalISel/CombinerInfo.h"
 #include "llvm/CodeGen/GlobalISel/GISelKnownBits.h"
 #include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
+#include "llvm/CodeGen/GlobalISel/Utils.h"
 #include "llvm/CodeGen/MachineDominators.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/CodeGen/TargetPassConfig.h"
@@ -43,29 +44,6 @@ struct ShuffleVectorPseudo {
   ShuffleVectorPseudo() {}
 };
 
-/// \returns The splat index of a G_SHUFFLE_VECTOR \p MI when \p MI is a splat.
-/// If \p MI is not a splat, returns None.
-static Optional<int> getSplatIndex(MachineInstr &MI) {
-  assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR &&
-         "Only G_SHUFFLE_VECTOR can have a splat index!");
-  ArrayRef<int> Mask = MI.getOperand(3).getShuffleMask();
-  auto FirstDefinedIdx = find_if(Mask, [](int Elt) { return Elt >= 0; });
-
-  // If all elements are undefined, this shuffle can be considered a splat.
-  // Return 0 for better potential for callers to simplify.
-  if (FirstDefinedIdx == Mask.end())
-    return 0;
-
-  // Make sure all remaining elements are either undef or the same
-  // as the first non-undef value.
-  int SplatValue = *FirstDefinedIdx;
-  if (any_of(make_range(std::next(FirstDefinedIdx), Mask.end()),
-             [&SplatValue](int Elt) { return Elt >= 0 && Elt != SplatValue; }))
-    return None;
-
-  return SplatValue;
-}
-
 /// Check if a vector shuffle corresponds to a REV instruction with the
 /// specified blocksize.
 static bool isREVMask(ArrayRef<int> M, unsigned EltSize, unsigned NumElts,
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/select-shuffle-to-duplane.mir b/llvm/test/CodeGen/AArch64/GlobalISel/select-shuffle-to-duplane.mir
new file mode 100644
index 0000000000000..5743c489fa206
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/select-shuffle-to-duplane.mir
@@ -0,0 +1,103 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=aarch64 -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s
+...
+---
+name:            duplane_v16i8
+alignment:       4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+liveins:
+  - { reg: '$q0' }
+body:             |
+  bb.1:
+    liveins: $q0
+
+    ; CHECK-LABEL: name: duplane_v16i8
+    ; CHECK: liveins: $q0
+    ; CHECK: [[COPY:%[0-9]+]]:fpr128 = COPY $q0
+    ; CHECK: [[DUPv16i8lane:%[0-9]+]]:fpr128 = DUPv16i8lane [[COPY]], 0
+    ; CHECK: $q0 = COPY [[DUPv16i8lane]]
+    ; CHECK: RET_ReallyLR implicit $q0
+    %0:fpr(<16 x s8>) = COPY $q0
+    %2:fpr(<16 x s8>) = G_IMPLICIT_DEF
+    %1:fpr(<16 x s8>) = G_SHUFFLE_VECTOR %0(<16 x s8>), %2, shufflemask(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)
+    $q0 = COPY %1(<16 x s8>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            duplane_v8i16
+alignment:       4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+liveins:
+  - { reg: '$q0' }
+body:             |
+  bb.1:
+    liveins: $q0
+
+    ; CHECK-LABEL: name: duplane_v8i16
+    ; CHECK: liveins: $q0
+    ; CHECK: [[COPY:%[0-9]+]]:fpr128 = COPY $q0
+    ; CHECK: [[DUPv8i16lane:%[0-9]+]]:fpr128 = DUPv8i16lane [[COPY]], 0
+    ; CHECK: $q0 = COPY [[DUPv8i16lane]]
+    ; CHECK: RET_ReallyLR implicit $q0
+    %0:fpr(<8 x s16>) = COPY $q0
+    %2:fpr(<8 x s16>) = G_IMPLICIT_DEF
+    %1:fpr(<8 x s16>) = G_SHUFFLE_VECTOR %0(<8 x s16>), %2, shufflemask(0, 0, 0, 0, 0, 0, 0, 0)
+    $q0 = COPY %1(<8 x s16>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            duplane_v4f32
+alignment:       4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+liveins:
+  - { reg: '$q0' }
+body:             |
+  bb.1:
+    liveins: $q0
+
+    ; CHECK-LABEL: name: duplane_v4f32
+    ; CHECK: liveins: $q0
+    ; CHECK: [[COPY:%[0-9]+]]:fpr128 = COPY $q0
+    ; CHECK: [[DUPv4i32lane:%[0-9]+]]:fpr128 = DUPv4i32lane [[COPY]], 0
+    ; CHECK: $q0 = COPY [[DUPv4i32lane]]
+    ; CHECK: RET_ReallyLR implicit $q0
+    %0:fpr(<4 x s32>) = COPY $q0
+    %2:fpr(<4 x s32>) = G_IMPLICIT_DEF
+    %1:fpr(<4 x s32>) = G_SHUFFLE_VECTOR %0(<4 x s32>), %2, shufflemask(0, 0, 0, 0)
+    $q0 = COPY %1(<4 x s32>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            duplane_v2i64
+alignment:       4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+liveins:
+  - { reg: '$q0' }
+body:             |
+  bb.1:
+    liveins: $q0
+
+    ; CHECK-LABEL: name: duplane_v2i64
+    ; CHECK: liveins: $q0
+    ; CHECK: [[COPY:%[0-9]+]]:fpr128 = COPY $q0
+    ; CHECK: [[DUPv2i64lane:%[0-9]+]]:fpr128 = DUPv2i64lane [[COPY]], 0
+    ; CHECK: $q0 = COPY [[DUPv2i64lane]]
+    ; CHECK: RET_ReallyLR implicit $q0
+    %0:fpr(<2 x s64>) = COPY $q0
+    %2:fpr(<2 x s64>) = G_IMPLICIT_DEF
+    %1:fpr(<2 x s64>) = G_SHUFFLE_VECTOR %0(<2 x s64>), %2, shufflemask(0, 0)
+    $q0 = COPY %1(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/select-shuffle-vector.mir b/llvm/test/CodeGen/AArch64/GlobalISel/select-shuffle-vector.mir
index c66d334d16b0b..345cc774f9963 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/select-shuffle-vector.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/select-shuffle-vector.mir
@@ -140,7 +140,7 @@ body:             |
 
     ; CHECK-LABEL: name: shuffle_v2i64
     ; CHECK: constants:
-    ; CHECK: value:           '<16 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7>'
+    ; CHECK: value:           '<16 x i8> <i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7>'
     ; CHECK: alignment:       16
     ; CHECK: isTargetSpecific: false
     ; CHECK: liveins: $q0, $q1
@@ -154,7 +154,7 @@ body:             |
     ; CHECK: RET_ReallyLR implicit $q0
     %0:fpr(<2 x s64>) = COPY $q0
     %1:fpr(<2 x s64>) = COPY $q1
-    %2:fpr(<2 x s64>) = G_SHUFFLE_VECTOR %0(<2 x s64>), %1, shufflemask(0, 0)
+    %2:fpr(<2 x s64>) = G_SHUFFLE_VECTOR %0(<2 x s64>), %1, shufflemask(1, 0)
     $q0 = COPY %2(<2 x s64>)
     RET_ReallyLR implicit $q0
 

From 8fe2abc190f95226fe48d920b7355d8134d03ed1 Mon Sep 17 00:00:00 2001
From: Philip Reames <listmail@philipreames.com>
Date: Wed, 29 Jul 2020 11:41:40 -0700
Subject: [PATCH 0519/1035] [Statepoint] Consolidate relocation type tracking
 [NFC]

Change the way we track how a particular pointer was relocated at a statepoint in selection dag.  Previously, we used an optional<location> for the spill lowering, and a block local Register for the newly introduced vreg lowering.  Combine all three lowerings (norelocate, spill, and vreg) into a single helper class, and keep a single copy of the information.

This is submitted separately as it really does make the code more readible on it's own, but the indirect motivation is to move vreg tracking from StatepointLowering to FunctionLoweringInfo.  This is the last piece needed to support cross block relocations with vregs; that will follow in a separate (non-NFC) patch.
---
 .../llvm/CodeGen/FunctionLoweringInfo.h       |  34 ++++--
 .../SelectionDAG/FunctionLoweringInfo.cpp     |   2 +-
 .../SelectionDAG/StatepointLowering.cpp       | 113 +++++++++---------
 .../CodeGen/SelectionDAG/StatepointLowering.h |   4 -
 4 files changed, 86 insertions(+), 67 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/FunctionLoweringInfo.h b/llvm/include/llvm/CodeGen/FunctionLoweringInfo.h
index c99ca00eac29f..b6bde0249f883 100644
--- a/llvm/include/llvm/CodeGen/FunctionLoweringInfo.h
+++ b/llvm/include/llvm/CodeGen/FunctionLoweringInfo.h
@@ -91,13 +91,33 @@ class FunctionLoweringInfo {
   /// Track virtual registers created for exception pointers.
   DenseMap<const Value *, Register> CatchPadExceptionPointers;
 
-  /// Keep track of frame indices allocated for statepoints as they could be
-  /// used across basic block boundaries (e.g. for an invoke).  For each
-  /// gc.statepoint instruction, maps uniqued llvm IR values to the slots they
-  /// were spilled in.  If a value is mapped to None it means we visited the
-  /// value but didn't spill it (because it was a constant, for instance).
-  using StatepointSpillMapTy = DenseMap<const Value *, Optional<int>>;
-  DenseMap<const Instruction *, StatepointSpillMapTy> StatepointSpillMaps;
+  /// Helper object to track which of three possible relocation mechanisms are
+  /// used for a particular value being relocated over a statepoint.
+  struct StatepointRelocationRecord {
+    enum RelocType {
+      // Value did not need to be relocated and can be used directly.
+      NoRelocate,
+      // Value was spilled to stack and needs filled at the gc.relocate.
+      Spill,
+      // Value was lowered to tied def and gc.relocate should be replaced with
+      // copy from vreg.
+      VReg,
+    } type = NoRelocate;
+    // Payload contains either frame index of the stack slot in which the value
+    // was spilled, or virtual register which contains the re-definition.
+    union payload_t {
+      payload_t() : FI(-1) {}
+      int FI;
+      Register Reg;
+    } payload;
+  };
+
+  /// Keep track of each value which was relocated and the strategy used to
+  /// relocate that value.  This information is required when visiting
+  /// gc.relocates which may appear in following blocks.
+  using StatepointSpillMapTy =
+    DenseMap<const Value *, StatepointRelocationRecord>;
+  DenseMap<const Instruction *, StatepointSpillMapTy> StatepointRelocationMaps;
 
   /// StaticAllocaMap - Keep track of frame indices for fixed sized allocas in
   /// the entry block.  This allows the allocas to be efficiently referenced
diff --git a/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp b/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp
index 5cf83cff3a903..fd821ed5c9692 100644
--- a/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp
@@ -360,7 +360,7 @@ void FunctionLoweringInfo::clear() {
   RegFixups.clear();
   RegsWithFixups.clear();
   StatepointStackSlots.clear();
-  StatepointSpillMaps.clear();
+  StatepointRelocationMaps.clear();
   PreferredExtendType.clear();
 }
 
diff --git a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
index 5fd1ab6f20912..044376c666706 100644
--- a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
@@ -71,6 +71,8 @@ cl::opt<unsigned> MaxRegistersForGCPointers(
     "max-registers-for-gc-values", cl::Hidden, cl::init(0),
     cl::desc("Max number of VRegs allowed to pass GC pointer meta args in"));
 
+typedef FunctionLoweringInfo::StatepointRelocationRecord RecordType;
+
 static void pushStackMapConstant(SmallVectorImpl<SDValue>& Ops,
                                  SelectionDAGBuilder &Builder, uint64_t Value) {
   SDLoc L = Builder.getCurSDLoc();
@@ -90,13 +92,11 @@ void StatepointLoweringState::startNewStatepoint(SelectionDAGBuilder &Builder) {
   // FunctionLoweringInfo.  Also need to ensure used bits get cleared.
   AllocatedStackSlots.clear();
   AllocatedStackSlots.resize(Builder.FuncInfo.StatepointStackSlots.size());
-  VirtRegs.clear();
 }
 
 void StatepointLoweringState::clear() {
   Locations.clear();
   AllocatedStackSlots.clear();
-  VirtRegs.clear();
   assert(PendingGCRelocateCalls.empty() &&
          "cleared before statepoint sequence completed");
 }
@@ -162,14 +162,18 @@ static Optional<int> findPreviousSpillSlot(const Value *Val,
 
   // Spill location is known for gc relocates
   if (const auto *Relocate = dyn_cast<GCRelocateInst>(Val)) {
-    const auto &SpillMap =
-        Builder.FuncInfo.StatepointSpillMaps[Relocate->getStatepoint()];
+    const auto &RelocationMap =
+        Builder.FuncInfo.StatepointRelocationMaps[Relocate->getStatepoint()];
+
+    auto It = RelocationMap.find(Relocate->getDerivedPtr());
+    if (It == RelocationMap.end())
+      return None;
 
-    auto It = SpillMap.find(Relocate->getDerivedPtr());
-    if (It == SpillMap.end())
+    auto &Record = It->second;
+    if (Record.type != RecordType::Spill)
       return None;
 
-    return It->second;
+    return Record.payload.FI;
   }
 
   // Look through bitcast instructions.
@@ -666,39 +670,6 @@ lowerStatepointMetaArgs(SmallVectorImpl<SDValue> &Ops,
       MemRefs.push_back(MMO);
     }
   }
-
-  // Record computed locations for all lowered values.
-  // This can not be embedded in lowering loops as we need to record *all*
-  // values, while previous loops account only values with unique SDValues.
-  const Instruction *StatepointInstr = SI.StatepointInstr;
-  auto &SpillMap = Builder.FuncInfo.StatepointSpillMaps[StatepointInstr];
-
-  for (const GCRelocateInst *Relocate : SI.GCRelocates) {
-    const Value *V = Relocate->getDerivedPtr();
-    SDValue SDV = Builder.getValue(V);
-    SDValue Loc = Builder.StatepointLowering.getLocation(SDV);
-
-    if (Loc.getNode()) {
-      // If this is a value we spilled, remember where for when we visit the
-      // gc.relocate corresponding to this gc.statepoint
-      SpillMap[V] = cast<FrameIndexSDNode>(Loc)->getIndex();
-    } else {
-      // If we didn't spill the value - allocas, constants, and values lowered
-      // as tied vregs - mark them as visited, but not spilled.  Marking them
-      // visited (as opposed to simply missing in the map), allows tighter
-      // assertion checking.  
-      SpillMap[V] = None;
-
-      // Conservatively export all values used by gc.relocates outside this
-      // block.  This is currently only needed for expressions which don't need
-      // relocation (such as constants and allocas).
-      if (!LowerAsVReg.count(SDV) &&
-          Relocate->getParent() != StatepointInstr->getParent()) {
-        Builder.ExportFromCurrentBlock(V);
-        assert(!LowerAsVReg.count(SDV));
-      }
-    }
-  }
 }
 
 SDValue SelectionDAGBuilder::LowerAsSTATEPOINT(
@@ -856,12 +827,10 @@ SDValue SelectionDAGBuilder::LowerAsSTATEPOINT(
     DAG.getMachineNode(TargetOpcode::STATEPOINT, getCurSDLoc(), NodeTys, Ops);
   DAG.setNodeMemRefs(StatepointMCNode, MemRefs);
 
-  SDNode *SinkNode = StatepointMCNode;
 
-  // Remember the mapping between values relocated in registers and the virtual
-  // register holding the relocation.  Note that for simplicity, we *always*
-  // create a vreg even within a single block.
-  auto &VirtRegs = StatepointLowering.VirtRegs;
+  // For values lowered to tied-defs, create the virtual registers.  Note that
+  // for simplicity, we *always* create a vreg even within a single block. 
+  DenseMap<const Value *, Register> VirtRegs;
   for (const auto *Relocate : SI.GCRelocates) {
     Value *Derived = Relocate->getDerivedPtr();
     SDValue SD = getValue(Derived);
@@ -885,6 +854,39 @@ SDValue SelectionDAGBuilder::LowerAsSTATEPOINT(
     VirtRegs[Derived] = Reg; 
   }
 
+  // Record for later use how each relocation was lowered.  This is needed to
+  // allow later gc.relocates to mirror the lowering chosen.
+  const Instruction *StatepointInstr = SI.StatepointInstr;
+  auto &RelocationMap = FuncInfo.StatepointRelocationMaps[StatepointInstr];
+  for (const GCRelocateInst *Relocate : SI.GCRelocates) {
+    const Value *V = Relocate->getDerivedPtr();
+    SDValue SDV = getValue(V);
+    SDValue Loc = StatepointLowering.getLocation(SDV);
+
+    RecordType Record;
+    if (Loc.getNode()) {
+      Record.type = RecordType::Spill;
+      Record.payload.FI = cast<FrameIndexSDNode>(Loc)->getIndex();
+    } else if (LowerAsVReg.count(SDV)) {
+      Record.type = RecordType::VReg;
+      assert(VirtRegs.count(V));
+      Record.payload.Reg = VirtRegs[V];
+    } else {
+      Record.type = RecordType::NoRelocate;
+      // If we didn't relocate a value, we'll essentialy end up inserting an
+      // additional use of the original value when lowering the gc.relocate.
+      // We need to make sure the value is available at the new use, which
+      // might be in another block.
+      if (Relocate->getParent() != StatepointInstr->getParent())
+        ExportFromCurrentBlock(V);
+    }
+    RelocationMap[V] = Record;
+  }
+
+  
+
+  SDNode *SinkNode = StatepointMCNode;
+
   // Build the GC_TRANSITION_END node if necessary.
   //
   // See the comment above regarding GC_TRANSITION_START for the layout of
@@ -1118,12 +1120,15 @@ void SelectionDAGBuilder::visitGCRelocate(const GCRelocateInst &Relocate) {
 #endif
 
   const Value *DerivedPtr = Relocate.getDerivedPtr();
+  auto &RelocationMap =
+    FuncInfo.StatepointRelocationMaps[Relocate.getStatepoint()];
+  auto SlotIt = RelocationMap.find(DerivedPtr);
+  assert(SlotIt != RelocationMap.end() && "Relocating not lowered gc value");
+  const RecordType &Record = SlotIt->second;
 
   // If relocation was done via virtual register..
-  auto &VirtRegs = StatepointLowering.VirtRegs;
-  auto It = VirtRegs.find(DerivedPtr);
-  if (It != VirtRegs.end()) {
-    Register InReg = It->second;
+  if (Record.type == RecordType::VReg) {
+    Register InReg = Record.payload.Reg;
     RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(),
                      DAG.getDataLayout(), InReg, Relocate.getType(),
                      None); // This is not an ABI copy.
@@ -1143,19 +1148,17 @@ void SelectionDAGBuilder::visitGCRelocate(const GCRelocateInst &Relocate) {
     return;
   }
 
-  auto &SpillMap = FuncInfo.StatepointSpillMaps[Relocate.getStatepoint()];
-  auto SlotIt = SpillMap.find(DerivedPtr);
-  assert(SlotIt != SpillMap.end() && "Relocating not lowered gc value");
-  Optional<int> DerivedPtrLocation = SlotIt->second;
 
   // We didn't need to spill these special cases (constants and allocas).
   // See the handling in spillIncomingValueForStatepoint for detail.
-  if (!DerivedPtrLocation) {
+  if (Record.type == RecordType::NoRelocate) {
     setValue(&Relocate, SD);
     return;
   }
 
-  unsigned Index = *DerivedPtrLocation;
+  assert(Record.type == RecordType::Spill);
+
+  unsigned Index = Record.payload.FI;;
   SDValue SpillSlot = DAG.getTargetFrameIndex(Index, getFrameIndexTy());
 
   // All the reloads are independent and are reading memory only modified by
diff --git a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.h b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.h
index 521a5c60053da..634ef87f3840e 100644
--- a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.h
+++ b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.h
@@ -103,10 +103,6 @@ class StatepointLoweringState {
     return AllocatedStackSlots.test(Offset);
   }
 
-  /// Maps pre-relocated value to virtual register holding it's relocation if
-  /// vreg lowering was used.
-  DenseMap<const Value *, Register> VirtRegs;
-
 private:
   /// Maps pre-relocation value (gc pointer directly incoming into statepoint)
   /// into it's location (currently only stack slots)

From 0c0e36061a8ebb9e81da9d057ebcd4c3d2a1f416 Mon Sep 17 00:00:00 2001
From: Amara Emerson <amara@apple.com>
Date: Fri, 24 Jul 2020 13:00:12 -0700
Subject: [PATCH 0520/1035] [GlobalISel] Add G_INTRINSIC_LRINT and translate
 from llvm.lrint

Differential Revision: https://reviews.llvm.org/D84551
---
 llvm/include/llvm/Support/TargetOpcodes.def            |  3 +++
 llvm/include/llvm/Target/GenericOpcodes.td             |  6 ++++++
 llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp           |  2 ++
 .../CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll   | 10 ++++++++++
 .../AArch64/GlobalISel/legalizer-info-validation.mir   |  3 +++
 5 files changed, 24 insertions(+)

diff --git a/llvm/include/llvm/Support/TargetOpcodes.def b/llvm/include/llvm/Support/TargetOpcodes.def
index 5eb3398562ba2..9470b1d52bbad 100644
--- a/llvm/include/llvm/Support/TargetOpcodes.def
+++ b/llvm/include/llvm/Support/TargetOpcodes.def
@@ -294,6 +294,9 @@ HANDLE_TARGET_OPCODE(G_INTRINSIC_TRUNC)
 /// INTRINSIC round intrinsic.
 HANDLE_TARGET_OPCODE(G_INTRINSIC_ROUND)
 
+/// INTRINSIC round to integer intrinsic.
+HANDLE_TARGET_OPCODE(G_INTRINSIC_LRINT)
+
 /// INTRINSIC readcyclecounter
 HANDLE_TARGET_OPCODE(G_READCYCLECOUNTER)
 
diff --git a/llvm/include/llvm/Target/GenericOpcodes.td b/llvm/include/llvm/Target/GenericOpcodes.td
index 5b04713d40e81..469ac13cca492 100644
--- a/llvm/include/llvm/Target/GenericOpcodes.td
+++ b/llvm/include/llvm/Target/GenericOpcodes.td
@@ -912,6 +912,12 @@ def G_INTRINSIC_ROUND : GenericInstruction {
   let hasSideEffects = 0;
 }
 
+def G_INTRINSIC_LRINT : GenericInstruction {
+  let OutOperandList = (outs type0:$dst);
+  let InOperandList = (ins type1:$src);
+  let hasSideEffects = 0;
+}
+
 def G_READCYCLECOUNTER : GenericInstruction {
   let OutOperandList = (outs type0:$dst);
   let InOperandList = (ins);
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index dba341660a550..6338f276b85da 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -1290,6 +1290,8 @@ unsigned IRTranslator::getSimpleIntrinsicOpcode(Intrinsic::ID ID) {
       return TargetOpcode::G_READCYCLECOUNTER;
     case Intrinsic::ptrmask:
       return TargetOpcode::G_PTRMASK;
+    case Intrinsic::lrint:
+      return TargetOpcode::G_INTRINSIC_LRINT;
   }
   return Intrinsic::not_intrinsic;
 }
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll
index 8975b2d0bc9ae..11ffacae7b866 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll
@@ -1391,6 +1391,16 @@ define float @test_intrinsic_round(float %a) {
   ret float %res
 }
 
+declare i32 @llvm.lrint.i32.f32(float)
+define i32 @test_intrinsic_lrint(float %a) {
+; CHECK-LABEL: name: test_intrinsic_lrint
+; CHECK: [[A:%[0-9]+]]:_(s32) = COPY $s0
+; CHECK: [[RES:%[0-9]+]]:_(s32) = G_INTRINSIC_LRINT [[A]]
+; CHECK: $w0 = COPY [[RES]]
+  %res = call i32 @llvm.lrint.i32.f32(float %a)
+  ret i32 %res
+}
+
 declare i32 @llvm.ctlz.i32(i32, i1)
 define i32 @test_ctlz_intrinsic_zero_not_undef(i32 %a) {
 ; CHECK-LABEL: name: test_ctlz_intrinsic_zero_not_undef
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
index b21f66e53fd48..08e416fd63163 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
@@ -129,6 +129,9 @@
 # DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
 # DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: G_INTRINSIC_LRINT (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
+# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT:.. imm index coverage check SKIPPED: no rules defined
 
 # DEBUG-NEXT: G_READCYCLECOUNTER (opcode {{[0-9]+}}): 1 type index, 0 imm indices
 # DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined

From d5c1f686e34fdd9ed8ec9b7a195799baf492e854 Mon Sep 17 00:00:00 2001
From: Tatyana Krasnukha <tatyana@synopsys.com>
Date: Wed, 29 Jul 2020 21:52:40 +0300
Subject: [PATCH 0521/1035] [lldb/BreakpointSite] Handle all ways of control
 flow

---
 lldb/source/Breakpoint/BreakpointSite.cpp | 59 ++++++++++++-----------
 1 file changed, 31 insertions(+), 28 deletions(-)

diff --git a/lldb/source/Breakpoint/BreakpointSite.cpp b/lldb/source/Breakpoint/BreakpointSite.cpp
index bdcabd7cce5ee..faa7e4c261e73 100644
--- a/lldb/source/Breakpoint/BreakpointSite.cpp
+++ b/lldb/source/Breakpoint/BreakpointSite.cpp
@@ -170,36 +170,39 @@ bool BreakpointSite::IntersectsRange(lldb::addr_t addr, size_t size,
   // The function should be called only for software breakpoints.
   lldbassert(GetType() == Type::eSoftware);
 
-  if (m_byte_size > 0) {
-    const lldb::addr_t bp_end_addr = m_addr + m_byte_size;
-    const lldb::addr_t end_addr = addr + size;
-    // Is the breakpoint end address before the passed in start address?
-    if (bp_end_addr <= addr)
-      return false;
-    // Is the breakpoint start address after passed in end address?
-    if (end_addr <= m_addr)
-      return false;
-    if (intersect_addr || intersect_size || opcode_offset) {
-      if (m_addr < addr) {
-        if (intersect_addr)
-          *intersect_addr = addr;
-        if (intersect_size)
-          *intersect_size =
-              std::min<lldb::addr_t>(bp_end_addr, end_addr) - addr;
-        if (opcode_offset)
-          *opcode_offset = addr - m_addr;
-      } else {
-        if (intersect_addr)
-          *intersect_addr = m_addr;
-        if (intersect_size)
-          *intersect_size =
-              std::min<lldb::addr_t>(bp_end_addr, end_addr) - m_addr;
-        if (opcode_offset)
-          *opcode_offset = 0;
-      }
+  if (m_byte_size == 0)
+    return false;
+
+  const lldb::addr_t bp_end_addr = m_addr + m_byte_size;
+  const lldb::addr_t end_addr = addr + size;
+  // Is the breakpoint end address before the passed in start address?
+  if (bp_end_addr <= addr)
+    return false;
+
+  // Is the breakpoint start address after passed in end address?
+  if (end_addr <= m_addr)
+    return false;
+
+  if (intersect_addr || intersect_size || opcode_offset) {
+    if (m_addr < addr) {
+      if (intersect_addr)
+        *intersect_addr = addr;
+      if (intersect_size)
+        *intersect_size =
+            std::min<lldb::addr_t>(bp_end_addr, end_addr) - addr;
+      if (opcode_offset)
+        *opcode_offset = addr - m_addr;
+    } else {
+      if (intersect_addr)
+        *intersect_addr = m_addr;
+      if (intersect_size)
+        *intersect_size =
+            std::min<lldb::addr_t>(bp_end_addr, end_addr) - m_addr;
+      if (opcode_offset)
+        *opcode_offset = 0;
     }
-    return true;
   }
+  return true;
 }
 
 size_t

From e2d0b44a7cd261218c9e527d23eb5d13425afe8b Mon Sep 17 00:00:00 2001
From: Matt Morehouse <mascasa@google.com>
Date: Wed, 29 Jul 2020 18:58:14 +0000
Subject: [PATCH 0522/1035] [DFSan] Add efficient fast16labels instrumentation
 mode.

Adds the -fast-16-labels flag, which enables efficient instrumentation
for DFSan when the user needs <=16 labels.  The instrumentation
eliminates most branches and most calls to __dfsan_union or
__dfsan_union_load.

Reviewed By: vitalybuka

Differential Revision: https://reviews.llvm.org/D84371
---
 clang/docs/DataFlowSanitizer.rst              |  52 +++++++++
 compiler-rt/lib/dfsan/dfsan.cpp               |  34 +++---
 compiler-rt/lib/dfsan/dfsan_flags.inc         |   4 -
 compiler-rt/lib/dfsan/done_abilist.txt        |   2 +
 .../lib/fuzzer/FuzzerDataFlowTrace.cpp        |   2 +-
 compiler-rt/lib/fuzzer/dataflow/DataFlow.cpp  |   6 +-
 compiler-rt/test/dfsan/fast16labels.c         |  30 +-----
 compiler-rt/test/fuzzer/dataflow.test         |   8 +-
 .../test/fuzzer/only-some-bytes-fork.test     |   4 +-
 compiler-rt/test/fuzzer/only-some-bytes.test  |   4 +-
 .../Instrumentation/DataFlowSanitizer.cpp     |  54 +++++++++-
 .../DataFlowSanitizer/fast16labels.ll         | 100 ++++++++++++++++++
 12 files changed, 235 insertions(+), 65 deletions(-)
 create mode 100644 llvm/test/Instrumentation/DataFlowSanitizer/fast16labels.ll

diff --git a/clang/docs/DataFlowSanitizer.rst b/clang/docs/DataFlowSanitizer.rst
index 44956037490a8..cd8c5869a017b 100644
--- a/clang/docs/DataFlowSanitizer.rst
+++ b/clang/docs/DataFlowSanitizer.rst
@@ -174,6 +174,58 @@ the correct labels are propagated.
     return 0;
   }
 
+fast16labels mode
+=================
+
+If you need 16 or fewer labels, you can use fast16labels instrumentation for
+less CPU and code size overhead.  To use fast16labels instrumentation, you'll
+need to specify `-fsanitize=dataflow -mllvm -dfsan-fast-16-labels` in your
+compile and link commands and use a modified API for creating and managing
+labels.
+
+In fast16labels mode, base labels are simply 16-bit unsigned integers that are
+powers of 2 (i.e. 1, 2, 4, 8, ..., 32768), and union labels are created by ORing
+base labels.  In this mode DFSan does not manage any label metadata, so the
+functions `dfsan_create_label`, `dfsan_union`, `dfsan_get_label_info`,
+`dfsan_has_label`, `dfsan_has_label_with_desc`, `dfsan_get_label_count`, and
+`dfsan_dump_labels` are unsupported.  Instead of using them, the user should
+maintain any necessary metadata about base labels themselves.
+
+For example:
+
+.. code-block:: c++
+
+  #include <sanitizer/dfsan_interface.h>
+  #include <assert.h>
+
+  int main(void) {
+    int i = 100;
+    int j = 200;
+    int k = 300;
+    dfsan_label i_label = 1;
+    dfsan_label j_label = 2;
+    dfsan_label k_label = 4;
+    dfsan_set_label(i_label, &i, sizeof(i));
+    dfsan_set_label(j_label, &j, sizeof(j));
+    dfsan_set_label(k_label, &k, sizeof(k));
+
+    dfsan_label ij_label = dfsan_get_label(i + j);
+
+    assert(ij_label & i_label);  // ij_label has i_label
+    assert(ij_label & j_label);  // ij_label has j_label
+    assert(!(ij_label & k_label));  // ij_label doesn't have k_label
+    assert(ij_label == 3)  // Verifies all of the above
+
+    dfsan_label ijk_label = dfsan_get_label(i + j + k);
+
+    assert(ijk_label & i_label);  // ijk_label has i_label
+    assert(ijk_label & j_label);  // ijk_label has j_label
+    assert(ijk_label & k_label);  // ijk_label has k_label
+    assert(ijk_label == 7);  // Verifies all of the above
+
+    return 0;
+  }
+
 Current status
 ==============
 
diff --git a/compiler-rt/lib/dfsan/dfsan.cpp b/compiler-rt/lib/dfsan/dfsan.cpp
index 105989c93ab10..678f6c1183e01 100644
--- a/compiler-rt/lib/dfsan/dfsan.cpp
+++ b/compiler-rt/lib/dfsan/dfsan.cpp
@@ -18,15 +18,16 @@
 // prefixed __dfsan_.
 //===----------------------------------------------------------------------===//
 
+#include "dfsan/dfsan.h"
+
 #include "sanitizer_common/sanitizer_atomic.h"
 #include "sanitizer_common/sanitizer_common.h"
 #include "sanitizer_common/sanitizer_file.h"
-#include "sanitizer_common/sanitizer_flags.h"
 #include "sanitizer_common/sanitizer_flag_parser.h"
+#include "sanitizer_common/sanitizer_flags.h"
+#include "sanitizer_common/sanitizer_internal_defs.h"
 #include "sanitizer_common/sanitizer_libc.h"
 
-#include "dfsan/dfsan.h"
-
 using namespace __dfsan;
 
 typedef atomic_uint16_t atomic_dfsan_label;
@@ -158,18 +159,10 @@ static void dfsan_check_label(dfsan_label label) {
   }
 }
 
-static void ReportUnsupportedFast16(const char *func) {
-  Report("FATAL: DataFlowSanitizer: %s is unsupported in fast16labels mode\n",
-         func);
-  Die();
-}
-
 // Resolves the union of two unequal labels.  Nonequality is a precondition for
 // this function (the instrumentation pass inlines the equality test).
 extern "C" SANITIZER_INTERFACE_ATTRIBUTE
 dfsan_label __dfsan_union(dfsan_label l1, dfsan_label l2) {
-  if (flags().fast16labels)
-    return l1 | l2;
   DCHECK_NE(l1, l2);
 
   if (l1 == 0)
@@ -224,6 +217,14 @@ dfsan_label __dfsan_union_load(const dfsan_label *ls, uptr n) {
   return label;
 }
 
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE
+dfsan_label __dfsan_union_load_fast16labels(const dfsan_label *ls, uptr n) {
+  dfsan_label label = ls[0];
+  for (uptr i = 1; i != n; ++i)
+    label |= ls[i];
+  return label;
+}
+
 extern "C" SANITIZER_INTERFACE_ATTRIBUTE
 void __dfsan_unimplemented(char *fname) {
   if (flags().warn_unimplemented)
@@ -259,8 +260,6 @@ dfsan_union(dfsan_label l1, dfsan_label l2) {
 
 extern "C" SANITIZER_INTERFACE_ATTRIBUTE
 dfsan_label dfsan_create_label(const char *desc, void *userdata) {
-  if (flags().fast16labels)
-    ReportUnsupportedFast16("dfsan_create_label");
   dfsan_label label =
       atomic_fetch_add(&__dfsan_last_label, 1, memory_order_relaxed) + 1;
   dfsan_check_label(label);
@@ -319,15 +318,11 @@ dfsan_read_label(const void *addr, uptr size) {
 
 extern "C" SANITIZER_INTERFACE_ATTRIBUTE
 const struct dfsan_label_info *dfsan_get_label_info(dfsan_label label) {
-  if (flags().fast16labels)
-    ReportUnsupportedFast16("dfsan_get_label_info");
   return &__dfsan_label_info[label];
 }
 
 extern "C" SANITIZER_INTERFACE_ATTRIBUTE int
 dfsan_has_label(dfsan_label label, dfsan_label elem) {
-  if (flags().fast16labels)
-    return label & elem;
   if (label == elem)
     return true;
   const dfsan_label_info *info = dfsan_get_label_info(label);
@@ -340,8 +335,6 @@ dfsan_has_label(dfsan_label label, dfsan_label elem) {
 
 extern "C" SANITIZER_INTERFACE_ATTRIBUTE dfsan_label
 dfsan_has_label_with_desc(dfsan_label label, const char *desc) {
-  if (flags().fast16labels)
-    ReportUnsupportedFast16("dfsan_has_label_with_desc");
   const dfsan_label_info *info = dfsan_get_label_info(label);
   if (info->l1 != 0) {
     return dfsan_has_label_with_desc(info->l1, desc) ||
@@ -361,9 +354,6 @@ dfsan_get_label_count(void) {
 
 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
 dfsan_dump_labels(int fd) {
-  if (flags().fast16labels)
-    return;
-
   dfsan_label last_label =
       atomic_load(&__dfsan_last_label, memory_order_relaxed);
   for (uptr l = 1; l <= last_label; ++l) {
diff --git a/compiler-rt/lib/dfsan/dfsan_flags.inc b/compiler-rt/lib/dfsan/dfsan_flags.inc
index 29db73b98278a..cdd0035c9b2d0 100644
--- a/compiler-rt/lib/dfsan/dfsan_flags.inc
+++ b/compiler-rt/lib/dfsan/dfsan_flags.inc
@@ -29,7 +29,3 @@ DFSAN_FLAG(
 DFSAN_FLAG(const char *, dump_labels_at_exit, "", "The path of the file where "
                                                   "to dump the labels when the "
                                                   "program terminates.")
-DFSAN_FLAG(bool, fast16labels, false,
-    "Enables experimental mode where DFSan supports only 16 power-of-2 labels "
-    "(1, 2, 4, 8, ... 32768) and the label union is computed as a bit-wise OR."
-)
diff --git a/compiler-rt/lib/dfsan/done_abilist.txt b/compiler-rt/lib/dfsan/done_abilist.txt
index 52f3ff5ef2395..540f0a4c6bfcd 100644
--- a/compiler-rt/lib/dfsan/done_abilist.txt
+++ b/compiler-rt/lib/dfsan/done_abilist.txt
@@ -28,6 +28,8 @@ fun:dfsan_set_write_callback=uninstrumented
 fun:dfsan_set_write_callback=custom
 fun:dfsan_flush=uninstrumented
 fun:dfsan_flush=discard
+fun:dfsan_use_fast16labels=uninstrumented
+fun:dfsan_use_fast16labels=discard
 
 ###############################################################################
 # glibc
diff --git a/compiler-rt/lib/fuzzer/FuzzerDataFlowTrace.cpp b/compiler-rt/lib/fuzzer/FuzzerDataFlowTrace.cpp
index 48df8e668604c..0e9cdf7e66b15 100644
--- a/compiler-rt/lib/fuzzer/FuzzerDataFlowTrace.cpp
+++ b/compiler-rt/lib/fuzzer/FuzzerDataFlowTrace.cpp
@@ -253,7 +253,7 @@ int CollectDataFlow(const std::string &DFTBinary, const std::string &DirPath,
     return 1;
   }
 
-  static char DFSanEnv[] = "DFSAN_OPTIONS=fast16labels=1:warn_unimplemented=0";
+  static char DFSanEnv[] = "DFSAN_OPTIONS=warn_unimplemented=0";
   putenv(DFSanEnv);
   MkDir(DirPath);
   for (auto &F : CorporaFiles) {
diff --git a/compiler-rt/lib/fuzzer/dataflow/DataFlow.cpp b/compiler-rt/lib/fuzzer/dataflow/DataFlow.cpp
index 8bf4e25b8b15b..78b3f9abd5cf5 100644
--- a/compiler-rt/lib/fuzzer/dataflow/DataFlow.cpp
+++ b/compiler-rt/lib/fuzzer/dataflow/DataFlow.cpp
@@ -17,9 +17,11 @@
 // and also provides basic-block coverage for every input.
 //
 // Build:
-//   1. Compile this file (DataFlow.cpp) with -fsanitize=dataflow and -O2.
+//   1. Compile this file (DataFlow.cpp) with -fsanitize=dataflow -mllvm
+//       -dfsan-fast-16-labels and -O2.
 //   2. Compile DataFlowCallbacks.cpp with -O2 -fPIC.
 //   3. Build the fuzz target with -g -fsanitize=dataflow
+//       -mllvm -dfsan-fast-16-labels
 //       -fsanitize-coverage=trace-pc-guard,pc-table,bb,trace-cmp
 //   4. Link those together with -fsanitize=dataflow
 //
@@ -36,7 +38,7 @@
 // Run:
 //   # Collect data flow and coverage for INPUT_FILE
 //   # write to OUTPUT_FILE (default: stdout)
-//   export DFSAN_OPTIONS=fast16labels=1:warn_unimplemented=0
+//   export DFSAN_OPTIONS=warn_unimplemented=0
 //   ./a.out INPUT_FILE [OUTPUT_FILE]
 //
 //   # Print all instrumented functions. llvm-symbolizer must be present in PATH
diff --git a/compiler-rt/test/dfsan/fast16labels.c b/compiler-rt/test/dfsan/fast16labels.c
index 269d1e2efc0af..72aaf38bfed42 100644
--- a/compiler-rt/test/dfsan/fast16labels.c
+++ b/compiler-rt/test/dfsan/fast16labels.c
@@ -1,14 +1,8 @@
-// RUN: %clang_dfsan %s -o %t
-// RUN: DFSAN_OPTIONS=fast16labels=1 %run %t
-// RUN: DFSAN_OPTIONS=fast16labels=1 not %run %t dfsan_create_label 2>&1 \
-// RUN:   | FileCheck %s --check-prefix=CREATE-LABEL
-// RUN: DFSAN_OPTIONS=fast16labels=1 not %run %t dfsan_get_label_info 2>&1 \
-// RUN:   | FileCheck %s --check-prefix=GET-LABEL-INFO
-// RUN: DFSAN_OPTIONS=fast16labels=1 not %run %t dfsan_has_label_with_desc \
-// RUN:   2>&1 | FileCheck %s --check-prefix=HAS-LABEL-WITH-DESC
-//
-// Tests DFSAN_OPTIONS=fast16labels=1
+// RUN: %clang_dfsan %s -mllvm -dfsan-fast-16-labels -o %t
+// RUN: %run %t
 //
+// Tests fast16labels mode.
+
 #include <sanitizer/dfsan_interface.h>
 
 #include <assert.h>
@@ -20,19 +14,6 @@ int foo(int a, int b) {
 }
 
 int main(int argc, char *argv[]) {
-  // Death tests for unsupported API usage.
-  const char *command = (argc < 2) ? "" : argv[1];
-  // CREATE-LABEL: FATAL: DataFlowSanitizer: dfsan_create_label is unsupported
-  if (strcmp(command, "dfsan_create_label") == 0)
-    dfsan_create_label("", NULL);
-  // GET-LABEL-INFO: FATAL: DataFlowSanitizer: dfsan_get_label_info is unsupported
-  if (strcmp(command, "dfsan_get_label_info") == 0)
-    dfsan_get_label_info(1);
-  // HAS-LABEL-WITH-DESC: FATAL: DataFlowSanitizer: dfsan_has_label_with_desc is unsupported
-  if (strcmp(command, "dfsan_has_label_with_desc") == 0)
-    dfsan_has_label_with_desc(1, "");
-
-  // Supported usage.
   int a = 10;
   int b = 20;
   dfsan_set_label(8, &a, sizeof(a));
@@ -43,7 +24,4 @@ int main(int argc, char *argv[]) {
   dfsan_label l = dfsan_get_label(c);
   printf("C: 0x%x\n", l);
   assert(l == 520);  // OR of the other two labels.
-  assert(dfsan_has_label(l, 8));
-  assert(dfsan_has_label(l, 512));
-  assert(!dfsan_has_label(l, 1));
 }
diff --git a/compiler-rt/test/fuzzer/dataflow.test b/compiler-rt/test/fuzzer/dataflow.test
index 93cd580b9d52e..391f160d0044a 100644
--- a/compiler-rt/test/fuzzer/dataflow.test
+++ b/compiler-rt/test/fuzzer/dataflow.test
@@ -2,10 +2,10 @@
 REQUIRES: linux, x86_64
 
 # Build the tracer and the test.
-RUN: %no_fuzzer_cpp_compiler -c -fno-sanitize=all -fsanitize=dataflow  %S/../../lib/fuzzer/dataflow/DataFlow.cpp -o  %t-DataFlow.o
+RUN: %no_fuzzer_cpp_compiler -c -fno-sanitize=all -fsanitize=dataflow -mllvm -dfsan-fast-16-labels  %S/../../lib/fuzzer/dataflow/DataFlow.cpp -o  %t-DataFlow.o
 RUN: %no_fuzzer_cpp_compiler -c -fno-sanitize=all -fPIC                %S/../../lib/fuzzer/dataflow/DataFlowCallbacks.cpp -o  %t-DataFlowCallbacks.o
-RUN: %no_fuzzer_cpp_compiler    -fno-sanitize=all -fsanitize=dataflow -fsanitize-coverage=trace-pc-guard,pc-table,bb,trace-cmp   %S/ThreeFunctionsTest.cpp     %t-DataFlow*.o -o %t-ThreeFunctionsTestDF
-RUN: %no_fuzzer_cpp_compiler    -fno-sanitize=all -fsanitize=dataflow -fsanitize-coverage=trace-pc-guard,pc-table,bb,trace-cmp   %S/Labels20Test.cpp     %t-DataFlow*.o -o %t-Labels20TestDF
+RUN: %no_fuzzer_cpp_compiler    -fno-sanitize=all -fsanitize=dataflow -mllvm -dfsan-fast-16-labels -fsanitize-coverage=trace-pc-guard,pc-table,bb,trace-cmp   %S/ThreeFunctionsTest.cpp     %t-DataFlow*.o -o %t-ThreeFunctionsTestDF
+RUN: %no_fuzzer_cpp_compiler    -fno-sanitize=all -fsanitize=dataflow -mllvm -dfsan-fast-16-labels -fsanitize-coverage=trace-pc-guard,pc-table,bb,trace-cmp   %S/Labels20Test.cpp     %t-DataFlow*.o -o %t-Labels20TestDF
 RUN: %cpp_compiler %S/ThreeFunctionsTest.cpp -o %t-ThreeFunctionsTest
 
 # Dump the function list.
@@ -30,7 +30,7 @@ RUN: echo -n FUZxxxxxxxxxxxxxxxxx > %t/IN20/FUZxxxxxxxxxxxxxxxxx
 RUN: echo -n FUxxxxxxxxxxxxxxxxxx > %t/IN20/FUxxxxxxxxxxxxxxxxxx
 
 
-RUN: export DFSAN_OPTIONS=fast16labels=1:warn_unimplemented=0
+RUN: export DFSAN_OPTIONS=warn_unimplemented=0
 
 # This test assumes that the functions in ThreeFunctionsTestDF are instrumented
 # in a specific order:
diff --git a/compiler-rt/test/fuzzer/only-some-bytes-fork.test b/compiler-rt/test/fuzzer/only-some-bytes-fork.test
index ef03c24fd2c63..a4142dbbb3b9a 100644
--- a/compiler-rt/test/fuzzer/only-some-bytes-fork.test
+++ b/compiler-rt/test/fuzzer/only-some-bytes-fork.test
@@ -2,9 +2,9 @@
 REQUIRES: linux, x86_64
 
 # Build the tracer and the test.
-RUN: %no_fuzzer_cpp_compiler -c -fno-sanitize=all -fsanitize=dataflow  %S/../../lib/fuzzer/dataflow/DataFlow.cpp -o  %t-DataFlow.o
+RUN: %no_fuzzer_cpp_compiler -c -fno-sanitize=all -fsanitize=dataflow -mllvm -dfsan-fast-16-labels %S/../../lib/fuzzer/dataflow/DataFlow.cpp -o  %t-DataFlow.o
 RUN: %no_fuzzer_cpp_compiler -c -fno-sanitize=all -fPIC                %S/../../lib/fuzzer/dataflow/DataFlowCallbacks.cpp -o  %t-DataFlowCallbacks.o
-RUN: %no_fuzzer_cpp_compiler    -fno-sanitize=all -fsanitize=dataflow -fsanitize-coverage=trace-pc-guard,pc-table,bb,trace-cmp   %S/OnlySomeBytesTest.cpp     %t-DataFlow*.o -o %t-DFT
+RUN: %no_fuzzer_cpp_compiler    -fno-sanitize=all -fsanitize=dataflow -mllvm -dfsan-fast-16-labels -fsanitize-coverage=trace-pc-guard,pc-table,bb,trace-cmp   %S/OnlySomeBytesTest.cpp     %t-DataFlow*.o -o %t-DFT
 RUN: %cpp_compiler %S/OnlySomeBytesTest.cpp -o %t-Fuzz
 
 # Test that the fork mode can collect and use the DFT
diff --git a/compiler-rt/test/fuzzer/only-some-bytes.test b/compiler-rt/test/fuzzer/only-some-bytes.test
index 62703f398f76b..9a31ebfc93c1e 100644
--- a/compiler-rt/test/fuzzer/only-some-bytes.test
+++ b/compiler-rt/test/fuzzer/only-some-bytes.test
@@ -2,9 +2,9 @@
 REQUIRES: linux, x86_64
 
 # Build the tracer and the test.
-RUN: %no_fuzzer_cpp_compiler -c -fno-sanitize=all -fsanitize=dataflow  %S/../../lib/fuzzer/dataflow/DataFlow.cpp -o  %t-DataFlow.o
+RUN: %no_fuzzer_cpp_compiler -c -fno-sanitize=all -fsanitize=dataflow -mllvm -dfsan-fast-16-labels %S/../../lib/fuzzer/dataflow/DataFlow.cpp -o  %t-DataFlow.o
 RUN: %no_fuzzer_cpp_compiler -c -fno-sanitize=all -fPIC                %S/../../lib/fuzzer/dataflow/DataFlowCallbacks.cpp -o  %t-DataFlowCallbacks.o
-RUN: %no_fuzzer_cpp_compiler    -fno-sanitize=all -fsanitize=dataflow -fsanitize-coverage=trace-pc-guard,pc-table,bb,trace-cmp   %S/OnlySomeBytesTest.cpp     %t-DataFlow*.o -o %t-DFT
+RUN: %no_fuzzer_cpp_compiler    -fno-sanitize=all -fsanitize=dataflow -mllvm -dfsan-fast-16-labels -fsanitize-coverage=trace-pc-guard,pc-table,bb,trace-cmp   %S/OnlySomeBytesTest.cpp     %t-DataFlow*.o -o %t-DFT
 RUN: %cpp_compiler %S/OnlySomeBytesTest.cpp -o %t-Fuzz
 
 # Prepare the inputs.
diff --git a/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
index 317b0904876ae..feb1abb18777b 100644
--- a/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
@@ -178,6 +178,14 @@ static cl::opt<bool> ClEventCallbacks(
     cl::desc("Insert calls to __dfsan_*_callback functions on data events."),
     cl::Hidden, cl::init(false));
 
+// Use a distinct bit for each base label, enabling faster unions with less
+// instrumentation.  Limits the max number of base labels to 16.
+static cl::opt<bool> ClFast16Labels(
+    "dfsan-fast-16-labels",
+    cl::desc("Use more efficient instrumentation, limiting the number of "
+             "labels to 16."),
+    cl::Hidden, cl::init(false));
+
 static StringRef GetGlobalTypeString(const GlobalValue &G) {
   // Types of GlobalVariables are always pointer types.
   Type *GType = G.getValueType();
@@ -362,6 +370,7 @@ class DataFlowSanitizer {
   FunctionCallee DFSanUnionFn;
   FunctionCallee DFSanCheckedUnionFn;
   FunctionCallee DFSanUnionLoadFn;
+  FunctionCallee DFSanUnionLoadFast16LabelsFn;
   FunctionCallee DFSanUnimplementedFn;
   FunctionCallee DFSanSetLabelFn;
   FunctionCallee DFSanNonzeroLabelFn;
@@ -748,6 +757,17 @@ void DataFlowSanitizer::initializeRuntimeFunctions(Module &M) {
     DFSanUnionLoadFn =
         Mod->getOrInsertFunction("__dfsan_union_load", DFSanUnionLoadFnTy, AL);
   }
+  {
+    AttributeList AL;
+    AL = AL.addAttribute(M.getContext(), AttributeList::FunctionIndex,
+                         Attribute::NoUnwind);
+    AL = AL.addAttribute(M.getContext(), AttributeList::FunctionIndex,
+                         Attribute::ReadOnly);
+    AL = AL.addAttribute(M.getContext(), AttributeList::ReturnIndex,
+                         Attribute::ZExt);
+    DFSanUnionLoadFast16LabelsFn = Mod->getOrInsertFunction(
+        "__dfsan_union_load_fast16labels", DFSanUnionLoadFnTy, AL);
+  }
   DFSanUnimplementedFn =
       Mod->getOrInsertFunction("__dfsan_unimplemented", DFSanUnimplementedFnTy);
   {
@@ -810,6 +830,7 @@ bool DataFlowSanitizer::runImpl(Module &M) {
         &i != DFSanUnionFn.getCallee()->stripPointerCasts() &&
         &i != DFSanCheckedUnionFn.getCallee()->stripPointerCasts() &&
         &i != DFSanUnionLoadFn.getCallee()->stripPointerCasts() &&
+        &i != DFSanUnionLoadFast16LabelsFn.getCallee()->stripPointerCasts() &&
         &i != DFSanUnimplementedFn.getCallee()->stripPointerCasts() &&
         &i != DFSanSetLabelFn.getCallee()->stripPointerCasts() &&
         &i != DFSanNonzeroLabelFn.getCallee()->stripPointerCasts() &&
@@ -1144,7 +1165,10 @@ Value *DFSanFunction::combineShadows(Value *V1, Value *V2, Instruction *Pos) {
     return CCS.Shadow;
 
   IRBuilder<> IRB(Pos);
-  if (AvoidNewBlocks) {
+  if (ClFast16Labels) {
+    CCS.Block = Pos->getParent();
+    CCS.Shadow = IRB.CreateOr(V1, V2);
+  } else if (AvoidNewBlocks) {
     CallInst *Call = IRB.CreateCall(DFS.DFSanCheckedUnionFn, {V1, V2});
     Call->addAttribute(AttributeList::ReturnIndex, Attribute::ZExt);
     Call->addParamAttr(0, Attribute::ZExt);
@@ -1254,6 +1278,30 @@ Value *DFSanFunction::loadShadow(Value *Addr, uint64_t Size, uint64_t Align,
         IRB.CreateAlignedLoad(DFS.ShadowTy, ShadowAddr1, ShadowAlign), Pos);
   }
   }
+
+  if (ClFast16Labels && Size % (64 / DFS.ShadowWidthBits) == 0) {
+    // First OR all the WideShadows, then OR individual shadows within the
+    // combined WideShadow.  This is fewer instructions than ORing shadows
+    // individually.
+    IRBuilder<> IRB(Pos);
+    Value *WideAddr =
+        IRB.CreateBitCast(ShadowAddr, Type::getInt64PtrTy(*DFS.Ctx));
+    Value *CombinedWideShadow =
+        IRB.CreateAlignedLoad(IRB.getInt64Ty(), WideAddr, ShadowAlign);
+    for (uint64_t Ofs = 64 / DFS.ShadowWidthBits; Ofs != Size;
+         Ofs += 64 / DFS.ShadowWidthBits) {
+      WideAddr = IRB.CreateGEP(Type::getInt64Ty(*DFS.Ctx), WideAddr,
+                               ConstantInt::get(DFS.IntptrTy, 1));
+      Value *NextWideShadow =
+          IRB.CreateAlignedLoad(IRB.getInt64Ty(), WideAddr, ShadowAlign);
+      CombinedWideShadow = IRB.CreateOr(CombinedWideShadow, NextWideShadow);
+    }
+    for (unsigned Width = 32; Width >= DFS.ShadowWidthBits; Width >>= 1) {
+      Value *ShrShadow = IRB.CreateLShr(CombinedWideShadow, Width);
+      CombinedWideShadow = IRB.CreateOr(CombinedWideShadow, ShrShadow);
+    }
+    return IRB.CreateTrunc(CombinedWideShadow, DFS.ShadowTy);
+  }
   if (!AvoidNewBlocks && Size % (64 / DFS.ShadowWidthBits) == 0) {
     // Fast path for the common case where each byte has identical shadow: load
     // shadow 64 bits at a time, fall out to a __dfsan_union_load call if any
@@ -1320,8 +1368,10 @@ Value *DFSanFunction::loadShadow(Value *Addr, uint64_t Size, uint64_t Align,
   }
 
   IRBuilder<> IRB(Pos);
+  FunctionCallee &UnionLoadFn =
+      ClFast16Labels ? DFS.DFSanUnionLoadFast16LabelsFn : DFS.DFSanUnionLoadFn;
   CallInst *FallbackCall = IRB.CreateCall(
-      DFS.DFSanUnionLoadFn, {ShadowAddr, ConstantInt::get(DFS.IntptrTy, Size)});
+      UnionLoadFn, {ShadowAddr, ConstantInt::get(DFS.IntptrTy, Size)});
   FallbackCall->addAttribute(AttributeList::ReturnIndex, Attribute::ZExt);
   return FallbackCall;
 }
diff --git a/llvm/test/Instrumentation/DataFlowSanitizer/fast16labels.ll b/llvm/test/Instrumentation/DataFlowSanitizer/fast16labels.ll
new file mode 100644
index 0000000000000..85f99f5b1828e
--- /dev/null
+++ b/llvm/test/Instrumentation/DataFlowSanitizer/fast16labels.ll
@@ -0,0 +1,100 @@
+; Test that -dfsan-fast-16-labels mode uses inline ORs rather than calling
+; __dfsan_union or __dfsan_union_load.
+; RUN: opt < %s -dfsan -dfsan-fast-16-labels -S | FileCheck %s --implicit-check-not="call{{.*}}__dfsan_union"
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i8 @add(i8 %a, i8 %b) {
+  ; CHECK-LABEL: define i8 @"dfs$add"
+  ; CHECK-DAG: %[[ALABEL:.*]] = load{{.*}}__dfsan_arg_tls, i64 0, i64 0
+  ; CHECK-DAG: %[[BLABEL:.*]] = load{{.*}}__dfsan_arg_tls, i64 0, i64 1
+  ; CHECK: %[[ADDLABEL:.*]] = or i16 %[[ALABEL]], %[[BLABEL]]
+  ; CHECK: add i8
+  ; CHECK: store i16 %[[ADDLABEL]], i16* @__dfsan_retval_tls
+  ; CHECK: ret i8
+  %c = add i8 %a, %b
+  ret i8 %c
+}
+
+define i8 @load8(i8* %p) {
+  ; CHECK-LABEL: define i8 @"dfs$load8"
+  ; CHECK: load i16, i16*
+  ; CHECK: ptrtoint i8* {{.*}} to i64
+  ; CHECK: and i64
+  ; CHECK: mul i64
+  ; CHECK: inttoptr i64
+  ; CHECK: load i16, i16*
+  ; CHECK: or i16
+  ; CHECK: load i8, i8*
+  ; CHECK: store i16 {{.*}} @__dfsan_retval_tls
+  ; CHECK: ret i8
+
+  %a = load i8, i8* %p
+  ret i8 %a
+}
+
+define i16 @load16(i16* %p) {
+  ; CHECK-LABEL: define i16 @"dfs$load16"
+  ; CHECK: ptrtoint i16*
+  ; CHECK: and i64
+  ; CHECK: mul i64
+  ; CHECK: inttoptr i64 {{.*}} i16*
+  ; CHECK: getelementptr i16
+  ; CHECK: load i16, i16*
+  ; CHECK: load i16, i16*
+  ; CHECK: or i16
+  ; CHECK: or i16
+  ; CHECK: load i16, i16*
+  ; CHECK: store {{.*}} @__dfsan_retval_tls
+  ; CHECK: ret i16
+
+  %a = load i16, i16* %p
+  ret i16 %a
+}
+
+define i32 @load32(i32* %p) {
+  ; CHECK-LABEL: define i32 @"dfs$load32"
+  ; CHECK: ptrtoint i32*
+  ; CHECK: and i64
+  ; CHECK: mul i64
+  ; CHECK: inttoptr i64 {{.*}} i16*
+  ; CHECK: bitcast i16* {{.*}} i64*
+  ; CHECK: load i64, i64*
+  ; CHECK: lshr i64 {{.*}}, 32
+  ; CHECK: or i64
+  ; CHECK: lshr i64 {{.*}}, 16
+  ; CHECK: or i64
+  ; CHECK: trunc i64 {{.*}} i16
+  ; CHECK: or i16
+  ; CHECK: load i32, i32*
+  ; CHECK: store i16 {{.*}} @__dfsan_retval_tls
+  ; CHECK: ret i32
+
+  %a = load i32, i32* %p
+  ret i32 %a
+}
+
+define i64 @load64(i64* %p) {
+  ; CHECK-LABEL: define i64 @"dfs$load64"
+  ; CHECK: ptrtoint i64*
+  ; CHECK: and i64
+  ; CHECK: mul i64
+  ; CHECK: inttoptr i64 {{.*}} i16*
+  ; CHECK: bitcast i16* {{.*}} i64*
+  ; CHECK: load i64, i64*
+  ; CHECK: getelementptr i64, i64* {{.*}}, i64 1
+  ; CHECK: load i64, i64*
+  ; CHECK: or i64
+  ; CHECK: lshr i64 {{.*}}, 32
+  ; CHECK: or i64
+  ; CHECK: lshr i64 {{.*}}, 16
+  ; CHECK: or i64
+  ; CHECK: trunc i64 {{.*}} i16
+  ; CHECK: or i16
+  ; CHECK: load i64, i64*
+  ; CHECK: store i16 {{.*}} @__dfsan_retval_tls
+  ; CHECK: ret i64
+
+  %a = load i64, i64* %p
+  ret i64 %a
+}

From 7aaa85627b5937364d4c944d8f6828a6a3759785 Mon Sep 17 00:00:00 2001
From: Baptiste Saleil <baptiste.saleil@ibm.com>
Date: Tue, 28 Jul 2020 16:02:50 -0500
Subject: [PATCH 0523/1035] [PowerPC] Add options to control paired vector
 memops support

Adds frontend and backend options to enable and disable the
PowerPC paired vector memory operations added in ISA 3.1.
Instructions using these options will be added in subsequent patches.

Differential Revision: https://reviews.llvm.org/D83722
---
 clang/include/clang/Driver/Options.td             |  4 ++++
 clang/lib/Basic/Targets/PPC.cpp                   | 15 ++++++++++++---
 clang/lib/Basic/Targets/PPC.h                     |  1 +
 clang/test/Driver/ppc-dependent-options.cpp       |  5 +++++
 llvm/lib/Target/PowerPC/PPC.td                    |  6 +++++-
 llvm/lib/Target/PowerPC/PPCInstrPrefix.td         |  1 +
 llvm/lib/Target/PowerPC/PPCScheduleP9.td          |  6 ++++--
 llvm/lib/Target/PowerPC/PPCSubtarget.cpp          |  1 +
 llvm/lib/Target/PowerPC/PPCSubtarget.h            |  2 ++
 .../test/CodeGen/PowerPC/future-check-features.ll | 12 ++++++------
 10 files changed, 41 insertions(+), 12 deletions(-)

diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index b6e31700c0a62..7d91bad1f848f 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -2453,6 +2453,10 @@ def mdirect_move : Flag<["-"], "mdirect-move">,
     Group<m_ppc_Features_Group>;
 def mnodirect_move : Flag<["-"], "mno-direct-move">,
     Group<m_ppc_Features_Group>;
+def mpaired_vector_memops: Flag<["-"], "mpaired-vector-memops">,
+    Group<m_ppc_Features_Group>;
+def mnopaired_vector_memops: Flag<["-"], "mno-paired-vector-memops">,
+    Group<m_ppc_Features_Group>;
 def mhtm : Flag<["-"], "mhtm">, Group<m_ppc_Features_Group>;
 def mno_htm : Flag<["-"], "mno-htm">, Group<m_ppc_Features_Group>;
 def mfprnd : Flag<["-"], "mfprnd">, Group<m_ppc_Features_Group>;
diff --git a/clang/lib/Basic/Targets/PPC.cpp b/clang/lib/Basic/Targets/PPC.cpp
index 5f716a541ae92..c5ad4a5a2d270 100644
--- a/clang/lib/Basic/Targets/PPC.cpp
+++ b/clang/lib/Basic/Targets/PPC.cpp
@@ -62,6 +62,8 @@ bool PPCTargetInfo::handleTargetFeatures(std::vector<std::string> &Features,
       LongDoubleFormat = &llvm::APFloat::IEEEdouble();
     } else if (Feature == "-hard-float") {
       FloatABI = SoftFloat;
+    } else if (Feature == "+paired-vector-memops") {
+      PairedVectorMemops = true;
     }
     // TODO: Finish this list and add an assert that we've handled them
     // all.
@@ -218,6 +220,7 @@ void PPCTargetInfo::getTargetDefines(const LangOptions &Opts,
 // - direct-move
 // - float128
 // - power9-vector
+// - paired-vector-memops
 // - power10-vector
 // then go ahead and error since the customer has expressed an incompatible
 // set of options.
@@ -240,6 +243,7 @@ static bool ppcUserFeaturesCheck(DiagnosticsEngine &Diags,
   Found |= FindVSXSubfeature("+direct-move", "-mdirect-move");
   Found |= FindVSXSubfeature("+float128", "-mfloat128");
   Found |= FindVSXSubfeature("+power9-vector", "-mpower9-vector");
+  Found |= FindVSXSubfeature("+paired-vector-memops", "-mpaired-vector-memops");
   Found |= FindVSXSubfeature("+power10-vector", "-mpower10-vector");
 
   // Return false if any vsx subfeatures was found.
@@ -340,6 +344,7 @@ bool PPCTargetInfo::initFeatureMap(
 void PPCTargetInfo::addP10SpecificFeatures(
     llvm::StringMap<bool> &Features) const {
   Features["htm"] = false; // HTM was removed for P10.
+  Features["paired-vector-memops"] = true;
   Features["power10-vector"] = true;
   Features["pcrelative-memops"] = true;
   return;
@@ -364,6 +369,7 @@ bool PPCTargetInfo::hasFeature(StringRef Feature) const {
       .Case("extdiv", HasExtDiv)
       .Case("float128", HasFloat128)
       .Case("power9-vector", HasP9Vector)
+      .Case("paired-vector-memops", PairedVectorMemops)
       .Case("power10-vector", HasP10Vector)
       .Case("pcrelative-memops", HasPCRelativeMemops)
       .Case("spe", HasSPE)
@@ -380,6 +386,7 @@ void PPCTargetInfo::setFeatureEnabled(llvm::StringMap<bool> &Features,
                              .Case("direct-move", true)
                              .Case("power8-vector", true)
                              .Case("power9-vector", true)
+                             .Case("paired-vector-memops", true)
                              .Case("power10-vector", true)
                              .Case("float128", true)
                              .Default(false);
@@ -399,11 +406,13 @@ void PPCTargetInfo::setFeatureEnabled(llvm::StringMap<bool> &Features,
     if ((Name == "altivec") || (Name == "vsx"))
       Features["vsx"] = Features["direct-move"] = Features["power8-vector"] =
           Features["float128"] = Features["power9-vector"] =
-              Features["power10-vector"] = false;
+              Features["paired-vector-memops"] = Features["power10-vector"] =
+                  false;
     if (Name == "power8-vector")
-      Features["power9-vector"] = Features["power10-vector"] = false;
+      Features["power9-vector"] = Features["paired-vector-memops"] =
+          Features["power10-vector"] = false;
     else if (Name == "power9-vector")
-      Features["power10-vector"] = false;
+      Features["paired-vector-memops"] = Features["power10-vector"] = false;
     if (Name == "pcrel")
       Features["pcrelative-memops"] = false;
     else
diff --git a/clang/lib/Basic/Targets/PPC.h b/clang/lib/Basic/Targets/PPC.h
index c2048b2145918..da4d37bbdcfc8 100644
--- a/clang/lib/Basic/Targets/PPC.h
+++ b/clang/lib/Basic/Targets/PPC.h
@@ -67,6 +67,7 @@ class LLVM_LIBRARY_VISIBILITY PPCTargetInfo : public TargetInfo {
   bool HasExtDiv = false;
   bool HasP9Vector = false;
   bool HasSPE = false;
+  bool PairedVectorMemops = false;
   bool HasP10Vector = false;
   bool HasPCRelativeMemops = false;
 
diff --git a/clang/test/Driver/ppc-dependent-options.cpp b/clang/test/Driver/ppc-dependent-options.cpp
index 05e4b5b10fc49..1c1a0c38cdcb0 100644
--- a/clang/test/Driver/ppc-dependent-options.cpp
+++ b/clang/test/Driver/ppc-dependent-options.cpp
@@ -54,6 +54,10 @@
 // RUN: -mcpu=power9 -std=c++11 -mno-vsx -mfloat128 %s 2>&1 | \
 // RUN: FileCheck %s -check-prefix=CHECK-NVSX-FLT128
 
+// RUN: not %clang -target powerpc64le-unknown-unknown -fsyntax-only \
+// RUN: -mcpu=power10 -std=c++11 -mno-vsx -mpaired-vector-memops %s 2>&1 | \
+// RUN: FileCheck %s -check-prefix=CHECK-NVSX-PAIRED-VEC-MEMOPS
+
 // RUN: not %clang -target powerpc64le-unknown-unknown -fsyntax-only \
 // RUN: -mcpu=power9 -std=c++11 -mno-vsx -mfloat128 -mpower9-vector %s 2>&1 | \
 // RUN: FileCheck %s -check-prefix=CHECK-NVSX-MULTI
@@ -96,6 +100,7 @@ static_assert(false, "Neither enabled");
 // CHECK-NVSX-P10V: error: option '-mpower10-vector' cannot be specified with '-mno-vsx'
 // CHECK-NVSX-FLT128: error: option '-mfloat128' cannot be specified with '-mno-vsx'
 // CHECK-NVSX-DMV: error: option '-mdirect-move' cannot be specified with '-mno-vsx'
+// CHECK-NVSX-PAIRED-VEC-MEMOPS: error: option '-mpaired-vector-memops' cannot be specified with '-mno-vsx'
 // CHECK-NVSX-MULTI: error: option '-mfloat128' cannot be specified with '-mno-vsx'
 // CHECK-NVSX-MULTI: error: option '-mpower9-vector' cannot be specified with '-mno-vsx'
 // CHECK-NVSX: Neither enabled
diff --git a/llvm/lib/Target/PowerPC/PPC.td b/llvm/lib/Target/PowerPC/PPC.td
index adb9366217d51..cda809e2472d4 100644
--- a/llvm/lib/Target/PowerPC/PPC.td
+++ b/llvm/lib/Target/PowerPC/PPC.td
@@ -234,6 +234,10 @@ def FeaturePCRelativeMemops :
   SubtargetFeature<"pcrelative-memops", "HasPCRelativeMemops", "true",
                    "Enable PC relative Memory Ops",
                    [FeatureISA3_0]>;
+def FeaturePairedVectorMemops:
+  SubtargetFeature<"paired-vector-memops", "PairedVectorMemops", "true",
+                   "32Byte load and store instructions",
+                   [FeatureISA3_0]>;
 
 def FeaturePredictableSelectIsExpensive :
   SubtargetFeature<"predictable-select-expensive",
@@ -339,7 +343,7 @@ def ProcessorFeatures {
   // still exist with the exception of those we know are Power9 specific.
   list<SubtargetFeature> P10AdditionalFeatures =
     [DirectivePwr10, FeatureISA3_1, FeaturePrefixInstrs,
-     FeaturePCRelativeMemops, FeatureP10Vector];
+     FeaturePCRelativeMemops, FeatureP10Vector, FeaturePairedVectorMemops];
   list<SubtargetFeature> P10SpecificFeatures = [];
   list<SubtargetFeature> P10InheritableFeatures =
     !listconcat(P9InheritableFeatures, P10AdditionalFeatures);
diff --git a/llvm/lib/Target/PowerPC/PPCInstrPrefix.td b/llvm/lib/Target/PowerPC/PPCInstrPrefix.td
index fa21c54efc28b..8ff0d460f47fb 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrPrefix.td
+++ b/llvm/lib/Target/PowerPC/PPCInstrPrefix.td
@@ -454,6 +454,7 @@ multiclass 8LS_DForm_R_SI34_XT6_RA5_p<bits<5> opcode, dag OOL, dag IOL,
 
 def PrefixInstrs : Predicate<"Subtarget->hasPrefixInstrs()">;
 def IsISA3_1 : Predicate<"Subtarget->isISA3_1()">;
+def PairedVectorMemops : Predicate<"PPCSubTarget->pairedVectorMemops()">;
 
 let Predicates = [PrefixInstrs] in {
   let Interpretation64Bit = 1, isCodeGenOnly = 1 in {
diff --git a/llvm/lib/Target/PowerPC/PPCScheduleP9.td b/llvm/lib/Target/PowerPC/PPCScheduleP9.td
index 311d5cf165f63..c79d55f56b2a6 100644
--- a/llvm/lib/Target/PowerPC/PPCScheduleP9.td
+++ b/llvm/lib/Target/PowerPC/PPCScheduleP9.td
@@ -41,8 +41,10 @@ def P9Model : SchedMachineModel {
   let CompleteModel = 1;
 
   // Do not support SPE (Signal Processing Engine), prefixed instructions on
-  // Power 9, PC relative mem ops, or instructions introduced in ISA 3.1.
-  let UnsupportedFeatures = [HasSPE, PrefixInstrs, PCRelativeMemops, IsISA3_1];
+  // Power 9, paired vector mem ops, PC relative mem ops, or instructions
+  // introduced in ISA 3.1.
+  let UnsupportedFeatures = [HasSPE, PrefixInstrs, PairedVectorMemops,
+                             PCRelativeMemops, IsISA3_1];
 }
 
 let SchedModel = P9Model in {
diff --git a/llvm/lib/Target/PowerPC/PPCSubtarget.cpp b/llvm/lib/Target/PowerPC/PPCSubtarget.cpp
index 85d2966654970..6577b37f908f2 100644
--- a/llvm/lib/Target/PowerPC/PPCSubtarget.cpp
+++ b/llvm/lib/Target/PowerPC/PPCSubtarget.cpp
@@ -116,6 +116,7 @@ void PPCSubtarget::initializeEnvironment() {
   VectorsUseTwoUnits = false;
   UsePPCPreRASchedStrategy = false;
   UsePPCPostRASchedStrategy = false;
+  PairedVectorMemops = false;
   PredictableSelectIsExpensive = false;
 
   HasPOPCNTD = POPCNTD_Unavailable;
diff --git a/llvm/lib/Target/PowerPC/PPCSubtarget.h b/llvm/lib/Target/PowerPC/PPCSubtarget.h
index 8a4041518e3c2..a457214e86302 100644
--- a/llvm/lib/Target/PowerPC/PPCSubtarget.h
+++ b/llvm/lib/Target/PowerPC/PPCSubtarget.h
@@ -145,6 +145,7 @@ class PPCSubtarget : public PPCGenSubtargetInfo {
   bool VectorsUseTwoUnits;
   bool UsePPCPreRASchedStrategy;
   bool UsePPCPostRASchedStrategy;
+  bool PairedVectorMemops;
   bool PredictableSelectIsExpensive;
 
   POPCNTDKind HasPOPCNTD;
@@ -259,6 +260,7 @@ class PPCSubtarget : public PPCGenSubtargetInfo {
   bool hasP10Vector() const { return HasP10Vector; }
   bool hasPrefixInstrs() const { return HasPrefixInstrs; }
   bool hasPCRelativeMemops() const { return HasPCRelativeMemops; }
+  bool pairedVectorMemops() const { return PairedVectorMemops; }
   bool hasMFOCRF() const { return HasMFOCRF; }
   bool hasISEL() const { return HasISEL; }
   bool hasBPERMD() const { return HasBPERMD; }
diff --git a/llvm/test/CodeGen/PowerPC/future-check-features.ll b/llvm/test/CodeGen/PowerPC/future-check-features.ll
index cfaf4eab93c1e..4d9b6e7555924 100644
--- a/llvm/test/CodeGen/PowerPC/future-check-features.ll
+++ b/llvm/test/CodeGen/PowerPC/future-check-features.ll
@@ -1,9 +1,9 @@
-; RUN: llc -mattr=pcrelative-memops,prefix-instrs -verify-machineinstrs \
-; RUN:   -mtriple=powerpc64le-unknown-unknown -ppc-asm-full-reg-names \
-; RUN:   %s -o - 2>&1 | FileCheck %s
-; RUN: llc -mattr=pcrelative-memops,prefix-instrs -verify-machineinstrs \
-; RUN:   -mtriple=powerpc64-unknown-unknown -ppc-asm-full-reg-names \
-; RUN:   %s -o - 2>&1 | FileCheck %s
+; RUN: llc -mattr=pcrelative-memops,prefix-instrs,paired-vector-memops \
+; RUN:   -verify-machineinstrs -mtriple=powerpc64le-unknown-unknown \
+; RUN:   -ppc-asm-full-reg-names %s -o - 2>&1 | FileCheck %s
+; RUN: llc -mattr=pcrelative-memops,prefix-instrs,paired-vector-memops \
+; RUN:   -verify-machineinstrs -mtriple=powerpc64-unknown-unknown \
+; RUN:   -ppc-asm-full-reg-names %s -o - 2>&1 | FileCheck %s
 
 define dso_local signext i32 @f() {
 entry:

From e69138dad5a535c4027a1931f05cabab0f484974 Mon Sep 17 00:00:00 2001
From: Richard Smith <richard@metafoo.co.uk>
Date: Wed, 29 Jul 2020 12:01:47 -0700
Subject: [PATCH 0524/1035] PR46859: Fix crash if declaring a template in a
 DeclScope with no DeclContext.

This can happen during error recovery; it could also happen at block
scope if we ever parsed a template declaration at block scope.
---
 clang/lib/Sema/SemaTemplate.cpp | 5 ++---
 clang/test/SemaCXX/extern-c.cpp | 5 +++++
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp
index c05ed0b14e3eb..76ba12303d84b 100644
--- a/clang/lib/Sema/SemaTemplate.cpp
+++ b/clang/lib/Sema/SemaTemplate.cpp
@@ -7774,15 +7774,14 @@ Sema::CheckTemplateDeclScope(Scope *S, TemplateParameterList *TemplateParams) {
   // C++ [temp]p4:
   //   A template [...] shall not have C linkage.
   DeclContext *Ctx = S->getEntity();
-  assert(Ctx && "Unknown context");
-  if (Ctx->isExternCContext()) {
+  if (Ctx && Ctx->isExternCContext()) {
     Diag(TemplateParams->getTemplateLoc(), diag::err_template_linkage)
         << TemplateParams->getSourceRange();
     if (const LinkageSpecDecl *LSD = Ctx->getExternCContext())
       Diag(LSD->getExternLoc(), diag::note_extern_c_begins_here);
     return true;
   }
-  Ctx = Ctx->getRedeclContext();
+  Ctx = Ctx ? Ctx->getRedeclContext() : nullptr;
 
   // C++ [temp]p2:
   //   A template-declaration can appear only as a namespace scope or
diff --git a/clang/test/SemaCXX/extern-c.cpp b/clang/test/SemaCXX/extern-c.cpp
index 1a32beec7baab..68d1494b94918 100644
--- a/clang/test/SemaCXX/extern-c.cpp
+++ b/clang/test/SemaCXX/extern-c.cpp
@@ -259,3 +259,8 @@ namespace PR35697 {
     }
   }
 }
+
+namespace PR46859 {
+  extern "bogus" // expected-error {{unknown linkage language}}
+    template<int> struct X {}; // expected-error {{templates can only be declared in namespace or class scope}}
+}

From da0bba5c9abb161ff824cf450537cf2ccf50d457 Mon Sep 17 00:00:00 2001
From: Tatyana Krasnukha <tatyana@synopsys.com>
Date: Thu, 23 Jul 2020 18:41:14 +0300
Subject: [PATCH 0525/1035] [lldb/Breakpoint] Rename StoppointLocation to
 StoppointSite and drop its relationship with BreakpointLocation

Both of BreakpointLocation and BreakpointSite were inherited from StoppointLocation. However, the only thing
they shared was hit counting logic. The patch encapsulates those logic into StoppointHitCounter, renames
StoppointLocation to StoppointSite, and stops BreakpointLocation's inheriting from it.

Differential Revision: https://reviews.llvm.org/D84527
---
 lldb/include/lldb/Breakpoint/Breakpoint.h     | 18 ++--
 .../lldb/Breakpoint/BreakpointLocation.h      | 30 ++++---
 lldb/include/lldb/Breakpoint/BreakpointSite.h |  6 +-
 .../lldb/Breakpoint/StoppointHitCounter.h     | 43 ++++++++++
 .../lldb/Breakpoint/StoppointLocation.h       | 85 -------------------
 lldb/include/lldb/Breakpoint/StoppointSite.h  | 81 ++++++++++++++++++
 lldb/include/lldb/Breakpoint/Watchpoint.h     |  8 +-
 lldb/include/lldb/lldb-forward.h              |  2 -
 lldb/source/Breakpoint/Breakpoint.cpp         |  6 +-
 lldb/source/Breakpoint/BreakpointLocation.cpp | 44 +++++-----
 lldb/source/Breakpoint/BreakpointSite.cpp     | 11 +--
 lldb/source/Breakpoint/CMakeLists.txt         |  2 +-
 lldb/source/Breakpoint/StoppointLocation.cpp  | 32 -------
 lldb/source/Breakpoint/StoppointSite.cpp      | 23 +++++
 lldb/source/Breakpoint/Watchpoint.cpp         | 14 ++-
 15 files changed, 208 insertions(+), 197 deletions(-)
 create mode 100644 lldb/include/lldb/Breakpoint/StoppointHitCounter.h
 delete mode 100644 lldb/include/lldb/Breakpoint/StoppointLocation.h
 create mode 100644 lldb/include/lldb/Breakpoint/StoppointSite.h
 delete mode 100644 lldb/source/Breakpoint/StoppointLocation.cpp
 create mode 100644 lldb/source/Breakpoint/StoppointSite.cpp

diff --git a/lldb/include/lldb/Breakpoint/Breakpoint.h b/lldb/include/lldb/Breakpoint/Breakpoint.h
index d29d21070fd7c..5d3e596c474be 100644
--- a/lldb/include/lldb/Breakpoint/Breakpoint.h
+++ b/lldb/include/lldb/Breakpoint/Breakpoint.h
@@ -20,6 +20,7 @@
 #include "lldb/Breakpoint/BreakpointName.h"
 #include "lldb/Breakpoint/BreakpointOptions.h"
 #include "lldb/Breakpoint/Stoppoint.h"
+#include "lldb/Breakpoint/StoppointHitCounter.h"
 #include "lldb/Core/SearchFilter.h"
 #include "lldb/Utility/Event.h"
 #include "lldb/Utility/StringList.h"
@@ -624,13 +625,6 @@ class Breakpoint : public std::enable_shared_from_this<Breakpoint>,
 
   bool IgnoreCountShouldStop();
 
-  void IncrementHitCount() { m_hit_count++; }
-
-  void DecrementHitCount() {
-    assert(m_hit_count > 0);
-    m_hit_count--;
-  }
-
 private:
   // To call from CopyFromBreakpoint.
   Breakpoint(Target &new_target, const Breakpoint &bp_to_copy_from);
@@ -660,10 +654,12 @@ class Breakpoint : public std::enable_shared_from_this<Breakpoint>,
       m_locations; // The list of locations currently found for this breakpoint.
   std::string m_kind_description;
   bool m_resolve_indirect_symbols;
-  uint32_t m_hit_count; // Number of times this breakpoint/watchpoint has been
-                        // hit.  This is kept
-  // separately from the locations hit counts, since locations can go away when
-  // their backing library gets unloaded, and we would lose hit counts.
+
+  /// Number of times this breakpoint has been hit. This is kept separately
+  /// from the locations hit counts, since locations can go away when their
+  /// backing library gets unloaded, and we would lose hit counts.
+  StoppointHitCounter m_hit_counter;
+
   BreakpointName::Permissions m_permissions;
 
   void SendBreakpointChangedEvent(lldb::BreakpointEventType eventKind);
diff --git a/lldb/include/lldb/Breakpoint/BreakpointLocation.h b/lldb/include/lldb/Breakpoint/BreakpointLocation.h
index 04b2b89068998..4e1c57a404352 100644
--- a/lldb/include/lldb/Breakpoint/BreakpointLocation.h
+++ b/lldb/include/lldb/Breakpoint/BreakpointLocation.h
@@ -13,7 +13,7 @@
 #include <mutex>
 
 #include "lldb/Breakpoint/BreakpointOptions.h"
-#include "lldb/Breakpoint/StoppointLocation.h"
+#include "lldb/Breakpoint/StoppointHitCounter.h"
 #include "lldb/Core/Address.h"
 #include "lldb/Utility/UserID.h"
 #include "lldb/lldb-private.h"
@@ -35,15 +35,14 @@ namespace lldb_private {
 /// be useful if you've set options on the locations.
 
 class BreakpointLocation
-    : public std::enable_shared_from_this<BreakpointLocation>,
-      public StoppointLocation {
+    : public std::enable_shared_from_this<BreakpointLocation> {
 public:
-  ~BreakpointLocation() override;
+  ~BreakpointLocation();
 
   /// Gets the load address for this breakpoint location \return
   ///     Returns breakpoint location load address, \b
   ///     LLDB_INVALID_ADDRESS if not yet set.
-  lldb::addr_t GetLoadAddress() const override;
+  lldb::addr_t GetLoadAddress() const;
 
   /// Gets the Address for this breakpoint location \return
   ///     Returns breakpoint location Address.
@@ -63,7 +62,7 @@ class BreakpointLocation
   /// \return
   ///     \b true if this breakpoint location thinks we should stop,
   ///     \b false otherwise.
-  bool ShouldStop(StoppointCallbackContext *context) override;
+  bool ShouldStop(StoppointCallbackContext *context);
 
   // The next section deals with various breakpoint options.
 
@@ -76,12 +75,6 @@ class BreakpointLocation
   ///     \b true if the breakpoint is enabled, \b false if disabled.
   bool IsEnabled() const;
 
-  /// Check if it is a hardware breakpoint.
-  ///
-  /// \return
-  ///     \b true if the breakpoint is a harware breakpoint.
-  bool IsHardware() const override;
-
   /// If \a auto_continue is \b true, set the breakpoint to continue when hit.
   void SetAutoContinue(bool auto_continue);
 
@@ -91,11 +84,14 @@ class BreakpointLocation
   ///     \b true if the breakpoint is set to auto-continue, \b false if not.
   bool IsAutoContinue() const;
 
+  /// Return the current Hit Count.
+  uint32_t GetHitCount() const { return m_hit_counter.GetValue(); }
+
   /// Return the current Ignore Count.
   ///
   /// \return
   ///     The number of breakpoint hits to be ignored.
-  uint32_t GetIgnoreCount();
+  uint32_t GetIgnoreCount() const;
 
   /// Set the breakpoint to ignore the next \a count breakpoint hits.
   ///
@@ -198,7 +194,7 @@ class BreakpointLocation
   void GetDescription(Stream *s, lldb::DescriptionLevel level);
 
   /// Standard "Dump" method.  At present it does nothing.
-  void Dump(Stream *s) const override;
+  void Dump(Stream *s) const;
 
   /// Use this to set location specific breakpoint options.
   ///
@@ -274,6 +270,9 @@ class BreakpointLocation
   ///     \b true or \b false as given in the description above.
   bool EquivalentToLocation(BreakpointLocation &location);
 
+  /// Returns the breakpoint location ID.
+  lldb::break_id_t GetID() const { return m_loc_id; }
+
 protected:
   friend class BreakpointSite;
   friend class BreakpointLocationList;
@@ -344,6 +343,9 @@ class BreakpointLocation
                                 /// multiple processes.
   size_t m_condition_hash; ///< For testing whether the condition source code
                            ///changed.
+  lldb::break_id_t m_loc_id; ///< Breakpoint location ID.
+  StoppointHitCounter m_hit_counter; ///< Number of times this breakpoint
+                                     /// location has been hit.
 
   void SetShouldResolveIndirectFunctions(bool do_resolve) {
     m_should_resolve_indirect_functions = do_resolve;
diff --git a/lldb/include/lldb/Breakpoint/BreakpointSite.h b/lldb/include/lldb/Breakpoint/BreakpointSite.h
index 6ddab5dc14606..fc32c04ffe359 100644
--- a/lldb/include/lldb/Breakpoint/BreakpointSite.h
+++ b/lldb/include/lldb/Breakpoint/BreakpointSite.h
@@ -14,7 +14,7 @@
 
 
 #include "lldb/Breakpoint/BreakpointLocationCollection.h"
-#include "lldb/Breakpoint/StoppointLocation.h"
+#include "lldb/Breakpoint/StoppointSite.h"
 #include "lldb/Utility/LLDBAssert.h"
 #include "lldb/Utility/UserID.h"
 #include "lldb/lldb-forward.h"
@@ -33,7 +33,7 @@ namespace lldb_private {
 /// by the process.
 
 class BreakpointSite : public std::enable_shared_from_this<BreakpointSite>,
-                       public StoppointLocation {
+                       public StoppointSite {
 public:
   enum Type {
     eSoftware, // Breakpoint opcode has been written to memory and
@@ -61,8 +61,6 @@ class BreakpointSite : public std::enable_shared_from_this<BreakpointSite>,
   /// Sets the trap opcode
   bool SetTrapOpcode(const uint8_t *trap_opcode, uint32_t trap_opcode_size);
 
-  void SetHardwareIndex(uint32_t index) override;
-
   /// Gets the original instruction bytes that were overwritten by the trap
   uint8_t *GetSavedOpcodeBytes();
 
diff --git a/lldb/include/lldb/Breakpoint/StoppointHitCounter.h b/lldb/include/lldb/Breakpoint/StoppointHitCounter.h
new file mode 100644
index 0000000000000..26f816da6430d
--- /dev/null
+++ b/lldb/include/lldb/Breakpoint/StoppointHitCounter.h
@@ -0,0 +1,43 @@
+//===-- StoppointHitCounter.h -----------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLDB_BREAKPOINT_STOPPOINT_HIT_COUNTER_H
+#define LLDB_BREAKPOINT_STOPPOINT_HIT_COUNTER_H
+
+#include <assert.h>
+#include <cstdint>
+#include <limits>
+
+#include "lldb/Utility/LLDBAssert.h"
+
+namespace lldb_private {
+
+class StoppointHitCounter {
+public:
+  uint32_t GetValue() const { return m_hit_count; }
+
+  void Increment(uint32_t difference = 1) {
+    lldbassert(std::numeric_limits<uint32_t>::max() - m_hit_count >= difference);
+    m_hit_count += difference;
+  }
+
+  void Decrement(uint32_t difference = 1) {
+    lldbassert(m_hit_count >= difference);
+    m_hit_count -= difference;
+  }
+
+  void Reset() { m_hit_count = 0; }
+
+private:
+  /// Number of times this breakpoint/watchpoint has been hit.
+  uint32_t m_hit_count = 0;
+};
+
+} // namespace lldb_private
+
+#endif // LLDB_BREAKPOINT_STOPPOINT_HIT_COUNTER_H
diff --git a/lldb/include/lldb/Breakpoint/StoppointLocation.h b/lldb/include/lldb/Breakpoint/StoppointLocation.h
deleted file mode 100644
index 4392c7b84f9e8..0000000000000
--- a/lldb/include/lldb/Breakpoint/StoppointLocation.h
+++ /dev/null
@@ -1,85 +0,0 @@
-//===-- StoppointLocation.h -------------------------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLDB_BREAKPOINT_STOPPOINTLOCATION_H
-#define LLDB_BREAKPOINT_STOPPOINTLOCATION_H
-
-#include "lldb/Utility/UserID.h"
-#include "lldb/lldb-private.h"
-// #include "lldb/Breakpoint/BreakpointOptions.h"
-
-namespace lldb_private {
-
-class StoppointLocation {
-public:
-  // Constructors and Destructors
-  StoppointLocation(lldb::break_id_t bid, lldb::addr_t m_addr, bool hardware);
-
-  StoppointLocation(lldb::break_id_t bid, lldb::addr_t m_addr,
-                    uint32_t byte_size, bool hardware);
-
-  virtual ~StoppointLocation();
-
-  // Operators
-
-  // Methods
-  virtual lldb::addr_t GetLoadAddress() const { return m_addr; }
-
-  virtual void SetLoadAddress(lldb::addr_t addr) { m_addr = addr; }
-
-  uint32_t GetByteSize() const { return m_byte_size; }
-
-  uint32_t GetHitCount() const { return m_hit_count; }
-
-  uint32_t GetHardwareIndex() const { return m_hardware_index; }
-
-  bool HardwareRequired() const { return m_hardware; }
-
-  virtual bool IsHardware() const = 0;
-
-  virtual bool ShouldStop(StoppointCallbackContext *context) { return true; }
-
-  virtual void Dump(Stream *stream) const {}
-
-  virtual void SetHardwareIndex(uint32_t index) { m_hardware_index = index; }
-
-  lldb::break_id_t GetID() const { return m_loc_id; }
-
-protected:
-  // Classes that inherit from StoppointLocation can see and modify these
-  lldb::break_id_t m_loc_id; // Stoppoint location ID
-  lldb::addr_t
-      m_addr; // The load address of this stop point. The base Stoppoint doesn't
-  // store a full Address since that's not needed for the breakpoint sites.
-  bool m_hardware; // True if this point has been is required to use hardware
-                   // (which may fail due to lack of resources)
-  uint32_t m_hardware_index; // The hardware resource index for this
-                             // breakpoint/watchpoint
-  uint32_t m_byte_size; // The size in bytes of stop location.  e.g. the length
-                        // of the trap opcode for
-  // software breakpoints, or the optional length in bytes for hardware
-  // breakpoints, or the length of the watchpoint.
-  uint32_t
-      m_hit_count; // Number of times this breakpoint/watchpoint has been hit
-
-  // If you override this, be sure to call the base class to increment the
-  // internal counter.
-  void IncrementHitCount() { ++m_hit_count; }
-
-  void DecrementHitCount();
-
-private:
-  // For StoppointLocation only
-  StoppointLocation(const StoppointLocation &) = delete;
-  const StoppointLocation &operator=(const StoppointLocation &) = delete;
-  StoppointLocation() = delete;
-};
-
-} // namespace lldb_private
-
-#endif // LLDB_BREAKPOINT_STOPPOINTLOCATION_H
diff --git a/lldb/include/lldb/Breakpoint/StoppointSite.h b/lldb/include/lldb/Breakpoint/StoppointSite.h
new file mode 100644
index 0000000000000..7e5e33486345d
--- /dev/null
+++ b/lldb/include/lldb/Breakpoint/StoppointSite.h
@@ -0,0 +1,81 @@
+//===-- StoppointSite.h -----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLDB_BREAKPOINT_STOPPOINTSITE_H
+#define LLDB_BREAKPOINT_STOPPOINTSITE_H
+
+#include "lldb/Breakpoint/StoppointHitCounter.h"
+#include "lldb/Utility/UserID.h"
+#include "lldb/lldb-private.h"
+
+namespace lldb_private {
+
+class StoppointSite {
+public:
+  StoppointSite(lldb::break_id_t bid, lldb::addr_t m_addr, bool hardware);
+
+  StoppointSite(lldb::break_id_t bid, lldb::addr_t m_addr,
+                uint32_t byte_size, bool hardware);
+
+  virtual ~StoppointSite() = default;
+
+  virtual lldb::addr_t GetLoadAddress() const { return m_addr; }
+
+  virtual void SetLoadAddress(lldb::addr_t addr) { m_addr = addr; }
+
+  uint32_t GetByteSize() const { return m_byte_size; }
+
+  uint32_t GetHitCount() const { return m_hit_counter.GetValue(); }
+
+  void ResetHitCount() { m_hit_counter.Reset(); }
+
+  bool HardwareRequired() const { return m_is_hardware_required; }
+
+  virtual bool IsHardware() const = 0;
+
+  uint32_t GetHardwareIndex() const { return m_hardware_index; }
+
+  void SetHardwareIndex(uint32_t index) { m_hardware_index = index; }
+
+  virtual bool ShouldStop(StoppointCallbackContext* context) = 0;
+
+  virtual void Dump(Stream* stream) const = 0;
+
+  lldb::break_id_t GetID() const { return m_id; }
+
+protected:
+  /// Stoppoint site ID.
+  lldb::break_id_t m_id;
+
+  /// The load address of this stop point.
+  lldb::addr_t m_addr;
+
+  /// True if this point is required to use hardware (which may fail due to
+  /// the lack of resources).
+  bool m_is_hardware_required;
+
+  /// The hardware resource index for this breakpoint/watchpoint.
+  uint32_t m_hardware_index;
+
+  /// The size in bytes of stoppoint, e.g. the length of the trap opcode for
+  /// software breakpoints, or the optional length in bytes for hardware
+  /// breakpoints, or the length of the watchpoint.
+  uint32_t m_byte_size;
+
+  /// Number of times this breakpoint/watchpoint has been hit.
+  StoppointHitCounter m_hit_counter;
+
+private:
+  StoppointSite(const StoppointSite &) = delete;
+  const StoppointSite &operator=(const StoppointSite &) = delete;
+  StoppointSite() = delete;
+};
+
+} // namespace lldb_private
+
+#endif // LLDB_BREAKPOINT_STOPPOINTSITE_H
diff --git a/lldb/include/lldb/Breakpoint/Watchpoint.h b/lldb/include/lldb/Breakpoint/Watchpoint.h
index bce15f0a85dde..41b723a66b6a3 100644
--- a/lldb/include/lldb/Breakpoint/Watchpoint.h
+++ b/lldb/include/lldb/Breakpoint/Watchpoint.h
@@ -12,7 +12,7 @@
 #include <memory>
 #include <string>
 
-#include "lldb/Breakpoint/StoppointLocation.h"
+#include "lldb/Breakpoint/StoppointSite.h"
 #include "lldb/Breakpoint/WatchpointOptions.h"
 #include "lldb/Symbol/CompilerType.h"
 #include "lldb/Target/Target.h"
@@ -22,7 +22,7 @@
 namespace lldb_private {
 
 class Watchpoint : public std::enable_shared_from_this<Watchpoint>,
-                   public StoppointLocation {
+                   public StoppointSite {
 public:
   class WatchpointEventData : public EventData {
   public:
@@ -158,8 +158,6 @@ class Watchpoint : public std::enable_shared_from_this<Watchpoint>,
   friend class Target;
   friend class WatchpointList;
 
-  void ResetHitCount() { m_hit_count = 0; }
-
   void ResetHistoricValues() {
     m_old_value_sp.reset();
     m_new_value_sp.reset();
@@ -199,7 +197,7 @@ class Watchpoint : public std::enable_shared_from_this<Watchpoint>,
 
   std::unique_ptr<UserExpression> m_condition_up; // The condition to test.
 
-  void SetID(lldb::watch_id_t id) { m_loc_id = id; }
+  void SetID(lldb::watch_id_t id) { m_id = id; }
 
   void SendWatchpointChangedEvent(lldb::WatchpointEventType eventKind);
 
diff --git a/lldb/include/lldb/lldb-forward.h b/lldb/include/lldb/lldb-forward.h
index 0bf9e2c105377..6682413820d6b 100644
--- a/lldb/include/lldb/lldb-forward.h
+++ b/lldb/include/lldb/lldb-forward.h
@@ -192,7 +192,6 @@ class Status;
 class StopInfo;
 class Stoppoint;
 class StoppointCallbackContext;
-class StoppointLocation;
 class Stream;
 class StreamFile;
 class StreamString;
@@ -405,7 +404,6 @@ typedef std::shared_ptr<lldb_private::StackFrameRecognizer>
 typedef std::unique_ptr<lldb_private::StackFrameRecognizerManager>
     StackFrameRecognizerManagerUP;
 typedef std::shared_ptr<lldb_private::StopInfo> StopInfoSP;
-typedef std::shared_ptr<lldb_private::StoppointLocation> StoppointLocationSP;
 typedef std::shared_ptr<lldb_private::Stream> StreamSP;
 typedef std::weak_ptr<lldb_private::Stream> StreamWP;
 typedef std::shared_ptr<lldb_private::StreamFile> StreamFileSP;
diff --git a/lldb/source/Breakpoint/Breakpoint.cpp b/lldb/source/Breakpoint/Breakpoint.cpp
index 88c7f3352f426..9ed9549a789ea 100644
--- a/lldb/source/Breakpoint/Breakpoint.cpp
+++ b/lldb/source/Breakpoint/Breakpoint.cpp
@@ -51,7 +51,7 @@ Breakpoint::Breakpoint(Target &target, SearchFilterSP &filter_sp,
     : m_being_created(true), m_hardware(hardware), m_target(target),
       m_filter_sp(filter_sp), m_resolver_sp(resolver_sp),
       m_options_up(new BreakpointOptions(true)), m_locations(*this),
-      m_resolve_indirect_symbols(resolve_indirect_symbols), m_hit_count(0) {
+      m_resolve_indirect_symbols(resolve_indirect_symbols), m_hit_counter() {
   m_being_created = false;
 }
 
@@ -61,7 +61,7 @@ Breakpoint::Breakpoint(Target &new_target, const Breakpoint &source_bp)
       m_options_up(new BreakpointOptions(*source_bp.m_options_up)),
       m_locations(*this),
       m_resolve_indirect_symbols(source_bp.m_resolve_indirect_symbols),
-      m_hit_count(0) {}
+      m_hit_counter() {}
 
 // Destructor
 Breakpoint::~Breakpoint() = default;
@@ -341,7 +341,7 @@ bool Breakpoint::IgnoreCountShouldStop() {
     return true;
 }
 
-uint32_t Breakpoint::GetHitCount() const { return m_hit_count; }
+uint32_t Breakpoint::GetHitCount() const { return m_hit_counter.GetValue(); }
 
 bool Breakpoint::IsOneShot() const { return m_options_up->IsOneShot(); }
 
diff --git a/lldb/source/Breakpoint/BreakpointLocation.cpp b/lldb/source/Breakpoint/BreakpointLocation.cpp
index eae1c1e033ad9..d3d6ea08bdb38 100644
--- a/lldb/source/Breakpoint/BreakpointLocation.cpp
+++ b/lldb/source/Breakpoint/BreakpointLocation.cpp
@@ -31,11 +31,10 @@ using namespace lldb_private;
 BreakpointLocation::BreakpointLocation(break_id_t loc_id, Breakpoint &owner,
                                        const Address &addr, lldb::tid_t tid,
                                        bool hardware, bool check_for_resolver)
-    : StoppointLocation(loc_id, addr.GetOpcodeLoadAddress(&owner.GetTarget()),
-                        hardware),
-      m_being_created(true), m_should_resolve_indirect_functions(false),
+    : m_being_created(true), m_should_resolve_indirect_functions(false),
       m_is_reexported(false), m_is_indirect(false), m_address(addr),
-      m_owner(owner), m_options_up(), m_bp_site_sp(), m_condition_mutex() {
+      m_owner(owner), m_options_up(), m_bp_site_sp(), m_condition_mutex(),
+      m_condition_hash(0), m_loc_id(loc_id), m_hit_counter() {
   if (check_for_resolver) {
     Symbol *symbol = m_address.CalculateSymbolContextSymbol();
     if (symbol && symbol->IsIndirect()) {
@@ -68,14 +67,6 @@ Breakpoint &BreakpointLocation::GetBreakpoint() { return m_owner; }
 
 Target &BreakpointLocation::GetTarget() { return m_owner.GetTarget(); }
 
-bool BreakpointLocation::IsHardware() const {
-  if (m_bp_site_sp)
-    return m_bp_site_sp->IsHardware();
-
-  // If breakpoint location is not resolved yet, it cannot be hardware.
-  return false;
-}
-
 bool BreakpointLocation::IsEnabled() const {
   if (!m_owner.IsEnabled())
     return false;
@@ -340,7 +331,7 @@ bool BreakpointLocation::ConditionSaysStop(ExecutionContext &exe_ctx,
   return ret;
 }
 
-uint32_t BreakpointLocation::GetIgnoreCount() {
+uint32_t BreakpointLocation::GetIgnoreCount() const {
   return GetOptionsSpecifyingKind(BreakpointOptions::eIgnoreCount)
       ->GetIgnoreCount();
 }
@@ -425,16 +416,16 @@ bool BreakpointLocation::ShouldStop(StoppointCallbackContext *context) {
 void BreakpointLocation::BumpHitCount() {
   if (IsEnabled()) {
     // Step our hit count, and also step the hit count of the owner.
-    IncrementHitCount();
-    m_owner.IncrementHitCount();
+    m_hit_counter.Increment();
+    m_owner.m_hit_counter.Increment();
   }
 }
 
 void BreakpointLocation::UndoBumpHitCount() {
   if (IsEnabled()) {
     // Step our hit count, and also step the hit count of the owner.
-    DecrementHitCount();
-    m_owner.DecrementHitCount();
+    m_hit_counter.Decrement();
+    m_owner.m_hit_counter.Decrement();
   }
 }
 
@@ -601,12 +592,15 @@ void BreakpointLocation::GetDescription(Stream *s,
     }
   }
 
+  bool is_resolved = IsResolved();
+  bool is_hardware = is_resolved && m_bp_site_sp->IsHardware();
+
   if (level == lldb::eDescriptionLevelVerbose) {
     s->EOL();
     s->Indent();
-    s->Printf("resolved = %s\n", IsResolved() ? "true" : "false");
+    s->Printf("resolved = %s\n", is_resolved ? "true" : "false");
     s->Indent();
-    s->Printf("hardware = %s\n", IsHardware() ? "true" : "false");
+    s->Printf("hardware = %s\n", is_hardware ? "true" : "false");
     s->Indent();
     s->Printf("hit count = %-4u\n", GetHitCount());
 
@@ -617,8 +611,8 @@ void BreakpointLocation::GetDescription(Stream *s,
     }
     s->IndentLess();
   } else if (level != eDescriptionLevelInitial) {
-    s->Printf(", %sresolved, %shit count = %u ", (IsResolved() ? "" : "un"),
-              (IsHardware() ? "hardware, " : ""), GetHitCount());
+    s->Printf(", %sresolved, %shit count = %u ", (is_resolved ? "" : "un"),
+              (is_hardware ? "hardware, " : ""), GetHitCount());
     if (m_options_up) {
       m_options_up->GetDescription(s, level);
     }
@@ -629,6 +623,11 @@ void BreakpointLocation::Dump(Stream *s) const {
   if (s == nullptr)
     return;
 
+  bool is_resolved = IsResolved();
+  bool is_hardware = is_resolved && m_bp_site_sp->IsHardware();
+  auto hardware_index = is_resolved ?
+      m_bp_site_sp->GetHardwareIndex() : LLDB_INVALID_INDEX32;
+
   lldb::tid_t tid = GetOptionsSpecifyingKind(BreakpointOptions::eThreadSpec)
       ->GetThreadSpecNoCreate()->GetTID();
   s->Printf("BreakpointLocation %u: tid = %4.4" PRIx64
@@ -639,8 +638,7 @@ void BreakpointLocation::Dump(Stream *s) const {
             (m_options_up ? m_options_up->IsEnabled() : m_owner.IsEnabled())
                 ? "enabled "
                 : "disabled",
-            IsHardware() ? "hardware" : "software", GetHardwareIndex(),
-            GetHitCount(),
+            is_hardware ? "hardware" : "software", hardware_index, GetHitCount(),
             GetOptionsSpecifyingKind(BreakpointOptions::eIgnoreCount)
                 ->GetIgnoreCount());
 }
diff --git a/lldb/source/Breakpoint/BreakpointSite.cpp b/lldb/source/Breakpoint/BreakpointSite.cpp
index faa7e4c261e73..fb3f0cd068971 100644
--- a/lldb/source/Breakpoint/BreakpointSite.cpp
+++ b/lldb/source/Breakpoint/BreakpointSite.cpp
@@ -21,7 +21,7 @@ using namespace lldb_private;
 BreakpointSite::BreakpointSite(BreakpointSiteList *list,
                                const BreakpointLocationSP &owner,
                                lldb::addr_t addr, bool use_hardware)
-    : StoppointLocation(GetNextID(), addr, 0, use_hardware),
+    : StoppointSite(GetNextID(), addr, 0, use_hardware),
       m_type(eSoftware), // Process subclasses need to set this correctly using
                          // SetType()
       m_saved_opcode(), m_trap_opcode(),
@@ -48,7 +48,7 @@ break_id_t BreakpointSite::GetNextID() {
 // should continue.
 
 bool BreakpointSite::ShouldStop(StoppointCallbackContext *context) {
-  IncrementHitCount();
+  m_hit_counter.Increment();
   // ShouldStop can do a lot of work, and might even come come back and hit
   // this breakpoint site again.  So don't hold the m_owners_mutex the whole
   // while.  Instead make a local copy of the collection and call ShouldStop on
@@ -156,13 +156,6 @@ void BreakpointSite::BumpHitCounts() {
   }
 }
 
-void BreakpointSite::SetHardwareIndex(uint32_t index) {
-  std::lock_guard<std::recursive_mutex> guard(m_owners_mutex);
-  for (BreakpointLocationSP loc_sp : m_owners.BreakpointLocations()) {
-    loc_sp->SetHardwareIndex(index);
-  }
-}
-
 bool BreakpointSite::IntersectsRange(lldb::addr_t addr, size_t size,
                                      lldb::addr_t *intersect_addr,
                                      size_t *intersect_size,
diff --git a/lldb/source/Breakpoint/CMakeLists.txt b/lldb/source/Breakpoint/CMakeLists.txt
index a7c0baf21af61..4862c2b364033 100644
--- a/lldb/source/Breakpoint/CMakeLists.txt
+++ b/lldb/source/Breakpoint/CMakeLists.txt
@@ -19,7 +19,7 @@ add_lldb_library(lldbBreakpoint
   BreakpointSiteList.cpp
   Stoppoint.cpp
   StoppointCallbackContext.cpp
-  StoppointLocation.cpp
+  StoppointSite.cpp
   Watchpoint.cpp
   WatchpointList.cpp
   WatchpointOptions.cpp
diff --git a/lldb/source/Breakpoint/StoppointLocation.cpp b/lldb/source/Breakpoint/StoppointLocation.cpp
deleted file mode 100644
index 5bb4c78548404..0000000000000
--- a/lldb/source/Breakpoint/StoppointLocation.cpp
+++ /dev/null
@@ -1,32 +0,0 @@
-//===-- StoppointLocation.cpp ---------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "lldb/Breakpoint/StoppointLocation.h"
-
-
-using namespace lldb;
-using namespace lldb_private;
-
-// StoppointLocation constructor
-StoppointLocation::StoppointLocation(break_id_t bid, addr_t addr, bool hardware)
-    : m_loc_id(bid), m_addr(addr), m_hardware(hardware),
-      m_hardware_index(LLDB_INVALID_INDEX32), m_byte_size(0), m_hit_count(0) {}
-
-StoppointLocation::StoppointLocation(break_id_t bid, addr_t addr,
-                                     uint32_t byte_size, bool hardware)
-    : m_loc_id(bid), m_addr(addr), m_hardware(hardware),
-      m_hardware_index(LLDB_INVALID_INDEX32), m_byte_size(byte_size),
-      m_hit_count(0) {}
-
-// Destructor
-StoppointLocation::~StoppointLocation() {}
-
-void StoppointLocation::DecrementHitCount() {
-  assert(m_hit_count > 0);
-  --m_hit_count;
-}
diff --git a/lldb/source/Breakpoint/StoppointSite.cpp b/lldb/source/Breakpoint/StoppointSite.cpp
new file mode 100644
index 0000000000000..ba8c48326bdb6
--- /dev/null
+++ b/lldb/source/Breakpoint/StoppointSite.cpp
@@ -0,0 +1,23 @@
+//===-- StoppointSite.cpp ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "lldb/Breakpoint/StoppointSite.h"
+
+
+using namespace lldb;
+using namespace lldb_private;
+
+StoppointSite::StoppointSite(break_id_t id, addr_t addr, bool hardware)
+    : m_id(id), m_addr(addr), m_is_hardware_required(hardware),
+      m_hardware_index(LLDB_INVALID_INDEX32), m_byte_size(0), m_hit_counter() {}
+
+StoppointSite::StoppointSite(break_id_t id, addr_t addr,
+                             uint32_t byte_size, bool hardware)
+    : m_id(id), m_addr(addr), m_is_hardware_required(hardware),
+      m_hardware_index(LLDB_INVALID_INDEX32), m_byte_size(byte_size),
+      m_hit_counter() {}
diff --git a/lldb/source/Breakpoint/Watchpoint.cpp b/lldb/source/Breakpoint/Watchpoint.cpp
index 481c7b7eba8c0..29ae1ef3df26e 100644
--- a/lldb/source/Breakpoint/Watchpoint.cpp
+++ b/lldb/source/Breakpoint/Watchpoint.cpp
@@ -25,7 +25,7 @@ using namespace lldb_private;
 
 Watchpoint::Watchpoint(Target &target, lldb::addr_t addr, uint32_t size,
                        const CompilerType *type, bool hardware)
-    : StoppointLocation(0, addr, size, hardware), m_target(target),
+    : StoppointSite(0, addr, size, hardware), m_target(target),
       m_enabled(false), m_is_hardware(hardware), m_is_watch_variable(false),
       m_is_ephemeral(false), m_disabled_count(0), m_watch_read(0),
       m_watch_write(0), m_watch_was_read(0), m_watch_was_written(0),
@@ -93,8 +93,6 @@ void Watchpoint::SetWatchSpec(const std::string &str) {
   m_watch_spec_str = str;
 }
 
-// Override default impl of StoppointLocation::IsHardware() since m_is_hardware
-// member field is more accurate.
 bool Watchpoint::IsHardware() const {
   lldbassert(m_is_hardware || !HardwareRequired());
   return m_is_hardware;
@@ -126,12 +124,12 @@ bool Watchpoint::CaptureWatchedValue(const ExecutionContext &exe_ctx) {
 void Watchpoint::IncrementFalseAlarmsAndReviseHitCount() {
   ++m_false_alarms;
   if (m_false_alarms) {
-    if (m_hit_count >= m_false_alarms) {
-      m_hit_count -= m_false_alarms;
+    if (m_hit_counter.GetValue() >= m_false_alarms) {
+      m_hit_counter.Decrement(m_false_alarms);
       m_false_alarms = 0;
     } else {
-      m_false_alarms -= m_hit_count;
-      m_hit_count = 0;
+      m_false_alarms -= m_hit_counter.GetValue();
+      m_hit_counter.Reset();
     }
   }
 }
@@ -140,7 +138,7 @@ void Watchpoint::IncrementFalseAlarmsAndReviseHitCount() {
 // should continue.
 
 bool Watchpoint::ShouldStop(StoppointCallbackContext *context) {
-  IncrementHitCount();
+  m_hit_counter.Increment();
 
   return IsEnabled();
 }

From bdad24c2cc9f759a974cb63aaa3fff62b978726d Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Wed, 29 Jul 2020 14:40:51 -0400
Subject: [PATCH 0526/1035] GlobalISel: Fix insert point in CSEMIRBuilder unit
 test

This was using invalid MIR for the test instructions. The test add was
the first instruction in the block, before the trunc inputs or copies
from physical registers which I assume was not intended.
---
 llvm/unittests/CodeGen/GlobalISel/CSETest.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/llvm/unittests/CodeGen/GlobalISel/CSETest.cpp b/llvm/unittests/CodeGen/GlobalISel/CSETest.cpp
index 69c21d6afb009..0ef7b3d207fef 100644
--- a/llvm/unittests/CodeGen/GlobalISel/CSETest.cpp
+++ b/llvm/unittests/CodeGen/GlobalISel/CSETest.cpp
@@ -27,8 +27,8 @@ TEST_F(AArch64GISelMITest, TestCSE) {
   B.setCSEInfo(&CSEInfo);
   CSEMIRBuilder CSEB(B.getState());
 
-  CSEB.setInsertPt(*EntryMBB, EntryMBB->begin());
-  unsigned AddReg = MRI->createGenericVirtualRegister(s16);
+  CSEB.setInsertPt(B.getMBB(), B.getInsertPt());
+  Register AddReg = MRI->createGenericVirtualRegister(s16);
   auto MIBAddCopy =
       CSEB.buildInstr(TargetOpcode::G_ADD, {AddReg}, {MIBInput, MIBInput});
   EXPECT_EQ(MIBAddCopy->getOpcode(), TargetOpcode::COPY);

From a5faf3c849298d1106e74068f3ecffa13e1684a6 Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne@apple.com>
Date: Wed, 29 Jul 2020 14:24:02 -0400
Subject: [PATCH 0527/1035] [libc++] Re-enable tests for C11 math macros in
 <float.h> and <cfloat>

Fixes http://llvm.org/PR38572.
---
 libcxx/test/std/depr/depr.c.headers/float_h.pass.cpp        | 6 +++---
 .../support.limits/c.limits/cfloat.pass.cpp                 | 6 +++---
 libcxx/test/support/test_macros.h                           | 1 -
 3 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/libcxx/test/std/depr/depr.c.headers/float_h.pass.cpp b/libcxx/test/std/depr/depr.c.headers/float_h.pass.cpp
index 1069a679263fb..2b7edcadc2176 100644
--- a/libcxx/test/std/depr/depr.c.headers/float_h.pass.cpp
+++ b/libcxx/test/std/depr/depr.c.headers/float_h.pass.cpp
@@ -24,7 +24,7 @@
 #error FLT_RADIX not defined
 #endif
 
-#if TEST_STD_VER > 14 && 0
+#if TEST_STD_VER > 14
 #ifndef FLT_HAS_SUBNORM
 #error FLT_HAS_SUBNORM not defined
 #endif
@@ -54,7 +54,7 @@
 #error DECIMAL_DIG not defined
 #endif
 
-#if TEST_STD_VER > 14 && 0
+#if TEST_STD_VER > 14
 #ifndef FLT_DECIMAL_DIG
 #error FLT_DECIMAL_DIG not defined
 #endif
@@ -164,7 +164,7 @@
 #error LDBL_MIN not defined
 #endif
 
-#if TEST_STD_VER > 14 && 0
+#if TEST_STD_VER > 14
 #ifndef FLT_TRUE_MIN
 #error FLT_TRUE_MIN not defined
 #endif
diff --git a/libcxx/test/std/language.support/support.limits/c.limits/cfloat.pass.cpp b/libcxx/test/std/language.support/support.limits/c.limits/cfloat.pass.cpp
index 6ddd8c60834b0..99fe6a37243c5 100644
--- a/libcxx/test/std/language.support/support.limits/c.limits/cfloat.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/c.limits/cfloat.pass.cpp
@@ -24,7 +24,7 @@
 #error FLT_RADIX not defined
 #endif
 
-#if TEST_STD_VER > 14 && 0
+#if TEST_STD_VER > 14
 #ifndef FLT_HAS_SUBNORM
 #error FLT_HAS_SUBNORM not defined
 #endif
@@ -54,7 +54,7 @@
 #error DECIMAL_DIG not defined
 #endif
 
-#if TEST_STD_VER > 14 && 0
+#if TEST_STD_VER > 14
 #ifndef FLT_DECIMAL_DIG
 #error FLT_DECIMAL_DIG not defined
 #endif
@@ -164,7 +164,7 @@
 #error LDBL_MIN not defined
 #endif
 
-#if TEST_STD_VER > 14 && 0
+#if TEST_STD_VER > 14
 #ifndef FLT_TRUE_MIN
 #error FLT_TRUE_MIN not defined
 #endif
diff --git a/libcxx/test/support/test_macros.h b/libcxx/test/support/test_macros.h
index 6fe1c56a5588e..00873607fb11c 100644
--- a/libcxx/test/support/test_macros.h
+++ b/libcxx/test/support/test_macros.h
@@ -162,7 +162,6 @@
 #endif
 
 // Sniff out to see if the underlying C library has C11 features
-// Note that at this time (July 2018), MacOS X and iOS do NOT.
 // This is cribbed from __config; but lives here as well because we can't assume libc++
 #if __ISO_C_VISIBLE >= 2011 || TEST_STD_VER >= 11
 #  if defined(__FreeBSD__)

From 13b63be472233762024ef196dd88612369a51807 Mon Sep 17 00:00:00 2001
From: Stanislav Mekhanoshin <Stanislav.Mekhanoshin@amd.com>
Date: Wed, 29 Jul 2020 11:47:18 -0700
Subject: [PATCH 0528/1035] [AMDGPU] prefer non-mfma in post-RA schedule

MFMA instructions shall not be scheduled back to back
to avoid MAI SIMD stall. Tell post-RA schedule we would
prefer some other instruction instead.

Differential Revision: https://reviews.llvm.org/D84883
---
 .../lib/Target/AMDGPU/GCNHazardRecognizer.cpp | 24 +++++++++++++++++++
 llvm/lib/Target/AMDGPU/GCNHazardRecognizer.h  |  1 +
 .../CodeGen/AMDGPU/sched-prefer-non-mfma.mir  | 18 ++++++++++++++
 3 files changed, 43 insertions(+)
 create mode 100644 llvm/test/CodeGen/AMDGPU/sched-prefer-non-mfma.mir

diff --git a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
index 8b446a4342986..6c95c3310be64 100644
--- a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
@@ -1383,3 +1383,27 @@ int GCNHazardRecognizer::checkMAILdStHazards(MachineInstr *MI) {
 
   return WaitStatesNeeded;
 }
+
+bool GCNHazardRecognizer::ShouldPreferAnother(SUnit *SU) {
+  if (!SU->isInstr())
+    return false;
+
+  MachineInstr *MAI = nullptr;
+  auto IsMFMAFn = [&MAI] (MachineInstr *MI) {
+    MAI = nullptr;
+    if (SIInstrInfo::isMAI(*MI) &&
+        MI->getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32 &&
+        MI->getOpcode() != AMDGPU::V_ACCVGPR_READ_B32)
+      MAI = MI;
+    return MAI != nullptr;
+  };
+
+  MachineInstr *MI = SU->getInstr();
+  if (IsMFMAFn(MI)) {
+    int W = getWaitStatesSince(IsMFMAFn, 16);
+    if (MAI)
+      return W < (int)TSchedModel.computeInstrLatency(MAI);
+  }
+
+  return false;
+}
diff --git a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.h b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.h
index 59d6d4f90755a..629fae92e0058 100644
--- a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.h
+++ b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.h
@@ -108,6 +108,7 @@ class GCNHazardRecognizer final : public ScheduleHazardRecognizer {
   unsigned PreEmitNoopsCommon(MachineInstr *);
   void AdvanceCycle() override;
   void RecedeCycle() override;
+  bool ShouldPreferAnother(SUnit *SU) override;
 };
 
 } // end namespace llvm
diff --git a/llvm/test/CodeGen/AMDGPU/sched-prefer-non-mfma.mir b/llvm/test/CodeGen/AMDGPU/sched-prefer-non-mfma.mir
new file mode 100644
index 0000000000000..35d18c0c4c16d
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/sched-prefer-non-mfma.mir
@@ -0,0 +1,18 @@
+# RUN: llc -march=amdgcn -mcpu=gfx908 -run-pass post-RA-sched -amdgpu-disable-power-sched -verify-machineinstrs -o - %s | FileCheck -check-prefix=GCN %s
+
+# GCN-LABEL: name: unrelated_mfma
+# GCN: V_MFMA_F32_32X32X1F32
+# GCN: S_CMP_LG_U32
+# GCN: V_MFMA_F32_32X32X1F32
+
+---
+name:            unrelated_mfma
+body:             |
+  bb.0.entry:
+    renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 = V_MFMA_F32_32X32X1F32 $vgpr67, $vgpr66, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31, 0, 0, 0, implicit $mode, implicit $exec
+    renamable $agpr32_agpr33_agpr34_agpr35_agpr36_agpr37_agpr38_agpr39_agpr40_agpr41_agpr42_agpr43_agpr44_agpr45_agpr46_agpr47_agpr48_agpr49_agpr50_agpr51_agpr52_agpr53_agpr54_agpr55_agpr56_agpr57_agpr58_agpr59_agpr60_agpr61_agpr62_agpr63 = V_MFMA_F32_32X32X1F32 $vgpr69, $vgpr68, killed $agpr32_agpr33_agpr34_agpr35_agpr36_agpr37_agpr38_agpr39_agpr40_agpr41_agpr42_agpr43_agpr44_agpr45_agpr46_agpr47_agpr48_agpr49_agpr50_agpr51_agpr52_agpr53_agpr54_agpr55_agpr56_agpr57_agpr58_agpr59_agpr60_agpr61_agpr62_agpr63, 0, 0, 0, implicit $mode, implicit $exec
+    renamable $sgpr2 = S_ADD_U32 renamable $sgpr2, 4, implicit-def $scc
+    renamable $sgpr3 = S_ADDC_U32 renamable $sgpr3, 0, implicit-def dead $scc, implicit killed $scc
+    S_CMP_LG_U32 renamable $sgpr2, 64, implicit-def $scc
+
+...

From decfdb8ce3d511739b976d47862a98fe5674e2aa Mon Sep 17 00:00:00 2001
From: Stanislav Mekhanoshin <Stanislav.Mekhanoshin@amd.com>
Date: Wed, 29 Jul 2020 12:21:28 -0700
Subject: [PATCH 0529/1035] [AMDGPU] Fixed formatting in
 GCNHazardRecognizer.cpp. NFC.

---
 llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
index 6c95c3310be64..30a563fdc1dfd 100644
--- a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
@@ -715,8 +715,9 @@ int GCNHazardRecognizer::createsVALUHazard(const MachineInstr &MI) {
   return -1;
 }
 
-int GCNHazardRecognizer::checkVALUHazardsHelper(const MachineOperand &Def,
-						const MachineRegisterInfo &MRI) {
+int
+GCNHazardRecognizer::checkVALUHazardsHelper(const MachineOperand &Def,
+                                            const MachineRegisterInfo &MRI) {
   // Helper to check for the hazard where VMEM instructions that store more than
   // 8 bytes can have there store data over written by the next instruction.
   const SIRegisterInfo *TRI = ST.getRegisterInfo();

From 1cbdf932b41eb58ed94fbc240e93b63653d95bd3 Mon Sep 17 00:00:00 2001
From: Richard Smith <richard@metafoo.co.uk>
Date: Wed, 29 Jul 2020 12:29:05 -0700
Subject: [PATCH 0530/1035] PR46231: Promote diagnostic for 'template<...>;'
 from ExtWarn to Error.

No other compiler accepts this as an extension, not even in permissive
mode. We're not doing anyone any favors by allowing this, and it's
unlikely to be at all common, even in Clang-only code, in the wild.
---
 clang/include/clang/Basic/DiagnosticSemaKinds.td |  1 +
 clang/lib/Sema/SemaDecl.cpp                      |  5 ++++-
 clang/test/Parser/cxx-template-decl.cpp          | 11 ++++++++++-
 clang/test/SemaCXX/PR16677.cpp                   |  2 +-
 clang/test/SemaCXX/invalid-template-params.cpp   |  4 ++--
 clang/test/SemaTemplate/template-decl-fail.cpp   |  2 +-
 6 files changed, 19 insertions(+), 6 deletions(-)

diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 794ef23f07619..91112860a2d02 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -793,6 +793,7 @@ def ext_main_used : Extension<
 /// parser diagnostics
 def ext_no_declarators : ExtWarn<"declaration does not declare anything">,
   InGroup<MissingDeclarations>;
+def err_no_declarators : Error<"declaration does not declare anything">;
 def ext_typedef_without_a_name : ExtWarn<"typedef requires a name">,
   InGroup<MissingDeclarations>;
 def err_typedef_not_identifier : Error<"typedef name must be an identifier">;
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index ddbf086ea6385..9b9d164dcbc1b 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -4766,7 +4766,10 @@ Sema::ParsedFreeStandingDeclSpec(Scope *S, AccessSpecifier AS, DeclSpec &DS,
   if (!DeclaresAnything) {
     // In C, we allow this as a (popular) extension / bug. Don't bother
     // producing further diagnostics for redundant qualifiers after this.
-    Diag(DS.getBeginLoc(), diag::ext_no_declarators) << DS.getSourceRange();
+    Diag(DS.getBeginLoc(), (IsExplicitInstantiation || !TemplateParams.empty())
+                               ? diag::err_no_declarators
+                               : diag::ext_no_declarators)
+        << DS.getSourceRange();
     return TagD;
   }
 
diff --git a/clang/test/Parser/cxx-template-decl.cpp b/clang/test/Parser/cxx-template-decl.cpp
index 64e7ca921f575..7455b1d0ffe72 100644
--- a/clang/test/Parser/cxx-template-decl.cpp
+++ b/clang/test/Parser/cxx-template-decl.cpp
@@ -12,7 +12,7 @@ export template x;      // expected-error {{expected '<' after 'template'}}
 export template<class T> class x0; // expected-warning {{exported templates are unsupported}}
 template < ;            // expected-error {{expected template parameter}} \
 // expected-error{{expected ',' or '>' in template-parameter-list}} \
-// expected-warning {{declaration does not declare anything}}
+// expected-error {{declaration does not declare anything}}
 template <int +> struct x1; // expected-error {{expected ',' or '>' in template-parameter-list}}
 
 // verifies that we only walk to the ',' & still produce errors on the rest of the template parameters
@@ -286,3 +286,12 @@ namespace PR45239 {
   template<int> int b;
   template<int> auto f() -> b<0>; // expected-error +{{}}
 }
+
+namespace PR46231 {
+  template; // expected-error {{declaration does not declare anything}}
+  template<>; // expected-error {{declaration does not declare anything}}
+  template<int>; // expected-error {{declaration does not declare anything}}
+  template int; // expected-error {{declaration does not declare anything}}
+  template<> int; // expected-error {{declaration does not declare anything}}
+  template<int> int; // expected-error {{declaration does not declare anything}}
+}
diff --git a/clang/test/SemaCXX/PR16677.cpp b/clang/test/SemaCXX/PR16677.cpp
index efa4faaacd693..3ad76c6d861d7 100644
--- a/clang/test/SemaCXX/PR16677.cpp
+++ b/clang/test/SemaCXX/PR16677.cpp
@@ -11,5 +11,5 @@ template<class T,  // Should be angle bracket instead of comma
 class Derived : public Base<T> { // expected-error{{'Derived' cannot be defined in a type specifier}}
   Class_With_Destructor member;
 }; // expected-error{{expected ',' or '>' in template-parameter-list}}
-   // expected-warning@-1{{declaration does not declare anything}}
+   // expected-error@-1{{declaration does not declare anything}}
 
diff --git a/clang/test/SemaCXX/invalid-template-params.cpp b/clang/test/SemaCXX/invalid-template-params.cpp
index 0c463fe13d5b0..21220f3fea345 100644
--- a/clang/test/SemaCXX/invalid-template-params.cpp
+++ b/clang/test/SemaCXX/invalid-template-params.cpp
@@ -5,7 +5,7 @@ template<class> class Foo {
                       // expected-note@-1 {{'UBar' declared here}}
   void foo1(); // expected-error {{a non-type template parameter cannot have type 'class UBar'}}
                // expected-error@-1 {{expected ',' or '>' in template-parameter-list}}
-               // expected-warning@-2 {{declaration does not declare anything}}
+               // expected-error@-2 {{declaration does not declare anything}}
 };
 
 Foo<int>::UBar g1; // expected-error {{no type named 'UBar' in 'Foo<int>'}}
@@ -16,7 +16,7 @@ class C0 {
   struct S0 {}; // expected-error {{'S0' cannot be defined in a type specifier}}
                 // expected-error@-1 {{cannot combine with previous 'type-name' declaration specifier}}
                 // expected-error@-2 {{expected ',' or '>' in template-parameter-list}}
-                // expected-warning@-3 {{declaration does not declare anything}}
+                // expected-error@-3 {{declaration does not declare anything}}
   C0() : m(new S0<int>) {} // expected-error {{expected '(' for function-style cast or type construction}}
                            // expected-error@-1 {{expected expression}}
   S0<int> *m; // expected-error {{expected member name or ';' after declaration specifiers}}
diff --git a/clang/test/SemaTemplate/template-decl-fail.cpp b/clang/test/SemaTemplate/template-decl-fail.cpp
index ad134cdf225c2..7019caa2c2f31 100644
--- a/clang/test/SemaTemplate/template-decl-fail.cpp
+++ b/clang/test/SemaTemplate/template-decl-fail.cpp
@@ -4,7 +4,7 @@ template<typename T> typedef T X; // expected-error{{typedef cannot be a templat
 
 template<typename T>
 enum t0 { A = T::x }; // expected-error{{enumeration cannot be a template}} \
-                      // expected-warning{{declaration does not declare anything}}
+                      // expected-error{{declaration does not declare anything}}
 
 enum e0 {};
 template<int x> enum e0 f0(int a=x) {}

From f75564ad4e4799465cf14b96f761e3fae13f6976 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Wed, 29 Jul 2020 19:35:14 +0100
Subject: [PATCH 0531/1035] Reland "[SCEVExpander] Add option to preserve LCSSA
 directly."

This reverts the revert commit dc2867576886247cbe351e7c63618c09ab6af808.

It includes a fix for Polly, which uses SCEVExpander on IR that is not
in LCSSA form. Set PreserveLCSSA = false in that case, to ensure we do
not introduce LCSSA phis where there were none before.
---
 .../Utils/ScalarEvolutionExpander.h           |  41 ++++-
 .../Transforms/Scalar/LoopStrengthReduce.cpp  |   6 +-
 .../Utils/ScalarEvolutionExpander.cpp         | 173 +++++++++++++-----
 .../Utils/ScalarEvolutionExpanderTest.cpp     |   7 +-
 polly/lib/Support/ScopHelper.cpp              |   4 +-
 5 files changed, 168 insertions(+), 63 deletions(-)

diff --git a/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h b/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h
index 6e53d15d25fc7..cb212b2705eed 100644
--- a/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h
+++ b/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h
@@ -52,6 +52,9 @@ class SCEVExpander : public SCEVVisitor<SCEVExpander, Value *> {
   // New instructions receive a name to identify them with the current pass.
   const char *IVName;
 
+  /// Indicates whether LCSSA phis should be created for inserted values.
+  bool PreserveLCSSA;
+
   // InsertedExpressions caches Values for reuse, so must track RAUW.
   DenseMap<std::pair<const SCEV *, Instruction *>, TrackingVH<Value>>
       InsertedExpressions;
@@ -146,9 +149,10 @@ class SCEVExpander : public SCEVVisitor<SCEVExpander, Value *> {
 public:
   /// Construct a SCEVExpander in "canonical" mode.
   explicit SCEVExpander(ScalarEvolution &se, const DataLayout &DL,
-                        const char *name)
-      : SE(se), DL(DL), IVName(name), IVIncInsertLoop(nullptr),
-        IVIncInsertPos(nullptr), CanonicalMode(true), LSRMode(false),
+                        const char *name, bool PreserveLCSSA = true)
+      : SE(se), DL(DL), IVName(name), PreserveLCSSA(PreserveLCSSA),
+        IVIncInsertLoop(nullptr), IVIncInsertPos(nullptr), CanonicalMode(true),
+        LSRMode(false),
         Builder(se.getContext(), TargetFolder(DL),
                 IRBuilderCallbackInserter(
                     [this](Instruction *I) { rememberInstruction(I); })) {
@@ -223,14 +227,18 @@ class SCEVExpander : public SCEVVisitor<SCEVExpander, Value *> {
                                const TargetTransformInfo *TTI = nullptr);
 
   /// Insert code to directly compute the specified SCEV expression into the
-  /// program.  The inserted code is inserted into the specified block.
-  Value *expandCodeFor(const SCEV *SH, Type *Ty, Instruction *I);
+  /// program.  The code is inserted into the specified block.
+  Value *expandCodeFor(const SCEV *SH, Type *Ty, Instruction *I) {
+    return expandCodeForImpl(SH, Ty, I, true);
+  }
 
   /// Insert code to directly compute the specified SCEV expression into the
-  /// program.  The inserted code is inserted into the SCEVExpander's current
+  /// program.  The code is inserted into the SCEVExpander's current
   /// insertion point. If a type is specified, the result will be expanded to
   /// have that type, with a cast if necessary.
-  Value *expandCodeFor(const SCEV *SH, Type *Ty = nullptr);
+  Value *expandCodeFor(const SCEV *SH, Type *Ty = nullptr) {
+    return expandCodeForImpl(SH, Ty, true);
+  }
 
   /// Generates a code sequence that evaluates this predicate.  The inserted
   /// instructions will be at position \p Loc.  The result will be of type i1
@@ -338,6 +346,20 @@ class SCEVExpander : public SCEVVisitor<SCEVExpander, Value *> {
 private:
   LLVMContext &getContext() const { return SE.getContext(); }
 
+  /// Insert code to directly compute the specified SCEV expression into the
+  /// program. The code is inserted into the SCEVExpander's current
+  /// insertion point. If a type is specified, the result will be expanded to
+  /// have that type, with a cast if necessary. If \p Root is true, this
+  /// indicates that \p SH is the top-level expression to expand passed from
+  /// an external client call.
+  Value *expandCodeForImpl(const SCEV *SH, Type *Ty, bool Root);
+
+  /// Insert code to directly compute the specified SCEV expression into the
+  /// program. The code is inserted into the specified block. If \p
+  /// Root is true, this indicates that \p SH is the top-level expression to
+  /// expand passed from an external client call.
+  Value *expandCodeForImpl(const SCEV *SH, Type *Ty, Instruction *I, bool Root);
+
   /// Recursive helper function for isHighCostExpansion.
   bool isHighCostExpansionHelper(const SCEV *S, Loop *L, const Instruction &At,
                                  int &BudgetRemaining,
@@ -419,6 +441,11 @@ class SCEVExpander : public SCEVVisitor<SCEVExpander, Value *> {
                       Instruction *Pos, PHINode *LoopPhi);
 
   void fixupInsertPoints(Instruction *I);
+
+  /// If required, create LCSSA PHIs for \p Users' operand \p OpIdx. If new
+  /// LCSSA PHIs have been created, return the LCSSA PHI available at \p User.
+  /// If no PHIs have been created, return the unchanged operand \p OpIdx.
+  Value *fixupLCSSAFormFor(Instruction *User, unsigned OpIdx);
 };
 } // namespace llvm
 
diff --git a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
index cf02ef1e83f3f..c3e46c1fadef3 100644
--- a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
@@ -5514,8 +5514,8 @@ void LSRInstance::ImplementSolution(
   // we can remove them after we are done working.
   SmallVector<WeakTrackingVH, 16> DeadInsts;
 
-  SCEVExpander Rewriter(SE, L->getHeader()->getModule()->getDataLayout(),
-                        "lsr");
+  SCEVExpander Rewriter(SE, L->getHeader()->getModule()->getDataLayout(), "lsr",
+                        false);
 #ifndef NDEBUG
   Rewriter.setDebugType(DEBUG_TYPE);
 #endif
@@ -5780,7 +5780,7 @@ static bool ReduceLoopStrength(Loop *L, IVUsers &IU, ScalarEvolution &SE,
   if (EnablePhiElim && L->isLoopSimplifyForm()) {
     SmallVector<WeakTrackingVH, 16> DeadInsts;
     const DataLayout &DL = L->getHeader()->getModule()->getDataLayout();
-    SCEVExpander Rewriter(SE, DL, "lsr");
+    SCEVExpander Rewriter(SE, DL, "lsr", false);
 #ifndef NDEBUG
     Rewriter.setDebugType(DEBUG_TYPE);
 #endif
diff --git a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
index c99d612a7768f..1a10e580c68c0 100644
--- a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
+++ b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
@@ -27,6 +27,7 @@
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
+#include "llvm/Transforms/Utils/LoopUtils.h"
 
 using namespace llvm;
 
@@ -461,9 +462,10 @@ Value *SCEVExpander::expandAddToGEP(const SCEV *const *op_begin,
     // we didn't find any operands that could be factored, tentatively
     // assume that element zero was selected (since the zero offset
     // would obviously be folded away).
-    Value *Scaled = ScaledOps.empty() ?
-                    Constant::getNullValue(Ty) :
-                    expandCodeFor(SE.getAddExpr(ScaledOps), Ty);
+    Value *Scaled =
+        ScaledOps.empty()
+            ? Constant::getNullValue(Ty)
+            : expandCodeForImpl(SE.getAddExpr(ScaledOps), Ty, false);
     GepIndices.push_back(Scaled);
 
     // Collect struct field index operands.
@@ -522,7 +524,7 @@ Value *SCEVExpander::expandAddToGEP(const SCEV *const *op_begin,
            SE.DT.dominates(cast<Instruction>(V), &*Builder.GetInsertPoint()));
 
     // Expand the operands for a plain byte offset.
-    Value *Idx = expandCodeFor(SE.getAddExpr(Ops), Ty);
+    Value *Idx = expandCodeForImpl(SE.getAddExpr(Ops), Ty, false);
 
     // Fold a GEP with constant operands.
     if (Constant *CLHS = dyn_cast<Constant>(V))
@@ -743,14 +745,14 @@ Value *SCEVExpander::visitAddExpr(const SCEVAddExpr *S) {
       Sum = expandAddToGEP(NewOps.begin(), NewOps.end(), PTy, Ty, expand(Op));
     } else if (Op->isNonConstantNegative()) {
       // Instead of doing a negate and add, just do a subtract.
-      Value *W = expandCodeFor(SE.getNegativeSCEV(Op), Ty);
+      Value *W = expandCodeForImpl(SE.getNegativeSCEV(Op), Ty, false);
       Sum = InsertNoopCastOfTo(Sum, Ty);
       Sum = InsertBinop(Instruction::Sub, Sum, W, SCEV::FlagAnyWrap,
                         /*IsSafeToHoist*/ true);
       ++I;
     } else {
       // A simple add.
-      Value *W = expandCodeFor(Op, Ty);
+      Value *W = expandCodeForImpl(Op, Ty, false);
       Sum = InsertNoopCastOfTo(Sum, Ty);
       // Canonicalize a constant to the RHS.
       if (isa<Constant>(Sum)) std::swap(Sum, W);
@@ -802,7 +804,7 @@ Value *SCEVExpander::visitMulExpr(const SCEVMulExpr *S) {
 
     // Calculate powers with exponents 1, 2, 4, 8 etc. and include those of them
     // that are needed into the result.
-    Value *P = expandCodeFor(I->second, Ty);
+    Value *P = expandCodeForImpl(I->second, Ty, false);
     Value *Result = nullptr;
     if (Exponent & 1)
       Result = P;
@@ -861,7 +863,7 @@ Value *SCEVExpander::visitMulExpr(const SCEVMulExpr *S) {
 Value *SCEVExpander::visitUDivExpr(const SCEVUDivExpr *S) {
   Type *Ty = SE.getEffectiveSCEVType(S->getType());
 
-  Value *LHS = expandCodeFor(S->getLHS(), Ty);
+  Value *LHS = expandCodeForImpl(S->getLHS(), Ty, false);
   if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(S->getRHS())) {
     const APInt &RHS = SC->getAPInt();
     if (RHS.isPowerOf2())
@@ -870,7 +872,7 @@ Value *SCEVExpander::visitUDivExpr(const SCEVUDivExpr *S) {
                          SCEV::FlagAnyWrap, /*IsSafeToHoist*/ true);
   }
 
-  Value *RHS = expandCodeFor(S->getRHS(), Ty);
+  Value *RHS = expandCodeForImpl(S->getRHS(), Ty, false);
   return InsertBinop(Instruction::UDiv, LHS, RHS, SCEV::FlagAnyWrap,
                      /*IsSafeToHoist*/ SE.isKnownNonZero(S->getRHS()));
 }
@@ -1265,8 +1267,9 @@ SCEVExpander::getAddRecExprPHILiterally(const SCEVAddRecExpr *Normalized,
   // Expand code for the start value into the loop preheader.
   assert(L->getLoopPreheader() &&
          "Can't expand add recurrences without a loop preheader!");
-  Value *StartV = expandCodeFor(Normalized->getStart(), ExpandTy,
-                                L->getLoopPreheader()->getTerminator());
+  Value *StartV =
+      expandCodeForImpl(Normalized->getStart(), ExpandTy,
+                        L->getLoopPreheader()->getTerminator(), false);
 
   // StartV must have been be inserted into L's preheader to dominate the new
   // phi.
@@ -1284,8 +1287,8 @@ SCEVExpander::getAddRecExprPHILiterally(const SCEVAddRecExpr *Normalized,
   if (useSubtract)
     Step = SE.getNegativeSCEV(Step);
   // Expand the step somewhere that dominates the loop header.
-  Value *StepV = expandCodeFor(Step, IntTy,
-                               &*L->getHeader()->getFirstInsertionPt());
+  Value *StepV = expandCodeForImpl(
+      Step, IntTy, &*L->getHeader()->getFirstInsertionPt(), false);
 
   // The no-wrap behavior proved by IsIncrement(NUW|NSW) is only applicable if
   // we actually do emit an addition.  It does not apply if we emit a
@@ -1430,8 +1433,8 @@ Value *SCEVExpander::expandAddRecExprLiterally(const SCEVAddRecExpr *S) {
       {
         // Expand the step somewhere that dominates the loop header.
         SCEVInsertPointGuard Guard(Builder, this);
-        StepV = expandCodeFor(Step, IntTy,
-                              &*L->getHeader()->getFirstInsertionPt());
+        StepV = expandCodeForImpl(
+            Step, IntTy, &*L->getHeader()->getFirstInsertionPt(), false);
       }
       Result = expandIVInc(PN, StepV, L, ExpandTy, IntTy, useSubtract);
     }
@@ -1450,8 +1453,8 @@ Value *SCEVExpander::expandAddRecExprLiterally(const SCEVAddRecExpr *S) {
 
     // Invert the result.
     if (InvertStep)
-      Result = Builder.CreateSub(expandCodeFor(Normalized->getStart(), TruncTy),
-                                 Result);
+      Result = Builder.CreateSub(
+          expandCodeForImpl(Normalized->getStart(), TruncTy, false), Result);
   }
 
   // Re-apply any non-loop-dominating scale.
@@ -1459,22 +1462,22 @@ Value *SCEVExpander::expandAddRecExprLiterally(const SCEVAddRecExpr *S) {
     assert(S->isAffine() && "Can't linearly scale non-affine recurrences.");
     Result = InsertNoopCastOfTo(Result, IntTy);
     Result = Builder.CreateMul(Result,
-                               expandCodeFor(PostLoopScale, IntTy));
+                               expandCodeForImpl(PostLoopScale, IntTy, false));
   }
 
   // Re-apply any non-loop-dominating offset.
   if (PostLoopOffset) {
     if (PointerType *PTy = dyn_cast<PointerType>(ExpandTy)) {
       if (Result->getType()->isIntegerTy()) {
-        Value *Base = expandCodeFor(PostLoopOffset, ExpandTy);
+        Value *Base = expandCodeForImpl(PostLoopOffset, ExpandTy, false);
         Result = expandAddToGEP(SE.getUnknown(Result), PTy, IntTy, Base);
       } else {
         Result = expandAddToGEP(PostLoopOffset, PTy, IntTy, Result);
       }
     } else {
       Result = InsertNoopCastOfTo(Result, IntTy);
-      Result = Builder.CreateAdd(Result,
-                                 expandCodeFor(PostLoopOffset, IntTy));
+      Result = Builder.CreateAdd(
+          Result, expandCodeForImpl(PostLoopOffset, IntTy, false));
     }
   }
 
@@ -1516,8 +1519,8 @@ Value *SCEVExpander::visitAddRecExpr(const SCEVAddRecExpr *S) {
                                        S->getNoWrapFlags(SCEV::FlagNW)));
     BasicBlock::iterator NewInsertPt =
         findInsertPointAfter(cast<Instruction>(V), Builder.GetInsertBlock());
-    V = expandCodeFor(SE.getTruncateExpr(SE.getUnknown(V), Ty), nullptr,
-                      &*NewInsertPt);
+    V = expandCodeForImpl(SE.getTruncateExpr(SE.getUnknown(V), Ty), nullptr,
+                          &*NewInsertPt, false);
     return V;
   }
 
@@ -1632,22 +1635,25 @@ Value *SCEVExpander::visitAddRecExpr(const SCEVAddRecExpr *S) {
 
 Value *SCEVExpander::visitTruncateExpr(const SCEVTruncateExpr *S) {
   Type *Ty = SE.getEffectiveSCEVType(S->getType());
-  Value *V = expandCodeFor(S->getOperand(),
-                           SE.getEffectiveSCEVType(S->getOperand()->getType()));
+  Value *V = expandCodeForImpl(
+      S->getOperand(), SE.getEffectiveSCEVType(S->getOperand()->getType()),
+      false);
   return Builder.CreateTrunc(V, Ty);
 }
 
 Value *SCEVExpander::visitZeroExtendExpr(const SCEVZeroExtendExpr *S) {
   Type *Ty = SE.getEffectiveSCEVType(S->getType());
-  Value *V = expandCodeFor(S->getOperand(),
-                           SE.getEffectiveSCEVType(S->getOperand()->getType()));
+  Value *V = expandCodeForImpl(
+      S->getOperand(), SE.getEffectiveSCEVType(S->getOperand()->getType()),
+      false);
   return Builder.CreateZExt(V, Ty);
 }
 
 Value *SCEVExpander::visitSignExtendExpr(const SCEVSignExtendExpr *S) {
   Type *Ty = SE.getEffectiveSCEVType(S->getType());
-  Value *V = expandCodeFor(S->getOperand(),
-                           SE.getEffectiveSCEVType(S->getOperand()->getType()));
+  Value *V = expandCodeForImpl(
+      S->getOperand(), SE.getEffectiveSCEVType(S->getOperand()->getType()),
+      false);
   return Builder.CreateSExt(V, Ty);
 }
 
@@ -1662,7 +1668,7 @@ Value *SCEVExpander::visitSMaxExpr(const SCEVSMaxExpr *S) {
       Ty = SE.getEffectiveSCEVType(Ty);
       LHS = InsertNoopCastOfTo(LHS, Ty);
     }
-    Value *RHS = expandCodeFor(S->getOperand(i), Ty);
+    Value *RHS = expandCodeForImpl(S->getOperand(i), Ty, false);
     Value *ICmp = Builder.CreateICmpSGT(LHS, RHS);
     Value *Sel = Builder.CreateSelect(ICmp, LHS, RHS, "smax");
     LHS = Sel;
@@ -1685,7 +1691,7 @@ Value *SCEVExpander::visitUMaxExpr(const SCEVUMaxExpr *S) {
       Ty = SE.getEffectiveSCEVType(Ty);
       LHS = InsertNoopCastOfTo(LHS, Ty);
     }
-    Value *RHS = expandCodeFor(S->getOperand(i), Ty);
+    Value *RHS = expandCodeForImpl(S->getOperand(i), Ty, false);
     Value *ICmp = Builder.CreateICmpUGT(LHS, RHS);
     Value *Sel = Builder.CreateSelect(ICmp, LHS, RHS, "umax");
     LHS = Sel;
@@ -1708,7 +1714,7 @@ Value *SCEVExpander::visitSMinExpr(const SCEVSMinExpr *S) {
       Ty = SE.getEffectiveSCEVType(Ty);
       LHS = InsertNoopCastOfTo(LHS, Ty);
     }
-    Value *RHS = expandCodeFor(S->getOperand(i), Ty);
+    Value *RHS = expandCodeForImpl(S->getOperand(i), Ty, false);
     Value *ICmp = Builder.CreateICmpSLT(LHS, RHS);
     Value *Sel = Builder.CreateSelect(ICmp, LHS, RHS, "smin");
     LHS = Sel;
@@ -1731,7 +1737,7 @@ Value *SCEVExpander::visitUMinExpr(const SCEVUMinExpr *S) {
       Ty = SE.getEffectiveSCEVType(Ty);
       LHS = InsertNoopCastOfTo(LHS, Ty);
     }
-    Value *RHS = expandCodeFor(S->getOperand(i), Ty);
+    Value *RHS = expandCodeForImpl(S->getOperand(i), Ty, false);
     Value *ICmp = Builder.CreateICmpULT(LHS, RHS);
     Value *Sel = Builder.CreateSelect(ICmp, LHS, RHS, "umin");
     LHS = Sel;
@@ -1743,15 +1749,43 @@ Value *SCEVExpander::visitUMinExpr(const SCEVUMinExpr *S) {
   return LHS;
 }
 
-Value *SCEVExpander::expandCodeFor(const SCEV *SH, Type *Ty,
-                                   Instruction *IP) {
+Value *SCEVExpander::expandCodeForImpl(const SCEV *SH, Type *Ty,
+                                       Instruction *IP, bool Root) {
   setInsertPoint(IP);
-  return expandCodeFor(SH, Ty);
+  Value *V = expandCodeForImpl(SH, Ty, Root);
+  return V;
 }
 
-Value *SCEVExpander::expandCodeFor(const SCEV *SH, Type *Ty) {
+Value *SCEVExpander::expandCodeForImpl(const SCEV *SH, Type *Ty, bool Root) {
   // Expand the code for this SCEV.
   Value *V = expand(SH);
+
+  if (PreserveLCSSA) {
+    if (auto *Inst = dyn_cast<Instruction>(V)) {
+      // Create a temporary instruction to at the current insertion point, so we
+      // can hand it off to the helper to create LCSSA PHIs if required for the
+      // new use.
+      // FIXME: Ideally formLCSSAForInstructions (used in fixupLCSSAFormFor)
+      // would accept a insertion point and return an LCSSA phi for that
+      // insertion point, so there is no need to insert & remove the temporary
+      // instruction.
+      Instruction *Tmp;
+      if (Inst->getType()->isIntegerTy())
+        Tmp = cast<Instruction>(Builder.CreateAdd(Inst, Inst));
+      else {
+        assert(Inst->getType()->isPointerTy());
+        Tmp = cast<Instruction>(Builder.CreateGEP(Inst, Builder.getInt32(1)));
+      }
+      V = fixupLCSSAFormFor(Tmp, 0);
+
+      // Clean up temporary instruction.
+      InsertedValues.erase(Tmp);
+      InsertedPostIncValues.erase(Tmp);
+      Tmp->eraseFromParent();
+    }
+  }
+
+  InsertedExpressions[std::make_pair(SH, &*Builder.GetInsertPoint())] = V;
   if (Ty) {
     assert(SE.getTypeSizeInBits(Ty) == SE.getTypeSizeInBits(SH->getType()) &&
            "non-trivial casts should be done with the SCEVs directly!");
@@ -1891,10 +1925,28 @@ Value *SCEVExpander::expand(const SCEV *S) {
 }
 
 void SCEVExpander::rememberInstruction(Value *I) {
-  if (!PostIncLoops.empty())
-    InsertedPostIncValues.insert(I);
-  else
-    InsertedValues.insert(I);
+  auto DoInsert = [this](Value *V) {
+    if (!PostIncLoops.empty())
+      InsertedPostIncValues.insert(V);
+    else
+      InsertedValues.insert(V);
+  };
+  DoInsert(I);
+
+  if (!PreserveLCSSA)
+    return;
+
+  if (auto *Inst = dyn_cast<Instruction>(I)) {
+    // A new instruction has been added, which might introduce new uses outside
+    // a defining loop. Fix LCSSA from for each operand of the new instruction,
+    // if required.
+    for (unsigned OpIdx = 0, OpEnd = Inst->getNumOperands(); OpIdx != OpEnd;
+         OpIdx++) {
+      auto *V = fixupLCSSAFormFor(Inst, OpIdx);
+      if (V != I)
+        DoInsert(V);
+    }
+  }
 }
 
 /// getOrInsertCanonicalInductionVariable - This method returns the
@@ -1913,9 +1965,8 @@ SCEVExpander::getOrInsertCanonicalInductionVariable(const Loop *L,
 
   // Emit code for it.
   SCEVInsertPointGuard Guard(Builder, this);
-  PHINode *V =
-      cast<PHINode>(expandCodeFor(H, nullptr,
-                                  &*L->getHeader()->getFirstInsertionPt()));
+  PHINode *V = cast<PHINode>(expandCodeForImpl(
+      H, nullptr, &*L->getHeader()->getFirstInsertionPt(), false));
 
   return V;
 }
@@ -2315,8 +2366,10 @@ Value *SCEVExpander::expandCodeForPredicate(const SCEVPredicate *Pred,
 
 Value *SCEVExpander::expandEqualPredicate(const SCEVEqualPredicate *Pred,
                                           Instruction *IP) {
-  Value *Expr0 = expandCodeFor(Pred->getLHS(), Pred->getLHS()->getType(), IP);
-  Value *Expr1 = expandCodeFor(Pred->getRHS(), Pred->getRHS()->getType(), IP);
+  Value *Expr0 =
+      expandCodeForImpl(Pred->getLHS(), Pred->getLHS()->getType(), IP, false);
+  Value *Expr1 =
+      expandCodeForImpl(Pred->getRHS(), Pred->getRHS()->getType(), IP, false);
 
   Builder.SetInsertPoint(IP);
   auto *I = Builder.CreateICmpNE(Expr0, Expr1, "ident.check");
@@ -2348,15 +2401,16 @@ Value *SCEVExpander::generateOverflowCheck(const SCEVAddRecExpr *AR,
 
   IntegerType *CountTy = IntegerType::get(Loc->getContext(), SrcBits);
   Builder.SetInsertPoint(Loc);
-  Value *TripCountVal = expandCodeFor(ExitCount, CountTy, Loc);
+  Value *TripCountVal = expandCodeForImpl(ExitCount, CountTy, Loc, false);
 
   IntegerType *Ty =
       IntegerType::get(Loc->getContext(), SE.getTypeSizeInBits(ARTy));
   Type *ARExpandTy = DL.isNonIntegralPointerType(ARTy) ? ARTy : Ty;
 
-  Value *StepValue = expandCodeFor(Step, Ty, Loc);
-  Value *NegStepValue = expandCodeFor(SE.getNegativeSCEV(Step), Ty, Loc);
-  Value *StartValue = expandCodeFor(Start, ARExpandTy, Loc);
+  Value *StepValue = expandCodeForImpl(Step, Ty, Loc, false);
+  Value *NegStepValue =
+      expandCodeForImpl(SE.getNegativeSCEV(Step), Ty, Loc, false);
+  Value *StartValue = expandCodeForImpl(Start, ARExpandTy, Loc, false);
 
   ConstantInt *Zero =
       ConstantInt::get(Loc->getContext(), APInt::getNullValue(DstBits));
@@ -2459,6 +2513,25 @@ Value *SCEVExpander::expandUnionPredicate(const SCEVUnionPredicate *Union,
   return Check;
 }
 
+Value *SCEVExpander::fixupLCSSAFormFor(Instruction *User, unsigned OpIdx) {
+  assert(PreserveLCSSA);
+  SmallVector<Instruction *, 1> ToUpdate;
+
+  auto *OpV = User->getOperand(OpIdx);
+  auto *OpI = dyn_cast<Instruction>(OpV);
+  if (!OpI)
+    return OpV;
+
+  Loop *DefLoop = SE.LI.getLoopFor(OpI->getParent());
+  Loop *UseLoop = SE.LI.getLoopFor(User->getParent());
+  if (!DefLoop || UseLoop == DefLoop || DefLoop->contains(UseLoop))
+    return OpV;
+
+  ToUpdate.push_back(OpI);
+  formLCSSAForInstructions(ToUpdate, SE.DT, SE.LI, &SE);
+  return User->getOperand(OpIdx);
+}
+
 namespace {
 // Search for a SCEV subexpression that is not safe to expand.  Any expression
 // that may expand to a !isSafeToSpeculativelyExecute value is unsafe, namely
diff --git a/llvm/unittests/Transforms/Utils/ScalarEvolutionExpanderTest.cpp b/llvm/unittests/Transforms/Utils/ScalarEvolutionExpanderTest.cpp
index c146d155ffbea..6e83370dfeb95 100644
--- a/llvm/unittests/Transforms/Utils/ScalarEvolutionExpanderTest.cpp
+++ b/llvm/unittests/Transforms/Utils/ScalarEvolutionExpanderTest.cpp
@@ -265,7 +265,7 @@ TEST_F(ScalarEvolutionExpanderTest, SCEVExpanderIsSafeToExpandAt) {
   Phi->addIncoming(Add, L);
 
   Builder.SetInsertPoint(Post);
-  Builder.CreateRetVoid();
+  Instruction *Ret = Builder.CreateRetVoid();
 
   ScalarEvolution SE = buildSE(*F);
   const SCEV *S = SE.getSCEV(Phi);
@@ -276,6 +276,11 @@ TEST_F(ScalarEvolutionExpanderTest, SCEVExpanderIsSafeToExpandAt) {
   EXPECT_FALSE(isSafeToExpandAt(AR, LPh->getTerminator(), SE));
   EXPECT_TRUE(isSafeToExpandAt(AR, L->getTerminator(), SE));
   EXPECT_TRUE(isSafeToExpandAt(AR, Post->getTerminator(), SE));
+
+  EXPECT_TRUE(LI->getLoopFor(L)->isLCSSAForm(*DT));
+  SCEVExpander Exp(SE, M.getDataLayout(), "expander");
+  Exp.expandCodeFor(SE.getSCEV(Add), nullptr, Ret);
+  EXPECT_TRUE(LI->getLoopFor(L)->isLCSSAForm(*DT));
 }
 
 // Check that SCEV expander does not use the nuw instruction
diff --git a/polly/lib/Support/ScopHelper.cpp b/polly/lib/Support/ScopHelper.cpp
index 78af966522dda..bf2d82e446600 100644
--- a/polly/lib/Support/ScopHelper.cpp
+++ b/polly/lib/Support/ScopHelper.cpp
@@ -243,8 +243,8 @@ struct ScopExpander : SCEVVisitor<ScopExpander, const SCEV *> {
   explicit ScopExpander(const Region &R, ScalarEvolution &SE,
                         const DataLayout &DL, const char *Name, ValueMapT *VMap,
                         BasicBlock *RTCBB)
-      : Expander(SE, DL, Name), SE(SE), Name(Name), R(R), VMap(VMap),
-        RTCBB(RTCBB) {}
+      : Expander(SE, DL, Name, /*PreserveLCSSA=*/false), SE(SE), Name(Name),
+        R(R), VMap(VMap), RTCBB(RTCBB) {}
 
   Value *expandCodeFor(const SCEV *E, Type *Ty, Instruction *I) {
     // If we generate code in the region we will immediately fall back to the

From e980913831c1dfe5ce443a0b4f7a551c992133b8 Mon Sep 17 00:00:00 2001
From: Philip Reames <listmail@philipreames.com>
Date: Wed, 29 Jul 2020 12:44:49 -0700
Subject: [PATCH 0532/1035] [Tests] Split a file for ease of update

---
 .../CodeGen/X86/statepoint-vreg-details.ll    |  399 +++++++
 llvm/test/CodeGen/X86/statepoint-vreg.ll      | 1004 ++++-------------
 2 files changed, 638 insertions(+), 765 deletions(-)
 create mode 100644 llvm/test/CodeGen/X86/statepoint-vreg-details.ll

diff --git a/llvm/test/CodeGen/X86/statepoint-vreg-details.ll b/llvm/test/CodeGen/X86/statepoint-vreg-details.ll
new file mode 100644
index 0000000000000..bfd08f983103b
--- /dev/null
+++ b/llvm/test/CodeGen/X86/statepoint-vreg-details.ll
@@ -0,0 +1,399 @@
+; This file contains some of the same basic tests as statepoint-vreg.ll, but
+; focuses on examining the intermediate representation.  It's separate so that
+; the main file is easy to update with update_llc_test_checks.py
+
+; This run is to demonstrate what MIR SSA looks like.
+; RUN: llc -max-registers-for-gc-values=4 -stop-after finalize-isel < %s | FileCheck --check-prefix=CHECK-VREG %s
+; This run is to demonstrate register allocator work.
+; RUN: llc -max-registers-for-gc-values=4 -stop-after virtregrewriter < %s | FileCheck --check-prefix=CHECK-PREG %s
+
+target datalayout = "e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-linux-gnu"
+
+declare i1 @return_i1()
+declare void @func()
+declare void @consume(i32 addrspace(1)*)
+declare void @consume2(i32 addrspace(1)*, i32 addrspace(1)*)
+declare void @consume5(i32 addrspace(1)*, i32 addrspace(1)*, i32 addrspace(1)*, i32 addrspace(1)*, i32 addrspace(1)*)
+declare void @use1(i32 addrspace(1)*, i8 addrspace(1)*)
+
+; test most simple relocate
+define i1 @test_relocate(i32 addrspace(1)* %a) gc "statepoint-example" {
+; CHECK-VREG-LABEL: name:            test_relocate
+; CHECK-VREG:    %0:gr64 = COPY $rdi
+; CHECK-VREG:    %1:gr64 = STATEPOINT 0, 0, 0, @return_i1, 2, 0, 2, 0, 2, 0, %0, %0(tied-def 0), csr_64, implicit-def $rsp, implicit-def $ssp, implicit-def $al
+; CHECK-VREG:    %2:gr8 = COPY $al
+; CHECK-VREG:    $rdi = COPY %1
+; CHECK-VREG:    CALL64pcrel32 @consume, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit-def $rsp, implicit-def $ssp
+
+; CHECK-PREG-LABEL: name:            test_relocate
+; CHECK-PREG:    renamable $rbx = COPY $rdi
+; CHECK-PREG:    renamable $rbx = STATEPOINT 0, 0, 0, @return_i1, 2, 0, 2, 0, 2, 0, killed renamable $rbx, renamable $rbx(tied-def 0), csr_64, implicit-def $rsp, implicit-def $ssp, implicit-def $al
+; CHECK-PREG:    renamable $bpl = COPY killed $al
+; CHECK-PREG:    $rdi = COPY killed renamable $rbx
+; CHECK-PREG:    CALL64pcrel32 @consume, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit-def $rsp, implicit-def $ssp
+
+entry:
+  %safepoint_token = tail call token (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0) ["gc-live" (i32 addrspace(1)* %a)]
+  %rel1 = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 0, i32 0)
+  %res1 = call zeroext i1 @llvm.experimental.gc.result.i1(token %safepoint_token)
+  call void @consume(i32 addrspace(1)* %rel1)
+  ret i1 %res1
+}
+; test pointer variables intermixed with pointer constants
+define void @test_mixed(i32 addrspace(1)* %a, i32 addrspace(1)* %b, i32 addrspace(1)* %c) gc "statepoint-example" {
+; CHECK-VREG-LABEL: name:            test_mixed
+; CHECK-VREG:    %2:gr64 = COPY $rdx
+; CHECK-VREG:    %1:gr64 = COPY $rsi
+; CHECK-VREG:    %0:gr64 = COPY $rdi
+; CHECK-VREG:    %3:gr64, %4:gr64, %5:gr64 = STATEPOINT 0, 0, 0, @func, 2, 0, 2, 0, 2, 0, %2, %2(tied-def 0), 2, 0, 2, 0, %1, %1(tied-def 1), %0, %0(tied-def 2), csr_64
+; CHECK-VREG:    %6:gr32 = MOV32r0 implicit-def dead $eflags
+; CHECK-VREG:    %7:gr64 = SUBREG_TO_REG 0, killed %6, %subreg.sub_32bit
+; CHECK-VREG:    $rdi = COPY %5
+; CHECK-VREG:    $rsi = COPY %7
+; CHECK-VREG:    $rdx = COPY %4
+; CHECK-VREG:    $rcx = COPY %7
+; CHECK-VREG:    $r8 = COPY %3
+; CHECK-VREG:    CALL64pcrel32 @consume5, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit $rsi, implicit $rdx, implicit $rcx, implicit $r8, implicit-def $rsp, implicit-def $ssp
+
+; CHECK-PREG-LABEL: name:            test_mixed
+; CHECK-PREG:    renamable $r14 = COPY $rdx
+; CHECK-PREG:    renamable $r15 = COPY $rsi
+; CHECK-PREG:    renamable $rbx = COPY $rdi
+; CHECK-PREG:    renamable $r14, renamable $r15, renamable $rbx = STATEPOINT 0, 0, 0, @func, 2, 0, 2, 0, 2, 0, killed renamable $r14, renamable $r14(tied-def 0), 2, 0, 2, 0, killed renamable $r15, renamable $r15(tied-def 1), killed renamable $rbx, renamable $rbx(tied-def 2), csr_64, implicit-def $rsp, implicit-def $ssp
+; CHECK-PREG:    $rdi = COPY killed renamable $rbx
+; CHECK-PREG:    dead $esi = MOV32r0 implicit-def dead $eflags, implicit-def $rsi
+; CHECK-PREG:    $rdx = COPY killed renamable $r15
+; CHECK-PREG:    dead $ecx = MOV32r0 implicit-def dead $eflags, implicit-def $rcx
+; CHECK-PREG:    $r8 = COPY killed renamable $r14
+; CHECK-PREG:    CALL64pcrel32 @consume5, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit $rsi, implicit $rdx, implicit killed $rcx, implicit killed $r8, implicit-def $rsp, implicit-def $ssp
+
+entry:
+  %safepoint_token = tail call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0) ["gc-live" (i32 addrspace(1)* %a, i32 addrspace(1)* null, i32 addrspace(1)* %b, i32 addrspace(1)* null, i32 addrspace(1)* %c)]
+  %rel1 = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 0, i32 0)
+  %rel2 = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 1, i32 1)
+  %rel3 = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 2, i32 2)
+  %rel4 = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 3, i32 3)
+  %rel5 = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 4, i32 4)
+  call void @consume5(i32 addrspace(1)* %rel1, i32 addrspace(1)* %rel2, i32 addrspace(1)* %rel3, i32 addrspace(1)* %rel4, i32 addrspace(1)* %rel5)
+  ret void
+}
+
+; same as above, but for alloca
+define i32 addrspace(1)* @test_alloca(i32 addrspace(1)* %ptr) gc "statepoint-example" {
+; CHECK-VREG-LABEL: name:            test_alloca
+; CHECK-VREG:    %0:gr64 = COPY $rdi
+; CHECK-VREG:    MOV64mr %stack.0.alloca, 1, $noreg, 0, $noreg, %0 :: (store 8 into %ir.alloca)
+; CHECK-VREG:    %1:gr64 = STATEPOINT 0, 0, 0, @return_i1, 2, 0, 2, 0, 2, 0, %0, %0(tied-def 0), 0, %stack.0.alloca, 0, csr_64, implicit-def $rsp, implicit-def $ssp, implicit-def $al :: (volatile load store 8 on %stack.0.alloca)
+; CHECK-VREG:    %2:gr8 = COPY $al
+; CHECK-VREG:    %3:gr64 = MOV64rm %stack.0.alloca, 1, $noreg, 0, $noreg :: (dereferenceable load 8 from %ir.alloca)
+; CHECK-VREG:    $rdi = COPY %1
+; CHECK-VREG:    CALL64pcrel32 @consume, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit-def $rsp, implicit-def $ssp
+
+; CHECK-PREG-LABEL: name:            test_alloca
+; CHECK-PREG:    renamable $rbx = COPY $rdi
+; CHECK-PREG:    MOV64mr %stack.0.alloca, 1, $noreg, 0, $noreg, renamable $rbx :: (store 8 into %ir.alloca)
+; CHECK-PREG:    renamable $rbx = STATEPOINT 0, 0, 0, @return_i1, 2, 0, 2, 0, 2, 0, killed renamable $rbx, renamable $rbx(tied-def 0), 0, %stack.0.alloca, 0, csr_64, implicit-def $rsp, implicit-def $ssp, implicit-def dead $al :: (volatile load store 8 on %stack.0.alloca)
+; CHECK-PREG:    renamable $r14 = MOV64rm %stack.0.alloca, 1, $noreg, 0, $noreg :: (dereferenceable load 8 from %ir.alloca)
+; CHECK-PREG:    $rdi = COPY killed renamable $rbx
+; CHECK-PREG:    CALL64pcrel32 @consume, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit-def $rsp, implicit-def $ssp
+
+entry:
+  %alloca = alloca i32 addrspace(1)*, align 8
+  store i32 addrspace(1)* %ptr, i32 addrspace(1)** %alloca
+  %safepoint_token = call token (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0) ["gc-live" (i32 addrspace(1)** %alloca, i32 addrspace(1)* %ptr)]
+  %rel1 = load i32 addrspace(1)*, i32 addrspace(1)** %alloca
+  %rel2 = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 1, i32 1)
+  call void @consume(i32 addrspace(1)* %rel2)
+  ret i32 addrspace(1)* %rel1
+}
+
+; test base != derived
+define void @test_base_derived(i32 addrspace(1)* %base, i32 addrspace(1)* %derived) gc "statepoint-example" {
+; CHECK-VREG-LABEL: name:            test_base_derived
+; CHECK-VREG:    %1:gr64 = COPY $rsi
+; CHECK-VREG:    %0:gr64 = COPY $rdi
+; CHECK-VREG:    MOV64mr %stack.0, 1, $noreg, 0, $noreg, %0 :: (store 8 into %stack.0)
+; CHECK-VREG:    %2:gr64 = STATEPOINT 0, 0, 0, @func, 2, 0, 2, 0, 2, 0, 1, 8, %stack.0, 0, %1(tied-def 0), csr_64, implicit-def $rsp, implicit-def $ssp :: (volatile load store 8 on %stack.0)
+; CHECK-VREG:    $rdi = COPY %2
+; CHECK-VREG:    CALL64pcrel32 @consume, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit-def $rsp, implicit-def $ssp
+
+; CHECK-PREG-LABEL: name:            test_base_derived
+; CHECK-PREG:    renamable $rbx = COPY $rsi
+; CHECK-PREG:    MOV64mr %stack.0, 1, $noreg, 0, $noreg, killed renamable $rdi :: (store 8 into %stack.0)
+; CHECK-PREG:    renamable $rbx = STATEPOINT 0, 0, 0, @func, 2, 0, 2, 0, 2, 0, 1, 8, %stack.0, 0, killed renamable $rbx(tied-def 0), csr_64, implicit-def $rsp, implicit-def $ssp :: (volatile load store 8 on %stack.0)
+; CHECK-PREG:    $rdi = COPY killed renamable $rbx
+; CHECK-PREG:    CALL64pcrel32 @consume, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit-def $rsp, implicit-def $ssp
+
+  %safepoint_token = tail call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0) ["gc-live" (i32 addrspace(1)* %base, i32 addrspace(1)* %derived)]
+  %reloc = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 0, i32 1)
+  call void @consume(i32 addrspace(1)* %reloc)
+  ret void
+}
+
+; deopt GC pointer not present in GC args must be spilled
+define void @test_deopt_gcpointer(i32 addrspace(1)* %a, i32 addrspace(1)* %b) gc "statepoint-example" {
+; CHECK-VREG-LABEL: name:            test_deopt_gcpointer
+; CHECK-VREG:    %1:gr64 = COPY $rsi
+; CHECK-VREG:    %0:gr64 = COPY $rdi
+; CHECK-VREG:    MOV64mr %stack.0, 1, $noreg, 0, $noreg, %0 :: (store 8 into %stack.0)
+; CHECK-VREG:    %2:gr64 = STATEPOINT 0, 0, 0, @func, 2, 0, 2, 0, 2, 1, 1, 8, %stack.0, 0, %1, %1(tied-def 0), csr_64, implicit-def $rsp, implicit-def $ssp :: (volatile load store 8 on %stack.0)
+; CHECK-VREG:    $rdi = COPY %2
+; CHECK-VREG:    CALL64pcrel32 @consume, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit-def $rsp, implicit-def $ssp
+; CHECK-VREG:    RET 0
+
+; CHECK-PREG-LABEL: name:            test_deopt_gcpointer
+; CHECK-PREG:    renamable $rbx = COPY $rsi
+; CHECK-PREG:    MOV64mr %stack.0, 1, $noreg, 0, $noreg, killed renamable $rdi :: (store 8 into %stack.0)
+; CHECK-PREG:    renamable $rbx = STATEPOINT 0, 0, 0, @func, 2, 0, 2, 0, 2, 1, 1, 8, %stack.0, 0, killed renamable $rbx, renamable $rbx(tied-def 0), csr_64, implicit-def $rsp, implicit-def $ssp :: (volatile load store 8 on %stack.0)
+; CHECK-PREG:    $rdi = COPY killed renamable $rbx
+; CHECK-PREG:    CALL64pcrel32 @consume, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit-def $rsp, implicit-def $ssp
+
+  %safepoint_token = tail call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0) ["deopt" (i32 addrspace(1)* %a), "gc-live" (i32 addrspace(1)* %b)]
+  %rel = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 0, i32 0)
+  call void @consume(i32 addrspace(1)* %rel)
+  ret void
+}
+
+;; Two gc.relocates of the same input, should require only a single spill/fill
+define void @test_gcrelocate_uniqueing(i32 addrspace(1)* %ptr) gc "statepoint-example" {
+; CHECK-VREG-LABEL: name:            test_gcrelocate_uniqueing
+; CHECK-VREG:    %0:gr64 = COPY $rdi
+; CHECK-VREG:    %1:gr64 = STATEPOINT 0, 0, 0, @func, 2, 0, 2, 0, 2, 2, %0, 2, 4278124286, %0, %0(tied-def 0), csr_64, implicit-def $rsp, implicit-def $ssp
+; CHECK-VREG:    $rdi = COPY %1
+; CHECK-VREG:    $rsi = COPY %1
+; CHECK-VREG:    CALL64pcrel32 @consume2, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit $rsi, implicit-def $rsp, implicit-def $ssp
+
+; CHECK-PREG-LABEL: name:            test_gcrelocate_uniqueing
+; CHECK-PREG:    renamable $rbx = COPY $rdi
+; CHECK-PREG:    renamable $rbx = STATEPOINT 0, 0, 0, @func, 2, 0, 2, 0, 2, 2, killed renamable $rbx, 2, 4278124286, renamable $rbx, renamable $rbx(tied-def 0), csr_64, implicit-def $rsp, implicit-def $ssp
+; CHECK-PREG:    $rdi = COPY renamable $rbx
+; CHECK-PREG:    $rsi = COPY killed renamable $rbx
+; CHECK-PREG:    CALL64pcrel32 @consume2, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit killed $rsi, implicit-def $rsp, implicit-def $ssp
+
+  %tok = tail call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0) ["deopt" (i32 addrspace(1)* %ptr, i32 undef), "gc-live" (i32 addrspace(1)* %ptr, i32 addrspace(1)* %ptr)]
+  %a = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %tok, i32 0, i32 0)
+  %b = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %tok, i32 1, i32 1)
+  call void @consume2(i32 addrspace(1)* %a, i32 addrspace(1)* %b)
+  ret void
+}
+
+; Two gc.relocates of a bitcasted pointer should only require a single spill/fill
+define void @test_gcptr_uniqueing(i32 addrspace(1)* %ptr) gc "statepoint-example" {
+; CHECK-VREG-LABEL: name:            test_gcptr_uniqueing
+; CHECK-VREG:    %0:gr64 = COPY $rdi
+; CHECK-VREG:    ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+; CHECK-VREG:    %2:gr64 = STATEPOINT 0, 0, 0, @func, 2, 0, 2, 0, 2, 2, %0, 2, 4278124286, %0, %0(tied-def 0), csr_64, implicit-def $rsp, implicit-def $ssp
+; CHECK-VREG:    ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+; CHECK-VREG:    %1:gr64 = COPY %2
+; CHECK-VREG:    ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+; CHECK-VREG:    $rdi = COPY %2
+; CHECK-VREG:    $rsi = COPY %1
+; CHECK-VREG:    CALL64pcrel32 @use1, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit $rsi, implicit-def $rsp, implicit-def $ssp
+
+; CHECK-PREG-LABEL: name:            test_gcptr_uniqueing
+; CHECK-PREG:    renamable $rbx = COPY $rdi
+; CHECK-PREG:    renamable $rbx = STATEPOINT 0, 0, 0, @func, 2, 0, 2, 0, 2, 2, killed renamable $rbx, 2, 4278124286, renamable $rbx, renamable $rbx(tied-def 0), csr_64, implicit-def $rsp, implicit-def $ssp
+; CHECK-PREG:    $rdi = COPY renamable $rbx
+; CHECK-PREG:    $rsi = COPY killed renamable $rbx
+; CHECK-PREG:    CALL64pcrel32 @use1, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit killed $rsi, implicit-def $rsp, implicit-def $ssp
+
+  %ptr2 = bitcast i32 addrspace(1)* %ptr to i8 addrspace(1)*
+  %tok = tail call token (i64, i32, void ()*, i32, i32, ...)
+      @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0) ["deopt" (i32 addrspace(1)* %ptr, i32 undef), "gc-live" (i32 addrspace(1)* %ptr, i8 addrspace(1)* %ptr2)]
+  %a = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %tok, i32 0, i32 0)
+  %b = call i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %tok, i32 1, i32 1)
+  call void @use1(i32 addrspace(1)* %a, i8 addrspace(1)* %b)
+  ret void
+}
+
+;
+; Cross-basicblock relocates are handled with spilling for now.
+; No need to check post-RA output
+define i1 @test_cross_bb(i32 addrspace(1)* %a, i1 %external_cond) gc "statepoint-example" {
+; CHECK-VREG-LABEL: name:            test_cross_bb
+; CHECK-VREG:  bb.0.entry:
+; CHECK-VREG:         %1:gr32 = COPY $esi
+; CHECK-VREG-NEXT:    %0:gr64 = COPY $rdi
+; CHECK-VREG-NEXT:    %3:gr8 = COPY %1.sub_8bit
+; CHECK-VREG-NEXT:    MOV64mr %stack.0, 1, $noreg, 0, $noreg, %0 :: (store 8 into %stack.0)
+; CHECK-VREG-NEXT:    ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+; CHECK-VREG-NEXT:    STATEPOINT 0, 0, 0, @return_i1, 2, 0, 2, 0, 2, 0, 1, 8, %stack.0, 0, 1, 8, %stack.0, 0, csr_64, implicit-def $rsp, implicit-def $ssp, implicit-def $al :: (volatile load store 8 on %stack.0)
+; CHECK-VREG-NEXT:    ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+; CHECK-VREG-NEXT:    %4:gr8 = COPY $al
+; CHECK-VREG-NEXT:    %2:gr8 = COPY %4
+; CHECK-VREG-NEXT:    TEST8ri killed %3, 1, implicit-def $eflags
+; CHECK-VREG-NEXT:    JCC_1 %bb.2, 4, implicit $eflags
+; CHECK-VREG-NEXT:    JMP_1 %bb.1
+; CHECK-VREG:       bb.1.left:
+; CHECK-VREG-NEXT:    %6:gr64 = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load 8 from %stack.0)
+; CHECK-VREG-NEXT:    ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+; CHECK-VREG-NEXT:    $rdi = COPY %6
+; CHECK-VREG-NEXT:    CALL64pcrel32 @consume, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit-def $rsp, implicit-def $ssp
+; CHECK-VREG-NEXT:    ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+; CHECK-VREG-NEXT:    $al = COPY %2
+; CHECK-VREG-NEXT:    RET 0, $al
+; CHECK-VREG:       bb.2.right:
+; CHECK-VREG-NEXT:    %5:gr8 = MOV8ri 1
+; CHECK-VREG-NEXT:    $al = COPY %5
+; CHECK-VREG-NEXT:    RET 0, $al
+
+entry:
+  %safepoint_token = tail call token (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0) ["gc-live" (i32 addrspace(1)* %a)]
+  br i1 %external_cond, label %left, label %right
+
+left:
+  %call1 = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 0, i32 0)
+  %call2 = call zeroext i1 @llvm.experimental.gc.result.i1(token %safepoint_token)
+  call void @consume(i32 addrspace(1)* %call1)
+  ret i1 %call2
+
+right:
+  ret i1 true
+}
+
+; No need to check post-regalloc output as it is the same
+define i1 @duplicate_reloc() gc "statepoint-example" {
+; CHECK-VREG-LABEL: name:            duplicate_reloc
+; CHECK-VREG:  bb.0.entry:
+; CHECK-VREG:    STATEPOINT 0, 0, 0, @func, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, csr_64, implicit-def $rsp, implicit-def $ssp
+; CHECK-VREG:    STATEPOINT 0, 0, 0, @func, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, csr_64, implicit-def $rsp, implicit-def $ssp
+; CHECK-VREG:    %0:gr8 = MOV8ri 1
+; CHECK-VREG:    $al = COPY %0
+; CHECK-VREG:    RET 0, $al
+
+entry:
+  %safepoint_token = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0) ["gc-live" (i32 addrspace(1)* null, i32 addrspace(1)* null)]
+  %base = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 0, i32 0)
+  %derived = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 0, i32 1)
+  %safepoint_token2 = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0) ["gc-live" (i32 addrspace(1)* %base, i32 addrspace(1)* %derived)]
+  %base_reloc = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token2,  i32 0, i32 0)
+  %derived_reloc = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token2,  i32 0, i32 1)
+  %cmp1 = icmp eq i32 addrspace(1)* %base_reloc, null
+  %cmp2 = icmp eq i32 addrspace(1)* %derived_reloc, null
+  %cmp = and i1 %cmp1, %cmp2
+  ret i1 %cmp
+}
+
+; Vectors cannot go in VRegs
+; No need to check post-regalloc output as it is lowered using old scheme
+define <2 x i8 addrspace(1)*> @test_vector(<2 x i8 addrspace(1)*> %obj) gc "statepoint-example" {
+; CHECK-VREG-LABEL: name:            test_vector
+; CHECK-VREG:    %0:vr128 = COPY $xmm0
+; CHECK-VREG:    MOVAPSmr %stack.0, 1, $noreg, 0, $noreg, %0 :: (store 16 into %stack.0)
+; CHECK-VREG:    STATEPOINT 0, 0, 0, @func, 2, 0, 2, 0, 2, 0, 1, 16, %stack.0, 0, 1, 16, %stack.0, 0, csr_64, implicit-def $rsp, implicit-def $ssp :: (volatile load store 16 on %stack.0)
+; CHECK-VREG:    %1:vr128 = MOVAPSrm %stack.0, 1, $noreg, 0, $noreg :: (load 16 from %stack.0)
+; CHECK-VREG:    $xmm0 = COPY %1
+; CHECK-VREG:    RET 0, $xmm0
+
+entry:
+  %safepoint_token = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0) ["gc-live" (<2 x i8 addrspace(1)*> %obj)]
+  %obj.relocated = call coldcc <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token %safepoint_token, i32 0, i32 0) ; (%obj, %obj)
+  ret <2 x i8 addrspace(1)*> %obj.relocated
+}
+
+
+; test limit on amount of vregs
+define void @test_limit(i32 addrspace(1)* %a, i32 addrspace(1)* %b, i32 addrspace(1)* %c, i32 addrspace(1)* %d, i32 addrspace(1)*  %e) gc "statepoint-example" {
+; CHECK-VREG-LABEL: name:            test_limit
+; CHECK-VREG:    %4:gr64 = COPY $r8
+; CHECK-VREG:    %3:gr64 = COPY $rcx
+; CHECK-VREG:    %2:gr64 = COPY $rdx
+; CHECK-VREG:    %1:gr64 = COPY $rsi
+; CHECK-VREG:    %0:gr64 = COPY $rdi
+; CHECK-VREG:    MOV64mr %stack.0, 1, $noreg, 0, $noreg, %0 :: (store 8 into %stack.0)
+; CHECK-VREG:    %5:gr64, %6:gr64, %7:gr64, %8:gr64 = STATEPOINT 0, 0, 0, @func, 2, 0, 2, 0, 2, 0, %4, %4(tied-def 0), %3, %3(tied-def 1), %2, %2(tied-def 2), %1, %1(tied-def 3), 1, 8, %stack.0, 0, 1, 8, %stack.0, 0, csr_64, implicit-def $rsp, implicit-def $ssp :: (volatile load store 8 on %stack.0)
+; CHECK-VREG:    %9:gr64 = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load 8 from %stack.0)
+; CHECK-VREG:    $rdi = COPY %9
+; CHECK-VREG:    $rsi = COPY %8
+; CHECK-VREG:    $rdx = COPY %7
+; CHECK-VREG:    $rcx = COPY %6
+; CHECK-VREG:    $r8 = COPY %5
+; CHECK-VREG:    CALL64pcrel32 @consume5, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit $rsi, implicit $rdx, implicit $rcx, implicit $r8, implicit-def $rsp, implicit-def $ssp
+; CHECK-VREG:    RET 0
+entry:
+  %safepoint_token = tail call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0) ["gc-live" (i32 addrspace(1)* %a, i32 addrspace(1)* %b, i32 addrspace(1)* %c, i32 addrspace(1)* %d, i32 addrspace(1)* %e)]
+  %rel1 = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 0, i32 0)
+  %rel2 = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 1, i32 1)
+  %rel3 = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 2, i32 2)
+  %rel4 = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 3, i32 3)
+  %rel5 = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 4, i32 4)
+  call void @consume5(i32 addrspace(1)* %rel1, i32 addrspace(1)* %rel2, i32 addrspace(1)* %rel3, i32 addrspace(1)* %rel4, i32 addrspace(1)* %rel5)
+  ret void
+}
+
+declare token @llvm.experimental.gc.statepoint.p0f_i1f(i64, i32, i1 ()*, i32, i32, ...)
+declare token @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...)
+declare i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token, i32, i32)
+declare i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token, i32, i32)
+declare <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token, i32, i32)
+declare i1 @llvm.experimental.gc.result.i1(token)
+
+; Entry for test_relocate
+; Num locations
+; Location 1 Constant 0
+; Location 2 Constant 0
+; Location 3 Constant 0
+; Location 4 Register $rbx
+; Location 5 Register $rbx
+;  Entry for test_mixed
+; Num locations
+; Location 1 Constant 0
+; Location 2 Constant 0
+; Location 3 Constant 0
+; Location 4 Register $r14
+; Location 5 Register $r14
+; Location 6 Constant 0
+; Location 7 Constant 0
+; Location 8 Register $r15
+; Location 9 Register $r15
+; Location 10 Register $rbx
+; Location 11 Register $rbx
+; Entry for test_alloca
+; Num locations
+; Location 1 Constant 0
+; Location 2 Constant 0
+; Location 3 Constant 0
+; Location 4 Register $rbx
+; Location 5 Register $rbx
+; Location 6 Direct $rsp + 0
+; Entry for test_base_derive
+; Num locations
+; Location 1 Constant 0
+; Location 2 Constant 0
+; Location 3 Constant 0
+; Location 4 Indirect $rsp + 8
+; Location 5 Register $rbx
+; Entry for test_deopt_gcpointer
+; Num locations
+; Location 1 Constant 0
+; Location 2 Constant 0
+; Location 3 Constant 1
+; Location 4Indirect $rsp + 8
+; Location 5 Register $rbx
+; Location 6
+; Entry for test_gcrelocate_uniqueing
+; Num locations
+; Location 1 Constant 0
+; Location 2 Constant 0
+; Location 3 Constant 2
+; Location 4 Register $rbx
+; Location 5 Constant Index 0
+; Location 6 Register $rbx
+; Location 7 Register $rbx
+; Entry for test_gcptr_uniqueing
+; Num locations
+; Location 1 Constant 0
+; Location 2 Constant 0
+; Location 3 Constant 2
+; Location 4 Register $rbx
+; Location 5 Constant Index 0
+; Location 6 Register $rbx
+; Location 7 Register $rbx
+; Entry for test_cross_bb
+; Num locations
+; Location 1 Constant 0
+; Location 2 Constant 0
+; Location 3 Constant 0
+; Location 4 Indirect $rsp + 0
+; Location 5 Indirect $rsp + 0
diff --git a/llvm/test/CodeGen/X86/statepoint-vreg.ll b/llvm/test/CodeGen/X86/statepoint-vreg.ll
index 2d8dea4179a07..8c69af77c2f2e 100644
--- a/llvm/test/CodeGen/X86/statepoint-vreg.ll
+++ b/llvm/test/CodeGen/X86/statepoint-vreg.ll
@@ -1,10 +1,5 @@
-; This run is to demonstrate what MIR SSA looks like.
-; RUN: llc -max-registers-for-gc-values=4 -stop-after finalize-isel < %s | FileCheck --check-prefix=CHECK-VREG %s
-; This run is to demonstrate register allocator work.
-; RUN: llc -max-registers-for-gc-values=4 -stop-after virtregrewriter < %s | FileCheck --check-prefix=CHECK-PREG %s
-; This run is to demonstrate resulting assembly/stackmaps.
-; NOTE: When D81647 is landed this run line will need to be adjusted!
-; RUN: llc -max-registers-for-gc-values=4 < %s | FileCheck --check-prefix=CHECK-ASM %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -max-registers-for-gc-values=4 < %s | FileCheck %s
 
 target datalayout = "e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-pc-linux-gnu"
@@ -18,44 +13,30 @@ declare void @use1(i32 addrspace(1)*, i8 addrspace(1)*)
 
 ; test most simple relocate
 define i1 @test_relocate(i32 addrspace(1)* %a) gc "statepoint-example" {
-; CHECK-VREG-LABEL: name:            test_relocate
-; CHECK-VREG:    %0:gr64 = COPY $rdi
-; CHECK-VREG:    %1:gr64 = STATEPOINT 0, 0, 0, @return_i1, 2, 0, 2, 0, 2, 0, %0, %0(tied-def 0), csr_64, implicit-def $rsp, implicit-def $ssp, implicit-def $al
-; CHECK-VREG:    %2:gr8 = COPY $al
-; CHECK-VREG:    $rdi = COPY %1
-; CHECK-VREG:    CALL64pcrel32 @consume, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit-def $rsp, implicit-def $ssp
-
-; CHECK-PREG-LABEL: name:            test_relocate
-; CHECK-PREG:    renamable $rbx = COPY $rdi
-; CHECK-PREG:    renamable $rbx = STATEPOINT 0, 0, 0, @return_i1, 2, 0, 2, 0, 2, 0, killed renamable $rbx, renamable $rbx(tied-def 0), csr_64, implicit-def $rsp, implicit-def $ssp, implicit-def $al
-; CHECK-PREG:    renamable $bpl = COPY killed $al
-; CHECK-PREG:    $rdi = COPY killed renamable $rbx
-; CHECK-PREG:    CALL64pcrel32 @consume, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit-def $rsp, implicit-def $ssp
-
-; CHECK-ASM-LABEL: test_relocate:
-; CHECK-ASM:       # %bb.0:
-; CHECK-ASM-NEXT:	pushq	%rbp
-; CHECK-ASM-NEXT:	.cfi_def_cfa_offset 16
-; CHECK-ASM-NEXT:	pushq	%rbx
-; CHECK-ASM-NEXT:	.cfi_def_cfa_offset 24
-; CHECK-ASM-NEXT:	pushq	%rax
-; CHECK-ASM-NEXT:	.cfi_def_cfa_offset 32
-; CHECK-ASM-NEXT:	.cfi_offset %rbx, -24
-; CHECK-ASM-NEXT:	.cfi_offset %rbp, -16
-; CHECK-ASM-NEXT:	movq	%rdi, %rbx
-; CHECK-ASM-NEXT:	callq	return_i1
-; CHECK-ASM-NEXT:  .Ltmp0:
-; CHECK-ASM-NEXT:	movl	%eax, %ebp
-; CHECK-ASM-NEXT:	movq	%rbx, %rdi
-; CHECK-ASM-NEXT:	callq	consume
-; CHECK-ASM-NEXT:	movl	%ebp, %eax
-; CHECK-ASM-NEXT:	addq	$8, %rsp
-; CHECK-ASM-NEXT:	.cfi_def_cfa_offset 24
-; CHECK-ASM-NEXT:	popq	%rbx
-; CHECK-ASM-NEXT:	.cfi_def_cfa_offset 16
-; CHECK-ASM-NEXT:	popq	%rbp
-; CHECK-ASM-NEXT:	.cfi_def_cfa_offset 8
-; CHECK-ASM-NEXT:	retq
+; CHECK-LABEL: test_relocate:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    .cfi_offset %rbx, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq %rdi, %rbx
+; CHECK-NEXT:    callq return_i1
+; CHECK-NEXT:  .Ltmp0:
+; CHECK-NEXT:    movl %eax, %ebp
+; CHECK-NEXT:    movq %rbx, %rdi
+; CHECK-NEXT:    callq consume
+; CHECK-NEXT:    movl %ebp, %eax
+; CHECK-NEXT:    addq $8, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
 entry:
   %safepoint_token = tail call token (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0) ["gc-live" (i32 addrspace(1)* %a)]
   %rel1 = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 0, i32 0)
@@ -65,61 +46,35 @@ entry:
 }
 ; test pointer variables intermixed with pointer constants
 define void @test_mixed(i32 addrspace(1)* %a, i32 addrspace(1)* %b, i32 addrspace(1)* %c) gc "statepoint-example" {
-; CHECK-VREG-LABEL: name:            test_mixed
-; CHECK-VREG:    %2:gr64 = COPY $rdx
-; CHECK-VREG:    %1:gr64 = COPY $rsi
-; CHECK-VREG:    %0:gr64 = COPY $rdi
-; CHECK-VREG:    %3:gr64, %4:gr64, %5:gr64 = STATEPOINT 0, 0, 0, @func, 2, 0, 2, 0, 2, 0, %2, %2(tied-def 0), 2, 0, 2, 0, %1, %1(tied-def 1), %0, %0(tied-def 2), csr_64
-; CHECK-VREG:    %6:gr32 = MOV32r0 implicit-def dead $eflags
-; CHECK-VREG:    %7:gr64 = SUBREG_TO_REG 0, killed %6, %subreg.sub_32bit
-; CHECK-VREG:    $rdi = COPY %5
-; CHECK-VREG:    $rsi = COPY %7
-; CHECK-VREG:    $rdx = COPY %4
-; CHECK-VREG:    $rcx = COPY %7
-; CHECK-VREG:    $r8 = COPY %3
-; CHECK-VREG:    CALL64pcrel32 @consume5, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit $rsi, implicit $rdx, implicit $rcx, implicit $r8, implicit-def $rsp, implicit-def $ssp
-
-; CHECK-PREG-LABEL: name:            test_mixed
-; CHECK-PREG:    renamable $r14 = COPY $rdx
-; CHECK-PREG:    renamable $r15 = COPY $rsi
-; CHECK-PREG:    renamable $rbx = COPY $rdi
-; CHECK-PREG:    renamable $r14, renamable $r15, renamable $rbx = STATEPOINT 0, 0, 0, @func, 2, 0, 2, 0, 2, 0, killed renamable $r14, renamable $r14(tied-def 0), 2, 0, 2, 0, killed renamable $r15, renamable $r15(tied-def 1), killed renamable $rbx, renamable $rbx(tied-def 2), csr_64, implicit-def $rsp, implicit-def $ssp
-; CHECK-PREG:    $rdi = COPY killed renamable $rbx
-; CHECK-PREG:    dead $esi = MOV32r0 implicit-def dead $eflags, implicit-def $rsi
-; CHECK-PREG:    $rdx = COPY killed renamable $r15
-; CHECK-PREG:    dead $ecx = MOV32r0 implicit-def dead $eflags, implicit-def $rcx
-; CHECK-PREG:    $r8 = COPY killed renamable $r14
-; CHECK-PREG:    CALL64pcrel32 @consume5, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit $rsi, implicit $rdx, implicit killed $rcx, implicit killed $r8, implicit-def $rsp, implicit-def $ssp
-
-; CHECK-ASM-LABEL: test_mixed:
-; CHECK-ASM:        # %bb.0:                                # %entry
-; CHECK-ASM-NEXT:	pushq	%r15
-; CHECK-ASM-NEXT:	.cfi_def_cfa_offset 16
-; CHECK-ASM-NEXT:	pushq	%r14
-; CHECK-ASM-NEXT:	.cfi_def_cfa_offset 24
-; CHECK-ASM-NEXT:	pushq	%rbx
-; CHECK-ASM-NEXT:	.cfi_def_cfa_offset 32
-; CHECK-ASM-NEXT:	.cfi_offset %rbx, -32
-; CHECK-ASM-NEXT:	.cfi_offset %r14, -24
-; CHECK-ASM-NEXT:	.cfi_offset %r15, -16
-; CHECK-ASM-NEXT:	movq	%rdx, %r14
-; CHECK-ASM-NEXT:	movq	%rsi, %r15
-; CHECK-ASM-NEXT:	movq	%rdi, %rbx
-; CHECK-ASM-NEXT:	callq	func
-; CHECK-ASM-NEXT:.Ltmp1:
-; CHECK-ASM-NEXT:	movq	%rbx, %rdi
-; CHECK-ASM-NEXT:	xorl	%esi, %esi
-; CHECK-ASM-NEXT:	movq	%r15, %rdx
-; CHECK-ASM-NEXT:	xorl	%ecx, %ecx
-; CHECK-ASM-NEXT:	movq	%r14, %r8
-; CHECK-ASM-NEXT:	callq	consume5
-; CHECK-ASM-NEXT:	popq	%rbx
-; CHECK-ASM-NEXT:	.cfi_def_cfa_offset 24
-; CHECK-ASM-NEXT:	popq	%r14
-; CHECK-ASM-NEXT:	.cfi_def_cfa_offset 16
-; CHECK-ASM-NEXT:	popq	%r15
-; CHECK-ASM-NEXT:	.cfi_def_cfa_offset 8
-; CHECK-ASM-NEXT:	retq
+; CHECK-LABEL: test_mixed:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    .cfi_offset %rbx, -32
+; CHECK-NEXT:    .cfi_offset %r14, -24
+; CHECK-NEXT:    .cfi_offset %r15, -16
+; CHECK-NEXT:    movq %rdx, %r14
+; CHECK-NEXT:    movq %rsi, %r15
+; CHECK-NEXT:    movq %rdi, %rbx
+; CHECK-NEXT:    callq func
+; CHECK-NEXT:  .Ltmp1:
+; CHECK-NEXT:    movq %rbx, %rdi
+; CHECK-NEXT:    xorl %esi, %esi
+; CHECK-NEXT:    movq %r15, %rdx
+; CHECK-NEXT:    xorl %ecx, %ecx
+; CHECK-NEXT:    movq %r14, %r8
+; CHECK-NEXT:    callq consume5
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
 entry:
   %safepoint_token = tail call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0) ["gc-live" (i32 addrspace(1)* %a, i32 addrspace(1)* null, i32 addrspace(1)* %b, i32 addrspace(1)* null, i32 addrspace(1)* %c)]
   %rel1 = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 0, i32 0)
@@ -133,48 +88,31 @@ entry:
 
 ; same as above, but for alloca
 define i32 addrspace(1)* @test_alloca(i32 addrspace(1)* %ptr) gc "statepoint-example" {
-; CHECK-VREG-LABEL: name:            test_alloca
-; CHECK-VREG:    %0:gr64 = COPY $rdi
-; CHECK-VREG:    MOV64mr %stack.0.alloca, 1, $noreg, 0, $noreg, %0 :: (store 8 into %ir.alloca)
-; CHECK-VREG:    %1:gr64 = STATEPOINT 0, 0, 0, @return_i1, 2, 0, 2, 0, 2, 0, %0, %0(tied-def 0), 0, %stack.0.alloca, 0, csr_64, implicit-def $rsp, implicit-def $ssp, implicit-def $al :: (volatile load store 8 on %stack.0.alloca)
-; CHECK-VREG:    %2:gr8 = COPY $al
-; CHECK-VREG:    %3:gr64 = MOV64rm %stack.0.alloca, 1, $noreg, 0, $noreg :: (dereferenceable load 8 from %ir.alloca)
-; CHECK-VREG:    $rdi = COPY %1
-; CHECK-VREG:    CALL64pcrel32 @consume, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit-def $rsp, implicit-def $ssp
-
-; CHECK-PREG-LABEL: name:            test_alloca
-; CHECK-PREG:    renamable $rbx = COPY $rdi
-; CHECK-PREG:    MOV64mr %stack.0.alloca, 1, $noreg, 0, $noreg, renamable $rbx :: (store 8 into %ir.alloca)
-; CHECK-PREG:    renamable $rbx = STATEPOINT 0, 0, 0, @return_i1, 2, 0, 2, 0, 2, 0, killed renamable $rbx, renamable $rbx(tied-def 0), 0, %stack.0.alloca, 0, csr_64, implicit-def $rsp, implicit-def $ssp, implicit-def dead $al :: (volatile load store 8 on %stack.0.alloca)
-; CHECK-PREG:    renamable $r14 = MOV64rm %stack.0.alloca, 1, $noreg, 0, $noreg :: (dereferenceable load 8 from %ir.alloca)
-; CHECK-PREG:    $rdi = COPY killed renamable $rbx
-; CHECK-PREG:    CALL64pcrel32 @consume, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit-def $rsp, implicit-def $ssp
-
-; CHECK-ASM-LABEL: test_alloca:
-; CHECK-ASM:       # %bb.0: # %entry
-; CHECK-ASM-NEXT:	pushq	%r14
-; CHECK-ASM-NEXT:	.cfi_def_cfa_offset 16
-; CHECK-ASM-NEXT:	pushq	%rbx
-; CHECK-ASM-NEXT:	.cfi_def_cfa_offset 24
-; CHECK-ASM-NEXT:	pushq	%rax
-; CHECK-ASM-NEXT:	.cfi_def_cfa_offset 32
-; CHECK-ASM-NEXT:	.cfi_offset %rbx, -24
-; CHECK-ASM-NEXT:	.cfi_offset %r14, -16
-; CHECK-ASM-NEXT:	movq	%rdi, %rbx
-; CHECK-ASM-NEXT:	movq	%rdi, (%rsp)
-; CHECK-ASM-NEXT:	callq	return_i1
-; CHECK-ASM-NEXT:  .Ltmp2:
-; CHECK-ASM-NEXT:	movq	(%rsp), %r14
-; CHECK-ASM-NEXT:	movq	%rbx, %rdi
-; CHECK-ASM-NEXT:	callq	consume
-; CHECK-ASM-NEXT:	movq	%r14, %rax
-; CHECK-ASM-NEXT:	addq	$8, %rsp
-; CHECK-ASM-NEXT:	.cfi_def_cfa_offset 24
-; CHECK-ASM-NEXT:	popq	%rbx
-; CHECK-ASM-NEXT:	.cfi_def_cfa_offset 16
-; CHECK-ASM-NEXT:	popq	%r14
-; CHECK-ASM-NEXT:	.cfi_def_cfa_offset 8
-; CHECK-ASM-NEXT:	retq
+; CHECK-LABEL: test_alloca:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    .cfi_offset %rbx, -24
+; CHECK-NEXT:    .cfi_offset %r14, -16
+; CHECK-NEXT:    movq %rdi, %rbx
+; CHECK-NEXT:    movq %rdi, (%rsp)
+; CHECK-NEXT:    callq return_i1
+; CHECK-NEXT:  .Ltmp2:
+; CHECK-NEXT:    movq (%rsp), %r14
+; CHECK-NEXT:    movq %rbx, %rdi
+; CHECK-NEXT:    callq consume
+; CHECK-NEXT:    movq %r14, %rax
+; CHECK-NEXT:    addq $8, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
 entry:
   %alloca = alloca i32 addrspace(1)*, align 8
   store i32 addrspace(1)* %ptr, i32 addrspace(1)** %alloca
@@ -187,39 +125,24 @@ entry:
 
 ; test base != derived
 define void @test_base_derived(i32 addrspace(1)* %base, i32 addrspace(1)* %derived) gc "statepoint-example" {
-; CHECK-VREG-LABEL: name:            test_base_derived
-; CHECK-VREG:    %1:gr64 = COPY $rsi
-; CHECK-VREG:    %0:gr64 = COPY $rdi
-; CHECK-VREG:    MOV64mr %stack.0, 1, $noreg, 0, $noreg, %0 :: (store 8 into %stack.0)
-; CHECK-VREG:    %2:gr64 = STATEPOINT 0, 0, 0, @func, 2, 0, 2, 0, 2, 0, 1, 8, %stack.0, 0, %1(tied-def 0), csr_64, implicit-def $rsp, implicit-def $ssp :: (volatile load store 8 on %stack.0)
-; CHECK-VREG:    $rdi = COPY %2
-; CHECK-VREG:    CALL64pcrel32 @consume, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit-def $rsp, implicit-def $ssp
-
-; CHECK-PREG-LABEL: name:            test_base_derived
-; CHECK-PREG:    renamable $rbx = COPY $rsi
-; CHECK-PREG:    MOV64mr %stack.0, 1, $noreg, 0, $noreg, killed renamable $rdi :: (store 8 into %stack.0)
-; CHECK-PREG:    renamable $rbx = STATEPOINT 0, 0, 0, @func, 2, 0, 2, 0, 2, 0, 1, 8, %stack.0, 0, killed renamable $rbx(tied-def 0), csr_64, implicit-def $rsp, implicit-def $ssp :: (volatile load store 8 on %stack.0)
-; CHECK-PREG:    $rdi = COPY killed renamable $rbx
-; CHECK-PREG:    CALL64pcrel32 @consume, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit-def $rsp, implicit-def $ssp
-
-; CHECK-ASM-LABEL: test_base_derived:
-; CHECK-ASM:       # %bb.0:
-; CHECK-ASM-NEXT:	pushq	%rbx
-; CHECK-ASM-NEXT:	.cfi_def_cfa_offset 16
-; CHECK-ASM-NEXT:	subq	$16, %rsp
-; CHECK-ASM-NEXT:	.cfi_def_cfa_offset 32
-; CHECK-ASM-NEXT:	.cfi_offset %rbx, -16
-; CHECK-ASM-NEXT:	movq	%rsi, %rbx
-; CHECK-ASM-NEXT:	movq	%rdi, 8(%rsp)
-; CHECK-ASM-NEXT:	callq	func
-; CHECK-ASM-NEXT:  .Ltmp3:
-; CHECK-ASM-NEXT:	movq	%rbx, %rdi
-; CHECK-ASM-NEXT:	callq	consume
-; CHECK-ASM-NEXT:	addq	$16, %rsp
-; CHECK-ASM-NEXT:	.cfi_def_cfa_offset 16
-; CHECK-ASM-NEXT:	popq	%rbx
-; CHECK-ASM-NEXT:	.cfi_def_cfa_offset 8
-; CHECK-ASM-NEXT:	retq
+; CHECK-LABEL: test_base_derived:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    subq $16, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    .cfi_offset %rbx, -16
+; CHECK-NEXT:    movq %rsi, %rbx
+; CHECK-NEXT:    movq %rdi, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    callq func
+; CHECK-NEXT:  .Ltmp3:
+; CHECK-NEXT:    movq %rbx, %rdi
+; CHECK-NEXT:    callq consume
+; CHECK-NEXT:    addq $16, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %safepoint_token = tail call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0) ["gc-live" (i32 addrspace(1)* %base, i32 addrspace(1)* %derived)]
   %reloc = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 0, i32 1)
   call void @consume(i32 addrspace(1)* %reloc)
@@ -228,40 +151,24 @@ define void @test_base_derived(i32 addrspace(1)* %base, i32 addrspace(1)* %deriv
 
 ; deopt GC pointer not present in GC args must be spilled
 define void @test_deopt_gcpointer(i32 addrspace(1)* %a, i32 addrspace(1)* %b) gc "statepoint-example" {
-; CHECK-VREG-LABEL: name:            test_deopt_gcpointer
-; CHECK-VREG:    %1:gr64 = COPY $rsi
-; CHECK-VREG:    %0:gr64 = COPY $rdi
-; CHECK-VREG:    MOV64mr %stack.0, 1, $noreg, 0, $noreg, %0 :: (store 8 into %stack.0)
-; CHECK-VREG:    %2:gr64 = STATEPOINT 0, 0, 0, @func, 2, 0, 2, 0, 2, 1, 1, 8, %stack.0, 0, %1, %1(tied-def 0), csr_64, implicit-def $rsp, implicit-def $ssp :: (volatile load store 8 on %stack.0)
-; CHECK-VREG:    $rdi = COPY %2
-; CHECK-VREG:    CALL64pcrel32 @consume, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit-def $rsp, implicit-def $ssp
-; CHECK-VREG:    RET 0
-
-; CHECK-PREG-LABEL: name:            test_deopt_gcpointer
-; CHECK-PREG:    renamable $rbx = COPY $rsi
-; CHECK-PREG:    MOV64mr %stack.0, 1, $noreg, 0, $noreg, killed renamable $rdi :: (store 8 into %stack.0)
-; CHECK-PREG:    renamable $rbx = STATEPOINT 0, 0, 0, @func, 2, 0, 2, 0, 2, 1, 1, 8, %stack.0, 0, killed renamable $rbx, renamable $rbx(tied-def 0), csr_64, implicit-def $rsp, implicit-def $ssp :: (volatile load store 8 on %stack.0)
-; CHECK-PREG:    $rdi = COPY killed renamable $rbx
-; CHECK-PREG:    CALL64pcrel32 @consume, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit-def $rsp, implicit-def $ssp
-
-; CHECK-ASM-LABEL: test_deopt_gcpointer:
-; CHECK-ASM:       # %bb.0:
-; CHECK-ASM-NEXT:	pushq	%rbx
-; CHECK-ASM-NEXT:	.cfi_def_cfa_offset 16
-; CHECK-ASM-NEXT:	subq	$16, %rsp
-; CHECK-ASM-NEXT:	.cfi_def_cfa_offset 32
-; CHECK-ASM-NEXT:	.cfi_offset %rbx, -16
-; CHECK-ASM-NEXT:	movq	%rsi, %rbx
-; CHECK-ASM-NEXT:	movq	%rdi, 8(%rsp)
-; CHECK-ASM-NEXT:	callq	func
-; CHECK-ASM-NEXT:  .Ltmp4:
-; CHECK-ASM-NEXT:	movq	%rbx, %rdi
-; CHECK-ASM-NEXT:	callq	consume
-; CHECK-ASM-NEXT:	addq	$16, %rsp
-; CHECK-ASM-NEXT:	.cfi_def_cfa_offset 16
-; CHECK-ASM-NEXT:	popq	%rbx
-; CHECK-ASM-NEXT:	.cfi_def_cfa_offset 8
-; CHECK-ASM-NEXT:	retq
+; CHECK-LABEL: test_deopt_gcpointer:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    subq $16, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    .cfi_offset %rbx, -16
+; CHECK-NEXT:    movq %rsi, %rbx
+; CHECK-NEXT:    movq %rdi, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    callq func
+; CHECK-NEXT:  .Ltmp4:
+; CHECK-NEXT:    movq %rbx, %rdi
+; CHECK-NEXT:    callq consume
+; CHECK-NEXT:    addq $16, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %safepoint_token = tail call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0) ["deopt" (i32 addrspace(1)* %a), "gc-live" (i32 addrspace(1)* %b)]
   %rel = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 0, i32 0)
   call void @consume(i32 addrspace(1)* %rel)
@@ -270,34 +177,20 @@ define void @test_deopt_gcpointer(i32 addrspace(1)* %a, i32 addrspace(1)* %b) gc
 
 ;; Two gc.relocates of the same input, should require only a single spill/fill
 define void @test_gcrelocate_uniqueing(i32 addrspace(1)* %ptr) gc "statepoint-example" {
-; CHECK-VREG-LABEL: name:            test_gcrelocate_uniqueing
-; CHECK-VREG:    %0:gr64 = COPY $rdi
-; CHECK-VREG:    %1:gr64 = STATEPOINT 0, 0, 0, @func, 2, 0, 2, 0, 2, 2, %0, 2, 4278124286, %0, %0(tied-def 0), csr_64, implicit-def $rsp, implicit-def $ssp
-; CHECK-VREG:    $rdi = COPY %1
-; CHECK-VREG:    $rsi = COPY %1
-; CHECK-VREG:    CALL64pcrel32 @consume2, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit $rsi, implicit-def $rsp, implicit-def $ssp
-
-; CHECK-PREG-LABEL: name:            test_gcrelocate_uniqueing
-; CHECK-PREG:    renamable $rbx = COPY $rdi
-; CHECK-PREG:    renamable $rbx = STATEPOINT 0, 0, 0, @func, 2, 0, 2, 0, 2, 2, killed renamable $rbx, 2, 4278124286, renamable $rbx, renamable $rbx(tied-def 0), csr_64, implicit-def $rsp, implicit-def $ssp
-; CHECK-PREG:    $rdi = COPY renamable $rbx
-; CHECK-PREG:    $rsi = COPY killed renamable $rbx
-; CHECK-PREG:    CALL64pcrel32 @consume2, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit killed $rsi, implicit-def $rsp, implicit-def $ssp
-
-; CHECK-ASM-LABEL: test_gcrelocate_uniqueing:
-; CHECK-ASM:       # %bb.0:
-; CHECK-ASM-NEXT:	pushq	%rbx
-; CHECK-ASM-NEXT:	.cfi_def_cfa_offset 16
-; CHECK-ASM-NEXT:	.cfi_offset %rbx, -16
-; CHECK-ASM-NEXT:	movq	%rdi, %rbx
-; CHECK-ASM-NEXT:	callq	func
-; CHECK-ASM-NEXT: .Ltmp5:
-; CHECK-ASM-NEXT:	movq	%rbx, %rdi
-; CHECK-ASM-NEXT:	movq	%rbx, %rsi
-; CHECK-ASM-NEXT:	callq	consume2
-; CHECK-ASM-NEXT:	popq	%rbx
-; CHECK-ASM-NEXT:	.cfi_def_cfa_offset 8
-; CHECK-ASM-NEXT:	retq
+; CHECK-LABEL: test_gcrelocate_uniqueing:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset %rbx, -16
+; CHECK-NEXT:    movq %rdi, %rbx
+; CHECK-NEXT:    callq func
+; CHECK-NEXT:  .Ltmp5:
+; CHECK-NEXT:    movq %rbx, %rdi
+; CHECK-NEXT:    movq %rbx, %rsi
+; CHECK-NEXT:    callq consume2
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %tok = tail call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0) ["deopt" (i32 addrspace(1)* %ptr, i32 undef), "gc-live" (i32 addrspace(1)* %ptr, i32 addrspace(1)* %ptr)]
   %a = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %tok, i32 0, i32 0)
   %b = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %tok, i32 1, i32 1)
@@ -307,38 +200,20 @@ define void @test_gcrelocate_uniqueing(i32 addrspace(1)* %ptr) gc "statepoint-ex
 
 ; Two gc.relocates of a bitcasted pointer should only require a single spill/fill
 define void @test_gcptr_uniqueing(i32 addrspace(1)* %ptr) gc "statepoint-example" {
-; CHECK-VREG-LABEL: name:            test_gcptr_uniqueing
-; CHECK-VREG:    %0:gr64 = COPY $rdi
-; CHECK-VREG:    ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
-; CHECK-VREG:    %2:gr64 = STATEPOINT 0, 0, 0, @func, 2, 0, 2, 0, 2, 2, %0, 2, 4278124286, %0, %0(tied-def 0), csr_64, implicit-def $rsp, implicit-def $ssp
-; CHECK-VREG:    ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
-; CHECK-VREG:    %1:gr64 = COPY %2
-; CHECK-VREG:    ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
-; CHECK-VREG:    $rdi = COPY %2
-; CHECK-VREG:    $rsi = COPY %1
-; CHECK-VREG:    CALL64pcrel32 @use1, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit $rsi, implicit-def $rsp, implicit-def $ssp
-
-; CHECK-PREG-LABEL: name:            test_gcptr_uniqueing
-; CHECK-PREG:    renamable $rbx = COPY $rdi
-; CHECK-PREG:    renamable $rbx = STATEPOINT 0, 0, 0, @func, 2, 0, 2, 0, 2, 2, killed renamable $rbx, 2, 4278124286, renamable $rbx, renamable $rbx(tied-def 0), csr_64, implicit-def $rsp, implicit-def $ssp
-; CHECK-PREG:    $rdi = COPY renamable $rbx
-; CHECK-PREG:    $rsi = COPY killed renamable $rbx
-; CHECK-PREG:    CALL64pcrel32 @use1, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit killed $rsi, implicit-def $rsp, implicit-def $ssp
-
-; CHECK-ASM-LABEL: test_gcptr_uniqueing:
-; CHECK-ASM:       # %bb.0:
-; CHECK-ASM-NEXT:	pushq	%rbx
-; CHECK-ASM-NEXT:	.cfi_def_cfa_offset 16
-; CHECK-ASM-NEXT:	.cfi_offset %rbx, -16
-; CHECK-ASM-NEXT:	movq	%rdi, %rbx
-; CHECK-ASM-NEXT:	callq	func
-; CHECK-ASM-NEXT:  .Ltmp6:
-; CHECK-ASM-NEXT:	movq	%rbx, %rdi
-; CHECK-ASM-NEXT:	movq	%rbx, %rsi
-; CHECK-ASM-NEXT:	callq	use1
-; CHECK-ASM-NEXT:	popq	%rbx
-; CHECK-ASM-NEXT:	.cfi_def_cfa_offset 8
-; CHECK-ASM-NEXT:	retq
+; CHECK-LABEL: test_gcptr_uniqueing:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset %rbx, -16
+; CHECK-NEXT:    movq %rdi, %rbx
+; CHECK-NEXT:    callq func
+; CHECK-NEXT:  .Ltmp6:
+; CHECK-NEXT:    movq %rbx, %rdi
+; CHECK-NEXT:    movq %rbx, %rsi
+; CHECK-NEXT:    callq use1
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %ptr2 = bitcast i32 addrspace(1)* %ptr to i8 addrspace(1)*
   %tok = tail call token (i64, i32, void ()*, i32, i32, ...)
       @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0) ["deopt" (i32 addrspace(1)* %ptr, i32 undef), "gc-live" (i32 addrspace(1)* %ptr, i8 addrspace(1)* %ptr2)]
@@ -352,33 +227,38 @@ define void @test_gcptr_uniqueing(i32 addrspace(1)* %ptr) gc "statepoint-example
 ; Cross-basicblock relocates are handled with spilling for now.
 ; No need to check post-RA output
 define i1 @test_cross_bb(i32 addrspace(1)* %a, i1 %external_cond) gc "statepoint-example" {
-; CHECK-VREG-LABEL: name:            test_cross_bb
-; CHECK-VREG:  bb.0.entry:
-; CHECK-VREG:         %1:gr32 = COPY $esi
-; CHECK-VREG-NEXT:    %0:gr64 = COPY $rdi
-; CHECK-VREG-NEXT:    %3:gr8 = COPY %1.sub_8bit
-; CHECK-VREG-NEXT:    MOV64mr %stack.0, 1, $noreg, 0, $noreg, %0 :: (store 8 into %stack.0)
-; CHECK-VREG-NEXT:    ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
-; CHECK-VREG-NEXT:    STATEPOINT 0, 0, 0, @return_i1, 2, 0, 2, 0, 2, 0, 1, 8, %stack.0, 0, 1, 8, %stack.0, 0, csr_64, implicit-def $rsp, implicit-def $ssp, implicit-def $al :: (volatile load store 8 on %stack.0)
-; CHECK-VREG-NEXT:    ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
-; CHECK-VREG-NEXT:    %4:gr8 = COPY $al
-; CHECK-VREG-NEXT:    %2:gr8 = COPY %4
-; CHECK-VREG-NEXT:    TEST8ri killed %3, 1, implicit-def $eflags
-; CHECK-VREG-NEXT:    JCC_1 %bb.2, 4, implicit $eflags
-; CHECK-VREG-NEXT:    JMP_1 %bb.1
-; CHECK-VREG:       bb.1.left:
-; CHECK-VREG-NEXT:    %6:gr64 = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load 8 from %stack.0)
-; CHECK-VREG-NEXT:    ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
-; CHECK-VREG-NEXT:    $rdi = COPY %6
-; CHECK-VREG-NEXT:    CALL64pcrel32 @consume, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit-def $rsp, implicit-def $ssp
-; CHECK-VREG-NEXT:    ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
-; CHECK-VREG-NEXT:    $al = COPY %2
-; CHECK-VREG-NEXT:    RET 0, $al
-; CHECK-VREG:       bb.2.right:
-; CHECK-VREG-NEXT:    %5:gr8 = MOV8ri 1
-; CHECK-VREG-NEXT:    $al = COPY %5
-; CHECK-VREG-NEXT:    RET 0, $al
-
+; CHECK-LABEL: test_cross_bb:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    .cfi_offset %rbx, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movl %esi, %ebp
+; CHECK-NEXT:    movq %rdi, (%rsp)
+; CHECK-NEXT:    callq return_i1
+; CHECK-NEXT:  .Ltmp7:
+; CHECK-NEXT:    testb $1, %bpl
+; CHECK-NEXT:    je .LBB7_2
+; CHECK-NEXT:  # %bb.1: # %left
+; CHECK-NEXT:    movl %eax, %ebx
+; CHECK-NEXT:    movq (%rsp), %rdi
+; CHECK-NEXT:    callq consume
+; CHECK-NEXT:    movl %ebx, %eax
+; CHECK-NEXT:    jmp .LBB7_3
+; CHECK-NEXT:  .LBB7_2: # %right
+; CHECK-NEXT:    movb $1, %al
+; CHECK-NEXT:  .LBB7_3: # %right
+; CHECK-NEXT:    addq $8, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
 entry:
   %safepoint_token = tail call token (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0) ["gc-live" (i32 addrspace(1)* %a)]
   br i1 %external_cond, label %left, label %right
@@ -395,26 +275,18 @@ right:
 
 ; No need to check post-regalloc output as it is the same
 define i1 @duplicate_reloc() gc "statepoint-example" {
-; CHECK-VREG-LABEL: name:            duplicate_reloc
-; CHECK-VREG:  bb.0.entry:
-; CHECK-VREG:    STATEPOINT 0, 0, 0, @func, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, csr_64, implicit-def $rsp, implicit-def $ssp
-; CHECK-VREG:    STATEPOINT 0, 0, 0, @func, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, csr_64, implicit-def $rsp, implicit-def $ssp
-; CHECK-VREG:    %0:gr8 = MOV8ri 1
-; CHECK-VREG:    $al = COPY %0
-; CHECK-VREG:    RET 0, $al
-
-; CHECK-ASM-LABEL: duplicate_reloc:
-; CHECK-ASM:       # %bb.0: # %entry
-; CHECK-ASM-NEXT:          pushq	%rax
-; CHECK-ASM-NEXT:          .cfi_def_cfa_offset 16
-; CHECK-ASM-NEXT:          callq	func
-; CHECK-ASM-NEXT:  .Ltmp8:
-; CHECK-ASM-NEXT:          callq	func
-; CHECK-ASM-NEXT:  .Ltmp9:
-; CHECK-ASM-NEXT:          movb	$1, %al
-; CHECK-ASM-NEXT:          popq	%rcx
-; CHECK-ASM-NEXT:          .cfi_def_cfa_offset 8
-; CHECK-ASM-NEXT:          retq
+; CHECK-LABEL: duplicate_reloc:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    callq func
+; CHECK-NEXT:  .Ltmp8:
+; CHECK-NEXT:    callq func
+; CHECK-NEXT:  .Ltmp9:
+; CHECK-NEXT:    movb $1, %al
+; CHECK-NEXT:    popq %rcx
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
 entry:
   %safepoint_token = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0) ["gc-live" (i32 addrspace(1)* null, i32 addrspace(1)* null)]
   %base = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 0, i32 0)
@@ -431,25 +303,17 @@ entry:
 ; Vectors cannot go in VRegs
 ; No need to check post-regalloc output as it is lowered using old scheme
 define <2 x i8 addrspace(1)*> @test_vector(<2 x i8 addrspace(1)*> %obj) gc "statepoint-example" {
-; CHECK-VREG-LABEL: name:            test_vector
-; CHECK-VREG:    %0:vr128 = COPY $xmm0
-; CHECK-VREG:    MOVAPSmr %stack.0, 1, $noreg, 0, $noreg, %0 :: (store 16 into %stack.0)
-; CHECK-VREG:    STATEPOINT 0, 0, 0, @func, 2, 0, 2, 0, 2, 0, 1, 16, %stack.0, 0, 1, 16, %stack.0, 0, csr_64, implicit-def $rsp, implicit-def $ssp :: (volatile load store 16 on %stack.0)
-; CHECK-VREG:    %1:vr128 = MOVAPSrm %stack.0, 1, $noreg, 0, $noreg :: (load 16 from %stack.0)
-; CHECK-VREG:    $xmm0 = COPY %1
-; CHECK-VREG:    RET 0, $xmm0
-
-; CHECK-ASM-LABEL: test_vector:
-; CHECK-ASM:       # %bb.0: # %entry
-; CHECK-ASM-NEXT:          subq	$24, %rsp
-; CHECK-ASM-NEXT:          .cfi_def_cfa_offset 32
-; CHECK-ASM-NEXT:          movaps	%xmm0, (%rsp)
-; CHECK-ASM-NEXT:          callq	func
-; CHECK-ASM-NEXT:  .Ltmp10:
-; CHECK-ASM-NEXT:          movaps	(%rsp), %xmm0
-; CHECK-ASM-NEXT:          addq	$24, %rsp
-; CHECK-ASM-NEXT:          .cfi_def_cfa_offset 8
-; CHECK-ASM-NEXT:          retq
+; CHECK-LABEL: test_vector:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    subq $24, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    movaps %xmm0, (%rsp)
+; CHECK-NEXT:    callq func
+; CHECK-NEXT:  .Ltmp10:
+; CHECK-NEXT:    movaps (%rsp), %xmm0
+; CHECK-NEXT:    addq $24, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
 entry:
   %safepoint_token = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0) ["gc-live" (<2 x i8 addrspace(1)*> %obj)]
   %obj.relocated = call coldcc <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token %safepoint_token, i32 0, i32 0) ; (%obj, %obj)
@@ -459,22 +323,46 @@ entry:
 
 ; test limit on amount of vregs
 define void @test_limit(i32 addrspace(1)* %a, i32 addrspace(1)* %b, i32 addrspace(1)* %c, i32 addrspace(1)* %d, i32 addrspace(1)*  %e) gc "statepoint-example" {
-; CHECK-VREG-LABEL: name:            test_limit
-; CHECK-VREG:    %4:gr64 = COPY $r8
-; CHECK-VREG:    %3:gr64 = COPY $rcx
-; CHECK-VREG:    %2:gr64 = COPY $rdx
-; CHECK-VREG:    %1:gr64 = COPY $rsi
-; CHECK-VREG:    %0:gr64 = COPY $rdi
-; CHECK-VREG:    MOV64mr %stack.0, 1, $noreg, 0, $noreg, %0 :: (store 8 into %stack.0)
-; CHECK-VREG:    %5:gr64, %6:gr64, %7:gr64, %8:gr64 = STATEPOINT 0, 0, 0, @func, 2, 0, 2, 0, 2, 0, %4, %4(tied-def 0), %3, %3(tied-def 1), %2, %2(tied-def 2), %1, %1(tied-def 3), 1, 8, %stack.0, 0, 1, 8, %stack.0, 0, csr_64, implicit-def $rsp, implicit-def $ssp :: (volatile load store 8 on %stack.0)
-; CHECK-VREG:    %9:gr64 = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load 8 from %stack.0)
-; CHECK-VREG:    $rdi = COPY %9
-; CHECK-VREG:    $rsi = COPY %8
-; CHECK-VREG:    $rdx = COPY %7
-; CHECK-VREG:    $rcx = COPY %6
-; CHECK-VREG:    $r8 = COPY %5
-; CHECK-VREG:    CALL64pcrel32 @consume5, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit $rsi, implicit $rdx, implicit $rcx, implicit $r8, implicit-def $rsp, implicit-def $ssp
-; CHECK-VREG:    RET 0
+; CHECK-LABEL: test_limit:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    .cfi_offset %rbx, -40
+; CHECK-NEXT:    .cfi_offset %r12, -32
+; CHECK-NEXT:    .cfi_offset %r14, -24
+; CHECK-NEXT:    .cfi_offset %r15, -16
+; CHECK-NEXT:    movq %r8, %r14
+; CHECK-NEXT:    movq %rcx, %r15
+; CHECK-NEXT:    movq %rdx, %r12
+; CHECK-NEXT:    movq %rsi, %rbx
+; CHECK-NEXT:    movq %rdi, (%rsp)
+; CHECK-NEXT:    callq func
+; CHECK-NEXT:  .Ltmp11:
+; CHECK-NEXT:    movq (%rsp), %rdi
+; CHECK-NEXT:    movq %rbx, %rsi
+; CHECK-NEXT:    movq %r12, %rdx
+; CHECK-NEXT:    movq %r15, %rcx
+; CHECK-NEXT:    movq %r14, %r8
+; CHECK-NEXT:    callq consume5
+; CHECK-NEXT:    addq $8, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
 entry:
   %safepoint_token = tail call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0) ["gc-live" (i32 addrspace(1)* %a, i32 addrspace(1)* %b, i32 addrspace(1)* %c, i32 addrspace(1)* %d, i32 addrspace(1)* %e)]
   %rel1 = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 0, i32 0)
@@ -492,417 +380,3 @@ declare i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token, i32, i32)
 declare i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token, i32, i32)
 declare <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token, i32, i32)
 declare i1 @llvm.experimental.gc.result.i1(token)
-
-; CHECK-ASM-LABEL: .section .llvm_stackmaps
-; CHECK-ASM-NEXT:  __LLVM_StackMaps:
-; Entry for test_relocate
-; CHECK-ASM:	        .quad	0
-; CHECK-ASM-NEXT:     	.long	.Ltmp0-test_relocate
-; CHECK-ASM-NEXT:	.short	0
-; Num locations
-; CHECK-ASM-NEXT:	.short	5
-; Location 1 Constant 0
-; CHECK-ASM-NEXT:	.byte	4
-; CHECK-ASM-NEXT:	.byte	0
-; CHECK-ASM-NEXT:	.short	8
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.long	0
-; Location 2 Constant 0
-; CHECK-ASM-NEXT:	.byte	4
-; CHECK-ASM-NEXT:	.byte	0
-; CHECK-ASM-NEXT:	.short	8
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.long	0
-; Location 3 Constant 0
-; CHECK-ASM-NEXT:	.byte	4
-; CHECK-ASM-NEXT:	.byte	0
-; CHECK-ASM-NEXT:	.short	8
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.long	0
-; Location 4 Register $rbx
-; CHECK-ASM-NEXT:	.byte	1
-; CHECK-ASM-NEXT:	.byte	0
-; CHECK-ASM-NEXT:	.short	8
-; CHECK-ASM-NEXT:	.short	3
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.long	0
-; Location 5 Register $rbx
-; CHECK-ASM-NEXT:	.byte	1
-; CHECK-ASM-NEXT:	.byte	0
-; CHECK-ASM-NEXT:	.short	8
-; CHECK-ASM-NEXT:	.short	3
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.long	0
-;  Entry for test_mixed
-; CHECK-ASM:     	.quad	0
-; CHECK-ASM-NEXT:	.long	.Ltmp1-test_mixed
-; CHECK-ASM-NEXT:	.short	0
-; Num locations
-; CHECK-ASM-NEXT:	.short	11
-; Location 1 Constant 0
-; CHECK-ASM-NEXT:	.byte	4
-; CHECK-ASM-NEXT:	.byte	0
-; CHECK-ASM-NEXT:	.short	8
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.long	0
-; Location 2 Constant 0
-; CHECK-ASM-NEXT:	.byte	4
-; CHECK-ASM-NEXT:	.byte	0
-; CHECK-ASM-NEXT:	.short	8
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.long	0
-; Location 3 Constant 0
-; CHECK-ASM-NEXT:	.byte	4
-; CHECK-ASM-NEXT:	.byte	0
-; CHECK-ASM-NEXT:	.short	8
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.long	0
-; Location 4 Register $r14
-; CHECK-ASM-NEXT:	.byte	1
-; CHECK-ASM-NEXT:	.byte	0
-; CHECK-ASM-NEXT:	.short	8
-; CHECK-ASM-NEXT:	.short	14
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.long	0
-; Location 5 Register $r14
-; CHECK-ASM-NEXT:	.byte	1
-; CHECK-ASM-NEXT:	.byte	0
-; CHECK-ASM-NEXT:	.short	8
-; CHECK-ASM-NEXT:	.short	14
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.long	0
-; Location 6 Constant 0
-; CHECK-ASM-NEXT:	.byte	4
-; CHECK-ASM-NEXT:	.byte	0
-; CHECK-ASM-NEXT:	.short	8
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.long	0
-; Location 7 Constant 0
-; CHECK-ASM-NEXT:	.byte	4
-; CHECK-ASM-NEXT:	.byte	0
-; CHECK-ASM-NEXT:	.short	8
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.long	0
-; Location 8 Register $r15
-; CHECK-ASM-NEXT:	.byte	1
-; CHECK-ASM-NEXT:	.byte	0
-; CHECK-ASM-NEXT:	.short	8
-; CHECK-ASM-NEXT:	.short	15
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.long	0
-; Location 9 Register $r15
-; CHECK-ASM-NEXT:	.byte	1
-; CHECK-ASM-NEXT:	.byte	0
-; CHECK-ASM-NEXT:	.short	8
-; CHECK-ASM-NEXT:	.short	15
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.long	0
-; Location 10 Register $rbx
-; CHECK-ASM-NEXT:	.byte	1
-; CHECK-ASM-NEXT:	.byte	0
-; CHECK-ASM-NEXT:	.short	8
-; CHECK-ASM-NEXT:	.short	3
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.long	0
-; Location 11 Register $rbx
-; CHECK-ASM-NEXT:	.byte	1
-; CHECK-ASM-NEXT:	.byte	0
-; CHECK-ASM-NEXT:	.short	8
-; CHECK-ASM-NEXT:	.short	3
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.long	0
-; Entry for test_alloca
-; CHECK-ASM:     	.quad	0
-; CHECK-ASM-NEXT:	.long	.Ltmp2-test_alloca
-; CHECK-ASM-NEXT:	.short	0
-; Num locations
-; CHECK-ASM-NEXT:	.short	6
-; Location 1 Constant 0
-; CHECK-ASM-NEXT:	.byte	4
-; CHECK-ASM-NEXT:	.byte	0
-; CHECK-ASM-NEXT:	.short	8
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.long	0
-; Location 2 Constant 0
-; CHECK-ASM-NEXT:	.byte	4
-; CHECK-ASM-NEXT:	.byte	0
-; CHECK-ASM-NEXT:	.short	8
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.long	0
-; Location 3 Constant 0
-; CHECK-ASM-NEXT:	.byte	4
-; CHECK-ASM-NEXT:	.byte	0
-; CHECK-ASM-NEXT:	.short	8
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.long	0
-; Location 4 Register $rbx
-; CHECK-ASM-NEXT:	.byte	1
-; CHECK-ASM-NEXT:	.byte	0
-; CHECK-ASM-NEXT:	.short	8
-; CHECK-ASM-NEXT:	.short	3
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.long	0
-; Location 5 Register $rbx
-; CHECK-ASM-NEXT:	.byte	1
-; CHECK-ASM-NEXT:	.byte	0
-; CHECK-ASM-NEXT:	.short	8
-; CHECK-ASM-NEXT:	.short	3
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.long	0
-; Location 6 Direct $rsp + 0
-; CHECK-ASM-NEXT:	.byte	2
-; CHECK-ASM-NEXT:	.byte	0
-; CHECK-ASM-NEXT:	.short	8
-; CHECK-ASM-NEXT:	.short	7
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.long	0
-; Entry for test_base_derive
-; CHECK-ASM:     	.quad	0
-; CHECK-ASM-NEXT:	.long	.Ltmp3-test_base_derived
-; CHECK-ASM-NEXT:	.short	0
-; Num locations
-; CHECK-ASM-NEXT:	.short	5
-; Location 1 Constant 0
-; CHECK-ASM-NEXT:	.byte	4
-; CHECK-ASM-NEXT:	.byte	0
-; CHECK-ASM-NEXT:	.short	8
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.long	0
-; Location 2 Constant 0
-; CHECK-ASM-NEXT:	.byte	4
-; CHECK-ASM-NEXT:	.byte	0
-; CHECK-ASM-NEXT:	.short	8
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.long	0
-; Location 3 Constant 0
-; CHECK-ASM-NEXT:	.byte	4
-; CHECK-ASM-NEXT:	.byte	0
-; CHECK-ASM-NEXT:	.short	8
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.long	0
-; Location 4 Indirect $rsp + 8
-; CHECK-ASM-NEXT:	.byte	3
-; CHECK-ASM-NEXT:	.byte	0
-; CHECK-ASM-NEXT:	.short	8
-; CHECK-ASM-NEXT:	.short	7
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.long	8
-; Location 5 Register $rbx
-; CHECK-ASM-NEXT:	.byte	1
-; CHECK-ASM-NEXT:	.byte	0
-; CHECK-ASM-NEXT:	.short	8
-; CHECK-ASM-NEXT:	.short	3
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.long	0
-; Entry for test_deopt_gcpointer
-; CHECK-ASM:     	.quad	0
-; CHECK-ASM-NEXT:	.long	.Ltmp4-test_deopt_gcpointer
-; CHECK-ASM-NEXT:	.short	0
-; Num locations
-; CHECK-ASM-NEXT:	.short	6
-; Location 1 Constant 0
-; CHECK-ASM-NEXT:	.byte	4
-; CHECK-ASM-NEXT:	.byte	0
-; CHECK-ASM-NEXT:	.short	8
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.long	0
-; Location 2 Constant 0
-; CHECK-ASM-NEXT:	.byte	4
-; CHECK-ASM-NEXT:	.byte	0
-; CHECK-ASM-NEXT:	.short	8
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.long	0
-; Location 3 Constant 1
-; CHECK-ASM-NEXT:	.byte	4
-; CHECK-ASM-NEXT:	.byte	0
-; CHECK-ASM-NEXT:	.short	8
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.long	1
-; Location 4Indirect $rsp + 8
-; CHECK-ASM-NEXT:	.byte	3
-; CHECK-ASM-NEXT:	.byte	0
-; CHECK-ASM-NEXT:	.short	8
-; CHECK-ASM-NEXT:	.short	7
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.long	8
-; Location 5 Register $rbx
-; CHECK-ASM-NEXT:	.byte	1
-; CHECK-ASM-NEXT:	.byte	0
-; CHECK-ASM-NEXT:	.short	8
-; CHECK-ASM-NEXT:	.short	3
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.long	0
-; Location 6
-; CHECK-ASM-NEXT:	.byte	1
-; CHECK-ASM-NEXT:	.byte	0
-; CHECK-ASM-NEXT:	.short	8
-; CHECK-ASM-NEXT:	.short	3
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.long	0
-; Entry for test_gcrelocate_uniqueing
-; CHECK-ASM:     	.quad	0
-; CHECK-ASM-NEXT:	.long	.Ltmp5-test_gcrelocate_uniqueing
-; CHECK-ASM-NEXT:	.short	0
-; Num locations
-; CHECK-ASM-NEXT:	.short	7
-; Location 1 Constant 0
-; CHECK-ASM-NEXT:	.byte	4
-; CHECK-ASM-NEXT:	.byte	0
-; CHECK-ASM-NEXT:	.short	8
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.long	0
-; Location 2 Constant 0
-; CHECK-ASM-NEXT:	.byte	4
-; CHECK-ASM-NEXT:	.byte	0
-; CHECK-ASM-NEXT:	.short	8
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.long	0
-; Location 3 Constant 2
-; CHECK-ASM-NEXT:	.byte	4
-; CHECK-ASM-NEXT:	.byte	0
-; CHECK-ASM-NEXT:	.short	8
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.long	2
-; Location 4 Register $rbx
-; CHECK-ASM-NEXT:	.byte	1
-; CHECK-ASM-NEXT:	.byte	0
-; CHECK-ASM-NEXT:	.short	8
-; CHECK-ASM-NEXT:	.short	3
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.long	0
-; Location 5 Constant Index 0
-; CHECK-ASM-NEXT:	.byte	5
-; CHECK-ASM-NEXT:	.byte	0
-; CHECK-ASM-NEXT:	.short	8
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.long	0
-; Location 6 Register $rbx
-; CHECK-ASM-NEXT:	.byte	1
-; CHECK-ASM-NEXT:	.byte	0
-; CHECK-ASM-NEXT:	.short	8
-; CHECK-ASM-NEXT:	.short	3
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.long	0
-; Location 7 Register $rbx
-; CHECK-ASM-NEXT:	.byte	1
-; CHECK-ASM-NEXT:	.byte	0
-; CHECK-ASM-NEXT:	.short	8
-; CHECK-ASM-NEXT:	.short	3
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.long	0
-; Entry for test_gcptr_uniqueing
-; CHECK-ASM:     	.long	.Ltmp6-test_gcptr_uniqueing
-; CHECK-ASM-NEXT:	.short	0
-; Num locations
-; CHECK-ASM-NEXT:	.short	7
-; Location 1 Constant 0
-; CHECK-ASM-NEXT:	.byte	4
-; CHECK-ASM-NEXT:	.byte	0
-; CHECK-ASM-NEXT:	.short	8
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.long	0
-; Location 2 Constant 0
-; CHECK-ASM-NEXT:	.byte	4
-; CHECK-ASM-NEXT:	.byte	0
-; CHECK-ASM-NEXT:	.short	8
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.long	0
-; Location 3 Constant 2
-; CHECK-ASM-NEXT:	.byte	4
-; CHECK-ASM-NEXT:	.byte	0
-; CHECK-ASM-NEXT:	.short	8
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.long	2
-; Location 4 Register $rbx
-; CHECK-ASM-NEXT:	.byte	1
-; CHECK-ASM-NEXT:	.byte	0
-; CHECK-ASM-NEXT:	.short	8
-; CHECK-ASM-NEXT:	.short	3
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.long	0
-; Location 5 Constant Index 0
-; CHECK-ASM-NEXT:	.byte	5
-; CHECK-ASM-NEXT:	.byte	0
-; CHECK-ASM-NEXT:	.short	8
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.long	0
-; Location 6 Register $rbx
-; CHECK-ASM-NEXT:	.byte	1
-; CHECK-ASM-NEXT:	.byte	0
-; CHECK-ASM-NEXT:	.short	8
-; CHECK-ASM-NEXT:	.short	3
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.long	0
-; Location 7 Register $rbx
-; CHECK-ASM-NEXT:	.byte	1
-; CHECK-ASM-NEXT:	.byte	0
-; CHECK-ASM-NEXT:	.short	8
-; CHECK-ASM-NEXT:	.short	3
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.long	0
-; Entry for test_cross_bb
-; CHECK-ASM:     	.quad	0
-; CHECK-ASM-NEXT:	.long	.Ltmp7-test_cross_bb
-; CHECK-ASM-NEXT:	.short	0
-; Num locations
-; CHECK-ASM-NEXT:	.short	5
-; Location 1 Constant 0
-; CHECK-ASM-NEXT:	.byte	4
-; CHECK-ASM-NEXT:	.byte	0
-; CHECK-ASM-NEXT:	.short	8
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.long	0
-; Location 2 Constant 0
-; CHECK-ASM-NEXT:	.byte	4
-; CHECK-ASM-NEXT:	.byte	0
-; CHECK-ASM-NEXT:	.short	8
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.long	0
-; Location 3 Constant 0
-; CHECK-ASM-NEXT:	.byte	4
-; CHECK-ASM-NEXT:	.byte	0
-; CHECK-ASM-NEXT:	.short	8
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.long	0
-; Location 4 Indirect $rsp + 0
-; CHECK-ASM-NEXT:	.byte	3
-; CHECK-ASM-NEXT:	.byte	0
-; CHECK-ASM-NEXT:	.short	8
-; CHECK-ASM-NEXT:	.short	7
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.long	0
-; Location 5 Indirect $rsp + 0
-; CHECK-ASM-NEXT:	.byte	3
-; CHECK-ASM-NEXT:	.byte	0
-; CHECK-ASM-NEXT:	.short	8
-; CHECK-ASM-NEXT:	.short	7
-; CHECK-ASM-NEXT:	.short	0
-; CHECK-ASM-NEXT:	.long	0

From a1c9529e602adcc25162f93fd4563fbe3d04ab62 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Wed, 29 Jul 2020 19:52:11 +0100
Subject: [PATCH 0533/1035] [X86][AVX] isHorizontalBinOp - relax
 no-lane-crossing limit for AVX1-only targets.

Instead of never accepting v8f32/v4f64 FHADD/FHSUB if the input shuffle masks cross lanes, perform the matching and determine if the post shuffle mask simplifies to a 'whole lane shuffle' mask - in which case we are guaranteed to cheaply perform this as a VPERM2F128 shuffle.
---
 llvm/lib/Target/X86/X86ISelLowering.cpp | 19 +++++++++++++------
 llvm/test/CodeGen/X86/haddsub-4.ll      | 12 +++++-------
 2 files changed, 18 insertions(+), 13 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 556bb58468817..8d95c066f5a0b 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -5625,6 +5625,11 @@ static bool scaleShuffleElements(ArrayRef<int> Mask, unsigned NumDstElts,
   return false;
 }
 
+static bool canScaleShuffleElements(ArrayRef<int> Mask, unsigned NumDstElts) {
+  SmallVector<int, 32> WidenedMask;
+  return scaleShuffleElements(Mask, NumDstElts, WidenedMask);
+}
+
 /// Returns true if Elt is a constant zero or a floating point constant +0.0.
 bool X86::isZeroNode(SDValue Elt) {
   return isNullConstant(Elt) || isNullFPConstant(Elt);
@@ -44486,12 +44491,6 @@ static bool isHorizontalBinOp(SDValue &LHS, SDValue &RHS, SelectionDAG &DAG,
       RMask.push_back(i);
   }
 
-  // Avoid 128-bit lane crossing if pre-AVX2 and FP (integer will split).
-  if (!Subtarget.hasAVX2() && VT.isFloatingPoint() &&
-      (isLaneCrossingShuffleMask(128, VT.getScalarSizeInBits(), LMask) ||
-       isLaneCrossingShuffleMask(128, VT.getScalarSizeInBits(), RMask)))
-    return false;
-
   // If A and B occur in reverse order in RHS, then canonicalize by commuting
   // RHS operands and shuffle mask.
   if (A != C) {
@@ -44554,6 +44553,14 @@ static bool isHorizontalBinOp(SDValue &LHS, SDValue &RHS, SelectionDAG &DAG,
   if (IsIdentityPostShuffle)
     PostShuffleMask.clear();
 
+  // Avoid 128-bit lane crossing if pre-AVX2 and FP (integer will split), unless
+  // the shuffle can widen to shuffle entire lanes, which should still be quick.
+  if (!IsIdentityPostShuffle && !Subtarget.hasAVX2() && VT.isFloatingPoint() &&
+      isLaneCrossingShuffleMask(128, VT.getScalarSizeInBits(),
+                                PostShuffleMask) &&
+      !canScaleShuffleElements(PostShuffleMask, 2))
+    return false;
+
   // Assume a SingleSource HOP if we only shuffle one input and don't need to
   // shuffle the result.
   if (!shouldUseHorizontalOp(LHS == RHS &&
diff --git a/llvm/test/CodeGen/X86/haddsub-4.ll b/llvm/test/CodeGen/X86/haddsub-4.ll
index 4c1dc71982aa4..31b6617b45a96 100644
--- a/llvm/test/CodeGen/X86/haddsub-4.ll
+++ b/llvm/test/CodeGen/X86/haddsub-4.ll
@@ -151,13 +151,11 @@ define <8 x double> @hadd_reverse_v8f64(<8 x double> %a0, <8 x double> %a1) noun
 ;
 ; AVX1-LABEL: hadd_reverse_v8f64:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm2[2,3,0,1]
-; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,0,1]
-; AVX1-NEXT:    vhaddpd %ymm2, %ymm0, %ymm2
-; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm3[2,3,0,1]
-; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm1[2,3,0,1]
-; AVX1-NEXT:    vhaddpd %ymm0, %ymm1, %ymm0
-; AVX1-NEXT:    vmovapd %ymm2, %ymm1
+; AVX1-NEXT:    vhaddpd %ymm3, %ymm1, %ymm1
+; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm3 = ymm1[2,3,0,1]
+; AVX1-NEXT:    vhaddpd %ymm2, %ymm0, %ymm0
+; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm0[2,3,0,1]
+; AVX1-NEXT:    vmovapd %ymm3, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: hadd_reverse_v8f64:

From 8dbea4785c107aa975b6671025a3cea7829b8370 Mon Sep 17 00:00:00 2001
From: Victor Huang <wei.huang@ibm.com>
Date: Wed, 29 Jul 2020 18:38:55 +0000
Subject: [PATCH 0534/1035] [PowerPC] Support for R_PPC64_REL24_NOTOC calls
 where the caller has no TOC and the callee is not DSO local

This patch supports the situation where caller does not have a valid TOC and
calls using the R_PPC64_REL24_NOTOC relocation and the callee is not DSO local.
In this case the call cannot be made directly since the callee may or may not
require a valid TOC pointer. As a result this situation require a PC-relative
plt stub to set up r12.

Reviewed By: sfertile, MaskRay, stefanp

Differential Revision: https://reviews.llvm.org/D83669
---
 lld/ELF/Arch/PPC64.cpp                    |   5 -
 lld/ELF/Thunks.cpp                        |  35 +++++-
 lld/test/ELF/ppc64-pcrel-call-to-extern.s | 131 ++++++++++++++++++++++
 llvm/include/llvm/Object/ELF.h            |   1 +
 4 files changed, 166 insertions(+), 6 deletions(-)
 create mode 100644 lld/test/ELF/ppc64-pcrel-call-to-extern.s

diff --git a/lld/ELF/Arch/PPC64.cpp b/lld/ELF/Arch/PPC64.cpp
index da0b510d96a10..c1ad72eaa2c9d 100644
--- a/lld/ELF/Arch/PPC64.cpp
+++ b/lld/ELF/Arch/PPC64.cpp
@@ -1035,11 +1035,6 @@ bool PPC64::needsThunk(RelExpr expr, RelType type, const InputFile *file,
       type != R_PPC64_REL24_NOTOC)
     return false;
 
-  // FIXME: Remove the fatal error once the call protocol is implemented.
-  if (type == R_PPC64_REL24_NOTOC && s.isInPlt())
-    fatal("unimplemented feature: external function call with the reltype"
-          " R_PPC64_REL24_NOTOC");
-
   // If a function is in the Plt it needs to be called with a call-stub.
   if (s.isInPlt())
     return true;
diff --git a/lld/ELF/Thunks.cpp b/lld/ELF/Thunks.cpp
index 091a291b26cd4..cbdb286557f8d 100644
--- a/lld/ELF/Thunks.cpp
+++ b/lld/ELF/Thunks.cpp
@@ -305,6 +305,21 @@ class PPC64R12SetupStub final : public Thunk {
   void addSymbols(ThunkSection &isec) override;
 };
 
+// PPC64 PC-relative PLT Stub
+// When a caller that does not maintain a toc-pointer performs an extern call
+// then this stub is needed for:
+// 1) Loading the target functions address from the procedure linkage table into
+//    r12 for use by the target functions global entry point, and into the count
+//    register with pc-relative instructions.
+// 2) Transferring control to the target function through an indirect branch.
+class PPC64PCRelPLTStub final : public Thunk {
+public:
+  PPC64PCRelPLTStub(Symbol &dest) : Thunk(dest, 0) {}
+  uint32_t size() override { return 16; }
+  void writeTo(uint8_t *buf) override;
+  void addSymbols(ThunkSection &isec) override;
+};
+
 // A bl instruction uses a signed 24 bit offset, with an implicit 4 byte
 // alignment. This gives a possible 26 bits of 'reach'. If the call offset is
 // larger then that we need to emit a long-branch thunk. The target address
@@ -880,6 +895,23 @@ void PPC64R12SetupStub::addSymbols(ThunkSection &isec) {
             isec);
 }
 
+void PPC64PCRelPLTStub::writeTo(uint8_t *buf) {
+  int64_t offset = destination.getGotPltVA() - getThunkTargetSym()->getVA();
+  if (!isInt<34>(offset))
+    fatal("offset must fit in 34 bits to encode in the instruction");
+  uint64_t pld =
+      PLD_R12_NO_DISP | (((offset >> 16) & 0x3ffff) << 32) | (offset & 0xffff);
+
+  writePrefixedInstruction(buf + 0, pld); // pld r12, func@plt@pcrel
+  write32(buf + 8, MTCTR_R12);            // mtctr r12
+  write32(buf + 12, BCTR);                // bctr
+}
+
+void PPC64PCRelPLTStub::addSymbols(ThunkSection &isec) {
+  addSymbol(saver.save("__plt_pcrel_" + destination.getName()), STT_FUNC, 0,
+            isec);
+}
+
 void PPC64LongBranchThunk::writeTo(uint8_t *buf) {
   int64_t offset = in.ppc64LongBranchTarget->getEntryVA(&destination, addend) -
                    getPPC64TocBase();
@@ -1007,7 +1039,8 @@ static Thunk *addThunkPPC64(RelType type, Symbol &s, int64_t a) {
           type == R_PPC64_REL24_NOTOC) &&
          "unexpected relocation type for thunk");
   if (s.isInPlt())
-    return make<PPC64PltCallStub>(s);
+    return type == R_PPC64_REL24_NOTOC ? (Thunk *)make<PPC64PCRelPLTStub>(s)
+                                       : (Thunk *)make<PPC64PltCallStub>(s);
 
   // This check looks at the st_other bits of the callee. If the value is 1
   // then the callee clobbers the TOC and we need an R2 save stub.
diff --git a/lld/test/ELF/ppc64-pcrel-call-to-extern.s b/lld/test/ELF/ppc64-pcrel-call-to-extern.s
new file mode 100644
index 0000000000000..ab2eaf09a6acc
--- /dev/null
+++ b/lld/test/ELF/ppc64-pcrel-call-to-extern.s
@@ -0,0 +1,131 @@
+# REQUIRES: ppc
+# RUN: echo 'SECTIONS { \
+# RUN:   .text_caller1 0x10010000 : { *(.text_caller1) } \
+# RUN:   .text_caller2 0x10020000 : { *(.text_caller2) } \
+# RUN:   .text_caller3 0x10030000 : { *(.text_caller3) } \
+# RUN:   }' > %t.script
+
+# RUN: llvm-mc -filetype=obj -triple=powerpc64le --defsym AUX=1 %s -o %t1.o
+# RUN: llvm-mc -filetype=obj -triple=powerpc64le %s -o %t2.o
+# RUN: ld.lld --shared %t2.o -o %t2.so
+# RUN: ld.lld -T %t.script %t1.o %t2.so -o %t
+# RUN: llvm-readelf -s %t | FileCheck %s --check-prefix=SYMBOL
+# RUN: llvm-readelf -S -d %t | FileCheck %s --check-prefix=SEC
+# RUN: llvm-readobj -r %t | FileCheck %s --check-prefix=REL
+# RUN: llvm-objdump -d --no-show-raw-insn --mcpu=pwr10 %t | FileCheck %s
+
+# RUN: llvm-mc -filetype=obj -triple=powerpc64 --defsym AUX=1 %s -o %t1.o
+# RUN: llvm-mc -filetype=obj -triple=powerpc64 %s -o %t2.o
+# RUN: ld.lld --shared %t2.o -o %t2.so
+# RUN: ld.lld -T %t.script %t1.o %t2.so -o %t
+# RUN: llvm-readelf -s %t | FileCheck %s --check-prefix=SYMBOL
+# RUN: llvm-readelf -S -d %t | FileCheck %s --check-prefix=SEC
+# RUN: llvm-readobj -r %t | FileCheck %s --check-prefix=REL
+# RUN: llvm-objdump -d --no-show-raw-insn --mcpu=pwr10 %t | FileCheck %s
+
+## The test is created to check that when a function without TOC access an
+## external function, a r12 setup stub is inserted.
+
+# SYMBOL: Symbol table '.dynsym' contains 4 entries:
+# SYMBOL:      1: 0000000000000000     0 NOTYPE  GLOBAL DEFAULT [<other: 0x60>] UND callee_global_TOC
+# SYMBOL-NEXT: 2: 0000000000000000     0 NOTYPE  GLOBAL DEFAULT                UND callee_global_stother0
+# SYMBOL-NEXT: 3: 0000000000000000     0 NOTYPE  GLOBAL DEFAULT [<other: 0x20>] UND callee_global_stother1
+
+# SYMBOL: Symbol table '.symtab' contains 12 entries:
+# SYMBOL:      2: 0000000010010000     0 NOTYPE  LOCAL  DEFAULT [<other: 0x20>]   6 caller1
+# SYMBOL-NEXT: 3: 0000000010020000     0 NOTYPE  LOCAL  DEFAULT [<other: 0x20>]   7 caller2
+# SYMBOL-NEXT: 4: 0000000010030000     0 NOTYPE  LOCAL  DEFAULT [<other: 0x20>]   8 caller3
+# SYMBOL:      6: 0000000010010008    16 FUNC    LOCAL  DEFAULT                  6 __plt_pcrel_callee_global_stother0
+# SYMBOL-NEXT: 7: 0000000010020008    16 FUNC    LOCAL  DEFAULT                  7 __plt_pcrel_callee_global_stother1
+# SYMBOL-NEXT: 8: 0000000010030008    16 FUNC    LOCAL  DEFAULT                  8 __plt_pcrel_callee_global_TOC
+# SYMBOL-NEXT: 9: 0000000000000000     0 NOTYPE  GLOBAL DEFAULT [<other: 0x60>] UND callee_global_TOC
+# SYMBOL-NEXT: 10: 0000000000000000     0 NOTYPE  GLOBAL DEFAULT                UND callee_global_stother0
+# SYMBOL-NEXT: 11: 0000000000000000     0 NOTYPE  GLOBAL DEFAULT [<other: 0x20>] UND callee_global_stother1
+
+## DT_PLTGOT points to .plt
+# SEC: .plt              NOBITS          0000000010030140 040140 000028 00  WA  0   0  8
+# SEC: 0x0000000000000003 (PLTGOT)      0x10030140
+
+## The first 2 entries in the .plt are reserved for the dynamic linkers
+## usage. The JMP_SLOT relocations are stored at .plt[2], .plt[3], .plt[4].
+## Check that we emit 3 R_PPC64_JMP_SLOT in .rela.plt.
+# REL:      .rela.plt {
+# REL-NEXT:   0x10030150 R_PPC64_JMP_SLOT callee_global_stother0 0x0
+# REL-NEXT:   0x10030158 R_PPC64_JMP_SLOT callee_global_stother1 0x0
+# REL-NEXT:   0x10030160 R_PPC64_JMP_SLOT callee_global_TOC 0x0
+# REL-NEXT: }
+
+# CHECK-LABEL: <caller1>:
+# CHECK:       10010000: bl 0x10010008
+# CHECK-NEXT:  10010004: blr
+
+## .plt[2] - 0x10010008 = 0x10030150 - 0x10010008 = 0x20148 = 131400
+# CHECK-LABEL: <__plt_pcrel_callee_global_stother0>:
+# CHECK:       10010008: pld 12, 131400(0), 1
+# CHECK-NEXT:  10010010: mtctr 12
+# CHECK-NEXT:  10010014: bctr
+
+# CHECK-LABEL: <caller2>:
+# CHECK:       10020000: bl 0x10020008
+# CHECK-NEXT:  10020004: blr
+
+## .plt[3] - 0x10020008 = 0x10030158 - 0x10020008 = 0x10150 = 65872
+# CHECK-LABEL: <__plt_pcrel_callee_global_stother1>:
+# CHECK:       10020008: pld 12, 65872(0), 1
+# CHECK-NEXT:  10020010: mtctr 12
+# CHECK-NEXT:  10020014: bctr
+
+# CHECK-LABEL: <caller3>:
+# CHECK:       10030000: bl 0x10030008
+# CHECK-NEXT:  10030004: blr
+
+## .plt[4] - 0x10030008 = 0x10030160 - 0x10030008 = 0x158 = 344
+# CHECK-LABEL: <__plt_pcrel_callee_global_TOC>:
+# CHECK:       10030008: pld 12, 344(0), 1
+# CHECK-NEXT:  10030010: mtctr 12
+# CHECK-NEXT:  10030014: bctr
+
+.ifdef AUX
+.section .text_caller1, "ax", %progbits
+caller1:
+  .localentry caller1, 1
+  bl callee_global_stother0@notoc
+  blr
+.section .text_caller2, "ax", %progbits
+caller2:
+  .localentry caller2, 1
+  bl callee_global_stother1@notoc
+  blr
+
+.section .text_caller3, "ax", %progbits
+caller3:
+  .localentry caller3, 1
+  bl callee_global_TOC@notoc
+  blr
+
+.else
+func_extern:
+  blr
+.globl callee_global_stother0
+callee_global_stother0:
+  blr
+.globl callee_global_stother1
+callee_global_stother1:
+  .localentry callee_global_stother1, 1
+  ## nop is not needed after bl for R_PPC64_REL24_NOTOC
+  bl func_extern@notoc
+  blr
+.globl callee_global_TOC
+callee_global_TOC:
+.Lfunc_gep1:
+  addis 2, 12, .TOC.-.Lfunc_gep1@ha
+  addi 2, 2, .TOC.-.Lfunc_gep1@l
+.Lfunc_lep1:
+  .localentry callee_global_TOC, .Lfunc_lep1-.Lfunc_gep1
+  addis 4, 2, global@toc@ha
+  lwz 3, global@toc@l(4)
+  blr
+global:
+  .long  0
+  .size  global, 4
+.endif
diff --git a/llvm/include/llvm/Object/ELF.h b/llvm/include/llvm/Object/ELF.h
index 3dab0471a2391..0080d0ba58f34 100644
--- a/llvm/include/llvm/Object/ELF.h
+++ b/llvm/include/llvm/Object/ELF.h
@@ -50,6 +50,7 @@ static inline Error createError(const Twine &Err) {
 
 enum PPCInstrMasks : uint64_t {
   PADDI_R12_NO_DISP = 0x0610000039800000,
+  PLD_R12_NO_DISP = 0x04100000E5800000,
   MTCTR_R12 = 0x7D8903A6,
   BCTR = 0x4E800420,
 };

From ed7bde0e4b40cbf8a7c833fd8240c957fcda176e Mon Sep 17 00:00:00 2001
From: Fangrui Song <maskray@google.com>
Date: Wed, 29 Jul 2020 13:01:31 -0700
Subject: [PATCH 0535/1035] [ELF][test] Fix ppc64-reloc-pcrel34-overflow.s

---
 lld/test/ELF/ppc64-reloc-pcrel34-overflow.s | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/lld/test/ELF/ppc64-reloc-pcrel34-overflow.s b/lld/test/ELF/ppc64-reloc-pcrel34-overflow.s
index a97160a430dd0..ad80ed720b637 100644
--- a/lld/test/ELF/ppc64-reloc-pcrel34-overflow.s
+++ b/lld/test/ELF/ppc64-reloc-pcrel34-overflow.s
@@ -1,21 +1,22 @@
 # REQUIRES: ppc
 # RUN: echo 'SECTIONS { \
-# RUN:       .text_low 0x10010000: { *(.text_low) } \
-# RUN:       .text_overflow 0x1000000000 : { *(.text_overflow) } \
-# RUN:       }' > %t.script
+# RUN:   .text 0x10000: { *(.text) } \
+# RUN:   .data 0x200010000 : { *(.data) } \
+# RUN: }' > %t.script
 
 # RUN: llvm-mc -filetype=obj -triple=powerpc64le %s -o %t.o
-# RUN: not ld.lld -T %t.script %t.o -o %t
+# RUN: not ld.lld -T %t.script %t.o -o /dev/null 2>&1 | FileCheck %s
 
 # RUN: llvm-mc -filetype=obj -triple=powerpc64 %s -o %t.o
-# RUN: not ld.lld -T %t.script %t.o -o %t
+# RUN: not ld.lld -T %t.script %t.o -o /dev/null 2>&1 | FileCheck %s
 
-.section .text_low, "ax", %progbits
-# CHECK: relocation R_PPC64_PCREL34 out of range
-GlobIntOverflow:
+# CHECK: relocation R_PPC64_PCREL34 out of range: 8589934592 is not in [-8589934592, 8589934591]
 	plwa 3, glob_overflow@PCREL(0), 1
-	blr
-.section .text_overflow, "ax", %progbits
+
+# CHECK-NOT: relocation
+	plwa 3, .data@PCREL(0), 1
+
+.data
 glob_overflow:
 	.long	0
 	.size	glob_overflow, 4

From 6ce97eee35463be2ccd82ae659034506ae9b25f1 Mon Sep 17 00:00:00 2001
From: Richard Smith <richard@metafoo.co.uk>
Date: Wed, 29 Jul 2020 13:06:52 -0700
Subject: [PATCH 0536/1035] Fix invalid attempted explicit instantiation, which
 Clang now rejects.

---
 .../clang-tidy/checkers/bugprone-string-integer-assignment.cpp  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone-string-integer-assignment.cpp b/clang-tools-extra/test/clang-tidy/checkers/bugprone-string-integer-assignment.cpp
index 2c288e0bbddf9..8db47ea190d8f 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/bugprone-string-integer-assignment.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone-string-integer-assignment.cpp
@@ -107,5 +107,5 @@ struct S {
   // CHECK-FIXES: {{^}}  void f(char c) { s += std::to_string(c | static_cast<int>(t)); } 
 };
 
-template S<int>;
+template struct S<int>;
 }

From 77c3b016c42412edd43568b001fe358425a113dd Mon Sep 17 00:00:00 2001
From: Tobias Gysi <tobias.gysi@inf.ethz.ch>
Date: Wed, 29 Jul 2020 22:00:48 +0200
Subject: [PATCH 0537/1035] [mlir] fix error handling in rocm runtime wrapper

The patch fixes minor issues in the rocm runtime wrapper due to api differences between CUDA and HIP.

Reviewed By: herhut

Differential Revision: https://reviews.llvm.org/D84861
---
 mlir/tools/mlir-rocm-runner/rocm-runtime-wrappers.cpp | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/mlir/tools/mlir-rocm-runner/rocm-runtime-wrappers.cpp b/mlir/tools/mlir-rocm-runner/rocm-runtime-wrappers.cpp
index b97ce695ac42c..a64815007661e 100644
--- a/mlir/tools/mlir-rocm-runner/rocm-runtime-wrappers.cpp
+++ b/mlir/tools/mlir-rocm-runner/rocm-runtime-wrappers.cpp
@@ -25,8 +25,7 @@
   [](hipError_t result) {                                                      \
     if (!result)                                                               \
       return;                                                                  \
-    const char *name = nullptr;                                                \
-    hipGetErrorName(result, &name);                                            \
+    const char *name = hipGetErrorName(result);                                \
     if (!name)                                                                 \
       name = "<unknown>";                                                      \
     llvm::errs() << "'" << #expr << "' failed with '" << name << "'\n";        \
@@ -74,8 +73,7 @@ extern "C" void mgpuStreamSynchronize(hipStream_t stream) {
 // Allows to register byte array with the ROCM runtime. Helpful until we have
 // transfer functions implemented.
 extern "C" void mgpuMemHostRegister(void *ptr, uint64_t sizeBytes) {
-  HIP_REPORT_IF_ERROR(hipHostRegister(ptr, sizeBytes, /*flags=*/0),
-                      "MemHostRegister");
+  HIP_REPORT_IF_ERROR(hipHostRegister(ptr, sizeBytes, /*flags=*/0));
 }
 
 // Allows to register a MemRef with the ROCM runtime. Initializes array with
@@ -116,10 +114,9 @@ extern "C" void mgpuMemHostRegisterInt32(int64_t rank, void *ptr) {
 
 template <typename T>
 void mgpuMemGetDevicePointer(T *hostPtr, T **devicePtr) {
-  HIP_REPORT_IF_ERROR(hipSetDevice(0), "hipSetDevice");
+  HIP_REPORT_IF_ERROR(hipSetDevice(0));
   HIP_REPORT_IF_ERROR(
-      hipHostGetDevicePointer((void **)devicePtr, hostPtr, /*flags=*/0),
-      "hipHostGetDevicePointer");
+      hipHostGetDevicePointer((void **)devicePtr, hostPtr, /*flags=*/0));
 }
 
 extern "C" StridedMemRefType<float, 1>

From 897bdca4b81dff200714a5b1cb22b20e9c4c4594 Mon Sep 17 00:00:00 2001
From: Nikita Popov <nikita.ppv@gmail.com>
Date: Sat, 25 Jul 2020 16:21:55 +0200
Subject: [PATCH 0538/1035] [ConstantRange] Add API for intrinsics (NFC)

This adds a common API for compute constant ranges of intrinsics.
The intention here is that
a) we can reuse the same code across different passes that handle
   constant ranges, i.e. this can be reused in SCCP
b) we only have to add knowledge about supported intrinsics to
   ConstantRange, not any consumers.

Differential Revision: https://reviews.llvm.org/D84587
---
 llvm/include/llvm/IR/ConstantRange.h |  8 +++++
 llvm/lib/Analysis/LazyValueInfo.cpp  | 51 ++++++++--------------------
 llvm/lib/IR/ConstantRange.cpp        | 30 ++++++++++++++++
 3 files changed, 53 insertions(+), 36 deletions(-)

diff --git a/llvm/include/llvm/IR/ConstantRange.h b/llvm/include/llvm/IR/ConstantRange.h
index 8ecb9aa0ce020..a027292ea43f4 100644
--- a/llvm/include/llvm/IR/ConstantRange.h
+++ b/llvm/include/llvm/IR/ConstantRange.h
@@ -150,6 +150,14 @@ class LLVM_NODISCARD ConstantRange {
                                              const APInt &Other,
                                              unsigned NoWrapKind);
 
+  /// Returns true if ConstantRange calculations are supported for intrinsic
+  /// with \p IntrinsicID.
+  static bool isIntrinsicSupported(Intrinsic::ID IntrinsicID);
+
+  /// Compute range of intrinsic result for the given operand ranges.
+  static ConstantRange intrinsic(Intrinsic::ID IntrinsicID,
+                                 ArrayRef<ConstantRange> Ops);
+
   /// Set up \p Pred and \p RHS such that
   /// ConstantRange::makeExactICmpRegion(Pred, RHS) == *this.  Return true if
   /// successful.
diff --git a/llvm/lib/Analysis/LazyValueInfo.cpp b/llvm/lib/Analysis/LazyValueInfo.cpp
index 34cc81c4bf2a3..d326a442051a8 100644
--- a/llvm/lib/Analysis/LazyValueInfo.cpp
+++ b/llvm/lib/Analysis/LazyValueInfo.cpp
@@ -400,8 +400,6 @@ class LazyValueInfoImpl {
                                                     BasicBlock *BB);
   Optional<ValueLatticeElement> solveBlockValueOverflowIntrinsic(
       WithOverflowInst *WO, BasicBlock *BB);
-  Optional<ValueLatticeElement> solveBlockValueSaturatingIntrinsic(
-      SaturatingInst *SI, BasicBlock *BB);
   Optional<ValueLatticeElement> solveBlockValueIntrinsic(IntrinsicInst *II,
                                                          BasicBlock *BB);
   Optional<ValueLatticeElement> solveBlockValueExtractValue(
@@ -1035,43 +1033,24 @@ LazyValueInfoImpl::solveBlockValueOverflowIntrinsic(WithOverflowInst *WO,
       });
 }
 
-Optional<ValueLatticeElement>
-LazyValueInfoImpl::solveBlockValueSaturatingIntrinsic(SaturatingInst *SI,
-                                                      BasicBlock *BB) {
-  switch (SI->getIntrinsicID()) {
-  case Intrinsic::uadd_sat:
-    return solveBlockValueBinaryOpImpl(
-        SI, BB, [](const ConstantRange &CR1, const ConstantRange &CR2) {
-          return CR1.uadd_sat(CR2);
-        });
-  case Intrinsic::usub_sat:
-    return solveBlockValueBinaryOpImpl(
-        SI, BB, [](const ConstantRange &CR1, const ConstantRange &CR2) {
-          return CR1.usub_sat(CR2);
-        });
-  case Intrinsic::sadd_sat:
-    return solveBlockValueBinaryOpImpl(
-        SI, BB, [](const ConstantRange &CR1, const ConstantRange &CR2) {
-          return CR1.sadd_sat(CR2);
-        });
-  case Intrinsic::ssub_sat:
-    return solveBlockValueBinaryOpImpl(
-        SI, BB, [](const ConstantRange &CR1, const ConstantRange &CR2) {
-          return CR1.ssub_sat(CR2);
-        });
-  default:
-    llvm_unreachable("All llvm.sat intrinsic are handled.");
-  }
-}
-
 Optional<ValueLatticeElement> LazyValueInfoImpl::solveBlockValueIntrinsic(
     IntrinsicInst *II, BasicBlock *BB) {
-  if (auto *SI = dyn_cast<SaturatingInst>(II))
-    return solveBlockValueSaturatingIntrinsic(SI, BB);
+  if (!ConstantRange::isIntrinsicSupported(II->getIntrinsicID())) {
+    LLVM_DEBUG(dbgs() << " compute BB '" << BB->getName()
+                      << "' - overdefined (unknown intrinsic).\n");
+    return ValueLatticeElement::getOverdefined();
+  }
 
-  LLVM_DEBUG(dbgs() << " compute BB '" << BB->getName()
-                    << "' - overdefined (unknown intrinsic).\n");
-  return ValueLatticeElement::getOverdefined();
+  SmallVector<ConstantRange, 2> OpRanges;
+  for (Value *Op : II->args()) {
+    Optional<ConstantRange> Range = getRangeFor(Op, II, BB);
+    if (!Range)
+      return None;
+    OpRanges.push_back(*Range);
+  }
+
+  return ValueLatticeElement::getRange(
+      ConstantRange::intrinsic(II->getIntrinsicID(), OpRanges));
 }
 
 Optional<ValueLatticeElement> LazyValueInfoImpl::solveBlockValueExtractValue(
diff --git a/llvm/lib/IR/ConstantRange.cpp b/llvm/lib/IR/ConstantRange.cpp
index eabaaa203927f..26cff911cd675 100644
--- a/llvm/lib/IR/ConstantRange.cpp
+++ b/llvm/lib/IR/ConstantRange.cpp
@@ -26,6 +26,7 @@
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/InstrTypes.h"
 #include "llvm/IR/Instruction.h"
+#include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/Metadata.h"
 #include "llvm/IR/Operator.h"
 #include "llvm/Support/Compiler.h"
@@ -835,6 +836,35 @@ ConstantRange ConstantRange::overflowingBinaryOp(Instruction::BinaryOps BinOp,
   }
 }
 
+bool ConstantRange::isIntrinsicSupported(Intrinsic::ID IntrinsicID) {
+  switch (IntrinsicID) {
+  case Intrinsic::uadd_sat:
+  case Intrinsic::usub_sat:
+  case Intrinsic::sadd_sat:
+  case Intrinsic::ssub_sat:
+    return true;
+  default:
+    return false;
+  }
+}
+
+ConstantRange ConstantRange::intrinsic(Intrinsic::ID IntrinsicID,
+                                       ArrayRef<ConstantRange> Ops) {
+  switch (IntrinsicID) {
+  case Intrinsic::uadd_sat:
+    return Ops[0].uadd_sat(Ops[1]);
+  case Intrinsic::usub_sat:
+    return Ops[0].usub_sat(Ops[1]);
+  case Intrinsic::sadd_sat:
+    return Ops[0].sadd_sat(Ops[1]);
+  case Intrinsic::ssub_sat:
+    return Ops[0].ssub_sat(Ops[1]);
+  default:
+    assert(!isIntrinsicSupported(IntrinsicID) && "Shouldn't be supported");
+    llvm_unreachable("Unsupported intrinsic");
+  }
+}
+
 ConstantRange
 ConstantRange::add(const ConstantRange &Other) const {
   if (isEmptySet() || Other.isEmptySet())

From 8723280b68b1e5ed97a699466720b36a32a9e406 Mon Sep 17 00:00:00 2001
From: Johannes Doerfert <johannes@jdoerfert.de>
Date: Wed, 29 Jul 2020 15:18:20 -0500
Subject: [PATCH 0539/1035] [OpenMP] Fix D83281 issue on windows by allowing
 `dso_local` in CHECK

---
 .../declare_variant_device_isa_codegen_1.c       | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/clang/test/OpenMP/declare_variant_device_isa_codegen_1.c b/clang/test/OpenMP/declare_variant_device_isa_codegen_1.c
index baa5eb8f88307..25fc3941dd504 100644
--- a/clang/test/OpenMP/declare_variant_device_isa_codegen_1.c
+++ b/clang/test/OpenMP/declare_variant_device_isa_codegen_1.c
@@ -31,18 +31,18 @@ void avx512_saxpy(int n, float s, float *x, float *y) {
 }
 
 void caller(int n, float s, float *x, float *y) {
-  // GENERIC:     define void @{{.*}}caller
-  // GENERIC:      call void @{{.*}}base_saxpy
-  // WITHFEATURE: define void @{{.*}}caller
-  // WITHFEATURE:  call void @{{.*}}avx512_saxpy
+  // GENERIC:     define void {{.*}}caller
+  // GENERIC:      call void {{.*}}base_saxpy
+  // WITHFEATURE: define void {{.*}}caller
+  // WITHFEATURE:  call void {{.*}}avx512_saxpy
   base_saxpy(n, s, x, y);
 }
 
 __attribute__((target("avx512f"))) void variant_caller(int n, float s, float *x, float *y) {
-  // GENERIC:     define void @{{.*}}variant_caller
-  // GENERIC:      call void @{{.*}}avx512_saxpy
-  // WITHFEATURE: define void @{{.*}}variant_caller
-  // WITHFEATURE:  call void @{{.*}}avx512_saxpy
+  // GENERIC:     define void {{.*}}variant_caller
+  // GENERIC:      call void {{.*}}avx512_saxpy
+  // WITHFEATURE: define void {{.*}}variant_caller
+  // WITHFEATURE:  call void {{.*}}avx512_saxpy
   base_saxpy(n, s, x, y);
 }
 

From 4f92ad508fcc543541be18fb766ab428d08e46e3 Mon Sep 17 00:00:00 2001
From: aartbik <ajcbik@google.com>
Date: Wed, 29 Jul 2020 13:25:31 -0700
Subject: [PATCH 0540/1035] [mlir] [VectorOps] [integration_test] Sparse matrix
 times vector (jagged SAXPY version)

Transposed jagged diagonal storage yields longer vector lengths. Also, in
contrast with naive SAXPY (one gather/scatter), this only performs one gather.

Reviewed By: reidtatge

Differential Revision: https://reviews.llvm.org/D84699
---
 .../CPU/test-sparse-saxpy-jagged-matvec.mlir  | 232 ++++++++++++++++++
 1 file changed, 232 insertions(+)
 create mode 100644 mlir/integration_test/Dialect/Vector/CPU/test-sparse-saxpy-jagged-matvec.mlir

diff --git a/mlir/integration_test/Dialect/Vector/CPU/test-sparse-saxpy-jagged-matvec.mlir b/mlir/integration_test/Dialect/Vector/CPU/test-sparse-saxpy-jagged-matvec.mlir
new file mode 100644
index 0000000000000..5b02b5df4b73a
--- /dev/null
+++ b/mlir/integration_test/Dialect/Vector/CPU/test-sparse-saxpy-jagged-matvec.mlir
@@ -0,0 +1,232 @@
+// RUN: mlir-opt %s -convert-scf-to-std -convert-vector-to-llvm -convert-std-to-llvm | \
+// RUN: mlir-cpu-runner -e entry -entry-point-result=void \
+// RUN:   -shared-libs=%mlir_integration_test_dir/libmlir_c_runner_utils%shlibext | \
+// RUN: FileCheck %s
+
+// Illustrates an 8x8 Sparse Matrix x Vector implemented with only operations
+// of the vector dialect (and some std/scf). Essentially, this example performs
+// the following multiplication:
+//
+//     0  1  2  3  4  5  6  7
+//   +------------------------+
+// 0 | 1  0  2  0  0  1  0  1 |   | 1 |   | 21 |
+// 1 | 1  8  0  0  3  0  1  0 |   | 2 |   | 39 |
+// 2 | 0  0  1  0  0  2  6  2 |   | 3 |   | 73 |
+// 3 | 0  3  0  1  0  1  0  1 | x | 4 | = | 24 |
+// 4 | 5  0  0  1  1  1  0  0 |   | 5 |   | 20 |
+// 5 | 0  3  0  0  2  1  2  0 |   | 6 |   | 36 |
+// 6 | 4  0  7  0  1  0  1  0 |   | 7 |   | 37 |
+// 7 | 0  3  0  2  0  0  1  1 |   | 8 |   | 29 |
+//   +------------------------+
+//
+// The sparse storage scheme used is an extended column scheme (also referred
+// to as jagged diagonal, which is essentially a vector friendly variant of
+// the general sparse row-wise scheme (also called compressed row storage),
+// using fixed length vectors and no explicit pointer indexing into the
+// value array to find the rows.
+//
+// The extended column storage for the matrix shown above is as follows.
+//
+//      VALUE           INDEX
+//   +---------+     +---------+
+// 0 | 1 2 1 1 |     | 0 2 5 7 |
+// 1 | 1 8 3 1 |     | 0 1 4 6 |
+// 2 | 1 2 6 2 |     | 2 5 6 7 |
+// 3 | 3 1 1 1 |     | 1 3 5 7 |
+// 4 | 5 1 1 1 |     | 0 3 4 5 |
+// 5 | 3 2 1 2 |     | 1 4 5 6 |
+// 6 | 4 7 1 1 |     | 0 2 4 6 |
+// 7 | 3 2 1 1 |     | 1 3 6 7 |
+//   +---------+     +---------+
+//
+// This example illustrates an effective SAXPY version that operates
+// on the transposed jagged diagonal storage to obtain higher vector
+// lengths. Another example in this directory illustrates a DOT
+// version of the operation.
+
+func @spmv8x8(%AVAL: memref<4xvector<8xf32>>,
+              %AIDX: memref<4xvector<8xi32>>,
+	      %X: memref<?xf32>, %B: memref<1xvector<8xf32>>) {
+  %c0 = constant 0 : index
+  %c1 = constant 1 : index
+  %cn = constant 4 : index
+  %mask = vector.constant_mask [8] : vector<8xi1>
+  %b = load %B[%c0] : memref<1xvector<8xf32>>
+  %b_out = scf.for %k = %c0 to %cn step %c1 iter_args(%b_iter = %b) -> (vector<8xf32>) {
+    %aval = load %AVAL[%k] : memref<4xvector<8xf32>>
+    %aidx = load %AIDX[%k] : memref<4xvector<8xi32>>
+    %0 = vector.gather %X, %aidx, %mask : (memref<?xf32>, vector<8xi32>, vector<8xi1>) -> vector<8xf32>
+    %b_new = vector.fma %aval, %0, %b_iter : vector<8xf32>
+    scf.yield %b_new : vector<8xf32>
+  }
+  store %b_out, %B[%c0] : memref<1xvector<8xf32>>
+  return
+}
+
+func @entry() {
+  %c0 = constant 0 : index
+  %c1 = constant 1 : index
+  %c2 = constant 2 : index
+  %c3 = constant 3 : index
+  %c4 = constant 4 : index
+  %c5 = constant 5 : index
+  %c6 = constant 6 : index
+  %c7 = constant 7 : index
+  %c8 = constant 8 : index
+
+  %f0 = constant 0.0 : f32
+  %f1 = constant 1.0 : f32
+  %f2 = constant 2.0 : f32
+  %f3 = constant 3.0 : f32
+  %f4 = constant 4.0 : f32
+  %f5 = constant 5.0 : f32
+  %f6 = constant 6.0 : f32
+  %f7 = constant 7.0 : f32
+  %f8 = constant 8.0 : f32
+
+  %i0 = constant 0 : i32
+  %i1 = constant 1 : i32
+  %i2 = constant 2 : i32
+  %i3 = constant 3 : i32
+  %i4 = constant 4 : i32
+  %i5 = constant 5 : i32
+  %i6 = constant 6 : i32
+  %i7 = constant 7 : i32
+
+  //
+  // Allocate.
+  //
+
+  %AVAL = alloc()    {alignment = 64} : memref<4xvector<8xf32>>
+  %AIDX = alloc()    {alignment = 64} : memref<4xvector<8xi32>>
+  %X    = alloc(%c8) {alignment = 64} : memref<?xf32>
+  %B    = alloc()    {alignment = 64} : memref<1xvector<8xf32>>
+
+  //
+  // Initialize.
+  //
+
+  %vf1 = vector.broadcast %f1 : f32 to vector<8xf32>
+
+  %0 = vector.insert %f3, %vf1[3] : f32 into vector<8xf32>
+  %1 = vector.insert %f5, %0[4] : f32 into vector<8xf32>
+  %2 = vector.insert %f3, %1[5] : f32 into vector<8xf32>
+  %3 = vector.insert %f4, %2[6] : f32 into vector<8xf32>
+  %4 = vector.insert %f3, %3[7] : f32 into vector<8xf32>
+  store %4, %AVAL[%c0] : memref<4xvector<8xf32>>
+
+  %5 = vector.insert %f2, %vf1[0] : f32 into vector<8xf32>
+  %6 = vector.insert %f8, %5[1] : f32 into vector<8xf32>
+  %7 = vector.insert %f2, %6[2] : f32 into vector<8xf32>
+  %8 = vector.insert %f2, %7[5] : f32 into vector<8xf32>
+  %9 = vector.insert %f7, %8[6] : f32 into vector<8xf32>
+  %10 = vector.insert %f2, %9[7] : f32 into vector<8xf32>
+  store %10, %AVAL[%c1] : memref<4xvector<8xf32>>
+
+  %11 = vector.insert %f3, %vf1[1] : f32 into vector<8xf32>
+  %12 = vector.insert %f6, %11[2] : f32 into vector<8xf32>
+  store %12, %AVAL[%c2] : memref<4xvector<8xf32>>
+
+  %13 = vector.insert %f2, %vf1[2] : f32 into vector<8xf32>
+  %14 = vector.insert %f2, %13[5] : f32 into vector<8xf32>
+  store %14, %AVAL[%c3] : memref<4xvector<8xf32>>
+
+  %vi0 = vector.broadcast %i0 : i32 to vector<8xi32>
+
+  %20 = vector.insert %i2, %vi0[2] : i32 into vector<8xi32>
+  %21 = vector.insert %i1, %20[3] : i32 into vector<8xi32>
+  %22 = vector.insert %i1, %21[5] : i32 into vector<8xi32>
+  %23 = vector.insert %i1, %22[7] : i32 into vector<8xi32>
+  store %23, %AIDX[%c0] : memref<4xvector<8xi32>>
+
+  %24 = vector.insert %i2, %vi0[0] : i32 into vector<8xi32>
+  %25 = vector.insert %i1, %24[1] : i32 into vector<8xi32>
+  %26 = vector.insert %i5, %25[2] : i32 into vector<8xi32>
+  %27 = vector.insert %i3, %26[3] : i32 into vector<8xi32>
+  %28 = vector.insert %i3, %27[4] : i32 into vector<8xi32>
+  %29 = vector.insert %i4, %28[5] : i32 into vector<8xi32>
+  %30 = vector.insert %i2, %29[6] : i32 into vector<8xi32>
+  %31 = vector.insert %i3, %30[7] : i32 into vector<8xi32>
+  store %31, %AIDX[%c1] : memref<4xvector<8xi32>>
+
+  %32 = vector.insert %i5, %vi0[0] : i32 into vector<8xi32>
+  %33 = vector.insert %i4, %32[1] : i32 into vector<8xi32>
+  %34 = vector.insert %i6, %33[2] : i32 into vector<8xi32>
+  %35 = vector.insert %i5, %34[3] : i32 into vector<8xi32>
+  %36 = vector.insert %i4, %35[4] : i32 into vector<8xi32>
+  %37 = vector.insert %i5, %36[5] : i32 into vector<8xi32>
+  %38 = vector.insert %i4, %37[6] : i32 into vector<8xi32>
+  %39 = vector.insert %i6, %38[7] : i32 into vector<8xi32>
+  store %39, %AIDX[%c2] : memref<4xvector<8xi32>>
+
+  %40 = vector.insert %i7, %vi0[0] : i32 into vector<8xi32>
+  %41 = vector.insert %i6, %40[1] : i32 into vector<8xi32>
+  %42 = vector.insert %i7, %41[2] : i32 into vector<8xi32>
+  %43 = vector.insert %i7, %42[3] : i32 into vector<8xi32>
+  %44 = vector.insert %i5, %43[4] : i32 into vector<8xi32>
+  %45 = vector.insert %i6, %44[5] : i32 into vector<8xi32>
+  %46 = vector.insert %i6, %45[6] : i32 into vector<8xi32>
+  %47 = vector.insert %i7, %46[7] : i32 into vector<8xi32>
+  store %47, %AIDX[%c3] : memref<4xvector<8xi32>>
+
+  %vf0 = vector.broadcast %f0 : f32 to vector<8xf32>
+  store %vf0, %B[%c0] : memref<1xvector<8xf32>>
+
+  scf.for %i = %c0 to %c8 step %c1 {
+    %ix = addi %i, %c1 : index
+    %kx = index_cast %ix : index to i32
+    %fx = sitofp %kx : i32 to f32
+    store %fx, %X[%i] : memref<?xf32>
+  }
+
+  //
+  // Multiply.
+  //
+
+  call @spmv8x8(%AVAL, %AIDX, %X, %B) : (memref<4xvector<8xf32>>,
+                                         memref<4xvector<8xi32>>,
+					 memref<?xf32>,
+					 memref<1xvector<8xf32>>) -> ()
+
+  //
+  // Print and verify.
+  //
+
+  scf.for %i = %c0 to %c4 step %c1 {
+    %aval = load %AVAL[%i] : memref<4xvector<8xf32>>
+    vector.print %aval : vector<8xf32>
+  }
+
+  scf.for %i = %c0 to %c4 step %c1 {
+    %aidx = load %AIDX[%i] : memref<4xvector<8xi32>>
+    vector.print %aidx : vector<8xi32>
+  }
+
+  %ldb = load %B[%c0] : memref<1xvector<8xf32>>
+  vector.print %ldb : vector<8xf32>
+
+  //
+  // CHECK:      ( 1, 1, 1, 3, 5, 3, 4, 3 )
+  // CHECK-NEXT: ( 2, 8, 2, 1, 1, 2, 7, 2 )
+  // CHECK-NEXT: ( 1, 3, 6, 1, 1, 1, 1, 1 )
+  // CHECK-NEXT: ( 1, 1, 2, 1, 1, 2, 1, 1 )
+  //
+  // CHECK-NEXT: ( 0, 0, 2, 1, 0, 1, 0, 1 )
+  // CHECK-NEXT: ( 2, 1, 5, 3, 3, 4, 2, 3 )
+  // CHECK-NEXT: ( 5, 4, 6, 5, 4, 5, 4, 6 )
+  // CHECK-NEXT: ( 7, 6, 7, 7, 5, 6, 6, 7 )
+  //
+  // CHECK-NEXT: ( 21, 39, 73, 24, 20, 36, 37, 29 )
+  //
+
+  //
+  // Free.
+  //
+
+  dealloc %AVAL : memref<4xvector<8xf32>>
+  dealloc %AIDX : memref<4xvector<8xi32>>
+  dealloc %X    : memref<?xf32>
+  dealloc %B    : memref<1xvector<8xf32>>
+
+  return
+}

From 47d2c7cd5db4abe11f44fd540bceb73b8b4372d9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Lubo=C5=A1=20Lu=C5=88=C3=A1k?= <l.lunak@centrum.cz>
Date: Fri, 11 Oct 2019 21:42:56 +0200
Subject: [PATCH 0541/1035] [lldb] remove somewhat dangerous 'd'(etach) and
 'k'(ill) shortcuts

'd' would be much better used for up/down shortcuts, and this also removes
the possibility of ruining the whole debugging session by accidentally
hitting 'd' or 'k'. Also change menu to have both 'detach and resume'
and 'detach suspended' to make it clear which one is which. See
discussion at https://reviews.llvm.org/D68541 .

Differential Revision: https://reviews.llvm.org/D68908
---
 lldb/source/Core/IOHandlerCursesGUI.cpp | 32 ++++++++++---------------
 1 file changed, 13 insertions(+), 19 deletions(-)

diff --git a/lldb/source/Core/IOHandlerCursesGUI.cpp b/lldb/source/Core/IOHandlerCursesGUI.cpp
index de4bd2a79095a..52d772c41ebc9 100644
--- a/lldb/source/Core/IOHandlerCursesGUI.cpp
+++ b/lldb/source/Core/IOHandlerCursesGUI.cpp
@@ -2829,7 +2829,8 @@ class ApplicationDelegate : public WindowDelegate, public MenuDelegate {
 
     eMenuID_Process,
     eMenuID_ProcessAttach,
-    eMenuID_ProcessDetach,
+    eMenuID_ProcessDetachResume,
+    eMenuID_ProcessDetachSuspended,
     eMenuID_ProcessLaunch,
     eMenuID_ProcessContinue,
     eMenuID_ProcessHalt,
@@ -2974,13 +2975,15 @@ class ApplicationDelegate : public WindowDelegate, public MenuDelegate {
     }
       return MenuActionResult::Handled;
 
-    case eMenuID_ProcessDetach: {
+    case eMenuID_ProcessDetachResume:
+    case eMenuID_ProcessDetachSuspended: {
       ExecutionContext exe_ctx =
           m_debugger.GetCommandInterpreter().GetExecutionContext();
       if (exe_ctx.HasProcessScope()) {
         Process *process = exe_ctx.GetProcessPtr();
         if (process && process->IsAlive())
-          process->Detach(false);
+          process->Detach(menu.GetIdentifier() ==
+                          eMenuID_ProcessDetachSuspended);
       }
     }
       return MenuActionResult::Handled;
@@ -3233,10 +3236,8 @@ class SourceFileWindowDelegate : public WindowDelegate {
         {KEY_NPAGE, "Page down"},
         {'b', "Set breakpoint on selected source/disassembly line"},
         {'c', "Continue process"},
-        {'d', "Detach and resume process"},
         {'D', "Detach with process suspended"},
         {'h', "Show help dialog"},
-        {'k', "Kill process"},
         {'n', "Step over (source line)"},
         {'N', "Step over (single instruction)"},
         {'o', "Step out"},
@@ -3798,26 +3799,15 @@ class SourceFileWindowDelegate : public WindowDelegate {
       }
       return eKeyHandled;
 
-    case 'd': // 'd' == detach and let run
     case 'D': // 'D' == detach and keep stopped
     {
       ExecutionContext exe_ctx =
           m_debugger.GetCommandInterpreter().GetExecutionContext();
       if (exe_ctx.HasProcessScope())
-        exe_ctx.GetProcessRef().Detach(c == 'D');
+        exe_ctx.GetProcessRef().Detach(true);
     }
       return eKeyHandled;
 
-    case 'k':
-      // 'k' == kill
-      {
-        ExecutionContext exe_ctx =
-            m_debugger.GetCommandInterpreter().GetExecutionContext();
-        if (exe_ctx.HasProcessScope())
-          exe_ctx.GetProcessRef().Destroy(false);
-      }
-      return eKeyHandled;
-
     case 'c':
       // 'c' == continue
       {
@@ -3937,8 +3927,12 @@ void IOHandlerCursesGUI::Activate() {
                                     ApplicationDelegate::eMenuID_Process));
     process_menu_sp->AddSubmenu(MenuSP(new Menu(
         "Attach", nullptr, 'a', ApplicationDelegate::eMenuID_ProcessAttach)));
-    process_menu_sp->AddSubmenu(MenuSP(new Menu(
-        "Detach", nullptr, 'd', ApplicationDelegate::eMenuID_ProcessDetach)));
+    process_menu_sp->AddSubmenu(
+        MenuSP(new Menu("Detach and resume", nullptr, 'd',
+                        ApplicationDelegate::eMenuID_ProcessDetachResume)));
+    process_menu_sp->AddSubmenu(
+        MenuSP(new Menu("Detach suspended", nullptr, 's',
+                        ApplicationDelegate::eMenuID_ProcessDetachSuspended)));
     process_menu_sp->AddSubmenu(MenuSP(new Menu(
         "Launch", nullptr, 'l', ApplicationDelegate::eMenuID_ProcessLaunch)));
     process_menu_sp->AddSubmenu(MenuSP(new Menu(Menu::Type::Separator)));

From 336c7029ef3b7eff981d6c6990a6a255a4de236e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Lubo=C5=A1=20Lu=C5=88=C3=A1k?= <l.lunak@centrum.cz>
Date: Fri, 11 Oct 2019 21:59:50 +0200
Subject: [PATCH 0542/1035] [lldb] change shortcut for 'step out' from 'o' to
 'f'

This makes it consistent with gdb tui, where 'f' is 'finish'.
See the discussion at https://reviews.llvm.org/D68541 .

Differential Revision: https://reviews.llvm.org/D68909
---
 lldb/source/Core/IOHandlerCursesGUI.cpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lldb/source/Core/IOHandlerCursesGUI.cpp b/lldb/source/Core/IOHandlerCursesGUI.cpp
index 52d772c41ebc9..52bf889456cea 100644
--- a/lldb/source/Core/IOHandlerCursesGUI.cpp
+++ b/lldb/source/Core/IOHandlerCursesGUI.cpp
@@ -3240,7 +3240,7 @@ class SourceFileWindowDelegate : public WindowDelegate {
         {'h', "Show help dialog"},
         {'n', "Step over (source line)"},
         {'N', "Step over (single instruction)"},
-        {'o', "Step out"},
+        {'f', "Step out (finish)"},
         {'s', "Step in (source line)"},
         {'S', "Step in (single instruction)"},
         {',', "Page up"},
@@ -3818,8 +3818,8 @@ class SourceFileWindowDelegate : public WindowDelegate {
       }
       return eKeyHandled;
 
-    case 'o':
-      // 'o' == step out
+    case 'f':
+      // 'f' == step out (finish)
       {
         ExecutionContext exe_ctx =
             m_debugger.GetCommandInterpreter().GetExecutionContext();

From 13978643b678e226500d9d1ae8a003c8ed4b05ef Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Lubo=C5=A1=20Lu=C5=88=C3=A1k?= <l.lunak@centrum.cz>
Date: Sat, 5 Oct 2019 21:14:01 +0200
Subject: [PATCH 0543/1035] [lldb] implement 'up' and 'down' shortcuts in lldb
 gui

Also add a unittest.

Differential Revision: https://reviews.llvm.org/D68541
---
 .../test/commands/gui/basicdebug/Makefile     |  2 +
 .../gui/basicdebug/TestGuiBasicDebug.py       | 42 +++++++++++++++++++
 .../test/commands/gui/basicdebug/func.c       |  3 ++
 .../test/commands/gui/basicdebug/main.c       |  7 ++++
 lldb/source/Core/IOHandlerCursesGUI.cpp       | 22 ++++++++++
 5 files changed, 76 insertions(+)
 create mode 100644 lldb/packages/Python/lldbsuite/test/commands/gui/basicdebug/Makefile
 create mode 100644 lldb/packages/Python/lldbsuite/test/commands/gui/basicdebug/TestGuiBasicDebug.py
 create mode 100644 lldb/packages/Python/lldbsuite/test/commands/gui/basicdebug/func.c
 create mode 100644 lldb/packages/Python/lldbsuite/test/commands/gui/basicdebug/main.c

diff --git a/lldb/packages/Python/lldbsuite/test/commands/gui/basicdebug/Makefile b/lldb/packages/Python/lldbsuite/test/commands/gui/basicdebug/Makefile
new file mode 100644
index 0000000000000..7b75ed77849f2
--- /dev/null
+++ b/lldb/packages/Python/lldbsuite/test/commands/gui/basicdebug/Makefile
@@ -0,0 +1,2 @@
+C_SOURCES := main.c func.c
+include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/commands/gui/basicdebug/TestGuiBasicDebug.py b/lldb/packages/Python/lldbsuite/test/commands/gui/basicdebug/TestGuiBasicDebug.py
new file mode 100644
index 0000000000000..54c763fd2afc2
--- /dev/null
+++ b/lldb/packages/Python/lldbsuite/test/commands/gui/basicdebug/TestGuiBasicDebug.py
@@ -0,0 +1,42 @@
+"""
+Test the 'gui' shortcuts 's','n','f','u','d' (step in, step over, step out, up, down)
+"""
+
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test.lldbpexpect import PExpectTest
+
+class TestGuiBasicDebugCommandTest(PExpectTest):
+
+    mydir = TestBase.compute_mydir(__file__)
+
+    @skipIfCursesSupportMissing
+    def test_gui(self):
+        self.build()
+
+        self.launch(executable=self.getBuildArtifact("a.out"), dimensions=(100,500))
+        self.expect('br set -f main.c -p "// Break here"', substrs=["Breakpoint 1", "address ="])
+        self.expect("run", substrs=["stop reason ="])
+
+        escape_key = chr(27).encode()
+
+        # Start the GUI and close the welcome window.
+        self.child.sendline("gui")
+        self.child.send(escape_key)
+
+        # Simulate a simple debugging session.
+        self.child.send("s") # step
+        self.child.expect("return 1; // In function[^\r\n]+<<< Thread 1: step in")
+        self.child.send("u") # up
+        self.child.expect_exact("func(); // Break here")
+        self.child.send("d") # down
+        self.child.expect_exact("return 1; // In function")
+        self.child.send("f") # finish
+        self.child.expect("func\(\); // Break here[^\r\n]+<<< Thread 1: step out")
+        self.child.send("s") # move onto the second one
+        self.child.expect("func\(\); // Second[^\r\n]+<<< Thread 1: step in")
+        self.child.send("n") # step over
+        self.child.expect("return 0;[^\r\n]+<<< Thread 1: step over")
+
+        self.quit()
diff --git a/lldb/packages/Python/lldbsuite/test/commands/gui/basicdebug/func.c b/lldb/packages/Python/lldbsuite/test/commands/gui/basicdebug/func.c
new file mode 100644
index 0000000000000..f404a177af4f7
--- /dev/null
+++ b/lldb/packages/Python/lldbsuite/test/commands/gui/basicdebug/func.c
@@ -0,0 +1,3 @@
+int func() {
+  return 1; // In function
+}
diff --git a/lldb/packages/Python/lldbsuite/test/commands/gui/basicdebug/main.c b/lldb/packages/Python/lldbsuite/test/commands/gui/basicdebug/main.c
new file mode 100644
index 0000000000000..f776fb99898de
--- /dev/null
+++ b/lldb/packages/Python/lldbsuite/test/commands/gui/basicdebug/main.c
@@ -0,0 +1,7 @@
+extern int func();
+
+int main(int argc, char **argv) {
+  func(); // Break here
+  func(); // Second
+  return 0;
+}
diff --git a/lldb/source/Core/IOHandlerCursesGUI.cpp b/lldb/source/Core/IOHandlerCursesGUI.cpp
index 52bf889456cea..fda3aa1886795 100644
--- a/lldb/source/Core/IOHandlerCursesGUI.cpp
+++ b/lldb/source/Core/IOHandlerCursesGUI.cpp
@@ -3243,6 +3243,8 @@ class SourceFileWindowDelegate : public WindowDelegate {
         {'f', "Step out (finish)"},
         {'s', "Step in (source line)"},
         {'S', "Step in (single instruction)"},
+        {'u', "Frame up"},
+        {'d', "Frame down"},
         {',', "Page up"},
         {'.', "Page down"},
         {'\0', nullptr}};
@@ -3856,6 +3858,26 @@ class SourceFileWindowDelegate : public WindowDelegate {
     }
       return eKeyHandled;
 
+    case 'u': // 'u' == frame up
+    case 'd': // 'd' == frame down
+    {
+      ExecutionContext exe_ctx =
+          m_debugger.GetCommandInterpreter().GetExecutionContext();
+      if (exe_ctx.HasThreadScope()) {
+        Thread *thread = exe_ctx.GetThreadPtr();
+        uint32_t frame_idx = thread->GetSelectedFrameIndex();
+        if (frame_idx == UINT32_MAX)
+          frame_idx = 0;
+        if (c == 'u' && frame_idx + 1 < thread->GetStackFrameCount())
+          ++frame_idx;
+        else if (c == 'd' && frame_idx > 0)
+          --frame_idx;
+        if (thread->SetSelectedFrameByIndex(frame_idx, true))
+          exe_ctx.SetFrameSP(thread->GetSelectedFrame());
+      }
+    }
+      return eKeyHandled;
+
     case 'h':
       window.CreateHelpSubwindow();
       return eKeyHandled;

From 755f91f12cf08da9c6553de901135945f2c6083a Mon Sep 17 00:00:00 2001
From: Philip Reames <listmail@philipreames.com>
Date: Wed, 29 Jul 2020 13:28:26 -0700
Subject: [PATCH 0544/1035] [Statepoint] Enable cross block relocates w/vreg
 lowering

This change is mechanical, it just removes the restriction and updates tests.  The key building blocks were submitted in 31342eb and 8fe2abc.

Note that this (and preceeding changes) entirely subsumes D83965.  I did includes a couple of it's tests.

From the codegen changes, an interesting observation: this doesn't actual reduce spilling, it just let's the register allocator do it's job.  That results in a slightly different overall result which has both pros and cons over the eager spill lowering.  (i.e. We'll have some perf tuning to do once this is stable.)
---
 .../SelectionDAG/StatepointLowering.cpp       |  10 -
 .../CodeGen/X86/statepoint-vreg-details.ll    |  20 +-
 llvm/test/CodeGen/X86/statepoint-vreg.ll      | 179 +++++++++++++++++-
 3 files changed, 179 insertions(+), 30 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
index 044376c666706..f5d29c11790c3 100644
--- a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
@@ -545,16 +545,6 @@ lowerStatepointMetaArgs(SmallVectorImpl<SDValue> &Ops,
   const unsigned MaxTiedRegs = 15; // Max  number of tied regs MI can have.
   unsigned MaxVRegPtrs =
       std::min(MaxTiedRegs, MaxRegistersForGCPointers.getValue());
-  // Use old spill scheme for cross-block relocates.
-  if (SI.StatepointInstr) {
-    const BasicBlock *BB = SI.StatepointInstr->getParent();
-    bool NonLocalReloc =
-        llvm::any_of(SI.GCRelocates, [BB](const GCRelocateInst *R) {
-          return R->getParent() != BB;
-        });
-    if (NonLocalReloc)
-      MaxVRegPtrs = 0;
-  }
 
   LLVM_DEBUG(dbgs() << "Desiding how to lower GC Pointers:\n");
   unsigned CurNumVRegs = 0;
diff --git a/llvm/test/CodeGen/X86/statepoint-vreg-details.ll b/llvm/test/CodeGen/X86/statepoint-vreg-details.ll
index bfd08f983103b..16939d770fed1 100644
--- a/llvm/test/CodeGen/X86/statepoint-vreg-details.ll
+++ b/llvm/test/CodeGen/X86/statepoint-vreg-details.ll
@@ -215,27 +215,25 @@ define i1 @test_cross_bb(i32 addrspace(1)* %a, i1 %external_cond) gc "statepoint
 ; CHECK-VREG:  bb.0.entry:
 ; CHECK-VREG:         %1:gr32 = COPY $esi
 ; CHECK-VREG-NEXT:    %0:gr64 = COPY $rdi
-; CHECK-VREG-NEXT:    %3:gr8 = COPY %1.sub_8bit
-; CHECK-VREG-NEXT:    MOV64mr %stack.0, 1, $noreg, 0, $noreg, %0 :: (store 8 into %stack.0)
+; CHECK-VREG-NEXT:    %4:gr8 = COPY %1.sub_8bit
 ; CHECK-VREG-NEXT:    ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
-; CHECK-VREG-NEXT:    STATEPOINT 0, 0, 0, @return_i1, 2, 0, 2, 0, 2, 0, 1, 8, %stack.0, 0, 1, 8, %stack.0, 0, csr_64, implicit-def $rsp, implicit-def $ssp, implicit-def $al :: (volatile load store 8 on %stack.0)
+; CHECK-VREG-NEXT:    %2:gr64 = STATEPOINT 0, 0, 0, @return_i1, 2, 0, 2, 0, 2, 0, %0, %0(tied-def 0), csr_64, implicit-def $rsp, implicit-def $ssp, implicit-def $al
 ; CHECK-VREG-NEXT:    ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
-; CHECK-VREG-NEXT:    %4:gr8 = COPY $al
-; CHECK-VREG-NEXT:    %2:gr8 = COPY %4
-; CHECK-VREG-NEXT:    TEST8ri killed %3, 1, implicit-def $eflags
+; CHECK-VREG-NEXT:    %5:gr8 = COPY $al
+; CHECK-VREG-NEXT:    %3:gr8 = COPY %5
+; CHECK-VREG-NEXT:    TEST8ri killed %4, 1, implicit-def $eflags
 ; CHECK-VREG-NEXT:    JCC_1 %bb.2, 4, implicit $eflags
 ; CHECK-VREG-NEXT:    JMP_1 %bb.1
 ; CHECK-VREG:       bb.1.left:
-; CHECK-VREG-NEXT:    %6:gr64 = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load 8 from %stack.0)
 ; CHECK-VREG-NEXT:    ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
-; CHECK-VREG-NEXT:    $rdi = COPY %6
+; CHECK-VREG-NEXT:    $rdi = COPY %2
 ; CHECK-VREG-NEXT:    CALL64pcrel32 @consume, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit-def $rsp, implicit-def $ssp
 ; CHECK-VREG-NEXT:    ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
-; CHECK-VREG-NEXT:    $al = COPY %2
+; CHECK-VREG-NEXT:    $al = COPY %3
 ; CHECK-VREG-NEXT:    RET 0, $al
 ; CHECK-VREG:       bb.2.right:
-; CHECK-VREG-NEXT:    %5:gr8 = MOV8ri 1
-; CHECK-VREG-NEXT:    $al = COPY %5
+; CHECK-VREG-NEXT:    %6:gr8 = MOV8ri 1
+; CHECK-VREG-NEXT:    $al = COPY %6
 ; CHECK-VREG-NEXT:    RET 0, $al
 
 entry:
diff --git a/llvm/test/CodeGen/X86/statepoint-vreg.ll b/llvm/test/CodeGen/X86/statepoint-vreg.ll
index 8c69af77c2f2e..6e6240d737ec4 100644
--- a/llvm/test/CodeGen/X86/statepoint-vreg.ll
+++ b/llvm/test/CodeGen/X86/statepoint-vreg.ll
@@ -6,10 +6,12 @@ target triple = "x86_64-pc-linux-gnu"
 
 declare i1 @return_i1()
 declare void @func()
+declare void @"some_call"(i64 addrspace(1)*)
 declare void @consume(i32 addrspace(1)*)
 declare void @consume2(i32 addrspace(1)*, i32 addrspace(1)*)
 declare void @consume5(i32 addrspace(1)*, i32 addrspace(1)*, i32 addrspace(1)*, i32 addrspace(1)*, i32 addrspace(1)*)
 declare void @use1(i32 addrspace(1)*, i8 addrspace(1)*)
+declare i32 @"personality_function"()
 
 ; test most simple relocate
 define i1 @test_relocate(i32 addrspace(1)* %a) gc "statepoint-example" {
@@ -231,30 +233,31 @@ define i1 @test_cross_bb(i32 addrspace(1)* %a, i1 %external_cond) gc "statepoint
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    pushq %rbp
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    pushq %r14
 ; CHECK-NEXT:    .cfi_def_cfa_offset 24
-; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    pushq %rbx
 ; CHECK-NEXT:    .cfi_def_cfa_offset 32
-; CHECK-NEXT:    .cfi_offset %rbx, -24
+; CHECK-NEXT:    .cfi_offset %rbx, -32
+; CHECK-NEXT:    .cfi_offset %r14, -24
 ; CHECK-NEXT:    .cfi_offset %rbp, -16
 ; CHECK-NEXT:    movl %esi, %ebp
-; CHECK-NEXT:    movq %rdi, (%rsp)
+; CHECK-NEXT:    movq %rdi, %rbx
 ; CHECK-NEXT:    callq return_i1
 ; CHECK-NEXT:  .Ltmp7:
 ; CHECK-NEXT:    testb $1, %bpl
 ; CHECK-NEXT:    je .LBB7_2
 ; CHECK-NEXT:  # %bb.1: # %left
-; CHECK-NEXT:    movl %eax, %ebx
-; CHECK-NEXT:    movq (%rsp), %rdi
+; CHECK-NEXT:    movl %eax, %r14d
+; CHECK-NEXT:    movq %rbx, %rdi
 ; CHECK-NEXT:    callq consume
-; CHECK-NEXT:    movl %ebx, %eax
+; CHECK-NEXT:    movl %r14d, %eax
 ; CHECK-NEXT:    jmp .LBB7_3
 ; CHECK-NEXT:  .LBB7_2: # %right
 ; CHECK-NEXT:    movb $1, %al
 ; CHECK-NEXT:  .LBB7_3: # %right
-; CHECK-NEXT:    addq $8, %rsp
-; CHECK-NEXT:    .cfi_def_cfa_offset 24
 ; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r14
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    popq %rbp
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
@@ -374,9 +377,167 @@ entry:
   ret void
 }
 
+define i64 addrspace(1)* @test_basic_invoke(i64 addrspace(1)* %obj,
+; CHECK-LABEL: test_basic_invoke:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    .cfi_offset %rbx, -24
+; CHECK-NEXT:    .cfi_offset %r14, -16
+; CHECK-NEXT:    movq %rsi, %r14
+; CHECK-NEXT:    movq %rdi, %rbx
+; CHECK-NEXT:  .Ltmp12:
+; CHECK-NEXT:    callq some_call
+; CHECK-NEXT:  .Ltmp15:
+; CHECK-NEXT:  .Ltmp13:
+; CHECK-NEXT:  # %bb.1: # %normal_return
+; CHECK-NEXT:    movq %rbx, %rax
+; CHECK-NEXT:  .LBB11_2: # %normal_return
+; CHECK-NEXT:    addq $8, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB11_3: # %exceptional_return
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:  .Ltmp14:
+; CHECK-NEXT:    movq %r14, %rax
+; CHECK-NEXT:    jmp .LBB11_2
+                                            i64 addrspace(1)* %obj1) gc "statepoint-example" personality i32 ()* @"personality_function" {
+entry:
+  %0 = invoke token (i64, i32, void (i64 addrspace(1)*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidp1i64f(i64 0, i32 0, void (i64 addrspace(1)*)* @some_call, i32 1, i32 0, i64 addrspace(1)* %obj, i32 0, i32 0) ["gc-live" (i64 addrspace(1)* %obj, i64 addrspace(1)* %obj1), "deopt" (i32 0, i32 -1, i32 0, i32 0, i32 0)]
+          to label %invoke_safepoint_normal_dest unwind label %exceptional_return
+
+invoke_safepoint_normal_dest:
+  %obj.relocated = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(token %0, i32 0, i32 0)
+  %obj1.relocated = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(token %0, i32 1, i32 1)
+  br label %normal_return
+
+normal_return:
+  ret i64 addrspace(1)* %obj.relocated
+
+exceptional_return:
+  %landing_pad = landingpad token
+          cleanup
+  %obj.relocated1 = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(token %landing_pad, i32 0, i32 0)
+  %obj1.relocated1 = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(token %landing_pad, i32 1, i32 1)
+  ret i64 addrspace(1)* %obj1.relocated1
+}
+
+define i64 addrspace(1)* @test_invoke_same_val(i1 %cond, i64 addrspace(1)* %val1, i64 addrspace(1)* %val2, i64 addrspace(1)* %val3) gc "statepoint-example" personality i32 ()* @"personality_function" {
+; CHECK-LABEL: test_invoke_same_val:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    .cfi_offset %rbx, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq %rdx, %rbx
+; CHECK-NEXT:    movq %rsi, %rbp
+; CHECK-NEXT:    movl %edi, %r14d
+; CHECK-NEXT:    testb $1, %r14b
+; CHECK-NEXT:    je .LBB12_2
+; CHECK-NEXT:  # %bb.1: # %left
+; CHECK-NEXT:  .Ltmp19:
+; CHECK-NEXT:    movq %rbp, %rdi
+; CHECK-NEXT:    callq some_call
+; CHECK-NEXT:  .Ltmp22:
+; CHECK-NEXT:  .Ltmp20:
+; CHECK-NEXT:    jmp .LBB12_4
+; CHECK-NEXT:  .LBB12_2: # %right
+; CHECK-NEXT:    movq %rcx, %r15
+; CHECK-NEXT:  .Ltmp16:
+; CHECK-NEXT:    movq %rbp, %rdi
+; CHECK-NEXT:    callq some_call
+; CHECK-NEXT:  .Ltmp23:
+; CHECK-NEXT:  .Ltmp17:
+; CHECK-NEXT:  # %bb.3: # %right.relocs
+; CHECK-NEXT:    movq %r15, %rbp
+; CHECK-NEXT:  .LBB12_4: # %normal_return
+; CHECK-NEXT:    testb $1, %r14b
+; CHECK-NEXT:    cmoveq %rbx, %rbp
+; CHECK-NEXT:  .LBB12_6: # %exceptional_return.left
+; CHECK-NEXT:    movq %rbp, %rax
+; CHECK-NEXT:  .LBB12_7: # %exceptional_return.left
+; CHECK-NEXT:    addq $8, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB12_8: # %exceptional_return.right
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:  .Ltmp18:
+; CHECK-NEXT:    movq %rbx, %rax
+; CHECK-NEXT:    jmp .LBB12_7
+; CHECK-NEXT:  .LBB12_5: # %exceptional_return.left
+; CHECK-NEXT:  .Ltmp21:
+; CHECK-NEXT:    jmp .LBB12_6
+entry:
+  br i1 %cond, label %left, label %right
+
+left:
+  %sp1 = invoke token (i64, i32, void (i64 addrspace(1)*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidp1i64f(i64 0, i32 0, void (i64 addrspace(1)*)* @some_call, i32 1, i32 0, i64 addrspace(1)* %val1, i32 0, i32 0) ["gc-live"(i64 addrspace(1)* %val1, i64 addrspace(1)* %val2)]
+           to label %left.relocs unwind label %exceptional_return.left
+
+left.relocs:
+  %val1.relocated = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(token %sp1, i32 0, i32 0)
+  %val2.relocated_left = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(token %sp1, i32 1, i32 1)
+  br label %normal_return
+
+right:
+  %sp2 = invoke token (i64, i32, void (i64 addrspace(1)*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidp1i64f(i64 0, i32 0, void (i64 addrspace(1)*)* @some_call, i32 1, i32 0, i64 addrspace(1)* %val1, i32 0, i32 0) ["gc-live"(i64 addrspace(1)* %val2, i64 addrspace(1)* %val3)]
+           to label %right.relocs unwind label %exceptional_return.right
+
+right.relocs:
+  %val2.relocated_right = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(token %sp2, i32 0, i32 0)
+  %val3.relocated = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(token %sp2, i32 1, i32 1)
+  br label %normal_return
+
+normal_return:
+  %a1 = phi i64 addrspace(1)* [%val1.relocated, %left.relocs], [%val3.relocated, %right.relocs]
+  %a2 = phi i64 addrspace(1)* [%val2.relocated_left, %left.relocs], [%val2.relocated_right, %right.relocs]
+  %ret = select i1 %cond, i64 addrspace(1)* %a1, i64 addrspace(1)* %a2
+  ret i64 addrspace(1)* %ret
+
+exceptional_return.left:
+  %landing_pad = landingpad token
+          cleanup
+  %val.relocated2 = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(token %landing_pad, i32 0, i32 0)
+  ret i64 addrspace(1)* %val.relocated2
+
+exceptional_return.right:
+  %landing_pad1 = landingpad token
+          cleanup
+  %val.relocated3 = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(token %landing_pad1, i32 0, i32 0)
+  ret i64 addrspace(1)* %val.relocated3
+}
+
 declare token @llvm.experimental.gc.statepoint.p0f_i1f(i64, i32, i1 ()*, i32, i32, ...)
 declare token @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...)
+declare token @llvm.experimental.gc.statepoint.p0f_isVoidp1i64f(i64, i32, void (i64 addrspace(1)*)*, i32, i32, ...)
 declare i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token, i32, i32)
+declare i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(token, i32, i32)
 declare i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token, i32, i32)
 declare <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token, i32, i32)
 declare i1 @llvm.experimental.gc.result.i1(token)

From e14a7ff7627500eb4d4f58041d9f5288d9c78659 Mon Sep 17 00:00:00 2001
From: cgyurgyik <gyurgyikcp@gmail.com>
Date: Wed, 29 Jul 2020 16:29:19 -0400
Subject: [PATCH 0545/1035] [libc] Adds fuzz test for strstr and alphabetizes
 string fuzz CMakeList.

Reviewed By: sivachandra

Differential Revision: https://reviews.llvm.org/D84611
---
 libc/fuzzing/string/CMakeLists.txt  | 15 ++++-
 libc/fuzzing/string/strstr_fuzz.cpp | 85 +++++++++++++++++++++++++++++
 2 files changed, 97 insertions(+), 3 deletions(-)
 create mode 100644 libc/fuzzing/string/strstr_fuzz.cpp

diff --git a/libc/fuzzing/string/CMakeLists.txt b/libc/fuzzing/string/CMakeLists.txt
index db3def324fccd..1885ee5f66ebf 100644
--- a/libc/fuzzing/string/CMakeLists.txt
+++ b/libc/fuzzing/string/CMakeLists.txt
@@ -1,3 +1,11 @@
+add_libc_fuzzer(
+  strcmp_fuzz
+  SRCS
+    strcmp_fuzz.cpp
+  DEPENDS
+    libc.src.string.strcmp
+)
+
 add_libc_fuzzer(
   strcpy_fuzz
   SRCS
@@ -9,9 +17,10 @@ add_libc_fuzzer(
 )
 
 add_libc_fuzzer(
-  strcmp_fuzz
+  strstr_fuzz
   SRCS
-    strcmp_fuzz.cpp
+    strstr_fuzz.cpp
   DEPENDS
-    libc.src.string.strcmp
+    libc.src.string.strstr
+    libc.src.string.strlen
 )
diff --git a/libc/fuzzing/string/strstr_fuzz.cpp b/libc/fuzzing/string/strstr_fuzz.cpp
new file mode 100644
index 0000000000000..5ccf85c7dc0fc
--- /dev/null
+++ b/libc/fuzzing/string/strstr_fuzz.cpp
@@ -0,0 +1,85 @@
+//===-- strstr_fuzz.cpp ---------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// Fuzzing test for llvm-libc strstr implementation.
+///
+//===----------------------------------------------------------------------===//
+
+#include "src/string/strlen.h"
+#include "src/string/strstr.h"
+#include <stddef.h>
+#include <stdint.h>
+
+// Simple loop to compare two strings up to a size n.
+static int simple_memcmp(const char *left, const char *right, size_t n) {
+  for (; n && *left == *right; ++left, ++right, --n)
+    ;
+  return n ? *left - *right : 0;
+}
+
+// The general structure is to take the value of the first byte, set size1 to
+// that value, and add the null terminator. size2 will then contain the rest of
+// the bytes in data.
+// For example, with inputs (data={2, 6, 4, 8, 0}, size=5):
+//         size1: data[0] = 2
+//         data1: {2, 6} + '\0' = {2, 6, '\0'}
+//         size2: size - size1 = 3
+//         data2: {4, 8, '\0'}
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  // Verify the size is at least 1 and the data is null terminated.
+  if (!size || data[size - 1] != '\0')
+    return 0;
+  const size_t size1 = (data[0] <= size ? data[0] : size);
+  // The first size will always be at least 1 since
+  // we need to append the null terminator. The second size
+  // needs to be checked since it must also contain the null
+  // terminator.
+  if (size - size1 == 0)
+    return 0;
+
+  // Copy the data into a new container.
+  uint8_t *container = new uint8_t[size1 + 1];
+  if (!container)
+    __builtin_trap();
+
+  size_t i;
+  for (i = 0; i < size1; ++i)
+    container[i] = data[i];
+  container[size1] = '\0'; // Add null terminator to container.
+
+  const char *needle = reinterpret_cast<const char *>(container);
+  const char *haystack = reinterpret_cast<const char *>(data + i);
+  const char *result = __llvm_libc::strstr(haystack, needle);
+
+  // A null terminator may exist earlier in each, so this needs to be recorded.
+  const size_t haystack_size = __llvm_libc::strlen(haystack);
+  const size_t needle_size = __llvm_libc::strlen(needle);
+
+  if (result) {
+    // The needle is in the haystack.
+    // 1. Verify that the result matches the needle.
+    if (simple_memcmp(needle, result, needle_size) != 0)
+      __builtin_trap();
+
+    const char *haystack_ptr = haystack;
+    // 2. Verify that the result is the first occurrence of the needle.
+    for (; haystack_ptr != result; ++haystack_ptr) {
+      if (simple_memcmp(needle, haystack_ptr, needle_size) == 0)
+        __builtin_trap(); // There was an earlier occurrence of the needle.
+    }
+  } else {
+    // No result was found. Verify that the needle doesn't exist within the
+    // haystack.
+    for (size_t i = 0; i + needle_size < haystack_size; ++i) {
+      if (simple_memcmp(needle, haystack + i, needle_size) == 0)
+        __builtin_trap(); // There was an earlier occurrence of the needle.
+    }
+  }
+  delete[] container;
+  return 0;
+}

From b08abf4c808e98718b8806dafcae1626328676d4 Mon Sep 17 00:00:00 2001
From: Johannes Doerfert <johannes@jdoerfert.de>
Date: Wed, 29 Jul 2020 15:46:43 -0500
Subject: [PATCH 0546/1035] [OpenMP] Fix D83281 issue on windows by allowing
 `dso_local` in CHECK [2/1]

The problem with 8723280b68b1e5ed97a699466720b36a32a9e406 was that the
`dso_local` is *before* the void not after. Hope this works.
---
 .../declare_variant_device_isa_codegen_1.c       | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/clang/test/OpenMP/declare_variant_device_isa_codegen_1.c b/clang/test/OpenMP/declare_variant_device_isa_codegen_1.c
index 25fc3941dd504..76a3eedeae301 100644
--- a/clang/test/OpenMP/declare_variant_device_isa_codegen_1.c
+++ b/clang/test/OpenMP/declare_variant_device_isa_codegen_1.c
@@ -31,18 +31,18 @@ void avx512_saxpy(int n, float s, float *x, float *y) {
 }
 
 void caller(int n, float s, float *x, float *y) {
-  // GENERIC:     define void {{.*}}caller
-  // GENERIC:      call void {{.*}}base_saxpy
-  // WITHFEATURE: define void {{.*}}caller
-  // WITHFEATURE:  call void {{.*}}avx512_saxpy
+  // GENERIC:     define {{.*}}void @{{.*}}caller
+  // GENERIC:      call void @{{.*}}base_saxpy
+  // WITHFEATURE: define {{.*}}void @{{.*}}caller
+  // WITHFEATURE:  call void @{{.*}}avx512_saxpy
   base_saxpy(n, s, x, y);
 }
 
 __attribute__((target("avx512f"))) void variant_caller(int n, float s, float *x, float *y) {
-  // GENERIC:     define void {{.*}}variant_caller
-  // GENERIC:      call void {{.*}}avx512_saxpy
-  // WITHFEATURE: define void {{.*}}variant_caller
-  // WITHFEATURE:  call void {{.*}}avx512_saxpy
+  // GENERIC:     define {{.*}}void @{{.*}}variant_caller
+  // GENERIC:      call void @{{.*}}avx512_saxpy
+  // WITHFEATURE: define {{.*}}void @{{.*}}variant_caller
+  // WITHFEATURE:  call void @{{.*}}avx512_saxpy
   base_saxpy(n, s, x, y);
 }
 

From 01aa14784b0cfc44a977080a2a17d6dbb0c77818 Mon Sep 17 00:00:00 2001
From: Arthur Eubanks <aeubanks@google.com>
Date: Wed, 22 Jul 2020 08:40:30 -0700
Subject: [PATCH 0547/1035] [NewPM][opt] Revert to legacy PM when any codegen
 passes are specified

This reduces the number of check-llvm failures by 500.

Ideally we'd have a codegen version of PassRegistry.def, or have all the
codegen passes ported and put into PassRegistry.def. But since that
doesn't exist yet, hardcode the list of codegen IR passes.

There are still codegen passes missing from this list, I'll add them
later as I stumble upon them.

Reviewed By: asbirlea, ychen

Differential Revision: https://reviews.llvm.org/D84872
---
 llvm/tools/opt/opt.cpp | 42 +++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 41 insertions(+), 1 deletion(-)

diff --git a/llvm/tools/opt/opt.cpp b/llvm/tools/opt/opt.cpp
index ed3e506a609fc..4a985c50fcd51 100644
--- a/llvm/tools/opt/opt.cpp
+++ b/llvm/tools/opt/opt.cpp
@@ -486,6 +486,41 @@ struct TimeTracerRAII {
   }
 };
 
+// For use in NPM transition.
+// TODO: use a codegen version of PassRegistry.def/PassBuilder::is*Pass() once
+// it exists.
+static bool IsCodegenPass(StringRef Pass) {
+  std::vector<StringRef> PassNamePrefix = {
+      "x86-",    "xcore-", "wasm-",  "systemz-", "ppc-",    "nvvm-",
+      "nvptx-",  "mips-",  "lanai-", "hexagon-", "bpf-",    "avr-",
+      "thumb2-", "arm-",   "si-",    "gcn-",     "amdgpu-", "aarch64-"};
+  std::vector<StringRef> PassNameContain = {"ehprepare"};
+  std::vector<StringRef> PassNameExact = {
+      "safe-stack",           "cost-model",
+      "codegenprepare",       "interleaved-load-combine",
+      "unreachableblockelim", "sclaraized-masked-mem-intrin"};
+  for (const auto &P : PassNamePrefix)
+    if (Pass.startswith(P))
+      return true;
+  for (const auto &P : PassNameContain)
+    if (Pass.contains(P))
+      return true;
+  for (const auto &P : PassNamePrefix)
+    if (Pass == P)
+      return true;
+  return false;
+}
+
+// For use in NPM transition.
+static bool CodegenPassSpecifiedInPassList() {
+  for (const auto &P : PassList) {
+    StringRef Arg = P->getPassArgument();
+    if (IsCodegenPass(Arg))
+      return true;
+  }
+  return false;
+}
+
 //===----------------------------------------------------------------------===//
 // main for opt
 //
@@ -685,7 +720,12 @@ int main(int argc, char **argv) {
   if (OutputThinLTOBC)
     M->addModuleFlag(Module::Error, "EnableSplitLTOUnit", SplitLTOUnit);
 
-  if (EnableNewPassManager || PassPipeline.getNumOccurrences() > 0) {
+  // If `-passes=` is specified, use NPM.
+  // If `-enable-new-pm` is specified and there are no codegen passes, use NPM.
+  // e.g. `-enable-new-pm -sroa` will use NPM.
+  // but `-enable-new-pm -codegenprepare` will still revert to legacy PM.
+  if ((EnableNewPassManager && !CodegenPassSpecifiedInPassList()) ||
+      PassPipeline.getNumOccurrences() > 0) {
     if (PassPipeline.getNumOccurrences() > 0 && PassList.size() > 0) {
       errs()
           << "Cannot specify passes via both -foo-pass and --passes=foo-pass";

From 120e66b3418b37b95fc1dbbb23e296a602a24fa8 Mon Sep 17 00:00:00 2001
From: Hiroshi Yamauchi <yamauchi@google.com>
Date: Tue, 28 Jul 2020 10:09:49 -0700
Subject: [PATCH 0548/1035] [PGO] Include the mem ops into the function hash.

To avoid hash collisions when the only difference is in mem ops.

Differential Revision: https://reviews.llvm.org/D84782
---
 .../CodeGen/Inputs/thinlto_expect1.proftext   |  2 +-
 .../CodeGen/Inputs/thinlto_expect2.proftext   |  4 +-
 .../CodeGenCXX/Inputs/profile-remap.proftext  |  2 +-
 .../Inputs/profile-remap_entry.proftext       |  2 +-
 .../Inputs/gcc-flag-compatibility_IR.proftext |  2 +-
 .../gcc-flag-compatibility_IR_entry.proftext  |  2 +-
 .../profile/Linux/instrprof-value-merge.c     |  2 +-
 .../Instrumentation/PGOInstrumentation.cpp    | 49 +++++++++++++++----
 .../PGOProfile/Inputs/PR41279.proftext        |  2 +-
 .../PGOProfile/Inputs/PR41279_2.proftext      |  2 +-
 .../PGOProfile/Inputs/branch1.proftext        |  2 +-
 .../Inputs/branch1_large_count.proftext       |  2 +-
 .../PGOProfile/Inputs/branch2.proftext        |  2 +-
 .../PGOProfile/Inputs/branch2_entry.proftext  |  2 +-
 .../PGOProfile/Inputs/criticaledge.proftext   |  4 +-
 .../Inputs/criticaledge_entry.proftext        |  4 +-
 .../PGOProfile/Inputs/cspgo.proftext          | 16 +++---
 .../Inputs/diag_no_value_sites.proftext       |  2 +-
 .../Inputs/fix_entry_count.proftext           |  2 +-
 .../PGOProfile/Inputs/func_entry.proftext     |  6 +--
 .../PGOProfile/Inputs/indirect_call.proftext  |  8 +--
 .../PGOProfile/Inputs/indirectbr.proftext     |  2 +-
 .../Inputs/indirectbr_entry.proftext          |  2 +-
 .../PGOProfile/Inputs/irreducible.proftext    |  4 +-
 .../Inputs/irreducible_entry.proftext         |  4 +-
 .../PGOProfile/Inputs/landingpad.proftext     |  4 +-
 .../Inputs/landingpad_entry.proftext          |  4 +-
 .../Inputs/large_count_remarks.proftext       |  2 +-
 .../PGOProfile/Inputs/loop1.proftext          |  2 +-
 .../PGOProfile/Inputs/loop1_entry.proftext    |  2 +-
 .../PGOProfile/Inputs/loop2.proftext          |  2 +-
 .../PGOProfile/Inputs/loop2_entry.proftext    |  2 +-
 .../Inputs/memop_size_annotation.proftext     |  2 +-
 .../Inputs/misexpect-branch-correct.proftext  |  2 +-
 .../Inputs/misexpect-branch.proftext          |  2 +-
 .../Inputs/misexpect-branch_entry.proftext    |  2 +-
 .../Inputs/misexpect-switch-correct.proftext  |  2 +-
 .../misexpect-switch-correct_entry.proftext   |  2 +-
 .../Inputs/misexpect-switch.proftext          |  2 +-
 .../Inputs/misexpect-switch_entry.proftext    |  2 +-
 .../Inputs/multiple_hash_profile.proftext     | 32 +++++++++++-
 .../PGOProfile/Inputs/noreturncall.proftext   |  2 +-
 .../PGOProfile/Inputs/remap.proftext          |  2 +-
 .../PGOProfile/Inputs/select1.proftext        |  2 +-
 .../PGOProfile/Inputs/select2.proftext        |  2 +-
 .../PGOProfile/Inputs/suppl-profile.proftext  |  2 +-
 .../PGOProfile/Inputs/switch.proftext         |  2 +-
 .../PGOProfile/Inputs/switch_entry.proftext   |  2 +-
 .../PGOProfile/Inputs/thinlto_cs.proftext     |  8 +--
 .../PGOProfile/multiple_hash_profile.ll       |  5 ++
 50 files changed, 144 insertions(+), 80 deletions(-)

diff --git a/clang/test/CodeGen/Inputs/thinlto_expect1.proftext b/clang/test/CodeGen/Inputs/thinlto_expect1.proftext
index e7ce3a4ee2379..0c904e2ea1c89 100644
--- a/clang/test/CodeGen/Inputs/thinlto_expect1.proftext
+++ b/clang/test/CodeGen/Inputs/thinlto_expect1.proftext
@@ -2,7 +2,7 @@
 :ir
 foo
 # Func Hash:
-25571299074
+784007059655560962
 # Num Counters:
 2
 # Counter Values:
diff --git a/clang/test/CodeGen/Inputs/thinlto_expect2.proftext b/clang/test/CodeGen/Inputs/thinlto_expect2.proftext
index f9de785587abe..c240a442c465f 100644
--- a/clang/test/CodeGen/Inputs/thinlto_expect2.proftext
+++ b/clang/test/CodeGen/Inputs/thinlto_expect2.proftext
@@ -2,7 +2,7 @@
 :csir
 foo
 # Func Hash:
-25571299074
+784007059655560962
 # Num Counters:
 2
 # Counter Values:
@@ -11,7 +11,7 @@ foo
 
 foo
 # Func Hash:
-1152921530178146050
+1936928564262407938
 # Num Counters:
 2
 # Counter Values:
diff --git a/clang/test/CodeGenCXX/Inputs/profile-remap.proftext b/clang/test/CodeGenCXX/Inputs/profile-remap.proftext
index a1f90cfa6e9ed..bf57fc696c49e 100644
--- a/clang/test/CodeGenCXX/Inputs/profile-remap.proftext
+++ b/clang/test/CodeGenCXX/Inputs/profile-remap.proftext
@@ -1,6 +1,6 @@
 :ir
 _ZN3Foo8functionENS_1XE
-29667547796
+146835647075900052
 2
 10
 90
diff --git a/clang/test/CodeGenCXX/Inputs/profile-remap_entry.proftext b/clang/test/CodeGenCXX/Inputs/profile-remap_entry.proftext
index 65bc6ff46711b..89772cd7644f2 100644
--- a/clang/test/CodeGenCXX/Inputs/profile-remap_entry.proftext
+++ b/clang/test/CodeGenCXX/Inputs/profile-remap_entry.proftext
@@ -1,7 +1,7 @@
 :ir
 :entry_first
 _ZN3Foo8functionENS_1XE
-29667547796
+146835647075900052
 2
 100
 90
diff --git a/clang/test/Profile/Inputs/gcc-flag-compatibility_IR.proftext b/clang/test/Profile/Inputs/gcc-flag-compatibility_IR.proftext
index 64d18116ed647..0f39af77715dd 100644
--- a/clang/test/Profile/Inputs/gcc-flag-compatibility_IR.proftext
+++ b/clang/test/Profile/Inputs/gcc-flag-compatibility_IR.proftext
@@ -2,7 +2,7 @@
 :ir
 main
 # Func Hash:
-34137660316
+1063705162469825436
 # Num Counters:
 2
 # Counter Values:
diff --git a/clang/test/Profile/Inputs/gcc-flag-compatibility_IR_entry.proftext b/clang/test/Profile/Inputs/gcc-flag-compatibility_IR_entry.proftext
index 4903f56c419e0..d60162c0f5294 100644
--- a/clang/test/Profile/Inputs/gcc-flag-compatibility_IR_entry.proftext
+++ b/clang/test/Profile/Inputs/gcc-flag-compatibility_IR_entry.proftext
@@ -3,7 +3,7 @@
 :entry_first
 main
 # Func Hash:
-34137660316
+1063705162469825436
 # Num Counters:
 2
 # Counter Values:
diff --git a/compiler-rt/test/profile/Linux/instrprof-value-merge.c b/compiler-rt/test/profile/Linux/instrprof-value-merge.c
index 902430a4a9686..2619a1d00336e 100644
--- a/compiler-rt/test/profile/Linux/instrprof-value-merge.c
+++ b/compiler-rt/test/profile/Linux/instrprof-value-merge.c
@@ -45,7 +45,7 @@ int main(int argc, char *argv[]) {
 
 // CHECK: Counters:
 // CHECK:   main:
-// CHECK:     Hash: 0x00030012a7ab6e87
+// CHECK:     Hash: 0x0a9bd81e87ab6e87
 // CHECK:     Counters: 6
 // CHECK:     Indirect Call Site Count: 3
 // CHECK:     Number of Memory Intrinsics Calls: 3
diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
index 7a14f777b565a..a99201991823f 100644
--- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
+++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
@@ -261,6 +261,10 @@ extern cl::opt<PGOViewCountsType> PGOViewCounts;
 // Defined in Analysis/BlockFrequencyInfo.cpp:  -view-bfi-func-name=
 extern cl::opt<std::string> ViewBlockFreqFuncName;
 
+static cl::opt<bool>
+    PGOOldCFGHashing("pgo-instr-old-cfg-hashing", cl::init(false), cl::Hidden,
+                     cl::desc("Use the old CFG function hashing"));
+
 // Return a string describing the branch condition that can be
 // used in static branch probability heuristics:
 static std::string getBranchCondString(Instruction *TI) {
@@ -620,7 +624,8 @@ template <class Edge, class BBInfo> class FuncPGOInstrumentation {
 } // end anonymous namespace
 
 // Compute Hash value for the CFG: the lower 32 bits are CRC32 of the index
-// value of each BB in the CFG. The higher 32 bits record the number of edges.
+// value of each BB in the CFG. The higher 32 bits are the CRC32 of the numbers
+// of selects, indirect calls, mem ops and edges.
 template <class Edge, class BBInfo>
 void FuncPGOInstrumentation<Edge, BBInfo>::computeCFGHash() {
   std::vector<uint8_t> Indexes;
@@ -639,12 +644,34 @@ void FuncPGOInstrumentation<Edge, BBInfo>::computeCFGHash() {
   }
   JC.update(Indexes);
 
-  // Hash format for context sensitive profile. Reserve 4 bits for other
-  // information.
-  FunctionHash = (uint64_t)SIVisitor.getNumOfSelectInsts() << 56 |
-                 (uint64_t)ValueSites[IPVK_IndirectCallTarget].size() << 48 |
-                 //(uint64_t)ValueSites[IPVK_MemOPSize].size() << 40 |
-                 (uint64_t)MST.AllEdges.size() << 32 | JC.getCRC();
+  JamCRC JCH;
+  if (PGOOldCFGHashing) {
+    // Hash format for context sensitive profile. Reserve 4 bits for other
+    // information.
+    FunctionHash = (uint64_t)SIVisitor.getNumOfSelectInsts() << 56 |
+                   (uint64_t)ValueSites[IPVK_IndirectCallTarget].size() << 48 |
+                   //(uint64_t)ValueSites[IPVK_MemOPSize].size() << 40 |
+                   (uint64_t)MST.AllEdges.size() << 32 | JC.getCRC();
+  } else {
+    // The higher 32 bits.
+    union {
+      uint64_t N;
+      uint8_t C[8];
+    } Data;
+    Data.N = (uint64_t)SIVisitor.getNumOfSelectInsts();
+    JCH.update(Data.C);
+    Data.N = (uint64_t)ValueSites[IPVK_IndirectCallTarget].size();
+    JCH.update(Data.C);
+    Data.N = (uint64_t)ValueSites[IPVK_MemOPSize].size();
+    JCH.update(Data.C);
+    Data.N = (uint64_t)MST.AllEdges.size();
+    JCH.update(Data.C);
+
+    // Hash format for context sensitive profile. Reserve 4 bits for other
+    // information.
+    FunctionHash = (((uint64_t)JCH.getCRC()) << 28) + JC.getCRC();
+  }
+
   // Reserve bit 60-63 for other information purpose.
   FunctionHash &= 0x0FFFFFFFFFFFFFFF;
   if (IsCS)
@@ -653,8 +680,12 @@ void FuncPGOInstrumentation<Edge, BBInfo>::computeCFGHash() {
                     << " CRC = " << JC.getCRC()
                     << ", Selects = " << SIVisitor.getNumOfSelectInsts()
                     << ", Edges = " << MST.AllEdges.size() << ", ICSites = "
-                    << ValueSites[IPVK_IndirectCallTarget].size()
-                    << ", Hash = " << FunctionHash << "\n";);
+                    << ValueSites[IPVK_IndirectCallTarget].size());
+  if (!PGOOldCFGHashing) {
+    LLVM_DEBUG(dbgs() << ", Memops = " << ValueSites[IPVK_MemOPSize].size()
+                      << ", High32 CRC = " << JCH.getCRC());
+  }
+  LLVM_DEBUG(dbgs() << ", Hash = " << FunctionHash << "\n";);
 }
 
 // Check if we can safely rename this Comdat function.
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/PR41279.proftext b/llvm/test/Transforms/PGOProfile/Inputs/PR41279.proftext
index 58c3e89f089cc..ce657e720c571 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/PR41279.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/PR41279.proftext
@@ -1,6 +1,6 @@
 :ir
 foo
-60927483247
+1096621588030135663
 4
 3
 2
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/PR41279_2.proftext b/llvm/test/Transforms/PGOProfile/Inputs/PR41279_2.proftext
index adc16bbad92c3..c6cb02aaddd1d 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/PR41279_2.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/PR41279_2.proftext
@@ -1,6 +1,6 @@
 :ir
 f
-62077759478
+1096621589180411894
 2
 3
 2
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/branch1.proftext b/llvm/test/Transforms/PGOProfile/Inputs/branch1.proftext
index 8ca9db9c0515b..d5f9123fcc24f 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/branch1.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/branch1.proftext
@@ -1,7 +1,7 @@
 # :ir is the flag to indicate this is IR level profile.
 :ir
 test_br_1
-25571299074
+784007059655560962
 2
 3
 2
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/branch1_large_count.proftext b/llvm/test/Transforms/PGOProfile/Inputs/branch1_large_count.proftext
index a5b1f8204e808..4fd528ee076f2 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/branch1_large_count.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/branch1_large_count.proftext
@@ -1,7 +1,7 @@
 # :ir is the flag to indicate this is IR level profile.
 :ir
 test_br_1
-25571299074
+784007059655560962
 2
 12884901888
 8589934592
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/branch2.proftext b/llvm/test/Transforms/PGOProfile/Inputs/branch2.proftext
index b5fee2b6f4faf..3e20359b1b84f 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/branch2.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/branch2.proftext
@@ -1,7 +1,7 @@
 # :ir is the flag to indicate this is IR level profile.
 :ir
 test_br_2
-29667547796
+146835647075900052
 2
 1
 1
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/branch2_entry.proftext b/llvm/test/Transforms/PGOProfile/Inputs/branch2_entry.proftext
index 51de52d824926..31710fc8f4bde 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/branch2_entry.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/branch2_entry.proftext
@@ -2,7 +2,7 @@
 :ir
 :entry_first
 test_br_2
-29667547796
+146835647075900052
 2
 2
 1
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/criticaledge.proftext b/llvm/test/Transforms/PGOProfile/Inputs/criticaledge.proftext
index 7613b643b163e..85b9779abeece 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/criticaledge.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/criticaledge.proftext
@@ -1,7 +1,7 @@
 # :ir is the flag to indicate this is IR level profile.
 :ir
 test_criticalEdge
-82323253069
+93478046750287693
 8
 2
 1
@@ -13,7 +13,7 @@ test_criticalEdge
 1
 
 <stdin>:bar
-12884901887
+742261418966908927
 1
 7
 
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/criticaledge_entry.proftext b/llvm/test/Transforms/PGOProfile/Inputs/criticaledge_entry.proftext
index 79754261d2f88..f1497d6c01c9f 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/criticaledge_entry.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/criticaledge_entry.proftext
@@ -2,7 +2,7 @@
 :ir
 :entry_first
 test_criticalEdge
-82323253069
+93478046750287693
 8
 7
 2
@@ -14,7 +14,7 @@ test_criticalEdge
 1
 
 <stdin>:bar
-12884901887
+742261418966908927
 1
 7
 
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/cspgo.proftext b/llvm/test/Transforms/PGOProfile/Inputs/cspgo.proftext
index e40a0f29fccc0..6d6d86cfb50bb 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/cspgo.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/cspgo.proftext
@@ -54,7 +54,7 @@ bar_m2
 
 foo
 # Func Hash:
-1152921640672869708
+1456607294772657484
 # Num Counters:
 10
 # Counter Values:
@@ -71,7 +71,7 @@ foo
 
 foo
 # Func Hash:
-29212902728
+146835646621254984
 # Num Counters:
 2
 # Counter Values:
@@ -80,7 +80,7 @@ foo
 
 bar
 # Func Hash:
-1152921569533132113
+1440408129826749777
 # Num Counters:
 5
 # Counter Values:
@@ -92,7 +92,7 @@ bar
 
 bar
 # Func Hash:
-56228292833
+567185239050791137
 # Num Counters:
 4
 # Counter Values:
@@ -103,7 +103,7 @@ bar
 
 main
 # Func Hash:
-1152921517491748863
+1895182923573755903
 # Num Counters:
 1
 # Counter Values:
@@ -111,7 +111,7 @@ main
 
 main
 # Func Hash:
-12884901887
+742261418966908927
 # Num Counters:
 1
 # Counter Values:
@@ -135,7 +135,7 @@ csfdo_plain.c:barbar
 
 goo
 # Func Hash:
-1152921517491748863
+1895182923573755903
 # Num Counters:
 1
 # Counter Values:
@@ -143,7 +143,7 @@ goo
 
 goo
 # Func Hash:
-12884901887
+742261418966908927
 # Num Counters:
 1
 # Counter Values:
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/diag_no_value_sites.proftext b/llvm/test/Transforms/PGOProfile/Inputs/diag_no_value_sites.proftext
index b7ba652292ca9..182778755f4bb 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/diag_no_value_sites.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/diag_no_value_sites.proftext
@@ -1,6 +1,6 @@
 # :ir is the flag to indicate this is IR level profile.
 :ir
 foo
-12884901887
+48277136972185599
 1
 1
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/fix_entry_count.proftext b/llvm/test/Transforms/PGOProfile/Inputs/fix_entry_count.proftext
index de4771beaacc2..7ca97aa29dbcd 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/fix_entry_count.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/fix_entry_count.proftext
@@ -1,7 +1,7 @@
 :ir
 :entry_first
 test_simple_for
-34137660316
+1063705162469825436
 2
 0
 96
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/func_entry.proftext b/llvm/test/Transforms/PGOProfile/Inputs/func_entry.proftext
index 60c49bf730954..6d92958e89b87 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/func_entry.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/func_entry.proftext
@@ -3,7 +3,7 @@
 :entry_first
 hot
 # Func Hash:
-12884901887
+742261418966908927
 # Num Counters:
 1
 # Counter Values:
@@ -11,7 +11,7 @@ hot
 
 cold
 # Func Hash:
-12884901887
+742261418966908927
 # Num Counters:
 1
 # Counter Values:
@@ -19,7 +19,7 @@ cold
 
 med
 # Func Hash:
-12884901887
+742261418966908927
 # Num Counters:
 1
 # Counter Values:
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/indirect_call.proftext b/llvm/test/Transforms/PGOProfile/Inputs/indirect_call.proftext
index d453090d1c582..bb48ccc57a8bf 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/indirect_call.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/indirect_call.proftext
@@ -1,7 +1,7 @@
 :ir
 bar
 # Func Hash:
-281487861612543
+170957022131388415
 # Num Counters:
 1
 # Counter Values:
@@ -19,7 +19,7 @@ func3:20
 
 func1
 # Func Hash:
-12884901887
+742261418966908927
 # Num Counters:
 1
 # Counter Values:
@@ -27,7 +27,7 @@ func1
 
 func2
 # Func Hash:
-12884901887
+742261418966908927
 # Num Counters:
 1
 # Counter Values:
@@ -35,7 +35,7 @@ func2
 
 func3
 # Func Hash:
-12884901887
+742261418966908927
 # Num Counters:
 1
 # Counter Values:
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/indirectbr.proftext b/llvm/test/Transforms/PGOProfile/Inputs/indirectbr.proftext
index 3909968476272..49fafd9d99bf9 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/indirectbr.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/indirectbr.proftext
@@ -2,7 +2,7 @@
 :ir
 foo
 # Func Hash:
-47485104005
+844982796158316421
 # Num Counters:
 4
 # Counter Values:
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/indirectbr_entry.proftext b/llvm/test/Transforms/PGOProfile/Inputs/indirectbr_entry.proftext
index 7173269d20cc9..6910f7e21d677 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/indirectbr_entry.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/indirectbr_entry.proftext
@@ -3,7 +3,7 @@
 :entry_first
 foo
 # Func Hash:
-47485104005
+844982796158316421
 # Num Counters:
 4
 # Counter Values:
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/irreducible.proftext b/llvm/test/Transforms/PGOProfile/Inputs/irreducible.proftext
index 9b0210d9a309f..24eacb886489e 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/irreducible.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/irreducible.proftext
@@ -1,7 +1,7 @@
 :ir
 _Z11irreducibleii
 # Func Hash:
-64451410787
+287486624745028451
 # Num Counters:
 6
 # Counter Values:
@@ -14,7 +14,7 @@ _Z11irreducibleii
 
 _Z11irreduciblePh
 # Func Hash:
-104649601521
+331779889035882993
 # Num Counters:
 9
 # Counter Values:
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/irreducible_entry.proftext b/llvm/test/Transforms/PGOProfile/Inputs/irreducible_entry.proftext
index d04cfddda896d..702c42d1addc8 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/irreducible_entry.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/irreducible_entry.proftext
@@ -2,7 +2,7 @@
 :entry_first
 _Z11irreducibleii
 # Func Hash:
-64451410787
+287486624745028451
 # Num Counters:
 6
 # Counter Values:
@@ -15,7 +15,7 @@ _Z11irreducibleii
 
 _Z11irreduciblePh
 # Func Hash:
-104649601521
+331779889035882993
 # Num Counters:
 9
 # Counter Values:
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/landingpad.proftext b/llvm/test/Transforms/PGOProfile/Inputs/landingpad.proftext
index c71fd04170bee..7a74b296e55fe 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/landingpad.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/landingpad.proftext
@@ -1,7 +1,7 @@
 # :ir is the flag to indicate this is IR level profile.
 :ir
 foo
-59130013419
+567185241952511723
 4
 3
 1
@@ -9,7 +9,7 @@ foo
 0
 
 bar
-24868915205
+784007058953177093
 2
 3
 2
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/landingpad_entry.proftext b/llvm/test/Transforms/PGOProfile/Inputs/landingpad_entry.proftext
index 3175d9c0d630b..9b604038abcc4 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/landingpad_entry.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/landingpad_entry.proftext
@@ -2,7 +2,7 @@
 :ir
 :entry_first
 foo
-59130013419
+567185241952511723
 4
 5
 1
@@ -10,7 +10,7 @@ foo
 0
 
 bar
-24868915205
+784007058953177093
 2
 3
 2
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/large_count_remarks.proftext b/llvm/test/Transforms/PGOProfile/Inputs/large_count_remarks.proftext
index 814af7402b28a..8e23ae6906b0c 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/large_count_remarks.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/large_count_remarks.proftext
@@ -1,7 +1,7 @@
 # :ir is the flag to indicate this is IR level profile.
 :ir
 test
-25571299074
+784007059655560962
 2
 40000000000
 20000000000
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/loop1.proftext b/llvm/test/Transforms/PGOProfile/Inputs/loop1.proftext
index c19737149ca25..25eeefe33c11f 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/loop1.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/loop1.proftext
@@ -1,7 +1,7 @@
 # :ir is the flag to indicate this is IR level profile.
 :ir
 test_simple_for
-34137660316
+1063705162469825436
 2
 96
 4
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/loop1_entry.proftext b/llvm/test/Transforms/PGOProfile/Inputs/loop1_entry.proftext
index b61634d94689d..56f04e808aee8 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/loop1_entry.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/loop1_entry.proftext
@@ -2,7 +2,7 @@
 :ir
 :entry_first
 test_simple_for
-34137660316
+1063705162469825436
 2
 4
 96
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/loop2.proftext b/llvm/test/Transforms/PGOProfile/Inputs/loop2.proftext
index af3a71df1e6b9..7ab9cd6b946e4 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/loop2.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/loop2.proftext
@@ -1,7 +1,7 @@
 # :ir is the flag to indicate this is IR level profile.
 :ir
 test_nested_for
-53929068288
+798733566382720768
 3
 33
 10
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/loop2_entry.proftext b/llvm/test/Transforms/PGOProfile/Inputs/loop2_entry.proftext
index ac8536a52675b..bde1b7e4a4c12 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/loop2_entry.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/loop2_entry.proftext
@@ -2,7 +2,7 @@
 :ir
 :entry_first
 test_nested_for
-53929068288
+798733566382720768
 3
 6
 33
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/memop_size_annotation.proftext b/llvm/test/Transforms/PGOProfile/Inputs/memop_size_annotation.proftext
index cce1a67a94574..83aec54f23283 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/memop_size_annotation.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/memop_size_annotation.proftext
@@ -2,7 +2,7 @@
 :ir
 foo
 # Func Hash:
-53929068288
+687116424982578944
 # Num Counters:
 3
 # Counter Values:
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/misexpect-branch-correct.proftext b/llvm/test/Transforms/PGOProfile/Inputs/misexpect-branch-correct.proftext
index c785a40efea59..24c9746262bda 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/misexpect-branch-correct.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/misexpect-branch-correct.proftext
@@ -2,7 +2,7 @@
 :ir
 bar
 # Func Hash:
-29667547796
+146835647075900052
 # Num Counters:
 2
 # Counter Values:
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/misexpect-branch.proftext b/llvm/test/Transforms/PGOProfile/Inputs/misexpect-branch.proftext
index 36eaa3462782f..5505df7375271 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/misexpect-branch.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/misexpect-branch.proftext
@@ -2,7 +2,7 @@
 :ir
 bar
 # Func Hash:
-29667547796
+146835647075900052
 # Num Counters:
 2
 # Counter Values:
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/misexpect-branch_entry.proftext b/llvm/test/Transforms/PGOProfile/Inputs/misexpect-branch_entry.proftext
index df7d6cf80466b..752b885ad2b2c 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/misexpect-branch_entry.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/misexpect-branch_entry.proftext
@@ -3,7 +3,7 @@
 :entry_first
 bar
 # Func Hash:
-29667547796
+146835647075900052
 # Num Counters:
 2
 # Counter Values:
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/misexpect-switch-correct.proftext b/llvm/test/Transforms/PGOProfile/Inputs/misexpect-switch-correct.proftext
index b0924938ea831..39864486bb96b 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/misexpect-switch-correct.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/misexpect-switch-correct.proftext
@@ -2,7 +2,7 @@
 :ir
 main
 # Func Hash:
-74054140268
+391331300939170156
 # Num Counters:
 7
 # Counter Values:
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/misexpect-switch-correct_entry.proftext b/llvm/test/Transforms/PGOProfile/Inputs/misexpect-switch-correct_entry.proftext
index 4686fec3bee78..c74f3230939ee 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/misexpect-switch-correct_entry.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/misexpect-switch-correct_entry.proftext
@@ -3,7 +3,7 @@
 :entry_first
 main
 # Func Hash:
-74054140268
+391331300939170156
 # Num Counters:
 7
 # Counter Values:
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/misexpect-switch.proftext b/llvm/test/Transforms/PGOProfile/Inputs/misexpect-switch.proftext
index 8e8049f3dc62e..ea50f6b6043f4 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/misexpect-switch.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/misexpect-switch.proftext
@@ -2,7 +2,7 @@
 :ir
 main
 # Func Hash:
-74054140268
+391331300939170156
 # Num Counters:
 7
 # Counter Values:
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/misexpect-switch_entry.proftext b/llvm/test/Transforms/PGOProfile/Inputs/misexpect-switch_entry.proftext
index 3da4c10ac19d1..6afb6c9019f24 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/misexpect-switch_entry.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/misexpect-switch_entry.proftext
@@ -3,7 +3,7 @@
 :entry_first
 main
 # Func Hash:
-74054140268
+391331300939170156
 # Num Counters:
 7
 # Counter Values:
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/multiple_hash_profile.proftext b/llvm/test/Transforms/PGOProfile/Inputs/multiple_hash_profile.proftext
index 5bf67fb2bfaf2..77f8d5a5ade3b 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/multiple_hash_profile.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/multiple_hash_profile.proftext
@@ -2,13 +2,23 @@
 :ir
 _Z3fooi
 # Func Hash:
-72057606922829823
+382993475055910911
 # Num Counters:
 2
 # Counter Values:
 18
 12
 
+# For -pgo-instr-old-cfg-hashing=true
+_Z3fooi
+# Func Hash:
+72057606922829823
+# Num Counters:
+2
+# Counter Values:
+18
+6
+
 _Z3fooi
 # Func Hash:
 12884901887
@@ -17,6 +27,16 @@ _Z3fooi
 # Counter Values:
 0
 
+_Z3bari
+# Func Hash:
+382993475055910911
+# Num Counters:
+2
+# Counter Values:
+0
+0
+
+# For -pgo-instr-old-cfg-hashing=true
 _Z3bari
 # Func Hash:
 72057606922829823
@@ -28,9 +48,17 @@ _Z3bari
 
 _Z4m2f1v
 # Func Hash:
-12884901887
+742261418966908927
 # Num Counters:
 1
 # Counter Values:
 1
 
+# For -pgo-instr-old-cfg-hashing=true
+_Z4m2f1v
+# Func Hash:
+12884901887
+# Num Counters:
+1
+# Counter Values:
+1
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/noreturncall.proftext b/llvm/test/Transforms/PGOProfile/Inputs/noreturncall.proftext
index 1bedce5b9f9b3..bb4b0719cf129 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/noreturncall.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/noreturncall.proftext
@@ -2,7 +2,7 @@
 :ir
 foo
 # Func Hash:
-36496524737
+238984482720105921
 # Num Counters:
 3
 # Counter Values:
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/remap.proftext b/llvm/test/Transforms/PGOProfile/Inputs/remap.proftext
index 40054d78f5aab..d9b22637d3a7d 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/remap.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/remap.proftext
@@ -1,7 +1,7 @@
 # :ir is the flag to indicate this is IR level profile.
 :ir
 _ZN3foo3barERKN1N1XINS_4quuxEEE
-25571299074
+784007059655560962
 2
 3
 2
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/select1.proftext b/llvm/test/Transforms/PGOProfile/Inputs/select1.proftext
index 60663e02ffb49..c4d6d56073b01 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/select1.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/select1.proftext
@@ -1,7 +1,7 @@
 :ir
 :entry_first
 test_br_2
-72057623705475732
+942389667449461396
 3
 5
 1
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/select2.proftext b/llvm/test/Transforms/PGOProfile/Inputs/select2.proftext
index 8f3c562097fae..0d4b003ec0667 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/select2.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/select2.proftext
@@ -3,7 +3,7 @@
 :entry_first
 foo
 # Func Hash:
-72057628175588252
+134732432632142748
 # Num Counters:
 3
 # Counter Values:
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/suppl-profile.proftext b/llvm/test/Transforms/PGOProfile/Inputs/suppl-profile.proftext
index c82311b3e0c06..41f4fbe655fac 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/suppl-profile.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/suppl-profile.proftext
@@ -1,7 +1,7 @@
 # :ir is the flag to indicate this is IR level profile.
 :ir
 test_simple_for
-34137660316
+1063705162469825436
 2
 0
 0
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/switch.proftext b/llvm/test/Transforms/PGOProfile/Inputs/switch.proftext
index bebd65fa4c9fd..cf73ed083794e 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/switch.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/switch.proftext
@@ -1,7 +1,7 @@
 # :ir is the flag to indicate this is IR level profile.
 :ir
 test_switch
-46200943743
+536873293052540031
 4
 0
 5
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/switch_entry.proftext b/llvm/test/Transforms/PGOProfile/Inputs/switch_entry.proftext
index 20df873608c19..cf079893e3404 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/switch_entry.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/switch_entry.proftext
@@ -2,7 +2,7 @@
 :ir
 :entry_first
 test_switch
-46200943743
+536873293052540031
 4
 10
 5
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/thinlto_cs.proftext b/llvm/test/Transforms/PGOProfile/Inputs/thinlto_cs.proftext
index 4717752da716d..1b9f19e7f7fa5 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/thinlto_cs.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/thinlto_cs.proftext
@@ -10,7 +10,7 @@ cond.llvm.11253644763537639171
 
 foo
 # Func Hash:
-29212902728
+1720106746050921044
 # Num Counters:
 2
 # Counter Values:
@@ -19,7 +19,7 @@ foo
 
 bar
 # Func Hash:
-1152921534274394772
+1299757151682747028
 # Num Counters:
 2
 # Counter Values:
@@ -45,7 +45,7 @@ main
 
 main
 # Func Hash:
-12884901887
+1895182923573755903
 # Num Counters:
 1
 # Counter Values:
@@ -53,7 +53,7 @@ main
 
 cspgo.c:foo
 # Func Hash:
-1152921563228422740
+1720106746050921044
 # Num Counters:
 4
 # Counter Values:
diff --git a/llvm/test/Transforms/PGOProfile/multiple_hash_profile.ll b/llvm/test/Transforms/PGOProfile/multiple_hash_profile.ll
index fdfb16b9e2e55..f0011114f0d01 100644
--- a/llvm/test/Transforms/PGOProfile/multiple_hash_profile.ll
+++ b/llvm/test/Transforms/PGOProfile/multiple_hash_profile.ll
@@ -1,6 +1,8 @@
 ; RUN: llvm-profdata merge %S/Inputs/multiple_hash_profile.proftext -o %t.profdata
 ; RUN: opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata  -S | FileCheck %s
+; RUN: opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata -pgo-instr-old-cfg-hashing=true -S | FileCheck -check-prefix=CHECKOLDHASH %s
 ; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s
+; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -pgo-instr-old-cfg-hashing=true -S | FileCheck -check-prefix=CHECKOLDHASH %s
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
@@ -29,6 +31,9 @@ entry:
 ; CHECK: %mul.i = select i1 %cmp.i, i32 1, i32 %i
 ; CHECK-SAME: !prof ![[BW:[0-9]+]]
 ; CHECK: ![[BW]] = !{!"branch_weights", i32 12, i32 6}
+; CHECKOLDHASH: %mul.i = select i1 %cmp.i, i32 1, i32 %i
+; CHECKOLDHASH-SAME: !prof ![[BW:[0-9]+]]
+; CHECKOLDHASH: ![[BW]] = !{!"branch_weights", i32 6, i32 12}
   %retval.0.i = mul nsw i32 %mul.i, %i
   ret i32 %retval.0.i
 }

From d8e8e32d85f1cbde2a6b67af010fba3c3a3c86e9 Mon Sep 17 00:00:00 2001
From: Saleem Abdulrasool <compnerd@compnerd.org>
Date: Wed, 22 Jul 2020 08:54:36 -0700
Subject: [PATCH 0549/1035] build: always add support for assembly targets on
 Darwin

When attempting to build compiler-rt on a developer transition kit, the
build would fail due to `.S` files not being handled properly by the
Ninja generator.  Rather than conditionalising on Xcode, conditionalise
to Darwin.  Because we know that the system compiler is clang based, it
will always properly handle the pre-processing based on the extension.

Differential Revision: https://reviews.llvm.org/D84333
---
 compiler-rt/cmake/Modules/AddCompilerRT.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/compiler-rt/cmake/Modules/AddCompilerRT.cmake b/compiler-rt/cmake/Modules/AddCompilerRT.cmake
index efb6608182705..9c135ef931a03 100644
--- a/compiler-rt/cmake/Modules/AddCompilerRT.cmake
+++ b/compiler-rt/cmake/Modules/AddCompilerRT.cmake
@@ -110,7 +110,7 @@ endfunction()
 function(add_asm_sources output)
   set(${output} ${ARGN} PARENT_SCOPE)
   # Xcode will try to compile asm files as C ('clang -x c'), and that will fail.
-  if (${CMAKE_GENERATOR} STREQUAL "Xcode")
+  if(CMAKE_SYSTEM_NAME STREQUAL Darwin)
     enable_language(ASM)
   else()
     # Pass ASM file directly to the C++ compiler.

From 5cd695dd7fbd52af0f58ad75cc8d63ad2baa619f Mon Sep 17 00:00:00 2001
From: Sanjay Patel <spatel@rotateright.com>
Date: Wed, 29 Jul 2020 13:43:33 -0400
Subject: [PATCH 0550/1035] [InstSimplify] fold min/max with opposite of limit
 value

---
 llvm/lib/Analysis/InstructionSimplify.cpp     | 11 ++++++
 .../InstSimplify/maxmin_intrinsics.ll         | 36 +++++++------------
 2 files changed, 23 insertions(+), 24 deletions(-)

diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp
index de4e23aac9826..9d9999ea4372b 100644
--- a/llvm/lib/Analysis/InstructionSimplify.cpp
+++ b/llvm/lib/Analysis/InstructionSimplify.cpp
@@ -5270,12 +5270,23 @@ static Value *simplifyBinaryIntrinsic(Function *F, Value *Op0, Value *Op1,
     if (!match(Op1, m_APIntAllowUndef(C)))
       break;
 
+    // Clamp to limit value. For example:
+    // umax(i8 %x, i8 255) --> 255
     if ((IID == Intrinsic::smax && C->isMaxSignedValue()) ||
         (IID == Intrinsic::smin && C->isMinSignedValue()) ||
         (IID == Intrinsic::umax && C->isMaxValue()) ||
         (IID == Intrinsic::umin && C->isMinValue()))
       return ConstantInt::get(ReturnType, *C);
 
+    // If the constant op is the opposite of the limit value, the other must be
+    // larger/smaller or equal. For example:
+    // umin(i8 %x, i8 255) --> %x
+    if ((IID == Intrinsic::smax && C->isMinSignedValue()) ||
+        (IID == Intrinsic::smin && C->isMaxSignedValue()) ||
+        (IID == Intrinsic::umax && C->isMinValue()) ||
+        (IID == Intrinsic::umin && C->isMaxValue()))
+      return Op0;
+
     break;
   }
   case Intrinsic::usub_with_overflow:
diff --git a/llvm/test/Transforms/InstSimplify/maxmin_intrinsics.ll b/llvm/test/Transforms/InstSimplify/maxmin_intrinsics.ll
index 312056d745a90..a376ef8adf1ab 100644
--- a/llvm/test/Transforms/InstSimplify/maxmin_intrinsics.ll
+++ b/llvm/test/Transforms/InstSimplify/maxmin_intrinsics.ll
@@ -146,8 +146,7 @@ define <2 x i8> @umin_minval_commute(<2 x i8> %x) {
 
 define i8 @smax_minval(i8 %x) {
 ; CHECK-LABEL: @smax_minval(
-; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.smax.i8(i8 [[X:%.*]], i8 -128)
-; CHECK-NEXT:    ret i8 [[R]]
+; CHECK-NEXT:    ret i8 [[X:%.*]]
 ;
   %r = call i8 @llvm.smax.i8(i8 %x, i8 -128)
   ret i8 %r
@@ -155,8 +154,7 @@ define i8 @smax_minval(i8 %x) {
 
 define <2 x i8> @smax_minval_commute(<2 x i8> %x) {
 ; CHECK-LABEL: @smax_minval_commute(
-; CHECK-NEXT:    [[R:%.*]] = call <2 x i8> @llvm.smax.v2i8(<2 x i8> <i8 -128, i8 -128>, <2 x i8> [[X:%.*]])
-; CHECK-NEXT:    ret <2 x i8> [[R]]
+; CHECK-NEXT:    ret <2 x i8> [[X:%.*]]
 ;
   %r = call <2 x i8> @llvm.smax.v2i8(<2 x i8> <i8 -128, i8 -128>, <2 x i8> %x)
   ret <2 x i8> %r
@@ -164,8 +162,7 @@ define <2 x i8> @smax_minval_commute(<2 x i8> %x) {
 
 define i8 @smin_maxval(i8 %x) {
 ; CHECK-LABEL: @smin_maxval(
-; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.smin.i8(i8 127, i8 [[X:%.*]])
-; CHECK-NEXT:    ret i8 [[R]]
+; CHECK-NEXT:    ret i8 [[X:%.*]]
 ;
   %r = call i8 @llvm.smin.i8(i8 127, i8 %x)
   ret i8 %r
@@ -173,8 +170,7 @@ define i8 @smin_maxval(i8 %x) {
 
 define <2 x i8> @smin_maxval_commute(<2 x i8> %x) {
 ; CHECK-LABEL: @smin_maxval_commute(
-; CHECK-NEXT:    [[R:%.*]] = call <2 x i8> @llvm.smin.v2i8(<2 x i8> [[X:%.*]], <2 x i8> <i8 127, i8 127>)
-; CHECK-NEXT:    ret <2 x i8> [[R]]
+; CHECK-NEXT:    ret <2 x i8> [[X:%.*]]
 ;
   %r = call <2 x i8> @llvm.smin.v2i8(<2 x i8> %x, <2 x i8> <i8 127, i8 127>)
   ret <2 x i8> %r
@@ -182,8 +178,7 @@ define <2 x i8> @smin_maxval_commute(<2 x i8> %x) {
 
 define i8 @umax_minval(i8 %x) {
 ; CHECK-LABEL: @umax_minval(
-; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.umax.i8(i8 [[X:%.*]], i8 0)
-; CHECK-NEXT:    ret i8 [[R]]
+; CHECK-NEXT:    ret i8 [[X:%.*]]
 ;
   %r = call i8 @llvm.umax.i8(i8 %x, i8 0)
   ret i8 %r
@@ -191,8 +186,7 @@ define i8 @umax_minval(i8 %x) {
 
 define <2 x i8> @umax_minval_commute(<2 x i8> %x) {
 ; CHECK-LABEL: @umax_minval_commute(
-; CHECK-NEXT:    [[R:%.*]] = call <2 x i8> @llvm.umax.v2i8(<2 x i8> zeroinitializer, <2 x i8> [[X:%.*]])
-; CHECK-NEXT:    ret <2 x i8> [[R]]
+; CHECK-NEXT:    ret <2 x i8> [[X:%.*]]
 ;
   %r = call <2 x i8> @llvm.umax.v2i8(<2 x i8> zeroinitializer, <2 x i8> %x)
   ret <2 x i8> %r
@@ -200,8 +194,7 @@ define <2 x i8> @umax_minval_commute(<2 x i8> %x) {
 
 define i8 @umin_maxval(i8 %x) {
 ; CHECK-LABEL: @umin_maxval(
-; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.umin.i8(i8 -1, i8 [[X:%.*]])
-; CHECK-NEXT:    ret i8 [[R]]
+; CHECK-NEXT:    ret i8 [[X:%.*]]
 ;
   %r = call i8 @llvm.umin.i8(i8 255, i8 %x)
   ret i8 %r
@@ -209,8 +202,7 @@ define i8 @umin_maxval(i8 %x) {
 
 define <2 x i8> @umin_maxval_commute(<2 x i8> %x) {
 ; CHECK-LABEL: @umin_maxval_commute(
-; CHECK-NEXT:    [[R:%.*]] = call <2 x i8> @llvm.umin.v2i8(<2 x i8> [[X:%.*]], <2 x i8> <i8 -1, i8 -1>)
-; CHECK-NEXT:    ret <2 x i8> [[R]]
+; CHECK-NEXT:    ret <2 x i8> [[X:%.*]]
 ;
   %r = call <2 x i8> @llvm.umin.v2i8(<2 x i8> %x, <2 x i8> <i8 255, i8 255>)
   ret <2 x i8> %r
@@ -250,8 +242,7 @@ define <2 x i8> @umin_minval_partial_undef(<2 x i8> %x) {
 
 define <2 x i8> @smax_minval_partial_undef(<2 x i8> %x) {
 ; CHECK-LABEL: @smax_minval_partial_undef(
-; CHECK-NEXT:    [[R:%.*]] = call <2 x i8> @llvm.smax.v2i8(<2 x i8> <i8 undef, i8 -128>, <2 x i8> [[X:%.*]])
-; CHECK-NEXT:    ret <2 x i8> [[R]]
+; CHECK-NEXT:    ret <2 x i8> [[X:%.*]]
 ;
   %r = call <2 x i8> @llvm.smax.v2i8(<2 x i8> <i8 undef, i8 -128>, <2 x i8> %x)
   ret <2 x i8> %r
@@ -259,8 +250,7 @@ define <2 x i8> @smax_minval_partial_undef(<2 x i8> %x) {
 
 define <2 x i8> @smin_maxval_partial_undef(<2 x i8> %x) {
 ; CHECK-LABEL: @smin_maxval_partial_undef(
-; CHECK-NEXT:    [[R:%.*]] = call <2 x i8> @llvm.smin.v2i8(<2 x i8> [[X:%.*]], <2 x i8> <i8 undef, i8 127>)
-; CHECK-NEXT:    ret <2 x i8> [[R]]
+; CHECK-NEXT:    ret <2 x i8> [[X:%.*]]
 ;
   %r = call <2 x i8> @llvm.smin.v2i8(<2 x i8> %x, <2 x i8> <i8 undef, i8 127>)
   ret <2 x i8> %r
@@ -268,8 +258,7 @@ define <2 x i8> @smin_maxval_partial_undef(<2 x i8> %x) {
 
 define <2 x i8> @umax_minval_partial_undef(<2 x i8> %x) {
 ; CHECK-LABEL: @umax_minval_partial_undef(
-; CHECK-NEXT:    [[R:%.*]] = call <2 x i8> @llvm.umax.v2i8(<2 x i8> <i8 0, i8 undef>, <2 x i8> [[X:%.*]])
-; CHECK-NEXT:    ret <2 x i8> [[R]]
+; CHECK-NEXT:    ret <2 x i8> [[X:%.*]]
 ;
   %r = call <2 x i8> @llvm.umax.v2i8(<2 x i8> <i8 0, i8 undef>, <2 x i8> %x)
   ret <2 x i8> %r
@@ -277,8 +266,7 @@ define <2 x i8> @umax_minval_partial_undef(<2 x i8> %x) {
 
 define <2 x i8> @umin_maxval_partial_undef(<2 x i8> %x) {
 ; CHECK-LABEL: @umin_maxval_partial_undef(
-; CHECK-NEXT:    [[R:%.*]] = call <2 x i8> @llvm.umin.v2i8(<2 x i8> [[X:%.*]], <2 x i8> <i8 -1, i8 undef>)
-; CHECK-NEXT:    ret <2 x i8> [[R]]
+; CHECK-NEXT:    ret <2 x i8> [[X:%.*]]
 ;
   %r = call <2 x i8> @llvm.umin.v2i8(<2 x i8> %x, <2 x i8> <i8 255, i8 undef>)
   ret <2 x i8> %r

From fef513f5ccb799853a8aed46df2463de1cb4aed9 Mon Sep 17 00:00:00 2001
From: Sanjay Patel <spatel@rotateright.com>
Date: Wed, 29 Jul 2020 16:42:11 -0400
Subject: [PATCH 0551/1035] [InstSimplify] fold min/max intrinsic with undef
 operand

---
 llvm/lib/Analysis/InstructionSimplify.cpp           | 13 +++++++++++++
 .../Transforms/InstSimplify/maxmin_intrinsics.ll    | 12 ++++--------
 2 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp
index 9d9999ea4372b..178b5bed20067 100644
--- a/llvm/lib/Analysis/InstructionSimplify.cpp
+++ b/llvm/lib/Analysis/InstructionSimplify.cpp
@@ -5253,6 +5253,7 @@ static Value *simplifyBinaryIntrinsic(Function *F, Value *Op0, Value *Op1,
                                       const SimplifyQuery &Q) {
   Intrinsic::ID IID = F->getIntrinsicID();
   Type *ReturnType = F->getReturnType();
+  unsigned BitWidth = ReturnType->getScalarSizeInBits();
   switch (IID) {
   case Intrinsic::smax:
   case Intrinsic::smin:
@@ -5266,6 +5267,18 @@ static Value *simplifyBinaryIntrinsic(Function *F, Value *Op0, Value *Op1,
     if (isa<Constant>(Op0))
       std::swap(Op0, Op1);
 
+    // Assume undef is the limit value.
+    if (isa<UndefValue>(Op1)) {
+      if (IID == Intrinsic::smax)
+        return ConstantInt::get(ReturnType, APInt::getSignedMaxValue(BitWidth));
+      if (IID == Intrinsic::smin)
+        return ConstantInt::get(ReturnType, APInt::getSignedMinValue(BitWidth));
+      if (IID == Intrinsic::umax)
+        return ConstantInt::get(ReturnType, APInt::getMaxValue(BitWidth));
+      if (IID == Intrinsic::umin)
+        return ConstantInt::get(ReturnType, APInt::getMinValue(BitWidth));
+    }
+
     const APInt *C;
     if (!match(Op1, m_APIntAllowUndef(C)))
       break;
diff --git a/llvm/test/Transforms/InstSimplify/maxmin_intrinsics.ll b/llvm/test/Transforms/InstSimplify/maxmin_intrinsics.ll
index a376ef8adf1ab..40dacb2d07ba6 100644
--- a/llvm/test/Transforms/InstSimplify/maxmin_intrinsics.ll
+++ b/llvm/test/Transforms/InstSimplify/maxmin_intrinsics.ll
@@ -46,8 +46,7 @@ define <2 x i8> @umin_sameval(<2 x i8> %x) {
 
 define i81 @smax_undef(i81 %x) {
 ; CHECK-LABEL: @smax_undef(
-; CHECK-NEXT:    [[R:%.*]] = call i81 @llvm.smax.i81(i81 undef, i81 [[X:%.*]])
-; CHECK-NEXT:    ret i81 [[R]]
+; CHECK-NEXT:    ret i81 1208925819614629174706175
 ;
   %r = call i81 @llvm.smax.i81(i81 undef, i81 %x)
   ret i81 %r
@@ -55,8 +54,7 @@ define i81 @smax_undef(i81 %x) {
 
 define i3 @smin_undef(i3 %x) {
 ; CHECK-LABEL: @smin_undef(
-; CHECK-NEXT:    [[R:%.*]] = call i3 @llvm.smin.i3(i3 [[X:%.*]], i3 undef)
-; CHECK-NEXT:    ret i3 [[R]]
+; CHECK-NEXT:    ret i3 -4
 ;
   %r = call i3 @llvm.smin.i3(i3 %x, i3 undef)
   ret i3 %r
@@ -64,8 +62,7 @@ define i3 @smin_undef(i3 %x) {
 
 define <2 x i8> @umax_undef(<2 x i8> %x) {
 ; CHECK-LABEL: @umax_undef(
-; CHECK-NEXT:    [[R:%.*]] = call <2 x i8> @llvm.umax.v2i8(<2 x i8> undef, <2 x i8> [[X:%.*]])
-; CHECK-NEXT:    ret <2 x i8> [[R]]
+; CHECK-NEXT:    ret <2 x i8> <i8 -1, i8 -1>
 ;
   %r = call <2 x i8> @llvm.umax.v2i8(<2 x i8> undef, <2 x i8> %x)
   ret <2 x i8> %r
@@ -73,8 +70,7 @@ define <2 x i8> @umax_undef(<2 x i8> %x) {
 
 define <2 x i8> @umin_undef(<2 x i8> %x) {
 ; CHECK-LABEL: @umin_undef(
-; CHECK-NEXT:    [[R:%.*]] = call <2 x i8> @llvm.umin.v2i8(<2 x i8> [[X:%.*]], <2 x i8> undef)
-; CHECK-NEXT:    ret <2 x i8> [[R]]
+; CHECK-NEXT:    ret <2 x i8> zeroinitializer
 ;
   %r = call <2 x i8> @llvm.umin.v2i8(<2 x i8> %x, <2 x i8> undef)
   ret <2 x i8> %r

From 0dcd4a21d57e5b0fd8c4c6f740a71bf9fc8acb06 Mon Sep 17 00:00:00 2001
From: Arthur Eubanks <aeubanks@google.com>
Date: Wed, 29 Jul 2020 14:13:08 -0700
Subject: [PATCH 0552/1035] [opt][NewPM] Fix typo

From https://reviews.llvm.org/D84872.
---
 llvm/tools/opt/opt.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/tools/opt/opt.cpp b/llvm/tools/opt/opt.cpp
index 4a985c50fcd51..e42c90e61188a 100644
--- a/llvm/tools/opt/opt.cpp
+++ b/llvm/tools/opt/opt.cpp
@@ -505,7 +505,7 @@ static bool IsCodegenPass(StringRef Pass) {
   for (const auto &P : PassNameContain)
     if (Pass.contains(P))
       return true;
-  for (const auto &P : PassNamePrefix)
+  for (const auto &P : PassNameExact)
     if (Pass == P)
       return true;
   return false;

From b1c1825b9972cf2c77917b8f2fc220946ab70bac Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@intel.com>
Date: Wed, 29 Jul 2020 13:08:14 -0700
Subject: [PATCH 0553/1035] [X86] Remove unused argument from
 HandleAVX512Operand in the assembly parser.

---
 llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp b/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
index bb9919a25847b..0950cf9e5710d 100644
--- a/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
+++ b/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
@@ -1000,8 +1000,7 @@ class X86AsmParser : public MCTargetAsmParser {
   /// Parses AVX512 specific operand primitives: masked registers ({%k<NUM>}, {z})
   /// and memory broadcasting ({1to<NUM>}) primitives, updating Operands vector if required.
   /// return false if no parsing errors occurred, true otherwise.
-  bool HandleAVX512Operand(OperandVector &Operands,
-                           const MCParsedAsmOperand &Op);
+  bool HandleAVX512Operand(OperandVector &Operands);
 
   bool ParseZ(std::unique_ptr<X86Operand> &Z, const SMLoc &StartLoc);
 
@@ -2313,8 +2312,7 @@ bool X86AsmParser::ParseZ(std::unique_ptr<X86Operand> &Z,
 }
 
 // true on failure, false otherwise
-bool X86AsmParser::HandleAVX512Operand(OperandVector &Operands,
-                                       const MCParsedAsmOperand &Op) {
+bool X86AsmParser::HandleAVX512Operand(OperandVector &Operands) {
   MCAsmParser &Parser = getParser();
   if (getLexer().is(AsmToken::LCurly)) {
     // Eat "{" and mark the current place.
@@ -2893,7 +2891,7 @@ bool X86AsmParser::ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
     while(1) {
       if (std::unique_ptr<X86Operand> Op = ParseOperand()) {
         Operands.push_back(std::move(Op));
-        if (HandleAVX512Operand(Operands, *Operands.back()))
+        if (HandleAVX512Operand(Operands))
           return true;
       } else {
          return true;

From 6700f4b9fe6321ef704efa4890af5bc351a124f0 Mon Sep 17 00:00:00 2001
From: shafik <syaghmour@apple.com>
Date: Wed, 29 Jul 2020 13:29:16 -0700
Subject: [PATCH 0554/1035] [LLDB] Add checks for ValueObjectSP in Cocoa
 summary providers

We saw a crash recently (rdar://problem/65276489) that looks related to an invalid ValueObjectSP in a summary providers in Cocoa.cpp e.g. NSBundleSummaryProvider(...).
This adds checks before we use them usually by calling NSStringSummaryProvider.

Differential Revision: https://reviews.llvm.org/D84272
---
 lldb/source/Plugins/Language/ObjC/Cocoa.cpp | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/lldb/source/Plugins/Language/ObjC/Cocoa.cpp b/lldb/source/Plugins/Language/ObjC/Cocoa.cpp
index 648fc4adf24fd..ca4f7332f2588 100644
--- a/lldb/source/Plugins/Language/ObjC/Cocoa.cpp
+++ b/lldb/source/Plugins/Language/ObjC/Cocoa.cpp
@@ -72,6 +72,9 @@ bool lldb_private::formatters::NSBundleSummaryProvider(
         valobj.GetCompilerType().GetBasicTypeFromAST(lldb::eBasicTypeObjCID),
         true));
 
+    if (!text)
+      return false;
+
     StreamString summary_stream;
     bool was_nsstring_ok =
         NSStringSummaryProvider(*text, summary_stream, options);
@@ -117,6 +120,10 @@ bool lldb_private::formatters::NSTimeZoneSummaryProvider(
     uint64_t offset = ptr_size;
     ValueObjectSP text(valobj.GetSyntheticChildAtOffset(
         offset, valobj.GetCompilerType(), true));
+
+    if (!text)
+      return false;
+
     StreamString summary_stream;
     bool was_nsstring_ok =
         NSStringSummaryProvider(*text, summary_stream, options);
@@ -162,6 +169,10 @@ bool lldb_private::formatters::NSNotificationSummaryProvider(
     uint64_t offset = ptr_size;
     ValueObjectSP text(valobj.GetSyntheticChildAtOffset(
         offset, valobj.GetCompilerType(), true));
+
+    if (!text)
+      return false;
+
     StreamString summary_stream;
     bool was_nsstring_ok =
         NSStringSummaryProvider(*text, summary_stream, options);

From ae7589e1f100b30a4ae13da713c9273733e69fe1 Mon Sep 17 00:00:00 2001
From: Hiroshi Yamauchi <yamauchi@google.com>
Date: Wed, 29 Jul 2020 15:04:57 -0700
Subject: [PATCH 0555/1035] Revert "[PGO] Include the mem ops into the function
 hash."

This reverts commit 120e66b3418b37b95fc1dbbb23e296a602a24fa8.

Due to a buildbot failure.
---
 .../CodeGen/Inputs/thinlto_expect1.proftext   |  2 +-
 .../CodeGen/Inputs/thinlto_expect2.proftext   |  4 +-
 .../CodeGenCXX/Inputs/profile-remap.proftext  |  2 +-
 .../Inputs/profile-remap_entry.proftext       |  2 +-
 .../Inputs/gcc-flag-compatibility_IR.proftext |  2 +-
 .../gcc-flag-compatibility_IR_entry.proftext  |  2 +-
 .../profile/Linux/instrprof-value-merge.c     |  2 +-
 .../Instrumentation/PGOInstrumentation.cpp    | 49 ++++---------------
 .../PGOProfile/Inputs/PR41279.proftext        |  2 +-
 .../PGOProfile/Inputs/PR41279_2.proftext      |  2 +-
 .../PGOProfile/Inputs/branch1.proftext        |  2 +-
 .../Inputs/branch1_large_count.proftext       |  2 +-
 .../PGOProfile/Inputs/branch2.proftext        |  2 +-
 .../PGOProfile/Inputs/branch2_entry.proftext  |  2 +-
 .../PGOProfile/Inputs/criticaledge.proftext   |  4 +-
 .../Inputs/criticaledge_entry.proftext        |  4 +-
 .../PGOProfile/Inputs/cspgo.proftext          | 16 +++---
 .../Inputs/diag_no_value_sites.proftext       |  2 +-
 .../Inputs/fix_entry_count.proftext           |  2 +-
 .../PGOProfile/Inputs/func_entry.proftext     |  6 +--
 .../PGOProfile/Inputs/indirect_call.proftext  |  8 +--
 .../PGOProfile/Inputs/indirectbr.proftext     |  2 +-
 .../Inputs/indirectbr_entry.proftext          |  2 +-
 .../PGOProfile/Inputs/irreducible.proftext    |  4 +-
 .../Inputs/irreducible_entry.proftext         |  4 +-
 .../PGOProfile/Inputs/landingpad.proftext     |  4 +-
 .../Inputs/landingpad_entry.proftext          |  4 +-
 .../Inputs/large_count_remarks.proftext       |  2 +-
 .../PGOProfile/Inputs/loop1.proftext          |  2 +-
 .../PGOProfile/Inputs/loop1_entry.proftext    |  2 +-
 .../PGOProfile/Inputs/loop2.proftext          |  2 +-
 .../PGOProfile/Inputs/loop2_entry.proftext    |  2 +-
 .../Inputs/memop_size_annotation.proftext     |  2 +-
 .../Inputs/misexpect-branch-correct.proftext  |  2 +-
 .../Inputs/misexpect-branch.proftext          |  2 +-
 .../Inputs/misexpect-branch_entry.proftext    |  2 +-
 .../Inputs/misexpect-switch-correct.proftext  |  2 +-
 .../misexpect-switch-correct_entry.proftext   |  2 +-
 .../Inputs/misexpect-switch.proftext          |  2 +-
 .../Inputs/misexpect-switch_entry.proftext    |  2 +-
 .../Inputs/multiple_hash_profile.proftext     | 32 +-----------
 .../PGOProfile/Inputs/noreturncall.proftext   |  2 +-
 .../PGOProfile/Inputs/remap.proftext          |  2 +-
 .../PGOProfile/Inputs/select1.proftext        |  2 +-
 .../PGOProfile/Inputs/select2.proftext        |  2 +-
 .../PGOProfile/Inputs/suppl-profile.proftext  |  2 +-
 .../PGOProfile/Inputs/switch.proftext         |  2 +-
 .../PGOProfile/Inputs/switch_entry.proftext   |  2 +-
 .../PGOProfile/Inputs/thinlto_cs.proftext     |  8 +--
 .../PGOProfile/multiple_hash_profile.ll       |  5 --
 50 files changed, 80 insertions(+), 144 deletions(-)

diff --git a/clang/test/CodeGen/Inputs/thinlto_expect1.proftext b/clang/test/CodeGen/Inputs/thinlto_expect1.proftext
index 0c904e2ea1c89..e7ce3a4ee2379 100644
--- a/clang/test/CodeGen/Inputs/thinlto_expect1.proftext
+++ b/clang/test/CodeGen/Inputs/thinlto_expect1.proftext
@@ -2,7 +2,7 @@
 :ir
 foo
 # Func Hash:
-784007059655560962
+25571299074
 # Num Counters:
 2
 # Counter Values:
diff --git a/clang/test/CodeGen/Inputs/thinlto_expect2.proftext b/clang/test/CodeGen/Inputs/thinlto_expect2.proftext
index c240a442c465f..f9de785587abe 100644
--- a/clang/test/CodeGen/Inputs/thinlto_expect2.proftext
+++ b/clang/test/CodeGen/Inputs/thinlto_expect2.proftext
@@ -2,7 +2,7 @@
 :csir
 foo
 # Func Hash:
-784007059655560962
+25571299074
 # Num Counters:
 2
 # Counter Values:
@@ -11,7 +11,7 @@ foo
 
 foo
 # Func Hash:
-1936928564262407938
+1152921530178146050
 # Num Counters:
 2
 # Counter Values:
diff --git a/clang/test/CodeGenCXX/Inputs/profile-remap.proftext b/clang/test/CodeGenCXX/Inputs/profile-remap.proftext
index bf57fc696c49e..a1f90cfa6e9ed 100644
--- a/clang/test/CodeGenCXX/Inputs/profile-remap.proftext
+++ b/clang/test/CodeGenCXX/Inputs/profile-remap.proftext
@@ -1,6 +1,6 @@
 :ir
 _ZN3Foo8functionENS_1XE
-146835647075900052
+29667547796
 2
 10
 90
diff --git a/clang/test/CodeGenCXX/Inputs/profile-remap_entry.proftext b/clang/test/CodeGenCXX/Inputs/profile-remap_entry.proftext
index 89772cd7644f2..65bc6ff46711b 100644
--- a/clang/test/CodeGenCXX/Inputs/profile-remap_entry.proftext
+++ b/clang/test/CodeGenCXX/Inputs/profile-remap_entry.proftext
@@ -1,7 +1,7 @@
 :ir
 :entry_first
 _ZN3Foo8functionENS_1XE
-146835647075900052
+29667547796
 2
 100
 90
diff --git a/clang/test/Profile/Inputs/gcc-flag-compatibility_IR.proftext b/clang/test/Profile/Inputs/gcc-flag-compatibility_IR.proftext
index 0f39af77715dd..64d18116ed647 100644
--- a/clang/test/Profile/Inputs/gcc-flag-compatibility_IR.proftext
+++ b/clang/test/Profile/Inputs/gcc-flag-compatibility_IR.proftext
@@ -2,7 +2,7 @@
 :ir
 main
 # Func Hash:
-1063705162469825436
+34137660316
 # Num Counters:
 2
 # Counter Values:
diff --git a/clang/test/Profile/Inputs/gcc-flag-compatibility_IR_entry.proftext b/clang/test/Profile/Inputs/gcc-flag-compatibility_IR_entry.proftext
index d60162c0f5294..4903f56c419e0 100644
--- a/clang/test/Profile/Inputs/gcc-flag-compatibility_IR_entry.proftext
+++ b/clang/test/Profile/Inputs/gcc-flag-compatibility_IR_entry.proftext
@@ -3,7 +3,7 @@
 :entry_first
 main
 # Func Hash:
-1063705162469825436
+34137660316
 # Num Counters:
 2
 # Counter Values:
diff --git a/compiler-rt/test/profile/Linux/instrprof-value-merge.c b/compiler-rt/test/profile/Linux/instrprof-value-merge.c
index 2619a1d00336e..902430a4a9686 100644
--- a/compiler-rt/test/profile/Linux/instrprof-value-merge.c
+++ b/compiler-rt/test/profile/Linux/instrprof-value-merge.c
@@ -45,7 +45,7 @@ int main(int argc, char *argv[]) {
 
 // CHECK: Counters:
 // CHECK:   main:
-// CHECK:     Hash: 0x0a9bd81e87ab6e87
+// CHECK:     Hash: 0x00030012a7ab6e87
 // CHECK:     Counters: 6
 // CHECK:     Indirect Call Site Count: 3
 // CHECK:     Number of Memory Intrinsics Calls: 3
diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
index a99201991823f..7a14f777b565a 100644
--- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
+++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
@@ -261,10 +261,6 @@ extern cl::opt<PGOViewCountsType> PGOViewCounts;
 // Defined in Analysis/BlockFrequencyInfo.cpp:  -view-bfi-func-name=
 extern cl::opt<std::string> ViewBlockFreqFuncName;
 
-static cl::opt<bool>
-    PGOOldCFGHashing("pgo-instr-old-cfg-hashing", cl::init(false), cl::Hidden,
-                     cl::desc("Use the old CFG function hashing"));
-
 // Return a string describing the branch condition that can be
 // used in static branch probability heuristics:
 static std::string getBranchCondString(Instruction *TI) {
@@ -624,8 +620,7 @@ template <class Edge, class BBInfo> class FuncPGOInstrumentation {
 } // end anonymous namespace
 
 // Compute Hash value for the CFG: the lower 32 bits are CRC32 of the index
-// value of each BB in the CFG. The higher 32 bits are the CRC32 of the numbers
-// of selects, indirect calls, mem ops and edges.
+// value of each BB in the CFG. The higher 32 bits record the number of edges.
 template <class Edge, class BBInfo>
 void FuncPGOInstrumentation<Edge, BBInfo>::computeCFGHash() {
   std::vector<uint8_t> Indexes;
@@ -644,34 +639,12 @@ void FuncPGOInstrumentation<Edge, BBInfo>::computeCFGHash() {
   }
   JC.update(Indexes);
 
-  JamCRC JCH;
-  if (PGOOldCFGHashing) {
-    // Hash format for context sensitive profile. Reserve 4 bits for other
-    // information.
-    FunctionHash = (uint64_t)SIVisitor.getNumOfSelectInsts() << 56 |
-                   (uint64_t)ValueSites[IPVK_IndirectCallTarget].size() << 48 |
-                   //(uint64_t)ValueSites[IPVK_MemOPSize].size() << 40 |
-                   (uint64_t)MST.AllEdges.size() << 32 | JC.getCRC();
-  } else {
-    // The higher 32 bits.
-    union {
-      uint64_t N;
-      uint8_t C[8];
-    } Data;
-    Data.N = (uint64_t)SIVisitor.getNumOfSelectInsts();
-    JCH.update(Data.C);
-    Data.N = (uint64_t)ValueSites[IPVK_IndirectCallTarget].size();
-    JCH.update(Data.C);
-    Data.N = (uint64_t)ValueSites[IPVK_MemOPSize].size();
-    JCH.update(Data.C);
-    Data.N = (uint64_t)MST.AllEdges.size();
-    JCH.update(Data.C);
-
-    // Hash format for context sensitive profile. Reserve 4 bits for other
-    // information.
-    FunctionHash = (((uint64_t)JCH.getCRC()) << 28) + JC.getCRC();
-  }
-
+  // Hash format for context sensitive profile. Reserve 4 bits for other
+  // information.
+  FunctionHash = (uint64_t)SIVisitor.getNumOfSelectInsts() << 56 |
+                 (uint64_t)ValueSites[IPVK_IndirectCallTarget].size() << 48 |
+                 //(uint64_t)ValueSites[IPVK_MemOPSize].size() << 40 |
+                 (uint64_t)MST.AllEdges.size() << 32 | JC.getCRC();
   // Reserve bit 60-63 for other information purpose.
   FunctionHash &= 0x0FFFFFFFFFFFFFFF;
   if (IsCS)
@@ -680,12 +653,8 @@ void FuncPGOInstrumentation<Edge, BBInfo>::computeCFGHash() {
                     << " CRC = " << JC.getCRC()
                     << ", Selects = " << SIVisitor.getNumOfSelectInsts()
                     << ", Edges = " << MST.AllEdges.size() << ", ICSites = "
-                    << ValueSites[IPVK_IndirectCallTarget].size());
-  if (!PGOOldCFGHashing) {
-    LLVM_DEBUG(dbgs() << ", Memops = " << ValueSites[IPVK_MemOPSize].size()
-                      << ", High32 CRC = " << JCH.getCRC());
-  }
-  LLVM_DEBUG(dbgs() << ", Hash = " << FunctionHash << "\n";);
+                    << ValueSites[IPVK_IndirectCallTarget].size()
+                    << ", Hash = " << FunctionHash << "\n";);
 }
 
 // Check if we can safely rename this Comdat function.
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/PR41279.proftext b/llvm/test/Transforms/PGOProfile/Inputs/PR41279.proftext
index ce657e720c571..58c3e89f089cc 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/PR41279.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/PR41279.proftext
@@ -1,6 +1,6 @@
 :ir
 foo
-1096621588030135663
+60927483247
 4
 3
 2
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/PR41279_2.proftext b/llvm/test/Transforms/PGOProfile/Inputs/PR41279_2.proftext
index c6cb02aaddd1d..adc16bbad92c3 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/PR41279_2.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/PR41279_2.proftext
@@ -1,6 +1,6 @@
 :ir
 f
-1096621589180411894
+62077759478
 2
 3
 2
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/branch1.proftext b/llvm/test/Transforms/PGOProfile/Inputs/branch1.proftext
index d5f9123fcc24f..8ca9db9c0515b 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/branch1.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/branch1.proftext
@@ -1,7 +1,7 @@
 # :ir is the flag to indicate this is IR level profile.
 :ir
 test_br_1
-784007059655560962
+25571299074
 2
 3
 2
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/branch1_large_count.proftext b/llvm/test/Transforms/PGOProfile/Inputs/branch1_large_count.proftext
index 4fd528ee076f2..a5b1f8204e808 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/branch1_large_count.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/branch1_large_count.proftext
@@ -1,7 +1,7 @@
 # :ir is the flag to indicate this is IR level profile.
 :ir
 test_br_1
-784007059655560962
+25571299074
 2
 12884901888
 8589934592
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/branch2.proftext b/llvm/test/Transforms/PGOProfile/Inputs/branch2.proftext
index 3e20359b1b84f..b5fee2b6f4faf 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/branch2.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/branch2.proftext
@@ -1,7 +1,7 @@
 # :ir is the flag to indicate this is IR level profile.
 :ir
 test_br_2
-146835647075900052
+29667547796
 2
 1
 1
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/branch2_entry.proftext b/llvm/test/Transforms/PGOProfile/Inputs/branch2_entry.proftext
index 31710fc8f4bde..51de52d824926 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/branch2_entry.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/branch2_entry.proftext
@@ -2,7 +2,7 @@
 :ir
 :entry_first
 test_br_2
-146835647075900052
+29667547796
 2
 2
 1
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/criticaledge.proftext b/llvm/test/Transforms/PGOProfile/Inputs/criticaledge.proftext
index 85b9779abeece..7613b643b163e 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/criticaledge.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/criticaledge.proftext
@@ -1,7 +1,7 @@
 # :ir is the flag to indicate this is IR level profile.
 :ir
 test_criticalEdge
-93478046750287693
+82323253069
 8
 2
 1
@@ -13,7 +13,7 @@ test_criticalEdge
 1
 
 <stdin>:bar
-742261418966908927
+12884901887
 1
 7
 
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/criticaledge_entry.proftext b/llvm/test/Transforms/PGOProfile/Inputs/criticaledge_entry.proftext
index f1497d6c01c9f..79754261d2f88 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/criticaledge_entry.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/criticaledge_entry.proftext
@@ -2,7 +2,7 @@
 :ir
 :entry_first
 test_criticalEdge
-93478046750287693
+82323253069
 8
 7
 2
@@ -14,7 +14,7 @@ test_criticalEdge
 1
 
 <stdin>:bar
-742261418966908927
+12884901887
 1
 7
 
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/cspgo.proftext b/llvm/test/Transforms/PGOProfile/Inputs/cspgo.proftext
index 6d6d86cfb50bb..e40a0f29fccc0 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/cspgo.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/cspgo.proftext
@@ -54,7 +54,7 @@ bar_m2
 
 foo
 # Func Hash:
-1456607294772657484
+1152921640672869708
 # Num Counters:
 10
 # Counter Values:
@@ -71,7 +71,7 @@ foo
 
 foo
 # Func Hash:
-146835646621254984
+29212902728
 # Num Counters:
 2
 # Counter Values:
@@ -80,7 +80,7 @@ foo
 
 bar
 # Func Hash:
-1440408129826749777
+1152921569533132113
 # Num Counters:
 5
 # Counter Values:
@@ -92,7 +92,7 @@ bar
 
 bar
 # Func Hash:
-567185239050791137
+56228292833
 # Num Counters:
 4
 # Counter Values:
@@ -103,7 +103,7 @@ bar
 
 main
 # Func Hash:
-1895182923573755903
+1152921517491748863
 # Num Counters:
 1
 # Counter Values:
@@ -111,7 +111,7 @@ main
 
 main
 # Func Hash:
-742261418966908927
+12884901887
 # Num Counters:
 1
 # Counter Values:
@@ -135,7 +135,7 @@ csfdo_plain.c:barbar
 
 goo
 # Func Hash:
-1895182923573755903
+1152921517491748863
 # Num Counters:
 1
 # Counter Values:
@@ -143,7 +143,7 @@ goo
 
 goo
 # Func Hash:
-742261418966908927
+12884901887
 # Num Counters:
 1
 # Counter Values:
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/diag_no_value_sites.proftext b/llvm/test/Transforms/PGOProfile/Inputs/diag_no_value_sites.proftext
index 182778755f4bb..b7ba652292ca9 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/diag_no_value_sites.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/diag_no_value_sites.proftext
@@ -1,6 +1,6 @@
 # :ir is the flag to indicate this is IR level profile.
 :ir
 foo
-48277136972185599
+12884901887
 1
 1
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/fix_entry_count.proftext b/llvm/test/Transforms/PGOProfile/Inputs/fix_entry_count.proftext
index 7ca97aa29dbcd..de4771beaacc2 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/fix_entry_count.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/fix_entry_count.proftext
@@ -1,7 +1,7 @@
 :ir
 :entry_first
 test_simple_for
-1063705162469825436
+34137660316
 2
 0
 96
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/func_entry.proftext b/llvm/test/Transforms/PGOProfile/Inputs/func_entry.proftext
index 6d92958e89b87..60c49bf730954 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/func_entry.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/func_entry.proftext
@@ -3,7 +3,7 @@
 :entry_first
 hot
 # Func Hash:
-742261418966908927
+12884901887
 # Num Counters:
 1
 # Counter Values:
@@ -11,7 +11,7 @@ hot
 
 cold
 # Func Hash:
-742261418966908927
+12884901887
 # Num Counters:
 1
 # Counter Values:
@@ -19,7 +19,7 @@ cold
 
 med
 # Func Hash:
-742261418966908927
+12884901887
 # Num Counters:
 1
 # Counter Values:
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/indirect_call.proftext b/llvm/test/Transforms/PGOProfile/Inputs/indirect_call.proftext
index bb48ccc57a8bf..d453090d1c582 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/indirect_call.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/indirect_call.proftext
@@ -1,7 +1,7 @@
 :ir
 bar
 # Func Hash:
-170957022131388415
+281487861612543
 # Num Counters:
 1
 # Counter Values:
@@ -19,7 +19,7 @@ func3:20
 
 func1
 # Func Hash:
-742261418966908927
+12884901887
 # Num Counters:
 1
 # Counter Values:
@@ -27,7 +27,7 @@ func1
 
 func2
 # Func Hash:
-742261418966908927
+12884901887
 # Num Counters:
 1
 # Counter Values:
@@ -35,7 +35,7 @@ func2
 
 func3
 # Func Hash:
-742261418966908927
+12884901887
 # Num Counters:
 1
 # Counter Values:
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/indirectbr.proftext b/llvm/test/Transforms/PGOProfile/Inputs/indirectbr.proftext
index 49fafd9d99bf9..3909968476272 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/indirectbr.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/indirectbr.proftext
@@ -2,7 +2,7 @@
 :ir
 foo
 # Func Hash:
-844982796158316421
+47485104005
 # Num Counters:
 4
 # Counter Values:
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/indirectbr_entry.proftext b/llvm/test/Transforms/PGOProfile/Inputs/indirectbr_entry.proftext
index 6910f7e21d677..7173269d20cc9 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/indirectbr_entry.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/indirectbr_entry.proftext
@@ -3,7 +3,7 @@
 :entry_first
 foo
 # Func Hash:
-844982796158316421
+47485104005
 # Num Counters:
 4
 # Counter Values:
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/irreducible.proftext b/llvm/test/Transforms/PGOProfile/Inputs/irreducible.proftext
index 24eacb886489e..9b0210d9a309f 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/irreducible.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/irreducible.proftext
@@ -1,7 +1,7 @@
 :ir
 _Z11irreducibleii
 # Func Hash:
-287486624745028451
+64451410787
 # Num Counters:
 6
 # Counter Values:
@@ -14,7 +14,7 @@ _Z11irreducibleii
 
 _Z11irreduciblePh
 # Func Hash:
-331779889035882993
+104649601521
 # Num Counters:
 9
 # Counter Values:
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/irreducible_entry.proftext b/llvm/test/Transforms/PGOProfile/Inputs/irreducible_entry.proftext
index 702c42d1addc8..d04cfddda896d 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/irreducible_entry.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/irreducible_entry.proftext
@@ -2,7 +2,7 @@
 :entry_first
 _Z11irreducibleii
 # Func Hash:
-287486624745028451
+64451410787
 # Num Counters:
 6
 # Counter Values:
@@ -15,7 +15,7 @@ _Z11irreducibleii
 
 _Z11irreduciblePh
 # Func Hash:
-331779889035882993
+104649601521
 # Num Counters:
 9
 # Counter Values:
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/landingpad.proftext b/llvm/test/Transforms/PGOProfile/Inputs/landingpad.proftext
index 7a74b296e55fe..c71fd04170bee 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/landingpad.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/landingpad.proftext
@@ -1,7 +1,7 @@
 # :ir is the flag to indicate this is IR level profile.
 :ir
 foo
-567185241952511723
+59130013419
 4
 3
 1
@@ -9,7 +9,7 @@ foo
 0
 
 bar
-784007058953177093
+24868915205
 2
 3
 2
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/landingpad_entry.proftext b/llvm/test/Transforms/PGOProfile/Inputs/landingpad_entry.proftext
index 9b604038abcc4..3175d9c0d630b 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/landingpad_entry.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/landingpad_entry.proftext
@@ -2,7 +2,7 @@
 :ir
 :entry_first
 foo
-567185241952511723
+59130013419
 4
 5
 1
@@ -10,7 +10,7 @@ foo
 0
 
 bar
-784007058953177093
+24868915205
 2
 3
 2
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/large_count_remarks.proftext b/llvm/test/Transforms/PGOProfile/Inputs/large_count_remarks.proftext
index 8e23ae6906b0c..814af7402b28a 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/large_count_remarks.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/large_count_remarks.proftext
@@ -1,7 +1,7 @@
 # :ir is the flag to indicate this is IR level profile.
 :ir
 test
-784007059655560962
+25571299074
 2
 40000000000
 20000000000
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/loop1.proftext b/llvm/test/Transforms/PGOProfile/Inputs/loop1.proftext
index 25eeefe33c11f..c19737149ca25 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/loop1.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/loop1.proftext
@@ -1,7 +1,7 @@
 # :ir is the flag to indicate this is IR level profile.
 :ir
 test_simple_for
-1063705162469825436
+34137660316
 2
 96
 4
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/loop1_entry.proftext b/llvm/test/Transforms/PGOProfile/Inputs/loop1_entry.proftext
index 56f04e808aee8..b61634d94689d 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/loop1_entry.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/loop1_entry.proftext
@@ -2,7 +2,7 @@
 :ir
 :entry_first
 test_simple_for
-1063705162469825436
+34137660316
 2
 4
 96
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/loop2.proftext b/llvm/test/Transforms/PGOProfile/Inputs/loop2.proftext
index 7ab9cd6b946e4..af3a71df1e6b9 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/loop2.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/loop2.proftext
@@ -1,7 +1,7 @@
 # :ir is the flag to indicate this is IR level profile.
 :ir
 test_nested_for
-798733566382720768
+53929068288
 3
 33
 10
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/loop2_entry.proftext b/llvm/test/Transforms/PGOProfile/Inputs/loop2_entry.proftext
index bde1b7e4a4c12..ac8536a52675b 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/loop2_entry.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/loop2_entry.proftext
@@ -2,7 +2,7 @@
 :ir
 :entry_first
 test_nested_for
-798733566382720768
+53929068288
 3
 6
 33
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/memop_size_annotation.proftext b/llvm/test/Transforms/PGOProfile/Inputs/memop_size_annotation.proftext
index 83aec54f23283..cce1a67a94574 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/memop_size_annotation.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/memop_size_annotation.proftext
@@ -2,7 +2,7 @@
 :ir
 foo
 # Func Hash:
-687116424982578944
+53929068288
 # Num Counters:
 3
 # Counter Values:
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/misexpect-branch-correct.proftext b/llvm/test/Transforms/PGOProfile/Inputs/misexpect-branch-correct.proftext
index 24c9746262bda..c785a40efea59 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/misexpect-branch-correct.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/misexpect-branch-correct.proftext
@@ -2,7 +2,7 @@
 :ir
 bar
 # Func Hash:
-146835647075900052
+29667547796
 # Num Counters:
 2
 # Counter Values:
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/misexpect-branch.proftext b/llvm/test/Transforms/PGOProfile/Inputs/misexpect-branch.proftext
index 5505df7375271..36eaa3462782f 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/misexpect-branch.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/misexpect-branch.proftext
@@ -2,7 +2,7 @@
 :ir
 bar
 # Func Hash:
-146835647075900052
+29667547796
 # Num Counters:
 2
 # Counter Values:
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/misexpect-branch_entry.proftext b/llvm/test/Transforms/PGOProfile/Inputs/misexpect-branch_entry.proftext
index 752b885ad2b2c..df7d6cf80466b 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/misexpect-branch_entry.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/misexpect-branch_entry.proftext
@@ -3,7 +3,7 @@
 :entry_first
 bar
 # Func Hash:
-146835647075900052
+29667547796
 # Num Counters:
 2
 # Counter Values:
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/misexpect-switch-correct.proftext b/llvm/test/Transforms/PGOProfile/Inputs/misexpect-switch-correct.proftext
index 39864486bb96b..b0924938ea831 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/misexpect-switch-correct.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/misexpect-switch-correct.proftext
@@ -2,7 +2,7 @@
 :ir
 main
 # Func Hash:
-391331300939170156
+74054140268
 # Num Counters:
 7
 # Counter Values:
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/misexpect-switch-correct_entry.proftext b/llvm/test/Transforms/PGOProfile/Inputs/misexpect-switch-correct_entry.proftext
index c74f3230939ee..4686fec3bee78 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/misexpect-switch-correct_entry.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/misexpect-switch-correct_entry.proftext
@@ -3,7 +3,7 @@
 :entry_first
 main
 # Func Hash:
-391331300939170156
+74054140268
 # Num Counters:
 7
 # Counter Values:
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/misexpect-switch.proftext b/llvm/test/Transforms/PGOProfile/Inputs/misexpect-switch.proftext
index ea50f6b6043f4..8e8049f3dc62e 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/misexpect-switch.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/misexpect-switch.proftext
@@ -2,7 +2,7 @@
 :ir
 main
 # Func Hash:
-391331300939170156
+74054140268
 # Num Counters:
 7
 # Counter Values:
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/misexpect-switch_entry.proftext b/llvm/test/Transforms/PGOProfile/Inputs/misexpect-switch_entry.proftext
index 6afb6c9019f24..3da4c10ac19d1 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/misexpect-switch_entry.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/misexpect-switch_entry.proftext
@@ -3,7 +3,7 @@
 :entry_first
 main
 # Func Hash:
-391331300939170156
+74054140268
 # Num Counters:
 7
 # Counter Values:
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/multiple_hash_profile.proftext b/llvm/test/Transforms/PGOProfile/Inputs/multiple_hash_profile.proftext
index 77f8d5a5ade3b..5bf67fb2bfaf2 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/multiple_hash_profile.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/multiple_hash_profile.proftext
@@ -2,22 +2,12 @@
 :ir
 _Z3fooi
 # Func Hash:
-382993475055910911
-# Num Counters:
-2
-# Counter Values:
-18
-12
-
-# For -pgo-instr-old-cfg-hashing=true
-_Z3fooi
-# Func Hash:
 72057606922829823
 # Num Counters:
 2
 # Counter Values:
 18
-6
+12
 
 _Z3fooi
 # Func Hash:
@@ -27,16 +17,6 @@ _Z3fooi
 # Counter Values:
 0
 
-_Z3bari
-# Func Hash:
-382993475055910911
-# Num Counters:
-2
-# Counter Values:
-0
-0
-
-# For -pgo-instr-old-cfg-hashing=true
 _Z3bari
 # Func Hash:
 72057606922829823
@@ -46,15 +26,6 @@ _Z3bari
 0
 0
 
-_Z4m2f1v
-# Func Hash:
-742261418966908927
-# Num Counters:
-1
-# Counter Values:
-1
-
-# For -pgo-instr-old-cfg-hashing=true
 _Z4m2f1v
 # Func Hash:
 12884901887
@@ -62,3 +33,4 @@ _Z4m2f1v
 1
 # Counter Values:
 1
+
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/noreturncall.proftext b/llvm/test/Transforms/PGOProfile/Inputs/noreturncall.proftext
index bb4b0719cf129..1bedce5b9f9b3 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/noreturncall.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/noreturncall.proftext
@@ -2,7 +2,7 @@
 :ir
 foo
 # Func Hash:
-238984482720105921
+36496524737
 # Num Counters:
 3
 # Counter Values:
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/remap.proftext b/llvm/test/Transforms/PGOProfile/Inputs/remap.proftext
index d9b22637d3a7d..40054d78f5aab 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/remap.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/remap.proftext
@@ -1,7 +1,7 @@
 # :ir is the flag to indicate this is IR level profile.
 :ir
 _ZN3foo3barERKN1N1XINS_4quuxEEE
-784007059655560962
+25571299074
 2
 3
 2
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/select1.proftext b/llvm/test/Transforms/PGOProfile/Inputs/select1.proftext
index c4d6d56073b01..60663e02ffb49 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/select1.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/select1.proftext
@@ -1,7 +1,7 @@
 :ir
 :entry_first
 test_br_2
-942389667449461396
+72057623705475732
 3
 5
 1
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/select2.proftext b/llvm/test/Transforms/PGOProfile/Inputs/select2.proftext
index 0d4b003ec0667..8f3c562097fae 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/select2.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/select2.proftext
@@ -3,7 +3,7 @@
 :entry_first
 foo
 # Func Hash:
-134732432632142748
+72057628175588252
 # Num Counters:
 3
 # Counter Values:
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/suppl-profile.proftext b/llvm/test/Transforms/PGOProfile/Inputs/suppl-profile.proftext
index 41f4fbe655fac..c82311b3e0c06 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/suppl-profile.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/suppl-profile.proftext
@@ -1,7 +1,7 @@
 # :ir is the flag to indicate this is IR level profile.
 :ir
 test_simple_for
-1063705162469825436
+34137660316
 2
 0
 0
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/switch.proftext b/llvm/test/Transforms/PGOProfile/Inputs/switch.proftext
index cf73ed083794e..bebd65fa4c9fd 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/switch.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/switch.proftext
@@ -1,7 +1,7 @@
 # :ir is the flag to indicate this is IR level profile.
 :ir
 test_switch
-536873293052540031
+46200943743
 4
 0
 5
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/switch_entry.proftext b/llvm/test/Transforms/PGOProfile/Inputs/switch_entry.proftext
index cf079893e3404..20df873608c19 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/switch_entry.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/switch_entry.proftext
@@ -2,7 +2,7 @@
 :ir
 :entry_first
 test_switch
-536873293052540031
+46200943743
 4
 10
 5
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/thinlto_cs.proftext b/llvm/test/Transforms/PGOProfile/Inputs/thinlto_cs.proftext
index 1b9f19e7f7fa5..4717752da716d 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/thinlto_cs.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/thinlto_cs.proftext
@@ -10,7 +10,7 @@ cond.llvm.11253644763537639171
 
 foo
 # Func Hash:
-1720106746050921044
+29212902728
 # Num Counters:
 2
 # Counter Values:
@@ -19,7 +19,7 @@ foo
 
 bar
 # Func Hash:
-1299757151682747028
+1152921534274394772
 # Num Counters:
 2
 # Counter Values:
@@ -45,7 +45,7 @@ main
 
 main
 # Func Hash:
-1895182923573755903
+12884901887
 # Num Counters:
 1
 # Counter Values:
@@ -53,7 +53,7 @@ main
 
 cspgo.c:foo
 # Func Hash:
-1720106746050921044
+1152921563228422740
 # Num Counters:
 4
 # Counter Values:
diff --git a/llvm/test/Transforms/PGOProfile/multiple_hash_profile.ll b/llvm/test/Transforms/PGOProfile/multiple_hash_profile.ll
index f0011114f0d01..fdfb16b9e2e55 100644
--- a/llvm/test/Transforms/PGOProfile/multiple_hash_profile.ll
+++ b/llvm/test/Transforms/PGOProfile/multiple_hash_profile.ll
@@ -1,8 +1,6 @@
 ; RUN: llvm-profdata merge %S/Inputs/multiple_hash_profile.proftext -o %t.profdata
 ; RUN: opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata  -S | FileCheck %s
-; RUN: opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata -pgo-instr-old-cfg-hashing=true -S | FileCheck -check-prefix=CHECKOLDHASH %s
 ; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s
-; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -pgo-instr-old-cfg-hashing=true -S | FileCheck -check-prefix=CHECKOLDHASH %s
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
@@ -31,9 +29,6 @@ entry:
 ; CHECK: %mul.i = select i1 %cmp.i, i32 1, i32 %i
 ; CHECK-SAME: !prof ![[BW:[0-9]+]]
 ; CHECK: ![[BW]] = !{!"branch_weights", i32 12, i32 6}
-; CHECKOLDHASH: %mul.i = select i1 %cmp.i, i32 1, i32 %i
-; CHECKOLDHASH-SAME: !prof ![[BW:[0-9]+]]
-; CHECKOLDHASH: ![[BW]] = !{!"branch_weights", i32 6, i32 12}
   %retval.0.i = mul nsw i32 %mul.i, %i
   ret i32 %retval.0.i
 }

From dc22182d1f7489239388f7bdd04170573c0652a3 Mon Sep 17 00:00:00 2001
From: Alex Lorenz <arphaman@gmail.com>
Date: Fri, 24 Jul 2020 16:01:48 -0700
Subject: [PATCH 0556/1035] [darwin] build and link with a separate compiler-rt
 builtins library for device simulators

This change separates out the iOS/tvOS/watchOS simulator slices from the "libclang_rt.<os>.a"
fat archive, by moving them out to their own "libclang_rt.<os>sim.a" static archive.
This allows us to build and to link with an arm64 device simulator slice for the simulators running
on Apple Silicons, and to distribute it in one archive alongside the Intel simulator slices.

Differential Revision: https://reviews.llvm.org/D84564
---
 clang/lib/Driver/ToolChains/Darwin.cpp        |  5 ++--
 .../lib/darwin/libclang_rt.iossim.a           |  0
 clang/test/Driver/darwin-ld.c                 |  2 +-
 .../cmake/Modules/CompilerRTDarwinUtils.cmake | 24 ++++++++++++-------
 4 files changed, 18 insertions(+), 13 deletions(-)
 create mode 100644 clang/test/Driver/Inputs/resource_dir/lib/darwin/libclang_rt.iossim.a

diff --git a/clang/lib/Driver/ToolChains/Darwin.cpp b/clang/lib/Driver/ToolChains/Darwin.cpp
index 325dcb7df545c..cf8a3e6aee273 100644
--- a/clang/lib/Driver/ToolChains/Darwin.cpp
+++ b/clang/lib/Driver/ToolChains/Darwin.cpp
@@ -1062,10 +1062,9 @@ void MachO::AddLinkRuntimeLib(const ArgList &Args, ArgStringList &CmdArgs,
     DarwinLibName += Component;
     if (!(Opts & RLO_IsEmbedded))
       DarwinLibName += "_";
-    DarwinLibName += getOSLibraryNameSuffix();
-  } else
-    DarwinLibName += getOSLibraryNameSuffix(true);
+  }
 
+  DarwinLibName += getOSLibraryNameSuffix();
   DarwinLibName += IsShared ? "_dynamic.dylib" : ".a";
   SmallString<128> Dir(getDriver().ResourceDir);
   llvm::sys::path::append(
diff --git a/clang/test/Driver/Inputs/resource_dir/lib/darwin/libclang_rt.iossim.a b/clang/test/Driver/Inputs/resource_dir/lib/darwin/libclang_rt.iossim.a
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/clang/test/Driver/darwin-ld.c b/clang/test/Driver/darwin-ld.c
index ea71142e88c19..82cd36d59f80d 100644
--- a/clang/test/Driver/darwin-ld.c
+++ b/clang/test/Driver/darwin-ld.c
@@ -156,7 +156,7 @@
 // RUN: FileCheck -check-prefix=LINK_IOSSIM_PROFILE %s < %t.log
 // LINK_IOSSIM_PROFILE: {{ld(.exe)?"}}
 // LINK_IOSSIM_PROFILE: libclang_rt.profile_iossim.a
-// LINK_IOSSIM_PROFILE: libclang_rt.ios.a
+// LINK_IOSSIM_PROFILE: libclang_rt.iossim.a
 
 // RUN: %clang -target arm64-apple-tvos8.3 -mlinker-version=400 -mtvos-version-min=8.3 -resource-dir=%S/Inputs/resource_dir -### %t.o 2> %t.log
 // RUN: FileCheck -check-prefix=LINK_TVOS_ARM64 %s < %t.log
diff --git a/compiler-rt/cmake/Modules/CompilerRTDarwinUtils.cmake b/compiler-rt/cmake/Modules/CompilerRTDarwinUtils.cmake
index be8d7e733c7a0..c37b5e98bcc68 100644
--- a/compiler-rt/cmake/Modules/CompilerRTDarwinUtils.cmake
+++ b/compiler-rt/cmake/Modules/CompilerRTDarwinUtils.cmake
@@ -289,6 +289,15 @@ macro(darwin_add_builtin_library name suffix)
     endforeach(cflag)
   endif()
 
+  if ("${LIB_OS}" MATCHES ".*sim$")
+    # Pass an explicit -simulator environment to the -target option to ensure
+    # that we don't rely on the architecture to infer whether we're building
+    # for the simulator.
+    string(REGEX REPLACE "sim" "" base_os "${LIB_OS}")
+    list(APPEND builtin_cflags
+         -target "${LIB_ARCH}-apple-${base_os}${DARWIN_${LIBOS}_BUILTIN_MIN_VER}-simulator")
+  endif()
+
   set_target_compile_flags(${libname}
     ${sysroot_flag}
     ${DARWIN_${LIB_OS}_BUILTIN_MIN_VER_FLAG}
@@ -449,16 +458,13 @@ macro(darwin_add_builtin_libraries)
     endif()
   endforeach()
 
-  # We put the x86 sim slices into the archives for their base OS
   foreach (os ${ARGN})
-    if(NOT ${os} MATCHES ".*sim$")
-      darwin_lipo_libs(clang_rt.${os}
-                        PARENT_TARGET builtins
-                        LIPO_FLAGS ${${os}_builtins_lipo_flags} ${${os}sim_builtins_lipo_flags}
-                        DEPENDS ${${os}_builtins_libs} ${${os}sim_builtins_libs}
-                        OUTPUT_DIR ${COMPILER_RT_LIBRARY_OUTPUT_DIR}
-                        INSTALL_DIR ${COMPILER_RT_LIBRARY_INSTALL_DIR})
-    endif()
+    darwin_lipo_libs(clang_rt.${os}
+                     PARENT_TARGET builtins
+                     LIPO_FLAGS ${${os}_builtins_lipo_flags}
+                     DEPENDS ${${os}_builtins_libs}
+                     OUTPUT_DIR ${COMPILER_RT_LIBRARY_OUTPUT_DIR}
+                     INSTALL_DIR ${COMPILER_RT_LIBRARY_INSTALL_DIR})
   endforeach()
   darwin_add_embedded_builtin_libraries()
 endmacro()

From 6587ff77ea7aea508dde4ff68bf89e301756a52c Mon Sep 17 00:00:00 2001
From: Jonas Devlieghere <jonas@devlieghere.com>
Date: Wed, 29 Jul 2020 15:37:11 -0700
Subject: [PATCH 0557/1035] [lldb] Fix llvm-lit in standalone builds

LLVM's make_paths_relative uses Python3_EXECUTABLE which isn't set in
standalone LLDB builds.
---
 lldb/test/CMakeLists.txt | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/lldb/test/CMakeLists.txt b/lldb/test/CMakeLists.txt
index 96931cbaeaf99..dc5af5e2defe7 100644
--- a/lldb/test/CMakeLists.txt
+++ b/lldb/test/CMakeLists.txt
@@ -216,6 +216,9 @@ add_dependencies(check-lldb-reproducers check-lldb-reproducers-capture)
 if(LLDB_BUILT_STANDALONE)
   # This has to happen *AFTER* add_lit_testsuite.
   if (EXISTS ${LLVM_MAIN_SRC_DIR}/utils/llvm-lit)
+    # LLVM's make_paths_relative uses Python3_EXECUTABLE which isn't set in a
+    # standalone LLDB build.
+    set(Python3_EXECUTABLE ${PYTHON_EXECUTABLE})
     add_subdirectory(${LLVM_MAIN_SRC_DIR}/utils/llvm-lit ${CMAKE_CURRENT_BINARY_DIR}/llvm-lit)
   endif()
 endif()

From 926c14798d317a8c60592f420bb79cd80c150285 Mon Sep 17 00:00:00 2001
From: Francis Visoiu Mistrih <francisvm@yahoo.com>
Date: Wed, 29 Jul 2020 15:41:04 -0700
Subject: [PATCH 0558/1035] Revert "build: always add support for assembly
 targets on Darwin"

This reverts commit d8e8e32d85f1cbde2a6b67af010fba3c3a3c86e9.

This breaks the build on GreenDragon:
http://green.lab.llvm.org/green/job/clang-stage1-RA/13171/console
---
 compiler-rt/cmake/Modules/AddCompilerRT.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/compiler-rt/cmake/Modules/AddCompilerRT.cmake b/compiler-rt/cmake/Modules/AddCompilerRT.cmake
index 9c135ef931a03..efb6608182705 100644
--- a/compiler-rt/cmake/Modules/AddCompilerRT.cmake
+++ b/compiler-rt/cmake/Modules/AddCompilerRT.cmake
@@ -110,7 +110,7 @@ endfunction()
 function(add_asm_sources output)
   set(${output} ${ARGN} PARENT_SCOPE)
   # Xcode will try to compile asm files as C ('clang -x c'), and that will fail.
-  if(CMAKE_SYSTEM_NAME STREQUAL Darwin)
+  if (${CMAKE_GENERATOR} STREQUAL "Xcode")
     enable_language(ASM)
   else()
     # Pass ASM file directly to the C++ compiler.

From 005991a3fe21f2e027c248f20e44314f19ce8f0b Mon Sep 17 00:00:00 2001
From: Matt Morehouse <mascasa@google.com>
Date: Wed, 29 Jul 2020 23:18:07 +0000
Subject: [PATCH 0559/1035] [DFSan] Remove dfsan_use_fast16labels from abilist.

Its implementation was scrapped in the final fast16labels
instrumentation patch.
---
 compiler-rt/lib/dfsan/done_abilist.txt | 2 --
 1 file changed, 2 deletions(-)

diff --git a/compiler-rt/lib/dfsan/done_abilist.txt b/compiler-rt/lib/dfsan/done_abilist.txt
index 540f0a4c6bfcd..52f3ff5ef2395 100644
--- a/compiler-rt/lib/dfsan/done_abilist.txt
+++ b/compiler-rt/lib/dfsan/done_abilist.txt
@@ -28,8 +28,6 @@ fun:dfsan_set_write_callback=uninstrumented
 fun:dfsan_set_write_callback=custom
 fun:dfsan_flush=uninstrumented
 fun:dfsan_flush=discard
-fun:dfsan_use_fast16labels=uninstrumented
-fun:dfsan_use_fast16labels=discard
 
 ###############################################################################
 # glibc

From 71059257bd4b9780ddf8de4248e415535d8dbbfe Mon Sep 17 00:00:00 2001
From: Mircea Trofin <mtrofin@google.com>
Date: Wed, 29 Jul 2020 16:29:21 -0700
Subject: [PATCH 0560/1035] [llvm][NFC] TensorSpec abstraction for ML evaluator

Further abstracting the specification of a tensor, to more easily
support different types and shapes of tensor, and also to perform
initialization up-front, at TFModelEvaluator construction time.

Differential Revision: https://reviews.llvm.org/D84685
---
 llvm/include/llvm/Analysis/Utils/TFUtils.h    | 72 ++++++++++-----
 .../Analysis/DevelopmentModeInlineAdvisor.cpp | 60 +++----------
 .../Analysis/InlineSizeEstimatorAnalysis.cpp  | 15 ++--
 llvm/lib/Analysis/TFUtils.cpp                 | 90 ++++++++-----------
 llvm/unittests/Analysis/TFUtilsTest.cpp       | 22 +++--
 5 files changed, 115 insertions(+), 144 deletions(-)

diff --git a/llvm/include/llvm/Analysis/Utils/TFUtils.h b/llvm/include/llvm/Analysis/Utils/TFUtils.h
index b93ca88ec1c93..512f45bb5671a 100644
--- a/llvm/include/llvm/Analysis/Utils/TFUtils.h
+++ b/llvm/include/llvm/Analysis/Utils/TFUtils.h
@@ -36,6 +36,43 @@ namespace llvm {
 class TFModelEvaluatorImpl;
 class EvaluationResultImpl;
 
+/// TensorSpec encapsulates the specification of a tensor: its dimensions, or
+/// "shape" (row-major), its type (see TensorSpec::getDataType specializations
+/// for supported types), its name and port (see "TensorFlow: Large-Scale
+/// Machine Learning on Heterogeneous Distributed Systems", section 4.2, para 2:
+/// https://static.googleusercontent.com/media/research.google.com/en//pubs/archive/45166.pdf)
+///
+/// TensorSpec is used to set up a TFModelEvaluator by describing the expected
+/// inputs and outputs.
+class TensorSpec final {
+public:
+  template <typename T>
+  static TensorSpec createSpec(const std::string &Name,
+                               const std::vector<int64_t> &Shape,
+                               int Port = 0) {
+    return TensorSpec(Name, Port, getDataType<T>(), Shape);
+  }
+
+  const std::string &name() const { return Name; }
+  int port() const { return Port; }
+  int typeIndex() const { return TypeIndex; }
+  const std::vector<int64_t> &shape() const { return Shape; }
+
+private:
+  TensorSpec(const std::string &Name, int Port, int TypeIndex,
+             const std::vector<int64_t> &Shape)
+      : Name(Name), Port(Port), TypeIndex(TypeIndex), Shape(Shape) {}
+
+  template <typename T> static int getDataType() {
+    llvm_unreachable("Undefined tensor type");
+  }
+
+  std::string Name;
+  int Port = 0;
+  int TypeIndex = 0;
+  std::vector<int64_t> Shape;
+};
+
 class TFModelEvaluator final {
 public:
   /// The result of a model evaluation. Handles the lifetime of the output
@@ -60,8 +97,8 @@ class TFModelEvaluator final {
   };
 
   TFModelEvaluator(StringRef SavedModelPath,
-                   const std::vector<std::string> &InputNames,
-                   const std::vector<std::string> &OutputNames,
+                   const std::vector<TensorSpec> &InputSpecs,
+                   const std::vector<TensorSpec> &OutputSpecs,
                    const char *Tags = "serve");
   ~TFModelEvaluator();
   TFModelEvaluator(const TFModelEvaluator &) = delete;
@@ -82,32 +119,21 @@ class TFModelEvaluator final {
   /// otherwise.
   bool isValid() const { return !!Impl; }
 
-  /// Initialize the input at Index as a tensor of the given type and
-  /// dimensions.
-  template <typename T>
-  void initInput(size_t Index, const std::vector<int64_t> &Dimensions) {
-    return initInput(Index, getModelTypeIndex<T>(), Dimensions);
-  }
-
 private:
   void *getUntypedInput(size_t Index);
-  template <typename T> int getModelTypeIndex();
-  void initInput(size_t Index, int TypeIndex,
-                 const std::vector<int64_t> &Dimensions);
-
   std::unique_ptr<TFModelEvaluatorImpl> Impl;
 };
 
-template <> int TFModelEvaluator::getModelTypeIndex<float>();
-template <> int TFModelEvaluator::getModelTypeIndex<double>();
-template <> int TFModelEvaluator::getModelTypeIndex<int8_t>();
-template <> int TFModelEvaluator::getModelTypeIndex<uint8_t>();
-template <> int TFModelEvaluator::getModelTypeIndex<int16_t>();
-template <> int TFModelEvaluator::getModelTypeIndex<uint16_t>();
-template <> int TFModelEvaluator::getModelTypeIndex<int32_t>();
-template <> int TFModelEvaluator::getModelTypeIndex<uint32_t>();
-template <> int TFModelEvaluator::getModelTypeIndex<int64_t>();
-template <> int TFModelEvaluator::getModelTypeIndex<uint64_t>();
+template <> int TensorSpec::getDataType<float>();
+template <> int TensorSpec::getDataType<double>();
+template <> int TensorSpec::getDataType<int8_t>();
+template <> int TensorSpec::getDataType<uint8_t>();
+template <> int TensorSpec::getDataType<int16_t>();
+template <> int TensorSpec::getDataType<uint16_t>();
+template <> int TensorSpec::getDataType<int32_t>();
+template <> int TensorSpec::getDataType<uint32_t>();
+template <> int TensorSpec::getDataType<int64_t>();
+template <> int TensorSpec::getDataType<uint64_t>();
 
 } // namespace llvm
 
diff --git a/llvm/lib/Analysis/DevelopmentModeInlineAdvisor.cpp b/llvm/lib/Analysis/DevelopmentModeInlineAdvisor.cpp
index ba9343543dbbc..32fd16810dc35 100644
--- a/llvm/lib/Analysis/DevelopmentModeInlineAdvisor.cpp
+++ b/llvm/lib/Analysis/DevelopmentModeInlineAdvisor.cpp
@@ -298,35 +298,12 @@ class ModelUnderTrainingRunner final : public MLModelRunner {
 private:
   std::unique_ptr<TFModelEvaluator> Evaluator;
 
-  // The training framework needs some additional features, that just need to
-  // be set to 0.
-  struct TensorSpec {
-    std::string Name;
-    std::function<void(TFModelEvaluator *, size_t Index,
-                       const std::vector<int64_t> &Dim)>
-        Initializer;
-  };
-
+  // The training framework needs some additional features.
   const std::vector<TensorSpec> TrainingOnlyFeatures{
-      {"inlining_default",
-       [](TFModelEvaluator *Evaluator, size_t Index,
-          const std::vector<int64_t> &Dim) {
-         Evaluator->initInput<int64_t>(Index, Dim);
-       }},
-      {"discount",
-       [](TFModelEvaluator *Evaluator, size_t Index,
-          const std::vector<int64_t> &Dim) {
-         Evaluator->initInput<float>(Index, Dim);
-       }},
-      {"reward",
-       [](TFModelEvaluator *Evaluator, size_t Index,
-          const std::vector<int64_t> &Dim) {
-         Evaluator->initInput<float>(Index, Dim);
-       }},
-      {"step_type", [](TFModelEvaluator *Evaluator, size_t Index,
-                       const std::vector<int64_t> &Dim) {
-         Evaluator->initInput<int32_t>(Index, Dim);
-       }}};
+      TensorSpec::createSpec<int64_t>(TFFeedPrefix + "inlining_default", {1}),
+      TensorSpec::createSpec<float>(TFFeedPrefix + "discount", {1}),
+      TensorSpec::createSpec<float>(TFFeedPrefix + "reward", {1}),
+      TensorSpec::createSpec<int32_t>(TFFeedPrefix + "step_type", {1})};
 };
 } // namespace
 
@@ -409,33 +386,22 @@ size_t DevelopmentModeMLInlineAdvisor::getTotalSizeEstimate() {
 ModelUnderTrainingRunner::ModelUnderTrainingRunner(LLVMContext &Ctx,
                                                    const std::string &ModelPath)
     : MLModelRunner(Ctx) {
-  std::vector<std::string> InputNames;
-  std::vector<std::string> OutputNames;
+  std::vector<TensorSpec> InputSpecs;
+  std::vector<TensorSpec> OutputSpecs;
   for (size_t I = 0; I < NumberOfFeatures; ++I)
-    InputNames.push_back(TFFeedPrefix + FeatureNameMap[I]);
-  for (size_t I = 0; I < TrainingOnlyFeatures.size(); ++I)
-    InputNames.push_back(TFFeedPrefix + TrainingOnlyFeatures[I].Name);
-  OutputNames.push_back(TFDecisionName);
+    InputSpecs.push_back(
+        TensorSpec::createSpec<int64_t>(TFFeedPrefix + FeatureNameMap[I], {1}));
+  InputSpecs.insert(InputSpecs.end(), TrainingOnlyFeatures.begin(),
+                    TrainingOnlyFeatures.end());
+  OutputSpecs.push_back(TensorSpec::createSpec<int64_t>(TFDecisionName, {1}));
 
   Evaluator =
-      std::make_unique<TFModelEvaluator>(ModelPath, InputNames, OutputNames);
+      std::make_unique<TFModelEvaluator>(ModelPath, InputSpecs, OutputSpecs);
   if (!Evaluator || !Evaluator->isValid()) {
     Ctx.emitError("Failed to create inliner saved model evaluator");
     Evaluator.reset();
     return;
   }
-
-  static const std::vector<int64_t> Dim{1};
-
-  size_t InputIndex = 0;
-  for (; InputIndex < NumberOfFeatures; ++InputIndex) {
-    Evaluator->initInput<int64_t>(InputIndex, Dim);
-  }
-
-  for (; InputIndex < InputNames.size(); ++InputIndex) {
-    TrainingOnlyFeatures[InputIndex - NumberOfFeatures].Initializer(
-        Evaluator.get(), InputIndex, Dim);
-  }
 }
 
 bool ModelUnderTrainingRunner::run() {
diff --git a/llvm/lib/Analysis/InlineSizeEstimatorAnalysis.cpp b/llvm/lib/Analysis/InlineSizeEstimatorAnalysis.cpp
index cffdbe4116086..dc426aaccb22a 100644
--- a/llvm/lib/Analysis/InlineSizeEstimatorAnalysis.cpp
+++ b/llvm/lib/Analysis/InlineSizeEstimatorAnalysis.cpp
@@ -244,19 +244,18 @@ InlineSizeEstimatorAnalysis::InlineSizeEstimatorAnalysis() {
   if (!isEvaluatorRequested()) {
     return;
   }
-  std::vector<std::string> InputNames{"serving_default_input_1"};
-  std::vector<std::string> OutputName{"StatefulPartitionedCall"};
+  std::vector<TensorSpec> InputSpecs{TensorSpec::createSpec<int32_t>(
+      "serving_default_input_1",
+      {1, static_cast<int64_t>(
+              IRToNativeSizeLearning::FunctionFeatures::FeatureCount)})};
+  std::vector<TensorSpec> OutputSpecs{
+      TensorSpec::createSpec<float>("StatefulPartitionedCall", {1})};
   Evaluator = std::make_unique<TFModelEvaluator>(
-      TFIR2NativeModelPath.getValue().c_str(), InputNames, OutputName);
+      TFIR2NativeModelPath.getValue().c_str(), InputSpecs, OutputSpecs);
   if (!Evaluator || !Evaluator->isValid()) {
     Evaluator.reset();
     return;
   }
-  static const std::vector<int64_t> Dim{
-      1, static_cast<int64_t>(
-             IRToNativeSizeLearning::FunctionFeatures::FeatureCount)};
-
-  Evaluator->initInput<int32_t>(0, Dim);
 }
 
 InlineSizeEstimatorAnalysis::Result
diff --git a/llvm/lib/Analysis/TFUtils.cpp b/llvm/lib/Analysis/TFUtils.cpp
index a6f5b2956399e..b0ff19857963b 100644
--- a/llvm/lib/Analysis/TFUtils.cpp
+++ b/llvm/lib/Analysis/TFUtils.cpp
@@ -86,8 +86,8 @@ class EvaluationResultImpl {
 class TFModelEvaluatorImpl {
 public:
   TFModelEvaluatorImpl(StringRef SavedModelPath,
-                       const std::vector<std::string> &InputNames,
-                       const std::vector<std::string> &OutputNames,
+                       const std::vector<TensorSpec> &InputSpecs,
+                       const std::vector<TensorSpec> &OutputSpecs,
                        const char *Tags);
 
   bool isValid() const { return IsValid; }
@@ -132,16 +132,17 @@ class TFModelEvaluatorImpl {
 
   /// Reusable utility for ensuring we can bind the requested Name to a node in
   /// the SavedModel Graph.
-  bool checkReportAndInvalidate(const TF_Output &Output, StringRef Name);
+  bool checkReportAndInvalidate(const TF_Output &Output,
+                                const TensorSpec &OutputSpec);
 };
 } // namespace llvm
 
 TFModelEvaluatorImpl::TFModelEvaluatorImpl(
-    StringRef SavedModelPath, const std::vector<std::string> &InputNames,
-    const std::vector<std::string> &OutputNames, const char *Tags)
+    StringRef SavedModelPath, const std::vector<TensorSpec> &InputSpecs,
+    const std::vector<TensorSpec> &OutputSpecs, const char *Tags)
     : Graph(createTFGraph()), Options(createTFSessionOptions()),
-      InputFeed(InputNames.size()), Input(InputNames.size()),
-      OutputFeed(OutputNames.size()) {
+      InputFeed(InputSpecs.size()), Input(InputSpecs.size()),
+      OutputFeed(OutputSpecs.size()) {
   if (!ensureInitTF()) {
     errs() << "Tensorflow should have been initialized";
     return;
@@ -155,25 +156,31 @@ TFModelEvaluatorImpl::TFModelEvaluatorImpl(
     errs() << TF_Message(Status.get());
     invalidate();
   }
-  for (size_t I = 0; I < InputNames.size(); ++I) {
+  for (size_t I = 0; I < InputSpecs.size(); ++I) {
+    auto &InputSpec = InputSpecs[I];
     InputFeed[I] = {
-        TF_GraphOperationByName(Graph.get(), (InputNames[I]).c_str()), 0};
-    if (!checkReportAndInvalidate(InputFeed[I], InputNames[I]))
+        TF_GraphOperationByName(Graph.get(), (InputSpec.name()).c_str()),
+        InputSpec.port()};
+    if (!checkReportAndInvalidate(InputFeed[I], InputSpec))
       return;
+    initInput(I, static_cast<TF_DataType>(InputSpec.typeIndex()),
+              InputSpec.shape());
   }
-  for (size_t I = 0; I < OutputNames.size(); ++I) {
+  for (size_t I = 0; I < OutputSpecs.size(); ++I) {
+    auto &OutputSpec = OutputSpecs[I];
     OutputFeed[I] = {
-        TF_GraphOperationByName(Graph.get(), (OutputNames[I]).c_str()), 0};
-    if (!checkReportAndInvalidate(OutputFeed[I], OutputNames[I]))
+        TF_GraphOperationByName(Graph.get(), (OutputSpec.name()).c_str()),
+        OutputSpec.port()};
+    if (!checkReportAndInvalidate(OutputFeed[I], OutputSpec))
       return;
   }
 }
 
 TFModelEvaluator::TFModelEvaluator(StringRef SavedModelPath,
-                                   const std::vector<std::string> &InputNames,
-                                   const std::vector<std::string> &OutputNames,
+                                   const std::vector<TensorSpec> &InputSpecs,
+                                   const std::vector<TensorSpec> &OutputSpecs,
                                    const char *Tags)
-    : Impl(new TFModelEvaluatorImpl(SavedModelPath, InputNames, OutputNames,
+    : Impl(new TFModelEvaluatorImpl(SavedModelPath, InputSpecs, OutputSpecs,
                                     Tags)) {
   if (!Impl->isValid())
     Impl.reset();
@@ -192,11 +199,11 @@ TFModelEvaluatorImpl::~TFModelEvaluatorImpl() {
     errs() << "Could not delete TF session";
 }
 
-bool TFModelEvaluatorImpl::checkReportAndInvalidate(const TF_Output &Output,
-                                                    StringRef Name) {
+bool TFModelEvaluatorImpl::checkReportAndInvalidate(
+    const TF_Output &Output, const TensorSpec &OutputSpec) {
   if (Output.oper)
     return true;
-  errs() << "Could not find TF_Output named: " + Name;
+  errs() << "Could not find TF_Output named: " + OutputSpec.name();
   IsValid = false;
   return IsValid;
 }
@@ -242,50 +249,25 @@ void *TFModelEvaluator::EvaluationResult::getUntypedTensorValue(size_t Index) {
   return TF_TensorData(Impl->getOutput()[Index]);
 }
 
-void TFModelEvaluator::initInput(size_t Index, int TypeIndex,
-                                 const std::vector<int64_t> &Dimensions) {
-  Impl->initInput(Index, static_cast<TF_DataType>(TypeIndex), Dimensions);
-}
-
-template <> int TFModelEvaluator::getModelTypeIndex<float>() {
-  return TF_FLOAT;
-}
+template <> int TensorSpec::getDataType<float>() { return TF_FLOAT; }
 
-template <> int TFModelEvaluator::getModelTypeIndex<double>() {
-  return TF_DOUBLE;
-}
+template <> int TensorSpec::getDataType<double>() { return TF_DOUBLE; }
 
-template <> int TFModelEvaluator::getModelTypeIndex<int8_t>() {
-  return TF_INT8;
-}
+template <> int TensorSpec::getDataType<int8_t>() { return TF_INT8; }
 
-template <> int TFModelEvaluator::getModelTypeIndex<uint8_t>() {
-  return TF_UINT8;
-}
+template <> int TensorSpec::getDataType<uint8_t>() { return TF_UINT8; }
 
-template <> int TFModelEvaluator::getModelTypeIndex<int16_t>() {
-  return TF_INT16;
-}
+template <> int TensorSpec::getDataType<int16_t>() { return TF_INT16; }
 
-template <> int TFModelEvaluator::getModelTypeIndex<uint16_t>() {
-  return TF_UINT16;
-}
+template <> int TensorSpec::getDataType<uint16_t>() { return TF_UINT16; }
 
-template <> int TFModelEvaluator::getModelTypeIndex<int32_t>() {
-  return TF_INT32;
-}
+template <> int TensorSpec::getDataType<int32_t>() { return TF_INT32; }
 
-template <> int TFModelEvaluator::getModelTypeIndex<uint32_t>() {
-  return TF_UINT32;
-}
+template <> int TensorSpec::getDataType<uint32_t>() { return TF_UINT32; }
 
-template <> int TFModelEvaluator::getModelTypeIndex<int64_t>() {
-  return TF_INT64;
-}
+template <> int TensorSpec::getDataType<int64_t>() { return TF_INT64; }
 
-template <> int TFModelEvaluator::getModelTypeIndex<uint64_t>() {
-  return TF_UINT64;
-}
+template <> int TensorSpec::getDataType<uint64_t>() { return TF_UINT64; }
 
 TFModelEvaluator::EvaluationResult::~EvaluationResult() {}
 TFModelEvaluator::~TFModelEvaluator() {}
diff --git a/llvm/unittests/Analysis/TFUtilsTest.cpp b/llvm/unittests/Analysis/TFUtilsTest.cpp
index 1e54f1885b2ca..e96d34092c7e5 100644
--- a/llvm/unittests/Analysis/TFUtilsTest.cpp
+++ b/llvm/unittests/Analysis/TFUtilsTest.cpp
@@ -37,15 +37,14 @@ TEST(TFUtilsTest, NoModel) {
 TEST(TFUtilsTest, LoadAndExecuteTest) {
   // We use the ir2native model for test. We know it has one feature of
   // dimension (1, 214)
-  std::vector<std::string> InputNames{"serving_default_input_1"};
-  std::vector<std::string> OutputName{"StatefulPartitionedCall"};
   const static int64_t KnownSize = 214;
+  std::vector<TensorSpec> InputSpecs{TensorSpec::createSpec<int32_t>(
+      "serving_default_input_1", {1, KnownSize})};
+  std::vector<TensorSpec> OutputSpecs{
+      TensorSpec::createSpec<float>("StatefulPartitionedCall", {1})};
 
-  TFModelEvaluator Evaluator(getModelPath(), InputNames, OutputName);
-  static const std::vector<int64_t> Dim{1, KnownSize};
-
+  TFModelEvaluator Evaluator(getModelPath(), InputSpecs, OutputSpecs);
   EXPECT_TRUE(Evaluator.isValid());
-  Evaluator.initInput<int32_t>(0, Dim);
 
   int32_t *V = Evaluator.getInput<int32_t>(0);
   // Fill it up with 1's, we know the output.
@@ -77,15 +76,14 @@ TEST(TFUtilsTest, LoadAndExecuteTest) {
 TEST(TFUtilsTest, EvalError) {
   // We use the ir2native model for test. We know it has one feature of
   // dimension (1, 214)
-  std::vector<std::string> InputNames{"serving_default_input_1"};
-  std::vector<std::string> OutputName{"StatefulPartitionedCall"};
   const static int64_t KnownSize = 213;
+  std::vector<TensorSpec> InputSpecs{TensorSpec::createSpec<int32_t>(
+      "serving_default_input_1", {1, KnownSize})};
+  std::vector<TensorSpec> OutputSpecs{
+      TensorSpec::createSpec<float>("StatefulPartitionedCall", {1})};
 
-  TFModelEvaluator Evaluator(getModelPath(), InputNames, OutputName);
-  static const std::vector<int64_t> Dim{1, KnownSize};
-
+  TFModelEvaluator Evaluator(getModelPath(), InputSpecs, OutputSpecs);
   EXPECT_TRUE(Evaluator.isValid());
-  Evaluator.initInput<int32_t>(0, Dim);
 
   int32_t *V = Evaluator.getInput<int32_t>(0);
   // Fill it up with 1's, we know the output.

From 0da582d9b6cc8a96682628ff90b5e0e5d155358c Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Sun, 19 Jul 2020 09:56:15 -0400
Subject: [PATCH 0561/1035] GlobalISel: Handle llvm.roundeven

I still think it's highly questionable that we have two intrinsics
with identical behavior and only vary by the name of the libcall used
if it happens to be lowered that way, but try to reduce the feature
delta between SDAG and GlobalISel for recently added intrinsics. I'm
not sure which opcode should be considered the canonical one, but
lower roundeven back to round.
---
 .../llvm/CodeGen/GlobalISel/LegalizerHelper.h |   2 +
 .../llvm/CodeGen/GlobalISel/LegalizerInfo.h   |   9 +
 llvm/include/llvm/Support/TargetOpcodes.def   |   3 +
 llvm/include/llvm/Target/GenericOpcodes.td    |   6 +
 llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp  |   2 +
 .../CodeGen/GlobalISel/LegalizerHelper.cpp    |  54 +-
 .../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp |   3 +-
 llvm/lib/Target/X86/X86LegalizerInfo.cpp      |   5 +
 .../GlobalISel/legalizer-info-validation.mir  |   3 +
 .../CodeGen/AMDGPU/GlobalISel/roundeven.ll    | 566 ++++++++++++++++++
 llvm/test/CodeGen/X86/GlobalISel/roundeven.ll |  68 +++
 11 files changed, 710 insertions(+), 11 deletions(-)
 create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/roundeven.ll
 create mode 100644 llvm/test/CodeGen/X86/GlobalISel/roundeven.ll

diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
index d5f5ebe2a081e..d925c53a57502 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
@@ -228,6 +228,8 @@ class LegalizerHelper {
                          ArrayRef<Register> Src1Regs,
                          ArrayRef<Register> Src2Regs, LLT NarrowTy);
 
+  void changeOpcode(MachineInstr &MI, unsigned NewOpcode);
+
 public:
   /// Return the alignment to use for a stack temporary object with the given
   /// type.
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h
index 7123aff36cd14..57cc693e1817b 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h
@@ -667,6 +667,15 @@ class LegalizeRuleSet {
                                      Types2);
   }
 
+  /// The instruction is emitted as a library call.
+  LegalizeRuleSet &libcall() {
+    using namespace LegalizeMutations;
+    // We have no choice but conservatively assume that predicate-less lowering
+    // properly handles all type indices by design:
+    markAllIdxsAsCovered();
+    return actionIf(LegalizeAction::Libcall, always);
+  }
+
   /// Like legalIf, but for the Libcall action.
   LegalizeRuleSet &libcallIf(LegalityPredicate Predicate) {
     // We have no choice but conservatively assume that a libcall with a
diff --git a/llvm/include/llvm/Support/TargetOpcodes.def b/llvm/include/llvm/Support/TargetOpcodes.def
index 9470b1d52bbad..572da717bc0b4 100644
--- a/llvm/include/llvm/Support/TargetOpcodes.def
+++ b/llvm/include/llvm/Support/TargetOpcodes.def
@@ -297,6 +297,9 @@ HANDLE_TARGET_OPCODE(G_INTRINSIC_ROUND)
 /// INTRINSIC round to integer intrinsic.
 HANDLE_TARGET_OPCODE(G_INTRINSIC_LRINT)
 
+/// INTRINSIC roundeven intrinsic.
+HANDLE_TARGET_OPCODE(G_INTRINSIC_ROUNDEVEN)
+
 /// INTRINSIC readcyclecounter
 HANDLE_TARGET_OPCODE(G_READCYCLECOUNTER)
 
diff --git a/llvm/include/llvm/Target/GenericOpcodes.td b/llvm/include/llvm/Target/GenericOpcodes.td
index 469ac13cca492..841a314ecc90c 100644
--- a/llvm/include/llvm/Target/GenericOpcodes.td
+++ b/llvm/include/llvm/Target/GenericOpcodes.td
@@ -918,6 +918,12 @@ def G_INTRINSIC_LRINT : GenericInstruction {
   let hasSideEffects = 0;
 }
 
+def G_INTRINSIC_ROUNDEVEN : GenericInstruction {
+  let OutOperandList = (outs type0:$dst);
+  let InOperandList = (ins type0:$src1);
+  let hasSideEffects = 0;
+}
+
 def G_READCYCLECOUNTER : GenericInstruction {
   let OutOperandList = (outs type0:$dst);
   let InOperandList = (ins);
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 6338f276b85da..9adf3d1a89a33 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -1280,6 +1280,8 @@ unsigned IRTranslator::getSimpleIntrinsicOpcode(Intrinsic::ID ID) {
       return TargetOpcode::G_FRINT;
     case Intrinsic::round:
       return TargetOpcode::G_INTRINSIC_ROUND;
+    case Intrinsic::roundeven:
+      return TargetOpcode::G_INTRINSIC_ROUNDEVEN;
     case Intrinsic::sin:
       return TargetOpcode::G_FSIN;
     case Intrinsic::sqrt:
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index 7609adfc7c655..abb983dac6bdc 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -77,6 +77,8 @@ static Type *getFloatTypeForLLT(LLVMContext &Ctx, LLT Ty) {
     return Type::getFloatTy(Ctx);
   case 64:
     return Type::getDoubleTy(Ctx);
+  case 80:
+    return Type::getX86_FP80Ty(Ctx);
   case 128:
     return Type::getFP128Ty(Ctx);
   default:
@@ -386,7 +388,7 @@ void LegalizerHelper::buildWidenedRemergeToDst(Register DstReg, LLT LCMTy,
 }
 
 static RTLIB::Libcall getRTLibDesc(unsigned Opcode, unsigned Size) {
-#define RTLIBCASE(LibcallPrefix)                                               \
+#define RTLIBCASE_INT(LibcallPrefix)                                           \
   do {                                                                         \
     switch (Size) {                                                            \
     case 32:                                                                   \
@@ -400,19 +402,33 @@ static RTLIB::Libcall getRTLibDesc(unsigned Opcode, unsigned Size) {
     }                                                                          \
   } while (0)
 
-  assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size");
+#define RTLIBCASE(LibcallPrefix)                                               \
+  do {                                                                         \
+    switch (Size) {                                                            \
+    case 32:                                                                   \
+      return RTLIB::LibcallPrefix##32;                                         \
+    case 64:                                                                   \
+      return RTLIB::LibcallPrefix##64;                                         \
+    case 80:                                                                   \
+      return RTLIB::LibcallPrefix##80;                                         \
+    case 128:                                                                  \
+      return RTLIB::LibcallPrefix##128;                                        \
+    default:                                                                   \
+      llvm_unreachable("unexpected size");                                     \
+    }                                                                          \
+  } while (0)
 
   switch (Opcode) {
   case TargetOpcode::G_SDIV:
-    RTLIBCASE(SDIV_I);
+    RTLIBCASE_INT(SDIV_I);
   case TargetOpcode::G_UDIV:
-    RTLIBCASE(UDIV_I);
+    RTLIBCASE_INT(UDIV_I);
   case TargetOpcode::G_SREM:
-    RTLIBCASE(SREM_I);
+    RTLIBCASE_INT(SREM_I);
   case TargetOpcode::G_UREM:
-    RTLIBCASE(UREM_I);
+    RTLIBCASE_INT(UREM_I);
   case TargetOpcode::G_CTLZ_ZERO_UNDEF:
-    RTLIBCASE(CTLZ_I);
+    RTLIBCASE_INT(CTLZ_I);
   case TargetOpcode::G_FADD:
     RTLIBCASE(ADD_F);
   case TargetOpcode::G_FSUB:
@@ -455,6 +471,8 @@ static RTLIB::Libcall getRTLibDesc(unsigned Opcode, unsigned Size) {
     RTLIBCASE(RINT_F);
   case TargetOpcode::G_FNEARBYINT:
     RTLIBCASE(NEARBYINT_F);
+  case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
+    RTLIBCASE(ROUNDEVEN_F);
   }
   llvm_unreachable("Unknown libcall function");
 }
@@ -670,10 +688,11 @@ LegalizerHelper::libcall(MachineInstr &MI) {
   case TargetOpcode::G_FMAXNUM:
   case TargetOpcode::G_FSQRT:
   case TargetOpcode::G_FRINT:
-  case TargetOpcode::G_FNEARBYINT: {
+  case TargetOpcode::G_FNEARBYINT:
+  case TargetOpcode::G_INTRINSIC_ROUNDEVEN: {
     Type *HLTy = getFloatTypeForLLT(Ctx, LLTy);
-    if (!HLTy || (Size != 32 && Size != 64 && Size != 128)) {
-      LLVM_DEBUG(dbgs() << "No libcall available for size " << Size << ".\n");
+    if (!HLTy || (Size != 32 && Size != 64 && Size != 80 && Size != 128)) {
+      LLVM_DEBUG(dbgs() << "No libcall available for type " << LLTy << ".\n");
       return UnableToLegalize;
     }
     auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy);
@@ -2163,6 +2182,7 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
   case TargetOpcode::G_FPOW:
   case TargetOpcode::G_INTRINSIC_TRUNC:
   case TargetOpcode::G_INTRINSIC_ROUND:
+  case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
     assert(TypeIdx == 0);
     Observer.changingInstr(MI);
 
@@ -2363,6 +2383,13 @@ LegalizerHelper::bitcast(MachineInstr &MI, unsigned TypeIdx, LLT CastTy) {
   }
 }
 
+// Legalize an instruction by changing the opcode in place.
+void LegalizerHelper::changeOpcode(MachineInstr &MI, unsigned NewOpcode) {
+    Observer.changingInstr(MI);
+    MI.setDesc(MIRBuilder.getTII().get(NewOpcode));
+    Observer.changedInstr(MI);
+}
+
 LegalizerHelper::LegalizeResult
 LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
   using namespace TargetOpcode;
@@ -2461,6 +2488,12 @@ LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
     return lowerFFloor(MI);
   case TargetOpcode::G_INTRINSIC_ROUND:
     return lowerIntrinsicRound(MI);
+  case TargetOpcode::G_INTRINSIC_ROUNDEVEN: {
+    // Since round even is the assumed rounding mode for unconstrained FP
+    // operations, rint and roundeven are the same operation.
+    changeOpcode(MI, TargetOpcode::G_FRINT);
+    return Legalized;
+  }
   case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: {
     Register OldValRes = MI.getOperand(0).getReg();
     Register SuccessRes = MI.getOperand(1).getReg();
@@ -3557,6 +3590,7 @@ LegalizerHelper::fewerElementsVector(MachineInstr &MI, unsigned TypeIdx,
   case G_FFLOOR:
   case G_FRINT:
   case G_INTRINSIC_ROUND:
+  case G_INTRINSIC_ROUNDEVEN:
   case G_INTRINSIC_TRUNC:
   case G_FCOS:
   case G_FSIN:
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index bac4f9ab5c25b..4320151d5758e 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -711,7 +711,8 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
        .scalarize(0)
        .lower();
 
-  getActionDefinitionsBuilder(G_INTRINSIC_ROUND)
+  // Lower roundeven into G_FRINT
+  getActionDefinitionsBuilder({G_INTRINSIC_ROUND, G_INTRINSIC_ROUNDEVEN})
     .scalarize(0)
     .lower();
 
diff --git a/llvm/lib/Target/X86/X86LegalizerInfo.cpp b/llvm/lib/Target/X86/X86LegalizerInfo.cpp
index 84f560f2f9ee7..96c9a8580f8f4 100644
--- a/llvm/lib/Target/X86/X86LegalizerInfo.cpp
+++ b/llvm/lib/Target/X86/X86LegalizerInfo.cpp
@@ -70,6 +70,11 @@ X86LegalizerInfo::X86LegalizerInfo(const X86Subtarget &STI,
   setLegalizerInfoAVX512DQ();
   setLegalizerInfoAVX512BW();
 
+  getActionDefinitionsBuilder(G_INTRINSIC_ROUNDEVEN)
+    .scalarize(0)
+    .minScalar(0, LLT::scalar(32))
+    .libcall();
+
   setLegalizeScalarToDifferentSizeStrategy(G_PHI, 0, widen_1);
   for (unsigned BinOp : {G_SUB, G_MUL, G_AND, G_OR, G_XOR})
     setLegalizeScalarToDifferentSizeStrategy(BinOp, 0, widen_1);
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
index 08e416fd63163..705f7cd1a0589 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
@@ -132,6 +132,9 @@
 # DEBUG-NEXT: G_INTRINSIC_LRINT (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
 # DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
 # DEBUG-NEXT:.. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: G_INTRINSIC_ROUNDEVEN (opcode {{[0-9]+}}): 1 type index, 0 imm indices
+# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
 
 # DEBUG-NEXT: G_READCYCLECOUNTER (opcode {{[0-9]+}}): 1 type index, 0 imm indices
 # DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/roundeven.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/roundeven.ll
new file mode 100644
index 0000000000000..09eda4be61975
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/roundeven.ll
@@ -0,0 +1,566 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel -march=amdgcn -mcpu=tahiti < %s | FileCheck -check-prefix=GFX6 %s
+; RUN: llc -global-isel -march=amdgcn -mcpu=hawaii < %s | FileCheck -check-prefix=GFX7 %s
+; RUN: llc -global-isel -march=amdgcn -mcpu=fiji < %s | FileCheck -check-prefix=GFX8 %s
+; RUN: llc -global-isel -march=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefix=GFX9 %s
+
+define float @v_roundeven_f32(float %x) {
+; GFX6-LABEL: v_roundeven_f32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_rndne_f32_e32 v0, v0
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: v_roundeven_f32:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_rndne_f32_e32 v0, v0
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_roundeven_f32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_rndne_f32_e32 v0, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_roundeven_f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_rndne_f32_e32 v0, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %roundeven = call float @llvm.roundeven.f32(float %x)
+  ret float %roundeven
+}
+
+define <2 x float> @v_roundeven_v2f32(<2 x float> %x) {
+; GFX6-LABEL: v_roundeven_v2f32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_rndne_f32_e32 v0, v0
+; GFX6-NEXT:    v_rndne_f32_e32 v1, v1
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: v_roundeven_v2f32:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_rndne_f32_e32 v0, v0
+; GFX7-NEXT:    v_rndne_f32_e32 v1, v1
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_roundeven_v2f32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_rndne_f32_e32 v0, v0
+; GFX8-NEXT:    v_rndne_f32_e32 v1, v1
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_roundeven_v2f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_rndne_f32_e32 v0, v0
+; GFX9-NEXT:    v_rndne_f32_e32 v1, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %roundeven = call <2 x float> @llvm.roundeven.v2f32(<2 x float> %x)
+  ret <2 x float> %roundeven
+}
+
+define <3 x float> @v_roundeven_v3f32(<3 x float> %x) {
+; GFX6-LABEL: v_roundeven_v3f32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_rndne_f32_e32 v0, v0
+; GFX6-NEXT:    v_rndne_f32_e32 v1, v1
+; GFX6-NEXT:    v_rndne_f32_e32 v2, v2
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: v_roundeven_v3f32:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_rndne_f32_e32 v0, v0
+; GFX7-NEXT:    v_rndne_f32_e32 v1, v1
+; GFX7-NEXT:    v_rndne_f32_e32 v2, v2
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_roundeven_v3f32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_rndne_f32_e32 v0, v0
+; GFX8-NEXT:    v_rndne_f32_e32 v1, v1
+; GFX8-NEXT:    v_rndne_f32_e32 v2, v2
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_roundeven_v3f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_rndne_f32_e32 v0, v0
+; GFX9-NEXT:    v_rndne_f32_e32 v1, v1
+; GFX9-NEXT:    v_rndne_f32_e32 v2, v2
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %roundeven = call <3 x float> @llvm.roundeven.v3f32(<3 x float> %x)
+  ret <3 x float> %roundeven
+}
+
+define <4 x float> @v_roundeven_v4f32(<4 x float> %x) {
+; GFX6-LABEL: v_roundeven_v4f32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_rndne_f32_e32 v0, v0
+; GFX6-NEXT:    v_rndne_f32_e32 v1, v1
+; GFX6-NEXT:    v_rndne_f32_e32 v2, v2
+; GFX6-NEXT:    v_rndne_f32_e32 v3, v3
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: v_roundeven_v4f32:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_rndne_f32_e32 v0, v0
+; GFX7-NEXT:    v_rndne_f32_e32 v1, v1
+; GFX7-NEXT:    v_rndne_f32_e32 v2, v2
+; GFX7-NEXT:    v_rndne_f32_e32 v3, v3
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_roundeven_v4f32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_rndne_f32_e32 v0, v0
+; GFX8-NEXT:    v_rndne_f32_e32 v1, v1
+; GFX8-NEXT:    v_rndne_f32_e32 v2, v2
+; GFX8-NEXT:    v_rndne_f32_e32 v3, v3
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_roundeven_v4f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_rndne_f32_e32 v0, v0
+; GFX9-NEXT:    v_rndne_f32_e32 v1, v1
+; GFX9-NEXT:    v_rndne_f32_e32 v2, v2
+; GFX9-NEXT:    v_rndne_f32_e32 v3, v3
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %roundeven = call <4 x float> @llvm.roundeven.v4f32(<4 x float> %x)
+  ret <4 x float> %roundeven
+}
+
+define half @v_roundeven_f16(half %x) {
+; GFX6-LABEL: v_roundeven_f16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX6-NEXT:    v_rndne_f32_e32 v0, v0
+; GFX6-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: v_roundeven_f16:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX7-NEXT:    v_rndne_f32_e32 v0, v0
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_roundeven_f16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_rndne_f16_e32 v0, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_roundeven_f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_rndne_f16_e32 v0, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %roundeven = call half @llvm.roundeven.f16(half %x)
+  ret half %roundeven
+}
+
+define <2 x half> @v_roundeven_v2f16(<2 x half> %x) {
+; GFX6-LABEL: v_roundeven_v2f16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX6-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GFX6-NEXT:    v_rndne_f32_e32 v0, v0
+; GFX6-NEXT:    v_rndne_f32_e32 v1, v1
+; GFX6-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX6-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: v_roundeven_v2f16:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GFX7-NEXT:    v_rndne_f32_e32 v0, v0
+; GFX7-NEXT:    v_rndne_f32_e32 v1, v1
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_roundeven_v2f16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_rndne_f16_e32 v1, v0
+; GFX8-NEXT:    v_rndne_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX8-NEXT:    v_mov_b32_e32 v2, 16
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_roundeven_v2f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_rndne_f16_e32 v1, v0
+; GFX9-NEXT:    v_rndne_f16_sdwa v0, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0xffff
+; GFX9-NEXT:    v_and_or_b32 v0, v1, v2, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %roundeven = call <2 x half> @llvm.roundeven.v2f16(<2 x half> %x)
+  ret <2 x half> %roundeven
+}
+
+define <2 x half> @v_roundeven_v2f16_fneg(<2 x half> %x) {
+; GFX6-LABEL: v_roundeven_v2f16_fneg:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX6-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX6-NEXT:    v_xor_b32_e32 v0, 0x80008000, v0
+; GFX6-NEXT:    v_cvt_f32_f16_e32 v1, v0
+; GFX6-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX6-NEXT:    v_cvt_f32_f16_e32 v2, v0
+; GFX6-NEXT:    v_rndne_f32_e32 v0, v1
+; GFX6-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX6-NEXT:    v_rndne_f32_e32 v1, v2
+; GFX6-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: v_roundeven_v2f16_fneg:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX7-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX7-NEXT:    v_xor_b32_e32 v0, 0x80008000, v0
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v1, v0
+; GFX7-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v2, v0
+; GFX7-NEXT:    v_rndne_f32_e32 v0, v1
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX7-NEXT:    v_rndne_f32_e32 v1, v2
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_roundeven_v2f16_fneg:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_xor_b32_e32 v0, 0x80008000, v0
+; GFX8-NEXT:    v_rndne_f16_e32 v1, v0
+; GFX8-NEXT:    v_rndne_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX8-NEXT:    v_mov_b32_e32 v2, 16
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_roundeven_v2f16_fneg:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_xor_b32_e32 v0, 0x80008000, v0
+; GFX9-NEXT:    v_rndne_f16_e32 v1, v0
+; GFX9-NEXT:    v_rndne_f16_sdwa v0, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0xffff
+; GFX9-NEXT:    v_and_or_b32 v0, v1, v2, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %x.fneg = fneg <2 x half> %x
+  %roundeven = call <2 x half> @llvm.roundeven.v2f16(<2 x half> %x.fneg)
+  ret <2 x half> %roundeven
+}
+
+define <4 x half> @v_roundeven_v4f16(<4 x half> %x) {
+; GFX6-LABEL: v_roundeven_v4f16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX6-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GFX6-NEXT:    v_cvt_f32_f16_e32 v2, v2
+; GFX6-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GFX6-NEXT:    v_rndne_f32_e32 v0, v0
+; GFX6-NEXT:    v_rndne_f32_e32 v1, v1
+; GFX6-NEXT:    v_rndne_f32_e32 v2, v2
+; GFX6-NEXT:    v_rndne_f32_e32 v3, v3
+; GFX6-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX6-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GFX6-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GFX6-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: v_roundeven_v4f16:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v2, v2
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GFX7-NEXT:    v_rndne_f32_e32 v0, v0
+; GFX7-NEXT:    v_rndne_f32_e32 v1, v1
+; GFX7-NEXT:    v_rndne_f32_e32 v2, v2
+; GFX7-NEXT:    v_rndne_f32_e32 v3, v3
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_roundeven_v4f16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_rndne_f16_e32 v2, v0
+; GFX8-NEXT:    v_rndne_f16_e32 v3, v1
+; GFX8-NEXT:    v_rndne_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX8-NEXT:    v_mov_b32_e32 v4, 16
+; GFX8-NEXT:    v_rndne_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_roundeven_v4f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_rndne_f16_e32 v2, v0
+; GFX9-NEXT:    v_rndne_f16_e32 v3, v1
+; GFX9-NEXT:    v_rndne_f16_sdwa v0, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0xffff
+; GFX9-NEXT:    v_rndne_f16_sdwa v1, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-NEXT:    v_and_or_b32 v0, v2, v4, v0
+; GFX9-NEXT:    v_and_or_b32 v1, v3, v4, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %roundeven = call <4 x half> @llvm.roundeven.v4f16(<4 x half> %x)
+  ret <4 x half> %roundeven
+}
+
+
+define float @v_roundeven_f32_fabs(float %x) {
+; GFX6-LABEL: v_roundeven_f32_fabs:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_rndne_f32_e64 v0, |v0|
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: v_roundeven_f32_fabs:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_rndne_f32_e64 v0, |v0|
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_roundeven_f32_fabs:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_rndne_f32_e64 v0, |v0|
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_roundeven_f32_fabs:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_rndne_f32_e64 v0, |v0|
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %fabs.x = call float @llvm.fabs.f32(float %x)
+  %roundeven = call float @llvm.roundeven.f32(float %fabs.x)
+  ret float %roundeven
+}
+
+define amdgpu_ps float @s_roundeven_f32(float inreg %x) {
+; GFX6-LABEL: s_roundeven_f32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    v_rndne_f32_e32 v0, s0
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX7-LABEL: s_roundeven_f32:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    v_rndne_f32_e32 v0, s0
+; GFX7-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_roundeven_f32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    v_rndne_f32_e32 v0, s0
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_roundeven_f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_rndne_f32_e32 v0, s0
+; GFX9-NEXT:    ; return to shader part epilog
+  %roundeven = call float @llvm.roundeven.f32(float %x)
+  ret float %roundeven
+}
+
+define float @v_roundeven_f32_fneg(float %x) {
+; GFX6-LABEL: v_roundeven_f32_fneg:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_rndne_f32_e64 v0, -v0
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: v_roundeven_f32_fneg:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_rndne_f32_e64 v0, -v0
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_roundeven_f32_fneg:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_rndne_f32_e64 v0, -v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_roundeven_f32_fneg:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_rndne_f32_e64 v0, -v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %neg.x = fneg float %x
+  %roundeven = call float @llvm.roundeven.f32(float %neg.x)
+  ret float %roundeven
+}
+
+define double @v_roundeven_f64(double %x) {
+; GFX6-LABEL: v_roundeven_f64:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_and_b32_e32 v3, 0x80000000, v1
+; GFX6-NEXT:    v_mov_b32_e32 v2, 0
+; GFX6-NEXT:    v_or_b32_e32 v3, 0x43300000, v3
+; GFX6-NEXT:    v_add_f64 v[4:5], v[0:1], v[2:3]
+; GFX6-NEXT:    s_mov_b32 s4, -1
+; GFX6-NEXT:    s_mov_b32 s5, 0x432fffff
+; GFX6-NEXT:    v_add_f64 v[2:3], v[4:5], -v[2:3]
+; GFX6-NEXT:    v_cmp_gt_f64_e64 vcc, |v[0:1]|, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: v_roundeven_f64:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_rndne_f64_e32 v[0:1], v[0:1]
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_roundeven_f64:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_rndne_f64_e32 v[0:1], v[0:1]
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_roundeven_f64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_rndne_f64_e32 v[0:1], v[0:1]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %roundeven = call double @llvm.roundeven.f64(double %x)
+  ret double %roundeven
+}
+
+define double @v_roundeven_f64_fneg(double %x) {
+; GFX6-LABEL: v_roundeven_f64_fneg:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_xor_b32_e32 v2, 0x80000000, v1
+; GFX6-NEXT:    v_and_b32_e32 v4, 0x80000000, v2
+; GFX6-NEXT:    v_mov_b32_e32 v3, 0
+; GFX6-NEXT:    v_or_b32_e32 v4, 0x43300000, v4
+; GFX6-NEXT:    v_add_f64 v[5:6], -v[0:1], v[3:4]
+; GFX6-NEXT:    v_mov_b32_e32 v1, v0
+; GFX6-NEXT:    s_mov_b32 s4, -1
+; GFX6-NEXT:    s_mov_b32 s5, 0x432fffff
+; GFX6-NEXT:    v_add_f64 v[3:4], v[5:6], -v[3:4]
+; GFX6-NEXT:    v_cmp_gt_f64_e64 vcc, |v[1:2]|, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v4, v2, vcc
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: v_roundeven_f64_fneg:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_rndne_f64_e64 v[0:1], -v[0:1]
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_roundeven_f64_fneg:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_rndne_f64_e64 v[0:1], -v[0:1]
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_roundeven_f64_fneg:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_rndne_f64_e64 v[0:1], -v[0:1]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %neg.x = fneg double %x
+  %roundeven = call double @llvm.roundeven.f64(double %neg.x)
+  ret double %roundeven
+}
+
+define <2 x double> @v_roundeven_v2f64(<2 x double> %x) {
+; GFX6-LABEL: v_roundeven_v2f64:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    s_brev_b32 s6, 1
+; GFX6-NEXT:    s_mov_b32 s7, 0x43300000
+; GFX6-NEXT:    v_and_b32_e32 v5, s6, v1
+; GFX6-NEXT:    v_mov_b32_e32 v4, 0
+; GFX6-NEXT:    v_or_b32_e32 v5, s7, v5
+; GFX6-NEXT:    v_add_f64 v[6:7], v[0:1], v[4:5]
+; GFX6-NEXT:    s_mov_b32 s4, -1
+; GFX6-NEXT:    s_mov_b32 s5, 0x432fffff
+; GFX6-NEXT:    v_add_f64 v[5:6], v[6:7], -v[4:5]
+; GFX6-NEXT:    v_cmp_gt_f64_e64 vcc, |v[0:1]|, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
+; GFX6-NEXT:    v_and_b32_e32 v5, s6, v3
+; GFX6-NEXT:    v_or_b32_e32 v5, s7, v5
+; GFX6-NEXT:    v_add_f64 v[7:8], v[2:3], v[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
+; GFX6-NEXT:    v_add_f64 v[4:5], v[7:8], -v[4:5]
+; GFX6-NEXT:    v_cmp_gt_f64_e64 vcc, |v[2:3]|, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v3, v5, v3, vcc
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: v_roundeven_v2f64:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_rndne_f64_e32 v[0:1], v[0:1]
+; GFX7-NEXT:    v_rndne_f64_e32 v[2:3], v[2:3]
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_roundeven_v2f64:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_rndne_f64_e32 v[0:1], v[0:1]
+; GFX8-NEXT:    v_rndne_f64_e32 v[2:3], v[2:3]
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_roundeven_v2f64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_rndne_f64_e32 v[0:1], v[0:1]
+; GFX9-NEXT:    v_rndne_f64_e32 v[2:3], v[2:3]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %roundeven = call <2 x double> @llvm.roundeven.v2f64(<2 x double> %x)
+  ret <2 x double> %roundeven
+}
+
+declare half @llvm.roundeven.f16(half) #0
+declare <2 x half> @llvm.roundeven.v2f16(<2 x half>) #0
+declare <4 x half> @llvm.roundeven.v4f16(<4 x half>) #0
+
+declare float @llvm.roundeven.f32(float) #0
+declare <2 x float> @llvm.roundeven.v2f32(<2 x float>) #0
+declare <3 x float> @llvm.roundeven.v3f32(<3 x float>) #0
+declare <4 x float> @llvm.roundeven.v4f32(<4 x float>) #0
+
+declare double @llvm.roundeven.f64(double) #0
+declare <2 x double> @llvm.roundeven.v2f64(<2 x double>) #0
+
+declare half @llvm.fabs.f16(half) #0
+declare float @llvm.fabs.f32(float) #0
+
+attributes #0 = { nounwind readnone speculatable willreturn }
diff --git a/llvm/test/CodeGen/X86/GlobalISel/roundeven.ll b/llvm/test/CodeGen/X86/GlobalISel/roundeven.ll
new file mode 100644
index 0000000000000..119821e91b38c
--- /dev/null
+++ b/llvm/test/CodeGen/X86/GlobalISel/roundeven.ll
@@ -0,0 +1,68 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel -mtriple=x86_64-linux-gnu < %s | FileCheck %s
+
+; FIXME: Calling convention lowering fails
+; define half @roundeven_f16(half %x) {
+;   %roundeven = call half @llvm.roundeven.f16(half %x)
+;   ret half %roundeven
+; }
+
+define float @roundeven_f32(float %x) {
+; CHECK-LABEL: roundeven_f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    callq roundevenf
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
+  %roundeven = call float @llvm.roundeven.f32(float %x)
+  ret float %roundeven
+}
+
+define double @roundeven_f64(double %x) {
+; CHECK-LABEL: roundeven_f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    callq roundeven
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
+  %roundeven = call double @llvm.roundeven.f64(double %x)
+  ret double %roundeven
+}
+
+; FIXME: Insert fails
+; define x86_fp80 @roundeven_fp80(x86_fp80 %x) {
+;   %roundeven = call x86_fp80 @llvm.roundeven.f80(x86_fp80 %x)
+;   ret x86_fp80 %roundeven
+; }
+
+define fp128 @roundeven_f128(fp128 %x) {
+; CHECK-LABEL: roundeven_f128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    callq roundevenl
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
+  %roundeven = call fp128 @llvm.roundeven.f128(fp128 %x)
+  ret fp128 %roundeven
+}
+
+; FIXME: Fails on build_vector
+; define <4 x float> @roundeven_v4f32(<4 x float> %x) {
+;   %roundeven = call <4 x float> @llvm.roundeven.v4f32(<4 x float> %x)
+;   ret <4 x float> %roundeven
+; }
+
+declare half @llvm.roundeven.f16(half) #0
+declare float @llvm.roundeven.f32(float) #0
+declare <4 x float> @llvm.roundeven.v4f32(<4 x float>) #0
+declare double @llvm.roundeven.f64(double) #0
+declare x86_fp80 @llvm.roundeven.f80(x86_fp80) #0
+declare fp128 @llvm.roundeven.f128(fp128) #0
+
+attributes #0 = { nounwind readnone speculatable willreturn }

From 44e7d449973948ddaa5f7c84a56dd79f8bb82e43 Mon Sep 17 00:00:00 2001
From: Julian Lettner <julian.lettner@apple.com>
Date: Wed, 29 Jul 2020 12:50:49 -0700
Subject: [PATCH 0562/1035] [Darwin][compiler-rt] kern.osproductversion
 unavailable pre macOS 10.13

Add a fallback for `sysctl kern.osproductversion` for XNU 17 (macOS
10.13) and below, which do not provide this property.

Unfortunately, this means we have to take the detour via Darwin kernel
version again (at least for the fallback).

Reviewed By: delcypher

Differential Revision: https://reviews.llvm.org/D84892
---
 .../lib/sanitizer_common/sanitizer_mac.cpp    | 21 ++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp
index 5b06fee62c13f..a10ba774b9552 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp
@@ -606,6 +606,14 @@ HandleSignalMode GetHandleSignalMode(int signum) {
   return result;
 }
 
+// Offset example:
+// XNU 17 -- macOS 10.13 -- iOS 11 -- tvOS 11 -- watchOS 4
+constexpr u16 GetOSMajorKernelOffset() {
+  if (TARGET_OS_OSX) return 4;
+  if (SANITIZER_IOS || SANITIZER_TVOS) return 6;
+  if (SANITIZER_WATCHOS) return 13;
+}
+
 using VersStr = char[64];
 
 static void GetOSVersion(VersStr vers) {
@@ -621,7 +629,18 @@ static void GetOSVersion(VersStr vers) {
   } else {
     int res =
         internal_sysctlbyname("kern.osproductversion", vers, &len, nullptr, 0);
-    CHECK_EQ(res, 0);
+    if (res) {
+      // Fallback for XNU 17 (macOS 10.13) and below that do not provide the
+      // `kern.osproductversion` property.
+      u16 kernel_major = GetDarwinKernelVersion().major;
+      u16 offset = GetOSMajorKernelOffset();
+      CHECK_LE(kernel_major, 17);
+      CHECK_GE(kernel_major, offset);
+      u16 os_major = kernel_major - offset;
+
+      auto format = TARGET_OS_OSX ? "10.%d" : "%d.0";
+      len = internal_snprintf(vers, len, format, os_major);
+    }
   }
   CHECK_LT(len, sizeof(VersStr));
 }

From 618a0c0d3bd364dbab26af6f3a7398d0e9165c9c Mon Sep 17 00:00:00 2001
From: Vedant Kumar <vsk@apple.com>
Date: Wed, 29 Jul 2020 17:22:27 -0700
Subject: [PATCH 0563/1035] [profile] Add InstrProfilingInternal.c.o to Darwin
 kext builtins

Fixes a build failure in the Darwin kernel. Tested with:

% nm -m ./lib/libclang_rt.cc_kext_x86_64_osx.a | grep lprofSetProfileDumped

rdar://66249602
---
 compiler-rt/cmake/Modules/CompilerRTDarwinUtils.cmake  | 3 ++-
 compiler-rt/lib/profile/InstrProfiling.c               | 4 +++-
 compiler-rt/lib/profile/InstrProfilingBuffer.c         | 3 +++
 compiler-rt/lib/profile/InstrProfilingInternal.c       | 3 +++
 compiler-rt/lib/profile/InstrProfilingPlatformDarwin.c | 3 +++
 compiler-rt/lib/profile/InstrProfilingWriter.c         | 3 +++
 6 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/compiler-rt/cmake/Modules/CompilerRTDarwinUtils.cmake b/compiler-rt/cmake/Modules/CompilerRTDarwinUtils.cmake
index c37b5e98bcc68..a31bcc3963e17 100644
--- a/compiler-rt/cmake/Modules/CompilerRTDarwinUtils.cmake
+++ b/compiler-rt/cmake/Modules/CompilerRTDarwinUtils.cmake
@@ -398,7 +398,8 @@ macro(darwin_add_builtin_libraries)
   set(PROFILE_SOURCES ../profile/InstrProfiling 
                       ../profile/InstrProfilingBuffer
                       ../profile/InstrProfilingPlatformDarwin
-                      ../profile/InstrProfilingWriter)
+                      ../profile/InstrProfilingWriter
+                      ../profile/InstrProfilingInternal)
   foreach (os ${ARGN})
     list_intersect(DARWIN_BUILTIN_ARCHS DARWIN_${os}_BUILTIN_ARCHS BUILTIN_SUPPORTED_ARCH)
     foreach (arch ${DARWIN_BUILTIN_ARCHS})
diff --git a/compiler-rt/lib/profile/InstrProfiling.c b/compiler-rt/lib/profile/InstrProfiling.c
index 92ad25f62cd1b..6df65f66df732 100644
--- a/compiler-rt/lib/profile/InstrProfiling.c
+++ b/compiler-rt/lib/profile/InstrProfiling.c
@@ -6,6 +6,9 @@
 |*
 \*===----------------------------------------------------------------------===*/
 
+// Note: This is linked into the Darwin kernel, and must remain compatible
+// with freestanding compilation. See `darwin_add_builtin_libraries`.
+
 #include <limits.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -17,7 +20,6 @@
 #define INSTR_PROF_VALUE_PROF_DATA
 #include "profile/InstrProfData.inc"
 
-
 COMPILER_RT_VISIBILITY uint64_t __llvm_profile_get_magic(void) {
   return sizeof(void *) == sizeof(uint64_t) ? (INSTR_PROF_RAW_MAGIC_64)
                                             : (INSTR_PROF_RAW_MAGIC_32);
diff --git a/compiler-rt/lib/profile/InstrProfilingBuffer.c b/compiler-rt/lib/profile/InstrProfilingBuffer.c
index 5ee44785a7ab9..800ae22f2746c 100644
--- a/compiler-rt/lib/profile/InstrProfilingBuffer.c
+++ b/compiler-rt/lib/profile/InstrProfilingBuffer.c
@@ -6,6 +6,9 @@
 |*
 \*===----------------------------------------------------------------------===*/
 
+// Note: This is linked into the Darwin kernel, and must remain compatible
+// with freestanding compilation. See `darwin_add_builtin_libraries`.
+
 #include "InstrProfiling.h"
 #include "InstrProfilingInternal.h"
 #include "InstrProfilingPort.h"
diff --git a/compiler-rt/lib/profile/InstrProfilingInternal.c b/compiler-rt/lib/profile/InstrProfilingInternal.c
index d58bc19ad11e6..6a54697df7f01 100644
--- a/compiler-rt/lib/profile/InstrProfilingInternal.c
+++ b/compiler-rt/lib/profile/InstrProfilingInternal.c
@@ -6,6 +6,9 @@
 |*
 \*===----------------------------------------------------------------------===*/
 
+// Note: This is linked into the Darwin kernel, and must remain compatible
+// with freestanding compilation. See `darwin_add_builtin_libraries`.
+
 #if !defined(__Fuchsia__)
 
 #include "InstrProfilingInternal.h"
diff --git a/compiler-rt/lib/profile/InstrProfilingPlatformDarwin.c b/compiler-rt/lib/profile/InstrProfilingPlatformDarwin.c
index 23bdb7f37179d..29541c74d5a63 100644
--- a/compiler-rt/lib/profile/InstrProfilingPlatformDarwin.c
+++ b/compiler-rt/lib/profile/InstrProfilingPlatformDarwin.c
@@ -6,6 +6,9 @@
 |*
 \*===----------------------------------------------------------------------===*/
 
+// Note: This is linked into the Darwin kernel, and must remain compatible
+// with freestanding compilation. See `darwin_add_builtin_libraries`.
+
 #include "InstrProfiling.h"
 
 #if defined(__APPLE__)
diff --git a/compiler-rt/lib/profile/InstrProfilingWriter.c b/compiler-rt/lib/profile/InstrProfilingWriter.c
index c34e110a69590..16ad965ff6083 100644
--- a/compiler-rt/lib/profile/InstrProfilingWriter.c
+++ b/compiler-rt/lib/profile/InstrProfilingWriter.c
@@ -6,6 +6,9 @@
 |*
 \*===----------------------------------------------------------------------===*/
 
+// Note: This is linked into the Darwin kernel, and must remain compatible
+// with freestanding compilation. See `darwin_add_builtin_libraries`.
+
 #ifdef _MSC_VER
 /* For _alloca */
 #include <malloc.h>

From 111a02decd4f4ed1cf3995dd0010a6238b542914 Mon Sep 17 00:00:00 2001
From: Juneyoung Lee <aqjune@gmail.com>
Date: Thu, 30 Jul 2020 09:38:50 +0900
Subject: [PATCH 0564/1035] [JumpThreading] Fold br(freeze(undef))

This patch makes JumpThreading fold br(freeze(undef)) if the freeze instruction
is only used by the branch.

Reviewed By: efriedma

Differential Revision: https://reviews.llvm.org/D84818
---
 llvm/lib/Transforms/Scalar/JumpThreading.cpp | 10 +++++++---
 llvm/test/Transforms/JumpThreading/freeze.ll | 14 ++++----------
 2 files changed, 11 insertions(+), 13 deletions(-)

diff --git a/llvm/lib/Transforms/Scalar/JumpThreading.cpp b/llvm/lib/Transforms/Scalar/JumpThreading.cpp
index 7399c7ab609e5..37a588a582997 100644
--- a/llvm/lib/Transforms/Scalar/JumpThreading.cpp
+++ b/llvm/lib/Transforms/Scalar/JumpThreading.cpp
@@ -1057,9 +1057,11 @@ bool JumpThreadingPass::ProcessBlock(BasicBlock *BB) {
     }
   }
 
-  // If the terminator is branching on an undef, we can pick any of the
-  // successors to branch to.  Let GetBestDestForJumpOnUndef decide.
-  if (isa<UndefValue>(Condition)) {
+  // If the terminator is branching on an undef or freeze undef, we can pick any
+  // of the successors to branch to.  Let GetBestDestForJumpOnUndef decide.
+  auto *FI = dyn_cast<FreezeInst>(Condition);
+  if (isa<UndefValue>(Condition) ||
+      (FI && isa<UndefValue>(FI->getOperand(0)) && FI->hasOneUse())) {
     unsigned BestSucc = GetBestDestForJumpOnUndef(BB);
     std::vector<DominatorTree::UpdateType> Updates;
 
@@ -1078,6 +1080,8 @@ bool JumpThreadingPass::ProcessBlock(BasicBlock *BB) {
     BranchInst::Create(BBTerm->getSuccessor(BestSucc), BBTerm);
     BBTerm->eraseFromParent();
     DTU->applyUpdatesPermissive(Updates);
+    if (FI)
+      FI->eraseFromParent();
     return true;
   }
 
diff --git a/llvm/test/Transforms/JumpThreading/freeze.ll b/llvm/test/Transforms/JumpThreading/freeze.ll
index 8f8f3f5566568..99df46f38a8ec 100644
--- a/llvm/test/Transforms/JumpThreading/freeze.ll
+++ b/llvm/test/Transforms/JumpThreading/freeze.ll
@@ -128,19 +128,13 @@ F2:
 
 define i32 @test1_undef(i1 %cond) {
 ; CHECK-LABEL: @test1_undef(
-; CHECK-NEXT:    br i1 [[COND:%.*]], label [[MERGE_THREAD:%.*]], label [[MERGE:%.*]]
-; CHECK:       Merge.thread:
-; CHECK-NEXT:    [[V1:%.*]] = call i32 @f1()
-; CHECK-NEXT:    br label [[T2:%.*]]
-; CHECK:       Merge:
-; CHECK-NEXT:    [[V2:%.*]] = call i32 @f2()
-; CHECK-NEXT:    [[A_FR:%.*]] = freeze i1 undef
-; CHECK-NEXT:    br i1 [[A_FR]], label [[T2]], label [[F2:%.*]]
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[T2:%.*]], label [[F2:%.*]]
 ; CHECK:       T2:
-; CHECK-NEXT:    [[B4:%.*]] = phi i32 [ [[V1]], [[MERGE_THREAD]] ], [ [[V2]], [[MERGE]] ]
+; CHECK-NEXT:    [[V1:%.*]] = call i32 @f1()
 ; CHECK-NEXT:    call void @f3()
-; CHECK-NEXT:    ret i32 [[B4]]
+; CHECK-NEXT:    ret i32 [[V1]]
 ; CHECK:       F2:
+; CHECK-NEXT:    [[V2:%.*]] = call i32 @f2()
 ; CHECK-NEXT:    ret i32 [[V2]]
 ;
   br i1 %cond, label %T1, label %F1

From a64883431369f28f3fac311c496a4dfad480058f Mon Sep 17 00:00:00 2001
From: Richard Smith <richard@metafoo.co.uk>
Date: Wed, 29 Jul 2020 16:49:05 -0700
Subject: [PATCH 0565/1035] PR46729: Reject explicit and partial
 specializations with C linkage.

---
 clang/lib/Sema/SemaDecl.cpp                   | 32 +++++---
 clang/lib/Sema/SemaTemplate.cpp               | 15 +++-
 clang/test/CXX/temp/temp.pre/p6.cpp           | 79 +++++++++++++++++++
 .../test/SemaTemplate/class-template-decl.cpp | 15 +++-
 4 files changed, 126 insertions(+), 15 deletions(-)
 create mode 100644 clang/test/CXX/temp/temp.pre/p6.cpp

diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index 9b9d164dcbc1b..531c2801bf929 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -6994,19 +6994,18 @@ NamedDecl *Sema::ActOnVariableDeclarator(
                          TemplateParams->getRAngleLoc());
         TemplateParams = nullptr;
       } else {
+        // Check that we can declare a template here.
+        if (CheckTemplateDeclScope(S, TemplateParams))
+          return nullptr;
+
         if (D.getName().getKind() == UnqualifiedIdKind::IK_TemplateId) {
           // This is an explicit specialization or a partial specialization.
-          // FIXME: Check that we can declare a specialization here.
           IsVariableTemplateSpecialization = true;
           IsPartialSpecialization = TemplateParams->size() > 0;
         } else { // if (TemplateParams->size() > 0)
           // This is a template declaration.
           IsVariableTemplate = true;
 
-          // Check that we can declare a template here.
-          if (CheckTemplateDeclScope(S, TemplateParams))
-            return nullptr;
-
           // Only C++1y supports variable templates (N3651).
           Diag(D.getIdentifierLoc(),
                getLangOpts().CPlusPlus14
@@ -7015,6 +7014,10 @@ NamedDecl *Sema::ActOnVariableDeclarator(
         }
       }
     } else {
+      // Check that we can declare a member specialization here.
+      if (!TemplateParamLists.empty() && IsMemberSpecialization &&
+          CheckTemplateDeclScope(S, TemplateParamLists.back()))
+        return nullptr;
       assert((Invalid ||
               D.getName().getKind() != UnqualifiedIdKind::IK_TemplateId) &&
              "should have a 'template<>' for this decl");
@@ -8941,13 +8944,13 @@ Sema::ActOnFunctionDeclarator(Scope *S, Declarator &D, DeclContext *DC,
             TemplateParamLists, isFriend, isMemberSpecialization,
             Invalid);
     if (TemplateParams) {
+      // Check that we can declare a template here.
+      if (CheckTemplateDeclScope(S, TemplateParams))
+        NewFD->setInvalidDecl();
+
       if (TemplateParams->size() > 0) {
         // This is a function template
 
-        // Check that we can declare a template here.
-        if (CheckTemplateDeclScope(S, TemplateParams))
-          NewFD->setInvalidDecl();
-
         // A destructor cannot be a template.
         if (Name.getNameKind() == DeclarationName::CXXDestructorName) {
           Diag(NewFD->getLocation(), diag::err_destructor_template);
@@ -9006,6 +9009,11 @@ Sema::ActOnFunctionDeclarator(Scope *S, Declarator &D, DeclContext *DC,
         }
       }
     } else {
+      // Check that we can declare a template here.
+      if (!TemplateParamLists.empty() && isMemberSpecialization &&
+          CheckTemplateDeclScope(S, TemplateParamLists.back()))
+        NewFD->setInvalidDecl();
+
       // All template param lists were matched against the scope specifier:
       // this is NOT (an explicit specialization of) a template.
       if (TemplateParamLists.size() > 0)
@@ -15301,6 +15309,10 @@ Decl *Sema::ActOnTag(Scope *S, unsigned TagSpec, TagUseKind TUK,
         isMemberSpecialization = true;
       }
     }
+
+    if (!TemplateParameterLists.empty() && isMemberSpecialization &&
+        CheckTemplateDeclScope(S, TemplateParameterLists.back()))
+      return nullptr;
   }
 
   // Figure out the underlying type if this a enum declaration. We need to do
@@ -17300,7 +17312,7 @@ void Sema::ActOnFields(Scope *S, SourceLocation RecLoc, Decl *EnclosingDecl,
       CXXRecordDecl *CXXRecord = cast<CXXRecordDecl>(Record);
       CheckForZeroSize =
           CXXRecord->getLexicalDeclContext()->isExternCContext() &&
-          !CXXRecord->isDependentType() &&
+          !CXXRecord->isDependentType() && !inTemplateInstantiation() &&
           CXXRecord->isCLike();
     }
     if (CheckForZeroSize) {
diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp
index 76ba12303d84b..3991f2b479778 100644
--- a/clang/lib/Sema/SemaTemplate.cpp
+++ b/clang/lib/Sema/SemaTemplate.cpp
@@ -7771,8 +7771,9 @@ Sema::CheckTemplateDeclScope(Scope *S, TemplateParameterList *TemplateParams) {
          (S->getFlags() & Scope::TemplateParamScope) != 0)
     S = S->getParent();
 
-  // C++ [temp]p4:
-  //   A template [...] shall not have C linkage.
+  // C++ [temp.pre]p6: [P2096]
+  //   A template, explicit specialization, or partial specialization shall not
+  //   have C linkage.
   DeclContext *Ctx = S->getEntity();
   if (Ctx && Ctx->isExternCContext()) {
     Diag(TemplateParams->getTemplateLoc(), diag::err_template_linkage)
@@ -7786,6 +7787,12 @@ Sema::CheckTemplateDeclScope(Scope *S, TemplateParameterList *TemplateParams) {
   // C++ [temp]p2:
   //   A template-declaration can appear only as a namespace scope or
   //   class scope declaration.
+  // C++ [temp.expl.spec]p3:
+  //   An explicit specialization may be declared in any scope in which the
+  //   corresponding primary template may be defined.
+  // C++ [temp.class.spec]p6: [P2096]
+  //   A partial specialization may be declared in any scope in which the
+  //   corresponding primary template may be defined.
   if (Ctx) {
     if (Ctx->isFileContext())
       return false;
@@ -8105,6 +8112,10 @@ DeclResult Sema::ActOnClassTemplateSpecialization(
   if (Invalid)
     return true;
 
+  // Check that we can declare a template specialization here.
+  if (TemplateParams && CheckTemplateDeclScope(S, TemplateParams))
+    return true;
+
   if (TemplateParams && TemplateParams->size() > 0) {
     isPartialSpecialization = true;
 
diff --git a/clang/test/CXX/temp/temp.pre/p6.cpp b/clang/test/CXX/temp/temp.pre/p6.cpp
new file mode 100644
index 0000000000000..cb8c70ca3abed
--- /dev/null
+++ b/clang/test/CXX/temp/temp.pre/p6.cpp
@@ -0,0 +1,79 @@
+// RUN: %clang_cc1 -std=c++20 -verify %s
+
+// Templates and partial and explicit specializations can't have C linkage.
+namespace extern_c_templates {
+
+template<typename T> struct A {
+  static int a;
+  struct b;
+  void c();
+  enum class d;
+
+  template<typename U> static int e;
+  template<typename U> struct f;
+  template<typename U> void g();
+};
+
+template<typename T> int B;
+template<typename T> void C();
+
+extern "C" { // expected-note 1+{{begins here}}
+  // templates
+  template<typename T> struct A; // expected-error {{templates must have C++ linkage}}
+  template<typename T> int B; // expected-error {{templates must have C++ linkage}}
+  template<typename T> void C(); // expected-error {{templates must have C++ linkage}}
+
+  // non-template members of a template
+  // FIXME: Should these really be valid?
+  template<typename T> int A<T>::a;
+  template<typename T> struct A<T>::b {};
+  template<typename T> void A<T>::c() {}
+  template<typename T> enum class A<T>::d {};
+
+  // templates
+  template<typename T> template<typename U> int A<T>::e; // expected-error {{templates must have C++ linkage}}
+  template<typename T> template<typename U> struct A<T>::f {}; // expected-error {{templates must have C++ linkage}}
+  template<typename T> template<typename U> void A<T>::g() {} // expected-error {{templates must have C++ linkage}}
+
+  // partial specializations
+  template<typename T> struct A<int*>; // expected-error {{templates must have C++ linkage}}
+  template<typename T> int B<int*>; // expected-error {{templates must have C++ linkage}}
+  template<typename T> template<typename U> int A<T>::e<U*>; // expected-error {{templates must have C++ linkage}}
+  template<typename T> template<typename U> struct A<T>::f<U*> {}; // expected-error {{templates must have C++ linkage}}
+
+  // explicit specializations of templates
+  template<> struct A<char> {}; // expected-error {{templates must have C++ linkage}}
+  template<> int B<char>; // expected-error {{templates must have C++ linkage}}
+  template<> void C<char>() {} // expected-error {{templates must have C++ linkage}}
+
+  // explicit specializations of members of a template
+  template<> int A<int>::a; // expected-error {{templates must have C++ linkage}}
+  template<> struct A<int>::b {}; // expected-error {{templates must have C++ linkage}}
+  template<> void A<int>::c() {} // expected-error {{templates must have C++ linkage}}
+  template<> enum class A<int>::d {}; // expected-error {{templates must have C++ linkage}}
+
+  // explicit specializations of member templates
+  template<> template<typename U> int A<int>::e; // expected-error {{templates must have C++ linkage}}
+  template<> template<typename U> struct A<int>::f {}; // expected-error {{templates must have C++ linkage}}
+  template<> template<typename U> void A<int>::g() {} // expected-error {{templates must have C++ linkage}}
+}
+
+// Provide valid definitions for the explicit instantiations below.
+// FIXME: Our recovery from the invalid definitions above isn't very good.
+template<typename T> template<typename U> int A<T>::e;
+template<typename T> template<typename U> struct A<T>::f {};
+template<typename T> template<typename U> void A<T>::g() {}
+
+extern "C" {
+  // explicit instantiations
+  // FIXME: Should these really be valid?
+  template struct A<double>;
+  template int A<float>::a;
+  template struct A<float>::b;
+  template void A<float>::c();
+  template int A<float>::e<float>;
+  template struct A<float>::f<float>;
+  template void A<float>::g<float>();
+}
+
+}
diff --git a/clang/test/SemaTemplate/class-template-decl.cpp b/clang/test/SemaTemplate/class-template-decl.cpp
index dd9dcd2de9f4c..453218ac3b40e 100644
--- a/clang/test/SemaTemplate/class-template-decl.cpp
+++ b/clang/test/SemaTemplate/class-template-decl.cpp
@@ -1,17 +1,26 @@
 // RUN: %clang_cc1 -std=c++14 -fsyntax-only -verify %s
 
-template<typename T> class A;
+template<typename T> class A {};
 
 extern "C++" {
-  template<typename T> class B;
+  template<typename T> class B {};
+  template<typename T> class A<T *>;
+  template<> class A<int[1]>;
+  template class A<int[2]>;
+  template<typename T> class B<T *>;
+  template<> class B<int[1]>;
+  template class B<int[2]>;
 }
 
 namespace N {
   template<typename T> class C;
 }
 
-extern "C" { // expected-note {{extern "C" language linkage specification begins here}}
+extern "C" { // expected-note 3 {{extern "C" language linkage specification begins here}}
   template<typename T> class D; // expected-error{{templates must have C++ linkage}}
+  template<typename T> class A<T **>; // expected-error{{templates must have C++ linkage}}
+  template<> class A<int[3]>; // expected-error{{templates must have C++ linkage}}
+  template class A<int[4]>; // OK (surprisingly) FIXME: Should we warn on this?
 }
 
 extern "C" { // expected-note 2 {{extern "C" language linkage specification begins here}}

From 9a3dbc972322413045bb5672b0fd3ba8c216c987 Mon Sep 17 00:00:00 2001
From: Jonas Devlieghere <jonas@devlieghere.com>
Date: Wed, 29 Jul 2020 17:56:38 -0700
Subject: [PATCH 0566/1035] [lldb] Move finish_swig logic into a function in
 the binding dir

Move the finish_swig logic into a function in the bindings directory. By
making this a function I can reuse the logic internally where we ship
two Python versions and therefore need to finish the bindings twice.
---
 lldb/CMakeLists.txt          | 168 +++--------------------------------
 lldb/bindings/CMakeLists.txt | 168 +++++++++++++++++++++++++++++++++++
 2 files changed, 178 insertions(+), 158 deletions(-)

diff --git a/lldb/CMakeLists.txt b/lldb/CMakeLists.txt
index 14cfba1f64ff5..df289747cf3a7 100644
--- a/lldb/CMakeLists.txt
+++ b/lldb/CMakeLists.txt
@@ -85,6 +85,16 @@ add_subdirectory(source)
 add_subdirectory(tools)
 add_subdirectory(docs)
 
+if (LLDB_ENABLE_PYTHON OR LLDB_ENABLE_LUA)
+  if(LLDB_BUILD_FRAMEWORK)
+    set(lldb_python_target_dir "${LLDB_FRAMEWORK_ABSOLUTE_BUILD_DIR}/LLDB.framework/Resources/Python/lldb")
+  else()
+    set(lldb_python_target_dir "${CMAKE_BINARY_DIR}/${CMAKE_CFG_INTDIR}/${LLDB_PYTHON_RELATIVE_PATH}/lldb")
+  endif()
+  get_target_property(lldb_bindings_dir swig_wrapper BINARY_DIR)
+  finish_swig("finish_swig" "${lldb_bindings_dir}" "${lldb_python_target_dir}")
+endif()
+
 option(LLDB_INCLUDE_TESTS "Generate build targets for the LLDB unit tests." ${LLVM_INCLUDE_TESTS})
 if(LLDB_INCLUDE_TESTS)
   add_subdirectory(test)
@@ -92,164 +102,6 @@ if(LLDB_INCLUDE_TESTS)
   add_subdirectory(utils)
 endif()
 
-if (LLDB_ENABLE_PYTHON)
-  get_target_property(lldb_bindings_dir swig_wrapper BINARY_DIR)
-
-  if(LLDB_BUILD_FRAMEWORK)
-    set(lldb_python_build_path "${LLDB_FRAMEWORK_ABSOLUTE_BUILD_DIR}/LLDB.framework/Resources/Python/lldb")
-  else()
-    set(lldb_python_build_path "${CMAKE_BINARY_DIR}/${CMAKE_CFG_INTDIR}/${LLDB_PYTHON_RELATIVE_PATH}/lldb")
-  endif()
-
-  # Add a Post-Build Event to copy over Python files and create the symlink
-  # to liblldb.so for the Python API(hardlink on Windows).
-  add_custom_target(finish_swig ALL VERBATIM
-    COMMAND ${CMAKE_COMMAND} -E make_directory ${lldb_python_build_path}
-    DEPENDS ${lldb_bindings_dir}/lldb.py
-    COMMENT "Python script sym-linking LLDB Python API")
-
-  if(NOT LLDB_USE_SYSTEM_SIX)
-    add_custom_command(TARGET finish_swig POST_BUILD VERBATIM
-      COMMAND ${CMAKE_COMMAND} -E copy
-        "${LLDB_SOURCE_DIR}/third_party/Python/module/six/six.py"
-        "${lldb_python_build_path}/../six.py")
-  endif()
-
-  add_custom_command(TARGET finish_swig POST_BUILD VERBATIM
-    COMMAND ${CMAKE_COMMAND} -E copy
-      "${lldb_bindings_dir}/lldb.py"
-      "${lldb_python_build_path}/__init__.py")
-
-  function(create_python_package pkg_dir)
-    cmake_parse_arguments(ARG "NOINIT" "" "FILES" ${ARGN})
-    if(ARG_FILES)
-      set(copy_cmd COMMAND ${CMAKE_COMMAND} -E copy ${ARG_FILES} ${pkg_dir})
-    endif()
-    if(NOT ARG_NOINIT)
-      set(init_cmd COMMAND ${PYTHON_EXECUTABLE}
-          "${LLDB_SOURCE_DIR}/bindings/python/createPythonInit.py"
-          "${pkg_dir}" ${ARG_FILES})
-    endif()
-    add_custom_command(TARGET finish_swig POST_BUILD VERBATIM
-      COMMAND ${CMAKE_COMMAND} -E make_directory ${pkg_dir}
-      ${copy_cmd}
-      ${init_cmd}
-      WORKING_DIRECTORY ${lldb_python_build_path})
-  endfunction()
-
-  add_custom_command(TARGET finish_swig POST_BUILD VERBATIM
-    COMMAND ${CMAKE_COMMAND} -E copy
-      "${LLDB_SOURCE_DIR}/source/Interpreter/embedded_interpreter.py" ${lldb_python_build_path})
-
-  # Distribute the examples as python packages.
-  create_python_package("formatters/cpp"
-    FILES "${LLDB_SOURCE_DIR}/examples/synthetic/gnu_libstdcpp.py"
-          "${LLDB_SOURCE_DIR}/examples/synthetic/libcxx.py")
-
-  create_python_package("formatters"
-    FILES "${LLDB_SOURCE_DIR}/examples/summaries/cocoa/cache.py"
-          "${LLDB_SOURCE_DIR}/examples/summaries/synth.py"
-          "${LLDB_SOURCE_DIR}/examples/summaries/cocoa/metrics.py"
-          "${LLDB_SOURCE_DIR}/examples/summaries/cocoa/attrib_fromdict.py"
-          "${LLDB_SOURCE_DIR}/examples/summaries/cocoa/Logger.py")
-
-  create_python_package("utils"
-    FILES "${LLDB_SOURCE_DIR}/examples/python/in_call_stack.py"
-          "${LLDB_SOURCE_DIR}/examples/python/symbolication.py")
-
-  if(APPLE)
-    create_python_package("macosx"
-      FILES "${LLDB_SOURCE_DIR}/examples/python/crashlog.py"
-            "${LLDB_SOURCE_DIR}/examples/darwin/heap_find/heap.py")
-
-    create_python_package("macosx/heap"
-      FILES "${LLDB_SOURCE_DIR}/examples/darwin/heap_find/heap/heap_find.cpp"
-            "${LLDB_SOURCE_DIR}/examples/darwin/heap_find/heap/Makefile"
-            NOINIT)
-
-    create_python_package("diagnose"
-      FILES "${LLDB_SOURCE_DIR}/examples/python/diagnose_unwind.py"
-            "${LLDB_SOURCE_DIR}/examples/python/diagnose_nsstring.py")
-  endif()
-
-  function(create_relative_symlink target dest_file output_dir output_name)
-    get_filename_component(dest_file ${dest_file} ABSOLUTE)
-    get_filename_component(output_dir ${output_dir} ABSOLUTE)
-    file(RELATIVE_PATH rel_dest_file ${output_dir} ${dest_file})
-    if(CMAKE_HOST_UNIX)
-      set(LLVM_LINK_OR_COPY create_symlink)
-    else()
-      set(LLVM_LINK_OR_COPY copy)
-    endif()
-    add_custom_command(TARGET ${target} POST_BUILD VERBATIM
-      COMMAND ${CMAKE_COMMAND} -E ${LLVM_LINK_OR_COPY} ${rel_dest_file} ${output_name}
-      WORKING_DIRECTORY ${output_dir})
-  endfunction()
-
-  if(LLDB_BUILD_FRAMEWORK)
-    set(LIBLLDB_SYMLINK_DEST "${LLDB_FRAMEWORK_ABSOLUTE_BUILD_DIR}/LLDB.framework/LLDB")
-  else()
-    set(LIBLLDB_SYMLINK_DEST "${LLVM_SHLIB_OUTPUT_INTDIR}/liblldb${CMAKE_SHARED_LIBRARY_SUFFIX}")
-  endif()
-  if(WIN32)
-    if(CMAKE_BUILD_TYPE STREQUAL Debug)
-      set(LIBLLDB_SYMLINK_OUTPUT_FILE "_lldb_d.pyd")
-    else()
-      set(LIBLLDB_SYMLINK_OUTPUT_FILE "_lldb.pyd")
-    endif()
-  else()
-    set(LIBLLDB_SYMLINK_OUTPUT_FILE "_lldb.so")
-  endif()
-  create_relative_symlink(finish_swig ${LIBLLDB_SYMLINK_DEST}
-                          ${lldb_python_build_path} ${LIBLLDB_SYMLINK_OUTPUT_FILE})
-
-  if(NOT LLDB_BUILD_FRAMEWORK)
-    set(LLDB_ARGDUMPER_FILENAME "lldb-argdumper${CMAKE_EXECUTABLE_SUFFIX}")
-    create_relative_symlink(finish_swig "${LLVM_RUNTIME_OUTPUT_INTDIR}/${LLDB_ARGDUMPER_FILENAME}"
-                            ${lldb_python_build_path} ${LLDB_ARGDUMPER_FILENAME})
-  endif()
-
-  add_dependencies(finish_swig swig_wrapper liblldb lldb-argdumper)
-  set_target_properties(finish_swig swig_wrapper PROPERTIES FOLDER "lldb misc")
-
-  # Ensure we do the python post-build step when building lldb.
-  add_dependencies(lldb finish_swig)
-
-  # Install the LLDB python module
-  if(LLDB_BUILD_FRAMEWORK)
-    set(LLDB_PYTHON_INSTALL_PATH ${LLDB_FRAMEWORK_INSTALL_DIR}/LLDB.framework/Resources/Python)
-  else()
-    set(LLDB_PYTHON_INSTALL_PATH ${LLDB_PYTHON_RELATIVE_PATH})
-  endif()
-  if (NOT CMAKE_CFG_INTDIR STREQUAL  ".")
-    string(REPLACE ${CMAKE_CFG_INTDIR} "\$\{CMAKE_INSTALL_CONFIG_NAME\}" LLDB_PYTHON_INSTALL_PATH ${LLDB_PYTHON_INSTALL_PATH})
-    string(REPLACE ${CMAKE_CFG_INTDIR} "\$\{CMAKE_INSTALL_CONFIG_NAME\}" lldb_python_build_path ${lldb_python_build_path})
-  endif()
-  add_custom_target(lldb-python-scripts)
-  add_dependencies(lldb-python-scripts finish_swig)
-  install(DIRECTORY ${lldb_python_build_path}/../
-          DESTINATION ${LLDB_PYTHON_INSTALL_PATH}
-          COMPONENT lldb-python-scripts)
-  if (NOT LLVM_ENABLE_IDE)
-    add_llvm_install_targets(install-lldb-python-scripts
-                             COMPONENT lldb-python-scripts
-                             DEPENDS lldb-python-scripts)
-  endif()
-
-  # Add a Post-Build Event to copy the custom Python DLL to the lldb binaries dir so that Windows can find it when launching
-  # lldb.exe or any other executables that were linked with liblldb.
-  if (WIN32 AND NOT "${PYTHON_DLL}" STREQUAL "")
-    # When using the Visual Studio CMake generator the lldb binaries end up in Release/bin, Debug/bin etc.
-    file(TO_NATIVE_PATH "${CMAKE_BINARY_DIR}/${CMAKE_CFG_INTDIR}/bin" LLDB_BIN_DIR)
-    file(TO_NATIVE_PATH "${PYTHON_DLL}" PYTHON_DLL_NATIVE_PATH)
-    add_custom_command(
-      TARGET finish_swig
-      POST_BUILD
-      COMMAND ${CMAKE_COMMAND} -E copy ${PYTHON_DLL_NATIVE_PATH} ${LLDB_BIN_DIR} VERBATIM
-      COMMENT "Copying Python DLL to LLDB binaries directory.")
-  endif ()
-endif ()
-
 if(LLDB_BUILT_STANDALONE AND NOT LLVM_ENABLE_IDE)
   llvm_distribution_add_targets()
 endif()
diff --git a/lldb/bindings/CMakeLists.txt b/lldb/bindings/CMakeLists.txt
index a2e51c263f7e6..3440f605bca5f 100644
--- a/lldb/bindings/CMakeLists.txt
+++ b/lldb/bindings/CMakeLists.txt
@@ -75,3 +75,171 @@ if (LLDB_ENABLE_LUA)
     ${CMAKE_CURRENT_BINARY_DIR}/LLDBWrapLua.cpp
   )
 endif()
+
+function(create_python_package swig_target working_dir pkg_dir)
+  cmake_parse_arguments(ARG "NOINIT" "" "FILES" ${ARGN})
+  if(ARG_FILES)
+    set(copy_cmd COMMAND ${CMAKE_COMMAND} -E copy ${ARG_FILES} ${pkg_dir})
+  endif()
+  if(NOT ARG_NOINIT)
+    set(init_cmd COMMAND ${PYTHON_EXECUTABLE}
+        "${LLDB_SOURCE_DIR}/bindings/python/createPythonInit.py"
+        "${pkg_dir}" ${ARG_FILES})
+  endif()
+  add_custom_command(TARGET ${swig_target} POST_BUILD VERBATIM
+    COMMAND ${CMAKE_COMMAND} -E make_directory ${pkg_dir}
+    ${copy_cmd}
+    ${init_cmd}
+    WORKING_DIRECTORY ${working_dir})
+endfunction()
+
+function(create_relative_symlink swig_target dest_file output_dir output_name)
+  get_filename_component(dest_file ${dest_file} ABSOLUTE)
+  get_filename_component(output_dir ${output_dir} ABSOLUTE)
+  file(RELATIVE_PATH rel_dest_file ${output_dir} ${dest_file})
+  if(CMAKE_HOST_UNIX)
+    set(LLVM_LINK_OR_COPY create_symlink)
+  else()
+    set(LLVM_LINK_OR_COPY copy)
+  endif()
+  add_custom_command(TARGET ${swig_target} POST_BUILD VERBATIM
+    COMMAND ${CMAKE_COMMAND} -E ${LLVM_LINK_OR_COPY} ${rel_dest_file} ${output_name}
+    WORKING_DIRECTORY ${output_dir})
+endfunction()
+
+function(finish_swig swig_target lldb_bindings_dir lldb_python_target_dir)
+  # Add a Post-Build Event to copy over Python files and create the symlink to
+  # liblldb.so for the Python API(hardlink on Windows).
+  add_custom_target(${swig_target} ALL VERBATIM
+    COMMAND ${CMAKE_COMMAND} -E make_directory ${lldb_python_target_dir}
+    DEPENDS ${lldb_bindings_dir}/lldb.py
+    COMMENT "Python script sym-linking LLDB Python API")
+
+  if(NOT LLDB_USE_SYSTEM_SIX)
+    add_custom_command(TARGET ${swig_target} POST_BUILD VERBATIM
+      COMMAND ${CMAKE_COMMAND} -E copy
+        "${LLDB_SOURCE_DIR}/third_party/Python/module/six/six.py"
+        "${lldb_python_target_dir}/../six.py")
+  endif()
+
+  add_custom_command(TARGET ${swig_target} POST_BUILD VERBATIM
+    COMMAND ${CMAKE_COMMAND} -E copy
+      "${lldb_bindings_dir}/lldb.py"
+      "${lldb_python_target_dir}/__init__.py")
+
+  add_custom_command(TARGET ${swig_target} POST_BUILD VERBATIM
+    COMMAND ${CMAKE_COMMAND} -E copy
+      "${LLDB_SOURCE_DIR}/source/Interpreter/embedded_interpreter.py"
+      "${lldb_python_target_dir}")
+
+  # Distribute the examples as python packages.
+  create_python_package(
+    ${swig_target}
+    ${lldb_python_target_dir}
+    "formatters/cpp"
+    FILES "${LLDB_SOURCE_DIR}/examples/synthetic/gnu_libstdcpp.py"
+          "${LLDB_SOURCE_DIR}/examples/synthetic/libcxx.py")
+
+  create_python_package(
+    ${swig_target}
+    ${lldb_python_target_dir}
+    "formatters"
+    FILES "${LLDB_SOURCE_DIR}/examples/summaries/cocoa/cache.py"
+          "${LLDB_SOURCE_DIR}/examples/summaries/synth.py"
+          "${LLDB_SOURCE_DIR}/examples/summaries/cocoa/metrics.py"
+          "${LLDB_SOURCE_DIR}/examples/summaries/cocoa/attrib_fromdict.py"
+          "${LLDB_SOURCE_DIR}/examples/summaries/cocoa/Logger.py")
+
+  create_python_package(
+    ${swig_target}
+    ${lldb_python_target_dir}
+    "utils"
+    FILES "${LLDB_SOURCE_DIR}/examples/python/in_call_stack.py"
+          "${LLDB_SOURCE_DIR}/examples/python/symbolication.py")
+
+  if(APPLE)
+    create_python_package(
+      ${swig_target}
+      ${lldb_python_target_dir} "macosx"
+      FILES "${LLDB_SOURCE_DIR}/examples/python/crashlog.py"
+            "${LLDB_SOURCE_DIR}/examples/darwin/heap_find/heap.py")
+
+    create_python_package(
+      ${swig_target}
+      ${lldb_python_target_dir} "macosx/heap"
+      FILES "${LLDB_SOURCE_DIR}/examples/darwin/heap_find/heap/heap_find.cpp"
+            "${LLDB_SOURCE_DIR}/examples/darwin/heap_find/heap/Makefile"
+            NOINIT)
+
+    create_python_package(
+      ${swig_target}
+      ${lldb_python_target_dir} "diagnose"
+      FILES "${LLDB_SOURCE_DIR}/examples/python/diagnose_unwind.py"
+            "${LLDB_SOURCE_DIR}/examples/python/diagnose_nsstring.py")
+  endif()
+
+  if(LLDB_BUILD_FRAMEWORK)
+    set(LIBLLDB_SYMLINK_DEST "${LLDB_FRAMEWORK_ABSOLUTE_BUILD_DIR}/LLDB.framework/LLDB")
+  else()
+    set(LIBLLDB_SYMLINK_DEST "${LLVM_SHLIB_OUTPUT_INTDIR}/liblldb${CMAKE_SHARED_LIBRARY_SUFFIX}")
+  endif()
+  if(WIN32)
+    if(CMAKE_BUILD_TYPE STREQUAL Debug)
+      set(LIBLLDB_SYMLINK_OUTPUT_FILE "_lldb_d.pyd")
+    else()
+      set(LIBLLDB_SYMLINK_OUTPUT_FILE "_lldb.pyd")
+    endif()
+  else()
+    set(LIBLLDB_SYMLINK_OUTPUT_FILE "_lldb.so")
+  endif()
+  create_relative_symlink(${swig_target} ${LIBLLDB_SYMLINK_DEST}
+                          ${lldb_python_target_dir} ${LIBLLDB_SYMLINK_OUTPUT_FILE})
+
+  if(NOT LLDB_BUILD_FRAMEWORK)
+    set(LLDB_ARGDUMPER_FILENAME "lldb-argdumper${CMAKE_EXECUTABLE_SUFFIX}")
+    create_relative_symlink(${swig_target} "${LLVM_RUNTIME_OUTPUT_INTDIR}/${LLDB_ARGDUMPER_FILENAME}"
+                            ${lldb_python_target_dir} ${LLDB_ARGDUMPER_FILENAME})
+  endif()
+
+  add_dependencies(${swig_target} swig_wrapper liblldb lldb-argdumper)
+  set_target_properties(${swig_target} swig_wrapper PROPERTIES FOLDER "lldb misc")
+
+  # Ensure we do the python post-build step when building lldb.
+  add_dependencies(lldb ${swig_target})
+
+  # Install the LLDB python module
+  if(LLDB_BUILD_FRAMEWORK)
+    set(LLDB_PYTHON_INSTALL_PATH ${LLDB_FRAMEWORK_INSTALL_DIR}/LLDB.framework/Resources/Python)
+  else()
+    set(LLDB_PYTHON_INSTALL_PATH ${LLDB_PYTHON_RELATIVE_PATH})
+  endif()
+  if (NOT CMAKE_CFG_INTDIR STREQUAL  ".")
+    string(REPLACE ${CMAKE_CFG_INTDIR} "\$\{CMAKE_INSTALL_CONFIG_NAME\}" LLDB_PYTHON_INSTALL_PATH ${LLDB_PYTHON_INSTALL_PATH})
+    string(REPLACE ${CMAKE_CFG_INTDIR} "\$\{CMAKE_INSTALL_CONFIG_NAME\}" lldb_python_target_dir ${lldb_python_target_dir})
+  endif()
+  set(swig_scripts_target "${swig_target}_scripts")
+  set(swig_scripts_install_target "${swig_target}_scripts_install")
+  add_custom_target(${swig_scripts_target})
+  add_dependencies(${swig_scripts_target} ${swig_target})
+  install(DIRECTORY ${lldb_python_target_dir}/../
+          DESTINATION ${LLDB_PYTHON_INSTALL_PATH}
+          COMPONENT ${swig_scripts_target})
+  if (NOT LLVM_ENABLE_IDE)
+    add_llvm_install_targets(${swig_scripts_install_target}
+                             COMPONENT ${swig_scripts_target}
+                             DEPENDS ${swig_scripts_target})
+  endif()
+
+  # Add a Post-Build Event to copy the custom Python DLL to the lldb binaries dir so that Windows can find it when launching
+  # lldb.exe or any other executables that were linked with liblldb.
+  if (WIN32 AND NOT "${PYTHON_DLL}" STREQUAL "")
+    # When using the Visual Studio CMake generator the lldb binaries end up in Release/bin, Debug/bin etc.
+    file(TO_NATIVE_PATH "${CMAKE_BINARY_DIR}/${CMAKE_CFG_INTDIR}/bin" LLDB_BIN_DIR)
+    file(TO_NATIVE_PATH "${PYTHON_DLL}" PYTHON_DLL_NATIVE_PATH)
+    add_custom_command(
+      TARGET ${swig_target}
+      POST_BUILD
+      COMMAND ${CMAKE_COMMAND} -E copy ${PYTHON_DLL_NATIVE_PATH} ${LLDB_BIN_DIR} VERBATIM
+      COMMENT "Copying Python DLL to LLDB binaries directory.")
+  endif()
+endfunction()

From 66c572af5504b1eff32d221291041081af835256 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Wed, 29 Jul 2020 08:55:02 -0400
Subject: [PATCH 0567/1035] GlobalISel: Handle assorted no-op intrinsics

SelectionDAGBuilder just drops these, so do the same.
---
 llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp  | 10 +++
 .../irtranslator-no-op-intrinsics.ll          | 79 +++++++++++++++++++
 2 files changed, 89 insertions(+)
 create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-no-op-intrinsics.ll

diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 9adf3d1a89a33..f9b0843178196 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -1645,6 +1645,16 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
   }
   case Intrinsic::invariant_end:
     return true;
+  case Intrinsic::expect:
+  case Intrinsic::annotation:
+  case Intrinsic::ptr_annotation:
+  case Intrinsic::launder_invariant_group:
+  case Intrinsic::strip_invariant_group: {
+    // Drop the intrinsic, but forward the value.
+    MIRBuilder.buildCopy(getOrCreateVReg(CI),
+                         getOrCreateVReg(*CI.getArgOperand(0)));
+    return true;
+  }
   case Intrinsic::assume:
   case Intrinsic::var_annotation:
   case Intrinsic::sideeffect:
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-no-op-intrinsics.ll b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-no-op-intrinsics.ll
new file mode 100644
index 0000000000000..dc243b74375d1
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-no-op-intrinsics.ll
@@ -0,0 +1,79 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -global-isel -O0 -mtriple=aarch64-- -stop-after=irtranslator -verify-machineinstrs -o - %s | FileCheck %s
+
+define i64 @expect_i64(i64 %arg0) {
+  ; CHECK-LABEL: name: expect_i64
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $x0
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s64) = COPY $x0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s64) = COPY [[COPY]](s64)
+  ; CHECK:   $x0 = COPY [[COPY1]](s64)
+  ; CHECK:   RET_ReallyLR implicit $x0
+  %expval = call i64 @llvm.expect.i64(i64 %arg0, i64 1)
+  ret i64 %expval
+}
+
+define i8* @ptr_annotate(i8* %arg0, i8* %arg1, i8* %arg2, i32 %arg3) {
+  ; CHECK-LABEL: name: ptr_annotate
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $w3, $x0, $x1, $x2
+  ; CHECK:   [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(p0) = COPY $x1
+  ; CHECK:   [[COPY2:%[0-9]+]]:_(p0) = COPY $x2
+  ; CHECK:   [[COPY3:%[0-9]+]]:_(s32) = COPY $w3
+  ; CHECK:   [[COPY4:%[0-9]+]]:_(p0) = COPY [[COPY]](p0)
+  ; CHECK:   $x0 = COPY [[COPY4]](p0)
+  ; CHECK:   RET_ReallyLR implicit $x0
+  %call = call i8* @llvm.ptr.annotation.p0i8(i8* %arg0, i8* %arg1, i8* %arg2, i32 %arg3)
+  ret i8* %call
+}
+
+@.str = private unnamed_addr constant [4 x i8] c"sth\00", section "llvm.metadata"
+@.str1 = private unnamed_addr constant [4 x i8] c"t.c\00", section "llvm.metadata"
+
+define i32 @annotation(i32 %a) {
+  ; CHECK-LABEL: name: annotation
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $w0
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $w0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   $w0 = COPY [[COPY1]](s32)
+  ; CHECK:   RET_ReallyLR implicit $w0
+  %call = call i32 @llvm.annotation.i32(i32 %a, i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i32 0, i32 0), i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str1, i32 0, i32 0), i32 2)
+  ret i32 %call
+}
+
+define i8* @launder_invariant_group(i8* %p) {
+  ; CHECK-LABEL: name: launder_invariant_group
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $x0
+  ; CHECK:   [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(p0) = COPY [[COPY]](p0)
+  ; CHECK:   $x0 = COPY [[COPY1]](p0)
+  ; CHECK:   RET_ReallyLR implicit $x0
+  %q = call i8* @llvm.launder.invariant.group.p0i8(i8* %p)
+  ret i8* %q
+}
+
+define i8* @strip_invariant_group(i8* %p) {
+  ; CHECK-LABEL: name: strip_invariant_group
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $x0
+  ; CHECK:   [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(p0) = COPY [[COPY]](p0)
+  ; CHECK:   $x0 = COPY [[COPY1]](p0)
+  ; CHECK:   RET_ReallyLR implicit $x0
+  %q = call i8* @llvm.strip.invariant.group.p0i8(i8* %p)
+  ret i8* %q
+}
+
+declare i64 @llvm.expect.i64(i64, i64) #0
+declare i8* @llvm.ptr.annotation.p0i8(i8*, i8*, i8*, i32) #1
+declare i32 @llvm.annotation.i32(i32, i8*, i8*, i32) #1
+declare i8* @llvm.launder.invariant.group.p0i8(i8*) #2
+declare i8* @llvm.strip.invariant.group.p0i8(i8*) #3
+
+attributes #0 = { nounwind readnone willreturn }
+attributes #1 = { nounwind willreturn }
+attributes #2 = { inaccessiblememonly nounwind speculatable willreturn }
+attributes #3 = { nounwind readnone speculatable willreturn }

From 7d0b32c268ab4cfbc0630b2836890f2aef128d8b Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Wed, 29 Jul 2020 09:42:03 -0400
Subject: [PATCH 0568/1035] GlobalISel: Use result of find rather than
 rechecking map

---
 llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index f9b0843178196..264af2b562b3f 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -222,8 +222,9 @@ ArrayRef<Register> IRTranslator::getOrCreateVRegs(const Value &Val) {
 }
 
 int IRTranslator::getOrCreateFrameIndex(const AllocaInst &AI) {
-  if (FrameIndices.find(&AI) != FrameIndices.end())
-    return FrameIndices[&AI];
+  auto MapEntry = FrameIndices.find(&AI);
+  if (MapEntry != FrameIndices.end())
+    return MapEntry->second;
 
   uint64_t ElementSize = DL->getTypeAllocSize(AI.getAllocatedType());
   uint64_t Size =

From 5aa2fd88cfb745659fb36d700326b21fcd148afd Mon Sep 17 00:00:00 2001
From: Tony <Tony.Tye@amd.com>
Date: Fri, 3 Jul 2020 22:31:53 +0000
Subject: [PATCH 0569/1035] [AMDGPU] DWARF proposal changes for expression
 context

- Clarify what context is used in DWARF expression evaluation.

- Define location descriptions to fully resolve the context and so
  include the context in their result.

- As a consequence of location descriptions being fully resoved,
  change address spaces so only a swizzled and unswizzled private
  address space is defined. The lane is now part of the location
  description context.

- Clarify how call frame information is used to fully resolve
  expressions that specify registers.

Reviewed By: scott.linder

Differential Revision: https://reviews.llvm.org/D70523
---
 ...DwarfProposalForHeterogeneousDebugging.rst | 1437 +++++++++++------
 llvm/docs/AMDGPUUsage.rst                     |   10 -
 2 files changed, 930 insertions(+), 517 deletions(-)

diff --git a/llvm/docs/AMDGPUDwarfProposalForHeterogeneousDebugging.rst b/llvm/docs/AMDGPUDwarfProposalForHeterogeneousDebugging.rst
index a2c48f97717a4..8253519f7dfe9 100644
--- a/llvm/docs/AMDGPUDwarfProposalForHeterogeneousDebugging.rst
+++ b/llvm/docs/AMDGPUDwarfProposalForHeterogeneousDebugging.rst
@@ -103,7 +103,7 @@ compiler [:ref:`GCC <amdgpu-dwarf-GCC>`] and the Perforce TotalView HPC debugger
 [:ref:`Perforce-TotalView <amdgpu-dwarf-Perforce-TotalView>`].
 
 However, the proposal is intended to be vendor and architecture neutral. It is
-believed to apply to other heterogeous hardware devices including GPUs, DSPs,
+believed to apply to other heterogenous hardware devices including GPUs, DSPs,
 FPGAs, and other specialized hardware. These collectively include similar
 characteristics and requirements as AMDGPU devices. Parts of the proposal can
 also apply to traditional CPU hardware that supports large vector registers.
@@ -162,11 +162,11 @@ required. See ``DW_AT_LLVM_vector_size``.
 
 If the source language is mapped onto the AMDGPU wavefronts in a SIMT manner,
 then the variable DWARF location expressions must compute the location for a
-single lane of the wavefront. Therefore, a DWARF operation is required to
-denote the current lane, much like ``DW_OP_push_object_address`` denotes the
-current object. The ``DW_OP_*piece`` operations only allow literal indices.
-Therefore, a way to use a computed offset of an arbitrary location description
-(such as a vector register) is required. See ``DW_OP_LLVM_push_lane``,
+single lane of the wavefront. Therefore, a DWARF operation is required to denote
+the current lane, much like ``DW_OP_push_object_address`` denotes the current
+object. The ``DW_OP_*piece`` operations only allow literal indices. Therefore, a
+way to use a computed offset of an arbitrary location description (such as a
+vector register) is required. See ``DW_OP_LLVM_push_lane``,
 ``DW_OP_LLVM_offset``, ``DW_OP_LLVM_offset_uconst``, and
 ``DW_OP_LLVM_bit_offset``.
 
@@ -250,12 +250,12 @@ describe when bytes are in registers, are implicit, or a composite of these, the
 define that integer overflow causes wrap-around. The offset operations can
 operate on location storage of any size. For example, implicit location storage
 could be any number of bits in size. It is simpler to define offsets that exceed
-the size of the location storage as being ill-formed, than having to force an
-implementation to support potentially infinite precision offsets to allow it to
-correctly track a series of positive and negative offsets that may transiently
-overflow or underflow, but end up in range. This is simple for the arithmetic
-operations as they are defined in terms of two's compliment arithmetic on a base
-type of a fixed size.
+the size of the location storage as being an evaluation error, than having to
+force an implementation to support potentially infinite precision offsets to
+allow it to correctly track a series of positive and negative offsets that may
+transiently overflow or underflow, but end up in range. This is simple for the
+arithmetic operations as they are defined in terms of two's compliment
+arithmetic on a base type of a fixed size.
 
 Having the offset operations allows ``DW_OP_push_object_address`` to push a
 location description that may be in a register, or be an implicit value, and the
@@ -276,7 +276,7 @@ larger than the generic type. See ``DW_OP_LLVM_form_aspace_address``.
 If the ``DW_OP_LLVM_form_aspace_address`` operation had to produce a value
 that can be implicitly converted to a memory location description, then it
 would be limited to the size of the generic type which matches the size of the
-default address space. Its value would be unspecified and likely not match any
+default address space. Its value would be undefined and likely not match any
 value in the actual program. By making the result a location description, it
 allows a consumer great freedom in how it implements it. The implicit
 conversion back to a value can be limited only to the default address space to
@@ -317,7 +317,7 @@ consumer on how to implement them. They allow the address space to encode lane
 information so they can be used to read memory with only the memory
 description and no extra arguments. The same set of operations can operate on
 locations independent of their kind of storage. The ``DW_OP_deref*`` therefore
-can be used on any storage kind. ``DW_OP_xderef*`` is unnecessary except to
+can be used on any storage kind. ``DW_OP_xderef*`` is unnecessary, except to
 become a more compact way to convert a non-default address space address
 followed by dereferencing it.
 
@@ -462,10 +462,10 @@ DWARF Expressions
 
 .. note::
 
-  This section, and its nested sections, replaces DWARF Version 5 section 2.5 and
-  section 2.6. The new proposed DWARF expression operations are defined as well
-  as clarifying the extensions to already existing DWARF Version 5 operations. It is
-  based on the text of the existing DWARF Version 5 standard.
+  This section, and its nested sections, replaces DWARF Version 5 section 2.5
+  and section 2.6. The new proposed DWARF expression operations are defined as
+  well as clarifying the extensions to already existing DWARF Version 5
+  operations. It is based on the text of the existing DWARF Version 5 standard.
 
 DWARF expressions describe how to compute a value or specify a location.
 
@@ -473,155 +473,215 @@ DWARF expressions describe how to compute a value or specify a location.
 value of an array bound, the length of a dynamic string, the desired value
 itself, and so on.*
 
-The evaluation of a DWARF expression can either result in a value or a location
-description:
+If the evaluation of a DWARF expression does not encounter an error, then it can
+either result in a value (see :ref:`amdgpu-dwarf-expression-value`) or a
+location description (see :ref:`amdgpu-dwarf-location-description`). When a
+DWARF expression is evaluated, it may be specified whether a value or location
+description is required as the result kind.
 
-*value*
+If a result kind is specified, and the result of the evaluation does not match
+the specified result kind, then the implicit conversions described in
+:ref:`amdgpu-dwarf-memory-location-description-operations` are performed if
+valid. Otherwise, the DWARF expression is ill-formed.
 
-  A value has a type and a literal value. It can represent a literal value of
-  any supported base type of the target architecture. The base type specifies
-  the size and encoding of the literal value.
+If the evaluation of a DWARF expression encounters an evaluation error, then the
+result is an evaluation error.
 
-  .. note::
+.. note::
 
-    It may be desirable to add an implicit pointer base type encoding. It would
-    be used for the type of the value that is produced when the ``DW_OP_deref*``
-    operation retrieves the full contents of an implicit pointer location
-    storage created by the ``DW_OP_implicit_pointer`` or
-    ``DW_OP_LLVM_aspace_implicit_pointer`` operations. The literal value would
-    record the debugging information entry and byte dispacement specified by the
-    associated ``DW_OP_implicit_pointer`` or
-    ``DW_OP_LLVM_aspace_implicit_pointer`` operations.
+  Decided to define the concept of an evaluation error. An alternative is to
+  introduce an undefined value base type in a similar way to location
+  descriptions having an undefined location description. Then operations that
+  encounter an evaluation error can return the undefined location description or
+  value with an undefined base type.
 
-  Instead of a base type, a value can have a distinguished generic type, which
-  is an integral type that has the size of an address in the target architecture
-  default address space and unspecified signedness.
+  All operations that act on values would return an undefined entity if given an
+  undefined value. The expression would then always evaluate to completion, and
+  can be tested to determine if it is an undefined entity.
 
-  *The generic type is the same as the unspecified type used for stack
-  operations defined in DWARF Version 4 and before.*
+  However, this would add considerable additional complexity and does not match
+  that GDB throws an exception when these evaluation errors occur.
 
-  An integral type is a base type that has an encoding of ``DW_ATE_signed``,
-  ``DW_ATE_signed_char``, ``DW_ATE_unsigned``, ``DW_ATE_unsigned_char``,
-  ``DW_ATE_boolean``, or any target architecture defined integral encoding in
-  the inclusive range ``DW_ATE_lo_user`` to ``DW_ATE_hi_user``.
+If a DWARF expression is ill-formed, then the result is undefined.
 
-  .. note::
+The following sections detail the rules for when a DWARF expression is
+ill-formed or results in an evaluation error.
 
-    It is unclear if ``DW_ATE_address`` is an integral type. GDB does not seem
-    to consider it as integral.
+A DWARF expression can either be encoded as a operation expression (see
+:ref:`amdgpu-dwarf-operation-expressions`), or as a location list expression
+(see :ref:`amdgpu-dwarf-location-list-expressions`).
 
-*location description*
+.. _amdgpu-dwarf-expression-evaluation-context:
 
-  *Debugging information must provide consumers a way to find the location of
-  program variables, determine the bounds of dynamic arrays and strings, and
-  possibly to find the base address of a subprogram’s stack frame or the return
-  address of a subprogram. Furthermore, to meet the needs of recent computer
-  architectures and optimization techniques, debugging information must be able
-  to describe the location of an object whose location changes over the object’s
-  lifetime, and may reside at multiple locations simultaneously during parts of
-  an object's lifetime.*
+DWARF Expression Evaluation Context
++++++++++++++++++++++++++++++++++++
 
-  Information about the location of program objects is provided by location
-  descriptions.
+A DWARF expression is evaluated in a context that can include a number of
+context elements.  If multiple context elements are specified then they must be
+self consistent or the result of the evaluation is undefined. The context
+elements that can be specified are:
 
-  Location descriptions can consist of one or more single location descriptions.
+*A current result kind*
 
-  A single location description specifies the location storage that holds a
-  program object and a position within the location storage where the program
-  object starts. The position within the location storage is expressed as a bit
-  offset relative to the start of the location storage.
+  The kind of result required by the DWARF expression evaluation. If specified
+  it can be a location description or a value.
 
-  A location storage is a linear stream of bits that can hold values. Each
-  location storage has a size in bits and can be accessed using a zero-based bit
-  offset. The ordering of bits within a location storage uses the bit numbering
-  and direction conventions that are appropriate to the current language on the
-  target architecture.
+*A current thread*
 
-  There are five kinds of location storage:
+  The target architecture thread identifier of the source program thread of
+  execution for which a user presented expression is currently being evaluated.
 
-  *memory location storage*
-    Corresponds to the target architecture memory address spaces.
+  It is required for operations that are related to target architecture threads.
 
-  *register location storage*
-    Corresponds to the target architecture registers.
+  *For example, the* ``DW_OP_form_tls_address`` *operation and*
+  ``DW_OP_LLVM_form_aspace_address`` *operation when given an address space that
+  is thread specific.*
 
-  *implicit location storage*
-    Corresponds to fixed values that can only be read.
+*A current lane*
 
-  *undefined location storage*
-    Indicates no value is available and therefore cannot be read or written.
+  The target architecture lane identifier of the source program thread of
+  execution for which a user presented expression is currently being evaluated.
+  This applies to languages that are implemented using a SIMD or SIMT execution
+  model.
 
-  *composite location storage*
-    Allows a mixture of these where some bits come from one location storage and
-    some from another location storage, or from disjoint parts of the same
-    location storage.
+  It is required for operations that are related to target architecture lanes.
 
-  .. note::
+  *For example, the* ``DW_OP_LLVM_push_lane`` *operation and*
+  ``DW_OP_LLVM_form_aspace_address`` *operation when given an address space that
+  is lane specific.*
 
-    It may be better to add an implicit pointer location storage kind used by
-    the ``DW_OP_implicit_pointer`` and ``DW_OP_LLVM_aspace_implicit_pointer``
-    operations. It would specify the debugger information entry and byte offset
-    provided by the operations.
-
-  *Location descriptions are a language independent representation of addressing
-  rules. They are created using DWARF operation expressions of arbitrary
-  complexity. They can be the result of evaluting a debugger information entry
-  attribute that specifies an operation expression. In this usage they can
-  describe the location of an object as long as its lifetime is either static or
-  the same as the lexical block (see DWARF Version 5 section 3.5) that owns it,
-  and it does not move during its lifetime. They can be the result of evaluating
-  a debugger information entry attribute that specifies a location list
-  expression. In this usage they can describe the location of an object that has
-  a limited lifetime, changes its location during its lifetime, or has multiple
-  locations over part or all of its lifetime.*
-
-  If a location description has more than one single location description, the
-  DWARF expression is ill-formed if the object value held in each single
-  location description's position within the associated location storage is not
-  the same value, except for the parts of the value that are uninitialized.
-
-  *A location description that has more than one single location description can
-  only be created by a location list expression that has overlapping program
-  location ranges, or certain expression operations that act on a location
-  description that has more than one single location description. There are no
-  operation expression operations that can directly create a location
-  description with more than one single location description.*
-
-  *A location description with more than one single location description can be
-  used to describe objects that reside in more than one piece of storage at the
-  same time. An object may have more than one location as a result of
-  optimization. For example, a value that is only read may be promoted from
-  memory to a register for some region of code, but later code may revert to
-  reading the value from memory as the register may be used for other purposes.
-  For the code region where the value is in a register, any change to the object
-  value must be made in both the register and the memory so both regions of code
-  will read the updated value.*
-
-  *A consumer of a location description with more than one single location
-  description can read the object's value from any of the single location
-  descriptions (since they all refer to location storage that has the same
-  value), but must write any changed value to all the single location
-  descriptions.*
+  If specified, it must be consistent with any specified current thread and
+  current target architecture. It is consistent with a thread if it identifies a
+  lane of the thread. It is consistent with a target architecture if it is a
+  valid lane identifier of the target architecture. Otherwise the result is
+  undefined.
 
-A DWARF expression can either be encoded as a operation expression (see
-:ref:`amdgpu-dwarf-operation-expressions`), or as a location list expression
-(see :ref:`amdgpu-dwarf-location-list-expressions`).
+*A current call frame*
 
-A DWARF expression is evaluated in the context of:
-
-*A current subprogram*
-  This may be used in the evaluation of register access operations to support
-  virtual unwinding of the call stack (see
+  The target architecture call frame identifier. It identifies a call frame that
+  corresponds to an active invocation of a subprogram in the current thread. It
+  is identified by its address on the call stack. The address is referred to as
+  the Canonical Frame Address (CFA). The call frame information is used to
+  determine the CFA for the call frames of the current thread's call stack (see
   :ref:`amdgpu-dwarf-call-frame-information`).
 
+  It is required for operations that specify target architecture registers to
+  support virtual unwinding of the call stack.
+
+  *For example, the* ``DW_OP_*reg*`` *operations.*
+
+  If specified, it must be an active call frame in the current thread. If the
+  current lane is specified, then that lane must have been active on entry to
+  the call frame (see the ``DW_AT_LLVM_lane_pc`` attribute). Otherwise the
+  result is undefined.
+
+  If it is the currently executing call frame, then it is termed the top call
+  frame.
+
 *A current program location*
-  This may be used in the evaluation of location list expressions to select
-  amongst multiple program location ranges. It should be the program location
-  corresponding to the current subprogram. If the current subprogram was reached
-  by virtual call stack unwinding, then the program location will correspond to
-  the associated call site.
+
+  The target architecture program location corresponding to the current call
+  frame of the current thread.
+
+  The program location of the top call frame is the target architecture program
+  counter for the current thread. The call frame information is used to obtain
+  the value of the return address register to determine the program location of
+  the other call frames (see :ref:`amdgpu-dwarf-call-frame-information`).
+
+  It is required for the evaluation of location list expressions to select
+  amongst multiple program location ranges. It is required for operations that
+  specify target architecture registers to support virtual unwinding of the call
+  stack (see :ref:`amdgpu-dwarf-call-frame-information`).
+
+  If specified:
+
+  * If the current lane is not specified:
+
+    * If the current call frame is the top call frame, it must be the current
+      target architecture program location.
+
+    * If the current call frame F is not the top call frame, it must be the
+      program location associated with the call site in the current caller frame
+      F that invoked the callee frame.
+
+  * If the current lane is specified and the architecture program location LPC
+    computed by the ``DW_AT_LLVM_lane_pc`` attribute for the current lane is not
+    the undefined location description (indicating the lane was not active on
+    entry to the call frame), it must be LPC.
+
+  * Otherwise the result is undefined.
+
+*A current compilation unit*
+
+  The compilation unit debug information entry that contains the DWARF expression
+  being evaluated.
+
+  It is required for operations that reference debug information associated with
+  the same compilation unit, including indicating if such references use the
+  32-bit or 64-bit DWARF format. It can also provide the default address space
+  address size if no current target architecture is specified.
+
+  *For example, the* ``DW_OP_constx`` *and* ``DW_OP_addrx`` *operations.*
+
+  *Note that this compilation unit may not be the same as the compilation unit
+  determined from the loaded code object corresponding to the current program
+  location. For example, the evaluation of the expression E associated with a
+  ``DW_AT_location`` attribute of the debug information entry operand of the
+  ``DW_OP_call*`` operations is evaluated with the compilation unit that
+  contains E and not the one that contains the ``DW_OP_call*`` operation
+  expression.*
+
+*A current target architecture*
+
+  The target architecture.
+
+  It is required for operations that specify target architecture specific
+  entities.
+
+  *For example, target architecture specific entities include DWARF register
+  identifiers, DWARF lane identifiers, DWARF address space identifiers, the
+  default address space, and the address space address sizes.*
+
+  If specified:
+
+  * If the current thread is specified, then the current target architecture
+    must be the same as the target architecture of the current thread.
+
+  * If the current compilation unit is specified, then the current target
+    architecture default address space address size must be the same as he
+    ``address_size`` field in the header of the current compilation unit and any
+    associated entry in the ``.debug_aranges`` section.
+
+  * If the current program location is specified, then the current target
+    architecture must be the same as the target architecture of any line number
+    information entry (see :ref:`amdgpu-dwarf-line-number-information`)
+    corresponding to the current program location.
+
+  * If the current program location is specified, then the current target
+    architecture default address space address size must be the same as he
+    ``address_size`` field in the header of any entry corresponding to the
+    current program location in the ``.debug_addr``, ``.debug_line``,
+    ``.debug_rnglists``, ``.debug_rnglists.dwo``, ``.debug_loclists``, and
+    ``.debug_loclists.dwo`` sections.
+
+  * Otherwise the result is undefined.
+
+*A current object*
+
+  The location description of a program object.
+
+  It is required for the ``DW_OP_push_object_address`` operation.
+
+  *For example, the* ``DW_AT_data_location`` *attribute on type debug
+  information entries specifies the the program object corresponding to a
+  runtime descriptor as the current object when it evaluates its associated
+  expression.*
+
+  The result is undefined if the location descriptor is invalid (see
+  :ref:`amdgpu-dwarf-location-description`).
 
 *An initial stack*
+
   This is a list of values or location descriptions that will be pushed on the
   operation expression evaluation stack in the order provided before evaluation
   of an operation expression starts.
@@ -630,13 +690,204 @@ A DWARF expression is evaluated in the context of:
   expression value with initial stack entries. In all other cases the initial
   stack is empty.
 
-When a DWARF expression is evaluated, it may be specified whether a value or
-location description is required as the result kind.
+  The result is undefined if any location descriptors are invalid (see
+  :ref:`amdgpu-dwarf-location-description`).
 
-If a result kind is specified, and the result of the evaluation does not match
-the specified result kind, then the implicit conversions described in
-:ref:`amdgpu-dwarf-memory-location-description-operations` are performed if
-valid. Otherwise, the DWARF expression is ill-formed.
+If the evaluation requires a context element that is not specified, then the
+result of the evaluation is an error.
+
+*A DWARF expression for the location description may be able to be evaluated
+without a thread, lane, call frame, program location, or architecture context.
+For example, the location of a global variable may be able to be evaluated
+without such context. If the expression evaluates with an error then it may
+indicate the variable has been optimized and so requires more context.*
+
+*The DWARF expression for call frame information (see
+:ref:`amdgpu-dwarf-call-frame-information`) operations are restricted to those
+that do not require the compilation unit context to be specified.*
+
+The DWARF is ill-formed if all the ``address_size`` fields in the headers of all
+the entries in the ``.debug_info``, ``.debug_addr``, ``.debug_line``,
+``.debug_rnglists``, ``.debug_rnglists.dwo``, ``.debug_loclists``, and
+``.debug_loclists.dwo`` sections corresponding to any given program location do
+not match.
+
+.. _amdgpu-dwarf-expression-value:
+
+DWARF Expression Value
+++++++++++++++++++++++
+
+A value has a type and a literal value. It can represent a literal value of any
+supported base type of the target architecture. The base type specifies the size
+and encoding of the literal value.
+
+.. note::
+
+  It may be desirable to add an implicit pointer base type encoding. It would be
+  used for the type of the value that is produced when the ``DW_OP_deref*``
+  operation retrieves the full contents of an implicit pointer location storage
+  created by the ``DW_OP_implicit_pointer`` or
+  ``DW_OP_LLVM_aspace_implicit_pointer`` operations. The literal value would
+  record the debugging information entry and byte displacement specified by the
+  associated ``DW_OP_implicit_pointer`` or
+  ``DW_OP_LLVM_aspace_implicit_pointer`` operations.
+
+There is a distinguished base type termed the generic type, which is an integral
+type that has the size of an address in the target architecture default address
+space and unspecified signedness.
+
+*The generic type is the same as the unspecified type used for stack operations
+defined in DWARF Version 4 and before.*
+
+An integral type is a base type that has an encoding of ``DW_ATE_signed``,
+``DW_ATE_signed_char``, ``DW_ATE_unsigned``, ``DW_ATE_unsigned_char``,
+``DW_ATE_boolean``, or any target architecture defined integral encoding in the
+inclusive range ``DW_ATE_lo_user`` to ``DW_ATE_hi_user``.
+
+.. note::
+
+  It is unclear if ``DW_ATE_address`` is an integral type. GDB does not seem to
+  consider it as integral.
+
+.. _amdgpu-dwarf-location-description:
+
+DWARF Location Description
+++++++++++++++++++++++++++
+
+*Debugging information must provide consumers a way to find the location of
+program variables, determine the bounds of dynamic arrays and strings, and
+possibly to find the base address of a subprogram’s call frame or the return
+address of a subprogram. Furthermore, to meet the needs of recent computer
+architectures and optimization techniques, debugging information must be able to
+describe the location of an object whose location changes over the object’s
+lifetime, and may reside at multiple locations simultaneously during parts of an
+object's lifetime.*
+
+Information about the location of program objects is provided by location
+descriptions.
+
+Location descriptions can consist of one or more single location descriptions.
+
+A single location description specifies the location storage that holds a
+program object and a position within the location storage where the program
+object starts. The position within the location storage is expressed as a bit
+offset relative to the start of the location storage.
+
+A location storage is a linear stream of bits that can hold values. Each
+location storage has a size in bits and can be accessed using a zero-based bit
+offset. The ordering of bits within a location storage uses the bit numbering
+and direction conventions that are appropriate to the current language on the
+target architecture.
+
+There are five kinds of location storage:
+
+*memory location storage*
+  Corresponds to the target architecture memory address spaces.
+
+*register location storage*
+  Corresponds to the target architecture registers.
+
+*implicit location storage*
+  Corresponds to fixed values that can only be read.
+
+*undefined location storage*
+  Indicates no value is available and therefore cannot be read or written.
+
+*composite location storage*
+  Allows a mixture of these where some bits come from one location storage and
+  some from another location storage, or from disjoint parts of the same
+  location storage.
+
+.. note::
+
+  It may be better to add an implicit pointer location storage kind used by the
+  ``DW_OP_implicit_pointer`` and ``DW_OP_LLVM_aspace_implicit_pointer``
+  operations. It would specify the debugger information entry and byte offset
+  provided by the operations.
+
+*Location descriptions are a language independent representation of addressing
+rules. They are created using DWARF operation expressions of arbitrary
+complexity. They can be the result of evaluating a debugger information entry
+attribute that specifies an operation expression. In this usage they can
+describe the location of an object as long as its lifetime is either static or
+the same as the lexical block (see DWARF Version 5 section 3.5) that owns it,
+and it does not move during its lifetime. They can be the result of evaluating a
+debugger information entry attribute that specifies a location list expression.
+In this usage they can describe the location of an object that has a limited
+lifetime, changes its location during its lifetime, or has multiple locations
+over part or all of its lifetime.*
+
+If a location description has more than one single location description, the
+DWARF expression is ill-formed if the object value held in each single location
+description's position within the associated location storage is not the same
+value, except for the parts of the value that are uninitialized.
+
+*A location description that has more than one single location description can
+only be created by a location list expression that has overlapping program
+location ranges, or certain expression operations that act on a location
+description that has more than one single location description. There are no
+operation expression operations that can directly create a location description
+with more than one single location description.*
+
+*A location description with more than one single location description can be
+used to describe objects that reside in more than one piece of storage at the
+same time. An object may have more than one location as a result of
+optimization. For example, a value that is only read may be promoted from memory
+to a register for some region of code, but later code may revert to reading the
+value from memory as the register may be used for other purposes. For the code
+region where the value is in a register, any change to the object value must be
+made in both the register and the memory so both regions of code will read the
+updated value.*
+
+*A consumer of a location description with more than one single location
+description can read the object's value from any of the single location
+descriptions (since they all refer to location storage that has the same value),
+but must write any changed value to all the single location descriptions.*
+
+The evaluation of an expression may require context elements to create a
+location description. If such a location description is accessed, the storage it
+denotes is that associated with the context element values specified when the
+location description was created, which may differ from the context at the time
+it is accessed.
+
+*For example, creating a register location description requires the thread
+context: the location storage is for the specified register of that thread.
+Creating a memory location description for an address space may required a
+thread and a lane context: the location storage is the memory associated with
+that thread and lane.*
+
+If any of the context elements required to create a location description change,
+the location description becomes invalid and accessing it is undefined.
+
+*Examples of context that can invalidate a location description are:*
+
+* *The thread context is required and execution causes the thread to terminate.*
+* *The call frame context is required and further execution causes the call
+  frame to return to the calling frame.*
+* *The program location is required and further execution of the thread occurs.
+  That could change the location list entry or call frame information entry that
+  applies.*
+* *An operation uses call frame information:*
+
+  * *Any of the frames used in the virtual call frame unwinding return.*
+  * *The top call frame is used, the program location is used to select the call
+    frame information entry, and further execution of the thread occurs.*
+
+*A DWARF expression can be used to compute a location description for an object.
+A subsequent DWARF expression evaluation can be given the object location
+description as the object context or initial stack context to compute a
+component of the object. The final result is undefined if the object location
+description becomes invalid between the two expression evaluations.*
+
+A change of a thread's program location may not make a location description
+invalid, yet may still render it as no longer meaningful. Accessing such a
+location description, or using it as the object context or initial stack context
+of an expression evaluation, may produce an undefined result.
+
+*For example, a location description may specify a register that no longer holds
+the intended program object after a program location change. One way to avoid
+such problems is to recompute location descriptions associated with threads when
+their program locations change.*
 
 .. _amdgpu-dwarf-operation-expressions:
 
@@ -658,10 +909,16 @@ operation expression is ill-formed.
 Evaluation of an operation expression starts with an empty stack on which the
 entries from the initial stack provided by the context are pushed in the order
 provided. Then the operations are evaluated, starting with the first operation
-of the stream, until one past the last operation of the stream is reached. The
-result of the evaluation is:
+of the stream. Evaluation continues until either an operation has an evaluation
+error, or until one past the last operation of the stream is reached.
 
-* If evaluation of the DWARF expression requires a location description, then:
+The result of the evaluation is:
+
+* If an operation has an evaluation error, or an operation evaluates an
+  expression that has an evaluation error, then the result is an evaluation
+  error.
+
+* If the current result kind specifies a location description, then:
 
   * If the stack is empty, the result is a location description with one
     undefined location description.
@@ -682,7 +939,7 @@ result of the evaluation is:
       Could define this case as returning an implicit location description as
       if the ``DW_OP_implicit`` operation is performed.
 
-* If evaluation of the DWARF expression requires a value, then:
+* If the current result kind specifies a value, then:
 
   * If the top stack entry is a value, or can be converted to one (see
     :ref:`amdgpu-dwarf-memory-location-description-operations`), then the result
@@ -691,8 +948,7 @@ result of the evaluation is:
 
   * Otherwise the DWARF expression is ill-formed.
 
-* If evaluation of the DWARF expression does not specify if a value or location
-  description is required, then:
+* If the current result kind is not specified, then:
 
   * If the stack is empty, the result is a location description with one
     undefined location description.
@@ -837,24 +1093,43 @@ expression.
     ``DW_OP_call2``, ``DW_OP_call4``, and ``DW_OP_call_ref`` perform DWARF
     procedure calls during evaluation of a DWARF expression.
 
-    ``DW_OP_call2`` and ``DW_OP_call4``, have one operand that is a 2- or 4-byte
-    unsigned offset, respectively, of a debugging information entry D in the
-    current compilation unit.
+    ``DW_OP_call2`` and ``DW_OP_call4``, have one operand that is, respectively,
+    a 2-byte or 4-byte unsigned offset DR that represents the byte offset of a
+    debugging information entry D relative to the beginning of the current
+    compilation unit.
 
     ``DW_OP_call_ref`` has one operand that is a 4-byte unsigned value in the
     32-bit DWARF format, or an 8-byte unsigned value in the 64-bit DWARF format,
-    that represents an offset of a debugging information entry D in a
-    ``.debug_info`` section, which may be contained in an executable or shared
-    object file other than that containing the operation. For references from
-    one executable or shared object file to another, the relocation must be
-    performed by the consumer.
+    that represents the byte offset DR of a debugging information entry D
+    relative to the beginning of the ``.debug_info`` section that contains the
+    current compilation unit. D may not be in the current compilation unit.
 
     .. note:
 
-      It is unclear how crossing from one executable or shared object file to
-      another can work. How would a consumer know which executable or shared
-      object file is being referenced? In an ELF file the DWARF is in a
-      non-ALLOC segment so standard dynamic relocations cannot be used.
+      DWARF Version 5 states that DR can be an offset in a ``.debug_info``
+      section other than the one that contains the current compilation unit. It
+      states that relocation of references from one executable or shared object
+      file to another must be performed by the consumer. But given that DR is
+      defined as an offset in a ``.debug_info`` section this seems impossible.
+      If DR was defined as an implementation defined value, then the consumer
+      could choose to interpret the value in an implementation defined manner to
+      reference a debug information in another executable or shared object.
+
+      In ELF the ``.debug_info`` section is in a non-\ ``PT_LOAD`` segment so
+      standard dynamic relocations cannot be used. But even if they were loaded
+      segments and dynamic relocations were used, DR would need to be the
+      address of D, not an offset in a ``.debug_info`` section. That would also
+      need DR to be the size of a global address. So it would not be possible to
+      use the 32-bit DWARF format in a 64-bit global address space. In addition,
+      the consumer would need to determine what executable or shared object the
+      relocated address was in so it could determine the containing compilation
+      unit.
+
+      GDB only interprets DR as an offset in the ``.debug_info`` section that
+      contains the current compilation unit.
+
+      This comment also applies to ``DW_OP_implicit_pointer`` and
+      ``DW_OP_LLVM_aspace_implicit_pointer``.
 
     *Operand interpretation of* ``DW_OP_call2``\ *,* ``DW_OP_call4``\ *, and*
     ``DW_OP_call_ref`` *is exactly like that for* ``DW_FORM_ref2``\ *,
@@ -867,8 +1142,13 @@ expression.
       operation expression continues from the first operation of E. Execution
       continues until one past the last operation of E is reached, at which
       point execution continues with the operation following the call operation.
-      Since E is evaluated on the same stack as the call, E can use, add, and/or
-      remove entries already on the stack.
+      The operations of E are evaluated with the same current context, except
+      current compilation unit is the one that contains D and the stack is the
+      same as that being used by the call operation. After the call operation
+      has been evaluated, the stack is therefore as it is left by the evaluation
+      of the operations of E. Since E is evaluated on the same stack as the call
+      operation, E can use, and/or remove entries already on the stack, and can
+      add new entries to the stack.
 
       *Values on the stack at the time of the call may be used as parameters by
       the called expression and values left on the stack by the called expression
@@ -877,10 +1157,10 @@ expression.
 
     * If D has a ``DW_AT_location`` attribute that is encoded as a ``loclist`` or
       ``loclistsptr``, then the specified location list expression E is
-      evaluated, and the resulting location description is pushed on the stack.
-      The evaluation of E uses a context that has the same current frame and
-      current program location as the current operation expression, but an empty
-      initial stack.
+      evaluated. The evaluation of E uses the current context, except the result
+      kind is a location description, the compilation unit is the one that
+      contains D, and the initial stack is empty. The location description
+      result is pushed on the stack.
 
       .. note::
 
@@ -909,7 +1189,7 @@ expression.
 
         GDB implements ``DW_OP_call*`` by always executing E on the same stack.
         If the location list has multiple matching entries, it simply picks the
-        first one and ignores the rest. This seems fundementally at odds with
+        first one and ignores the rest. This seems fundamentally at odds with
         the desire to supporting multiple places for variables.
 
         So, it feels like ``DW_OP_call*`` should both support pushing a location
@@ -930,7 +1210,7 @@ expression.
         ``exprproc``, ``loclistproc``, and ``loclistsptrproc`` to indicate that
         the expression is executed on the same stack. ``exprproc`` is the same
         encoding as ``exprloc``. ``loclistproc`` and ``loclistsptrproc`` are the
-        same encoding as their non-\ ``proc`` counterparts except the DWARF is
+        same encoding as their non-\ ``proc`` counterparts, except the DWARF is
         ill-formed if the location list does not match exactly one location list
         entry and a default entry is required. These forms indicate explicitly
         that the matched single operation expression must be executed on the
@@ -951,10 +1231,10 @@ expression.
         information entry. The DWARF would be ill-formed if E is a location list
         expression that does not match exactly one location list entry. In all
         other cases the evaluation of an expression E that is the value of a
-        ``DW_AT_location`` attribute would evaluate E with a context that has
-        the same current frame and current program location as the current
-        operation expression, but an empty initial stack, and push the resulting
-        location description on the stack.
+        ``DW_AT_location`` attribute would evaluate E with the current context,
+        except the result kind is a location description, the compilation unit
+        is the one that contains D, and the initial stack is empty. The location
+        description result is pushed on the stack.
 
     * If D has a ``DW_AT_const_value`` attribute with a value V, then it is as
       if a ``DW_OP_implicit_value V`` operation was executed.
@@ -1070,18 +1350,18 @@ size and the low-order bits used.
 9.  ``DW_OP_const_type``
 
     ``DW_OP_const_type`` has three operands. The first is an unsigned LEB128
-    integer that represents the offset of a debugging information entry D in the
-    current compilation unit, that provides the type of the constant value. The
-    second is a 1-byte unsigned integral constant S. The third is a block of
-    bytes B, with a length equal to S.
+    integer DR that represents the byte offset of a debugging information entry
+    D relative to the beginning of the current compilation unit, that provides
+    the type T of the constant value. The second is a 1-byte unsigned integral
+    constant S. The third is a block of bytes B, with a length equal to S.
 
-    T is the bit size of the type D. The least significant T bits of B are
+    TS is the bit size of the type T. The least significant TS bits of B are
     interpreted as a value V of the type D. It pushes the value V with the type
     D.
 
     The DWARF is ill-formed if D is not a ``DW_TAG_base_type`` debugging
-    information entry, or if T divided by 8 and rounded up to a multiple of 8
-    (the byte size) is not equal to S.
+    information entry in the current compilation unit, or if TS divided by 8
+    (the byte size) and rounded up to a whole number is not equal to S.
 
     *While the size of the byte block B can be inferred from the type D
     definition, it is encoded explicitly into the operation so that the
@@ -1090,9 +1370,8 @@ size and the low-order bits used.
 
 10. ``DW_OP_LLVM_push_lane`` *New*
 
-    ``DW_OP_LLVM_push_lane`` pushes a value with the generic type that is the
-    target architecture specific lane identifier of the thread of execution for
-    which a user presented expression is currently being evaluated.
+    ``DW_OP_LLVM_push_lane`` pushes the target architecture lane identifier of
+    the current lane as a value with the generic type.
 
     *For languages that are implemented using a SIMD or SIMT execution model,
     this is the lane number that corresponds to the source language thread of
@@ -1127,123 +1406,137 @@ There are these special value operations currently defined:
 
     ``DW_OP_regval_type`` has two operands. The first is an unsigned LEB128
     integer that represents a register number R. The second is an unsigned
-    LEB128 integer that represents the offset of a debugging information entry D
-    in the current compilation unit, that provides the type of the register
-    value.
+    LEB128 integer DR that represents the byte offset of a debugging information
+    entry D relative to the beginning of the current compilation unit, that
+    provides the type T of the register value.
 
-    The contents of register R are interpreted as a value V of the type D. The
-    value V is pushed on the stack with the type D.
-
-    The DWARF is ill-formed if D is not a ``DW_TAG_base_type`` debugging
-    information entry, or if the size of type D is not the same as the size of
-    register R.
+    The operation is equivalent to performing ``DW_OP_regx R; DW_OP_deref_type
+    DR``.
 
     .. note::
 
-      Should DWARF allow the type D to be a different size to the size of the
-      register R? Requiring them to be the same bit size avoids any issue of
-      conversion as the bit contents of the register is simply interpreted as a
-      value of the specified type. If a conversion is wanted it can be done
+      Should DWARF allow the type T to be a larger size than the size of the
+      register R? Restricting a larger bit size avoids any issue of conversion
+      as the, possibly truncated, bit contents of the register is simply
+      interpreted as a value of T. If a conversion is wanted it can be done
       explicitly using a ``DW_OP_convert`` operation.
 
       GDB has a per register hook that allows a target specific conversion on a
-      register by register basis. It defaults to truncation of bigger registers,
-      and to actually reading bytes from the next register (or reads out of
-      bounds for the last register) for smaller registers. There are no GDB
-      tests that read a register out of bounds (except an illegal hand written
-      assembly test).
+      register by register basis. It defaults to truncation of bigger registers.
+      Removing use of the target hook does not cause any test failures in common
+      architectures. If the compiler for a target architecture did want some
+      form of conversion, including a larger result type, it could always
+      explicitly used the ``DW_OP_convert`` operation.
+
+      If T is a larger type than the register size, then the default GDB
+      register hook reads bytes from the next register (or reads out of bounds
+      for the last register!). Removing use of the target hook does not cause
+      any test failures in common architectures (except an illegal hand written
+      assembly test). If a target architecture requires this behavior, this
+      proposal allows a composite location description to be used to combine
+      multiple registers.
 
 2.  ``DW_OP_deref``
 
-    The ``DW_OP_deref`` operation pops one stack entry that must be a location
-    description L.
+    S is the bit size of the generic type divided by 8 (the byte size) and
+    rounded up to a whole number. DR is the offset of a hypothetical debug
+    information entry D in the current compilation unit for a base type of the
+    generic type.
 
-    A value of the bit size of the generic type is retrieved from the location
-    storage specified by L. The value V retrieved is pushed on the stack with
-    the generic type.
-
-    If any bit of the value is retrieved from the undefined location storage, or
-    the offset of any bit exceeds the size of the location storage specified by
-    L, then the DWARF expression is ill-formed.
-
-    See :ref:`amdgpu-dwarf-implicit-location-descriptions` for special rules
-    concerning implicit location descriptions created by the
-    ``DW_OP_implicit_pointer`` and ``DW_OP_LLVM_implicit_aspace_pointer``
-    operations.
-
-    *If L, or the location description of any composite location description
-    part that is a subcomponent of L, has more than one single location
-    description, then any one of them can be selected as they are required to
-    all have the same value. For any single location description SL, bits are
-    retrieved from the associated storage location starting at the bit offset
-    specified by SL. For a composite location description, the retrieved bits
-    are the concatenation of the N bits from each composite location part PL,
-    where N is limited to the size of PL.*
+    The operation is equivalent to performing ``DW_OP_deref_type S, DR``.
 
 3.  ``DW_OP_deref_size``
 
     ``DW_OP_deref_size`` has a single 1-byte unsigned integral constant that
     represents a byte result size S.
 
-    It pops one stack entry that must be a location description L.
-
-    T is the smaller of the generic type size and S scaled by 8 (the byte size).
-    A value V of T bits is retrieved from the location storage specified by L.
-    If V is smaller than the size of the generic type, V is zero-extended to the
-    generic type size. V is pushed onto the stack with the generic type.
-
-    The DWARF expression is ill-formed if any bit of the value is retrieved from
-    the undefined location storage, or if the offset of any bit exceeds the size
-    of the location storage specified by L.
+    TS is the smaller of the generic type bit size and S scaled by 8 (the byte
+    size). If TS is smaller than the generic type bit size then T is an unsigned
+    integral type of bit size TS, otherwise T is the generic type. DR is the
+    offset of a hypothetical debug information entry D in the current
+    compilation unit for a base type T.
 
     .. note::
 
       Truncating the value when S is larger than the generic type matches what
-      GDB does. This allows the generic type size to not be a integral byte
+      GDB does. This allows the generic type size to not be an integral byte
       size. It does allow S to be arbitrarily large. Should S be restricted to
       the size of the generic type rounded up to a multiple of 8?
 
-    See :ref:`amdgpu-dwarf-implicit-location-descriptions` for special rules
-    concerning implicit location descriptions created by the
-    ``DW_OP_implicit_pointer`` and ``DW_OP_LLVM_implicit_aspace_pointer``
-    operations.
+    The operation is equivalent to performing ``DW_OP_deref_type S, DR``, except
+    if T is not the generic type, the value V pushed is zero-extended to the
+    generic type bit size and its type changed to the generic type.
 
 4.  ``DW_OP_deref_type``
 
     ``DW_OP_deref_type`` has two operands. The first is a 1-byte unsigned
-    integral constant S. The second is an unsigned LEB128 integer that
-    represents the offset of a debugging information entry D in the current
-    compilation unit, that provides the type of the result value.
+    integral constant S. The second is an unsigned LEB128 integer DR that
+    represents the byte offset of a debugging information entry D relative to
+    the beginning of the current compilation unit, that provides the type T of
+    the result value.
 
-    It pops one stack entry that must be a location description L. T is the bit
-    size of the type D. A value V of T bits is retrieved from the location
-    storage specified by L. V is pushed on the stack with the type D.
+    TS is the bit size of the type T.
 
-    The DWARF is ill-formed if D is not a ``DW_TAG_base_type`` debugging
-    information entry, if T divided by 8 and rounded up to a multiple of 8 (the
-    byte size) is not equal to S, if any bit of the value is retrieved from the
-    undefined location storage, or if the offset of any bit exceeds the size of
-    the location storage specified by L.
-
-    See :ref:`amdgpu-dwarf-implicit-location-descriptions` for special rules
-    concerning implicit location descriptions created by the
-    ``DW_OP_implicit_pointer`` and ``DW_OP_LLVM_implicit_aspace_pointer``
-    operations.
-
-    *While the size of the pushed value V can be inferred from the type D
-    definition, it is encoded explicitly into the operation so that the
-    operation can be parsed easily without reference to the* ``.debug_info``
-    *section.*
+    *While the size of the pushed value V can be inferred from the type T, it is
+    encoded explicitly as the operand S so that the operation can be parsed
+    easily without reference to the* ``.debug_info`` *section.*
 
     .. note::
 
       It is unclear why the operand S is needed. Unlike ``DW_OP_const_type``,
       the size is not needed for parsing. Any evaluation needs to get the base
-      type to record with the value to know its encoding and bit size.
+      type T to push with the value to know its encoding and bit size.
+
+    It pops one stack entry that must be a location description L.
+
+    A value V of TS bits is retrieved from the location storage LS specified by
+    one of the single location descriptions SL of L.
+
+    *If L, or the location description of any composite location description
+    part that is a subcomponent of L, has more than one single location
+    description, then any one of them can be selected as they are required to
+    all have the same value. For any single location description SL, bits are
+    retrieved from the associated storage location starting at the bit offset
+    specified by SL. For a composite location description, the retrieved bits
+    are the concatenation of the N bits from each composite location part PL,
+    where N is limited to the size of PL.*
+
+    V is pushed on the stack with the type T.
+
+    .. note::
+
+      This definition makes it an evaluation error if L is a register location
+      description that has less than TS bits remaining in the register storage.
+      Particularly since this proposal extends location descriptions to have a
+      bit offset, it would be odd to define this as performing sign extension
+      based on the type, or be target architecture dependent, as the number of
+      remaining bits could be any number. This matches the GDB implementation
+      for ``DW_OP_deref_type``.
+
+      This proposal defines ``DW_OP_*breg*`` in terms of ``DW_OP_regval_type``.
+      ``DW_OP_regval_type`` is defined in terms of ``DW_OP_regx``, which uses a
+      0 bit offset, and ``DW_OP_deref_type``. Therefore, it requires the
+      register size to be greater or equal to the address size of the address
+      space. This matches the GDB implementation for ``DW_OP_*breg*``.
+
+    The DWARF is ill-formed if D is not in the current compilation unit, D is
+    not a ``DW_TAG_base_type`` debugging information entry, or if TS divided by
+    8 (the byte size) and rounded up to a whole number is not equal to S.
+
+    .. note::
 
       This definition allows the base type to be a bit size since there seems no
       reason to restrict it.
 
+    It is an evaluation error if any bit of the value is retrieved from the
+    undefined location storage or the offset of any bit exceeds the size of the
+    location storage LS specified by any single location description SL of L.
+
+    See :ref:`amdgpu-dwarf-implicit-location-descriptions` for special rules
+    concerning implicit location descriptions created by the
+    ``DW_OP_implicit_pointer`` and ``DW_OP_LLVM_implicit_aspace_pointer``
+    operations.
+
 5.  ``DW_OP_xderef`` *Deprecated*
 
     ``DW_OP_xderef`` pops two stack entries. The first must be an integral type
@@ -1277,9 +1570,10 @@ There are these special value operations currently defined:
 7.  ``DW_OP_xderef_type`` *Deprecated*
 
     ``DW_OP_xderef_type`` has two operands. The first is a 1-byte unsigned
-    integral constant S. The second operand is an unsigned LEB128
-    integer R that represents the offset of a debugging information entry D in
-    the current compilation unit, that provides the type of the result value.
+    integral constant S. The second operand is an unsigned LEB128 integer DR
+    that represents the byte offset of a debugging information entry D relative
+    to the beginning of the current compilation unit, that provides the type T
+    of the result value.
 
     It pops two stack entries. The first must be an integral type value that
     represents an address A. The second must be an integral type value that
@@ -1294,51 +1588,43 @@ There are these special value operations currently defined:
 
 8.  ``DW_OP_entry_value`` *Deprecated*
 
-    ``DW_OP_entry_value`` pushes the value that the described location held upon
-    entering the current subprogram.
+    ``DW_OP_entry_value`` pushes the value of an expression that is evaluated in
+    the context of the calling frame.
+
+    *It may be used to determine the value of arguments on entry to the current
+    call frame provided they are not clobbered.*
 
     It has two operands. The first is an unsigned LEB128 integer S. The second
     is a block of bytes, with a length equal S, interpreted as a DWARF
     operation expression E.
 
-    E is evaluated as if it had been evaluated upon entering the current
-    subprogram with an empty initial stack.
-
-    .. note::
+    E is evaluated with the current context, except the result kind is
+    unspecified, the call frame is the one that called the current frame, the
+    program location is the call site in the calling frame, the object is
+    unspecified, and the initial stack is empty. The calling frame information
+    is obtained by virtually unwinding the current call frame using the call
+    frame information (see :ref:`amdgpu-dwarf-call-frame-information`).
 
-      It is unclear what this means. What is the current program location and
-      current frame that must be used? Does this require reverse execution so
-      the register and memory state are as it was on entry to the current
-      subprogram?
+    If the result of E is a location description L (see
+    :ref:`amdgpu-dwarf-register-location-descriptions`), and the last operation
+    executed by E is a ``DW_OP_reg*`` for register R with a target architecture
+    specific base type of T, then the contents of the register are retrieved as
+    if a ``DW_OP_deref_type DR`` operation was performed where DR is the offset
+    of a hypothetical debug information entry in the current compilation unit
+    for T. The resulting value V s pushed on the stack.
 
-    The DWARF expression is ill-formed if the evaluation of E executes a
-    ``DW_OP_push_object_address`` operation.
+    *Using* ``DW_OP_reg*`` *provides a more compact form for the case where the
+    value was in a register on entry to the subprogram.*
 
-    If the result of E is a location description with one register location
-    description (see :ref:`amdgpu-dwarf-register-location-descriptions`),
-    ``DW_OP_entry_value`` pushes the value that register had upon entering the
-    current subprogram. The value entry type is the target architecture register
-    base type. If the register value is undefined or the register location
-    description bit offset is not 0, then the DWARF expression is ill-formed.
+    .. note:
 
-    *The register location description provides a more compact form for the case
-    where the value was in a register on entry to the subprogram.*
+      It is unclear how this provides a more compact expression, as
+      ``DW_OP_regval_type`` could be used which is marginally larger.
 
-    If the result of E is a value V, ``DW_OP_entry_value`` pushes V on the
-    stack.
+    If the result of E is a value V, then V is pushed on the stack.
 
     Otherwise, the DWARF expression is ill-formed.
 
-    *The values needed to evaluate* ``DW_OP_entry_value`` *could be obtained in
-    several ways. The consumer could suspend execution on entry to the
-    subprogram, record values needed by* ``DW_OP_entry_value`` *expressions
-    within the subprogram, and then continue. When evaluating*
-    ``DW_OP_entry_value``\ *, the consumer would use these recorded values
-    rather than the current values. Or, when evaluating* ``DW_OP_entry_value``\
-    *, the consumer could virtually unwind using the Call Frame Information
-    (see* :ref:`amdgpu-dwarf-call-frame-information`\ *) to recover register
-    values that might have been clobbered since the subprogram entry point.*
-
     *The* ``DW_OP_entry_value`` *operation is deprecated as its main usage is
     provided by other means. DWARF Version 5 added the*
     ``DW_TAG_call_site_parameter`` *debugger information entry for call sites
@@ -1351,29 +1637,8 @@ There are these special value operations currently defined:
 
     .. note::
 
-      It is unclear why this operation is defined this way. How would a consumer
-      know what values have to be saved on entry to the subprogram? Does it have
-      to parse every expression of every ``DW_OP_entry_value`` operation to
-      capture all the possible results needed? Or does it have to implement
-      reverse execution so it can evaluate the expression in the context of the
-      entry of the subprogram so it can obtain the entry point register and
-      memory values? Or does the compiler somehow instruct the consumer how to
-      create the saved copies of the variables on entry?
-
-      If the expression is simply using existing variables, then it is just a
-      regular expression and no special operation is needed. If the main purpose
-      is only to read the entry value of a register using CFI then it would be
-      better to have an operation that explicitly does just that such as the
-      proposed ``DW_OP_LLVM_call_frame_entry_reg`` operation.
-
-      GDB only seems to implement ``DW_OP_entry_value`` when E is exactly
-      ``DW_OP_reg*`` or ``DW_OP_breg*; DW_OP_deref*``. It evaluates E in the
-      context of the calling subprogram and the calling call site program
-      location. But the wording suggests that is not the intention.
-
-      Given these issues it is suggested ``DW_OP_entry_value`` is deprecated in
-      favor of using the new facities that have well defined semantics and
-      implementations.
+      GDB only implements ``DW_OP_entry_value`` when E is exactly
+      ``DW_OP_reg*`` or ``DW_OP_breg*; DW_OP_deref*``.
 
 .. _amdgpu-dwarf-location-description-operations:
 
@@ -1395,9 +1660,9 @@ General Location Description Operations
     It adds the value of B scaled by 8 (the byte size) to the bit offset of each
     single location description SL of L, and pushes the updated L.
 
-    If the updated bit offset of any SL is less than 0 or greater than or equal
-    to the size of the location storage specified by SL, then the DWARF
-    expression is ill-formed.
+    It is an evaluation error if the updated bit offset of any SL is less than 0
+    or greater than or equal to the size of the location storage specified by
+    SL.
 
 2.  ``DW_OP_LLVM_offset_uconst`` *New*
 
@@ -1426,40 +1691,53 @@ General Location Description Operations
     It adds the value of B to the bit offset of each single location description
     SL of L, and pushes the updated L.
 
-    If the updated bit offset of any SL is less than 0 or greater than or equal
-    to the size of the location storage specified by SL, then the DWARF
-    expression is ill-formed.
+    It is an evaluation error if the updated bit offset of any SL is less than 0
+    or greater than or equal to the size of the location storage specified by
+    SL.
 
 4.  ``DW_OP_push_object_address``
 
     ``DW_OP_push_object_address`` pushes the location description L of the
-    object currently being evaluated as part of evaluation of a user presented
-    expression.
+    current object.
 
-    This object may correspond to an independent variable described by its own
-    debugging information entry or it may be a component of an array, structure,
-    or class whose address has been dynamically determined by an earlier step
-    during user expression evaluation.
+    *This object may correspond to an independent variable that is part of a
+    user presented expression that is being evaluated. The object location
+    description may be determined from the variable's own debugging information
+    entry or it may be a component of an array, structure, or class whose
+    address has been dynamically determined by an earlier step during user
+    expression evaluation.*
 
     *This operation provides explicit functionality (especially for arrays
     involving descriptions) that is analogous to the implicit push of the base
     location description of a structure prior to evaluation of a
     ``DW_AT_data_member_location`` to access a data member of a structure.*
 
+    .. note::
+
+      This operation could be removed and the object location description
+      specified as the initial stack as for ``DW_AT_data_member_location``.
+
+      The only attribute that specifies a current object is
+      ``DW_AT_data_location`` so the non-normative text seems to overstate how
+      this is being used. Or are there other attributes that need to state they
+      pass an object?
+
 5.  ``DW_OP_LLVM_call_frame_entry_reg`` *New*
 
     ``DW_OP_LLVM_call_frame_entry_reg`` has a single unsigned LEB128 integer
     operand that represents a target architecture register number R.
 
     It pushes a location description L that holds the value of register R on
-    entry to the current subprogram as defined by the Call Frame Information
+    entry to the current subprogram as defined by the call frame information
     (see :ref:`amdgpu-dwarf-call-frame-information`).
 
-    *If there is no Call Frame Information defined, then the default rules for
+    *If there is no call frame information defined, then the default rules for
     the target architecture are used. If the register rule is* undefined\ *, then
     the undefined location description is pushed. If the register rule is* same
     value\ *, then a register location description for R is pushed.*
 
+.. _amdgpu-dwarf-undefined-location-description-operations:
+
 Undefined Location Description Operations
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
@@ -1616,8 +1894,19 @@ type.
     least significant S bits as a twos-complement unsigned value A'.
 
     It pushes a location description L with one memory location description SL
-    on the stack. SL specifies the memory location storage that corresponds to
-    AS with a bit offset equal to A' scaled by 8 (the byte size).
+    on the stack. SL specifies the memory location storage LS that corresponds
+    to AS with a bit offset equal to A' scaled by 8 (the byte size).
+
+    If AS is an address space that is specific to context elements, then LS
+    corresponds to the location storage associated with the current context.
+
+    *For example, if AS is for per thread storage then LS is the location
+    storage for the current thread. For languages that are implemented using a
+    SIMD or SIMT execution model, then if AS is for per lane storage then LS is
+    the location storage for the current lane of the current thread. Therefore,
+    if L is accessed by an operation, the location storage selected when the
+    location description was created is accessed, and not the location storage
+    associated with the current context of the access operation.*
 
     The DWARF expression is ill-formed if AS is not one of the values defined by
     the target architecture specific ``DW_ASPACE_*`` values.
@@ -1630,16 +1919,17 @@ type.
 4.  ``DW_OP_form_tls_address``
 
     ``DW_OP_form_tls_address`` pops one stack entry that must be an integral
-    type value and treats it as a thread-local storage address T.
+    type value and treats it as a thread-local storage address TA.
 
     It pushes a location description L with one memory location description SL
     on the stack. SL is the target architecture specific memory location
-    description that corresponds to the thread-local storage address T.
+    description that corresponds to the thread-local storage address TA.
 
-    The meaning of the thread-local storage address T is defined by the run-time
-    environment. If the run-time environment supports multiple thread-local
-    storage blocks for a single thread, then the block corresponding to the
-    executable or shared library containing this DWARF expression is used.
+    The meaning of the thread-local storage address TA is defined by the
+    run-time environment. If the run-time environment supports multiple
+    thread-local storage blocks for a single thread, then the block
+    corresponding to the executable or shared library containing this DWARF
+    expression is used.
 
     *Some implementations of C, C++, Fortran, and other languages support a
     thread-local storage class. Variables with this storage class have distinct
@@ -1661,15 +1951,15 @@ type.
 5.  ``DW_OP_call_frame_cfa``
 
     ``DW_OP_call_frame_cfa`` pushes the location description L of the Canonical
-    Frame Address (CFA) of the current subprogram, obtained from the Call Frame
-    Information on the stack. See :ref:`amdgpu-dwarf-call-frame-information`.
+    Frame Address (CFA) of the current subprogram, obtained from the call frame
+    information on the stack. See :ref:`amdgpu-dwarf-call-frame-information`.
 
     *Although the value of the* ``DW_AT_frame_base`` *attribute of the debugger
     information entry corresponding to the current subprogram can be computed
     using a location list expression, in some cases this would require an
     extensive location list because the values of the registers used in
-    computing the CFA change during a subprogram execution. If the Call Frame
-    Information is present, then it already encodes such changes, and it is
+    computing the CFA change during a subprogram execution. If the call frame
+    information is present, then it already encodes such changes, and it is
     space efficient to reference that using the* ``DW_OP_call_frame_cfa``
     *operation.*
 
@@ -1701,10 +1991,12 @@ type.
     The address size S is defined as the address bit size of the target
     architecture specific address space corresponding to AS.
 
-    The contents of the register specified by R are retrieved as a
-    twos-complement unsigned value and zero extended to S bits. B is added and
-    the least significant S bits are treated as a twos-complement unsigned value
-    to be used as an address A.
+    The contents of the register specified by R are retrieved as if a
+    ``DW_OP_regval_type R, DR`` operation was performed where DR is the offset
+    of a hypothetical debug information entry in the current compilation unit
+    for an unsigned integral base type of size S bits. B is added and the least
+    significant S bits are treated as an unsigned value to be used as an address
+    A.
 
     They push a location description L comprising one memory location
     description LS on the stack. LS specifies the memory location storage that
@@ -1716,8 +2008,8 @@ type.
     that represents a register number R. The second is a signed LEB128
     integer that represents a byte displacement B.
 
-    The action is the same as for ``DW_OP_breg<N>`` except that R is used as the
-    register number and B is used as the byte displacement.
+    The action is the same as for ``DW_OP_breg<N>``, except that R is used as
+    the register number and B is used as the byte displacement.
 
 9.  ``DW_OP_LLVM_aspace_bregx`` *New*
 
@@ -1727,9 +2019,9 @@ type.
     entry that is required to be an integral type value that represents a target
     architecture specific address space identifier AS.
 
-    The action is the same as for ``DW_OP_breg<N>`` except that R is used as the
-    register number, B is used as the byte displacement, and AS is used as the
-    address space identifier.
+    The action is the same as for ``DW_OP_breg<N>``, except that R is used as
+    the register number, B is used as the byte displacement, and AS is used as
+    the address space identifier.
 
     The DWARF expression is ill-formed if AS is not one of the values defined by
     the target architecture specific ``DW_ASPACE_*`` values.
@@ -1759,18 +2051,32 @@ register starting at the specified bit offset.
     numbered from 0 through 31, inclusive. The target architecture register
     number R corresponds to the N in the operation name.
 
-    They push a location description L that specifies one register location
-    description SL on the stack. SL specifies the register location storage that
-    corresponds to R with a bit offset of 0.
+    The operation is equivalent to performing ``DW_OP_regx R``.
 
 2.  ``DW_OP_regx``
 
     ``DW_OP_regx`` has a single unsigned LEB128 integer operand that represents
     a target architecture register number R.
 
-    It pushes a location description L that specifies one register location
-    description SL on the stack. SL specifies the register location storage that
-    corresponds to R with a bit offset of 0.
+    If the current call frame is the top call frame, it pushes a location
+    description L that specifies one register location description SL on the
+    stack. SL specifies the register location storage that corresponds to R with
+    a bit offset of 0 for the current thread.
+
+    If the current call frame is not the top call frame, call frame information
+    (see :ref:`amdgpu-dwarf-call-frame-information`) is used to determine the
+    location description that holds the register for the current call frame and
+    current program location of the current thread. The resulting location
+    description L is pushed.
+
+    *Note that if call frame information is used, the resulting location
+    description may be register, memory, or undefined.*
+
+    *An implementation may evaluate the call frame information immediately, or
+    may defer evaluation until L is accessed by an operation. If evaluation is
+    defered, R and the current context can be recorded in L. When accessed, the
+    recorded context is used to evaluate the call frame information, not the
+    current context of the access operation.*
 
 *These operations obtain a register location. To fetch the contents of a
 register, it is necessary to use* ``DW_OP_regval_type``\ *, use one of the*
@@ -1844,16 +2150,14 @@ implicit storage value starting at the bit offset.
     to access the dereferenced pointer, even when it cannot access the pointer
     itself.*
 
-    ``DW_OP_implicit_pointer`` has two operands. The first is a 4-byte unsigned
-    value in the 32-bit DWARF format, or an 8-byte unsigned value in the 64-bit
-    DWARF format, that represents a debugging information entry reference R. The
-    second is a signed LEB128 integer that represents a byte displacement B.
+    ``DW_OP_implicit_pointer`` has two operands. The first operand is a 4-byte
+    unsigned value in the 32-bit DWARF format, or an 8-byte unsigned value in
+    the 64-bit DWARF format, that represents the byte offset DR of a debugging
+    information entry D relative to the beginning of the ``.debug_info`` section
+    that contains the current compilation unit. The second operand is a signed
+    LEB128 integer that represents a byte displacement B.
 
-    R is used as the offset of a debugging information entry D in a
-    ``.debug_info`` section, which may be contained in an executable or shared
-    object file other than that containing the operation. For references from one
-    executable or shared object file to another, the relocation must be
-    performed by the consumer.
+    *Note that D may not be in the current compilation unit.*
 
     *The first operand interpretation is exactly like that for*
     ``DW_FORM_ref_addr``\ *.*
@@ -1870,8 +2174,10 @@ implicit storage value starting at the bit offset.
     It pushes a location description L that comprises one implicit location
     description SL on the stack. SL specifies LS with a bit offset of 0.
 
-    If a ``DW_OP_deref*`` operation pops a location description L', and
-    retrieves S bits where both:
+    It is an evaluation error if a ``DW_OP_deref*`` operation pops a location
+    description L', and retrieves S bits, such that any retrieved bits come from
+    an implicit location storage that is the same as LS, unless both the
+    following conditions are met:
 
     1.  All retrieved bits come from an implicit location description that
         refers to an implicit location storage that is the same as LS.
@@ -1884,26 +2190,29 @@ implicit storage value starting at the bit offset.
 
     *These rules are equivalent to retrieving the complete contents of LS.*
 
-    Then the value V pushed by the ``DW_OP_deref*`` operation is an implicit
-    pointer value IPV with a target architecture specific address space of AS, a
-    debugging information entry of D, and a base type of T. If AS is the target
-    architecture default address space, then T is the generic type. Otherwise, T
-    is a target architecture specific integral type with a bit size equal to S.
-
-    Otherwise, if a ``DW_OP_deref*`` operation is applied to a location
-    description such that some retrieved bits come from an implicit location
-    storage that is the same as LS, then the DWARF expression is ill-formed.
+    If both the above conditions are met, then the value V pushed by the
+    ``DW_OP_deref*`` operation is an implicit pointer value IPV with a target
+    architecture specific address space of AS, a debugging information entry of
+    D, and a base type of T. If AS is the target architecture default address
+    space, then T is the generic type. Otherwise, T is a target architecture
+    specific integral type with a bit size equal to S.
 
     If IPV is either implicitly converted to a location description (only done
     if AS is the target architecture default address space) or used by
-    ``DW_OP_LLVM_form_aspace_address`` (only done if the address space specified
-    is AS), then the resulting location description RL is:
+    ``DW_OP_LLVM_form_aspace_address`` (only done if the address space popped by
+    ``DW_OP_LLVM_form_aspace_address`` is AS), then the resulting location
+    description RL is:
 
     * If D has a ``DW_AT_location`` attribute, the DWARF expression E from the
-      ``DW_AT_location`` attribute is evaluated as a location description. The
-      current subprogram and current program location of the evaluation context
-      that is accessing IPV is used for the evaluation context of E, together
-      with an empty initial stack. RL is the expression result.
+      ``DW_AT_location`` attribute is evaluated with the current context, except
+      that the result kind is a location description, the compilation unit is
+      the one that contains D, the object is unspecified, and the initial stack
+      is empty. RL is the expression result.
+
+      *Note that E is evaluated with the context of the expression accessing
+      IPV, and not the context of the expression that contained the*
+      ``DW_OP_implicit_pointer`` *or* ``DW_OP_LLVM_aspace_implicit_pointer``
+      *operation that created L.*
 
     * If D has a ``DW_AT_const_value`` attribute, then an implicit location
       storage RLS is created from the ``DW_AT_const_value`` attribute's value
@@ -1917,7 +2226,7 @@ implicit storage value starting at the bit offset.
         deprecated and instead ``DW_AT_location`` is used with an implicit
         location description, then this rule would not be required.
 
-    * Otherwise the DWARF expression is ill-formed.
+    * Otherwise, it is an evaluation error.
 
     The bit offset of RL is updated as if the ``DW_OP_LLVM_offset_uconst B``
     operation was applied.
@@ -1925,8 +2234,7 @@ implicit storage value starting at the bit offset.
     If a ``DW_OP_stack_value`` operation pops a value that is the same as IPV,
     then it pushes a location description that is the same as L.
 
-    The DWARF expression is ill-formed if it accesses LS or IPV in any other
-    manner.
+    It is an evaluation error if LS or IPV is accessed in any other manner.
 
     *The restrictions on how an implicit pointer location description created
     by* ``DW_OP_implicit_pointer`` *and* ``DW_OP_LLVM_aspace_implicit_pointer``
@@ -1942,12 +2250,18 @@ implicit storage value starting at the bit offset.
     a target architecture specific address space identifier AS.
 
     The location description L that is pushed on the stack is the same as for
-    ``DW_OP_implicit_pointer`` except that the address space identifier used is
+    ``DW_OP_implicit_pointer``, except that the address space identifier used is
     AS.
 
     The DWARF expression is ill-formed if AS is not one of the values defined by
     the target architecture specific ``DW_ASPACE_*`` values.
 
+    .. note::
+
+      This definition of ``DW_OP_LLVM_aspace_implicit_pointer`` may change when
+      full support for address classes is added as required for languages such
+      as OpenCL/SyCL.
+
 *Typically a* ``DW_OP_implicit_pointer`` *or*
 ``DW_OP_LLVM_aspace_implicit_pointer`` *operation is used in a DWARF expression
 E*\ :sub:`1` *of a* ``DW_TAG_variable`` *or* ``DW_TAG_formal_parameter``
@@ -2111,7 +2425,7 @@ compatible with the definitions in DWARF Version 5.*
     integer that represents the part bit size S. The second is an unsigned
     LEB128 integer that represents a bit displacement B.
 
-    The action is the same as for ``DW_OP_piece`` except that any part created
+    The action is the same as for ``DW_OP_piece``, except that any part created
     has the bit size S, and the location description PL of any created part is
     updated as if the ``DW_OP_constu B; DW_OP_LLVM_bit_offset`` operations were
     applied.
@@ -2252,17 +2566,31 @@ defined, and there is no following default location description entry, it is
 assumed that the object is not available for the portion of the range that is
 not covered.
 
-The operation expression of each matching location list entry is evaluated as a
-location description and its result is returned as the result of the location
-list entry. The operation expression is evaluated with the same context as the
-location list expression, including the same current frame, current program
-location, and initial stack.
+The result of the evaluation of a DWARF location list expression is:
+
+* If the current program location is not specified, then it is an evaluation
+  error.
+
+  .. note::
+
+    If the location list only has a single default entry, should that be
+    considered a match if there is no program location? If there are non-default
+    entries then it seems it has to be an evaluation error when there is no
+    program location as that indicates the location depends on the program
+    location which is not known.
 
-The result of the evaluation of a DWARF location list expression is a location
-description that is comprised of the union of the single location descriptions
-of the location description result of each matching location list entry. If
-there are no matching location list entries, then the result is a location
-description that comprises one undefined location description.
+* If there are no matching location list entries, then the result is a location
+  description that comprises one undefined location description.
+
+* Otherwise, the operation expression E of each matching location list entry is
+  evaluated with the current context, except that the result kind is a location
+  description, the object is unspecified, and the initial stack is empty. The
+  location list entry result is the location description returned by the
+  evaluation of E.
+
+  The result is a location description that is comprised of the union of the
+  single location descriptions of the location description result of each
+  matching location list entry.
 
 A location list expression can only be used as the value of a debugger
 information entry attribute that is encoded using class ``loclist`` or
@@ -2471,6 +2799,26 @@ DWARF address space identifiers are used by:
   ``DW_AT_address_space`` would be target architecture specific and the same as
   used in ``DW_OP_xderef*``.
 
+.. note::
+
+  Some additional changes will be made to support languages such as OpenCL/SyCL
+  that allow address class pointer casting and queries.
+
+  This requires the compiler to provide the mapping from address space to
+  address class which may be runtime and not target architecture dependent. Some
+  implementations may have a one-to-one mapping from source language address
+  class to target architecture address space, and some may have a many-to-one
+  mapping which requires knowledge of the address class when determining if
+  pointer address class casts are allowed.
+
+  The changes will likely add an attribute that has an expression provided by
+  the compiler to map from address class to address space. The
+  ``DW_OP_implicit_pointer`` and ``DW_OP_LLVM_aspace_implicit_pointer``
+  operations may be changed as the current IPV definition may not provide enough
+  information when used to cast between address classes. Other attributes and
+  operations may be needed. The legal casts between address classes may need to
+  be defined on a per language address class basis.
+
 .. _amdgpu-dwarf-debugging-information-entry-attributes:
 
 Debugging Information Entry Attributes
@@ -2488,10 +2836,12 @@ Debugging Information Entry Attributes
     variables and parameters) or common blocks may have a ``DW_AT_location``
     attribute, whose value is a DWARF expression E.
 
-    The result of the attribute is obtained by evaluating E as a location
-    description in the context of the current subprogram, current program
-    location, and with an empty initial stack. See
-    :ref:`amdgpu-dwarf-expressions`.
+    The result of the attribute is obtained by evaluating E with a context that
+    has a result kind of a location description, an unspecified object, the
+    compilation unit that contains E, an empty initial stack, and other context
+    elements corresponding to the source language thread of execution upon which
+    the user is focused, if any. The result of the evaluation is the location
+    description of the base of the data object.
 
     See :ref:`amdgpu-dwarf-control-flow-operations` for special evaluation rules
     used by the ``DW_OP_call*`` operations.
@@ -2535,9 +2885,11 @@ Debugging Information Entry Attributes
     may have a ``DW_AT_frame_base`` attribute, whose value is a DWARF expression
     E.
 
-    The result of the attribute is obtained by evaluating E as a location
-    description in the context of the current subprogram, current program
-    location, and with an empty initial stack.
+    The result of the attribute is obtained by evaluating E with a context that
+    has a result kind of a location description, an unspecified object, the
+    compilation unit that contains E, an empty initial stack, and other context
+    elements corresponding to the source language thread of execution upon which
+    the user is focused, if any.
 
     The DWARF is ill-formed if E contains an ``DW_OP_fbreg`` operation, or the
     resulting location description L is not comprised of one single location
@@ -2584,10 +2936,13 @@ Debugging Information Entry Attributes
         byte.*
 
     2.  Otherwise, the attribute must be a DWARF expression E which is evaluated
-        with a context of the current frame, current program location, and an
-        initial stack comprising the location description of the beginning of
-        the containing entity. The result of the evaluation is the location
-        description of the base of the member entry.
+        with a context that has a result kind of a location description, an
+        unspecified object, the compilation unit that contains E, an initial
+        stack comprising the location description of the beginning of the
+        containing entity, and other context elements corresponding to the
+        source language thread of execution upon which the user is focused, if
+        any. The result of the evaluation is the location description of the
+        base of the member entry.
 
     .. note::
 
@@ -2613,12 +2968,16 @@ Debugging Information Entry Attributes
     location description for a particular object of the given pointer to member
     type and for a particular structure or class instance.
 
-    The result of the attribute is obtained by evaluating E as a location
-    description with the context of the current subprogram, current program
-    location, and an initial stack comprising two entries. The first entry is
-    the value of the pointer to member object itself. The second entry is the
-    location description of the base of the entire class, structure, or union
-    instance containing the member whose location is being calculated.
+    The result of the attribute is obtained by evaluating E with a context that
+    has a result kind of a location description, an unspecified object, the
+    compilation unit that contains E, an initial stack comprising two entries,
+    and other context elements corresponding to the source language thread of
+    execution upon which the user is focused, if any. The first stack entry is
+    the value of the pointer to member object itself. The second stack entry is
+    the location description of the base of the entire class, structure, or
+    union instance containing the member whose location is being calculated. The
+    result of the evaluation is the location description of the member of the
+    class to which the pointer to member entry points.
 
 6.  ``DW_AT_data_location``
 
@@ -2629,9 +2988,13 @@ Debugging Information Entry Attributes
     attribute is omitted, the location description of the data is the same as
     the location description of the object.
 
-    The result of the attribute is obtained by evaluating E as a location
-    description with the context of the current subprogram, current program
-    location, and an empty initial stack.
+    The result of the attribute is obtained by evaluating E with a context that
+    has a result kind of a location description, an object that is the location
+    description of the data descriptor, the compilation unit that contains E, an
+    empty initial stack, and other context elements corresponding to the source
+    language thread of execution upon which the user is focused, if any. The
+    result of the evaluation is the location description of the base of the
+    member entry.
 
     *E will typically involve an operation expression that begins with a*
     ``DW_OP_push_object_address`` *operation which loads the location
@@ -2657,13 +3020,14 @@ Debugging Information Entry Attributes
     An entry for a virtual function also has a ``DW_AT_vtable_elem_location``
     attribute whose value is a DWARF expression E.
 
-    The result of the attribute is obtained by evaluating E as a location
-    description with the context of the current subprogram, current program
-    location, and an initial stack comprising the location description of the
-    object of the enclosing type.
-
-    The resulting location description is the slot for the function within the
-    virtual function table for the enclosing class.
+    The result of the attribute is obtained by evaluating E with a context that
+    has a result kind of a location description, an unspecified object, the
+    compilation unit that contains E, an initial stack comprising the location
+    description of the object of the enclosing type, and other context elements
+    corresponding to the source language thread of execution upon which the user
+    is focused, if any. The result of the evaluation is the location description
+    of the slot for the function within the virtual function table for the
+    enclosing class.
 
 8.  ``DW_AT_static_link``
 
@@ -2671,17 +3035,18 @@ Debugging Information Entry Attributes
     entry is lexically nested, it may have a ``DW_AT_static_link`` attribute,
     whose value is a DWARF expression E.
 
-    The result of the attribute is obtained by evaluating E as a location
-    description with the context of the current subprogram, current program
-    location, and an empty initial stack.
+    The result of the attribute is obtained by evaluating E with a context that
+    has a result kind of a location description, an unspecified object, the
+    compilation unit that contains E, an empty initial stack, and other context
+    elements corresponding to the source language thread of execution upon which
+    the user is focused, if any. The result of the evaluation is the location
+    description L of the *canonical frame address* (see
+    :ref:`amdgpu-dwarf-call-frame-information`) of the relevant call frame of
+    the subprogram instance that immediately lexically encloses the current call
+    frame's subprogram or entry point.
 
-    The DWARF is ill-formed if the resulting location description L is is not
-    comprised of one memory location description in any of the target
-    architecture specific address spaces.
-
-    The resulting L is the *frame base* of the relevant instance of the
-    subprogram that immediately lexically encloses the subprogram or entry
-    point.
+    The DWARF is ill-formed if L is is not comprised of one memory location
+    description for one of the target architecture specific address spaces.
 
 9.  ``DW_AT_return_addr``
 
@@ -2689,45 +3054,39 @@ Debugging Information Entry Attributes
     ``DW_TAG_entry_point`` debugger information entry may have a
     ``DW_AT_return_addr`` attribute, whose value is a DWARF expression E.
 
-    The result of the attribute is obtained by evaluating E as a location
-    description with the context of the current subprogram, current program
-    location, and an empty initial stack.
-
-    The DWARF is ill-formed if the resulting location description L is not
-    comprised one memory location description in any of the target architecture
-    specific address spaces.
+    The result of the attribute is obtained by evaluating E with a context that
+    has a result kind of a location description, an unspecified object, the
+    compilation unit that contains E, an empty initial stack, and other context
+    elements corresponding to the source language thread of execution upon which
+    the user is focused, if any. The result of the evaluation is the location
+    description L of the place where the return address for the current call
+    frame's subprogram or entry point is stored.
 
-    The resulting L is the place where the return address for the subprogram or
-    entry point is stored.
+    The DWARF is ill-formed if L is not comprised of one memory location
+    description for one of the target architecture specific address spaces.
 
     .. note::
 
       It is unclear why ``DW_TAG_inlined_subroutine`` has a
       ``DW_AT_return_addr`` attribute but not a ``DW_AT_frame_base`` or
       ``DW_AT_static_link`` attribute. Seems it would either have all of them or
-      none. Since inlined subprograms do not have a frame it seems they would
-      have none of these attributes.
+      none. Since inlined subprograms do not have a call frame it seems they
+      would have none of these attributes.
 
-10. ``DW_AT_call_value``, ``DW_AT_call_data_location``, and ``DW_AT_call_data_value``
+10. ``DW_AT_call_value``, ``DW_AT_call_data_location``, and
+    ``DW_AT_call_data_value``
 
     A ``DW_TAG_call_site_parameter`` debugger information entry may have a
     ``DW_AT_call_value`` attribute, whose value is a DWARF operation expression
     E\ :sub:`1`\ .
 
     The result of the ``DW_AT_call_value`` attribute is obtained by evaluating
-    E\ :sub:`1` as a value with the context of the call site subprogram, call
-    site program location, and an empty initial stack.
-
-    The call site subprogram is the subprogram containing the
-    ``DW_TAG_call_site_parameter`` debugger information entry. The call site
-    program location is the location of call site in the call site subprogram.
-
-    *The consumer may have to virtually unwind to the call site in order to
-    evaluate the attribute. This will provide both the call site subprogram and
-    call site program location needed to evaluate the expression.*
-
-    The resulting value V\ :sub:`1` is the value of the parameter at the time of
-    the call made by the call site.
+    E\ :sub:`1` with a context that has a result kind of a value, an unspecified
+    object, the compilation unit that contains E, an empty initial stack, and
+    other context elements corresponding to the source language thread of
+    execution upon which the user is focused, if any. The resulting value V\
+    :sub:`1` is the value of the parameter at the time of the call made by the
+    call site.
 
     For parameters passed by reference, where the code passes a pointer to a
     location which contains the parameter, or for reference type parameters, the
@@ -2737,21 +3096,34 @@ Debugging Information Entry Attributes
     value is a DWARF operation expression E\ :sub:`3`\ .
 
     The value of the ``DW_AT_call_data_location`` attribute is obtained by
-    evaluating E\ :sub:`2` as a location description with the context of the
-    call site subprogram, call site program location, and an empty initial
-    stack.
-
-    The resulting location description L\ :sub:`2` is the location where the
+    evaluating E\ :sub:`2` with a context that has a result kind of a location
+    description, an unspecified object, the compilation unit that contains E, an
+    empty initial stack, and other context elements corresponding to the source
+    language thread of execution upon which the user is focused, if any. The
+    resulting location description L\ :sub:`2` is the location where the
     referenced parameter lives during the call made by the call site. If E\
     :sub:`2` would just be a ``DW_OP_push_object_address``, then the
     ``DW_AT_call_data_location`` attribute may be omitted.
 
     The value of the ``DW_AT_call_data_value`` attribute is obtained by
-    evaluating E\ :sub:`3` as a value with the context of the call site
-    subprogram, call site program location, and an empty initial stack.
-
-    The resulting value V\ :sub:`3` is the value in L\ :sub:`2` at the time of
-    the call made by the call site.
+    evaluating E\ :sub:`3` with a context that has a result kind of a value, an
+    unspecified object, the compilation unit that contains E, an empty initial
+    stack, and other context elements corresponding to the source language
+    thread of execution upon which the user is focused, if any. The resulting
+    value V\ :sub:`3` is the value in L\ :sub:`2` at the time of the call made
+    by the call site.
+
+    The result of these attributes is undefined if the current call frame is
+    not for the subprogram containing the ``DW_TAG_call_site_parameter``
+    debugger information entry or the current program location is not for the
+    call site containing the ``DW_TAG_call_site_parameter`` debugger information
+    entry in the current call frame.
+
+    *The consumer may have to virtually unwind to the call site (see*
+    :ref:`amdgpu-dwarf-call-frame-information`\ *) in order to evaluate these
+    attributes. This will ensure the source language thread of execution upon
+    which the user is focused corresponds to the call site needed to evaluate
+    the expression.*
 
     If it is not possible to avoid the expressions of these attributes from
     accessing registers or memory locations that might be clobbered by the
@@ -2787,9 +3159,11 @@ Debugging Information Entry Attributes
     ``DW_TAG_entry_point`` debugging information entry may have a
     ``DW_AT_LLVM_lane_pc`` attribute whose value is a DWARF expression E.
 
-    The result of the attribute is obtained by evaluating E as a location
-    description with the context of the current subprogram, current program
-    location, and an empty initial stack.
+    The result of the attribute is obtained by evaluating E with a context that
+    has a result kind of a location description, an unspecified object, the
+    compilation unit that contains E, an empty initial stack, and other context
+    elements corresponding to the source language thread of execution upon which
+    the user is focused, if any.
 
     The resulting location description L is for a thread lane count sized vector
     of generic type elements. The thread lane count is the value of the
@@ -2817,9 +3191,11 @@ Debugging Information Entry Attributes
     ``DW_TAG_entry_point`` debugger information entry may have a
     ``DW_AT_LLVM_active_lane`` attribute whose value is a DWARF expression E.
 
-    The result of the attribute is obtained by evaluating E as a value with the
-    context of the current subprogram, current program location, and an empty
-    initial stack.
+    The result of the attribute is obtained by evaluating E with a context that
+    has a result kind of a value, an unspecified object, the compilation unit
+    that contains E, an empty initial stack, and other context elements
+    corresponding to the source language thread of execution upon which the user
+    is focused, if any.
 
     The DWARF is ill-formed if the resulting value V is not an integral value.
 
@@ -3042,7 +3418,7 @@ Call Frame Information
 
 .. note::
 
-  This section provides changes to existing Call Frame Information and defines
+  This section provides changes to existing call frame information and defines
   instructions added by the proposal. Additional support is added for address
   spaces. Register unwind DWARF expressions are generalized to allow any
   location description, including those with composite and implicit location
@@ -3050,6 +3426,8 @@ Call Frame Information
 
   These changes would be incorporated into the DWARF Version 5 section 6.1.
 
+.. _amdgpu-dwarf-structure_of-call-frame-information:
+
 Structure of Call Frame Information
 +++++++++++++++++++++++++++++++++++
 
@@ -3057,24 +3435,43 @@ The register rules are:
 
 *undefined*
   A register that has this rule has no recoverable value in the previous frame.
-  (By convention, it is not preserved by a callee.)
+  The previous value of this register is the undefined location description (see
+  :ref:`amdgpu-dwarf-undefined-location-description-operations`).
+
+  *By convention, the register is not preserved by a callee.*
 
 *same value*
-  This register has not been modified from the previous frame. (By convention,
-  it is preserved by the callee, but the callee has not modified it.)
+  This register has not been modified from the previous caller frame.
+
+  If the current frame is the top frame, then the previous value of this
+  register is the location description L that specifies one register location
+  description SL. SL specifies the register location storage that corresponds to
+  the register with a bit offset of 0 for the current thread.
+
+  If the current frame is not the top frame, then the previous value of this
+  register is the location description obtained using the call frame information
+  for the callee frame and callee program location invoked by the current caller
+  frame for the same register.
+
+  *By convention, the register is preserved by the callee, but the callee has
+  not modified it.*
 
 *offset(N)*
   N is a signed byte offset. The previous value of this register is saved at the
   location description computed as if the DWARF operation expression
-  ``DW_OP_LLVM_offset N`` is evaluated as a location description with an initial
-  stack comprising the location description of the current CFA (see
+  ``DW_OP_LLVM_offset N`` is evaluated with the current context, except the
+  result kind is a location description, the compilation unit is unspecified,
+  the object is unspecified, and an initial stack comprising the location
+  description of the current CFA (see
   :ref:`amdgpu-dwarf-operation-expressions`).
 
 *val_offset(N)*
   N is a signed byte offset. The previous value of this register is the memory
   byte address of the location description computed as if the DWARF operation
-  expression ``DW_OP_LLVM_offset N`` is evaluated as a location description with
-  an initial stack comprising the location description of the current CFA (see
+  expression ``DW_OP_LLVM_offset N`` is evaluated with the current context,
+  except the result kind is a location description, the compilation unit is
+  unspecified, the object is unspecified, and an initial stack comprising the
+  location description of the current CFA (see
   :ref:`amdgpu-dwarf-operation-expressions`).
 
   The DWARF is ill-formed if the CFA location description is not a memory byte
@@ -3103,26 +3500,40 @@ The register rules are:
     test).
 
 *register(R)*
-  The previous value of this register is stored in another register numbered R.
+  This register has been stored in another register numbered R.
 
-  The DWARF is ill-formed if the register sizes do not match.
+  The previous value of this register is the location description obtained using
+  the call frame information for the current frame and current program location
+  for register R.
+
+  The DWARF is ill-formed if the size of this register does not match the size
+  of register R or if there is a cyclic dependency in the call frame
+  information.
+
+  .. note::
+
+    Should this also allow R to be larger than this register? If so is the value
+    stored in the low order bits and it is undefined what is stored in the
+    extra upper bits?
 
 *expression(E)*
   The previous value of this register is located at the location description
   produced by evaluating the DWARF operation expression E (see
   :ref:`amdgpu-dwarf-operation-expressions`).
 
-  E is evaluated as a location description in the context of the current
-  subprogram, current program location, and with an initial stack comprising the
-  location description of the current CFA.
+  E is evaluated with the current context, except the result kind is a location
+  description, the compilation unit is unspecified, the object is unspecified,
+  and an initial stack comprising the location description of the current CFA
+  (see :ref:`amdgpu-dwarf-operation-expressions`).
 
 *val_expression(E)*
   The previous value of this register is the value produced by evaluating the
   DWARF operation expression E (see :ref:`amdgpu-dwarf-operation-expressions`).
 
-  E is evaluated as a value in the context of the current subprogram, current
-  program location, and with an initial stack comprising the location
-  description of the current CFA.
+  E is evaluated with the current context, except the result kind is a value,
+  the compilation unit is unspecified, the object is unspecified, and an initial
+  stack comprising the location description of the current CFA (see
+  :ref:`amdgpu-dwarf-operation-expressions`).
 
   The DWARF is ill-formed if the resulting value type size does not match the
   register size.
@@ -3139,8 +3550,8 @@ The register rules are:
 *architectural*
   The rule is defined externally to this specification by the augmenter.
 
-A Common Information Entry holds information that is shared among many Frame
-Description Entries. There is at least one CIE in every non-empty
+A Common Information Entry (CIE) holds information that is shared among many
+Frame Description Entries (FDE). There is at least one CIE in every non-empty
 ``.debug_frame`` section. A CIE contains the following fields, in order:
 
 1.  ``length`` (initial length)
@@ -3236,6 +3647,10 @@ Description Entries. There is at least one CIE in every non-empty
     represents the return address of the subprogram. Note that this column might
     not correspond to an actual machine register.
 
+    The value of the return address register is used to determine the program
+    location of the caller frame. The program location of the top frame is the
+    target architecture program counter value of the current thread.
+
 10. ``initial_instructions`` (array of ubyte)
 
     A sequence of rules that are interpreted to create the initial setting of
@@ -3356,7 +3771,7 @@ CFA Definition Instructions
     result of evaluating the DWARF operation expression ``DW_OP_constu AS;
     DW_OP_aspace_bregx R, B*data_alignment_factor`` as a location description.
 
-    *The action is the same as* ``DW_CFA_def_cfa`` *except that the second
+    *The action is the same as* ``DW_CFA_def_cfa``\ *, except that the second
     operand is signed and factored.*
 
 3.  ``DW_CFA_def_aspace_cfa`` *New*
@@ -3384,7 +3799,7 @@ CFA Definition Instructions
     If AS is not one of the values defined by the target architecture specific
     ``DW_ASPACE_*`` values, then the DWARF expression is ill-formed.
 
-    *The action is the same as* ``DW_CFA_aspace_def_cfa`` *except that the
+    *The action is the same as* ``DW_CFA_aspace_def_cfa``\ *, except that the
     second operand is signed and factored.*
 
 5.  ``DW_CFA_def_cfa_register``
@@ -3423,7 +3838,7 @@ CFA Definition Instructions
     If the subprogram has no current CFA rule, or the rule was defined by a
     ``DW_CFA_def_cfa_expression`` instruction, then the DWARF is ill-formed.
 
-    *The action is the same as* ``DW_CFA_def_cfa_offset`` *except that the
+    *The action is the same as* ``DW_CFA_def_cfa_offset``\ *, except that the
     operand is signed and factored.*
 
 8.  ``DW_CFA_def_cfa_expression``
@@ -3431,8 +3846,9 @@ CFA Definition Instructions
     The ``DW_CFA_def_cfa_expression`` instruction takes a single operand encoded
     as a ``DW_FORM_exprloc`` value representing a DWARF operation expression E.
     The required action is to define the current CFA rule to be the result of
-    evaluating E as a location description in the context of the current
-    subprogram, current program location, and an empty initial stack.
+    evaluating E with the current context, except the result kind is a location
+    description, the compilation unit is unspecified, the object is unspecified,
+    and an empty initial stack.
 
     *See* :ref:`amdgpu-dwarf-call-frame-instructions` *regarding restrictions on
     the DWARF expression operations that can be used in E.*
@@ -3473,7 +3889,7 @@ Register Rule Instructions
 
     The ``DW_CFA_offset_extended`` instruction takes two unsigned LEB128
     operands representing a register number R and a factored displacement B.
-    This instruction is identical to ``DW_CFA_offset`` except for the encoding
+    This instruction is identical to ``DW_CFA_offset``, except for the encoding
     and size of the register operand.
 
     .. note::
@@ -3486,7 +3902,7 @@ Register Rule Instructions
     The ``DW_CFA_offset_extended_sf`` instruction takes two operands: an
     unsigned LEB128 value representing a register number R and a signed LEB128
     factored displacement B. This instruction is identical to
-    ``DW_CFA_offset_extended`` except that B is signed.
+    ``DW_CFA_offset_extended``, except that B is signed.
 
 6.  ``DW_CFA_val_offset``
 
@@ -3510,7 +3926,7 @@ Register Rule Instructions
 
     The ``DW_CFA_val_offset_sf`` instruction takes two operands: an unsigned
     LEB128 value representing a register number R and a signed LEB128 factored
-    displacement B. This instruction is identical to ``DW_CFA_val_offset``
+    displacement B. This instruction is identical to ``DW_CFA_val_offset``,
     except that B is signed.
 
 8.  ``DW_CFA_register``
@@ -3560,7 +3976,7 @@ Register Rule Instructions
 
     The ``DW_CFA_restore_extended`` instruction takes a single unsigned LEB128
     operand that represents a register number R. This instruction is identical
-    to ``DW_CFA_restore`` except for the encoding and size of the register
+    to ``DW_CFA_restore``, except for the encoding and size of the register
     operand.
 
 Row State Instructions
@@ -3692,7 +4108,7 @@ operations.
    DW_OP_LLVM_undefined               0xe7     0
    DW_OP_LLVM_aspace_bregx            0xe8     2     ULEB128 register number,
                                                      ULEB128 byte displacement
-   DW_OP_LLVM_aspace_implicit_pointer 0xe9     2     4- or 8-byte offset of DIE,
+   DW_OP_LLVM_aspace_implicit_pointer 0xe9     2     4-byte or 8-byte offset of DIE,
                                                      SLEB128 byte displacement
    DW_OP_LLVM_piece_end               0xea     0
    DW_OP_LLVM_extend                  0xeb     2     ULEB128 bit size,
@@ -3830,6 +4246,13 @@ Examples
 The AMD GPU specific usage of the features in the proposal, including examples,
 is available at :ref:`amdgpu-dwarf-debug-information`.
 
+.. note::
+
+  Change examples to use ``DW_OP_LLVM_offset`` instead of ``DW_OP_add`` when
+  acting on a location description.
+
+  Need to provide examples of new features.
+
 .. _amdgpu-dwarf-references:
 
 References
diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst
index af764c0856000..f709bb8cf8e2f 100644
--- a/llvm/docs/AMDGPUUsage.rst
+++ b/llvm/docs/AMDGPUUsage.rst
@@ -1367,10 +1367,6 @@ The DWARF address space mapping used for AMDGPU is defined in
    *Reserved*                              0x04
    ``DW_ASPACE_AMDGPU_private_lane``       0x05  4       4        Private (Scratch) *focused lane*
    ``DW_ASPACE_AMDGPU_private_wave``       0x06  4       4        Private (Scratch) *unswizzled wavefront*
-   *Reserved*                              0x07-
-                                           0x1F
-   ``DW_ASPACE_AMDGPU_private_lane<0-63>`` 0x20- 4       4        Private (Scratch) *specific lane*
-                                           0x5F
    ======================================= ===== ======= ======== ================= =======================
 
 See :ref:`amdgpu-address-spaces` for information on the AMDGPU address spaces
@@ -1442,12 +1438,6 @@ address may have to be converted as the size of a
 ``DW_ASPACE_AMDGPU_private_lane`` address may be smaller than the size of a
 ``DW_ASPACE_AMDGPU_private_wave`` address.
 
-The ``DW_ASPACE_AMDGPU_private_lane<N>`` address space allows location
-expressions to specify the private address space corresponding to a specific
-lane N. For example, this can be used when the compiler spills scalar registers
-to scratch memory, with each scalar register being saved to a different lane's
-scratch memory.
-
 .. _amdgpu-dwarf-lane-identifier:
 
 Lane identifier

From a18953c1c05e520406783acbb5e6b06954f9150d Mon Sep 17 00:00:00 2001
From: Kang Zhang <shkzhang@cn.ibm.com>
Date: Thu, 30 Jul 2020 02:10:49 +0000
Subject: [PATCH 0570/1035] [PowerPC] Fix RM operands for some instructions

Summary:
Some instructions have set the wrong [RM] flag, this patch is to fix it.

Instructions x(v|s)r(d|s)pi[zmp]? and fri[npzm] use fixed rounding
directions without referencing current rounding mode.

Also, the SETRNDi, SETRND, BCLRn, MTFSFI, MTFSB0, MTFSB1, MTFSFb,
MTFSFI, MTFSFI_rec, MTFSF, MTFSF_rec should also fix the RM flag.

Reviewed By: jsji

Differential Revision: https://reviews.llvm.org/D81360
---
 llvm/lib/Target/PowerPC/PPCISelLowering.cpp   |  15 ++-
 llvm/lib/Target/PowerPC/PPCInstrInfo.td       |  63 ++++++-----
 llvm/lib/Target/PowerPC/PPCInstrVSX.td        | 105 +++++++++---------
 llvm/test/CodeGen/PowerPC/rounding-rm-flag.ll |  26 +++++
 llvm/test/CodeGen/PowerPC/setrnd.ll           |   6 +-
 5 files changed, 129 insertions(+), 86 deletions(-)
 create mode 100644 llvm/test/CodeGen/PowerPC/rounding-rm-flag.ll

diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index 7f93831f44f65..13dbcbe10d337 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -11931,8 +11931,13 @@ PPCTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
     BuildMI(*BB, MI, dl, TII->get(PPC::MFFS), MFFSReg);
 
     // Set rounding mode to round-to-zero.
-    BuildMI(*BB, MI, dl, TII->get(PPC::MTFSB1)).addImm(31);
-    BuildMI(*BB, MI, dl, TII->get(PPC::MTFSB0)).addImm(30);
+    BuildMI(*BB, MI, dl, TII->get(PPC::MTFSB1))
+        .addImm(31)
+        .addReg(PPC::RM, RegState::ImplicitDefine);
+
+    BuildMI(*BB, MI, dl, TII->get(PPC::MTFSB0))
+        .addImm(30)
+        .addReg(PPC::RM, RegState::ImplicitDefine);
 
     // Perform addition.
     BuildMI(*BB, MI, dl, TII->get(PPC::FADD), Dest).addReg(Src1).addReg(Src2);
@@ -11994,10 +11999,12 @@ PPCTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
     // the immediate to set the bits 62:63 of FPSCR.
     unsigned Mode = MI.getOperand(1).getImm();
     BuildMI(*BB, MI, dl, TII->get((Mode & 1) ? PPC::MTFSB1 : PPC::MTFSB0))
-      .addImm(31);
+        .addImm(31)
+        .addReg(PPC::RM, RegState::ImplicitDefine);
 
     BuildMI(*BB, MI, dl, TII->get((Mode & 2) ? PPC::MTFSB1 : PPC::MTFSB0))
-      .addImm(30);
+        .addImm(30)
+        .addReg(PPC::RM, RegState::ImplicitDefine);
   } else if (MI.getOpcode() == PPC::SETRND) {
     DebugLoc dl = MI.getDebugLoc();
 
diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.td b/llvm/lib/Target/PowerPC/PPCInstrInfo.td
index e4ec0f1e5ea13..be1e59d7fae71 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrInfo.td
+++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.td
@@ -1563,11 +1563,12 @@ let isBranch = 1, isTerminator = 1, hasCtrlDep = 1, PPC970_Unit = 7 in {
     def BCn : BForm_4<16, 4, 0, 0, (outs), (ins crbitrc:$bi, condbrtarget:$dst),
              "bc 4, $bi, $dst">;
 
-    let isReturn = 1, Uses = [LR, RM] in
+    let isReturn = 1, Uses = [LR, RM] in {
     def BCLR  : XLForm_2_br2<19, 16, 12, 0, (outs), (ins crbitrc:$bi),
                              "bclr 12, $bi, 0", IIC_BrB, []>;
     def BCLRn : XLForm_2_br2<19, 16, 4, 0, (outs), (ins crbitrc:$bi),
                              "bclr 4, $bi, 0", IIC_BrB, []>;
+    }
   }
 
   let isReturn = 1, Defs = [CTR], Uses = [CTR, LR, RM] in {
@@ -2584,22 +2585,7 @@ def FTDIV: XForm_17<63, 128, (outs crrc:$crD), (ins f8rc:$fA, f8rc:$fB),
 def FTSQRT: XForm_17a<63, 160, (outs crrc:$crD), (ins f8rc:$fB),
                       "ftsqrt $crD, $fB", IIC_FPCompare>;
 
-let Uses = [RM], mayRaiseFPException = 1 in {
-  let hasSideEffects = 0 in {
-  defm FCTIW  : XForm_26r<63, 14, (outs f8rc:$frD), (ins f8rc:$frB),
-                          "fctiw", "$frD, $frB", IIC_FPGeneral,
-                          []>;
-  defm FCTIWU  : XForm_26r<63, 142, (outs f8rc:$frD), (ins f8rc:$frB),
-                          "fctiwu", "$frD, $frB", IIC_FPGeneral,
-                          []>;
-  defm FCTIWZ : XForm_26r<63, 15, (outs f8rc:$frD), (ins f8rc:$frB),
-                          "fctiwz", "$frD, $frB", IIC_FPGeneral,
-                          [(set f64:$frD, (PPCfctiwz f64:$frB))]>;
-
-  defm FRSP   : XForm_26r<63, 12, (outs f4rc:$frD), (ins f8rc:$frB),
-                          "frsp", "$frD, $frB", IIC_FPGeneral,
-                          [(set f32:$frD, (any_fpround f64:$frB))]>;
-
+let mayRaiseFPException = 1, hasSideEffects = 0 in {
   let Interpretation64Bit = 1, isCodeGenOnly = 1 in
   defm FRIND  : XForm_26r<63, 392, (outs f8rc:$frD), (ins f8rc:$frB),
                           "frin", "$frD, $frB", IIC_FPGeneral,
@@ -2607,9 +2593,7 @@ let Uses = [RM], mayRaiseFPException = 1 in {
   defm FRINS  : XForm_26r<63, 392, (outs f4rc:$frD), (ins f4rc:$frB),
                           "frin", "$frD, $frB", IIC_FPGeneral,
                           [(set f32:$frD, (any_fround f32:$frB))]>;
-  }
 
-  let hasSideEffects = 0 in {
   let Interpretation64Bit = 1, isCodeGenOnly = 1 in
   defm FRIPD  : XForm_26r<63, 456, (outs f8rc:$frD), (ins f8rc:$frB),
                           "frip", "$frD, $frB", IIC_FPGeneral,
@@ -2631,6 +2615,22 @@ let Uses = [RM], mayRaiseFPException = 1 in {
   defm FRIMS  : XForm_26r<63, 488, (outs f4rc:$frD), (ins f4rc:$frB),
                           "frim", "$frD, $frB", IIC_FPGeneral,
                           [(set f32:$frD, (any_ffloor f32:$frB))]>;
+}
+
+let Uses = [RM], mayRaiseFPException = 1, hasSideEffects = 0 in {
+  defm FCTIW  : XForm_26r<63, 14, (outs f8rc:$frD), (ins f8rc:$frB),
+                          "fctiw", "$frD, $frB", IIC_FPGeneral,
+                          []>;
+  defm FCTIWU  : XForm_26r<63, 142, (outs f8rc:$frD), (ins f8rc:$frB),
+                          "fctiwu", "$frD, $frB", IIC_FPGeneral,
+                          []>;
+  defm FCTIWZ : XForm_26r<63, 15, (outs f8rc:$frD), (ins f8rc:$frB),
+                          "fctiwz", "$frD, $frB", IIC_FPGeneral,
+                          [(set f64:$frD, (PPCfctiwz f64:$frB))]>;
+
+  defm FRSP   : XForm_26r<63, 12, (outs f4rc:$frD), (ins f8rc:$frB),
+                          "frsp", "$frD, $frB", IIC_FPGeneral,
+                          [(set f32:$frD, (any_fpround f64:$frB))]>;
 
   defm FSQRT  : XForm_26r<63, 22, (outs f8rc:$frD), (ins f8rc:$frB),
                           "fsqrt", "$frD, $frB", IIC_FPSqrtD,
@@ -2638,8 +2638,7 @@ let Uses = [RM], mayRaiseFPException = 1 in {
   defm FSQRTS : XForm_26r<59, 22, (outs f4rc:$frD), (ins f4rc:$frB),
                           "fsqrts", "$frD, $frB", IIC_FPSqrtS,
                           [(set f32:$frD, (any_fsqrt f32:$frB))]>;
-  }
-  }
+}
 }
 
 /// Note that FMR is defined as pseudo-ops on the PPC970 because they are
@@ -2916,13 +2915,17 @@ let Uses = [RM] in {
 
 // The above pseudo gets expanded to make use of the following instructions
 // to manipulate FPSCR.  Note that FPSCR is not modeled at the DAG level.
-let Uses = [RM], Defs = [RM] in {
-  def MTFSB0 : XForm_43<63, 70, (outs), (ins u5imm:$FM),
-                        "mtfsb0 $FM", IIC_IntMTFSB0, []>,
-               PPC970_DGroup_Single, PPC970_Unit_FPU;
-  def MTFSB1 : XForm_43<63, 38, (outs), (ins u5imm:$FM),
-                        "mtfsb1 $FM", IIC_IntMTFSB0, []>,
-               PPC970_DGroup_Single, PPC970_Unit_FPU;
+
+// When FM is 30/31, we are setting the 62/63 bit of FPSCR, the implicit-def
+// RM should be set.
+def MTFSB0 : XForm_43<63, 70, (outs), (ins u5imm:$FM),
+                      "mtfsb0 $FM", IIC_IntMTFSB0, []>,
+             PPC970_DGroup_Single, PPC970_Unit_FPU;
+def MTFSB1 : XForm_43<63, 38, (outs), (ins u5imm:$FM),
+                      "mtfsb1 $FM", IIC_IntMTFSB0, []>,
+             PPC970_DGroup_Single, PPC970_Unit_FPU;
+
+let Defs = [RM] in {
   let isCodeGenOnly = 1 in
   def MTFSFb  : XFLForm<63, 711, (outs), (ins i32imm:$FM, f8rc:$rT),
                         "mtfsf $FM, $rT", IIC_IntMTFSB0, []>,
@@ -4318,6 +4321,8 @@ def MTMSRD : XForm_mtmsr<31, 178, (outs), (ins gprc:$RS, i32imm:$L),
 def MCRFS : XLForm_3<63, 64, (outs crrc:$BF), (ins crrc:$BFA),
                      "mcrfs $BF, $BFA", IIC_BrMCR>;
 
+// If W is 0 and BF is 7, the 60:63 bits will be set, we should set the
+// implicit-def RM.
 def MTFSFI : XLForm_4<63, 134, (outs crrc:$BF), (ins i32imm:$U, i32imm:$W),
                       "mtfsfi $BF, $U, $W", IIC_IntMFFS>;
 let Defs = [CR1] in
@@ -4328,6 +4333,7 @@ def : InstAlias<"mtfsfi $BF, $U", (MTFSFI crrc:$BF, i32imm:$U, 0)>;
 def : InstAlias<"mtfsfi. $BF, $U", (MTFSFI_rec crrc:$BF, i32imm:$U, 0)>;
 
 let Predicates = [HasFPU] in {
+let Defs = [RM] in {
 def MTFSF : XFLForm_1<63, 711, (outs),
                       (ins i32imm:$FLM, f8rc:$FRB, i32imm:$L, i32imm:$W),
                       "mtfsf $FLM, $FRB, $L, $W", IIC_IntMFFS, []>;
@@ -4335,6 +4341,7 @@ let Defs = [CR1] in
 def MTFSF_rec : XFLForm_1<63, 711, (outs),
                        (ins i32imm:$FLM, f8rc:$FRB, i32imm:$L, i32imm:$W),
                        "mtfsf. $FLM, $FRB, $L, $W", IIC_IntMFFS, []>, isRecordForm;
+}
 
 def : InstAlias<"mtfsf $FLM, $FRB", (MTFSF i32imm:$FLM, f8rc:$FRB, 0, 0)>;
 def : InstAlias<"mtfsf. $FLM, $FRB", (MTFSF_rec i32imm:$FLM, f8rc:$FRB, 0, 0)>;
diff --git a/llvm/lib/Target/PowerPC/PPCInstrVSX.td b/llvm/lib/Target/PowerPC/PPCInstrVSX.td
index 9ba5058a6f812..272f8b1c0aac1 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrVSX.td
+++ b/llvm/lib/Target/PowerPC/PPCInstrVSX.td
@@ -362,7 +362,8 @@ let hasSideEffects = 0 in {
     }
   } // mayStore
 
-  let Uses = [RM], mayRaiseFPException = 1 in {
+  let mayRaiseFPException = 1 in {
+  let Uses = [RM] in {
   // Add/Mul Instructions
   let isCommutable = 1 in {
     def XSADDDP : XX3Form<60, 32,
@@ -885,15 +886,61 @@ let hasSideEffects = 0 in {
                       "xvcvuxwsp $XT, $XB", IIC_VecFP,
                       [(set v4f32:$XT, (uint_to_fp v4i32:$XB))]>;
 
-  // Rounding Instructions
-  def XSRDPI : XX2Form<60, 73,
-                      (outs vsfrc:$XT), (ins vsfrc:$XB),
-                      "xsrdpi $XT, $XB", IIC_VecFP,
-                      [(set f64:$XT, (any_fround f64:$XB))]>;
+  // Rounding Instructions respecting current rounding mode
   def XSRDPIC : XX2Form<60, 107,
                       (outs vsfrc:$XT), (ins vsfrc:$XB),
                       "xsrdpic $XT, $XB", IIC_VecFP,
                       [(set f64:$XT, (any_fnearbyint f64:$XB))]>;
+  def XVRDPIC : XX2Form<60, 235,
+                      (outs vsrc:$XT), (ins vsrc:$XB),
+                      "xvrdpic $XT, $XB", IIC_VecFP,
+                      [(set v2f64:$XT, (any_fnearbyint v2f64:$XB))]>;
+  def XVRSPIC : XX2Form<60, 171,
+                      (outs vsrc:$XT), (ins vsrc:$XB),
+                      "xvrspic $XT, $XB", IIC_VecFP,
+                      [(set v4f32:$XT, (any_fnearbyint v4f32:$XB))]>;
+  // Max/Min Instructions
+  let isCommutable = 1 in {
+  def XSMAXDP : XX3Form<60, 160,
+                        (outs vsfrc:$XT), (ins vsfrc:$XA, vsfrc:$XB),
+                        "xsmaxdp $XT, $XA, $XB", IIC_VecFP,
+                        [(set vsfrc:$XT,
+                              (int_ppc_vsx_xsmaxdp vsfrc:$XA, vsfrc:$XB))]>;
+  def XSMINDP : XX3Form<60, 168,
+                        (outs vsfrc:$XT), (ins vsfrc:$XA, vsfrc:$XB),
+                        "xsmindp $XT, $XA, $XB", IIC_VecFP,
+                        [(set vsfrc:$XT,
+                              (int_ppc_vsx_xsmindp vsfrc:$XA, vsfrc:$XB))]>;
+
+  def XVMAXDP : XX3Form<60, 224,
+                        (outs vsrc:$XT), (ins vsrc:$XA, vsrc:$XB),
+                        "xvmaxdp $XT, $XA, $XB", IIC_VecFP,
+                        [(set vsrc:$XT,
+                              (int_ppc_vsx_xvmaxdp vsrc:$XA, vsrc:$XB))]>;
+  def XVMINDP : XX3Form<60, 232,
+                        (outs vsrc:$XT), (ins vsrc:$XA, vsrc:$XB),
+                        "xvmindp $XT, $XA, $XB", IIC_VecFP,
+                        [(set vsrc:$XT,
+                              (int_ppc_vsx_xvmindp vsrc:$XA, vsrc:$XB))]>;
+
+  def XVMAXSP : XX3Form<60, 192,
+                        (outs vsrc:$XT), (ins vsrc:$XA, vsrc:$XB),
+                        "xvmaxsp $XT, $XA, $XB", IIC_VecFP,
+                        [(set vsrc:$XT,
+                              (int_ppc_vsx_xvmaxsp vsrc:$XA, vsrc:$XB))]>;
+  def XVMINSP : XX3Form<60, 200,
+                        (outs vsrc:$XT), (ins vsrc:$XA, vsrc:$XB),
+                        "xvminsp $XT, $XA, $XB", IIC_VecFP,
+                        [(set vsrc:$XT,
+                              (int_ppc_vsx_xvminsp vsrc:$XA, vsrc:$XB))]>;
+  } // isCommutable
+  } // Uses = [RM]
+
+  // Rounding Instructions with static direction.
+  def XSRDPI : XX2Form<60, 73,
+                      (outs vsfrc:$XT), (ins vsfrc:$XB),
+                      "xsrdpi $XT, $XB", IIC_VecFP,
+                      [(set f64:$XT, (any_fround f64:$XB))]>;
   def XSRDPIM : XX2Form<60, 121,
                       (outs vsfrc:$XT), (ins vsfrc:$XB),
                       "xsrdpim $XT, $XB", IIC_VecFP,
@@ -911,10 +958,6 @@ let hasSideEffects = 0 in {
                       (outs vsrc:$XT), (ins vsrc:$XB),
                       "xvrdpi $XT, $XB", IIC_VecFP,
                       [(set v2f64:$XT, (any_fround v2f64:$XB))]>;
-  def XVRDPIC : XX2Form<60, 235,
-                      (outs vsrc:$XT), (ins vsrc:$XB),
-                      "xvrdpic $XT, $XB", IIC_VecFP,
-                      [(set v2f64:$XT, (any_fnearbyint v2f64:$XB))]>;
   def XVRDPIM : XX2Form<60, 249,
                       (outs vsrc:$XT), (ins vsrc:$XB),
                       "xvrdpim $XT, $XB", IIC_VecFP,
@@ -932,10 +975,6 @@ let hasSideEffects = 0 in {
                       (outs vsrc:$XT), (ins vsrc:$XB),
                       "xvrspi $XT, $XB", IIC_VecFP,
                       [(set v4f32:$XT, (any_fround v4f32:$XB))]>;
-  def XVRSPIC : XX2Form<60, 171,
-                      (outs vsrc:$XT), (ins vsrc:$XB),
-                      "xvrspic $XT, $XB", IIC_VecFP,
-                      [(set v4f32:$XT, (any_fnearbyint v4f32:$XB))]>;
   def XVRSPIM : XX2Form<60, 185,
                       (outs vsrc:$XT), (ins vsrc:$XB),
                       "xvrspim $XT, $XB", IIC_VecFP,
@@ -948,43 +987,7 @@ let hasSideEffects = 0 in {
                       (outs vsrc:$XT), (ins vsrc:$XB),
                       "xvrspiz $XT, $XB", IIC_VecFP,
                       [(set v4f32:$XT, (any_ftrunc v4f32:$XB))]>;
-
-  // Max/Min Instructions
-  let isCommutable = 1 in {
-  def XSMAXDP : XX3Form<60, 160,
-                        (outs vsfrc:$XT), (ins vsfrc:$XA, vsfrc:$XB),
-                        "xsmaxdp $XT, $XA, $XB", IIC_VecFP,
-                        [(set vsfrc:$XT,
-                              (int_ppc_vsx_xsmaxdp vsfrc:$XA, vsfrc:$XB))]>;
-  def XSMINDP : XX3Form<60, 168,
-                        (outs vsfrc:$XT), (ins vsfrc:$XA, vsfrc:$XB),
-                        "xsmindp $XT, $XA, $XB", IIC_VecFP,
-                        [(set vsfrc:$XT,
-                              (int_ppc_vsx_xsmindp vsfrc:$XA, vsfrc:$XB))]>;
-
-  def XVMAXDP : XX3Form<60, 224,
-                        (outs vsrc:$XT), (ins vsrc:$XA, vsrc:$XB),
-                        "xvmaxdp $XT, $XA, $XB", IIC_VecFP,
-                        [(set vsrc:$XT,
-                              (int_ppc_vsx_xvmaxdp vsrc:$XA, vsrc:$XB))]>;
-  def XVMINDP : XX3Form<60, 232,
-                        (outs vsrc:$XT), (ins vsrc:$XA, vsrc:$XB),
-                        "xvmindp $XT, $XA, $XB", IIC_VecFP,
-                        [(set vsrc:$XT,
-                              (int_ppc_vsx_xvmindp vsrc:$XA, vsrc:$XB))]>;
-
-  def XVMAXSP : XX3Form<60, 192,
-                        (outs vsrc:$XT), (ins vsrc:$XA, vsrc:$XB),
-                        "xvmaxsp $XT, $XA, $XB", IIC_VecFP,
-                        [(set vsrc:$XT,
-                              (int_ppc_vsx_xvmaxsp vsrc:$XA, vsrc:$XB))]>;
-  def XVMINSP : XX3Form<60, 200,
-                        (outs vsrc:$XT), (ins vsrc:$XA, vsrc:$XB),
-                        "xvminsp $XT, $XA, $XB", IIC_VecFP,
-                        [(set vsrc:$XT,
-                              (int_ppc_vsx_xvminsp vsrc:$XA, vsrc:$XB))]>;
-  } // isCommutable
-  } // Uses = [RM], mayRaiseFPException
+  } // mayRaiseFPException
 
   // Logical Instructions
   let isCommutable = 1 in
diff --git a/llvm/test/CodeGen/PowerPC/rounding-rm-flag.ll b/llvm/test/CodeGen/PowerPC/rounding-rm-flag.ll
new file mode 100644
index 0000000000000..0af781f6657b6
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/rounding-rm-flag.ll
@@ -0,0 +1,26 @@
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu \
+; RUN:     -mcpu=pwr9 -stop-after=early-ifcvt < %s | FileCheck %s
+
+define float @test_XSRDPI(float %f) {
+entry:
+  %0 = tail call float @llvm.round.f32(float %f)
+  ret float %0
+
+; CHECK-LABEL: name:            test_XSRDPI
+; CHECK-NOT:  %2:vsfrc = nofpexcept XSRDPI killed %1, implicit $rm
+; CHECK:      %2:vsfrc = nofpexcept XSRDPI killed %1
+}
+
+define double @test_XSRDPIM(double %d) {
+entry:
+  %0 = tail call double @llvm.floor.f64(double %d)
+  ret double %0
+
+; CHECK-LABEL: name:            test_XSRDPIM
+; CHECK-NOT:  %1:vsfrc = nofpexcept XSRDPIM %0, implicit $rm
+; CHECK:      %1:vsfrc = nofpexcept XSRDPIM %0
+}
+
+declare float @llvm.round.f32(float)
+declare double @llvm.floor.f64(double)
+
diff --git a/llvm/test/CodeGen/PowerPC/setrnd.ll b/llvm/test/CodeGen/PowerPC/setrnd.ll
index a732e3f73c992..9080a4b0ee51f 100644
--- a/llvm/test/CodeGen/PowerPC/setrnd.ll
+++ b/llvm/test/CodeGen/PowerPC/setrnd.ll
@@ -19,8 +19,8 @@ entry:
 
 ; AFTER-FINALIZE-ISEL:  test_setrndi
 ; AFTER-FINALIZE-ISEL:  MFFS implicit $rm
-; AFTER-FINALIZE-ISEL:  MTFSB0 31, implicit-def $rm, implicit $rm
-; AFTER-FINALIZE-ISEL:  MTFSB1 30, implicit-def $rm, implicit $rm
+; AFTER-FINALIZE-ISEL:  MTFSB0 31, implicit-def $rm
+; AFTER-FINALIZE-ISEL:  MTFSB1 30, implicit-def $rm
 
 ; CHECK-LABEL: @test_setrndi
 ; CHECK:      # %bb.0:
@@ -40,7 +40,7 @@ entry:
 
 ; AFTER-FINALIZE-ISEL: test_setrnd
 ; AFTER-FINALIZE-ISEL: MFFS implicit $rm
-; AFTER-FINALIZE-ISEL: MTFSF 255, %7, 0, 0
+; AFTER-FINALIZE-ISEL: MTFSF 255, %7, 0, 0, implicit-def $rm
 
 ; CHECK-LABEL: @test_setrnd
 ; CHECK:      # %bb.0:

From eee7cc95d65e29d2493eea41559935b33e39b57e Mon Sep 17 00:00:00 2001
From: Yuanfang Chen <yuanfang.chen@sony.com>
Date: Sat, 25 Jul 2020 12:24:37 -0700
Subject: [PATCH 0571/1035] [llvm-readobj] NFC. Add -help description of
 --hex-dump and --string-dump

---
 llvm/tools/llvm-readobj/llvm-readobj.cpp | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/llvm/tools/llvm-readobj/llvm-readobj.cpp b/llvm/tools/llvm-readobj/llvm-readobj.cpp
index 6b8a883af01df..d6fc610bc2ca4 100644
--- a/llvm/tools/llvm-readobj/llvm-readobj.cpp
+++ b/llvm/tools/llvm-readobj/llvm-readobj.cpp
@@ -183,14 +183,18 @@ namespace opts {
                           cl::aliasopt(ProgramHeaders));
 
   // --string-dump, -p
-  cl::list<std::string> StringDump("string-dump", cl::desc("<number|name>"),
-                                   cl::ZeroOrMore);
+  cl::list<std::string> StringDump(
+      "string-dump", cl::value_desc("number|name"),
+      cl::desc("Display the specified section(s) as a list of strings"),
+      cl::ZeroOrMore);
   cl::alias StringDumpShort("p", cl::desc("Alias for --string-dump"),
                             cl::aliasopt(StringDump), cl::Prefix);
 
   // --hex-dump, -x
-  cl::list<std::string> HexDump("hex-dump", cl::desc("<number|name>"),
-                                cl::ZeroOrMore);
+  cl::list<std::string>
+      HexDump("hex-dump", cl::value_desc("number|name"),
+              cl::desc("Display the specified section(s) as hexadecimal bytes"),
+              cl::ZeroOrMore);
   cl::alias HexDumpShort("x", cl::desc("Alias for --hex-dump"),
                          cl::aliasopt(HexDump), cl::Prefix);
 

From ec932939423bdea77c46cf79be4198266ea83e9f Mon Sep 17 00:00:00 2001
From: Yuanfang Chen <yuanfang.chen@sony.com>
Date: Tue, 28 Jul 2020 16:52:13 -0700
Subject: [PATCH 0572/1035] NFC. Fix comments

---
 llvm/include/llvm/IR/PassManager.h | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/llvm/include/llvm/IR/PassManager.h b/llvm/include/llvm/IR/PassManager.h
index c2f535037a56d..d3b98ed243b8e 100644
--- a/llvm/include/llvm/IR/PassManager.h
+++ b/llvm/include/llvm/IR/PassManager.h
@@ -651,7 +651,7 @@ template <typename IRUnitT, typename... ExtraArgTs> class AnalysisManager {
   /// when any of its embedded analysis results end up invalidated. We pass an
   /// \c Invalidator object as an argument to \c invalidate() in order to let
   /// the analysis results themselves define the dependency graph on the fly.
-  /// This lets us avoid building building an explicit representation of the
+  /// This lets us avoid building an explicit representation of the
   /// dependencies between analysis results.
   class Invalidator {
   public:
@@ -846,7 +846,7 @@ template <typename IRUnitT, typename... ExtraArgTs> class AnalysisManager {
     return true;
   }
 
-  /// Invalidate a specific analysis pass for an IR module.
+  /// Invalidate a specific analysis pass for an IR unit.
   ///
   /// Note that the analysis result can disregard invalidation, if it determines
   /// it is in fact still valid.
@@ -890,7 +890,7 @@ template <typename IRUnitT, typename... ExtraArgTs> class AnalysisManager {
     return RI == AnalysisResults.end() ? nullptr : &*RI->second->second;
   }
 
-  /// Invalidate a function pass result.
+  /// Invalidate a pass result for a IR unit.
   void invalidateImpl(AnalysisKey *ID, IRUnitT &IR) {
     typename AnalysisResultMapT::iterator RI =
         AnalysisResults.find({ID, &IR});
@@ -904,20 +904,20 @@ template <typename IRUnitT, typename... ExtraArgTs> class AnalysisManager {
     AnalysisResults.erase(RI);
   }
 
-  /// Map type from module analysis pass ID to pass concept pointer.
+  /// Map type from analysis pass ID to pass concept pointer.
   using AnalysisPassMapT =
       DenseMap<AnalysisKey *, std::unique_ptr<PassConceptT>>;
 
-  /// Collection of module analysis passes, indexed by ID.
+  /// Collection of analysis passes, indexed by ID.
   AnalysisPassMapT AnalysisPasses;
 
-  /// Map from function to a list of function analysis results.
+  /// Map from IR unit to a list of analysis results.
   ///
-  /// Provides linear time removal of all analysis results for a function and
+  /// Provides linear time removal of all analysis results for a IR unit and
   /// the ultimate storage for a particular cached analysis result.
   AnalysisResultListMapT AnalysisResultLists;
 
-  /// Map from an analysis ID and function to a particular cached
+  /// Map from an analysis ID and IR unit to a particular cached
   /// analysis result.
   AnalysisResultMapT AnalysisResults;
 

From 8224c5047e9cef2db4b0e31427cdf90a2568a341 Mon Sep 17 00:00:00 2001
From: Yuanfang Chen <yuanfang.chen@sony.com>
Date: Wed, 29 Jul 2020 18:34:32 -0700
Subject: [PATCH 0573/1035] For some tests targeting SystemZ, -march=z13 --->
 -mcpu=z13

z13 is not a target. It is a CPU.
---
 llvm/test/Analysis/MemorySSA/pr40509.ll                       | 2 +-
 .../Instrumentation/MemorySanitizer/SystemZ/vararg-kernel.ll  | 4 ++--
 llvm/test/Instrumentation/MemorySanitizer/SystemZ/vararg.ll   | 4 ++--
 llvm/test/Transforms/LICM/pr40317.ll                          | 2 +-
 llvm/test/Transforms/LoopSimplifyCFG/pr39783.ll               | 2 +-
 5 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/llvm/test/Analysis/MemorySSA/pr40509.ll b/llvm/test/Analysis/MemorySSA/pr40509.ll
index 4827acf1b802b..1dbb6cfba3f61 100644
--- a/llvm/test/Analysis/MemorySSA/pr40509.ll
+++ b/llvm/test/Analysis/MemorySSA/pr40509.ll
@@ -1,5 +1,5 @@
 ; REQUIRES: asserts
-; RUN: opt -mtriple=systemz-unknown -march=z13 -O3 -enable-mssa-loop-dependency -disable-output %s
+; RUN: opt -mtriple=systemz-unknown -mcpu=z13 -O3 -enable-mssa-loop-dependency -disable-output %s
 
 ; During transform to LCSSA, an access becomes obfuscated to:
 ; (2 = phi (phi(val), val)), which BasicAA fails to analyze.
diff --git a/llvm/test/Instrumentation/MemorySanitizer/SystemZ/vararg-kernel.ll b/llvm/test/Instrumentation/MemorySanitizer/SystemZ/vararg-kernel.ll
index 25ee925d68609..b538f9a26b156 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/SystemZ/vararg-kernel.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/SystemZ/vararg-kernel.ll
@@ -1,5 +1,5 @@
-; RUN: opt < %s -S -march=z13 -msan-kernel=1 -float-abi=soft -passes=msan 2>&1 | FileCheck %s
-; RUN: opt < %s -msan -S -march=z13 -msan-kernel=1 -float-abi=soft | FileCheck %s
+; RUN: opt < %s -S -mcpu=z13 -msan-kernel=1 -float-abi=soft -passes=msan 2>&1 | FileCheck %s
+; RUN: opt < %s -msan -S -mcpu=z13 -msan-kernel=1 -float-abi=soft | FileCheck %s
 
 target datalayout = "E-m:e-i1:8:16-i8:8:16-i64:64-f128:64-a:8:16-n32:64"
 target triple = "s390x-unknown-linux-gnu"
diff --git a/llvm/test/Instrumentation/MemorySanitizer/SystemZ/vararg.ll b/llvm/test/Instrumentation/MemorySanitizer/SystemZ/vararg.ll
index c2c9779fb79e9..be5c006c02c8d 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/SystemZ/vararg.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/SystemZ/vararg.ll
@@ -1,5 +1,5 @@
-; RUN: opt < %s -S -march=z13 -passes=msan 2>&1 | FileCheck %s
-; RUN: opt < %s -msan -S -march=z13 | FileCheck %s
+; RUN: opt < %s -S -mcpu=z13 -passes=msan 2>&1 | FileCheck %s
+; RUN: opt < %s -msan -S -mcpu=z13 | FileCheck %s
 
 target datalayout = "E-m:e-i1:8:16-i8:8:16-i64:64-f128:64-a:8:16-n32:64"
 target triple = "s390x-unknown-linux-gnu"
diff --git a/llvm/test/Transforms/LICM/pr40317.ll b/llvm/test/Transforms/LICM/pr40317.ll
index 88afbdc496caf..487cbb23eb0b1 100644
--- a/llvm/test/Transforms/LICM/pr40317.ll
+++ b/llvm/test/Transforms/LICM/pr40317.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -march=z13 -tbaa -licm -enable-mssa-loop-dependency -licm-control-flow-hoisting -verify-memoryssa < %s | FileCheck %s
+; RUN: opt -S -mcpu=z13 -tbaa -licm -enable-mssa-loop-dependency -licm-control-flow-hoisting -verify-memoryssa < %s | FileCheck %s
 
 target datalayout = "E-m:e-i1:8:16-i8:8:16-i64:64-f128:64-v128:64-a:8:16-n32:64"
 target triple = "s390x-ibm-linux"
diff --git a/llvm/test/Transforms/LoopSimplifyCFG/pr39783.ll b/llvm/test/Transforms/LoopSimplifyCFG/pr39783.ll
index 9ebcc2a66a935..570d57cde2dc9 100644
--- a/llvm/test/Transforms/LoopSimplifyCFG/pr39783.ll
+++ b/llvm/test/Transforms/LoopSimplifyCFG/pr39783.ll
@@ -1,5 +1,5 @@
 ; REQUIRES: asserts
-; RUN: opt -march=z13 -S -loop-simplifycfg -enable-mssa-loop-dependency -enable-loop-simplifycfg-term-folding -verify-memoryssa 2>&1 < %s | FileCheck %s
+; RUN: opt -mcpu=z13 -S -loop-simplifycfg -enable-mssa-loop-dependency -enable-loop-simplifycfg-term-folding -verify-memoryssa 2>&1 < %s | FileCheck %s
 target datalayout = "E-m:e-i1:8:16-i8:8:16-i64:64-f128:64-v128:64-a:8:16-n32:64"
 
 @global = external dso_local local_unnamed_addr global i8, align 2

From f71deb43abea588901c083d3e6d6d25371b486ae Mon Sep 17 00:00:00 2001
From: Amy Huang <akhuang@google.com>
Date: Wed, 29 Jul 2020 19:54:08 -0700
Subject: [PATCH 0574/1035] [DebugInfo] Fix to ctor homing to ignore classes
 with trivial ctors.

Previously ctor homing was omitting debug info for classes if they
have both trival and nontrivial constructors, but we should only omit debug
info if the class doesn't have any trivial constructors.

retained types list.

bug: https://bugs.llvm.org/show_bug.cgi?id=46537

Differential Revision: https://reviews.llvm.org/D84870
---
 clang/lib/CodeGen/CGDebugInfo.cpp                 | 13 ++++++++++---
 clang/test/CodeGenCXX/debug-info-limited-ctor.cpp |  7 +++++++
 2 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/clang/lib/CodeGen/CGDebugInfo.cpp b/clang/lib/CodeGen/CGDebugInfo.cpp
index 780e0c692c051..461f2eee965d2 100644
--- a/clang/lib/CodeGen/CGDebugInfo.cpp
+++ b/clang/lib/CodeGen/CGDebugInfo.cpp
@@ -2296,12 +2296,19 @@ static bool shouldOmitDefinition(codegenoptions::DebugInfoKind DebugKind,
   // In constructor debug mode, only emit debug info for a class when its
   // constructor is emitted. Skip this optimization if the class or any of
   // its methods are marked dllimport.
+  //
+  // This applies to classes that don't have any trivial constructors and have
+  // at least one constructor.
   if (DebugKind == codegenoptions::DebugInfoConstructor &&
       !CXXDecl->isLambda() && !CXXDecl->hasConstexprNonCopyMoveConstructor() &&
-      !isClassOrMethodDLLImport(CXXDecl))
+      !isClassOrMethodDLLImport(CXXDecl)) {
+    if (CXXDecl->ctors().empty())
+      return false;
     for (const auto *Ctor : CXXDecl->ctors())
-      if (Ctor->isUserProvided())
-        return true;
+      if (Ctor->isTrivial() && !Ctor->isCopyOrMoveConstructor())
+        return false;
+    return true;
+  }
 
   TemplateSpecializationKind Spec = TSK_Undeclared;
   if (const auto *SD = dyn_cast<ClassTemplateSpecializationDecl>(RD))
diff --git a/clang/test/CodeGenCXX/debug-info-limited-ctor.cpp b/clang/test/CodeGenCXX/debug-info-limited-ctor.cpp
index 25b4ebdb54a32..89dd2b16b75b0 100644
--- a/clang/test/CodeGenCXX/debug-info-limited-ctor.cpp
+++ b/clang/test/CodeGenCXX/debug-info-limited-ctor.cpp
@@ -24,3 +24,10 @@ D::D() {}
 struct E {
   constexpr E(){};
 } TestE;
+
+// CHECK-DAG: !DICompositeType(tag: DW_TAG_structure_type, name: "F"{{.*}}DIFlagTypePassByValue
+struct F {
+  F() = default;
+  F(int) {}
+  int i;
+} TestF;

From 9611ee5f40ea0b0878725dd06657e55363b4065f Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@intel.com>
Date: Wed, 29 Jul 2020 21:14:21 -0700
Subject: [PATCH 0575/1035] [X86] Teach the assembler parser to handle a '*'
 between segment register and base/index/displacement part of an address

A '*' after the segment is equivalent to a '*' before the segment register. To make the AsmMatcher table work we need to place the '*' token into the operand vector before the full memory operand. To accomplish this I've modified some portions of operand parsing to expose the operand vector to ParseATTOperand so that the token can be pushed to the vector after parsing the segment register and before creating the memory operand using that segment register.

Fixes PR46879

Reviewed By: MaskRay

Differential Revision: https://reviews.llvm.org/D84895
---
 .../lib/Target/X86/AsmParser/X86AsmParser.cpp | 69 ++++++++++++-------
 llvm/test/MC/X86/x86-32.s                     |  4 ++
 llvm/test/MC/X86/x86-64.s                     |  4 ++
 3 files changed, 52 insertions(+), 25 deletions(-)

diff --git a/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp b/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
index 0950cf9e5710d..445bb482055ce 100644
--- a/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
+++ b/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
@@ -906,8 +906,8 @@ class X86AsmParser : public MCTargetAsmParser {
                             std::unique_ptr<llvm::MCParsedAsmOperand> &&Dst);
   bool VerifyAndAdjustOperands(OperandVector &OrigOperands,
                                OperandVector &FinalOperands);
-  std::unique_ptr<X86Operand> ParseOperand();
-  std::unique_ptr<X86Operand> ParseATTOperand();
+  bool ParseOperand(OperandVector &Operands);
+  bool ParseATTOperand(OperandVector &Operands);
   std::unique_ptr<X86Operand> ParseIntelOperand();
   bool ParseIntelOffsetOperator(const MCExpr *&Val, StringRef &ID,
                                 InlineAsmIdentifierInfo &Info, SMLoc &End);
@@ -1531,10 +1531,16 @@ bool X86AsmParser::VerifyAndAdjustOperands(OperandVector &OrigOperands,
   return false;
 }
 
-std::unique_ptr<X86Operand> X86AsmParser::ParseOperand() {
-  if (isParsingIntelSyntax())
-    return ParseIntelOperand();
-  return ParseATTOperand();
+bool X86AsmParser::ParseOperand(OperandVector &Operands) {
+  if (isParsingIntelSyntax()) {
+    if (std::unique_ptr<X86Operand> Op = ParseIntelOperand()) {
+      Operands.push_back(std::move(Op));
+      return false;
+    }
+    return true;
+  }
+
+  return ParseATTOperand(Operands);
 }
 
 std::unique_ptr<X86Operand> X86AsmParser::CreateMemForMSInlineAsm(
@@ -2206,7 +2212,7 @@ std::unique_ptr<X86Operand> X86AsmParser::ParseIntelOperand() {
                                BaseReg, IndexReg, Scale, Start, End, Size);
 }
 
-std::unique_ptr<X86Operand> X86AsmParser::ParseATTOperand() {
+bool X86AsmParser::ParseATTOperand(OperandVector &Operands) {
   MCAsmParser &Parser = getParser();
   switch (getLexer().getKind()) {
   case AsmToken::Dollar: {
@@ -2221,12 +2227,17 @@ std::unique_ptr<X86Operand> X86AsmParser::ParseATTOperand() {
               "expected immediate expression") ||
         getParser().parseExpression(Val, End) ||
         check(isa<X86MCExpr>(Val), L, "expected immediate expression"))
-      return nullptr;
-    return X86Operand::CreateImm(Val, Start, End);
+      return true;
+    Operands.push_back(X86Operand::CreateImm(Val, Start, End));
+    return false;
   }
   case AsmToken::LCurly: {
     SMLoc Start = Parser.getTok().getLoc();
-    return ParseRoundingModeOp(Start);
+    if (std::unique_ptr<X86Operand> Op = ParseRoundingModeOp(Start)) {
+      Operands.push_back(std::move(Op));
+      return false;
+    }
+    return true;
   }
   default: {
     // This a memory operand or a register. We have some parsing complications
@@ -2240,7 +2251,7 @@ std::unique_ptr<X86Operand> X86AsmParser::ParseATTOperand() {
     if (getLexer().isNot(AsmToken::LParen)) {
       // No '(' so this is either a displacement expression or a register.
       if (Parser.parseExpression(Expr, EndLoc))
-        return nullptr;
+        return true;
       if (auto *RE = dyn_cast<X86MCExpr>(Expr)) {
         // Segment Register. Reset Expr and copy value to register.
         Expr = nullptr;
@@ -2248,21 +2259,31 @@ std::unique_ptr<X86Operand> X86AsmParser::ParseATTOperand() {
 
         // Sanity check register.
         if (Reg == X86::EIZ || Reg == X86::RIZ)
-          return ErrorOperand(
+          return Error(
               Loc, "%eiz and %riz can only be used as index registers",
               SMRange(Loc, EndLoc));
         if (Reg == X86::RIP)
-          return ErrorOperand(Loc, "%rip can only be used as a base register",
-                              SMRange(Loc, EndLoc));
+          return Error(Loc, "%rip can only be used as a base register",
+                       SMRange(Loc, EndLoc));
         // Return register that are not segment prefixes immediately.
-        if (!Parser.parseOptionalToken(AsmToken::Colon))
-          return X86Operand::CreateReg(Reg, Loc, EndLoc);
+        if (!Parser.parseOptionalToken(AsmToken::Colon)) {
+          Operands.push_back(X86Operand::CreateReg(Reg, Loc, EndLoc));
+          return false;
+        }
         if (!X86MCRegisterClasses[X86::SEGMENT_REGRegClassID].contains(Reg))
-          return ErrorOperand(Loc, "invalid segment register");
+          return Error(Loc, "invalid segment register");
+        // Accept a '*' absolute memory reference after the segment. Place it
+        // before the full memory operand.
+        if (getLexer().is(AsmToken::Star))
+          Operands.push_back(X86Operand::CreateToken("*", consumeToken()));
       }
     }
     // This is a Memory operand.
-    return ParseMemOperand(Reg, Expr, Loc, EndLoc);
+    if (std::unique_ptr<X86Operand> Op = ParseMemOperand(Reg, Expr, Loc, EndLoc)) {
+      Operands.push_back(std::move(Op));
+      return false;
+    }
+    return true;
   }
   }
 }
@@ -2889,13 +2910,11 @@ bool X86AsmParser::ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
 
     // Read the operands.
     while(1) {
-      if (std::unique_ptr<X86Operand> Op = ParseOperand()) {
-        Operands.push_back(std::move(Op));
-        if (HandleAVX512Operand(Operands))
-          return true;
-      } else {
-         return true;
-      }
+      if (ParseOperand(Operands))
+        return true;
+      if (HandleAVX512Operand(Operands))
+        return true;
+
       // check for comma and eat it
       if (getLexer().is(AsmToken::Comma))
         Parser.Lex();
diff --git a/llvm/test/MC/X86/x86-32.s b/llvm/test/MC/X86/x86-32.s
index 256d8351e74d8..ef9f5ebfdd92d 100644
--- a/llvm/test/MC/X86/x86-32.s
+++ b/llvm/test/MC/X86/x86-32.s
@@ -1120,3 +1120,7 @@ foo:
 // CHECK:  encoding: [0x0f,0x84,A,A,A,A]
 // CHECK:  fixup A - offset: 2, value: foo-4, kind: FK_PCRel_4
 {disp32} je foo
+
+// CHECK: ljmpl *%cs:305419896
+// CHECK:  encoding: [0x2e,0xff,0x2d,0x78,0x56,0x34,0x12]
+ljmp %cs:*0x12345678
diff --git a/llvm/test/MC/X86/x86-64.s b/llvm/test/MC/X86/x86-64.s
index 54b7c3388a480..38d5c6173f3b4 100644
--- a/llvm/test/MC/X86/x86-64.s
+++ b/llvm/test/MC/X86/x86-64.s
@@ -399,6 +399,10 @@ ljmp *0xbadeface
 lcall *(%rax)
 ljmpl *(%rax)
 
+// CHECK: ljmpl *%cs:305419896
+// CHECK:  encoding: [0x2e,0xff,0x2c,0x25,0x78,0x56,0x34,0x12]
+ljmp %cs:*0x12345678
+
 // rdar://8444631
 // CHECK: enter	$31438, $0
 // CHECK: encoding: [0xc8,0xce,0x7a,0x00]

From e24f5f314914cff02958b023f2ecfc3cb413372e Mon Sep 17 00:00:00 2001
From: Tony <Tony.Tye@amd.com>
Date: Fri, 3 Jul 2020 22:31:53 +0000
Subject: [PATCH 0576/1035] [AMDGPU] DWARF proposal changes

- Clarify that these are extensions to DWARF 5 and not as yet a
  proposal.

Reviewed By: scott.linder

Differential Revision: https://reviews.llvm.org/D70523
---
 ...rfExtensionsForHeterogeneousDebugging.rst} | 155 +++++++++---------
 llvm/docs/AMDGPUUsage.rst                     |  31 ++--
 2 files changed, 91 insertions(+), 95 deletions(-)
 rename llvm/docs/{AMDGPUDwarfProposalForHeterogeneousDebugging.rst => AMDGPUDwarfExtensionsForHeterogeneousDebugging.rst} (97%)

diff --git a/llvm/docs/AMDGPUDwarfProposalForHeterogeneousDebugging.rst b/llvm/docs/AMDGPUDwarfExtensionsForHeterogeneousDebugging.rst
similarity index 97%
rename from llvm/docs/AMDGPUDwarfProposalForHeterogeneousDebugging.rst
rename to llvm/docs/AMDGPUDwarfExtensionsForHeterogeneousDebugging.rst
index 8253519f7dfe9..b6df0c92201ae 100644
--- a/llvm/docs/AMDGPUDwarfProposalForHeterogeneousDebugging.rst
+++ b/llvm/docs/AMDGPUDwarfExtensionsForHeterogeneousDebugging.rst
@@ -1,15 +1,15 @@
-.. _amdgpu-dwarf-proposal-for-heterogeneous-debugging:
+.. _amdgpu-dwarf-extensions-for-heterogeneous-debugging:
 
-******************************************
-DWARF Proposal For Heterogeneous Debugging
-******************************************
+********************************************
+DWARF Extensions For Heterogeneous Debugging
+********************************************
 
 .. contents::
    :local:
 
 .. warning::
 
-   This document describes a **provisional proposal** for DWARF Version 6
+   This document describes **provisional extensions** to DWARF Version 5
    [:ref:`DWARF <amdgpu-dwarf-DWARF>`] to support heterogeneous debugging. It is
    not currently fully implemented and is subject to change.
 
@@ -37,43 +37,37 @@ and the Perforce TotalView HPC debugger [:ref:`Perforce-TotalView
 
 To support debugging heterogeneous programs several features that are not
 provided by current DWARF Version 5 [:ref:`DWARF <amdgpu-dwarf-DWARF>`] have
-been identified. This document contains a collection of proposals to address
+been identified. This document contains a collection of extensions to address
 providing those features.
 
 The :ref:`amdgpu-dwarf-motivation` section describes the issues that are being
 addressed for heterogeneous computing. That is followed by the
-:ref:`amdgpu-dwarf-proposed-changes-relative-to-dwarf-version-5` section
-containing the proposed textual changes relative to the DWARF Version 5
-standard. Then there is an :ref:`amdgpu-dwarf-examples` section that links to
-the AMD GPU specific usage of the features in the proposal that includes an
-example. Finally, there is a :ref:`amdgpu-dwarf-references` section. There are a
-number of notes included that raise open questions, or provide alternative
-approaches considered. The draft proposal seeks to be general in nature and
-backwards compatible with DWARF Version 5. Its goal is to be applicable to
-meeting the needs of any heterogeneous system and not be vendor or architecture
-specific.
-
-A fundamental aspect of the draft proposal is that it allows DWARF expression
-location descriptions as stack elements. The draft proposal is based on DWARF
+:ref:`amdgpu-dwarf-changes-relative-to-dwarf-version-5` section containing the
+textual changes for the extensions relative to the DWARF Version 5 standard.
+Then there is an :ref:`amdgpu-dwarf-examples` section that links to the AMD GPU
+specific usage of the extensions that includes an example. Finally, there is a
+:ref:`amdgpu-dwarf-references` section. There are a number of notes included
+that raise open questions, or provide alternative approaches considered. The
+extensions seek to be general in nature and backwards compatible with DWARF
+Version 5. The goal is to be applicable to meeting the needs of any
+heterogeneous system and not be vendor or architecture specific.
+
+A fundamental aspect of the extensions is that it allows DWARF expression
+location descriptions as stack elements. The extensions are based on DWARF
 Version 5 and maintains compatibility with DWARF Version 5. After attempting
-several alternatives, the current thinking is that such an addition to DWARF
-Version 5 is the simplest and cleanest way to support debugging optimized GPU
+several alternatives, the current thinking is that such extensions to DWARF
+Version 5 are the simplest and cleanest ways to support debugging optimized GPU
 code. It also appears to be generally useful and may be able to address other
 reported DWARF issues, as well as being helpful in providing better optimization
 support for non-GPU code.
 
-General feedback on this draft proposal is sought, together with suggestions on
-how to clarify, simplify, or organize it before submitting it as a formal DWARF
-proposal. The current draft proposal is large and may need to be split into
-separate proposals before formal submission. Any suggestions on how best to do
-that are appreciated. However, at the initial review stage it is believed there
-is value in presenting a unified proposal as there are mutual dependencies
-between the various parts that would not be as apparent if it was broken up into
-separate independent proposals.
+General feedback on these extensions is sought, together with suggestions on how
+to clarify, simplify, or organize them. If their is general interest then some
+or all of these extensions could be submitted as future DWARF proposals.
 
-We are in the process of modifying LLVM and GDB to support this draft proposal
+We are in the process of modifying LLVM and GDB to support these extensions
 which is providing experience and insights. We plan to upstream the changes to
-those projects for any final form of the proposal.
+those projects for any final form of the extensions.
 
 The author very much appreciates the input provided so far by many others which
 has been incorporated into this current version.
@@ -83,11 +77,10 @@ has been incorporated into this current version.
 Motivation
 ==========
 
-This document proposes a set of backwards compatible extensions to DWARF Version
-5 [:ref:`DWARF <amdgpu-dwarf-DWARF>`] for consideration of inclusion into a
-future DWARF Version 6 standard to support heterogeneous debugging.
+This document presents a set of backwards compatible extensions to DWARF Version
+5 [:ref:`DWARF <amdgpu-dwarf-DWARF>`] to support heterogeneous debugging.
 
-The remainder of this section provides motivation for each proposed feature in
+The remainder of this section provides motivation for each extension in
 terms of heterogeneous debugging on commercially available AMD GPU hardware
 (AMDGPU). The goal is to add support to the AMD [:ref:`AMD <amdgpu-dwarf-AMD>`]
 open source Radeon Open Compute Platform (ROCm) [:ref:`AMD-ROCm
@@ -102,21 +95,21 @@ to work with third parties to enable support for AMDGPU debugging in the GCC
 compiler [:ref:`GCC <amdgpu-dwarf-GCC>`] and the Perforce TotalView HPC debugger
 [:ref:`Perforce-TotalView <amdgpu-dwarf-Perforce-TotalView>`].
 
-However, the proposal is intended to be vendor and architecture neutral. It is
-believed to apply to other heterogenous hardware devices including GPUs, DSPs,
-FPGAs, and other specialized hardware. These collectively include similar
-characteristics and requirements as AMDGPU devices. Parts of the proposal can
+However, the extensions are intended to be vendor and architecture neutral. They
+are believed to apply to other heterogenous hardware devices including GPUs,
+DSPs, FPGAs, and other specialized hardware. These collectively include similar
+characteristics and requirements as AMDGPU devices. Some of the extension can
 also apply to traditional CPU hardware that supports large vector registers.
 Compilers can map source languages and extensions that describe large scale
 parallel execution onto the lanes of the vector registers. This is common in
-programming languages used in ML and HPC. The proposal also includes improved
+programming languages used in ML and HPC. The extensions also include improved
 support for optimized code on any architecture. Some of the generalizations may
 also benefit other issues that have been raised.
 
-The proposal has evolved though collaboration with many individuals and active
-prototyping within the GDB debugger and LLVM compiler. Input has also been very
-much appreciated from the developers working on the Perforce TotalView HPC
-Debugger and GCC compiler.
+The extensions have evolved though collaboration with many individuals and
+active prototyping within the GDB debugger and LLVM compiler. Input has also
+been very much appreciated from the developers working on the Perforce TotalView
+HPC Debugger and GCC compiler.
 
 The AMDGPU has several features that require additional DWARF functionality in
 order to support optimized code.
@@ -332,8 +325,8 @@ attributes such as ``DW_AT_data_member_location``, ``DW_AT_use_location``, and
 on the expression stack before evaluating the expression. However, DWARF
 Version 5 only allows the stack to contain values and so only a single memory
 address can be on the stack which makes these incapable of handling location
-descriptions with multiple places, or places other than memory. Since this
-proposal allows the stack to contain location descriptions, the operations are
+descriptions with multiple places, or places other than memory. Since these
+extensions allow the stack to contain location descriptions, the operations are
 generalized to support location descriptions that can have multiple places.
 This is backwards compatible with DWARF Version 5 and allows objects with
 multiple places to be supported. For example, the expression that describes
@@ -345,8 +338,8 @@ unified into a single section that describes DWARF expressions in general.
 This unification seems to be a natural consequence and a necessity of allowing
 location descriptions to be part of the evaluation stack.
 
-For those familiar with the definition of location descriptions in DWARF
-Version 5, the definition in this proposal is presented differently, but does
+For those familiar with the definition of location descriptions in DWARF Version
+5, the definitions in these extensions are presented differently, but does
 in fact define the same concept with the same fundamental semantics. However,
 it does so in a way that allows the concept to extend to support address
 spaces, bit addressing, the ability for composite location descriptions to be
@@ -354,7 +347,7 @@ composed of any kind of location description, and the ability to support
 objects located at multiple places. Collectively these changes expand the set
 of processors that can be supported and improves support for optimized code.
 
-Several approaches were considered, and the one proposed appears to be the
+Several approaches were considered, and the one presented appears to be the
 cleanest and offers the greatest improvement of DWARF's ability to support
 optimized code. Examining the GDB debugger and LLVM compiler, it appears only
 to require modest changes as they both already have to support general use of
@@ -412,22 +405,22 @@ style based on the DWARF Version 5 specification. Non-normative text is shown
 in *italics*.
 
 The names for the new operations, attributes, and constants include "\
-``LLVM``\ " and are encoded with vendor specific codes so this proposal can be
-implemented as an LLVM vendor extension to DWARF Version 5. If accepted these
+``LLVM``\ " and are encoded with vendor specific codes so these extensions can
+be implemented as an LLVM vendor extension to DWARF Version 5. If accepted these
 names would not include the "\ ``LLVM``\ " and would not use encodings in the
 vendor range.
 
-The proposal is described in
-:ref:`amdgpu-dwarf-proposed-changes-relative-to-dwarf-version-5` and is
+The extensions are described in
+:ref:`amdgpu-dwarf-changes-relative-to-dwarf-version-5` and are
 organized to follow the section ordering of DWARF Version 5. It includes notes
 to indicate the corresponding DWARF Version 5 sections to which they pertain.
 Other notes describe additional changes that may be worth considering, and to
 raise questions.
 
-.. _amdgpu-dwarf-proposed-changes-relative-to-dwarf-version-5:
+.. _amdgpu-dwarf-changes-relative-to-dwarf-version-5:
 
-Proposed Changes Relative to DWARF Version 5
-============================================
+Changes Relative to DWARF Version 5
+===================================
 
 General Description
 -------------------
@@ -463,7 +456,7 @@ DWARF Expressions
 .. note::
 
   This section, and its nested sections, replaces DWARF Version 5 section 2.5
-  and section 2.6. The new proposed DWARF expression operations are defined as
+  and section 2.6. The new DWARF expression operation extensions are defined as
   well as clarifying the extensions to already existing DWARF Version 5
   operations. It is based on the text of the existing DWARF Version 5 standard.
 
@@ -1432,8 +1425,8 @@ There are these special value operations currently defined:
       register hook reads bytes from the next register (or reads out of bounds
       for the last register!). Removing use of the target hook does not cause
       any test failures in common architectures (except an illegal hand written
-      assembly test). If a target architecture requires this behavior, this
-      proposal allows a composite location description to be used to combine
+      assembly test). If a target architecture requires this behavior, these
+      extensions allow a composite location description to be used to combine
       multiple registers.
 
 2.  ``DW_OP_deref``
@@ -1507,17 +1500,18 @@ There are these special value operations currently defined:
 
       This definition makes it an evaluation error if L is a register location
       description that has less than TS bits remaining in the register storage.
-      Particularly since this proposal extends location descriptions to have a
-      bit offset, it would be odd to define this as performing sign extension
+      Particularly since these extensions extend location descriptions to have
+      a bit offset, it would be odd to define this as performing sign extension
       based on the type, or be target architecture dependent, as the number of
       remaining bits could be any number. This matches the GDB implementation
       for ``DW_OP_deref_type``.
 
-      This proposal defines ``DW_OP_*breg*`` in terms of ``DW_OP_regval_type``.
-      ``DW_OP_regval_type`` is defined in terms of ``DW_OP_regx``, which uses a
-      0 bit offset, and ``DW_OP_deref_type``. Therefore, it requires the
-      register size to be greater or equal to the address size of the address
-      space. This matches the GDB implementation for ``DW_OP_*breg*``.
+      These extensions define ``DW_OP_*breg*`` in terms of
+      ``DW_OP_regval_type``. ``DW_OP_regval_type`` is defined in terms of
+      ``DW_OP_regx``, which uses a 0 bit offset, and ``DW_OP_deref_type``.
+      Therefore, it requires the register size to be greater or equal to the
+      address size of the address space. This matches the GDB implementation for
+      ``DW_OP_*breg*``.
 
     The DWARF is ill-formed if D is not in the current compilation unit, D is
     not a ``DW_TAG_base_type`` debugging information entry, or if TS divided by
@@ -2410,7 +2404,7 @@ compatible with the definitions in DWARF Version 5.*
 
     .. note::
 
-      Since this proposal allows location descriptions to be entries on the
+      Since these extensions allow location descriptions to be entries on the
       stack, a simpler operation to create composite location descriptions. For
       example, just one operation that specifies how many parts, and pops pairs
       of stack entries for the part size and location description. Not only
@@ -2693,15 +2687,15 @@ DWARF address space identifiers are used by:
 
 .. note::
 
-  With the definition of DWARF address classes and DWARF address spaces in this
-  proposal, DWARF Version 5 table 2.7 needs to be updated. It seems it is an
+  With the definition of DWARF address classes and DWARF address spaces in these
+  extensions, DWARF Version 5 table 2.7 needs to be updated. It seems it is an
   example of DWARF address spaces and not DWARF address classes.
 
 .. note::
 
-  With the expanded support for DWARF address spaces in this proposal, it may be
-  worth examining if DWARF segments can be eliminated and DWARF address spaces
-  used instead.
+  With the expanded support for DWARF address spaces in these extensions, it may
+  be worth examining if DWARF segments can be eliminated and DWARF address
+  spaces used instead.
 
   That may involve extending DWARF address spaces to also be used to specify
   code locations. In target architectures that use different memory areas for
@@ -2751,7 +2745,7 @@ DWARF address space identifiers are used by:
   debugger information entry type modifier that can be applied to a pointer type
   and reference type. The ``DW_AT_address_class`` attribute could be re-defined
   to not be target architecture specific and instead define generalized language
-  values (as is proposed above for DWARF address classes in the table
+  values (as presented above for DWARF address classes in the table
   :ref:`amdgpu-dwarf-address-class-table`) that will support OpenCL and other
   languages using memory spaces. The ``DW_AT_address_class`` attribute could be
   defined to not be applied to pointer types or reference types, but instead
@@ -2772,7 +2766,7 @@ DWARF address space identifiers are used by:
   variable allocation in address classes. Such variable allocation would result
   in the variable's location description needing an address space.
 
-  The approach proposed in :ref:`amdgpu-dwarf-address-class-table` is to define
+  The approach presented in :ref:`amdgpu-dwarf-address-class-table` is to define
   the default ``DW_ADDR_none`` to be the generic address class and not the
   global address class. This matches how CLANG and LLVM have added support for
   CUDA-like languages on top of existing C++ language support. This allows all
@@ -2827,7 +2821,7 @@ Debugging Information Entry Attributes
 .. note::
 
   This section provides changes to existing debugger information entry
-  attributes and defines attributes added by the proposal. These would be
+  attributes and defines attributes added by these extensions. These would be
   incorporated into the appropriate DWARF Version 5 chapter 2 sections.
 
 1.  ``DW_AT_location``
@@ -3419,8 +3413,8 @@ Call Frame Information
 .. note::
 
   This section provides changes to existing call frame information and defines
-  instructions added by the proposal. Additional support is added for address
-  spaces. Register unwind DWARF expressions are generalized to allow any
+  instructions added by these extensions. Additional support is added for
+  address spaces. Register unwind DWARF expressions are generalized to allow any
   location description, including those with composite and implicit location
   descriptions.
 
@@ -3578,7 +3572,7 @@ Frame Description Entries (FDE). There is at least one CIE in every non-empty
 
     .. note::
 
-      Would this be increased to 5 to reflect the changes in the proposal?
+      Would this be increased to 5 to reflect the changes in these extensions?
 
 4.  ``augmentation`` (sequence of UTF-8 characters)
 
@@ -4243,8 +4237,9 @@ debugger information entries.
 Examples
 ========
 
-The AMD GPU specific usage of the features in the proposal, including examples,
-is available at :ref:`amdgpu-dwarf-debug-information`.
+The AMD GPU specific usage of the features in these extensions, including
+examples, is available at *User Guide for AMDGPU Backend* section
+:ref:`amdgpu-dwarf-debug-information`.
 
 .. note::
 
diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst
index f709bb8cf8e2f..0a29f9e8f4131 100644
--- a/llvm/docs/AMDGPUUsage.rst
+++ b/llvm/docs/AMDGPUUsage.rst
@@ -21,7 +21,7 @@ User Guide for AMDGPU Backend
    AMDGPUOperandSyntax
    AMDGPUInstructionSyntax
    AMDGPUInstructionNotation
-   AMDGPUDwarfProposalForHeterogeneousDebugging
+   AMDGPUDwarfExtensionsForHeterogeneousDebugging
 
 Introduction
 ============
@@ -1160,14 +1160,14 @@ DWARF Debug Information
 
 .. warning::
 
-   This section describes a **provisional proposal** for AMDGPU DWARF [DWARF]_
-   that is not currently fully implemented and is subject to change.
+   This section describes **provisional support** for AMDGPU DWARF [DWARF]_ that
+   is not currently fully implemented and is subject to change.
 
 AMDGPU generates DWARF [DWARF]_ debugging information ELF sections (see
 :ref:`amdgpu-elf-code-object`) which contain information that maps the code
 object executable code and data to the source language constructs. It can be
 used by tools such as debuggers and profilers. It uses features defined in
-:doc:`AMDGPUDwarfProposalForHeterogeneousDebugging` that are made available in
+:doc:`AMDGPUDwarfExtensionsForHeterogeneousDebugging` that are made available in
 DWARF Version 4 and DWARF Version 5 as an LLVM vendor extension.
 
 This section defines the AMDGPU target architecture specific DWARF mappings.
@@ -1299,8 +1299,8 @@ Address Class Identifier
 ------------------------
 
 The DWARF address class represents the source language memory space. See DWARF
-Version 5 section 2.12 which is updated by the propoal in
-:ref:`amdgpu-dwarf-segment_addresses`.
+Version 5 section 2.12 which is updated by the *DWARF Extensions For
+Heterogeneous Debugging* section :ref:`amdgpu-dwarf-segment_addresses`.
 
 The DWARF address class mapping used for AMDGPU is defined in
 :ref:`amdgpu-dwarf-address-class-mapping-table`.
@@ -1321,8 +1321,8 @@ The DWARF address class mapping used for AMDGPU is defined in
    ``DW_ADDR_AMDGPU_region`` 0x8000 Region (GDS)
    ========================= ====== =================
 
-The DWARF address class values defined in the proposal at
-:ref:`amdgpu-dwarf-segment_addresses` are used.
+The DWARF address class values defined in the *DWARF Extensions For
+Heterogeneous Debugging* section :ref:`amdgpu-dwarf-segment_addresses` are used.
 
 In addition, ``DW_ADDR_AMDGPU_region`` is encoded as a vendor extension. This is
 available for use for the AMD extension for access to the hardware GDS memory
@@ -1341,8 +1341,8 @@ Address Space Identifier
 ------------------------
 
 DWARF address spaces correspond to target architecture specific linear
-addressable memory areas. See DWARF Version 5 section 2.12 and
-:ref:`amdgpu-dwarf-segment_addresses`.
+addressable memory areas. See DWARF Version 5 section 2.12 and *DWARF Extensions
+For Heterogeneous Debugging* section :ref:`amdgpu-dwarf-segment_addresses`.
 
 The DWARF address space mapping used for AMDGPU is defined in
 :ref:`amdgpu-dwarf-address-space-mapping-table`.
@@ -1447,8 +1447,8 @@ DWARF lane identifies specify a target architecture lane position for hardware
 that executes in a SIMD or SIMT manner, and on which a source language maps its
 threads of execution onto those lanes. The DWARF lane identifier is pushed by
 the ``DW_OP_LLVM_push_lane`` DWARF expression operation. See DWARF Version 5
-section 2.5 which is updated by the proposal in
-:ref:`amdgpu-dwarf-operation-expressions`.
+section 2.5 which is updated by *DWARF Extensions For Heterogeneous Debugging*
+section :ref:`amdgpu-dwarf-operation-expressions`.
 
 For AMDGPU, the lane identifier corresponds to the hardware lane ID of a
 wavefront. It is numbered from 0 to the wavefront size minus 1.
@@ -1483,7 +1483,8 @@ Debugger Information Entry Attributes
 
 This section describes how certain debugger information entry attributes are
 used by AMDGPU. See the sections in DWARF Version 5 section 2 which are updated
-by the proposal in :ref:`amdgpu-dwarf-debugging-information-entry-attributes`.
+by *DWARF Extensions For Heterogeneous Debugging* section
+:ref:`amdgpu-dwarf-debugging-information-entry-attributes`.
 
 .. _amdgpu-dwarf-dw-at-llvm-lane-pc:
 
@@ -1938,8 +1939,8 @@ DWARF Version 5 section 6.2.4):
 
 Source text for online-compiled programs (for example, those compiled by the
 OpenCL language runtime) may be embedded into the DWARF Version 5 line table.
-See DWARF Version 5 section 6.2.4.1 which is updated by the proposal in
-:ref:`DW_LNCT_LLVM_source
+See DWARF Version 5 section 6.2.4.1 which is updated by *DWARF Extensions For
+Heterogeneous Debugging* section :ref:`DW_LNCT_LLVM_source
 <amdgpu-dwarf-line-number-information-dw-lnct-llvm-source>`.
 
 The Clang option used to control source embedding in AMDGPU is defined in

From 629467eb981ac5d6bccebd524a166e02bef1820f Mon Sep 17 00:00:00 2001
From: Tony <Tony.Tye@amd.com>
Date: Thu, 30 Jul 2020 05:10:21 +0000
Subject: [PATCH 0577/1035] [AMDGPU] Fix DWARF extensions User Guide table of
 contents

---
 llvm/docs/UserGuides.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/llvm/docs/UserGuides.rst b/llvm/docs/UserGuides.rst
index 6e4329128fab4..2e0cffb711ef9 100644
--- a/llvm/docs/UserGuides.rst
+++ b/llvm/docs/UserGuides.rst
@@ -198,6 +198,6 @@ Additional Topics
 :doc:`AMDGPUUsage`
    This document describes using the AMDGPU backend to compile GPU kernels.
 
-:doc:`AMDGPUDwarfProposalForHeterogeneousDebugging`
-   This document describes a DWARF proposal to support heterogeneous debugging
+:doc:`AMDGPUDwarfExtensionsForHeterogeneousDebugging`
+   This document describes DWARF extensions to support heterogeneous debugging
    for targets such as the AMDGPU backend.

From b22b97b3d0c08485d478073d6a2a6e769af7a2af Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bal=C3=A1zs=20K=C3=A9ri?= <1.int32@gmail.com>
Date: Thu, 30 Jul 2020 08:31:39 +0200
Subject: [PATCH 0578/1035] [Analyzer] Use of BugType in DereferenceChecker
 (NFC).

Use of BuiltinBug is replaced by BugType.
Class BuiltinBug seems to have no benefits and is confusing.

Reviewed By: Szelethus, martong, NoQ, vsavchenko

Differential Revision: https://reviews.llvm.org/D84494
---
 .../Checkers/DereferenceChecker.cpp           | 21 +++++++------------
 1 file changed, 7 insertions(+), 14 deletions(-)

diff --git a/clang/lib/StaticAnalyzer/Checkers/DereferenceChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/DereferenceChecker.cpp
index 2411f0e2d058f..9a87729de8fda 100644
--- a/clang/lib/StaticAnalyzer/Checkers/DereferenceChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/DereferenceChecker.cpp
@@ -30,8 +30,9 @@ class DereferenceChecker
     : public Checker< check::Location,
                       check::Bind,
                       EventDispatcher<ImplicitNullDerefEvent> > {
-  mutable std::unique_ptr<BuiltinBug> BT_null;
-  mutable std::unique_ptr<BuiltinBug> BT_undef;
+  BugType BT_Null{this, "Dereference of null pointer", categories::LogicError};
+  BugType BT_Undef{this, "Dereference of undefined pointer value",
+                   categories::LogicError};
 
   void reportBug(ProgramStateRef State, const Stmt *S, CheckerContext &C) const;
 
@@ -123,11 +124,6 @@ void DereferenceChecker::reportBug(ProgramStateRef State, const Stmt *S,
   if (!N)
     return;
 
-  // We know that 'location' cannot be non-null.  This is what
-  // we call an "explicit" null dereference.
-  if (!BT_null)
-    BT_null.reset(new BuiltinBug(this, "Dereference of null pointer"));
-
   SmallString<100> buf;
   llvm::raw_svector_ostream os(buf);
 
@@ -180,7 +176,7 @@ void DereferenceChecker::reportBug(ProgramStateRef State, const Stmt *S,
   }
 
   auto report = std::make_unique<PathSensitiveBugReport>(
-      *BT_null, buf.empty() ? BT_null->getDescription() : StringRef(buf), N);
+      BT_Null, buf.empty() ? BT_Null.getDescription() : StringRef(buf), N);
 
   bugreporter::trackExpressionValue(N, bugreporter::getDerefExpr(S), *report);
 
@@ -196,12 +192,8 @@ void DereferenceChecker::checkLocation(SVal l, bool isLoad, const Stmt* S,
   // Check for dereference of an undefined value.
   if (l.isUndef()) {
     if (ExplodedNode *N = C.generateErrorNode()) {
-      if (!BT_undef)
-        BT_undef.reset(
-            new BuiltinBug(this, "Dereference of undefined pointer value"));
-
       auto report = std::make_unique<PathSensitiveBugReport>(
-          *BT_undef, BT_undef->getDescription(), N);
+          BT_Undef, BT_Undef.getDescription(), N);
       bugreporter::trackExpressionValue(N, bugreporter::getDerefExpr(S), *report);
       C.emitReport(std::move(report));
     }
@@ -219,9 +211,10 @@ void DereferenceChecker::checkLocation(SVal l, bool isLoad, const Stmt* S,
   ProgramStateRef notNullState, nullState;
   std::tie(notNullState, nullState) = state->assume(location);
 
-  // The explicit NULL case.
   if (nullState) {
     if (!notNullState) {
+      // We know that 'location' can only be null.  This is what
+      // we call an "explicit" null dereference.
       const Expr *expr = getDereferenceExpr(S);
       if (!suppressReport(expr)) {
         reportBug(nullState, expr, C);

From 3678ad88a6a1cbc8f5171519666b10e8997289dc Mon Sep 17 00:00:00 2001
From: Max Kazantsev <mkazantsev@azul.com>
Date: Thu, 30 Jul 2020 13:31:16 +0700
Subject: [PATCH 0579/1035] [NFC] Remove unused variable

---
 llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp b/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp
index 6c6d6ca9cf656..c7d23c9532910 100644
--- a/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp
+++ b/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp
@@ -2844,7 +2844,6 @@ static bool unswitchLoop(Loop &L, DominatorTree &DT, LoopInfo &LI,
                          ScalarEvolution *SE, MemorySSAUpdater *MSSAU) {
   assert(L.isRecursivelyLCSSAForm(DT, LI) &&
          "Loops must be in LCSSA form before unswitching.");
-  bool Changed = false;
 
   // Must be in loop simplified form: we need a preheader and dedicated exits.
   if (!L.isLoopSimplifyForm())
@@ -2876,7 +2875,7 @@ static bool unswitchLoop(Loop &L, DominatorTree &DT, LoopInfo &LI,
     return true;
 
   // No other opportunities to unswitch.
-  return Changed;
+  return false;
 }
 
 PreservedAnalyses SimpleLoopUnswitchPass::run(Loop &L, LoopAnalysisManager &AM,

From 032ed39defc7403b94c367fa43432606c418e946 Mon Sep 17 00:00:00 2001
From: Serge Pavlov <sepavloff@gmail.com>
Date: Wed, 29 Apr 2020 11:26:19 +0700
Subject: [PATCH 0580/1035] [Support] Class to facilitate file locking

This change define RAII class `FileLocker` and methods `lock` and
`tryLockFor` of the class `raw_fd_stream` to facilitate using file locks.

Differential Revision: https://reviews.llvm.org/D79066
---
 llvm/include/llvm/Support/FileSystem.h  | 29 +++++++++++++++
 llvm/include/llvm/Support/raw_ostream.h | 37 ++++++++++++++++++-
 llvm/lib/Support/raw_ostream.cpp        | 15 ++++++++
 llvm/unittests/Support/Path.cpp         | 47 +++++++++++++++++++++++++
 4 files changed, 127 insertions(+), 1 deletion(-)

diff --git a/llvm/include/llvm/Support/FileSystem.h b/llvm/include/llvm/Support/FileSystem.h
index 18d787a75f611..5a8842ee26e21 100644
--- a/llvm/include/llvm/Support/FileSystem.h
+++ b/llvm/include/llvm/Support/FileSystem.h
@@ -1179,6 +1179,35 @@ std::error_code unlockFile(int FD);
 /// means that the filesystem may have failed to perform some buffered writes.
 std::error_code closeFile(file_t &F);
 
+/// RAII class that facilitates file locking.
+class FileLocker {
+  int FD; ///< Locked file handle.
+  FileLocker(int FD) : FD(FD) {}
+  friend class llvm::raw_fd_ostream;
+
+public:
+  FileLocker(const FileLocker &L) = delete;
+  FileLocker(FileLocker &&L) : FD(L.FD) { L.FD = -1; }
+  ~FileLocker() {
+    if (FD != -1)
+      unlockFile(FD);
+  }
+  FileLocker &operator=(FileLocker &&L) {
+    FD = L.FD;
+    L.FD = -1;
+    return *this;
+  }
+  FileLocker &operator=(const FileLocker &L) = delete;
+  std::error_code unlock() {
+    if (FD != -1) {
+      std::error_code Result = unlockFile(FD);
+      FD = -1;
+      return Result;
+    }
+    return std::error_code();
+  }
+};
+
 std::error_code getUniqueID(const Twine Path, UniqueID &Result);
 
 /// Get disk space usage information.
diff --git a/llvm/include/llvm/Support/raw_ostream.h b/llvm/include/llvm/Support/raw_ostream.h
index 8d289f7c765f9..e9c710d0f38fe 100644
--- a/llvm/include/llvm/Support/raw_ostream.h
+++ b/llvm/include/llvm/Support/raw_ostream.h
@@ -16,6 +16,7 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 #include <cassert>
+#include <chrono>
 #include <cstddef>
 #include <cstdint>
 #include <cstring>
@@ -30,12 +31,14 @@ class format_object_base;
 class FormattedString;
 class FormattedNumber;
 class FormattedBytes;
+template <class T> class LLVM_NODISCARD Expected;
 
 namespace sys {
 namespace fs {
 enum FileAccess : unsigned;
 enum OpenFlags : unsigned;
 enum CreationDisposition : unsigned;
+class FileLocker;
 } // end namespace fs
 } // end namespace sys
 
@@ -468,7 +471,7 @@ class raw_fd_ostream : public raw_pwrite_stream {
   /// fsync.
   void close();
 
-  bool supportsSeeking() { return SupportsSeeking; }
+  bool supportsSeeking() const { return SupportsSeeking; }
 
   /// Flushes the stream and repositions the underlying file descriptor position
   /// to the offset specified from the beginning of the file.
@@ -496,6 +499,38 @@ class raw_fd_ostream : public raw_pwrite_stream {
   ///      - from The Zen of Python, by Tim Peters
   ///
   void clear_error() { EC = std::error_code(); }
+
+  /// Locks the underlying file.
+  ///
+  /// @returns RAII object that releases the lock upon leaving the scope, if the
+  ///          locking was successful. Otherwise returns corresponding
+  ///          error code.
+  ///
+  /// The function blocks the current thread until the lock become available or
+  /// error occurs.
+  ///
+  /// Possible use of this function may be as follows:
+  ///
+  ///   @code{.cpp}
+  ///   if (auto L = stream.lock()) {
+  ///     // ... do action that require file to be locked.
+  ///   } else {
+  ///     handleAllErrors(std::move(L.takeError()), [&](ErrorInfoBase &EIB) {
+  ///       // ... handle lock error.
+  ///     });
+  ///   }
+  ///   @endcode
+  LLVM_NODISCARD Expected<sys::fs::FileLocker> lock();
+
+  /// Tries to lock the underlying file within the specified period.
+  ///
+  /// @returns RAII object that releases the lock upon leaving the scope, if the
+  ///          locking was successful. Otherwise returns corresponding
+  ///          error code.
+  ///
+  /// It is used as @ref lock.
+  LLVM_NODISCARD
+  Expected<sys::fs::FileLocker> tryLockFor(std::chrono::milliseconds Timeout);
 };
 
 /// This returns a reference to a raw_fd_ostream for standard output. Use it
diff --git a/llvm/lib/Support/raw_ostream.cpp b/llvm/lib/Support/raw_ostream.cpp
index f2d78d7732397..86c48993957a0 100644
--- a/llvm/lib/Support/raw_ostream.cpp
+++ b/llvm/lib/Support/raw_ostream.cpp
@@ -861,6 +861,21 @@ bool raw_fd_ostream::has_colors() const {
   return sys::Process::FileDescriptorHasColors(FD);
 }
 
+Expected<sys::fs::FileLocker> raw_fd_ostream::lock() {
+  std::error_code EC = sys::fs::lockFile(FD);
+  if (!EC)
+    return sys::fs::FileLocker(FD);
+  return errorCodeToError(EC);
+}
+
+Expected<sys::fs::FileLocker>
+raw_fd_ostream::tryLockFor(std::chrono::milliseconds Timeout) {
+  std::error_code EC = sys::fs::tryLockFile(FD, Timeout);
+  if (!EC)
+    return sys::fs::FileLocker(FD);
+  return errorCodeToError(EC);
+}
+
 void raw_fd_ostream::anchor() {}
 
 //===----------------------------------------------------------------------===//
diff --git a/llvm/unittests/Support/Path.cpp b/llvm/unittests/Support/Path.cpp
index 19ff49d254263..9d7ab1a60090e 100644
--- a/llvm/unittests/Support/Path.cpp
+++ b/llvm/unittests/Support/Path.cpp
@@ -2179,4 +2179,51 @@ TEST_F(FileSystemTest, widenPath) {
 }
 #endif
 
+#ifdef _WIN32
+// Windows refuses lock request if file region is already locked by the same
+// process. POSIX system in this case updates the existing lock.
+TEST_F(FileSystemTest, FileLocker) {
+  using namespace std::chrono;
+  int FD;
+  std::error_code EC;
+  SmallString<64> TempPath;
+  EC = fs::createTemporaryFile("test", "temp", FD, TempPath);
+  ASSERT_NO_ERROR(EC);
+  FileRemover Cleanup(TempPath);
+  raw_fd_ostream Stream(TempPath, EC);
+
+  EC = fs::tryLockFile(FD);
+  ASSERT_NO_ERROR(EC);
+  EC = fs::unlockFile(FD);
+  ASSERT_NO_ERROR(EC);
+
+  if (auto L = Stream.lock()) {
+    ASSERT_ERROR(fs::tryLockFile(FD));
+    ASSERT_NO_ERROR(L->unlock());
+    ASSERT_NO_ERROR(fs::tryLockFile(FD));
+    ASSERT_NO_ERROR(fs::unlockFile(FD));
+  } else {
+    ADD_FAILURE();
+    handleAllErrors(L.takeError(), [&](ErrorInfoBase &EIB) {});
+  }
+
+  ASSERT_NO_ERROR(fs::tryLockFile(FD));
+  ASSERT_NO_ERROR(fs::unlockFile(FD));
+
+  {
+    Expected<fs::FileLocker> L1 = Stream.lock();
+    ASSERT_THAT_EXPECTED(L1, Succeeded());
+    raw_fd_ostream Stream2(FD, false);
+    Expected<fs::FileLocker> L2 = Stream2.tryLockFor(250ms);
+    ASSERT_THAT_EXPECTED(L2, Failed());
+    ASSERT_NO_ERROR(L1->unlock());
+    Expected<fs::FileLocker> L3 = Stream.tryLockFor(0ms);
+    ASSERT_THAT_EXPECTED(L3, Succeeded());
+  }
+
+  ASSERT_NO_ERROR(fs::tryLockFile(FD));
+  ASSERT_NO_ERROR(fs::unlockFile(FD));
+}
+#endif
+
 } // anonymous namespace

From 17597442db159829dd167abb638b198ad0afc309 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@intel.com>
Date: Wed, 29 Jul 2020 22:57:28 -0700
Subject: [PATCH 0581/1035] [X86] Don't pass some many parameters to
 ParseMemOperand by reference.

Pointers and SMLocs are cheap to copy. Even though the function
modifies some of these the caller doesn't use them after the call.
---
 llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp b/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
index 445bb482055ce..0479db857a875 100644
--- a/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
+++ b/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
@@ -926,9 +926,9 @@ class X86AsmParser : public MCTargetAsmParser {
                                      bool IsParsingOffsetOperator = false);
 
   std::unique_ptr<X86Operand> ParseMemOperand(unsigned SegReg,
-                                              const MCExpr *&Disp,
-                                              const SMLoc &StartLoc,
-                                              SMLoc &EndLoc);
+                                              const MCExpr *Disp,
+                                              SMLoc StartLoc,
+                                              SMLoc EndLoc);
 
   X86::CondCode ParseConditionCode(StringRef CCode);
 
@@ -2418,9 +2418,9 @@ bool X86AsmParser::HandleAVX512Operand(OperandVector &Operands) {
 /// ParseMemOperand: 'seg : disp(basereg, indexreg, scale)'.  The '%ds:' prefix
 /// has already been parsed if present. disp may be provided as well.
 std::unique_ptr<X86Operand> X86AsmParser::ParseMemOperand(unsigned SegReg,
-                                                          const MCExpr *&Disp,
-                                                          const SMLoc &StartLoc,
-                                                          SMLoc &EndLoc) {
+                                                          const MCExpr *Disp,
+                                                          SMLoc StartLoc,
+                                                          SMLoc EndLoc) {
   MCAsmParser &Parser = getParser();
   SMLoc Loc;
   // Based on the initial passed values, we may be in any of these cases, we are

From 07bb8240a0b2f4262764efb828b09650f24a03c5 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@intel.com>
Date: Wed, 29 Jul 2020 23:19:43 -0700
Subject: [PATCH 0582/1035] [X86] Pass the OperandVector to ParseMemOperand
 instead of returning the operand. NFCI

Continue the change made to ParseATTOperand to take the vector by
reference. Let ParseMemOperand add its memory operand to the
vector and just return true/false to indicate error.
---
 .../lib/Target/X86/AsmParser/X86AsmParser.cpp | 75 ++++++++++---------
 1 file changed, 38 insertions(+), 37 deletions(-)

diff --git a/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp b/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
index 0479db857a875..a278c46aaf97c 100644
--- a/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
+++ b/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
@@ -925,10 +925,8 @@ class X86AsmParser : public MCTargetAsmParser {
                                      bool IsUnevaluatedOperand, SMLoc &End,
                                      bool IsParsingOffsetOperator = false);
 
-  std::unique_ptr<X86Operand> ParseMemOperand(unsigned SegReg,
-                                              const MCExpr *Disp,
-                                              SMLoc StartLoc,
-                                              SMLoc EndLoc);
+  bool ParseMemOperand(unsigned SegReg, const MCExpr *Disp, SMLoc StartLoc,
+                       SMLoc EndLoc, OperandVector &Operands);
 
   X86::CondCode ParseConditionCode(StringRef CCode);
 
@@ -2279,11 +2277,7 @@ bool X86AsmParser::ParseATTOperand(OperandVector &Operands) {
       }
     }
     // This is a Memory operand.
-    if (std::unique_ptr<X86Operand> Op = ParseMemOperand(Reg, Expr, Loc, EndLoc)) {
-      Operands.push_back(std::move(Op));
-      return false;
-    }
-    return true;
+    return ParseMemOperand(Reg, Expr, Loc, EndLoc, Operands);
   }
   }
 }
@@ -2417,10 +2411,9 @@ bool X86AsmParser::HandleAVX512Operand(OperandVector &Operands) {
 
 /// ParseMemOperand: 'seg : disp(basereg, indexreg, scale)'.  The '%ds:' prefix
 /// has already been parsed if present. disp may be provided as well.
-std::unique_ptr<X86Operand> X86AsmParser::ParseMemOperand(unsigned SegReg,
-                                                          const MCExpr *Disp,
-                                                          SMLoc StartLoc,
-                                                          SMLoc EndLoc) {
+bool X86AsmParser::ParseMemOperand(unsigned SegReg, const MCExpr *Disp,
+                                   SMLoc StartLoc, SMLoc EndLoc,
+                                   OperandVector &Operands) {
   MCAsmParser &Parser = getParser();
   SMLoc Loc;
   // Based on the initial passed values, we may be in any of these cases, we are
@@ -2482,7 +2475,7 @@ std::unique_ptr<X86Operand> X86AsmParser::ParseMemOperand(unsigned SegReg,
     // Parse immediate if we're not at a mem operand yet.
     if (!isAtMemOperand()) {
       if (Parser.parseTokenLoc(Loc) || Parser.parseExpression(Disp, EndLoc))
-        return nullptr;
+        return true;
       assert(!isa<X86MCExpr>(Disp) && "Expected non-register here.");
     } else {
       // Disp is implicitly zero if we haven't parsed it yet.
@@ -2495,9 +2488,12 @@ std::unique_ptr<X86Operand> X86AsmParser::ParseMemOperand(unsigned SegReg,
 
   if (!parseOptionalToken(AsmToken::LParen)) {
     if (SegReg == 0)
-      return X86Operand::CreateMem(getPointerWidth(), Disp, StartLoc, EndLoc);
-    return X86Operand::CreateMem(getPointerWidth(), SegReg, Disp, 0, 0, 1,
-                                 StartLoc, EndLoc);
+      Operands.push_back(
+          X86Operand::CreateMem(getPointerWidth(), Disp, StartLoc, EndLoc));
+    else
+      Operands.push_back(X86Operand::CreateMem(getPointerWidth(), SegReg, Disp,
+                                               0, 0, 1, StartLoc, EndLoc));
+    return false;
   }
 
   // If we reached here, then eat the '(' and Process
@@ -2511,14 +2507,13 @@ std::unique_ptr<X86Operand> X86AsmParser::ParseMemOperand(unsigned SegReg,
   if (getLexer().isNot(AsmToken::Comma) && getLexer().isNot(AsmToken::RParen)) {
     if (Parser.parseExpression(E, EndLoc) ||
         check(!isa<X86MCExpr>(E), BaseLoc, "expected register here"))
-      return nullptr;
+      return true;
 
     // Sanity check register.
     BaseReg = cast<X86MCExpr>(E)->getRegNo();
     if (BaseReg == X86::EIZ || BaseReg == X86::RIZ)
-      return ErrorOperand(BaseLoc,
-                          "eiz and riz can only be used as index registers",
-                          SMRange(BaseLoc, EndLoc));
+      return Error(BaseLoc, "eiz and riz can only be used as index registers",
+                   SMRange(BaseLoc, EndLoc));
   }
 
   if (parseOptionalToken(AsmToken::Comma)) {
@@ -2530,14 +2525,14 @@ std::unique_ptr<X86Operand> X86AsmParser::ParseMemOperand(unsigned SegReg,
     // "1(%eax,,1)", the assembler doesn't. Use "eiz" or "riz" for this.
     if (getLexer().isNot(AsmToken::RParen)) {
       if (Parser.parseTokenLoc(Loc) || Parser.parseExpression(E, EndLoc))
-        return nullptr;
+        return true;
 
       if (!isa<X86MCExpr>(E)) {
         // We've parsed an unexpected Scale Value instead of an index
         // register. Interpret it as an absolute.
         int64_t ScaleVal;
         if (!E->evaluateAsAbsolute(ScaleVal, getStreamer().getAssemblerPtr()))
-          return ErrorOperand(Loc, "expected absolute expression");
+          return Error(Loc, "expected absolute expression");
         if (ScaleVal != 1)
           Warning(Loc, "scale factor without index register is ignored");
         Scale = 1;
@@ -2545,10 +2540,10 @@ std::unique_ptr<X86Operand> X86AsmParser::ParseMemOperand(unsigned SegReg,
         IndexReg = cast<X86MCExpr>(E)->getRegNo();
 
         if (BaseReg == X86::RIP)
-          return ErrorOperand(
-              Loc, "%rip as base register can not have an index register");
+          return Error(Loc,
+                       "%rip as base register can not have an index register");
         if (IndexReg == X86::RIP)
-          return ErrorOperand(Loc, "%rip is not allowed as an index register");
+          return Error(Loc, "%rip is not allowed as an index register");
 
         if (parseOptionalToken(AsmToken::Comma)) {
           // Parse the scale amount:
@@ -2559,15 +2554,14 @@ std::unique_ptr<X86Operand> X86AsmParser::ParseMemOperand(unsigned SegReg,
             int64_t ScaleVal;
             if (Parser.parseTokenLoc(Loc) ||
                 Parser.parseAbsoluteExpression(ScaleVal))
-              return ErrorOperand(Loc, "expected scale expression");
+              return Error(Loc, "expected scale expression");
             Scale = (unsigned)ScaleVal;
             // Validate the scale amount.
             if (X86MCRegisterClasses[X86::GR16RegClassID].contains(BaseReg) &&
                 Scale != 1)
-              return ErrorOperand(Loc,
-                                  "scale factor in 16-bit address must be 1");
+              return Error(Loc, "scale factor in 16-bit address must be 1");
             if (checkScale(Scale, ErrMsg))
-              return ErrorOperand(Loc, ErrMsg);
+              return Error(Loc, ErrMsg);
           }
         }
       }
@@ -2576,23 +2570,30 @@ std::unique_ptr<X86Operand> X86AsmParser::ParseMemOperand(unsigned SegReg,
 
   // Ok, we've eaten the memory operand, verify we have a ')' and eat it too.
   if (parseToken(AsmToken::RParen, "unexpected token in memory operand"))
-    return nullptr;
+    return true;
 
   // This is to support otherwise illegal operand (%dx) found in various
   // unofficial manuals examples (e.g. "out[s]?[bwl]? %al, (%dx)") and must now
   // be supported. Mark such DX variants separately fix only in special cases.
   if (BaseReg == X86::DX && IndexReg == 0 && Scale == 1 && SegReg == 0 &&
-      isa<MCConstantExpr>(Disp) && cast<MCConstantExpr>(Disp)->getValue() == 0)
-    return X86Operand::CreateDXReg(BaseLoc, BaseLoc);
+      isa<MCConstantExpr>(Disp) &&
+      cast<MCConstantExpr>(Disp)->getValue() == 0) {
+    Operands.push_back(X86Operand::CreateDXReg(BaseLoc, BaseLoc));
+    return false;
+  }
 
   if (CheckBaseRegAndIndexRegAndScale(BaseReg, IndexReg, Scale, is64BitMode(),
                                       ErrMsg))
-    return ErrorOperand(BaseLoc, ErrMsg);
+    return Error(BaseLoc, ErrMsg);
 
   if (SegReg || BaseReg || IndexReg)
-    return X86Operand::CreateMem(getPointerWidth(), SegReg, Disp, BaseReg,
-                                 IndexReg, Scale, StartLoc, EndLoc);
-  return X86Operand::CreateMem(getPointerWidth(), Disp, StartLoc, EndLoc);
+    Operands.push_back(X86Operand::CreateMem(getPointerWidth(), SegReg, Disp,
+                                             BaseReg, IndexReg, Scale, StartLoc,
+                                             EndLoc));
+  else
+    Operands.push_back(
+        X86Operand::CreateMem(getPointerWidth(), Disp, StartLoc, EndLoc));
+  return false;
 }
 
 // Parse either a standard primary expression or a register.

From cd4e8d7f6f5ef108919f9f53db35ac73d1edea3d Mon Sep 17 00:00:00 2001
From: Haojian Wu <hokein.wu@gmail.com>
Date: Wed, 29 Jul 2020 13:58:12 +0200
Subject: [PATCH 0583/1035] [clangd] Fix an assertion failure in TargetFinder's
 heuristic resolution of dependent type.

The assertion is not true anymore after D82739, this patch just removes
it, and rename related functions.

And also fixes a missing cases.

Differential Revision: https://reviews.llvm.org/D84837
---
 clang-tools-extra/clangd/FindTarget.cpp       | 28 +++++++++----------
 .../clangd/unittests/FindTargetTests.cpp      | 14 ++++++++++
 2 files changed, 28 insertions(+), 14 deletions(-)

diff --git a/clang-tools-extra/clangd/FindTarget.cpp b/clang-tools-extra/clangd/FindTarget.cpp
index c6022b2463e85..a346d6b662e9a 100644
--- a/clang-tools-extra/clangd/FindTarget.cpp
+++ b/clang-tools-extra/clangd/FindTarget.cpp
@@ -163,13 +163,12 @@ const Type *getPointeeType(const Type *T) {
 }
 
 // Forward declaration, needed as this function is mutually recursive
-// with resolveDependentExprToDecls.
-const Type *resolveDependentExprToType(const Expr *E);
+// with resolveExprToDecls.
+const Type *resolveExprToType(const Expr *E);
 
-// Try to heuristically resolve a dependent expression `E` to one
+// Try to heuristically resolve a possibly-dependent expression `E` to one
 // or more declarations that it likely references.
-std::vector<const NamedDecl *> resolveDependentExprToDecls(const Expr *E) {
-  assert(E->isTypeDependent());
+std::vector<const NamedDecl *> resolveExprToDecls(const Expr *E) {
   if (const auto *ME = dyn_cast<CXXDependentScopeMemberExpr>(E)) {
     const Type *BaseType = ME->getBaseType().getTypePtrOrNull();
     if (ME->isArrow()) {
@@ -183,7 +182,7 @@ std::vector<const NamedDecl *> resolveDependentExprToDecls(const Expr *E) {
       // can get further by analyzing the depedent expression.
       Expr *Base = ME->isImplicitAccess() ? nullptr : ME->getBase();
       if (Base && BT->getKind() == BuiltinType::Dependent) {
-        BaseType = resolveDependentExprToType(Base);
+        BaseType = resolveExprToType(Base);
       }
     }
     return getMembersReferencedViaDependentName(
@@ -197,7 +196,7 @@ std::vector<const NamedDecl *> resolveDependentExprToDecls(const Expr *E) {
         /*IsNonstaticMember=*/false);
   }
   if (const auto *CE = dyn_cast<CallExpr>(E)) {
-    const auto *CalleeType = resolveDependentExprToType(CE->getCallee());
+    const auto *CalleeType = resolveExprToType(CE->getCallee());
     if (!CalleeType)
       return {};
     if (const auto *FnTypePtr = CalleeType->getAs<PointerType>())
@@ -209,15 +208,16 @@ std::vector<const NamedDecl *> resolveDependentExprToDecls(const Expr *E) {
       }
     }
   }
-  if (const auto *ME = dyn_cast<MemberExpr>(E)) {
+  if (const auto *ME = dyn_cast<MemberExpr>(E))
     return {ME->getMemberDecl()};
-  }
+  if (const auto *DRE = dyn_cast<DeclRefExpr>(E))
+    return {DRE->getFoundDecl()};
   return {};
 }
 
-// Try to heuristically resolve the type of a dependent expression `E`.
-const Type *resolveDependentExprToType(const Expr *E) {
-  std::vector<const NamedDecl *> Decls = resolveDependentExprToDecls(E);
+// Try to heuristically resolve the type of a possibly-dependent expression `E`.
+const Type *resolveExprToType(const Expr *E) {
+  std::vector<const NamedDecl *> Decls = resolveExprToDecls(E);
   if (Decls.size() != 1) // Names an overload set -- just bail.
     return nullptr;
   if (const auto *TD = dyn_cast<TypeDecl>(Decls[0])) {
@@ -426,12 +426,12 @@ struct TargetFinder {
       }
       void
       VisitCXXDependentScopeMemberExpr(const CXXDependentScopeMemberExpr *E) {
-        for (const NamedDecl *D : resolveDependentExprToDecls(E)) {
+        for (const NamedDecl *D : resolveExprToDecls(E)) {
           Outer.add(D, Flags);
         }
       }
       void VisitDependentScopeDeclRefExpr(const DependentScopeDeclRefExpr *E) {
-        for (const NamedDecl *D : resolveDependentExprToDecls(E)) {
+        for (const NamedDecl *D : resolveExprToDecls(E)) {
           Outer.add(D, Flags);
         }
       }
diff --git a/clang-tools-extra/clangd/unittests/FindTargetTests.cpp b/clang-tools-extra/clangd/unittests/FindTargetTests.cpp
index f7af77e8b57be..92095e871e201 100644
--- a/clang-tools-extra/clangd/unittests/FindTargetTests.cpp
+++ b/clang-tools-extra/clangd/unittests/FindTargetTests.cpp
@@ -627,6 +627,20 @@ TEST_F(TargetDeclTest, DependentExprs) {
         };
       )cpp";
   EXPECT_DECLS("CXXDependentScopeMemberExpr", "int aaaa");
+
+  Code = R"cpp(
+        class Foo {
+        public:
+          static Foo k(int);
+          template <typename T> T convert() const;
+        };
+        template <typename T>
+        void test() {
+          Foo::k(T()).template [[convert]]<T>();
+        }
+      )cpp";
+  EXPECT_DECLS("CXXDependentScopeMemberExpr",
+               "template <typename T> T convert() const");
 }
 
 TEST_F(TargetDeclTest, ObjC) {

From 647e9a54c758a6fdd85a569f019f00a653b2bc40 Mon Sep 17 00:00:00 2001
From: George Mitenkov <georgemitenk0v@gmail.com>
Date: Thu, 30 Jul 2020 09:22:44 +0300
Subject: [PATCH 0584/1035] [MLIR][SPIRVToLLVM] Conversion patterns for GLSL
 ops

This is the first patch that adds support for GLSL extended
instruction set ops. These are direct conversions, apart from `spv.Tan`
that is lowered to `sin() / cos()`.

Reviewed By: antiagainst

Differential Revision: https://reviews.llvm.org/D84627
---
 .../SPIRVToLLVM/ConvertSPIRVToLLVM.cpp        |  28 +++++
 .../SPIRVToLLVM/glsl-ops-to-llvm.mlir         | 105 ++++++++++++++++++
 2 files changed, 133 insertions(+)
 create mode 100644 mlir/test/Conversion/SPIRVToLLVM/glsl-ops-to-llvm.mlir

diff --git a/mlir/lib/Conversion/SPIRVToLLVM/ConvertSPIRVToLLVM.cpp b/mlir/lib/Conversion/SPIRVToLLVM/ConvertSPIRVToLLVM.cpp
index fd0e96b79d2b5..803b05a032dac 100644
--- a/mlir/lib/Conversion/SPIRVToLLVM/ConvertSPIRVToLLVM.cpp
+++ b/mlir/lib/Conversion/SPIRVToLLVM/ConvertSPIRVToLLVM.cpp
@@ -802,6 +802,25 @@ class ShiftPattern : public SPIRVToLLVMConversion<SPIRVOp> {
   }
 };
 
+class TanPattern : public SPIRVToLLVMConversion<spirv::GLSLTanOp> {
+public:
+  using SPIRVToLLVMConversion<spirv::GLSLTanOp>::SPIRVToLLVMConversion;
+
+  LogicalResult
+  matchAndRewrite(spirv::GLSLTanOp tanOp, ArrayRef<Value> operands,
+                  ConversionPatternRewriter &rewriter) const override {
+    auto dstType = typeConverter.convertType(tanOp.getType());
+    if (!dstType)
+      return failure();
+
+    Location loc = tanOp.getLoc();
+    Value sin = rewriter.create<LLVM::SinOp>(loc, dstType, tanOp.operand());
+    Value cos = rewriter.create<LLVM::CosOp>(loc, dstType, tanOp.operand());
+    rewriter.replaceOpWithNewOp<LLVM::FDivOp>(tanOp, dstType, sin, cos);
+    return success();
+  }
+};
+
 class VariablePattern : public SPIRVToLLVMConversion<spirv::VariableOp> {
 public:
   using SPIRVToLLVMConversion<spirv::VariableOp>::SPIRVToLLVMConversion;
@@ -1026,6 +1045,15 @@ void mlir::populateSPIRVToLLVMConversionPatterns(
       // Function Call op
       FunctionCallPattern,
 
+      // GLSL extended instruction set ops
+      DirectConversionPattern<spirv::GLSLCeilOp, LLVM::FCeilOp>,
+      DirectConversionPattern<spirv::GLSLCosOp, LLVM::CosOp>,
+      DirectConversionPattern<spirv::GLSLExpOp, LLVM::ExpOp>,
+      DirectConversionPattern<spirv::GLSLFAbsOp, LLVM::FAbsOp>,
+      DirectConversionPattern<spirv::GLSLLogOp, LLVM::LogOp>,
+      DirectConversionPattern<spirv::GLSLSinOp, LLVM::SinOp>,
+      DirectConversionPattern<spirv::GLSLSqrtOp, LLVM::SqrtOp>, TanPattern,
+
       // Logical ops
       DirectConversionPattern<spirv::LogicalAndOp, LLVM::AndOp>,
       DirectConversionPattern<spirv::LogicalOrOp, LLVM::OrOp>,
diff --git a/mlir/test/Conversion/SPIRVToLLVM/glsl-ops-to-llvm.mlir b/mlir/test/Conversion/SPIRVToLLVM/glsl-ops-to-llvm.mlir
new file mode 100644
index 0000000000000..1907619445c92
--- /dev/null
+++ b/mlir/test/Conversion/SPIRVToLLVM/glsl-ops-to-llvm.mlir
@@ -0,0 +1,105 @@
+// RUN: mlir-opt -convert-spirv-to-llvm %s | FileCheck %s
+
+//===----------------------------------------------------------------------===//
+// spv.GLSL.Ceil
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: @ceil
+func @ceil(%arg0: f32, %arg1: vector<3xf16>) {
+	// CHECK: "llvm.intr.ceil"(%{{.*}}) : (!llvm.float) -> !llvm.float
+	%0 = spv.GLSL.Ceil %arg0 : f32
+	// CHECK: "llvm.intr.ceil"(%{{.*}}) : (!llvm<"<3 x half>">) -> !llvm<"<3 x half>">
+	%1 = spv.GLSL.Ceil %arg1 : vector<3xf16>
+	return
+}
+
+//===----------------------------------------------------------------------===//
+// spv.GLSL.Cos
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: @cos
+func @cos(%arg0: f32, %arg1: vector<3xf16>) {
+	// CHECK: "llvm.intr.cos"(%{{.*}}) : (!llvm.float) -> !llvm.float
+	%0 = spv.GLSL.Cos %arg0 : f32
+	// CHECK: "llvm.intr.cos"(%{{.*}}) : (!llvm<"<3 x half>">) -> !llvm<"<3 x half>">
+	%1 = spv.GLSL.Cos %arg1 : vector<3xf16>
+	return
+}
+
+//===----------------------------------------------------------------------===//
+// spv.GLSL.Exp
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: @exp
+func @exp(%arg0: f32, %arg1: vector<3xf16>) {
+	// CHECK: "llvm.intr.exp"(%{{.*}}) : (!llvm.float) -> !llvm.float
+	%0 = spv.GLSL.Exp %arg0 : f32
+	// CHECK: "llvm.intr.exp"(%{{.*}}) : (!llvm<"<3 x half>">) -> !llvm<"<3 x half>">
+	%1 = spv.GLSL.Exp %arg1 : vector<3xf16>
+	return
+}
+
+//===----------------------------------------------------------------------===//
+// spv.GLSL.FAbs
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: @fabs
+func @fabs(%arg0: f32, %arg1: vector<3xf16>) {
+	// CHECK: "llvm.intr.fabs"(%{{.*}}) : (!llvm.float) -> !llvm.float
+	%0 = spv.GLSL.FAbs %arg0 : f32
+	// CHECK: "llvm.intr.fabs"(%{{.*}}) : (!llvm<"<3 x half>">) -> !llvm<"<3 x half>">
+	%1 = spv.GLSL.FAbs %arg1 : vector<3xf16>
+	return
+}
+
+//===----------------------------------------------------------------------===//
+// spv.GLSL.Log
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: @log
+func @log(%arg0: f32, %arg1: vector<3xf16>) {
+	// CHECK: "llvm.intr.log"(%{{.*}}) : (!llvm.float) -> !llvm.float
+	%0 = spv.GLSL.Log %arg0 : f32
+	// CHECK: "llvm.intr.log"(%{{.*}}) : (!llvm<"<3 x half>">) -> !llvm<"<3 x half>">
+	%1 = spv.GLSL.Log %arg1 : vector<3xf16>
+	return
+}
+
+//===----------------------------------------------------------------------===//
+// spv.GLSL.Sin
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: @sin
+func @sin(%arg0: f32, %arg1: vector<3xf16>) {
+	// CHECK: "llvm.intr.sin"(%{{.*}}) : (!llvm.float) -> !llvm.float
+	%0 = spv.GLSL.Sin %arg0 : f32
+	// CHECK: "llvm.intr.sin"(%{{.*}}) : (!llvm<"<3 x half>">) -> !llvm<"<3 x half>">
+	%1 = spv.GLSL.Sin %arg1 : vector<3xf16>
+	return
+}
+
+//===----------------------------------------------------------------------===//
+// spv.GLSL.Sqrt
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: @sqrt
+func @sqrt(%arg0: f32, %arg1: vector<3xf16>) {
+	// CHECK: "llvm.intr.sqrt"(%{{.*}}) : (!llvm.float) -> !llvm.float
+	%0 = spv.GLSL.Sqrt %arg0 : f32
+	// CHECK: "llvm.intr.sqrt"(%{{.*}}) : (!llvm<"<3 x half>">) -> !llvm<"<3 x half>">
+	%1 = spv.GLSL.Sqrt %arg1 : vector<3xf16>
+	return
+}
+
+//===----------------------------------------------------------------------===//
+// spv.GLSL.Tan
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: @tan
+func @tan(%arg0: f32) {
+	// CHECK: %[[SIN:.*]] = "llvm.intr.sin"(%{{.*}}) : (!llvm.float) -> !llvm.float
+  // CHECK: %[[COS:.*]] = "llvm.intr.cos"(%{{.*}}) : (!llvm.float) -> !llvm.float
+	// CHECK: llvm.fdiv %[[SIN]], %[[COS]] : !llvm.float
+	%0 = spv.GLSL.Tan %arg0 : f32
+	return
+}

From 73c12bd8ff1a9cd8375a357ea06f171e127ec1b8 Mon Sep 17 00:00:00 2001
From: Haojian Wu <hokein.wu@gmail.com>
Date: Thu, 30 Jul 2020 09:19:58 +0200
Subject: [PATCH 0585/1035] [Concepts] Fix a deserialization crash.

`TemplateTypeParmDecl::hasTypeConstraint` is not a safe guard for
checking `TemplateTypeParmDecl::getTypeConstraint()` result is null.

in somecases (e.g. implicit deduction guide templates synthesized from the
constructor, immediately-declared constraint is not formed because of an error),
hasTypeConstraint returns false, and getTypeConstraint returns a nullptr.

Fix https://bugs.llvm.org/show_bug.cgi?id=46790

Differential Revision: https://reviews.llvm.org/D84455
---
 clang/lib/Serialization/ASTReaderDecl.cpp  |  8 +++---
 clang/test/PCH/cxx2a-constraints-crash.cpp | 29 ++++++++++++++++++++++
 2 files changed, 34 insertions(+), 3 deletions(-)
 create mode 100644 clang/test/PCH/cxx2a-constraints-crash.cpp

diff --git a/clang/lib/Serialization/ASTReaderDecl.cpp b/clang/lib/Serialization/ASTReaderDecl.cpp
index 0b87161ddeea3..5413b28ffde25 100644
--- a/clang/lib/Serialization/ASTReaderDecl.cpp
+++ b/clang/lib/Serialization/ASTReaderDecl.cpp
@@ -2909,9 +2909,11 @@ static bool isSameTemplateParameter(const NamedDecl *X,
       return false;
     if (TX->hasTypeConstraint() != TY->hasTypeConstraint())
       return false;
-    if (TX->hasTypeConstraint()) {
-      const TypeConstraint *TXTC = TX->getTypeConstraint();
-      const TypeConstraint *TYTC = TY->getTypeConstraint();
+    const TypeConstraint *TXTC = TX->getTypeConstraint();
+    const TypeConstraint *TYTC = TY->getTypeConstraint();
+    if (!TXTC != !TYTC)
+      return false;
+    if (TXTC && TYTC) {
       if (TXTC->getNamedConcept() != TYTC->getNamedConcept())
         return false;
       if (TXTC->hasExplicitTemplateArgs() != TYTC->hasExplicitTemplateArgs())
diff --git a/clang/test/PCH/cxx2a-constraints-crash.cpp b/clang/test/PCH/cxx2a-constraints-crash.cpp
new file mode 100644
index 0000000000000..637c55f0c879c
--- /dev/null
+++ b/clang/test/PCH/cxx2a-constraints-crash.cpp
@@ -0,0 +1,29 @@
+// RUN: %clang_cc1 -std=c++2a -emit-pch %s -o %t
+// RUN: %clang_cc1 -std=c++2a -include-pch %t -verify %s
+
+// expected-no-diagnostics
+
+#ifndef HEADER
+#define HEADER
+
+template <typename T, typename U>
+concept not_same_as = true;
+
+template <int Kind>
+struct subrange {
+  template <not_same_as<int> R>
+  subrange(R) requires(Kind == 0);
+
+  template <not_same_as<int> R>
+  subrange(R) requires(Kind != 0);
+};
+
+template <typename R>
+subrange(R) -> subrange<42>;
+
+int main() {
+  int c;
+  subrange s(c);
+}
+
+#endif

From 23ad660b5d34930b2b5362f1bba63daee78f6aa4 Mon Sep 17 00:00:00 2001
From: David Sherwood <david.sherwood@arm.com>
Date: Tue, 28 Jul 2020 13:28:14 +0100
Subject: [PATCH 0586/1035] [SVE][CodeGen] At -O0 fallback to DAG ISel when
 translating alloca with scalable types

When building code at -O0 We weren't falling back to DAG ISel correctly
when encountering alloca instructions with scalable vector types. This
is because the alloca has no operands that are scalable. I've fixed this by
adding a check in AArch64ISelLowering::fallBackToDAGISel for alloca
instructions with scalable types.

Differential Revision: https://reviews.llvm.org/D84746
---
 llvm/lib/Target/AArch64/AArch64ISelLowering.cpp        | 5 +++++
 llvm/test/CodeGen/AArch64/GlobalISel/arm64-fallback.ll | 8 ++++++++
 2 files changed, 13 insertions(+)

diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index f080abd8e6271..8b533e97b928d 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -14975,6 +14975,11 @@ bool AArch64TargetLowering::fallBackToDAGISel(const Instruction &Inst) const {
     if (isa<ScalableVectorType>(Inst.getOperand(i)->getType()))
       return true;
 
+  if (const AllocaInst *AI = dyn_cast<AllocaInst>(&Inst)) {
+    if (isa<ScalableVectorType>(AI->getAllocatedType()))
+      return true;
+  }
+
   return false;
 }
 
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-fallback.ll b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-fallback.ll
index 8ac3e50ea0b74..f555856f34e58 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-fallback.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-fallback.ll
@@ -244,6 +244,14 @@ define i8 @scalable_call(i8* %addr) #1 {
   ret i8 %res
 }
 
+; FALLBACK-WITH-REPORT-ERR: remark: <unknown>:0:0: unable to translate instruction{{.*}}scalable_alloca
+; FALLBACK-WITH-REPORT-OUT-LABEL: scalable_alloca
+define void @scalable_alloca() #1 {
+  %local0 = alloca <vscale x 16 x i8>
+  load volatile <vscale x 16 x i8>, <vscale x 16 x i8>* %local0
+  ret void
+}
+
 ; FALLBACK-WITH-REPORT-ERR: remark: <unknown>:0:0: unable to translate instruction{{.*}}asm_indirect_output
 ; FALLBACK-WITH-REPORT-OUT-LABEL: asm_indirect_output
 define void @asm_indirect_output() {

From 1745ba41b196d80d8a6739dffcbb6f613d371f29 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bal=C3=A1zs=20K=C3=A9ri?= <1.int32@gmail.com>
Date: Thu, 30 Jul 2020 08:36:05 +0200
Subject: [PATCH 0587/1035] [Analyzer] Remove inclusion of uniqueing decl from
 diagnostic profile.

The uniqueing decl in PathDiagnostic is the declaration with the
uniqueing loc, as stated by documentation comments.
It is enough to include the uniqueing loc in the profile. It is possible
to have objects with different uniqueing decl but same location, at
least with templates. These belong to the same class and should have
same profile.

Reviewed By: vsavchenko, NoQ

Differential Revision: https://reviews.llvm.org/D84843
---
 clang/lib/Analysis/PathDiagnostic.cpp    |  1 -
 clang/test/Analysis/report-uniqueing.cpp | 16 ++++++++++++++++
 2 files changed, 16 insertions(+), 1 deletion(-)
 create mode 100644 clang/test/Analysis/report-uniqueing.cpp

diff --git a/clang/lib/Analysis/PathDiagnostic.cpp b/clang/lib/Analysis/PathDiagnostic.cpp
index 9aa3386129d7f..f80b99b998063 100644
--- a/clang/lib/Analysis/PathDiagnostic.cpp
+++ b/clang/lib/Analysis/PathDiagnostic.cpp
@@ -1134,7 +1134,6 @@ void PathDiagnosticPopUpPiece::Profile(llvm::FoldingSetNodeID &ID) const {
 void PathDiagnostic::Profile(llvm::FoldingSetNodeID &ID) const {
   ID.Add(getLocation());
   ID.Add(getUniqueingLoc());
-  ID.AddPointer(getUniqueingLoc().isValid() ? getUniqueingDecl() : nullptr);
   ID.AddString(BugType);
   ID.AddString(VerboseDesc);
   ID.AddString(Category);
diff --git a/clang/test/Analysis/report-uniqueing.cpp b/clang/test/Analysis/report-uniqueing.cpp
new file mode 100644
index 0000000000000..0e4d50e13a20d
--- /dev/null
+++ b/clang/test/Analysis/report-uniqueing.cpp
@@ -0,0 +1,16 @@
+// RUN: %clang_analyze_cc1 -verify %s \
+// RUN:   -analyzer-checker=security
+
+void bzero(void *, unsigned long);
+
+template <typename T> void foo(T l) {
+  // The warning comes from multiple instances and with
+  // different declarations that have same source location.
+  // One instance should be shown.
+  bzero(l, 1); // expected-warning{{The bzero() function is obsoleted}}
+}
+
+void p(int *p, unsigned *q) {
+  foo(p);
+  foo(q);
+}

From 3aab320557e7441bc2ce0b51fd6d82838fd0d484 Mon Sep 17 00:00:00 2001
From: George Mitenkov <georgemitenk0v@gmail.com>
Date: Thu, 30 Jul 2020 10:24:10 +0300
Subject: [PATCH 0588/1035] [MLIR][SPIRVToLLVM] Conversion for inverse sqrt and
 tanh

This is a second patch on conversion of GLSL ops to LLVM dialect.
It introduces patterns to convert `spv.InverseSqrt` and `spv.Tanh`.

Reviewed By: antiagainst

Differential Revision: https://reviews.llvm.org/D84633
---
 .../SPIRVToLLVM/ConvertSPIRVToLLVM.cpp        | 73 ++++++++++++++++++-
 .../SPIRVToLLVM/glsl-ops-to-llvm.mlir         | 30 ++++++++
 2 files changed, 102 insertions(+), 1 deletion(-)

diff --git a/mlir/lib/Conversion/SPIRVToLLVM/ConvertSPIRVToLLVM.cpp b/mlir/lib/Conversion/SPIRVToLLVM/ConvertSPIRVToLLVM.cpp
index 803b05a032dac..58d160d30a494 100644
--- a/mlir/lib/Conversion/SPIRVToLLVM/ConvertSPIRVToLLVM.cpp
+++ b/mlir/lib/Conversion/SPIRVToLLVM/ConvertSPIRVToLLVM.cpp
@@ -91,6 +91,21 @@ static Value createConstantAllBitsSet(Location loc, Type srcType, Type dstType,
       loc, dstType, minusOneIntegerAttribute(srcType, rewriter));
 }
 
+/// Creates `llvm.mlir.constant` with a floating-point scalar or vector value.
+static Value createFPConstant(Location loc, Type srcType, Type dstType,
+                              PatternRewriter &rewriter, double value) {
+  if (auto vecType = srcType.dyn_cast<VectorType>()) {
+    auto floatType = vecType.getElementType().cast<FloatType>();
+    return rewriter.create<LLVM::ConstantOp>(
+        loc, dstType,
+        SplatElementsAttr::get(vecType,
+                               rewriter.getFloatAttr(floatType, value)));
+  }
+  auto floatType = srcType.cast<FloatType>();
+  return rewriter.create<LLVM::ConstantOp>(
+      loc, dstType, rewriter.getFloatAttr(floatType, value));
+}
+
 /// Utility function for bitfiled ops:
 ///   - `BitFieldInsert`
 ///   - `BitFieldSExtract`
@@ -590,6 +605,27 @@ class IComparePattern : public SPIRVToLLVMConversion<SPIRVOp> {
   }
 };
 
+class InverseSqrtPattern
+    : public SPIRVToLLVMConversion<spirv::GLSLInverseSqrtOp> {
+public:
+  using SPIRVToLLVMConversion<spirv::GLSLInverseSqrtOp>::SPIRVToLLVMConversion;
+
+  LogicalResult
+  matchAndRewrite(spirv::GLSLInverseSqrtOp op, ArrayRef<Value> operands,
+                  ConversionPatternRewriter &rewriter) const override {
+    auto srcType = op.getType();
+    auto dstType = typeConverter.convertType(srcType);
+    if (!dstType)
+      return failure();
+
+    Location loc = op.getLoc();
+    Value one = createFPConstant(loc, srcType, dstType, rewriter, 1.0);
+    Value sqrt = rewriter.create<LLVM::SqrtOp>(loc, dstType, op.operand());
+    rewriter.replaceOpWithNewOp<LLVM::FDivOp>(op, dstType, one, sqrt);
+    return success();
+  }
+};
+
 /// Converts `spv.Load` and `spv.Store` to LLVM dialect.
 template <typename SPIRVop>
 class LoadStorePattern : public SPIRVToLLVMConversion<SPIRVop> {
@@ -821,6 +857,40 @@ class TanPattern : public SPIRVToLLVMConversion<spirv::GLSLTanOp> {
   }
 };
 
+/// Convert `spv.Tanh` to
+///
+///   exp(2x) - 1
+///   -----------
+///   exp(2x) + 1
+///
+class TanhPattern : public SPIRVToLLVMConversion<spirv::GLSLTanhOp> {
+public:
+  using SPIRVToLLVMConversion<spirv::GLSLTanhOp>::SPIRVToLLVMConversion;
+
+  LogicalResult
+  matchAndRewrite(spirv::GLSLTanhOp tanhOp, ArrayRef<Value> operands,
+                  ConversionPatternRewriter &rewriter) const override {
+    auto srcType = tanhOp.getType();
+    auto dstType = typeConverter.convertType(srcType);
+    if (!dstType)
+      return failure();
+
+    Location loc = tanhOp.getLoc();
+    Value two = createFPConstant(loc, srcType, dstType, rewriter, 2.0);
+    Value multiplied =
+        rewriter.create<LLVM::FMulOp>(loc, dstType, two, tanhOp.operand());
+    Value exponential = rewriter.create<LLVM::ExpOp>(loc, dstType, multiplied);
+    Value one = createFPConstant(loc, srcType, dstType, rewriter, 1.0);
+    Value numerator =
+        rewriter.create<LLVM::FSubOp>(loc, dstType, exponential, one);
+    Value denominator =
+        rewriter.create<LLVM::FAddOp>(loc, dstType, exponential, one);
+    rewriter.replaceOpWithNewOp<LLVM::FDivOp>(tanhOp, dstType, numerator,
+                                              denominator);
+    return success();
+  }
+};
+
 class VariablePattern : public SPIRVToLLVMConversion<spirv::VariableOp> {
 public:
   using SPIRVToLLVMConversion<spirv::VariableOp>::SPIRVToLLVMConversion;
@@ -1052,7 +1122,8 @@ void mlir::populateSPIRVToLLVMConversionPatterns(
       DirectConversionPattern<spirv::GLSLFAbsOp, LLVM::FAbsOp>,
       DirectConversionPattern<spirv::GLSLLogOp, LLVM::LogOp>,
       DirectConversionPattern<spirv::GLSLSinOp, LLVM::SinOp>,
-      DirectConversionPattern<spirv::GLSLSqrtOp, LLVM::SqrtOp>, TanPattern,
+      DirectConversionPattern<spirv::GLSLSqrtOp, LLVM::SqrtOp>,
+      InverseSqrtPattern, TanPattern, TanhPattern,
 
       // Logical ops
       DirectConversionPattern<spirv::LogicalAndOp, LLVM::AndOp>,
diff --git a/mlir/test/Conversion/SPIRVToLLVM/glsl-ops-to-llvm.mlir b/mlir/test/Conversion/SPIRVToLLVM/glsl-ops-to-llvm.mlir
index 1907619445c92..ab501b8aabfd0 100644
--- a/mlir/test/Conversion/SPIRVToLLVM/glsl-ops-to-llvm.mlir
+++ b/mlir/test/Conversion/SPIRVToLLVM/glsl-ops-to-llvm.mlir
@@ -103,3 +103,33 @@ func @tan(%arg0: f32) {
 	%0 = spv.GLSL.Tan %arg0 : f32
 	return
 }
+
+//===----------------------------------------------------------------------===//
+// spv.GLSL.Tanh
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: @tanh
+func @tanh(%arg0: f32) {
+	// CHECK: %[[TWO:.*]] = llvm.mlir.constant(2.000000e+00 : f32) : !llvm.float
+  // CHECK: %[[X2:.*]] = llvm.fmul %[[TWO]], %{{.*}} : !llvm.float
+  // CHECK: %[[EXP:.*]] = "llvm.intr.exp"(%[[X2]]) : (!llvm.float) -> !llvm.float
+  // CHECK: %[[ONE:.*]] = llvm.mlir.constant(1.000000e+00 : f32) : !llvm.float
+  // CHECK: %[[T0:.*]] = llvm.fsub %[[EXP]], %[[ONE]] : !llvm.float
+  // CHECK: %[[T1:.*]] = llvm.fadd %[[EXP]], %[[ONE]] : !llvm.float
+  // CHECK: llvm.fdiv %[[T0]], %[[T1]] : !llvm.float
+	%0 = spv.GLSL.Tanh %arg0 : f32
+	return
+}
+
+//===----------------------------------------------------------------------===//
+// spv.GLSL.InverseSqrt
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: @inverse_sqrt
+func @inverse_sqrt(%arg0: f32) {
+	// CHECK: %[[ONE:.*]] = llvm.mlir.constant(1.000000e+00 : f32) : !llvm.float
+  // CHECK: %[[SQRT:.*]] = "llvm.intr.sqrt"(%{{.*}}) : (!llvm.float) -> !llvm.float
+	// CHECK: llvm.fdiv %[[ONE]], %[[SQRT]] : !llvm.float
+	%0 = spv.GLSL.InverseSqrt %arg0 : f32
+	return
+}

From 0037a5f894345eef5552d6620548cc8ad5900b41 Mon Sep 17 00:00:00 2001
From: Kang Zhang <shkzhang@cn.ibm.com>
Date: Thu, 30 Jul 2020 08:18:50 +0000
Subject: [PATCH 0589/1035] [PHIElimination] Fix the killed flag for
 LowerPHINode()

Summary:
In the phi-node-elimination pass, we set the killed flag incorrectly.
When we eliminate the PHI node, we replace the PHI with a copy for the
incoming value.

Before this patch, we will set incoming value as killed(PHICopy). And
we will remove the killed flag from last using incoming value(OldKill).
This is correct, only if the new PHICopy is after the OldKill.

Reviewed By: bjope

Differential Revision: https://reviews.llvm.org/D80886
---
 llvm/lib/CodeGen/PHIElimination.cpp         |  48 +++-
 llvm/test/CodeGen/PowerPC/phi-eliminate.mir | 295 ++++++++++++++++++++
 2 files changed, 330 insertions(+), 13 deletions(-)
 create mode 100644 llvm/test/CodeGen/PowerPC/phi-eliminate.mir

diff --git a/llvm/lib/CodeGen/PHIElimination.cpp b/llvm/lib/CodeGen/PHIElimination.cpp
index 311b87fa9e3b0..521ae367cde33 100644
--- a/llvm/lib/CodeGen/PHIElimination.cpp
+++ b/llvm/lib/CodeGen/PHIElimination.cpp
@@ -324,21 +324,43 @@ void PHIElimination::LowerPHINode(MachineBasicBlock &MBB,
       // Increment use count of the newly created virtual register.
       LV->setPHIJoin(IncomingReg);
 
-      // When we are reusing the incoming register, it may already have been
-      // killed in this block. The old kill will also have been inserted at
-      // AfterPHIsIt, so it appears before the current PHICopy.
-      if (reusedIncoming)
-        if (MachineInstr *OldKill = VI.findKill(&MBB)) {
-          LLVM_DEBUG(dbgs() << "Remove old kill from " << *OldKill);
-          LV->removeVirtualRegisterKilled(IncomingReg, *OldKill);
-          LLVM_DEBUG(MBB.dump());
+      MachineInstr *OldKill = nullptr;
+      bool IsPHICopyAfterOldKill = false;
+
+      if (reusedIncoming && (OldKill = VI.findKill(&MBB))) {
+        // Calculate whether the PHICopy is after the OldKill.
+        // In general, the PHICopy is inserted as the first non-phi instruction
+        // by default, so it's before the OldKill. But some Target hooks for
+        // createPHIDestinationCopy() may modify the default insert position of
+        // PHICopy.
+        for (auto I = MBB.SkipPHIsAndLabels(MBB.begin()), E = MBB.end();
+             I != E; ++I) {
+          if (I == PHICopy)
+            break;
+
+          if (I == OldKill) {
+            IsPHICopyAfterOldKill = true;
+            break;
+          }
         }
+      }
+
+      // When we are reusing the incoming register and it has been marked killed
+      // by OldKill, if the PHICopy is after the OldKill, we should remove the
+      // killed flag from OldKill.
+      if (IsPHICopyAfterOldKill) {
+        LLVM_DEBUG(dbgs() << "Remove old kill from " << *OldKill);
+        LV->removeVirtualRegisterKilled(IncomingReg, *OldKill);
+        LLVM_DEBUG(MBB.dump());
+      }
 
-      // Add information to LiveVariables to know that the incoming value is
-      // killed.  Note that because the value is defined in several places (once
-      // each for each incoming block), the "def" block and instruction fields
-      // for the VarInfo is not filled in.
-      LV->addVirtualRegisterKilled(IncomingReg, *PHICopy);
+      // Add information to LiveVariables to know that the first used incoming
+      // value or the resued incoming value whose PHICopy is after the OldKIll
+      // is killed. Note that because the value is defined in several places
+      // (once each for each incoming block), the "def" block and instruction
+      // fields for the VarInfo is not filled in.
+      if (!OldKill || IsPHICopyAfterOldKill)
+        LV->addVirtualRegisterKilled(IncomingReg, *PHICopy);
     }
 
     // Since we are going to be deleting the PHI node, if it is the last use of
diff --git a/llvm/test/CodeGen/PowerPC/phi-eliminate.mir b/llvm/test/CodeGen/PowerPC/phi-eliminate.mir
new file mode 100644
index 0000000000000..e74ca4ca0e5d9
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/phi-eliminate.mir
@@ -0,0 +1,295 @@
+# RUN: llc -mtriple=powerpc64le-unknown-linux-gnu -mcpu=pwr9 %s -o - \
+# RUN:   -run-pass=livevars,phi-node-elimination | FileCheck %s
+
+--- |
+  define void @phi_eliminate(i32 %0, i32 %1, i8* %2) {
+    %scevgep3 = getelementptr i8, i8* %2, i64 undef
+    call void @llvm.set.loop.iterations.i64(i64 undef)
+    br label %4
+
+  4:                                                ; preds = %4, %3
+    %5 = phi i32 [ %8, %4 ], [ %0, %3 ]
+    %6 = phi i8* [ %scevgep3, %3 ], [ %7, %4 ]
+    %7 = getelementptr i8, i8* %6, i64 -1
+    %8 = sdiv i32 %5, %1
+    %9 = mul nsw i32 %8, %1
+    %10 = sub nsw i32 %5, %9
+    %11 = icmp ult i32 %10, 10
+    %12 = trunc i32 %10 to i8
+    %13 = select i1 %11, i8 48, i8 55
+    %14 = add i8 %13, %12
+    store i8 %14, i8* %7, align 1
+    %15 = call i1 @llvm.loop.decrement.i64(i64 1)
+    br i1 %15, label %4, label %16
+
+  16:                                               ; preds = %4
+    ret void
+  }
+
+  declare void @llvm.set.loop.iterations.i64(i64)
+
+  declare i1 @llvm.loop.decrement.i64(i64)
+
+  declare void @llvm.stackprotector(i8*, i8**)
+...
+---
+name:            phi_eliminate
+alignment:       16
+exposesReturnsTwice: false
+legalized:       false
+regBankSelected: false
+selected:        false
+failedISel:      false
+tracksRegLiveness: true
+hasWinCFI:       false
+registers:
+  - { id: 0, class: g8rc_and_g8rc_nox0, preferred-register: '' }
+  - { id: 1, class: gprc, preferred-register: '' }
+  - { id: 2, class: g8rc_and_g8rc_nox0, preferred-register: '' }
+  - { id: 3, class: g8rc, preferred-register: '' }
+  - { id: 4, class: gprc, preferred-register: '' }
+  - { id: 5, class: g8rc, preferred-register: '' }
+  - { id: 6, class: g8rc, preferred-register: '' }
+  - { id: 7, class: g8rc, preferred-register: '' }
+  - { id: 8, class: gprc, preferred-register: '' }
+  - { id: 9, class: gprc, preferred-register: '' }
+  - { id: 10, class: g8rc, preferred-register: '' }
+  - { id: 11, class: gprc, preferred-register: '' }
+  - { id: 12, class: gprc, preferred-register: '' }
+  - { id: 13, class: crrc, preferred-register: '' }
+  - { id: 14, class: gprc_and_gprc_nor0, preferred-register: '' }
+  - { id: 15, class: gprc_and_gprc_nor0, preferred-register: '' }
+  - { id: 16, class: gprc, preferred-register: '' }
+  - { id: 17, class: gprc, preferred-register: '' }
+  - { id: 18, class: g8rc_and_g8rc_nox0, preferred-register: '' }
+  - { id: 19, class: gprc, preferred-register: '' }
+  - { id: 20, class: gprc, preferred-register: '' }
+  - { id: 21, class: gprc, preferred-register: '' }
+  - { id: 22, class: crrc, preferred-register: '' }
+  - { id: 23, class: gprc, preferred-register: '' }
+  - { id: 24, class: gprc, preferred-register: '' }
+  - { id: 25, class: g8rc_and_g8rc_nox0, preferred-register: '' }
+  - { id: 26, class: gprc, preferred-register: '' }
+  - { id: 27, class: g8rc_and_g8rc_nox0, preferred-register: '' }
+  - { id: 28, class: gprc, preferred-register: '' }
+  - { id: 29, class: gprc, preferred-register: '' }
+  - { id: 30, class: gprc, preferred-register: '' }
+  - { id: 31, class: crrc, preferred-register: '' }
+  - { id: 32, class: gprc, preferred-register: '' }
+  - { id: 33, class: gprc, preferred-register: '' }
+  - { id: 34, class: g8rc_and_g8rc_nox0, preferred-register: '' }
+  - { id: 35, class: gprc, preferred-register: '' }
+  - { id: 36, class: gprc, preferred-register: '' }
+  - { id: 37, class: gprc, preferred-register: '' }
+  - { id: 38, class: g8rc_and_g8rc_nox0, preferred-register: '' }
+  - { id: 39, class: g8rc, preferred-register: '' }
+  - { id: 40, class: gprc, preferred-register: '' }
+  - { id: 41, class: gprc, preferred-register: '' }
+  - { id: 42, class: g8rc_and_g8rc_nox0, preferred-register: '' }
+  - { id: 43, class: gprc, preferred-register: '' }
+  - { id: 44, class: g8rc_and_g8rc_nox0, preferred-register: '' }
+  - { id: 45, class: gprc, preferred-register: '' }
+  - { id: 46, class: gprc, preferred-register: '' }
+  - { id: 47, class: crrc, preferred-register: '' }
+  - { id: 48, class: gprc, preferred-register: '' }
+  - { id: 49, class: gprc, preferred-register: '' }
+  - { id: 50, class: g8rc_and_g8rc_nox0, preferred-register: '' }
+  - { id: 51, class: g8rc, preferred-register: '' }
+  - { id: 52, class: gprc, preferred-register: '' }
+  - { id: 53, class: gprc, preferred-register: '' }
+  - { id: 54, class: g8rc_and_g8rc_nox0, preferred-register: '' }
+  - { id: 55, class: gprc, preferred-register: '' }
+  - { id: 56, class: gprc, preferred-register: '' }
+liveins:
+  - { reg: '$x3', virtual-reg: '%5' }
+  - { reg: '$x4', virtual-reg: '%6' }
+frameInfo:
+  isFrameAddressTaken: false
+  isReturnAddressTaken: false
+  hasStackMap:     false
+  hasPatchPoint:   false
+  stackSize:       0
+  offsetAdjustment: 0
+  maxAlignment:    1
+  adjustsStack:    false
+  hasCalls:        false
+  stackProtector:  ''
+  maxCallFrameSize: 4294967295
+  cvBytesOfCalleeSavedRegisters: 0
+  hasOpaqueSPAdjustment: false
+  hasVAStart:      false
+  hasMustTailInVarArgFunc: false
+  localFrameSize:  0
+  savePoint:       ''
+  restorePoint:    ''
+fixedStack:      []
+stack:           []
+callSites:       []
+constants:       []
+machineFunctionInfo: {}
+body:             |
+  ; CHECK-LABEL: name: phi_eliminate
+  ; CHECK: bb.0 (%ir-block.3):
+  ; CHECK:   successors: %bb.1(0x80000000)
+  ; CHECK:   liveins: $x3, $x4
+  ; CHECK:   %6:g8rc = COPY killed $x4
+  ; CHECK:   %5:g8rc = COPY killed $x3
+  ; CHECK:   %9:gprc = COPY killed %6.sub_32
+  ; CHECK:   %8:gprc = COPY killed %5.sub_32
+  ; CHECK:   MTCTR8loop undef %10:g8rc, implicit-def dead $ctr8
+  ; CHECK:   %14:gprc_and_gprc_nor0 = LI 55
+  ; CHECK:   %15:gprc_and_gprc_nor0 = LI 48
+
+  ; CHECK: bb.1 (%ir-block.4):
+  ; CHECK:   successors: %bb.2(0x40000000), %bb.7(0x40000000)
+  ; CHECK:   %19:gprc = DIVW %8, %9
+  ; CHECK:   BDNZ8 %bb.2, implicit-def $ctr8, implicit $ctr8
+
+  ; CHECK: bb.7:
+  ; CHECK:   successors: %bb.5(0x80000000)
+  ; CHECK:   %61:gprc = COPY killed %8
+  ; CHECK:   %62:g8rc_and_g8rc_nox0 = IMPLICIT_DEF
+  ; CHECK:   %63:gprc = COPY killed %19
+  ; CHECK:   B %bb.5
+
+  ; CHECK: bb.2 (%ir-block.4):
+  ; CHECK:   successors: %bb.3(0x40000000), %bb.4(0x40000000)
+  ; CHECK:   %20:gprc = nsw MULLW %19, %9
+  ; CHECK:   %21:gprc = SUBF killed %20, killed %8
+  ; CHECK:   %22:crrc = CMPLWI %21, 10
+  ; CHECK:   %23:gprc = ISEL %15, %14, killed %22.sub_lt
+  ; CEHCK:   %24:gprc = ADD4 killed %23, killed %21
+  ; CHECK:   %25:g8rc_and_g8rc_nox0 = STBU killed %24, -1, undef %0:g8rc_and_g8rc_nox0 :: (store 1 into %ir.7)
+  ; CHECK:   %26:gprc = DIVW %19, %9
+  ; CHECK:   %57:gprc = COPY killed %26
+  ; CHECK:   %58:gprc = COPY %19
+  ; CHECK:   %59:g8rc_and_g8rc_nox0 = COPY killed %25
+  ; CHECK:   %60:gprc = COPY killed %19
+  ; CHECK:   BDZ8 %bb.4, implicit-def $ctr8, implicit $ctr8
+  ; CHECK:   B %bb.3
+
+  ; CHECK: bb.3 (%ir-block.4):
+  ; CHECK:   successors: %bb.3(0x7c000000), %bb.4(0x04000000)
+  ; CHECK:   %38:g8rc_and_g8rc_nox0 = COPY killed %59
+  ; CHECK:   %37:gprc = COPY %57
+  ; CHECK:   %36:gprc = COPY killed %58
+  ; CHECK:   %35:gprc = COPY killed %57
+  ; CHECK:   %27:g8rc_and_g8rc_nox0 = COPY killed %38
+  ; CHECK:   %56:gprc = COPY %35
+  ; CHECK:   %28:gprc = DIVW %56, %9
+  ; CHECK:   %29:gprc = nsw MULLW killed %37, %9
+  ; CHECK:   %30:gprc = SUBF killed %29, killed %36
+  ; CHECK:   %31:crrc = CMPLWI %30, 10
+  ; CHECK:   %32:gprc = ISEL %15, %14, killed %31.sub_lt
+  ; CHECK:   %33:gprc = ADD4 killed %32, killed %30
+  ; CHECK:   %34:g8rc_and_g8rc_nox0 = STBU killed %33, -1, killed %27 :: (store unknown-size into %ir.7, align 1)
+  ; CHECK:   %57:gprc = COPY killed %28
+  ; CHECK:   %58:gprc = COPY killed %35
+  ; CHECK:   %59:g8rc_and_g8rc_nox0 = COPY killed %34
+  ; CHECK:   %60:gprc = COPY killed %56
+  ; CHECK:   BDNZ8 %bb.3, implicit-def $ctr8, implicit $ctr8
+  ; CHECK:   B %bb.4
+
+  ; CHECK: bb.4:
+  ; CHECK:   successors: %bb.5(0x80000000)
+  ; CHECK:   %44:g8rc_and_g8rc_nox0 = COPY killed %59
+  ; CHECK:   %43:gprc = COPY killed %57
+  ; CHECK:   %41:gprc = COPY killed %60
+  ; CHECK:   %39:g8rc = COPY killed %44
+  ; CHECK:   %61:gprc = COPY killed %41
+  ; CHECK:   %62:g8rc_and_g8rc_nox0 = COPY killed %39
+  ; CHECK:   %63:gprc = COPY killed %43
+
+  ; CHECK: bb.5:
+  ; CHECK:   successors: %bb.6(0x80000000)
+  ; CHECK:   %55:gprc = COPY killed %63
+  ; CHECK:   %54:g8rc_and_g8rc_nox0 = COPY killed %62
+  ; CHECK:   %53:gprc = COPY killed %61
+  ; CHECK:   %45:gprc = nsw MULLW killed %55, killed %9
+  ; CHECK:   %46:gprc = SUBF killed %45, killed %53
+  ; CHECK:   %47:crrc = CMPLWI %46, 10
+  ; CHECK:   %48:gprc = ISEL killed %15, killed %14, killed %47.sub_lt
+  ; CHECK:   %49:gprc = ADD4 killed %48, killed %46
+  ; CHECK:   dead %50:g8rc_and_g8rc_nox0 = STBU killed %49, -1, killed %54 :: (store unknown-size into %ir.7, align 1)
+  ; CHECK:   B %bb.6
+
+  ; CHECK: bb.6 (%ir-block.16):
+  ; CHECK:   BLR8 implicit $lr8, implicit $rm
+
+  bb.0 (%ir-block.3):
+    successors: %bb.1(0x80000000)
+    liveins: $x3, $x4
+
+    %6:g8rc = COPY killed $x4
+    %5:g8rc = COPY killed $x3
+    %9:gprc = COPY killed %6.sub_32
+    %8:gprc = COPY killed %5.sub_32
+    MTCTR8loop undef %10:g8rc, implicit-def dead $ctr8
+    %14:gprc_and_gprc_nor0 = LI 55
+    %15:gprc_and_gprc_nor0 = LI 48
+
+  bb.1 (%ir-block.4):
+    successors: %bb.2(0x40000000), %bb.5(0x40000000)
+
+    %19:gprc = DIVW %8, %9
+    BDZ8 %bb.5, implicit-def $ctr8, implicit $ctr8
+    B %bb.2
+
+  bb.2 (%ir-block.4):
+    successors: %bb.3(0x40000000), %bb.4(0x40000000)
+
+    %20:gprc = nsw MULLW %19, %9
+    %21:gprc = SUBF killed %20, killed %8
+    %22:crrc = CMPLWI %21, 10
+    %23:gprc = ISEL %15, %14, killed %22.sub_lt
+    %24:gprc = ADD4 killed %23, killed %21
+    %25:g8rc_and_g8rc_nox0 = STBU killed %24, -1, undef %0:g8rc_and_g8rc_nox0 :: (store 1 into %ir.7)
+    %26:gprc = DIVW %19, %9
+    BDZ8 %bb.4, implicit-def $ctr8, implicit $ctr8
+    B %bb.3
+
+  bb.3 (%ir-block.4):
+    successors: %bb.3(0x7c000000), %bb.4(0x04000000)
+
+    %35:gprc = PHI %26, %bb.2, %28, %bb.3
+    %36:gprc = PHI %19, %bb.2, %35, %bb.3
+    %37:gprc = PHI %26, %bb.2, %28, %bb.3
+    %38:g8rc_and_g8rc_nox0 = PHI %25, %bb.2, %34, %bb.3
+    %27:g8rc_and_g8rc_nox0 = COPY killed %38
+    %56:gprc = COPY %35
+    %28:gprc = DIVW %56, %9
+    %29:gprc = nsw MULLW killed %37, %9
+    %30:gprc = SUBF killed %29, killed %36
+    %31:crrc = CMPLWI %30, 10
+    %32:gprc = ISEL %15, %14, killed %31.sub_lt
+    %33:gprc = ADD4 killed %32, killed %30
+    %34:g8rc_and_g8rc_nox0 = STBU killed %33, -1, killed %27 :: (store unknown-size into %ir.7, align 1)
+    BDNZ8 %bb.3, implicit-def $ctr8, implicit $ctr8
+    B %bb.4
+
+  bb.4:
+    successors: %bb.5(0x80000000)
+
+    %41:gprc = PHI %19, %bb.2, %56, %bb.3
+    %43:gprc = PHI %26, %bb.2, %28, %bb.3
+    %44:g8rc_and_g8rc_nox0 = PHI %25, %bb.2, %34, %bb.3
+    %39:g8rc = COPY killed %44
+
+  bb.5:
+    successors: %bb.6(0x80000000)
+
+    %53:gprc = PHI %8, %bb.1, %41, %bb.4
+    %54:g8rc_and_g8rc_nox0 = PHI undef %0:g8rc_and_g8rc_nox0, %bb.1, %39, %bb.4
+    %55:gprc = PHI %19, %bb.1, %43, %bb.4
+    %45:gprc = nsw MULLW killed %55, killed %9
+    %46:gprc = SUBF killed %45, killed %53
+    %47:crrc = CMPLWI %46, 10
+    %48:gprc = ISEL killed %15, killed %14, killed %47.sub_lt
+    %49:gprc = ADD4 killed %48, killed %46
+    dead %50:g8rc_and_g8rc_nox0 = STBU killed %49, -1, killed %54 :: (store unknown-size into %ir.7, align 1)
+    B %bb.6
+
+  bb.6 (%ir-block.16):
+    BLR8 implicit $lr8, implicit $rm
+
+...

From 18805320365314ecbbebc65279c142a12ae9c0c2 Mon Sep 17 00:00:00 2001
From: George Mitenkov <georgemitenk0v@gmail.com>
Date: Thu, 30 Jul 2020 10:53:06 +0300
Subject: [PATCH 0590/1035] [MLIR][SPIRVToLLVM] Conversion of GLSL ops to LLVM
 intrinsics

This patch introduces new intrinsics in LLVM dialect:
-  `llvm.intr.floor`
-  `llvm.intr.maxnum`
-  `llvm.intr.minnum`
-  `llvm.intr.smax`
-  `llvm.intr.smin`
These intrinsics correspond to SPIR-V ops from GLSL
extended instruction set (`spv.GLSL.Floor`, `spv.GLSL.FMax`,
`spv.GLSL.FMin`,  `spv.GLSL.SMax` and `spv.GLSL.SMin`
respectively). Also conversion patterns for them were added.

Reviewed By: antiagainst

Differential Revision: https://reviews.llvm.org/D84661
---
 mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td   |  5 ++
 .../SPIRVToLLVM/ConvertSPIRVToLLVM.cpp        |  5 ++
 .../SPIRVToLLVM/glsl-ops-to-llvm.mlir         | 65 +++++++++++++++++++
 mlir/test/Target/llvmir-intrinsics.mlir       | 45 +++++++++++++
 4 files changed, 120 insertions(+)

diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
index 4da90575524b7..4b1a6efe002f2 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
@@ -852,6 +852,7 @@ def LLVM_ExpOp : LLVM_UnaryIntrinsicOp<"exp">;
 def LLVM_Exp2Op : LLVM_UnaryIntrinsicOp<"exp2">;
 def LLVM_FAbsOp : LLVM_UnaryIntrinsicOp<"fabs">;
 def LLVM_FCeilOp : LLVM_UnaryIntrinsicOp<"ceil">;
+def LLVM_FFloorOp : LLVM_UnaryIntrinsicOp<"floor">;
 def LLVM_FMAOp : LLVM_TernarySameArgsIntrinsicOp<"fma">;
 def LLVM_FMulAddOp : LLVM_TernarySameArgsIntrinsicOp<"fmuladd">;
 def LLVM_Log10Op : LLVM_UnaryIntrinsicOp<"log10">;
@@ -865,6 +866,10 @@ def LLVM_SqrtOp : LLVM_UnaryIntrinsicOp<"sqrt">;
 def LLVM_PowOp : LLVM_BinarySameArgsIntrinsicOp<"pow">;
 def LLVM_BitReverseOp : LLVM_UnaryIntrinsicOp<"bitreverse">;
 def LLVM_CtPopOp : LLVM_UnaryIntrinsicOp<"ctpop">;
+def LLVM_MaxNumOp : LLVM_BinarySameArgsIntrinsicOp<"maxnum">;
+def LLVM_MinNumOp : LLVM_BinarySameArgsIntrinsicOp<"minnum">;
+def LLVM_SMaxOp : LLVM_BinarySameArgsIntrinsicOp<"smax">;
+def LLVM_SMinOp : LLVM_BinarySameArgsIntrinsicOp<"smin">;
 
 def LLVM_MemcpyOp : LLVM_ZeroResultIntrOp<"memcpy", [0, 1, 2]>,
                     Arguments<(ins LLVM_Type:$dst, LLVM_Type:$src,
diff --git a/mlir/lib/Conversion/SPIRVToLLVM/ConvertSPIRVToLLVM.cpp b/mlir/lib/Conversion/SPIRVToLLVM/ConvertSPIRVToLLVM.cpp
index 58d160d30a494..45b9f41121514 100644
--- a/mlir/lib/Conversion/SPIRVToLLVM/ConvertSPIRVToLLVM.cpp
+++ b/mlir/lib/Conversion/SPIRVToLLVM/ConvertSPIRVToLLVM.cpp
@@ -1120,8 +1120,13 @@ void mlir::populateSPIRVToLLVMConversionPatterns(
       DirectConversionPattern<spirv::GLSLCosOp, LLVM::CosOp>,
       DirectConversionPattern<spirv::GLSLExpOp, LLVM::ExpOp>,
       DirectConversionPattern<spirv::GLSLFAbsOp, LLVM::FAbsOp>,
+      DirectConversionPattern<spirv::GLSLFloorOp, LLVM::FFloorOp>,
+      DirectConversionPattern<spirv::GLSLFMaxOp, LLVM::MaxNumOp>,
+      DirectConversionPattern<spirv::GLSLFMinOp, LLVM::MinNumOp>,
       DirectConversionPattern<spirv::GLSLLogOp, LLVM::LogOp>,
       DirectConversionPattern<spirv::GLSLSinOp, LLVM::SinOp>,
+      DirectConversionPattern<spirv::GLSLSMaxOp, LLVM::SMaxOp>,
+      DirectConversionPattern<spirv::GLSLSMinOp, LLVM::SMinOp>,
       DirectConversionPattern<spirv::GLSLSqrtOp, LLVM::SqrtOp>,
       InverseSqrtPattern, TanPattern, TanhPattern,
 
diff --git a/mlir/test/Conversion/SPIRVToLLVM/glsl-ops-to-llvm.mlir b/mlir/test/Conversion/SPIRVToLLVM/glsl-ops-to-llvm.mlir
index ab501b8aabfd0..ecb9843d64fdd 100644
--- a/mlir/test/Conversion/SPIRVToLLVM/glsl-ops-to-llvm.mlir
+++ b/mlir/test/Conversion/SPIRVToLLVM/glsl-ops-to-llvm.mlir
@@ -52,6 +52,45 @@ func @fabs(%arg0: f32, %arg1: vector<3xf16>) {
 	return
 }
 
+//===----------------------------------------------------------------------===//
+// spv.GLSL.Floor
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: @floor
+func @floor(%arg0: f32, %arg1: vector<3xf16>) {
+	// CHECK: "llvm.intr.floor"(%{{.*}}) : (!llvm.float) -> !llvm.float
+	%0 = spv.GLSL.Floor %arg0 : f32
+	// CHECK: "llvm.intr.floor"(%{{.*}}) : (!llvm<"<3 x half>">) -> !llvm<"<3 x half>">
+	%1 = spv.GLSL.Floor %arg1 : vector<3xf16>
+	return
+}
+
+//===----------------------------------------------------------------------===//
+// spv.GLSL.FMax
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: @fmax
+func @fmax(%arg0: f32, %arg1: vector<3xf16>) {
+	// CHECK: "llvm.intr.maxnum"(%{{.*}}, %{{.*}}) : (!llvm.float, !llvm.float) -> !llvm.float
+	%0 = spv.GLSL.FMax %arg0, %arg0 : f32
+	// CHECK: "llvm.intr.maxnum"(%{{.*}}, %{{.*}}) : (!llvm<"<3 x half>">, !llvm<"<3 x half>">) -> !llvm<"<3 x half>">
+	%1 = spv.GLSL.FMax %arg1, %arg1 : vector<3xf16>
+	return
+}
+
+//===----------------------------------------------------------------------===//
+// spv.GLSL.FMin
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: @fmin
+func @fmin(%arg0: f32, %arg1: vector<3xf16>) {
+	// CHECK: "llvm.intr.minnum"(%{{.*}}, %{{.*}}) : (!llvm.float, !llvm.float) -> !llvm.float
+	%0 = spv.GLSL.FMin %arg0, %arg0 : f32
+	// CHECK: "llvm.intr.minnum"(%{{.*}}, %{{.*}}) : (!llvm<"<3 x half>">, !llvm<"<3 x half>">) -> !llvm<"<3 x half>">
+	%1 = spv.GLSL.FMin %arg1, %arg1 : vector<3xf16>
+	return
+}
+
 //===----------------------------------------------------------------------===//
 // spv.GLSL.Log
 //===----------------------------------------------------------------------===//
@@ -78,6 +117,32 @@ func @sin(%arg0: f32, %arg1: vector<3xf16>) {
 	return
 }
 
+//===----------------------------------------------------------------------===//
+// spv.GLSL.SMax
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: @smax
+func @smax(%arg0: i16, %arg1: vector<3xi32>) {
+	// CHECK: "llvm.intr.smax"(%{{.*}}, %{{.*}}) : (!llvm.i16, !llvm.i16) -> !llvm.i16
+	%0 = spv.GLSL.SMax %arg0, %arg0 : i16
+	// CHECK: "llvm.intr.smax"(%{{.*}}, %{{.*}}) : (!llvm<"<3 x i32>">, !llvm<"<3 x i32>">) -> !llvm<"<3 x i32>">
+	%1 = spv.GLSL.SMax %arg1, %arg1 : vector<3xi32>
+	return
+}
+
+//===----------------------------------------------------------------------===//
+// spv.GLSL.SMin
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: @smin
+func @smin(%arg0: i16, %arg1: vector<3xi32>) {
+	// CHECK: "llvm.intr.smin"(%{{.*}}, %{{.*}}) : (!llvm.i16, !llvm.i16) -> !llvm.i16
+	%0 = spv.GLSL.SMin %arg0, %arg0 : i16
+	// CHECK: "llvm.intr.smin"(%{{.*}}, %{{.*}}) : (!llvm<"<3 x i32>">, !llvm<"<3 x i32>">) -> !llvm<"<3 x i32>">
+	%1 = spv.GLSL.SMin %arg1, %arg1 : vector<3xi32>
+	return
+}
+
 //===----------------------------------------------------------------------===//
 // spv.GLSL.Sqrt
 //===----------------------------------------------------------------------===//
diff --git a/mlir/test/Target/llvmir-intrinsics.mlir b/mlir/test/Target/llvmir-intrinsics.mlir
index 79b7edb3c7010..286d8fbe5f68b 100644
--- a/mlir/test/Target/llvmir-intrinsics.mlir
+++ b/mlir/test/Target/llvmir-intrinsics.mlir
@@ -90,6 +90,15 @@ llvm.func @ceil_test(%arg0: !llvm.float, %arg1: !llvm<"<8 x float>">) {
   llvm.return
 }
 
+// CHECK-LABEL: @floor_test
+llvm.func @floor_test(%arg0: !llvm.float, %arg1: !llvm<"<8 x float>">) {
+  // CHECK: call float @llvm.floor.f32
+  "llvm.intr.floor"(%arg0) : (!llvm.float) -> !llvm.float
+  // CHECK: call <8 x float> @llvm.floor.v8f32
+  "llvm.intr.floor"(%arg1) : (!llvm<"<8 x float>">) -> !llvm<"<8 x float>">
+  llvm.return
+}
+
 // CHECK-LABEL: @cos_test
 llvm.func @cos_test(%arg0: !llvm.float, %arg1: !llvm<"<8 x float>">) {
   // CHECK: call float @llvm.cos.f32
@@ -135,6 +144,42 @@ llvm.func @ctpop_test(%arg0: !llvm.i32, %arg1: !llvm<"<8 x i32>">) {
   llvm.return
 }
 
+// CHECK-LABEL: @maxnum_test
+llvm.func @maxnum_test(%arg0: !llvm.float, %arg1: !llvm.float, %arg2: !llvm<"<8 x float>">, %arg3: !llvm<"<8 x float>">) {
+  // CHECK: call float @llvm.maxnum.f32
+  "llvm.intr.maxnum"(%arg0, %arg1) : (!llvm.float, !llvm.float) -> !llvm.float
+  // CHECK: call <8 x float> @llvm.maxnum.v8f32
+  "llvm.intr.maxnum"(%arg2, %arg3) : (!llvm<"<8 x float>">, !llvm<"<8 x float>">) -> !llvm<"<8 x float>">
+  llvm.return
+}
+
+// CHECK-LABEL: @minnum_test
+llvm.func @minnum_test(%arg0: !llvm.float, %arg1: !llvm.float, %arg2: !llvm<"<8 x float>">, %arg3: !llvm<"<8 x float>">) {
+  // CHECK: call float @llvm.minnum.f32
+  "llvm.intr.minnum"(%arg0, %arg1) : (!llvm.float, !llvm.float) -> !llvm.float
+  // CHECK: call <8 x float> @llvm.minnum.v8f32
+  "llvm.intr.minnum"(%arg2, %arg3) : (!llvm<"<8 x float>">, !llvm<"<8 x float>">) -> !llvm<"<8 x float>">
+  llvm.return
+}
+
+// CHECK-LABEL: @smax_test
+llvm.func @smax_test(%arg0: !llvm.i32, %arg1: !llvm.i32, %arg2: !llvm<"<8 x i32>">, %arg3: !llvm<"<8 x i32>">) {
+  // CHECK: call i32 @llvm.smax.i32
+  "llvm.intr.smax"(%arg0, %arg1) : (!llvm.i32, !llvm.i32) -> !llvm.i32
+  // CHECK: call <8 x i32> @llvm.smax.v8i32
+  "llvm.intr.smax"(%arg2, %arg3) : (!llvm<"<8 x i32>">, !llvm<"<8 x i32>">) -> !llvm<"<8 x i32>">
+  llvm.return
+}
+
+// CHECK-LABEL: @smin_test
+llvm.func @smin_test(%arg0: !llvm.i32, %arg1: !llvm.i32, %arg2: !llvm<"<8 x i32>">, %arg3: !llvm<"<8 x i32>">) {
+  // CHECK: call i32 @llvm.smin.i32
+  "llvm.intr.smin"(%arg0, %arg1) : (!llvm.i32, !llvm.i32) -> !llvm.i32
+  // CHECK: call <8 x i32> @llvm.smin.v8i32
+  "llvm.intr.smin"(%arg2, %arg3) : (!llvm<"<8 x i32>">, !llvm<"<8 x i32>">) -> !llvm<"<8 x i32>">
+  llvm.return
+}
+
 // CHECK-LABEL: @vector_reductions
 llvm.func @vector_reductions(%arg0: !llvm.float, %arg1: !llvm<"<8 x float>">, %arg2: !llvm<"<8 x i32>">) {
   // CHECK: call i32 @llvm.experimental.vector.reduce.add.v8i32

From bbe509920721cf5d42990339e4daf43e27073ec4 Mon Sep 17 00:00:00 2001
From: Rainer Orth <ro@gcc.gnu.org>
Date: Thu, 30 Jul 2020 10:27:36 +0200
Subject: [PATCH 0591/1035] [compiler-rt][cmake] Simplify
 COMPILER_RT_HAS_VERSION_SCRIPT tests

As requested in the review, this patch removes the additional conditions in
the `COMPILER_RT_HAS_VERSION_SCRIPT` tests.

Tested on `amd64-pc-solaris2.11` and `x86_64-pc-linux-gnu`.

Differential Revision: https://reviews.llvm.org/D84559
---
 compiler-rt/lib/asan/CMakeLists.txt  | 2 +-
 compiler-rt/lib/ubsan/CMakeLists.txt | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/compiler-rt/lib/asan/CMakeLists.txt b/compiler-rt/lib/asan/CMakeLists.txt
index 0c29893ebfe23..41e889eb79e6f 100644
--- a/compiler-rt/lib/asan/CMakeLists.txt
+++ b/compiler-rt/lib/asan/CMakeLists.txt
@@ -224,7 +224,7 @@ else()
     PARENT_TARGET asan)
 
   foreach(arch ${ASAN_SUPPORTED_ARCH})
-    if (UNIX AND COMPILER_RT_HAS_VERSION_SCRIPT)
+    if (COMPILER_RT_HAS_VERSION_SCRIPT)
       add_sanitizer_rt_version_list(clang_rt.asan-dynamic-${arch}
                                     LIBS clang_rt.asan-${arch} clang_rt.asan_cxx-${arch}
                                     EXTRA asan.syms.extra)
diff --git a/compiler-rt/lib/ubsan/CMakeLists.txt b/compiler-rt/lib/ubsan/CMakeLists.txt
index 00b74a63cbe53..b5342f2e9e645 100644
--- a/compiler-rt/lib/ubsan/CMakeLists.txt
+++ b/compiler-rt/lib/ubsan/CMakeLists.txt
@@ -200,7 +200,7 @@ else()
       CFLAGS ${UBSAN_CXXFLAGS}
       PARENT_TARGET ubsan)
 
-    if ((FUCHSIA OR UNIX) AND COMPILER_RT_HAS_VERSION_SCRIPT)
+    if (COMPILER_RT_HAS_VERSION_SCRIPT)
       file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/dummy.cpp "")
       add_compiler_rt_object_libraries(RTUbsan_dynamic_version_script_dummy
         ARCHS ${UBSAN_SUPPORTED_ARCH}

From 1ac72a0774b5e6d770bb65d2debccca561bc1c31 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Thu, 9 Jul 2020 17:29:19 +0100
Subject: [PATCH 0592/1035] [IPConstProp] Regenerate check lines.

Preparation for D84447.
---
 .../IPConstantProp/2008-06-09-WeakProp.ll     | 18 +++-
 .../IPConstantProp/arg-count-mismatch.ll      | 23 +++--
 .../Transforms/IPConstantProp/comdat-ipo.ll   | 14 ++-
 .../test/Transforms/IPConstantProp/deadarg.ll |  5 +-
 .../IPConstantProp/multiple_callbacks.ll      | 44 ++++++---
 .../Transforms/IPConstantProp/naked-return.ll | 18 +++-
 .../IPConstantProp/openmp_parallel_for.ll     | 72 +++++++++++++-
 .../Transforms/IPConstantProp/pthreads.ll     | 18 +++-
 .../Transforms/IPConstantProp/recursion.ll    | 20 +++-
 .../IPConstantProp/return-argument.ll         | 99 ++++++++++++-------
 .../IPConstantProp/return-constant.ll         | 50 +++++++---
 .../IPConstantProp/return-constants.ll        | 36 ++++++-
 .../IPConstantProp/thread_local_acs.ll        | 16 ++-
 13 files changed, 336 insertions(+), 97 deletions(-)

diff --git a/llvm/test/Transforms/IPConstantProp/2008-06-09-WeakProp.ll b/llvm/test/Transforms/IPConstantProp/2008-06-09-WeakProp.ll
index 6471ed9257976..270115cf5ddd1 100644
--- a/llvm/test/Transforms/IPConstantProp/2008-06-09-WeakProp.ll
+++ b/llvm/test/Transforms/IPConstantProp/2008-06-09-WeakProp.ll
@@ -1,17 +1,25 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -ipconstprop -S | FileCheck %s
 ; Should not propagate the result of a weak function.
 ; PR2411
 
-; CHECK: ret i32 %r
-
 define weak i32 @foo() nounwind  {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret i32 1
+;
 entry:
-        ret i32 1
+  ret i32 1
 }
 
 define i32 @main() nounwind  {
+; CHECK-LABEL: @main(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[R:%.*]] = call i32 @foo() #0
+; CHECK-NEXT:    ret i32 [[R]]
+;
 entry:
-        %r = call i32 @foo( ) nounwind
-        ret i32 %r
+  %r = call i32 @foo( ) nounwind
+  ret i32 %r
 }
 
diff --git a/llvm/test/Transforms/IPConstantProp/arg-count-mismatch.ll b/llvm/test/Transforms/IPConstantProp/arg-count-mismatch.ll
index 2a7d92d3f9b6f..7afe858e52d11 100644
--- a/llvm/test/Transforms/IPConstantProp/arg-count-mismatch.ll
+++ b/llvm/test/Transforms/IPConstantProp/arg-count-mismatch.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -ipconstprop -S -o - | FileCheck %s
 
 ; The original C source looked like this:
@@ -50,22 +51,28 @@ define internal i16 @bar(i16 %p1, i16 %p2) {
 ; in argument count due to varargs (as long as all non-variadic arguments have
 ; been provided),
 
+define internal i16 @vararg_prop(i16 %p1, ...) {
+; CHECK-LABEL: @vararg_prop(
+; CHECK-NEXT:    ret i16 7
+;
+  ret i16 %p1
+}
+
 define dso_local i16 @vararg_tests(i16 %a) {
+; CHECK-LABEL: @vararg_tests(
+; CHECK-NEXT:    [[CALL1:%.*]] = call i16 (i16, ...) @vararg_prop(i16 7, i16 8, i16 [[A:%.*]])
+; CHECK-NEXT:    [[CALL2:%.*]] = call i16 bitcast (i16 (i16, i16, ...)* @vararg_no_prop to i16 (i16)*)(i16 7)
+; CHECK-NEXT:    [[ADD:%.*]] = add i16 7, [[CALL2]]
+; CHECK-NEXT:    ret i16 [[ADD]]
+;
   %call1 = call i16 (i16, ...) @vararg_prop(i16 7, i16 8, i16 %a)
   %call2 = call i16 bitcast (i16 (i16, i16, ...) * @vararg_no_prop to i16 (i16) *) (i16 7)
   %add = add i16 %call1, %call2
   ret i16 %add
 }
 
-define internal i16 @vararg_prop(i16 %p1, ...) {
-; CHECK-LABEL: define internal i16 @vararg_prop(
-; CHECK-NEXT:    ret i16 7
-;
-  ret i16 %p1
-}
-
 define internal i16 @vararg_no_prop(i16 %p1, i16 %p2, ...) {
-; CHECK-LABEL: define internal i16 @vararg_no_prop(
+; CHECK-LABEL: @vararg_no_prop(
 ; CHECK-NEXT:    ret i16 [[P1:%.*]]
 ;
   ret i16 %p1
diff --git a/llvm/test/Transforms/IPConstantProp/comdat-ipo.ll b/llvm/test/Transforms/IPConstantProp/comdat-ipo.ll
index 6c4c44c33e258..a19c89cb9bcf8 100644
--- a/llvm/test/Transforms/IPConstantProp/comdat-ipo.ll
+++ b/llvm/test/Transforms/IPConstantProp/comdat-ipo.ll
@@ -1,8 +1,12 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -ipconstprop -S | FileCheck %s
 
 ; See PR26774
 
 define i32 @baz() {
+; CHECK-LABEL: @baz(
+; CHECK-NEXT:    ret i32 10
+;
   ret i32 10
 }
 
@@ -11,8 +15,9 @@ define i32 @baz() {
 
 define linkonce_odr i32 @foo() {
 ; CHECK-LABEL: @foo(
-; CHECK-NEXT:  %val = call i32 @baz()
-; CHECK-NEXT:  ret i32 10
+; CHECK-NEXT:    [[VAL:%.*]] = call i32 @baz()
+; CHECK-NEXT:    ret i32 10
+;
 
   %val = call i32 @baz()
   ret i32 %val
@@ -20,8 +25,9 @@ define linkonce_odr i32 @foo() {
 
 define i32 @bar() {
 ; CHECK-LABEL: @bar(
-; CHECK-NEXT:  %val = call i32 @foo()
-; CHECK-NEXT:  ret i32 %val
+; CHECK-NEXT:    [[VAL:%.*]] = call i32 @foo()
+; CHECK-NEXT:    ret i32 [[VAL]]
+;
 
   %val = call i32 @foo()
   ret i32 %val
diff --git a/llvm/test/Transforms/IPConstantProp/deadarg.ll b/llvm/test/Transforms/IPConstantProp/deadarg.ll
index 4b9938e09e90c..25b9749b50790 100644
--- a/llvm/test/Transforms/IPConstantProp/deadarg.ll
+++ b/llvm/test/Transforms/IPConstantProp/deadarg.ll
@@ -1,6 +1,7 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -ipconstprop -disable-output
 define internal void @foo(i32 %X) {
-        call void @foo( i32 %X )
-        ret void
+  call void @foo( i32 %X )
+  ret void
 }
 
diff --git a/llvm/test/Transforms/IPConstantProp/multiple_callbacks.ll b/llvm/test/Transforms/IPConstantProp/multiple_callbacks.ll
index 6bc3f4477db08..6684044e24ce0 100644
--- a/llvm/test/Transforms/IPConstantProp/multiple_callbacks.ll
+++ b/llvm/test/Transforms/IPConstantProp/multiple_callbacks.ll
@@ -1,9 +1,5 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -ipconstprop -S < %s | FileCheck %s
-; FIXME: There seems to be another instance of nondeterminism which causes the number of iterations to be either 1 or 3, depending on the system. 
-;        This needs to be investigated and resolved. In the meantime we do not verify the number of iterations.
-; opt -S -passes=attributor -aa-pipeline='basic-aa'  -attributor-max-iterations-verify -attributor-max-iterations=1 < %s | FileCheck %s
-; RUN: opt -S -passes=attributor -aa-pipeline='basic-aa'  -attributor-max-iterations-verify=false -attributor-max-iterations=3 < %s | FileCheck %s
-;
 ;
 ;                            /---------------------------------------|
 ;                            |                /----------------------|----|
@@ -40,39 +36,63 @@
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
 define internal i32 @cb0(i32 %zero) {
+; CHECK-LABEL: @cb0(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret i32 0
+;
 entry:
-; CHECK:      @cb0
-; CHECK-NEXT: entry
-; CHECK-NEXT: ret i32 0
   ret i32 %zero
 }
 
 define internal i32 @cb1(i32 %unknown) {
+; CHECK-LABEL: @cb1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret i32 [[UNKNOWN:%.*]]
+;
 entry:
-; CHECK: ret i32 %unknown
   ret i32 %unknown
 }
 
 define internal i32 @cb2(i32 %unknown) {
+; CHECK-LABEL: @cb2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @cb0(i32 0)
+; CHECK-NEXT:    ret i32 [[UNKNOWN:%.*]]
+;
 entry:
   %call = call i32 @cb0(i32 0)
-; CHECK: ret i32 %unknown
   ret i32 %unknown
 }
 
 define internal i32 @cb3(i32 %unknown) {
+; CHECK-LABEL: @cb3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret i32 [[UNKNOWN:%.*]]
+;
 entry:
-; CHECK: ret i32 %unknown
   ret i32 %unknown
 }
 
 define internal i32 @cb4(i32 %unknown) {
+; CHECK-LABEL: @cb4(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret i32 [[UNKNOWN:%.*]]
+;
 entry:
-; CHECK: ret i32 %unknown
   ret i32 %unknown
 }
 
 define void @foo() {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @cb0(i32 0)
+; CHECK-NEXT:    [[CALL1:%.*]] = call i32 @cb3(i32 1)
+; CHECK-NEXT:    call void @broker(i32 (i32)* nonnull @cb0, i32 (i32)* nonnull @cb1, i32 (i32)* nonnull @cb0, i32 0, i32 1)
+; CHECK-NEXT:    call void @broker(i32 (i32)* nonnull @cb1, i32 (i32)* nonnull @cb2, i32 (i32)* nonnull @cb2, i32 0, i32 1)
+; CHECK-NEXT:    call void @broker(i32 (i32)* nonnull @cb3, i32 (i32)* nonnull @cb2, i32 (i32)* nonnull @cb3, i32 0, i32 1)
+; CHECK-NEXT:    call void @broker(i32 (i32)* nonnull @cb4, i32 (i32)* nonnull @cb4, i32 (i32)* nonnull @cb4, i32 0, i32 1)
+; CHECK-NEXT:    ret void
+;
 entry:
   %call = call i32 @cb0(i32 0)
   %call1 = call i32 @cb3(i32 1)
diff --git a/llvm/test/Transforms/IPConstantProp/naked-return.ll b/llvm/test/Transforms/IPConstantProp/naked-return.ll
index ac3dc062abf79..133662a211b41 100644
--- a/llvm/test/Transforms/IPConstantProp/naked-return.ll
+++ b/llvm/test/Transforms/IPConstantProp/naked-return.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -ipsccp -S %s | FileCheck %s
 ; RUN: opt -ipconstprop -S %s | FileCheck %s
 
@@ -5,22 +6,37 @@ target datalayout = "e-m:x-p:32:32-i64:64-f80:32-n8:16:32-a:0:32-S32"
 target triple = "i686-pc-windows-msvc19.0.24215"
 
 define i32 @dipsy(i32, i32) local_unnamed_addr #0 {
+; CHECK-LABEL: @dipsy(
+; CHECK-NEXT:  BasicBlock0:
+; CHECK-NEXT:    call void asm "\0D\0Apushl %ebp\0D\0Amovl 8(%esp),%eax\0D\0Amovl 12(%esp), %ebp\0D\0Acalll *%eax\0D\0Apopl %ebp\0D\0Aretl\0D\0A", ""()
+; CHECK-NEXT:    ret i32 0
+;
 BasicBlock0:
   call void asm "\0D\0Apushl %ebp\0D\0Amovl 8(%esp),%eax\0D\0Amovl 12(%esp), %ebp\0D\0Acalll *%eax\0D\0Apopl %ebp\0D\0Aretl\0D\0A", ""()
   ret i32 0
 }
 
 define void @tinkywinky(i32, i32, i32) local_unnamed_addr #0 {
+; CHECK-LABEL: @tinkywinky(
+; CHECK-NEXT:  BasicBlock1:
+; CHECK-NEXT:    call void asm "\0D\0A movl 12(%esp), %ebp\0D\0A movl 4(%esp), %eax\0D\0A movl 8(%esp), %esp\0D\0A jmpl *%eax\0D\0A", ""()
+; CHECK-NEXT:    ret void
+;
 BasicBlock1:
   call void asm "\0D\0A    movl 12(%esp), %ebp\0D\0A    movl 4(%esp), %eax\0D\0A    movl 8(%esp), %esp\0D\0A    jmpl *%eax\0D\0A", ""()
   ret void
 }
 
 define void @patatino(i32, i32, i32) local_unnamed_addr #1 {
+; CHECK-LABEL: @patatino(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 @dipsy(i32 [[TMP0:%.*]], i32 [[TMP1:%.*]]) #0
+; CHECK-NEXT:    tail call void @tinkywinky(i32 [[TMP3]], i32 [[TMP2:%.*]], i32 [[TMP1]]) #0
+; CHECK-NEXT:    ret void
+;
 bb:
   %3 = tail call i32 @dipsy(i32 %0, i32 %1) #0
 ; Check that we don't accidentally propagate zero.
-; CHECK: @tinkywinky(i32 %3, i32 %2, i32 %1) #0
   tail call void @tinkywinky(i32 %3, i32 %2, i32 %1) #0
   ret void
 }
diff --git a/llvm/test/Transforms/IPConstantProp/openmp_parallel_for.ll b/llvm/test/Transforms/IPConstantProp/openmp_parallel_for.ll
index 3c7ffe27f5d2f..338cc8886e29d 100644
--- a/llvm/test/Transforms/IPConstantProp/openmp_parallel_for.ll
+++ b/llvm/test/Transforms/IPConstantProp/openmp_parallel_for.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S -ipconstprop < %s | FileCheck %s
 ;
 ;    void bar(int, float, double);
@@ -24,6 +25,16 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 @1 = private unnamed_addr global %struct.ident_t { i32 0, i32 2, i32 0, i32 0, i8* getelementptr inbounds ([23 x i8], [23 x i8]* @.str, i32 0, i32 0) }, align 8
 
 define dso_local void @foo(i32 %N) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[N_ADDR:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[P:%.*]] = alloca float, align 4
+; CHECK-NEXT:    store i32 [[N:%.*]], i32* [[N_ADDR]], align 4
+; CHECK-NEXT:    store float 3.000000e+00, float* [[P]], align 4
+; CHECK-NEXT:    store i32 7, i32* [[N_ADDR]], align 4
+; CHECK-NEXT:    call void (%struct.ident_t*, i32, void (i32*, i32*, ...)*, ...) @__kmpc_fork_call(%struct.ident_t* nonnull @1, i32 3, void (i32*, i32*, ...)* bitcast (void (i32*, i32*, i32*, float*, i64)* @.omp_outlined. to void (i32*, i32*, ...)*), i32* nonnull [[N_ADDR]], float* nonnull [[P]], i64 4617315517961601024)
+; CHECK-NEXT:    ret void
+;
 entry:
   %N.addr = alloca i32, align 4
   %p = alloca float, align 4
@@ -35,13 +46,72 @@ entry:
 }
 
 define internal void @.omp_outlined.(i32* noalias %.global_tid., i32* noalias %.bound_tid., i32* dereferenceable(4) %N, float* dereferenceable(4) %p, i64 %q) {
+; CHECK-LABEL: @.omp_outlined.(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[Q_ADDR:%.*]] = alloca i64, align 8
+; CHECK-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[DOTOMP_UB:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    store i64 4617315517961601024, i64* [[Q_ADDR]], align 8
+; CHECK-NEXT:    [[CONV:%.*]] = bitcast i64* [[Q_ADDR]] to double*
+; CHECK-NEXT:    [[TMP:%.*]] = load i32, i32* [[N:%.*]], align 4
+; CHECK-NEXT:    [[SUB3:%.*]] = add nsw i32 [[TMP]], -3
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP]], 2
+; CHECK-NEXT:    br i1 [[CMP]], label [[OMP_PRECOND_THEN:%.*]], label [[OMP_PRECOND_END:%.*]]
+; CHECK:       omp.precond.then:
+; CHECK-NEXT:    store i32 0, i32* [[DOTOMP_LB]], align 4
+; CHECK-NEXT:    store i32 [[SUB3]], i32* [[DOTOMP_UB]], align 4
+; CHECK-NEXT:    store i32 1, i32* [[DOTOMP_STRIDE]], align 4
+; CHECK-NEXT:    store i32 0, i32* [[DOTOMP_IS_LAST]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = load i32, i32* [[DOTGLOBAL_TID_:%.*]], align 4
+; CHECK-NEXT:    call void @__kmpc_for_static_init_4(%struct.ident_t* nonnull @0, i32 [[TMP5]], i32 34, i32* nonnull [[DOTOMP_IS_LAST]], i32* nonnull [[DOTOMP_LB]], i32* nonnull [[DOTOMP_UB]], i32* nonnull [[DOTOMP_STRIDE]], i32 1, i32 1)
+; CHECK-NEXT:    [[TMP6:%.*]] = load i32, i32* [[DOTOMP_UB]], align 4
+; CHECK-NEXT:    [[CMP6:%.*]] = icmp sgt i32 [[TMP6]], [[SUB3]]
+; CHECK-NEXT:    br i1 [[CMP6]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+; CHECK:       cond.true:
+; CHECK-NEXT:    br label [[COND_END:%.*]]
+; CHECK:       cond.false:
+; CHECK-NEXT:    [[TMP7:%.*]] = load i32, i32* [[DOTOMP_UB]], align 4
+; CHECK-NEXT:    br label [[COND_END]]
+; CHECK:       cond.end:
+; CHECK-NEXT:    [[COND:%.*]] = phi i32 [ [[SUB3]], [[COND_TRUE]] ], [ [[TMP7]], [[COND_FALSE]] ]
+; CHECK-NEXT:    store i32 [[COND]], i32* [[DOTOMP_UB]], align 4
+; CHECK-NEXT:    [[TMP8:%.*]] = load i32, i32* [[DOTOMP_LB]], align 4
+; CHECK-NEXT:    br label [[OMP_INNER_FOR_COND:%.*]]
+; CHECK:       omp.inner.for.cond:
+; CHECK-NEXT:    [[DOTOMP_IV_0:%.*]] = phi i32 [ [[TMP8]], [[COND_END]] ], [ [[ADD11:%.*]], [[OMP_INNER_FOR_INC:%.*]] ]
+; CHECK-NEXT:    [[TMP9:%.*]] = load i32, i32* [[DOTOMP_UB]], align 4
+; CHECK-NEXT:    [[CMP8:%.*]] = icmp sgt i32 [[DOTOMP_IV_0]], [[TMP9]]
+; CHECK-NEXT:    br i1 [[CMP8]], label [[OMP_INNER_FOR_COND_CLEANUP:%.*]], label [[OMP_INNER_FOR_BODY:%.*]]
+; CHECK:       omp.inner.for.cond.cleanup:
+; CHECK-NEXT:    br label [[OMP_INNER_FOR_END:%.*]]
+; CHECK:       omp.inner.for.body:
+; CHECK-NEXT:    [[ADD10:%.*]] = add nsw i32 [[DOTOMP_IV_0]], 2
+; CHECK-NEXT:    [[TMP10:%.*]] = load float, float* [[P:%.*]], align 4
+; CHECK-NEXT:    [[TMP11:%.*]] = load double, double* [[CONV]], align 8
+; CHECK-NEXT:    call void @bar(i32 [[ADD10]], float [[TMP10]], double [[TMP11]])
+; CHECK-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
+; CHECK:       omp.body.continue:
+; CHECK-NEXT:    br label [[OMP_INNER_FOR_INC]]
+; CHECK:       omp.inner.for.inc:
+; CHECK-NEXT:    [[ADD11]] = add nsw i32 [[DOTOMP_IV_0]], 1
+; CHECK-NEXT:    br label [[OMP_INNER_FOR_COND]]
+; CHECK:       omp.inner.for.end:
+; CHECK-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
+; CHECK:       omp.loop.exit:
+; CHECK-NEXT:    [[TMP12:%.*]] = load i32, i32* [[DOTGLOBAL_TID_]], align 4
+; CHECK-NEXT:    call void @__kmpc_for_static_fini(%struct.ident_t* nonnull @0, i32 [[TMP12]])
+; CHECK-NEXT:    br label [[OMP_PRECOND_END]]
+; CHECK:       omp.precond.end:
+; CHECK-NEXT:    ret void
+;
 entry:
   %q.addr = alloca i64, align 8
   %.omp.lb = alloca i32, align 4
   %.omp.ub = alloca i32, align 4
   %.omp.stride = alloca i32, align 4
   %.omp.is_last = alloca i32, align 4
-; CHECK: store i64 4617315517961601024, i64* %q.addr, align 8
   store i64 %q, i64* %q.addr, align 8
   %conv = bitcast i64* %q.addr to double*
   %tmp = load i32, i32* %N, align 4
diff --git a/llvm/test/Transforms/IPConstantProp/pthreads.ll b/llvm/test/Transforms/IPConstantProp/pthreads.ll
index 0af2c166a0887..dcad3858da13a 100644
--- a/llvm/test/Transforms/IPConstantProp/pthreads.ll
+++ b/llvm/test/Transforms/IPConstantProp/pthreads.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -ipconstprop -S < %s | FileCheck %s
 ;
 ;    #include <pthread.h>
@@ -24,6 +25,13 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 @GlobalVPtr = common dso_local global i8* null, align 8
 
 define dso_local i32 @main() {
+; CHECK-LABEL: @main(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[THREAD:%.*]] = alloca i64, align 8
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @pthread_create(i64* nonnull [[THREAD]], %union.pthread_attr_t* null, i8* (i8*)* nonnull @foo, i8* null)
+; CHECK-NEXT:    [[CALL1:%.*]] = call i32 @pthread_create(i64* nonnull [[THREAD]], %union.pthread_attr_t* null, i8* (i8*)* nonnull @bar, i8* bitcast (i8** @GlobalVPtr to i8*))
+; CHECK-NEXT:    ret i32 0
+;
 entry:
   %thread = alloca i64, align 8
   %call = call i32 @pthread_create(i64* nonnull %thread, %union.pthread_attr_t* null, i8* (i8*)* nonnull @foo, i8* null)
@@ -34,14 +42,20 @@ entry:
 declare !callback !0 dso_local i32 @pthread_create(i64*, %union.pthread_attr_t*, i8* (i8*)*, i8*)
 
 define internal i8* @foo(i8* %arg) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret i8* null
+;
 entry:
-; CHECK: ret i8* null
   ret i8* %arg
 }
 
 define internal i8* @bar(i8* %arg) {
+; CHECK-LABEL: @bar(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret i8* bitcast (i8** @GlobalVPtr to i8*)
+;
 entry:
-; CHECK: ret i8* bitcast (i8** @GlobalVPtr to i8*)
   ret i8* %arg
 }
 
diff --git a/llvm/test/Transforms/IPConstantProp/recursion.ll b/llvm/test/Transforms/IPConstantProp/recursion.ll
index 13aea8fc7b4a4..ac8ff9ca00e32 100644
--- a/llvm/test/Transforms/IPConstantProp/recursion.ll
+++ b/llvm/test/Transforms/IPConstantProp/recursion.ll
@@ -1,15 +1,25 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -ipconstprop -deadargelim -S | FileCheck %s
 
 ; CHECK-NOT: %X
 
 define internal i32 @foo(i32 %X) {
-        %Y = call i32 @foo( i32 %X )            ; <i32> [#uses=1]
-        %Z = add i32 %Y, 1              ; <i32> [#uses=1]
-        ret i32 %Z
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:    [[Y:%.*]] = call i32 @foo()
+; CHECK-NEXT:    [[Z:%.*]] = add i32 [[Y]], 1
+; CHECK-NEXT:    ret i32 [[Z]]
+;
+  %Y = call i32 @foo( i32 %X )            ; <i32> [#uses=1]
+  %Z = add i32 %Y, 1              ; <i32> [#uses=1]
+  ret i32 %Z
 }
 
 define void @bar() {
-        call i32 @foo( i32 17 )         ; <i32>:1 [#uses=0]
-        ret void
+; CHECK-LABEL: @bar(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @foo()
+; CHECK-NEXT:    ret void
+;
+  call i32 @foo( i32 17 )         ; <i32>:1 [#uses=0]
+  ret void
 }
 
diff --git a/llvm/test/Transforms/IPConstantProp/return-argument.ll b/llvm/test/Transforms/IPConstantProp/return-argument.ll
index 830a6547b632f..6a3eac0c120d0 100644
--- a/llvm/test/Transforms/IPConstantProp/return-argument.ll
+++ b/llvm/test/Transforms/IPConstantProp/return-argument.ll
@@ -1,58 +1,91 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -ipconstprop -S | FileCheck %s
 
-; CHECK: add i32 1, 3
-; CHECK: store i32 %Z, i32* %Q
-
 ;; This function returns its second argument on all return statements
 define internal i32* @incdec(i1 %C, i32* %V) {
-        %X = load i32, i32* %V
-        br i1 %C, label %T, label %F
+; CHECK-LABEL: @incdec(
+; CHECK-NEXT:    [[X:%.*]] = load i32, i32* [[V:%.*]], align 4
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[T:%.*]], label [[F:%.*]]
+; CHECK:       T:
+; CHECK-NEXT:    [[X1:%.*]] = add i32 [[X]], 1
+; CHECK-NEXT:    store i32 [[X1]], i32* [[V]], align 4
+; CHECK-NEXT:    ret i32* [[V]]
+; CHECK:       F:
+; CHECK-NEXT:    [[X2:%.*]] = sub i32 [[X]], 1
+; CHECK-NEXT:    store i32 [[X2]], i32* [[V]], align 4
+; CHECK-NEXT:    ret i32* [[V]]
+;
+  %X = load i32, i32* %V
+  br i1 %C, label %T, label %F
 
 T:              ; preds = %0
-        %X1 = add i32 %X, 1
-        store i32 %X1, i32* %V
-        ret i32* %V
+  %X1 = add i32 %X, 1
+  store i32 %X1, i32* %V
+  ret i32* %V
 
 F:              ; preds = %0
-        %X2 = sub i32 %X, 1
-        store i32 %X2, i32* %V
-        ret i32* %V
+  %X2 = sub i32 %X, 1
+  store i32 %X2, i32* %V
+  ret i32* %V
 }
 
 ;; This function returns its first argument as a part of a multiple return
 ;; value
 define internal { i32, i32 } @foo(i32 %A, i32 %B) {
-        %X = add i32 %A, %B
-        %Y = insertvalue { i32, i32 } undef, i32 %A, 0
-        %Z = insertvalue { i32, i32 } %Y, i32 %X, 1
-        ret { i32, i32 } %Z
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:    [[X:%.*]] = add i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[Y:%.*]] = insertvalue { i32, i32 } undef, i32 [[A]], 0
+; CHECK-NEXT:    [[Z:%.*]] = insertvalue { i32, i32 } [[Y]], i32 [[X]], 1
+; CHECK-NEXT:    ret { i32, i32 } [[Z]]
+;
+  %X = add i32 %A, %B
+  %Y = insertvalue { i32, i32 } undef, i32 %A, 0
+  %Z = insertvalue { i32, i32 } %Y, i32 %X, 1
+  ret { i32, i32 } %Z
 }
 
 define void @caller(i1 %C) personality i32 (...)* @__gxx_personality_v0 {
-        %Q = alloca i32
-        ;; Call incdec to see if %W is properly replaced by %Q
-        %W = call i32* @incdec(i1 %C, i32* %Q )             ; <i32> [#uses=1]
-        ;; Call @foo twice, to prevent the arguments from propagating into the
-        ;; function (so we can check the returned argument is properly
-        ;; propagated per-caller).
-        %S1 = call { i32, i32 } @foo(i32 1, i32 2)
-        %X1 = extractvalue { i32, i32 } %S1, 0
-        %S2 = invoke { i32, i32 } @foo(i32 3, i32 4) to label %OK unwind label %LPAD
+; CHECK-LABEL: @caller(
+; CHECK-NEXT:    [[Q:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[W:%.*]] = call i32* @incdec(i1 [[C:%.*]], i32* [[Q]])
+; CHECK-NEXT:    [[S1:%.*]] = call { i32, i32 } @foo(i32 1, i32 2)
+; CHECK-NEXT:    [[S2:%.*]] = invoke { i32, i32 } @foo(i32 3, i32 4)
+; CHECK-NEXT:    to label [[OK:%.*]] unwind label [[LPAD:%.*]]
+; CHECK:       OK:
+; CHECK-NEXT:    [[Z:%.*]] = add i32 1, 3
+; CHECK-NEXT:    store i32 [[Z]], i32* [[Q]], align 4
+; CHECK-NEXT:    br label [[RET:%.*]]
+; CHECK:       LPAD:
+; CHECK-NEXT:    [[EXN:%.*]] = landingpad { i8*, i32 }
+; CHECK-NEXT:    cleanup
+; CHECK-NEXT:    br label [[RET]]
+; CHECK:       RET:
+; CHECK-NEXT:    ret void
+;
+  %Q = alloca i32
+  ;; Call incdec to see if %W is properly replaced by %Q
+  %W = call i32* @incdec(i1 %C, i32* %Q )             ; <i32> [#uses=1]
+  ;; Call @foo twice, to prevent the arguments from propagating into the
+  ;; function (so we can check the returned argument is properly
+  ;; propagated per-caller).
+  %S1 = call { i32, i32 } @foo(i32 1, i32 2)
+  %X1 = extractvalue { i32, i32 } %S1, 0
+  %S2 = invoke { i32, i32 } @foo(i32 3, i32 4) to label %OK unwind label %LPAD
 
 OK:
-        %X2 = extractvalue { i32, i32 } %S2, 0
-        ;; Do some stuff with the returned values which we can grep for
-        %Z  = add i32 %X1, %X2
-        store i32 %Z, i32* %W
-        br label %RET
+  %X2 = extractvalue { i32, i32 } %S2, 0
+  ;; Do some stuff with the returned values which we can grep for
+  %Z  = add i32 %X1, %X2
+  store i32 %Z, i32* %W
+  br label %RET
 
 LPAD:
-        %exn = landingpad {i8*, i32}
-                 cleanup
-        br label %RET
+  %exn = landingpad {i8*, i32}
+  cleanup
+  br label %RET
 
 RET:
-        ret void
+  ret void
 }
 
 declare i32 @__gxx_personality_v0(...)
diff --git a/llvm/test/Transforms/IPConstantProp/return-constant.ll b/llvm/test/Transforms/IPConstantProp/return-constant.ll
index 155a3011dd668..d75aa9b969317 100644
--- a/llvm/test/Transforms/IPConstantProp/return-constant.ll
+++ b/llvm/test/Transforms/IPConstantProp/return-constant.ll
@@ -1,34 +1,52 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -ipconstprop -instcombine -S | FileCheck %s
 
-; CHECK: ret i1 true
-; CHECK: ret i1 true
-; CHECK-NOT: ret i1 true
-
 define internal i32 @foo(i1 %C) {
-        br i1 %C, label %T, label %F
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[T:%.*]], label [[F:%.*]]
+; CHECK:       T:
+; CHECK-NEXT:    ret i32 52
+; CHECK:       F:
+; CHECK-NEXT:    ret i32 52
+;
+  br i1 %C, label %T, label %F
 
 T:              ; preds = %0
-        ret i32 52
+  ret i32 52
 
 F:              ; preds = %0
-        ret i32 52
+  ret i32 52
 }
 
 define i1 @caller(i1 %C) {
-        %X = call i32 @foo( i1 %C )             ; <i32> [#uses=1]
-        %Y = icmp ne i32 %X, 0          ; <i1> [#uses=1]
-        ret i1 %Y
+; CHECK-LABEL: @caller(
+; CHECK-NEXT:    [[X:%.*]] = call i32 @foo(i1 [[C:%.*]])
+; CHECK-NEXT:    ret i1 true
+;
+  %X = call i32 @foo( i1 %C )             ; <i32> [#uses=1]
+  %Y = icmp ne i32 %X, 0          ; <i1> [#uses=1]
+  ret i1 %Y
 }
 
 define i1 @invokecaller(i1 %C) personality i32 (...)* @__gxx_personality_v0 {
-        %X = invoke i32 @foo( i1 %C ) to label %OK unwind label %FAIL             ; <i32> [#uses=1]
+; CHECK-LABEL: @invokecaller(
+; CHECK-NEXT:    [[X:%.*]] = invoke i32 @foo(i1 [[C:%.*]])
+; CHECK-NEXT:    to label [[OK:%.*]] unwind label [[FAIL:%.*]]
+; CHECK:       OK:
+; CHECK-NEXT:    ret i1 true
+; CHECK:       FAIL:
+; CHECK-NEXT:    [[EXN:%.*]] = landingpad { i8*, i32 }
+; CHECK-NEXT:    cleanup
+; CHECK-NEXT:    ret i1 false
+;
+  %X = invoke i32 @foo( i1 %C ) to label %OK unwind label %FAIL             ; <i32> [#uses=1]
 OK:
-        %Y = icmp ne i32 %X, 0          ; <i1> [#uses=1]
-        ret i1 %Y 
+  %Y = icmp ne i32 %X, 0          ; <i1> [#uses=1]
+  ret i1 %Y
 FAIL:
-        %exn = landingpad {i8*, i32}
-                 cleanup
-        ret i1 false
+  %exn = landingpad {i8*, i32}
+  cleanup
+  ret i1 false
 }
 
 declare i32 @__gxx_personality_v0(...)
diff --git a/llvm/test/Transforms/IPConstantProp/return-constants.ll b/llvm/test/Transforms/IPConstantProp/return-constants.ll
index 40b97ad04a9da..4611067316141 100644
--- a/llvm/test/Transforms/IPConstantProp/return-constants.ll
+++ b/llvm/test/Transforms/IPConstantProp/return-constants.ll
@@ -1,13 +1,20 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -ipconstprop -S | FileCheck %s
 
-;; Check that the 21 constants got propagated properly
-; CHECK: %M = add i32 21, 21
-;; Check that the second return values didn't get propagated
-; CHECK: %N = add i32 %B, %D
-
 %0 = type { i32, i32 }
 
 define internal %0 @foo(i1 %Q) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:    br i1 [[Q:%.*]], label [[T:%.*]], label [[F:%.*]]
+; CHECK:       T:
+; CHECK-NEXT:    [[MRV:%.*]] = insertvalue [[TMP0:%.*]] undef, i32 21, 0
+; CHECK-NEXT:    [[MRV1:%.*]] = insertvalue [[TMP0]] %mrv, i32 22, 1
+; CHECK-NEXT:    ret [[TMP0]] %mrv1
+; CHECK:       F:
+; CHECK-NEXT:    [[MRV2:%.*]] = insertvalue [[TMP0]] undef, i32 21, 0
+; CHECK-NEXT:    [[MRV3:%.*]] = insertvalue [[TMP0]] %mrv2, i32 23, 1
+; CHECK-NEXT:    ret [[TMP0]] %mrv3
+;
   br i1 %Q, label %T, label %F
 
 T:                                                ; preds = %0
@@ -22,6 +29,16 @@ F:                                                ; preds = %0
 }
 
 define internal %0 @bar(i1 %Q) {
+; CHECK-LABEL: @bar(
+; CHECK-NEXT:    [[A:%.*]] = insertvalue [[TMP0:%.*]] undef, i32 21, 0
+; CHECK-NEXT:    br i1 [[Q:%.*]], label [[T:%.*]], label [[F:%.*]]
+; CHECK:       T:
+; CHECK-NEXT:    [[B:%.*]] = insertvalue [[TMP0]] %A, i32 22, 1
+; CHECK-NEXT:    ret [[TMP0]] %B
+; CHECK:       F:
+; CHECK-NEXT:    [[C:%.*]] = insertvalue [[TMP0]] %A, i32 23, 1
+; CHECK-NEXT:    ret [[TMP0]] %C
+;
   %A = insertvalue %0 undef, i32 21, 0
   br i1 %Q, label %T, label %F
 
@@ -35,6 +52,15 @@ F:                                                ; preds = %0
 }
 
 define %0 @caller(i1 %Q) {
+; CHECK-LABEL: @caller(
+; CHECK-NEXT:    [[X:%.*]] = call [[TMP0:%.*]] @foo(i1 [[Q:%.*]])
+; CHECK-NEXT:    [[B:%.*]] = extractvalue [[TMP0]] %X, 1
+; CHECK-NEXT:    [[Y:%.*]] = call [[TMP0]] @bar(i1 [[Q]])
+; CHECK-NEXT:    [[D:%.*]] = extractvalue [[TMP0]] %Y, 1
+; CHECK-NEXT:    [[M:%.*]] = add i32 21, 21
+; CHECK-NEXT:    [[N:%.*]] = add i32 [[B]], [[D]]
+; CHECK-NEXT:    ret [[TMP0]] %X
+;
   %X = call %0 @foo(i1 %Q)
   %A = extractvalue %0 %X, 0
   %B = extractvalue %0 %X, 1
diff --git a/llvm/test/Transforms/IPConstantProp/thread_local_acs.ll b/llvm/test/Transforms/IPConstantProp/thread_local_acs.ll
index 0595a5ca7f13c..3f843d26077ad 100644
--- a/llvm/test/Transforms/IPConstantProp/thread_local_acs.ll
+++ b/llvm/test/Transforms/IPConstantProp/thread_local_acs.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -ipconstprop -S < %s | FileCheck %s
 ;
 ;    #include <threads.h>
@@ -20,10 +21,14 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 @gsh = dso_local global i32 0, align 4
 
 define internal i32 @callee(i32* %thread_local_ptr, i32* %shared_ptr) {
+; CHECK-LABEL: @callee(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP:%.*]] = load i32, i32* [[THREAD_LOCAL_PTR:%.*]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* @gsh, align 4
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP]], [[TMP1]]
+; CHECK-NEXT:    ret i32 [[ADD]]
+;
 entry:
-; CHECK:  %tmp = load i32, i32* %thread_local_ptr, align 4
-; CHECK:  %tmp1 = load i32, i32* @gsh, align 4
-; CHECK:  %add = add nsw i32 %tmp, %tmp1
   %tmp = load i32, i32* %thread_local_ptr, align 4
   %tmp1 = load i32, i32* %shared_ptr, align 4
   %add = add nsw i32 %tmp, %tmp1
@@ -31,6 +36,11 @@ entry:
 }
 
 define dso_local void @caller() {
+; CHECK-LABEL: @caller(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    call void @broker(i32* nonnull @gtl, i32 (i32*, i32*)* nonnull @callee, i32* nonnull @gsh)
+; CHECK-NEXT:    ret void
+;
 entry:
   call void @broker(i32* nonnull @gtl, i32 (i32*, i32*)* nonnull @callee, i32* nonnull @gsh)
   ret void

From 332170356e35ea9cdc2c1d612b61a50ec5ea322e Mon Sep 17 00:00:00 2001
From: David Truby <david.truby@arm.com>
Date: Thu, 16 Jul 2020 16:13:09 +0100
Subject: [PATCH 0593/1035] [flang] Fix multi-config generator builds.

Summary:
Currently the binaries are output directly into the bin subdirectory of the
build directory. This doesn't work correctly with multi-config generators which
should output the binaries into <CONFIG_NAME>/bin instead.

Reviewers: sscalpone, richard.barton.arm

Subscribers: mgorny, llvm-commits

Tags: #llvm

Differential Revision: https://reviews.llvm.org/D84022
---
 flang/CMakeLists.txt           |  4 +---
 flang/test/lit.cfg.py          |  8 +-------
 flang/test/lit.site.cfg.py.in  |  1 -
 flang/tools/f18/CMakeLists.txt | 10 ++++++++--
 4 files changed, 10 insertions(+), 13 deletions(-)

diff --git a/flang/CMakeLists.txt b/flang/CMakeLists.txt
index f1aaa5c6473fe..337311fc0a63d 100644
--- a/flang/CMakeLists.txt
+++ b/flang/CMakeLists.txt
@@ -180,7 +180,7 @@ else()
          ${LLVM_INCLUDE_TESTS})
   set(FLANG_GTEST_AVAIL 1)
 
-  set(FLANG_BINARY_DIR ${CMAKE_BINARY_DIR}/tools/flang)
+  set(FLANG_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR})
   set(BACKEND_PACKAGE_STRING "${PACKAGE_STRING}")
   if (LINK_WITH_FIR)
     set(MLIR_MAIN_SRC_DIR ${LLVM_MAIN_SRC_DIR}/../mlir/include ) # --src-root
@@ -194,8 +194,6 @@ endif()
 
 if(LINK_WITH_FIR)
   # tco tool and FIR lib output directories
-  set(LLVM_RUNTIME_OUTPUT_INTDIR ${CMAKE_BINARY_DIR}/bin)
-  set(LLVM_LIBRARY_OUTPUT_INTDIR ${CMAKE_BINARY_DIR}/lib)
   # Always build tco tool
   set(LLVM_BUILD_TOOLS ON)
   message(STATUS "Linking driver with FIR and LLVM")
diff --git a/flang/test/lit.cfg.py b/flang/test/lit.cfg.py
index 8ad5a9b6357f9..521ae968d1cb9 100644
--- a/flang/test/lit.cfg.py
+++ b/flang/test/lit.cfg.py
@@ -48,12 +48,6 @@
 llvm_config.with_environment('PATH', config.flang_tools_dir, append_path=True)
 llvm_config.with_environment('PATH', config.llvm_tools_dir, append_path=True)
 
-# For builds with FIR, set path for tco and enable related tests
-if config.flang_llvm_tools_dir != "" :
-  config.available_features.add('fir')
-  if config.llvm_tools_dir != config.flang_llvm_tools_dir :
-    llvm_config.with_environment('PATH', config.flang_llvm_tools_dir, append_path=True)
-
 # For each occurrence of a flang tool name, replace it with the full path to
 # the build directory holding that tool.
 tools = [
@@ -61,7 +55,7 @@
     extra_args=["-intrinsic-module-directory "+config.flang_intrinsic_modules_dir],
     unresolved='fatal')
 ]
-llvm_config.add_tool_substitutions(tools, [config.flang_llvm_tools_dir])
+llvm_config.add_tool_substitutions(tools, config.llvm_tools_dir)
 
 # Enable libpgmath testing
 result = lit_config.params.get("LIBPGMATH")
diff --git a/flang/test/lit.site.cfg.py.in b/flang/test/lit.site.cfg.py.in
index e8e2945a2cbf0..34bcdab11b55b 100644
--- a/flang/test/lit.site.cfg.py.in
+++ b/flang/test/lit.site.cfg.py.in
@@ -7,7 +7,6 @@ config.flang_obj_root = "@FLANG_BINARY_DIR@"
 config.flang_src_dir = "@FLANG_SOURCE_DIR@"
 config.flang_tools_dir = "@FLANG_TOOLS_DIR@"
 config.flang_intrinsic_modules_dir = "@FLANG_INTRINSIC_MODULES_DIR@"
-config.flang_llvm_tools_dir = "@CMAKE_BINARY_DIR@/bin"
 config.python_executable = "@PYTHON_EXECUTABLE@"
 
 # Support substitution of the tools_dir with user parameters. This is
diff --git a/flang/tools/f18/CMakeLists.txt b/flang/tools/f18/CMakeLists.txt
index 46c38fa43a2e5..8e8b3d6b6ab11 100644
--- a/flang/tools/f18/CMakeLists.txt
+++ b/flang/tools/f18/CMakeLists.txt
@@ -59,8 +59,14 @@ add_custom_target(module_files ALL DEPENDS ${MODULE_FILES})
 install(TARGETS f18 DESTINATION bin)
 
 set(FLANG_INTRINSIC_MODULES_DIR ${FLANG_BINARY_DIR}/include/flang)
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/flang.sh.in ${CMAKE_BINARY_DIR}/tools/flang/bin/flang @ONLY)
-file(COPY ${CMAKE_BINARY_DIR}/tools/flang/bin/flang DESTINATION ${CMAKE_BINARY_DIR}/bin FILE_PERMISSIONS OWNER_EXECUTE OWNER_READ OWNER_WRITE)
+# This flang shell script will only work in a POSIX shell.
+if (NOT WIN32)
+  configure_file(${CMAKE_CURRENT_SOURCE_DIR}/flang.sh.in ${CMAKE_CURRENT_BINARY_DIR}/tools/flang/bin/flang @ONLY)
+add_custom_command(TARGET f18
+  POST_BUILD
+  COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_BINARY_DIR}/tools/flang/bin/flang ${LLVM_RUNTIME_OUTPUT_INTDIR}/flang
+  COMMAND chmod +x ${LLVM_RUNTIME_OUTPUT_INTDIR}/flang)
+endif()
 # The flang script to be installed needs a different path to the headers.
 set(FLANG_INTRINSIC_MODULES_DIR ${CMAKE_INSTALL_PREFIX}/include/flang)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/flang.sh.in ${FLANG_BINARY_DIR}/bin/flang-install.sh @ONLY)

From 45a720a864320bbbeb596abe412786fa91858980 Mon Sep 17 00:00:00 2001
From: Nathan James <n.james93@hotmail.co.uk>
Date: Thu, 30 Jul 2020 10:31:12 +0100
Subject: [PATCH 0594/1035] [clang-tidy] Use StringMap for
 ClangTidyOptions::OptionsMap

Ordering of options isn't important so an `llvm::StringMap` is a much better container for this purpose.

Reviewed By: gribozavr2

Differential Revision: https://reviews.llvm.org/D84868
---
 clang-tools-extra/clang-tidy/ClangTidy.cpp    |  7 +++---
 .../clang-tidy/ClangTidyCheck.cpp             | 16 ++++++-------
 .../clang-tidy/ClangTidyOptions.cpp           |  8 ++++---
 .../clang-tidy/ClangTidyOptions.h             |  3 +--
 .../clang-tidy/checkers/google-module.cpp     | 12 ++++------
 .../infrastructure/config-files.cpp           | 23 +++++++------------
 6 files changed, 29 insertions(+), 40 deletions(-)

diff --git a/clang-tools-extra/clang-tidy/ClangTidy.cpp b/clang-tools-extra/clang-tidy/ClangTidy.cpp
index d6913dfd3c078..63c83a0b9954e 100644
--- a/clang-tools-extra/clang-tidy/ClangTidy.cpp
+++ b/clang-tools-extra/clang-tidy/ClangTidy.cpp
@@ -329,12 +329,11 @@ static void setStaticAnalyzerCheckerOpts(const ClangTidyOptions &Opts,
                                          AnalyzerOptionsRef AnalyzerOptions) {
   StringRef AnalyzerPrefix(AnalyzerCheckNamePrefix);
   for (const auto &Opt : Opts.CheckOptions) {
-    StringRef OptName(Opt.first);
-    if (!OptName.startswith(AnalyzerPrefix))
+    StringRef OptName(Opt.getKey());
+    if (!OptName.consume_front(AnalyzerPrefix))
       continue;
     // Analyzer options are always local options so we can ignore priority.
-    AnalyzerOptions->Config[OptName.substr(AnalyzerPrefix.size())] =
-        Opt.second.Value;
+    AnalyzerOptions->Config[OptName] = Opt.getValue().Value;
   }
 }
 
diff --git a/clang-tools-extra/clang-tidy/ClangTidyCheck.cpp b/clang-tools-extra/clang-tidy/ClangTidyCheck.cpp
index ffd5bf974ba2f..737d85e092d97 100644
--- a/clang-tools-extra/clang-tidy/ClangTidyCheck.cpp
+++ b/clang-tools-extra/clang-tidy/ClangTidyCheck.cpp
@@ -72,7 +72,7 @@ llvm::Expected<std::string>
 ClangTidyCheck::OptionsView::get(StringRef LocalName) const {
   const auto &Iter = CheckOptions.find(NamePrefix + LocalName.str());
   if (Iter != CheckOptions.end())
-    return Iter->second.Value;
+    return Iter->getValue().Value;
   return llvm::make_error<MissingOptionError>((NamePrefix + LocalName).str());
 }
 
@@ -85,7 +85,7 @@ findPriorityOption(const ClangTidyOptions::OptionMap &Options, StringRef NamePre
     return IterGlobal;
   if (IterGlobal == Options.end())
     return IterLocal;
-  if (IterLocal->second.Priority >= IterGlobal->second.Priority)
+  if (IterLocal->getValue().Priority >= IterGlobal->getValue().Priority)
     return IterLocal;
   return IterGlobal;
 }
@@ -94,7 +94,7 @@ llvm::Expected<std::string>
 ClangTidyCheck::OptionsView::getLocalOrGlobal(StringRef LocalName) const {
   auto Iter = findPriorityOption(CheckOptions, NamePrefix, LocalName);
   if (Iter != CheckOptions.end())
-    return Iter->second.Value;
+    return Iter->getValue().Value;
   return llvm::make_error<MissingOptionError>((NamePrefix + LocalName).str());
 }
 
@@ -135,7 +135,7 @@ llvm::Expected<bool>
 ClangTidyCheck::OptionsView::getLocalOrGlobal<bool>(StringRef LocalName) const {
   auto Iter = findPriorityOption(CheckOptions, NamePrefix, LocalName);
   if (Iter != CheckOptions.end())
-    return getAsBool(Iter->second.Value, Iter->first);
+    return getAsBool(Iter->getValue().Value, Iter->getKey());
   return llvm::make_error<MissingOptionError>((NamePrefix + LocalName).str());
 }
 
@@ -177,7 +177,7 @@ llvm::Expected<int64_t> ClangTidyCheck::OptionsView::getEnumInt(
   if (Iter == CheckOptions.end())
     return llvm::make_error<MissingOptionError>((NamePrefix + LocalName).str());
 
-  StringRef Value = Iter->second.Value;
+  StringRef Value = Iter->getValue().Value;
   StringRef Closest;
   unsigned EditDistance = -1;
   for (const auto &NameAndEnum : Mapping) {
@@ -199,9 +199,9 @@ llvm::Expected<int64_t> ClangTidyCheck::OptionsView::getEnumInt(
   }
   if (EditDistance < 3)
     return llvm::make_error<UnparseableEnumOptionError>(
-        Iter->first, Iter->second.Value, std::string(Closest));
-  return llvm::make_error<UnparseableEnumOptionError>(Iter->first,
-                                                      Iter->second.Value);
+        Iter->getKey().str(), Iter->getValue().Value, Closest.str());
+  return llvm::make_error<UnparseableEnumOptionError>(Iter->getKey().str(),
+                                                      Iter->getValue().Value);
 }
 
 void ClangTidyCheck::OptionsView::logErrToStdErr(llvm::Error &&Err) {
diff --git a/clang-tools-extra/clang-tidy/ClangTidyOptions.cpp b/clang-tools-extra/clang-tidy/ClangTidyOptions.cpp
index bb5a4b513967d..19ba47f005dc8 100644
--- a/clang-tools-extra/clang-tidy/ClangTidyOptions.cpp
+++ b/clang-tools-extra/clang-tidy/ClangTidyOptions.cpp
@@ -70,7 +70,7 @@ struct NOptionMap {
   NOptionMap(IO &, const ClangTidyOptions::OptionMap &OptionMap) {
     Options.reserve(OptionMap.size());
     for (const auto &KeyValue : OptionMap)
-      Options.emplace_back(KeyValue.first, KeyValue.second.Value);
+      Options.emplace_back(KeyValue.getKey(), KeyValue.getValue().Value);
   }
   ClangTidyOptions::OptionMap denormalize(IO &) {
     ClangTidyOptions::OptionMap Map;
@@ -157,8 +157,10 @@ ClangTidyOptions ClangTidyOptions::mergeWith(const ClangTidyOptions &Other,
   mergeVectors(Result.ExtraArgsBefore, Other.ExtraArgsBefore);
 
   for (const auto &KeyValue : Other.CheckOptions) {
-    Result.CheckOptions[KeyValue.first] = ClangTidyValue(
-        KeyValue.second.Value, KeyValue.second.Priority + Priority);
+    Result.CheckOptions.insert_or_assign(
+        KeyValue.getKey(),
+        ClangTidyValue(KeyValue.getValue().Value,
+                       KeyValue.getValue().Priority + Priority));
   }
 
   return Result;
diff --git a/clang-tools-extra/clang-tidy/ClangTidyOptions.h b/clang-tools-extra/clang-tidy/ClangTidyOptions.h
index 0f3c1d413ec30..6bfcae0162e20 100644
--- a/clang-tools-extra/clang-tidy/ClangTidyOptions.h
+++ b/clang-tools-extra/clang-tidy/ClangTidyOptions.h
@@ -16,7 +16,6 @@
 #include "llvm/Support/ErrorOr.h"
 #include "llvm/Support/VirtualFileSystem.h"
 #include <functional>
-#include <map>
 #include <string>
 #include <system_error>
 #include <utility>
@@ -108,7 +107,7 @@ struct ClangTidyOptions {
     unsigned Priority;
   };
   typedef std::pair<std::string, std::string> StringPair;
-  typedef std::map<std::string, ClangTidyValue> OptionMap;
+  typedef llvm::StringMap<ClangTidyValue> OptionMap;
 
   /// Key-value mapping used to store check-specific options.
   OptionMap CheckOptions;
diff --git a/clang-tools-extra/test/clang-tidy/checkers/google-module.cpp b/clang-tools-extra/test/clang-tidy/checkers/google-module.cpp
index 494ac48e17633..2c82237e4186d 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/google-module.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/google-module.cpp
@@ -1,10 +1,6 @@
 // RUN: clang-tidy -checks='-*,google*' -config='{}' -dump-config - -- | FileCheck %s
 // CHECK: CheckOptions:
-// CHECK: {{- key: *google-readability-braces-around-statements.ShortStatementLines}}
-// CHECK-NEXT: {{value: *'1'}}
-// CHECK: {{- key: *google-readability-function-size.StatementThreshold}}
-// CHECK-NEXT: {{value: *'800'}}
-// CHECK: {{- key: *google-readability-namespace-comments.ShortNamespaceLines}}
-// CHECK-NEXT: {{value: *'10'}}
-// CHECK: {{- key: *google-readability-namespace-comments.SpacesBeforeComments}}
-// CHECK-NEXT: {{value: *'2'}}
+// CHECK-DAG: {{- key: *google-readability-braces-around-statements.ShortStatementLines *[[:space:]] *value: *'1'}}
+// CHECK-DAG: {{- key: *google-readability-function-size.StatementThreshold *[[:space:]] *value: *'800'}}
+// CHECK-DAG: {{- key: *google-readability-namespace-comments.ShortNamespaceLines *[[:space:]] *value: *'10'}}
+// CHECK-DAG: {{- key: *google-readability-namespace-comments.SpacesBeforeComments *[[:space:]] *value: *'2'}}
diff --git a/clang-tools-extra/test/clang-tidy/infrastructure/config-files.cpp b/clang-tools-extra/test/clang-tidy/infrastructure/config-files.cpp
index d2a0a8c2a150f..d708ec8777c9a 100644
--- a/clang-tools-extra/test/clang-tidy/infrastructure/config-files.cpp
+++ b/clang-tools-extra/test/clang-tidy/infrastructure/config-files.cpp
@@ -17,14 +17,10 @@
 // For this test we have to use names of the real checks because otherwise values are ignored.
 // RUN: clang-tidy -dump-config %S/Inputs/config-files/4/44/- -- | FileCheck %s -check-prefix=CHECK-CHILD4
 // CHECK-CHILD4: Checks: {{.*}}modernize-loop-convert,modernize-use-using,llvm-qualified-auto
-// CHECK-CHILD4: - key: llvm-qualified-auto.AddConstToQualified
-// CHECK-CHILD4-NEXT: value: 'true'
-// CHECK-CHILD4: - key: modernize-loop-convert.MaxCopySize
-// CHECK-CHILD4-NEXT: value: '20'
-// CHECK-CHILD4: - key: modernize-loop-convert.MinConfidence
-// CHECK-CHILD4-NEXT: value: reasonable
-// CHECK-CHILD4: - key: modernize-use-using.IgnoreMacros
-// CHECK-CHILD4-NEXT: value: 'false'
+// CHECK-CHILD4-DAG: - key: llvm-qualified-auto.AddConstToQualified{{ *[[:space:]] *}}value: 'true'
+// CHECK-CHILD4-DAG: - key: modernize-loop-convert.MaxCopySize{{ *[[:space:]] *}}value: '20'
+// CHECK-CHILD4-DAG: - key: modernize-loop-convert.MinConfidence{{ *[[:space:]] *}}value: reasonable
+// CHECK-CHILD4-DAG: - key: modernize-use-using.IgnoreMacros{{ *[[:space:]] *}}value: 'false'
 
 // RUN: clang-tidy --explain-config %S/Inputs/config-files/4/44/- -- | FileCheck %s -check-prefix=CHECK-EXPLAIN
 // CHECK-EXPLAIN: 'llvm-qualified-auto' is enabled in the {{.*}}{{[/\\]}}Inputs{{[/\\]}}config-files{{[/\\]}}4{{[/\\]}}44{{[/\\]}}.clang-tidy.
@@ -37,16 +33,13 @@
 // RUN: CheckOptions: [{key: modernize-loop-convert.MaxCopySize, value: 21}]}' \
 // RUN: %S/Inputs/config-files/4/44/- -- | FileCheck %s -check-prefix=CHECK-CHILD5
 // CHECK-CHILD5: Checks: {{.*}}modernize-loop-convert,modernize-use-using,llvm-qualified-auto,-llvm-qualified-auto
-// CHECK-CHILD5: - key: modernize-loop-convert.MaxCopySize
-// CHECK-CHILD5-NEXT: value: '21'
-// CHECK-CHILD5: - key: modernize-loop-convert.MinConfidence
-// CHECK-CHILD5-NEXT: value: reasonable
-// CHECK-CHILD5: - key: modernize-use-using.IgnoreMacros
-// CHECK-CHILD5-NEXT: value: 'false'
+// CHECK-CHILD5-DAG: - key: modernize-loop-convert.MaxCopySize{{ *[[:space:]] *}}value: '21'
+// CHECK-CHILD5-DAG: - key: modernize-loop-convert.MinConfidence{{ *[[:space:]] *}}value: reasonable
+// CHECK-CHILD5-DAG: - key: modernize-use-using.IgnoreMacros{{ *[[:space:]] *}}value: 'false'
 
 // RUN: clang-tidy -dump-config \
 // RUN: --config='{InheritParentConfig: false, \
 // RUN: Checks: -llvm-qualified-auto}' \
 // RUN: %S/Inputs/config-files/4/44/- -- | FileCheck %s -check-prefix=CHECK-CHILD6
-// CHECK-CHILD6: Checks: {{.*}}-llvm-qualified-auto
+// CHECK-CHILD6: Checks: {{.*-llvm-qualified-auto'? *$}}
 // CHECK-CHILD6-NOT: - key: modernize-use-using.IgnoreMacros

From 276ed5f7e4ee82e02abaa37ffc2dfa090ea6167e Mon Sep 17 00:00:00 2001
From: Sam Tebbs <samuel.tebbs@arm.com>
Date: Wed, 22 Jul 2020 10:36:14 +0100
Subject: [PATCH 0595/1035] [DAGCombiner] Fold sext_inreg of a masked load into
 a sign extended masked load

This patch adds a DAG combine fold for a sext(masked_load) into a sign extended masked load.

Differential Revision: https://reviews.llvm.org/D84332
---
 llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp |  16 +++
 .../CodeGen/Thumb2/mve-sext-masked-load.ll    | 102 ++++++++++++++++++
 2 files changed, 118 insertions(+)
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-sext-masked-load.ll

diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 8a5a7f098bd74..b22d978d37366 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -11129,6 +11129,22 @@ SDValue DAGCombiner::visitSIGN_EXTEND_INREG(SDNode *N) {
     return SDValue(N, 0);   // Return N so it doesn't get rechecked!
   }
 
+  // fold (sext_inreg (masked_load x)) -> (sext_masked_load x)
+  // ignore it if the masked load is already sign extended
+  if (MaskedLoadSDNode *Ld = dyn_cast<MaskedLoadSDNode>(N0)) {
+    if (ExtVT == Ld->getMemoryVT() && N0.hasOneUse() &&
+        Ld->getExtensionType() != ISD::LoadExtType::NON_EXTLOAD &&
+        TLI.isLoadExtLegal(ISD::SEXTLOAD, VT, ExtVT)) {
+      SDValue ExtMaskedLoad = DAG.getMaskedLoad(
+          VT, SDLoc(N), Ld->getChain(), Ld->getBasePtr(), Ld->getOffset(),
+          Ld->getMask(), Ld->getPassThru(), ExtVT, Ld->getMemOperand(),
+          Ld->getAddressingMode(), ISD::SEXTLOAD, Ld->isExpandingLoad());
+      CombineTo(N, ExtMaskedLoad);
+      CombineTo(N0.getNode(), ExtMaskedLoad, ExtMaskedLoad.getValue(1));
+      return SDValue(N, 0); // Return N so it doesn't get rechecked!
+    }
+  }
+
   // Form (sext_inreg (bswap >> 16)) or (sext_inreg (rotl (bswap) 16))
   if (ExtVTBits <= 16 && N0.getOpcode() == ISD::OR) {
     if (SDValue BSwap = MatchBSwapHWordLow(N0.getNode(), N0.getOperand(0),
diff --git a/llvm/test/CodeGen/Thumb2/mve-sext-masked-load.ll b/llvm/test/CodeGen/Thumb2/mve-sext-masked-load.ll
new file mode 100644
index 0000000000000..54249151d448e
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-sext-masked-load.ll
@@ -0,0 +1,102 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve.fp,+fp64 -verify-machineinstrs -o - %s | FileCheck %s
+
+define arm_aapcs_vfpcc <4 x float> @foo_v4i16(<4 x i16>* nocapture readonly %pSrc, i32 %blockSize, <4 x i16> %a) {
+; CHECK-LABEL: foo_v4i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmovlb.s16 q0, q0
+; CHECK-NEXT:    vpt.s32 lt, q0, zr
+; CHECK-NEXT:    vldrht.s32 q0, [r0]
+; CHECK-NEXT:    vcvt.f32.s32 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %active.lane.mask = icmp slt <4 x i16> %a, zeroinitializer
+  %wide.masked.load = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %pSrc, i32 2, <4 x i1> %active.lane.mask, <4 x i16> undef)
+  %0 = sitofp <4 x i16> %wide.masked.load to <4 x float>
+  ret <4 x float> %0
+}
+
+define arm_aapcs_vfpcc <8 x half> @foo_v8i8(<8 x i8>* nocapture readonly %pSrc, i32 %blockSize, <8 x i8> %a) {
+; CHECK-LABEL: foo_v8i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmovlb.s8 q0, q0
+; CHECK-NEXT:    vpt.s16 lt, q0, zr
+; CHECK-NEXT:    vldrbt.s16 q0, [r0]
+; CHECK-NEXT:    vcvt.f16.s16 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %active.lane.mask = icmp slt <8 x i8> %a, zeroinitializer
+  %wide.masked.load = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %pSrc, i32 1, <8 x i1> %active.lane.mask, <8 x i8> undef)
+  %0 = sitofp <8 x i8> %wide.masked.load to <8 x half>
+  ret <8 x half> %0
+}
+
+define arm_aapcs_vfpcc <4 x float> @foo_v4i8(<4 x i8>* nocapture readonly %pSrc, i32 %blockSize, <4 x i8> %a) {
+; CHECK-LABEL: foo_v4i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmovlb.s8 q0, q0
+; CHECK-NEXT:    vmovlb.s16 q0, q0
+; CHECK-NEXT:    vpt.s32 lt, q0, zr
+; CHECK-NEXT:    vldrbt.s32 q0, [r0]
+; CHECK-NEXT:    vcvt.f32.s32 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %active.lane.mask = icmp slt <4 x i8> %a, zeroinitializer
+  %wide.masked.load = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %pSrc, i32 1, <4 x i1> %active.lane.mask, <4 x i8> undef)
+  %0 = sitofp <4 x i8> %wide.masked.load to <4 x float>
+  ret <4 x float> %0
+}
+
+define arm_aapcs_vfpcc <4 x double> @foo_v4i32(<4 x i32>* nocapture readonly %pSrc, i32 %blockSize, <4 x i32> %a) {
+; CHECK-LABEL: foo_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT:    vpt.s32 lt, q0, zr
+; CHECK-NEXT:    vldrwt.u32 q5, [r0]
+; CHECK-NEXT:    vmov.f64 d8, d10
+; CHECK-NEXT:    vmov.f32 s18, s21
+; CHECK-NEXT:    vmov r0, s18
+; CHECK-NEXT:    asrs r1, r0, #31
+; CHECK-NEXT:    bl __aeabi_l2d
+; CHECK-NEXT:    vmov r2, s16
+; CHECK-NEXT:    vmov d9, r0, r1
+; CHECK-NEXT:    asrs r3, r2, #31
+; CHECK-NEXT:    mov r0, r2
+; CHECK-NEXT:    mov r1, r3
+; CHECK-NEXT:    bl __aeabi_l2d
+; CHECK-NEXT:    vmov.f64 d12, d11
+; CHECK-NEXT:    vmov.f32 s26, s23
+; CHECK-NEXT:    vmov d8, r0, r1
+; CHECK-NEXT:    vmov r2, s26
+; CHECK-NEXT:    asrs r3, r2, #31
+; CHECK-NEXT:    mov r0, r2
+; CHECK-NEXT:    mov r1, r3
+; CHECK-NEXT:    bl __aeabi_l2d
+; CHECK-NEXT:    vmov r2, s24
+; CHECK-NEXT:    vmov d11, r0, r1
+; CHECK-NEXT:    asrs r3, r2, #31
+; CHECK-NEXT:    mov r0, r2
+; CHECK-NEXT:    mov r1, r3
+; CHECK-NEXT:    bl __aeabi_l2d
+; CHECK-NEXT:    vmov d10, r0, r1
+; CHECK-NEXT:    vmov q0, q4
+; CHECK-NEXT:    vmov q1, q5
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %active.lane.mask = icmp slt <4 x i32> %a, zeroinitializer
+  %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %pSrc, i32 4, <4 x i1> %active.lane.mask, <4 x i32> undef)
+  %0 = sitofp <4 x i32> %wide.masked.load to <4 x double>
+  ret <4 x double> %0
+}
+
+declare <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>*, i32 immarg, <4 x i1>, <4 x i16>)
+
+declare <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>*, i32 immarg, <8 x i1>, <8 x i8>)
+
+declare <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>*, i32 immarg, <4 x i1>, <4 x i8>)
+
+declare <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>*, i32 immarg, <4 x i1>, <4 x i32>)

From 006f6f8ac6fe920e978f65d92bb44825833f2933 Mon Sep 17 00:00:00 2001
From: Xing GUO <higuoxing@gmail.com>
Date: Thu, 30 Jul 2020 17:39:39 +0800
Subject: [PATCH 0596/1035] [DWARFYAML] Make the 'AddressSize',
 'SegmentSelectorSize' fields optional.

This patch makes the 'AddressSize' and 'SegmentSelectorSize' fields of
address range table optional.

Reviewed By: jhenderson

Differential Revision: https://reviews.llvm.org/D84907
---
 llvm/include/llvm/ObjectYAML/DWARFYAML.h      |  2 +-
 llvm/lib/ObjectYAML/DWARFEmitter.cpp          | 19 ++++---
 llvm/lib/ObjectYAML/DWARFYAML.cpp             |  4 +-
 .../ObjectYAML/MachO/DWARF-debug_aranges.yaml |  1 -
 .../yaml2obj/ELF/DWARF/debug-aranges.yaml     | 52 +++++++++++++++++++
 5 files changed, 68 insertions(+), 10 deletions(-)

diff --git a/llvm/include/llvm/ObjectYAML/DWARFYAML.h b/llvm/include/llvm/ObjectYAML/DWARFYAML.h
index 6b8ab07274acc..5375074817fa7 100644
--- a/llvm/include/llvm/ObjectYAML/DWARFYAML.h
+++ b/llvm/include/llvm/ObjectYAML/DWARFYAML.h
@@ -69,7 +69,7 @@ struct ARange {
   yaml::Hex64 Length;
   uint16_t Version;
   yaml::Hex64 CuOffset;
-  yaml::Hex8 AddrSize;
+  Optional<yaml::Hex8> AddrSize;
   yaml::Hex8 SegSize;
   std::vector<ARangeDescriptor> Descriptors;
 };
diff --git a/llvm/lib/ObjectYAML/DWARFEmitter.cpp b/llvm/lib/ObjectYAML/DWARFEmitter.cpp
index 66fabe875f7b0..0790eb6234400 100644
--- a/llvm/lib/ObjectYAML/DWARFEmitter.cpp
+++ b/llvm/lib/ObjectYAML/DWARFEmitter.cpp
@@ -127,26 +127,33 @@ Error DWARFYAML::emitDebugAbbrev(raw_ostream &OS, const DWARFYAML::Data &DI) {
 Error DWARFYAML::emitDebugAranges(raw_ostream &OS, const DWARFYAML::Data &DI) {
   for (auto Range : DI.ARanges) {
     auto HeaderStart = OS.tell();
+
+    uint8_t AddrSize;
+    if (Range.AddrSize)
+      AddrSize = *Range.AddrSize;
+    else
+      AddrSize = DI.Is64BitAddrSize ? 8 : 4;
+
     writeInitialLength(Range.Format, Range.Length, OS, DI.IsLittleEndian);
     writeInteger((uint16_t)Range.Version, OS, DI.IsLittleEndian);
     writeDWARFOffset(Range.CuOffset, Range.Format, OS, DI.IsLittleEndian);
-    writeInteger((uint8_t)Range.AddrSize, OS, DI.IsLittleEndian);
+    writeInteger((uint8_t)AddrSize, OS, DI.IsLittleEndian);
     writeInteger((uint8_t)Range.SegSize, OS, DI.IsLittleEndian);
 
     auto HeaderSize = OS.tell() - HeaderStart;
-    auto FirstDescriptor = alignTo(HeaderSize, Range.AddrSize * 2);
+    auto FirstDescriptor = alignTo(HeaderSize, AddrSize * 2);
     ZeroFillBytes(OS, FirstDescriptor - HeaderSize);
 
     for (auto Descriptor : Range.Descriptors) {
-      if (Error Err = writeVariableSizedInteger(
-              Descriptor.Address, Range.AddrSize, OS, DI.IsLittleEndian))
+      if (Error Err = writeVariableSizedInteger(Descriptor.Address, AddrSize,
+                                                OS, DI.IsLittleEndian))
         return createStringError(errc::not_supported,
                                  "unable to write debug_aranges address: %s",
                                  toString(std::move(Err)).c_str());
-      cantFail(writeVariableSizedInteger(Descriptor.Length, Range.AddrSize, OS,
+      cantFail(writeVariableSizedInteger(Descriptor.Length, AddrSize, OS,
                                          DI.IsLittleEndian));
     }
-    ZeroFillBytes(OS, Range.AddrSize * 2);
+    ZeroFillBytes(OS, AddrSize * 2);
   }
 
   return Error::success();
diff --git a/llvm/lib/ObjectYAML/DWARFYAML.cpp b/llvm/lib/ObjectYAML/DWARFYAML.cpp
index fda1ba797a916..5c2675e1d1eb3 100644
--- a/llvm/lib/ObjectYAML/DWARFYAML.cpp
+++ b/llvm/lib/ObjectYAML/DWARFYAML.cpp
@@ -106,8 +106,8 @@ void MappingTraits<DWARFYAML::ARange>::mapping(IO &IO,
   IO.mapRequired("Length", ARange.Length);
   IO.mapRequired("Version", ARange.Version);
   IO.mapRequired("CuOffset", ARange.CuOffset);
-  IO.mapRequired("AddressSize", ARange.AddrSize);
-  IO.mapRequired("SegmentSelectorSize", ARange.SegSize);
+  IO.mapOptional("AddressSize", ARange.AddrSize);
+  IO.mapOptional("SegmentSelectorSize", ARange.SegSize, 0);
   IO.mapRequired("Descriptors", ARange.Descriptors);
 }
 
diff --git a/llvm/test/ObjectYAML/MachO/DWARF-debug_aranges.yaml b/llvm/test/ObjectYAML/MachO/DWARF-debug_aranges.yaml
index 868054b08a991..ecdd88a68dd0c 100644
--- a/llvm/test/ObjectYAML/MachO/DWARF-debug_aranges.yaml
+++ b/llvm/test/ObjectYAML/MachO/DWARF-debug_aranges.yaml
@@ -328,7 +328,6 @@ DWARF:
 # CHECK-NEXT:       Version:             2
 # CHECK-NEXT:       CuOffset:            0x0000000000000000
 # CHECK-NEXT:       AddressSize:         0x08
-# CHECK-NEXT:       SegmentSelectorSize: 0x00
 # CHECK-NEXT:       Descriptors:
 # CHECK-NEXT:         - Address: 0x0000000100000F50
 # CHECK-NEXT:           Length:  52
diff --git a/llvm/test/tools/yaml2obj/ELF/DWARF/debug-aranges.yaml b/llvm/test/tools/yaml2obj/ELF/DWARF/debug-aranges.yaml
index b1064a9ae2153..b11d107f5badf 100644
--- a/llvm/test/tools/yaml2obj/ELF/DWARF/debug-aranges.yaml
+++ b/llvm/test/tools/yaml2obj/ELF/DWARF/debug-aranges.yaml
@@ -458,3 +458,55 @@ DWARF:
       Descriptors:
         - Address: 0x1234
           Length:  0x1000
+
+## j) Test the default value of address_size and segment_selector_size in a 64/32-bit object file.
+
+# RUN: yaml2obj --docnum=10 -DCLASS=ELFCLASS64 %s -o %t10.64-bit.o
+# RUN: llvm-readelf --hex-dump=.debug_aranges %t10.64-bit.o | \
+# RUN:   FileCheck %s --check-prefix=DEFAULT64
+
+#      DEFAULT64: Hex dump of section '.debug_aranges':
+# DEFAULT64-NEXT: 0x00000000 2c000000 02000000 00000800 00000000 ,...............
+##                           ^-------                            unit_length (4-byte)
+##                                    ^---                       version (2-byte)
+##                                        ^--------              debug_info_offset (4-byte)
+##                                                 ^-            address_size (1-byte)
+##                                                   ^-          segment_selector_size (1-byte)
+##                                                      ^------- padding (4-byte)
+# DEFAULT64-NEXT: 0x00000010 34120000 00000000 00100000 00000000 4...............
+##                           ^----------------                   address (8-byte)
+##                                             ^---------------- length (8-byte)
+# DEFAULT64-NEXT: 0x00000020 00000000 00000000 00000000 00000000 ................
+##                           ^---------------------------------- terminating entry (16-byte)
+
+# RUN: yaml2obj --docnum=10 -DCLASS=ELFCLASS32 %s -o %t10.32-bit.o
+# RUN: llvm-readelf --hex-dump=.debug_aranges %t10.32-bit.o | \
+# RUN:   FileCheck %s --check-prefix=DEFAULT32
+
+#      DEFAULT32: Hex dump of section '.debug_aranges':
+# DEFAULT32-NEXT: 0x00000000 2c000000 02000000 00000400 00000000 ,...............
+##                           ^-------                            unit_length (4-byte)
+##                                    ^---                       version (2-byte)
+##                                        ^--------              debug_info_offset (4-byte)
+##                                                 ^-            address_size (1-byte)
+##                                                   ^-          segment_selector_size (1-byte)
+##                                                      ^------- padding (4-byte)
+# DEFAULT32-NEXT: 0x00000010 34120000 00100000 00000000 00000000 4...............
+##                           ^-------                            address (4-byte)
+##                                    ^-------                   length (4-byte)
+##                                             ^---------------- terminating entry (8-byte)
+
+--- !ELF
+FileHeader:
+  Class:   [[CLASS]]
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+DWARF:
+  debug_aranges:
+    - Length:              0x2c
+      Version:             2
+      CuOffset:            0
+      Descriptors:
+        - Address: 0x1234
+          Length:  0x1000

From 3da6a974db86ba7d1f9a17a1e229daeade52a038 Mon Sep 17 00:00:00 2001
From: Xing GUO <higuoxing@gmail.com>
Date: Thu, 30 Jul 2020 17:42:18 +0800
Subject: [PATCH 0597/1035] [DWARFYAML] Make the 'Length' field of the address
 range table optional.

This patch makes the 'Length' field of the address range table optional.

Reviewed By: jhenderson

Differential Revision: https://reviews.llvm.org/D84911
---
 llvm/include/llvm/ObjectYAML/DWARFYAML.h      |  2 +-
 llvm/lib/ObjectYAML/DWARFEmitter.cpp          | 27 ++++--
 llvm/lib/ObjectYAML/DWARFYAML.cpp             |  2 +-
 .../yaml2obj/ELF/DWARF/debug-aranges.yaml     | 85 +++++++++++++++++++
 4 files changed, 107 insertions(+), 9 deletions(-)

diff --git a/llvm/include/llvm/ObjectYAML/DWARFYAML.h b/llvm/include/llvm/ObjectYAML/DWARFYAML.h
index 5375074817fa7..32382da6de504 100644
--- a/llvm/include/llvm/ObjectYAML/DWARFYAML.h
+++ b/llvm/include/llvm/ObjectYAML/DWARFYAML.h
@@ -66,7 +66,7 @@ struct ARangeDescriptor {
 
 struct ARange {
   dwarf::DwarfFormat Format;
-  yaml::Hex64 Length;
+  Optional<yaml::Hex64> Length;
   uint16_t Version;
   yaml::Hex64 CuOffset;
   Optional<yaml::Hex8> AddrSize;
diff --git a/llvm/lib/ObjectYAML/DWARFEmitter.cpp b/llvm/lib/ObjectYAML/DWARFEmitter.cpp
index 0790eb6234400..be9581fa38083 100644
--- a/llvm/lib/ObjectYAML/DWARFEmitter.cpp
+++ b/llvm/lib/ObjectYAML/DWARFEmitter.cpp
@@ -126,23 +126,36 @@ Error DWARFYAML::emitDebugAbbrev(raw_ostream &OS, const DWARFYAML::Data &DI) {
 
 Error DWARFYAML::emitDebugAranges(raw_ostream &OS, const DWARFYAML::Data &DI) {
   for (auto Range : DI.ARanges) {
-    auto HeaderStart = OS.tell();
-
     uint8_t AddrSize;
     if (Range.AddrSize)
       AddrSize = *Range.AddrSize;
     else
       AddrSize = DI.Is64BitAddrSize ? 8 : 4;
 
-    writeInitialLength(Range.Format, Range.Length, OS, DI.IsLittleEndian);
+    uint64_t Length = 4; // sizeof(version) 2 + sizeof(address_size) 1 +
+                         // sizeof(segment_selector_size) 1
+    Length +=
+        Range.Format == dwarf::DWARF64 ? 8 : 4; // sizeof(debug_info_offset)
+
+    const uint64_t HeaderLength =
+        Length + (Range.Format == dwarf::DWARF64
+                      ? 12
+                      : 4); // sizeof(unit_header) = 12 (DWARF64) or 4 (DWARF32)
+    const uint64_t PaddedHeaderLength = alignTo(HeaderLength, AddrSize * 2);
+
+    if (Range.Length) {
+      Length = *Range.Length;
+    } else {
+      Length += PaddedHeaderLength - HeaderLength;
+      Length += AddrSize * 2 * (Range.Descriptors.size() + 1);
+    }
+
+    writeInitialLength(Range.Format, Length, OS, DI.IsLittleEndian);
     writeInteger((uint16_t)Range.Version, OS, DI.IsLittleEndian);
     writeDWARFOffset(Range.CuOffset, Range.Format, OS, DI.IsLittleEndian);
     writeInteger((uint8_t)AddrSize, OS, DI.IsLittleEndian);
     writeInteger((uint8_t)Range.SegSize, OS, DI.IsLittleEndian);
-
-    auto HeaderSize = OS.tell() - HeaderStart;
-    auto FirstDescriptor = alignTo(HeaderSize, AddrSize * 2);
-    ZeroFillBytes(OS, FirstDescriptor - HeaderSize);
+    ZeroFillBytes(OS, PaddedHeaderLength - HeaderLength);
 
     for (auto Descriptor : Range.Descriptors) {
       if (Error Err = writeVariableSizedInteger(Descriptor.Address, AddrSize,
diff --git a/llvm/lib/ObjectYAML/DWARFYAML.cpp b/llvm/lib/ObjectYAML/DWARFYAML.cpp
index 5c2675e1d1eb3..7dd289ff6973b 100644
--- a/llvm/lib/ObjectYAML/DWARFYAML.cpp
+++ b/llvm/lib/ObjectYAML/DWARFYAML.cpp
@@ -103,7 +103,7 @@ void MappingTraits<DWARFYAML::ARangeDescriptor>::mapping(
 void MappingTraits<DWARFYAML::ARange>::mapping(IO &IO,
                                                DWARFYAML::ARange &ARange) {
   IO.mapOptional("Format", ARange.Format, dwarf::DWARF32);
-  IO.mapRequired("Length", ARange.Length);
+  IO.mapOptional("Length", ARange.Length);
   IO.mapRequired("Version", ARange.Version);
   IO.mapRequired("CuOffset", ARange.CuOffset);
   IO.mapOptional("AddressSize", ARange.AddrSize);
diff --git a/llvm/test/tools/yaml2obj/ELF/DWARF/debug-aranges.yaml b/llvm/test/tools/yaml2obj/ELF/DWARF/debug-aranges.yaml
index b11d107f5badf..9fa86449053d0 100644
--- a/llvm/test/tools/yaml2obj/ELF/DWARF/debug-aranges.yaml
+++ b/llvm/test/tools/yaml2obj/ELF/DWARF/debug-aranges.yaml
@@ -510,3 +510,88 @@ DWARF:
       Descriptors:
         - Address: 0x1234
           Length:  0x1000
+
+## k) Test that yaml2obj is able to determine the correct length for the address range table
+## if the 'Length' field isn't specified.
+
+# RUN: yaml2obj --docnum=11 -DCLASS=ELFCLASS64 -DADDRSIZE=4 %s -o %t11.64-bit.o
+# RUN: llvm-readelf --hex-dump=.debug_aranges %t11.64-bit.o | \
+# RUN:   FileCheck %s --check-prefix=LENGTH64
+
+#      LENGTH64: Hex dump of section '.debug_aranges':
+# LENGTH64-NEXT: 0x00000000 2c000000 02000000 00000800 00000000 ,...............
+##                          ^-------                            unit_length (4-byte)
+##                                   ^---                       version (2-byte)
+##                                       ^--------              debug_info_offset (4-byte)
+##                                                ^-            address_size (1-byte)
+##                                                  ^-          segment_selector_size (1-byte)
+##                                                     ^------- padding (4-byte)
+# LENGTH64-NEXT: 0x00000010 34120000 00000000 00100000 00000000 4...............
+##                          ^----------------                   address (8-byte)
+##                                            ^---------------- length (8-byte)
+# LENGTH64-NEXT: 0x00000020 00000000 00000000 00000000 00000000 ................
+##                          ^---------------------------------- terminating entry (16-byte)
+# LENGTH64-NEXT: 0x00000030 ffffffff 1c000000 00000000 02000000 ................
+##                          ^-------------------------          unit_length (12-byte)
+##                                                     ^---     version (2-byte)
+##                                                         ^--- debug_info_offset (8-byte)
+# LENGTH64-NEXT: 0x00000040 00000000 00000400 34120000 21430000 ........4...!C..
+##                          -------------
+##                                       ^-                     address_size (1-byte)
+##                                         ^-                   segment_selector_size (1-byte)
+##                                            ^-------          address (4-byte)
+##                                                     ^------- length (4-byte)
+# LENGTH64-NEXT: 0x00000050 00000000 00000000                   ........
+##                          ^----------------                   terminating entry (8-byte)
+
+# RUN: yaml2obj --docnum=11 -DCLASS=ELFCLASS32 -DADDRSIZE=8 %s -o %t11.32-bit.o
+# RUN: llvm-readelf --hex-dump=.debug_aranges %t11.32-bit.o | \
+# RUN:   FileCheck %s --check-prefix=LENGTH32
+
+#      LENGTH32: Hex dump of section '.debug_aranges':
+# LENGTH32-NEXT: 0x00000000 1c000000 02000000 00000400 00000000 ................
+##                          ^-------                            unit_length (4-byte)
+##                                   ^---                       version (2-byte)
+##                                       ^--------              debug_info_offset (4-byte)
+##                                                ^-            address_size (1-byte)
+##                                                  ^-          segment_selector_size (1-byte)
+##                                                     ^------- padding (4-byte)
+# LENGTH32-NEXT: 0x00000010 34120000 00100000 00000000 00000000 4...............
+##                          ^-------                            address (4-byte)
+##                                   ^-------                   length (4-byte)
+##                                            ^---------------- terminating entry (8-byte)
+# LENGTH32-NEXT: 0x00000020 ffffffff 34000000 00000000 02000000 ....4...........
+##                          ^-------------------------          unit_length (12-byte)
+##                                                     ^---     version (2-byte)
+##                                                         ^--- debug_info_offset (8-byte)
+# LENGTH32-NEXT: 0x00000030 00000000 00000800 00000000 00000000 ................
+##                          -------------
+##                                       ^-                     address_size (1-byte)
+##                                         ^-                   segment_selector_size (1-byte)
+##                                            ^---------------- padding (8-byte)
+# LENGTH32-NEXT: 0x00000040 34120000 00000000 21430000 00000000 4.......!C......
+##                          ^----------------                   address (8-byte)
+##                                            ^---------------- length (8-byte)
+# LENGTH32-NEXT: 0x00000050 00000000 00000000 00000000 00000000 ................
+##                          ^---------------------------------- terminating entry (16-byte)
+
+--- !ELF
+FileHeader:
+  Class:   [[CLASS]]
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+DWARF:
+  debug_aranges:
+    - Version:  2
+      CuOffset: 0
+      Descriptors:
+        - Address: 0x1234
+          Length:  0x1000
+    - Format:      DWARF64
+      Version:     2
+      AddressSize: [[ADDRSIZE]]
+      CuOffset:    0
+      Descriptors:
+        - Address: 0x1234
+          Length:  0x4321

From 05b173466142596b3297ab02e423574cb74b3799 Mon Sep 17 00:00:00 2001
From: Aleksandr Platonov <platonov.aleksandr@huawei.com>
Date: Thu, 30 Jul 2020 12:45:07 +0300
Subject: [PATCH 0598/1035] [clangd] findNearbyIdentifier(): fix the word
 search in the token stream.

Without this patch the word occurrence search always returns the first token of the file.
Despite of that, `findNeardyIdentifier()` returns the correct result (but inefficently) until there are several matched tokens with the same value `floor(log2(<token line> - <word line>))` (e.g. several matched tokens on the same line).

Reviewed By: kadircet

Differential Revision: https://reviews.llvm.org/D84912
---
 clang-tools-extra/clangd/XRefs.cpp                | 2 +-
 clang-tools-extra/clangd/unittests/XRefsTests.cpp | 9 ++++++++-
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/clang-tools-extra/clangd/XRefs.cpp b/clang-tools-extra/clangd/XRefs.cpp
index aeff7ebc32a26..cf747b607f4a1 100644
--- a/clang-tools-extra/clangd/XRefs.cpp
+++ b/clang-tools-extra/clangd/XRefs.cpp
@@ -518,7 +518,7 @@ const syntax::Token *findNearbyIdentifier(const SpelledWord &Word,
   // Find where the word occurred in the token stream, to search forward & back.
   auto *I = llvm::partition_point(SpelledTokens, [&](const syntax::Token &T) {
     assert(SM.getFileID(T.location()) == SM.getFileID(Word.Location));
-    return T.location() >= Word.Location; // Comparison OK: same file.
+    return T.location() < Word.Location; // Comparison OK: same file.
   });
   // Search for matches after the cursor.
   for (const syntax::Token &Tok : llvm::makeArrayRef(I, SpelledTokens.end()))
diff --git a/clang-tools-extra/clangd/unittests/XRefsTests.cpp b/clang-tools-extra/clangd/unittests/XRefsTests.cpp
index 0428303f5b0af..0a8f85ed53176 100644
--- a/clang-tools-extra/clangd/unittests/XRefsTests.cpp
+++ b/clang-tools-extra/clangd/unittests/XRefsTests.cpp
@@ -1197,7 +1197,14 @@ TEST(LocateSymbol, NearbyIdentifier) {
 
       // h^i
     )cpp",
-  };
+      R"cpp(
+      // prefer nearest occurrence even if several matched tokens
+      // have the same value of `floor(log2(<token line> - <word line>))`.
+      int hello;
+      int x = hello, y = hello;
+      int z = [[hello]];
+      // h^ello
+    )cpp"};
   for (const char *Test : Tests) {
     Annotations T(Test);
     auto AST = TestTU::withCode(T.code()).build();

From 141b64a340a42ef59fe42f54faa91059ede44980 Mon Sep 17 00:00:00 2001
From: Esme-Yi <esme.yi@ibm.com>
Date: Thu, 30 Jul 2020 10:05:04 +0000
Subject: [PATCH 0599/1035] [NFC] Failed cases for some patterns defined in
 DAGCombiner.cpp

---
 .../test/CodeGen/PowerPC/combine_ext_trunc.ll | 67 +++++++++++++++++++
 1 file changed, 67 insertions(+)
 create mode 100644 llvm/test/CodeGen/PowerPC/combine_ext_trunc.ll

diff --git a/llvm/test/CodeGen/PowerPC/combine_ext_trunc.ll b/llvm/test/CodeGen/PowerPC/combine_ext_trunc.ll
new file mode 100644
index 0000000000000..67768bfe51f38
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/combine_ext_trunc.ll
@@ -0,0 +1,67 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=powerpc64le-unknown-unknown -verify-machineinstrs | FileCheck %s
+
+; FIXME: We have implemented the following patterns in DAGCombiner.cpp,
+; but we can't get results as expected.
+
+; fold (or (and X, (xor Y, -1)), Y) to (or X, Y)
+define i32 @pattern1(i32 %x, i32 %y){
+; CHECK-LABEL: pattern1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xori 5, 4, 65535
+; CHECK-NEXT:    xoris 5, 5, 65535
+; CHECK-NEXT:    and 3, 3, 5
+; CHECK-NEXT:    or 3, 3, 4
+; CHECK-NEXT:    blr
+    %a = xor i32 %y, -1
+    %b = and i32 %x, %a
+    %c = or i32 %b, %y
+    ret i32 %c
+}
+
+; fold (or (and (xor Y, -1), X), Y) to (or X, Y)
+define i32 @pattern2(i32 %x, i32 %y){
+; CHECK-LABEL: pattern2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xori 5, 4, 65535
+; CHECK-NEXT:    xoris 5, 5, 65535
+; CHECK-NEXT:    and 3, 5, 3
+; CHECK-NEXT:    or 3, 3, 4
+; CHECK-NEXT:    blr
+    %a = xor i32 %y, -1
+    %b = and i32 %a, %x
+    %c = or i32 %b, %y
+    ret i32 %c
+}
+
+; fold (and (select Cond, 0, -1), X) to (select Cond, 0, X)
+define i32 @pattern3(i1 %cond, i32 %x) {
+; CHECK-LABEL: pattern3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li 5, 0
+; CHECK-NEXT:    andi. 3, 3, 1
+; CHECK-NEXT:    oris 3, 5, 65535
+; CHECK-NEXT:    ori 3, 3, 65535
+; CHECK-NEXT:    iselgt 3, 0, 3
+; CHECK-NEXT:    and 3, 3, 4
+; CHECK-NEXT:    blr
+  %sel = select i1 %cond, i32 0, i32 -1
+  %res = and i32 %sel, %x
+  ret i32 %res
+}
+
+; fold (or X, (select Cond, -1, 0)) to (select Cond, -1, X)
+define i32 @pattern4(i1 %cond, i32 %x) {
+; CHECK-LABEL: pattern4:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li 5, 0
+; CHECK-NEXT:    andi. 3, 3, 1
+; CHECK-NEXT:    oris 3, 5, 65535
+; CHECK-NEXT:    ori 3, 3, 65535
+; CHECK-NEXT:    iselgt 3, 3, 5
+; CHECK-NEXT:    or 3, 4, 3
+; CHECK-NEXT:    blr
+  %sel = select i1 %cond, i32 -1, i32 0
+  %res = or i32 %x, %sel
+  ret i32 %res
+}

From a4a0844248d4a68a866b9c4e18ae89fa49a83ec0 Mon Sep 17 00:00:00 2001
From: Raphael Isemann <teemperor@gmail.com>
Date: Thu, 30 Jul 2020 11:50:53 +0200
Subject: [PATCH 0600/1035] [lldb] Don't use static locals for return value
 storage in some *AsCString functions

Let's just return a std::string to make this safe. formatv seemed overkill for formatting
the return values as they all just append an integer value to a constant string.

Reviewed By: labath

Differential Revision: https://reviews.llvm.org/D84505
---
 lldb/include/lldb/Core/Communication.h             |  2 +-
 lldb/include/lldb/Target/Thread.h                  |  4 ++--
 lldb/source/Core/Communication.cpp                 | 13 +++++--------
 .../Process/MacOSX-Kernel/CommunicationKDP.cpp     | 11 +++++------
 .../Process/gdb-remote/GDBRemoteCommunication.cpp  |  2 +-
 lldb/source/Target/Thread.cpp                      | 14 ++++----------
 lldb/source/Target/ThreadPlanCallFunction.cpp      |  6 +++---
 .../source/Target/ThreadPlanStepOverBreakpoint.cpp |  4 ++--
 8 files changed, 23 insertions(+), 33 deletions(-)

diff --git a/lldb/include/lldb/Core/Communication.h b/lldb/include/lldb/Core/Communication.h
index 6b65974f95225..354c4bbcc283c 100644
--- a/lldb/include/lldb/Core/Communication.h
+++ b/lldb/include/lldb/Core/Communication.h
@@ -285,7 +285,7 @@ class Communication : public Broadcaster {
   ///
   void SynchronizeWithReadThread();
 
-  static const char *ConnectionStatusAsCString(lldb::ConnectionStatus status);
+  static std::string ConnectionStatusAsString(lldb::ConnectionStatus status);
 
   bool GetCloseOnEOF() const { return m_close_on_eof; }
 
diff --git a/lldb/include/lldb/Target/Thread.h b/lldb/include/lldb/Target/Thread.h
index 205a0d965c63a..066b8e1845c04 100644
--- a/lldb/include/lldb/Target/Thread.h
+++ b/lldb/include/lldb/Target/Thread.h
@@ -253,9 +253,9 @@ class Thread : public std::enable_shared_from_this<Thread>,
 
   bool ThreadStoppedForAReason();
 
-  static const char *RunModeAsCString(lldb::RunMode mode);
+  static std::string RunModeAsString(lldb::RunMode mode);
 
-  static const char *StopReasonAsCString(lldb::StopReason reason);
+  static std::string StopReasonAsString(lldb::StopReason reason);
 
   virtual const char *GetInfo() { return nullptr; }
 
diff --git a/lldb/source/Core/Communication.cpp b/lldb/source/Core/Communication.cpp
index 859f5be74b439..b50cd0ecab5ce 100644
--- a/lldb/source/Core/Communication.cpp
+++ b/lldb/source/Core/Communication.cpp
@@ -339,7 +339,7 @@ lldb::thread_result_t Communication::ReadThread(lldb::thread_arg_t p) {
       }
       if (error.Fail())
         LLDB_LOG(log, "error: {0}, status = {1}", error,
-                 Communication::ConnectionStatusAsCString(status));
+                 Communication::ConnectionStatusAsString(status));
       break;
     case eConnectionStatusInterrupted: // Synchronization signal from
                                        // SynchronizeWithReadThread()
@@ -355,7 +355,7 @@ lldb::thread_result_t Communication::ReadThread(lldb::thread_arg_t p) {
     case eConnectionStatusTimedOut: // Request timed out
       if (error.Fail())
         LLDB_LOG(log, "error: {0}, status = {1}", error,
-                 Communication::ConnectionStatusAsCString(status));
+                 Communication::ConnectionStatusAsString(status));
       break;
     }
   }
@@ -416,8 +416,8 @@ void Communication::SetConnection(std::unique_ptr<Connection> connection) {
   m_connection_sp = std::move(connection);
 }
 
-const char *
-Communication::ConnectionStatusAsCString(lldb::ConnectionStatus status) {
+std::string
+Communication::ConnectionStatusAsString(lldb::ConnectionStatus status) {
   switch (status) {
   case eConnectionStatusSuccess:
     return "success";
@@ -435,8 +435,5 @@ Communication::ConnectionStatusAsCString(lldb::ConnectionStatus status) {
     return "interrupted";
   }
 
-  static char unknown_state_string[64];
-  snprintf(unknown_state_string, sizeof(unknown_state_string),
-           "ConnectionStatus = %i", status);
-  return unknown_state_string;
+  return "@" + std::to_string(status);
 }
diff --git a/lldb/source/Plugins/Process/MacOSX-Kernel/CommunicationKDP.cpp b/lldb/source/Plugins/Process/MacOSX-Kernel/CommunicationKDP.cpp
index 06190d0c036de..dc283fce81046 100644
--- a/lldb/source/Plugins/Process/MacOSX-Kernel/CommunicationKDP.cpp
+++ b/lldb/source/Plugins/Process/MacOSX-Kernel/CommunicationKDP.cpp
@@ -194,12 +194,11 @@ size_t CommunicationKDP::WaitForPacketWithTimeoutMicroSecondsNoLock(
                                  : std::chrono::microseconds(timeout_usec),
                              status, &error);
 
-    LLDB_LOGV(log, 
-      "Read (buffer, sizeof(buffer), timeout_usec = 0x{0:x}, "
-                  "status = {1}, error = {2}) => bytes_read = {4}",
-                  timeout_usec,
-                  Communication::ConnectionStatusAsCString(status),
-                  error, bytes_read);
+    LLDB_LOGV(log,
+              "Read (buffer, sizeof(buffer), timeout_usec = 0x{0:x}, "
+              "status = {1}, error = {2}) => bytes_read = {4}",
+              timeout_usec, Communication::ConnectionStatusAsString(status),
+              error, bytes_read);
 
     if (bytes_read > 0) {
       if (CheckForPacket(buffer, bytes_read, packet))
diff --git a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunication.cpp b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunication.cpp
index 6a60502416959..832760f7f0dcc 100644
--- a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunication.cpp
+++ b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunication.cpp
@@ -284,7 +284,7 @@ GDBRemoteCommunication::WaitForPacketNoLock(StringExtractorGDBRemote &packet,
     LLDB_LOGV(log,
               "Read(buffer, sizeof(buffer), timeout = {0}, "
               "status = {1}, error = {2}) => bytes_read = {3}",
-              timeout, Communication::ConnectionStatusAsCString(status), error,
+              timeout, Communication::ConnectionStatusAsString(status), error,
               bytes_read);
 
     if (bytes_read > 0) {
diff --git a/lldb/source/Target/Thread.cpp b/lldb/source/Target/Thread.cpp
index 24cf4bf3ee1ed..ad28603cc02ec 100644
--- a/lldb/source/Target/Thread.cpp
+++ b/lldb/source/Target/Thread.cpp
@@ -1661,7 +1661,7 @@ Thread::GetStackFrameSPForStackFramePtr(StackFrame *stack_frame_ptr) {
   return GetStackFrameList()->GetStackFrameSPForStackFramePtr(stack_frame_ptr);
 }
 
-const char *Thread::StopReasonAsCString(lldb::StopReason reason) {
+std::string Thread::StopReasonAsString(lldb::StopReason reason) {
   switch (reason) {
   case eStopReasonInvalid:
     return "invalid";
@@ -1687,13 +1687,10 @@ const char *Thread::StopReasonAsCString(lldb::StopReason reason) {
     return "instrumentation break";
   }
 
-  static char unknown_state_string[64];
-  snprintf(unknown_state_string, sizeof(unknown_state_string),
-           "StopReason = %i", reason);
-  return unknown_state_string;
+  return "StopReason = " + std::to_string(reason);
 }
 
-const char *Thread::RunModeAsCString(lldb::RunMode mode) {
+std::string Thread::RunModeAsString(lldb::RunMode mode) {
   switch (mode) {
   case eOnlyThisThread:
     return "only this thread";
@@ -1703,10 +1700,7 @@ const char *Thread::RunModeAsCString(lldb::RunMode mode) {
     return "only during stepping";
   }
 
-  static char unknown_state_string[64];
-  snprintf(unknown_state_string, sizeof(unknown_state_string), "RunMode = %i",
-           mode);
-  return unknown_state_string;
+  return "RunMode = " + std::to_string(mode);
 }
 
 size_t Thread::GetStatus(Stream &strm, uint32_t start_frame,
diff --git a/lldb/source/Target/ThreadPlanCallFunction.cpp b/lldb/source/Target/ThreadPlanCallFunction.cpp
index dbe26f42c9bf7..f525173f8a51f 100644
--- a/lldb/source/Target/ThreadPlanCallFunction.cpp
+++ b/lldb/source/Target/ThreadPlanCallFunction.cpp
@@ -260,9 +260,9 @@ bool ThreadPlanCallFunction::DoPlanExplainsStop(Event *event_ptr) {
     stop_reason = eStopReasonNone;
   else
     stop_reason = m_real_stop_info_sp->GetStopReason();
-  LLDB_LOGF(log,
-            "ThreadPlanCallFunction::PlanExplainsStop: Got stop reason - %s.",
-            Thread::StopReasonAsCString(stop_reason));
+  LLDB_LOG(log,
+           "ThreadPlanCallFunction::PlanExplainsStop: Got stop reason - {0}.",
+           Thread::StopReasonAsString(stop_reason));
 
   if (stop_reason == eStopReasonBreakpoint && BreakpointsExplainStop())
     return true;
diff --git a/lldb/source/Target/ThreadPlanStepOverBreakpoint.cpp b/lldb/source/Target/ThreadPlanStepOverBreakpoint.cpp
index f3d35a91fcbc3..f188d827faae4 100644
--- a/lldb/source/Target/ThreadPlanStepOverBreakpoint.cpp
+++ b/lldb/source/Target/ThreadPlanStepOverBreakpoint.cpp
@@ -62,8 +62,8 @@ bool ThreadPlanStepOverBreakpoint::DoPlanExplainsStop(Event *event_ptr) {
     StopReason reason = stop_info_sp->GetStopReason();
 
     Log *log(lldb_private::GetLogIfAllCategoriesSet(LIBLLDB_LOG_STEP));
-    LLDB_LOGF(log, "Step over breakpoint stopped for reason: %s.",
-              Thread::StopReasonAsCString(reason));
+    LLDB_LOG(log, "Step over breakpoint stopped for reason: {0}.",
+             Thread::StopReasonAsString(reason));
 
     switch (reason) {
       case eStopReasonTrace:

From 44a4ba859d8b0247e631f84645b37b433bdefc7a Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Thu, 30 Jul 2020 11:17:52 +0100
Subject: [PATCH 0601/1035] [AArch64] Add machine-combiner tests with
 instruction level FMFs.

---
 .../AArch64/machine-combiner-instr-fmf.mir    | 405 ++++++++++++++++++
 1 file changed, 405 insertions(+)
 create mode 100644 llvm/test/CodeGen/AArch64/machine-combiner-instr-fmf.mir

diff --git a/llvm/test/CodeGen/AArch64/machine-combiner-instr-fmf.mir b/llvm/test/CodeGen/AArch64/machine-combiner-instr-fmf.mir
new file mode 100644
index 0000000000000..23ed96a42b513
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/machine-combiner-instr-fmf.mir
@@ -0,0 +1,405 @@
+# RUN: llc -run-pass=machine-combiner -o - -simplify-mir -mtriple=arm64-apple-iphoneos %s | FileCheck %s
+
+# Can create FMADD, because both the fmul and fadd have all fast-math flags.
+#
+# CHECK-LABEL: name: scalar_fmadd_fast
+# CHECK:        [[C:%.*]]:fpr32 = COPY $s2
+# CHECK-NEXT:   [[B:%.*]]:fpr32 = COPY $s1
+# CHECK-NEXT:   [[A:%.*]]:fpr32 = COPY $s0
+# CHECK-NEXT:   [[MUL:%.*]]:fpr32 = nnan ninf nsz arcp contract afn reassoc FMULSrr [[B]], [[A]]
+# CHECK-NEXT:   fpr32 = nnan ninf nsz arcp contract afn reassoc FADDSrr killed [[MUL]], [[C]]
+---
+name:            scalar_fmadd_fast
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fpr32 }
+  - { id: 1, class: fpr32 }
+  - { id: 2, class: fpr32 }
+  - { id: 3, class: fpr32 }
+  - { id: 4, class: fpr32 }
+liveins:
+  - { reg: '$s0', virtual-reg: '%0' }
+  - { reg: '$s1', virtual-reg: '%1' }
+  - { reg: '$s2', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+  maxCallFrameSize: 0
+machineFunctionInfo: {}
+body:             |
+  bb.0.entry:
+    liveins: $s0, $s1, $s2
+
+    %2:fpr32 = COPY $s2
+    %1:fpr32 = COPY $s1
+    %0:fpr32 = COPY $s0
+    %3:fpr32 = nnan ninf nsz arcp contract afn reassoc FMULSrr %1, %0
+    %4:fpr32 = nnan ninf nsz arcp contract afn reassoc FADDSrr killed %3, %2
+    $s0 = COPY %4
+    RET_ReallyLR implicit $s0
+
+...
+
+# Can create FMADD, because both the fmul and fadd have the contract fast-math flag.
+#
+# CHECK-LABEL: name: scalar_fmadd_contract
+# CHECK:        [[C:%.*]]:fpr32 = COPY $s2
+# CHECK-NEXT:   [[B:%.*]]:fpr32 = COPY $s1
+# CHECK-NEXT:   [[A:%.*]]:fpr32 = COPY $s0
+# CHECK-NEXT:   [[MUL:%.*]]:fpr32 = contract FMULSrr [[B]], [[A]]
+# CHECK-NEXT:   fpr32 = contract FADDSrr killed [[MUL]], [[C]]
+
+---
+name:            scalar_fmadd_contract
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fpr32 }
+  - { id: 1, class: fpr32 }
+  - { id: 2, class: fpr32 }
+  - { id: 3, class: fpr32 }
+  - { id: 4, class: fpr32 }
+liveins:
+  - { reg: '$s0', virtual-reg: '%0' }
+  - { reg: '$s1', virtual-reg: '%1' }
+  - { reg: '$s2', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+  maxCallFrameSize: 0
+machineFunctionInfo: {}
+body:             |
+  bb.0.entry:
+    liveins: $s0, $s1, $s2
+
+    %2:fpr32 = COPY $s2
+    %1:fpr32 = COPY $s1
+    %0:fpr32 = COPY $s0
+    %3:fpr32 = contract FMULSrr %1, %0
+    %4:fpr32 = contract FADDSrr killed %3, %2
+    $s0 = COPY %4
+    RET_ReallyLR implicit $s0
+
+...
+
+# Do not create FMADD, because we don't have the contract flag on moth instructions.
+
+# CHECK-LABEL: name: scalar_fmadd_contract_op0
+# CHECK:        [[C:%.*]]:fpr32 = COPY $s2
+# CHECK-NEXT:   [[B:%.*]]:fpr32 = COPY $s1
+# CHECK-NEXT:   [[A:%.*]]:fpr32 = COPY $s0
+# CHECK-NEXT:   [[MUL:%.*]]:fpr32 = contract FMULSrr [[B]], [[A]]
+# CHECK-NEXT:   fpr32 = FADDSrr killed [[MUL]], [[C]]
+---
+name:            scalar_fmadd_contract_op0
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fpr32 }
+  - { id: 1, class: fpr32 }
+  - { id: 2, class: fpr32 }
+  - { id: 3, class: fpr32 }
+  - { id: 4, class: fpr32 }
+liveins:
+  - { reg: '$s0', virtual-reg: '%0' }
+  - { reg: '$s1', virtual-reg: '%1' }
+  - { reg: '$s2', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+  maxCallFrameSize: 0
+machineFunctionInfo: {}
+body:             |
+  bb.0.entry:
+    liveins: $s0, $s1, $s2
+
+    %2:fpr32 = COPY $s2
+    %1:fpr32 = COPY $s1
+    %0:fpr32 = COPY $s0
+    %3:fpr32 = contract FMULSrr %1, %0
+    %4:fpr32 = FADDSrr killed %3, %2
+    $s0 = COPY %4
+    RET_ReallyLR implicit $s0
+
+...
+
+# Do not create FMADD, because we don't have the contract flag on moth instructions.
+#
+# CHECK-LABEL: name: scalar_fmadd_contract_op1
+# CHECK:        [[C:%.*]]:fpr32 = COPY $s2
+# CHECK-NEXT:   [[B:%.*]]:fpr32 = COPY $s1
+# CHECK-NEXT:   [[A:%.*]]:fpr32 = COPY $s0
+# CHECK-NEXT:   [[MUL:%.*]]:fpr32 = FMULSrr [[B]], [[A]]
+# CHECK-NEXT:   fpr32 = contract FADDSrr killed [[MUL]], [[C]]
+
+---
+name:            scalar_fmadd_contract_op1
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fpr32 }
+  - { id: 1, class: fpr32 }
+  - { id: 2, class: fpr32 }
+  - { id: 3, class: fpr32 }
+  - { id: 4, class: fpr32 }
+liveins:
+  - { reg: '$s0', virtual-reg: '%0' }
+  - { reg: '$s1', virtual-reg: '%1' }
+  - { reg: '$s2', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+  maxCallFrameSize: 0
+machineFunctionInfo: {}
+body:             |
+  bb.0.entry:
+    liveins: $s0, $s1, $s2
+
+    %2:fpr32 = COPY $s2
+    %1:fpr32 = COPY $s1
+    %0:fpr32 = COPY $s0
+    %3:fpr32 = FMULSrr %1, %0
+    %4:fpr32 = contract FADDSrr killed %3, %2
+    $s0 = COPY %4
+    RET_ReallyLR implicit $s0
+
+...
+
+# Do not create FMADD, as nsz flag does not allow it.
+#
+# CHECK-LABEL: name: scalar_fmadd_nsz
+# CHECK:        [[C:%.*]]:fpr32 = COPY $s2
+# CHECK-NEXT:   [[B:%.*]]:fpr32 = COPY $s1
+# CHECK-NEXT:   [[A:%.*]]:fpr32 = COPY $s0
+# CHECK-NEXT:   [[MUL:%.*]]:fpr32 = nsz FMULSrr [[B]], [[A]]
+# CHECK-NEXT:   fpr32 = nsz FADDSrr killed [[MUL]], [[C]]
+
+---
+name:            scalar_fmadd_nsz
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fpr32 }
+  - { id: 1, class: fpr32 }
+  - { id: 2, class: fpr32 }
+  - { id: 3, class: fpr32 }
+  - { id: 4, class: fpr32 }
+liveins:
+  - { reg: '$s0', virtual-reg: '%0' }
+  - { reg: '$s1', virtual-reg: '%1' }
+  - { reg: '$s2', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+  maxCallFrameSize: 0
+machineFunctionInfo: {}
+body:             |
+  bb.0.entry:
+    liveins: $s0, $s1, $s2
+
+    %2:fpr32 = COPY $s2
+    %1:fpr32 = COPY $s1
+    %0:fpr32 = COPY $s0
+    %3:fpr32 = nsz FMULSrr %1, %0
+    %4:fpr32 = nsz FADDSrr killed %3, %2
+    $s0 = COPY %4
+    RET_ReallyLR implicit $s0
+
+...
+
+# Can create FMADD, because both the fmul and fadd have all fast-math flags.
+#
+# CHECK-LABEL: name: vector_fmadd_fast
+# CHECK:       [[C:%.*]]:fpr128 = COPY $q2
+# CHECK-NEXT:  [[B:%.*]]:fpr128 = COPY $q1
+# CHECK-NEXT:  [[A:%.*]]:fpr128 = COPY $q0
+# CHECK-NEXT:  [[MUL:%.*]]:fpr128 = nnan ninf nsz arcp contract afn reassoc FMULv2f64 [[B]], [[A]]
+# CHECK-NEXT:  fpr128 = nnan ninf nsz arcp contract afn reassoc FADDv2f64 killed [[MUL]], [[C]]
+---
+name:            vector_fmadd_fast
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fpr128 }
+  - { id: 1, class: fpr128 }
+  - { id: 2, class: fpr128 }
+  - { id: 3, class: fpr128 }
+  - { id: 4, class: fpr128 }
+liveins:
+  - { reg: '$q0', virtual-reg: '%0' }
+  - { reg: '$q1', virtual-reg: '%1' }
+  - { reg: '$q2', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+  maxCallFrameSize: 0
+machineFunctionInfo: {}
+body:             |
+  bb.0.entry:
+    liveins: $q0, $q1, $q2
+
+    %2:fpr128 = COPY $q2
+    %1:fpr128 = COPY $q1
+    %0:fpr128 = COPY $q0
+    %3:fpr128 = nnan ninf nsz arcp contract afn reassoc FMULv2f64 %1, %0
+    %4:fpr128 = nnan ninf nsz arcp contract afn reassoc FADDv2f64 killed %3, %2
+    $q0 = COPY %4
+    RET_ReallyLR implicit $q0
+
+...
+
+# Can create FMADD, because both the fmul and fadd have the contract fast-math flag.
+#
+# CHECK-LABEL: name: vector_fmadd_contract
+# CHECK:       [[C:%.*]]:fpr128 = COPY $q2
+# CHECK-NEXT:  [[B:%.*]]:fpr128 = COPY $q1
+# CHECK-NEXT:  [[A:%.*]]:fpr128 = COPY $q0
+# CHECK-NEXT:  [[MUL:%.*]]:fpr128 = contract FMULv2f64 [[B]], [[A]]
+# CHECK-NEXT:  fpr128 = contract FADDv2f64 killed [[MUL]], [[C]]
+---
+name:            vector_fmadd_contract
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fpr128 }
+  - { id: 1, class: fpr128 }
+  - { id: 2, class: fpr128 }
+  - { id: 3, class: fpr128 }
+  - { id: 4, class: fpr128 }
+liveins:
+  - { reg: '$q0', virtual-reg: '%0' }
+  - { reg: '$q1', virtual-reg: '%1' }
+  - { reg: '$q2', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+  maxCallFrameSize: 0
+machineFunctionInfo: {}
+body:             |
+  bb.0.entry:
+    liveins: $q0, $q1, $q2
+
+    %2:fpr128 = COPY $q2
+    %1:fpr128 = COPY $q1
+    %0:fpr128 = COPY $q0
+    %3:fpr128 = contract FMULv2f64 %1, %0
+    %4:fpr128 = contract FADDv2f64 killed %3, %2
+    $q0 = COPY %4
+    RET_ReallyLR implicit $q0
+
+...
+
+# Do not create FMADD, because we don't have the contract flag on moth instructions.
+#
+# CHECK-LABEL: name: vector_fmadd_contract_op0
+# CHECK:       [[C:%.*]]:fpr128 = COPY $q2
+# CHECK-NEXT:  [[B:%.*]]:fpr128 = COPY $q1
+# CHECK-NEXT:  [[A:%.*]]:fpr128 = COPY $q0
+# CHECK-NEXT:  [[MUL:%.*]]:fpr128 = contract FMULv2f64 [[B]], [[A]]
+# CHECK-NEXT:  fpr128 = FADDv2f64 killed [[MUL]], [[C]]
+---
+name:            vector_fmadd_contract_op0
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fpr128 }
+  - { id: 1, class: fpr128 }
+  - { id: 2, class: fpr128 }
+  - { id: 3, class: fpr128 }
+  - { id: 4, class: fpr128 }
+liveins:
+  - { reg: '$q0', virtual-reg: '%0' }
+  - { reg: '$q1', virtual-reg: '%1' }
+  - { reg: '$q2', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+  maxCallFrameSize: 0
+machineFunctionInfo: {}
+body:             |
+  bb.0.entry:
+    liveins: $q0, $q1, $q2
+
+    %2:fpr128 = COPY $q2
+    %1:fpr128 = COPY $q1
+    %0:fpr128 = COPY $q0
+    %3:fpr128 = contract FMULv2f64 %1, %0
+    %4:fpr128 = FADDv2f64 killed %3, %2
+    $q0 = COPY %4
+    RET_ReallyLR implicit $q0
+
+...
+
+# Do not create FMADD, because we don't have the contract flag on moth instructions.
+#
+# CHECK-LABEL: name: vector_fmadd_contract_op1
+# CHECK:       [[C:%.*]]:fpr128 = COPY $q2
+# CHECK-NEXT:  [[B:%.*]]:fpr128 = COPY $q1
+# CHECK-NEXT:  [[A:%.*]]:fpr128 = COPY $q0
+# CHECK-NEXT:  [[MUL:%.*]]:fpr128 = FMULv2f64 [[B]], [[A]]
+# CHECK-NEXT:  fpr128 = contract FADDv2f64 killed [[MUL]], [[C]]
+
+---
+name:            vector_fmadd_contract_op1
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fpr128 }
+  - { id: 1, class: fpr128 }
+  - { id: 2, class: fpr128 }
+  - { id: 3, class: fpr128 }
+  - { id: 4, class: fpr128 }
+liveins:
+  - { reg: '$q0', virtual-reg: '%0' }
+  - { reg: '$q1', virtual-reg: '%1' }
+  - { reg: '$q2', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+  maxCallFrameSize: 0
+machineFunctionInfo: {}
+body:             |
+  bb.0.entry:
+    liveins: $q0, $q1, $q2
+
+    %2:fpr128 = COPY $q2
+    %1:fpr128 = COPY $q1
+    %0:fpr128 = COPY $q0
+    %3:fpr128 = FMULv2f64 %1, %0
+    %4:fpr128 = contract FADDv2f64 killed %3, %2
+    $q0 = COPY %4
+    RET_ReallyLR implicit $q0
+
+...
+
+# Do not create FMADD, as nsz flag does not allow it.
+#
+# CHECK-LABEL: name: vector_fmadd_nsz
+# CHECK:       [[C:%.*]]:fpr128 = COPY $q2
+# CHECK-NEXT:  [[B:%.*]]:fpr128 = COPY $q1
+# CHECK-NEXT:  [[A:%.*]]:fpr128 = COPY $q0
+# CHECK-NEXT:  [[MUL:%.*]]:fpr128 = nsz FMULv2f64 [[B]], [[A]]
+# CHECK-NEXT:  fpr128 = nsz FADDv2f64 killed [[MUL]], [[C]]
+---
+name:            vector_fmadd_nsz
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fpr128 }
+  - { id: 1, class: fpr128 }
+  - { id: 2, class: fpr128 }
+  - { id: 3, class: fpr128 }
+  - { id: 4, class: fpr128 }
+liveins:
+  - { reg: '$q0', virtual-reg: '%0' }
+  - { reg: '$q1', virtual-reg: '%1' }
+  - { reg: '$q2', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+  maxCallFrameSize: 0
+machineFunctionInfo: {}
+body:             |
+  bb.0.entry:
+    liveins: $q0, $q1, $q2
+
+    %2:fpr128 = COPY $q2
+    %1:fpr128 = COPY $q1
+    %0:fpr128 = COPY $q0
+    %3:fpr128 = nsz FMULv2f64 %1, %0
+    %4:fpr128 = nsz FADDv2f64 killed %3, %2
+    $q0 = COPY %4
+    RET_ReallyLR implicit $q0
+
+...

From a262f0fea46ce08008f3462c336c3d7107e98b27 Mon Sep 17 00:00:00 2001
From: Kirill Bobyrev <kbobyrev@google.com>
Date: Thu, 30 Jul 2020 12:57:20 +0200
Subject: [PATCH 0602/1035] [clangd] Implement Relations request for remote
 index

This is the last missing bit in the core remote index implementation. The only
remaining bits are some API refactorings (replacing Optional with Expected and
being better at reporting errors).

Reviewed By: kadircet

Differential Revision: https://reviews.llvm.org/D84894
---
 .../clangd/index/remote/Client.cpp            |  13 +-
 .../clangd/index/remote/Index.proto           |  24 +++
 .../index/remote/marshalling/Marshalling.cpp  |  65 +++++++-
 .../index/remote/marshalling/Marshalling.h    |  10 +-
 .../clangd/index/remote/server/Server.cpp     |  32 ++++
 .../unittests/remote/MarshallingTests.cpp     | 149 ++++++++++++------
 6 files changed, 234 insertions(+), 59 deletions(-)

diff --git a/clang-tools-extra/clangd/index/remote/Client.cpp b/clang-tools-extra/clangd/index/remote/Client.cpp
index 5a33fd2eaf143..af58645d17959 100644
--- a/clang-tools-extra/clangd/index/remote/Client.cpp
+++ b/clang-tools-extra/clangd/index/remote/Client.cpp
@@ -91,11 +91,16 @@ class IndexClient : public clangd::SymbolIndex {
     return streamRPC(Request, &remote::SymbolIndex::Stub::Refs, Callback);
   }
 
-  // FIXME(kirillbobyrev): Implement this.
   void
-  relations(const clangd::RelationsRequest &,
-            llvm::function_ref<void(const SymbolID &, const clangd::Symbol &)>)
-      const {}
+  relations(const clangd::RelationsRequest &Request,
+            llvm::function_ref<void(const SymbolID &, const clangd::Symbol &)>
+                Callback) const {
+    streamRPC(Request, &remote::SymbolIndex::Stub::Relations,
+              // Unpack protobuf Relation.
+              [&](std::pair<SymbolID, clangd::Symbol> SubjectAndObject) {
+                Callback(SubjectAndObject.first, SubjectAndObject.second);
+              });
+  }
 
   // IndexClient does not take any space since the data is stored on the
   // server.
diff --git a/clang-tools-extra/clangd/index/remote/Index.proto b/clang-tools-extra/clangd/index/remote/Index.proto
index 99c4e3329d670..305164ffef779 100644
--- a/clang-tools-extra/clangd/index/remote/Index.proto
+++ b/clang-tools-extra/clangd/index/remote/Index.proto
@@ -18,6 +18,8 @@ service SymbolIndex {
   rpc FuzzyFind(FuzzyFindRequest) returns (stream FuzzyFindReply) {}
 
   rpc Refs(RefsRequest) returns (stream RefsReply) {}
+
+  rpc Relations(RelationsRequest) returns (stream RelationsReply) {}
 }
 
 message LookupRequest { repeated string ids = 1; }
@@ -114,3 +116,25 @@ message HeaderWithReferences {
   string header = 1;
   uint32 references = 2;
 }
+
+message RelationsRequest {
+  repeated string subjects = 1;
+  uint32 predicate = 2;
+  uint32 limit = 3;
+}
+
+// The response is a stream of reference messages, and one terminating has_more
+// message.
+message RelationsReply {
+  oneof kind {
+    Relation stream_result = 1;
+    bool final_result = 2; // HasMore
+  }
+}
+
+// This struct does not mirror clangd::Relation but rather the arguments of
+// SymbolIndex::relations callback.
+message Relation {
+  string subject_id = 1;
+  Symbol object = 2;
+}
diff --git a/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp b/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp
index b2085bc21f486..270e15347ee03 100644
--- a/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp
+++ b/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp
@@ -10,6 +10,7 @@
 #include "Headers.h"
 #include "Index.pb.h"
 #include "Protocol.h"
+#include "index/Index.h"
 #include "index/Serialization.h"
 #include "index/Symbol.h"
 #include "index/SymbolID.h"
@@ -33,10 +34,10 @@ namespace remote {
 
 namespace {
 
-template <typename MessageT>
-llvm::Expected<llvm::DenseSet<SymbolID>> getIDs(MessageT *Message) {
+template <typename IDRange>
+llvm::Expected<llvm::DenseSet<SymbolID>> getIDs(IDRange IDs) {
   llvm::DenseSet<SymbolID> Result;
-  for (const auto &ID : Message->ids()) {
+  for (const auto &ID : IDs) {
     auto SID = SymbolID::fromStr(StringRef(ID));
     if (!SID)
       return SID.takeError();
@@ -69,7 +70,7 @@ Marshaller::Marshaller(llvm::StringRef RemoteIndexRoot,
 llvm::Expected<clangd::LookupRequest>
 Marshaller::fromProtobuf(const LookupRequest *Message) {
   clangd::LookupRequest Req;
-  auto IDs = getIDs(Message);
+  auto IDs = getIDs(Message->ids());
   if (!IDs)
     return IDs.takeError();
   Req.IDs = std::move(*IDs);
@@ -100,7 +101,7 @@ Marshaller::fromProtobuf(const FuzzyFindRequest *Message) {
 llvm::Expected<clangd::RefsRequest>
 Marshaller::fromProtobuf(const RefsRequest *Message) {
   clangd::RefsRequest Req;
-  auto IDs = getIDs(Message);
+  auto IDs = getIDs(Message->ids());
   if (!IDs)
     return IDs.takeError();
   Req.IDs = std::move(*IDs);
@@ -110,6 +111,19 @@ Marshaller::fromProtobuf(const RefsRequest *Message) {
   return Req;
 }
 
+llvm::Expected<clangd::RelationsRequest>
+Marshaller::fromProtobuf(const RelationsRequest *Message) {
+  clangd::RelationsRequest Req;
+  auto IDs = getIDs(Message->subjects());
+  if (!IDs)
+    return IDs.takeError();
+  Req.Subjects = std::move(*IDs);
+  Req.Predicate = static_cast<RelationKind>(Message->predicate());
+  if (Message->limit())
+    Req.Limit = Message->limit();
+  return Req;
+}
+
 llvm::Optional<clangd::Symbol> Marshaller::fromProtobuf(const Symbol &Message) {
   if (!Message.has_info() || !Message.has_canonical_declaration()) {
     elog("Cannot convert Symbol from protobuf (missing info, definition or "
@@ -178,6 +192,24 @@ llvm::Optional<clangd::Ref> Marshaller::fromProtobuf(const Ref &Message) {
   return Result;
 }
 
+llvm::Optional<std::pair<clangd::SymbolID, clangd::Symbol>>
+Marshaller::fromProtobuf(const Relation &Message) {
+  auto SubjectID = SymbolID::fromStr(Message.subject_id());
+  if (!SubjectID) {
+    elog("Cannot convert Relation from protobuf (invalid Subject ID): {0}",
+         SubjectID.takeError());
+    return llvm::None;
+  }
+  if (!Message.has_object()) {
+    elog("Cannot convert Relation from protobuf (missing Object): {0}");
+    return llvm::None;
+  }
+  auto Object = fromProtobuf(Message.object());
+  if (!Object)
+    return llvm::None;
+  return std::make_pair(*SubjectID, *Object);
+}
+
 LookupRequest Marshaller::toProtobuf(const clangd::LookupRequest &From) {
   LookupRequest RPCRequest;
   for (const auto &SymbolID : From.IDs)
@@ -216,6 +248,16 @@ RefsRequest Marshaller::toProtobuf(const clangd::RefsRequest &From) {
   return RPCRequest;
 }
 
+RelationsRequest Marshaller::toProtobuf(const clangd::RelationsRequest &From) {
+  RelationsRequest RPCRequest;
+  for (const auto &ID : From.Subjects)
+    RPCRequest.add_subjects(ID.str());
+  RPCRequest.set_predicate(static_cast<uint32_t>(From.Predicate));
+  if (From.Limit)
+    RPCRequest.set_limit(*From.Limit);
+  return RPCRequest;
+}
+
 llvm::Optional<Symbol> Marshaller::toProtobuf(const clangd::Symbol &From) {
   Symbol Result;
   Result.set_id(From.ID.str());
@@ -274,6 +316,19 @@ llvm::Optional<Ref> Marshaller::toProtobuf(const clangd::Ref &From) {
   return Result;
 }
 
+llvm::Optional<Relation> Marshaller::toProtobuf(const clangd::SymbolID &Subject,
+                                                const clangd::Symbol &Object) {
+  Relation Result;
+  *Result.mutable_subject_id() = Subject.str();
+  auto SerializedObject = toProtobuf(Object);
+  if (!SerializedObject) {
+    elog("Can not convert Relation to protobuf (invalid symbol): {0}", Object);
+    return llvm::None;
+  }
+  *Result.mutable_object() = *SerializedObject;
+  return Result;
+}
+
 llvm::Optional<std::string>
 Marshaller::relativePathToURI(llvm::StringRef RelativePath) {
   assert(LocalIndexRoot);
diff --git a/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.h b/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.h
index 5d82cdb7e7650..9588f1ef64113 100644
--- a/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.h
+++ b/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.h
@@ -38,14 +38,19 @@ class Marshaller {
   Marshaller() = delete;
   Marshaller(llvm::StringRef RemoteIndexRoot, llvm::StringRef LocalIndexRoot);
 
+  // FIXME(kirillbobyrev): Switch from Optional to Expected.
   llvm::Optional<clangd::Symbol> fromProtobuf(const Symbol &Message);
   llvm::Optional<clangd::Ref> fromProtobuf(const Ref &Message);
+  llvm::Optional<std::pair<clangd::SymbolID, clangd::Symbol>>
+  fromProtobuf(const Relation &Message);
 
   llvm::Expected<clangd::LookupRequest>
   fromProtobuf(const LookupRequest *Message);
   llvm::Expected<clangd::FuzzyFindRequest>
   fromProtobuf(const FuzzyFindRequest *Message);
   llvm::Expected<clangd::RefsRequest> fromProtobuf(const RefsRequest *Message);
+  llvm::Expected<clangd::RelationsRequest>
+  fromProtobuf(const RelationsRequest *Message);
 
   /// toProtobuf() functions serialize native clangd types and strip IndexRoot
   /// from the file paths specific to indexing machine. fromProtobuf() functions
@@ -54,9 +59,12 @@ class Marshaller {
   LookupRequest toProtobuf(const clangd::LookupRequest &From);
   FuzzyFindRequest toProtobuf(const clangd::FuzzyFindRequest &From);
   RefsRequest toProtobuf(const clangd::RefsRequest &From);
+  RelationsRequest toProtobuf(const clangd::RelationsRequest &From);
 
-  llvm::Optional<Ref> toProtobuf(const clangd::Ref &From);
   llvm::Optional<Symbol> toProtobuf(const clangd::Symbol &From);
+  llvm::Optional<Ref> toProtobuf(const clangd::Ref &From);
+  llvm::Optional<Relation> toProtobuf(const clangd::SymbolID &Subject,
+                                      const clangd::Symbol &Object);
 
   /// Translates \p RelativePath into the absolute path and builds URI for the
   /// user machine. This translation happens on the client side with the
diff --git a/clang-tools-extra/clangd/index/remote/server/Server.cpp b/clang-tools-extra/clangd/index/remote/server/Server.cpp
index c3f9efeb7ec05..077d58ea7db0c 100644
--- a/clang-tools-extra/clangd/index/remote/server/Server.cpp
+++ b/clang-tools-extra/clangd/index/remote/server/Server.cpp
@@ -169,6 +169,38 @@ class RemoteIndexServer final : public SymbolIndex::Service {
     return grpc::Status::OK;
   }
 
+  grpc::Status Relations(grpc::ServerContext *Context,
+                         const RelationsRequest *Request,
+                         grpc::ServerWriter<RelationsReply> *Reply) override {
+    trace::Span Tracer(RelationsRequest::descriptor()->name());
+    auto Req = ProtobufMarshaller->fromProtobuf(Request);
+    if (!Req) {
+      elog("Can not parse RelationsRequest from protobuf: {0}",
+           Req.takeError());
+      return grpc::Status::CANCELLED;
+    }
+    unsigned Sent = 0;
+    unsigned FailedToSend = 0;
+    Index->relations(
+        *Req, [&](const SymbolID &Subject, const clangd::Symbol &Object) {
+          auto SerializedItem = ProtobufMarshaller->toProtobuf(Subject, Object);
+          if (!SerializedItem) {
+            ++FailedToSend;
+            return;
+          }
+          RelationsReply NextMessage;
+          *NextMessage.mutable_stream_result() = *SerializedItem;
+          Reply->Write(NextMessage);
+          ++Sent;
+        });
+    RelationsReply LastMessage;
+    LastMessage.set_final_result(true);
+    Reply->Write(LastMessage);
+    SPAN_ATTACH(Tracer, "Sent", Sent);
+    SPAN_ATTACH(Tracer, "Failed to send", FailedToSend);
+    return grpc::Status::OK;
+  }
+
   std::unique_ptr<clangd::SymbolIndex> Index;
   std::unique_ptr<Marshaller> ProtobufMarshaller;
 };
diff --git a/clang-tools-extra/clangd/unittests/remote/MarshallingTests.cpp b/clang-tools-extra/clangd/unittests/remote/MarshallingTests.cpp
index f975d1c35e1ed..6abf16a8dd497 100644
--- a/clang-tools-extra/clangd/unittests/remote/MarshallingTests.cpp
+++ b/clang-tools-extra/clangd/unittests/remote/MarshallingTests.cpp
@@ -10,6 +10,7 @@
 #include "TestFS.h"
 #include "index/Index.h"
 #include "index/Ref.h"
+#include "index/Relation.h"
 #include "index/Serialization.h"
 #include "index/Symbol.h"
 #include "index/SymbolID.h"
@@ -18,6 +19,7 @@
 #include "clang/Index/IndexSymbol.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/Twine.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/StringSaver.h"
@@ -38,6 +40,51 @@ const char *testPathURI(llvm::StringRef Path,
   return Strings.save(URI.toString()).begin();
 }
 
+clangd::Symbol createSymbol(llvm::StringRef PathPrefix,
+                            llvm::UniqueStringSaver &Strings) {
+  clangd::Symbol Sym;
+  Sym.ID = llvm::cantFail(SymbolID::fromStr("057557CEBF6E6B2D"));
+
+  index::SymbolInfo Info;
+  Info.Kind = index::SymbolKind::Function;
+  Info.SubKind = index::SymbolSubKind::AccessorGetter;
+  Info.Lang = index::SymbolLanguage::CXX;
+  Info.Properties = static_cast<index::SymbolPropertySet>(
+      index::SymbolProperty::TemplateSpecialization);
+  Sym.SymInfo = Info;
+
+  Sym.Name = Strings.save("Foo");
+  Sym.Scope = Strings.save("llvm::foo::bar::");
+
+  clangd::SymbolLocation Location;
+  Location.Start.setLine(1);
+  Location.Start.setColumn(15);
+  Location.End.setLine(3);
+  Location.End.setColumn(121);
+  Location.FileURI = testPathURI(PathPrefix.str() + "Definition.cpp", Strings);
+  Sym.Definition = Location;
+
+  Location.Start.setLine(42);
+  Location.Start.setColumn(31);
+  Location.End.setLine(20);
+  Location.End.setColumn(400);
+  Location.FileURI = testPathURI(PathPrefix.str() + "Declaration.h", Strings);
+  Sym.CanonicalDeclaration = Location;
+
+  Sym.References = 9000;
+  Sym.Origin = clangd::SymbolOrigin::Static;
+  Sym.Signature = Strings.save("(int X, char Y, Type T)");
+  Sym.TemplateSpecializationArgs = Strings.save("<int, char, bool, Type>");
+  Sym.CompletionSnippetSuffix =
+      Strings.save("({1: int X}, {2: char Y}, {3: Type T})");
+  Sym.Documentation = Strings.save("This is my amazing Foo constructor!");
+  Sym.ReturnType = Strings.save("Foo");
+
+  Sym.Flags = clangd::Symbol::SymbolFlag::IndexedForCodeCompletion;
+
+  return Sym;
+}
+
 TEST(RemoteMarshallingTest, URITranslation) {
   llvm::BumpPtrAllocator Arena;
   llvm::UniqueStringSaver Strings(Arena);
@@ -86,52 +133,10 @@ TEST(RemoteMarshallingTest, URITranslation) {
 }
 
 TEST(RemoteMarshallingTest, SymbolSerialization) {
-  clangd::Symbol Sym;
-
-  auto ID = SymbolID::fromStr("057557CEBF6E6B2D");
-  ASSERT_TRUE(bool(ID));
-  Sym.ID = *ID;
-
-  index::SymbolInfo Info;
-  Info.Kind = index::SymbolKind::Function;
-  Info.SubKind = index::SymbolSubKind::AccessorGetter;
-  Info.Lang = index::SymbolLanguage::CXX;
-  Info.Properties = static_cast<index::SymbolPropertySet>(
-      index::SymbolProperty::TemplateSpecialization);
-  Sym.SymInfo = Info;
-
   llvm::BumpPtrAllocator Arena;
   llvm::UniqueStringSaver Strings(Arena);
 
-  Sym.Name = Strings.save("Foo");
-  Sym.Scope = Strings.save("llvm::foo::bar::");
-
-  clangd::SymbolLocation Location;
-  Location.Start.setLine(1);
-  Location.Start.setColumn(15);
-  Location.End.setLine(3);
-  Location.End.setColumn(121);
-  Location.FileURI = testPathURI("home/Definition.cpp", Strings);
-  Sym.Definition = Location;
-
-  Location.Start.setLine(42);
-  Location.Start.setColumn(31);
-  Location.End.setLine(20);
-  Location.End.setColumn(400);
-  Location.FileURI = testPathURI("home/Declaration.h", Strings);
-  Sym.CanonicalDeclaration = Location;
-
-  Sym.References = 9000;
-  Sym.Origin = clangd::SymbolOrigin::Static;
-  Sym.Signature = Strings.save("(int X, char Y, Type T)");
-  Sym.TemplateSpecializationArgs = Strings.save("<int, char, bool, Type>");
-  Sym.CompletionSnippetSuffix =
-      Strings.save("({1: int X}, {2: char Y}, {3: Type T})");
-  Sym.Documentation = Strings.save("This is my amazing Foo constructor!");
-  Sym.ReturnType = Strings.save("Foo");
-
-  Sym.Flags = clangd::Symbol::SymbolFlag::IndexedForCodeCompletion;
-
+  clangd::Symbol Sym = createSymbol("home/", Strings);
   Marshaller ProtobufMarshaller(testPath("home/"), testPath("home/"));
 
   // Check that symbols are exactly the same if the path to indexed project is
@@ -160,20 +165,17 @@ TEST(RemoteMarshallingTest, SymbolSerialization) {
   EXPECT_FALSE(ProtobufMarshaller.fromProtobuf(*Serialized));
 
   // Fail with an invalid URI.
-  Location.FileURI = "Not A URI";
-  Sym.Definition = Location;
+  Sym.Definition.FileURI = "Not A URI";
   EXPECT_FALSE(ProtobufMarshaller.toProtobuf(Sym));
 
   // Schemes other than "file" can not be used.
   auto UnittestURI = URI::create(testPath("home/SomePath.h"), "unittest");
   ASSERT_TRUE(bool(UnittestURI));
-  Location.FileURI = Strings.save(UnittestURI->toString()).begin();
-  Sym.Definition = Location;
+  Sym.Definition.FileURI = Strings.save(UnittestURI->toString()).begin();
   EXPECT_FALSE(ProtobufMarshaller.toProtobuf(Sym));
 
   // Passing root that is not prefix of the original file path.
-  Location.FileURI = testPathURI("home/File.h", Strings);
-  Sym.Definition = Location;
+  Sym.Definition.FileURI = testPathURI("home/File.h", Strings);
   // Check that the symbol is valid and passing the correct path works.
   Serialized = ProtobufMarshaller.toProtobuf(Sym);
   ASSERT_TRUE(Serialized);
@@ -342,6 +344,55 @@ TEST(RemoteMarshallingTest, RefsRequestFailingSerialization) {
   llvm::consumeError(Deserialized.takeError());
 }
 
+TEST(RemoteMarshallingTest, RelationsRequestSerialization) {
+  clangd::RelationsRequest Request;
+  Request.Subjects.insert(
+      llvm::cantFail(SymbolID::fromStr("0000000000000001")));
+  Request.Subjects.insert(
+      llvm::cantFail(SymbolID::fromStr("0000000000000002")));
+
+  Request.Limit = 9000;
+  Request.Predicate = RelationKind::BaseOf;
+
+  Marshaller ProtobufMarshaller(testPath("remote/"), testPath("local/"));
+
+  auto Serialized = ProtobufMarshaller.toProtobuf(Request);
+  EXPECT_EQ(static_cast<unsigned>(Serialized.subjects_size()),
+            Request.Subjects.size());
+  EXPECT_EQ(Serialized.limit(), Request.Limit);
+  EXPECT_EQ(static_cast<RelationKind>(Serialized.predicate()),
+            Request.Predicate);
+  auto Deserialized = ProtobufMarshaller.fromProtobuf(&Serialized);
+  ASSERT_TRUE(bool(Deserialized));
+  EXPECT_EQ(Deserialized->Subjects, Request.Subjects);
+  ASSERT_TRUE(Deserialized->Limit);
+  EXPECT_EQ(*Deserialized->Limit, Request.Limit);
+  EXPECT_EQ(Deserialized->Predicate, Request.Predicate);
+}
+
+TEST(RemoteMarshallingTest, RelationsRequestFailingSerialization) {
+  RelationsRequest Serialized;
+  Serialized.add_subjects("ZZZZZZZZZZZZZZZZ");
+  Marshaller ProtobufMarshaller(testPath("remote/"), testPath("local/"));
+  auto Deserialized = ProtobufMarshaller.fromProtobuf(&Serialized);
+  EXPECT_FALSE(Deserialized);
+  llvm::consumeError(Deserialized.takeError());
+}
+
+TEST(RemoteMarshallingTest, RelationsSerializion) {
+  llvm::BumpPtrAllocator Arena;
+  llvm::UniqueStringSaver Strings(Arena);
+
+  clangd::Symbol Sym = createSymbol("remote/", Strings);
+  SymbolID ID = llvm::cantFail(SymbolID::fromStr("0000000000000002"));
+  Marshaller ProtobufMarshaller(testPath("remote/"), testPath("local/"));
+  auto Serialized = ProtobufMarshaller.toProtobuf(ID, Sym);
+  auto Deserialized = ProtobufMarshaller.fromProtobuf(*Serialized);
+  ASSERT_TRUE(Deserialized);
+  EXPECT_THAT(Deserialized->first, ID);
+  EXPECT_THAT(Deserialized->second.ID, Sym.ID);
+}
+
 TEST(RemoteMarshallingTest, RelativePathToURITranslation) {
   Marshaller ProtobufMarshaller(/*RemoteIndexRoot=*/"",
                                 /*LocalIndexRoot=*/testPath("home/project/"));

From 2dec72ba5cd271f619a10d5cc230e384d3d86d1d Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Thu, 30 Jul 2020 11:27:37 +0100
Subject: [PATCH 0603/1035] [X86][SSE] combineExtractWithShuffle - extend
 extract(truncate(x),0) for any source vector size

As long as we can extract the lowest 128-bit subvector from the pre-truncated source vector, then we don't care what size it is.

The next stage will be to support non-zero extraction indices, as long as its still coming from the lowest 128-bit subvector.
---
 llvm/lib/Target/X86/X86ISelLowering.cpp    |  8 ++++---
 llvm/test/CodeGen/X86/vector-reduce-mul.ll | 26 +++++++++-------------
 2 files changed, 16 insertions(+), 18 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 8d95c066f5a0b..493d934e5381a 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -39081,10 +39081,12 @@ static SDValue combineExtractWithShuffle(SDNode *N, SelectionDAG &DAG,
   // Handle extract(truncate(x)) for 0'th index.
   // TODO: Treat this as a faux shuffle?
   // TODO: When can we use this for general indices?
-  if (ISD::TRUNCATE == Src.getOpcode() && SrcVT.is128BitVector() && IdxC == 0) {
+  if (ISD::TRUNCATE == Src.getOpcode() && IdxC == 0 &&
+      (SrcVT.getSizeInBits() % 128) == 0) {
     Src = extract128BitVector(Src.getOperand(0), 0, DAG, dl);
-    Src = DAG.getBitcast(SrcVT, Src);
-    return DAG.getNode(N->getOpcode(), dl, VT, Src, Idx);
+    MVT ExtractVT = MVT::getVectorVT(SrcSVT.getSimpleVT(), 128 / SrcEltBits);
+    return DAG.getNode(N->getOpcode(), dl, VT, DAG.getBitcast(ExtractVT, Src),
+                       Idx);
   }
 
   // Resolve the target shuffle inputs and mask.
diff --git a/llvm/test/CodeGen/X86/vector-reduce-mul.ll b/llvm/test/CodeGen/X86/vector-reduce-mul.ll
index b0e3a79b6ff19..a3cec079ab091 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-mul.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-mul.ll
@@ -1995,9 +1995,7 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm1
 ; AVX512BW-NEXT:    vpsrlw $8, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
-; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vmovd %xmm0, %eax
 ; AVX512BW-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512BW-NEXT:    vzeroupper
@@ -2006,26 +2004,24 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX512BWVL-LABEL: test_v32i8:
 ; AVX512BWVL:       # %bb.0:
 ; AVX512BWVL-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
-; AVX512BWVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
+; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
+; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
 ; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm1
 ; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
-; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512BWVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
+; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
 ; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm1
 ; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512BWVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
+; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
 ; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm1
 ; AVX512BWVL-NEXT:    vpsrld $16, %xmm1, %xmm1
-; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512BWVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
+; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
 ; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm1
 ; AVX512BWVL-NEXT:    vpsrlw $8, %xmm1, %xmm1
-; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512BWVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
-; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm0
+; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    vmovd %xmm0, %eax
 ; AVX512BWVL-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512BWVL-NEXT:    vzeroupper

From cc529285fd621c6f4f5de77c8de10ac0bb7ce190 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Thu, 30 Jul 2020 12:11:34 +0100
Subject: [PATCH 0604/1035] VectorUtils.h - reduce unnecessary includes. NFC.

Replace TargetLibraryInfo.h include with forward declaration and fix implicit dependencies.

Reduce SmallSet.h include to SmallVector.h include.
---
 llvm/include/llvm/Analysis/VectorUtils.h                    | 4 ++--
 llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp          | 2 ++
 llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp | 2 ++
 3 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/llvm/include/llvm/Analysis/VectorUtils.h b/llvm/include/llvm/Analysis/VectorUtils.h
index b1d7850442fba..a860652b2332a 100644
--- a/llvm/include/llvm/Analysis/VectorUtils.h
+++ b/llvm/include/llvm/Analysis/VectorUtils.h
@@ -14,12 +14,12 @@
 #define LLVM_ANALYSIS_VECTORUTILS_H
 
 #include "llvm/ADT/MapVector.h"
-#include "llvm/ADT/SmallSet.h"
+#include "llvm/ADT/SmallVector.h"
 #include "llvm/Analysis/LoopAccessAnalysis.h"
-#include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/Support/CheckedArithmetic.h"
 
 namespace llvm {
+class TargetLibraryInfo;
 
 /// Describes the type of Parameters
 enum class VFParamKind {
diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
index 4171d4a35b25f..04b3dceae4754 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
+++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
@@ -8,6 +8,7 @@
 
 #include "PPCTargetTransformInfo.h"
 #include "llvm/Analysis/CodeMetrics.h"
+#include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/CodeGen/BasicTTIImpl.h"
 #include "llvm/CodeGen/CostTable.h"
@@ -18,6 +19,7 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/Transforms/InstCombine/InstCombiner.h"
 #include "llvm/Transforms/Utils/Local.h"
+
 using namespace llvm;
 
 #define DEBUG_TYPE "ppctti"
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 120b544808bed..e2a95d6f67a86 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -13,9 +13,11 @@
 // pass. It should be easy to create an analysis pass around it if there
 // is a need (but D45420 needs to happen first).
 //
+
 #include "llvm/Transforms/Vectorize/LoopVectorizationLegality.h"
 #include "llvm/Analysis/Loads.h"
 #include "llvm/Analysis/LoopInfo.h"
+#include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/Analysis/VectorUtils.h"
 #include "llvm/IR/IntrinsicInst.h"

From e77624a3be942c7abba48942b3a8da3462070a3f Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Thu, 30 Jul 2020 10:15:47 +0100
Subject: [PATCH 0605/1035] [IPConstProp] Remove and move tests to SCCP.

As far as I know, ipconstprop has not been used in years and ipsccp has
been used instead. This has the potential for confusion and sometimes
leads people to spend time finding & reporting bugs as well as
updating it to work with the latest API changes.

This patch moves the tests over to SCCP. There's one functional difference
I am aware of: ipconstprop propagates for each call-site individually, so
for functions that are called with different constant arguments it can sometimes
produce better results than ipsccp (at much higher compile-time cost).But
IPSCCP can be thought to do so as well for internal functions and as mentioned
earlier, the pass seems unused in practice (and there are no plans on working
towards enabling it anytime).

Also discussed on llvm-dev:
http://lists.llvm.org/pipermail/llvm-dev/2020-July/143773.html

Reviewed By: jdoerfert

Differential Revision: https://reviews.llvm.org/D84447
---
 llvm/bindings/go/llvm/transforms_ipo.go       |   1 -
 .../bindings/ocaml/transforms/ipo/ipo_ocaml.c |   6 -
 .../ocaml/transforms/ipo/llvm_ipo.mli         |   5 -
 llvm/docs/Passes.rst                          |   9 -
 llvm/include/llvm-c/Transforms/IPO.h          |   3 -
 llvm/include/llvm/InitializePasses.h          |   1 -
 llvm/include/llvm/LinkAllPasses.h             |   1 -
 llvm/include/llvm/Transforms/IPO.h            |   6 -
 llvm/lib/Transforms/IPO/CMakeLists.txt        |   1 -
 .../Transforms/IPO/IPConstantPropagation.cpp  | 308 ------------------
 llvm/lib/Transforms/IPO/IPO.cpp               |   5 -
 .../Transforms/IPConstantProp/comdat-ipo.ll   |  34 --
 .../2008-06-09-WeakProp.ll                    |   2 +-
 .../{IPConstantProp => SCCP}/PR43857.ll       |   4 +-
 .../arg-count-mismatch.ll                     |   4 +-
 .../arg-type-mismatch.ll                      |   2 +-
 llvm/test/Transforms/SCCP/comdat-ipo.ll       |  14 +-
 .../{IPConstantProp => SCCP}/deadarg.ll       |   2 +-
 .../multiple_callbacks.ll                     |   7 +-
 .../{IPConstantProp => SCCP}/naked-return.ll  |   1 -
 .../openmp_parallel_for.ll                    |   4 +-
 .../{IPConstantProp => SCCP}/pthreads.ll      |   6 +-
 .../{IPConstantProp => SCCP}/recursion.ll     |   2 +-
 .../return-argument.ll                        |   8 +-
 .../return-constant.ll                        |   6 +-
 .../return-constants.ll                       |  18 +-
 .../thread_local_acs.ll                       |   4 +-
 llvm/utils/findoptdiff                        |   2 +-
 .../llvm/lib/Transforms/IPO/BUILD.gn          |   1 -
 29 files changed, 43 insertions(+), 424 deletions(-)
 delete mode 100644 llvm/lib/Transforms/IPO/IPConstantPropagation.cpp
 delete mode 100644 llvm/test/Transforms/IPConstantProp/comdat-ipo.ll
 rename llvm/test/Transforms/{IPConstantProp => SCCP}/2008-06-09-WeakProp.ll (91%)
 rename llvm/test/Transforms/{IPConstantProp => SCCP}/PR43857.ll (86%)
 rename llvm/test/Transforms/{IPConstantProp => SCCP}/arg-count-mismatch.ll (96%)
 rename llvm/test/Transforms/{IPConstantProp => SCCP}/arg-type-mismatch.ll (92%)
 rename llvm/test/Transforms/{IPConstantProp => SCCP}/deadarg.ll (77%)
 rename llvm/test/Transforms/{IPConstantProp => SCCP}/multiple_callbacks.ll (96%)
 rename llvm/test/Transforms/{IPConstantProp => SCCP}/naked-return.ll (97%)
 rename llvm/test/Transforms/{IPConstantProp => SCCP}/openmp_parallel_for.ll (98%)
 rename llvm/test/Transforms/{IPConstantProp => SCCP}/pthreads.ll (93%)
 rename llvm/test/Transforms/{IPConstantProp => SCCP}/recursion.ll (90%)
 rename llvm/test/Transforms/{IPConstantProp => SCCP}/return-argument.ll (91%)
 rename llvm/test/Transforms/{IPConstantProp => SCCP}/return-constant.ll (91%)
 rename llvm/test/Transforms/{IPConstantProp => SCCP}/return-constants.ll (70%)
 rename llvm/test/Transforms/{IPConstantProp => SCCP}/thread_local_acs.ll (92%)

diff --git a/llvm/bindings/go/llvm/transforms_ipo.go b/llvm/bindings/go/llvm/transforms_ipo.go
index 1dcb2af8bf243..8a158f208bef4 100644
--- a/llvm/bindings/go/llvm/transforms_ipo.go
+++ b/llvm/bindings/go/llvm/transforms_ipo.go
@@ -32,7 +32,6 @@ func (pm PassManager) AddFunctionAttrsPass()         { C.LLVMAddFunctionAttrsPas
 func (pm PassManager) AddFunctionInliningPass()      { C.LLVMAddFunctionInliningPass(pm.C) }
 func (pm PassManager) AddGlobalDCEPass()             { C.LLVMAddGlobalDCEPass(pm.C) }
 func (pm PassManager) AddGlobalOptimizerPass()       { C.LLVMAddGlobalOptimizerPass(pm.C) }
-func (pm PassManager) AddIPConstantPropagationPass() { C.LLVMAddIPConstantPropagationPass(pm.C) }
 func (pm PassManager) AddPruneEHPass()               { C.LLVMAddPruneEHPass(pm.C) }
 func (pm PassManager) AddIPSCCPPass()                { C.LLVMAddIPSCCPPass(pm.C) }
 func (pm PassManager) AddInternalizePass(allButMain bool) {
diff --git a/llvm/bindings/ocaml/transforms/ipo/ipo_ocaml.c b/llvm/bindings/ocaml/transforms/ipo/ipo_ocaml.c
index 9fcaa10534f6b..c0e213714ed02 100644
--- a/llvm/bindings/ocaml/transforms/ipo/ipo_ocaml.c
+++ b/llvm/bindings/ocaml/transforms/ipo/ipo_ocaml.c
@@ -73,12 +73,6 @@ CAMLprim value llvm_add_global_optimizer(LLVMPassManagerRef PM) {
   return Val_unit;
 }
 
-/* [`Module] Llvm.PassManager.t -> unit */
-CAMLprim value llvm_add_ip_constant_propagation(LLVMPassManagerRef PM) {
-  LLVMAddIPConstantPropagationPass(PM);
-  return Val_unit;
-}
-
 /* [`Module] Llvm.PassManager.t -> unit */
 CAMLprim value llvm_add_prune_eh(LLVMPassManagerRef PM) {
   LLVMAddPruneEHPass(PM);
diff --git a/llvm/bindings/ocaml/transforms/ipo/llvm_ipo.mli b/llvm/bindings/ocaml/transforms/ipo/llvm_ipo.mli
index 6507c5d92c2b1..a581924c6d549 100644
--- a/llvm/bindings/ocaml/transforms/ipo/llvm_ipo.mli
+++ b/llvm/bindings/ocaml/transforms/ipo/llvm_ipo.mli
@@ -56,11 +56,6 @@ external add_global_optimizer
   : [ `Module ] Llvm.PassManager.t -> unit
   = "llvm_add_global_optimizer"
 
-(** See the [llvm::createIPConstantPropagationPass] function. *)
-external add_ipc_propagation
-  : [ `Module ] Llvm.PassManager.t -> unit
-  = "llvm_add_ip_constant_propagation"
-
 (** See the [llvm::createPruneEHPass] function. *)
 external add_prune_eh
   : [ `Module ] Llvm.PassManager.t -> unit
diff --git a/llvm/docs/Passes.rst b/llvm/docs/Passes.rst
index e45adad98c157..13317ecc966e7 100644
--- a/llvm/docs/Passes.rst
+++ b/llvm/docs/Passes.rst
@@ -676,15 +676,6 @@ This pass loops over all of the functions in the input module, looking for a
 main function.  If a main function is found, all other functions and all global
 variables with initializers are marked as internal.
 
-``-ipconstprop``: Interprocedural constant propagation
-------------------------------------------------------
-
-This pass implements an *extremely* simple interprocedural constant propagation
-pass.  It could certainly be improved in many different ways, like using a
-worklist.  This pass makes arguments dead, but does not remove them.  The
-existing dead argument elimination pass should be run after this to clean up
-the mess.
-
 ``-ipsccp``: Interprocedural Sparse Conditional Constant Propagation
 --------------------------------------------------------------------
 
diff --git a/llvm/include/llvm-c/Transforms/IPO.h b/llvm/include/llvm-c/Transforms/IPO.h
index cde3d24609208..3f2cadf32366b 100644
--- a/llvm/include/llvm-c/Transforms/IPO.h
+++ b/llvm/include/llvm-c/Transforms/IPO.h
@@ -57,9 +57,6 @@ void LLVMAddGlobalDCEPass(LLVMPassManagerRef PM);
 /** See llvm::createGlobalOptimizerPass function. */
 void LLVMAddGlobalOptimizerPass(LLVMPassManagerRef PM);
 
-/** See llvm::createIPConstantPropagationPass function. */
-void LLVMAddIPConstantPropagationPass(LLVMPassManagerRef PM);
-
 /** See llvm::createPruneEHPass function. */
 void LLVMAddPruneEHPass(LLVMPassManagerRef PM);
 
diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h
index cce6a43504021..d23ecfb98ce92 100644
--- a/llvm/include/llvm/InitializePasses.h
+++ b/llvm/include/llvm/InitializePasses.h
@@ -179,7 +179,6 @@ void initializeGuardWideningLegacyPassPass(PassRegistry&);
 void initializeHardwareLoopsPass(PassRegistry&);
 void initializeHotColdSplittingLegacyPassPass(PassRegistry&);
 void initializeHWAddressSanitizerLegacyPassPass(PassRegistry &);
-void initializeIPCPPass(PassRegistry&);
 void initializeIPSCCPLegacyPassPass(PassRegistry&);
 void initializeIRCELegacyPassPass(PassRegistry&);
 void initializeIRTranslatorPass(PassRegistry&);
diff --git a/llvm/include/llvm/LinkAllPasses.h b/llvm/include/llvm/LinkAllPasses.h
index 90e2e24294d40..9d7ac2b3f3b99 100644
--- a/llvm/include/llvm/LinkAllPasses.h
+++ b/llvm/include/llvm/LinkAllPasses.h
@@ -116,7 +116,6 @@ namespace {
       (void) llvm::createGlobalsAAWrapperPass();
       (void) llvm::createGuardWideningPass();
       (void) llvm::createLoopGuardWideningPass();
-      (void) llvm::createIPConstantPropagationPass();
       (void) llvm::createIPSCCPPass();
       (void) llvm::createInductiveRangeCheckEliminationPass();
       (void) llvm::createIndVarSimplifyPass();
diff --git a/llvm/include/llvm/Transforms/IPO.h b/llvm/include/llvm/Transforms/IPO.h
index 28e454d3b0fc7..7b73eeaf8e45b 100644
--- a/llvm/include/llvm/Transforms/IPO.h
+++ b/llvm/include/llvm/Transforms/IPO.h
@@ -155,12 +155,6 @@ Pass *createArgumentPromotionPass(unsigned maxElements = 3);
 /// createOpenMPOptLegacyPass - OpenMP specific optimizations.
 Pass *createOpenMPOptLegacyPass();
 
-//===----------------------------------------------------------------------===//
-/// createIPConstantPropagationPass - This pass propagates constants from call
-/// sites into the bodies of functions.
-///
-ModulePass *createIPConstantPropagationPass();
-
 //===----------------------------------------------------------------------===//
 /// createIPSCCPPass - This pass propagates constants from call sites into the
 /// bodies of functions, and keeps track of whether basic blocks are executable
diff --git a/llvm/lib/Transforms/IPO/CMakeLists.txt b/llvm/lib/Transforms/IPO/CMakeLists.txt
index 3f30c0289e8c3..a17ef63371063 100644
--- a/llvm/lib/Transforms/IPO/CMakeLists.txt
+++ b/llvm/lib/Transforms/IPO/CMakeLists.txt
@@ -18,7 +18,6 @@ add_llvm_component_library(LLVMipo
   GlobalOpt.cpp
   GlobalSplit.cpp
   HotColdSplitting.cpp
-  IPConstantPropagation.cpp
   IPO.cpp
   InferFunctionAttrs.cpp
   InlineSimple.cpp
diff --git a/llvm/lib/Transforms/IPO/IPConstantPropagation.cpp b/llvm/lib/Transforms/IPO/IPConstantPropagation.cpp
deleted file mode 100644
index 8d05a72d68dac..0000000000000
--- a/llvm/lib/Transforms/IPO/IPConstantPropagation.cpp
+++ /dev/null
@@ -1,308 +0,0 @@
-//===-- IPConstantPropagation.cpp - Propagate constants through calls -----===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This pass implements an _extremely_ simple interprocedural constant
-// propagation pass.  It could certainly be improved in many different ways,
-// like using a worklist.  This pass makes arguments dead, but does not remove
-// them.  The existing dead argument elimination pass should be run after this
-// to clean up the mess.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/Statistic.h"
-#include "llvm/Analysis/ValueTracking.h"
-#include "llvm/IR/AbstractCallSite.h"
-#include "llvm/IR/Constants.h"
-#include "llvm/IR/Instructions.h"
-#include "llvm/IR/Module.h"
-#include "llvm/InitializePasses.h"
-#include "llvm/Pass.h"
-#include "llvm/Transforms/IPO.h"
-using namespace llvm;
-
-#define DEBUG_TYPE "ipconstprop"
-
-STATISTIC(NumArgumentsProped, "Number of args turned into constants");
-STATISTIC(NumReturnValProped, "Number of return values turned into constants");
-
-namespace {
-  /// IPCP - The interprocedural constant propagation pass
-  ///
-  struct IPCP : public ModulePass {
-    static char ID; // Pass identification, replacement for typeid
-    IPCP() : ModulePass(ID) {
-      initializeIPCPPass(*PassRegistry::getPassRegistry());
-    }
-
-    bool runOnModule(Module &M) override;
-  };
-}
-
-/// PropagateConstantsIntoArguments - Look at all uses of the specified
-/// function.  If all uses are direct call sites, and all pass a particular
-/// constant in for an argument, propagate that constant in as the argument.
-///
-static bool PropagateConstantsIntoArguments(Function &F) {
-  if (F.arg_empty() || F.use_empty()) return false; // No arguments? Early exit.
-
-  // For each argument, keep track of its constant value and whether it is a
-  // constant or not.  The bool is driven to true when found to be non-constant.
-  SmallVector<PointerIntPair<Constant *, 1, bool>, 16> ArgumentConstants;
-  ArgumentConstants.resize(F.arg_size());
-
-  unsigned NumNonconstant = 0;
-  for (Use &U : F.uses()) {
-    User *UR = U.getUser();
-    // Ignore blockaddress uses.
-    if (isa<BlockAddress>(UR)) continue;
-
-    // If no abstract call site was created we did not understand the use, bail.
-    AbstractCallSite ACS(&U);
-    if (!ACS)
-      return false;
-
-    // Mismatched argument count is undefined behavior. Simply bail out to avoid
-    // handling of such situations below (avoiding asserts/crashes).
-    unsigned NumActualArgs = ACS.getNumArgOperands();
-    if (F.isVarArg() ? ArgumentConstants.size() > NumActualArgs
-                     : ArgumentConstants.size() != NumActualArgs)
-      return false;
-
-    // Check out all of the potentially constant arguments.  Note that we don't
-    // inspect varargs here.
-    Function::arg_iterator Arg = F.arg_begin();
-    for (unsigned i = 0, e = ArgumentConstants.size(); i != e; ++i, ++Arg) {
-
-      // If this argument is known non-constant, ignore it.
-      if (ArgumentConstants[i].getInt())
-        continue;
-
-      Value *V = ACS.getCallArgOperand(i);
-      Constant *C = dyn_cast_or_null<Constant>(V);
-
-      // Mismatched argument type is undefined behavior. Simply bail out to avoid
-      // handling of such situations below (avoiding asserts/crashes).
-      if (C && Arg->getType() != C->getType())
-        return false;
-
-      // We can only propagate thread independent values through callbacks.
-      // This is different to direct/indirect call sites because for them we
-      // know the thread executing the caller and callee is the same. For
-      // callbacks this is not guaranteed, thus a thread dependent value could
-      // be different for the caller and callee, making it invalid to propagate.
-      if (C && ACS.isCallbackCall() && C->isThreadDependent()) {
-        // Argument became non-constant. If all arguments are non-constant now,
-        // give up on this function.
-        if (++NumNonconstant == ArgumentConstants.size())
-          return false;
-
-        ArgumentConstants[i].setInt(true);
-        continue;
-      }
-
-      if (C && ArgumentConstants[i].getPointer() == nullptr) {
-        ArgumentConstants[i].setPointer(C); // First constant seen.
-      } else if (C && ArgumentConstants[i].getPointer() == C) {
-        // Still the constant value we think it is.
-      } else if (V == &*Arg) {
-        // Ignore recursive calls passing argument down.
-      } else {
-        // Argument became non-constant.  If all arguments are non-constant now,
-        // give up on this function.
-        if (++NumNonconstant == ArgumentConstants.size())
-          return false;
-        ArgumentConstants[i].setInt(true);
-      }
-    }
-  }
-
-  // If we got to this point, there is a constant argument!
-  assert(NumNonconstant != ArgumentConstants.size());
-  bool MadeChange = false;
-  Function::arg_iterator AI = F.arg_begin();
-  for (unsigned i = 0, e = ArgumentConstants.size(); i != e; ++i, ++AI) {
-    // Do we have a constant argument?
-    if (ArgumentConstants[i].getInt() || AI->use_empty() ||
-        (AI->hasByValAttr() && !F.onlyReadsMemory()))
-      continue;
-
-    Value *V = ArgumentConstants[i].getPointer();
-    if (!V) V = UndefValue::get(AI->getType());
-    AI->replaceAllUsesWith(V);
-    ++NumArgumentsProped;
-    MadeChange = true;
-  }
-  return MadeChange;
-}
-
-
-// Check to see if this function returns one or more constants. If so, replace
-// all callers that use those return values with the constant value. This will
-// leave in the actual return values and instructions, but deadargelim will
-// clean that up.
-//
-// Additionally if a function always returns one of its arguments directly,
-// callers will be updated to use the value they pass in directly instead of
-// using the return value.
-static bool PropagateConstantReturn(Function &F) {
-  if (F.getReturnType()->isVoidTy())
-    return false; // No return value.
-
-  // We can infer and propagate the return value only when we know that the
-  // definition we'll get at link time is *exactly* the definition we see now.
-  // For more details, see GlobalValue::mayBeDerefined.
-  if (!F.isDefinitionExact())
-    return false;
-
-  // Don't touch naked functions. The may contain asm returning
-  // value we don't see, so we may end up interprocedurally propagating
-  // the return value incorrectly.
-  if (F.hasFnAttribute(Attribute::Naked))
-    return false;
-
-  // Check to see if this function returns a constant.
-  SmallVector<Value *,4> RetVals;
-  StructType *STy = dyn_cast<StructType>(F.getReturnType());
-  if (STy)
-    for (unsigned i = 0, e = STy->getNumElements(); i < e; ++i)
-      RetVals.push_back(UndefValue::get(STy->getElementType(i)));
-  else
-    RetVals.push_back(UndefValue::get(F.getReturnType()));
-
-  unsigned NumNonConstant = 0;
-  for (BasicBlock &BB : F)
-    if (ReturnInst *RI = dyn_cast<ReturnInst>(BB.getTerminator())) {
-      for (unsigned i = 0, e = RetVals.size(); i != e; ++i) {
-        // Already found conflicting return values?
-        Value *RV = RetVals[i];
-        if (!RV)
-          continue;
-
-        // Find the returned value
-        Value *V;
-        if (!STy)
-          V = RI->getOperand(0);
-        else
-          V = FindInsertedValue(RI->getOperand(0), i);
-
-        if (V) {
-          // Ignore undefs, we can change them into anything
-          if (isa<UndefValue>(V))
-            continue;
-
-          // Try to see if all the rets return the same constant or argument.
-          if (isa<Constant>(V) || isa<Argument>(V)) {
-            if (isa<UndefValue>(RV)) {
-              // No value found yet? Try the current one.
-              RetVals[i] = V;
-              continue;
-            }
-            // Returning the same value? Good.
-            if (RV == V)
-              continue;
-          }
-        }
-        // Different or no known return value? Don't propagate this return
-        // value.
-        RetVals[i] = nullptr;
-        // All values non-constant? Stop looking.
-        if (++NumNonConstant == RetVals.size())
-          return false;
-      }
-    }
-
-  // If we got here, the function returns at least one constant value.  Loop
-  // over all users, replacing any uses of the return value with the returned
-  // constant.
-  bool MadeChange = false;
-  for (Use &U : F.uses()) {
-    CallBase *CB = dyn_cast<CallBase>(U.getUser());
-
-    // Not a call instruction or a call instruction that's not calling F
-    // directly?
-    if (!CB || !CB->isCallee(&U))
-      continue;
-
-    // Call result not used?
-    if (CB->use_empty())
-      continue;
-
-    MadeChange = true;
-
-    if (!STy) {
-      Value* New = RetVals[0];
-      if (Argument *A = dyn_cast<Argument>(New))
-        // Was an argument returned? Then find the corresponding argument in
-        // the call instruction and use that.
-        New = CB->getArgOperand(A->getArgNo());
-      CB->replaceAllUsesWith(New);
-      continue;
-    }
-
-    for (auto I = CB->user_begin(), E = CB->user_end(); I != E;) {
-      Instruction *Ins = cast<Instruction>(*I);
-
-      // Increment now, so we can remove the use
-      ++I;
-
-      // Find the index of the retval to replace with
-      int index = -1;
-      if (ExtractValueInst *EV = dyn_cast<ExtractValueInst>(Ins))
-        if (EV->getNumIndices() == 1)
-          index = *EV->idx_begin();
-
-      // If this use uses a specific return value, and we have a replacement,
-      // replace it.
-      if (index != -1) {
-        Value *New = RetVals[index];
-        if (New) {
-          if (Argument *A = dyn_cast<Argument>(New))
-            // Was an argument returned? Then find the corresponding argument in
-            // the call instruction and use that.
-            New = CB->getArgOperand(A->getArgNo());
-          Ins->replaceAllUsesWith(New);
-          Ins->eraseFromParent();
-        }
-      }
-    }
-  }
-
-  if (MadeChange) ++NumReturnValProped;
-  return MadeChange;
-}
-
-char IPCP::ID = 0;
-INITIALIZE_PASS(IPCP, "ipconstprop",
-                "Interprocedural constant propagation", false, false)
-
-ModulePass *llvm::createIPConstantPropagationPass() { return new IPCP(); }
-
-bool IPCP::runOnModule(Module &M) {
-  if (skipModule(M))
-    return false;
-
-  bool Changed = false;
-  bool LocalChange = true;
-
-  // FIXME: instead of using smart algorithms, we just iterate until we stop
-  // making changes.
-  while (LocalChange) {
-    LocalChange = false;
-    for (Function &F : M)
-      if (!F.isDeclaration()) {
-        // Delete any klingons.
-        F.removeDeadConstantUsers();
-        if (F.hasLocalLinkage())
-          LocalChange |= PropagateConstantsIntoArguments(F);
-        Changed |= PropagateConstantReturn(F);
-      }
-    Changed |= LocalChange;
-  }
-  return Changed;
-}
diff --git a/llvm/lib/Transforms/IPO/IPO.cpp b/llvm/lib/Transforms/IPO/IPO.cpp
index d37b9236380d4..45aca100086d4 100644
--- a/llvm/lib/Transforms/IPO/IPO.cpp
+++ b/llvm/lib/Transforms/IPO/IPO.cpp
@@ -35,7 +35,6 @@ void llvm::initializeIPO(PassRegistry &Registry) {
   initializeGlobalOptLegacyPassPass(Registry);
   initializeGlobalSplitPass(Registry);
   initializeHotColdSplittingLegacyPassPass(Registry);
-  initializeIPCPPass(Registry);
   initializeAlwaysInlinerLegacyPassPass(Registry);
   initializeSimpleInlinerPass(Registry);
   initializeInferFunctionAttrsLegacyPassPass(Registry);
@@ -104,10 +103,6 @@ void LLVMAddGlobalOptimizerPass(LLVMPassManagerRef PM) {
   unwrap(PM)->add(createGlobalOptimizerPass());
 }
 
-void LLVMAddIPConstantPropagationPass(LLVMPassManagerRef PM) {
-  unwrap(PM)->add(createIPConstantPropagationPass());
-}
-
 void LLVMAddPruneEHPass(LLVMPassManagerRef PM) {
   unwrap(PM)->add(createPruneEHPass());
 }
diff --git a/llvm/test/Transforms/IPConstantProp/comdat-ipo.ll b/llvm/test/Transforms/IPConstantProp/comdat-ipo.ll
deleted file mode 100644
index a19c89cb9bcf8..0000000000000
--- a/llvm/test/Transforms/IPConstantProp/comdat-ipo.ll
+++ /dev/null
@@ -1,34 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -ipconstprop -S | FileCheck %s
-
-; See PR26774
-
-define i32 @baz() {
-; CHECK-LABEL: @baz(
-; CHECK-NEXT:    ret i32 10
-;
-  ret i32 10
-}
-
-; We can const-prop @baz's return value *into* @foo, but cannot
-; constprop @foo's return value into bar.
-
-define linkonce_odr i32 @foo() {
-; CHECK-LABEL: @foo(
-; CHECK-NEXT:    [[VAL:%.*]] = call i32 @baz()
-; CHECK-NEXT:    ret i32 10
-;
-
-  %val = call i32 @baz()
-  ret i32 %val
-}
-
-define i32 @bar() {
-; CHECK-LABEL: @bar(
-; CHECK-NEXT:    [[VAL:%.*]] = call i32 @foo()
-; CHECK-NEXT:    ret i32 [[VAL]]
-;
-
-  %val = call i32 @foo()
-  ret i32 %val
-}
diff --git a/llvm/test/Transforms/IPConstantProp/2008-06-09-WeakProp.ll b/llvm/test/Transforms/SCCP/2008-06-09-WeakProp.ll
similarity index 91%
rename from llvm/test/Transforms/IPConstantProp/2008-06-09-WeakProp.ll
rename to llvm/test/Transforms/SCCP/2008-06-09-WeakProp.ll
index 270115cf5ddd1..b6c5299ae7dc4 100644
--- a/llvm/test/Transforms/IPConstantProp/2008-06-09-WeakProp.ll
+++ b/llvm/test/Transforms/SCCP/2008-06-09-WeakProp.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -ipconstprop -S | FileCheck %s
+; RUN: opt < %s -ipsccp -S | FileCheck %s
 ; Should not propagate the result of a weak function.
 ; PR2411
 
diff --git a/llvm/test/Transforms/IPConstantProp/PR43857.ll b/llvm/test/Transforms/SCCP/PR43857.ll
similarity index 86%
rename from llvm/test/Transforms/IPConstantProp/PR43857.ll
rename to llvm/test/Transforms/SCCP/PR43857.ll
index 0d0d14d398dea..efe4bca4f514c 100644
--- a/llvm/test/Transforms/IPConstantProp/PR43857.ll
+++ b/llvm/test/Transforms/SCCP/PR43857.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -S -ipconstprop | FileCheck %s
+; RUN: opt < %s -S -ipsccp | FileCheck %s
 
 %struct.wobble = type { i32 }
 %struct.zot = type { %struct.wobble, %struct.wobble, %struct.wobble }
@@ -19,7 +19,7 @@ define void @baz(<8 x i32> %arg) local_unnamed_addr {
 ; CHECK-LABEL: @baz(
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[TMP:%.*]] = call [[STRUCT_ZOT:%.*]] @widget(<8 x i32> [[ARG:%.*]])
-; CHECK-NEXT:    [[TMP1:%.*]] = extractvalue [[STRUCT_ZOT]] %tmp, 0, 0
+; CHECK-NEXT:    [[TMP1:%.*]] = extractvalue [[STRUCT_ZOT]] undef, 0, 0
 ; CHECK-NEXT:    ret void
 ;
 bb:
diff --git a/llvm/test/Transforms/IPConstantProp/arg-count-mismatch.ll b/llvm/test/Transforms/SCCP/arg-count-mismatch.ll
similarity index 96%
rename from llvm/test/Transforms/IPConstantProp/arg-count-mismatch.ll
rename to llvm/test/Transforms/SCCP/arg-count-mismatch.ll
index 7afe858e52d11..ba5f1a6d83f72 100644
--- a/llvm/test/Transforms/IPConstantProp/arg-count-mismatch.ll
+++ b/llvm/test/Transforms/SCCP/arg-count-mismatch.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -ipconstprop -S -o - | FileCheck %s
+; RUN: opt < %s -ipsccp -S -o - | FileCheck %s
 
 ; The original C source looked like this:
 ;
@@ -53,7 +53,7 @@ define internal i16 @bar(i16 %p1, i16 %p2) {
 
 define internal i16 @vararg_prop(i16 %p1, ...) {
 ; CHECK-LABEL: @vararg_prop(
-; CHECK-NEXT:    ret i16 7
+; CHECK-NEXT:    ret i16 undef
 ;
   ret i16 %p1
 }
diff --git a/llvm/test/Transforms/IPConstantProp/arg-type-mismatch.ll b/llvm/test/Transforms/SCCP/arg-type-mismatch.ll
similarity index 92%
rename from llvm/test/Transforms/IPConstantProp/arg-type-mismatch.ll
rename to llvm/test/Transforms/SCCP/arg-type-mismatch.ll
index ff924d73390b0..9a9da52174c2a 100644
--- a/llvm/test/Transforms/IPConstantProp/arg-type-mismatch.ll
+++ b/llvm/test/Transforms/SCCP/arg-type-mismatch.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -ipconstprop -S -o - | FileCheck %s
+; RUN: opt < %s -ipsccp -S -o - | FileCheck %s
 
 ; This test is just to verify that we do not crash/assert due to mismatch in
 ; argument type between the caller and callee.
diff --git a/llvm/test/Transforms/SCCP/comdat-ipo.ll b/llvm/test/Transforms/SCCP/comdat-ipo.ll
index 618075fd5e3fe..fc715f45406f3 100644
--- a/llvm/test/Transforms/SCCP/comdat-ipo.ll
+++ b/llvm/test/Transforms/SCCP/comdat-ipo.ll
@@ -1,8 +1,12 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -ipsccp -S | FileCheck %s
 
 ; See PR26774
 
 define i32 @baz() {
+; CHECK-LABEL: @baz(
+; CHECK-NEXT:    ret i32 10
+;
   ret i32 10
 }
 
@@ -11,8 +15,9 @@ define i32 @baz() {
 
 define linkonce_odr i32 @foo() {
 ; CHECK-LABEL: @foo(
-; CHECK-NEXT:  %val = call i32 @baz()
-; CHECK-NEXT:  ret i32 10
+; CHECK-NEXT:    [[VAL:%.*]] = call i32 @baz()
+; CHECK-NEXT:    ret i32 10
+;
 
   %val = call i32 @baz()
   ret i32 %val
@@ -20,8 +25,9 @@ define linkonce_odr i32 @foo() {
 
 define i32 @bar() {
 ; CHECK-LABEL: @bar(
-; CHECK-NEXT:  %val = call i32 @foo()
-; CHECK-NEXT:  ret i32 %val
+; CHECK-NEXT:    [[VAL:%.*]] = call i32 @foo()
+; CHECK-NEXT:    ret i32 [[VAL]]
+;
 
   %val = call i32 @foo()
   ret i32 %val
diff --git a/llvm/test/Transforms/IPConstantProp/deadarg.ll b/llvm/test/Transforms/SCCP/deadarg.ll
similarity index 77%
rename from llvm/test/Transforms/IPConstantProp/deadarg.ll
rename to llvm/test/Transforms/SCCP/deadarg.ll
index 25b9749b50790..1117acc7d0137 100644
--- a/llvm/test/Transforms/IPConstantProp/deadarg.ll
+++ b/llvm/test/Transforms/SCCP/deadarg.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -ipconstprop -disable-output
+; RUN: opt < %s -ipsccp -disable-output
 define internal void @foo(i32 %X) {
   call void @foo( i32 %X )
   ret void
diff --git a/llvm/test/Transforms/IPConstantProp/multiple_callbacks.ll b/llvm/test/Transforms/SCCP/multiple_callbacks.ll
similarity index 96%
rename from llvm/test/Transforms/IPConstantProp/multiple_callbacks.ll
rename to llvm/test/Transforms/SCCP/multiple_callbacks.ll
index 6684044e24ce0..3d196f86e4a1c 100644
--- a/llvm/test/Transforms/IPConstantProp/multiple_callbacks.ll
+++ b/llvm/test/Transforms/SCCP/multiple_callbacks.ll
@@ -1,5 +1,6 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -ipconstprop -S < %s | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-attributes
+; RUN: opt -ipsccp -S %s | FileCheck %s
+;
 ;
 ;                            /---------------------------------------|
 ;                            |                /----------------------|----|
@@ -38,7 +39,7 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 define internal i32 @cb0(i32 %zero) {
 ; CHECK-LABEL: @cb0(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    ret i32 0
+; CHECK-NEXT:    ret i32 [[ZERO:%.*]]
 ;
 entry:
   ret i32 %zero
diff --git a/llvm/test/Transforms/IPConstantProp/naked-return.ll b/llvm/test/Transforms/SCCP/naked-return.ll
similarity index 97%
rename from llvm/test/Transforms/IPConstantProp/naked-return.ll
rename to llvm/test/Transforms/SCCP/naked-return.ll
index 133662a211b41..daeb176b09972 100644
--- a/llvm/test/Transforms/IPConstantProp/naked-return.ll
+++ b/llvm/test/Transforms/SCCP/naked-return.ll
@@ -1,6 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -ipsccp -S %s | FileCheck %s
-; RUN: opt -ipconstprop -S %s | FileCheck %s
 
 target datalayout = "e-m:x-p:32:32-i64:64-f80:32-n8:16:32-a:0:32-S32"
 target triple = "i686-pc-windows-msvc19.0.24215"
diff --git a/llvm/test/Transforms/IPConstantProp/openmp_parallel_for.ll b/llvm/test/Transforms/SCCP/openmp_parallel_for.ll
similarity index 98%
rename from llvm/test/Transforms/IPConstantProp/openmp_parallel_for.ll
rename to llvm/test/Transforms/SCCP/openmp_parallel_for.ll
index 338cc8886e29d..27831c6e66196 100644
--- a/llvm/test/Transforms/IPConstantProp/openmp_parallel_for.ll
+++ b/llvm/test/Transforms/SCCP/openmp_parallel_for.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S -ipconstprop < %s | FileCheck %s
+; RUN: opt -S -ipsccp < %s | FileCheck %s
 ;
 ;    void bar(int, float, double);
 ;
@@ -53,7 +53,7 @@ define internal void @.omp_outlined.(i32* noalias %.global_tid., i32* noalias %.
 ; CHECK-NEXT:    [[DOTOMP_UB:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    store i64 4617315517961601024, i64* [[Q_ADDR]], align 8
+; CHECK-NEXT:    store i64 [[Q:%.*]], i64* [[Q_ADDR]], align 8
 ; CHECK-NEXT:    [[CONV:%.*]] = bitcast i64* [[Q_ADDR]] to double*
 ; CHECK-NEXT:    [[TMP:%.*]] = load i32, i32* [[N:%.*]], align 4
 ; CHECK-NEXT:    [[SUB3:%.*]] = add nsw i32 [[TMP]], -3
diff --git a/llvm/test/Transforms/IPConstantProp/pthreads.ll b/llvm/test/Transforms/SCCP/pthreads.ll
similarity index 93%
rename from llvm/test/Transforms/IPConstantProp/pthreads.ll
rename to llvm/test/Transforms/SCCP/pthreads.ll
index dcad3858da13a..a9d2b942c1a5a 100644
--- a/llvm/test/Transforms/IPConstantProp/pthreads.ll
+++ b/llvm/test/Transforms/SCCP/pthreads.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -ipconstprop -S < %s | FileCheck %s
+; RUN: opt -ipsccp -S < %s | FileCheck %s
 ;
 ;    #include <pthread.h>
 ;
@@ -44,7 +44,7 @@ declare !callback !0 dso_local i32 @pthread_create(i64*, %union.pthread_attr_t*,
 define internal i8* @foo(i8* %arg) {
 ; CHECK-LABEL: @foo(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    ret i8* null
+; CHECK-NEXT:    ret i8* [[ARG:%.*]]
 ;
 entry:
   ret i8* %arg
@@ -53,7 +53,7 @@ entry:
 define internal i8* @bar(i8* %arg) {
 ; CHECK-LABEL: @bar(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    ret i8* bitcast (i8** @GlobalVPtr to i8*)
+; CHECK-NEXT:    ret i8* [[ARG:%.*]]
 ;
 entry:
   ret i8* %arg
diff --git a/llvm/test/Transforms/IPConstantProp/recursion.ll b/llvm/test/Transforms/SCCP/recursion.ll
similarity index 90%
rename from llvm/test/Transforms/IPConstantProp/recursion.ll
rename to llvm/test/Transforms/SCCP/recursion.ll
index ac8ff9ca00e32..e4f5f1d240e7c 100644
--- a/llvm/test/Transforms/IPConstantProp/recursion.ll
+++ b/llvm/test/Transforms/SCCP/recursion.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -ipconstprop -deadargelim -S | FileCheck %s
+; RUN: opt < %s -ipsccp -deadargelim -S | FileCheck %s
 
 ; CHECK-NOT: %X
 
diff --git a/llvm/test/Transforms/IPConstantProp/return-argument.ll b/llvm/test/Transforms/SCCP/return-argument.ll
similarity index 91%
rename from llvm/test/Transforms/IPConstantProp/return-argument.ll
rename to llvm/test/Transforms/SCCP/return-argument.ll
index 6a3eac0c120d0..764b4898c9616 100644
--- a/llvm/test/Transforms/IPConstantProp/return-argument.ll
+++ b/llvm/test/Transforms/SCCP/return-argument.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -ipconstprop -S | FileCheck %s
+; RUN: opt < %s -ipsccp -S | FileCheck %s
 
 ;; This function returns its second argument on all return statements
 define internal i32* @incdec(i1 %C, i32* %V) {
@@ -49,11 +49,13 @@ define void @caller(i1 %C) personality i32 (...)* @__gxx_personality_v0 {
 ; CHECK-NEXT:    [[Q:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[W:%.*]] = call i32* @incdec(i1 [[C:%.*]], i32* [[Q]])
 ; CHECK-NEXT:    [[S1:%.*]] = call { i32, i32 } @foo(i32 1, i32 2)
+; CHECK-NEXT:    [[X1:%.*]] = extractvalue { i32, i32 } [[S1]], 0
 ; CHECK-NEXT:    [[S2:%.*]] = invoke { i32, i32 } @foo(i32 3, i32 4)
 ; CHECK-NEXT:    to label [[OK:%.*]] unwind label [[LPAD:%.*]]
 ; CHECK:       OK:
-; CHECK-NEXT:    [[Z:%.*]] = add i32 1, 3
-; CHECK-NEXT:    store i32 [[Z]], i32* [[Q]], align 4
+; CHECK-NEXT:    [[X2:%.*]] = extractvalue { i32, i32 } [[S2]], 0
+; CHECK-NEXT:    [[Z:%.*]] = add i32 [[X1]], [[X2]]
+; CHECK-NEXT:    store i32 [[Z]], i32* [[W]], align 4
 ; CHECK-NEXT:    br label [[RET:%.*]]
 ; CHECK:       LPAD:
 ; CHECK-NEXT:    [[EXN:%.*]] = landingpad { i8*, i32 }
diff --git a/llvm/test/Transforms/IPConstantProp/return-constant.ll b/llvm/test/Transforms/SCCP/return-constant.ll
similarity index 91%
rename from llvm/test/Transforms/IPConstantProp/return-constant.ll
rename to llvm/test/Transforms/SCCP/return-constant.ll
index d75aa9b969317..5cf53eea12b77 100644
--- a/llvm/test/Transforms/IPConstantProp/return-constant.ll
+++ b/llvm/test/Transforms/SCCP/return-constant.ll
@@ -1,13 +1,13 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -ipconstprop -instcombine -S | FileCheck %s
+; RUN: opt < %s -ipsccp -instcombine -S | FileCheck %s
 
 define internal i32 @foo(i1 %C) {
 ; CHECK-LABEL: @foo(
 ; CHECK-NEXT:    br i1 [[C:%.*]], label [[T:%.*]], label [[F:%.*]]
 ; CHECK:       T:
-; CHECK-NEXT:    ret i32 52
+; CHECK-NEXT:    ret i32 undef
 ; CHECK:       F:
-; CHECK-NEXT:    ret i32 52
+; CHECK-NEXT:    ret i32 undef
 ;
   br i1 %C, label %T, label %F
 
diff --git a/llvm/test/Transforms/IPConstantProp/return-constants.ll b/llvm/test/Transforms/SCCP/return-constants.ll
similarity index 70%
rename from llvm/test/Transforms/IPConstantProp/return-constants.ll
rename to llvm/test/Transforms/SCCP/return-constants.ll
index 4611067316141..cbf178d2efcaf 100644
--- a/llvm/test/Transforms/IPConstantProp/return-constants.ll
+++ b/llvm/test/Transforms/SCCP/return-constants.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -ipconstprop -S | FileCheck %s
+; RUN: opt < %s -ipsccp -S | FileCheck %s
 
 %0 = type { i32, i32 }
 
@@ -7,13 +7,9 @@ define internal %0 @foo(i1 %Q) {
 ; CHECK-LABEL: @foo(
 ; CHECK-NEXT:    br i1 [[Q:%.*]], label [[T:%.*]], label [[F:%.*]]
 ; CHECK:       T:
-; CHECK-NEXT:    [[MRV:%.*]] = insertvalue [[TMP0:%.*]] undef, i32 21, 0
-; CHECK-NEXT:    [[MRV1:%.*]] = insertvalue [[TMP0]] %mrv, i32 22, 1
-; CHECK-NEXT:    ret [[TMP0]] %mrv1
+; CHECK-NEXT:    ret [[TMP0:%.*]] { i32 21, i32 22 }
 ; CHECK:       F:
-; CHECK-NEXT:    [[MRV2:%.*]] = insertvalue [[TMP0]] undef, i32 21, 0
-; CHECK-NEXT:    [[MRV3:%.*]] = insertvalue [[TMP0]] %mrv2, i32 23, 1
-; CHECK-NEXT:    ret [[TMP0]] %mrv3
+; CHECK-NEXT:    ret [[TMP0]] { i32 21, i32 23 }
 ;
   br i1 %Q, label %T, label %F
 
@@ -30,14 +26,11 @@ F:                                                ; preds = %0
 
 define internal %0 @bar(i1 %Q) {
 ; CHECK-LABEL: @bar(
-; CHECK-NEXT:    [[A:%.*]] = insertvalue [[TMP0:%.*]] undef, i32 21, 0
 ; CHECK-NEXT:    br i1 [[Q:%.*]], label [[T:%.*]], label [[F:%.*]]
 ; CHECK:       T:
-; CHECK-NEXT:    [[B:%.*]] = insertvalue [[TMP0]] %A, i32 22, 1
-; CHECK-NEXT:    ret [[TMP0]] %B
+; CHECK-NEXT:    ret [[TMP0:%.*]] { i32 21, i32 22 }
 ; CHECK:       F:
-; CHECK-NEXT:    [[C:%.*]] = insertvalue [[TMP0]] %A, i32 23, 1
-; CHECK-NEXT:    ret [[TMP0]] %C
+; CHECK-NEXT:    ret [[TMP0]] { i32 21, i32 23 }
 ;
   %A = insertvalue %0 undef, i32 21, 0
   br i1 %Q, label %T, label %F
@@ -57,7 +50,6 @@ define %0 @caller(i1 %Q) {
 ; CHECK-NEXT:    [[B:%.*]] = extractvalue [[TMP0]] %X, 1
 ; CHECK-NEXT:    [[Y:%.*]] = call [[TMP0]] @bar(i1 [[Q]])
 ; CHECK-NEXT:    [[D:%.*]] = extractvalue [[TMP0]] %Y, 1
-; CHECK-NEXT:    [[M:%.*]] = add i32 21, 21
 ; CHECK-NEXT:    [[N:%.*]] = add i32 [[B]], [[D]]
 ; CHECK-NEXT:    ret [[TMP0]] %X
 ;
diff --git a/llvm/test/Transforms/IPConstantProp/thread_local_acs.ll b/llvm/test/Transforms/SCCP/thread_local_acs.ll
similarity index 92%
rename from llvm/test/Transforms/IPConstantProp/thread_local_acs.ll
rename to llvm/test/Transforms/SCCP/thread_local_acs.ll
index 3f843d26077ad..69c0cfeec163e 100644
--- a/llvm/test/Transforms/IPConstantProp/thread_local_acs.ll
+++ b/llvm/test/Transforms/SCCP/thread_local_acs.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -ipconstprop -S < %s | FileCheck %s
+; RUN: opt -ipsccp -S < %s | FileCheck %s
 ;
 ;    #include <threads.h>
 ;    thread_local int gtl = 0;
@@ -24,7 +24,7 @@ define internal i32 @callee(i32* %thread_local_ptr, i32* %shared_ptr) {
 ; CHECK-LABEL: @callee(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP:%.*]] = load i32, i32* [[THREAD_LOCAL_PTR:%.*]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* @gsh, align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[SHARED_PTR:%.*]], align 4
 ; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP]], [[TMP1]]
 ; CHECK-NEXT:    ret i32 [[ADD]]
 ;
diff --git a/llvm/utils/findoptdiff b/llvm/utils/findoptdiff
index 9a8803184384e..925e2dce4bf3f 100755
--- a/llvm/utils/findoptdiff
+++ b/llvm/utils/findoptdiff
@@ -70,7 +70,7 @@ dis2="$llvm2/Debug/bin/llvm-dis"
 opt1="$llvm1/Debug/bin/opt"
 opt2="$llvm2/Debug/bin/opt"
 
-all_switches="-verify -lowersetjmp -simplifycfg -mem2reg -globalopt -globaldce -ipconstprop -deadargelim -instcombine -simplifycfg -prune-eh -inline -simplify-libcalls -argpromotion -tailduplicate -simplifycfg -sroa -instcombine -predsimplify -condprop -tailcallelim -simplifycfg -reassociate -licm -loop-unswitch -instcombine -indvars -loop-unroll -instcombine -load-vn -gcse -sccp -instcombine -condprop -dse -dce -simplifycfg -deadtypeelim -constmerge -internalize -ipsccp -globalopt -constmerge -deadargelim -inline -prune-eh -globalopt -globaldce -argpromotion -instcombine -predsimplify -sroa -globalsmodref-aa -licm -load-vn -gcse -dse -instcombine -simplifycfg -verify"
+all_switches="-verify -lowersetjmp -simplifycfg -mem2reg -globalopt -globaldce -deadargelim -instcombine -simplifycfg -prune-eh -inline -simplify-libcalls -argpromotion -tailduplicate -simplifycfg -sroa -instcombine -predsimplify -condprop -tailcallelim -simplifycfg -reassociate -licm -loop-unswitch -instcombine -indvars -loop-unroll -instcombine -load-vn -gcse -sccp -instcombine -condprop -dse -dce -simplifycfg -deadtypeelim -constmerge -internalize -ipsccp -globalopt -constmerge -deadargelim -inline -prune-eh -globalopt -globaldce -argpromotion -instcombine -predsimplify -sroa -globalsmodref-aa -licm -load-vn -gcse -dse -instcombine -simplifycfg -verify"
 
 #counter=0
 function tryit {
diff --git a/llvm/utils/gn/secondary/llvm/lib/Transforms/IPO/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Transforms/IPO/BUILD.gn
index 7d48256b6cfa2..168bb499dcffd 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Transforms/IPO/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Transforms/IPO/BUILD.gn
@@ -38,7 +38,6 @@ static_library("IPO") {
     "GlobalOpt.cpp",
     "GlobalSplit.cpp",
     "HotColdSplitting.cpp",
-    "IPConstantPropagation.cpp",
     "IPO.cpp",
     "InferFunctionAttrs.cpp",
     "InlineSimple.cpp",

From a97940d4e0ed0507eb9beba7f22e34d62449a58f Mon Sep 17 00:00:00 2001
From: Frederik Gossen <frgossen@google.com>
Date: Thu, 30 Jul 2020 11:40:16 +0000
Subject: [PATCH 0606/1035] [MLIR][Shape] Limit `shape.rank` lowering to its
 extent tensor variant

When lowering to the standard dialect, we currently support only the extent
tensor variant of the shape.rank operation. This change lets the conversion
pattern fail in a well-defined manner.

Differential Revision: https://reviews.llvm.org/D84852
---
 .../lib/Conversion/ShapeToStandard/ShapeToStandard.cpp |  4 ++++
 .../Conversion/ShapeToStandard/shape-to-standard.mlir  | 10 ++++++++++
 2 files changed, 14 insertions(+)

diff --git a/mlir/lib/Conversion/ShapeToStandard/ShapeToStandard.cpp b/mlir/lib/Conversion/ShapeToStandard/ShapeToStandard.cpp
index 41d4d90b33d3b..debdedc89e87d 100644
--- a/mlir/lib/Conversion/ShapeToStandard/ShapeToStandard.cpp
+++ b/mlir/lib/Conversion/ShapeToStandard/ShapeToStandard.cpp
@@ -231,6 +231,10 @@ class RankOpConverter : public OpConversionPattern<shape::RankOp> {
 LogicalResult
 RankOpConverter::matchAndRewrite(shape::RankOp op, ArrayRef<Value> operands,
                                  ConversionPatternRewriter &rewriter) const {
+  // For now, this lowering supports only error-free types.
+  if (op.getType().isa<SizeType>())
+    return failure();
+
   shape::RankOp::Adaptor transformed(operands);
   rewriter.replaceOpWithNewOp<DimOp>(op, transformed.shape(), 0);
   return success();
diff --git a/mlir/test/Conversion/ShapeToStandard/shape-to-standard.mlir b/mlir/test/Conversion/ShapeToStandard/shape-to-standard.mlir
index 3f19de9c52f00..b0fb5bac9071b 100644
--- a/mlir/test/Conversion/ShapeToStandard/shape-to-standard.mlir
+++ b/mlir/test/Conversion/ShapeToStandard/shape-to-standard.mlir
@@ -90,6 +90,16 @@ func @get_extent(%shape : tensor<?xindex>, %idx : !shape.size) -> !shape.size {
 
 // -----
 
+// Don't lower `rank` if type is not error-free.
+// CHECK-LABEL: @rank
+func @rank(%shape : !shape.shape) {
+  // CHECK: shape.rank
+  %rank = shape.rank %shape : !shape.shape -> !shape.size
+  return
+}
+
+// -----
+
 // Express `get_extent` as `std.dim` when it relies directly on the outcome of a
 // `shape_of` operation.
 // CHECK-LABEL: @get_extent_shape_of

From 59d6e814ce0e7b40b7cc3ab136b9af2ffab9c6f8 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Thu, 30 Jul 2020 13:06:15 +0100
Subject: [PATCH 0607/1035] Revert "[IPConstProp] Remove and move tests to
 SCCP."

This reverts commit e77624a3be942c7abba48942b3a8da3462070a3f.

Looks like some clang tests manually invoke -ipconstprop via opt.....
---
 llvm/bindings/go/llvm/transforms_ipo.go       |   1 +
 .../bindings/ocaml/transforms/ipo/ipo_ocaml.c |   6 +
 .../ocaml/transforms/ipo/llvm_ipo.mli         |   5 +
 llvm/docs/Passes.rst                          |   9 +
 llvm/include/llvm-c/Transforms/IPO.h          |   3 +
 llvm/include/llvm/InitializePasses.h          |   1 +
 llvm/include/llvm/LinkAllPasses.h             |   1 +
 llvm/include/llvm/Transforms/IPO.h            |   6 +
 llvm/lib/Transforms/IPO/CMakeLists.txt        |   1 +
 .../Transforms/IPO/IPConstantPropagation.cpp  | 308 ++++++++++++++++++
 llvm/lib/Transforms/IPO/IPO.cpp               |   5 +
 .../2008-06-09-WeakProp.ll                    |   2 +-
 .../{SCCP => IPConstantProp}/PR43857.ll       |   4 +-
 .../arg-count-mismatch.ll                     |   4 +-
 .../arg-type-mismatch.ll                      |   2 +-
 .../Transforms/IPConstantProp/comdat-ipo.ll   |  34 ++
 .../{SCCP => IPConstantProp}/deadarg.ll       |   2 +-
 .../multiple_callbacks.ll                     |   7 +-
 .../{SCCP => IPConstantProp}/naked-return.ll  |   1 +
 .../openmp_parallel_for.ll                    |   4 +-
 .../{SCCP => IPConstantProp}/pthreads.ll      |   6 +-
 .../{SCCP => IPConstantProp}/recursion.ll     |   2 +-
 .../return-argument.ll                        |   8 +-
 .../return-constant.ll                        |   6 +-
 .../return-constants.ll                       |  18 +-
 .../thread_local_acs.ll                       |   4 +-
 llvm/test/Transforms/SCCP/comdat-ipo.ll       |  14 +-
 llvm/utils/findoptdiff                        |   2 +-
 .../llvm/lib/Transforms/IPO/BUILD.gn          |   1 +
 29 files changed, 424 insertions(+), 43 deletions(-)
 create mode 100644 llvm/lib/Transforms/IPO/IPConstantPropagation.cpp
 rename llvm/test/Transforms/{SCCP => IPConstantProp}/2008-06-09-WeakProp.ll (91%)
 rename llvm/test/Transforms/{SCCP => IPConstantProp}/PR43857.ll (86%)
 rename llvm/test/Transforms/{SCCP => IPConstantProp}/arg-count-mismatch.ll (96%)
 rename llvm/test/Transforms/{SCCP => IPConstantProp}/arg-type-mismatch.ll (92%)
 create mode 100644 llvm/test/Transforms/IPConstantProp/comdat-ipo.ll
 rename llvm/test/Transforms/{SCCP => IPConstantProp}/deadarg.ll (77%)
 rename llvm/test/Transforms/{SCCP => IPConstantProp}/multiple_callbacks.ll (96%)
 rename llvm/test/Transforms/{SCCP => IPConstantProp}/naked-return.ll (97%)
 rename llvm/test/Transforms/{SCCP => IPConstantProp}/openmp_parallel_for.ll (98%)
 rename llvm/test/Transforms/{SCCP => IPConstantProp}/pthreads.ll (93%)
 rename llvm/test/Transforms/{SCCP => IPConstantProp}/recursion.ll (90%)
 rename llvm/test/Transforms/{SCCP => IPConstantProp}/return-argument.ll (91%)
 rename llvm/test/Transforms/{SCCP => IPConstantProp}/return-constant.ll (91%)
 rename llvm/test/Transforms/{SCCP => IPConstantProp}/return-constants.ll (70%)
 rename llvm/test/Transforms/{SCCP => IPConstantProp}/thread_local_acs.ll (92%)

diff --git a/llvm/bindings/go/llvm/transforms_ipo.go b/llvm/bindings/go/llvm/transforms_ipo.go
index 8a158f208bef4..1dcb2af8bf243 100644
--- a/llvm/bindings/go/llvm/transforms_ipo.go
+++ b/llvm/bindings/go/llvm/transforms_ipo.go
@@ -32,6 +32,7 @@ func (pm PassManager) AddFunctionAttrsPass()         { C.LLVMAddFunctionAttrsPas
 func (pm PassManager) AddFunctionInliningPass()      { C.LLVMAddFunctionInliningPass(pm.C) }
 func (pm PassManager) AddGlobalDCEPass()             { C.LLVMAddGlobalDCEPass(pm.C) }
 func (pm PassManager) AddGlobalOptimizerPass()       { C.LLVMAddGlobalOptimizerPass(pm.C) }
+func (pm PassManager) AddIPConstantPropagationPass() { C.LLVMAddIPConstantPropagationPass(pm.C) }
 func (pm PassManager) AddPruneEHPass()               { C.LLVMAddPruneEHPass(pm.C) }
 func (pm PassManager) AddIPSCCPPass()                { C.LLVMAddIPSCCPPass(pm.C) }
 func (pm PassManager) AddInternalizePass(allButMain bool) {
diff --git a/llvm/bindings/ocaml/transforms/ipo/ipo_ocaml.c b/llvm/bindings/ocaml/transforms/ipo/ipo_ocaml.c
index c0e213714ed02..9fcaa10534f6b 100644
--- a/llvm/bindings/ocaml/transforms/ipo/ipo_ocaml.c
+++ b/llvm/bindings/ocaml/transforms/ipo/ipo_ocaml.c
@@ -73,6 +73,12 @@ CAMLprim value llvm_add_global_optimizer(LLVMPassManagerRef PM) {
   return Val_unit;
 }
 
+/* [`Module] Llvm.PassManager.t -> unit */
+CAMLprim value llvm_add_ip_constant_propagation(LLVMPassManagerRef PM) {
+  LLVMAddIPConstantPropagationPass(PM);
+  return Val_unit;
+}
+
 /* [`Module] Llvm.PassManager.t -> unit */
 CAMLprim value llvm_add_prune_eh(LLVMPassManagerRef PM) {
   LLVMAddPruneEHPass(PM);
diff --git a/llvm/bindings/ocaml/transforms/ipo/llvm_ipo.mli b/llvm/bindings/ocaml/transforms/ipo/llvm_ipo.mli
index a581924c6d549..6507c5d92c2b1 100644
--- a/llvm/bindings/ocaml/transforms/ipo/llvm_ipo.mli
+++ b/llvm/bindings/ocaml/transforms/ipo/llvm_ipo.mli
@@ -56,6 +56,11 @@ external add_global_optimizer
   : [ `Module ] Llvm.PassManager.t -> unit
   = "llvm_add_global_optimizer"
 
+(** See the [llvm::createIPConstantPropagationPass] function. *)
+external add_ipc_propagation
+  : [ `Module ] Llvm.PassManager.t -> unit
+  = "llvm_add_ip_constant_propagation"
+
 (** See the [llvm::createPruneEHPass] function. *)
 external add_prune_eh
   : [ `Module ] Llvm.PassManager.t -> unit
diff --git a/llvm/docs/Passes.rst b/llvm/docs/Passes.rst
index 13317ecc966e7..e45adad98c157 100644
--- a/llvm/docs/Passes.rst
+++ b/llvm/docs/Passes.rst
@@ -676,6 +676,15 @@ This pass loops over all of the functions in the input module, looking for a
 main function.  If a main function is found, all other functions and all global
 variables with initializers are marked as internal.
 
+``-ipconstprop``: Interprocedural constant propagation
+------------------------------------------------------
+
+This pass implements an *extremely* simple interprocedural constant propagation
+pass.  It could certainly be improved in many different ways, like using a
+worklist.  This pass makes arguments dead, but does not remove them.  The
+existing dead argument elimination pass should be run after this to clean up
+the mess.
+
 ``-ipsccp``: Interprocedural Sparse Conditional Constant Propagation
 --------------------------------------------------------------------
 
diff --git a/llvm/include/llvm-c/Transforms/IPO.h b/llvm/include/llvm-c/Transforms/IPO.h
index 3f2cadf32366b..cde3d24609208 100644
--- a/llvm/include/llvm-c/Transforms/IPO.h
+++ b/llvm/include/llvm-c/Transforms/IPO.h
@@ -57,6 +57,9 @@ void LLVMAddGlobalDCEPass(LLVMPassManagerRef PM);
 /** See llvm::createGlobalOptimizerPass function. */
 void LLVMAddGlobalOptimizerPass(LLVMPassManagerRef PM);
 
+/** See llvm::createIPConstantPropagationPass function. */
+void LLVMAddIPConstantPropagationPass(LLVMPassManagerRef PM);
+
 /** See llvm::createPruneEHPass function. */
 void LLVMAddPruneEHPass(LLVMPassManagerRef PM);
 
diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h
index d23ecfb98ce92..cce6a43504021 100644
--- a/llvm/include/llvm/InitializePasses.h
+++ b/llvm/include/llvm/InitializePasses.h
@@ -179,6 +179,7 @@ void initializeGuardWideningLegacyPassPass(PassRegistry&);
 void initializeHardwareLoopsPass(PassRegistry&);
 void initializeHotColdSplittingLegacyPassPass(PassRegistry&);
 void initializeHWAddressSanitizerLegacyPassPass(PassRegistry &);
+void initializeIPCPPass(PassRegistry&);
 void initializeIPSCCPLegacyPassPass(PassRegistry&);
 void initializeIRCELegacyPassPass(PassRegistry&);
 void initializeIRTranslatorPass(PassRegistry&);
diff --git a/llvm/include/llvm/LinkAllPasses.h b/llvm/include/llvm/LinkAllPasses.h
index 9d7ac2b3f3b99..90e2e24294d40 100644
--- a/llvm/include/llvm/LinkAllPasses.h
+++ b/llvm/include/llvm/LinkAllPasses.h
@@ -116,6 +116,7 @@ namespace {
       (void) llvm::createGlobalsAAWrapperPass();
       (void) llvm::createGuardWideningPass();
       (void) llvm::createLoopGuardWideningPass();
+      (void) llvm::createIPConstantPropagationPass();
       (void) llvm::createIPSCCPPass();
       (void) llvm::createInductiveRangeCheckEliminationPass();
       (void) llvm::createIndVarSimplifyPass();
diff --git a/llvm/include/llvm/Transforms/IPO.h b/llvm/include/llvm/Transforms/IPO.h
index 7b73eeaf8e45b..28e454d3b0fc7 100644
--- a/llvm/include/llvm/Transforms/IPO.h
+++ b/llvm/include/llvm/Transforms/IPO.h
@@ -155,6 +155,12 @@ Pass *createArgumentPromotionPass(unsigned maxElements = 3);
 /// createOpenMPOptLegacyPass - OpenMP specific optimizations.
 Pass *createOpenMPOptLegacyPass();
 
+//===----------------------------------------------------------------------===//
+/// createIPConstantPropagationPass - This pass propagates constants from call
+/// sites into the bodies of functions.
+///
+ModulePass *createIPConstantPropagationPass();
+
 //===----------------------------------------------------------------------===//
 /// createIPSCCPPass - This pass propagates constants from call sites into the
 /// bodies of functions, and keeps track of whether basic blocks are executable
diff --git a/llvm/lib/Transforms/IPO/CMakeLists.txt b/llvm/lib/Transforms/IPO/CMakeLists.txt
index a17ef63371063..3f30c0289e8c3 100644
--- a/llvm/lib/Transforms/IPO/CMakeLists.txt
+++ b/llvm/lib/Transforms/IPO/CMakeLists.txt
@@ -18,6 +18,7 @@ add_llvm_component_library(LLVMipo
   GlobalOpt.cpp
   GlobalSplit.cpp
   HotColdSplitting.cpp
+  IPConstantPropagation.cpp
   IPO.cpp
   InferFunctionAttrs.cpp
   InlineSimple.cpp
diff --git a/llvm/lib/Transforms/IPO/IPConstantPropagation.cpp b/llvm/lib/Transforms/IPO/IPConstantPropagation.cpp
new file mode 100644
index 0000000000000..8d05a72d68dac
--- /dev/null
+++ b/llvm/lib/Transforms/IPO/IPConstantPropagation.cpp
@@ -0,0 +1,308 @@
+//===-- IPConstantPropagation.cpp - Propagate constants through calls -----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This pass implements an _extremely_ simple interprocedural constant
+// propagation pass.  It could certainly be improved in many different ways,
+// like using a worklist.  This pass makes arguments dead, but does not remove
+// them.  The existing dead argument elimination pass should be run after this
+// to clean up the mess.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/Analysis/ValueTracking.h"
+#include "llvm/IR/AbstractCallSite.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/Module.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/Pass.h"
+#include "llvm/Transforms/IPO.h"
+using namespace llvm;
+
+#define DEBUG_TYPE "ipconstprop"
+
+STATISTIC(NumArgumentsProped, "Number of args turned into constants");
+STATISTIC(NumReturnValProped, "Number of return values turned into constants");
+
+namespace {
+  /// IPCP - The interprocedural constant propagation pass
+  ///
+  struct IPCP : public ModulePass {
+    static char ID; // Pass identification, replacement for typeid
+    IPCP() : ModulePass(ID) {
+      initializeIPCPPass(*PassRegistry::getPassRegistry());
+    }
+
+    bool runOnModule(Module &M) override;
+  };
+}
+
+/// PropagateConstantsIntoArguments - Look at all uses of the specified
+/// function.  If all uses are direct call sites, and all pass a particular
+/// constant in for an argument, propagate that constant in as the argument.
+///
+static bool PropagateConstantsIntoArguments(Function &F) {
+  if (F.arg_empty() || F.use_empty()) return false; // No arguments? Early exit.
+
+  // For each argument, keep track of its constant value and whether it is a
+  // constant or not.  The bool is driven to true when found to be non-constant.
+  SmallVector<PointerIntPair<Constant *, 1, bool>, 16> ArgumentConstants;
+  ArgumentConstants.resize(F.arg_size());
+
+  unsigned NumNonconstant = 0;
+  for (Use &U : F.uses()) {
+    User *UR = U.getUser();
+    // Ignore blockaddress uses.
+    if (isa<BlockAddress>(UR)) continue;
+
+    // If no abstract call site was created we did not understand the use, bail.
+    AbstractCallSite ACS(&U);
+    if (!ACS)
+      return false;
+
+    // Mismatched argument count is undefined behavior. Simply bail out to avoid
+    // handling of such situations below (avoiding asserts/crashes).
+    unsigned NumActualArgs = ACS.getNumArgOperands();
+    if (F.isVarArg() ? ArgumentConstants.size() > NumActualArgs
+                     : ArgumentConstants.size() != NumActualArgs)
+      return false;
+
+    // Check out all of the potentially constant arguments.  Note that we don't
+    // inspect varargs here.
+    Function::arg_iterator Arg = F.arg_begin();
+    for (unsigned i = 0, e = ArgumentConstants.size(); i != e; ++i, ++Arg) {
+
+      // If this argument is known non-constant, ignore it.
+      if (ArgumentConstants[i].getInt())
+        continue;
+
+      Value *V = ACS.getCallArgOperand(i);
+      Constant *C = dyn_cast_or_null<Constant>(V);
+
+      // Mismatched argument type is undefined behavior. Simply bail out to avoid
+      // handling of such situations below (avoiding asserts/crashes).
+      if (C && Arg->getType() != C->getType())
+        return false;
+
+      // We can only propagate thread independent values through callbacks.
+      // This is different to direct/indirect call sites because for them we
+      // know the thread executing the caller and callee is the same. For
+      // callbacks this is not guaranteed, thus a thread dependent value could
+      // be different for the caller and callee, making it invalid to propagate.
+      if (C && ACS.isCallbackCall() && C->isThreadDependent()) {
+        // Argument became non-constant. If all arguments are non-constant now,
+        // give up on this function.
+        if (++NumNonconstant == ArgumentConstants.size())
+          return false;
+
+        ArgumentConstants[i].setInt(true);
+        continue;
+      }
+
+      if (C && ArgumentConstants[i].getPointer() == nullptr) {
+        ArgumentConstants[i].setPointer(C); // First constant seen.
+      } else if (C && ArgumentConstants[i].getPointer() == C) {
+        // Still the constant value we think it is.
+      } else if (V == &*Arg) {
+        // Ignore recursive calls passing argument down.
+      } else {
+        // Argument became non-constant.  If all arguments are non-constant now,
+        // give up on this function.
+        if (++NumNonconstant == ArgumentConstants.size())
+          return false;
+        ArgumentConstants[i].setInt(true);
+      }
+    }
+  }
+
+  // If we got to this point, there is a constant argument!
+  assert(NumNonconstant != ArgumentConstants.size());
+  bool MadeChange = false;
+  Function::arg_iterator AI = F.arg_begin();
+  for (unsigned i = 0, e = ArgumentConstants.size(); i != e; ++i, ++AI) {
+    // Do we have a constant argument?
+    if (ArgumentConstants[i].getInt() || AI->use_empty() ||
+        (AI->hasByValAttr() && !F.onlyReadsMemory()))
+      continue;
+
+    Value *V = ArgumentConstants[i].getPointer();
+    if (!V) V = UndefValue::get(AI->getType());
+    AI->replaceAllUsesWith(V);
+    ++NumArgumentsProped;
+    MadeChange = true;
+  }
+  return MadeChange;
+}
+
+
+// Check to see if this function returns one or more constants. If so, replace
+// all callers that use those return values with the constant value. This will
+// leave in the actual return values and instructions, but deadargelim will
+// clean that up.
+//
+// Additionally if a function always returns one of its arguments directly,
+// callers will be updated to use the value they pass in directly instead of
+// using the return value.
+static bool PropagateConstantReturn(Function &F) {
+  if (F.getReturnType()->isVoidTy())
+    return false; // No return value.
+
+  // We can infer and propagate the return value only when we know that the
+  // definition we'll get at link time is *exactly* the definition we see now.
+  // For more details, see GlobalValue::mayBeDerefined.
+  if (!F.isDefinitionExact())
+    return false;
+
+  // Don't touch naked functions. The may contain asm returning
+  // value we don't see, so we may end up interprocedurally propagating
+  // the return value incorrectly.
+  if (F.hasFnAttribute(Attribute::Naked))
+    return false;
+
+  // Check to see if this function returns a constant.
+  SmallVector<Value *,4> RetVals;
+  StructType *STy = dyn_cast<StructType>(F.getReturnType());
+  if (STy)
+    for (unsigned i = 0, e = STy->getNumElements(); i < e; ++i)
+      RetVals.push_back(UndefValue::get(STy->getElementType(i)));
+  else
+    RetVals.push_back(UndefValue::get(F.getReturnType()));
+
+  unsigned NumNonConstant = 0;
+  for (BasicBlock &BB : F)
+    if (ReturnInst *RI = dyn_cast<ReturnInst>(BB.getTerminator())) {
+      for (unsigned i = 0, e = RetVals.size(); i != e; ++i) {
+        // Already found conflicting return values?
+        Value *RV = RetVals[i];
+        if (!RV)
+          continue;
+
+        // Find the returned value
+        Value *V;
+        if (!STy)
+          V = RI->getOperand(0);
+        else
+          V = FindInsertedValue(RI->getOperand(0), i);
+
+        if (V) {
+          // Ignore undefs, we can change them into anything
+          if (isa<UndefValue>(V))
+            continue;
+
+          // Try to see if all the rets return the same constant or argument.
+          if (isa<Constant>(V) || isa<Argument>(V)) {
+            if (isa<UndefValue>(RV)) {
+              // No value found yet? Try the current one.
+              RetVals[i] = V;
+              continue;
+            }
+            // Returning the same value? Good.
+            if (RV == V)
+              continue;
+          }
+        }
+        // Different or no known return value? Don't propagate this return
+        // value.
+        RetVals[i] = nullptr;
+        // All values non-constant? Stop looking.
+        if (++NumNonConstant == RetVals.size())
+          return false;
+      }
+    }
+
+  // If we got here, the function returns at least one constant value.  Loop
+  // over all users, replacing any uses of the return value with the returned
+  // constant.
+  bool MadeChange = false;
+  for (Use &U : F.uses()) {
+    CallBase *CB = dyn_cast<CallBase>(U.getUser());
+
+    // Not a call instruction or a call instruction that's not calling F
+    // directly?
+    if (!CB || !CB->isCallee(&U))
+      continue;
+
+    // Call result not used?
+    if (CB->use_empty())
+      continue;
+
+    MadeChange = true;
+
+    if (!STy) {
+      Value* New = RetVals[0];
+      if (Argument *A = dyn_cast<Argument>(New))
+        // Was an argument returned? Then find the corresponding argument in
+        // the call instruction and use that.
+        New = CB->getArgOperand(A->getArgNo());
+      CB->replaceAllUsesWith(New);
+      continue;
+    }
+
+    for (auto I = CB->user_begin(), E = CB->user_end(); I != E;) {
+      Instruction *Ins = cast<Instruction>(*I);
+
+      // Increment now, so we can remove the use
+      ++I;
+
+      // Find the index of the retval to replace with
+      int index = -1;
+      if (ExtractValueInst *EV = dyn_cast<ExtractValueInst>(Ins))
+        if (EV->getNumIndices() == 1)
+          index = *EV->idx_begin();
+
+      // If this use uses a specific return value, and we have a replacement,
+      // replace it.
+      if (index != -1) {
+        Value *New = RetVals[index];
+        if (New) {
+          if (Argument *A = dyn_cast<Argument>(New))
+            // Was an argument returned? Then find the corresponding argument in
+            // the call instruction and use that.
+            New = CB->getArgOperand(A->getArgNo());
+          Ins->replaceAllUsesWith(New);
+          Ins->eraseFromParent();
+        }
+      }
+    }
+  }
+
+  if (MadeChange) ++NumReturnValProped;
+  return MadeChange;
+}
+
+char IPCP::ID = 0;
+INITIALIZE_PASS(IPCP, "ipconstprop",
+                "Interprocedural constant propagation", false, false)
+
+ModulePass *llvm::createIPConstantPropagationPass() { return new IPCP(); }
+
+bool IPCP::runOnModule(Module &M) {
+  if (skipModule(M))
+    return false;
+
+  bool Changed = false;
+  bool LocalChange = true;
+
+  // FIXME: instead of using smart algorithms, we just iterate until we stop
+  // making changes.
+  while (LocalChange) {
+    LocalChange = false;
+    for (Function &F : M)
+      if (!F.isDeclaration()) {
+        // Delete any klingons.
+        F.removeDeadConstantUsers();
+        if (F.hasLocalLinkage())
+          LocalChange |= PropagateConstantsIntoArguments(F);
+        Changed |= PropagateConstantReturn(F);
+      }
+    Changed |= LocalChange;
+  }
+  return Changed;
+}
diff --git a/llvm/lib/Transforms/IPO/IPO.cpp b/llvm/lib/Transforms/IPO/IPO.cpp
index 45aca100086d4..d37b9236380d4 100644
--- a/llvm/lib/Transforms/IPO/IPO.cpp
+++ b/llvm/lib/Transforms/IPO/IPO.cpp
@@ -35,6 +35,7 @@ void llvm::initializeIPO(PassRegistry &Registry) {
   initializeGlobalOptLegacyPassPass(Registry);
   initializeGlobalSplitPass(Registry);
   initializeHotColdSplittingLegacyPassPass(Registry);
+  initializeIPCPPass(Registry);
   initializeAlwaysInlinerLegacyPassPass(Registry);
   initializeSimpleInlinerPass(Registry);
   initializeInferFunctionAttrsLegacyPassPass(Registry);
@@ -103,6 +104,10 @@ void LLVMAddGlobalOptimizerPass(LLVMPassManagerRef PM) {
   unwrap(PM)->add(createGlobalOptimizerPass());
 }
 
+void LLVMAddIPConstantPropagationPass(LLVMPassManagerRef PM) {
+  unwrap(PM)->add(createIPConstantPropagationPass());
+}
+
 void LLVMAddPruneEHPass(LLVMPassManagerRef PM) {
   unwrap(PM)->add(createPruneEHPass());
 }
diff --git a/llvm/test/Transforms/SCCP/2008-06-09-WeakProp.ll b/llvm/test/Transforms/IPConstantProp/2008-06-09-WeakProp.ll
similarity index 91%
rename from llvm/test/Transforms/SCCP/2008-06-09-WeakProp.ll
rename to llvm/test/Transforms/IPConstantProp/2008-06-09-WeakProp.ll
index b6c5299ae7dc4..270115cf5ddd1 100644
--- a/llvm/test/Transforms/SCCP/2008-06-09-WeakProp.ll
+++ b/llvm/test/Transforms/IPConstantProp/2008-06-09-WeakProp.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -ipsccp -S | FileCheck %s
+; RUN: opt < %s -ipconstprop -S | FileCheck %s
 ; Should not propagate the result of a weak function.
 ; PR2411
 
diff --git a/llvm/test/Transforms/SCCP/PR43857.ll b/llvm/test/Transforms/IPConstantProp/PR43857.ll
similarity index 86%
rename from llvm/test/Transforms/SCCP/PR43857.ll
rename to llvm/test/Transforms/IPConstantProp/PR43857.ll
index efe4bca4f514c..0d0d14d398dea 100644
--- a/llvm/test/Transforms/SCCP/PR43857.ll
+++ b/llvm/test/Transforms/IPConstantProp/PR43857.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -S -ipsccp | FileCheck %s
+; RUN: opt < %s -S -ipconstprop | FileCheck %s
 
 %struct.wobble = type { i32 }
 %struct.zot = type { %struct.wobble, %struct.wobble, %struct.wobble }
@@ -19,7 +19,7 @@ define void @baz(<8 x i32> %arg) local_unnamed_addr {
 ; CHECK-LABEL: @baz(
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[TMP:%.*]] = call [[STRUCT_ZOT:%.*]] @widget(<8 x i32> [[ARG:%.*]])
-; CHECK-NEXT:    [[TMP1:%.*]] = extractvalue [[STRUCT_ZOT]] undef, 0, 0
+; CHECK-NEXT:    [[TMP1:%.*]] = extractvalue [[STRUCT_ZOT]] %tmp, 0, 0
 ; CHECK-NEXT:    ret void
 ;
 bb:
diff --git a/llvm/test/Transforms/SCCP/arg-count-mismatch.ll b/llvm/test/Transforms/IPConstantProp/arg-count-mismatch.ll
similarity index 96%
rename from llvm/test/Transforms/SCCP/arg-count-mismatch.ll
rename to llvm/test/Transforms/IPConstantProp/arg-count-mismatch.ll
index ba5f1a6d83f72..7afe858e52d11 100644
--- a/llvm/test/Transforms/SCCP/arg-count-mismatch.ll
+++ b/llvm/test/Transforms/IPConstantProp/arg-count-mismatch.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -ipsccp -S -o - | FileCheck %s
+; RUN: opt < %s -ipconstprop -S -o - | FileCheck %s
 
 ; The original C source looked like this:
 ;
@@ -53,7 +53,7 @@ define internal i16 @bar(i16 %p1, i16 %p2) {
 
 define internal i16 @vararg_prop(i16 %p1, ...) {
 ; CHECK-LABEL: @vararg_prop(
-; CHECK-NEXT:    ret i16 undef
+; CHECK-NEXT:    ret i16 7
 ;
   ret i16 %p1
 }
diff --git a/llvm/test/Transforms/SCCP/arg-type-mismatch.ll b/llvm/test/Transforms/IPConstantProp/arg-type-mismatch.ll
similarity index 92%
rename from llvm/test/Transforms/SCCP/arg-type-mismatch.ll
rename to llvm/test/Transforms/IPConstantProp/arg-type-mismatch.ll
index 9a9da52174c2a..ff924d73390b0 100644
--- a/llvm/test/Transforms/SCCP/arg-type-mismatch.ll
+++ b/llvm/test/Transforms/IPConstantProp/arg-type-mismatch.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -ipsccp -S -o - | FileCheck %s
+; RUN: opt < %s -ipconstprop -S -o - | FileCheck %s
 
 ; This test is just to verify that we do not crash/assert due to mismatch in
 ; argument type between the caller and callee.
diff --git a/llvm/test/Transforms/IPConstantProp/comdat-ipo.ll b/llvm/test/Transforms/IPConstantProp/comdat-ipo.ll
new file mode 100644
index 0000000000000..a19c89cb9bcf8
--- /dev/null
+++ b/llvm/test/Transforms/IPConstantProp/comdat-ipo.ll
@@ -0,0 +1,34 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -ipconstprop -S | FileCheck %s
+
+; See PR26774
+
+define i32 @baz() {
+; CHECK-LABEL: @baz(
+; CHECK-NEXT:    ret i32 10
+;
+  ret i32 10
+}
+
+; We can const-prop @baz's return value *into* @foo, but cannot
+; constprop @foo's return value into bar.
+
+define linkonce_odr i32 @foo() {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:    [[VAL:%.*]] = call i32 @baz()
+; CHECK-NEXT:    ret i32 10
+;
+
+  %val = call i32 @baz()
+  ret i32 %val
+}
+
+define i32 @bar() {
+; CHECK-LABEL: @bar(
+; CHECK-NEXT:    [[VAL:%.*]] = call i32 @foo()
+; CHECK-NEXT:    ret i32 [[VAL]]
+;
+
+  %val = call i32 @foo()
+  ret i32 %val
+}
diff --git a/llvm/test/Transforms/SCCP/deadarg.ll b/llvm/test/Transforms/IPConstantProp/deadarg.ll
similarity index 77%
rename from llvm/test/Transforms/SCCP/deadarg.ll
rename to llvm/test/Transforms/IPConstantProp/deadarg.ll
index 1117acc7d0137..25b9749b50790 100644
--- a/llvm/test/Transforms/SCCP/deadarg.ll
+++ b/llvm/test/Transforms/IPConstantProp/deadarg.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -ipsccp -disable-output
+; RUN: opt < %s -ipconstprop -disable-output
 define internal void @foo(i32 %X) {
   call void @foo( i32 %X )
   ret void
diff --git a/llvm/test/Transforms/SCCP/multiple_callbacks.ll b/llvm/test/Transforms/IPConstantProp/multiple_callbacks.ll
similarity index 96%
rename from llvm/test/Transforms/SCCP/multiple_callbacks.ll
rename to llvm/test/Transforms/IPConstantProp/multiple_callbacks.ll
index 3d196f86e4a1c..6684044e24ce0 100644
--- a/llvm/test/Transforms/SCCP/multiple_callbacks.ll
+++ b/llvm/test/Transforms/IPConstantProp/multiple_callbacks.ll
@@ -1,6 +1,5 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-attributes
-; RUN: opt -ipsccp -S %s | FileCheck %s
-;
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -ipconstprop -S < %s | FileCheck %s
 ;
 ;                            /---------------------------------------|
 ;                            |                /----------------------|----|
@@ -39,7 +38,7 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 define internal i32 @cb0(i32 %zero) {
 ; CHECK-LABEL: @cb0(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    ret i32 [[ZERO:%.*]]
+; CHECK-NEXT:    ret i32 0
 ;
 entry:
   ret i32 %zero
diff --git a/llvm/test/Transforms/SCCP/naked-return.ll b/llvm/test/Transforms/IPConstantProp/naked-return.ll
similarity index 97%
rename from llvm/test/Transforms/SCCP/naked-return.ll
rename to llvm/test/Transforms/IPConstantProp/naked-return.ll
index daeb176b09972..133662a211b41 100644
--- a/llvm/test/Transforms/SCCP/naked-return.ll
+++ b/llvm/test/Transforms/IPConstantProp/naked-return.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -ipsccp -S %s | FileCheck %s
+; RUN: opt -ipconstprop -S %s | FileCheck %s
 
 target datalayout = "e-m:x-p:32:32-i64:64-f80:32-n8:16:32-a:0:32-S32"
 target triple = "i686-pc-windows-msvc19.0.24215"
diff --git a/llvm/test/Transforms/SCCP/openmp_parallel_for.ll b/llvm/test/Transforms/IPConstantProp/openmp_parallel_for.ll
similarity index 98%
rename from llvm/test/Transforms/SCCP/openmp_parallel_for.ll
rename to llvm/test/Transforms/IPConstantProp/openmp_parallel_for.ll
index 27831c6e66196..338cc8886e29d 100644
--- a/llvm/test/Transforms/SCCP/openmp_parallel_for.ll
+++ b/llvm/test/Transforms/IPConstantProp/openmp_parallel_for.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S -ipsccp < %s | FileCheck %s
+; RUN: opt -S -ipconstprop < %s | FileCheck %s
 ;
 ;    void bar(int, float, double);
 ;
@@ -53,7 +53,7 @@ define internal void @.omp_outlined.(i32* noalias %.global_tid., i32* noalias %.
 ; CHECK-NEXT:    [[DOTOMP_UB:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    store i64 [[Q:%.*]], i64* [[Q_ADDR]], align 8
+; CHECK-NEXT:    store i64 4617315517961601024, i64* [[Q_ADDR]], align 8
 ; CHECK-NEXT:    [[CONV:%.*]] = bitcast i64* [[Q_ADDR]] to double*
 ; CHECK-NEXT:    [[TMP:%.*]] = load i32, i32* [[N:%.*]], align 4
 ; CHECK-NEXT:    [[SUB3:%.*]] = add nsw i32 [[TMP]], -3
diff --git a/llvm/test/Transforms/SCCP/pthreads.ll b/llvm/test/Transforms/IPConstantProp/pthreads.ll
similarity index 93%
rename from llvm/test/Transforms/SCCP/pthreads.ll
rename to llvm/test/Transforms/IPConstantProp/pthreads.ll
index a9d2b942c1a5a..dcad3858da13a 100644
--- a/llvm/test/Transforms/SCCP/pthreads.ll
+++ b/llvm/test/Transforms/IPConstantProp/pthreads.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -ipsccp -S < %s | FileCheck %s
+; RUN: opt -ipconstprop -S < %s | FileCheck %s
 ;
 ;    #include <pthread.h>
 ;
@@ -44,7 +44,7 @@ declare !callback !0 dso_local i32 @pthread_create(i64*, %union.pthread_attr_t*,
 define internal i8* @foo(i8* %arg) {
 ; CHECK-LABEL: @foo(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    ret i8* [[ARG:%.*]]
+; CHECK-NEXT:    ret i8* null
 ;
 entry:
   ret i8* %arg
@@ -53,7 +53,7 @@ entry:
 define internal i8* @bar(i8* %arg) {
 ; CHECK-LABEL: @bar(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    ret i8* [[ARG:%.*]]
+; CHECK-NEXT:    ret i8* bitcast (i8** @GlobalVPtr to i8*)
 ;
 entry:
   ret i8* %arg
diff --git a/llvm/test/Transforms/SCCP/recursion.ll b/llvm/test/Transforms/IPConstantProp/recursion.ll
similarity index 90%
rename from llvm/test/Transforms/SCCP/recursion.ll
rename to llvm/test/Transforms/IPConstantProp/recursion.ll
index e4f5f1d240e7c..ac8ff9ca00e32 100644
--- a/llvm/test/Transforms/SCCP/recursion.ll
+++ b/llvm/test/Transforms/IPConstantProp/recursion.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -ipsccp -deadargelim -S | FileCheck %s
+; RUN: opt < %s -ipconstprop -deadargelim -S | FileCheck %s
 
 ; CHECK-NOT: %X
 
diff --git a/llvm/test/Transforms/SCCP/return-argument.ll b/llvm/test/Transforms/IPConstantProp/return-argument.ll
similarity index 91%
rename from llvm/test/Transforms/SCCP/return-argument.ll
rename to llvm/test/Transforms/IPConstantProp/return-argument.ll
index 764b4898c9616..6a3eac0c120d0 100644
--- a/llvm/test/Transforms/SCCP/return-argument.ll
+++ b/llvm/test/Transforms/IPConstantProp/return-argument.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -ipsccp -S | FileCheck %s
+; RUN: opt < %s -ipconstprop -S | FileCheck %s
 
 ;; This function returns its second argument on all return statements
 define internal i32* @incdec(i1 %C, i32* %V) {
@@ -49,13 +49,11 @@ define void @caller(i1 %C) personality i32 (...)* @__gxx_personality_v0 {
 ; CHECK-NEXT:    [[Q:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[W:%.*]] = call i32* @incdec(i1 [[C:%.*]], i32* [[Q]])
 ; CHECK-NEXT:    [[S1:%.*]] = call { i32, i32 } @foo(i32 1, i32 2)
-; CHECK-NEXT:    [[X1:%.*]] = extractvalue { i32, i32 } [[S1]], 0
 ; CHECK-NEXT:    [[S2:%.*]] = invoke { i32, i32 } @foo(i32 3, i32 4)
 ; CHECK-NEXT:    to label [[OK:%.*]] unwind label [[LPAD:%.*]]
 ; CHECK:       OK:
-; CHECK-NEXT:    [[X2:%.*]] = extractvalue { i32, i32 } [[S2]], 0
-; CHECK-NEXT:    [[Z:%.*]] = add i32 [[X1]], [[X2]]
-; CHECK-NEXT:    store i32 [[Z]], i32* [[W]], align 4
+; CHECK-NEXT:    [[Z:%.*]] = add i32 1, 3
+; CHECK-NEXT:    store i32 [[Z]], i32* [[Q]], align 4
 ; CHECK-NEXT:    br label [[RET:%.*]]
 ; CHECK:       LPAD:
 ; CHECK-NEXT:    [[EXN:%.*]] = landingpad { i8*, i32 }
diff --git a/llvm/test/Transforms/SCCP/return-constant.ll b/llvm/test/Transforms/IPConstantProp/return-constant.ll
similarity index 91%
rename from llvm/test/Transforms/SCCP/return-constant.ll
rename to llvm/test/Transforms/IPConstantProp/return-constant.ll
index 5cf53eea12b77..d75aa9b969317 100644
--- a/llvm/test/Transforms/SCCP/return-constant.ll
+++ b/llvm/test/Transforms/IPConstantProp/return-constant.ll
@@ -1,13 +1,13 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -ipsccp -instcombine -S | FileCheck %s
+; RUN: opt < %s -ipconstprop -instcombine -S | FileCheck %s
 
 define internal i32 @foo(i1 %C) {
 ; CHECK-LABEL: @foo(
 ; CHECK-NEXT:    br i1 [[C:%.*]], label [[T:%.*]], label [[F:%.*]]
 ; CHECK:       T:
-; CHECK-NEXT:    ret i32 undef
+; CHECK-NEXT:    ret i32 52
 ; CHECK:       F:
-; CHECK-NEXT:    ret i32 undef
+; CHECK-NEXT:    ret i32 52
 ;
   br i1 %C, label %T, label %F
 
diff --git a/llvm/test/Transforms/SCCP/return-constants.ll b/llvm/test/Transforms/IPConstantProp/return-constants.ll
similarity index 70%
rename from llvm/test/Transforms/SCCP/return-constants.ll
rename to llvm/test/Transforms/IPConstantProp/return-constants.ll
index cbf178d2efcaf..4611067316141 100644
--- a/llvm/test/Transforms/SCCP/return-constants.ll
+++ b/llvm/test/Transforms/IPConstantProp/return-constants.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -ipsccp -S | FileCheck %s
+; RUN: opt < %s -ipconstprop -S | FileCheck %s
 
 %0 = type { i32, i32 }
 
@@ -7,9 +7,13 @@ define internal %0 @foo(i1 %Q) {
 ; CHECK-LABEL: @foo(
 ; CHECK-NEXT:    br i1 [[Q:%.*]], label [[T:%.*]], label [[F:%.*]]
 ; CHECK:       T:
-; CHECK-NEXT:    ret [[TMP0:%.*]] { i32 21, i32 22 }
+; CHECK-NEXT:    [[MRV:%.*]] = insertvalue [[TMP0:%.*]] undef, i32 21, 0
+; CHECK-NEXT:    [[MRV1:%.*]] = insertvalue [[TMP0]] %mrv, i32 22, 1
+; CHECK-NEXT:    ret [[TMP0]] %mrv1
 ; CHECK:       F:
-; CHECK-NEXT:    ret [[TMP0]] { i32 21, i32 23 }
+; CHECK-NEXT:    [[MRV2:%.*]] = insertvalue [[TMP0]] undef, i32 21, 0
+; CHECK-NEXT:    [[MRV3:%.*]] = insertvalue [[TMP0]] %mrv2, i32 23, 1
+; CHECK-NEXT:    ret [[TMP0]] %mrv3
 ;
   br i1 %Q, label %T, label %F
 
@@ -26,11 +30,14 @@ F:                                                ; preds = %0
 
 define internal %0 @bar(i1 %Q) {
 ; CHECK-LABEL: @bar(
+; CHECK-NEXT:    [[A:%.*]] = insertvalue [[TMP0:%.*]] undef, i32 21, 0
 ; CHECK-NEXT:    br i1 [[Q:%.*]], label [[T:%.*]], label [[F:%.*]]
 ; CHECK:       T:
-; CHECK-NEXT:    ret [[TMP0:%.*]] { i32 21, i32 22 }
+; CHECK-NEXT:    [[B:%.*]] = insertvalue [[TMP0]] %A, i32 22, 1
+; CHECK-NEXT:    ret [[TMP0]] %B
 ; CHECK:       F:
-; CHECK-NEXT:    ret [[TMP0]] { i32 21, i32 23 }
+; CHECK-NEXT:    [[C:%.*]] = insertvalue [[TMP0]] %A, i32 23, 1
+; CHECK-NEXT:    ret [[TMP0]] %C
 ;
   %A = insertvalue %0 undef, i32 21, 0
   br i1 %Q, label %T, label %F
@@ -50,6 +57,7 @@ define %0 @caller(i1 %Q) {
 ; CHECK-NEXT:    [[B:%.*]] = extractvalue [[TMP0]] %X, 1
 ; CHECK-NEXT:    [[Y:%.*]] = call [[TMP0]] @bar(i1 [[Q]])
 ; CHECK-NEXT:    [[D:%.*]] = extractvalue [[TMP0]] %Y, 1
+; CHECK-NEXT:    [[M:%.*]] = add i32 21, 21
 ; CHECK-NEXT:    [[N:%.*]] = add i32 [[B]], [[D]]
 ; CHECK-NEXT:    ret [[TMP0]] %X
 ;
diff --git a/llvm/test/Transforms/SCCP/thread_local_acs.ll b/llvm/test/Transforms/IPConstantProp/thread_local_acs.ll
similarity index 92%
rename from llvm/test/Transforms/SCCP/thread_local_acs.ll
rename to llvm/test/Transforms/IPConstantProp/thread_local_acs.ll
index 69c0cfeec163e..3f843d26077ad 100644
--- a/llvm/test/Transforms/SCCP/thread_local_acs.ll
+++ b/llvm/test/Transforms/IPConstantProp/thread_local_acs.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -ipsccp -S < %s | FileCheck %s
+; RUN: opt -ipconstprop -S < %s | FileCheck %s
 ;
 ;    #include <threads.h>
 ;    thread_local int gtl = 0;
@@ -24,7 +24,7 @@ define internal i32 @callee(i32* %thread_local_ptr, i32* %shared_ptr) {
 ; CHECK-LABEL: @callee(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP:%.*]] = load i32, i32* [[THREAD_LOCAL_PTR:%.*]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[SHARED_PTR:%.*]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* @gsh, align 4
 ; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP]], [[TMP1]]
 ; CHECK-NEXT:    ret i32 [[ADD]]
 ;
diff --git a/llvm/test/Transforms/SCCP/comdat-ipo.ll b/llvm/test/Transforms/SCCP/comdat-ipo.ll
index fc715f45406f3..618075fd5e3fe 100644
--- a/llvm/test/Transforms/SCCP/comdat-ipo.ll
+++ b/llvm/test/Transforms/SCCP/comdat-ipo.ll
@@ -1,12 +1,8 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -ipsccp -S | FileCheck %s
 
 ; See PR26774
 
 define i32 @baz() {
-; CHECK-LABEL: @baz(
-; CHECK-NEXT:    ret i32 10
-;
   ret i32 10
 }
 
@@ -15,9 +11,8 @@ define i32 @baz() {
 
 define linkonce_odr i32 @foo() {
 ; CHECK-LABEL: @foo(
-; CHECK-NEXT:    [[VAL:%.*]] = call i32 @baz()
-; CHECK-NEXT:    ret i32 10
-;
+; CHECK-NEXT:  %val = call i32 @baz()
+; CHECK-NEXT:  ret i32 10
 
   %val = call i32 @baz()
   ret i32 %val
@@ -25,9 +20,8 @@ define linkonce_odr i32 @foo() {
 
 define i32 @bar() {
 ; CHECK-LABEL: @bar(
-; CHECK-NEXT:    [[VAL:%.*]] = call i32 @foo()
-; CHECK-NEXT:    ret i32 [[VAL]]
-;
+; CHECK-NEXT:  %val = call i32 @foo()
+; CHECK-NEXT:  ret i32 %val
 
   %val = call i32 @foo()
   ret i32 %val
diff --git a/llvm/utils/findoptdiff b/llvm/utils/findoptdiff
index 925e2dce4bf3f..9a8803184384e 100755
--- a/llvm/utils/findoptdiff
+++ b/llvm/utils/findoptdiff
@@ -70,7 +70,7 @@ dis2="$llvm2/Debug/bin/llvm-dis"
 opt1="$llvm1/Debug/bin/opt"
 opt2="$llvm2/Debug/bin/opt"
 
-all_switches="-verify -lowersetjmp -simplifycfg -mem2reg -globalopt -globaldce -deadargelim -instcombine -simplifycfg -prune-eh -inline -simplify-libcalls -argpromotion -tailduplicate -simplifycfg -sroa -instcombine -predsimplify -condprop -tailcallelim -simplifycfg -reassociate -licm -loop-unswitch -instcombine -indvars -loop-unroll -instcombine -load-vn -gcse -sccp -instcombine -condprop -dse -dce -simplifycfg -deadtypeelim -constmerge -internalize -ipsccp -globalopt -constmerge -deadargelim -inline -prune-eh -globalopt -globaldce -argpromotion -instcombine -predsimplify -sroa -globalsmodref-aa -licm -load-vn -gcse -dse -instcombine -simplifycfg -verify"
+all_switches="-verify -lowersetjmp -simplifycfg -mem2reg -globalopt -globaldce -ipconstprop -deadargelim -instcombine -simplifycfg -prune-eh -inline -simplify-libcalls -argpromotion -tailduplicate -simplifycfg -sroa -instcombine -predsimplify -condprop -tailcallelim -simplifycfg -reassociate -licm -loop-unswitch -instcombine -indvars -loop-unroll -instcombine -load-vn -gcse -sccp -instcombine -condprop -dse -dce -simplifycfg -deadtypeelim -constmerge -internalize -ipsccp -globalopt -constmerge -deadargelim -inline -prune-eh -globalopt -globaldce -argpromotion -instcombine -predsimplify -sroa -globalsmodref-aa -licm -load-vn -gcse -dse -instcombine -simplifycfg -verify"
 
 #counter=0
 function tryit {
diff --git a/llvm/utils/gn/secondary/llvm/lib/Transforms/IPO/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Transforms/IPO/BUILD.gn
index 168bb499dcffd..7d48256b6cfa2 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Transforms/IPO/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Transforms/IPO/BUILD.gn
@@ -38,6 +38,7 @@ static_library("IPO") {
     "GlobalOpt.cpp",
     "GlobalSplit.cpp",
     "HotColdSplitting.cpp",
+    "IPConstantPropagation.cpp",
     "IPO.cpp",
     "InferFunctionAttrs.cpp",
     "InlineSimple.cpp",

From 67e41df4e03e934b2990cd7f0b7f5d4e0a58e14f Mon Sep 17 00:00:00 2001
From: Jean Perier <jperier@nvidia.com>
Date: Thu, 30 Jul 2020 14:29:24 +0200
Subject: [PATCH 0608/1035] [flang] Expose specific to generic intrinsic name
 mapping

The intrinsic lowering facility is based on the generic intrinsic names to avoid
duplicating implementations. Specific intrinsics call are re-written to call to
the generic versions by the front-end but this cannot be done when specific intrinsics
are passed as arguments (the rewrite would give illegal/ambiguous unparsed Fortran).
Solve the issue by making the specific to generic name mapping accessible to lowering
and can be later used to generate the unrestricted intrinsic functions.

Reviewed By: schweitz

Differential Revision: https://reviews.llvm.org/D84842
---
 flang/include/flang/Evaluate/intrinsics.h |  5 +++
 flang/lib/Evaluate/intrinsics.cpp         | 17 +++++++++++
 flang/unittests/Evaluate/intrinsics.cpp   | 37 +++++++++++++++++++++++
 3 files changed, 59 insertions(+)

diff --git a/flang/include/flang/Evaluate/intrinsics.h b/flang/include/flang/Evaluate/intrinsics.h
index 88d6a7af13eb7..50212a1434c33 100644
--- a/flang/include/flang/Evaluate/intrinsics.h
+++ b/flang/include/flang/Evaluate/intrinsics.h
@@ -76,6 +76,11 @@ class IntrinsicProcTable {
   // Inquiry intrinsics are defined in section 16.7, table 16.1
   IntrinsicClass GetIntrinsicClass(const std::string &) const;
 
+  // Return the generic name of a specific intrinsic name.
+  // The name provided is returned if it is a generic intrinsic name or is
+  // not known to be an intrinsic.
+  std::string GetGenericIntrinsicName(const std::string &) const;
+
   // Probe the intrinsics for a match against a specific call.
   // On success, the actual arguments are transferred to the result
   // in dummy argument order; on failure, the actual arguments remain
diff --git a/flang/lib/Evaluate/intrinsics.cpp b/flang/lib/Evaluate/intrinsics.cpp
index 6237499d12ef7..0ad5c195ae861 100644
--- a/flang/lib/Evaluate/intrinsics.cpp
+++ b/flang/lib/Evaluate/intrinsics.cpp
@@ -1568,6 +1568,7 @@ class IntrinsicProcTable::Implementation {
   bool IsIntrinsic(const std::string &) const;
 
   IntrinsicClass GetIntrinsicClass(const std::string &) const;
+  std::string GetGenericIntrinsicName(const std::string &) const;
 
   std::optional<SpecificCall> Probe(const CallCharacteristics &,
       ActualArguments &, FoldingContext &, const IntrinsicProcTable &) const;
@@ -1625,6 +1626,17 @@ IntrinsicClass IntrinsicProcTable::Implementation::GetIntrinsicClass(
   return IntrinsicClass::noClass;
 }
 
+std::string IntrinsicProcTable::Implementation::GetGenericIntrinsicName(
+    const std::string &name) const {
+  auto specificIntrinsic{specificFuncs_.find(name)};
+  if (specificIntrinsic != specificFuncs_.end()) {
+    if (const char *genericName{specificIntrinsic->second->generic}) {
+      return {genericName};
+    }
+  }
+  return name;
+}
+
 bool CheckAndRearrangeArguments(ActualArguments &arguments,
     parser::ContextualMessages &messages, const char *const dummyKeywords[],
     std::size_t trailingOptionals) {
@@ -2075,6 +2087,11 @@ IntrinsicClass IntrinsicProcTable::GetIntrinsicClass(
   return DEREF(impl_).GetIntrinsicClass(name);
 }
 
+std::string IntrinsicProcTable::GetGenericIntrinsicName(
+    const std::string &name) const {
+  return DEREF(impl_).GetGenericIntrinsicName(name);
+}
+
 std::optional<SpecificCall> IntrinsicProcTable::Probe(
     const CallCharacteristics &call, ActualArguments &arguments,
     FoldingContext &context) const {
diff --git a/flang/unittests/Evaluate/intrinsics.cpp b/flang/unittests/Evaluate/intrinsics.cpp
index 57f71965f8d94..7b8895b8da4c0 100644
--- a/flang/unittests/Evaluate/intrinsics.cpp
+++ b/flang/unittests/Evaluate/intrinsics.cpp
@@ -292,6 +292,43 @@ void TestIntrinsics() {
       .DoCall(); // bad type
 
   // TODO: test other intrinsics
+
+  // Test unrestricted specific to generic name mapping (table 16.2).
+  TEST(table.GetGenericIntrinsicName("alog") == "log");
+  TEST(table.GetGenericIntrinsicName("alog10") == "log10");
+  TEST(table.GetGenericIntrinsicName("amod") == "mod");
+  TEST(table.GetGenericIntrinsicName("cabs") == "abs");
+  TEST(table.GetGenericIntrinsicName("ccos") == "cos");
+  TEST(table.GetGenericIntrinsicName("cexp") == "exp");
+  TEST(table.GetGenericIntrinsicName("clog") == "log");
+  TEST(table.GetGenericIntrinsicName("csin") == "sin");
+  TEST(table.GetGenericIntrinsicName("csqrt") == "sqrt");
+  TEST(table.GetGenericIntrinsicName("dabs") == "abs");
+  TEST(table.GetGenericIntrinsicName("dacos") == "acos");
+  TEST(table.GetGenericIntrinsicName("dasin") == "asin");
+  TEST(table.GetGenericIntrinsicName("datan") == "atan");
+  TEST(table.GetGenericIntrinsicName("datan2") == "atan2");
+  TEST(table.GetGenericIntrinsicName("dcos") == "cos");
+  TEST(table.GetGenericIntrinsicName("dcosh") == "cosh");
+  TEST(table.GetGenericIntrinsicName("ddim") == "dim");
+  TEST(table.GetGenericIntrinsicName("dexp") == "exp");
+  TEST(table.GetGenericIntrinsicName("dint") == "aint");
+  TEST(table.GetGenericIntrinsicName("dlog") == "log");
+  TEST(table.GetGenericIntrinsicName("dlog10") == "log10");
+  TEST(table.GetGenericIntrinsicName("dmod") == "mod");
+  TEST(table.GetGenericIntrinsicName("dnint") == "anint");
+  TEST(table.GetGenericIntrinsicName("dsign") == "sign");
+  TEST(table.GetGenericIntrinsicName("dsin") == "sin");
+  TEST(table.GetGenericIntrinsicName("dsinh") == "sinh");
+  TEST(table.GetGenericIntrinsicName("dsqrt") == "sqrt");
+  TEST(table.GetGenericIntrinsicName("dtan") == "tan");
+  TEST(table.GetGenericIntrinsicName("dtanh") == "tanh");
+  TEST(table.GetGenericIntrinsicName("iabs") == "abs");
+  TEST(table.GetGenericIntrinsicName("idim") == "dim");
+  TEST(table.GetGenericIntrinsicName("idnint") == "nint");
+  TEST(table.GetGenericIntrinsicName("isign") == "sign");
+  // Test a case where specific and generic name are the same.
+  TEST(table.GetGenericIntrinsicName("acos") == "acos");
 }
 } // namespace Fortran::evaluate
 

From e12db3ed997de473b2b7189781dbec7a239a3994 Mon Sep 17 00:00:00 2001
From: Stephan Herhut <herhut@google.com>
Date: Thu, 30 Jul 2020 14:02:46 +0200
Subject: [PATCH 0609/1035] [mlir] Allow index as element type of memref

Differential Revision: https://reviews.llvm.org/D84934
---
 mlir/docs/Rationale/Rationale.md              | 29 +++++--------------
 mlir/lib/IR/StandardTypes.cpp                 |  2 +-
 mlir/lib/Parser/TypeParser.cpp                |  2 +-
 .../StandardToLLVM/convert-to-llvmir.mlir     | 14 +++++++++
 mlir/test/IR/invalid.mlir                     |  4 ---
 mlir/test/IR/parser.mlir                      |  3 ++
 6 files changed, 27 insertions(+), 27 deletions(-)

diff --git a/mlir/docs/Rationale/Rationale.md b/mlir/docs/Rationale/Rationale.md
index 22e21383e9036..e906559a14a7f 100644
--- a/mlir/docs/Rationale/Rationale.md
+++ b/mlir/docs/Rationale/Rationale.md
@@ -202,32 +202,19 @@ and described in
 interest
 [starts here](https://www.google.com/url?q=https://youtu.be/Ntj8ab-5cvE?t%3D596&sa=D&ust=1529450150971000&usg=AFQjCNFQHEWL7m8q3eO-1DiKw9zqC2v24Q).
 
-### Index type disallowed in vector/memref types
+### Index type disallowed in vector types
 
-Index types are not allowed as elements of `vector` and `memref` types. Index
+Index types are not allowed as elements of `vector` types. Index
 types are intended to be used for platform-specific "size" values and may appear
 in subscripts, sizes of aggregate types and affine expressions. They are also
 tightly coupled with `affine.apply` and affine.load/store operations; having
 `index` type is a necessary precondition of a value to be acceptable by these
-operations. While it may be useful to have `memref<?xindex>` to express indirect
-accesses, e.g. sparse matrix manipulations or lookup tables, it creates problems
-MLIR is not ready to address yet. MLIR needs to internally store constants of
-aggregate types and emit code operating on values of those types, which are
-subject to target-specific size and alignment constraints. Since MLIR does not
-have a target description mechanism at the moment, it cannot reliably emit such
-code. Moreover, some platforms may not support vectors of type equivalent to
-`index`.
-
-Indirect access use cases can be alternatively supported by providing and
-`index_cast` instruction that allows for conversion between `index` and
-fixed-width integer types, at the SSA value level. It has an additional benefit
-of supporting smaller integer types, e.g. `i8` or `i16`, for small indices
-instead of (presumably larger) `index` type.
-
-Index types are allowed as element types of `tensor` types. The `tensor` type
-specifically abstracts the target-specific aspects that intersect with the
-code-generation-related/lowering-related concerns explained above. In fact, the
-`tensor` type even allows dialect-specific types as element types.
+operations.
+
+We allow `index` types in tensors and memrefs as a code generation strategy has
+to map `index` to an implementation type and hence needs to be able to
+materialize corresponding values. However, the target might lack support for
+`vector` values with the target specfic equivalent of the `index` type.
 
 ### Bit width of a non-primitive type and `index` is undefined
 
diff --git a/mlir/lib/IR/StandardTypes.cpp b/mlir/lib/IR/StandardTypes.cpp
index 5a9d22148b76f..2d1f8d8eb6f05 100644
--- a/mlir/lib/IR/StandardTypes.cpp
+++ b/mlir/lib/IR/StandardTypes.cpp
@@ -398,7 +398,7 @@ MemRefType MemRefType::getImpl(ArrayRef<int64_t> shape, Type elementType,
   auto *context = elementType.getContext();
 
   // Check that memref is formed from allowed types.
-  if (!elementType.isIntOrFloat() &&
+  if (!elementType.isIntOrIndexOrFloat() &&
       !elementType.isa<VectorType, ComplexType>())
     return emitOptionalError(location, "invalid memref element type"),
            MemRefType();
diff --git a/mlir/lib/Parser/TypeParser.cpp b/mlir/lib/Parser/TypeParser.cpp
index 9d8d198aa1c84..f5c98f3c6f9dd 100644
--- a/mlir/lib/Parser/TypeParser.cpp
+++ b/mlir/lib/Parser/TypeParser.cpp
@@ -217,7 +217,7 @@ Type Parser::parseMemRefType() {
     return nullptr;
 
   // Check that memref is formed from allowed types.
-  if (!elementType.isIntOrFloat() &&
+  if (!elementType.isIntOrIndexOrFloat() &&
       !elementType.isa<VectorType, ComplexType>())
     return emitError(typeLoc, "invalid memref element type"), nullptr;
 
diff --git a/mlir/test/Conversion/StandardToLLVM/convert-to-llvmir.mlir b/mlir/test/Conversion/StandardToLLVM/convert-to-llvmir.mlir
index 2129cf6819a9c..c1ec558da86f1 100644
--- a/mlir/test/Conversion/StandardToLLVM/convert-to-llvmir.mlir
+++ b/mlir/test/Conversion/StandardToLLVM/convert-to-llvmir.mlir
@@ -1277,3 +1277,17 @@ func @bfloat(%arg0: bf16) -> bf16 {
   return %arg0 : bf16
 }
 // CHECK-NEXT: return %{{.*}} : !llvm.bfloat
+
+// -----
+
+// CHECK-LABEL: func @memref_index
+// CHECK-SAME: %arg0: !llvm<"i64*">, %arg1: !llvm<"i64*">,
+// CHECK-SAME: %arg2: !llvm.i64, %arg3: !llvm.i64, %arg4: !llvm.i64)
+// CHECK-SAME: -> !llvm<"{ i64*, i64*, i64, [1 x i64], [1 x i64] }">
+// CHECK32-LABEL: func @memref_index
+// CHECK32-SAME: %arg0: !llvm<"i32*">, %arg1: !llvm<"i32*">,
+// CHECK32-SAME: %arg2: !llvm.i32, %arg3: !llvm.i32, %arg4: !llvm.i32)
+// CHECK32-SAME: -> !llvm<"{ i32*, i32*, i32, [1 x i32], [1 x i32] }">
+func @memref_index(%arg0: memref<32xindex>) -> memref<32xindex> {
+  return %arg0 : memref<32xindex>
+}
diff --git a/mlir/test/IR/invalid.mlir b/mlir/test/IR/invalid.mlir
index 2d8474c655f60..dcf04735c9010 100644
--- a/mlir/test/IR/invalid.mlir
+++ b/mlir/test/IR/invalid.mlir
@@ -19,10 +19,6 @@ func @nestedtensor(tensor<tensor<i8>>) -> () // expected-error {{invalid tensor
 
 func @indexvector(vector<4 x index>) -> () // expected-error {{vector elements must be int or float type}}
 
-// -----
-
-func @indexmemref(memref<? x index>) -> () // expected-error {{invalid memref element type}}
-
 // -----
 // Test no map in memref type.
 func @memrefs(memref<2x4xi8, >) // expected-error {{expected list element}}
diff --git a/mlir/test/IR/parser.mlir b/mlir/test/IR/parser.mlir
index 93db23fd5d0db..8d3d161ef27e0 100644
--- a/mlir/test/IR/parser.mlir
+++ b/mlir/test/IR/parser.mlir
@@ -140,6 +140,9 @@ func @memrefs_compose_with_id(memref<2x2xi8, affine_map<(d0, d1) -> (d0, d1)>,
 func @complex_types(complex<i1>) -> complex<f32>
 
 
+// CHECK: func @memref_with_index_elems(memref<1x?xindex>)
+func @memref_with_index_elems(memref<1x?xindex>)
+
 // CHECK: func @memref_with_complex_elems(memref<1x?xcomplex<f32>>)
 func @memref_with_complex_elems(memref<1x?xcomplex<f32>>)
 

From 76d07503f0c69f6632e6d8d4736e2a4cb4055a92 Mon Sep 17 00:00:00 2001
From: Abhishek Varma <abhishek.varma@polymagelabs.com>
Date: Thu, 30 Jul 2020 18:06:59 +0530
Subject: [PATCH 0610/1035] [MLIR] Introduce inter-procedural memref layout
 normalization

-- Introduces a pass that normalizes the affine layout maps to the identity layout map both within and across functions by rewriting function arguments and call operands where necessary.
-- Memref normalization is now implemented entirely in the module pass '-normalize-memrefs' and the limited intra-procedural version has been removed from '-simplify-affine-structures'.
-- Run using -normalize-memrefs.
-- Return ops are not handled and would be handled in the subsequent revisions.

Signed-off-by: Abhishek Varma <abhishek.varma@polymagelabs.com>

Differential Revision: https://reviews.llvm.org/D84490
---
 mlir/include/mlir/Transforms/Passes.h         |   8 +-
 mlir/include/mlir/Transforms/Passes.td        |   6 +-
 mlir/include/mlir/Transforms/Utils.h          |  43 +++-
 .../Transforms/SimplifyAffineStructures.cpp   |   9 -
 mlir/lib/Transforms/CMakeLists.txt            |   1 +
 mlir/lib/Transforms/NormalizeMemRefs.cpp      | 218 ++++++++++++++++++
 mlir/lib/Transforms/Utils/Utils.cpp           | 180 +++++++++------
 ...-normalize.mlir => normalize-memrefs.mlir} |  63 ++++-
 8 files changed, 428 insertions(+), 100 deletions(-)
 create mode 100644 mlir/lib/Transforms/NormalizeMemRefs.cpp
 rename mlir/test/Transforms/{memref-normalize.mlir => normalize-memrefs.mlir} (65%)

diff --git a/mlir/include/mlir/Transforms/Passes.h b/mlir/include/mlir/Transforms/Passes.h
index 955b0e99a1d17..1ffff1a25a6d5 100644
--- a/mlir/include/mlir/Transforms/Passes.h
+++ b/mlir/include/mlir/Transforms/Passes.h
@@ -24,7 +24,9 @@ class AffineForOp;
 class FuncOp;
 class ModuleOp;
 class Pass;
-template <typename T> class OperationPass;
+
+template <typename T>
+class OperationPass;
 
 /// Creates an instance of the BufferPlacement pass.
 std::unique_ptr<Pass> createBufferPlacementPass();
@@ -89,6 +91,10 @@ std::unique_ptr<Pass> createSCCPPass();
 /// Creates a pass which delete symbol operations that are unreachable. This
 /// pass may *only* be scheduled on an operation that defines a SymbolTable.
 std::unique_ptr<Pass> createSymbolDCEPass();
+
+/// Creates an interprocedural pass to normalize memrefs to have a trivial
+/// (identity) layout map.
+std::unique_ptr<OperationPass<ModuleOp>> createNormalizeMemRefsPass();
 } // end namespace mlir
 
 #endif // MLIR_TRANSFORMS_PASSES_H
diff --git a/mlir/include/mlir/Transforms/Passes.td b/mlir/include/mlir/Transforms/Passes.td
index 9e0d5c40d61fc..bd905b0e20f65 100644
--- a/mlir/include/mlir/Transforms/Passes.td
+++ b/mlir/include/mlir/Transforms/Passes.td
@@ -309,6 +309,11 @@ def MemRefDataFlowOpt : FunctionPass<"memref-dataflow-opt"> {
   let constructor = "mlir::createMemRefDataFlowOptPass()";
 }
 
+def NormalizeMemRefs : Pass<"normalize-memrefs", "ModuleOp"> {
+  let summary = "Normalize memrefs";
+  let constructor = "mlir::createNormalizeMemRefsPass()";
+}
+
 def ParallelLoopCollapsing : Pass<"parallel-loop-collapsing"> {
   let summary = "Collapse parallel loops to use less induction variables";
   let constructor = "mlir::createParallelLoopCollapsingPass()";
@@ -405,5 +410,4 @@ def SymbolDCE : Pass<"symbol-dce"> {
   }];
   let constructor = "mlir::createSymbolDCEPass()";
 }
-
 #endif // MLIR_TRANSFORMS_PASSES
diff --git a/mlir/include/mlir/Transforms/Utils.h b/mlir/include/mlir/Transforms/Utils.h
index 6f29c1b41ae63..81b4dfd0a01b3 100644
--- a/mlir/include/mlir/Transforms/Utils.h
+++ b/mlir/include/mlir/Transforms/Utils.h
@@ -45,10 +45,19 @@ class OpBuilder;
 /// operations that are dominated by the former; similarly, `postDomInstFilter`
 /// restricts replacement to only those operations that are postdominated by it.
 ///
+/// 'allowNonDereferencingOps', if set, allows replacement of non-dereferencing
+/// uses of a memref without any requirement for access index rewrites. The
+/// default value of this flag variable is false.
+///
+/// 'replaceInDeallocOp', if set, lets DeallocOp, a non-dereferencing user, to
+/// also be a candidate for replacement. The default value of this flag is
+/// false.
+///
 /// Returns true on success and false if the replacement is not possible,
-/// whenever a memref is used as an operand in a non-dereferencing context,
-/// except for dealloc's on the memref which are left untouched. See comments at
-/// function definition for an example.
+/// whenever a memref is used as an operand in a non-dereferencing context and
+/// 'allowNonDereferencingOps' is false, except for dealloc's on the memref
+/// which are left untouched. See comments at function definition for an
+/// example.
 //
 //  Ex: to replace load %A[%i, %j] with load %Abuf[%t mod 2, %ii - %i, %j]:
 //  The SSA value corresponding to '%t mod 2' should be in 'extraIndices', and
@@ -57,28 +66,38 @@ class OpBuilder;
 //  extra operands, note that 'indexRemap' would just be applied to existing
 //  indices (%i, %j).
 //  TODO: allow extraIndices to be added at any position.
-LogicalResult replaceAllMemRefUsesWith(Value oldMemRef, Value newMemRef,
-                                       ArrayRef<Value> extraIndices = {},
-                                       AffineMap indexRemap = AffineMap(),
-                                       ArrayRef<Value> extraOperands = {},
-                                       ArrayRef<Value> symbolOperands = {},
-                                       Operation *domInstFilter = nullptr,
-                                       Operation *postDomInstFilter = nullptr);
+LogicalResult replaceAllMemRefUsesWith(
+    Value oldMemRef, Value newMemRef, ArrayRef<Value> extraIndices = {},
+    AffineMap indexRemap = AffineMap(), ArrayRef<Value> extraOperands = {},
+    ArrayRef<Value> symbolOperands = {}, Operation *domInstFilter = nullptr,
+    Operation *postDomInstFilter = nullptr,
+    bool allowNonDereferencingOps = false, bool replaceInDeallocOp = false);
 
 /// Performs the same replacement as the other version above but only for the
-/// dereferencing uses of `oldMemRef` in `op`.
+/// dereferencing uses of `oldMemRef` in `op`, except in cases where
+/// 'allowNonDereferencingOps' is set to true where we replace the
+/// non-dereferencing uses as well.
 LogicalResult replaceAllMemRefUsesWith(Value oldMemRef, Value newMemRef,
                                        Operation *op,
                                        ArrayRef<Value> extraIndices = {},
                                        AffineMap indexRemap = AffineMap(),
                                        ArrayRef<Value> extraOperands = {},
-                                       ArrayRef<Value> symbolOperands = {});
+                                       ArrayRef<Value> symbolOperands = {},
+                                       bool allowNonDereferencingOps = false);
 
 /// Rewrites the memref defined by this alloc op to have an identity layout map
 /// and updates all its indexing uses. Returns failure if any of its uses
 /// escape (while leaving the IR in a valid state).
 LogicalResult normalizeMemRef(AllocOp op);
 
+/// Uses the old memref type map layout and computes the new memref type to have
+/// a new shape and a layout map, where the old layout map has been normalized
+/// to an identity layout map. It returns the old memref in case no
+/// normalization was needed or a failure occurs while transforming the old map
+/// layout to an identity layout map.
+MemRefType normalizeMemRefType(MemRefType memrefType, OpBuilder builder,
+                               unsigned numSymbolicOperands);
+
 /// Creates and inserts into 'builder' a new AffineApplyOp, with the number of
 /// its results equal to the number of operands, as a composition
 /// of all other AffineApplyOps reachable from input parameter 'operands'. If
diff --git a/mlir/lib/Dialect/Affine/Transforms/SimplifyAffineStructures.cpp b/mlir/lib/Dialect/Affine/Transforms/SimplifyAffineStructures.cpp
index 0cd59b52d543d..d8ffb9742fae2 100644
--- a/mlir/lib/Dialect/Affine/Transforms/SimplifyAffineStructures.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/SimplifyAffineStructures.cpp
@@ -96,13 +96,4 @@ void SimplifyAffineStructures::runOnFunction() {
     if (isa<AffineForOp, AffineIfOp, AffineApplyOp>(op))
       applyOpPatternsAndFold(op, patterns);
   });
-
-  // Turn memrefs' non-identity layouts maps into ones with identity. Collect
-  // alloc ops first and then process since normalizeMemRef replaces/erases ops
-  // during memref rewriting.
-  SmallVector<AllocOp, 4> allocOps;
-  func.walk([&](AllocOp op) { allocOps.push_back(op); });
-  for (auto allocOp : allocOps) {
-    normalizeMemRef(allocOp);
-  }
 }
diff --git a/mlir/lib/Transforms/CMakeLists.txt b/mlir/lib/Transforms/CMakeLists.txt
index 3c6b3933de2a8..58c5fa6720883 100644
--- a/mlir/lib/Transforms/CMakeLists.txt
+++ b/mlir/lib/Transforms/CMakeLists.txt
@@ -12,6 +12,7 @@ add_mlir_library(MLIRTransforms
   LoopFusion.cpp
   LoopInvariantCodeMotion.cpp
   MemRefDataFlowOpt.cpp
+  NormalizeMemRefs.cpp
   OpStats.cpp
   ParallelLoopCollapsing.cpp
   PipelineDataTransfer.cpp
diff --git a/mlir/lib/Transforms/NormalizeMemRefs.cpp b/mlir/lib/Transforms/NormalizeMemRefs.cpp
new file mode 100644
index 0000000000000..1484dcb9e10c6
--- /dev/null
+++ b/mlir/lib/Transforms/NormalizeMemRefs.cpp
@@ -0,0 +1,218 @@
+//===- NormalizeMemRefs.cpp -----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements an interprocedural pass to normalize memrefs to have
+// identity layout maps.
+//
+//===----------------------------------------------------------------------===//
+
+#include "PassDetail.h"
+#include "mlir/Dialect/Affine/IR/AffineOps.h"
+#include "mlir/Transforms/Passes.h"
+#include "mlir/Transforms/Utils.h"
+
+#define DEBUG_TYPE "normalize-memrefs"
+
+using namespace mlir;
+
+namespace {
+
+/// All memrefs passed across functions with non-trivial layout maps are
+/// converted to ones with trivial identity layout ones.
+
+// Input :-
+// #tile = affine_map<(i) -> (i floordiv 4, i mod 4)>
+// func @matmul(%A: memref<16xf64, #tile>, %B: index, %C: memref<16xf64>) ->
+// (memref<16xf64, #tile>) {
+//   affine.for %arg3 = 0 to 16 {
+//         %a = affine.load %A[%arg3] : memref<16xf64, #tile>
+//         %p = mulf %a, %a : f64
+//         affine.store %p, %A[%arg3] : memref<16xf64, #tile>
+//   }
+//   %c = alloc() : memref<16xf64, #tile>
+//   %d = affine.load %c[0] : memref<16xf64, #tile>
+//   return %A: memref<16xf64, #tile>
+// }
+
+// Output :-
+//   func @matmul(%arg0: memref<4x4xf64>, %arg1: index, %arg2: memref<16xf64>)
+//   -> memref<4x4xf64> {
+//     affine.for %arg3 = 0 to 16 {
+//       %2 = affine.load %arg0[%arg3 floordiv 4, %arg3 mod 4] : memref<4x4xf64>
+//       %3 = mulf %2, %2 : f64
+//       affine.store %3, %arg0[%arg3 floordiv 4, %arg3 mod 4] : memref<4x4xf64>
+//     }
+//     %0 = alloc() : memref<16xf64, #map0>
+//     %1 = affine.load %0[0] : memref<16xf64, #map0>
+//     return %arg0 : memref<4x4xf64>
+//   }
+
+struct NormalizeMemRefs : public NormalizeMemRefsBase<NormalizeMemRefs> {
+  void runOnOperation() override;
+  void runOnFunction(FuncOp funcOp);
+  bool areMemRefsNormalizable(FuncOp funcOp);
+  void updateFunctionSignature(FuncOp funcOp);
+};
+
+} // end anonymous namespace
+
+std::unique_ptr<OperationPass<ModuleOp>> mlir::createNormalizeMemRefsPass() {
+  return std::make_unique<NormalizeMemRefs>();
+}
+
+void NormalizeMemRefs::runOnOperation() {
+  ModuleOp moduleOp = getOperation();
+  // We traverse each function within the module in order to normalize the
+  // memref type arguments.
+  // TODO: Handle external functions.
+  moduleOp.walk([&](FuncOp funcOp) {
+    if (areMemRefsNormalizable(funcOp))
+      runOnFunction(funcOp);
+  });
+}
+
+// Return true if this operation dereferences one or more memref's.
+// TODO: Temporary utility, will be replaced when this is modeled through
+// side-effects/op traits.
+static bool isMemRefDereferencingOp(Operation &op) {
+  return isa<AffineReadOpInterface, AffineWriteOpInterface, AffineDmaStartOp,
+             AffineDmaWaitOp>(op);
+}
+
+// Check whether all the uses of oldMemRef are either dereferencing uses or the
+// op is of type : DeallocOp, CallOp. Only if these constraints are satisfied
+// will the value become a candidate for replacement.
+static bool isMemRefNormalizable(Value::user_range opUsers) {
+  if (llvm::any_of(opUsers, [](Operation *op) {
+        if (isMemRefDereferencingOp(*op))
+          return false;
+        return !isa<DeallocOp, CallOp>(*op);
+      }))
+    return false;
+  return true;
+}
+
+// Check whether all the uses of AllocOps, CallOps and function arguments of a
+// function are either of dereferencing type or of type: DeallocOp, CallOp. Only
+// if these constraints are satisfied will the function become a candidate for
+// normalization.
+bool NormalizeMemRefs::areMemRefsNormalizable(FuncOp funcOp) {
+  if (funcOp
+          .walk([&](AllocOp allocOp) -> WalkResult {
+            Value oldMemRef = allocOp.getResult();
+            if (!isMemRefNormalizable(oldMemRef.getUsers()))
+              return WalkResult::interrupt();
+            return WalkResult::advance();
+          })
+          .wasInterrupted())
+    return false;
+
+  if (funcOp
+          .walk([&](CallOp callOp) -> WalkResult {
+            for (unsigned resIndex :
+                 llvm::seq<unsigned>(0, callOp.getNumResults())) {
+              Value oldMemRef = callOp.getResult(resIndex);
+              if (oldMemRef.getType().isa<MemRefType>())
+                if (!isMemRefNormalizable(oldMemRef.getUsers()))
+                  return WalkResult::interrupt();
+            }
+            return WalkResult::advance();
+          })
+          .wasInterrupted())
+    return false;
+
+  for (unsigned argIndex : llvm::seq<unsigned>(0, funcOp.getNumArguments())) {
+    BlockArgument oldMemRef = funcOp.getArgument(argIndex);
+    if (oldMemRef.getType().isa<MemRefType>())
+      if (!isMemRefNormalizable(oldMemRef.getUsers()))
+        return false;
+  }
+
+  return true;
+}
+
+// Fetch the updated argument list and result of the function and update the
+// function signature.
+void NormalizeMemRefs::updateFunctionSignature(FuncOp funcOp) {
+  FunctionType functionType = funcOp.getType();
+  SmallVector<Type, 8> argTypes;
+  SmallVector<Type, 4> resultTypes;
+
+  for (const auto &arg : llvm::enumerate(funcOp.getArguments()))
+    argTypes.push_back(arg.value().getType());
+
+  resultTypes = llvm::to_vector<4>(functionType.getResults());
+  // We create a new function type and modify the function signature with this
+  // new type.
+  FunctionType newFuncType = FunctionType::get(/*inputs=*/argTypes,
+                                               /*results=*/resultTypes,
+                                               /*context=*/&getContext());
+
+  // TODO: Handle ReturnOps to update function results the caller site.
+  funcOp.setType(newFuncType);
+}
+
+void NormalizeMemRefs::runOnFunction(FuncOp funcOp) {
+  // Turn memrefs' non-identity layouts maps into ones with identity. Collect
+  // alloc ops first and then process since normalizeMemRef replaces/erases ops
+  // during memref rewriting.
+  SmallVector<AllocOp, 4> allocOps;
+  funcOp.walk([&](AllocOp op) { allocOps.push_back(op); });
+  for (AllocOp allocOp : allocOps)
+    normalizeMemRef(allocOp);
+
+  // We use this OpBuilder to create new memref layout later.
+  OpBuilder b(funcOp);
+
+  // Walk over each argument of a function to perform memref normalization (if
+  // any).
+  for (unsigned argIndex : llvm::seq<unsigned>(0, funcOp.getNumArguments())) {
+    Type argType = funcOp.getArgument(argIndex).getType();
+    MemRefType memrefType = argType.dyn_cast<MemRefType>();
+    // Check whether argument is of MemRef type. Any other argument type can
+    // simply be part of the final function signature.
+    if (!memrefType)
+      continue;
+    // Fetch a new memref type after normalizing the old memref to have an
+    // identity map layout.
+    MemRefType newMemRefType = normalizeMemRefType(memrefType, b,
+                                                   /*numSymbolicOperands=*/0);
+    if (newMemRefType == memrefType) {
+      // Either memrefType already had an identity map or the map couldn't be
+      // transformed to an identity map.
+      continue;
+    }
+
+    // Insert a new temporary argument with the new memref type.
+    BlockArgument newMemRef =
+        funcOp.front().insertArgument(argIndex, newMemRefType);
+    BlockArgument oldMemRef = funcOp.getArgument(argIndex + 1);
+    AffineMap layoutMap = memrefType.getAffineMaps().front();
+    // Replace all uses of the old memref.
+    if (failed(replaceAllMemRefUsesWith(oldMemRef, /*newMemRef=*/newMemRef,
+                                        /*extraIndices=*/{},
+                                        /*indexRemap=*/layoutMap,
+                                        /*extraOperands=*/{},
+                                        /*symbolOperands=*/{},
+                                        /*domInstFilter=*/nullptr,
+                                        /*postDomInstFilter=*/nullptr,
+                                        /*allowNonDereferencingOps=*/true,
+                                        /*handleDeallocOp=*/true))) {
+      // If it failed (due to escapes for example), bail out. Removing the
+      // temporary argument inserted previously.
+      funcOp.front().eraseArgument(argIndex);
+      continue;
+    }
+
+    // All uses for the argument with old memref type were replaced
+    // successfully. So we remove the old argument now.
+    funcOp.front().eraseArgument(argIndex + 1);
+  }
+
+  updateFunctionSignature(funcOp);
+}
diff --git a/mlir/lib/Transforms/Utils/Utils.cpp b/mlir/lib/Transforms/Utils/Utils.cpp
index 35853c17232bd..73df4fa939bfb 100644
--- a/mlir/lib/Transforms/Utils/Utils.cpp
+++ b/mlir/lib/Transforms/Utils/Utils.cpp
@@ -48,7 +48,8 @@ LogicalResult mlir::replaceAllMemRefUsesWith(Value oldMemRef, Value newMemRef,
                                              ArrayRef<Value> extraIndices,
                                              AffineMap indexRemap,
                                              ArrayRef<Value> extraOperands,
-                                             ArrayRef<Value> symbolOperands) {
+                                             ArrayRef<Value> symbolOperands,
+                                             bool allowNonDereferencingOps) {
   unsigned newMemRefRank = newMemRef.getType().cast<MemRefType>().getRank();
   (void)newMemRefRank; // unused in opt mode
   unsigned oldMemRefRank = oldMemRef.getType().cast<MemRefType>().getRank();
@@ -67,11 +68,6 @@ LogicalResult mlir::replaceAllMemRefUsesWith(Value oldMemRef, Value newMemRef,
   assert(oldMemRef.getType().cast<MemRefType>().getElementType() ==
          newMemRef.getType().cast<MemRefType>().getElementType());
 
-  if (!isMemRefDereferencingOp(*op))
-    // Failure: memref used in a non-dereferencing context (potentially
-    // escapes); no replacement in these cases.
-    return failure();
-
   SmallVector<unsigned, 2> usePositions;
   for (const auto &opEntry : llvm::enumerate(op->getOperands())) {
     if (opEntry.value() == oldMemRef)
@@ -91,6 +87,18 @@ LogicalResult mlir::replaceAllMemRefUsesWith(Value oldMemRef, Value newMemRef,
   unsigned memRefOperandPos = usePositions.front();
 
   OpBuilder builder(op);
+  // The following checks if op is dereferencing memref and performs the access
+  // index rewrites.
+  if (!isMemRefDereferencingOp(*op)) {
+    if (!allowNonDereferencingOps)
+      // Failure: memref used in a non-dereferencing context (potentially
+      // escapes); no replacement in these cases unless allowNonDereferencingOps
+      // is set.
+      return failure();
+    op->setOperand(memRefOperandPos, newMemRef);
+    return success();
+  }
+  // Perform index rewrites for the dereferencing op and then replace the op
   NamedAttribute oldMapAttrPair = getAffineMapAttrForMemRef(op, oldMemRef);
   AffineMap oldMap = oldMapAttrPair.second.cast<AffineMapAttr>().getValue();
   unsigned oldMapNumInputs = oldMap.getNumInputs();
@@ -112,7 +120,7 @@ LogicalResult mlir::replaceAllMemRefUsesWith(Value oldMemRef, Value newMemRef,
       affineApplyOps.push_back(afOp);
     }
   } else {
-    oldMemRefOperands.append(oldMapOperands.begin(), oldMapOperands.end());
+    oldMemRefOperands.assign(oldMapOperands.begin(), oldMapOperands.end());
   }
 
   // Construct new indices as a remap of the old ones if a remapping has been
@@ -141,14 +149,14 @@ LogicalResult mlir::replaceAllMemRefUsesWith(Value oldMemRef, Value newMemRef,
     }
   } else {
     // No remapping specified.
-    remapOutputs.append(remapOperands.begin(), remapOperands.end());
+    remapOutputs.assign(remapOperands.begin(), remapOperands.end());
   }
 
   SmallVector<Value, 4> newMapOperands;
   newMapOperands.reserve(newMemRefRank);
 
   // Prepend 'extraIndices' in 'newMapOperands'.
-  for (auto extraIndex : extraIndices) {
+  for (Value extraIndex : extraIndices) {
     assert(extraIndex.getDefiningOp()->getNumResults() == 1 &&
            "single result op's expected to generate these indices");
     assert((isValidDim(extraIndex) || isValidSymbol(extraIndex)) &&
@@ -167,12 +175,12 @@ LogicalResult mlir::replaceAllMemRefUsesWith(Value oldMemRef, Value newMemRef,
   newMap = simplifyAffineMap(newMap);
   canonicalizeMapAndOperands(&newMap, &newMapOperands);
   // Remove any affine.apply's that became dead as a result of composition.
-  for (auto value : affineApplyOps)
+  for (Value value : affineApplyOps)
     if (value.use_empty())
       value.getDefiningOp()->erase();
 
-  // Construct the new operation using this memref.
   OperationState state(op->getLoc(), op->getName());
+  // Construct the new operation using this memref.
   state.operands.reserve(op->getNumOperands() + extraIndices.size());
   // Insert the non-memref operands.
   state.operands.append(op->operand_begin(),
@@ -196,11 +204,10 @@ LogicalResult mlir::replaceAllMemRefUsesWith(Value oldMemRef, Value newMemRef,
   // Add attribute for 'newMap', other Attributes do not change.
   auto newMapAttr = AffineMapAttr::get(newMap);
   for (auto namedAttr : op->getAttrs()) {
-    if (namedAttr.first == oldMapAttrPair.first) {
+    if (namedAttr.first == oldMapAttrPair.first)
       state.attributes.push_back({namedAttr.first, newMapAttr});
-    } else {
+    else
       state.attributes.push_back(namedAttr);
-    }
   }
 
   // Create the new operation.
@@ -211,13 +218,12 @@ LogicalResult mlir::replaceAllMemRefUsesWith(Value oldMemRef, Value newMemRef,
   return success();
 }
 
-LogicalResult mlir::replaceAllMemRefUsesWith(Value oldMemRef, Value newMemRef,
-                                             ArrayRef<Value> extraIndices,
-                                             AffineMap indexRemap,
-                                             ArrayRef<Value> extraOperands,
-                                             ArrayRef<Value> symbolOperands,
-                                             Operation *domInstFilter,
-                                             Operation *postDomInstFilter) {
+LogicalResult mlir::replaceAllMemRefUsesWith(
+    Value oldMemRef, Value newMemRef, ArrayRef<Value> extraIndices,
+    AffineMap indexRemap, ArrayRef<Value> extraOperands,
+    ArrayRef<Value> symbolOperands, Operation *domInstFilter,
+    Operation *postDomInstFilter, bool allowNonDereferencingOps,
+    bool replaceInDeallocOp) {
   unsigned newMemRefRank = newMemRef.getType().cast<MemRefType>().getRank();
   (void)newMemRefRank; // unused in opt mode
   unsigned oldMemRefRank = oldMemRef.getType().cast<MemRefType>().getRank();
@@ -261,16 +267,21 @@ LogicalResult mlir::replaceAllMemRefUsesWith(Value oldMemRef, Value newMemRef,
 
     // Skip dealloc's - no replacement is necessary, and a memref replacement
     // at other uses doesn't hurt these dealloc's.
-    if (isa<DeallocOp>(op))
+    if (isa<DeallocOp>(op) && !replaceInDeallocOp)
       continue;
 
     // Check if the memref was used in a non-dereferencing context. It is fine
     // for the memref to be used in a non-dereferencing way outside of the
     // region where this replacement is happening.
-    if (!isMemRefDereferencingOp(*op))
-      // Failure: memref used in a non-dereferencing op (potentially escapes);
-      // no replacement in these cases.
-      return failure();
+    if (!isMemRefDereferencingOp(*op)) {
+      // Currently we support the following non-dereferencing types to be a
+      // candidate for replacement: Dealloc and CallOp.
+      // TODO: Add support for other kinds of ops.
+      if (!allowNonDereferencingOps)
+        return failure();
+      if (!(isa<DeallocOp, CallOp>(*op)))
+        return failure();
+    }
 
     // We'll first collect and then replace --- since replacement erases the op
     // that has the use, and that op could be postDomFilter or domFilter itself!
@@ -278,9 +289,9 @@ LogicalResult mlir::replaceAllMemRefUsesWith(Value oldMemRef, Value newMemRef,
   }
 
   for (auto *op : opsToReplace) {
-    if (failed(replaceAllMemRefUsesWith(oldMemRef, newMemRef, op, extraIndices,
-                                        indexRemap, extraOperands,
-                                        symbolOperands)))
+    if (failed(replaceAllMemRefUsesWith(
+            oldMemRef, newMemRef, op, extraIndices, indexRemap, extraOperands,
+            symbolOperands, allowNonDereferencingOps)))
       llvm_unreachable("memref replacement guaranteed to succeed here");
   }
 
@@ -385,85 +396,102 @@ void mlir::createAffineComputationSlice(
 // TODO: Currently works for static memrefs with a single layout map.
 LogicalResult mlir::normalizeMemRef(AllocOp allocOp) {
   MemRefType memrefType = allocOp.getType();
-  unsigned rank = memrefType.getRank();
-  if (rank == 0)
-    return success();
-
-  auto layoutMaps = memrefType.getAffineMaps();
   OpBuilder b(allocOp);
-  if (layoutMaps.size() != 1)
+
+  // Fetch a new memref type after normalizing the old memref to have an
+  // identity map layout.
+  MemRefType newMemRefType =
+      normalizeMemRefType(memrefType, b, allocOp.getNumSymbolicOperands());
+  if (newMemRefType == memrefType)
+    // Either memrefType already had an identity map or the map couldn't be
+    // transformed to an identity map.
     return failure();
 
-  AffineMap layoutMap = layoutMaps.front();
+  Value oldMemRef = allocOp.getResult();
 
-  // Nothing to do for identity layout maps.
-  if (layoutMap == b.getMultiDimIdentityMap(rank))
-    return success();
+  SmallVector<Value, 4> symbolOperands(allocOp.getSymbolicOperands());
+  AllocOp newAlloc = b.create<AllocOp>(allocOp.getLoc(), newMemRefType,
+                                       llvm::None, allocOp.alignmentAttr());
+  AffineMap layoutMap = memrefType.getAffineMaps().front();
+  // Replace all uses of the old memref.
+  if (failed(replaceAllMemRefUsesWith(oldMemRef, /*newMemRef=*/newAlloc,
+                                      /*extraIndices=*/{},
+                                      /*indexRemap=*/layoutMap,
+                                      /*extraOperands=*/{},
+                                      /*symbolOperands=*/symbolOperands,
+                                      /*domInstFilter=*/nullptr,
+                                      /*postDomInstFilter=*/nullptr,
+                                      /*allowDereferencingOps=*/true))) {
+    // If it failed (due to escapes for example), bail out.
+    newAlloc.erase();
+    return failure();
+  }
+  // Replace any uses of the original alloc op and erase it. All remaining uses
+  // have to be dealloc's; RAMUW above would've failed otherwise.
+  assert(llvm::all_of(oldMemRef.getUsers(),
+                      [](Operation *op) { return isa<DeallocOp>(op); }));
+  oldMemRef.replaceAllUsesWith(newAlloc);
+  allocOp.erase();
+  return success();
+}
+
+MemRefType mlir::normalizeMemRefType(MemRefType memrefType, OpBuilder b,
+                                     unsigned numSymbolicOperands) {
+  unsigned rank = memrefType.getRank();
+  if (rank == 0)
+    return memrefType;
+
+  ArrayRef<AffineMap> layoutMaps = memrefType.getAffineMaps();
+  if (layoutMaps.empty() ||
+      layoutMaps.front() == b.getMultiDimIdentityMap(rank)) {
+    // Either no maps is associated with this memref or this memref has
+    // a trivial (identity) map.
+    return memrefType;
+  }
 
   // We don't do any checks for one-to-one'ness; we assume that it is
   // one-to-one.
 
   // TODO: Only for static memref's for now.
   if (memrefType.getNumDynamicDims() > 0)
-    return failure();
+    return memrefType;
 
-  // We have a single map that is not an identity map. Create a new memref with
-  // the right shape and an identity layout map.
-  auto shape = memrefType.getShape();
-  FlatAffineConstraints fac(rank, allocOp.getNumSymbolicOperands());
+  // We have a single map that is not an identity map. Create a new memref
+  // with the right shape and an identity layout map.
+  ArrayRef<int64_t> shape = memrefType.getShape();
+  // FlatAffineConstraint may later on use symbolicOperands.
+  FlatAffineConstraints fac(rank, numSymbolicOperands);
   for (unsigned d = 0; d < rank; ++d) {
     fac.addConstantLowerBound(d, 0);
     fac.addConstantUpperBound(d, shape[d] - 1);
   }
-
-  // We compose this map with the original index (logical) space to derive the
-  // upper bounds for the new index space.
+  // We compose this map with the original index (logical) space to derive
+  // the upper bounds for the new index space.
+  AffineMap layoutMap = layoutMaps.front();
   unsigned newRank = layoutMap.getNumResults();
   if (failed(fac.composeMatchingMap(layoutMap)))
-    // TODO: semi-affine maps.
-    return failure();
-
+    return memrefType;
+  // TODO: Handle semi-affine maps.
   // Project out the old data dimensions.
   fac.projectOut(newRank, fac.getNumIds() - newRank - fac.getNumLocalIds());
   SmallVector<int64_t, 4> newShape(newRank);
   for (unsigned d = 0; d < newRank; ++d) {
     // The lower bound for the shape is always zero.
     auto ubConst = fac.getConstantUpperBound(d);
-    // For a static memref and an affine map with no symbols, this is always
-    // bounded.
+    // For a static memref and an affine map with no symbols, this is
+    // always bounded.
     assert(ubConst.hasValue() && "should always have an upper bound");
     if (ubConst.getValue() < 0)
       // This is due to an invalid map that maps to a negative space.
-      return failure();
+      return memrefType;
     newShape[d] = ubConst.getValue() + 1;
   }
 
-  auto oldMemRef = allocOp.getResult();
-  SmallVector<Value, 4> symbolOperands(allocOp.getSymbolicOperands());
-
+  // Create the new memref type after trivializing the old layout map.
   MemRefType newMemRefType =
       MemRefType::Builder(memrefType)
           .setShape(newShape)
           .setAffineMaps(b.getMultiDimIdentityMap(newRank));
 
-  auto newAlloc = b.create<AllocOp>(allocOp.getLoc(), newMemRefType, llvm::None,
-                                    allocOp.alignmentAttr());
-
-  // Replace all uses of the old memref.
-  if (failed(replaceAllMemRefUsesWith(oldMemRef, /*newMemRef=*/newAlloc,
-                                      /*extraIndices=*/{},
-                                      /*indexRemap=*/layoutMap,
-                                      /*extraOperands=*/{},
-                                      /*symbolOperands=*/symbolOperands))) {
-    // If it failed (due to escapes for example), bail out.
-    newAlloc.erase();
-    return failure();
-  }
-  // Replace any uses of the original alloc op and erase it. All remaining uses
-  // have to be dealloc's; RAMUW above would've failed otherwise.
-  assert(llvm::all_of(oldMemRef.getUsers(),
-                      [](Operation *op) { return isa<DeallocOp>(op); }));
-  oldMemRef.replaceAllUsesWith(newAlloc);
-  allocOp.erase();
-  return success();
+  return newMemRefType;
 }
diff --git a/mlir/test/Transforms/memref-normalize.mlir b/mlir/test/Transforms/normalize-memrefs.mlir
similarity index 65%
rename from mlir/test/Transforms/memref-normalize.mlir
rename to mlir/test/Transforms/normalize-memrefs.mlir
index 375bd3ef0e6c6..7d56c88939409 100644
--- a/mlir/test/Transforms/memref-normalize.mlir
+++ b/mlir/test/Transforms/normalize-memrefs.mlir
@@ -1,4 +1,7 @@
-// RUN: mlir-opt -allow-unregistered-dialect -simplify-affine-structures %s | FileCheck %s
+// RUN: mlir-opt -normalize-memrefs -allow-unregistered-dialect %s | FileCheck %s
+
+// This file tests whether the memref type having non-trivial map layouts
+// are normalized to trivial (identity) layouts.
 
 // CHECK-LABEL: func @permute()
 func @permute() {
@@ -150,3 +153,61 @@ func @alignment() {
   // CHECK-NEXT: alloc() {alignment = 32 : i64} : memref<256x64x128xf32>
   return
 }
+
+#tile = affine_map < (i)->(i floordiv 4, i mod 4) >
+
+// Following test cases check the inter-procedural memref normalization.
+
+// Test case 1: Check normalization for multiple memrefs in a function argument list.
+// CHECK-LABEL: func @multiple_argument_type
+// CHECK-SAME:  (%[[A:arg[0-9]+]]: memref<4x4xf64>, %[[B:arg[0-9]+]]: f64, %[[C:arg[0-9]+]]: memref<2x4xf64>, %[[D:arg[0-9]+]]: memref<24xf64>) -> f64
+func @multiple_argument_type(%A: memref<16xf64, #tile>, %B: f64, %C: memref<8xf64, #tile>, %D: memref<24xf64>) -> f64 {
+  %a = affine.load %A[0] : memref<16xf64, #tile>
+  %p = mulf %a, %a : f64
+  affine.store %p, %A[10] : memref<16xf64, #tile>
+  call @single_argument_type(%C): (memref<8xf64, #tile>) -> ()
+  return %B : f64
+}
+
+// CHECK: %[[a:[0-9]+]] = affine.load %[[A]][0, 0] : memref<4x4xf64>
+// CHECK: %[[p:[0-9]+]] = mulf %[[a]], %[[a]] : f64
+// CHECK: affine.store %[[p]], %[[A]][2, 2] : memref<4x4xf64>
+// CHECK: call @single_argument_type(%[[C]]) : (memref<2x4xf64>) -> ()
+// CHECK: return %[[B]] : f64
+
+// Test case 2: Check normalization for single memref argument in a function.
+// CHECK-LABEL: func @single_argument_type
+// CHECK-SAME: (%[[C:arg[0-9]+]]: memref<2x4xf64>)
+func @single_argument_type(%C : memref<8xf64, #tile>) {
+  %a = alloc(): memref<8xf64, #tile>
+  %b = alloc(): memref<16xf64, #tile>
+  %d = constant 23.0 : f64
+  %e = alloc(): memref<24xf64>
+  call @single_argument_type(%a): (memref<8xf64, #tile>) -> ()
+  call @single_argument_type(%C): (memref<8xf64, #tile>) -> ()
+  call @multiple_argument_type(%b, %d, %a, %e): (memref<16xf64, #tile>, f64, memref<8xf64, #tile>, memref<24xf64>) -> f64
+  return
+}
+
+// CHECK: %[[a:[0-9]+]] = alloc() : memref<2x4xf64>
+// CHECK: %[[b:[0-9]+]] = alloc() : memref<4x4xf64>
+// CHECK: %cst = constant 2.300000e+01 : f64
+// CHECK: %[[e:[0-9]+]] = alloc() : memref<24xf64>
+// CHECK: call @single_argument_type(%[[a]]) : (memref<2x4xf64>) -> ()
+// CHECK: call @single_argument_type(%[[C]]) : (memref<2x4xf64>) -> ()
+// CHECK: call @multiple_argument_type(%[[b]], %cst, %[[a]], %[[e]]) : (memref<4x4xf64>, f64, memref<2x4xf64>, memref<24xf64>) -> f64
+
+// Test case 3: Check function returning any other type except memref.
+// CHECK-LABEL: func @non_memref_ret
+// CHECK-SAME: (%[[C:arg[0-9]+]]: memref<2x4xf64>) -> i1
+func @non_memref_ret(%A: memref<8xf64, #tile>) -> i1 {
+  %d = constant 1 : i1
+  return %d : i1
+}
+
+// Test case 4: No normalization should take place because the function is returning the memref.
+// CHECK-LABEL: func @memref_used_in_return
+// CHECK-SAME: (%[[A:arg[0-9]+]]: memref<8xf64, #map{{[0-9]+}}>) -> memref<8xf64, #map{{[0-9]+}}>
+func @memref_used_in_return(%A: memref<8xf64, #tile>) -> (memref<8xf64, #tile>) {
+  return %A : memref<8xf64, #tile>
+}

From 13a3d88666e2162a91dfcae3824198c9ba74cc7b Mon Sep 17 00:00:00 2001
From: Christian Sigg <csigg@google.com>
Date: Thu, 30 Jul 2020 10:09:29 +0200
Subject: [PATCH 0611/1035] [MLIR] Don't pass separate LowerToLLVMOptions when
 we already pass a LLVMTypeConverter which contains those options already.

This also prevents passing inconsistent options.

Reviewed By: ftynse

Differential Revision: https://reviews.llvm.org/D84915
---
 .../StandardToLLVM/ConvertStandardToLLVM.h    | 10 +---
 .../ConvertStandardToLLVMPass.h               | 15 ++---
 .../StandardToLLVM/StandardToLLVM.cpp         | 59 ++++++++-----------
 3 files changed, 32 insertions(+), 52 deletions(-)

diff --git a/mlir/include/mlir/Conversion/StandardToLLVM/ConvertStandardToLLVM.h b/mlir/include/mlir/Conversion/StandardToLLVM/ConvertStandardToLLVM.h
index 72c6079db9ac5..298899b721d3f 100644
--- a/mlir/include/mlir/Conversion/StandardToLLVM/ConvertStandardToLLVM.h
+++ b/mlir/include/mlir/Conversion/StandardToLLVM/ConvertStandardToLLVM.h
@@ -87,6 +87,8 @@ class LLVMTypeConverter : public TypeConverter {
   /// Returns the LLVM dialect.
   LLVM::LLVMDialect *getDialect() { return llvmDialect; }
 
+  const LowerToLLVMOptions &getOptions() const { return options; }
+
   /// Promote the LLVM struct representation of all MemRef descriptors to stack
   /// and use pointers to struct to avoid the complexity of the
   /// platform-specific C/C++ ABI lowering related to struct argument passing.
@@ -390,8 +392,6 @@ class ConvertToLLVMPattern : public ConversionPattern {
 public:
   ConvertToLLVMPattern(StringRef rootOpName, MLIRContext *context,
                        LLVMTypeConverter &typeConverter,
-                       const LowerToLLVMOptions &options =
-                           LowerToLLVMOptions::getDefaultOptions(),
                        PatternBenefit benefit = 1);
 
   /// Returns the LLVM dialect.
@@ -443,9 +443,6 @@ class ConvertToLLVMPattern : public ConversionPattern {
 protected:
   /// Reference to the type converter, with potential extensions.
   LLVMTypeConverter &typeConverter;
-
-  /// Reference to the llvm lowering options.
-  const LowerToLLVMOptions &options;
 };
 
 /// Utility class for operation conversions targeting the LLVM dialect that
@@ -454,11 +451,10 @@ template <typename OpTy>
 class ConvertOpToLLVMPattern : public ConvertToLLVMPattern {
 public:
   ConvertOpToLLVMPattern(LLVMTypeConverter &typeConverter,
-                         const LowerToLLVMOptions &options,
                          PatternBenefit benefit = 1)
       : ConvertToLLVMPattern(OpTy::getOperationName(),
                              &typeConverter.getContext(), typeConverter,
-                             options, benefit) {}
+                             benefit) {}
 };
 
 namespace LLVM {
diff --git a/mlir/include/mlir/Conversion/StandardToLLVM/ConvertStandardToLLVMPass.h b/mlir/include/mlir/Conversion/StandardToLLVM/ConvertStandardToLLVMPass.h
index 1f76b929679ef..3d8312c6e7f51 100644
--- a/mlir/include/mlir/Conversion/StandardToLLVM/ConvertStandardToLLVMPass.h
+++ b/mlir/include/mlir/Conversion/StandardToLLVM/ConvertStandardToLLVMPass.h
@@ -42,31 +42,26 @@ struct LowerToLLVMOptions {
 /// Standard dialect to the LLVM dialect, excluding non-memory-related
 /// operations and FuncOp.
 void populateStdToLLVMMemoryConversionPatterns(
-    LLVMTypeConverter &converter, OwningRewritePatternList &patterns,
-    const LowerToLLVMOptions &options);
+    LLVMTypeConverter &converter, OwningRewritePatternList &patterns);
 
 /// Collect a set of patterns to convert from the Standard dialect to the LLVM
 /// dialect, excluding the memory-related operations.
 void populateStdToLLVMNonMemoryConversionPatterns(
-    LLVMTypeConverter &converter, OwningRewritePatternList &patterns,
-    const LowerToLLVMOptions &options);
+    LLVMTypeConverter &converter, OwningRewritePatternList &patterns);
 
 /// Collect the default pattern to convert a FuncOp to the LLVM dialect. If
 /// `emitCWrappers` is set, the pattern will also produce functions
 /// that pass memref descriptors by pointer-to-structure in addition to the
 /// default unpacked form.
 void populateStdToLLVMFuncOpConversionPattern(
-    LLVMTypeConverter &converter, OwningRewritePatternList &patterns,
-    const LowerToLLVMOptions &options);
+    LLVMTypeConverter &converter, OwningRewritePatternList &patterns);
 
 /// Collect the patterns to convert from the Standard dialect to LLVM. The
 /// conversion patterns capture the LLVMTypeConverter and the LowerToLLVMOptions
 /// by reference meaning the references have to remain alive during the entire
 /// pattern lifetime.
-void populateStdToLLVMConversionPatterns(
-    LLVMTypeConverter &converter, OwningRewritePatternList &patterns,
-    const LowerToLLVMOptions &options =
-        LowerToLLVMOptions::getDefaultOptions());
+void populateStdToLLVMConversionPatterns(LLVMTypeConverter &converter,
+                                         OwningRewritePatternList &patterns);
 
 /// Creates a pass to convert the Standard dialect into the LLVMIR dialect.
 /// stdlib malloc/free is used by default for allocating memrefs allocated with
diff --git a/mlir/lib/Conversion/StandardToLLVM/StandardToLLVM.cpp b/mlir/lib/Conversion/StandardToLLVM/StandardToLLVM.cpp
index 0d154796f049c..143f7fa3aabd0 100644
--- a/mlir/lib/Conversion/StandardToLLVM/StandardToLLVM.cpp
+++ b/mlir/lib/Conversion/StandardToLLVM/StandardToLLVM.cpp
@@ -424,10 +424,9 @@ Type LLVMTypeConverter::convertVectorType(VectorType type) {
 ConvertToLLVMPattern::ConvertToLLVMPattern(StringRef rootOpName,
                                            MLIRContext *context,
                                            LLVMTypeConverter &typeConverter,
-                                           const LowerToLLVMOptions &options,
                                            PatternBenefit benefit)
     : ConversionPattern(rootOpName, benefit, typeConverter, context),
-      typeConverter(typeConverter), options(options) {}
+      typeConverter(typeConverter) {}
 
 /*============================================================================*/
 /* StructBuilder implementation                                               */
@@ -1124,10 +1123,8 @@ struct FuncOpConversionBase : public ConvertOpToLLVMPattern<FuncOp> {
 /// information.
 static constexpr StringRef kEmitIfaceAttrName = "llvm.emit_c_interface";
 struct FuncOpConversion : public FuncOpConversionBase {
-  FuncOpConversion(LLVMTypeConverter &converter,
-                   const LowerToLLVMOptions &options)
-      : FuncOpConversionBase(converter, options) {}
-  using ConvertOpToLLVMPattern<FuncOp>::options;
+  FuncOpConversion(LLVMTypeConverter &converter)
+      : FuncOpConversionBase(converter) {}
 
   LogicalResult
   matchAndRewrite(Operation *op, ArrayRef<Value> operands,
@@ -1138,7 +1135,7 @@ struct FuncOpConversion : public FuncOpConversionBase {
     if (!newFuncOp)
       return failure();
 
-    if (options.emitCWrappers ||
+    if (typeConverter.getOptions().emitCWrappers ||
         funcOp.getAttrOfType<UnitAttr>(kEmitIfaceAttrName)) {
       if (newFuncOp.isExternal())
         wrapExternalFunction(rewriter, op->getLoc(), typeConverter, funcOp,
@@ -1652,11 +1649,9 @@ struct AllocLikeOpLowering : public ConvertOpToLLVMPattern<AllocLikeOp> {
   using ConvertOpToLLVMPattern<AllocLikeOp>::getIndexType;
   using ConvertOpToLLVMPattern<AllocLikeOp>::typeConverter;
   using ConvertOpToLLVMPattern<AllocLikeOp>::getVoidPtrType;
-  using ConvertOpToLLVMPattern<AllocLikeOp>::options;
 
-  explicit AllocLikeOpLowering(LLVMTypeConverter &converter,
-                               const LowerToLLVMOptions &options)
-      : ConvertOpToLLVMPattern<AllocLikeOp>(converter, options) {}
+  explicit AllocLikeOpLowering(LLVMTypeConverter &converter)
+      : ConvertOpToLLVMPattern<AllocLikeOp>(converter) {}
 
   LogicalResult match(Operation *op) const override {
     MemRefType memRefType = cast<AllocLikeOp>(op).getType();
@@ -1823,7 +1818,7 @@ struct AllocLikeOpLowering : public ConvertOpToLLVMPattern<AllocLikeOp> {
   /// allocation size to be a multiple of alignment,
   Optional<int64_t> getAllocationAlignment(AllocOp allocOp) const {
     // No alignment can be used for the 'malloc' call itself.
-    if (!options.useAlignedAlloc)
+    if (!typeConverter.getOptions().useAlignedAlloc)
       return None;
 
     if (allocOp.alignment())
@@ -2002,9 +1997,8 @@ struct AllocLikeOpLowering : public ConvertOpToLLVMPattern<AllocLikeOp> {
 };
 
 struct AllocOpLowering : public AllocLikeOpLowering<AllocOp> {
-  explicit AllocOpLowering(LLVMTypeConverter &converter,
-                           const LowerToLLVMOptions &options)
-      : AllocLikeOpLowering<AllocOp>(converter, options) {}
+  explicit AllocOpLowering(LLVMTypeConverter &converter)
+      : AllocLikeOpLowering<AllocOp>(converter) {}
 };
 
 using AllocaOpLowering = AllocLikeOpLowering<AllocaOp>;
@@ -2174,9 +2168,8 @@ struct CallIndirectOpLowering : public CallOpInterfaceLowering<CallIndirectOp> {
 struct DeallocOpLowering : public ConvertOpToLLVMPattern<DeallocOp> {
   using ConvertOpToLLVMPattern<DeallocOp>::ConvertOpToLLVMPattern;
 
-  explicit DeallocOpLowering(LLVMTypeConverter &converter,
-                             const LowerToLLVMOptions &options)
-      : ConvertOpToLLVMPattern<DeallocOp>(converter, options) {}
+  explicit DeallocOpLowering(LLVMTypeConverter &converter)
+      : ConvertOpToLLVMPattern<DeallocOp>(converter) {}
 
   LogicalResult
   matchAndRewrite(Operation *op, ArrayRef<Value> operands,
@@ -3200,8 +3193,7 @@ struct GenericAtomicRMWOpLowering
 
 /// Collect a set of patterns to convert from the Standard dialect to LLVM.
 void mlir::populateStdToLLVMNonMemoryConversionPatterns(
-    LLVMTypeConverter &converter, OwningRewritePatternList &patterns,
-    const LowerToLLVMOptions &options) {
+    LLVMTypeConverter &converter, OwningRewritePatternList &patterns) {
   // FIXME: this should be tablegen'ed
   // clang-format off
   patterns.insert<
@@ -3265,13 +3257,12 @@ void mlir::populateStdToLLVMNonMemoryConversionPatterns(
       UnsignedRemIOpLowering,
       UnsignedShiftRightOpLowering,
       XOrOpLowering,
-      ZeroExtendIOpLowering>(converter, options);
+      ZeroExtendIOpLowering>(converter);
   // clang-format on
 }
 
 void mlir::populateStdToLLVMMemoryConversionPatterns(
-    LLVMTypeConverter &converter, OwningRewritePatternList &patterns,
-    const LowerToLLVMOptions &options) {
+    LLVMTypeConverter &converter, OwningRewritePatternList &patterns) {
   // clang-format off
   patterns.insert<
       AssumeAlignmentOpLowering,
@@ -3282,25 +3273,23 @@ void mlir::populateStdToLLVMMemoryConversionPatterns(
       StoreOpLowering,
       SubViewOpLowering,
       ViewOpLowering,
-      AllocOpLowering>(converter, options);
+      AllocOpLowering>(converter);
   // clang-format on
 }
 
 void mlir::populateStdToLLVMFuncOpConversionPattern(
-    LLVMTypeConverter &converter, OwningRewritePatternList &patterns,
-    const LowerToLLVMOptions &options) {
-  if (options.useBarePtrCallConv)
-    patterns.insert<BarePtrFuncOpConversion>(converter, options);
+    LLVMTypeConverter &converter, OwningRewritePatternList &patterns) {
+  if (converter.getOptions().useBarePtrCallConv)
+    patterns.insert<BarePtrFuncOpConversion>(converter);
   else
-    patterns.insert<FuncOpConversion>(converter, options);
+    patterns.insert<FuncOpConversion>(converter);
 }
 
 void mlir::populateStdToLLVMConversionPatterns(
-    LLVMTypeConverter &converter, OwningRewritePatternList &patterns,
-    const LowerToLLVMOptions &options) {
-  populateStdToLLVMFuncOpConversionPattern(converter, patterns, options);
-  populateStdToLLVMNonMemoryConversionPatterns(converter, patterns, options);
-  populateStdToLLVMMemoryConversionPatterns(converter, patterns, options);
+    LLVMTypeConverter &converter, OwningRewritePatternList &patterns) {
+  populateStdToLLVMFuncOpConversionPattern(converter, patterns);
+  populateStdToLLVMNonMemoryConversionPatterns(converter, patterns);
+  populateStdToLLVMMemoryConversionPatterns(converter, patterns);
 }
 
 // Create an LLVM IR structure type if there is more than one result.
@@ -3395,7 +3384,7 @@ struct LLVMLoweringPass : public ConvertStandardToLLVMBase<LLVMLoweringPass> {
     LLVMTypeConverter typeConverter(&getContext(), options);
 
     OwningRewritePatternList patterns;
-    populateStdToLLVMConversionPatterns(typeConverter, patterns, options);
+    populateStdToLLVMConversionPatterns(typeConverter, patterns);
 
     LLVMConversionTarget target(getContext());
     if (failed(applyPartialConversion(m, target, patterns)))

From 85defd23aa09f2fa8bf48fed7a04ed4a5851cd0c Mon Sep 17 00:00:00 2001
From: Stephan Herhut <herhut@google.com>
Date: Thu, 30 Jul 2020 14:38:12 +0200
Subject: [PATCH 0612/1035] [mlir][shape] Use memref of index in shape lowering

Now that we can have a memref of index type, we no longer need to materialize shapes in i64 and then index_cast.

Differential Revision: https://reviews.llvm.org/D84938
---
 mlir/lib/Conversion/ShapeToSCF/ShapeToSCF.cpp     | 11 ++++-------
 mlir/test/Conversion/ShapeToSCF/shape-to-scf.mlir |  8 +++-----
 2 files changed, 7 insertions(+), 12 deletions(-)

diff --git a/mlir/lib/Conversion/ShapeToSCF/ShapeToSCF.cpp b/mlir/lib/Conversion/ShapeToSCF/ShapeToSCF.cpp
index 0101c9e7fdc01..a6c667f5641c3 100644
--- a/mlir/lib/Conversion/ShapeToSCF/ShapeToSCF.cpp
+++ b/mlir/lib/Conversion/ShapeToSCF/ShapeToSCF.cpp
@@ -186,8 +186,8 @@ ShapeOfOpConverter::matchAndRewrite(ShapeOfOp op, ArrayRef<Value> operands,
   // Allocate stack memory.
   auto loc = op.getLoc();
   Value rank = rewriter.create<mlir::RankOp>(loc, arg);
-  Type i64Ty = rewriter.getI64Type();
-  Type memTy = MemRefType::get({ShapedType::kDynamicSize}, i64Ty);
+  Type indexTy = rewriter.getIndexType();
+  Type memTy = MemRefType::get({ShapedType::kDynamicSize}, indexTy);
   Value mem = rewriter.create<AllocaOp>(loc, memTy, ValueRange{rank});
 
   // Copy shape extents to stack-allocated memory.
@@ -197,15 +197,12 @@ ShapeOfOpConverter::matchAndRewrite(ShapeOfOp op, ArrayRef<Value> operands,
       loc, zero, rank, one, llvm::None,
       [&](OpBuilder &b, Location loc, Value iv, ValueRange args) {
         Value dim = rewriter.create<DimOp>(loc, arg, iv);
-        Value dimInt = rewriter.create<IndexCastOp>(loc, dim, i64Ty);
-        rewriter.create<StoreOp>(loc, dimInt, mem, ValueRange{iv});
+        rewriter.create<StoreOp>(loc, dim, mem, ValueRange{iv});
         rewriter.create<scf::YieldOp>(loc);
       });
 
   // Load extents to tensor value.
-  Value extentTensorInt = rewriter.create<TensorLoadOp>(loc, mem);
-  rewriter.replaceOpWithNewOp<IndexCastOp>(op.getOperation(), extentTensorInt,
-                                           op.getType());
+  rewriter.replaceOpWithNewOp<TensorLoadOp>(op.getOperation(), mem);
   return success();
 }
 
diff --git a/mlir/test/Conversion/ShapeToSCF/shape-to-scf.mlir b/mlir/test/Conversion/ShapeToSCF/shape-to-scf.mlir
index 97d2bce5a0948..768a627208b8e 100644
--- a/mlir/test/Conversion/ShapeToSCF/shape-to-scf.mlir
+++ b/mlir/test/Conversion/ShapeToSCF/shape-to-scf.mlir
@@ -40,16 +40,14 @@ func @shape_of(%arg : tensor<*xf32>) {
 // CHECK-SAME: (%[[ARG:.*]]: tensor<*xf32>)
 func @shape_of_unranked(%arg : tensor<*xf32>) {
   // CHECK: %[[RANK:.*]] = rank %[[ARG]] : tensor<*xf32>
-  // CHECK: %[[SHAPE_MEM:.*]] = alloca(%[[RANK]]) : memref<?xi64>
+  // CHECK: %[[SHAPE_MEM:.*]] = alloca(%[[RANK]]) : memref<?xindex>
   // CHECK: %[[C0:.*]] = constant 0 : index
   // CHECK: %[[C1:.*]] = constant 1 : index
   // CHECK: scf.for %[[I:.*]] = %[[C0]] to %[[RANK]] step %[[C1]] {
   // CHECK:   %[[DIM:.]] = dim %[[ARG]], %[[I]] : tensor<*xf32>
-  // CHECK:   %[[DIM_INT:.*]] = index_cast %[[DIM]] : index to i64
-  // CHECK:   store %[[DIM_INT]], %[[SHAPE_MEM]][%[[I]]] : memref<?xi64>
+  // CHECK:   store %[[DIM]], %[[SHAPE_MEM]][%[[I]]] : memref<?xindex>
   // CHECK: }
-  // CHECK: %[[SHAPE_INT:.*]] = tensor_load %[[SHAPE_MEM]] : memref<?xi64>
-  // CHECK: %[[SHAPE:.*]] = index_cast %[[SHAPE_INT]] : tensor<?xi64> to tensor<?xindex>
+  // CHECK: %[[SHAPE:.*]] = tensor_load %[[SHAPE_MEM]] : memref<?xindex>
   %shape = shape.shape_of %arg : tensor<*xf32> -> tensor<?xindex>
   return
 }

From 9be1bdc24075b8a99f251cee5416615574afa295 Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne@apple.com>
Date: Thu, 30 Jul 2020 09:26:34 -0400
Subject: [PATCH 0613/1035] [libc++] Add XFAIL for <float.h> and <cfloat> tests
 on older Clangs

---
 libcxx/test/std/depr/depr.c.headers/float_h.pass.cpp         | 5 ++++-
 .../language.support/support.limits/c.limits/cfloat.pass.cpp | 5 ++++-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/libcxx/test/std/depr/depr.c.headers/float_h.pass.cpp b/libcxx/test/std/depr/depr.c.headers/float_h.pass.cpp
index 2b7edcadc2176..ae061ed851590 100644
--- a/libcxx/test/std/depr/depr.c.headers/float_h.pass.cpp
+++ b/libcxx/test/std/depr/depr.c.headers/float_h.pass.cpp
@@ -6,7 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
- // test <float.h>
+// Before Clang 9.0, <float.h> does not define FLT_HAS_SUBNORM & friends in C++.
+// XFAIL: clang-4, clang-5, clang-6, clang-7, clang-8
+
+// test <float.h>
 
 #include <float.h>
 
diff --git a/libcxx/test/std/language.support/support.limits/c.limits/cfloat.pass.cpp b/libcxx/test/std/language.support/support.limits/c.limits/cfloat.pass.cpp
index 99fe6a37243c5..1406908b52464 100644
--- a/libcxx/test/std/language.support/support.limits/c.limits/cfloat.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/c.limits/cfloat.pass.cpp
@@ -6,7 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
- // test cfloat
+// Before Clang 9.0, <float.h> does not define FLT_HAS_SUBNORM & friends in C++.
+// XFAIL: clang-4, clang-5, clang-6, clang-7, clang-8
+
+// test cfloat
 
 #include <cfloat>
 

From 1da0c47fa2e6e3316294e146ce7065dd49b2d5a2 Mon Sep 17 00:00:00 2001
From: David Green <david.green@arm.com>
Date: Thu, 30 Jul 2020 14:28:08 +0100
Subject: [PATCH 0614/1035] [LoopVectorizer] Don't create unused block masks
 for reductions. NFC

This removes some unneeded block masks when we don't have any
reductions. It should not have any effect on codegen as the values
created are dead anyway.

Differential Revision: https://reviews.llvm.org/D81415
---
 .../Transforms/Vectorize/LoopVectorize.cpp    |  2 +-
 .../Transforms/LoopVectorize/icmp-uniforms.ll |  3 +--
 .../tail-folding-vectorization-factor-1.ll    | 24 +++++--------------
 3 files changed, 8 insertions(+), 21 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 4b519514d543c..5fffcc8cf0f3a 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7459,7 +7459,7 @@ VPlanPtr LoopVectorizationPlanner::buildVPlanWithVPRecipes(
 
   // Finally, if tail is folded by masking, introduce selects between the phi
   // and the live-out instruction of each reduction, at the end of the latch.
-  if (CM.foldTailByMasking()) {
+  if (CM.foldTailByMasking() && !Legal->getReductionVars().empty()) {
     Builder.setInsertPoint(VPBB);
     auto *Cond = RecipeBuilder.createBlockInMask(OrigLoop->getHeader(), Plan);
     for (auto &Reduction : Legal->getReductionVars()) {
diff --git a/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll b/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll
index 3636446e5c93a..a5b57432ff6e5 100644
--- a/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll
+++ b/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll
@@ -39,8 +39,7 @@ for.end:
 ; CHECK-NEXT:    "loop:\n" +
 ; CHECK-NEXT:      "WIDEN-INDUCTION %iv = phi 0, %iv.next\l" +
 ; CHECK-NEXT:      "WIDEN\l""  %cond0 = icmp %iv, 13\l" +
-; CHECK-NEXT:      "WIDEN-SELECT%s = select %cond0, 10, 20\l" +
-; CHECK-NEXT:      "EMIT vp<%1> = icmp ule ir<%iv> vp<%0>\l"
+; CHECK-NEXT:      "WIDEN-SELECT%s = select %cond0, 10, 20\l"
 ; CHECK-NEXT:  ]
 define void @test() {
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/tail-folding-vectorization-factor-1.ll b/llvm/test/Transforms/LoopVectorize/tail-folding-vectorization-factor-1.ll
index 973d7013837a6..bb901ed70a827 100644
--- a/llvm/test/Transforms/LoopVectorize/tail-folding-vectorization-factor-1.ll
+++ b/llvm/test/Transforms/LoopVectorize/tail-folding-vectorization-factor-1.ll
@@ -10,7 +10,7 @@
 ; CHECK-REMARKS-NOT:  remark: {{.*}} vectorized loop
 
 define void @VF1-VPlanExe() {
-; CHECK-LABEL: @VF1-VPlanExe
+; CHECK-LABEL: @VF1-VPlanExe(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
@@ -21,13 +21,9 @@ define void @VF1-VPlanExe() {
 ; CHECK-NEXT:    [[INDUCTION1:%.*]] = add i64 [[INDEX]], 1
 ; CHECK-NEXT:    [[INDUCTION2:%.*]] = add i64 [[INDEX]], 2
 ; CHECK-NEXT:    [[INDUCTION3:%.*]] = add i64 [[INDEX]], 3
-; CHECK-NEXT:    [[TMP0:%.*]] = icmp ule i64 [[INDUCTION]], 14
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp ule i64 [[INDUCTION1]], 14
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp ule i64 [[INDUCTION2]], 14
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp ule i64 [[INDUCTION3]], 14
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
-; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16
-; CHECK-NEXT:    br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !0
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16
+; CHECK-NEXT:    br i1 [[TMP0]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !0
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
@@ -55,7 +51,7 @@ for.body:
 }
 
 define void @VF1-VPWidenCanonicalIVRecipeExe(double* %ptr1) {
-; CHECK-LABEL: @VF1-VPWidenCanonicalIVRecipeExe
+; CHECK-LABEL: @VF1-VPWidenCanonicalIVRecipeExe(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[PTR2:%.*]] = getelementptr inbounds double, double* [[PTR1:%.*]], i64 15
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
@@ -72,17 +68,9 @@ define void @VF1-VPWidenCanonicalIVRecipeExe(double* %ptr1) {
 ; CHECK-NEXT:    [[NEXT_GEP2:%.*]] = getelementptr double, double* [[PTR1]], i64 [[TMP2]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[INDEX]], 3
 ; CHECK-NEXT:    [[NEXT_GEP3:%.*]] = getelementptr double, double* [[PTR1]], i64 [[TMP3]]
-; CHECK-NEXT:    [[VEC_IV:%.*]] = add i64 [[INDEX]], 0
-; CHECK-NEXT:    [[VEC_IV4:%.*]] = add i64 [[INDEX]], 1
-; CHECK-NEXT:    [[VEC_IV5:%.*]] = add i64 [[INDEX]], 2
-; CHECK-NEXT:    [[VEC_IV6:%.*]] = add i64 [[INDEX]], 3
-; CHECK-NEXT:    [[TMP4:%.*]] = icmp ule i64 [[VEC_IV]], 14
-; CHECK-NEXT:    [[TMP5:%.*]] = icmp ule i64 [[VEC_IV4]], 14
-; CHECK-NEXT:    [[TMP6:%.*]] = icmp ule i64 [[VEC_IV5]], 14
-; CHECK-NEXT:    [[TMP7:%.*]] = icmp ule i64 [[VEC_IV6]], 14
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
-; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16
-; CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !3
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16
+; CHECK-NEXT:    br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !3
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:

From f7237ee74fa629a4e79abf4bd3bc2e4d2b5d8a0a Mon Sep 17 00:00:00 2001
From: Sanjay Patel <spatel@rotateright.com>
Date: Thu, 30 Jul 2020 08:37:02 -0400
Subject: [PATCH 0615/1035] [ConstantFolding] add tests for abs intrinsic; NFC

---
 llvm/test/Analysis/ConstantFolding/abs.ll | 50 +++++++++++++++++++++++
 1 file changed, 50 insertions(+)
 create mode 100644 llvm/test/Analysis/ConstantFolding/abs.ll

diff --git a/llvm/test/Analysis/ConstantFolding/abs.ll b/llvm/test/Analysis/ConstantFolding/abs.ll
new file mode 100644
index 0000000000000..24171b1d4c6e4
--- /dev/null
+++ b/llvm/test/Analysis/ConstantFolding/abs.ll
@@ -0,0 +1,50 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -constprop -S | FileCheck %s
+
+declare i8 @llvm.abs.i8(i8, i1)
+declare <8 x i8> @llvm.abs.v8i8(<8 x i8>, i1)
+
+define i8 @undef_val_min_poison() {
+; CHECK-LABEL: @undef_val_min_poison(
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.abs.i8(i8 undef, i1 true)
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %r = call i8 @llvm.abs.i8(i8 undef, i1 true)
+  ret i8 %r
+}
+
+define i8 @undef_val_min_not_poison() {
+; CHECK-LABEL: @undef_val_min_not_poison(
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.abs.i8(i8 undef, i1 false)
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %r = call i8 @llvm.abs.i8(i8 undef, i1 false)
+  ret i8 %r
+}
+
+define i8 @min_val_min_poison() {
+; CHECK-LABEL: @min_val_min_poison(
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.abs.i8(i8 -128, i1 true)
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %r = call i8 @llvm.abs.i8(i8 -128, i1 true)
+  ret i8 %r
+}
+
+define i8 @min_val_min_not_poison() {
+; CHECK-LABEL: @min_val_min_not_poison(
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.abs.i8(i8 -128, i1 false)
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %r = call i8 @llvm.abs.i8(i8 -128, i1 false)
+  ret i8 %r
+}
+
+define <8 x i8> @vec_const() {
+; CHECK-LABEL: @vec_const(
+; CHECK-NEXT:    [[R:%.*]] = call <8 x i8> @llvm.abs.v8i8(<8 x i8> <i8 -127, i8 -126, i8 -42, i8 -1, i8 0, i8 1, i8 42, i8 127>, i1 true)
+; CHECK-NEXT:    ret <8 x i8> [[R]]
+;
+  %r = call <8 x i8> @llvm.abs.v8i8(<8 x i8> <i8 -127, i8 -126, i8 -42, i8 -1, i8 0, i8 1, i8 42, i8 127>, i1 1)
+  ret <8 x i8> %r
+}

From 04dc9691ebef740cc8dc85e9a5d7a3e1ed595765 Mon Sep 17 00:00:00 2001
From: jasonliu <jasonliu.development@gmail.com>
Date: Wed, 29 Jul 2020 15:06:04 +0000
Subject: [PATCH 0616/1035] [XCOFF][AIX] Enable -ffunction-sections

Summary:
This patch implements -ffunction-sections on AIX.
This patch focuses on assembly generation.
Follow-on patch needs to handle:
1. -ffunction-sections implication for jump table.
2. Object file generation path and associated testing.

Differential Revision: https://reviews.llvm.org/D83875
---
 .../CodeGen/TargetLoweringObjectFileImpl.cpp  | 27 +++++-
 llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp     |  6 +-
 .../CodeGen/PowerPC/aix-xcoff-funcsect.ll     | 95 +++++++++++++++++++
 3 files changed, 124 insertions(+), 4 deletions(-)
 create mode 100644 llvm/test/CodeGen/PowerPC/aix-xcoff-funcsect.ll

diff --git a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
index bf3b89e006405..48599199626ce 100644
--- a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
@@ -2027,8 +2027,8 @@ MCSection *TargetLoweringObjectFileXCOFF::getSectionForExternalReference(
 
 MCSection *TargetLoweringObjectFileXCOFF::SelectSectionForGlobal(
     const GlobalObject *GO, SectionKind Kind, const TargetMachine &TM) const {
-  assert(!TM.getFunctionSections() && !TM.getDataSections() &&
-         "XCOFF unique sections not yet implemented.");
+  assert(!TM.getDataSections() &&
+         "XCOFF unique data sections not yet implemented.");
 
   // Common symbols go into a csect with matching name which will get mapped
   // into the .bss section.
@@ -2057,8 +2057,13 @@ MCSection *TargetLoweringObjectFileXCOFF::SelectSectionForGlobal(
         Kind, /* BeginSymbolName */ nullptr);
   }
 
-  if (Kind.isText())
+  if (Kind.isText()) {
+    if (TM.getFunctionSections()) {
+      return cast<MCSymbolXCOFF>(getFunctionEntryPointSymbol(GO, TM))
+          ->getRepresentedCsect();
+    }
     return TextSection;
+  }
 
   if (Kind.isData() || Kind.isReadOnlyWithRel())
     // TODO: We may put this under option control, because user may want to
@@ -2161,6 +2166,22 @@ MCSymbol *TargetLoweringObjectFileXCOFF::getFunctionEntryPointSymbol(
   SmallString<128> NameStr;
   NameStr.push_back('.');
   getNameWithPrefix(NameStr, Func, TM);
+
+  // When -function-sections is enabled, it's not necessary to emit
+  // function entry point label any more. We will use function entry
+  // point csect instead. For function delcarations, it's okay to continue
+  // using label semantic because undefined symbols gets treated as csect with
+  // XTY_ER property anyway.
+  if (TM.getFunctionSections() && !Func->isDeclaration() &&
+      isa<Function>(Func)) {
+    XCOFF::StorageClass SC =
+        TargetLoweringObjectFileXCOFF::getStorageClassForGlobal(Func);
+    return cast<MCSectionXCOFF>(getContext().getXCOFFSection(
+                                    NameStr, XCOFF::XMC_PR, XCOFF::XTY_SD, SC,
+                                    SectionKind::getText()))
+        ->getQualNameSymbol();
+  }
+
   return getContext().getOrCreateSymbol(NameStr);
 }
 
diff --git a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
index 5affddd8d147a..10962df7b872c 100644
--- a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
+++ b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
@@ -1779,7 +1779,11 @@ void PPCAIXAsmPrinter::emitFunctionDescriptor() {
 }
 
 void PPCAIXAsmPrinter::emitFunctionEntryLabel() {
-  PPCAsmPrinter::emitFunctionEntryLabel();
+  // It's not necessary to emit the label when we have individual
+  // function in its own csect.
+  if (!TM.getFunctionSections())
+    PPCAsmPrinter::emitFunctionEntryLabel();
+
   // Emit aliasing label for function entry point label.
   llvm::for_each(
       GOAliasMap[&MF->getFunction()], [this](const GlobalAlias *Alias) {
diff --git a/llvm/test/CodeGen/PowerPC/aix-xcoff-funcsect.ll b/llvm/test/CodeGen/PowerPC/aix-xcoff-funcsect.ll
new file mode 100644
index 0000000000000..acc521e028859
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/aix-xcoff-funcsect.ll
@@ -0,0 +1,95 @@
+; RUN: llc -verify-machineinstrs -mtriple powerpc-ibm-aix-xcoff -mcpu=pwr4 \
+; RUN:     -mattr=-altivec -function-sections < %s | \
+; RUN:   FileCheck --check-prefix=ASM %s
+; RUN: llc -verify-machineinstrs -mtriple powerpc64-ibm-aix-xcoff -mcpu=pwr4 \
+; RUN:     -mattr=-altivec -function-sections < %s | \
+; RUN:   FileCheck --check-prefix=ASM %s
+
+@alias_foo = alias void (...), bitcast (void ()* @foo to void (...)*)
+
+define void @foo() {
+entry:
+  ret void
+}
+
+define hidden void @hidden_foo() {
+entry:
+  ret void
+}
+
+define void @bar() {
+entry:
+  call void @foo()
+  call void @static_overalign_foo()
+  call void bitcast (void (...)* @alias_foo to void ()*)()
+  call void bitcast (void (...)* @extern_foo to void ()*)()
+  call void @hidden_foo()
+  ret void
+}
+
+declare void @extern_foo(...)
+
+define internal void @static_overalign_foo() align 64 {
+entry:
+  ret void
+}
+
+; ASM:        .csect .foo[PR],2
+; ASM-NEXT:  	.globl	foo[DS]                         # -- Begin function foo
+; ASM-NEXT:  	.globl	.foo[PR]
+; ASM-NEXT:  	.align	4
+; ASM-NEXT:  	.csect foo[DS]
+; ASM-NEXT:  alias_foo:                                # @foo
+; ASM-NEXT:  	.vbyte	{{[0-9]+}}, .foo[PR]
+; ASM-NEXT:  	.vbyte	{{[0-9]+}}, TOC[TC0]
+; ASM-NEXT:  	.vbyte	{{[0-9]+}}, 0
+; ASM-NEXT:  	.csect .foo[PR],2
+; ASM-NEXT:  .alias_foo:
+; ASM-NEXT:  # %bb.0:                                # %entry
+; ASM-NEXT:  	blr
+; ASM:        .csect .hidden_foo[PR],2
+; ASM-NEXT:  	.globl	hidden_foo[DS],hidden           # -- Begin function hidden_foo
+; ASM-NEXT:  	.globl	.hidden_foo[PR],hidden
+; ASM-NEXT:  	.align	4
+; ASM-NEXT:  	.csect hidden_foo[DS]
+; ASM-NEXT:  	.vbyte	{{[0-9]+}}, .hidden_foo[PR]              # @hidden_foo
+; ASM-NEXT:  	.vbyte	{{[0-9]+}}, TOC[TC0]
+; ASM-NEXT:  	.vbyte	{{[0-9]+}}, 0
+; ASM-NEXT:  	.csect .hidden_foo[PR]
+; ASM-NEXT:  # %bb.0:                                # %entry
+; ASM-NEXT:  	blr
+; ASM:        .csect .bar[PR],2
+; ASM-NEXT:  	.globl	bar[DS]                         # -- Begin function bar
+; ASM-NEXT:  	.globl	.bar[PR]
+; ASM-NEXT:  	.align	4
+; ASM-NEXT:  	.csect bar[DS]
+; ASM-NEXT:  	.vbyte	{{[0-9]+}}, .bar[PR]                     # @bar
+; ASM-NEXT:  	.vbyte	{{[0-9]+}}, TOC[TC0]
+; ASM-NEXT:  	.vbyte	{{[0-9]+}}, 0
+; ASM-NEXT:  	.csect .bar[PR],2
+; ASM-NEXT:  # %bb.0:                                # %entry
+; ASM:        bl .foo[PR]
+; ASM-NEXT:  	nop
+; ASM-NEXT:  	bl .static_overalign_foo[PR]
+; ASM-NEXT:  	nop
+; ASM-NEXT:  	bl .alias_foo
+; ASM-NEXT:  	nop
+; ASM-NEXT:  	bl .extern_foo
+; ASM-NEXT:  	nop
+; ASM-NEXT:  	bl .hidden_foo[PR]
+; ASM-NEXT:  	nop
+; ASM:        .csect .static_overalign_foo[PR],6
+; ASM-NEXT:  	.lglobl	static_overalign_foo[DS]                  # -- Begin function static_overalign_foo
+; ASM-NEXT:  	.lglobl	.static_overalign_foo[PR]
+; ASM-NEXT:  	.align	6
+; ASM-NEXT:  	.csect static_overalign_foo[DS]
+; ASM-NEXT:  	.vbyte	{{[0-9]+}}, .static_overalign_foo[PR]              # @static_overalign_foo
+; ASM-NEXT:  	.vbyte	{{[0-9]+}}, TOC[TC0]
+; ASM-NEXT:  	.vbyte	{{[0-9]+}}, 0
+; ASM-NEXT:  	.csect .static_overalign_foo[PR],6
+; ASM-NEXT:  # %bb.0:                                # %entry
+; ASM-NEXT:  	blr
+; ASM:        .extern	.extern_foo
+; ASM-NEXT:  	.extern	extern_foo[DS]
+; ASM-NEXT:  	.globl	alias_foo
+; ASM-NEXT:  	.globl	.alias_foo

From 142d0d3ed8e07aca2476bc4ecc1a12d15577a84a Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev@hotmail.com>
Date: Fri, 24 Jul 2020 17:46:19 -0400
Subject: [PATCH 0617/1035] [OPENMP]Fix PR46824: Global declare target pointer
 cannot be accessed in target region.

Need to map the base pointer for all directives, not only target
data-based ones.
The base pointer is mapped for array sections, array subscript, array
shaping and other array-like constructs with the base pointer. Also,
codegen for use_device_ptr clause was modified to correctly handle
mapping combination of array like constructs + use_device_ptr clause.
The data for use_device_ptr clause is emitted as the last records in the
data mapping array.
It applies only for global pointers.

Differential Revision: https://reviews.llvm.org/D84767
---
 clang/lib/CodeGen/CGOpenMPRuntime.cpp         |  78 ++++--
 clang/test/OpenMP/target_data_codegen.cpp     |  18 +-
 .../target_data_use_device_ptr_codegen.cpp    |  48 ++--
 clang/test/OpenMP/target_map_codegen.cpp      |  21 +-
 clang/test/OpenMP/target_update_codegen.cpp   | 242 ++++++++----------
 openmp/libomptarget/src/omptarget.cpp         |   9 +-
 .../test/env/base_ptr_ref_count.c             |  51 ++++
 7 files changed, 255 insertions(+), 212 deletions(-)
 create mode 100644 openmp/libomptarget/test/env/base_ptr_ref_count.c

diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
index b8e33948c21ca..6c5c54951921e 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -7392,10 +7392,9 @@ class MappableExprsHandler {
     // &p, &p, sizeof(float*), TARGET_PARAM | TO | FROM
     //
     // map(p[1:24])
+    // &p, &p[1], 24*sizeof(float), TARGET_PARAM | TO | FROM | PTR_AND_OBJ
+    // in unified shared memory mode or for local pointers
     // p, &p[1], 24*sizeof(float), TARGET_PARAM | TO | FROM
-    // for data directives
-    // p, p, sizeof(float*), TARGET_PARAM | TO | FROM
-    // p, &p[1], 24*sizeof(float), PTR_AND_OBJ | TO | FROM
     //
     // map(s)
     // &s, &s, sizeof(S2), TARGET_PARAM | TO | FROM
@@ -7530,6 +7529,7 @@ class MappableExprsHandler {
     // Track if the map information being generated is the first for a list of
     // components.
     bool IsExpressionFirstInfo = true;
+    bool FirstPointerInComplexData = false;
     Address BP = Address::invalid();
     const Expr *AssocExpr = I->getAssociatedExpression();
     const auto *AE = dyn_cast<ArraySubscriptExpr>(AssocExpr);
@@ -7572,17 +7572,16 @@ class MappableExprsHandler {
       QualType Ty =
           I->getAssociatedDeclaration()->getType().getNonReferenceType();
       if (Ty->isAnyPointerType() && std::next(I) != CE) {
-        BP = CGF.EmitLoadOfPointer(BP, Ty->castAs<PointerType>());
-
-        // For non-data directives, we do not need to generate individual map
-        // information for the pointer, it can be associated with the combined
-        // storage.
+        // No need to generate individual map information for the pointer, it
+        // can be associated with the combined storage if shared memory mode is
+        // active or the base declaration is not global variable.
+        const auto *VD = dyn_cast<VarDecl>(I->getAssociatedDeclaration());
         if (CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory() ||
-            !CurDir.is<const OMPExecutableDirective *>() ||
-            !isOpenMPTargetDataManagementDirective(
-                CurDir.get<const OMPExecutableDirective *>()
-                    ->getDirectiveKind()))
-          ++I;
+            !VD || VD->hasLocalStorage())
+          BP = CGF.EmitLoadOfPointer(BP, Ty->castAs<PointerType>());
+        else
+          FirstPointerInComplexData = true;
+        ++I;
       }
     }
 
@@ -7617,8 +7616,19 @@ class MappableExprsHandler {
         EncounteredME = dyn_cast<MemberExpr>(I->getAssociatedExpression());
         // If we encounter a PTR_AND_OBJ entry from now on it should be marked
         // as MEMBER_OF the parent struct.
-        if (EncounteredME)
+        if (EncounteredME) {
           ShouldBeMemberOf = true;
+          // Do not emit as complex pointer if this is actually not array-like
+          // expression.
+          if (FirstPointerInComplexData) {
+            QualType Ty = std::prev(I)
+                              ->getAssociatedDeclaration()
+                              ->getType()
+                              .getNonReferenceType();
+            BP = CGF.EmitLoadOfPointer(BP, Ty->castAs<PointerType>());
+            FirstPointerInComplexData = false;
+          }
+        }
       }
 
       auto Next = std::next(I);
@@ -7760,7 +7770,8 @@ class MappableExprsHandler {
           // (there is a set of entries for each capture).
           OpenMPOffloadMappingFlags Flags =
               getMapTypeBits(MapType, MapModifiers, MotionModifiers, IsImplicit,
-                             !IsExpressionFirstInfo || RequiresReference,
+                             !IsExpressionFirstInfo || RequiresReference ||
+                                 FirstPointerInComplexData,
                              IsCaptureFirstInfo && !RequiresReference);
 
           if (!IsExpressionFirstInfo) {
@@ -7819,6 +7830,7 @@ class MappableExprsHandler {
 
         IsExpressionFirstInfo = false;
         IsCaptureFirstInfo = false;
+        FirstPointerInComplexData = false;
       }
     }
   }
@@ -8067,6 +8079,7 @@ class MappableExprsHandler {
     // emission of that entry until the whole struct has been processed.
     llvm::MapVector<const ValueDecl *, SmallVector<DeferredDevicePtrEntryTy, 4>>
         DeferredInfo;
+    MapCombinedInfoTy UseDevicePtrCombinedInfo;
 
     for (const auto *C :
          CurExecDir->getClausesOfKind<OMPUseDevicePtrClause>()) {
@@ -8086,13 +8099,24 @@ class MappableExprsHandler {
         // We potentially have map information for this declaration already.
         // Look for the first set of components that refer to it.
         if (It != Info.end()) {
-          auto CI = std::find_if(
-              It->second.begin(), It->second.end(), [VD](const MapInfo &MI) {
-                return MI.Components.back().getAssociatedDeclaration() == VD;
-              });
+          auto *CI = llvm::find_if(It->second, [VD](const MapInfo &MI) {
+            return MI.Components.back().getAssociatedDeclaration() == VD;
+          });
           // If we found a map entry, signal that the pointer has to be returned
           // and move on to the next declaration.
-          if (CI != It->second.end()) {
+          // Exclude cases where the base pointer is mapped as array subscript,
+          // array section or array shaping. The base address is passed as a
+          // pointer to base in this case and cannot be used as a base for
+          // use_device_ptr list item.
+          auto PrevCI = std::next(CI->Components.rbegin());
+          const auto *VarD = dyn_cast<VarDecl>(VD);
+          if (CI != It->second.end() &&
+              (CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory() ||
+               isa<MemberExpr>(IE) ||
+               !VD->getType().getNonReferenceType()->isPointerType() ||
+               PrevCI == CI->Components.rend() ||
+               isa<MemberExpr>(PrevCI->getAssociatedExpression()) || !VarD ||
+               VarD->hasLocalStorage())) {
             CI->ReturnDevicePointer = true;
             continue;
           }
@@ -8115,13 +8139,13 @@ class MappableExprsHandler {
         } else {
           llvm::Value *Ptr =
               CGF.EmitLoadOfScalar(CGF.EmitLValue(IE), IE->getExprLoc());
-          CombinedInfo.BasePointers.emplace_back(Ptr, VD);
-          CombinedInfo.Pointers.push_back(Ptr);
-          CombinedInfo.Sizes.push_back(
+          UseDevicePtrCombinedInfo.BasePointers.emplace_back(Ptr, VD);
+          UseDevicePtrCombinedInfo.Pointers.push_back(Ptr);
+          UseDevicePtrCombinedInfo.Sizes.push_back(
               llvm::Constant::getNullValue(CGF.Int64Ty));
-          CombinedInfo.Types.push_back(OMP_MAP_RETURN_PARAM |
-                                       OMP_MAP_TARGET_PARAM);
-          CombinedInfo.Mappers.push_back(nullptr);
+          UseDevicePtrCombinedInfo.Types.push_back(OMP_MAP_RETURN_PARAM |
+                                                   OMP_MAP_TARGET_PARAM);
+          UseDevicePtrCombinedInfo.Mappers.push_back(nullptr);
         }
       }
     }
@@ -8273,6 +8297,8 @@ class MappableExprsHandler {
       // We need to append the results of this capture to what we already have.
       CombinedInfo.append(CurInfo);
     }
+    // Append data for use_device_ptr clauses.
+    CombinedInfo.append(UseDevicePtrCombinedInfo);
   }
 
   /// Generate all the base pointers, section pointers, sizes, map types, and
diff --git a/clang/test/OpenMP/target_data_codegen.cpp b/clang/test/OpenMP/target_data_codegen.cpp
index 80e674c0b49d9..a2bb8cdf5ba88 100644
--- a/clang/test/OpenMP/target_data_codegen.cpp
+++ b/clang/test/OpenMP/target_data_codegen.cpp
@@ -555,7 +555,7 @@ struct S2 {
 
 void test_close_modifier(int arg) {
   S2 *ps;
-  // CK5: private unnamed_addr constant [6 x i64] [i64 1059, i64 32, i64 562949953422339, i64 562949953421328, i64 16, i64 1043]
+  // CK5: private unnamed_addr constant [5 x i64] [i64 1059, i64 32, i64 562949953421328, i64 16, i64 1043]
   #pragma omp target data map(close,tofrom: arg, ps->ps->ps->ps->s)
   {
     ++(arg);
@@ -634,20 +634,17 @@ void test_present_modifier(int arg) {
   // Make sure the struct picks up present even if another element of the struct
   // doesn't have present.
 
-  // CK8: private unnamed_addr constant [15 x i64]
+  // CK8: private unnamed_addr constant [11 x i64]
 
   // ps1
   //
   // PRESENT=0x1000 | TARGET_PARAM=0x20 = 0x1020
   // MEMBER_OF_1=0x1000000000000 | FROM=0x2 | TO=0x1 = 0x1000000000003
-  // MEMBER_OF_1=0x1000000000000 | PTR_AND_OBJ=0x10 | FROM=0x2 | TO=0x1 = 0x1000000000013
-  // MEMBER_OF_1=0x1000000000000 | PRESENT=0x1000 | FROM=0x2 | TO=0x1 = 0x1000000001003
   // MEMBER_OF_1=0x1000000000000 | PRESENT=0x1000 | PTR_AND_OBJ=0x10 = 0x1000000001010
   // PRESENT=0x1000 | PTR_AND_OBJ=0x10 = 0x1010
   // PRESENT=0x1000 | PTR_AND_OBJ=0x10 | FROM=0x2 | TO=0x1 = 0x1013
   //
   // CK8-SAME: {{^}} [i64 [[#0x1020]], i64 [[#0x1000000000003]],
-  // CK8-SAME: {{^}} i64 [[#0x1000000000013]], i64 [[#0x1000000001003]],
   // CK8-SAME: {{^}} i64 [[#0x1000000001010]], i64 [[#0x1010]], i64 [[#0x1013]],
 
   // arg
@@ -659,16 +656,13 @@ void test_present_modifier(int arg) {
   // ps2
   //
   // PRESENT=0x1000 | TARGET_PARAM=0x20 = 0x1020
-  // MEMBER_OF_9=0x9000000000000 | PRESENT=0x1000 | FROM=0x2 | TO=0x1 = 0x9000000001003
-  // MEMBER_OF_9=0x9000000000000 | PRESENT=0x1000 | PTR_AND_OBJ=0x10 | FROM=0x2 | TO=0x1 = 0x9000000001013
-  // MEMBER_OF_9=0x9000000000000 | FROM=0x2 | TO=0x1 = 0x9000000000003
-  // MEMBER_OF_9=0x9000000000000 | PTR_AND_OBJ=0x10 = 0x9000000000010
+  // MEMBER_OF_7=0x7000000000000 | PRESENT=0x1000 | FROM=0x2 | TO=0x1 = 0x7000000001003
+  // MEMBER_OF_7=0x7000000000000 | PTR_AND_OBJ=0x10 = 0x7000000000010
   // PTR_AND_OBJ=0x10 = 0x10
   // PTR_AND_OBJ=0x10 | FROM=0x2 | TO=0x1 = 0x13
   //
-  // CK8-SAME: {{^}} i64 [[#0x1020]], i64 [[#0x9000000001003]],
-  // CK8-SAME: {{^}} i64 [[#0x9000000001013]], i64 [[#0x9000000000003]],
-  // CK8-SAME: {{^}} i64 [[#0x9000000000010]], i64 [[#0x10]], i64 [[#0x13]]]
+  // CK8-SAME: {{^}} i64 [[#0x1020]], i64 [[#0x7000000001003]],
+  // CK8-SAME: {{^}} i64 [[#0x7000000000010]], i64 [[#0x10]], i64 [[#0x13]]]
   #pragma omp target data map(tofrom: ps1->s) \
                           map(present,tofrom: arg, ps1->ps->ps->ps->s, ps2->s) \
                           map(tofrom: ps2->ps->ps->ps->s)
diff --git a/clang/test/OpenMP/target_data_use_device_ptr_codegen.cpp b/clang/test/OpenMP/target_data_use_device_ptr_codegen.cpp
index fe6ea01b43c91..ca5536f927a10 100644
--- a/clang/test/OpenMP/target_data_use_device_ptr_codegen.cpp
+++ b/clang/test/OpenMP/target_data_use_device_ptr_codegen.cpp
@@ -22,18 +22,18 @@
 double *g;
 
 // CK1: @g = global double*
-// CK1: [[MTYPE00:@.+]] = {{.*}}constant [2 x i64] [i64 99, i64 19]
-// CK1: [[MTYPE01:@.+]] = {{.*}}constant [2 x i64] [i64 99, i64 19]
-// CK1: [[MTYPE03:@.+]] = {{.*}}constant [2 x i64] [i64 99, i64 19]
-// CK1: [[MTYPE04:@.+]] = {{.*}}constant [2 x i64] [i64 99, i64 19]
-// CK1: [[MTYPE05:@.+]] = {{.*}}constant [2 x i64] [i64 99, i64 19]
-// CK1: [[MTYPE06:@.+]] = {{.*}}constant [2 x i64] [i64 99, i64 19]
-// CK1: [[MTYPE07:@.+]] = {{.*}}constant [2 x i64] [i64 99, i64 19]
-// CK1: [[MTYPE08:@.+]] = {{.*}}constant [4 x i64] [i64 99, i64 19, i64 35, i64 19]
-// CK1: [[MTYPE09:@.+]] = {{.*}}constant [4 x i64] [i64 99, i64 19, i64 99, i64 19]
-// CK1: [[MTYPE10:@.+]] = {{.*}}constant [4 x i64] [i64 99, i64 19, i64 99, i64 19]
-// CK1: [[MTYPE11:@.+]] = {{.*}}constant [3 x i64] [i64 96, i64 35, i64 19]
-// CK1: [[MTYPE12:@.+]] = {{.*}}constant [3 x i64] [i64 96, i64 35, i64 19]
+// CK1: [[MTYPE00:@.+]] = {{.*}}constant [2 x i64] [i64 51, i64 96]
+// CK1: [[MTYPE01:@.+]] = {{.*}}constant [1 x i64] [i64 99]
+// CK1: [[MTYPE03:@.+]] = {{.*}}constant [1 x i64] [i64 99]
+// CK1: [[MTYPE04:@.+]] = {{.*}}constant [1 x i64] [i64 99]
+// CK1: [[MTYPE05:@.+]] = {{.*}}constant [1 x i64] [i64 99]
+// CK1: [[MTYPE06:@.+]] = {{.*}}constant [1 x i64] [i64 99]
+// CK1: [[MTYPE07:@.+]] = {{.*}}constant [1 x i64] [i64 99]
+// CK1: [[MTYPE08:@.+]] = {{.*}}constant [2 x i64] [i64 99, i64 35]
+// CK1: [[MTYPE09:@.+]] = {{.*}}constant [2 x i64] [i64 99, i64 99]
+// CK1: [[MTYPE10:@.+]] = {{.*}}constant [2 x i64] [i64 99, i64 99]
+// CK1: [[MTYPE11:@.+]] = {{.*}}constant [2 x i64] [i64 35, i64 96]
+// CK1: [[MTYPE12:@.+]] = {{.*}}constant [2 x i64] [i64 35, i64 96]
 
 // CK1-LABEL: @_Z3foo
 template<typename T>
@@ -42,7 +42,7 @@ void foo(float *&lr, T *&tr) {
   T *t;
 
   // CK1:     [[T:%.+]] = load double*, double** [[DECL:@g]],
-  // CK1:     [[BP:%.+]] = getelementptr inbounds [2 x i8*], [2 x i8*]* %{{.+}}, i32 0, i32 0
+  // CK1:     [[BP:%.+]] = getelementptr inbounds [2 x i8*], [2 x i8*]* %{{.+}}, i32 0, i32 1
   // CK1:     [[CBP:%.+]] = bitcast i8** [[BP]] to double**
   // CK1:     store double* [[T]], double** [[CBP]],
   // CK1:     call void @__tgt_target_data_begin{{.+}}[[MTYPE00]]
@@ -61,7 +61,7 @@ void foo(float *&lr, T *&tr) {
   ++g;
 
   // CK1:     [[T1:%.+]] = load float*, float** [[DECL:%.+]],
-  // CK1:     [[BP:%.+]] = getelementptr inbounds [2 x i8*], [2 x i8*]* %{{.+}}, i32 0, i32 0
+  // CK1:     [[BP:%.+]] = getelementptr inbounds [1 x i8*], [1 x i8*]* %{{.+}}, i32 0, i32 0
   // CK1:     [[CBP:%.+]] = bitcast i8** [[BP]] to float**
   // CK1:     store float* [[T1]], float** [[CBP]],
   // CK1:     call void @__tgt_target_data_begin{{.+}}[[MTYPE01]]
@@ -92,7 +92,7 @@ void foo(float *&lr, T *&tr) {
   ++l;
 
   // CK1:     [[T1:%.+]] = load float*, float** [[DECL:%.+]],
-  // CK1:     [[BP:%.+]] = getelementptr inbounds [2 x i8*], [2 x i8*]* %{{.+}}, i32 0, i32 0
+  // CK1:     [[BP:%.+]] = getelementptr inbounds [1 x i8*], [1 x i8*]* %{{.+}}, i32 0, i32 0
   // CK1:     [[CBP:%.+]] = bitcast i8** [[BP]] to float**
   // CK1:     store float* [[T1]], float** [[CBP]],
   // CK1:     call void @__tgt_target_data_begin{{.+}}[[MTYPE03]]
@@ -115,7 +115,7 @@ void foo(float *&lr, T *&tr) {
 
   // CK1:     [[BTHEN]]:
   // CK1:     [[T1:%.+]] = load float*, float** [[DECL:%.+]],
-  // CK1:     [[BP:%.+]] = getelementptr inbounds [2 x i8*], [2 x i8*]* %{{.+}}, i32 0, i32 0
+  // CK1:     [[BP:%.+]] = getelementptr inbounds [1 x i8*], [1 x i8*]* %{{.+}}, i32 0, i32 0
   // CK1:     [[CBP:%.+]] = bitcast i8** [[BP]] to float**
   // CK1:     store float* [[T1]], float** [[CBP]],
   // CK1:     call void @__tgt_target_data_begin{{.+}}[[MTYPE04]]
@@ -152,7 +152,7 @@ void foo(float *&lr, T *&tr) {
 
   // CK1:     [[T2:%.+]] = load float**, float*** [[DECL:%.+]],
   // CK1:     [[T1:%.+]] = load float*, float** [[T2]],
-  // CK1:     [[BP:%.+]] = getelementptr inbounds [2 x i8*], [2 x i8*]* %{{.+}}, i32 0, i32 0
+  // CK1:     [[BP:%.+]] = getelementptr inbounds [1 x i8*], [1 x i8*]* %{{.+}}, i32 0, i32 0
   // CK1:     [[CBP:%.+]] = bitcast i8** [[BP]] to float**
   // CK1:     store float* [[T1]], float** [[CBP]],
   // CK1:     call void @__tgt_target_data_begin{{.+}}[[MTYPE05]]
@@ -174,7 +174,7 @@ void foo(float *&lr, T *&tr) {
   ++lr;
 
   // CK1:     [[T1:%.+]] = load i32*, i32** [[DECL:%.+]],
-  // CK1:     [[BP:%.+]] = getelementptr inbounds [2 x i8*], [2 x i8*]* %{{.+}}, i32 0, i32 0
+  // CK1:     [[BP:%.+]] = getelementptr inbounds [1 x i8*], [1 x i8*]* %{{.+}}, i32 0, i32 0
   // CK1:     [[CBP:%.+]] = bitcast i8** [[BP]] to i32**
   // CK1:     store i32* [[T1]], i32** [[CBP]],
   // CK1:     call void @__tgt_target_data_begin{{.+}}[[MTYPE06]]
@@ -194,7 +194,7 @@ void foo(float *&lr, T *&tr) {
 
   // CK1:     [[T2:%.+]] = load i32**, i32*** [[DECL:%.+]],
   // CK1:     [[T1:%.+]] = load i32*, i32** [[T2]],
-  // CK1:     [[BP:%.+]] = getelementptr inbounds [2 x i8*], [2 x i8*]* %{{.+}}, i32 0, i32 0
+  // CK1:     [[BP:%.+]] = getelementptr inbounds [1 x i8*], [1 x i8*]* %{{.+}}, i32 0, i32 0
   // CK1:     [[CBP:%.+]] = bitcast i8** [[BP]] to i32**
   // CK1:     store i32* [[T1]], i32** [[CBP]],
   // CK1:     call void @__tgt_target_data_begin{{.+}}[[MTYPE07]]
@@ -216,7 +216,7 @@ void foo(float *&lr, T *&tr) {
   ++tr;
 
   // CK1:     [[T1:%.+]] = load float*, float** [[DECL:%.+]],
-  // CK1:     [[BP:%.+]] = getelementptr inbounds [4 x i8*], [4 x i8*]* %{{.+}}, i32 0, i32
+  // CK1:     [[BP:%.+]] = getelementptr inbounds [2 x i8*], [2 x i8*]* %{{.+}}, i32 0, i32 0
   // CK1:     [[CBP:%.+]] = bitcast i8** [[BP]] to float**
   // CK1:     store float* [[T1]], float** [[CBP]],
   // CK1:     call void @__tgt_target_data_begin{{.+}}[[MTYPE08]]
@@ -280,7 +280,7 @@ void foo(float *&lr, T *&tr) {
   ++l; ++t;
 
   // CK1:     [[T1:%.+]] = load i32*, i32** [[DECL:%.+]],
-  // CK1:     [[BP:%.+]] = getelementptr inbounds [3 x i8*], [3 x i8*]* %{{.+}}, i32 0, i32 0
+  // CK1:     [[BP:%.+]] = getelementptr inbounds [2 x i8*], [2 x i8*]* %{{.+}}, i32 0, i32 1
   // CK1:     [[CBP:%.+]] = bitcast i8** [[BP]] to i32**
   // CK1:     store i32* [[T1]], i32** [[CBP]],
   // CK1:     call void @__tgt_target_data_begin{{.+}}[[MTYPE11]]
@@ -300,7 +300,7 @@ void foo(float *&lr, T *&tr) {
 
   // CK1:     [[T2:%.+]] = load i32**, i32*** [[DECL:%.+]],
   // CK1:     [[T1:%.+]] = load i32*, i32** [[T2]],
-  // CK1:     [[BP:%.+]] = getelementptr inbounds [3 x i8*], [3 x i8*]* %{{.+}}, i32 0, i32 0
+  // CK1:     [[BP:%.+]] = getelementptr inbounds [2 x i8*], [2 x i8*]* %{{.+}}, i32 0, i32 1
   // CK1:     [[CBP:%.+]] = bitcast i8** [[BP]] to i32**
   // CK1:     store i32* [[T1]], i32** [[CBP]],
   // CK1:     call void @__tgt_target_data_begin{{.+}}[[MTYPE12]]
@@ -348,7 +348,7 @@ void bar(float *&a, int *&b) {
 // CK2: [[ST:%.+]] = type { double*, double** }
 // CK2: [[MTYPE00:@.+]] = {{.*}}constant [2 x i64] [i64 32, i64 281474976710739]
 // CK2: [[MTYPE01:@.+]] = {{.*}}constant [2 x i64] [i64 32, i64 281474976710739]
-// CK2: [[MTYPE02:@.+]] = {{.*}}constant [4 x i64] [i64 35, i64 19, i64 32, i64 844424930132048]
+// CK2: [[MTYPE02:@.+]] = {{.*}}constant [3 x i64] [i64 35, i64 32, i64 562949953421392]
 // CK2: [[MTYPE03:@.+]] = {{.*}}constant [3 x i64] [i64 32, i64 281474976710739, i64 281474976710736]
 
 template <typename T>
@@ -404,7 +404,7 @@ struct ST {
     // CK2:     getelementptr inbounds double, double* [[TTTT]], i32 1
     b++;
 
-    // CK2:     [[BP:%.+]] = getelementptr inbounds [4 x i8*], [4 x i8*]* %{{.+}}, i32 0, i32 3
+    // CK2:     [[BP:%.+]] = getelementptr inbounds [3 x i8*], [3 x i8*]* %{{.+}}, i32 0, i32 2
     // CK2:     [[CBP:%.+]] = bitcast i8** [[BP]] to double***
     // CK2:     store double** [[RVAL:%.+]], double*** [[CBP]],
     // CK2:     call void @__tgt_target_data_begin{{.+}}[[MTYPE02]]
diff --git a/clang/test/OpenMP/target_map_codegen.cpp b/clang/test/OpenMP/target_map_codegen.cpp
index e63f19ad73c0a..a394f5b1c3d53 100644
--- a/clang/test/OpenMP/target_map_codegen.cpp
+++ b/clang/test/OpenMP/target_map_codegen.cpp
@@ -3874,7 +3874,7 @@ int explicit_maps_template_args_and_members(int a){
 
 // CK22-LABEL: @.__omp_offloading_{{.*}}explicit_maps_globals{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK22: [[SIZE04:@.+]] = private {{.*}}constant [1 x i64] [i64 20]
-// CK22: [[MTYPE04:@.+]] = private {{.*}}constant [1 x i64] [i64 35]
+// CK22: [[MTYPE04:@.+]] = private {{.*}}constant [1 x i64] [i64 51]
 
 // CK22-LABEL: @.__omp_offloading_{{.*}}explicit_maps_globals{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK22: [[SIZE05:@.+]] = private {{.*}}constant [1 x i64] [i64 4]
@@ -3894,7 +3894,7 @@ int explicit_maps_template_args_and_members(int a){
 
 // CK22-LABEL: @.__omp_offloading_{{.*}}explicit_maps_globals{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK22: [[SIZE09:@.+]] = private {{.*}}constant [1 x i64] [i64 20]
-// CK22: [[MTYPE09:@.+]] = private {{.*}}constant [1 x i64] [i64 35]
+// CK22: [[MTYPE09:@.+]] = private {{.*}}constant [1 x i64] [i64 51]
 
 // CK22-LABEL: @.__omp_offloading_{{.*}}explicit_maps_globals{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK22: [[SIZE10:@.+]] = private {{.*}}constant [1 x i64] [i64 4]
@@ -3914,7 +3914,7 @@ int explicit_maps_template_args_and_members(int a){
 
 // CK22-LABEL: @.__omp_offloading_{{.*}}explicit_maps_globals{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK22: [[SIZE14:@.+]] = private {{.*}}constant [1 x i64] [i64 20]
-// CK22: [[MTYPE14:@.+]] = private {{.*}}constant [1 x i64] [i64 35]
+// CK22: [[MTYPE14:@.+]] = private {{.*}}constant [1 x i64] [i64 51]
 
 int a;
 int c[100];
@@ -4010,11 +4010,10 @@ int explicit_maps_globals(void){
 
   // CK22-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 0
   // CK22-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 0
-  // CK22-DAG: [[CBP0:%.+]] = bitcast i8** [[BP0]] to i32**
+  // CK22-DAG: [[CBP0:%.+]] = bitcast i8** [[BP0]] to i32***
   // CK22-DAG: [[CP0:%.+]] = bitcast i8** [[P0]] to i32**
-  // CK22-DAG: store i32* [[RVAR0:%.+]], i32** [[CBP0]]
+  // CK22-DAG: store i32** @d, i32*** [[CBP0]]
   // CK22-DAG: store i32* [[SEC0:%.+]], i32** [[CP0]]
-  // CK22-DAG: [[RVAR0]] = load i32*, i32** @d
   // CK22-DAG: [[SEC0]] = getelementptr {{.*}}i32* [[RVAR00:%.+]], i{{.+}} 2
   // CK22-DAG: [[RVAR00]] = load i32*, i32** @d
 
@@ -4093,11 +4092,10 @@ int explicit_maps_globals(void){
 
   // CK22-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 0
   // CK22-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 0
-  // CK22-DAG: [[CBP0:%.+]] = bitcast i8** [[BP0]] to [[ST]]**
+  // CK22-DAG: [[CBP0:%.+]] = bitcast i8** [[BP0]] to [[ST]]***
   // CK22-DAG: [[CP0:%.+]] = bitcast i8** [[P0]] to [[ST]]**
-  // CK22-DAG: store [[ST]]* [[RVAR0:%.+]], [[ST]]** [[CBP0]]
+  // CK22-DAG: store [[ST]]** @sd, [[ST]]*** [[CBP0]]
   // CK22-DAG: store [[ST]]* [[SEC0:%.+]], [[ST]]** [[CP0]]
-  // CK22-DAG: [[RVAR0]] = load [[ST]]*, [[ST]]** @sd
   // CK22-DAG: [[SEC0]] = getelementptr {{.*}}[[ST]]* [[RVAR00:%.+]], i{{.+}} 2
   // CK22-DAG: [[RVAR00]] = load [[ST]]*, [[ST]]** @sd
 
@@ -4176,11 +4174,10 @@ int explicit_maps_globals(void){
 
   // CK22-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 0
   // CK22-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 0
-  // CK22-DAG: [[CBP0:%.+]] = bitcast i8** [[BP0]] to [[STT]]**
+  // CK22-DAG: [[CBP0:%.+]] = bitcast i8** [[BP0]] to [[STT]]***
   // CK22-DAG: [[CP0:%.+]] = bitcast i8** [[P0]] to [[STT]]**
-  // CK22-DAG: store [[STT]]* [[RVAR0:%.+]], [[STT]]** [[CBP0]]
+  // CK22-DAG: store [[STT]]** @std, [[STT]]*** [[CBP0]]
   // CK22-DAG: store [[STT]]* [[SEC0:%.+]], [[STT]]** [[CP0]]
-  // CK22-DAG: [[RVAR0]] = load [[STT]]*, [[STT]]** @std
   // CK22-DAG: [[SEC0]] = getelementptr {{.*}}[[STT]]* [[RVAR00:%.+]], i{{.+}} 2
   // CK22-DAG: [[RVAR00]] = load [[STT]]*, [[STT]]** @std
 
diff --git a/clang/test/OpenMP/target_update_codegen.cpp b/clang/test/OpenMP/target_update_codegen.cpp
index feea1129be4cc..9eab8b3367c4e 100644
--- a/clang/test/OpenMP/target_update_codegen.cpp
+++ b/clang/test/OpenMP/target_update_codegen.cpp
@@ -310,22 +310,23 @@ void device_side_scan(int arg) {
 
 #ifdef CK5
 
-// CK5: [[SIZE00:@.+]] = {{.+}}constant [2 x i[[sz:64|32]]] [i{{64|32}} {{8|4}}, i{{64|32}} 4]
-// CK5: [[MTYPE00:@.+]] = {{.+}}constant [2 x i64] [i64 33, i64 17]
+// CK5: [[SIZE00:@.+]] = {{.+}}constant [1 x i[[sz:64|32]]] [i{{64|32}} 4]
+// CK5: [[MTYPE00:@.+]] = {{.+}}constant [1 x i64] [i64 33]
 
 // CK5-LABEL: lvalue
 void lvalue(int *B, int l, int e) {
 
-  // CK5-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 2, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}]* [[SIZE00]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
+  // CK5-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 1, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[SIZE00]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
   // CK5-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
   // CK5-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
 
-  // CK5-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
-  // CK5-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
-  // CK5-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to i32***
+  // CK5-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 0
+  // CK5-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 0
+  // CK5-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to i32**
   // CK5-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to i32**
-  // CK5-DAG: store i32** [[B_ADDR:%.+]], i32*** [[BPC0]]
+  // CK5-DAG: store i32* [[B_VAL:%.+]], i32** [[BPC0]]
   // CK5-DAG: store i32* [[B_VAL_2:%.+]], i32** [[PC0]]
+  // CK5-DAG: [[B_VAL]] = load i32*, i32** [[B_ADDR:%.+]]
   // CK5-DAG: [[B_VAL_2]] = load i32*, i32** [[B_ADDR]]
   #pragma omp target update to(*B)
   *B += e;
@@ -351,28 +352,29 @@ void lvalue(int *B, int l, int e) {
 
 #ifdef CK6
 
-// CK6: [[SIZE00:@.+]] = {{.+}}constant [2 x i[[sz:64|32]]] [i{{64|32}} {{8|4}}, i{{64|32}} 4]
-// CK6: [[MTYPE00:@.+]] = {{.+}}constant [2 x i64] [i64 33, i64 17]
+// CK6: [[SIZE00:@.+]] = {{.+}}constant [1 x i[[sz:64|32]]] [i{{64|32}} 4]
+// CK6: [[MTYPE00:@.+]] = {{.+}}constant [1 x i64] [i64 33]
 
 // CK6-LABEL: lvalue
 void lvalue(int *B, int l, int e) {
 
-  // CK6-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 2, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}]* [[SIZE00]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
+  // CK6-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 1, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[SIZE00]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
   // CK6-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
   // CK6-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
 
-  // CK6-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
-  // CK6-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
-  // CK6-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to i32***
+  // CK6-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 0
+  // CK6-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 0
+  // CK6-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to i32**
   // CK6-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to i32**
-  // CK6-DAG: store i32** [[B_ADDR:%.+]], i32*** [[BPC0]]
+  // CK6-DAG: store i32* [[TWO:%.+]], i32** [[BPC0]]
   // CK6-DAG: store i32* [[ADD_PTR:%.+]], i32** [[PC0]]
   // CK6-64-DAG: [[ADD_PTR]] = getelementptr inbounds i32, i32* [[ONE:%.+]], i{{32|64}} [[IDX_EXT:%.+]]
   // CK6-32-DAG: [[ADD_PTR]] = getelementptr inbounds i32, i32* [[ONE:%.+]], i{{32|64}} [[L_VAL:%.+]]
   // CK6-64-DAG: [[IDX_EXT]] = sext i32 [[L_VAL:%.+]] to i64
   // CK6-DAG: [[L_VAL]] = load i32, i32* [[L_ADDR:%.+]]
   // CK6-DAG: store i32 {{.+}}, i32* [[L_ADDR]]
-  // CK6-DAG: [[ONE]] = load i32*, i32** [[B_ADDR]]
+  // CK6-DAG: [[ONE]] = load i32*, i32** [[B_ADDR:%.+]]
+  // CK6-DAG: [[TWO]] = load i32*, i32** [[B_ADDR]]
   #pragma omp target update to(*(B+l))
   *(B+l) += e;
   #pragma omp target update from(*(B+l))
@@ -397,25 +399,26 @@ void lvalue(int *B, int l, int e) {
 
 #ifdef CK7
 
-// CK7: [[SIZE00:@.+]] = {{.+}}constant [2 x i[[sz:64|32]]] [i{{64|32}} {{8|4}}, i{{64|32}} 4]
-// CK7: [[MTYPE00:@.+]] = {{.+}}constant [2 x i64] [i64 33, i64 17]
+// CK7: [[SIZE00:@.+]] = {{.+}}constant [1 x i[[sz:64|32]]] [i{{64|32}} 4]
+// CK7: [[MTYPE00:@.+]] = {{.+}}constant [1 x i64] [i64 33]
 
 // CK7-LABEL: lvalue
 void lvalue(int *B, int l, int e) {
 
-  // CK7-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 2, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}]* [[SIZE00]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
+  // CK7-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 1, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[SIZE00]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
   // CK7-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
   // CK7-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
 
-  // CK7-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
-  // CK7-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
-  // CK7-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to i32***
+  // CK7-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 0
+  // CK7-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 0
+  // CK7-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to i32**
   // CK7-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to i32**
-  // CK7-DAG: store i32** [[B_ADDR:%.+]], i32*** [[BPC0]]
+  // CK7-DAG: store i32* [[B_VAL:%.+]], i32** [[BPC0]]
   // CK7-DAG: store i32* [[ARRAY_IDX:%.+]], i32** [[PC0]]
   // CK7-DAG: [[ARRAY_IDX]] = getelementptr inbounds i32, i32* [[ADD_PTR:%.+]], i{{32|64}} [[IDX_PROM:%.+]]
   // CK7-64-DAG: [[ADD_PTR]] = getelementptr inbounds i32, i32* [[ONE:%.+]], i64 [[IDX_EXT:%.+]]
   // CK7-32-DAG: [[ADD_PTR]] = getelementptr inbounds i32, i32* [[B_VAL_2:%.+]], i32 [[L_VAL:%.+]]
+  // CK7-32-DAG: [[B_VAL]] = load i32*, i32** [[B_ADDR:%.+]]
   // CK7-32-DAG: [[B_VAL_2]] = load i32*, i32** [[B_ADDR]]
   // CK7-32-DAG: [[L_VAL]] = load i32, i32* [[L_ADDR:%.+]]
   // CK7-32-DAG: [[IDX_PROM]] = load i32, i32* [[L_ADDR]]
@@ -446,18 +449,18 @@ void lvalue(int *B, int l, int e) {
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 
 #ifdef CK8
-// CK8: [[SIZE00:@.+]] = {{.+}}constant [3 x i[[sz:64|32]]] [i{{64|32}} {{8|4}}, i{{64|32}} {{8|4}}, i{{64|32}} 4]
-// CK8: [[MTYPE00:@.+]] = {{.+}}constant [3 x i64] [i64 33, i64 16, i64 17]
+// CK8: [[SIZE00:@.+]] = {{.+}}constant [2 x i[[sz:64|32]]] [i{{64|32}} {{8|4}}, i{{64|32}} 4]
+// CK8: [[MTYPE00:@.+]] = {{.+}}constant [2 x i64] [i64 33, i64 17]
 
 // CK8-LABEL: lvalue
 void lvalue(int **B, int l, int e) {
 
-  // CK8-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 3, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], {{.+}}getelementptr {{.+}}[3 x i{{.+}}], [3 x i{{.+}}]* [[SIZE00]], {{.+}}getelementptr {{.+}}[3 x i{{.+}}, [3 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
+  // CK8-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 2, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}], [2 x i{{.+}}]* [[SIZE00]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}, [2 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
   // CK8-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
   // CK8-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
 
-  // CK8-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 2
-  // CK8-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 2
+  // CK8-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
+  // CK8-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
   // CK8-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to i32***
   // CK8-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to i32**
   // CK8-DAG: store i32** [[ARRAY_IDX_1:%.+]], i32*** [[BPC0]]
@@ -501,19 +504,19 @@ struct S {
   double *p;
 };
 
-// CK9: [[MTYPE00:@.+]] = {{.+}}constant [3 x i64] [i64 32, i64 281474976710657, i64 281474976710673]
+// CK9: [[MTYPE00:@.+]] = {{.+}}constant [2 x i64] [i64 32, i64 281474976710673]
 
 // CK9-LABEL: lvalue
 void lvalue(struct S *s, int l, int e) {
 
-  // CK9-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 3, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i{{.+}} [[GSIZE:%.+]], {{.+}}getelementptr {{.+}}[3 x i{{.+}}, [3 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
+  // CK9-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 2, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i{{.+}} [[GSIZE:%.+]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}, [2 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
   // CK9-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
   // CK9-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
   // CK9-DAG: [[GSIZE]] = getelementptr inbounds {{.+}}[[SIZE:%[^,]+]]
   //
-  // CK9-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 2
-  // CK9-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 2
-  // CK9-DAG: [[SIZE0:%.+]] = getelementptr inbounds {{.+}}[[SIZE]], i{{.+}} 0, i{{.+}} 2
+  // CK9-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
+  // CK9-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
+  // CK9-DAG: [[SIZE0:%.+]] = getelementptr inbounds {{.+}}[[SIZE]], i{{.+}} 0, i{{.+}} 1
   // CK9-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to double***
   // CK9-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to double**
   // CK9-DAG: store double** [[P:%.+]], double*** [[BPC0]]
@@ -551,19 +554,19 @@ struct S {
   double *p;
 };
 
-// CK10: [[MTYPE00:@.+]] = {{.+}}constant [3 x i64] [i64 32, i64 281474976710657, i64 281474976710673]
+// CK10: [[MTYPE00:@.+]] = {{.+}}constant [2 x i64] [i64 32, i64 281474976710673]
 
 // CK10-LABEL: lvalue
 void lvalue(struct S *s, int l, int e) {
 
-  // CK10-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 3, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i{{.+}} [[GSIZE:%.+]], {{.+}}getelementptr {{.+}}[3 x i{{.+}}, [3 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
+  // CK10-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 2, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i{{.+}} [[GSIZE:%.+]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}, [2 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
   // CK10-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
   // CK10-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
   // CK10-DAG: [[GSIZE]] = getelementptr inbounds {{.+}}[[SIZE:%[^,]+]]
   //
-  // CK10-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 2
-  // CK10-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 2
-  // CK10-DAG: [[SIZE0:%.+]] = getelementptr inbounds {{.+}}[[SIZE]], i{{.+}} 0, i{{.+}} 2
+  // CK10-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
+  // CK10-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
+  // CK10-DAG: [[SIZE0:%.+]] = getelementptr inbounds {{.+}}[[SIZE]], i{{.+}} 0, i{{.+}} 1
   // CK10-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to double***
   // CK10-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to double**
   // CK10-DAG: store double** [[P_VAL:%.+]], double*** [[BPC0]]
@@ -601,19 +604,19 @@ void lvalue(struct S *s, int l, int e) {
 struct S {
   double *p;
 };
-// CK11: [[MTYPE00:@.+]] = {{.+}}constant [3 x i64] [i64 32, i64 281474976710657, i64 281474976710673]
+// CK11: [[MTYPE00:@.+]] = {{.+}}constant [2 x i64] [i64 32, i64 281474976710673]
 
 // CK11-LABEL: lvalue
 void lvalue(struct S *s, int l, int e) {
 
-  // CK11-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 3, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i{{.+}} [[GSIZE:%.+]], {{.+}}getelementptr {{.+}}[3 x i{{.+}}, [3 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
+  // CK11-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 2, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i{{.+}} [[GSIZE:%.+]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}, [2 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
   // CK11-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
   // CK11-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
   // CK11-DAG: [[GSIZE]] = getelementptr inbounds {{.+}}[[SIZE:%[^,]+]]
   //
-  // CK11-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 2
-  // CK11-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 2
-  // CK11-DAG: [[SIZE0:%.+]] = getelementptr inbounds {{.+}}[[SIZE]], i{{.+}} 0, i{{.+}} 2
+  // CK11-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
+  // CK11-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
+  // CK11-DAG: [[SIZE0:%.+]] = getelementptr inbounds {{.+}}[[SIZE]], i{{.+}} 0, i{{.+}} 1
   // CK11-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to double***
   // CK11-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to double**
   // CK11-DAG: store double** [[P:%.+]], double*** [[BPC0]]
@@ -653,41 +656,44 @@ struct S {
   double *p;
   struct S *sp;
 };
-// CK12: [[MTYPE00:@.+]] = {{.+}}constant [4 x i64] [i64 32, i64 281474976710657, i64 281474976710672, i64 17]
+// CK12: [[MTYPE00:@.+]] = {{.+}}constant [3 x i64] [i64 32, i64 281474976710672, i64 17]
 
 // CK12-LABEL: lvalue
 void lvalue(struct S *s, int l, int e) {
 
-  // CK12-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 4, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i{{.+}} [[GSIZE:%.+]], {{.+}}getelementptr {{.+}}[4 x i{{.+}}, [4 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
+  // CK12-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 3, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i{{.+}} [[GSIZE:%.+]], {{.+}}getelementptr {{.+}}[3 x i{{.+}}, [3 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
   // CK12-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
   // CK12-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
   // CK12-DAG: [[GSIZE]] = getelementptr inbounds {{.+}}[[SIZE:%[^,]+]]
   //
-  // CK12-DAG: [[BP2:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 3
-  // CK12-DAG: [[P2:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 3
-  // CK12-DAG: [[SIZE2:%.+]] = getelementptr inbounds {{.+}}[[SIZE]], i{{.+}} 0, i{{.+}} 3
+  // CK12-DAG: [[BP2:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 2
+  // CK12-DAG: [[P2:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 2
+  // CK12-DAG: [[SIZE2:%.+]] = getelementptr inbounds {{.+}}[[SIZE]], i{{.+}} 0, i{{.+}} 2
   // CK12-DAG: [[BPC2:%.+]] = bitcast i8** [[BP2]] to double***
   // CK12-DAG: [[PC2:%.+]] = bitcast i8** [[P2]] to double**
   // CK12-DAG: store double** [[P_VAL:%.+]], double*** [[BPC2]]
   // CK12-DAG: store double* [[SIX:%.+]], double** [[PC2]]
   // CK12-DAG: store i{{.+}} 8, i{{.+}}* [[SIZE2]]
-  // CK12-DAG: [[BP1:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 2
-  // CK12-DAG: [[P1:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 2
-  // CK12-DAG: [[SIZE1:%.+]] = getelementptr inbounds {{.+}}[[SIZE]], i{{.+}} 0, i{{.+}} 2
+  // CK12-DAG: [[BP1:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
+  // CK12-DAG: [[P1:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
+  // CK12-DAG: [[SIZE1:%.+]] = getelementptr inbounds {{.+}}[[SIZE]], i{{.+}} 0, i{{.+}} 1
   // CK12-DAG: [[BPC1:%.+]] = bitcast i8** [[BP1]] to [[STRUCT_S:%.+]]***
   // CK12-DAG: [[PC1:%.+]] = bitcast i8** [[P1]] to double***
   // CK12-DAG: store [[STRUCT_S]]** [[SP:%.+]], [[STRUCT_S]]*** [[BPC1]]
   // CK12-DAG: store double** [[P_VAL:%.+]], double*** [[PC1]]
   // CK12-DAG: store i{{.+}} {{4|8}}, i{{.+}}* [[SIZE1]]
-  // CK12-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
-  // CK12-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
-  // CK12-DAG: [[SIZE0:%.+]] = getelementptr inbounds {{.+}}[[SIZE]], i{{.+}} 0, i{{.+}} 1
+  // CK12-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 0
+  // CK12-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 0
+  // CK12-DAG: [[SIZE0:%.+]] = getelementptr inbounds {{.+}}[[SIZE]], i{{.+}} 0, i{{.+}} 0
   // CK12-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to [[STRUCT_S:%.+]]**
   // CK12-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to [[STRUCT_S]]***
+  // CK12-DAG: store [[STRUCT_S]]* [[ZERO:%.+]], [[STRUCT_S]]** [[BPC0]]
+  // CK12-DAG: store [[STRUCT_S]]** [[SP]], [[STRUCT_S]]*** [[PC0]]
   // CK12-DAG: store [[STRUCT_S]]** [[S:%.+]], [[STRUCT_S]]*** [[S_VAL:%.+]]
   // CK12-DAG: store i{{.+}} {{.+}}, i{{.+}}* [[SIZE0]]
   // CK12-DAG: [[SP]] = getelementptr inbounds [[STRUCT_S]], [[STRUCT_S]]* [[ONE:%.+]], i32 0, i32 1
-  // CK12-DAG: [[ONE]] = load %struct.S*, %struct.S** [[S]],
+  // CK12-DAG: [[ONE]] = load [[STRUCT_S]]*, [[STRUCT_S]]** [[S:%.+]],
+  // CK12-DAG: [[ZERO]] = load [[STRUCT_S]]*, [[STRUCT_S]]** [[S]],
   #pragma omp target update to(*(s->sp->p))
     *(s->sp->p) = e;
   #pragma omp target update from(*(s->sp->p))
@@ -711,21 +717,21 @@ void lvalue(struct S *s, int l, int e) {
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 #ifdef CK13
 
-// CK13: [[SIZE00:@.+]] = {{.+}}constant [2 x i64] [i64 {{8|4}}, i64 4]
-// CK13: [[MTYPE00:@.+]] = {{.+}}constant [2 x i64] [i64 33, i64 17]
+// CK13: [[SIZE00:@.+]] = {{.+}}constant [1 x i64] [i64 4]
+// CK13: [[MTYPE00:@.+]] = {{.+}}constant [1 x i64] [i64 33]
 
 // CK13-LABEL: lvalue
 void lvalue(int **BB, int a, int b) {
 
-  // CK13-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 2, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}]* [[SIZE00]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
+  // CK13-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 1, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[SIZE00]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
   // CK13-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
   // CK13-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
 
-  // CK13-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
-  // CK13-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
-  // CK13-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to i32****
+  // CK13-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 0
+  // CK13-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 0
+  // CK13-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to i32***
   // CK13-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to i32**
-  // CK13-DAG: store i32*** [[BB_ADDR:%.+]], i32**** [[BPC0]]
+  // CK13-DAG: store i32** [[B_VAL1:%.+]], i32*** [[BPC0]]
   // CK13-DAG: store i32* [[ADD_PTR_2:%.+]], i32** [[PC0]]
   // CK13-64-DAG: [[ADD_PTR_2]] = getelementptr inbounds i32, i32* [[RESULT:%.+]], i64 [[IDX_EXT_1:%.+]]
   // CK13-32-DAG: [[ADD_PTR_2]] = getelementptr inbounds i32, i32* [[RESULT:%.+]], i32 [[B_ADDR:%.+]]
@@ -734,7 +740,8 @@ void lvalue(int **BB, int a, int b) {
   // CK13-64-DAG: [[ADD_PTR]] = getelementptr inbounds i32*, i32** [[B_VAL:%.+]], i64 [[IDX_EXT:%.+]]
   // CK13-32-DAG: [[ADD_PTR]] = getelementptr inbounds i32*, i32** [[B_VAL:%.+]], i32 [[A_ADDR:%.+]]
   // CK13-64-DAG: [[IDX_EXT]] = sext i32 [[TWO:%.+]] to i64
-  // CK13-DAG: [[B_VAL]] = load i32**, i32*** [[BB_ADDR]]
+  // CK13-DAG: [[B_VAL]] = load i32**, i32*** [[BB_ADDR:%.+]]
+  // CK13-DAG: [[B_VAL1]] = load i32**, i32*** [[BB_ADDR]]
   #pragma omp target update to(*(*(BB+a)+b))
   *(*(BB+a)+b) = 1;
   #pragma omp target update from(*(*(BB+a)+b))
@@ -831,7 +838,7 @@ void lvalue_member(SSA *sap) {
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 #ifdef CK15
 
-// CK15: [[MTYPE00:@.+]] = {{.+}}constant [3 x i64] [i64 32, i64 281474976710657, i64 281474976710673]
+// CK15: [[MTYPE00:@.+]] = {{.+}}constant [2 x i64] [i64 32, i64 281474976710673]
 
 struct SSA {
   double *p;
@@ -842,36 +849,27 @@ struct SSA {
 //CK-15-LABEL: lvalue_member
 void lvalue_member(SSA *sap) {
 
-  // CK15-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 3, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i64* [[GSIZE:%.+]], {{.+}}getelementptr {{.+}}[3 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
+  // CK15-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 2, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i64* [[GSIZE:%.+]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
   // CK15-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
   // CK15-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
   // CK15-DAG: [[GSIZE]] = getelementptr inbounds {{.+}}[[SIZE:%[^,]+]]
 
-  // CK15-DAG: [[BP2:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 2
-  // CK15-DAG: [[P2:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 2
-  // CK15-DAG: [[SIZE2:%.+]] = getelementptr inbounds {{.+}}[[SIZE]], i{{.+}} 0, i{{.+}} 2
-  // CK15-DAG: [[BPC2:%.+]] = bitcast i8** [[BP2]] to double***
-  // CK15-DAG: [[PC2:%.+]] = bitcast i8** [[P2]] to double**
-  // CK15-DAG: store double** [[P_VAL:%.+]], double*** [[BPC2]]
-  // CK15-DAG: store double* [[ADD_PTR:%.+]], double** [[PC2]]
-  // CK15-DAG: store i64 8, i64* [[SIZE2]]
-  // CK15-DAG: [[ADD_PTR]] = getelementptr inbounds double, double* [[THREE:%.+]], i{{.+}} 3
-  // CK15-DAG: [[THREE]] = load double*, double** [[P_VAL_1:%.+]]
-  // CK15-DAG: [[P_VAL]] = getelementptr inbounds [[SSA:%.+]], [[SSA:%.+]]* [[THIS:%.+]], i32 0, i32 0
   // CK15-DAG: [[BP1:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
   // CK15-DAG: [[P1:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
   // CK15-DAG: [[SIZE1:%.+]] = getelementptr inbounds {{.+}}[[SIZE]], i{{.+}} 0, i{{.+}} 1
-  // CK15-DAG: [[BPC1:%.+]] = bitcast i8** [[BP1]] to [[SSA]]**
-  // CK15-DAG: [[PC1:%.+]] = bitcast i8** [[P1]] to [[SSA]]***
-  // CK15-DAG: store [[SSA]]* [[SAP_VAL:%.+]], [[SSA]]** [[BPC1]],
-  // CK15-DAG: store [[SSA]]** [[SAP_ADDR:%.+]], [[SSA]]*** [[PC1]]
-  // CK15-DAG: store i{{.+}} {{8|4}}, i{{.+}}* [[SIZE1]]
-  // CK15-DAG: [[SAP_VAL]] = load [[SSA]]*, [[SSA]]** [[SAP_ADDR]],
+  // CK15-DAG: [[BPC1:%.+]] = bitcast i8** [[BP1]] to double***
+  // CK15-DAG: [[PC1:%.+]] = bitcast i8** [[P1]] to double**
+  // CK15-DAG: store double** [[P_VAL:%.+]], double*** [[BPC1]]
+  // CK15-DAG: store double* [[ADD_PTR:%.+]], double** [[PC1]]
+  // CK15-DAG: store i64 {{4|8}}, i64* [[SIZE1]]
+  // CK15-DAG: [[ADD_PTR]] = getelementptr inbounds double, double* [[THREE:%.+]], i{{.+}} 3
+  // CK15-DAG: [[THREE]] = load double*, double** [[P_VAL_1:%.+]]
+  // CK15-DAG: [[P_VAL]] = getelementptr inbounds [[SSA:%.+]], [[SSA:%.+]]* [[THIS:%.+]], i32 0, i32 0
   // CK15-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 0
   // CK15-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 0
   // CK15-DAG: [[SIZE0:%.+]] = getelementptr inbounds {{.+}}[[SIZE]], i{{.+}} 0, i{{.+}} 0
-  // CK15-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to [[SSA]]***
-  // CK15-DAG: store [[SSA]]** [[SAP_ADDR]], [[SSA]]*** [[BPC0]],
+  // CK15-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to [[SSA]]**
+  // CK15-DAG: store [[SSA]]* [[ZERO:%.+]], [[SSA]]** [[BPC0]],
   // CK15-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to double***
   // CK15-DAG: store double** [[P_VAL]], double*** [[PC0]],
   // CK15-DAG: store i{{.+}} [[COMPUTE_SIZE:%.+]], i{{.+}}* [[SIZE0]]
@@ -881,6 +879,7 @@ void lvalue_member(SSA *sap) {
   // CK15-DAG: [[EIGHT]] = ptrtoint i8* [[FIVE:%.+]] to i64
   // CK15-DAG: [[SIX]] = bitcast double** {{.+}} to i8*
   // CK15-DAG: [[FIVE]] = bitcast double** {{.+}} to i8*
+  // CK15-DAG: [[ZERO]] = load [[SSA]]*, [[SSA]]** %{{.+}},
   #pragma omp target update to(*(3+sap->p))
   *(3+sap->p) = 1;
   #pragma omp target update from(*(3+sap->p))
@@ -904,25 +903,26 @@ void lvalue_member(SSA *sap) {
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 #ifdef CK16
 
-// CK16: [[SIZE00:@.+]] = {{.+}}constant [2 x i64] [i64 {{8|4}}, i64 4]
-// CK16: [[MTYPE00:@.+]] = {{.+}}constant [2 x i64] [i64 33, i64 17]
+// CK16: [[SIZE00:@.+]] = {{.+}}constant [1 x i64] [i64 4]
+// CK16: [[MTYPE00:@.+]] = {{.+}}constant [1 x i64] [i64 33]
 
 //CK16-LABEL: lvalue_find_base
 void lvalue_find_base(float *f, int *i) {
 
-  // CK16-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 2, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}]* [[SIZE00]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
+  // CK16-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 1, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[SIZE00]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
   // CK16-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
   // CK16-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
 
-  // CK16-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
-  // CK16-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
-  // CK16-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to float***
+  // CK16-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 0
+  // CK16-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 0
+  // CK16-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to float**
   // CK16-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to float**
-  // CK16-DAG: store float** [[F_ADDR:%.+]], float*** [[BPC0]]
+  // CK16-DAG: store float* [[F:%.+]], float** [[BPC0]]
   // CK16-DAG: store float* [[ADD_PTR:%.+]], float** [[PC0]]
   // CK16-32-DAG: [[ADD_PTR]] = getelementptr inbounds float, float* [[THREE:%.+]], i32 [[I:%.+]]
   // CK16-64-DAG: [[ADD_PTR]] = getelementptr inbounds float, float* [[THREE:%.+]], i64 [[IDX_EXT:%.+]]
-  // CK16-DAG: [[THREE]] = load float*, float** [[F_ADDR]],
+  // CK16-DAG: [[THREE]] = load float*, float** [[F_ADDR:%.+]],
+  // CK16-DAG: [[F]] = load float*, float** [[F_ADDR]],
   // CK16-64-DAG: [[IDX_EXT]] = sext i32 [[I:%.+]] to i64
 
   #pragma omp target update to(*(*i+f))
@@ -948,8 +948,8 @@ void lvalue_find_base(float *f, int *i) {
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 #ifdef CK17
 
-// CK17: [[SIZE00:@.+]] = {{.+}}constant [2 x i64] [i64 {{4|8}}, i64 4]
-// CK17: [[MTYPE00:@.+]] = {{.+}}constant [2 x i64] [i64 33, i64 17]
+// CK17: [[SIZE00:@.+]] = {{.+}}constant [1 x i64] [i64 4]
+// CK17: [[MTYPE00:@.+]] = {{.+}}constant [1 x i64] [i64 33]
 
 struct SSA {
   int i;
@@ -959,15 +959,15 @@ struct SSA {
 //CK17-LABEL: lvalue_find_base
 void lvalue_find_base(float **f, SSA *sa) {
 
-  // CK17-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 2, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}]* [[SIZE00]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
+  // CK17-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 1, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[SIZE00]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
   // CK17-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
   // CK17-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
 
-  // CK17-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
-  // CK17-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
-  // CK17-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to float****
+  // CK17-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 0
+  // CK17-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 0
+  // CK17-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to float***
   // CK17-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to float**
-  // CK17-DAG: store float*** [[F_ADDR:%.+]], float**** [[BPC0]],
+  // CK17-DAG: store float** [[F_VAL:%.+]], float*** [[BPC0]],
   // CK17-DAG: store float* [[ADD_PTR_4:%.+]], float** [[PC0]],
   // CK17-64-DAG: [[ADD_PTR_4]] = getelementptr inbounds float, float* [[SEVEN:%.+]], i64 [[IDX_EXT_3:%.+]]
   // CK17-64-DAG: [[IDX_EXT_3]] = sext i32 [[I_VAL:%.+]] to i64
@@ -981,6 +981,8 @@ void lvalue_find_base(float **f, SSA *sa) {
   // CK17-DAG: [[FIVE]] = load i32, i32* [[I_2:%.+]],
   // CK17-DAG: [[I_2]] = getelementptr inbounds [[SSA:%.+]], [[SSA]]* [[FOUR:%.+]], i32 0, i32 0
   // CK17-DAG: [[FOUR]] = load [[SSA]]*, [[SSA]]** [[SSA_ADDR:%.+]],
+  // CK17-DAG: [[F]] = load float**, float*** [[F_ADDR:%.+]],
+  // CK17-DAG: [[F_VAL]] = load float**, float*** [[F_ADDR]],
 
   #pragma omp target update to(*(sa->sa->i+*(1+sa->i+f)))
   *(sa->sa->i+*(1+sa->i+f)) = 1;
@@ -1005,13 +1007,13 @@ void lvalue_find_base(float **f, SSA *sa) {
 // SIMD-ONLY18-NOT: {{__kmpc|__tgt}}
 #ifdef CK18
 
-// CK18-DAG: [[MTYPE_TO:@.+]] = {{.+}}constant [2 x i64] [i64 33, i64 16]
-// CK18-DAG: [[MTYPE_FROM:@.+]] = {{.+}}constant [2 x i64] [i64 34, i64 16]
+// CK18-DAG: [[MTYPE_TO:@.+]] = {{.+}}constant [1 x i64] [i64 33]
+// CK18-DAG: [[MTYPE_FROM:@.+]] = {{.+}}constant [1 x i64] [i64 34]
 
 //CK18-LABEL: array_shaping
 void array_shaping(float *f, int sa) {
 
-  // CK18-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 2, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i64* [[GEPS:%.+]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}]* [[MTYPE_TO]]{{.+}}, i8** null)
+  // CK18-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 1, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i64* [[GEPS:%.+]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[MTYPE_TO]]{{.+}}, i8** null)
   // CK18-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
   // CK18-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
   // CK18-DAG: [[GEPS]] = getelementptr inbounds {{.+}}[[S:%[^,]+]]
@@ -1021,23 +1023,12 @@ void array_shaping(float *f, int sa) {
   // CK18-DAG: [[S0:%.+]] = getelementptr inbounds {{.+}}[[S]], i{{.+}} 0, i{{.+}} 0
 
   // CK18-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to float**
-  // CK18-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to float***
+  // CK18-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to float**
 
   // CK18-DAG: store float* [[F1:%.+]], float** [[BPC0]],
-  // CK18-DAG: store float** [[F_ADDR:%.+]], float*** [[PC0]],
-  // CK18-DAG: store i64 {{8|4}}, i64* [[S0]],
-  // CK18-DAG: [[F1]] = load float*, float** [[F_ADDR]],
-
-  // CK18-DAG: [[BP1:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
-  // CK18-DAG: [[P1:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
-  // CK18-DAG: [[S1:%.+]] = getelementptr inbounds {{.+}}[[S]], i{{.+}} 0, i{{.+}} 1
-
-  // CK18-DAG: [[BPC1:%.+]] = bitcast i8** [[BP1]] to float***
-  // CK18-DAG: [[PC1:%.+]] = bitcast i8** [[P1]] to float**
-
-  // CK18-DAG: store float** [[F_ADDR]], float*** [[BPC1]],
-  // CK18-DAG: store float* [[F2:%.+]], float** [[PC1]],
-  // CK18-DAG: store i64 [[SIZE:%.+]], i64* [[S1]],
+  // CK18-DAG: store float* [[F2:%.+]], float** [[PC0]],
+  // CK18-DAG: store i64 [[SIZE:%.+]], i64* [[S0]],
+  // CK18-DAG: [[F1]] = load float*, float** [[F_ADDR:%.+]],
   // CK18-DAG: [[F2]] = load float*, float** [[F_ADDR]],
 
   // CK18-64-DAG: [[SIZE]] = mul nuw i64 [[SZ1:%.+]], 4
@@ -1047,7 +1038,7 @@ void array_shaping(float *f, int sa) {
   // CK18-32-DAG: [[SZ2]] = mul nuw i32 12, %{{.+}}
   #pragma omp target update to(([3][sa][4])f)
   sa = 1;
-  // CK18-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 2, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i64* [[GEPS:%.+]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}]* [[MTYPE_FROM]]{{.+}}, i8** null)
+  // CK18-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 1, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i64* [[GEPS:%.+]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[MTYPE_FROM]]{{.+}}, i8** null)
   // CK18-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
   // CK18-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
   // CK18-DAG: [[GEPS]] = getelementptr inbounds {{.+}}[[S:%[^,]+]]
@@ -1057,23 +1048,12 @@ void array_shaping(float *f, int sa) {
   // CK18-DAG: [[S0:%.+]] = getelementptr inbounds {{.+}}[[S]], i{{.+}} 0, i{{.+}} 0
 
   // CK18-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to float**
-  // CK18-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to float***
+  // CK18-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to float**
 
   // CK18-DAG: store float* [[F1:%.+]], float** [[BPC0]],
-  // CK18-DAG: store float** [[F_ADDR:%.+]], float*** [[PC0]],
-  // CK18-DAG: store i64 {{8|4}}, i64* [[S0]],
+  // CK18-DAG: store float* [[F2:%.+]], float** [[PC0]],
+  // CK18-DAG: store i64 [[SIZE:%.+]], i64* [[S0]],
   // CK18-DAG: [[F1]] = load float*, float** [[F_ADDR]],
-
-  // CK18-DAG: [[BP1:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
-  // CK18-DAG: [[P1:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
-  // CK18-DAG: [[S1:%.+]] = getelementptr inbounds {{.+}}[[S]], i{{.+}} 0, i{{.+}} 1
-
-  // CK18-DAG: [[BPC1:%.+]] = bitcast i8** [[BP1]] to float***
-  // CK18-DAG: [[PC1:%.+]] = bitcast i8** [[P1]] to float**
-
-  // CK18-DAG: store float** [[F_ADDR]], float*** [[BPC1]],
-  // CK18-DAG: store float* [[F2:%.+]], float** [[PC1]],
-  // CK18-DAG: store i64 [[SIZE:%.+]], i64* [[S1]],
   // CK18-DAG: [[F2]] = load float*, float** [[F_ADDR]],
 
   // CK18-64-DAG: [[SIZE]] = mul nuw i64 [[SZ1:%.+]], 5
diff --git a/openmp/libomptarget/src/omptarget.cpp b/openmp/libomptarget/src/omptarget.cpp
index 409089500015e..9c7ab6309503a 100644
--- a/openmp/libomptarget/src/omptarget.cpp
+++ b/openmp/libomptarget/src/omptarget.cpp
@@ -880,14 +880,9 @@ int target(int64_t DeviceId, void *HostPtr, int32_t ArgNum, void **ArgBases,
           return OFFLOAD_FAIL;
         }
       }
-    } else if (ArgTypes[I] & OMP_TGT_MAPTYPE_PTR_AND_OBJ) {
-      TgtPtrBegin = Device.getTgtPtrBegin(HstPtrBase, sizeof(void *), IsLast,
-                                          false, IsHostPtr);
-      TgtBaseOffset = 0; // no offset for ptrs.
-      DP("Obtained target argument " DPxMOD " from host pointer " DPxMOD " to "
-         "object " DPxMOD "\n",
-         DPxPTR(TgtPtrBegin), DPxPTR(HstPtrBase), DPxPTR(HstPtrBase));
     } else {
+      if (ArgTypes[I] & OMP_TGT_MAPTYPE_PTR_AND_OBJ)
+        HstPtrBase = *reinterpret_cast<void **>(HstPtrBase);
       TgtPtrBegin = Device.getTgtPtrBegin(HstPtrBegin, ArgSizes[I], IsLast,
                                           false, IsHostPtr);
       TgtBaseOffset = (intptr_t)HstPtrBase - (intptr_t)HstPtrBegin;
diff --git a/openmp/libomptarget/test/env/base_ptr_ref_count.c b/openmp/libomptarget/test/env/base_ptr_ref_count.c
new file mode 100644
index 0000000000000..4036954182faf
--- /dev/null
+++ b/openmp/libomptarget/test/env/base_ptr_ref_count.c
@@ -0,0 +1,51 @@
+// RUN: %libomptarget-compile-aarch64-unknown-linux-gnu && env LIBOMPTARGET_DEBUG=1 %libomptarget-run-aarch64-unknown-linux-gnu 2>&1 | %fcheck-aarch64-unknown-linux-gnu
+// RUN: %libomptarget-compile-powerpc64-ibm-linux-gnu && env LIBOMPTARGET_DEBUG=1 %libomptarget-run-powerpc64-ibm-linux-gnu 2>&1 | %fcheck-powerpc64-ibm-linux-gnu
+// RUN: %libomptarget-compile-powerpc64le-ibm-linux-gnu && env LIBOMPTARGET_DEBUG=1 %libomptarget-run-powerpc64le-ibm-linux-gnu 2>&1 | %fcheck-powerpc64le-ibm-linux-gnu
+// RUN: %libomptarget-compile-x86_64-pc-linux-gnu && env LIBOMPTARGET_DEBUG=1 %libomptarget-run-x86_64-pc-linux-gnu 2>&1 | %fcheck-x86_64-pc-linux-gnu
+// RUN: %libomptarget-compile-nvptx64-nvidia-cuda && env LIBOMPTARGET_DEBUG=1 %libomptarget-run-nvptx64-nvidia-cuda 2>&1 | %fcheck-nvptx64-nvidia-cuda
+// REQUIRES: libomptarget-debug
+
+#include <stdlib.h>
+
+int *allocate(size_t n) {
+  int *ptr = malloc(sizeof(int) * n);
+#pragma omp target enter data map(to : ptr[:n])
+  return ptr;
+}
+
+void deallocate(int *ptr, size_t n) {
+#pragma omp target exit data map(delete : ptr[:n])
+  free(ptr);
+}
+
+#pragma omp declare target
+int *cnt;
+void foo() {
+  ++(*cnt);
+}
+#pragma omp end declare target
+
+int main(void) {
+  int *A = allocate(10);
+  int *V = allocate(10);
+  deallocate(A, 10);
+  deallocate(V, 10);
+// CHECK-NOT: RefCount=2
+  cnt = malloc(sizeof(int));
+  *cnt = 0;
+#pragma omp target map(cnt[:1])
+  foo();
+  printf("Cnt = %d.\n", *cnt);
+// CHECK: Cnt = 1.
+  *cnt = 0;
+#pragma omp target data map(cnt[:1])
+#pragma omp target
+  foo();
+  printf("Cnt = %d.\n", *cnt);
+// CHECK: Cnt = 1.
+  free(cnt);
+
+  return 0;
+}
+
+

From 38272f45fed3f2233d54ff1495a4433f87215667 Mon Sep 17 00:00:00 2001
From: Tim Keith <tkeith@nvidia.com>
Date: Thu, 30 Jul 2020 07:12:24 -0700
Subject: [PATCH 0618/1035] [flang] Create HostAssoc symbols for uplevel
 references

To make it easier for lowering to identify which symbols from the host
are captured by internal subprograms, create HostAssocDetails for them.

In particular, if a symbol is referenced and it is contained in a
subprogram or main program that is not the same as the containing
program unit of the reference, a HostAssocDetails symbol is created
in the current scope.

Differential Revision: https://reviews.llvm.org/D84889
---
 flang/lib/Semantics/resolve-names.cpp | 16 +++++++++++++++-
 flang/lib/Semantics/tools.cpp         | 13 ++++++++++++-
 flang/test/Semantics/symbol02.f90     |  2 +-
 flang/test/Semantics/symbol03.f90     |  9 ++++++++-
 flang/test/Semantics/symbol05.f90     |  2 +-
 5 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp
index 50ea735d81d5e..744eee19ba43c 100644
--- a/flang/lib/Semantics/resolve-names.cpp
+++ b/flang/lib/Semantics/resolve-names.cpp
@@ -909,6 +909,7 @@ class DeclarationVisitor : public ArraySpecVisitor,
   void AddSaveName(std::set<SourceName> &, const SourceName &);
   void SetSaveAttr(Symbol &);
   bool HandleUnrestrictedSpecificIntrinsicFunction(const parser::Name &);
+  bool IsUplevelReference(const Symbol &);
   const parser::Name *FindComponent(const parser::Name *, const parser::Name &);
   bool CheckInitialDataTarget(const Symbol &, const SomeExpr &, SourceName);
   void CheckInitialProcTarget(const Symbol &, const parser::Name &, SourceName);
@@ -5429,7 +5430,10 @@ const parser::Name *DeclarationVisitor::ResolveName(const parser::Name &name) {
     if (CheckUseError(name)) {
       return nullptr; // reported an error
     }
-    if (IsDummy(*symbol) ||
+    if (IsUplevelReference(*symbol)) {
+      name.symbol = nullptr;
+      MakeSymbol(name, HostAssocDetails{*symbol});
+    } else if (IsDummy(*symbol) ||
         (!symbol->GetType() && FindCommonBlockContaining(*symbol))) {
       ConvertToObjectEntity(*symbol);
       ApplyImplicitRules(*symbol);
@@ -5453,6 +5457,16 @@ const parser::Name *DeclarationVisitor::ResolveName(const parser::Name &name) {
   return &name;
 }
 
+bool DeclarationVisitor::IsUplevelReference(const Symbol &symbol) {
+  const Scope *symbolUnit{FindProgramUnitContaining(symbol)};
+  if (symbolUnit == FindProgramUnitContaining(currScope())) {
+    return false;
+  } else {
+    Scope::Kind kind{DEREF(symbolUnit).kind()};
+    return kind == Scope::Kind::Subprogram || kind == Scope::Kind::MainProgram;
+  }
+}
+
 // base is a part-ref of a derived type; find the named component in its type.
 // Also handles intrinsic type parameter inquiries (%kind, %len) and
 // COMPLEX component references (%re, %im).
diff --git a/flang/lib/Semantics/tools.cpp b/flang/lib/Semantics/tools.cpp
index 2607011333964..d5ef9c76aa34c 100644
--- a/flang/lib/Semantics/tools.cpp
+++ b/flang/lib/Semantics/tools.cpp
@@ -179,10 +179,21 @@ bool DoesScopeContain(const Scope *maybeAncestor, const Symbol &symbol) {
   return DoesScopeContain(maybeAncestor, symbol.owner());
 }
 
+static const Symbol &FollowHostAssoc(const Symbol &symbol) {
+  for (const Symbol *s{&symbol};;) {
+    const auto *details{s->detailsIf<HostAssocDetails>()};
+    if (!details) {
+      return *s;
+    }
+    s = &details->symbol();
+  }
+}
+
 bool IsHostAssociated(const Symbol &symbol, const Scope &scope) {
   const Scope *subprogram{FindProgramUnitContaining(scope)};
   return subprogram &&
-      DoesScopeContain(FindProgramUnitContaining(symbol), *subprogram);
+      DoesScopeContain(
+          FindProgramUnitContaining(FollowHostAssoc(symbol)), *subprogram);
 }
 
 bool IsInStmtFunction(const Symbol &symbol) {
diff --git a/flang/test/Semantics/symbol02.f90 b/flang/test/Semantics/symbol02.f90
index 4d73edd660446..add42939e4647 100644
--- a/flang/test/Semantics/symbol02.f90
+++ b/flang/test/Semantics/symbol02.f90
@@ -44,7 +44,7 @@ subroutine s2
    !REF: /m/x
    z = x
    !REF: /m/s/s2/z
-   !REF: /m/s/y
+   !DEF: /m/s/s2/y HostAssoc TYPE(t)
    z = y
    !REF: /m/s/s
    call s
diff --git a/flang/test/Semantics/symbol03.f90 b/flang/test/Semantics/symbol03.f90
index 2f3620284a833..a398d8db40b0d 100644
--- a/flang/test/Semantics/symbol03.f90
+++ b/flang/test/Semantics/symbol03.f90
@@ -11,7 +11,14 @@ program main
  !REF: /main/s
  subroutine s
   !DEF: /main/s/y (Implicit) ObjectEntity REAL(4)
-  !REF: /main/x
+  !DEF: /main/s/x HostAssoc INTEGER(4)
   y = x
+ contains
+  !DEF: /main/s/s2 (Subroutine) Subprogram
+  subroutine s2
+   !DEF: /main/s/s2/z (Implicit) ObjectEntity REAL(4)
+   !DEF: /main/s/s2/x HostAssoc INTEGER(4)
+   z = x
+  end subroutine
  end subroutine
 end program
diff --git a/flang/test/Semantics/symbol05.f90 b/flang/test/Semantics/symbol05.f90
index 0c36384a6c4eb..03f6fbbd989e3 100644
--- a/flang/test/Semantics/symbol05.f90
+++ b/flang/test/Semantics/symbol05.f90
@@ -33,7 +33,7 @@ subroutine s2
 contains
  !DEF: /s2/s (Subroutine) Subprogram
  subroutine s
-  !REF: /s2/x
+  !DEF: /s2/s/x HostAssoc INTEGER(4)
   x = 1
   !DEF: /s2/s/w (Implicit) ObjectEntity INTEGER(4)
   w = 1

From b7292f2db02d37c9291afc0613a3fbce0a4ad4e8 Mon Sep 17 00:00:00 2001
From: Hans Wennborg <hans@chromium.org>
Date: Thu, 30 Jul 2020 16:11:15 +0200
Subject: [PATCH 0619/1035] Speculative GCC 5 build fix

It's complaining about specializing the template in a different namespace.
---
 llvm/lib/Target/WebAssembly/WebAssemblySortRegion.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/Target/WebAssembly/WebAssemblySortRegion.cpp b/llvm/lib/Target/WebAssembly/WebAssemblySortRegion.cpp
index 475f9120f7b27..6b3957e3ec557 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblySortRegion.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblySortRegion.cpp
@@ -5,7 +5,8 @@
 using namespace llvm;
 using namespace WebAssembly;
 
-template <> bool ConcreteSortRegion<MachineLoop>::isLoop() const {
+template <>
+bool llvm::WebAssembly::ConcreteSortRegion<MachineLoop>::isLoop() const {
   return true;
 }
 

From dab8d6104bd743b8caa29cf232ddde051c2d404b Mon Sep 17 00:00:00 2001
From: Jinsong Ji <jji@us.ibm.com>
Date: Thu, 30 Jul 2020 02:46:39 +0000
Subject: [PATCH 0620/1035] [PowerPC][AIX] Move the testcase to proper dir

---
 {test => llvm/test}/CodeGen/PowerPC/aix-complex.ll | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename {test => llvm/test}/CodeGen/PowerPC/aix-complex.ll (100%)

diff --git a/test/CodeGen/PowerPC/aix-complex.ll b/llvm/test/CodeGen/PowerPC/aix-complex.ll
similarity index 100%
rename from test/CodeGen/PowerPC/aix-complex.ll
rename to llvm/test/CodeGen/PowerPC/aix-complex.ll

From 6316b0023ec081159843e5ca9609c6f35198a4c4 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Thu, 30 Jul 2020 14:25:08 +0100
Subject: [PATCH 0621/1035] Attributor.h - remove unnecessary includes. NFCI.

Fix implicit cpp include dependencies.
---
 llvm/include/llvm/Transforms/IPO/Attributor.h      | 4 ----
 llvm/lib/Transforms/IPO/Attributor.cpp             | 1 +
 llvm/lib/Transforms/IPO/AttributorAttributes.cpp   | 3 +++
 llvm/unittests/Transforms/IPO/AttributorTestBase.h | 1 +
 4 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/llvm/include/llvm/Transforms/IPO/Attributor.h b/llvm/include/llvm/Transforms/IPO/Attributor.h
index a8076d33018cd..0fa9c99a882cc 100644
--- a/llvm/include/llvm/Transforms/IPO/Attributor.h
+++ b/llvm/include/llvm/Transforms/IPO/Attributor.h
@@ -99,21 +99,17 @@
 
 #include "llvm/ADT/GraphTraits.h"
 #include "llvm/ADT/MapVector.h"
-#include "llvm/ADT/SCCIterator.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Analysis/AssumeBundleQueries.h"
 #include "llvm/Analysis/CFG.h"
 #include "llvm/Analysis/CGSCCPassManager.h"
-#include "llvm/Analysis/CallGraph.h"
-#include "llvm/Analysis/InlineCost.h"
 #include "llvm/Analysis/LazyCallGraph.h"
 #include "llvm/Analysis/LoopInfo.h"
 #include "llvm/Analysis/MustExecute.h"
 #include "llvm/Analysis/PostDominators.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
-#include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/IR/AbstractCallSite.h"
 #include "llvm/IR/ConstantRange.h"
 #include "llvm/IR/PassManager.h"
diff --git a/llvm/lib/Transforms/IPO/Attributor.cpp b/llvm/lib/Transforms/IPO/Attributor.cpp
index 003baf39e8bdf..1be3782e77d71 100644
--- a/llvm/lib/Transforms/IPO/Attributor.cpp
+++ b/llvm/lib/Transforms/IPO/Attributor.cpp
@@ -19,6 +19,7 @@
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/ADT/TinyPtrVector.h"
+#include "llvm/Analysis/InlineCost.h"
 #include "llvm/Analysis/LazyValueInfo.h"
 #include "llvm/Analysis/MustExecute.h"
 #include "llvm/Analysis/ValueTracking.h"
diff --git a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
index cb25030ce71c4..142a4e48217c7 100644
--- a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
+++ b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
@@ -13,13 +13,16 @@
 
 #include "llvm/Transforms/IPO/Attributor.h"
 
+#include "llvm/ADT/SCCIterator.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Analysis/AssumeBundleQueries.h"
+#include "llvm/Analysis/AssumptionCache.h"
 #include "llvm/Analysis/CaptureTracking.h"
 #include "llvm/Analysis/LazyValueInfo.h"
 #include "llvm/Analysis/MemoryBuiltins.h"
 #include "llvm/Analysis/ScalarEvolution.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/IntrinsicInst.h"
diff --git a/llvm/unittests/Transforms/IPO/AttributorTestBase.h b/llvm/unittests/Transforms/IPO/AttributorTestBase.h
index d12a2b481af0b..9220b60c497ca 100644
--- a/llvm/unittests/Transforms/IPO/AttributorTestBase.h
+++ b/llvm/unittests/Transforms/IPO/AttributorTestBase.h
@@ -18,6 +18,7 @@
 #include "llvm/IR/Module.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/Support/Allocator.h"
+#include "llvm/Support/SourceMgr.h"
 #include "llvm/Testing/Support/Error.h"
 #include "llvm/Transforms/IPO/Attributor.h"
 #include "llvm/Transforms/Utils/CallGraphUpdater.h"

From b8c8d1b30986a25ef392c786daf178beff230f6d Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Thu, 30 Jul 2020 09:23:19 -0400
Subject: [PATCH 0622/1035] AMDGPU: Convert some tests to use new buffer
 intrinsics

The legacy not struct or raw buffer intrinsics should now all be
consolidated into the tests specifically for those intrinsics.
---
 .../AMDGPU/amdgcn-load-offset-from-reg.ll     |  2 -
 llvm/test/CodeGen/AMDGPU/fix-wwm-vgpr-copy.ll | 18 ++++---
 llvm/test/CodeGen/AMDGPU/merge-store-crash.ll |  7 ++-
 .../test/CodeGen/AMDGPU/merge-store-usedef.ll |  5 +-
 llvm/test/CodeGen/AMDGPU/mubuf.ll             | 30 +++++++-----
 .../AMDGPU/scheduler-subrange-crash.ll        | 28 +++++------
 .../AMDGPU/si-triv-disjoint-mem-access.ll     | 47 +++++++++----------
 7 files changed, 67 insertions(+), 70 deletions(-)

diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn-load-offset-from-reg.ll b/llvm/test/CodeGen/AMDGPU/amdgcn-load-offset-from-reg.ll
index fde3ab8c6d4a4..be819ceb88ccf 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn-load-offset-from-reg.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn-load-offset-from-reg.ll
@@ -23,8 +23,6 @@ define amdgpu_cs void @test_load_zext(i32 inreg %0, i32 inreg %1, i32 inreg %res
   ret void
 }
 
-declare <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32>, i32, i32, i1, i1) #0
-; Function Attrs: nounwind writeonly
 declare void @llvm.amdgcn.raw.buffer.store.v4i32(<4 x i32>, <4 x i32>, i32, i32, i32 immarg) #1
 
 ; Function Attrs: nounwind readnone speculatable
diff --git a/llvm/test/CodeGen/AMDGPU/fix-wwm-vgpr-copy.ll b/llvm/test/CodeGen/AMDGPU/fix-wwm-vgpr-copy.ll
index 3375efa14dd9b..f4c8f67bbd635 100644
--- a/llvm/test/CodeGen/AMDGPU/fix-wwm-vgpr-copy.ll
+++ b/llvm/test/CodeGen/AMDGPU/fix-wwm-vgpr-copy.ll
@@ -1,11 +1,5 @@
 ; RUN: llc -mtriple=amdgcn--amdpal -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
 
-declare i64 @llvm.amdgcn.icmp.i32(i32, i32, i32) #0
-declare i32 @llvm.amdgcn.set.inactive.i32(i32, i32) #0
-declare i32 @llvm.amdgcn.wwm.i32(i32) #1
-declare void @llvm.amdgcn.tbuffer.store.f32(float, <4 x i32>, i32, i32, i32, i32, i32, i32, i1, i1) #2
-declare void @llvm.amdgcn.exp.f32(i32, i32, float, float, float, float, i1, i1) #2
-
 define amdgpu_hs void @foo(i32 inreg %arg, <4 x i32> inreg %buffer) {
 entry:
   br label %work
@@ -19,7 +13,7 @@ bb602:
   br i1 %tmp607, label %bb49, label %bb54
 
 bb49:
-  tail call void @llvm.amdgcn.tbuffer.store.f32(float 1.000000e+00, <4 x i32> %buffer, i32 0, i32 1, i32 1, i32 4, i32 4, i32 7, i1 true, i1 false) #7
+  call void @llvm.amdgcn.raw.tbuffer.store.f32(float 1.0, <4 x i32> %buffer, i32 4, i32 1, i32 116, i32 1)
   ret void
 
 bb54:
@@ -42,6 +36,10 @@ work:
   br i1 %tmp34, label %bb602, label %bb42
 }
 
-attributes #0 = { convergent nounwind readnone }
-attributes #1 = { nounwind readnone speculatable }
-attributes #2 = { nounwind writeonly }
+declare i32 @llvm.amdgcn.set.inactive.i32(i32, i32) #0
+declare i32 @llvm.amdgcn.wwm.i32(i32) #1
+declare void @llvm.amdgcn.raw.tbuffer.store.f32(float, <4 x i32>, i32, i32, i32 immarg, i32 immarg) #2
+
+attributes #0 = { convergent nounwind readnone willreturn }
+attributes #1 = { convergent nounwind readnone speculatable willreturn }
+attributes #2 = { nounwind willreturn writeonly }
diff --git a/llvm/test/CodeGen/AMDGPU/merge-store-crash.ll b/llvm/test/CodeGen/AMDGPU/merge-store-crash.ll
index fea47db60d9ea..73f4f9e0cfc06 100644
--- a/llvm/test/CodeGen/AMDGPU/merge-store-crash.ll
+++ b/llvm/test/CodeGen/AMDGPU/merge-store-crash.ll
@@ -27,11 +27,10 @@ main_body:
   %tmp9 = insertelement <4 x i32> %tmp8, i32 %tmp7, i32 1
   %tmp10 = insertelement <4 x i32> %tmp9, i32 undef, i32 2
   %tmp11 = insertelement <4 x i32> %tmp10, i32 undef, i32 3
-  call void @llvm.amdgcn.tbuffer.store.v4i32(<4 x i32> %tmp11, <4 x i32> undef, i32 undef, i32 0, i32 %arg, i32 0, i32 14, i32 4, i1 1, i1 1)
+  call void @llvm.amdgcn.struct.tbuffer.store.v4i32(<4 x i32> %tmp11, <4 x i32> undef, i32 0, i32 0, i32 %arg, i32 78, i32 3) #2
   ret void
 }
 
-; Function Attrs: nounwind
-declare void @llvm.amdgcn.tbuffer.store.v4i32(<4 x i32>, <4 x i32>, i32, i32, i32, i32, i32, i32, i1, i1) #0
+declare void @llvm.amdgcn.struct.tbuffer.store.v4i32(<4 x i32>, <4 x i32>, i32, i32, i32, i32 immarg, i32 immarg) #0
 
-attributes #0 = { nounwind }
+attributes #0 = { nounwind willreturn writeonly }
diff --git a/llvm/test/CodeGen/AMDGPU/merge-store-usedef.ll b/llvm/test/CodeGen/AMDGPU/merge-store-usedef.ll
index 958692e0c92b1..e363d039548b0 100644
--- a/llvm/test/CodeGen/AMDGPU/merge-store-usedef.ll
+++ b/llvm/test/CodeGen/AMDGPU/merge-store-usedef.ll
@@ -11,13 +11,14 @@ define amdgpu_vs void @test1(i32 %v) #0 {
 
   store i32 %v, i32 addrspace(3)* %p0
 
-  call void @llvm.amdgcn.tbuffer.store.i32(i32 %v, <4 x i32> undef, i32 0, i32 0, i32 0, i32 0, i32 4, i32 4, i1 1, i1 0)
+  call void @llvm.amdgcn.raw.tbuffer.store.i32(i32 %v, <4 x i32> undef, i32 0, i32 0, i32 68, i32 1)
 
   %w = load i32, i32 addrspace(3)* %p0
   store i32 %w, i32 addrspace(3)* %p1
   ret void
 }
 
-declare void @llvm.amdgcn.tbuffer.store.i32(i32, <4 x i32>, i32, i32, i32, i32, i32, i32, i1, i1) #0
+declare void @llvm.amdgcn.raw.tbuffer.store.i32(i32, <4 x i32>, i32, i32, i32 immarg, i32 immarg) #1
 
 attributes #0 = { nounwind }
+attributes #1 = { nounwind willreturn writeonly }
diff --git a/llvm/test/CodeGen/AMDGPU/mubuf.ll b/llvm/test/CodeGen/AMDGPU/mubuf.ll
index b28e8fa72ec77..90da8406c4b78 100644
--- a/llvm/test/CodeGen/AMDGPU/mubuf.ll
+++ b/llvm/test/CodeGen/AMDGPU/mubuf.ll
@@ -1,7 +1,5 @@
 ; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -show-mc-encoding -verify-machineinstrs < %s | FileCheck %s
 
-declare i32 @llvm.amdgcn.workitem.id.x() readnone
-
 ;;;==========================================================================;;;
 ;;; MUBUF LOAD TESTS
 ;;;==========================================================================;;;
@@ -60,10 +58,10 @@ main_body:
   %tmp0 = getelementptr [6 x <4 x i32>], [6 x <4 x i32>] addrspace(4)* %0, i32 0, i32 0
   %tmp1 = load <4 x i32>, <4 x i32> addrspace(4)* %tmp0
   %tmp2 = shl i32 %6, 2
-  %tmp3 = call i32 @llvm.amdgcn.raw.buffer.load.dword.i32(<4 x i32> %tmp1, i32 %tmp2, i32 64, i32 1)
+  %tmp3 = call i32 @llvm.amdgcn.raw.buffer.load.i32(<4 x i32> %tmp1, i32 %tmp2, i32 64, i32 1)
   %tmp4 = add i32 %6, 16
   %tmp1.4xi32 = bitcast <4 x i32> %tmp1 to <4 x i32>
-  call void @llvm.amdgcn.tbuffer.store.i32(i32 %tmp3, <4 x i32> %tmp1.4xi32, i32 0, i32 %tmp4, i32 %4, i32 0, i32 4, i32 4, i1 1, i1 1)
+  call void @llvm.amdgcn.raw.tbuffer.store.i32(i32 %tmp3, <4 x i32> %tmp1.4xi32, i32 %tmp4, i32 %4, i32 68, i32 3)
   ret void
 }
 
@@ -79,10 +77,10 @@ main_body:
   %tmp0 = getelementptr [6 x <4 x i32>], [6 x <4 x i32>] addrspace(4)* %0, i32 0, i32 0
   %tmp1 = load <4 x i32>, <4 x i32> addrspace(4)* %tmp0
   %tmp2 = shl i32 %6, 2
-  %tmp3 = call i32 @llvm.amdgcn.raw.buffer.load.dword.i32(<4 x i32> %tmp1, i32 %tmp2, i32 65, i32 1)
+  %tmp3 = call i32 @llvm.amdgcn.raw.buffer.load.i32(<4 x i32> %tmp1, i32 %tmp2, i32 65, i32 1)
   %tmp4 = add i32 %6, 16
   %tmp1.4xi32 = bitcast <4 x i32> %tmp1 to <4 x i32>
-  call void @llvm.amdgcn.tbuffer.store.i32(i32 %tmp3, <4 x i32> %tmp1.4xi32, i32 0, i32 %tmp4, i32 %4, i32 0, i32 4, i32 4, i1 1, i1 1)
+  call void @llvm.amdgcn.raw.tbuffer.store.i32(i32 %tmp3, <4 x i32> %tmp1.4xi32, i32 %tmp4, i32 %4, i32 68, i32 3)
   ret void
 }
 
@@ -136,14 +134,14 @@ entry:
 
 ; CHECK-LABEL: {{^}}store_sgpr_ptr:
 ; CHECK: buffer_store_dword v{{[0-9]+}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0
-define amdgpu_kernel void @store_sgpr_ptr(i32 addrspace(1)* %out) #0 {
+define amdgpu_kernel void @store_sgpr_ptr(i32 addrspace(1)* %out) {
   store i32 99, i32 addrspace(1)* %out, align 4
   ret void
 }
 
 ; CHECK-LABEL: {{^}}store_sgpr_ptr_offset:
 ; CHECK: buffer_store_dword v{{[0-9]+}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0 offset:40
-define amdgpu_kernel void @store_sgpr_ptr_offset(i32 addrspace(1)* %out) #0 {
+define amdgpu_kernel void @store_sgpr_ptr_offset(i32 addrspace(1)* %out) {
   %out.gep = getelementptr i32, i32 addrspace(1)* %out, i32 10
   store i32 99, i32 addrspace(1)* %out.gep, align 4
   ret void
@@ -152,7 +150,7 @@ define amdgpu_kernel void @store_sgpr_ptr_offset(i32 addrspace(1)* %out) #0 {
 ; CHECK-LABEL: {{^}}store_sgpr_ptr_large_offset:
 ; CHECK: s_mov_b32 [[SOFFSET:s[0-9]+]], 0x20000
 ; CHECK: buffer_store_dword v{{[0-9]+}}, off, s{{\[[0-9]+:[0-9]+\]}}, [[SOFFSET]]
-define amdgpu_kernel void @store_sgpr_ptr_large_offset(i32 addrspace(1)* %out) #0 {
+define amdgpu_kernel void @store_sgpr_ptr_large_offset(i32 addrspace(1)* %out) {
   %out.gep = getelementptr i32, i32 addrspace(1)* %out, i32 32768
   store i32 99, i32 addrspace(1)* %out.gep, align 4
   ret void
@@ -161,7 +159,7 @@ define amdgpu_kernel void @store_sgpr_ptr_large_offset(i32 addrspace(1)* %out) #
 ; CHECK-LABEL: {{^}}store_sgpr_ptr_large_offset_atomic:
 ; CHECK: s_mov_b32 [[SOFFSET:s[0-9]+]], 0x20000
 ; CHECK: buffer_atomic_add v{{[0-9]+}}, off, s{{\[[0-9]+:[0-9]+\]}}, [[SOFFSET]]
-define amdgpu_kernel void @store_sgpr_ptr_large_offset_atomic(i32 addrspace(1)* %out) #0 {
+define amdgpu_kernel void @store_sgpr_ptr_large_offset_atomic(i32 addrspace(1)* %out) {
   %gep = getelementptr i32, i32 addrspace(1)* %out, i32 32768
   %val = atomicrmw volatile add i32 addrspace(1)* %gep, i32 5 seq_cst
   ret void
@@ -169,14 +167,20 @@ define amdgpu_kernel void @store_sgpr_ptr_large_offset_atomic(i32 addrspace(1)*
 
 ; CHECK-LABEL: {{^}}store_vgpr_ptr:
 ; CHECK: buffer_store_dword v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64
-define amdgpu_kernel void @store_vgpr_ptr(i32 addrspace(1)* %out) #0 {
+define amdgpu_kernel void @store_vgpr_ptr(i32 addrspace(1)* %out) {
   %tid = call i32 @llvm.amdgcn.workitem.id.x() readnone
   %out.gep = getelementptr i32, i32 addrspace(1)* %out, i32 %tid
   store i32 99, i32 addrspace(1)* %out.gep, align 4
   ret void
 }
 
-declare i32 @llvm.amdgcn.raw.buffer.load.dword.i32(<4 x i32>, i32, i32, i32) #0
-declare void @llvm.amdgcn.tbuffer.store.i32(i32, <4 x i32>, i32, i32, i32, i32, i32, i32, i1, i1)
+
+declare i32 @llvm.amdgcn.workitem.id.x() #1
+declare void @llvm.amdgcn.raw.tbuffer.store.i32(i32, <4 x i32>, i32, i32, i32 immarg, i32 immarg) #2
+declare i32 @llvm.amdgcn.raw.buffer.load.i32(<4 x i32>, i32, i32, i32 immarg) #3
 
 attributes #0 = { nounwind readonly }
+attributes #1 = { nounwind readnone speculatable willreturn }
+attributes #2 = { nounwind willreturn writeonly }
+attributes #3 = { nounwind readonly willreturn }
+attributes #4 = { readnone }
diff --git a/llvm/test/CodeGen/AMDGPU/scheduler-subrange-crash.ll b/llvm/test/CodeGen/AMDGPU/scheduler-subrange-crash.ll
index be76371c42df3..8aa48d4d24caa 100644
--- a/llvm/test/CodeGen/AMDGPU/scheduler-subrange-crash.ll
+++ b/llvm/test/CodeGen/AMDGPU/scheduler-subrange-crash.ll
@@ -25,31 +25,31 @@ main_body:
   %array_vector10 = insertelement <4 x float> %array_vector9, float 0.000000e+00, i32 2
   %array_vector11 = insertelement <4 x float> %array_vector10, float undef, i32 3
   %tmp3 = call i32 @llvm.amdgcn.raw.buffer.load.i32(<4 x i32> undef, i32 undef, i32 4864, i32 0)
-  call void @llvm.amdgcn.tbuffer.store.i32(i32 %tmp3, <4 x i32> undef, i32 0, i32 0, i32 %arg, i32 36, i32 4, i32 4, i1 1, i1 1)
+  call void @llvm.amdgcn.raw.tbuffer.store.i32(i32 %tmp3, <4 x i32> undef, i32 36, i32 %arg, i32 68, i32 3)
   %bc = bitcast <4 x float> %array_vector3 to <4 x i32>
   %tmp4 = extractelement <4 x i32> %bc, i32 undef
-  call void @llvm.amdgcn.tbuffer.store.i32(i32 %tmp4, <4 x i32> undef, i32 0, i32 0, i32 %arg, i32 48, i32 4, i32 4, i1 1, i1 1)
+  call void @llvm.amdgcn.raw.tbuffer.store.i32(i32 %tmp4, <4 x i32> undef, i32 48, i32 %arg, i32 68, i32 3)
   %bc49 = bitcast <4 x float> %array_vector11 to <4 x i32>
   %tmp5 = extractelement <4 x i32> %bc49, i32 undef
-  call void @llvm.amdgcn.tbuffer.store.i32(i32 %tmp5, <4 x i32> undef, i32 0, i32 0, i32 %arg, i32 72, i32 4, i32 4, i1 1, i1 1)
+  call void @llvm.amdgcn.raw.tbuffer.store.i32(i32 %tmp5, <4 x i32> undef, i32 72, i32 %arg, i32 68, i32 3)
   %array_vector21 = insertelement <4 x float> <float 0.000000e+00, float undef, float undef, float undef>, float %tmp, i32 1
   %array_vector22 = insertelement <4 x float> %array_vector21, float undef, i32 2
   %array_vector23 = insertelement <4 x float> %array_vector22, float undef, i32 3
-  call void @llvm.amdgcn.tbuffer.store.i32(i32 undef, <4 x i32> undef, i32 0, i32 0, i32 %arg, i32 28, i32 4, i32 4, i1 1, i1 1)
+  call void @llvm.amdgcn.raw.tbuffer.store.i32(i32 undef, <4 x i32> undef, i32 28, i32 %arg, i32 68, i32 3)
   %bc52 = bitcast <4 x float> %array_vector23 to <4 x i32>
   %tmp6 = extractelement <4 x i32> %bc52, i32 undef
-  call void @llvm.amdgcn.tbuffer.store.i32(i32 %tmp6, <4 x i32> undef, i32 0, i32 0, i32 %arg, i32 64, i32 4, i32 4, i1 1, i1 1)
-  call void @llvm.amdgcn.tbuffer.store.i32(i32 undef, <4 x i32> undef, i32 0, i32 0, i32 %arg, i32 20, i32 4, i32 4, i1 1, i1 1)
-  call void @llvm.amdgcn.tbuffer.store.i32(i32 undef, <4 x i32> undef, i32 0, i32 0, i32 %arg, i32 56, i32 4, i32 4, i1 1, i1 1)
-  call void @llvm.amdgcn.tbuffer.store.i32(i32 undef, <4 x i32> undef, i32 0, i32 0, i32 %arg, i32 92, i32 4, i32 4, i1 1, i1 1)
+  call void @llvm.amdgcn.raw.tbuffer.store.i32(i32 %tmp6, <4 x i32> undef, i32 64, i32 %arg, i32 68, i32 3)
+  call void @llvm.amdgcn.raw.tbuffer.store.i32(i32 undef, <4 x i32> undef, i32 20, i32 %arg, i32 68, i32 3)
+  call void @llvm.amdgcn.raw.tbuffer.store.i32(i32 undef, <4 x i32> undef, i32 56, i32 %arg, i32 68, i32 3)
+  call void @llvm.amdgcn.raw.tbuffer.store.i32(i32 undef, <4 x i32> undef, i32 92, i32 %arg, i32 68, i32 3)
   ret void
 }
 
-declare float @llvm.amdgcn.s.buffer.load.f32(<4 x i32>, i32, i32) #1
-declare i32 @llvm.amdgcn.raw.buffer.load.i32(<4 x i32>, i32, i32, i32) #2
-declare void @llvm.amdgcn.tbuffer.store.i32(i32, <4 x i32>, i32, i32, i32, i32, i32, i32, i1, i1) #3
+declare float @llvm.amdgcn.s.buffer.load.f32(<4 x i32>, i32, i32 immarg) #1
+declare i32 @llvm.amdgcn.raw.buffer.load.i32(<4 x i32>, i32, i32, i32 immarg) #2
+declare void @llvm.amdgcn.raw.tbuffer.store.i32(i32, <4 x i32>, i32, i32, i32 immarg, i32 immarg) #3
 
 attributes #0 = { nounwind "target-cpu"="tonga" }
-attributes #1 = { nounwind readnone }
-attributes #2 = { nounwind readonly }
-attributes #3 = { nounwind }
+attributes #1 = { nounwind readnone willreturn }
+attributes #2 = { nounwind readonly willreturn }
+attributes #3 = { nounwind willreturn writeonly }
diff --git a/llvm/test/CodeGen/AMDGPU/si-triv-disjoint-mem-access.ll b/llvm/test/CodeGen/AMDGPU/si-triv-disjoint-mem-access.ll
index ec2dbb995c98c..225d7cc0d9a2f 100644
--- a/llvm/test/CodeGen/AMDGPU/si-triv-disjoint-mem-access.ll
+++ b/llvm/test/CodeGen/AMDGPU/si-triv-disjoint-mem-access.ll
@@ -1,12 +1,6 @@
 ; RUN: llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=bonaire -enable-amdgpu-aa=0 -verify-machineinstrs -enable-misched -enable-aa-sched-mi < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CI %s
 ; RUN: llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=gfx900 -enable-amdgpu-aa=0 -verify-machineinstrs -enable-misched -enable-aa-sched-mi -amdgpu-enable-global-sgpr-addr < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9 %s
 
-declare void @llvm.amdgcn.tbuffer.store.i32(i32, <4 x i32>, i32, i32, i32, i32, i32, i32, i1, i1)
-declare void @llvm.amdgcn.tbuffer.store.v4i32(<4 x i32>, <4 x i32>, i32, i32, i32, i32, i32, i32, i1, i1)
-declare void @llvm.amdgcn.s.barrier() #1
-declare i32 @llvm.amdgcn.workitem.id.x() #2
-
-
 @stored_lds_ptr = addrspace(3) global i32 addrspace(3)* undef, align 4
 @stored_constant_ptr = addrspace(3) global i32 addrspace(4)* undef, align 8
 @stored_global_ptr = addrspace(3) global i32 addrspace(1)* undef, align 8
@@ -296,30 +290,33 @@ define amdgpu_kernel void @reorder_global_offsets_addr64_soffset0(i32 addrspace(
   ret void
 }
 
-; XGCN-LABEL: {{^}}reorder_local_load_tbuffer_store_local_load:
-; XCI: ds_read_b32 {{v[0-9]+}}, {{v[0-9]+}}, 0x4
-; XCI: TBUFFER_STORE_FORMAT
-; XCI: ds_read_b32 {{v[0-9]+}}, {{v[0-9]+}}, 0x8
-; define amdgpu_vs void @reorder_local_load_tbuffer_store_local_load(i32 addrspace(1)* %out, i32 %a1, i32 %vaddr) #0 {
-;   %ptr0 = load i32 addrspace(3)*, i32 addrspace(3)* addrspace(3)* @stored_lds_ptr, align 4
+; GCN-LABEL: {{^}}reorder_local_load_tbuffer_store_local_load:
+; GCN: tbuffer_store_format
+; GCN: ds_read2_b32 {{v\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}} offset0:1 offset1:2
+define amdgpu_vs void @reorder_local_load_tbuffer_store_local_load(i32 addrspace(1)* %out, i32 %a1, i32 %vaddr) #0 {
+  %ptr0 = load i32 addrspace(3)*, i32 addrspace(3)* addrspace(3)* @stored_lds_ptr, align 4
 
-;   %ptr1 = getelementptr inbounds i32, i32 addrspace(3)* %ptr0, i32 1
-;   %ptr2 = getelementptr inbounds i32, i32 addrspace(3)* %ptr0, i32 2
+  %ptr1 = getelementptr inbounds i32, i32 addrspace(3)* %ptr0, i32 1
+  %ptr2 = getelementptr inbounds i32, i32 addrspace(3)* %ptr0, i32 2
 
-;   %tmp1 = load i32, i32 addrspace(3)* %ptr1, align 4
+  %tmp1 = load i32, i32 addrspace(3)* %ptr1, align 4
 
-;   %vdata = insertelement <4 x i32> undef, i32 %a1, i32 0
-;   call void @llvm.amdgcn.tbuffer.store.v4i32(<4 x i32> %vdata, <4 x i32> undef,
-;         i32 %vaddr, i32 0, i32 0, i32 32, i32 14, i32 4, i1 1, i1 1)
+  %vdata = insertelement <4 x i32> undef, i32 %a1, i32 0
+  %vaddr.add = add i32 %vaddr, 32
+  call void @llvm.amdgcn.struct.tbuffer.store.v4i32(<4 x i32> %vdata, <4 x i32> undef, i32 %vaddr.add, i32 0, i32 0, i32 228, i32 3)
 
-;   %tmp2 = load i32, i32 addrspace(3)* %ptr2, align 4
+  %tmp2 = load i32, i32 addrspace(3)* %ptr2, align 4
 
-;   %add = add nsw i32 %tmp1, %tmp2
+  %add = add nsw i32 %tmp1, %tmp2
+  store i32 %add, i32 addrspace(1)* %out, align 4
+  ret void
+}
 
-;   store i32 %add, i32 addrspace(1)* %out, align 4
-;   ret void
-; }
+declare void @llvm.amdgcn.s.barrier() #1
+declare i32 @llvm.amdgcn.workitem.id.x() #2
+declare void @llvm.amdgcn.struct.tbuffer.store.v4i32(<4 x i32>, <4 x i32>, i32, i32, i32, i32 immarg, i32 immarg) #3
 
 attributes #0 = { nounwind }
-attributes #1 = { nounwind convergent }
-attributes #2 = { nounwind readnone }
+attributes #1 = { convergent nounwind willreturn }
+attributes #2 = { nounwind readnone speculatable willreturn }
+attributes #3 = { nounwind willreturn writeonly }

From ef4e66543541603a426bdc5c170e9ab65e65c42e Mon Sep 17 00:00:00 2001
From: Momchil Velikov <momchil.velikov@arm.com>
Date: Thu, 30 Jul 2020 14:43:15 +0100
Subject: [PATCH 0623/1035] [AArch64] Fix operand definitions of XPACI/XPACD

The operand to these instructions is both input and output.

These are not yet emitted by the compiler and the assembler already
works fine, so can't test in this patch.  But D75044 will use XPACI
and provide test coverage for this patch as well.

Differential Revision: https://reviews.llvm.org/D84298
---
 llvm/lib/Target/AArch64/AArch64InstrFormats.td | 9 +++++++++
 llvm/lib/Target/AArch64/AArch64InstrInfo.td    | 4 ++--
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64InstrFormats.td b/llvm/lib/Target/AArch64/AArch64InstrFormats.td
index e2403c5f6347f..088c129bc5f3c 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrFormats.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrFormats.td
@@ -1937,6 +1937,15 @@ class SignAuthTwoOperand<bits<4> opc, string asm,
   let Inst{4-0}   = Rd;
 }
 
+class ClearAuth<bits<1> data, string asm>
+  : I<(outs GPR64:$Rd), (ins GPR64:$Rn), asm, "\t$Rd", "$Rd = $Rn", []>, Sched<[]> {
+  bits<5> Rd;
+  let Inst{31-11} = 0b110110101100000101000;
+  let Inst{10} = data;
+  let Inst{9-5} = 0b11111;
+  let Inst{4-0} = Rd;
+}
+
 // Base class for the Armv8.4-A 8 and 16-bit flag manipulation instructions
 class BaseFlagManipulation<bit sf, bit sz, dag iops, string asm, string ops>
     : I<(outs), iops, asm, ops, "", []>,
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index 9ea497838b634..61a43ed9df67f 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -994,8 +994,8 @@ let Predicates = [HasPA] in {
   defm PAC : SignAuth<0b000, 0b010, "pac">;
   defm AUT : SignAuth<0b001, 0b011, "aut">;
 
-  def XPACI : SignAuthZero<0b100, 0b00, "xpaci">;
-  def XPACD : SignAuthZero<0b100, 0b01, "xpacd">;
+  def XPACI : ClearAuth<0, "xpaci">;
+  def XPACD : ClearAuth<1, "xpacd">;
   def PACGA : SignAuthTwoOperand<0b1100, "pacga", null_frag>;
 
   // Combined Instructions

From 4e6176fd912a68de0764fff43a129a73f5cab800 Mon Sep 17 00:00:00 2001
From: Xiangling Liao <Xiangling.Liao@ibm.com>
Date: Wed, 29 Jul 2020 14:05:20 -0400
Subject: [PATCH 0624/1035] [AIX] Temporarily disable IncrementalProcessingTest
 partially

Temporarily disable IncrementalProcessingTest partially until the static
initialization implementation on AIX is recovered.

Differential Revision: https://reviews.llvm.org/D84880
---
 clang/unittests/CodeGen/IncrementalProcessingTest.cpp | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/clang/unittests/CodeGen/IncrementalProcessingTest.cpp b/clang/unittests/CodeGen/IncrementalProcessingTest.cpp
index 045ed9bbc7606..d1d921bb03c69 100644
--- a/clang/unittests/CodeGen/IncrementalProcessingTest.cpp
+++ b/clang/unittests/CodeGen/IncrementalProcessingTest.cpp
@@ -159,6 +159,11 @@ TEST(IncrementalProcessing, EmitCXXGlobalInitFunc) {
     // First code should not end up in second module:
     ASSERT_FALSE(M[2]->getFunction("funcForProg1"));
 
+    // TODO: Remove this after the static initialization frontend implementation
+    // is recovered on AIX.
+    if (compiler.getTarget().getTriple().isOSAIX())
+      return;
+
     // Make sure global inits exist and are unique:
     const Function* GlobalInit1 = getGlobalInit(*M[1]);
     ASSERT_TRUE(GlobalInit1);

From b69357c2f4f2aa0c4999d6827a40fe748641fdb1 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev@hotmail.com>
Date: Thu, 30 Jul 2020 10:57:21 -0400
Subject: [PATCH 0625/1035] Revert "[OPENMP]Fix PR46824: Global declare target
 pointer cannot be accessed in target region."

This reverts commit 142d0d3ed8e07aca2476bc4ecc1a12d15577a84a to
investigate undefined behavior revealed by buildbots.
---
 clang/lib/CodeGen/CGOpenMPRuntime.cpp         |  78 ++----
 clang/test/OpenMP/target_data_codegen.cpp     |  18 +-
 .../target_data_use_device_ptr_codegen.cpp    |  48 ++--
 clang/test/OpenMP/target_map_codegen.cpp      |  21 +-
 clang/test/OpenMP/target_update_codegen.cpp   | 242 ++++++++++--------
 openmp/libomptarget/src/omptarget.cpp         |   9 +-
 .../test/env/base_ptr_ref_count.c             |  51 ----
 7 files changed, 212 insertions(+), 255 deletions(-)
 delete mode 100644 openmp/libomptarget/test/env/base_ptr_ref_count.c

diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
index 6c5c54951921e..b8e33948c21ca 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -7392,9 +7392,10 @@ class MappableExprsHandler {
     // &p, &p, sizeof(float*), TARGET_PARAM | TO | FROM
     //
     // map(p[1:24])
-    // &p, &p[1], 24*sizeof(float), TARGET_PARAM | TO | FROM | PTR_AND_OBJ
-    // in unified shared memory mode or for local pointers
     // p, &p[1], 24*sizeof(float), TARGET_PARAM | TO | FROM
+    // for data directives
+    // p, p, sizeof(float*), TARGET_PARAM | TO | FROM
+    // p, &p[1], 24*sizeof(float), PTR_AND_OBJ | TO | FROM
     //
     // map(s)
     // &s, &s, sizeof(S2), TARGET_PARAM | TO | FROM
@@ -7529,7 +7530,6 @@ class MappableExprsHandler {
     // Track if the map information being generated is the first for a list of
     // components.
     bool IsExpressionFirstInfo = true;
-    bool FirstPointerInComplexData = false;
     Address BP = Address::invalid();
     const Expr *AssocExpr = I->getAssociatedExpression();
     const auto *AE = dyn_cast<ArraySubscriptExpr>(AssocExpr);
@@ -7572,16 +7572,17 @@ class MappableExprsHandler {
       QualType Ty =
           I->getAssociatedDeclaration()->getType().getNonReferenceType();
       if (Ty->isAnyPointerType() && std::next(I) != CE) {
-        // No need to generate individual map information for the pointer, it
-        // can be associated with the combined storage if shared memory mode is
-        // active or the base declaration is not global variable.
-        const auto *VD = dyn_cast<VarDecl>(I->getAssociatedDeclaration());
+        BP = CGF.EmitLoadOfPointer(BP, Ty->castAs<PointerType>());
+
+        // For non-data directives, we do not need to generate individual map
+        // information for the pointer, it can be associated with the combined
+        // storage.
         if (CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory() ||
-            !VD || VD->hasLocalStorage())
-          BP = CGF.EmitLoadOfPointer(BP, Ty->castAs<PointerType>());
-        else
-          FirstPointerInComplexData = true;
-        ++I;
+            !CurDir.is<const OMPExecutableDirective *>() ||
+            !isOpenMPTargetDataManagementDirective(
+                CurDir.get<const OMPExecutableDirective *>()
+                    ->getDirectiveKind()))
+          ++I;
       }
     }
 
@@ -7616,19 +7617,8 @@ class MappableExprsHandler {
         EncounteredME = dyn_cast<MemberExpr>(I->getAssociatedExpression());
         // If we encounter a PTR_AND_OBJ entry from now on it should be marked
         // as MEMBER_OF the parent struct.
-        if (EncounteredME) {
+        if (EncounteredME)
           ShouldBeMemberOf = true;
-          // Do not emit as complex pointer if this is actually not array-like
-          // expression.
-          if (FirstPointerInComplexData) {
-            QualType Ty = std::prev(I)
-                              ->getAssociatedDeclaration()
-                              ->getType()
-                              .getNonReferenceType();
-            BP = CGF.EmitLoadOfPointer(BP, Ty->castAs<PointerType>());
-            FirstPointerInComplexData = false;
-          }
-        }
       }
 
       auto Next = std::next(I);
@@ -7770,8 +7760,7 @@ class MappableExprsHandler {
           // (there is a set of entries for each capture).
           OpenMPOffloadMappingFlags Flags =
               getMapTypeBits(MapType, MapModifiers, MotionModifiers, IsImplicit,
-                             !IsExpressionFirstInfo || RequiresReference ||
-                                 FirstPointerInComplexData,
+                             !IsExpressionFirstInfo || RequiresReference,
                              IsCaptureFirstInfo && !RequiresReference);
 
           if (!IsExpressionFirstInfo) {
@@ -7830,7 +7819,6 @@ class MappableExprsHandler {
 
         IsExpressionFirstInfo = false;
         IsCaptureFirstInfo = false;
-        FirstPointerInComplexData = false;
       }
     }
   }
@@ -8079,7 +8067,6 @@ class MappableExprsHandler {
     // emission of that entry until the whole struct has been processed.
     llvm::MapVector<const ValueDecl *, SmallVector<DeferredDevicePtrEntryTy, 4>>
         DeferredInfo;
-    MapCombinedInfoTy UseDevicePtrCombinedInfo;
 
     for (const auto *C :
          CurExecDir->getClausesOfKind<OMPUseDevicePtrClause>()) {
@@ -8099,24 +8086,13 @@ class MappableExprsHandler {
         // We potentially have map information for this declaration already.
         // Look for the first set of components that refer to it.
         if (It != Info.end()) {
-          auto *CI = llvm::find_if(It->second, [VD](const MapInfo &MI) {
-            return MI.Components.back().getAssociatedDeclaration() == VD;
-          });
+          auto CI = std::find_if(
+              It->second.begin(), It->second.end(), [VD](const MapInfo &MI) {
+                return MI.Components.back().getAssociatedDeclaration() == VD;
+              });
           // If we found a map entry, signal that the pointer has to be returned
           // and move on to the next declaration.
-          // Exclude cases where the base pointer is mapped as array subscript,
-          // array section or array shaping. The base address is passed as a
-          // pointer to base in this case and cannot be used as a base for
-          // use_device_ptr list item.
-          auto PrevCI = std::next(CI->Components.rbegin());
-          const auto *VarD = dyn_cast<VarDecl>(VD);
-          if (CI != It->second.end() &&
-              (CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory() ||
-               isa<MemberExpr>(IE) ||
-               !VD->getType().getNonReferenceType()->isPointerType() ||
-               PrevCI == CI->Components.rend() ||
-               isa<MemberExpr>(PrevCI->getAssociatedExpression()) || !VarD ||
-               VarD->hasLocalStorage())) {
+          if (CI != It->second.end()) {
             CI->ReturnDevicePointer = true;
             continue;
           }
@@ -8139,13 +8115,13 @@ class MappableExprsHandler {
         } else {
           llvm::Value *Ptr =
               CGF.EmitLoadOfScalar(CGF.EmitLValue(IE), IE->getExprLoc());
-          UseDevicePtrCombinedInfo.BasePointers.emplace_back(Ptr, VD);
-          UseDevicePtrCombinedInfo.Pointers.push_back(Ptr);
-          UseDevicePtrCombinedInfo.Sizes.push_back(
+          CombinedInfo.BasePointers.emplace_back(Ptr, VD);
+          CombinedInfo.Pointers.push_back(Ptr);
+          CombinedInfo.Sizes.push_back(
               llvm::Constant::getNullValue(CGF.Int64Ty));
-          UseDevicePtrCombinedInfo.Types.push_back(OMP_MAP_RETURN_PARAM |
-                                                   OMP_MAP_TARGET_PARAM);
-          UseDevicePtrCombinedInfo.Mappers.push_back(nullptr);
+          CombinedInfo.Types.push_back(OMP_MAP_RETURN_PARAM |
+                                       OMP_MAP_TARGET_PARAM);
+          CombinedInfo.Mappers.push_back(nullptr);
         }
       }
     }
@@ -8297,8 +8273,6 @@ class MappableExprsHandler {
       // We need to append the results of this capture to what we already have.
       CombinedInfo.append(CurInfo);
     }
-    // Append data for use_device_ptr clauses.
-    CombinedInfo.append(UseDevicePtrCombinedInfo);
   }
 
   /// Generate all the base pointers, section pointers, sizes, map types, and
diff --git a/clang/test/OpenMP/target_data_codegen.cpp b/clang/test/OpenMP/target_data_codegen.cpp
index a2bb8cdf5ba88..80e674c0b49d9 100644
--- a/clang/test/OpenMP/target_data_codegen.cpp
+++ b/clang/test/OpenMP/target_data_codegen.cpp
@@ -555,7 +555,7 @@ struct S2 {
 
 void test_close_modifier(int arg) {
   S2 *ps;
-  // CK5: private unnamed_addr constant [5 x i64] [i64 1059, i64 32, i64 562949953421328, i64 16, i64 1043]
+  // CK5: private unnamed_addr constant [6 x i64] [i64 1059, i64 32, i64 562949953422339, i64 562949953421328, i64 16, i64 1043]
   #pragma omp target data map(close,tofrom: arg, ps->ps->ps->ps->s)
   {
     ++(arg);
@@ -634,17 +634,20 @@ void test_present_modifier(int arg) {
   // Make sure the struct picks up present even if another element of the struct
   // doesn't have present.
 
-  // CK8: private unnamed_addr constant [11 x i64]
+  // CK8: private unnamed_addr constant [15 x i64]
 
   // ps1
   //
   // PRESENT=0x1000 | TARGET_PARAM=0x20 = 0x1020
   // MEMBER_OF_1=0x1000000000000 | FROM=0x2 | TO=0x1 = 0x1000000000003
+  // MEMBER_OF_1=0x1000000000000 | PTR_AND_OBJ=0x10 | FROM=0x2 | TO=0x1 = 0x1000000000013
+  // MEMBER_OF_1=0x1000000000000 | PRESENT=0x1000 | FROM=0x2 | TO=0x1 = 0x1000000001003
   // MEMBER_OF_1=0x1000000000000 | PRESENT=0x1000 | PTR_AND_OBJ=0x10 = 0x1000000001010
   // PRESENT=0x1000 | PTR_AND_OBJ=0x10 = 0x1010
   // PRESENT=0x1000 | PTR_AND_OBJ=0x10 | FROM=0x2 | TO=0x1 = 0x1013
   //
   // CK8-SAME: {{^}} [i64 [[#0x1020]], i64 [[#0x1000000000003]],
+  // CK8-SAME: {{^}} i64 [[#0x1000000000013]], i64 [[#0x1000000001003]],
   // CK8-SAME: {{^}} i64 [[#0x1000000001010]], i64 [[#0x1010]], i64 [[#0x1013]],
 
   // arg
@@ -656,13 +659,16 @@ void test_present_modifier(int arg) {
   // ps2
   //
   // PRESENT=0x1000 | TARGET_PARAM=0x20 = 0x1020
-  // MEMBER_OF_7=0x7000000000000 | PRESENT=0x1000 | FROM=0x2 | TO=0x1 = 0x7000000001003
-  // MEMBER_OF_7=0x7000000000000 | PTR_AND_OBJ=0x10 = 0x7000000000010
+  // MEMBER_OF_9=0x9000000000000 | PRESENT=0x1000 | FROM=0x2 | TO=0x1 = 0x9000000001003
+  // MEMBER_OF_9=0x9000000000000 | PRESENT=0x1000 | PTR_AND_OBJ=0x10 | FROM=0x2 | TO=0x1 = 0x9000000001013
+  // MEMBER_OF_9=0x9000000000000 | FROM=0x2 | TO=0x1 = 0x9000000000003
+  // MEMBER_OF_9=0x9000000000000 | PTR_AND_OBJ=0x10 = 0x9000000000010
   // PTR_AND_OBJ=0x10 = 0x10
   // PTR_AND_OBJ=0x10 | FROM=0x2 | TO=0x1 = 0x13
   //
-  // CK8-SAME: {{^}} i64 [[#0x1020]], i64 [[#0x7000000001003]],
-  // CK8-SAME: {{^}} i64 [[#0x7000000000010]], i64 [[#0x10]], i64 [[#0x13]]]
+  // CK8-SAME: {{^}} i64 [[#0x1020]], i64 [[#0x9000000001003]],
+  // CK8-SAME: {{^}} i64 [[#0x9000000001013]], i64 [[#0x9000000000003]],
+  // CK8-SAME: {{^}} i64 [[#0x9000000000010]], i64 [[#0x10]], i64 [[#0x13]]]
   #pragma omp target data map(tofrom: ps1->s) \
                           map(present,tofrom: arg, ps1->ps->ps->ps->s, ps2->s) \
                           map(tofrom: ps2->ps->ps->ps->s)
diff --git a/clang/test/OpenMP/target_data_use_device_ptr_codegen.cpp b/clang/test/OpenMP/target_data_use_device_ptr_codegen.cpp
index ca5536f927a10..fe6ea01b43c91 100644
--- a/clang/test/OpenMP/target_data_use_device_ptr_codegen.cpp
+++ b/clang/test/OpenMP/target_data_use_device_ptr_codegen.cpp
@@ -22,18 +22,18 @@
 double *g;
 
 // CK1: @g = global double*
-// CK1: [[MTYPE00:@.+]] = {{.*}}constant [2 x i64] [i64 51, i64 96]
-// CK1: [[MTYPE01:@.+]] = {{.*}}constant [1 x i64] [i64 99]
-// CK1: [[MTYPE03:@.+]] = {{.*}}constant [1 x i64] [i64 99]
-// CK1: [[MTYPE04:@.+]] = {{.*}}constant [1 x i64] [i64 99]
-// CK1: [[MTYPE05:@.+]] = {{.*}}constant [1 x i64] [i64 99]
-// CK1: [[MTYPE06:@.+]] = {{.*}}constant [1 x i64] [i64 99]
-// CK1: [[MTYPE07:@.+]] = {{.*}}constant [1 x i64] [i64 99]
-// CK1: [[MTYPE08:@.+]] = {{.*}}constant [2 x i64] [i64 99, i64 35]
-// CK1: [[MTYPE09:@.+]] = {{.*}}constant [2 x i64] [i64 99, i64 99]
-// CK1: [[MTYPE10:@.+]] = {{.*}}constant [2 x i64] [i64 99, i64 99]
-// CK1: [[MTYPE11:@.+]] = {{.*}}constant [2 x i64] [i64 35, i64 96]
-// CK1: [[MTYPE12:@.+]] = {{.*}}constant [2 x i64] [i64 35, i64 96]
+// CK1: [[MTYPE00:@.+]] = {{.*}}constant [2 x i64] [i64 99, i64 19]
+// CK1: [[MTYPE01:@.+]] = {{.*}}constant [2 x i64] [i64 99, i64 19]
+// CK1: [[MTYPE03:@.+]] = {{.*}}constant [2 x i64] [i64 99, i64 19]
+// CK1: [[MTYPE04:@.+]] = {{.*}}constant [2 x i64] [i64 99, i64 19]
+// CK1: [[MTYPE05:@.+]] = {{.*}}constant [2 x i64] [i64 99, i64 19]
+// CK1: [[MTYPE06:@.+]] = {{.*}}constant [2 x i64] [i64 99, i64 19]
+// CK1: [[MTYPE07:@.+]] = {{.*}}constant [2 x i64] [i64 99, i64 19]
+// CK1: [[MTYPE08:@.+]] = {{.*}}constant [4 x i64] [i64 99, i64 19, i64 35, i64 19]
+// CK1: [[MTYPE09:@.+]] = {{.*}}constant [4 x i64] [i64 99, i64 19, i64 99, i64 19]
+// CK1: [[MTYPE10:@.+]] = {{.*}}constant [4 x i64] [i64 99, i64 19, i64 99, i64 19]
+// CK1: [[MTYPE11:@.+]] = {{.*}}constant [3 x i64] [i64 96, i64 35, i64 19]
+// CK1: [[MTYPE12:@.+]] = {{.*}}constant [3 x i64] [i64 96, i64 35, i64 19]
 
 // CK1-LABEL: @_Z3foo
 template<typename T>
@@ -42,7 +42,7 @@ void foo(float *&lr, T *&tr) {
   T *t;
 
   // CK1:     [[T:%.+]] = load double*, double** [[DECL:@g]],
-  // CK1:     [[BP:%.+]] = getelementptr inbounds [2 x i8*], [2 x i8*]* %{{.+}}, i32 0, i32 1
+  // CK1:     [[BP:%.+]] = getelementptr inbounds [2 x i8*], [2 x i8*]* %{{.+}}, i32 0, i32 0
   // CK1:     [[CBP:%.+]] = bitcast i8** [[BP]] to double**
   // CK1:     store double* [[T]], double** [[CBP]],
   // CK1:     call void @__tgt_target_data_begin{{.+}}[[MTYPE00]]
@@ -61,7 +61,7 @@ void foo(float *&lr, T *&tr) {
   ++g;
 
   // CK1:     [[T1:%.+]] = load float*, float** [[DECL:%.+]],
-  // CK1:     [[BP:%.+]] = getelementptr inbounds [1 x i8*], [1 x i8*]* %{{.+}}, i32 0, i32 0
+  // CK1:     [[BP:%.+]] = getelementptr inbounds [2 x i8*], [2 x i8*]* %{{.+}}, i32 0, i32 0
   // CK1:     [[CBP:%.+]] = bitcast i8** [[BP]] to float**
   // CK1:     store float* [[T1]], float** [[CBP]],
   // CK1:     call void @__tgt_target_data_begin{{.+}}[[MTYPE01]]
@@ -92,7 +92,7 @@ void foo(float *&lr, T *&tr) {
   ++l;
 
   // CK1:     [[T1:%.+]] = load float*, float** [[DECL:%.+]],
-  // CK1:     [[BP:%.+]] = getelementptr inbounds [1 x i8*], [1 x i8*]* %{{.+}}, i32 0, i32 0
+  // CK1:     [[BP:%.+]] = getelementptr inbounds [2 x i8*], [2 x i8*]* %{{.+}}, i32 0, i32 0
   // CK1:     [[CBP:%.+]] = bitcast i8** [[BP]] to float**
   // CK1:     store float* [[T1]], float** [[CBP]],
   // CK1:     call void @__tgt_target_data_begin{{.+}}[[MTYPE03]]
@@ -115,7 +115,7 @@ void foo(float *&lr, T *&tr) {
 
   // CK1:     [[BTHEN]]:
   // CK1:     [[T1:%.+]] = load float*, float** [[DECL:%.+]],
-  // CK1:     [[BP:%.+]] = getelementptr inbounds [1 x i8*], [1 x i8*]* %{{.+}}, i32 0, i32 0
+  // CK1:     [[BP:%.+]] = getelementptr inbounds [2 x i8*], [2 x i8*]* %{{.+}}, i32 0, i32 0
   // CK1:     [[CBP:%.+]] = bitcast i8** [[BP]] to float**
   // CK1:     store float* [[T1]], float** [[CBP]],
   // CK1:     call void @__tgt_target_data_begin{{.+}}[[MTYPE04]]
@@ -152,7 +152,7 @@ void foo(float *&lr, T *&tr) {
 
   // CK1:     [[T2:%.+]] = load float**, float*** [[DECL:%.+]],
   // CK1:     [[T1:%.+]] = load float*, float** [[T2]],
-  // CK1:     [[BP:%.+]] = getelementptr inbounds [1 x i8*], [1 x i8*]* %{{.+}}, i32 0, i32 0
+  // CK1:     [[BP:%.+]] = getelementptr inbounds [2 x i8*], [2 x i8*]* %{{.+}}, i32 0, i32 0
   // CK1:     [[CBP:%.+]] = bitcast i8** [[BP]] to float**
   // CK1:     store float* [[T1]], float** [[CBP]],
   // CK1:     call void @__tgt_target_data_begin{{.+}}[[MTYPE05]]
@@ -174,7 +174,7 @@ void foo(float *&lr, T *&tr) {
   ++lr;
 
   // CK1:     [[T1:%.+]] = load i32*, i32** [[DECL:%.+]],
-  // CK1:     [[BP:%.+]] = getelementptr inbounds [1 x i8*], [1 x i8*]* %{{.+}}, i32 0, i32 0
+  // CK1:     [[BP:%.+]] = getelementptr inbounds [2 x i8*], [2 x i8*]* %{{.+}}, i32 0, i32 0
   // CK1:     [[CBP:%.+]] = bitcast i8** [[BP]] to i32**
   // CK1:     store i32* [[T1]], i32** [[CBP]],
   // CK1:     call void @__tgt_target_data_begin{{.+}}[[MTYPE06]]
@@ -194,7 +194,7 @@ void foo(float *&lr, T *&tr) {
 
   // CK1:     [[T2:%.+]] = load i32**, i32*** [[DECL:%.+]],
   // CK1:     [[T1:%.+]] = load i32*, i32** [[T2]],
-  // CK1:     [[BP:%.+]] = getelementptr inbounds [1 x i8*], [1 x i8*]* %{{.+}}, i32 0, i32 0
+  // CK1:     [[BP:%.+]] = getelementptr inbounds [2 x i8*], [2 x i8*]* %{{.+}}, i32 0, i32 0
   // CK1:     [[CBP:%.+]] = bitcast i8** [[BP]] to i32**
   // CK1:     store i32* [[T1]], i32** [[CBP]],
   // CK1:     call void @__tgt_target_data_begin{{.+}}[[MTYPE07]]
@@ -216,7 +216,7 @@ void foo(float *&lr, T *&tr) {
   ++tr;
 
   // CK1:     [[T1:%.+]] = load float*, float** [[DECL:%.+]],
-  // CK1:     [[BP:%.+]] = getelementptr inbounds [2 x i8*], [2 x i8*]* %{{.+}}, i32 0, i32 0
+  // CK1:     [[BP:%.+]] = getelementptr inbounds [4 x i8*], [4 x i8*]* %{{.+}}, i32 0, i32
   // CK1:     [[CBP:%.+]] = bitcast i8** [[BP]] to float**
   // CK1:     store float* [[T1]], float** [[CBP]],
   // CK1:     call void @__tgt_target_data_begin{{.+}}[[MTYPE08]]
@@ -280,7 +280,7 @@ void foo(float *&lr, T *&tr) {
   ++l; ++t;
 
   // CK1:     [[T1:%.+]] = load i32*, i32** [[DECL:%.+]],
-  // CK1:     [[BP:%.+]] = getelementptr inbounds [2 x i8*], [2 x i8*]* %{{.+}}, i32 0, i32 1
+  // CK1:     [[BP:%.+]] = getelementptr inbounds [3 x i8*], [3 x i8*]* %{{.+}}, i32 0, i32 0
   // CK1:     [[CBP:%.+]] = bitcast i8** [[BP]] to i32**
   // CK1:     store i32* [[T1]], i32** [[CBP]],
   // CK1:     call void @__tgt_target_data_begin{{.+}}[[MTYPE11]]
@@ -300,7 +300,7 @@ void foo(float *&lr, T *&tr) {
 
   // CK1:     [[T2:%.+]] = load i32**, i32*** [[DECL:%.+]],
   // CK1:     [[T1:%.+]] = load i32*, i32** [[T2]],
-  // CK1:     [[BP:%.+]] = getelementptr inbounds [2 x i8*], [2 x i8*]* %{{.+}}, i32 0, i32 1
+  // CK1:     [[BP:%.+]] = getelementptr inbounds [3 x i8*], [3 x i8*]* %{{.+}}, i32 0, i32 0
   // CK1:     [[CBP:%.+]] = bitcast i8** [[BP]] to i32**
   // CK1:     store i32* [[T1]], i32** [[CBP]],
   // CK1:     call void @__tgt_target_data_begin{{.+}}[[MTYPE12]]
@@ -348,7 +348,7 @@ void bar(float *&a, int *&b) {
 // CK2: [[ST:%.+]] = type { double*, double** }
 // CK2: [[MTYPE00:@.+]] = {{.*}}constant [2 x i64] [i64 32, i64 281474976710739]
 // CK2: [[MTYPE01:@.+]] = {{.*}}constant [2 x i64] [i64 32, i64 281474976710739]
-// CK2: [[MTYPE02:@.+]] = {{.*}}constant [3 x i64] [i64 35, i64 32, i64 562949953421392]
+// CK2: [[MTYPE02:@.+]] = {{.*}}constant [4 x i64] [i64 35, i64 19, i64 32, i64 844424930132048]
 // CK2: [[MTYPE03:@.+]] = {{.*}}constant [3 x i64] [i64 32, i64 281474976710739, i64 281474976710736]
 
 template <typename T>
@@ -404,7 +404,7 @@ struct ST {
     // CK2:     getelementptr inbounds double, double* [[TTTT]], i32 1
     b++;
 
-    // CK2:     [[BP:%.+]] = getelementptr inbounds [3 x i8*], [3 x i8*]* %{{.+}}, i32 0, i32 2
+    // CK2:     [[BP:%.+]] = getelementptr inbounds [4 x i8*], [4 x i8*]* %{{.+}}, i32 0, i32 3
     // CK2:     [[CBP:%.+]] = bitcast i8** [[BP]] to double***
     // CK2:     store double** [[RVAL:%.+]], double*** [[CBP]],
     // CK2:     call void @__tgt_target_data_begin{{.+}}[[MTYPE02]]
diff --git a/clang/test/OpenMP/target_map_codegen.cpp b/clang/test/OpenMP/target_map_codegen.cpp
index a394f5b1c3d53..e63f19ad73c0a 100644
--- a/clang/test/OpenMP/target_map_codegen.cpp
+++ b/clang/test/OpenMP/target_map_codegen.cpp
@@ -3874,7 +3874,7 @@ int explicit_maps_template_args_and_members(int a){
 
 // CK22-LABEL: @.__omp_offloading_{{.*}}explicit_maps_globals{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK22: [[SIZE04:@.+]] = private {{.*}}constant [1 x i64] [i64 20]
-// CK22: [[MTYPE04:@.+]] = private {{.*}}constant [1 x i64] [i64 51]
+// CK22: [[MTYPE04:@.+]] = private {{.*}}constant [1 x i64] [i64 35]
 
 // CK22-LABEL: @.__omp_offloading_{{.*}}explicit_maps_globals{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK22: [[SIZE05:@.+]] = private {{.*}}constant [1 x i64] [i64 4]
@@ -3894,7 +3894,7 @@ int explicit_maps_template_args_and_members(int a){
 
 // CK22-LABEL: @.__omp_offloading_{{.*}}explicit_maps_globals{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK22: [[SIZE09:@.+]] = private {{.*}}constant [1 x i64] [i64 20]
-// CK22: [[MTYPE09:@.+]] = private {{.*}}constant [1 x i64] [i64 51]
+// CK22: [[MTYPE09:@.+]] = private {{.*}}constant [1 x i64] [i64 35]
 
 // CK22-LABEL: @.__omp_offloading_{{.*}}explicit_maps_globals{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK22: [[SIZE10:@.+]] = private {{.*}}constant [1 x i64] [i64 4]
@@ -3914,7 +3914,7 @@ int explicit_maps_template_args_and_members(int a){
 
 // CK22-LABEL: @.__omp_offloading_{{.*}}explicit_maps_globals{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK22: [[SIZE14:@.+]] = private {{.*}}constant [1 x i64] [i64 20]
-// CK22: [[MTYPE14:@.+]] = private {{.*}}constant [1 x i64] [i64 51]
+// CK22: [[MTYPE14:@.+]] = private {{.*}}constant [1 x i64] [i64 35]
 
 int a;
 int c[100];
@@ -4010,10 +4010,11 @@ int explicit_maps_globals(void){
 
   // CK22-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 0
   // CK22-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 0
-  // CK22-DAG: [[CBP0:%.+]] = bitcast i8** [[BP0]] to i32***
+  // CK22-DAG: [[CBP0:%.+]] = bitcast i8** [[BP0]] to i32**
   // CK22-DAG: [[CP0:%.+]] = bitcast i8** [[P0]] to i32**
-  // CK22-DAG: store i32** @d, i32*** [[CBP0]]
+  // CK22-DAG: store i32* [[RVAR0:%.+]], i32** [[CBP0]]
   // CK22-DAG: store i32* [[SEC0:%.+]], i32** [[CP0]]
+  // CK22-DAG: [[RVAR0]] = load i32*, i32** @d
   // CK22-DAG: [[SEC0]] = getelementptr {{.*}}i32* [[RVAR00:%.+]], i{{.+}} 2
   // CK22-DAG: [[RVAR00]] = load i32*, i32** @d
 
@@ -4092,10 +4093,11 @@ int explicit_maps_globals(void){
 
   // CK22-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 0
   // CK22-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 0
-  // CK22-DAG: [[CBP0:%.+]] = bitcast i8** [[BP0]] to [[ST]]***
+  // CK22-DAG: [[CBP0:%.+]] = bitcast i8** [[BP0]] to [[ST]]**
   // CK22-DAG: [[CP0:%.+]] = bitcast i8** [[P0]] to [[ST]]**
-  // CK22-DAG: store [[ST]]** @sd, [[ST]]*** [[CBP0]]
+  // CK22-DAG: store [[ST]]* [[RVAR0:%.+]], [[ST]]** [[CBP0]]
   // CK22-DAG: store [[ST]]* [[SEC0:%.+]], [[ST]]** [[CP0]]
+  // CK22-DAG: [[RVAR0]] = load [[ST]]*, [[ST]]** @sd
   // CK22-DAG: [[SEC0]] = getelementptr {{.*}}[[ST]]* [[RVAR00:%.+]], i{{.+}} 2
   // CK22-DAG: [[RVAR00]] = load [[ST]]*, [[ST]]** @sd
 
@@ -4174,10 +4176,11 @@ int explicit_maps_globals(void){
 
   // CK22-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 0
   // CK22-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 0
-  // CK22-DAG: [[CBP0:%.+]] = bitcast i8** [[BP0]] to [[STT]]***
+  // CK22-DAG: [[CBP0:%.+]] = bitcast i8** [[BP0]] to [[STT]]**
   // CK22-DAG: [[CP0:%.+]] = bitcast i8** [[P0]] to [[STT]]**
-  // CK22-DAG: store [[STT]]** @std, [[STT]]*** [[CBP0]]
+  // CK22-DAG: store [[STT]]* [[RVAR0:%.+]], [[STT]]** [[CBP0]]
   // CK22-DAG: store [[STT]]* [[SEC0:%.+]], [[STT]]** [[CP0]]
+  // CK22-DAG: [[RVAR0]] = load [[STT]]*, [[STT]]** @std
   // CK22-DAG: [[SEC0]] = getelementptr {{.*}}[[STT]]* [[RVAR00:%.+]], i{{.+}} 2
   // CK22-DAG: [[RVAR00]] = load [[STT]]*, [[STT]]** @std
 
diff --git a/clang/test/OpenMP/target_update_codegen.cpp b/clang/test/OpenMP/target_update_codegen.cpp
index 9eab8b3367c4e..feea1129be4cc 100644
--- a/clang/test/OpenMP/target_update_codegen.cpp
+++ b/clang/test/OpenMP/target_update_codegen.cpp
@@ -310,23 +310,22 @@ void device_side_scan(int arg) {
 
 #ifdef CK5
 
-// CK5: [[SIZE00:@.+]] = {{.+}}constant [1 x i[[sz:64|32]]] [i{{64|32}} 4]
-// CK5: [[MTYPE00:@.+]] = {{.+}}constant [1 x i64] [i64 33]
+// CK5: [[SIZE00:@.+]] = {{.+}}constant [2 x i[[sz:64|32]]] [i{{64|32}} {{8|4}}, i{{64|32}} 4]
+// CK5: [[MTYPE00:@.+]] = {{.+}}constant [2 x i64] [i64 33, i64 17]
 
 // CK5-LABEL: lvalue
 void lvalue(int *B, int l, int e) {
 
-  // CK5-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 1, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[SIZE00]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
+  // CK5-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 2, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}]* [[SIZE00]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
   // CK5-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
   // CK5-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
 
-  // CK5-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 0
-  // CK5-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 0
-  // CK5-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to i32**
+  // CK5-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
+  // CK5-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
+  // CK5-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to i32***
   // CK5-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to i32**
-  // CK5-DAG: store i32* [[B_VAL:%.+]], i32** [[BPC0]]
+  // CK5-DAG: store i32** [[B_ADDR:%.+]], i32*** [[BPC0]]
   // CK5-DAG: store i32* [[B_VAL_2:%.+]], i32** [[PC0]]
-  // CK5-DAG: [[B_VAL]] = load i32*, i32** [[B_ADDR:%.+]]
   // CK5-DAG: [[B_VAL_2]] = load i32*, i32** [[B_ADDR]]
   #pragma omp target update to(*B)
   *B += e;
@@ -352,29 +351,28 @@ void lvalue(int *B, int l, int e) {
 
 #ifdef CK6
 
-// CK6: [[SIZE00:@.+]] = {{.+}}constant [1 x i[[sz:64|32]]] [i{{64|32}} 4]
-// CK6: [[MTYPE00:@.+]] = {{.+}}constant [1 x i64] [i64 33]
+// CK6: [[SIZE00:@.+]] = {{.+}}constant [2 x i[[sz:64|32]]] [i{{64|32}} {{8|4}}, i{{64|32}} 4]
+// CK6: [[MTYPE00:@.+]] = {{.+}}constant [2 x i64] [i64 33, i64 17]
 
 // CK6-LABEL: lvalue
 void lvalue(int *B, int l, int e) {
 
-  // CK6-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 1, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[SIZE00]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
+  // CK6-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 2, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}]* [[SIZE00]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
   // CK6-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
   // CK6-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
 
-  // CK6-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 0
-  // CK6-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 0
-  // CK6-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to i32**
+  // CK6-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
+  // CK6-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
+  // CK6-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to i32***
   // CK6-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to i32**
-  // CK6-DAG: store i32* [[TWO:%.+]], i32** [[BPC0]]
+  // CK6-DAG: store i32** [[B_ADDR:%.+]], i32*** [[BPC0]]
   // CK6-DAG: store i32* [[ADD_PTR:%.+]], i32** [[PC0]]
   // CK6-64-DAG: [[ADD_PTR]] = getelementptr inbounds i32, i32* [[ONE:%.+]], i{{32|64}} [[IDX_EXT:%.+]]
   // CK6-32-DAG: [[ADD_PTR]] = getelementptr inbounds i32, i32* [[ONE:%.+]], i{{32|64}} [[L_VAL:%.+]]
   // CK6-64-DAG: [[IDX_EXT]] = sext i32 [[L_VAL:%.+]] to i64
   // CK6-DAG: [[L_VAL]] = load i32, i32* [[L_ADDR:%.+]]
   // CK6-DAG: store i32 {{.+}}, i32* [[L_ADDR]]
-  // CK6-DAG: [[ONE]] = load i32*, i32** [[B_ADDR:%.+]]
-  // CK6-DAG: [[TWO]] = load i32*, i32** [[B_ADDR]]
+  // CK6-DAG: [[ONE]] = load i32*, i32** [[B_ADDR]]
   #pragma omp target update to(*(B+l))
   *(B+l) += e;
   #pragma omp target update from(*(B+l))
@@ -399,26 +397,25 @@ void lvalue(int *B, int l, int e) {
 
 #ifdef CK7
 
-// CK7: [[SIZE00:@.+]] = {{.+}}constant [1 x i[[sz:64|32]]] [i{{64|32}} 4]
-// CK7: [[MTYPE00:@.+]] = {{.+}}constant [1 x i64] [i64 33]
+// CK7: [[SIZE00:@.+]] = {{.+}}constant [2 x i[[sz:64|32]]] [i{{64|32}} {{8|4}}, i{{64|32}} 4]
+// CK7: [[MTYPE00:@.+]] = {{.+}}constant [2 x i64] [i64 33, i64 17]
 
 // CK7-LABEL: lvalue
 void lvalue(int *B, int l, int e) {
 
-  // CK7-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 1, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[SIZE00]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
+  // CK7-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 2, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}]* [[SIZE00]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
   // CK7-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
   // CK7-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
 
-  // CK7-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 0
-  // CK7-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 0
-  // CK7-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to i32**
+  // CK7-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
+  // CK7-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
+  // CK7-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to i32***
   // CK7-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to i32**
-  // CK7-DAG: store i32* [[B_VAL:%.+]], i32** [[BPC0]]
+  // CK7-DAG: store i32** [[B_ADDR:%.+]], i32*** [[BPC0]]
   // CK7-DAG: store i32* [[ARRAY_IDX:%.+]], i32** [[PC0]]
   // CK7-DAG: [[ARRAY_IDX]] = getelementptr inbounds i32, i32* [[ADD_PTR:%.+]], i{{32|64}} [[IDX_PROM:%.+]]
   // CK7-64-DAG: [[ADD_PTR]] = getelementptr inbounds i32, i32* [[ONE:%.+]], i64 [[IDX_EXT:%.+]]
   // CK7-32-DAG: [[ADD_PTR]] = getelementptr inbounds i32, i32* [[B_VAL_2:%.+]], i32 [[L_VAL:%.+]]
-  // CK7-32-DAG: [[B_VAL]] = load i32*, i32** [[B_ADDR:%.+]]
   // CK7-32-DAG: [[B_VAL_2]] = load i32*, i32** [[B_ADDR]]
   // CK7-32-DAG: [[L_VAL]] = load i32, i32* [[L_ADDR:%.+]]
   // CK7-32-DAG: [[IDX_PROM]] = load i32, i32* [[L_ADDR]]
@@ -449,18 +446,18 @@ void lvalue(int *B, int l, int e) {
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 
 #ifdef CK8
-// CK8: [[SIZE00:@.+]] = {{.+}}constant [2 x i[[sz:64|32]]] [i{{64|32}} {{8|4}}, i{{64|32}} 4]
-// CK8: [[MTYPE00:@.+]] = {{.+}}constant [2 x i64] [i64 33, i64 17]
+// CK8: [[SIZE00:@.+]] = {{.+}}constant [3 x i[[sz:64|32]]] [i{{64|32}} {{8|4}}, i{{64|32}} {{8|4}}, i{{64|32}} 4]
+// CK8: [[MTYPE00:@.+]] = {{.+}}constant [3 x i64] [i64 33, i64 16, i64 17]
 
 // CK8-LABEL: lvalue
 void lvalue(int **B, int l, int e) {
 
-  // CK8-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 2, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}], [2 x i{{.+}}]* [[SIZE00]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}, [2 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
+  // CK8-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 3, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], {{.+}}getelementptr {{.+}}[3 x i{{.+}}], [3 x i{{.+}}]* [[SIZE00]], {{.+}}getelementptr {{.+}}[3 x i{{.+}}, [3 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
   // CK8-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
   // CK8-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
 
-  // CK8-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
-  // CK8-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
+  // CK8-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 2
+  // CK8-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 2
   // CK8-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to i32***
   // CK8-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to i32**
   // CK8-DAG: store i32** [[ARRAY_IDX_1:%.+]], i32*** [[BPC0]]
@@ -504,19 +501,19 @@ struct S {
   double *p;
 };
 
-// CK9: [[MTYPE00:@.+]] = {{.+}}constant [2 x i64] [i64 32, i64 281474976710673]
+// CK9: [[MTYPE00:@.+]] = {{.+}}constant [3 x i64] [i64 32, i64 281474976710657, i64 281474976710673]
 
 // CK9-LABEL: lvalue
 void lvalue(struct S *s, int l, int e) {
 
-  // CK9-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 2, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i{{.+}} [[GSIZE:%.+]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}, [2 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
+  // CK9-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 3, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i{{.+}} [[GSIZE:%.+]], {{.+}}getelementptr {{.+}}[3 x i{{.+}}, [3 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
   // CK9-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
   // CK9-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
   // CK9-DAG: [[GSIZE]] = getelementptr inbounds {{.+}}[[SIZE:%[^,]+]]
   //
-  // CK9-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
-  // CK9-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
-  // CK9-DAG: [[SIZE0:%.+]] = getelementptr inbounds {{.+}}[[SIZE]], i{{.+}} 0, i{{.+}} 1
+  // CK9-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 2
+  // CK9-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 2
+  // CK9-DAG: [[SIZE0:%.+]] = getelementptr inbounds {{.+}}[[SIZE]], i{{.+}} 0, i{{.+}} 2
   // CK9-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to double***
   // CK9-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to double**
   // CK9-DAG: store double** [[P:%.+]], double*** [[BPC0]]
@@ -554,19 +551,19 @@ struct S {
   double *p;
 };
 
-// CK10: [[MTYPE00:@.+]] = {{.+}}constant [2 x i64] [i64 32, i64 281474976710673]
+// CK10: [[MTYPE00:@.+]] = {{.+}}constant [3 x i64] [i64 32, i64 281474976710657, i64 281474976710673]
 
 // CK10-LABEL: lvalue
 void lvalue(struct S *s, int l, int e) {
 
-  // CK10-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 2, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i{{.+}} [[GSIZE:%.+]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}, [2 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
+  // CK10-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 3, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i{{.+}} [[GSIZE:%.+]], {{.+}}getelementptr {{.+}}[3 x i{{.+}}, [3 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
   // CK10-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
   // CK10-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
   // CK10-DAG: [[GSIZE]] = getelementptr inbounds {{.+}}[[SIZE:%[^,]+]]
   //
-  // CK10-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
-  // CK10-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
-  // CK10-DAG: [[SIZE0:%.+]] = getelementptr inbounds {{.+}}[[SIZE]], i{{.+}} 0, i{{.+}} 1
+  // CK10-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 2
+  // CK10-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 2
+  // CK10-DAG: [[SIZE0:%.+]] = getelementptr inbounds {{.+}}[[SIZE]], i{{.+}} 0, i{{.+}} 2
   // CK10-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to double***
   // CK10-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to double**
   // CK10-DAG: store double** [[P_VAL:%.+]], double*** [[BPC0]]
@@ -604,19 +601,19 @@ void lvalue(struct S *s, int l, int e) {
 struct S {
   double *p;
 };
-// CK11: [[MTYPE00:@.+]] = {{.+}}constant [2 x i64] [i64 32, i64 281474976710673]
+// CK11: [[MTYPE00:@.+]] = {{.+}}constant [3 x i64] [i64 32, i64 281474976710657, i64 281474976710673]
 
 // CK11-LABEL: lvalue
 void lvalue(struct S *s, int l, int e) {
 
-  // CK11-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 2, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i{{.+}} [[GSIZE:%.+]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}, [2 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
+  // CK11-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 3, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i{{.+}} [[GSIZE:%.+]], {{.+}}getelementptr {{.+}}[3 x i{{.+}}, [3 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
   // CK11-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
   // CK11-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
   // CK11-DAG: [[GSIZE]] = getelementptr inbounds {{.+}}[[SIZE:%[^,]+]]
   //
-  // CK11-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
-  // CK11-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
-  // CK11-DAG: [[SIZE0:%.+]] = getelementptr inbounds {{.+}}[[SIZE]], i{{.+}} 0, i{{.+}} 1
+  // CK11-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 2
+  // CK11-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 2
+  // CK11-DAG: [[SIZE0:%.+]] = getelementptr inbounds {{.+}}[[SIZE]], i{{.+}} 0, i{{.+}} 2
   // CK11-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to double***
   // CK11-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to double**
   // CK11-DAG: store double** [[P:%.+]], double*** [[BPC0]]
@@ -656,44 +653,41 @@ struct S {
   double *p;
   struct S *sp;
 };
-// CK12: [[MTYPE00:@.+]] = {{.+}}constant [3 x i64] [i64 32, i64 281474976710672, i64 17]
+// CK12: [[MTYPE00:@.+]] = {{.+}}constant [4 x i64] [i64 32, i64 281474976710657, i64 281474976710672, i64 17]
 
 // CK12-LABEL: lvalue
 void lvalue(struct S *s, int l, int e) {
 
-  // CK12-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 3, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i{{.+}} [[GSIZE:%.+]], {{.+}}getelementptr {{.+}}[3 x i{{.+}}, [3 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
+  // CK12-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 4, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i{{.+}} [[GSIZE:%.+]], {{.+}}getelementptr {{.+}}[4 x i{{.+}}, [4 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
   // CK12-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
   // CK12-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
   // CK12-DAG: [[GSIZE]] = getelementptr inbounds {{.+}}[[SIZE:%[^,]+]]
   //
-  // CK12-DAG: [[BP2:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 2
-  // CK12-DAG: [[P2:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 2
-  // CK12-DAG: [[SIZE2:%.+]] = getelementptr inbounds {{.+}}[[SIZE]], i{{.+}} 0, i{{.+}} 2
+  // CK12-DAG: [[BP2:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 3
+  // CK12-DAG: [[P2:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 3
+  // CK12-DAG: [[SIZE2:%.+]] = getelementptr inbounds {{.+}}[[SIZE]], i{{.+}} 0, i{{.+}} 3
   // CK12-DAG: [[BPC2:%.+]] = bitcast i8** [[BP2]] to double***
   // CK12-DAG: [[PC2:%.+]] = bitcast i8** [[P2]] to double**
   // CK12-DAG: store double** [[P_VAL:%.+]], double*** [[BPC2]]
   // CK12-DAG: store double* [[SIX:%.+]], double** [[PC2]]
   // CK12-DAG: store i{{.+}} 8, i{{.+}}* [[SIZE2]]
-  // CK12-DAG: [[BP1:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
-  // CK12-DAG: [[P1:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
-  // CK12-DAG: [[SIZE1:%.+]] = getelementptr inbounds {{.+}}[[SIZE]], i{{.+}} 0, i{{.+}} 1
+  // CK12-DAG: [[BP1:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 2
+  // CK12-DAG: [[P1:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 2
+  // CK12-DAG: [[SIZE1:%.+]] = getelementptr inbounds {{.+}}[[SIZE]], i{{.+}} 0, i{{.+}} 2
   // CK12-DAG: [[BPC1:%.+]] = bitcast i8** [[BP1]] to [[STRUCT_S:%.+]]***
   // CK12-DAG: [[PC1:%.+]] = bitcast i8** [[P1]] to double***
   // CK12-DAG: store [[STRUCT_S]]** [[SP:%.+]], [[STRUCT_S]]*** [[BPC1]]
   // CK12-DAG: store double** [[P_VAL:%.+]], double*** [[PC1]]
   // CK12-DAG: store i{{.+}} {{4|8}}, i{{.+}}* [[SIZE1]]
-  // CK12-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 0
-  // CK12-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 0
-  // CK12-DAG: [[SIZE0:%.+]] = getelementptr inbounds {{.+}}[[SIZE]], i{{.+}} 0, i{{.+}} 0
+  // CK12-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
+  // CK12-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
+  // CK12-DAG: [[SIZE0:%.+]] = getelementptr inbounds {{.+}}[[SIZE]], i{{.+}} 0, i{{.+}} 1
   // CK12-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to [[STRUCT_S:%.+]]**
   // CK12-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to [[STRUCT_S]]***
-  // CK12-DAG: store [[STRUCT_S]]* [[ZERO:%.+]], [[STRUCT_S]]** [[BPC0]]
-  // CK12-DAG: store [[STRUCT_S]]** [[SP]], [[STRUCT_S]]*** [[PC0]]
   // CK12-DAG: store [[STRUCT_S]]** [[S:%.+]], [[STRUCT_S]]*** [[S_VAL:%.+]]
   // CK12-DAG: store i{{.+}} {{.+}}, i{{.+}}* [[SIZE0]]
   // CK12-DAG: [[SP]] = getelementptr inbounds [[STRUCT_S]], [[STRUCT_S]]* [[ONE:%.+]], i32 0, i32 1
-  // CK12-DAG: [[ONE]] = load [[STRUCT_S]]*, [[STRUCT_S]]** [[S:%.+]],
-  // CK12-DAG: [[ZERO]] = load [[STRUCT_S]]*, [[STRUCT_S]]** [[S]],
+  // CK12-DAG: [[ONE]] = load %struct.S*, %struct.S** [[S]],
   #pragma omp target update to(*(s->sp->p))
     *(s->sp->p) = e;
   #pragma omp target update from(*(s->sp->p))
@@ -717,21 +711,21 @@ void lvalue(struct S *s, int l, int e) {
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 #ifdef CK13
 
-// CK13: [[SIZE00:@.+]] = {{.+}}constant [1 x i64] [i64 4]
-// CK13: [[MTYPE00:@.+]] = {{.+}}constant [1 x i64] [i64 33]
+// CK13: [[SIZE00:@.+]] = {{.+}}constant [2 x i64] [i64 {{8|4}}, i64 4]
+// CK13: [[MTYPE00:@.+]] = {{.+}}constant [2 x i64] [i64 33, i64 17]
 
 // CK13-LABEL: lvalue
 void lvalue(int **BB, int a, int b) {
 
-  // CK13-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 1, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[SIZE00]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
+  // CK13-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 2, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}]* [[SIZE00]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
   // CK13-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
   // CK13-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
 
-  // CK13-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 0
-  // CK13-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 0
-  // CK13-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to i32***
+  // CK13-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
+  // CK13-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
+  // CK13-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to i32****
   // CK13-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to i32**
-  // CK13-DAG: store i32** [[B_VAL1:%.+]], i32*** [[BPC0]]
+  // CK13-DAG: store i32*** [[BB_ADDR:%.+]], i32**** [[BPC0]]
   // CK13-DAG: store i32* [[ADD_PTR_2:%.+]], i32** [[PC0]]
   // CK13-64-DAG: [[ADD_PTR_2]] = getelementptr inbounds i32, i32* [[RESULT:%.+]], i64 [[IDX_EXT_1:%.+]]
   // CK13-32-DAG: [[ADD_PTR_2]] = getelementptr inbounds i32, i32* [[RESULT:%.+]], i32 [[B_ADDR:%.+]]
@@ -740,8 +734,7 @@ void lvalue(int **BB, int a, int b) {
   // CK13-64-DAG: [[ADD_PTR]] = getelementptr inbounds i32*, i32** [[B_VAL:%.+]], i64 [[IDX_EXT:%.+]]
   // CK13-32-DAG: [[ADD_PTR]] = getelementptr inbounds i32*, i32** [[B_VAL:%.+]], i32 [[A_ADDR:%.+]]
   // CK13-64-DAG: [[IDX_EXT]] = sext i32 [[TWO:%.+]] to i64
-  // CK13-DAG: [[B_VAL]] = load i32**, i32*** [[BB_ADDR:%.+]]
-  // CK13-DAG: [[B_VAL1]] = load i32**, i32*** [[BB_ADDR]]
+  // CK13-DAG: [[B_VAL]] = load i32**, i32*** [[BB_ADDR]]
   #pragma omp target update to(*(*(BB+a)+b))
   *(*(BB+a)+b) = 1;
   #pragma omp target update from(*(*(BB+a)+b))
@@ -838,7 +831,7 @@ void lvalue_member(SSA *sap) {
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 #ifdef CK15
 
-// CK15: [[MTYPE00:@.+]] = {{.+}}constant [2 x i64] [i64 32, i64 281474976710673]
+// CK15: [[MTYPE00:@.+]] = {{.+}}constant [3 x i64] [i64 32, i64 281474976710657, i64 281474976710673]
 
 struct SSA {
   double *p;
@@ -849,27 +842,36 @@ struct SSA {
 //CK-15-LABEL: lvalue_member
 void lvalue_member(SSA *sap) {
 
-  // CK15-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 2, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i64* [[GSIZE:%.+]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
+  // CK15-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 3, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i64* [[GSIZE:%.+]], {{.+}}getelementptr {{.+}}[3 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
   // CK15-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
   // CK15-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
   // CK15-DAG: [[GSIZE]] = getelementptr inbounds {{.+}}[[SIZE:%[^,]+]]
 
-  // CK15-DAG: [[BP1:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
-  // CK15-DAG: [[P1:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
-  // CK15-DAG: [[SIZE1:%.+]] = getelementptr inbounds {{.+}}[[SIZE]], i{{.+}} 0, i{{.+}} 1
-  // CK15-DAG: [[BPC1:%.+]] = bitcast i8** [[BP1]] to double***
-  // CK15-DAG: [[PC1:%.+]] = bitcast i8** [[P1]] to double**
-  // CK15-DAG: store double** [[P_VAL:%.+]], double*** [[BPC1]]
-  // CK15-DAG: store double* [[ADD_PTR:%.+]], double** [[PC1]]
-  // CK15-DAG: store i64 {{4|8}}, i64* [[SIZE1]]
+  // CK15-DAG: [[BP2:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 2
+  // CK15-DAG: [[P2:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 2
+  // CK15-DAG: [[SIZE2:%.+]] = getelementptr inbounds {{.+}}[[SIZE]], i{{.+}} 0, i{{.+}} 2
+  // CK15-DAG: [[BPC2:%.+]] = bitcast i8** [[BP2]] to double***
+  // CK15-DAG: [[PC2:%.+]] = bitcast i8** [[P2]] to double**
+  // CK15-DAG: store double** [[P_VAL:%.+]], double*** [[BPC2]]
+  // CK15-DAG: store double* [[ADD_PTR:%.+]], double** [[PC2]]
+  // CK15-DAG: store i64 8, i64* [[SIZE2]]
   // CK15-DAG: [[ADD_PTR]] = getelementptr inbounds double, double* [[THREE:%.+]], i{{.+}} 3
   // CK15-DAG: [[THREE]] = load double*, double** [[P_VAL_1:%.+]]
   // CK15-DAG: [[P_VAL]] = getelementptr inbounds [[SSA:%.+]], [[SSA:%.+]]* [[THIS:%.+]], i32 0, i32 0
+  // CK15-DAG: [[BP1:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
+  // CK15-DAG: [[P1:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
+  // CK15-DAG: [[SIZE1:%.+]] = getelementptr inbounds {{.+}}[[SIZE]], i{{.+}} 0, i{{.+}} 1
+  // CK15-DAG: [[BPC1:%.+]] = bitcast i8** [[BP1]] to [[SSA]]**
+  // CK15-DAG: [[PC1:%.+]] = bitcast i8** [[P1]] to [[SSA]]***
+  // CK15-DAG: store [[SSA]]* [[SAP_VAL:%.+]], [[SSA]]** [[BPC1]],
+  // CK15-DAG: store [[SSA]]** [[SAP_ADDR:%.+]], [[SSA]]*** [[PC1]]
+  // CK15-DAG: store i{{.+}} {{8|4}}, i{{.+}}* [[SIZE1]]
+  // CK15-DAG: [[SAP_VAL]] = load [[SSA]]*, [[SSA]]** [[SAP_ADDR]],
   // CK15-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 0
   // CK15-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 0
   // CK15-DAG: [[SIZE0:%.+]] = getelementptr inbounds {{.+}}[[SIZE]], i{{.+}} 0, i{{.+}} 0
-  // CK15-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to [[SSA]]**
-  // CK15-DAG: store [[SSA]]* [[ZERO:%.+]], [[SSA]]** [[BPC0]],
+  // CK15-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to [[SSA]]***
+  // CK15-DAG: store [[SSA]]** [[SAP_ADDR]], [[SSA]]*** [[BPC0]],
   // CK15-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to double***
   // CK15-DAG: store double** [[P_VAL]], double*** [[PC0]],
   // CK15-DAG: store i{{.+}} [[COMPUTE_SIZE:%.+]], i{{.+}}* [[SIZE0]]
@@ -879,7 +881,6 @@ void lvalue_member(SSA *sap) {
   // CK15-DAG: [[EIGHT]] = ptrtoint i8* [[FIVE:%.+]] to i64
   // CK15-DAG: [[SIX]] = bitcast double** {{.+}} to i8*
   // CK15-DAG: [[FIVE]] = bitcast double** {{.+}} to i8*
-  // CK15-DAG: [[ZERO]] = load [[SSA]]*, [[SSA]]** %{{.+}},
   #pragma omp target update to(*(3+sap->p))
   *(3+sap->p) = 1;
   #pragma omp target update from(*(3+sap->p))
@@ -903,26 +904,25 @@ void lvalue_member(SSA *sap) {
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 #ifdef CK16
 
-// CK16: [[SIZE00:@.+]] = {{.+}}constant [1 x i64] [i64 4]
-// CK16: [[MTYPE00:@.+]] = {{.+}}constant [1 x i64] [i64 33]
+// CK16: [[SIZE00:@.+]] = {{.+}}constant [2 x i64] [i64 {{8|4}}, i64 4]
+// CK16: [[MTYPE00:@.+]] = {{.+}}constant [2 x i64] [i64 33, i64 17]
 
 //CK16-LABEL: lvalue_find_base
 void lvalue_find_base(float *f, int *i) {
 
-  // CK16-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 1, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[SIZE00]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
+  // CK16-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 2, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}]* [[SIZE00]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
   // CK16-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
   // CK16-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
 
-  // CK16-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 0
-  // CK16-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 0
-  // CK16-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to float**
+  // CK16-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
+  // CK16-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
+  // CK16-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to float***
   // CK16-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to float**
-  // CK16-DAG: store float* [[F:%.+]], float** [[BPC0]]
+  // CK16-DAG: store float** [[F_ADDR:%.+]], float*** [[BPC0]]
   // CK16-DAG: store float* [[ADD_PTR:%.+]], float** [[PC0]]
   // CK16-32-DAG: [[ADD_PTR]] = getelementptr inbounds float, float* [[THREE:%.+]], i32 [[I:%.+]]
   // CK16-64-DAG: [[ADD_PTR]] = getelementptr inbounds float, float* [[THREE:%.+]], i64 [[IDX_EXT:%.+]]
-  // CK16-DAG: [[THREE]] = load float*, float** [[F_ADDR:%.+]],
-  // CK16-DAG: [[F]] = load float*, float** [[F_ADDR]],
+  // CK16-DAG: [[THREE]] = load float*, float** [[F_ADDR]],
   // CK16-64-DAG: [[IDX_EXT]] = sext i32 [[I:%.+]] to i64
 
   #pragma omp target update to(*(*i+f))
@@ -948,8 +948,8 @@ void lvalue_find_base(float *f, int *i) {
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 #ifdef CK17
 
-// CK17: [[SIZE00:@.+]] = {{.+}}constant [1 x i64] [i64 4]
-// CK17: [[MTYPE00:@.+]] = {{.+}}constant [1 x i64] [i64 33]
+// CK17: [[SIZE00:@.+]] = {{.+}}constant [2 x i64] [i64 {{4|8}}, i64 4]
+// CK17: [[MTYPE00:@.+]] = {{.+}}constant [2 x i64] [i64 33, i64 17]
 
 struct SSA {
   int i;
@@ -959,15 +959,15 @@ struct SSA {
 //CK17-LABEL: lvalue_find_base
 void lvalue_find_base(float **f, SSA *sa) {
 
-  // CK17-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 1, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[SIZE00]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
+  // CK17-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 2, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}]* [[SIZE00]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
   // CK17-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
   // CK17-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
 
-  // CK17-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 0
-  // CK17-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 0
-  // CK17-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to float***
+  // CK17-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
+  // CK17-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
+  // CK17-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to float****
   // CK17-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to float**
-  // CK17-DAG: store float** [[F_VAL:%.+]], float*** [[BPC0]],
+  // CK17-DAG: store float*** [[F_ADDR:%.+]], float**** [[BPC0]],
   // CK17-DAG: store float* [[ADD_PTR_4:%.+]], float** [[PC0]],
   // CK17-64-DAG: [[ADD_PTR_4]] = getelementptr inbounds float, float* [[SEVEN:%.+]], i64 [[IDX_EXT_3:%.+]]
   // CK17-64-DAG: [[IDX_EXT_3]] = sext i32 [[I_VAL:%.+]] to i64
@@ -981,8 +981,6 @@ void lvalue_find_base(float **f, SSA *sa) {
   // CK17-DAG: [[FIVE]] = load i32, i32* [[I_2:%.+]],
   // CK17-DAG: [[I_2]] = getelementptr inbounds [[SSA:%.+]], [[SSA]]* [[FOUR:%.+]], i32 0, i32 0
   // CK17-DAG: [[FOUR]] = load [[SSA]]*, [[SSA]]** [[SSA_ADDR:%.+]],
-  // CK17-DAG: [[F]] = load float**, float*** [[F_ADDR:%.+]],
-  // CK17-DAG: [[F_VAL]] = load float**, float*** [[F_ADDR]],
 
   #pragma omp target update to(*(sa->sa->i+*(1+sa->i+f)))
   *(sa->sa->i+*(1+sa->i+f)) = 1;
@@ -1007,13 +1005,13 @@ void lvalue_find_base(float **f, SSA *sa) {
 // SIMD-ONLY18-NOT: {{__kmpc|__tgt}}
 #ifdef CK18
 
-// CK18-DAG: [[MTYPE_TO:@.+]] = {{.+}}constant [1 x i64] [i64 33]
-// CK18-DAG: [[MTYPE_FROM:@.+]] = {{.+}}constant [1 x i64] [i64 34]
+// CK18-DAG: [[MTYPE_TO:@.+]] = {{.+}}constant [2 x i64] [i64 33, i64 16]
+// CK18-DAG: [[MTYPE_FROM:@.+]] = {{.+}}constant [2 x i64] [i64 34, i64 16]
 
 //CK18-LABEL: array_shaping
 void array_shaping(float *f, int sa) {
 
-  // CK18-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 1, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i64* [[GEPS:%.+]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[MTYPE_TO]]{{.+}}, i8** null)
+  // CK18-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 2, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i64* [[GEPS:%.+]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}]* [[MTYPE_TO]]{{.+}}, i8** null)
   // CK18-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
   // CK18-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
   // CK18-DAG: [[GEPS]] = getelementptr inbounds {{.+}}[[S:%[^,]+]]
@@ -1023,12 +1021,23 @@ void array_shaping(float *f, int sa) {
   // CK18-DAG: [[S0:%.+]] = getelementptr inbounds {{.+}}[[S]], i{{.+}} 0, i{{.+}} 0
 
   // CK18-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to float**
-  // CK18-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to float**
+  // CK18-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to float***
 
   // CK18-DAG: store float* [[F1:%.+]], float** [[BPC0]],
-  // CK18-DAG: store float* [[F2:%.+]], float** [[PC0]],
-  // CK18-DAG: store i64 [[SIZE:%.+]], i64* [[S0]],
-  // CK18-DAG: [[F1]] = load float*, float** [[F_ADDR:%.+]],
+  // CK18-DAG: store float** [[F_ADDR:%.+]], float*** [[PC0]],
+  // CK18-DAG: store i64 {{8|4}}, i64* [[S0]],
+  // CK18-DAG: [[F1]] = load float*, float** [[F_ADDR]],
+
+  // CK18-DAG: [[BP1:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
+  // CK18-DAG: [[P1:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
+  // CK18-DAG: [[S1:%.+]] = getelementptr inbounds {{.+}}[[S]], i{{.+}} 0, i{{.+}} 1
+
+  // CK18-DAG: [[BPC1:%.+]] = bitcast i8** [[BP1]] to float***
+  // CK18-DAG: [[PC1:%.+]] = bitcast i8** [[P1]] to float**
+
+  // CK18-DAG: store float** [[F_ADDR]], float*** [[BPC1]],
+  // CK18-DAG: store float* [[F2:%.+]], float** [[PC1]],
+  // CK18-DAG: store i64 [[SIZE:%.+]], i64* [[S1]],
   // CK18-DAG: [[F2]] = load float*, float** [[F_ADDR]],
 
   // CK18-64-DAG: [[SIZE]] = mul nuw i64 [[SZ1:%.+]], 4
@@ -1038,7 +1047,7 @@ void array_shaping(float *f, int sa) {
   // CK18-32-DAG: [[SZ2]] = mul nuw i32 12, %{{.+}}
   #pragma omp target update to(([3][sa][4])f)
   sa = 1;
-  // CK18-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 1, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i64* [[GEPS:%.+]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[MTYPE_FROM]]{{.+}}, i8** null)
+  // CK18-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 2, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i64* [[GEPS:%.+]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}]* [[MTYPE_FROM]]{{.+}}, i8** null)
   // CK18-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
   // CK18-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
   // CK18-DAG: [[GEPS]] = getelementptr inbounds {{.+}}[[S:%[^,]+]]
@@ -1048,12 +1057,23 @@ void array_shaping(float *f, int sa) {
   // CK18-DAG: [[S0:%.+]] = getelementptr inbounds {{.+}}[[S]], i{{.+}} 0, i{{.+}} 0
 
   // CK18-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to float**
-  // CK18-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to float**
+  // CK18-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to float***
 
   // CK18-DAG: store float* [[F1:%.+]], float** [[BPC0]],
-  // CK18-DAG: store float* [[F2:%.+]], float** [[PC0]],
-  // CK18-DAG: store i64 [[SIZE:%.+]], i64* [[S0]],
+  // CK18-DAG: store float** [[F_ADDR:%.+]], float*** [[PC0]],
+  // CK18-DAG: store i64 {{8|4}}, i64* [[S0]],
   // CK18-DAG: [[F1]] = load float*, float** [[F_ADDR]],
+
+  // CK18-DAG: [[BP1:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
+  // CK18-DAG: [[P1:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
+  // CK18-DAG: [[S1:%.+]] = getelementptr inbounds {{.+}}[[S]], i{{.+}} 0, i{{.+}} 1
+
+  // CK18-DAG: [[BPC1:%.+]] = bitcast i8** [[BP1]] to float***
+  // CK18-DAG: [[PC1:%.+]] = bitcast i8** [[P1]] to float**
+
+  // CK18-DAG: store float** [[F_ADDR]], float*** [[BPC1]],
+  // CK18-DAG: store float* [[F2:%.+]], float** [[PC1]],
+  // CK18-DAG: store i64 [[SIZE:%.+]], i64* [[S1]],
   // CK18-DAG: [[F2]] = load float*, float** [[F_ADDR]],
 
   // CK18-64-DAG: [[SIZE]] = mul nuw i64 [[SZ1:%.+]], 5
diff --git a/openmp/libomptarget/src/omptarget.cpp b/openmp/libomptarget/src/omptarget.cpp
index 9c7ab6309503a..409089500015e 100644
--- a/openmp/libomptarget/src/omptarget.cpp
+++ b/openmp/libomptarget/src/omptarget.cpp
@@ -880,9 +880,14 @@ int target(int64_t DeviceId, void *HostPtr, int32_t ArgNum, void **ArgBases,
           return OFFLOAD_FAIL;
         }
       }
+    } else if (ArgTypes[I] & OMP_TGT_MAPTYPE_PTR_AND_OBJ) {
+      TgtPtrBegin = Device.getTgtPtrBegin(HstPtrBase, sizeof(void *), IsLast,
+                                          false, IsHostPtr);
+      TgtBaseOffset = 0; // no offset for ptrs.
+      DP("Obtained target argument " DPxMOD " from host pointer " DPxMOD " to "
+         "object " DPxMOD "\n",
+         DPxPTR(TgtPtrBegin), DPxPTR(HstPtrBase), DPxPTR(HstPtrBase));
     } else {
-      if (ArgTypes[I] & OMP_TGT_MAPTYPE_PTR_AND_OBJ)
-        HstPtrBase = *reinterpret_cast<void **>(HstPtrBase);
       TgtPtrBegin = Device.getTgtPtrBegin(HstPtrBegin, ArgSizes[I], IsLast,
                                           false, IsHostPtr);
       TgtBaseOffset = (intptr_t)HstPtrBase - (intptr_t)HstPtrBegin;
diff --git a/openmp/libomptarget/test/env/base_ptr_ref_count.c b/openmp/libomptarget/test/env/base_ptr_ref_count.c
deleted file mode 100644
index 4036954182faf..0000000000000
--- a/openmp/libomptarget/test/env/base_ptr_ref_count.c
+++ /dev/null
@@ -1,51 +0,0 @@
-// RUN: %libomptarget-compile-aarch64-unknown-linux-gnu && env LIBOMPTARGET_DEBUG=1 %libomptarget-run-aarch64-unknown-linux-gnu 2>&1 | %fcheck-aarch64-unknown-linux-gnu
-// RUN: %libomptarget-compile-powerpc64-ibm-linux-gnu && env LIBOMPTARGET_DEBUG=1 %libomptarget-run-powerpc64-ibm-linux-gnu 2>&1 | %fcheck-powerpc64-ibm-linux-gnu
-// RUN: %libomptarget-compile-powerpc64le-ibm-linux-gnu && env LIBOMPTARGET_DEBUG=1 %libomptarget-run-powerpc64le-ibm-linux-gnu 2>&1 | %fcheck-powerpc64le-ibm-linux-gnu
-// RUN: %libomptarget-compile-x86_64-pc-linux-gnu && env LIBOMPTARGET_DEBUG=1 %libomptarget-run-x86_64-pc-linux-gnu 2>&1 | %fcheck-x86_64-pc-linux-gnu
-// RUN: %libomptarget-compile-nvptx64-nvidia-cuda && env LIBOMPTARGET_DEBUG=1 %libomptarget-run-nvptx64-nvidia-cuda 2>&1 | %fcheck-nvptx64-nvidia-cuda
-// REQUIRES: libomptarget-debug
-
-#include <stdlib.h>
-
-int *allocate(size_t n) {
-  int *ptr = malloc(sizeof(int) * n);
-#pragma omp target enter data map(to : ptr[:n])
-  return ptr;
-}
-
-void deallocate(int *ptr, size_t n) {
-#pragma omp target exit data map(delete : ptr[:n])
-  free(ptr);
-}
-
-#pragma omp declare target
-int *cnt;
-void foo() {
-  ++(*cnt);
-}
-#pragma omp end declare target
-
-int main(void) {
-  int *A = allocate(10);
-  int *V = allocate(10);
-  deallocate(A, 10);
-  deallocate(V, 10);
-// CHECK-NOT: RefCount=2
-  cnt = malloc(sizeof(int));
-  *cnt = 0;
-#pragma omp target map(cnt[:1])
-  foo();
-  printf("Cnt = %d.\n", *cnt);
-// CHECK: Cnt = 1.
-  *cnt = 0;
-#pragma omp target data map(cnt[:1])
-#pragma omp target
-  foo();
-  printf("Cnt = %d.\n", *cnt);
-// CHECK: Cnt = 1.
-  free(cnt);
-
-  return 0;
-}
-
-

From c4b7bfdff65e5883bfe64248ac244b9fadf531ee Mon Sep 17 00:00:00 2001
From: Kirill Bobyrev <kbobyrev@google.com>
Date: Thu, 30 Jul 2020 17:12:29 +0200
Subject: [PATCH 0626/1035] [clangd] NFC: Spell out types in index callback
 arguments

---
 clang-tools-extra/clangd/index/remote/server/Server.cpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/clang-tools-extra/clangd/index/remote/server/Server.cpp b/clang-tools-extra/clangd/index/remote/server/Server.cpp
index 077d58ea7db0c..27e89cbbb48ef 100644
--- a/clang-tools-extra/clangd/index/remote/server/Server.cpp
+++ b/clang-tools-extra/clangd/index/remote/server/Server.cpp
@@ -90,7 +90,7 @@ class RemoteIndexServer final : public SymbolIndex::Service {
     }
     unsigned Sent = 0;
     unsigned FailedToSend = 0;
-    Index->lookup(*Req, [&](const auto &Item) {
+    Index->lookup(*Req, [&](const clangd::Symbol &Item) {
       auto SerializedItem = ProtobufMarshaller->toProtobuf(Item);
       if (!SerializedItem) {
         ++FailedToSend;
@@ -121,7 +121,7 @@ class RemoteIndexServer final : public SymbolIndex::Service {
     }
     unsigned Sent = 0;
     unsigned FailedToSend = 0;
-    bool HasMore = Index->fuzzyFind(*Req, [&](const auto &Item) {
+    bool HasMore = Index->fuzzyFind(*Req, [&](const clangd::Symbol &Item) {
       auto SerializedItem = ProtobufMarshaller->toProtobuf(Item);
       if (!SerializedItem) {
         ++FailedToSend;
@@ -150,7 +150,7 @@ class RemoteIndexServer final : public SymbolIndex::Service {
     }
     unsigned Sent = 0;
     unsigned FailedToSend = 0;
-    bool HasMore = Index->refs(*Req, [&](const auto &Item) {
+    bool HasMore = Index->refs(*Req, [&](const clangd::Ref &Item) {
       auto SerializedItem = ProtobufMarshaller->toProtobuf(Item);
       if (!SerializedItem) {
         ++FailedToSend;

From 622e46156d9a91206c877a604d069bb3e2dbf294 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev@hotmail.com>
Date: Thu, 30 Jul 2020 11:00:21 -0400
Subject: [PATCH 0627/1035] [OPENMP]Fix PR46824: Global declare target pointer
 cannot be accessed in target region.

Need to map the base pointer for all directives, not only target
data-based ones.
The base pointer is mapped for array sections, array subscript, array
shaping and other array-like constructs with the base pointer. Also,
codegen for use_device_ptr clause was modified to correctly handle
mapping combination of array like constructs + use_device_ptr clause.
The data for use_device_ptr clause is emitted as the last records in the
data mapping array.

Reviewed By: ye-luo

Differential Revision: https://reviews.llvm.org/D84767
---
 clang/lib/CodeGen/CGOpenMPRuntime.cpp         |  81 ++++--
 clang/test/OpenMP/target_data_codegen.cpp     |  18 +-
 .../target_data_use_device_ptr_codegen.cpp    |  48 ++--
 clang/test/OpenMP/target_map_codegen.cpp      |  21 +-
 clang/test/OpenMP/target_update_codegen.cpp   | 242 ++++++++----------
 openmp/libomptarget/src/omptarget.cpp         |   9 +-
 .../test/env/base_ptr_ref_count.c             |  51 ++++
 7 files changed, 257 insertions(+), 213 deletions(-)
 create mode 100644 openmp/libomptarget/test/env/base_ptr_ref_count.c

diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
index b8e33948c21ca..dc12286c72be8 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -7392,10 +7392,9 @@ class MappableExprsHandler {
     // &p, &p, sizeof(float*), TARGET_PARAM | TO | FROM
     //
     // map(p[1:24])
+    // &p, &p[1], 24*sizeof(float), TARGET_PARAM | TO | FROM | PTR_AND_OBJ
+    // in unified shared memory mode or for local pointers
     // p, &p[1], 24*sizeof(float), TARGET_PARAM | TO | FROM
-    // for data directives
-    // p, p, sizeof(float*), TARGET_PARAM | TO | FROM
-    // p, &p[1], 24*sizeof(float), PTR_AND_OBJ | TO | FROM
     //
     // map(s)
     // &s, &s, sizeof(S2), TARGET_PARAM | TO | FROM
@@ -7530,6 +7529,7 @@ class MappableExprsHandler {
     // Track if the map information being generated is the first for a list of
     // components.
     bool IsExpressionFirstInfo = true;
+    bool FirstPointerInComplexData = false;
     Address BP = Address::invalid();
     const Expr *AssocExpr = I->getAssociatedExpression();
     const auto *AE = dyn_cast<ArraySubscriptExpr>(AssocExpr);
@@ -7572,17 +7572,16 @@ class MappableExprsHandler {
       QualType Ty =
           I->getAssociatedDeclaration()->getType().getNonReferenceType();
       if (Ty->isAnyPointerType() && std::next(I) != CE) {
-        BP = CGF.EmitLoadOfPointer(BP, Ty->castAs<PointerType>());
-
-        // For non-data directives, we do not need to generate individual map
-        // information for the pointer, it can be associated with the combined
-        // storage.
+        // No need to generate individual map information for the pointer, it
+        // can be associated with the combined storage if shared memory mode is
+        // active or the base declaration is not global variable.
+        const auto *VD = dyn_cast<VarDecl>(I->getAssociatedDeclaration());
         if (CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory() ||
-            !CurDir.is<const OMPExecutableDirective *>() ||
-            !isOpenMPTargetDataManagementDirective(
-                CurDir.get<const OMPExecutableDirective *>()
-                    ->getDirectiveKind()))
-          ++I;
+            !VD || VD->hasLocalStorage())
+          BP = CGF.EmitLoadOfPointer(BP, Ty->castAs<PointerType>());
+        else
+          FirstPointerInComplexData = true;
+        ++I;
       }
     }
 
@@ -7617,8 +7616,19 @@ class MappableExprsHandler {
         EncounteredME = dyn_cast<MemberExpr>(I->getAssociatedExpression());
         // If we encounter a PTR_AND_OBJ entry from now on it should be marked
         // as MEMBER_OF the parent struct.
-        if (EncounteredME)
+        if (EncounteredME) {
           ShouldBeMemberOf = true;
+          // Do not emit as complex pointer if this is actually not array-like
+          // expression.
+          if (FirstPointerInComplexData) {
+            QualType Ty = std::prev(I)
+                              ->getAssociatedDeclaration()
+                              ->getType()
+                              .getNonReferenceType();
+            BP = CGF.EmitLoadOfPointer(BP, Ty->castAs<PointerType>());
+            FirstPointerInComplexData = false;
+          }
+        }
       }
 
       auto Next = std::next(I);
@@ -7760,7 +7770,8 @@ class MappableExprsHandler {
           // (there is a set of entries for each capture).
           OpenMPOffloadMappingFlags Flags =
               getMapTypeBits(MapType, MapModifiers, MotionModifiers, IsImplicit,
-                             !IsExpressionFirstInfo || RequiresReference,
+                             !IsExpressionFirstInfo || RequiresReference ||
+                                 FirstPointerInComplexData,
                              IsCaptureFirstInfo && !RequiresReference);
 
           if (!IsExpressionFirstInfo) {
@@ -7819,6 +7830,7 @@ class MappableExprsHandler {
 
         IsExpressionFirstInfo = false;
         IsCaptureFirstInfo = false;
+        FirstPointerInComplexData = false;
       }
     }
   }
@@ -8067,6 +8079,7 @@ class MappableExprsHandler {
     // emission of that entry until the whole struct has been processed.
     llvm::MapVector<const ValueDecl *, SmallVector<DeferredDevicePtrEntryTy, 4>>
         DeferredInfo;
+    MapCombinedInfoTy UseDevicePtrCombinedInfo;
 
     for (const auto *C :
          CurExecDir->getClausesOfKind<OMPUseDevicePtrClause>()) {
@@ -8086,15 +8099,27 @@ class MappableExprsHandler {
         // We potentially have map information for this declaration already.
         // Look for the first set of components that refer to it.
         if (It != Info.end()) {
-          auto CI = std::find_if(
-              It->second.begin(), It->second.end(), [VD](const MapInfo &MI) {
-                return MI.Components.back().getAssociatedDeclaration() == VD;
-              });
+          auto *CI = llvm::find_if(It->second, [VD](const MapInfo &MI) {
+            return MI.Components.back().getAssociatedDeclaration() == VD;
+          });
           // If we found a map entry, signal that the pointer has to be returned
           // and move on to the next declaration.
+          // Exclude cases where the base pointer is mapped as array subscript,
+          // array section or array shaping. The base address is passed as a
+          // pointer to base in this case and cannot be used as a base for
+          // use_device_ptr list item.
           if (CI != It->second.end()) {
-            CI->ReturnDevicePointer = true;
-            continue;
+            auto PrevCI = std::next(CI->Components.rbegin());
+            const auto *VarD = dyn_cast<VarDecl>(VD);
+            if (CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory() ||
+                isa<MemberExpr>(IE) ||
+                !VD->getType().getNonReferenceType()->isPointerType() ||
+                PrevCI == CI->Components.rend() ||
+                isa<MemberExpr>(PrevCI->getAssociatedExpression()) || !VarD ||
+                VarD->hasLocalStorage()) {
+              CI->ReturnDevicePointer = true;
+              continue;
+            }
           }
         }
 
@@ -8115,13 +8140,13 @@ class MappableExprsHandler {
         } else {
           llvm::Value *Ptr =
               CGF.EmitLoadOfScalar(CGF.EmitLValue(IE), IE->getExprLoc());
-          CombinedInfo.BasePointers.emplace_back(Ptr, VD);
-          CombinedInfo.Pointers.push_back(Ptr);
-          CombinedInfo.Sizes.push_back(
+          UseDevicePtrCombinedInfo.BasePointers.emplace_back(Ptr, VD);
+          UseDevicePtrCombinedInfo.Pointers.push_back(Ptr);
+          UseDevicePtrCombinedInfo.Sizes.push_back(
               llvm::Constant::getNullValue(CGF.Int64Ty));
-          CombinedInfo.Types.push_back(OMP_MAP_RETURN_PARAM |
-                                       OMP_MAP_TARGET_PARAM);
-          CombinedInfo.Mappers.push_back(nullptr);
+          UseDevicePtrCombinedInfo.Types.push_back(OMP_MAP_RETURN_PARAM |
+                                                   OMP_MAP_TARGET_PARAM);
+          UseDevicePtrCombinedInfo.Mappers.push_back(nullptr);
         }
       }
     }
@@ -8273,6 +8298,8 @@ class MappableExprsHandler {
       // We need to append the results of this capture to what we already have.
       CombinedInfo.append(CurInfo);
     }
+    // Append data for use_device_ptr clauses.
+    CombinedInfo.append(UseDevicePtrCombinedInfo);
   }
 
   /// Generate all the base pointers, section pointers, sizes, map types, and
diff --git a/clang/test/OpenMP/target_data_codegen.cpp b/clang/test/OpenMP/target_data_codegen.cpp
index 80e674c0b49d9..a2bb8cdf5ba88 100644
--- a/clang/test/OpenMP/target_data_codegen.cpp
+++ b/clang/test/OpenMP/target_data_codegen.cpp
@@ -555,7 +555,7 @@ struct S2 {
 
 void test_close_modifier(int arg) {
   S2 *ps;
-  // CK5: private unnamed_addr constant [6 x i64] [i64 1059, i64 32, i64 562949953422339, i64 562949953421328, i64 16, i64 1043]
+  // CK5: private unnamed_addr constant [5 x i64] [i64 1059, i64 32, i64 562949953421328, i64 16, i64 1043]
   #pragma omp target data map(close,tofrom: arg, ps->ps->ps->ps->s)
   {
     ++(arg);
@@ -634,20 +634,17 @@ void test_present_modifier(int arg) {
   // Make sure the struct picks up present even if another element of the struct
   // doesn't have present.
 
-  // CK8: private unnamed_addr constant [15 x i64]
+  // CK8: private unnamed_addr constant [11 x i64]
 
   // ps1
   //
   // PRESENT=0x1000 | TARGET_PARAM=0x20 = 0x1020
   // MEMBER_OF_1=0x1000000000000 | FROM=0x2 | TO=0x1 = 0x1000000000003
-  // MEMBER_OF_1=0x1000000000000 | PTR_AND_OBJ=0x10 | FROM=0x2 | TO=0x1 = 0x1000000000013
-  // MEMBER_OF_1=0x1000000000000 | PRESENT=0x1000 | FROM=0x2 | TO=0x1 = 0x1000000001003
   // MEMBER_OF_1=0x1000000000000 | PRESENT=0x1000 | PTR_AND_OBJ=0x10 = 0x1000000001010
   // PRESENT=0x1000 | PTR_AND_OBJ=0x10 = 0x1010
   // PRESENT=0x1000 | PTR_AND_OBJ=0x10 | FROM=0x2 | TO=0x1 = 0x1013
   //
   // CK8-SAME: {{^}} [i64 [[#0x1020]], i64 [[#0x1000000000003]],
-  // CK8-SAME: {{^}} i64 [[#0x1000000000013]], i64 [[#0x1000000001003]],
   // CK8-SAME: {{^}} i64 [[#0x1000000001010]], i64 [[#0x1010]], i64 [[#0x1013]],
 
   // arg
@@ -659,16 +656,13 @@ void test_present_modifier(int arg) {
   // ps2
   //
   // PRESENT=0x1000 | TARGET_PARAM=0x20 = 0x1020
-  // MEMBER_OF_9=0x9000000000000 | PRESENT=0x1000 | FROM=0x2 | TO=0x1 = 0x9000000001003
-  // MEMBER_OF_9=0x9000000000000 | PRESENT=0x1000 | PTR_AND_OBJ=0x10 | FROM=0x2 | TO=0x1 = 0x9000000001013
-  // MEMBER_OF_9=0x9000000000000 | FROM=0x2 | TO=0x1 = 0x9000000000003
-  // MEMBER_OF_9=0x9000000000000 | PTR_AND_OBJ=0x10 = 0x9000000000010
+  // MEMBER_OF_7=0x7000000000000 | PRESENT=0x1000 | FROM=0x2 | TO=0x1 = 0x7000000001003
+  // MEMBER_OF_7=0x7000000000000 | PTR_AND_OBJ=0x10 = 0x7000000000010
   // PTR_AND_OBJ=0x10 = 0x10
   // PTR_AND_OBJ=0x10 | FROM=0x2 | TO=0x1 = 0x13
   //
-  // CK8-SAME: {{^}} i64 [[#0x1020]], i64 [[#0x9000000001003]],
-  // CK8-SAME: {{^}} i64 [[#0x9000000001013]], i64 [[#0x9000000000003]],
-  // CK8-SAME: {{^}} i64 [[#0x9000000000010]], i64 [[#0x10]], i64 [[#0x13]]]
+  // CK8-SAME: {{^}} i64 [[#0x1020]], i64 [[#0x7000000001003]],
+  // CK8-SAME: {{^}} i64 [[#0x7000000000010]], i64 [[#0x10]], i64 [[#0x13]]]
   #pragma omp target data map(tofrom: ps1->s) \
                           map(present,tofrom: arg, ps1->ps->ps->ps->s, ps2->s) \
                           map(tofrom: ps2->ps->ps->ps->s)
diff --git a/clang/test/OpenMP/target_data_use_device_ptr_codegen.cpp b/clang/test/OpenMP/target_data_use_device_ptr_codegen.cpp
index fe6ea01b43c91..ca5536f927a10 100644
--- a/clang/test/OpenMP/target_data_use_device_ptr_codegen.cpp
+++ b/clang/test/OpenMP/target_data_use_device_ptr_codegen.cpp
@@ -22,18 +22,18 @@
 double *g;
 
 // CK1: @g = global double*
-// CK1: [[MTYPE00:@.+]] = {{.*}}constant [2 x i64] [i64 99, i64 19]
-// CK1: [[MTYPE01:@.+]] = {{.*}}constant [2 x i64] [i64 99, i64 19]
-// CK1: [[MTYPE03:@.+]] = {{.*}}constant [2 x i64] [i64 99, i64 19]
-// CK1: [[MTYPE04:@.+]] = {{.*}}constant [2 x i64] [i64 99, i64 19]
-// CK1: [[MTYPE05:@.+]] = {{.*}}constant [2 x i64] [i64 99, i64 19]
-// CK1: [[MTYPE06:@.+]] = {{.*}}constant [2 x i64] [i64 99, i64 19]
-// CK1: [[MTYPE07:@.+]] = {{.*}}constant [2 x i64] [i64 99, i64 19]
-// CK1: [[MTYPE08:@.+]] = {{.*}}constant [4 x i64] [i64 99, i64 19, i64 35, i64 19]
-// CK1: [[MTYPE09:@.+]] = {{.*}}constant [4 x i64] [i64 99, i64 19, i64 99, i64 19]
-// CK1: [[MTYPE10:@.+]] = {{.*}}constant [4 x i64] [i64 99, i64 19, i64 99, i64 19]
-// CK1: [[MTYPE11:@.+]] = {{.*}}constant [3 x i64] [i64 96, i64 35, i64 19]
-// CK1: [[MTYPE12:@.+]] = {{.*}}constant [3 x i64] [i64 96, i64 35, i64 19]
+// CK1: [[MTYPE00:@.+]] = {{.*}}constant [2 x i64] [i64 51, i64 96]
+// CK1: [[MTYPE01:@.+]] = {{.*}}constant [1 x i64] [i64 99]
+// CK1: [[MTYPE03:@.+]] = {{.*}}constant [1 x i64] [i64 99]
+// CK1: [[MTYPE04:@.+]] = {{.*}}constant [1 x i64] [i64 99]
+// CK1: [[MTYPE05:@.+]] = {{.*}}constant [1 x i64] [i64 99]
+// CK1: [[MTYPE06:@.+]] = {{.*}}constant [1 x i64] [i64 99]
+// CK1: [[MTYPE07:@.+]] = {{.*}}constant [1 x i64] [i64 99]
+// CK1: [[MTYPE08:@.+]] = {{.*}}constant [2 x i64] [i64 99, i64 35]
+// CK1: [[MTYPE09:@.+]] = {{.*}}constant [2 x i64] [i64 99, i64 99]
+// CK1: [[MTYPE10:@.+]] = {{.*}}constant [2 x i64] [i64 99, i64 99]
+// CK1: [[MTYPE11:@.+]] = {{.*}}constant [2 x i64] [i64 35, i64 96]
+// CK1: [[MTYPE12:@.+]] = {{.*}}constant [2 x i64] [i64 35, i64 96]
 
 // CK1-LABEL: @_Z3foo
 template<typename T>
@@ -42,7 +42,7 @@ void foo(float *&lr, T *&tr) {
   T *t;
 
   // CK1:     [[T:%.+]] = load double*, double** [[DECL:@g]],
-  // CK1:     [[BP:%.+]] = getelementptr inbounds [2 x i8*], [2 x i8*]* %{{.+}}, i32 0, i32 0
+  // CK1:     [[BP:%.+]] = getelementptr inbounds [2 x i8*], [2 x i8*]* %{{.+}}, i32 0, i32 1
   // CK1:     [[CBP:%.+]] = bitcast i8** [[BP]] to double**
   // CK1:     store double* [[T]], double** [[CBP]],
   // CK1:     call void @__tgt_target_data_begin{{.+}}[[MTYPE00]]
@@ -61,7 +61,7 @@ void foo(float *&lr, T *&tr) {
   ++g;
 
   // CK1:     [[T1:%.+]] = load float*, float** [[DECL:%.+]],
-  // CK1:     [[BP:%.+]] = getelementptr inbounds [2 x i8*], [2 x i8*]* %{{.+}}, i32 0, i32 0
+  // CK1:     [[BP:%.+]] = getelementptr inbounds [1 x i8*], [1 x i8*]* %{{.+}}, i32 0, i32 0
   // CK1:     [[CBP:%.+]] = bitcast i8** [[BP]] to float**
   // CK1:     store float* [[T1]], float** [[CBP]],
   // CK1:     call void @__tgt_target_data_begin{{.+}}[[MTYPE01]]
@@ -92,7 +92,7 @@ void foo(float *&lr, T *&tr) {
   ++l;
 
   // CK1:     [[T1:%.+]] = load float*, float** [[DECL:%.+]],
-  // CK1:     [[BP:%.+]] = getelementptr inbounds [2 x i8*], [2 x i8*]* %{{.+}}, i32 0, i32 0
+  // CK1:     [[BP:%.+]] = getelementptr inbounds [1 x i8*], [1 x i8*]* %{{.+}}, i32 0, i32 0
   // CK1:     [[CBP:%.+]] = bitcast i8** [[BP]] to float**
   // CK1:     store float* [[T1]], float** [[CBP]],
   // CK1:     call void @__tgt_target_data_begin{{.+}}[[MTYPE03]]
@@ -115,7 +115,7 @@ void foo(float *&lr, T *&tr) {
 
   // CK1:     [[BTHEN]]:
   // CK1:     [[T1:%.+]] = load float*, float** [[DECL:%.+]],
-  // CK1:     [[BP:%.+]] = getelementptr inbounds [2 x i8*], [2 x i8*]* %{{.+}}, i32 0, i32 0
+  // CK1:     [[BP:%.+]] = getelementptr inbounds [1 x i8*], [1 x i8*]* %{{.+}}, i32 0, i32 0
   // CK1:     [[CBP:%.+]] = bitcast i8** [[BP]] to float**
   // CK1:     store float* [[T1]], float** [[CBP]],
   // CK1:     call void @__tgt_target_data_begin{{.+}}[[MTYPE04]]
@@ -152,7 +152,7 @@ void foo(float *&lr, T *&tr) {
 
   // CK1:     [[T2:%.+]] = load float**, float*** [[DECL:%.+]],
   // CK1:     [[T1:%.+]] = load float*, float** [[T2]],
-  // CK1:     [[BP:%.+]] = getelementptr inbounds [2 x i8*], [2 x i8*]* %{{.+}}, i32 0, i32 0
+  // CK1:     [[BP:%.+]] = getelementptr inbounds [1 x i8*], [1 x i8*]* %{{.+}}, i32 0, i32 0
   // CK1:     [[CBP:%.+]] = bitcast i8** [[BP]] to float**
   // CK1:     store float* [[T1]], float** [[CBP]],
   // CK1:     call void @__tgt_target_data_begin{{.+}}[[MTYPE05]]
@@ -174,7 +174,7 @@ void foo(float *&lr, T *&tr) {
   ++lr;
 
   // CK1:     [[T1:%.+]] = load i32*, i32** [[DECL:%.+]],
-  // CK1:     [[BP:%.+]] = getelementptr inbounds [2 x i8*], [2 x i8*]* %{{.+}}, i32 0, i32 0
+  // CK1:     [[BP:%.+]] = getelementptr inbounds [1 x i8*], [1 x i8*]* %{{.+}}, i32 0, i32 0
   // CK1:     [[CBP:%.+]] = bitcast i8** [[BP]] to i32**
   // CK1:     store i32* [[T1]], i32** [[CBP]],
   // CK1:     call void @__tgt_target_data_begin{{.+}}[[MTYPE06]]
@@ -194,7 +194,7 @@ void foo(float *&lr, T *&tr) {
 
   // CK1:     [[T2:%.+]] = load i32**, i32*** [[DECL:%.+]],
   // CK1:     [[T1:%.+]] = load i32*, i32** [[T2]],
-  // CK1:     [[BP:%.+]] = getelementptr inbounds [2 x i8*], [2 x i8*]* %{{.+}}, i32 0, i32 0
+  // CK1:     [[BP:%.+]] = getelementptr inbounds [1 x i8*], [1 x i8*]* %{{.+}}, i32 0, i32 0
   // CK1:     [[CBP:%.+]] = bitcast i8** [[BP]] to i32**
   // CK1:     store i32* [[T1]], i32** [[CBP]],
   // CK1:     call void @__tgt_target_data_begin{{.+}}[[MTYPE07]]
@@ -216,7 +216,7 @@ void foo(float *&lr, T *&tr) {
   ++tr;
 
   // CK1:     [[T1:%.+]] = load float*, float** [[DECL:%.+]],
-  // CK1:     [[BP:%.+]] = getelementptr inbounds [4 x i8*], [4 x i8*]* %{{.+}}, i32 0, i32
+  // CK1:     [[BP:%.+]] = getelementptr inbounds [2 x i8*], [2 x i8*]* %{{.+}}, i32 0, i32 0
   // CK1:     [[CBP:%.+]] = bitcast i8** [[BP]] to float**
   // CK1:     store float* [[T1]], float** [[CBP]],
   // CK1:     call void @__tgt_target_data_begin{{.+}}[[MTYPE08]]
@@ -280,7 +280,7 @@ void foo(float *&lr, T *&tr) {
   ++l; ++t;
 
   // CK1:     [[T1:%.+]] = load i32*, i32** [[DECL:%.+]],
-  // CK1:     [[BP:%.+]] = getelementptr inbounds [3 x i8*], [3 x i8*]* %{{.+}}, i32 0, i32 0
+  // CK1:     [[BP:%.+]] = getelementptr inbounds [2 x i8*], [2 x i8*]* %{{.+}}, i32 0, i32 1
   // CK1:     [[CBP:%.+]] = bitcast i8** [[BP]] to i32**
   // CK1:     store i32* [[T1]], i32** [[CBP]],
   // CK1:     call void @__tgt_target_data_begin{{.+}}[[MTYPE11]]
@@ -300,7 +300,7 @@ void foo(float *&lr, T *&tr) {
 
   // CK1:     [[T2:%.+]] = load i32**, i32*** [[DECL:%.+]],
   // CK1:     [[T1:%.+]] = load i32*, i32** [[T2]],
-  // CK1:     [[BP:%.+]] = getelementptr inbounds [3 x i8*], [3 x i8*]* %{{.+}}, i32 0, i32 0
+  // CK1:     [[BP:%.+]] = getelementptr inbounds [2 x i8*], [2 x i8*]* %{{.+}}, i32 0, i32 1
   // CK1:     [[CBP:%.+]] = bitcast i8** [[BP]] to i32**
   // CK1:     store i32* [[T1]], i32** [[CBP]],
   // CK1:     call void @__tgt_target_data_begin{{.+}}[[MTYPE12]]
@@ -348,7 +348,7 @@ void bar(float *&a, int *&b) {
 // CK2: [[ST:%.+]] = type { double*, double** }
 // CK2: [[MTYPE00:@.+]] = {{.*}}constant [2 x i64] [i64 32, i64 281474976710739]
 // CK2: [[MTYPE01:@.+]] = {{.*}}constant [2 x i64] [i64 32, i64 281474976710739]
-// CK2: [[MTYPE02:@.+]] = {{.*}}constant [4 x i64] [i64 35, i64 19, i64 32, i64 844424930132048]
+// CK2: [[MTYPE02:@.+]] = {{.*}}constant [3 x i64] [i64 35, i64 32, i64 562949953421392]
 // CK2: [[MTYPE03:@.+]] = {{.*}}constant [3 x i64] [i64 32, i64 281474976710739, i64 281474976710736]
 
 template <typename T>
@@ -404,7 +404,7 @@ struct ST {
     // CK2:     getelementptr inbounds double, double* [[TTTT]], i32 1
     b++;
 
-    // CK2:     [[BP:%.+]] = getelementptr inbounds [4 x i8*], [4 x i8*]* %{{.+}}, i32 0, i32 3
+    // CK2:     [[BP:%.+]] = getelementptr inbounds [3 x i8*], [3 x i8*]* %{{.+}}, i32 0, i32 2
     // CK2:     [[CBP:%.+]] = bitcast i8** [[BP]] to double***
     // CK2:     store double** [[RVAL:%.+]], double*** [[CBP]],
     // CK2:     call void @__tgt_target_data_begin{{.+}}[[MTYPE02]]
diff --git a/clang/test/OpenMP/target_map_codegen.cpp b/clang/test/OpenMP/target_map_codegen.cpp
index e63f19ad73c0a..a394f5b1c3d53 100644
--- a/clang/test/OpenMP/target_map_codegen.cpp
+++ b/clang/test/OpenMP/target_map_codegen.cpp
@@ -3874,7 +3874,7 @@ int explicit_maps_template_args_and_members(int a){
 
 // CK22-LABEL: @.__omp_offloading_{{.*}}explicit_maps_globals{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK22: [[SIZE04:@.+]] = private {{.*}}constant [1 x i64] [i64 20]
-// CK22: [[MTYPE04:@.+]] = private {{.*}}constant [1 x i64] [i64 35]
+// CK22: [[MTYPE04:@.+]] = private {{.*}}constant [1 x i64] [i64 51]
 
 // CK22-LABEL: @.__omp_offloading_{{.*}}explicit_maps_globals{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK22: [[SIZE05:@.+]] = private {{.*}}constant [1 x i64] [i64 4]
@@ -3894,7 +3894,7 @@ int explicit_maps_template_args_and_members(int a){
 
 // CK22-LABEL: @.__omp_offloading_{{.*}}explicit_maps_globals{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK22: [[SIZE09:@.+]] = private {{.*}}constant [1 x i64] [i64 20]
-// CK22: [[MTYPE09:@.+]] = private {{.*}}constant [1 x i64] [i64 35]
+// CK22: [[MTYPE09:@.+]] = private {{.*}}constant [1 x i64] [i64 51]
 
 // CK22-LABEL: @.__omp_offloading_{{.*}}explicit_maps_globals{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK22: [[SIZE10:@.+]] = private {{.*}}constant [1 x i64] [i64 4]
@@ -3914,7 +3914,7 @@ int explicit_maps_template_args_and_members(int a){
 
 // CK22-LABEL: @.__omp_offloading_{{.*}}explicit_maps_globals{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 // CK22: [[SIZE14:@.+]] = private {{.*}}constant [1 x i64] [i64 20]
-// CK22: [[MTYPE14:@.+]] = private {{.*}}constant [1 x i64] [i64 35]
+// CK22: [[MTYPE14:@.+]] = private {{.*}}constant [1 x i64] [i64 51]
 
 int a;
 int c[100];
@@ -4010,11 +4010,10 @@ int explicit_maps_globals(void){
 
   // CK22-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 0
   // CK22-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 0
-  // CK22-DAG: [[CBP0:%.+]] = bitcast i8** [[BP0]] to i32**
+  // CK22-DAG: [[CBP0:%.+]] = bitcast i8** [[BP0]] to i32***
   // CK22-DAG: [[CP0:%.+]] = bitcast i8** [[P0]] to i32**
-  // CK22-DAG: store i32* [[RVAR0:%.+]], i32** [[CBP0]]
+  // CK22-DAG: store i32** @d, i32*** [[CBP0]]
   // CK22-DAG: store i32* [[SEC0:%.+]], i32** [[CP0]]
-  // CK22-DAG: [[RVAR0]] = load i32*, i32** @d
   // CK22-DAG: [[SEC0]] = getelementptr {{.*}}i32* [[RVAR00:%.+]], i{{.+}} 2
   // CK22-DAG: [[RVAR00]] = load i32*, i32** @d
 
@@ -4093,11 +4092,10 @@ int explicit_maps_globals(void){
 
   // CK22-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 0
   // CK22-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 0
-  // CK22-DAG: [[CBP0:%.+]] = bitcast i8** [[BP0]] to [[ST]]**
+  // CK22-DAG: [[CBP0:%.+]] = bitcast i8** [[BP0]] to [[ST]]***
   // CK22-DAG: [[CP0:%.+]] = bitcast i8** [[P0]] to [[ST]]**
-  // CK22-DAG: store [[ST]]* [[RVAR0:%.+]], [[ST]]** [[CBP0]]
+  // CK22-DAG: store [[ST]]** @sd, [[ST]]*** [[CBP0]]
   // CK22-DAG: store [[ST]]* [[SEC0:%.+]], [[ST]]** [[CP0]]
-  // CK22-DAG: [[RVAR0]] = load [[ST]]*, [[ST]]** @sd
   // CK22-DAG: [[SEC0]] = getelementptr {{.*}}[[ST]]* [[RVAR00:%.+]], i{{.+}} 2
   // CK22-DAG: [[RVAR00]] = load [[ST]]*, [[ST]]** @sd
 
@@ -4176,11 +4174,10 @@ int explicit_maps_globals(void){
 
   // CK22-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 0
   // CK22-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 0
-  // CK22-DAG: [[CBP0:%.+]] = bitcast i8** [[BP0]] to [[STT]]**
+  // CK22-DAG: [[CBP0:%.+]] = bitcast i8** [[BP0]] to [[STT]]***
   // CK22-DAG: [[CP0:%.+]] = bitcast i8** [[P0]] to [[STT]]**
-  // CK22-DAG: store [[STT]]* [[RVAR0:%.+]], [[STT]]** [[CBP0]]
+  // CK22-DAG: store [[STT]]** @std, [[STT]]*** [[CBP0]]
   // CK22-DAG: store [[STT]]* [[SEC0:%.+]], [[STT]]** [[CP0]]
-  // CK22-DAG: [[RVAR0]] = load [[STT]]*, [[STT]]** @std
   // CK22-DAG: [[SEC0]] = getelementptr {{.*}}[[STT]]* [[RVAR00:%.+]], i{{.+}} 2
   // CK22-DAG: [[RVAR00]] = load [[STT]]*, [[STT]]** @std
 
diff --git a/clang/test/OpenMP/target_update_codegen.cpp b/clang/test/OpenMP/target_update_codegen.cpp
index feea1129be4cc..9eab8b3367c4e 100644
--- a/clang/test/OpenMP/target_update_codegen.cpp
+++ b/clang/test/OpenMP/target_update_codegen.cpp
@@ -310,22 +310,23 @@ void device_side_scan(int arg) {
 
 #ifdef CK5
 
-// CK5: [[SIZE00:@.+]] = {{.+}}constant [2 x i[[sz:64|32]]] [i{{64|32}} {{8|4}}, i{{64|32}} 4]
-// CK5: [[MTYPE00:@.+]] = {{.+}}constant [2 x i64] [i64 33, i64 17]
+// CK5: [[SIZE00:@.+]] = {{.+}}constant [1 x i[[sz:64|32]]] [i{{64|32}} 4]
+// CK5: [[MTYPE00:@.+]] = {{.+}}constant [1 x i64] [i64 33]
 
 // CK5-LABEL: lvalue
 void lvalue(int *B, int l, int e) {
 
-  // CK5-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 2, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}]* [[SIZE00]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
+  // CK5-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 1, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[SIZE00]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
   // CK5-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
   // CK5-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
 
-  // CK5-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
-  // CK5-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
-  // CK5-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to i32***
+  // CK5-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 0
+  // CK5-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 0
+  // CK5-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to i32**
   // CK5-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to i32**
-  // CK5-DAG: store i32** [[B_ADDR:%.+]], i32*** [[BPC0]]
+  // CK5-DAG: store i32* [[B_VAL:%.+]], i32** [[BPC0]]
   // CK5-DAG: store i32* [[B_VAL_2:%.+]], i32** [[PC0]]
+  // CK5-DAG: [[B_VAL]] = load i32*, i32** [[B_ADDR:%.+]]
   // CK5-DAG: [[B_VAL_2]] = load i32*, i32** [[B_ADDR]]
   #pragma omp target update to(*B)
   *B += e;
@@ -351,28 +352,29 @@ void lvalue(int *B, int l, int e) {
 
 #ifdef CK6
 
-// CK6: [[SIZE00:@.+]] = {{.+}}constant [2 x i[[sz:64|32]]] [i{{64|32}} {{8|4}}, i{{64|32}} 4]
-// CK6: [[MTYPE00:@.+]] = {{.+}}constant [2 x i64] [i64 33, i64 17]
+// CK6: [[SIZE00:@.+]] = {{.+}}constant [1 x i[[sz:64|32]]] [i{{64|32}} 4]
+// CK6: [[MTYPE00:@.+]] = {{.+}}constant [1 x i64] [i64 33]
 
 // CK6-LABEL: lvalue
 void lvalue(int *B, int l, int e) {
 
-  // CK6-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 2, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}]* [[SIZE00]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
+  // CK6-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 1, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[SIZE00]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
   // CK6-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
   // CK6-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
 
-  // CK6-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
-  // CK6-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
-  // CK6-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to i32***
+  // CK6-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 0
+  // CK6-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 0
+  // CK6-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to i32**
   // CK6-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to i32**
-  // CK6-DAG: store i32** [[B_ADDR:%.+]], i32*** [[BPC0]]
+  // CK6-DAG: store i32* [[TWO:%.+]], i32** [[BPC0]]
   // CK6-DAG: store i32* [[ADD_PTR:%.+]], i32** [[PC0]]
   // CK6-64-DAG: [[ADD_PTR]] = getelementptr inbounds i32, i32* [[ONE:%.+]], i{{32|64}} [[IDX_EXT:%.+]]
   // CK6-32-DAG: [[ADD_PTR]] = getelementptr inbounds i32, i32* [[ONE:%.+]], i{{32|64}} [[L_VAL:%.+]]
   // CK6-64-DAG: [[IDX_EXT]] = sext i32 [[L_VAL:%.+]] to i64
   // CK6-DAG: [[L_VAL]] = load i32, i32* [[L_ADDR:%.+]]
   // CK6-DAG: store i32 {{.+}}, i32* [[L_ADDR]]
-  // CK6-DAG: [[ONE]] = load i32*, i32** [[B_ADDR]]
+  // CK6-DAG: [[ONE]] = load i32*, i32** [[B_ADDR:%.+]]
+  // CK6-DAG: [[TWO]] = load i32*, i32** [[B_ADDR]]
   #pragma omp target update to(*(B+l))
   *(B+l) += e;
   #pragma omp target update from(*(B+l))
@@ -397,25 +399,26 @@ void lvalue(int *B, int l, int e) {
 
 #ifdef CK7
 
-// CK7: [[SIZE00:@.+]] = {{.+}}constant [2 x i[[sz:64|32]]] [i{{64|32}} {{8|4}}, i{{64|32}} 4]
-// CK7: [[MTYPE00:@.+]] = {{.+}}constant [2 x i64] [i64 33, i64 17]
+// CK7: [[SIZE00:@.+]] = {{.+}}constant [1 x i[[sz:64|32]]] [i{{64|32}} 4]
+// CK7: [[MTYPE00:@.+]] = {{.+}}constant [1 x i64] [i64 33]
 
 // CK7-LABEL: lvalue
 void lvalue(int *B, int l, int e) {
 
-  // CK7-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 2, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}]* [[SIZE00]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
+  // CK7-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 1, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[SIZE00]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
   // CK7-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
   // CK7-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
 
-  // CK7-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
-  // CK7-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
-  // CK7-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to i32***
+  // CK7-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 0
+  // CK7-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 0
+  // CK7-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to i32**
   // CK7-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to i32**
-  // CK7-DAG: store i32** [[B_ADDR:%.+]], i32*** [[BPC0]]
+  // CK7-DAG: store i32* [[B_VAL:%.+]], i32** [[BPC0]]
   // CK7-DAG: store i32* [[ARRAY_IDX:%.+]], i32** [[PC0]]
   // CK7-DAG: [[ARRAY_IDX]] = getelementptr inbounds i32, i32* [[ADD_PTR:%.+]], i{{32|64}} [[IDX_PROM:%.+]]
   // CK7-64-DAG: [[ADD_PTR]] = getelementptr inbounds i32, i32* [[ONE:%.+]], i64 [[IDX_EXT:%.+]]
   // CK7-32-DAG: [[ADD_PTR]] = getelementptr inbounds i32, i32* [[B_VAL_2:%.+]], i32 [[L_VAL:%.+]]
+  // CK7-32-DAG: [[B_VAL]] = load i32*, i32** [[B_ADDR:%.+]]
   // CK7-32-DAG: [[B_VAL_2]] = load i32*, i32** [[B_ADDR]]
   // CK7-32-DAG: [[L_VAL]] = load i32, i32* [[L_ADDR:%.+]]
   // CK7-32-DAG: [[IDX_PROM]] = load i32, i32* [[L_ADDR]]
@@ -446,18 +449,18 @@ void lvalue(int *B, int l, int e) {
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 
 #ifdef CK8
-// CK8: [[SIZE00:@.+]] = {{.+}}constant [3 x i[[sz:64|32]]] [i{{64|32}} {{8|4}}, i{{64|32}} {{8|4}}, i{{64|32}} 4]
-// CK8: [[MTYPE00:@.+]] = {{.+}}constant [3 x i64] [i64 33, i64 16, i64 17]
+// CK8: [[SIZE00:@.+]] = {{.+}}constant [2 x i[[sz:64|32]]] [i{{64|32}} {{8|4}}, i{{64|32}} 4]
+// CK8: [[MTYPE00:@.+]] = {{.+}}constant [2 x i64] [i64 33, i64 17]
 
 // CK8-LABEL: lvalue
 void lvalue(int **B, int l, int e) {
 
-  // CK8-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 3, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], {{.+}}getelementptr {{.+}}[3 x i{{.+}}], [3 x i{{.+}}]* [[SIZE00]], {{.+}}getelementptr {{.+}}[3 x i{{.+}}, [3 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
+  // CK8-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 2, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}], [2 x i{{.+}}]* [[SIZE00]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}, [2 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
   // CK8-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
   // CK8-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
 
-  // CK8-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 2
-  // CK8-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 2
+  // CK8-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
+  // CK8-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
   // CK8-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to i32***
   // CK8-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to i32**
   // CK8-DAG: store i32** [[ARRAY_IDX_1:%.+]], i32*** [[BPC0]]
@@ -501,19 +504,19 @@ struct S {
   double *p;
 };
 
-// CK9: [[MTYPE00:@.+]] = {{.+}}constant [3 x i64] [i64 32, i64 281474976710657, i64 281474976710673]
+// CK9: [[MTYPE00:@.+]] = {{.+}}constant [2 x i64] [i64 32, i64 281474976710673]
 
 // CK9-LABEL: lvalue
 void lvalue(struct S *s, int l, int e) {
 
-  // CK9-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 3, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i{{.+}} [[GSIZE:%.+]], {{.+}}getelementptr {{.+}}[3 x i{{.+}}, [3 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
+  // CK9-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 2, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i{{.+}} [[GSIZE:%.+]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}, [2 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
   // CK9-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
   // CK9-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
   // CK9-DAG: [[GSIZE]] = getelementptr inbounds {{.+}}[[SIZE:%[^,]+]]
   //
-  // CK9-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 2
-  // CK9-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 2
-  // CK9-DAG: [[SIZE0:%.+]] = getelementptr inbounds {{.+}}[[SIZE]], i{{.+}} 0, i{{.+}} 2
+  // CK9-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
+  // CK9-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
+  // CK9-DAG: [[SIZE0:%.+]] = getelementptr inbounds {{.+}}[[SIZE]], i{{.+}} 0, i{{.+}} 1
   // CK9-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to double***
   // CK9-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to double**
   // CK9-DAG: store double** [[P:%.+]], double*** [[BPC0]]
@@ -551,19 +554,19 @@ struct S {
   double *p;
 };
 
-// CK10: [[MTYPE00:@.+]] = {{.+}}constant [3 x i64] [i64 32, i64 281474976710657, i64 281474976710673]
+// CK10: [[MTYPE00:@.+]] = {{.+}}constant [2 x i64] [i64 32, i64 281474976710673]
 
 // CK10-LABEL: lvalue
 void lvalue(struct S *s, int l, int e) {
 
-  // CK10-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 3, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i{{.+}} [[GSIZE:%.+]], {{.+}}getelementptr {{.+}}[3 x i{{.+}}, [3 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
+  // CK10-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 2, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i{{.+}} [[GSIZE:%.+]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}, [2 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
   // CK10-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
   // CK10-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
   // CK10-DAG: [[GSIZE]] = getelementptr inbounds {{.+}}[[SIZE:%[^,]+]]
   //
-  // CK10-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 2
-  // CK10-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 2
-  // CK10-DAG: [[SIZE0:%.+]] = getelementptr inbounds {{.+}}[[SIZE]], i{{.+}} 0, i{{.+}} 2
+  // CK10-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
+  // CK10-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
+  // CK10-DAG: [[SIZE0:%.+]] = getelementptr inbounds {{.+}}[[SIZE]], i{{.+}} 0, i{{.+}} 1
   // CK10-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to double***
   // CK10-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to double**
   // CK10-DAG: store double** [[P_VAL:%.+]], double*** [[BPC0]]
@@ -601,19 +604,19 @@ void lvalue(struct S *s, int l, int e) {
 struct S {
   double *p;
 };
-// CK11: [[MTYPE00:@.+]] = {{.+}}constant [3 x i64] [i64 32, i64 281474976710657, i64 281474976710673]
+// CK11: [[MTYPE00:@.+]] = {{.+}}constant [2 x i64] [i64 32, i64 281474976710673]
 
 // CK11-LABEL: lvalue
 void lvalue(struct S *s, int l, int e) {
 
-  // CK11-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 3, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i{{.+}} [[GSIZE:%.+]], {{.+}}getelementptr {{.+}}[3 x i{{.+}}, [3 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
+  // CK11-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 2, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i{{.+}} [[GSIZE:%.+]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}, [2 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
   // CK11-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
   // CK11-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
   // CK11-DAG: [[GSIZE]] = getelementptr inbounds {{.+}}[[SIZE:%[^,]+]]
   //
-  // CK11-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 2
-  // CK11-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 2
-  // CK11-DAG: [[SIZE0:%.+]] = getelementptr inbounds {{.+}}[[SIZE]], i{{.+}} 0, i{{.+}} 2
+  // CK11-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
+  // CK11-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
+  // CK11-DAG: [[SIZE0:%.+]] = getelementptr inbounds {{.+}}[[SIZE]], i{{.+}} 0, i{{.+}} 1
   // CK11-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to double***
   // CK11-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to double**
   // CK11-DAG: store double** [[P:%.+]], double*** [[BPC0]]
@@ -653,41 +656,44 @@ struct S {
   double *p;
   struct S *sp;
 };
-// CK12: [[MTYPE00:@.+]] = {{.+}}constant [4 x i64] [i64 32, i64 281474976710657, i64 281474976710672, i64 17]
+// CK12: [[MTYPE00:@.+]] = {{.+}}constant [3 x i64] [i64 32, i64 281474976710672, i64 17]
 
 // CK12-LABEL: lvalue
 void lvalue(struct S *s, int l, int e) {
 
-  // CK12-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 4, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i{{.+}} [[GSIZE:%.+]], {{.+}}getelementptr {{.+}}[4 x i{{.+}}, [4 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
+  // CK12-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 3, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i{{.+}} [[GSIZE:%.+]], {{.+}}getelementptr {{.+}}[3 x i{{.+}}, [3 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
   // CK12-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
   // CK12-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
   // CK12-DAG: [[GSIZE]] = getelementptr inbounds {{.+}}[[SIZE:%[^,]+]]
   //
-  // CK12-DAG: [[BP2:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 3
-  // CK12-DAG: [[P2:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 3
-  // CK12-DAG: [[SIZE2:%.+]] = getelementptr inbounds {{.+}}[[SIZE]], i{{.+}} 0, i{{.+}} 3
+  // CK12-DAG: [[BP2:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 2
+  // CK12-DAG: [[P2:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 2
+  // CK12-DAG: [[SIZE2:%.+]] = getelementptr inbounds {{.+}}[[SIZE]], i{{.+}} 0, i{{.+}} 2
   // CK12-DAG: [[BPC2:%.+]] = bitcast i8** [[BP2]] to double***
   // CK12-DAG: [[PC2:%.+]] = bitcast i8** [[P2]] to double**
   // CK12-DAG: store double** [[P_VAL:%.+]], double*** [[BPC2]]
   // CK12-DAG: store double* [[SIX:%.+]], double** [[PC2]]
   // CK12-DAG: store i{{.+}} 8, i{{.+}}* [[SIZE2]]
-  // CK12-DAG: [[BP1:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 2
-  // CK12-DAG: [[P1:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 2
-  // CK12-DAG: [[SIZE1:%.+]] = getelementptr inbounds {{.+}}[[SIZE]], i{{.+}} 0, i{{.+}} 2
+  // CK12-DAG: [[BP1:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
+  // CK12-DAG: [[P1:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
+  // CK12-DAG: [[SIZE1:%.+]] = getelementptr inbounds {{.+}}[[SIZE]], i{{.+}} 0, i{{.+}} 1
   // CK12-DAG: [[BPC1:%.+]] = bitcast i8** [[BP1]] to [[STRUCT_S:%.+]]***
   // CK12-DAG: [[PC1:%.+]] = bitcast i8** [[P1]] to double***
   // CK12-DAG: store [[STRUCT_S]]** [[SP:%.+]], [[STRUCT_S]]*** [[BPC1]]
   // CK12-DAG: store double** [[P_VAL:%.+]], double*** [[PC1]]
   // CK12-DAG: store i{{.+}} {{4|8}}, i{{.+}}* [[SIZE1]]
-  // CK12-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
-  // CK12-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
-  // CK12-DAG: [[SIZE0:%.+]] = getelementptr inbounds {{.+}}[[SIZE]], i{{.+}} 0, i{{.+}} 1
+  // CK12-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 0
+  // CK12-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 0
+  // CK12-DAG: [[SIZE0:%.+]] = getelementptr inbounds {{.+}}[[SIZE]], i{{.+}} 0, i{{.+}} 0
   // CK12-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to [[STRUCT_S:%.+]]**
   // CK12-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to [[STRUCT_S]]***
+  // CK12-DAG: store [[STRUCT_S]]* [[ZERO:%.+]], [[STRUCT_S]]** [[BPC0]]
+  // CK12-DAG: store [[STRUCT_S]]** [[SP]], [[STRUCT_S]]*** [[PC0]]
   // CK12-DAG: store [[STRUCT_S]]** [[S:%.+]], [[STRUCT_S]]*** [[S_VAL:%.+]]
   // CK12-DAG: store i{{.+}} {{.+}}, i{{.+}}* [[SIZE0]]
   // CK12-DAG: [[SP]] = getelementptr inbounds [[STRUCT_S]], [[STRUCT_S]]* [[ONE:%.+]], i32 0, i32 1
-  // CK12-DAG: [[ONE]] = load %struct.S*, %struct.S** [[S]],
+  // CK12-DAG: [[ONE]] = load [[STRUCT_S]]*, [[STRUCT_S]]** [[S:%.+]],
+  // CK12-DAG: [[ZERO]] = load [[STRUCT_S]]*, [[STRUCT_S]]** [[S]],
   #pragma omp target update to(*(s->sp->p))
     *(s->sp->p) = e;
   #pragma omp target update from(*(s->sp->p))
@@ -711,21 +717,21 @@ void lvalue(struct S *s, int l, int e) {
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 #ifdef CK13
 
-// CK13: [[SIZE00:@.+]] = {{.+}}constant [2 x i64] [i64 {{8|4}}, i64 4]
-// CK13: [[MTYPE00:@.+]] = {{.+}}constant [2 x i64] [i64 33, i64 17]
+// CK13: [[SIZE00:@.+]] = {{.+}}constant [1 x i64] [i64 4]
+// CK13: [[MTYPE00:@.+]] = {{.+}}constant [1 x i64] [i64 33]
 
 // CK13-LABEL: lvalue
 void lvalue(int **BB, int a, int b) {
 
-  // CK13-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 2, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}]* [[SIZE00]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
+  // CK13-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 1, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[SIZE00]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
   // CK13-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
   // CK13-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
 
-  // CK13-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
-  // CK13-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
-  // CK13-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to i32****
+  // CK13-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 0
+  // CK13-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 0
+  // CK13-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to i32***
   // CK13-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to i32**
-  // CK13-DAG: store i32*** [[BB_ADDR:%.+]], i32**** [[BPC0]]
+  // CK13-DAG: store i32** [[B_VAL1:%.+]], i32*** [[BPC0]]
   // CK13-DAG: store i32* [[ADD_PTR_2:%.+]], i32** [[PC0]]
   // CK13-64-DAG: [[ADD_PTR_2]] = getelementptr inbounds i32, i32* [[RESULT:%.+]], i64 [[IDX_EXT_1:%.+]]
   // CK13-32-DAG: [[ADD_PTR_2]] = getelementptr inbounds i32, i32* [[RESULT:%.+]], i32 [[B_ADDR:%.+]]
@@ -734,7 +740,8 @@ void lvalue(int **BB, int a, int b) {
   // CK13-64-DAG: [[ADD_PTR]] = getelementptr inbounds i32*, i32** [[B_VAL:%.+]], i64 [[IDX_EXT:%.+]]
   // CK13-32-DAG: [[ADD_PTR]] = getelementptr inbounds i32*, i32** [[B_VAL:%.+]], i32 [[A_ADDR:%.+]]
   // CK13-64-DAG: [[IDX_EXT]] = sext i32 [[TWO:%.+]] to i64
-  // CK13-DAG: [[B_VAL]] = load i32**, i32*** [[BB_ADDR]]
+  // CK13-DAG: [[B_VAL]] = load i32**, i32*** [[BB_ADDR:%.+]]
+  // CK13-DAG: [[B_VAL1]] = load i32**, i32*** [[BB_ADDR]]
   #pragma omp target update to(*(*(BB+a)+b))
   *(*(BB+a)+b) = 1;
   #pragma omp target update from(*(*(BB+a)+b))
@@ -831,7 +838,7 @@ void lvalue_member(SSA *sap) {
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 #ifdef CK15
 
-// CK15: [[MTYPE00:@.+]] = {{.+}}constant [3 x i64] [i64 32, i64 281474976710657, i64 281474976710673]
+// CK15: [[MTYPE00:@.+]] = {{.+}}constant [2 x i64] [i64 32, i64 281474976710673]
 
 struct SSA {
   double *p;
@@ -842,36 +849,27 @@ struct SSA {
 //CK-15-LABEL: lvalue_member
 void lvalue_member(SSA *sap) {
 
-  // CK15-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 3, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i64* [[GSIZE:%.+]], {{.+}}getelementptr {{.+}}[3 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
+  // CK15-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 2, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i64* [[GSIZE:%.+]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
   // CK15-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
   // CK15-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
   // CK15-DAG: [[GSIZE]] = getelementptr inbounds {{.+}}[[SIZE:%[^,]+]]
 
-  // CK15-DAG: [[BP2:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 2
-  // CK15-DAG: [[P2:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 2
-  // CK15-DAG: [[SIZE2:%.+]] = getelementptr inbounds {{.+}}[[SIZE]], i{{.+}} 0, i{{.+}} 2
-  // CK15-DAG: [[BPC2:%.+]] = bitcast i8** [[BP2]] to double***
-  // CK15-DAG: [[PC2:%.+]] = bitcast i8** [[P2]] to double**
-  // CK15-DAG: store double** [[P_VAL:%.+]], double*** [[BPC2]]
-  // CK15-DAG: store double* [[ADD_PTR:%.+]], double** [[PC2]]
-  // CK15-DAG: store i64 8, i64* [[SIZE2]]
-  // CK15-DAG: [[ADD_PTR]] = getelementptr inbounds double, double* [[THREE:%.+]], i{{.+}} 3
-  // CK15-DAG: [[THREE]] = load double*, double** [[P_VAL_1:%.+]]
-  // CK15-DAG: [[P_VAL]] = getelementptr inbounds [[SSA:%.+]], [[SSA:%.+]]* [[THIS:%.+]], i32 0, i32 0
   // CK15-DAG: [[BP1:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
   // CK15-DAG: [[P1:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
   // CK15-DAG: [[SIZE1:%.+]] = getelementptr inbounds {{.+}}[[SIZE]], i{{.+}} 0, i{{.+}} 1
-  // CK15-DAG: [[BPC1:%.+]] = bitcast i8** [[BP1]] to [[SSA]]**
-  // CK15-DAG: [[PC1:%.+]] = bitcast i8** [[P1]] to [[SSA]]***
-  // CK15-DAG: store [[SSA]]* [[SAP_VAL:%.+]], [[SSA]]** [[BPC1]],
-  // CK15-DAG: store [[SSA]]** [[SAP_ADDR:%.+]], [[SSA]]*** [[PC1]]
-  // CK15-DAG: store i{{.+}} {{8|4}}, i{{.+}}* [[SIZE1]]
-  // CK15-DAG: [[SAP_VAL]] = load [[SSA]]*, [[SSA]]** [[SAP_ADDR]],
+  // CK15-DAG: [[BPC1:%.+]] = bitcast i8** [[BP1]] to double***
+  // CK15-DAG: [[PC1:%.+]] = bitcast i8** [[P1]] to double**
+  // CK15-DAG: store double** [[P_VAL:%.+]], double*** [[BPC1]]
+  // CK15-DAG: store double* [[ADD_PTR:%.+]], double** [[PC1]]
+  // CK15-DAG: store i64 {{4|8}}, i64* [[SIZE1]]
+  // CK15-DAG: [[ADD_PTR]] = getelementptr inbounds double, double* [[THREE:%.+]], i{{.+}} 3
+  // CK15-DAG: [[THREE]] = load double*, double** [[P_VAL_1:%.+]]
+  // CK15-DAG: [[P_VAL]] = getelementptr inbounds [[SSA:%.+]], [[SSA:%.+]]* [[THIS:%.+]], i32 0, i32 0
   // CK15-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 0
   // CK15-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 0
   // CK15-DAG: [[SIZE0:%.+]] = getelementptr inbounds {{.+}}[[SIZE]], i{{.+}} 0, i{{.+}} 0
-  // CK15-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to [[SSA]]***
-  // CK15-DAG: store [[SSA]]** [[SAP_ADDR]], [[SSA]]*** [[BPC0]],
+  // CK15-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to [[SSA]]**
+  // CK15-DAG: store [[SSA]]* [[ZERO:%.+]], [[SSA]]** [[BPC0]],
   // CK15-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to double***
   // CK15-DAG: store double** [[P_VAL]], double*** [[PC0]],
   // CK15-DAG: store i{{.+}} [[COMPUTE_SIZE:%.+]], i{{.+}}* [[SIZE0]]
@@ -881,6 +879,7 @@ void lvalue_member(SSA *sap) {
   // CK15-DAG: [[EIGHT]] = ptrtoint i8* [[FIVE:%.+]] to i64
   // CK15-DAG: [[SIX]] = bitcast double** {{.+}} to i8*
   // CK15-DAG: [[FIVE]] = bitcast double** {{.+}} to i8*
+  // CK15-DAG: [[ZERO]] = load [[SSA]]*, [[SSA]]** %{{.+}},
   #pragma omp target update to(*(3+sap->p))
   *(3+sap->p) = 1;
   #pragma omp target update from(*(3+sap->p))
@@ -904,25 +903,26 @@ void lvalue_member(SSA *sap) {
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 #ifdef CK16
 
-// CK16: [[SIZE00:@.+]] = {{.+}}constant [2 x i64] [i64 {{8|4}}, i64 4]
-// CK16: [[MTYPE00:@.+]] = {{.+}}constant [2 x i64] [i64 33, i64 17]
+// CK16: [[SIZE00:@.+]] = {{.+}}constant [1 x i64] [i64 4]
+// CK16: [[MTYPE00:@.+]] = {{.+}}constant [1 x i64] [i64 33]
 
 //CK16-LABEL: lvalue_find_base
 void lvalue_find_base(float *f, int *i) {
 
-  // CK16-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 2, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}]* [[SIZE00]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
+  // CK16-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 1, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[SIZE00]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
   // CK16-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
   // CK16-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
 
-  // CK16-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
-  // CK16-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
-  // CK16-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to float***
+  // CK16-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 0
+  // CK16-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 0
+  // CK16-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to float**
   // CK16-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to float**
-  // CK16-DAG: store float** [[F_ADDR:%.+]], float*** [[BPC0]]
+  // CK16-DAG: store float* [[F:%.+]], float** [[BPC0]]
   // CK16-DAG: store float* [[ADD_PTR:%.+]], float** [[PC0]]
   // CK16-32-DAG: [[ADD_PTR]] = getelementptr inbounds float, float* [[THREE:%.+]], i32 [[I:%.+]]
   // CK16-64-DAG: [[ADD_PTR]] = getelementptr inbounds float, float* [[THREE:%.+]], i64 [[IDX_EXT:%.+]]
-  // CK16-DAG: [[THREE]] = load float*, float** [[F_ADDR]],
+  // CK16-DAG: [[THREE]] = load float*, float** [[F_ADDR:%.+]],
+  // CK16-DAG: [[F]] = load float*, float** [[F_ADDR]],
   // CK16-64-DAG: [[IDX_EXT]] = sext i32 [[I:%.+]] to i64
 
   #pragma omp target update to(*(*i+f))
@@ -948,8 +948,8 @@ void lvalue_find_base(float *f, int *i) {
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 #ifdef CK17
 
-// CK17: [[SIZE00:@.+]] = {{.+}}constant [2 x i64] [i64 {{4|8}}, i64 4]
-// CK17: [[MTYPE00:@.+]] = {{.+}}constant [2 x i64] [i64 33, i64 17]
+// CK17: [[SIZE00:@.+]] = {{.+}}constant [1 x i64] [i64 4]
+// CK17: [[MTYPE00:@.+]] = {{.+}}constant [1 x i64] [i64 33]
 
 struct SSA {
   int i;
@@ -959,15 +959,15 @@ struct SSA {
 //CK17-LABEL: lvalue_find_base
 void lvalue_find_base(float **f, SSA *sa) {
 
-  // CK17-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 2, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}]* [[SIZE00]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
+  // CK17-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 1, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[SIZE00]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
   // CK17-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
   // CK17-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
 
-  // CK17-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
-  // CK17-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
-  // CK17-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to float****
+  // CK17-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 0
+  // CK17-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 0
+  // CK17-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to float***
   // CK17-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to float**
-  // CK17-DAG: store float*** [[F_ADDR:%.+]], float**** [[BPC0]],
+  // CK17-DAG: store float** [[F_VAL:%.+]], float*** [[BPC0]],
   // CK17-DAG: store float* [[ADD_PTR_4:%.+]], float** [[PC0]],
   // CK17-64-DAG: [[ADD_PTR_4]] = getelementptr inbounds float, float* [[SEVEN:%.+]], i64 [[IDX_EXT_3:%.+]]
   // CK17-64-DAG: [[IDX_EXT_3]] = sext i32 [[I_VAL:%.+]] to i64
@@ -981,6 +981,8 @@ void lvalue_find_base(float **f, SSA *sa) {
   // CK17-DAG: [[FIVE]] = load i32, i32* [[I_2:%.+]],
   // CK17-DAG: [[I_2]] = getelementptr inbounds [[SSA:%.+]], [[SSA]]* [[FOUR:%.+]], i32 0, i32 0
   // CK17-DAG: [[FOUR]] = load [[SSA]]*, [[SSA]]** [[SSA_ADDR:%.+]],
+  // CK17-DAG: [[F]] = load float**, float*** [[F_ADDR:%.+]],
+  // CK17-DAG: [[F_VAL]] = load float**, float*** [[F_ADDR]],
 
   #pragma omp target update to(*(sa->sa->i+*(1+sa->i+f)))
   *(sa->sa->i+*(1+sa->i+f)) = 1;
@@ -1005,13 +1007,13 @@ void lvalue_find_base(float **f, SSA *sa) {
 // SIMD-ONLY18-NOT: {{__kmpc|__tgt}}
 #ifdef CK18
 
-// CK18-DAG: [[MTYPE_TO:@.+]] = {{.+}}constant [2 x i64] [i64 33, i64 16]
-// CK18-DAG: [[MTYPE_FROM:@.+]] = {{.+}}constant [2 x i64] [i64 34, i64 16]
+// CK18-DAG: [[MTYPE_TO:@.+]] = {{.+}}constant [1 x i64] [i64 33]
+// CK18-DAG: [[MTYPE_FROM:@.+]] = {{.+}}constant [1 x i64] [i64 34]
 
 //CK18-LABEL: array_shaping
 void array_shaping(float *f, int sa) {
 
-  // CK18-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 2, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i64* [[GEPS:%.+]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}]* [[MTYPE_TO]]{{.+}}, i8** null)
+  // CK18-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 1, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i64* [[GEPS:%.+]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[MTYPE_TO]]{{.+}}, i8** null)
   // CK18-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
   // CK18-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
   // CK18-DAG: [[GEPS]] = getelementptr inbounds {{.+}}[[S:%[^,]+]]
@@ -1021,23 +1023,12 @@ void array_shaping(float *f, int sa) {
   // CK18-DAG: [[S0:%.+]] = getelementptr inbounds {{.+}}[[S]], i{{.+}} 0, i{{.+}} 0
 
   // CK18-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to float**
-  // CK18-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to float***
+  // CK18-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to float**
 
   // CK18-DAG: store float* [[F1:%.+]], float** [[BPC0]],
-  // CK18-DAG: store float** [[F_ADDR:%.+]], float*** [[PC0]],
-  // CK18-DAG: store i64 {{8|4}}, i64* [[S0]],
-  // CK18-DAG: [[F1]] = load float*, float** [[F_ADDR]],
-
-  // CK18-DAG: [[BP1:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
-  // CK18-DAG: [[P1:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
-  // CK18-DAG: [[S1:%.+]] = getelementptr inbounds {{.+}}[[S]], i{{.+}} 0, i{{.+}} 1
-
-  // CK18-DAG: [[BPC1:%.+]] = bitcast i8** [[BP1]] to float***
-  // CK18-DAG: [[PC1:%.+]] = bitcast i8** [[P1]] to float**
-
-  // CK18-DAG: store float** [[F_ADDR]], float*** [[BPC1]],
-  // CK18-DAG: store float* [[F2:%.+]], float** [[PC1]],
-  // CK18-DAG: store i64 [[SIZE:%.+]], i64* [[S1]],
+  // CK18-DAG: store float* [[F2:%.+]], float** [[PC0]],
+  // CK18-DAG: store i64 [[SIZE:%.+]], i64* [[S0]],
+  // CK18-DAG: [[F1]] = load float*, float** [[F_ADDR:%.+]],
   // CK18-DAG: [[F2]] = load float*, float** [[F_ADDR]],
 
   // CK18-64-DAG: [[SIZE]] = mul nuw i64 [[SZ1:%.+]], 4
@@ -1047,7 +1038,7 @@ void array_shaping(float *f, int sa) {
   // CK18-32-DAG: [[SZ2]] = mul nuw i32 12, %{{.+}}
   #pragma omp target update to(([3][sa][4])f)
   sa = 1;
-  // CK18-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 2, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i64* [[GEPS:%.+]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}]* [[MTYPE_FROM]]{{.+}}, i8** null)
+  // CK18-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 1, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i64* [[GEPS:%.+]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[MTYPE_FROM]]{{.+}}, i8** null)
   // CK18-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
   // CK18-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
   // CK18-DAG: [[GEPS]] = getelementptr inbounds {{.+}}[[S:%[^,]+]]
@@ -1057,23 +1048,12 @@ void array_shaping(float *f, int sa) {
   // CK18-DAG: [[S0:%.+]] = getelementptr inbounds {{.+}}[[S]], i{{.+}} 0, i{{.+}} 0
 
   // CK18-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to float**
-  // CK18-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to float***
+  // CK18-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to float**
 
   // CK18-DAG: store float* [[F1:%.+]], float** [[BPC0]],
-  // CK18-DAG: store float** [[F_ADDR:%.+]], float*** [[PC0]],
-  // CK18-DAG: store i64 {{8|4}}, i64* [[S0]],
+  // CK18-DAG: store float* [[F2:%.+]], float** [[PC0]],
+  // CK18-DAG: store i64 [[SIZE:%.+]], i64* [[S0]],
   // CK18-DAG: [[F1]] = load float*, float** [[F_ADDR]],
-
-  // CK18-DAG: [[BP1:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
-  // CK18-DAG: [[P1:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
-  // CK18-DAG: [[S1:%.+]] = getelementptr inbounds {{.+}}[[S]], i{{.+}} 0, i{{.+}} 1
-
-  // CK18-DAG: [[BPC1:%.+]] = bitcast i8** [[BP1]] to float***
-  // CK18-DAG: [[PC1:%.+]] = bitcast i8** [[P1]] to float**
-
-  // CK18-DAG: store float** [[F_ADDR]], float*** [[BPC1]],
-  // CK18-DAG: store float* [[F2:%.+]], float** [[PC1]],
-  // CK18-DAG: store i64 [[SIZE:%.+]], i64* [[S1]],
   // CK18-DAG: [[F2]] = load float*, float** [[F_ADDR]],
 
   // CK18-64-DAG: [[SIZE]] = mul nuw i64 [[SZ1:%.+]], 5
diff --git a/openmp/libomptarget/src/omptarget.cpp b/openmp/libomptarget/src/omptarget.cpp
index 409089500015e..9c7ab6309503a 100644
--- a/openmp/libomptarget/src/omptarget.cpp
+++ b/openmp/libomptarget/src/omptarget.cpp
@@ -880,14 +880,9 @@ int target(int64_t DeviceId, void *HostPtr, int32_t ArgNum, void **ArgBases,
           return OFFLOAD_FAIL;
         }
       }
-    } else if (ArgTypes[I] & OMP_TGT_MAPTYPE_PTR_AND_OBJ) {
-      TgtPtrBegin = Device.getTgtPtrBegin(HstPtrBase, sizeof(void *), IsLast,
-                                          false, IsHostPtr);
-      TgtBaseOffset = 0; // no offset for ptrs.
-      DP("Obtained target argument " DPxMOD " from host pointer " DPxMOD " to "
-         "object " DPxMOD "\n",
-         DPxPTR(TgtPtrBegin), DPxPTR(HstPtrBase), DPxPTR(HstPtrBase));
     } else {
+      if (ArgTypes[I] & OMP_TGT_MAPTYPE_PTR_AND_OBJ)
+        HstPtrBase = *reinterpret_cast<void **>(HstPtrBase);
       TgtPtrBegin = Device.getTgtPtrBegin(HstPtrBegin, ArgSizes[I], IsLast,
                                           false, IsHostPtr);
       TgtBaseOffset = (intptr_t)HstPtrBase - (intptr_t)HstPtrBegin;
diff --git a/openmp/libomptarget/test/env/base_ptr_ref_count.c b/openmp/libomptarget/test/env/base_ptr_ref_count.c
new file mode 100644
index 0000000000000..4036954182faf
--- /dev/null
+++ b/openmp/libomptarget/test/env/base_ptr_ref_count.c
@@ -0,0 +1,51 @@
+// RUN: %libomptarget-compile-aarch64-unknown-linux-gnu && env LIBOMPTARGET_DEBUG=1 %libomptarget-run-aarch64-unknown-linux-gnu 2>&1 | %fcheck-aarch64-unknown-linux-gnu
+// RUN: %libomptarget-compile-powerpc64-ibm-linux-gnu && env LIBOMPTARGET_DEBUG=1 %libomptarget-run-powerpc64-ibm-linux-gnu 2>&1 | %fcheck-powerpc64-ibm-linux-gnu
+// RUN: %libomptarget-compile-powerpc64le-ibm-linux-gnu && env LIBOMPTARGET_DEBUG=1 %libomptarget-run-powerpc64le-ibm-linux-gnu 2>&1 | %fcheck-powerpc64le-ibm-linux-gnu
+// RUN: %libomptarget-compile-x86_64-pc-linux-gnu && env LIBOMPTARGET_DEBUG=1 %libomptarget-run-x86_64-pc-linux-gnu 2>&1 | %fcheck-x86_64-pc-linux-gnu
+// RUN: %libomptarget-compile-nvptx64-nvidia-cuda && env LIBOMPTARGET_DEBUG=1 %libomptarget-run-nvptx64-nvidia-cuda 2>&1 | %fcheck-nvptx64-nvidia-cuda
+// REQUIRES: libomptarget-debug
+
+#include <stdlib.h>
+
+int *allocate(size_t n) {
+  int *ptr = malloc(sizeof(int) * n);
+#pragma omp target enter data map(to : ptr[:n])
+  return ptr;
+}
+
+void deallocate(int *ptr, size_t n) {
+#pragma omp target exit data map(delete : ptr[:n])
+  free(ptr);
+}
+
+#pragma omp declare target
+int *cnt;
+void foo() {
+  ++(*cnt);
+}
+#pragma omp end declare target
+
+int main(void) {
+  int *A = allocate(10);
+  int *V = allocate(10);
+  deallocate(A, 10);
+  deallocate(V, 10);
+// CHECK-NOT: RefCount=2
+  cnt = malloc(sizeof(int));
+  *cnt = 0;
+#pragma omp target map(cnt[:1])
+  foo();
+  printf("Cnt = %d.\n", *cnt);
+// CHECK: Cnt = 1.
+  *cnt = 0;
+#pragma omp target data map(cnt[:1])
+#pragma omp target
+  foo();
+  printf("Cnt = %d.\n", *cnt);
+// CHECK: Cnt = 1.
+  free(cnt);
+
+  return 0;
+}
+
+

From 19756ef53a498b7aa1fbac9e3a7cd3aa8e110fad Mon Sep 17 00:00:00 2001
From: Johannes Doerfert <johannes@jdoerfert.de>
Date: Fri, 19 Jun 2020 09:38:04 -0500
Subject: [PATCH 0628/1035] [OpenMP][IRBuilder] Support allocas in nested
 parallel regions

We need to keep track of the alloca insertion point (which we already
communicate via the callback to the user) as we place allocas as well.

Reviewed By: fghanim, SouraVX

Differential Revision: https://reviews.llvm.org/D82470
---
 clang/lib/CodeGen/CGStmtOpenMP.cpp            |   8 +-
 .../llvm/Frontend/OpenMP/OMPIRBuilder.h       |  10 +-
 llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp     |  25 +-
 .../Frontend/OpenMPIRBuilderTest.cpp          | 226 +++++++++++++++++-
 4 files changed, 244 insertions(+), 25 deletions(-)

diff --git a/clang/lib/CodeGen/CGStmtOpenMP.cpp b/clang/lib/CodeGen/CGStmtOpenMP.cpp
index 0ee1133ebaa16..df1cc1666de4e 100644
--- a/clang/lib/CodeGen/CGStmtOpenMP.cpp
+++ b/clang/lib/CodeGen/CGStmtOpenMP.cpp
@@ -1707,9 +1707,11 @@ void CodeGenFunction::EmitOMPParallelDirective(const OMPParallelDirective &S) {
 
     CGCapturedStmtInfo CGSI(*CS, CR_OpenMP);
     CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(*this, &CGSI);
-    Builder.restoreIP(OMPBuilder.CreateParallel(Builder, BodyGenCB, PrivCB,
-                                                FiniCB, IfCond, NumThreads,
-                                                ProcBind, S.hasCancel()));
+    llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
+        AllocaInsertPt->getParent(), AllocaInsertPt->getIterator());
+    Builder.restoreIP(
+        OMPBuilder.CreateParallel(Builder, AllocaIP, BodyGenCB, PrivCB, FiniCB,
+                                  IfCond, NumThreads, ProcBind, S.hasCancel()));
     return;
   }
 
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
index 95eed59f1b3d0..f813a730342e2 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
@@ -156,6 +156,7 @@ class OpenMPIRBuilder {
   /// Generator for '#omp parallel'
   ///
   /// \param Loc The insert and source location description.
+  /// \param AllocaIP The insertion points to be used for alloca instructions.
   /// \param BodyGenCB Callback that will generate the region code.
   /// \param PrivCB Callback to copy a given variable (think copy constructor).
   /// \param FiniCB Callback to finalize variable copies.
@@ -166,10 +167,11 @@ class OpenMPIRBuilder {
   ///
   /// \returns The insertion position *after* the parallel.
   IRBuilder<>::InsertPoint
-  CreateParallel(const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB,
-                 PrivatizeCallbackTy PrivCB, FinalizeCallbackTy FiniCB,
-                 Value *IfCondition, Value *NumThreads,
-                 omp::ProcBindKind ProcBind, bool IsCancellable);
+  CreateParallel(const LocationDescription &Loc, InsertPointTy AllocaIP,
+                 BodyGenCallbackTy BodyGenCB, PrivatizeCallbackTy PrivCB,
+                 FinalizeCallbackTy FiniCB, Value *IfCondition,
+                 Value *NumThreads, omp::ProcBindKind ProcBind,
+                 bool IsCancellable);
 
   /// Generator for '#omp flush'
   ///
diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index 9468a3aa3c8dd..a5fe4ec87c46b 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -394,9 +394,10 @@ void OpenMPIRBuilder::emitCancelationCheckImpl(
 }
 
 IRBuilder<>::InsertPoint OpenMPIRBuilder::CreateParallel(
-    const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB,
-    PrivatizeCallbackTy PrivCB, FinalizeCallbackTy FiniCB, Value *IfCondition,
-    Value *NumThreads, omp::ProcBindKind ProcBind, bool IsCancellable) {
+    const LocationDescription &Loc, InsertPointTy OuterAllocaIP,
+    BodyGenCallbackTy BodyGenCB, PrivatizeCallbackTy PrivCB,
+    FinalizeCallbackTy FiniCB, Value *IfCondition, Value *NumThreads,
+    omp::ProcBindKind ProcBind, bool IsCancellable) {
   if (!updateToLocation(Loc))
     return Loc.IP;
 
@@ -429,7 +430,9 @@ IRBuilder<>::InsertPoint OpenMPIRBuilder::CreateParallel(
   // we want to delete at the end.
   SmallVector<Instruction *, 4> ToBeDeleted;
 
-  Builder.SetInsertPoint(OuterFn->getEntryBlock().getFirstNonPHI());
+  // Change the location to the outer alloca insertion point to create and
+  // initialize the allocas we pass into the parallel region.
+  Builder.restoreIP(OuterAllocaIP);
   AllocaInst *TIDAddr = Builder.CreateAlloca(Int32, nullptr, "tid.addr");
   AllocaInst *ZeroAddr = Builder.CreateAlloca(Int32, nullptr, "zero.addr");
 
@@ -481,9 +484,9 @@ IRBuilder<>::InsertPoint OpenMPIRBuilder::CreateParallel(
 
   // Generate the privatization allocas in the block that will become the entry
   // of the outlined function.
-  InsertPointTy AllocaIP(PRegEntryBB,
-                         PRegEntryBB->getTerminator()->getIterator());
-  Builder.restoreIP(AllocaIP);
+  Builder.SetInsertPoint(PRegEntryBB->getTerminator());
+  InsertPointTy InnerAllocaIP = Builder.saveIP();
+
   AllocaInst *PrivTIDAddr =
       Builder.CreateAlloca(Int32, nullptr, "tid.addr.local");
   Instruction *PrivTID = Builder.CreateLoad(PrivTIDAddr, "tid");
@@ -512,7 +515,7 @@ IRBuilder<>::InsertPoint OpenMPIRBuilder::CreateParallel(
   // Let the caller create the body.
   assert(BodyGenCB && "Expected body generation callback!");
   InsertPointTy CodeGenIP(PRegBodyBB, PRegBodyBB->begin());
-  BodyGenCB(AllocaIP, CodeGenIP, *PRegPreFiniBB);
+  BodyGenCB(InnerAllocaIP, CodeGenIP, *PRegPreFiniBB);
 
   LLVM_DEBUG(dbgs() << "After  body codegen: " << *OuterFn << "\n");
 
@@ -671,7 +674,7 @@ IRBuilder<>::InsertPoint OpenMPIRBuilder::CreateParallel(
       ReplacementValue = PrivTID;
     } else {
       Builder.restoreIP(
-          PrivCB(AllocaIP, Builder.saveIP(), V, ReplacementValue));
+          PrivCB(InnerAllocaIP, Builder.saveIP(), V, ReplacementValue));
       assert(ReplacementValue &&
              "Expected copy/create callback to set replacement value!");
       if (ReplacementValue == &V)
@@ -686,6 +689,10 @@ IRBuilder<>::InsertPoint OpenMPIRBuilder::CreateParallel(
     LLVM_DEBUG(dbgs() << "Captured input: " << *Input << "\n");
     PrivHelper(*Input);
   }
+  LLVM_DEBUG({
+    for (Value *Output : Outputs)
+      LLVM_DEBUG(dbgs() << "Captured output: " << *Output << "\n");
+  });
   assert(Outputs.empty() &&
          "OpenMP outlining should not produce live-out values!");
 
diff --git a/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp b/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp
index 2ba9d85a0f9ee..edd2c8f5dd886 100644
--- a/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp
+++ b/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp
@@ -6,13 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "llvm/Frontend/OpenMP/OMPConstants.h"
 #include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/DIBuilder.h"
 #include "llvm/IR/Function.h"
+#include "llvm/IR/InstIterator.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Module.h"
-#include "llvm/Frontend/OpenMP/OMPConstants.h"
 #include "llvm/IR/Verifier.h"
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
 #include "gtest/gtest.h"
@@ -360,9 +361,11 @@ TEST_F(OpenMPIRBuilderTest, ParallelSimple) {
 
   auto FiniCB = [&](InsertPointTy CodeGenIP) { ++NumFinalizationPoints; };
 
+  IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
+                                    F->getEntryBlock().getFirstInsertionPt());
   IRBuilder<>::InsertPoint AfterIP =
-      OMPBuilder.CreateParallel(Loc, BodyGenCB, PrivCB, FiniCB, nullptr,
-                                nullptr, OMP_PROC_BIND_default, false);
+      OMPBuilder.CreateParallel(Loc, AllocaIP, BodyGenCB, PrivCB, FiniCB,
+                                nullptr, nullptr, OMP_PROC_BIND_default, false);
   EXPECT_EQ(NumBodiesGenerated, 1U);
   EXPECT_EQ(NumPrivatizedVars, 1U);
   EXPECT_EQ(NumFinalizationPoints, 1U);
@@ -400,6 +403,205 @@ TEST_F(OpenMPIRBuilderTest, ParallelSimple) {
   EXPECT_EQ(ForkCI->getArgOperand(3), F->arg_begin());
 }
 
+TEST_F(OpenMPIRBuilderTest, ParallelNested) {
+  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
+  OpenMPIRBuilder OMPBuilder(*M);
+  OMPBuilder.initialize();
+  F->setName("func");
+  IRBuilder<> Builder(BB);
+
+  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
+
+  unsigned NumInnerBodiesGenerated = 0;
+  unsigned NumOuterBodiesGenerated = 0;
+  unsigned NumFinalizationPoints = 0;
+
+  auto InnerBodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
+                            BasicBlock &ContinuationIP) {
+    ++NumInnerBodiesGenerated;
+  };
+
+  auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
+                    Value &VPtr, Value *&ReplacementValue) -> InsertPointTy {
+    // Trivial copy (=firstprivate).
+    Builder.restoreIP(AllocaIP);
+    Type *VTy = VPtr.getType()->getPointerElementType();
+    Value *V = Builder.CreateLoad(VTy, &VPtr, VPtr.getName() + ".reload");
+    ReplacementValue = Builder.CreateAlloca(VTy, 0, VPtr.getName() + ".copy");
+    Builder.restoreIP(CodeGenIP);
+    Builder.CreateStore(V, ReplacementValue);
+    return CodeGenIP;
+  };
+
+  auto FiniCB = [&](InsertPointTy CodeGenIP) { ++NumFinalizationPoints; };
+
+  auto OuterBodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
+                            BasicBlock &ContinuationIP) {
+    ++NumOuterBodiesGenerated;
+    Builder.restoreIP(CodeGenIP);
+    BasicBlock *CGBB = CodeGenIP.getBlock();
+    BasicBlock *NewBB = SplitBlock(CGBB, &*CodeGenIP.getPoint());
+    CGBB->getTerminator()->eraseFromParent();
+    ;
+
+    IRBuilder<>::InsertPoint AfterIP = OMPBuilder.CreateParallel(
+        InsertPointTy(CGBB, CGBB->end()), AllocaIP, InnerBodyGenCB, PrivCB,
+        FiniCB, nullptr, nullptr, OMP_PROC_BIND_default, false);
+
+    Builder.restoreIP(AfterIP);
+    Builder.CreateBr(NewBB);
+  };
+
+  IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
+                                    F->getEntryBlock().getFirstInsertionPt());
+  IRBuilder<>::InsertPoint AfterIP =
+      OMPBuilder.CreateParallel(Loc, AllocaIP, OuterBodyGenCB, PrivCB, FiniCB,
+                                nullptr, nullptr, OMP_PROC_BIND_default, false);
+
+  EXPECT_EQ(NumInnerBodiesGenerated, 1U);
+  EXPECT_EQ(NumOuterBodiesGenerated, 1U);
+  EXPECT_EQ(NumFinalizationPoints, 2U);
+
+  Builder.restoreIP(AfterIP);
+  Builder.CreateRetVoid();
+
+  OMPBuilder.finalize();
+
+  EXPECT_EQ(M->size(), 5U);
+  for (Function &OutlinedFn : *M) {
+    if (F == &OutlinedFn || OutlinedFn.isDeclaration())
+      continue;
+    EXPECT_FALSE(verifyModule(*M, &errs()));
+    EXPECT_TRUE(OutlinedFn.hasFnAttribute(Attribute::NoUnwind));
+    EXPECT_TRUE(OutlinedFn.hasFnAttribute(Attribute::NoRecurse));
+    EXPECT_TRUE(OutlinedFn.hasParamAttribute(0, Attribute::NoAlias));
+    EXPECT_TRUE(OutlinedFn.hasParamAttribute(1, Attribute::NoAlias));
+
+    EXPECT_TRUE(OutlinedFn.hasInternalLinkage());
+    EXPECT_EQ(OutlinedFn.arg_size(), 2U);
+
+    EXPECT_EQ(OutlinedFn.getNumUses(), 1U);
+    User *Usr = OutlinedFn.user_back();
+    ASSERT_TRUE(isa<ConstantExpr>(Usr));
+    CallInst *ForkCI = dyn_cast<CallInst>(Usr->user_back());
+    ASSERT_NE(ForkCI, nullptr);
+
+    EXPECT_EQ(ForkCI->getCalledFunction()->getName(), "__kmpc_fork_call");
+    EXPECT_EQ(ForkCI->getNumArgOperands(), 3U);
+    EXPECT_TRUE(isa<GlobalVariable>(ForkCI->getArgOperand(0)));
+    EXPECT_EQ(ForkCI->getArgOperand(1),
+              ConstantInt::get(Type::getInt32Ty(Ctx), 0U));
+    EXPECT_EQ(ForkCI->getArgOperand(2), Usr);
+  }
+}
+
+TEST_F(OpenMPIRBuilderTest, ParallelNested2Inner) {
+  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
+  OpenMPIRBuilder OMPBuilder(*M);
+  OMPBuilder.initialize();
+  F->setName("func");
+  IRBuilder<> Builder(BB);
+
+  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
+
+  unsigned NumInnerBodiesGenerated = 0;
+  unsigned NumOuterBodiesGenerated = 0;
+  unsigned NumFinalizationPoints = 0;
+
+  auto InnerBodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
+                            BasicBlock &ContinuationIP) {
+    ++NumInnerBodiesGenerated;
+  };
+
+  auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
+                    Value &VPtr, Value *&ReplacementValue) -> InsertPointTy {
+    // Trivial copy (=firstprivate).
+    Builder.restoreIP(AllocaIP);
+    Type *VTy = VPtr.getType()->getPointerElementType();
+    Value *V = Builder.CreateLoad(VTy, &VPtr, VPtr.getName() + ".reload");
+    ReplacementValue = Builder.CreateAlloca(VTy, 0, VPtr.getName() + ".copy");
+    Builder.restoreIP(CodeGenIP);
+    Builder.CreateStore(V, ReplacementValue);
+    return CodeGenIP;
+  };
+
+  auto FiniCB = [&](InsertPointTy CodeGenIP) { ++NumFinalizationPoints; };
+
+  auto OuterBodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
+                            BasicBlock &ContinuationIP) {
+    ++NumOuterBodiesGenerated;
+    Builder.restoreIP(CodeGenIP);
+    BasicBlock *CGBB = CodeGenIP.getBlock();
+    BasicBlock *NewBB1 = SplitBlock(CGBB, &*CodeGenIP.getPoint());
+    BasicBlock *NewBB2 = SplitBlock(NewBB1, &*NewBB1->getFirstInsertionPt());
+    CGBB->getTerminator()->eraseFromParent();
+    ;
+    NewBB1->getTerminator()->eraseFromParent();
+    ;
+
+    IRBuilder<>::InsertPoint AfterIP1 = OMPBuilder.CreateParallel(
+        InsertPointTy(CGBB, CGBB->end()), AllocaIP, InnerBodyGenCB, PrivCB,
+        FiniCB, nullptr, nullptr, OMP_PROC_BIND_default, false);
+
+    Builder.restoreIP(AfterIP1);
+    Builder.CreateBr(NewBB1);
+
+    IRBuilder<>::InsertPoint AfterIP2 = OMPBuilder.CreateParallel(
+        InsertPointTy(NewBB1, NewBB1->end()), AllocaIP, InnerBodyGenCB, PrivCB,
+        FiniCB, nullptr, nullptr, OMP_PROC_BIND_default, false);
+
+    Builder.restoreIP(AfterIP2);
+    Builder.CreateBr(NewBB2);
+  };
+
+  IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
+                                    F->getEntryBlock().getFirstInsertionPt());
+  IRBuilder<>::InsertPoint AfterIP =
+      OMPBuilder.CreateParallel(Loc, AllocaIP, OuterBodyGenCB, PrivCB, FiniCB,
+                                nullptr, nullptr, OMP_PROC_BIND_default, false);
+
+  EXPECT_EQ(NumInnerBodiesGenerated, 2U);
+  EXPECT_EQ(NumOuterBodiesGenerated, 1U);
+  EXPECT_EQ(NumFinalizationPoints, 3U);
+
+  Builder.restoreIP(AfterIP);
+  Builder.CreateRetVoid();
+
+  OMPBuilder.finalize();
+
+  EXPECT_EQ(M->size(), 6U);
+  for (Function &OutlinedFn : *M) {
+    if (F == &OutlinedFn || OutlinedFn.isDeclaration())
+      continue;
+    EXPECT_FALSE(verifyModule(*M, &errs()));
+    EXPECT_TRUE(OutlinedFn.hasFnAttribute(Attribute::NoUnwind));
+    EXPECT_TRUE(OutlinedFn.hasFnAttribute(Attribute::NoRecurse));
+    EXPECT_TRUE(OutlinedFn.hasParamAttribute(0, Attribute::NoAlias));
+    EXPECT_TRUE(OutlinedFn.hasParamAttribute(1, Attribute::NoAlias));
+
+    EXPECT_TRUE(OutlinedFn.hasInternalLinkage());
+    EXPECT_EQ(OutlinedFn.arg_size(), 2U);
+
+    unsigned NumAllocas = 0;
+    for (Instruction &I : instructions(OutlinedFn))
+      NumAllocas += isa<AllocaInst>(I);
+    EXPECT_EQ(NumAllocas, 1U);
+
+    EXPECT_EQ(OutlinedFn.getNumUses(), 1U);
+    User *Usr = OutlinedFn.user_back();
+    ASSERT_TRUE(isa<ConstantExpr>(Usr));
+    CallInst *ForkCI = dyn_cast<CallInst>(Usr->user_back());
+    ASSERT_NE(ForkCI, nullptr);
+
+    EXPECT_EQ(ForkCI->getCalledFunction()->getName(), "__kmpc_fork_call");
+    EXPECT_EQ(ForkCI->getNumArgOperands(), 3U);
+    EXPECT_TRUE(isa<GlobalVariable>(ForkCI->getArgOperand(0)));
+    EXPECT_EQ(ForkCI->getArgOperand(1),
+              ConstantInt::get(Type::getInt32Ty(Ctx), 0U));
+    EXPECT_EQ(ForkCI->getArgOperand(2), Usr);
+  }
+}
+
 TEST_F(OpenMPIRBuilderTest, ParallelIfCond) {
   using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
   OpenMPIRBuilder OMPBuilder(*M);
@@ -460,9 +662,12 @@ TEST_F(OpenMPIRBuilderTest, ParallelIfCond) {
     // No destructors.
   };
 
-  IRBuilder<>::InsertPoint AfterIP = OMPBuilder.CreateParallel(
-      Loc, BodyGenCB, PrivCB, FiniCB, Builder.CreateIsNotNull(F->arg_begin()),
-      nullptr, OMP_PROC_BIND_default, false);
+  IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
+                                    F->getEntryBlock().getFirstInsertionPt());
+  IRBuilder<>::InsertPoint AfterIP =
+      OMPBuilder.CreateParallel(Loc, AllocaIP, BodyGenCB, PrivCB, FiniCB,
+                                Builder.CreateIsNotNull(F->arg_begin()),
+                                nullptr, OMP_PROC_BIND_default, false);
 
   EXPECT_EQ(NumBodiesGenerated, 1U);
   EXPECT_EQ(NumPrivatizedVars, 1U);
@@ -585,9 +790,12 @@ TEST_F(OpenMPIRBuilderTest, ParallelCancelBarrier) {
                        {Builder.getInt32(NumFinalizationPoints)});
   };
 
-  IRBuilder<>::InsertPoint AfterIP = OMPBuilder.CreateParallel(
-      Loc, BodyGenCB, PrivCB, FiniCB, Builder.CreateIsNotNull(F->arg_begin()),
-      nullptr, OMP_PROC_BIND_default, true);
+  IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
+                                    F->getEntryBlock().getFirstInsertionPt());
+  IRBuilder<>::InsertPoint AfterIP =
+      OMPBuilder.CreateParallel(Loc, AllocaIP, BodyGenCB, PrivCB, FiniCB,
+                                Builder.CreateIsNotNull(F->arg_begin()),
+                                nullptr, OMP_PROC_BIND_default, true);
 
   EXPECT_EQ(NumBodiesGenerated, 1U);
   EXPECT_EQ(NumPrivatizedVars, 0U);

From ebad64dfe133e64d1df6b82e6ef2fb031d635b08 Mon Sep 17 00:00:00 2001
From: Johannes Doerfert <johannes@jdoerfert.de>
Date: Mon, 29 Jun 2020 13:14:46 -0500
Subject: [PATCH 0629/1035] [OpenMP][FIX] Consistently use OpenMPIRBuilder if
 requested

When we use the OpenMPIRBuilder for the parallel region we need to also
use it to get the thread ID (among other things) in the body. This is
because CGOpenMPRuntime::getThreadID() and
CGOpenMPRuntime::emitUpdateLocation implicitly assumes that if they are
called from within a parallel region there is a certain structure to the
code and certain members of the OMPRegionInfo are initialized. It might
make sense to initialize them even if we use the OpenMPIRBuilder but we
would preferably get rid of such state instead.

Bug reported by Anchu Rajendran Sudhakumari.

Depends on D82470.

Reviewed By: anchu-rajendran

Differential Revision: https://reviews.llvm.org/D82822
---
 clang/lib/CodeGen/CGOpenMPRuntime.cpp         |  45 ++-
 clang/test/OpenMP/cancel_codegen.cpp          |   9 +-
 .../OpenMP/irbuilder_nested_parallel_for.c    | 299 ++++++++++++++++++
 clang/test/OpenMP/task_codegen.cpp            |  49 ++-
 4 files changed, 363 insertions(+), 39 deletions(-)
 create mode 100644 clang/test/OpenMP/irbuilder_nested_parallel_for.c

diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
index dc12286c72be8..60c7081b135bf 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -1455,6 +1455,19 @@ void CGOpenMPRuntime::clearLocThreadIdInsertPt(CodeGenFunction &CGF) {
   }
 }
 
+static StringRef getIdentStringFromSourceLocation(CodeGenFunction &CGF,
+                                                  SourceLocation Loc,
+                                                  SmallString<128> &Buffer) {
+  llvm::raw_svector_ostream OS(Buffer);
+  // Build debug location
+  PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc);
+  OS << ";" << PLoc.getFilename() << ";";
+  if (const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.CurFuncDecl))
+    OS << FD->getQualifiedNameAsString();
+  OS << ";" << PLoc.getLine() << ";" << PLoc.getColumn() << ";;";
+  return OS.str();
+}
+
 llvm::Value *CGOpenMPRuntime::emitUpdateLocation(CodeGenFunction &CGF,
                                                  SourceLocation Loc,
                                                  unsigned Flags) {
@@ -1464,6 +1477,16 @@ llvm::Value *CGOpenMPRuntime::emitUpdateLocation(CodeGenFunction &CGF,
       Loc.isInvalid())
     return getOrCreateDefaultLocation(Flags).getPointer();
 
+  // If the OpenMPIRBuilder is used we need to use it for all location handling
+  // as the clang invariants used below might be broken.
+  if (CGM.getLangOpts().OpenMPIRBuilder) {
+    SmallString<128> Buffer;
+    OMPBuilder.updateToLocation(CGF.Builder.saveIP());
+    auto *SrcLocStr = OMPBuilder.getOrCreateSrcLocStr(
+        getIdentStringFromSourceLocation(CGF, Loc, Buffer));
+    return OMPBuilder.getOrCreateIdent(SrcLocStr, IdentFlag(Flags));
+  }
+
   assert(CGF.CurFn && "No function in current CodeGenFunction.");
 
   CharUnits Align = CGM.getContext().getTypeAlignInChars(IdentQTy);
@@ -1497,15 +1520,9 @@ llvm::Value *CGOpenMPRuntime::emitUpdateLocation(CodeGenFunction &CGF,
 
   llvm::Value *OMPDebugLoc = OpenMPDebugLocMap.lookup(Loc.getRawEncoding());
   if (OMPDebugLoc == nullptr) {
-    SmallString<128> Buffer2;
-    llvm::raw_svector_ostream OS2(Buffer2);
-    // Build debug location
-    PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc);
-    OS2 << ";" << PLoc.getFilename() << ";";
-    if (const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.CurFuncDecl))
-      OS2 << FD->getQualifiedNameAsString();
-    OS2 << ";" << PLoc.getLine() << ";" << PLoc.getColumn() << ";;";
-    OMPDebugLoc = CGF.Builder.CreateGlobalStringPtr(OS2.str());
+    SmallString<128> Buffer;
+    OMPDebugLoc = CGF.Builder.CreateGlobalStringPtr(
+        getIdentStringFromSourceLocation(CGF, Loc, Buffer));
     OpenMPDebugLocMap[Loc.getRawEncoding()] = OMPDebugLoc;
   }
   // *psource = ";<File>;<Function>;<Line>;<Column>;;";
@@ -1519,6 +1536,16 @@ llvm::Value *CGOpenMPRuntime::emitUpdateLocation(CodeGenFunction &CGF,
 llvm::Value *CGOpenMPRuntime::getThreadID(CodeGenFunction &CGF,
                                           SourceLocation Loc) {
   assert(CGF.CurFn && "No function in current CodeGenFunction.");
+  // If the OpenMPIRBuilder is used we need to use it for all thread id calls as
+  // the clang invariants used below might be broken.
+  if (CGM.getLangOpts().OpenMPIRBuilder) {
+    SmallString<128> Buffer;
+    OMPBuilder.updateToLocation(CGF.Builder.saveIP());
+    auto *SrcLocStr = OMPBuilder.getOrCreateSrcLocStr(
+        getIdentStringFromSourceLocation(CGF, Loc, Buffer));
+    return OMPBuilder.getOrCreateThreadID(
+        OMPBuilder.getOrCreateIdent(SrcLocStr));
+  }
 
   llvm::Value *ThreadID = nullptr;
   // Check whether we've already cached a load of the thread id in this
diff --git a/clang/test/OpenMP/cancel_codegen.cpp b/clang/test/OpenMP/cancel_codegen.cpp
index a21a9db1e39a8..0942c7cf42369 100644
--- a/clang/test/OpenMP/cancel_codegen.cpp
+++ b/clang/test/OpenMP/cancel_codegen.cpp
@@ -16,7 +16,6 @@
 
 float flag;
 int main (int argc, char **argv) {
-// ALL: [[GTID:%.+]] = call i32 @__kmpc_global_thread_num(
 #pragma omp parallel
 {
 #pragma omp cancel parallel if(flag)
@@ -42,14 +41,14 @@ int main (int argc, char **argv) {
   }
 }
 // ALL: call void @__kmpc_for_static_init_4(
-// ALL: [[RES:%.+]] = call i32 @__kmpc_cancel(%struct.ident_t* {{[^,]+}}, i32 [[GTID]], i32 3)
+// ALL: [[RES:%.+]] = call i32 @__kmpc_cancel(%struct.ident_t* {{[^,]+}}, i32 [[GTID:%.*]], i32 3)
 // ALL: [[CMP:%.+]] = icmp ne i32 [[RES]], 0
 // ALL: br i1 [[CMP]], label %[[EXIT:[^,].+]], label %[[CONTINUE:.+]]
 // ALL: [[EXIT]]
 // ALL: br label
 // ALL: [[CONTINUE]]
 // ALL: br label
-// ALL: [[RES:%.+]] = call i32 @__kmpc_cancel(%struct.ident_t* {{[^,]+}}, i32 [[GTID]], i32 3)
+// ALL: [[RES:%.+]] = call i32 @__kmpc_cancel(%struct.ident_t* {{[^,]+}}, i32 [[GTID:%.*]], i32 3)
 // ALL: [[CMP:%.+]] = icmp ne i32 [[RES]], 0
 // ALL: br i1 [[CMP]], label %[[EXIT:[^,].+]], label %[[CONTINUE:.+]]
 // ALL: [[EXIT]]
@@ -66,7 +65,7 @@ for (int i = 0; i < argc; ++i) {
 // ALL: [[BOOL:%.+]] = fcmp une float [[FLAG]], 0.000000e+00
 // ALL: br i1 [[BOOL]], label %[[THEN:[^,]+]], label %[[ELSE:[^,]+]]
 // ALL: [[THEN]]
-// ALL: [[RES:%.+]] = call i32 @__kmpc_cancel(%struct.ident_t* {{[^,]+}}, i32 [[GTID]], i32 2)
+// ALL: [[RES:%.+]] = call i32 @__kmpc_cancel(%struct.ident_t* {{[^,]+}}, i32 [[GTID:%.*]], i32 2)
 // ALL: [[CMP:%.+]] = icmp ne i32 [[RES]], 0
 // ALL: br i1 [[CMP]], label %[[EXIT:[^,].+]], label %[[CONTINUE:.+]]
 // ALL: [[EXIT]]
@@ -148,7 +147,7 @@ for (int i = 0; i < argc; ++i) {
 // CHECK: br label
 // CHECK: [[CONTINUE]]
 // CHECK: br label
-// CHECK: [[RES:%.+]] = call i32 @__kmpc_cancel(%struct.ident_t* {{[^,]+}}, i32 [[GTID]], i32 3)
+// CHECK: [[RES:%.+]] = call i32 @__kmpc_cancel(%struct.ident_t* {{[^,]+}}, i32 [[GTID:%.*]], i32 3)
 // CHECK: [[CMP:%.+]] = icmp ne i32 [[RES]], 0
 // CHECK: br i1 [[CMP]], label %[[EXIT:[^,].+]], label %[[CONTINUE:.+]]
 // CHECK: [[EXIT]]
diff --git a/clang/test/OpenMP/irbuilder_nested_parallel_for.c b/clang/test/OpenMP/irbuilder_nested_parallel_for.c
new file mode 100644
index 0000000000000..929a92827689b
--- /dev/null
+++ b/clang/test/OpenMP/irbuilder_nested_parallel_for.c
@@ -0,0 +1,299 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-enable-irbuilder -x c++ -emit-llvm %s -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -o - | FileCheck --check-prefixes=CHECK %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-enable-irbuilder -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -debug-info-kind=limited -std=c++11 -verify %s -emit-llvm -o - | FileCheck --check-prefixes=CHECK-DEBUG %s
+
+// expected-no-diagnostics
+
+// TODO: Teach the update script to check new functions too.
+
+#ifndef HEADER
+#define HEADER
+
+// CHECK-LABEL: @_Z14parallel_for_0v(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[OMP_GLOBAL_THREAD_NUM:%.*]] = call i32 @__kmpc_global_thread_num(%struct.ident_t* @1)
+// CHECK-NEXT:    br label [[OMP_PARALLEL:%.*]]
+// CHECK:       omp_parallel:
+// CHECK-NEXT:    call void (%struct.ident_t*, i32, void (i32*, i32*, ...)*, ...) @__kmpc_fork_call(%struct.ident_t* @1, i32 0, void (i32*, i32*, ...)* bitcast (void (i32*, i32*)* @_Z14parallel_for_0v..omp_par to void (i32*, i32*, ...)*))
+// CHECK-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT:%.*]]
+// CHECK:       omp.par.outlined.exit:
+// CHECK-NEXT:    br label [[OMP_PAR_EXIT_SPLIT:%.*]]
+// CHECK:       omp.par.exit.split:
+// CHECK-NEXT:    ret void
+//
+// CHECK-DEBUG-LABEL: @_Z14parallel_for_0v(
+// CHECK-DEBUG-NEXT:  entry:
+// CHECK-DEBUG-NEXT:    [[OMP_GLOBAL_THREAD_NUM:%.*]] = call i32 @__kmpc_global_thread_num(%struct.ident_t* @1), !dbg !{{[0-9]*}}
+// CHECK-DEBUG-NEXT:    br label [[OMP_PARALLEL:%.*]]
+// CHECK-DEBUG:       omp_parallel:
+// CHECK-DEBUG-NEXT:    call void (%struct.ident_t*, i32, void (i32*, i32*, ...)*, ...) @__kmpc_fork_call(%struct.ident_t* @1, i32 0, void (i32*, i32*, ...)* bitcast (void (i32*, i32*)* @_Z14parallel_for_0v..omp_par to void (i32*, i32*, ...)*)), !dbg !{{[0-9]*}}
+// CHECK-DEBUG-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT:%.*]]
+// CHECK-DEBUG:       omp.par.outlined.exit:
+// CHECK-DEBUG-NEXT:    br label [[OMP_PAR_EXIT_SPLIT:%.*]]
+// CHECK-DEBUG:       omp.par.exit.split:
+// CHECK-DEBUG-NEXT:    ret void, !dbg !{{[0-9]*}}
+//
+void parallel_for_0(void) {
+#pragma omp parallel
+  {
+#pragma omp for
+    for (int i = 0; i < 100; ++i) {
+    }
+  }
+}
+
+// CHECK-LABEL: @_Z14parallel_for_1Pfid(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[R_ADDR:%.*]] = alloca float*, align 8
+// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[B_ADDR:%.*]] = alloca double, align 8
+// CHECK-NEXT:    store float* [[R:%.*]], float** [[R_ADDR]], align 8
+// CHECK-NEXT:    store i32 [[A:%.*]], i32* [[A_ADDR]], align 4
+// CHECK-NEXT:    store double [[B:%.*]], double* [[B_ADDR]], align 8
+// CHECK-NEXT:    [[OMP_GLOBAL_THREAD_NUM:%.*]] = call i32 @__kmpc_global_thread_num(%struct.ident_t* @1)
+// CHECK-NEXT:    br label [[OMP_PARALLEL:%.*]]
+// CHECK:       omp_parallel:
+// CHECK-NEXT:    call void (%struct.ident_t*, i32, void (i32*, i32*, ...)*, ...) @__kmpc_fork_call(%struct.ident_t* @1, i32 3, void (i32*, i32*, ...)* bitcast (void (i32*, i32*, i32*, double*, float**)* @_Z14parallel_for_1Pfid..omp_par.1 to void (i32*, i32*, ...)*), i32* [[A_ADDR]], double* [[B_ADDR]], float** [[R_ADDR]])
+// CHECK-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT19:%.*]]
+// CHECK:       omp.par.outlined.exit19:
+// CHECK-NEXT:    br label [[OMP_PAR_EXIT_SPLIT:%.*]]
+// CHECK:       omp.par.exit.split:
+// CHECK-NEXT:    ret void
+//
+// CHECK-DEBUG-LABEL: @_Z14parallel_for_1Pfid(
+// CHECK-DEBUG-NEXT:  entry:
+// CHECK-DEBUG-NEXT:    [[R_ADDR:%.*]] = alloca float*, align 8
+// CHECK-DEBUG-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
+// CHECK-DEBUG-NEXT:    [[B_ADDR:%.*]] = alloca double, align 8
+// CHECK-DEBUG-NEXT:    store float* [[R:%.*]], float** [[R_ADDR]], align 8
+// CHECK-DEBUG-NEXT:    call void @llvm.dbg.declare(metadata float** [[R_ADDR]], metadata !{{[0-9]*}}, metadata !DIExpression()), !dbg !{{[0-9]*}}
+// CHECK-DEBUG-NEXT:    store i32 [[A:%.*]], i32* [[A_ADDR]], align 4
+// CHECK-DEBUG-NEXT:    call void @llvm.dbg.declare(metadata i32* [[A_ADDR]], metadata !{{[0-9]*}}, metadata !DIExpression()), !dbg !{{[0-9]*}}
+// CHECK-DEBUG-NEXT:    store double [[B:%.*]], double* [[B_ADDR]], align 8
+// CHECK-DEBUG-NEXT:    call void @llvm.dbg.declare(metadata double* [[B_ADDR]], metadata !{{[0-9]*}}, metadata !DIExpression()), !dbg !{{[0-9]*}}
+// CHECK-DEBUG-NEXT:    [[OMP_GLOBAL_THREAD_NUM:%.*]] = call i32 @__kmpc_global_thread_num(%struct.ident_t* @12), !dbg !{{[0-9]*}}
+// CHECK-DEBUG-NEXT:    br label [[OMP_PARALLEL:%.*]]
+// CHECK-DEBUG:       omp_parallel:
+// CHECK-DEBUG-NEXT:    call void (%struct.ident_t*, i32, void (i32*, i32*, ...)*, ...) @__kmpc_fork_call(%struct.ident_t* @12, i32 3, void (i32*, i32*, ...)* bitcast (void (i32*, i32*, i32*, double*, float**)* @_Z14parallel_for_1Pfid..omp_par.1 to void (i32*, i32*, ...)*), i32* [[A_ADDR]], double* [[B_ADDR]], float** [[R_ADDR]]), !dbg !{{[0-9]*}}
+// CHECK-DEBUG-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT19:%.*]]
+// CHECK-DEBUG:       omp.par.outlined.exit19:
+// CHECK-DEBUG-NEXT:    br label [[OMP_PAR_EXIT_SPLIT:%.*]]
+// CHECK-DEBUG:       omp.par.exit.split:
+// CHECK-DEBUG-NEXT:    ret void, !dbg !{{[0-9]*}}
+//
+void parallel_for_1(float *r, int a, double b) {
+#pragma omp parallel
+  {
+#pragma omp parallel
+    {
+#pragma omp for
+      for (int i = 0; i < 100; ++i) {
+        *r = a + b;
+      }
+    }
+  }
+}
+
+// CHECK-LABEL: @_Z14parallel_for_2Pfid(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[R_ADDR:%.*]] = alloca float*, align 8
+// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[B_ADDR:%.*]] = alloca double, align 8
+// CHECK-NEXT:    [[DOTOMP_IV212:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[TMP213:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[DOTOMP_LB214:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[DOTOMP_UB215:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[DOTOMP_STRIDE216:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[DOTOMP_IS_LAST217:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[I218:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    store float* [[R:%.*]], float** [[R_ADDR]], align 8
+// CHECK-NEXT:    store i32 [[A:%.*]], i32* [[A_ADDR]], align 4
+// CHECK-NEXT:    store double [[B:%.*]], double* [[B_ADDR]], align 8
+// CHECK-NEXT:    [[OMP_GLOBAL_THREAD_NUM:%.*]] = call i32 @__kmpc_global_thread_num(%struct.ident_t* @1)
+// CHECK-NEXT:    br label [[OMP_PARALLEL:%.*]]
+// CHECK:       omp_parallel:
+// CHECK-NEXT:    call void (%struct.ident_t*, i32, void (i32*, i32*, ...)*, ...) @__kmpc_fork_call(%struct.ident_t* @1, i32 3, void (i32*, i32*, ...)* bitcast (void (i32*, i32*, i32*, double*, float**)* @_Z14parallel_for_2Pfid..omp_par.4 to void (i32*, i32*, ...)*), i32* [[A_ADDR]], double* [[B_ADDR]], float** [[R_ADDR]])
+// CHECK-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT211:%.*]]
+// CHECK:       omp.par.outlined.exit211:
+// CHECK-NEXT:    br label [[OMP_PAR_EXIT_SPLIT:%.*]]
+// CHECK:       omp.par.exit.split:
+// CHECK-NEXT:    store i32 0, i32* [[DOTOMP_LB214]], align 4
+// CHECK-NEXT:    store i32 99, i32* [[DOTOMP_UB215]], align 4
+// CHECK-NEXT:    store i32 1, i32* [[DOTOMP_STRIDE216]], align 4
+// CHECK-NEXT:    store i32 0, i32* [[DOTOMP_IS_LAST217]], align 4
+// CHECK-NEXT:    [[OMP_GLOBAL_THREAD_NUM219:%.*]] = call i32 @__kmpc_global_thread_num(%struct.ident_t* @41)
+// CHECK-NEXT:    call void @__kmpc_for_static_init_4(%struct.ident_t* @2, i32 [[OMP_GLOBAL_THREAD_NUM219]], i32 34, i32* [[DOTOMP_IS_LAST217]], i32* [[DOTOMP_LB214]], i32* [[DOTOMP_UB215]], i32* [[DOTOMP_STRIDE216]], i32 1, i32 1)
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[DOTOMP_UB215]], align 4
+// CHECK-NEXT:    [[CMP220:%.*]] = icmp sgt i32 [[TMP0]], 99
+// CHECK-NEXT:    br i1 [[CMP220]], label [[COND_TRUE221:%.*]], label [[COND_FALSE222:%.*]]
+// CHECK:       cond.true221:
+// CHECK-NEXT:    br label [[COND_END223:%.*]]
+// CHECK:       cond.false222:
+// CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[DOTOMP_UB215]], align 4
+// CHECK-NEXT:    br label [[COND_END223]]
+// CHECK:       cond.end223:
+// CHECK-NEXT:    [[COND224:%.*]] = phi i32 [ 99, [[COND_TRUE221]] ], [ [[TMP1]], [[COND_FALSE222]] ]
+// CHECK-NEXT:    store i32 [[COND224]], i32* [[DOTOMP_UB215]], align 4
+// CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* [[DOTOMP_LB214]], align 4
+// CHECK-NEXT:    store i32 [[TMP2]], i32* [[DOTOMP_IV212]], align 4
+// CHECK-NEXT:    br label [[OMP_INNER_FOR_COND225:%.*]]
+// CHECK:       omp.inner.for.cond225:
+// CHECK-NEXT:    [[TMP3:%.*]] = load i32, i32* [[DOTOMP_IV212]], align 4
+// CHECK-NEXT:    [[TMP4:%.*]] = load i32, i32* [[DOTOMP_UB215]], align 4
+// CHECK-NEXT:    [[CMP226:%.*]] = icmp sle i32 [[TMP3]], [[TMP4]]
+// CHECK-NEXT:    br i1 [[CMP226]], label [[OMP_INNER_FOR_BODY227:%.*]], label [[OMP_INNER_FOR_END236:%.*]]
+// CHECK:       omp.inner.for.body227:
+// CHECK-NEXT:    [[TMP5:%.*]] = load i32, i32* [[DOTOMP_IV212]], align 4
+// CHECK-NEXT:    [[MUL228:%.*]] = mul nsw i32 [[TMP5]], 1
+// CHECK-NEXT:    [[ADD229:%.*]] = add nsw i32 0, [[MUL228]]
+// CHECK-NEXT:    store i32 [[ADD229]], i32* [[I218]], align 4
+// CHECK-NEXT:    [[TMP6:%.*]] = load i32, i32* [[A_ADDR]], align 4
+// CHECK-NEXT:    [[CONV230:%.*]] = sitofp i32 [[TMP6]] to double
+// CHECK-NEXT:    [[TMP7:%.*]] = load double, double* [[B_ADDR]], align 8
+// CHECK-NEXT:    [[ADD231:%.*]] = fadd double [[CONV230]], [[TMP7]]
+// CHECK-NEXT:    [[CONV232:%.*]] = fptrunc double [[ADD231]] to float
+// CHECK-NEXT:    [[TMP8:%.*]] = load float*, float** [[R_ADDR]], align 8
+// CHECK-NEXT:    store float [[CONV232]], float* [[TMP8]], align 4
+// CHECK-NEXT:    br label [[OMP_BODY_CONTINUE233:%.*]]
+// CHECK:       omp.body.continue233:
+// CHECK-NEXT:    br label [[OMP_INNER_FOR_INC234:%.*]]
+// CHECK:       omp.inner.for.inc234:
+// CHECK-NEXT:    [[TMP9:%.*]] = load i32, i32* [[DOTOMP_IV212]], align 4
+// CHECK-NEXT:    [[ADD235:%.*]] = add nsw i32 [[TMP9]], 1
+// CHECK-NEXT:    store i32 [[ADD235]], i32* [[DOTOMP_IV212]], align 4
+// CHECK-NEXT:    br label [[OMP_INNER_FOR_COND225]]
+// CHECK:       omp.inner.for.end236:
+// CHECK-NEXT:    br label [[OMP_LOOP_EXIT237:%.*]]
+// CHECK:       omp.loop.exit237:
+// CHECK-NEXT:    [[OMP_GLOBAL_THREAD_NUM238:%.*]] = call i32 @__kmpc_global_thread_num(%struct.ident_t* @43)
+// CHECK-NEXT:    call void @__kmpc_for_static_fini(%struct.ident_t* @2, i32 [[OMP_GLOBAL_THREAD_NUM238]])
+// CHECK-NEXT:    [[OMP_GLOBAL_THREAD_NUM239:%.*]] = call i32 @__kmpc_global_thread_num(%struct.ident_t* @1)
+// CHECK-NEXT:    call void @__kmpc_barrier(%struct.ident_t* @7, i32 [[OMP_GLOBAL_THREAD_NUM239]])
+// CHECK-NEXT:    ret void
+//
+// CHECK-DEBUG-LABEL: @_Z14parallel_for_2Pfid(
+// CHECK-DEBUG-NEXT:  entry:
+// CHECK-DEBUG-NEXT:    [[R_ADDR:%.*]] = alloca float*, align 8
+// CHECK-DEBUG-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
+// CHECK-DEBUG-NEXT:    [[B_ADDR:%.*]] = alloca double, align 8
+// CHECK-DEBUG-NEXT:    [[DOTOMP_IV212:%.*]] = alloca i32, align 4
+// CHECK-DEBUG-NEXT:    [[TMP213:%.*]] = alloca i32, align 4
+// CHECK-DEBUG-NEXT:    [[DOTOMP_LB214:%.*]] = alloca i32, align 4
+// CHECK-DEBUG-NEXT:    [[DOTOMP_UB215:%.*]] = alloca i32, align 4
+// CHECK-DEBUG-NEXT:    [[DOTOMP_STRIDE216:%.*]] = alloca i32, align 4
+// CHECK-DEBUG-NEXT:    [[DOTOMP_IS_LAST217:%.*]] = alloca i32, align 4
+// CHECK-DEBUG-NEXT:    [[I218:%.*]] = alloca i32, align 4
+// CHECK-DEBUG-NEXT:    store float* [[R:%.*]], float** [[R_ADDR]], align 8
+// CHECK-DEBUG-NEXT:    call void @llvm.dbg.declare(metadata float** [[R_ADDR]], metadata !{{[0-9]*}}, metadata !DIExpression()), !dbg !{{[0-9]*}}
+// CHECK-DEBUG-NEXT:    store i32 [[A:%.*]], i32* [[A_ADDR]], align 4
+// CHECK-DEBUG-NEXT:    call void @llvm.dbg.declare(metadata i32* [[A_ADDR]], metadata !{{[0-9]*}}, metadata !DIExpression()), !dbg !{{[0-9]*}}
+// CHECK-DEBUG-NEXT:    store double [[B:%.*]], double* [[B_ADDR]], align 8
+// CHECK-DEBUG-NEXT:    call void @llvm.dbg.declare(metadata double* [[B_ADDR]], metadata !{{[0-9]*}}, metadata !DIExpression()), !dbg !{{[0-9]*}}
+// CHECK-DEBUG-NEXT:    [[OMP_GLOBAL_THREAD_NUM:%.*]] = call i32 @__kmpc_global_thread_num(%struct.ident_t* @25), !dbg !{{[0-9]*}}
+// CHECK-DEBUG-NEXT:    br label [[OMP_PARALLEL:%.*]]
+// CHECK-DEBUG:       omp_parallel:
+// CHECK-DEBUG-NEXT:    call void (%struct.ident_t*, i32, void (i32*, i32*, ...)*, ...) @__kmpc_fork_call(%struct.ident_t* @25, i32 3, void (i32*, i32*, ...)* bitcast (void (i32*, i32*, i32*, double*, float**)* @_Z14parallel_for_2Pfid..omp_par.4 to void (i32*, i32*, ...)*), i32* [[A_ADDR]], double* [[B_ADDR]], float** [[R_ADDR]]), !dbg !{{[0-9]*}}
+// CHECK-DEBUG-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT211:%.*]]
+// CHECK-DEBUG:       omp.par.outlined.exit211:
+// CHECK-DEBUG-NEXT:    br label [[OMP_PAR_EXIT_SPLIT:%.*]]
+// CHECK-DEBUG:       omp.par.exit.split:
+// CHECK-DEBUG-NEXT:    call void @llvm.dbg.declare(metadata i32* [[DOTOMP_IV212]], metadata !{{[0-9]*}}, metadata !DIExpression()), !dbg !{{[0-9]*}}
+// CHECK-DEBUG-NEXT:    call void @llvm.dbg.declare(metadata i32* [[DOTOMP_LB214]], metadata !{{[0-9]*}}, metadata !DIExpression()), !dbg !{{[0-9]*}}
+// CHECK-DEBUG-NEXT:    store i32 0, i32* [[DOTOMP_LB214]], align 4, !dbg !{{[0-9]*}}
+// CHECK-DEBUG-NEXT:    call void @llvm.dbg.declare(metadata i32* [[DOTOMP_UB215]], metadata !{{[0-9]*}}, metadata !DIExpression()), !dbg !{{[0-9]*}}
+// CHECK-DEBUG-NEXT:    store i32 99, i32* [[DOTOMP_UB215]], align 4, !dbg !{{[0-9]*}}
+// CHECK-DEBUG-NEXT:    call void @llvm.dbg.declare(metadata i32* [[DOTOMP_STRIDE216]], metadata !{{[0-9]*}}, metadata !DIExpression()), !dbg !{{[0-9]*}}
+// CHECK-DEBUG-NEXT:    store i32 1, i32* [[DOTOMP_STRIDE216]], align 4, !dbg !{{[0-9]*}}
+// CHECK-DEBUG-NEXT:    call void @llvm.dbg.declare(metadata i32* [[DOTOMP_IS_LAST217]], metadata !{{[0-9]*}}, metadata !DIExpression()), !dbg !{{[0-9]*}}
+// CHECK-DEBUG-NEXT:    store i32 0, i32* [[DOTOMP_IS_LAST217]], align 4, !dbg !{{[0-9]*}}
+// CHECK-DEBUG-NEXT:    call void @llvm.dbg.declare(metadata i32* [[I218]], metadata !{{[0-9]*}}, metadata !DIExpression()), !dbg !{{[0-9]*}}
+// CHECK-DEBUG-NEXT:    [[OMP_GLOBAL_THREAD_NUM219:%.*]] = call i32 @__kmpc_global_thread_num(%struct.ident_t* @97)
+// CHECK-DEBUG-NEXT:    call void @__kmpc_for_static_init_4(%struct.ident_t* @96, i32 [[OMP_GLOBAL_THREAD_NUM219]], i32 34, i32* [[DOTOMP_IS_LAST217]], i32* [[DOTOMP_LB214]], i32* [[DOTOMP_UB215]], i32* [[DOTOMP_STRIDE216]], i32 1, i32 1), !dbg !{{[0-9]*}}
+// CHECK-DEBUG-NEXT:    [[TMP0:%.*]] = load i32, i32* [[DOTOMP_UB215]], align 4, !dbg !{{[0-9]*}}
+// CHECK-DEBUG-NEXT:    [[CMP220:%.*]] = icmp sgt i32 [[TMP0]], 99, !dbg !{{[0-9]*}}
+// CHECK-DEBUG-NEXT:    br i1 [[CMP220]], label [[COND_TRUE221:%.*]], label [[COND_FALSE222:%.*]], !dbg !{{[0-9]*}}
+// CHECK-DEBUG:       cond.true221:
+// CHECK-DEBUG-NEXT:    br label [[COND_END223:%.*]], !dbg !{{[0-9]*}}
+// CHECK-DEBUG:       cond.false222:
+// CHECK-DEBUG-NEXT:    [[TMP1:%.*]] = load i32, i32* [[DOTOMP_UB215]], align 4, !dbg !{{[0-9]*}}
+// CHECK-DEBUG-NEXT:    br label [[COND_END223]], !dbg !{{[0-9]*}}
+// CHECK-DEBUG:       cond.end223:
+// CHECK-DEBUG-NEXT:    [[COND224:%.*]] = phi i32 [ 99, [[COND_TRUE221]] ], [ [[TMP1]], [[COND_FALSE222]] ], !dbg !{{[0-9]*}}
+// CHECK-DEBUG-NEXT:    store i32 [[COND224]], i32* [[DOTOMP_UB215]], align 4, !dbg !{{[0-9]*}}
+// CHECK-DEBUG-NEXT:    [[TMP2:%.*]] = load i32, i32* [[DOTOMP_LB214]], align 4, !dbg !{{[0-9]*}}
+// CHECK-DEBUG-NEXT:    store i32 [[TMP2]], i32* [[DOTOMP_IV212]], align 4, !dbg !{{[0-9]*}}
+// CHECK-DEBUG-NEXT:    br label [[OMP_INNER_FOR_COND225:%.*]], !dbg !{{[0-9]*}}
+// CHECK-DEBUG:       omp.inner.for.cond225:
+// CHECK-DEBUG-NEXT:    [[TMP3:%.*]] = load i32, i32* [[DOTOMP_IV212]], align 4, !dbg !{{[0-9]*}}
+// CHECK-DEBUG-NEXT:    [[TMP4:%.*]] = load i32, i32* [[DOTOMP_UB215]], align 4, !dbg !{{[0-9]*}}
+// CHECK-DEBUG-NEXT:    [[CMP226:%.*]] = icmp sle i32 [[TMP3]], [[TMP4]], !dbg !{{[0-9]*}}
+// CHECK-DEBUG-NEXT:    br i1 [[CMP226]], label [[OMP_INNER_FOR_BODY227:%.*]], label [[OMP_INNER_FOR_END236:%.*]], !dbg !{{[0-9]*}}
+// CHECK-DEBUG:       omp.inner.for.body227:
+// CHECK-DEBUG-NEXT:    [[TMP5:%.*]] = load i32, i32* [[DOTOMP_IV212]], align 4, !dbg !{{[0-9]*}}
+// CHECK-DEBUG-NEXT:    [[MUL228:%.*]] = mul nsw i32 [[TMP5]], 1, !dbg !{{[0-9]*}}
+// CHECK-DEBUG-NEXT:    [[ADD229:%.*]] = add nsw i32 0, [[MUL228]], !dbg !{{[0-9]*}}
+// CHECK-DEBUG-NEXT:    store i32 [[ADD229]], i32* [[I218]], align 4, !dbg !{{[0-9]*}}
+// CHECK-DEBUG-NEXT:    [[TMP6:%.*]] = load i32, i32* [[A_ADDR]], align 4, !dbg !{{[0-9]*}}
+// CHECK-DEBUG-NEXT:    [[CONV230:%.*]] = sitofp i32 [[TMP6]] to double, !dbg !{{[0-9]*}}
+// CHECK-DEBUG-NEXT:    [[TMP7:%.*]] = load double, double* [[B_ADDR]], align 8, !dbg !{{[0-9]*}}
+// CHECK-DEBUG-NEXT:    [[ADD231:%.*]] = fadd double [[CONV230]], [[TMP7]], !dbg !{{[0-9]*}}
+// CHECK-DEBUG-NEXT:    [[CONV232:%.*]] = fptrunc double [[ADD231]] to float, !dbg !{{[0-9]*}}
+// CHECK-DEBUG-NEXT:    [[TMP8:%.*]] = load float*, float** [[R_ADDR]], align 8, !dbg !{{[0-9]*}}
+// CHECK-DEBUG-NEXT:    store float [[CONV232]], float* [[TMP8]], align 4, !dbg !{{[0-9]*}}
+// CHECK-DEBUG-NEXT:    br label [[OMP_BODY_CONTINUE233:%.*]], !dbg !{{[0-9]*}}
+// CHECK-DEBUG:       omp.body.continue233:
+// CHECK-DEBUG-NEXT:    br label [[OMP_INNER_FOR_INC234:%.*]], !dbg !{{[0-9]*}}
+// CHECK-DEBUG:       omp.inner.for.inc234:
+// CHECK-DEBUG-NEXT:    [[TMP9:%.*]] = load i32, i32* [[DOTOMP_IV212]], align 4, !dbg !{{[0-9]*}}
+// CHECK-DEBUG-NEXT:    [[ADD235:%.*]] = add nsw i32 [[TMP9]], 1, !dbg !{{[0-9]*}}
+// CHECK-DEBUG-NEXT:    store i32 [[ADD235]], i32* [[DOTOMP_IV212]], align 4, !dbg !{{[0-9]*}}
+// CHECK-DEBUG-NEXT:    br label [[OMP_INNER_FOR_COND225]], !dbg !{{[0-9]*}}, !llvm.loop !{{[0-9]*}}
+// CHECK-DEBUG:       omp.inner.for.end236:
+// CHECK-DEBUG-NEXT:    br label [[OMP_LOOP_EXIT237:%.*]], !dbg !{{[0-9]*}}
+// CHECK-DEBUG:       omp.loop.exit237:
+// CHECK-DEBUG-NEXT:    [[OMP_GLOBAL_THREAD_NUM238:%.*]] = call i32 @__kmpc_global_thread_num(%struct.ident_t* @100)
+// CHECK-DEBUG-NEXT:    call void @__kmpc_for_static_fini(%struct.ident_t* @99, i32 [[OMP_GLOBAL_THREAD_NUM238]]), !dbg !{{[0-9]*}}
+// CHECK-DEBUG-NEXT:    [[OMP_GLOBAL_THREAD_NUM239:%.*]] = call i32 @__kmpc_global_thread_num(%struct.ident_t* @103), !dbg !{{[0-9]*}}
+// CHECK-DEBUG-NEXT:    call void @__kmpc_barrier(%struct.ident_t* @102, i32 [[OMP_GLOBAL_THREAD_NUM239]]), !dbg !{{[0-9]*}}
+// CHECK-DEBUG-NEXT:    ret void, !dbg !{{[0-9]*}}
+//
+void parallel_for_2(float *r, int a, double b) {
+#pragma omp parallel
+  {
+#pragma omp for
+    for (int i = 0; i < 100; ++i)
+      *r = a + b;
+#pragma omp parallel
+    {
+#pragma omp for
+      for (int i = 0; i < 100; ++i)
+        *r = a + b;
+#pragma omp parallel
+      {
+#pragma omp for
+        for (int i = 0; i < 100; ++i)
+          *r = a + b;
+      }
+#pragma omp for
+      for (int i = 0; i < 100; ++i)
+        *r = a + b;
+#pragma omp parallel
+      {
+#pragma omp for
+        for (int i = 0; i < 100; ++i)
+          *r = a + b;
+      }
+#pragma omp for
+      for (int i = 0; i < 100; ++i)
+        *r = a + b;
+    }
+#pragma omp for
+    for (int i = 0; i < 100; ++i)
+      *r = a + b;
+  }
+#pragma omp for
+  for (int i = 0; i < 100; ++i)
+    *r = a + b;
+}
+
+#endif
diff --git a/clang/test/OpenMP/task_codegen.cpp b/clang/test/OpenMP/task_codegen.cpp
index 2a561ce7744e6..922623499d494 100644
--- a/clang/test/OpenMP/task_codegen.cpp
+++ b/clang/test/OpenMP/task_codegen.cpp
@@ -33,12 +33,11 @@ int main() {
   char b;
   S s[2];
   int arr[10][a];
-// CHECK: [[GTID:%.+]] = call i32 @__kmpc_global_thread_num([[IDENT_T]]* @{{.+}})
 // CHECK: [[B_REF:%.+]] = getelementptr inbounds [[STRUCT_SHAREDS]], [[STRUCT_SHAREDS]]* [[CAPTURES:%.+]], i32 0, i32 0
 // CHECK: store i8* [[B]], i8** [[B_REF]]
 // CHECK: [[S_REF:%.+]] = getelementptr inbounds [[STRUCT_SHAREDS]], [[STRUCT_SHAREDS]]* [[CAPTURES]], i32 0, i32 1
 // CHECK: store [2 x [[STRUCT_S]]]* [[S]], [2 x [[STRUCT_S]]]** [[S_REF]]
-// CHECK: [[ORIG_TASK_PTR:%.+]] = call i8* @__kmpc_omp_task_alloc([[IDENT_T]]* @{{.+}}, i32 [[GTID]], i32 33, i64 40, i64 16, i32 (i32, i8*)* bitcast (i32 (i32, [[KMP_TASK_T]]{{.*}}*)* [[TASK_ENTRY1:@.+]] to i32 (i32, i8*)*))
+// CHECK: [[ORIG_TASK_PTR:%.+]] = call i8* @__kmpc_omp_task_alloc([[IDENT_T]]* @{{.+}}, i32 {{%.*}}, i32 33, i64 40, i64 16, i32 (i32, i8*)* bitcast (i32 (i32, [[KMP_TASK_T]]{{.*}}*)* [[TASK_ENTRY1:@.+]] to i32 (i32, i8*)*))
 // CHECK: [[SHAREDS_REF_PTR:%.+]] = getelementptr inbounds [[KMP_TASK_T]], [[KMP_TASK_T]]* [[TASK_PTR:%.+]], i32 0, i32 0
 // CHECK: [[SHAREDS_REF:%.+]] = load i8*, i8** [[SHAREDS_REF_PTR]]
 // CHECK: [[BITCAST:%.+]] = bitcast [[STRUCT_SHAREDS]]* [[CAPTURES]] to i8*
@@ -46,7 +45,7 @@ int main() {
 // CHECK: [[PRIORITY_REF_PTR:%.+]] = getelementptr inbounds [[KMP_TASK_T]], [[KMP_TASK_T]]* [[TASK_PTR]], i32 0, i32 4
 // CHECK: [[PRIORITY:%.+]] = bitcast %union{{.+}}* [[PRIORITY_REF_PTR]] to i32*
 // CHECK: store i32 {{.+}}, i32* [[PRIORITY]]
-// CHECK: call i32 @__kmpc_omp_task([[IDENT_T]]* @{{.+}}, i32 [[GTID]], i8* [[ORIG_TASK_PTR]])
+// CHECK: call i32 @__kmpc_omp_task([[IDENT_T]]* @{{.+}}, i32 {{%.*}}, i8* [[ORIG_TASK_PTR]])
 #pragma omp task shared(a, b, s) priority(b)
   {
     a = 15;
@@ -55,7 +54,7 @@ int main() {
   }
 // CHECK: [[S_REF:%.+]] = getelementptr inbounds [[STRUCT_SHAREDS1]], [[STRUCT_SHAREDS1]]* [[CAPTURES:%.+]], i32 0, i32 0
 // CHECK: store [2 x [[STRUCT_S]]]* [[S]], [2 x [[STRUCT_S]]]** [[S_REF]]
-// CHECK: [[ORIG_TASK_PTR:%.+]] = call i8* @__kmpc_omp_task_alloc([[IDENT_T]]* @{{[^,]+}}, i32 [[GTID]], i32 1, i64 40, i64 8,
+// CHECK: [[ORIG_TASK_PTR:%.+]] = call i8* @__kmpc_omp_task_alloc([[IDENT_T]]* @{{[^,]+}}, i32 {{%.*}}, i32 1, i64 40, i64 8,
 // CHECK: [[SHAREDS_REF_PTR:%.+]] = getelementptr inbounds [[KMP_TASK_T]], [[KMP_TASK_T]]* [[TASK_PTR:%.+]], i32 0, i32 0
 // CHECK: [[SHAREDS_REF:%.+]] = load i8*, i8** [[SHAREDS_REF_PTR]]
 // CHECK: [[BITCAST:%.+]] = bitcast [[STRUCT_SHAREDS1]]* [[CAPTURES]] to i8*
@@ -101,20 +100,20 @@ int main() {
 // CHECK: [[T0:%.*]] = getelementptr inbounds [[KMP_DEPEND_INFO]], [[KMP_DEPEND_INFO]]* %{{[^,]+}}, i32 0, i32 2
 // CHECK: store i8 1, i8* [[T0]]
 // CHECK: bitcast [[KMP_DEPEND_INFO]]* [[DEP_BASE]] to i8*
-// CHECK: call i32 @__kmpc_omp_task_with_deps([[IDENT_T]]* @{{.+}}, i32 [[GTID]], i8* [[ORIG_TASK_PTR]], i32 4, i8* %{{[^,]+}}, i32 0, i8* null)
+// CHECK: call i32 @__kmpc_omp_task_with_deps([[IDENT_T]]* @{{.+}}, i32 {{%.*}}, i8* [[ORIG_TASK_PTR]], i32 4, i8* %{{[^,]+}}, i32 0, i8* null)
 #pragma omp task shared(a, s) depend(in : a, b, s, arr[:])
   {
     a = 15;
     s[1].a = 10;
   }
-// CHECK: [[ORIG_TASK_PTR:%.+]] = call i8* @__kmpc_omp_task_alloc([[IDENT_T]]* @{{.+}}, i32 [[GTID]], i32 0, i64 40, i64 1, i32 (i32, i8*)* bitcast (i32 (i32, [[KMP_TASK_T]]{{.*}}*)* [[TASK_ENTRY2:@.+]] to i32 (i32, i8*)*))
-// CHECK: call i32 @__kmpc_omp_task([[IDENT_T]]* @{{.+}}, i32 [[GTID]], i8* [[ORIG_TASK_PTR]])
+// CHECK: [[ORIG_TASK_PTR:%.+]] = call i8* @__kmpc_omp_task_alloc([[IDENT_T]]* @{{.+}}, i32 {{%.*}}, i32 0, i64 40, i64 1, i32 (i32, i8*)* bitcast (i32 (i32, [[KMP_TASK_T]]{{.*}}*)* [[TASK_ENTRY2:@.+]] to i32 (i32, i8*)*))
+// CHECK: call i32 @__kmpc_omp_task([[IDENT_T]]* @{{.+}}, i32 {{%.*}}, i8* [[ORIG_TASK_PTR]])
 #pragma omp task untied
   {
 #pragma omp critical
     a = 1;
   }
-// CHECK: [[ORIG_TASK_PTR:%.+]] = call i8* @__kmpc_omp_task_alloc([[IDENT_T]]* @{{.+}}, i32 [[GTID]], i32 0, i64 40, i64 1,
+// CHECK: [[ORIG_TASK_PTR:%.+]] = call i8* @__kmpc_omp_task_alloc([[IDENT_T]]* @{{.+}}, i32 {{%.*}}, i32 0, i64 40, i64 1,
 // CHECK: getelementptr inbounds [2 x [[STRUCT_S]]], [2 x [[STRUCT_S]]]* [[S]], i64 0, i64 0
 // CHECK: getelementptr inbounds [[KMP_DEPEND_INFO]], [[KMP_DEPEND_INFO]]* %{{[^,]+}}, i32 0, i32 0
 // CHECK: ptrtoint [[STRUCT_S]]* %{{.+}} to i64
@@ -146,12 +145,12 @@ int main() {
 // CHECK: getelementptr inbounds [[KMP_DEPEND_INFO]], [[KMP_DEPEND_INFO]]* %{{[^,]+}}, i32 0, i32 2
 // CHECK: store i8 3, i8*
 // CHECK: bitcast [[KMP_DEPEND_INFO]]* %{{.+}} to i8*
-// CHECK: call i32 @__kmpc_omp_task_with_deps([[IDENT_T]]* @{{.+}}, i32 [[GTID]], i8* [[ORIG_TASK_PTR]], i32 2, i8* %{{[^,]+}}, i32 0, i8* null)
+// CHECK: call i32 @__kmpc_omp_task_with_deps([[IDENT_T]]* @{{.+}}, i32 {{%.*}}, i8* [[ORIG_TASK_PTR]], i32 2, i8* %{{[^,]+}}, i32 0, i8* null)
 #pragma omp task untied depend(out : s[0], arr[4:][b])
   {
     a = 1;
   }
-// CHECK: [[ORIG_TASK_PTR:%.+]] = call i8* @__kmpc_omp_task_alloc([[IDENT_T]]* @{{.+}}, i32 [[GTID]], i32 0, i64 40, i64 1,
+// CHECK: [[ORIG_TASK_PTR:%.+]] = call i8* @__kmpc_omp_task_alloc([[IDENT_T]]* @{{.+}}, i32 {{%.*}}, i32 0, i64 40, i64 1,
 // CHECK: getelementptr inbounds [2 x [[STRUCT_S]]], [2 x [[STRUCT_S]]]* [[S]], i64 0, i64 0
 // CHECK: getelementptr inbounds [[KMP_DEPEND_INFO]], [[KMP_DEPEND_INFO]]* %{{[^,]+}}, i32 0, i32 0
 // CHECK: ptrtoint [[STRUCT_S]]* %{{.+}} to i64
@@ -183,12 +182,12 @@ int main() {
 // CHECK: getelementptr inbounds [[KMP_DEPEND_INFO]], [[KMP_DEPEND_INFO]]* %{{[^,]+}}, i32 0, i32 2
 // CHECK: store i8 4, i8*
 // CHECK: bitcast [[KMP_DEPEND_INFO]]* %{{.+}} to i8*
-// CHECK: call i32 @__kmpc_omp_task_with_deps([[IDENT_T]]* @{{.+}}, i32 [[GTID]], i8* [[ORIG_TASK_PTR]], i32 2, i8* %{{[^,]+}}, i32 0, i8* null)
+// CHECK: call i32 @__kmpc_omp_task_with_deps([[IDENT_T]]* @{{.+}}, i32 {{%.*}}, i8* [[ORIG_TASK_PTR]], i32 2, i8* %{{[^,]+}}, i32 0, i8* null)
 #pragma omp task untied depend(mutexinoutset: s[0], arr[4:][b])
   {
     a = 1;
   }
-// CHECK: [[ORIG_TASK_PTR:%.+]] = call i8* @__kmpc_omp_task_alloc([[IDENT_T]]* @{{.+}}, i32 [[GTID]], i32 3, i64 40, i64 1,
+// CHECK: [[ORIG_TASK_PTR:%.+]] = call i8* @__kmpc_omp_task_alloc([[IDENT_T]]* @{{.+}}, i32 {{%.*}}, i32 3, i64 40, i64 1,
 // CHECK: getelementptr inbounds [[KMP_DEPEND_INFO]], [[KMP_DEPEND_INFO]]* %{{[^,]+}}, i32 0, i32 0
 // CHECK: store i64 ptrtoint (i32* @{{.+}} to i64), i64*
 // CHECK: getelementptr inbounds [[KMP_DEPEND_INFO]], [[KMP_DEPEND_INFO]]* %{{[^,]+}}, i32 0, i32 1
@@ -229,38 +228,38 @@ int main() {
 // CHECK: getelementptr inbounds [[KMP_DEPEND_INFO]], [[KMP_DEPEND_INFO]]* %{{[^,]+}}, i32 0, i32 2
 // CHECK: store i8 3, i8*
 // CHECK: bitcast [[KMP_DEPEND_INFO]]* %{{.+}} to i8*
-// CHECK: call i32 @__kmpc_omp_task_with_deps([[IDENT_T]]* @{{.+}}, i32 [[GTID]], i8* [[ORIG_TASK_PTR]], i32 3, i8* %{{[^,]+}}, i32 0, i8* null)
+// CHECK: call i32 @__kmpc_omp_task_with_deps([[IDENT_T]]* @{{.+}}, i32 {{%.*}}, i8* [[ORIG_TASK_PTR]], i32 3, i8* %{{[^,]+}}, i32 0, i8* null)
 #pragma omp task final(true) depend(inout: a, s[1], arr[:a][3:])
   {
     a = 2;
   }
-// CHECK: [[ORIG_TASK_PTR:%.+]] = call i8* @__kmpc_omp_task_alloc([[IDENT_T]]* @{{.+}}, i32 [[GTID]], i32 3, i64 40, i64 1, i32 (i32, i8*)* bitcast (i32 (i32, [[KMP_TASK_T]]{{.*}}*)* [[TASK_ENTRY3:@.+]] to i32 (i32, i8*)*))
-// CHECK: call i32 @__kmpc_omp_task([[IDENT_T]]* @{{.+}}, i32 [[GTID]], i8* [[ORIG_TASK_PTR]])
+// CHECK: [[ORIG_TASK_PTR:%.+]] = call i8* @__kmpc_omp_task_alloc([[IDENT_T]]* @{{.+}}, i32 {{%.*}}, i32 3, i64 40, i64 1, i32 (i32, i8*)* bitcast (i32 (i32, [[KMP_TASK_T]]{{.*}}*)* [[TASK_ENTRY3:@.+]] to i32 (i32, i8*)*))
+// CHECK: call i32 @__kmpc_omp_task([[IDENT_T]]* @{{.+}}, i32 {{%.*}}, i8* [[ORIG_TASK_PTR]])
 #pragma omp task final(true)
   {
     a = 2;
   }
-// CHECK: [[ORIG_TASK_PTR:%.+]] = call i8* @__kmpc_omp_task_alloc([[IDENT_T]]* @{{.+}}, i32 [[GTID]], i32 1, i64 40, i64 1, i32 (i32, i8*)* bitcast (i32 (i32, [[KMP_TASK_T]]{{.*}}*)* [[TASK_ENTRY4:@.+]] to i32 (i32, i8*)*))
-// CHECK: call i32 @__kmpc_omp_task([[IDENT_T]]* @{{.+}}, i32 [[GTID]], i8* [[ORIG_TASK_PTR]])
+  // CHECK: [[ORIG_TASK_PTR:%.+]] = call i8* @__kmpc_omp_task_alloc([[IDENT_T]]* @{{.+}}, i32 {{%.*}}, i32 1, i64 40, i64 1, i32 (i32, i8*)* bitcast (i32 (i32, [[KMP_TASK_T]]{{.*}}*)* [[TASK_ENTRY4:@.+]] to i32 (i32, i8*)*))
+  // CHECK: call i32 @__kmpc_omp_task([[IDENT_T]]* @{{.+}}, i32 {{%.*}}, i8* [[ORIG_TASK_PTR]])
   const bool flag = false;
 #pragma omp task final(flag)
   {
     a = 3;
   }
-// CHECK: [[B_VAL:%.+]] = load i8, i8* [[B]]
-// CHECK: [[CMP:%.+]] = icmp ne i8 [[B_VAL]], 0
-// CHECK: [[FINAL:%.+]] = select i1 [[CMP]], i32 2, i32 0
-// CHECK: [[FLAGS:%.+]] = or i32 [[FINAL]], 1
-// CHECK: [[ORIG_TASK_PTR:%.+]] = call i8* @__kmpc_omp_task_alloc([[IDENT_T]]* @{{.+}}, i32 [[GTID]], i32 [[FLAGS]], i64 40, i64 8, i32 (i32, i8*)* bitcast (i32 (i32, [[KMP_TASK_T]]{{.*}}*)* [[TASK_ENTRY5:@.+]] to i32 (i32, i8*)*))
-// CHECK: call i32 @__kmpc_omp_task([[IDENT_T]]* @{{.+}}, i32 [[GTID]], i8* [[ORIG_TASK_PTR]])
+  // CHECK: [[B_VAL:%.+]] = load i8, i8* [[B]]
+  // CHECK: [[CMP:%.+]] = icmp ne i8 [[B_VAL]], 0
+  // CHECK: [[FINAL:%.+]] = select i1 [[CMP]], i32 2, i32 0
+  // CHECK: [[FLAGS:%.+]] = or i32 [[FINAL]], 1
+  // CHECK: [[ORIG_TASK_PTR:%.+]] = call i8* @__kmpc_omp_task_alloc([[IDENT_T]]* @{{.+}}, i32 {{%.*}}, i32 [[FLAGS]], i64 40, i64 8, i32 (i32, i8*)* bitcast (i32 (i32, [[KMP_TASK_T]]{{.*}}*)* [[TASK_ENTRY5:@.+]] to i32 (i32, i8*)*))
+  // CHECK: call i32 @__kmpc_omp_task([[IDENT_T]]* @{{.+}}, i32 {{%.*}}, i8* [[ORIG_TASK_PTR]])
   int c __attribute__((aligned(128)));
 #pragma omp task final(b) shared(c)
   {
     a = 4;
     c = 5;
   }
-// CHECK: [[ORIG_TASK_PTR:%.+]] = call i8* @__kmpc_omp_task_alloc([[IDENT_T]]* @{{.+}}, i32 [[GTID]], i32 0, i64 40, i64 1, i32 (i32, i8*)* bitcast (i32 (i32, [[KMP_TASK_T]]{{.*}}*)* [[TASK_ENTRY6:@.+]] to i32 (i32, i8*)*))
-// CHECK: call i32 @__kmpc_omp_task([[IDENT_T]]* @{{.+}}, i32 [[GTID]], i8* [[ORIG_TASK_PTR]])
+// CHECK: [[ORIG_TASK_PTR:%.+]] = call i8* @__kmpc_omp_task_alloc([[IDENT_T]]* @{{.+}}, i32 {{%.*}}, i32 0, i64 40, i64 1, i32 (i32, i8*)* bitcast (i32 (i32, [[KMP_TASK_T]]{{.*}}*)* [[TASK_ENTRY6:@.+]] to i32 (i32, i8*)*))
+// CHECK: call i32 @__kmpc_omp_task([[IDENT_T]]* @{{.+}}, i32 {{%.*}}, i8* [[ORIG_TASK_PTR]])
 #pragma omp task untied
   {
     S s1;

From d2c22487225b05159c1d7d0da7fac7cae3d2049b Mon Sep 17 00:00:00 2001
From: Fangrui Song <maskray@google.com>
Date: Thu, 30 Jul 2020 08:30:06 -0700
Subject: [PATCH 0630/1035] [X86] Parse and ignore .arch directives

We parse .arch so that some `.arch i386; .code32` code can assemble. It seems
that X86AsmParser does not do a good job tracking what features are needed to
assemble instructions. GNU as's x86 port supports a very wide range of .arch
operands. Ignore the operand for now.

Reviewed By: craig.topper

Differential Revision: https://reviews.llvm.org/D84900
---
 llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp |  9 +++++++++
 llvm/test/MC/X86/directive-arch.s              | 10 ++++++++++
 2 files changed, 19 insertions(+)
 create mode 100644 llvm/test/MC/X86/directive-arch.s

diff --git a/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp b/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
index a278c46aaf97c..f52735068c8bb 100644
--- a/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
+++ b/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
@@ -937,6 +937,7 @@ class X86AsmParser : public MCTargetAsmParser {
                           SMLoc End, unsigned Size, StringRef Identifier,
                           const InlineAsmIdentifierInfo &Info);
 
+  bool parseDirectiveArch();
   bool parseDirectiveEven(SMLoc L);
   bool ParseDirectiveCode(StringRef IDVal, SMLoc L);
 
@@ -3993,6 +3994,8 @@ bool X86AsmParser::OmitRegisterFromClobberLists(unsigned RegNo) {
 bool X86AsmParser::ParseDirective(AsmToken DirectiveID) {
   MCAsmParser &Parser = getParser();
   StringRef IDVal = DirectiveID.getIdentifier();
+  if (IDVal.startswith(".arch"))
+    return parseDirectiveArch();
   if (IDVal.startswith(".code"))
     return ParseDirectiveCode(IDVal, DirectiveID.getLoc());
   else if (IDVal.startswith(".att_syntax")) {
@@ -4047,6 +4050,12 @@ bool X86AsmParser::ParseDirective(AsmToken DirectiveID) {
   return true;
 }
 
+bool X86AsmParser::parseDirectiveArch() {
+  // Ignore .arch for now.
+  getParser().parseStringToEndOfStatement();
+  return false;
+}
+
 /// parseDirectiveEven
 ///  ::= .even
 bool X86AsmParser::parseDirectiveEven(SMLoc L) {
diff --git a/llvm/test/MC/X86/directive-arch.s b/llvm/test/MC/X86/directive-arch.s
new file mode 100644
index 0000000000000..5f148a2ffb17b
--- /dev/null
+++ b/llvm/test/MC/X86/directive-arch.s
@@ -0,0 +1,10 @@
+## We currently parse but ignore .arch directives.
+# RUN: llvm-mc -triple=x86_64 %s | FileCheck /dev/null --implicit-check-not=.arch
+
+.arch i286
+.arch generic32
+
+.arch .avx512vl
+.arch .noavx512bw
+.arch .nop
+.arch .sse4.2

From 7b114446c320de542c50c4c02f566e5d18adee33 Mon Sep 17 00:00:00 2001
From: Brendon Cahoon <bcahoon@quicinc.com>
Date: Thu, 30 Jul 2020 09:50:59 -0500
Subject: [PATCH 0631/1035] Align store conditional address

In cases where the alignment of the datatype is smaller than
expected by the instruction, the address is aligned. The aligned
address is used for the load, but wasn't used for the store
conditional, which resulted in a run-time alignment exception.
---
 llvm/lib/CodeGen/AtomicExpandPass.cpp          |  3 ++-
 llvm/test/CodeGen/Hexagon/atomic-store-byte.ll | 18 ++++++++++++++++++
 2 files changed, 20 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/CodeGen/Hexagon/atomic-store-byte.ll

diff --git a/llvm/lib/CodeGen/AtomicExpandPass.cpp b/llvm/lib/CodeGen/AtomicExpandPass.cpp
index a5030305435c1..c61531c5141a9 100644
--- a/llvm/lib/CodeGen/AtomicExpandPass.cpp
+++ b/llvm/lib/CodeGen/AtomicExpandPass.cpp
@@ -1239,7 +1239,8 @@ bool AtomicExpand::expandAtomicCmpXchg(AtomicCmpXchgInst *CI) {
   Value *NewValueInsert =
       insertMaskedValue(Builder, LoadedTryStore, CI->getNewValOperand(), PMV);
   Value *StoreSuccess =
-      TLI->emitStoreConditional(Builder, NewValueInsert, Addr, MemOpOrder);
+      TLI->emitStoreConditional(Builder, NewValueInsert, PMV.AlignedAddr,
+                                MemOpOrder);
   StoreSuccess = Builder.CreateICmpEQ(
       StoreSuccess, ConstantInt::get(Type::getInt32Ty(Ctx), 0), "success");
   BasicBlock *RetryBB = HasReleasedLoadBB ? ReleasedLoadBB : StartBB;
diff --git a/llvm/test/CodeGen/Hexagon/atomic-store-byte.ll b/llvm/test/CodeGen/Hexagon/atomic-store-byte.ll
new file mode 100644
index 0000000000000..e3febe0264ad4
--- /dev/null
+++ b/llvm/test/CodeGen/Hexagon/atomic-store-byte.ll
@@ -0,0 +1,18 @@
+; RUN: llc -mtriple=hexagon < %s | FileCheck %s
+
+; Test that the address for a store conditional for a byte is aligned
+; correctly to use the memw_locked instruction.
+
+; CHECK: [[REG:(r[0-9]+)]] = and(r{{[0-9]+}},#-4)
+; CHECK: = memw_locked([[REG]])
+; CHECK: memw_locked([[REG]],p{{[0-4]}}) =
+
+@foo.a00 = internal global i8 0, align 1
+
+; Function Attrs: nofree norecurse nounwind
+define dso_local void @foo() local_unnamed_addr #0 {
+entry:
+  %0 = cmpxchg volatile i8* @foo.a00, i8 0, i8 1 seq_cst seq_cst
+  ret void
+}
+

From b6635b5b15cb1c160776493a982302a854df332e Mon Sep 17 00:00:00 2001
From: Jonas Devlieghere <jonas@devlieghere.com>
Date: Thu, 30 Jul 2020 08:46:02 -0700
Subject: [PATCH 0632/1035] [lldb] Add SBCommandInterpreterRunOptions to LLDB.h

---
 lldb/include/lldb/API/LLDB.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lldb/include/lldb/API/LLDB.h b/lldb/include/lldb/API/LLDB.h
index 83c38d3b61664..f7390cfabf01c 100644
--- a/lldb/include/lldb/API/LLDB.h
+++ b/lldb/include/lldb/API/LLDB.h
@@ -17,6 +17,7 @@
 #include "lldb/API/SBBreakpointName.h"
 #include "lldb/API/SBBroadcaster.h"
 #include "lldb/API/SBCommandInterpreter.h"
+#include "lldb/API/SBCommandInterpreterRunOptions.h"
 #include "lldb/API/SBCommandReturnObject.h"
 #include "lldb/API/SBCommunication.h"
 #include "lldb/API/SBCompileUnit.h"

From 6a043ecc0cf4d257d06c4fe0c3d5e1d9a8c7ea94 Mon Sep 17 00:00:00 2001
From: Artem Dergachev <artem.dergachev@gmail.com>
Date: Wed, 29 Jul 2020 18:05:42 -0700
Subject: [PATCH 0633/1035] [clang-tidy] Fix ODR violation in unittests.

Both tests define clang::tidy::test::TestCheck::registerMatchers().
This is UB and causes linker to sometimes choose the wrong overload.

Put classes into anonymous namespaces to avoid the problem.

Differential Revision: https://reviews.llvm.org/D84902
---
 .../clang-tidy/ClangTidyDiagnosticConsumerTest.cpp  | 13 ++-----------
 .../unittests/clang-tidy/ClangTidyOptionsTest.cpp   |  6 ++----
 2 files changed, 4 insertions(+), 15 deletions(-)

diff --git a/clang-tools-extra/unittests/clang-tidy/ClangTidyDiagnosticConsumerTest.cpp b/clang-tools-extra/unittests/clang-tidy/ClangTidyDiagnosticConsumerTest.cpp
index a8729660bdcec..0894e5fd5eb91 100644
--- a/clang-tools-extra/unittests/clang-tidy/ClangTidyDiagnosticConsumerTest.cpp
+++ b/clang-tools-extra/unittests/clang-tidy/ClangTidyDiagnosticConsumerTest.cpp
@@ -6,6 +6,7 @@ namespace clang {
 namespace tidy {
 namespace test {
 
+namespace {
 class TestCheck : public ClangTidyCheck {
 public:
   TestCheck(StringRef Name, ClangTidyContext *Context)
@@ -20,17 +21,8 @@ class TestCheck : public ClangTidyCheck {
     diag(Var->getTypeSpecStartLoc(), "type specifier");
   }
 };
+} // namespace
 
-// FIXME: This test seems to cause a strange linking interference
-// with the ValidConfiguration.ValidEnumOptions test on macOS.
-// If both tests are enabled, this test will fail as if
-// runCheckOnCode() is not invoked at all. Looks like a linker bug.
-// For now both tests are disabled on macOS. It is not sufficient
-// to only disable the other test because this test keeps failing
-// under Address Sanitizer, which may be an indication of more
-// such linking interference with other tests and this test
-// seems to be in the center of it.
-#ifndef __APPLE__
 TEST(ClangTidyDiagnosticConsumer, SortsErrors) {
   std::vector<ClangTidyError> Errors;
   runCheckOnCode<TestCheck>("int a;", &Errors);
@@ -38,7 +30,6 @@ TEST(ClangTidyDiagnosticConsumer, SortsErrors) {
   EXPECT_EQ("type specifier", Errors[0].Message.Message);
   EXPECT_EQ("variable", Errors[1].Message.Message);
 }
-#endif
 
 } // namespace test
 } // namespace tidy
diff --git a/clang-tools-extra/unittests/clang-tidy/ClangTidyOptionsTest.cpp b/clang-tools-extra/unittests/clang-tidy/ClangTidyOptionsTest.cpp
index c4239af0e7673..bfa594098fb71 100644
--- a/clang-tools-extra/unittests/clang-tidy/ClangTidyOptionsTest.cpp
+++ b/clang-tools-extra/unittests/clang-tidy/ClangTidyOptionsTest.cpp
@@ -118,6 +118,7 @@ TEST(ParseConfiguration, MergeConfigurations) {
   EXPECT_TRUE(*Options.UseColor);
 }
 
+namespace {
 class TestCheck : public ClangTidyCheck {
 public:
   TestCheck(ClangTidyContext *Context) : ClangTidyCheck("test", Context) {}
@@ -140,6 +141,7 @@ class TestCheck : public ClangTidyCheck {
     return Options.getLocalOrGlobal<IntType>(std::forward<Args>(Arguments)...);
   }
 };
+} // namespace
 
 #define CHECK_VAL(Value, Expected)                                             \
   do {                                                                         \
@@ -222,9 +224,6 @@ TEST(CheckOptionsValidation, ValidIntOptions) {
 #undef CHECK_ERROR_INT
 }
 
-// FIXME: Figure out why this test causes crashes on mac os.
-// See also comments around the ClangTidyDiagnosticConsumer.SortsErrors test.
-#ifndef __APPLE__
 TEST(ValidConfiguration, ValidEnumOptions) {
 
   ClangTidyOptions Options;
@@ -276,7 +275,6 @@ TEST(ValidConfiguration, ValidEnumOptions) {
 
 #undef CHECK_ERROR_ENUM
 }
-#endif
 
 #undef CHECK_VAL
 #undef CHECK_ERROR

From 0a00a7d57791e2bf2a5c86c09dbfb52aac57862d Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne@apple.com>
Date: Fri, 11 Oct 2019 14:42:26 -0400
Subject: [PATCH 0634/1035] [libc++] Use generator expression in Linker script
 generation

This is an alternative to the workaround in 34a3b24a90c6.

Differential Revision: https://reviews.llvm.org/D68880
---
 libcxx/cmake/Modules/DefineLinkerScript.cmake | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/libcxx/cmake/Modules/DefineLinkerScript.cmake b/libcxx/cmake/Modules/DefineLinkerScript.cmake
index 11a6ca57dfc0d..41426bf067149 100644
--- a/libcxx/cmake/Modules/DefineLinkerScript.cmake
+++ b/libcxx/cmake/Modules/DefineLinkerScript.cmake
@@ -34,13 +34,8 @@ function(define_linker_script target)
       if ("${lib}" STREQUAL "cxx-headers")
         continue()
       endif()
-      if (TARGET "${lib}" OR
-          (${lib} MATCHES "cxxabi(_static|_shared)?" AND HAVE_LIBCXXABI) OR
-          (${lib} MATCHES "unwind(_static|_shared)?" AND HAVE_LIBUNWIND))
-        list(APPEND link_libraries "${CMAKE_LINK_LIBRARY_FLAG}$<TARGET_PROPERTY:${lib},OUTPUT_NAME>")
-      else()
-        list(APPEND link_libraries "${CMAKE_LINK_LIBRARY_FLAG}${lib}")
-      endif()
+      set(libname "$<IF:$<TARGET_EXISTS:${lib}>,$<TARGET_PROPERTY:${lib},OUTPUT_NAME>,${lib}>")
+      list(APPEND link_libraries "${CMAKE_LINK_LIBRARY_FLAG}${libname}")
     endforeach()
   endif()
   string(REPLACE ";" " " link_libraries "${link_libraries}")

From 1260a155c37f567111902e532f32d7facdb0f087 Mon Sep 17 00:00:00 2001
From: Kuba Mracek <mracek@apple.com>
Date: Thu, 30 Jul 2020 09:00:14 -0700
Subject: [PATCH 0635/1035] [tsan] Fix the open and open64 interceptors to have
 correct declarations (variadic functions)

Not matching the (real) variadic declaration makes the interceptor take garbage inputs on Darwin/AArch64.

Differential Revision: https://reviews.llvm.org/D84570
---
 .../lib/tsan/rtl/tsan_interceptors_posix.cpp  | 23 +++++++++++++-----
 compiler-rt/test/tsan/variadic-open.cpp       | 24 +++++++++++++++++++
 2 files changed, 41 insertions(+), 6 deletions(-)
 create mode 100644 compiler-rt/test/tsan/variadic-open.cpp

diff --git a/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp b/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp
index 9c3e0369bc6cd..d485af8b60520 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp
@@ -31,6 +31,8 @@
 #include "tsan_mman.h"
 #include "tsan_fd.h"
 
+#include <stdarg.h>
+
 using namespace __tsan;
 
 #if SANITIZER_FREEBSD || SANITIZER_MAC
@@ -135,6 +137,7 @@ const int PTHREAD_BARRIER_SERIAL_THREAD = -1;
 #endif
 const int MAP_FIXED = 0x10;
 typedef long long_t;
+typedef __sanitizer::u16 mode_t;
 
 // From /usr/include/unistd.h
 # define F_ULOCK 0      /* Unlock a previously locked region.  */
@@ -1508,20 +1511,28 @@ TSAN_INTERCEPTOR(int, fstat64, int fd, void *buf) {
 #define TSAN_MAYBE_INTERCEPT_FSTAT64
 #endif
 
-TSAN_INTERCEPTOR(int, open, const char *name, int flags, int mode) {
-  SCOPED_TSAN_INTERCEPTOR(open, name, flags, mode);
+TSAN_INTERCEPTOR(int, open, const char *name, int oflag, ...) {
+  va_list ap;
+  va_start(ap, oflag);
+  mode_t mode = va_arg(ap, int);
+  va_end(ap);
+  SCOPED_TSAN_INTERCEPTOR(open, name, oflag, mode);
   READ_STRING(thr, pc, name, 0);
-  int fd = REAL(open)(name, flags, mode);
+  int fd = REAL(open)(name, oflag, mode);
   if (fd >= 0)
     FdFileCreate(thr, pc, fd);
   return fd;
 }
 
 #if SANITIZER_LINUX
-TSAN_INTERCEPTOR(int, open64, const char *name, int flags, int mode) {
-  SCOPED_TSAN_INTERCEPTOR(open64, name, flags, mode);
+TSAN_INTERCEPTOR(int, open64, const char *name, int oflag, ...) {
+  va_list ap;
+  va_start(ap, oflag);
+  mode_t mode = va_arg(ap, int);
+  va_end(ap);
+  SCOPED_TSAN_INTERCEPTOR(open64, name, oflag, mode);
   READ_STRING(thr, pc, name, 0);
-  int fd = REAL(open64)(name, flags, mode);
+  int fd = REAL(open64)(name, oflag, mode);
   if (fd >= 0)
     FdFileCreate(thr, pc, fd);
   return fd;
diff --git a/compiler-rt/test/tsan/variadic-open.cpp b/compiler-rt/test/tsan/variadic-open.cpp
new file mode 100644
index 0000000000000..cd7684005ceca
--- /dev/null
+++ b/compiler-rt/test/tsan/variadic-open.cpp
@@ -0,0 +1,24 @@
+// RUN: %clangxx_tsan -O1 %s -o %t && %run %t %t.tmp 2>&1 | FileCheck %s
+#include <stdio.h>
+#include <assert.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <sys/stat.h>
+
+int main(int argc, char *argv[]) {
+  fprintf(stderr, "Hello world.\n");
+  assert(argv[1]);
+  unlink(argv[1]);
+  int fd = open(argv[1], O_RDWR | O_CREAT, 0600);
+  assert(fd != -1);
+  struct stat info;
+  int result = fstat(fd, &info);
+  fprintf(stderr, "permissions = 0%o\n", info.st_mode & ~S_IFMT);
+  assert(result == 0);
+  close(fd);
+  fprintf(stderr, "Done.\n");
+}
+
+// CHECK: Hello world.
+// CHECK: permissions = 0600
+// CHECK: Done.

From 33fd4a18e7d373344c8af0012dd97c1c739f2916 Mon Sep 17 00:00:00 2001
From: hsmahesha <mahesha.comp@gmail.com>
Date: Thu, 30 Jul 2020 21:39:34 +0530
Subject: [PATCH 0636/1035] [AMDGPU/MemOpsCluster] Clean-up fixme's around mem
 ops clustering logic

Get rid of all fixmes and base heuristic on `num-clustered-dwords`. The main intuition behind this is as
follows. The existing heuristic roughly summarizes as below:

* Assume, all the mem ops instructions participating in the clustering process,  loads/stores same num bytes
* If num bytes loaded by each mem op is 4 bytes, then cluster at max 5 mem ops, that is at max 20 bytes
* If num bytes loaded by each mem op is 8 bytes, then cluster at max 3 mem ops, that is at max 24 bytes
* If num bytes loaded by each mem op is 16 bytes, then cluster at max 2 mem ops, that is at max 32 bytes

So, we need to make sure that the new heuristic do not completey deviate away from the above one, and it
properly handles both the sub-word loads and the wide loads.

Reviewed By: arsenm, rampitec

Differential Revision: https://reviews.llvm.org/D84354
---
 llvm/lib/Target/AMDGPU/SIInstrInfo.cpp        |  70 +--
 .../AMDGPU/GlobalISel/llvm.amdgcn.ubfe.ll     |  32 +-
 .../AMDGPU/GlobalISel/load-constant.96.ll     | 547 +++++++++---------
 .../CodeGen/AMDGPU/amdhsa-trap-num-sgprs.ll   |   2 +-
 .../CodeGen/AMDGPU/call-argument-types.ll     |  12 +-
 llvm/test/CodeGen/AMDGPU/cttz_zero_undef.ll   |   6 +-
 llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll     | 215 +++----
 llvm/test/CodeGen/AMDGPU/fshr.ll              | 314 +++++-----
 .../CodeGen/AMDGPU/indirect-addressing-si.ll  |   4 +-
 llvm/test/CodeGen/AMDGPU/kernel-args.ll       |   4 +-
 llvm/test/CodeGen/AMDGPU/llvm.round.f64.ll    | 448 +++++++-------
 llvm/test/CodeGen/AMDGPU/merge-stores.ll      |   1 -
 .../AMDGPU/promote-constOffset-to-imm.ll      |  10 +-
 llvm/test/CodeGen/AMDGPU/store-weird-sizes.ll | 254 ++++----
 .../AMDGPU/token-factor-inline-limit-test.ll  |  12 +-
 llvm/test/CodeGen/AMDGPU/udivrem.ll           |   7 +-
 16 files changed, 949 insertions(+), 989 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 59915cb48324d..d3dbf47370678 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -474,65 +474,29 @@ bool SIInstrInfo::shouldClusterMemOps(ArrayRef<const MachineOperand *> BaseOps1,
                                       ArrayRef<const MachineOperand *> BaseOps2,
                                       unsigned NumLoads,
                                       unsigned NumBytes) const {
+  // If the mem ops (to be clustered) do not have the same base ptr, then they
+  // should not be clustered
   assert(!BaseOps1.empty() && !BaseOps2.empty());
   const MachineInstr &FirstLdSt = *BaseOps1.front()->getParent();
   const MachineInstr &SecondLdSt = *BaseOps2.front()->getParent();
-
   if (!memOpsHaveSameBasePtr(FirstLdSt, BaseOps1, SecondLdSt, BaseOps2))
     return false;
 
-  const MachineOperand *FirstDst = nullptr;
-  const MachineOperand *SecondDst = nullptr;
-
-  if ((isMUBUF(FirstLdSt) && isMUBUF(SecondLdSt)) ||
-      (isMTBUF(FirstLdSt) && isMTBUF(SecondLdSt)) ||
-      (isMIMG(FirstLdSt) && isMIMG(SecondLdSt)) ||
-      (isFLAT(FirstLdSt) && isFLAT(SecondLdSt))) {
-    const unsigned MaxGlobalLoadCluster = 7;
-    if (NumLoads > MaxGlobalLoadCluster)
-      return false;
-
-    FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::vdata);
-    if (!FirstDst)
-      FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::vdst);
-    SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::vdata);
-    if (!SecondDst)
-      SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::vdst);
-  } else if (isSMRD(FirstLdSt) && isSMRD(SecondLdSt)) {
-    FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::sdst);
-    SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::sdst);
-  } else if (isDS(FirstLdSt) && isDS(SecondLdSt)) {
-    FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::vdst);
-    SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::vdst);
-  }
-
-  if (!FirstDst || !SecondDst)
-    return false;
-
-  // Try to limit clustering based on the total number of bytes loaded
-  // rather than the number of instructions.  This is done to help reduce
-  // register pressure.  The method used is somewhat inexact, though,
-  // because it assumes that all loads in the cluster will load the
-  // same number of bytes as FirstLdSt.
-
-  // The unit of this value is bytes.
-  // FIXME: This needs finer tuning.
-  unsigned LoadClusterThreshold = 16;
-
-  const MachineRegisterInfo &MRI =
-      FirstLdSt.getParent()->getParent()->getRegInfo();
-
-  const Register Reg = FirstDst->getReg();
-
-  const TargetRegisterClass *DstRC = Register::isVirtualRegister(Reg)
-                                         ? MRI.getRegClass(Reg)
-                                         : RI.getPhysRegClass(Reg);
-
-  // FIXME: NumLoads should not be subtracted 1. This is to match behavior
-  // of clusterNeighboringMemOps which was previosly passing cluster length
-  // less 1. LoadClusterThreshold should be tuned instead.
-  return ((NumLoads - 1) * (RI.getRegSizeInBits(*DstRC) / 8)) <=
-         LoadClusterThreshold;
+  // In order to avoid regester pressure, on an average, the number of DWORDS
+  // loaded together by all clustered mem ops should not exceed 8. This is an
+  // empirical value based on certain observations and performance related
+  // experiments.
+  // The good thing about this heuristic is - it avoids clustering of too many
+  // sub-word loads, and also avoids clustering of wide loads. Below is the
+  // brief summary of how the heuristic behaves for various `LoadSize`.
+  // (1) 1 <= LoadSize <= 4: cluster at max 8 mem ops
+  // (2) 5 <= LoadSize <= 8: cluster at max 4 mem ops
+  // (3) 9 <= LoadSize <= 12: cluster at max 2 mem ops
+  // (4) 13 <= LoadSize <= 16: cluster at max 2 mem ops
+  // (5) LoadSize >= 17: do not cluster
+  const unsigned LoadSize = NumBytes / NumLoads;
+  const unsigned NumDWORDs = ((LoadSize + 3) / 4) * NumLoads;
+  return NumDWORDs <= 8;
 }
 
 // FIXME: This behaves strangely. If, for example, you have 32 load + stores,
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ubfe.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ubfe.ll
index 6fc1cd5753083..ab3fbc03e81d5 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ubfe.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ubfe.ll
@@ -950,22 +950,22 @@ define amdgpu_kernel void @bfe_u32_constant_fold_test_18(i32 addrspace(1)* %out)
 define amdgpu_kernel void @simplify_bfe_u32_multi_use_arg(i32 addrspace(1)* %out0,
 ; GFX6-LABEL: simplify_bfe_u32_multi_use_arg:
 ; GFX6:       ; %bb.0:
-; GFX6-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0xd
-; GFX6-NEXT:    s_mov_b32 s6, -1
-; GFX6-NEXT:    s_mov_b32 s7, 0xf000
-; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX6-NEXT:    s_load_dword s8, s[2:3], 0x0
-; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
-; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
-; GFX6-NEXT:    s_mov_b64 s[2:3], s[6:7]
-; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX6-NEXT:    s_and_b32 s8, s8, 63
-; GFX6-NEXT:    s_bfe_u32 s9, s8, 0x20002
-; GFX6-NEXT:    v_mov_b32_e32 v1, s9
-; GFX6-NEXT:    v_mov_b32_e32 v0, s8
-; GFX6-NEXT:    buffer_store_dword v1, off, s[4:7], 0
-; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
-; GFX6-NEXT:    s_endpgm
+; GFX6-NEXT:        s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:        s_load_dwordx2 s[8:9], s[0:1], 0xb
+; GFX6-NEXT:        s_load_dwordx2 s[0:1], s[0:1], 0xd
+; GFX6-NEXT:        s_mov_b32 s6, -1
+; GFX6-NEXT:        s_mov_b32 s7, 0xf000
+; GFX6-NEXT:        s_mov_b64 s[10:11], s[6:7]
+; GFX6-NEXT:        s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:        s_load_dword s0, s[0:1], 0x0
+; GFX6-NEXT:        s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:        s_and_b32 s0, s0, 63
+; GFX6-NEXT:        s_bfe_u32 s1, s0, 0x20002
+; GFX6-NEXT:        v_mov_b32_e32 v1, s1
+; GFX6-NEXT:        v_mov_b32_e32 v0, s0
+; GFX6-NEXT:        buffer_store_dword v1, off, s[4:7], 0
+; GFX6-NEXT:        buffer_store_dword v0, off, s[8:11], 0
+; GFX6-NEXT:        s_endpgm
                                             i32 addrspace(1)* %out1,
                                             i32 addrspace(1)* %in) #0 {
   %src = load i32, i32 addrspace(1)* %in, align 4
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/load-constant.96.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/load-constant.96.ll
index af8b4f0f9e5ae..6dceaf2e22fa8 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/load-constant.96.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/load-constant.96.ll
@@ -17,57 +17,56 @@ define <3 x i32> @v_load_constant_v3i32_align1(<3 x i32> addrspace(4)* %ptr) {
 ;
 ; GFX9-NOUNALIGNED-LABEL: v_load_constant_v3i32_align1:
 ; GFX9-NOUNALIGNED:       ; %bb.0:
-; GFX9-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NOUNALIGNED-NEXT:    v_add_co_u32_e32 v2, vcc, 11, v0
-; GFX9-NOUNALIGNED-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v1, vcc
-; GFX9-NOUNALIGNED-NEXT:    global_load_ubyte v4, v[2:3], off offset:-6
-; GFX9-NOUNALIGNED-NEXT:    global_load_ubyte v5, v[2:3], off offset:-5
-; GFX9-NOUNALIGNED-NEXT:    global_load_ubyte v6, v[2:3], off offset:-4
-; GFX9-NOUNALIGNED-NEXT:    global_load_ubyte v7, v[2:3], off offset:-3
-; GFX9-NOUNALIGNED-NEXT:    global_load_ubyte v8, v[2:3], off offset:-2
-; GFX9-NOUNALIGNED-NEXT:    global_load_ubyte v9, v[2:3], off offset:-1
-; GFX9-NOUNALIGNED-NEXT:    global_load_ubyte v10, v[2:3], off
-; GFX9-NOUNALIGNED-NEXT:    global_load_ubyte v0, v[0:1], off
-; GFX9-NOUNALIGNED-NEXT:    global_load_ubyte v1, v[2:3], off offset:-10
-; GFX9-NOUNALIGNED-NEXT:    global_load_ubyte v11, v[2:3], off offset:-9
-; GFX9-NOUNALIGNED-NEXT:    global_load_ubyte v12, v[2:3], off offset:-8
-; GFX9-NOUNALIGNED-NEXT:    global_load_ubyte v2, v[2:3], off offset:-7
-; GFX9-NOUNALIGNED-NEXT:    v_mov_b32_e32 v3, 0xff
-; GFX9-NOUNALIGNED-NEXT:    s_movk_i32 s4, 0xff
-; GFX9-NOUNALIGNED-NEXT:    v_mov_b32_e32 v13, 8
-; GFX9-NOUNALIGNED-NEXT:    s_mov_b32 s5, 8
-; GFX9-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(11)
-; GFX9-NOUNALIGNED-NEXT:    v_lshlrev_b32_sdwa v4, v13, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(10)
-; GFX9-NOUNALIGNED-NEXT:    v_and_b32_e32 v5, v5, v3
-; GFX9-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(9)
-; GFX9-NOUNALIGNED-NEXT:    v_and_b32_e32 v6, v6, v3
-; GFX9-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(7)
-; GFX9-NOUNALIGNED-NEXT:    v_lshlrev_b32_sdwa v8, v13, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(6)
-; GFX9-NOUNALIGNED-NEXT:    v_and_b32_e32 v9, v9, v3
-; GFX9-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(5)
-; GFX9-NOUNALIGNED-NEXT:    v_and_b32_e32 v10, v10, v3
-; GFX9-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(3)
-; GFX9-NOUNALIGNED-NEXT:    v_lshlrev_b32_sdwa v1, s5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NOUNALIGNED-NEXT:    v_and_b32_e32 v11, s4, v11
-; GFX9-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(1)
-; GFX9-NOUNALIGNED-NEXT:    v_and_b32_e32 v12, s4, v12
-; GFX9-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NOUNALIGNED-NEXT:    v_and_or_b32 v2, v2, v3, v4
-; GFX9-NOUNALIGNED-NEXT:    v_and_or_b32 v0, v0, s4, v1
-; GFX9-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v1, 16, v11
-; GFX9-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v11, 24, v12
-; GFX9-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v4, 16, v5
-; GFX9-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v5, 24, v6
-; GFX9-NOUNALIGNED-NEXT:    v_and_or_b32 v3, v7, v3, v8
-; GFX9-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v6, 16, v9
-; GFX9-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v7, 24, v10
-; GFX9-NOUNALIGNED-NEXT:    v_or3_b32 v0, v0, v1, v11
-; GFX9-NOUNALIGNED-NEXT:    v_or3_b32 v1, v2, v4, v5
-; GFX9-NOUNALIGNED-NEXT:    v_or3_b32 v2, v3, v6, v7
-; GFX9-NOUNALIGNED-NEXT:    s_setpc_b64 s[30:31]
+; GFX9-NOUNALIGNED-NEXT:	s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NOUNALIGNED-NEXT:	v_add_co_u32_e32 v2, vcc, 11, v0
+; GFX9-NOUNALIGNED-NEXT:	v_addc_co_u32_e32 v3, vcc, 0, v1, vcc
+; GFX9-NOUNALIGNED-NEXT:	global_load_ubyte v0, v[0:1], off
+; GFX9-NOUNALIGNED-NEXT:	global_load_ubyte v1, v[2:3], off offset:-10
+; GFX9-NOUNALIGNED-NEXT:	global_load_ubyte v4, v[2:3], off offset:-9
+; GFX9-NOUNALIGNED-NEXT:	global_load_ubyte v5, v[2:3], off offset:-8
+; GFX9-NOUNALIGNED-NEXT:	global_load_ubyte v6, v[2:3], off offset:-7
+; GFX9-NOUNALIGNED-NEXT:	global_load_ubyte v7, v[2:3], off offset:-6
+; GFX9-NOUNALIGNED-NEXT:	global_load_ubyte v8, v[2:3], off offset:-5
+; GFX9-NOUNALIGNED-NEXT:	global_load_ubyte v9, v[2:3], off offset:-4
+; GFX9-NOUNALIGNED-NEXT:	global_load_ubyte v10, v[2:3], off offset:-3
+; GFX9-NOUNALIGNED-NEXT:	global_load_ubyte v11, v[2:3], off offset:-2
+; GFX9-NOUNALIGNED-NEXT:	global_load_ubyte v12, v[2:3], off offset:-1
+; GFX9-NOUNALIGNED-NEXT:	global_load_ubyte v2, v[2:3], off
+; GFX9-NOUNALIGNED-NEXT:	v_mov_b32_e32 v3, 0xff
+; GFX9-NOUNALIGNED-NEXT:	s_movk_i32 s4, 0xff
+; GFX9-NOUNALIGNED-NEXT:	v_mov_b32_e32 v13, 8
+; GFX9-NOUNALIGNED-NEXT:	s_mov_b32 s5, 8
+; GFX9-NOUNALIGNED-NEXT:	s_waitcnt vmcnt(10)
+; GFX9-NOUNALIGNED-NEXT:	v_lshlrev_b32_sdwa v1, s5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NOUNALIGNED-NEXT:	s_waitcnt vmcnt(9)
+; GFX9-NOUNALIGNED-NEXT:	v_and_b32_e32 v4, s4, v4
+; GFX9-NOUNALIGNED-NEXT:	s_waitcnt vmcnt(8)
+; GFX9-NOUNALIGNED-NEXT:	v_and_b32_e32 v5, s4, v5
+; GFX9-NOUNALIGNED-NEXT:	v_and_or_b32 v0, v0, s4, v1
+; GFX9-NOUNALIGNED-NEXT:	s_waitcnt vmcnt(6)
+; GFX9-NOUNALIGNED-NEXT:	v_lshlrev_b32_sdwa v7, v13, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NOUNALIGNED-NEXT:	s_waitcnt vmcnt(5)
+; GFX9-NOUNALIGNED-NEXT:	v_and_b32_e32 v8, v8, v3
+; GFX9-NOUNALIGNED-NEXT:	s_waitcnt vmcnt(4)
+; GFX9-NOUNALIGNED-NEXT:	v_and_b32_e32 v9, v9, v3
+; GFX9-NOUNALIGNED-NEXT:	v_lshlrev_b32_e32 v1, 16, v4
+; GFX9-NOUNALIGNED-NEXT:	s_waitcnt vmcnt(2)
+; GFX9-NOUNALIGNED-NEXT:	v_lshlrev_b32_sdwa v11, v13, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NOUNALIGNED-NEXT:	s_waitcnt vmcnt(1)
+; GFX9-NOUNALIGNED-NEXT:	v_and_b32_e32 v12, v12, v3
+; GFX9-NOUNALIGNED-NEXT:	s_waitcnt vmcnt(0)
+; GFX9-NOUNALIGNED-NEXT:	v_and_b32_e32 v2, v2, v3
+; GFX9-NOUNALIGNED-NEXT:	v_lshlrev_b32_e32 v4, 24, v5
+; GFX9-NOUNALIGNED-NEXT:	v_and_or_b32 v5, v6, v3, v7
+; GFX9-NOUNALIGNED-NEXT:	v_lshlrev_b32_e32 v6, 16, v8
+; GFX9-NOUNALIGNED-NEXT:	v_lshlrev_b32_e32 v7, 24, v9
+; GFX9-NOUNALIGNED-NEXT:	v_and_or_b32 v3, v10, v3, v11
+; GFX9-NOUNALIGNED-NEXT:	v_lshlrev_b32_e32 v8, 16, v12
+; GFX9-NOUNALIGNED-NEXT:	v_lshlrev_b32_e32 v2, 24, v2
+; GFX9-NOUNALIGNED-NEXT:	v_or3_b32 v0, v0, v1, v4
+; GFX9-NOUNALIGNED-NEXT:	v_or3_b32 v1, v5, v6, v7
+; GFX9-NOUNALIGNED-NEXT:	v_or3_b32 v2, v3, v8, v2
+; GFX9-NOUNALIGNED-NEXT:	s_setpc_b64 s[30:31]
 ;
 ; GFX7-UNALIGNED-LABEL: v_load_constant_v3i32_align1:
 ; GFX7-UNALIGNED:       ; %bb.0:
@@ -156,31 +155,30 @@ define <3 x i32> @v_load_constant_v3i32_align2(<3 x i32> addrspace(4)* %ptr) {
 ;
 ; GFX9-NOUNALIGNED-LABEL: v_load_constant_v3i32_align2:
 ; GFX9-NOUNALIGNED:       ; %bb.0:
-; GFX9-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NOUNALIGNED-NEXT:    v_add_co_u32_e32 v2, vcc, 10, v0
-; GFX9-NOUNALIGNED-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v1, vcc
-; GFX9-NOUNALIGNED-NEXT:    global_load_ushort v4, v[2:3], off
-; GFX9-NOUNALIGNED-NEXT:    global_load_ushort v0, v[0:1], off
-; GFX9-NOUNALIGNED-NEXT:    global_load_ushort v1, v[2:3], off offset:-8
-; GFX9-NOUNALIGNED-NEXT:    global_load_ushort v5, v[2:3], off offset:-6
-; GFX9-NOUNALIGNED-NEXT:    global_load_ushort v6, v[2:3], off offset:-4
-; GFX9-NOUNALIGNED-NEXT:    global_load_ushort v2, v[2:3], off offset:-2
-; GFX9-NOUNALIGNED-NEXT:    v_mov_b32_e32 v3, 0xffff
-; GFX9-NOUNALIGNED-NEXT:    s_mov_b32 s4, 0xffff
-; GFX9-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(5)
-; GFX9-NOUNALIGNED-NEXT:    v_and_b32_e32 v4, v4, v3
-; GFX9-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX9-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(3)
-; GFX9-NOUNALIGNED-NEXT:    v_and_b32_e32 v1, s4, v1
-; GFX9-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX9-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(1)
-; GFX9-NOUNALIGNED-NEXT:    v_and_b32_e32 v6, v6, v3
-; GFX9-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX9-NOUNALIGNED-NEXT:    v_and_or_b32 v0, v0, s4, v1
-; GFX9-NOUNALIGNED-NEXT:    v_and_or_b32 v1, v5, v3, v6
-; GFX9-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NOUNALIGNED-NEXT:    v_and_or_b32 v2, v2, v3, v4
-; GFX9-NOUNALIGNED-NEXT:    s_setpc_b64 s[30:31]
+; GFX9-NOUNALIGNED-NEXT:        s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NOUNALIGNED-NEXT:        v_add_co_u32_e32 v2, vcc, 10, v0
+; GFX9-NOUNALIGNED-NEXT:        v_addc_co_u32_e32 v3, vcc, 0, v1, vcc
+; GFX9-NOUNALIGNED-NEXT:        global_load_ushort v0, v[0:1], off
+; GFX9-NOUNALIGNED-NEXT:        global_load_ushort v1, v[2:3], off offset:-8
+; GFX9-NOUNALIGNED-NEXT:        global_load_ushort v4, v[2:3], off offset:-6
+; GFX9-NOUNALIGNED-NEXT:        global_load_ushort v5, v[2:3], off offset:-4
+; GFX9-NOUNALIGNED-NEXT:        global_load_ushort v6, v[2:3], off offset:-2
+; GFX9-NOUNALIGNED-NEXT:        global_load_ushort v2, v[2:3], off
+; GFX9-NOUNALIGNED-NEXT:        v_mov_b32_e32 v3, 0xffff
+; GFX9-NOUNALIGNED-NEXT:        s_mov_b32 s4, 0xffff
+; GFX9-NOUNALIGNED-NEXT:        s_waitcnt vmcnt(4)
+; GFX9-NOUNALIGNED-NEXT:        v_and_b32_e32 v1, s4, v1
+; GFX9-NOUNALIGNED-NEXT:        v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NOUNALIGNED-NEXT:        s_waitcnt vmcnt(2)
+; GFX9-NOUNALIGNED-NEXT:        v_and_b32_e32 v5, v5, v3
+; GFX9-NOUNALIGNED-NEXT:        v_lshlrev_b32_e32 v5, 16, v5
+; GFX9-NOUNALIGNED-NEXT:        s_waitcnt vmcnt(0)
+; GFX9-NOUNALIGNED-NEXT:        v_and_b32_e32 v2, v2, v3
+; GFX9-NOUNALIGNED-NEXT:        v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NOUNALIGNED-NEXT:        v_and_or_b32 v0, v0, s4, v1
+; GFX9-NOUNALIGNED-NEXT:        v_and_or_b32 v1, v4, v3, v5
+; GFX9-NOUNALIGNED-NEXT:        v_and_or_b32 v2, v6, v3, v2
+; GFX9-NOUNALIGNED-NEXT:        s_setpc_b64 s[30:31]
 ;
 ; GFX7-UNALIGNED-LABEL: v_load_constant_v3i32_align2:
 ; GFX7-UNALIGNED:       ; %bb.0:
@@ -194,35 +192,36 @@ define <3 x i32> @v_load_constant_v3i32_align2(<3 x i32> addrspace(4)* %ptr) {
 ;
 ; GFX7-NOUNALIGNED-LABEL: v_load_constant_v3i32_align2:
 ; GFX7-NOUNALIGNED:       ; %bb.0:
-; GFX7-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NOUNALIGNED-NEXT:    s_mov_b32 s6, 0
-; GFX7-NOUNALIGNED-NEXT:    s_mov_b32 s7, 0xf000
-; GFX7-NOUNALIGNED-NEXT:    s_mov_b64 s[4:5], 0
-; GFX7-NOUNALIGNED-NEXT:    buffer_load_ushort v2, v[0:1], s[4:7], 0 addr64 offset:10
-; GFX7-NOUNALIGNED-NEXT:    buffer_load_ushort v3, v[0:1], s[4:7], 0 addr64
-; GFX7-NOUNALIGNED-NEXT:    buffer_load_ushort v4, v[0:1], s[4:7], 0 addr64 offset:2
-; GFX7-NOUNALIGNED-NEXT:    buffer_load_ushort v5, v[0:1], s[4:7], 0 addr64 offset:4
-; GFX7-NOUNALIGNED-NEXT:    buffer_load_ushort v6, v[0:1], s[4:7], 0 addr64 offset:6
-; GFX7-NOUNALIGNED-NEXT:    buffer_load_ushort v0, v[0:1], s[4:7], 0 addr64 offset:8
-; GFX7-NOUNALIGNED-NEXT:    s_mov_b32 s4, 0xffff
-; GFX7-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(4)
-; GFX7-NOUNALIGNED-NEXT:    v_and_b32_e32 v1, s4, v3
-; GFX7-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(3)
-; GFX7-NOUNALIGNED-NEXT:    v_and_b32_e32 v3, s4, v4
-; GFX7-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(2)
-; GFX7-NOUNALIGNED-NEXT:    v_and_b32_e32 v4, s4, v5
-; GFX7-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(1)
-; GFX7-NOUNALIGNED-NEXT:    v_and_b32_e32 v5, s4, v6
-; GFX7-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(0)
-; GFX7-NOUNALIGNED-NEXT:    v_and_b32_e32 v6, s4, v0
-; GFX7-NOUNALIGNED-NEXT:    v_and_b32_e32 v0, s4, v2
-; GFX7-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v2, 16, v3
-; GFX7-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v3, 16, v5
-; GFX7-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v5, 16, v0
-; GFX7-NOUNALIGNED-NEXT:    v_or_b32_e32 v0, v1, v2
-; GFX7-NOUNALIGNED-NEXT:    v_or_b32_e32 v1, v4, v3
-; GFX7-NOUNALIGNED-NEXT:    v_or_b32_e32 v2, v6, v5
-; GFX7-NOUNALIGNED-NEXT:    s_setpc_b64 s[30:31]
+; GFX7-NOUNALIGNED-NEXT:        s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NOUNALIGNED-NEXT:        s_mov_b32 s6, 0
+; GFX7-NOUNALIGNED-NEXT:        s_mov_b32 s7, 0xf000
+; GFX7-NOUNALIGNED-NEXT:        s_mov_b64 s[4:5], 0
+; GFX7-NOUNALIGNED-NEXT:        buffer_load_ushort v2, v[0:1], s[4:7], 0 addr64
+; GFX7-NOUNALIGNED-NEXT:        buffer_load_ushort v3, v[0:1], s[4:7], 0 addr64 offset:2
+; GFX7-NOUNALIGNED-NEXT:        buffer_load_ushort v4, v[0:1], s[4:7], 0 addr64 offset:4
+; GFX7-NOUNALIGNED-NEXT:        buffer_load_ushort v5, v[0:1], s[4:7], 0 addr64 offset:6
+; GFX7-NOUNALIGNED-NEXT:        buffer_load_ushort v6, v[0:1], s[4:7], 0 addr64 offset:8
+; GFX7-NOUNALIGNED-NEXT:        buffer_load_ushort v0, v[0:1], s[4:7], 0 addr64 offset:10
+; GFX7-NOUNALIGNED-NEXT:        s_mov_b32 s4, 0xffff
+; GFX7-NOUNALIGNED-NEXT:        s_waitcnt vmcnt(5)
+; GFX7-NOUNALIGNED-NEXT:        v_and_b32_e32 v1, s4, v2
+; GFX7-NOUNALIGNED-NEXT:        s_waitcnt vmcnt(4)
+; GFX7-NOUNALIGNED-NEXT:        v_and_b32_e32 v2, s4, v3
+; GFX7-NOUNALIGNED-NEXT:        s_waitcnt vmcnt(3)
+; GFX7-NOUNALIGNED-NEXT:        v_and_b32_e32 v3, s4, v4
+; GFX7-NOUNALIGNED-NEXT:        s_waitcnt vmcnt(2)
+; GFX7-NOUNALIGNED-NEXT:        v_and_b32_e32 v4, s4, v5
+; GFX7-NOUNALIGNED-NEXT:        s_waitcnt vmcnt(1)
+; GFX7-NOUNALIGNED-NEXT:        v_and_b32_e32 v5, s4, v6
+; GFX7-NOUNALIGNED-NEXT:        s_waitcnt vmcnt(0)
+; GFX7-NOUNALIGNED-NEXT:        v_and_b32_e32 v0, s4, v0
+; GFX7-NOUNALIGNED-NEXT:        v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NOUNALIGNED-NEXT:        v_lshlrev_b32_e32 v6, 16, v0
+; GFX7-NOUNALIGNED-NEXT:        v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NOUNALIGNED-NEXT:        v_or_b32_e32 v0, v1, v2
+; GFX7-NOUNALIGNED-NEXT:        v_or_b32_e32 v1, v3, v4
+; GFX7-NOUNALIGNED-NEXT:        v_or_b32_e32 v2, v5, v6
+; GFX7-NOUNALIGNED-NEXT:        s_setpc_b64 s[30:31]
   %load = load <3 x i32>, <3 x i32> addrspace(4)* %ptr, align 2
   ret <3 x i32> %load
 }
@@ -399,97 +398,101 @@ define amdgpu_ps <3 x i32> @s_load_constant_v3i32_align1(<3 x i32> addrspace(4)*
 ;
 ; GFX9-NOUNALIGNED-LABEL: s_load_constant_v3i32_align1:
 ; GFX9-NOUNALIGNED:       ; %bb.0:
-; GFX9-NOUNALIGNED-NEXT:    s_add_u32 s2, s0, 1
-; GFX9-NOUNALIGNED-NEXT:    s_addc_u32 s3, s1, 0
-; GFX9-NOUNALIGNED-NEXT:    v_mov_b32_e32 v2, s2
-; GFX9-NOUNALIGNED-NEXT:    v_mov_b32_e32 v3, s3
-; GFX9-NOUNALIGNED-NEXT:    s_add_u32 s2, s0, 2
-; GFX9-NOUNALIGNED-NEXT:    s_addc_u32 s3, s1, 0
-; GFX9-NOUNALIGNED-NEXT:    v_mov_b32_e32 v5, s3
-; GFX9-NOUNALIGNED-NEXT:    v_mov_b32_e32 v4, s2
-; GFX9-NOUNALIGNED-NEXT:    s_add_u32 s2, s0, 3
-; GFX9-NOUNALIGNED-NEXT:    s_addc_u32 s3, s1, 0
-; GFX9-NOUNALIGNED-NEXT:    v_mov_b32_e32 v7, s3
-; GFX9-NOUNALIGNED-NEXT:    v_mov_b32_e32 v6, s2
-; GFX9-NOUNALIGNED-NEXT:    s_add_u32 s2, s0, 4
-; GFX9-NOUNALIGNED-NEXT:    s_addc_u32 s3, s1, 0
-; GFX9-NOUNALIGNED-NEXT:    v_mov_b32_e32 v9, s3
-; GFX9-NOUNALIGNED-NEXT:    v_mov_b32_e32 v8, s2
-; GFX9-NOUNALIGNED-NEXT:    s_add_u32 s2, s0, 5
-; GFX9-NOUNALIGNED-NEXT:    s_addc_u32 s3, s1, 0
-; GFX9-NOUNALIGNED-NEXT:    v_mov_b32_e32 v11, s3
-; GFX9-NOUNALIGNED-NEXT:    v_mov_b32_e32 v10, s2
-; GFX9-NOUNALIGNED-NEXT:    s_add_u32 s2, s0, 6
-; GFX9-NOUNALIGNED-NEXT:    s_addc_u32 s3, s1, 0
-; GFX9-NOUNALIGNED-NEXT:    v_mov_b32_e32 v13, s3
-; GFX9-NOUNALIGNED-NEXT:    v_mov_b32_e32 v12, s2
-; GFX9-NOUNALIGNED-NEXT:    s_add_u32 s2, s0, 7
-; GFX9-NOUNALIGNED-NEXT:    s_addc_u32 s3, s1, 0
-; GFX9-NOUNALIGNED-NEXT:    v_mov_b32_e32 v15, s3
-; GFX9-NOUNALIGNED-NEXT:    v_mov_b32_e32 v14, s2
-; GFX9-NOUNALIGNED-NEXT:    s_add_u32 s2, s0, 8
-; GFX9-NOUNALIGNED-NEXT:    s_addc_u32 s3, s1, 0
-; GFX9-NOUNALIGNED-NEXT:    v_mov_b32_e32 v17, s3
-; GFX9-NOUNALIGNED-NEXT:    v_mov_b32_e32 v16, s2
-; GFX9-NOUNALIGNED-NEXT:    s_add_u32 s2, s0, 9
-; GFX9-NOUNALIGNED-NEXT:    s_addc_u32 s3, s1, 0
-; GFX9-NOUNALIGNED-NEXT:    v_mov_b32_e32 v19, s3
-; GFX9-NOUNALIGNED-NEXT:    v_mov_b32_e32 v18, s2
-; GFX9-NOUNALIGNED-NEXT:    s_add_u32 s2, s0, 10
-; GFX9-NOUNALIGNED-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-NOUNALIGNED-NEXT:    s_addc_u32 s3, s1, 0
-; GFX9-NOUNALIGNED-NEXT:    v_mov_b32_e32 v1, s1
-; GFX9-NOUNALIGNED-NEXT:    s_add_u32 s0, s0, 11
-; GFX9-NOUNALIGNED-NEXT:    global_load_ubyte v20, v[10:11], off
-; GFX9-NOUNALIGNED-NEXT:    global_load_ubyte v21, v[12:13], off
-; GFX9-NOUNALIGNED-NEXT:    global_load_ubyte v14, v[14:15], off
-; GFX9-NOUNALIGNED-NEXT:    global_load_ubyte v15, v[16:17], off
-; GFX9-NOUNALIGNED-NEXT:    global_load_ubyte v16, v[18:19], off
-; GFX9-NOUNALIGNED-NEXT:    s_addc_u32 s1, s1, 0
-; GFX9-NOUNALIGNED-NEXT:    v_mov_b32_e32 v11, s3
-; GFX9-NOUNALIGNED-NEXT:    v_mov_b32_e32 v13, s1
-; GFX9-NOUNALIGNED-NEXT:    v_mov_b32_e32 v10, s2
-; GFX9-NOUNALIGNED-NEXT:    v_mov_b32_e32 v12, s0
-; GFX9-NOUNALIGNED-NEXT:    global_load_ubyte v10, v[10:11], off
-; GFX9-NOUNALIGNED-NEXT:    global_load_ubyte v11, v[12:13], off
-; GFX9-NOUNALIGNED-NEXT:    global_load_ubyte v0, v[0:1], off
-; GFX9-NOUNALIGNED-NEXT:    global_load_ubyte v1, v[2:3], off
-; GFX9-NOUNALIGNED-NEXT:    global_load_ubyte v2, v[4:5], off
-; GFX9-NOUNALIGNED-NEXT:    global_load_ubyte v3, v[6:7], off
-; GFX9-NOUNALIGNED-NEXT:    global_load_ubyte v4, v[8:9], off
-; GFX9-NOUNALIGNED-NEXT:    s_mov_b32 s1, 8
-; GFX9-NOUNALIGNED-NEXT:    s_movk_i32 s0, 0xff
-; GFX9-NOUNALIGNED-NEXT:    v_mov_b32_e32 v5, 0xff
-; GFX9-NOUNALIGNED-NEXT:    v_mov_b32_e32 v6, 8
-; GFX9-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(3)
-; GFX9-NOUNALIGNED-NEXT:    v_lshlrev_b32_sdwa v1, s1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NOUNALIGNED-NEXT:    v_and_or_b32 v0, v0, s0, v1
-; GFX9-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NOUNALIGNED-NEXT:    v_and_b32_e32 v1, s0, v2
-; GFX9-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(1)
-; GFX9-NOUNALIGNED-NEXT:    v_and_b32_e32 v2, s0, v3
-; GFX9-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX9-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v2, 24, v2
-; GFX9-NOUNALIGNED-NEXT:    v_or3_b32 v0, v0, v1, v2
-; GFX9-NOUNALIGNED-NEXT:    v_and_b32_e32 v1, v21, v5
-; GFX9-NOUNALIGNED-NEXT:    v_and_b32_e32 v2, v14, v5
-; GFX9-NOUNALIGNED-NEXT:    v_readfirstlane_b32 s0, v0
-; GFX9-NOUNALIGNED-NEXT:    v_lshlrev_b32_sdwa v0, v6, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NOUNALIGNED-NEXT:    v_and_or_b32 v0, v4, v5, v0
-; GFX9-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX9-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v2, 24, v2
-; GFX9-NOUNALIGNED-NEXT:    v_or3_b32 v1, v0, v1, v2
-; GFX9-NOUNALIGNED-NEXT:    v_lshlrev_b32_sdwa v0, v6, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NOUNALIGNED-NEXT:    v_and_b32_e32 v2, v11, v5
-; GFX9-NOUNALIGNED-NEXT:    v_readfirstlane_b32 s1, v1
-; GFX9-NOUNALIGNED-NEXT:    v_and_b32_e32 v1, v10, v5
-; GFX9-NOUNALIGNED-NEXT:    v_and_or_b32 v0, v15, v5, v0
-; GFX9-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX9-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v2, 24, v2
-; GFX9-NOUNALIGNED-NEXT:    v_or3_b32 v2, v0, v1, v2
-; GFX9-NOUNALIGNED-NEXT:    v_readfirstlane_b32 s2, v2
-; GFX9-NOUNALIGNED-NEXT:    ; return to shader part epilog
+; GFX9-NOUNALIGNED-NEXT:	s_add_u32 s2, s0, 1
+; GFX9-NOUNALIGNED-NEXT:	s_addc_u32 s3, s1, 0
+; GFX9-NOUNALIGNED-NEXT:	v_mov_b32_e32 v2, s2
+; GFX9-NOUNALIGNED-NEXT:	v_mov_b32_e32 v3, s3
+; GFX9-NOUNALIGNED-NEXT:	s_add_u32 s2, s0, 2
+; GFX9-NOUNALIGNED-NEXT:	s_addc_u32 s3, s1, 0
+; GFX9-NOUNALIGNED-NEXT:	v_mov_b32_e32 v5, s3
+; GFX9-NOUNALIGNED-NEXT:	v_mov_b32_e32 v4, s2
+; GFX9-NOUNALIGNED-NEXT:	s_add_u32 s2, s0, 3
+; GFX9-NOUNALIGNED-NEXT:	s_addc_u32 s3, s1, 0
+; GFX9-NOUNALIGNED-NEXT:	v_mov_b32_e32 v7, s3
+; GFX9-NOUNALIGNED-NEXT:	v_mov_b32_e32 v6, s2
+; GFX9-NOUNALIGNED-NEXT:	s_add_u32 s2, s0, 4
+; GFX9-NOUNALIGNED-NEXT:	s_addc_u32 s3, s1, 0
+; GFX9-NOUNALIGNED-NEXT:	v_mov_b32_e32 v9, s3
+; GFX9-NOUNALIGNED-NEXT:	v_mov_b32_e32 v8, s2
+; GFX9-NOUNALIGNED-NEXT:	s_add_u32 s2, s0, 5
+; GFX9-NOUNALIGNED-NEXT:	s_addc_u32 s3, s1, 0
+; GFX9-NOUNALIGNED-NEXT:	v_mov_b32_e32 v11, s3
+; GFX9-NOUNALIGNED-NEXT:	v_mov_b32_e32 v10, s2
+; GFX9-NOUNALIGNED-NEXT:	s_add_u32 s2, s0, 6
+; GFX9-NOUNALIGNED-NEXT:	s_addc_u32 s3, s1, 0
+; GFX9-NOUNALIGNED-NEXT:	v_mov_b32_e32 v13, s3
+; GFX9-NOUNALIGNED-NEXT:	v_mov_b32_e32 v12, s2
+; GFX9-NOUNALIGNED-NEXT:	s_add_u32 s2, s0, 7
+; GFX9-NOUNALIGNED-NEXT:	s_addc_u32 s3, s1, 0
+; GFX9-NOUNALIGNED-NEXT:	v_mov_b32_e32 v0, s0
+; GFX9-NOUNALIGNED-NEXT:	v_mov_b32_e32 v15, s3
+; GFX9-NOUNALIGNED-NEXT:	v_mov_b32_e32 v14, s2
+; GFX9-NOUNALIGNED-NEXT:	s_add_u32 s2, s0, 8
+; GFX9-NOUNALIGNED-NEXT:	v_mov_b32_e32 v1, s1
+; GFX9-NOUNALIGNED-NEXT:	global_load_ubyte v16, v[0:1], off
+; GFX9-NOUNALIGNED-NEXT:	global_load_ubyte v17, v[2:3], off
+; GFX9-NOUNALIGNED-NEXT:	global_load_ubyte v18, v[4:5], off
+; GFX9-NOUNALIGNED-NEXT:	global_load_ubyte v19, v[6:7], off
+; GFX9-NOUNALIGNED-NEXT:	global_load_ubyte v8, v[8:9], off
+; GFX9-NOUNALIGNED-NEXT:	global_load_ubyte v9, v[10:11], off
+; GFX9-NOUNALIGNED-NEXT:	global_load_ubyte v10, v[12:13], off
+; GFX9-NOUNALIGNED-NEXT:	global_load_ubyte v11, v[14:15], off
+; GFX9-NOUNALIGNED-NEXT:	s_addc_u32 s3, s1, 0
+; GFX9-NOUNALIGNED-NEXT:	v_mov_b32_e32 v0, s2
+; GFX9-NOUNALIGNED-NEXT:	v_mov_b32_e32 v1, s3
+; GFX9-NOUNALIGNED-NEXT:	s_add_u32 s2, s0, 9
+; GFX9-NOUNALIGNED-NEXT:	s_addc_u32 s3, s1, 0
+; GFX9-NOUNALIGNED-NEXT:	v_mov_b32_e32 v2, s2
+; GFX9-NOUNALIGNED-NEXT:	v_mov_b32_e32 v3, s3
+; GFX9-NOUNALIGNED-NEXT:	s_add_u32 s2, s0, 10
+; GFX9-NOUNALIGNED-NEXT:	s_addc_u32 s3, s1, 0
+; GFX9-NOUNALIGNED-NEXT:	s_add_u32 s0, s0, 11
+; GFX9-NOUNALIGNED-NEXT:	s_addc_u32 s1, s1, 0
+; GFX9-NOUNALIGNED-NEXT:	v_mov_b32_e32 v5, s3
+; GFX9-NOUNALIGNED-NEXT:	v_mov_b32_e32 v7, s1
+; GFX9-NOUNALIGNED-NEXT:	v_mov_b32_e32 v4, s2
+; GFX9-NOUNALIGNED-NEXT:	v_mov_b32_e32 v6, s0
+; GFX9-NOUNALIGNED-NEXT:	global_load_ubyte v12, v[0:1], off
+; GFX9-NOUNALIGNED-NEXT:	global_load_ubyte v2, v[2:3], off
+; GFX9-NOUNALIGNED-NEXT:	global_load_ubyte v3, v[4:5], off
+; GFX9-NOUNALIGNED-NEXT:	global_load_ubyte v4, v[6:7], off
+; GFX9-NOUNALIGNED-NEXT:	s_movk_i32 s0, 0xff
+; GFX9-NOUNALIGNED-NEXT:	s_mov_b32 s1, 8
+; GFX9-NOUNALIGNED-NEXT:	v_mov_b32_e32 v5, 0xff
+; GFX9-NOUNALIGNED-NEXT:	v_mov_b32_e32 v6, 8
+; GFX9-NOUNALIGNED-NEXT:	s_waitcnt vmcnt(10)
+; GFX9-NOUNALIGNED-NEXT:	v_lshlrev_b32_sdwa v0, s1, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NOUNALIGNED-NEXT:	s_waitcnt vmcnt(9)
+; GFX9-NOUNALIGNED-NEXT:	v_and_b32_e32 v1, s0, v18
+; GFX9-NOUNALIGNED-NEXT:	s_waitcnt vmcnt(8)
+; GFX9-NOUNALIGNED-NEXT:	v_and_b32_e32 v7, s0, v19
+; GFX9-NOUNALIGNED-NEXT:	v_and_or_b32 v0, v16, s0, v0
+; GFX9-NOUNALIGNED-NEXT:	v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NOUNALIGNED-NEXT:	v_lshlrev_b32_e32 v7, 24, v7
+; GFX9-NOUNALIGNED-NEXT:	v_or3_b32 v0, v0, v1, v7
+; GFX9-NOUNALIGNED-NEXT:	s_waitcnt vmcnt(5)
+; GFX9-NOUNALIGNED-NEXT:	v_and_b32_e32 v1, v10, v5
+; GFX9-NOUNALIGNED-NEXT:	s_waitcnt vmcnt(4)
+; GFX9-NOUNALIGNED-NEXT:	v_and_b32_e32 v7, v11, v5
+; GFX9-NOUNALIGNED-NEXT:	v_readfirstlane_b32 s0, v0
+; GFX9-NOUNALIGNED-NEXT:	v_lshlrev_b32_sdwa v0, v6, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NOUNALIGNED-NEXT:	v_and_or_b32 v0, v8, v5, v0
+; GFX9-NOUNALIGNED-NEXT:	v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NOUNALIGNED-NEXT:	v_lshlrev_b32_e32 v7, 24, v7
+; GFX9-NOUNALIGNED-NEXT:	v_or3_b32 v1, v0, v1, v7
+; GFX9-NOUNALIGNED-NEXT:	v_readfirstlane_b32 s1, v1
+; GFX9-NOUNALIGNED-NEXT:	s_waitcnt vmcnt(2)
+; GFX9-NOUNALIGNED-NEXT:	v_lshlrev_b32_sdwa v0, v6, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NOUNALIGNED-NEXT:	s_waitcnt vmcnt(1)
+; GFX9-NOUNALIGNED-NEXT:	v_and_b32_e32 v1, v3, v5
+; GFX9-NOUNALIGNED-NEXT:	s_waitcnt vmcnt(0)
+; GFX9-NOUNALIGNED-NEXT:	v_and_b32_e32 v2, v4, v5
+; GFX9-NOUNALIGNED-NEXT:	v_and_or_b32 v0, v12, v5, v0
+; GFX9-NOUNALIGNED-NEXT:	v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NOUNALIGNED-NEXT:	v_lshlrev_b32_e32 v2, 24, v2
+; GFX9-NOUNALIGNED-NEXT:	v_or3_b32 v2, v0, v1, v2
+; GFX9-NOUNALIGNED-NEXT:	v_readfirstlane_b32 s2, v2
+; GFX9-NOUNALIGNED-NEXT:	; return to shader part epilog
 ;
 ; GFX7-UNALIGNED-LABEL: s_load_constant_v3i32_align1:
 ; GFX7-UNALIGNED:       ; %bb.0:
@@ -585,52 +588,52 @@ define amdgpu_ps <3 x i32> @s_load_constant_v3i32_align2(<3 x i32> addrspace(4)*
 ;
 ; GFX9-NOUNALIGNED-LABEL: s_load_constant_v3i32_align2:
 ; GFX9-NOUNALIGNED:       ; %bb.0:
-; GFX9-NOUNALIGNED-NEXT:    s_add_u32 s2, s0, 2
-; GFX9-NOUNALIGNED-NEXT:    s_addc_u32 s3, s1, 0
-; GFX9-NOUNALIGNED-NEXT:    v_mov_b32_e32 v2, s2
-; GFX9-NOUNALIGNED-NEXT:    v_mov_b32_e32 v3, s3
-; GFX9-NOUNALIGNED-NEXT:    s_add_u32 s2, s0, 4
-; GFX9-NOUNALIGNED-NEXT:    s_addc_u32 s3, s1, 0
-; GFX9-NOUNALIGNED-NEXT:    v_mov_b32_e32 v5, s3
-; GFX9-NOUNALIGNED-NEXT:    v_mov_b32_e32 v4, s2
-; GFX9-NOUNALIGNED-NEXT:    s_add_u32 s2, s0, 6
-; GFX9-NOUNALIGNED-NEXT:    s_addc_u32 s3, s1, 0
-; GFX9-NOUNALIGNED-NEXT:    v_mov_b32_e32 v7, s3
-; GFX9-NOUNALIGNED-NEXT:    v_mov_b32_e32 v6, s2
-; GFX9-NOUNALIGNED-NEXT:    s_add_u32 s2, s0, 8
-; GFX9-NOUNALIGNED-NEXT:    s_addc_u32 s3, s1, 0
-; GFX9-NOUNALIGNED-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-NOUNALIGNED-NEXT:    v_mov_b32_e32 v1, s1
-; GFX9-NOUNALIGNED-NEXT:    s_add_u32 s0, s0, 10
-; GFX9-NOUNALIGNED-NEXT:    s_addc_u32 s1, s1, 0
-; GFX9-NOUNALIGNED-NEXT:    v_mov_b32_e32 v11, s1
-; GFX9-NOUNALIGNED-NEXT:    v_mov_b32_e32 v9, s3
-; GFX9-NOUNALIGNED-NEXT:    v_mov_b32_e32 v10, s0
-; GFX9-NOUNALIGNED-NEXT:    v_mov_b32_e32 v8, s2
-; GFX9-NOUNALIGNED-NEXT:    global_load_ushort v10, v[10:11], off
-; GFX9-NOUNALIGNED-NEXT:    global_load_ushort v0, v[0:1], off
-; GFX9-NOUNALIGNED-NEXT:    global_load_ushort v1, v[2:3], off
-; GFX9-NOUNALIGNED-NEXT:    global_load_ushort v2, v[4:5], off
-; GFX9-NOUNALIGNED-NEXT:    global_load_ushort v3, v[6:7], off
-; GFX9-NOUNALIGNED-NEXT:    global_load_ushort v4, v[8:9], off
-; GFX9-NOUNALIGNED-NEXT:    s_mov_b32 s0, 0xffff
-; GFX9-NOUNALIGNED-NEXT:    v_mov_b32_e32 v5, 0xffff
-; GFX9-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(3)
-; GFX9-NOUNALIGNED-NEXT:    v_and_b32_e32 v1, s0, v1
-; GFX9-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX9-NOUNALIGNED-NEXT:    v_and_or_b32 v0, v0, s0, v1
-; GFX9-NOUNALIGNED-NEXT:    v_readfirstlane_b32 s0, v0
-; GFX9-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(1)
-; GFX9-NOUNALIGNED-NEXT:    v_and_b32_e32 v0, v3, v5
-; GFX9-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; GFX9-NOUNALIGNED-NEXT:    v_and_or_b32 v1, v2, v5, v0
-; GFX9-NOUNALIGNED-NEXT:    v_and_b32_e32 v0, v10, v5
-; GFX9-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; GFX9-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NOUNALIGNED-NEXT:    v_and_or_b32 v2, v4, v5, v0
-; GFX9-NOUNALIGNED-NEXT:    v_readfirstlane_b32 s1, v1
-; GFX9-NOUNALIGNED-NEXT:    v_readfirstlane_b32 s2, v2
-; GFX9-NOUNALIGNED-NEXT:    ; return to shader part epilog
+; GFX9-NOUNALIGNED-NEXT:	s_add_u32 s2, s0, 2
+; GFX9-NOUNALIGNED-NEXT:	s_addc_u32 s3, s1, 0
+; GFX9-NOUNALIGNED-NEXT:	v_mov_b32_e32 v2, s2
+; GFX9-NOUNALIGNED-NEXT:	v_mov_b32_e32 v3, s3
+; GFX9-NOUNALIGNED-NEXT:	s_add_u32 s2, s0, 4
+; GFX9-NOUNALIGNED-NEXT:	s_addc_u32 s3, s1, 0
+; GFX9-NOUNALIGNED-NEXT:	v_mov_b32_e32 v5, s3
+; GFX9-NOUNALIGNED-NEXT:	v_mov_b32_e32 v4, s2
+; GFX9-NOUNALIGNED-NEXT:	s_add_u32 s2, s0, 6
+; GFX9-NOUNALIGNED-NEXT:	s_addc_u32 s3, s1, 0
+; GFX9-NOUNALIGNED-NEXT:	v_mov_b32_e32 v7, s3
+; GFX9-NOUNALIGNED-NEXT:	v_mov_b32_e32 v6, s2
+; GFX9-NOUNALIGNED-NEXT:	s_add_u32 s2, s0, 8
+; GFX9-NOUNALIGNED-NEXT:	s_addc_u32 s3, s1, 0
+; GFX9-NOUNALIGNED-NEXT:	v_mov_b32_e32 v0, s0
+; GFX9-NOUNALIGNED-NEXT:	v_mov_b32_e32 v1, s1
+; GFX9-NOUNALIGNED-NEXT:	s_add_u32 s0, s0, 10
+; GFX9-NOUNALIGNED-NEXT:	s_addc_u32 s1, s1, 0
+; GFX9-NOUNALIGNED-NEXT:	v_mov_b32_e32 v9, s3
+; GFX9-NOUNALIGNED-NEXT:	v_mov_b32_e32 v11, s1
+; GFX9-NOUNALIGNED-NEXT:	v_mov_b32_e32 v10, s0
+; GFX9-NOUNALIGNED-NEXT:	v_mov_b32_e32 v8, s2
+; GFX9-NOUNALIGNED-NEXT:	global_load_ushort v0, v[0:1], off
+; GFX9-NOUNALIGNED-NEXT:	global_load_ushort v1, v[2:3], off
+; GFX9-NOUNALIGNED-NEXT:	global_load_ushort v2, v[4:5], off
+; GFX9-NOUNALIGNED-NEXT:	global_load_ushort v3, v[6:7], off
+; GFX9-NOUNALIGNED-NEXT:	global_load_ushort v4, v[8:9], off
+; GFX9-NOUNALIGNED-NEXT:	global_load_ushort v5, v[10:11], off
+; GFX9-NOUNALIGNED-NEXT:	s_mov_b32 s0, 0xffff
+; GFX9-NOUNALIGNED-NEXT:	v_mov_b32_e32 v6, 0xffff
+; GFX9-NOUNALIGNED-NEXT:	s_waitcnt vmcnt(4)
+; GFX9-NOUNALIGNED-NEXT:	v_and_b32_e32 v1, s0, v1
+; GFX9-NOUNALIGNED-NEXT:	v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NOUNALIGNED-NEXT:	v_and_or_b32 v0, v0, s0, v1
+; GFX9-NOUNALIGNED-NEXT:	v_readfirstlane_b32 s0, v0
+; GFX9-NOUNALIGNED-NEXT:	s_waitcnt vmcnt(2)
+; GFX9-NOUNALIGNED-NEXT:	v_and_b32_e32 v0, v3, v6
+; GFX9-NOUNALIGNED-NEXT:	v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NOUNALIGNED-NEXT:	v_and_or_b32 v1, v2, v6, v0
+; GFX9-NOUNALIGNED-NEXT:	s_waitcnt vmcnt(0)
+; GFX9-NOUNALIGNED-NEXT:	v_and_b32_e32 v0, v5, v6
+; GFX9-NOUNALIGNED-NEXT:	v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NOUNALIGNED-NEXT:	v_and_or_b32 v2, v4, v6, v0
+; GFX9-NOUNALIGNED-NEXT:	v_readfirstlane_b32 s1, v1
+; GFX9-NOUNALIGNED-NEXT:	v_readfirstlane_b32 s2, v2
+; GFX9-NOUNALIGNED-NEXT:	; return to shader part epilog
 ;
 ; GFX7-UNALIGNED-LABEL: s_load_constant_v3i32_align2:
 ; GFX7-UNALIGNED:       ; %bb.0:
@@ -648,37 +651,35 @@ define amdgpu_ps <3 x i32> @s_load_constant_v3i32_align2(<3 x i32> addrspace(4)*
 ;
 ; GFX7-NOUNALIGNED-LABEL: s_load_constant_v3i32_align2:
 ; GFX7-NOUNALIGNED:       ; %bb.0:
-; GFX7-NOUNALIGNED-NEXT:    s_mov_b32 s2, -1
-; GFX7-NOUNALIGNED-NEXT:    s_mov_b32 s3, 0xf000
-; GFX7-NOUNALIGNED-NEXT:    buffer_load_ushort v0, off, s[0:3], 0 offset:10
-; GFX7-NOUNALIGNED-NEXT:    buffer_load_ushort v1, off, s[0:3], 0
-; GFX7-NOUNALIGNED-NEXT:    buffer_load_ushort v2, off, s[0:3], 0 offset:2
-; GFX7-NOUNALIGNED-NEXT:    buffer_load_ushort v3, off, s[0:3], 0 offset:4
-; GFX7-NOUNALIGNED-NEXT:    buffer_load_ushort v4, off, s[0:3], 0 offset:6
-; GFX7-NOUNALIGNED-NEXT:    buffer_load_ushort v5, off, s[0:3], 0 offset:8
-; GFX7-NOUNALIGNED-NEXT:    s_mov_b32 s0, 0xffff
-; GFX7-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(5)
-; GFX7-NOUNALIGNED-NEXT:    v_and_b32_e32 v0, s0, v0
-; GFX7-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(4)
-; GFX7-NOUNALIGNED-NEXT:    v_and_b32_e32 v1, s0, v1
-; GFX7-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(3)
-; GFX7-NOUNALIGNED-NEXT:    v_and_b32_e32 v2, s0, v2
-; GFX7-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(2)
-; GFX7-NOUNALIGNED-NEXT:    v_and_b32_e32 v3, s0, v3
-; GFX7-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(1)
-; GFX7-NOUNALIGNED-NEXT:    v_and_b32_e32 v4, s0, v4
-; GFX7-NOUNALIGNED-NEXT:    s_waitcnt vmcnt(0)
-; GFX7-NOUNALIGNED-NEXT:    v_and_b32_e32 v5, s0, v5
-; GFX7-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v6, 16, v0
-; GFX7-NOUNALIGNED-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX7-NOUNALIGNED-NEXT:    v_or_b32_e32 v0, v1, v2
-; GFX7-NOUNALIGNED-NEXT:    v_or_b32_e32 v1, v3, v4
-; GFX7-NOUNALIGNED-NEXT:    v_or_b32_e32 v2, v5, v6
-; GFX7-NOUNALIGNED-NEXT:    v_readfirstlane_b32 s0, v0
-; GFX7-NOUNALIGNED-NEXT:    v_readfirstlane_b32 s1, v1
-; GFX7-NOUNALIGNED-NEXT:    v_readfirstlane_b32 s2, v2
-; GFX7-NOUNALIGNED-NEXT:    ; return to shader part epilog
+; GFX7-NOUNALIGNED-NEXT:        s_mov_b32 s2, -1
+; GFX7-NOUNALIGNED-NEXT:        s_mov_b32 s3, 0xf000
+; GFX7-NOUNALIGNED-NEXT:        buffer_load_ushort v0, off, s[0:3], 0
+; GFX7-NOUNALIGNED-NEXT:        buffer_load_ushort v1, off, s[0:3], 0 offset:2
+; GFX7-NOUNALIGNED-NEXT:        buffer_load_ushort v2, off, s[0:3], 0 offset:4
+; GFX7-NOUNALIGNED-NEXT:        buffer_load_ushort v3, off, s[0:3], 0 offset:6
+; GFX7-NOUNALIGNED-NEXT:        buffer_load_ushort v4, off, s[0:3], 0 offset:8
+; GFX7-NOUNALIGNED-NEXT:        buffer_load_ushort v5, off, s[0:3], 0 offset:10
+; GFX7-NOUNALIGNED-NEXT:        s_mov_b32 s0, 0xffff
+; GFX7-NOUNALIGNED-NEXT:        s_waitcnt vmcnt(5)
+; GFX7-NOUNALIGNED-NEXT:        v_and_b32_e32 v0, s0, v0
+; GFX7-NOUNALIGNED-NEXT:        s_waitcnt vmcnt(4)
+; GFX7-NOUNALIGNED-NEXT:        v_and_b32_e32 v1, s0, v1
+; GFX7-NOUNALIGNED-NEXT:        v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-NOUNALIGNED-NEXT:        s_waitcnt vmcnt(2)
+; GFX7-NOUNALIGNED-NEXT:        v_and_b32_e32 v3, s0, v3
+; GFX7-NOUNALIGNED-NEXT:        v_and_b32_e32 v2, s0, v2
+; GFX7-NOUNALIGNED-NEXT:        s_waitcnt vmcnt(0)
+; GFX7-NOUNALIGNED-NEXT:        v_and_b32_e32 v5, s0, v5
+; GFX7-NOUNALIGNED-NEXT:        v_and_b32_e32 v4, s0, v4
+; GFX7-NOUNALIGNED-NEXT:        v_lshlrev_b32_e32 v3, 16, v3
+; GFX7-NOUNALIGNED-NEXT:        v_lshlrev_b32_e32 v5, 16, v5
+; GFX7-NOUNALIGNED-NEXT:        v_or_b32_e32 v0, v0, v1
+; GFX7-NOUNALIGNED-NEXT:        v_or_b32_e32 v1, v2, v3
+; GFX7-NOUNALIGNED-NEXT:        v_or_b32_e32 v2, v4, v5
+; GFX7-NOUNALIGNED-NEXT:        v_readfirstlane_b32 s0, v0
+; GFX7-NOUNALIGNED-NEXT:        v_readfirstlane_b32 s1, v1
+; GFX7-NOUNALIGNED-NEXT:        v_readfirstlane_b32 s2, v2
+; GFX7-NOUNALIGNED-NEXT:        ; return to shader part epilog
   %load = load <3 x i32>, <3 x i32> addrspace(4)* %ptr, align 2
   ret <3 x i32> %load
 }
diff --git a/llvm/test/CodeGen/AMDGPU/amdhsa-trap-num-sgprs.ll b/llvm/test/CodeGen/AMDGPU/amdhsa-trap-num-sgprs.ll
index 3d75eca93cb48..8f4f1c3915351 100644
--- a/llvm/test/CodeGen/AMDGPU/amdhsa-trap-num-sgprs.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdhsa-trap-num-sgprs.ll
@@ -3,7 +3,7 @@
 
 ; GCN-LABEL: {{^}}amdhsa_trap_num_sgprs
 ; TRAP-HANDLER-ENABLE:  NumSgprs: 61
-; TRAP-HANDLER-DISABLE: NumSgprs: 79
+; TRAP-HANDLER-DISABLE: NumSgprs: 77
 define amdgpu_kernel void @amdhsa_trap_num_sgprs(
     i32 addrspace(1)* %out0, i32 %in0,
     i32 addrspace(1)* %out1, i32 %in1,
diff --git a/llvm/test/CodeGen/AMDGPU/call-argument-types.ll b/llvm/test/CodeGen/AMDGPU/call-argument-types.ll
index 84eb94228decc..2986cb48a86ea 100644
--- a/llvm/test/CodeGen/AMDGPU/call-argument-types.ll
+++ b/llvm/test/CodeGen/AMDGPU/call-argument-types.ll
@@ -840,14 +840,14 @@ entry:
 ; GCN: v_mov_b32_e32 [[REG10:v[0-9]+]], 10
 ; GCN: v_mov_b32_e32 [[REG11:v[0-9]+]], 11
 ; GCN: v_mov_b32_e32 [[REG12:v[0-9]+]], 12
+; GCN: v_mov_b32_e32 [[REG13:v[0-9]+]], 13
+; GCN: v_mov_b32_e32 [[REG14:v[0-9]+]], 14
+; GCN: v_mov_b32_e32 [[REG15:v[0-9]+]], 15
 ; GCN: buffer_store_dword [[REG8]], {{.*$}}
 ; GCN: buffer_store_dword [[REG9]], {{.*}} offset:4
 ; GCN: buffer_store_dword [[REG10]], {{.*}} offset:8
 ; GCN: buffer_store_dword [[REG11]], {{.*}} offset:12
 ; GCN: buffer_store_dword [[REG12]], {{.*}} offset:16
-; GCN: v_mov_b32_e32 [[REG13:v[0-9]+]], 13
-; GCN: v_mov_b32_e32 [[REG14:v[0-9]+]], 14
-; GCN: v_mov_b32_e32 [[REG15:v[0-9]+]], 15
 ; GCN: buffer_store_dword [[REG13]], {{.*}} offset:20
 ; GCN: buffer_store_dword [[REG14]], {{.*}} offset:24
 ; GCN: buffer_store_dword [[REG15]], {{.*}} offset:28
@@ -874,14 +874,14 @@ entry:
 ; GCN: v_mov_b32_e32 [[REG10:v[0-9]+]], 0x41200000
 ; GCN: v_mov_b32_e32 [[REG11:v[0-9]+]], 0x41300000
 ; GCN: v_mov_b32_e32 [[REG12:v[0-9]+]], 0x41400000
+; GCN: v_mov_b32_e32 [[REG13:v[0-9]+]], 0x41500000
+; GCN: v_mov_b32_e32 [[REG14:v[0-9]+]], 0x41600000
+; GCN: v_mov_b32_e32 [[REG15:v[0-9]+]], 0x41700000
 ; GCN: buffer_store_dword [[REG8]], {{.*$}}
 ; GCN: buffer_store_dword [[REG9]], {{.*}} offset:4
 ; GCN: buffer_store_dword [[REG10]], {{.*}} offset:8
 ; GCN: buffer_store_dword [[REG11]], {{.*}} offset:12
 ; GCN: buffer_store_dword [[REG12]], {{.*}} offset:16
-; GCN: v_mov_b32_e32 [[REG13:v[0-9]+]], 0x41500000
-; GCN: v_mov_b32_e32 [[REG14:v[0-9]+]], 0x41600000
-; GCN: v_mov_b32_e32 [[REG15:v[0-9]+]], 0x41700000
 ; GCN: buffer_store_dword [[REG13]], {{.*}} offset:20
 ; GCN: buffer_store_dword [[REG14]], {{.*}} offset:24
 ; GCN: buffer_store_dword [[REG15]], {{.*}} offset:28
diff --git a/llvm/test/CodeGen/AMDGPU/cttz_zero_undef.ll b/llvm/test/CodeGen/AMDGPU/cttz_zero_undef.ll
index 6bc3073eb18e0..27698c58e3c03 100644
--- a/llvm/test/CodeGen/AMDGPU/cttz_zero_undef.ll
+++ b/llvm/test/CodeGen/AMDGPU/cttz_zero_undef.ll
@@ -177,13 +177,13 @@ define amdgpu_kernel void @v_cttz_zero_undef_i32_with_select(i32 addrspace(1)* n
 ; SI-NOSDWA: v_or_b32_e32 [[VAL2:v[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}
 ; SI-NOSDWA: v_ffbl_b32_e32 v{{[0-9]+}}, [[VAL1]]
 ; SI-NOSDWA: v_ffbl_b32_e32 v{{[0-9]+}}, [[VAL2]]
-; SI-SDWA: v_or_b32_sdwa
 ; SI-SDWA: v_or_b32_e32
+; SI-SDWA: v_or_b32_sdwa
 ; SI-SDWA: v_or_b32_e32
-; SI-SDWA: v_or_b32_e32 [[VAL1:v[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}
-; SI-SDWA: v_ffbl_b32_e32 v{{[0-9]+}}, [[VAL1]]
 ; SI-SDWA: v_or_b32_sdwa
+; SI-SDWA: v_or_b32_e32 [[VAL1:v[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}
 ; SI-SDWA: v_or_b32_e32 [[VAL2:v[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}
+; SI-SDWA: v_ffbl_b32_e32 v{{[0-9]+}}, [[VAL1]]
 ; SI-SDWA: v_ffbl_b32_e32 v{{[0-9]+}}, [[VAL2]]
 ; SI: v_cmp_eq_u32_e32 vcc, 0
 ; SI: v_cmp_ne_u64_e32 vcc, 0
diff --git a/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll b/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
index 21360aa85cbc1..7ea072bffecb5 100644
--- a/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
+++ b/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
@@ -636,81 +636,81 @@ define amdgpu_kernel void @load_v4i8_to_v4f32_unaligned(<4 x float> addrspace(1)
 define amdgpu_kernel void @load_v4i8_to_v4f32_2_uses(<4 x float> addrspace(1)* noalias %out, <4 x i8> addrspace(1)* noalias %out2, <4 x i8> addrspace(1)* noalias %in) nounwind {
 ; SI-LABEL: load_v4i8_to_v4f32_2_uses:
 ; SI:       ; %bb.0:
-; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xd
-; SI-NEXT:    s_mov_b32 s3, 0xf000
-; SI-NEXT:    s_mov_b32 s6, 0
-; SI-NEXT:    s_mov_b32 s7, s3
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; SI-NEXT:    v_mov_b32_e32 v1, 0
-; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
-; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
-; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
-; SI-NEXT:    s_mov_b32 s2, -1
-; SI-NEXT:    s_movk_i32 s8, 0xff
-; SI-NEXT:    s_mov_b32 s6, s2
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_lshrrev_b32_e32 v5, 16, v4
-; SI-NEXT:    v_lshrrev_b32_e32 v6, 24, v4
-; SI-NEXT:    v_and_b32_e32 v7, 0xff00, v4
-; SI-NEXT:    v_cvt_f32_ubyte3_e32 v3, v4
-; SI-NEXT:    v_cvt_f32_ubyte2_e32 v2, v4
-; SI-NEXT:    v_cvt_f32_ubyte1_e32 v1, v4
-; SI-NEXT:    v_cvt_f32_ubyte0_e32 v0, v4
-; SI-NEXT:    v_add_i32_e32 v4, vcc, 9, v4
-; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v0, s8, v4
-; SI-NEXT:    v_add_i32_e32 v2, vcc, 9, v5
-; SI-NEXT:    v_or_b32_e32 v0, v7, v0
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v6
-; SI-NEXT:    v_and_b32_e32 v2, s8, v2
-; SI-NEXT:    v_add_i32_e32 v0, vcc, 0x900, v0
-; SI-NEXT:    v_or_b32_e32 v1, v1, v2
-; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT:    v_or_b32_e32 v0, v1, v0
-; SI-NEXT:    v_add_i32_e32 v0, vcc, 0x9000000, v0
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
-; SI-NEXT:    s_endpgm
+; SI-NEXT:        s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SI-NEXT:        s_load_dwordx2 s[8:9], s[0:1], 0xb
+; SI-NEXT:        s_load_dwordx2 s[0:1], s[0:1], 0xd
+; SI-NEXT:        s_mov_b32 s11, 0xf000
+; SI-NEXT:        s_mov_b32 s2, 0
+; SI-NEXT:        s_mov_b32 s3, s11
+; SI-NEXT:        v_lshlrev_b32_e32 v0, 2, v0
+; SI-NEXT:        v_mov_b32_e32 v1, 0
+; SI-NEXT:        s_waitcnt lgkmcnt(0)
+; SI-NEXT:        buffer_load_dword v4, v[0:1], s[0:3], 0 addr64
+; SI-NEXT:        s_mov_b32 s10, -1
+; SI-NEXT:        s_movk_i32 s0, 0xff
+; SI-NEXT:        s_mov_b32 s6, s10
+; SI-NEXT:        s_mov_b32 s7, s11
+; SI-NEXT:        s_waitcnt vmcnt(0)
+; SI-NEXT:        v_lshrrev_b32_e32 v5, 16, v4
+; SI-NEXT:        v_lshrrev_b32_e32 v6, 24, v4
+; SI-NEXT:        v_and_b32_e32 v7, 0xff00, v4
+; SI-NEXT:        v_cvt_f32_ubyte3_e32 v3, v4
+; SI-NEXT:        v_cvt_f32_ubyte2_e32 v2, v4
+; SI-NEXT:        v_cvt_f32_ubyte1_e32 v1, v4
+; SI-NEXT:        v_cvt_f32_ubyte0_e32 v0, v4
+; SI-NEXT:        v_add_i32_e32 v4, vcc, 9, v4
+; SI-NEXT:        buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; SI-NEXT:        s_waitcnt expcnt(0)
+; SI-NEXT:        v_and_b32_e32 v0, s0, v4
+; SI-NEXT:        v_add_i32_e32 v2, vcc, 9, v5
+; SI-NEXT:        v_or_b32_e32 v0, v7, v0
+; SI-NEXT:        v_lshlrev_b32_e32 v1, 8, v6
+; SI-NEXT:        v_and_b32_e32 v2, s0, v2
+; SI-NEXT:        v_add_i32_e32 v0, vcc, 0x900, v0
+; SI-NEXT:        v_or_b32_e32 v1, v1, v2
+; SI-NEXT:        v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT:        v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT:        v_or_b32_e32 v0, v1, v0
+; SI-NEXT:        v_add_i32_e32 v0, vcc, 0x9000000, v0
+; SI-NEXT:        buffer_store_dword v0, off, s[8:11], 0
+; SI-NEXT:        s_endpgm
 ;
 ; VI-LABEL: load_v4i8_to_v4f32_2_uses:
 ; VI:       ; %bb.0:
-; VI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
-; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; VI-NEXT:    v_mov_b32_e32 v5, 9
-; VI-NEXT:    s_movk_i32 s8, 0x900
-; VI-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v1, s3
-; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v0
-; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT:    flat_load_dword v4, v[0:1]
-; VI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
-; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2c
-; VI-NEXT:    s_mov_b32 s3, 0xf000
-; VI-NEXT:    s_mov_b32 s2, -1
-; VI-NEXT:    s_mov_b32 s6, s2
-; VI-NEXT:    s_mov_b32 s7, s3
-; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT:    v_lshrrev_b32_e32 v6, 24, v4
-; VI-NEXT:    v_cvt_f32_ubyte3_e32 v3, v4
-; VI-NEXT:    v_cvt_f32_ubyte2_e32 v2, v4
-; VI-NEXT:    v_cvt_f32_ubyte1_e32 v1, v4
-; VI-NEXT:    v_cvt_f32_ubyte0_e32 v0, v4
-; VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
-; VI-NEXT:    v_and_b32_e32 v7, 0xffffff00, v4
-; VI-NEXT:    v_add_u16_e32 v8, 9, v4
-; VI-NEXT:    v_add_u16_sdwa v4, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v6
-; VI-NEXT:    v_or_b32_sdwa v0, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_or_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:    v_mov_b32_e32 v2, s8
-; VI-NEXT:    v_add_u16_e32 v0, s8, v0
-; VI-NEXT:    v_add_u16_sdwa v1, v1, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:    v_or_b32_e32 v0, v0, v1
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
-; VI-NEXT:    s_endpgm
+; VI-NEXT:        s_load_dwordx2 s[4:5], s[0:1], 0x24
+; VI-NEXT:        s_load_dwordx2 s[8:9], s[0:1], 0x2c
+; VI-NEXT:        s_load_dwordx2 s[0:1], s[0:1], 0x34
+; VI-NEXT:        v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT:        s_mov_b32 s11, 0xf000
+; VI-NEXT:        s_mov_b32 s10, -1
+; VI-NEXT:        v_mov_b32_e32 v5, 9
+; VI-NEXT:        s_waitcnt lgkmcnt(0)
+; VI-NEXT:        v_add_u32_e32 v0, vcc, s0, v0
+; VI-NEXT:        v_mov_b32_e32 v1, s1
+; VI-NEXT:        v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:        flat_load_dword v4, v[0:1]
+; VI-NEXT:        s_mov_b32 s6, s10
+; VI-NEXT:        s_mov_b32 s7, s11
+; VI-NEXT:        s_movk_i32 s0, 0x900
+; VI-NEXT:        s_waitcnt vmcnt(0) lgkmcnt(0)
+; VI-NEXT:        v_lshrrev_b32_e32 v6, 24, v4
+; VI-NEXT:        v_cvt_f32_ubyte3_e32 v3, v4
+; VI-NEXT:        v_cvt_f32_ubyte2_e32 v2, v4
+; VI-NEXT:        v_cvt_f32_ubyte1_e32 v1, v4
+; VI-NEXT:        v_cvt_f32_ubyte0_e32 v0, v4
+; VI-NEXT:        buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; VI-NEXT:        v_and_b32_e32 v7, 0xffffff00, v4
+; VI-NEXT:        v_add_u16_e32 v8, 9, v4
+; VI-NEXT:        v_add_u16_sdwa v4, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:        v_lshlrev_b16_e32 v1, 8, v6
+; VI-NEXT:        v_or_b32_sdwa v0, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:        v_or_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:        v_mov_b32_e32 v2, s0
+; VI-NEXT:        v_add_u16_e32 v0, s0, v0
+; VI-NEXT:        v_add_u16_sdwa v1, v1, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT:        v_or_b32_e32 v0, v0, v1
+; VI-NEXT:        buffer_store_dword v0, off, s[8:11], 0
+; VI-NEXT:        s_endpgm
   %tid.x = call i32 @llvm.amdgcn.workitem.id.x()
   %in.ptr = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %in, i32 %tid.x
   %load = load <4 x i8>, <4 x i8> addrspace(1)* %in.ptr, align 4
@@ -725,41 +725,42 @@ define amdgpu_kernel void @load_v4i8_to_v4f32_2_uses(<4 x float> addrspace(1)* n
 define amdgpu_kernel void @load_v7i8_to_v7f32(<7 x float> addrspace(1)* noalias %out, <7 x i8> addrspace(1)* noalias %in) nounwind {
 ; SI-LABEL: load_v7i8_to_v7f32:
 ; SI:       ; %bb.0:
-; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
-; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
-; SI-NEXT:    s_mov_b32 s7, 0xf000
-; SI-NEXT:    s_mov_b32 s2, 0
-; SI-NEXT:    s_mov_b32 s3, s7
-; SI-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
-; SI-NEXT:    v_mov_b32_e32 v1, 0
-; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    buffer_load_ubyte v2, v[0:1], s[0:3], 0 addr64
-; SI-NEXT:    buffer_load_ubyte v3, v[0:1], s[0:3], 0 addr64 offset:1
-; SI-NEXT:    buffer_load_ubyte v6, v[0:1], s[0:3], 0 addr64 offset:2
-; SI-NEXT:    buffer_load_ubyte v4, v[0:1], s[0:3], 0 addr64 offset:3
-; SI-NEXT:    buffer_load_ubyte v7, v[0:1], s[0:3], 0 addr64 offset:4
-; SI-NEXT:    buffer_load_ubyte v5, v[0:1], s[0:3], 0 addr64 offset:5
-; SI-NEXT:    buffer_load_ubyte v8, v[0:1], s[0:3], 0 addr64 offset:6
-; SI-NEXT:    s_mov_b32 s6, -1
-; SI-NEXT:    s_waitcnt vmcnt(6)
-; SI-NEXT:    v_cvt_f32_ubyte0_e32 v0, v2
-; SI-NEXT:    s_waitcnt vmcnt(5)
-; SI-NEXT:    v_cvt_f32_ubyte2_e32 v1, v3
-; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_lshlrev_b32_e32 v9, 8, v4
-; SI-NEXT:    v_or_b32_e32 v2, v9, v6
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_cvt_f32_ubyte0_e32 v4, v7
-; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_cvt_f32_ubyte0_e32 v7, v8
-; SI-NEXT:    v_cvt_f32_ubyte3_e32 v3, v2
-; SI-NEXT:    v_cvt_f32_ubyte2_e32 v5, v5
-; SI-NEXT:    v_cvt_f32_ubyte2_e32 v2, v2
-; SI-NEXT:    buffer_store_dword v7, off, s[4:7], 0 offset:24
-; SI-NEXT:    buffer_store_dwordx2 v[4:5], off, s[4:7], 0 offset:16
-; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
-; SI-NEXT:    s_endpgm
+; SI-NEXT:        s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SI-NEXT:        s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SI-NEXT:        s_mov_b32 s7, 0xf000
+; SI-NEXT:        s_mov_b32 s2, 0
+; SI-NEXT:        s_mov_b32 s3, s7
+; SI-NEXT:        v_lshlrev_b32_e32 v0, 3, v0
+; SI-NEXT:        v_mov_b32_e32 v1, 0
+; SI-NEXT:        s_waitcnt lgkmcnt(0)
+; SI-NEXT:        buffer_load_ubyte v2, v[0:1], s[0:3], 0 addr64
+; SI-NEXT:        buffer_load_ubyte v3, v[0:1], s[0:3], 0 addr64 offset:1
+; SI-NEXT:        buffer_load_ubyte v6, v[0:1], s[0:3], 0 addr64 offset:2
+; SI-NEXT:        buffer_load_ubyte v4, v[0:1], s[0:3], 0 addr64 offset:3
+; SI-NEXT:        buffer_load_ubyte v7, v[0:1], s[0:3], 0 addr64 offset:4
+; SI-NEXT:        buffer_load_ubyte v5, v[0:1], s[0:3], 0 addr64 offset:5
+; SI-NEXT:        buffer_load_ubyte v8, v[0:1], s[0:3], 0 addr64 offset:6
+; SI-NEXT:        s_mov_b32 s6, -1
+; SI-NEXT:        s_waitcnt vmcnt(6)
+; SI-NEXT:        v_cvt_f32_ubyte0_e32 v0, v2
+; SI-NEXT:        s_waitcnt vmcnt(5)
+; SI-NEXT:        v_cvt_f32_ubyte2_e32 v1, v3
+; SI-NEXT:        s_waitcnt vmcnt(3)
+; SI-NEXT:        v_lshlrev_b32_e32 v9, 8, v4
+; SI-NEXT:        v_or_b32_e32 v3, v9, v6
+; SI-NEXT:        s_waitcnt vmcnt(1)
+; SI-NEXT:        v_cvt_f32_ubyte2_e32 v5, v5
+; SI-NEXT:        s_waitcnt vmcnt(0)
+; SI-NEXT:        v_cvt_f32_ubyte0_e32 v2, v8
+; SI-NEXT:        buffer_store_dword v2, off, s[4:7], 0 offset:24
+; SI-NEXT:        s_waitcnt expcnt(0)
+; SI-NEXT:        v_lshlrev_b32_e32 v2, 16, v3
+; SI-NEXT:        v_cvt_f32_ubyte0_e32 v4, v7
+; SI-NEXT:        v_cvt_f32_ubyte3_e32 v3, v2
+; SI-NEXT:        v_cvt_f32_ubyte2_e32 v2, v2
+; SI-NEXT:        buffer_store_dwordx2 v[4:5], off, s[4:7], 0 offset:16
+; SI-NEXT:        buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; SI-NEXT:        s_endpgm
 ;
 ; VI-LABEL: load_v7i8_to_v7f32:
 ; VI:       ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/fshr.ll b/llvm/test/CodeGen/AMDGPU/fshr.ll
index 83b3b56172c92..444421443b4f2 100644
--- a/llvm/test/CodeGen/AMDGPU/fshr.ll
+++ b/llvm/test/CodeGen/AMDGPU/fshr.ll
@@ -1210,171 +1210,167 @@ define i24 @v_fshr_i24(i24 %src0, i24 %src1, i24 %src2) {
 define <2 x i24> @v_fshr_v2i24(<2 x i24> %src0, <2 x i24> %src1, <2 x i24> %src2) {
 ; SI-LABEL: v_fshr_v2i24:
 ; SI:       ; %bb.0:
-; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:8
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:16
-; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:20
-; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:12
-; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32
-; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:4
-; SI-NEXT:    s_mov_b32 s4, 0xffffff
-; SI-NEXT:    s_mov_b32 s5, 0xaaaaaaab
-; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v0
-; SI-NEXT:    v_add_i32_e32 v8, vcc, 4, v0
-; SI-NEXT:    v_add_i32_e32 v9, vcc, 5, v0
-; SI-NEXT:    v_add_i32_e32 v10, vcc, 2, v0
-; SI-NEXT:    s_waitcnt vmcnt(5)
-; SI-NEXT:    v_and_b32_e32 v14, s4, v1
-; SI-NEXT:    s_waitcnt vmcnt(4)
-; SI-NEXT:    v_and_b32_e32 v2, s4, v2
-; SI-NEXT:    v_mul_hi_u32 v12, v2, s5
-; SI-NEXT:    s_waitcnt vmcnt(3)
-; SI-NEXT:    v_and_b32_e32 v3, s4, v3
-; SI-NEXT:    v_mul_hi_u32 v13, v3, s5
-; SI-NEXT:    s_waitcnt vmcnt(2)
-; SI-NEXT:    v_and_b32_e32 v11, s4, v4
-; SI-NEXT:    v_lshrrev_b32_e32 v12, 4, v12
-; SI-NEXT:    v_mul_lo_u32 v12, v12, 24
-; SI-NEXT:    v_lshrrev_b32_e32 v13, 4, v13
-; SI-NEXT:    v_mul_lo_u32 v13, v13, 24
-; SI-NEXT:    v_sub_i32_e32 v2, vcc, v2, v12
-; SI-NEXT:    v_lshr_b32_e32 v12, v14, v2
-; SI-NEXT:    v_sub_i32_e32 v3, vcc, v3, v13
-; SI-NEXT:    v_sub_i32_e32 v13, vcc, 24, v2
-; SI-NEXT:    v_sub_i32_e32 v14, vcc, 24, v3
-; SI-NEXT:    v_and_b32_e32 v13, s4, v13
-; SI-NEXT:    s_waitcnt vmcnt(1)
-; SI-NEXT:    v_lshl_b32_e32 v5, v5, v13
-; SI-NEXT:    v_and_b32_e32 v14, 0xffffff, v14
-; SI-NEXT:    v_lshr_b32_e32 v11, v11, v3
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_lshl_b32_e32 v6, v6, v14
-; SI-NEXT:    v_or_b32_e32 v5, v5, v12
-; SI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
-; SI-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; SI-NEXT:    v_or_b32_e32 v6, v6, v11
-; SI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; SI-NEXT:    v_cndmask_b32_e32 v2, v6, v4, vcc
-; SI-NEXT:    buffer_store_byte v2, v7, s[0:3], 0 offen
-; SI-NEXT:    buffer_store_short v1, v0, s[0:3], 0 offen
-; SI-NEXT:    v_lshrrev_b32_e32 v0, 8, v2
-; SI-NEXT:    s_waitcnt expcnt(1)
-; SI-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT:    buffer_store_byte v0, v8, s[0:3], 0 offen
-; SI-NEXT:    buffer_store_byte v2, v9, s[0:3], 0 offen
-; SI-NEXT:    buffer_store_byte v1, v10, s[0:3], 0 offen
-; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
-; SI-NEXT:    s_setpc_b64 s[30:31]
+; SI-NEXT:	s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT:	buffer_load_dword v1, off, s[0:3], s32 offset:8
+; SI-NEXT:	buffer_load_dword v2, off, s[0:3], s32 offset:16
+; SI-NEXT:	buffer_load_dword v3, off, s[0:3], s32 offset:20
+; SI-NEXT:	buffer_load_dword v4, off, s[0:3], s32 offset:4
+; SI-NEXT:	buffer_load_dword v5, off, s[0:3], s32 offset:12
+; SI-NEXT:	buffer_load_dword v6, off, s[0:3], s32
+; SI-NEXT:	s_mov_b32 s4, 0xffffff
+; SI-NEXT:	s_mov_b32 s5, 0xaaaaaaab
+; SI-NEXT:	v_add_i32_e32 v7, vcc, 3, v0
+; SI-NEXT:	v_add_i32_e32 v8, vcc, 4, v0
+; SI-NEXT:	v_add_i32_e32 v9, vcc, 5, v0
+; SI-NEXT:	v_add_i32_e32 v10, vcc, 2, v0
+; SI-NEXT:	s_waitcnt vmcnt(5)
+; SI-NEXT:	v_and_b32_e32 v14, s4, v1
+; SI-NEXT:	s_waitcnt vmcnt(4)
+; SI-NEXT:	v_and_b32_e32 v2, s4, v2
+; SI-NEXT:	v_mul_hi_u32 v12, v2, s5
+; SI-NEXT:	s_waitcnt vmcnt(3)
+; SI-NEXT:	v_and_b32_e32 v3, s4, v3
+; SI-NEXT:	v_mul_hi_u32 v13, v3, s5
+; SI-NEXT:	s_waitcnt vmcnt(1)
+; SI-NEXT:	v_and_b32_e32 v11, s4, v5
+; SI-NEXT:	v_lshrrev_b32_e32 v12, 4, v12
+; SI-NEXT:	v_mul_lo_u32 v12, v12, 24
+; SI-NEXT:	v_lshrrev_b32_e32 v13, 4, v13
+; SI-NEXT:	v_mul_lo_u32 v13, v13, 24
+; SI-NEXT:	v_sub_i32_e32 v2, vcc, v2, v12
+; SI-NEXT:	v_lshr_b32_e32 v12, v14, v2
+; SI-NEXT:	v_sub_i32_e32 v3, vcc, v3, v13
+; SI-NEXT:	v_sub_i32_e32 v13, vcc, 24, v2
+; SI-NEXT:	v_sub_i32_e32 v14, vcc, 24, v3
+; SI-NEXT:	v_and_b32_e32 v13, s4, v13
+; SI-NEXT:	s_waitcnt vmcnt(0)
+; SI-NEXT:	v_lshl_b32_e32 v6, v6, v13
+; SI-NEXT:	v_and_b32_e32 v14, 0xffffff, v14
+; SI-NEXT:	v_lshr_b32_e32 v11, v11, v3
+; SI-NEXT:	v_lshl_b32_e32 v4, v4, v14
+; SI-NEXT:	v_or_b32_e32 v6, v6, v12
+; SI-NEXT:	v_cmp_eq_u32_e32 vcc, 0, v2
+; SI-NEXT:	v_cndmask_b32_e32 v1, v6, v1, vcc
+; SI-NEXT:	v_or_b32_e32 v4, v4, v11
+; SI-NEXT:	v_cmp_eq_u32_e32 vcc, 0, v3
+; SI-NEXT:	v_cndmask_b32_e32 v2, v4, v5, vcc
+; SI-NEXT:	buffer_store_byte v2, v7, s[0:3], 0 offen
+; SI-NEXT:	buffer_store_short v1, v0, s[0:3], 0 offen
+; SI-NEXT:	v_lshrrev_b32_e32 v0, 8, v2
+; SI-NEXT:	s_waitcnt expcnt(1)
+; SI-NEXT:	v_lshrrev_b32_e32 v2, 16, v2
+; SI-NEXT:	s_waitcnt expcnt(0)
+; SI-NEXT:	v_lshrrev_b32_e32 v1, 16, v1
+; SI-NEXT:	buffer_store_byte v0, v8, s[0:3], 0 offen
+; SI-NEXT:	buffer_store_byte v2, v9, s[0:3], 0 offen
+; SI-NEXT:	buffer_store_byte v1, v10, s[0:3], 0 offen
+; SI-NEXT:	s_waitcnt vmcnt(0) expcnt(0)
+; SI-NEXT:	s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: v_fshr_v2i24:
 ; VI:       ; %bb.0:
-; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:8
-; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:16
-; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:20
-; VI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:12
-; VI-NEXT:    buffer_load_dword v5, off, s[0:3], s32
-; VI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 offset:4
-; VI-NEXT:    s_mov_b32 s4, 0xffffff
-; VI-NEXT:    s_mov_b32 s5, 0xaaaaaaab
-; VI-NEXT:    v_add_u32_e32 v7, vcc, 3, v0
-; VI-NEXT:    v_add_u32_e32 v8, vcc, 4, v0
-; VI-NEXT:    v_add_u32_e32 v9, vcc, 5, v0
-; VI-NEXT:    v_add_u32_e32 v10, vcc, 2, v0
-; VI-NEXT:    s_waitcnt vmcnt(5)
-; VI-NEXT:    v_and_b32_e32 v14, s4, v1
-; VI-NEXT:    s_waitcnt vmcnt(4)
-; VI-NEXT:    v_and_b32_e32 v2, s4, v2
-; VI-NEXT:    v_mul_hi_u32 v12, v2, s5
-; VI-NEXT:    s_waitcnt vmcnt(3)
-; VI-NEXT:    v_and_b32_e32 v3, s4, v3
-; VI-NEXT:    v_mul_hi_u32 v13, v3, s5
-; VI-NEXT:    s_waitcnt vmcnt(2)
-; VI-NEXT:    v_and_b32_e32 v11, s4, v4
-; VI-NEXT:    v_lshrrev_b32_e32 v12, 4, v12
-; VI-NEXT:    v_mul_lo_u32 v12, v12, 24
-; VI-NEXT:    v_lshrrev_b32_e32 v13, 4, v13
-; VI-NEXT:    v_mul_lo_u32 v13, v13, 24
-; VI-NEXT:    v_sub_u32_e32 v2, vcc, v2, v12
-; VI-NEXT:    v_lshrrev_b32_e32 v12, v2, v14
-; VI-NEXT:    v_sub_u32_e32 v3, vcc, v3, v13
-; VI-NEXT:    v_sub_u32_e32 v13, vcc, 24, v2
-; VI-NEXT:    v_sub_u32_e32 v14, vcc, 24, v3
-; VI-NEXT:    v_and_b32_e32 v13, s4, v13
-; VI-NEXT:    s_waitcnt vmcnt(1)
-; VI-NEXT:    v_lshlrev_b32_e32 v5, v13, v5
-; VI-NEXT:    v_and_b32_e32 v14, 0xffffff, v14
-; VI-NEXT:    v_lshrrev_b32_e32 v11, v3, v11
-; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_lshlrev_b32_e32 v6, v14, v6
-; VI-NEXT:    v_or_b32_e32 v5, v5, v12
-; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
-; VI-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; VI-NEXT:    v_or_b32_e32 v6, v6, v11
-; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; VI-NEXT:    v_cndmask_b32_e32 v2, v6, v4, vcc
-; VI-NEXT:    buffer_store_byte v2, v7, s[0:3], 0 offen
-; VI-NEXT:    buffer_store_short v1, v0, s[0:3], 0 offen
-; VI-NEXT:    v_lshrrev_b32_e32 v0, 8, v2
-; VI-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
-; VI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; VI-NEXT:    buffer_store_byte v0, v8, s[0:3], 0 offen
-; VI-NEXT:    buffer_store_byte v2, v9, s[0:3], 0 offen
-; VI-NEXT:    buffer_store_byte v1, v10, s[0:3], 0 offen
-; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    s_setpc_b64 s[30:31]
+; VI-NEXT:	s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:	buffer_load_dword v1, off, s[0:3], s32 offset:8
+; VI-NEXT:	buffer_load_dword v2, off, s[0:3], s32 offset:16
+; VI-NEXT:	buffer_load_dword v3, off, s[0:3], s32 offset:20
+; VI-NEXT:	buffer_load_dword v4, off, s[0:3], s32 offset:4
+; VI-NEXT:	buffer_load_dword v5, off, s[0:3], s32 offset:12
+; VI-NEXT:	buffer_load_dword v6, off, s[0:3], s32
+; VI-NEXT:	s_mov_b32 s4, 0xffffff
+; VI-NEXT:	s_mov_b32 s5, 0xaaaaaaab
+; VI-NEXT:	v_add_u32_e32 v7, vcc, 3, v0
+; VI-NEXT:	v_add_u32_e32 v8, vcc, 4, v0
+; VI-NEXT:	v_add_u32_e32 v9, vcc, 5, v0
+; VI-NEXT:	v_add_u32_e32 v10, vcc, 2, v0
+; VI-NEXT:	s_waitcnt vmcnt(5)
+; VI-NEXT:	v_and_b32_e32 v14, s4, v1
+; VI-NEXT:	s_waitcnt vmcnt(4)
+; VI-NEXT:	v_and_b32_e32 v2, s4, v2
+; VI-NEXT:	v_mul_hi_u32 v12, v2, s5
+; VI-NEXT:	s_waitcnt vmcnt(3)
+; VI-NEXT:	v_and_b32_e32 v3, s4, v3
+; VI-NEXT:	v_mul_hi_u32 v13, v3, s5
+; VI-NEXT:	s_waitcnt vmcnt(1)
+; VI-NEXT:	v_and_b32_e32 v11, s4, v5
+; VI-NEXT:	v_lshrrev_b32_e32 v12, 4, v12
+; VI-NEXT:	v_mul_lo_u32 v12, v12, 24
+; VI-NEXT:	v_lshrrev_b32_e32 v13, 4, v13
+; VI-NEXT:	v_mul_lo_u32 v13, v13, 24
+; VI-NEXT:	v_sub_u32_e32 v2, vcc, v2, v12
+; VI-NEXT:	v_lshrrev_b32_e32 v12, v2, v14
+; VI-NEXT:	v_sub_u32_e32 v3, vcc, v3, v13
+; VI-NEXT:	v_sub_u32_e32 v13, vcc, 24, v2
+; VI-NEXT:	v_sub_u32_e32 v14, vcc, 24, v3
+; VI-NEXT:	v_and_b32_e32 v13, s4, v13
+; VI-NEXT:	s_waitcnt vmcnt(0)
+; VI-NEXT:	v_lshlrev_b32_e32 v6, v13, v6
+; VI-NEXT:	v_and_b32_e32 v14, 0xffffff, v14
+; VI-NEXT:	v_lshrrev_b32_e32 v11, v3, v11
+; VI-NEXT:	v_lshlrev_b32_e32 v4, v14, v4
+; VI-NEXT:	v_or_b32_e32 v6, v6, v12
+; VI-NEXT:	v_cmp_eq_u32_e32 vcc, 0, v2
+; VI-NEXT:	v_cndmask_b32_e32 v1, v6, v1, vcc
+; VI-NEXT:	v_or_b32_e32 v4, v4, v11
+; VI-NEXT:	v_cmp_eq_u32_e32 vcc, 0, v3
+; VI-NEXT:	v_cndmask_b32_e32 v2, v4, v5, vcc
+; VI-NEXT:	buffer_store_byte v2, v7, s[0:3], 0 offen
+; VI-NEXT:	buffer_store_short v1, v0, s[0:3], 0 offen
+; VI-NEXT:	v_lshrrev_b32_e32 v0, 8, v2
+; VI-NEXT:	v_lshrrev_b32_e32 v2, 16, v2
+; VI-NEXT:	v_lshrrev_b32_e32 v1, 16, v1
+; VI-NEXT:	buffer_store_byte v0, v8, s[0:3], 0 offen
+; VI-NEXT:	buffer_store_byte v2, v9, s[0:3], 0 offen
+; VI-NEXT:	buffer_store_byte v1, v10, s[0:3], 0 offen
+; VI-NEXT:	s_waitcnt vmcnt(0)
+; VI-NEXT:	s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_fshr_v2i24:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:8
-; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:16
-; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:20
-; GFX9-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:12
-; GFX9-NEXT:    buffer_load_dword v5, off, s[0:3], s32
-; GFX9-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:4
-; GFX9-NEXT:    s_mov_b32 s4, 0xffffff
-; GFX9-NEXT:    s_mov_b32 s5, 0xaaaaaaab
-; GFX9-NEXT:    s_waitcnt vmcnt(5)
-; GFX9-NEXT:    v_and_b32_e32 v10, s4, v1
-; GFX9-NEXT:    s_waitcnt vmcnt(4)
-; GFX9-NEXT:    v_and_b32_e32 v2, s4, v2
-; GFX9-NEXT:    v_mul_hi_u32 v6, v2, s5
-; GFX9-NEXT:    s_waitcnt vmcnt(3)
-; GFX9-NEXT:    v_and_b32_e32 v3, s4, v3
-; GFX9-NEXT:    v_mul_hi_u32 v7, v3, s5
-; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    v_and_b32_e32 v9, s4, v4
-; GFX9-NEXT:    v_lshrrev_b32_e32 v6, 4, v6
-; GFX9-NEXT:    v_mul_lo_u32 v6, v6, 24
-; GFX9-NEXT:    v_lshrrev_b32_e32 v7, 4, v7
-; GFX9-NEXT:    v_mul_lo_u32 v7, v7, 24
-; GFX9-NEXT:    v_sub_u32_e32 v2, v2, v6
-; GFX9-NEXT:    v_lshrrev_b32_e32 v6, v2, v10
-; GFX9-NEXT:    v_sub_u32_e32 v3, v3, v7
-; GFX9-NEXT:    v_sub_u32_e32 v7, 24, v2
-; GFX9-NEXT:    v_sub_u32_e32 v10, 24, v3
-; GFX9-NEXT:    v_and_b32_e32 v7, s4, v7
-; GFX9-NEXT:    v_lshrrev_b32_e32 v9, v3, v9
-; GFX9-NEXT:    v_and_b32_e32 v10, 0xffffff, v10
-; GFX9-NEXT:    s_waitcnt vmcnt(1)
-; GFX9-NEXT:    v_lshl_or_b32 v5, v5, v7, v6
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_lshl_or_b32 v6, v8, v10, v9
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v6, v4, vcc
-; GFX9-NEXT:    v_lshrrev_b32_e32 v3, 8, v2
-; GFX9-NEXT:    buffer_store_byte_d16_hi v2, v0, s[0:3], 0 offen offset:5
-; GFX9-NEXT:    buffer_store_byte v3, v0, s[0:3], 0 offen offset:4
-; GFX9-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:3
-; GFX9-NEXT:    buffer_store_byte_d16_hi v1, v0, s[0:3], 0 offen offset:2
-; GFX9-NEXT:    buffer_store_short v1, v0, s[0:3], 0 offen
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    s_setpc_b64 s[30:31]
+; GFX9-NEXT:        s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:        buffer_load_dword v1, off, s[0:3], s32 offset:16
+; GFX9-NEXT:        buffer_load_dword v2, off, s[0:3], s32 offset:20
+; GFX9-NEXT:        buffer_load_dword v3, off, s[0:3], s32 offset:4
+; GFX9-NEXT:        buffer_load_dword v4, off, s[0:3], s32 offset:12
+; GFX9-NEXT:        buffer_load_dword v5, off, s[0:3], s32
+; GFX9-NEXT:        buffer_load_dword v8, off, s[0:3], s32 offset:8
+; GFX9-NEXT:        s_mov_b32 s4, 0xffffff
+; GFX9-NEXT:        s_mov_b32 s5, 0xaaaaaaab
+; GFX9-NEXT:        s_waitcnt vmcnt(5)
+; GFX9-NEXT:        v_and_b32_e32 v1, s4, v1
+; GFX9-NEXT:        v_mul_hi_u32 v6, v1, s5
+; GFX9-NEXT:        s_waitcnt vmcnt(4)
+; GFX9-NEXT:        v_and_b32_e32 v2, s4, v2
+; GFX9-NEXT:        v_mul_hi_u32 v7, v2, s5
+; GFX9-NEXT:        s_waitcnt vmcnt(2)
+; GFX9-NEXT:        v_and_b32_e32 v9, s4, v4
+; GFX9-NEXT:        v_lshrrev_b32_e32 v6, 4, v6
+; GFX9-NEXT:        v_mul_lo_u32 v6, v6, 24
+; GFX9-NEXT:        v_lshrrev_b32_e32 v7, 4, v7
+; GFX9-NEXT:        v_mul_lo_u32 v7, v7, 24
+; GFX9-NEXT:        s_waitcnt vmcnt(0)
+; GFX9-NEXT:        v_and_b32_e32 v10, s4, v8
+; GFX9-NEXT:        v_sub_u32_e32 v1, v1, v6
+; GFX9-NEXT:        v_lshrrev_b32_e32 v6, v1, v10
+; GFX9-NEXT:        v_sub_u32_e32 v2, v2, v7
+; GFX9-NEXT:        v_sub_u32_e32 v7, 24, v1
+; GFX9-NEXT:        v_sub_u32_e32 v10, 24, v2
+; GFX9-NEXT:        v_and_b32_e32 v7, s4, v7
+; GFX9-NEXT:        v_lshrrev_b32_e32 v9, v2, v9
+; GFX9-NEXT:        v_and_b32_e32 v10, 0xffffff, v10
+; GFX9-NEXT:        v_lshl_or_b32 v5, v5, v7, v6
+; GFX9-NEXT:        v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-NEXT:        v_cndmask_b32_e32 v1, v5, v8, vcc
+; GFX9-NEXT:        v_lshl_or_b32 v3, v3, v10, v9
+; GFX9-NEXT:        v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-NEXT:        v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT:        v_lshrrev_b32_e32 v3, 8, v2
+; GFX9-NEXT:        buffer_store_byte_d16_hi v2, v0, s[0:3], 0 offen offset:5
+; GFX9-NEXT:        buffer_store_byte v3, v0, s[0:3], 0 offen offset:4
+; GFX9-NEXT:        buffer_store_byte v2, v0, s[0:3], 0 offen offset:3
+; GFX9-NEXT:        buffer_store_byte_d16_hi v1, v0, s[0:3], 0 offen offset:2
+; GFX9-NEXT:        buffer_store_short v1, v0, s[0:3], 0 offen
+; GFX9-NEXT:        s_waitcnt vmcnt(0)
+; GFX9-NEXT:        s_setpc_b64 s[30:31]
 ;
 ; R600-LABEL: v_fshr_v2i24:
 ; R600:       ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/indirect-addressing-si.ll b/llvm/test/CodeGen/AMDGPU/indirect-addressing-si.ll
index bca00f69e25cd..bfc42e48bca86 100644
--- a/llvm/test/CodeGen/AMDGPU/indirect-addressing-si.ll
+++ b/llvm/test/CodeGen/AMDGPU/indirect-addressing-si.ll
@@ -273,8 +273,8 @@ entry:
 ; MOVREL: s_add_i32 m0, s{{[0-9]+}}, 0xfffffe{{[0-9a-z]+}}
 ; MOVREL: v_movreld_b32_e32 v0, 5
 
-; IDXMODE: s_addk_i32 [[ADD_IDX:s[0-9]+]], 0xfe00{{$}}
-; IDXMODE: s_set_gpr_idx_on [[ADD_IDX]], gpr_idx(DST)
+; IDXMODE: s_add_i32 s{{[0-9]+}}, s{{[0-9]+}}, 0xfffffe00{{$}}
+; IDXMODE: s_set_gpr_idx_on s{{[0-9]+}}, gpr_idx(DST)
 ; IDXMODE-NEXT: v_mov_b32_e32 v0, 5
 ; IDXMODE-NEXT: s_set_gpr_idx_off
 define amdgpu_kernel void @insert_neg_offset_sgpr_loadreg(i32 addrspace(1)* %in, <16 x i32> addrspace(1)* %out, <16 x i32> %vec, i32 %offset) {
diff --git a/llvm/test/CodeGen/AMDGPU/kernel-args.ll b/llvm/test/CodeGen/AMDGPU/kernel-args.ll
index 72f8c9cfc55ff..9aee862dbe119 100644
--- a/llvm/test/CodeGen/AMDGPU/kernel-args.ll
+++ b/llvm/test/CodeGen/AMDGPU/kernel-args.ll
@@ -855,10 +855,10 @@ define amdgpu_kernel void @struct_argument_alignment({i32, i64} %arg0, i8, {i32,
 ; multiple.
 ; FUNC-LABEL: {{^}}packed_struct_argument_alignment:
 ; HSA-GFX9: kernarg_segment_byte_size = 28
-; HSA-GFX9: global_load_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, off offset:17
-; HSA-GFX9: global_load_dword v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, off offset:13
 ; HSA-GFX9: s_load_dword s{{[0-9]+}}, s[4:5], 0x0
 ; HSA-GFX9: s_load_dwordx2 s{{\[[0-9]+:[0-9]+\]}}, s[4:5], 0x4
+; HSA-GFX9: global_load_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, off offset:17
+; HSA-GFX9: global_load_dword v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, off offset:13
 define amdgpu_kernel void @packed_struct_argument_alignment(<{i32, i64}> %arg0, i8, <{i32, i64}> %arg1) {
   %val0 = extractvalue <{i32, i64}> %arg0, 0
   %val1 = extractvalue <{i32, i64}> %arg0, 1
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.round.f64.ll b/llvm/test/CodeGen/AMDGPU/llvm.round.f64.ll
index 76a218760e8e8..667425d86c692 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.round.f64.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.round.f64.ll
@@ -237,157 +237,157 @@ define amdgpu_kernel void @round_v2f64(<2 x double> addrspace(1)* %out, <2 x dou
 define amdgpu_kernel void @round_v4f64(<4 x double> addrspace(1)* %out, <4 x double> %in) #0 {
 ; SI-LABEL: round_v4f64:
 ; SI:       ; %bb.0:
-; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
-; SI-NEXT:    s_load_dwordx8 s[8:15], s[0:1], 0x11
-; SI-NEXT:    s_mov_b32 s6, -1
-; SI-NEXT:    s_movk_i32 s18, 0xfc01
-; SI-NEXT:    s_mov_b32 s3, 0xfffff
-; SI-NEXT:    s_mov_b32 s2, s6
-; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    s_bfe_u32 s0, s11, 0xb0014
-; SI-NEXT:    s_add_i32 s19, s0, s18
-; SI-NEXT:    s_lshr_b64 s[0:1], s[2:3], s19
-; SI-NEXT:    s_brev_b32 s20, 1
-; SI-NEXT:    s_andn2_b64 s[16:17], s[10:11], s[0:1]
-; SI-NEXT:    s_and_b32 s0, s11, s20
-; SI-NEXT:    v_mov_b32_e32 v1, s0
-; SI-NEXT:    v_mov_b32_e32 v0, s17
-; SI-NEXT:    v_cmp_lt_i32_e64 vcc, s19, 0
-; SI-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; SI-NEXT:    v_mov_b32_e32 v1, s11
-; SI-NEXT:    v_cmp_gt_i32_e64 s[0:1], s19, 51
-; SI-NEXT:    v_cndmask_b32_e64 v1, v0, v1, s[0:1]
-; SI-NEXT:    v_mov_b32_e32 v0, s16
-; SI-NEXT:    v_cndmask_b32_e64 v0, v0, 0, vcc
-; SI-NEXT:    v_mov_b32_e32 v2, s10
-; SI-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
-; SI-NEXT:    v_add_f64 v[2:3], s[10:11], -v[0:1]
-; SI-NEXT:    s_bfe_u32 s0, s9, 0xb0014
-; SI-NEXT:    s_add_i32 s17, s0, s18
-; SI-NEXT:    v_cmp_ge_f64_e64 vcc, |v[2:3]|, 0.5
-; SI-NEXT:    s_brev_b32 s16, -2
-; SI-NEXT:    v_mov_b32_e32 v12, 0x3ff00000
-; SI-NEXT:    v_mov_b32_e32 v4, s11
-; SI-NEXT:    v_bfi_b32 v4, s16, v12, v4
-; SI-NEXT:    s_lshr_b64 s[0:1], s[2:3], s17
-; SI-NEXT:    v_cndmask_b32_e32 v3, 0, v4, vcc
-; SI-NEXT:    v_mov_b32_e32 v2, 0
-; SI-NEXT:    s_andn2_b64 s[10:11], s[8:9], s[0:1]
-; SI-NEXT:    s_and_b32 s0, s9, s20
-; SI-NEXT:    v_add_f64 v[2:3], v[0:1], v[2:3]
-; SI-NEXT:    v_mov_b32_e32 v1, s0
-; SI-NEXT:    v_mov_b32_e32 v0, s11
-; SI-NEXT:    v_cmp_lt_i32_e64 vcc, s17, 0
-; SI-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; SI-NEXT:    v_mov_b32_e32 v1, s9
-; SI-NEXT:    v_cmp_gt_i32_e64 s[0:1], s17, 51
-; SI-NEXT:    v_cndmask_b32_e64 v1, v0, v1, s[0:1]
-; SI-NEXT:    v_mov_b32_e32 v0, s10
-; SI-NEXT:    v_cndmask_b32_e64 v0, v0, 0, vcc
-; SI-NEXT:    v_mov_b32_e32 v4, s8
-; SI-NEXT:    v_cndmask_b32_e64 v0, v0, v4, s[0:1]
-; SI-NEXT:    v_add_f64 v[4:5], s[8:9], -v[0:1]
-; SI-NEXT:    s_bfe_u32 s0, s15, 0xb0014
-; SI-NEXT:    s_add_i32 s10, s0, s18
-; SI-NEXT:    v_mov_b32_e32 v6, s9
-; SI-NEXT:    s_lshr_b64 s[0:1], s[2:3], s10
-; SI-NEXT:    v_cmp_ge_f64_e64 vcc, |v[4:5]|, 0.5
-; SI-NEXT:    s_andn2_b64 s[8:9], s[14:15], s[0:1]
-; SI-NEXT:    v_bfi_b32 v6, s16, v12, v6
-; SI-NEXT:    s_and_b32 s0, s15, s20
-; SI-NEXT:    v_cndmask_b32_e32 v9, 0, v6, vcc
-; SI-NEXT:    v_mov_b32_e32 v5, s0
-; SI-NEXT:    v_mov_b32_e32 v4, s9
-; SI-NEXT:    v_cmp_lt_i32_e64 vcc, s10, 0
-; SI-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc
-; SI-NEXT:    v_mov_b32_e32 v5, s15
-; SI-NEXT:    v_cmp_gt_i32_e64 s[0:1], s10, 51
-; SI-NEXT:    v_cndmask_b32_e64 v5, v4, v5, s[0:1]
-; SI-NEXT:    v_mov_b32_e32 v4, s8
-; SI-NEXT:    v_cndmask_b32_e64 v4, v4, 0, vcc
-; SI-NEXT:    v_mov_b32_e32 v6, s14
-; SI-NEXT:    v_cndmask_b32_e64 v4, v4, v6, s[0:1]
-; SI-NEXT:    v_add_f64 v[6:7], s[14:15], -v[4:5]
-; SI-NEXT:    s_bfe_u32 s0, s13, 0xb0014
-; SI-NEXT:    v_mov_b32_e32 v10, s15
-; SI-NEXT:    s_add_i32 s8, s0, s18
-; SI-NEXT:    v_cmp_ge_f64_e64 vcc, |v[6:7]|, 0.5
-; SI-NEXT:    s_lshr_b64 s[0:1], s[2:3], s8
-; SI-NEXT:    v_bfi_b32 v10, s16, v12, v10
-; SI-NEXT:    v_cndmask_b32_e32 v7, 0, v10, vcc
-; SI-NEXT:    v_mov_b32_e32 v6, 0
-; SI-NEXT:    s_andn2_b64 s[2:3], s[12:13], s[0:1]
-; SI-NEXT:    s_and_b32 s0, s13, s20
-; SI-NEXT:    v_add_f64 v[6:7], v[4:5], v[6:7]
-; SI-NEXT:    v_mov_b32_e32 v5, s0
-; SI-NEXT:    v_mov_b32_e32 v4, s3
-; SI-NEXT:    v_cmp_lt_i32_e64 vcc, s8, 0
-; SI-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc
-; SI-NEXT:    v_mov_b32_e32 v5, s13
-; SI-NEXT:    v_cmp_gt_i32_e64 s[0:1], s8, 51
-; SI-NEXT:    v_cndmask_b32_e64 v5, v4, v5, s[0:1]
-; SI-NEXT:    v_mov_b32_e32 v4, s2
-; SI-NEXT:    v_cndmask_b32_e64 v4, v4, 0, vcc
-; SI-NEXT:    v_mov_b32_e32 v10, s12
-; SI-NEXT:    v_cndmask_b32_e64 v4, v4, v10, s[0:1]
-; SI-NEXT:    v_add_f64 v[10:11], s[12:13], -v[4:5]
-; SI-NEXT:    v_mov_b32_e32 v13, s13
-; SI-NEXT:    v_cmp_ge_f64_e64 vcc, |v[10:11]|, 0.5
-; SI-NEXT:    v_bfi_b32 v12, s16, v12, v13
-; SI-NEXT:    v_cndmask_b32_e32 v11, 0, v12, vcc
-; SI-NEXT:    v_mov_b32_e32 v10, 0
-; SI-NEXT:    v_mov_b32_e32 v8, 0
-; SI-NEXT:    v_add_f64 v[4:5], v[4:5], v[10:11]
-; SI-NEXT:    s_mov_b32 s7, 0xf000
-; SI-NEXT:    v_add_f64 v[0:1], v[0:1], v[8:9]
-; SI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[4:7], 0 offset:16
-; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
-; SI-NEXT:    s_endpgm
+; SI-NEXT:	s_load_dwordx8 s[8:15], s[0:1], 0x11
+; SI-NEXT:	s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SI-NEXT:	s_mov_b32 s6, -1
+; SI-NEXT:	s_movk_i32 s18, 0xfc01
+; SI-NEXT:	s_mov_b32 s3, 0xfffff
+; SI-NEXT:	s_waitcnt lgkmcnt(0)
+; SI-NEXT:	s_bfe_u32 s0, s11, 0xb0014
+; SI-NEXT:	s_add_i32 s19, s0, s18
+; SI-NEXT:	s_mov_b32 s2, s6
+; SI-NEXT:	s_lshr_b64 s[0:1], s[2:3], s19
+; SI-NEXT:	s_brev_b32 s20, 1
+; SI-NEXT:	s_andn2_b64 s[16:17], s[10:11], s[0:1]
+; SI-NEXT:	s_and_b32 s0, s11, s20
+; SI-NEXT:	v_mov_b32_e32 v1, s0
+; SI-NEXT:	v_mov_b32_e32 v0, s17
+; SI-NEXT:	v_cmp_lt_i32_e64 vcc, s19, 0
+; SI-NEXT:	v_cndmask_b32_e32 v0, v0, v1, vcc
+; SI-NEXT:	v_mov_b32_e32 v1, s11
+; SI-NEXT:	v_cmp_gt_i32_e64 s[0:1], s19, 51
+; SI-NEXT:	v_cndmask_b32_e64 v1, v0, v1, s[0:1]
+; SI-NEXT:	v_mov_b32_e32 v0, s16
+; SI-NEXT:	v_cndmask_b32_e64 v0, v0, 0, vcc
+; SI-NEXT:	v_mov_b32_e32 v2, s10
+; SI-NEXT:	v_cndmask_b32_e64 v0, v0, v2, s[0:1]
+; SI-NEXT:	v_add_f64 v[2:3], s[10:11], -v[0:1]
+; SI-NEXT:	s_bfe_u32 s0, s9, 0xb0014
+; SI-NEXT:	s_add_i32 s17, s0, s18
+; SI-NEXT:	v_cmp_ge_f64_e64 vcc, |v[2:3]|, 0.5
+; SI-NEXT:	s_brev_b32 s16, -2
+; SI-NEXT:	v_mov_b32_e32 v12, 0x3ff00000
+; SI-NEXT:	v_mov_b32_e32 v4, s11
+; SI-NEXT:	v_bfi_b32 v4, s16, v12, v4
+; SI-NEXT:	s_lshr_b64 s[0:1], s[2:3], s17
+; SI-NEXT:	v_cndmask_b32_e32 v3, 0, v4, vcc
+; SI-NEXT:	v_mov_b32_e32 v2, 0
+; SI-NEXT:	s_andn2_b64 s[10:11], s[8:9], s[0:1]
+; SI-NEXT:	s_and_b32 s0, s9, s20
+; SI-NEXT:	v_add_f64 v[2:3], v[0:1], v[2:3]
+; SI-NEXT:	v_mov_b32_e32 v1, s0
+; SI-NEXT:	v_mov_b32_e32 v0, s11
+; SI-NEXT:	v_cmp_lt_i32_e64 vcc, s17, 0
+; SI-NEXT:	v_cndmask_b32_e32 v0, v0, v1, vcc
+; SI-NEXT:	v_mov_b32_e32 v1, s9
+; SI-NEXT:	v_cmp_gt_i32_e64 s[0:1], s17, 51
+; SI-NEXT:	v_cndmask_b32_e64 v1, v0, v1, s[0:1]
+; SI-NEXT:	v_mov_b32_e32 v0, s10
+; SI-NEXT:	v_cndmask_b32_e64 v0, v0, 0, vcc
+; SI-NEXT:	v_mov_b32_e32 v4, s8
+; SI-NEXT:	v_cndmask_b32_e64 v0, v0, v4, s[0:1]
+; SI-NEXT:	v_add_f64 v[4:5], s[8:9], -v[0:1]
+; SI-NEXT:	s_bfe_u32 s0, s15, 0xb0014
+; SI-NEXT:	s_add_i32 s10, s0, s18
+; SI-NEXT:	v_mov_b32_e32 v6, s9
+; SI-NEXT:	s_lshr_b64 s[0:1], s[2:3], s10
+; SI-NEXT:	v_cmp_ge_f64_e64 vcc, |v[4:5]|, 0.5
+; SI-NEXT:	s_andn2_b64 s[8:9], s[14:15], s[0:1]
+; SI-NEXT:	v_bfi_b32 v6, s16, v12, v6
+; SI-NEXT:	s_and_b32 s0, s15, s20
+; SI-NEXT:	v_cndmask_b32_e32 v9, 0, v6, vcc
+; SI-NEXT:	v_mov_b32_e32 v5, s0
+; SI-NEXT:	v_mov_b32_e32 v4, s9
+; SI-NEXT:	v_cmp_lt_i32_e64 vcc, s10, 0
+; SI-NEXT:	v_cndmask_b32_e32 v4, v4, v5, vcc
+; SI-NEXT:	v_mov_b32_e32 v5, s15
+; SI-NEXT:	v_cmp_gt_i32_e64 s[0:1], s10, 51
+; SI-NEXT:	v_cndmask_b32_e64 v5, v4, v5, s[0:1]
+; SI-NEXT:	v_mov_b32_e32 v4, s8
+; SI-NEXT:	v_cndmask_b32_e64 v4, v4, 0, vcc
+; SI-NEXT:	v_mov_b32_e32 v6, s14
+; SI-NEXT:	v_cndmask_b32_e64 v4, v4, v6, s[0:1]
+; SI-NEXT:	v_add_f64 v[6:7], s[14:15], -v[4:5]
+; SI-NEXT:	s_bfe_u32 s0, s13, 0xb0014
+; SI-NEXT:	v_mov_b32_e32 v10, s15
+; SI-NEXT:	s_add_i32 s8, s0, s18
+; SI-NEXT:	v_cmp_ge_f64_e64 vcc, |v[6:7]|, 0.5
+; SI-NEXT:	s_lshr_b64 s[0:1], s[2:3], s8
+; SI-NEXT:	v_bfi_b32 v10, s16, v12, v10
+; SI-NEXT:	v_cndmask_b32_e32 v7, 0, v10, vcc
+; SI-NEXT:	v_mov_b32_e32 v6, 0
+; SI-NEXT:	s_andn2_b64 s[2:3], s[12:13], s[0:1]
+; SI-NEXT:	s_and_b32 s0, s13, s20
+; SI-NEXT:	v_add_f64 v[6:7], v[4:5], v[6:7]
+; SI-NEXT:	v_mov_b32_e32 v5, s0
+; SI-NEXT:	v_mov_b32_e32 v4, s3
+; SI-NEXT:	v_cmp_lt_i32_e64 vcc, s8, 0
+; SI-NEXT:	v_cndmask_b32_e32 v4, v4, v5, vcc
+; SI-NEXT:	v_mov_b32_e32 v5, s13
+; SI-NEXT:	v_cmp_gt_i32_e64 s[0:1], s8, 51
+; SI-NEXT:	v_cndmask_b32_e64 v5, v4, v5, s[0:1]
+; SI-NEXT:	v_mov_b32_e32 v4, s2
+; SI-NEXT:	v_cndmask_b32_e64 v4, v4, 0, vcc
+; SI-NEXT:	v_mov_b32_e32 v10, s12
+; SI-NEXT:	v_cndmask_b32_e64 v4, v4, v10, s[0:1]
+; SI-NEXT:	v_add_f64 v[10:11], s[12:13], -v[4:5]
+; SI-NEXT:	v_mov_b32_e32 v13, s13
+; SI-NEXT:	v_cmp_ge_f64_e64 vcc, |v[10:11]|, 0.5
+; SI-NEXT:	v_bfi_b32 v12, s16, v12, v13
+; SI-NEXT:	v_cndmask_b32_e32 v11, 0, v12, vcc
+; SI-NEXT:	v_mov_b32_e32 v10, 0
+; SI-NEXT:	v_mov_b32_e32 v8, 0
+; SI-NEXT:	v_add_f64 v[4:5], v[4:5], v[10:11]
+; SI-NEXT:	s_mov_b32 s7, 0xf000
+; SI-NEXT:	v_add_f64 v[0:1], v[0:1], v[8:9]
+; SI-NEXT:	buffer_store_dwordx4 v[4:7], off, s[4:7], 0 offset:16
+; SI-NEXT:	buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; SI-NEXT:	s_endpgm
 ;
 ; CI-LABEL: round_v4f64:
 ; CI:       ; %bb.0:
-; CI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
-; CI-NEXT:    s_load_dwordx8 s[8:15], s[0:1], 0x11
-; CI-NEXT:    s_brev_b32 s2, -2
-; CI-NEXT:    v_mov_b32_e32 v12, 0x3ff00000
-; CI-NEXT:    s_mov_b32 s7, 0xf000
-; CI-NEXT:    s_mov_b32 s6, -1
-; CI-NEXT:    s_waitcnt lgkmcnt(0)
-; CI-NEXT:    v_trunc_f64_e32 v[0:1], s[10:11]
-; CI-NEXT:    v_mov_b32_e32 v4, s11
-; CI-NEXT:    v_add_f64 v[2:3], s[10:11], -v[0:1]
-; CI-NEXT:    v_bfi_b32 v4, s2, v12, v4
-; CI-NEXT:    v_cmp_ge_f64_e64 vcc, |v[2:3]|, 0.5
-; CI-NEXT:    v_trunc_f64_e32 v[8:9], s[8:9]
-; CI-NEXT:    v_cndmask_b32_e32 v3, 0, v4, vcc
-; CI-NEXT:    v_mov_b32_e32 v2, 0
-; CI-NEXT:    v_add_f64 v[2:3], v[0:1], v[2:3]
-; CI-NEXT:    v_add_f64 v[0:1], s[8:9], -v[8:9]
-; CI-NEXT:    v_mov_b32_e32 v4, s9
-; CI-NEXT:    v_cmp_ge_f64_e64 vcc, |v[0:1]|, 0.5
-; CI-NEXT:    v_bfi_b32 v4, s2, v12, v4
-; CI-NEXT:    v_cndmask_b32_e32 v1, 0, v4, vcc
-; CI-NEXT:    v_trunc_f64_e32 v[4:5], s[14:15]
-; CI-NEXT:    v_mov_b32_e32 v10, s15
-; CI-NEXT:    v_add_f64 v[6:7], s[14:15], -v[4:5]
-; CI-NEXT:    v_bfi_b32 v10, s2, v12, v10
-; CI-NEXT:    v_cmp_ge_f64_e64 vcc, |v[6:7]|, 0.5
-; CI-NEXT:    v_mov_b32_e32 v6, 0
-; CI-NEXT:    v_cndmask_b32_e32 v7, 0, v10, vcc
-; CI-NEXT:    v_trunc_f64_e32 v[10:11], s[12:13]
-; CI-NEXT:    v_add_f64 v[6:7], v[4:5], v[6:7]
-; CI-NEXT:    v_add_f64 v[4:5], s[12:13], -v[10:11]
-; CI-NEXT:    v_mov_b32_e32 v13, s13
-; CI-NEXT:    v_cmp_ge_f64_e64 vcc, |v[4:5]|, 0.5
-; CI-NEXT:    v_bfi_b32 v12, s2, v12, v13
-; CI-NEXT:    v_cndmask_b32_e32 v5, 0, v12, vcc
-; CI-NEXT:    v_mov_b32_e32 v4, 0
-; CI-NEXT:    v_mov_b32_e32 v0, 0
-; CI-NEXT:    v_add_f64 v[4:5], v[10:11], v[4:5]
-; CI-NEXT:    v_add_f64 v[0:1], v[8:9], v[0:1]
-; CI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[4:7], 0 offset:16
-; CI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
-; CI-NEXT:    s_endpgm
+; CI-NEXT:	s_load_dwordx8 s[4:11], s[0:1], 0x11
+; CI-NEXT:	s_brev_b32 s12, -2
+; CI-NEXT:	v_mov_b32_e32 v12, 0x3ff00000
+; CI-NEXT:	s_load_dwordx2 s[0:1], s[0:1], 0x9
+; CI-NEXT:	s_mov_b32 s3, 0xf000
+; CI-NEXT:	s_waitcnt lgkmcnt(0)
+; CI-NEXT:	v_trunc_f64_e32 v[0:1], s[6:7]
+; CI-NEXT:	v_mov_b32_e32 v4, s7
+; CI-NEXT:	v_add_f64 v[2:3], s[6:7], -v[0:1]
+; CI-NEXT:	v_bfi_b32 v4, s12, v12, v4
+; CI-NEXT:	v_cmp_ge_f64_e64 vcc, |v[2:3]|, 0.5
+; CI-NEXT:	v_trunc_f64_e32 v[8:9], s[4:5]
+; CI-NEXT:	v_cndmask_b32_e32 v3, 0, v4, vcc
+; CI-NEXT:	v_mov_b32_e32 v2, 0
+; CI-NEXT:	v_add_f64 v[2:3], v[0:1], v[2:3]
+; CI-NEXT:	v_add_f64 v[0:1], s[4:5], -v[8:9]
+; CI-NEXT:	v_mov_b32_e32 v4, s5
+; CI-NEXT:	v_cmp_ge_f64_e64 vcc, |v[0:1]|, 0.5
+; CI-NEXT:	v_bfi_b32 v4, s12, v12, v4
+; CI-NEXT:	v_cndmask_b32_e32 v1, 0, v4, vcc
+; CI-NEXT:	v_trunc_f64_e32 v[4:5], s[10:11]
+; CI-NEXT:	v_mov_b32_e32 v10, s11
+; CI-NEXT:	v_add_f64 v[6:7], s[10:11], -v[4:5]
+; CI-NEXT:	v_bfi_b32 v10, s12, v12, v10
+; CI-NEXT:	v_cmp_ge_f64_e64 vcc, |v[6:7]|, 0.5
+; CI-NEXT:	v_mov_b32_e32 v6, 0
+; CI-NEXT:	v_cndmask_b32_e32 v7, 0, v10, vcc
+; CI-NEXT:	v_trunc_f64_e32 v[10:11], s[8:9]
+; CI-NEXT:	v_add_f64 v[6:7], v[4:5], v[6:7]
+; CI-NEXT:	v_add_f64 v[4:5], s[8:9], -v[10:11]
+; CI-NEXT:	v_mov_b32_e32 v13, s9
+; CI-NEXT:	v_cmp_ge_f64_e64 vcc, |v[4:5]|, 0.5
+; CI-NEXT:	v_bfi_b32 v12, s12, v12, v13
+; CI-NEXT:	v_cndmask_b32_e32 v5, 0, v12, vcc
+; CI-NEXT:	v_mov_b32_e32 v4, 0
+; CI-NEXT:	v_mov_b32_e32 v0, 0
+; CI-NEXT:	v_add_f64 v[4:5], v[10:11], v[4:5]
+; CI-NEXT:	s_mov_b32 s2, -1
+; CI-NEXT:	v_add_f64 v[0:1], v[8:9], v[0:1]
+; CI-NEXT:	buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:16
+; CI-NEXT:	buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; CI-NEXT:	s_endpgm
   %result = call <4 x double> @llvm.round.v4f64(<4 x double> %in) #1
   store <4 x double> %result, <4 x double> addrspace(1)* %out
   ret void
@@ -600,82 +600,82 @@ define amdgpu_kernel void @round_v8f64(<8 x double> addrspace(1)* %out, <8 x dou
 ;
 ; CI-LABEL: round_v8f64:
 ; CI:       ; %bb.0:
-; CI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
-; CI-NEXT:    s_load_dwordx16 s[8:23], s[0:1], 0x19
-; CI-NEXT:    s_brev_b32 s2, -2
-; CI-NEXT:    v_mov_b32_e32 v16, 0x3ff00000
-; CI-NEXT:    s_mov_b32 s7, 0xf000
-; CI-NEXT:    s_mov_b32 s6, -1
-; CI-NEXT:    s_waitcnt lgkmcnt(0)
-; CI-NEXT:    v_trunc_f64_e32 v[0:1], s[10:11]
-; CI-NEXT:    v_mov_b32_e32 v4, s11
-; CI-NEXT:    v_add_f64 v[2:3], s[10:11], -v[0:1]
-; CI-NEXT:    v_bfi_b32 v4, s2, v16, v4
-; CI-NEXT:    v_cmp_ge_f64_e64 vcc, |v[2:3]|, 0.5
-; CI-NEXT:    v_mov_b32_e32 v2, 0
-; CI-NEXT:    v_cndmask_b32_e32 v3, 0, v4, vcc
-; CI-NEXT:    v_trunc_f64_e32 v[4:5], s[8:9]
-; CI-NEXT:    v_add_f64 v[2:3], v[0:1], v[2:3]
-; CI-NEXT:    v_add_f64 v[0:1], s[8:9], -v[4:5]
-; CI-NEXT:    v_mov_b32_e32 v6, s9
-; CI-NEXT:    v_cmp_ge_f64_e64 vcc, |v[0:1]|, 0.5
-; CI-NEXT:    v_bfi_b32 v6, s2, v16, v6
-; CI-NEXT:    v_cndmask_b32_e32 v1, 0, v6, vcc
-; CI-NEXT:    v_trunc_f64_e32 v[6:7], s[14:15]
-; CI-NEXT:    v_mov_b32_e32 v0, 0
-; CI-NEXT:    v_add_f64 v[0:1], v[4:5], v[0:1]
-; CI-NEXT:    v_add_f64 v[4:5], s[14:15], -v[6:7]
-; CI-NEXT:    v_mov_b32_e32 v8, s15
-; CI-NEXT:    v_cmp_ge_f64_e64 vcc, |v[4:5]|, 0.5
-; CI-NEXT:    v_bfi_b32 v8, s2, v16, v8
-; CI-NEXT:    v_cndmask_b32_e32 v5, 0, v8, vcc
-; CI-NEXT:    v_trunc_f64_e32 v[8:9], s[12:13]
-; CI-NEXT:    v_mov_b32_e32 v4, 0
-; CI-NEXT:    v_add_f64 v[6:7], v[6:7], v[4:5]
-; CI-NEXT:    v_add_f64 v[4:5], s[12:13], -v[8:9]
-; CI-NEXT:    v_mov_b32_e32 v10, s13
-; CI-NEXT:    v_cmp_ge_f64_e64 vcc, |v[4:5]|, 0.5
-; CI-NEXT:    v_bfi_b32 v10, s2, v16, v10
-; CI-NEXT:    v_cndmask_b32_e32 v5, 0, v10, vcc
-; CI-NEXT:    v_mov_b32_e32 v4, 0
-; CI-NEXT:    v_add_f64 v[4:5], v[8:9], v[4:5]
-; CI-NEXT:    v_mov_b32_e32 v8, s19
-; CI-NEXT:    v_bfi_b32 v18, s2, v16, v8
-; CI-NEXT:    v_trunc_f64_e32 v[8:9], s[20:21]
-; CI-NEXT:    v_trunc_f64_e32 v[10:11], s[22:23]
-; CI-NEXT:    v_add_f64 v[14:15], s[20:21], -v[8:9]
-; CI-NEXT:    v_mov_b32_e32 v19, s23
-; CI-NEXT:    v_cmp_ge_f64_e64 vcc, |v[14:15]|, 0.5
-; CI-NEXT:    v_add_f64 v[14:15], s[22:23], -v[10:11]
-; CI-NEXT:    v_mov_b32_e32 v17, s21
-; CI-NEXT:    v_cmp_ge_f64_e64 s[0:1], |v[14:15]|, 0.5
-; CI-NEXT:    v_bfi_b32 v19, s2, v16, v19
-; CI-NEXT:    v_trunc_f64_e32 v[12:13], s[16:17]
-; CI-NEXT:    v_bfi_b32 v17, s2, v16, v17
-; CI-NEXT:    v_cndmask_b32_e64 v15, 0, v19, s[0:1]
-; CI-NEXT:    v_mov_b32_e32 v14, 0
-; CI-NEXT:    v_add_f64 v[10:11], v[10:11], v[14:15]
-; CI-NEXT:    v_cndmask_b32_e32 v15, 0, v17, vcc
-; CI-NEXT:    v_mov_b32_e32 v14, 0
-; CI-NEXT:    v_mov_b32_e32 v17, s17
-; CI-NEXT:    v_add_f64 v[8:9], v[8:9], v[14:15]
-; CI-NEXT:    v_add_f64 v[14:15], s[16:17], -v[12:13]
-; CI-NEXT:    v_bfi_b32 v19, s2, v16, v17
-; CI-NEXT:    v_trunc_f64_e32 v[16:17], s[18:19]
-; CI-NEXT:    v_cmp_ge_f64_e64 vcc, |v[14:15]|, 0.5
-; CI-NEXT:    v_add_f64 v[14:15], s[18:19], -v[16:17]
-; CI-NEXT:    v_cmp_ge_f64_e64 s[0:1], |v[14:15]|, 0.5
-; CI-NEXT:    v_mov_b32_e32 v14, 0
-; CI-NEXT:    v_cndmask_b32_e64 v15, 0, v18, s[0:1]
-; CI-NEXT:    v_add_f64 v[14:15], v[16:17], v[14:15]
-; CI-NEXT:    v_cndmask_b32_e32 v17, 0, v19, vcc
-; CI-NEXT:    v_mov_b32_e32 v16, 0
-; CI-NEXT:    v_add_f64 v[12:13], v[12:13], v[16:17]
-; CI-NEXT:    buffer_store_dwordx4 v[8:11], off, s[4:7], 0 offset:48
-; CI-NEXT:    buffer_store_dwordx4 v[12:15], off, s[4:7], 0 offset:32
-; CI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[4:7], 0 offset:16
-; CI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
-; CI-NEXT:    s_endpgm
+; CI-NEXT:	s_load_dwordx16 s[8:23], s[0:1], 0x19
+; CI-NEXT:	s_brev_b32 s2, -2
+; CI-NEXT:	v_mov_b32_e32 v16, 0x3ff00000
+; CI-NEXT:	s_load_dwordx2 s[4:5], s[0:1], 0x9
+; CI-NEXT:	s_mov_b32 s7, 0xf000
+; CI-NEXT:	s_waitcnt lgkmcnt(0)
+; CI-NEXT:	v_trunc_f64_e32 v[0:1], s[10:11]
+; CI-NEXT:	v_mov_b32_e32 v4, s11
+; CI-NEXT:	v_add_f64 v[2:3], s[10:11], -v[0:1]
+; CI-NEXT:	v_bfi_b32 v4, s2, v16, v4
+; CI-NEXT:	v_cmp_ge_f64_e64 vcc, |v[2:3]|, 0.5
+; CI-NEXT:	v_mov_b32_e32 v2, 0
+; CI-NEXT:	v_cndmask_b32_e32 v3, 0, v4, vcc
+; CI-NEXT:	v_trunc_f64_e32 v[4:5], s[8:9]
+; CI-NEXT:	v_add_f64 v[2:3], v[0:1], v[2:3]
+; CI-NEXT:	v_add_f64 v[0:1], s[8:9], -v[4:5]
+; CI-NEXT:	v_mov_b32_e32 v6, s9
+; CI-NEXT:	v_cmp_ge_f64_e64 vcc, |v[0:1]|, 0.5
+; CI-NEXT:	v_bfi_b32 v6, s2, v16, v6
+; CI-NEXT:	v_cndmask_b32_e32 v1, 0, v6, vcc
+; CI-NEXT:	v_trunc_f64_e32 v[6:7], s[14:15]
+; CI-NEXT:	v_mov_b32_e32 v0, 0
+; CI-NEXT:	v_add_f64 v[0:1], v[4:5], v[0:1]
+; CI-NEXT:	v_add_f64 v[4:5], s[14:15], -v[6:7]
+; CI-NEXT:	v_mov_b32_e32 v8, s15
+; CI-NEXT:	v_cmp_ge_f64_e64 vcc, |v[4:5]|, 0.5
+; CI-NEXT:	v_bfi_b32 v8, s2, v16, v8
+; CI-NEXT:	v_cndmask_b32_e32 v5, 0, v8, vcc
+; CI-NEXT:	v_trunc_f64_e32 v[8:9], s[12:13]
+; CI-NEXT:	v_mov_b32_e32 v4, 0
+; CI-NEXT:	v_add_f64 v[6:7], v[6:7], v[4:5]
+; CI-NEXT:	v_add_f64 v[4:5], s[12:13], -v[8:9]
+; CI-NEXT:	v_mov_b32_e32 v10, s13
+; CI-NEXT:	v_cmp_ge_f64_e64 vcc, |v[4:5]|, 0.5
+; CI-NEXT:	v_bfi_b32 v10, s2, v16, v10
+; CI-NEXT:	v_cndmask_b32_e32 v5, 0, v10, vcc
+; CI-NEXT:	v_mov_b32_e32 v4, 0
+; CI-NEXT:	v_add_f64 v[4:5], v[8:9], v[4:5]
+; CI-NEXT:	v_mov_b32_e32 v8, s19
+; CI-NEXT:	v_bfi_b32 v18, s2, v16, v8
+; CI-NEXT:	v_trunc_f64_e32 v[8:9], s[20:21]
+; CI-NEXT:	v_trunc_f64_e32 v[10:11], s[22:23]
+; CI-NEXT:	v_add_f64 v[14:15], s[20:21], -v[8:9]
+; CI-NEXT:	v_mov_b32_e32 v19, s23
+; CI-NEXT:	v_cmp_ge_f64_e64 vcc, |v[14:15]|, 0.5
+; CI-NEXT:	v_add_f64 v[14:15], s[22:23], -v[10:11]
+; CI-NEXT:	v_mov_b32_e32 v17, s21
+; CI-NEXT:	v_cmp_ge_f64_e64 s[0:1], |v[14:15]|, 0.5
+; CI-NEXT:	v_bfi_b32 v19, s2, v16, v19
+; CI-NEXT:	v_trunc_f64_e32 v[12:13], s[16:17]
+; CI-NEXT:	v_bfi_b32 v17, s2, v16, v17
+; CI-NEXT:	v_cndmask_b32_e64 v15, 0, v19, s[0:1]
+; CI-NEXT:	v_mov_b32_e32 v14, 0
+; CI-NEXT:	v_add_f64 v[10:11], v[10:11], v[14:15]
+; CI-NEXT:	v_cndmask_b32_e32 v15, 0, v17, vcc
+; CI-NEXT:	v_mov_b32_e32 v14, 0
+; CI-NEXT:	v_mov_b32_e32 v17, s17
+; CI-NEXT:	v_add_f64 v[8:9], v[8:9], v[14:15]
+; CI-NEXT:	v_add_f64 v[14:15], s[16:17], -v[12:13]
+; CI-NEXT:	v_bfi_b32 v19, s2, v16, v17
+; CI-NEXT:	v_trunc_f64_e32 v[16:17], s[18:19]
+; CI-NEXT:	v_cmp_ge_f64_e64 vcc, |v[14:15]|, 0.5
+; CI-NEXT:	v_add_f64 v[14:15], s[18:19], -v[16:17]
+; CI-NEXT:	s_mov_b32 s6, -1
+; CI-NEXT:	v_cmp_ge_f64_e64 s[0:1], |v[14:15]|, 0.5
+; CI-NEXT:	v_mov_b32_e32 v14, 0
+; CI-NEXT:	v_cndmask_b32_e64 v15, 0, v18, s[0:1]
+; CI-NEXT:	v_add_f64 v[14:15], v[16:17], v[14:15]
+; CI-NEXT:	v_cndmask_b32_e32 v17, 0, v19, vcc
+; CI-NEXT:	v_mov_b32_e32 v16, 0
+; CI-NEXT:	v_add_f64 v[12:13], v[12:13], v[16:17]
+; CI-NEXT:	buffer_store_dwordx4 v[8:11], off, s[4:7], 0 offset:48
+; CI-NEXT:	buffer_store_dwordx4 v[12:15], off, s[4:7], 0 offset:32
+; CI-NEXT:	buffer_store_dwordx4 v[4:7], off, s[4:7], 0 offset:16
+; CI-NEXT:	buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; CI-NEXT:	s_endpgm
   %result = call <8 x double> @llvm.round.v8f64(<8 x double> %in) #1
   store <8 x double> %result, <8 x double> addrspace(1)* %out
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/merge-stores.ll b/llvm/test/CodeGen/AMDGPU/merge-stores.ll
index cb45d1172e6c5..925a2daa93da7 100644
--- a/llvm/test/CodeGen/AMDGPU/merge-stores.ll
+++ b/llvm/test/CodeGen/AMDGPU/merge-stores.ll
@@ -566,7 +566,6 @@ define amdgpu_kernel void @merge_global_store_6_constants_i32(i32 addrspace(1)*
 ; GCN-LABEL: {{^}}merge_global_store_7_constants_i32:
 ; GCN: buffer_store_dwordx4
 ; SI-DAG: buffer_store_dwordx2
-; SI-DAG: buffer_store_dword v
 ; CI: buffer_store_dwordx3
 define amdgpu_kernel void @merge_global_store_7_constants_i32(i32 addrspace(1)* %out) {
   store i32 34, i32 addrspace(1)* %out, align 4
diff --git a/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.ll b/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.ll
index a9a60b93ef54d..50df07e0a8f43 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.ll
+++ b/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.ll
@@ -17,9 +17,9 @@ define amdgpu_kernel void @clmem_read_simplified(i8 addrspace(1)*  %buffer) {
 ;
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:2048
-; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:2048
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:2048
+; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:2048
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:-4096
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:2048
@@ -86,14 +86,14 @@ define hidden amdgpu_kernel void @clmem_read(i8 addrspace(1)*  %buffer) {
 ; GFX8:    flat_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}]
 ; GFX8:    flat_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}]
 ;
+; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:-4096
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:-2048
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:-2048
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:-2048
-; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:-2048
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
-; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:-4096
+; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:-2048
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:-4096
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:-2048
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
@@ -300,9 +300,9 @@ define amdgpu_kernel void @Offset64(i8 addrspace(1)*  %buffer) {
 ; GFX8:    flat_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}]
 ;
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
+; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:-4096
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:2048
-; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
 ;
 ; GFX10:   global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
 ; GFX10:   global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:-2048
@@ -456,10 +456,10 @@ define amdgpu_kernel void @ReverseOrder(i8 addrspace(1)* %buffer) {
 ;
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:2048
-; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:2048
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:2048
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
+; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:2048
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
 ; GFX9:    global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:2048
 ;
diff --git a/llvm/test/CodeGen/AMDGPU/store-weird-sizes.ll b/llvm/test/CodeGen/AMDGPU/store-weird-sizes.ll
index 2b8eba5f90149..a4d08ec980f44 100644
--- a/llvm/test/CodeGen/AMDGPU/store-weird-sizes.ll
+++ b/llvm/test/CodeGen/AMDGPU/store-weird-sizes.ll
@@ -6,14 +6,14 @@
 define void @local_store_i56(i56 addrspace(3)* %ptr, i56 %arg) #0 {
 ; CIVI-LABEL: local_store_i56:
 ; CIVI:       ; %bb.0:
-; CIVI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CIVI-NEXT:    s_mov_b32 m0, -1
-; CIVI-NEXT:    ds_write_b16 v0, v2 offset:4
-; CIVI-NEXT:    ds_write_b32 v0, v1
-; CIVI-NEXT:    v_lshrrev_b32_e32 v1, 16, v2
-; CIVI-NEXT:    ds_write_b8 v0, v1 offset:6
-; CIVI-NEXT:    s_waitcnt lgkmcnt(0)
-; CIVI-NEXT:    s_setpc_b64 s[30:31]
+; CIVI-NEXT:        s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CIVI-NEXT:        v_lshrrev_b32_e32 v3, 16, v2
+; CIVI-NEXT:        s_mov_b32 m0, -1
+; CIVI-NEXT:        ds_write_b8 v0, v3 offset:6
+; CIVI-NEXT:        ds_write_b16 v0, v2 offset:4
+; CIVI-NEXT:        ds_write_b32 v0, v1
+; CIVI-NEXT:        s_waitcnt lgkmcnt(0)
+; CIVI-NEXT:        s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: local_store_i56:
 ; GFX9:       ; %bb.0:
@@ -30,70 +30,70 @@ define void @local_store_i56(i56 addrspace(3)* %ptr, i56 %arg) #0 {
 define amdgpu_kernel void @local_store_i55(i55 addrspace(3)* %ptr, i55 %arg) #0 {
 ; HAWAII-LABEL: local_store_i55:
 ; HAWAII:       ; %bb.0:
-; HAWAII-NEXT:    s_or_b32 s0, s4, 14
-; HAWAII-NEXT:    v_mov_b32_e32 v0, s0
-; HAWAII-NEXT:    v_mov_b32_e32 v1, s5
-; HAWAII-NEXT:    flat_load_ubyte v0, v[0:1]
-; HAWAII-NEXT:    s_load_dword s0, s[4:5], 0x0
-; HAWAII-NEXT:    s_load_dword s1, s[4:5], 0x2
-; HAWAII-NEXT:    s_load_dword s2, s[4:5], 0x3
-; HAWAII-NEXT:    s_mov_b32 m0, -1
-; HAWAII-NEXT:    s_waitcnt lgkmcnt(0)
-; HAWAII-NEXT:    v_mov_b32_e32 v1, s0
-; HAWAII-NEXT:    v_mov_b32_e32 v3, s1
-; HAWAII-NEXT:    v_mov_b32_e32 v2, s2
-; HAWAII-NEXT:    ds_write_b16 v1, v2 offset:4
-; HAWAII-NEXT:    s_waitcnt vmcnt(0)
-; HAWAII-NEXT:    v_and_b32_e32 v0, 0x7f, v0
-; HAWAII-NEXT:    ds_write_b8 v1, v0 offset:6
-; HAWAII-NEXT:    ds_write_b32 v1, v3
-; HAWAII-NEXT:    s_endpgm
+; HAWAII-NEXT:        s_or_b32 s0, s4, 14
+; HAWAII-NEXT:        v_mov_b32_e32 v0, s0
+; HAWAII-NEXT:        v_mov_b32_e32 v1, s5
+; HAWAII-NEXT:        flat_load_ubyte v0, v[0:1]
+; HAWAII-NEXT:        s_load_dword s0, s[4:5], 0x0
+; HAWAII-NEXT:        s_load_dword s1, s[4:5], 0x2
+; HAWAII-NEXT:        s_load_dword s2, s[4:5], 0x3
+; HAWAII-NEXT:        s_mov_b32 m0, -1
+; HAWAII-NEXT:        s_waitcnt lgkmcnt(0)
+; HAWAII-NEXT:        v_mov_b32_e32 v1, s0
+; HAWAII-NEXT:        v_mov_b32_e32 v2, s1
+; HAWAII-NEXT:        v_mov_b32_e32 v3, s2
+; HAWAII-NEXT:        s_waitcnt vmcnt(0)
+; HAWAII-NEXT:        v_and_b32_e32 v0, 0x7f, v0
+; HAWAII-NEXT:        ds_write_b8 v1, v0 offset:6
+; HAWAII-NEXT:        ds_write_b16 v1, v3 offset:4
+; HAWAII-NEXT:        ds_write_b32 v1, v2
+; HAWAII-NEXT:        s_endpgm
 ;
 ; FIJI-LABEL: local_store_i55:
 ; FIJI:       ; %bb.0:
-; FIJI-NEXT:    s_or_b32 s0, s4, 14
-; FIJI-NEXT:    v_mov_b32_e32 v0, s0
-; FIJI-NEXT:    v_mov_b32_e32 v1, s5
-; FIJI-NEXT:    flat_load_ubyte v0, v[0:1]
-; FIJI-NEXT:    s_load_dword s0, s[4:5], 0x0
-; FIJI-NEXT:    s_load_dword s1, s[4:5], 0x8
-; FIJI-NEXT:    s_load_dword s2, s[4:5], 0xc
-; FIJI-NEXT:    s_mov_b32 m0, -1
-; FIJI-NEXT:    s_waitcnt lgkmcnt(0)
-; FIJI-NEXT:    v_mov_b32_e32 v1, s0
-; FIJI-NEXT:    v_mov_b32_e32 v3, s1
-; FIJI-NEXT:    s_and_b32 s3, s2, 0xffff
-; FIJI-NEXT:    v_mov_b32_e32 v2, s2
-; FIJI-NEXT:    ds_write_b16 v1, v2 offset:4
-; FIJI-NEXT:    s_waitcnt vmcnt(0)
-; FIJI-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; FIJI-NEXT:    v_or_b32_e32 v0, s3, v0
-; FIJI-NEXT:    v_bfe_u32 v0, v0, 16, 7
-; FIJI-NEXT:    ds_write_b8 v1, v0 offset:6
-; FIJI-NEXT:    ds_write_b32 v1, v3
-; FIJI-NEXT:    s_endpgm
+; FIJI-NEXT:        s_or_b32 s0, s4, 14
+; FIJI-NEXT:        v_mov_b32_e32 v0, s0
+; FIJI-NEXT:        v_mov_b32_e32 v1, s5
+; FIJI-NEXT:        flat_load_ubyte v0, v[0:1]
+; FIJI-NEXT:        s_load_dword s0, s[4:5], 0x0
+; FIJI-NEXT:        s_load_dword s1, s[4:5], 0x8
+; FIJI-NEXT:        s_load_dword s2, s[4:5], 0xc
+; FIJI-NEXT:        s_mov_b32 m0, -1
+; FIJI-NEXT:        s_waitcnt lgkmcnt(0)
+; FIJI-NEXT:        v_mov_b32_e32 v1, s0
+; FIJI-NEXT:        v_mov_b32_e32 v3, s1
+; FIJI-NEXT:        s_and_b32 s3, s2, 0xffff
+; FIJI-NEXT:        v_mov_b32_e32 v2, s2
+; FIJI-NEXT:        s_waitcnt vmcnt(0)
+; FIJI-NEXT:        v_lshlrev_b32_e32 v0, 16, v0
+; FIJI-NEXT:        v_or_b32_e32 v0, s3, v0
+; FIJI-NEXT:        v_bfe_u32 v0, v0, 16, 7
+; FIJI-NEXT:        ds_write_b8 v1, v0 offset:6
+; FIJI-NEXT:        ds_write_b16 v1, v2 offset:4
+; FIJI-NEXT:        ds_write_b32 v1, v3
+; FIJI-NEXT:        s_endpgm
 ;
 ; GFX9-LABEL: local_store_i55:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    v_mov_b32_e32 v0, s4
-; GFX9-NEXT:    v_mov_b32_e32 v1, s5
-; GFX9-NEXT:    v_mov_b32_e32 v2, 0
-; GFX9-NEXT:    global_load_ubyte_d16_hi v2, v[0:1], off offset:14
-; GFX9-NEXT:    s_load_dword s0, s[4:5], 0x0
-; GFX9-NEXT:    s_load_dword s1, s[4:5], 0x8
-; GFX9-NEXT:    s_load_dword s2, s[4:5], 0xc
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-NEXT:    v_mov_b32_e32 v3, s1
-; GFX9-NEXT:    v_mov_b32_e32 v1, s2
-; GFX9-NEXT:    s_and_b32 s3, s2, 0xffff
-; GFX9-NEXT:    ds_write_b16 v0, v1 offset:4
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_or_b32_e32 v1, s3, v2
-; GFX9-NEXT:    v_and_b32_e32 v1, 0x7fffff, v1
-; GFX9-NEXT:    ds_write_b8_d16_hi v0, v1 offset:6
-; GFX9-NEXT:    ds_write_b32 v0, v3
-; GFX9-NEXT:    s_endpgm
+; GFX9-NEXT:        v_mov_b32_e32 v0, s4
+; GFX9-NEXT:        v_mov_b32_e32 v1, s5
+; GFX9-NEXT:        v_mov_b32_e32 v2, 0
+; GFX9-NEXT:        global_load_ubyte_d16_hi v2, v[0:1], off offset:14
+; GFX9-NEXT:        s_load_dword s0, s[4:5], 0x0
+; GFX9-NEXT:        s_load_dword s1, s[4:5], 0x8
+; GFX9-NEXT:        s_load_dword s2, s[4:5], 0xc
+; GFX9-NEXT:        s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:        v_mov_b32_e32 v0, s0
+; GFX9-NEXT:        v_mov_b32_e32 v3, s1
+; GFX9-NEXT:        s_and_b32 s3, s2, 0xffff
+; GFX9-NEXT:        v_mov_b32_e32 v1, s2
+; GFX9-NEXT:        s_waitcnt vmcnt(0)
+; GFX9-NEXT:        v_or_b32_e32 v2, s3, v2
+; GFX9-NEXT:        v_and_b32_e32 v2, 0x7fffff, v2
+; GFX9-NEXT:        ds_write_b8_d16_hi v0, v2 offset:6
+; GFX9-NEXT:        ds_write_b16 v0, v1 offset:4
+; GFX9-NEXT:        ds_write_b32 v0, v3
+; GFX9-NEXT:        s_endpgm
   store i55 %arg, i55 addrspace(3)* %ptr, align 8
   ret void
 }
@@ -101,31 +101,31 @@ define amdgpu_kernel void @local_store_i55(i55 addrspace(3)* %ptr, i55 %arg) #0
 define amdgpu_kernel void @local_store_i48(i48 addrspace(3)* %ptr, i48 %arg) #0 {
 ; HAWAII-LABEL: local_store_i48:
 ; HAWAII:       ; %bb.0:
-; HAWAII-NEXT:    s_load_dword s0, s[4:5], 0x0
-; HAWAII-NEXT:    s_load_dword s1, s[4:5], 0x2
-; HAWAII-NEXT:    s_load_dword s2, s[4:5], 0x3
-; HAWAII-NEXT:    s_mov_b32 m0, -1
-; HAWAII-NEXT:    s_waitcnt lgkmcnt(0)
-; HAWAII-NEXT:    v_mov_b32_e32 v0, s0
-; HAWAII-NEXT:    v_mov_b32_e32 v1, s2
-; HAWAII-NEXT:    ds_write_b16 v0, v1 offset:4
-; HAWAII-NEXT:    v_mov_b32_e32 v1, s1
-; HAWAII-NEXT:    ds_write_b32 v0, v1
-; HAWAII-NEXT:    s_endpgm
+; HAWAII-NEXT:        s_load_dword s0, s[4:5], 0x0
+; HAWAII-NEXT:        s_load_dword s1, s[4:5], 0x2
+; HAWAII-NEXT:        s_load_dword s2, s[4:5], 0x3
+; HAWAII-NEXT:        s_mov_b32 m0, -1
+; HAWAII-NEXT:        s_waitcnt lgkmcnt(0)
+; HAWAII-NEXT:        v_mov_b32_e32 v0, s0
+; HAWAII-NEXT:        v_mov_b32_e32 v2, s1
+; HAWAII-NEXT:        v_mov_b32_e32 v1, s2
+; HAWAII-NEXT:        ds_write_b16 v0, v1 offset:4
+; HAWAII-NEXT:        ds_write_b32 v0, v2
+; HAWAII-NEXT:        s_endpgm
 ;
 ; FIJI-LABEL: local_store_i48:
 ; FIJI:       ; %bb.0:
-; FIJI-NEXT:    s_load_dword s0, s[4:5], 0x0
-; FIJI-NEXT:    s_load_dword s1, s[4:5], 0x8
-; FIJI-NEXT:    s_load_dword s2, s[4:5], 0xc
-; FIJI-NEXT:    s_mov_b32 m0, -1
-; FIJI-NEXT:    s_waitcnt lgkmcnt(0)
-; FIJI-NEXT:    v_mov_b32_e32 v0, s0
-; FIJI-NEXT:    v_mov_b32_e32 v1, s2
-; FIJI-NEXT:    ds_write_b16 v0, v1 offset:4
-; FIJI-NEXT:    v_mov_b32_e32 v1, s1
-; FIJI-NEXT:    ds_write_b32 v0, v1
-; FIJI-NEXT:    s_endpgm
+; FIJI-NEXT:        s_load_dword s0, s[4:5], 0x0
+; FIJI-NEXT:        s_load_dword s1, s[4:5], 0x8
+; FIJI-NEXT:        s_load_dword s2, s[4:5], 0xc
+; FIJI-NEXT:        s_mov_b32 m0, -1
+; FIJI-NEXT:        s_waitcnt lgkmcnt(0)
+; FIJI-NEXT:        v_mov_b32_e32 v0, s0
+; FIJI-NEXT:        v_mov_b32_e32 v2, s1
+; FIJI-NEXT:        v_mov_b32_e32 v1, s2
+; FIJI-NEXT:        ds_write_b16 v0, v1 offset:4
+; FIJI-NEXT:        ds_write_b32 v0, v2
+; FIJI-NEXT:        s_endpgm
 ;
 ; GFX9-LABEL: local_store_i48:
 ; GFX9:       ; %bb.0:
@@ -146,35 +146,35 @@ define amdgpu_kernel void @local_store_i48(i48 addrspace(3)* %ptr, i48 %arg) #0
 define amdgpu_kernel void @local_store_i65(i65 addrspace(3)* %ptr, i65 %arg) #0 {
 ; HAWAII-LABEL: local_store_i65:
 ; HAWAII:       ; %bb.0:
-; HAWAII-NEXT:    s_load_dword s2, s[4:5], 0x0
-; HAWAII-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
-; HAWAII-NEXT:    s_load_dword s3, s[4:5], 0x4
-; HAWAII-NEXT:    s_mov_b32 m0, -1
-; HAWAII-NEXT:    s_waitcnt lgkmcnt(0)
-; HAWAII-NEXT:    v_mov_b32_e32 v2, s2
-; HAWAII-NEXT:    s_and_b32 s3, s3, 1
-; HAWAII-NEXT:    v_mov_b32_e32 v0, s3
-; HAWAII-NEXT:    ds_write_b8 v2, v0 offset:8
-; HAWAII-NEXT:    v_mov_b32_e32 v0, s0
-; HAWAII-NEXT:    v_mov_b32_e32 v1, s1
-; HAWAII-NEXT:    ds_write_b64 v2, v[0:1]
-; HAWAII-NEXT:    s_endpgm
+; HAWAII-NEXT:        s_load_dword s2, s[4:5], 0x0
+; HAWAII-NEXT:        s_load_dwordx2 s[0:1], s[4:5], 0x2
+; HAWAII-NEXT:        s_load_dword s3, s[4:5], 0x4
+; HAWAII-NEXT:        s_mov_b32 m0, -1
+; HAWAII-NEXT:        s_waitcnt lgkmcnt(0)
+; HAWAII-NEXT:        v_mov_b32_e32 v2, s2
+; HAWAII-NEXT:        v_mov_b32_e32 v0, s0
+; HAWAII-NEXT:        s_and_b32 s3, s3, 1
+; HAWAII-NEXT:        v_mov_b32_e32 v3, s3
+; HAWAII-NEXT:        v_mov_b32_e32 v1, s1
+; HAWAII-NEXT:        ds_write_b8 v2, v3 offset:8
+; HAWAII-NEXT:        ds_write_b64 v2, v[0:1]
+; HAWAII-NEXT:        s_endpgm
 ;
 ; FIJI-LABEL: local_store_i65:
 ; FIJI:       ; %bb.0:
-; FIJI-NEXT:    s_load_dword s2, s[4:5], 0x0
-; FIJI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
-; FIJI-NEXT:    s_load_dword s3, s[4:5], 0x10
-; FIJI-NEXT:    s_mov_b32 m0, -1
-; FIJI-NEXT:    s_waitcnt lgkmcnt(0)
-; FIJI-NEXT:    v_mov_b32_e32 v2, s2
-; FIJI-NEXT:    s_and_b32 s3, s3, 1
-; FIJI-NEXT:    v_mov_b32_e32 v0, s3
-; FIJI-NEXT:    ds_write_b8 v2, v0 offset:8
-; FIJI-NEXT:    v_mov_b32_e32 v0, s0
-; FIJI-NEXT:    v_mov_b32_e32 v1, s1
-; FIJI-NEXT:    ds_write_b64 v2, v[0:1]
-; FIJI-NEXT:    s_endpgm
+; FIJI-NEXT:        s_load_dword s2, s[4:5], 0x0
+; FIJI-NEXT:        s_load_dwordx2 s[0:1], s[4:5], 0x8
+; FIJI-NEXT:        s_load_dword s3, s[4:5], 0x10
+; FIJI-NEXT:        s_mov_b32 m0, -1
+; FIJI-NEXT:        s_waitcnt lgkmcnt(0)
+; FIJI-NEXT:        v_mov_b32_e32 v2, s2
+; FIJI-NEXT:        v_mov_b32_e32 v0, s0
+; FIJI-NEXT:        s_and_b32 s3, s3, 1
+; FIJI-NEXT:        v_mov_b32_e32 v3, s3
+; FIJI-NEXT:        v_mov_b32_e32 v1, s1
+; FIJI-NEXT:        ds_write_b8 v2, v3 offset:8
+; FIJI-NEXT:        ds_write_b64 v2, v[0:1]
+; FIJI-NEXT:        s_endpgm
 ;
 ; GFX9-LABEL: local_store_i65:
 ; GFX9:       ; %bb.0:
@@ -218,22 +218,22 @@ define void @local_store_i13(i13 addrspace(3)* %ptr, i13 %arg) #0 {
 define void @local_store_i17(i17 addrspace(3)* %ptr, i17 %arg) #0 {
 ; CIVI-LABEL: local_store_i17:
 ; CIVI:       ; %bb.0:
-; CIVI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CIVI-NEXT:    s_mov_b32 m0, -1
-; CIVI-NEXT:    ds_write_b16 v0, v1
-; CIVI-NEXT:    v_bfe_u32 v1, v1, 16, 1
-; CIVI-NEXT:    ds_write_b8 v0, v1 offset:2
-; CIVI-NEXT:    s_waitcnt lgkmcnt(0)
-; CIVI-NEXT:    s_setpc_b64 s[30:31]
+; CIVI-NEXT:        s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CIVI-NEXT:        s_mov_b32 m0, -1
+; CIVI-NEXT:        v_bfe_u32 v2, v1, 16, 1
+; CIVI-NEXT:        ds_write_b16 v0, v1
+; CIVI-NEXT:        ds_write_b8 v0, v2 offset:2
+; CIVI-NEXT:        s_waitcnt lgkmcnt(0)
+; CIVI-NEXT:        s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: local_store_i17:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    ds_write_b16 v0, v1
-; GFX9-NEXT:    v_and_b32_e32 v1, 0x1ffff, v1
-; GFX9-NEXT:    ds_write_b8_d16_hi v0, v1 offset:2
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_setpc_b64 s[30:31]
+; GFX9-NEXT:        s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:        v_and_b32_e32 v2, 0x1ffff, v1
+; GFX9-NEXT:        ds_write_b16 v0, v1
+; GFX9-NEXT:        ds_write_b8_d16_hi v0, v2 offset:2
+; GFX9-NEXT:        s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:        s_setpc_b64 s[30:31]
   store i17 %arg, i17 addrspace(3)* %ptr, align 8
   ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/token-factor-inline-limit-test.ll b/llvm/test/CodeGen/AMDGPU/token-factor-inline-limit-test.ll
index 88d9eeb8f2667..1648c7fe37ccb 100644
--- a/llvm/test/CodeGen/AMDGPU/token-factor-inline-limit-test.ll
+++ b/llvm/test/CodeGen/AMDGPU/token-factor-inline-limit-test.ll
@@ -9,14 +9,14 @@
 ; GCN-TFILD: v_mov_b32_e32 [[REG10:v[0-9]+]], 10
 ; GCN-TFILD: v_mov_b32_e32 [[REG11:v[0-9]+]], 11
 ; GCN-TFILD: v_mov_b32_e32 [[REG12:v[0-9]+]], 12
+; GCN-TFILD: v_mov_b32_e32 [[REG13:v[0-9]+]], 13
+; GCN-TFILD: v_mov_b32_e32 [[REG14:v[0-9]+]], 14
+; GCN-TFILD: v_mov_b32_e32 [[REG15:v[0-9]+]], 15
 ; GCN-TFILD: buffer_store_dword [[REG8]], {{.*$}}
 ; GCN-TFILD: buffer_store_dword [[REG9]], {{.*}} offset:4
 ; GCN-TFILD: buffer_store_dword [[REG10]], {{.*}} offset:8
 ; GCN-TFILD: buffer_store_dword [[REG11]], {{.*}} offset:12
 ; GCN-TFILD: buffer_store_dword [[REG12]], {{.*}} offset:16
-; GCN-TFILD: v_mov_b32_e32 [[REG13:v[0-9]+]], 13
-; GCN-TFILD: v_mov_b32_e32 [[REG14:v[0-9]+]], 14
-; GCN-TFILD: v_mov_b32_e32 [[REG15:v[0-9]+]], 15
 ; GCN-TFILD: buffer_store_dword [[REG13]], {{.*}} offset:20
 ; GCN-TFILD: buffer_store_dword [[REG14]], {{.*}} offset:24
 ; GCN-TFILD: buffer_store_dword [[REG15]], {{.*}} offset:28
@@ -24,14 +24,14 @@
 ; GCN-TFIL7: v_mov_b32_e32 [[REG15:v[0-9]+]], 15
 ; GCN-TFIL7: v_mov_b32_e32 [[REG14:v[0-9]+]], 14
 ; GCN-TFIL7: v_mov_b32_e32 [[REG13:v[0-9]+]], 13
-; GCN-TFIL7: buffer_store_dword [[REG15]], {{.*}} offset:28
-; GCN-TFIL7: buffer_store_dword [[REG14]], {{.*}} offset:24
-; GCN-TFIL7: buffer_store_dword [[REG13]], {{.*}} offset:20
 ; GCN-TFIL7: v_mov_b32_e32 [[REG12:v[0-9]+]], 12
 ; GCN-TFIL7: v_mov_b32_e32 [[REG11:v[0-9]+]], 11
 ; GCN-TFIL7: v_mov_b32_e32 [[REG10:v[0-9]+]], 10
 ; GCN-TFIL7: v_mov_b32_e32 [[REG9:v[0-9]+]], 9
 ; GCN-TFIL7: v_mov_b32_e32 [[REG8:v[0-9]+]], 8
+; GCN-TFIL7: buffer_store_dword [[REG15]], {{.*}} offset:28
+; GCN-TFIL7: buffer_store_dword [[REG14]], {{.*}} offset:24
+; GCN-TFIL7: buffer_store_dword [[REG13]], {{.*}} offset:20
 ; GCN-TFIL7: buffer_store_dword [[REG12]], {{.*}} offset:16
 ; GCN-TFIL7: buffer_store_dword [[REG11]], {{.*}} offset:12
 ; GCN-TFIL7: buffer_store_dword [[REG10]], {{.*}} offset:8
diff --git a/llvm/test/CodeGen/AMDGPU/udivrem.ll b/llvm/test/CodeGen/AMDGPU/udivrem.ll
index 559f1092e6e44..fe613f2579e45 100644
--- a/llvm/test/CodeGen/AMDGPU/udivrem.ll
+++ b/llvm/test/CodeGen/AMDGPU/udivrem.ll
@@ -39,6 +39,7 @@ define amdgpu_kernel void @test_udivrem(i32 addrspace(1)* %out0, [8 x i32], i32
 ; GFX6-NEXT:        s_load_dword s3, s[0:1], 0x26
 ; GFX6-NEXT:        s_load_dwordx2 s[4:5], s[0:1], 0x9
 ; GFX6-NEXT:        s_load_dwordx2 s[8:9], s[0:1], 0x13
+; GFX6-NEXT:        s_load_dword s0, s[0:1], 0x1d
 ; GFX6-NEXT:        s_mov_b32 s7, 0xf000
 ; GFX6-NEXT:        s_mov_b32 s6, -1
 ; GFX6-NEXT:        s_mov_b32 s10, s6
@@ -50,14 +51,12 @@ define amdgpu_kernel void @test_udivrem(i32 addrspace(1)* %out0, [8 x i32], i32
 ; GFX6-NEXT:        v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GFX6-NEXT:        v_cvt_u32_f32_e32 v0, v0
 ; GFX6-NEXT:        v_mul_lo_u32 v1, s2, v0
-; GFX6-NEXT:        s_load_dword s2, s[0:1], 0x1d
 ; GFX6-NEXT:        v_mul_hi_u32 v1, v0, v1
 ; GFX6-NEXT:        v_add_i32_e32 v0, vcc, v1, v0
-; GFX6-NEXT:        s_waitcnt lgkmcnt(0)
-; GFX6-NEXT:        v_mul_hi_u32 v0, s2, v0
+; GFX6-NEXT:        v_mul_hi_u32 v0, s0, v0
 ; GFX6-NEXT:        v_mul_lo_u32 v1, v0, s3
 ; GFX6-NEXT:        v_add_i32_e32 v2, vcc, 1, v0
-; GFX6-NEXT:        v_sub_i32_e32 v1, vcc, s2, v1
+; GFX6-NEXT:        v_sub_i32_e32 v1, vcc, s0, v1
 ; GFX6-NEXT:        v_cmp_le_u32_e64 s[0:1], s3, v1
 ; GFX6-NEXT:        v_cndmask_b32_e64 v0, v0, v2, s[0:1]
 ; GFX6-NEXT:        v_subrev_i32_e32 v2, vcc, s3, v1

From 3d06fc0049c6bb94e6efd77388453206037f43ad Mon Sep 17 00:00:00 2001
From: "Joel E. Denny" <jdenny.ornl@gmail.com>
Date: Thu, 30 Jul 2020 12:20:46 -0400
Subject: [PATCH 0637/1035] [OpenMP][Docs] Mark `present` motion modifier as
 done

---
 clang/docs/OpenMPSupport.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/docs/OpenMPSupport.rst b/clang/docs/OpenMPSupport.rst
index bda52e934c266..28fbd7baebb24 100644
--- a/clang/docs/OpenMPSupport.rst
+++ b/clang/docs/OpenMPSupport.rst
@@ -270,5 +270,5 @@ want to help with the implementation.
 +------------------------------+--------------------------------------------------------------+--------------------------+-----------------------------------------------------------------------+
 | device extension             | 'present' map type modifier                                  | :part:`mostly done`      | D83061, D83062, D84422                                                |
 +------------------------------+--------------------------------------------------------------+--------------------------+-----------------------------------------------------------------------+
-| device extension             | 'present' motion modifier                                    | :part:`worked on`        | D84711, D84712                                                        |
+| device extension             | 'present' motion modifier                                    | :good:`done`             | D84711, D84712                                                        |
 +------------------------------+--------------------------------------------------------------+--------------------------+-----------------------------------------------------------------------+

From 3d6f53018f845e893ad34f64ff2851a2e5c3ba1d Mon Sep 17 00:00:00 2001
From: Hiroshi Yamauchi <yamauchi@google.com>
Date: Wed, 29 Jul 2020 15:22:13 -0700
Subject: [PATCH 0638/1035] [PGO] Include the mem ops into the function hash.

To avoid hash collisions when the only difference is in mem ops.
---
 .../CodeGen/Inputs/thinlto_expect1.proftext   |  2 +-
 .../CodeGen/Inputs/thinlto_expect2.proftext   |  4 +-
 .../CodeGenCXX/Inputs/profile-remap.proftext  |  2 +-
 .../Inputs/profile-remap_entry.proftext       |  2 +-
 .../Inputs/gcc-flag-compatibility_IR.proftext |  2 +-
 .../gcc-flag-compatibility_IR_entry.proftext  |  2 +-
 .../profile/Linux/instrprof-value-merge.c     |  2 +-
 .../Instrumentation/PGOInstrumentation.cpp    | 46 +++++++++++++++----
 .../PGOProfile/Inputs/PR41279.proftext        |  2 +-
 .../PGOProfile/Inputs/PR41279_2.proftext      |  2 +-
 .../PGOProfile/Inputs/branch1.proftext        |  2 +-
 .../Inputs/branch1_large_count.proftext       |  2 +-
 .../PGOProfile/Inputs/branch2.proftext        |  2 +-
 .../PGOProfile/Inputs/branch2_entry.proftext  |  2 +-
 .../PGOProfile/Inputs/criticaledge.proftext   |  4 +-
 .../Inputs/criticaledge_entry.proftext        |  4 +-
 .../PGOProfile/Inputs/cspgo.proftext          | 16 +++----
 .../Inputs/diag_no_value_sites.proftext       |  2 +-
 .../Inputs/fix_entry_count.proftext           |  2 +-
 .../PGOProfile/Inputs/func_entry.proftext     |  6 +--
 .../PGOProfile/Inputs/indirect_call.proftext  |  8 ++--
 .../PGOProfile/Inputs/indirectbr.proftext     |  2 +-
 .../Inputs/indirectbr_entry.proftext          |  2 +-
 .../PGOProfile/Inputs/irreducible.proftext    |  4 +-
 .../Inputs/irreducible_entry.proftext         |  4 +-
 .../PGOProfile/Inputs/landingpad.proftext     |  4 +-
 .../Inputs/landingpad_entry.proftext          |  4 +-
 .../Inputs/large_count_remarks.proftext       |  2 +-
 .../PGOProfile/Inputs/loop1.proftext          |  2 +-
 .../PGOProfile/Inputs/loop1_entry.proftext    |  2 +-
 .../PGOProfile/Inputs/loop2.proftext          |  2 +-
 .../PGOProfile/Inputs/loop2_entry.proftext    |  2 +-
 .../Inputs/memop_size_annotation.proftext     |  2 +-
 .../Inputs/misexpect-branch-correct.proftext  |  2 +-
 .../Inputs/misexpect-branch.proftext          |  2 +-
 .../Inputs/misexpect-branch_entry.proftext    |  2 +-
 .../Inputs/misexpect-switch-correct.proftext  |  2 +-
 .../misexpect-switch-correct_entry.proftext   |  2 +-
 .../Inputs/misexpect-switch.proftext          |  2 +-
 .../Inputs/misexpect-switch_entry.proftext    |  2 +-
 .../Inputs/multiple_hash_profile.proftext     | 32 ++++++++++++-
 .../PGOProfile/Inputs/noreturncall.proftext   |  2 +-
 .../PGOProfile/Inputs/remap.proftext          |  2 +-
 .../PGOProfile/Inputs/select1.proftext        |  2 +-
 .../PGOProfile/Inputs/select2.proftext        |  2 +-
 .../PGOProfile/Inputs/suppl-profile.proftext  |  2 +-
 .../PGOProfile/Inputs/switch.proftext         |  2 +-
 .../PGOProfile/Inputs/switch_entry.proftext   |  2 +-
 .../PGOProfile/Inputs/thinlto_cs.proftext     |  8 ++--
 .../PGOProfile/multiple_hash_profile.ll       |  5 ++
 50 files changed, 141 insertions(+), 80 deletions(-)

diff --git a/clang/test/CodeGen/Inputs/thinlto_expect1.proftext b/clang/test/CodeGen/Inputs/thinlto_expect1.proftext
index e7ce3a4ee2379..0c904e2ea1c89 100644
--- a/clang/test/CodeGen/Inputs/thinlto_expect1.proftext
+++ b/clang/test/CodeGen/Inputs/thinlto_expect1.proftext
@@ -2,7 +2,7 @@
 :ir
 foo
 # Func Hash:
-25571299074
+784007059655560962
 # Num Counters:
 2
 # Counter Values:
diff --git a/clang/test/CodeGen/Inputs/thinlto_expect2.proftext b/clang/test/CodeGen/Inputs/thinlto_expect2.proftext
index f9de785587abe..c240a442c465f 100644
--- a/clang/test/CodeGen/Inputs/thinlto_expect2.proftext
+++ b/clang/test/CodeGen/Inputs/thinlto_expect2.proftext
@@ -2,7 +2,7 @@
 :csir
 foo
 # Func Hash:
-25571299074
+784007059655560962
 # Num Counters:
 2
 # Counter Values:
@@ -11,7 +11,7 @@ foo
 
 foo
 # Func Hash:
-1152921530178146050
+1936928564262407938
 # Num Counters:
 2
 # Counter Values:
diff --git a/clang/test/CodeGenCXX/Inputs/profile-remap.proftext b/clang/test/CodeGenCXX/Inputs/profile-remap.proftext
index a1f90cfa6e9ed..bf57fc696c49e 100644
--- a/clang/test/CodeGenCXX/Inputs/profile-remap.proftext
+++ b/clang/test/CodeGenCXX/Inputs/profile-remap.proftext
@@ -1,6 +1,6 @@
 :ir
 _ZN3Foo8functionENS_1XE
-29667547796
+146835647075900052
 2
 10
 90
diff --git a/clang/test/CodeGenCXX/Inputs/profile-remap_entry.proftext b/clang/test/CodeGenCXX/Inputs/profile-remap_entry.proftext
index 65bc6ff46711b..89772cd7644f2 100644
--- a/clang/test/CodeGenCXX/Inputs/profile-remap_entry.proftext
+++ b/clang/test/CodeGenCXX/Inputs/profile-remap_entry.proftext
@@ -1,7 +1,7 @@
 :ir
 :entry_first
 _ZN3Foo8functionENS_1XE
-29667547796
+146835647075900052
 2
 100
 90
diff --git a/clang/test/Profile/Inputs/gcc-flag-compatibility_IR.proftext b/clang/test/Profile/Inputs/gcc-flag-compatibility_IR.proftext
index 64d18116ed647..0f39af77715dd 100644
--- a/clang/test/Profile/Inputs/gcc-flag-compatibility_IR.proftext
+++ b/clang/test/Profile/Inputs/gcc-flag-compatibility_IR.proftext
@@ -2,7 +2,7 @@
 :ir
 main
 # Func Hash:
-34137660316
+1063705162469825436
 # Num Counters:
 2
 # Counter Values:
diff --git a/clang/test/Profile/Inputs/gcc-flag-compatibility_IR_entry.proftext b/clang/test/Profile/Inputs/gcc-flag-compatibility_IR_entry.proftext
index 4903f56c419e0..d60162c0f5294 100644
--- a/clang/test/Profile/Inputs/gcc-flag-compatibility_IR_entry.proftext
+++ b/clang/test/Profile/Inputs/gcc-flag-compatibility_IR_entry.proftext
@@ -3,7 +3,7 @@
 :entry_first
 main
 # Func Hash:
-34137660316
+1063705162469825436
 # Num Counters:
 2
 # Counter Values:
diff --git a/compiler-rt/test/profile/Linux/instrprof-value-merge.c b/compiler-rt/test/profile/Linux/instrprof-value-merge.c
index 902430a4a9686..2619a1d00336e 100644
--- a/compiler-rt/test/profile/Linux/instrprof-value-merge.c
+++ b/compiler-rt/test/profile/Linux/instrprof-value-merge.c
@@ -45,7 +45,7 @@ int main(int argc, char *argv[]) {
 
 // CHECK: Counters:
 // CHECK:   main:
-// CHECK:     Hash: 0x00030012a7ab6e87
+// CHECK:     Hash: 0x0a9bd81e87ab6e87
 // CHECK:     Counters: 6
 // CHECK:     Indirect Call Site Count: 3
 // CHECK:     Number of Memory Intrinsics Calls: 3
diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
index 7a14f777b565a..5a0185600e068 100644
--- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
+++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
@@ -261,6 +261,10 @@ extern cl::opt<PGOViewCountsType> PGOViewCounts;
 // Defined in Analysis/BlockFrequencyInfo.cpp:  -view-bfi-func-name=
 extern cl::opt<std::string> ViewBlockFreqFuncName;
 
+static cl::opt<bool>
+    PGOOldCFGHashing("pgo-instr-old-cfg-hashing", cl::init(false), cl::Hidden,
+                     cl::desc("Use the old CFG function hashing"));
+
 // Return a string describing the branch condition that can be
 // used in static branch probability heuristics:
 static std::string getBranchCondString(Instruction *TI) {
@@ -620,7 +624,8 @@ template <class Edge, class BBInfo> class FuncPGOInstrumentation {
 } // end anonymous namespace
 
 // Compute Hash value for the CFG: the lower 32 bits are CRC32 of the index
-// value of each BB in the CFG. The higher 32 bits record the number of edges.
+// value of each BB in the CFG. The higher 32 bits are the CRC32 of the numbers
+// of selects, indirect calls, mem ops and edges.
 template <class Edge, class BBInfo>
 void FuncPGOInstrumentation<Edge, BBInfo>::computeCFGHash() {
   std::vector<uint8_t> Indexes;
@@ -639,12 +644,31 @@ void FuncPGOInstrumentation<Edge, BBInfo>::computeCFGHash() {
   }
   JC.update(Indexes);
 
-  // Hash format for context sensitive profile. Reserve 4 bits for other
-  // information.
-  FunctionHash = (uint64_t)SIVisitor.getNumOfSelectInsts() << 56 |
-                 (uint64_t)ValueSites[IPVK_IndirectCallTarget].size() << 48 |
-                 //(uint64_t)ValueSites[IPVK_MemOPSize].size() << 40 |
-                 (uint64_t)MST.AllEdges.size() << 32 | JC.getCRC();
+  JamCRC JCH;
+  if (PGOOldCFGHashing) {
+    // Hash format for context sensitive profile. Reserve 4 bits for other
+    // information.
+    FunctionHash = (uint64_t)SIVisitor.getNumOfSelectInsts() << 56 |
+                   (uint64_t)ValueSites[IPVK_IndirectCallTarget].size() << 48 |
+                   //(uint64_t)ValueSites[IPVK_MemOPSize].size() << 40 |
+                   (uint64_t)MST.AllEdges.size() << 32 | JC.getCRC();
+  } else {
+    // The higher 32 bits.
+    auto updateJCH = [&JCH](uint64_t Num) {
+      uint8_t Data[8];
+      support::endian::write64le(Data, Num);
+      JCH.update(Data);
+    };
+    updateJCH((uint64_t)SIVisitor.getNumOfSelectInsts());
+    updateJCH((uint64_t)ValueSites[IPVK_IndirectCallTarget].size());
+    updateJCH((uint64_t)ValueSites[IPVK_MemOPSize].size());
+    updateJCH((uint64_t)MST.AllEdges.size());
+
+    // Hash format for context sensitive profile. Reserve 4 bits for other
+    // information.
+    FunctionHash = (((uint64_t)JCH.getCRC()) << 28) + JC.getCRC();
+  }
+
   // Reserve bit 60-63 for other information purpose.
   FunctionHash &= 0x0FFFFFFFFFFFFFFF;
   if (IsCS)
@@ -653,8 +677,12 @@ void FuncPGOInstrumentation<Edge, BBInfo>::computeCFGHash() {
                     << " CRC = " << JC.getCRC()
                     << ", Selects = " << SIVisitor.getNumOfSelectInsts()
                     << ", Edges = " << MST.AllEdges.size() << ", ICSites = "
-                    << ValueSites[IPVK_IndirectCallTarget].size()
-                    << ", Hash = " << FunctionHash << "\n";);
+                    << ValueSites[IPVK_IndirectCallTarget].size());
+  if (!PGOOldCFGHashing) {
+    LLVM_DEBUG(dbgs() << ", Memops = " << ValueSites[IPVK_MemOPSize].size()
+                      << ", High32 CRC = " << JCH.getCRC());
+  }
+  LLVM_DEBUG(dbgs() << ", Hash = " << FunctionHash << "\n";);
 }
 
 // Check if we can safely rename this Comdat function.
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/PR41279.proftext b/llvm/test/Transforms/PGOProfile/Inputs/PR41279.proftext
index 58c3e89f089cc..ce657e720c571 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/PR41279.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/PR41279.proftext
@@ -1,6 +1,6 @@
 :ir
 foo
-60927483247
+1096621588030135663
 4
 3
 2
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/PR41279_2.proftext b/llvm/test/Transforms/PGOProfile/Inputs/PR41279_2.proftext
index adc16bbad92c3..c6cb02aaddd1d 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/PR41279_2.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/PR41279_2.proftext
@@ -1,6 +1,6 @@
 :ir
 f
-62077759478
+1096621589180411894
 2
 3
 2
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/branch1.proftext b/llvm/test/Transforms/PGOProfile/Inputs/branch1.proftext
index 8ca9db9c0515b..d5f9123fcc24f 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/branch1.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/branch1.proftext
@@ -1,7 +1,7 @@
 # :ir is the flag to indicate this is IR level profile.
 :ir
 test_br_1
-25571299074
+784007059655560962
 2
 3
 2
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/branch1_large_count.proftext b/llvm/test/Transforms/PGOProfile/Inputs/branch1_large_count.proftext
index a5b1f8204e808..4fd528ee076f2 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/branch1_large_count.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/branch1_large_count.proftext
@@ -1,7 +1,7 @@
 # :ir is the flag to indicate this is IR level profile.
 :ir
 test_br_1
-25571299074
+784007059655560962
 2
 12884901888
 8589934592
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/branch2.proftext b/llvm/test/Transforms/PGOProfile/Inputs/branch2.proftext
index b5fee2b6f4faf..3e20359b1b84f 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/branch2.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/branch2.proftext
@@ -1,7 +1,7 @@
 # :ir is the flag to indicate this is IR level profile.
 :ir
 test_br_2
-29667547796
+146835647075900052
 2
 1
 1
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/branch2_entry.proftext b/llvm/test/Transforms/PGOProfile/Inputs/branch2_entry.proftext
index 51de52d824926..31710fc8f4bde 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/branch2_entry.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/branch2_entry.proftext
@@ -2,7 +2,7 @@
 :ir
 :entry_first
 test_br_2
-29667547796
+146835647075900052
 2
 2
 1
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/criticaledge.proftext b/llvm/test/Transforms/PGOProfile/Inputs/criticaledge.proftext
index 7613b643b163e..85b9779abeece 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/criticaledge.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/criticaledge.proftext
@@ -1,7 +1,7 @@
 # :ir is the flag to indicate this is IR level profile.
 :ir
 test_criticalEdge
-82323253069
+93478046750287693
 8
 2
 1
@@ -13,7 +13,7 @@ test_criticalEdge
 1
 
 <stdin>:bar
-12884901887
+742261418966908927
 1
 7
 
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/criticaledge_entry.proftext b/llvm/test/Transforms/PGOProfile/Inputs/criticaledge_entry.proftext
index 79754261d2f88..f1497d6c01c9f 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/criticaledge_entry.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/criticaledge_entry.proftext
@@ -2,7 +2,7 @@
 :ir
 :entry_first
 test_criticalEdge
-82323253069
+93478046750287693
 8
 7
 2
@@ -14,7 +14,7 @@ test_criticalEdge
 1
 
 <stdin>:bar
-12884901887
+742261418966908927
 1
 7
 
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/cspgo.proftext b/llvm/test/Transforms/PGOProfile/Inputs/cspgo.proftext
index e40a0f29fccc0..6d6d86cfb50bb 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/cspgo.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/cspgo.proftext
@@ -54,7 +54,7 @@ bar_m2
 
 foo
 # Func Hash:
-1152921640672869708
+1456607294772657484
 # Num Counters:
 10
 # Counter Values:
@@ -71,7 +71,7 @@ foo
 
 foo
 # Func Hash:
-29212902728
+146835646621254984
 # Num Counters:
 2
 # Counter Values:
@@ -80,7 +80,7 @@ foo
 
 bar
 # Func Hash:
-1152921569533132113
+1440408129826749777
 # Num Counters:
 5
 # Counter Values:
@@ -92,7 +92,7 @@ bar
 
 bar
 # Func Hash:
-56228292833
+567185239050791137
 # Num Counters:
 4
 # Counter Values:
@@ -103,7 +103,7 @@ bar
 
 main
 # Func Hash:
-1152921517491748863
+1895182923573755903
 # Num Counters:
 1
 # Counter Values:
@@ -111,7 +111,7 @@ main
 
 main
 # Func Hash:
-12884901887
+742261418966908927
 # Num Counters:
 1
 # Counter Values:
@@ -135,7 +135,7 @@ csfdo_plain.c:barbar
 
 goo
 # Func Hash:
-1152921517491748863
+1895182923573755903
 # Num Counters:
 1
 # Counter Values:
@@ -143,7 +143,7 @@ goo
 
 goo
 # Func Hash:
-12884901887
+742261418966908927
 # Num Counters:
 1
 # Counter Values:
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/diag_no_value_sites.proftext b/llvm/test/Transforms/PGOProfile/Inputs/diag_no_value_sites.proftext
index b7ba652292ca9..182778755f4bb 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/diag_no_value_sites.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/diag_no_value_sites.proftext
@@ -1,6 +1,6 @@
 # :ir is the flag to indicate this is IR level profile.
 :ir
 foo
-12884901887
+48277136972185599
 1
 1
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/fix_entry_count.proftext b/llvm/test/Transforms/PGOProfile/Inputs/fix_entry_count.proftext
index de4771beaacc2..7ca97aa29dbcd 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/fix_entry_count.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/fix_entry_count.proftext
@@ -1,7 +1,7 @@
 :ir
 :entry_first
 test_simple_for
-34137660316
+1063705162469825436
 2
 0
 96
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/func_entry.proftext b/llvm/test/Transforms/PGOProfile/Inputs/func_entry.proftext
index 60c49bf730954..6d92958e89b87 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/func_entry.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/func_entry.proftext
@@ -3,7 +3,7 @@
 :entry_first
 hot
 # Func Hash:
-12884901887
+742261418966908927
 # Num Counters:
 1
 # Counter Values:
@@ -11,7 +11,7 @@ hot
 
 cold
 # Func Hash:
-12884901887
+742261418966908927
 # Num Counters:
 1
 # Counter Values:
@@ -19,7 +19,7 @@ cold
 
 med
 # Func Hash:
-12884901887
+742261418966908927
 # Num Counters:
 1
 # Counter Values:
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/indirect_call.proftext b/llvm/test/Transforms/PGOProfile/Inputs/indirect_call.proftext
index d453090d1c582..bb48ccc57a8bf 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/indirect_call.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/indirect_call.proftext
@@ -1,7 +1,7 @@
 :ir
 bar
 # Func Hash:
-281487861612543
+170957022131388415
 # Num Counters:
 1
 # Counter Values:
@@ -19,7 +19,7 @@ func3:20
 
 func1
 # Func Hash:
-12884901887
+742261418966908927
 # Num Counters:
 1
 # Counter Values:
@@ -27,7 +27,7 @@ func1
 
 func2
 # Func Hash:
-12884901887
+742261418966908927
 # Num Counters:
 1
 # Counter Values:
@@ -35,7 +35,7 @@ func2
 
 func3
 # Func Hash:
-12884901887
+742261418966908927
 # Num Counters:
 1
 # Counter Values:
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/indirectbr.proftext b/llvm/test/Transforms/PGOProfile/Inputs/indirectbr.proftext
index 3909968476272..49fafd9d99bf9 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/indirectbr.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/indirectbr.proftext
@@ -2,7 +2,7 @@
 :ir
 foo
 # Func Hash:
-47485104005
+844982796158316421
 # Num Counters:
 4
 # Counter Values:
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/indirectbr_entry.proftext b/llvm/test/Transforms/PGOProfile/Inputs/indirectbr_entry.proftext
index 7173269d20cc9..6910f7e21d677 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/indirectbr_entry.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/indirectbr_entry.proftext
@@ -3,7 +3,7 @@
 :entry_first
 foo
 # Func Hash:
-47485104005
+844982796158316421
 # Num Counters:
 4
 # Counter Values:
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/irreducible.proftext b/llvm/test/Transforms/PGOProfile/Inputs/irreducible.proftext
index 9b0210d9a309f..24eacb886489e 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/irreducible.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/irreducible.proftext
@@ -1,7 +1,7 @@
 :ir
 _Z11irreducibleii
 # Func Hash:
-64451410787
+287486624745028451
 # Num Counters:
 6
 # Counter Values:
@@ -14,7 +14,7 @@ _Z11irreducibleii
 
 _Z11irreduciblePh
 # Func Hash:
-104649601521
+331779889035882993
 # Num Counters:
 9
 # Counter Values:
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/irreducible_entry.proftext b/llvm/test/Transforms/PGOProfile/Inputs/irreducible_entry.proftext
index d04cfddda896d..702c42d1addc8 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/irreducible_entry.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/irreducible_entry.proftext
@@ -2,7 +2,7 @@
 :entry_first
 _Z11irreducibleii
 # Func Hash:
-64451410787
+287486624745028451
 # Num Counters:
 6
 # Counter Values:
@@ -15,7 +15,7 @@ _Z11irreducibleii
 
 _Z11irreduciblePh
 # Func Hash:
-104649601521
+331779889035882993
 # Num Counters:
 9
 # Counter Values:
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/landingpad.proftext b/llvm/test/Transforms/PGOProfile/Inputs/landingpad.proftext
index c71fd04170bee..7a74b296e55fe 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/landingpad.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/landingpad.proftext
@@ -1,7 +1,7 @@
 # :ir is the flag to indicate this is IR level profile.
 :ir
 foo
-59130013419
+567185241952511723
 4
 3
 1
@@ -9,7 +9,7 @@ foo
 0
 
 bar
-24868915205
+784007058953177093
 2
 3
 2
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/landingpad_entry.proftext b/llvm/test/Transforms/PGOProfile/Inputs/landingpad_entry.proftext
index 3175d9c0d630b..9b604038abcc4 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/landingpad_entry.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/landingpad_entry.proftext
@@ -2,7 +2,7 @@
 :ir
 :entry_first
 foo
-59130013419
+567185241952511723
 4
 5
 1
@@ -10,7 +10,7 @@ foo
 0
 
 bar
-24868915205
+784007058953177093
 2
 3
 2
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/large_count_remarks.proftext b/llvm/test/Transforms/PGOProfile/Inputs/large_count_remarks.proftext
index 814af7402b28a..8e23ae6906b0c 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/large_count_remarks.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/large_count_remarks.proftext
@@ -1,7 +1,7 @@
 # :ir is the flag to indicate this is IR level profile.
 :ir
 test
-25571299074
+784007059655560962
 2
 40000000000
 20000000000
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/loop1.proftext b/llvm/test/Transforms/PGOProfile/Inputs/loop1.proftext
index c19737149ca25..25eeefe33c11f 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/loop1.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/loop1.proftext
@@ -1,7 +1,7 @@
 # :ir is the flag to indicate this is IR level profile.
 :ir
 test_simple_for
-34137660316
+1063705162469825436
 2
 96
 4
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/loop1_entry.proftext b/llvm/test/Transforms/PGOProfile/Inputs/loop1_entry.proftext
index b61634d94689d..56f04e808aee8 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/loop1_entry.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/loop1_entry.proftext
@@ -2,7 +2,7 @@
 :ir
 :entry_first
 test_simple_for
-34137660316
+1063705162469825436
 2
 4
 96
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/loop2.proftext b/llvm/test/Transforms/PGOProfile/Inputs/loop2.proftext
index af3a71df1e6b9..7ab9cd6b946e4 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/loop2.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/loop2.proftext
@@ -1,7 +1,7 @@
 # :ir is the flag to indicate this is IR level profile.
 :ir
 test_nested_for
-53929068288
+798733566382720768
 3
 33
 10
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/loop2_entry.proftext b/llvm/test/Transforms/PGOProfile/Inputs/loop2_entry.proftext
index ac8536a52675b..bde1b7e4a4c12 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/loop2_entry.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/loop2_entry.proftext
@@ -2,7 +2,7 @@
 :ir
 :entry_first
 test_nested_for
-53929068288
+798733566382720768
 3
 6
 33
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/memop_size_annotation.proftext b/llvm/test/Transforms/PGOProfile/Inputs/memop_size_annotation.proftext
index cce1a67a94574..83aec54f23283 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/memop_size_annotation.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/memop_size_annotation.proftext
@@ -2,7 +2,7 @@
 :ir
 foo
 # Func Hash:
-53929068288
+687116424982578944
 # Num Counters:
 3
 # Counter Values:
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/misexpect-branch-correct.proftext b/llvm/test/Transforms/PGOProfile/Inputs/misexpect-branch-correct.proftext
index c785a40efea59..24c9746262bda 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/misexpect-branch-correct.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/misexpect-branch-correct.proftext
@@ -2,7 +2,7 @@
 :ir
 bar
 # Func Hash:
-29667547796
+146835647075900052
 # Num Counters:
 2
 # Counter Values:
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/misexpect-branch.proftext b/llvm/test/Transforms/PGOProfile/Inputs/misexpect-branch.proftext
index 36eaa3462782f..5505df7375271 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/misexpect-branch.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/misexpect-branch.proftext
@@ -2,7 +2,7 @@
 :ir
 bar
 # Func Hash:
-29667547796
+146835647075900052
 # Num Counters:
 2
 # Counter Values:
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/misexpect-branch_entry.proftext b/llvm/test/Transforms/PGOProfile/Inputs/misexpect-branch_entry.proftext
index df7d6cf80466b..752b885ad2b2c 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/misexpect-branch_entry.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/misexpect-branch_entry.proftext
@@ -3,7 +3,7 @@
 :entry_first
 bar
 # Func Hash:
-29667547796
+146835647075900052
 # Num Counters:
 2
 # Counter Values:
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/misexpect-switch-correct.proftext b/llvm/test/Transforms/PGOProfile/Inputs/misexpect-switch-correct.proftext
index b0924938ea831..39864486bb96b 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/misexpect-switch-correct.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/misexpect-switch-correct.proftext
@@ -2,7 +2,7 @@
 :ir
 main
 # Func Hash:
-74054140268
+391331300939170156
 # Num Counters:
 7
 # Counter Values:
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/misexpect-switch-correct_entry.proftext b/llvm/test/Transforms/PGOProfile/Inputs/misexpect-switch-correct_entry.proftext
index 4686fec3bee78..c74f3230939ee 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/misexpect-switch-correct_entry.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/misexpect-switch-correct_entry.proftext
@@ -3,7 +3,7 @@
 :entry_first
 main
 # Func Hash:
-74054140268
+391331300939170156
 # Num Counters:
 7
 # Counter Values:
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/misexpect-switch.proftext b/llvm/test/Transforms/PGOProfile/Inputs/misexpect-switch.proftext
index 8e8049f3dc62e..ea50f6b6043f4 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/misexpect-switch.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/misexpect-switch.proftext
@@ -2,7 +2,7 @@
 :ir
 main
 # Func Hash:
-74054140268
+391331300939170156
 # Num Counters:
 7
 # Counter Values:
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/misexpect-switch_entry.proftext b/llvm/test/Transforms/PGOProfile/Inputs/misexpect-switch_entry.proftext
index 3da4c10ac19d1..6afb6c9019f24 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/misexpect-switch_entry.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/misexpect-switch_entry.proftext
@@ -3,7 +3,7 @@
 :entry_first
 main
 # Func Hash:
-74054140268
+391331300939170156
 # Num Counters:
 7
 # Counter Values:
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/multiple_hash_profile.proftext b/llvm/test/Transforms/PGOProfile/Inputs/multiple_hash_profile.proftext
index 5bf67fb2bfaf2..77f8d5a5ade3b 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/multiple_hash_profile.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/multiple_hash_profile.proftext
@@ -2,13 +2,23 @@
 :ir
 _Z3fooi
 # Func Hash:
-72057606922829823
+382993475055910911
 # Num Counters:
 2
 # Counter Values:
 18
 12
 
+# For -pgo-instr-old-cfg-hashing=true
+_Z3fooi
+# Func Hash:
+72057606922829823
+# Num Counters:
+2
+# Counter Values:
+18
+6
+
 _Z3fooi
 # Func Hash:
 12884901887
@@ -17,6 +27,16 @@ _Z3fooi
 # Counter Values:
 0
 
+_Z3bari
+# Func Hash:
+382993475055910911
+# Num Counters:
+2
+# Counter Values:
+0
+0
+
+# For -pgo-instr-old-cfg-hashing=true
 _Z3bari
 # Func Hash:
 72057606922829823
@@ -28,9 +48,17 @@ _Z3bari
 
 _Z4m2f1v
 # Func Hash:
-12884901887
+742261418966908927
 # Num Counters:
 1
 # Counter Values:
 1
 
+# For -pgo-instr-old-cfg-hashing=true
+_Z4m2f1v
+# Func Hash:
+12884901887
+# Num Counters:
+1
+# Counter Values:
+1
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/noreturncall.proftext b/llvm/test/Transforms/PGOProfile/Inputs/noreturncall.proftext
index 1bedce5b9f9b3..bb4b0719cf129 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/noreturncall.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/noreturncall.proftext
@@ -2,7 +2,7 @@
 :ir
 foo
 # Func Hash:
-36496524737
+238984482720105921
 # Num Counters:
 3
 # Counter Values:
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/remap.proftext b/llvm/test/Transforms/PGOProfile/Inputs/remap.proftext
index 40054d78f5aab..d9b22637d3a7d 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/remap.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/remap.proftext
@@ -1,7 +1,7 @@
 # :ir is the flag to indicate this is IR level profile.
 :ir
 _ZN3foo3barERKN1N1XINS_4quuxEEE
-25571299074
+784007059655560962
 2
 3
 2
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/select1.proftext b/llvm/test/Transforms/PGOProfile/Inputs/select1.proftext
index 60663e02ffb49..c4d6d56073b01 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/select1.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/select1.proftext
@@ -1,7 +1,7 @@
 :ir
 :entry_first
 test_br_2
-72057623705475732
+942389667449461396
 3
 5
 1
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/select2.proftext b/llvm/test/Transforms/PGOProfile/Inputs/select2.proftext
index 8f3c562097fae..0d4b003ec0667 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/select2.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/select2.proftext
@@ -3,7 +3,7 @@
 :entry_first
 foo
 # Func Hash:
-72057628175588252
+134732432632142748
 # Num Counters:
 3
 # Counter Values:
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/suppl-profile.proftext b/llvm/test/Transforms/PGOProfile/Inputs/suppl-profile.proftext
index c82311b3e0c06..41f4fbe655fac 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/suppl-profile.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/suppl-profile.proftext
@@ -1,7 +1,7 @@
 # :ir is the flag to indicate this is IR level profile.
 :ir
 test_simple_for
-34137660316
+1063705162469825436
 2
 0
 0
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/switch.proftext b/llvm/test/Transforms/PGOProfile/Inputs/switch.proftext
index bebd65fa4c9fd..cf73ed083794e 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/switch.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/switch.proftext
@@ -1,7 +1,7 @@
 # :ir is the flag to indicate this is IR level profile.
 :ir
 test_switch
-46200943743
+536873293052540031
 4
 0
 5
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/switch_entry.proftext b/llvm/test/Transforms/PGOProfile/Inputs/switch_entry.proftext
index 20df873608c19..cf079893e3404 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/switch_entry.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/switch_entry.proftext
@@ -2,7 +2,7 @@
 :ir
 :entry_first
 test_switch
-46200943743
+536873293052540031
 4
 10
 5
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/thinlto_cs.proftext b/llvm/test/Transforms/PGOProfile/Inputs/thinlto_cs.proftext
index 4717752da716d..1b9f19e7f7fa5 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/thinlto_cs.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/thinlto_cs.proftext
@@ -10,7 +10,7 @@ cond.llvm.11253644763537639171
 
 foo
 # Func Hash:
-29212902728
+1720106746050921044
 # Num Counters:
 2
 # Counter Values:
@@ -19,7 +19,7 @@ foo
 
 bar
 # Func Hash:
-1152921534274394772
+1299757151682747028
 # Num Counters:
 2
 # Counter Values:
@@ -45,7 +45,7 @@ main
 
 main
 # Func Hash:
-12884901887
+1895182923573755903
 # Num Counters:
 1
 # Counter Values:
@@ -53,7 +53,7 @@ main
 
 cspgo.c:foo
 # Func Hash:
-1152921563228422740
+1720106746050921044
 # Num Counters:
 4
 # Counter Values:
diff --git a/llvm/test/Transforms/PGOProfile/multiple_hash_profile.ll b/llvm/test/Transforms/PGOProfile/multiple_hash_profile.ll
index fdfb16b9e2e55..f0011114f0d01 100644
--- a/llvm/test/Transforms/PGOProfile/multiple_hash_profile.ll
+++ b/llvm/test/Transforms/PGOProfile/multiple_hash_profile.ll
@@ -1,6 +1,8 @@
 ; RUN: llvm-profdata merge %S/Inputs/multiple_hash_profile.proftext -o %t.profdata
 ; RUN: opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata  -S | FileCheck %s
+; RUN: opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata -pgo-instr-old-cfg-hashing=true -S | FileCheck -check-prefix=CHECKOLDHASH %s
 ; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s
+; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -pgo-instr-old-cfg-hashing=true -S | FileCheck -check-prefix=CHECKOLDHASH %s
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
@@ -29,6 +31,9 @@ entry:
 ; CHECK: %mul.i = select i1 %cmp.i, i32 1, i32 %i
 ; CHECK-SAME: !prof ![[BW:[0-9]+]]
 ; CHECK: ![[BW]] = !{!"branch_weights", i32 12, i32 6}
+; CHECKOLDHASH: %mul.i = select i1 %cmp.i, i32 1, i32 %i
+; CHECKOLDHASH-SAME: !prof ![[BW:[0-9]+]]
+; CHECKOLDHASH: ![[BW]] = !{!"branch_weights", i32 6, i32 12}
   %retval.0.i = mul nsw i32 %mul.i, %i
   ret i32 %retval.0.i
 }

From 1e8a9c3e026f4523eb9745d3a3f4c13f934dcbec Mon Sep 17 00:00:00 2001
From: Kuba Mracek <mracek@apple.com>
Date: Thu, 30 Jul 2020 09:32:11 -0700
Subject: [PATCH 0639/1035] [tsan] Fixup for 1260a155: Move variadic-open.cpp
 test into Darwin/ directory

---
 compiler-rt/test/tsan/{ => Darwin}/variadic-open.cpp | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename compiler-rt/test/tsan/{ => Darwin}/variadic-open.cpp (100%)

diff --git a/compiler-rt/test/tsan/variadic-open.cpp b/compiler-rt/test/tsan/Darwin/variadic-open.cpp
similarity index 100%
rename from compiler-rt/test/tsan/variadic-open.cpp
rename to compiler-rt/test/tsan/Darwin/variadic-open.cpp

From e263dcc3efeae419b691730da5e37977e1a681f6 Mon Sep 17 00:00:00 2001
From: cgyurgyik <gyurgyikcp@gmail.com>
Date: Thu, 30 Jul 2020 12:36:28 -0400
Subject: [PATCH 0640/1035] [libc] Implements isdigit and isalnum. Adds a
 utility header to inline functions to avoid overhead of function calls.

Reviewed By: sivachandra

Differential Revision: https://reviews.llvm.org/D84893
---
 libc/config/linux/aarch64/entrypoints.txt |  2 ++
 libc/config/linux/api.td                  |  2 ++
 libc/config/linux/x86_64/entrypoints.txt  |  2 ++
 libc/spec/stdc.td                         | 10 +++++++
 libc/src/ctype/CMakeLists.txt             | 28 +++++++++++++++++++
 libc/src/ctype/ctype_utils.h              | 34 +++++++++++++++++++++++
 libc/src/ctype/isalnum.cpp                | 22 +++++++++++++++
 libc/src/ctype/isalnum.h                  | 18 ++++++++++++
 libc/src/ctype/isalpha.cpp                |  6 ++--
 libc/src/ctype/isdigit.cpp                | 19 +++++++++++++
 libc/src/ctype/isdigit.h                  | 18 ++++++++++++
 libc/test/src/ctype/CMakeLists.txt        | 20 +++++++++++++
 libc/test/src/ctype/isalnum_test.cpp      | 27 ++++++++++++++++++
 libc/test/src/ctype/isdigit_test.cpp      | 26 +++++++++++++++++
 14 files changed, 230 insertions(+), 4 deletions(-)
 create mode 100644 libc/src/ctype/ctype_utils.h
 create mode 100644 libc/src/ctype/isalnum.cpp
 create mode 100644 libc/src/ctype/isalnum.h
 create mode 100644 libc/src/ctype/isdigit.cpp
 create mode 100644 libc/src/ctype/isdigit.h
 create mode 100644 libc/test/src/ctype/isalnum_test.cpp
 create mode 100644 libc/test/src/ctype/isdigit_test.cpp

diff --git a/libc/config/linux/aarch64/entrypoints.txt b/libc/config/linux/aarch64/entrypoints.txt
index 7e3589bcf0ecc..8d2cc16189465 100644
--- a/libc/config/linux/aarch64/entrypoints.txt
+++ b/libc/config/linux/aarch64/entrypoints.txt
@@ -1,6 +1,8 @@
 set(TARGET_LIBC_ENTRYPOINTS
     # ctype.h entrypoints
+    libc.src.ctype.isalnum
     libc.src.ctype.isalpha
+    libc.src.ctype.isdigit
     
     # errno.h entrypoints
     libc.src.errno.__errno_location
diff --git a/libc/config/linux/api.td b/libc/config/linux/api.td
index aea3fed1b7c3a..01e6eab92f016 100644
--- a/libc/config/linux/api.td
+++ b/libc/config/linux/api.td
@@ -88,7 +88,9 @@ def AssertAPI : PublicAPI<"assert.h"> {
 
 def CTypeAPI : PublicAPI<"ctype.h"> {
   let Functions = [
+    "isalnum",
     "isalpha",
+    "isdigit",
   ];
 }
 
diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt
index c361fce3dbec0..4484f6cbf1a79 100644
--- a/libc/config/linux/x86_64/entrypoints.txt
+++ b/libc/config/linux/x86_64/entrypoints.txt
@@ -3,7 +3,9 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.assert.__assert_fail
     
     # ctype.h entrypoints
+    libc.src.ctype.isalnum
     libc.src.ctype.isalpha
+    libc.src.ctype.isdigit
 
     # errno.h entrypoints
     libc.src.errno.__errno_location
diff --git a/libc/spec/stdc.td b/libc/spec/stdc.td
index 1b93960c9a4f0..fed24cb2dbdd3 100644
--- a/libc/spec/stdc.td
+++ b/libc/spec/stdc.td
@@ -46,11 +46,21 @@ def StdC : StandardSpec<"stdc"> {
       [], // Types
       [], // Enumerations
       [
+          FunctionSpec<
+              "isalnum",
+              RetValSpec<IntType>,
+              [ArgSpec<IntType>]
+          >,
           FunctionSpec<
               "isalpha",
               RetValSpec<IntType>,
               [ArgSpec<IntType>]
           >,
+          FunctionSpec<
+              "isdigit",
+              RetValSpec<IntType>,
+              [ArgSpec<IntType>]
+          >,
       ]
   >;
      
diff --git a/libc/src/ctype/CMakeLists.txt b/libc/src/ctype/CMakeLists.txt
index 686ec78785123..53161b8b179fd 100644
--- a/libc/src/ctype/CMakeLists.txt
+++ b/libc/src/ctype/CMakeLists.txt
@@ -1,7 +1,35 @@
+add_header_library(
+  ctype_utils
+  HDRS
+    ctype_utils.h
+)
+
+add_entrypoint_object(
+  isalnum
+  SRCS
+    isalnum.cpp
+  HDRS
+    isalnum.h
+  DEPENDS
+    .ctype_utils
+)
+
 add_entrypoint_object(
   isalpha
   SRCS
     isalpha.cpp
   HDRS
     isalpha.h
+  DEPENDS
+    .ctype_utils
+)
+
+add_entrypoint_object(
+  isdigit
+  SRCS
+    isdigit.cpp
+  HDRS
+    isdigit.h
+  DEPENDS
+    .ctype_utils
 )
diff --git a/libc/src/ctype/ctype_utils.h b/libc/src/ctype/ctype_utils.h
new file mode 100644
index 0000000000000..4e8d3960bb704
--- /dev/null
+++ b/libc/src/ctype/ctype_utils.h
@@ -0,0 +1,34 @@
+//===-- Collection of utils for implementing ctype functions-------*-C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_CTYPE_CTYPE_UTILS_H
+#define LLVM_LIBC_SRC_CTYPE_CTYPE_UTILS_H
+
+namespace __llvm_libc {
+namespace internal {
+
+// ------------------------------------------------------
+// Rationale: Since these classification functions are
+// called in other functions, we will avoid the overhead
+// of a function call by inlining them.
+// ------------------------------------------------------
+
+static inline int isdigit(int c) {
+  const unsigned ch = c;
+  return (ch - '0') < 10;
+}
+
+static inline int isalpha(int c) {
+  const unsigned ch = c;
+  return (ch | 32) - 'a' < 26;
+}
+
+} // namespace internal
+} // namespace __llvm_libc
+
+#endif //  LLVM_LIBC_SRC_CTYPE_CTYPE_UTILS_H
diff --git a/libc/src/ctype/isalnum.cpp b/libc/src/ctype/isalnum.cpp
new file mode 100644
index 0000000000000..08b6520e44267
--- /dev/null
+++ b/libc/src/ctype/isalnum.cpp
@@ -0,0 +1,22 @@
+//===-- Implementation of isalnum------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/ctype/isalnum.h"
+#include "src/ctype/ctype_utils.h"
+
+#include "src/__support/common.h"
+
+namespace __llvm_libc {
+
+// TODO: Currently restricted to default locale.
+// These should be extended using locale information.
+int LLVM_LIBC_ENTRYPOINT(isalnum)(int c) {
+  return internal::isalpha(c) || internal::isdigit(c);
+}
+
+} // namespace __llvm_libc
diff --git a/libc/src/ctype/isalnum.h b/libc/src/ctype/isalnum.h
new file mode 100644
index 0000000000000..beb202a7529da
--- /dev/null
+++ b/libc/src/ctype/isalnum.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for isalnum -------------------------*-C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_CTYPE_ISALNUM_H
+#define LLVM_LIBC_SRC_CTYPE_ISALNUM_H
+
+namespace __llvm_libc {
+
+int isalnum(int c);
+
+} // namespace __llvm_libc
+
+#endif //  LLVM_LIBC_SRC_CTYPE_ISALNUM_H
diff --git a/libc/src/ctype/isalpha.cpp b/libc/src/ctype/isalpha.cpp
index e945ca98ffb1c..4b3725477a7d4 100644
--- a/libc/src/ctype/isalpha.cpp
+++ b/libc/src/ctype/isalpha.cpp
@@ -9,14 +9,12 @@
 #include "src/ctype/isalpha.h"
 
 #include "src/__support/common.h"
+#include "src/ctype/ctype_utils.h"
 
 namespace __llvm_libc {
 
 // TODO: Currently restricted to default locale.
 // These should be extended using locale information.
-int LLVM_LIBC_ENTRYPOINT(isalpha)(int c) {
-  const unsigned ch = c;
-  return (ch | 32) - 'a' < 26;
-}
+int LLVM_LIBC_ENTRYPOINT(isalpha)(int c) { return internal::isalpha(c); }
 
 } // namespace __llvm_libc
diff --git a/libc/src/ctype/isdigit.cpp b/libc/src/ctype/isdigit.cpp
new file mode 100644
index 0000000000000..94ec42a0a3d8d
--- /dev/null
+++ b/libc/src/ctype/isdigit.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of isdigit------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/ctype/isdigit.h"
+#include "src/__support/common.h"
+#include "src/ctype/ctype_utils.h"
+
+namespace __llvm_libc {
+
+// TODO: Currently restricted to default locale.
+// These should be extended using locale information.
+int LLVM_LIBC_ENTRYPOINT(isdigit)(int c) { return internal::isdigit(c); }
+
+} // namespace __llvm_libc
diff --git a/libc/src/ctype/isdigit.h b/libc/src/ctype/isdigit.h
new file mode 100644
index 0000000000000..32a76235e0592
--- /dev/null
+++ b/libc/src/ctype/isdigit.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for isdigit -------------------------*-C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_CTYPE_ISDIGIT_H
+#define LLVM_LIBC_SRC_CTYPE_ISDIGIT_H
+
+namespace __llvm_libc {
+
+int isdigit(int c);
+
+} // namespace __llvm_libc
+
+#endif //  LLVM_LIBC_SRC_CTYPE_ISDIGIT_H
diff --git a/libc/test/src/ctype/CMakeLists.txt b/libc/test/src/ctype/CMakeLists.txt
index b7572d3c5bfad..7834746ab1d73 100644
--- a/libc/test/src/ctype/CMakeLists.txt
+++ b/libc/test/src/ctype/CMakeLists.txt
@@ -1,5 +1,15 @@
 add_libc_testsuite(libc_ctype_unittests)
 
+add_libc_unittest(
+  isalnum
+  SUITE
+    libc_ctype_unittests
+  SRCS
+    isalnum_test.cpp
+  DEPENDS
+    libc.src.ctype.isalnum
+)
+
 add_libc_unittest(
   isalpha
   SUITE
@@ -9,3 +19,13 @@ add_libc_unittest(
   DEPENDS
     libc.src.ctype.isalpha
 )
+
+add_libc_unittest(
+  isdigit
+  SUITE
+    libc_ctype_unittests
+  SRCS
+    isdigit_test.cpp
+  DEPENDS
+    libc.src.ctype.isdigit
+)
diff --git a/libc/test/src/ctype/isalnum_test.cpp b/libc/test/src/ctype/isalnum_test.cpp
new file mode 100644
index 0000000000000..1c4ad7d3ff8ac
--- /dev/null
+++ b/libc/test/src/ctype/isalnum_test.cpp
@@ -0,0 +1,27 @@
+//===-- Unittests for isalnum----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/ctype/isalnum.h"
+#include "utils/UnitTest/Test.h"
+
+// Helper function that makes a call to isalnum a bit cleaner
+// for use with testing utilities, since it explicitly requires
+// a boolean value for EXPECT_TRUE and EXPECT_FALSE.
+bool call_isalnum(int c) { return __llvm_libc::isalnum(c); }
+
+TEST(IsAlNum, DefaultLocale) {
+  // Loops through all characters, verifying that numbers and letters
+  // return true and everything else returns false.
+  for (int c = 0; c < 255; ++c) {
+    if (('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') ||
+        ('0' <= c && c <= '9'))
+      EXPECT_TRUE(call_isalnum(c));
+    else
+      EXPECT_FALSE(call_isalnum(c));
+  }
+}
diff --git a/libc/test/src/ctype/isdigit_test.cpp b/libc/test/src/ctype/isdigit_test.cpp
new file mode 100644
index 0000000000000..6fea9564db674
--- /dev/null
+++ b/libc/test/src/ctype/isdigit_test.cpp
@@ -0,0 +1,26 @@
+//===-- Unittests for isdigit----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/ctype/isdigit.h"
+#include "utils/UnitTest/Test.h"
+
+// Helper function that makes a call to isdigit a bit cleaner
+// for use with testing utilities, since it explicitly requires
+// a boolean value for EXPECT_TRUE and EXPECT_FALSE.
+bool call_isdigit(int c) { return __llvm_libc::isdigit(c); }
+
+TEST(IsDigit, DefaultLocale) {
+  // Loops through all characters, verifying that numbers return true
+  // and everything else returns false.
+  for (int ch = 0; ch < 255; ++ch) {
+    if ('0' <= ch && ch <= '9')
+      EXPECT_TRUE(call_isdigit(ch));
+    else
+      EXPECT_FALSE(call_isdigit(ch));
+  }
+}

From 09cb6f233d3d18cb17ba00188cbc32a414ec0c82 Mon Sep 17 00:00:00 2001
From: Jordan Rupprecht <rupprecht@google.com>
Date: Wed, 29 Jul 2020 16:58:29 -0700
Subject: [PATCH 0641/1035] [lldb][NFC][test] Fix comment referring to
 FileCheck instead of yaml2obj

---
 lldb/packages/Python/lldbsuite/test/lldbtest.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lldb/packages/Python/lldbsuite/test/lldbtest.py b/lldb/packages/Python/lldbsuite/test/lldbtest.py
index 4aad2867cf8f4..e4bf6eb569cf3 100644
--- a/lldb/packages/Python/lldbsuite/test/lldbtest.py
+++ b/lldb/packages/Python/lldbsuite/test/lldbtest.py
@@ -1608,7 +1608,7 @@ def yaml2obj(self, yaml_path, obj_path):
         """
         yaml2obj_bin = configuration.get_yaml2obj_path()
         if not yaml2obj_bin:
-            self.assertTrue(False, "No valid FileCheck executable specified")
+            self.assertTrue(False, "No valid yaml2obj executable specified")
         command = [yaml2obj_bin, "-o=%s" % obj_path, yaml_path]
         system([command])
 

From 4d83aa4771d84940626d86c883193af390812281 Mon Sep 17 00:00:00 2001
From: Johannes Doerfert <johannes@jdoerfert.de>
Date: Thu, 30 Jul 2020 11:53:06 -0500
Subject: [PATCH 0642/1035] [MLIR][OpenMP] Fix OpenMPIRBuilder usage after
 D82470

---
 mlir/lib/Target/LLVMIR/ModuleTranslation.cpp | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
index a0aefc988a5da..2c61f2a4ac11b 100644
--- a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
@@ -456,8 +456,12 @@ ModuleTranslation::convertOmpParallel(Operation &opInst,
   llvm::Value *ifCond = nullptr;
   llvm::Value *numThreads = nullptr;
   bool isCancellable = false;
+  // TODO: Determine the actual alloca insertion point, e.g., the function
+  // entry or the alloca insertion point as provided by the body callback
+  // above.
+  llvm::OpenMPIRBuilder::InsertPointTy allocaIP(builder.saveIP());
   builder.restoreIP(ompBuilder->CreateParallel(
-      builder, bodyGenCB, privCB, finiCB, ifCond, numThreads,
+      builder, allocaIP, bodyGenCB, privCB, finiCB, ifCond, numThreads,
       llvm::omp::OMP_PROC_BIND_default, isCancellable));
   return success();
 }

From 3632f765dc6a95a903864585056d1caa9185bd00 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@intel.com>
Date: Thu, 30 Jul 2020 09:54:02 -0700
Subject: [PATCH 0643/1035] [WebAssembly] Fix GCC 5 build.

Hans' speculative fix in b7292f2db02d37c9291afc0613a3fbce0a4ad4e8
didn't work for me. This seems to.
---
 llvm/lib/Target/WebAssembly/WebAssemblySortRegion.cpp | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/Target/WebAssembly/WebAssemblySortRegion.cpp b/llvm/lib/Target/WebAssembly/WebAssemblySortRegion.cpp
index 6b3957e3ec557..cd84e68aed140 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblySortRegion.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblySortRegion.cpp
@@ -5,10 +5,14 @@
 using namespace llvm;
 using namespace WebAssembly;
 
+namespace llvm {
+namespace WebAssembly {
 template <>
-bool llvm::WebAssembly::ConcreteSortRegion<MachineLoop>::isLoop() const {
+bool ConcreteSortRegion<MachineLoop>::isLoop() const {
   return true;
 }
+} // end namespace WebAssembly
+} // end namespace llvm
 
 const SortRegion *SortRegionInfo::getRegionFor(const MachineBasicBlock *MBB) {
   const auto *ML = MLI.getLoopFor(MBB);

From 555cf42f380d86f35e761c3a2179c761356ab152 Mon Sep 17 00:00:00 2001
From: Yuanfang Chen <yuanfang.chen@sony.com>
Date: Tue, 28 Jul 2020 17:08:24 -0700
Subject: [PATCH 0644/1035] [NewPM][PassInstrument] Add PrintPass callback to
 StandardInstrumentations

Problem:
Right now, our "Running pass" is not accurate when passes are wrapped in adaptor because adaptor is never skipped and a pass could be skipped. The other problem is that "Running pass" for a adaptor is before any "Running pass" of passes/analyses it depends on. (for example, FunctionToLoopPassAdaptor). So the order of printing is not the actual order.

Solution:
Doing things like PassManager::Debuglogging is very intrusive because we need to specify Debuglogging whenever adaptor is created. (Actually, right now we're not specifying Debuglogging for some sub-PassManagers. Check PassBuilder)

This patch move debug logging for pass as a PassInstrument callback. We could be sure that all running passes are logged and in the correct order.

This could also be used to implement hierarchy pass logging in legacy PM. We could also move logging of pass manager to this if we want.

The test fixes looks messy. It includes changes:
- Remove PassInstrumentationAnalysis
- Remove PassAdaptor
- If a PassAdaptor is for a real pass, the pass is added
- Pass reorder (to the correct order), related to PassAdaptor
- Add missing passes (due to Debuglogging not passed down)

Reviewed By: asbirlea, aeubanks

Differential Revision: https://reviews.llvm.org/D84774
---
 .../test/CodeGen/thinlto-distributed-newpm.ll | 37 ++++-----
 llvm/include/llvm/IR/PassInstrumentation.h    |  2 +
 llvm/include/llvm/IR/PassManager.h            |  4 -
 llvm/include/llvm/IR/PassManagerImpl.h        |  3 -
 .../llvm/Passes/StandardInstrumentations.h    | 12 ++-
 llvm/lib/Analysis/CGSCCPassManager.cpp        |  3 -
 llvm/lib/IR/PassInstrumentation.cpp           |  9 +++
 llvm/lib/IR/PassTimingInfo.cpp                | 15 +---
 llvm/lib/Passes/StandardInstrumentations.cpp  | 76 ++++++++++++++++---
 .../lib/Transforms/Scalar/LoopPassManager.cpp |  3 -
 llvm/test/Other/loop-pm-invalidation.ll       | 46 +++--------
 llvm/test/Other/new-pass-manager.ll           | 15 ----
 llvm/test/Other/new-pm-defaults.ll            | 26 ++-----
 llvm/test/Other/new-pm-lto-defaults.ll        | 33 ++++----
 llvm/test/Other/new-pm-pgo.ll                 |  6 +-
 llvm/test/Other/new-pm-thinlto-defaults.ll    | 31 ++------
 .../new-pm-thinlto-postlink-pgo-defaults.ll   | 34 +++------
 ...-pm-thinlto-postlink-samplepgo-defaults.ll | 28 ++-----
 .../new-pm-thinlto-prelink-pgo-defaults.ll    | 29 ++-----
 ...w-pm-thinlto-prelink-samplepgo-defaults.ll | 22 ++----
 llvm/test/Other/pass-pipeline-parsing.ll      | 25 +++---
 llvm/test/Transforms/LoopRotate/pr35210.ll    |  4 -
 llvm/test/Transforms/LoopUnroll/revisit.ll    |  4 +-
 .../LoopUnroll/unroll-loop-invalidation.ll    | 12 ++-
 .../SCCP/ipsccp-preserve-analysis.ll          |  1 -
 25 files changed, 196 insertions(+), 284 deletions(-)

diff --git a/clang/test/CodeGen/thinlto-distributed-newpm.ll b/clang/test/CodeGen/thinlto-distributed-newpm.ll
index 6269fd677ca82..3dcc6a196365c 100644
--- a/clang/test/CodeGen/thinlto-distributed-newpm.ll
+++ b/clang/test/CodeGen/thinlto-distributed-newpm.ll
@@ -1,3 +1,4 @@
+; FIXME: This test should use CHECK-NEXT to keep up-to-date.
 ; REQUIRES: x86-registered-target
 
 ; Validate ThinLTO post link pipeline at O2 and O3
@@ -18,7 +19,6 @@
 ; RUN:   -c -fthinlto-index=%t.o.thinlto.bc \
 ; RUN:   -o %t.native.o -x ir %t.o 2>&1 | FileCheck -check-prefixes=CHECK-O,CHECK-O3 %s --dump-input=fail
 
-; CHECK-O: Running analysis: PassInstrumentationAnalysis
 ; CHECK-O: Starting {{.*}}Module pass manager run.
 ; CHECK-O: Running pass: WholeProgramDevirtPass
 ; CHECK-O: Running analysis: InnerAnalysisManagerProxy
@@ -26,15 +26,12 @@
 ; CHECK-O: Invalidating all non-preserved analyses for:
 ; CHECK-O: Invalidating analysis: InnerAnalysisManagerProxy
 ; CHECK-O: Running pass: ForceFunctionAttrsPass
-; CHECK-O: Running pass: PassManager<{{.*}}Module>
 ; CHECK-O: Starting {{.*}}Module pass manager run.
 ; CHECK-O: Running pass: PGOIndirectCallPromotion
 ; CHECK-O: Running analysis: ProfileSummaryAnalysis
 ; CHECK-O: Running analysis: InnerAnalysisManagerProxy
 ; CHECK-O: Running analysis: OptimizationRemarkEmitterAnalysis on main
-; CHECK-O: Running analysis: PassInstrumentationAnalysis on main
 ; CHECK-O: Running pass: InferFunctionAttrsPass
-; CHECK-O: Running pass: ModuleToFunctionPassAdaptor<{{.*}}PassManager<{{.*}}Function>{{ ?}}>
 ; CHECK-O: Starting {{.*}}Function pass manager run.
 ; CHECK-O: Running pass: SimplifyCFGPass on main
 ; CHECK-O: Running analysis: TargetIRAnalysis on main
@@ -46,18 +43,17 @@
 ; CHECK-O: Running pass: LowerExpectIntrinsicPass on main
 ; CHECK-O3: Running pass: CallSiteSplittingPass on main
 ; CHECK-O: Finished {{.*}}Function pass manager run.
+; CHECK-O: Running pass: LowerTypeTestsPass
 ; CHECK-O: Running pass: IPSCCPPass
 ; CHECK-O: Running pass: CalledValuePropagationPass
 ; CHECK-O: Running pass: GlobalOptPass
 ; CHECK-O: Invalidating all non-preserved analyses for:
 ; CHECK-O: Invalidating analysis: InnerAnalysisManagerProxy
-; CHECK-O: Running pass: ModuleToFunctionPassAdaptor<{{.*}}PromotePass>
 ; CHECK-O: Running analysis: InnerAnalysisManagerProxy
+; CHECK-O: Running pass: PromotePass
 ; CHECK-O: Running analysis: DominatorTreeAnalysis on main
-; CHECK-O: Running analysis: PassInstrumentationAnalysis on main
 ; CHECK-O: Running analysis: AssumptionAnalysis on main
 ; CHECK-O: Running pass: DeadArgumentEliminationPass
-; CHECK-O: Running pass: ModuleToFunctionPassAdaptor<{{.*}}PassManager<{{.*}}Function>{{ ?}}>
 ; CHECK-O: Starting {{.*}}Function pass manager run.
 ; CHECK-O: Running pass: InstCombinePass on main
 ; CHECK-O: Running analysis: TargetLibraryAnalysis on main
@@ -74,11 +70,9 @@
 ; CHECK-O: Running analysis: GlobalsAA
 ; CHECK-O: Running analysis: CallGraphAnalysis
 ; CHECK-O: Running pass: RequireAnalysisPass<{{.*}}ProfileSummaryAnalysis
-; CHECK-O: Running pass: ModuleToPostOrderCGSCCPassAdaptor<{{.*}}DevirtSCCRepeatedPass<{{.*}}PassManager<{{.*}}LazyCallGraph::SCC
 ; CHECK-O: Running analysis: InnerAnalysisManagerProxy
 ; CHECK-O: Running analysis: LazyCallGraphAnalysis
 ; CHECK-O: Running analysis: FunctionAnalysisManagerCGSCCProxy on (main)
-; CHECK-O: Running analysis: PassInstrumentationAnalysis on (main)
 ; CHECK-O: Running analysis: OuterAnalysisManagerProxy
 ; CHECK-O: Starting CGSCC pass manager run.
 ; CHECK-O: Running pass: InlinerPass on (main)
@@ -87,8 +81,6 @@
 ; CHECK-O: Clearing all analysis results for: main
 ; CHECK-O3: Running pass: ArgumentPromotionPass on (main)
 ; CHECK-O3: Running analysis: TargetIRAnalysis on main
-; CHECK-O: Running analysis: PassInstrumentationAnalysis on main
-; CHECK-O3: Running pass: CGSCCToFunctionPassAdaptor<{{.*}}PassManager{{.*}}>
 ; CHECK-O: Starting {{.*}}Function pass manager run.
 ; CHECK-O: Running pass: SROA on main
 ; These next two can appear in any order since they are accessed as parameters
@@ -117,7 +109,6 @@
 ; CHECK-O: Running pass: SimplifyCFGPass on main
 ; CHECK-O: Running pass: ReassociatePass on main
 ; CHECK-O: Running pass: RequireAnalysisPass<{{.*}}OptimizationRemarkEmitterAnalysis
-; CHECK-O: Running pass: FunctionToLoopPassAdaptor<{{.*}}PassManager<{{.*}}Loop
 ; CHECK-O: Starting {{.*}}Function pass manager run.
 ; CHECK-O: Running pass: LoopSimplifyPass on main
 ; CHECK-O: Running analysis: LoopAnalysis on main
@@ -125,7 +116,6 @@
 ; CHECK-O: Finished {{.*}}Function pass manager run.
 ; CHECK-O: Running pass: SimplifyCFGPass on main
 ; CHECK-O: Running pass: InstCombinePass on main
-; CHECK-O: Running pass: FunctionToLoopPassAdaptor<{{.*}}PassManager<{{.*}}Loop
 ; CHECK-O: Starting {{.*}}Function pass manager run.
 ; CHECK-O: Running pass: LoopSimplifyPass on main
 ; CHECK-O: Running pass: LCSSAPass on main
@@ -143,7 +133,6 @@
 ; CHECK-O: Running pass: JumpThreadingPass on main
 ; CHECK-O: Running pass: CorrelatedValuePropagationPass on main
 ; CHECK-O: Running pass: DSEPass on main
-; CHECK-O: Running pass: FunctionToLoopPassAdaptor<{{.*}}LICMPass> on main
 ; CHECK-O: Starting {{.*}}Function pass manager run.
 ; CHECK-O: Running pass: LoopSimplifyPass on main
 ; CHECK-O: Running pass: LCSSAPass on main
@@ -168,7 +157,6 @@
 ; CHECK-O: Invalidating analysis: PostDominatorTreeAnalysis on main
 ; CHECK-O: Invalidating analysis: CallGraphAnalysis
 ; CHECK-O: Finished {{.*}}Module pass manager run.
-; CHECK-O: Running pass: PassManager<{{.*}}Module>
 ; CHECK-O: Starting {{.*}}Module pass manager run.
 ; CHECK-O: Running pass: GlobalOptPass
 ; CHECK-O: Running pass: GlobalDCEPass
@@ -176,40 +164,43 @@
 ; CHECK-O: Running pass: ReversePostOrderFunctionAttrsPass
 ; CHECK-O: Running analysis: CallGraphAnalysis
 ; CHECK-O: Running pass: RequireAnalysisPass<{{.*}}GlobalsAA
-; CHECK-O: Running pass: ModuleToFunctionPassAdaptor<{{.*}}PassManager<{{.*}}Function>{{ ?}}>
 ; CHECK-O: Starting {{.*}}Function pass manager run.
 ; CHECK-O: Running pass: Float2IntPass on main
 ; CHECK-O: Running pass: LowerConstantIntrinsicsPass on main
-; CHECK-O: Running pass: FunctionToLoopPassAdaptor<{{.*}}LoopRotatePass> on main
 ; CHECK-O: Starting {{.*}}Function pass manager run.
 ; CHECK-O: Running pass: LoopSimplifyPass on main
 ; CHECK-O: Running analysis: LoopAnalysis on main
 ; CHECK-O: Running pass: LCSSAPass on main
 ; CHECK-O: Finished {{.*}}Function pass manager run.
-; CHECK-O: Running pass: LoopDistributePass on main
-; CHECK-O: Running analysis: ScalarEvolutionAnalysis on main
+; CHECK-O: Running analysis: MemorySSAAnalysis on main
 ; CHECK-O: Running analysis: AAManager on main
 ; CHECK-O: Running analysis: BasicAA on main
+; CHECK-O: Running analysis: ScalarEvolutionAnalysis on main
 ; CHECK-O: Running analysis: InnerAnalysisManagerProxy
+; CHECK-O: Running pass: LoopRotatePass on Loop at depth 1 containing: %b
+; CHECK-O: Running pass: LoopDistributePass on main
+; CHECK-O: Running pass: InjectTLIMappings on main
 ; CHECK-O: Running pass: LoopVectorizePass on main
 ; CHECK-O: Running analysis: BlockFrequencyAnalysis on main
 ; CHECK-O: Running analysis: BranchProbabilityAnalysis on main
+; CHECK-O: Running analysis: PostDominatorTreeAnalysis on main
 ; CHECK-O: Running analysis: DemandedBitsAnalysis on main
-; CHECK-O: Running analysis: MemorySSAAnalysis on main
 ; CHECK-O: Running pass: LoopLoadEliminationPass on main
+; CHECK-O: Running analysis: LoopAccessAnalysis on Loop at depth 1 containing: %b
 ; CHECK-O: Running pass: InstCombinePass on main
 ; CHECK-O: Running pass: SimplifyCFGPass on main
 ; CHECK-O: Running pass: SLPVectorizerPass on main
+; CHECK-O: Running pass: VectorCombinePass on main
 ; CHECK-O: Running pass: InstCombinePass on main
 ; CHECK-O: Running pass: LoopUnrollPass on main
 ; CHECK-O: Running pass: WarnMissedTransformationsPass on main
 ; CHECK-O: Running pass: InstCombinePass on main
 ; CHECK-O: Running pass: RequireAnalysisPass<{{.*}}OptimizationRemarkEmitterAnalysis
-; CHECK-O: Running pass: FunctionToLoopPassAdaptor<{{.*}}LICMPass> on main
 ; CHECK-O: Starting {{.*}}Function pass manager run.
 ; CHECK-O: Running pass: LoopSimplifyPass on main
 ; CHECK-O: Running pass: LCSSAPass on main
 ; CHECK-O: Finished {{.*}}Function pass manager run.
+; CHECK-O: Running pass: LICMPass on Loop at depth 1 containing: %b
 ; CHECK-O: Running pass: AlignmentFromAssumptionsPass on main
 ; CHECK-O: Running pass: LoopSinkPass on main
 ; CHECK-O: Running pass: InstSimplifyPass on main
@@ -227,6 +218,8 @@ target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16
 target triple = "x86_64-grtev4-linux-gnu"
 
 define i32 @main() {
-entry:
+  br label %b
+b:
+  br label %b
   ret i32 0
 }
diff --git a/llvm/include/llvm/IR/PassInstrumentation.h b/llvm/include/llvm/IR/PassInstrumentation.h
index df5a0ceb7d016..4df073c91766d 100644
--- a/llvm/include/llvm/IR/PassInstrumentation.h
+++ b/llvm/include/llvm/IR/PassInstrumentation.h
@@ -234,6 +234,8 @@ class PassInstrumentation {
   }
 };
 
+bool isSpecialPass(StringRef PassID, const std::vector<StringRef> &Specials);
+
 } // namespace llvm
 
 #endif
diff --git a/llvm/include/llvm/IR/PassManager.h b/llvm/include/llvm/IR/PassManager.h
index d3b98ed243b8e..c410818c470ca 100644
--- a/llvm/include/llvm/IR/PassManager.h
+++ b/llvm/include/llvm/IR/PassManager.h
@@ -510,10 +510,6 @@ class PassManager : public PassInfoMixin<
       if (!PI.runBeforePass<IRUnitT>(*P, IR))
         continue;
 
-      if (DebugLogging)
-        dbgs() << "Running pass: " << P->name() << " on " << IR.getName()
-               << "\n";
-
       PreservedAnalyses PassPA;
       {
         TimeTraceScope TimeScope(P->name(), IR.getName());
diff --git a/llvm/include/llvm/IR/PassManagerImpl.h b/llvm/include/llvm/IR/PassManagerImpl.h
index 978655ac69c46..80831b31c6b07 100644
--- a/llvm/include/llvm/IR/PassManagerImpl.h
+++ b/llvm/include/llvm/IR/PassManagerImpl.h
@@ -64,9 +64,6 @@ AnalysisManager<IRUnitT, ExtraArgTs...>::getResultImpl(
   // run it to produce a result, which we then add to the cache.
   if (Inserted) {
     auto &P = this->lookUpPass(ID);
-    if (DebugLogging)
-      dbgs() << "Running analysis: " << P.name() << " on " << IR.getName()
-             << "\n";
 
     PassInstrumentation PI;
     if (ID != PassInstrumentationAnalysis::ID()) {
diff --git a/llvm/include/llvm/Passes/StandardInstrumentations.h b/llvm/include/llvm/Passes/StandardInstrumentations.h
index 9c29bfc35941b..fc0f21e404010 100644
--- a/llvm/include/llvm/Passes/StandardInstrumentations.h
+++ b/llvm/include/llvm/Passes/StandardInstrumentations.h
@@ -61,7 +61,16 @@ class OptNoneInstrumentation {
 
 private:
   bool skip(StringRef PassID, Any IR);
+  bool DebugLogging;
+};
+
+// Debug logging for transformation and analysis passes.
+class PrintPassInstrumentation {
+public:
+  PrintPassInstrumentation(bool DebugLogging) : DebugLogging(DebugLogging) {}
+  void registerCallbacks(PassInstrumentationCallbacks &PIC);
 
+private:
   bool DebugLogging;
 };
 
@@ -69,12 +78,13 @@ class OptNoneInstrumentation {
 /// instrumentations and manages their state (if any).
 class StandardInstrumentations {
   PrintIRInstrumentation PrintIR;
+  PrintPassInstrumentation PrintPass;
   TimePassesHandler TimePasses;
   OptNoneInstrumentation OptNone;
 
 public:
   StandardInstrumentations(bool DebugLogging)
-      : PrintIR(), TimePasses(), OptNone(DebugLogging) {}
+      : PrintPass(DebugLogging), OptNone(DebugLogging) {}
 
   void registerCallbacks(PassInstrumentationCallbacks &PIC);
 
diff --git a/llvm/lib/Analysis/CGSCCPassManager.cpp b/llvm/lib/Analysis/CGSCCPassManager.cpp
index fd3166f8cd0cd..a862de643c081 100644
--- a/llvm/lib/Analysis/CGSCCPassManager.cpp
+++ b/llvm/lib/Analysis/CGSCCPassManager.cpp
@@ -78,9 +78,6 @@ PassManager<LazyCallGraph::SCC, CGSCCAnalysisManager, LazyCallGraph &,
     if (!PI.runBeforePass(*Pass, *C))
       continue;
 
-    if (DebugLogging)
-      dbgs() << "Running pass: " << Pass->name() << " on " << *C << "\n";
-
     PreservedAnalyses PassPA;
     {
       TimeTraceScope TimeScope(Pass->name());
diff --git a/llvm/lib/IR/PassInstrumentation.cpp b/llvm/lib/IR/PassInstrumentation.cpp
index 49cc6ec04d90c..d3867de44bcac 100644
--- a/llvm/lib/IR/PassInstrumentation.cpp
+++ b/llvm/lib/IR/PassInstrumentation.cpp
@@ -12,10 +12,19 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/IR/PassInstrumentation.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/IR/PassManager.h"
 
 namespace llvm {
 
 AnalysisKey PassInstrumentationAnalysis::Key;
 
+bool isSpecialPass(StringRef PassID, const std::vector<StringRef> &Specials) {
+  size_t Pos = PassID.find('<');
+  if (Pos == StringRef::npos)
+    return false;
+  StringRef Prefix = PassID.substr(0, Pos);
+  return any_of(Specials, [Prefix](StringRef S) { return Prefix.endswith(S); });
+}
+
 } // namespace llvm
diff --git a/llvm/lib/IR/PassTimingInfo.cpp b/llvm/lib/IR/PassTimingInfo.cpp
index b3cb4ae601647..fb551ddf5c727 100644
--- a/llvm/lib/IR/PassTimingInfo.cpp
+++ b/llvm/lib/IR/PassTimingInfo.cpp
@@ -231,17 +231,9 @@ void TimePassesHandler::stopTimer(StringRef PassID) {
     MyTimer->stopTimer();
 }
 
-static bool matchPassManager(StringRef PassID) {
-  size_t prefix_pos = PassID.find('<');
-  if (prefix_pos == StringRef::npos)
-    return false;
-  StringRef Prefix = PassID.substr(0, prefix_pos);
-  return Prefix.endswith("PassManager") || Prefix.endswith("PassAdaptor") ||
-         Prefix.endswith("AnalysisManagerProxy");
-}
-
 void TimePassesHandler::runBeforePass(StringRef PassID) {
-  if (matchPassManager(PassID))
+  if (isSpecialPass(PassID,
+                    {"PassManager", "PassAdaptor", "AnalysisManagerProxy"}))
     return;
 
   startTimer(PassID);
@@ -251,7 +243,8 @@ void TimePassesHandler::runBeforePass(StringRef PassID) {
 }
 
 void TimePassesHandler::runAfterPass(StringRef PassID) {
-  if (matchPassManager(PassID))
+  if (isSpecialPass(PassID,
+                    {"PassManager", "PassAdaptor", "AnalysisManagerProxy"}))
     return;
 
   stopTimer(PassID);
diff --git a/llvm/lib/Passes/StandardInstrumentations.cpp b/llvm/lib/Passes/StandardInstrumentations.cpp
index 2f802c9eee8a5..d4a6784985a23 100644
--- a/llvm/lib/Passes/StandardInstrumentations.cpp
+++ b/llvm/lib/Passes/StandardInstrumentations.cpp
@@ -26,6 +26,7 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/raw_ostream.h"
+#include <vector>
 
 using namespace llvm;
 
@@ -35,6 +36,13 @@ static cl::opt<bool>
                   cl::desc("Enable skipping optional passes optnone functions "
                            "under new pass manager"));
 
+// FIXME: Change `-debug-pass-manager` from boolean to enum type. Similar to
+// `-debug-pass` in legacy PM.
+static cl::opt<bool>
+    DebugPMVerbose("debug-pass-manager-verbose", cl::Hidden, cl::init(false),
+                   cl::desc("Print all pass management debugging information. "
+                            "`-debug-pass-manager` must also be specified"));
+
 namespace {
 
 /// Extracting Module out of \p IR unit. Also fills a textual description
@@ -78,14 +86,25 @@ Optional<std::pair<const Module *, std::string>> unwrapModule(Any IR) {
   llvm_unreachable("Unknown IR unit");
 }
 
-void printIR(const Function *F, StringRef Banner,
-             StringRef Extra = StringRef()) {
+void printIR(const Function *F, StringRef Banner, StringRef Extra = StringRef(),
+             bool Brief = false) {
+  if (Brief) {
+    dbgs() << F->getName() << '\n';
+    return;
+  }
+
   if (!llvm::isFunctionInPrintList(F->getName()))
     return;
   dbgs() << Banner << Extra << "\n" << static_cast<const Value &>(*F);
 }
 
-void printIR(const Module *M, StringRef Banner, StringRef Extra = StringRef()) {
+void printIR(const Module *M, StringRef Banner, StringRef Extra = StringRef(),
+             bool Brief = false) {
+  if (Brief) {
+    dbgs() << M->getName() << '\n';
+    return;
+  }
+
   if (llvm::isFunctionInPrintList("*") || llvm::forcePrintModuleIR()) {
     dbgs() << Banner << Extra << "\n";
     M->print(dbgs(), nullptr, false);
@@ -97,7 +116,12 @@ void printIR(const Module *M, StringRef Banner, StringRef Extra = StringRef()) {
 }
 
 void printIR(const LazyCallGraph::SCC *C, StringRef Banner,
-             StringRef Extra = StringRef()) {
+             StringRef Extra = StringRef(), bool Brief = false) {
+  if (Brief) {
+    dbgs() << *C << '\n';
+    return;
+  }
+
   bool BannerPrinted = false;
   for (const LazyCallGraph::Node &N : *C) {
     const Function &F = N.getFunction();
@@ -110,7 +134,13 @@ void printIR(const LazyCallGraph::SCC *C, StringRef Banner,
     }
   }
 }
-void printIR(const Loop *L, StringRef Banner) {
+
+void printIR(const Loop *L, StringRef Banner, bool Brief = false) {
+  if (Brief) {
+    dbgs() << *L;
+    return;
+  }
+
   const Function *F = L->getHeader()->getParent();
   if (!llvm::isFunctionInPrintList(F->getName()))
     return;
@@ -119,7 +149,8 @@ void printIR(const Loop *L, StringRef Banner) {
 
 /// Generic IR-printing helper that unpacks a pointer to IRUnit wrapped into
 /// llvm::Any and does actual print job.
-void unwrapAndPrint(Any IR, StringRef Banner, bool ForceModule = false) {
+void unwrapAndPrint(Any IR, StringRef Banner, bool ForceModule = false,
+                    bool Brief = false) {
   if (ForceModule) {
     if (auto UnwrappedModule = unwrapModule(IR))
       printIR(UnwrappedModule->first, Banner, UnwrappedModule->second);
@@ -129,14 +160,14 @@ void unwrapAndPrint(Any IR, StringRef Banner, bool ForceModule = false) {
   if (any_isa<const Module *>(IR)) {
     const Module *M = any_cast<const Module *>(IR);
     assert(M && "module should be valid for printing");
-    printIR(M, Banner);
+    printIR(M, Banner, "", Brief);
     return;
   }
 
   if (any_isa<const Function *>(IR)) {
     const Function *F = any_cast<const Function *>(IR);
     assert(F && "function should be valid for printing");
-    printIR(F, Banner);
+    printIR(F, Banner, "", Brief);
     return;
   }
 
@@ -144,14 +175,14 @@ void unwrapAndPrint(Any IR, StringRef Banner, bool ForceModule = false) {
     const LazyCallGraph::SCC *C = any_cast<const LazyCallGraph::SCC *>(IR);
     assert(C && "scc should be valid for printing");
     std::string Extra = std::string(formatv(" (scc: {0})", C->getName()));
-    printIR(C, Banner, Extra);
+    printIR(C, Banner, Extra, Brief);
     return;
   }
 
   if (any_isa<const Loop *>(IR)) {
     const Loop *L = any_cast<const Loop *>(IR);
     assert(L && "Loop should be valid for printing");
-    printIR(L, Banner);
+    printIR(L, Banner, Brief);
     return;
   }
   llvm_unreachable("Unknown wrapped IR type");
@@ -274,9 +305,34 @@ bool OptNoneInstrumentation::skip(StringRef PassID, Any IR) {
   return true;
 }
 
+void PrintPassInstrumentation::registerCallbacks(
+    PassInstrumentationCallbacks &PIC) {
+  if (!DebugLogging)
+    return;
+
+  std::vector<StringRef> SpecialPasses = {"PassManager"};
+  if (!DebugPMVerbose)
+    SpecialPasses.emplace_back("PassAdaptor");
+
+  PIC.registerBeforeNonSkippedPassCallback(
+      [SpecialPasses](StringRef PassID, Any IR) {
+        if (isSpecialPass(PassID, SpecialPasses))
+          return;
+
+        dbgs() << "Running pass: " << PassID << " on ";
+        unwrapAndPrint(IR, "", false, true);
+      });
+
+  PIC.registerBeforeAnalysisCallback([](StringRef PassID, Any IR) {
+    dbgs() << "Running analysis: " << PassID << " on ";
+    unwrapAndPrint(IR, "", false, true);
+  });
+}
+
 void StandardInstrumentations::registerCallbacks(
     PassInstrumentationCallbacks &PIC) {
   PrintIR.registerCallbacks(PIC);
+  PrintPass.registerCallbacks(PIC);
   TimePasses.registerCallbacks(PIC);
   OptNone.registerCallbacks(PIC);
 }
diff --git a/llvm/lib/Transforms/Scalar/LoopPassManager.cpp b/llvm/lib/Transforms/Scalar/LoopPassManager.cpp
index 98889a9df1163..91de5715a6ae3 100644
--- a/llvm/lib/Transforms/Scalar/LoopPassManager.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopPassManager.cpp
@@ -39,9 +39,6 @@ PassManager<Loop, LoopAnalysisManager, LoopStandardAnalysisResults &,
     if (!PI.runBeforePass<Loop>(*Pass, L))
       continue;
 
-    if (DebugLogging)
-      dbgs() << "Running pass: " << Pass->name() << " on " << L;
-
     PreservedAnalyses PassPA;
     {
       TimeTraceScope TimeScope(Pass->name(), L.getName());
diff --git a/llvm/test/Other/loop-pm-invalidation.ll b/llvm/test/Other/loop-pm-invalidation.ll
index 20acb3573531e..3fef7a4a5082c 100644
--- a/llvm/test/Other/loop-pm-invalidation.ll
+++ b/llvm/test/Other/loop-pm-invalidation.ll
@@ -18,7 +18,7 @@
 ; RUN:     | FileCheck %s --check-prefix=CHECK-SCEV-INV-AFTER-DELETE
 
 define void @no_loops() {
-; CHECK-LOOP-INV-LABEL: Running pass: FunctionToLoopPassAdaptor<{{.*}}> on no_loops
+; CHECK-LOOP-INV: Starting {{.*}}Function pass manager run
 ; CHECK-LOOP-INV-NEXT: Starting {{.*}}Function pass manager run
 ; CHECK-LOOP-INV-NEXT: Running pass: LoopSimplifyPass
 ; CHECK-LOOP-INV-NEXT: Running analysis: LoopAnalysis
@@ -29,7 +29,6 @@ define void @no_loops() {
 ; CHECK-LOOP-INV-NEXT: Running pass: InvalidateAnalysisPass<{{.*}}LoopAnalysis
 ; CHECK-LOOP-INV-NEXT: Invalidating all non-preserved analyses
 ; CHECK-LOOP-INV-NEXT: Invalidating analysis: LoopAnalysis
-; CHECK-LOOP-INV-NEXT: Running pass: FunctionToLoopPassAdaptor<{{.*}}> on no_loops
 ; CHECK-LOOP-INV-NEXT: Starting {{.*}}Function pass manager run
 ; CHECK-LOOP-INV-NEXT: Running pass: LoopSimplifyPass
 ; CHECK-LOOP-INV-NEXT: Running analysis: LoopAnalysis
@@ -37,7 +36,7 @@ define void @no_loops() {
 ; CHECK-LOOP-INV-NEXT: Finished {{.*}}Function pass manager run
 ; CHECK-LOOP-INV-NEXT: Finished {{.*}}Function pass manager run.
 ;
-; CHECK-SCEV-INV-LABEL: Running pass: FunctionToLoopPassAdaptor<{{.*}}> on no_loops
+; CHECK-SCEV-INV: Starting {{.*}}Function pass manager run
 ; CHECK-SCEV-INV-NEXT: Starting {{.*}}Function pass manager run
 ; CHECK-SCEV-INV-NEXT: Running pass: LoopSimplifyPass
 ; CHECK-SCEV-INV-NEXT: Running analysis: LoopAnalysis
@@ -47,7 +46,6 @@ define void @no_loops() {
 ; CHECK-SCEV-INV-NEXT: Finished {{.*}}Function pass manager run
 ; CHECK-SCEV-INV-NEXT: Running pass: InvalidateAnalysisPass<{{.*}}ScalarEvolutionAnalysis
 ; CHECK-SCEV-INV-NEXT: Invalidating all non-preserved analyses
-; CHECK-SCEV-INV-NEXT: Running pass: FunctionToLoopPassAdaptor<{{.*}}> on no_loops
 ; CHECK-SCEV-INV-NEXT: Starting {{.*}}Function pass manager run
 ; CHECK-SCEV-INV-NEXT: Running pass: LoopSimplifyPass
 ; CHECK-SCEV-INV-NEXT: Running pass: LCSSAPass
@@ -59,7 +57,7 @@ entry:
 }
 
 define void @one_loop(i1* %ptr) {
-; CHECK-LOOP-INV-LABEL: Running pass: FunctionToLoopPassAdaptor<{{.*}}> on one_loop
+; CHECK-LOOP-INV: Starting {{.*}}Function pass manager run
 ; CHECK-LOOP-INV-NEXT: Starting {{.*}}Function pass manager run
 ; CHECK-LOOP-INV-NEXT: Running pass: LoopSimplifyPass
 ; CHECK-LOOP-INV-NEXT: Running analysis: LoopAnalysis
@@ -73,7 +71,6 @@ define void @one_loop(i1* %ptr) {
 ; CHECK-LOOP-INV-NEXT: Running analysis: TargetIRAnalysis
 ; CHECK-LOOP-INV-NEXT: Running analysis: InnerAnalysisManagerProxy<{{.*}}Loop
 ; CHECK-LOOP-INV-NEXT: Starting {{.*}}Loop pass manager run.
-; CHECK-LOOP-INV-NEXT: Running analysis: PassInstrumentationAnalysis
 ; CHECK-LOOP-INV-NEXT: Running pass: NoOpLoopPass
 ; CHECK-LOOP-INV-NEXT: Finished {{.*}}Loop pass manager run.
 ; CHECK-LOOP-INV-NEXT: Running pass: InvalidateAnalysisPass<{{.*}}LoopAnalysis
@@ -82,7 +79,6 @@ define void @one_loop(i1* %ptr) {
 ; CHECK-LOOP-INV-NEXT: Invalidating analysis: LoopAnalysis
 ; CHECK-LOOP-INV-NEXT: Invalidating analysis: ScalarEvolutionAnalysis
 ; CHECK-LOOP-INV-NEXT: Invalidating analysis: InnerAnalysisManagerProxy<{{.*}}Loop
-; CHECK-LOOP-INV-NEXT: Running pass: FunctionToLoopPassAdaptor<{{.*}}> on one_loop
 ; CHECK-LOOP-INV-NEXT: Starting {{.*}}Function pass manager run
 ; CHECK-LOOP-INV-NEXT: Running pass: LoopSimplifyPass
 ; CHECK-LOOP-INV-NEXT: Running analysis: LoopAnalysis
@@ -91,12 +87,11 @@ define void @one_loop(i1* %ptr) {
 ; CHECK-LOOP-INV-NEXT: Running analysis: ScalarEvolutionAnalysis
 ; CHECK-LOOP-INV-NEXT: Running analysis: InnerAnalysisManagerProxy<{{.*}}Loop
 ; CHECK-LOOP-INV-NEXT: Starting {{.*}}Loop pass manager run.
-; CHECK-LOOP-INV-NEXT: Running analysis: PassInstrumentationAnalysis
 ; CHECK-LOOP-INV-NEXT: Running pass: NoOpLoopPass
 ; CHECK-LOOP-INV-NEXT: Finished {{.*}}Loop pass manager run.
 ; CHECK-LOOP-INV-NEXT: Finished {{.*}}Function pass manager run.
 ;
-; CHECK-SCEV-INV-LABEL: Running pass: FunctionToLoopPassAdaptor<{{.*}}> on one_loop
+; CHECK-SCEV-INV: Starting {{.*}}Function pass manager run
 ; CHECK-SCEV-INV-NEXT: Starting {{.*}}Function pass manager run
 ; CHECK-SCEV-INV-NEXT: Running pass: LoopSimplifyPass
 ; CHECK-SCEV-INV-NEXT: Running analysis: LoopAnalysis
@@ -110,7 +105,6 @@ define void @one_loop(i1* %ptr) {
 ; CHECK-SCEV-INV-NEXT: Running analysis: TargetIRAnalysis
 ; CHECK-SCEV-INV-NEXT: Running analysis: InnerAnalysisManagerProxy<{{.*}}Loop
 ; CHECK-SCEV-INV-NEXT: Starting {{.*}}Loop pass manager run.
-; CHECK-SCEV-INV-NEXT: Running analysis: PassInstrumentationAnalysis
 ; CHECK-SCEV-INV-NEXT: Running pass: NoOpLoopPass
 ; CHECK-SCEV-INV-NEXT: Finished {{.*}}Loop pass manager run.
 ; CHECK-SCEV-INV-NEXT: Running pass: InvalidateAnalysisPass<{{.*}}ScalarEvolutionAnalysis
@@ -118,7 +112,6 @@ define void @one_loop(i1* %ptr) {
 ; CHECK-SCEV-INV-NEXT: Clearing all analysis results for: <possibly invalidated loop>
 ; CHECK-SCEV-INV-NEXT: Invalidating analysis: ScalarEvolutionAnalysis
 ; CHECK-SCEV-INV-NEXT: Invalidating analysis: InnerAnalysisManagerProxy<{{.*}}Loop
-; CHECK-SCEV-INV-NEXT: Running pass: FunctionToLoopPassAdaptor<{{.*}}> on one_loop
 ; CHECK-SCEV-INV-NEXT: Starting {{.*}}Function pass manager run
 ; CHECK-SCEV-INV-NEXT: Running pass: LoopSimplifyPass
 ; CHECK-SCEV-INV-NEXT: Running pass: LCSSAPass
@@ -126,7 +119,6 @@ define void @one_loop(i1* %ptr) {
 ; CHECK-SCEV-INV-NEXT: Running analysis: ScalarEvolutionAnalysis
 ; CHECK-SCEV-INV-NEXT: Running analysis: InnerAnalysisManagerProxy<{{.*}}Loop
 ; CHECK-SCEV-INV-NEXT: Starting {{.*}}Loop pass manager run.
-; CHECK-SCEV-INV-NEXT: Running analysis: PassInstrumentationAnalysis
 ; CHECK-SCEV-INV-NEXT: Running pass: NoOpLoopPass
 ; CHECK-SCEV-INV-NEXT: Finished {{.*}}Loop pass manager run.
 ; CHECK-SCEV-INV-NEXT: Finished {{.*}}Function pass manager run.
@@ -143,7 +135,7 @@ exit:
 }
 
 define void @nested_loops(i1* %ptr) {
-; CHECK-LOOP-INV-LABEL: Running pass: FunctionToLoopPassAdaptor<{{.*}}> on nested_loops
+; CHECK-LOOP-INV: Starting {{.*}}Function pass manager run
 ; CHECK-LOOP-INV-NEXT: Starting {{.*}}Function pass manager run
 ; CHECK-LOOP-INV-NEXT: Running pass: LoopSimplifyPass
 ; CHECK-LOOP-INV-NEXT: Running analysis: LoopAnalysis
@@ -157,11 +149,9 @@ define void @nested_loops(i1* %ptr) {
 ; CHECK-LOOP-INV-NEXT: Running analysis: TargetIRAnalysis
 ; CHECK-LOOP-INV-NEXT: Running analysis: InnerAnalysisManagerProxy<{{.*}}Loop
 ; CHECK-LOOP-INV-NEXT: Starting {{.*}}Loop pass manager run.
-; CHECK-LOOP-INV-NEXT: Running analysis: PassInstrumentationAnalysis
 ; CHECK-LOOP-INV-NEXT: Running pass: NoOpLoopPass
 ; CHECK-LOOP-INV-NEXT: Finished {{.*}}Loop pass manager run.
 ; CHECK-LOOP-INV-NEXT: Starting {{.*}}Loop pass manager run.
-; CHECK-LOOP-INV-NEXT: Running analysis: PassInstrumentationAnalysis
 ; CHECK-LOOP-INV-NEXT: Running pass: NoOpLoopPass
 ; CHECK-LOOP-INV: Finished {{.*}}Loop pass manager run.
 ; CHECK-LOOP-INV-NEXT: Running pass: InvalidateAnalysisPass<{{.*}}LoopAnalysis
@@ -171,7 +161,6 @@ define void @nested_loops(i1* %ptr) {
 ; CHECK-LOOP-INV-NEXT: Invalidating analysis: LoopAnalysis
 ; CHECK-LOOP-INV-NEXT: Invalidating analysis: ScalarEvolutionAnalysis
 ; CHECK-LOOP-INV-NEXT: Invalidating analysis: InnerAnalysisManagerProxy<{{.*}}Loop
-; CHECK-LOOP-INV-NEXT: Running pass: FunctionToLoopPassAdaptor<{{.*}}> on nested_loops
 ; CHECK-LOOP-INV-NEXT: Starting {{.*}}Function pass manager run
 ; CHECK-LOOP-INV-NEXT: Running pass: LoopSimplifyPass
 ; CHECK-LOOP-INV-NEXT: Running analysis: LoopAnalysis
@@ -180,16 +169,14 @@ define void @nested_loops(i1* %ptr) {
 ; CHECK-LOOP-INV-NEXT: Running analysis: ScalarEvolutionAnalysis
 ; CHECK-LOOP-INV-NEXT: Running analysis: InnerAnalysisManagerProxy<{{.*}}Loop
 ; CHECK-LOOP-INV-NEXT: Starting {{.*}}Loop pass manager run.
-; CHECK-LOOP-INV-NEXT: Running analysis: PassInstrumentationAnalysis
 ; CHECK-LOOP-INV-NEXT: Running pass: NoOpLoopPass
 ; CHECK-LOOP-INV-NEXT: Finished {{.*}}Loop pass manager run.
 ; CHECK-LOOP-INV-NEXT: Starting {{.*}}Loop pass manager run.
-; CHECK-LOOP-INV-NEXT: Running analysis: PassInstrumentationAnalysis
 ; CHECK-LOOP-INV-NEXT: Running pass: NoOpLoopPass
 ; CHECK-LOOP-INV: Finished {{.*}}Loop pass manager run.
 ; CHECK-LOOP-INV-NEXT: Finished {{.*}}Function pass manager run.
 ;
-; CHECK-SCEV-INV-LABEL: Running pass: FunctionToLoopPassAdaptor<{{.*}}> on nested_loops
+; CHECK-SCEV-INV: Starting {{.*}}Function pass manager run
 ; CHECK-SCEV-INV-NEXT: Starting {{.*}}Function pass manager run
 ; CHECK-SCEV-INV-NEXT: Running pass: LoopSimplifyPass
 ; CHECK-SCEV-INV-NEXT: Running analysis: LoopAnalysis
@@ -203,11 +190,9 @@ define void @nested_loops(i1* %ptr) {
 ; CHECK-SCEV-INV-NEXT: Running analysis: TargetIRAnalysis
 ; CHECK-SCEV-INV-NEXT: Running analysis: InnerAnalysisManagerProxy<{{.*}}Loop
 ; CHECK-SCEV-INV-NEXT: Starting {{.*}}Loop pass manager run.
-; CHECK-SCEV-INV-NEXT: Running analysis: PassInstrumentationAnalysis
 ; CHECK-SCEV-INV-NEXT: Running pass: NoOpLoopPass
 ; CHECK-SCEV-INV-NEXT: Finished {{.*}}Loop pass manager run.
 ; CHECK-SCEV-INV-NEXT: Starting {{.*}}Loop pass manager run.
-; CHECK-SCEV-INV-NEXT: Running analysis: PassInstrumentationAnalysis
 ; CHECK-SCEV-INV-NEXT: Running pass: NoOpLoopPass
 ; CHECK-SCEV-INV: Finished {{.*}}Loop pass manager run.
 ; CHECK-SCEV-INV-NEXT: Running pass: InvalidateAnalysisPass<{{.*}}ScalarEvolutionAnalysis
@@ -216,7 +201,6 @@ define void @nested_loops(i1* %ptr) {
 ; CHECK-SCEV-INV-NEXT: Clearing all analysis results for: <possibly invalidated loop>
 ; CHECK-SCEV-INV-NEXT: Invalidating analysis: ScalarEvolutionAnalysis
 ; CHECK-SCEV-INV-NEXT: Invalidating analysis: InnerAnalysisManagerProxy<{{.*}}Loop
-; CHECK-SCEV-INV-NEXT: Running pass: FunctionToLoopPassAdaptor<{{.*}}> on nested_loops
 ; CHECK-SCEV-INV-NEXT: Starting {{.*}}Function pass manager run
 ; CHECK-SCEV-INV-NEXT: Running pass: LoopSimplifyPass
 ; CHECK-SCEV-INV-NEXT: Running pass: LCSSAPass
@@ -224,11 +208,9 @@ define void @nested_loops(i1* %ptr) {
 ; CHECK-SCEV-INV-NEXT: Running analysis: ScalarEvolutionAnalysis
 ; CHECK-SCEV-INV-NEXT: Running analysis: InnerAnalysisManagerProxy<{{.*}}Loop
 ; CHECK-SCEV-INV-NEXT: Starting {{.*}}Loop pass manager run.
-; CHECK-SCEV-INV-NEXT: Running analysis: PassInstrumentationAnalysis
 ; CHECK-SCEV-INV-NEXT: Running pass: NoOpLoopPass
 ; CHECK-SCEV-INV-NEXT: Finished {{.*}}Loop pass manager run.
 ; CHECK-SCEV-INV-NEXT: Starting {{.*}}Loop pass manager run.
-; CHECK-SCEV-INV-NEXT: Running analysis: PassInstrumentationAnalysis
 ; CHECK-SCEV-INV-NEXT: Running pass: NoOpLoopPass
 ; CHECK-SCEV-INV: Finished {{.*}}Loop pass manager run.
 ; CHECK-SCEV-INV-NEXT: Finished {{.*}}Function pass manager run.
@@ -252,7 +234,7 @@ exit:
 }
 
 define void @dead_loop() {
-; CHECK-LOOP-INV-LABEL: Running pass: FunctionToLoopPassAdaptor<{{.*}}> on dead_loop
+; CHECK-LOOP-INV: Starting {{.*}}Function pass manager run
 ; CHECK-LOOP-INV-NEXT: Starting {{.*}}Function pass manager run
 ; CHECK-LOOP-INV-NEXT: Running pass: LoopSimplifyPass
 ; CHECK-LOOP-INV-NEXT: Running analysis: LoopAnalysis
@@ -266,7 +248,6 @@ define void @dead_loop() {
 ; CHECK-LOOP-INV-NEXT: Running analysis: TargetIRAnalysis
 ; CHECK-LOOP-INV-NEXT: Running analysis: InnerAnalysisManagerProxy<{{.*}}Loop
 ; CHECK-LOOP-INV-NEXT: Starting {{.*}}Loop pass manager run.
-; CHECK-LOOP-INV-NEXT: Running analysis: PassInstrumentationAnalysis
 ; CHECK-LOOP-INV-NEXT: Running pass: NoOpLoopPass
 ; CHECK-LOOP-INV-NEXT: Finished {{.*}}Loop pass manager run.
 ; CHECK-LOOP-INV-NEXT: Running pass: InvalidateAnalysisPass<{{.*}}LoopAnalysis
@@ -275,7 +256,6 @@ define void @dead_loop() {
 ; CHECK-LOOP-INV-NEXT: Invalidating analysis: LoopAnalysis
 ; CHECK-LOOP-INV-NEXT: Invalidating analysis: ScalarEvolutionAnalysis
 ; CHECK-LOOP-INV-NEXT: Invalidating analysis: InnerAnalysisManagerProxy<{{.*}}Loop
-; CHECK-LOOP-INV-NEXT: Running pass: FunctionToLoopPassAdaptor<{{.*}}> on dead_loop
 ; CHECK-LOOP-INV-NEXT: Starting {{.*}}Function pass manager run
 ; CHECK-LOOP-INV-NEXT: Running pass: LoopSimplifyPass
 ; CHECK-LOOP-INV-NEXT: Running analysis: LoopAnalysis
@@ -284,12 +264,11 @@ define void @dead_loop() {
 ; CHECK-LOOP-INV-NEXT: Running analysis: ScalarEvolutionAnalysis
 ; CHECK-LOOP-INV-NEXT: Running analysis: InnerAnalysisManagerProxy<{{.*}}Loop
 ; CHECK-LOOP-INV-NEXT: Starting {{.*}}Loop pass manager run.
-; CHECK-LOOP-INV-NEXT: Running analysis: PassInstrumentationAnalysis
 ; CHECK-LOOP-INV-NEXT: Running pass: NoOpLoopPass
 ; CHECK-LOOP-INV-NEXT: Finished {{.*}}Loop pass manager run.
 ; CHECK-LOOP-INV-NEXT: Finished {{.*}}Function pass manager run.
 ;
-; CHECK-SCEV-INV-LABEL: Running pass: FunctionToLoopPassAdaptor<{{.*}}> on dead_loop
+; CHECK-SCEV-INV: Starting {{.*}}Function pass manager run
 ; CHECK-SCEV-INV-NEXT: Starting {{.*}}Function pass manager run
 ; CHECK-SCEV-INV-NEXT: Running pass: LoopSimplifyPass
 ; CHECK-SCEV-INV-NEXT: Running analysis: LoopAnalysis
@@ -303,7 +282,6 @@ define void @dead_loop() {
 ; CHECK-SCEV-INV-NEXT: Running analysis: TargetIRAnalysis
 ; CHECK-SCEV-INV-NEXT: Running analysis: InnerAnalysisManagerProxy<{{.*}}Loop
 ; CHECK-SCEV-INV-NEXT: Starting {{.*}}Loop pass manager run.
-; CHECK-SCEV-INV-NEXT: Running analysis: PassInstrumentationAnalysis
 ; CHECK-SCEV-INV-NEXT: Running pass: NoOpLoopPass
 ; CHECK-SCEV-INV-NEXT: Finished {{.*}}Loop pass manager run.
 ; CHECK-SCEV-INV-NEXT: Running pass: InvalidateAnalysisPass<{{.*}}ScalarEvolutionAnalysis
@@ -311,7 +289,6 @@ define void @dead_loop() {
 ; CHECK-SCEV-INV-NEXT: Clearing all analysis results for: <possibly invalidated loop>
 ; CHECK-SCEV-INV-NEXT: Invalidating analysis: ScalarEvolutionAnalysis
 ; CHECK-SCEV-INV-NEXT: Invalidating analysis: InnerAnalysisManagerProxy<{{.*}}Loop
-; CHECK-SCEV-INV-NEXT: Running pass: FunctionToLoopPassAdaptor<{{.*}}> on dead_loop
 ; CHECK-SCEV-INV-NEXT: Starting {{.*}}Function pass manager run
 ; CHECK-SCEV-INV-NEXT: Running pass: LoopSimplifyPass
 ; CHECK-SCEV-INV-NEXT: Running pass: LCSSAPass
@@ -319,14 +296,11 @@ define void @dead_loop() {
 ; CHECK-SCEV-INV-NEXT: Running analysis: ScalarEvolutionAnalysis
 ; CHECK-SCEV-INV-NEXT: Running analysis: InnerAnalysisManagerProxy<{{.*}}Loop
 ; CHECK-SCEV-INV-NEXT: Starting {{.*}}Loop pass manager run.
-; CHECK-SCEV-INV-NEXT: Running analysis: PassInstrumentationAnalysis
 ; CHECK-SCEV-INV-NEXT: Running pass: NoOpLoopPass
 ; CHECK-SCEV-INV-NEXT: Finished {{.*}}Loop pass manager run.
 ; CHECK-SCEV-INV-NEXT: Finished {{.*}}Function pass manager run.
 ;
-; CHECK-SCEV-INV-AFTER-DELETE-LABEL: Running pass: FunctionToLoopPassAdaptor<{{.*}}> on dead_loop
-; CHECK-SCEV-INV-AFTER-DELETE-NEXT: Starting {{.*}}Function pass manager run
-; CHECK-SCEV-INV-AFTER-DELETE-NEXT: Running pass: LoopSimplifyPass
+; CHECK-SCEV-INV-AFTER-DELETE-LABEL: Running pass: LoopSimplifyPass on dead_loop
 ; CHECK-SCEV-INV-AFTER-DELETE-NEXT: Running analysis: LoopAnalysis
 ; CHECK-SCEV-INV-AFTER-DELETE-NEXT: Running analysis: DominatorTreeAnalysis
 ; CHECK-SCEV-INV-AFTER-DELETE-NEXT: Running analysis: AssumptionAnalysis
@@ -338,7 +312,6 @@ define void @dead_loop() {
 ; CHECK-SCEV-INV-AFTER-DELETE-NEXT: Running analysis: TargetIRAnalysis
 ; CHECK-SCEV-INV-AFTER-DELETE-NEXT: Running analysis: InnerAnalysisManagerProxy<{{.*}}Loop
 ; CHECK-SCEV-INV-AFTER-DELETE-NEXT: Starting {{.*}}Loop pass manager run.
-; CHECK-SCEV-INV-AFTER-DELETE-NEXT: Running analysis: PassInstrumentationAnalysis
 ; CHECK-SCEV-INV-AFTER-DELETE-NEXT: Running pass: NoOpLoopPass
 ; CHECK-SCEV-INV-AFTER-DELETE-NEXT: Running pass: LoopDeletionPass
 ; CHECK-SCEV-INV-AFTER-DELETE-NEXT: Clearing all analysis results for:
@@ -348,7 +321,6 @@ define void @dead_loop() {
 ; CHECK-SCEV-INV-AFTER-DELETE-NEXT: Invalidating all non-preserved analyses
 ; CHECK-SCEV-INV-AFTER-DELETE-NEXT: Invalidating analysis: ScalarEvolutionAnalysis
 ; CHECK-SCEV-INV-AFTER-DELETE-NEXT: Invalidating analysis: InnerAnalysisManagerProxy<{{.*}}Loop
-; CHECK-SCEV-INV-AFTER-DELETE-NEXT: Running pass: FunctionToLoopPassAdaptor<{{.*}}> on dead_loop
 ; CHECK-SCEV-INV-AFTER-DELETE-NEXT: Starting {{.*}}Function pass manager run
 ; CHECK-SCEV-INV-AFTER-DELETE-NEXT: Running pass: LoopSimplifyPass
 ; CHECK-SCEV-INV-AFTER-DELETE-NEXT: Running pass: LCSSAPass
diff --git a/llvm/test/Other/new-pass-manager.ll b/llvm/test/Other/new-pass-manager.ll
index d769319ea1ca4..1c9477dc9da22 100644
--- a/llvm/test/Other/new-pass-manager.ll
+++ b/llvm/test/Other/new-pass-manager.ll
@@ -19,14 +19,11 @@
 ; RUN:     -passes='cgscc(no-op-cgscc)' %s 2>&1 \
 ; RUN:     | FileCheck %s --check-prefix=CHECK-CGSCC-PASS
 ; CHECK-CGSCC-PASS: Starting llvm::Module pass manager run
-; CHECK-CGSCC-PASS-NEXT: Running pass: ModuleToPostOrderCGSCCPassAdaptor
 ; CHECK-CGSCC-PASS-NEXT: Running analysis: InnerAnalysisManagerProxy<{{.*(CGSCCAnalysisManager|AnalysisManager<.*LazyCallGraph::SCC.*>).*}},{{.*}}Module>
 ; CHECK-CGSCC-PASS-NEXT: Running analysis: InnerAnalysisManagerProxy<{{.*(FunctionAnalysisManager|AnalysisManager<.*Function.*>).*}},{{.*}}Module>
 ; CHECK-CGSCC-PASS-NEXT: Running analysis: LazyCallGraphAnalysis
 ; CHECK-CGSCC-PASS-NEXT: Running analysis: TargetLibraryAnalysis
-; CHECK-CGSCC-PASS-NEXT: Running analysis: PassInstrumentationAnalysis
 ; CHECK-CGSCC-PASS-NEXT: Running analysis: FunctionAnalysisManagerCGSCCProxy
-; CHECK-CGSCC-PASS-NEXT: Running analysis: PassInstrumentationAnalysis
 ; CHECK-CGSCC-PASS-NEXT: Running analysis: OuterAnalysisManagerProxy<{{.*}}LazyCallGraph::SCC{{.*}}>
 ; CHECK-CGSCC-PASS-NEXT: Starting CGSCC pass manager run
 ; CHECK-CGSCC-PASS-NEXT: Running pass: NoOpCGSCCPass
@@ -40,9 +37,7 @@
 ; RUN:     -passes='function(no-op-function)' %s 2>&1 \
 ; RUN:     | FileCheck %s --check-prefix=CHECK-FUNCTION-PASS
 ; CHECK-FUNCTION-PASS: Starting llvm::Module pass manager run
-; CHECK-FUNCTION-PASS-NEXT: Running pass: ModuleToFunctionPassAdaptor
 ; CHECK-FUNCTION-PASS-NEXT: Running analysis: InnerAnalysisManagerProxy<{{.*}}>
-; CHECK-FUNCTION-PASS-NEXT: Running analysis: PassInstrumentationAnalysis
 ; CHECK-FUNCTION-PASS-NEXT: Starting llvm::Function pass manager run
 ; CHECK-FUNCTION-PASS-NEXT: Running pass: NoOpFunctionPass
 ; CHECK-FUNCTION-PASS-NEXT: Finished llvm::Function pass manager run
@@ -71,7 +66,6 @@
 ; RUN:     | FileCheck %s --check-prefix=CHECK-FUNCTION-PRINT
 ; CHECK-FUNCTION-PRINT: Starting llvm::Module pass manager run
 ; CHECK-FUNCTION-PRINT: Running pass: VerifierPass
-; CHECK-FUNCTION-PRINT: Running pass: ModuleToFunctionPassAdaptor
 ; CHECK-FUNCTION-PRINT: Running analysis: InnerAnalysisManagerProxy<{{.*}}>
 ; CHECK-FUNCTION-PRINT: Starting llvm::Function pass manager run
 ; CHECK-FUNCTION-PRINT: Running pass: PrintFunctionPass
@@ -406,14 +400,11 @@
 ; RUN:     -passes='cgscc(repeat<3>(no-op-cgscc))' %s 2>&1 \
 ; RUN:     | FileCheck %s --check-prefix=CHECK-REPEAT-CGSCC-PASS
 ; CHECK-REPEAT-CGSCC-PASS: Starting llvm::Module pass manager run
-; CHECK-REPEAT-CGSCC-PASS-NEXT: Running pass: ModuleToPostOrderCGSCCPassAdaptor
 ; CHECK-REPEAT-CGSCC-PASS-NEXT: Running analysis: InnerAnalysisManagerProxy<{{.*(CGSCCAnalysisManager|AnalysisManager<.*LazyCallGraph::SCC.*>).*}},{{.*}}Module>
 ; CHECK-REPEAT-CGSCC-PASS-NEXT: Running analysis: InnerAnalysisManagerProxy<{{.*(FunctionAnalysisManager|AnalysisManager<.*Function.*>).*}},{{.*}}Module>
 ; CHECK-REPEAT-CGSCC-PASS-NEXT: Running analysis: LazyCallGraphAnalysis
 ; CHECK-REPEAT-CGSCC-PASS-NEXT: Running analysis: TargetLibraryAnalysis
-; CHECK-REPEAT-CGSCC-PASS-NEXT: Running analysis: PassInstrumentationAnalysis
 ; CHECK-REPEAT-CGSCC-PASS-NEXT: Running analysis: FunctionAnalysisManagerCGSCCProxy
-; CHECK-REPEAT-CGSCC-PASS-NEXT: Running analysis: PassInstrumentationAnalysis
 ; CHECK-REPEAT-CGSCC-PASS-NEXT: Running analysis: OuterAnalysisManagerProxy<{{.*}}LazyCallGraph::SCC{{.*}}>
 ; CHECK-REPEAT-CGSCC-PASS-NEXT: Starting CGSCC pass manager run
 ; CHECK-REPEAT-CGSCC-PASS-NEXT: Running pass: RepeatedPass
@@ -433,9 +424,7 @@
 ; RUN:     -passes='function(repeat<3>(no-op-function))' %s 2>&1 \
 ; RUN:     | FileCheck %s --check-prefix=CHECK-REPEAT-FUNCTION-PASS
 ; CHECK-REPEAT-FUNCTION-PASS: Starting llvm::Module pass manager run
-; CHECK-REPEAT-FUNCTION-PASS-NEXT: Running pass: ModuleToFunctionPassAdaptor
 ; CHECK-REPEAT-FUNCTION-PASS-NEXT: Running analysis: InnerAnalysisManagerProxy<{{.*}}>
-; CHECK-REPEAT-FUNCTION-PASS-NEXT: Running analysis: PassInstrumentationAnalysis
 ; CHECK-REPEAT-FUNCTION-PASS-NEXT: Starting llvm::Function pass manager run
 ; CHECK-REPEAT-FUNCTION-PASS-NEXT: Running pass: RepeatedPass
 ; CHECK-REPEAT-FUNCTION-PASS-NEXT: Starting llvm::Function pass manager run
@@ -454,11 +443,8 @@
 ; RUN:     -passes='loop(repeat<3>(no-op-loop))' %s 2>&1 \
 ; RUN:     | FileCheck %s --check-prefix=CHECK-REPEAT-LOOP-PASS
 ; CHECK-REPEAT-LOOP-PASS: Starting llvm::Module pass manager run
-; CHECK-REPEAT-LOOP-PASS-NEXT: Running pass: ModuleToFunctionPassAdaptor
 ; CHECK-REPEAT-LOOP-PASS-NEXT: Running analysis: InnerAnalysisManagerProxy<{{.*}}>
-; CHECK-REPEAT-LOOP-PASS-NEXT: Running analysis: PassInstrumentationAnalysis
 ; CHECK-REPEAT-LOOP-PASS-NEXT: Starting llvm::Function pass manager run
-; CHECK-REPEAT-LOOP-PASS-NEXT: Running pass: FunctionToLoopPassAdaptor
 ; CHECK-REPEAT-LOOP-PASS-NEXT: Starting llvm::Function pass manager run
 ; CHECK-REPEAT-LOOP-PASS-NEXT: Running pass: LoopSimplify
 ; CHECK-REPEAT-LOOP-PASS-NEXT: Running analysis: LoopAnalysis
@@ -473,7 +459,6 @@
 ; CHECK-REPEAT-LOOP-PASS-NEXT: Running analysis: TargetIRAnalysis
 ; CHECK-REPEAT-LOOP-PASS-NEXT: Running analysis: InnerAnalysisManagerProxy<{{.*}}>
 ; CHECK-REPEAT-LOOP-PASS-NEXT: Starting Loop pass manager run
-; CHECK-REPEAT-LOOP-PASS-NEXT: Running analysis: PassInstrumentationAnalysis
 ; CHECK-REPEAT-LOOP-PASS-NEXT: Running pass: RepeatedPass
 ; CHECK-REPEAT-LOOP-PASS-NEXT: Starting Loop pass manager run
 ; CHECK-REPEAT-LOOP-PASS-NEXT: Running pass: NoOpLoopPass
diff --git a/llvm/test/Other/new-pm-defaults.ll b/llvm/test/Other/new-pm-defaults.ll
index 56679e6a8e435..8f75e3ce0bf36 100644
--- a/llvm/test/Other/new-pm-defaults.ll
+++ b/llvm/test/Other/new-pm-defaults.ll
@@ -88,20 +88,14 @@
 ; RUN:     --check-prefix=%llvmcheckext \
 ; RUN:     --check-prefix=CHECK-EP-OPTIMIZER-LAST --check-prefix=CHECK-O23SZ
 
-; CHECK-O: Running analysis: PassInstrumentationAnalysis
-; CHECK-O-NEXT: Starting llvm::Module pass manager run.
-; CHECK-O-NEXT: Running pass: PassManager<{{.*}}Module{{.*}}>
+; CHECK-O: Starting llvm::Module pass manager run.
 ; CHECK-O-NEXT: Starting llvm::Module pass manager run.
 ; CHECK-O-NEXT: Running pass: ForceFunctionAttrsPass
 ; CHECK-EP-PIPELINE-START-NEXT: Running pass: NoOpModulePass
-; CHECK-O-NEXT: Running pass: PassManager<{{.*}}Module{{.*}}>
 ; CHECK-O-NEXT: Starting llvm::Module pass manager run.
 ; CHECK-O-NEXT: Running pass: InferFunctionAttrsPass
 ; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy
 ; CHECK-O-NEXT: Running analysis: TargetLibraryAnalysis
-; CHECK-O-NEXT: Running analysis: PassInstrumentationAnalysis
-; CHECK-O-NEXT: Running pass: ModuleToFunctionPassAdaptor<{{.*}}PassManager{{.*}}>
-; CHECK-O-NEXT: Running analysis: PassInstrumentationAnalysis
 ; CHECK-O-NEXT: Starting llvm::Function pass manager run.
 ; CHECK-O-NEXT: Running pass: SimplifyCFGPass
 ; CHECK-O-NEXT: Running analysis: TargetIRAnalysis
@@ -116,9 +110,8 @@
 ; CHECK-O-NEXT: Running pass: IPSCCPPass
 ; CHECK-O-NEXT: Running pass: CalledValuePropagationPass
 ; CHECK-O-NEXT: Running pass: GlobalOptPass
-; CHECK-O-NEXT: Running pass: ModuleToFunctionPassAdaptor<{{.*}}PromotePass>
+; CHECK-O-NEXT: Running pass: PromotePass
 ; CHECK-O-NEXT: Running pass: DeadArgumentEliminationPass
-; CHECK-O-NEXT: Running pass: ModuleToFunctionPassAdaptor<{{.*}}PassManager{{.*}}>
 ; CHECK-O-NEXT: Starting llvm::Function pass manager run.
 ; CHECK-O-NEXT: Running pass: InstCombinePass
 ; CHECK-O-NEXT: Running analysis: OptimizationRemarkEmitterAnalysis
@@ -135,19 +128,17 @@
 ; CHECK-O-NEXT: Running analysis: CallGraphAnalysis
 ; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}ProfileSummaryAnalysis
 ; CHECK-O-NEXT: Running analysis: ProfileSummaryAnalysis
-; CHECK-O-NEXT: Running pass: ModuleToPostOrderCGSCCPassAdaptor<{{.*}}LazyCallGraph{{.*}}>
 ; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy
 ; CHECK-O-NEXT: Running analysis: LazyCallGraphAnalysis
 ; CHECK-O-NEXT: Running analysis: FunctionAnalysisManagerCGSCCProxy
-; CHECK-O-NEXT: Running analysis: PassInstrumentationAnalysis
 ; CHECK-O-NEXT: Running analysis: OuterAnalysisManagerProxy<{{.*}}LazyCallGraph::SCC{{.*}}>
+; CHECK-O-NEXT: Running pass: DevirtSCCRepeatedPass
 ; CHECK-O-NEXT: Starting CGSCC pass manager run.
 ; CHECK-O-NEXT: Running pass: InlinerPass
 ; CHECK-O-NEXT: Running pass: PostOrderFunctionAttrsPass
 ; CHECK-O3-NEXT: Running pass: ArgumentPromotionPass
 ; CHECK-O2-NEXT: Running pass: OpenMPOptPass on (foo)
 ; CHECK-O3-NEXT: Running pass: OpenMPOptPass on (foo)
-; CHECK-O-NEXT: Running pass: CGSCCToFunctionPassAdaptor<{{.*}}PassManager{{.*}}>
 ; CHECK-O-NEXT: Starting llvm::Function pass manager run.
 ; CHECK-O-NEXT: Running pass: SROA
 ; CHECK-O-NEXT: Running pass: EarlyCSEPass
@@ -167,7 +158,6 @@
 ; CHECK-O-NEXT: Running pass: SimplifyCFGPass
 ; CHECK-O-NEXT: Running pass: ReassociatePass
 ; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}OptimizationRemarkEmitterAnalysis
-; CHECK-O-NEXT: Running pass: FunctionToLoopPassAdaptor<{{.*}}LoopStandardAnalysisResults{{.*}}>
 ; CHECK-O-NEXT: Starting llvm::Function pass manager run.
 ; CHECK-O-NEXT: Running pass: LoopSimplifyPass
 ; CHECK-O-NEXT: Running analysis: LoopAnalysis
@@ -176,7 +166,6 @@
 ; CHECK-O-NEXT: Running analysis: ScalarEvolutionAnalysis
 ; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy
 ; CHECK-O-NEXT: Starting Loop pass manager run.
-; CHECK-O-NEXT: Running analysis: PassInstrumentationAnalysis
 ; CHECK-O-NEXT: Running pass: LoopInstSimplifyPass
 ; CHECK-O-NEXT: Running pass: LoopSimplifyCFGPass
 ; CHECK-O-NEXT: Running pass: LoopRotatePass
@@ -185,7 +174,6 @@
 ; CHECK-O-NEXT: Finished Loop pass manager run.
 ; CHECK-O-NEXT: Running pass: SimplifyCFGPass
 ; CHECK-O-NEXT: Running pass: InstCombinePass
-; CHECK-O-NEXT: Running pass: FunctionToLoopPassAdaptor<{{.*}}LoopStandardAnalysisResults{{.*}}>
 ; CHECK-O-NEXT: Starting llvm::Function pass manager run.
 ; CHECK-O-NEXT: Running pass: LoopSimplifyPass
 ; CHECK-O-NEXT: Running pass: LCSSAPass
@@ -214,11 +202,11 @@
 ; CHECK-O23SZ-NEXT: Running pass: JumpThreadingPass
 ; CHECK-O23SZ-NEXT: Running pass: CorrelatedValuePropagationPass
 ; CHECK-O23SZ-NEXT: Running pass: DSEPass
-; CHECK-O23SZ-NEXT: Running pass: FunctionToLoopPassAdaptor<{{.*}}LICMPass{{.*}}>
 ; CHECK-O23SZ-NEXT: Starting llvm::Function pass manager run.
 ; CHECK-O23SZ-NEXT: Running pass: LoopSimplifyPass
 ; CHECK-O23SZ-NEXT: Running pass: LCSSAPass
 ; CHECK-O23SZ-NEXT: Finished llvm::Function pass manager run.
+; CHECK-O23SZ-NEXT: Running pass: LICMPass
 ; CHECK-EP-SCALAR-LATE-NEXT: Running pass: NoOpFunctionPass
 ; CHECK-O-NEXT: Running pass: ADCEPass
 ; CHECK-O-NEXT: Running analysis: PostDominatorTreeAnalysis
@@ -230,24 +218,22 @@
 ; CHECK-O-NEXT: Finished CGSCC pass manager run.
 ; CHECK-O-NEXT: Finished llvm::Module pass manager run.
 ; CHECK-O-NEXT: Finished llvm::Module pass manager run.
-; CHECK-O-NEXT: Running pass: PassManager<{{.*}}Module{{.*}}>
 ; CHECK-O-NEXT: Starting llvm::Module pass manager run.
 ; CHECK-O-NEXT: Running pass: GlobalOptPass
 ; CHECK-O-NEXT: Running pass: GlobalDCEPass
 ; CHECK-O2-LTO-NOT: Running pass: EliminateAvailableExternallyPass
 ; CHECK-O: Running pass: ReversePostOrderFunctionAttrsPass
 ; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}GlobalsAA
-; CHECK-O-NEXT: Running pass: ModuleToFunctionPassAdaptor<{{.*}}PassManager{{.*}}>
 ; CHECK-O-NEXT: Starting llvm::Function pass manager run.
 ; CHECK-O-NEXT: Running pass: Float2IntPass
 ; CHECK-O-NEXT: Running pass: LowerConstantIntrinsicsPass on foo
 ; CHECK-EP-VECTORIZER-START-NEXT: Running pass: NoOpFunctionPass
 ; CHECK-EXT: Running pass: {{.*}}::Bye on foo
-; CHECK-O-NEXT: Running pass: FunctionToLoopPassAdaptor<{{.*}}LoopRotatePass
 ; CHECK-O-NEXT: Starting llvm::Function pass manager run.
 ; CHECK-O-NEXT: Running pass: LoopSimplifyPass
 ; CHECK-O-NEXT: Running pass: LCSSAPass
 ; CHECK-O-NEXT: Finished llvm::Function pass manager run.
+; CHECK-O-NEXT: Running pass: LoopRotatePass
 ; CHECK-O-NEXT: Running pass: LoopDistributePass
 ; CHECK-O-NEXT: Running pass: InjectTLIMappings
 ; CHECK-O-NEXT: Running pass: LoopVectorizePass
@@ -266,11 +252,11 @@
 ; CHECK-O-NEXT: Running pass: WarnMissedTransformationsPass
 ; CHECK-O-NEXT: Running pass: InstCombinePass
 ; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}OptimizationRemarkEmitterAnalysis
-; CHECK-O-NEXT: Running pass: FunctionToLoopPassAdaptor<{{.*}}LICMPass
 ; CHECK-O-NEXT: Starting llvm::Function pass manager run.
 ; CHECK-O-NEXT: Running pass: LoopSimplifyPass
 ; CHECK-O-NEXT: Running pass: LCSSAPass
 ; CHECK-O-NEXT: Finished llvm::Function pass manager run.
+; CHECK-O-NEXT: Running pass: LICMPass
 ; CHECK-O-NEXT: Running pass: AlignmentFromAssumptionsPass
 ; CHECK-O-NEXT: Running pass: LoopSinkPass
 ; CHECK-O-NEXT: Running pass: InstSimplifyPass
diff --git a/llvm/test/Other/new-pm-lto-defaults.ll b/llvm/test/Other/new-pm-lto-defaults.ll
index 0283bc8d0a107..a3be19ca29f1f 100644
--- a/llvm/test/Other/new-pm-lto-defaults.ll
+++ b/llvm/test/Other/new-pm-lto-defaults.ll
@@ -23,19 +23,13 @@
 ; RUN:     | FileCheck %s --check-prefix=CHECK-O --check-prefix=CHECK-O2 \
 ; RUN:     --check-prefix=CHECK-O3 --check-prefix=CHECK-EP-Peephole
 
-; CHECK-O: Running analysis: PassInstrumentationAnalysis
-; CHECK-O-NEXT: Starting llvm::Module pass manager run.
-; CHECK-O-NEXT: Running pass: PassManager<{{.*}}Module
+; CHECK-O: Starting llvm::Module pass manager run.
 ; CHECK-O-NEXT: Starting llvm::Module pass manager run.
 ; CHECK-O-NEXT: Running pass: GlobalDCEPass
 ; CHECK-O-NEXT: Running pass: ForceFunctionAttrsPass
 ; CHECK-O-NEXT: Running pass: InferFunctionAttrsPass
 ; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy<{{.*}}Module
 ; CHECK-O-NEXT: Running analysis: TargetLibraryAnalysis
-; CHECK-O-NEXT: Running analysis: PassInstrumentationAnalysis
-; CHECK-O1-NEXT: Running pass: ModuleToPostOrderCGSCCPassAdaptor<{{.*}}PostOrderFunctionAttrsPass>
-; CHECK-O2-NEXT: Running pass: ModuleToFunctionPassAdaptor<{{.*}}PassManager{{.*}}>
-; CHECK-O2-NEXT: Running analysis: PassInstrumentationAnalysis
 ; CHECK-O2-NEXT: Starting llvm::Function pass manager run.
 ; CHECK-O2-NEXT: Running pass: CallSiteSplittingPass on foo
 ; CHECK-O2-NEXT: Running analysis: TargetLibraryAnalysis on foo
@@ -48,14 +42,12 @@
 ; CHECK-O2-NEXT: Running pass: IPSCCPPass
 ; CHECK-O2-NEXT: Running analysis: AssumptionAnalysis on foo
 ; CHECK-O2-NEXT: Running pass: CalledValuePropagationPass
-; CHECK-O2-NEXT: Running pass: ModuleToPostOrderCGSCCPassAdaptor<{{.*}}PostOrderFunctionAttrsPass>
 ; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy<{{.*}}SCC
 ; CHECK-O-NEXT: Running analysis: LazyCallGraphAnalysis
 ; CHECK-O1-NEXT: Running analysis: TargetLibraryAnalysis
-; CHECK-O1-NEXT: Running analysis: PassInstrumentationAnalysis
 ; CHECK-O-NEXT: Running analysis: FunctionAnalysisManagerCGSCCProxy
-; CHECK-O-NEXT: Running analysis: PassInstrumentationAnalysis
 ; CHECK-O-NEXT: Running analysis: OuterAnalysisManagerProxy<{{.*}}LazyCallGraph{{.*}}>
+; CHECK-O-NEXT: Running pass: PostOrderFunctionAttrsPass
 ; CHECK-O-NEXT: Running analysis: AAManager
 ; CHECK-O-NEXT: Running pass: ReversePostOrderFunctionAttrsPass
 ; CHECK-O-NEXT: Running analysis: CallGraphAnalysis
@@ -63,10 +55,9 @@
 ; CHECK-O-NEXT: Running pass: WholeProgramDevirtPass
 ; CHECK-O1-NEXT: Running pass: LowerTypeTestsPass
 ; CHECK-O2-NEXT: Running pass: GlobalOptPass
-; CHECK-O2-NEXT: Running pass: ModuleToFunctionPassAdaptor<{{.*}}PromotePass>
+; CHECK-O2-NEXT: Running pass: PromotePass
 ; CHECK-O2-NEXT: Running pass: ConstantMergePass
 ; CHECK-O2-NEXT: Running pass: DeadArgumentEliminationPass
-; CHECK-O2-NEXT: Running pass: ModuleToFunctionPassAdaptor<{{.*}}PassManager{{.*}}>
 ; CHECK-O2-NEXT: Starting llvm::Function pass manager run.
 ; CHECK-O3-NEXT: Running pass: AggressiveInstCombinePass
 ; CHECK-O2-NEXT: Running pass: InstCombinePass
@@ -76,14 +67,12 @@
 ; CHECK-O2-NEXT: Running pass: ModuleInlinerWrapperPass
 ; CHECK-O2-NEXT: Running analysis: InlineAdvisorAnalysis
 ; CHECK-O2-NEXT: Starting llvm::Module pass manager run.
-; CHECK-O2-NEXT: Running pass: ModuleToPostOrderCGSCCPassAdaptor<{{.*}}PassManager{{.*}}>
 ; CHECK-O2-NEXT: Starting CGSCC pass manager run.
 ; CHECK-O2-NEXT: Running pass: InlinerPass
 ; CHECK-O2-NEXT: Finished CGSCC pass manager run.
 ; CHECK-O2-NEXT: Finished llvm::Module pass manager run.
 ; CHECK-O2-NEXT: Running pass: GlobalOptPass
 ; CHECK-O2-NEXT: Running pass: GlobalDCEPass
-; CHECK-O2-NEXT: Running pass: ModuleToFunctionPassAdaptor<{{.*}}PassManager{{.*}}>
 ; CHECK-O2-NEXT: Starting llvm::Function pass manager run.
 ; CHECK-O2-NEXT: Running pass: InstCombinePass
 ; CHECK-EP-Peephole-NEXT: Running pass: NoOpFunctionPass
@@ -92,15 +81,25 @@
 ; CHECK-O2-NEXT: Running pass: SROA on foo
 ; CHECK-O2-NEXT: Running pass: TailCallElimPass on foo
 ; CHECK-O2-NEXT: Finished llvm::Function pass manager run.
-; CHECK-O2-NEXT: Running pass: ModuleToPostOrderCGSCCPassAdaptor<{{.*}}PostOrderFunctionAttrsPass>
-; CHECK-O2-NEXT: Running pass: ModuleToFunctionPassAdaptor<{{.*}}PassManager{{.*}}>
+; CHECK-O2-NEXT: Running pass: PostOrderFunctionAttrsPass
+; CHECK-O2-NEXT: Running pass: GVN on foo
 ; CHECK-O2-NEXT: Running analysis: MemoryDependenceAnalysis
 ; CHECK-O2-NEXT: Running analysis: PhiValuesAnalysis
+; CHECK-O2-NEXT: Running pass: MemCpyOptPass on foo
+; CHECK-O2-NEXT: Running pass: DSEPass on foo
+; CHECK-O2-NEXT: Running pass: InstCombinePass on foo
+; CHECK-O2-NEXT: Running pass: SimplifyCFGPass on foo
+; CHECK-O2-NEXT: Running pass: SCCPPass on foo
+; CHECK-O2-NEXT: Running pass: InstCombinePass on foo
+; CHECK-O2-NEXT: Running pass: BDCEPass on foo
 ; CHECK-O2-NEXT: Running analysis: DemandedBitsAnalysis
+; CHECK-O2-NEXT: Running pass: InstCombinePass
+; CHECK-EP-Peephole-NEXT: Running pass: NoOpFunctionPass
+; CHECK-O2-NEXT: Running pass: JumpThreadingPass
 ; CHECK-O2-NEXT: Running pass: CrossDSOCFIPass
 ; CHECK-O2-NEXT: Running pass: LowerTypeTestsPass
 ; CHECK-O-NEXT: Running pass: LowerTypeTestsPass
-; CHECK-O2-NEXT: Running pass: ModuleToFunctionPassAdaptor<{{.*}}SimplifyCFGPass>
+; CHECK-O2-NEXT: Running pass: SimplifyCFGPass
 ; CHECK-O2-NEXT: Running pass: EliminateAvailableExternallyPass
 ; CHECK-O2-NEXT: Running pass: GlobalDCEPass
 ; CHECK-O-NEXT: Finished llvm::Module pass manager run.
diff --git a/llvm/test/Other/new-pm-pgo.ll b/llvm/test/Other/new-pm-pgo.ll
index 610f200c19da0..09fd103e172b5 100644
--- a/llvm/test/Other/new-pm-pgo.ll
+++ b/llvm/test/Other/new-pm-pgo.ll
@@ -18,8 +18,8 @@
 ; USE: Running pass: PGOMemOPSizeOpt
 ; USE_POST_LINK: Running pass: PGOIndirectCallPromotion
 ; USE_POST_LINK: Running pass: PGOMemOPSizeOpt
-; SAMPLE_USE_O: Running pass: ModuleToFunctionPassAdaptor<{{.*}}AddDiscriminatorsPass{{.*}}>
-; SAMPLE_USE_PRE_LINK: Running pass: ModuleToFunctionPassAdaptor<{{.*}}AddDiscriminatorsPass{{.*}}>
+; SAMPLE_USE_O: Running pass: AddDiscriminatorsPass
+; SAMPLE_USE_PRE_LINK: Running pass: AddDiscriminatorsPass
 ; SAMPLE_USE: Running pass: SimplifyCFGPass
 ; SAMPLE_USE: Running pass: SROA
 ; SAMPLE_USE: Running pass: EarlyCSEPass
@@ -30,7 +30,7 @@
 ; SAMPLE_USE_POST_LINK-NOT: Running pass: GlobalOptPass
 ; SAMPLE_USE_POST_LINK: Running pass: PGOIndirectCallPromotion
 ; SAMPLE_USE_PRE_LINK-NOT: Running pass: PGOIndirectCallPromotion
-; SAMPLE_GEN: Running pass: ModuleToFunctionPassAdaptor<{{.*}}AddDiscriminatorsPass{{.*}}>
+; SAMPLE_GEN: Running pass: AddDiscriminatorsPass
 ; SPLIT: Running pass: HotColdSplittingPass
 
 define void @foo() {
diff --git a/llvm/test/Other/new-pm-thinlto-defaults.ll b/llvm/test/Other/new-pm-thinlto-defaults.ll
index 78f9022c741dc..a39656cd26f66 100644
--- a/llvm/test/Other/new-pm-thinlto-defaults.ll
+++ b/llvm/test/Other/new-pm-thinlto-defaults.ll
@@ -47,27 +47,20 @@
 ; RUN:     -passes='thinlto<O2>' -S  %s 2>&1 \
 ; RUN:     | FileCheck %s --check-prefixes=CHECK-O,CHECK-O2,CHECK-O23SZ,CHECK-POSTLINK-O,%llvmcheckext,CHECK-POSTLINK-O2
 ;
-; CHECK-O: Running analysis: PassInstrumentationAnalysis
-; CHECK-O-NEXT: Starting llvm::Module pass manager run.
-; CHECK-O-NEXT: Running pass: PassManager<{{.*}}Module{{.*}}>
+; CHECK-O: Starting llvm::Module pass manager run.
 ; CHECK-O-NEXT: Starting llvm::Module pass manager run.
 ; CHECK-O-NEXT: Running pass: ForceFunctionAttrsPass
 ; CHECK-EP-PIPELINE-START-NEXT: Running pass: NoOpModulePass
-; CHECK-DIS-NEXT: Running pass: ModuleToFunctionPassAdaptor<{{.*}}AddDiscriminatorsPass{{.*}}>
 ; CHECK-DIS-NEXT: Running analysis: InnerAnalysisManagerProxy
-; CHECK-O-NEXT: Running pass: PassManager<{{.*}}Module{{.*}}>
+; CHECK-DIS-NEXT: Running pass: AddDiscriminatorsPass
 ; CHECK-O-NEXT: Starting llvm::Module pass manager run.
 ; CHECK-POSTLINK-O-NEXT: Running pass: PGOIndirectCallPromotion
 ; CHECK-POSTLINK-O-NEXT: Running analysis: ProfileSummaryAnalysis
 ; CHECK-POSTLINK-O-NEXT: Running analysis: InnerAnalysisManagerProxy
 ; CHECK-POSTLINK-O-NEXT: Running analysis: OptimizationRemarkEmitterAnalysis
-; CHECK-POSTLINK-O-NEXT: Running analysis: PassInstrumentationAnalysis
 ; CHECK-O-NEXT: Running pass: InferFunctionAttrsPass
 ; CHECK-PRELINK-O-NODIS-NEXT: Running analysis: InnerAnalysisManagerProxy
 ; CHECK-O-NEXT: Running analysis: TargetLibraryAnalysis
-; CHECK-O-NEXT: Running analysis: PassInstrumentationAnalysis
-; CHECK-O-NEXT: Running pass: ModuleToFunctionPassAdaptor<{{.*}}PassManager{{.*}}>
-; CHECK-PRELINK-O-NEXT: Running analysis: PassInstrumentationAnalysis
 ; CHECK-O-NEXT: Starting llvm::Function pass manager run.
 ; CHECK-O-NEXT: Running pass: SimplifyCFGPass
 ; CHECK-O-NEXT: Running analysis: TargetIRAnalysis
@@ -83,9 +76,8 @@
 ; CHECK-O-NEXT: Running pass: IPSCCPPass
 ; CHECK-O-NEXT: Running pass: CalledValuePropagationPass
 ; CHECK-O-NEXT: Running pass: GlobalOptPass
-; CHECK-O-NEXT: Running pass: ModuleToFunctionPassAdaptor<{{.*}}PromotePass>
+; CHECK-O-NEXT: Running pass: PromotePass
 ; CHECK-O-NEXT: Running pass: DeadArgumentEliminationPass
-; CHECK-O-NEXT: Running pass: ModuleToFunctionPassAdaptor<{{.*}}PassManager{{.*}}>
 ; CHECK-O-NEXT: Starting llvm::Function pass manager run.
 ; CHECK-O-NEXT: Running pass: InstCombinePass
 ; CHECK-PRELINK-O-NEXT: Running analysis: OptimizationRemarkEmitterAnalysis
@@ -101,19 +93,17 @@
 ; CHECK-O-NEXT: Running analysis: CallGraphAnalysis
 ; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}ProfileSummaryAnalysis
 ; CHECK-PRELINK-O-NEXT: Running analysis: ProfileSummaryAnalysis
-; CHECK-O-NEXT: Running pass: ModuleToPostOrderCGSCCPassAdaptor<{{.*}}LazyCallGraph{{.*}}>
 ; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy
 ; CHECK-O-NEXT: Running analysis: LazyCallGraphAnalysis
 ; CHECK-O-NEXT: Running analysis: FunctionAnalysisManagerCGSCCProxy
-; CHECK-O-NEXT: Running analysis: PassInstrumentationAnalysis
-; CHECK-O-NEXT: Running analysis: OuterAnalysisManagerProxy<{{.*}}LazyCallGraph::SCC{{.*}}>
+; CHECK-O-NEXT: Running analysis: OuterAnalysisManagerProxy
+; CHECK-O-NEXT: Running pass: DevirtSCCRepeatedPass
 ; CHECK-O-NEXT: Starting CGSCC pass manager run.
 ; CHECK-O-NEXT: Running pass: InlinerPass
 ; CHECK-O-NEXT: Running pass: PostOrderFunctionAttrsPass
 ; CHECK-O3-NEXT: Running pass: ArgumentPromotionPass
 ; CHECK-O2-NEXT: Running pass: OpenMPOptPass on (foo)
 ; CHECK-O3-NEXT: Running pass: OpenMPOptPass on (foo)
-; CHECK-O-NEXT: Running pass: CGSCCToFunctionPassAdaptor<{{.*}}PassManager{{.*}}>
 ; CHECK-O-NEXT: Starting llvm::Function pass manager run.
 ; CHECK-O-NEXT: Running pass: SROA
 ; CHECK-O-NEXT: Running pass: EarlyCSEPass
@@ -132,7 +122,6 @@
 ; CHECK-O-NEXT: Running pass: SimplifyCFGPass
 ; CHECK-O-NEXT: Running pass: ReassociatePass
 ; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}OptimizationRemarkEmitterAnalysis
-; CHECK-O-NEXT: Running pass: FunctionToLoopPassAdaptor<{{.*}}LoopStandardAnalysisResults{{.*}}>
 ; CHECK-O-NEXT: Starting llvm::Function pass manager run
 ; CHECK-O-NEXT: Running pass: LoopSimplifyPass
 ; CHECK-O-NEXT: Running analysis: LoopAnalysis
@@ -141,7 +130,6 @@
 ; CHECK-O-NEXT: Running analysis: ScalarEvolutionAnalysis
 ; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy
 ; CHECK-O-NEXT: Starting Loop pass manager run.
-; CHECK-O-NEXT: Running analysis: PassInstrumentationAnalysis
 ; CHECK-O-NEXT: Running pass: LoopInstSimplifyPass
 ; CHECK-O-NEXT: Running pass: LoopSimplifyCFGPass
 ; CHECK-O-NEXT: Running pass: LoopRotatePass
@@ -150,7 +138,6 @@
 ; CHECK-O-NEXT: Finished Loop pass manager run.
 ; CHECK-O-NEXT: Running pass: SimplifyCFGPass
 ; CHECK-O-NEXT: Running pass: InstCombinePass
-; CHECK-O-NEXT: Running pass: FunctionToLoopPassAdaptor<{{.*}}LoopStandardAnalysisResults{{.*}}>
 ; CHECK-O-NEXT: Starting llvm::Function pass manager run
 ; CHECK-O-NEXT: Running pass: LoopSimplifyPass
 ; CHECK-O-NEXT: Running pass: LCSSAPass
@@ -188,11 +175,11 @@
 ; CHECK-O23SZ-NEXT: Running pass: JumpThreadingPass
 ; CHECK-O23SZ-NEXT: Running pass: CorrelatedValuePropagationPass
 ; CHECK-O23SZ-NEXT: Running pass: DSEPass
-; CHECK-O23SZ-NEXT: Running pass: FunctionToLoopPassAdaptor<{{.*}}LICMPass{{.*}}>
 ; CHECK-O23SZ-NEXT: Starting llvm::Function pass manager run
 ; CHECK-O23SZ-NEXT: Running pass: LoopSimplifyPass
 ; CHECK-O23SZ-NEXT: Running pass: LCSSAPass
 ; CHECK-O23SZ-NEXT: Finished llvm::Function pass manager run
+; CHECK-O23SZ-NEXT: Running pass: LICMPass on Loop at depth 1 containing: %loop
 ; CHECK-O-NEXT: Running pass: ADCEPass
 ; CHECK-O-NEXT: Running analysis: PostDominatorTreeAnalysis
 ; CHECK-O-NEXT: Running pass: SimplifyCFGPass
@@ -202,23 +189,21 @@
 ; CHECK-O-NEXT: Finished llvm::Module pass manager run.
 ; CHECK-O-NEXT: Finished llvm::Module pass manager run.
 ; CHECK-PRELINK-O-NEXT: Running pass: GlobalOptPass
-; CHECK-POSTLINK-O-NEXT: Running pass: PassManager<{{.*}}Module{{.*}}>
 ; CHECK-POSTLINK-O-NEXT: Starting llvm::Module pass manager run.
 ; CHECK-POSTLINK-O-NEXT: Running pass: GlobalOptPass
 ; CHECK-POSTLINK-O-NEXT: Running pass: GlobalDCEPass
 ; CHECK-POSTLINK-O-NEXT: Running pass: EliminateAvailableExternallyPass
 ; CHECK-POSTLINK-O-NEXT: Running pass: ReversePostOrderFunctionAttrsPass
 ; CHECK-POSTLINK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}GlobalsAA
-; CHECK-POSTLINK-O-NEXT: Running pass: ModuleToFunctionPassAdaptor<{{.*}}PassManager{{.*}}>
 ; CHECK-POSTLINK-O-NEXT: Starting llvm::Function pass manager run.
 ; CHECK-POSTLINK-O-NEXT: Running pass: Float2IntPass
 ; CHECK-POSTLINK-O-NEXT: Running pass: LowerConstantIntrinsicsPass
 ; CHECK-EXT: Running pass: {{.*}}::Bye
-; CHECK-POSTLINK-O-NEXT: Running pass: FunctionToLoopPassAdaptor<{{.*}}LoopRotatePass
 ; CHECK-POSTLINK-O-NEXT: Starting llvm::Function pass manager run
 ; CHECK-POSTLINK-O-NEXT: Running pass: LoopSimplifyPass
 ; CHECK-POSTLINK-O-NEXT: Running pass: LCSSAPass
 ; CHECK-POSTLINK-O-NEXT: Finished llvm::Function pass manager run
+; CHECK-POSTLINK-O-NEXT: Running pass: LoopRotatePass
 ; CHECK-POSTLINK-O-NEXT: Running pass: LoopDistributePass
 ; CHECK-POSTLINK-O-NEXT: Running pass: InjectTLIMappings
 ; CHECK-POSTLINK-O-NEXT: Running pass: LoopVectorizePass
@@ -237,11 +222,11 @@
 ; CHECK-POSTLINK-O-NEXT: Running pass: WarnMissedTransformationsPass
 ; CHECK-POSTLINK-O-NEXT: Running pass: InstCombinePass
 ; CHECK-POSTLINK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}OptimizationRemarkEmitterAnalysis
-; CHECK-POSTLINK-O-NEXT: Running pass: FunctionToLoopPassAdaptor<{{.*}}LICMPass
 ; CHECK-POSTLINK-O-NEXT: Starting llvm::Function pass manager run
 ; CHECK-POSTLINK-O-NEXT: Running pass: LoopSimplifyPass
 ; CHECK-POSTLINK-O-NEXT: Running pass: LCSSAPass
 ; CHECK-POSTLINK-O-NEXT: Finished llvm::Function pass manager run
+; CHECK-POSTLINK-O-NEXT: Running pass: LICMPass
 ; CHECK-POSTLINK-O-NEXT: Running pass: AlignmentFromAssumptionsPass
 ; CHECK-POSTLINK-O-NEXT: Running pass: LoopSinkPass
 ; CHECK-POSTLINK-O-NEXT: Running pass: InstSimplifyPass
diff --git a/llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll b/llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll
index cbb6d8ac081cb..333f4c0d413ed 100644
--- a/llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll
+++ b/llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll
@@ -20,23 +20,17 @@
 ; RUN:     -passes='thinlto<O2>' -S  %s 2>&1 \
 ; RUN:     | FileCheck %s --check-prefixes=CHECK-O,CHECK-O2,CHECK-O23SZ,%llvmcheckext --dump-input=fail
 ;
-; CHECK-O: Running analysis: PassInstrumentationAnalysis
-; CHECK-O-NEXT: Starting {{.*}}Module pass manager run.
-; CHECK-O-NEXT: Running pass: PassManager<{{.*}}Module{{.*}}>
+; CHECK-O: Starting {{.*}}Module pass manager run.
 ; CHECK-O-NEXT: Starting {{.*}}Module pass manager run.
 ; CHECK-O-NEXT: Running pass: ForceFunctionAttrsPass
 ; CHECK-EP-PIPELINE-START-NEXT: Running pass: NoOpModulePass
-; CHECK-O-NEXT: Running pass: PassManager<{{.*}}Module{{.*}}>
 ; CHECK-O-NEXT: Starting {{.*}}Module pass manager run.
 ; CHECK-O-NEXT: Running pass: PGOIndirectCallPromotion
 ; CHECK-O-NEXT: Running analysis: ProfileSummaryAnalysis
 ; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy
 ; CHECK-O-NEXT: Running analysis: OptimizationRemarkEmitterAnalysis
-; CHECK-O-NEXT: Running analysis: PassInstrumentationAnalysis
 ; CHECK-O-NEXT: Running pass: InferFunctionAttrsPass
 ; CHECK-O-NEXT: Running analysis: TargetLibraryAnalysis
-; CHECK-O-NEXT: Running analysis: PassInstrumentationAnalysis
-; CHECK-O-NEXT: Running pass: ModuleToFunctionPassAdaptor<{{.*}}PassManager{{.*}}>
 ; CHECK-O-NEXT: Starting {{.*}}Function pass manager run.
 ; CHECK-O-NEXT: Running pass: SimplifyCFGPass
 ; CHECK-O-NEXT: Running analysis: TargetIRAnalysis
@@ -52,9 +46,8 @@
 ; CHECK-O-NEXT: Running pass: IPSCCPPass
 ; CHECK-O-NEXT: Running pass: CalledValuePropagationPass
 ; CHECK-O-NEXT: Running pass: GlobalOptPass
-; CHECK-O-NEXT: Running pass: ModuleToFunctionPassAdaptor<{{.*}}PromotePass>
+; CHECK-O-NEXT: Running pass: PromotePass
 ; CHECK-O-NEXT: Running pass: DeadArgumentEliminationPass
-; CHECK-O-NEXT: Running pass: ModuleToFunctionPassAdaptor<{{.*}}PassManager{{.*}}>
 ; CHECK-O-NEXT: Starting {{.*}}Function pass manager run.
 ; CHECK-O-NEXT: Running pass: InstCombinePass
 ; CHECK-O-NEXT: Running analysis: AAManager
@@ -74,19 +67,17 @@
 ; CHECK-O-NEXT: Running analysis: GlobalsAA
 ; CHECK-O-NEXT: Running analysis: CallGraphAnalysis
 ; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}ProfileSummaryAnalysis
-; CHECK-O-NEXT: Running pass: ModuleToPostOrderCGSCCPassAdaptor<{{.*}}LazyCallGraph{{.*}}>
 ; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy
 ; CHECK-O-NEXT: Running analysis: LazyCallGraphAnalysis
 ; CHECK-O-NEXT: Running analysis: FunctionAnalysisManagerCGSCCProxy
-; CHECK-O-NEXT: Running analysis: PassInstrumentationAnalysis
 ; CHECK-O-NEXT: Running analysis: OuterAnalysisManagerProxy<{{.*}}LazyCallGraph::SCC{{.*}}>
+; CHECK-O-NEXT: Running pass: DevirtSCCRepeatedPass
 ; CHECK-O-NEXT: Starting CGSCC pass manager run.
 ; CHECK-O-NEXT: Running pass: InlinerPass
 ; CHECK-O-NEXT: Running pass: PostOrderFunctionAttrsPass
 ; CHECK-O3-NEXT: Running pass: ArgumentPromotionPass
 ; CHECK-O2-NEXT: Running pass: OpenMPOptPass
 ; CHECK-O3-NEXT: Running pass: OpenMPOptPass
-; CHECK-O-NEXT: Running pass: CGSCCToFunctionPassAdaptor<{{.*}}PassManager{{.*}}>
 ; CHECK-O-NEXT: Starting {{.*}}Function pass manager run.
 ; CHECK-O-NEXT: Running pass: SROA
 ; CHECK-O-NEXT: Running pass: EarlyCSEPass
@@ -105,7 +96,6 @@
 ; CHECK-O-NEXT: Running pass: SimplifyCFGPass
 ; CHECK-O-NEXT: Running pass: ReassociatePass
 ; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}OptimizationRemarkEmitterAnalysis
-; CHECK-O-NEXT: Running pass: FunctionToLoopPassAdaptor<{{.*}}LoopStandardAnalysisResults{{.*}}>
 ; CHECK-O-NEXT: Starting {{.*}}Function pass manager run
 ; CHECK-O-NEXT: Running pass: LoopSimplifyPass
 ; CHECK-O-NEXT: Running pass: LCSSAPass
@@ -113,7 +103,6 @@
 ; CHECK-O-NEXT: Running analysis: ScalarEvolutionAnalysis
 ; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy
 ; CHECK-O-NEXT: Starting Loop pass manager run.
-; CHECK-O-NEXT: Running analysis: PassInstrumentationAnalysis
 ; CHECK-O-NEXT: Running pass: LoopInstSimplifyPass
 ; CHECK-O-NEXT: Running pass: LoopSimplifyCFGPass
 ; CHECK-O-NEXT: Running pass: LoopRotatePass
@@ -122,7 +111,6 @@
 ; CHECK-O-NEXT: Finished Loop pass manager run.
 ; CHECK-O-NEXT: Running pass: SimplifyCFGPass
 ; CHECK-O-NEXT: Running pass: InstCombinePass
-; CHECK-O-NEXT: Running pass: FunctionToLoopPassAdaptor<{{.*}}LoopStandardAnalysisResults{{.*}}>
 ; CHECK-O-NEXT: Starting {{.*}}Function pass manager run
 ; CHECK-O-NEXT: Running pass: LoopSimplifyPass
 ; CHECK-O-NEXT: Running pass: LCSSAPass
@@ -160,11 +148,11 @@
 ; CHECK-O23SZ-NEXT: Running pass: JumpThreadingPass
 ; CHECK-O23SZ-NEXT: Running pass: CorrelatedValuePropagationPass
 ; CHECK-O23SZ-NEXT: Running pass: DSEPass
-; CHECK-O23SZ-NEXT: Running pass: FunctionToLoopPassAdaptor<{{.*}}LICMPass{{.*}}>
 ; CHECK-O23SZ-NEXT: Starting {{.*}}Function pass manager run
 ; CHECK-O23SZ-NEXT: Running pass: LoopSimplifyPass
 ; CHECK-O23SZ-NEXT: Running pass: LCSSAPass
 ; CHECK-O23SZ-NEXT: Finished {{.*}}Function pass manager run
+; CHECK-O23SZ-NEXT: Running pass: LICMPass
 ; CHECK-O-NEXT: Running pass: ADCEPass
 ; CHECK-O-NEXT: Running pass: SimplifyCFGPass
 ; CHECK-O-NEXT: Running pass: InstCombinePass
@@ -172,23 +160,21 @@
 ; CHECK-O-NEXT: Finished CGSCC pass manager run.
 ; CHECK-O-NEXT: Finished {{.*}}Module pass manager run.
 ; CHECK-O-NEXT: Finished {{.*}}Module pass manager run.
-; CHECK-O-NEXT: Running pass: PassManager<{{.*}}Module{{.*}}>
 ; CHECK-O-NEXT: Starting {{.*}}Module pass manager run.
 ; CHECK-O-NEXT: Running pass: GlobalOptPass
 ; CHECK-O-NEXT: Running pass: GlobalDCEPass
 ; CHECK-O-NEXT: Running pass: EliminateAvailableExternallyPass
 ; CHECK-O-NEXT: Running pass: ReversePostOrderFunctionAttrsPass
 ; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}GlobalsAA
-; CHECK-O-NEXT: Running pass: ModuleToFunctionPassAdaptor<{{.*}}PassManager{{.*}}>
 ; CHECK-O-NEXT: Starting {{.*}}Function pass manager run.
 ; CHECK-O-NEXT: Running pass: Float2IntPass
 ; CHECK-O-NEXT: Running pass: LowerConstantIntrinsicsPass
 ; CHECK-EXT: Running pass: {{.*}}::Bye
-; CHECK-O-NEXT: Running pass: FunctionToLoopPassAdaptor<{{.*}}LoopRotatePass
-; CHECK-O-NEXT: Starting {{.*}}Function pass manager run
-; CHECK-O-NEXT: Running pass: LoopSimplifyPass
-; CHECK-O-NEXT: Running pass: LCSSAPass
-; CHECK-O-NEXT: Finished {{.*}}Function pass manager run
+; CHECK-O-NEXT: Starting {{.*}}Function pass manager run.
+; CHECK-O-NEXT: Running pass: LoopSimplifyPass on foo
+; CHECK-O-NEXT: Running pass: LCSSAPass on foo
+; CHECK-O-NEXT: Finished {{.*}}Function pass manager run.
+; CHECK-O-NEXT: Running pass: LoopRotatePass
 ; CHECK-O-NEXT: Running pass: LoopDistributePass
 ; CHECK-O-NEXT: Running pass: InjectTLIMappings
 ; CHECK-O-NEXT: Running pass: LoopVectorizePass
@@ -205,11 +191,11 @@
 ; CHECK-O-NEXT: Running pass: WarnMissedTransformationsPass
 ; CHECK-O-NEXT: Running pass: InstCombinePass
 ; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}OptimizationRemarkEmitterAnalysis
-; CHECK-O-NEXT: Running pass: FunctionToLoopPassAdaptor<{{.*}}LICMPass
 ; CHECK-O-NEXT: Starting {{.*}}Function pass manager run
 ; CHECK-O-NEXT: Running pass: LoopSimplifyPass
 ; CHECK-O-NEXT: Running pass: LCSSAPass
 ; CHECK-O-NEXT: Finished {{.*}}Function pass manager run
+; CHECK-O-NEXT: Running pass: LICMPass
 ; CHECK-O-NEXT: Running pass: AlignmentFromAssumptionsPass
 ; CHECK-O-NEXT: Running pass: LoopSinkPass
 ; CHECK-O-NEXT: Running pass: InstSimplifyPass
diff --git a/llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll b/llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll
index 295a65eb76683..360f7ee07037f 100644
--- a/llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll
+++ b/llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll
@@ -25,20 +25,14 @@
 ; RUN:     -passes='thinlto<O2>' -S  %s 2>&1 \
 ; RUN:     | FileCheck %s --check-prefixes=CHECK-O,CHECK-O2,CHECK-O23SZ,%llvmcheckext --dump-input=fail
 ;
-; CHECK-O: Running analysis: PassInstrumentationAnalysis
-; CHECK-O-NEXT: Starting {{.*}}Module pass manager run.
-; CHECK-O-NEXT: Running pass: PassManager<{{.*}}Module{{.*}}>
+; CHECK-O: Starting {{.*}}Module pass manager run.
 ; CHECK-O-NEXT: Starting {{.*}}Module pass manager run.
 ; CHECK-O-NEXT: Running pass: ForceFunctionAttrsPass
 ; CHECK-EP-PIPELINE-START-NEXT: Running pass: NoOpModulePass
-; CHECK-O-NEXT: Running pass: PassManager<{{.*}}Module{{.*}}>
 ; CHECK-O-NEXT: Starting {{.*}}Module pass manager run.
 ; CHECK-O-NEXT: Running pass: InferFunctionAttrsPass
 ; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy
 ; CHECK-O-NEXT: Running analysis: TargetLibraryAnalysis
-; CHECK-O-NEXT: Running analysis: PassInstrumentationAnalysis
-; CHECK-O-NEXT: Running pass: ModuleToFunctionPassAdaptor<{{.*}}PassManager{{.*}}>
-; CHECK-O-NEXT: Running analysis: PassInstrumentationAnalysis
 ; CHECK-O-NEXT: Starting {{.*}}Function pass manager run.
 ; CHECK-O-NEXT: Running pass: SimplifyCFGPass
 ; CHECK-O-NEXT: Running analysis: TargetIRAnalysis
@@ -63,9 +57,8 @@
 ; CHECK-O-NEXT: Running pass: IPSCCPPass
 ; CHECK-O-NEXT: Running pass: CalledValuePropagationPass
 ; CHECK-O-NEXT: Running pass: GlobalOptPass
-; CHECK-O-NEXT: Running pass: ModuleToFunctionPassAdaptor<{{.*}}PromotePass>
+; CHECK-O-NEXT: Running pass: PromotePass
 ; CHECK-O-NEXT: Running pass: DeadArgumentEliminationPass
-; CHECK-O-NEXT: Running pass: ModuleToFunctionPassAdaptor<{{.*}}PassManager{{.*}}>
 ; CHECK-O-NEXT: Starting {{.*}}Function pass manager run.
 ; CHECK-O-NEXT: Running pass: InstCombinePass
 ; CHECK-O-NEXT: Running analysis: BlockFrequencyAnalysis on foo
@@ -82,19 +75,17 @@
 ; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}GlobalsAA
 ; CHECK-O-NEXT: Running analysis: GlobalsAA
 ; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}ProfileSummaryAnalysis
-; CHECK-O-NEXT: Running pass: ModuleToPostOrderCGSCCPassAdaptor<{{.*}}LazyCallGraph{{.*}}>
 ; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy
 ; CHECK-O-NEXT: Running analysis: LazyCallGraphAnalysis
 ; CHECK-O-NEXT: Running analysis: FunctionAnalysisManagerCGSCCProxy
-; CHECK-O-NEXT: Running analysis: PassInstrumentationAnalysis
-; CHECK-O-NEXT: Running analysis: OuterAnalysisManagerProxy<{{.*}}LazyCallGraph::SCC{{.*}}>
+; CHECK-O-NEXT: Running analysis: OuterAnalysisManagerProxy
+; CHECK-O-NEXT: Running pass: DevirtSCCRepeatedPass
 ; CHECK-O-NEXT: Starting CGSCC pass manager run.
 ; CHECK-O-NEXT: Running pass: InlinerPass
 ; CHECK-O-NEXT: Running pass: PostOrderFunctionAttrsPass
 ; CHECK-O3-NEXT: Running pass: ArgumentPromotionPass
 ; CHECK-O2-NEXT: Running pass: OpenMPOptPass
 ; CHECK-O3-NEXT: Running pass: OpenMPOptPass
-; CHECK-O-NEXT: Running pass: CGSCCToFunctionPassAdaptor<{{.*}}PassManager{{.*}}>
 ; CHECK-O-NEXT: Starting {{.*}}Function pass manager run.
 ; CHECK-O-NEXT: Running pass: SROA
 ; CHECK-O-NEXT: Running pass: EarlyCSEPass
@@ -113,7 +104,6 @@
 ; CHECK-O-NEXT: Running pass: SimplifyCFGPass
 ; CHECK-O-NEXT: Running pass: ReassociatePass
 ; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}OptimizationRemarkEmitterAnalysis
-; CHECK-O-NEXT: Running pass: FunctionToLoopPassAdaptor<{{.*}}LoopStandardAnalysisResults{{.*}}>
 ; CHECK-O-NEXT: Starting {{.*}}Function pass manager run
 ; CHECK-O-NEXT: Running pass: LoopSimplifyPass
 ; CHECK-O-NEXT: Running pass: LCSSAPass
@@ -121,7 +111,6 @@
 ; CHECK-O-NEXT: Running analysis: ScalarEvolutionAnalysis
 ; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy
 ; CHECK-O-NEXT: Starting Loop pass manager run.
-; CHECK-O-NEXT: Running analysis: PassInstrumentationAnalysis
 ; CHECK-O-NEXT: Running pass: LoopInstSimplifyPass
 ; CHECK-O-NEXT: Running pass: LoopSimplifyCFGPass
 ; CHECK-O-NEXT: Running pass: LoopRotatePass
@@ -130,7 +119,6 @@
 ; CHECK-O-NEXT: Finished Loop pass manager run.
 ; CHECK-O-NEXT: Running pass: SimplifyCFGPass
 ; CHECK-O-NEXT: Running pass: InstCombinePass
-; CHECK-O-NEXT: Running pass: FunctionToLoopPassAdaptor<{{.*}}LoopStandardAnalysisResults{{.*}}>
 ; CHECK-O-NEXT: Starting {{.*}}Function pass manager run
 ; CHECK-O-NEXT: Running pass: LoopSimplifyPass
 ; CHECK-O-NEXT: Running pass: LCSSAPass
@@ -168,11 +156,11 @@
 ; CHECK-O23SZ-NEXT: Running pass: JumpThreadingPass
 ; CHECK-O23SZ-NEXT: Running pass: CorrelatedValuePropagationPass
 ; CHECK-O23SZ-NEXT: Running pass: DSEPass
-; CHECK-O23SZ-NEXT: Running pass: FunctionToLoopPassAdaptor<{{.*}}LICMPass{{.*}}>
 ; CHECK-O23SZ-NEXT: Starting {{.*}}Function pass manager run
 ; CHECK-O23SZ-NEXT: Running pass: LoopSimplifyPass
 ; CHECK-O23SZ-NEXT: Running pass: LCSSAPass
 ; CHECK-O23SZ-NEXT: Finished {{.*}}Function pass manager run
+; CHECK-O23SZ-NEXT: Running pass: LICMPass
 ; CHECK-O-NEXT: Running pass: ADCEPass
 ; CHECK-O-NEXT: Running pass: SimplifyCFGPass
 ; CHECK-O-NEXT: Running pass: InstCombinePass
@@ -183,23 +171,21 @@
 ; CHECK-O-NEXT: Finished CGSCC pass manager run.
 ; CHECK-O-NEXT: Finished {{.*}}Module pass manager run.
 ; CHECK-O-NEXT: Finished {{.*}}Module pass manager run.
-; CHECK-O-NEXT: Running pass: PassManager<{{.*}}Module{{.*}}>
 ; CHECK-O-NEXT: Starting {{.*}}Module pass manager run.
 ; CHECK-O-NEXT: Running pass: GlobalOptPass
 ; CHECK-O-NEXT: Running pass: GlobalDCEPass
 ; CHECK-O-NEXT: Running pass: EliminateAvailableExternallyPass
 ; CHECK-O-NEXT: Running pass: ReversePostOrderFunctionAttrsPass
 ; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}GlobalsAA
-; CHECK-O-NEXT: Running pass: ModuleToFunctionPassAdaptor<{{.*}}PassManager{{.*}}>
 ; CHECK-O-NEXT: Starting {{.*}}Function pass manager run.
 ; CHECK-O-NEXT: Running pass: Float2IntPass
 ; CHECK-O-NEXT: Running pass: LowerConstantIntrinsicsPass
 ; CHECK-EXT: Running pass: {{.*}}::Bye
-; CHECK-O-NEXT: Running pass: FunctionToLoopPassAdaptor<{{.*}}LoopRotatePass
 ; CHECK-O-NEXT: Starting {{.*}}Function pass manager run
 ; CHECK-O-NEXT: Running pass: LoopSimplifyPass
 ; CHECK-O-NEXT: Running pass: LCSSAPass
 ; CHECK-O-NEXT: Finished {{.*}}Function pass manager run
+; CHECK-O-NEXT: Running pass: LoopRotatePass
 ; CHECK-O-NEXT: Running pass: LoopDistributePass
 ; CHECK-O-NEXT: Running pass: InjectTLIMappings
 ; CHECK-O-NEXT: Running pass: LoopVectorizePass
@@ -216,11 +202,11 @@
 ; CHECK-O-NEXT: Running pass: WarnMissedTransformationsPass
 ; CHECK-O-NEXT: Running pass: InstCombinePass
 ; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}OptimizationRemarkEmitterAnalysis
-; CHECK-O-NEXT: Running pass: FunctionToLoopPassAdaptor<{{.*}}LICMPass
 ; CHECK-O-NEXT: Starting {{.*}}Function pass manager run
 ; CHECK-O-NEXT: Running pass: LoopSimplifyPass
 ; CHECK-O-NEXT: Running pass: LCSSAPass
 ; CHECK-O-NEXT: Finished {{.*}}Function pass manager run
+; CHECK-O-NEXT: Running pass: LICMPass
 ; CHECK-O-NEXT: Running pass: AlignmentFromAssumptionsPass
 ; CHECK-O-NEXT: Running pass: LoopSinkPass
 ; CHECK-O-NEXT: Running pass: InstSimplifyPass
diff --git a/llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll b/llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll
index 0baf7df0a3d70..e5791095201b8 100644
--- a/llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll
+++ b/llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll
@@ -27,20 +27,14 @@
 ; RUN:     -passes='thinlto-pre-link<O2>,name-anon-globals' -S  %s 2>&1 \
 ; RUN:     | FileCheck %s --check-prefixes=CHECK-DIS,CHECK-O,CHECK-O2,CHECK-O23SZ,CHECK-O123 --dump-input=fail
 ;
-; CHECK-O: Running analysis: PassInstrumentationAnalysis
-; CHECK-O-NEXT: Starting {{.*}}Module pass manager run.
-; CHECK-O-NEXT: Running pass: PassManager<{{.*}}Module{{.*}}>
+; CHECK-O: Starting {{.*}}Module pass manager run.
 ; CHECK-O-NEXT: Starting {{.*}}Module pass manager run.
 ; CHECK-O-NEXT: Running pass: ForceFunctionAttrsPass
 ; CHECK-EP-PIPELINE-START-NEXT: Running pass: NoOpModulePass
-; CHECK-O-NEXT: Running pass: PassManager<{{.*}}Module{{.*}}>
 ; CHECK-O-NEXT: Starting {{.*}}Module pass manager run.
 ; CHECK-O-NEXT: Running pass: InferFunctionAttrsPass
 ; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy
 ; CHECK-O-NEXT: Running analysis: TargetLibraryAnalysis
-; CHECK-O-NEXT: Running analysis: PassInstrumentationAnalysis
-; CHECK-O-NEXT: Running pass: ModuleToFunctionPassAdaptor<{{.*}}PassManager{{.*}}>
-; CHECK-O-NEXT: Running analysis: PassInstrumentationAnalysis
 ; CHECK-O-NEXT: Starting {{.*}}Function pass manager run.
 ; CHECK-O-NEXT: Running pass: SimplifyCFGPass
 ; CHECK-O-NEXT: Running analysis: TargetIRAnalysis
@@ -55,9 +49,8 @@
 ; CHECK-O-NEXT: Running pass: IPSCCPPass
 ; CHECK-O-NEXT: Running pass: CalledValuePropagationPass
 ; CHECK-O-NEXT: Running pass: GlobalOptPass
-; CHECK-O-NEXT: Running pass: ModuleToFunctionPassAdaptor<{{.*}}PromotePass>
+; CHECK-O-NEXT: Running pass: PromotePass
 ; CHECK-O-NEXT: Running pass: DeadArgumentEliminationPass
-; CHECK-O-NEXT: Running pass: ModuleToFunctionPassAdaptor<{{.*}}PassManager{{.*}}>
 ; CHECK-O-NEXT: Starting {{.*}}Function pass manager run.
 ; CHECK-O-NEXT: Running pass: InstCombinePass
 ; CHECK-O-NEXT: Running analysis: OptimizationRemarkEmitterAnalysis
@@ -68,15 +61,16 @@
 ; CHECK-O123-NEXT: Running pass: ModuleInlinerWrapperPass
 ; CHECK-O123-NEXT: Running analysis: InlineAdvisorAnalysis
 ; CHECK-O123-NEXT: Starting {{.*}}Module pass manager run.
-; CHECK-O123-NEXT: Running pass: ModuleToPostOrderCGSCCPassAdaptor<{{.*}}PassManager<{{.*}}LazyCallGraph::SCC
 ; CHECK-O123-NEXT: Running analysis: InnerAnalysisManagerProxy
 ; CHECK-O123-NEXT: Running analysis: LazyCallGraphAnalysis
 ; CHECK-O123-NEXT: Running analysis: FunctionAnalysisManagerCGSCCProxy on (foo)
-; CHECK-O123-NEXT: Running analysis: PassInstrumentationAnalysis on (foo)
 ; CHECK-O123-NEXT: Running analysis: OuterAnalysisManagerProxy
 ; CHECK-O123-NEXT: Starting CGSCC pass manager run.
 ; CHECK-O123-NEXT: Running pass: InlinerPass on (foo)
-; CHECK-O123-NEXT: Running pass: CGSCCToFunctionPassAdaptor<{{.*}}PassManager{{.*}}>
+; CHECK-O123-NEXT: Running pass: SROA on foo
+; CHECK-O123-NEXT: Running pass: EarlyCSEPass on foo
+; CHECK-O123-NEXT: Running pass: SimplifyCFGPass on foo
+; CHECK-O123-NEXT: Running pass: InstCombinePass on foo
 ; CHECK-O123-NEXT: Finished CGSCC pass manager run.
 ; CHECK-O123-NEXT: Finished {{.*}}Module pass manager run.
 ; CHECK-O123-NEXT: Running pass: GlobalDCEPass
@@ -96,7 +90,6 @@
 ; CHECK-O-NEXT: Running pass: PGOIndirectCallPromotion on
 ; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy
 ; CHECK-O-NEXT: Running analysis: OptimizationRemarkEmitterAnalysis on foo
-; CHECK-O-NEXT: Running analysis: PassInstrumentationAnalysis on foo
 ; CHECK-O-NEXT: Running pass: ModuleInlinerWrapperPass
 ; CHECK-Os-NEXT: Running analysis: InlineAdvisorAnalysis
 ; CHECK-Oz-NEXT: Running analysis: InlineAdvisorAnalysis
@@ -105,13 +98,12 @@
 ; CHECK-O-NEXT: Running analysis: GlobalsAA
 ; CHECK-O-NEXT: Running analysis: CallGraphAnalysis
 ; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}ProfileSummaryAnalysis
-; CHECK-O-NEXT: Running pass: ModuleToPostOrderCGSCCPassAdaptor<{{.*}}LazyCallGraph{{.*}}>
 ; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy
 ; CHECK-O-NEXT: Running analysis: LazyCallGraphAnalysis
 ; CHECK-O-NEXT: Running analysis: TargetLibraryAnalysis on foo
 ; CHECK-O-NEXT: Running analysis: FunctionAnalysisManagerCGSCCProxy
-; CHECK-O-NEXT: Running analysis: PassInstrumentationAnalysis
 ; CHECK-O-NEXT: Running analysis: OuterAnalysisManagerProxy<{{.*}}LazyCallGraph::SCC{{.*}}>
+; CHECK-O-NEXT: Running pass: DevirtSCCRepeatedPass
 ; CHECK-O-NEXT: Starting CGSCC pass manager run.
 ; CHECK-O-NEXT: Running pass: InlinerPass
 ; CHECK-O-NEXT: Running pass: PostOrderFunctionAttrsPass
@@ -120,7 +112,6 @@
 ; CHECK-O3-NEXT: Running analysis: TargetIRAnalysis
 ; CHECK-O2-NEXT: Running pass: OpenMPOptPass
 ; CHECK-O3-NEXT: Running pass: OpenMPOptPass
-; CHECK-O-NEXT: Running pass: CGSCCToFunctionPassAdaptor<{{.*}}PassManager{{.*}}>
 ; CHECK-O-NEXT: Starting {{.*}}Function pass manager run.
 ; CHECK-O-NEXT: Running pass: SROA
 ; These next two can appear in any order since they are accessed as parameters
@@ -156,7 +147,6 @@
 ; CHECK-O-NEXT: Running pass: SimplifyCFGPass
 ; CHECK-O-NEXT: Running pass: ReassociatePass
 ; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}OptimizationRemarkEmitterAnalysis
-; CHECK-O-NEXT: Running pass: FunctionToLoopPassAdaptor<{{.*}}LoopStandardAnalysisResults{{.*}}>
 ; CHECK-O-NEXT: Starting {{.*}}Function pass manager run
 ; CHECK-O-NEXT: Running pass: LoopSimplifyPass
 ; CHECK-O-NEXT: Running pass: LCSSAPass
@@ -164,7 +154,6 @@
 ; CHECK-O-NEXT: Running analysis: ScalarEvolutionAnalysis
 ; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy
 ; CHECK-O-NEXT: Starting Loop pass manager run.
-; CHECK-O-NEXT: Running analysis: PassInstrumentationAnalysis
 ; CHECK-O-NEXT: Running pass: LoopInstSimplifyPass
 ; CHECK-O-NEXT: Running pass: LoopSimplifyCFGPass
 ; CHECK-O-NEXT: Running pass: LoopRotatePass
@@ -173,7 +162,6 @@
 ; CHECK-O-NEXT: Finished Loop pass manager run.
 ; CHECK-O-NEXT: Running pass: SimplifyCFGPass
 ; CHECK-O-NEXT: Running pass: InstCombinePass
-; CHECK-O-NEXT: Running pass: FunctionToLoopPassAdaptor<{{.*}}LoopStandardAnalysisResults{{.*}}>
 ; CHECK-O-NEXT: Starting {{.*}}Function pass manager run
 ; CHECK-O-NEXT: Running pass: LoopSimplifyPass
 ; CHECK-O-NEXT: Running pass: LCSSAPass
@@ -211,11 +199,11 @@
 ; CHECK-O23SZ-NEXT: Running pass: JumpThreadingPass
 ; CHECK-O23SZ-NEXT: Running pass: CorrelatedValuePropagationPass
 ; CHECK-O23SZ-NEXT: Running pass: DSEPass
-; CHECK-O23SZ-NEXT: Running pass: FunctionToLoopPassAdaptor<{{.*}}LICMPass{{.*}}>
 ; CHECK-O23SZ-NEXT: Starting {{.*}}Function pass manager run
 ; CHECK-O23SZ-NEXT: Running pass: LoopSimplifyPass
 ; CHECK-O23SZ-NEXT: Running pass: LCSSAPass
 ; CHECK-O23SZ-NEXT: Finished {{.*}}Function pass manager run
+; CHECK-O23SZ-NEXT: Running pass: LICMPass
 ; CHECK-O-NEXT: Running pass: ADCEPass
 ; CHECK-O-NEXT: Running pass: SimplifyCFGPass
 ; CHECK-O-NEXT: Running pass: InstCombinePass
@@ -228,7 +216,6 @@
 ; CHECK-O-NEXT: Finished {{.*}}Module pass manager run.
 ; CHECK-O-NEXT: Running pass: GlobalOptPass
 ; CHECK-O-NEXT: Running analysis: TargetLibraryAnalysis on bar
-; CHECK-O-NEXT: Running analysis: PassInstrumentationAnalysis on bar
 ; CHECK-EXT: Running pass: {{.*}}::Bye
 ; CHECK-O-NEXT: Finished {{.*}}Module pass manager run.
 ; CHECK-O-NEXT: Running pass: NameAnonGlobalPass
diff --git a/llvm/test/Other/new-pm-thinlto-prelink-samplepgo-defaults.ll b/llvm/test/Other/new-pm-thinlto-prelink-samplepgo-defaults.ll
index 439b553903c9e..b062afdfb49cd 100644
--- a/llvm/test/Other/new-pm-thinlto-prelink-samplepgo-defaults.ll
+++ b/llvm/test/Other/new-pm-thinlto-prelink-samplepgo-defaults.ll
@@ -25,21 +25,15 @@
 ; RUN:     -passes='thinlto-pre-link<O2>,name-anon-globals' -S  %s 2>&1 \
 ; RUN:     | FileCheck %s --check-prefixes=CHECK-DIS,CHECK-O,CHECK-O2,CHECK-O23SZ,CHECK-O123 --dump-input=fail
 ;
-; CHECK-O: Running analysis: PassInstrumentationAnalysis
-; CHECK-O-NEXT: Starting {{.*}}Module pass manager run.
-; CHECK-O-NEXT: Running pass: PassManager<{{.*}}Module{{.*}}>
+; CHECK-O: Starting {{.*}}Module pass manager run.
 ; CHECK-O-NEXT: Starting {{.*}}Module pass manager run.
 ; CHECK-O-NEXT: Running pass: ForceFunctionAttrsPass
-; CHECK-O-NEXT: Running pass: ModuleToFunctionPassAdaptor<{{.*}}AddDiscriminatorsPass>
 ; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy
+; CHECK-O-NEXT: Running pass: AddDiscriminatorsPass
 ; CHECK-EP-PIPELINE-START-NEXT: Running pass: NoOpModulePass
-; CHECK-O-NEXT: Running pass: PassManager<{{.*}}Module{{.*}}>
 ; CHECK-O-NEXT: Starting {{.*}}Module pass manager run.
 ; CHECK-O-NEXT: Running pass: InferFunctionAttrsPass
 ; CHECK-O-NEXT: Running analysis: TargetLibraryAnalysis
-; CHECK-O-NEXT: Running analysis: PassInstrumentationAnalysis
-; CHECK-O-NEXT: Running pass: ModuleToFunctionPassAdaptor<{{.*}}PassManager{{.*}}>
-; CHECK-O-NEXT: Running analysis: PassInstrumentationAnalysis
 ; CHECK-O-NEXT: Starting {{.*}}Function pass manager run.
 ; CHECK-O-NEXT: Running pass: SimplifyCFGPass
 ; CHECK-O-NEXT: Running analysis: TargetIRAnalysis
@@ -62,9 +56,8 @@
 ; CHECK-O-NEXT: Running pass: IPSCCPPass
 ; CHECK-O-NEXT: Running pass: CalledValuePropagationPass
 ; CHECK-O-NEXT: Running pass: GlobalOptPass
-; CHECK-O-NEXT: Running pass: ModuleToFunctionPassAdaptor<{{.*}}PromotePass>
+; CHECK-O-NEXT: Running pass: PromotePass
 ; CHECK-O-NEXT: Running pass: DeadArgumentEliminationPass
-; CHECK-O-NEXT: Running pass: ModuleToFunctionPassAdaptor<{{.*}}PassManager{{.*}}>
 ; CHECK-O-NEXT: Starting {{.*}}Function pass manager run.
 ; CHECK-O-NEXT: Running pass: InstCombinePass
 ; CHECK-O-NEXT: Running analysis: BlockFrequencyAnalysis on foo
@@ -81,19 +74,17 @@
 ; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}GlobalsAA
 ; CHECK-O-NEXT: Running analysis: GlobalsAA
 ; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}ProfileSummaryAnalysis
-; CHECK-O-NEXT: Running pass: ModuleToPostOrderCGSCCPassAdaptor<{{.*}}LazyCallGraph{{.*}}>
 ; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy
 ; CHECK-O-NEXT: Running analysis: LazyCallGraphAnalysis
 ; CHECK-O-NEXT: Running analysis: FunctionAnalysisManagerCGSCCProxy
-; CHECK-O-NEXT: Running analysis: PassInstrumentationAnalysis
 ; CHECK-O-NEXT: Running analysis: OuterAnalysisManagerProxy<{{.*}}LazyCallGraph::SCC{{.*}}>
+; CHECK-O-NEXT: Running pass: DevirtSCCRepeatedPass
 ; CHECK-O-NEXT: Starting CGSCC pass manager run.
 ; CHECK-O-NEXT: Running pass: InlinerPass
 ; CHECK-O-NEXT: Running pass: PostOrderFunctionAttrsPass
 ; CHECK-O3-NEXT: Running pass: ArgumentPromotionPass
 ; CHECK-O2-NEXT: Running pass: OpenMPOptPass
 ; CHECK-O3-NEXT: Running pass: OpenMPOptPass
-; CHECK-O-NEXT: Running pass: CGSCCToFunctionPassAdaptor<{{.*}}PassManager{{.*}}>
 ; CHECK-O-NEXT: Starting {{.*}}Function pass manager run.
 ; CHECK-O-NEXT: Running pass: SROA
 ; CHECK-O-NEXT: Running pass: EarlyCSEPass
@@ -112,7 +103,6 @@
 ; CHECK-O-NEXT: Running pass: SimplifyCFGPass
 ; CHECK-O-NEXT: Running pass: ReassociatePass
 ; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}OptimizationRemarkEmitterAnalysis
-; CHECK-O-NEXT: Running pass: FunctionToLoopPassAdaptor<{{.*}}LoopStandardAnalysisResults{{.*}}>
 ; CHECK-O-NEXT: Starting {{.*}}Function pass manager run
 ; CHECK-O-NEXT: Running pass: LoopSimplifyPass
 ; CHECK-O-NEXT: Running pass: LCSSAPass
@@ -120,7 +110,6 @@
 ; CHECK-O-NEXT: Running analysis: ScalarEvolutionAnalysis
 ; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy
 ; CHECK-O-NEXT: Starting Loop pass manager run.
-; CHECK-O-NEXT: Running analysis: PassInstrumentationAnalysis
 ; CHECK-O-NEXT: Running pass: LoopInstSimplifyPass
 ; CHECK-O-NEXT: Running pass: LoopSimplifyCFGPass
 ; CHECK-O-NEXT: Running pass: LoopRotatePass
@@ -129,7 +118,6 @@
 ; CHECK-O-NEXT: Finished Loop pass manager run.
 ; CHECK-O-NEXT: Running pass: SimplifyCFGPass
 ; CHECK-O-NEXT: Running pass: InstCombinePass
-; CHECK-O-NEXT: Running pass: FunctionToLoopPassAdaptor<{{.*}}LoopStandardAnalysisResults{{.*}}>
 ; CHECK-O-NEXT: Starting {{.*}}Function pass manager run
 ; CHECK-O-NEXT: Running pass: LoopSimplifyPass
 ; CHECK-O-NEXT: Running pass: LCSSAPass
@@ -166,11 +154,11 @@
 ; CHECK-O23SZ-NEXT: Running pass: JumpThreadingPass
 ; CHECK-O23SZ-NEXT: Running pass: CorrelatedValuePropagationPass
 ; CHECK-O23SZ-NEXT: Running pass: DSEPass
-; CHECK-O23SZ-NEXT: Running pass: FunctionToLoopPassAdaptor<{{.*}}LICMPass{{.*}}>
 ; CHECK-O23SZ-NEXT: Starting {{.*}}Function pass manager run
 ; CHECK-O23SZ-NEXT: Running pass: LoopSimplifyPass
 ; CHECK-O23SZ-NEXT: Running pass: LCSSAPass
 ; CHECK-O23SZ-NEXT: Finished {{.*}}Function pass manager run
+; CHECK-O23SZ-NEXT: Running pass: LICMPass
 ; CHECK-O-NEXT: Running pass: ADCEPass
 ; CHECK-O-NEXT: Running pass: SimplifyCFGPass
 ; CHECK-O-NEXT: Running pass: InstCombinePass
diff --git a/llvm/test/Other/pass-pipeline-parsing.ll b/llvm/test/Other/pass-pipeline-parsing.ll
index 902bd9b3eabb1..eb1d07c01ab9e 100644
--- a/llvm/test/Other/pass-pipeline-parsing.ll
+++ b/llvm/test/Other/pass-pipeline-parsing.ll
@@ -20,7 +20,6 @@
 ; RUN:     -passes=no-op-function,no-op-function %s 2>&1 \
 ; RUN:     | FileCheck %s --check-prefix=CHECK-TWO-NOOP-FP
 ; CHECK-TWO-NOOP-FP: Starting llvm::Module pass manager run
-; CHECK-TWO-NOOP-FP: Running pass: ModuleToFunctionPassAdaptor
 ; CHECK-TWO-NOOP-FP: Starting llvm::Function pass manager run
 ; CHECK-TWO-NOOP-FP: Running pass: NoOpFunctionPass
 ; CHECK-TWO-NOOP-FP: Running pass: NoOpFunctionPass
@@ -31,7 +30,6 @@
 ; RUN:     -passes='function(no-op-function,no-op-function)' %s 2>&1 \
 ; RUN:     | FileCheck %s --check-prefix=CHECK-NESTED-TWO-NOOP-FP
 ; CHECK-NESTED-TWO-NOOP-FP: Starting llvm::Module pass manager run
-; CHECK-NESTED-TWO-NOOP-FP: Running pass: ModuleToFunctionPassAdaptor
 ; CHECK-NESTED-TWO-NOOP-FP: Starting llvm::Function pass manager run
 ; CHECK-NESTED-TWO-NOOP-FP: Running pass: NoOpFunctionPass
 ; CHECK-NESTED-TWO-NOOP-FP: Running pass: NoOpFunctionPass
@@ -43,7 +41,6 @@
 ; RUN:     | FileCheck %s --check-prefix=CHECK-MIXED-FP-AND-MP
 ; CHECK-MIXED-FP-AND-MP: Starting llvm::Module pass manager run
 ; CHECK-MIXED-FP-AND-MP: Running pass: NoOpModulePass
-; CHECK-MIXED-FP-AND-MP: Running pass: ModuleToFunctionPassAdaptor
 ; CHECK-MIXED-FP-AND-MP: Starting llvm::Function pass manager run
 ; CHECK-MIXED-FP-AND-MP: Running pass: NoOpFunctionPass
 ; CHECK-MIXED-FP-AND-MP: Running pass: NoOpFunctionPass
@@ -105,7 +102,6 @@
 ; RUN:     -passes=no-op-cgscc,no-op-cgscc %s 2>&1 \
 ; RUN:     | FileCheck %s --check-prefix=CHECK-TWO-NOOP-CG
 ; CHECK-TWO-NOOP-CG: Starting llvm::Module pass manager run
-; CHECK-TWO-NOOP-CG: Running pass: ModuleToPostOrderCGSCCPassAdaptor
 ; CHECK-TWO-NOOP-CG: Starting CGSCC pass manager run
 ; CHECK-TWO-NOOP-CG: Running pass: NoOpCGSCCPass
 ; CHECK-TWO-NOOP-CG: Running pass: NoOpCGSCCPass
@@ -117,20 +113,16 @@
 ; RUN:     | FileCheck %s --check-prefix=CHECK-NESTED-MP-CG-FP
 ; CHECK-NESTED-MP-CG-FP: Starting llvm::Module pass manager run
 ; CHECK-NESTED-MP-CG-FP: Starting llvm::Module pass manager run
-; CHECK-NESTED-MP-CG-FP: Running pass: ModuleToFunctionPassAdaptor
 ; CHECK-NESTED-MP-CG-FP: Starting llvm::Function pass manager run
 ; CHECK-NESTED-MP-CG-FP: Running pass: NoOpFunctionPass
 ; CHECK-NESTED-MP-CG-FP: Finished llvm::Function pass manager run
-; CHECK-NESTED-MP-CG-FP: Running pass: ModuleToPostOrderCGSCCPassAdaptor
 ; CHECK-NESTED-MP-CG-FP: Starting CGSCC pass manager run
 ; CHECK-NESTED-MP-CG-FP: Running pass: NoOpCGSCCPass
-; CHECK-NESTED-MP-CG-FP: Running pass: CGSCCToFunctionPassAdaptor
 ; CHECK-NESTED-MP-CG-FP: Starting llvm::Function pass manager run
 ; CHECK-NESTED-MP-CG-FP: Running pass: NoOpFunctionPass
 ; CHECK-NESTED-MP-CG-FP: Finished llvm::Function pass manager run
 ; CHECK-NESTED-MP-CG-FP: Running pass: NoOpCGSCCPass
 ; CHECK-NESTED-MP-CG-FP: Finished CGSCC pass manager run
-; CHECK-NESTED-MP-CG-FP: Running pass: ModuleToFunctionPassAdaptor
 ; CHECK-NESTED-MP-CG-FP: Starting llvm::Function pass manager run
 ; CHECK-NESTED-MP-CG-FP: Running pass: NoOpFunctionPass
 ; CHECK-NESTED-MP-CG-FP: Finished llvm::Function pass manager run
@@ -141,9 +133,7 @@
 ; RUN:     -passes='no-op-loop,no-op-loop' %s 2>&1 \
 ; RUN:     | FileCheck %s --check-prefix=CHECK-TWO-NOOP-LOOP
 ; CHECK-TWO-NOOP-LOOP: Starting llvm::Module pass manager run
-; CHECK-TWO-NOOP-LOOP: Running pass: ModuleToFunctionPassAdaptor
 ; CHECK-TWO-NOOP-LOOP: Starting llvm::Function pass manager run
-; CHECK-TWO-NOOP-LOOP: Running pass: FunctionToLoopPassAdaptor
 ; CHECK-TWO-NOOP-LOOP: Starting Loop pass manager run
 ; CHECK-TWO-NOOP-LOOP: Running pass: NoOpLoopPass
 ; CHECK-TWO-NOOP-LOOP: Running pass: NoOpLoopPass
@@ -164,31 +154,37 @@
 ; RUN:     -passes='no-op-loop' %s 2>&1 \
 ; RUN:     | FileCheck %s --check-prefix=CHECK-NESTED-FP-LP
 ; CHECK-NESTED-FP-LP: Starting llvm::Module pass manager run
-; CHECK-NESTED-FP-LP: Running pass: ModuleToFunctionPassAdaptor
 ; CHECK-NESTED-FP-LP: Starting llvm::Function pass manager run
-; CHECK-NESTED-FP-LP: Running pass: FunctionToLoopPassAdaptor
 ; CHECK-NESTED-FP-LP: Starting Loop pass manager run
 ; CHECK-NESTED-FP-LP: Running pass: NoOpLoopPass
 ; CHECK-NESTED-FP-LP: Finished Loop pass manager run
 ; CHECK-NESTED-FP-LP: Finished llvm::Function pass manager run
 ; CHECK-NESTED-FP-LP: Finished llvm::Module pass manager run
 
-; RUN: opt -disable-output -debug-pass-manager \
+; RUN: opt -disable-output -debug-pass-manager -debug-pass-manager-verbose \
 ; RUN:     -passes='module(no-op-function,no-op-loop,no-op-cgscc,cgscc(no-op-function,no-op-loop),function(no-op-loop))' %s 2>&1 \
 ; RUN:     | FileCheck %s --check-prefix=CHECK-ADAPTORS
 ; CHECK-ADAPTORS: Starting llvm::Module pass manager run
 ; CHECK-ADAPTORS: Starting llvm::Module pass manager run
 ; CHECK-ADAPTORS: Running pass: ModuleToFunctionPassAdaptor<{{.*}}NoOpFunctionPass>
+; CHECK-ADAPTORS: Running pass: NoOpFunctionPass
 ; CHECK-ADAPTORS: Running pass: ModuleToFunctionPassAdaptor<{{.*}}FunctionToLoopPassAdaptor<{{.*}}NoOpLoopPass>{{.*}}>
+; CHECK-ADAPTORS: Running pass: FunctionToLoopPassAdaptor<{{.*}}NoOpLoopPass>
+; CHECK-ADAPTORS: Running pass: NoOpLoopPass on Loop at depth 1 containing: %loop
 ; CHECK-ADAPTORS: Running pass: ModuleToPostOrderCGSCCPassAdaptor<{{.*}}NoOpCGSCCPass>
+; CHECK-ADAPTORS: Running pass: NoOpCGSCCPass
 ; CHECK-ADAPTORS: Running pass: ModuleToPostOrderCGSCCPassAdaptor<{{.*}}LazyCallGraph{{.*}}>
 ; CHECK-ADAPTORS: Starting CGSCC pass manager run
 ; CHECK-ADAPTORS: Running pass: CGSCCToFunctionPassAdaptor<{{.*}}NoOpFunctionPass>
+; CHECK-ADAPTORS: Running pass: NoOpFunctionPass
 ; CHECK-ADAPTORS: Running pass: CGSCCToFunctionPassAdaptor<{{.*}}FunctionToLoopPassAdaptor<{{.*}}NoOpLoopPass>{{.*}}>
+; CHECK-ADAPTORS: Running pass: FunctionToLoopPassAdaptor<{{.*}}NoOpLoopPass>
+; CHECK-ADAPTORS: Running pass: NoOpLoopPass on Loop at depth 1 containing: %loop
 ; CHECK-ADAPTORS: Finished CGSCC pass manager run
 ; CHECK-ADAPTORS: Running pass: ModuleToFunctionPassAdaptor<{{.*}}PassManager{{.*}}>
 ; CHECK-ADAPTORS: Starting llvm::Function pass manager run
 ; CHECK-ADAPTORS: Running pass: FunctionToLoopPassAdaptor<{{.*}}NoOpLoopPass>
+; CHECK-ADAPTORS: Running pass: NoOpLoopPass on Loop at depth 1 containing: %loop
 ; CHECK-ADAPTORS: Finished llvm::Function pass manager run
 ; CHECK-ADAPTORS: Finished llvm::Module pass manager run
 ; CHECK-ADAPTORS: Finished llvm::Module pass manager run
@@ -197,9 +193,8 @@
 ; RUN:     -passes='cgscc(print)' %s 2>&1 \
 ; RUN:     | FileCheck %s --check-prefix=CHECK-PRINT-IN-CGSCC
 ; CHECK-PRINT-IN-CGSCC: Starting llvm::Module pass manager run
-; CHECK-PRINT-IN-CGSCC: Running pass: ModuleToPostOrderCGSCCPassAdaptor<{{.*}}LazyCallGraph{{.*}}>
 ; CHECK-PRINT-IN-CGSCC: Starting CGSCC pass manager run
-; CHECK-PRINT-IN-CGSCC: Running pass: CGSCCToFunctionPassAdaptor<{{.*}}PrintFunctionPass>
+; CHECK-PRINT-IN-CGSCC: Running pass: PrintFunctionPass
 ; CHECK-PRINT-IN-CGSCC: Finished CGSCC pass manager run
 ; CHECK-PRINT-IN-CGSCC: Running pass: VerifierPass
 ; CHECK-PRINT-IN-CGSCC: Finished llvm::Module pass manager run
diff --git a/llvm/test/Transforms/LoopRotate/pr35210.ll b/llvm/test/Transforms/LoopRotate/pr35210.ll
index 6fddeeeb90893..a615e53aec2b9 100644
--- a/llvm/test/Transforms/LoopRotate/pr35210.ll
+++ b/llvm/test/Transforms/LoopRotate/pr35210.ll
@@ -8,7 +8,6 @@
 ; CHECK: Starting llvm::Function pass manager run.
 ; CHECK-NEXT: Running pass: ADCEPass on f
 ; CHECK-NEXT: Running analysis: PostDominatorTreeAnalysis on f
-; CHECK-NEXT: Running pass: FunctionToLoopPassAdaptor{{.*}} on f
 ; CHECK-NEXT: Starting llvm::Function pass manager run.
 ; CHECK-NEXT: Running pass: LoopSimplifyPass on f
 ; CHECK-NEXT: Running analysis: LoopAnalysis on f
@@ -22,7 +21,6 @@
 ; CHECK-NEXT: Running analysis: TargetIRAnalysis on f
 ; CHECK-NEXT: Running analysis: InnerAnalysisManagerProxy{{.*}} on f
 ; CHECK-NEXT: Starting Loop pass manager run.
-; CHECK-NEXT: Running analysis: PassInstrumentationAnalysis on bb
 ; CHECK-NEXT: Running pass: LoopRotatePass on Loop at depth 1 containing: %bb<header><exiting>,%bb4<latch>
 ; CHECK-NEXT: Folding loop latch bb4 into bb
 ; CHECK-NEXT: Invalidating all non-preserved analyses for: bb
@@ -36,7 +34,6 @@
 ; MSSA: Starting llvm::Function pass manager run.
 ; MSSA-NEXT: Running pass: ADCEPass on f
 ; MSSA-NEXT: Running analysis: PostDominatorTreeAnalysis on f
-; MSSA-NEXT: Running pass: FunctionToLoopPassAdaptor{{.*}} on f
 ; MSSA-NEXT: Starting llvm::Function pass manager run.
 ; MSSA-NEXT: Running pass: LoopSimplifyPass on f
 ; MSSA-NEXT: Running analysis: LoopAnalysis on f
@@ -51,7 +48,6 @@
 ; MSSA-NEXT: Running analysis: TargetIRAnalysis on f
 ; MSSA-NEXT: Running analysis: InnerAnalysisManagerProxy{{.*}} on f
 ; MSSA-NEXT: Starting Loop pass manager run.
-; MSSA-NEXT: Running analysis: PassInstrumentationAnalysis on bb
 ; MSSA-NEXT: Running pass: LoopRotatePass on Loop at depth 1 containing: %bb<header><exiting>,%bb4<latch>
 ; MSSA-NEXT: Folding loop latch bb4 into bb
 ; MSSA-NEXT: Invalidating all non-preserved analyses for: bb
diff --git a/llvm/test/Transforms/LoopUnroll/revisit.ll b/llvm/test/Transforms/LoopUnroll/revisit.ll
index 8c4f04d10854a..4ad867fa72a4c 100644
--- a/llvm/test/Transforms/LoopUnroll/revisit.ll
+++ b/llvm/test/Transforms/LoopUnroll/revisit.ll
@@ -15,7 +15,7 @@
 ; Basic test is fully unrolled and we revisit the post-unroll new sibling
 ; loops, including the ones that used to be child loops.
 define void @full_unroll(i1* %ptr) {
-; CHECK-LABEL: FunctionToLoopPassAdaptor{{.*}} on full_unroll
+; CHECK-LABEL: OptimizationRemarkEmitterAnalysis on full_unroll
 ; CHECK-NOT: LoopFullUnrollPass
 
 entry:
@@ -81,7 +81,7 @@ exit:
 ; duplicating child loops without changing their structure and so they aren't by
 ; default visited, but will be visited with a special parameter.
 define void @partial_unroll(i32 %count, i1* %ptr) {
-; CHECK-LABEL: FunctionToLoopPassAdaptor{{.*}} on partial_unroll
+; CHECK-LABEL: OptimizationRemarkEmitterAnalysis on partial_unroll
 ; CHECK-NOT: LoopFullUnrollPass
 
 entry:
diff --git a/llvm/test/Transforms/LoopUnroll/unroll-loop-invalidation.ll b/llvm/test/Transforms/LoopUnroll/unroll-loop-invalidation.ll
index ec377ffb2235b..8a3e6e17f505d 100644
--- a/llvm/test/Transforms/LoopUnroll/unroll-loop-invalidation.ll
+++ b/llvm/test/Transforms/LoopUnroll/unroll-loop-invalidation.ll
@@ -6,20 +6,19 @@
 ; RUN: opt -S -passes='loop(require<access-info>),loop-unroll,loop(print-access-info)' -debug-pass-manager < %s 2>&1 | FileCheck %s
 ;
 ; CHECK: Starting llvm::Function pass manager run.
-; CHECK: Running pass: FunctionToLoopPassAdaptor
 ; CHECK: Running analysis: LoopAnalysis
 ; CHECK: Running analysis: InnerAnalysisManagerProxy<
 ; CHECK: Starting Loop pass manager run.
 ; CHECK: Running pass: RequireAnalysisPass<{{.*}}LoopAccessAnalysis
-; CHECK: Running analysis: LoopAccessAnalysis on inner1.header
+; CHECK: Running analysis: LoopAccessAnalysis on Loop at depth 2 containing: %inner1.header
 ; CHECK: Finished Loop pass manager run.
 ; CHECK: Starting Loop pass manager run.
 ; CHECK: Running pass: RequireAnalysisPass<{{.*}}LoopAccessAnalysis
-; CHECK: Running analysis: LoopAccessAnalysis on inner2.header
+; CHECK: Running analysis: LoopAccessAnalysis on Loop at depth 2 containing: %inner2.header
 ; CHECK: Finished Loop pass manager run.
 ; CHECK: Starting Loop pass manager run.
 ; CHECK: Running pass: RequireAnalysisPass<{{.*}}LoopAccessAnalysis
-; CHECK: Running analysis: LoopAccessAnalysis on outer.header
+; CHECK: Running analysis: LoopAccessAnalysis on Loop at depth 1 containing: %outer.header
 ; CHECK: Finished Loop pass manager run.
 ; CHECK: Running pass: LoopUnrollPass
 ; CHECK: Clearing all analysis results for: inner2.header
@@ -29,16 +28,15 @@
 ; CHECK: Invalidating analysis: LoopAccessAnalysis on inner1.header
 ; CHECK: Invalidating all non-preserved analyses for: inner1.header.1
 ; CHECK-NOT: Invalidating analysis: LoopAccessAnalysis on inner1.header.1
-; CHECK: Running pass: FunctionToLoopPassAdaptor
 ; CHECK: Starting Loop pass manager run.
 ; CHECK: Running pass: LoopAccessInfoPrinterPass
-; CHECK: Running analysis: LoopAccessAnalysis on inner1.header
+; CHECK: Running analysis: LoopAccessAnalysis on Loop at depth 1 containing: %inner1.header
 ; CHECK: Loop access info in function 'test':
 ; CHECK:   inner1.header:
 ; CHECK: Finished Loop pass manager run.
 ; CHECK: Starting Loop pass manager run.
 ; CHECK: Running pass: LoopAccessInfoPrinterPass
-; CHECK: Running analysis: LoopAccessAnalysis on inner1.header.1
+; CHECK: Running analysis: LoopAccessAnalysis on Loop at depth 1 containing: %inner1.header.1
 ; CHECK: Loop access info in function 'test':
 ; CHECK:   inner1.header.1:
 ; CHECK: Finished Loop pass manager run.
diff --git a/llvm/test/Transforms/SCCP/ipsccp-preserve-analysis.ll b/llvm/test/Transforms/SCCP/ipsccp-preserve-analysis.ll
index b840e5e09fdc2..78c02d9b8a42c 100644
--- a/llvm/test/Transforms/SCCP/ipsccp-preserve-analysis.ll
+++ b/llvm/test/Transforms/SCCP/ipsccp-preserve-analysis.ll
@@ -19,7 +19,6 @@
 ; NEW-PM-NEXT: Invalidating all non-preserved analyses for:
 ; NEW-PM-NEXT: Invalidating all non-preserved analyses for: f1
 ; NEW-PM-NEXT: Invalidating all non-preserved analyses for: f2
-; NEW-PM-NEXT: Running pass: ModuleToFunctionPassAdaptor
 ; NEW-PM-NOT: Running analysis:
 
 ; IR-LABEL: @f1

From 4a161bd8b3c2c31da788072b9641e1dcde62ce08 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Thu, 30 Jul 2020 17:37:57 +0100
Subject: [PATCH 0645/1035] LoopUnroll.cpp - pass std::vector by const
 reference to needToInsertPhisForLCSSA helper. NFCI.

Avoid an unnecessary pass by value.
---
 llvm/lib/Transforms/Utils/LoopUnroll.cpp | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Transforms/Utils/LoopUnroll.cpp b/llvm/lib/Transforms/Utils/LoopUnroll.cpp
index 3875c631f839b..8e8aeea15dbf9 100644
--- a/llvm/lib/Transforms/Utils/LoopUnroll.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUnroll.cpp
@@ -108,14 +108,15 @@ UnrollVerifyDomtree("unroll-verify-domtree", cl::Hidden,
 /// insert a phi-node, otherwise LCSSA will be broken.
 /// The function is just a helper function for llvm::UnrollLoop that returns
 /// true if this situation occurs, indicating that LCSSA needs to be fixed.
-static bool needToInsertPhisForLCSSA(Loop *L, std::vector<BasicBlock *> Blocks,
+static bool needToInsertPhisForLCSSA(Loop *L,
+                                     const std::vector<BasicBlock *> &Blocks,
                                      LoopInfo *LI) {
   for (BasicBlock *BB : Blocks) {
     if (LI->getLoopFor(BB) == L)
       continue;
     for (Instruction &I : *BB) {
       for (Use &U : I.operands()) {
-        if (auto Def = dyn_cast<Instruction>(U)) {
+        if (const auto *Def = dyn_cast<Instruction>(U)) {
           Loop *DefLoop = LI->getLoopFor(Def->getParent());
           if (!DefLoop)
             continue;

From be198e03ebba264baadfd2ed383b1725633c43ea Mon Sep 17 00:00:00 2001
From: Jordan Rupprecht <rupprecht@google.com>
Date: Thu, 30 Jul 2020 10:25:28 -0700
Subject: [PATCH 0646/1035] [lldb][test] Move registers-target-xml-reading
 target to the correct test location.

This test was added in D74217 (and the `.categories` file later added in ccf1c30cde6e1e763e7c9cdd48a609a805166699) around the same time I moved the test tree from `lldb/packages/Python/lldbsuite/test` to `lldb/test/API` (D71151). Since this got lost in the move, it isn't running. (I introduced an intentional syntax error, and `ninja check-lldb` passes).

I moved it to the correct location, and now it runs and passes -- locally, at least -- as `ninja check-lldb-api-tools-lldb-server-registers-target-xml-reading`.
---
 .../lldbsuite/test => test/API}/tools/lldb-server/.categories   | 0
 .../tools/lldb-server/registers-target-xml-reading/Makefile     | 0
 .../TestGdbRemoteTargetXmlPacket.py                             | 2 +-
 .../tools/lldb-server/registers-target-xml-reading/main.cpp     | 0
 4 files changed, 1 insertion(+), 1 deletion(-)
 rename lldb/{packages/Python/lldbsuite/test => test/API}/tools/lldb-server/.categories (100%)
 rename lldb/{packages/Python/lldbsuite/test => test/API}/tools/lldb-server/registers-target-xml-reading/Makefile (100%)
 rename lldb/{packages/Python/lldbsuite/test => test/API}/tools/lldb-server/registers-target-xml-reading/TestGdbRemoteTargetXmlPacket.py (99%)
 rename lldb/{packages/Python/lldbsuite/test => test/API}/tools/lldb-server/registers-target-xml-reading/main.cpp (100%)

diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/.categories b/lldb/test/API/tools/lldb-server/.categories
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-server/.categories
rename to lldb/test/API/tools/lldb-server/.categories
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/registers-target-xml-reading/Makefile b/lldb/test/API/tools/lldb-server/registers-target-xml-reading/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-server/registers-target-xml-reading/Makefile
rename to lldb/test/API/tools/lldb-server/registers-target-xml-reading/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/registers-target-xml-reading/TestGdbRemoteTargetXmlPacket.py b/lldb/test/API/tools/lldb-server/registers-target-xml-reading/TestGdbRemoteTargetXmlPacket.py
similarity index 99%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-server/registers-target-xml-reading/TestGdbRemoteTargetXmlPacket.py
rename to lldb/test/API/tools/lldb-server/registers-target-xml-reading/TestGdbRemoteTargetXmlPacket.py
index 530e2ce80023b..51f3d35956406 100644
--- a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/registers-target-xml-reading/TestGdbRemoteTargetXmlPacket.py
+++ b/lldb/test/API/tools/lldb-server/registers-target-xml-reading/TestGdbRemoteTargetXmlPacket.py
@@ -66,4 +66,4 @@ def test_g_target_xml_returns_correct_data(self):
             self.assertEqual(q_info_reg["format"], xml_info_reg.get("format"))
             self.assertEqual(q_info_reg["bitsize"], xml_info_reg.get("bitsize"))
             self.assertEqual(q_info_reg["offset"], xml_info_reg.get("offset"))
-            self.assertEqual(q_info_reg["encoding"], xml_info_reg.get("encoding"))
\ No newline at end of file
+            self.assertEqual(q_info_reg["encoding"], xml_info_reg.get("encoding"))
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/registers-target-xml-reading/main.cpp b/lldb/test/API/tools/lldb-server/registers-target-xml-reading/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-server/registers-target-xml-reading/main.cpp
rename to lldb/test/API/tools/lldb-server/registers-target-xml-reading/main.cpp

From 41909e96824b73162431e84cbba071b1ff4f6341 Mon Sep 17 00:00:00 2001
From: Jonas Devlieghere <jonas@devlieghere.com>
Date: Thu, 30 Jul 2020 10:34:32 -0700
Subject: [PATCH 0647/1035] [lldb] Add copy ctor/assignment operator to
 SBCommandInterpreterRunOptions

---
 .../lldb/API/SBCommandInterpreterRunOptions.h |  4 ++
 .../API/SBCommandInterpreterRunOptions.cpp    | 37 ++++++++++++++++---
 2 files changed, 35 insertions(+), 6 deletions(-)

diff --git a/lldb/include/lldb/API/SBCommandInterpreterRunOptions.h b/lldb/include/lldb/API/SBCommandInterpreterRunOptions.h
index 82d6feedc02e7..3c00513faaa71 100644
--- a/lldb/include/lldb/API/SBCommandInterpreterRunOptions.h
+++ b/lldb/include/lldb/API/SBCommandInterpreterRunOptions.h
@@ -26,8 +26,12 @@ class LLDB_API SBCommandInterpreterRunOptions {
 
 public:
   SBCommandInterpreterRunOptions();
+  SBCommandInterpreterRunOptions(const SBCommandInterpreterRunOptions &rhs);
   ~SBCommandInterpreterRunOptions();
 
+  SBCommandInterpreterRunOptions &
+  operator=(const SBCommandInterpreterRunOptions &rhs);
+
   bool GetStopOnContinue() const;
 
   void SetStopOnContinue(bool);
diff --git a/lldb/source/API/SBCommandInterpreterRunOptions.cpp b/lldb/source/API/SBCommandInterpreterRunOptions.cpp
index fcfbf5e5401ae..da800e8b78046 100644
--- a/lldb/source/API/SBCommandInterpreterRunOptions.cpp
+++ b/lldb/source/API/SBCommandInterpreterRunOptions.cpp
@@ -24,8 +24,29 @@ SBCommandInterpreterRunOptions::SBCommandInterpreterRunOptions() {
   m_opaque_up = std::make_unique<CommandInterpreterRunOptions>();
 }
 
+SBCommandInterpreterRunOptions::SBCommandInterpreterRunOptions(
+    const SBCommandInterpreterRunOptions &rhs)
+    : m_opaque_up() {
+  LLDB_RECORD_CONSTRUCTOR(SBCommandInterpreterRunOptions,
+                          (const lldb::SBCommandInterpreterRunOptions &), rhs);
+
+  m_opaque_up = std::make_unique<CommandInterpreterRunOptions>(rhs.ref());
+}
+
 SBCommandInterpreterRunOptions::~SBCommandInterpreterRunOptions() = default;
 
+SBCommandInterpreterRunOptions &SBCommandInterpreterRunOptions::operator=(
+    const SBCommandInterpreterRunOptions &rhs) {
+  LLDB_RECORD_METHOD(lldb::SBCommandInterpreterRunOptions &,
+                     SBCommandInterpreterRunOptions, operator=,
+                     (const lldb::SBCommandInterpreterRunOptions &), rhs);
+
+  if (this == &rhs)
+    return LLDB_RECORD_RESULT(*this);
+  *m_opaque_up = *rhs.m_opaque_up;
+  return LLDB_RECORD_RESULT(*this);
+}
+
 bool SBCommandInterpreterRunOptions::GetStopOnContinue() const {
   LLDB_RECORD_METHOD_CONST_NO_ARGS(bool, SBCommandInterpreterRunOptions,
                                    GetStopOnContinue);
@@ -190,12 +211,11 @@ SBCommandInterpreterRunResult::~SBCommandInterpreterRunResult() = default;
 SBCommandInterpreterRunResult &SBCommandInterpreterRunResult::operator=(
     const SBCommandInterpreterRunResult &rhs) {
   LLDB_RECORD_METHOD(lldb::SBCommandInterpreterRunResult &,
-                     SBCommandInterpreterRunResult,
-                     operator=,(const lldb::SBCommandInterpreterRunResult &),
-                     rhs);
+                     SBCommandInterpreterRunResult, operator=,
+                     (const lldb::SBCommandInterpreterRunResult &), rhs);
 
   if (this == &rhs)
-    return *this;
+    return LLDB_RECORD_RESULT(*this);
   *m_opaque_up = *rhs.m_opaque_up;
   return LLDB_RECORD_RESULT(*this);
 }
@@ -220,6 +240,11 @@ namespace repro {
 
 template <> void RegisterMethods<SBCommandInterpreterRunOptions>(Registry &R) {
   LLDB_REGISTER_CONSTRUCTOR(SBCommandInterpreterRunOptions, ());
+  LLDB_REGISTER_CONSTRUCTOR(SBCommandInterpreterRunOptions,
+                            (const lldb::SBCommandInterpreterRunOptions &));
+  LLDB_REGISTER_METHOD(lldb::SBCommandInterpreterRunOptions &,
+                       SBCommandInterpreterRunOptions, operator=,
+                       (const lldb::SBCommandInterpreterRunOptions &));
   LLDB_REGISTER_METHOD_CONST(bool, SBCommandInterpreterRunOptions,
                              GetStopOnContinue, ());
   LLDB_REGISTER_METHOD(void, SBCommandInterpreterRunOptions, SetStopOnContinue,
@@ -260,8 +285,8 @@ template <> void RegisterMethods<SBCommandInterpreterRunOptions>(Registry &R) {
   LLDB_REGISTER_CONSTRUCTOR(SBCommandInterpreterRunResult,
                             (const lldb::SBCommandInterpreterRunResult &));
   LLDB_REGISTER_METHOD(lldb::SBCommandInterpreterRunResult &,
-                       SBCommandInterpreterRunResult,
-                       operator=,(const lldb::SBCommandInterpreterRunResult &));
+                       SBCommandInterpreterRunResult, operator=,
+                       (const lldb::SBCommandInterpreterRunResult &));
   LLDB_REGISTER_METHOD_CONST(int, SBCommandInterpreterRunResult,
                              GetNumberOfErrors, ());
   LLDB_REGISTER_METHOD_CONST(lldb::CommandInterpreterResult,

From 5b32518f96ca79d3b3c1d60940c3a6764e500c1e Mon Sep 17 00:00:00 2001
From: Stanislav Mekhanoshin <Stanislav.Mekhanoshin@amd.com>
Date: Wed, 29 Jul 2020 17:17:45 -0700
Subject: [PATCH 0648/1035] [AMDGPU] Do not use undef on indirect source

We are using undef on the indirect move source subreg and then
using implicit super-reg. This creates a problem in RA when
Greedy decides to split the register. It reassigns the implicit
super-reg but does not bother to change undef source because
it is really does not matter. The fix is to stop lying to RA and
drop undef flag.

This has also hit a problem in SIFoldOperands as it can fold
immediate into an indirect move since there is no undef flag
anymore. That results in multiple test failures, so added the
check for this case.

Differential Revision: https://reviews.llvm.org/D84899
---
 .../AMDGPU/AMDGPUInstructionSelector.cpp      |  4 +-
 llvm/lib/Target/AMDGPU/SIFoldOperands.cpp     | 13 +++++-
 llvm/lib/Target/AMDGPU/SIISelLowering.cpp     |  8 ++--
 .../inst-select-extract-vector-elt.mir        | 44 +++++++++----------
 .../AMDGPU/indirect-addressing-term.ll        |  2 +-
 5 files changed, 41 insertions(+), 30 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
index 23899a7e4dd71..709329b4c0c6d 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
@@ -2542,7 +2542,7 @@ bool AMDGPUInstructionSelector::selectG_EXTRACT_VECTOR_ELT(
     BuildMI(*BB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0)
       .addReg(IdxReg);
     BuildMI(*BB, &MI, DL, TII.get(AMDGPU::V_MOVRELS_B32_e32), DstReg)
-      .addReg(SrcReg, RegState::Undef, SubReg)
+      .addReg(SrcReg, 0, SubReg)
       .addReg(SrcReg, RegState::Implicit);
     MI.eraseFromParent();
     return true;
@@ -2552,7 +2552,7 @@ bool AMDGPUInstructionSelector::selectG_EXTRACT_VECTOR_ELT(
     .addReg(IdxReg)
     .addImm(AMDGPU::VGPRIndexMode::SRC0_ENABLE);
   BuildMI(*BB, MI, DL, TII.get(AMDGPU::V_MOV_B32_e32), DstReg)
-    .addReg(SrcReg, RegState::Undef, SubReg)
+    .addReg(SrcReg, 0, SubReg)
     .addReg(SrcReg, RegState::Implicit)
     .addReg(AMDGPU::M0, RegState::Implicit);
   BuildMI(*BB, MI, DL, TII.get(AMDGPU::S_SET_GPR_IDX_OFF));
diff --git a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
index 88d7304550f60..67b91e14fcca8 100644
--- a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
@@ -463,7 +463,18 @@ static bool tryAddToFoldList(SmallVectorImpl<FoldCandidate> &FoldList,
 static bool isUseSafeToFold(const SIInstrInfo *TII,
                             const MachineInstr &MI,
                             const MachineOperand &UseMO) {
-  return !UseMO.isUndef() && !TII->isSDWA(MI);
+  if (UseMO.isUndef() || TII->isSDWA(MI))
+    return false;
+
+  switch (MI.getOpcode()) {
+  case AMDGPU::V_MOV_B32_e32:
+  case AMDGPU::V_MOV_B32_e64:
+  case AMDGPU::V_MOV_B64_PSEUDO:
+    // Do not fold into an indirect mov.
+    return !MI.hasRegisterImplicitUseOperand(AMDGPU::M0);
+  }
+
+  return true;
   //return !MI.hasRegisterImplicitUseOperand(UseMO.getReg());
 }
 
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 367e4fc974bb5..4ea44373b976a 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -3658,13 +3658,13 @@ static MachineBasicBlock *emitIndirectSrc(MachineInstr &MI,
       // to avoid interfering with other uses, so probably requires a new
       // optimization pass.
       BuildMI(MBB, I, DL, TII->get(AMDGPU::V_MOV_B32_e32), Dst)
-        .addReg(SrcReg, RegState::Undef, SubReg)
+        .addReg(SrcReg, 0, SubReg)
         .addReg(SrcReg, RegState::Implicit)
         .addReg(AMDGPU::M0, RegState::Implicit);
       BuildMI(MBB, I, DL, TII->get(AMDGPU::S_SET_GPR_IDX_OFF));
     } else {
       BuildMI(MBB, I, DL, TII->get(AMDGPU::V_MOVRELS_B32_e32), Dst)
-        .addReg(SrcReg, RegState::Undef, SubReg)
+        .addReg(SrcReg, 0, SubReg)
         .addReg(SrcReg, RegState::Implicit);
     }
 
@@ -3687,13 +3687,13 @@ static MachineBasicBlock *emitIndirectSrc(MachineInstr &MI,
 
   if (UseGPRIdxMode) {
     BuildMI(*LoopBB, InsPt, DL, TII->get(AMDGPU::V_MOV_B32_e32), Dst)
-      .addReg(SrcReg, RegState::Undef, SubReg)
+      .addReg(SrcReg, 0, SubReg)
       .addReg(SrcReg, RegState::Implicit)
       .addReg(AMDGPU::M0, RegState::Implicit);
     BuildMI(*LoopBB, InsPt, DL, TII->get(AMDGPU::S_SET_GPR_IDX_OFF));
   } else {
     BuildMI(*LoopBB, InsPt, DL, TII->get(AMDGPU::V_MOVRELS_B32_e32), Dst)
-      .addReg(SrcReg, RegState::Undef, SubReg)
+      .addReg(SrcReg, 0, SubReg)
       .addReg(SrcReg, RegState::Implicit);
   }
 
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-extract-vector-elt.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-extract-vector-elt.mir
index 10e4cbdc14676..9edad11a23418 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-extract-vector-elt.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-extract-vector-elt.mir
@@ -502,13 +502,13 @@ body: |
     ; MOVREL: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; MOVREL: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
     ; MOVREL: $m0 = COPY [[COPY1]]
-    ; MOVREL: [[V_MOVRELS_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOVRELS_B32_e32 undef [[COPY]].sub0, implicit $m0, implicit $exec, implicit [[COPY]]
+    ; MOVREL: [[V_MOVRELS_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOVRELS_B32_e32 [[COPY]].sub0, implicit $m0, implicit $exec, implicit [[COPY]]
     ; MOVREL: S_ENDPGM 0, implicit [[V_MOVRELS_B32_e32_]]
     ; GPRIDX-LABEL: name: extract_vector_elt_v_s32_v2s32
     ; GPRIDX: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GPRIDX: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
     ; GPRIDX: S_SET_GPR_IDX_ON [[COPY1]], 1, implicit-def $m0, implicit-def $mode, implicit $m0, implicit $mode
-    ; GPRIDX: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 undef [[COPY]].sub0, implicit $exec, implicit [[COPY]], implicit $m0
+    ; GPRIDX: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 [[COPY]].sub0, implicit $exec, implicit [[COPY]], implicit $m0
     ; GPRIDX: S_SET_GPR_IDX_OFF implicit-def $mode, implicit $mode
     ; GPRIDX: S_ENDPGM 0, implicit [[V_MOV_B32_e32_]]
     %0:vgpr(<2 x s32>) = COPY $vgpr0_vgpr1
@@ -530,13 +530,13 @@ body: |
     ; MOVREL: [[COPY:%[0-9]+]]:vreg_96 = COPY $vgpr0_vgpr1_vgpr2
     ; MOVREL: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
     ; MOVREL: $m0 = COPY [[COPY1]]
-    ; MOVREL: [[V_MOVRELS_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOVRELS_B32_e32 undef [[COPY]].sub0, implicit $m0, implicit $exec, implicit [[COPY]]
+    ; MOVREL: [[V_MOVRELS_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOVRELS_B32_e32 [[COPY]].sub0, implicit $m0, implicit $exec, implicit [[COPY]]
     ; MOVREL: S_ENDPGM 0, implicit [[V_MOVRELS_B32_e32_]]
     ; GPRIDX-LABEL: name: extract_vector_elt_v_s32_v3s32
     ; GPRIDX: [[COPY:%[0-9]+]]:vreg_96 = COPY $vgpr0_vgpr1_vgpr2
     ; GPRIDX: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
     ; GPRIDX: S_SET_GPR_IDX_ON [[COPY1]], 1, implicit-def $m0, implicit-def $mode, implicit $m0, implicit $mode
-    ; GPRIDX: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 undef [[COPY]].sub0, implicit $exec, implicit [[COPY]], implicit $m0
+    ; GPRIDX: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 [[COPY]].sub0, implicit $exec, implicit [[COPY]], implicit $m0
     ; GPRIDX: S_SET_GPR_IDX_OFF implicit-def $mode, implicit $mode
     ; GPRIDX: S_ENDPGM 0, implicit [[V_MOV_B32_e32_]]
     %0:vgpr(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
@@ -558,13 +558,13 @@ body: |
     ; MOVREL: [[COPY:%[0-9]+]]:vreg_128 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     ; MOVREL: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr4
     ; MOVREL: $m0 = COPY [[COPY1]]
-    ; MOVREL: [[V_MOVRELS_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOVRELS_B32_e32 undef [[COPY]].sub0, implicit $m0, implicit $exec, implicit [[COPY]]
+    ; MOVREL: [[V_MOVRELS_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOVRELS_B32_e32 [[COPY]].sub0, implicit $m0, implicit $exec, implicit [[COPY]]
     ; MOVREL: S_ENDPGM 0, implicit [[V_MOVRELS_B32_e32_]]
     ; GPRIDX-LABEL: name: extract_vector_elt_v_s32_v4s32
     ; GPRIDX: [[COPY:%[0-9]+]]:vreg_128 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     ; GPRIDX: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr4
     ; GPRIDX: S_SET_GPR_IDX_ON [[COPY1]], 1, implicit-def $m0, implicit-def $mode, implicit $m0, implicit $mode
-    ; GPRIDX: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 undef [[COPY]].sub0, implicit $exec, implicit [[COPY]], implicit $m0
+    ; GPRIDX: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 [[COPY]].sub0, implicit $exec, implicit [[COPY]], implicit $m0
     ; GPRIDX: S_SET_GPR_IDX_OFF implicit-def $mode, implicit $mode
     ; GPRIDX: S_ENDPGM 0, implicit [[V_MOV_B32_e32_]]
     %0:vgpr(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
@@ -586,13 +586,13 @@ body: |
     ; MOVREL: [[COPY:%[0-9]+]]:vreg_256 = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
     ; MOVREL: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr8
     ; MOVREL: $m0 = COPY [[COPY1]]
-    ; MOVREL: [[V_MOVRELS_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOVRELS_B32_e32 undef [[COPY]].sub0, implicit $m0, implicit $exec, implicit [[COPY]]
+    ; MOVREL: [[V_MOVRELS_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOVRELS_B32_e32 [[COPY]].sub0, implicit $m0, implicit $exec, implicit [[COPY]]
     ; MOVREL: S_ENDPGM 0, implicit [[V_MOVRELS_B32_e32_]]
     ; GPRIDX-LABEL: name: extract_vector_elt_v_s32_v8s32
     ; GPRIDX: [[COPY:%[0-9]+]]:vreg_256 = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
     ; GPRIDX: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr8
     ; GPRIDX: S_SET_GPR_IDX_ON [[COPY1]], 1, implicit-def $m0, implicit-def $mode, implicit $m0, implicit $mode
-    ; GPRIDX: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 undef [[COPY]].sub0, implicit $exec, implicit [[COPY]], implicit $m0
+    ; GPRIDX: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 [[COPY]].sub0, implicit $exec, implicit [[COPY]], implicit $m0
     ; GPRIDX: S_SET_GPR_IDX_OFF implicit-def $mode, implicit $mode
     ; GPRIDX: S_ENDPGM 0, implicit [[V_MOV_B32_e32_]]
     %0:vgpr(<8 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
@@ -614,13 +614,13 @@ body: |
     ; MOVREL: [[COPY:%[0-9]+]]:vreg_512 = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
     ; MOVREL: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr8
     ; MOVREL: $m0 = COPY [[COPY1]]
-    ; MOVREL: [[V_MOVRELS_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOVRELS_B32_e32 undef [[COPY]].sub0, implicit $m0, implicit $exec, implicit [[COPY]]
+    ; MOVREL: [[V_MOVRELS_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOVRELS_B32_e32 [[COPY]].sub0, implicit $m0, implicit $exec, implicit [[COPY]]
     ; MOVREL: S_ENDPGM 0, implicit [[V_MOVRELS_B32_e32_]]
     ; GPRIDX-LABEL: name: extract_vector_elt_v_s32_v16s32
     ; GPRIDX: [[COPY:%[0-9]+]]:vreg_512 = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
     ; GPRIDX: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr8
     ; GPRIDX: S_SET_GPR_IDX_ON [[COPY1]], 1, implicit-def $m0, implicit-def $mode, implicit $m0, implicit $mode
-    ; GPRIDX: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 undef [[COPY]].sub0, implicit $exec, implicit [[COPY]], implicit $m0
+    ; GPRIDX: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 [[COPY]].sub0, implicit $exec, implicit [[COPY]], implicit $m0
     ; GPRIDX: S_SET_GPR_IDX_OFF implicit-def $mode, implicit $mode
     ; GPRIDX: S_ENDPGM 0, implicit [[V_MOV_B32_e32_]]
     %0:vgpr(<16 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
@@ -642,13 +642,13 @@ body: |
     ; MOVREL: [[COPY:%[0-9]+]]:vreg_1024 = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
     ; MOVREL: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr40
     ; MOVREL: $m0 = COPY [[COPY1]]
-    ; MOVREL: [[V_MOVRELS_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOVRELS_B32_e32 undef [[COPY]].sub0, implicit $m0, implicit $exec, implicit [[COPY]]
+    ; MOVREL: [[V_MOVRELS_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOVRELS_B32_e32 [[COPY]].sub0, implicit $m0, implicit $exec, implicit [[COPY]]
     ; MOVREL: S_ENDPGM 0, implicit [[V_MOVRELS_B32_e32_]]
     ; GPRIDX-LABEL: name: extract_vector_elt_v_s32_v32s32
     ; GPRIDX: [[COPY:%[0-9]+]]:vreg_1024 = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
     ; GPRIDX: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr40
     ; GPRIDX: S_SET_GPR_IDX_ON [[COPY1]], 1, implicit-def $m0, implicit-def $mode, implicit $m0, implicit $mode
-    ; GPRIDX: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 undef [[COPY]].sub0, implicit $exec, implicit [[COPY]], implicit $m0
+    ; GPRIDX: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 [[COPY]].sub0, implicit $exec, implicit [[COPY]], implicit $m0
     ; GPRIDX: S_SET_GPR_IDX_OFF implicit-def $mode, implicit $mode
     ; GPRIDX: S_ENDPGM 0, implicit [[V_MOV_B32_e32_]]
     %0:vgpr(<32 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
@@ -670,13 +670,13 @@ body: |
     ; MOVREL: [[COPY:%[0-9]+]]:vreg_256 = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
     ; MOVREL: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr8
     ; MOVREL: $m0 = COPY [[COPY1]]
-    ; MOVREL: [[V_MOVRELS_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOVRELS_B32_e32 undef [[COPY]].sub1, implicit $m0, implicit $exec, implicit [[COPY]]
+    ; MOVREL: [[V_MOVRELS_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOVRELS_B32_e32 [[COPY]].sub1, implicit $m0, implicit $exec, implicit [[COPY]]
     ; MOVREL: S_ENDPGM 0, implicit [[V_MOVRELS_B32_e32_]]
     ; GPRIDX-LABEL: name: extract_vector_elt_v_s32_v8s32_idx_offset_1
     ; GPRIDX: [[COPY:%[0-9]+]]:vreg_256 = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
     ; GPRIDX: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr8
     ; GPRIDX: S_SET_GPR_IDX_ON [[COPY1]], 1, implicit-def $m0, implicit-def $mode, implicit $m0, implicit $mode
-    ; GPRIDX: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 undef [[COPY]].sub1, implicit $exec, implicit [[COPY]], implicit $m0
+    ; GPRIDX: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 [[COPY]].sub1, implicit $exec, implicit [[COPY]], implicit $m0
     ; GPRIDX: S_SET_GPR_IDX_OFF implicit-def $mode, implicit $mode
     ; GPRIDX: S_ENDPGM 0, implicit [[V_MOV_B32_e32_]]
     %0:vgpr(<8 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
@@ -702,7 +702,7 @@ body: |
     ; MOVREL: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
     ; MOVREL: [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[COPY1]], [[S_MOV_B32_]], implicit-def $scc
     ; MOVREL: $m0 = COPY [[S_ADD_I32_]]
-    ; MOVREL: [[V_MOVRELS_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOVRELS_B32_e32 undef [[COPY]].sub0, implicit $m0, implicit $exec, implicit [[COPY]]
+    ; MOVREL: [[V_MOVRELS_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOVRELS_B32_e32 [[COPY]].sub0, implicit $m0, implicit $exec, implicit [[COPY]]
     ; MOVREL: S_ENDPGM 0, implicit [[V_MOVRELS_B32_e32_]]
     ; GPRIDX-LABEL: name: extract_vector_elt_v_s32_v8s32_idx_offset_m1
     ; GPRIDX: [[COPY:%[0-9]+]]:vreg_256 = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
@@ -710,7 +710,7 @@ body: |
     ; GPRIDX: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
     ; GPRIDX: [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[COPY1]], [[S_MOV_B32_]], implicit-def $scc
     ; GPRIDX: S_SET_GPR_IDX_ON [[S_ADD_I32_]], 1, implicit-def $m0, implicit-def $mode, implicit $m0, implicit $mode
-    ; GPRIDX: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 undef [[COPY]].sub0, implicit $exec, implicit [[COPY]], implicit $m0
+    ; GPRIDX: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 [[COPY]].sub0, implicit $exec, implicit [[COPY]], implicit $m0
     ; GPRIDX: S_SET_GPR_IDX_OFF implicit-def $mode, implicit $mode
     ; GPRIDX: S_ENDPGM 0, implicit [[V_MOV_B32_e32_]]
     %0:vgpr(<8 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
@@ -734,13 +734,13 @@ body: |
     ; MOVREL: [[COPY:%[0-9]+]]:vreg_256 = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
     ; MOVREL: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr8
     ; MOVREL: $m0 = COPY [[COPY1]]
-    ; MOVREL: [[V_MOVRELS_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOVRELS_B32_e32 undef [[COPY]].sub7, implicit $m0, implicit $exec, implicit [[COPY]]
+    ; MOVREL: [[V_MOVRELS_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOVRELS_B32_e32 [[COPY]].sub7, implicit $m0, implicit $exec, implicit [[COPY]]
     ; MOVREL: S_ENDPGM 0, implicit [[V_MOVRELS_B32_e32_]]
     ; GPRIDX-LABEL: name: extract_vector_elt_v_s32_v8s32_idx_offset_7
     ; GPRIDX: [[COPY:%[0-9]+]]:vreg_256 = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
     ; GPRIDX: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr8
     ; GPRIDX: S_SET_GPR_IDX_ON [[COPY1]], 1, implicit-def $m0, implicit-def $mode, implicit $m0, implicit $mode
-    ; GPRIDX: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 undef [[COPY]].sub7, implicit $exec, implicit [[COPY]], implicit $m0
+    ; GPRIDX: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 [[COPY]].sub7, implicit $exec, implicit [[COPY]], implicit $m0
     ; GPRIDX: S_SET_GPR_IDX_OFF implicit-def $mode, implicit $mode
     ; GPRIDX: S_ENDPGM 0, implicit [[V_MOV_B32_e32_]]
     %0:vgpr(<8 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
@@ -766,7 +766,7 @@ body: |
     ; MOVREL: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 8
     ; MOVREL: [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[COPY1]], [[S_MOV_B32_]], implicit-def $scc
     ; MOVREL: $m0 = COPY [[S_ADD_I32_]]
-    ; MOVREL: [[V_MOVRELS_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOVRELS_B32_e32 undef [[COPY]].sub0, implicit $m0, implicit $exec, implicit [[COPY]]
+    ; MOVREL: [[V_MOVRELS_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOVRELS_B32_e32 [[COPY]].sub0, implicit $m0, implicit $exec, implicit [[COPY]]
     ; MOVREL: S_ENDPGM 0, implicit [[V_MOVRELS_B32_e32_]]
     ; GPRIDX-LABEL: name: extract_vector_elt_v_s32_v8s32_idx_offset_8
     ; GPRIDX: [[COPY:%[0-9]+]]:vreg_256 = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
@@ -774,7 +774,7 @@ body: |
     ; GPRIDX: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 8
     ; GPRIDX: [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[COPY1]], [[S_MOV_B32_]], implicit-def $scc
     ; GPRIDX: S_SET_GPR_IDX_ON [[S_ADD_I32_]], 1, implicit-def $m0, implicit-def $mode, implicit $m0, implicit $mode
-    ; GPRIDX: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 undef [[COPY]].sub0, implicit $exec, implicit [[COPY]], implicit $m0
+    ; GPRIDX: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 [[COPY]].sub0, implicit $exec, implicit [[COPY]], implicit $m0
     ; GPRIDX: S_SET_GPR_IDX_OFF implicit-def $mode, implicit $mode
     ; GPRIDX: S_ENDPGM 0, implicit [[V_MOV_B32_e32_]]
     %0:vgpr(<8 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
@@ -825,13 +825,13 @@ body: |
     ; MOVREL: [[COPY:%[0-9]+]]:vreg_128 = COPY $sgpr0_sgpr1_sgpr2_sgpr3
     ; MOVREL: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
     ; MOVREL: $m0 = COPY [[S_MOV_B32_]]
-    ; MOVREL: [[V_MOVRELS_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOVRELS_B32_e32 undef [[COPY]].sub0, implicit $m0, implicit $exec, implicit [[COPY]]
+    ; MOVREL: [[V_MOVRELS_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOVRELS_B32_e32 [[COPY]].sub0, implicit $m0, implicit $exec, implicit [[COPY]]
     ; MOVREL: S_ENDPGM 0, implicit [[V_MOVRELS_B32_e32_]]
     ; GPRIDX-LABEL: name: extract_vector_elt_v_s32_v4s32_const_idx
     ; GPRIDX: [[COPY:%[0-9]+]]:vreg_128 = COPY $sgpr0_sgpr1_sgpr2_sgpr3
     ; GPRIDX: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
     ; GPRIDX: S_SET_GPR_IDX_ON [[S_MOV_B32_]], 1, implicit-def $m0, implicit-def $mode, implicit $m0, implicit $mode
-    ; GPRIDX: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 undef [[COPY]].sub0, implicit $exec, implicit [[COPY]], implicit $m0
+    ; GPRIDX: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 [[COPY]].sub0, implicit $exec, implicit [[COPY]], implicit $m0
     ; GPRIDX: S_SET_GPR_IDX_OFF implicit-def $mode, implicit $mode
     ; GPRIDX: S_ENDPGM 0, implicit [[V_MOV_B32_e32_]]
     %0:vgpr(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
diff --git a/llvm/test/CodeGen/AMDGPU/indirect-addressing-term.ll b/llvm/test/CodeGen/AMDGPU/indirect-addressing-term.ll
index e26b1c9471049..3d3b511ab34b7 100644
--- a/llvm/test/CodeGen/AMDGPU/indirect-addressing-term.ll
+++ b/llvm/test/CodeGen/AMDGPU/indirect-addressing-term.ll
@@ -88,7 +88,7 @@ define amdgpu_kernel void @extract_w_offset_vgpr(i32 addrspace(1)* %out) {
   ; GCN:   renamable $sgpr4_sgpr5 = S_AND_SAVEEXEC_B64 killed renamable $sgpr4_sgpr5, implicit-def $exec, implicit-def $scc, implicit $exec
   ; GCN:   S_SET_GPR_IDX_ON killed renamable $sgpr2, 1, implicit-def $m0, implicit-def undef $mode, implicit $m0, implicit $mode
   ; GCN:   $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17 = SI_SPILL_V512_RESTORE %stack.2, $sgpr96_sgpr97_sgpr98_sgpr99, $sgpr32, 0, implicit $exec :: (load 64 from %stack.2, align 4, addrspace 5)
-  ; GCN:   renamable $vgpr18 = V_MOV_B32_e32 undef $vgpr3, implicit $exec, implicit killed $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17, implicit $m0
+  ; GCN:   renamable $vgpr18 = V_MOV_B32_e32 $vgpr3, implicit $exec, implicit killed $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17, implicit $m0
   ; GCN:   S_SET_GPR_IDX_OFF implicit-def $mode, implicit $mode
   ; GCN:   renamable $vgpr19 = COPY renamable $vgpr18
   ; GCN:   renamable $sgpr2_sgpr3 = COPY renamable $sgpr4_sgpr5

From ce1eb7af9dc1fb1c4167fa21824bda7dcf92bbce Mon Sep 17 00:00:00 2001
From: Wouter van Oortmerssen <aardappel@gmail.com>
Date: Mon, 27 Jul 2020 14:59:31 -0700
Subject: [PATCH 0649/1035] [WebAssembly] Fixed 64-bit indices in br_table

LLVM selection dag assumes "switch" indices are pointer sized, which causes problems for our 32-bit br_table. The new function ensures 32-bit operands don't get unnecessarily extended, and 64-bit operands get truncated.

Note that the changes to the existing test test exactly that: the addition of -NEXT in 2 places ensures no extension is inserted (which the test previously ignored) and that the wrap is present (previously omitted in wasm64 mode).

Differential Revision: https://reviews.llvm.org/D84705
---
 .../WebAssemblyFixBrTableDefaults.cpp         |  45 +++++-
 llvm/test/CodeGen/WebAssembly/switch.ll       | 138 +++++++++---------
 2 files changed, 110 insertions(+), 73 deletions(-)

diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyFixBrTableDefaults.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyFixBrTableDefaults.cpp
index 7f805b34b4995..52aa3534c78ef 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyFixBrTableDefaults.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyFixBrTableDefaults.cpp
@@ -41,13 +41,51 @@ class WebAssemblyFixBrTableDefaults final : public MachineFunctionPass {
 
 char WebAssemblyFixBrTableDefaults::ID = 0;
 
+// Target indepedent selection dag assumes that it is ok to use PointerTy
+// as the index for a "switch", whereas Wasm so far only has a 32-bit br_table.
+// See e.g. SelectionDAGBuilder::visitJumpTableHeader
+// We have a 64-bit br_table in the tablegen defs as a result, which does get
+// selected, and thus we get incorrect truncates/extensions happening on
+// wasm64. Here we fix that.
+void fixBrTableIndex(MachineInstr &MI, MachineBasicBlock *MBB,
+                     MachineFunction &MF) {
+  // Only happens on wasm64.
+  auto &WST = MF.getSubtarget<WebAssemblySubtarget>();
+  if (!WST.hasAddr64())
+    return;
+
+  assert(MI.getDesc().getOpcode() == WebAssembly::BR_TABLE_I64 &&
+         "64-bit br_table pseudo instruction expected");
+
+  // Find extension op, if any. It sits in the previous BB before the branch.
+  auto ExtMI = MF.getRegInfo().getVRegDef(MI.getOperand(0).getReg());
+  if (ExtMI->getOpcode() == WebAssembly::I64_EXTEND_U_I32) {
+    // Unnecessarily extending a 32-bit value to 64, remove it.
+    assert(MI.getOperand(0).getReg() == ExtMI->getOperand(0).getReg());
+    MI.getOperand(0).setReg(ExtMI->getOperand(1).getReg());
+    ExtMI->eraseFromParent();
+  } else {
+    // Incoming 64-bit value that needs to be truncated.
+    Register Reg32 =
+        MF.getRegInfo().createVirtualRegister(&WebAssembly::I32RegClass);
+    BuildMI(*MBB, MI.getIterator(), MI.getDebugLoc(),
+            WST.getInstrInfo()->get(WebAssembly::I32_WRAP_I64), Reg32)
+        .addReg(MI.getOperand(0).getReg());
+    MI.getOperand(0).setReg(Reg32);
+  }
+
+  // We now have a 32-bit operand in all cases, so change the instruction
+  // accordingly.
+  MI.setDesc(WST.getInstrInfo()->get(WebAssembly::BR_TABLE_I32));
+}
+
 // `MI` is a br_table instruction with a dummy default target argument. This
 // function finds and adds the default target argument and removes any redundant
 // range check preceding the br_table. Returns the MBB that the br_table is
 // moved into so it can be removed from further consideration, or nullptr if the
 // br_table cannot be optimized.
-MachineBasicBlock *fixBrTable(MachineInstr &MI, MachineBasicBlock *MBB,
-                              MachineFunction &MF) {
+MachineBasicBlock *fixBrTableDefault(MachineInstr &MI, MachineBasicBlock *MBB,
+                                     MachineFunction &MF) {
   // Get the header block, which contains the redundant range check.
   assert(MBB->pred_size() == 1 && "Expected a single guard predecessor");
   auto *HeaderMBB = *MBB->pred_begin();
@@ -125,7 +163,8 @@ bool WebAssemblyFixBrTableDefaults::runOnMachineFunction(MachineFunction &MF) {
     MBBSet.erase(MBB);
     for (auto &MI : *MBB) {
       if (WebAssembly::isBrTable(MI)) {
-        auto *Fixed = fixBrTable(MI, MBB, MF);
+        fixBrTableIndex(MI, MBB, MF);
+        auto *Fixed = fixBrTableDefault(MI, MBB, MF);
         if (Fixed != nullptr) {
           MBBSet.erase(Fixed);
           Changed = true;
diff --git a/llvm/test/CodeGen/WebAssembly/switch.ll b/llvm/test/CodeGen/WebAssembly/switch.ll
index fbb59ddd10610..717ddcda6f321 100644
--- a/llvm/test/CodeGen/WebAssembly/switch.ll
+++ b/llvm/test/CodeGen/WebAssembly/switch.ll
@@ -1,11 +1,9 @@
-; RUN: llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -disable-block-placement -verify-machineinstrs | FileCheck %s
+; RUN: llc < %s --mtriple=wasm32-unknown-unknown -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -disable-block-placement -verify-machineinstrs | FileCheck %s
+; RUN: llc < %s --mtriple=wasm64-unknown-unknown -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -disable-block-placement -verify-machineinstrs | FileCheck %s
 
 ; Test switch instructions. Block placement is disabled because it reorders
 ; the blocks in a way that isn't interesting here.
 
-target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
-target triple = "wasm32-unknown-unknown"
-
 declare void @foo0()
 declare void @foo1()
 declare void @foo2()
@@ -14,28 +12,28 @@ declare void @foo4()
 declare void @foo5()
 
 ; CHECK-LABEL: bar32:
-; CHECK: block   {{$}}
-; CHECK: block   {{$}}
-; CHECK: block   {{$}}
-; CHECK: block   {{$}}
-; CHECK: block   {{$}}
-; CHECK: block   {{$}}
-; CHECK: block   {{$}}
-; CHECK: br_table {{[^,]+}}, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 4, 5, 6{{$}}
-; CHECK: .LBB{{[0-9]+}}_1:
-; CHECK:   call foo0{{$}}
-; CHECK: .LBB{{[0-9]+}}_2:
-; CHECK:   call foo1{{$}}
-; CHECK: .LBB{{[0-9]+}}_3:
-; CHECK:   call foo2{{$}}
-; CHECK: .LBB{{[0-9]+}}_4:
-; CHECK:   call foo3{{$}}
-; CHECK: .LBB{{[0-9]+}}_5:
-; CHECK:   call foo4{{$}}
-; CHECK: .LBB{{[0-9]+}}_6:
-; CHECK:   call foo5{{$}}
-; CHECK: .LBB{{[0-9]+}}_7:
-; CHECK:   return{{$}}
+; CHECK:      block   {{$}}
+; CHECK:      block   {{$}}
+; CHECK:      block   {{$}}
+; CHECK:      block   {{$}}
+; CHECK:      block   {{$}}
+; CHECK:      block   {{$}}
+; CHECK:      block   {{$}}
+; CHECK-NEXT: br_table {{[^,]+}}, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 4, 5, 6{{$}}
+; CHECK:      .LBB{{[0-9]+}}_1:
+; CHECK:        call foo0{{$}}
+; CHECK:      .LBB{{[0-9]+}}_2:
+; CHECK:        call foo1{{$}}
+; CHECK:      .LBB{{[0-9]+}}_3:
+; CHECK:        call foo2{{$}}
+; CHECK:      .LBB{{[0-9]+}}_4:
+; CHECK:        call foo3{{$}}
+; CHECK:      .LBB{{[0-9]+}}_5:
+; CHECK:        call foo4{{$}}
+; CHECK:      .LBB{{[0-9]+}}_6:
+; CHECK:        call foo5{{$}}
+; CHECK:      .LBB{{[0-9]+}}_7:
+; CHECK:        return{{$}}
 define void @bar32(i32 %n) {
 entry:
   switch i32 %n, label %sw.epilog [
@@ -94,32 +92,32 @@ sw.epilog:                                        ; preds = %entry, %sw.bb.5, %s
 }
 
 ; CHECK-LABEL: bar64:
-; CHECK: block   {{$}}
-; CHECK: i64.const
-; CHECK: i64.gt_u
-; CHECK: br_if 0
-; CHECK: block   {{$}}
-; CHECK: block   {{$}}
-; CHECK: block   {{$}}
-; CHECK: block   {{$}}
-; CHECK: block   {{$}}
-; CHECK: block   {{$}}
-; CHECK: i32.wrap_i64
-; CHECK: br_table {{[^,]+}}, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 4, 5, 0{{$}}
-; CHECK: .LBB{{[0-9]+}}_2:
-; CHECK:   call foo0{{$}}
-; CHECK: .LBB{{[0-9]+}}_3:
-; CHECK:   call foo1{{$}}
-; CHECK: .LBB{{[0-9]+}}_4:
-; CHECK:   call foo2{{$}}
-; CHECK: .LBB{{[0-9]+}}_5:
-; CHECK:   call foo3{{$}}
-; CHECK: .LBB{{[0-9]+}}_6:
-; CHECK:   call foo4{{$}}
-; CHECK: .LBB{{[0-9]+}}_7:
-; CHECK:   call foo5{{$}}
-; CHECK: .LBB{{[0-9]+}}_8:
-; CHECK:   return{{$}}
+; CHECK:      block   {{$}}
+; CHECK:      i64.const
+; CHECK:      i64.gt_u
+; CHECK:      br_if 0
+; CHECK:      block   {{$}}
+; CHECK:      block   {{$}}
+; CHECK:      block   {{$}}
+; CHECK:      block   {{$}}
+; CHECK:      block   {{$}}
+; CHECK:      block   {{$}}
+; CHECK:      i32.wrap_i64
+; CHECK-NEXT: br_table {{[^,]+}}, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 4, 5, 0{{$}}
+; CHECK:      .LBB{{[0-9]+}}_2:
+; CHECK:        call foo0{{$}}
+; CHECK:      .LBB{{[0-9]+}}_3:
+; CHECK:        call foo1{{$}}
+; CHECK:      .LBB{{[0-9]+}}_4:
+; CHECK:        call foo2{{$}}
+; CHECK:      .LBB{{[0-9]+}}_5:
+; CHECK:        call foo3{{$}}
+; CHECK:      .LBB{{[0-9]+}}_6:
+; CHECK:        call foo4{{$}}
+; CHECK:      .LBB{{[0-9]+}}_7:
+; CHECK:        call foo5{{$}}
+; CHECK:      .LBB{{[0-9]+}}_8:
+; CHECK:        return{{$}}
 define void @bar64(i64 %n) {
 entry:
   switch i64 %n, label %sw.epilog [
@@ -178,24 +176,24 @@ sw.epilog:                                        ; preds = %entry, %sw.bb.5, %s
 }
 
 ; CHECK-LABEL: truncated:
-; CHECK:   block
-; CHECK:   block
-; CHECK:   block
-; CHECK:   i32.wrap_i64
-; CHECK:   br_table {{[^,]+}}, 0, 1, 2{{$}}
-; CHECK: .LBB{{[0-9]+}}_1
-; CHECK:   end_block
-; CHECK:   call foo0{{$}}
-; CHECK:   return{{$}}
-; CHECK: .LBB{{[0-9]+}}_2
-; CHECK:   end_block
-; CHECK:   call foo1{{$}}
-; CHECK:   return{{$}}
-; CHECK: .LBB{{[0-9]+}}_3
-; CHECK:   end_block
-; CHECK:   call foo2{{$}}
-; CHECK:   return{{$}}
-; CHECK:   end_function
+; CHECK:        block
+; CHECK:        block
+; CHECK:        block
+; CHECK:        i32.wrap_i64
+; CHECK-NEXT:   br_table {{[^,]+}}, 0, 1, 2{{$}}
+; CHECK:      .LBB{{[0-9]+}}_1
+; CHECK:        end_block
+; CHECK:        call foo0{{$}}
+; CHECK:        return{{$}}
+; CHECK:      .LBB{{[0-9]+}}_2
+; CHECK:        end_block
+; CHECK:        call foo1{{$}}
+; CHECK:        return{{$}}
+; CHECK:      .LBB{{[0-9]+}}_3
+; CHECK:        end_block
+; CHECK:        call foo2{{$}}
+; CHECK:        return{{$}}
+; CHECK:        end_function
 define void @truncated(i64 %n) {
 entry:
   %m = trunc i64 %n to i32

From 36a4f1037628683f2a10c0f3a84904f0463432ff Mon Sep 17 00:00:00 2001
From: Ettore Tiotto <etiotto@ca.ibm.com>
Date: Thu, 30 Jul 2020 15:36:09 +0000
Subject: [PATCH 0650/1035] Fix computeHostNumPhysicalCores() for Linux on
 POWER and Linux on Z
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

ThinLTO is run using a single thread on Linux on Power. The
compute_thread_count() routine calls getHostNumPhysicalCores which
returns -1 by default, and so `MaxThreadCount is set to 1.

unsigned llvm::ThreadPoolStrategy::compute_thread_count() const {
    int MaxThreadCount = UseHyperThreads
          ? computeHostNumHardwareThreads()
          : sys::getHostNumPhysicalCores();
     if (MaxThreadCount <= 0)
        MaxThreadCount = 1;
   …
}
Fix: provide custom implementation of getHostNumPhysicalCores for
Linux on Power and Linux on Z.

Reviewed By: Kai, uweigand

Differential Revision: https://reviews.llvm.org/D84764
---
 llvm/lib/Support/Host.cpp       | 6 ++++++
 llvm/unittests/Support/Host.cpp | 4 +++-
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/Support/Host.cpp b/llvm/lib/Support/Host.cpp
index 658c1ee74cfec..01ede62c755a6 100644
--- a/llvm/lib/Support/Host.cpp
+++ b/llvm/lib/Support/Host.cpp
@@ -1271,6 +1271,12 @@ int computeHostNumPhysicalCores() {
   }
   return CPU_COUNT(&Enabled);
 }
+#elif (defined(__linux__) &&                                                   \
+       (defined(__ppc__) || defined(__powerpc__) || defined(__s390x__)))
+#include <unistd.h>
+
+// Gets the number of *physical cores* on the machine.
+int computeHostNumPhysicalCores() { return sysconf(_SC_NPROCESSORS_ONLN); }
 #elif defined(__APPLE__) && defined(__x86_64__)
 #include <sys/param.h>
 #include <sys/sysctl.h>
diff --git a/llvm/unittests/Support/Host.cpp b/llvm/unittests/Support/Host.cpp
index 7d43366631c58..41c7227dd94ef 100644
--- a/llvm/unittests/Support/Host.cpp
+++ b/llvm/unittests/Support/Host.cpp
@@ -40,7 +40,9 @@ class HostTest : public testing::Test {
     // x86_64 Linux and Darwin.
     return (Host.isOSWindows() && llvm_is_multithreaded()) ||
            (Host.isX86() &&
-            (Host.isOSDarwin() || Host.getOS() == Triple::Linux));
+            (Host.isOSDarwin() || Host.getOS() == Triple::Linux)) ||
+           (Host.getOS() == Triple::Linux &&
+            (Host.isPPC64() || Host.isSystemZ()));
   }
 
   HostTest() : Host(Triple::normalize(sys::getProcessTriple())) {}

From 7551fd5ef8fdc450483b987d6cd44b76b0671ad9 Mon Sep 17 00:00:00 2001
From: Sanjay Patel <spatel@rotateright.com>
Date: Thu, 30 Jul 2020 14:16:18 -0400
Subject: [PATCH 0651/1035] [InstCombine] update test checks; NFC

---
 .../test/Transforms/InstCombine/distribute.ll | 27 ++++++++++++-------
 1 file changed, 17 insertions(+), 10 deletions(-)

diff --git a/llvm/test/Transforms/InstCombine/distribute.ll b/llvm/test/Transforms/InstCombine/distribute.ll
index e6360f8ba64ed..c598b975ce80d 100644
--- a/llvm/test/Transforms/InstCombine/distribute.ll
+++ b/llvm/test/Transforms/InstCombine/distribute.ll
@@ -1,68 +1,75 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -instcombine -S | FileCheck %s
 
 define i32 @factorize(i32 %x, i32 %y) {
 ; CHECK-LABEL: @factorize(
+; CHECK-NEXT:    ret i32 [[X:%.*]]
+;
 ; (X | 1) & (X | 2) -> X | (1 & 2) -> X
   %l = or i32 %x, 1
   %r = or i32 %x, 2
   %z = and i32 %l, %r
   ret i32 %z
-; CHECK: ret i32 %x
 }
 
 define i32 @factorize2(i32 %x) {
 ; CHECK-LABEL: @factorize2(
+; CHECK-NEXT:    ret i32 [[X:%.*]]
+;
 ; 3*X - 2*X -> X
   %l = mul i32 3, %x
   %r = mul i32 2, %x
   %z = sub i32 %l, %r
   ret i32 %z
-; CHECK: ret i32 %x
 }
 
 define i32 @factorize3(i32 %x, i32 %a, i32 %b) {
 ; CHECK-LABEL: @factorize3(
+; CHECK-NEXT:    [[Z:%.*]] = or i32 [[B:%.*]], [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[Z]]
+;
 ; (X | (A|B)) & (X | B) -> X | ((A|B) & B) -> X | B
   %aORb = or i32 %a, %b
   %l = or i32 %x, %aORb
   %r = or i32 %x, %b
   %z = and i32 %l, %r
   ret i32 %z
-; CHECK: %z = or i32 %b, %x
-; CHECK: ret i32 %z
 }
 
 define i32 @factorize4(i32 %x, i32 %y) {
 ; CHECK-LABEL: @factorize4(
+; CHECK-NEXT:    [[S:%.*]] = mul i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[S]]
+;
 ; ((Y << 1) * X) - (X * Y) -> (X * (Y * 2 - Y)) -> (X * Y)
   %sh = shl i32 %y, 1
   %ml = mul i32 %sh, %x
   %mr = mul i32 %x, %y
   %s = sub i32 %ml, %mr
   ret i32 %s
-; CHECK: %s = mul i32 %y, %x
-; CHECK: ret i32 %s
 }
 
 define i32 @factorize5(i32 %x, i32 %y) {
 ; CHECK-LABEL: @factorize5(
+; CHECK-NEXT:    [[S:%.*]] = mul i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[S]]
+;
 ; ((Y * 2) * X) - (X * Y) -> (X * Y)
   %sh = mul i32 %y, 2
   %ml = mul i32 %sh, %x
   %mr = mul i32 %x, %y
   %s = sub i32 %ml, %mr
   ret i32 %s
-; CHECK: %s = mul i32 %y, %x
-; CHECK: ret i32 %s
 }
 
 define i32 @expand(i32 %x) {
 ; CHECK-LABEL: @expand(
+; CHECK-NEXT:    [[A:%.*]] = and i32 [[X:%.*]], 1
+; CHECK-NEXT:    ret i32 [[A]]
+;
 ; ((X & 1) | 2) & 1 -> ((X & 1) & 1) | (2 & 1) -> (X & 1) | 0 -> X & 1
   %a = and i32 %x, 1
   %b = or i32 %a, 2
   %c = and i32 %b, 1
   ret i32 %c
-; CHECK: %a = and i32 %x, 1
-; CHECK: ret i32 %a
 }

From 6b8c641d8ea2b1a0294f556450941199e06e65a5 Mon Sep 17 00:00:00 2001
From: Alexander Belyaev <pifon@google.com>
Date: Thu, 30 Jul 2020 20:16:51 +0200
Subject: [PATCH 0652/1035] [mlir] NFC: Expose `getElementPtrType` and
 `getSizes` methods of AllocOpLowering.

Differential Revision: https://reviews.llvm.org/D84917
---
 .../StandardToLLVM/ConvertStandardToLLVM.h    |  14 +++
 .../StandardToLLVM/StandardToLLVM.cpp         | 106 +++++++++---------
 .../convert-dynamic-memref-ops.mlir           |   4 +
 .../convert-static-memref-ops.mlir            |  15 ++-
 4 files changed, 83 insertions(+), 56 deletions(-)

diff --git a/mlir/include/mlir/Conversion/StandardToLLVM/ConvertStandardToLLVM.h b/mlir/include/mlir/Conversion/StandardToLLVM/ConvertStandardToLLVM.h
index 298899b721d3f..a1b3ec44d3eb8 100644
--- a/mlir/include/mlir/Conversion/StandardToLLVM/ConvertStandardToLLVM.h
+++ b/mlir/include/mlir/Conversion/StandardToLLVM/ConvertStandardToLLVM.h
@@ -440,6 +440,20 @@ class ConvertToLLVMPattern : public ConversionPattern {
                    ValueRange indices, ConversionPatternRewriter &rewriter,
                    llvm::Module &module) const;
 
+  /// Returns the type of a pointer to an element of the memref.
+  Type getElementPtrType(MemRefType type) const;
+
+  /// Determines sizes to be used in the memref descriptor.
+  void getMemRefDescriptorSizes(Location loc, MemRefType memRefType,
+                                ArrayRef<Value> dynSizes,
+                                ConversionPatternRewriter &rewriter,
+                                SmallVectorImpl<Value> &sizes) const;
+
+  /// Computes total size in bytes of to store the given shape.
+  Value getCumulativeSizeInBytes(Location loc, Type elementType,
+                                 ArrayRef<Value> shape,
+                                 ConversionPatternRewriter &rewriter) const;
+
 protected:
   /// Reference to the type converter, with potential extensions.
   LLVMTypeConverter &typeConverter;
diff --git a/mlir/lib/Conversion/StandardToLLVM/StandardToLLVM.cpp b/mlir/lib/Conversion/StandardToLLVM/StandardToLLVM.cpp
index 143f7fa3aabd0..42c4d49b8941b 100644
--- a/mlir/lib/Conversion/StandardToLLVM/StandardToLLVM.cpp
+++ b/mlir/lib/Conversion/StandardToLLVM/StandardToLLVM.cpp
@@ -924,6 +924,52 @@ Value ConvertToLLVMPattern::getDataPtr(Location loc, MemRefType type,
                               offset, rewriter);
 }
 
+Type ConvertToLLVMPattern::getElementPtrType(MemRefType type) const {
+  auto elementType = type.getElementType();
+  auto structElementType = typeConverter.convertType(elementType);
+  return structElementType.cast<LLVM::LLVMType>().getPointerTo(
+      type.getMemorySpace());
+}
+
+void ConvertToLLVMPattern::getMemRefDescriptorSizes(
+    Location loc, MemRefType memRefType, ArrayRef<Value> dynSizes,
+    ConversionPatternRewriter &rewriter, SmallVectorImpl<Value> &sizes) const {
+  sizes.reserve(memRefType.getRank());
+  unsigned i = 0;
+  for (int64_t s : memRefType.getShape())
+    sizes.push_back(s == ShapedType::kDynamicSize
+                        ? dynSizes[i++]
+                        : createIndexConstant(rewriter, loc, s));
+}
+
+Value ConvertToLLVMPattern::getCumulativeSizeInBytes(
+    Location loc, Type elementType, ArrayRef<Value> sizes,
+    ConversionPatternRewriter &rewriter) const {
+  // Compute the total number of memref elements.
+  Value cumulativeSizeInBytes =
+      sizes.empty() ? createIndexConstant(rewriter, loc, 1) : sizes.front();
+  for (unsigned i = 1, e = sizes.size(); i < e; ++i)
+    cumulativeSizeInBytes = rewriter.create<LLVM::MulOp>(
+        loc, getIndexType(), ArrayRef<Value>{cumulativeSizeInBytes, sizes[i]});
+
+  // Compute the size of an individual element. This emits the MLIR equivalent
+  // of the following sizeof(...) implementation in LLVM IR:
+  //   %0 = getelementptr %elementType* null, %indexType 1
+  //   %1 = ptrtoint %elementType* %0 to %indexType
+  // which is a common pattern of getting the size of a type in bytes.
+  auto convertedPtrType = typeConverter.convertType(elementType)
+                              .cast<LLVM::LLVMType>()
+                              .getPointerTo();
+  auto nullPtr = rewriter.create<LLVM::NullOp>(loc, convertedPtrType);
+  auto gep = rewriter.create<LLVM::GEPOp>(
+      loc, convertedPtrType,
+      ArrayRef<Value>{nullPtr, createIndexConstant(rewriter, loc, 1)});
+  auto elementSize =
+      rewriter.create<LLVM::PtrToIntOp>(loc, getIndexType(), gep);
+  return rewriter.create<LLVM::MulOp>(
+      loc, getIndexType(), ArrayRef<Value>{cumulativeSizeInBytes, elementSize});
+}
+
 /// Only retain those attributes that are not constructed by
 /// `LLVMFuncOp::build`. If `filterArgAttrs` is set, also filter out argument
 /// attributes.
@@ -1693,7 +1739,7 @@ struct AllocLikeOpLowering : public ConvertOpToLLVMPattern<AllocLikeOp> {
       Location loc, ConversionPatternRewriter &rewriter, MemRefType memRefType,
       Value allocatedTypePtr, Value allocatedBytePtr, Value accessAlignment,
       uint64_t offset, ArrayRef<int64_t> strides, ArrayRef<Value> sizes) const {
-    auto elementPtrType = getElementPtrType(memRefType);
+    auto elementPtrType = this->getElementPtrType(memRefType);
     auto structType = typeConverter.convertType(memRefType);
     auto memRefDescriptor = MemRefDescriptor::undef(rewriter, loc, structType);
 
@@ -1751,52 +1797,6 @@ struct AllocLikeOpLowering : public ConvertOpToLLVMPattern<AllocLikeOp> {
     return memRefDescriptor;
   }
 
-  /// Determines sizes to be used in the memref descriptor.
-  void getSizes(Location loc, MemRefType memRefType, ArrayRef<Value> operands,
-                ConversionPatternRewriter &rewriter,
-                SmallVectorImpl<Value> &sizes, Value &cumulativeSize,
-                Value &one) const {
-    sizes.reserve(memRefType.getRank());
-    unsigned i = 0;
-    for (int64_t s : memRefType.getShape())
-      sizes.push_back(s == -1 ? operands[i++]
-                              : createIndexConstant(rewriter, loc, s));
-    if (sizes.empty())
-      sizes.push_back(createIndexConstant(rewriter, loc, 1));
-
-    // Compute the total number of memref elements.
-    cumulativeSize = sizes.front();
-    for (unsigned i = 1, e = sizes.size(); i < e; ++i)
-      cumulativeSize = rewriter.create<LLVM::MulOp>(
-          loc, getIndexType(), ArrayRef<Value>{cumulativeSize, sizes[i]});
-
-    // Compute the size of an individual element. This emits the MLIR equivalent
-    // of the following sizeof(...) implementation in LLVM IR:
-    //   %0 = getelementptr %elementType* null, %indexType 1
-    //   %1 = ptrtoint %elementType* %0 to %indexType
-    // which is a common pattern of getting the size of a type in bytes.
-    auto elementType = memRefType.getElementType();
-    auto convertedPtrType = typeConverter.convertType(elementType)
-                                .template cast<LLVM::LLVMType>()
-                                .getPointerTo();
-    auto nullPtr = rewriter.create<LLVM::NullOp>(loc, convertedPtrType);
-    one = createIndexConstant(rewriter, loc, 1);
-    auto gep = rewriter.create<LLVM::GEPOp>(loc, convertedPtrType,
-                                            ArrayRef<Value>{nullPtr, one});
-    auto elementSize =
-        rewriter.create<LLVM::PtrToIntOp>(loc, getIndexType(), gep);
-    cumulativeSize = rewriter.create<LLVM::MulOp>(
-        loc, getIndexType(), ArrayRef<Value>{cumulativeSize, elementSize});
-  }
-
-  /// Returns the type of a pointer to an element of the memref.
-  Type getElementPtrType(MemRefType memRefType) const {
-    auto elementType = memRefType.getElementType();
-    auto structElementType = typeConverter.convertType(elementType);
-    return structElementType.template cast<LLVM::LLVMType>().getPointerTo(
-        memRefType.getMemorySpace());
-  }
-
   /// Returns the memref's element size in bytes.
   // TODO: there are other places where this is used. Expose publicly?
   static unsigned getMemRefEltSizeInBytes(MemRefType memRefType) {
@@ -1851,7 +1851,7 @@ struct AllocLikeOpLowering : public ConvertOpToLLVMPattern<AllocLikeOp> {
                        MemRefType memRefType, Value one, Value &accessAlignment,
                        Value &allocatedBytePtr,
                        ConversionPatternRewriter &rewriter) const {
-    auto elementPtrType = getElementPtrType(memRefType);
+    auto elementPtrType = this->getElementPtrType(memRefType);
 
     // With alloca, one gets a pointer to the element type right away.
     // For stack allocations.
@@ -1954,9 +1954,10 @@ struct AllocLikeOpLowering : public ConvertOpToLLVMPattern<AllocLikeOp> {
     // values and dynamic sizes are passed to 'alloc' as operands.  In case of
     // zero-dimensional memref, assume a scalar (size 1).
     SmallVector<Value, 4> sizes;
-    Value cumulativeSize, one;
-    getSizes(loc, memRefType, operands, rewriter, sizes, cumulativeSize, one);
+    this->getMemRefDescriptorSizes(loc, memRefType, operands, rewriter, sizes);
 
+    Value cumulativeSize = this->getCumulativeSizeInBytes(
+        loc, memRefType.getElementType(), sizes, rewriter);
     // Allocate the underlying buffer.
     // Value holding the alignment that has to be performed post allocation
     // (in conjunction with allocators that do not support alignment, eg.
@@ -1965,8 +1966,9 @@ struct AllocLikeOpLowering : public ConvertOpToLLVMPattern<AllocLikeOp> {
     // Byte pointer to the allocated buffer.
     Value allocatedBytePtr;
     Value allocatedTypePtr =
-        allocateBuffer(loc, cumulativeSize, op, memRefType, one,
-                       accessAlignment, allocatedBytePtr, rewriter);
+        allocateBuffer(loc, cumulativeSize, op, memRefType,
+                       createIndexConstant(rewriter, loc, 1), accessAlignment,
+                       allocatedBytePtr, rewriter);
 
     int64_t offset;
     SmallVector<int64_t, 4> strides;
diff --git a/mlir/test/Conversion/StandardToLLVM/convert-dynamic-memref-ops.mlir b/mlir/test/Conversion/StandardToLLVM/convert-dynamic-memref-ops.mlir
index a1a9634818b33..32365f362d047 100644
--- a/mlir/test/Conversion/StandardToLLVM/convert-dynamic-memref-ops.mlir
+++ b/mlir/test/Conversion/StandardToLLVM/convert-dynamic-memref-ops.mlir
@@ -36,6 +36,7 @@ func @mixed_alloc(%arg0: index, %arg1: index) -> memref<?x42x?xf32> {
 //  CHECK-NEXT:  %[[gep:.*]] = llvm.getelementptr %[[null]][%[[one]]] : (!llvm<"float*">, !llvm.i64) -> !llvm<"float*">
 //  CHECK-NEXT:  %[[sizeof:.*]] = llvm.ptrtoint %[[gep]] : !llvm<"float*"> to !llvm.i64
 //  CHECK-NEXT:  %[[sz_bytes:.*]] = llvm.mul %[[sz]], %[[sizeof]] : !llvm.i64
+//  CHECK-NEXT:  %[[one_1:.*]] = llvm.mlir.constant(1 : index) : !llvm.i64
 //  CHECK-NEXT:  llvm.call @malloc(%[[sz_bytes]]) : (!llvm.i64) -> !llvm<"i8*">
 //  CHECK-NEXT:  llvm.bitcast %{{.*}} : !llvm<"i8*"> to !llvm<"float*">
 //  CHECK-NEXT:  llvm.mlir.undef : !llvm<"{ float*, float*, i64, [3 x i64], [3 x i64] }">
@@ -76,6 +77,7 @@ func @dynamic_alloc(%arg0: index, %arg1: index) -> memref<?x?xf32> {
 //  CHECK-NEXT:  %[[gep:.*]] = llvm.getelementptr %[[null]][%[[one]]] : (!llvm<"float*">, !llvm.i64) -> !llvm<"float*">
 //  CHECK-NEXT:  %[[sizeof:.*]] = llvm.ptrtoint %[[gep]] : !llvm<"float*"> to !llvm.i64
 //  CHECK-NEXT:  %[[sz_bytes:.*]] = llvm.mul %[[sz]], %[[sizeof]] : !llvm.i64
+//  CHECK-NEXT:  %[[one_1:.*]] = llvm.mlir.constant(1 : index) : !llvm.i64
 //  CHECK-NEXT:  llvm.call @malloc(%[[sz_bytes]]) : (!llvm.i64) -> !llvm<"i8*">
 //  CHECK-NEXT:  llvm.bitcast %{{.*}} : !llvm<"i8*"> to !llvm<"float*">
 //  CHECK-NEXT:  llvm.mlir.undef : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
@@ -105,6 +107,7 @@ func @dynamic_alloca(%arg0: index, %arg1: index) -> memref<?x?xf32> {
 //  CHECK-NEXT:  %[[gep:.*]] = llvm.getelementptr %[[null]][%[[one]]] : (!llvm<"float*">, !llvm.i64) -> !llvm<"float*">
 //  CHECK-NEXT:  %[[sizeof:.*]] = llvm.ptrtoint %[[gep]] : !llvm<"float*"> to !llvm.i64
 //  CHECK-NEXT:  %[[sz_bytes:.*]] = llvm.mul %[[num_elems]], %[[sizeof]] : !llvm.i64
+//  CHECK-NEXT:  %[[one_1:.*]] = llvm.mlir.constant(1 : index) : !llvm.i64
 //  CHECK-NEXT:  %[[allocated:.*]] = llvm.alloca %[[sz_bytes]] x !llvm.float : (!llvm.i64) -> !llvm<"float*">
 //  CHECK-NEXT:  llvm.mlir.undef : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
 //  CHECK-NEXT:  llvm.insertvalue %[[allocated]], %{{.*}}[0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
@@ -150,6 +153,7 @@ func @stdlib_aligned_alloc(%N : index) -> memref<32x18xf32> {
 // ALIGNED-ALLOC-NEXT:  %[[gep:.*]] = llvm.getelementptr %[[null]][%[[one]]] : (!llvm<"float*">, !llvm.i64) -> !llvm<"float*">
 // ALIGNED-ALLOC-NEXT:  %[[sizeof:.*]] = llvm.ptrtoint %[[gep]] : !llvm<"float*"> to !llvm.i64
 // ALIGNED-ALLOC-NEXT:  %[[bytes:.*]] = llvm.mul %[[num_elems]], %[[sizeof]] : !llvm.i64
+// ALIGNED-ALLOC-NEXT:  %[[one_1:.*]] = llvm.mlir.constant(1 : index) : !llvm.i64
 // ALIGNED-ALLOC-NEXT:  %[[alignment:.*]] = llvm.mlir.constant(32 : i64) : !llvm.i64
 // ALIGNED-ALLOC-NEXT:  %[[allocated:.*]] = llvm.call @aligned_alloc(%[[alignment]], %[[bytes]]) : (!llvm.i64, !llvm.i64) -> !llvm<"i8*">
 // ALIGNED-ALLOC-NEXT:  llvm.bitcast %[[allocated]] : !llvm<"i8*"> to !llvm<"float*">
diff --git a/mlir/test/Conversion/StandardToLLVM/convert-static-memref-ops.mlir b/mlir/test/Conversion/StandardToLLVM/convert-static-memref-ops.mlir
index f9f9dfffbbbf5..dbf8fe6884289 100644
--- a/mlir/test/Conversion/StandardToLLVM/convert-static-memref-ops.mlir
+++ b/mlir/test/Conversion/StandardToLLVM/convert-static-memref-ops.mlir
@@ -74,6 +74,7 @@ func @zero_d_alloc() -> memref<f32> {
 // CHECK-NEXT:  %[[gep:.*]] = llvm.getelementptr %[[null]][%[[one]]] : (!llvm<"float*">, !llvm.i64) -> !llvm<"float*">
 // CHECK-NEXT:  %[[sizeof:.*]] = llvm.ptrtoint %[[gep]] : !llvm<"float*"> to !llvm.i64
 // CHECK-NEXT:  llvm.mul %{{.*}}, %[[sizeof]] : !llvm.i64
+// CHECK-NEXT:  %[[one_1:.*]] = llvm.mlir.constant(1 : index) : !llvm.i64
 // CHECK-NEXT:  llvm.call @malloc(%{{.*}}) : (!llvm.i64) -> !llvm<"i8*">
 // CHECK-NEXT:  %[[ptr:.*]] = llvm.bitcast %{{.*}} : !llvm<"i8*"> to !llvm<"float*">
 // CHECK-NEXT:  llvm.mlir.undef : !llvm<"{ float*, float*, i64 }">
@@ -88,6 +89,7 @@ func @zero_d_alloc() -> memref<f32> {
 // BAREPTR-NEXT:  %[[gep:.*]] = llvm.getelementptr %[[null]][%[[one]]] : (!llvm<"float*">, !llvm.i64) -> !llvm<"float*">
 // BAREPTR-NEXT:  %[[sizeof:.*]] = llvm.ptrtoint %[[gep]] : !llvm<"float*"> to !llvm.i64
 // BAREPTR-NEXT:  llvm.mul %{{.*}}, %[[sizeof]] : !llvm.i64
+// BAREPTR-NEXT:  %[[one_1:.*]] = llvm.mlir.constant(1 : index) : !llvm.i64
 // BAREPTR-NEXT:  llvm.call @malloc(%{{.*}}) : (!llvm.i64) -> !llvm<"i8*">
 // BAREPTR-NEXT:  %[[ptr:.*]] = llvm.bitcast %{{.*}} : !llvm<"i8*"> to !llvm<"float*">
 // BAREPTR-NEXT:  llvm.mlir.undef : !llvm<"{ float*, float*, i64 }">
@@ -126,9 +128,10 @@ func @aligned_1d_alloc() -> memref<42xf32> {
 // CHECK-NEXT:  %[[gep:.*]] = llvm.getelementptr %[[null]][%[[one]]] : (!llvm<"float*">, !llvm.i64) -> !llvm<"float*">
 // CHECK-NEXT:  %[[sizeof:.*]] = llvm.ptrtoint %[[gep]] : !llvm<"float*"> to !llvm.i64
 // CHECK-NEXT:  llvm.mul %{{.*}}, %[[sizeof]] : !llvm.i64
+// CHECK-NEXT:  %[[one_1:.*]] = llvm.mlir.constant(1 : index) : !llvm.i64
 // CHECK-NEXT:  %[[alignment:.*]] = llvm.mlir.constant(8 : index) : !llvm.i64
 // CHECK-NEXT:  %[[alignmentMinus1:.*]] = llvm.add {{.*}}, %[[alignment]] : !llvm.i64
-// CHECK-NEXT:  %[[allocsize:.*]] = llvm.sub %[[alignmentMinus1]], %[[one]] : !llvm.i64
+// CHECK-NEXT:  %[[allocsize:.*]] = llvm.sub %[[alignmentMinus1]], %[[one_1]] : !llvm.i64
 // CHECK-NEXT:  %[[allocated:.*]] = llvm.call @malloc(%[[allocsize]]) : (!llvm.i64) -> !llvm<"i8*">
 // CHECK-NEXT:  %[[ptr:.*]] = llvm.bitcast %{{.*}} : !llvm<"i8*"> to !llvm<"float*">
 // CHECK-NEXT:  llvm.mlir.undef : !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }">
@@ -137,7 +140,7 @@ func @aligned_1d_alloc() -> memref<42xf32> {
 // CHECK-NEXT:  %[[alignAdj1:.*]] = llvm.urem %[[allocatedAsInt]], %[[alignment]] : !llvm.i64
 // CHECK-NEXT:  %[[alignAdj2:.*]] = llvm.sub %[[alignment]], %[[alignAdj1]] : !llvm.i64
 // CHECK-NEXT:  %[[alignAdj3:.*]] = llvm.urem %[[alignAdj2]], %[[alignment]] : !llvm.i64
-// CHECK-NEXT:  %[[aligned:.*]] = llvm.getelementptr %9[%[[alignAdj3]]] : (!llvm<"i8*">, !llvm.i64) -> !llvm<"i8*">
+// CHECK-NEXT:  %[[aligned:.*]] = llvm.getelementptr %[[allocated]][%[[alignAdj3]]] : (!llvm<"i8*">, !llvm.i64) -> !llvm<"i8*">
 // CHECK-NEXT:  %[[alignedBitCast:.*]] = llvm.bitcast %[[aligned]] : !llvm<"i8*"> to !llvm<"float*">
 // CHECK-NEXT:  llvm.insertvalue %[[alignedBitCast]], %{{.*}}[1] : !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }">
 // CHECK-NEXT:  %[[c0:.*]] = llvm.mlir.constant(0 : index) : !llvm.i64
@@ -149,9 +152,10 @@ func @aligned_1d_alloc() -> memref<42xf32> {
 // BAREPTR-NEXT:  %[[gep:.*]] = llvm.getelementptr %[[null]][%[[one]]] : (!llvm<"float*">, !llvm.i64) -> !llvm<"float*">
 // BAREPTR-NEXT:  %[[sizeof:.*]] = llvm.ptrtoint %[[gep]] : !llvm<"float*"> to !llvm.i64
 // BAREPTR-NEXT:  llvm.mul %{{.*}}, %[[sizeof]] : !llvm.i64
+// BAREPTR-NEXT:  %[[one_1:.*]] = llvm.mlir.constant(1 : index) : !llvm.i64
 // BAREPTR-NEXT:  %[[alignment:.*]] = llvm.mlir.constant(8 : index) : !llvm.i64
 // BAREPTR-NEXT:  %[[alignmentMinus1:.*]] = llvm.add {{.*}}, %[[alignment]] : !llvm.i64
-// BAREPTR-NEXT:  %[[allocsize:.*]] = llvm.sub %[[alignmentMinus1]], %[[one]] : !llvm.i64
+// BAREPTR-NEXT:  %[[allocsize:.*]] = llvm.sub %[[alignmentMinus1]], %[[one_1]] : !llvm.i64
 // BAREPTR-NEXT:  %[[allocated:.*]] = llvm.call @malloc(%[[allocsize]]) : (!llvm.i64) -> !llvm<"i8*">
 // BAREPTR-NEXT:  %[[ptr:.*]] = llvm.bitcast %{{.*}} : !llvm<"i8*"> to !llvm<"float*">
 // BAREPTR-NEXT:  llvm.mlir.undef : !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }">
@@ -160,7 +164,7 @@ func @aligned_1d_alloc() -> memref<42xf32> {
 // BAREPTR-NEXT:  %[[alignAdj1:.*]] = llvm.urem %[[allocatedAsInt]], %[[alignment]] : !llvm.i64
 // BAREPTR-NEXT:  %[[alignAdj2:.*]] = llvm.sub %[[alignment]], %[[alignAdj1]] : !llvm.i64
 // BAREPTR-NEXT:  %[[alignAdj3:.*]] = llvm.urem %[[alignAdj2]], %[[alignment]] : !llvm.i64
-// BAREPTR-NEXT:  %[[aligned:.*]] = llvm.getelementptr %9[%[[alignAdj3]]] : (!llvm<"i8*">, !llvm.i64) -> !llvm<"i8*">
+// BAREPTR-NEXT:  %[[aligned:.*]] = llvm.getelementptr %[[allocated]][%[[alignAdj3]]] : (!llvm<"i8*">, !llvm.i64) -> !llvm<"i8*">
 // BAREPTR-NEXT:  %[[alignedBitCast:.*]] = llvm.bitcast %[[aligned]] : !llvm<"i8*"> to !llvm<"float*">
 // BAREPTR-NEXT:  llvm.insertvalue %[[alignedBitCast]], %{{.*}}[1] : !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }">
 // BAREPTR-NEXT:  %[[c0:.*]] = llvm.mlir.constant(0 : index) : !llvm.i64
@@ -182,6 +186,7 @@ func @static_alloc() -> memref<32x18xf32> {
 // CHECK-NEXT:  %[[gep:.*]] = llvm.getelementptr %[[null]][%[[one]]] : (!llvm<"float*">, !llvm.i64) -> !llvm<"float*">
 //  CHECK-NEXT:  %[[sizeof:.*]] = llvm.ptrtoint %[[gep]] : !llvm<"float*"> to !llvm.i64
 // CHECK-NEXT:  %[[bytes:.*]] = llvm.mul %[[num_elems]], %[[sizeof]] : !llvm.i64
+// CHECK-NEXT:  %[[one_1:.*]] = llvm.mlir.constant(1 : index) : !llvm.i64
 // CHECK-NEXT:  %[[allocated:.*]] = llvm.call @malloc(%[[bytes]]) : (!llvm.i64) -> !llvm<"i8*">
 // CHECK-NEXT:  llvm.bitcast %[[allocated]] : !llvm<"i8*"> to !llvm<"float*">
 
@@ -193,6 +198,7 @@ func @static_alloc() -> memref<32x18xf32> {
 // BAREPTR-NEXT: %[[gep:.*]] = llvm.getelementptr %[[null]][%[[one]]] : (!llvm<"float*">, !llvm.i64) -> !llvm<"float*">
 // BAREPTR-NEXT: %[[sizeof:.*]] = llvm.ptrtoint %[[gep]] : !llvm<"float*"> to !llvm.i64
 // BAREPTR-NEXT: %[[bytes:.*]] = llvm.mul %[[num_elems]], %[[sizeof]] : !llvm.i64
+// BAREPTR-NEXT:  %[[one_1:.*]] = llvm.mlir.constant(1 : index) : !llvm.i64
 // BAREPTR-NEXT: %[[allocated:.*]] = llvm.call @malloc(%[[bytes]]) : (!llvm.i64) -> !llvm<"i8*">
 // BAREPTR-NEXT: llvm.bitcast %[[allocated]] : !llvm<"i8*"> to !llvm<"float*">
  %0 = alloc() : memref<32x18xf32>
@@ -211,6 +217,7 @@ func @static_alloca() -> memref<32x18xf32> {
 // CHECK-NEXT:  %[[gep:.*]] = llvm.getelementptr %[[null]][%[[one]]] : (!llvm<"float*">, !llvm.i64) -> !llvm<"float*">
 // CHECK-NEXT:  %[[sizeof:.*]] = llvm.ptrtoint %[[gep]] : !llvm<"float*"> to !llvm.i64
 // CHECK-NEXT:  %[[bytes:.*]] = llvm.mul %[[num_elems]], %[[sizeof]] : !llvm.i64
+// CHECK-NEXT:  %[[one_1:.*]] = llvm.mlir.constant(1 : index) : !llvm.i64
 // CHECK-NEXT:  %[[allocated:.*]] = llvm.alloca %[[bytes]] x !llvm.float : (!llvm.i64) -> !llvm<"float*">
  %0 = alloca() : memref<32x18xf32>
 

From 2062b3707c1ef698deaa9abc571b937fdd077168 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Thu, 30 Jul 2020 19:12:28 +0100
Subject: [PATCH 0653/1035] [LAA] Avoid adding pointers to the checks if they
 are not needed.

Currently we skip alias sets with only reads or a single write and no
reads, but still add the pointers to the list of pointers in RtCheck.

This can lead to cases where we try to access a pointer that does not
exist when grouping checks.  In most cases, the way we access
PositionMap masked that, as the value would default to index 0.

But in the example in PR46854 it causes a crash.

This patch updates the logic to avoid adding pointers for alias sets
that do not need any checks. It makes things slightly more verbose, by
first checking the numbers of reads/writes and bailing out early if we don't
need checks for the alias set.

I think this makes the logic a bit simpler to follow.

Reviewed By: anemet

Differential Revision: https://reviews.llvm.org/D84608
---
 llvm/lib/Analysis/LoopAccessAnalysis.cpp      | 60 ++++++++-------
 .../LoopLoadElim/pr46854-adress-spaces.ll     | 77 +++++++++++++++++++
 2 files changed, 110 insertions(+), 27 deletions(-)
 create mode 100644 llvm/test/Transforms/LoopLoadElim/pr46854-adress-spaces.ll

diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index ae282a7a10952..f409cd322146e 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -393,7 +393,10 @@ void RuntimePointerChecking::groupChecks(
     // equivalence class, the iteration order is deterministic.
     for (auto MI = DepCands.member_begin(LeaderI), ME = DepCands.member_end();
          MI != ME; ++MI) {
-      unsigned Pointer = PositionMap[MI->getPointer()];
+      auto PointerI = PositionMap.find(MI->getPointer());
+      assert(PointerI != PositionMap.end() &&
+             "pointer in equivalence class not found in PositionMap");
+      unsigned Pointer = PointerI->second;
       bool Merged = false;
       // Mark this pointer as seen.
       Seen.insert(Pointer);
@@ -726,52 +729,55 @@ bool AccessAnalysis::canCheckPtrAtRT(RuntimePointerChecking &RtCheck,
 
     SmallVector<MemAccessInfo, 4> Retries;
 
+    // First, count how many write and read accesses are in the alias set. Also
+    // collect MemAccessInfos for later.
+    SmallVector<MemAccessInfo, 4> AccessInfos;
     for (auto A : AS) {
       Value *Ptr = A.getValue();
       bool IsWrite = Accesses.count(MemAccessInfo(Ptr, true));
-      MemAccessInfo Access(Ptr, IsWrite);
 
       if (IsWrite)
         ++NumWritePtrChecks;
       else
         ++NumReadPtrChecks;
+      AccessInfos.emplace_back(Ptr, IsWrite);
+    }
 
+    // We do not need runtime checks for this alias set, if there are no writes
+    // or a single write and no reads.
+    if (NumWritePtrChecks == 0 ||
+        (NumWritePtrChecks == 1 && NumReadPtrChecks == 0)) {
+      assert((AS.size() <= 1 ||
+              all_of(AS,
+                     [this](auto AC) {
+                       MemAccessInfo AccessWrite(AC.getValue(), true);
+                       return DepCands.findValue(AccessWrite) == DepCands.end();
+                     })) &&
+             "Can only skip updating CanDoRT below, if all entries in AS "
+             "are reads or there is at most 1 entry");
+      continue;
+    }
+
+    for (auto &Access : AccessInfos) {
       if (!createCheckForAccess(RtCheck, Access, StridesMap, DepSetId, TheLoop,
                                 RunningDepId, ASId, ShouldCheckWrap, false)) {
-        LLVM_DEBUG(dbgs() << "LAA: Can't find bounds for ptr:" << *Ptr << '\n');
+        LLVM_DEBUG(dbgs() << "LAA: Can't find bounds for ptr:"
+                          << *Access.getPointer() << '\n');
         Retries.push_back(Access);
         CanDoAliasSetRT = false;
       }
     }
 
-    // If we have at least two writes or one write and a read then we need to
-    // check them.  But there is no need to checks if there is only one
-    // dependence set for this alias set.
-    //
     // Note that this function computes CanDoRT and MayNeedRTCheck
     // independently. For example CanDoRT=false, MayNeedRTCheck=false means that
     // we have a pointer for which we couldn't find the bounds but we don't
     // actually need to emit any checks so it does not matter.
-    bool NeedsAliasSetRTCheck = false;
-    if (!(IsDepCheckNeeded && CanDoAliasSetRT && RunningDepId == 2)) {
-      NeedsAliasSetRTCheck = (NumWritePtrChecks >= 2 ||
-                             (NumReadPtrChecks >= 1 && NumWritePtrChecks >= 1));
-      // For alias sets without at least 2 writes or 1 write and 1 read, there
-      // is no need to generate RT checks and CanDoAliasSetRT for this alias set
-      // does not impact whether runtime checks can be generated.
-      if (!NeedsAliasSetRTCheck) {
-        assert((AS.size() <= 1 ||
-                all_of(AS,
-                       [this](auto AC) {
-                         MemAccessInfo AccessWrite(AC.getValue(), true);
-                         return DepCands.findValue(AccessWrite) ==
-                                DepCands.end();
-                       })) &&
-               "Can only skip updating CanDoRT below, if all entries in AS "
-               "are reads or there is at most 1 entry");
-        continue;
-      }
-    }
+    //
+    // We need runtime checks for this alias set, if there are at least 2
+    // dependence sets (in which case RunningDepId > 2) or if we need to re-try
+    // any bound checks (because in that case the number of dependence sets is
+    // incomplete).
+    bool NeedsAliasSetRTCheck = RunningDepId > 2 || !Retries.empty();
 
     // We need to perform run-time alias checks, but some pointers had bounds
     // that couldn't be checked.
diff --git a/llvm/test/Transforms/LoopLoadElim/pr46854-adress-spaces.ll b/llvm/test/Transforms/LoopLoadElim/pr46854-adress-spaces.ll
new file mode 100644
index 0000000000000..396899d8d2806
--- /dev/null
+++ b/llvm/test/Transforms/LoopLoadElim/pr46854-adress-spaces.ll
@@ -0,0 +1,77 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; REQUIRES: amdgpu-registered-target
+
+; RUN: opt -globals-aa -loop-load-elim -S %s | FileCheck %s
+
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-ni:7"
+target triple = "amdgcn-amd-amdhsa"
+
+%struct.foo = type { %struct.pluto, i8, i8*, i32 }
+%struct.pluto = type { i32, i32, i32, %struct.wombat*, i32, i32, i32 }
+%struct.wombat = type { %struct.barney }
+%struct.barney = type { <2 x float> }
+
+@global = external protected local_unnamed_addr addrspace(4) externally_initialized global %struct.foo, align 8
+@global.1 = internal unnamed_addr addrspace(3) constant [4000 x float] undef, align 16
+
+; Function Attrs: nounwind
+define protected amdgpu_kernel void @widget(i32 %arg, i32 %arg1) #0 {
+; CHECK-LABEL: @widget(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP:%.*]] = getelementptr inbounds [4000 x float], [4000 x float] addrspace(3)* @global.1, i32 0, i32 [[ARG:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = load %struct.wombat*, %struct.wombat* addrspace(4)* getelementptr inbounds (%struct.foo, [[STRUCT_FOO:%.*]] addrspace(4)* @global, i64 0, i32 0, i32 3), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds [[STRUCT_WOMBAT:%.*]], %struct.wombat* [[TMP2]], i64 undef, i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast %struct.barney* [[TMP3]] to i64*
+; CHECK-NEXT:    br label [[BB5:%.*]]
+; CHECK:       bb5.loopexit:
+; CHECK-NEXT:    br label [[BB5]]
+; CHECK:       bb5:
+; CHECK-NEXT:    br label [[BB6:%.*]]
+; CHECK:       bb6:
+; CHECK-NEXT:    [[TMP7:%.*]] = phi i32 [ undef, [[BB5]] ], [ [[TMP19:%.*]], [[BB6]] ]
+; CHECK-NEXT:    [[TMP8:%.*]] = mul nsw i32 [[TMP7]], undef
+; CHECK-NEXT:    [[TMP9:%.*]] = add i32 [[TMP8]], undef
+; CHECK-NEXT:    [[TMP10:%.*]] = sext i32 [[TMP9]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds [[STRUCT_WOMBAT]], %struct.wombat* [[TMP2]], i64 [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast %struct.wombat* [[TMP11]] to i64*
+; CHECK-NEXT:    [[TMP13:%.*]] = load i64, i64* [[TMP12]], align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = srem i32 1, [[ARG1:%.*]]
+; CHECK-NEXT:    [[TMP15:%.*]] = add nuw nsw i32 [[TMP14]], 1
+; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds [4000 x float], [4000 x float] addrspace(3)* @global.1, i32 0, i32 [[TMP15]]
+; CHECK-NEXT:    [[TMP17:%.*]] = load float, float addrspace(3)* [[TMP16]], align 4
+; CHECK-NEXT:    [[TMP18:%.*]] = load float, float addrspace(3)* [[TMP]], align 4
+; CHECK-NEXT:    store i64 [[TMP13]], i64* [[TMP4]], align 8
+; CHECK-NEXT:    [[TMP19]] = add nsw i32 [[TMP7]], 1
+; CHECK-NEXT:    [[TMP20:%.*]] = icmp slt i32 [[TMP7]], 3
+; CHECK-NEXT:    br i1 [[TMP20]], label [[BB6]], label [[BB5_LOOPEXIT:%.*]]
+;
+bb:
+  %tmp = getelementptr inbounds [4000 x float], [4000 x float] addrspace(3)* @global.1, i32 0, i32 %arg
+  %tmp2 = load %struct.wombat*, %struct.wombat* addrspace(4)* getelementptr inbounds (%struct.foo, %struct.foo addrspace(4)* @global, i64 0, i32 0, i32 3), align 8
+  %tmp3 = getelementptr inbounds %struct.wombat, %struct.wombat* %tmp2, i64 undef, i32 0
+  %tmp4 = bitcast %struct.barney* %tmp3 to i64*
+  br label %bb5
+
+bb5:                                              ; preds = %bb6, %bb
+  br label %bb6
+
+bb6:                                              ; preds = %bb6, %bb5
+  %tmp7 = phi i32 [ undef, %bb5 ], [ %tmp19, %bb6 ]
+  %tmp8 = mul nsw i32 %tmp7, undef
+  %tmp9 = add i32 %tmp8, undef
+  %tmp10 = sext i32 %tmp9 to i64
+  %tmp11 = getelementptr inbounds %struct.wombat, %struct.wombat* %tmp2, i64 %tmp10
+  %tmp12 = bitcast %struct.wombat* %tmp11 to i64*
+  %tmp13 = load i64, i64* %tmp12, align 8
+  %tmp14 = srem i32 1, %arg1
+  %tmp15 = add nuw nsw i32 %tmp14, 1
+  %tmp16 = getelementptr inbounds [4000 x float], [4000 x float] addrspace(3)* @global.1, i32 0, i32 %tmp15
+  %tmp17 = load float, float addrspace(3)* %tmp16, align 4
+  %tmp18 = load float, float addrspace(3)* %tmp, align 4
+  store i64 %tmp13, i64* %tmp4, align 8
+  %tmp19 = add nsw i32 %tmp7, 1
+  %tmp20 = icmp slt i32 %tmp7, 3
+  br i1 %tmp20, label %bb6, label %bb5
+}
+
+attributes #0 = { nounwind }

From 3bb48898bc8af58884d4aa6279d4281938c07e76 Mon Sep 17 00:00:00 2001
From: Jonas Devlieghere <jonas@devlieghere.com>
Date: Thu, 30 Jul 2020 11:47:55 -0700
Subject: [PATCH 0654/1035] [lldb/Test] Use self.assertIn in
 TestGdbRemoteTargetXmlPacket

On the ARM buildbot the returned architecture is `armv8l` while
getArchitecture() just returns `arm`.
---
 .../TestGdbRemoteTargetXmlPacket.py                             | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lldb/test/API/tools/lldb-server/registers-target-xml-reading/TestGdbRemoteTargetXmlPacket.py b/lldb/test/API/tools/lldb-server/registers-target-xml-reading/TestGdbRemoteTargetXmlPacket.py
index 51f3d35956406..5ef074c1802cf 100644
--- a/lldb/test/API/tools/lldb-server/registers-target-xml-reading/TestGdbRemoteTargetXmlPacket.py
+++ b/lldb/test/API/tools/lldb-server/registers-target-xml-reading/TestGdbRemoteTargetXmlPacket.py
@@ -43,7 +43,7 @@ def test_g_target_xml_returns_correct_data(self):
 
         architecture = root.find("architecture")
         self.assertIsNotNone(architecture)
-        self.assertEqual(architecture.text, self.getArchitecture())
+        self.assertIn(self.getArchitecture(), architecture.text)
 
         feature = root.find("feature")
         self.assertIsNotNone(feature)

From 4c16eafe12a5f0c82303511d9d3d7acb5ffe84e4 Mon Sep 17 00:00:00 2001
From: Nikita Popov <nikita.ppv@gmail.com>
Date: Sun, 19 Jul 2020 21:28:14 +0200
Subject: [PATCH 0655/1035] [SCCP] Remove dead switch cases based on range
 information

Determine whether switch edges are feasible based on range information,
and remove non-feasible edges lateron.

This does not try to determine whether the default edge is dead,
as we'd have to determine that the range is fully covered by the
cases for that.

Another limitation here is that we don't remove dead cases that
have the same successor as a live case. I'm not handling this
because I wanted to keep the edge removal based on feasible edges
only, rather than inspecting ranges again there -- this does not
seem like a particularly useful case to handle.

Differential Revision: https://reviews.llvm.org/D84270
---
 llvm/lib/Transforms/Scalar/SCCP.cpp | 46 ++++++++++++++++++++-----
 llvm/test/Transforms/SCCP/switch.ll | 53 +++++++++++------------------
 2 files changed, 58 insertions(+), 41 deletions(-)

diff --git a/llvm/lib/Transforms/Scalar/SCCP.cpp b/llvm/lib/Transforms/Scalar/SCCP.cpp
index c4f5c522e4509..9c9f483ab103d 100644
--- a/llvm/lib/Transforms/Scalar/SCCP.cpp
+++ b/llvm/lib/Transforms/Scalar/SCCP.cpp
@@ -649,17 +649,30 @@ void SCCPSolver::getFeasibleSuccessors(Instruction &TI,
       Succs[0] = true;
       return;
     }
-    ValueLatticeElement SCValue = getValueState(SI->getCondition());
-    ConstantInt *CI = getConstantInt(SCValue);
+    const ValueLatticeElement &SCValue = getValueState(SI->getCondition());
+    if (ConstantInt *CI = getConstantInt(SCValue)) {
+      Succs[SI->findCaseValue(CI)->getSuccessorIndex()] = true;
+      return;
+    }
 
-    if (!CI) {   // Overdefined or unknown condition?
-      // All destinations are executable!
-      if (!SCValue.isUnknownOrUndef())
-        Succs.assign(TI.getNumSuccessors(), true);
+    // TODO: Switch on undef is UB. Stop passing false once the rest of LLVM
+    // is ready.
+    if (SCValue.isConstantRange(/*UndefAllowed=*/false)) {
+      const ConstantRange &Range = SCValue.getConstantRange();
+      for (const auto &Case : SI->cases()) {
+        const APInt &CaseValue = Case.getCaseValue()->getValue();
+        if (Range.contains(CaseValue))
+          Succs[Case.getSuccessorIndex()] = true;
+      }
+
+      // TODO: Determine whether default case is reachable.
+      Succs[SI->case_default()->getSuccessorIndex()] = true;
       return;
     }
 
-    Succs[SI->findCaseValue(CI)->getSuccessorIndex()] = true;
+    // Overdefined or unknown condition? All destinations are executable!
+    if (!SCValue.isUnknownOrUndef())
+      Succs.assign(TI.getNumSuccessors(), true);
     return;
   }
 
@@ -1847,9 +1860,26 @@ static bool removeNonFeasibleEdges(const SCCPSolver &Solver, BasicBlock *BB,
 
     BranchInst::Create(OnlyFeasibleSuccessor, BB);
     TI->eraseFromParent();
+    DTU.applyUpdatesPermissive(Updates);
+  } else if (FeasibleSuccessors.size() > 1) {
+    SwitchInstProfUpdateWrapper SI(*cast<SwitchInst>(TI));
+    SmallVector<DominatorTree::UpdateType, 8> Updates;
+    for (auto CI = SI->case_begin(); CI != SI->case_end();) {
+      if (FeasibleSuccessors.contains(CI->getCaseSuccessor())) {
+        ++CI;
+        continue;
+      }
+
+      BasicBlock *Succ = CI->getCaseSuccessor();
+      Succ->removePredecessor(BB);
+      Updates.push_back({DominatorTree::Delete, BB, Succ});
+      SI.removeCase(CI);
+      // Don't increment CI, as we removed a case.
+    }
+
     DTU.applyUpdatesPermissive(Updates);
   } else {
-    llvm_unreachable("Either all successors are feasible, or exactly one is");
+    llvm_unreachable("Must have at least one feasible successor");
   }
   return true;
 }
diff --git a/llvm/test/Transforms/SCCP/switch.ll b/llvm/test/Transforms/SCCP/switch.ll
index 3587587bcb91d..17653014b090d 100644
--- a/llvm/test/Transforms/SCCP/switch.ll
+++ b/llvm/test/Transforms/SCCP/switch.ll
@@ -73,34 +73,29 @@ end:
   ret i32 %phi
 }
 
-define i32 @test_duplicate_successors_phi_3(i1 %c1, i32 %x) {
+define i32 @test_duplicate_successors_phi_3(i1 %c1, i32* %p, i32 %y) {
 ; CHECK-LABEL: @test_duplicate_successors_phi_3(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br i1 [[C1:%.*]], label [[SWITCH:%.*]], label [[SWITCH_1:%.*]]
 ; CHECK:       switch:
-; CHECK-NEXT:    [[C2:%.*]] = icmp ult i32 [[X:%.*]], 3
-; CHECK-NEXT:    call void @llvm.assume(i1 [[C2]])
+; CHECK-NEXT:    [[X:%.*]] = load i32, i32* [[P:%.*]], align 4, !range !0
 ; CHECK-NEXT:    switch i32 [[X]], label [[SWITCH_DEFAULT:%.*]] [
 ; CHECK-NEXT:    i32 0, label [[SWITCH_DEFAULT]]
 ; CHECK-NEXT:    i32 1, label [[SWITCH_0:%.*]]
 ; CHECK-NEXT:    i32 2, label [[SWITCH_0]]
-; CHECK-NEXT:    i32 3, label [[SWITCH_1]]
-; CHECK-NEXT:    i32 4, label [[SWITCH_1]]
 ; CHECK-NEXT:    ]
 ; CHECK:       switch.default:
 ; CHECK-NEXT:    ret i32 -1
 ; CHECK:       switch.0:
 ; CHECK-NEXT:    ret i32 0
 ; CHECK:       switch.1:
-; CHECK-NEXT:    [[PHI:%.*]] = phi i32 [ [[X]], [[ENTRY:%.*]] ], [ 0, [[SWITCH]] ], [ 0, [[SWITCH]] ]
-; CHECK-NEXT:    ret i32 [[PHI]]
+; CHECK-NEXT:    ret i32 [[Y:%.*]]
 ;
 entry:
   br i1 %c1, label %switch, label %switch.1
 
 switch:
-  %c2 = icmp ult i32 %x, 3
-  call void @llvm.assume(i1 %c2)
+  %x = load i32, i32* %p, !range !{i32 0, i32 3}
   switch i32 %x, label %switch.default [
   i32 0, label %switch.default
   i32 1, label %switch.0
@@ -116,19 +111,18 @@ switch.0:
   ret i32 0
 
 switch.1:
-  %phi = phi i32 [ %x, %entry ], [ 0, %switch ], [ 0, %switch ]
+  %phi = phi i32 [ %y, %entry ], [ 0, %switch ], [ 0, %switch ]
   ret i32 %phi
 }
 
-define i32 @test_local_range(i32 %x) {
+; TODO: Determine that the default destination is dead.
+define i32 @test_local_range(i32* %p) {
 ; CHECK-LABEL: @test_local_range(
-; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 [[X:%.*]], 3
-; CHECK-NEXT:    call void @llvm.assume(i1 [[C]])
+; CHECK-NEXT:    [[X:%.*]] = load i32, i32* [[P:%.*]], align 4, !range !0
 ; CHECK-NEXT:    switch i32 [[X]], label [[SWITCH_DEFAULT:%.*]] [
 ; CHECK-NEXT:    i32 0, label [[SWITCH_0:%.*]]
 ; CHECK-NEXT:    i32 1, label [[SWITCH_1:%.*]]
 ; CHECK-NEXT:    i32 2, label [[SWITCH_2:%.*]]
-; CHECK-NEXT:    i32 3, label [[SWITCH_3:%.*]]
 ; CHECK-NEXT:    ]
 ; CHECK:       switch.default:
 ; CHECK-NEXT:    ret i32 -1
@@ -138,11 +132,8 @@ define i32 @test_local_range(i32 %x) {
 ; CHECK-NEXT:    ret i32 1
 ; CHECK:       switch.2:
 ; CHECK-NEXT:    ret i32 2
-; CHECK:       switch.3:
-; CHECK-NEXT:    ret i32 3
 ;
-  %c = icmp ult i32 %x, 3
-  call void @llvm.assume(i1 %c)
+  %x = load i32, i32* %p, !range !{i32 0, i32 3}
   switch i32 %x, label %switch.default [
   i32 0, label %switch.0
   i32 1, label %switch.1
@@ -166,17 +157,15 @@ switch.3:
   ret i32 3
 }
 
-define i32 @test_duplicate_successors(i32 %x) {
+; TODO: Determine that case i3 is dead, even though the edge is shared?
+define i32 @test_duplicate_successors(i32* %p) {
 ; CHECK-LABEL: @test_duplicate_successors(
-; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 [[X:%.*]], 3
-; CHECK-NEXT:    call void @llvm.assume(i1 [[C]])
+; CHECK-NEXT:    [[X:%.*]] = load i32, i32* [[P:%.*]], align 4, !range !0
 ; CHECK-NEXT:    switch i32 [[X]], label [[SWITCH_DEFAULT:%.*]] [
 ; CHECK-NEXT:    i32 0, label [[SWITCH_0:%.*]]
 ; CHECK-NEXT:    i32 1, label [[SWITCH_0]]
 ; CHECK-NEXT:    i32 2, label [[SWITCH_1:%.*]]
 ; CHECK-NEXT:    i32 3, label [[SWITCH_1]]
-; CHECK-NEXT:    i32 4, label [[SWITCH_2:%.*]]
-; CHECK-NEXT:    i32 5, label [[SWITCH_2]]
 ; CHECK-NEXT:    ]
 ; CHECK:       switch.default:
 ; CHECK-NEXT:    ret i32 -1
@@ -184,11 +173,8 @@ define i32 @test_duplicate_successors(i32 %x) {
 ; CHECK-NEXT:    ret i32 0
 ; CHECK:       switch.1:
 ; CHECK-NEXT:    ret i32 1
-; CHECK:       switch.2:
-; CHECK-NEXT:    ret i32 2
 ;
-  %c = icmp ult i32 %x, 3
-  call void @llvm.assume(i1 %c)
+  %x = load i32, i32* %p, !range !{i32 0, i32 3}
   switch i32 %x, label %switch.default [
   i32 0, label %switch.0
   i32 1, label %switch.0
@@ -211,18 +197,17 @@ switch.2:
   ret i32 2
 }
 
+; Case i32 2 is dead as well, but this cannot be determined based on
+; range information.
 define internal i32 @test_ip_range(i32 %x) {
 ; CHECK-LABEL: @test_ip_range(
 ; CHECK-NEXT:    switch i32 [[X:%.*]], label [[SWITCH_DEFAULT:%.*]] [
-; CHECK-NEXT:    i32 0, label [[SWITCH_0:%.*]]
+; CHECK-NEXT:    i32 3, label [[SWITCH_3:%.*]]
 ; CHECK-NEXT:    i32 1, label [[SWITCH_1:%.*]]
 ; CHECK-NEXT:    i32 2, label [[SWITCH_2:%.*]]
-; CHECK-NEXT:    i32 3, label [[SWITCH_3:%.*]]
-; CHECK-NEXT:    ]
+; CHECK-NEXT:    ], !prof !1
 ; CHECK:       switch.default:
 ; CHECK-NEXT:    ret i32 -1
-; CHECK:       switch.0:
-; CHECK-NEXT:    ret i32 0
 ; CHECK:       switch.1:
 ; CHECK-NEXT:    ret i32 1
 ; CHECK:       switch.2:
@@ -235,7 +220,7 @@ define internal i32 @test_ip_range(i32 %x) {
   i32 1, label %switch.1
   i32 2, label %switch.2
   i32 3, label %switch.3
-  ]
+  ], !prof !{!"branch_weights", i32 1, i32 2, i32 3, i32 4, i32 5}
 
 switch.default:
   ret i32 -1
@@ -265,3 +250,5 @@ define void @call_test_ip_range() {
 }
 
 declare void @llvm.assume(i1)
+
+; CHECK: !1 = !{!"branch_weights", i32 1, i32 5, i32 3, i32 4}

From b4c7657ba602acde1c2ea5391c973949b9c3ce09 Mon Sep 17 00:00:00 2001
From: Petr Hosek <phosek@google.com>
Date: Tue, 23 Jun 2020 20:00:04 -0700
Subject: [PATCH 0656/1035] [ELF] Add --dependency-file option

Clang and GCC have a feature (-MD flag) to create a dependency file
in a format that build systems such as Make or Ninja can read, which
specifies all the additional inputs such .h files.

This change introduces the same functionality to lld bringing it to
feature parity with ld and gold which gained this feature recently.
See https://sourceware.org/bugzilla/show_bug.cgi?id=22843 for more
details and discussion.

The implementation corresponds to -MD -MP compiler flag where the
generated dependency file also includes phony targets which works
around the errors where the dependency is removed. This matches the
format used by ld and gold.

Fixes PR42806

Differential Revision: https://reviews.llvm.org/D82437
---
 lld/ELF/Config.h               |  5 ++-
 lld/ELF/Driver.cpp             | 75 ++++++++++++++++++++++++++++++++++
 lld/ELF/InputFiles.cpp         |  1 +
 lld/ELF/Options.td             |  3 ++
 lld/test/ELF/dependency-file.s | 21 ++++++++++
 5 files changed, 104 insertions(+), 1 deletion(-)
 create mode 100644 lld/test/ELF/dependency-file.s

diff --git a/lld/ELF/Config.h b/lld/ELF/Config.h
index e74a4a0c5b223..8eebecf3eb060 100644
--- a/lld/ELF/Config.h
+++ b/lld/ELF/Config.h
@@ -11,6 +11,7 @@
 
 #include "lld/Common/ErrorHandler.h"
 #include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/StringSet.h"
 #include "llvm/BinaryFormat/ELF.h"
@@ -90,11 +91,13 @@ struct Configuration {
   uint8_t osabi = 0;
   uint32_t andFeatures = 0;
   llvm::CachePruningPolicy thinLTOCachePolicy;
+  llvm::SetVector<StringRef> dependencyFiles; // for --dependency-file
   llvm::StringMap<uint64_t> sectionStartMap;
   llvm::StringRef bfdname;
   llvm::StringRef chroot;
-  llvm::StringRef dynamicLinker;
+  llvm::StringRef dependencyFile;
   llvm::StringRef dwoDir;
+  llvm::StringRef dynamicLinker;
   llvm::StringRef entry;
   llvm::StringRef emulation;
   llvm::StringRef fini;
diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp
index 64a41ba77ba2f..75aa89ff3c863 100644
--- a/lld/ELF/Driver.cpp
+++ b/lld/ELF/Driver.cpp
@@ -918,6 +918,7 @@ static void readConfigs(opt::InputArgList &args) {
   config->optimizeBBJumps =
       args.hasFlag(OPT_optimize_bb_jumps, OPT_no_optimize_bb_jumps, false);
   config->demangle = args.hasFlag(OPT_demangle, OPT_no_demangle, true);
+  config->dependencyFile = args.getLastArgValue(OPT_dependency_file);
   config->dependentLibraries = args.hasFlag(OPT_dependent_libraries, OPT_no_dependent_libraries, true);
   config->disableVerify = args.hasArg(OPT_disable_verify);
   config->discard = getDiscard(args);
@@ -1564,6 +1565,75 @@ static void handleLibcall(StringRef name) {
     sym->fetch();
 }
 
+// Handle --dependency-file=<path>. If that option is given, lld creates a
+// file at a given path with the following contents:
+//
+//   <output-file>: <input-file> ...
+//
+//   <input-file>:
+//
+// where <output-file> is a pathname of an output file and <input-file>
+// ... is a list of pathnames of all input files. `make` command can read a
+// file in the above format and interpret it as a dependency info. We write
+// phony targets for every <input-file> to avoid an error when that file is
+// removed.
+//
+// This option is useful if you want to make your final executable to depend
+// on all input files including system libraries. Here is why.
+//
+// When you write a Makefile, you usually write it so that the final
+// executable depends on all user-generated object files. Normally, you
+// don't make your executable to depend on system libraries (such as libc)
+// because you don't know the exact paths of libraries, even though system
+// libraries that are linked to your executable statically are technically a
+// part of your program. By using --dependency-file option, you can make
+// lld to dump dependency info so that you can maintain exact dependencies
+// easily.
+static void writeDependencyFile() {
+  std::error_code ec;
+  raw_fd_ostream os(config->dependencyFile, ec, sys::fs::F_None);
+  if (ec) {
+    error("cannot open " + config->dependencyFile + ": " + ec.message());
+    return;
+  }
+
+  // We use the same escape rules as Clang/GCC which are accepted by Make/Ninja:
+  // * A space is escaped by a backslash which itself must be escaped.
+  // * A hash sign is escaped by a single backslash.
+  // * $ is escapes as $$.
+  auto printFilename = [](raw_fd_ostream &os, StringRef filename) {
+    llvm::SmallString<256> nativePath;
+    llvm::sys::path::native(filename.str(), nativePath);
+    llvm::sys::path::remove_dots(nativePath, /*remove_dot_dot=*/true);
+    for (unsigned i = 0, e = nativePath.size(); i != e; ++i) {
+      if (nativePath[i] == '#') {
+        os << '\\';
+      } else if (nativePath[i] == ' ') {
+        os << '\\';
+        unsigned j = i;
+        while (j > 0 && nativePath[--j] == '\\')
+          os << '\\';
+      } else if (nativePath[i] == '$') {
+        os << '$';
+      }
+      os << nativePath[i];
+    }
+  };
+
+  os << config->outputFile << ":";
+  for (StringRef path : config->dependencyFiles) {
+    os << " \\\n ";
+    printFilename(os, path);
+  }
+  os << "\n";
+
+  for (StringRef path : config->dependencyFiles) {
+    os << "\n";
+    printFilename(os, path);
+    os << ":\n";
+  }
+}
+
 // Replaces common symbols with defined symbols reside in .bss sections.
 // This function is called after all symbol names are resolved. As a
 // result, the passes after the symbol resolution won't see any
@@ -2064,6 +2134,11 @@ template <class ELFT> void LinkerDriver::link(opt::InputArgList &args) {
     return false;
   });
 
+  // Since we now have a complete set of input files, we can create
+  // a .d file to record build dependencies.
+  if (!config->dependencyFile.empty())
+    writeDependencyFile();
+
   // Now that the number of partitions is fixed, save a pointer to the main
   // partition.
   mainPart = &partitions[0];
diff --git a/lld/ELF/InputFiles.cpp b/lld/ELF/InputFiles.cpp
index c142c00517ccf..79cc3c00cb2bc 100644
--- a/lld/ELF/InputFiles.cpp
+++ b/lld/ELF/InputFiles.cpp
@@ -110,6 +110,7 @@ Optional<MemoryBufferRef> elf::readFile(StringRef path) {
     path = saver.save(config->chroot + path);
 
   log(path);
+  config->dependencyFiles.insert(path);
 
   auto mbOrErr = MemoryBuffer::getFile(path, -1, false);
   if (auto ec = mbOrErr.getError()) {
diff --git a/lld/ELF/Options.td b/lld/ELF/Options.td
index c3c1309aca1a7..3a8c9e83b08e0 100644
--- a/lld/ELF/Options.td
+++ b/lld/ELF/Options.td
@@ -132,6 +132,9 @@ defm demangle: B<"demangle",
     "Demangle symbol names (default)",
     "Do not demangle symbol names">;
 
+defm dependency_file: EEq<"dependency-file", "Write a dependency file">,
+  MetaVarName<"<path>">;
+
 def disable_new_dtags: F<"disable-new-dtags">,
   HelpText<"Disable new dynamic tags">;
 
diff --git a/lld/test/ELF/dependency-file.s b/lld/test/ELF/dependency-file.s
new file mode 100644
index 0000000000000..14048ead49430
--- /dev/null
+++ b/lld/test/ELF/dependency-file.s
@@ -0,0 +1,21 @@
+# REQUIRES: x86
+# RUN: mkdir -p %t
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o %t/foo.o
+# RUN: llvm-mc -filetype=obj -triple=x86_64 /dev/null -o "%t/bar baz.o"
+# RUN: llvm-mc -filetype=obj -triple=x86_64 /dev/null -o "%t/#quux$.o"
+# RUN: ld.lld -o %t/foo.exe %t/foo.o %t/"bar baz.o" "%t/#quux$.o" --dependency-file=%t/foo.d
+# RUN: FileCheck --match-full-lines -DFILE=%t %s < %t/foo.d
+
+# CHECK:      [[FILE]]{{/|\\\\}}foo.exe: \
+# CHECK-NEXT:   [[FILE]]{{/|\\\\}}foo.o \
+# CHECK-NEXT:   [[FILE]]{{/|\\\\}}bar\ baz.o \
+# CHECK-NEXT:   [[FILE]]{{/|\\\\}}\#quux$$.o
+# CHECK-EMPTY:
+# CHECK-NEXT: [[FILE]]{{/|\\\\}}foo.o:
+# CHECK-EMPTY:
+# CHECK-NEXT: [[FILE]]{{/|\\\\}}bar\ baz.o:
+# CHECK-EMPTY:
+# CHECK-NEXT: [[FILE]]{{/|\\\\}}\#quux$$.o:
+
+.global _start
+_start:

From fac84536bcb9fbca2b3517ca83d382329d0e6c55 Mon Sep 17 00:00:00 2001
From: Peter Steinfeld <psteinfeld@nvidia.com>
Date: Thu, 30 Jul 2020 10:51:44 -0700
Subject: [PATCH 0657/1035] [flang] Fix an assert on duplicate initializations

When declaring the same variable twice with an initialization, we were failing
an internal check.  I fixed this by checking to see if the associated symbol
already had an error.

I added tests for pointer and non-pointer initialization of duplicate names.

Differential Revision: https://reviews.llvm.org/D84969
---
 flang/lib/Semantics/resolve-names.cpp | 33 +++++++++++++++------------
 flang/test/Semantics/resolve91.f90    | 11 +++++++++
 2 files changed, 29 insertions(+), 15 deletions(-)

diff --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp
index 744eee19ba43c..eb7dd697b2748 100644
--- a/flang/lib/Semantics/resolve-names.cpp
+++ b/flang/lib/Semantics/resolve-names.cpp
@@ -5680,22 +5680,25 @@ void DeclarationVisitor::NonPointerInitialization(const parser::Name &name,
     const parser::ConstantExpr &expr, bool inComponentDecl) {
   if (name.symbol) {
     Symbol &ultimate{name.symbol->GetUltimate()};
-    if (IsPointer(ultimate)) {
-      Say(name, "'%s' is a pointer but is not initialized like one"_err_en_US);
-    } else if (auto *details{ultimate.detailsIf<ObjectEntityDetails>()}) {
-      CHECK(!details->init());
-      Walk(expr);
-      // TODO: check C762 - all bounds and type parameters of component
-      // are colons or constant expressions if component is initialized
-      if (inComponentDecl) {
-        // Can't convert to type of component, which might not yet
-        // be known; that's done later during instantiation.
-        if (MaybeExpr value{EvaluateExpr(expr)}) {
-          details->set_init(std::move(*value));
+    if (!context().HasError(ultimate)) {
+      if (IsPointer(ultimate)) {
+        Say(name,
+            "'%s' is a pointer but is not initialized like one"_err_en_US);
+      } else if (auto *details{ultimate.detailsIf<ObjectEntityDetails>()}) {
+        CHECK(!details->init());
+        Walk(expr);
+        // TODO: check C762 - all bounds and type parameters of component
+        // are colons or constant expressions if component is initialized
+        if (inComponentDecl) {
+          // Can't convert to type of component, which might not yet
+          // be known; that's done later during instantiation.
+          if (MaybeExpr value{EvaluateExpr(expr)}) {
+            details->set_init(std::move(*value));
+          }
+        } else if (MaybeExpr folded{EvaluateConvertedExpr(
+                       ultimate, expr, expr.thing.value().source)}) {
+          details->set_init(std::move(*folded));
         }
-      } else if (MaybeExpr folded{EvaluateConvertedExpr(
-                     ultimate, expr, expr.thing.value().source)}) {
-        details->set_init(std::move(*folded));
       }
     }
   }
diff --git a/flang/test/Semantics/resolve91.f90 b/flang/test/Semantics/resolve91.f90
index 2fbcaa384dfc1..c94cda458f3f0 100644
--- a/flang/test/Semantics/resolve91.f90
+++ b/flang/test/Semantics/resolve91.f90
@@ -63,3 +63,14 @@ module m7
   !ERROR: Derived type 'ubound' not found
   integer :: ivar = ubound(iarray)(1)
 end module m7
+
+module m8
+  integer :: iVar = 3
+  !ERROR: The type of 'ivar' has already been declared
+  integer :: iVar = 4
+  integer, target :: jVar = 5
+  integer, target :: kVar = 5
+  integer, pointer :: pVar => jVar
+  !ERROR: The type of 'pvar' has already been declared
+  integer, pointer :: pVar => kVar
+end module m8

From 3ff7dcded213844bcb0c1d55a1a66ad8fd9b483b Mon Sep 17 00:00:00 2001
From: Petr Hosek <phosek@google.com>
Date: Thu, 16 Jul 2020 19:50:34 -0700
Subject: [PATCH 0658/1035] [CMake][Fuchsia] Include additional tools in the
 toolchain

These are needed on Windows.

Differential Revision: https://reviews.llvm.org/D83999
---
 clang/cmake/caches/Fuchsia-stage2.cmake | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/clang/cmake/caches/Fuchsia-stage2.cmake b/clang/cmake/caches/Fuchsia-stage2.cmake
index 259684ff2b0d9..e00b64073ca52 100644
--- a/clang/cmake/caches/Fuchsia-stage2.cmake
+++ b/clang/cmake/caches/Fuchsia-stage2.cmake
@@ -233,14 +233,17 @@ set(LLVM_TOOLCHAIN_TOOLS
   llvm-ar
   llvm-cov
   llvm-cxxfilt
+  llvm-dlltool
   llvm-dwarfdump
   llvm-dwp
   llvm-gsymutil
   llvm-lib
+  llvm-mt
   llvm-nm
   llvm-objcopy
   llvm-objdump
   llvm-profdata
+  llvm-rc
   llvm-ranlib
   llvm-readelf
   llvm-readobj

From 95d1e668ed502af481edabb64d3a2093a57329ad Mon Sep 17 00:00:00 2001
From: Nikita Popov <nikita.ppv@gmail.com>
Date: Thu, 30 Jul 2020 22:16:11 +0200
Subject: [PATCH 0659/1035] [CVP] Add tests for min/max/abs intrinsic
 comparisons (NFC)

---
 .../CorrelatedValuePropagation/minmaxabs.ll   | 165 ++++++++++++++++++
 1 file changed, 165 insertions(+)
 create mode 100644 llvm/test/Transforms/CorrelatedValuePropagation/minmaxabs.ll

diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/minmaxabs.ll b/llvm/test/Transforms/CorrelatedValuePropagation/minmaxabs.ll
new file mode 100644
index 0000000000000..6ccfd7a6e901b
--- /dev/null
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/minmaxabs.ll
@@ -0,0 +1,165 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -correlated-propagation < %s | FileCheck %s
+
+declare i32 @llvm.umin.i32(i32, i32)
+declare i32 @llvm.umax.i32(i32, i32)
+declare i32 @llvm.smin.i32(i32, i32)
+declare i32 @llvm.smax.i32(i32, i32)
+declare i32 @llvm.abs.i32(i32, i1)
+declare void @use(i1)
+
+define void @test_umin(i32 %x) {
+; CHECK-LABEL: @test_umin(
+; CHECK-NEXT:    [[M:%.*]] = call i32 @llvm.umin.i32(i32 [[X:%.*]], i32 10)
+; CHECK-NEXT:    br label [[SPLIT:%.*]]
+; CHECK:       split:
+; CHECK-NEXT:    [[C1:%.*]] = icmp ule i32 [[M]], 10
+; CHECK-NEXT:    call void @use(i1 [[C1]])
+; CHECK-NEXT:    [[C2:%.*]] = icmp ult i32 [[M]], 10
+; CHECK-NEXT:    call void @use(i1 [[C2]])
+; CHECK-NEXT:    ret void
+;
+  %m = call i32 @llvm.umin.i32(i32 %x, i32 10)
+  br label %split
+
+split:
+  %c1 = icmp ule i32 %m, 10
+  call void @use(i1 %c1)
+  %c2 = icmp ult i32 %m, 10
+  call void @use(i1 %c2)
+  ret void
+}
+
+define void @test_umax(i32 %x) {
+; CHECK-LABEL: @test_umax(
+; CHECK-NEXT:    [[M:%.*]] = call i32 @llvm.umax.i32(i32 [[X:%.*]], i32 10)
+; CHECK-NEXT:    br label [[SPLIT:%.*]]
+; CHECK:       split:
+; CHECK-NEXT:    [[C1:%.*]] = icmp uge i32 [[M]], 10
+; CHECK-NEXT:    call void @use(i1 [[C1]])
+; CHECK-NEXT:    [[C2:%.*]] = icmp ugt i32 [[M]], 10
+; CHECK-NEXT:    call void @use(i1 [[C2]])
+; CHECK-NEXT:    ret void
+;
+  %m = call i32 @llvm.umax.i32(i32 %x, i32 10)
+  br label %split
+
+split:
+  %c1 = icmp uge i32 %m, 10
+  call void @use(i1 %c1)
+  %c2 = icmp ugt i32 %m, 10
+  call void @use(i1 %c2)
+  ret void
+}
+
+define void @test_smin(i32 %x) {
+; CHECK-LABEL: @test_smin(
+; CHECK-NEXT:    [[M:%.*]] = call i32 @llvm.smin.i32(i32 [[X:%.*]], i32 10)
+; CHECK-NEXT:    br label [[SPLIT:%.*]]
+; CHECK:       split:
+; CHECK-NEXT:    [[C1:%.*]] = icmp sle i32 [[M]], 10
+; CHECK-NEXT:    call void @use(i1 [[C1]])
+; CHECK-NEXT:    [[C2:%.*]] = icmp slt i32 [[M]], 10
+; CHECK-NEXT:    call void @use(i1 [[C2]])
+; CHECK-NEXT:    ret void
+;
+  %m = call i32 @llvm.smin.i32(i32 %x, i32 10)
+  br label %split
+
+split:
+  %c1 = icmp sle i32 %m, 10
+  call void @use(i1 %c1)
+  %c2 = icmp slt i32 %m, 10
+  call void @use(i1 %c2)
+  ret void
+}
+
+define void @test_smax(i32 %x) {
+; CHECK-LABEL: @test_smax(
+; CHECK-NEXT:    [[M:%.*]] = call i32 @llvm.smax.i32(i32 [[X:%.*]], i32 10)
+; CHECK-NEXT:    br label [[SPLIT:%.*]]
+; CHECK:       split:
+; CHECK-NEXT:    [[C1:%.*]] = icmp sge i32 [[M]], 10
+; CHECK-NEXT:    call void @use(i1 [[C1]])
+; CHECK-NEXT:    [[C2:%.*]] = icmp sgt i32 [[M]], 10
+; CHECK-NEXT:    call void @use(i1 [[C2]])
+; CHECK-NEXT:    ret void
+;
+  %m = call i32 @llvm.smax.i32(i32 %x, i32 10)
+  br label %split
+
+split:
+  %c1 = icmp sge i32 %m, 10
+  call void @use(i1 %c1)
+  %c2 = icmp sgt i32 %m, 10
+  call void @use(i1 %c2)
+  ret void
+}
+
+define void @test_abs1(i32* %p) {
+; CHECK-LABEL: @test_abs1(
+; CHECK-NEXT:    [[X:%.*]] = load i32, i32* [[P:%.*]], align 4, !range !0
+; CHECK-NEXT:    [[A:%.*]] = call i32 @llvm.abs.i32(i32 [[X]], i1 false)
+; CHECK-NEXT:    br label [[SPLIT:%.*]]
+; CHECK:       split:
+; CHECK-NEXT:    [[C1:%.*]] = icmp ule i32 [[A]], 15
+; CHECK-NEXT:    call void @use(i1 [[C1]])
+; CHECK-NEXT:    [[C2:%.*]] = icmp ult i32 [[A]], 15
+; CHECK-NEXT:    call void @use(i1 [[C2]])
+; CHECK-NEXT:    ret void
+;
+  %x = load i32, i32* %p, !range !{i32 -15, i32 10}
+  %a = call i32 @llvm.abs.i32(i32 %x, i1 false)
+  br label %split
+
+split:
+  %c1 = icmp ule i32 %a, 15
+  call void @use(i1 %c1)
+  %c2 = icmp ult i32 %a, 15
+  call void @use(i1 %c2)
+  ret void
+}
+
+define void @test_abs2(i32 %x) {
+; CHECK-LABEL: @test_abs2(
+; CHECK-NEXT:    [[A:%.*]] = call i32 @llvm.abs.i32(i32 [[X:%.*]], i1 false)
+; CHECK-NEXT:    br label [[SPLIT:%.*]]
+; CHECK:       split:
+; CHECK-NEXT:    [[C1:%.*]] = icmp ule i32 [[A]], -2147483648
+; CHECK-NEXT:    call void @use(i1 [[C1]])
+; CHECK-NEXT:    [[C2:%.*]] = icmp ult i32 [[A]], -2147483648
+; CHECK-NEXT:    call void @use(i1 [[C2]])
+; CHECK-NEXT:    ret void
+;
+  %a = call i32 @llvm.abs.i32(i32 %x, i1 false)
+  br label %split
+
+split:
+  %c1 = icmp ule i32 %a, 2147483648
+  call void @use(i1 %c1)
+  %c2 = icmp ult i32 %a, 2147483648
+  call void @use(i1 %c2)
+  ret void
+}
+
+define void @test_abs3(i32 %x) {
+; CHECK-LABEL: @test_abs3(
+; CHECK-NEXT:    [[A:%.*]] = call i32 @llvm.abs.i32(i32 [[X:%.*]], i1 true)
+; CHECK-NEXT:    br label [[SPLIT:%.*]]
+; CHECK:       split:
+; CHECK-NEXT:    [[C1:%.*]] = icmp sge i32 [[A]], 0
+; CHECK-NEXT:    call void @use(i1 [[C1]])
+; CHECK-NEXT:    [[C2:%.*]] = icmp sgt i32 [[A]], 0
+; CHECK-NEXT:    call void @use(i1 [[C2]])
+; CHECK-NEXT:    ret void
+;
+  %a = call i32 @llvm.abs.i32(i32 %x, i1 true)
+  br label %split
+
+split:
+  %c1 = icmp sge i32 %a, 0
+  call void @use(i1 %c1)
+  %c2 = icmp sgt i32 %a, 0
+  call void @use(i1 %c2)
+  ret void
+}

From d8a98a9c351be0a535ce5c3b9b200a76e9b16d3a Mon Sep 17 00:00:00 2001
From: Nikita Popov <nikita.ppv@gmail.com>
Date: Thu, 30 Jul 2020 22:15:06 +0200
Subject: [PATCH 0660/1035] [ConstantRange][CVP] Compute min/max/abs intrinsic
 ranges

Wire up ConstantRange::intrinsic() to the existing primitives for
min, max and abs.

The poison flag on abs is not yet taken into account.
---
 llvm/lib/IR/ConstantRange.cpp                  | 16 ++++++++++++++++
 .../CorrelatedValuePropagation/minmaxabs.ll    | 18 ++++++------------
 2 files changed, 22 insertions(+), 12 deletions(-)

diff --git a/llvm/lib/IR/ConstantRange.cpp b/llvm/lib/IR/ConstantRange.cpp
index 26cff911cd675..d379140639dc9 100644
--- a/llvm/lib/IR/ConstantRange.cpp
+++ b/llvm/lib/IR/ConstantRange.cpp
@@ -842,6 +842,11 @@ bool ConstantRange::isIntrinsicSupported(Intrinsic::ID IntrinsicID) {
   case Intrinsic::usub_sat:
   case Intrinsic::sadd_sat:
   case Intrinsic::ssub_sat:
+  case Intrinsic::umin:
+  case Intrinsic::umax:
+  case Intrinsic::smin:
+  case Intrinsic::smax:
+  case Intrinsic::abs:
     return true;
   default:
     return false;
@@ -859,6 +864,17 @@ ConstantRange ConstantRange::intrinsic(Intrinsic::ID IntrinsicID,
     return Ops[0].sadd_sat(Ops[1]);
   case Intrinsic::ssub_sat:
     return Ops[0].ssub_sat(Ops[1]);
+  case Intrinsic::umin:
+    return Ops[0].umin(Ops[1]);
+  case Intrinsic::umax:
+    return Ops[0].umax(Ops[1]);
+  case Intrinsic::smin:
+    return Ops[0].smin(Ops[1]);
+  case Intrinsic::smax:
+    return Ops[0].smax(Ops[1]);
+  case Intrinsic::abs:
+    // TODO: Make use of poison flag.
+    return Ops[0].abs();
   default:
     assert(!isIntrinsicSupported(IntrinsicID) && "Shouldn't be supported");
     llvm_unreachable("Unsupported intrinsic");
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/minmaxabs.ll b/llvm/test/Transforms/CorrelatedValuePropagation/minmaxabs.ll
index 6ccfd7a6e901b..ef5bf4d7c2a16 100644
--- a/llvm/test/Transforms/CorrelatedValuePropagation/minmaxabs.ll
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/minmaxabs.ll
@@ -13,8 +13,7 @@ define void @test_umin(i32 %x) {
 ; CHECK-NEXT:    [[M:%.*]] = call i32 @llvm.umin.i32(i32 [[X:%.*]], i32 10)
 ; CHECK-NEXT:    br label [[SPLIT:%.*]]
 ; CHECK:       split:
-; CHECK-NEXT:    [[C1:%.*]] = icmp ule i32 [[M]], 10
-; CHECK-NEXT:    call void @use(i1 [[C1]])
+; CHECK-NEXT:    call void @use(i1 true)
 ; CHECK-NEXT:    [[C2:%.*]] = icmp ult i32 [[M]], 10
 ; CHECK-NEXT:    call void @use(i1 [[C2]])
 ; CHECK-NEXT:    ret void
@@ -35,8 +34,7 @@ define void @test_umax(i32 %x) {
 ; CHECK-NEXT:    [[M:%.*]] = call i32 @llvm.umax.i32(i32 [[X:%.*]], i32 10)
 ; CHECK-NEXT:    br label [[SPLIT:%.*]]
 ; CHECK:       split:
-; CHECK-NEXT:    [[C1:%.*]] = icmp uge i32 [[M]], 10
-; CHECK-NEXT:    call void @use(i1 [[C1]])
+; CHECK-NEXT:    call void @use(i1 true)
 ; CHECK-NEXT:    [[C2:%.*]] = icmp ugt i32 [[M]], 10
 ; CHECK-NEXT:    call void @use(i1 [[C2]])
 ; CHECK-NEXT:    ret void
@@ -57,8 +55,7 @@ define void @test_smin(i32 %x) {
 ; CHECK-NEXT:    [[M:%.*]] = call i32 @llvm.smin.i32(i32 [[X:%.*]], i32 10)
 ; CHECK-NEXT:    br label [[SPLIT:%.*]]
 ; CHECK:       split:
-; CHECK-NEXT:    [[C1:%.*]] = icmp sle i32 [[M]], 10
-; CHECK-NEXT:    call void @use(i1 [[C1]])
+; CHECK-NEXT:    call void @use(i1 true)
 ; CHECK-NEXT:    [[C2:%.*]] = icmp slt i32 [[M]], 10
 ; CHECK-NEXT:    call void @use(i1 [[C2]])
 ; CHECK-NEXT:    ret void
@@ -79,8 +76,7 @@ define void @test_smax(i32 %x) {
 ; CHECK-NEXT:    [[M:%.*]] = call i32 @llvm.smax.i32(i32 [[X:%.*]], i32 10)
 ; CHECK-NEXT:    br label [[SPLIT:%.*]]
 ; CHECK:       split:
-; CHECK-NEXT:    [[C1:%.*]] = icmp sge i32 [[M]], 10
-; CHECK-NEXT:    call void @use(i1 [[C1]])
+; CHECK-NEXT:    call void @use(i1 true)
 ; CHECK-NEXT:    [[C2:%.*]] = icmp sgt i32 [[M]], 10
 ; CHECK-NEXT:    call void @use(i1 [[C2]])
 ; CHECK-NEXT:    ret void
@@ -102,8 +98,7 @@ define void @test_abs1(i32* %p) {
 ; CHECK-NEXT:    [[A:%.*]] = call i32 @llvm.abs.i32(i32 [[X]], i1 false)
 ; CHECK-NEXT:    br label [[SPLIT:%.*]]
 ; CHECK:       split:
-; CHECK-NEXT:    [[C1:%.*]] = icmp ule i32 [[A]], 15
-; CHECK-NEXT:    call void @use(i1 [[C1]])
+; CHECK-NEXT:    call void @use(i1 true)
 ; CHECK-NEXT:    [[C2:%.*]] = icmp ult i32 [[A]], 15
 ; CHECK-NEXT:    call void @use(i1 [[C2]])
 ; CHECK-NEXT:    ret void
@@ -125,8 +120,7 @@ define void @test_abs2(i32 %x) {
 ; CHECK-NEXT:    [[A:%.*]] = call i32 @llvm.abs.i32(i32 [[X:%.*]], i1 false)
 ; CHECK-NEXT:    br label [[SPLIT:%.*]]
 ; CHECK:       split:
-; CHECK-NEXT:    [[C1:%.*]] = icmp ule i32 [[A]], -2147483648
-; CHECK-NEXT:    call void @use(i1 [[C1]])
+; CHECK-NEXT:    call void @use(i1 true)
 ; CHECK-NEXT:    [[C2:%.*]] = icmp ult i32 [[A]], -2147483648
 ; CHECK-NEXT:    call void @use(i1 [[C2]])
 ; CHECK-NEXT:    ret void

From da324f9904634855a0a3549284758cb079723cdf Mon Sep 17 00:00:00 2001
From: Peiyuan Song <squallatf@gmail.com>
Date: Thu, 30 Jul 2020 23:32:37 +0300
Subject: [PATCH 0661/1035] [LLD] [Mingw] Don't export symbols from profile
 generate

Differential Revision: https://reviews.llvm.org/D84756
---
 lld/COFF/MinGW.cpp | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/lld/COFF/MinGW.cpp b/lld/COFF/MinGW.cpp
index bded985f04d0f..e24cdca6ee34c 100644
--- a/lld/COFF/MinGW.cpp
+++ b/lld/COFF/MinGW.cpp
@@ -34,6 +34,11 @@ AutoExporter::AutoExporter() {
       "libclang_rt.builtins-arm",
       "libclang_rt.builtins-i386",
       "libclang_rt.builtins-x86_64",
+      "libclang_rt.profile",
+      "libclang_rt.profile-aarch64",
+      "libclang_rt.profile-arm",
+      "libclang_rt.profile-i386",
+      "libclang_rt.profile-x86_64",
       "libc++",
       "libc++abi",
       "libunwind",
@@ -57,6 +62,10 @@ AutoExporter::AutoExporter() {
       "__builtin_",
       // Artificial symbols such as .refptr
       ".",
+      // profile generate symbols
+      "__profc_",
+      "__profd_",
+      "__profvp_",
   };
 
   excludeSymbolSuffixes = {

From 14c1b4017422cbf374086ea4c4fa74e16fb56779 Mon Sep 17 00:00:00 2001
From: Peiyuan Song <squallatf@gmail.com>
Date: Thu, 30 Jul 2020 23:37:17 +0300
Subject: [PATCH 0662/1035] [compiler-rt] [profile] fix profile generate for
 mingw x86_64

Differential Revision: https://reviews.llvm.org/D84757
---
 compiler-rt/lib/profile/CMakeLists.txt       | 6 +++---
 compiler-rt/lib/profile/InstrProfilingPort.h | 6 +++---
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/compiler-rt/lib/profile/CMakeLists.txt b/compiler-rt/lib/profile/CMakeLists.txt
index a968009f9ea90..3457bb6444de6 100644
--- a/compiler-rt/lib/profile/CMakeLists.txt
+++ b/compiler-rt/lib/profile/CMakeLists.txt
@@ -1,11 +1,11 @@
 
 CHECK_CXX_SOURCE_COMPILES("
-#ifdef _MSC_VER
-#include <Intrin.h> /* Workaround for PR19898. */
+#ifdef _WIN32
+#include <intrin.h> /* Workaround for PR19898. */
 #include <windows.h>
 #endif
 int main() {
-#ifdef _MSC_VER
+#ifdef _WIN32
         volatile LONG val = 1;
         MemoryBarrier();
         InterlockedCompareExchange(&val, 0, 1);
diff --git a/compiler-rt/lib/profile/InstrProfilingPort.h b/compiler-rt/lib/profile/InstrProfilingPort.h
index 20cf5d660c6ae..4493dd512ff0d 100644
--- a/compiler-rt/lib/profile/InstrProfilingPort.h
+++ b/compiler-rt/lib/profile/InstrProfilingPort.h
@@ -53,9 +53,9 @@
 #endif
 
 #if COMPILER_RT_HAS_ATOMICS == 1
-#ifdef _MSC_VER
+#ifdef _WIN32
 #include <windows.h>
-#if _MSC_VER < 1900
+#if defined(_MSC_VER) && _MSC_VER < 1900
 #define snprintf _snprintf
 #endif
 #if defined(_WIN64)
@@ -73,7 +73,7 @@
   (DomType *)InterlockedExchangeAdd((LONG volatile *)&PtrVar,                  \
                                     (LONG)sizeof(DomType) * PtrIncr)
 #endif
-#else /* !defined(_MSC_VER) */
+#else /* !defined(_WIN32) */
 #define COMPILER_RT_BOOL_CMPXCHG(Ptr, OldV, NewV)                              \
   __sync_bool_compare_and_swap(Ptr, OldV, NewV)
 #define COMPILER_RT_PTR_FETCH_ADD(DomType, PtrVar, PtrIncr)                    \

From 8c1a31d83313e4d43509177193f1de14cf897c05 Mon Sep 17 00:00:00 2001
From: Jonas Devlieghere <jonas@devlieghere.com>
Date: Thu, 30 Jul 2020 13:46:47 -0700
Subject: [PATCH 0663/1035] [lldb/Docs] Add lldb-arm-ubuntu to the list of bots

---
 lldb/docs/resources/bots.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lldb/docs/resources/bots.rst b/lldb/docs/resources/bots.rst
index efe8c7116b01e..6df7dade9814a 100644
--- a/lldb/docs/resources/bots.rst
+++ b/lldb/docs/resources/bots.rst
@@ -10,6 +10,7 @@ LLVM Buildbot is the place where volunteers provide build machines. Everyone can
 * `lldb-x64-windows-ninja <http://lab.llvm.org:8011/builders/lldb-x64-windows-ninja>`_
 * `lldb-x86_64-debian <http://lab.llvm.org:8011/builders/lldb-x86_64-debian>`_
 * `lldb-aarch64-ubuntu <http://lab.llvm.org:8011/builders/lldb-aarch64-ubuntu/>`_
+* `lldb-arm-ubuntu <http://lab.llvm.org:8011/builders/lldb-arm-ubuntu>`_
 * `lldb-x86_64-fedora <http://lab.llvm.org:8014/builders/lldb-x86_64-fedora>`_
 
 Documentation

From 94f8120cb9d368602df5aefa32211e001338e296 Mon Sep 17 00:00:00 2001
From: Nikita Popov <nikita.ppv@gmail.com>
Date: Thu, 30 Jul 2020 22:47:33 +0200
Subject: [PATCH 0664/1035] [ConstantRange] Support abs with poison flag

This just adds the ConstantRange support, including exhaustive
testing. It's not wired up to the IR intrinsic flag yet.
---
 llvm/include/llvm/IR/ConstantRange.h    |  5 +-
 llvm/lib/IR/ConstantRange.cpp           | 17 +++++--
 llvm/unittests/IR/ConstantRangeTest.cpp | 62 ++++++++++++++++---------
 3 files changed, 56 insertions(+), 28 deletions(-)

diff --git a/llvm/include/llvm/IR/ConstantRange.h b/llvm/include/llvm/IR/ConstantRange.h
index a027292ea43f4..318532b24e838 100644
--- a/llvm/include/llvm/IR/ConstantRange.h
+++ b/llvm/include/llvm/IR/ConstantRange.h
@@ -464,8 +464,9 @@ class LLVM_NODISCARD ConstantRange {
   ConstantRange inverse() const;
 
   /// Calculate absolute value range. If the original range contains signed
-  /// min, then the resulting range will also contain signed min.
-  ConstantRange abs() const;
+  /// min, then the resulting range will contain signed min if and only if
+  /// \p IntMinIsPoison is false.
+  ConstantRange abs(bool IntMinIsPoison = false) const;
 
   /// Represents whether an operation on the given constant range is known to
   /// always or never overflow.
diff --git a/llvm/lib/IR/ConstantRange.cpp b/llvm/lib/IR/ConstantRange.cpp
index d379140639dc9..6e0b5a0fc8600 100644
--- a/llvm/lib/IR/ConstantRange.cpp
+++ b/llvm/lib/IR/ConstantRange.cpp
@@ -1464,7 +1464,7 @@ ConstantRange ConstantRange::inverse() const {
   return ConstantRange(Upper, Lower);
 }
 
-ConstantRange ConstantRange::abs() const {
+ConstantRange ConstantRange::abs(bool IntMinIsPoison) const {
   if (isEmptySet())
     return getEmpty();
 
@@ -1476,12 +1476,23 @@ ConstantRange ConstantRange::abs() const {
     else
       Lo = APIntOps::umin(Lower, -Upper + 1);
 
-    // SignedMin is included in the result range.
-    return ConstantRange(Lo, APInt::getSignedMinValue(getBitWidth()) + 1);
+    // If SignedMin is not poison, then it is included in the result range.
+    if (IntMinIsPoison)
+      return ConstantRange(Lo, APInt::getSignedMinValue(getBitWidth()));
+    else
+      return ConstantRange(Lo, APInt::getSignedMinValue(getBitWidth()) + 1);
   }
 
   APInt SMin = getSignedMin(), SMax = getSignedMax();
 
+  // Skip SignedMin if it is poison.
+  if (IntMinIsPoison && SMin.isMinSignedValue()) {
+    // The range may become empty if it *only* contains SignedMin.
+    if (SMax.isMinSignedValue())
+      return getEmpty();
+    ++SMin;
+  }
+
   // All non-negative.
   if (SMin.isNonNegative())
     return *this;
diff --git a/llvm/unittests/IR/ConstantRangeTest.cpp b/llvm/unittests/IR/ConstantRangeTest.cpp
index 4d19dd59cae64..8bcc6efdc2653 100644
--- a/llvm/unittests/IR/ConstantRangeTest.cpp
+++ b/llvm/unittests/IR/ConstantRangeTest.cpp
@@ -59,6 +59,35 @@ static void ForeachNumInConstantRange(const ConstantRange &CR, Fn TestFn) {
   }
 }
 
+template<typename Fn1, typename Fn2>
+static void TestUnsignedUnaryOpExhaustive(
+    Fn1 RangeFn, Fn2 IntFn, bool SkipSignedIntMin = false) {
+  unsigned Bits = 4;
+  EnumerateConstantRanges(Bits, [&](const ConstantRange &CR) {
+    APInt Min = APInt::getMaxValue(Bits);
+    APInt Max = APInt::getMinValue(Bits);
+    ForeachNumInConstantRange(CR, [&](const APInt &N) {
+      if (SkipSignedIntMin && N.isMinSignedValue())
+        return;
+
+      APInt AbsN = IntFn(N);
+      if (AbsN.ult(Min))
+        Min = AbsN;
+      if (AbsN.ugt(Max))
+        Max = AbsN;
+    });
+
+    ConstantRange ResultCR = RangeFn(CR);
+    if (Min.ugt(Max)) {
+      EXPECT_TRUE(ResultCR.isEmptySet());
+      return;
+    }
+
+    ConstantRange Exact = ConstantRange::getNonEmpty(Min, Max + 1);
+    EXPECT_EQ(Exact, ResultCR);
+  });
+}
+
 template<typename Fn1, typename Fn2>
 static void TestUnsignedBinOpExhaustive(
     Fn1 RangeFn, Fn2 IntFn,
@@ -2270,29 +2299,16 @@ TEST_F(ConstantRangeTest, SShlSat) {
 }
 
 TEST_F(ConstantRangeTest, Abs) {
-  unsigned Bits = 4;
-  EnumerateConstantRanges(Bits, [&](const ConstantRange &CR) {
-    // We're working with unsigned integers here, because it makes the signed
-    // min case non-wrapping.
-    APInt Min = APInt::getMaxValue(Bits);
-    APInt Max = APInt::getMinValue(Bits);
-    ForeachNumInConstantRange(CR, [&](const APInt &N) {
-      APInt AbsN = N.abs();
-      if (AbsN.ult(Min))
-        Min = AbsN;
-      if (AbsN.ugt(Max))
-        Max = AbsN;
-    });
-
-    ConstantRange AbsCR = CR.abs();
-    if (Min.ugt(Max)) {
-      EXPECT_TRUE(AbsCR.isEmptySet());
-      return;
-    }
-
-    ConstantRange Exact = ConstantRange::getNonEmpty(Min, Max + 1);
-    EXPECT_EQ(Exact, AbsCR);
-  });
+  // We're working with unsigned integers here, because it makes the signed
+  // min case non-wrapping.
+  TestUnsignedUnaryOpExhaustive(
+      [](const ConstantRange &CR) { return CR.abs(); },
+      [](const APInt &N) { return N.abs(); });
+
+  TestUnsignedUnaryOpExhaustive(
+      [](const ConstantRange &CR) { return CR.abs(/*IntMinIsPoison=*/true); },
+      [](const APInt &N) { return N.abs(); },
+      /*SkipSignedIntMin=*/true);
 }
 
 TEST_F(ConstantRangeTest, castOps) {

From 02c1bba67009041a999220f370e81376883d7a65 Mon Sep 17 00:00:00 2001
From: Jonas Devlieghere <jonas@devlieghere.com>
Date: Thu, 30 Jul 2020 13:50:05 -0700
Subject: [PATCH 0665/1035] [lldb/Docs] Remove stale bot on GreenDragon and add
 reproducer one

 - Remove the link to the Python 3 job which no longer exists.
 - Add a link to the reproducer job.
---
 lldb/docs/resources/bots.rst | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/lldb/docs/resources/bots.rst b/lldb/docs/resources/bots.rst
index 6df7dade9814a..d9ddcde41abcc 100644
--- a/lldb/docs/resources/bots.rst
+++ b/lldb/docs/resources/bots.rst
@@ -13,14 +13,6 @@ LLVM Buildbot is the place where volunteers provide build machines. Everyone can
 * `lldb-arm-ubuntu <http://lab.llvm.org:8011/builders/lldb-arm-ubuntu>`_
 * `lldb-x86_64-fedora <http://lab.llvm.org:8014/builders/lldb-x86_64-fedora>`_
 
-Documentation
--------------
-
-The documentation bot validates that the website builds correctly with Sphinx.
-It does not generate the website itself, which happens on a separate server.
-
-* `lldb-sphinx-docs <http://lab.llvm.org:8011/builders/lldb-sphinx-docs>`_
-
 GreenDragon
 -----------
 
@@ -29,7 +21,14 @@ GreenDragon builds and tests LLDB on macOS. It has a `dedicated tab
 
 * `lldb-cmake <http://green.lab.llvm.org/green/view/LLDB/job/lldb-cmake/>`_
 * `lldb-cmake-matrix <http://green.lab.llvm.org/green/view/LLDB/job/lldb-cmake-matrix/>`_
-* `lldb-cmake-python3 <http://green.lab.llvm.org/green/view/LLDB/job/lldb-cmake-python3/>`_
+* `lldb-cmake-reproducers <http://green.lab.llvm.org/green/view/LLDB/job/lldb-cmake-reproducers/>`_
 * `lldb-cmake-standalone <http://green.lab.llvm.org/green/view/LLDB/job/lldb-cmake-standalone/>`_
 * `lldb-cmake-sanitized <http://green.lab.llvm.org/green/view/LLDB/job/lldb-cmake-sanitized/>`_
 
+Documentation
+-------------
+
+The documentation bot validates that the website builds correctly with Sphinx.
+It does not generate the website itself, which happens on a separate server.
+
+* `lldb-sphinx-docs <http://lab.llvm.org:8011/builders/lldb-sphinx-docs>`_

From afae6d97fa55984003321245ec6ceb5af9cc5856 Mon Sep 17 00:00:00 2001
From: Jon Roelofs <jonathan_roelofs@apple.com>
Date: Wed, 29 Jul 2020 13:14:17 -0600
Subject: [PATCH 0666/1035] [SelectionDAG] Fix lowering of vector geps

This fixes an assertion failure that was being triggered in
SelectionDAG::getZeroExtendInReg(), where it was trying to extend the <2xi32>
to i64 (which should have been <2xi64>).

Fixes: rdar://66016901

Differential Revision: https://reviews.llvm.org/D84884
---
 .../SelectionDAG/SelectionDAGBuilder.cpp      |  9 ++++++--
 llvm/test/CodeGen/AArch64/vector-gep.ll       | 21 +++++++++++++++++++
 2 files changed, 28 insertions(+), 2 deletions(-)
 create mode 100644 llvm/test/CodeGen/AArch64/vector-gep.ll

diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 9d2f64b94df16..d231e61a103b0 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -3753,8 +3753,6 @@ void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
   SDValue N = getValue(Op0);
   SDLoc dl = getCurSDLoc();
   auto &TLI = DAG.getTargetLoweringInfo();
-  MVT PtrTy = TLI.getPointerTy(DAG.getDataLayout(), AS);
-  MVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout(), AS);
 
   // Normalize Vector GEP - all scalar operands should be converted to the
   // splat vector.
@@ -3880,6 +3878,13 @@ void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
     }
   }
 
+  MVT PtrTy = TLI.getPointerTy(DAG.getDataLayout(), AS);
+  MVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout(), AS);
+  if (IsVectorGEP) {
+    PtrTy = MVT::getVectorVT(PtrTy, VectorElementCount);
+    PtrMemTy = MVT::getVectorVT(PtrMemTy, VectorElementCount);
+  }
+
   if (PtrMemTy != PtrTy && !cast<GEPOperator>(I).isInBounds())
     N = DAG.getPtrExtendInReg(N, dl, PtrMemTy);
 
diff --git a/llvm/test/CodeGen/AArch64/vector-gep.ll b/llvm/test/CodeGen/AArch64/vector-gep.ll
new file mode 100644
index 0000000000000..c69405524bcd3
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/vector-gep.ll
@@ -0,0 +1,21 @@
+; RUN: llc < %s -mtriple=arm64_32-apple-watchos2.0.0 --aarch64-neon-syntax=generic | FileCheck %s
+
+target datalayout = "e-m:o-p:32:32-i64:64-i128:128-n32:64-S128"
+target triple = "arm64_32-apple-watchos2.0.0"
+
+; CHECK-LABEL: lCPI0_0:
+; CHECK-NEXT:    .quad 36
+; CHECK-NEXT:    .quad 4804
+
+define <2 x i8*> @vector_gep(<2 x i8*> %0) {
+; CHECK-LABEL: vector_gep:
+; CHECK:         adrp x[[REG8:[123]?[0-9]]], lCPI0_0@PAGE
+; CHECK:         ldr q[[REG1:[0-9]+]], [x[[REG8]], lCPI0_0@PAGEOFF]
+; CHECK:         add v[[REG0:[0-9]+]].2d, v[[REG0]].2d, v[[REG1]].2d
+; CHECK:         movi v[[REG1]].2d, #0x000000ffffffff
+; CHECK:         and v[[REG0]].16b, v[[REG0]].16b, v[[REG1]].16b
+; CHECK:         ret
+entry:
+  %1 = getelementptr i8, <2 x i8*> %0, <2 x i32> <i32 36, i32 4804>
+  ret <2 x i8*> %1
+}

From 9ebeac678855a1bc2492a9630f0300518e2755dc Mon Sep 17 00:00:00 2001
From: Nikita Popov <nikita.ppv@gmail.com>
Date: Thu, 30 Jul 2020 22:54:53 +0200
Subject: [PATCH 0667/1035] [ConstantRange][CVP] Make use of abs poison flag

Pass the abs poison flag to the underlying ConstantRange
implementation, allowing CVP to simplify based on it.

Importantly, this recognizes that abs with poison flag is actually
non-negative...
---
 llvm/lib/IR/ConstantRange.cpp                            | 9 ++++++---
 .../Transforms/CorrelatedValuePropagation/minmaxabs.ll   | 3 +--
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/llvm/lib/IR/ConstantRange.cpp b/llvm/lib/IR/ConstantRange.cpp
index 6e0b5a0fc8600..396c39b5b3a85 100644
--- a/llvm/lib/IR/ConstantRange.cpp
+++ b/llvm/lib/IR/ConstantRange.cpp
@@ -872,9 +872,12 @@ ConstantRange ConstantRange::intrinsic(Intrinsic::ID IntrinsicID,
     return Ops[0].smin(Ops[1]);
   case Intrinsic::smax:
     return Ops[0].smax(Ops[1]);
-  case Intrinsic::abs:
-    // TODO: Make use of poison flag.
-    return Ops[0].abs();
+  case Intrinsic::abs: {
+    const APInt *IntMinIsPoison = Ops[1].getSingleElement();
+    assert(IntMinIsPoison && "Must be known (immarg)");
+    assert(IntMinIsPoison->getBitWidth() == 1 && "Must be boolean");
+    return Ops[0].abs(IntMinIsPoison->getBoolValue());
+  }
   default:
     assert(!isIntrinsicSupported(IntrinsicID) && "Shouldn't be supported");
     llvm_unreachable("Unsupported intrinsic");
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/minmaxabs.ll b/llvm/test/Transforms/CorrelatedValuePropagation/minmaxabs.ll
index ef5bf4d7c2a16..040bc8ae9016b 100644
--- a/llvm/test/Transforms/CorrelatedValuePropagation/minmaxabs.ll
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/minmaxabs.ll
@@ -141,8 +141,7 @@ define void @test_abs3(i32 %x) {
 ; CHECK-NEXT:    [[A:%.*]] = call i32 @llvm.abs.i32(i32 [[X:%.*]], i1 true)
 ; CHECK-NEXT:    br label [[SPLIT:%.*]]
 ; CHECK:       split:
-; CHECK-NEXT:    [[C1:%.*]] = icmp sge i32 [[A]], 0
-; CHECK-NEXT:    call void @use(i1 [[C1]])
+; CHECK-NEXT:    call void @use(i1 true)
 ; CHECK-NEXT:    [[C2:%.*]] = icmp sgt i32 [[A]], 0
 ; CHECK-NEXT:    call void @use(i1 [[C2]])
 ; CHECK-NEXT:    ret void

From 3587de22819869a2925994d8bd75fa1386464660 Mon Sep 17 00:00:00 2001
From: Jez Ng <jezng@fb.com>
Date: Thu, 30 Jul 2020 14:28:41 -0700
Subject: [PATCH 0668/1035] [lld-macho] Support __dso_handle for C++

The C++ ABI requires dylibs to pass a pointer to __cxa_atexit which does
e.g. cleanup of static global variables. The C++ spec says that the pointer
can point to any address in one of the dylib's segments, but in practice
ld64 seems to set it to point to the header, so that's what's implemented
here.

Reviewed By: #lld-macho, smeenai

Differential Revision: https://reviews.llvm.org/D83603
---
 lld/MachO/Driver.cpp                          |  2 +
 lld/MachO/SymbolTable.cpp                     | 13 +++++
 lld/MachO/SymbolTable.h                       |  3 ++
 lld/MachO/Symbols.cpp                         |  7 +++
 lld/MachO/Symbols.h                           | 49 ++++++++++++++-----
 lld/MachO/SyntheticSections.h                 |  1 +
 lld/MachO/Writer.cpp                          | 24 +++++----
 lld/test/MachO/dso-handle.s                   | 16 ++++++
 lld/test/MachO/invalid/dso-handle-duplicate.s | 20 ++++++++
 9 files changed, 110 insertions(+), 25 deletions(-)
 create mode 100644 lld/test/MachO/dso-handle.s
 create mode 100644 lld/test/MachO/invalid/dso-handle-duplicate.s

diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp
index d76e0115d10fc..a6d3eb69b8a0b 100644
--- a/lld/MachO/Driver.cpp
+++ b/lld/MachO/Driver.cpp
@@ -13,6 +13,7 @@
 #include "OutputSegment.h"
 #include "SymbolTable.h"
 #include "Symbols.h"
+#include "SyntheticSections.h"
 #include "Target.h"
 #include "Writer.h"
 
@@ -479,6 +480,7 @@ bool macho::link(llvm::ArrayRef<const char *> argsArr, bool canExitEarly,
   }
 
   createSyntheticSections();
+  symtab->addDSOHandle(in.header);
 
   // Initialize InputSections.
   for (InputFile *file : inputFiles) {
diff --git a/lld/MachO/SymbolTable.cpp b/lld/MachO/SymbolTable.cpp
index 061642d73f441..1a8a1d5ac0650 100644
--- a/lld/MachO/SymbolTable.cpp
+++ b/lld/MachO/SymbolTable.cpp
@@ -94,4 +94,17 @@ Symbol *SymbolTable::addLazy(StringRef name, ArchiveFile *file,
   return s;
 }
 
+Symbol *SymbolTable::addDSOHandle(const MachHeaderSection *header) {
+  Symbol *s;
+  bool wasInserted;
+  std::tie(s, wasInserted) = insert(DSOHandle::name);
+  if (!wasInserted) {
+    if (auto *defined = dyn_cast<Defined>(s))
+      error("found defined symbol from " + defined->isec->file->getName() +
+            " with illegal name " + DSOHandle::name);
+  }
+  replaceSymbol<DSOHandle>(s, header);
+  return s;
+}
+
 SymbolTable *macho::symtab;
diff --git a/lld/MachO/SymbolTable.h b/lld/MachO/SymbolTable.h
index 088b0e97c8406..822eb5b35dac5 100644
--- a/lld/MachO/SymbolTable.h
+++ b/lld/MachO/SymbolTable.h
@@ -20,6 +20,7 @@ namespace macho {
 class ArchiveFile;
 class DylibFile;
 class InputSection;
+class MachHeaderSection;
 class Symbol;
 
 /*
@@ -40,6 +41,8 @@ class SymbolTable {
   Symbol *addLazy(StringRef name, ArchiveFile *file,
                   const llvm::object::Archive::Symbol &sym);
 
+  Symbol *addDSOHandle(const MachHeaderSection *);
+
   ArrayRef<Symbol *> getSymbols() const { return symVector; }
   Symbol *find(StringRef name);
 
diff --git a/lld/MachO/Symbols.cpp b/lld/MachO/Symbols.cpp
index fbafa8a92a4fa..af5d9d25029f5 100644
--- a/lld/MachO/Symbols.cpp
+++ b/lld/MachO/Symbols.cpp
@@ -8,6 +8,7 @@
 
 #include "Symbols.h"
 #include "InputFiles.h"
+#include "SyntheticSections.h"
 
 using namespace llvm;
 using namespace lld;
@@ -21,3 +22,9 @@ std::string lld::toString(const Symbol &sym) {
     return *s;
   return std::string(sym.getName());
 }
+
+uint64_t DSOHandle::getVA() const { return header->addr; }
+
+uint64_t DSOHandle::getFileOffset() const { return header->fileOff; }
+
+constexpr StringRef DSOHandle::name;
diff --git a/lld/MachO/Symbols.h b/lld/MachO/Symbols.h
index 2dcccd03a8d02..1e0767a6a12b9 100644
--- a/lld/MachO/Symbols.h
+++ b/lld/MachO/Symbols.h
@@ -19,6 +19,7 @@ namespace lld {
 namespace macho {
 
 class InputSection;
+class MachHeaderSection;
 class DylibFile;
 class ArchiveFile;
 
@@ -37,6 +38,7 @@ class Symbol {
     UndefinedKind,
     DylibKind,
     LazyKind,
+    DSOHandleKind,
   };
 
   virtual ~Symbol() {}
@@ -45,9 +47,11 @@ class Symbol {
 
   StringRef getName() const { return {name.data, name.size}; }
 
-  uint64_t getVA() const;
+  virtual uint64_t getVA() const { return 0; }
 
-  uint64_t getFileOffset() const;
+  virtual uint64_t getFileOffset() const {
+    llvm_unreachable("attempt to get an offset from a non-defined symbol");
+  }
 
   virtual bool isWeakDef() const { llvm_unreachable("cannot be weak"); }
 
@@ -70,6 +74,12 @@ class Defined : public Symbol {
 
   static bool classof(const Symbol *s) { return s->kind() == DefinedKind; }
 
+  uint64_t getVA() const override { return isec->getVA() + value; }
+
+  uint64_t getFileOffset() const override {
+    return isec->getFileOffset() + value;
+  }
+
   InputSection *isec;
   uint32_t value;
 
@@ -115,17 +125,32 @@ class LazySymbol : public Symbol {
   const llvm::object::Archive::Symbol sym;
 };
 
-inline uint64_t Symbol::getVA() const {
-  if (auto *d = dyn_cast<Defined>(this))
-    return d->isec->getVA() + d->value;
-  return 0;
-}
+// The Itanium C++ ABI requires dylibs to pass a pointer to __cxa_atexit which
+// does e.g. cleanup of static global variables. The ABI document says that the
+// pointer can point to any address in one of the dylib's segments, but in
+// practice ld64 seems to set it to point to the header, so that's what's
+// implemented here.
+//
+// The ARM C++ ABI uses __dso_handle similarly, but I (int3) have not yet
+// tested this on an ARM platform.
+//
+// DSOHandle effectively functions like a Defined symbol, but it doesn't belong
+// to an InputSection.
+class DSOHandle : public Symbol {
+public:
+  DSOHandle(const MachHeaderSection *header)
+      : Symbol(DSOHandleKind, name), header(header) {}
 
-inline uint64_t Symbol::getFileOffset() const {
-  if (auto *d = dyn_cast<Defined>(this))
-    return d->isec->getFileOffset() + d->value;
-  llvm_unreachable("attempt to get an offset from an undefined symbol");
-}
+  const MachHeaderSection *header;
+
+  uint64_t getVA() const override;
+
+  uint64_t getFileOffset() const override;
+
+  static constexpr StringRef name = "___dso_handle";
+
+  static bool classof(const Symbol *s) { return s->kind() == DefinedKind; }
+};
 
 union SymbolUnion {
   alignas(Defined) char a[sizeof(Defined)];
diff --git a/lld/MachO/SyntheticSections.h b/lld/MachO/SyntheticSections.h
index a8fbf6c8a2657..f3052000be5ff 100644
--- a/lld/MachO/SyntheticSections.h
+++ b/lld/MachO/SyntheticSections.h
@@ -273,6 +273,7 @@ class SymtabSection : public SyntheticSection {
 };
 
 struct InStruct {
+  MachHeaderSection *header = nullptr;
   BindingSection *binding = nullptr;
   GotSection *got = nullptr;
   LazyPointerSection *lazyPointers = nullptr;
diff --git a/lld/MachO/Writer.cpp b/lld/MachO/Writer.cpp
index c9070e90f97e0..593e24f6b859e 100644
--- a/lld/MachO/Writer.cpp
+++ b/lld/MachO/Writer.cpp
@@ -53,7 +53,7 @@ class Writer {
   std::unique_ptr<FileOutputBuffer> &buffer;
   uint64_t addr = 0;
   uint64_t fileOff = 0;
-  MachHeaderSection *headerSection = nullptr;
+  MachHeaderSection *header = nullptr;
   LazyBindingSection *lazyBindingSection = nullptr;
   ExportSection *exportSection = nullptr;
   StringTableSection *stringTableSection = nullptr;
@@ -264,20 +264,18 @@ void Writer::scanRelocations() {
 }
 
 void Writer::createLoadCommands() {
-  headerSection->addLoadCommand(
+  in.header->addLoadCommand(
       make<LCDyldInfo>(in.binding, lazyBindingSection, exportSection));
-  headerSection->addLoadCommand(
-      make<LCSymtab>(symtabSection, stringTableSection));
-  headerSection->addLoadCommand(make<LCDysymtab>());
+  in.header->addLoadCommand(make<LCSymtab>(symtabSection, stringTableSection));
+  in.header->addLoadCommand(make<LCDysymtab>());
 
   switch (config->outputType) {
   case MH_EXECUTE:
-    headerSection->addLoadCommand(make<LCMain>());
-    headerSection->addLoadCommand(make<LCLoadDylinker>());
+    in.header->addLoadCommand(make<LCMain>());
+    in.header->addLoadCommand(make<LCLoadDylinker>());
     break;
   case MH_DYLIB:
-    headerSection->addLoadCommand(
-        make<LCDylib>(LC_ID_DYLIB, config->installName));
+    in.header->addLoadCommand(make<LCDylib>(LC_ID_DYLIB, config->installName));
     break;
   default:
     llvm_unreachable("unhandled output file type");
@@ -285,19 +283,19 @@ void Writer::createLoadCommands() {
 
   uint8_t segIndex = 0;
   for (OutputSegment *seg : outputSegments) {
-    headerSection->addLoadCommand(make<LCSegment>(seg->name, seg));
+    in.header->addLoadCommand(make<LCSegment>(seg->name, seg));
     seg->index = segIndex++;
   }
 
   uint64_t dylibOrdinal = 1;
   for (InputFile *file : inputFiles) {
     if (auto *dylibFile = dyn_cast<DylibFile>(file)) {
-      headerSection->addLoadCommand(
+      in.header->addLoadCommand(
           make<LCDylib>(LC_LOAD_DYLIB, dylibFile->dylibName));
       dylibFile->ordinal = dylibOrdinal++;
 
       if (dylibFile->reexport)
-        headerSection->addLoadCommand(
+        in.header->addLoadCommand(
             make<LCDylib>(LC_REEXPORT_DYLIB, dylibFile->dylibName));
     }
   }
@@ -406,7 +404,6 @@ static void sortSegmentsAndSections() {
 
 void Writer::createOutputSections() {
   // First, create hidden sections
-  headerSection = make<MachHeaderSection>();
   lazyBindingSection = make<LazyBindingSection>();
   stringTableSection = make<StringTableSection>();
   symtabSection = make<SymtabSection>(*stringTableSection);
@@ -539,6 +536,7 @@ void Writer::run() {
 void macho::writeResult() { Writer().run(); }
 
 void macho::createSyntheticSections() {
+  in.header = make<MachHeaderSection>();
   in.binding = make<BindingSection>();
   in.got = make<GotSection>();
   in.lazyPointers = make<LazyPointerSection>();
diff --git a/lld/test/MachO/dso-handle.s b/lld/test/MachO/dso-handle.s
new file mode 100644
index 0000000000000..f57ec7260fe06
--- /dev/null
+++ b/lld/test/MachO/dso-handle.s
@@ -0,0 +1,16 @@
+# REQUIRES: x86
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %s -o %t.o
+
+# RUN: lld -flavor darwinnew %t.o -o %t
+# RUN: llvm-objdump -d --no-show-raw-insn %t | FileCheck %s
+# CHECK: leaq {{.*}} # 100000000
+
+# RUN: lld -flavor darwinnew -dylib %t.o -o %t.dylib
+# RUN: llvm-objdump -d --no-show-raw-insn %t.dylib | FileCheck %s --check-prefix=DYLIB-CHECK
+# DYLIB-CHECK: leaq {{.*}} # 0
+
+.globl _main
+.text
+_main:
+  leaq ___dso_handle(%rip), %rdx
+  ret
diff --git a/lld/test/MachO/invalid/dso-handle-duplicate.s b/lld/test/MachO/invalid/dso-handle-duplicate.s
new file mode 100644
index 0000000000000..5991c6faff882
--- /dev/null
+++ b/lld/test/MachO/invalid/dso-handle-duplicate.s
@@ -0,0 +1,20 @@
+# REQUIRES: x86
+
+## If for some bizarre reason the input file defines its own ___dso_handle, we
+## should raise an error. At least, we've implemented this behavior if the
+## conflicting symbol is a global. A local symbol of the same name will still
+## take priority in our implementation, unlike in ld64. But that's a pretty
+## far-out edge case that should be safe to ignore.
+
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %s -o %t.o
+# RUN: not lld -flavor darwinnew -dylib %t.o -o %t.dylib 2>&1 | FileCheck %s -DFILE=%t.o
+# CHECK: error: found defined symbol from [[FILE]] with illegal name ___dso_handle
+
+.globl _main, ___dso_handle
+.text
+_main:
+  leaq ___dso_handle(%rip), %rdx
+  ret
+
+___dso_handle:
+  .space 1

From 22e6648a1834aa6680064eaf83e1e051a7248e17 Mon Sep 17 00:00:00 2001
From: Jez Ng <jezng@fb.com>
Date: Thu, 30 Jul 2020 14:28:45 -0700
Subject: [PATCH 0669/1035] [lld-macho] Implement -headerpad

Tools like `install_name_tool` and `codesign` may modify the Mach-O
header and increase its size. The linker has to provide padding to make this
possible. This diff does that, plus sets its default value to 32 bytes (which
is what ld64 does).

Unlike ld64, however, we lay out our sections *exactly* `-headerpad` bytes from
the header, whereas ld64 just treats the padding requirement as a lower bound.
ld64 actually starts laying out the non-header sections in the __TEXT segment
from the end of the (page-aligned) segment rather than the front, so its
binaries typically have more than `-headerpad` bytes of actual padding.
We should consider implementing the same alignment behavior.

Reviewed By: #lld-macho, compnerd

Differential Revision: https://reviews.llvm.org/D84714
---
 lld/Common/Args.cpp             | 16 +++++++++++++---
 lld/MachO/Config.h              |  1 +
 lld/MachO/Driver.cpp            |  2 ++
 lld/MachO/Options.td            |  1 -
 lld/MachO/SyntheticSections.cpp |  2 +-
 lld/include/lld/Common/Args.h   |  2 ++
 lld/test/MachO/headerpad.s      | 34 +++++++++++++++++++++++++++++++++
 lld/test/MachO/local-got.s      |  4 ++--
 lld/test/MachO/relocations.s    |  2 +-
 9 files changed, 56 insertions(+), 8 deletions(-)
 create mode 100644 lld/test/MachO/headerpad.s

diff --git a/lld/Common/Args.cpp b/lld/Common/Args.cpp
index 4ea3a435c7ae0..507830f9da346 100644
--- a/lld/Common/Args.cpp
+++ b/lld/Common/Args.cpp
@@ -26,14 +26,14 @@ CodeGenOpt::Level lld::args::getCGOptLevel(int optLevelLTO) {
   return CodeGenOpt::Default;
 }
 
-int64_t lld::args::getInteger(opt::InputArgList &args, unsigned key,
-                              int64_t Default) {
+static int64_t getInteger(opt::InputArgList &args, unsigned key,
+                          int64_t Default, unsigned base) {
   auto *a = args.getLastArg(key);
   if (!a)
     return Default;
 
   int64_t v;
-  if (to_integer(a->getValue(), v, 10))
+  if (to_integer(a->getValue(), v, base))
     return v;
 
   StringRef spelling = args.getArgString(a->getIndex());
@@ -41,6 +41,16 @@ int64_t lld::args::getInteger(opt::InputArgList &args, unsigned key,
   return 0;
 }
 
+int64_t lld::args::getInteger(opt::InputArgList &args, unsigned key,
+                              int64_t Default) {
+  return ::getInteger(args, key, Default, 10);
+}
+
+int64_t lld::args::getHex(opt::InputArgList &args, unsigned key,
+                          int64_t Default) {
+  return ::getInteger(args, key, Default, 16);
+}
+
 std::vector<StringRef> lld::args::getStrings(opt::InputArgList &args, int id) {
   std::vector<StringRef> v;
   for (auto *arg : args.filtered(id))
diff --git a/lld/MachO/Config.h b/lld/MachO/Config.h
index c66991b581fc0..f65ff7c5c1047 100644
--- a/lld/MachO/Config.h
+++ b/lld/MachO/Config.h
@@ -25,6 +25,7 @@ struct SymbolPriorityEntry;
 struct Configuration {
   Symbol *entry;
   bool hasReexports = false;
+  uint32_t headerPad;
   llvm::StringRef installName;
   llvm::StringRef outputFile;
   llvm::MachO::Architecture arch;
diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp
index a6d3eb69b8a0b..afb914b55a16a 100644
--- a/lld/MachO/Driver.cpp
+++ b/lld/MachO/Driver.cpp
@@ -396,6 +396,7 @@ bool macho::link(llvm::ArrayRef<const char *> argsArr, bool canExitEarly,
   config->outputFile = args.getLastArgValue(OPT_o, "a.out");
   config->installName =
       args.getLastArgValue(OPT_install_name, config->outputFile);
+  config->headerPad = args::getHex(args, OPT_headerpad, 32);
   getLibrarySearchPaths(config->librarySearchPaths, args);
   getFrameworkSearchPaths(config->frameworkSearchPaths, args);
   config->outputType = args.hasArg(OPT_dylib) ? MH_DYLIB : MH_EXECUTE;
@@ -450,6 +451,7 @@ bool macho::link(llvm::ArrayRef<const char *> argsArr, bool canExitEarly,
     case OPT_e:
     case OPT_F:
     case OPT_L:
+    case OPT_headerpad:
     case OPT_install_name:
     case OPT_Z:
     case OPT_arch:
diff --git a/lld/MachO/Options.td b/lld/MachO/Options.td
index 7d81ca7c14270..c27ca0d0195f9 100644
--- a/lld/MachO/Options.td
+++ b/lld/MachO/Options.td
@@ -693,7 +693,6 @@ def umbrella : Separate<["-"], "umbrella">,
 def headerpad : Separate<["-"], "headerpad">,
      MetaVarName<"<size>">,
      HelpText<"Allocate hex <size> extra space for future expansion of the load commands via install_name_tool">,
-     Flags<[HelpHidden]>,
      Group<grp_rare>;
 def headerpad_max_install_names : Flag<["-"], "headerpad_max_install_names">,
      HelpText<"Allocate extra space so all load-command paths can expand to MAXPATHLEN via install_name_tool">,
diff --git a/lld/MachO/SyntheticSections.cpp b/lld/MachO/SyntheticSections.cpp
index a2d8bf42e9edf..94df6c50e0107 100644
--- a/lld/MachO/SyntheticSections.cpp
+++ b/lld/MachO/SyntheticSections.cpp
@@ -47,7 +47,7 @@ void MachHeaderSection::addLoadCommand(LoadCommand *lc) {
 }
 
 uint64_t MachHeaderSection::getSize() const {
-  return sizeof(MachO::mach_header_64) + sizeOfCmds;
+  return sizeof(MachO::mach_header_64) + sizeOfCmds + config->headerPad;
 }
 
 void MachHeaderSection::writeTo(uint8_t *buf) const {
diff --git a/lld/include/lld/Common/Args.h b/lld/include/lld/Common/Args.h
index b3c8686e57e22..48f7b4079cd97 100644
--- a/lld/include/lld/Common/Args.h
+++ b/lld/include/lld/Common/Args.h
@@ -28,6 +28,8 @@ llvm::CodeGenOpt::Level getCGOptLevel(int optLevelLTO);
 int64_t getInteger(llvm::opt::InputArgList &args, unsigned key,
                    int64_t Default);
 
+int64_t getHex(llvm::opt::InputArgList &args, unsigned key, int64_t Default);
+
 std::vector<StringRef> getStrings(llvm::opt::InputArgList &args, int id);
 
 uint64_t getZOptionValue(llvm::opt::InputArgList &args, int id, StringRef key,
diff --git a/lld/test/MachO/headerpad.s b/lld/test/MachO/headerpad.s
new file mode 100644
index 0000000000000..963ccf9ce4e70
--- /dev/null
+++ b/lld/test/MachO/headerpad.s
@@ -0,0 +1,34 @@
+# REQUIRES: x86
+
+## NOTE: Here we check that the first non-header section -- __text -- appears
+## *exactly* `-headerpad` bytes from the end of the header. ld64 actually
+## starts laying out the non-header sections in the __TEXT segment from the end
+## of the (page-aligned) segment rather than the front, so its binaries
+## typically have more than `-headerpad` bytes of actual padding. `-headerpad`
+## just enforces a lower bound. We should consider implementing the same
+## alignment behavior.
+
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %s -o %t.o
+# RUN: lld -flavor darwinnew -o %t %t.o -headerpad 0
+# RUN: llvm-objdump --macho --all-headers %t | FileCheck %s --check-prefix=PAD0
+# PAD0:      magic        cputype  cpusubtype  caps    filetype ncmds sizeofcmds
+# PAD0-NEXT: MH_MAGIC_64  X86_64   ALL         LIB64   EXECUTE  8     [[#%u, CMDSIZE:]] {{.*}}
+# PAD0:      sectname __text
+# PAD0-NEXT: segname __TEXT
+# PAD0-NEXT: addr
+# PAD0-NEXT: size
+# PAD0-NEXT: offset [[#%u, CMDSIZE + 32]]
+
+# RUN: lld -flavor darwinnew -o %t %t.o -headerpad 11
+# RUN: llvm-objdump --macho --all-headers %t | FileCheck %s --check-prefix=PAD11
+# PAD11:      magic        cputype  cpusubtype  caps    filetype ncmds sizeofcmds
+# PAD11-NEXT: MH_MAGIC_64  X86_64   ALL         LIB64   EXECUTE  8     [[#%u, CMDSIZE:]] {{.*}}
+# PAD11:      sectname __text
+# PAD11-NEXT: segname __TEXT
+# PAD11-NEXT: addr
+# PAD11-NEXT: size
+# PAD11-NEXT: offset [[#%u, CMDSIZE + 32 + 0x11]]
+
+.globl _main
+_main:
+  ret
diff --git a/lld/test/MachO/local-got.s b/lld/test/MachO/local-got.s
index 6099a6bf18c7b..baf4b3336b7ab 100644
--- a/lld/test/MachO/local-got.s
+++ b/lld/test/MachO/local-got.s
@@ -12,12 +12,12 @@
 ## address offset and the contents at that address very similarly, so am using
 ## --match-full-lines to make sure we match on the right thing.
 # CHECK:      Contents of section __cstring:
-# CHECK-NEXT: 1000003cc {{.*}}
+# CHECK-NEXT: 1000003ec {{.*}}
 
 ## 1st 8 bytes refer to the start of __cstring + 0xe, 2nd 8 bytes refer to the
 ## start of __cstring
 # CHECK:      Contents of section __got:
-# CHECK-NEXT: [[#%X,ADDR:]]  da030000 01000000 cc030000 01000000 {{.*}}
+# CHECK-NEXT: [[#%X,ADDR:]]  fa030000 01000000 ec030000 01000000 {{.*}}
 # CHECK-NEXT: [[#ADDR + 16]] 00000000 00000000 {{.*}}
 
 ## Check that a non-locally-defined symbol is still bound at the correct offset:
diff --git a/lld/test/MachO/relocations.s b/lld/test/MachO/relocations.s
index edaa308935608..93be4998db68f 100644
--- a/lld/test/MachO/relocations.s
+++ b/lld/test/MachO/relocations.s
@@ -21,7 +21,7 @@
 
 # RUN: llvm-objdump --section=__const --full-contents -d %t | FileCheck %s --check-prefix=NONPCREL
 # NONPCREL:      Contents of section __const:
-# NONPCREL-NEXT: 100001000 b0030000 01000000 b0030000 01000000
+# NONPCREL-NEXT: 100001000 d0030000 01000000 d0030000 01000000
 
 .section __TEXT,__text
 .globl _main, _f

From 98210796e108b59bad56fed9df98e920359afc6b Mon Sep 17 00:00:00 2001
From: Jez Ng <jezng@fb.com>
Date: Thu, 30 Jul 2020 14:29:14 -0700
Subject: [PATCH 0670/1035] [lld-macho] Make __LINKEDIT sections contiguous

codesign (or more specifically libstuff) checks that each section in
__LINKEDIT ends where the next one starts -- no gaps are permitted. This
diff achieves it by aligning every section's start and end points to
WordSize.

Remarks: ld64 appears to satisfy the constraint by adding padding bytes
when generating the __LINKEDIT data, e.g. by emitting BIND_OPCODE_DONE
(which is a 0x0 byte) repeatedly. I think the approach this diff takes
is a bit more elegant, but I'm not sure if it's too restrictive. In
particular, it assumes padding always uses the zero byte. But we can
revisit this later.

Reviewed By: #lld-macho, compnerd

Differential Revision: https://reviews.llvm.org/D84718
---
 lld/MachO/SyntheticSections.cpp      | 14 ++++------
 lld/MachO/SyntheticSections.h        | 37 +++++++++++++++++++------
 lld/MachO/Writer.cpp                 |  3 +-
 lld/test/MachO/linkedit-contiguity.s | 41 ++++++++++++++++++++++++++++
 4 files changed, 77 insertions(+), 18 deletions(-)
 create mode 100644 lld/test/MachO/linkedit-contiguity.s

diff --git a/lld/MachO/SyntheticSections.cpp b/lld/MachO/SyntheticSections.cpp
index 94df6c50e0107..ac772da0129f8 100644
--- a/lld/MachO/SyntheticSections.cpp
+++ b/lld/MachO/SyntheticSections.cpp
@@ -94,7 +94,7 @@ void GotSection::writeTo(uint8_t *buf) const {
 }
 
 BindingSection::BindingSection()
-    : SyntheticSection(segment_names::linkEdit, section_names::binding) {}
+    : LinkEditSection(segment_names::linkEdit, section_names::binding) {}
 
 bool BindingSection::isNeeded() const {
   return bindings.size() != 0 || in.got->isNeeded();
@@ -301,7 +301,7 @@ void LazyPointerSection::writeTo(uint8_t *buf) const {
 }
 
 LazyBindingSection::LazyBindingSection()
-    : SyntheticSection(segment_names::linkEdit, section_names::lazyBinding) {}
+    : LinkEditSection(segment_names::linkEdit, section_names::lazyBinding) {}
 
 bool LazyBindingSection::isNeeded() const { return in.stubs->isNeeded(); }
 
@@ -344,7 +344,7 @@ uint32_t LazyBindingSection::encode(const DylibSymbol &sym) {
 }
 
 ExportSection::ExportSection()
-    : SyntheticSection(segment_names::linkEdit, section_names::export_) {}
+    : LinkEditSection(segment_names::linkEdit, section_names::export_) {}
 
 void ExportSection::finalizeContents() {
   // TODO: We should check symbol visibility.
@@ -358,11 +358,7 @@ void ExportSection::writeTo(uint8_t *buf) const { trieBuilder.writeTo(buf); }
 
 SymtabSection::SymtabSection(StringTableSection &stringTableSection)
     : SyntheticSection(segment_names::linkEdit, section_names::symbolTable),
-      stringTableSection(stringTableSection) {
-  // TODO: When we introduce the SyntheticSections superclass, we should make
-  // all synthetic sections aligned to WordSize by default.
-  align = WordSize;
-}
+      stringTableSection(stringTableSection) {}
 
 uint64_t SymtabSection::getSize() const {
   return symbols.size() * sizeof(structs::nlist_64);
@@ -392,7 +388,7 @@ void SymtabSection::writeTo(uint8_t *buf) const {
 }
 
 StringTableSection::StringTableSection()
-    : SyntheticSection(segment_names::linkEdit, section_names::stringTable) {}
+    : LinkEditSection(segment_names::linkEdit, section_names::stringTable) {}
 
 uint32_t StringTableSection::addString(StringRef str) {
   uint32_t strx = size;
diff --git a/lld/MachO/SyntheticSections.h b/lld/MachO/SyntheticSections.h
index f3052000be5ff..db2ffbe57d442 100644
--- a/lld/MachO/SyntheticSections.h
+++ b/lld/MachO/SyntheticSections.h
@@ -49,6 +49,27 @@ class SyntheticSection : public OutputSection {
   const StringRef segname;
 };
 
+// All sections in __LINKEDIT should inherit from this.
+class LinkEditSection : public SyntheticSection {
+public:
+  LinkEditSection(const char *segname, const char *name)
+      : SyntheticSection(segname, name) {
+    align = WordSize;
+  }
+
+  virtual uint64_t getRawSize() const = 0;
+
+  // codesign (or more specifically libstuff) checks that each section in
+  // __LINKEDIT ends where the next one starts -- no gaps are permitted. We
+  // therefore align every section's start and end points to WordSize.
+  //
+  // NOTE: This assumes that the extra bytes required for alignment can be
+  // zero-valued bytes.
+  uint64_t getSize() const override final {
+    return llvm::alignTo(getRawSize(), WordSize);
+  }
+};
+
 // The header of the Mach-O file, which must have a file offset of zero.
 class MachHeaderSection : public SyntheticSection {
 public:
@@ -105,11 +126,11 @@ struct BindingEntry {
 };
 
 // Stores bind opcodes for telling dyld which symbols to load non-lazily.
-class BindingSection : public SyntheticSection {
+class BindingSection : public LinkEditSection {
 public:
   BindingSection();
   void finalizeContents();
-  uint64_t getSize() const override { return contents.size(); }
+  uint64_t getRawSize() const override { return contents.size(); }
   // Like other sections in __LINKEDIT, the binding section is special: its
   // offsets are recorded in the LC_DYLD_INFO_ONLY load command, instead of in
   // section headers.
@@ -194,11 +215,11 @@ class LazyPointerSection : public SyntheticSection {
   void writeTo(uint8_t *buf) const override;
 };
 
-class LazyBindingSection : public SyntheticSection {
+class LazyBindingSection : public LinkEditSection {
 public:
   LazyBindingSection();
   void finalizeContents();
-  uint64_t getSize() const override { return contents.size(); }
+  uint64_t getRawSize() const override { return contents.size(); }
   uint32_t encode(const DylibSymbol &);
   // Like other sections in __LINKEDIT, the lazy binding section is special: its
   // offsets are recorded in the LC_DYLD_INFO_ONLY load command, instead of in
@@ -213,11 +234,11 @@ class LazyBindingSection : public SyntheticSection {
 };
 
 // Stores a trie that describes the set of exported symbols.
-class ExportSection : public SyntheticSection {
+class ExportSection : public LinkEditSection {
 public:
   ExportSection();
   void finalizeContents();
-  uint64_t getSize() const override { return size; }
+  uint64_t getRawSize() const override { return size; }
   // Like other sections in __LINKEDIT, the export section is special: its
   // offsets are recorded in the LC_DYLD_INFO_ONLY load command, instead of in
   // section headers.
@@ -230,12 +251,12 @@ class ExportSection : public SyntheticSection {
 };
 
 // Stores the strings referenced by the symbol table.
-class StringTableSection : public SyntheticSection {
+class StringTableSection : public LinkEditSection {
 public:
   StringTableSection();
   // Returns the start offset of the added string.
   uint32_t addString(StringRef);
-  uint64_t getSize() const override { return size; }
+  uint64_t getRawSize() const override { return size; }
   // Like other sections in __LINKEDIT, the string table section is special: its
   // offsets are recorded in the LC_SYMTAB load command, instead of in section
   // headers.
diff --git a/lld/MachO/Writer.cpp b/lld/MachO/Writer.cpp
index 593e24f6b859e..886ca482136b7 100644
--- a/lld/MachO/Writer.cpp
+++ b/lld/MachO/Writer.cpp
@@ -353,7 +353,8 @@ static int sectionOrder(OutputSection *osec) {
       return -1;
   } else if (segname == segment_names::linkEdit) {
     return StringSwitch<int>(osec->name)
-        .Case(section_names::binding, -4)
+        .Case(section_names::binding, -5)
+        .Case(section_names::lazyBinding, -4)
         .Case(section_names::export_, -3)
         .Case(section_names::symbolTable, -2)
         .Case(section_names::stringTable, -1)
diff --git a/lld/test/MachO/linkedit-contiguity.s b/lld/test/MachO/linkedit-contiguity.s
new file mode 100644
index 0000000000000..f815e5555a859
--- /dev/null
+++ b/lld/test/MachO/linkedit-contiguity.s
@@ -0,0 +1,41 @@
+# REQUIRES: x86
+# RUN: mkdir -p %t
+
+## codesign requires that each setion in __LINKEDIT ends where the next one
+## starts. This test enforces that invariant.
+## TODO: Test other __LINKEDIT sections here as support for them gets added.
+## Examples of such sections include the data for LC_CODE_SIGNATURE and
+## LC_DATA_IN_CODE.
+
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %p/Inputs/libhello.s \
+# RUN:   -o %t/libhello.o
+# RUN: lld -flavor darwinnew -dylib -L%S/Inputs/MacOSX.sdk/usr/lib \
+# RUN:   -install_name @executable_path/libhello.dylib %t/libhello.o \
+# RUN:   -o %t/libhello.dylib
+
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %s -o %t/test.o
+# RUN: lld -flavor darwinnew -o %t/test \
+# RUN:   -L%S/Inputs/MacOSX.sdk/usr/lib -L%t -lhello %t/test.o -lSystem
+
+# RUN: llvm-objdump --macho --all-headers %t/test | FileCheck %s
+
+# CHECK:      cmd LC_DYLD_INFO_ONLY
+# CHECK-NEXT: cmdsize 48
+# CHECK-NEXT: rebase_off 0
+# CHECK-NEXT: rebase_size 0
+# CHECK-NEXT: bind_off [[#BIND_OFF:]]
+# CHECK-NEXT: bind_size [[#BIND_SIZE:]]
+# CHECK-NEXT: weak_bind_off 0
+# CHECK-NEXT: weak_bind_size 0
+# CHECK-NEXT: lazy_bind_off [[#LAZY_OFF: BIND_OFF + BIND_SIZE]]
+# CHECK-NEXT: lazy_bind_size [[#LAZY_SIZE:]]
+# CHECK-NEXT: export_off [[#EXPORT_OFF: LAZY_OFF + LAZY_SIZE]]
+# CHECK-NEXT: export_size [[#]]
+
+.text
+.globl _main
+_main:
+  sub $8, %rsp # 16-byte-align the stack; dyld checks for this
+  callq _print_hello
+  add $8, %rsp
+  ret

From 243376cdc7b719d443f42c8c4667e5d96af53dcc Mon Sep 17 00:00:00 2001
From: Changpeng Fang <changpeng.fang@gmail.com>
Date: Thu, 30 Jul 2020 14:37:06 -0700
Subject: [PATCH 0671/1035] AMDGPU: Put inexpensive ops first in
 AMDGPUAnnotateUniformValues::visitLoadInst

Summary:
  This is in response to the review of https://reviews.llvm.org/D84873:
The expensive check should be reordered last

Reviewers:
  arsenm

Differential Revision:
  https://reviews.llvm.org/D84890
---
 .../AMDGPU/AMDGPUAnnotateUniformValues.cpp       | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAnnotateUniformValues.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAnnotateUniformValues.cpp
index b09e92c07f9ba..45f515c5115ed 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAnnotateUniformValues.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAnnotateUniformValues.cpp
@@ -131,10 +131,20 @@ void AMDGPUAnnotateUniformValues::visitLoadInst(LoadInst &I) {
   // We're tracking up to the Function boundaries, and cannot go beyond because
   // of FunctionPass restrictions. We can ensure that is memory not clobbered
   // for memory operations that are live in to entry points only.
-  bool NotClobbered = isEntryFunc && !isClobberedInFunction(&I);
   Instruction *PtrI = dyn_cast<Instruction>(Ptr);
-  if (!PtrI && NotClobbered && isGlobalLoad(I)) {
-    if (isa<Argument>(Ptr) || isa<GlobalValue>(Ptr)) {
+
+  if (!isEntryFunc) {
+    if (PtrI)
+      setUniformMetadata(PtrI);
+    return;
+  }
+
+  bool NotClobbered = false;
+  if (PtrI)
+    NotClobbered = !isClobberedInFunction(&I);
+  else if (isa<Argument>(Ptr) || isa<GlobalValue>(Ptr)) {
+    if (isGlobalLoad(I) && !isClobberedInFunction(&I)) {
+      NotClobbered = true;
       // Lookup for the existing GEP
       if (noClobberClones.count(Ptr)) {
         PtrI = noClobberClones[Ptr];

From c89e46e767e25c426628f6365c577a3248634f1f Mon Sep 17 00:00:00 2001
From: Jez Ng <jezng@fb.com>
Date: Thu, 30 Jul 2020 14:38:58 -0700
Subject: [PATCH 0672/1035] [lld-macho] Add comment for literal argument

---
 lld/MachO/Driver.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp
index afb914b55a16a..6b25cd55ccfc7 100644
--- a/lld/MachO/Driver.cpp
+++ b/lld/MachO/Driver.cpp
@@ -396,7 +396,7 @@ bool macho::link(llvm::ArrayRef<const char *> argsArr, bool canExitEarly,
   config->outputFile = args.getLastArgValue(OPT_o, "a.out");
   config->installName =
       args.getLastArgValue(OPT_install_name, config->outputFile);
-  config->headerPad = args::getHex(args, OPT_headerpad, 32);
+  config->headerPad = args::getHex(args, OPT_headerpad, /*Default=*/32);
   getLibrarySearchPaths(config->librarySearchPaths, args);
   getFrameworkSearchPaths(config->frameworkSearchPaths, args);
   config->outputType = args.hasArg(OPT_dylib) ? MH_DYLIB : MH_EXECUTE;

From 1e7f026c3b59bea22491819f86d8388d0c3e0c07 Mon Sep 17 00:00:00 2001
From: Richard Smith <richard@metafoo.co.uk>
Date: Thu, 30 Jul 2020 14:17:26 -0700
Subject: [PATCH 0673/1035] PR46908: Emit undef destroying_delete_t as an
 aggregate RValue.

We previously used a non-aggregate RValue to represent the passed value,
which violated the assumptions of call arg lowering in some cases, in
particular on 32-bit Windows, where we'd end up producing an FCA store
with TBAA metadata, that the IR verifier would reject.
---
 clang/lib/CodeGen/CGExprCXX.cpp               | 14 +++--
 .../CodeGenCXX/cxx2a-destroying-delete.cpp    | 53 +++++++++++++------
 2 files changed, 47 insertions(+), 20 deletions(-)

diff --git a/clang/lib/CodeGen/CGExprCXX.cpp b/clang/lib/CodeGen/CGExprCXX.cpp
index d59aa6ce0fb94..d0e0c7d6c0603 100644
--- a/clang/lib/CodeGen/CGExprCXX.cpp
+++ b/clang/lib/CodeGen/CGExprCXX.cpp
@@ -1788,11 +1788,14 @@ void CodeGenFunction::EmitDeleteCall(const FunctionDecl *DeleteFD,
   DeleteArgs.add(RValue::get(DeletePtr), ArgTy);
 
   // Pass the std::destroying_delete tag if present.
+  llvm::AllocaInst *DestroyingDeleteTag = nullptr;
   if (Params.DestroyingDelete) {
     QualType DDTag = *ParamTypeIt++;
-    // Just pass an 'undef'. We expect the tag type to be an empty struct.
-    auto *V = llvm::UndefValue::get(getTypes().ConvertType(DDTag));
-    DeleteArgs.add(RValue::get(V), DDTag);
+    llvm::Type *Ty = getTypes().ConvertType(DDTag);
+    CharUnits Align = CGM.getNaturalTypeAlignment(DDTag);
+    DestroyingDeleteTag = CreateTempAlloca(Ty, "destroying.delete.tag");
+    DestroyingDeleteTag->setAlignment(Align.getAsAlign());
+    DeleteArgs.add(RValue::getAggregate(Address(DestroyingDeleteTag, Align)), DDTag);
   }
 
   // Pass the size if the delete function has a size_t parameter.
@@ -1829,6 +1832,11 @@ void CodeGenFunction::EmitDeleteCall(const FunctionDecl *DeleteFD,
 
   // Emit the call to delete.
   EmitNewDeleteCall(*this, DeleteFD, DeleteFTy, DeleteArgs);
+
+  // If call argument lowering didn't use the destroying_delete_t alloca,
+  // remove it again.
+  if (DestroyingDeleteTag && DestroyingDeleteTag->use_empty())
+    DestroyingDeleteTag->eraseFromParent();
 }
 
 namespace {
diff --git a/clang/test/CodeGenCXX/cxx2a-destroying-delete.cpp b/clang/test/CodeGenCXX/cxx2a-destroying-delete.cpp
index 60c5970cb94f4..e689afa620924 100644
--- a/clang/test/CodeGenCXX/cxx2a-destroying-delete.cpp
+++ b/clang/test/CodeGenCXX/cxx2a-destroying-delete.cpp
@@ -1,5 +1,11 @@
-// RUN: %clang_cc1 -std=c++2a -emit-llvm %s -triple x86_64-linux-gnu -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-ITANIUM
-// RUN: %clang_cc1 -std=c++2a -emit-llvm %s -triple x86_64-windows -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-MSABI
+// RUN: %clang_cc1 -std=c++2a -emit-llvm %s -triple x86_64-linux-gnu -o - | FileCheck %s --check-prefixes=CHECK,CHECK-ITANIUM,CHECK-64BIT
+// RUN: %clang_cc1 -std=c++2a -emit-llvm %s -triple x86_64-windows -o - | FileCheck %s --check-prefixes=CHECK,CHECK-MSABI,CHECK-MSABI64,CHECK-64BIT
+// RUN: %clang_cc1 -std=c++2a -emit-llvm %s -triple i386-windows -o - | FileCheck %s --check-prefixes=CHECK,CHECK-MSABI,CHECK-MSABI32,CHECK-32BIT
+
+// PR46908: ensure the IR passes the verifier with optimizations enabled.
+// RUN: %clang_cc1 -std=c++2a -emit-llvm-only %s -triple x86_64-linux-gnu -O2
+// RUN: %clang_cc1 -std=c++2a -emit-llvm-only %s -triple x86_64-windows -O2
+// RUN: %clang_cc1 -std=c++2a -emit-llvm-only %s -triple i386-windows -O2
 
 namespace std {
   using size_t = decltype(sizeof(0));
@@ -21,7 +27,8 @@ void delete_A(A *a) { delete a; }
 // Ensure that we call the destroying delete and not the destructor.
 // CHECK-NOT: call
 // CHECK-ITANIUM: call void @_ZN1AdlEPS_St19destroying_delete_t(%{{.*}}* %[[a]])
-// CHECK-MSABI: call void @"??3A@@SAXPEAU0@Udestroying_delete_t@std@@@Z"(%{{.*}}* %[[a]], i8
+// CHECK-MSABI64: call void @"??3A@@SAXPEAU0@Udestroying_delete_t@std@@@Z"(%{{.*}}* %[[a]], i8
+// CHECK-MSABI32: call void @"??3A@@SAXPAU0@Udestroying_delete_t@std@@@Z"(%{{.*}}* %[[a]], %{{.*}}* byval(%{{.*}}) align 4 %{{.*}})
 // CHECK-NOT: call
 // CHECK: }
 
@@ -39,7 +46,7 @@ void delete_B(B *b) { delete b; }
 // CHECK-NOT: call
 // CHECK: %[[VTABLE:.*]] = load
 // CHECK: %[[DTOR:.*]] = load
-// CHECK: call {{void|i8\*}} %[[DTOR]](%{{.*}}* %[[b]]
+// CHECK: call {{void|i8\*|x86_thiscallcc i8\*}} %[[DTOR]](%{{.*}}* %[[b]]
 // CHECK-MSABI-SAME: , i32 1)
 // CHECK-NOT: call
 // CHECK: }
@@ -56,7 +63,8 @@ void delete_C(C *c) { delete c; }
 // CHECK: icmp eq %{{.*}} %[[c]], null
 // CHECK: br i1
 //
-// CHECK: %[[base:.*]] = getelementptr {{.*}}, i64 8
+// CHECK-64BIT: %[[base:.*]] = getelementptr {{.*}}, i64 8
+// CHECK-32BIT: %[[base:.*]] = getelementptr {{.*}}, i32 4
 // CHECK: %[[castbase:.*]] = bitcast {{.*}} %[[base]]
 //
 // CHECK: %[[a:.*]] = phi {{.*}} %[[castbase]]
@@ -65,7 +73,8 @@ void delete_C(C *c) { delete c; }
 //
 // CHECK-NOT: call
 // CHECK-ITANIUM: call void @_ZN1AdlEPS_St19destroying_delete_t(%{{.*}}* %[[a]])
-// CHECK-MSABI: call void @"??3A@@SAXPEAU0@Udestroying_delete_t@std@@@Z"(%{{.*}}* %[[a]], i8
+// CHECK-MSABI64: call void @"??3A@@SAXPEAU0@Udestroying_delete_t@std@@@Z"(%{{.*}}* %[[a]], i8
+// CHECK-MSABI32: call void @"??3A@@SAXPAU0@Udestroying_delete_t@std@@@Z"(%{{.*}}* %[[a]], %{{.*}}* byval(%{{.*}}) align 4 %{{.*}})
 // CHECK-NOT: call
 // CHECK: }
 
@@ -81,13 +90,14 @@ void delete_D(D *d) { delete d; }
 // For MS, we don't add a new vtable slot to the primary vtable for the virtual
 // destructor. Instead we cast to the VDel base class.
 // CHECK-MSABI: bitcast {{.*}} %[[d]]
-// CHECK-MSABI-NEXT: getelementptr {{.*}}, i64 8
+// CHECK-MSABI64-NEXT: getelementptr {{.*}}, i64 8
+// CHECK-MSABI32-NEXT: getelementptr {{.*}}, i32 4
 // CHECK-MSABI-NEXT: %[[d:.*]] = bitcast i8*
 //
 // CHECK: %[[VTABLE:.*]] = load
 // CHECK: %[[DTOR:.*]] = load
 //
-// CHECK: call {{void|i8\*}} %[[DTOR]](%{{.*}}* %[[d]]
+// CHECK: call {{void|i8\*|x86_thiscallcc i8\*}} %[[DTOR]](%{{.*}}* %[[d]]
 // CHECK-MSABI-SAME: , i32 1)
 // CHECK-NOT: call
 // CHECK: }
@@ -100,7 +110,8 @@ void delete_G(G *g) { delete g; }
 // CHECK-LABEL: define {{.*}}delete_G
 // CHECK-NOT: call
 // CHECK-ITANIUM: call void @_ZN1FdlEPS_St19destroying_delete_tmSt11align_val_t(%{{.*}}* %[[a]], i64 32, i64 16)
-// CHECK-MSABI: call void @"??3F@@SAXPEAU0@Udestroying_delete_t@std@@_KW4align_val_t@2@@Z"(%{{.*}}* %[[a]], i8 {{[^,]*}}, i64 32, i64 16)
+// CHECK-MSABI64: call void @"??3F@@SAXPEAU0@Udestroying_delete_t@std@@_KW4align_val_t@2@@Z"(%{{.*}}* %[[a]], i8 {{[^,]*}}, i64 32, i64 16)
+// CHECK-MSABI32: call void @"??3F@@SAXPAU0@Udestroying_delete_t@std@@IW4align_val_t@2@@Z"(%{{.*}}* %[[a]], %{{.*}}* byval(%{{.*}}) align 4 %{{.*}}, i32 16, i32 16)
 // CHECK-NOT: call
 // CHECK: }
 
@@ -116,19 +127,23 @@ H::~H() { call_in_dtor(); }
 // CHECK-ITANIUM-NOT: call
 // CHECK-ITANIUM: }
 
-// CHECK-MSABI: define {{.*}} @"??_GH@@UEAAPEAXI@Z"(
+// CHECK-MSABI64: define {{.*}} @"??_GH@@UEAAPEAXI@Z"(
+// CHECK-MSABI32: define {{.*}} @"??_GH@@UAEPAXI@Z"(
 // CHECK-MSABI-NOT: call{{ }}
 // CHECK-MSABI: load i32
 // CHECK-MSABI: icmp eq i32 {{.*}}, 0
 // CHECK-MSABI: br i1
 //
 // CHECK-MSABI-NOT: call{{ }}
-// CHECK-MSABI: getelementptr {{.*}}, i64 24
+// CHECK-MSABI64: getelementptr {{.*}}, i64 24
+// CHECK-MSABI32: getelementptr {{.*}}, i32 20
 // CHECK-MSABI-NOT: call{{ }}
-// CHECK-MSABI: call void @"??3F@@SAXPEAU0@Udestroying_delete_t@std@@_KW4align_val_t@2@@Z"({{.*}}, i64 48, i64 16)
+// CHECK-MSABI64: call void @"??3F@@SAXPEAU0@Udestroying_delete_t@std@@_KW4align_val_t@2@@Z"({{.*}}, i64 48, i64 16)
+// CHECK-MSABI32: call void @"??3F@@SAXPAU0@Udestroying_delete_t@std@@IW4align_val_t@2@@Z"({{.*}}, i32 32, i32 16)
 // CHECK-MSABI: br label %[[RETURN:.*]]
 //
-// CHECK-MSABI: call void @"??1H@@UEAA@XZ"(
+// CHECK-MSABI64: call void @"??1H@@UEAA@XZ"(
+// CHECK-MSABI32: call x86_thiscallcc void @"??1H@@UAE@XZ"(
 // CHECK-MSABI: br label %[[RETURN]]
 //
 // CHECK-MSABI: }
@@ -143,19 +158,23 @@ I::~I() { call_in_dtor(); }
 // CHECK-ITANIUM-NOT: call
 // CHECK-ITANIUM: }
 
-// CHECK-MSABI: define {{.*}} @"??_GI@@UEAAPEAXI@Z"(
+// CHECK-MSABI64: define {{.*}} @"??_GI@@UEAAPEAXI@Z"(
+// CHECK-MSABI32: define {{.*}} @"??_GI@@UAEPAXI@Z"(
 // CHECK-MSABI-NOT: call{{ }}
 // CHECK-MSABI: load i32
 // CHECK-MSABI: icmp eq i32 {{.*}}, 0
 // CHECK-MSABI: br i1
 //
 // CHECK-MSABI-NOT: call{{ }}
-// CHECK-MSABI: getelementptr {{.*}}, i64 24
+// CHECK-MSABI64: getelementptr {{.*}}, i64 24
+// CHECK-MSABI32: getelementptr {{.*}}, i32 20
 // CHECK-MSABI-NOT: call{{ }}
-// CHECK-MSABI: call void @"??3F@@SAXPEAU0@Udestroying_delete_t@std@@_KW4align_val_t@2@@Z"({{.*}}, i64 96, i64 32)
+// CHECK-MSABI64: call void @"??3F@@SAXPEAU0@Udestroying_delete_t@std@@_KW4align_val_t@2@@Z"({{.*}}, i64 96, i64 32)
+// CHECK-MSABI32: call void @"??3F@@SAXPAU0@Udestroying_delete_t@std@@IW4align_val_t@2@@Z"({{.*}}, i32 64, i32 32)
 // CHECK-MSABI: br label %[[RETURN:.*]]
 //
-// CHECK-MSABI: call void @"??1I@@UEAA@XZ"(
+// CHECK-MSABI64: call void @"??1I@@UEAA@XZ"(
+// CHECK-MSABI32: call x86_thiscallcc void @"??1I@@UAE@XZ"(
 // CHECK-MSABI: br label %[[RETURN]]
 //
 // CHECK-MSABI: }

From c23ae3f18ee3ff11671f4c62ffc66d150b1bcdc2 Mon Sep 17 00:00:00 2001
From: Nathan James <n.james93@hotmail.co.uk>
Date: Thu, 30 Jul 2020 22:57:32 +0100
Subject: [PATCH 0674/1035] [clang-tidy][NFC] Use StringMap for
 ClangTidyCheckFactories::FacoryMap

Reviewed By: gribozavr2

Differential Revision: https://reviews.llvm.org/D84926
---
 clang-tools-extra/clang-tidy/ClangTidy.cpp       | 4 ++--
 clang-tools-extra/clang-tidy/ClangTidyModule.cpp | 6 +++---
 clang-tools-extra/clang-tidy/ClangTidyModule.h   | 2 +-
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/clang-tools-extra/clang-tidy/ClangTidy.cpp b/clang-tools-extra/clang-tidy/ClangTidy.cpp
index 63c83a0b9954e..90b39347bc9ac 100644
--- a/clang-tools-extra/clang-tidy/ClangTidy.cpp
+++ b/clang-tools-extra/clang-tidy/ClangTidy.cpp
@@ -449,8 +449,8 @@ ClangTidyASTConsumerFactory::CreateASTConsumer(
 std::vector<std::string> ClangTidyASTConsumerFactory::getCheckNames() {
   std::vector<std::string> CheckNames;
   for (const auto &CheckFactory : *CheckFactories) {
-    if (Context.isCheckEnabled(CheckFactory.first))
-      CheckNames.push_back(CheckFactory.first);
+    if (Context.isCheckEnabled(CheckFactory.getKey()))
+      CheckNames.emplace_back(CheckFactory.getKey());
   }
 
 #if CLANG_ENABLE_STATIC_ANALYZER
diff --git a/clang-tools-extra/clang-tidy/ClangTidyModule.cpp b/clang-tools-extra/clang-tidy/ClangTidyModule.cpp
index e9031d498eeff..c72bcd3e188ce 100644
--- a/clang-tools-extra/clang-tidy/ClangTidyModule.cpp
+++ b/clang-tools-extra/clang-tidy/ClangTidyModule.cpp
@@ -18,15 +18,15 @@ namespace tidy {
 
 void ClangTidyCheckFactories::registerCheckFactory(StringRef Name,
                                                    CheckFactory Factory) {
-  Factories[std::string(Name)] = std::move(Factory);
+  Factories.insert_or_assign(Name, std::move(Factory));
 }
 
 std::vector<std::unique_ptr<ClangTidyCheck>>
 ClangTidyCheckFactories::createChecks(ClangTidyContext *Context) {
   std::vector<std::unique_ptr<ClangTidyCheck>> Checks;
   for (const auto &Factory : Factories) {
-    if (Context->isCheckEnabled(Factory.first))
-      Checks.emplace_back(Factory.second(Factory.first, Context));
+    if (Context->isCheckEnabled(Factory.getKey()))
+      Checks.emplace_back(Factory.getValue()(Factory.getKey(), Context));
   }
   return Checks;
 }
diff --git a/clang-tools-extra/clang-tidy/ClangTidyModule.h b/clang-tools-extra/clang-tidy/ClangTidyModule.h
index 31cf4774a885a..7fd16c2a7b3c7 100644
--- a/clang-tools-extra/clang-tidy/ClangTidyModule.h
+++ b/clang-tools-extra/clang-tidy/ClangTidyModule.h
@@ -69,7 +69,7 @@ class ClangTidyCheckFactories {
   std::vector<std::unique_ptr<ClangTidyCheck>>
   createChecks(ClangTidyContext *Context);
 
-  typedef std::map<std::string, CheckFactory> FactoryMap;
+  typedef llvm::StringMap<CheckFactory> FactoryMap;
   FactoryMap::const_iterator begin() const { return Factories.begin(); }
   FactoryMap::const_iterator end() const { return Factories.end(); }
   bool empty() const { return Factories.empty(); }

From 59156bad03ffe37558b95fca62b5df4394de280c Mon Sep 17 00:00:00 2001
From: Thomas Raoux <thomasraoux@google.com>
Date: Thu, 30 Jul 2020 14:56:50 -0700
Subject: [PATCH 0675/1035] [mlir][spirv] Add support for converting memref of
 vector to SPIR-V

This allow declaring buffers and alloc of vectors so that we can support vector
load/store.

Differential Revision: https://reviews.llvm.org/D84982
---
 .../ConvertStandardToSPIRV.cpp                | 14 +++---
 mlir/lib/Dialect/SPIRV/SPIRVLowering.cpp      | 34 +++++++++-----
 .../Conversion/StandardToSPIRV/alloc.mlir     | 24 ++++++++++
 .../StandardToSPIRV/std-types-to-spirv.mlir   | 45 +++++++++++++++++++
 4 files changed, 101 insertions(+), 16 deletions(-)

diff --git a/mlir/lib/Conversion/StandardToSPIRV/ConvertStandardToSPIRV.cpp b/mlir/lib/Conversion/StandardToSPIRV/ConvertStandardToSPIRV.cpp
index e59830fcef89a..543b23acabebc 100644
--- a/mlir/lib/Conversion/StandardToSPIRV/ConvertStandardToSPIRV.cpp
+++ b/mlir/lib/Conversion/StandardToSPIRV/ConvertStandardToSPIRV.cpp
@@ -217,11 +217,15 @@ CHECK_UNSIGNED_OP(spirv::UModOp)
 /// Returns true if the allocations of type `t` can be lowered to SPIR-V.
 static bool isAllocationSupported(MemRefType t) {
   // Currently only support workgroup local memory allocations with static
-  // shape and int or float element type.
-  return t.hasStaticShape() &&
-         SPIRVTypeConverter::getMemorySpaceForStorageClass(
-             spirv::StorageClass::Workgroup) == t.getMemorySpace() &&
-         t.getElementType().isIntOrFloat();
+  // shape and int or float or vector of int or float element type.
+  if (!(t.hasStaticShape() &&
+        SPIRVTypeConverter::getMemorySpaceForStorageClass(
+            spirv::StorageClass::Workgroup) == t.getMemorySpace()))
+    return false;
+  Type elementType = t.getElementType();
+  if (auto vecType = elementType.dyn_cast<VectorType>())
+    elementType = vecType.getElementType();
+  return elementType.isIntOrFloat();
 }
 
 /// Returns the scope to use for atomic operations use for emulating store
diff --git a/mlir/lib/Dialect/SPIRV/SPIRVLowering.cpp b/mlir/lib/Dialect/SPIRV/SPIRVLowering.cpp
index d31f9c28362a3..3d7535f9110e4 100644
--- a/mlir/lib/Dialect/SPIRV/SPIRVLowering.cpp
+++ b/mlir/lib/Dialect/SPIRV/SPIRVLowering.cpp
@@ -170,7 +170,14 @@ static Optional<int64_t> getTypeNumBytes(Type t) {
       return llvm::None;
     }
     return bitWidth / 8;
-  } else if (auto memRefType = t.dyn_cast<MemRefType>()) {
+  }
+  if (auto vecType = t.dyn_cast<VectorType>()) {
+    auto elementSize = getTypeNumBytes(vecType.getElementType());
+    if (!elementSize)
+      return llvm::None;
+    return vecType.getNumElements() * *elementSize;
+  }
+  if (auto memRefType = t.dyn_cast<MemRefType>()) {
     // TODO: Layout should also be controlled by the ABI attributes. For now
     // using the layout from MemRef.
     int64_t offset;
@@ -343,26 +350,31 @@ static Optional<Type> convertMemrefType(const spirv::TargetEnv &targetEnv,
     return llvm::None;
   }
 
-  auto scalarType = type.getElementType().dyn_cast<spirv::ScalarType>();
-  if (!scalarType) {
-    LLVM_DEBUG(llvm::dbgs()
-               << type << " illegal: cannot convert non-scalar element type\n");
+  Optional<Type> arrayElemType;
+  Type elementType = type.getElementType();
+  if (auto vecType = elementType.dyn_cast<VectorType>()) {
+    arrayElemType = convertVectorType(targetEnv, vecType, storageClass);
+  } else if (auto scalarType = elementType.dyn_cast<spirv::ScalarType>()) {
+    arrayElemType = convertScalarType(targetEnv, scalarType, storageClass);
+  } else {
+    LLVM_DEBUG(
+        llvm::dbgs()
+        << type
+        << " unhandled: can only convert scalar or vector element type\n");
     return llvm::None;
   }
-
-  auto arrayElemType = convertScalarType(targetEnv, scalarType, storageClass);
   if (!arrayElemType)
     return llvm::None;
 
-  Optional<int64_t> scalarSize = getTypeNumBytes(scalarType);
-  if (!scalarSize) {
+  Optional<int64_t> elementSize = getTypeNumBytes(elementType);
+  if (!elementSize) {
     LLVM_DEBUG(llvm::dbgs()
                << type << " illegal: cannot deduce element size\n");
     return llvm::None;
   }
 
   if (!type.hasStaticShape()) {
-    auto arrayType = spirv::RuntimeArrayType::get(*arrayElemType, *scalarSize);
+    auto arrayType = spirv::RuntimeArrayType::get(*arrayElemType, *elementSize);
     // Wrap in a struct to satisfy Vulkan interface requirements.
     auto structType = spirv::StructType::get(arrayType, 0);
     return spirv::PointerType::get(structType, *storageClass);
@@ -375,7 +387,7 @@ static Optional<Type> convertMemrefType(const spirv::TargetEnv &targetEnv,
     return llvm::None;
   }
 
-  auto arrayElemCount = *memrefSize / *scalarSize;
+  auto arrayElemCount = *memrefSize / *elementSize;
 
   Optional<int64_t> arrayElemSize = getTypeNumBytes(*arrayElemType);
   if (!arrayElemSize) {
diff --git a/mlir/test/Conversion/StandardToSPIRV/alloc.mlir b/mlir/test/Conversion/StandardToSPIRV/alloc.mlir
index 3cbeda1cafb07..fe4c9d125a269 100644
--- a/mlir/test/Conversion/StandardToSPIRV/alloc.mlir
+++ b/mlir/test/Conversion/StandardToSPIRV/alloc.mlir
@@ -75,6 +75,30 @@ module attributes {
 //      CHECK: spv.func @two_allocs()
 //      CHECK: spv.Return
 
+// -----
+
+module attributes {
+  spv.target_env = #spv.target_env<
+    #spv.vce<v1.0, [Shader], [SPV_KHR_storage_buffer_storage_class]>,
+    {max_compute_workgroup_invocations = 128 : i32,
+     max_compute_workgroup_size = dense<[128, 128, 64]> : vector<3xi32>}>
+  }
+{
+  func @two_allocs_vector() {
+    %0 = alloc() : memref<4xvector<4xf32>, 3>
+    %1 = alloc() : memref<2xvector<2xi32>, 3>
+    return
+  }
+}
+
+//  CHECK-DAG: spv.globalVariable @__workgroup_mem__{{[0-9]+}}
+// CHECK-SAME:   !spv.ptr<!spv.struct<!spv.array<2 x vector<2xi32>, stride=8>>, Workgroup>
+//  CHECK-DAG: spv.globalVariable @__workgroup_mem__{{[0-9]+}}
+// CHECK-SAME:   !spv.ptr<!spv.struct<!spv.array<4 x vector<4xf32>, stride=16>>, Workgroup>
+//      CHECK: spv.func @two_allocs_vector()
+//      CHECK: spv.Return
+
+
 // -----
 
 module attributes {
diff --git a/mlir/test/Conversion/StandardToSPIRV/std-types-to-spirv.mlir b/mlir/test/Conversion/StandardToSPIRV/std-types-to-spirv.mlir
index b98a20a56c6a7..5ea44c18c6183 100644
--- a/mlir/test/Conversion/StandardToSPIRV/std-types-to-spirv.mlir
+++ b/mlir/test/Conversion/StandardToSPIRV/std-types-to-spirv.mlir
@@ -510,6 +510,51 @@ func @dynamic_dim_memref(%arg0: memref<8x?xi32>,
 
 // -----
 
+// Vector types
+module attributes {
+  spv.target_env = #spv.target_env<
+    #spv.vce<v1.0, [], []>,
+    {max_compute_workgroup_invocations = 128 : i32,
+     max_compute_workgroup_size = dense<[128, 128, 64]> : vector<3xi32>}>
+} {
+
+// CHECK-LABEL: func @memref_vector
+// CHECK-SAME: !spv.ptr<!spv.struct<!spv.array<4 x vector<2xf32>, stride=8> [0]>, StorageBuffer>
+// CHECK-SAME: !spv.ptr<!spv.struct<!spv.array<4 x vector<4xf32>, stride=16> [0]>, Uniform>
+func @memref_vector(
+    %arg0: memref<4xvector<2xf32>, 0>,
+    %arg1: memref<4xvector<4xf32>, 4>)
+{ return }
+
+// CHECK-LABEL: func @dynamic_dim_memref_vector
+// CHECK-SAME: !spv.ptr<!spv.struct<!spv.rtarray<vector<4xi32>, stride=16> [0]>, StorageBuffer>
+// CHECK-SAME: !spv.ptr<!spv.struct<!spv.rtarray<vector<2xf32>, stride=8> [0]>, StorageBuffer>
+func @dynamic_dim_memref_vector(%arg0: memref<8x?xvector<4xi32>>,
+                         %arg1: memref<?x?xvector<2xf32>>)
+{ return }
+
+} // end module
+
+// -----
+
+// Vector types, check that sizes not available in SPIR-V are not transformed.
+module attributes {
+  spv.target_env = #spv.target_env<
+    #spv.vce<v1.0, [], []>,
+    {max_compute_workgroup_invocations = 128 : i32,
+     max_compute_workgroup_size = dense<[128, 128, 64]> : vector<3xi32>}>
+} {
+
+// CHECK-LABEL: func @memref_vector_wrong_size
+// CHECK-SAME: memref<4xvector<5xf32>>
+func @memref_vector_wrong_size(
+    %arg0: memref<4xvector<5xf32>, 0>)
+{ return }
+
+} // end module
+
+// -----
+
 //===----------------------------------------------------------------------===//
 // Tensor types
 //===----------------------------------------------------------------------===//

From e56e9022bc54c575ad8ecc3934e9cf84b8542e62 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Wed, 1 Jul 2020 12:48:42 -0400
Subject: [PATCH 0676/1035] AMDGPU: Fix liveness errors when copying AGPR
 tuples

Avoid recursively calling copyPhysReg for AGPR handling. This was
dropping the necessary super register implicit defs to avoid liveness
verifier errors.
---
 llvm/lib/Target/AMDGPU/SIInstrInfo.cpp    |  77 ++++--
 llvm/test/CodeGen/AMDGPU/accvgpr-copy.mir | 292 +++++++++++++++-------
 2 files changed, 266 insertions(+), 103 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index d3dbf47370678..745a526085566 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -542,7 +542,9 @@ static void indirectCopyToAGPR(const SIInstrInfo &TII,
                                MachineBasicBlock::iterator MI,
                                const DebugLoc &DL, MCRegister DestReg,
                                MCRegister SrcReg, bool KillSrc,
-                               RegScavenger &RS) {
+                               RegScavenger &RS,
+                               Register ImpDefSuperReg = Register(),
+                               Register ImpUseSuperReg = Register()) {
   const SIRegisterInfo &RI = TII.getRegisterInfo();
 
   assert(AMDGPU::SReg_32RegClass.contains(SrcReg) ||
@@ -573,8 +575,17 @@ static void indirectCopyToAGPR(const SIInstrInfo &TII,
       DefOp.setIsKill(false);
     }
 
-    BuildMI(MBB, MI, DL, TII.get(AMDGPU::V_ACCVGPR_WRITE_B32), DestReg)
+    MachineInstrBuilder Builder =
+      BuildMI(MBB, MI, DL, TII.get(AMDGPU::V_ACCVGPR_WRITE_B32), DestReg)
       .add(DefOp);
+    if (ImpDefSuperReg)
+      Builder.addReg(ImpDefSuperReg, RegState::Define | RegState::Implicit);
+
+    if (ImpUseSuperReg) {
+      Builder.addReg(ImpUseSuperReg,
+                     getKillRegState(KillSrc) | RegState::Implicit);
+    }
+
     return;
   }
 
@@ -604,9 +615,27 @@ static void indirectCopyToAGPR(const SIInstrInfo &TII,
     RS.setRegUsed(Tmp);
   }
 
-  TII.copyPhysReg(MBB, MI, DL, Tmp, SrcReg, KillSrc);
-  BuildMI(MBB, MI, DL, TII.get(AMDGPU::V_ACCVGPR_WRITE_B32), DestReg)
+  // Insert copy to temporary VGPR.
+  unsigned TmpCopyOp = AMDGPU::V_MOV_B32_e32;
+  if (AMDGPU::AGPR_32RegClass.contains(SrcReg)) {
+    TmpCopyOp = AMDGPU::V_ACCVGPR_READ_B32;
+  } else {
+    assert(AMDGPU::SReg_32RegClass.contains(SrcReg));
+  }
+
+  MachineInstrBuilder UseBuilder = BuildMI(MBB, MI, DL, TII.get(TmpCopyOp), Tmp)
+    .addReg(SrcReg, getKillRegState(KillSrc));
+  if (ImpUseSuperReg) {
+    UseBuilder.addReg(ImpUseSuperReg,
+                      getKillRegState(KillSrc) | RegState::Implicit);
+  }
+
+  MachineInstrBuilder DefBuilder
+    = BuildMI(MBB, MI, DL, TII.get(AMDGPU::V_ACCVGPR_WRITE_B32), DestReg)
     .addReg(Tmp, RegState::Kill);
+
+  if (ImpDefSuperReg)
+    DefBuilder.addReg(ImpDefSuperReg, RegState::Define | RegState::Implicit);
 }
 
 void SIInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
@@ -827,11 +856,21 @@ void SIInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
     }
   } else if (RI.hasAGPRs(RC)) {
     Opcode = RI.hasVGPRs(RI.getPhysRegClass(SrcReg)) ?
-      AMDGPU::V_ACCVGPR_WRITE_B32 : AMDGPU::COPY;
+      AMDGPU::V_ACCVGPR_WRITE_B32 : AMDGPU::INSTRUCTION_LIST_END;
   } else if (RI.hasVGPRs(RC) && RI.hasAGPRs(RI.getPhysRegClass(SrcReg))) {
     Opcode = AMDGPU::V_ACCVGPR_READ_B32;
   }
 
+  // For the cases where we need an intermediate instruction/temporary register
+  // (the result is an SGPR, and the source is either an SGPR or AGPR), we need
+  // a scavenger.
+  //
+  // FIXME: The pass should maintain this for us so we don't have to re-scan the
+  // whole block for every handled copy.
+  std::unique_ptr<RegScavenger> RS;
+  if (Opcode == AMDGPU::INSTRUCTION_LIST_END)
+    RS.reset(new RegScavenger());
+
   ArrayRef<int16_t> SubIndices = RI.getRegSplitParts(RC, EltSize);
   bool Forward = RI.getHWRegIndex(DestReg) <= RI.getHWRegIndex(SrcReg);
 
@@ -842,22 +881,24 @@ void SIInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
     else
       SubIdx = SubIndices[SubIndices.size() - Idx - 1];
 
-    if (Opcode == TargetOpcode::COPY) {
-      copyPhysReg(MBB, MI, DL, RI.getSubReg(DestReg, SubIdx),
-                  RI.getSubReg(SrcReg, SubIdx), KillSrc);
-      continue;
-    }
-
-    MachineInstrBuilder Builder = BuildMI(MBB, MI, DL,
-      get(Opcode), RI.getSubReg(DestReg, SubIdx));
 
-    Builder.addReg(RI.getSubReg(SrcReg, SubIdx));
+    bool UseKill = KillSrc && Idx == SubIndices.size() - 1;
 
-    if (Idx == 0)
-      Builder.addReg(DestReg, RegState::Define | RegState::Implicit);
+    if (Opcode == AMDGPU::INSTRUCTION_LIST_END) {
+      Register ImpDefSuper = Idx == 0 ? Register(DestReg) : Register();
+      Register ImpUseSuper = SrcReg;
+      indirectCopyToAGPR(*this, MBB, MI, DL, RI.getSubReg(DestReg, SubIdx),
+                         RI.getSubReg(SrcReg, SubIdx), UseKill, *RS,
+                         ImpDefSuper, ImpUseSuper);
+    } else {
+      MachineInstrBuilder Builder =
+        BuildMI(MBB, MI, DL, get(Opcode), RI.getSubReg(DestReg, SubIdx))
+        .addReg(RI.getSubReg(SrcReg, SubIdx));
+      if (Idx == 0)
+        Builder.addReg(DestReg, RegState::Define | RegState::Implicit);
 
-    bool UseKill = KillSrc && Idx == SubIndices.size() - 1;
-    Builder.addReg(SrcReg, getKillRegState(UseKill) | RegState::Implicit);
+      Builder.addReg(SrcReg, getKillRegState(UseKill) | RegState::Implicit);
+    }
   }
 }
 
diff --git a/llvm/test/CodeGen/AMDGPU/accvgpr-copy.mir b/llvm/test/CodeGen/AMDGPU/accvgpr-copy.mir
index 667a22ea0c1da..de839d9253e80 100644
--- a/llvm/test/CodeGen/AMDGPU/accvgpr-copy.mir
+++ b/llvm/test/CodeGen/AMDGPU/accvgpr-copy.mir
@@ -28,10 +28,16 @@
     define amdgpu_kernel void @a2_to_a2() #0 { ret void }
     define amdgpu_kernel void @a3_to_a3() #0 { ret void }
     define amdgpu_kernel void @a4_to_a4() #0 { ret void }
+    define amdgpu_kernel void @a4_to_a4_overlap() #0 { ret void }
     define amdgpu_kernel void @a8_to_a8() #0 { ret void }
     define amdgpu_kernel void @a16_to_a16() #0 { ret void }
 
     define amdgpu_kernel void @a_to_a_spill() #0 { ret void }
+    define amdgpu_kernel void @copy_sgpr_to_agpr_tuple() #0 { ret void }
+    define amdgpu_kernel void @copy_sgpr_to_agpr_tuple_kill() #0 { ret void }
+    define amdgpu_kernel void @copy_agpr_to_agpr_tuple() #0 { ret void }
+    define amdgpu_kernel void @copy_agpr_to_agpr_tuple_kill() #0 { ret void }
+
     attributes #0 = { "amdgpu-flat-work-group-size"="1,256" }
 ...
 
@@ -285,9 +291,9 @@ body:             |
     liveins: $sgpr0_sgpr1
     ; GCN-LABEL: name: s2_to_a2
     ; GCN: liveins: $sgpr0_sgpr1
-    ; GCN: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec
-    ; GCN: $agpr0 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec
-    ; GCN: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec
+    ; GCN: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit $sgpr0_sgpr1
+    ; GCN: $agpr0 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1
+    ; GCN: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit killed $sgpr0_sgpr1
     ; GCN: $agpr1 = V_ACCVGPR_WRITE_B32 killed $vgpr1, implicit $exec, implicit $exec
     ; GCN: S_ENDPGM 0, implicit $agpr0_agpr1
     $agpr0_agpr1 = COPY killed $sgpr0_sgpr1, implicit $exec
@@ -302,11 +308,11 @@ body:             |
     liveins: $sgpr0_sgpr1_sgpr2
     ; GCN-LABEL: name: s3_to_a3
     ; GCN: liveins: $sgpr0_sgpr1_sgpr2
-    ; GCN: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec
-    ; GCN: $agpr0 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec
-    ; GCN: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec
+    ; GCN: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit $sgpr0_sgpr1_sgpr2
+    ; GCN: $agpr0 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2
+    ; GCN: $vgpr1 = V_MOV_B32_e32 $sgpr1, implicit $exec, implicit $sgpr0_sgpr1_sgpr2
     ; GCN: $agpr1 = V_ACCVGPR_WRITE_B32 killed $vgpr1, implicit $exec
-    ; GCN: $vgpr2 = V_MOV_B32_e32 killed $sgpr2, implicit $exec
+    ; GCN: $vgpr2 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit killed $sgpr0_sgpr1_sgpr2
     ; GCN: $agpr2 = V_ACCVGPR_WRITE_B32 killed $vgpr2, implicit $exec, implicit $exec
     ; GCN: S_ENDPGM 0, implicit $agpr0_agpr1_agpr2
     $agpr0_agpr1_agpr2 = COPY killed $sgpr0_sgpr1_sgpr2, implicit $exec
@@ -321,13 +327,13 @@ body:             |
     liveins: $sgpr0_sgpr1_sgpr2_sgpr3
     ; GCN-LABEL: name: s4_to_a4
     ; GCN: liveins: $sgpr0_sgpr1_sgpr2_sgpr3
-    ; GCN: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec
-    ; GCN: $agpr0 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec
-    ; GCN: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec
+    ; GCN: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3
+    ; GCN: $agpr0 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3
+    ; GCN: $vgpr1 = V_MOV_B32_e32 $sgpr1, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3
     ; GCN: $agpr1 = V_ACCVGPR_WRITE_B32 killed $vgpr1, implicit $exec
-    ; GCN: $vgpr2 = V_MOV_B32_e32 killed $sgpr2, implicit $exec
+    ; GCN: $vgpr2 = V_MOV_B32_e32 $sgpr2, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3
     ; GCN: $agpr2 = V_ACCVGPR_WRITE_B32 killed $vgpr2, implicit $exec
-    ; GCN: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec
+    ; GCN: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit killed $sgpr0_sgpr1_sgpr2_sgpr3
     ; GCN: $agpr3 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit $exec
     ; GCN: S_ENDPGM 0, implicit $agpr0_agpr1_agpr2_agpr3
     $agpr0_agpr1_agpr2_agpr3 = COPY killed $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec
@@ -342,17 +348,17 @@ body:             |
     liveins: $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5
     ; GCN-LABEL: name: s6_to_a6
     ; GCN: liveins: $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5
-    ; GCN: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec
-    ; GCN: $agpr0 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec
-    ; GCN: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec
+    ; GCN: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5
+    ; GCN: $agpr0 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5
+    ; GCN: $vgpr1 = V_MOV_B32_e32 $sgpr1, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5
     ; GCN: $agpr1 = V_ACCVGPR_WRITE_B32 killed $vgpr1, implicit $exec
-    ; GCN: $vgpr2 = V_MOV_B32_e32 killed $sgpr2, implicit $exec
+    ; GCN: $vgpr2 = V_MOV_B32_e32 $sgpr2, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5
     ; GCN: $agpr2 = V_ACCVGPR_WRITE_B32 killed $vgpr2, implicit $exec
-    ; GCN: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec
+    ; GCN: $vgpr0 = V_MOV_B32_e32 $sgpr3, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5
     ; GCN: $agpr3 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec
-    ; GCN: $vgpr1 = V_MOV_B32_e32 killed $sgpr4, implicit $exec
+    ; GCN: $vgpr1 = V_MOV_B32_e32 $sgpr4, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5
     ; GCN: $agpr4 = V_ACCVGPR_WRITE_B32 killed $vgpr1, implicit $exec
-    ; GCN: $vgpr2 = V_MOV_B32_e32 killed $sgpr5, implicit $exec
+    ; GCN: $vgpr2 = V_MOV_B32_e32 killed $sgpr5, implicit $exec, implicit killed $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5
     ; GCN: $agpr5 = V_ACCVGPR_WRITE_B32 killed $vgpr2, implicit $exec, implicit $exec
     ; GCN: S_ENDPGM 0, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5
     $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5 = COPY killed $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5, implicit $exec
@@ -367,21 +373,21 @@ body:             |
     liveins: $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     ; GCN-LABEL: name: s8_to_a8
     ; GCN: liveins: $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
-    ; GCN: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec
-    ; GCN: $agpr0 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec
-    ; GCN: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec
+    ; GCN: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
+    ; GCN: $agpr0 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+    ; GCN: $vgpr1 = V_MOV_B32_e32 $sgpr1, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     ; GCN: $agpr1 = V_ACCVGPR_WRITE_B32 killed $vgpr1, implicit $exec
-    ; GCN: $vgpr2 = V_MOV_B32_e32 killed $sgpr2, implicit $exec
+    ; GCN: $vgpr2 = V_MOV_B32_e32 $sgpr2, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     ; GCN: $agpr2 = V_ACCVGPR_WRITE_B32 killed $vgpr2, implicit $exec
-    ; GCN: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec
+    ; GCN: $vgpr0 = V_MOV_B32_e32 $sgpr3, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     ; GCN: $agpr3 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec
-    ; GCN: $vgpr1 = V_MOV_B32_e32 killed $sgpr4, implicit $exec
+    ; GCN: $vgpr1 = V_MOV_B32_e32 $sgpr4, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     ; GCN: $agpr4 = V_ACCVGPR_WRITE_B32 killed $vgpr1, implicit $exec
-    ; GCN: $vgpr2 = V_MOV_B32_e32 killed $sgpr5, implicit $exec
+    ; GCN: $vgpr2 = V_MOV_B32_e32 $sgpr5, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     ; GCN: $agpr5 = V_ACCVGPR_WRITE_B32 killed $vgpr2, implicit $exec
-    ; GCN: $vgpr0 = V_MOV_B32_e32 killed $sgpr6, implicit $exec
+    ; GCN: $vgpr0 = V_MOV_B32_e32 $sgpr6, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     ; GCN: $agpr6 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec
-    ; GCN: $vgpr1 = V_MOV_B32_e32 killed $sgpr7, implicit $exec
+    ; GCN: $vgpr1 = V_MOV_B32_e32 killed $sgpr7, implicit $exec, implicit killed $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     ; GCN: $agpr7 = V_ACCVGPR_WRITE_B32 killed $vgpr1, implicit $exec, implicit $exec
     ; GCN: S_ENDPGM 0, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
     $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7 = COPY killed $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, implicit $exec
@@ -396,37 +402,37 @@ body:             |
     liveins: $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
     ; GCN-LABEL: name: s16_to_a16
     ; GCN: liveins: $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
-    ; GCN: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec
-    ; GCN: $agpr0 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec
-    ; GCN: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec
+    ; GCN: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
+    ; GCN: $agpr0 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; GCN: $vgpr1 = V_MOV_B32_e32 $sgpr1, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
     ; GCN: $agpr1 = V_ACCVGPR_WRITE_B32 killed $vgpr1, implicit $exec
-    ; GCN: $vgpr2 = V_MOV_B32_e32 killed $sgpr2, implicit $exec
+    ; GCN: $vgpr2 = V_MOV_B32_e32 $sgpr2, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
     ; GCN: $agpr2 = V_ACCVGPR_WRITE_B32 killed $vgpr2, implicit $exec
-    ; GCN: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec
+    ; GCN: $vgpr0 = V_MOV_B32_e32 $sgpr3, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
     ; GCN: $agpr3 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec
-    ; GCN: $vgpr1 = V_MOV_B32_e32 killed $sgpr4, implicit $exec
+    ; GCN: $vgpr1 = V_MOV_B32_e32 $sgpr4, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
     ; GCN: $agpr4 = V_ACCVGPR_WRITE_B32 killed $vgpr1, implicit $exec
-    ; GCN: $vgpr2 = V_MOV_B32_e32 killed $sgpr5, implicit $exec
+    ; GCN: $vgpr2 = V_MOV_B32_e32 $sgpr5, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
     ; GCN: $agpr5 = V_ACCVGPR_WRITE_B32 killed $vgpr2, implicit $exec
-    ; GCN: $vgpr0 = V_MOV_B32_e32 killed $sgpr6, implicit $exec
+    ; GCN: $vgpr0 = V_MOV_B32_e32 $sgpr6, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
     ; GCN: $agpr6 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec
-    ; GCN: $vgpr1 = V_MOV_B32_e32 killed $sgpr7, implicit $exec
+    ; GCN: $vgpr1 = V_MOV_B32_e32 $sgpr7, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
     ; GCN: $agpr7 = V_ACCVGPR_WRITE_B32 killed $vgpr1, implicit $exec
-    ; GCN: $vgpr2 = V_MOV_B32_e32 killed $sgpr8, implicit $exec
+    ; GCN: $vgpr2 = V_MOV_B32_e32 $sgpr8, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
     ; GCN: $agpr8 = V_ACCVGPR_WRITE_B32 killed $vgpr2, implicit $exec
-    ; GCN: $vgpr0 = V_MOV_B32_e32 killed $sgpr9, implicit $exec
+    ; GCN: $vgpr0 = V_MOV_B32_e32 $sgpr9, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
     ; GCN: $agpr9 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec
-    ; GCN: $vgpr1 = V_MOV_B32_e32 killed $sgpr10, implicit $exec
+    ; GCN: $vgpr1 = V_MOV_B32_e32 $sgpr10, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
     ; GCN: $agpr10 = V_ACCVGPR_WRITE_B32 killed $vgpr1, implicit $exec
-    ; GCN: $vgpr2 = V_MOV_B32_e32 killed $sgpr11, implicit $exec
+    ; GCN: $vgpr2 = V_MOV_B32_e32 $sgpr11, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
     ; GCN: $agpr11 = V_ACCVGPR_WRITE_B32 killed $vgpr2, implicit $exec
-    ; GCN: $vgpr0 = V_MOV_B32_e32 killed $sgpr12, implicit $exec
+    ; GCN: $vgpr0 = V_MOV_B32_e32 $sgpr12, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
     ; GCN: $agpr12 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec
-    ; GCN: $vgpr1 = V_MOV_B32_e32 killed $sgpr13, implicit $exec
+    ; GCN: $vgpr1 = V_MOV_B32_e32 $sgpr13, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
     ; GCN: $agpr13 = V_ACCVGPR_WRITE_B32 killed $vgpr1, implicit $exec
-    ; GCN: $vgpr2 = V_MOV_B32_e32 killed $sgpr14, implicit $exec
+    ; GCN: $vgpr2 = V_MOV_B32_e32 $sgpr14, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
     ; GCN: $agpr14 = V_ACCVGPR_WRITE_B32 killed $vgpr2, implicit $exec
-    ; GCN: $vgpr0 = V_MOV_B32_e32 killed $sgpr15, implicit $exec
+    ; GCN: $vgpr0 = V_MOV_B32_e32 killed $sgpr15, implicit $exec, implicit killed $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
     ; GCN: $agpr15 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit $exec
     ; GCN: S_ENDPGM 0, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
     $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = COPY killed $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15, implicit $exec
@@ -455,9 +461,9 @@ body:             |
   bb.0:
     ; GCN-LABEL: name: a2_to_a2
     ; GCN: $agpr0_agpr1 = IMPLICIT_DEF
-    ; GCN: $vgpr2 = V_ACCVGPR_READ_B32 killed $agpr1, implicit $exec
-    ; GCN: $agpr2 = V_ACCVGPR_WRITE_B32 killed $vgpr2, implicit $exec
-    ; GCN: $vgpr1 = V_ACCVGPR_READ_B32 killed $agpr0, implicit $exec
+    ; GCN: $vgpr2 = V_ACCVGPR_READ_B32 $agpr1, implicit $exec, implicit $agpr0_agpr1
+    ; GCN: $agpr2 = V_ACCVGPR_WRITE_B32 killed $vgpr2, implicit $exec, implicit-def $agpr1_agpr2
+    ; GCN: $vgpr1 = V_ACCVGPR_READ_B32 killed $agpr0, implicit $exec, implicit killed $agpr0_agpr1
     ; GCN: $agpr1 = V_ACCVGPR_WRITE_B32 killed $vgpr1, implicit $exec, implicit $exec
     ; GCN: S_ENDPGM 0, implicit $agpr1_agpr2
     $agpr0_agpr1 = IMPLICIT_DEF
@@ -472,11 +478,11 @@ body:             |
   bb.0:
     ; GCN-LABEL: name: a3_to_a3
     ; GCN: $agpr0_agpr1_agpr2 = IMPLICIT_DEF
-    ; GCN: $vgpr1 = V_ACCVGPR_READ_B32 killed $agpr2, implicit $exec
-    ; GCN: $agpr4 = V_ACCVGPR_WRITE_B32 killed $vgpr1, implicit $exec
-    ; GCN: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr1, implicit $exec
+    ; GCN: $vgpr1 = V_ACCVGPR_READ_B32 $agpr2, implicit $exec, implicit $agpr0_agpr1_agpr2
+    ; GCN: $agpr4 = V_ACCVGPR_WRITE_B32 killed $vgpr1, implicit $exec, implicit-def $agpr2_agpr3_agpr4
+    ; GCN: $vgpr0 = V_ACCVGPR_READ_B32 $agpr1, implicit $exec, implicit $agpr0_agpr1_agpr2
     ; GCN: $agpr3 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec
-    ; GCN: $vgpr2 = V_ACCVGPR_READ_B32 killed $agpr0, implicit $exec
+    ; GCN: $vgpr2 = V_ACCVGPR_READ_B32 killed $agpr0, implicit $exec, implicit killed $agpr0_agpr1_agpr2
     ; GCN: $agpr2 = V_ACCVGPR_WRITE_B32 killed $vgpr2, implicit $exec, implicit $exec
     ; GCN: S_ENDPGM 0, implicit $agpr2_agpr3_agpr4
     $agpr0_agpr1_agpr2 = IMPLICIT_DEF
@@ -491,20 +497,39 @@ body:             |
   bb.0:
     ; GCN-LABEL: name: a4_to_a4
     ; GCN: $agpr0_agpr1_agpr2_agpr3 = IMPLICIT_DEF
-    ; GCN: $vgpr2 = V_ACCVGPR_READ_B32 killed $agpr3, implicit $exec
-    ; GCN: $agpr5 = V_ACCVGPR_WRITE_B32 killed $vgpr2, implicit $exec
-    ; GCN: $vgpr1 = V_ACCVGPR_READ_B32 killed $agpr2, implicit $exec
-    ; GCN: $agpr4 = V_ACCVGPR_WRITE_B32 killed $vgpr1, implicit $exec
-    ; GCN: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr1, implicit $exec
+    ; GCN: $vgpr2 = V_ACCVGPR_READ_B32 $agpr3, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3
+    ; GCN: $agpr5 = V_ACCVGPR_WRITE_B32 $vgpr2, implicit $exec, implicit-def $agpr2_agpr3_agpr4_agpr5
+    ; GCN: $agpr4 = V_ACCVGPR_WRITE_B32 $vgpr2, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3
+    ; GCN: $vgpr0 = V_ACCVGPR_READ_B32 $agpr1, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3
     ; GCN: $agpr3 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec
-    ; GCN: $vgpr2 = V_ACCVGPR_READ_B32 killed $agpr0, implicit $exec
-    ; GCN: $agpr2 = V_ACCVGPR_WRITE_B32 killed $vgpr2, implicit $exec, implicit $exec
+    ; GCN: $vgpr3 = V_ACCVGPR_READ_B32 killed $agpr0, implicit $exec, implicit killed $agpr0_agpr1_agpr2_agpr3
+    ; GCN: $agpr2 = V_ACCVGPR_WRITE_B32 killed $vgpr3, implicit $exec, implicit $exec
     ; GCN: S_ENDPGM 0, implicit $agpr2_agpr3_agpr4_agpr5
     $agpr0_agpr1_agpr2_agpr3 = IMPLICIT_DEF
     $agpr2_agpr3_agpr4_agpr5 = COPY killed $agpr0_agpr1_agpr2_agpr3, implicit $exec
     S_ENDPGM 0, implicit $agpr2_agpr3_agpr4_agpr5
 ...
 
+---
+name:            a4_to_a4_overlap
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $agpr0_agpr1_agpr2_agpr3
+    ; GCN-LABEL: name: a4_to_a4_overlap
+    ; GCN: liveins: $agpr0_agpr1_agpr2_agpr3
+    ; GCN: $vgpr2 = V_ACCVGPR_READ_B32 $agpr3, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3
+    ; GCN: $agpr5 = V_ACCVGPR_WRITE_B32 $vgpr2, implicit $exec, implicit-def $agpr2_agpr3_agpr4_agpr5
+    ; GCN: $agpr4 = V_ACCVGPR_WRITE_B32 $vgpr2, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3
+    ; GCN: $vgpr0 = V_ACCVGPR_READ_B32 $agpr1, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3
+    ; GCN: $agpr3 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec
+    ; GCN: $vgpr3 = V_ACCVGPR_READ_B32 $agpr0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3
+    ; GCN: $agpr2 = V_ACCVGPR_WRITE_B32 killed $vgpr3, implicit $exec, implicit $exec
+    ; GCN: S_ENDPGM 0, implicit $agpr0, implicit $agpr1, implicit $agpr2, implicit $agpr3, implicit $agpr4, implicit $agpr5
+    $agpr2_agpr3_agpr4_agpr5 = COPY $agpr0_agpr1_agpr2_agpr3, implicit $exec
+    S_ENDPGM 0, implicit $agpr0, implicit $agpr1, implicit $agpr2, implicit $agpr3, implicit $agpr4, implicit $agpr5
+...
+
 ---
 name:            a8_to_a8
 tracksRegLiveness: true
@@ -512,21 +537,21 @@ body:             |
   bb.0:
     ; GCN-LABEL: name: a8_to_a8
     ; GCN: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7 = IMPLICIT_DEF
-    ; GCN: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr7, implicit $exec
-    ; GCN: $agpr15 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec
-    ; GCN: $vgpr2 = V_ACCVGPR_READ_B32 killed $agpr6, implicit $exec
+    ; GCN: $vgpr0 = V_ACCVGPR_READ_B32 $agpr7, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+    ; GCN: $agpr15 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; GCN: $vgpr2 = V_ACCVGPR_READ_B32 $agpr6, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
     ; GCN: $agpr14 = V_ACCVGPR_WRITE_B32 killed $vgpr2, implicit $exec
-    ; GCN: $vgpr1 = V_ACCVGPR_READ_B32 killed $agpr5, implicit $exec
+    ; GCN: $vgpr1 = V_ACCVGPR_READ_B32 $agpr5, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
     ; GCN: $agpr13 = V_ACCVGPR_WRITE_B32 killed $vgpr1, implicit $exec
-    ; GCN: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr4, implicit $exec
+    ; GCN: $vgpr0 = V_ACCVGPR_READ_B32 $agpr4, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
     ; GCN: $agpr12 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec
-    ; GCN: $vgpr2 = V_ACCVGPR_READ_B32 killed $agpr3, implicit $exec
+    ; GCN: $vgpr2 = V_ACCVGPR_READ_B32 $agpr3, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
     ; GCN: $agpr11 = V_ACCVGPR_WRITE_B32 killed $vgpr2, implicit $exec
-    ; GCN: $vgpr1 = V_ACCVGPR_READ_B32 killed $agpr2, implicit $exec
+    ; GCN: $vgpr1 = V_ACCVGPR_READ_B32 $agpr2, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
     ; GCN: $agpr10 = V_ACCVGPR_WRITE_B32 killed $vgpr1, implicit $exec
-    ; GCN: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr1, implicit $exec
+    ; GCN: $vgpr0 = V_ACCVGPR_READ_B32 $agpr1, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
     ; GCN: $agpr9 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec
-    ; GCN: $vgpr2 = V_ACCVGPR_READ_B32 killed $agpr0, implicit $exec
+    ; GCN: $vgpr2 = V_ACCVGPR_READ_B32 killed $agpr0, implicit $exec, implicit killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
     ; GCN: $agpr8 = V_ACCVGPR_WRITE_B32 killed $vgpr2, implicit $exec, implicit $exec
     ; GCN: S_ENDPGM 0, implicit $agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
     $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7 = IMPLICIT_DEF
@@ -542,37 +567,37 @@ body:             |
 
     ; GCN-LABEL: name: a16_to_a16
     ; GCN: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = IMPLICIT_DEF
-    ; GCN: $vgpr1 = V_ACCVGPR_READ_B32 killed $agpr15, implicit $exec
-    ; GCN: $agpr31 = V_ACCVGPR_WRITE_B32 killed $vgpr1, implicit $exec
-    ; GCN: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr14, implicit $exec
+    ; GCN: $vgpr1 = V_ACCVGPR_READ_B32 $agpr15, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; GCN: $agpr31 = V_ACCVGPR_WRITE_B32 killed $vgpr1, implicit $exec, implicit-def $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; GCN: $vgpr0 = V_ACCVGPR_READ_B32 $agpr14, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
     ; GCN: $agpr30 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec
-    ; GCN: $vgpr2 = V_ACCVGPR_READ_B32 killed $agpr13, implicit $exec
+    ; GCN: $vgpr2 = V_ACCVGPR_READ_B32 $agpr13, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
     ; GCN: $agpr29 = V_ACCVGPR_WRITE_B32 killed $vgpr2, implicit $exec
-    ; GCN: $vgpr1 = V_ACCVGPR_READ_B32 killed $agpr12, implicit $exec
+    ; GCN: $vgpr1 = V_ACCVGPR_READ_B32 $agpr12, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
     ; GCN: $agpr28 = V_ACCVGPR_WRITE_B32 killed $vgpr1, implicit $exec
-    ; GCN: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr11, implicit $exec
+    ; GCN: $vgpr0 = V_ACCVGPR_READ_B32 $agpr11, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
     ; GCN: $agpr27 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec
-    ; GCN: $vgpr2 = V_ACCVGPR_READ_B32 killed $agpr10, implicit $exec
+    ; GCN: $vgpr2 = V_ACCVGPR_READ_B32 $agpr10, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
     ; GCN: $agpr26 = V_ACCVGPR_WRITE_B32 killed $vgpr2, implicit $exec
-    ; GCN: $vgpr1 = V_ACCVGPR_READ_B32 killed $agpr9, implicit $exec
+    ; GCN: $vgpr1 = V_ACCVGPR_READ_B32 $agpr9, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
     ; GCN: $agpr25 = V_ACCVGPR_WRITE_B32 killed $vgpr1, implicit $exec
-    ; GCN: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr8, implicit $exec
+    ; GCN: $vgpr0 = V_ACCVGPR_READ_B32 $agpr8, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
     ; GCN: $agpr24 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec
-    ; GCN: $vgpr2 = V_ACCVGPR_READ_B32 killed $agpr7, implicit $exec
+    ; GCN: $vgpr2 = V_ACCVGPR_READ_B32 $agpr7, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
     ; GCN: $agpr23 = V_ACCVGPR_WRITE_B32 killed $vgpr2, implicit $exec
-    ; GCN: $vgpr1 = V_ACCVGPR_READ_B32 killed $agpr6, implicit $exec
+    ; GCN: $vgpr1 = V_ACCVGPR_READ_B32 $agpr6, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
     ; GCN: $agpr22 = V_ACCVGPR_WRITE_B32 killed $vgpr1, implicit $exec
-    ; GCN: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr5, implicit $exec
+    ; GCN: $vgpr0 = V_ACCVGPR_READ_B32 $agpr5, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
     ; GCN: $agpr21 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec
-    ; GCN: $vgpr2 = V_ACCVGPR_READ_B32 killed $agpr4, implicit $exec
+    ; GCN: $vgpr2 = V_ACCVGPR_READ_B32 $agpr4, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
     ; GCN: $agpr20 = V_ACCVGPR_WRITE_B32 killed $vgpr2, implicit $exec
-    ; GCN: $vgpr1 = V_ACCVGPR_READ_B32 killed $agpr3, implicit $exec
+    ; GCN: $vgpr1 = V_ACCVGPR_READ_B32 $agpr3, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
     ; GCN: $agpr19 = V_ACCVGPR_WRITE_B32 killed $vgpr1, implicit $exec
-    ; GCN: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr2, implicit $exec
+    ; GCN: $vgpr0 = V_ACCVGPR_READ_B32 $agpr2, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
     ; GCN: $agpr18 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec
-    ; GCN: $vgpr2 = V_ACCVGPR_READ_B32 killed $agpr1, implicit $exec
+    ; GCN: $vgpr2 = V_ACCVGPR_READ_B32 $agpr1, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
     ; GCN: $agpr17 = V_ACCVGPR_WRITE_B32 killed $vgpr2, implicit $exec
-    ; GCN: $vgpr1 = V_ACCVGPR_READ_B32 killed $agpr0, implicit $exec
+    ; GCN: $vgpr1 = V_ACCVGPR_READ_B32 killed $agpr0, implicit $exec, implicit killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
     ; GCN: $agpr16 = V_ACCVGPR_WRITE_B32 killed $vgpr1, implicit $exec, implicit $exec
     ; GCN: S_ENDPGM 0, implicit $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
     $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = IMPLICIT_DEF
@@ -599,3 +624,100 @@ body:             |
     $agpr0 = COPY killed $agpr1, implicit $exec
     S_ENDPGM 0, implicit $agpr0
 ...
+
+---
+name: copy_sgpr_to_agpr_tuple
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $agpr0, $sgpr2_sgpr3
+
+    ; GCN-LABEL: name: copy_sgpr_to_agpr_tuple
+    ; GCN: liveins: $agpr0, $sgpr2_sgpr3
+    ; GCN: S_NOP 0, implicit-def dead $sgpr0_sgpr1
+    ; GCN: $vgpr1 = V_MOV_B32_e32 $sgpr3, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3
+    ; GCN: $agpr7 = V_ACCVGPR_WRITE_B32 killed $vgpr1, implicit $exec, implicit-def $agpr4_agpr5_agpr6_agpr7
+    ; GCN: $vgpr0 = V_MOV_B32_e32 $sgpr2, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3
+    ; GCN: $agpr6 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec
+    ; GCN: $vgpr2 = V_MOV_B32_e32 $sgpr1, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3
+    ; GCN: $agpr5 = V_ACCVGPR_WRITE_B32 killed $vgpr2, implicit $exec
+    ; GCN: $vgpr1 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3
+    ; GCN: $agpr4 = V_ACCVGPR_WRITE_B32 killed $vgpr1, implicit $exec, implicit $exec
+    ; GCN: S_ENDPGM 0, implicit $agpr4_agpr5_agpr6_agpr7, implicit $sgpr0_sgpr1_sgpr2_sgpr3
+    S_NOP 0, implicit-def dead $sgpr0_sgpr1
+    renamable $agpr4_agpr5_agpr6_agpr7 = COPY renamable $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec
+    S_ENDPGM 0, implicit $agpr4_agpr5_agpr6_agpr7, implicit $sgpr0_sgpr1_sgpr2_sgpr3
+...
+---
+name: copy_sgpr_to_agpr_tuple_kill
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $agpr0, $sgpr2_sgpr3
+
+    ; GCN-LABEL: name: copy_sgpr_to_agpr_tuple_kill
+    ; GCN: liveins: $agpr0, $sgpr2_sgpr3
+    ; GCN: S_NOP 0, implicit-def dead $sgpr0_sgpr1
+    ; GCN: $vgpr1 = V_MOV_B32_e32 $sgpr3, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3
+    ; GCN: $agpr7 = V_ACCVGPR_WRITE_B32 killed $vgpr1, implicit $exec, implicit-def $agpr4_agpr5_agpr6_agpr7
+    ; GCN: $vgpr0 = V_MOV_B32_e32 $sgpr2, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3
+    ; GCN: $agpr6 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec
+    ; GCN: $vgpr2 = V_MOV_B32_e32 $sgpr1, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3
+    ; GCN: $agpr5 = V_ACCVGPR_WRITE_B32 killed $vgpr2, implicit $exec
+    ; GCN: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit killed $sgpr0_sgpr1_sgpr2_sgpr3
+    ; GCN: $agpr4 = V_ACCVGPR_WRITE_B32 killed $vgpr1, implicit $exec, implicit $exec
+    ; GCN: S_ENDPGM 0, implicit $agpr4_agpr5_agpr6_agpr7
+    S_NOP 0, implicit-def dead $sgpr0_sgpr1
+    renamable $agpr4_agpr5_agpr6_agpr7 = COPY renamable killed $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec
+    S_ENDPGM 0, implicit $agpr4_agpr5_agpr6_agpr7
+...
+
+---
+name: copy_agpr_to_agpr_tuple
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $agpr0, $agpr2_agpr3
+
+    ; GCN-LABEL: name: copy_agpr_to_agpr_tuple
+    ; GCN: liveins: $agpr0, $agpr2_agpr3
+    ; GCN: S_NOP 0, implicit-def dead $agpr0_agpr1
+    ; GCN: $vgpr1 = V_ACCVGPR_READ_B32 $agpr3, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3
+    ; GCN: $agpr7 = V_ACCVGPR_WRITE_B32 killed $vgpr1, implicit $exec, implicit-def $agpr4_agpr5_agpr6_agpr7
+    ; GCN: $vgpr0 = V_ACCVGPR_READ_B32 $agpr2, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3
+    ; GCN: $agpr6 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec
+    ; GCN: $vgpr2 = V_ACCVGPR_READ_B32 $agpr1, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3
+    ; GCN: $agpr5 = V_ACCVGPR_WRITE_B32 killed $vgpr2, implicit $exec
+    ; GCN: $vgpr1 = V_ACCVGPR_READ_B32 $agpr0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3
+    ; GCN: $agpr4 = V_ACCVGPR_WRITE_B32 killed $vgpr1, implicit $exec, implicit $exec
+    ; GCN: S_ENDPGM 0, implicit $agpr4_agpr5_agpr6_agpr7, implicit $agpr0_agpr1_agpr2_agpr3
+    S_NOP 0, implicit-def dead $agpr0_agpr1
+    renamable $agpr4_agpr5_agpr6_agpr7 = COPY renamable $agpr0_agpr1_agpr2_agpr3, implicit $exec
+    S_ENDPGM 0, implicit $agpr4_agpr5_agpr6_agpr7, implicit $agpr0_agpr1_agpr2_agpr3
+...
+
+---
+name: copy_agpr_to_agpr_tuple_kill
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $agpr0, $agpr2_agpr3
+
+    ; GCN-LABEL: name: copy_agpr_to_agpr_tuple_kill
+    ; GCN: liveins: $agpr0, $agpr2_agpr3
+    ; GCN: S_NOP 0, implicit-def dead $agpr0_agpr1
+    ; GCN: $vgpr1 = V_ACCVGPR_READ_B32 $agpr3, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3
+    ; GCN: $agpr7 = V_ACCVGPR_WRITE_B32 killed $vgpr1, implicit $exec, implicit-def $agpr4_agpr5_agpr6_agpr7
+    ; GCN: $vgpr0 = V_ACCVGPR_READ_B32 $agpr2, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3
+    ; GCN: $agpr6 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec
+    ; GCN: $vgpr2 = V_ACCVGPR_READ_B32 $agpr1, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3
+    ; GCN: $agpr5 = V_ACCVGPR_WRITE_B32 killed $vgpr2, implicit $exec
+    ; GCN: $vgpr1 = V_ACCVGPR_READ_B32 killed $agpr0, implicit $exec, implicit killed $agpr0_agpr1_agpr2_agpr3
+    ; GCN: $agpr4 = V_ACCVGPR_WRITE_B32 killed $vgpr1, implicit $exec, implicit $exec
+    ; GCN: S_ENDPGM 0, implicit $agpr4_agpr5_agpr6_agpr7
+    S_NOP 0, implicit-def dead $agpr0_agpr1
+    renamable $agpr4_agpr5_agpr6_agpr7 = COPY renamable killed $agpr0_agpr1_agpr2_agpr3, implicit $exec
+    S_ENDPGM 0, implicit $agpr4_agpr5_agpr6_agpr7
+...
+
+---

From a34a8d52605a203cfde7d9a8600e1e3e0ee63df7 Mon Sep 17 00:00:00 2001
From: Rahul Joshi <jurahul@google.com>
Date: Thu, 30 Jul 2020 14:18:33 -0700
Subject: [PATCH 0677/1035] [MLIR][NFC] Add SymbolUse::UseRange::empty()

Differential Revision: https://reviews.llvm.org/D84984
---
 mlir/include/mlir/IR/SymbolTable.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/mlir/include/mlir/IR/SymbolTable.h b/mlir/include/mlir/IR/SymbolTable.h
index 7e52011f81ffb..4e41fe55d2c59 100644
--- a/mlir/include/mlir/IR/SymbolTable.h
+++ b/mlir/include/mlir/IR/SymbolTable.h
@@ -150,6 +150,7 @@ class SymbolTable {
     using iterator = std::vector<SymbolUse>::const_iterator;
     iterator begin() const { return uses.begin(); }
     iterator end() const { return uses.end(); }
+    bool empty() const { return uses.empty(); }
 
   private:
     std::vector<SymbolUse> uses;

From 763671f387fdd00329c5b715d3ec6c62680f3f8e Mon Sep 17 00:00:00 2001
From: Zequan Wu <zequanwu@google.com>
Date: Tue, 21 Jul 2020 13:46:11 -0700
Subject: [PATCH 0678/1035] [COFF] Port CallGraphSort to COFF from ELF

---
 lld/COFF/CMakeLists.txt                |   1 +
 lld/COFF/CallGraphSort.cpp             | 245 +++++++++++++++++++++++++
 lld/COFF/CallGraphSort.h               |  22 +++
 lld/COFF/Config.h                      |  11 ++
 lld/COFF/Driver.cpp                    |  94 +++++++++-
 lld/COFF/InputFiles.cpp                |   5 +
 lld/COFF/InputFiles.h                  |   2 +
 lld/COFF/Options.td                    |  11 ++
 lld/COFF/Writer.cpp                    |  20 +-
 lld/ELF/CallGraphSort.cpp              |   6 +-
 lld/test/COFF/cgprofile-bad-clusters.s |  61 ++++++
 lld/test/COFF/cgprofile-err.s          |  11 ++
 lld/test/COFF/cgprofile-icf.s          |  45 +++++
 lld/test/COFF/cgprofile-obj.s          |  45 +++++
 lld/test/COFF/cgprofile-print.s        |  34 ++++
 lld/test/COFF/cgprofile-txt.s          |  43 +++++
 16 files changed, 646 insertions(+), 10 deletions(-)
 create mode 100644 lld/COFF/CallGraphSort.cpp
 create mode 100644 lld/COFF/CallGraphSort.h
 create mode 100644 lld/test/COFF/cgprofile-bad-clusters.s
 create mode 100644 lld/test/COFF/cgprofile-err.s
 create mode 100644 lld/test/COFF/cgprofile-icf.s
 create mode 100644 lld/test/COFF/cgprofile-obj.s
 create mode 100644 lld/test/COFF/cgprofile-print.s
 create mode 100644 lld/test/COFF/cgprofile-txt.s

diff --git a/lld/COFF/CMakeLists.txt b/lld/COFF/CMakeLists.txt
index 796f7a82a3de1..bbcd337b210f3 100644
--- a/lld/COFF/CMakeLists.txt
+++ b/lld/COFF/CMakeLists.txt
@@ -3,6 +3,7 @@ tablegen(LLVM Options.inc -gen-opt-parser-defs)
 add_public_tablegen_target(COFFOptionsTableGen)
 
 add_lld_library(lldCOFF
+  CallGraphSort.cpp
   Chunks.cpp
   DebugTypes.cpp
   DLL.cpp
diff --git a/lld/COFF/CallGraphSort.cpp b/lld/COFF/CallGraphSort.cpp
new file mode 100644
index 0000000000000..d3e5312ce7fdd
--- /dev/null
+++ b/lld/COFF/CallGraphSort.cpp
@@ -0,0 +1,245 @@
+//===- CallGraphSort.cpp --------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// This is based on the ELF port, see ELF/CallGraphSort.cpp for the details
+/// about the algorithm.
+///
+//===----------------------------------------------------------------------===//
+
+#include "CallGraphSort.h"
+#include "InputFiles.h"
+#include "SymbolTable.h"
+#include "Symbols.h"
+#include "lld/Common/ErrorHandler.h"
+
+#include <numeric>
+
+using namespace llvm;
+using namespace lld;
+using namespace lld::coff;
+
+namespace {
+struct Edge {
+  int from;
+  uint64_t weight;
+};
+
+struct Cluster {
+  Cluster(int sec, size_t s) : next(sec), prev(sec), size(s) {}
+
+  double getDensity() const {
+    if (size == 0)
+      return 0;
+    return double(weight) / double(size);
+  }
+
+  int next;
+  int prev;
+  uint64_t size;
+  uint64_t weight = 0;
+  uint64_t initialWeight = 0;
+  Edge bestPred = {-1, 0};
+};
+
+class CallGraphSort {
+public:
+  CallGraphSort();
+
+  DenseMap<const SectionChunk *, int> run();
+
+private:
+  std::vector<Cluster> clusters;
+  std::vector<const SectionChunk *> sections;
+};
+
+// Maximum amount the combined cluster density can be worse than the original
+// cluster to consider merging.
+constexpr int MAX_DENSITY_DEGRADATION = 8;
+
+// Maximum cluster size in bytes.
+constexpr uint64_t MAX_CLUSTER_SIZE = 1024 * 1024;
+} // end anonymous namespace
+
+using SectionPair = std::pair<const SectionChunk *, const SectionChunk *>;
+
+// Take the edge list in Config->CallGraphProfile, resolve symbol names to
+// Symbols, and generate a graph between InputSections with the provided
+// weights.
+CallGraphSort::CallGraphSort() {
+  MapVector<SectionPair, uint64_t> &profile = config->callGraphProfile;
+  DenseMap<const SectionChunk *, int> secToCluster;
+
+  auto getOrCreateNode = [&](const SectionChunk *isec) -> int {
+    auto res = secToCluster.try_emplace(isec, clusters.size());
+    if (res.second) {
+      sections.push_back(isec);
+      clusters.emplace_back(clusters.size(), isec->getSize());
+    }
+    return res.first->second;
+  };
+
+  // Create the graph.
+  for (std::pair<SectionPair, uint64_t> &c : profile) {
+    const auto *fromSec = cast<SectionChunk>(c.first.first->repl);
+    const auto *toSec = cast<SectionChunk>(c.first.second->repl);
+    uint64_t weight = c.second;
+
+    // Ignore edges between input sections belonging to different output
+    // sections.  This is done because otherwise we would end up with clusters
+    // containing input sections that can't actually be placed adjacently in the
+    // output.  This messes with the cluster size and density calculations.  We
+    // would also end up moving input sections in other output sections without
+    // moving them closer to what calls them.
+    if (fromSec->getOutputSection() != toSec->getOutputSection())
+      continue;
+
+    int from = getOrCreateNode(fromSec);
+    int to = getOrCreateNode(toSec);
+
+    clusters[to].weight += weight;
+
+    if (from == to)
+      continue;
+
+    // Remember the best edge.
+    Cluster &toC = clusters[to];
+    if (toC.bestPred.from == -1 || toC.bestPred.weight < weight) {
+      toC.bestPred.from = from;
+      toC.bestPred.weight = weight;
+    }
+  }
+  for (Cluster &c : clusters)
+    c.initialWeight = c.weight;
+}
+
+// It's bad to merge clusters which would degrade the density too much.
+static bool isNewDensityBad(Cluster &a, Cluster &b) {
+  double newDensity = double(a.weight + b.weight) / double(a.size + b.size);
+  return newDensity < a.getDensity() / MAX_DENSITY_DEGRADATION;
+}
+
+// Find the leader of V's belonged cluster (represented as an equivalence
+// class). We apply union-find path-halving technique (simple to implement) in
+// the meantime as it decreases depths and the time complexity.
+static int getLeader(std::vector<int> &leaders, int v) {
+  while (leaders[v] != v) {
+    leaders[v] = leaders[leaders[v]];
+    v = leaders[v];
+  }
+  return v;
+}
+
+static void mergeClusters(std::vector<Cluster> &cs, Cluster &into, int intoIdx,
+                          Cluster &from, int fromIdx) {
+  int tail1 = into.prev, tail2 = from.prev;
+  into.prev = tail2;
+  cs[tail2].next = intoIdx;
+  from.prev = tail1;
+  cs[tail1].next = fromIdx;
+  into.size += from.size;
+  into.weight += from.weight;
+  from.size = 0;
+  from.weight = 0;
+}
+
+// Group InputSections into clusters using the Call-Chain Clustering heuristic
+// then sort the clusters by density.
+DenseMap<const SectionChunk *, int> CallGraphSort::run() {
+  std::vector<int> sorted(clusters.size());
+  std::vector<int> leaders(clusters.size());
+
+  std::iota(leaders.begin(), leaders.end(), 0);
+  std::iota(sorted.begin(), sorted.end(), 0);
+  llvm::stable_sort(sorted, [&](int a, int b) {
+    return clusters[a].getDensity() > clusters[b].getDensity();
+  });
+
+  for (int l : sorted) {
+    // The cluster index is the same as the index of its leader here because
+    // clusters[L] has not been merged into another cluster yet.
+    Cluster &c = clusters[l];
+
+    // Don't consider merging if the edge is unlikely.
+    if (c.bestPred.from == -1 || c.bestPred.weight * 10 <= c.initialWeight)
+      continue;
+
+    int predL = getLeader(leaders, c.bestPred.from);
+    if (l == predL)
+      continue;
+
+    Cluster *predC = &clusters[predL];
+    if (c.size + predC->size > MAX_CLUSTER_SIZE)
+      continue;
+
+    if (isNewDensityBad(*predC, c))
+      continue;
+
+    leaders[l] = predL;
+    mergeClusters(clusters, *predC, predL, c, l);
+  }
+
+  // Sort remaining non-empty clusters by density.
+  sorted.clear();
+  for (int i = 0, e = (int)clusters.size(); i != e; ++i)
+    if (clusters[i].size > 0)
+      sorted.push_back(i);
+  llvm::stable_sort(sorted, [&](int a, int b) {
+    return clusters[a].getDensity() > clusters[b].getDensity();
+  });
+
+  DenseMap<const SectionChunk *, int> orderMap;
+  // Sections will be sorted by increasing order. Absent sections will have
+  // priority 0 and be placed at the end of sections.
+  int curOrder = INT_MIN;
+  for (int leader : sorted) {
+    for (int i = leader;;) {
+      orderMap[sections[i]] = curOrder++;
+      i = clusters[i].next;
+      if (i == leader)
+        break;
+    }
+  }
+  if (!config->printSymbolOrder.empty()) {
+    std::error_code ec;
+    raw_fd_ostream os(config->printSymbolOrder, ec, sys::fs::OF_None);
+    if (ec) {
+      error("cannot open " + config->printSymbolOrder + ": " + ec.message());
+      return orderMap;
+    }
+    // Print the symbols ordered by C3, in the order of increasing curOrder
+    // Instead of sorting all the orderMap, just repeat the loops above.
+    for (int leader : sorted)
+      for (int i = leader;;) {
+        const SectionChunk *sc = sections[i];
+
+        // Search all the symbols in the file of the section
+        // and find out a DefinedCOFF symbol with name that is within the
+        // section.
+        for (Symbol *sym : sc->file->getSymbols())
+          if (auto *d = dyn_cast_or_null<DefinedCOFF>(sym))
+            // Filter out non-COMDAT symbols and section symbols.
+            if (d->isCOMDAT && !d->getCOFFSymbol().isSection() &&
+                sc == d->getChunk())
+              os << sym->getName() << "\n";
+        i = clusters[i].next;
+        if (i == leader)
+          break;
+      }
+  }
+
+  return orderMap;
+}
+
+// Sort sections by the profile data provided by  /call-graph-ordering-file
+//
+// This first builds a call graph based on the profile data then merges sections
+// according to the C³ heuristic. All clusters are then sorted by a density
+// metric to further improve locality.
+DenseMap<const SectionChunk *, int> coff::computeCallGraphProfileOrder() {
+  return CallGraphSort().run();
+}
diff --git a/lld/COFF/CallGraphSort.h b/lld/COFF/CallGraphSort.h
new file mode 100644
index 0000000000000..e4f372137448c
--- /dev/null
+++ b/lld/COFF/CallGraphSort.h
@@ -0,0 +1,22 @@
+//===- CallGraphSort.h ------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLD_COFF_CALL_GRAPH_SORT_H
+#define LLD_COFF_CALL_GRAPH_SORT_H
+
+#include "llvm/ADT/DenseMap.h"
+
+namespace lld {
+namespace coff {
+class SectionChunk;
+
+llvm::DenseMap<const SectionChunk *, int> computeCallGraphProfileOrder();
+} // namespace coff
+} // namespace lld
+
+#endif
diff --git a/lld/COFF/Config.h b/lld/COFF/Config.h
index 7c439176f3a45..286b67b437a89 100644
--- a/lld/COFF/Config.h
+++ b/lld/COFF/Config.h
@@ -9,6 +9,7 @@
 #ifndef LLD_COFF_CONFIG_H
 #define LLD_COFF_CONFIG_H
 
+#include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Object/COFF.h"
@@ -29,6 +30,7 @@ class DefinedRelative;
 class StringChunk;
 class Symbol;
 class InputFile;
+class SectionChunk;
 
 // Short aliases.
 static const auto AMD64 = llvm::COFF::IMAGE_FILE_MACHINE_AMD64;
@@ -201,6 +203,15 @@ struct Configuration {
   // Used for /lto-obj-path:
   llvm::StringRef ltoObjPath;
 
+  // Used for /call-graph-ordering-file:
+  llvm::MapVector<std::pair<const SectionChunk *, const SectionChunk *>,
+                  uint64_t>
+      callGraphProfile;
+  bool callGraphProfileSort = false;
+
+  // Used for /print-symbol-order:
+  StringRef printSymbolOrder;
+
   uint64_t align = 4096;
   uint64_t imageBase = -1;
   uint64_t fileAlign = 512;
diff --git a/lld/COFF/Driver.cpp b/lld/COFF/Driver.cpp
index 9ceccef867797..55e97d50c2263 100644
--- a/lld/COFF/Driver.cpp
+++ b/lld/COFF/Driver.cpp
@@ -34,6 +34,7 @@
 #include "llvm/Option/Arg.h"
 #include "llvm/Option/ArgList.h"
 #include "llvm/Option/Option.h"
+#include "llvm/Support/BinaryStreamReader.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/LEB128.h"
@@ -924,6 +925,75 @@ static void parseOrderFile(StringRef arg) {
   }
 }
 
+static void parseCallGraphFile(StringRef path) {
+  std::unique_ptr<MemoryBuffer> mb = CHECK(
+      MemoryBuffer::getFile(path, -1, false, true), "could not open " + path);
+
+  // Build a map from symbol name to section.
+  DenseMap<StringRef, Symbol *> map;
+  for (ObjFile *file : ObjFile::instances)
+    for (Symbol *sym : file->getSymbols())
+      if (sym)
+        map[sym->getName()] = sym;
+
+  auto findSection = [&](StringRef name) -> SectionChunk * {
+    Symbol *sym = map.lookup(name);
+    if (!sym) {
+      if (config->warnMissingOrderSymbol)
+        warn(path + ": no such symbol: " + name);
+      return nullptr;
+    }
+
+    if (DefinedCOFF *dr = dyn_cast_or_null<DefinedCOFF>(sym))
+      return dyn_cast_or_null<SectionChunk>(dr->getChunk());
+    return nullptr;
+  };
+
+  for (StringRef line : args::getLines(*mb)) {
+    SmallVector<StringRef, 3> fields;
+    line.split(fields, ' ');
+    uint64_t count;
+
+    if (fields.size() != 3 || !to_integer(fields[2], count)) {
+      error(path + ": parse error");
+      return;
+    }
+
+    if (SectionChunk *from = findSection(fields[0]))
+      if (SectionChunk *to = findSection(fields[1]))
+        config->callGraphProfile[{from, to}] += count;
+  }
+}
+
+static void readCallGraphsFromObjectFiles() {
+  for (ObjFile *obj : ObjFile::instances) {
+    if (obj->callgraphSec) {
+      ArrayRef<uint8_t> contents;
+      cantFail(
+          obj->getCOFFObj()->getSectionContents(obj->callgraphSec, contents));
+      BinaryStreamReader reader(contents, support::little);
+      while (!reader.empty()) {
+        uint32_t fromIndex, toIndex;
+        uint64_t count;
+        if (Error err = reader.readInteger(fromIndex))
+          fatal(toString(obj) + ": Expected 32-bit integer");
+        if (Error err = reader.readInteger(toIndex))
+          fatal(toString(obj) + ": Expected 32-bit integer");
+        if (Error err = reader.readInteger(count))
+          fatal(toString(obj) + ": Expected 64-bit integer");
+        auto *fromSym = dyn_cast_or_null<Defined>(obj->getSymbol(fromIndex));
+        auto *toSym = dyn_cast_or_null<Defined>(obj->getSymbol(toIndex));
+        if (!fromSym || !toSym)
+          continue;
+        auto *from = dyn_cast_or_null<SectionChunk>(fromSym->getChunk());
+        auto *to = dyn_cast_or_null<SectionChunk>(toSym->getChunk());
+        if (from && to)
+          config->callGraphProfile[{from, to}] += count;
+      }
+    }
+  }
+}
+
 static void markAddrsig(Symbol *s) {
   if (auto *d = dyn_cast_or_null<Defined>(s))
     if (SectionChunk *c = dyn_cast_or_null<SectionChunk>(d->getChunk()))
@@ -1587,9 +1657,11 @@ void LinkerDriver::link(ArrayRef<const char *> argsArr) {
       args.hasFlag(OPT_auto_import, OPT_auto_import_no, config->mingw);
   config->pseudoRelocs = args.hasFlag(
       OPT_runtime_pseudo_reloc, OPT_runtime_pseudo_reloc_no, config->mingw);
+  config->callGraphProfileSort = args.hasFlag(
+      OPT_call_graph_profile_sort, OPT_call_graph_profile_sort_no, true);
 
-  // Don't warn about long section names, such as .debug_info, for mingw or when
-  // -debug:dwarf is requested.
+  // Don't warn about long section names, such as .debug_info, for mingw or
+  // when -debug:dwarf is requested.
   if (config->mingw || config->debugDwarf)
     config->warnLongSectionNames = false;
 
@@ -2024,8 +2096,24 @@ void LinkerDriver::link(ArrayRef<const char *> argsArr) {
   // Handle /order. We want to do this at this moment because we
   // need a complete list of comdat sections to warn on nonexistent
   // functions.
-  if (auto *arg = args.getLastArg(OPT_order))
+  if (auto *arg = args.getLastArg(OPT_order)) {
+    if (args.hasArg(OPT_call_graph_ordering_file))
+      error("/order and /call-graph-order-file may not be used together");
     parseOrderFile(arg->getValue());
+    config->callGraphProfileSort = false;
+  }
+
+  // Handle /call-graph-ordering-file and /call-graph-profile-sort (default on).
+  if (config->callGraphProfileSort) {
+    if (auto *arg = args.getLastArg(OPT_call_graph_ordering_file)) {
+      parseCallGraphFile(arg->getValue());
+    }
+    readCallGraphsFromObjectFiles();
+  }
+
+  // Handle /print-symbol-order.
+  if (auto *arg = args.getLastArg(OPT_print_symbol_order))
+    config->printSymbolOrder = arg->getValue();
 
   // Identify unreferenced COMDAT sections.
   if (config->doGC)
diff --git a/lld/COFF/InputFiles.cpp b/lld/COFF/InputFiles.cpp
index 4346b3a2ffa7b..0bcc6c940bbab 100644
--- a/lld/COFF/InputFiles.cpp
+++ b/lld/COFF/InputFiles.cpp
@@ -249,6 +249,11 @@ SectionChunk *ObjFile::readSection(uint32_t sectionNumber,
     return nullptr;
   }
 
+  if (name == ".llvm.call-graph-profile") {
+    callgraphSec = sec;
+    return nullptr;
+  }
+
   // Object files may have DWARF debug info or MS CodeView debug info
   // (or both).
   //
diff --git a/lld/COFF/InputFiles.h b/lld/COFF/InputFiles.h
index 50323f596e2cc..1e0b97a82be29 100644
--- a/lld/COFF/InputFiles.h
+++ b/lld/COFF/InputFiles.h
@@ -191,6 +191,8 @@ class ObjFile : public InputFile {
 
   const coff_section *addrsigSec = nullptr;
 
+  const coff_section *callgraphSec = nullptr;
+
   // When using Microsoft precompiled headers, this is the PCH's key.
   // The same key is used by both the precompiled object, and objects using the
   // precompiled object. Any difference indicates out-of-date objects.
diff --git a/lld/COFF/Options.td b/lld/COFF/Options.td
index 087d53b5d2ddd..d1badf0fdd2f0 100644
--- a/lld/COFF/Options.td
+++ b/lld/COFF/Options.td
@@ -235,6 +235,17 @@ def dash_dash_version : Flag<["--"], "version">,
 def threads
     : P<"threads", "Number of threads. '1' disables multi-threading. By "
                    "default all available hardware threads are used">;
+def call_graph_ordering_file: P<
+    "call-graph-ordering-file", 
+    "Layout sections to optimize the given callgraph">;
+defm call_graph_profile_sort: B<
+    "call-graph-profile-sort",
+    "Reorder sections with call graph profile (default)",
+    "Do not reorder sections with call graph profile">;
+def print_symbol_order: P<
+    "print-symbol-order",
+    "Print a symbol order specified by /call-graph-ordering-file and "
+    "/call-graph-profile-sort into the specified file">;
 
 // Flags for debugging
 def lldmap : F<"lldmap">;
diff --git a/lld/COFF/Writer.cpp b/lld/COFF/Writer.cpp
index 082de5b8c1d62..36ecdcd13512c 100644
--- a/lld/COFF/Writer.cpp
+++ b/lld/COFF/Writer.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "Writer.h"
+#include "CallGraphSort.h"
 #include "Config.h"
 #include "DLL.h"
 #include "InputFiles.h"
@@ -229,6 +230,7 @@ class Writer {
   void setSectionPermissions();
   void writeSections();
   void writeBuildId();
+  void sortSections();
   void sortExceptionTable();
   void sortCRTSectionChunks(std::vector<Chunk *> &chunks);
   void addSyntheticIdata();
@@ -798,6 +800,19 @@ static bool shouldStripSectionSuffix(SectionChunk *sc, StringRef name) {
          name.startswith(".xdata$") || name.startswith(".eh_frame$");
 }
 
+void Writer::sortSections() {
+  if (!config->callGraphProfile.empty()) {
+    DenseMap<const SectionChunk *, int> order = computeCallGraphProfileOrder();
+    for (auto it : order) {
+      if (DefinedRegular *sym = it.first->sym)
+        config->order[sym->getName()] = it.second;
+    }
+  }
+  if (!config->order.empty())
+    for (auto it : partialSections)
+      sortBySectionOrder(it.second->chunks);
+}
+
 // Create output section objects and add them to OutputSections.
 void Writer::createSections() {
   // First, create the builtin sections.
@@ -861,10 +876,7 @@ void Writer::createSections() {
   if (hasIdata)
     addSyntheticIdata();
 
-  // Process an /order option.
-  if (!config->order.empty())
-    for (auto it : partialSections)
-      sortBySectionOrder(it.second->chunks);
+  sortSections();
 
   if (hasIdata)
     locateImportTables();
diff --git a/lld/ELF/CallGraphSort.cpp b/lld/ELF/CallGraphSort.cpp
index 21c641b5161f8..15da4d2414ab4 100644
--- a/lld/ELF/CallGraphSort.cpp
+++ b/lld/ELF/CallGraphSort.cpp
@@ -68,7 +68,7 @@ struct Cluster {
 
   int next;
   int prev;
-  size_t size = 0;
+  uint64_t size;
   uint64_t weight = 0;
   uint64_t initialWeight = 0;
   Edge bestPred = {-1, 0};
@@ -223,14 +223,14 @@ DenseMap<const InputSectionBase *, int> CallGraphSort::run() {
 
   DenseMap<const InputSectionBase *, int> orderMap;
   int curOrder = 1;
-  for (int leader : sorted)
+  for (int leader : sorted) {
     for (int i = leader;;) {
       orderMap[sections[i]] = curOrder++;
       i = clusters[i].next;
       if (i == leader)
         break;
     }
-
+  }
   if (!config->printSymbolOrder.empty()) {
     std::error_code ec;
     raw_fd_ostream os(config->printSymbolOrder, ec, sys::fs::OF_None);
diff --git a/lld/test/COFF/cgprofile-bad-clusters.s b/lld/test/COFF/cgprofile-bad-clusters.s
new file mode 100644
index 0000000000000..12c05424095df
--- /dev/null
+++ b/lld/test/COFF/cgprofile-bad-clusters.s
@@ -0,0 +1,61 @@
+# REQUIRES: x86
+# This test checks that CallGraphSort ignores edges that would form "bad"
+# clusters.
+
+# RUN: llvm-mc -filetype=obj -triple=x86_64-pc-win32 %s -o %t
+# RUN: echo "A C 1" > %t.call_graph
+# RUN: echo "E B 4" >> %t.call_graph
+# RUN: echo "C D 2" >> %t.call_graph
+# RUN: echo "B D 1" >> %t.call_graph
+# RUN: echo "F G 6" >> %t.call_graph
+# RUN: echo "G H 5" >> %t.call_graph
+# RUN: echo "H I 4" >> %t.call_graph
+# RUN: lld-link /subsystem:console /entry:A %t /call-graph-ordering-file:%t.call_graph /out:%t2 /debug:symtab
+# RUN: llvm-nm --numeric-sort %t2 | FileCheck %s
+
+    .section    .text,"ax",one_only,A
+    .globl A
+A:
+    retq
+
+    .section    .text,"ax",one_only,D
+D:
+    .fill 1000, 1, 0
+
+    .section    .text,"ax",one_only,E
+E:
+    retq
+
+    .section    .text,"ax",one_only,C
+C:
+    retq
+
+    .section    .text,"ax",one_only,B
+B:
+    .fill 1000, 1, 0
+
+    .section    .text,"ax",one_only,F
+F:
+    .fill (1024 * 1024) - 1, 1, 0
+
+    .section    .text,"ax",one_only,G
+G:
+    retq
+
+    .section    .text,"ax",one_only,H
+H:
+    retq
+
+    .section    .text,"ax",one_only,I
+I:
+    .fill 13, 1, 0
+
+# CHECK: 140001000 t H
+# CHECK: 140001001 t I
+# CHECK: 14000100e T A
+# CHECK: 14000100f t C
+# CHECK: 140001010 t E
+# CHECK: 140001011 t B
+# CHECK: 1400013f9 t D
+# CHECK: 1400017e1 t F
+# CHECK: 1401017e0 t G
diff --git a/lld/test/COFF/cgprofile-err.s b/lld/test/COFF/cgprofile-err.s
new file mode 100644
index 0000000000000..94c1c2a688627
--- /dev/null
+++ b/lld/test/COFF/cgprofile-err.s
@@ -0,0 +1,11 @@
+# REQUIRES: x86
+
+# RUN: llvm-mc -filetype=obj -triple=x86_64-pc-win32 %s -o %t
+
+# RUN: echo "A B C 100" > %t.call_graph
+# RUN: not lld-link /dll /noentry  /subsystem:console %t /call-graph-ordering-file:%t.call_graph /out:/dev/null 2>&1 | FileCheck %s
+
+# CHECK: {{.*}}.call_graph: parse error
+
+# RUN: echo "A B C" > %t.call_graph
+# RUN: not lld-link /dll /noentry  /subsystem:console %t /call-graph-ordering-file:%t.call_graph /out:/dev/null 2>&1 | FileCheck %s
diff --git a/lld/test/COFF/cgprofile-icf.s b/lld/test/COFF/cgprofile-icf.s
new file mode 100644
index 0000000000000..19cdd0f26e117
--- /dev/null
+++ b/lld/test/COFF/cgprofile-icf.s
@@ -0,0 +1,45 @@
+# REQUIRES: x86
+# Test the compatibility of ICF and cgprofile.
+
+# RUN: llvm-mc -filetype=obj -triple=x86_64-pc-win32 %s -o %t
+# RUN: echo "A B 100" > %t.call_graph
+# RUN: echo "A C 40" >> %t.call_graph
+# RUN: echo "C D 61" >> %t.call_graph
+# RUN: lld-link /subsystem:console /entry:A %t /call-graph-ordering-file:%t.call_graph /out:%t2 /debug:symtab /opt:icf
+# RUN: llvm-nm --numeric-sort %t2 | FileCheck %s
+# RUN: lld-link /subsystem:console /entry:A %t /call-graph-ordering-file:%t.call_graph /out:%t2 /debug:symtab
+# RUN: llvm-nm --numeric-sort %t2 | FileCheck %s --check-prefix=NOICF
+
+    .section    .text,"x",one_only,D
+    .globl  D
+D:
+    mov $60, %rax
+    retq
+
+    .section    .text,"x",one_only,C
+    .globl  C
+C:
+    mov $60, %rax
+    retq
+
+    .section    .text,"x",one_only,B
+    .globl  B
+B:
+    mov $2, %rax
+    retq
+
+    .section    .text,"x",one_only,A
+    .globl  A
+A:
+    mov $42, %rax
+    retq
+
+# CHECK: 140001000 T A
+# CHECK: 140001008 T C
+# CHECK: 140001008 T D
+# CHECK: 140001010 T B
+
+# NOICF: 140001000 T A
+# NOICF: 140001008 T B
+# NOICF: 140001010 T C
+# NOICF: 140001018 T D
diff --git a/lld/test/COFF/cgprofile-obj.s b/lld/test/COFF/cgprofile-obj.s
new file mode 100644
index 0000000000000..b267850c46382
--- /dev/null
+++ b/lld/test/COFF/cgprofile-obj.s
@@ -0,0 +1,45 @@
+# REQUIRES: x86
+
+# RUN: llvm-mc -filetype=obj -triple=x86_64-pc-win32 %s -o %t
+# RUN: lld-link /subsystem:console /entry:A %t /out:%t2 /debug:symtab
+# RUN: llvm-nm --numeric-sort %t2 | FileCheck %s
+# RUN: lld-link /call-graph-profile-sort:no /subsystem:console /entry:A %t /out:%t3 /debug:symtab
+# RUN: llvm-nm --numeric-sort %t3 | FileCheck %s --check-prefix=NO-CG
+
+    .section    .text,"ax", one_only, D
+D:
+ retq
+
+    .section    .text,"ax", one_only, C
+    .globl  C
+C:
+ retq
+
+    .section    .text,"ax", one_only, B
+    .globl  B
+B:
+ retq
+
+    .section    .text,"ax", one_only, A
+    .globl  A
+A:
+Aa:
+ retq
+
+    .cg_profile A, B, 10
+    .cg_profile A, B, 10
+    .cg_profile Aa, B, 80
+    .cg_profile A, C, 40
+    .cg_profile B, C, 30
+    .cg_profile C, D, 90
+
+# CHECK: 140001000 T A
+# CHECK: 140001001 T B
+# CHECK: 140001002 T C
+# CHECK: 140001003 t D
+
+
+# NO-CG: 140001000 t D
+# NO-CG: 140001001 T C
+# NO-CG: 140001002 T B
+# NO-CG: 140001003 T A
diff --git a/lld/test/COFF/cgprofile-print.s b/lld/test/COFF/cgprofile-print.s
new file mode 100644
index 0000000000000..e82185c0f30b1
--- /dev/null
+++ b/lld/test/COFF/cgprofile-print.s
@@ -0,0 +1,34 @@
+# REQUIRES: x86
+
+# RUN: llvm-mc -filetype=obj -triple=x86_64-pc-win32 %s -o %t
+# RUN: echo "A B 5" > %t.call_graph
+# RUN: echo "B C 50" >> %t.call_graph
+# RUN: echo "C D 40" >> %t.call_graph
+# RUN: echo "D B 10" >> %t.call_graph
+# RUN: lld-link /subsystem:console /entry:A %t /call-graph-ordering-file:%t.call_graph /out:%t2 /print-symbol-order:%t3
+# RUN: FileCheck %s --input-file %t3
+
+# CHECK: B
+# CHECK-NEXT: C
+# CHECK-NEXT: D
+# CHECK-NEXT: A
+
+.section    .text, "x", one_only, A
+.globl  A
+A:
+ nop
+
+.section    .text, "x", one_only, B
+.globl  B
+B:
+ nop
+
+.section    .text, "x", one_only, C
+.globl  C
+C:
+ nop
+
+.section    .text, "x", one_only, D
+.globl  D
+D:
+ nop
diff --git a/lld/test/COFF/cgprofile-txt.s b/lld/test/COFF/cgprofile-txt.s
new file mode 100644
index 0000000000000..49cade9dc52de
--- /dev/null
+++ b/lld/test/COFF/cgprofile-txt.s
@@ -0,0 +1,43 @@
+# REQUIRES: x86
+# Test correctness of call graph ordering.
+
+# RUN: llvm-mc -filetype=obj -triple=x86_64-pc-win32 %s -o %t
+# RUN: lld-link /subsystem:console /entry:A %t /out:%t2 /debug:symtab
+# RUN: llvm-nm --numeric-sort %t2 | FileCheck %s --check-prefix=NOSORT 
+
+# RUN: echo "A B 5" > %t.call_graph
+# RUN: echo "B C 50" >> %t.call_graph
+# RUN: echo "C D 40" >> %t.call_graph
+# RUN: echo "D B 10" >> %t.call_graph
+# RUN: lld-link /subsystem:console /entry:A %t /call-graph-ordering-file:%t.call_graph /out:%t2 /debug:symtab
+# RUN: llvm-nm --numeric-sort %t2 | FileCheck %s 
+
+# NOSORT: 140001000 T A
+# NOSORT: 140001001 T B
+# NOSORT: 140001002 T C
+# NOSORT: 140001003 T D
+
+# CHECK: 140001000 T B
+# CHECK: 140001001 T C
+# CHECK: 140001002 T D
+# CHECK: 140001003 T A
+
+.section    .text, "x", one_only, A
+.globl  A
+A:
+ nop
+
+.section    .text, "x", one_only, B
+.globl  B
+B:
+ nop
+
+.section    .text, "x", one_only, C
+.globl  C
+C:
+ nop
+
+.section    .text, "x", one_only, D
+.globl  D
+D:
+ nop

From 9f1dcdca71c4109354ec916ad27c24caf269c705 Mon Sep 17 00:00:00 2001
From: Lang Hames <lhames@gmail.com>
Date: Wed, 29 Jul 2020 20:46:56 -0700
Subject: [PATCH 0679/1035] [JITLink] Allow JITLinkContext::notifyResolved to
 return an Error.

This allows clients to detect invalid transformations applied by JITLink passes
(e.g. inserting or removing symbols in unexpected ways) and terminate linking
with an error.

This change is used to simplify the error propagation logic in
ObjectLinkingLayer.
---
 .../llvm/ExecutionEngine/JITLink/JITLink.h    |  6 +++-
 .../JITLink/JITLinkGeneric.cpp                |  4 ++-
 .../Orc/ObjectLinkingLayer.cpp                | 31 +++++++------------
 3 files changed, 20 insertions(+), 21 deletions(-)

diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h b/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h
index 76f9dea4160f7..46b5747507351 100644
--- a/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h
+++ b/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h
@@ -1279,7 +1279,11 @@ class JITLinkContext {
   /// their final memory locations in the target process. At this point the
   /// LinkGraph can be inspected to build a symbol table, however the block
   /// content will not generally have been copied to the target location yet.
-  virtual void notifyResolved(LinkGraph &G) = 0;
+  ///
+  /// If the client detects an error in the LinkGraph state (e.g. unexpected or
+  /// missing symbols) they may return an error here. The error will be
+  /// propagated to notifyFailed and the linker will bail out.
+  virtual Error notifyResolved(LinkGraph &G) = 0;
 
   /// Called by JITLink to notify the context that the object has been
   /// finalized (i.e. emitted to memory and memory permissions set). If all of
diff --git a/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp b/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp
index 1d76a49939dc0..e74dd4d5c8b13 100644
--- a/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp
@@ -67,7 +67,9 @@ void JITLinkerBase::linkPhase1(std::unique_ptr<JITLinkerBase> Self) {
   // Notify client that the defined symbols have been assigned addresses.
   LLVM_DEBUG(
       { dbgs() << "Resolving symbols defined in " << G->getName() << "\n"; });
-  Ctx->notifyResolved(*G);
+
+  if (auto Err = Ctx->notifyResolved(*G))
+    return Ctx->notifyFailed(std::move(Err));
 
   auto ExternalSymbols = getExternalSymbolNames();
 
diff --git a/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp b/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp
index 8ad2e338ca82a..5b828ed84462a 100644
--- a/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp
@@ -97,7 +97,7 @@ class ObjectLinkingLayerJITLinkContext final : public JITLinkContext {
               });
   }
 
-  void notifyResolved(LinkGraph &G) override {
+  Error notifyResolved(LinkGraph &G) override {
     auto &ES = Layer.getExecutionSession();
 
     SymbolFlagsMap ExtraSymbolsToClaim;
@@ -143,7 +143,7 @@ class ObjectLinkingLayerJITLinkContext final : public JITLinkContext {
 
     if (!ExtraSymbolsToClaim.empty())
       if (auto Err = MR.defineMaterializing(ExtraSymbolsToClaim))
-        return notifyFailed(std::move(Err));
+        return Err;
 
     {
 
@@ -169,12 +169,9 @@ class ObjectLinkingLayerJITLinkContext final : public JITLinkContext {
       }
 
       // If there were missing symbols then report the error.
-      if (!MissingSymbols.empty()) {
-        ES.reportError(make_error<MissingSymbolDefinitions>(
-            G.getName(), std::move(MissingSymbols)));
-        MR.failMaterialization();
-        return;
-      }
+      if (!MissingSymbols.empty())
+        return make_error<MissingSymbolDefinitions>(G.getName(),
+                                                    std::move(MissingSymbols));
 
       // If there are more definitions than expected, add them to the
       // ExtraSymbols vector.
@@ -186,20 +183,16 @@ class ObjectLinkingLayerJITLinkContext final : public JITLinkContext {
       }
 
       // If there were extra definitions then report the error.
-      if (!ExtraSymbols.empty()) {
-        ES.reportError(make_error<UnexpectedSymbolDefinitions>(
-            G.getName(), std::move(ExtraSymbols)));
-        MR.failMaterialization();
-        return;
-      }
+      if (!ExtraSymbols.empty())
+        return make_error<UnexpectedSymbolDefinitions>(G.getName(),
+                                                       std::move(ExtraSymbols));
     }
 
-    if (auto Err = MR.notifyResolved(InternedResult)) {
-      Layer.getExecutionSession().reportError(std::move(Err));
-      MR.failMaterialization();
-      return;
-    }
+    if (auto Err = MR.notifyResolved(InternedResult))
+      return Err;
+
     Layer.notifyLoaded(MR);
+    return Error::success();
   }
 
   void notifyFinalized(

From 8ce8cee1e1d44d480ea87f1ac47eddcf07c433d1 Mon Sep 17 00:00:00 2001
From: Lang Hames <lhames@gmail.com>
Date: Wed, 29 Jul 2020 22:55:33 -0700
Subject: [PATCH 0680/1035] [llvm-jitlink] Add -harness option to llvm-jitlink.

The -harness option enables new testing use-cases for llvm-jitlink. It takes a
list of objects to treat as a test harness for any regular objects passed to
llvm-jitlink.

If any files are passed using the -harness option then the following
transformations are applied to all other files:

  (1) Symbols definitions that are referenced by the harness files are promoted
      to default scope. (This enables access to statics from test harness).

  (2) Symbols definitions that clash with definitions in the harness files are
      deleted. (This enables interposition by test harness).

  (3) All other definitions in regular files are demoted to local scope.
      (This causes untested code to be dead stripped, reducing memory cost and
      eliminating spurious unresolved symbol errors from untested code).

These transformations allow the harness files to reference and interpose
symbols in the regular object files, which can be used to support execution
tests (including fuzz tests) of functions in relocatable objects produced by a
build.
---
 .../llvm/ExecutionEngine/JITLink/JITLink.h    |   4 +
 .../JITLink/JITLinkGeneric.cpp                |  31 +++-
 .../X86/Inputs/MachO_test_harness_test.s      |  40 +++++
 .../JITLink/X86/MachO_test_harness_harness.s  |  65 +++++++
 llvm/tools/llvm-jitlink/llvm-jitlink.cpp      | 167 +++++++++++++++++-
 llvm/tools/llvm-jitlink/llvm-jitlink.h        |  24 ++-
 6 files changed, 320 insertions(+), 11 deletions(-)
 create mode 100644 llvm/test/ExecutionEngine/JITLink/X86/Inputs/MachO_test_harness_test.s
 create mode 100644 llvm/test/ExecutionEngine/JITLink/X86/MachO_test_harness_harness.s

diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h b/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h
index 46b5747507351..76f1eb6356576 100644
--- a/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h
+++ b/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h
@@ -395,6 +395,10 @@ class Symbol {
     return Name;
   }
 
+  /// Rename this symbol. The client is responsible for updating scope and
+  /// linkage if this name-change requires it.
+  void setName(StringRef Name) { this->Name = Name; }
+
   /// Returns true if this Symbol has content (potentially) defined within this
   /// object file (i.e. is anything but an external or absolute symbol).
   bool isDefined() const {
diff --git a/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp b/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp
index e74dd4d5c8b13..e0901186347f5 100644
--- a/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp
@@ -24,7 +24,10 @@ JITLinkerBase::~JITLinkerBase() {}
 
 void JITLinkerBase::linkPhase1(std::unique_ptr<JITLinkerBase> Self) {
 
-  LLVM_DEBUG({ dbgs() << "Building jitlink graph for new input...\n"; });
+  LLVM_DEBUG({
+    dbgs() << "Building jitlink graph for new input "
+           << Ctx->getObjectBuffer().getBufferIdentifier() << "...\n";
+  });
 
   // Build the link graph.
   if (auto GraphOrErr = buildGraph(Ctx->getObjectBuffer()))
@@ -447,16 +450,19 @@ void prune(LinkGraph &G) {
     VisitedBlocks.insert(&B);
 
     for (auto &E : Sym->getBlock().edges()) {
-      if (E.getTarget().isDefined() && !E.getTarget().isLive()) {
-        E.getTarget().setLive(true);
+      // If the edge target is a defined symbol that is being newly marked live
+      // then add it to the worklist.
+      if (E.getTarget().isDefined() && !E.getTarget().isLive())
         Worklist.push_back(&E.getTarget());
-      }
+
+      // Mark the target live.
+      E.getTarget().setLive(true);
     }
   }
 
-  // Collect all the symbols to remove, then remove them.
+  // Collect all defined symbols to remove, then remove them.
   {
-    LLVM_DEBUG(dbgs() << "Dead-stripping symbols:\n");
+    LLVM_DEBUG(dbgs() << "Dead-stripping defined symbols:\n");
     std::vector<Symbol *> SymbolsToRemove;
     for (auto *Sym : G.defined_symbols())
       if (!Sym->isLive())
@@ -479,6 +485,19 @@ void prune(LinkGraph &G) {
       G.removeBlock(*B);
     }
   }
+
+  // Collect all external symbols to remove, then remove them.
+  {
+    LLVM_DEBUG(dbgs() << "Removing unused external symbols:\n");
+    std::vector<Symbol *> SymbolsToRemove;
+    for (auto *Sym : G.external_symbols())
+      if (!Sym->isLive())
+        SymbolsToRemove.push_back(Sym);
+    for (auto *Sym : SymbolsToRemove) {
+      LLVM_DEBUG(dbgs() << "  " << *Sym << "...\n");
+      G.removeExternalSymbol(*Sym);
+    }
+  }
 }
 
 } // end namespace jitlink
diff --git a/llvm/test/ExecutionEngine/JITLink/X86/Inputs/MachO_test_harness_test.s b/llvm/test/ExecutionEngine/JITLink/X86/Inputs/MachO_test_harness_test.s
new file mode 100644
index 0000000000000..e0764a93fa435
--- /dev/null
+++ b/llvm/test/ExecutionEngine/JITLink/X86/Inputs/MachO_test_harness_test.s
@@ -0,0 +1,40 @@
+	.section	__TEXT,__text,regular,pure_instructions
+	.globl	_unused_public_function
+	.p2align	4, 0x90
+_unused_public_function:
+	jmp	_unresolvable_external
+
+	.p2align	4, 0x90
+_unused_private_function:
+	jmp	_unresolvable_external
+
+	.globl	_public_func_to_interpose
+	.p2align	4, 0x90
+_public_func_to_interpose:
+	retq
+
+  .p2align	4, 0x90
+_private_func_to_interpose:
+	retq
+
+	.globl	_public_func_to_test
+	.p2align	4, 0x90
+_public_func_to_test:
+	jmp	_public_func_to_interpose
+
+	.p2align	4, 0x90
+_private_func_to_test:
+	jmp	_private_func_to_interpose
+
+	.section	__DATA,__data
+	.globl	_public_func_to_interpose_as_seen_by_test
+	.p2align	3
+_public_func_to_interpose_as_seen_by_test:
+	.quad	_public_func_to_interpose
+
+	.globl	_private_func_to_interpose_as_seen_by_test
+	.p2align	3
+_private_func_to_interpose_as_seen_by_test:
+	.quad	_private_func_to_interpose
+
+.subsections_via_symbols
diff --git a/llvm/test/ExecutionEngine/JITLink/X86/MachO_test_harness_harness.s b/llvm/test/ExecutionEngine/JITLink/X86/MachO_test_harness_harness.s
new file mode 100644
index 0000000000000..7fdddf2a64ca7
--- /dev/null
+++ b/llvm/test/ExecutionEngine/JITLink/X86/MachO_test_harness_harness.s
@@ -0,0 +1,65 @@
+# RUN: rm -rf %t && mkdir -p %t
+# RUN: llvm-mc -triple=x86_64-apple-macosx10.9 -filetype=obj \
+# RUN:   -o %t/file_to_test.o %S/Inputs/MachO_test_harness_test.s
+# RUN: llvm-mc -triple=x86_64-apple-macosx10.9 -filetype=obj \
+# RUN:   -o %t/test_harness.o %s
+# RUN: llvm-jitlink -noexec -check %s %t/file_to_test.o \
+# RUN:    -harness %t/test_harness.o
+#
+# Check that we
+#   (1) Can call global symbols in the test object.
+#   (2) Can call private symbols in the test object.
+#   (3) Can interpose global symbols in the test object.
+#   (4) Can interpose private symbols in the test object.
+#   (5) Don't need to resolve unused externals in the test object.
+
+.section	__TEXT,__text,regular,pure_instructions
+
+  .globl	_public_func_to_interpose
+	.p2align	4, 0x90
+_public_func_to_interpose:
+	retq
+
+	.globl	_private_func_to_interpose
+	.p2align	4, 0x90
+_private_func_to_interpose:
+	retq
+
+	.globl	_main
+	.p2align	4, 0x90
+_main:
+	callq	_public_func_to_test
+	callq	_private_func_to_test
+  xorl  %eax, %eax
+	retq
+
+	.section	__DATA,__data
+
+# Check that the harness and test file agree on the address of the addresses
+# of the interposes:
+
+# jitlink-check: *{8}_public_func_to_interpose_as_seen_by_harness = \
+# jitlink-check:   *{8}_public_func_to_interpose_as_seen_by_test
+
+# jitlink-check: *{8}_private_func_to_interpose_as_seen_by_harness = \
+# jitlink-check:   *{8}_private_func_to_interpose_as_seen_by_test
+
+  .globl	_public_func_to_interpose_as_seen_by_harness
+	.p2align	3
+_public_func_to_interpose_as_seen_by_harness:
+	.quad	_public_func_to_interpose
+
+	.globl	_private_func_to_interpose_as_seen_by_harness
+	.p2align	3
+_private_func_to_interpose_as_seen_by_harness:
+	.quad	_private_func_to_interpose
+
+# We need to reference the *_as_seen_by_test pointers used above to ensure
+# that they're not dead-stripped as unused.
+  .globl  _anchor_test_case_pointers
+  .p2align  3
+_anchor_test_case_pointers:
+  .quad _public_func_to_interpose_as_seen_by_test
+  .quad _private_func_to_interpose_as_seen_by_test
+
+.subsections_via_symbols
diff --git a/llvm/tools/llvm-jitlink/llvm-jitlink.cpp b/llvm/tools/llvm-jitlink/llvm-jitlink.cpp
index 04132f0769662..6828944ced238 100644
--- a/llvm/tools/llvm-jitlink/llvm-jitlink.cpp
+++ b/llvm/tools/llvm-jitlink/llvm-jitlink.cpp
@@ -86,6 +86,11 @@ static cl::list<std::string> AbsoluteDefs(
     cl::desc("Inject absolute symbol definitions (syntax: <name>=<addr>)"),
     cl::ZeroOrMore);
 
+static cl::list<std::string> TestHarnesses("harness", cl::Positional,
+                                           cl::desc("Test harness files"),
+                                           cl::ZeroOrMore,
+                                           cl::PositionalEatsArgs);
+
 static cl::opt<bool> ShowInitialExecutionSessionState(
     "show-init-es",
     cl::desc("Print ExecutionSession state before resolving entry point"),
@@ -166,6 +171,43 @@ operator<<(raw_ostream &OS, const Session::FileInfoMap &FIM) {
   return OS;
 }
 
+static Error applyHarnessPromotions(Session &S, LinkGraph &G) {
+
+  // If this graph is part of the test harness there's nothing to do.
+  if (S.HarnessFiles.empty() || S.HarnessFiles.count(G.getName()))
+    return Error::success();
+
+  LLVM_DEBUG(dbgs() << "Appling promotions to graph " << G.getName() << "\n");
+
+  // If it isn't then promote any symbols referenced by the harness to default
+  // scope, remove all symbols that clash with harness definitions, and demote
+  // all others.
+  std::vector<Symbol *> DefinitionsToRemove;
+  for (auto *Sym : G.defined_symbols()) {
+
+    if (!Sym->hasName())
+      continue;
+
+    if (S.HarnessExternals.count(Sym->getName())) {
+      LLVM_DEBUG(dbgs() << "  Promoting " << Sym->getName() << "\n");
+      Sym->setScope(Scope::Default);
+      Sym->setLive(true);
+    } else if (S.HarnessDefinitions.count(Sym->getName())) {
+      LLVM_DEBUG(dbgs() << "  Externalizing " << Sym->getName() << "\n");
+      DefinitionsToRemove.push_back(Sym);
+    } else {
+      LLVM_DEBUG(dbgs() << "  Demoting " << Sym->getName() << "\n");
+      Sym->setScope(Scope::Local);
+      Sym->setLive(false);
+    }
+  }
+
+  for (auto *Sym : DefinitionsToRemove)
+    G.makeExternal(*Sym);
+
+  return Error::success();
+}
+
 static uint64_t computeTotalBlockSizes(LinkGraph &G) {
   uint64_t TotalSize = 0;
   for (auto *B : G.blocks())
@@ -408,12 +450,85 @@ Expected<uint64_t> getSlabAllocSize(StringRef SizeString) {
   return SlabSize * Units;
 }
 
-static std::unique_ptr<jitlink::JITLinkMemoryManager> createMemoryManager() {
+static std::unique_ptr<JITLinkMemoryManager> createMemoryManager() {
   if (!SlabAllocateSizeString.empty()) {
     auto SlabSize = ExitOnErr(getSlabAllocSize(SlabAllocateSizeString));
     return ExitOnErr(JITLinkSlabAllocator::Create(SlabSize));
   }
-  return std::make_unique<jitlink::InProcessMemoryManager>();
+  return std::make_unique<InProcessMemoryManager>();
+}
+
+LLVMJITLinkObjectLinkingLayer::LLVMJITLinkObjectLinkingLayer(
+    Session &S, std::unique_ptr<JITLinkMemoryManager> MemMgr)
+    : ObjectLinkingLayer(S.ES, std::move(MemMgr)), S(S) {}
+
+Error LLVMJITLinkObjectLinkingLayer::add(JITDylib &JD,
+                                         std::unique_ptr<MemoryBuffer> O,
+                                         VModuleKey K) {
+
+  if (S.HarnessFiles.empty() || S.HarnessFiles.count(O->getBufferIdentifier()))
+    return ObjectLinkingLayer::add(JD, std::move(O), std::move(K));
+
+  // Use getObjectSymbolInfo to compute the init symbol, but ignore
+  // the symbols field. We'll handle that manually to include promotion.
+  auto ObjSymInfo =
+      getObjectSymbolInfo(getExecutionSession(), O->getMemBufferRef());
+
+  if (!ObjSymInfo)
+    return ObjSymInfo.takeError();
+
+  auto &InitSymbol = ObjSymInfo->second;
+
+  // If creating an object file was going to fail it would have happened above,
+  // so we can 'cantFail' this.
+  auto Obj =
+      cantFail(object::ObjectFile::createObjectFile(O->getMemBufferRef()));
+
+  SymbolFlagsMap SymbolFlags;
+
+  // The init symbol must be included in the SymbolFlags map if present.
+  if (InitSymbol)
+    SymbolFlags[InitSymbol] = JITSymbolFlags::MaterializationSideEffectsOnly;
+
+  for (auto &Sym : Obj->symbols()) {
+    Expected<uint32_t> SymFlagsOrErr = Sym.getFlags();
+    if (!SymFlagsOrErr)
+      // TODO: Test this error.
+      return SymFlagsOrErr.takeError();
+
+    // Skip symbols not defined in this object file.
+    if (*SymFlagsOrErr & object::BasicSymbolRef::SF_Undefined)
+      continue;
+
+    auto Name = Sym.getName();
+    if (!Name)
+      return Name.takeError();
+
+    // Skip symbols that aren't in the HarnessExternals set.
+    if (!S.HarnessExternals.count(*Name))
+      continue;
+
+    // Skip symbols that have type SF_File.
+    if (auto SymType = Sym.getType()) {
+      if (*SymType == object::SymbolRef::ST_File)
+        continue;
+    } else
+      return SymType.takeError();
+
+    auto InternedName = S.ES.intern(*Name);
+    auto SymFlags = JITSymbolFlags::fromObjectSymbol(Sym);
+    if (!SymFlags)
+      return SymFlags.takeError();
+
+    *SymFlags |= JITSymbolFlags::Exported;
+
+    SymbolFlags[InternedName] = std::move(*SymFlags);
+  }
+
+  auto MU = std::make_unique<BasicObjectLayerMaterializationUnit>(
+      *this, K, std::move(O), std::move(SymbolFlags), std::move(InitSymbol));
+
+  return JD.define(std::move(MU));
 }
 
 Expected<std::unique_ptr<Session>> Session::Create(Triple TT) {
@@ -427,7 +542,7 @@ Expected<std::unique_ptr<Session>> Session::Create(Triple TT) {
 // FIXME: Move to createJITDylib if/when we start using Platform support in
 // llvm-jitlink.
 Session::Session(Triple TT, Error &Err)
-    : ObjLayer(ES, createMemoryManager()), TT(std::move(TT)) {
+    : ObjLayer(*this, createMemoryManager()), TT(std::move(TT)) {
 
   /// Local ObjectLinkingLayer::Plugin class to forward modifyPassConfig to the
   /// Session.
@@ -457,6 +572,39 @@ Session::Session(Triple TT, Error &Err)
         InProcessEHFrameRegistrar::getInstance()));
 
   ObjLayer.addPlugin(std::make_unique<JITLinkSessionPlugin>(*this));
+
+  // Process any harness files.
+  for (auto &HarnessFile : TestHarnesses) {
+    HarnessFiles.insert(HarnessFile);
+
+    auto ObjBuffer =
+        ExitOnErr(errorOrToExpected(MemoryBuffer::getFile(HarnessFile)));
+
+    auto ObjSymbolInfo =
+        ExitOnErr(getObjectSymbolInfo(ES, ObjBuffer->getMemBufferRef()));
+
+    for (auto &KV : ObjSymbolInfo.first)
+      HarnessDefinitions.insert(*KV.first);
+
+    auto Obj = ExitOnErr(
+        object::ObjectFile::createObjectFile(ObjBuffer->getMemBufferRef()));
+
+    for (auto &Sym : Obj->symbols()) {
+      uint32_t SymFlags = ExitOnErr(Sym.getFlags());
+      auto Name = ExitOnErr(Sym.getName());
+
+      if (Name.empty())
+        continue;
+
+      if (SymFlags & object::BasicSymbolRef::SF_Undefined)
+        HarnessExternals.insert(Name);
+    }
+  }
+
+  // If a name is defined by some harness file then it's a definition, not an
+  // external.
+  for (auto &DefName : HarnessDefinitions)
+    HarnessExternals.erase(DefName.getKey());
 }
 
 void Session::dumpSessionInfo(raw_ostream &OS) {
@@ -481,11 +629,14 @@ void Session::modifyPassConfig(const Triple &FTT,
 
   if (ShowLinkGraph)
     PassConfig.PostFixupPasses.push_back([](LinkGraph &G) -> Error {
-      outs() << "Link graph post-fixup:\n";
+      outs() << "Link graph \"" << G.getName() << "\" post-fixup:\n";
       G.dump(outs());
       return Error::success();
     });
 
+  PassConfig.PrePrunePasses.push_back(
+      [this](LinkGraph &G) { return applyHarnessPromotions(*this, G); });
+
   if (ShowSizes) {
     PassConfig.PrePrunePasses.push_back([this](LinkGraph &G) -> Error {
       SizeBeforePruning += computeTotalBlockSizes(G);
@@ -672,6 +823,14 @@ Error loadObjects(Session &S) {
     }
   }
 
+  LLVM_DEBUG(dbgs() << "Adding test harness objects...\n");
+  for (auto HarnessFile : TestHarnesses) {
+    LLVM_DEBUG(dbgs() << "  " << HarnessFile << "\n");
+    auto ObjBuffer =
+        ExitOnErr(errorOrToExpected(MemoryBuffer::getFile(HarnessFile)));
+    ExitOnErr(S.ObjLayer.add(*S.MainJD, std::move(ObjBuffer)));
+  }
+
   // Load each object into the corresponding JITDylib..
   LLVM_DEBUG(dbgs() << "Adding objects...\n");
   for (auto InputFileItr = InputFiles.begin(), InputFileEnd = InputFiles.end();
diff --git a/llvm/tools/llvm-jitlink/llvm-jitlink.h b/llvm/tools/llvm-jitlink/llvm-jitlink.h
index 5884e164a44d1..c16aed9f2b504 100644
--- a/llvm/tools/llvm-jitlink/llvm-jitlink.h
+++ b/llvm/tools/llvm-jitlink/llvm-jitlink.h
@@ -13,21 +13,39 @@
 #ifndef LLVM_TOOLS_LLVM_JITLINK_LLVM_JITLINK_H
 #define LLVM_TOOLS_LLVM_JITLINK_LLVM_JITLINK_H
 
+#include "llvm/ADT/Optional.h"
+#include "llvm/ADT/StringSet.h"
 #include "llvm/ADT/Triple.h"
 #include "llvm/ExecutionEngine/Orc/Core.h"
 #include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h"
 #include "llvm/ExecutionEngine/RuntimeDyldChecker.h"
 #include "llvm/Support/Error.h"
+#include "llvm/Support/Regex.h"
 #include "llvm/Support/raw_ostream.h"
 
 #include <vector>
 
 namespace llvm {
 
+struct Session;
+
+/// ObjectLinkingLayer with additional support for symbol promotion.
+class LLVMJITLinkObjectLinkingLayer : public orc::ObjectLinkingLayer {
+public:
+  LLVMJITLinkObjectLinkingLayer(
+      Session &S, std::unique_ptr<jitlink::JITLinkMemoryManager> MemMgr);
+
+  Error add(orc::JITDylib &JD, std::unique_ptr<MemoryBuffer> O,
+            orc::VModuleKey K = orc::VModuleKey()) override;
+
+private:
+  Session &S;
+};
+
 struct Session {
   orc::ExecutionSession ES;
   orc::JITDylib *MainJD;
-  orc::ObjectLinkingLayer ObjLayer;
+  LLVMJITLinkObjectLinkingLayer ObjLayer;
   std::vector<orc::JITDylib *> JDSearchOrder;
   Triple TT;
 
@@ -65,6 +83,10 @@ struct Session {
   uint64_t SizeBeforePruning = 0;
   uint64_t SizeAfterFixups = 0;
 
+  StringSet<> HarnessFiles;
+  StringSet<> HarnessExternals;
+  StringSet<> HarnessDefinitions;
+
 private:
   Session(Triple TT, Error &Err);
 };

From b811736f8bbf6ec9596c191a6e9406f9df9523ae Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
Date: Thu, 30 Jul 2020 22:29:22 +0000
Subject: [PATCH 0681/1035] [gn build] Port 763671f387f

---
 llvm/utils/gn/secondary/lld/COFF/BUILD.gn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/utils/gn/secondary/lld/COFF/BUILD.gn b/llvm/utils/gn/secondary/lld/COFF/BUILD.gn
index ceb1b22d81fd9..0f19a77741b03 100644
--- a/llvm/utils/gn/secondary/lld/COFF/BUILD.gn
+++ b/llvm/utils/gn/secondary/lld/COFF/BUILD.gn
@@ -27,6 +27,7 @@ static_library("COFF") {
     "//llvm/lib/WindowsManifest",
   ]
   sources = [
+    "CallGraphSort.cpp",
     "Chunks.cpp",
     "DLL.cpp",
     "DebugTypes.cpp",

From 8dfb5d767e70dd862096c8872fd3e3bead99741d Mon Sep 17 00:00:00 2001
From: Eli Friedman <efriedma@quicinc.com>
Date: Mon, 27 Jul 2020 14:01:46 -0700
Subject: [PATCH 0682/1035] [clang codegen][AArch64] Use
 llvm.aarch64.neon.fcvtzs/u where it's necessary

fptosi/fptoui have similar, but not identical, semantics.  In
particular, the behavior on overflow is different.

Fixes https://bugs.llvm.org/show_bug.cgi?id=46844 for 64-bit.  (The
corresponding patch for 32-bit is more involved because the equivalent
intrinsics don't exist, as far as I can tell.)

Differential Revision: https://reviews.llvm.org/D84703
---
 clang/lib/CodeGen/CGBuiltin.cpp               | 69 +++++--------------
 .../CodeGen/aarch64-neon-fcvt-intrinsics.c    |  8 +--
 clang/test/CodeGen/aarch64-neon-intrinsics.c  |  4 +-
 clang/test/CodeGen/aarch64-neon-misc.c        | 12 ++--
 .../CodeGen/aarch64-v8.2a-fp16-intrinsics.c   | 18 ++---
 .../CodeGen/aarch64-v8.2a-neon-intrinsics.c   |  8 +--
 6 files changed, 45 insertions(+), 74 deletions(-)

diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index ecc0b5bf2dc44..042b41a09f198 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -5251,6 +5251,8 @@ static const ARMVectorIntrinsicInfo AArch64SISDIntrinsicMap[] = {
   NEONMAP1(vcvtd_n_f64_u64, aarch64_neon_vcvtfxu2fp, AddRetType | Add1ArgType),
   NEONMAP1(vcvtd_n_s64_f64, aarch64_neon_vcvtfp2fxs, AddRetType | Add1ArgType),
   NEONMAP1(vcvtd_n_u64_f64, aarch64_neon_vcvtfp2fxu, AddRetType | Add1ArgType),
+  NEONMAP1(vcvtd_s64_f64, aarch64_neon_fcvtzs, AddRetType | Add1ArgType),
+  NEONMAP1(vcvtd_u64_f64, aarch64_neon_fcvtzu, AddRetType | Add1ArgType),
   NEONMAP1(vcvth_bf16_f32, aarch64_neon_bfcvt, 0),
   NEONMAP1(vcvtmd_s64_f64, aarch64_neon_fcvtms, AddRetType | Add1ArgType),
   NEONMAP1(vcvtmd_u64_f64, aarch64_neon_fcvtmu, AddRetType | Add1ArgType),
@@ -5268,6 +5270,8 @@ static const ARMVectorIntrinsicInfo AArch64SISDIntrinsicMap[] = {
   NEONMAP1(vcvts_n_f32_u32, aarch64_neon_vcvtfxu2fp, AddRetType | Add1ArgType),
   NEONMAP1(vcvts_n_s32_f32, aarch64_neon_vcvtfp2fxs, AddRetType | Add1ArgType),
   NEONMAP1(vcvts_n_u32_f32, aarch64_neon_vcvtfp2fxu, AddRetType | Add1ArgType),
+  NEONMAP1(vcvts_s32_f32, aarch64_neon_fcvtzs, AddRetType | Add1ArgType),
+  NEONMAP1(vcvts_u32_f32, aarch64_neon_fcvtzu, AddRetType | Add1ArgType),
   NEONMAP1(vcvtxd_f32_f64, aarch64_sisd_fcvtxn, 0),
   NEONMAP1(vmaxnmv_f32, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType),
   NEONMAP1(vmaxnmvq_f32, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType),
@@ -5426,6 +5430,10 @@ static const ARMVectorIntrinsicInfo AArch64SISDIntrinsicMap[] = {
   NEONMAP1(vcvth_n_s64_f16, aarch64_neon_vcvtfp2fxs, AddRetType | Add1ArgType),
   NEONMAP1(vcvth_n_u32_f16, aarch64_neon_vcvtfp2fxu, AddRetType | Add1ArgType),
   NEONMAP1(vcvth_n_u64_f16, aarch64_neon_vcvtfp2fxu, AddRetType | Add1ArgType),
+  NEONMAP1(vcvth_s32_f16, aarch64_neon_fcvtzs, AddRetType | Add1ArgType),
+  NEONMAP1(vcvth_s64_f16, aarch64_neon_fcvtzs, AddRetType | Add1ArgType),
+  NEONMAP1(vcvth_u32_f16, aarch64_neon_fcvtzu, AddRetType | Add1ArgType),
+  NEONMAP1(vcvth_u64_f16, aarch64_neon_fcvtzu, AddRetType | Add1ArgType),
   NEONMAP1(vcvtmh_s32_f16, aarch64_neon_fcvtms, AddRetType | Add1ArgType),
   NEONMAP1(vcvtmh_s64_f16, aarch64_neon_fcvtms, AddRetType | Add1ArgType),
   NEONMAP1(vcvtmh_u32_f16, aarch64_neon_fcvtmu, AddRetType | Add1ArgType),
@@ -8995,21 +9003,6 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
     Value *Ptr = Builder.CreateBitCast(Ops[0], Int128PTy);
     return Builder.CreateDefaultAlignedStore(EmitScalarExpr(E->getArg(1)), Ptr);
   }
-  case NEON::BI__builtin_neon_vcvts_u32_f32:
-  case NEON::BI__builtin_neon_vcvtd_u64_f64:
-    usgn = true;
-    LLVM_FALLTHROUGH;
-  case NEON::BI__builtin_neon_vcvts_s32_f32:
-  case NEON::BI__builtin_neon_vcvtd_s64_f64: {
-    Ops.push_back(EmitScalarExpr(E->getArg(0)));
-    bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64;
-    llvm::Type *InTy = Is64 ? Int64Ty : Int32Ty;
-    llvm::Type *FTy = Is64 ? DoubleTy : FloatTy;
-    Ops[0] = Builder.CreateBitCast(Ops[0], FTy);
-    if (usgn)
-      return Builder.CreateFPToUI(Ops[0], InTy);
-    return Builder.CreateFPToSI(Ops[0], InTy);
-  }
   case NEON::BI__builtin_neon_vcvts_f32_u32:
   case NEON::BI__builtin_neon_vcvtd_f64_u64:
     usgn = true;
@@ -9047,44 +9040,16 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
       return Builder.CreateUIToFP(Ops[0], FTy);
     return Builder.CreateSIToFP(Ops[0], FTy);
   }
-  case NEON::BI__builtin_neon_vcvth_u16_f16:
-    usgn = true;
-    LLVM_FALLTHROUGH;
-  case NEON::BI__builtin_neon_vcvth_s16_f16: {
-    Ops.push_back(EmitScalarExpr(E->getArg(0)));
-    Ops[0] = Builder.CreateBitCast(Ops[0], HalfTy);
-    if (usgn)
-      return Builder.CreateFPToUI(Ops[0], Int16Ty);
-    return Builder.CreateFPToSI(Ops[0], Int16Ty);
-  }
-  case NEON::BI__builtin_neon_vcvth_u32_f16:
-    usgn = true;
-    LLVM_FALLTHROUGH;
-  case NEON::BI__builtin_neon_vcvth_s32_f16: {
-    Ops.push_back(EmitScalarExpr(E->getArg(0)));
-    Ops[0] = Builder.CreateBitCast(Ops[0], HalfTy);
-    if (usgn)
-      return Builder.CreateFPToUI(Ops[0], Int32Ty);
-    return Builder.CreateFPToSI(Ops[0], Int32Ty);
-  }
-  case NEON::BI__builtin_neon_vcvth_u64_f16:
-    usgn = true;
-    LLVM_FALLTHROUGH;
-  case NEON::BI__builtin_neon_vcvth_s64_f16: {
-    Ops.push_back(EmitScalarExpr(E->getArg(0)));
-    Ops[0] = Builder.CreateBitCast(Ops[0], HalfTy);
-    if (usgn)
-      return Builder.CreateFPToUI(Ops[0], Int64Ty);
-    return Builder.CreateFPToSI(Ops[0], Int64Ty);
-  }
   case NEON::BI__builtin_neon_vcvtah_u16_f16:
   case NEON::BI__builtin_neon_vcvtmh_u16_f16:
   case NEON::BI__builtin_neon_vcvtnh_u16_f16:
   case NEON::BI__builtin_neon_vcvtph_u16_f16:
+  case NEON::BI__builtin_neon_vcvth_u16_f16:
   case NEON::BI__builtin_neon_vcvtah_s16_f16:
   case NEON::BI__builtin_neon_vcvtmh_s16_f16:
   case NEON::BI__builtin_neon_vcvtnh_s16_f16:
-  case NEON::BI__builtin_neon_vcvtph_s16_f16: {
+  case NEON::BI__builtin_neon_vcvtph_s16_f16:
+  case NEON::BI__builtin_neon_vcvth_s16_f16: {
     unsigned Int;
     llvm::Type* InTy = Int32Ty;
     llvm::Type* FTy  = HalfTy;
@@ -9100,6 +9065,8 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
       Int = Intrinsic::aarch64_neon_fcvtnu; break;
     case NEON::BI__builtin_neon_vcvtph_u16_f16:
       Int = Intrinsic::aarch64_neon_fcvtpu; break;
+    case NEON::BI__builtin_neon_vcvth_u16_f16:
+      Int = Intrinsic::aarch64_neon_fcvtzu; break;
     case NEON::BI__builtin_neon_vcvtah_s16_f16:
       Int = Intrinsic::aarch64_neon_fcvtas; break;
     case NEON::BI__builtin_neon_vcvtmh_s16_f16:
@@ -9108,6 +9075,8 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
       Int = Intrinsic::aarch64_neon_fcvtns; break;
     case NEON::BI__builtin_neon_vcvtph_s16_f16:
       Int = Intrinsic::aarch64_neon_fcvtps; break;
+    case NEON::BI__builtin_neon_vcvth_s16_f16:
+      Int = Intrinsic::aarch64_neon_fcvtzs; break;
     }
     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "fcvt");
     return Builder.CreateTrunc(Ops[0], Int16Ty);
@@ -10148,10 +10117,10 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
   case NEON::BI__builtin_neon_vcvtq_u64_v:
   case NEON::BI__builtin_neon_vcvtq_s16_v:
   case NEON::BI__builtin_neon_vcvtq_u16_v: {
-    Ops[0] = Builder.CreateBitCast(Ops[0], GetFloatNeonType(this, Type));
-    if (usgn)
-      return Builder.CreateFPToUI(Ops[0], Ty);
-    return Builder.CreateFPToSI(Ops[0], Ty);
+    Int =
+        usgn ? Intrinsic::aarch64_neon_fcvtzu : Intrinsic::aarch64_neon_fcvtzs;
+    llvm::Type *Tys[2] = {Ty, GetFloatNeonType(this, Type)};
+    return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtz");
   }
   case NEON::BI__builtin_neon_vcvta_s16_v:
   case NEON::BI__builtin_neon_vcvta_u16_v:
diff --git a/clang/test/CodeGen/aarch64-neon-fcvt-intrinsics.c b/clang/test/CodeGen/aarch64-neon-fcvt-intrinsics.c
index 929a6501df77f..f06432c914f25 100644
--- a/clang/test/CodeGen/aarch64-neon-fcvt-intrinsics.c
+++ b/clang/test/CodeGen/aarch64-neon-fcvt-intrinsics.c
@@ -125,28 +125,28 @@ uint64_t test_vcvtpd_u64_f64(float64_t a) {
 }
 
 // CHECK-LABEL: define i32 @test_vcvts_s32_f32(float %a) #0 {
-// CHECK:   [[TMP0:%.*]] = fptosi float %a to i32
+// CHECK:   [[TMP0:%.*]] = call i32 @llvm.aarch64.neon.fcvtzs.i32.f32(float %a)
 // CHECK:   ret i32 [[TMP0]]
 int32_t test_vcvts_s32_f32(float32_t a) {
   return (int32_t)vcvts_s32_f32(a);
 }
 
 // CHECK-LABEL: define i64 @test_vcvtd_s64_f64(double %a) #0 {
-// CHECK:   [[TMP0:%.*]] = fptosi double %a to i64
+// CHECK:   [[TMP0:%.*]] = call i64 @llvm.aarch64.neon.fcvtzs.i64.f64(double %a)
 // CHECK:   ret i64 [[TMP0]]
 int64_t test_vcvtd_s64_f64(float64_t a) {
   return (int64_t)vcvtd_s64_f64(a);
 }
 
 // CHECK-LABEL: define i32 @test_vcvts_u32_f32(float %a) #0 {
-// CHECK:   [[TMP0:%.*]] = fptoui float %a to i32
+// CHECK:   [[TMP0:%.*]] = call i32 @llvm.aarch64.neon.fcvtzu.i32.f32(float %a)
 // CHECK:   ret i32 [[TMP0]]
 uint32_t test_vcvts_u32_f32(float32_t a) {
   return (uint32_t)vcvts_u32_f32(a);
 }
 
 // CHECK-LABEL: define i64 @test_vcvtd_u64_f64(double %a) #0 {
-// CHECK:   [[TMP0:%.*]] = fptoui double %a to i64
+// CHECK:   [[TMP0:%.*]] = call i64 @llvm.aarch64.neon.fcvtzu.i64.f64(double %a)
 // CHECK:   ret i64 [[TMP0]]
 uint64_t test_vcvtd_u64_f64(float64_t a) {
   return (uint64_t)vcvtd_u64_f64(a);
diff --git a/clang/test/CodeGen/aarch64-neon-intrinsics.c b/clang/test/CodeGen/aarch64-neon-intrinsics.c
index fbf3c2c5b63e8..de7e770256058 100644
--- a/clang/test/CodeGen/aarch64-neon-intrinsics.c
+++ b/clang/test/CodeGen/aarch64-neon-intrinsics.c
@@ -18023,7 +18023,7 @@ float64x1_t test_vneg_f64(float64x1_t a) {
 
 // CHECK-LABEL: @test_vcvt_s64_f64(
 // CHECK:   [[TMP0:%.*]] = bitcast <1 x double> %a to <8 x i8>
-// CHECK:   [[TMP1:%.*]] = fptosi <1 x double> %a to <1 x i64>
+// CHECK:   [[TMP1:%.*]] = call <1 x i64> @llvm.aarch64.neon.fcvtzs.v1i64.v1f64(<1 x double> %a)
 // CHECK:   ret <1 x i64> [[TMP1]]
 int64x1_t test_vcvt_s64_f64(float64x1_t a) {
   return vcvt_s64_f64(a);
@@ -18031,7 +18031,7 @@ int64x1_t test_vcvt_s64_f64(float64x1_t a) {
 
 // CHECK-LABEL: @test_vcvt_u64_f64(
 // CHECK:   [[TMP0:%.*]] = bitcast <1 x double> %a to <8 x i8>
-// CHECK:   [[TMP1:%.*]] = fptoui <1 x double> %a to <1 x i64>
+// CHECK:   [[TMP1:%.*]] = call <1 x i64> @llvm.aarch64.neon.fcvtzu.v1i64.v1f64(<1 x double> %a)
 // CHECK:   ret <1 x i64> [[TMP1]]
 uint64x1_t test_vcvt_u64_f64(float64x1_t a) {
   return vcvt_u64_f64(a);
diff --git a/clang/test/CodeGen/aarch64-neon-misc.c b/clang/test/CodeGen/aarch64-neon-misc.c
index acc3c0b7d2d4e..14b5a357f61ee 100644
--- a/clang/test/CodeGen/aarch64-neon-misc.c
+++ b/clang/test/CodeGen/aarch64-neon-misc.c
@@ -2311,7 +2311,7 @@ float64x2_t test_vrndiq_f64(float64x2_t a) {
 
 // CHECK-LABEL: @test_vcvt_s32_f32(
 // CHECK:   [[TMP0:%.*]] = bitcast <2 x float> %a to <8 x i8>
-// CHECK:   [[TMP1:%.*]] = fptosi <2 x float> %a to <2 x i32>
+// CHECK:   [[TMP1:%.*]] = call <2 x i32> @llvm.aarch64.neon.fcvtzs.v2i32.v2f32(<2 x float> %a)
 // CHECK:   ret <2 x i32> [[TMP1]]
 int32x2_t test_vcvt_s32_f32(float32x2_t a) {
   return vcvt_s32_f32(a);
@@ -2319,7 +2319,7 @@ int32x2_t test_vcvt_s32_f32(float32x2_t a) {
 
 // CHECK-LABEL: @test_vcvtq_s32_f32(
 // CHECK:   [[TMP0:%.*]] = bitcast <4 x float> %a to <16 x i8>
-// CHECK:   [[TMP1:%.*]] = fptosi <4 x float> %a to <4 x i32>
+// CHECK:   [[TMP1:%.*]] = call <4 x i32> @llvm.aarch64.neon.fcvtzs.v4i32.v4f32(<4 x float> %a)
 // CHECK:   ret <4 x i32> [[TMP1]]
 int32x4_t test_vcvtq_s32_f32(float32x4_t a) {
   return vcvtq_s32_f32(a);
@@ -2327,7 +2327,7 @@ int32x4_t test_vcvtq_s32_f32(float32x4_t a) {
 
 // CHECK-LABEL: @test_vcvtq_s64_f64(
 // CHECK:   [[TMP0:%.*]] = bitcast <2 x double> %a to <16 x i8>
-// CHECK:   [[TMP1:%.*]] = fptosi <2 x double> %a to <2 x i64>
+// CHECK:   [[TMP1:%.*]] = call <2 x i64> @llvm.aarch64.neon.fcvtzs.v2i64.v2f64(<2 x double> %a)
 // CHECK:   ret <2 x i64> [[TMP1]]
 int64x2_t test_vcvtq_s64_f64(float64x2_t a) {
   return vcvtq_s64_f64(a);
@@ -2335,7 +2335,7 @@ int64x2_t test_vcvtq_s64_f64(float64x2_t a) {
 
 // CHECK-LABEL: @test_vcvt_u32_f32(
 // CHECK:   [[TMP0:%.*]] = bitcast <2 x float> %a to <8 x i8>
-// CHECK:   [[TMP1:%.*]] = fptoui <2 x float> %a to <2 x i32>
+// CHECK:   [[TMP1:%.*]] = call <2 x i32> @llvm.aarch64.neon.fcvtzu.v2i32.v2f32(<2 x float> %a)
 // CHECK:   ret <2 x i32> [[TMP1]]
 uint32x2_t test_vcvt_u32_f32(float32x2_t a) {
   return vcvt_u32_f32(a);
@@ -2343,7 +2343,7 @@ uint32x2_t test_vcvt_u32_f32(float32x2_t a) {
 
 // CHECK-LABEL: @test_vcvtq_u32_f32(
 // CHECK:   [[TMP0:%.*]] = bitcast <4 x float> %a to <16 x i8>
-// CHECK:   [[TMP1:%.*]] = fptoui <4 x float> %a to <4 x i32>
+// CHECK:   [[TMP1:%.*]] = call <4 x i32> @llvm.aarch64.neon.fcvtzu.v4i32.v4f32(<4 x float> %a)
 // CHECK:   ret <4 x i32> [[TMP1]]
 uint32x4_t test_vcvtq_u32_f32(float32x4_t a) {
   return vcvtq_u32_f32(a);
@@ -2351,7 +2351,7 @@ uint32x4_t test_vcvtq_u32_f32(float32x4_t a) {
 
 // CHECK-LABEL: @test_vcvtq_u64_f64(
 // CHECK:   [[TMP0:%.*]] = bitcast <2 x double> %a to <16 x i8>
-// CHECK:   [[TMP1:%.*]] = fptoui <2 x double> %a to <2 x i64>
+// CHECK:   [[TMP1:%.*]] = call <2 x i64> @llvm.aarch64.neon.fcvtzu.v2i64.v2f64(<2 x double> %a)
 // CHECK:   ret <2 x i64> [[TMP1]]
 uint64x2_t test_vcvtq_u64_f64(float64x2_t a) {
   return vcvtq_u64_f64(a);
diff --git a/clang/test/CodeGen/aarch64-v8.2a-fp16-intrinsics.c b/clang/test/CodeGen/aarch64-v8.2a-fp16-intrinsics.c
index 45b18453b89dd..32161146ef450 100644
--- a/clang/test/CodeGen/aarch64-v8.2a-fp16-intrinsics.c
+++ b/clang/test/CodeGen/aarch64-v8.2a-fp16-intrinsics.c
@@ -97,42 +97,44 @@ float16_t test_vcvth_f16_u64 (uint64_t a) {
 }
 
 // CHECK-LABEL: test_vcvth_s16_f16
-// CHECK:  [[VCVT:%.*]] = fptosi half %a to i16
-// CHECK:  ret i16 [[VCVT]]
+// CHECK:  [[VCVT:%.*]] = call i32 @llvm.aarch64.neon.fcvtzs.i32.f16(half %a)
+// CHECK:  [[TRUNC:%.*]] = trunc i32 [[VCVT]] to i16
+// CHECK:  ret i16 [[TRUNC]]
 int16_t test_vcvth_s16_f16 (float16_t a) {
   return vcvth_s16_f16(a);
 }
 
 // CHECK-LABEL: test_vcvth_s32_f16
-// CHECK:  [[VCVT:%.*]] = fptosi half %a to i32
+// CHECK:  [[VCVT:%.*]] = call i32 @llvm.aarch64.neon.fcvtzs.i32.f16(half %a)
 // CHECK:  ret i32 [[VCVT]]
 int32_t test_vcvth_s32_f16 (float16_t a) {
   return vcvth_s32_f16(a);
 }
 
 // CHECK-LABEL: test_vcvth_s64_f16
-// CHECK:  [[VCVT:%.*]] = fptosi half %a to i64
+// CHECK:  [[VCVT:%.*]] = call i64 @llvm.aarch64.neon.fcvtzs.i64.f16(half %a)
 // CHECK:  ret i64 [[VCVT]]
 int64_t test_vcvth_s64_f16 (float16_t a) {
   return vcvth_s64_f16(a);
 }
 
 // CHECK-LABEL: test_vcvth_u16_f16
-// CHECK:  [[VCVT:%.*]] = fptoui half %a to i16
-// CHECK:  ret i16 [[VCVT]]
+// CHECK:  [[VCVT:%.*]] = call i32 @llvm.aarch64.neon.fcvtzu.i32.f16(half %a)
+// CHECK:  [[TRUNC:%.*]] = trunc i32 [[VCVT]] to i16
+// CHECK:  ret i16 [[TRUNC]]
 uint16_t test_vcvth_u16_f16 (float16_t a) {
   return vcvth_u16_f16(a);
 }
 
 // CHECK-LABEL: test_vcvth_u32_f16
-// CHECK:  [[VCVT:%.*]] = fptoui half %a to i32
+// CHECK:  [[VCVT:%.*]] = call i32 @llvm.aarch64.neon.fcvtzu.i32.f16(half %a)
 // CHECK:  ret i32 [[VCVT]]
 uint32_t test_vcvth_u32_f16 (float16_t a) {
   return vcvth_u32_f16(a);
 }
 
 // CHECK-LABEL: test_vcvth_u64_f16
-// CHECK:  [[VCVT:%.*]] = fptoui half %a to i64
+// CHECK:  [[VCVT:%.*]] = call i64 @llvm.aarch64.neon.fcvtzu.i64.f16(half %a)
 // CHECK:  ret i64 [[VCVT]]
 uint64_t test_vcvth_u64_f16 (float16_t a) {
   return vcvth_u64_f16(a);
diff --git a/clang/test/CodeGen/aarch64-v8.2a-neon-intrinsics.c b/clang/test/CodeGen/aarch64-v8.2a-neon-intrinsics.c
index 59b3dfec80cb9..5c4f9053a9aec 100644
--- a/clang/test/CodeGen/aarch64-v8.2a-neon-intrinsics.c
+++ b/clang/test/CodeGen/aarch64-v8.2a-neon-intrinsics.c
@@ -130,28 +130,28 @@ float16x8_t test_vcvtq_f16_u16 (uint16x8_t a) {
 }
 
 // CHECK-LABEL: test_vcvt_s16_f16
-// CHECK:  [[VCVT:%.*]] = fptosi <4 x half> %a to <4 x i16>
+// CHECK:  [[VCVT:%.*]] = call <4 x i16> @llvm.aarch64.neon.fcvtzs.v4i16.v4f16(<4 x half> %a)
 // CHECK:  ret <4 x i16> [[VCVT]]
 int16x4_t test_vcvt_s16_f16 (float16x4_t a) {
   return vcvt_s16_f16(a);
 }
 
 // CHECK-LABEL: test_vcvtq_s16_f16
-// CHECK:  [[VCVT:%.*]] = fptosi <8 x half> %a to <8 x i16>
+// CHECK:  [[VCVT:%.*]] = call <8 x i16> @llvm.aarch64.neon.fcvtzs.v8i16.v8f16(<8 x half> %a)
 // CHECK:  ret <8 x i16> [[VCVT]]
 int16x8_t test_vcvtq_s16_f16 (float16x8_t a) {
   return vcvtq_s16_f16(a);
 }
 
 // CHECK-LABEL: test_vcvt_u16_f16
-// CHECK:  [[VCVT:%.*]] = fptoui <4 x half> %a to <4 x i16>
+// CHECK:  [[VCVT:%.*]] = call <4 x i16> @llvm.aarch64.neon.fcvtzu.v4i16.v4f16(<4 x half> %a)
 // CHECK:  ret <4 x i16> [[VCVT]]
 uint16x4_t test_vcvt_u16_f16 (float16x4_t a) {
   return vcvt_u16_f16(a);
 }
 
 // CHECK-LABEL: test_vcvtq_u16_f16
-// CHECK:  [[VCVT:%.*]] = fptoui <8 x half> %a to <8 x i16>
+// CHECK:  [[VCVT:%.*]] = call <8 x i16> @llvm.aarch64.neon.fcvtzu.v8i16.v8f16(<8 x half> %a)
 // CHECK:  ret <8 x i16> [[VCVT]]
 uint16x8_t test_vcvtq_u16_f16 (float16x8_t a) {
   return vcvtq_u16_f16(a);

From a32af8252fde237a3936c4f2f5bee1dfb0cf5ea8 Mon Sep 17 00:00:00 2001
From: Siva Chandra Reddy <sivachandra@google.com>
Date: Tue, 28 Jul 2020 23:42:11 -0700
Subject: [PATCH 0683/1035] [libc] Add a tool called WrapperGen.

This tool will be used to generate C wrappers for the C++ LLVM libc
implementations. This change does not hook this tool up to anything yet.
However, it can be useful for cases where one does not want to run the
objcopy step (to insert the C symbol in the object file) but can make use
of LTO to eliminate the cost of the additional wrapper call. This can be
relevant for certain downstream platforms. If this tool can benefit other
libc platforms in general, then it can be integrated into the build system
with options to use or not use the wrappers. An example of such a
platform is CUDA.

Reviewed By: abrachet

Differential Revision: https://reviews.llvm.org/D84848
---
 libc/utils/CMakeLists.txt                  |  1 +
 libc/utils/LibcTableGenUtil/APIIndexer.cpp |  5 +-
 libc/utils/LibcTableGenUtil/APIIndexer.h   |  2 +
 libc/utils/tools/CMakeLists.txt            |  1 +
 libc/utils/tools/WrapperGen/CMakeLists.txt |  8 +++
 libc/utils/tools/WrapperGen/Main.cpp       | 72 ++++++++++++++++++++++
 6 files changed, 87 insertions(+), 2 deletions(-)
 create mode 100644 libc/utils/tools/CMakeLists.txt
 create mode 100644 libc/utils/tools/WrapperGen/CMakeLists.txt
 create mode 100644 libc/utils/tools/WrapperGen/Main.cpp

diff --git a/libc/utils/CMakeLists.txt b/libc/utils/CMakeLists.txt
index ec1308927bf92..a55182065d20f 100644
--- a/libc/utils/CMakeLists.txt
+++ b/libc/utils/CMakeLists.txt
@@ -4,4 +4,5 @@ add_subdirectory(LibcTableGenUtil)
 add_subdirectory(HdrGen)
 add_subdirectory(MPFRWrapper)
 add_subdirectory(testutils)
+add_subdirectory(tools)
 add_subdirectory(UnitTest)
diff --git a/libc/utils/LibcTableGenUtil/APIIndexer.cpp b/libc/utils/LibcTableGenUtil/APIIndexer.cpp
index 6ef9a6576b87f..16aef5880bffe 100644
--- a/libc/utils/LibcTableGenUtil/APIIndexer.cpp
+++ b/libc/utils/LibcTableGenUtil/APIIndexer.cpp
@@ -97,8 +97,9 @@ void APIIndexer::indexStandardSpecDef(llvm::Record *StandardSpec) {
 
       auto FunctionSpecList = HeaderSpec->getValueAsListOfDefs("Functions");
       for (llvm::Record *FunctionSpec : FunctionSpecList) {
-        FunctionSpecMap[std::string(FunctionSpec->getValueAsString("Name"))] =
-            FunctionSpec;
+        auto FunctionName = std::string(FunctionSpec->getValueAsString("Name"));
+        FunctionSpecMap[FunctionName] = FunctionSpec;
+        FunctionToHeaderMap[FunctionName] = std::string(Header);
       }
 
       auto EnumerationSpecList =
diff --git a/libc/utils/LibcTableGenUtil/APIIndexer.h b/libc/utils/LibcTableGenUtil/APIIndexer.h
index 8fa87113610e1..7b4d62a38c615 100644
--- a/libc/utils/LibcTableGenUtil/APIIndexer.h
+++ b/libc/utils/LibcTableGenUtil/APIIndexer.h
@@ -65,6 +65,8 @@ class APIIndexer {
   NameToRecordMapping MacroDefsMap;
   NameToRecordMapping TypeDeclsMap;
 
+  std::unordered_map<std::string, std::string> FunctionToHeaderMap;
+
   NameSet Structs;
   NameSet Enumerations;
   NameSet Functions;
diff --git a/libc/utils/tools/CMakeLists.txt b/libc/utils/tools/CMakeLists.txt
new file mode 100644
index 0000000000000..e975b7f293af9
--- /dev/null
+++ b/libc/utils/tools/CMakeLists.txt
@@ -0,0 +1 @@
+add_subdirectory(WrapperGen)
diff --git a/libc/utils/tools/WrapperGen/CMakeLists.txt b/libc/utils/tools/WrapperGen/CMakeLists.txt
new file mode 100644
index 0000000000000..fe8ffcce94a58
--- /dev/null
+++ b/libc/utils/tools/WrapperGen/CMakeLists.txt
@@ -0,0 +1,8 @@
+set(LLVM_LINK_COMPONENTS Support)
+
+add_tablegen(libc-wrappergen llvm-libc
+  Main.cpp
+)
+
+target_include_directories(libc-wrappergen PRIVATE ${LIBC_SOURCE_DIR})
+target_link_libraries(libc-wrappergen PRIVATE LibcTableGenUtil)
diff --git a/libc/utils/tools/WrapperGen/Main.cpp b/libc/utils/tools/WrapperGen/Main.cpp
new file mode 100644
index 0000000000000..ae606d1b66fa1
--- /dev/null
+++ b/libc/utils/tools/WrapperGen/Main.cpp
@@ -0,0 +1,72 @@
+//===-- "main" function of libc-wrappergen --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "utils/LibcTableGenUtil/APIIndexer.h"
+
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/TableGen/Error.h"
+#include "llvm/TableGen/Main.h"
+
+#include <sstream>
+#include <string>
+
+llvm::cl::opt<std::string>
+    FunctionName("name", llvm::cl::desc("Name of the function to be wrapped."),
+                 llvm::cl::value_desc("<function name>"), llvm::cl::Required);
+
+static bool WrapperGenMain(llvm::raw_ostream &OS, llvm::RecordKeeper &Records) {
+  llvm_libc::APIIndexer Indexer(Records);
+  auto Iter = Indexer.FunctionSpecMap.find(FunctionName);
+  if (Iter == Indexer.FunctionSpecMap.end()) {
+    llvm::PrintFatalError("Function '" + FunctionName +
+                          "' not found in any standard spec.");
+  }
+
+  // To avoid all confusion, we include the implementation header using the
+  // full path (relative the libc directory.)
+  std::string Header = Indexer.FunctionToHeaderMap[FunctionName];
+  auto RelPath = llvm::StringRef(Header).drop_back(2); // Drop the ".h" suffix.
+  OS << "#include \"src/" << RelPath << "/" << FunctionName << ".h\"\n";
+
+  auto &NameSpecPair = *Iter;
+  llvm::Record *FunctionSpec = NameSpecPair.second;
+  llvm::Record *RetValSpec = FunctionSpec->getValueAsDef("Return");
+  llvm::Record *ReturnType = RetValSpec->getValueAsDef("ReturnType");
+  OS << "extern \"C\" " << Indexer.getTypeAsString(ReturnType) << " "
+     << FunctionName << "(";
+
+  auto ArgsList = FunctionSpec->getValueAsListOfDefs("Args");
+  std::stringstream CallArgs;
+  std::string ArgPrefix("__arg");
+  for (size_t i = 0; i < ArgsList.size(); ++i) {
+    llvm::Record *ArgType = ArgsList[i]->getValueAsDef("ArgType");
+    auto TypeName = Indexer.getTypeAsString(ArgType);
+    OS << TypeName << " " << ArgPrefix << i;
+    CallArgs << ArgPrefix << i;
+    if (i < ArgsList.size() - 1) {
+      OS << ", ";
+      CallArgs << ", ";
+    }
+  }
+
+  // TODO: Arg types of the C++ implementation functions need not
+  // match the standard types. Either handle such differences here, or
+  // avoid such a thing in the implementations.
+  OS << ") {\n"
+     << "  return __llvm_libc::" << FunctionName << "(" << CallArgs.str()
+     << ");\n"
+     << "}\n";
+
+  return false;
+}
+
+int main(int argc, char *argv[]) {
+  llvm::cl::ParseCommandLineOptions(argc, argv);
+  return TableGenMain(argv[0], WrapperGenMain);
+}

From abb812823701cbbb615fea0d6fe731725eb4162f Mon Sep 17 00:00:00 2001
From: Mircea Trofin <mtrofin@google.com>
Date: Thu, 30 Jul 2020 16:08:06 -0700
Subject: [PATCH 0684/1035] [doc] Describe the header guard style

clang-tidy's llvm-header-guard rule references the LLVM style - where it's
missing.

Differential Revision: https://reviews.llvm.org/D84989
---
 llvm/docs/CodingStandards.rst | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/llvm/docs/CodingStandards.rst b/llvm/docs/CodingStandards.rst
index a9884cd9f3b5a..9c95e3ebf28a3 100644
--- a/llvm/docs/CodingStandards.rst
+++ b/llvm/docs/CodingStandards.rst
@@ -174,6 +174,16 @@ used as an abstract.  Any additional information should be separated by a blank
 line.  If an algorithm is based on a paper or is described in another source,
 provide a reference.
 
+Header Guard
+""""""""""""
+
+The header file's guard should be the all-caps path that a user of this header
+would #include, using '_' instead of path separator and extension marker. 
+For example, the header file
+``llvm/include/llvm/Analysis/Utils/Local.h`` would be ``#include``-ed as 
+``#include "llvm/Analysis/Utils/Local.h"``, so its guard is 
+``LLVM_ANALYSIS_UTILS_LOCAL_H``.
+
 Class overviews
 """""""""""""""
 

From 09f9f7dd1bfe3ec2625fec4de36f4d9c63593bf7 Mon Sep 17 00:00:00 2001
From: Amara Emerson <amara@apple.com>
Date: Fri, 24 Jul 2020 13:01:36 -0700
Subject: [PATCH 0685/1035] [AArch64][GlobalISel] Add legalization & selection
 support for G_INTRINSIC_LRINT.

Differential Revision: https://reviews.llvm.org/D84552
---
 .../Target/GlobalISel/SelectionDAGCompat.td   |  1 +
 .../AArch64/GISel/AArch64LegalizerInfo.cpp    |  2 +-
 .../AArch64/GlobalISel/legalize-lrint.mir     | 94 +++++++++++++++++++
 .../GlobalISel/legalizer-info-validation.mir  |  5 +-
 llvm/test/CodeGen/AArch64/lrint-conv.ll       |  5 +
 5 files changed, 104 insertions(+), 3 deletions(-)
 create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/legalize-lrint.mir

diff --git a/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td b/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td
index 14718556810f5..d3ca771ad3adb 100644
--- a/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td
+++ b/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td
@@ -125,6 +125,7 @@ def : GINodeEquiv<G_FRINT, frint>;
 def : GINodeEquiv<G_FNEARBYINT, fnearbyint>;
 def : GINodeEquiv<G_INTRINSIC_TRUNC, ftrunc>;
 def : GINodeEquiv<G_INTRINSIC_ROUND, fround>;
+def : GINodeEquiv<G_INTRINSIC_LRINT, lrint>;
 def : GINodeEquiv<G_FCOPYSIGN, fcopysign>;
 def : GINodeEquiv<G_SMIN, smin>;
 def : GINodeEquiv<G_SMAX, smax>;
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index 54e368cd7768f..6e5563af43638 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -156,7 +156,7 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
 
   getActionDefinitionsBuilder({G_FCEIL, G_FABS, G_FSQRT, G_FFLOOR, G_FRINT,
                                G_FMA, G_INTRINSIC_TRUNC, G_INTRINSIC_ROUND,
-                               G_FNEARBYINT})
+                               G_FNEARBYINT, G_INTRINSIC_LRINT})
       // If we don't have full FP16 support, then scalarize the elements of
       // vectors containing fp16 types.
       .fewerElementsIf(
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-lrint.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-lrint.mir
new file mode 100644
index 0000000000000..ddc4f07e1a146
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-lrint.mir
@@ -0,0 +1,94 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -verify-machineinstrs -mtriple aarch64-unknown-unknown -run-pass=legalizer %s -o - | FileCheck %s
+---
+name:            testmsws
+alignment:       4
+tracksRegLiveness: true
+liveins:
+  - { reg: '$s0' }
+body:             |
+  bb.1:
+    liveins: $s0
+
+    ; CHECK-LABEL: name: testmsws
+    ; CHECK: liveins: $s0
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK: [[INTRINSIC_LRINT:%[0-9]+]]:_(s64) = G_INTRINSIC_LRINT [[COPY]](s32)
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[INTRINSIC_LRINT]](s64)
+    ; CHECK: $w0 = COPY [[TRUNC]](s32)
+    ; CHECK: RET_ReallyLR implicit $w0
+    %0:_(s32) = COPY $s0
+    %1:_(s64) = G_INTRINSIC_LRINT %0(s32)
+    %2:_(s32) = G_TRUNC %1(s64)
+    $w0 = COPY %2(s32)
+    RET_ReallyLR implicit $w0
+
+...
+---
+name:            testmsxs
+alignment:       4
+tracksRegLiveness: true
+liveins:
+  - { reg: '$s0' }
+body:             |
+  bb.1:
+    liveins: $s0
+
+    ; CHECK-LABEL: name: testmsxs
+    ; CHECK: liveins: $s0
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK: [[INTRINSIC_LRINT:%[0-9]+]]:_(s64) = G_INTRINSIC_LRINT [[COPY]](s32)
+    ; CHECK: $x0 = COPY [[INTRINSIC_LRINT]](s64)
+    ; CHECK: RET_ReallyLR implicit $x0
+    %0:_(s32) = COPY $s0
+    %1:_(s64) = G_INTRINSIC_LRINT %0(s32)
+    $x0 = COPY %1(s64)
+    RET_ReallyLR implicit $x0
+
+...
+---
+name:            testmswd
+alignment:       4
+tracksRegLiveness: true
+liveins:
+  - { reg: '$d0' }
+body:             |
+  bb.1:
+    liveins: $d0
+
+    ; CHECK-LABEL: name: testmswd
+    ; CHECK: liveins: $d0
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $d0
+    ; CHECK: [[INTRINSIC_LRINT:%[0-9]+]]:_(s64) = G_INTRINSIC_LRINT [[COPY]](s64)
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[INTRINSIC_LRINT]](s64)
+    ; CHECK: $w0 = COPY [[TRUNC]](s32)
+    ; CHECK: RET_ReallyLR implicit $w0
+    %0:_(s64) = COPY $d0
+    %1:_(s64) = G_INTRINSIC_LRINT %0(s64)
+    %2:_(s32) = G_TRUNC %1(s64)
+    $w0 = COPY %2(s32)
+    RET_ReallyLR implicit $w0
+
+...
+---
+name:            testmsxd
+alignment:       4
+tracksRegLiveness: true
+liveins:
+  - { reg: '$d0' }
+body:             |
+  bb.1:
+    liveins: $d0
+
+    ; CHECK-LABEL: name: testmsxd
+    ; CHECK: liveins: $d0
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $d0
+    ; CHECK: [[INTRINSIC_LRINT:%[0-9]+]]:_(s64) = G_INTRINSIC_LRINT [[COPY]](s64)
+    ; CHECK: $x0 = COPY [[INTRINSIC_LRINT]](s64)
+    ; CHECK: RET_ReallyLR implicit $x0
+    %0:_(s64) = COPY $d0
+    %1:_(s64) = G_INTRINSIC_LRINT %0(s64)
+    $x0 = COPY %1(s64)
+    RET_ReallyLR implicit $x0
+
+...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
index 705f7cd1a0589..110c57408c056 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
@@ -130,8 +130,9 @@
 # DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: G_INTRINSIC_LRINT (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
-# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
-# DEBUG-NEXT:.. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: G_INTRINSIC_ROUNDEVEN (opcode {{[0-9]+}}): 1 type index, 0 imm indices
 # DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
diff --git a/llvm/test/CodeGen/AArch64/lrint-conv.ll b/llvm/test/CodeGen/AArch64/lrint-conv.ll
index 14d078b96ff1c..80f1e8b8fc18e 100644
--- a/llvm/test/CodeGen/AArch64/lrint-conv.ll
+++ b/llvm/test/CodeGen/AArch64/lrint-conv.ll
@@ -1,9 +1,11 @@
 ; RUN: llc < %s -mtriple=aarch64 -mattr=+neon | FileCheck %s
+; RUN: llc < %s -global-isel -global-isel-abort=2 -pass-remarks-missed=gisel* -mtriple=aarch64 | FileCheck %s --check-prefixes=FALLBACK,CHECK
 
 ; CHECK-LABEL: testmsws:
 ; CHECK:       frintx  [[REG:s[0-9]]], s0
 ; CHECK-NEXT:  fcvtzs  x0, [[REG]]
 ; CHECK:       ret
+; FALLBACK-NOT: remark{{.*}}testmsws
 define i32 @testmsws(float %x) {
 entry:
   %0 = tail call i64 @llvm.lrint.i64.f32(float %x)
@@ -15,6 +17,7 @@ entry:
 ; CHECK:       frintx  [[REG:s[0-9]]], s0
 ; CHECK-NEXT:  fcvtzs  x0, [[REG]]
 ; CHECK-NEXT:  ret
+; FALLBACK-NOT: remark{{.*}}testmsxs
 define i64 @testmsxs(float %x) {
 entry:
   %0 = tail call i64 @llvm.lrint.i64.f32(float %x)
@@ -25,6 +28,7 @@ entry:
 ; CHECK:       frintx  [[REG:d[0-9]]], d0
 ; CHECK-NEXT:  fcvtzs  x0, [[REG]]
 ; CHECK:       ret
+; FALLBACK-NOT: remark{{.*}}testmswd
 define i32 @testmswd(double %x) {
 entry:
   %0 = tail call i64 @llvm.lrint.i64.f64(double %x)
@@ -36,6 +40,7 @@ entry:
 ; CHECK:       frintx  [[REG:d[0-9]]], d0
 ; CHECK-NEXT:  fcvtzs  x0, [[REG]]
 ; CHECK-NEXT:  ret
+; FALLBACK-NOT: remark{{.*}}testmsxd
 define i64 @testmsxd(double %x) {
 entry:
   %0 = tail call i64 @llvm.lrint.i64.f64(double %x)

From 7e88efa7c5313aab5d840021085f3924d00609bf Mon Sep 17 00:00:00 2001
From: Eli Friedman <efriedma@quicinc.com>
Date: Mon, 27 Jul 2020 14:08:31 -0700
Subject: [PATCH 0686/1035] [LegalizeTypes][SVE] Support widen/split
 legalization for SPLAT_VECTOR

Just the obvious implementation that rewrites the result type. Also fix
warning from EXTRACT_SUBVECTOR legalization that triggers on the test.

Differential Revision: https://reviews.llvm.org/D84706
---
 llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h |  4 +--
 .../SelectionDAG/LegalizeVectorTypes.cpp      | 33 ++++++++++++++-----
 llvm/test/CodeGen/AArch64/sve-vector-splat.ll | 13 ++++++++
 3 files changed, 39 insertions(+), 11 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
index 0fa6d653a8364..9e2ae6b8f16f0 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
@@ -806,7 +806,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
   void SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo, SDValue &Hi);
   void SplitVecRes_MLOAD(MaskedLoadSDNode *MLD, SDValue &Lo, SDValue &Hi);
   void SplitVecRes_MGATHER(MaskedGatherSDNode *MGT, SDValue &Lo, SDValue &Hi);
-  void SplitVecRes_SCALAR_TO_VECTOR(SDNode *N, SDValue &Lo, SDValue &Hi);
+  void SplitVecRes_ScalarOp(SDNode *N, SDValue &Lo, SDValue &Hi);
   void SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi);
   void SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N, SDValue &Lo,
                                   SDValue &Hi);
@@ -862,7 +862,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
   SDValue WidenVecRes_LOAD(SDNode* N);
   SDValue WidenVecRes_MLOAD(MaskedLoadSDNode* N);
   SDValue WidenVecRes_MGATHER(MaskedGatherSDNode* N);
-  SDValue WidenVecRes_SCALAR_TO_VECTOR(SDNode* N);
+  SDValue WidenVecRes_ScalarOp(SDNode* N);
   SDValue WidenVecRes_SELECT(SDNode* N);
   SDValue WidenVSELECTAndMask(SDNode *N);
   SDValue WidenVecRes_SELECT_CC(SDNode* N);
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index c81d03cac81bc..94c91854466eb 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -836,7 +836,10 @@ void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
   case ISD::FPOWI:             SplitVecRes_FPOWI(N, Lo, Hi); break;
   case ISD::FCOPYSIGN:         SplitVecRes_FCOPYSIGN(N, Lo, Hi); break;
   case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break;
-  case ISD::SCALAR_TO_VECTOR:  SplitVecRes_SCALAR_TO_VECTOR(N, Lo, Hi); break;
+  case ISD::SPLAT_VECTOR:
+  case ISD::SCALAR_TO_VECTOR:
+    SplitVecRes_ScalarOp(N, Lo, Hi);
+    break;
   case ISD::SIGN_EXTEND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break;
   case ISD::LOAD:
     SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi);
@@ -1517,13 +1520,18 @@ void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo,
     Hi = DAG.getNode(ISD::TRUNCATE, dl, HiVT, Hi);
 }
 
-void DAGTypeLegalizer::SplitVecRes_SCALAR_TO_VECTOR(SDNode *N, SDValue &Lo,
-                                                    SDValue &Hi) {
+void DAGTypeLegalizer::SplitVecRes_ScalarOp(SDNode *N, SDValue &Lo,
+                                            SDValue &Hi) {
   EVT LoVT, HiVT;
   SDLoc dl(N);
   std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
-  Lo = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LoVT, N->getOperand(0));
-  Hi = DAG.getUNDEF(HiVT);
+  Lo = DAG.getNode(N->getOpcode(), dl, LoVT, N->getOperand(0));
+  if (N->getOpcode() == ISD::SCALAR_TO_VECTOR) {
+    Hi = DAG.getUNDEF(HiVT);
+  } else {
+    assert(N->getOpcode() == ISD::SPLAT_VECTOR && "Unexpected opcode");
+    Hi = Lo;
+  }
 }
 
 void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo,
@@ -2194,6 +2202,11 @@ SDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) {
 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
   // We know that the extracted result type is legal.
   EVT SubVT = N->getValueType(0);
+
+  if (SubVT.isScalableVector() !=
+      N->getOperand(0).getValueType().isScalableVector())
+    report_fatal_error("Extracting fixed from scalable not implemented");
+
   SDValue Idx = N->getOperand(1);
   SDLoc dl(N);
   SDValue Lo, Hi;
@@ -2739,7 +2752,10 @@ void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
   case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break;
   case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break;
   case ISD::LOAD:              Res = WidenVecRes_LOAD(N); break;
-  case ISD::SCALAR_TO_VECTOR:  Res = WidenVecRes_SCALAR_TO_VECTOR(N); break;
+  case ISD::SPLAT_VECTOR:
+  case ISD::SCALAR_TO_VECTOR:
+    Res = WidenVecRes_ScalarOp(N);
+    break;
   case ISD::SIGN_EXTEND_INREG: Res = WidenVecRes_InregOp(N); break;
   case ISD::VSELECT:
   case ISD::SELECT:            Res = WidenVecRes_SELECT(N); break;
@@ -3830,10 +3846,9 @@ SDValue DAGTypeLegalizer::WidenVecRes_MGATHER(MaskedGatherSDNode *N) {
   return Res;
 }
 
-SDValue DAGTypeLegalizer::WidenVecRes_SCALAR_TO_VECTOR(SDNode *N) {
+SDValue DAGTypeLegalizer::WidenVecRes_ScalarOp(SDNode *N) {
   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
-  return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N),
-                     WidenVT, N->getOperand(0));
+  return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, N->getOperand(0));
 }
 
 // Return true is this is a SETCC node or a strict version of it.
diff --git a/llvm/test/CodeGen/AArch64/sve-vector-splat.ll b/llvm/test/CodeGen/AArch64/sve-vector-splat.ll
index 7a53e5879c908..043e4ea135f34 100644
--- a/llvm/test/CodeGen/AArch64/sve-vector-splat.ll
+++ b/llvm/test/CodeGen/AArch64/sve-vector-splat.ll
@@ -134,6 +134,19 @@ define <vscale x 2 x i32> @sve_splat_2xi32(i32 %val) {
   ret <vscale x 2 x i32> %splat
 }
 
+;; Widen/split splats of wide vector types.
+
+define <vscale x 12 x i32> @sve_splat_12xi32(i32 %val) {
+; CHECK-LABEL: @sve_splat_12xi32
+; CHECK: mov z0.s, w0
+; CHECK-NEXT: mov z1.d, z0.d
+; CHECK-NEXT: mov z2.d, z0.d
+; CHECK-NEXT: ret
+  %ins = insertelement <vscale x 12 x i32> undef, i32 %val, i32 0
+  %splat = shufflevector <vscale x 12 x i32> %ins, <vscale x 12 x i32> undef, <vscale x 12 x i32> zeroinitializer
+  ret <vscale x 12 x i32> %splat
+}
+
 define <vscale x 2 x i1> @sve_splat_2xi1(i1 %val) {
 ; CHECK-LABEL: @sve_splat_2xi1
 ; CHECK: sbfx x8, x0, #0, #1

From 57605758b5de3726eec1d6e587de1003af1ab5b7 Mon Sep 17 00:00:00 2001
From: Davide Italiano <dccitaliano@gmail.com>
Date: Thu, 30 Jul 2020 16:20:38 -0700
Subject: [PATCH 0687/1035] [debugserver/Apple Silicon] Handoff connections
 when attaching to translated processes

When we detect a process that the native debugserver cannot handle,
handoff the connection fd to the translated debugserver.
---
 lldb/tools/debugserver/source/DNB.cpp         | 33 +++++++++++++++++++
 lldb/tools/debugserver/source/debugserver.cpp |  3 +-
 2 files changed, 35 insertions(+), 1 deletion(-)

diff --git a/lldb/tools/debugserver/source/DNB.cpp b/lldb/tools/debugserver/source/DNB.cpp
index 0830ea36a91ae..3c1cd85dc310f 100644
--- a/lldb/tools/debugserver/source/DNB.cpp
+++ b/lldb/tools/debugserver/source/DNB.cpp
@@ -442,6 +442,39 @@ nub_process_t DNBProcessAttach(nub_process_t attach_pid,
   if (err_str && err_len > 0)
     err_str[0] = '\0';
 
+  if (getenv("LLDB_DEBUGSERVER_PATH") == NULL) {
+    int mib[] = {CTL_KERN, KERN_PROC, KERN_PROC_PID,
+                 static_cast<int>(attach_pid)};
+    struct kinfo_proc processInfo;
+    size_t bufsize = sizeof(processInfo);
+    if (sysctl(mib, (unsigned)(sizeof(mib) / sizeof(int)), &processInfo,
+               &bufsize, NULL, 0) == 0 &&
+        bufsize > 0) {
+
+      if ((processInfo.kp_proc.p_flag & P_TRANSLATED) == P_TRANSLATED) {
+        const char *translated_debugserver =
+            "/Library/Apple/usr/libexec/oah/debugserver";
+        char fdstr[16];
+        char pidstr[16];
+        extern int communication_fd;
+
+        if (communication_fd == -1) {
+          fprintf(stderr, "Trying to attach to a translated process with the "
+                          "native debugserver, exiting...\n");
+          exit(1);
+        }
+
+        snprintf(fdstr, sizeof(fdstr), "--fd=%d", communication_fd);
+        snprintf(pidstr, sizeof(pidstr), "--attach=%d", attach_pid);
+        execl(translated_debugserver, "--native-regs", "--setsid", fdstr,
+              "--handoff-attach-from-native", pidstr, (char *)0);
+        DNBLogThreadedIf(LOG_PROCESS, "Failed to launch debugserver for "
+                         "translated process: ", errno, strerror(errno));
+        __builtin_trap();
+      }
+    }
+  }
+
   pid_t pid = INVALID_NUB_PROCESS;
   MachProcessSP processSP(new MachProcess);
   if (processSP.get()) {
diff --git a/lldb/tools/debugserver/source/debugserver.cpp b/lldb/tools/debugserver/source/debugserver.cpp
index 42205dedf4bb8..410ea7c310fad 100644
--- a/lldb/tools/debugserver/source/debugserver.cpp
+++ b/lldb/tools/debugserver/source/debugserver.cpp
@@ -878,6 +878,8 @@ static struct option g_long_options[] = {
            // -F localhost:1234 -- /bin/ls"
     {NULL, 0, NULL, 0}};
 
+int communication_fd = -1;
+
 // main
 int main(int argc, char *argv[]) {
   // If debugserver is launched with DYLD_INSERT_LIBRARIES, unset it so we
@@ -944,7 +946,6 @@ int main(int argc, char *argv[]) {
   int ch;
   int long_option_index = 0;
   int debug = 0;
-  int communication_fd = -1;
   std::string compile_options;
   std::string waitfor_pid_name; // Wait for a process that starts with this name
   std::string attach_pid_name;

From 896f797b8bb7683f7e52c6dc94979f3c449bc37d Mon Sep 17 00:00:00 2001
From: Vedant Kumar <vsk@apple.com>
Date: Thu, 30 Jul 2020 16:19:05 -0700
Subject: [PATCH 0688/1035] [profile] Remove dependence on getpagesize from
 InstrProfilingBuffer.c.o

InstrProfilingBuffer.c.o is generic code that must support compilation
into freestanding projects. This gets rid of its dependence on the
_getpagesize symbol from libc, shifting it to InstrProfilingFile.c.o.

This fixes a build failure seen in a firmware project.

rdar://66249701
---
 compiler-rt/lib/profile/InstrProfiling.h      |  9 ++++++++
 .../lib/profile/InstrProfilingBuffer.c        | 21 ++++++++++++-------
 compiler-rt/lib/profile/InstrProfilingFile.c  |  1 +
 .../test/profile/instrprof-without-libc.c     |  1 +
 4 files changed, 24 insertions(+), 8 deletions(-)

diff --git a/compiler-rt/lib/profile/InstrProfiling.h b/compiler-rt/lib/profile/InstrProfiling.h
index d7a7c32332c1b..7d1c77a3fab3e 100644
--- a/compiler-rt/lib/profile/InstrProfiling.h
+++ b/compiler-rt/lib/profile/InstrProfiling.h
@@ -54,6 +54,15 @@ int __llvm_profile_is_continuous_mode_enabled(void);
  */
 void __llvm_profile_enable_continuous_mode(void);
 
+/*!
+ * \brief Set the page size.
+ *
+ * This is a pre-requisite for enabling continuous mode. The buffer size
+ * calculation code inside of libprofile cannot simply call getpagesize(), as
+ * it is not allowed to depend on libc.
+ */
+void __llvm_profile_set_page_size(unsigned PageSize);
+
 /*!
  * \brief Get number of bytes necessary to pad the argument to eight
  * byte boundary.
diff --git a/compiler-rt/lib/profile/InstrProfilingBuffer.c b/compiler-rt/lib/profile/InstrProfilingBuffer.c
index 800ae22f2746c..07bb4d4e4f1b4 100644
--- a/compiler-rt/lib/profile/InstrProfilingBuffer.c
+++ b/compiler-rt/lib/profile/InstrProfilingBuffer.c
@@ -21,14 +21,22 @@
  * layering is violated. */
 static int ContinuouslySyncProfile = 0;
 
+/* The system page size. Only valid when non-zero. If 0, the page size is
+ * unavailable. */
+static unsigned PageSize = 0;
+
 COMPILER_RT_VISIBILITY int __llvm_profile_is_continuous_mode_enabled(void) {
-  return ContinuouslySyncProfile;
+  return ContinuouslySyncProfile && PageSize;
 }
 
 COMPILER_RT_VISIBILITY void __llvm_profile_enable_continuous_mode(void) {
   ContinuouslySyncProfile = 1;
 }
 
+COMPILER_RT_VISIBILITY void __llvm_profile_set_page_size(unsigned PS) {
+  PageSize = PS;
+}
+
 COMPILER_RT_VISIBILITY
 uint64_t __llvm_profile_get_size_for_buffer(void) {
   const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
@@ -52,8 +60,7 @@ uint64_t __llvm_profile_get_data_size(const __llvm_profile_data *Begin,
 
 /// Calculate the number of padding bytes needed to add to \p Offset in order
 /// for (\p Offset + Padding) to be page-aligned.
-static uint64_t calculateBytesNeededToPageAlign(uint64_t Offset,
-                                                unsigned PageSize) {
+static uint64_t calculateBytesNeededToPageAlign(uint64_t Offset) {
   uint64_t OffsetModPage = Offset % PageSize;
   if (OffsetModPage > 0)
     return PageSize - OffsetModPage;
@@ -75,15 +82,13 @@ void __llvm_profile_get_padding_sizes_for_counters(
 
   // In continuous mode, the file offsets for headers and for the start of
   // counter sections need to be page-aligned.
-  unsigned PageSize = getpagesize();
   uint64_t DataSizeInBytes = DataSize * sizeof(__llvm_profile_data);
   uint64_t CountersSizeInBytes = CountersSize * sizeof(uint64_t);
   *PaddingBytesBeforeCounters = calculateBytesNeededToPageAlign(
-      sizeof(__llvm_profile_header) + DataSizeInBytes, PageSize);
+      sizeof(__llvm_profile_header) + DataSizeInBytes);
   *PaddingBytesAfterCounters =
-      calculateBytesNeededToPageAlign(CountersSizeInBytes, PageSize);
-  *PaddingBytesAfterNames =
-      calculateBytesNeededToPageAlign(NamesSize, PageSize);
+      calculateBytesNeededToPageAlign(CountersSizeInBytes);
+  *PaddingBytesAfterNames = calculateBytesNeededToPageAlign(NamesSize);
 }
 
 COMPILER_RT_VISIBILITY
diff --git a/compiler-rt/lib/profile/InstrProfilingFile.c b/compiler-rt/lib/profile/InstrProfilingFile.c
index 9e1a54a0c3737..8c7bb0c25de16 100644
--- a/compiler-rt/lib/profile/InstrProfilingFile.c
+++ b/compiler-rt/lib/profile/InstrProfilingFile.c
@@ -751,6 +751,7 @@ static int parseFilenamePattern(const char *FilenamePat,
           return -1;
         }
 
+        __llvm_profile_set_page_size(getpagesize());
         __llvm_profile_enable_continuous_mode();
         I++; /* advance to 'c' */
       } else {
diff --git a/compiler-rt/test/profile/instrprof-without-libc.c b/compiler-rt/test/profile/instrprof-without-libc.c
index 6e9c1dde01e6a..cd9fb5e1dd8e0 100644
--- a/compiler-rt/test/profile/instrprof-without-libc.c
+++ b/compiler-rt/test/profile/instrprof-without-libc.c
@@ -72,3 +72,4 @@ int main(int argc, const char *argv[]) {
 // CHECK-SYMBOLS-NOT: {{ }}_free
 // CHECK-SYMBOLS-NOT: {{ }}free
 // CHECK-SYMBOLS-NOT: {{ }}_open
+// CHECK-SYMBOLS-NOT: {{ }}_getpagesize

From 24f5235d936537148fbfea0a6583ad7f2f775402 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@intel.com>
Date: Thu, 30 Jul 2020 11:25:55 -0700
Subject: [PATCH 0689/1035] [ValueTracking] Add basic computeKnownBits support
 for llvm.abs intrinsic

This includes basic support for computeKnownBits on abs. I've left FIXMEs for more complicated things we could do.

Differential Revision: https://reviews.llvm.org/D84963
---
 llvm/lib/Analysis/ValueTracking.cpp           | 11 ++++
 .../Transforms/InstCombine/abs-intrinsic.ll   | 53 +++++++++++++++++++
 2 files changed, 64 insertions(+)
 create mode 100644 llvm/test/Transforms/InstCombine/abs-intrinsic.ll

diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index 3bf09b30b2f9c..3b8c0fcca1617 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -1642,6 +1642,17 @@ static void computeKnownBitsFromOperator(const Operator *I,
     if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) {
       switch (II->getIntrinsicID()) {
       default: break;
+      case Intrinsic::abs:
+        computeKnownBits(I->getOperand(0), Known2, Depth + 1, Q);
+        // Otherwise, if this call is undefined for INT_MIN, the result is
+        // positive.
+        if (match(II->getArgOperand(1), m_One()))
+          Known.Zero.setSignBit();
+        // Absolute value preserves trailing zero count.
+        Known.Zero.setLowBits(Known2.Zero.countTrailingOnes());
+        // FIXME: Handle known negative/non-negative input?
+        // FIXME: Calculate the negated Known bits and combine them?
+        break;
       case Intrinsic::bitreverse:
         computeKnownBits(I->getOperand(0), DemandedElts, Known2, Depth + 1, Q);
         Known.Zero |= Known2.Zero.reverseBits();
diff --git a/llvm/test/Transforms/InstCombine/abs-intrinsic.ll b/llvm/test/Transforms/InstCombine/abs-intrinsic.ll
new file mode 100644
index 0000000000000..780f28238bf60
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/abs-intrinsic.ll
@@ -0,0 +1,53 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+declare i32 @llvm.abs.i32(i32, i1)
+
+define i1 @abs_nsw_must_be_positive(i32 %x) {
+; CHECK-LABEL: @abs_nsw_must_be_positive(
+; CHECK-NEXT:    ret i1 true
+;
+  %abs = call i32 @llvm.abs.i32(i32 %x, i1 true)
+  %c2 = icmp sge i32 %abs, 0
+  ret i1 %c2
+}
+
+; Negative test, no nsw provides no information about the sign bit of the result.
+define i1 @abs_nonsw(i32 %x) {
+; CHECK-LABEL: @abs_nonsw(
+; CHECK-NEXT:    [[ABS:%.*]] = call i32 @llvm.abs.i32(i32 [[X:%.*]], i1 false)
+; CHECK-NEXT:    [[C2:%.*]] = icmp sgt i32 [[ABS]], -1
+; CHECK-NEXT:    ret i1 [[C2]]
+;
+  %abs = call i32 @llvm.abs.i32(i32 %x, i1 false)
+  %c2 = icmp sge i32 %abs, 0
+  ret i1 %c2
+}
+
+; abs preserves trailing zeros so the second and is unneeded
+define i32 @abs_trailing_zeros(i32 %x) {
+; CHECK-LABEL: @abs_trailing_zeros(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], -4
+; CHECK-NEXT:    [[ABS:%.*]] = call i32 @llvm.abs.i32(i32 [[AND]], i1 false)
+; CHECK-NEXT:    ret i32 [[ABS]]
+;
+  %and = and i32 %x, -4
+  %abs = call i32 @llvm.abs.i32(i32 %and, i1 false)
+  %and2 = and i32 %abs, -2
+  ret i32 %and2
+}
+
+; negative test, can't remove the second and based on trailing zeroes.
+; FIXME: Could remove the first and using demanded bits.
+define i32 @abs_trailing_zeros_negative(i32 %x) {
+; CHECK-LABEL: @abs_trailing_zeros_negative(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], -2
+; CHECK-NEXT:    [[ABS:%.*]] = call i32 @llvm.abs.i32(i32 [[AND]], i1 false)
+; CHECK-NEXT:    [[AND2:%.*]] = and i32 [[ABS]], -4
+; CHECK-NEXT:    ret i32 [[AND2]]
+;
+  %and = and i32 %x, -2
+  %abs = call i32 @llvm.abs.i32(i32 %and, i1 false)
+  %and2 = and i32 %abs, -4
+  ret i32 %and2
+}

From 49def10e02e35e56d1249405a650b161245f87b0 Mon Sep 17 00:00:00 2001
From: kuterd <kuterdinel@gmail.com>
Date: Thu, 30 Jul 2020 23:26:39 +0300
Subject: [PATCH 0690/1035] [Attributor] Add time trace support.

This patch addes time trace functionality to have a better understanding
of the analysis times.

Reviewed By: jdoerfert

Differential Revision: https://reviews.llvm.org/D84980
---
 llvm/include/llvm/Transforms/IPO/Attributor.h | 7 +++++--
 llvm/lib/Transforms/IPO/Attributor.cpp        | 7 +++++++
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/llvm/include/llvm/Transforms/IPO/Attributor.h b/llvm/include/llvm/Transforms/IPO/Attributor.h
index 0fa9c99a882cc..bd6ff03a31fad 100644
--- a/llvm/include/llvm/Transforms/IPO/Attributor.h
+++ b/llvm/include/llvm/Transforms/IPO/Attributor.h
@@ -117,6 +117,7 @@
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/DOTGraphTraits.h"
 #include "llvm/Support/GraphWriter.h"
+#include "llvm/Support/TimeProfiler.h"
 #include "llvm/Transforms/Utils/CallGraphUpdater.h"
 
 namespace llvm {
@@ -1001,8 +1002,10 @@ struct Attributor {
       return AA;
     }
 
-    AA.initialize(*this);
-
+    {
+      TimeTraceScope TimeScope(AA.getName() + "::initialize");
+      AA.initialize(*this);
+    }
     // We can initialize (=look at) code outside the current function set but
     // not call update because that would again spawn new abstract attributes in
     // potentially unconnected code regions (=SCCs).
diff --git a/llvm/lib/Transforms/IPO/Attributor.cpp b/llvm/lib/Transforms/IPO/Attributor.cpp
index 1be3782e77d71..89971cc9c2946 100644
--- a/llvm/lib/Transforms/IPO/Attributor.cpp
+++ b/llvm/lib/Transforms/IPO/Attributor.cpp
@@ -931,6 +931,7 @@ bool Attributor::checkForAllReadWriteInstructions(
 }
 
 void Attributor::runTillFixpoint() {
+  TimeTraceScope TimeScope("Attributor::runTillFixpoint");
   LLVM_DEBUG(dbgs() << "[Attributor] Identified and initialized "
                     << DG.SyntheticRoot.Deps.size()
                     << " abstract attributes.\n");
@@ -1067,6 +1068,7 @@ void Attributor::runTillFixpoint() {
 }
 
 ChangeStatus Attributor::manifestAttributes() {
+  TimeTraceScope TimeScope("Attributor::manifestAttributes");
   size_t NumFinalAAs = DG.SyntheticRoot.Deps.size();
 
   unsigned NumManifested = 0;
@@ -1129,6 +1131,7 @@ ChangeStatus Attributor::manifestAttributes() {
 }
 
 ChangeStatus Attributor::cleanupIR() {
+  TimeTraceScope TimeScope("Attributor::cleanupIR");
   // Delete stuff at the end to avoid invalid references and a nice order.
   LLVM_DEBUG(dbgs() << "\n[Attributor] Delete at least "
                     << ToBeDeletedFunctions.size() << " functions and "
@@ -1297,6 +1300,8 @@ ChangeStatus Attributor::cleanupIR() {
 }
 
 ChangeStatus Attributor::run() {
+  TimeTraceScope TimeScope("Attributor::run");
+
   SeedingPeriod = false;
   runTillFixpoint();
 
@@ -1316,6 +1321,8 @@ ChangeStatus Attributor::run() {
 }
 
 ChangeStatus Attributor::updateAA(AbstractAttribute &AA) {
+  TimeTraceScope TimeScope(AA.getName() + "::updateAA");
+
   // Use a new dependence vector for this update.
   DependenceVector DV;
   DependenceStack.push_back(&DV);

From 3ad09fd03c51823aeb0bcbd7898aada33e9228d6 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@intel.com>
Date: Thu, 30 Jul 2020 17:05:06 -0700
Subject: [PATCH 0691/1035] [X86] Separate CPU Feature lists in X86.td between
 architecture features and tuning features

After the recent change to the tuning settings for pentium4 to improve our default 32-bit behavior, I've decided to see about implementing -mtune support. This way we could have a default architecture CPU of "pentium4" or "x86-64" and a default tuning cpu of "generic". And we could change our "pentium4" tuning settings back to what they were before.

As a step to supporting this, this patch separates all of the features lists for the CPUs into 2 lists. I'm using the Proc class and a new ProcModel class to concat the 2 lists before passing to the target independent ProcessorModel. Future work to truly support mtune would change ProcessorModel to take 2 lists separately. I've diffed the X86GenSubtargetInfo.inc file before and after this patch to ensure that the final feature list for the CPUs isn't changed.

Differential Revision: https://reviews.llvm.org/D84879
---
 llvm/lib/Target/X86/X86.td | 763 +++++++++++++++++++------------------
 1 file changed, 402 insertions(+), 361 deletions(-)

diff --git a/llvm/lib/Target/X86/X86.td b/llvm/lib/Target/X86/X86.td
index f9ed55ea37d49..332033b8fe257 100644
--- a/llvm/lib/Target/X86/X86.td
+++ b/llvm/lib/Target/X86/X86.td
@@ -547,57 +547,49 @@ include "X86SchedSkylakeServer.td"
 
 def ProcessorFeatures {
   // Nehalem
-  list<SubtargetFeature> NHMInheritableFeatures = [FeatureX87,
-                                                   FeatureCMPXCHG8B,
-                                                   FeatureCMOV,
-                                                   FeatureMMX,
-                                                   FeatureSSE42,
-                                                   FeatureFXSR,
-                                                   FeatureNOPL,
-                                                   Feature64Bit,
-                                                   FeatureCMPXCHG16B,
-                                                   FeaturePOPCNT,
-                                                   FeatureLAHFSAHF,
-                                                   FeatureMacroFusion,
-                                                   FeatureInsertVZEROUPPER];
-  list<SubtargetFeature> NHMSpecificFeatures = [];
-  list<SubtargetFeature> NHMFeatures =
-    !listconcat(NHMInheritableFeatures, NHMSpecificFeatures);
+  list<SubtargetFeature> NHMFeatures = [FeatureX87,
+                                        FeatureCMPXCHG8B,
+                                        FeatureCMOV,
+                                        FeatureMMX,
+                                        FeatureSSE42,
+                                        FeatureFXSR,
+                                        FeatureNOPL,
+                                        Feature64Bit,
+                                        FeatureCMPXCHG16B,
+                                        FeaturePOPCNT,
+                                        FeatureLAHFSAHF];
+  list<SubtargetFeature> NHMTuning = [FeatureMacroFusion,
+                                      FeatureInsertVZEROUPPER];
 
   // Westmere
   list<SubtargetFeature> WSMAdditionalFeatures = [FeaturePCLMUL];
-  list<SubtargetFeature> WSMSpecificFeatures = [];
-  list<SubtargetFeature> WSMInheritableFeatures =
-    !listconcat(NHMInheritableFeatures, WSMAdditionalFeatures);
+  list<SubtargetFeature> WSMTuning = NHMTuning;
   list<SubtargetFeature> WSMFeatures =
-    !listconcat(WSMInheritableFeatures, WSMSpecificFeatures);
+    !listconcat(NHMFeatures, WSMAdditionalFeatures);
 
   // Sandybridge
   list<SubtargetFeature> SNBAdditionalFeatures = [FeatureAVX,
-                                                  FeatureSlowDivide64,
                                                   FeatureXSAVE,
-                                                  FeatureXSAVEOPT,
-                                                  FeatureSlow3OpsLEA,
-                                                  FeatureFastScalarFSQRT,
-                                                  FeatureFastSHLDRotate,
-                                                  FeatureFast15ByteNOP];
-  list<SubtargetFeature> SNBSpecificFeatures = [FeatureSlowUAMem32,
-                                                FeaturePOPCNTFalseDeps];
-  list<SubtargetFeature> SNBInheritableFeatures =
-    !listconcat(WSMInheritableFeatures, SNBAdditionalFeatures);
+                                                  FeatureXSAVEOPT];
+  list<SubtargetFeature> SNBTuning = [FeatureMacroFusion,
+                                      FeatureSlow3OpsLEA,
+                                      FeatureSlowDivide64,
+                                      FeatureSlowUAMem32,
+                                      FeatureFastScalarFSQRT,
+                                      FeatureFastSHLDRotate,
+                                      FeatureFast15ByteNOP,
+                                      FeaturePOPCNTFalseDeps,
+                                      FeatureInsertVZEROUPPER];
   list<SubtargetFeature> SNBFeatures =
-    !listconcat(SNBInheritableFeatures, SNBSpecificFeatures);
+    !listconcat(WSMFeatures, SNBAdditionalFeatures);
 
   // Ivybridge
   list<SubtargetFeature> IVBAdditionalFeatures = [FeatureRDRAND,
                                                   FeatureF16C,
                                                   FeatureFSGSBase];
-  list<SubtargetFeature> IVBSpecificFeatures = [FeatureSlowUAMem32,
-                                                FeaturePOPCNTFalseDeps];
-  list<SubtargetFeature> IVBInheritableFeatures =
-    !listconcat(SNBInheritableFeatures, IVBAdditionalFeatures);
+  list<SubtargetFeature> IVBTuning = SNBTuning;
   list<SubtargetFeature> IVBFeatures =
-    !listconcat(IVBInheritableFeatures, IVBSpecificFeatures);
+    !listconcat(SNBFeatures, IVBAdditionalFeatures);
 
   // Haswell
   list<SubtargetFeature> HSWAdditionalFeatures = [FeatureAVX2,
@@ -607,77 +599,89 @@ def ProcessorFeatures {
                                                   FeatureFMA,
                                                   FeatureINVPCID,
                                                   FeatureLZCNT,
-                                                  FeatureMOVBE,
-                                                  FeatureFastVariableShuffle];
-  list<SubtargetFeature> HSWSpecificFeatures = [FeaturePOPCNTFalseDeps,
-                                                FeatureLZCNTFalseDeps];
-  list<SubtargetFeature> HSWInheritableFeatures =
-    !listconcat(IVBInheritableFeatures, HSWAdditionalFeatures);
+                                                  FeatureMOVBE];
+  list<SubtargetFeature> HSWTuning = [FeatureMacroFusion,
+                                      FeatureSlow3OpsLEA,
+                                      FeatureSlowDivide64,
+                                      FeatureFastScalarFSQRT,
+                                      FeatureFastSHLDRotate,
+                                      FeatureFast15ByteNOP,
+                                      FeatureFastVariableShuffle,
+                                      FeaturePOPCNTFalseDeps,
+                                      FeatureLZCNTFalseDeps,
+                                      FeatureInsertVZEROUPPER];
   list<SubtargetFeature> HSWFeatures =
-    !listconcat(HSWInheritableFeatures, HSWSpecificFeatures);
+    !listconcat(IVBFeatures, HSWAdditionalFeatures);
 
   // Broadwell
   list<SubtargetFeature> BDWAdditionalFeatures = [FeatureADX,
                                                   FeatureRDSEED,
                                                   FeaturePRFCHW];
-  list<SubtargetFeature> BDWSpecificFeatures = [FeaturePOPCNTFalseDeps,
-                                                FeatureLZCNTFalseDeps];
-  list<SubtargetFeature> BDWInheritableFeatures =
-    !listconcat(HSWInheritableFeatures, BDWAdditionalFeatures);
+  list<SubtargetFeature> BDWTuning = HSWTuning;
   list<SubtargetFeature> BDWFeatures =
-    !listconcat(BDWInheritableFeatures, BDWSpecificFeatures);
+    !listconcat(HSWFeatures, BDWAdditionalFeatures);
 
   // Skylake
   list<SubtargetFeature> SKLAdditionalFeatures = [FeatureAES,
                                                   FeatureXSAVEC,
                                                   FeatureXSAVES,
                                                   FeatureCLFLUSHOPT,
-                                                  FeatureFastVectorFSQRT];
-  list<SubtargetFeature> SKLSpecificFeatures = [FeatureHasFastGather,
-                                                FeaturePOPCNTFalseDeps,
-                                                FeatureSGX];
-  list<SubtargetFeature> SKLInheritableFeatures =
-    !listconcat(BDWInheritableFeatures, SKLAdditionalFeatures);
+                                                  FeatureSGX];
+  list<SubtargetFeature> SKLTuning = [FeatureHasFastGather,
+                                      FeatureMacroFusion,
+                                      FeatureSlow3OpsLEA,
+                                      FeatureSlowDivide64,
+                                      FeatureFastScalarFSQRT,
+                                      FeatureFastVectorFSQRT,
+                                      FeatureFastSHLDRotate,
+                                      FeatureFast15ByteNOP,
+                                      FeatureFastVariableShuffle,
+                                      FeaturePOPCNTFalseDeps,
+                                      FeatureInsertVZEROUPPER];
   list<SubtargetFeature> SKLFeatures =
-    !listconcat(SKLInheritableFeatures, SKLSpecificFeatures);
+    !listconcat(BDWFeatures, SKLAdditionalFeatures);
 
   // Skylake-AVX512
-  list<SubtargetFeature> SKXAdditionalFeatures = [FeatureAVX512,
-                                                  FeaturePrefer256Bit,
+  list<SubtargetFeature> SKXAdditionalFeatures = [FeatureAES,
+                                                  FeatureXSAVEC,
+                                                  FeatureXSAVES,
+                                                  FeatureCLFLUSHOPT,
+                                                  FeatureAVX512,
                                                   FeatureCDI,
                                                   FeatureDQI,
                                                   FeatureBWI,
                                                   FeatureVLX,
                                                   FeaturePKU,
                                                   FeatureCLWB];
-  list<SubtargetFeature> SKXSpecificFeatures = [FeatureHasFastGather,
-                                                FeaturePOPCNTFalseDeps];
-  list<SubtargetFeature> SKXInheritableFeatures =
-    !listconcat(SKLInheritableFeatures, SKXAdditionalFeatures);
+  list<SubtargetFeature> SKXTuning = [FeatureHasFastGather,
+                                      FeatureMacroFusion,
+                                      FeatureSlow3OpsLEA,
+                                      FeatureSlowDivide64,
+                                      FeatureFastScalarFSQRT,
+                                      FeatureFastVectorFSQRT,
+                                      FeatureFastSHLDRotate,
+                                      FeatureFast15ByteNOP,
+                                      FeatureFastVariableShuffle,
+                                      FeaturePrefer256Bit,
+                                      FeaturePOPCNTFalseDeps,
+                                      FeatureInsertVZEROUPPER];
   list<SubtargetFeature> SKXFeatures =
-    !listconcat(SKXInheritableFeatures, SKXSpecificFeatures);
+    !listconcat(BDWFeatures, SKXAdditionalFeatures);
 
   // Cascadelake
   list<SubtargetFeature> CLXAdditionalFeatures = [FeatureVNNI];
-  list<SubtargetFeature> CLXSpecificFeatures = [FeatureHasFastGather,
-                                                FeaturePOPCNTFalseDeps];
-  list<SubtargetFeature> CLXInheritableFeatures =
-    !listconcat(SKXInheritableFeatures, CLXAdditionalFeatures);
+  list<SubtargetFeature> CLXTuning = SKXTuning;
   list<SubtargetFeature> CLXFeatures =
-    !listconcat(CLXInheritableFeatures, CLXSpecificFeatures);
+    !listconcat(SKXFeatures, CLXAdditionalFeatures);
 
   // Cooperlake
   list<SubtargetFeature> CPXAdditionalFeatures = [FeatureBF16];
-  list<SubtargetFeature> CPXSpecificFeatures = [FeatureHasFastGather,
-                                                FeaturePOPCNTFalseDeps];
-  list<SubtargetFeature> CPXInheritableFeatures =
-    !listconcat(CLXInheritableFeatures, CPXAdditionalFeatures);
+  list<SubtargetFeature> CPXTuning = SKXTuning;
   list<SubtargetFeature> CPXFeatures =
-    !listconcat(CPXInheritableFeatures, CPXSpecificFeatures);
+    !listconcat(CLXFeatures, CPXAdditionalFeatures);
 
   // Cannonlake
   list<SubtargetFeature> CNLAdditionalFeatures = [FeatureAVX512,
-                                                  FeaturePrefer256Bit,
                                                   FeatureCDI,
                                                   FeatureDQI,
                                                   FeatureBWI,
@@ -685,13 +689,20 @@ def ProcessorFeatures {
                                                   FeaturePKU,
                                                   FeatureVBMI,
                                                   FeatureIFMA,
-                                                  FeatureSHA,
-                                                  FeatureSGX];
-  list<SubtargetFeature> CNLSpecificFeatures = [FeatureHasFastGather];
-  list<SubtargetFeature> CNLInheritableFeatures =
-    !listconcat(SKLInheritableFeatures, CNLAdditionalFeatures);
+                                                  FeatureSHA];
+  list<SubtargetFeature> CNLTuning = [FeatureHasFastGather,
+                                      FeatureMacroFusion,
+                                      FeatureSlow3OpsLEA,
+                                      FeatureSlowDivide64,
+                                      FeatureFastScalarFSQRT,
+                                      FeatureFastVectorFSQRT,
+                                      FeatureFastSHLDRotate,
+                                      FeatureFast15ByteNOP,
+                                      FeatureFastVariableShuffle,
+                                      FeaturePrefer256Bit,
+                                      FeatureInsertVZEROUPPER];
   list<SubtargetFeature> CNLFeatures =
-    !listconcat(CNLInheritableFeatures, CNLSpecificFeatures);
+    !listconcat(SKLFeatures, CNLAdditionalFeatures);
 
   // Icelake
   list<SubtargetFeature> ICLAdditionalFeatures = [FeatureBITALG,
@@ -703,71 +714,65 @@ def ProcessorFeatures {
                                                   FeatureGFNI,
                                                   FeatureCLWB,
                                                   FeatureRDPID];
-  list<SubtargetFeature> ICLSpecificFeatures = [FeatureHasFastGather];
-  list<SubtargetFeature> ICLInheritableFeatures =
-    !listconcat(CNLInheritableFeatures, ICLAdditionalFeatures);
+  list<SubtargetFeature> ICLTuning = CNLTuning;
   list<SubtargetFeature> ICLFeatures =
-    !listconcat(ICLInheritableFeatures, ICLSpecificFeatures);
+    !listconcat(CNLFeatures, ICLAdditionalFeatures);
 
   // Icelake Server
-  list<SubtargetFeature> ICXSpecificFeatures = [FeaturePCONFIG,
-                                                FeatureWBNOINVD,
-                                                FeatureHasFastGather];
+  list<SubtargetFeature> ICXAdditionalFeatures = [FeaturePCONFIG,
+                                                  FeatureWBNOINVD];
+  list<SubtargetFeature> ICXTuning = CNLTuning;
   list<SubtargetFeature> ICXFeatures =
-    !listconcat(ICLInheritableFeatures, ICXSpecificFeatures);
+    !listconcat(ICLFeatures, ICXAdditionalFeatures);
 
   //Tigerlake
   list<SubtargetFeature> TGLAdditionalFeatures = [FeatureVP2INTERSECT,
                                                   FeatureMOVDIRI,
                                                   FeatureMOVDIR64B,
                                                   FeatureSHSTK];
-  list<SubtargetFeature> TGLSpecificFeatures = [FeatureHasFastGather];
-  list<SubtargetFeature> TGLInheritableFeatures =
-    !listconcat(TGLAdditionalFeatures ,TGLSpecificFeatures);
+  list<SubtargetFeature> TGLTuning = CNLTuning;
   list<SubtargetFeature> TGLFeatures =
-    !listconcat(ICLFeatures, TGLInheritableFeatures );
+    !listconcat(ICLFeatures, TGLAdditionalFeatures );
 
   // Atom
-  list<SubtargetFeature> AtomInheritableFeatures = [FeatureX87,
-                                                    FeatureCMPXCHG8B,
-                                                    FeatureCMOV,
-                                                    FeatureMMX,
-                                                    FeatureSSSE3,
-                                                    FeatureFXSR,
-                                                    FeatureNOPL,
-                                                    Feature64Bit,
-                                                    FeatureCMPXCHG16B,
-                                                    FeatureMOVBE,
-                                                    FeatureSlowTwoMemOps,
-                                                    FeatureLAHFSAHF,
-                                                    FeatureInsertVZEROUPPER];
-  list<SubtargetFeature> AtomSpecificFeatures = [ProcIntelAtom,
-                                                 FeatureSlowUAMem16,
-                                                 FeatureLEAForSP,
-                                                 FeatureSlowDivide32,
-                                                 FeatureSlowDivide64,
-                                                 FeatureLEAUsesAG,
-                                                 FeaturePadShortFunctions];
-  list<SubtargetFeature> AtomFeatures =
-    !listconcat(AtomInheritableFeatures, AtomSpecificFeatures);
+  list<SubtargetFeature> AtomFeatures = [FeatureX87,
+                                         FeatureCMPXCHG8B,
+                                         FeatureCMOV,
+                                         FeatureMMX,
+                                         FeatureSSSE3,
+                                         FeatureFXSR,
+                                         FeatureNOPL,
+                                         Feature64Bit,
+                                         FeatureCMPXCHG16B,
+                                         FeatureMOVBE,
+                                         FeatureLAHFSAHF];
+  list<SubtargetFeature> AtomTuning = [ProcIntelAtom,
+                                       FeatureSlowUAMem16,
+                                       FeatureLEAForSP,
+                                       FeatureSlowDivide32,
+                                       FeatureSlowDivide64,
+                                       FeatureSlowTwoMemOps,
+                                       FeatureLEAUsesAG,
+                                       FeaturePadShortFunctions,
+                                       FeatureInsertVZEROUPPER];
 
   // Silvermont
   list<SubtargetFeature> SLMAdditionalFeatures = [FeatureSSE42,
                                                   FeaturePOPCNT,
                                                   FeaturePCLMUL,
                                                   FeaturePRFCHW,
-                                                  FeatureSlowLEA,
-                                                  FeatureSlowIncDec,
                                                   FeatureRDRAND];
-  list<SubtargetFeature> SLMSpecificFeatures = [ProcIntelSLM,
-                                                FeatureSlowDivide64,
-                                                FeatureSlowPMULLD,
-                                                FeatureFast7ByteNOP,
-                                                FeaturePOPCNTFalseDeps];
-  list<SubtargetFeature> SLMInheritableFeatures =
-    !listconcat(AtomInheritableFeatures, SLMAdditionalFeatures);
+  list<SubtargetFeature> SLMTuning = [ProcIntelSLM,
+                                      FeatureSlowTwoMemOps,
+                                      FeatureSlowLEA,
+                                      FeatureSlowIncDec,
+                                      FeatureSlowDivide64,
+                                      FeatureSlowPMULLD,
+                                      FeatureFast7ByteNOP,
+                                      FeaturePOPCNTFalseDeps,
+                                      FeatureInsertVZEROUPPER];
   list<SubtargetFeature> SLMFeatures =
-    !listconcat(SLMInheritableFeatures, SLMSpecificFeatures);
+    !listconcat(AtomFeatures, SLMAdditionalFeatures);
 
   // Goldmont
   list<SubtargetFeature> GLMAdditionalFeatures = [FeatureAES,
@@ -779,31 +784,33 @@ def ProcessorFeatures {
                                                   FeatureXSAVES,
                                                   FeatureCLFLUSHOPT,
                                                   FeatureFSGSBase];
-  list<SubtargetFeature> GLMSpecificFeatures = [FeatureUseGLMDivSqrtCosts,
-                                                FeaturePOPCNTFalseDeps];
-  list<SubtargetFeature> GLMInheritableFeatures =
-    !listconcat(SLMInheritableFeatures, GLMAdditionalFeatures);
+  list<SubtargetFeature> GLMTuning = [FeatureUseGLMDivSqrtCosts,
+                                      FeatureSlowTwoMemOps,
+                                      FeatureSlowLEA,
+                                      FeatureSlowIncDec,
+                                      FeaturePOPCNTFalseDeps,
+                                      FeatureInsertVZEROUPPER];
   list<SubtargetFeature> GLMFeatures =
-    !listconcat(GLMInheritableFeatures, GLMSpecificFeatures);
+    !listconcat(SLMFeatures, GLMAdditionalFeatures);
 
   // Goldmont Plus
   list<SubtargetFeature> GLPAdditionalFeatures = [FeaturePTWRITE,
                                                   FeatureRDPID,
                                                   FeatureSGX];
-  list<SubtargetFeature> GLPSpecificFeatures = [FeatureUseGLMDivSqrtCosts];
-  list<SubtargetFeature> GLPInheritableFeatures =
-    !listconcat(GLMInheritableFeatures, GLPAdditionalFeatures);
+  list<SubtargetFeature> GLPTuning = [FeatureUseGLMDivSqrtCosts,
+                                      FeatureSlowTwoMemOps,
+                                      FeatureSlowLEA,
+                                      FeatureSlowIncDec,
+                                      FeatureInsertVZEROUPPER];
   list<SubtargetFeature> GLPFeatures =
-    !listconcat(GLPInheritableFeatures, GLPSpecificFeatures);
+    !listconcat(GLMFeatures, GLPAdditionalFeatures);
 
   // Tremont
   list<SubtargetFeature> TRMAdditionalFeatures = [FeatureCLWB,
                                                   FeatureGFNI];
-  list<SubtargetFeature> TRMSpecificFeatures = [FeatureUseGLMDivSqrtCosts];
-  list<SubtargetFeature> TRMInheritableFeatures =
-    !listconcat(GLPInheritableFeatures, TRMAdditionalFeatures);
+  list<SubtargetFeature> TRMTuning = GLPTuning;
   list<SubtargetFeature> TRMFeatures =
-    !listconcat(TRMInheritableFeatures, TRMSpecificFeatures);
+    !listconcat(GLPFeatures, TRMAdditionalFeatures);
 
   // Knights Landing
   list<SubtargetFeature> KNLFeatures = [FeatureX87,
@@ -815,13 +822,10 @@ def ProcessorFeatures {
                                         Feature64Bit,
                                         FeatureCMPXCHG16B,
                                         FeaturePOPCNT,
-                                        FeatureSlowDivide64,
                                         FeaturePCLMUL,
                                         FeatureXSAVE,
                                         FeatureXSAVEOPT,
                                         FeatureLAHFSAHF,
-                                        FeatureSlow3OpsLEA,
-                                        FeatureSlowIncDec,
                                         FeatureAES,
                                         FeatureRDRAND,
                                         FeatureF16C,
@@ -838,56 +842,56 @@ def ProcessorFeatures {
                                         FeatureBMI,
                                         FeatureBMI2,
                                         FeatureFMA,
-                                        FeaturePRFCHW,
-                                        FeaturePreferMaskRegisters,
-                                        FeatureSlowTwoMemOps,
-                                        FeatureHasFastGather,
-                                        FeatureSlowPMADDWD];
+                                        FeaturePRFCHW];
+  list<SubtargetFeature> KNLTuning = [FeatureSlowDivide64,
+                                      FeatureSlow3OpsLEA,
+                                      FeatureSlowIncDec,
+                                      FeatureSlowTwoMemOps,
+                                      FeaturePreferMaskRegisters,
+                                      FeatureHasFastGather,
+                                      FeatureSlowPMADDWD];
   // TODO Add AVX5124FMAPS/AVX5124VNNIW features
   list<SubtargetFeature> KNMFeatures =
     !listconcat(KNLFeatures, [FeatureVPOPCNTDQ]);
 
   // Barcelona
-  list<SubtargetFeature> BarcelonaInheritableFeatures = [FeatureX87,
-                                                         FeatureCMPXCHG8B,
-                                                         FeatureSSE4A,
-                                                         Feature3DNowA,
-                                                         FeatureFXSR,
-                                                         FeatureNOPL,
-                                                         FeatureCMPXCHG16B,
-                                                         FeaturePRFCHW,
-                                                         FeatureLZCNT,
-                                                         FeaturePOPCNT,
-                                                         FeatureSlowSHLD,
-                                                         FeatureLAHFSAHF,
-                                                         FeatureCMOV,
-                                                         Feature64Bit,
-                                                         FeatureFastScalarShiftMasks,
-                                                         FeatureInsertVZEROUPPER];
-  list<SubtargetFeature> BarcelonaFeatures = BarcelonaInheritableFeatures;
+  list<SubtargetFeature> BarcelonaFeatures = [FeatureX87,
+                                              FeatureCMPXCHG8B,
+                                              FeatureSSE4A,
+                                              Feature3DNowA,
+                                              FeatureFXSR,
+                                              FeatureNOPL,
+                                              FeatureCMPXCHG16B,
+                                              FeaturePRFCHW,
+                                              FeatureLZCNT,
+                                              FeaturePOPCNT,
+                                              FeatureLAHFSAHF,
+                                              FeatureCMOV,
+                                              Feature64Bit];
+  list<SubtargetFeature> BarcelonaTuning = [FeatureFastScalarShiftMasks,
+                                            FeatureSlowSHLD,
+                                            FeatureInsertVZEROUPPER];
 
   // Bobcat
-  list<SubtargetFeature> BtVer1InheritableFeatures = [FeatureX87,
-                                                      FeatureCMPXCHG8B,
-                                                      FeatureCMOV,
-                                                      FeatureMMX,
-                                                      FeatureSSSE3,
-                                                      FeatureSSE4A,
-                                                      FeatureFXSR,
-                                                      FeatureNOPL,
-                                                      Feature64Bit,
-                                                      FeatureCMPXCHG16B,
-                                                      FeaturePRFCHW,
-                                                      FeatureLZCNT,
-                                                      FeaturePOPCNT,
-                                                      FeatureSlowSHLD,
-                                                      FeatureLAHFSAHF,
-                                                      FeatureFast15ByteNOP,
-                                                      FeatureFastScalarShiftMasks,
-                                                      FeatureFastVectorShiftMasks];
-  list<SubtargetFeature> BtVer1SpecificFeatures = [FeatureInsertVZEROUPPER];
-  list<SubtargetFeature> BtVer1Features =
-    !listconcat(BtVer1InheritableFeatures, BtVer1SpecificFeatures);
+  list<SubtargetFeature> BtVer1Features = [FeatureX87,
+                                           FeatureCMPXCHG8B,
+                                           FeatureCMOV,
+                                           FeatureMMX,
+                                           FeatureSSSE3,
+                                           FeatureSSE4A,
+                                           FeatureFXSR,
+                                           FeatureNOPL,
+                                           Feature64Bit,
+                                           FeatureCMPXCHG16B,
+                                           FeaturePRFCHW,
+                                           FeatureLZCNT,
+                                           FeaturePOPCNT,
+                                           FeatureLAHFSAHF];
+  list<SubtargetFeature> BtVer1Tuning = [FeatureFast15ByteNOP,
+                                         FeatureFastScalarShiftMasks,
+                                         FeatureFastVectorShiftMasks,
+                                         FeatureSlowSHLD,
+                                         FeatureInsertVZEROUPPER];
 
   // Jaguar
   list<SubtargetFeature> BtVer2AdditionalFeatures = [FeatureAVX,
@@ -898,38 +902,39 @@ def ProcessorFeatures {
                                                      FeatureMOVBE,
                                                      FeatureXSAVE,
                                                      FeatureXSAVEOPT];
-  list<SubtargetFeature> BtVer2SpecificFeatures = [FeatureFastLZCNT,
-                                                   FeatureFastBEXTR,
-                                                   FeatureFastHorizontalOps];
-  list<SubtargetFeature> BtVer2InheritableFeatures =
-    !listconcat(BtVer1InheritableFeatures, BtVer2AdditionalFeatures);
+  list<SubtargetFeature> BtVer2Tuning = [FeatureFastLZCNT,
+                                         FeatureFastBEXTR,
+                                         FeatureFastHorizontalOps,
+                                         FeatureFast15ByteNOP,
+                                         FeatureFastScalarShiftMasks,
+                                         FeatureFastVectorShiftMasks,
+                                         FeatureSlowSHLD];
   list<SubtargetFeature> BtVer2Features =
-    !listconcat(BtVer2InheritableFeatures, BtVer2SpecificFeatures);
+    !listconcat(BtVer1Features, BtVer2AdditionalFeatures);
 
   // Bulldozer
-  list<SubtargetFeature> BdVer1InheritableFeatures = [FeatureX87,
-                                                      FeatureCMPXCHG8B,
-                                                      FeatureCMOV,
-                                                      FeatureXOP,
-                                                      Feature64Bit,
-                                                      FeatureCMPXCHG16B,
-                                                      FeatureAES,
-                                                      FeaturePRFCHW,
-                                                      FeaturePCLMUL,
-                                                      FeatureMMX,
-                                                      FeatureFXSR,
-                                                      FeatureNOPL,
-                                                      FeatureLZCNT,
-                                                      FeaturePOPCNT,
-                                                      FeatureXSAVE,
-                                                      FeatureLWP,
-                                                      FeatureSlowSHLD,
-                                                      FeatureLAHFSAHF,
-                                                      FeatureFast11ByteNOP,
-                                                      FeatureFastScalarShiftMasks,
-                                                      FeatureBranchFusion,
-                                                      FeatureInsertVZEROUPPER];
-  list<SubtargetFeature> BdVer1Features = BdVer1InheritableFeatures;
+  list<SubtargetFeature> BdVer1Features = [FeatureX87,
+                                           FeatureCMPXCHG8B,
+                                           FeatureCMOV,
+                                           FeatureXOP,
+                                           Feature64Bit,
+                                           FeatureCMPXCHG16B,
+                                           FeatureAES,
+                                           FeaturePRFCHW,
+                                           FeaturePCLMUL,
+                                           FeatureMMX,
+                                           FeatureFXSR,
+                                           FeatureNOPL,
+                                           FeatureLZCNT,
+                                           FeaturePOPCNT,
+                                           FeatureXSAVE,
+                                           FeatureLWP,
+                                           FeatureLAHFSAHF];
+  list<SubtargetFeature> BdVer1Tuning = [FeatureSlowSHLD,
+                                         FeatureFast11ByteNOP,
+                                         FeatureFastScalarShiftMasks,
+                                         FeatureBranchFusion,
+                                         FeatureInsertVZEROUPPER];
 
   // PileDriver
   list<SubtargetFeature> BdVer2AdditionalFeatures = [FeatureF16C,
@@ -937,16 +942,16 @@ def ProcessorFeatures {
                                                      FeatureTBM,
                                                      FeatureFMA,
                                                      FeatureFastBEXTR];
-  list<SubtargetFeature> BdVer2InheritableFeatures =
-    !listconcat(BdVer1InheritableFeatures, BdVer2AdditionalFeatures);
-  list<SubtargetFeature> BdVer2Features = BdVer2InheritableFeatures;
+  list<SubtargetFeature> BdVer2Tuning = BdVer1Tuning;
+  list<SubtargetFeature> BdVer2Features =
+    !listconcat(BdVer1Features, BdVer2AdditionalFeatures);
 
   // Steamroller
   list<SubtargetFeature> BdVer3AdditionalFeatures = [FeatureXSAVEOPT,
                                                      FeatureFSGSBase];
-  list<SubtargetFeature> BdVer3InheritableFeatures =
-    !listconcat(BdVer2InheritableFeatures, BdVer3AdditionalFeatures);
-  list<SubtargetFeature> BdVer3Features = BdVer3InheritableFeatures;
+  list<SubtargetFeature> BdVer3Tuning = BdVer2Tuning;
+  list<SubtargetFeature> BdVer3Features =
+    !listconcat(BdVer2Features, BdVer3AdditionalFeatures);
 
   // Excavator
   list<SubtargetFeature> BdVer4AdditionalFeatures = [FeatureAVX2,
@@ -954,9 +959,9 @@ def ProcessorFeatures {
                                                      FeatureMOVBE,
                                                      FeatureRDRAND,
                                                      FeatureMWAITX];
-  list<SubtargetFeature> BdVer4InheritableFeatures =
-    !listconcat(BdVer3InheritableFeatures, BdVer4AdditionalFeatures);
-  list<SubtargetFeature> BdVer4Features = BdVer4InheritableFeatures;
+  list<SubtargetFeature> BdVer4Tuning = BdVer3Tuning;
+  list<SubtargetFeature> BdVer4Features =
+    !listconcat(BdVer3Features, BdVer4AdditionalFeatures);
 
 
   // AMD Zen Processors common ISAs
@@ -975,13 +980,8 @@ def ProcessorFeatures {
                                        FeatureFSGSBase,
                                        FeatureFXSR,
                                        FeatureNOPL,
-                                       FeatureFastLZCNT,
                                        FeatureLAHFSAHF,
                                        FeatureLZCNT,
-                                       FeatureFastBEXTR,
-                                       FeatureFast15ByteNOP,
-                                       FeatureBranchFusion,
-                                       FeatureFastScalarShiftMasks,
                                        FeatureMMX,
                                        FeatureMOVBE,
                                        FeatureMWAITX,
@@ -992,16 +992,22 @@ def ProcessorFeatures {
                                        FeatureRDSEED,
                                        FeatureSHA,
                                        FeatureSSE4A,
-                                       FeatureSlowSHLD,
-                                       FeatureInsertVZEROUPPER,
                                        FeatureX87,
                                        FeatureXSAVE,
                                        FeatureXSAVEC,
                                        FeatureXSAVEOPT,
                                        FeatureXSAVES];
+  list<SubtargetFeature> ZNTuning = [FeatureFastLZCNT,
+                                     FeatureFastBEXTR,
+                                     FeatureFast15ByteNOP,
+                                     FeatureBranchFusion,
+                                     FeatureFastScalarShiftMasks,
+                                     FeatureSlowSHLD,
+                                     FeatureInsertVZEROUPPER];
   list<SubtargetFeature> ZN2AdditionalFeatures = [FeatureCLWB,
                                                   FeatureRDPID,
                                                   FeatureWBNOINVD];
+  list<SubtargetFeature> ZN2Tuning = ZNTuning;
   list<SubtargetFeature> ZN2Features =
     !listconcat(ZNFeatures, ZN2AdditionalFeatures);
 }
@@ -1010,42 +1016,47 @@ def ProcessorFeatures {
 // X86 processors supported.
 //===----------------------------------------------------------------------===//
 
-class Proc<string Name, list<SubtargetFeature> Features>
- : ProcessorModel<Name, GenericModel, Features>;
+class Proc<string Name, list<SubtargetFeature> Features,
+           list<SubtargetFeature> TuneFeatures>
+ : ProcessorModel<Name, GenericModel, !listconcat(Features, TuneFeatures)>;
+
+class ProcModel<string Name, SchedMachineModel Model,
+                list<SubtargetFeature> Features,
+                list<SubtargetFeature> TuneFeatures>
+ : ProcessorModel<Name, Model, !listconcat(Features, TuneFeatures)>;
 
 // NOTE: CMPXCHG8B is here for legacy compatibility so that it is only disabled
 // if i386/i486 is specifically requested.
 // NOTE: 64Bit is here as "generic" is the default llc CPU. The X86Subtarget
 // constructor checks that any CPU used in 64-bit mode has Feature64Bit enabled.
 // It has no effect on code generation.
-def : Proc<"generic",         [FeatureX87, FeatureSlowUAMem16,
-                               FeatureCMPXCHG8B, Feature64Bit,
-                               FeatureInsertVZEROUPPER]>;
-def : Proc<"i386",            [FeatureX87, FeatureSlowUAMem16,
-                               FeatureInsertVZEROUPPER]>;
-def : Proc<"i486",            [FeatureX87, FeatureSlowUAMem16,
-                               FeatureInsertVZEROUPPER]>;
-def : Proc<"i586",            [FeatureX87, FeatureSlowUAMem16,
-                               FeatureCMPXCHG8B, FeatureInsertVZEROUPPER]>;
-def : Proc<"pentium",         [FeatureX87, FeatureSlowUAMem16,
-                               FeatureCMPXCHG8B, FeatureInsertVZEROUPPER]>;
-def : Proc<"pentium-mmx",     [FeatureX87, FeatureSlowUAMem16,
-                               FeatureCMPXCHG8B, FeatureMMX,
-                               FeatureInsertVZEROUPPER]>;
-
-def : Proc<"i686", [FeatureX87, FeatureSlowUAMem16, FeatureCMPXCHG8B,
-                    FeatureCMOV, FeatureInsertVZEROUPPER]>;
-def : Proc<"pentiumpro", [FeatureX87, FeatureSlowUAMem16, FeatureCMPXCHG8B,
-                          FeatureCMOV, FeatureNOPL, FeatureInsertVZEROUPPER]>;
-
-def : Proc<"pentium2",        [FeatureX87, FeatureSlowUAMem16, FeatureCMPXCHG8B,
-                               FeatureMMX, FeatureCMOV, FeatureFXSR,
-                               FeatureNOPL, FeatureInsertVZEROUPPER]>;
+def : Proc<"generic",         [FeatureX87, FeatureCMPXCHG8B, Feature64Bit],
+                              [FeatureSlowUAMem16, FeatureInsertVZEROUPPER]>;
+def : Proc<"i386",            [FeatureX87],
+                              [FeatureSlowUAMem16, FeatureInsertVZEROUPPER]>;
+def : Proc<"i486",            [FeatureX87],
+                              [FeatureSlowUAMem16, FeatureInsertVZEROUPPER]>;
+def : Proc<"i586",            [FeatureX87, FeatureCMPXCHG8B],
+                              [FeatureSlowUAMem16, FeatureInsertVZEROUPPER]>;
+def : Proc<"pentium",         [FeatureX87, FeatureCMPXCHG8B],
+                              [FeatureSlowUAMem16, FeatureInsertVZEROUPPER]>;
+def : Proc<"pentium-mmx",     [FeatureX87, FeatureCMPXCHG8B, FeatureMMX],
+                              [FeatureSlowUAMem16, FeatureInsertVZEROUPPER]>;
+
+def : Proc<"i686", [FeatureX87, FeatureCMPXCHG8B, FeatureCMOV],
+                   [FeatureSlowUAMem16, FeatureInsertVZEROUPPER]>;
+def : Proc<"pentiumpro", [FeatureX87, FeatureCMPXCHG8B, FeatureCMOV,
+                          FeatureNOPL],
+                         [FeatureSlowUAMem16, FeatureInsertVZEROUPPER]>;
+
+def : Proc<"pentium2", [FeatureX87, FeatureCMPXCHG8B, FeatureMMX, FeatureCMOV,
+                        FeatureFXSR, FeatureNOPL],
+                       [FeatureSlowUAMem16, FeatureInsertVZEROUPPER]>;
 
 foreach P = ["pentium3", "pentium3m"] in {
-  def : Proc<P, [FeatureX87, FeatureSlowUAMem16, FeatureCMPXCHG8B,FeatureMMX,
-                 FeatureSSE1, FeatureFXSR, FeatureNOPL, FeatureCMOV,
-                 FeatureInsertVZEROUPPER]>;
+  def : Proc<P, [FeatureX87, FeatureCMPXCHG8B, FeatureMMX,
+                 FeatureSSE1, FeatureFXSR, FeatureNOPL, FeatureCMOV],
+                [FeatureSlowUAMem16, FeatureInsertVZEROUPPER]>;
 }
 
 // Enable the PostRAScheduler for SSE2 and SSE3 class cpus.
@@ -1058,46 +1069,45 @@ foreach P = ["pentium3", "pentium3m"] in {
 // measure to avoid performance surprises, in case clang's default cpu
 // changes slightly.
 
-def : ProcessorModel<"pentium-m", GenericPostRAModel,
-                     [FeatureX87, FeatureSlowUAMem16, FeatureCMPXCHG8B,
-                      FeatureMMX, FeatureSSE2, FeatureFXSR, FeatureNOPL,
-                      FeatureCMOV, FeatureInsertVZEROUPPER]>;
+def : ProcModel<"pentium-m", GenericPostRAModel,
+                [FeatureX87, FeatureCMPXCHG8B, FeatureMMX, FeatureSSE2,
+                FeatureFXSR, FeatureNOPL, FeatureCMOV],
+                [FeatureSlowUAMem16, FeatureInsertVZEROUPPER]>;
 
 foreach P = ["pentium4", "pentium4m"] in {
-//  def : ProcessorModel<P, GenericPostRAModel,
-//                       [FeatureX87, FeatureSlowUAMem16, FeatureCMPXCHG8B,
-//                        FeatureMMX, FeatureSSE2, FeatureFXSR, FeatureNOPL,
-//                        FeatureCMOV, FeatureInsertVZEROUPPER]>;
+//  def : ProcModel<P, GenericPostRAModel,
+//                  [FeatureX87, FeatureCMPXCHG8B, FeatureMMX, FeatureSSE2,
+//                   FeatureFXSR, FeatureNOPL, FeatureCMOV]>;
+//                  [FeatureSlowUAMem16, FeatureInsertVZEROUPPER]>;
 
   // Since 'pentium4' is the default 32-bit CPU on Linux and Windows,
   // give it more modern tunings.
   // FIXME: This wouldn't be needed if we supported mtune.
-  def : ProcessorModel<P, SandyBridgeModel,
-                       [FeatureX87, FeatureCMPXCHG8B,
-                        FeatureMMX, FeatureSSE2, FeatureFXSR, FeatureNOPL,
-                        FeatureCMOV, FeatureInsertVZEROUPPER,
-                        FeatureSlow3OpsLEA, FeatureSlowDivide64,
-                        FeatureSlowIncDec, FeatureMacroFusion]>;
+  def : ProcModel<P, SandyBridgeModel,
+                  [FeatureX87, FeatureCMPXCHG8B, FeatureMMX, FeatureSSE2,
+                   FeatureFXSR, FeatureNOPL, FeatureCMOV],
+                  [FeatureSlow3OpsLEA, FeatureSlowDivide64,
+                   FeatureSlowIncDec, FeatureMacroFusion,
+                   FeatureInsertVZEROUPPER]>;
 }
 
 // Intel Quark.
-def : Proc<"lakemont", [FeatureSlowUAMem16, FeatureCMPXCHG8B,
-                        FeatureInsertVZEROUPPER]>;
+def : Proc<"lakemont", [FeatureCMPXCHG8B],
+                       [FeatureSlowUAMem16, FeatureInsertVZEROUPPER]>;
 
 // Intel Core Duo.
-def : ProcessorModel<"yonah", SandyBridgeModel,
-                     [FeatureX87, FeatureSlowUAMem16, FeatureCMPXCHG8B,
-                      FeatureMMX, FeatureSSE3, FeatureFXSR, FeatureNOPL,
-                      FeatureCMOV, FeatureInsertVZEROUPPER]>;
+def : ProcModel<"yonah", SandyBridgeModel,
+                [FeatureX87, FeatureCMPXCHG8B, FeatureMMX, FeatureSSE3,
+                 FeatureFXSR, FeatureNOPL, FeatureCMOV],
+                [FeatureSlowUAMem16, FeatureInsertVZEROUPPER]>;
 
 // NetBurst.
-def : ProcessorModel<"prescott", GenericPostRAModel,
-                     [FeatureX87, FeatureSlowUAMem16, FeatureCMPXCHG8B,
-                      FeatureMMX, FeatureSSE3, FeatureFXSR, FeatureNOPL,
-                      FeatureCMOV, FeatureInsertVZEROUPPER]>;
-def : ProcessorModel<"nocona", GenericPostRAModel, [
+def : ProcModel<"prescott", GenericPostRAModel,
+                [FeatureX87, FeatureCMPXCHG8B, FeatureMMX, FeatureSSE3,
+                 FeatureFXSR, FeatureNOPL, FeatureCMOV],
+                [FeatureSlowUAMem16, FeatureInsertVZEROUPPER]>;
+def : ProcModel<"nocona", GenericPostRAModel, [
   FeatureX87,
-  FeatureSlowUAMem16,
   FeatureCMPXCHG8B,
   FeatureCMOV,
   FeatureMMX,
@@ -1106,13 +1116,15 @@ def : ProcessorModel<"nocona", GenericPostRAModel, [
   FeatureNOPL,
   Feature64Bit,
   FeatureCMPXCHG16B,
+],
+[
+  FeatureSlowUAMem16,
   FeatureInsertVZEROUPPER
 ]>;
 
 // Intel Core 2 Solo/Duo.
-def : ProcessorModel<"core2", SandyBridgeModel, [
+def : ProcModel<"core2", SandyBridgeModel, [
   FeatureX87,
-  FeatureSlowUAMem16,
   FeatureCMPXCHG8B,
   FeatureCMOV,
   FeatureMMX,
@@ -1121,13 +1133,15 @@ def : ProcessorModel<"core2", SandyBridgeModel, [
   FeatureNOPL,
   Feature64Bit,
   FeatureCMPXCHG16B,
-  FeatureLAHFSAHF,
+  FeatureLAHFSAHF
+],
+[
   FeatureMacroFusion,
+  FeatureSlowUAMem16,
   FeatureInsertVZEROUPPER
 ]>;
-def : ProcessorModel<"penryn", SandyBridgeModel, [
+def : ProcModel<"penryn", SandyBridgeModel, [
   FeatureX87,
-  FeatureSlowUAMem16,
   FeatureCMPXCHG8B,
   FeatureCMOV,
   FeatureMMX,
@@ -1136,140 +1150,165 @@ def : ProcessorModel<"penryn", SandyBridgeModel, [
   FeatureNOPL,
   Feature64Bit,
   FeatureCMPXCHG16B,
-  FeatureLAHFSAHF,
+  FeatureLAHFSAHF
+],
+[
   FeatureMacroFusion,
+  FeatureSlowUAMem16,
   FeatureInsertVZEROUPPER
 ]>;
 
 // Atom CPUs.
 foreach P = ["bonnell", "atom"] in {
-  def : ProcessorModel<P, AtomModel, ProcessorFeatures.AtomFeatures>;
+  def : ProcModel<P, AtomModel, ProcessorFeatures.AtomFeatures,
+                  ProcessorFeatures.AtomTuning>;
 }
 
 foreach P = ["silvermont", "slm"] in {
-  def : ProcessorModel<P, SLMModel, ProcessorFeatures.SLMFeatures>;
+  def : ProcModel<P, SLMModel, ProcessorFeatures.SLMFeatures,
+                  ProcessorFeatures.SLMTuning>;
 }
 
-def : ProcessorModel<"goldmont", SLMModel, ProcessorFeatures.GLMFeatures>;
-def : ProcessorModel<"goldmont-plus", SLMModel, ProcessorFeatures.GLPFeatures>;
-def : ProcessorModel<"tremont", SLMModel, ProcessorFeatures.TRMFeatures>;
+def : ProcModel<"goldmont", SLMModel, ProcessorFeatures.GLMFeatures,
+                ProcessorFeatures.GLMTuning>;
+def : ProcModel<"goldmont-plus", SLMModel, ProcessorFeatures.GLPFeatures,
+                ProcessorFeatures.GLPTuning>;
+def : ProcModel<"tremont", SLMModel, ProcessorFeatures.TRMFeatures,
+                ProcessorFeatures.TRMTuning>;
 
 // "Arrandale" along with corei3 and corei5
 foreach P = ["nehalem", "corei7"] in {
-  def : ProcessorModel<P, SandyBridgeModel, ProcessorFeatures.NHMFeatures>;
+  def : ProcModel<P, SandyBridgeModel, ProcessorFeatures.NHMFeatures,
+                  ProcessorFeatures.NHMTuning>;
 }
 
 // Westmere is the corei3/i5/i7 path from nehalem to sandybridge
-def : ProcessorModel<"westmere", SandyBridgeModel,
-                     ProcessorFeatures.WSMFeatures>;
+def : ProcModel<"westmere", SandyBridgeModel, ProcessorFeatures.WSMFeatures,
+                ProcessorFeatures.WSMTuning>;
 
 foreach P = ["sandybridge", "corei7-avx"] in {
-  def : ProcessorModel<P, SandyBridgeModel, ProcessorFeatures.SNBFeatures>;
+  def : ProcModel<P, SandyBridgeModel, ProcessorFeatures.SNBFeatures,
+                  ProcessorFeatures.SNBTuning>;
 }
 
 foreach P = ["ivybridge", "core-avx-i"] in {
-  def : ProcessorModel<P, SandyBridgeModel, ProcessorFeatures.IVBFeatures>;
+  def : ProcModel<P, SandyBridgeModel, ProcessorFeatures.IVBFeatures,
+                  ProcessorFeatures.IVBTuning>;
 }
 
 foreach P = ["haswell", "core-avx2"] in {
-  def : ProcessorModel<P, HaswellModel, ProcessorFeatures.HSWFeatures>;
+  def : ProcModel<P, HaswellModel, ProcessorFeatures.HSWFeatures,
+                  ProcessorFeatures.HSWTuning>;
 }
 
-def : ProcessorModel<"broadwell", BroadwellModel,
-                     ProcessorFeatures.BDWFeatures>;
+def : ProcModel<"broadwell", BroadwellModel, ProcessorFeatures.BDWFeatures,
+                ProcessorFeatures.BDWTuning>;
 
-def : ProcessorModel<"skylake", SkylakeClientModel,
-                     ProcessorFeatures.SKLFeatures>;
+def : ProcModel<"skylake", SkylakeClientModel, ProcessorFeatures.SKLFeatures,
+                ProcessorFeatures.SKLTuning>;
 
 // FIXME: define KNL scheduler model
-def : ProcessorModel<"knl", HaswellModel, ProcessorFeatures.KNLFeatures>;
-def : ProcessorModel<"knm", HaswellModel, ProcessorFeatures.KNMFeatures>;
+def : ProcModel<"knl", HaswellModel, ProcessorFeatures.KNLFeatures,
+                ProcessorFeatures.KNLTuning>;
+def : ProcModel<"knm", HaswellModel, ProcessorFeatures.KNMFeatures,
+                ProcessorFeatures.KNLTuning>;
 
 foreach P = ["skylake-avx512", "skx"] in {
-  def : ProcessorModel<P, SkylakeServerModel, ProcessorFeatures.SKXFeatures>;
+  def : ProcModel<P, SkylakeServerModel, ProcessorFeatures.SKXFeatures,
+                  ProcessorFeatures.SKXTuning>;
 }
 
-def : ProcessorModel<"cascadelake", SkylakeServerModel,
-                     ProcessorFeatures.CLXFeatures>;
-def : ProcessorModel<"cooperlake", SkylakeServerModel,
-                     ProcessorFeatures.CPXFeatures>;
-def : ProcessorModel<"cannonlake", SkylakeServerModel,
-                     ProcessorFeatures.CNLFeatures>;
-def : ProcessorModel<"icelake-client", SkylakeServerModel,
-                     ProcessorFeatures.ICLFeatures>;
-def : ProcessorModel<"icelake-server", SkylakeServerModel,
-                     ProcessorFeatures.ICXFeatures>;
-def : ProcessorModel<"tigerlake", SkylakeServerModel,
-                     ProcessorFeatures.TGLFeatures>;
+def : ProcModel<"cascadelake", SkylakeServerModel,
+                ProcessorFeatures.CLXFeatures, ProcessorFeatures.CLXTuning>;
+def : ProcModel<"cooperlake", SkylakeServerModel,
+                ProcessorFeatures.CPXFeatures, ProcessorFeatures.CPXTuning>;
+def : ProcModel<"cannonlake", SkylakeServerModel,
+                ProcessorFeatures.CNLFeatures, ProcessorFeatures.CNLTuning>;
+def : ProcModel<"icelake-client", SkylakeServerModel,
+                ProcessorFeatures.ICLFeatures, ProcessorFeatures.ICLTuning>;
+def : ProcModel<"icelake-server", SkylakeServerModel,
+                ProcessorFeatures.ICXFeatures, ProcessorFeatures.ICXTuning>;
+def : ProcModel<"tigerlake", SkylakeServerModel,
+                ProcessorFeatures.TGLFeatures, ProcessorFeatures.TGLTuning>;
 
 // AMD CPUs.
 
-def : Proc<"k6",   [FeatureX87, FeatureSlowUAMem16, FeatureCMPXCHG8B,
-                    FeatureMMX, FeatureInsertVZEROUPPER]>;
-def : Proc<"k6-2", [FeatureX87, FeatureSlowUAMem16, FeatureCMPXCHG8B,
-                    Feature3DNow, FeatureInsertVZEROUPPER]>;
-def : Proc<"k6-3", [FeatureX87, FeatureSlowUAMem16, FeatureCMPXCHG8B,
-                    Feature3DNow, FeatureInsertVZEROUPPER]>;
+def : Proc<"k6",   [FeatureX87, FeatureCMPXCHG8B, FeatureMMX],
+                   [FeatureSlowUAMem16, FeatureInsertVZEROUPPER]>;
+def : Proc<"k6-2", [FeatureX87, FeatureCMPXCHG8B, Feature3DNow],
+                   [FeatureSlowUAMem16, FeatureInsertVZEROUPPER]>;
+def : Proc<"k6-3", [FeatureX87, FeatureCMPXCHG8B, Feature3DNow],
+                   [FeatureSlowUAMem16, FeatureInsertVZEROUPPER]>;
 
 foreach P = ["athlon", "athlon-tbird"] in {
-  def : Proc<P, [FeatureX87, FeatureSlowUAMem16, FeatureCMPXCHG8B, FeatureCMOV,
-                 Feature3DNowA, FeatureNOPL, FeatureSlowSHLD,
-                 FeatureInsertVZEROUPPER]>;
+  def : Proc<P, [FeatureX87, FeatureCMPXCHG8B, FeatureCMOV, Feature3DNowA,
+                 FeatureNOPL],
+                [FeatureSlowSHLD, FeatureSlowUAMem16, FeatureInsertVZEROUPPER]>;
 }
 
 foreach P = ["athlon-4", "athlon-xp", "athlon-mp"] in {
-  def : Proc<P, [FeatureX87, FeatureSlowUAMem16, FeatureCMPXCHG8B, FeatureCMOV,
-                 FeatureSSE1, Feature3DNowA, FeatureFXSR, FeatureNOPL,
-                 FeatureSlowSHLD, FeatureInsertVZEROUPPER]>;
+  def : Proc<P, [FeatureX87, FeatureCMPXCHG8B, FeatureCMOV,
+                 FeatureSSE1, Feature3DNowA, FeatureFXSR, FeatureNOPL],
+                [FeatureSlowSHLD, FeatureSlowUAMem16, FeatureInsertVZEROUPPER]>;
 }
 
 foreach P = ["k8", "opteron", "athlon64", "athlon-fx"] in {
-  def : Proc<P, [FeatureX87, FeatureSlowUAMem16, FeatureCMPXCHG8B,
-                 FeatureSSE2, Feature3DNowA, FeatureFXSR, FeatureNOPL,
-                 Feature64Bit, FeatureSlowSHLD, FeatureCMOV,
-                 FeatureFastScalarShiftMasks, FeatureInsertVZEROUPPER]>;
+  def : Proc<P, [FeatureX87, FeatureCMPXCHG8B, FeatureSSE2, Feature3DNowA,
+                 FeatureFXSR, FeatureNOPL, Feature64Bit, FeatureCMOV],
+                [FeatureFastScalarShiftMasks, FeatureSlowSHLD, FeatureSlowUAMem16,
+                 FeatureInsertVZEROUPPER]>;
 }
 
 foreach P = ["k8-sse3", "opteron-sse3", "athlon64-sse3"] in {
-  def : Proc<P, [FeatureX87, FeatureSlowUAMem16, FeatureCMPXCHG8B, FeatureSSE3,
-                 Feature3DNowA, FeatureFXSR, FeatureNOPL, FeatureCMPXCHG16B,
-                 FeatureSlowSHLD, FeatureCMOV, Feature64Bit,
-                 FeatureFastScalarShiftMasks, FeatureInsertVZEROUPPER]>;
+  def : Proc<P, [FeatureX87, FeatureCMPXCHG8B, FeatureSSE3, Feature3DNowA,
+                 FeatureFXSR, FeatureNOPL, FeatureCMPXCHG16B, FeatureCMOV,
+                 Feature64Bit],
+                [FeatureFastScalarShiftMasks, FeatureSlowSHLD, FeatureSlowUAMem16,
+                 FeatureInsertVZEROUPPER]>;
 }
 
 foreach P = ["amdfam10", "barcelona"] in {
-  def : Proc<P, ProcessorFeatures.BarcelonaFeatures>;
+  def : Proc<P, ProcessorFeatures.BarcelonaFeatures,
+             ProcessorFeatures.BarcelonaTuning>;
 }
 
 // Bobcat
-def : Proc<"btver1", ProcessorFeatures.BtVer1Features>;
+def : Proc<"btver1", ProcessorFeatures.BtVer1Features,
+           ProcessorFeatures.BtVer1Tuning>;
 // Jaguar
-def : ProcessorModel<"btver2", BtVer2Model, ProcessorFeatures.BtVer2Features>;
+def : ProcModel<"btver2", BtVer2Model, ProcessorFeatures.BtVer2Features,
+                ProcessorFeatures.BtVer2Tuning>;
 
 // Bulldozer
-def : ProcessorModel<"bdver1", BdVer2Model, ProcessorFeatures.BdVer1Features>;
+def : ProcModel<"bdver1", BdVer2Model, ProcessorFeatures.BdVer1Features,
+                ProcessorFeatures.BdVer1Tuning>;
 // Piledriver
-def : ProcessorModel<"bdver2", BdVer2Model, ProcessorFeatures.BdVer2Features>;
+def : ProcModel<"bdver2", BdVer2Model, ProcessorFeatures.BdVer2Features,
+                ProcessorFeatures.BdVer2Tuning>;
 // Steamroller
-def : Proc<"bdver3", ProcessorFeatures.BdVer3Features>;
+def : Proc<"bdver3", ProcessorFeatures.BdVer3Features,
+           ProcessorFeatures.BdVer3Tuning>;
 // Excavator
-def : Proc<"bdver4", ProcessorFeatures.BdVer4Features>;
-
-def : ProcessorModel<"znver1", Znver1Model, ProcessorFeatures.ZNFeatures>;
-def : ProcessorModel<"znver2", Znver2Model, ProcessorFeatures.ZN2Features>;
-
-def : Proc<"geode",           [FeatureX87, FeatureSlowUAMem16, FeatureCMPXCHG8B,
-                               Feature3DNowA, FeatureInsertVZEROUPPER]>;
-
-def : Proc<"winchip-c6",      [FeatureX87, FeatureSlowUAMem16, FeatureMMX,
-                               FeatureInsertVZEROUPPER]>;
-def : Proc<"winchip2",        [FeatureX87, FeatureSlowUAMem16, Feature3DNow,
-                               FeatureInsertVZEROUPPER]>;
-def : Proc<"c3",              [FeatureX87, FeatureSlowUAMem16, Feature3DNow,
-                               FeatureInsertVZEROUPPER]>;
-def : Proc<"c3-2",            [FeatureX87, FeatureSlowUAMem16, FeatureCMPXCHG8B,
-                               FeatureMMX, FeatureSSE1, FeatureFXSR,
-                               FeatureCMOV, FeatureInsertVZEROUPPER]>;
+def : Proc<"bdver4", ProcessorFeatures.BdVer4Features,
+           ProcessorFeatures.BdVer4Tuning>;
+
+def : ProcModel<"znver1", Znver1Model, ProcessorFeatures.ZNFeatures,
+                ProcessorFeatures.ZNTuning>;
+def : ProcModel<"znver2", Znver2Model, ProcessorFeatures.ZN2Features,
+                ProcessorFeatures.ZN2Tuning>;
+
+def : Proc<"geode",           [FeatureX87, FeatureCMPXCHG8B, Feature3DNowA],
+                              [FeatureSlowUAMem16, FeatureInsertVZEROUPPER]>;
+
+def : Proc<"winchip-c6",      [FeatureX87, FeatureMMX],
+                              [FeatureSlowUAMem16, FeatureInsertVZEROUPPER]>;
+def : Proc<"winchip2",        [FeatureX87, Feature3DNow],
+                              [FeatureSlowUAMem16, FeatureInsertVZEROUPPER]>;
+def : Proc<"c3",              [FeatureX87, Feature3DNow],
+                              [FeatureSlowUAMem16, FeatureInsertVZEROUPPER]>;
+def : Proc<"c3-2",            [FeatureX87, FeatureCMPXCHG8B, FeatureMMX,
+                               FeatureSSE1, FeatureFXSR, FeatureCMOV],
+                              [FeatureSlowUAMem16, FeatureInsertVZEROUPPER]>;
 
 // We also provide a generic 64-bit specific x86 processor model which tries to
 // be good for modern chips without enabling instruction set encodings past the
@@ -1281,7 +1320,7 @@ def : Proc<"c3-2",            [FeatureX87, FeatureSlowUAMem16, FeatureCMPXCHG8B,
 // covers a huge swath of x86 processors. If there are specific scheduling
 // knobs which need to be tuned differently for AMD chips, we might consider
 // forming a common base for them.
-def : ProcessorModel<"x86-64", SandyBridgeModel, [
+def : ProcModel<"x86-64", SandyBridgeModel, [
   FeatureX87,
   FeatureCMPXCHG8B,
   FeatureCMOV,
@@ -1290,6 +1329,8 @@ def : ProcessorModel<"x86-64", SandyBridgeModel, [
   FeatureFXSR,
   FeatureNOPL,
   Feature64Bit,
+],
+[
   FeatureSlow3OpsLEA,
   FeatureSlowDivide64,
   FeatureSlowIncDec,

From ec1445c5afda7f145a414f11c9103c87a4c1823f Mon Sep 17 00:00:00 2001
From: Scott Constable <scott.d.constable@intel.com>
Date: Thu, 30 Jul 2020 17:21:48 -0700
Subject: [PATCH 0692/1035] [X86] Fix for ballooning compile times due to Load
 Value Injection (LVI) mitigations

Fix for the issue raised in https://github.com/rust-lang/rust/issues/74632.

The current heuristic for inserting LFENCEs uses a quadratic-time algorithm. This can apparently cause substantial compilation slowdowns for building Rust projects, where functions > 5000 LoC are apparently common.

The updated heuristic in this patch implements a linear-time algorithm. On a set of benchmarks, the slowdown factor for the generated code was comparable (2.55x geo mean for the quadratic-time heuristic, vs. 2.58x for the linear-time heuristic). Both heuristics offer the same security properties, namely, mitigating LVI.

This patch also includes some formatting fixes.

Differential Revision: https://reviews.llvm.org/D84471
---
 .../X86LoadValueInjectionLoadHardening.cpp    | 180 +++++++++---------
 1 file changed, 87 insertions(+), 93 deletions(-)

diff --git a/llvm/lib/Target/X86/X86LoadValueInjectionLoadHardening.cpp b/llvm/lib/Target/X86/X86LoadValueInjectionLoadHardening.cpp
index 50f8b3477acce..18fcc48bc9cdd 100644
--- a/llvm/lib/Target/X86/X86LoadValueInjectionLoadHardening.cpp
+++ b/llvm/lib/Target/X86/X86LoadValueInjectionLoadHardening.cpp
@@ -42,6 +42,7 @@
 #include "X86TargetMachine.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/ADT/StringRef.h"
@@ -104,9 +105,9 @@ static cl::opt<bool> EmitDotVerify(
     cl::init(false), cl::Hidden);
 
 static llvm::sys::DynamicLibrary OptimizeDL;
-typedef int (*OptimizeCutT)(unsigned int *nodes, unsigned int nodes_size,
-                            unsigned int *edges, int *edge_values,
-                            int *cut_edges /* out */, unsigned int edges_size);
+typedef int (*OptimizeCutT)(unsigned int *Nodes, unsigned int NodesSize,
+                            unsigned int *Edges, int *EdgeValues,
+                            int *CutEdges /* out */, unsigned int EdgesSize);
 static OptimizeCutT OptimizeCut = nullptr;
 
 namespace {
@@ -148,9 +149,10 @@ class X86LoadValueInjectionLoadHardeningPass : public MachineFunctionPass {
 
 private:
   using GraphBuilder = ImmutableGraphBuilder<MachineGadgetGraph>;
+  using Edge = MachineGadgetGraph::Edge;
+  using Node = MachineGadgetGraph::Node;
   using EdgeSet = MachineGadgetGraph::EdgeSet;
   using NodeSet = MachineGadgetGraph::NodeSet;
-  using Gadget = std::pair<MachineInstr *, MachineInstr *>;
 
   const X86Subtarget *STI;
   const TargetInstrInfo *TII;
@@ -162,8 +164,8 @@ class X86LoadValueInjectionLoadHardeningPass : public MachineFunctionPass {
                  const MachineDominanceFrontier &MDF) const;
   int hardenLoadsWithPlugin(MachineFunction &MF,
                             std::unique_ptr<MachineGadgetGraph> Graph) const;
-  int hardenLoadsWithGreedyHeuristic(
-      MachineFunction &MF, std::unique_ptr<MachineGadgetGraph> Graph) const;
+  int hardenLoadsWithHeuristic(MachineFunction &MF,
+                               std::unique_ptr<MachineGadgetGraph> Graph) const;
   int elimMitigatedEdgesAndNodes(MachineGadgetGraph &G,
                                  EdgeSet &ElimEdges /* in, out */,
                                  NodeSet &ElimNodes /* in, out */) const;
@@ -198,7 +200,7 @@ struct DOTGraphTraits<MachineGadgetGraph *> : DefaultDOTGraphTraits {
   using ChildIteratorType = typename Traits::ChildIteratorType;
   using ChildEdgeIteratorType = typename Traits::ChildEdgeIteratorType;
 
-  DOTGraphTraits(bool isSimple = false) : DefaultDOTGraphTraits(isSimple) {}
+  DOTGraphTraits(bool IsSimple = false) : DefaultDOTGraphTraits(IsSimple) {}
 
   std::string getNodeLabel(NodeRef Node, GraphType *) {
     if (Node->getValue() == MachineGadgetGraph::ArgNodeSentinel)
@@ -243,7 +245,7 @@ void X86LoadValueInjectionLoadHardeningPass::getAnalysisUsage(
   AU.setPreservesCFG();
 }
 
-static void WriteGadgetGraph(raw_ostream &OS, MachineFunction &MF,
+static void writeGadgetGraph(raw_ostream &OS, MachineFunction &MF,
                              MachineGadgetGraph *G) {
   WriteGraph(OS, G, /*ShortNames*/ false,
              "Speculative gadgets for \"" + MF.getName() + "\" function");
@@ -279,7 +281,7 @@ bool X86LoadValueInjectionLoadHardeningPass::runOnMachineFunction(
     return false; // didn't find any gadgets
 
   if (EmitDotVerify) {
-    WriteGadgetGraph(outs(), MF, Graph.get());
+    writeGadgetGraph(outs(), MF, Graph.get());
     return false;
   }
 
@@ -292,7 +294,7 @@ bool X86LoadValueInjectionLoadHardeningPass::runOnMachineFunction(
     raw_fd_ostream FileOut(FileName, FileError);
     if (FileError)
       errs() << FileError.message();
-    WriteGadgetGraph(FileOut, MF, Graph.get());
+    writeGadgetGraph(FileOut, MF, Graph.get());
     FileOut.close();
     LLVM_DEBUG(dbgs() << "Emitting gadget graph... Done\n");
     if (EmitDotOnly)
@@ -313,7 +315,7 @@ bool X86LoadValueInjectionLoadHardeningPass::runOnMachineFunction(
     }
     FencesInserted = hardenLoadsWithPlugin(MF, std::move(Graph));
   } else { // Use the default greedy heuristic
-    FencesInserted = hardenLoadsWithGreedyHeuristic(MF, std::move(Graph));
+    FencesInserted = hardenLoadsWithHeuristic(MF, std::move(Graph));
   }
 
   if (FencesInserted > 0)
@@ -540,17 +542,17 @@ X86LoadValueInjectionLoadHardeningPass::getGadgetGraph(
 
 // Returns the number of remaining gadget edges that could not be eliminated
 int X86LoadValueInjectionLoadHardeningPass::elimMitigatedEdgesAndNodes(
-    MachineGadgetGraph &G, MachineGadgetGraph::EdgeSet &ElimEdges /* in, out */,
-    MachineGadgetGraph::NodeSet &ElimNodes /* in, out */) const {
+    MachineGadgetGraph &G, EdgeSet &ElimEdges /* in, out */,
+    NodeSet &ElimNodes /* in, out */) const {
   if (G.NumFences > 0) {
     // Eliminate fences and CFG edges that ingress and egress the fence, as
     // they are trivially mitigated.
-    for (const auto &E : G.edges()) {
-      const MachineGadgetGraph::Node *Dest = E.getDest();
+    for (const Edge &E : G.edges()) {
+      const Node *Dest = E.getDest();
       if (isFence(Dest->getValue())) {
         ElimNodes.insert(*Dest);
         ElimEdges.insert(E);
-        for (const auto &DE : Dest->edges())
+        for (const Edge &DE : Dest->edges())
           ElimEdges.insert(DE);
       }
     }
@@ -558,29 +560,28 @@ int X86LoadValueInjectionLoadHardeningPass::elimMitigatedEdgesAndNodes(
 
   // Find and eliminate gadget edges that have been mitigated.
   int MitigatedGadgets = 0, RemainingGadgets = 0;
-  MachineGadgetGraph::NodeSet ReachableNodes{G};
-  for (const auto &RootN : G.nodes()) {
+  NodeSet ReachableNodes{G};
+  for (const Node &RootN : G.nodes()) {
     if (llvm::none_of(RootN.edges(), MachineGadgetGraph::isGadgetEdge))
       continue; // skip this node if it isn't a gadget source
 
     // Find all of the nodes that are CFG-reachable from RootN using DFS
     ReachableNodes.clear();
-    std::function<void(const MachineGadgetGraph::Node *, bool)>
-        FindReachableNodes =
-            [&](const MachineGadgetGraph::Node *N, bool FirstNode) {
-              if (!FirstNode)
-                ReachableNodes.insert(*N);
-              for (const auto &E : N->edges()) {
-                const MachineGadgetGraph::Node *Dest = E.getDest();
-                if (MachineGadgetGraph::isCFGEdge(E) &&
-                    !ElimEdges.contains(E) && !ReachableNodes.contains(*Dest))
-                  FindReachableNodes(Dest, false);
-              }
-            };
+    std::function<void(const Node *, bool)> FindReachableNodes =
+        [&](const Node *N, bool FirstNode) {
+          if (!FirstNode)
+            ReachableNodes.insert(*N);
+          for (const Edge &E : N->edges()) {
+            const Node *Dest = E.getDest();
+            if (MachineGadgetGraph::isCFGEdge(E) && !ElimEdges.contains(E) &&
+                !ReachableNodes.contains(*Dest))
+              FindReachableNodes(Dest, false);
+          }
+        };
     FindReachableNodes(&RootN, true);
 
     // Any gadget whose sink is unreachable has been mitigated
-    for (const auto &E : RootN.edges()) {
+    for (const Edge &E : RootN.edges()) {
       if (MachineGadgetGraph::isGadgetEdge(E)) {
         if (ReachableNodes.contains(*E.getDest())) {
           // This gadget's sink is reachable
@@ -598,8 +599,8 @@ int X86LoadValueInjectionLoadHardeningPass::elimMitigatedEdgesAndNodes(
 std::unique_ptr<MachineGadgetGraph>
 X86LoadValueInjectionLoadHardeningPass::trimMitigatedEdges(
     std::unique_ptr<MachineGadgetGraph> Graph) const {
-  MachineGadgetGraph::NodeSet ElimNodes{*Graph};
-  MachineGadgetGraph::EdgeSet ElimEdges{*Graph};
+  NodeSet ElimNodes{*Graph};
+  EdgeSet ElimEdges{*Graph};
   int RemainingGadgets =
       elimMitigatedEdgesAndNodes(*Graph, ElimEdges, ElimNodes);
   if (ElimEdges.empty() && ElimNodes.empty()) {
@@ -630,11 +631,11 @@ int X86LoadValueInjectionLoadHardeningPass::hardenLoadsWithPlugin(
     auto Edges = std::make_unique<unsigned int[]>(Graph->edges_size());
     auto EdgeCuts = std::make_unique<int[]>(Graph->edges_size());
     auto EdgeValues = std::make_unique<int[]>(Graph->edges_size());
-    for (const auto &N : Graph->nodes()) {
+    for (const Node &N : Graph->nodes()) {
       Nodes[Graph->getNodeIndex(N)] = Graph->getEdgeIndex(*N.edges_begin());
     }
     Nodes[Graph->nodes_size()] = Graph->edges_size(); // terminator node
-    for (const auto &E : Graph->edges()) {
+    for (const Edge &E : Graph->edges()) {
       Edges[Graph->getEdgeIndex(E)] = Graph->getNodeIndex(*E.getDest());
       EdgeValues[Graph->getEdgeIndex(E)] = E.getValue();
     }
@@ -651,74 +652,67 @@ int X86LoadValueInjectionLoadHardeningPass::hardenLoadsWithPlugin(
     LLVM_DEBUG(dbgs() << "Inserting LFENCEs... Done\n");
     LLVM_DEBUG(dbgs() << "Inserted " << FencesInserted << " fences\n");
 
-    Graph = GraphBuilder::trim(*Graph, MachineGadgetGraph::NodeSet{*Graph},
-                               CutEdges);
+    Graph = GraphBuilder::trim(*Graph, NodeSet{*Graph}, CutEdges);
   } while (true);
 
   return FencesInserted;
 }
 
-int X86LoadValueInjectionLoadHardeningPass::hardenLoadsWithGreedyHeuristic(
+int X86LoadValueInjectionLoadHardeningPass::hardenLoadsWithHeuristic(
     MachineFunction &MF, std::unique_ptr<MachineGadgetGraph> Graph) const {
-  LLVM_DEBUG(dbgs() << "Eliminating mitigated paths...\n");
-  Graph = trimMitigatedEdges(std::move(Graph));
-  LLVM_DEBUG(dbgs() << "Eliminating mitigated paths... Done\n");
+  // If `MF` does not have any fences, then no gadgets would have been
+  // mitigated at this point.
+  if (Graph->NumFences > 0) {
+    LLVM_DEBUG(dbgs() << "Eliminating mitigated paths...\n");
+    Graph = trimMitigatedEdges(std::move(Graph));
+    LLVM_DEBUG(dbgs() << "Eliminating mitigated paths... Done\n");
+  }
+
   if (Graph->NumGadgets == 0)
     return 0;
 
   LLVM_DEBUG(dbgs() << "Cutting edges...\n");
-  MachineGadgetGraph::NodeSet ElimNodes{*Graph}, GadgetSinks{*Graph};
-  MachineGadgetGraph::EdgeSet ElimEdges{*Graph}, CutEdges{*Graph};
-  auto IsCFGEdge = [&ElimEdges, &CutEdges](const MachineGadgetGraph::Edge &E) {
-    return !ElimEdges.contains(E) && !CutEdges.contains(E) &&
-           MachineGadgetGraph::isCFGEdge(E);
-  };
-  auto IsGadgetEdge = [&ElimEdges,
-                       &CutEdges](const MachineGadgetGraph::Edge &E) {
-    return !ElimEdges.contains(E) && !CutEdges.contains(E) &&
-           MachineGadgetGraph::isGadgetEdge(E);
-  };
-
-  // FIXME: this is O(E^2), we could probably do better.
-  do {
-    // Find the cheapest CFG edge that will eliminate a gadget (by being
-    // egress from a SOURCE node or ingress to a SINK node), and cut it.
-    const MachineGadgetGraph::Edge *CheapestSoFar = nullptr;
-
-    // First, collect all gadget source and sink nodes.
-    MachineGadgetGraph::NodeSet GadgetSources{*Graph}, GadgetSinks{*Graph};
-    for (const auto &N : Graph->nodes()) {
-      if (ElimNodes.contains(N))
+  EdgeSet CutEdges{*Graph};
+
+  // Begin by collecting all ingress CFG edges for each node
+  DenseMap<const Node *, SmallVector<const Edge *, 2>> IngressEdgeMap;
+  for (const Edge &E : Graph->edges())
+    if (MachineGadgetGraph::isCFGEdge(E))
+      IngressEdgeMap[E.getDest()].push_back(&E);
+
+  // For each gadget edge, make cuts that guarantee the gadget will be
+  // mitigated. A computationally efficient way to achieve this is to either:
+  // (a) cut all egress CFG edges from the gadget source, or
+  // (b) cut all ingress CFG edges to the gadget sink.
+  //
+  // Moreover, the algorithm tries not to make a cut into a loop by preferring
+  // to make a (b)-type cut if the gadget source resides at a greater loop depth
+  // than the gadget sink, or an (a)-type cut otherwise.
+  for (const Node &N : Graph->nodes()) {
+    for (const Edge &E : N.edges()) {
+      if (!MachineGadgetGraph::isGadgetEdge(E))
         continue;
-      for (const auto &E : N.edges()) {
-        if (IsGadgetEdge(E)) {
-          GadgetSources.insert(N);
-          GadgetSinks.insert(*E.getDest());
-        }
-      }
-    }
 
-    // Next, look for the cheapest CFG edge which, when cut, is guaranteed to
-    // mitigate at least one gadget by either:
-    // (a) being egress from a gadget source, or
-    // (b) being ingress to a gadget sink.
-    for (const auto &N : Graph->nodes()) {
-      if (ElimNodes.contains(N))
-        continue;
-      for (const auto &E : N.edges()) {
-        if (IsCFGEdge(E)) {
-          if (GadgetSources.contains(N) || GadgetSinks.contains(*E.getDest())) {
-            if (!CheapestSoFar || E.getValue() < CheapestSoFar->getValue())
-              CheapestSoFar = &E;
-          }
-        }
-      }
+      SmallVector<const Edge *, 2> EgressEdges;
+      SmallVector<const Edge *, 2> &IngressEdges = IngressEdgeMap[E.getDest()];
+      for (const Edge &EgressEdge : N.edges())
+        if (MachineGadgetGraph::isCFGEdge(EgressEdge))
+          EgressEdges.push_back(&EgressEdge);
+
+      int EgressCutCost = 0, IngressCutCost = 0;
+      for (const Edge *EgressEdge : EgressEdges)
+        if (!CutEdges.contains(*EgressEdge))
+          EgressCutCost += EgressEdge->getValue();
+      for (const Edge *IngressEdge : IngressEdges)
+        if (!CutEdges.contains(*IngressEdge))
+          IngressCutCost += IngressEdge->getValue();
+
+      auto &EdgesToCut =
+          IngressCutCost < EgressCutCost ? IngressEdges : EgressEdges;
+      for (const Edge *E : EdgesToCut)
+        CutEdges.insert(*E);
     }
-
-    assert(CheapestSoFar && "Failed to cut an edge");
-    CutEdges.insert(*CheapestSoFar);
-    ElimEdges.insert(*CheapestSoFar);
-  } while (elimMitigatedEdgesAndNodes(*Graph, ElimEdges, ElimNodes));
+  }
   LLVM_DEBUG(dbgs() << "Cutting edges... Done\n");
   LLVM_DEBUG(dbgs() << "Cut " << CutEdges.count() << " edges\n");
 
@@ -734,8 +728,8 @@ int X86LoadValueInjectionLoadHardeningPass::insertFences(
     MachineFunction &MF, MachineGadgetGraph &G,
     EdgeSet &CutEdges /* in, out */) const {
   int FencesInserted = 0;
-  for (const auto &N : G.nodes()) {
-    for (const auto &E : N.edges()) {
+  for (const Node &N : G.nodes()) {
+    for (const Edge &E : N.edges()) {
       if (CutEdges.contains(E)) {
         MachineInstr *MI = N.getValue(), *Prev;
         MachineBasicBlock *MBB;                  // Insert an LFENCE in this MBB
@@ -751,7 +745,7 @@ int X86LoadValueInjectionLoadHardeningPass::insertFences(
           Prev = MI->getPrevNode();
           // Remove all egress CFG edges from this branch because the inserted
           // LFENCE prevents gadgets from crossing the branch.
-          for (const auto &E : N.edges()) {
+          for (const Edge &E : N.edges()) {
             if (MachineGadgetGraph::isCFGEdge(E))
               CutEdges.insert(E);
           }

From aa77232a63e3f7ca2f0077434d46e0f269499eae Mon Sep 17 00:00:00 2001
From: dfukalov <daniil.fukalov@amd.com>
Date: Thu, 30 Jul 2020 04:12:17 +0300
Subject: [PATCH 0693/1035] [NFC][AMDGPU] Improve fused fmul+fadd tests.

Reviewed By: rampitec

Differential Revision: https://reviews.llvm.org/D84903
---
 llvm/test/CodeGen/AMDGPU/fmuladd.f16.ll   | 40 +++++++++++++++++++++++
 llvm/test/CodeGen/AMDGPU/fmuladd.f32.ll   | 18 ++++++++++
 llvm/test/CodeGen/AMDGPU/fmuladd.f64.ll   | 16 ++++++++-
 llvm/test/CodeGen/AMDGPU/fmuladd.v2f16.ll | 33 +++++++++++++++++++
 4 files changed, 106 insertions(+), 1 deletion(-)

diff --git a/llvm/test/CodeGen/AMDGPU/fmuladd.f16.ll b/llvm/test/CodeGen/AMDGPU/fmuladd.f16.ll
index 58d61ccaf97bb..cd19e237be709 100644
--- a/llvm/test/CodeGen/AMDGPU/fmuladd.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmuladd.f16.ll
@@ -32,6 +32,46 @@ define amdgpu_kernel void @fmuladd_f16(half addrspace(1)* %out, half addrspace(1
   ret void
 }
 
+; GCN-LABEL: {{^}}fmul_fadd_f16:
+; VI-FLUSH: v_mac_f16_e32 {{v[0-9]+, v[0-9]+, v[0-9]+}}
+
+; VI-DENORM-CONTRACT: v_fma_f16 {{v[0-9]+, v[0-9]+, v[0-9]+}}
+
+; GFX10-FLUSH:  v_mul_f16_e32
+; GFX10-FLUSH:  v_add_f16_e32
+; GFX10-DENORM-CONTRACT: v_fmac_f16_e32 {{v[0-9]+, v[0-9]+, v[0-9]+}}
+
+define amdgpu_kernel void @fmul_fadd_f16(half addrspace(1)* %out, half addrspace(1)* %in1,
+                         half addrspace(1)* %in2, half addrspace(1)* %in3) #0 {
+  %r0 = load half, half addrspace(1)* %in1
+  %r1 = load half, half addrspace(1)* %in2
+  %r2 = load half, half addrspace(1)* %in3
+  %mul = fmul half %r0, %r1
+  %add = fadd half %mul, %r2
+  store half %add, half addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}fmul_fadd_contract_f16:
+; VI-FLUSH: v_mac_f16_e32 {{v[0-9]+, v[0-9]+, v[0-9]+}}
+
+; VI-DENORM: v_fma_f16 {{v[0-9]+, v[0-9]+, v[0-9]+}}
+
+; GFX10-FLUSH:  v_mul_f16_e32
+; GFX10-FLUSH:  v_add_f16_e32
+; GFX10-DENORM: v_fmac_f16_e32 {{v[0-9]+, v[0-9]+, v[0-9]+}}
+
+define amdgpu_kernel void @fmul_fadd_contract_f16(half addrspace(1)* %out, half addrspace(1)* %in1,
+                         half addrspace(1)* %in2, half addrspace(1)* %in3) #0 {
+  %r0 = load half, half addrspace(1)* %in1
+  %r1 = load half, half addrspace(1)* %in2
+  %r2 = load half, half addrspace(1)* %in3
+  %mul = fmul half %r0, %r1
+  %add = fadd contract half %mul, %r2
+  store half %add, half addrspace(1)* %out
+  ret void
+}
+
 ; GCN-LABEL: {{^}}fmuladd_2.0_a_b_f16
 ; GCN: {{buffer|flat|global}}_load_ushort [[R1:v[0-9]+]],
 ; GCN: {{buffer|flat|global}}_load_ushort [[R2:v[0-9]+]],
diff --git a/llvm/test/CodeGen/AMDGPU/fmuladd.f32.ll b/llvm/test/CodeGen/AMDGPU/fmuladd.f32.ll
index 7b38b79c78a15..19862fbf13fe6 100644
--- a/llvm/test/CodeGen/AMDGPU/fmuladd.f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmuladd.f32.ll
@@ -69,6 +69,24 @@ define amdgpu_kernel void @fmul_fadd_f32(float addrspace(1)* %out, float addrspa
   ret void
 }
 
+; GCN-LABEL: {{^}}fmul_fadd_contract_f32:
+; GCN-FLUSH-FMAC: v_fmac_f32_e32
+
+; GCN-DENORM-SLOWFMA-CONTRACT: v_mul_f32_e32
+; GCN-DENORM-SLOWFMA-CONTRACT: v_add_f32_e32
+
+; GCN-DENORM-FASTFMA: v_fma_f32
+define amdgpu_kernel void @fmul_fadd_contract_f32(float addrspace(1)* %out, float addrspace(1)* %in1,
+                           float addrspace(1)* %in2, float addrspace(1)* %in3) #0 {
+  %r0 = load volatile float, float addrspace(1)* %in1
+  %r1 = load volatile float, float addrspace(1)* %in2
+  %r2 = load volatile float, float addrspace(1)* %in3
+  %mul = fmul float %r0, %r1
+  %add = fadd contract float %mul, %r2
+  store float %add, float addrspace(1)* %out
+  ret void
+}
+
 ; GCN-LABEL: {{^}}fmuladd_2.0_a_b_f32
 ; GCN: {{buffer|flat|global}}_load_dword [[R1:v[0-9]+]],
 ; GCN: {{buffer|flat|global}}_load_dword [[R2:v[0-9]+]],
diff --git a/llvm/test/CodeGen/AMDGPU/fmuladd.f64.ll b/llvm/test/CodeGen/AMDGPU/fmuladd.f64.ll
index 8d91a56ee4211..cdc4f3b5f31d0 100644
--- a/llvm/test/CodeGen/AMDGPU/fmuladd.f64.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmuladd.f64.ll
@@ -1,4 +1,4 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=tahiti -fp-contract=on -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GCN-STRICTSI %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=tahiti -fp-contract=on -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GCN-STRICT,SI %s
 ; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=verde  -fp-contract=on -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GCN-STRICT,SI %s
 ; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=tahiti -fp-contract=fast -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GCN-CONTRACT,SI %s
 ; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=verde  -fp-contract=fast -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GCN-CONTRACT,SI %s
@@ -33,6 +33,20 @@ define amdgpu_kernel void @fmul_fadd_f64(double addrspace(1)* %out, double addrs
   ret void
 }
 
+; GCN-LABEL: {{^}}fmul_fadd_contract_f64:
+; GCN: v_fma_f64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\]}}
+
+define amdgpu_kernel void @fmul_fadd_contract_f64(double addrspace(1)* %out, double addrspace(1)* %in1,
+                           double addrspace(1)* %in2, double addrspace(1)* %in3) #0 {
+  %r0 = load double, double addrspace(1)* %in1
+  %r1 = load double, double addrspace(1)* %in2
+  %r2 = load double, double addrspace(1)* %in3
+  %tmp = fmul double %r0, %r1
+  %r3 = fadd contract double %tmp, %r2
+  store double %r3, double addrspace(1)* %out
+  ret void
+}
+
 ; GCN-LABEL: {{^}}fadd_a_a_b_f64:
 ; GCN: {{buffer|flat}}_load_dwordx2 [[R1:v\[[0-9]+:[0-9]+\]]],
 ; GCN: {{buffer|flat}}_load_dwordx2 [[R2:v\[[0-9]+:[0-9]+\]]],
diff --git a/llvm/test/CodeGen/AMDGPU/fmuladd.v2f16.ll b/llvm/test/CodeGen/AMDGPU/fmuladd.v2f16.ll
index 95ea266c6eb72..74c91fbea44d1 100644
--- a/llvm/test/CodeGen/AMDGPU/fmuladd.v2f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmuladd.v2f16.ll
@@ -27,6 +27,39 @@ define amdgpu_kernel void @fmuladd_v2f16(<2 x half> addrspace(1)* %out, <2 x hal
   ret void
 }
 
+; GCN-LABEL: {{^}}fmul_fadd_v2f16:
+; GFX9-DENORM-STRICT: v_pk_mul_f16 {{v[0-9]+, v[0-9]+, v[0-9]+}}
+; GFX9-DENORM-STRICT: v_pk_add_f16 {{v[0-9]+, v[0-9]+, v[0-9]+}}
+
+; GFX9-DENORM-CONTRACT: v_pk_fma_f16 {{v[0-9]+, v[0-9]+, v[0-9]+}}
+define amdgpu_kernel void @fmul_fadd_v2f16(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %in1,
+                         <2 x half> addrspace(1)* %in2, <2 x half> addrspace(1)* %in3) #0 {
+  %r0 = load <2 x half>, <2 x half> addrspace(1)* %in1
+  %r1 = load <2 x half>, <2 x half> addrspace(1)* %in2
+  %r2 = load <2 x half>, <2 x half> addrspace(1)* %in3
+  %r3 = fmul <2 x half> %r0, %r1
+  %r4 = fadd <2 x half> %r3, %r2
+  store <2 x half> %r4, <2 x half> addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}fmul_fadd_contract_v2f16:
+; GFX9-FLUSH: v_pk_mul_f16 {{v[0-9]+, v[0-9]+, v[0-9]+}}
+; GFX9-FLUSH: v_pk_add_f16 {{v[0-9]+, v[0-9]+, v[0-9]+}}
+
+; GFX9-DENORM: v_pk_fma_f16 {{v[0-9]+, v[0-9]+, v[0-9]+}}
+define amdgpu_kernel void @fmul_fadd_contract_v2f16(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %in1,
+                         <2 x half> addrspace(1)* %in2, <2 x half> addrspace(1)* %in3) #0 {
+  %r0 = load <2 x half>, <2 x half> addrspace(1)* %in1
+  %r1 = load <2 x half>, <2 x half> addrspace(1)* %in2
+  %r2 = load <2 x half>, <2 x half> addrspace(1)* %in3
+  %r3 = fmul <2 x half> %r0, %r1
+  %r4 = fadd contract <2 x half> %r3, %r2
+  store <2 x half> %r4, <2 x half> addrspace(1)* %out
+  ret void
+}
+
+
 ; GCN-LABEL: {{^}}fmuladd_2.0_a_b_v2f16:
 ; GCN: {{buffer|flat|global}}_load_dword [[R1:v[0-9]+]],
 ; GCN: {{buffer|flat|global}}_load_dword [[R2:v[0-9]+]],

From 8218eee269c382472b9809cb3cce7a98eed7a31b Mon Sep 17 00:00:00 2001
From: Shilei Tian <tianshilei1992@gmail.com>
Date: Thu, 30 Jul 2020 21:05:30 -0400
Subject: [PATCH 0694/1035] [OpenMP] Refactored the function `target`

Refactored the function `target` to make preparation for fixing the
issue of ahead-of-time device memory deallocation.

Reviewed By: ye-luo

Differential Revision: https://reviews.llvm.org/D84816
---
 openmp/libomptarget/src/omptarget.cpp | 255 ++++++++++++++++----------
 1 file changed, 157 insertions(+), 98 deletions(-)

diff --git a/openmp/libomptarget/src/omptarget.cpp b/openmp/libomptarget/src/omptarget.cpp
index 9c7ab6309503a..ece7df6f15ac7 100644
--- a/openmp/libomptarget/src/omptarget.cpp
+++ b/openmp/libomptarget/src/omptarget.cpp
@@ -714,85 +714,82 @@ static bool isLambdaMapping(int64_t Mapping) {
   return (Mapping & LambdaMapping) == LambdaMapping;
 }
 
-/// performs the same actions as data_begin in case arg_num is
-/// non-zero and initiates run of the offloaded region on the target platform;
-/// if arg_num is non-zero after the region execution is done it also
-/// performs the same action as data_update and data_end above. This function
-/// returns 0 if it was able to transfer the execution to a target and an
-/// integer different from zero otherwise.
-int target(int64_t DeviceId, void *HostPtr, int32_t ArgNum, void **ArgBases,
-           void **Args, int64_t *ArgSizes, int64_t *ArgTypes, void **ArgMappers,
-           int32_t TeamNum, int32_t ThreadLimit, int IsTeamConstruct) {
-  DeviceTy &Device = Devices[DeviceId];
-
-  // Find the table information in the map or look it up in the translation
-  // tables.
-  TableMap *TM = 0;
-  {
-    std::lock_guard<std::mutex> TblMapLock(*TblMapMtx);
-    HostPtrToTableMapTy::iterator TableMapIt = HostPtrToTableMap->find(HostPtr);
-    if (TableMapIt == HostPtrToTableMap->end()) {
-      // We don't have a map. So search all the registered libraries.
-      std::lock_guard<std::mutex> TrlTblLock(*TrlTblMtx);
-      for (HostEntriesBeginToTransTableTy::iterator
-               II = HostEntriesBeginToTransTable->begin(),
-               IE = HostEntriesBeginToTransTable->end();
-           !TM && II != IE; ++II) {
-        // get the translation table (which contains all the good info).
-        TranslationTable *TransTable = &II->second;
-        // iterate over all the host table entries to see if we can locate the
-        // host_ptr.
-        __tgt_offload_entry *Begin = TransTable->HostTable.EntriesBegin;
-        __tgt_offload_entry *End = TransTable->HostTable.EntriesEnd;
-        __tgt_offload_entry *Cur = Begin;
-        for (uint32_t I = 0; Cur < End; ++Cur, ++I) {
-          if (Cur->addr != HostPtr)
-            continue;
-          // we got a match, now fill the HostPtrToTableMap so that we
-          // may avoid this search next time.
-          TM = &(*HostPtrToTableMap)[HostPtr];
-          TM->Table = TransTable;
-          TM->Index = I;
-          break;
-        }
-      }
-    } else {
-      TM = &TableMapIt->second;
+namespace {
+/// Find the table information in the map or look it up in the translation
+/// tables.
+TableMap *getTableMap(void *HostPtr) {
+  std::lock_guard<std::mutex> TblMapLock(*TblMapMtx);
+  HostPtrToTableMapTy::iterator TableMapIt = HostPtrToTableMap->find(HostPtr);
+
+  if (TableMapIt != HostPtrToTableMap->end())
+    return &TableMapIt->second;
+
+  // We don't have a map. So search all the registered libraries.
+  TableMap *TM = nullptr;
+  std::lock_guard<std::mutex> TrlTblLock(*TrlTblMtx);
+  for (HostEntriesBeginToTransTableTy::iterator Itr =
+           HostEntriesBeginToTransTable->begin();
+       Itr != HostEntriesBeginToTransTable->end(); ++Itr) {
+    // get the translation table (which contains all the good info).
+    TranslationTable *TransTable = &Itr->second;
+    // iterate over all the host table entries to see if we can locate the
+    // host_ptr.
+    __tgt_offload_entry *Cur = TransTable->HostTable.EntriesBegin;
+    for (uint32_t I = 0; Cur < TransTable->HostTable.EntriesEnd; ++Cur, ++I) {
+      if (Cur->addr != HostPtr)
+        continue;
+      // we got a match, now fill the HostPtrToTableMap so that we
+      // may avoid this search next time.
+      TM = &(*HostPtrToTableMap)[HostPtr];
+      TM->Table = TransTable;
+      TM->Index = I;
+      return TM;
     }
   }
 
-  // No map for this host pointer found!
-  if (!TM) {
-    DP("Host ptr " DPxMOD " does not have a matching target pointer.\n",
-       DPxPTR(HostPtr));
-    return OFFLOAD_FAIL;
-  }
+  return nullptr;
+}
+
+/// Get loop trip count
+/// FIXME: This function will not work right if calling
+/// __kmpc_push_target_tripcount in one thread but doing offloading in another
+/// thread, which might occur when we call task yield.
+uint64_t getLoopTripCount(int64_t DeviceId) {
+  DeviceTy &Device = Devices[DeviceId];
+  uint64_t LoopTripCount = 0;
 
-  // get target table.
-  __tgt_target_table *TargetTable = nullptr;
   {
-    std::lock_guard<std::mutex> TrlTblLock(*TrlTblMtx);
-    assert(TM->Table->TargetsTable.size() > (size_t)DeviceId &&
-           "Not expecting a device ID outside the table's bounds!");
-    TargetTable = TM->Table->TargetsTable[DeviceId];
+    std::lock_guard<std::mutex> TblMapLock(*TblMapMtx);
+    auto I = Device.LoopTripCnt.find(__kmpc_global_thread_num(NULL));
+    if (I != Device.LoopTripCnt.end()) {
+      LoopTripCount = I->second;
+      Device.LoopTripCnt.erase(I);
+      DP("loop trip count is %lu.\n", LoopTripCount);
+    }
   }
-  assert(TargetTable && "Global data has not been mapped\n");
 
-  __tgt_async_info AsyncInfo;
+  return LoopTripCount;
+}
 
-  // Move data to device.
+/// Process data before launching the kernel, including calling targetDataBegin
+/// to map and transfer data to target device, transferring (first-)private
+/// variables.
+int processDataBefore(int64_t DeviceId, void *HostPtr, int32_t ArgNum,
+                      void **ArgBases, void **Args, int64_t *ArgSizes,
+                      int64_t *ArgTypes, void **ArgMappers,
+                      std::vector<void *> &TgtArgs,
+                      std::vector<ptrdiff_t> &TgtOffsets,
+                      std::vector<void *> &FPArrays,
+                      __tgt_async_info *AsyncInfo) {
+  DeviceTy &Device = Devices[DeviceId];
   int Ret = targetDataBegin(Device, ArgNum, ArgBases, Args, ArgSizes, ArgTypes,
-                            ArgMappers, &AsyncInfo);
+                            ArgMappers, AsyncInfo);
   if (Ret != OFFLOAD_SUCCESS) {
     DP("Call to targetDataBegin failed, abort target.\n");
     return OFFLOAD_FAIL;
   }
 
-  std::vector<void *> TgtArgs;
-  std::vector<ptrdiff_t> TgtOffsets;
-
   // List of (first-)private arrays allocated for this target region
-  std::vector<void *> FPArrays;
   std::vector<int> TgtArgsPositions(ArgNum, -1);
 
   for (int32_t I = 0; I < ArgNum; ++I) {
@@ -833,7 +830,7 @@ int target(int64_t DeviceId, void *HostPtr, int32_t ArgNum, void **ArgBases,
         DP("Update lambda reference (" DPxMOD ") -> [" DPxMOD "]\n",
            DPxPTR(PointerTgtPtrBegin), DPxPTR(TgtPtrBegin));
         Ret = Device.submitData(TgtPtrBegin, &PointerTgtPtrBegin,
-                                sizeof(void *), &AsyncInfo);
+                                sizeof(void *), AsyncInfo);
         if (Ret != OFFLOAD_SUCCESS) {
           DP("Copying data to device failed.\n");
           return OFFLOAD_FAIL;
@@ -873,8 +870,8 @@ int target(int64_t DeviceId, void *HostPtr, int32_t ArgNum, void **ArgBases,
 #endif
       // If first-private, copy data from host
       if (ArgTypes[I] & OMP_TGT_MAPTYPE_TO) {
-        Ret = Device.submitData(TgtPtrBegin, HstPtrBegin, ArgSizes[I],
-                                &AsyncInfo);
+        Ret =
+            Device.submitData(TgtPtrBegin, HstPtrBegin, ArgSizes[I], AsyncInfo);
         if (Ret != OFFLOAD_SUCCESS) {
           DP("Copying data to device failed, failed.\n");
           return OFFLOAD_FAIL;
@@ -900,50 +897,112 @@ int target(int64_t DeviceId, void *HostPtr, int32_t ArgNum, void **ArgBases,
   assert(TgtArgs.size() == TgtOffsets.size() &&
          "Size mismatch in arguments and offsets");
 
-  // Pop loop trip count
-  uint64_t LoopTripCount = 0;
-  {
-    std::lock_guard<std::mutex> TblMapLock(*TblMapMtx);
-    auto I = Device.LoopTripCnt.find(__kmpc_global_thread_num(NULL));
-    if (I != Device.LoopTripCnt.end()) {
-      LoopTripCount = I->second;
-      Device.LoopTripCnt.erase(I);
-      DP("loop trip count is %lu.\n", LoopTripCount);
-    }
-  }
+  return OFFLOAD_SUCCESS;
+}
 
-  // Launch device execution.
-  DP("Launching target execution %s with pointer " DPxMOD " (index=%d).\n",
-     TargetTable->EntriesBegin[TM->Index].name,
-     DPxPTR(TargetTable->EntriesBegin[TM->Index].addr), TM->Index);
-  if (IsTeamConstruct) {
-    Ret = Device.runTeamRegion(TargetTable->EntriesBegin[TM->Index].addr,
-                               &TgtArgs[0], &TgtOffsets[0], TgtArgs.size(),
-                               TeamNum, ThreadLimit, LoopTripCount, &AsyncInfo);
-  } else {
-    Ret =
-        Device.runRegion(TargetTable->EntriesBegin[TM->Index].addr, &TgtArgs[0],
-                         &TgtOffsets[0], TgtArgs.size(), &AsyncInfo);
-  }
+/// Process data after launching the kernel, including transferring data back to
+/// host if needed and deallocating target memory of (first-)private variables.
+/// FIXME: This function has correctness issue that target memory might be
+/// deallocated when they're being used.
+int processDataAfter(int64_t DeviceId, void *HostPtr, int32_t ArgNum,
+                     void **ArgBases, void **Args, int64_t *ArgSizes,
+                     int64_t *ArgTypes, void **ArgMappers,
+                     std::vector<void *> &FPArrays,
+                     __tgt_async_info *AsyncInfo) {
+  DeviceTy &Device = Devices[DeviceId];
+
+  // Move data from device.
+  int Ret = targetDataEnd(Device, ArgNum, ArgBases, Args, ArgSizes, ArgTypes,
+                          ArgMappers, AsyncInfo);
   if (Ret != OFFLOAD_SUCCESS) {
-    DP("Executing target region abort target.\n");
+    DP("Call to targetDataEnd failed, abort targe.\n");
     return OFFLOAD_FAIL;
   }
 
   // Deallocate (first-)private arrays
-  for (auto Itr : FPArrays) {
-    Ret = Device.deleteData(Itr);
+  for (void *P : FPArrays) {
+    Ret = Device.deleteData(P);
     if (Ret != OFFLOAD_SUCCESS) {
       DP("Deallocation of (first-)private arrays failed.\n");
       return OFFLOAD_FAIL;
     }
   }
 
-  // Move data from device.
-  Ret = targetDataEnd(Device, ArgNum, ArgBases, Args, ArgSizes, ArgTypes,
-                      ArgMappers, &AsyncInfo);
+  return OFFLOAD_SUCCESS;
+}
+} // namespace
+
+/// performs the same actions as data_begin in case arg_num is
+/// non-zero and initiates run of the offloaded region on the target platform;
+/// if arg_num is non-zero after the region execution is done it also
+/// performs the same action as data_update and data_end above. This function
+/// returns 0 if it was able to transfer the execution to a target and an
+/// integer different from zero otherwise.
+int target(int64_t DeviceId, void *HostPtr, int32_t ArgNum, void **ArgBases,
+           void **Args, int64_t *ArgSizes, int64_t *ArgTypes, void **ArgMappers,
+           int32_t TeamNum, int32_t ThreadLimit, int IsTeamConstruct) {
+  DeviceTy &Device = Devices[DeviceId];
+
+  TableMap *TM = getTableMap(HostPtr);
+  // No map for this host pointer found!
+  if (!TM) {
+    DP("Host ptr " DPxMOD " does not have a matching target pointer.\n",
+       DPxPTR(HostPtr));
+    return OFFLOAD_FAIL;
+  }
+
+  // get target table.
+  __tgt_target_table *TargetTable = nullptr;
+  {
+    std::lock_guard<std::mutex> TrlTblLock(*TrlTblMtx);
+    assert(TM->Table->TargetsTable.size() > (size_t)DeviceId &&
+           "Not expecting a device ID outside the table's bounds!");
+    TargetTable = TM->Table->TargetsTable[DeviceId];
+  }
+  assert(TargetTable && "Global data has not been mapped\n");
+
+  __tgt_async_info AsyncInfo;
+
+  std::vector<void *> TgtArgs;
+  std::vector<ptrdiff_t> TgtOffsets;
+  std::vector<void *> FPArrays;
+
+  // Process data, such as data mapping, before launching the kernel
+  int Ret = processDataBefore(DeviceId, HostPtr, ArgNum, ArgBases, Args,
+                              ArgSizes, ArgTypes, ArgMappers, TgtArgs,
+                              TgtOffsets, FPArrays, &AsyncInfo);
   if (Ret != OFFLOAD_SUCCESS) {
-    DP("Call to targetDataEnd failed, abort targe.\n");
+    DP("Failed to process data before launching the kernel.\n");
+    return OFFLOAD_FAIL;
+  }
+
+  // Get loop trip count
+  uint64_t LoopTripCount = getLoopTripCount(DeviceId);
+
+  // Launch device execution.
+  void *TgtEntryPtr = TargetTable->EntriesBegin[TM->Index].addr;
+  DP("Launching target execution %s with pointer " DPxMOD " (index=%d).\n",
+     TargetTable->EntriesBegin[TM->Index].name, DPxPTR(TgtEntryPtr), TM->Index);
+
+  if (IsTeamConstruct)
+    Ret = Device.runTeamRegion(TgtEntryPtr, &TgtArgs[0], &TgtOffsets[0],
+                               TgtArgs.size(), TeamNum, ThreadLimit,
+                               LoopTripCount, &AsyncInfo);
+  else
+    Ret = Device.runRegion(TgtEntryPtr, &TgtArgs[0], &TgtOffsets[0],
+                           TgtArgs.size(), &AsyncInfo);
+
+  if (Ret != OFFLOAD_SUCCESS) {
+    DP("Executing target region abort target.\n");
+    return OFFLOAD_FAIL;
+  }
+
+  // Transfer data back and deallocate target memory for (first-)private
+  // variables
+  Ret = processDataAfter(DeviceId, HostPtr, ArgNum, ArgBases, Args, ArgSizes,
+                         ArgTypes, ArgMappers, FPArrays, &AsyncInfo);
+  if (Ret != OFFLOAD_SUCCESS) {
+    DP("Failed to process data after launching the kernel.\n");
     return OFFLOAD_FAIL;
   }
 

From 61cab352e377d5a37d47555bc56e11f052d7eb8b Mon Sep 17 00:00:00 2001
From: Vitaly Buka <vitalybuka@google.com>
Date: Thu, 30 Jul 2020 18:22:59 -0700
Subject: [PATCH 0695/1035] [NFC] Move findAllocaForValue into ValueTracking.h

Differential Revision: https://reviews.llvm.org/D84616
---
 llvm/include/llvm/Analysis/ValueTracking.h    |  5 +++
 llvm/include/llvm/Transforms/Utils/Local.h    |  4 --
 llvm/lib/Analysis/ValueTracking.cpp           | 35 +++++++++++++++++
 .../Instrumentation/MemorySanitizer.cpp       |  1 +
 llvm/lib/Transforms/Utils/Local.cpp           | 38 -------------------
 5 files changed, 41 insertions(+), 42 deletions(-)

diff --git a/llvm/include/llvm/Analysis/ValueTracking.h b/llvm/include/llvm/Analysis/ValueTracking.h
index 4d81bb692b47c..8008e6003e7dd 100644
--- a/llvm/include/llvm/Analysis/ValueTracking.h
+++ b/llvm/include/llvm/Analysis/ValueTracking.h
@@ -28,6 +28,7 @@
 namespace llvm {
 
 class AddOperator;
+class AllocaInst;
 class APInt;
 class AssumptionCache;
 class DominatorTree;
@@ -413,6 +414,10 @@ class Value;
                             SmallVectorImpl<Value *> &Objects,
                             const DataLayout &DL);
 
+  /// Finds alloca where the value comes from.
+  AllocaInst *
+  findAllocaForValue(Value *V, DenseMap<Value *, AllocaInst *> &AllocaForValue);
+
   /// Return true if the only users of this pointer are lifetime markers.
   bool onlyUsedByLifetimeMarkers(const Value *V);
 
diff --git a/llvm/include/llvm/Transforms/Utils/Local.h b/llvm/include/llvm/Transforms/Utils/Local.h
index 5cc8d1fa74376..546a5c1b96e90 100644
--- a/llvm/include/llvm/Transforms/Utils/Local.h
+++ b/llvm/include/llvm/Transforms/Utils/Local.h
@@ -305,10 +305,6 @@ bool replaceDbgDeclare(Value *Address, Value *NewAddress, DIBuilder &Builder,
 void replaceDbgValueForAlloca(AllocaInst *AI, Value *NewAllocaAddress,
                               DIBuilder &Builder, int Offset = 0);
 
-/// Finds alloca where the value comes from.
-AllocaInst *findAllocaForValue(Value *V,
-                               DenseMap<Value *, AllocaInst *> &AllocaForValue);
-
 /// Assuming the instruction \p I is going to be deleted, attempt to salvage
 /// debug users of \p I by writing the effect of \p I in a DIExpression. If it
 /// cannot be salvaged changes its debug uses to undef.
diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index 3b8c0fcca1617..6fb36fa826d09 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -4313,6 +4313,41 @@ bool llvm::getUnderlyingObjectsForCodeGen(const Value *V,
   return true;
 }
 
+AllocaInst *
+llvm::findAllocaForValue(Value *V,
+                         DenseMap<Value *, AllocaInst *> &AllocaForValue) {
+  if (AllocaInst *AI = dyn_cast<AllocaInst>(V))
+    return AI;
+  // See if we've already calculated (or started to calculate) alloca for a
+  // given value.
+  auto I = AllocaForValue.find(V);
+  if (I != AllocaForValue.end())
+    return I->second;
+  // Store 0 while we're calculating alloca for value V to avoid
+  // infinite recursion if the value references itself.
+  AllocaForValue[V] = nullptr;
+  AllocaInst *Res = nullptr;
+  if (CastInst *CI = dyn_cast<CastInst>(V))
+    Res = findAllocaForValue(CI->getOperand(0), AllocaForValue);
+  else if (PHINode *PN = dyn_cast<PHINode>(V)) {
+    for (Value *IncValue : PN->incoming_values()) {
+      // Allow self-referencing phi-nodes.
+      if (IncValue == PN)
+        continue;
+      AllocaInst *IncValueAI = findAllocaForValue(IncValue, AllocaForValue);
+      // AI for incoming values should exist and should all be equal.
+      if (IncValueAI == nullptr || (Res != nullptr && IncValueAI != Res))
+        return nullptr;
+      Res = IncValueAI;
+    }
+  } else if (GetElementPtrInst *EP = dyn_cast<GetElementPtrInst>(V)) {
+    Res = findAllocaForValue(EP->getPointerOperand(), AllocaForValue);
+  }
+  if (Res)
+    AllocaForValue[V] = Res;
+  return Res;
+}
+
 static bool onlyUsedByLifetimeMarkersOrDroppableInstsHelper(
     const Value *V, bool AllowLifetime, bool AllowDroppable) {
   for (const User *U : V->users()) {
diff --git a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
index aa06e8144f634..f31d88a633b9a 100644
--- a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
@@ -153,6 +153,7 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/Triple.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
+#include "llvm/Analysis/ValueTracking.h"
 #include "llvm/IR/Argument.h"
 #include "llvm/IR/Attributes.h"
 #include "llvm/IR/BasicBlock.h"
diff --git a/llvm/lib/Transforms/Utils/Local.cpp b/llvm/lib/Transforms/Utils/Local.cpp
index 3d163b8a86bcc..bc4351eec5866 100644
--- a/llvm/lib/Transforms/Utils/Local.cpp
+++ b/llvm/lib/Transforms/Utils/Local.cpp
@@ -3028,44 +3028,6 @@ bool llvm::canReplaceOperandWithVariable(const Instruction *I, unsigned OpIdx) {
   }
 }
 
-using AllocaForValueMapTy = DenseMap<Value *, AllocaInst *>;
-AllocaInst *llvm::findAllocaForValue(Value *V,
-                                     AllocaForValueMapTy &AllocaForValue) {
-  if (AllocaInst *AI = dyn_cast<AllocaInst>(V))
-    return AI;
-  // See if we've already calculated (or started to calculate) alloca for a
-  // given value.
-  AllocaForValueMapTy::iterator I = AllocaForValue.find(V);
-  if (I != AllocaForValue.end())
-    return I->second;
-  // Store 0 while we're calculating alloca for value V to avoid
-  // infinite recursion if the value references itself.
-  AllocaForValue[V] = nullptr;
-  AllocaInst *Res = nullptr;
-  if (CastInst *CI = dyn_cast<CastInst>(V))
-    Res = findAllocaForValue(CI->getOperand(0), AllocaForValue);
-  else if (PHINode *PN = dyn_cast<PHINode>(V)) {
-    for (Value *IncValue : PN->incoming_values()) {
-      // Allow self-referencing phi-nodes.
-      if (IncValue == PN)
-        continue;
-      AllocaInst *IncValueAI = findAllocaForValue(IncValue, AllocaForValue);
-      // AI for incoming values should exist and should all be equal.
-      if (IncValueAI == nullptr || (Res != nullptr && IncValueAI != Res))
-        return nullptr;
-      Res = IncValueAI;
-    }
-  } else if (GetElementPtrInst *EP = dyn_cast<GetElementPtrInst>(V)) {
-    Res = findAllocaForValue(EP->getPointerOperand(), AllocaForValue);
-  } else {
-    LLVM_DEBUG(dbgs() << "Alloca search cancelled on unknown instruction: "
-                      << *V << "\n");
-  }
-  if (Res)
-    AllocaForValue[V] = Res;
-  return Res;
-}
-
 Value *llvm::invertCondition(Value *Condition) {
   // First: Check if it's a constant
   if (Constant *C = dyn_cast<Constant>(Condition))

From 0f1016562648e0c5ab0618823d5d6b7280ca86ba Mon Sep 17 00:00:00 2001
From: Shilei Tian <tianshilei1992@gmail.com>
Date: Thu, 30 Jul 2020 21:37:01 -0400
Subject: [PATCH 0696/1035] [OpenMP] Refactored the function `targetDataEnd`

Refactored the function `targetDataEnd` to make preparation of fixing
the issue of ahead-of-time target memory deallocation. This patch only
renamed `targetDataEnd` related variables and functions to conform
with LLVM code standard.

Reviewed By: ye-luo

Differential Revision: https://reviews.llvm.org/D84991
---
 openmp/libomptarget/src/api.cpp       |   4 +-
 openmp/libomptarget/src/device.cpp    |   4 +-
 openmp/libomptarget/src/device.h      |   4 +-
 openmp/libomptarget/src/omptarget.cpp | 172 +++++++++++++-------------
 openmp/libomptarget/src/private.h     |   6 +-
 5 files changed, 95 insertions(+), 95 deletions(-)

diff --git a/openmp/libomptarget/src/api.cpp b/openmp/libomptarget/src/api.cpp
index da665dbee1e8d..d0f7324347c0c 100644
--- a/openmp/libomptarget/src/api.cpp
+++ b/openmp/libomptarget/src/api.cpp
@@ -163,7 +163,7 @@ EXTERN int omp_target_memcpy(void *dst, void *src, size_t length,
   } else if (dst_device == omp_get_initial_device()) {
     DP("copy from device to host\n");
     DeviceTy& SrcDev = Devices[src_device];
-    rc = SrcDev.data_retrieve(dstAddr, srcAddr, length, nullptr);
+    rc = SrcDev.retrieveData(dstAddr, srcAddr, length, nullptr);
   } else {
     DP("copy from device to device\n");
     DeviceTy &SrcDev = Devices[src_device];
@@ -177,7 +177,7 @@ EXTERN int omp_target_memcpy(void *dst, void *src, size_t length,
     }
 
     void *buffer = malloc(length);
-    rc = SrcDev.data_retrieve(buffer, srcAddr, length, nullptr);
+    rc = SrcDev.retrieveData(buffer, srcAddr, length, nullptr);
     if (rc == OFFLOAD_SUCCESS)
       rc = DstDev.submitData(dstAddr, buffer, length, nullptr);
     free(buffer);
diff --git a/openmp/libomptarget/src/device.cpp b/openmp/libomptarget/src/device.cpp
index c16a1a8cd764e..55d2e0162c9f4 100644
--- a/openmp/libomptarget/src/device.cpp
+++ b/openmp/libomptarget/src/device.cpp
@@ -370,8 +370,8 @@ int32_t DeviceTy::submitData(void *TgtPtrBegin, void *HstPtrBegin, int64_t Size,
 }
 
 // Retrieve data from device
-int32_t DeviceTy::data_retrieve(void *HstPtrBegin, void *TgtPtrBegin,
-                                int64_t Size, __tgt_async_info *AsyncInfoPtr) {
+int32_t DeviceTy::retrieveData(void *HstPtrBegin, void *TgtPtrBegin,
+                               int64_t Size, __tgt_async_info *AsyncInfoPtr) {
   if (!AsyncInfoPtr || !RTL->data_retrieve_async || !RTL->synchronize)
     return RTL->data_retrieve(RTLDeviceID, HstPtrBegin, TgtPtrBegin, Size);
   else
diff --git a/openmp/libomptarget/src/device.h b/openmp/libomptarget/src/device.h
index e4b188c6a8fbc..2cbb2f83e3650 100644
--- a/openmp/libomptarget/src/device.h
+++ b/openmp/libomptarget/src/device.h
@@ -210,8 +210,8 @@ struct DeviceTy {
   int32_t submitData(void *TgtPtrBegin, void *HstPtrBegin, int64_t Size,
                      __tgt_async_info *AsyncInfoPtr);
   // Copy data from device back to host
-  int32_t data_retrieve(void *HstPtrBegin, void *TgtPtrBegin, int64_t Size,
-                        __tgt_async_info *AsyncInfoPtr);
+  int32_t retrieveData(void *HstPtrBegin, void *TgtPtrBegin, int64_t Size,
+                       __tgt_async_info *AsyncInfoPtr);
   // Copy data from current device to destination device directly
   int32_t data_exchange(void *SrcPtr, DeviceTy DstDev, void *DstPtr,
                         int64_t Size, __tgt_async_info *AsyncInfoPtr);
diff --git a/openmp/libomptarget/src/omptarget.cpp b/openmp/libomptarget/src/omptarget.cpp
index ece7df6f15ac7..6704a6fd6e6b0 100644
--- a/openmp/libomptarget/src/omptarget.cpp
+++ b/openmp/libomptarget/src/omptarget.cpp
@@ -56,7 +56,7 @@ int DebugLevel = 0;
  * device will start at 0x200 with the padding (4 bytes), then &s1.b=0x204 and
  * &s1.p=0x208, as they should be to satisfy the alignment requirements.
  */
-static const int64_t alignment = 8;
+static const int64_t Alignment = 8;
 
 /// Map global data and execute pending ctors
 static int InitLibrary(DeviceTy& Device) {
@@ -216,9 +216,9 @@ static int32_t getParentIndex(int64_t type) {
 
 /// Call the user-defined mapper function followed by the appropriate
 // target_data_* function (target_data_{begin,end,update}).
-int target_data_mapper(DeviceTy &Device, void *arg_base,
-    void *arg, int64_t arg_size, int64_t arg_type, void *arg_mapper,
-    TargetDataFuncPtrTy target_data_function) {
+int targetDataMapper(DeviceTy &Device, void *arg_base, void *arg,
+                     int64_t arg_size, int64_t arg_type, void *arg_mapper,
+                     TargetDataFuncPtrTy target_data_function) {
   DP("Calling the mapper function " DPxMOD "\n", DPxPTR(arg_mapper));
 
   // The mapper function fills up Components.
@@ -263,16 +263,15 @@ int targetDataBegin(DeviceTy &Device, int32_t arg_num, void **args_base,
 
     if (arg_mappers && arg_mappers[i]) {
       // Instead of executing the regular path of targetDataBegin, call the
-      // target_data_mapper variant which will call targetDataBegin again
+      // targetDataMapper variant which will call targetDataBegin again
       // with new arguments.
-      DP("Calling target_data_mapper for the %dth argument\n", i);
+      DP("Calling targetDataMapper for the %dth argument\n", i);
 
-      int rc =
-          target_data_mapper(Device, args_base[i], args[i], arg_sizes[i],
-                             arg_types[i], arg_mappers[i], targetDataBegin);
+      int rc = targetDataMapper(Device, args_base[i], args[i], arg_sizes[i],
+                                arg_types[i], arg_mappers[i], targetDataBegin);
 
       if (rc != OFFLOAD_SUCCESS) {
-        DP("Call to targetDataBegin via target_data_mapper for custom mapper"
+        DP("Call to targetDataBegin via targetDataMapper for custom mapper"
            " failed.\n");
         return OFFLOAD_FAIL;
       }
@@ -292,7 +291,7 @@ int targetDataBegin(DeviceTy &Device, int32_t arg_num, void **args_base,
     const int next_i = i+1;
     if (getParentIndex(arg_types[i]) < 0 && next_i < arg_num &&
         getParentIndex(arg_types[next_i]) == i) {
-      padding = (int64_t)HstPtrBegin % alignment;
+      padding = (int64_t)HstPtrBegin % Alignment;
       if (padding) {
         DP("Using a padding of %" PRId64 " bytes for begin address " DPxMOD
             "\n", padding, DPxPTR(HstPtrBegin));
@@ -423,28 +422,29 @@ int targetDataBegin(DeviceTy &Device, int32_t arg_num, void **args_base,
 }
 
 /// Internal function to undo the mapping and retrieve the data from the device.
-int targetDataEnd(DeviceTy &Device, int32_t arg_num, void **args_base,
-                  void **args, int64_t *arg_sizes, int64_t *arg_types,
-                  void **arg_mappers, __tgt_async_info *async_info_ptr) {
+int targetDataEnd(DeviceTy &Device, int32_t ArgNum, void **ArgBases,
+                  void **Args, int64_t *ArgSizes, int64_t *ArgTypes,
+                  void **ArgMappers, __tgt_async_info *AsyncInfo) {
+  int Ret;
   // process each input.
-  for (int32_t i = arg_num - 1; i >= 0; --i) {
+  for (int32_t I = ArgNum - 1; I >= 0; --I) {
     // Ignore private variables and arrays - there is no mapping for them.
     // Also, ignore the use_device_ptr directive, it has no effect here.
-    if ((arg_types[i] & OMP_TGT_MAPTYPE_LITERAL) ||
-        (arg_types[i] & OMP_TGT_MAPTYPE_PRIVATE))
+    if ((ArgTypes[I] & OMP_TGT_MAPTYPE_LITERAL) ||
+        (ArgTypes[I] & OMP_TGT_MAPTYPE_PRIVATE))
       continue;
 
-    if (arg_mappers && arg_mappers[i]) {
+    if (ArgMappers && ArgMappers[I]) {
       // Instead of executing the regular path of targetDataEnd, call the
-      // target_data_mapper variant which will call targetDataEnd again
+      // targetDataMapper variant which will call targetDataEnd again
       // with new arguments.
-      DP("Calling target_data_mapper for the %dth argument\n", i);
+      DP("Calling targetDataMapper for the %dth argument\n", I);
 
-      int rc = target_data_mapper(Device, args_base[i], args[i], arg_sizes[i],
-                                  arg_types[i], arg_mappers[i], targetDataEnd);
+      Ret = targetDataMapper(Device, ArgBases[I], Args[I], ArgSizes[I],
+                             ArgTypes[I], ArgMappers[I], targetDataEnd);
 
-      if (rc != OFFLOAD_SUCCESS) {
-        DP("Call to targetDataEnd via target_data_mapper for custom mapper"
+      if (Ret != OFFLOAD_SUCCESS) {
+        DP("Call to targetDataEnd via targetDataMapper for custom mapper"
            " failed.\n");
         return OFFLOAD_FAIL;
       }
@@ -453,35 +453,35 @@ int targetDataEnd(DeviceTy &Device, int32_t arg_num, void **args_base,
       continue;
     }
 
-    void *HstPtrBegin = args[i];
-    int64_t data_size = arg_sizes[i];
+    void *HstPtrBegin = Args[I];
+    int64_t DataSize = ArgSizes[I];
     // Adjust for proper alignment if this is a combined entry (for structs).
     // Look at the next argument - if that is MEMBER_OF this one, then this one
     // is a combined entry.
-    int64_t padding = 0;
-    const int next_i = i+1;
-    if (getParentIndex(arg_types[i]) < 0 && next_i < arg_num &&
-        getParentIndex(arg_types[next_i]) == i) {
-      padding = (int64_t)HstPtrBegin % alignment;
-      if (padding) {
-        DP("Using a padding of %" PRId64 " bytes for begin address " DPxMOD
-            "\n", padding, DPxPTR(HstPtrBegin));
-        HstPtrBegin = (char *) HstPtrBegin - padding;
-        data_size += padding;
+    const int NextI = I + 1;
+    if (getParentIndex(ArgTypes[I]) < 0 && NextI < ArgNum &&
+        getParentIndex(ArgTypes[NextI]) == I) {
+      int64_t Padding = (int64_t)HstPtrBegin % Alignment;
+      if (Padding) {
+        DP("Using a Padding of %" PRId64 " bytes for begin address " DPxMOD
+           "\n",
+           Padding, DPxPTR(HstPtrBegin));
+        HstPtrBegin = (char *)HstPtrBegin - Padding;
+        DataSize += Padding;
       }
     }
 
     bool IsLast, IsHostPtr;
-    bool UpdateRef = !(arg_types[i] & OMP_TGT_MAPTYPE_MEMBER_OF) ||
-        (arg_types[i] & OMP_TGT_MAPTYPE_PTR_AND_OBJ);
-    bool ForceDelete = arg_types[i] & OMP_TGT_MAPTYPE_DELETE;
-    bool HasCloseModifier = arg_types[i] & OMP_TGT_MAPTYPE_CLOSE;
-    bool HasPresentModifier = arg_types[i] & OMP_TGT_MAPTYPE_PRESENT;
+    bool UpdateRef = !(ArgTypes[I] & OMP_TGT_MAPTYPE_MEMBER_OF) ||
+                     (ArgTypes[I] & OMP_TGT_MAPTYPE_PTR_AND_OBJ);
+    bool ForceDelete = ArgTypes[I] & OMP_TGT_MAPTYPE_DELETE;
+    bool HasCloseModifier = ArgTypes[I] & OMP_TGT_MAPTYPE_CLOSE;
+    bool HasPresentModifier = ArgTypes[I] & OMP_TGT_MAPTYPE_PRESENT;
 
     // If PTR_AND_OBJ, HstPtrBegin is address of pointee
-    void *TgtPtrBegin = Device.getTgtPtrBegin(HstPtrBegin, data_size, IsLast,
-        UpdateRef, IsHostPtr);
-    if (!TgtPtrBegin && (data_size || HasPresentModifier)) {
+    void *TgtPtrBegin = Device.getTgtPtrBegin(HstPtrBegin, DataSize, IsLast,
+                                              UpdateRef, IsHostPtr);
+    if (!TgtPtrBegin && (DataSize || HasPresentModifier)) {
       DP("Mapping does not exist (%s)\n",
          (HasPresentModifier ? "'present' map type modifier" : "ignored"));
       if (HasPresentModifier) {
@@ -489,38 +489,37 @@ int targetDataEnd(DeviceTy &Device, int32_t arg_num, void **args_base,
         // but it should be an error upon entering an "omp target exit data".
         MESSAGE("device mapping required by 'present' map type modifier does "
                 "not exist for host address " DPxMOD " (%ld bytes)",
-                DPxPTR(HstPtrBegin), data_size);
+                DPxPTR(HstPtrBegin), DataSize);
         return OFFLOAD_FAIL;
       }
     } else {
       DP("There are %" PRId64 " bytes allocated at target address " DPxMOD
          " - is%s last\n",
-         data_size, DPxPTR(TgtPtrBegin), (IsLast ? "" : " not"));
+         DataSize, DPxPTR(TgtPtrBegin), (IsLast ? "" : " not"));
     }
 
     bool DelEntry = IsLast || ForceDelete;
 
-    if ((arg_types[i] & OMP_TGT_MAPTYPE_MEMBER_OF) &&
-        !(arg_types[i] & OMP_TGT_MAPTYPE_PTR_AND_OBJ)) {
+    if ((ArgTypes[I] & OMP_TGT_MAPTYPE_MEMBER_OF) &&
+        !(ArgTypes[I] & OMP_TGT_MAPTYPE_PTR_AND_OBJ)) {
       DelEntry = false; // protect parent struct from being deallocated
     }
 
-    if ((arg_types[i] & OMP_TGT_MAPTYPE_FROM) || DelEntry) {
+    if ((ArgTypes[I] & OMP_TGT_MAPTYPE_FROM) || DelEntry) {
       // Move data back to the host
-      if (arg_types[i] & OMP_TGT_MAPTYPE_FROM) {
-        bool Always = arg_types[i] & OMP_TGT_MAPTYPE_ALWAYS;
+      if (ArgTypes[I] & OMP_TGT_MAPTYPE_FROM) {
+        bool Always = ArgTypes[I] & OMP_TGT_MAPTYPE_ALWAYS;
         bool CopyMember = false;
         if (!(RTLs->RequiresFlags & OMP_REQ_UNIFIED_SHARED_MEMORY) ||
             HasCloseModifier) {
-          if ((arg_types[i] & OMP_TGT_MAPTYPE_MEMBER_OF) &&
-              !(arg_types[i] & OMP_TGT_MAPTYPE_PTR_AND_OBJ)) {
+          if ((ArgTypes[I] & OMP_TGT_MAPTYPE_MEMBER_OF) &&
+              !(ArgTypes[I] & OMP_TGT_MAPTYPE_PTR_AND_OBJ)) {
             // Copy data only if the "parent" struct has RefCount==1.
-            int32_t parent_idx = getParentIndex(arg_types[i]);
-            uint64_t parent_rc = Device.getMapEntryRefCnt(args[parent_idx]);
-            assert(parent_rc > 0 && "parent struct not found");
-            if (parent_rc == 1) {
+            int32_t ParentIdx = getParentIndex(ArgTypes[I]);
+            uint64_t ParentRC = Device.getMapEntryRefCnt(Args[ParentIdx]);
+            assert(ParentRC > 0 && "parent struct not found");
+            if (ParentRC == 1)
               CopyMember = true;
-            }
           }
         }
 
@@ -528,10 +527,10 @@ int targetDataEnd(DeviceTy &Device, int32_t arg_num, void **args_base,
             !(RTLs->RequiresFlags & OMP_REQ_UNIFIED_SHARED_MEMORY &&
               TgtPtrBegin == HstPtrBegin)) {
           DP("Moving %" PRId64 " bytes (tgt:" DPxMOD ") -> (hst:" DPxMOD ")\n",
-             data_size, DPxPTR(TgtPtrBegin), DPxPTR(HstPtrBegin));
-          int rt = Device.data_retrieve(HstPtrBegin, TgtPtrBegin, data_size,
-                                        async_info_ptr);
-          if (rt != OFFLOAD_SUCCESS) {
+             DataSize, DPxPTR(TgtPtrBegin), DPxPTR(HstPtrBegin));
+          Ret = Device.retrieveData(HstPtrBegin, TgtPtrBegin, DataSize,
+                                    AsyncInfo);
+          if (Ret != OFFLOAD_SUCCESS) {
             DP("Copying data from device failed.\n");
             return OFFLOAD_FAIL;
           }
@@ -542,44 +541,44 @@ int targetDataEnd(DeviceTy &Device, int32_t arg_num, void **args_base,
       // need to restore the original host pointer values from their shadow
       // copies. If the struct is going to be deallocated, remove any remaining
       // shadow pointer entries for this struct.
-      uintptr_t lb = (uintptr_t) HstPtrBegin;
-      uintptr_t ub = (uintptr_t) HstPtrBegin + data_size;
+      uintptr_t LB = (uintptr_t)HstPtrBegin;
+      uintptr_t UB = (uintptr_t)HstPtrBegin + DataSize;
       Device.ShadowMtx.lock();
-      for (ShadowPtrListTy::iterator it = Device.ShadowPtrMap.begin();
-           it != Device.ShadowPtrMap.end();) {
-        void **ShadowHstPtrAddr = (void**) it->first;
+      for (ShadowPtrListTy::iterator Itr = Device.ShadowPtrMap.begin();
+           Itr != Device.ShadowPtrMap.end();) {
+        void **ShadowHstPtrAddr = (void **)Itr->first;
 
         // An STL map is sorted on its keys; use this property
         // to quickly determine when to break out of the loop.
-        if ((uintptr_t) ShadowHstPtrAddr < lb) {
-          ++it;
+        if ((uintptr_t)ShadowHstPtrAddr < LB) {
+          ++Itr;
           continue;
         }
-        if ((uintptr_t) ShadowHstPtrAddr >= ub)
+        if ((uintptr_t)ShadowHstPtrAddr >= UB)
           break;
 
         // If we copied the struct to the host, we need to restore the pointer.
-        if (arg_types[i] & OMP_TGT_MAPTYPE_FROM) {
+        if (ArgTypes[I] & OMP_TGT_MAPTYPE_FROM) {
           DP("Restoring original host pointer value " DPxMOD " for host "
-              "pointer " DPxMOD "\n", DPxPTR(it->second.HstPtrVal),
-              DPxPTR(ShadowHstPtrAddr));
-          *ShadowHstPtrAddr = it->second.HstPtrVal;
+             "pointer " DPxMOD "\n",
+             DPxPTR(Itr->second.HstPtrVal), DPxPTR(ShadowHstPtrAddr));
+          *ShadowHstPtrAddr = Itr->second.HstPtrVal;
         }
         // If the struct is to be deallocated, remove the shadow entry.
         if (DelEntry) {
           DP("Removing shadow pointer " DPxMOD "\n", DPxPTR(ShadowHstPtrAddr));
-          it = Device.ShadowPtrMap.erase(it);
+          Itr = Device.ShadowPtrMap.erase(Itr);
         } else {
-          ++it;
+          ++Itr;
         }
       }
       Device.ShadowMtx.unlock();
 
       // Deallocate map
       if (DelEntry) {
-        int rt = Device.deallocTgtPtr(HstPtrBegin, data_size, ForceDelete,
-                                      HasCloseModifier);
-        if (rt != OFFLOAD_SUCCESS) {
+        Ret = Device.deallocTgtPtr(HstPtrBegin, DataSize, ForceDelete,
+                                   HasCloseModifier);
+        if (Ret != OFFLOAD_SUCCESS) {
           DP("Deallocating data from device failed.\n");
           return OFFLOAD_FAIL;
         }
@@ -604,16 +603,17 @@ int target_data_update(DeviceTy &Device, int32_t arg_num,
 
     if (arg_mappers && arg_mappers[i]) {
       // Instead of executing the regular path of target_data_update, call the
-      // target_data_mapper variant which will call target_data_update again
+      // targetDataMapper variant which will call target_data_update again
       // with new arguments.
-      DP("Calling target_data_mapper for the %dth argument\n", i);
+      DP("Calling targetDataMapper for the %dth argument\n", i);
 
-      int rc = target_data_mapper(Device, args_base[i], args[i], arg_sizes[i],
-          arg_types[i], arg_mappers[i], target_data_update);
+      int rc =
+          targetDataMapper(Device, args_base[i], args[i], arg_sizes[i],
+                           arg_types[i], arg_mappers[i], target_data_update);
 
       if (rc != OFFLOAD_SUCCESS) {
-        DP("Call to target_data_update via target_data_mapper for custom mapper"
-            " failed.\n");
+        DP("Call to target_data_update via targetDataMapper for custom mapper"
+           " failed.\n");
         return OFFLOAD_FAIL;
       }
 
@@ -647,7 +647,7 @@ int target_data_update(DeviceTy &Device, int32_t arg_num,
     if (arg_types[i] & OMP_TGT_MAPTYPE_FROM) {
       DP("Moving %" PRId64 " bytes (tgt:" DPxMOD ") -> (hst:" DPxMOD ")\n",
           arg_sizes[i], DPxPTR(TgtPtrBegin), DPxPTR(HstPtrBegin));
-      int rt = Device.data_retrieve(HstPtrBegin, TgtPtrBegin, MapSize, nullptr);
+      int rt = Device.retrieveData(HstPtrBegin, TgtPtrBegin, MapSize, nullptr);
       if (rt != OFFLOAD_SUCCESS) {
         DP("Copying data from device failed.\n");
         return OFFLOAD_FAIL;
diff --git a/openmp/libomptarget/src/private.h b/openmp/libomptarget/src/private.h
index 41da68f24d595..c5ca21d12e5e5 100644
--- a/openmp/libomptarget/src/private.h
+++ b/openmp/libomptarget/src/private.h
@@ -22,9 +22,9 @@ extern int targetDataBegin(DeviceTy &Device, int32_t arg_num, void **args_base,
                            void **arg_mappers,
                            __tgt_async_info *async_info_ptr);
 
-extern int targetDataEnd(DeviceTy &Device, int32_t arg_num, void **args_base,
-                         void **args, int64_t *arg_sizes, int64_t *arg_types,
-                         void **arg_mappers, __tgt_async_info *async_info_ptr);
+extern int targetDataEnd(DeviceTy &Device, int32_t ArgNum, void **ArgBases,
+                         void **Args, int64_t *ArgSizes, int64_t *ArgTypes,
+                         void **ArgMappers, __tgt_async_info *AsyncInfo);
 
 extern int target_data_update(DeviceTy &Device, int32_t arg_num,
                               void **args_base, void **args,

From b256cb88a716dfd270745af1bb98b49c22521a99 Mon Sep 17 00:00:00 2001
From: Vitaly Buka <vitalybuka@google.com>
Date: Thu, 30 Jul 2020 18:48:34 -0700
Subject: [PATCH 0697/1035] [ValueTracking] Remove AllocaForValue parameter

findAllocaForValue uses AllocaForValue to cache resolved values.
The function is used only to resolve arguments of lifetime
intrinsic which usually are not fare for allocas. So result reuse
is likely unnoticeable.

In followup patches I'd like to replace the function with
GetUnderlyingObjects.

Depends on D84616.

Differential Revision: https://reviews.llvm.org/D84617
---
 llvm/include/llvm/Analysis/ValueTracking.h             |  6 ++++--
 llvm/lib/Analysis/ValueTracking.cpp                    | 10 +++++++---
 llvm/lib/Target/AArch64/AArch64StackTagging.cpp        |  4 +---
 .../Transforms/Instrumentation/AddressSanitizer.cpp    |  7 +------
 .../lib/Transforms/Instrumentation/MemorySanitizer.cpp |  4 +---
 5 files changed, 14 insertions(+), 17 deletions(-)

diff --git a/llvm/include/llvm/Analysis/ValueTracking.h b/llvm/include/llvm/Analysis/ValueTracking.h
index 8008e6003e7dd..92dc7c01e83bc 100644
--- a/llvm/include/llvm/Analysis/ValueTracking.h
+++ b/llvm/include/llvm/Analysis/ValueTracking.h
@@ -415,8 +415,10 @@ class Value;
                             const DataLayout &DL);
 
   /// Finds alloca where the value comes from.
-  AllocaInst *
-  findAllocaForValue(Value *V, DenseMap<Value *, AllocaInst *> &AllocaForValue);
+  AllocaInst *findAllocaForValue(Value *V);
+  inline const AllocaInst *findAllocaForValue(const Value *V) {
+    return findAllocaForValue(const_cast<Value *>(V));
+  }
 
   /// Return true if the only users of this pointer are lifetime markers.
   bool onlyUsedByLifetimeMarkers(const Value *V);
diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index 6fb36fa826d09..9dfe8fdcb1a5c 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -4313,9 +4313,8 @@ bool llvm::getUnderlyingObjectsForCodeGen(const Value *V,
   return true;
 }
 
-AllocaInst *
-llvm::findAllocaForValue(Value *V,
-                         DenseMap<Value *, AllocaInst *> &AllocaForValue) {
+static AllocaInst *
+findAllocaForValue(Value *V, DenseMap<Value *, AllocaInst *> &AllocaForValue) {
   if (AllocaInst *AI = dyn_cast<AllocaInst>(V))
     return AI;
   // See if we've already calculated (or started to calculate) alloca for a
@@ -4348,6 +4347,11 @@ llvm::findAllocaForValue(Value *V,
   return Res;
 }
 
+AllocaInst *llvm::findAllocaForValue(Value *V) {
+  DenseMap<Value *, AllocaInst *> AllocaForValue;
+  return ::findAllocaForValue(V, AllocaForValue);
+}
+
 static bool onlyUsedByLifetimeMarkersOrDroppableInstsHelper(
     const Value *V, bool AllowLifetime, bool AllowDroppable) {
   for (const User *U : V->users()) {
diff --git a/llvm/lib/Target/AArch64/AArch64StackTagging.cpp b/llvm/lib/Target/AArch64/AArch64StackTagging.cpp
index 0ed576fbc75f9..1b8c6417be38d 100644
--- a/llvm/lib/Target/AArch64/AArch64StackTagging.cpp
+++ b/llvm/lib/Target/AArch64/AArch64StackTagging.cpp
@@ -544,7 +544,6 @@ bool AArch64StackTagging::runOnFunction(Function &Fn) {
 
   MapVector<AllocaInst *, AllocaInfo> Allocas; // need stable iteration order
   SmallVector<Instruction *, 8> RetVec;
-  DenseMap<Value *, AllocaInst *> AllocaForValue;
   SmallVector<Instruction *, 4> UnrecognizedLifetimes;
 
   for (auto &BB : *F) {
@@ -566,8 +565,7 @@ bool AArch64StackTagging::runOnFunction(Function &Fn) {
       auto *II = dyn_cast<IntrinsicInst>(I);
       if (II && (II->getIntrinsicID() == Intrinsic::lifetime_start ||
                  II->getIntrinsicID() == Intrinsic::lifetime_end)) {
-        AllocaInst *AI =
-            llvm::findAllocaForValue(II->getArgOperand(1), AllocaForValue);
+        AllocaInst *AI = findAllocaForValue(II->getArgOperand(1));
         if (!AI) {
           UnrecognizedLifetimes.push_back(I);
           continue;
diff --git a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
index 647d25e6a24e0..053b9aa812bf4 100644
--- a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
@@ -911,10 +911,6 @@ struct FunctionStackPoisoner : public InstVisitor<FunctionStackPoisoner> {
   AllocaInst *DynamicAllocaLayout = nullptr;
   IntrinsicInst *LocalEscapeCall = nullptr;
 
-  // Maps Value to an AllocaInst from which the Value is originated.
-  using AllocaForValueMapTy = DenseMap<Value *, AllocaInst *>;
-  AllocaForValueMapTy AllocaForValue;
-
   bool HasInlineAsm = false;
   bool HasReturnsTwiceCall = false;
 
@@ -1065,8 +1061,7 @@ struct FunctionStackPoisoner : public InstVisitor<FunctionStackPoisoner> {
         !ConstantInt::isValueValidForType(IntptrTy, SizeValue))
       return;
     // Find alloca instruction that corresponds to llvm.lifetime argument.
-    AllocaInst *AI =
-        llvm::findAllocaForValue(II.getArgOperand(1), AllocaForValue);
+    AllocaInst *AI = findAllocaForValue(II.getArgOperand(1));
     if (!AI) {
       HasUntracedLifetimeIntrinsic = true;
       return;
diff --git a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
index f31d88a633b9a..edde80ce0ee8f 100644
--- a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
@@ -2693,9 +2693,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
   void handleLifetimeStart(IntrinsicInst &I) {
     if (!PoisonStack)
       return;
-    DenseMap<Value *, AllocaInst *> AllocaForValue;
-    AllocaInst *AI =
-        llvm::findAllocaForValue(I.getArgOperand(1), AllocaForValue);
+    AllocaInst *AI = llvm::findAllocaForValue(I.getArgOperand(1));
     if (!AI)
       InstrumentLifetimeStart = false;
     LifetimeStartList.push_back(std::make_pair(&I, AI));

From 2cf52504bb076d24c23f161e92340828052b69f7 Mon Sep 17 00:00:00 2001
From: Peter Steinfeld <psteinfeld@nvidia.com>
Date: Wed, 29 Jul 2020 14:46:36 -0700
Subject: [PATCH 0698/1035] [flang] Fixes for RESHAPE()

I fixed an assert caused by passing an empty array as the source= argument to
RESHAPE().  In the process, I noticed that there were no tests for RESHAPE(),
so I wrote a test that covers all the description in 16.9.163.  In the process,
I made the error messages more consistent and descriptive.  I also changed the
test to see if a reference to an intrinsic function was a constant to say that
it is a constant if it's a refererence to an invalid intrinsic.  This avoids
emitting multiple messages for the same erroneous source.

Differential Revision: https://reviews.llvm.org/D84904
---
 flang/include/flang/Evaluate/constant.h   |  2 +-
 flang/include/flang/Evaluate/intrinsics.h |  5 +++
 flang/lib/Evaluate/check-expression.cpp   |  5 ++-
 flang/lib/Evaluate/constant.cpp           |  6 +--
 flang/lib/Evaluate/fold-implementation.h  | 23 +++++++----
 flang/lib/Evaluate/intrinsics.cpp         |  6 ++-
 flang/test/Semantics/reshape.f90          | 50 +++++++++++++++++++++++
 7 files changed, 83 insertions(+), 14 deletions(-)
 create mode 100644 flang/test/Semantics/reshape.f90

diff --git a/flang/include/flang/Evaluate/constant.h b/flang/include/flang/Evaluate/constant.h
index 1617bdd086dcb..a25916f94ef79 100644
--- a/flang/include/flang/Evaluate/constant.h
+++ b/flang/include/flang/Evaluate/constant.h
@@ -54,7 +54,7 @@ std::size_t TotalElementCount(const ConstantSubscripts &);
 std::optional<std::vector<int>> ValidateDimensionOrder(
     int rank, const std::vector<int> &order);
 
-bool IsValidShape(const ConstantSubscripts &);
+bool HasNegativeExtent(const ConstantSubscripts &);
 
 class ConstantBounds {
 public:
diff --git a/flang/include/flang/Evaluate/intrinsics.h b/flang/include/flang/Evaluate/intrinsics.h
index 50212a1434c33..09f5691b1ea76 100644
--- a/flang/include/flang/Evaluate/intrinsics.h
+++ b/flang/include/flang/Evaluate/intrinsics.h
@@ -92,6 +92,11 @@ class IntrinsicProcTable {
   std::optional<SpecificIntrinsicFunctionInterface> IsSpecificIntrinsicFunction(
       const std::string &) const;
 
+  // Illegal name for an intrinsic used to avoid cascading error messages when
+  // constant folding.
+  static const inline std::string InvalidName{
+      "(invalid intrinsic function call)"};
+
   llvm::raw_ostream &Dump(llvm::raw_ostream &) const;
 
 private:
diff --git a/flang/lib/Evaluate/check-expression.cpp b/flang/lib/Evaluate/check-expression.cpp
index 9ac1a12e0f4ea..a7cc094033d05 100644
--- a/flang/lib/Evaluate/check-expression.cpp
+++ b/flang/lib/Evaluate/check-expression.cpp
@@ -43,7 +43,10 @@ class IsConstantExprHelper : public AllTraverse<IsConstantExprHelper, true> {
   }
   template <typename T> bool operator()(const FunctionRef<T> &call) const {
     if (const auto *intrinsic{std::get_if<SpecificIntrinsic>(&call.proc().u)}) {
-      return intrinsic->name == "kind";
+      // kind is always a constant, and we avoid cascading errors by calling
+      // invalid calls to intrinsics constant
+      return intrinsic->name == "kind" ||
+          intrinsic->name == IntrinsicProcTable::InvalidName;
       // TODO: other inquiry intrinsics
     } else {
       return false;
diff --git a/flang/lib/Evaluate/constant.cpp b/flang/lib/Evaluate/constant.cpp
index e7497630f5969..5b73979f1e2c2 100644
--- a/flang/lib/Evaluate/constant.cpp
+++ b/flang/lib/Evaluate/constant.cpp
@@ -93,13 +93,13 @@ std::optional<std::vector<int>> ValidateDimensionOrder(
   }
 }
 
-bool IsValidShape(const ConstantSubscripts &shape) {
+bool HasNegativeExtent(const ConstantSubscripts &shape) {
   for (ConstantSubscript extent : shape) {
     if (extent < 0) {
-      return false;
+      return true;
     }
   }
-  return shape.size() <= common::maxRank;
+  return false;
 }
 
 template <typename RESULT, typename ELEMENT>
diff --git a/flang/lib/Evaluate/fold-implementation.h b/flang/lib/Evaluate/fold-implementation.h
index 85e35613d6404..ebe826fcbc126 100644
--- a/flang/lib/Evaluate/fold-implementation.h
+++ b/flang/lib/Evaluate/fold-implementation.h
@@ -22,6 +22,7 @@
 #include "flang/Evaluate/expression.h"
 #include "flang/Evaluate/fold.h"
 #include "flang/Evaluate/formatting.h"
+#include "flang/Evaluate/intrinsics.h"
 #include "flang/Evaluate/shape.h"
 #include "flang/Evaluate/tools.h"
 #include "flang/Evaluate/traverse.h"
@@ -600,9 +601,9 @@ std::optional<std::vector<A>> GetIntegerVector(const B &x) {
 // gets re-folded.
 template <typename T> Expr<T> MakeInvalidIntrinsic(FunctionRef<T> &&funcRef) {
   SpecificIntrinsic invalid{std::get<SpecificIntrinsic>(funcRef.proc().u)};
-  invalid.name = "(invalid intrinsic function call)";
+  invalid.name = IntrinsicProcTable::InvalidName;
   return Expr<T>{FunctionRef<T>{ProcedureDesignator{std::move(invalid)},
-      ActualArguments{ActualArgument{AsGenericExpr(std::move(funcRef))}}}};
+      ActualArguments{std::move(funcRef.arguments())}}};
 }
 
 template <typename T> Expr<T> Folder<T>::Reshape(FunctionRef<T> &&funcRef) {
@@ -615,8 +616,13 @@ template <typename T> Expr<T> Folder<T>::Reshape(FunctionRef<T> &&funcRef) {
   std::optional<std::vector<int>> order{GetIntegerVector<int>(args[3])};
   if (!source || !shape || (args[2] && !pad) || (args[3] && !order)) {
     return Expr<T>{std::move(funcRef)}; // Non-constant arguments
-  } else if (!IsValidShape(shape.value())) {
-    context_.messages().Say("Invalid SHAPE in RESHAPE"_en_US);
+  } else if (shape.value().size() > common::maxRank) {
+    context_.messages().Say(
+        "Size of 'shape=' argument must not be greater than %d"_err_en_US,
+        common::maxRank);
+  } else if (HasNegativeExtent(shape.value())) {
+    context_.messages().Say(
+        "'shape=' argument must not have a negative extent"_err_en_US);
   } else {
     int rank{GetRank(shape.value())};
     std::size_t resultElements{TotalElementCount(shape.value())};
@@ -626,12 +632,13 @@ template <typename T> Expr<T> Folder<T>::Reshape(FunctionRef<T> &&funcRef) {
     }
     std::vector<int> *dimOrderPtr{dimOrder ? &dimOrder.value() : nullptr};
     if (order && !dimOrder) {
-      context_.messages().Say("Invalid ORDER in RESHAPE"_en_US);
+      context_.messages().Say("Invalid 'order=' argument in RESHAPE"_err_en_US);
     } else if (resultElements > source->size() && (!pad || pad->empty())) {
-      context_.messages().Say("Too few SOURCE elements in RESHAPE and PAD"
-                              "is not present or has null size"_en_US);
+      context_.messages().Say(
+          "Too few elements in 'source=' argument and 'pad=' "
+          "argument is not present or has null size"_err_en_US);
     } else {
-      Constant<T> result{!source->empty()
+      Constant<T> result{!source->empty() || !pad
               ? source->Reshape(std::move(shape.value()))
               : pad->Reshape(std::move(shape.value()))};
       ConstantSubscripts subscripts{result.lbounds()};
diff --git a/flang/lib/Evaluate/intrinsics.cpp b/flang/lib/Evaluate/intrinsics.cpp
index 0ad5c195ae861..35a69e4e9b937 100644
--- a/flang/lib/Evaluate/intrinsics.cpp
+++ b/flang/lib/Evaluate/intrinsics.cpp
@@ -1258,7 +1258,11 @@ std::optional<SpecificCall> IntrinsicInterface::Match(
         break;
       case Rank::shape:
         CHECK(!shapeArgSize);
-        if (rank == 1) {
+        if (rank != 1) {
+          messages.Say(
+              "'shape=' argument must be an array of rank 1"_err_en_US);
+          return std::nullopt;
+        } else {
           if (auto shape{GetShape(context, *arg)}) {
             if (auto constShape{AsConstantShape(context, *shape)}) {
               shapeArgSize = constShape->At(ConstantSubscripts{1}).ToInt64();
diff --git a/flang/test/Semantics/reshape.f90 b/flang/test/Semantics/reshape.f90
new file mode 100644
index 0000000000000..7749df60b0dca
--- /dev/null
+++ b/flang/test/Semantics/reshape.f90
@@ -0,0 +1,50 @@
+! RUN: %S/test_errors.sh %s %t %f18
+
+!Tests for RESHAPE
+program reshaper
+  ! RESHAPE with arguments SOURCE and SHAPE
+  integer, parameter :: array1(2,3) = RESHAPE([(n, n=1,6)], [2,3])
+  ! RESHAPE with arguments SOURCE, SHAPE, and PAD
+  integer :: array2(2,3) = RESHAPE([(n, n=1,4)], [2,3], [99])
+  ! RESHAPE with arguments SOURCE, SHAPE, PAD, and ORDER
+  integer :: array3(2,3) = RESHAPE([(n, n=1,4)], [2,3], [99], [2, 1])
+  !ERROR: Too few elements in 'source=' argument and 'pad=' argument is not present or has null size
+  integer :: array4(2,3) = RESHAPE([(n, n=1,5)], [2,3])
+  !ERROR: Actual argument for 'shape=' has bad type 'REAL(4)'
+  integer :: array5(2,3) = RESHAPE([(n, n=1,6)], [2.2,3.3])
+  !ERROR: 'shape=' argument must be an array of rank 1
+  integer :: array6(2,3) = RESHAPE([(n, n=1,6)], RESHAPE([(n, n=1,6)], [2,3]))
+  !ERROR: 'shape=' argument must be an array of rank 1
+  integer :: array7(2,3) = RESHAPE([(n, n=1,4)], 343)
+  !ERROR: Actual argument for 'pad=' has bad type or kind 'INTEGER(8)'
+  integer :: array8(2,3) = RESHAPE([(n, n=1,4)], [2,3], [99_8])
+  !ERROR: Actual argument for 'pad=' has bad type or kind 'REAL(4)'
+  real :: array9(2,3) = RESHAPE([(n, n=1,4)], [2,3], [99.9])
+  !ERROR: Invalid 'order=' argument in RESHAPE
+  real :: array10(2,3) = RESHAPE([(n,n=1,4)],[2,3],[99],[2,3])
+  !ERROR: Actual argument for 'order=' has bad type 'REAL(4)'
+  real :: array11(2,3) = RESHAPE([(n, n=1,4)], [2,3], [99], [2.2,3.3])
+  !ERROR: Invalid 'order=' argument in RESHAPE
+  real :: array12(2,3) = RESHAPE([(n, n=1,4)], [2,3], [99], [1])
+  !ERROR: Invalid 'order=' argument in RESHAPE
+  real :: array13(2,3) = RESHAPE([(n, n = 1, 4)], [2, 3], [99], [1, 1])
+
+  ! Examples that have caused problems
+  integer :: array14(0,0,0) = RESHAPE([(n,n=1,0)],[0,0,0])
+  integer, parameter :: array15(1) = RESHAPE([(n,n=1,2)],[1])
+  integer, parameter :: array16(1) = RESHAPE([(n,n=1,8)],[1], [0], array15)
+  integer, parameter, dimension(3,4) :: array17 = 3
+  integer, parameter, dimension(3,4) :: array18 = RESHAPE(array17, [3,4])
+  ! Implicit reshape of array of components
+  type :: dType
+    integer :: field(2)
+  end type dType
+  type(dType), parameter :: array19(*) = [dType::dType(field=[1,2])]
+  logical, parameter :: lVar = all(array19(:)%field(1) == [2])
+
+  !ERROR: Size of 'shape=' argument must not be greater than 15
+  CALL ext_sub(RESHAPE([(n, n=1,20)], &
+    [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]))
+  !ERROR: 'shape=' argument must not have a negative extent
+  CALL ext_sub(RESHAPE([(n, n=1,20)], [1, -5, 3]))
+end program reshaper

From b36c39260edcded47436f344e48f78cfbedac494 Mon Sep 17 00:00:00 2001
From: Arthur Eubanks <aeubanks@google.com>
Date: Thu, 30 Jul 2020 13:49:41 -0700
Subject: [PATCH 0699/1035] [NewPM] Don't print 'Invalidating all non-preserved
 analyses'

If an analysis is actually invalidated, there's already a log statement
for that: 'Invalidating analysis: FooAnalysis'.
Otherwise the statement is not very useful.

Reviewed By: asbirlea, ychen

Differential Revision: https://reviews.llvm.org/D84981
---
 llvm/include/llvm/IR/PassManagerImpl.h                 |  4 ----
 llvm/test/Other/loop-pm-invalidation.ll                | 10 ----------
 llvm/test/Other/new-pass-manager.ll                    |  3 ---
 llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll |  1 -
 .../Transforms/Inline/cgscc-incremental-invalidate.ll  |  5 -----
 llvm/test/Transforms/LoopRotate/pr35210.ll             |  4 ----
 .../Transforms/LoopUnroll/unroll-loop-invalidation.ll  |  3 ---
 .../LoopVectorize/novect-lcssa-cfg-invalidation.ll     |  1 -
 llvm/test/Transforms/SCCP/ipsccp-preserve-analysis.ll  |  3 ---
 llvm/test/Transforms/SROA/dead-inst.ll                 |  1 -
 10 files changed, 35 deletions(-)

diff --git a/llvm/include/llvm/IR/PassManagerImpl.h b/llvm/include/llvm/IR/PassManagerImpl.h
index 80831b31c6b07..71a86d1efb157 100644
--- a/llvm/include/llvm/IR/PassManagerImpl.h
+++ b/llvm/include/llvm/IR/PassManagerImpl.h
@@ -94,10 +94,6 @@ inline void AnalysisManager<IRUnitT, ExtraArgTs...>::invalidate(
   if (PA.allAnalysesInSetPreserved<AllAnalysesOn<IRUnitT>>())
     return;
 
-  if (DebugLogging)
-    dbgs() << "Invalidating all non-preserved analyses for: " << IR.getName()
-           << "\n";
-
   // Track whether each analysis's result is invalidated in
   // IsResultInvalidated.
   SmallDenseMap<AnalysisKey *, bool, 8> IsResultInvalidated;
diff --git a/llvm/test/Other/loop-pm-invalidation.ll b/llvm/test/Other/loop-pm-invalidation.ll
index 3fef7a4a5082c..9deb17f124d6b 100644
--- a/llvm/test/Other/loop-pm-invalidation.ll
+++ b/llvm/test/Other/loop-pm-invalidation.ll
@@ -27,7 +27,6 @@ define void @no_loops() {
 ; CHECK-LOOP-INV-NEXT: Running pass: LCSSAPass
 ; CHECK-LOOP-INV-NEXT: Finished {{.*}}Function pass manager run
 ; CHECK-LOOP-INV-NEXT: Running pass: InvalidateAnalysisPass<{{.*}}LoopAnalysis
-; CHECK-LOOP-INV-NEXT: Invalidating all non-preserved analyses
 ; CHECK-LOOP-INV-NEXT: Invalidating analysis: LoopAnalysis
 ; CHECK-LOOP-INV-NEXT: Starting {{.*}}Function pass manager run
 ; CHECK-LOOP-INV-NEXT: Running pass: LoopSimplifyPass
@@ -45,7 +44,6 @@ define void @no_loops() {
 ; CHECK-SCEV-INV-NEXT: Running pass: LCSSAPass
 ; CHECK-SCEV-INV-NEXT: Finished {{.*}}Function pass manager run
 ; CHECK-SCEV-INV-NEXT: Running pass: InvalidateAnalysisPass<{{.*}}ScalarEvolutionAnalysis
-; CHECK-SCEV-INV-NEXT: Invalidating all non-preserved analyses
 ; CHECK-SCEV-INV-NEXT: Starting {{.*}}Function pass manager run
 ; CHECK-SCEV-INV-NEXT: Running pass: LoopSimplifyPass
 ; CHECK-SCEV-INV-NEXT: Running pass: LCSSAPass
@@ -74,7 +72,6 @@ define void @one_loop(i1* %ptr) {
 ; CHECK-LOOP-INV-NEXT: Running pass: NoOpLoopPass
 ; CHECK-LOOP-INV-NEXT: Finished {{.*}}Loop pass manager run.
 ; CHECK-LOOP-INV-NEXT: Running pass: InvalidateAnalysisPass<{{.*}}LoopAnalysis
-; CHECK-LOOP-INV-NEXT: Invalidating all non-preserved analyses
 ; CHECK-LOOP-INV-NEXT: Clearing all analysis results for: <possibly invalidated loop>
 ; CHECK-LOOP-INV-NEXT: Invalidating analysis: LoopAnalysis
 ; CHECK-LOOP-INV-NEXT: Invalidating analysis: ScalarEvolutionAnalysis
@@ -108,7 +105,6 @@ define void @one_loop(i1* %ptr) {
 ; CHECK-SCEV-INV-NEXT: Running pass: NoOpLoopPass
 ; CHECK-SCEV-INV-NEXT: Finished {{.*}}Loop pass manager run.
 ; CHECK-SCEV-INV-NEXT: Running pass: InvalidateAnalysisPass<{{.*}}ScalarEvolutionAnalysis
-; CHECK-SCEV-INV-NEXT: Invalidating all non-preserved analyses
 ; CHECK-SCEV-INV-NEXT: Clearing all analysis results for: <possibly invalidated loop>
 ; CHECK-SCEV-INV-NEXT: Invalidating analysis: ScalarEvolutionAnalysis
 ; CHECK-SCEV-INV-NEXT: Invalidating analysis: InnerAnalysisManagerProxy<{{.*}}Loop
@@ -155,7 +151,6 @@ define void @nested_loops(i1* %ptr) {
 ; CHECK-LOOP-INV-NEXT: Running pass: NoOpLoopPass
 ; CHECK-LOOP-INV: Finished {{.*}}Loop pass manager run.
 ; CHECK-LOOP-INV-NEXT: Running pass: InvalidateAnalysisPass<{{.*}}LoopAnalysis
-; CHECK-LOOP-INV-NEXT: Invalidating all non-preserved analyses
 ; CHECK-LOOP-INV-NEXT: Clearing all analysis results for: <possibly invalidated loop>
 ; CHECK-LOOP-INV-NEXT: Clearing all analysis results for: <possibly invalidated loop>
 ; CHECK-LOOP-INV-NEXT: Invalidating analysis: LoopAnalysis
@@ -196,7 +191,6 @@ define void @nested_loops(i1* %ptr) {
 ; CHECK-SCEV-INV-NEXT: Running pass: NoOpLoopPass
 ; CHECK-SCEV-INV: Finished {{.*}}Loop pass manager run.
 ; CHECK-SCEV-INV-NEXT: Running pass: InvalidateAnalysisPass<{{.*}}ScalarEvolutionAnalysis
-; CHECK-SCEV-INV-NEXT: Invalidating all non-preserved analyses
 ; CHECK-SCEV-INV-NEXT: Clearing all analysis results for: <possibly invalidated loop>
 ; CHECK-SCEV-INV-NEXT: Clearing all analysis results for: <possibly invalidated loop>
 ; CHECK-SCEV-INV-NEXT: Invalidating analysis: ScalarEvolutionAnalysis
@@ -251,7 +245,6 @@ define void @dead_loop() {
 ; CHECK-LOOP-INV-NEXT: Running pass: NoOpLoopPass
 ; CHECK-LOOP-INV-NEXT: Finished {{.*}}Loop pass manager run.
 ; CHECK-LOOP-INV-NEXT: Running pass: InvalidateAnalysisPass<{{.*}}LoopAnalysis
-; CHECK-LOOP-INV-NEXT: Invalidating all non-preserved analyses
 ; CHECK-LOOP-INV-NEXT: Clearing all analysis results for: <possibly invalidated loop>
 ; CHECK-LOOP-INV-NEXT: Invalidating analysis: LoopAnalysis
 ; CHECK-LOOP-INV-NEXT: Invalidating analysis: ScalarEvolutionAnalysis
@@ -285,7 +278,6 @@ define void @dead_loop() {
 ; CHECK-SCEV-INV-NEXT: Running pass: NoOpLoopPass
 ; CHECK-SCEV-INV-NEXT: Finished {{.*}}Loop pass manager run.
 ; CHECK-SCEV-INV-NEXT: Running pass: InvalidateAnalysisPass<{{.*}}ScalarEvolutionAnalysis
-; CHECK-SCEV-INV-NEXT: Invalidating all non-preserved analyses
 ; CHECK-SCEV-INV-NEXT: Clearing all analysis results for: <possibly invalidated loop>
 ; CHECK-SCEV-INV-NEXT: Invalidating analysis: ScalarEvolutionAnalysis
 ; CHECK-SCEV-INV-NEXT: Invalidating analysis: InnerAnalysisManagerProxy<{{.*}}Loop
@@ -316,9 +308,7 @@ define void @dead_loop() {
 ; CHECK-SCEV-INV-AFTER-DELETE-NEXT: Running pass: LoopDeletionPass
 ; CHECK-SCEV-INV-AFTER-DELETE-NEXT: Clearing all analysis results for:
 ; CHECK-SCEV-INV-AFTER-DELETE-NEXT: Finished {{.*}}Loop pass manager run.
-; CHECK-SCEV-INV-AFTER-DELETE-NEXT: Invalidating all non-preserved analyses
 ; CHECK-SCEV-INV-AFTER-DELETE-NEXT: Running pass: InvalidateAnalysisPass<{{.*}}ScalarEvolutionAnalysis
-; CHECK-SCEV-INV-AFTER-DELETE-NEXT: Invalidating all non-preserved analyses
 ; CHECK-SCEV-INV-AFTER-DELETE-NEXT: Invalidating analysis: ScalarEvolutionAnalysis
 ; CHECK-SCEV-INV-AFTER-DELETE-NEXT: Invalidating analysis: InnerAnalysisManagerProxy<{{.*}}Loop
 ; CHECK-SCEV-INV-AFTER-DELETE-NEXT: Starting {{.*}}Function pass manager run
diff --git a/llvm/test/Other/new-pass-manager.ll b/llvm/test/Other/new-pass-manager.ll
index 1c9477dc9da22..31be3adb68978 100644
--- a/llvm/test/Other/new-pass-manager.ll
+++ b/llvm/test/Other/new-pass-manager.ll
@@ -149,7 +149,6 @@
 ; RUN:     -passes='require<no-op-module>,cgscc(require<no-op-cgscc>,function(require<no-op-function>))' %s 2>&1 \
 ; RUN:     | FileCheck %s --check-prefix=CHECK-NO-OP-INVALIDATION
 ; CHECK-NO-OP-INVALIDATION: Starting llvm::Module pass manager run
-; CHECK-NO-OP-INVALIDATION-NOT: Invalidating all non-preserved analyses
 
 ; RUN: opt -disable-output -debug-pass-manager \
 ; RUN:     -passes='require<no-op-module>,require<no-op-module>,require<no-op-module>' %s 2>&1 \
@@ -450,7 +449,6 @@
 ; CHECK-REPEAT-LOOP-PASS-NEXT: Running analysis: LoopAnalysis
 ; CHECK-REPEAT-LOOP-PASS-NEXT: Running analysis: DominatorTreeAnalysis
 ; CHECK-REPEAT-LOOP-PASS-NEXT: Running analysis: AssumptionAnalysis
-; CHECK-REPEAT-LOOP-PASS-NEXT: Invalidating all non-preserved analyses
 ; CHECK-REPEAT-LOOP-PASS-NEXT: Running pass: LCSSAPass
 ; CHECK-REPEAT-LOOP-PASS-NEXT: Finished llvm::Function pass manager run
 ; CHECK-REPEAT-LOOP-PASS-NEXT: Running analysis: AAManager
@@ -471,7 +469,6 @@
 ; CHECK-REPEAT-LOOP-PASS-NEXT: Finished Loop pass manager run
 ; CHECK-REPEAT-LOOP-PASS-NEXT: Finished Loop pass manager run
 ; CHECK-REPEAT-LOOP-PASS-NEXT: Finished llvm::Function pass manager run
-; CHECK-REPEAT-LOOP-PASS-NEXT: Invalidating all non-preserved analyses
 ; CHECK-REPEAT-LOOP-PASS-NEXT: Finished llvm::Module pass manager run
 
 define void @foo(i1 %x, i8* %p1, i8* %p2) {
diff --git a/llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll b/llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll
index e5791095201b8..f94b0992431fd 100644
--- a/llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll
+++ b/llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll
@@ -82,7 +82,6 @@
 ; CHECK-O-DAG: Running analysis: PostDominatorTreeAnalysis on foo
 ; CHECK-O-DAG: Running analysis: LoopAnalysis on foo
 ; CHECK-O-NEXT: Running analysis: BlockFrequencyAnalysis on foo
-; CHECK-O-NEXT: Invalidating all non-preserved analyses for:
 ; CHECK-O-NEXT: Invalidating analysis: InnerAnalysisManagerProxy
 ; CHECK-O123-NEXT: Invalidating analysis: LazyCallGraphAnalysis on
 ; CHECK-O123-NEXT: Invalidating analysis: InnerAnalysisManagerProxy
diff --git a/llvm/test/Transforms/Inline/cgscc-incremental-invalidate.ll b/llvm/test/Transforms/Inline/cgscc-incremental-invalidate.ll
index 5a429bc3a4f02..311de67ad507f 100644
--- a/llvm/test/Transforms/Inline/cgscc-incremental-invalidate.ll
+++ b/llvm/test/Transforms/Inline/cgscc-incremental-invalidate.ll
@@ -10,19 +10,14 @@
 ; CHECK: Running pass: InlinerPass on (test1_f, test1_g, test1_h)
 ; CHECK: Running analysis: DominatorTreeAnalysis on test1_f
 ; CHECK: Running analysis: DominatorTreeAnalysis on test1_g
-; CHECK: Invalidating all non-preserved analyses for: (test1_f)
-; CHECK: Invalidating all non-preserved analyses for: test1_f
 ; CHECK: Invalidating analysis: DominatorTreeAnalysis on test1_f
 ; CHECK: Invalidating analysis: LoopAnalysis on test1_f
 ; CHECK: Invalidating analysis: BranchProbabilityAnalysis on test1_f
 ; CHECK: Invalidating analysis: BlockFrequencyAnalysis on test1_f
-; CHECK: Invalidating all non-preserved analyses for: (test1_g, test1_h)
-; CHECK: Invalidating all non-preserved analyses for: test1_g
 ; CHECK: Invalidating analysis: DominatorTreeAnalysis on test1_g
 ; CHECK: Invalidating analysis: LoopAnalysis on test1_g
 ; CHECK: Invalidating analysis: BranchProbabilityAnalysis on test1_g
 ; CHECK: Invalidating analysis: BlockFrequencyAnalysis on test1_g
-; CHECK: Invalidating all non-preserved analyses for: test1_h
 ; CHECK: Invalidating analysis: DominatorTreeAnalysis on test1_h
 ; CHECK: Invalidating analysis: LoopAnalysis on test1_h
 ; CHECK: Invalidating analysis: BranchProbabilityAnalysis on test1_h
diff --git a/llvm/test/Transforms/LoopRotate/pr35210.ll b/llvm/test/Transforms/LoopRotate/pr35210.ll
index a615e53aec2b9..15b11b98b290b 100644
--- a/llvm/test/Transforms/LoopRotate/pr35210.ll
+++ b/llvm/test/Transforms/LoopRotate/pr35210.ll
@@ -23,9 +23,7 @@
 ; CHECK-NEXT: Starting Loop pass manager run.
 ; CHECK-NEXT: Running pass: LoopRotatePass on Loop at depth 1 containing: %bb<header><exiting>,%bb4<latch>
 ; CHECK-NEXT: Folding loop latch bb4 into bb
-; CHECK-NEXT: Invalidating all non-preserved analyses for: bb
 ; CHECK-NEXT: Finished Loop pass manager run.
-; CHECK-NEXT: Invalidating all non-preserved analyses for: f
 ; CHECK-NEXT: Invalidating analysis: PostDominatorTreeAnalysis on f
 ; CHECK-NEXT: Running pass: ADCEPass on f
 ; CHECK-NEXT: Running analysis: PostDominatorTreeAnalysis on f
@@ -50,9 +48,7 @@
 ; MSSA-NEXT: Starting Loop pass manager run.
 ; MSSA-NEXT: Running pass: LoopRotatePass on Loop at depth 1 containing: %bb<header><exiting>,%bb4<latch>
 ; MSSA-NEXT: Folding loop latch bb4 into bb
-; MSSA-NEXT: Invalidating all non-preserved analyses for: bb
 ; MSSA-NEXT: Finished Loop pass manager run.
-; MSSA-NEXT: Invalidating all non-preserved analyses for: f
 ; MSSA-NEXT: Invalidating analysis: PostDominatorTreeAnalysis on f
 ; MSSA-NEXT: Running pass: ADCEPass on f
 ; MSSA-NEXT: Running analysis: PostDominatorTreeAnalysis on f
diff --git a/llvm/test/Transforms/LoopUnroll/unroll-loop-invalidation.ll b/llvm/test/Transforms/LoopUnroll/unroll-loop-invalidation.ll
index 8a3e6e17f505d..3f30bbfc80220 100644
--- a/llvm/test/Transforms/LoopUnroll/unroll-loop-invalidation.ll
+++ b/llvm/test/Transforms/LoopUnroll/unroll-loop-invalidation.ll
@@ -23,10 +23,7 @@
 ; CHECK: Running pass: LoopUnrollPass
 ; CHECK: Clearing all analysis results for: inner2.header
 ; CHECK: Clearing all analysis results for: outer.header
-; CHECK: Invalidating all non-preserved analyses for: test
-; CHECK: Invalidating all non-preserved analyses for: inner1.header
 ; CHECK: Invalidating analysis: LoopAccessAnalysis on inner1.header
-; CHECK: Invalidating all non-preserved analyses for: inner1.header.1
 ; CHECK-NOT: Invalidating analysis: LoopAccessAnalysis on inner1.header.1
 ; CHECK: Starting Loop pass manager run.
 ; CHECK: Running pass: LoopAccessInfoPrinterPass
diff --git a/llvm/test/Transforms/LoopVectorize/novect-lcssa-cfg-invalidation.ll b/llvm/test/Transforms/LoopVectorize/novect-lcssa-cfg-invalidation.ll
index 05b36f8c550bf..12e1b5e7ff2af 100644
--- a/llvm/test/Transforms/LoopVectorize/novect-lcssa-cfg-invalidation.ll
+++ b/llvm/test/Transforms/LoopVectorize/novect-lcssa-cfg-invalidation.ll
@@ -8,7 +8,6 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 define i32 @novect(i32* %p) {
 
 ; CHECK:           Running pass: LoopVectorizePass on novect
-; CHECK:           Invalidating all non-preserved analyses for: novect
 ; CHECK:           Clearing all analysis results for: <possibly invalidated loop>
 ; CHECK:           Invalidating analysis: ScalarEvolutionAnalysis on novect
 ; CHECK-NOT:       Invalidating analysis: BranchProbabilityAnalysis on novect
diff --git a/llvm/test/Transforms/SCCP/ipsccp-preserve-analysis.ll b/llvm/test/Transforms/SCCP/ipsccp-preserve-analysis.ll
index 78c02d9b8a42c..0bc0754dfce80 100644
--- a/llvm/test/Transforms/SCCP/ipsccp-preserve-analysis.ll
+++ b/llvm/test/Transforms/SCCP/ipsccp-preserve-analysis.ll
@@ -16,9 +16,6 @@
 ; NEW-PM: Running pass: IPSCCPPass
 ; NEW-PM-DAG: Running analysis: AssumptionAnalysis on f1
 ; NEW-PM-DAG: Running analysis: AssumptionAnalysis on f2
-; NEW-PM-NEXT: Invalidating all non-preserved analyses for:
-; NEW-PM-NEXT: Invalidating all non-preserved analyses for: f1
-; NEW-PM-NEXT: Invalidating all non-preserved analyses for: f2
 ; NEW-PM-NOT: Running analysis:
 
 ; IR-LABEL: @f1
diff --git a/llvm/test/Transforms/SROA/dead-inst.ll b/llvm/test/Transforms/SROA/dead-inst.ll
index b1d106df3e683..043d8509f3f42 100644
--- a/llvm/test/Transforms/SROA/dead-inst.ll
+++ b/llvm/test/Transforms/SROA/dead-inst.ll
@@ -5,7 +5,6 @@
 ; CHECK: Running pass: BDCEPass on H
 ; CHECK: Running analysis: DemandedBitsAnalysis on H
 ; CHECK: Running pass: SROA on H
-; CHECK: Invalidating all non-preserved analyses for: H
 ; CHECK: Invalidating analysis: DemandedBitsAnalysis on H
 ; CHECK: Running pass: BDCEPass on H
 ; CHECK: Running analysis: DemandedBitsAnalysis on H

From 47acbcf09aa60ff13845740b1c02c7ca017e46d0 Mon Sep 17 00:00:00 2001
From: Arthur Eubanks <aeubanks@google.com>
Date: Wed, 29 Jul 2020 16:44:22 -0700
Subject: [PATCH 0700/1035] [tbaa] Rename type-based-aa -> tbaa

For consistency with legacy pass name.
Helps with 37 instances of "unknown pass name 'tbaa'" in check-llvm under NPM.

Reviewed By: ychen

Differential Revision: https://reviews.llvm.org/D84967
---
 llvm/lib/Passes/PassRegistry.def                            | 2 +-
 llvm/test/Analysis/DependenceAnalysis/AA.ll                 | 2 +-
 llvm/test/Analysis/TypeBasedAliasAnalysis/placement-tbaa.ll | 2 +-
 llvm/test/Transforms/LICM/promote-order.ll                  | 4 ++--
 llvm/test/Transforms/LICM/promote-tls.ll                    | 2 +-
 llvm/test/Transforms/LICM/scalar-promote-memmodel.ll        | 2 +-
 llvm/test/Transforms/LICM/scalar-promote.ll                 | 2 +-
 llvm/test/Transforms/LoopUnrollAndJam/pragma-explicit.ll    | 2 +-
 llvm/test/Transforms/LoopUnrollAndJam/unroll-and-jam.ll     | 2 +-
 9 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def
index 38a6575b831a5..e77182031f108 100644
--- a/llvm/lib/Passes/PassRegistry.def
+++ b/llvm/lib/Passes/PassRegistry.def
@@ -158,7 +158,7 @@ FUNCTION_ALIAS_ANALYSIS("cfl-anders-aa", CFLAndersAA())
 FUNCTION_ALIAS_ANALYSIS("cfl-steens-aa", CFLSteensAA())
 FUNCTION_ALIAS_ANALYSIS("scev-aa", SCEVAA())
 FUNCTION_ALIAS_ANALYSIS("scoped-noalias-aa", ScopedNoAliasAA())
-FUNCTION_ALIAS_ANALYSIS("type-based-aa", TypeBasedAA())
+FUNCTION_ALIAS_ANALYSIS("tbaa", TypeBasedAA())
 #undef FUNCTION_ALIAS_ANALYSIS
 #undef FUNCTION_ANALYSIS
 
diff --git a/llvm/test/Analysis/DependenceAnalysis/AA.ll b/llvm/test/Analysis/DependenceAnalysis/AA.ll
index ac04378b8a85d..efb5c8d1ef031 100644
--- a/llvm/test/Analysis/DependenceAnalysis/AA.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/AA.ll
@@ -1,5 +1,5 @@
 ; RUN: opt < %s -disable-output "-passes=print<da>"                            \
-; RUN: "-aa-pipeline=basic-aa,type-based-aa" 2>&1 | FileCheck %s
+; RUN: "-aa-pipeline=basic-aa,tbaa" 2>&1 | FileCheck %s
 ; RUN: opt < %s -analyze -basic-aa -tbaa -da | FileCheck %s
 
 ; CHECK-LABEL: 'Dependence Analysis' for function 'test_no_noalias'
diff --git a/llvm/test/Analysis/TypeBasedAliasAnalysis/placement-tbaa.ll b/llvm/test/Analysis/TypeBasedAliasAnalysis/placement-tbaa.ll
index 16cb3b741307f..f25024db4c013 100644
--- a/llvm/test/Analysis/TypeBasedAliasAnalysis/placement-tbaa.ll
+++ b/llvm/test/Analysis/TypeBasedAliasAnalysis/placement-tbaa.ll
@@ -1,5 +1,5 @@
 ; RUN: opt < %s -tbaa -basic-aa -aa-eval -evaluate-aa-metadata -print-no-aliases -print-may-aliases -disable-output 2>&1 | FileCheck %s
-; RUN: opt < %s -aa-pipeline=type-based-aa,basic-aa -passes=aa-eval -evaluate-aa-metadata -print-no-aliases -print-may-aliases -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -aa-pipeline=tbaa,basic-aa -passes=aa-eval -evaluate-aa-metadata -print-no-aliases -print-may-aliases -disable-output 2>&1 | FileCheck %s
 
 ; Generated with "clang -cc1 -disable-llvm-optzns -O1 -emit-llvm"
 ; #include <new>
diff --git a/llvm/test/Transforms/LICM/promote-order.ll b/llvm/test/Transforms/LICM/promote-order.ll
index 1a46f95386a83..ccb642b295a49 100644
--- a/llvm/test/Transforms/LICM/promote-order.ll
+++ b/llvm/test/Transforms/LICM/promote-order.ll
@@ -1,7 +1,7 @@
 ; RUN: opt -tbaa -basic-aa -licm -enable-mssa-loop-dependency=false -S < %s | FileCheck %s --check-prefixes=CHECK,AST
 ; RUN: opt -tbaa -basic-aa -licm -enable-mssa-loop-dependency=true -S < %s | FileCheck %s --check-prefixes=CHECK,MSSA
-; RUN: opt -aa-pipeline=type-based-aa,basic-aa -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' -S %s | FileCheck %s  --check-prefixes=CHECK,AST
-; RUN: opt -aa-pipeline=type-based-aa,basic-aa -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop-mssa(licm)' -S %s | FileCheck %s --check-prefixes=CHECK,MSSA
+; RUN: opt -aa-pipeline=tbaa,basic-aa -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' -S %s | FileCheck %s  --check-prefixes=CHECK,AST
+; RUN: opt -aa-pipeline=tbaa,basic-aa -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop-mssa(licm)' -S %s | FileCheck %s --check-prefixes=CHECK,MSSA
 
 ; LICM should keep the stores in their original order when it sinks/promotes them.
 ; rdar://12045203
diff --git a/llvm/test/Transforms/LICM/promote-tls.ll b/llvm/test/Transforms/LICM/promote-tls.ll
index ccd73733f62b4..7438162bff679 100644
--- a/llvm/test/Transforms/LICM/promote-tls.ll
+++ b/llvm/test/Transforms/LICM/promote-tls.ll
@@ -1,5 +1,5 @@
 ; RUN: opt -tbaa -basic-aa -licm -S < %s | FileCheck %s
-; RUN: opt -aa-pipeline=type-based-aa,basic-aa -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' -S %s | FileCheck %s
+; RUN: opt -aa-pipeline=tbaa,basic-aa -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' -S %s | FileCheck %s
 
 ; If we can prove a local is thread local, we can insert stores during
 ; promotion which wouldn't be legal otherwise.
diff --git a/llvm/test/Transforms/LICM/scalar-promote-memmodel.ll b/llvm/test/Transforms/LICM/scalar-promote-memmodel.ll
index 92724cf97330a..6f3fc9d152e33 100644
--- a/llvm/test/Transforms/LICM/scalar-promote-memmodel.ll
+++ b/llvm/test/Transforms/LICM/scalar-promote-memmodel.ll
@@ -1,5 +1,5 @@
 ; RUN: opt < %s -basic-aa -licm -S | FileCheck %s
-; RUN: opt -aa-pipeline=type-based-aa,basic-aa -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' -S %s | FileCheck %s
+; RUN: opt -aa-pipeline=tbaa,basic-aa -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' -S %s | FileCheck %s
 
 ; Make sure we don't hoist a conditionally-executed store out of the loop;
 ; it would violate the concurrency memory model
diff --git a/llvm/test/Transforms/LICM/scalar-promote.ll b/llvm/test/Transforms/LICM/scalar-promote.ll
index c0fc153e8722a..c6b600e1ca693 100644
--- a/llvm/test/Transforms/LICM/scalar-promote.ll
+++ b/llvm/test/Transforms/LICM/scalar-promote.ll
@@ -1,5 +1,5 @@
 ; RUN: opt < %s -basic-aa -tbaa -licm -S | FileCheck %s
-; RUN: opt -aa-pipeline=type-based-aa,basic-aa -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' -S %s | FileCheck %s
+; RUN: opt -aa-pipeline=tbaa,basic-aa -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' -S %s | FileCheck %s
 target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
 
 @X = global i32 7   ; <i32*> [#uses=4]
diff --git a/llvm/test/Transforms/LoopUnrollAndJam/pragma-explicit.ll b/llvm/test/Transforms/LoopUnrollAndJam/pragma-explicit.ll
index 91843dbd71c02..b607221a052d3 100644
--- a/llvm/test/Transforms/LoopUnrollAndJam/pragma-explicit.ll
+++ b/llvm/test/Transforms/LoopUnrollAndJam/pragma-explicit.ll
@@ -1,5 +1,5 @@
 ; RUN: opt -loop-unroll-and-jam -allow-unroll-and-jam -unroll-runtime -unroll-partial-threshold=60 < %s -S | FileCheck %s
-; RUN: opt -aa-pipeline=type-based-aa,basic-aa -passes='loop-unroll-and-jam' -allow-unroll-and-jam -unroll-runtime -unroll-partial-threshold=60 < %s -S | FileCheck %s
+; RUN: opt -aa-pipeline=tbaa,basic-aa -passes='loop-unroll-and-jam' -allow-unroll-and-jam -unroll-runtime -unroll-partial-threshold=60 < %s -S | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/Transforms/LoopUnrollAndJam/unroll-and-jam.ll b/llvm/test/Transforms/LoopUnrollAndJam/unroll-and-jam.ll
index 82f7bdf8ffce2..5086b2019ced0 100644
--- a/llvm/test/Transforms/LoopUnrollAndJam/unroll-and-jam.ll
+++ b/llvm/test/Transforms/LoopUnrollAndJam/unroll-and-jam.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -basic-aa -tbaa -loop-unroll-and-jam -allow-unroll-and-jam -unroll-and-jam-count=4 -unroll-remainder < %s -S | FileCheck %s
-; RUN: opt -aa-pipeline=type-based-aa,basic-aa -passes='loop-unroll-and-jam' -allow-unroll-and-jam -unroll-and-jam-count=4 -unroll-remainder < %s -S | FileCheck %s
+; RUN: opt -aa-pipeline=tbaa,basic-aa -passes='loop-unroll-and-jam' -allow-unroll-and-jam -unroll-and-jam-count=4 -unroll-remainder < %s -S | FileCheck %s
 
 target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
 

From 916d9e18774a10c837e50bb4f1b207e80f0db6e3 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@intel.com>
Date: Thu, 30 Jul 2020 19:42:57 -0700
Subject: [PATCH 0701/1035] [X86] Pass the OperandVector by reference to
 ParseIntelOperand and ParseRoundingMode. NFCI

Similar to what was recently done to ParseATTOperand. Make
ParseIntelOperand directly responsible for adding to the operand
vector instead of returning the operand. Return a bool for error.

Remove ErrorOperand since it is no longer used.
---
 .../lib/Target/X86/AsmParser/X86AsmParser.cpp | 145 +++++++++---------
 1 file changed, 75 insertions(+), 70 deletions(-)

diff --git a/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp b/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
index f52735068c8bb..6b4f2e33f67de 100644
--- a/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
+++ b/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
@@ -886,11 +886,6 @@ class X86AsmParser : public MCTargetAsmParser {
     return Parser.Error(L, Msg, Range);
   }
 
-  std::nullptr_t ErrorOperand(SMLoc Loc, StringRef Msg, SMRange R = SMRange()) {
-    Error(Loc, Msg, R);
-    return nullptr;
-  }
-
   bool MatchRegisterByName(unsigned &RegNo, StringRef RegName, SMLoc StartLoc,
                            SMLoc EndLoc);
   bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc,
@@ -908,13 +903,13 @@ class X86AsmParser : public MCTargetAsmParser {
                                OperandVector &FinalOperands);
   bool ParseOperand(OperandVector &Operands);
   bool ParseATTOperand(OperandVector &Operands);
-  std::unique_ptr<X86Operand> ParseIntelOperand();
+  bool ParseIntelOperand(OperandVector &Operands);
   bool ParseIntelOffsetOperator(const MCExpr *&Val, StringRef &ID,
                                 InlineAsmIdentifierInfo &Info, SMLoc &End);
   bool ParseIntelDotOperator(IntelExprStateMachine &SM, SMLoc &End);
   unsigned IdentifyIntelInlineAsmOperator(StringRef Name);
   unsigned ParseIntelInlineAsmOperator(unsigned OpKind);
-  std::unique_ptr<X86Operand> ParseRoundingModeOp(SMLoc Start);
+  bool ParseRoundingModeOp(SMLoc Start, OperandVector &Operands);
   bool ParseIntelNamedOperator(StringRef Name, IntelExprStateMachine &SM,
                                bool &ParseError, SMLoc &End);
   void RewriteIntelExpression(IntelExprStateMachine &SM, SMLoc Start,
@@ -931,11 +926,12 @@ class X86AsmParser : public MCTargetAsmParser {
   X86::CondCode ParseConditionCode(StringRef CCode);
 
   bool ParseIntelMemoryOperandSize(unsigned &Size);
-  std::unique_ptr<X86Operand>
-  CreateMemForMSInlineAsm(unsigned SegReg, const MCExpr *Disp, unsigned BaseReg,
-                          unsigned IndexReg, unsigned Scale, SMLoc Start,
-                          SMLoc End, unsigned Size, StringRef Identifier,
-                          const InlineAsmIdentifierInfo &Info);
+  bool CreateMemForMSInlineAsm(unsigned SegReg, const MCExpr *Disp,
+                               unsigned BaseReg, unsigned IndexReg,
+                               unsigned Scale, SMLoc Start, SMLoc End,
+                               unsigned Size, StringRef Identifier,
+                               const InlineAsmIdentifierInfo &Info,
+                               OperandVector &Operands);
 
   bool parseDirectiveArch();
   bool parseDirectiveEven(SMLoc L);
@@ -1531,21 +1527,16 @@ bool X86AsmParser::VerifyAndAdjustOperands(OperandVector &OrigOperands,
 }
 
 bool X86AsmParser::ParseOperand(OperandVector &Operands) {
-  if (isParsingIntelSyntax()) {
-    if (std::unique_ptr<X86Operand> Op = ParseIntelOperand()) {
-      Operands.push_back(std::move(Op));
-      return false;
-    }
-    return true;
-  }
+  if (isParsingIntelSyntax())
+    return ParseIntelOperand(Operands);
 
   return ParseATTOperand(Operands);
 }
 
-std::unique_ptr<X86Operand> X86AsmParser::CreateMemForMSInlineAsm(
+bool X86AsmParser::CreateMemForMSInlineAsm(
     unsigned SegReg, const MCExpr *Disp, unsigned BaseReg, unsigned IndexReg,
     unsigned Scale, SMLoc Start, SMLoc End, unsigned Size, StringRef Identifier,
-    const InlineAsmIdentifierInfo &Info) {
+    const InlineAsmIdentifierInfo &Info, OperandVector &Operands) {
   // If we found a decl other than a VarDecl, then assume it is a FuncDecl or
   // some other label reference.
   if (Info.isKind(InlineAsmIdentifierInfo::IK_Label)) {
@@ -1557,8 +1548,10 @@ std::unique_ptr<X86Operand> X86AsmParser::CreateMemForMSInlineAsm(
     }
     // Create an absolute memory reference in order to match against
     // instructions taking a PC relative operand.
-    return X86Operand::CreateMem(getPointerWidth(), Disp, Start, End, Size,
-                                 Identifier, Info.Label.Decl);
+    Operands.push_back(X86Operand::CreateMem(getPointerWidth(), Disp, Start,
+                                             End, Size, Identifier,
+                                             Info.Label.Decl));
+    return false;
   }
   // We either have a direct symbol reference, or an offset from a symbol.  The
   // parser always puts the symbol on the LHS, so look there for size
@@ -1575,17 +1568,19 @@ std::unique_ptr<X86Operand> X86AsmParser::CreateMemForMSInlineAsm(
   // It is widely common for MS InlineAsm to use a global variable and one/two
   // registers in a mmory expression, and though unaccessible via rip/eip.
   if (IsGlobalLV && (BaseReg || IndexReg)) {
-    return X86Operand::CreateMem(getPointerWidth(), Disp, Start, End);
+    Operands.push_back(
+        X86Operand::CreateMem(getPointerWidth(), Disp, Start, End));
+    return false;
+  }
   // Otherwise, we set the base register to a non-zero value
   // if we don't know the actual value at this time.  This is necessary to
   // get the matching correct in some cases.
-  } else {
-    BaseReg = BaseReg ? BaseReg : 1;
-    return X86Operand::CreateMem(getPointerWidth(), SegReg, Disp, BaseReg,
-                                 IndexReg, Scale, Start, End, Size,
-                                 /*DefaultBaseReg=*/X86::RIP, Identifier, Decl,
-                                 FrontendSize);
-  }
+  BaseReg = BaseReg ? BaseReg : 1;
+  Operands.push_back(X86Operand::CreateMem(
+      getPointerWidth(), SegReg, Disp, BaseReg, IndexReg, Scale, Start, End,
+      Size,
+      /*DefaultBaseReg=*/X86::RIP, Identifier, Decl, FrontendSize));
+  return false;
 }
 
 // Some binary bitwise operators have a named synonymous
@@ -1916,14 +1911,13 @@ bool X86AsmParser::ParseIntelInlineAsmIdentifier(
 }
 
 //ParseRoundingModeOp - Parse AVX-512 rounding mode operand
-std::unique_ptr<X86Operand>
-X86AsmParser::ParseRoundingModeOp(SMLoc Start) {
+bool X86AsmParser::ParseRoundingModeOp(SMLoc Start, OperandVector &Operands) {
   MCAsmParser &Parser = getParser();
   const AsmToken &Tok = Parser.getTok();
   // Eat "{" and mark the current place.
   const SMLoc consumedToken = consumeToken();
   if (Tok.isNot(AsmToken::Identifier))
-    return ErrorOperand(Tok.getLoc(), "Expected an identifier after {");
+    return Error(Tok.getLoc(), "Expected an identifier after {");
   if (Tok.getIdentifier().startswith("r")){
     int rndMode = StringSwitch<int>(Tok.getIdentifier())
       .Case("rn", X86::STATIC_ROUNDING::TO_NEAREST_INT)
@@ -1932,28 +1926,30 @@ X86AsmParser::ParseRoundingModeOp(SMLoc Start) {
       .Case("rz", X86::STATIC_ROUNDING::TO_ZERO)
       .Default(-1);
     if (-1 == rndMode)
-      return ErrorOperand(Tok.getLoc(), "Invalid rounding mode.");
+      return Error(Tok.getLoc(), "Invalid rounding mode.");
      Parser.Lex();  // Eat "r*" of r*-sae
     if (!getLexer().is(AsmToken::Minus))
-      return ErrorOperand(Tok.getLoc(), "Expected - at this point");
+      return Error(Tok.getLoc(), "Expected - at this point");
     Parser.Lex();  // Eat "-"
     Parser.Lex();  // Eat the sae
     if (!getLexer().is(AsmToken::RCurly))
-      return ErrorOperand(Tok.getLoc(), "Expected } at this point");
+      return Error(Tok.getLoc(), "Expected } at this point");
     SMLoc End = Tok.getEndLoc();
     Parser.Lex();  // Eat "}"
     const MCExpr *RndModeOp =
       MCConstantExpr::create(rndMode, Parser.getContext());
-    return X86Operand::CreateImm(RndModeOp, Start, End);
+    Operands.push_back(X86Operand::CreateImm(RndModeOp, Start, End));
+    return false;
   }
   if(Tok.getIdentifier().equals("sae")){
     Parser.Lex();  // Eat the sae
     if (!getLexer().is(AsmToken::RCurly))
-      return ErrorOperand(Tok.getLoc(), "Expected } at this point");
+      return Error(Tok.getLoc(), "Expected } at this point");
     Parser.Lex();  // Eat "}"
-    return X86Operand::CreateToken("{sae}", consumedToken);
+    Operands.push_back(X86Operand::CreateToken("{sae}", consumedToken));
+    return false;
   }
-  return ErrorOperand(Tok.getLoc(), "unknown token in expression");
+  return Error(Tok.getLoc(), "unknown token in expression");
 }
 
 /// Parse the '.' operator.
@@ -2088,7 +2084,7 @@ bool X86AsmParser::ParseIntelMemoryOperandSize(unsigned &Size) {
   return false;
 }
 
-std::unique_ptr<X86Operand> X86AsmParser::ParseIntelOperand() {
+bool X86AsmParser::ParseIntelOperand(OperandVector &Operands) {
   MCAsmParser &Parser = getParser();
   const AsmToken &Tok = Parser.getTok();
   SMLoc Start, End;
@@ -2096,28 +2092,31 @@ std::unique_ptr<X86Operand> X86AsmParser::ParseIntelOperand() {
   // Parse optional Size directive.
   unsigned Size;
   if (ParseIntelMemoryOperandSize(Size))
-    return nullptr;
+    return true;
   bool PtrInOperand = bool(Size);
 
   Start = Tok.getLoc();
 
   // Rounding mode operand.
   if (getLexer().is(AsmToken::LCurly))
-    return ParseRoundingModeOp(Start);
+    return ParseRoundingModeOp(Start, Operands);
 
   // Register operand.
   unsigned RegNo = 0;
   if (Tok.is(AsmToken::Identifier) && !ParseRegister(RegNo, Start, End)) {
     if (RegNo == X86::RIP)
-      return ErrorOperand(Start, "rip can only be used as a base register");
+      return Error(Start, "rip can only be used as a base register");
     // A Register followed by ':' is considered a segment override
-    if (Tok.isNot(AsmToken::Colon))
-      return !PtrInOperand ? X86Operand::CreateReg(RegNo, Start, End) :
-        ErrorOperand(Start, "expected memory operand after 'ptr', "
+    if (Tok.isNot(AsmToken::Colon)) {
+      if (PtrInOperand)
+        return Error(Start, "expected memory operand after 'ptr', "
                             "found register operand instead");
+      Operands.push_back(X86Operand::CreateReg(RegNo, Start, End));
+      return false;
+    }
     // An alleged segment override. check if we have a valid segment register
     if (!X86MCRegisterClasses[X86::SEGMENT_REGRegClassID].contains(RegNo))
-      return ErrorOperand(Start, "invalid segment register");
+      return Error(Start, "invalid segment register");
     // Eat ':' and update Start location
     Start = Lex().getLoc();
   }
@@ -2125,7 +2124,7 @@ std::unique_ptr<X86Operand> X86AsmParser::ParseIntelOperand() {
   // Immediates and Memory
   IntelExprStateMachine SM;
   if (ParseIntelExpression(SM, End))
-    return nullptr;
+    return true;
 
   if (isParsingMSInlineAsm())
     RewriteIntelExpression(SM, Start, Tok.getLoc());
@@ -2146,12 +2145,15 @@ std::unique_ptr<X86Operand> X86AsmParser::ParseIntelOperand() {
       if (Info.isKind(InlineAsmIdentifierInfo::IK_Var)) {
         // Disp includes the address of a variable; make sure this is recorded
         // for later handling.
-        return X86Operand::CreateImm(Disp, Start, End, SM.getSymName(),
-                                     Info.Var.Decl, Info.Var.IsGlobalLV);
+        Operands.push_back(X86Operand::CreateImm(Disp, Start, End,
+                                                 SM.getSymName(), Info.Var.Decl,
+                                                 Info.Var.IsGlobalLV));
+        return false;
       }
     }
 
-    return X86Operand::CreateImm(Disp, Start, End);
+    Operands.push_back(X86Operand::CreateImm(Disp, Start, End));
+    return false;
   }
 
   StringRef ErrMsg;
@@ -2176,7 +2178,7 @@ std::unique_ptr<X86Operand> X86AsmParser::ParseIntelOperand() {
 
   if (Scale != 0 &&
       X86MCRegisterClasses[X86::GR16RegClassID].contains(IndexReg))
-    return ErrorOperand(Start, "16-bit addresses cannot have a scale");
+    return Error(Start, "16-bit addresses cannot have a scale");
 
   // If there was no explicit scale specified, change it to 1.
   if (Scale == 0)
@@ -2192,23 +2194,30 @@ std::unique_ptr<X86Operand> X86AsmParser::ParseIntelOperand() {
   if ((BaseReg || IndexReg) &&
       CheckBaseRegAndIndexRegAndScale(BaseReg, IndexReg, Scale, is64BitMode(),
                                       ErrMsg))
-    return ErrorOperand(Start, ErrMsg);
+    return Error(Start, ErrMsg);
   if (isParsingMSInlineAsm())
     return CreateMemForMSInlineAsm(RegNo, Disp, BaseReg, IndexReg, Scale, Start,
                                    End, Size, SM.getSymName(),
-                                   SM.getIdentifierInfo());
+                                   SM.getIdentifierInfo(), Operands);
 
   // When parsing x64 MS-style assembly, all memory operands default to
   // RIP-relative when interpreted as non-absolute references.
-  if (Parser.isParsingMasm() && is64BitMode())
-    return X86Operand::CreateMem(getPointerWidth(), RegNo, Disp, BaseReg,
-                                 IndexReg, Scale, Start, End, Size,
-                                 /*DefaultBaseReg=*/X86::RIP);
-
-  if (!(BaseReg || IndexReg || RegNo))
-    return X86Operand::CreateMem(getPointerWidth(), Disp, Start, End, Size);
-  return X86Operand::CreateMem(getPointerWidth(), RegNo, Disp,
-                               BaseReg, IndexReg, Scale, Start, End, Size);
+  if (Parser.isParsingMasm() && is64BitMode()) {
+    Operands.push_back(X86Operand::CreateMem(getPointerWidth(), RegNo, Disp,
+                                             BaseReg, IndexReg, Scale, Start,
+                                             End, Size,
+                                             /*DefaultBaseReg=*/X86::RIP));
+    return false;
+  }
+
+  if ((BaseReg || IndexReg || RegNo))
+    Operands.push_back(X86Operand::CreateMem(getPointerWidth(), RegNo, Disp,
+                                             BaseReg, IndexReg, Scale, Start,
+                                             End, Size));
+  else
+    Operands.push_back(
+        X86Operand::CreateMem(getPointerWidth(), Disp, Start, End, Size));
+  return false;
 }
 
 bool X86AsmParser::ParseATTOperand(OperandVector &Operands) {
@@ -2232,11 +2241,7 @@ bool X86AsmParser::ParseATTOperand(OperandVector &Operands) {
   }
   case AsmToken::LCurly: {
     SMLoc Start = Parser.getTok().getLoc();
-    if (std::unique_ptr<X86Operand> Op = ParseRoundingModeOp(Start)) {
-      Operands.push_back(std::move(Op));
-      return false;
-    }
-    return true;
+    return ParseRoundingModeOp(Start, Operands);
   }
   default: {
     // This a memory operand or a register. We have some parsing complications

From c03d3aca7d00db61eece554b7c4a9d921c1aff05 Mon Sep 17 00:00:00 2001
From: Arthur Eubanks <aeubanks@google.com>
Date: Thu, 30 Jul 2020 20:09:34 -0700
Subject: [PATCH 0702/1035] [test] Fix thinlto-distributed-newpm.ll

Broken by https://reviews.llvm.org/D84981.
---
 clang/test/CodeGen/thinlto-distributed-newpm.ll | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/clang/test/CodeGen/thinlto-distributed-newpm.ll b/clang/test/CodeGen/thinlto-distributed-newpm.ll
index 3dcc6a196365c..caf294df8eb85 100644
--- a/clang/test/CodeGen/thinlto-distributed-newpm.ll
+++ b/clang/test/CodeGen/thinlto-distributed-newpm.ll
@@ -23,7 +23,6 @@
 ; CHECK-O: Running pass: WholeProgramDevirtPass
 ; CHECK-O: Running analysis: InnerAnalysisManagerProxy
 ; CHECK-O: Running pass: LowerTypeTestsPass
-; CHECK-O: Invalidating all non-preserved analyses for:
 ; CHECK-O: Invalidating analysis: InnerAnalysisManagerProxy
 ; CHECK-O: Running pass: ForceFunctionAttrsPass
 ; CHECK-O: Starting {{.*}}Module pass manager run.
@@ -47,7 +46,6 @@
 ; CHECK-O: Running pass: IPSCCPPass
 ; CHECK-O: Running pass: CalledValuePropagationPass
 ; CHECK-O: Running pass: GlobalOptPass
-; CHECK-O: Invalidating all non-preserved analyses for:
 ; CHECK-O: Invalidating analysis: InnerAnalysisManagerProxy
 ; CHECK-O: Running analysis: InnerAnalysisManagerProxy
 ; CHECK-O: Running pass: PromotePass
@@ -77,7 +75,6 @@
 ; CHECK-O: Starting CGSCC pass manager run.
 ; CHECK-O: Running pass: InlinerPass on (main)
 ; CHECK-O: Running pass: PostOrderFunctionAttrsPass on (main)
-; CHECK-O: Invalidating all non-preserved analyses for: (main)
 ; CHECK-O: Clearing all analysis results for: main
 ; CHECK-O3: Running pass: ArgumentPromotionPass on (main)
 ; CHECK-O3: Running analysis: TargetIRAnalysis on main
@@ -143,8 +140,6 @@
 ; CHECK-O: Running pass: InstCombinePass on main
 ; CHECK-O: Finished {{.*}}Function pass manager run.
 ; CHECK-O: Finished CGSCC pass manager run.
-; CHECK-O: Invalidating all non-preserved analyses for:
-; CHECK-O: Invalidating all non-preserved analyses for: main
 ; CHECK-O: Invalidating analysis: DominatorTreeAnalysis on main
 ; CHECK-O: Invalidating analysis: BasicAA on main
 ; CHECK-O: Invalidating analysis: AAManager on main

From 9f0225894254c4706519c85d6fc06c5382903fef Mon Sep 17 00:00:00 2001
From: Wolfgang Pieb <wolfgang_pieb@playstation.sony.com>
Date: Thu, 30 Jul 2020 20:36:31 -0700
Subject: [PATCH 0703/1035] Make a test case more flexible on Windows wrt the
 number of backslashes.

---
 lld/test/ELF/dependency-file.s | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/lld/test/ELF/dependency-file.s b/lld/test/ELF/dependency-file.s
index 14048ead49430..e7dbf9c7695f7 100644
--- a/lld/test/ELF/dependency-file.s
+++ b/lld/test/ELF/dependency-file.s
@@ -6,16 +6,16 @@
 # RUN: ld.lld -o %t/foo.exe %t/foo.o %t/"bar baz.o" "%t/#quux$.o" --dependency-file=%t/foo.d
 # RUN: FileCheck --match-full-lines -DFILE=%t %s < %t/foo.d
 
-# CHECK:      [[FILE]]{{/|\\\\}}foo.exe: \
-# CHECK-NEXT:   [[FILE]]{{/|\\\\}}foo.o \
-# CHECK-NEXT:   [[FILE]]{{/|\\\\}}bar\ baz.o \
-# CHECK-NEXT:   [[FILE]]{{/|\\\\}}\#quux$$.o
+# CHECK:      [[FILE]]{{/|(\\)+}}foo.exe: \
+# CHECK-NEXT:   [[FILE]]{{/|(\\)+}}foo.o \
+# CHECK-NEXT:   [[FILE]]{{/|(\\)+}}bar\ baz.o \
+# CHECK-NEXT:   [[FILE]]{{/|(\\)+}}\#quux$$.o
 # CHECK-EMPTY:
-# CHECK-NEXT: [[FILE]]{{/|\\\\}}foo.o:
+# CHECK-NEXT: [[FILE]]{{/|(\\)+}}foo.o:
 # CHECK-EMPTY:
-# CHECK-NEXT: [[FILE]]{{/|\\\\}}bar\ baz.o:
+# CHECK-NEXT: [[FILE]]{{/|(\\)+}}bar\ baz.o:
 # CHECK-EMPTY:
-# CHECK-NEXT: [[FILE]]{{/|\\\\}}\#quux$$.o:
+# CHECK-NEXT: [[FILE]]{{/|(\\)+}}\#quux$$.o:
 
 .global _start
 _start:

From 89051ebacea72733ff7088cf09a760b1be707acf Mon Sep 17 00:00:00 2001
From: Vitaly Buka <vitalybuka@google.com>
Date: Thu, 30 Jul 2020 21:07:10 -0700
Subject: [PATCH 0704/1035] [NFC] GetUnderlyingObject -> getUnderlyingObject

I am going to touch them in the next patch anyway
---
 .../llvm/Analysis/ObjCARCAnalysisUtils.h      |  2 +-
 llvm/include/llvm/Analysis/ValueTracking.h    | 12 ++++----
 llvm/lib/Analysis/AliasAnalysis.cpp           |  2 +-
 llvm/lib/Analysis/BasicAliasAnalysis.cpp      | 18 ++++++------
 llvm/lib/Analysis/CaptureTracking.cpp         |  2 +-
 llvm/lib/Analysis/ConstantFolding.cpp         |  2 +-
 llvm/lib/Analysis/DependenceAnalysis.cpp      |  4 +--
 llvm/lib/Analysis/GlobalsModRef.cpp           | 28 +++++++++----------
 llvm/lib/Analysis/InstructionSimplify.cpp     |  4 +--
 llvm/lib/Analysis/LazyValueInfo.cpp           | 14 +++++-----
 llvm/lib/Analysis/Lint.cpp                    |  2 +-
 llvm/lib/Analysis/LoopAccessAnalysis.cpp      |  8 +++---
 .../lib/Analysis/MemoryDependenceAnalysis.cpp |  2 +-
 llvm/lib/Analysis/ValueTracking.cpp           | 14 +++++-----
 llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp  |  2 +-
 llvm/lib/CodeGen/MachinePipeliner.cpp         |  6 ++--
 .../SelectionDAG/SelectionDAGBuilder.cpp      |  2 +-
 .../lib/Target/AMDGPU/AMDGPUAliasAnalysis.cpp |  2 +-
 llvm/lib/Target/AMDGPU/AMDGPUInline.cpp       |  2 +-
 .../lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp |  2 +-
 .../AMDGPU/AMDGPUTargetTransformInfo.cpp      |  2 +-
 llvm/lib/Target/AMDGPU/R600Instructions.td    |  4 +--
 llvm/lib/Target/AMDGPU/SIInstrInfo.cpp        |  4 +--
 llvm/lib/Target/Mips/MipsDelaySlotFiller.cpp  |  2 +-
 llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp   |  6 ++--
 llvm/lib/Target/NVPTX/NVPTXLowerArgs.cpp      |  2 +-
 .../Transforms/IPO/AttributorAttributes.cpp   |  2 +-
 .../InstCombine/InstCombineCompares.cpp       |  4 +--
 .../InstCombine/InstCombineSelect.cpp         |  5 ++--
 .../Instrumentation/AddressSanitizer.cpp      |  4 +--
 .../Instrumentation/DataFlowSanitizer.cpp     |  2 +-
 .../Instrumentation/ThreadSanitizer.cpp       |  2 +-
 .../Scalar/DeadStoreElimination.cpp           | 22 +++++++--------
 llvm/lib/Transforms/Scalar/LICM.cpp           |  4 +--
 .../Transforms/Scalar/LoopIdiomRecognize.cpp  |  4 +--
 .../Scalar/LowerMatrixIntrinsics.cpp          |  2 +-
 .../Transforms/Utils/AssumeBundleBuilder.cpp  |  4 +--
 llvm/lib/Transforms/Utils/InlineFunction.cpp  |  2 +-
 llvm/lib/Transforms/Utils/LoopUtils.cpp       |  4 +--
 llvm/lib/Transforms/Utils/VNCoercion.cpp      |  2 +-
 .../Vectorize/LoadStoreVectorizer.cpp         |  2 +-
 .../Transforms/Vectorize/SLPVectorizer.cpp    |  2 +-
 .../Analysis/LoopAccessAnalysis/nullptr.ll    |  4 +--
 .../AMDGPU/promote-alloca-to-lds-phi.ll       |  2 +-
 .../address-space-ptr-sze-gep-index-assert.ll |  4 +--
 45 files changed, 114 insertions(+), 113 deletions(-)

diff --git a/llvm/include/llvm/Analysis/ObjCARCAnalysisUtils.h b/llvm/include/llvm/Analysis/ObjCARCAnalysisUtils.h
index 945d41c376779..bd541eea9e81c 100644
--- a/llvm/include/llvm/Analysis/ObjCARCAnalysisUtils.h
+++ b/llvm/include/llvm/Analysis/ObjCARCAnalysisUtils.h
@@ -67,7 +67,7 @@ inline bool ModuleHasARC(const Module &M) {
 inline const Value *GetUnderlyingObjCPtr(const Value *V,
                                                 const DataLayout &DL) {
   for (;;) {
-    V = GetUnderlyingObject(V, DL);
+    V = getUnderlyingObject(V, DL);
     if (!IsForwarding(GetBasicARCInstKind(V)))
       break;
     V = cast<CallInst>(V)->getArgOperand(0);
diff --git a/llvm/include/llvm/Analysis/ValueTracking.h b/llvm/include/llvm/Analysis/ValueTracking.h
index 92dc7c01e83bc..2c1426be97341 100644
--- a/llvm/include/llvm/Analysis/ValueTracking.h
+++ b/llvm/include/llvm/Analysis/ValueTracking.h
@@ -368,14 +368,14 @@ class Value;
   /// that the returned value has pointer type if the specified value does. If
   /// the MaxLookup value is non-zero, it limits the number of instructions to
   /// be stripped off.
-  Value *GetUnderlyingObject(Value *V, const DataLayout &DL,
+  Value *getUnderlyingObject(Value *V, const DataLayout &DL,
                              unsigned MaxLookup = 6);
-  inline const Value *GetUnderlyingObject(const Value *V, const DataLayout &DL,
+  inline const Value *getUnderlyingObject(const Value *V, const DataLayout &DL,
                                           unsigned MaxLookup = 6) {
-    return GetUnderlyingObject(const_cast<Value *>(V), DL, MaxLookup);
+    return getUnderlyingObject(const_cast<Value *>(V), DL, MaxLookup);
   }
 
-  /// This method is similar to GetUnderlyingObject except that it can
+  /// This method is similar to getUnderlyingObject except that it can
   /// look through phi and select instructions and return multiple objects.
   ///
   /// If LoopInfo is passed, loop phis are further analyzed.  If a pointer
@@ -403,12 +403,12 @@ class Value;
   /// Since A[i] and A[i-1] are independent pointers, getUnderlyingObjects
   /// should not assume that Curr and Prev share the same underlying object thus
   /// it shouldn't look through the phi above.
-  void GetUnderlyingObjects(const Value *V,
+  void getUnderlyingObjects(const Value *V,
                             SmallVectorImpl<const Value *> &Objects,
                             const DataLayout &DL, LoopInfo *LI = nullptr,
                             unsigned MaxLookup = 6);
 
-  /// This is a wrapper around GetUnderlyingObjects and adds support for basic
+  /// This is a wrapper around getUnderlyingObjects and adds support for basic
   /// ptrtoint+arithmetic+inttoptr sequences.
   bool getUnderlyingObjectsForCodeGen(const Value *V,
                             SmallVectorImpl<Value *> &Objects,
diff --git a/llvm/lib/Analysis/AliasAnalysis.cpp b/llvm/lib/Analysis/AliasAnalysis.cpp
index fec2415a0e459..e77c203fc8d44 100644
--- a/llvm/lib/Analysis/AliasAnalysis.cpp
+++ b/llvm/lib/Analysis/AliasAnalysis.cpp
@@ -642,7 +642,7 @@ ModRefInfo AAResults::callCapturesBefore(const Instruction *I,
     return ModRefInfo::ModRef;
 
   const Value *Object =
-      GetUnderlyingObject(MemLoc.Ptr, I->getModule()->getDataLayout());
+      getUnderlyingObject(MemLoc.Ptr, I->getModule()->getDataLayout());
   if (!isIdentifiedObject(Object) || isa<GlobalValue>(Object) ||
       isa<Constant>(Object))
     return ModRefInfo::ModRef;
diff --git a/llvm/lib/Analysis/BasicAliasAnalysis.cpp b/llvm/lib/Analysis/BasicAliasAnalysis.cpp
index 33f122728d2aa..720ce0f688cfb 100644
--- a/llvm/lib/Analysis/BasicAliasAnalysis.cpp
+++ b/llvm/lib/Analysis/BasicAliasAnalysis.cpp
@@ -91,7 +91,7 @@ STATISTIC(SearchTimes, "Number of times a GEP is decomposed");
 const unsigned MaxNumPhiBBsValueReachabilityCheck = 20;
 
 // The max limit of the search depth in DecomposeGEPExpression() and
-// GetUnderlyingObject(), both functions need to use the same search
+// getUnderlyingObject(), both functions need to use the same search
 // depth otherwise the algorithm in aliasGEP will assert.
 static const unsigned MaxLookupSearchDepth = 6;
 
@@ -456,8 +456,8 @@ static unsigned getMaxPointerSize(const DataLayout &DL) {
 /// such, the gep cannot necessarily be reconstructed from its decomposed form.
 ///
 /// When DataLayout is around, this function is capable of analyzing everything
-/// that GetUnderlyingObject can look through. To be able to do that
-/// GetUnderlyingObject and DecomposeGEPExpression must use the same search
+/// that getUnderlyingObject can look through. To be able to do that
+/// getUnderlyingObject and DecomposeGEPExpression must use the same search
 /// depth (MaxLookupSearchDepth). When DataLayout not is around, it just looks
 /// through pointer casts.
 bool BasicAAResult::DecomposeGEPExpression(const Value *V,
@@ -661,7 +661,7 @@ bool BasicAAResult::pointsToConstantMemory(const MemoryLocation &Loc,
   SmallVector<const Value *, 16> Worklist;
   Worklist.push_back(Loc.Ptr);
   do {
-    const Value *V = GetUnderlyingObject(Worklist.pop_back_val(), DL);
+    const Value *V = getUnderlyingObject(Worklist.pop_back_val(), DL);
     if (!Visited.insert(V).second) {
       Visited.clear();
       return AAResultBase::pointsToConstantMemory(Loc, AAQI, OrLocal);
@@ -875,7 +875,7 @@ ModRefInfo BasicAAResult::getModRefInfo(const CallBase *Call,
   assert(notDifferentParent(Call, Loc.Ptr) &&
          "AliasAnalysis query involving multiple functions!");
 
-  const Value *Object = GetUnderlyingObject(Loc.Ptr, DL);
+  const Value *Object = getUnderlyingObject(Loc.Ptr, DL);
 
   // Calls marked 'tail' cannot read or write allocas from the current frame
   // because the current frame might be destroyed by the time they run. However,
@@ -1309,7 +1309,7 @@ bool BasicAAResult::isGEPBaseAtNegativeOffset(const GEPOperator *GEPOp,
 /// another pointer.
 ///
 /// We know that V1 is a GEP, but we don't know anything about V2.
-/// UnderlyingV1 is GetUnderlyingObject(GEP1, DL), UnderlyingV2 is the same for
+/// UnderlyingV1 is getUnderlyingObject(GEP1, DL), UnderlyingV2 is the same for
 /// V2.
 AliasResult BasicAAResult::aliasGEP(
     const GEPOperator *GEP1, LocationSize V1Size, const AAMDNodes &V1AAInfo,
@@ -1338,7 +1338,7 @@ AliasResult BasicAAResult::aliasGEP(
 
   assert(DecompGEP1.Base == UnderlyingV1 && DecompGEP2.Base == UnderlyingV2 &&
          "DecomposeGEPExpression returned a result different from "
-         "GetUnderlyingObject");
+         "getUnderlyingObject");
 
   // If the GEP's offset relative to its base is such that the base would
   // fall below the start of the object underlying V2, then the GEP and V2
@@ -1782,10 +1782,10 @@ AliasResult BasicAAResult::aliasCheck(const Value *V1, LocationSize V1Size,
 
   // Figure out what objects these things are pointing to if we can.
   if (O1 == nullptr)
-    O1 = GetUnderlyingObject(V1, DL, MaxLookupSearchDepth);
+    O1 = getUnderlyingObject(V1, DL, MaxLookupSearchDepth);
 
   if (O2 == nullptr)
-    O2 = GetUnderlyingObject(V2, DL, MaxLookupSearchDepth);
+    O2 = getUnderlyingObject(V2, DL, MaxLookupSearchDepth);
 
   // Null values in the default address space don't point to any object, so they
   // don't alias any other pointer.
diff --git a/llvm/lib/Analysis/CaptureTracking.cpp b/llvm/lib/Analysis/CaptureTracking.cpp
index 8b101e3b2cc47..6de6aaa48d4f8 100644
--- a/llvm/lib/Analysis/CaptureTracking.cpp
+++ b/llvm/lib/Analysis/CaptureTracking.cpp
@@ -273,7 +273,7 @@ void llvm::PointerMayBeCaptured(const Value *V, CaptureTracker *Tracker,
       // The pointer is not captured if returned pointer is not captured.
       // NOTE: CaptureTracking users should not assume that only functions
       // marked with nocapture do not capture. This means that places like
-      // GetUnderlyingObject in ValueTracking or DecomposeGEPExpression
+      // getUnderlyingObject in ValueTracking or DecomposeGEPExpression
       // in BasicAA also need to know about this property.
       if (isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(Call,
                                                                       true)) {
diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp
index a7458ffe4572c..1b3fa199d7a0c 100644
--- a/llvm/lib/Analysis/ConstantFolding.cpp
+++ b/llvm/lib/Analysis/ConstantFolding.cpp
@@ -718,7 +718,7 @@ Constant *llvm::ConstantFoldLoadFromConstPtr(Constant *C, Type *Ty,
 
   // If this load comes from anywhere in a constant global, and if the global
   // is all undef or zero, we know what it loads.
-  if (auto *GV = dyn_cast<GlobalVariable>(GetUnderlyingObject(CE, DL))) {
+  if (auto *GV = dyn_cast<GlobalVariable>(getUnderlyingObject(CE, DL))) {
     if (GV->isConstant() && GV->hasDefinitiveInitializer()) {
       if (GV->getInitializer()->isNullValue())
         return Constant::getNullValue(Ty);
diff --git a/llvm/lib/Analysis/DependenceAnalysis.cpp b/llvm/lib/Analysis/DependenceAnalysis.cpp
index bcfeef7fb8abc..3a67b4518c0ef 100644
--- a/llvm/lib/Analysis/DependenceAnalysis.cpp
+++ b/llvm/lib/Analysis/DependenceAnalysis.cpp
@@ -659,8 +659,8 @@ static AliasResult underlyingObjectsAlias(AAResults *AA,
     return NoAlias;
 
   // Check the underlying objects are the same
-  const Value *AObj = GetUnderlyingObject(LocA.Ptr, DL);
-  const Value *BObj = GetUnderlyingObject(LocB.Ptr, DL);
+  const Value *AObj = getUnderlyingObject(LocA.Ptr, DL);
+  const Value *BObj = getUnderlyingObject(LocB.Ptr, DL);
 
   // If the underlying objects are the same, they must alias
   if (AObj == BObj)
diff --git a/llvm/lib/Analysis/GlobalsModRef.cpp b/llvm/lib/Analysis/GlobalsModRef.cpp
index 8c8ccf04ebba2..b1cdfc1896d50 100644
--- a/llvm/lib/Analysis/GlobalsModRef.cpp
+++ b/llvm/lib/Analysis/GlobalsModRef.cpp
@@ -44,7 +44,7 @@ STATISTIC(NumIndirectGlobalVars, "Number of indirect global objects");
 // An option to enable unsafe alias results from the GlobalsModRef analysis.
 // When enabled, GlobalsModRef will provide no-alias results which in extremely
 // rare cases may not be conservatively correct. In particular, in the face of
-// transforms which cause assymetry between how effective GetUnderlyingObject
+// transforms which cause assymetry between how effective getUnderlyingObject
 // is for two pointers, it may produce incorrect results.
 //
 // These unsafe results have been returned by GMR for many years without
@@ -435,7 +435,7 @@ bool GlobalsAAResult::AnalyzeIndirectGlobalMemory(GlobalVariable *GV) {
         continue;
 
       // Check the value being stored.
-      Value *Ptr = GetUnderlyingObject(SI->getOperand(0),
+      Value *Ptr = getUnderlyingObject(SI->getOperand(0),
                                        GV->getParent()->getDataLayout());
 
       if (!isAllocLikeFn(Ptr, &GetTLI(*SI->getFunction())))
@@ -661,12 +661,12 @@ static bool isNonEscapingGlobalNoAliasWithLoad(const GlobalValue *GV,
       return false;
 
     if (auto *LI = dyn_cast<LoadInst>(Input)) {
-      Inputs.push_back(GetUnderlyingObject(LI->getPointerOperand(), DL));
+      Inputs.push_back(getUnderlyingObject(LI->getPointerOperand(), DL));
       continue;
     }
     if (auto *SI = dyn_cast<SelectInst>(Input)) {
-      const Value *LHS = GetUnderlyingObject(SI->getTrueValue(), DL);
-      const Value *RHS = GetUnderlyingObject(SI->getFalseValue(), DL);
+      const Value *LHS = getUnderlyingObject(SI->getTrueValue(), DL);
+      const Value *RHS = getUnderlyingObject(SI->getFalseValue(), DL);
       if (Visited.insert(LHS).second)
         Inputs.push_back(LHS);
       if (Visited.insert(RHS).second)
@@ -675,7 +675,7 @@ static bool isNonEscapingGlobalNoAliasWithLoad(const GlobalValue *GV,
     }
     if (auto *PN = dyn_cast<PHINode>(Input)) {
       for (const Value *Op : PN->incoming_values()) {
-        Op = GetUnderlyingObject(Op, DL);
+        Op = getUnderlyingObject(Op, DL);
         if (Visited.insert(Op).second)
           Inputs.push_back(Op);
       }
@@ -774,7 +774,7 @@ bool GlobalsAAResult::isNonEscapingGlobalNoAlias(const GlobalValue *GV,
     if (auto *LI = dyn_cast<LoadInst>(Input)) {
       // A pointer loaded from a global would have been captured, and we know
       // that the global is non-escaping, so no alias.
-      const Value *Ptr = GetUnderlyingObject(LI->getPointerOperand(), DL);
+      const Value *Ptr = getUnderlyingObject(LI->getPointerOperand(), DL);
       if (isNonEscapingGlobalNoAliasWithLoad(GV, Ptr, Depth, DL))
         // The load does not alias with GV.
         continue;
@@ -782,8 +782,8 @@ bool GlobalsAAResult::isNonEscapingGlobalNoAlias(const GlobalValue *GV,
       return false;
     }
     if (auto *SI = dyn_cast<SelectInst>(Input)) {
-      const Value *LHS = GetUnderlyingObject(SI->getTrueValue(), DL);
-      const Value *RHS = GetUnderlyingObject(SI->getFalseValue(), DL);
+      const Value *LHS = getUnderlyingObject(SI->getTrueValue(), DL);
+      const Value *RHS = getUnderlyingObject(SI->getFalseValue(), DL);
       if (Visited.insert(LHS).second)
         Inputs.push_back(LHS);
       if (Visited.insert(RHS).second)
@@ -792,7 +792,7 @@ bool GlobalsAAResult::isNonEscapingGlobalNoAlias(const GlobalValue *GV,
     }
     if (auto *PN = dyn_cast<PHINode>(Input)) {
       for (const Value *Op : PN->incoming_values()) {
-        Op = GetUnderlyingObject(Op, DL);
+        Op = getUnderlyingObject(Op, DL);
         if (Visited.insert(Op).second)
           Inputs.push_back(Op);
       }
@@ -827,8 +827,8 @@ AliasResult GlobalsAAResult::alias(const MemoryLocation &LocA,
                                    const MemoryLocation &LocB,
                                    AAQueryInfo &AAQI) {
   // Get the base object these pointers point to.
-  const Value *UV1 = GetUnderlyingObject(LocA.Ptr, DL);
-  const Value *UV2 = GetUnderlyingObject(LocB.Ptr, DL);
+  const Value *UV1 = getUnderlyingObject(LocA.Ptr, DL);
+  const Value *UV2 = getUnderlyingObject(LocB.Ptr, DL);
 
   // If either of the underlying values is a global, they may be non-addr-taken
   // globals, which we can answer queries about.
@@ -915,7 +915,7 @@ ModRefInfo GlobalsAAResult::getModRefInfoForArgument(const CallBase *Call,
   // is based on GV, return the conservative result.
   for (auto &A : Call->args()) {
     SmallVector<const Value*, 4> Objects;
-    GetUnderlyingObjects(A, Objects, DL);
+    getUnderlyingObjects(A, Objects, DL);
 
     // All objects must be identified.
     if (!all_of(Objects, isIdentifiedObject) &&
@@ -942,7 +942,7 @@ ModRefInfo GlobalsAAResult::getModRefInfo(const CallBase *Call,
   // If we are asking for mod/ref info of a direct call with a pointer to a
   // global we are tracking, return information if we have it.
   if (const GlobalValue *GV =
-          dyn_cast<GlobalValue>(GetUnderlyingObject(Loc.Ptr, DL)))
+          dyn_cast<GlobalValue>(getUnderlyingObject(Loc.Ptr, DL)))
     // If GV is internal to this IR and there is no function with local linkage
     // that has had their address taken, keep looking for a tighter ModRefInfo.
     if (GV->hasLocalLinkage() && !UnknownFunctionsWithLocalLinkage)
diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp
index 178b5bed20067..bf55bedd801eb 100644
--- a/llvm/lib/Analysis/InstructionSimplify.cpp
+++ b/llvm/lib/Analysis/InstructionSimplify.cpp
@@ -2524,8 +2524,8 @@ computePointerICmp(const DataLayout &DL, const TargetLibraryInfo *TLI,
     // memory within the lifetime of the current function (allocas, byval
     // arguments, globals), then determine the comparison result here.
     SmallVector<const Value *, 8> LHSUObjs, RHSUObjs;
-    GetUnderlyingObjects(LHS, LHSUObjs, DL);
-    GetUnderlyingObjects(RHS, RHSUObjs, DL);
+    getUnderlyingObjects(LHS, LHSUObjs, DL);
+    getUnderlyingObjects(RHS, RHSUObjs, DL);
 
     // Is the set of underlying objects all noalias calls?
     auto IsNAC = [](ArrayRef<const Value *> Objects) {
diff --git a/llvm/lib/Analysis/LazyValueInfo.cpp b/llvm/lib/Analysis/LazyValueInfo.cpp
index d326a442051a8..5011fc8782fe5 100644
--- a/llvm/lib/Analysis/LazyValueInfo.cpp
+++ b/llvm/lib/Analysis/LazyValueInfo.cpp
@@ -606,12 +606,12 @@ Optional<ValueLatticeElement> LazyValueInfoImpl::solveBlockValueImpl(
 static bool InstructionDereferencesPointer(Instruction *I, Value *Ptr) {
   if (LoadInst *L = dyn_cast<LoadInst>(I)) {
     return L->getPointerAddressSpace() == 0 &&
-           GetUnderlyingObject(L->getPointerOperand(),
+           getUnderlyingObject(L->getPointerOperand(),
                                L->getModule()->getDataLayout()) == Ptr;
   }
   if (StoreInst *S = dyn_cast<StoreInst>(I)) {
     return S->getPointerAddressSpace() == 0 &&
-           GetUnderlyingObject(S->getPointerOperand(),
+           getUnderlyingObject(S->getPointerOperand(),
                                S->getModule()->getDataLayout()) == Ptr;
   }
   if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(I)) {
@@ -622,12 +622,12 @@ static bool InstructionDereferencesPointer(Instruction *I, Value *Ptr) {
     if (!Len || Len->isZero()) return false;
 
     if (MI->getDestAddressSpace() == 0)
-      if (GetUnderlyingObject(MI->getRawDest(),
+      if (getUnderlyingObject(MI->getRawDest(),
                               MI->getModule()->getDataLayout()) == Ptr)
         return true;
     if (MemTransferInst *MTI = dyn_cast<MemTransferInst>(MI))
       if (MTI->getSourceAddressSpace() == 0)
-        if (GetUnderlyingObject(MTI->getRawSource(),
+        if (getUnderlyingObject(MTI->getRawSource(),
                                 MTI->getModule()->getDataLayout()) == Ptr)
           return true;
   }
@@ -642,10 +642,10 @@ static bool isObjectDereferencedInBlock(Value *Val, BasicBlock *BB) {
   assert(Val->getType()->isPointerTy());
 
   const DataLayout &DL = BB->getModule()->getDataLayout();
-  Value *UnderlyingVal = GetUnderlyingObject(Val, DL);
-  // If 'GetUnderlyingObject' didn't converge, skip it. It won't converge
+  Value *UnderlyingVal = getUnderlyingObject(Val, DL);
+  // If 'getUnderlyingObject' didn't converge, skip it. It won't converge
   // inside InstructionDereferencesPointer either.
-  if (UnderlyingVal == GetUnderlyingObject(UnderlyingVal, DL, 1))
+  if (UnderlyingVal == getUnderlyingObject(UnderlyingVal, DL, 1))
     for (Instruction &I : *BB)
       if (InstructionDereferencesPointer(&I, UnderlyingVal))
         return true;
diff --git a/llvm/lib/Analysis/Lint.cpp b/llvm/lib/Analysis/Lint.cpp
index 38017530c3227..a03f4257e97f1 100644
--- a/llvm/lib/Analysis/Lint.cpp
+++ b/llvm/lib/Analysis/Lint.cpp
@@ -673,7 +673,7 @@ Value *Lint::findValueImpl(Value *V, bool OffsetOk,
   // TODO: Look through eliminable cast pairs.
   // TODO: Look through calls with unique return values.
   // TODO: Look through vector insert/extract/shuffle.
-  V = OffsetOk ? GetUnderlyingObject(V, *DL) : V->stripPointerCasts();
+  V = OffsetOk ? getUnderlyingObject(V, *DL) : V->stripPointerCasts();
   if (LoadInst *L = dyn_cast<LoadInst>(V)) {
     BasicBlock::iterator BBI = L->getIterator();
     BasicBlock *BB = L->getParent();
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index f409cd322146e..e678b313c0fd2 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -938,7 +938,7 @@ void AccessAnalysis::processMemAccesses() {
           typedef SmallVector<const Value *, 16> ValueVector;
           ValueVector TempObjects;
 
-          GetUnderlyingObjects(Ptr, TempObjects, DL, LI);
+          getUnderlyingObjects(Ptr, TempObjects, DL, LI);
           LLVM_DEBUG(dbgs()
                      << "Underlying objects for pointer " << *Ptr << "\n");
           for (const Value *UnderlyingObj : TempObjects) {
@@ -1142,7 +1142,7 @@ bool llvm::sortPtrAccesses(ArrayRef<Value *> VL, const DataLayout &DL,
   // first pointer in the array.
   Value *Ptr0 = VL[0];
   const SCEV *Scev0 = SE.getSCEV(Ptr0);
-  Value *Obj0 = GetUnderlyingObject(Ptr0, DL);
+  Value *Obj0 = getUnderlyingObject(Ptr0, DL);
 
   llvm::SmallSet<int64_t, 4> Offsets;
   for (auto *Ptr : VL) {
@@ -1153,7 +1153,7 @@ bool llvm::sortPtrAccesses(ArrayRef<Value *> VL, const DataLayout &DL,
       return false;
     // If a pointer refers to a different underlying object, bail - the
     // pointers are by definition incomparable.
-    Value *CurrObj = GetUnderlyingObject(Ptr, DL);
+    Value *CurrObj = getUnderlyingObject(Ptr, DL);
     if (CurrObj != Obj0)
       return false;
 
@@ -1950,7 +1950,7 @@ void LoopAccessInfo::analyzeLoop(AAResults *AA, LoopInfo *LI,
   AccessAnalysis Accesses(TheLoop->getHeader()->getModule()->getDataLayout(),
                           TheLoop, AA, LI, DependentAccesses, *PSE);
 
-  // Holds the analyzed pointers. We don't want to call GetUnderlyingObjects
+  // Holds the analyzed pointers. We don't want to call getUnderlyingObjects
   // multiple times on the same object. If the ptr is accessed twice, once
   // for read and once for write, it will only appear once (on the write
   // list). This is okay, since we are going to check for conflicts between
diff --git a/llvm/lib/Analysis/MemoryDependenceAnalysis.cpp b/llvm/lib/Analysis/MemoryDependenceAnalysis.cpp
index 566eba5c54aff..645eb3d513717 100644
--- a/llvm/lib/Analysis/MemoryDependenceAnalysis.cpp
+++ b/llvm/lib/Analysis/MemoryDependenceAnalysis.cpp
@@ -576,7 +576,7 @@ MemDepResult MemoryDependenceResults::getSimplePointerDependencyFrom(
     // looking for a clobber in many cases; that's an alias property and is
     // handled by BasicAA.
     if (isa<AllocaInst>(Inst) || isNoAliasFn(Inst, &TLI)) {
-      const Value *AccessPtr = GetUnderlyingObject(MemLoc.Ptr, DL);
+      const Value *AccessPtr = getUnderlyingObject(MemLoc.Ptr, DL);
       if (AccessPtr == Inst || AA.isMustAlias(Inst, AccessPtr))
         return MemDepResult::getDef(Inst);
     }
diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index 9dfe8fdcb1a5c..54a2796ee2f65 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -4160,7 +4160,7 @@ static bool isSameUnderlyingObjectInLoop(const PHINode *PN,
   return true;
 }
 
-Value *llvm::GetUnderlyingObject(Value *V, const DataLayout &DL,
+Value *llvm::getUnderlyingObject(Value *V, const DataLayout &DL,
                                  unsigned MaxLookup) {
   if (!V->getType()->isPointerTy())
     return V;
@@ -4206,7 +4206,7 @@ Value *llvm::GetUnderlyingObject(Value *V, const DataLayout &DL,
   return V;
 }
 
-void llvm::GetUnderlyingObjects(const Value *V,
+void llvm::getUnderlyingObjects(const Value *V,
                                 SmallVectorImpl<const Value *> &Objects,
                                 const DataLayout &DL, LoopInfo *LI,
                                 unsigned MaxLookup) {
@@ -4215,7 +4215,7 @@ void llvm::GetUnderlyingObjects(const Value *V,
   Worklist.push_back(V);
   do {
     const Value *P = Worklist.pop_back_val();
-    P = GetUnderlyingObject(P, DL, MaxLookup);
+    P = getUnderlyingObject(P, DL, MaxLookup);
 
     if (!Visited.insert(P).second)
       continue;
@@ -4276,9 +4276,9 @@ static const Value *getUnderlyingObjectFromInt(const Value *V) {
   } while (true);
 }
 
-/// This is a wrapper around GetUnderlyingObjects and adds support for basic
+/// This is a wrapper around getUnderlyingObjects and adds support for basic
 /// ptrtoint+arithmetic+inttoptr sequences.
-/// It returns false if unidentified object is found in GetUnderlyingObjects.
+/// It returns false if unidentified object is found in getUnderlyingObjects.
 bool llvm::getUnderlyingObjectsForCodeGen(const Value *V,
                           SmallVectorImpl<Value *> &Objects,
                           const DataLayout &DL) {
@@ -4288,7 +4288,7 @@ bool llvm::getUnderlyingObjectsForCodeGen(const Value *V,
     V = Working.pop_back_val();
 
     SmallVector<const Value *, 4> Objs;
-    GetUnderlyingObjects(V, Objs, DL);
+    getUnderlyingObjects(V, Objs, DL);
 
     for (const Value *V : Objs) {
       if (!Visited.insert(V).second)
@@ -4301,7 +4301,7 @@ bool llvm::getUnderlyingObjectsForCodeGen(const Value *V,
           continue;
         }
       }
-      // If GetUnderlyingObjects fails to find an identifiable object,
+      // If getUnderlyingObjects fails to find an identifiable object,
       // getUnderlyingObjectsForCodeGen also fails for safety.
       if (!isIdentifiedObject(V)) {
         Objects.clear();
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 264af2b562b3f..61a1efc1295f8 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -1387,7 +1387,7 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
     // Get the underlying objects for the location passed on the lifetime
     // marker.
     SmallVector<const Value *, 4> Allocas;
-    GetUnderlyingObjects(CI.getArgOperand(1), Allocas, *DL);
+    getUnderlyingObjects(CI.getArgOperand(1), Allocas, *DL);
 
     // Iterate over each underlying object, creating lifetime markers for each
     // static alloca. Quit if we find a non-static alloca.
diff --git a/llvm/lib/CodeGen/MachinePipeliner.cpp b/llvm/lib/CodeGen/MachinePipeliner.cpp
index ef4b02ca9e3ef..908f4e45618ad 100644
--- a/llvm/lib/CodeGen/MachinePipeliner.cpp
+++ b/llvm/lib/CodeGen/MachinePipeliner.cpp
@@ -712,7 +712,7 @@ static void getUnderlyingObjects(const MachineInstr *MI,
   MachineMemOperand *MM = *MI->memoperands_begin();
   if (!MM->getValue())
     return;
-  GetUnderlyingObjects(MM->getValue(), Objs, DL);
+  getUnderlyingObjects(MM->getValue(), Objs, DL);
   for (const Value *V : Objs) {
     if (!isIdentifiedObject(V)) {
       Objs.clear();
@@ -736,7 +736,7 @@ void SwingSchedulerDAG::addLoopCarriedDependences(AliasAnalysis *AA) {
       PendingLoads.clear();
     else if (MI.mayLoad()) {
       SmallVector<const Value *, 4> Objs;
-      getUnderlyingObjects(&MI, Objs, MF.getDataLayout());
+      ::getUnderlyingObjects(&MI, Objs, MF.getDataLayout());
       if (Objs.empty())
         Objs.push_back(UnknownValue);
       for (auto V : Objs) {
@@ -745,7 +745,7 @@ void SwingSchedulerDAG::addLoopCarriedDependences(AliasAnalysis *AA) {
       }
     } else if (MI.mayStore()) {
       SmallVector<const Value *, 4> Objs;
-      getUnderlyingObjects(&MI, Objs, MF.getDataLayout());
+      ::getUnderlyingObjects(&MI, Objs, MF.getDataLayout());
       if (Objs.empty())
         Objs.push_back(UnknownValue);
       for (auto V : Objs) {
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index d231e61a103b0..c0ba4c55995d4 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -6636,7 +6636,7 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
         cast<ConstantInt>(I.getArgOperand(0))->getSExtValue();
     Value *const ObjectPtr = I.getArgOperand(1);
     SmallVector<const Value *, 4> Allocas;
-    GetUnderlyingObjects(ObjectPtr, Allocas, *DL);
+    getUnderlyingObjects(ObjectPtr, Allocas, *DL);
 
     for (SmallVectorImpl<const Value*>::iterator Object = Allocas.begin(),
            E = Allocas.end(); Object != E; ++Object) {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAliasAnalysis.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAliasAnalysis.cpp
index bb2aba0449748..e0159bb9b1130 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAliasAnalysis.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAliasAnalysis.cpp
@@ -96,7 +96,7 @@ bool AMDGPUAAResult::pointsToConstantMemory(const MemoryLocation &Loc,
       AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT)
     return true;
 
-  const Value *Base = GetUnderlyingObject(Loc.Ptr, DL);
+  const Value *Base = getUnderlyingObject(Loc.Ptr, DL);
   AS = Base->getType()->getPointerAddressSpace();
   if (AS == AMDGPUAS::CONSTANT_ADDRESS ||
       AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInline.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInline.cpp
index 3b5d91133a2f8..d859e798a74c8 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInline.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInline.cpp
@@ -134,7 +134,7 @@ unsigned AMDGPUInliner::getInlineThreshold(CallBase &CB) const {
                 Ty->getAddressSpace() != AMDGPUAS::FLAT_ADDRESS))
       continue;
 
-    PtrArg = GetUnderlyingObject(PtrArg, DL);
+    PtrArg = getUnderlyingObject(PtrArg, DL);
     if (const AllocaInst *AI = dyn_cast<AllocaInst>(PtrArg)) {
       if (!AI->isStaticAlloca() || !AIVisited.insert(AI).second)
         continue;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
index c9e2bd768ccec..965da223dac46 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
@@ -605,7 +605,7 @@ bool AMDGPUPromoteAlloca::binaryOpIsDerivedFromSameAlloca(Value *BaseAlloca,
   if (isa<ConstantPointerNull>(OtherOp))
     return true;
 
-  Value *OtherObj = GetUnderlyingObject(OtherOp, *DL);
+  Value *OtherObj = getUnderlyingObject(OtherOp, *DL);
   if (!isa<AllocaInst>(OtherObj))
     return false;
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index 4ae092ce55bff..18d8ad1d38482 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -169,7 +169,7 @@ void AMDGPUTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
       if (AS == AMDGPUAS::PRIVATE_ADDRESS) {
         const Value *Ptr = GEP->getPointerOperand();
         const AllocaInst *Alloca =
-            dyn_cast<AllocaInst>(GetUnderlyingObject(Ptr, DL));
+            dyn_cast<AllocaInst>(getUnderlyingObject(Ptr, DL));
         if (!Alloca || !Alloca->isStaticAlloca())
           continue;
         Type *Ty = Alloca->getAllocatedType();
diff --git a/llvm/lib/Target/AMDGPU/R600Instructions.td b/llvm/lib/Target/AMDGPU/R600Instructions.td
index 2cc21364c4397..03bfb2babf0cd 100644
--- a/llvm/lib/Target/AMDGPU/R600Instructions.td
+++ b/llvm/lib/Target/AMDGPU/R600Instructions.td
@@ -353,7 +353,7 @@ class LoadVtxId1 <PatFrag load> : PatFrag <
   const MemSDNode *LD = cast<MemSDNode>(N);
   return LD->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS ||
          (LD->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS &&
-           !isa<GlobalValue>(GetUnderlyingObject(
+           !isa<GlobalValue>(getUnderlyingObject(
            LD->getMemOperand()->getValue(), CurDAG->getDataLayout())));
 }]>;
 
@@ -365,7 +365,7 @@ class LoadVtxId2 <PatFrag load> : PatFrag <
   (ops node:$ptr), (load node:$ptr), [{
   const MemSDNode *LD = cast<MemSDNode>(N);
   return LD->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS &&
-         isa<GlobalValue>(GetUnderlyingObject(
+         isa<GlobalValue>(getUnderlyingObject(
          LD->getMemOperand()->getValue(), CurDAG->getDataLayout()));
 }]>;
 
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 745a526085566..17d1322d7e59e 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -461,8 +461,8 @@ static bool memOpsHaveSameBasePtr(const MachineInstr &MI1,
     return false;
   const MachineFunction &MF = *MI1.getParent()->getParent();
   const DataLayout &DL = MF.getFunction().getParent()->getDataLayout();
-  Base1 = GetUnderlyingObject(Base1, DL);
-  Base2 = GetUnderlyingObject(Base2, DL);
+  Base1 = getUnderlyingObject(Base1, DL);
+  Base2 = getUnderlyingObject(Base2, DL);
 
   if (isa<UndefValue>(Base1) || isa<UndefValue>(Base2))
     return false;
diff --git a/llvm/lib/Target/Mips/MipsDelaySlotFiller.cpp b/llvm/lib/Target/Mips/MipsDelaySlotFiller.cpp
index 155d19ba6959a..0fcbeba6db5c8 100644
--- a/llvm/lib/Target/Mips/MipsDelaySlotFiller.cpp
+++ b/llvm/lib/Target/Mips/MipsDelaySlotFiller.cpp
@@ -542,7 +542,7 @@ getUnderlyingObjects(const MachineInstr &MI,
 
   if (const Value *V = MMO.getValue()) {
     SmallVector<const Value *, 4> Objs;
-    GetUnderlyingObjects(V, Objs, DL);
+    ::getUnderlyingObjects(V, Objs, DL);
 
     for (const Value *UValue : Objs) {
       if (!isIdentifiedObject(V))
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
index 4296eca6a8dfd..4cdb8393ebb26 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
@@ -700,12 +700,12 @@ static bool canLowerToLDG(MemSDNode *N, const NVPTXSubtarget &Subtarget,
 
   bool IsKernelFn = isKernelFunction(F->getFunction());
 
-  // We use GetUnderlyingObjects() here instead of GetUnderlyingObject() mainly
+  // We use getUnderlyingObjects() here instead of getUnderlyingObject() mainly
   // because the former looks through phi nodes while the latter does not. We
   // need to look through phi nodes to handle pointer induction variables.
   SmallVector<const Value *, 8> Objs;
-  GetUnderlyingObjects(N->getMemOperand()->getValue(),
-                       Objs, F->getDataLayout());
+  getUnderlyingObjects(N->getMemOperand()->getValue(), Objs,
+                       F->getDataLayout());
 
   return all_of(Objs, [&](const Value *V) {
     if (auto *A = dyn_cast<const Argument>(V))
diff --git a/llvm/lib/Target/NVPTX/NVPTXLowerArgs.cpp b/llvm/lib/Target/NVPTX/NVPTXLowerArgs.cpp
index e60b5eeacdaee..453072818a534 100644
--- a/llvm/lib/Target/NVPTX/NVPTXLowerArgs.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXLowerArgs.cpp
@@ -214,7 +214,7 @@ bool NVPTXLowerArgs::runOnKernelFunction(Function &F) {
       for (auto &I : B) {
         if (LoadInst *LI = dyn_cast<LoadInst>(&I)) {
           if (LI->getType()->isPointerTy()) {
-            Value *UO = GetUnderlyingObject(LI->getPointerOperand(),
+            Value *UO = getUnderlyingObject(LI->getPointerOperand(),
                                             F.getParent()->getDataLayout());
             if (Argument *Arg = dyn_cast<Argument>(UO)) {
               if (Arg->hasByValAttr()) {
diff --git a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
index 142a4e48217c7..24b6bd4818dc5 100644
--- a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
+++ b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
@@ -5422,7 +5422,7 @@ struct AAPrivatizablePtrFloating : public AAPrivatizablePtrImpl {
   /// See AAPrivatizablePtrImpl::identifyPrivatizableType(...)
   Optional<Type *> identifyPrivatizableType(Attributor &A) override {
     Value *Obj =
-        GetUnderlyingObject(&getAssociatedValue(), A.getInfoCache().getDL());
+        getUnderlyingObject(&getAssociatedValue(), A.getInfoCache().getDL());
     if (!Obj) {
       LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] No underlying object found!\n");
       return nullptr;
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
index 2b8732fdb80d7..4c75bc97de66e 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
@@ -5643,10 +5643,10 @@ Instruction *InstCombinerImpl::visitICmpInst(ICmpInst &I) {
   // Try to optimize equality comparisons against alloca-based pointers.
   if (Op0->getType()->isPointerTy() && I.isEquality()) {
     assert(Op1->getType()->isPointerTy() && "Comparing pointer with non-pointer?");
-    if (auto *Alloca = dyn_cast<AllocaInst>(GetUnderlyingObject(Op0, DL)))
+    if (auto *Alloca = dyn_cast<AllocaInst>(getUnderlyingObject(Op0, DL)))
       if (Instruction *New = foldAllocaCmp(I, Alloca, Op1))
         return New;
-    if (auto *Alloca = dyn_cast<AllocaInst>(GetUnderlyingObject(Op1, DL)))
+    if (auto *Alloca = dyn_cast<AllocaInst>(getUnderlyingObject(Op1, DL)))
       if (Instruction *New = foldAllocaCmp(I, Alloca, Op0))
         return New;
   }
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
index e533198bd489d..c14a330542c9e 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
@@ -2849,8 +2849,9 @@ Instruction *InstCombinerImpl::visitSelectInst(SelectInst &SI) {
         return replaceOperand(SI, 1, TrueSI->getTrueValue());
       }
       // select(C0, select(C1, a, b), b) -> select(C0&C1, a, b)
-      // We choose this as normal form to enable folding on the And and shortening
-      // paths for the values (this helps GetUnderlyingObjects() for example).
+      // We choose this as normal form to enable folding on the And and
+      // shortening paths for the values (this helps getUnderlyingObjects() for
+      // example).
       if (TrueSI->getFalseValue() == FalseVal && TrueSI->hasOneUse()) {
         Value *And = Builder.CreateAnd(CondVal, TrueSI->getCondition());
         replaceOperand(SI, 0, And);
diff --git a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
index 053b9aa812bf4..0b542b638aee3 100644
--- a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
@@ -1556,7 +1556,7 @@ void AddressSanitizer::instrumentMop(ObjectSizeOffsetVisitor &ObjSizeVis,
   if (ClOpt && ClOptGlobals) {
     // If initialization order checking is disabled, a simple access to a
     // dynamically initialized global is always valid.
-    GlobalVariable *G = dyn_cast<GlobalVariable>(GetUnderlyingObject(Addr, DL));
+    GlobalVariable *G = dyn_cast<GlobalVariable>(getUnderlyingObject(Addr, DL));
     if (G && (!ClInitializers || GlobalIsLinkerInitialized(G)) &&
         isSafeAccess(ObjSizeVis, Addr, O.TypeSize)) {
       NumOptimizedAccessesToGlobalVar++;
@@ -1566,7 +1566,7 @@ void AddressSanitizer::instrumentMop(ObjectSizeOffsetVisitor &ObjSizeVis,
 
   if (ClOpt && ClOptStack) {
     // A direct inbounds access to a stack variable is always valid.
-    if (isa<AllocaInst>(GetUnderlyingObject(Addr, DL)) &&
+    if (isa<AllocaInst>(getUnderlyingObject(Addr, DL)) &&
         isSafeAccess(ObjSizeVis, Addr, O.TypeSize)) {
       NumOptimizedAccessesToStackVar++;
       return;
diff --git a/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
index feb1abb18777b..b1f5de810110e 100644
--- a/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
@@ -1246,7 +1246,7 @@ Value *DFSanFunction::loadShadow(Value *Addr, uint64_t Size, uint64_t Align,
 
   const llvm::Align ShadowAlign(Align * DFS.ShadowWidthBytes);
   SmallVector<const Value *, 2> Objs;
-  GetUnderlyingObjects(Addr, Objs, Pos->getModule()->getDataLayout());
+  getUnderlyingObjects(Addr, Objs, Pos->getModule()->getDataLayout());
   bool AllConstants = true;
   for (const Value *Obj : Objs) {
     if (isa<Function>(Obj) || isa<BlockAddress>(Obj))
diff --git a/llvm/lib/Transforms/Instrumentation/ThreadSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/ThreadSanitizer.cpp
index 8ce12c514f0bc..ca1535b3f9fac 100644
--- a/llvm/lib/Transforms/Instrumentation/ThreadSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/ThreadSanitizer.cpp
@@ -478,7 +478,7 @@ void ThreadSanitizer::chooseInstructionsToInstrument(
       }
     }
 
-    if (isa<AllocaInst>(GetUnderlyingObject(Addr, DL)) &&
+    if (isa<AllocaInst>(getUnderlyingObject(Addr, DL)) &&
         !PointerMayBeCaptured(Addr, true, true)) {
       // The variable is addressable but not captured, so it cannot be
       // referenced from a different thread and participate in a data race
diff --git a/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp b/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
index 258fd5b9454f0..ad80ef15362e5 100644
--- a/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
@@ -415,8 +415,8 @@ static OverwriteResult isOverwrite(const MemoryLocation &Later,
   // Check to see if the later store is to the entire object (either a global,
   // an alloca, or a byval/inalloca argument).  If so, then it clearly
   // overwrites any other store to the same object.
-  const Value *UO1 = GetUnderlyingObject(P1, DL),
-              *UO2 = GetUnderlyingObject(P2, DL);
+  const Value *UO1 = getUnderlyingObject(P1, DL),
+              *UO2 = getUnderlyingObject(P2, DL);
 
   // If we can't resolve the same pointers to the same object, then we can't
   // analyze them at all.
@@ -755,7 +755,7 @@ static bool handleFree(CallInst *F, AliasAnalysis *AA,
         break;
 
       Value *DepPointer =
-          GetUnderlyingObject(getStoredPointerOperand(Dependency), DL);
+          getUnderlyingObject(getStoredPointerOperand(Dependency), DL);
 
       // Check for aliasing.
       if (!AA->isMustAlias(F->getArgOperand(0), DepPointer))
@@ -795,7 +795,7 @@ static void removeAccessedObjects(const MemoryLocation &LoadedLoc,
                                   const DataLayout &DL, AliasAnalysis *AA,
                                   const TargetLibraryInfo *TLI,
                                   const Function *F) {
-  const Value *UnderlyingPointer = GetUnderlyingObject(LoadedLoc.Ptr, DL);
+  const Value *UnderlyingPointer = getUnderlyingObject(LoadedLoc.Ptr, DL);
 
   // A constant can't be in the dead pointer set.
   if (isa<Constant>(UnderlyingPointer))
@@ -861,7 +861,7 @@ static bool handleEndBlock(BasicBlock &BB, AliasAnalysis *AA,
     if (hasAnalyzableMemoryWrite(&*BBI, *TLI) && isRemovable(&*BBI)) {
       // See through pointer-to-pointer bitcasts
       SmallVector<const Value *, 4> Pointers;
-      GetUnderlyingObjects(getStoredPointerOperand(&*BBI), Pointers, DL);
+      getUnderlyingObjects(getStoredPointerOperand(&*BBI), Pointers, DL);
 
       // Stores to stack values are valid candidates for removal.
       bool AllDead = true;
@@ -1134,7 +1134,7 @@ static bool eliminateNoopStore(Instruction *Inst, BasicBlock::iterator &BBI,
   Constant *StoredConstant = dyn_cast<Constant>(SI->getValueOperand());
   if (StoredConstant && StoredConstant->isNullValue() && isRemovable(SI)) {
     Instruction *UnderlyingPointer =
-        dyn_cast<Instruction>(GetUnderlyingObject(SI->getPointerOperand(), DL));
+        dyn_cast<Instruction>(getUnderlyingObject(SI->getPointerOperand(), DL));
 
     if (UnderlyingPointer && isCallocLikeFn(UnderlyingPointer, TLI) &&
         memoryIsNotModifiedBetween(UnderlyingPointer, SI, AA, DL, DT)) {
@@ -1289,7 +1289,7 @@ static bool eliminateDeadStores(BasicBlock &BB, AliasAnalysis *AA,
       // to it is dead along the unwind edge. Otherwise, we need to preserve
       // the store.
       if (LastThrowing && DepWrite->comesBefore(LastThrowing)) {
-        const Value* Underlying = GetUnderlyingObject(DepLoc.Ptr, DL);
+        const Value *Underlying = getUnderlyingObject(DepLoc.Ptr, DL);
         bool IsStoreDeadOnUnwind = isa<AllocaInst>(Underlying);
         if (!IsStoreDeadOnUnwind) {
             // We're looking for a call to an allocation function
@@ -1715,7 +1715,7 @@ struct DSEState {
     // object can be considered terminated.
     if (MaybeTermLoc->second) {
       DataLayout DL = MaybeTerm->getParent()->getModule()->getDataLayout();
-      DefLoc = MemoryLocation(GetUnderlyingObject(DefLoc.Ptr, DL));
+      DefLoc = MemoryLocation(getUnderlyingObject(DefLoc.Ptr, DL));
     }
     return AA.isMustAlias(MaybeTermLoc->first, DefLoc);
   }
@@ -2047,7 +2047,7 @@ struct DSEState {
         Instruction *DefI = Def->getMemoryInst();
         // See through pointer-to-pointer bitcasts
         SmallVector<const Value *, 4> Pointers;
-        GetUnderlyingObjects(getLocForWriteEx(DefI)->Ptr, Pointers, DL);
+        getUnderlyingObjects(getLocForWriteEx(DefI)->Ptr, Pointers, DL);
 
         LLVM_DEBUG(dbgs() << "   ... MemoryDef is not accessed until the end "
                              "of the function\n");
@@ -2130,7 +2130,7 @@ bool eliminateDeadStoresMemorySSA(Function &F, AliasAnalysis &AA,
     }
     MemoryLocation SILoc = *MaybeSILoc;
     assert(SILoc.Ptr && "SILoc should not be null");
-    const Value *SILocUnd = GetUnderlyingObject(SILoc.Ptr, DL);
+    const Value *SILocUnd = getUnderlyingObject(SILoc.Ptr, DL);
 
     // Check if the store is a no-op.
     if (isRemovable(SI) && State.storeIsNoop(KillingDef, SILoc, SILocUnd)) {
@@ -2231,7 +2231,7 @@ bool eliminateDeadStoresMemorySSA(Function &F, AliasAnalysis &AA,
       MemoryLocation NILoc = *State.getLocForWriteEx(NI);
 
       if (State.isMemTerminatorInst(SI)) {
-        const Value *NIUnd = GetUnderlyingObject(NILoc.Ptr, DL);
+        const Value *NIUnd = getUnderlyingObject(NILoc.Ptr, DL);
         if (!SILocUnd || SILocUnd != NIUnd)
           continue;
         LLVM_DEBUG(dbgs() << "DSE: Remove Dead Store:\n  DEAD: " << *NI
diff --git a/llvm/lib/Transforms/Scalar/LICM.cpp b/llvm/lib/Transforms/Scalar/LICM.cpp
index 1a22edaf87266..043d3683aa2ec 100644
--- a/llvm/lib/Transforms/Scalar/LICM.cpp
+++ b/llvm/lib/Transforms/Scalar/LICM.cpp
@@ -1909,7 +1909,7 @@ bool llvm::promoteLoopAccessesToScalars(
     // we have to prove that the store is dead along the unwind edge.  We do
     // this by proving that the caller can't have a reference to the object
     // after return and thus can't possibly load from the object.
-    Value *Object = GetUnderlyingObject(SomePtr, MDL);
+    Value *Object = getUnderlyingObject(SomePtr, MDL);
     if (!isKnownNonEscaping(Object, TLI))
       return false;
     // Subtlety: Alloca's aren't visible to callers, but *are* potentially
@@ -2041,7 +2041,7 @@ bool llvm::promoteLoopAccessesToScalars(
     if (IsKnownThreadLocalObject)
       SafeToInsertStore = true;
     else {
-      Value *Object = GetUnderlyingObject(SomePtr, MDL);
+      Value *Object = getUnderlyingObject(SomePtr, MDL);
       SafeToInsertStore =
           (isAllocLikeFn(Object, TLI) || isa<AllocaInst>(Object)) &&
           !PointerMayBeCaptured(Object, true, true);
diff --git a/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp b/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
index fc980cf3689fb..99d3519eff153 100644
--- a/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
@@ -564,12 +564,12 @@ void LoopIdiomRecognize::collectStores(BasicBlock *BB) {
       break;
     case LegalStoreKind::Memset: {
       // Find the base pointer.
-      Value *Ptr = GetUnderlyingObject(SI->getPointerOperand(), *DL);
+      Value *Ptr = getUnderlyingObject(SI->getPointerOperand(), *DL);
       StoreRefsForMemset[Ptr].push_back(SI);
     } break;
     case LegalStoreKind::MemsetPattern: {
       // Find the base pointer.
-      Value *Ptr = GetUnderlyingObject(SI->getPointerOperand(), *DL);
+      Value *Ptr = getUnderlyingObject(SI->getPointerOperand(), *DL);
       StoreRefsForMemsetPattern[Ptr].push_back(SI);
     } break;
     case LegalStoreKind::Memcpy:
diff --git a/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp b/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp
index a9332356b3713..e0cd0253f9bf6 100644
--- a/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp
+++ b/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp
@@ -1562,7 +1562,7 @@ class LowerMatrixIntrinsics {
       if (Value *Ptr = getPointerOperand(V))
         return getUnderlyingObjectThroughLoads(Ptr);
       else if (V->getType()->isPointerTy())
-        return GetUnderlyingObject(V, DL);
+        return getUnderlyingObject(V, DL);
       return V;
     }
 
diff --git a/llvm/lib/Transforms/Utils/AssumeBundleBuilder.cpp b/llvm/lib/Transforms/Utils/AssumeBundleBuilder.cpp
index 7ff73fcdada79..762158f7e19ed 100644
--- a/llvm/lib/Transforms/Utils/AssumeBundleBuilder.cpp
+++ b/llvm/lib/Transforms/Utils/AssumeBundleBuilder.cpp
@@ -69,7 +69,7 @@ RetainedKnowledge canonicalizedKnowledge(RetainedKnowledge RK, Module *M) {
   default:
     return RK;
   case Attribute::NonNull:
-    RK.WasOn = GetUnderlyingObject(RK.WasOn, M->getDataLayout());
+    RK.WasOn = getUnderlyingObject(RK.WasOn, M->getDataLayout());
     return RK;
   case Attribute::Alignment: {
     Value *V = RK.WasOn->stripInBoundsOffsets([&](const Value *Strip) {
@@ -145,7 +145,7 @@ struct AssumeBuilderState {
     if (!RK.WasOn)
       return true;
     if (RK.WasOn->getType()->isPointerTy()) {
-      Value *UnderlyingPtr = GetUnderlyingObject(RK.WasOn, M->getDataLayout());
+      Value *UnderlyingPtr = getUnderlyingObject(RK.WasOn, M->getDataLayout());
       if (isa<AllocaInst>(UnderlyingPtr) || isa<GlobalValue>(UnderlyingPtr))
         return false;
     }
diff --git a/llvm/lib/Transforms/Utils/InlineFunction.cpp b/llvm/lib/Transforms/Utils/InlineFunction.cpp
index 5c9cb1245d01c..090ba9daa678f 100644
--- a/llvm/lib/Transforms/Utils/InlineFunction.cpp
+++ b/llvm/lib/Transforms/Utils/InlineFunction.cpp
@@ -1037,7 +1037,7 @@ static void AddAliasScopeMetadata(CallBase &CB, ValueToValueMapTy &VMap,
       SmallSetVector<const Argument *, 4> NAPtrArgs;
       for (const Value *V : PtrArgs) {
         SmallVector<const Value *, 4> Objects;
-        GetUnderlyingObjects(V, Objects, DL, /* LI = */ nullptr);
+        getUnderlyingObjects(V, Objects, DL, /* LI = */ nullptr);
 
         for (const Value *O : Objects)
           ObjSet.insert(O);
diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp
index 43363736684ee..d7cd9b19b8d51 100644
--- a/llvm/lib/Transforms/Utils/LoopUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp
@@ -1145,7 +1145,7 @@ static bool isValidRewrite(ScalarEvolution *SE, Value *FromVal, Value *ToVal) {
   // producing an expression involving multiple pointers. Until then, we must
   // bail out here.
   //
-  // Retrieve the pointer operand of the GEP. Don't use GetUnderlyingObject
+  // Retrieve the pointer operand of the GEP. Don't use getUnderlyingObject
   // because it understands lcssa phis while SCEV does not.
   Value *FromPtr = FromVal;
   Value *ToPtr = ToVal;
@@ -1162,7 +1162,7 @@ static bool isValidRewrite(ScalarEvolution *SE, Value *FromVal, Value *ToVal) {
 
     // SCEV may have rewritten an expression that produces the GEP's pointer
     // operand. That's ok as long as the pointer operand has the same base
-    // pointer. Unlike GetUnderlyingObject(), getPointerBase() will find the
+    // pointer. Unlike getUnderlyingObject(), getPointerBase() will find the
     // base of a recurrence. This handles the case in which SCEV expansion
     // converts a pointer type recurrence into a nonrecurrent pointer base
     // indexed by an integer recurrence.
diff --git a/llvm/lib/Transforms/Utils/VNCoercion.cpp b/llvm/lib/Transforms/Utils/VNCoercion.cpp
index 6ff08cd287124..11b079d3c0503 100644
--- a/llvm/lib/Transforms/Utils/VNCoercion.cpp
+++ b/llvm/lib/Transforms/Utils/VNCoercion.cpp
@@ -393,7 +393,7 @@ int analyzeLoadFromClobberingMemInst(Type *LoadTy, Value *LoadPtr,
   if (!Src)
     return -1;
 
-  GlobalVariable *GV = dyn_cast<GlobalVariable>(GetUnderlyingObject(Src, DL));
+  GlobalVariable *GV = dyn_cast<GlobalVariable>(getUnderlyingObject(Src, DL));
   if (!GV || !GV->isConstant() || !GV->hasDefinitiveInitializer())
     return -1;
 
diff --git a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
index 9b81afbb4b6cb..0b10e50bd1f77 100644
--- a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
@@ -763,7 +763,7 @@ Vectorizer::getVectorizablePrefix(ArrayRef<Instruction *> Chain) {
 }
 
 static ChainID getChainID(const Value *Ptr, const DataLayout &DL) {
-  const Value *ObjPtr = GetUnderlyingObject(Ptr, DL);
+  const Value *ObjPtr = getUnderlyingObject(Ptr, DL);
   if (const auto *Sel = dyn_cast<SelectInst>(ObjPtr)) {
     // The select's themselves are distinct instructions even if they share the
     // same condition and evaluate to consecutive pointers for true and false
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index dd6ff20b1defa..08c0021a41262 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -5912,7 +5912,7 @@ void SLPVectorizerPass::collectSeedInstructions(BasicBlock *BB) {
         continue;
       if (!isValidElementType(SI->getValueOperand()->getType()))
         continue;
-      Stores[GetUnderlyingObject(SI->getPointerOperand(), *DL)].push_back(SI);
+      Stores[getUnderlyingObject(SI->getPointerOperand(), *DL)].push_back(SI);
     }
 
     // Ignore getelementptr instructions that have more than one index, a
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/nullptr.ll b/llvm/test/Analysis/LoopAccessAnalysis/nullptr.ll
index 476cf3208c84c..8fbf47304e800 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/nullptr.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/nullptr.ll
@@ -2,9 +2,9 @@
 ; RUN: opt -passes='require<scalar-evolution>,require<aa>,loop(print-access-info)' -disable-output  < %s 2>&1 | FileCheck %s
 
 ; Test that the loop accesses are proven safe in this case.
-; The analyzer uses to be confused by the "diamond" because GetUnderlyingObjects
+; The analyzer uses to be confused by the "diamond" because getUnderlyingObjects
 ; is saying that the two pointers can both points to null. The loop analyzer
-; needs to ignore null in the results returned by GetUnderlyingObjects.
+; needs to ignore null in the results returned by getUnderlyingObjects.
 
 ; CHECK: Memory dependences are safe with run-time checks
 
diff --git a/llvm/test/CodeGen/AMDGPU/promote-alloca-to-lds-phi.ll b/llvm/test/CodeGen/AMDGPU/promote-alloca-to-lds-phi.ll
index 2fecbcb9d7a6b..bd37356b15ecb 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-alloca-to-lds-phi.ll
+++ b/llvm/test/CodeGen/AMDGPU/promote-alloca-to-lds-phi.ll
@@ -129,7 +129,7 @@ endif:
 ; }
 
 ; FIXME: This should be promotable. We need to use
-; GetUnderlyingObjects when looking at the icmp user.
+; getUnderlyingObjects when looking at the icmp user.
 
 ; CHECK-LABEL: @ptr_induction_var_same_alloca(
 ; CHECK: %alloca = alloca [64 x i32], align 4
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/address-space-ptr-sze-gep-index-assert.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/address-space-ptr-sze-gep-index-assert.ll
index 4c904b66f3d30..4f85482e3ae00 100644
--- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/address-space-ptr-sze-gep-index-assert.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/address-space-ptr-sze-gep-index-assert.ll
@@ -65,7 +65,7 @@ bb:
   ret void
 }
 
-; This should vectorize if using GetUnderlyingObject
+; This should vectorize if using getUnderlyingObject
 define void @multi_as_reduction_same_size(i32 addrspace(1)* %global, i64 %idx0, i64 %idx1) #0 {
 ; CHECK-LABEL: @multi_as_reduction_same_size(
 ; CHECK-NEXT:  bb:
@@ -106,7 +106,7 @@ bb:
   ret void
 }
 
-; This should vectorize if using GetUnderlyingObject
+; This should vectorize if using getUnderlyingObject
 ; The add is done in the same width, even though the address space size is smaller
 define void @multi_as_reduction_different_sized_noncanon(i32 addrspace(3)* %lds, i64 %idx0, i64 %idx1) #0 {
 ; CHECK-LABEL: @multi_as_reduction_different_sized_noncanon(

From abf546dd4f8326b821244b67eaa79ada7757d877 Mon Sep 17 00:00:00 2001
From: Jason Molenda <jason@molenda.com>
Date: Thu, 30 Jul 2020 20:55:41 -0700
Subject: [PATCH 0705/1035] debguserver's type sniffer to only treat .app
 things that end in .app

On an iOS device, if debugserver is left to figure out how to launch
the binary provided, it looks at the filename to see if it contains
".app" and asks FrontBoard to launch it.  However, if this is actually
a command line app with the characters ".app" in the name, it would
end up trying to launch that via the FrontBoard calls even though it
needed to be launched via posix_spawn.  For instance, a command line
program called com.application.tester.

Jim suggested this patch where we only send binaries that end in ".app"
to FrontBoard.

Often debugsever is invoked with a --launch command line argument to
specify the launch method, and none of this code is hit in that
instance.

<rdar://problem/65297100>
---
 lldb/tools/debugserver/source/debugserver.cpp | 26 ++++++++++++++-----
 1 file changed, 20 insertions(+), 6 deletions(-)

diff --git a/lldb/tools/debugserver/source/debugserver.cpp b/lldb/tools/debugserver/source/debugserver.cpp
index 410ea7c310fad..4e6aa39e52d37 100644
--- a/lldb/tools/debugserver/source/debugserver.cpp
+++ b/lldb/tools/debugserver/source/debugserver.cpp
@@ -156,6 +156,20 @@ RNBRunLoopMode RNBRunLoopGetStartModeFromRemote(RNBRemote *remote) {
   return eRNBRunLoopModeExit;
 }
 
+// Check the name to see if it ends with .app
+static bool is_dot_app (const char *app_name) {
+  size_t len = strlen(app_name);
+  if (len < 4)
+    return false;
+  
+  if (app_name[len - 4] == '.' &&
+      app_name[len - 3] == 'a' && 
+      app_name[len - 2] == 'p' &&
+      app_name[len - 1] == 'p')
+    return true;
+  return false;
+}
+
 // This run loop mode will wait for the process to launch and hit its
 // entry point. It will currently ignore all events except for the
 // process state changed event, where it watches for the process stopped
@@ -200,17 +214,17 @@ RNBRunLoopMode RNBRunLoopLaunchInferior(RNBRemote *remote,
 
 #if defined WITH_FBS
     // Check if we have an app bundle, if so launch using BackBoard Services.
-    if (strstr(inferior_argv[0], ".app")) {
+    if (is_dot_app(inferior_argv[0])) {
       launch_flavor = eLaunchFlavorFBS;
     }
 #elif defined WITH_BKS
     // Check if we have an app bundle, if so launch using BackBoard Services.
-    if (strstr(inferior_argv[0], ".app")) {
+    if (is_dot_app(inferior_argv[0])) {
       launch_flavor = eLaunchFlavorBKS;
     }
 #elif defined WITH_SPRINGBOARD
     // Check if we have an app bundle, if so launch using SpringBoard.
-    if (strstr(inferior_argv[0], ".app")) {
+    if (is_dot_app(inferior_argv[0])) {
       launch_flavor = eLaunchFlavorSpringBoard;
     }
 #endif
@@ -1499,17 +1513,17 @@ int main(int argc, char *argv[]) {
 
 #if defined WITH_FBS
           // Check if we have an app bundle, if so launch using SpringBoard.
-          if (waitfor_pid_name.find(".app") != std::string::npos) {
+          if (is_dot_app(waitfor_pid_name.c_str())) {
             launch_flavor = eLaunchFlavorFBS;
           }
 #elif defined WITH_BKS
           // Check if we have an app bundle, if so launch using SpringBoard.
-          if (waitfor_pid_name.find(".app") != std::string::npos) {
+          if (is_dot_app(waitfor_pid_name.c_str())) {
             launch_flavor = eLaunchFlavorBKS;
           }
 #elif defined WITH_SPRINGBOARD
           // Check if we have an app bundle, if so launch using SpringBoard.
-          if (waitfor_pid_name.find(".app") != std::string::npos) {
+          if (is_dot_app(waitfor_pid_name.c_str())) {
             launch_flavor = eLaunchFlavorSpringBoard;
           }
 #endif

From 836991d3676fda06d0f07c816ba855c8a0c3e0aa Mon Sep 17 00:00:00 2001
From: Wei Mi <wmi@google.com>
Date: Thu, 30 Jul 2020 17:47:27 -0700
Subject: [PATCH 0706/1035] Fix a crash when the sample profile uses md5 and
 -sample-profile-merge-inlinee is enabled.

When -sample-profile-merge-inlinee is enabled, new FunctionSamples may be
created during profile merge without GUIDToFuncNameMap being initialized.
That will occasionally cause compiler crash. The patch fixes it.

Differential Revision: https://reviews.llvm.org/D84994
---
 llvm/include/llvm/ProfileData/SampleProf.h        |   2 ++
 llvm/lib/Transforms/IPO/SampleProfile.cpp         |   2 ++
 .../Inputs/inline-mergeprof.md5.prof              | Bin 0 -> 372 bytes
 .../Transforms/SampleProfile/inline-mergeprof.ll  |   6 +++++-
 4 files changed, 9 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/Transforms/SampleProfile/Inputs/inline-mergeprof.md5.prof

diff --git a/llvm/include/llvm/ProfileData/SampleProf.h b/llvm/include/llvm/ProfileData/SampleProf.h
index 562468333ef47..fa5326038ada0 100644
--- a/llvm/include/llvm/ProfileData/SampleProf.h
+++ b/llvm/include/llvm/ProfileData/SampleProf.h
@@ -515,6 +515,8 @@ class FunctionSamples {
   sampleprof_error merge(const FunctionSamples &Other, uint64_t Weight = 1) {
     sampleprof_error Result = sampleprof_error::success;
     Name = Other.getName();
+    if (!GUIDToFuncNameMap)
+      GUIDToFuncNameMap = Other.GUIDToFuncNameMap;
     MergeResult(Result, addTotalSamples(Other.getTotalSamples(), Weight));
     MergeResult(Result, addHeadSamples(Other.getHeadSamples(), Weight));
     for (const auto &I : Other.getBodySamples()) {
diff --git a/llvm/lib/Transforms/IPO/SampleProfile.cpp b/llvm/lib/Transforms/IPO/SampleProfile.cpp
index b6871e260532d..ca60d35b8aafb 100644
--- a/llvm/lib/Transforms/IPO/SampleProfile.cpp
+++ b/llvm/lib/Transforms/IPO/SampleProfile.cpp
@@ -995,6 +995,8 @@ bool SampleProfileLoader::inlineHotFunctions(
         const FunctionSamples *FS = nullptr;
         if (auto *CB = dyn_cast<CallBase>(&I)) {
           if (!isa<IntrinsicInst>(I) && (FS = findCalleeFunctionSamples(*CB))) {
+            assert((!FunctionSamples::UseMD5 || FS->GUIDToFuncNameMap) &&
+                   "GUIDToFuncNameMap has to be populated");
             AllCandidates.push_back(CB);
             if (FS->getEntrySamples() > 0)
               localNotInlinedCallSites.try_emplace(CB, FS);
diff --git a/llvm/test/Transforms/SampleProfile/Inputs/inline-mergeprof.md5.prof b/llvm/test/Transforms/SampleProfile/Inputs/inline-mergeprof.md5.prof
new file mode 100644
index 0000000000000000000000000000000000000000..f8fa801beea7c52b237e5ece36beed0fa26fa3ed
GIT binary patch
literal 372
zcmZp9a$)0_lT%g%r?WCZ03(!!QJbKQWGKx9lK|2nwW3fBa!_#=BzX`sA1V(sUja?t
z7%FcA6=z11hv|n2w}frf7UN(NnBccjn{h!e8<0FOiw8(PSS|!48@5V-#1F}V#Luhr
zYcV~zqXi@zUKs$%1>enp<cGhOKyt%28z6b(j|~^won~7m_H8XdVn2vrzOmxs-Onw5
z&z)zyeP(Urq1)HbSN~eQeb2kLo2PCwGB9m^%*({YxKW#dnW0~cfsvV!QICm{K^({x
kWnct`4-+F$5d(u5BR4Z612>S##mK<L3<@nKCI&`L0O)2%U;qFB

literal 0
HcmV?d00001

diff --git a/llvm/test/Transforms/SampleProfile/inline-mergeprof.ll b/llvm/test/Transforms/SampleProfile/inline-mergeprof.ll
index d83fd23c33d36..dfd1f1f20209c 100644
--- a/llvm/test/Transforms/SampleProfile/inline-mergeprof.ll
+++ b/llvm/test/Transforms/SampleProfile/inline-mergeprof.ll
@@ -3,9 +3,13 @@
 ; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/inline-mergeprof.prof -sample-profile-merge-inlinee=true -S | FileCheck -check-prefix=SCALE %s
 ; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline-mergeprof.prof -sample-profile-merge-inlinee=false -S | FileCheck -check-prefix=SCALE %s
 
-; Test we properly merge not inlined profile properly with '-sample-profile-merge-inlinee'
+; Test we properly merge not inlined profile with '-sample-profile-merge-inlinee'
 ; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline-mergeprof.prof -sample-profile-merge-inlinee=true -S | FileCheck -check-prefix=MERGE  %s
 
+; Test we properly merge not inlined profile with '-sample-profile-merge-inlinee'
+; when the profile uses md5.
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline-mergeprof.md5.prof -sample-profile-merge-inlinee=true -S | FileCheck -check-prefix=MERGE  %s
+
 @.str = private unnamed_addr constant [11 x i8] c"sum is %d\0A\00", align 1
 
 define i32 @main() #0 !dbg !6 {

From d889e17eca8e8c43c6e4a2439fae5a1a40823623 Mon Sep 17 00:00:00 2001
From: Max Kazantsev <mkazantsev@azul.com>
Date: Fri, 31 Jul 2020 11:10:00 +0700
Subject: [PATCH 0707/1035] [SimpleLoopUnswitch] Drop make.implicit metadata in
 case of non-trivial unswitching

Non-trivial unswitching simply moves terminator being unswitch from the loop
up to the switch block. It also preserves all metadata that was there. It might not
be a correct thing to do for `make.implicit` metadata. Consider case:
```
for (...) {
  cond = // computed in loop
  if (cond) return X;
  if (p == null) throw_npe(); !make implicit
}
```
Before the unswitching, if `p` is null and we reach this check, we are guaranteed
to go to `throw_npe()` block. Now we unswitch on `p == null` condition:
```
if (p == null) !make implicit {
  for (...) {
    if (cond) return X;
    throw_npe()
  }
} else {
  for (...) {
    if (cond) return X;
  }
}
```
Now, following `true` branch of `p == null` does not always lead us to
`throw_npe()` because the loop has side exit. Now, if we run ImplicitNullCheck
pass on this code, it may end up making the unswitch condition implicit. This may
lead us to turning normal path to `return X` into signal-throwing path, which is
not efficient.

Note that this does not happen during trivial unswitch: it guarantees that we do not
have side exits before condition being unswitched.

This patch fixes this situation by unconditional dropping of `make.implicit` metadata
when we perform non-trivial unswitch. We could preserve it if we could prove that the
condition always executes. This can be done as a follow-up.

Differential Revision: https://reviews.llvm.org/D84916
Reviewed By: asbirlea
---
 .../Transforms/Scalar/SimpleLoopUnswitch.cpp  |   7 +
 .../implicit-null-checks.ll                   | 211 ++++++++++++++++++
 2 files changed, 218 insertions(+)
 create mode 100644 llvm/test/Transforms/SimpleLoopUnswitch/implicit-null-checks.ll

diff --git a/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp b/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp
index c7d23c9532910..c99fd634710aa 100644
--- a/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp
+++ b/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp
@@ -2070,6 +2070,13 @@ static void unswitchNontrivialInvariants(
         DominatingSucc, *VMaps.back(), DTUpdates, AC, DT, LI, MSSAU);
   }
 
+  // Drop metadata if we may break its semantics by moving this branch into the
+  // split block.
+  // TODO: We can keep make.implicit metadata if we prove that TI always
+  // executes and we cannot leave unswitched loop before getting to null check
+  // block. See @test_may_keep_make_implicit_non_trivial.
+  TI.setMetadata(LLVMContext::MD_make_implicit, nullptr);
+
   // The stitching of the branched code back together depends on whether we're
   // doing full unswitching or not with the exception that we always want to
   // nuke the initial terminator placed in the split block.
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/implicit-null-checks.ll b/llvm/test/Transforms/SimpleLoopUnswitch/implicit-null-checks.ll
new file mode 100644
index 0000000000000..32834d2428a47
--- /dev/null
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/implicit-null-checks.ll
@@ -0,0 +1,211 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -enable-nontrivial-unswitch=true -simple-loop-unswitch -S < %s | FileCheck %s
+; RUN: opt -enable-nontrivial-unswitch=true -passes='loop(unswitch),verify<loops>' -S < %s | FileCheck %s
+
+declare void @throw_npe()
+
+; It is illegal to preserve make_implicit notion of the condition being
+; unswitched because we may exit loop before we reach the condition, so
+; there is no guarantee that following implicit branch always means getting
+; to throw_npe block.
+define i32 @test_should_drop_make_implicit(i32* %p1, i32* %p2) {
+; CHECK-LABEL: @test_should_drop_make_implicit(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[NULL_CHECK:%.*]] = icmp eq i32* [[P2:%.*]], null
+; CHECK-NOT:     !make.implicit
+; CHECK-NEXT:    br i1 [[NULL_CHECK]], label [[ENTRY_SPLIT_US:%.*]], label [[ENTRY_SPLIT:%.*]]
+; CHECK:       entry.split.us:
+; CHECK-NEXT:    br label [[LOOP_US:%.*]]
+; CHECK:       loop.us:
+; CHECK-NEXT:    [[IV_US:%.*]] = phi i32 [ 0, [[ENTRY_SPLIT_US]] ]
+; CHECK-NEXT:    [[X_US:%.*]] = load i32, i32* [[P1:%.*]], align 4
+; CHECK-NEXT:    [[SIDE_EXIT_COND_US:%.*]] = icmp eq i32 [[X_US]], 0
+; CHECK-NEXT:    br i1 [[SIDE_EXIT_COND_US]], label [[SIDE_EXIT_SPLIT_US:%.*]], label [[NULL_CHECK_BLOCK_US:%.*]]
+; CHECK:       null_check_block.us:
+; CHECK-NEXT:    br label [[THROW_NPE_SPLIT_US:%.*]]
+; CHECK:       side_exit.split.us:
+; CHECK-NEXT:    br label [[SIDE_EXIT:%.*]]
+; CHECK:       throw_npe.split.us:
+; CHECK-NEXT:    br label [[THROW_NPE:%.*]]
+; CHECK:       entry.split:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY_SPLIT]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
+; CHECK-NEXT:    [[X:%.*]] = load i32, i32* [[P1]], align 4
+; CHECK-NEXT:    [[SIDE_EXIT_COND:%.*]] = icmp eq i32 [[X]], 0
+; CHECK-NEXT:    br i1 [[SIDE_EXIT_COND]], label [[SIDE_EXIT_SPLIT:%.*]], label [[NULL_CHECK_BLOCK:%.*]]
+; CHECK:       null_check_block:
+; CHECK-NEXT:    br label [[BACKEDGE]]
+; CHECK:       backedge:
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    [[LOOP_COND:%.*]] = icmp slt i32 [[IV_NEXT]], 10000
+; CHECK-NEXT:    br i1 [[LOOP_COND]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       side_exit.split:
+; CHECK-NEXT:    br label [[SIDE_EXIT]]
+; CHECK:       side_exit:
+; CHECK-NEXT:    ret i32 0
+; CHECK:       throw_npe:
+; CHECK-NEXT:    call void @throw_npe()
+; CHECK-NEXT:    unreachable
+; CHECK:       exit:
+; CHECK-NEXT:    [[X_LCSSA2:%.*]] = phi i32 [ [[X]], [[BACKEDGE]] ]
+; CHECK-NEXT:    ret i32 [[X_LCSSA2]]
+;
+entry:
+  %null_check = icmp eq i32* %p2, null
+  br label %loop
+loop:
+  %iv = phi i32 [0, %entry], [%iv.next, %backedge]
+  %x = load i32, i32* %p1
+  %side_exit_cond = icmp eq i32 %x, 0
+  br i1 %side_exit_cond, label %side_exit, label %null_check_block
+
+null_check_block:
+  br i1 %null_check, label %throw_npe, label %backedge, !make.implicit !0
+
+backedge:
+  %iv.next = add i32 %iv,1
+  %loop_cond = icmp slt i32 %iv.next, 10000
+  br i1 %loop_cond, label %loop, label %exit
+
+side_exit:
+  ret i32 0
+
+throw_npe:
+  call void @throw_npe()
+  unreachable
+
+exit:
+  ret i32 %x
+}
+
+; Here make.implicit notion may be preserved because we always get to throw_npe
+; after following true branch. This is a trivial unswitch.
+define i32 @test_may_keep_make_implicit_trivial(i32* %p1, i32* %p2) {
+; CHECK-LABEL: @test_may_keep_make_implicit_trivial(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[NULL_CHECK:%.*]] = icmp eq i32* [[P2:%.*]], null
+; CHECK-NEXT:    br i1 [[NULL_CHECK]], label [[THROW_NPE:%.*]], label [[ENTRY_SPLIT:%.*]], !make.implicit !0
+; CHECK:       entry.split:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY_SPLIT]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
+; CHECK-NEXT:    [[X:%.*]] = load i32, i32* [[P1:%.*]], align 4
+; CHECK-NEXT:    [[SIDE_EXIT_COND:%.*]] = icmp eq i32 [[X]], 0
+; CHECK-NEXT:    br label [[SIDE_EXIT_BLOCK:%.*]]
+; CHECK:       side_exit_block:
+; CHECK-NEXT:    br i1 [[SIDE_EXIT_COND]], label [[SIDE_EXIT:%.*]], label [[BACKEDGE]]
+; CHECK:       backedge:
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    [[LOOP_COND:%.*]] = icmp slt i32 [[IV_NEXT]], 10000
+; CHECK-NEXT:    br i1 [[LOOP_COND]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       side_exit:
+; CHECK-NEXT:    ret i32 0
+; CHECK:       throw_npe:
+; CHECK-NEXT:    call void @throw_npe()
+; CHECK-NEXT:    unreachable
+; CHECK:       exit:
+; CHECK-NEXT:    [[X_LCSSA2:%.*]] = phi i32 [ [[X]], [[BACKEDGE]] ]
+; CHECK-NEXT:    ret i32 [[X_LCSSA2]]
+;
+entry:
+  %null_check = icmp eq i32* %p2, null
+  br label %loop
+loop:
+  %iv = phi i32 [0, %entry], [%iv.next, %backedge]
+  %x = load i32, i32* %p1
+  %side_exit_cond = icmp eq i32 %x, 0
+  br i1 %null_check, label %throw_npe, label %side_exit_block, !make.implicit !0
+
+side_exit_block:
+  br i1 %side_exit_cond, label %side_exit, label %backedge
+
+backedge:
+  %iv.next = add i32 %iv,1
+  %loop_cond = icmp slt i32 %iv.next, 10000
+  br i1 %loop_cond, label %loop, label %exit
+
+side_exit:
+  ret i32 0
+
+throw_npe:
+  call void @throw_npe()
+  unreachable
+
+exit:
+  ret i32 %x
+}
+
+; TODO: This is a non-trivial unswitch, but it would still be legal to keep
+; !make.implicit in entry block. Currently we do not have enough analysis to
+; prove it.
+define i32 @test_may_keep_make_implicit_non_trivial(i32* %p1, i32* %p2) {
+; CHECK-LABEL: @test_may_keep_make_implicit_non_trivial(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[NULL_CHECK:%.*]] = icmp eq i32* [[P2:%.*]], null
+; CHECK-NOT:     !make.implicit
+; CHECK-NEXT:    br i1 [[NULL_CHECK]], label [[ENTRY_SPLIT_US:%.*]], label [[ENTRY_SPLIT:%.*]]
+; CHECK:       entry.split.us:
+; CHECK-NEXT:    br label [[LOOP_US:%.*]]
+; CHECK:       loop.us:
+; CHECK-NEXT:    [[IV_US:%.*]] = phi i32 [ 0, [[ENTRY_SPLIT_US]] ]
+; CHECK-NEXT:    [[X_US:%.*]] = load i32, i32* [[P1:%.*]], align 4
+; CHECK-NEXT:    [[INNER_BLOCK_COND_US:%.*]] = icmp eq i32 [[X_US]], 0
+; CHECK-NEXT:    br i1 [[INNER_BLOCK_COND_US]], label [[INNER_BLOCK_US:%.*]], label [[NULL_CHECK_BLOCK_US:%.*]]
+; CHECK:       inner_block.us:
+; CHECK-NEXT:    br label [[NULL_CHECK_BLOCK_US]]
+; CHECK:       null_check_block.us:
+; CHECK-NEXT:    br label [[THROW_NPE_SPLIT_US:%.*]]
+; CHECK:       throw_npe.split.us:
+; CHECK-NEXT:    br label [[THROW_NPE:%.*]]
+; CHECK:       entry.split:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY_SPLIT]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
+; CHECK-NEXT:    [[X:%.*]] = load i32, i32* [[P1]], align 4
+; CHECK-NEXT:    [[INNER_BLOCK_COND:%.*]] = icmp eq i32 [[X]], 0
+; CHECK-NEXT:    br i1 [[INNER_BLOCK_COND]], label [[INNER_BLOCK:%.*]], label [[NULL_CHECK_BLOCK:%.*]]
+; CHECK:       inner_block:
+; CHECK-NEXT:    br label [[NULL_CHECK_BLOCK]]
+; CHECK:       null_check_block:
+; CHECK-NEXT:    br label [[BACKEDGE]]
+; CHECK:       backedge:
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    [[LOOP_COND:%.*]] = icmp slt i32 [[IV_NEXT]], 10000
+; CHECK-NEXT:    br i1 [[LOOP_COND]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       throw_npe:
+; CHECK-NEXT:    call void @throw_npe()
+; CHECK-NEXT:    unreachable
+; CHECK:       exit:
+; CHECK-NEXT:    [[X_LCSSA1:%.*]] = phi i32 [ [[X]], [[BACKEDGE]] ]
+; CHECK-NEXT:    ret i32 [[X_LCSSA1]]
+;
+entry:
+  %null_check = icmp eq i32* %p2, null
+  br label %loop
+loop:
+  %iv = phi i32 [0, %entry], [%iv.next, %backedge]
+  %x = load i32, i32* %p1
+  %inner_block_cond = icmp eq i32 %x, 0
+  br i1 %inner_block_cond, label %inner_block, label %null_check_block
+
+inner_block:
+  br label %null_check_block
+
+null_check_block:
+  br i1 %null_check, label %throw_npe, label %backedge, !make.implicit !0
+
+backedge:
+  %iv.next = add i32 %iv,1
+  %loop_cond = icmp slt i32 %iv.next, 10000
+  br i1 %loop_cond, label %loop, label %exit
+
+throw_npe:
+  call void @throw_npe()
+  unreachable
+
+exit:
+  ret i32 %x
+}
+
+!0 = !{}

From 8aaeee5fb6d7c513d065e55b4bc2ad39bd6e760a Mon Sep 17 00:00:00 2001
From: Max Kazantsev <mkazantsev@azul.com>
Date: Fri, 31 Jul 2020 11:38:43 +0700
Subject: [PATCH 0708/1035] [SimpleLoopUnswitch] Preserve make.implicit in
 non-trivial unswitch if legal

We can preserve make.implicit metadata in the split block if it is
guaranteed that after following the branch we always reach the block
where processing of null case happens, which is equivalent to
"initial condition must execute if the loop is entered".

Differential Revision: https://reviews.llvm.org/D84925
Reviewed By: asbirlea
---
 .../Transforms/Scalar/SimpleLoopUnswitch.cpp  |  15 ++-
 .../implicit-null-checks.ll                   | 114 +++++++++++++++++-
 2 files changed, 119 insertions(+), 10 deletions(-)

diff --git a/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp b/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp
index c99fd634710aa..b1cbc714ec8e1 100644
--- a/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp
+++ b/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp
@@ -26,6 +26,7 @@
 #include "llvm/Analysis/LoopPass.h"
 #include "llvm/Analysis/MemorySSA.h"
 #include "llvm/Analysis/MemorySSAUpdater.h"
+#include "llvm/Analysis/MustExecute.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/Constant.h"
 #include "llvm/IR/Constants.h"
@@ -2070,12 +2071,16 @@ static void unswitchNontrivialInvariants(
         DominatingSucc, *VMaps.back(), DTUpdates, AC, DT, LI, MSSAU);
   }
 
-  // Drop metadata if we may break its semantics by moving this branch into the
+  // Drop metadata if we may break its semantics by moving this instr into the
   // split block.
-  // TODO: We can keep make.implicit metadata if we prove that TI always
-  // executes and we cannot leave unswitched loop before getting to null check
-  // block. See @test_may_keep_make_implicit_non_trivial.
-  TI.setMetadata(LLVMContext::MD_make_implicit, nullptr);
+  if (TI.getMetadata(LLVMContext::MD_make_implicit)) {
+    // It is only legal to preserve make.implicit metadata if we are guaranteed
+    // to reach implicit null check block after following this branch.
+    ICFLoopSafetyInfo SafetyInfo;
+    SafetyInfo.computeLoopSafetyInfo(&L);
+    if (!SafetyInfo.isGuaranteedToExecute(TI, &DT, &L))
+      TI.setMetadata(LLVMContext::MD_make_implicit, nullptr);
+  }
 
   // The stitching of the branched code back together depends on whether we're
   // doing full unswitching or not with the exception that we always want to
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/implicit-null-checks.ll b/llvm/test/Transforms/SimpleLoopUnswitch/implicit-null-checks.ll
index 32834d2428a47..002e57d54236f 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/implicit-null-checks.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/implicit-null-checks.ll
@@ -2,6 +2,7 @@
 ; RUN: opt -enable-nontrivial-unswitch=true -simple-loop-unswitch -S < %s | FileCheck %s
 ; RUN: opt -enable-nontrivial-unswitch=true -passes='loop(unswitch),verify<loops>' -S < %s | FileCheck %s
 
+declare void @may_exit()
 declare void @throw_npe()
 
 ; It is illegal to preserve make_implicit notion of the condition being
@@ -136,15 +137,11 @@ exit:
   ret i32 %x
 }
 
-; TODO: This is a non-trivial unswitch, but it would still be legal to keep
-; !make.implicit in entry block. Currently we do not have enough analysis to
-; prove it.
 define i32 @test_may_keep_make_implicit_non_trivial(i32* %p1, i32* %p2) {
 ; CHECK-LABEL: @test_may_keep_make_implicit_non_trivial(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[NULL_CHECK:%.*]] = icmp eq i32* [[P2:%.*]], null
-; CHECK-NOT:     !make.implicit
-; CHECK-NEXT:    br i1 [[NULL_CHECK]], label [[ENTRY_SPLIT_US:%.*]], label [[ENTRY_SPLIT:%.*]]
+; CHECK-NEXT:    br i1 [[NULL_CHECK]], label [[ENTRY_SPLIT_US:%.*]], label [[ENTRY_SPLIT:%.*]], !make.implicit !0
 ; CHECK:       entry.split.us:
 ; CHECK-NEXT:    br label [[LOOP_US:%.*]]
 ; CHECK:       loop.us:
@@ -208,4 +205,111 @@ exit:
   ret i32 %x
 }
 
+; Here make.implicit notion should be dropped because of exiting call.
+define i32 @test_should_drop_make_implicit_exiting_call(i32* %p1, i32* %p2) {
+; CHECK-LABEL: @test_should_drop_make_implicit_exiting_call(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[NULL_CHECK:%.*]] = icmp eq i32* [[P2:%.*]], null
+; CHECK-NOT:     !make.implicit
+; CHECK-NEXT:    br i1 [[NULL_CHECK]], label [[ENTRY_SPLIT_US:%.*]], label [[ENTRY_SPLIT:%.*]]
+; CHECK:       entry.split.us:
+; CHECK-NEXT:    br label [[LOOP_US:%.*]]
+; CHECK:       loop.us:
+; CHECK-NEXT:    [[IV_US:%.*]] = phi i32 [ 0, [[ENTRY_SPLIT_US]] ]
+; CHECK-NEXT:    call void @may_exit()
+; CHECK-NEXT:    [[X_US:%.*]] = load i32, i32* [[P1:%.*]], align 4
+; CHECK-NEXT:    [[SIDE_EXIT_COND_US:%.*]] = icmp eq i32 [[X_US]], 0
+; CHECK-NEXT:    br label [[THROW_NPE_SPLIT_US:%.*]]
+; CHECK:       throw_npe.split.us:
+; CHECK-NEXT:    br label [[THROW_NPE:%.*]]
+; CHECK:       entry.split:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY_SPLIT]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
+; CHECK-NEXT:    call void @may_exit()
+; CHECK-NEXT:    [[X:%.*]] = load i32, i32* [[P1]], align 4
+; CHECK-NEXT:    [[SIDE_EXIT_COND:%.*]] = icmp eq i32 [[X]], 0
+; CHECK-NEXT:    br label [[BACKEDGE]]
+; CHECK:       backedge:
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    [[LOOP_COND:%.*]] = icmp slt i32 [[IV_NEXT]], 10000
+; CHECK-NEXT:    br i1 [[LOOP_COND]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       throw_npe:
+; CHECK-NEXT:    call void @throw_npe()
+; CHECK-NEXT:    unreachable
+; CHECK:       exit:
+; CHECK-NEXT:    [[X_LCSSA1:%.*]] = phi i32 [ [[X]], [[BACKEDGE]] ]
+; CHECK-NEXT:    ret i32 [[X_LCSSA1]]
+;
+entry:
+  %null_check = icmp eq i32* %p2, null
+  br label %loop
+loop:
+  %iv = phi i32 [0, %entry], [%iv.next, %backedge]
+  call void @may_exit()
+  %x = load i32, i32* %p1
+  %side_exit_cond = icmp eq i32 %x, 0
+  br i1 %null_check, label %throw_npe, label %backedge, !make.implicit !0
+
+backedge:
+  %iv.next = add i32 %iv,1
+  %loop_cond = icmp slt i32 %iv.next, 10000
+  br i1 %loop_cond, label %loop, label %exit
+
+throw_npe:
+  call void @throw_npe()
+  unreachable
+
+exit:
+  ret i32 %x
+}
+
+; Here exiting call goes after the null check, so make.implicit may be preserved.
+define i32 @test_may_keep_make_implicit_exiting_call(i32* %p1, i32* %p2) {
+; CHECK-LABEL: @test_may_keep_make_implicit_exiting_call(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[NULL_CHECK:%.*]] = icmp eq i32* [[P2:%.*]], null
+; CHECK-NEXT:    br i1 [[NULL_CHECK]], label [[THROW_NPE:%.*]], label [[ENTRY_SPLIT:%.*]], !make.implicit !0
+; CHECK:       entry.split:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY_SPLIT]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
+; CHECK-NEXT:    [[X:%.*]] = load i32, i32* [[P1:%.*]], align 4
+; CHECK-NEXT:    [[SIDE_EXIT_COND:%.*]] = icmp eq i32 [[X]], 0
+; CHECK-NEXT:    br label [[BACKEDGE]]
+; CHECK:       backedge:
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    [[LOOP_COND:%.*]] = icmp slt i32 [[IV_NEXT]], 10000
+; CHECK-NEXT:    call void @may_exit()
+; CHECK-NEXT:    br i1 [[LOOP_COND]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       throw_npe:
+; CHECK-NEXT:    call void @throw_npe()
+; CHECK-NEXT:    unreachable
+; CHECK:       exit:
+; CHECK-NEXT:    [[X_LCSSA1:%.*]] = phi i32 [ [[X]], [[BACKEDGE]] ]
+; CHECK-NEXT:    ret i32 [[X_LCSSA1]]
+;
+entry:
+  %null_check = icmp eq i32* %p2, null
+  br label %loop
+loop:
+  %iv = phi i32 [0, %entry], [%iv.next, %backedge]
+  %x = load i32, i32* %p1
+  %side_exit_cond = icmp eq i32 %x, 0
+  br i1 %null_check, label %throw_npe, label %backedge, !make.implicit !0
+
+backedge:
+  %iv.next = add i32 %iv,1
+  %loop_cond = icmp slt i32 %iv.next, 10000
+  call void @may_exit()
+  br i1 %loop_cond, label %loop, label %exit
+
+throw_npe:
+  call void @throw_npe()
+  unreachable
+
+exit:
+  ret i32 %x
+}
+
 !0 = !{}

From 30a0dbb70d9e0c94a9bdc3d28e7c1c57d9891912 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@intel.com>
Date: Thu, 30 Jul 2020 21:50:17 -0700
Subject: [PATCH 0709/1035] [X86] Remove x86_sse42_crc32_64_64 from
 X86TTIImpl::simplifyDemandedUseBitsIntrinsic

It doesn't do any simplifying. It just computes known bits. We
can just let InstCombine call computeKnownBits which will handle
this just as well.
---
 llvm/lib/Target/X86/X86InstCombineIntrinsic.cpp | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/llvm/lib/Target/X86/X86InstCombineIntrinsic.cpp b/llvm/lib/Target/X86/X86InstCombineIntrinsic.cpp
index 91341f93a9943..aea355a297318 100644
--- a/llvm/lib/Target/X86/X86InstCombineIntrinsic.cpp
+++ b/llvm/lib/Target/X86/X86InstCombineIntrinsic.cpp
@@ -1785,10 +1785,6 @@ Optional<Value *> X86TTIImpl::simplifyDemandedUseBitsIntrinsic(
     KnownBitsComputed = true;
     break;
   }
-  case Intrinsic::x86_sse42_crc32_64_64:
-    Known.Zero.setBitsFrom(32);
-    KnownBitsComputed = true;
-    break;
   }
   return None;
 }

From c8fef0cc54faf51b769db3270c8fdbe6c65ef13d Mon Sep 17 00:00:00 2001
From: Lang Hames <lhames@gmail.com>
Date: Thu, 30 Jul 2020 22:46:04 -0700
Subject: [PATCH 0710/1035] [JITLink] Use correct Addressable constructor.

Calling createAddressable(false) generates an absolute symbol. We want
createAddressable(0, false), which generates an external symbol.
---
 llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h b/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h
index 76f1eb6356576..5e22837fa9309 100644
--- a/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h
+++ b/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h
@@ -963,7 +963,7 @@ class LinkGraph {
       Section &Sec = Sym.getBlock().getSection();
       Sec.removeSymbol(Sym);
     }
-    Sym.makeExternal(createAddressable(false));
+    Sym.makeExternal(createAddressable(0, false));
     ExternalSymbols.insert(&Sym);
   }
 

From f561713d7513110b6e791959734cca9e0bf7fe33 Mon Sep 17 00:00:00 2001
From: Juneyoung Lee <aqjune@gmail.com>
Date: Fri, 31 Jul 2020 15:20:38 +0900
Subject: [PATCH 0711/1035] [JumpThreading] Add a test for D84944 ; NFC

---
 .../test/Transforms/JumpThreading/pre-load.ll | 84 +++++++++++++++++++
 1 file changed, 84 insertions(+)
 create mode 100644 llvm/test/Transforms/JumpThreading/pre-load.ll

diff --git a/llvm/test/Transforms/JumpThreading/pre-load.ll b/llvm/test/Transforms/JumpThreading/pre-load.ll
new file mode 100644
index 0000000000000..0a8af988477fd
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/pre-load.ll
@@ -0,0 +1,84 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -jump-threading -S < %s | FileCheck %s
+
+@x = global i32 0
+@y = global i32 0
+
+declare void @f()
+declare void @g()
+
+define i32 @pre(i1 %cond, i32 %n) {
+; CHECK-LABEL: @pre(
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[C_THREAD:%.*]], label [[C:%.*]]
+; CHECK:       C.thread:
+; CHECK-NEXT:    store i32 0, i32* @x, align 4
+; CHECK-NEXT:    br label [[YES:%.*]]
+; CHECK:       C:
+; CHECK-NEXT:    [[A_PR:%.*]] = load i32, i32* @y, align 4
+; CHECK-NEXT:    [[COND2:%.*]] = icmp eq i32 [[A_PR]], 0
+; CHECK-NEXT:    br i1 [[COND2]], label [[YES]], label [[NO:%.*]]
+; CHECK:       YES:
+; CHECK-NEXT:    [[A4:%.*]] = phi i32 [ 0, [[C_THREAD]] ], [ [[A_PR]], [[C]] ]
+; CHECK-NEXT:    call void @f()
+; CHECK-NEXT:    ret i32 [[A4]]
+; CHECK:       NO:
+; CHECK-NEXT:    call void @g()
+; CHECK-NEXT:    ret i32 1
+;
+  br i1 %cond, label %A, label %B
+A:
+  store i32 0, i32* @x
+  br label %C
+B:
+  br label %C
+C:
+  %ptr = phi i32* [@x, %A], [@y, %B]
+  %a = load i32, i32* %ptr
+  %cond2 = icmp eq i32 %a, 0
+  br i1 %cond2, label %YES, label %NO
+YES:
+  call void @f()
+  ret i32 %a
+NO:
+  call void @g()
+  ret i32 1
+}
+
+define i32 @pre_freeze(i1 %cond, i32 %n) {
+; CHECK-LABEL: @pre_freeze(
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[A:%.*]], label [[C:%.*]]
+; CHECK:       A:
+; CHECK-NEXT:    store i32 0, i32* @x, align 4
+; CHECK-NEXT:    br label [[C]]
+; CHECK:       C:
+; CHECK-NEXT:    [[PTR:%.*]] = phi i32* [ @x, [[A]] ], [ @y, [[TMP0:%.*]] ]
+; CHECK-NEXT:    [[A:%.*]] = load i32, i32* [[PTR]], align 4
+; CHECK-NEXT:    [[COND2:%.*]] = icmp eq i32 [[A]], 0
+; CHECK-NEXT:    [[COND2_FR:%.*]] = freeze i1 [[COND2]]
+; CHECK-NEXT:    br i1 [[COND2_FR]], label [[YES:%.*]], label [[NO:%.*]]
+; CHECK:       YES:
+; CHECK-NEXT:    call void @f()
+; CHECK-NEXT:    ret i32 [[A]]
+; CHECK:       NO:
+; CHECK-NEXT:    call void @g()
+; CHECK-NEXT:    ret i32 1
+;
+  br i1 %cond, label %A, label %B
+A:
+  store i32 0, i32* @x
+  br label %C
+B:
+  br label %C
+C:
+  %ptr = phi i32* [@x, %A], [@y, %B]
+  %a = load i32, i32* %ptr
+  %cond2 = icmp eq i32 %a, 0
+  %cond2.fr = freeze i1 %cond2
+  br i1 %cond2.fr, label %YES, label %NO
+YES:
+  call void @f()
+  ret i32 %a
+NO:
+  call void @g()
+  ret i32 1
+}

From 1cc210383b281e9c78ff82c129d01efb598eee15 Mon Sep 17 00:00:00 2001
From: Fangrui Song <maskray@google.com>
Date: Thu, 30 Jul 2020 23:24:09 -0700
Subject: [PATCH 0712/1035] [MC] Support infix operator !

Disabled for Darwin mode.

Also disabled for ARM which has compatible aliases (implied 'sp' operand
in 'srs*' instructions like 'srsda #31!').
---
 llvm/include/llvm/MC/MCExpr.h       |  1 +
 llvm/lib/MC/MCExpr.cpp              |  2 ++
 llvm/lib/MC/MCParser/AsmParser.cpp  | 17 +++++++++++------
 llvm/lib/MC/MCParser/MasmParser.cpp |  2 --
 llvm/test/MC/AsmParser/exprs-gnu.s  |  4 ++++
 5 files changed, 18 insertions(+), 8 deletions(-)
 create mode 100644 llvm/test/MC/AsmParser/exprs-gnu.s

diff --git a/llvm/include/llvm/MC/MCExpr.h b/llvm/include/llvm/MC/MCExpr.h
index a424957bb0b22..843de5d5be92e 100644
--- a/llvm/include/llvm/MC/MCExpr.h
+++ b/llvm/include/llvm/MC/MCExpr.h
@@ -500,6 +500,7 @@ class MCBinaryExpr : public MCExpr {
     Mul,  ///< Multiplication.
     NE,   ///< Inequality comparison.
     Or,   ///< Bitwise or.
+    OrNot, ///< Bitwise or not.
     Shl,  ///< Shift left.
     AShr, ///< Arithmetic shift right.
     LShr, ///< Logical shift right.
diff --git a/llvm/lib/MC/MCExpr.cpp b/llvm/lib/MC/MCExpr.cpp
index 11fd32e48a4ae..a0aa7a3b95474 100644
--- a/llvm/lib/MC/MCExpr.cpp
+++ b/llvm/lib/MC/MCExpr.cpp
@@ -145,6 +145,7 @@ void MCExpr::print(raw_ostream &OS, const MCAsmInfo *MAI, bool InParens) const {
     case MCBinaryExpr::Mul:  OS <<  '*'; break;
     case MCBinaryExpr::NE:   OS << "!="; break;
     case MCBinaryExpr::Or:   OS <<  '|'; break;
+    case MCBinaryExpr::OrNot: OS << '!'; break;
     case MCBinaryExpr::Shl:  OS << "<<"; break;
     case MCBinaryExpr::Sub:  OS <<  '-'; break;
     case MCBinaryExpr::Xor:  OS <<  '^'; break;
@@ -920,6 +921,7 @@ bool MCExpr::evaluateAsRelocatableImpl(MCValue &Res, const MCAssembler *Asm,
     case MCBinaryExpr::Mul:  Result = LHS * RHS; break;
     case MCBinaryExpr::NE:   Result = LHS != RHS; break;
     case MCBinaryExpr::Or:   Result = LHS | RHS; break;
+    case MCBinaryExpr::OrNot: Result = LHS | ~RHS; break;
     case MCBinaryExpr::Shl:  Result = uint64_t(LHS) << uint64_t(RHS); break;
     case MCBinaryExpr::Sub:  Result = LHS - RHS; break;
     case MCBinaryExpr::Xor:  Result = LHS ^ RHS; break;
diff --git a/llvm/lib/MC/MCParser/AsmParser.cpp b/llvm/lib/MC/MCParser/AsmParser.cpp
index c05f26cbdda5c..497f73e411057 100644
--- a/llvm/lib/MC/MCParser/AsmParser.cpp
+++ b/llvm/lib/MC/MCParser/AsmParser.cpp
@@ -1497,8 +1497,6 @@ static unsigned getDarwinBinOpPrecedence(AsmToken::TokenKind K,
     return 1;
 
   // Low Precedence: |, &, ^
-  //
-  // FIXME: gas seems to support '!' as an infix operator?
   case AsmToken::Pipe:
     Kind = MCBinaryExpr::Or;
     return 2;
@@ -1559,7 +1557,8 @@ static unsigned getDarwinBinOpPrecedence(AsmToken::TokenKind K,
   }
 }
 
-static unsigned getGNUBinOpPrecedence(AsmToken::TokenKind K,
+static unsigned getGNUBinOpPrecedence(const MCAsmInfo &MAI,
+                                      AsmToken::TokenKind K,
                                       MCBinaryExpr::Opcode &Kind,
                                       bool ShouldUseLogicalShr) {
   switch (K) {
@@ -1603,12 +1602,18 @@ static unsigned getGNUBinOpPrecedence(AsmToken::TokenKind K,
     Kind = MCBinaryExpr::Sub;
     return 4;
 
-  // High Intermediate Precedence: |, &, ^
+  // High Intermediate Precedence: |, !, &, ^
   //
-  // FIXME: gas seems to support '!' as an infix operator?
   case AsmToken::Pipe:
     Kind = MCBinaryExpr::Or;
     return 5;
+  case AsmToken::Exclaim:
+    // Hack to support ARM compatible aliases (implied 'sp' operand in 'srs*'
+    // instructions like 'srsda #31!') and not parse ! as an infix operator.
+    if (MAI.getCommentString() == "@")
+      return 0;
+    Kind = MCBinaryExpr::OrNot;
+    return 5;
   case AsmToken::Caret:
     Kind = MCBinaryExpr::Xor;
     return 5;
@@ -1639,7 +1644,7 @@ unsigned AsmParser::getBinOpPrecedence(AsmToken::TokenKind K,
                                        MCBinaryExpr::Opcode &Kind) {
   bool ShouldUseLogicalShr = MAI.shouldUseLogicalShr();
   return IsDarwin ? getDarwinBinOpPrecedence(K, Kind, ShouldUseLogicalShr)
-                  : getGNUBinOpPrecedence(K, Kind, ShouldUseLogicalShr);
+                  : getGNUBinOpPrecedence(MAI, K, Kind, ShouldUseLogicalShr);
 }
 
 /// Parse all binary operators with precedence >= 'Precedence'.
diff --git a/llvm/lib/MC/MCParser/MasmParser.cpp b/llvm/lib/MC/MCParser/MasmParser.cpp
index fb3bb11d1f430..fb7aaae295dfa 100644
--- a/llvm/lib/MC/MCParser/MasmParser.cpp
+++ b/llvm/lib/MC/MCParser/MasmParser.cpp
@@ -1720,8 +1720,6 @@ static unsigned getGNUBinOpPrecedence(AsmToken::TokenKind K,
     return 4;
 
   // High Intermediate Precedence: |, &, ^
-  //
-  // FIXME: gas seems to support '!' as an infix operator?
   case AsmToken::Pipe:
     Kind = MCBinaryExpr::Or;
     return 5;
diff --git a/llvm/test/MC/AsmParser/exprs-gnu.s b/llvm/test/MC/AsmParser/exprs-gnu.s
new file mode 100644
index 0000000000000..3b5af48d36d4f
--- /dev/null
+++ b/llvm/test/MC/AsmParser/exprs-gnu.s
@@ -0,0 +1,4 @@
+# RUN: llvm-mc -triple=x86_64 %s | FileCheck %s
+
+# CHECK: movl %eax, 15
+movl %eax, 3 ! ~12

From ad48367722b23dd373e612c1c53b16290ae10194 Mon Sep 17 00:00:00 2001
From: Juneyoung Lee <aqjune@gmail.com>
Date: Fri, 31 Jul 2020 15:26:54 +0900
Subject: [PATCH 0713/1035] [JumpThreading] Let SimplifyPartiallyRedundantLoad
 look into freeze

This patch allows SimplifyPartiallyRedundantLoad work when
the branch condition was frozen.

Reviewed By: efriedma

Differential Revision: https://reviews.llvm.org/D84944
---
 llvm/lib/Transforms/Scalar/JumpThreading.cpp   |  5 +++++
 llvm/test/Transforms/JumpThreading/pre-load.ll | 16 ++++++++--------
 2 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/llvm/lib/Transforms/Scalar/JumpThreading.cpp b/llvm/lib/Transforms/Scalar/JumpThreading.cpp
index 37a588a582997..2a5bbfff1e1e7 100644
--- a/llvm/lib/Transforms/Scalar/JumpThreading.cpp
+++ b/llvm/lib/Transforms/Scalar/JumpThreading.cpp
@@ -1168,6 +1168,11 @@ bool JumpThreadingPass::ProcessBlock(BasicBlock *BB) {
   // we see one, check to see if it's partially redundant.  If so, insert a PHI
   // which can then be used to thread the values.
   Value *SimplifyValue = CondInst;
+
+  if (auto *FI = dyn_cast<FreezeInst>(SimplifyValue))
+    // Look into freeze's operand
+    SimplifyValue = FI->getOperand(0);
+
   if (CmpInst *CondCmp = dyn_cast<CmpInst>(SimplifyValue))
     if (isa<Constant>(CondCmp->getOperand(1)))
       SimplifyValue = CondCmp->getOperand(0);
diff --git a/llvm/test/Transforms/JumpThreading/pre-load.ll b/llvm/test/Transforms/JumpThreading/pre-load.ll
index 0a8af988477fd..d5df3acf6f7a1 100644
--- a/llvm/test/Transforms/JumpThreading/pre-load.ll
+++ b/llvm/test/Transforms/JumpThreading/pre-load.ll
@@ -46,19 +46,19 @@ NO:
 
 define i32 @pre_freeze(i1 %cond, i32 %n) {
 ; CHECK-LABEL: @pre_freeze(
-; CHECK-NEXT:    br i1 [[COND:%.*]], label [[A:%.*]], label [[C:%.*]]
-; CHECK:       A:
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[C_THREAD:%.*]], label [[C:%.*]]
+; CHECK:       C.thread:
 ; CHECK-NEXT:    store i32 0, i32* @x, align 4
-; CHECK-NEXT:    br label [[C]]
+; CHECK-NEXT:    br label [[YES:%.*]]
 ; CHECK:       C:
-; CHECK-NEXT:    [[PTR:%.*]] = phi i32* [ @x, [[A]] ], [ @y, [[TMP0:%.*]] ]
-; CHECK-NEXT:    [[A:%.*]] = load i32, i32* [[PTR]], align 4
-; CHECK-NEXT:    [[COND2:%.*]] = icmp eq i32 [[A]], 0
+; CHECK-NEXT:    [[A_PR:%.*]] = load i32, i32* @y, align 4
+; CHECK-NEXT:    [[COND2:%.*]] = icmp eq i32 [[A_PR]], 0
 ; CHECK-NEXT:    [[COND2_FR:%.*]] = freeze i1 [[COND2]]
-; CHECK-NEXT:    br i1 [[COND2_FR]], label [[YES:%.*]], label [[NO:%.*]]
+; CHECK-NEXT:    br i1 [[COND2_FR]], label [[YES]], label [[NO:%.*]]
 ; CHECK:       YES:
+; CHECK-NEXT:    [[A5:%.*]] = phi i32 [ 0, [[C_THREAD]] ], [ [[A_PR]], [[C]] ]
 ; CHECK-NEXT:    call void @f()
-; CHECK-NEXT:    ret i32 [[A]]
+; CHECK-NEXT:    ret i32 [[A5]]
 ; CHECK:       NO:
 ; CHECK-NEXT:    call void @g()
 ; CHECK-NEXT:    ret i32 1

From fcf8ada18f9cfb1261262e4b0399ae9ab40451f8 Mon Sep 17 00:00:00 2001
From: Fangrui Song <maskray@google.com>
Date: Thu, 30 Jul 2020 23:49:32 -0700
Subject: [PATCH 0714/1035] [Driver] Render -e for Gnu.cpp

---
 clang/include/clang/Driver/Options.td | 2 +-
 clang/test/Driver/Xlinker-args.c      | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 7d91bad1f848f..889035a0815e0 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -693,7 +693,7 @@ def emit_merged_ifs : Flag<["-"], "emit-merged-ifs">,
   HelpText<"Generate Interface Stub Files, emit merged text not binary.">;
 def interface_stub_version_EQ : JoinedOrSeparate<["-"], "interface-stub-version=">, Flags<[CC1Option]>;
 def exported__symbols__list : Separate<["-"], "exported_symbols_list">;
-def e : JoinedOrSeparate<["-"], "e">, Group<Link_Group>;
+def e : JoinedOrSeparate<["-"], "e">, Flags<[LinkerInput]>, Group<Link_Group>;
 def fmax_tokens_EQ : Joined<["-"], "fmax-tokens=">, Group<f_Group>, Flags<[CC1Option]>,
   HelpText<"Max total number of preprocessed tokens for -Wmax-tokens.">;
 def fPIC : Flag<["-"], "fPIC">, Group<f_Group>;
diff --git a/clang/test/Driver/Xlinker-args.c b/clang/test/Driver/Xlinker-args.c
index 72c2fd2698e88..a44957cd8aef1 100644
--- a/clang/test/Driver/Xlinker-args.c
+++ b/clang/test/Driver/Xlinker-args.c
@@ -8,13 +8,13 @@
 
 /// -T is reordered to the last to make sure -L takes precedence.
 // RUN: %clang -target x86_64-pc-linux-gnu -### \
-// RUN:   -T a.lds -Xlinker one -Xlinker --no-demangle \
+// RUN:   -e _start -T a.lds -Xlinker one -Xlinker --no-demangle \
 // RUN:   -Wl,two,--no-demangle,three -Xlinker four -z five -r %s 2> %t
 // RUN: FileCheck -check-prefix=LINUX < %t %s
 //
 // DARWIN-NOT: --no-demangle
 // DARWIN: "one" "two" "three" "four" "-z" "five" "-r"
-// LINUX: "--no-demangle" "one" "two" "three" "four" "-z" "five" "-r" {{.*}} "-T" "a.lds"
+// LINUX: "--no-demangle" "-e" "_start" "one" "two" "three" "four" "-z" "five" "-r" {{.*}} "-T" "a.lds"
 
 // Check that we forward '-Xlinker' and '-Wl,' on Windows.
 // RUN: %clang -target i686-pc-win32 -### \

From c06e22fe07aa5eb8a9f8ce824cbab1bfe5a96581 Mon Sep 17 00:00:00 2001
From: Fangrui Song <maskray@google.com>
Date: Fri, 31 Jul 2020 00:04:09 -0700
Subject: [PATCH 0715/1035] [Driver] Exclude options::LinkerInput for GCC
 linking

options::LinkerInput options may get duplicated after 6a75496836ea14bcfd2f4b59d35a1cad4ac58cee..
---
 clang/lib/Driver/ToolChains/Gnu.cpp | 3 +++
 clang/test/Driver/gcc_forward.c     | 4 ++--
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/clang/lib/Driver/ToolChains/Gnu.cpp b/clang/lib/Driver/ToolChains/Gnu.cpp
index 1806c14c395dc..9ca674511daba 100644
--- a/clang/lib/Driver/ToolChains/Gnu.cpp
+++ b/clang/lib/Driver/ToolChains/Gnu.cpp
@@ -38,6 +38,9 @@ using tools::addMultilibFlag;
 using tools::addPathIfExists;
 
 static bool forwardToGCC(const Option &O) {
+  // LinkerInput options have been forwarded. Don't duplicate.
+  if (O.hasFlag(options::LinkerInput))
+    return false;
   return O.matches(options::OPT_Link_Group) || O.hasFlag(options::LinkOption);
 }
 
diff --git a/clang/test/Driver/gcc_forward.c b/clang/test/Driver/gcc_forward.c
index 9579d0d60d1cf..a99944f8f5336 100644
--- a/clang/test/Driver/gcc_forward.c
+++ b/clang/test/Driver/gcc_forward.c
@@ -1,7 +1,7 @@
 // RUN: %clang -### %s -target aarch64-none-elf \
-// RUN:   --coverage -fuse-ld=lld --ld-path=ld -nostdlib -r -rdynamic -static -static-pie \
+// RUN:   --coverage -e _start -fuse-ld=lld --ld-path=ld -nostdlib -r -rdynamic -static -static-pie \
 // RUN:   2>&1 | FileCheck --check-prefix=FORWARD %s
-// FORWARD: gcc{{[^"]*}}" "--coverage" "-fuse-ld=lld" "--ld-path=ld" "-nostdlib" "-r" "-rdynamic" "-static" "-static-pie"
+// FORWARD: gcc{{[^"]*}}" "--coverage" "-fuse-ld=lld" "--ld-path=ld" "-nostdlib" "-rdynamic" "-static" "-static-pie" "-o" "a.out" "{{.*}}.o" "-e" "_start" "-r"
 
 // Check that we don't try to forward -Xclang or -mlinker-version to GCC.
 // PR12920 -- Check also we may not forward W_Group options to GCC.

From 9b04fec0021112d913058e81bf864e4f33c33dcc Mon Sep 17 00:00:00 2001
From: QingShan Zhang <qshanz@cn.ibm.com>
Date: Fri, 31 Jul 2020 07:05:45 +0000
Subject: [PATCH 0716/1035] [PowerPC] Retrieve the offset from load/store if it
 stores to stack slots

Scheduler will try to retrieve the offset and base addr to determine if two
loads/stores are disjoint memory access. PowerPC failed to handle this for
frame index which will bring extra memory dependency for loads/stores.

Reviewed By: jji

Differential Revision: https://reviews.llvm.org/D84308
---
 llvm/lib/Target/PowerPC/PPCInstrInfo.cpp      |  3 ++-
 .../CodeGen/PowerPC/aix-cc-byval-split.ll     | 10 +++----
 llvm/test/CodeGen/PowerPC/aix-cc-byval.ll     | 14 +++++-----
 .../CodeGen/PowerPC/aix32-cc-abi-vaarg.ll     |  4 +--
 .../CodeGen/PowerPC/aix64-cc-abi-vaarg.ll     | 26 +++++++++----------
 5 files changed, 29 insertions(+), 28 deletions(-)

diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp b/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
index 34618ed058050..888a77464f343 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
+++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
@@ -4624,7 +4624,8 @@ bool PPCInstrInfo::getMemOperandWithOffsetWidth(
   // Handle only loads/stores with base register followed by immediate offset.
   if (LdSt.getNumExplicitOperands() != 3)
     return false;
-  if (!LdSt.getOperand(1).isImm() || !LdSt.getOperand(2).isReg())
+  if (!LdSt.getOperand(1).isImm() ||
+      (!LdSt.getOperand(2).isReg() && !LdSt.getOperand(2).isFI()))
     return false;
 
   if (!LdSt.hasOneMemOperand())
diff --git a/llvm/test/CodeGen/PowerPC/aix-cc-byval-split.ll b/llvm/test/CodeGen/PowerPC/aix-cc-byval-split.ll
index e9155f236c9e1..6372f9ef17d6c 100644
--- a/llvm/test/CodeGen/PowerPC/aix-cc-byval-split.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-cc-byval-split.ll
@@ -36,18 +36,18 @@ entry:
 ; CHECK32:      bb.0.entry:
 ; CHECK32-NEXT:   liveins: $r3, $r4, $r5, $r6, $r7, $r8, $r9, $r10
 
+; CHECK32:     renamable $r[[REG1:[0-9]+]] = LWZ 84, %fixed-stack.0
 ; CHECK32-DAG: STW killed renamable $r3, 0, %fixed-stack.0 :: (store 4 into %fixed-stack.0
 ; CHECK32-DAG: STW killed renamable $r4, 4, %fixed-stack.0 :: (store 4 into %fixed-stack.0 + 4
+; CHECK32:     renamable $r[[REG2:[0-9]+]] = LWZ 80, %fixed-stack.0
 ; CHECK32-DAG: STW killed renamable $r5, 8, %fixed-stack.0 :: (store 4 into %fixed-stack.0 + 8
 ; CHECK32-DAG: STW killed renamable $r6, 12, %fixed-stack.0 :: (store 4 into %fixed-stack.0 + 12
 ; CHECK32-DAG: STW        renamable $r7, 16, %fixed-stack.0 :: (store 4 into %fixed-stack.0 + 16
 ; CHECK32-DAG: STW        renamable $r8, 20, %fixed-stack.0 :: (store 4 into %fixed-stack.0 + 20
 ; CHECK32-DAG: STW killed renamable $r9, 24, %fixed-stack.0 :: (store 4 into %fixed-stack.0 + 24
-; CHECK32-DAG: STW killed renamable $r10, 28, %fixed-stack.0 :: (store 4 into %fixed-stack.0 + 28
-; CHECK32:     renamable $r[[REG1:[0-9]+]] = LWZ 84, %fixed-stack.0
-; CHECK32:     renamable $r[[REG2:[0-9]+]] = LWZ 80, %fixed-stack.0
 ; CHECK32:     renamable $r4 = ADDC killed renamable $r8, killed renamable $r[[REG1]], implicit-def $carry
 ; CHECK32:     renamable $r3 = ADDE killed renamable $r7, killed renamable $r[[REG2]], implicit-def dead $carry, implicit killed $carry
+; CHECK32      STW killed renamable $r10, 28, %fixed-stack.0 :: (store 4 into %fixed-stack.0 + 28
 ; CHECK32:     BLR implicit $lr, implicit $rm, implicit $r3, implicit $r4
 
 
@@ -68,6 +68,7 @@ entry:
 ; CHECK64:  bb.0.entry:
 ; CHECK64:    liveins: $x3, $x4, $x5, $x6, $x7, $x8, $x9, $x10
 
+; CHECK64: renamable $x[[REG1:[0-9]+]] = LD 80, %fixed-stack.0
 ; CHECK64: STD killed renamable $x3, 0, %fixed-stack.0 :: (store 8 into %fixed-stack.0
 ; CHECK64: STD killed renamable $x4, 8, %fixed-stack.0 :: (store 8 into %fixed-stack.0 + 8
 ; CHECK64: STD renamable        $x5, 16, %fixed-stack.0 :: (store 8 into %fixed-stack.0 + 16
@@ -75,7 +76,6 @@ entry:
 ; CHECK64: STD killed renamable $x7, 32, %fixed-stack.0 :: (store 8 into %fixed-stack.0 + 32
 ; CHECK64: STD killed renamable $x8, 40, %fixed-stack.0 :: (store 8 into %fixed-stack.0 + 40
 ; CHECK64: STD killed renamable $x9, 48, %fixed-stack.0 :: (store 8 into %fixed-stack.0 + 48
-; CHECK64: STD killed renamable $x10, 56, %fixed-stack.0 :: (store 8 into %fixed-stack.0 + 56
-; CHECK64: renamable $x[[REG1:[0-9]+]] = LD 80, %fixed-stack.0
 ; CHECK64: renamable $x3 = ADD8 killed renamable $x5, killed renamable $x[[REG1]]
+; CHECK64: STD killed renamable $x10, 56, %fixed-stack.0 :: (store 8 into %fixed-stack.0 + 56
 ; CHECK64: BLR8 implicit $lr8, implicit $rm, implicit $x3
diff --git a/llvm/test/CodeGen/PowerPC/aix-cc-byval.ll b/llvm/test/CodeGen/PowerPC/aix-cc-byval.ll
index dd401d159fc9e..bccb28ef22906 100644
--- a/llvm/test/CodeGen/PowerPC/aix-cc-byval.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-cc-byval.ll
@@ -756,15 +756,15 @@ entry:
 
 ; 32BIT:      bb.0.entry:
 ; 32BIT-NEXT:   liveins: $r3, $r4, $r5, $r6, $r7, $r8, $r9, $r10
+; 32BIT:        STW killed renamable $r8,  20, %fixed-stack.0 :: (store 4 into %fixed-stack.0 + 20
 ; 32BIT-DAG:    STW killed renamable $r3,   0, %fixed-stack.0 :: (store 4 into %fixed-stack.0
 ; 32BIT-DAG:    STW killed renamable $r4,   4, %fixed-stack.0 :: (store 4 into %fixed-stack.0 + 4
 ; 32BIT-DAG:    STW killed renamable $r5,   8, %fixed-stack.0 :: (store 4 into %fixed-stack.0 + 8
 ; 32BIT-DAG:    STW killed renamable $r6,  12, %fixed-stack.0 :: (store 4 into %fixed-stack.0 + 12
 ; 32BIT-DAG:    STW killed renamable $r7,  16, %fixed-stack.0 :: (store 4 into %fixed-stack.0 + 16
-; 32BIT-DAG:    STW killed renamable $r8,  20, %fixed-stack.0 :: (store 4 into %fixed-stack.0 + 20
+; 32BIT:        renamable $r3 = LBZ 21, %fixed-stack.0 :: (dereferenceable load 1
 ; 32BIT-DAG:    STW killed renamable $r9,  24, %fixed-stack.0 :: (store 4 into %fixed-stack.0 + 24
 ; 32BIT-DAG:    STW killed renamable $r10, 28, %fixed-stack.0 :: (store 4 into %fixed-stack.0 + 28
-; 32BIT:        renamable $r3 = LBZ 21, %fixed-stack.0 :: (dereferenceable load 1
 ; 32BIT:        BLR
 
 ; 64BIT:      fixedStack:
@@ -773,11 +773,11 @@ entry:
 
 ; 64BIT:      bb.0.entry:
 ; 64BIT-NEXT:   liveins: $x3, $x4, $x5, $x6
+; 64BIT:        STD killed renamable $x5, 16, %fixed-stack.0 :: (store 8 into %fixed-stack.0 + 16
 ; 64BIT-DAG:    STD killed renamable $x3, 0, %fixed-stack.0 :: (store 8 into %fixed-stack.0
+; 64BIT-NEXT:   renamable $x3 = LBZ8 21, %fixed-stack.0 :: (dereferenceable load 1
 ; 64BIT-DAG:    STD killed renamable $x4, 8, %fixed-stack.0 :: (store 8 into %fixed-stack.0 + 8
-; 64BIT-DAG:    STD killed renamable $x5, 16, %fixed-stack.0 :: (store 8 into %fixed-stack.0 + 16
 ; 64BIT-DAG:    STD killed renamable $x6, 24, %fixed-stack.0 :: (store 8 into %fixed-stack.0 + 24
-; 64BIT-NEXT:   renamable $x3 = LBZ8 21, %fixed-stack.0 :: (dereferenceable load 1
 ; 64BIT-NEXT:   BLR8
 
 ; ASM-LABEL: .test_byval_32Byte:
@@ -901,9 +901,9 @@ entry:
 ; 32BIT-DAG:    STW killed renamable $r6,  12, %fixed-stack.0 :: (store 4 into %fixed-stack.0 + 12
 ; 32BIT-DAG:    STW killed renamable $r7,  16, %fixed-stack.0 :: (store 4 into %fixed-stack.0 + 16
 ; 32BIT-DAG:    STW killed renamable $r8,  20, %fixed-stack.0 :: (store 4 into %fixed-stack.0 + 20
+; 32BIT-NEXT:   renamable $f1 = LFD 16, %fixed-stack.0 :: (dereferenceable load 8
 ; 32BIT-DAG:    STW killed renamable $r9,  24, %fixed-stack.0 :: (store 4 into %fixed-stack.0 + 24
 ; 32BIT-DAG:    STW killed renamable $r10, 28, %fixed-stack.0 :: (store 4 into %fixed-stack.0 + 28
-; 32BIT-NEXT:   renamable $f1 = LFD 16, %fixed-stack.0 :: (dereferenceable load 8
 ; 32BIT-NEXT:   BLR
 
 ; 64BIT:      fixedStack:
@@ -913,10 +913,10 @@ entry:
 ; 64BIT:      bb.0.entry:
 ; 64BIT-NEXT:   liveins: $x3, $x4, $x5, $x6
 ; 64BIT-DAG:    STD killed renamable $x3,  0, %fixed-stack.0 :: (store 8 into %fixed-stack.0
-; 64BIT-DAG:    STD killed renamable $x4,  8, %fixed-stack.0 :: (store 8 into %fixed-stack.0 + 8
 ; 64BIT-DAG:    STD killed renamable $x5, 16, %fixed-stack.0 :: (store 8 into %fixed-stack.0 + 16
-; 64BIT-DAG:    STD killed renamable $x6, 24, %fixed-stack.0 :: (store 8 into %fixed-stack.0 + 24
 ; 64BIT-NEXT:   renamable $f1 = LFD 16, %fixed-stack.0 :: (dereferenceable load 8
+; 64BIT-DAG:    STD killed renamable $x4,  8, %fixed-stack.0 :: (store 8 into %fixed-stack.0 + 8
+; 64BIT-DAG:    STD killed renamable $x6, 24, %fixed-stack.0 :: (store 8 into %fixed-stack.0 + 24
 ; 64BIT-NEXT:   BLR8
 
 ; ASM32-LABEL: .test_byval_31Byte:
diff --git a/llvm/test/CodeGen/PowerPC/aix32-cc-abi-vaarg.ll b/llvm/test/CodeGen/PowerPC/aix32-cc-abi-vaarg.ll
index c276d4ccc3952..cb1544ef9756f 100644
--- a/llvm/test/CodeGen/PowerPC/aix32-cc-abi-vaarg.ll
+++ b/llvm/test/CodeGen/PowerPC/aix32-cc-abi-vaarg.ll
@@ -74,9 +74,9 @@
 ; 32BIT-DAG:     renamable $r11 = ADDI %fixed-stack.0, 0
 ; 32BIT-DAG:     STW renamable $r11, 0, %stack.0.arg1 :: (store 4 into %ir.0)
 ; 32BIT-DAG:     STW renamable $r4, 0, %stack.0.arg1 :: (store 4 into %ir.arg1)
-; 32BIT-DAG:     renamable $r5 = LWZ 0, %fixed-stack.0 :: (load 4 from %ir.2)
+; 32BIT-DAG:     renamable $r6 = LWZ 0, %fixed-stack.0 :: (load 4 from %ir.2)
 ; 32BIT-DAG:     renamable $r4 = LWZ 0, %fixed-stack.0 :: (load 4 from %ir.4)
-; 32BIT-DAG:     renamable $r3 = nsw ADD4 killed renamable $r5, killed renamable $r3
+; 32BIT-DAG:     renamable $r3 = nsw ADD4 killed renamable $r6, killed renamable $r3
 ; 32BIT-DAG:     renamable $r3 = nsw ADD4 killed renamable $r3, killed renamable $r4
 ; 32BIT-DAG:     BLR implicit $lr, implicit $rm, implicit $r3
 
diff --git a/llvm/test/CodeGen/PowerPC/aix64-cc-abi-vaarg.ll b/llvm/test/CodeGen/PowerPC/aix64-cc-abi-vaarg.ll
index 6d3a71193fa9b..7ee8dd0a3712a 100644
--- a/llvm/test/CodeGen/PowerPC/aix64-cc-abi-vaarg.ll
+++ b/llvm/test/CodeGen/PowerPC/aix64-cc-abi-vaarg.ll
@@ -63,15 +63,15 @@
 ; 64BIT-DAG:     STD killed renamable $x10, 48, %fixed-stack.0 :: (store 8)
 ; 64BIT-DAG:     renamable $x11 = ADDI8 %fixed-stack.0, 0
 ; 64BIT-DAG:     STD renamable $x11, 0, %stack.1.arg2 :: (store 8 into %ir.1)
-; 64BIT-DAG:     renamable $x4 = LD 0, %stack.1.arg2 :: (load 8 from %ir.arg2)
-; 64BIT-DAG:     renamable $x7 = ADDI8 renamable $x4, 4
-; 64BIT-DAG:     renamable $x5 = ADDI8 %fixed-stack.0, 4
-; 64BIT-DAG:     renamable $r6 = LWZ 0, %fixed-stack.0 :: (load 4 from %fixed-stack.0, align 8)
+; 64BIT-DAG:     renamable $x6 = LD 0, %stack.1.arg2 :: (load 8 from %ir.arg2)
+; 64BIT-DAG:     renamable $x9 = ADDI8 renamable $x6, 4
+; 64BIT-DAG:     renamable $x7 = ADDI8 %fixed-stack.0, 4
+; 64BIT-DAG:     renamable $r8 = LWZ 0, %fixed-stack.0 :: (load 4 from %fixed-stack.0, align 8)
 ; 64BIT-DAG:     STD killed renamable $x11, 0, %stack.0.arg1 :: (store 8 into %ir.0)
-; 64BIT-DAG:     STD killed renamable $x5, 0, %stack.0.arg1 :: (store 8 into %ir.arg1)
-; 64BIT-DAG:     STD killed renamable $x7, 0, %stack.1.arg2 :: (store 8 into %ir.arg2)
-; 64BIT-DAG:     renamable $r4 = LWZ 0, killed renamable $x4 :: (load 4)
-; 64BIT-DAG:     renamable $r3 = nsw ADD4 killed renamable $r6, renamable $r3, implicit killed $x3
+; 64BIT-DAG:     STD killed renamable $x7, 0, %stack.0.arg1 :: (store 8 into %ir.arg1)
+; 64BIT-DAG:     STD killed renamable $x9, 0, %stack.1.arg2 :: (store 8 into %ir.arg2)
+; 64BIT-DAG:     renamable $r4 = LWZ 0, killed renamable $x6 :: (load 4)
+; 64BIT-DAG:     renamable $r3 = nsw ADD4 killed renamable $r8, renamable $r3, implicit killed $x3
 ; 64BIT-DAG:     renamable $r4 = RLWINM killed renamable $r4, 1, 0, 30
 ; 64BIT-DAG:     renamable $r3 = nsw ADD4 killed renamable $r3, killed renamable $r4, implicit-def $x3
 ; 64BIT-DAG:     BLR8 implicit $lr8, implicit $rm, implicit $x3
@@ -225,14 +225,14 @@
 ; 64BIT-DAG:     STD killed renamable $x9, 40, %fixed-stack.0 :: (store 8)
 ; 64BIT-DAG:     STD killed renamable $x10, 48, %fixed-stack.0 :: (store 8)
 ; 64BIT-DAG:     STD renamable $x3, 0, %stack.1.arg2 :: (store 8 into %ir.1)
-; 64BIT-DAG:     renamable $x4 = LD 0, %stack.1.arg2 :: (load 8 from %ir.arg2)
-; 64BIT-DAG:     renamable $x5 = ADDI8 %fixed-stack.0, 8
+; 64BIT-DAG:     renamable $x6 = LD 0, %stack.1.arg2 :: (load 8 from %ir.arg2)
+; 64BIT-DAG:     renamable $x7 = ADDI8 %fixed-stack.0, 8
 ; 64BIT-DAG:     STD killed renamable $x3, 0, %stack.0.arg1 :: (store 8 into %ir.0)
-; 64BIT-DAG:     STD killed renamable $x5, 0, %stack.0.arg1 :: (store 8 into %ir.arg1)
+; 64BIT-DAG:     STD killed renamable $x7, 0, %stack.0.arg1 :: (store 8 into %ir.arg1)
 ; 64BIT-DAG:     renamable $f0 = LFD 0, %fixed-stack.0 :: (load 8)
-; 64BIT-DAG:     renamable $x3 = ADDI8 renamable $x4, 8
+; 64BIT-DAG:     renamable $x3 = ADDI8 renamable $x6, 8
 ; 64BIT-DAG:     STD killed renamable $x3, 0, %stack.1.arg2 :: (store 8 into %ir.arg2)
-; 64BIT-DAG:     renamable $f2 = LFD 0, killed renamable $x4 :: (load 8)
+; 64BIT-DAG:     renamable $f2 = LFD 0, killed renamable $x6 :: (load 8)
 ; 64BIT-DAG:     renamable $f0 = nofpexcept FADD killed renamable $f0, killed renamable $f1, implicit $rm
 ; 64BIT-DAG:     renamable $f1 = nofpexcept FADD killed renamable $f2, renamable $f2, implicit $rm
 ; 64BIT-DAG:     renamable $f1 = nofpexcept FADD killed renamable $f0, killed renamable $f1, implicit $rm

From 0bd918c828feb599b278c1876a33caa9c295be6a Mon Sep 17 00:00:00 2001
From: Petr Hosek <phosek@google.com>
Date: Fri, 31 Jul 2020 01:12:38 -0700
Subject: [PATCH 0717/1035] Revert "[ELF] Add --dependency-file option"

This reverts commit b4c7657ba602acde1c2ea5391c973949b9c3ce09 which
seems to be breaking certain bots with assertion error.
---
 lld/ELF/Config.h               |  5 +--
 lld/ELF/Driver.cpp             | 75 ----------------------------------
 lld/ELF/InputFiles.cpp         |  1 -
 lld/ELF/Options.td             |  3 --
 lld/test/ELF/dependency-file.s | 21 ----------
 5 files changed, 1 insertion(+), 104 deletions(-)
 delete mode 100644 lld/test/ELF/dependency-file.s

diff --git a/lld/ELF/Config.h b/lld/ELF/Config.h
index 8eebecf3eb060..e74a4a0c5b223 100644
--- a/lld/ELF/Config.h
+++ b/lld/ELF/Config.h
@@ -11,7 +11,6 @@
 
 #include "lld/Common/ErrorHandler.h"
 #include "llvm/ADT/MapVector.h"
-#include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/StringSet.h"
 #include "llvm/BinaryFormat/ELF.h"
@@ -91,13 +90,11 @@ struct Configuration {
   uint8_t osabi = 0;
   uint32_t andFeatures = 0;
   llvm::CachePruningPolicy thinLTOCachePolicy;
-  llvm::SetVector<StringRef> dependencyFiles; // for --dependency-file
   llvm::StringMap<uint64_t> sectionStartMap;
   llvm::StringRef bfdname;
   llvm::StringRef chroot;
-  llvm::StringRef dependencyFile;
-  llvm::StringRef dwoDir;
   llvm::StringRef dynamicLinker;
+  llvm::StringRef dwoDir;
   llvm::StringRef entry;
   llvm::StringRef emulation;
   llvm::StringRef fini;
diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp
index 75aa89ff3c863..64a41ba77ba2f 100644
--- a/lld/ELF/Driver.cpp
+++ b/lld/ELF/Driver.cpp
@@ -918,7 +918,6 @@ static void readConfigs(opt::InputArgList &args) {
   config->optimizeBBJumps =
       args.hasFlag(OPT_optimize_bb_jumps, OPT_no_optimize_bb_jumps, false);
   config->demangle = args.hasFlag(OPT_demangle, OPT_no_demangle, true);
-  config->dependencyFile = args.getLastArgValue(OPT_dependency_file);
   config->dependentLibraries = args.hasFlag(OPT_dependent_libraries, OPT_no_dependent_libraries, true);
   config->disableVerify = args.hasArg(OPT_disable_verify);
   config->discard = getDiscard(args);
@@ -1565,75 +1564,6 @@ static void handleLibcall(StringRef name) {
     sym->fetch();
 }
 
-// Handle --dependency-file=<path>. If that option is given, lld creates a
-// file at a given path with the following contents:
-//
-//   <output-file>: <input-file> ...
-//
-//   <input-file>:
-//
-// where <output-file> is a pathname of an output file and <input-file>
-// ... is a list of pathnames of all input files. `make` command can read a
-// file in the above format and interpret it as a dependency info. We write
-// phony targets for every <input-file> to avoid an error when that file is
-// removed.
-//
-// This option is useful if you want to make your final executable to depend
-// on all input files including system libraries. Here is why.
-//
-// When you write a Makefile, you usually write it so that the final
-// executable depends on all user-generated object files. Normally, you
-// don't make your executable to depend on system libraries (such as libc)
-// because you don't know the exact paths of libraries, even though system
-// libraries that are linked to your executable statically are technically a
-// part of your program. By using --dependency-file option, you can make
-// lld to dump dependency info so that you can maintain exact dependencies
-// easily.
-static void writeDependencyFile() {
-  std::error_code ec;
-  raw_fd_ostream os(config->dependencyFile, ec, sys::fs::F_None);
-  if (ec) {
-    error("cannot open " + config->dependencyFile + ": " + ec.message());
-    return;
-  }
-
-  // We use the same escape rules as Clang/GCC which are accepted by Make/Ninja:
-  // * A space is escaped by a backslash which itself must be escaped.
-  // * A hash sign is escaped by a single backslash.
-  // * $ is escapes as $$.
-  auto printFilename = [](raw_fd_ostream &os, StringRef filename) {
-    llvm::SmallString<256> nativePath;
-    llvm::sys::path::native(filename.str(), nativePath);
-    llvm::sys::path::remove_dots(nativePath, /*remove_dot_dot=*/true);
-    for (unsigned i = 0, e = nativePath.size(); i != e; ++i) {
-      if (nativePath[i] == '#') {
-        os << '\\';
-      } else if (nativePath[i] == ' ') {
-        os << '\\';
-        unsigned j = i;
-        while (j > 0 && nativePath[--j] == '\\')
-          os << '\\';
-      } else if (nativePath[i] == '$') {
-        os << '$';
-      }
-      os << nativePath[i];
-    }
-  };
-
-  os << config->outputFile << ":";
-  for (StringRef path : config->dependencyFiles) {
-    os << " \\\n ";
-    printFilename(os, path);
-  }
-  os << "\n";
-
-  for (StringRef path : config->dependencyFiles) {
-    os << "\n";
-    printFilename(os, path);
-    os << ":\n";
-  }
-}
-
 // Replaces common symbols with defined symbols reside in .bss sections.
 // This function is called after all symbol names are resolved. As a
 // result, the passes after the symbol resolution won't see any
@@ -2134,11 +2064,6 @@ template <class ELFT> void LinkerDriver::link(opt::InputArgList &args) {
     return false;
   });
 
-  // Since we now have a complete set of input files, we can create
-  // a .d file to record build dependencies.
-  if (!config->dependencyFile.empty())
-    writeDependencyFile();
-
   // Now that the number of partitions is fixed, save a pointer to the main
   // partition.
   mainPart = &partitions[0];
diff --git a/lld/ELF/InputFiles.cpp b/lld/ELF/InputFiles.cpp
index 79cc3c00cb2bc..c142c00517ccf 100644
--- a/lld/ELF/InputFiles.cpp
+++ b/lld/ELF/InputFiles.cpp
@@ -110,7 +110,6 @@ Optional<MemoryBufferRef> elf::readFile(StringRef path) {
     path = saver.save(config->chroot + path);
 
   log(path);
-  config->dependencyFiles.insert(path);
 
   auto mbOrErr = MemoryBuffer::getFile(path, -1, false);
   if (auto ec = mbOrErr.getError()) {
diff --git a/lld/ELF/Options.td b/lld/ELF/Options.td
index 3a8c9e83b08e0..c3c1309aca1a7 100644
--- a/lld/ELF/Options.td
+++ b/lld/ELF/Options.td
@@ -132,9 +132,6 @@ defm demangle: B<"demangle",
     "Demangle symbol names (default)",
     "Do not demangle symbol names">;
 
-defm dependency_file: EEq<"dependency-file", "Write a dependency file">,
-  MetaVarName<"<path>">;
-
 def disable_new_dtags: F<"disable-new-dtags">,
   HelpText<"Disable new dynamic tags">;
 
diff --git a/lld/test/ELF/dependency-file.s b/lld/test/ELF/dependency-file.s
deleted file mode 100644
index e7dbf9c7695f7..0000000000000
--- a/lld/test/ELF/dependency-file.s
+++ /dev/null
@@ -1,21 +0,0 @@
-# REQUIRES: x86
-# RUN: mkdir -p %t
-# RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o %t/foo.o
-# RUN: llvm-mc -filetype=obj -triple=x86_64 /dev/null -o "%t/bar baz.o"
-# RUN: llvm-mc -filetype=obj -triple=x86_64 /dev/null -o "%t/#quux$.o"
-# RUN: ld.lld -o %t/foo.exe %t/foo.o %t/"bar baz.o" "%t/#quux$.o" --dependency-file=%t/foo.d
-# RUN: FileCheck --match-full-lines -DFILE=%t %s < %t/foo.d
-
-# CHECK:      [[FILE]]{{/|(\\)+}}foo.exe: \
-# CHECK-NEXT:   [[FILE]]{{/|(\\)+}}foo.o \
-# CHECK-NEXT:   [[FILE]]{{/|(\\)+}}bar\ baz.o \
-# CHECK-NEXT:   [[FILE]]{{/|(\\)+}}\#quux$$.o
-# CHECK-EMPTY:
-# CHECK-NEXT: [[FILE]]{{/|(\\)+}}foo.o:
-# CHECK-EMPTY:
-# CHECK-NEXT: [[FILE]]{{/|(\\)+}}bar\ baz.o:
-# CHECK-EMPTY:
-# CHECK-NEXT: [[FILE]]{{/|(\\)+}}\#quux$$.o:
-
-.global _start
-_start:

From a50cec71ecea5680c15752d211db16d49d5b1f12 Mon Sep 17 00:00:00 2001
From: Jean Perier <jperier@nvidia.com>
Date: Fri, 31 Jul 2020 10:25:34 +0200
Subject: [PATCH 0718/1035] [flang] Fix CMPLX folding with complex arguments

CMPLX folding was expecting only one arguments in case X argument
is complex. This is wrong since there is also the optional KIND
argument.

Reviewed By: schweitz

Differential Revision: https://reviews.llvm.org/D84936
---
 flang/lib/Evaluate/fold-complex.cpp | 5 +++--
 flang/test/Evaluate/folding01.f90   | 8 ++++++++
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/flang/lib/Evaluate/fold-complex.cpp b/flang/lib/Evaluate/fold-complex.cpp
index ea96f045aa3e8..8058b2dbd13c4 100644
--- a/flang/lib/Evaluate/fold-complex.cpp
+++ b/flang/lib/Evaluate/fold-complex.cpp
@@ -36,7 +36,7 @@ Expr<Type<TypeCategory::Complex, KIND>> FoldIntrinsicFunction(
         context, std::move(funcRef), &Scalar<T>::CONJG);
   } else if (name == "cmplx") {
     using Part = typename T::Part;
-    if (args.size() == 1) {
+    if (args.size() == 2) { // CMPLX(X, [KIND])
       if (auto *x{UnwrapExpr<Expr<SomeComplex>>(args[0])}) {
         return Fold(context, ConvertToType<T>(std::move(*x)));
       }
@@ -46,7 +46,8 @@ Expr<Type<TypeCategory::Complex, KIND>> FoldIntrinsicFunction(
           Expr<T>{ComplexConstructor<KIND>{ToReal<KIND>(context, std::move(re)),
               ToReal<KIND>(context, std::move(im))}});
     }
-    CHECK(args.size() == 2 || args.size() == 3);
+    // CMPLX(X, [Y, KIND])
+    CHECK(args.size() == 3);
     Expr<SomeType> re{std::move(*args[0].value().UnwrapExpr())};
     Expr<SomeType> im{args[1] ? std::move(*args[1].value().UnwrapExpr())
                               : AsGenericExpr(Constant<Part>{Scalar<Part>{}})};
diff --git a/flang/test/Evaluate/folding01.f90 b/flang/test/Evaluate/folding01.f90
index 09004649e9d50..465b22752cecb 100644
--- a/flang/test/Evaluate/folding01.f90
+++ b/flang/test/Evaluate/folding01.f90
@@ -63,6 +63,14 @@ module m
   logical, parameter :: test_ne_i1 =.NOT.(2.NE.2)
   logical, parameter :: test_ne_i2 = -2.NE.2
 
+! Check conversions
+  logical, parameter :: test_cmplx1 = cmplx((1._4, -1._4)).EQ.((1._4, -1._4))
+  logical, parameter :: test_cmplx2 = cmplx((1._4, -1._4), 8).EQ.((1._8, -1._8))
+  logical, parameter :: test_cmplx3 = cmplx(1._4, -1._4).EQ.((1._4, -1._4))
+  logical, parameter :: test_cmplx4 = cmplx(1._4, -1._4, 8).EQ.((1._8, -1._8))
+  logical, parameter :: test_cmplx5 = cmplx(1._4).EQ.((1._4, 0._4))
+  logical, parameter :: test_cmplx6 = cmplx(1._4, kind=8).EQ.((1._8, 0._8))
+
 ! Check integer intrinsic operation folding
   logical, parameter :: test_unaryminus_i = (-(-1)).EQ.1
   logical, parameter :: test_unaryplus_i = (+1).EQ.1

From 63d3aeb529a7b0fb95c2092ca38ad21c1f5cfd74 Mon Sep 17 00:00:00 2001
From: Balazs Benics <benicsbalazs@gmail.com>
Date: Fri, 31 Jul 2020 10:28:14 +0200
Subject: [PATCH 0719/1035] [analyzer] Fix out-of-tree only clang build by not
 relaying on private header

It turned out that the D78704 included a private LLVM header, which is excluded
from the LLVM install target.
I'm substituting that `#include` with the public one by moving the necessary
`#define` into that. There was a discussion about this at D78704 and on the
cfe-dev mailing list.

I'm also placing a note to remind others of this pitfall.

Reviewed By: mgorny

Differential Revision: https://reviews.llvm.org/D84929
---
 .../StaticAnalyzer/FalsePositiveRefutationBRVisitorTest.cpp | 2 +-
 llvm/include/llvm/Config/config.h.cmake                     | 6 +++---
 llvm/include/llvm/Config/llvm-config.h.cmake                | 3 +++
 3 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/clang/unittests/StaticAnalyzer/FalsePositiveRefutationBRVisitorTest.cpp b/clang/unittests/StaticAnalyzer/FalsePositiveRefutationBRVisitorTest.cpp
index 7c151c1821133..e67dcacca0a9b 100644
--- a/clang/unittests/StaticAnalyzer/FalsePositiveRefutationBRVisitorTest.cpp
+++ b/clang/unittests/StaticAnalyzer/FalsePositiveRefutationBRVisitorTest.cpp
@@ -16,7 +16,7 @@
 #include "clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h"
 #include "clang/StaticAnalyzer/Frontend/AnalysisConsumer.h"
 #include "clang/StaticAnalyzer/Frontend/CheckerRegistry.h"
-#include "llvm/Config/config.h"
+#include "llvm/Config/llvm-config.h"
 #include "gtest/gtest.h"
 
 // FIXME: Use GTEST_SKIP() instead if GTest is updated to version 1.10.0
diff --git a/llvm/include/llvm/Config/config.h.cmake b/llvm/include/llvm/Config/config.h.cmake
index 4d76b27df6b61..b8c7e070eb341 100644
--- a/llvm/include/llvm/Config/config.h.cmake
+++ b/llvm/include/llvm/Config/config.h.cmake
@@ -1,6 +1,9 @@
 #ifndef CONFIG_H
 #define CONFIG_H
 
+// Include this header only under the llvm source tree.
+// This is a private header.
+
 /* Exported configuration */
 #include "llvm/Config/llvm-config.h"
 
@@ -335,9 +338,6 @@
 /* Whether GlobalISel rule coverage is being collected */
 #cmakedefine01 LLVM_GISEL_COV_ENABLED
 
-/* Define if we have z3 and want to build it */
-#cmakedefine LLVM_WITH_Z3 ${LLVM_WITH_Z3}
-
 /* Define to the default GlobalISel coverage file prefix */
 #cmakedefine LLVM_GISEL_COV_PREFIX "${LLVM_GISEL_COV_PREFIX}"
 
diff --git a/llvm/include/llvm/Config/llvm-config.h.cmake b/llvm/include/llvm/Config/llvm-config.h.cmake
index 01892bc5b610b..ee299876825ea 100644
--- a/llvm/include/llvm/Config/llvm-config.h.cmake
+++ b/llvm/include/llvm/Config/llvm-config.h.cmake
@@ -79,6 +79,9 @@
  */
 #cmakedefine01 LLVM_FORCE_ENABLE_STATS
 
+/* Define if we have z3 and want to build it */
+#cmakedefine LLVM_WITH_Z3 ${LLVM_WITH_Z3}
+
 /* Define if LLVM was built with a dependency to the libtensorflow dynamic library */
 #cmakedefine LLVM_HAVE_TF_API
 

From 4d6eec8e70fbf4f68fbaab5a5f80f28ddaaf586f Mon Sep 17 00:00:00 2001
From: Alexander Belyaev <pifon@google.com>
Date: Fri, 31 Jul 2020 11:00:15 +0200
Subject: [PATCH 0720/1035] [mlir] Add TFFramework dialect to
 DialectSymbolRegistry.

Differential Revision: https://reviews.llvm.org/D84918
---
 mlir/include/mlir/IR/DialectSymbolRegistry.def | 1 +
 1 file changed, 1 insertion(+)

diff --git a/mlir/include/mlir/IR/DialectSymbolRegistry.def b/mlir/include/mlir/IR/DialectSymbolRegistry.def
index d082e16026b10..7260b121930fb 100644
--- a/mlir/include/mlir/IR/DialectSymbolRegistry.def
+++ b/mlir/include/mlir/IR/DialectSymbolRegistry.def
@@ -25,6 +25,7 @@ DEFINE_SYM_KIND_RANGE(TOY) // Toy language (tutorial) Dialect
 DEFINE_SYM_KIND_RANGE(SPIRV) // SPIR-V dialect
 DEFINE_SYM_KIND_RANGE(XLA_HLO) // XLA HLO dialect
 DEFINE_SYM_KIND_RANGE(SHAPE) // Shape dialect
+DEFINE_SYM_KIND_RANGE(TF_FRAMEWORK) // TF Framework dialect
 
 // The following ranges are reserved for experimenting with MLIR dialects in a
 // private context without having to register them here.

From b0eb40ca3998c73b7e24b02e89ab3581f2daf647 Mon Sep 17 00:00:00 2001
From: Vitaly Buka <vitalybuka@google.com>
Date: Fri, 31 Jul 2020 02:09:54 -0700
Subject: [PATCH 0721/1035] [NFC] Remove unused GetUnderlyingObject paramenter

 Depends on D84617.

Differential Revision: https://reviews.llvm.org/D84621
---
 .../llvm/Analysis/ObjCARCAnalysisUtils.h      |  9 +++----
 llvm/include/llvm/Analysis/ValueTracking.h    | 13 ++++-----
 llvm/lib/Analysis/AliasAnalysis.cpp           |  3 +--
 llvm/lib/Analysis/BasicAliasAnalysis.cpp      | 10 +++----
 llvm/lib/Analysis/ConstantFolding.cpp         |  2 +-
 llvm/lib/Analysis/DependenceAnalysis.cpp      |  4 +--
 llvm/lib/Analysis/GlobalsModRef.cpp           | 27 +++++++++----------
 llvm/lib/Analysis/InstructionSimplify.cpp     |  4 +--
 llvm/lib/Analysis/LazyValueInfo.cpp           | 17 +++++-------
 llvm/lib/Analysis/Lint.cpp                    |  2 +-
 llvm/lib/Analysis/LoopAccessAnalysis.cpp      | 17 +++++-------
 .../lib/Analysis/MemoryDependenceAnalysis.cpp |  4 +--
 llvm/lib/Analysis/ObjCARCAliasAnalysis.cpp    |  6 ++---
 llvm/lib/Analysis/ValueTracking.cpp           | 13 ++++-----
 llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp  |  2 +-
 llvm/lib/CodeGen/MachinePipeliner.cpp         |  9 +++----
 llvm/lib/CodeGen/ScheduleDAGInstrs.cpp        |  2 +-
 .../SelectionDAG/SelectionDAGBuilder.cpp      |  2 +-
 llvm/lib/CodeGen/StackColoring.cpp            |  2 +-
 .../lib/Target/AMDGPU/AMDGPUAliasAnalysis.cpp |  2 +-
 llvm/lib/Target/AMDGPU/AMDGPUInline.cpp       |  2 +-
 .../lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp |  2 +-
 .../AMDGPU/AMDGPUTargetTransformInfo.cpp      |  2 +-
 llvm/lib/Target/AMDGPU/R600Instructions.td    |  4 +--
 llvm/lib/Target/AMDGPU/SIInstrInfo.cpp        |  6 ++---
 llvm/lib/Target/Mips/MipsDelaySlotFiller.cpp  | 13 +++++----
 llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp   |  3 +--
 llvm/lib/Target/NVPTX/NVPTXLowerArgs.cpp      |  3 +--
 .../Transforms/IPO/AttributorAttributes.cpp   |  3 +--
 .../InstCombine/InstCombineCompares.cpp       |  4 +--
 .../Instrumentation/AddressSanitizer.cpp      |  4 +--
 .../Instrumentation/DataFlowSanitizer.cpp     |  2 +-
 .../Instrumentation/ThreadSanitizer.cpp       |  2 +-
 .../Transforms/ObjCARC/DependencyAnalysis.cpp |  2 +-
 .../Transforms/ObjCARC/ProvenanceAnalysis.cpp |  4 +--
 .../Scalar/DeadStoreElimination.cpp           | 23 +++++++---------
 llvm/lib/Transforms/Scalar/LICM.cpp           |  4 +--
 .../Transforms/Scalar/LoopIdiomRecognize.cpp  |  4 +--
 .../Scalar/LowerMatrixIntrinsics.cpp          |  2 +-
 .../Transforms/Utils/AssumeBundleBuilder.cpp  |  4 +--
 llvm/lib/Transforms/Utils/InlineFunction.cpp  |  2 +-
 llvm/lib/Transforms/Utils/VNCoercion.cpp      |  2 +-
 .../Vectorize/LoadStoreVectorizer.cpp         |  8 +++---
 .../Transforms/Vectorize/SLPVectorizer.cpp    |  2 +-
 44 files changed, 114 insertions(+), 143 deletions(-)

diff --git a/llvm/include/llvm/Analysis/ObjCARCAnalysisUtils.h b/llvm/include/llvm/Analysis/ObjCARCAnalysisUtils.h
index bd541eea9e81c..156d5a560b9ec 100644
--- a/llvm/include/llvm/Analysis/ObjCARCAnalysisUtils.h
+++ b/llvm/include/llvm/Analysis/ObjCARCAnalysisUtils.h
@@ -64,10 +64,9 @@ inline bool ModuleHasARC(const Module &M) {
 /// This is a wrapper around getUnderlyingObject which also knows how to
 /// look through objc_retain and objc_autorelease calls, which we know to return
 /// their argument verbatim.
-inline const Value *GetUnderlyingObjCPtr(const Value *V,
-                                                const DataLayout &DL) {
+inline const Value *GetUnderlyingObjCPtr(const Value *V) {
   for (;;) {
-    V = getUnderlyingObject(V, DL);
+    V = getUnderlyingObject(V);
     if (!IsForwarding(GetBasicARCInstKind(V)))
       break;
     V = cast<CallInst>(V)->getArgOperand(0);
@@ -78,12 +77,12 @@ inline const Value *GetUnderlyingObjCPtr(const Value *V,
 
 /// A wrapper for GetUnderlyingObjCPtr used for results memoization.
 inline const Value *
-GetUnderlyingObjCPtrCached(const Value *V, const DataLayout &DL,
+GetUnderlyingObjCPtrCached(const Value *V,
                            DenseMap<const Value *, WeakTrackingVH> &Cache) {
   if (auto InCache = Cache.lookup(V))
     return InCache;
 
-  const Value *Computed = GetUnderlyingObjCPtr(V, DL);
+  const Value *Computed = GetUnderlyingObjCPtr(V);
   Cache[V] = const_cast<Value *>(Computed);
   return Computed;
 }
diff --git a/llvm/include/llvm/Analysis/ValueTracking.h b/llvm/include/llvm/Analysis/ValueTracking.h
index 2c1426be97341..0f95ad4b6c0ab 100644
--- a/llvm/include/llvm/Analysis/ValueTracking.h
+++ b/llvm/include/llvm/Analysis/ValueTracking.h
@@ -368,11 +368,10 @@ class Value;
   /// that the returned value has pointer type if the specified value does. If
   /// the MaxLookup value is non-zero, it limits the number of instructions to
   /// be stripped off.
-  Value *getUnderlyingObject(Value *V, const DataLayout &DL,
-                             unsigned MaxLookup = 6);
-  inline const Value *getUnderlyingObject(const Value *V, const DataLayout &DL,
+  Value *getUnderlyingObject(Value *V, unsigned MaxLookup = 6);
+  inline const Value *getUnderlyingObject(const Value *V,
                                           unsigned MaxLookup = 6) {
-    return getUnderlyingObject(const_cast<Value *>(V), DL, MaxLookup);
+    return getUnderlyingObject(const_cast<Value *>(V), MaxLookup);
   }
 
   /// This method is similar to getUnderlyingObject except that it can
@@ -405,14 +404,12 @@ class Value;
   /// it shouldn't look through the phi above.
   void getUnderlyingObjects(const Value *V,
                             SmallVectorImpl<const Value *> &Objects,
-                            const DataLayout &DL, LoopInfo *LI = nullptr,
-                            unsigned MaxLookup = 6);
+                            LoopInfo *LI = nullptr, unsigned MaxLookup = 6);
 
   /// This is a wrapper around getUnderlyingObjects and adds support for basic
   /// ptrtoint+arithmetic+inttoptr sequences.
   bool getUnderlyingObjectsForCodeGen(const Value *V,
-                            SmallVectorImpl<Value *> &Objects,
-                            const DataLayout &DL);
+                                      SmallVectorImpl<Value *> &Objects);
 
   /// Finds alloca where the value comes from.
   AllocaInst *findAllocaForValue(Value *V);
diff --git a/llvm/lib/Analysis/AliasAnalysis.cpp b/llvm/lib/Analysis/AliasAnalysis.cpp
index e77c203fc8d44..2593fc42afdb8 100644
--- a/llvm/lib/Analysis/AliasAnalysis.cpp
+++ b/llvm/lib/Analysis/AliasAnalysis.cpp
@@ -641,8 +641,7 @@ ModRefInfo AAResults::callCapturesBefore(const Instruction *I,
   if (!DT)
     return ModRefInfo::ModRef;
 
-  const Value *Object =
-      getUnderlyingObject(MemLoc.Ptr, I->getModule()->getDataLayout());
+  const Value *Object = getUnderlyingObject(MemLoc.Ptr);
   if (!isIdentifiedObject(Object) || isa<GlobalValue>(Object) ||
       isa<Constant>(Object))
     return ModRefInfo::ModRef;
diff --git a/llvm/lib/Analysis/BasicAliasAnalysis.cpp b/llvm/lib/Analysis/BasicAliasAnalysis.cpp
index 720ce0f688cfb..b1578f4b88e07 100644
--- a/llvm/lib/Analysis/BasicAliasAnalysis.cpp
+++ b/llvm/lib/Analysis/BasicAliasAnalysis.cpp
@@ -661,7 +661,7 @@ bool BasicAAResult::pointsToConstantMemory(const MemoryLocation &Loc,
   SmallVector<const Value *, 16> Worklist;
   Worklist.push_back(Loc.Ptr);
   do {
-    const Value *V = getUnderlyingObject(Worklist.pop_back_val(), DL);
+    const Value *V = getUnderlyingObject(Worklist.pop_back_val());
     if (!Visited.insert(V).second) {
       Visited.clear();
       return AAResultBase::pointsToConstantMemory(Loc, AAQI, OrLocal);
@@ -875,7 +875,7 @@ ModRefInfo BasicAAResult::getModRefInfo(const CallBase *Call,
   assert(notDifferentParent(Call, Loc.Ptr) &&
          "AliasAnalysis query involving multiple functions!");
 
-  const Value *Object = getUnderlyingObject(Loc.Ptr, DL);
+  const Value *Object = getUnderlyingObject(Loc.Ptr);
 
   // Calls marked 'tail' cannot read or write allocas from the current frame
   // because the current frame might be destroyed by the time they run. However,
@@ -1309,7 +1309,7 @@ bool BasicAAResult::isGEPBaseAtNegativeOffset(const GEPOperator *GEPOp,
 /// another pointer.
 ///
 /// We know that V1 is a GEP, but we don't know anything about V2.
-/// UnderlyingV1 is getUnderlyingObject(GEP1, DL), UnderlyingV2 is the same for
+/// UnderlyingV1 is getUnderlyingObject(GEP1), UnderlyingV2 is the same for
 /// V2.
 AliasResult BasicAAResult::aliasGEP(
     const GEPOperator *GEP1, LocationSize V1Size, const AAMDNodes &V1AAInfo,
@@ -1782,10 +1782,10 @@ AliasResult BasicAAResult::aliasCheck(const Value *V1, LocationSize V1Size,
 
   // Figure out what objects these things are pointing to if we can.
   if (O1 == nullptr)
-    O1 = getUnderlyingObject(V1, DL, MaxLookupSearchDepth);
+    O1 = getUnderlyingObject(V1, MaxLookupSearchDepth);
 
   if (O2 == nullptr)
-    O2 = getUnderlyingObject(V2, DL, MaxLookupSearchDepth);
+    O2 = getUnderlyingObject(V2, MaxLookupSearchDepth);
 
   // Null values in the default address space don't point to any object, so they
   // don't alias any other pointer.
diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp
index 1b3fa199d7a0c..b55ed6647d355 100644
--- a/llvm/lib/Analysis/ConstantFolding.cpp
+++ b/llvm/lib/Analysis/ConstantFolding.cpp
@@ -718,7 +718,7 @@ Constant *llvm::ConstantFoldLoadFromConstPtr(Constant *C, Type *Ty,
 
   // If this load comes from anywhere in a constant global, and if the global
   // is all undef or zero, we know what it loads.
-  if (auto *GV = dyn_cast<GlobalVariable>(getUnderlyingObject(CE, DL))) {
+  if (auto *GV = dyn_cast<GlobalVariable>(getUnderlyingObject(CE))) {
     if (GV->isConstant() && GV->hasDefinitiveInitializer()) {
       if (GV->getInitializer()->isNullValue())
         return Constant::getNullValue(Ty);
diff --git a/llvm/lib/Analysis/DependenceAnalysis.cpp b/llvm/lib/Analysis/DependenceAnalysis.cpp
index 3a67b4518c0ef..9b108e703e513 100644
--- a/llvm/lib/Analysis/DependenceAnalysis.cpp
+++ b/llvm/lib/Analysis/DependenceAnalysis.cpp
@@ -659,8 +659,8 @@ static AliasResult underlyingObjectsAlias(AAResults *AA,
     return NoAlias;
 
   // Check the underlying objects are the same
-  const Value *AObj = getUnderlyingObject(LocA.Ptr, DL);
-  const Value *BObj = getUnderlyingObject(LocB.Ptr, DL);
+  const Value *AObj = getUnderlyingObject(LocA.Ptr);
+  const Value *BObj = getUnderlyingObject(LocB.Ptr);
 
   // If the underlying objects are the same, they must alias
   if (AObj == BObj)
diff --git a/llvm/lib/Analysis/GlobalsModRef.cpp b/llvm/lib/Analysis/GlobalsModRef.cpp
index b1cdfc1896d50..a9df8238a4031 100644
--- a/llvm/lib/Analysis/GlobalsModRef.cpp
+++ b/llvm/lib/Analysis/GlobalsModRef.cpp
@@ -435,8 +435,7 @@ bool GlobalsAAResult::AnalyzeIndirectGlobalMemory(GlobalVariable *GV) {
         continue;
 
       // Check the value being stored.
-      Value *Ptr = getUnderlyingObject(SI->getOperand(0),
-                                       GV->getParent()->getDataLayout());
+      Value *Ptr = getUnderlyingObject(SI->getOperand(0));
 
       if (!isAllocLikeFn(Ptr, &GetTLI(*SI->getFunction())))
         return false; // Too hard to analyze.
@@ -661,12 +660,12 @@ static bool isNonEscapingGlobalNoAliasWithLoad(const GlobalValue *GV,
       return false;
 
     if (auto *LI = dyn_cast<LoadInst>(Input)) {
-      Inputs.push_back(getUnderlyingObject(LI->getPointerOperand(), DL));
+      Inputs.push_back(getUnderlyingObject(LI->getPointerOperand()));
       continue;
     }
     if (auto *SI = dyn_cast<SelectInst>(Input)) {
-      const Value *LHS = getUnderlyingObject(SI->getTrueValue(), DL);
-      const Value *RHS = getUnderlyingObject(SI->getFalseValue(), DL);
+      const Value *LHS = getUnderlyingObject(SI->getTrueValue());
+      const Value *RHS = getUnderlyingObject(SI->getFalseValue());
       if (Visited.insert(LHS).second)
         Inputs.push_back(LHS);
       if (Visited.insert(RHS).second)
@@ -675,7 +674,7 @@ static bool isNonEscapingGlobalNoAliasWithLoad(const GlobalValue *GV,
     }
     if (auto *PN = dyn_cast<PHINode>(Input)) {
       for (const Value *Op : PN->incoming_values()) {
-        Op = getUnderlyingObject(Op, DL);
+        Op = getUnderlyingObject(Op);
         if (Visited.insert(Op).second)
           Inputs.push_back(Op);
       }
@@ -774,7 +773,7 @@ bool GlobalsAAResult::isNonEscapingGlobalNoAlias(const GlobalValue *GV,
     if (auto *LI = dyn_cast<LoadInst>(Input)) {
       // A pointer loaded from a global would have been captured, and we know
       // that the global is non-escaping, so no alias.
-      const Value *Ptr = getUnderlyingObject(LI->getPointerOperand(), DL);
+      const Value *Ptr = getUnderlyingObject(LI->getPointerOperand());
       if (isNonEscapingGlobalNoAliasWithLoad(GV, Ptr, Depth, DL))
         // The load does not alias with GV.
         continue;
@@ -782,8 +781,8 @@ bool GlobalsAAResult::isNonEscapingGlobalNoAlias(const GlobalValue *GV,
       return false;
     }
     if (auto *SI = dyn_cast<SelectInst>(Input)) {
-      const Value *LHS = getUnderlyingObject(SI->getTrueValue(), DL);
-      const Value *RHS = getUnderlyingObject(SI->getFalseValue(), DL);
+      const Value *LHS = getUnderlyingObject(SI->getTrueValue());
+      const Value *RHS = getUnderlyingObject(SI->getFalseValue());
       if (Visited.insert(LHS).second)
         Inputs.push_back(LHS);
       if (Visited.insert(RHS).second)
@@ -792,7 +791,7 @@ bool GlobalsAAResult::isNonEscapingGlobalNoAlias(const GlobalValue *GV,
     }
     if (auto *PN = dyn_cast<PHINode>(Input)) {
       for (const Value *Op : PN->incoming_values()) {
-        Op = getUnderlyingObject(Op, DL);
+        Op = getUnderlyingObject(Op);
         if (Visited.insert(Op).second)
           Inputs.push_back(Op);
       }
@@ -827,8 +826,8 @@ AliasResult GlobalsAAResult::alias(const MemoryLocation &LocA,
                                    const MemoryLocation &LocB,
                                    AAQueryInfo &AAQI) {
   // Get the base object these pointers point to.
-  const Value *UV1 = getUnderlyingObject(LocA.Ptr, DL);
-  const Value *UV2 = getUnderlyingObject(LocB.Ptr, DL);
+  const Value *UV1 = getUnderlyingObject(LocA.Ptr);
+  const Value *UV2 = getUnderlyingObject(LocB.Ptr);
 
   // If either of the underlying values is a global, they may be non-addr-taken
   // globals, which we can answer queries about.
@@ -915,7 +914,7 @@ ModRefInfo GlobalsAAResult::getModRefInfoForArgument(const CallBase *Call,
   // is based on GV, return the conservative result.
   for (auto &A : Call->args()) {
     SmallVector<const Value*, 4> Objects;
-    getUnderlyingObjects(A, Objects, DL);
+    getUnderlyingObjects(A, Objects);
 
     // All objects must be identified.
     if (!all_of(Objects, isIdentifiedObject) &&
@@ -942,7 +941,7 @@ ModRefInfo GlobalsAAResult::getModRefInfo(const CallBase *Call,
   // If we are asking for mod/ref info of a direct call with a pointer to a
   // global we are tracking, return information if we have it.
   if (const GlobalValue *GV =
-          dyn_cast<GlobalValue>(getUnderlyingObject(Loc.Ptr, DL)))
+          dyn_cast<GlobalValue>(getUnderlyingObject(Loc.Ptr)))
     // If GV is internal to this IR and there is no function with local linkage
     // that has had their address taken, keep looking for a tighter ModRefInfo.
     if (GV->hasLocalLinkage() && !UnknownFunctionsWithLocalLinkage)
diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp
index bf55bedd801eb..38f3bbf4c6f53 100644
--- a/llvm/lib/Analysis/InstructionSimplify.cpp
+++ b/llvm/lib/Analysis/InstructionSimplify.cpp
@@ -2524,8 +2524,8 @@ computePointerICmp(const DataLayout &DL, const TargetLibraryInfo *TLI,
     // memory within the lifetime of the current function (allocas, byval
     // arguments, globals), then determine the comparison result here.
     SmallVector<const Value *, 8> LHSUObjs, RHSUObjs;
-    getUnderlyingObjects(LHS, LHSUObjs, DL);
-    getUnderlyingObjects(RHS, RHSUObjs, DL);
+    getUnderlyingObjects(LHS, LHSUObjs);
+    getUnderlyingObjects(RHS, RHSUObjs);
 
     // Is the set of underlying objects all noalias calls?
     auto IsNAC = [](ArrayRef<const Value *> Objects) {
diff --git a/llvm/lib/Analysis/LazyValueInfo.cpp b/llvm/lib/Analysis/LazyValueInfo.cpp
index 5011fc8782fe5..a95d1bca2fb51 100644
--- a/llvm/lib/Analysis/LazyValueInfo.cpp
+++ b/llvm/lib/Analysis/LazyValueInfo.cpp
@@ -606,13 +606,11 @@ Optional<ValueLatticeElement> LazyValueInfoImpl::solveBlockValueImpl(
 static bool InstructionDereferencesPointer(Instruction *I, Value *Ptr) {
   if (LoadInst *L = dyn_cast<LoadInst>(I)) {
     return L->getPointerAddressSpace() == 0 &&
-           getUnderlyingObject(L->getPointerOperand(),
-                               L->getModule()->getDataLayout()) == Ptr;
+           getUnderlyingObject(L->getPointerOperand()) == Ptr;
   }
   if (StoreInst *S = dyn_cast<StoreInst>(I)) {
     return S->getPointerAddressSpace() == 0 &&
-           getUnderlyingObject(S->getPointerOperand(),
-                               S->getModule()->getDataLayout()) == Ptr;
+           getUnderlyingObject(S->getPointerOperand()) == Ptr;
   }
   if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(I)) {
     if (MI->isVolatile()) return false;
@@ -622,13 +620,11 @@ static bool InstructionDereferencesPointer(Instruction *I, Value *Ptr) {
     if (!Len || Len->isZero()) return false;
 
     if (MI->getDestAddressSpace() == 0)
-      if (getUnderlyingObject(MI->getRawDest(),
-                              MI->getModule()->getDataLayout()) == Ptr)
+      if (getUnderlyingObject(MI->getRawDest()) == Ptr)
         return true;
     if (MemTransferInst *MTI = dyn_cast<MemTransferInst>(MI))
       if (MTI->getSourceAddressSpace() == 0)
-        if (getUnderlyingObject(MTI->getRawSource(),
-                                MTI->getModule()->getDataLayout()) == Ptr)
+        if (getUnderlyingObject(MTI->getRawSource()) == Ptr)
           return true;
   }
   return false;
@@ -641,11 +637,10 @@ static bool InstructionDereferencesPointer(Instruction *I, Value *Ptr) {
 static bool isObjectDereferencedInBlock(Value *Val, BasicBlock *BB) {
   assert(Val->getType()->isPointerTy());
 
-  const DataLayout &DL = BB->getModule()->getDataLayout();
-  Value *UnderlyingVal = getUnderlyingObject(Val, DL);
+  Value *UnderlyingVal = getUnderlyingObject(Val);
   // If 'getUnderlyingObject' didn't converge, skip it. It won't converge
   // inside InstructionDereferencesPointer either.
-  if (UnderlyingVal == getUnderlyingObject(UnderlyingVal, DL, 1))
+  if (UnderlyingVal == getUnderlyingObject(UnderlyingVal, 1))
     for (Instruction &I : *BB)
       if (InstructionDereferencesPointer(&I, UnderlyingVal))
         return true;
diff --git a/llvm/lib/Analysis/Lint.cpp b/llvm/lib/Analysis/Lint.cpp
index a03f4257e97f1..4a159d6035f0d 100644
--- a/llvm/lib/Analysis/Lint.cpp
+++ b/llvm/lib/Analysis/Lint.cpp
@@ -673,7 +673,7 @@ Value *Lint::findValueImpl(Value *V, bool OffsetOk,
   // TODO: Look through eliminable cast pairs.
   // TODO: Look through calls with unique return values.
   // TODO: Look through vector insert/extract/shuffle.
-  V = OffsetOk ? getUnderlyingObject(V, *DL) : V->stripPointerCasts();
+  V = OffsetOk ? getUnderlyingObject(V) : V->stripPointerCasts();
   if (LoadInst *L = dyn_cast<LoadInst>(V)) {
     BasicBlock::iterator BBI = L->getIterator();
     BasicBlock *BB = L->getParent();
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index e678b313c0fd2..85f658619e41e 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -508,10 +508,10 @@ class AccessAnalysis {
   typedef PointerIntPair<Value *, 1, bool> MemAccessInfo;
   typedef SmallVector<MemAccessInfo, 8> MemAccessInfoList;
 
-  AccessAnalysis(const DataLayout &Dl, Loop *TheLoop, AAResults *AA,
-                 LoopInfo *LI, MemoryDepChecker::DepCandidates &DA,
+  AccessAnalysis(Loop *TheLoop, AAResults *AA, LoopInfo *LI,
+                 MemoryDepChecker::DepCandidates &DA,
                  PredicatedScalarEvolution &PSE)
-      : DL(Dl), TheLoop(TheLoop), AST(*AA), LI(LI), DepCands(DA),
+      : TheLoop(TheLoop), AST(*AA), LI(LI), DepCands(DA),
         IsRTCheckAnalysisNeeded(false), PSE(PSE) {}
 
   /// Register a load  and whether it is only read from.
@@ -585,8 +585,6 @@ class AccessAnalysis {
   /// Set of all accesses.
   PtrAccessSet Accesses;
 
-  const DataLayout &DL;
-
   /// The loop being checked.
   const Loop *TheLoop;
 
@@ -938,7 +936,7 @@ void AccessAnalysis::processMemAccesses() {
           typedef SmallVector<const Value *, 16> ValueVector;
           ValueVector TempObjects;
 
-          getUnderlyingObjects(Ptr, TempObjects, DL, LI);
+          getUnderlyingObjects(Ptr, TempObjects, LI);
           LLVM_DEBUG(dbgs()
                      << "Underlying objects for pointer " << *Ptr << "\n");
           for (const Value *UnderlyingObj : TempObjects) {
@@ -1142,7 +1140,7 @@ bool llvm::sortPtrAccesses(ArrayRef<Value *> VL, const DataLayout &DL,
   // first pointer in the array.
   Value *Ptr0 = VL[0];
   const SCEV *Scev0 = SE.getSCEV(Ptr0);
-  Value *Obj0 = getUnderlyingObject(Ptr0, DL);
+  Value *Obj0 = getUnderlyingObject(Ptr0);
 
   llvm::SmallSet<int64_t, 4> Offsets;
   for (auto *Ptr : VL) {
@@ -1153,7 +1151,7 @@ bool llvm::sortPtrAccesses(ArrayRef<Value *> VL, const DataLayout &DL,
       return false;
     // If a pointer refers to a different underlying object, bail - the
     // pointers are by definition incomparable.
-    Value *CurrObj = getUnderlyingObject(Ptr, DL);
+    Value *CurrObj = getUnderlyingObject(Ptr);
     if (CurrObj != Obj0)
       return false;
 
@@ -1947,8 +1945,7 @@ void LoopAccessInfo::analyzeLoop(AAResults *AA, LoopInfo *LI,
   }
 
   MemoryDepChecker::DepCandidates DependentAccesses;
-  AccessAnalysis Accesses(TheLoop->getHeader()->getModule()->getDataLayout(),
-                          TheLoop, AA, LI, DependentAccesses, *PSE);
+  AccessAnalysis Accesses(TheLoop, AA, LI, DependentAccesses, *PSE);
 
   // Holds the analyzed pointers. We don't want to call getUnderlyingObjects
   // multiple times on the same object. If the ptr is accessed twice, once
diff --git a/llvm/lib/Analysis/MemoryDependenceAnalysis.cpp b/llvm/lib/Analysis/MemoryDependenceAnalysis.cpp
index 645eb3d513717..7f3de0fcf140a 100644
--- a/llvm/lib/Analysis/MemoryDependenceAnalysis.cpp
+++ b/llvm/lib/Analysis/MemoryDependenceAnalysis.cpp
@@ -406,8 +406,6 @@ MemDepResult MemoryDependenceResults::getSimplePointerDependencyFrom(
       isInvariantLoad = true;
   }
 
-  const DataLayout &DL = BB->getModule()->getDataLayout();
-
   // Return "true" if and only if the instruction I is either a non-simple
   // load or a non-simple store.
   auto isNonSimpleLoadOrStore = [](Instruction *I) -> bool {
@@ -576,7 +574,7 @@ MemDepResult MemoryDependenceResults::getSimplePointerDependencyFrom(
     // looking for a clobber in many cases; that's an alias property and is
     // handled by BasicAA.
     if (isa<AllocaInst>(Inst) || isNoAliasFn(Inst, &TLI)) {
-      const Value *AccessPtr = getUnderlyingObject(MemLoc.Ptr, DL);
+      const Value *AccessPtr = getUnderlyingObject(MemLoc.Ptr);
       if (AccessPtr == Inst || AA.isMustAlias(Inst, AccessPtr))
         return MemDepResult::getDef(Inst);
     }
diff --git a/llvm/lib/Analysis/ObjCARCAliasAnalysis.cpp b/llvm/lib/Analysis/ObjCARCAliasAnalysis.cpp
index 80e019f5fc921..58eb5281a717d 100644
--- a/llvm/lib/Analysis/ObjCARCAliasAnalysis.cpp
+++ b/llvm/lib/Analysis/ObjCARCAliasAnalysis.cpp
@@ -54,8 +54,8 @@ AliasResult ObjCARCAAResult::alias(const MemoryLocation &LocA,
 
   // If that failed, climb to the underlying object, including climbing through
   // ObjC-specific no-ops, and try making an imprecise alias query.
-  const Value *UA = GetUnderlyingObjCPtr(SA, DL);
-  const Value *UB = GetUnderlyingObjCPtr(SB, DL);
+  const Value *UA = GetUnderlyingObjCPtr(SA);
+  const Value *UB = GetUnderlyingObjCPtr(SB);
   if (UA != SA || UB != SB) {
     Result = AAResultBase::alias(MemoryLocation(UA), MemoryLocation(UB), AAQI);
     // We can't use MustAlias or PartialAlias results here because
@@ -83,7 +83,7 @@ bool ObjCARCAAResult::pointsToConstantMemory(const MemoryLocation &Loc,
 
   // If that failed, climb to the underlying object, including climbing through
   // ObjC-specific no-ops, and try making an imprecise alias query.
-  const Value *U = GetUnderlyingObjCPtr(S, DL);
+  const Value *U = GetUnderlyingObjCPtr(S);
   if (U != S)
     return AAResultBase::pointsToConstantMemory(MemoryLocation(U), AAQI,
                                                 OrLocal);
diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index 54a2796ee2f65..52132072a7d1e 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -4160,8 +4160,7 @@ static bool isSameUnderlyingObjectInLoop(const PHINode *PN,
   return true;
 }
 
-Value *llvm::getUnderlyingObject(Value *V, const DataLayout &DL,
-                                 unsigned MaxLookup) {
+Value *llvm::getUnderlyingObject(Value *V, unsigned MaxLookup) {
   if (!V->getType()->isPointerTy())
     return V;
   for (unsigned Count = 0; MaxLookup == 0 || Count < MaxLookup; ++Count) {
@@ -4208,14 +4207,13 @@ Value *llvm::getUnderlyingObject(Value *V, const DataLayout &DL,
 
 void llvm::getUnderlyingObjects(const Value *V,
                                 SmallVectorImpl<const Value *> &Objects,
-                                const DataLayout &DL, LoopInfo *LI,
-                                unsigned MaxLookup) {
+                                LoopInfo *LI, unsigned MaxLookup) {
   SmallPtrSet<const Value *, 4> Visited;
   SmallVector<const Value *, 4> Worklist;
   Worklist.push_back(V);
   do {
     const Value *P = Worklist.pop_back_val();
-    P = getUnderlyingObject(P, DL, MaxLookup);
+    P = getUnderlyingObject(P, MaxLookup);
 
     if (!Visited.insert(P).second)
       continue;
@@ -4280,15 +4278,14 @@ static const Value *getUnderlyingObjectFromInt(const Value *V) {
 /// ptrtoint+arithmetic+inttoptr sequences.
 /// It returns false if unidentified object is found in getUnderlyingObjects.
 bool llvm::getUnderlyingObjectsForCodeGen(const Value *V,
-                          SmallVectorImpl<Value *> &Objects,
-                          const DataLayout &DL) {
+                                          SmallVectorImpl<Value *> &Objects) {
   SmallPtrSet<const Value *, 16> Visited;
   SmallVector<const Value *, 4> Working(1, V);
   do {
     V = Working.pop_back_val();
 
     SmallVector<const Value *, 4> Objs;
-    getUnderlyingObjects(V, Objs, DL);
+    getUnderlyingObjects(V, Objs);
 
     for (const Value *V : Objs) {
       if (!Visited.insert(V).second)
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 61a1efc1295f8..b14492ce01236 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -1387,7 +1387,7 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
     // Get the underlying objects for the location passed on the lifetime
     // marker.
     SmallVector<const Value *, 4> Allocas;
-    getUnderlyingObjects(CI.getArgOperand(1), Allocas, *DL);
+    getUnderlyingObjects(CI.getArgOperand(1), Allocas);
 
     // Iterate over each underlying object, creating lifetime markers for each
     // static alloca. Quit if we find a non-static alloca.
diff --git a/llvm/lib/CodeGen/MachinePipeliner.cpp b/llvm/lib/CodeGen/MachinePipeliner.cpp
index 908f4e45618ad..45a5ef71d0fda 100644
--- a/llvm/lib/CodeGen/MachinePipeliner.cpp
+++ b/llvm/lib/CodeGen/MachinePipeliner.cpp
@@ -705,14 +705,13 @@ static bool isDependenceBarrier(MachineInstr &MI, AliasAnalysis *AA) {
 /// This function calls the code in ValueTracking, but first checks that the
 /// instruction has a memory operand.
 static void getUnderlyingObjects(const MachineInstr *MI,
-                                 SmallVectorImpl<const Value *> &Objs,
-                                 const DataLayout &DL) {
+                                 SmallVectorImpl<const Value *> &Objs) {
   if (!MI->hasOneMemOperand())
     return;
   MachineMemOperand *MM = *MI->memoperands_begin();
   if (!MM->getValue())
     return;
-  getUnderlyingObjects(MM->getValue(), Objs, DL);
+  getUnderlyingObjects(MM->getValue(), Objs);
   for (const Value *V : Objs) {
     if (!isIdentifiedObject(V)) {
       Objs.clear();
@@ -736,7 +735,7 @@ void SwingSchedulerDAG::addLoopCarriedDependences(AliasAnalysis *AA) {
       PendingLoads.clear();
     else if (MI.mayLoad()) {
       SmallVector<const Value *, 4> Objs;
-      ::getUnderlyingObjects(&MI, Objs, MF.getDataLayout());
+      ::getUnderlyingObjects(&MI, Objs);
       if (Objs.empty())
         Objs.push_back(UnknownValue);
       for (auto V : Objs) {
@@ -745,7 +744,7 @@ void SwingSchedulerDAG::addLoopCarriedDependences(AliasAnalysis *AA) {
       }
     } else if (MI.mayStore()) {
       SmallVector<const Value *, 4> Objs;
-      ::getUnderlyingObjects(&MI, Objs, MF.getDataLayout());
+      ::getUnderlyingObjects(&MI, Objs);
       if (Objs.empty())
         Objs.push_back(UnknownValue);
       for (auto V : Objs) {
diff --git a/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp b/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp
index 10da2d4217977..c3608f04c6480 100644
--- a/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp
+++ b/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp
@@ -154,7 +154,7 @@ static bool getUnderlyingObjectsForInstr(const MachineInstr *MI,
         Objects.push_back(UnderlyingObjectsVector::value_type(PSV, MayAlias));
       } else if (const Value *V = MMO->getValue()) {
         SmallVector<Value *, 4> Objs;
-        if (!getUnderlyingObjectsForCodeGen(V, Objs, DL))
+        if (!getUnderlyingObjectsForCodeGen(V, Objs))
           return false;
 
         for (Value *V : Objs) {
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index c0ba4c55995d4..d152cf8c4792a 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -6636,7 +6636,7 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
         cast<ConstantInt>(I.getArgOperand(0))->getSExtValue();
     Value *const ObjectPtr = I.getArgOperand(1);
     SmallVector<const Value *, 4> Allocas;
-    getUnderlyingObjects(ObjectPtr, Allocas, *DL);
+    getUnderlyingObjects(ObjectPtr, Allocas);
 
     for (SmallVectorImpl<const Value*>::iterator Object = Allocas.begin(),
            E = Allocas.end(); Object != E; ++Object) {
diff --git a/llvm/lib/CodeGen/StackColoring.cpp b/llvm/lib/CodeGen/StackColoring.cpp
index d720d93c306db..7258feafb7e90 100644
--- a/llvm/lib/CodeGen/StackColoring.cpp
+++ b/llvm/lib/CodeGen/StackColoring.cpp
@@ -1048,7 +1048,7 @@ void StackColoring::remapInstructions(DenseMap<int, int> &SlotRemap) {
         if (MMO->getAAInfo()) {
           if (const Value *MMOV = MMO->getValue()) {
             SmallVector<Value *, 4> Objs;
-            getUnderlyingObjectsForCodeGen(MMOV, Objs, MF->getDataLayout());
+            getUnderlyingObjectsForCodeGen(MMOV, Objs);
 
             if (Objs.empty())
               MayHaveConflictingAAMD = true;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAliasAnalysis.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAliasAnalysis.cpp
index e0159bb9b1130..90e7a39b30d79 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAliasAnalysis.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAliasAnalysis.cpp
@@ -96,7 +96,7 @@ bool AMDGPUAAResult::pointsToConstantMemory(const MemoryLocation &Loc,
       AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT)
     return true;
 
-  const Value *Base = getUnderlyingObject(Loc.Ptr, DL);
+  const Value *Base = getUnderlyingObject(Loc.Ptr);
   AS = Base->getType()->getPointerAddressSpace();
   if (AS == AMDGPUAS::CONSTANT_ADDRESS ||
       AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInline.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInline.cpp
index d859e798a74c8..04c41eb368327 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInline.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInline.cpp
@@ -134,7 +134,7 @@ unsigned AMDGPUInliner::getInlineThreshold(CallBase &CB) const {
                 Ty->getAddressSpace() != AMDGPUAS::FLAT_ADDRESS))
       continue;
 
-    PtrArg = getUnderlyingObject(PtrArg, DL);
+    PtrArg = getUnderlyingObject(PtrArg);
     if (const AllocaInst *AI = dyn_cast<AllocaInst>(PtrArg)) {
       if (!AI->isStaticAlloca() || !AIVisited.insert(AI).second)
         continue;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
index 965da223dac46..4cfe0edfc5330 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
@@ -605,7 +605,7 @@ bool AMDGPUPromoteAlloca::binaryOpIsDerivedFromSameAlloca(Value *BaseAlloca,
   if (isa<ConstantPointerNull>(OtherOp))
     return true;
 
-  Value *OtherObj = getUnderlyingObject(OtherOp, *DL);
+  Value *OtherObj = getUnderlyingObject(OtherOp);
   if (!isa<AllocaInst>(OtherObj))
     return false;
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index 18d8ad1d38482..ad15763e1a010 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -169,7 +169,7 @@ void AMDGPUTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
       if (AS == AMDGPUAS::PRIVATE_ADDRESS) {
         const Value *Ptr = GEP->getPointerOperand();
         const AllocaInst *Alloca =
-            dyn_cast<AllocaInst>(getUnderlyingObject(Ptr, DL));
+            dyn_cast<AllocaInst>(getUnderlyingObject(Ptr));
         if (!Alloca || !Alloca->isStaticAlloca())
           continue;
         Type *Ty = Alloca->getAllocatedType();
diff --git a/llvm/lib/Target/AMDGPU/R600Instructions.td b/llvm/lib/Target/AMDGPU/R600Instructions.td
index 03bfb2babf0cd..055e2de59ea1d 100644
--- a/llvm/lib/Target/AMDGPU/R600Instructions.td
+++ b/llvm/lib/Target/AMDGPU/R600Instructions.td
@@ -354,7 +354,7 @@ class LoadVtxId1 <PatFrag load> : PatFrag <
   return LD->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS ||
          (LD->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS &&
            !isa<GlobalValue>(getUnderlyingObject(
-           LD->getMemOperand()->getValue(), CurDAG->getDataLayout())));
+           LD->getMemOperand()->getValue())));
 }]>;
 
 def vtx_id1_az_extloadi8 : LoadVtxId1 <az_extloadi8>;
@@ -366,7 +366,7 @@ class LoadVtxId2 <PatFrag load> : PatFrag <
   const MemSDNode *LD = cast<MemSDNode>(N);
   return LD->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS &&
          isa<GlobalValue>(getUnderlyingObject(
-         LD->getMemOperand()->getValue(), CurDAG->getDataLayout()));
+         LD->getMemOperand()->getValue()));
 }]>;
 
 def vtx_id2_az_extloadi8 : LoadVtxId2 <az_extloadi8>;
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 17d1322d7e59e..5c096cae87c1d 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -459,10 +459,8 @@ static bool memOpsHaveSameBasePtr(const MachineInstr &MI1,
   auto Base2 = MO2->getValue();
   if (!Base1 || !Base2)
     return false;
-  const MachineFunction &MF = *MI1.getParent()->getParent();
-  const DataLayout &DL = MF.getFunction().getParent()->getDataLayout();
-  Base1 = getUnderlyingObject(Base1, DL);
-  Base2 = getUnderlyingObject(Base2, DL);
+  Base1 = getUnderlyingObject(Base1);
+  Base2 = getUnderlyingObject(Base2);
 
   if (isa<UndefValue>(Base1) || isa<UndefValue>(Base2))
     return false;
diff --git a/llvm/lib/Target/Mips/MipsDelaySlotFiller.cpp b/llvm/lib/Target/Mips/MipsDelaySlotFiller.cpp
index 0fcbeba6db5c8..18904d0051491 100644
--- a/llvm/lib/Target/Mips/MipsDelaySlotFiller.cpp
+++ b/llvm/lib/Target/Mips/MipsDelaySlotFiller.cpp
@@ -182,7 +182,7 @@ namespace {
   /// memory instruction can be moved to a delay slot.
   class MemDefsUses : public InspectMemInstr {
   public:
-    MemDefsUses(const DataLayout &DL, const MachineFrameInfo *MFI);
+    explicit MemDefsUses(const MachineFrameInfo *MFI);
 
   private:
     using ValueType = PointerUnion<const Value *, const PseudoSourceValue *>;
@@ -200,7 +200,6 @@ namespace {
 
     const MachineFrameInfo *MFI;
     SmallPtrSet<ValueType, 4> Uses, Defs;
-    const DataLayout &DL;
 
     /// Flags indicating whether loads or stores with no underlying objects have
     /// been seen.
@@ -492,8 +491,8 @@ bool LoadFromStackOrConst::hasHazard_(const MachineInstr &MI) {
   return true;
 }
 
-MemDefsUses::MemDefsUses(const DataLayout &DL, const MachineFrameInfo *MFI_)
-    : InspectMemInstr(false), MFI(MFI_), DL(DL) {}
+MemDefsUses::MemDefsUses(const MachineFrameInfo *MFI_)
+    : InspectMemInstr(false), MFI(MFI_) {}
 
 bool MemDefsUses::hasHazard_(const MachineInstr &MI) {
   bool HasHazard = false;
@@ -542,7 +541,7 @@ getUnderlyingObjects(const MachineInstr &MI,
 
   if (const Value *V = MMO.getValue()) {
     SmallVector<const Value *, 4> Objs;
-    ::getUnderlyingObjects(V, Objs, DL);
+    ::getUnderlyingObjects(V, Objs);
 
     for (const Value *UValue : Objs) {
       if (!isIdentifiedObject(V))
@@ -775,7 +774,7 @@ bool MipsDelaySlotFiller::searchBackward(MachineBasicBlock &MBB,
 
   auto *Fn = MBB.getParent();
   RegDefsUses RegDU(*Fn->getSubtarget().getRegisterInfo());
-  MemDefsUses MemDU(Fn->getDataLayout(), &Fn->getFrameInfo());
+  MemDefsUses MemDU(&Fn->getFrameInfo());
   ReverseIter Filler;
 
   RegDU.init(Slot);
@@ -851,7 +850,7 @@ bool MipsDelaySlotFiller::searchSuccBBs(MachineBasicBlock &MBB,
     IM.reset(new LoadFromStackOrConst());
   } else {
     const MachineFrameInfo &MFI = Fn->getFrameInfo();
-    IM.reset(new MemDefsUses(Fn->getDataLayout(), &MFI));
+    IM.reset(new MemDefsUses(&MFI));
   }
 
   if (!searchRange(MBB, SuccBB->begin(), SuccBB->end(), RegDU, *IM, Slot,
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
index 4cdb8393ebb26..2b2142cc6c212 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
@@ -704,8 +704,7 @@ static bool canLowerToLDG(MemSDNode *N, const NVPTXSubtarget &Subtarget,
   // because the former looks through phi nodes while the latter does not. We
   // need to look through phi nodes to handle pointer induction variables.
   SmallVector<const Value *, 8> Objs;
-  getUnderlyingObjects(N->getMemOperand()->getValue(), Objs,
-                       F->getDataLayout());
+  getUnderlyingObjects(N->getMemOperand()->getValue(), Objs);
 
   return all_of(Objs, [&](const Value *V) {
     if (auto *A = dyn_cast<const Argument>(V))
diff --git a/llvm/lib/Target/NVPTX/NVPTXLowerArgs.cpp b/llvm/lib/Target/NVPTX/NVPTXLowerArgs.cpp
index 453072818a534..d6859337f085f 100644
--- a/llvm/lib/Target/NVPTX/NVPTXLowerArgs.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXLowerArgs.cpp
@@ -214,8 +214,7 @@ bool NVPTXLowerArgs::runOnKernelFunction(Function &F) {
       for (auto &I : B) {
         if (LoadInst *LI = dyn_cast<LoadInst>(&I)) {
           if (LI->getType()->isPointerTy()) {
-            Value *UO = getUnderlyingObject(LI->getPointerOperand(),
-                                            F.getParent()->getDataLayout());
+            Value *UO = getUnderlyingObject(LI->getPointerOperand());
             if (Argument *Arg = dyn_cast<Argument>(UO)) {
               if (Arg->hasByValAttr()) {
                 // LI is a load from a pointer within a byval kernel parameter.
diff --git a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
index 24b6bd4818dc5..aa6bc94a3668e 100644
--- a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
+++ b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
@@ -5421,8 +5421,7 @@ struct AAPrivatizablePtrFloating : public AAPrivatizablePtrImpl {
 
   /// See AAPrivatizablePtrImpl::identifyPrivatizableType(...)
   Optional<Type *> identifyPrivatizableType(Attributor &A) override {
-    Value *Obj =
-        getUnderlyingObject(&getAssociatedValue(), A.getInfoCache().getDL());
+    Value *Obj = getUnderlyingObject(&getAssociatedValue());
     if (!Obj) {
       LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] No underlying object found!\n");
       return nullptr;
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
index 4c75bc97de66e..ca2bcb9bc4151 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
@@ -5643,10 +5643,10 @@ Instruction *InstCombinerImpl::visitICmpInst(ICmpInst &I) {
   // Try to optimize equality comparisons against alloca-based pointers.
   if (Op0->getType()->isPointerTy() && I.isEquality()) {
     assert(Op1->getType()->isPointerTy() && "Comparing pointer with non-pointer?");
-    if (auto *Alloca = dyn_cast<AllocaInst>(getUnderlyingObject(Op0, DL)))
+    if (auto *Alloca = dyn_cast<AllocaInst>(getUnderlyingObject(Op0)))
       if (Instruction *New = foldAllocaCmp(I, Alloca, Op1))
         return New;
-    if (auto *Alloca = dyn_cast<AllocaInst>(getUnderlyingObject(Op1, DL)))
+    if (auto *Alloca = dyn_cast<AllocaInst>(getUnderlyingObject(Op1)))
       if (Instruction *New = foldAllocaCmp(I, Alloca, Op0))
         return New;
   }
diff --git a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
index 0b542b638aee3..85ab19c20e84d 100644
--- a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
@@ -1556,7 +1556,7 @@ void AddressSanitizer::instrumentMop(ObjectSizeOffsetVisitor &ObjSizeVis,
   if (ClOpt && ClOptGlobals) {
     // If initialization order checking is disabled, a simple access to a
     // dynamically initialized global is always valid.
-    GlobalVariable *G = dyn_cast<GlobalVariable>(getUnderlyingObject(Addr, DL));
+    GlobalVariable *G = dyn_cast<GlobalVariable>(getUnderlyingObject(Addr));
     if (G && (!ClInitializers || GlobalIsLinkerInitialized(G)) &&
         isSafeAccess(ObjSizeVis, Addr, O.TypeSize)) {
       NumOptimizedAccessesToGlobalVar++;
@@ -1566,7 +1566,7 @@ void AddressSanitizer::instrumentMop(ObjectSizeOffsetVisitor &ObjSizeVis,
 
   if (ClOpt && ClOptStack) {
     // A direct inbounds access to a stack variable is always valid.
-    if (isa<AllocaInst>(getUnderlyingObject(Addr, DL)) &&
+    if (isa<AllocaInst>(getUnderlyingObject(Addr)) &&
         isSafeAccess(ObjSizeVis, Addr, O.TypeSize)) {
       NumOptimizedAccessesToStackVar++;
       return;
diff --git a/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
index b1f5de810110e..0af71e9b72b83 100644
--- a/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
@@ -1246,7 +1246,7 @@ Value *DFSanFunction::loadShadow(Value *Addr, uint64_t Size, uint64_t Align,
 
   const llvm::Align ShadowAlign(Align * DFS.ShadowWidthBytes);
   SmallVector<const Value *, 2> Objs;
-  getUnderlyingObjects(Addr, Objs, Pos->getModule()->getDataLayout());
+  getUnderlyingObjects(Addr, Objs);
   bool AllConstants = true;
   for (const Value *Obj : Objs) {
     if (isa<Function>(Obj) || isa<BlockAddress>(Obj))
diff --git a/llvm/lib/Transforms/Instrumentation/ThreadSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/ThreadSanitizer.cpp
index ca1535b3f9fac..b8d9b3061f07e 100644
--- a/llvm/lib/Transforms/Instrumentation/ThreadSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/ThreadSanitizer.cpp
@@ -478,7 +478,7 @@ void ThreadSanitizer::chooseInstructionsToInstrument(
       }
     }
 
-    if (isa<AllocaInst>(getUnderlyingObject(Addr, DL)) &&
+    if (isa<AllocaInst>(getUnderlyingObject(Addr)) &&
         !PointerMayBeCaptured(Addr, true, true)) {
       // The variable is addressable but not captured, so it cannot be
       // referenced from a different thread and participate in a data race
diff --git a/llvm/lib/Transforms/ObjCARC/DependencyAnalysis.cpp b/llvm/lib/Transforms/ObjCARC/DependencyAnalysis.cpp
index 46bc586fe688a..39ba2361a17a7 100644
--- a/llvm/lib/Transforms/ObjCARC/DependencyAnalysis.cpp
+++ b/llvm/lib/Transforms/ObjCARC/DependencyAnalysis.cpp
@@ -107,7 +107,7 @@ bool llvm::objcarc::CanUse(const Instruction *Inst, const Value *Ptr,
   } else if (const StoreInst *SI = dyn_cast<StoreInst>(Inst)) {
     // Special-case stores, because we don't care about the stored value, just
     // the store address.
-    const Value *Op = GetUnderlyingObjCPtr(SI->getPointerOperand(), DL);
+    const Value *Op = GetUnderlyingObjCPtr(SI->getPointerOperand());
     // If we can't tell what the underlying object was, assume there is a
     // dependence.
     return IsPotentialRetainableObjPtr(Op, *PA.getAA()) &&
diff --git a/llvm/lib/Transforms/ObjCARC/ProvenanceAnalysis.cpp b/llvm/lib/Transforms/ObjCARC/ProvenanceAnalysis.cpp
index c6138edba95ad..d528c65a90074 100644
--- a/llvm/lib/Transforms/ObjCARC/ProvenanceAnalysis.cpp
+++ b/llvm/lib/Transforms/ObjCARC/ProvenanceAnalysis.cpp
@@ -162,8 +162,8 @@ bool ProvenanceAnalysis::relatedCheck(const Value *A, const Value *B,
 
 bool ProvenanceAnalysis::related(const Value *A, const Value *B,
                                  const DataLayout &DL) {
-  A = GetUnderlyingObjCPtrCached(A, DL, UnderlyingObjCPtrCache);
-  B = GetUnderlyingObjCPtrCached(B, DL, UnderlyingObjCPtrCache);
+  A = GetUnderlyingObjCPtrCached(A, UnderlyingObjCPtrCache);
+  B = GetUnderlyingObjCPtrCached(B, UnderlyingObjCPtrCache);
 
   // Quick check.
   if (A == B)
diff --git a/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp b/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
index ad80ef15362e5..048b67a9f3b79 100644
--- a/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
@@ -415,8 +415,7 @@ static OverwriteResult isOverwrite(const MemoryLocation &Later,
   // Check to see if the later store is to the entire object (either a global,
   // an alloca, or a byval/inalloca argument).  If so, then it clearly
   // overwrites any other store to the same object.
-  const Value *UO1 = getUnderlyingObject(P1, DL),
-              *UO2 = getUnderlyingObject(P2, DL);
+  const Value *UO1 = getUnderlyingObject(P1), *UO2 = getUnderlyingObject(P2);
 
   // If we can't resolve the same pointers to the same object, then we can't
   // analyze them at all.
@@ -739,7 +738,6 @@ static bool handleFree(CallInst *F, AliasAnalysis *AA,
   MemoryLocation Loc = MemoryLocation(F->getOperand(0));
   SmallVector<BasicBlock *, 16> Blocks;
   Blocks.push_back(F->getParent());
-  const DataLayout &DL = F->getModule()->getDataLayout();
 
   while (!Blocks.empty()) {
     BasicBlock *BB = Blocks.pop_back_val();
@@ -755,7 +753,7 @@ static bool handleFree(CallInst *F, AliasAnalysis *AA,
         break;
 
       Value *DepPointer =
-          getUnderlyingObject(getStoredPointerOperand(Dependency), DL);
+          getUnderlyingObject(getStoredPointerOperand(Dependency));
 
       // Check for aliasing.
       if (!AA->isMustAlias(F->getArgOperand(0), DepPointer))
@@ -795,7 +793,7 @@ static void removeAccessedObjects(const MemoryLocation &LoadedLoc,
                                   const DataLayout &DL, AliasAnalysis *AA,
                                   const TargetLibraryInfo *TLI,
                                   const Function *F) {
-  const Value *UnderlyingPointer = getUnderlyingObject(LoadedLoc.Ptr, DL);
+  const Value *UnderlyingPointer = getUnderlyingObject(LoadedLoc.Ptr);
 
   // A constant can't be in the dead pointer set.
   if (isa<Constant>(UnderlyingPointer))
@@ -861,7 +859,7 @@ static bool handleEndBlock(BasicBlock &BB, AliasAnalysis *AA,
     if (hasAnalyzableMemoryWrite(&*BBI, *TLI) && isRemovable(&*BBI)) {
       // See through pointer-to-pointer bitcasts
       SmallVector<const Value *, 4> Pointers;
-      getUnderlyingObjects(getStoredPointerOperand(&*BBI), Pointers, DL);
+      getUnderlyingObjects(getStoredPointerOperand(&*BBI), Pointers);
 
       // Stores to stack values are valid candidates for removal.
       bool AllDead = true;
@@ -1134,7 +1132,7 @@ static bool eliminateNoopStore(Instruction *Inst, BasicBlock::iterator &BBI,
   Constant *StoredConstant = dyn_cast<Constant>(SI->getValueOperand());
   if (StoredConstant && StoredConstant->isNullValue() && isRemovable(SI)) {
     Instruction *UnderlyingPointer =
-        dyn_cast<Instruction>(getUnderlyingObject(SI->getPointerOperand(), DL));
+        dyn_cast<Instruction>(getUnderlyingObject(SI->getPointerOperand()));
 
     if (UnderlyingPointer && isCallocLikeFn(UnderlyingPointer, TLI) &&
         memoryIsNotModifiedBetween(UnderlyingPointer, SI, AA, DL, DT)) {
@@ -1289,7 +1287,7 @@ static bool eliminateDeadStores(BasicBlock &BB, AliasAnalysis *AA,
       // to it is dead along the unwind edge. Otherwise, we need to preserve
       // the store.
       if (LastThrowing && DepWrite->comesBefore(LastThrowing)) {
-        const Value *Underlying = getUnderlyingObject(DepLoc.Ptr, DL);
+        const Value *Underlying = getUnderlyingObject(DepLoc.Ptr);
         bool IsStoreDeadOnUnwind = isa<AllocaInst>(Underlying);
         if (!IsStoreDeadOnUnwind) {
             // We're looking for a call to an allocation function
@@ -1715,7 +1713,7 @@ struct DSEState {
     // object can be considered terminated.
     if (MaybeTermLoc->second) {
       DataLayout DL = MaybeTerm->getParent()->getModule()->getDataLayout();
-      DefLoc = MemoryLocation(getUnderlyingObject(DefLoc.Ptr, DL));
+      DefLoc = MemoryLocation(getUnderlyingObject(DefLoc.Ptr));
     }
     return AA.isMustAlias(MaybeTermLoc->first, DefLoc);
   }
@@ -2030,7 +2028,6 @@ struct DSEState {
   /// Eliminate writes to objects that are not visible in the caller and are not
   /// accessed before returning from the function.
   bool eliminateDeadWritesAtEndOfFunction() {
-    const DataLayout &DL = F.getParent()->getDataLayout();
     bool MadeChange = false;
     LLVM_DEBUG(
         dbgs()
@@ -2047,7 +2044,7 @@ struct DSEState {
         Instruction *DefI = Def->getMemoryInst();
         // See through pointer-to-pointer bitcasts
         SmallVector<const Value *, 4> Pointers;
-        getUnderlyingObjects(getLocForWriteEx(DefI)->Ptr, Pointers, DL);
+        getUnderlyingObjects(getLocForWriteEx(DefI)->Ptr, Pointers);
 
         LLVM_DEBUG(dbgs() << "   ... MemoryDef is not accessed until the end "
                              "of the function\n");
@@ -2130,7 +2127,7 @@ bool eliminateDeadStoresMemorySSA(Function &F, AliasAnalysis &AA,
     }
     MemoryLocation SILoc = *MaybeSILoc;
     assert(SILoc.Ptr && "SILoc should not be null");
-    const Value *SILocUnd = getUnderlyingObject(SILoc.Ptr, DL);
+    const Value *SILocUnd = getUnderlyingObject(SILoc.Ptr);
 
     // Check if the store is a no-op.
     if (isRemovable(SI) && State.storeIsNoop(KillingDef, SILoc, SILocUnd)) {
@@ -2231,7 +2228,7 @@ bool eliminateDeadStoresMemorySSA(Function &F, AliasAnalysis &AA,
       MemoryLocation NILoc = *State.getLocForWriteEx(NI);
 
       if (State.isMemTerminatorInst(SI)) {
-        const Value *NIUnd = getUnderlyingObject(NILoc.Ptr, DL);
+        const Value *NIUnd = getUnderlyingObject(NILoc.Ptr);
         if (!SILocUnd || SILocUnd != NIUnd)
           continue;
         LLVM_DEBUG(dbgs() << "DSE: Remove Dead Store:\n  DEAD: " << *NI
diff --git a/llvm/lib/Transforms/Scalar/LICM.cpp b/llvm/lib/Transforms/Scalar/LICM.cpp
index 043d3683aa2ec..4bf39ba8f151c 100644
--- a/llvm/lib/Transforms/Scalar/LICM.cpp
+++ b/llvm/lib/Transforms/Scalar/LICM.cpp
@@ -1909,7 +1909,7 @@ bool llvm::promoteLoopAccessesToScalars(
     // we have to prove that the store is dead along the unwind edge.  We do
     // this by proving that the caller can't have a reference to the object
     // after return and thus can't possibly load from the object.
-    Value *Object = getUnderlyingObject(SomePtr, MDL);
+    Value *Object = getUnderlyingObject(SomePtr);
     if (!isKnownNonEscaping(Object, TLI))
       return false;
     // Subtlety: Alloca's aren't visible to callers, but *are* potentially
@@ -2041,7 +2041,7 @@ bool llvm::promoteLoopAccessesToScalars(
     if (IsKnownThreadLocalObject)
       SafeToInsertStore = true;
     else {
-      Value *Object = getUnderlyingObject(SomePtr, MDL);
+      Value *Object = getUnderlyingObject(SomePtr);
       SafeToInsertStore =
           (isAllocLikeFn(Object, TLI) || isa<AllocaInst>(Object)) &&
           !PointerMayBeCaptured(Object, true, true);
diff --git a/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp b/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
index 99d3519eff153..52993c9dcb091 100644
--- a/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
@@ -564,12 +564,12 @@ void LoopIdiomRecognize::collectStores(BasicBlock *BB) {
       break;
     case LegalStoreKind::Memset: {
       // Find the base pointer.
-      Value *Ptr = getUnderlyingObject(SI->getPointerOperand(), *DL);
+      Value *Ptr = getUnderlyingObject(SI->getPointerOperand());
       StoreRefsForMemset[Ptr].push_back(SI);
     } break;
     case LegalStoreKind::MemsetPattern: {
       // Find the base pointer.
-      Value *Ptr = getUnderlyingObject(SI->getPointerOperand(), *DL);
+      Value *Ptr = getUnderlyingObject(SI->getPointerOperand());
       StoreRefsForMemsetPattern[Ptr].push_back(SI);
     } break;
     case LegalStoreKind::Memcpy:
diff --git a/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp b/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp
index e0cd0253f9bf6..053d96430b6b3 100644
--- a/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp
+++ b/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp
@@ -1562,7 +1562,7 @@ class LowerMatrixIntrinsics {
       if (Value *Ptr = getPointerOperand(V))
         return getUnderlyingObjectThroughLoads(Ptr);
       else if (V->getType()->isPointerTy())
-        return getUnderlyingObject(V, DL);
+        return getUnderlyingObject(V);
       return V;
     }
 
diff --git a/llvm/lib/Transforms/Utils/AssumeBundleBuilder.cpp b/llvm/lib/Transforms/Utils/AssumeBundleBuilder.cpp
index 762158f7e19ed..a3d0c0421e949 100644
--- a/llvm/lib/Transforms/Utils/AssumeBundleBuilder.cpp
+++ b/llvm/lib/Transforms/Utils/AssumeBundleBuilder.cpp
@@ -69,7 +69,7 @@ RetainedKnowledge canonicalizedKnowledge(RetainedKnowledge RK, Module *M) {
   default:
     return RK;
   case Attribute::NonNull:
-    RK.WasOn = getUnderlyingObject(RK.WasOn, M->getDataLayout());
+    RK.WasOn = getUnderlyingObject(RK.WasOn);
     return RK;
   case Attribute::Alignment: {
     Value *V = RK.WasOn->stripInBoundsOffsets([&](const Value *Strip) {
@@ -145,7 +145,7 @@ struct AssumeBuilderState {
     if (!RK.WasOn)
       return true;
     if (RK.WasOn->getType()->isPointerTy()) {
-      Value *UnderlyingPtr = getUnderlyingObject(RK.WasOn, M->getDataLayout());
+      Value *UnderlyingPtr = getUnderlyingObject(RK.WasOn);
       if (isa<AllocaInst>(UnderlyingPtr) || isa<GlobalValue>(UnderlyingPtr))
         return false;
     }
diff --git a/llvm/lib/Transforms/Utils/InlineFunction.cpp b/llvm/lib/Transforms/Utils/InlineFunction.cpp
index 090ba9daa678f..30726627bc829 100644
--- a/llvm/lib/Transforms/Utils/InlineFunction.cpp
+++ b/llvm/lib/Transforms/Utils/InlineFunction.cpp
@@ -1037,7 +1037,7 @@ static void AddAliasScopeMetadata(CallBase &CB, ValueToValueMapTy &VMap,
       SmallSetVector<const Argument *, 4> NAPtrArgs;
       for (const Value *V : PtrArgs) {
         SmallVector<const Value *, 4> Objects;
-        getUnderlyingObjects(V, Objects, DL, /* LI = */ nullptr);
+        getUnderlyingObjects(V, Objects, /* LI = */ nullptr);
 
         for (const Value *O : Objects)
           ObjSet.insert(O);
diff --git a/llvm/lib/Transforms/Utils/VNCoercion.cpp b/llvm/lib/Transforms/Utils/VNCoercion.cpp
index 11b079d3c0503..1939c0e3b504b 100644
--- a/llvm/lib/Transforms/Utils/VNCoercion.cpp
+++ b/llvm/lib/Transforms/Utils/VNCoercion.cpp
@@ -393,7 +393,7 @@ int analyzeLoadFromClobberingMemInst(Type *LoadTy, Value *LoadPtr,
   if (!Src)
     return -1;
 
-  GlobalVariable *GV = dyn_cast<GlobalVariable>(getUnderlyingObject(Src, DL));
+  GlobalVariable *GV = dyn_cast<GlobalVariable>(getUnderlyingObject(Src));
   if (!GV || !GV->isConstant() || !GV->hasDefinitiveInitializer())
     return -1;
 
diff --git a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
index 0b10e50bd1f77..26ffb044a35c2 100644
--- a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
@@ -762,8 +762,8 @@ Vectorizer::getVectorizablePrefix(ArrayRef<Instruction *> Chain) {
   return Chain.slice(0, ChainIdx);
 }
 
-static ChainID getChainID(const Value *Ptr, const DataLayout &DL) {
-  const Value *ObjPtr = getUnderlyingObject(Ptr, DL);
+static ChainID getChainID(const Value *Ptr) {
+  const Value *ObjPtr = getUnderlyingObject(Ptr);
   if (const auto *Sel = dyn_cast<SelectInst>(ObjPtr)) {
     // The select's themselves are distinct instructions even if they share the
     // same condition and evaluate to consecutive pointers for true and false
@@ -830,7 +830,7 @@ Vectorizer::collectInstructions(BasicBlock *BB) {
         continue;
 
       // Save the load locations.
-      const ChainID ID = getChainID(Ptr, DL);
+      const ChainID ID = getChainID(Ptr);
       LoadRefs[ID].push_back(LI);
     } else if (StoreInst *SI = dyn_cast<StoreInst>(&I)) {
       if (!SI->isSimple())
@@ -876,7 +876,7 @@ Vectorizer::collectInstructions(BasicBlock *BB) {
         continue;
 
       // Save store location.
-      const ChainID ID = getChainID(Ptr, DL);
+      const ChainID ID = getChainID(Ptr);
       StoreRefs[ID].push_back(SI);
     }
   }
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 08c0021a41262..b5b3fa59b8e92 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -5912,7 +5912,7 @@ void SLPVectorizerPass::collectSeedInstructions(BasicBlock *BB) {
         continue;
       if (!isValidElementType(SI->getValueOperand()->getType()))
         continue;
-      Stores[getUnderlyingObject(SI->getPointerOperand(), *DL)].push_back(SI);
+      Stores[getUnderlyingObject(SI->getPointerOperand())].push_back(SI);
     }
 
     // Ignore getelementptr instructions that have more than one index, a

From fb5588b0ad59522031d037b0d1a3fdcf8ada8a79 Mon Sep 17 00:00:00 2001
From: Kirill Bobyrev <kbobyrev@google.com>
Date: Fri, 31 Jul 2020 11:48:27 +0200
Subject: [PATCH 0722/1035] [clangd] Propagate remote index errors via Expected

This is a refactoring: errors should be logged only on the highest level.
Switch from Optional to Expected in the serialization code.

Reviewed By: kadircet

Differential Revision: https://reviews.llvm.org/D84939
---
 .../clangd/index/remote/Client.cpp            |   5 +-
 .../index/remote/marshalling/Marshalling.cpp  | 161 +++++++-----------
 .../index/remote/marshalling/Marshalling.h    |  25 ++-
 .../clangd/index/remote/server/Server.cpp     |  17 +-
 .../unittests/remote/MarshallingTests.cpp     | 160 +++++++++--------
 5 files changed, 182 insertions(+), 186 deletions(-)

diff --git a/clang-tools-extra/clangd/index/remote/Client.cpp b/clang-tools-extra/clangd/index/remote/Client.cpp
index af58645d17959..4c1741e715a55 100644
--- a/clang-tools-extra/clangd/index/remote/Client.cpp
+++ b/clang-tools-extra/clangd/index/remote/Client.cpp
@@ -16,6 +16,7 @@
 #include "support/Logger.h"
 #include "support/Trace.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Error.h"
 
 #include <chrono>
 
@@ -53,7 +54,9 @@ class IndexClient : public clangd::SymbolIndex {
       }
       auto Response = ProtobufMarshaller->fromProtobuf(Reply.stream_result());
       if (!Response) {
-        elog("Received invalid {0}", ReplyT::descriptor()->name());
+        elog("Received invalid {0}: {1}. Reason: {2}",
+             ReplyT::descriptor()->name(), Reply.stream_result().DebugString(),
+             Response.takeError());
         ++FailedToParse;
         continue;
       }
diff --git a/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp b/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp
index 270e15347ee03..bac5b7e6e958e 100644
--- a/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp
+++ b/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp
@@ -19,8 +19,6 @@
 #include "support/Logger.h"
 #include "clang/Index/IndexSymbol.h"
 #include "llvm/ADT/DenseSet.h"
-#include "llvm/ADT/None.h"
-#include "llvm/ADT/Optional.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
@@ -46,6 +44,11 @@ llvm::Expected<llvm::DenseSet<SymbolID>> getIDs(IDRange IDs) {
   return Result;
 }
 
+llvm::Error makeStringError(llvm::StringRef Message) {
+  return llvm::make_error<llvm::StringError>(Message,
+                                             llvm::inconvertibleErrorCode());
+}
+
 } // namespace
 
 Marshaller::Marshaller(llvm::StringRef RemoteIndexRoot,
@@ -124,20 +127,13 @@ Marshaller::fromProtobuf(const RelationsRequest *Message) {
   return Req;
 }
 
-llvm::Optional<clangd::Symbol> Marshaller::fromProtobuf(const Symbol &Message) {
-  if (!Message.has_info() || !Message.has_canonical_declaration()) {
-    elog("Cannot convert Symbol from protobuf (missing info, definition or "
-         "declaration): {0}",
-         Message.DebugString());
-    return llvm::None;
-  }
+llvm::Expected<clangd::Symbol> Marshaller::fromProtobuf(const Symbol &Message) {
+  if (!Message.has_info() || !Message.has_canonical_declaration())
+    return makeStringError("Missing info or declaration.");
   clangd::Symbol Result;
   auto ID = SymbolID::fromStr(Message.id());
-  if (!ID) {
-    elog("Cannot parse SymbolID {0} given protobuf: {1}", ID.takeError(),
-         Message.DebugString());
-    return llvm::None;
-  }
+  if (!ID)
+    return ID.takeError();
   Result.ID = *ID;
   Result.SymInfo = fromProtobuf(Message.info());
   Result.Name = Message.name();
@@ -148,11 +144,8 @@ llvm::Optional<clangd::Symbol> Marshaller::fromProtobuf(const Symbol &Message) {
       Result.Definition = *Definition;
   }
   auto Declaration = fromProtobuf(Message.canonical_declaration());
-  if (!Declaration) {
-    elog("Cannot convert Symbol from protobuf (invalid declaration): {0}",
-         Message.DebugString());
-    return llvm::None;
-  }
+  if (!Declaration)
+    return Declaration.takeError();
   Result.CanonicalDeclaration = *Declaration;
   Result.References = Message.references();
   Result.Origin = static_cast<clangd::SymbolOrigin>(Message.origin());
@@ -164,49 +157,36 @@ llvm::Optional<clangd::Symbol> Marshaller::fromProtobuf(const Symbol &Message) {
   Result.Type = Message.type();
   for (const auto &Header : Message.headers()) {
     auto SerializedHeader = fromProtobuf(Header);
-    if (SerializedHeader)
-      Result.IncludeHeaders.push_back(*SerializedHeader);
-    else
-      elog("Cannot convert HeaderWithIncludes from protobuf: {0}",
-           Header.DebugString());
+    if (!SerializedHeader)
+      return SerializedHeader.takeError();
+    Result.IncludeHeaders.push_back(*SerializedHeader);
   }
   Result.Flags = static_cast<clangd::Symbol::SymbolFlag>(Message.flags());
   return Result;
 }
 
-llvm::Optional<clangd::Ref> Marshaller::fromProtobuf(const Ref &Message) {
-  if (!Message.has_location()) {
-    elog("Cannot convert Ref from protobuf (missing location): {0}",
-         Message.DebugString());
-    return llvm::None;
-  }
+llvm::Expected<clangd::Ref> Marshaller::fromProtobuf(const Ref &Message) {
+  if (!Message.has_location())
+    return makeStringError("Missing location.");
   clangd::Ref Result;
   auto Location = fromProtobuf(Message.location());
-  if (!Location) {
-    elog("Cannot convert Ref from protobuf (invalid location): {0}",
-         Message.DebugString());
-    return llvm::None;
-  }
+  if (!Location)
+    return Location.takeError();
   Result.Location = *Location;
   Result.Kind = static_cast<clangd::RefKind>(Message.kind());
   return Result;
 }
 
-llvm::Optional<std::pair<clangd::SymbolID, clangd::Symbol>>
+llvm::Expected<std::pair<clangd::SymbolID, clangd::Symbol>>
 Marshaller::fromProtobuf(const Relation &Message) {
   auto SubjectID = SymbolID::fromStr(Message.subject_id());
-  if (!SubjectID) {
-    elog("Cannot convert Relation from protobuf (invalid Subject ID): {0}",
-         SubjectID.takeError());
-    return llvm::None;
-  }
-  if (!Message.has_object()) {
-    elog("Cannot convert Relation from protobuf (missing Object): {0}");
-    return llvm::None;
-  }
+  if (!SubjectID)
+    return SubjectID.takeError();
+  if (!Message.has_object())
+    return makeStringError("Missing Object.");
   auto Object = fromProtobuf(Message.object());
   if (!Object)
-    return llvm::None;
+    return Object.takeError();
   return std::make_pair(*SubjectID, *Object);
 }
 
@@ -258,27 +238,21 @@ RelationsRequest Marshaller::toProtobuf(const clangd::RelationsRequest &From) {
   return RPCRequest;
 }
 
-llvm::Optional<Symbol> Marshaller::toProtobuf(const clangd::Symbol &From) {
+llvm::Expected<Symbol> Marshaller::toProtobuf(const clangd::Symbol &From) {
   Symbol Result;
   Result.set_id(From.ID.str());
   *Result.mutable_info() = toProtobuf(From.SymInfo);
   Result.set_name(From.Name.str());
   if (*From.Definition.FileURI) {
     auto Definition = toProtobuf(From.Definition);
-    if (!Definition) {
-      elog("Can not convert Symbol to protobuf (invalid definition) {0}: {1}",
-           From, From.Definition);
-      return llvm::None;
-    }
+    if (!Definition)
+      return Definition.takeError();
     *Result.mutable_definition() = *Definition;
   }
   Result.set_scope(From.Scope.str());
   auto Declaration = toProtobuf(From.CanonicalDeclaration);
-  if (!Declaration) {
-    elog("Can not convert Symbol to protobuf (invalid declaration) {0}: {1}",
-         From, From.CanonicalDeclaration);
-    return llvm::None;
-  }
+  if (!Declaration)
+    return Declaration.takeError();
   *Result.mutable_canonical_declaration() = *Declaration;
   Result.set_references(From.References);
   Result.set_origin(static_cast<uint32_t>(From.Origin));
@@ -291,11 +265,8 @@ llvm::Optional<Symbol> Marshaller::toProtobuf(const clangd::Symbol &From) {
   Result.set_type(From.Type.str());
   for (const auto &Header : From.IncludeHeaders) {
     auto Serialized = toProtobuf(Header);
-    if (!Serialized) {
-      elog("Can not convert IncludeHeaderWithReferences to protobuf: {0}",
-           Header.IncludeHeader);
-      continue;
-    }
+    if (!Serialized)
+      return Serialized.takeError();
     auto *NextHeader = Result.add_headers();
     *NextHeader = *Serialized;
   }
@@ -303,64 +274,52 @@ llvm::Optional<Symbol> Marshaller::toProtobuf(const clangd::Symbol &From) {
   return Result;
 }
 
-llvm::Optional<Ref> Marshaller::toProtobuf(const clangd::Ref &From) {
+llvm::Expected<Ref> Marshaller::toProtobuf(const clangd::Ref &From) {
   Ref Result;
   Result.set_kind(static_cast<uint32_t>(From.Kind));
   auto Location = toProtobuf(From.Location);
-  if (!Location) {
-    elog("Can not convert Reference to protobuf (invalid location) {0}: {1}",
-         From, From.Location);
-    return llvm::None;
-  }
+  if (!Location)
+    return Location.takeError();
   *Result.mutable_location() = *Location;
   return Result;
 }
 
-llvm::Optional<Relation> Marshaller::toProtobuf(const clangd::SymbolID &Subject,
+llvm::Expected<Relation> Marshaller::toProtobuf(const clangd::SymbolID &Subject,
                                                 const clangd::Symbol &Object) {
   Relation Result;
   *Result.mutable_subject_id() = Subject.str();
   auto SerializedObject = toProtobuf(Object);
-  if (!SerializedObject) {
-    elog("Can not convert Relation to protobuf (invalid symbol): {0}", Object);
-    return llvm::None;
-  }
+  if (!SerializedObject)
+    return SerializedObject.takeError();
   *Result.mutable_object() = *SerializedObject;
   return Result;
 }
 
-llvm::Optional<std::string>
+llvm::Expected<std::string>
 Marshaller::relativePathToURI(llvm::StringRef RelativePath) {
   assert(LocalIndexRoot);
   assert(RelativePath == llvm::sys::path::convert_to_slash(
                              RelativePath, llvm::sys::path::Style::posix));
-  if (RelativePath.empty()) {
-    return llvm::None;
-  }
-  if (llvm::sys::path::is_absolute(RelativePath)) {
-    return llvm::None;
-  }
+  if (RelativePath.empty())
+    return makeStringError("Empty relative path.");
+  if (llvm::sys::path::is_absolute(RelativePath))
+    return makeStringError("RelativePath is absolute.");
   llvm::SmallString<256> FullPath = llvm::StringRef(*LocalIndexRoot);
   llvm::sys::path::append(FullPath, RelativePath);
   auto Result = URI::createFile(FullPath);
   return Result.toString();
 }
 
-llvm::Optional<std::string> Marshaller::uriToRelativePath(llvm::StringRef URI) {
+llvm::Expected<std::string> Marshaller::uriToRelativePath(llvm::StringRef URI) {
   assert(RemoteIndexRoot);
   auto ParsedURI = URI::parse(URI);
-  if (!ParsedURI) {
-    elog("Remote index got bad URI from client {0}: {1}", URI,
-         ParsedURI.takeError());
-    return llvm::None;
-  }
-  if (ParsedURI->scheme() != "file") {
-    return llvm::None;
-  }
+  if (!ParsedURI)
+    return ParsedURI.takeError();
+  if (ParsedURI->scheme() != "file")
+    return makeStringError("Can not use URI schemes other than file.");
   llvm::SmallString<256> Result = ParsedURI->body();
-  if (!llvm::sys::path::replace_path_prefix(Result, *RemoteIndexRoot, "")) {
-    return llvm::None;
-  }
+  if (!llvm::sys::path::replace_path_prefix(Result, *RemoteIndexRoot, ""))
+    return makeStringError("File path doesn't start with RemoteIndexRoot.");
   // Make sure the result has UNIX slashes.
   return llvm::sys::path::convert_to_slash(Result,
                                            llvm::sys::path::Style::posix);
@@ -401,31 +360,31 @@ SymbolInfo Marshaller::toProtobuf(const clang::index::SymbolInfo &Info) {
   return Result;
 }
 
-llvm::Optional<clangd::SymbolLocation>
+llvm::Expected<clangd::SymbolLocation>
 Marshaller::fromProtobuf(const SymbolLocation &Message) {
   clangd::SymbolLocation Location;
   auto URIString = relativePathToURI(Message.file_path());
   if (!URIString)
-    return llvm::None;
+    return URIString.takeError();
   Location.FileURI = Strings.save(*URIString).begin();
   Location.Start = fromProtobuf(Message.start());
   Location.End = fromProtobuf(Message.end());
   return Location;
 }
 
-llvm::Optional<SymbolLocation>
+llvm::Expected<SymbolLocation>
 Marshaller::toProtobuf(const clangd::SymbolLocation &Location) {
   remote::SymbolLocation Result;
   auto RelativePath = uriToRelativePath(Location.FileURI);
   if (!RelativePath)
-    return llvm::None;
+    return RelativePath.takeError();
   *Result.mutable_file_path() = *RelativePath;
   *Result.mutable_start() = toProtobuf(Location.Start);
   *Result.mutable_end() = toProtobuf(Location.End);
   return Result;
 }
 
-llvm::Optional<HeaderWithReferences> Marshaller::toProtobuf(
+llvm::Expected<HeaderWithReferences> Marshaller::toProtobuf(
     const clangd::Symbol::IncludeHeaderWithReferences &IncludeHeader) {
   HeaderWithReferences Result;
   Result.set_references(IncludeHeader.References);
@@ -436,18 +395,18 @@ llvm::Optional<HeaderWithReferences> Marshaller::toProtobuf(
   }
   auto RelativePath = uriToRelativePath(Header);
   if (!RelativePath)
-    return llvm::None;
+    return RelativePath.takeError();
   Result.set_header(*RelativePath);
   return Result;
 }
 
-llvm::Optional<clangd::Symbol::IncludeHeaderWithReferences>
+llvm::Expected<clangd::Symbol::IncludeHeaderWithReferences>
 Marshaller::fromProtobuf(const HeaderWithReferences &Message) {
   std::string Header = Message.header();
-  if (Header.front() != '<' && Header.front() != '"') {
+  if (!isLiteralInclude(Header)) {
     auto URIString = relativePathToURI(Header);
     if (!URIString)
-      return llvm::None;
+      return URIString.takeError();
     Header = *URIString;
   }
   return clangd::Symbol::IncludeHeaderWithReferences{Strings.save(Header),
diff --git a/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.h b/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.h
index 9588f1ef64113..18ce6074264c0 100644
--- a/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.h
+++ b/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.h
@@ -38,10 +38,9 @@ class Marshaller {
   Marshaller() = delete;
   Marshaller(llvm::StringRef RemoteIndexRoot, llvm::StringRef LocalIndexRoot);
 
-  // FIXME(kirillbobyrev): Switch from Optional to Expected.
-  llvm::Optional<clangd::Symbol> fromProtobuf(const Symbol &Message);
-  llvm::Optional<clangd::Ref> fromProtobuf(const Ref &Message);
-  llvm::Optional<std::pair<clangd::SymbolID, clangd::Symbol>>
+  llvm::Expected<clangd::Symbol> fromProtobuf(const Symbol &Message);
+  llvm::Expected<clangd::Ref> fromProtobuf(const Ref &Message);
+  llvm::Expected<std::pair<clangd::SymbolID, clangd::Symbol>>
   fromProtobuf(const Relation &Message);
 
   llvm::Expected<clangd::LookupRequest>
@@ -61,9 +60,9 @@ class Marshaller {
   RefsRequest toProtobuf(const clangd::RefsRequest &From);
   RelationsRequest toProtobuf(const clangd::RelationsRequest &From);
 
-  llvm::Optional<Symbol> toProtobuf(const clangd::Symbol &From);
-  llvm::Optional<Ref> toProtobuf(const clangd::Ref &From);
-  llvm::Optional<Relation> toProtobuf(const clangd::SymbolID &Subject,
+  llvm::Expected<Symbol> toProtobuf(const clangd::Symbol &From);
+  llvm::Expected<Ref> toProtobuf(const clangd::Ref &From);
+  llvm::Expected<Relation> toProtobuf(const clangd::SymbolID &Subject,
                                       const clangd::Symbol &Object);
 
   /// Translates \p RelativePath into the absolute path and builds URI for the
@@ -72,23 +71,23 @@ class Marshaller {
   /// provided by the client.
   ///
   /// The relative path passed over the wire has unix slashes.
-  llvm::Optional<std::string> relativePathToURI(llvm::StringRef RelativePath);
+  llvm::Expected<std::string> relativePathToURI(llvm::StringRef RelativePath);
   /// Translates a URI from the server's backing index to a relative path
   /// suitable to send over the wire to the client.
-  llvm::Optional<std::string> uriToRelativePath(llvm::StringRef URI);
+  llvm::Expected<std::string> uriToRelativePath(llvm::StringRef URI);
 
 private:
   clangd::SymbolLocation::Position fromProtobuf(const Position &Message);
   Position toProtobuf(const clangd::SymbolLocation::Position &Position);
   clang::index::SymbolInfo fromProtobuf(const SymbolInfo &Message);
   SymbolInfo toProtobuf(const clang::index::SymbolInfo &Info);
-  llvm::Optional<clangd::SymbolLocation>
+  llvm::Expected<clangd::SymbolLocation>
   fromProtobuf(const SymbolLocation &Message);
-  llvm::Optional<SymbolLocation>
+  llvm::Expected<SymbolLocation>
   toProtobuf(const clangd::SymbolLocation &Location);
-  llvm::Optional<HeaderWithReferences>
+  llvm::Expected<HeaderWithReferences>
   toProtobuf(const clangd::Symbol::IncludeHeaderWithReferences &IncludeHeader);
-  llvm::Optional<clangd::Symbol::IncludeHeaderWithReferences>
+  llvm::Expected<clangd::Symbol::IncludeHeaderWithReferences>
   fromProtobuf(const HeaderWithReferences &Message);
 
   /// RemoteIndexRoot and LocalIndexRoot are absolute paths to the project (on
diff --git a/clang-tools-extra/clangd/index/remote/server/Server.cpp b/clang-tools-extra/clangd/index/remote/server/Server.cpp
index 27e89cbbb48ef..e9838cce85e3d 100644
--- a/clang-tools-extra/clangd/index/remote/server/Server.cpp
+++ b/clang-tools-extra/clangd/index/remote/server/Server.cpp
@@ -15,6 +15,7 @@
 #include "support/Trace.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Error.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/Signals.h"
 
@@ -82,7 +83,7 @@ class RemoteIndexServer final : public SymbolIndex::Service {
   grpc::Status Lookup(grpc::ServerContext *Context,
                       const LookupRequest *Request,
                       grpc::ServerWriter<LookupReply> *Reply) override {
-    trace::Span Tracer(LookupRequest::descriptor()->name());
+    trace::Span Tracer("LookupRequest");
     auto Req = ProtobufMarshaller->fromProtobuf(Request);
     if (!Req) {
       elog("Can not parse LookupRequest from protobuf: {0}", Req.takeError());
@@ -93,6 +94,8 @@ class RemoteIndexServer final : public SymbolIndex::Service {
     Index->lookup(*Req, [&](const clangd::Symbol &Item) {
       auto SerializedItem = ProtobufMarshaller->toProtobuf(Item);
       if (!SerializedItem) {
+        elog("Unable to convert Symbol to protobuf: {0}",
+             SerializedItem.takeError());
         ++FailedToSend;
         return;
       }
@@ -112,7 +115,7 @@ class RemoteIndexServer final : public SymbolIndex::Service {
   grpc::Status FuzzyFind(grpc::ServerContext *Context,
                          const FuzzyFindRequest *Request,
                          grpc::ServerWriter<FuzzyFindReply> *Reply) override {
-    trace::Span Tracer(FuzzyFindRequest::descriptor()->name());
+    trace::Span Tracer("FuzzyFindRequest");
     auto Req = ProtobufMarshaller->fromProtobuf(Request);
     if (!Req) {
       elog("Can not parse FuzzyFindRequest from protobuf: {0}",
@@ -124,6 +127,8 @@ class RemoteIndexServer final : public SymbolIndex::Service {
     bool HasMore = Index->fuzzyFind(*Req, [&](const clangd::Symbol &Item) {
       auto SerializedItem = ProtobufMarshaller->toProtobuf(Item);
       if (!SerializedItem) {
+        elog("Unable to convert Symbol to protobuf: {0}",
+             SerializedItem.takeError());
         ++FailedToSend;
         return;
       }
@@ -142,7 +147,7 @@ class RemoteIndexServer final : public SymbolIndex::Service {
 
   grpc::Status Refs(grpc::ServerContext *Context, const RefsRequest *Request,
                     grpc::ServerWriter<RefsReply> *Reply) override {
-    trace::Span Tracer(RefsRequest::descriptor()->name());
+    trace::Span Tracer("RefsRequest");
     auto Req = ProtobufMarshaller->fromProtobuf(Request);
     if (!Req) {
       elog("Can not parse RefsRequest from protobuf: {0}", Req.takeError());
@@ -153,6 +158,8 @@ class RemoteIndexServer final : public SymbolIndex::Service {
     bool HasMore = Index->refs(*Req, [&](const clangd::Ref &Item) {
       auto SerializedItem = ProtobufMarshaller->toProtobuf(Item);
       if (!SerializedItem) {
+        elog("Unable to convert Ref to protobuf: {0}",
+             SerializedItem.takeError());
         ++FailedToSend;
         return;
       }
@@ -172,7 +179,7 @@ class RemoteIndexServer final : public SymbolIndex::Service {
   grpc::Status Relations(grpc::ServerContext *Context,
                          const RelationsRequest *Request,
                          grpc::ServerWriter<RelationsReply> *Reply) override {
-    trace::Span Tracer(RelationsRequest::descriptor()->name());
+    trace::Span Tracer("RelationsRequest");
     auto Req = ProtobufMarshaller->fromProtobuf(Request);
     if (!Req) {
       elog("Can not parse RelationsRequest from protobuf: {0}",
@@ -185,6 +192,8 @@ class RemoteIndexServer final : public SymbolIndex::Service {
         *Req, [&](const SymbolID &Subject, const clangd::Symbol &Object) {
           auto SerializedItem = ProtobufMarshaller->toProtobuf(Subject, Object);
           if (!SerializedItem) {
+            elog("Unable to convert Relation to protobuf: {0}",
+                 SerializedItem.takeError());
             ++FailedToSend;
             return;
           }
diff --git a/clang-tools-extra/clangd/unittests/remote/MarshallingTests.cpp b/clang-tools-extra/clangd/unittests/remote/MarshallingTests.cpp
index 6abf16a8dd497..ff819146621a2 100644
--- a/clang-tools-extra/clangd/unittests/remote/MarshallingTests.cpp
+++ b/clang-tools-extra/clangd/unittests/remote/MarshallingTests.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "../TestTU.h"
+#include "Index.pb.h"
 #include "TestFS.h"
 #include "index/Index.h"
 #include "index/Ref.h"
@@ -97,11 +98,11 @@ TEST(RemoteMarshallingTest, URITranslation) {
                   "clangd/unittests/remote/MarshallingTests.cpp",
                   Strings);
   auto Serialized = ProtobufMarshaller.toProtobuf(Original);
-  ASSERT_TRUE(Serialized);
+  ASSERT_TRUE(bool(Serialized));
   EXPECT_EQ(Serialized->location().file_path(),
             "clang-tools-extra/clangd/unittests/remote/MarshallingTests.cpp");
   auto Deserialized = ProtobufMarshaller.fromProtobuf(*Serialized);
-  ASSERT_TRUE(Deserialized);
+  ASSERT_TRUE(bool(Deserialized));
   EXPECT_STREQ(Deserialized->Location.FileURI,
                testPathURI("home/my-projects/llvm-project/clang-tools-extra/"
                            "clangd/unittests/remote/MarshallingTests.cpp",
@@ -109,12 +110,16 @@ TEST(RemoteMarshallingTest, URITranslation) {
 
   // Can't have empty paths.
   *Serialized->mutable_location()->mutable_file_path() = std::string();
-  EXPECT_FALSE(ProtobufMarshaller.fromProtobuf(*Serialized));
+  Deserialized = ProtobufMarshaller.fromProtobuf(*Serialized);
+  EXPECT_FALSE(bool(Deserialized));
+  llvm::consumeError(Deserialized.takeError());
 
   clangd::Ref WithInvalidURI;
   // Invalid URI results in serialization failure.
   WithInvalidURI.Location.FileURI = "This is not a URI";
-  EXPECT_FALSE(ProtobufMarshaller.toProtobuf(WithInvalidURI));
+  auto DeserializedRef = ProtobufMarshaller.toProtobuf(WithInvalidURI);
+  EXPECT_FALSE(bool(DeserializedRef));
+  llvm::consumeError(DeserializedRef.takeError());
 
   // Can not use URIs with scheme different from "file".
   auto UnittestURI =
@@ -122,14 +127,18 @@ TEST(RemoteMarshallingTest, URITranslation) {
   ASSERT_TRUE(bool(UnittestURI));
   WithInvalidURI.Location.FileURI =
       Strings.save(UnittestURI->toString()).begin();
-  EXPECT_FALSE(ProtobufMarshaller.toProtobuf(WithInvalidURI));
+  auto DeserializedSymbol = ProtobufMarshaller.toProtobuf(WithInvalidURI);
+  EXPECT_FALSE(bool(DeserializedSymbol));
+  llvm::consumeError(DeserializedSymbol.takeError());
 
   // Paths transmitted over the wire can not be absolute, they have to be
   // relative.
   Ref WithAbsolutePath;
   *WithAbsolutePath.mutable_location()->mutable_file_path() =
       "/usr/local/user/home/HelloWorld.cpp";
-  EXPECT_FALSE(ProtobufMarshaller.fromProtobuf(WithAbsolutePath));
+  Deserialized = ProtobufMarshaller.fromProtobuf(WithAbsolutePath);
+  EXPECT_FALSE(bool(Deserialized));
+  llvm::consumeError(Deserialized.takeError());
 }
 
 TEST(RemoteMarshallingTest, SymbolSerialization) {
@@ -142,9 +151,9 @@ TEST(RemoteMarshallingTest, SymbolSerialization) {
   // Check that symbols are exactly the same if the path to indexed project is
   // the same on indexing machine and the client.
   auto Serialized = ProtobufMarshaller.toProtobuf(Sym);
-  ASSERT_TRUE(Serialized);
+  ASSERT_TRUE(bool(Serialized));
   auto Deserialized = ProtobufMarshaller.fromProtobuf(*Serialized);
-  ASSERT_TRUE(Deserialized);
+  ASSERT_TRUE(bool(Deserialized));
   EXPECT_EQ(toYAML(Sym), toYAML(*Deserialized));
   // Serialized paths are relative and have UNIX slashes.
   EXPECT_EQ(convert_to_slash(Serialized->definition().file_path(),
@@ -156,36 +165,44 @@ TEST(RemoteMarshallingTest, SymbolSerialization) {
   // Missing definition is OK.
   Sym.Definition = clangd::SymbolLocation();
   Serialized = ProtobufMarshaller.toProtobuf(Sym);
-  ASSERT_TRUE(Serialized);
-  EXPECT_TRUE(ProtobufMarshaller.fromProtobuf(*Serialized));
+  ASSERT_TRUE(bool(Serialized));
+  ASSERT_TRUE(bool(ProtobufMarshaller.fromProtobuf(*Serialized)));
 
   // Relative path is absolute.
   *Serialized->mutable_canonical_declaration()->mutable_file_path() =
       convert_to_slash("/path/to/Declaration.h");
-  EXPECT_FALSE(ProtobufMarshaller.fromProtobuf(*Serialized));
+  Deserialized = ProtobufMarshaller.fromProtobuf(*Serialized);
+  EXPECT_FALSE(bool(Deserialized));
+  llvm::consumeError(Deserialized.takeError());
 
   // Fail with an invalid URI.
   Sym.Definition.FileURI = "Not A URI";
-  EXPECT_FALSE(ProtobufMarshaller.toProtobuf(Sym));
+  Serialized = ProtobufMarshaller.toProtobuf(Sym);
+  EXPECT_FALSE(bool(Serialized));
+  llvm::consumeError(Serialized.takeError());
 
   // Schemes other than "file" can not be used.
   auto UnittestURI = URI::create(testPath("home/SomePath.h"), "unittest");
   ASSERT_TRUE(bool(UnittestURI));
   Sym.Definition.FileURI = Strings.save(UnittestURI->toString()).begin();
-  EXPECT_FALSE(ProtobufMarshaller.toProtobuf(Sym));
+  Serialized = ProtobufMarshaller.toProtobuf(Sym);
+  EXPECT_FALSE(bool(Serialized));
+  llvm::consumeError(Serialized.takeError());
 
   // Passing root that is not prefix of the original file path.
   Sym.Definition.FileURI = testPathURI("home/File.h", Strings);
   // Check that the symbol is valid and passing the correct path works.
   Serialized = ProtobufMarshaller.toProtobuf(Sym);
-  ASSERT_TRUE(Serialized);
+  ASSERT_TRUE(bool(Serialized));
   Deserialized = ProtobufMarshaller.fromProtobuf(*Serialized);
-  ASSERT_TRUE(Deserialized);
+  ASSERT_TRUE(bool(Deserialized));
   EXPECT_STREQ(Deserialized->Definition.FileURI,
                testPathURI("home/File.h", Strings));
   // Fail with a wrong root.
   Marshaller WrongMarshaller(testPath("nothome/"), testPath("home/"));
-  EXPECT_FALSE(WrongMarshaller.toProtobuf(Sym));
+  Serialized = WrongMarshaller.toProtobuf(Sym);
+  EXPECT_FALSE(Serialized);
+  llvm::consumeError(Serialized.takeError());
 }
 
 TEST(RemoteMarshallingTest, RefSerialization) {
@@ -208,9 +225,9 @@ TEST(RemoteMarshallingTest, RefSerialization) {
                                 testPath("llvm-project/"));
 
   auto Serialized = ProtobufMarshaller.toProtobuf(Ref);
-  ASSERT_TRUE(Serialized);
+  ASSERT_TRUE(bool(Serialized));
   auto Deserialized = ProtobufMarshaller.fromProtobuf(*Serialized);
-  ASSERT_TRUE(Deserialized);
+  ASSERT_TRUE(bool(Deserialized));
   EXPECT_EQ(toYAML(Ref), toYAML(*Deserialized));
 }
 
@@ -218,60 +235,60 @@ TEST(RemoteMarshallingTest, IncludeHeaderURIs) {
   llvm::BumpPtrAllocator Arena;
   llvm::UniqueStringSaver Strings(Arena);
 
-  llvm::SmallVector<clangd::Symbol::IncludeHeaderWithReferences, 1>
-      ValidHeaders;
+  clangd::Symbol Sym = createSymbol("remote/", Strings);
+
   clangd::Symbol::IncludeHeaderWithReferences Header;
+  // Add only valid headers.
   Header.IncludeHeader = Strings.save(
       URI::createFile("/usr/local/user/home/project/Header.h").toString());
   Header.References = 21;
-  ValidHeaders.push_back(Header);
+  Sym.IncludeHeaders.push_back(Header);
   Header.IncludeHeader = Strings.save("<iostream>");
   Header.References = 100;
-  ValidHeaders.push_back(Header);
+  Sym.IncludeHeaders.push_back(Header);
   Header.IncludeHeader = Strings.save("\"cstdio\"");
   Header.References = 200;
-  ValidHeaders.push_back(Header);
+  Sym.IncludeHeaders.push_back(Header);
+
+  Marshaller ProtobufMarshaller(convert_to_slash("/"), convert_to_slash("/"));
+
+  auto Serialized = ProtobufMarshaller.toProtobuf(Sym);
+  ASSERT_TRUE(bool(Serialized));
+  EXPECT_EQ(static_cast<size_t>(Serialized->headers_size()),
+            Sym.IncludeHeaders.size());
+  auto Deserialized = ProtobufMarshaller.fromProtobuf(*Serialized);
+  ASSERT_TRUE(bool(Deserialized));
+  EXPECT_EQ(toYAML(Sym), toYAML(*Deserialized));
 
-  llvm::SmallVector<clangd::Symbol::IncludeHeaderWithReferences, 1>
-      InvalidHeaders;
   // This is an absolute path to a header: can not be transmitted over the wire.
   Header.IncludeHeader = Strings.save(testPath("project/include/Common.h"));
   Header.References = 42;
-  InvalidHeaders.push_back(Header);
+  Sym.IncludeHeaders.push_back(Header);
+  Serialized = ProtobufMarshaller.toProtobuf(Sym);
+  EXPECT_FALSE(bool(Serialized));
+  llvm::consumeError(Serialized.takeError());
+
+  // Remove last invalid header.
+  Sym.IncludeHeaders.pop_back();
   // This is not a valid header: can not be transmitted over the wire;
   Header.IncludeHeader = Strings.save("NotAHeader");
   Header.References = 5;
-  InvalidHeaders.push_back(Header);
-
-  clangd::Symbol Sym;
-  // Fill in definition and declaration, Symbool will be invalid otherwise.
-  clangd::SymbolLocation Location;
-  Location.Start.setLine(1);
-  Location.Start.setColumn(2);
-  Location.End.setLine(3);
-  Location.End.setColumn(4);
-  Location.FileURI = testPathURI("File.h", Strings);
-  Sym.Definition = Location;
-  Sym.CanonicalDeclaration = Location;
-
-  // Try to serialize all headers but only valid ones will end up in Protobuf
-  // message.
-  auto AllHeaders = ValidHeaders;
-  AllHeaders.insert(AllHeaders.end(), InvalidHeaders.begin(),
-                    InvalidHeaders.end());
-  Sym.IncludeHeaders = AllHeaders;
-
-  Marshaller ProtobufMarshaller(convert_to_slash("/"), convert_to_slash("/"));
-
-  auto Serialized = ProtobufMarshaller.toProtobuf(Sym);
-  ASSERT_TRUE(Serialized);
-  EXPECT_EQ(static_cast<size_t>(Serialized->headers_size()),
-            ValidHeaders.size());
-  auto Deserialized = ProtobufMarshaller.fromProtobuf(*Serialized);
-  ASSERT_TRUE(Deserialized);
+  Sym.IncludeHeaders.push_back(Header);
+  Serialized = ProtobufMarshaller.toProtobuf(Sym);
+  EXPECT_FALSE(bool(Serialized));
+  llvm::consumeError(Serialized.takeError());
 
-  Sym.IncludeHeaders = ValidHeaders;
-  EXPECT_EQ(toYAML(Sym), toYAML(*Deserialized));
+  // Try putting an invalid header into already serialized symbol.
+  Sym.IncludeHeaders.pop_back();
+  Serialized = ProtobufMarshaller.toProtobuf(Sym);
+  ASSERT_TRUE(bool(Serialized));
+  HeaderWithReferences InvalidHeader;
+  InvalidHeader.set_header(convert_to_slash("/absolute/path/Header.h"));
+  InvalidHeader.set_references(9000);
+  *Serialized->add_headers() = InvalidHeader;
+  Deserialized = ProtobufMarshaller.fromProtobuf(*Serialized);
+  EXPECT_FALSE(bool(Deserialized));
+  llvm::consumeError(Deserialized.takeError());
 }
 
 TEST(RemoteMarshallingTest, LookupRequestSerialization) {
@@ -294,7 +311,7 @@ TEST(RemoteMarshallingTest, LookupRequestFailingSerialization) {
   auto Serialized = ProtobufMarshaller.toProtobuf(Request);
   Serialized.add_ids("Invalid Symbol ID");
   auto Deserialized = ProtobufMarshaller.fromProtobuf(&Serialized);
-  EXPECT_FALSE(Deserialized);
+  EXPECT_FALSE(bool(Deserialized));
   llvm::consumeError(Deserialized.takeError());
 }
 
@@ -340,7 +357,7 @@ TEST(RemoteMarshallingTest, RefsRequestFailingSerialization) {
   auto Serialized = ProtobufMarshaller.toProtobuf(Request);
   Serialized.add_ids("Invalid Symbol ID");
   auto Deserialized = ProtobufMarshaller.fromProtobuf(&Serialized);
-  EXPECT_FALSE(Deserialized);
+  EXPECT_FALSE(bool(Deserialized));
   llvm::consumeError(Deserialized.takeError());
 }
 
@@ -375,7 +392,7 @@ TEST(RemoteMarshallingTest, RelationsRequestFailingSerialization) {
   Serialized.add_subjects("ZZZZZZZZZZZZZZZZ");
   Marshaller ProtobufMarshaller(testPath("remote/"), testPath("local/"));
   auto Deserialized = ProtobufMarshaller.fromProtobuf(&Serialized);
-  EXPECT_FALSE(Deserialized);
+  EXPECT_FALSE(bool(Deserialized));
   llvm::consumeError(Deserialized.takeError());
 }
 
@@ -387,8 +404,9 @@ TEST(RemoteMarshallingTest, RelationsSerializion) {
   SymbolID ID = llvm::cantFail(SymbolID::fromStr("0000000000000002"));
   Marshaller ProtobufMarshaller(testPath("remote/"), testPath("local/"));
   auto Serialized = ProtobufMarshaller.toProtobuf(ID, Sym);
+  ASSERT_TRUE(bool(Serialized));
   auto Deserialized = ProtobufMarshaller.fromProtobuf(*Serialized);
-  ASSERT_TRUE(Deserialized);
+  ASSERT_TRUE(bool(Deserialized));
   EXPECT_THAT(Deserialized->first, ID);
   EXPECT_THAT(Deserialized->second.ID, Sym.ID);
 }
@@ -396,11 +414,16 @@ TEST(RemoteMarshallingTest, RelationsSerializion) {
 TEST(RemoteMarshallingTest, RelativePathToURITranslation) {
   Marshaller ProtobufMarshaller(/*RemoteIndexRoot=*/"",
                                 /*LocalIndexRoot=*/testPath("home/project/"));
-  EXPECT_TRUE(ProtobufMarshaller.relativePathToURI("lib/File.cpp"));
+  auto URIString = ProtobufMarshaller.relativePathToURI("lib/File.cpp");
+  ASSERT_TRUE(bool(URIString));
   // RelativePath can not be absolute.
-  EXPECT_FALSE(ProtobufMarshaller.relativePathToURI("/lib/File.cpp"));
+  URIString = ProtobufMarshaller.relativePathToURI("/lib/File.cpp");
+  EXPECT_FALSE(bool(URIString));
+  llvm::consumeError(URIString.takeError());
   // RelativePath can not be empty.
-  EXPECT_FALSE(ProtobufMarshaller.relativePathToURI(std::string()));
+  URIString = ProtobufMarshaller.relativePathToURI(std::string());
+  EXPECT_FALSE(bool(URIString));
+  llvm::consumeError(URIString.takeError());
 }
 
 TEST(RemoteMarshallingTest, URIToRelativePathTranslation) {
@@ -408,14 +431,17 @@ TEST(RemoteMarshallingTest, URIToRelativePathTranslation) {
   llvm::UniqueStringSaver Strings(Arena);
   Marshaller ProtobufMarshaller(/*RemoteIndexRoot=*/testPath("remote/project/"),
                                 /*LocalIndexRoot=*/"");
-  EXPECT_TRUE(ProtobufMarshaller.uriToRelativePath(
-      testPathURI("remote/project/lib/File.cpp", Strings)));
+  auto RelativePath = ProtobufMarshaller.uriToRelativePath(
+      testPathURI("remote/project/lib/File.cpp", Strings));
+  ASSERT_TRUE(bool(RelativePath));
   // RemoteIndexRoot has to be be a prefix of the file path.
   Marshaller WrongMarshaller(
       /*RemoteIndexRoot=*/testPath("remote/other/project/"),
       /*LocalIndexRoot=*/"");
-  EXPECT_FALSE(WrongMarshaller.uriToRelativePath(
-      testPathURI("remote/project/lib/File.cpp", Strings)));
+  RelativePath = WrongMarshaller.uriToRelativePath(
+      testPathURI("remote/project/lib/File.cpp", Strings));
+  EXPECT_FALSE(bool(RelativePath));
+  llvm::consumeError(RelativePath.takeError());
 }
 
 } // namespace

From 161882816540fc011554e4a820ab896278491b6a Mon Sep 17 00:00:00 2001
From: Kadir Cetinkaya <kadircet@google.com>
Date: Thu, 30 Jul 2020 21:38:40 +0200
Subject: [PATCH 0723/1035] [clang][Syntax] syntax::Arena doesnt own
 TokenBuffer

Currently an Arena can only be built while consuming a TokenBuffer,
some users (like clangd) might want to share a TokenBuffer with multiple
compenents. This patch changes Arena's TokenBuffer member to be a reference so
that it can be created with read-only token buffers.

Differential Revision: https://reviews.llvm.org/D84973
---
 clang/include/clang/Tooling/Syntax/Tree.h   |  4 ++--
 clang/lib/Tooling/Syntax/Tree.cpp           |  4 ++--
 clang/unittests/Tooling/Syntax/TreeTest.cpp | 21 ++++++++++++++-------
 3 files changed, 18 insertions(+), 11 deletions(-)

diff --git a/clang/include/clang/Tooling/Syntax/Tree.h b/clang/include/clang/Tooling/Syntax/Tree.h
index bc581004c46e6..dd51e0a740708 100644
--- a/clang/include/clang/Tooling/Syntax/Tree.h
+++ b/clang/include/clang/Tooling/Syntax/Tree.h
@@ -39,7 +39,7 @@ namespace syntax {
 class Arena {
 public:
   Arena(SourceManager &SourceMgr, const LangOptions &LangOpts,
-        TokenBuffer Tokens);
+        const TokenBuffer &Tokens);
 
   const SourceManager &sourceManager() const { return SourceMgr; }
   const LangOptions &langOptions() const { return LangOpts; }
@@ -56,7 +56,7 @@ class Arena {
 private:
   SourceManager &SourceMgr;
   const LangOptions &LangOpts;
-  TokenBuffer Tokens;
+  const TokenBuffer &Tokens;
   /// IDs and storage for additional tokenized files.
   llvm::DenseMap<FileID, std::vector<syntax::Token>> ExtraTokens;
   /// Keeps all the allocated nodes and their intermediate data structures.
diff --git a/clang/lib/Tooling/Syntax/Tree.cpp b/clang/lib/Tooling/Syntax/Tree.cpp
index 37579e6145b65..2944819afc96c 100644
--- a/clang/lib/Tooling/Syntax/Tree.cpp
+++ b/clang/lib/Tooling/Syntax/Tree.cpp
@@ -33,8 +33,8 @@ static void traverse(syntax::Node *N,
 } // namespace
 
 syntax::Arena::Arena(SourceManager &SourceMgr, const LangOptions &LangOpts,
-                     TokenBuffer Tokens)
-    : SourceMgr(SourceMgr), LangOpts(LangOpts), Tokens(std::move(Tokens)) {}
+                     const TokenBuffer &Tokens)
+    : SourceMgr(SourceMgr), LangOpts(LangOpts), Tokens(Tokens) {}
 
 const clang::syntax::TokenBuffer &syntax::Arena::tokenBuffer() const {
   return Tokens;
diff --git a/clang/unittests/Tooling/Syntax/TreeTest.cpp b/clang/unittests/Tooling/Syntax/TreeTest.cpp
index bd639aa581661..a722ca2b1a45a 100644
--- a/clang/unittests/Tooling/Syntax/TreeTest.cpp
+++ b/clang/unittests/Tooling/Syntax/TreeTest.cpp
@@ -35,6 +35,7 @@
 #include "gmock/gmock.h"
 #include "gtest/gtest.h"
 #include <cstdlib>
+#include <memory>
 
 using namespace clang;
 
@@ -59,22 +60,25 @@ class SyntaxTreeTest : public ::testing::Test,
     class BuildSyntaxTree : public ASTConsumer {
     public:
       BuildSyntaxTree(syntax::TranslationUnit *&Root,
+                      std::unique_ptr<syntax::TokenBuffer> &TB,
                       std::unique_ptr<syntax::Arena> &Arena,
                       std::unique_ptr<syntax::TokenCollector> Tokens)
-          : Root(Root), Arena(Arena), Tokens(std::move(Tokens)) {
+          : Root(Root), TB(TB), Arena(Arena), Tokens(std::move(Tokens)) {
         assert(this->Tokens);
       }
 
       void HandleTranslationUnit(ASTContext &Ctx) override {
-        Arena = std::make_unique<syntax::Arena>(Ctx.getSourceManager(),
-                                                Ctx.getLangOpts(),
-                                                std::move(*Tokens).consume());
+        TB =
+            std::make_unique<syntax::TokenBuffer>(std::move(*Tokens).consume());
         Tokens = nullptr; // make sure we fail if this gets called twice.
+        Arena = std::make_unique<syntax::Arena>(Ctx.getSourceManager(),
+                                                Ctx.getLangOpts(), *TB);
         Root = syntax::buildSyntaxTree(*Arena, *Ctx.getTranslationUnitDecl());
       }
 
     private:
       syntax::TranslationUnit *&Root;
+      std::unique_ptr<syntax::TokenBuffer> &TB;
       std::unique_ptr<syntax::Arena> &Arena;
       std::unique_ptr<syntax::TokenCollector> Tokens;
     };
@@ -82,20 +86,22 @@ class SyntaxTreeTest : public ::testing::Test,
     class BuildSyntaxTreeAction : public ASTFrontendAction {
     public:
       BuildSyntaxTreeAction(syntax::TranslationUnit *&Root,
+                            std::unique_ptr<syntax::TokenBuffer> &TB,
                             std::unique_ptr<syntax::Arena> &Arena)
-          : Root(Root), Arena(Arena) {}
+          : Root(Root), TB(TB), Arena(Arena) {}
 
       std::unique_ptr<ASTConsumer>
       CreateASTConsumer(CompilerInstance &CI, StringRef InFile) override {
         // We start recording the tokens, ast consumer will take on the result.
         auto Tokens =
             std::make_unique<syntax::TokenCollector>(CI.getPreprocessor());
-        return std::make_unique<BuildSyntaxTree>(Root, Arena,
+        return std::make_unique<BuildSyntaxTree>(Root, TB, Arena,
                                                  std::move(Tokens));
       }
 
     private:
       syntax::TranslationUnit *&Root;
+      std::unique_ptr<syntax::TokenBuffer> &TB;
       std::unique_ptr<syntax::Arena> &Arena;
     };
 
@@ -132,7 +138,7 @@ class SyntaxTreeTest : public ::testing::Test,
     Compiler.setSourceManager(SourceMgr.get());
 
     syntax::TranslationUnit *Root = nullptr;
-    BuildSyntaxTreeAction Recorder(Root, this->Arena);
+    BuildSyntaxTreeAction Recorder(Root, this->TB, this->Arena);
 
     // Action could not be executed but the frontend didn't identify any errors
     // in the code ==> problem in setting up the action.
@@ -204,6 +210,7 @@ class SyntaxTreeTest : public ::testing::Test,
       new SourceManager(*Diags, *FileMgr);
   std::shared_ptr<CompilerInvocation> Invocation;
   // Set after calling buildTree().
+  std::unique_ptr<syntax::TokenBuffer> TB;
   std::unique_ptr<syntax::Arena> Arena;
 };
 

From 03116a9f8c2fc98577e153083aaf9b6a701ab8f9 Mon Sep 17 00:00:00 2001
From: Joachim Protze <protze@itc.rwth-aachen.de>
Date: Thu, 30 Jul 2020 09:28:17 +0200
Subject: [PATCH 0724/1035] [OpenMP] Use weak attribute in interface only for
 static library

This is to address the issue reported at:
https://bugs.llvm.org/show_bug.cgi?id=46863

Since weak is meaningless for a shared library interface function, this patch
disables the attribute, when the OpenMP library is built as shared library.

ompt_start_tool is not an interface function, but a internally called function
possibly implemented by an OMPT tool.
This function needs to be weak if possible to allow overwriting ompt_start_tool
with a function implementation built into the application.

Differential Revision: https://reviews.llvm.org/D84871
---
 openmp/runtime/src/kmp_ftn_entry.h   |  8 ++++----
 openmp/runtime/src/kmp_os.h          | 10 ++++++++--
 openmp/runtime/src/ompt-specific.cpp |  2 +-
 3 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/openmp/runtime/src/kmp_ftn_entry.h b/openmp/runtime/src/kmp_ftn_entry.h
index ab57907e088e3..b4b0dea0d1afa 100644
--- a/openmp/runtime/src/kmp_ftn_entry.h
+++ b/openmp/runtime/src/kmp_ftn_entry.h
@@ -939,7 +939,7 @@ void FTN_STDCALL KMP_EXPAND_NAME(FTN_SET_DEFAULT_DEVICE)(int KMP_DEREF arg) {
 
 // Get number of NON-HOST devices.
 // libomptarget, if loaded, provides this function in api.cpp.
-int FTN_STDCALL KMP_EXPAND_NAME(FTN_GET_NUM_DEVICES)(void) KMP_WEAK_ATTRIBUTE;
+int FTN_STDCALL KMP_EXPAND_NAME(FTN_GET_NUM_DEVICES)(void) KMP_WEAK_ATTRIBUTE_EXTERNAL;
 int FTN_STDCALL KMP_EXPAND_NAME(FTN_GET_NUM_DEVICES)(void) {
 #if KMP_MIC || KMP_OS_DARWIN || KMP_OS_WINDOWS || defined(KMP_STUB)
   return 0;
@@ -957,13 +957,13 @@ int FTN_STDCALL KMP_EXPAND_NAME(FTN_GET_NUM_DEVICES)(void) {
 
 // This function always returns true when called on host device.
 // Compiler/libomptarget should handle when it is called inside target region.
-int FTN_STDCALL KMP_EXPAND_NAME(FTN_IS_INITIAL_DEVICE)(void) KMP_WEAK_ATTRIBUTE;
+int FTN_STDCALL KMP_EXPAND_NAME(FTN_IS_INITIAL_DEVICE)(void) KMP_WEAK_ATTRIBUTE_EXTERNAL;
 int FTN_STDCALL KMP_EXPAND_NAME(FTN_IS_INITIAL_DEVICE)(void) {
   return 1; // This is the host
 }
 
 // libomptarget, if loaded, provides this function
-int FTN_STDCALL FTN_GET_INITIAL_DEVICE(void) KMP_WEAK_ATTRIBUTE;
+int FTN_STDCALL FTN_GET_INITIAL_DEVICE(void) KMP_WEAK_ATTRIBUTE_EXTERNAL;
 int FTN_STDCALL FTN_GET_INITIAL_DEVICE(void) {
 #if KMP_MIC || KMP_OS_DARWIN || KMP_OS_WINDOWS || defined(KMP_STUB)
   return KMP_HOST_DEVICE;
@@ -1318,7 +1318,7 @@ int FTN_STDCALL KMP_EXPAND_NAME(FTN_GET_MAX_TASK_PRIORITY)(void) {
 // This function will be defined in libomptarget. When libomptarget is not
 // loaded, we assume we are on the host and return KMP_HOST_DEVICE.
 // Compiler/libomptarget will handle this if called inside target.
-int FTN_STDCALL FTN_GET_DEVICE_NUM(void) KMP_WEAK_ATTRIBUTE;
+int FTN_STDCALL FTN_GET_DEVICE_NUM(void) KMP_WEAK_ATTRIBUTE_EXTERNAL;
 int FTN_STDCALL FTN_GET_DEVICE_NUM(void) { return KMP_HOST_DEVICE; }
 
 // Compiler will ensure that this is only called from host in sequential region
diff --git a/openmp/runtime/src/kmp_os.h b/openmp/runtime/src/kmp_os.h
index 33735cf455c7e..c658f90975489 100644
--- a/openmp/runtime/src/kmp_os.h
+++ b/openmp/runtime/src/kmp_os.h
@@ -338,10 +338,16 @@ extern "C" {
 #define KMP_ALIAS(alias_of) __attribute__((alias(alias_of)))
 #endif
 
+#if KMP_HAVE_WEAK_ATTRIBUTE && !KMP_DYNAMIC_LIB
+#define KMP_WEAK_ATTRIBUTE_EXTERNAL __attribute__((weak))
+#else
+#define KMP_WEAK_ATTRIBUTE_EXTERNAL /* Nothing */
+#endif
+
 #if KMP_HAVE_WEAK_ATTRIBUTE
-#define KMP_WEAK_ATTRIBUTE __attribute__((weak))
+#define KMP_WEAK_ATTRIBUTE_INTERNAL __attribute__((weak))
 #else
-#define KMP_WEAK_ATTRIBUTE /* Nothing */
+#define KMP_WEAK_ATTRIBUTE_INTERNAL /* Nothing */
 #endif
 
 // Define KMP_VERSION_SYMBOL and KMP_EXPAND_NAME
diff --git a/openmp/runtime/src/ompt-specific.cpp b/openmp/runtime/src/ompt-specific.cpp
index a7288f08a6614..9be699110fc6b 100644
--- a/openmp/runtime/src/ompt-specific.cpp
+++ b/openmp/runtime/src/ompt-specific.cpp
@@ -27,7 +27,7 @@
 #define THREAD_LOCAL __thread
 #endif
 
-#define OMPT_WEAK_ATTRIBUTE KMP_WEAK_ATTRIBUTE
+#define OMPT_WEAK_ATTRIBUTE KMP_WEAK_ATTRIBUTE_INTERNAL
 
 //******************************************************************************
 // macros

From eef1bfb2d219191cee16ee24efbf2d204488696c Mon Sep 17 00:00:00 2001
From: Jakub Lichman <limo@google.com>
Date: Fri, 31 Jul 2020 13:18:11 +0200
Subject: [PATCH 0725/1035] [mlir][Linalg] Conv {1,2,3}D ops defined with TC
 syntax

Replaced definition of named ND ConvOps with tensor comprehension
syntax which reduces boilerplate code significantly. Furthermore,
new ops to support TF convolutions added (without strides and dilations).

Reviewed By: nicolasvasilache

Differential Revision: https://reviews.llvm.org/D84628
---
 .../Linalg/IR/LinalgNamedStructuredOpsSpec.tc |  52 ++++++++
 .../mlir/Dialect/Linalg/IR/LinalgOps.h        |   8 --
 .../Dialect/Linalg/IR/LinalgStructuredOps.td  | 125 ------------------
 .../LinalgToStandard/LinalgToStandard.cpp     |   3 -
 mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp      |  80 +++++------
 mlir/lib/Dialect/Linalg/Transforms/Loops.cpp  |  73 +++-------
 mlir/test/Dialect/Linalg/invalid.mlir         |   8 --
 mlir/test/Dialect/Linalg/loops.mlir           |  82 ++++++------
 8 files changed, 146 insertions(+), 285 deletions(-)

diff --git a/mlir/include/mlir/Dialect/Linalg/IR/LinalgNamedStructuredOpsSpec.tc b/mlir/include/mlir/Dialect/Linalg/IR/LinalgNamedStructuredOpsSpec.tc
index 056f0723e92dd..27d4330a54d5f 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/LinalgNamedStructuredOpsSpec.tc
+++ b/mlir/include/mlir/Dialect/Linalg/IR/LinalgNamedStructuredOpsSpec.tc
@@ -17,3 +17,55 @@ ods_def<BatchMatmulOp>:
 def batch_matmul(A: f32(Batch, M, K), B: f32(Batch, K, N)) -> (C: f32(Batch, M, N)) {
   C(b, m, n) = std_addf<k>(std_mulf(A(b, m, k), B(b, k, n)));
 }
+
+ods_def<ConvWOp>:
+def conv_1d(I: f32(W), K: f32(KW)) -> (O: f32(W)) {
+  O(w) = std_addf(O(w), std_mulf(I(w + kw), K(kw)));
+}
+
+ods_def<ConvNWCOp>:
+def conv_1d_nwc(I: f32(N, W, C), K: f32(F, KW, C)) -> (O: f32(N, W, F)) {
+  O(n, w, f) = std_addf(O(n, w, f),
+    std_mulf(I(n, w + kw, c), K(f, kw, c)));
+}
+
+ods_def<ConvNCWOp>:
+def conv_1d_ncw(I: f32(N, C, W), K: f32(F, C, KW)) -> (O: f32(N, F, W)) {
+  O(n, f, w) = std_addf(O(n, f, w),
+    std_mulf(I(n, c, w + kw), K(f, c, kw)));
+}
+
+ods_def<ConvHWOp>:
+def conv_2d(I: f32(H, W), K: f32(KH, KW)) -> (O: f32(H, W)) {
+  O(h, w) = std_addf(O(h, w), std_mulf(I(h + kh, w + kw), K(kh, kw)));
+}
+
+ods_def<ConvNHWCOp>:
+def conv_2d_nhwc(I: f32(N, H, W, C), K: f32(F, KH, KW, C)) -> (O: f32(N, H, W, F)) {
+  O(n, h, w, f) = std_addf(O(n, h, w, f),
+    std_mulf(I(n, h + kh, w + kw, c), K(f, kh, kw, c)));
+}
+
+ods_def<ConvNCHWOp>:
+def conv_2d_nchw(I: f32(N, C, H, W), K: f32(F, C, KH, KW)) -> (O: f32(N, F, H, W)) {
+  O(n, f, h, w) = std_addf(O(n, f, h, w),
+    std_mulf(I(n, c, h + kh, w + kw), K(f, c, kh, kw)));
+}
+
+ods_def<ConvDHWOp>:
+def conv_3d(I: f32(D, H, W), K: f32(KD, KH, KW)) -> (O: f32(D, H, W)) {
+  O(d, h, w) = std_addf(O(d, h, w),
+    std_mulf(I(d + kd, h + kh, w + kw), K(kd, kh, kw)));
+}
+
+ods_def<ConvNDHWCOp>:
+def conv_3d_ndhwc(I: f32(N, D, H, W, C), K: f32(F, KD, KH, KW, C)) -> (O: f32(N, D, H, W, F)) {
+  O(n, d, h, w, f) = std_addf(O(n, d, h, w, f),
+    std_mulf(I(n, d + kd, h + kh, w + kw, c), K(f, kd, kh, kw, c)));
+}
+
+ods_def<ConvNCDHWOp>:
+def conv_3d_ncdhw(I: f32(N, C, D, H, W), K: f32(F, C, KD, KH, KW)) -> (O: f32(N, F, D, H, W)) {
+  O(n, f, d, h, w) = std_addf(O(n, f, d, h, w),
+    std_mulf(I(n, c, d + kd, h + kh, w + kw), K(f, c, kd, kh, kw)));
+}
\ No newline at end of file
diff --git a/mlir/include/mlir/Dialect/Linalg/IR/LinalgOps.h b/mlir/include/mlir/Dialect/Linalg/IR/LinalgOps.h
index 75e6599bf9fee..21bff4185abf8 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/LinalgOps.h
+++ b/mlir/include/mlir/Dialect/Linalg/IR/LinalgOps.h
@@ -85,14 +85,6 @@ AffineMap extractOrIdentityMap(Optional<AffineMap> maybeMap, unsigned rank,
 SmallVector<AffineExpr, 4> concat(ArrayRef<AffineExpr> a,
                                   ArrayRef<AffineExpr> b);
 
-/// Generates indexing maps for convolution with the following structure:
-/// input:   (m_1, ..., m_r, n_1, ..., n_r) -> (m_1 + n_1, ..., m_r + n_r)
-/// kernel:  (m_1, ..., m_r, n_1, ..., n_r) -> (n_1, ..., n_r)
-/// output:  (m_1, ..., m_r, n_1, ..., n_r) -> (m_1, ..., m_r)
-/// where r is the rank of the input, kernel and output
-llvm::Optional<SmallVector<AffineMap, 8>>
-createConvNDIndexingMaps(MLIRContext *context, unsigned rank);
-
 #include "mlir/Dialect/Linalg/IR/LinalgStructuredOpsInterfaces.h.inc"
 
 #define GET_OP_CLASSES
diff --git a/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOps.td b/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOps.td
index 84ae8e440bee6..1e3321af981e6 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOps.td
+++ b/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOps.td
@@ -180,131 +180,6 @@ def FillOp : LinalgStructured_Op<"fill", [NInputs<0>, NOutputs<1>]> {
   let hasFolder = 1;
 }
 
-class ConvOpBase<string mnemonic, int N>
-  : LinalgStructured_Op<mnemonic, [NInputs<2>, NOutputs<1>]> {
-  let description = [{
-    Base operation for any N-D Convolution implemented as a linalg.generic op.
-
-    Usage:
-
-    ```mlir
-    linalg.conv<N>D(%in, %filter, %out) : memref<(?x)+f32>,
-                                          memref<(?x)+f32>,
-                                          memref<(?x)+f32>
-    ```
-
-    where    %in:     input array
-             %filter: kernel or filter that will be applied on the input array
-             %out:    output array
-
-    and rank of the operands is *N*.
-
-    Every child convolution is expressed as:
-
-    ```mlir
-    #conv_trait = {
-      args_in = 2,
-      args_out = 1,
-      indexing_maps = #conv_accesses,
-      library_call  = "linalg_conv",
-      iterator_types = [("parallel", "parallel")+], // `2 * rank` iterators
-    }
-
-    linalg.generic #conv_trait %in, %filter, %out {
-      ^bb0(%a: f32, %b: f32, %c: f32) :
-        %d = mulf %a, %b : f32
-        %e = addf %c, %d : f32
-        linalg.yield %e : f32
-    } : memref<(?x)+f32>,
-        memref<(?x)+f32>,
-        memref<(?x)+f32>
-    ```
-
-    where #conv_accesses depend on the rank of the operands and thus
-    can be found in the documentation of each N-D case.
-    Please note that the input array is expected to be right-padded i.e.
-    the size of the input is greater than or equal to the size of the output
-    + size of the kernel - 1. If it is not padded the behavior of the op
-    is undefined.
-  }];
-
-  let arguments = (ins AnyStridedMemRefOfRank<N>,
-                       AnyStridedMemRefOfRank<N>,
-                       AnyStridedMemRefOfRank<N>);
-
-  let extraClassDeclaration = libraryCallName # [{
-    llvm::Optional<SmallVector<StringRef, 8>> referenceIterators() {
-      // There are always 2 loops for each dimension of the convolution. First
-      // iterates output and second kernel. Since ranks of all 3 operands must
-      // be the same it does not matter which operand is picked to get the rank.
-      // Loops iterating the output can be parallelized and thus are marked as
-      // "parallel" while loops iterating the kernel are accumulating the
-      // products and therefore are marked as "reduction".
-      unsigned rank = getInputShapedType(0).getRank();
-      SmallVector<StringRef, 8> parallel(rank, getParallelIteratorTypeName());
-      SmallVector<StringRef, 8> reduction(rank, getReductionIteratorTypeName());
-      parallel.insert(parallel.end(), reduction.begin(), reduction.end());
-      return parallel;
-    }
-
-    // Generates indexing maps with the following structure:
-    // input:   (m_1, ..., m_r, n_1, ..., n_r) -> (m_1 + n_1, ..., m_r + n_r)
-    // kernel:  (m_1, ..., m_r, n_1, ..., n_r) -> (n_1, ..., n_r)
-    // output:  (m_1, ..., m_r, n_1, ..., n_r) -> (m_1, ..., m_r)
-    // where r is the rank of the input, kernel and output
-    llvm::Optional<SmallVector<AffineMap, 8>> referenceIndexingMaps() {
-      MLIRContext *context = getContext();
-      unsigned rank = getInputShapedType(0).getRank();
-      return createConvNDIndexingMaps(context, rank);
-    }
-  }];
-
-  let hasFolder = 1;
-  let verifier = [{ return ::verify(*this); }];
-}
-
-def Conv1DOp : ConvOpBase<"conv1D", 1> {
-  let description = [{
-    *1D* convolution which uses following affine maps to access operands:
-
-    ```mlir
-    #conv_accesses = [
-      affine_map<(m, n) -> (m + n)>, // in
-      affine_map<(m, n) -> (n)>, // kernel
-      affine_map<(m, n) -> (m)> // out
-    ]
-    ```
-  }];
-}
-
-def Conv2DOp : ConvOpBase<"conv2D", 2> {
-  let description = [{
-    *2D* convolution which uses following affine maps to access operands:
-
-    ```mlir
-    #conv_accesses = [
-      affine_map<(m1, m2, n1, n2) -> (m1 + n1, m2 + n2)>, // in
-      affine_map<(m1, m2, n1, n2) -> (n1, n2)>, // kernel
-      affine_map<(m1, m2, n1, n2) -> (m1, m2) // out
-    ]
-    ```
-  }];
-}
-
-def Conv3DOp : ConvOpBase<"conv3D", 3> {
-  let description = [{
-    *3D* convolution which uses following affine maps to access operands:
-
-    ```mlir
-    #conv_accesses = [
-      affine_map<(m1, m2, m3, n1, n2, n3) -> (m1 + n1, m2 + n2, m3 + n3)>, // in
-      affine_map<(m1, m2, m3, n1, n2, n3) -> (n1, n2, n3)>, // kernel
-      affine_map<(m1, m2, m3, n1, n2, n3) -> (m1, m2, m3)> // out
-    ]
-    ```
-  }];
-}
-
 /// A base class for pooling operation such as conv. The arguments must contain
 /// optional arguments `strides`, `dilations` and `padding` with following type:
 ///   OptionalAttr<I64ArrayAttr>:$strides
diff --git a/mlir/lib/Conversion/LinalgToStandard/LinalgToStandard.cpp b/mlir/lib/Conversion/LinalgToStandard/LinalgToStandard.cpp
index 921445bd03b1f..55ffa3f8b6e61 100644
--- a/mlir/lib/Conversion/LinalgToStandard/LinalgToStandard.cpp
+++ b/mlir/lib/Conversion/LinalgToStandard/LinalgToStandard.cpp
@@ -236,9 +236,6 @@ void mlir::populateLinalgToStandardConversionPatterns(
       LinalgOpConversion<PoolingMinOp>,
       LinalgOpConversion<PoolingSumOp>,
       LinalgOpConversion<CopyOp>,
-      LinalgOpConversion<Conv1DOp>,
-      LinalgOpConversion<Conv2DOp>,
-      LinalgOpConversion<Conv3DOp>,
       LinalgOpConversion<FillOp>,
       LinalgOpConversion<GenericOp>,
       LinalgOpConversion<IndexedGenericOp>>(ctx);
diff --git a/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp b/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
index e67adf8c20420..03bd71f177169 100644
--- a/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
+++ b/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
@@ -986,17 +986,6 @@ static LogicalResult verifyStrideOrDilation(LinalgPoolingOp op,
   return success();
 }
 
-template <typename ConvNDOp>
-static LogicalResult verify(ConvNDOp op) {
-  auto outputType = op.getOutputShapedType(0).getElementType();
-  auto inputType = op.getInputShapedType(0).getElementType();
-  auto kernelType = op.getInputShapedType(1).getElementType();
-  if (outputType != inputType || inputType != kernelType)
-    return op.emitOpError("expected all element types of operands to match");
-
-  return success();
-}
-
 static LogicalResult verify(ConvOp op) {
   auto oType = op.output().getType().cast<MemRefType>();
   auto fType = op.filter().getType().cast<MemRefType>();
@@ -1107,27 +1096,6 @@ mlir::linalg::weightedPoolingInputIndex(PoolingOp op,
   return res;
 }
 
-llvm::Optional<SmallVector<AffineMap, 8>>
-mlir::linalg::createConvNDIndexingMaps(MLIRContext *context, unsigned rank) {
-  unsigned numDims = rank * 2, idx = 0;
-
-  SmallVector<AffineExpr, 8> dims, in, kernel, out;
-  dims = makeAffineDimExprs(numDims, idx, context);
-  in.reserve(rank);
-  kernel.reserve(rank);
-  out.reserve(rank);
-
-  for (unsigned i = 0; i < rank; i++) {
-    in.push_back(dims[i] + dims[rank + i]);
-    kernel.push_back(dims[rank + i]);
-    out.push_back(dims[i]);
-  }
-
-  return SmallVector<AffineMap, 8>{AffineMap::get(numDims, 0, in, context),
-                                   AffineMap::get(numDims, 0, kernel, context),
-                                   AffineMap::get(numDims, 0, out, context)};
-}
-
 #define INSTANTIATE_WEIGHTED_POOLING_INPUT_INDEX(OP_TYPE)                      \
   template SmallVector<AffineExpr, 4>                                          \
   mlir::linalg::weightedPoolingInputIndex<OP_TYPE>(                            \
@@ -1209,18 +1177,6 @@ LogicalResult FillOp::fold(ArrayRef<Attribute>,
                            SmallVectorImpl<OpFoldResult> &) {
   return foldMemRefCast(*this);
 }
-LogicalResult Conv1DOp::fold(ArrayRef<Attribute>,
-                             SmallVectorImpl<OpFoldResult> &) {
-  return foldMemRefCast(*this);
-}
-LogicalResult Conv2DOp::fold(ArrayRef<Attribute>,
-                             SmallVectorImpl<OpFoldResult> &) {
-  return foldMemRefCast(*this);
-}
-LogicalResult Conv3DOp::fold(ArrayRef<Attribute>,
-                             SmallVectorImpl<OpFoldResult> &) {
-  return foldMemRefCast(*this);
-}
 LogicalResult GenericOp::fold(ArrayRef<Attribute>,
                               SmallVectorImpl<OpFoldResult> &) {
   return foldMemRefCast(*this);
@@ -1362,3 +1318,39 @@ LogicalResult MatvecOp::fold(ArrayRef<Attribute>,
                              SmallVectorImpl<OpFoldResult> &) {
   return foldMemRefCast(*this);
 }
+LogicalResult ConvWOp::fold(ArrayRef<Attribute>,
+                            SmallVectorImpl<OpFoldResult> &) {
+  return foldMemRefCast(*this);
+}
+LogicalResult ConvNWCOp::fold(ArrayRef<Attribute>,
+                              SmallVectorImpl<OpFoldResult> &) {
+  return foldMemRefCast(*this);
+}
+LogicalResult ConvNCWOp::fold(ArrayRef<Attribute>,
+                              SmallVectorImpl<OpFoldResult> &) {
+  return foldMemRefCast(*this);
+}
+LogicalResult ConvHWOp::fold(ArrayRef<Attribute>,
+                             SmallVectorImpl<OpFoldResult> &) {
+  return foldMemRefCast(*this);
+}
+LogicalResult ConvNHWCOp::fold(ArrayRef<Attribute>,
+                               SmallVectorImpl<OpFoldResult> &) {
+  return foldMemRefCast(*this);
+}
+LogicalResult ConvNCHWOp::fold(ArrayRef<Attribute>,
+                               SmallVectorImpl<OpFoldResult> &) {
+  return foldMemRefCast(*this);
+}
+LogicalResult ConvDHWOp::fold(ArrayRef<Attribute>,
+                              SmallVectorImpl<OpFoldResult> &) {
+  return foldMemRefCast(*this);
+}
+LogicalResult ConvNDHWCOp::fold(ArrayRef<Attribute>,
+                                SmallVectorImpl<OpFoldResult> &) {
+  return foldMemRefCast(*this);
+}
+LogicalResult ConvNCDHWOp::fold(ArrayRef<Attribute>,
+                                SmallVectorImpl<OpFoldResult> &) {
+  return foldMemRefCast(*this);
+}
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp b/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp
index db29835e2caa7..281edd9a91f64 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp
@@ -295,61 +295,6 @@ void emitScalarImplementation(ArrayRef<Value> allIvs, FillOp fillOp) {
   nPar > 0 ? O(ivs) = fillOp.value() : O() = fillOp.value();
 }
 
-/// Following functions emit scalar part of the N-D convolution op.
-/// N-D convolution has 2N loops:
-///   1-N: Iterate over the output array *O* with iterators *m1, ..., mN*.
-///   N-2N:. Iterate over the kernel *K* with iterators *n1, ..., nN*.
-///
-/// The scalar part accumulates products of input array *I* values with kernel
-/// ones. The accumulation expression therefore looks like:
-///   O[m1, ..., mN] += I[m1 + n1, ..., mN + nN] * K[n1, ..., nN].
-/// Note that the input array has to be padded in order to prevent
-/// out of bounds accesses.
-template <typename IndexedValueType>
-void emitScalarImplementation(ArrayRef<Value> allIvs, Conv1DOp convOp) {
-  assert(convOp.hasBufferSemantics() &&
-         "expected linalg op with buffer semantics");
-  assert(allIvs.size() == 2);
-  Value m1(allIvs[0]);
-  Value n1(allIvs[1]);
-  IndexedValueType I(convOp.getInput(0)), K(convOp.getInput(1)),
-      O(convOp.getOutputBuffer(0));
-  // Emit scalar form for the 1D conv case.
-  Value i1 = m1 + n1;
-  O(m1) = O(m1) + I(i1) * K(n1);
-}
-
-template <typename IndexedValueType>
-void emitScalarImplementation(ArrayRef<Value> allIvs, Conv2DOp convOp) {
-  assert(convOp.hasBufferSemantics() &&
-         "expected linalg op with buffer semantics");
-  assert(allIvs.size() == 4);
-  Value m1(allIvs[0]), m2(allIvs[1]);
-  Value n1(allIvs[2]), n2(allIvs[3]);
-  IndexedValueType I(convOp.getInput(0)), K(convOp.getInput(1)),
-      O(convOp.getOutputBuffer(0));
-  // Emit scalar form for the 2D conv case.
-  Value i1 = m1 + n1;
-  Value i2 = m2 + n2;
-  O(m1, m2) = O(m1, m2) + I(i1, i2) * K(n1, n2);
-}
-
-template <typename IndexedValueType>
-void emitScalarImplementation(ArrayRef<Value> allIvs, Conv3DOp convOp) {
-  assert(convOp.hasBufferSemantics() &&
-         "expected linalg op with buffer semantics");
-  assert(allIvs.size() == 6);
-  Value m1(allIvs[0]), m2(allIvs[1]), m3(allIvs[2]);
-  Value n1(allIvs[3]), n2(allIvs[4]), n3(allIvs[5]);
-  IndexedValueType I(convOp.getInput(0)), K(convOp.getInput(1)),
-      O(convOp.getOutputBuffer(0));
-  // Emit scalar form for the 3D conv case.
-  Value i1 = m1 + n1;
-  Value i2 = m2 + n2;
-  Value i3 = m3 + n3;
-  O(m1, m2, m3) = O(m1, m2, m3) + I(i1, i2, i3) * K(n1, n2, n3);
-}
-
 template <typename IndexedValueType>
 Value getConvOpInput(ConvOp convOp, StdIndexedValue im,
                      MutableArrayRef<Value> imIdx) {
@@ -738,6 +683,24 @@ static Optional<LinalgLoops> linalgOpToLoopsImplSwitch(Operation *op,
     return linalgOpToLoopsImpl<LoopTy, DotOp>(op, builder);
   if (isa<BatchMatmulOp>(op))
     return linalgOpToLoopsImpl<LoopTy, BatchMatmulOp>(op, builder);
+  if (isa<ConvWOp>(op))
+    return linalgOpToLoopsImpl<LoopTy, ConvWOp>(op, builder);
+  if (isa<ConvNWCOp>(op))
+    return linalgOpToLoopsImpl<LoopTy, ConvNWCOp>(op, builder);
+  if (isa<ConvNCWOp>(op))
+    return linalgOpToLoopsImpl<LoopTy, ConvNCWOp>(op, builder);
+  if (isa<ConvHWOp>(op))
+    return linalgOpToLoopsImpl<LoopTy, ConvHWOp>(op, builder);
+  if (isa<ConvNHWCOp>(op))
+    return linalgOpToLoopsImpl<LoopTy, ConvNHWCOp>(op, builder);
+  if (isa<ConvNCHWOp>(op))
+    return linalgOpToLoopsImpl<LoopTy, ConvNCHWOp>(op, builder);
+  if (isa<ConvDHWOp>(op))
+    return linalgOpToLoopsImpl<LoopTy, ConvDHWOp>(op, builder);
+  if (isa<ConvNDHWCOp>(op))
+    return linalgOpToLoopsImpl<LoopTy, ConvNDHWCOp>(op, builder);
+  if (isa<ConvNCDHWOp>(op))
+    return linalgOpToLoopsImpl<LoopTy, ConvNCDHWOp>(op, builder);
   llvm_unreachable("Unexpected op in linalgOpToLoopsImpl");
 }
 
diff --git a/mlir/test/Dialect/Linalg/invalid.mlir b/mlir/test/Dialect/Linalg/invalid.mlir
index a5a6e9bee34f8..ca59ecd387ec3 100644
--- a/mlir/test/Dialect/Linalg/invalid.mlir
+++ b/mlir/test/Dialect/Linalg/invalid.mlir
@@ -507,11 +507,3 @@ func @named_ops(%a3: memref<?x?x?xf32>, %b3: memref<?x?xf32>, %c3: memref<?x?x?x
   linalg.batch_matmul %a3, %b3, %c3 : (memref<?x?x?xf32>, memref<?x?xf32>, memref<?x?x?xf32>) -> ()
   return
 }
-
-// -----
-
-func @conv_type_mismatch(%in: memref<?xi32>, %filter: memref<?xf32>, %out: memref<?xf32>) {
-  // expected-error @+1 {{expected all element types of operands to match}}
-  linalg.conv1D(%in, %filter, %out) : memref<?xi32>, memref<?xf32>, memref<?xf32>
-  return
-}
diff --git a/mlir/test/Dialect/Linalg/loops.mlir b/mlir/test/Dialect/Linalg/loops.mlir
index ee63d59ca8c46..6af53a2b8d222 100644
--- a/mlir/test/Dialect/Linalg/loops.mlir
+++ b/mlir/test/Dialect/Linalg/loops.mlir
@@ -1288,7 +1288,7 @@ func @conv4d(%in : memref<?x?x?x?xf32>, %filter : memref<?x?x?x?xf32>, %out :  m
 //       CHECKPARALLEL:   store %[[res]], %[[arg2]][%[[i0]], %[[i1]], %[[i2]], %[[i3]]] : memref<?x?x?x?xf32>
 
 func @conv1d_no_symbols(%in : memref<?xf32>, %filter : memref<?xf32>, %out : memref<?xf32>) -> () {
-  linalg.conv1D(%in, %filter, %out) : memref<?xf32>, memref<?xf32>, memref<?xf32>
+  linalg.conv_1d %in, %filter, %out : (memref<?xf32>, memref<?xf32>, memref<?xf32>)
   return
 }
 
@@ -1303,10 +1303,10 @@ func @conv1d_no_symbols(%in : memref<?xf32>, %filter : memref<?xf32>, %out : mem
 //       CHECKLOOP: scf.for %[[b:.*]] = %[[c0]] to %[[dim1]] step %[[c1]] {
 //       CHECKLOOP:   scf.for %[[m:.*]] = %[[c0]] to %[[dim0]] step %[[c1]] {
 //       CHECKLOOP:     %[[aff:.*]] = affine.apply #[[$stride1Dilation1]](%[[b]], %[[m]])
-//       CHECKLOOP:     %[[va:.*]] = load %[[arg1]][%[[m]]] : memref<?xf32>
 //       CHECKLOOP:     %[[vb:.*]] = load %[[arg0]][%[[aff]]] : memref<?xf32>
-//       CHECKLOOP:     %[[inc:.*]] = mulf %[[vb]], %[[va]] : f32
+//       CHECKLOOP:     %[[va:.*]] = load %[[arg1]][%[[m]]] : memref<?xf32>
 //       CHECKLOOP:     %[[vc:.*]] = load %[[arg2]][%[[b]]] : memref<?xf32>
+//       CHECKLOOP:     %[[inc:.*]] = mulf %[[vb]], %[[va]] : f32
 //       CHECKLOOP:     %[[res:.*]] = addf %[[vc]], %[[inc]] : f32
 //       CHECKLOOP:     store %[[res]], %[[arg2]][%[[b]]] : memref<?xf32>
 
@@ -1318,19 +1318,18 @@ func @conv1d_no_symbols(%in : memref<?xf32>, %filter : memref<?xf32>, %out : mem
 //       CHECKPARALLEL: %[[c1:.*]] = constant 1 : index
 //       CHECKPARALLEL: %[[dim0:.*]] = dim %[[arg1]], %[[c0]] : memref<?xf32>
 //       CHECKPARALLEL: %[[dim1:.*]] = dim %[[arg2]], %[[c0]] : memref<?xf32>
-//       CHECKPARALLEL: scf.parallel (%[[b:.*]]) = (%[[c0]]) to (%[[dim1]]) step (%[[c1]]) {
-//       CHECKPARALLEL:   scf.for %[[m:.*]] = %[[c0]] to %[[dim0]] step %[[c1]] {
-//       CHECKPARALLEL:     %[[aff:.*]] = affine.apply #[[$stride1Dilation1]](%[[b]], %[[m]])
-//       CHECKPARALLEL:     %[[va:.*]] = load %[[arg1]][%[[m]]] : memref<?xf32>
-//       CHECKPARALLEL:     %[[vb:.*]] = load %[[arg0]][%[[aff]]] : memref<?xf32>
-//       CHECKPARALLEL:     %[[inc:.*]] = mulf %[[vb]], %[[va]] : f32
-//       CHECKPARALLEL:     %[[vc:.*]] = load %[[arg2]][%[[b]]] : memref<?xf32>
-//       CHECKPARALLEL:     %[[res:.*]] = addf %[[vc]], %[[inc]] : f32
-//       CHECKPARALLEL:     store %[[res]], %[[arg2]][%[[b]]] : memref<?xf32>
+//       CHECKPARALLEL: scf.parallel (%[[b:.*]], %[[m:.*]]) = (%[[c0]], %[[c0]]) to (%[[dim1]], %[[dim0]]) step (%[[c1]], %[[c1]]) {
+//       CHECKPARALLEL:   %[[aff:.*]] = affine.apply #[[$stride1Dilation1]](%[[b]], %[[m]])
+//       CHECKPARALLEL:   %[[vb:.*]] = load %[[arg0]][%[[aff]]] : memref<?xf32>
+//       CHECKPARALLEL:   %[[va:.*]] = load %[[arg1]][%[[m]]] : memref<?xf32>
+//       CHECKPARALLEL:   %[[vc:.*]] = load %[[arg2]][%[[b]]] : memref<?xf32>
+//       CHECKPARALLEL:   %[[inc:.*]] = mulf %[[vb]], %[[va]] : f32
+//       CHECKPARALLEL:   %[[res:.*]] = addf %[[vc]], %[[inc]] : f32
+//       CHECKPARALLEL:   store %[[res]], %[[arg2]][%[[b]]] : memref<?xf32>
 
 
 func @conv2d_no_symbols(%in : memref<?x?xf32>, %filter : memref<?x?xf32>, %out : memref<?x?xf32>) -> () {
-  linalg.conv2D(%in, %filter, %out) : memref<?x?xf32>, memref<?x?xf32>, memref<?x?xf32>
+  linalg.conv_2d %in, %filter, %out : (memref<?x?xf32>, memref<?x?xf32>, memref<?x?xf32>)
   return
 }
 // CHECKLOOP-LABEL: @conv2d_no_symbols
@@ -1349,10 +1348,12 @@ func @conv2d_no_symbols(%in : memref<?x?xf32>, %filter : memref<?x?xf32>, %out :
 //       CHECKLOOP:       scf.for %[[arg6:.*]] = %[[c0]] to %[[dim1]] step %[[c1]] {
 //       CHECKLOOP:         %[[aff:.*]] = affine.apply #[[$stride1Dilation1]](%[[arg3]], %[[arg5]])
 //       CHECKLOOP:         %[[aff2:.*]] = affine.apply #[[$stride1Dilation1]](%[[arg4]], %[[arg6]])
-//       CHECKLOOP:         %[[va:.*]] = load %[[arg1]][%[[arg5]], %[[arg6]]] : memref<?x?xf32>
 //       CHECKLOOP:         %[[vb:.*]] = load %[[arg0]][%[[aff]], %[[aff2]]] : memref<?x?xf32>
-//       CHECKLOOP:         %[[inc:.*]] = mulf %[[vb]], %[[va]] : f32
+
+//       CHECKLOOP:         %[[va:.*]] = load %[[arg1]][%[[arg5]], %[[arg6]]] : memref<?x?xf32>
 //       CHECKLOOP:         %[[vc:.*]] = load %[[arg2]][%[[arg3]], %[[arg4]]] : memref<?x?xf32>
+
+//       CHECKLOOP:         %[[inc:.*]] = mulf %[[vb]], %[[va]] : f32
 //       CHECKLOOP:         %[[res:.*]] = addf %[[vc]], %[[inc]] : f32
 //       CHECKLOOP:         store %[[res]], %[[arg2]][%[[arg3]], %[[arg4]]] : memref<?x?xf32>
 
@@ -1366,21 +1367,19 @@ func @conv2d_no_symbols(%in : memref<?x?xf32>, %filter : memref<?x?xf32>, %out :
 //       CHECKPARALLEL: %[[dim1:.*]] = dim %[[arg1]], %[[c1]] : memref<?x?xf32>
 //       CHECKPARALLEL: %[[dim2:.*]] = dim %[[arg2]], %[[c0]] : memref<?x?xf32>
 //       CHECKPARALLEL: %[[dim3:.*]] = dim %[[arg2]], %[[c1]] : memref<?x?xf32>
-//       CHECKPARALLEL: scf.parallel (%[[arg3:.*]], %[[arg4:.*]]) = (%[[c0]], %[[c0]]) to (%[[dim2]], %[[dim3]]) step (%[[c1]], %[[c1]]) {
-//       CHECKPARALLEL:   scf.for %[[arg5:.*]] = %[[c0]] to %[[dim0]] step %[[c1]] {
-//       CHECKPARALLEL:     scf.for %[[arg6:.*]] = %[[c0]] to %[[dim1]] step %[[c1]] {
-//       CHECKPARALLEL:       %[[aff:.*]] = affine.apply #[[$stride1Dilation1]](%[[arg3]], %[[arg5]])
-//       CHECKPARALLEL:       %[[aff2:.*]] = affine.apply #[[$stride1Dilation1]](%[[arg4]], %[[arg6]])
-//       CHECKPARALLEL:       %[[va:.*]] = load %[[arg1]][%[[arg5]], %[[arg6]]] : memref<?x?xf32>
-//       CHECKPARALLEL:       %[[vb:.*]] = load %[[arg0]][%[[aff]], %[[aff2]]] : memref<?x?xf32>
-//       CHECKPARALLEL:       %[[inc:.*]] = mulf %[[vb]], %[[va]] : f32
-//       CHECKPARALLEL:       %[[vc:.*]] = load %[[arg2]][%[[arg3]], %[[arg4]]] : memref<?x?xf32>
-//       CHECKPARALLEL:       %[[res:.*]] = addf %[[vc]], %[[inc]] : f32
-//       CHECKPARALLEL:       store %[[res]], %[[arg2]][%[[arg3]], %[[arg4]]] : memref<?x?xf32>
+//       CHECKPARALLEL: scf.parallel (%[[arg3:.*]], %[[arg4:.*]], %[[arg5:.*]], %[[arg6:.*]]) = (%[[c0]], %[[c0]], %[[c0]], %[[c0]]) to (%[[dim2]], %[[dim3]], %[[dim0]], %[[dim1]]) step (%[[c1]], %[[c1]], %[[c1]], %[[c1]]) {
+//       CHECKPARALLEL:   %[[aff:.*]] = affine.apply #[[$stride1Dilation1]](%[[arg3]], %[[arg5]])
+//       CHECKPARALLEL:   %[[aff2:.*]] = affine.apply #[[$stride1Dilation1]](%[[arg4]], %[[arg6]])
+//       CHECKPARALLEL:   %[[vb:.*]] = load %[[arg0]][%[[aff]], %[[aff2]]] : memref<?x?xf32>
+//       CHECKPARALLEL:   %[[va:.*]] = load %[[arg1]][%[[arg5]], %[[arg6]]] : memref<?x?xf32>
+//       CHECKPARALLEL:   %[[vc:.*]] = load %[[arg2]][%[[arg3]], %[[arg4]]] : memref<?x?xf32>
+//       CHECKPARALLEL:   %[[inc:.*]] = mulf %[[vb]], %[[va]] : f32
+//       CHECKPARALLEL:   %[[res:.*]] = addf %[[vc]], %[[inc]] : f32
+//       CHECKPARALLEL:   store %[[res]], %[[arg2]][%[[arg3]], %[[arg4]]] : memref<?x?xf32>
 
 
 func @conv3d_no_symbols(%in : memref<?x?x?xf32>, %filter : memref<?x?x?xf32>, %out : memref<?x?x?xf32>) -> () {
-  linalg.conv3D(%in, %filter, %out) : memref<?x?x?xf32>, memref<?x?x?xf32>, memref<?x?x?xf32>
+  linalg.conv_3d %in, %filter, %out : (memref<?x?x?xf32>, memref<?x?x?xf32>, memref<?x?x?xf32>)
   return
 }
 
@@ -1406,10 +1405,12 @@ func @conv3d_no_symbols(%in : memref<?x?x?xf32>, %filter : memref<?x?x?xf32>, %o
 //       CHECKLOOP:             %[[aff:.*]] = affine.apply #[[$stride1Dilation1]](%[[arg3]], %[[arg6]])
 //       CHECKLOOP:             %[[aff2:.*]] = affine.apply #[[$stride1Dilation1]](%[[arg4]], %[[arg7]])
 //       CHECKLOOP:             %[[aff3:.*]] = affine.apply #[[$stride1Dilation1]](%[[arg5]], %[[arg8]])
-//       CHECKLOOP:             %[[va:.*]] = load %[[arg1]][%[[arg6]], %[[arg7]], %[[arg8]]] : memref<?x?x?xf32>
 //       CHECKLOOP:             %[[vb:.*]] = load %[[arg0]][%[[aff]], %[[aff2]], %[[aff3]]] : memref<?x?x?xf32>
-//       CHECKLOOP:             %[[inc:.*]] = mulf %[[vb]], %[[va]] : f32
+
+//       CHECKLOOP:             %[[va:.*]] = load %[[arg1]][%[[arg6]], %[[arg7]], %[[arg8]]] : memref<?x?x?xf32>
 //       CHECKLOOP:             %[[vc:.*]] = load %[[arg2]][%[[arg3]], %[[arg4]], %[[arg5]]] : memref<?x?x?xf32>
+
+//       CHECKLOOP:             %[[inc:.*]] = mulf %[[vb]], %[[va]] : f32
 //       CHECKLOOP:             %[[res:.*]] = addf %[[vc]], %[[inc]] : f32
 //       CHECKLOOP:             store %[[res]], %[[arg2]][%[[arg3]], %[[arg4]], %[[arg5]]] : memref<?x?x?xf32>
 
@@ -1426,16 +1427,13 @@ func @conv3d_no_symbols(%in : memref<?x?x?xf32>, %filter : memref<?x?x?xf32>, %o
 //       CHECKPARALLEL: %[[dim3:.*]] = dim %[[arg2]], %[[c0]] : memref<?x?x?xf32>
 //       CHECKPARALLEL: %[[dim4:.*]] = dim %[[arg2]], %[[c1]] : memref<?x?x?xf32>
 //       CHECKPARALLEL: %[[dim5:.*]] = dim %[[arg2]], %[[c2]] : memref<?x?x?xf32>
-//       CHECKPARALLEL: scf.parallel (%[[arg3:.*]], %[[arg4:.*]], %[[arg5:.*]]) = (%[[c0]], %[[c0]], %[[c0]]) to (%[[dim3]], %[[dim4]], %[[dim5]]) step (%[[c1]], %[[c1]], %[[c1]]) {
-//       CHECKPARALLEL:   scf.for %[[arg6:.*]] = %[[c0]] to %[[dim0]] step %[[c1]] {
-//       CHECKPARALLEL:     scf.for %[[arg7:.*]] = %[[c0]] to %[[dim1]] step %[[c1]] {
-//       CHECKPARALLEL:       scf.for %[[arg8:.*]] = %[[c0]] to %[[dim2]] step %[[c1]] {
-//       CHECKPARALLEL:         %[[aff:.*]] = affine.apply #[[$stride1Dilation1]](%[[arg3]], %[[arg6]])
-//       CHECKPARALLEL:         %[[aff2:.*]] = affine.apply #[[$stride1Dilation1]](%[[arg4]], %[[arg7]])
-//       CHECKPARALLEL:         %[[aff3:.*]] = affine.apply #[[$stride1Dilation1]](%[[arg5]], %[[arg8]])
-//       CHECKPARALLEL:         %[[va:.*]] = load %[[arg1]][%[[arg6]], %[[arg7]], %[[arg8]]] : memref<?x?x?xf32>
-//       CHECKPARALLEL:         %[[vb:.*]] = load %[[arg0]][%[[aff]], %[[aff2]], %[[aff3]]] : memref<?x?x?xf32>
-//       CHECKPARALLEL:         %[[inc:.*]] = mulf %[[vb]], %[[va]] : f32
-//       CHECKPARALLEL:         %[[vc:.*]] = load %[[arg2]][%[[arg3]], %[[arg4]], %[[arg5]]] : memref<?x?x?xf32>
-//       CHECKPARALLEL:         %[[res:.*]] = addf %[[vc]], %[[inc]] : f32
-//       CHECKPARALLEL:         store %[[res]], %[[arg2]][%[[arg3]], %[[arg4]], %[[arg5]]] : memref<?x?x?xf32>
+//       CHECKPARALLEL: scf.parallel (%[[arg3:.*]], %[[arg4:.*]], %[[arg5:.*]], %[[arg6:.*]], %[[arg7:.*]], %[[arg8:.*]]) = (%[[c0]], %[[c0]], %[[c0]], %[[c0]], %[[c0]], %[[c0]]) to (%[[dim3]], %[[dim4]], %[[dim5]], %[[dim0]], %[[dim1]], %[[dim2]]) step (%[[c1]], %[[c1]], %[[c1]], %[[c1]], %[[c1]], %[[c1]]) {
+//       CHECKPARALLEL:   %[[aff:.*]] = affine.apply #[[$stride1Dilation1]](%[[arg3]], %[[arg6]])
+//       CHECKPARALLEL:   %[[aff2:.*]] = affine.apply #[[$stride1Dilation1]](%[[arg4]], %[[arg7]])
+//       CHECKPARALLEL:   %[[aff3:.*]] = affine.apply #[[$stride1Dilation1]](%[[arg5]], %[[arg8]])
+//       CHECKPARALLEL:   %[[vb:.*]] = load %[[arg0]][%[[aff]], %[[aff2]], %[[aff3]]] : memref<?x?x?xf32>
+//       CHECKPARALLEL:   %[[va:.*]] = load %[[arg1]][%[[arg6]], %[[arg7]], %[[arg8]]] : memref<?x?x?xf32>
+//       CHECKPARALLEL:   %[[vc:.*]] = load %[[arg2]][%[[arg3]], %[[arg4]], %[[arg5]]] : memref<?x?x?xf32>
+//       CHECKPARALLEL:   %[[inc:.*]] = mulf %[[vb]], %[[va]] : f32
+//       CHECKPARALLEL:   %[[res:.*]] = addf %[[vc]], %[[inc]] : f32
+//       CHECKPARALLEL:   store %[[res]], %[[arg2]][%[[arg3]], %[[arg4]], %[[arg5]]] : memref<?x?x?xf32>

From acb3b8dce1cd7ed25a137c718678449dd2d77f79 Mon Sep 17 00:00:00 2001
From: Nemanja Ivanovic <nemanja.i.ibm@gmail.com>
Date: Fri, 31 Jul 2020 06:19:47 -0500
Subject: [PATCH 0726/1035] [Sanitizers] Fix lint failure with Python 3.6

There are some files in compiler-rt that use UTF-8 characters in some of the
comments. This causes lint failures with some versions of Python. This patch
just makes the encoding explicit in the call to open.
---
 compiler-rt/lib/sanitizer_common/scripts/litlint.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/compiler-rt/lib/sanitizer_common/scripts/litlint.py b/compiler-rt/lib/sanitizer_common/scripts/litlint.py
index 81b89c2144381..c443efb915a9c 100755
--- a/compiler-rt/lib/sanitizer_common/scripts/litlint.py
+++ b/compiler-rt/lib/sanitizer_common/scripts/litlint.py
@@ -45,7 +45,7 @@ def LintFile(p):
     The number of errors detected.
   """
   errs = 0
-  with open(p, 'r') as f:
+  with open(p, 'r', encoding='utf-8') as f:
     for i, s in enumerate(f.readlines(), start=1):
       msg, col = LintLine(s)
       if msg != None:

From 2978b10aa164f692d48041327e27b2811649a233 Mon Sep 17 00:00:00 2001
From: Kirill Bobyrev <kbobyrev@google.com>
Date: Fri, 31 Jul 2020 14:02:19 +0200
Subject: [PATCH 0727/1035] [clangd] Fix remote index build on macOS

macOS builds suddenly started failing:

https://github.com/kirillbobyrev/indexing-tools/runs/925090879

This patch makes use of imported libraries and fixes builds for macOS.

Landing this without a review since the patch is quite straightforward
and I've been testing it on my local macOS machine for a while.

Differential Revision: https://reviews.llvm.org/D84928
---
 llvm/cmake/modules/FindGRPC.cmake | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/llvm/cmake/modules/FindGRPC.cmake b/llvm/cmake/modules/FindGRPC.cmake
index 8a0ca593b2f4a..70d67ceb791d3 100644
--- a/llvm/cmake/modules/FindGRPC.cmake
+++ b/llvm/cmake/modules/FindGRPC.cmake
@@ -45,11 +45,25 @@ else()
       # system path.
       if (GRPC_HOMEBREW_RETURN_CODE EQUAL "0")
         include_directories(${GRPC_HOMEBREW_PATH}/include)
-        link_directories(${GRPC_HOMEBREW_PATH}/lib)
+        find_library(GRPC_LIBRARY
+                     grpc++
+                     PATHS ${GRPC_HOMEBREW_PATH}/lib
+                     NO_DEFAULT_PATH
+                     REQUIRED)
+        add_library(grpc++ UNKNOWN IMPORTED GLOBAL)
+        set_target_properties(grpc++ PROPERTIES
+                              IMPORTED_LOCATION ${GRPC_LIBRARY})
       endif()
       if (PROTOBUF_HOMEBREW_RETURN_CODE EQUAL "0")
         include_directories(${PROTOBUF_HOMEBREW_PATH}/include)
-        link_directories(${PROTOBUF_HOMEBREW_PATH}/lib)
+        find_library(PROTOBUF_LIBRARY
+                     protobuf
+                     PATHS ${PROTOBUF_HOMEBREW_PATH}/lib
+                     NO_DEFAULT_PATH
+                     REQUIRED)
+        add_library(protobuf UNKNOWN IMPORTED GLOBAL)
+        set_target_properties(protobuf PROPERTIES
+                              IMPORTED_LOCATION ${PROTOBUF_LIBRARY})
       endif()
     endif()
   endif()

From cbf5bf513b93cc5bfa360f4be8a57e50988e22f1 Mon Sep 17 00:00:00 2001
From: Xing GUO <higuoxing@gmail.com>
Date: Fri, 31 Jul 2020 20:02:20 +0800
Subject: [PATCH 0728/1035] [DWARFYAML] Add emitDebug[GNU]Pub[names/types]
 functions. NFC.

In this patch, emitDebugPubnames(), emitDebugPubtypes(),
emitDebugGNUPubnames(), emitDebugGNUPubtypes() are added.

Reviewed By: jhenderson

Differential Revision: https://reviews.llvm.org/D85003
---
 llvm/include/llvm/ObjectYAML/DWARFEmitter.h |  6 +++--
 llvm/lib/ObjectYAML/DWARFEmitter.cpp        | 27 ++++++++++++++++++---
 llvm/lib/ObjectYAML/ELFEmitter.cpp          | 10 +++-----
 llvm/lib/ObjectYAML/MachOEmitter.cpp        |  6 ++---
 4 files changed, 34 insertions(+), 15 deletions(-)

diff --git a/llvm/include/llvm/ObjectYAML/DWARFEmitter.h b/llvm/include/llvm/ObjectYAML/DWARFEmitter.h
index 5837c69ed59f9..5c29b0e757246 100644
--- a/llvm/include/llvm/ObjectYAML/DWARFEmitter.h
+++ b/llvm/include/llvm/ObjectYAML/DWARFEmitter.h
@@ -33,8 +33,10 @@ Error emitDebugStr(raw_ostream &OS, const Data &DI);
 
 Error emitDebugAranges(raw_ostream &OS, const Data &DI);
 Error emitDebugRanges(raw_ostream &OS, const Data &DI);
-Error emitPubSection(raw_ostream &OS, const PubSection &Sect,
-                     bool IsLittleEndian, bool IsGNUPubSec = false);
+Error emitDebugPubnames(raw_ostream &OS, const Data &DI);
+Error emitDebugPubtypes(raw_ostream &OS, const Data &DI);
+Error emitDebugGNUPubnames(raw_ostream &OS, const Data &DI);
+Error emitDebugGNUPubtypes(raw_ostream &OS, const Data &DI);
 Error emitDebugInfo(raw_ostream &OS, const Data &DI);
 Error emitDebugLine(raw_ostream &OS, const Data &DI);
 Error emitDebugAddr(raw_ostream &OS, const Data &DI);
diff --git a/llvm/lib/ObjectYAML/DWARFEmitter.cpp b/llvm/lib/ObjectYAML/DWARFEmitter.cpp
index be9581fa38083..9655b05317abf 100644
--- a/llvm/lib/ObjectYAML/DWARFEmitter.cpp
+++ b/llvm/lib/ObjectYAML/DWARFEmitter.cpp
@@ -209,9 +209,8 @@ Error DWARFYAML::emitDebugRanges(raw_ostream &OS, const DWARFYAML::Data &DI) {
   return Error::success();
 }
 
-Error DWARFYAML::emitPubSection(raw_ostream &OS,
-                                const DWARFYAML::PubSection &Sect,
-                                bool IsLittleEndian, bool IsGNUPubSec) {
+static Error emitPubSection(raw_ostream &OS, const DWARFYAML::PubSection &Sect,
+                            bool IsLittleEndian, bool IsGNUPubSec = false) {
   writeInitialLength(Sect.Length, OS, IsLittleEndian);
   writeInteger((uint16_t)Sect.Version, OS, IsLittleEndian);
   writeInteger((uint32_t)Sect.UnitOffset, OS, IsLittleEndian);
@@ -227,6 +226,28 @@ Error DWARFYAML::emitPubSection(raw_ostream &OS,
   return Error::success();
 }
 
+Error DWARFYAML::emitDebugPubnames(raw_ostream &OS, const Data &DI) {
+  assert(DI.PubNames && "unexpected emitDebugPubnames() call");
+  return emitPubSection(OS, *DI.PubNames, DI.IsLittleEndian);
+}
+
+Error DWARFYAML::emitDebugPubtypes(raw_ostream &OS, const Data &DI) {
+  assert(DI.PubTypes && "unexpected emitDebugPubtypes() call");
+  return emitPubSection(OS, *DI.PubTypes, DI.IsLittleEndian);
+}
+
+Error DWARFYAML::emitDebugGNUPubnames(raw_ostream &OS, const Data &DI) {
+  assert(DI.GNUPubNames && "unexpected emitDebugGNUPubnames() call");
+  return emitPubSection(OS, *DI.GNUPubNames, DI.IsLittleEndian,
+                        /*IsGNUStyle=*/true);
+}
+
+Error DWARFYAML::emitDebugGNUPubtypes(raw_ostream &OS, const Data &DI) {
+  assert(DI.GNUPubTypes && "unexpected emitDebugGNUPubtypes() call");
+  return emitPubSection(OS, *DI.GNUPubTypes, DI.IsLittleEndian,
+                        /*IsGNUStyle=*/true);
+}
+
 static Expected<uint64_t> writeDIE(ArrayRef<DWARFYAML::Abbrev> AbbrevDecls,
                                    const DWARFYAML::Unit &Unit,
                                    const DWARFYAML::Entry &Entry,
diff --git a/llvm/lib/ObjectYAML/ELFEmitter.cpp b/llvm/lib/ObjectYAML/ELFEmitter.cpp
index bc27c03cb6877..9fefd8c4c9965 100644
--- a/llvm/lib/ObjectYAML/ELFEmitter.cpp
+++ b/llvm/lib/ObjectYAML/ELFEmitter.cpp
@@ -967,15 +967,13 @@ Expected<uint64_t> emitDWARF(typename ELFT::Shdr &SHeader, StringRef Name,
   else if (Name == ".debug_info")
     Err = DWARFYAML::emitDebugInfo(*OS, DWARF);
   else if (Name == ".debug_pubnames")
-    Err = DWARFYAML::emitPubSection(*OS, *DWARF.PubNames, DWARF.IsLittleEndian);
+    Err = DWARFYAML::emitDebugPubnames(*OS, DWARF);
   else if (Name == ".debug_pubtypes")
-    Err = DWARFYAML::emitPubSection(*OS, *DWARF.PubTypes, DWARF.IsLittleEndian);
+    Err = DWARFYAML::emitDebugPubtypes(*OS, DWARF);
   else if (Name == ".debug_gnu_pubnames")
-    Err = DWARFYAML::emitPubSection(*OS, *DWARF.GNUPubNames,
-                                    DWARF.IsLittleEndian, /*IsGNUStyle=*/true);
+    Err = DWARFYAML::emitDebugGNUPubnames(*OS, DWARF);
   else if (Name == ".debug_gnu_pubtypes")
-    Err = DWARFYAML::emitPubSection(*OS, *DWARF.GNUPubTypes,
-                                    DWARF.IsLittleEndian, /*IsGNUStyle=*/true);
+    Err = DWARFYAML::emitDebugGNUPubtypes(*OS, DWARF);
   else if (Name == ".debug_str_offsets")
     Err = DWARFYAML::emitDebugStrOffsets(*OS, DWARF);
   else if (Name == ".debug_rnglists")
diff --git a/llvm/lib/ObjectYAML/MachOEmitter.cpp b/llvm/lib/ObjectYAML/MachOEmitter.cpp
index 680264484704b..619572a7532c4 100644
--- a/llvm/lib/ObjectYAML/MachOEmitter.cpp
+++ b/llvm/lib/ObjectYAML/MachOEmitter.cpp
@@ -299,12 +299,10 @@ Error MachOWriter::writeSectionData(raw_ostream &OS) {
             Err = DWARFYAML::emitDebugRanges(OS, Obj.DWARF);
           else if (0 == strncmp(&Sec.sectname[0], "__debug_pubnames", 16)) {
             if (Obj.DWARF.PubNames)
-              Err = DWARFYAML::emitPubSection(OS, *Obj.DWARF.PubNames,
-                                              Obj.IsLittleEndian);
+              Err = DWARFYAML::emitDebugPubnames(OS, Obj.DWARF);
           } else if (0 == strncmp(&Sec.sectname[0], "__debug_pubtypes", 16)) {
             if (Obj.DWARF.PubTypes)
-              Err = DWARFYAML::emitPubSection(OS, *Obj.DWARF.PubTypes,
-                                              Obj.IsLittleEndian);
+              Err = DWARFYAML::emitDebugPubtypes(OS, Obj.DWARF);
           } else if (0 == strncmp(&Sec.sectname[0], "__debug_info", 16))
             Err = DWARFYAML::emitDebugInfo(OS, Obj.DWARF);
           else if (0 == strncmp(&Sec.sectname[0], "__debug_line", 16))

From 760e4f22027ca1d609d21d220a28661ab091eb29 Mon Sep 17 00:00:00 2001
From: Xing GUO <higuoxing@gmail.com>
Date: Fri, 31 Jul 2020 20:06:30 +0800
Subject: [PATCH 0729/1035] [DWARFYAML] Add helper function
 getDWARFEmitterByName(). NFC.

In this patch, we add a helper function getDWARFEmitterByName(). This
function returns the proper DWARF section emitting method by the name.

Reviewed By: jhenderson

Differential Revision: https://reviews.llvm.org/D84952
---
 llvm/include/llvm/ObjectYAML/DWARFEmitter.h |  2 +
 llvm/lib/ObjectYAML/DWARFEmitter.cpp        | 56 +++++++++++++--------
 llvm/lib/ObjectYAML/ELFEmitter.cpp          | 34 +------------
 3 files changed, 40 insertions(+), 52 deletions(-)

diff --git a/llvm/include/llvm/ObjectYAML/DWARFEmitter.h b/llvm/include/llvm/ObjectYAML/DWARFEmitter.h
index 5c29b0e757246..89d01cecb9b7b 100644
--- a/llvm/include/llvm/ObjectYAML/DWARFEmitter.h
+++ b/llvm/include/llvm/ObjectYAML/DWARFEmitter.h
@@ -43,6 +43,8 @@ Error emitDebugAddr(raw_ostream &OS, const Data &DI);
 Error emitDebugStrOffsets(raw_ostream &OS, const Data &DI);
 Error emitDebugRnglists(raw_ostream &OS, const Data &DI);
 
+std::function<Error(raw_ostream &, const Data &)>
+getDWARFEmitterByName(StringRef SecName);
 Expected<StringMap<std::unique_ptr<MemoryBuffer>>>
 emitDebugSections(StringRef YAMLString,
                   bool IsLittleEndian = sys::IsLittleEndianHost);
diff --git a/llvm/lib/ObjectYAML/DWARFEmitter.cpp b/llvm/lib/ObjectYAML/DWARFEmitter.cpp
index 9655b05317abf..dbf417780cfc7 100644
--- a/llvm/lib/ObjectYAML/DWARFEmitter.cpp
+++ b/llvm/lib/ObjectYAML/DWARFEmitter.cpp
@@ -15,6 +15,7 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/StringSwitch.h"
 #include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/ObjectYAML/DWARFYAML.h"
 #include "llvm/Support/Errc.h"
@@ -762,14 +763,40 @@ Error DWARFYAML::emitDebugRnglists(raw_ostream &OS, const Data &DI) {
       OS, *DI.DebugRnglists, DI.IsLittleEndian, DI.Is64BitAddrSize);
 }
 
-using EmitFuncType = Error (*)(raw_ostream &, const DWARFYAML::Data &);
+std::function<Error(raw_ostream &, const DWARFYAML::Data &)>
+DWARFYAML::getDWARFEmitterByName(StringRef SecName) {
+  auto EmitFunc =
+      StringSwitch<
+          std::function<Error(raw_ostream &, const DWARFYAML::Data &)>>(SecName)
+          .Case("debug_abbrev", DWARFYAML::emitDebugAbbrev)
+          .Case("debug_addr", DWARFYAML::emitDebugAddr)
+          .Case("debug_aranges", DWARFYAML::emitDebugAranges)
+          .Case("debug_gnu_pubnames", DWARFYAML::emitDebugGNUPubnames)
+          .Case("debug_gnu_pubtypes", DWARFYAML::emitDebugGNUPubtypes)
+          .Case("debug_info", DWARFYAML::emitDebugInfo)
+          .Case("debug_line", DWARFYAML::emitDebugLine)
+          .Case("debug_pubnames", DWARFYAML::emitDebugPubnames)
+          .Case("debug_pubtypes", DWARFYAML::emitDebugPubtypes)
+          .Case("debug_ranges", DWARFYAML::emitDebugRanges)
+          .Case("debug_rnglists", DWARFYAML::emitDebugRnglists)
+          .Case("debug_str", DWARFYAML::emitDebugStr)
+          .Case("debug_str_offsets", DWARFYAML::emitDebugStrOffsets)
+          .Default([&](raw_ostream &, const DWARFYAML::Data &) {
+            return createStringError(errc::not_supported,
+                                     SecName + " is not supported");
+          });
+
+  return EmitFunc;
+}
 
 static Error
-emitDebugSectionImpl(const DWARFYAML::Data &DI, EmitFuncType EmitFunc,
-                     StringRef Sec,
+emitDebugSectionImpl(const DWARFYAML::Data &DI, StringRef Sec,
                      StringMap<std::unique_ptr<MemoryBuffer>> &OutputBuffers) {
   std::string Data;
   raw_string_ostream DebugInfoStream(Data);
+
+  auto EmitFunc = DWARFYAML::getDWARFEmitterByName(Sec);
+
   if (Error Err = EmitFunc(DebugInfoStream, DI))
     return Err;
   DebugInfoStream.flush();
@@ -796,23 +823,12 @@ DWARFYAML::emitDebugSections(StringRef YAMLString, bool IsLittleEndian) {
     return createStringError(YIn.error(), GeneratedDiag.getMessage());
 
   StringMap<std::unique_ptr<MemoryBuffer>> DebugSections;
-  Error Err = emitDebugSectionImpl(DI, &DWARFYAML::emitDebugInfo, "debug_info",
-                                   DebugSections);
-  Err = joinErrors(std::move(Err),
-                   emitDebugSectionImpl(DI, &DWARFYAML::emitDebugLine,
-                                        "debug_line", DebugSections));
-  Err = joinErrors(std::move(Err),
-                   emitDebugSectionImpl(DI, &DWARFYAML::emitDebugStr,
-                                        "debug_str", DebugSections));
-  Err = joinErrors(std::move(Err),
-                   emitDebugSectionImpl(DI, &DWARFYAML::emitDebugAbbrev,
-                                        "debug_abbrev", DebugSections));
-  Err = joinErrors(std::move(Err),
-                   emitDebugSectionImpl(DI, &DWARFYAML::emitDebugAranges,
-                                        "debug_aranges", DebugSections));
-  Err = joinErrors(std::move(Err),
-                   emitDebugSectionImpl(DI, &DWARFYAML::emitDebugRanges,
-                                        "debug_ranges", DebugSections));
+  Error Err = Error::success();
+  cantFail(std::move(Err));
+
+  for (StringRef SecName : DI.getNonEmptySectionNames())
+    Err = joinErrors(std::move(Err),
+                     emitDebugSectionImpl(DI, SecName, DebugSections));
 
   if (Err)
     return std::move(Err);
diff --git a/llvm/lib/ObjectYAML/ELFEmitter.cpp b/llvm/lib/ObjectYAML/ELFEmitter.cpp
index 9fefd8c4c9965..8cb7af91c56b0 100644
--- a/llvm/lib/ObjectYAML/ELFEmitter.cpp
+++ b/llvm/lib/ObjectYAML/ELFEmitter.cpp
@@ -949,39 +949,9 @@ Expected<uint64_t> emitDWARF(typename ELFT::Shdr &SHeader, StringRef Name,
     return 0;
 
   uint64_t BeginOffset = CBA.tell();
-  Error Err = Error::success();
-  cantFail(std::move(Err));
-
-  if (Name == ".debug_str")
-    Err = DWARFYAML::emitDebugStr(*OS, DWARF);
-  else if (Name == ".debug_aranges")
-    Err = DWARFYAML::emitDebugAranges(*OS, DWARF);
-  else if (Name == ".debug_ranges")
-    Err = DWARFYAML::emitDebugRanges(*OS, DWARF);
-  else if (Name == ".debug_line")
-    Err = DWARFYAML::emitDebugLine(*OS, DWARF);
-  else if (Name == ".debug_addr")
-    Err = DWARFYAML::emitDebugAddr(*OS, DWARF);
-  else if (Name == ".debug_abbrev")
-    Err = DWARFYAML::emitDebugAbbrev(*OS, DWARF);
-  else if (Name == ".debug_info")
-    Err = DWARFYAML::emitDebugInfo(*OS, DWARF);
-  else if (Name == ".debug_pubnames")
-    Err = DWARFYAML::emitDebugPubnames(*OS, DWARF);
-  else if (Name == ".debug_pubtypes")
-    Err = DWARFYAML::emitDebugPubtypes(*OS, DWARF);
-  else if (Name == ".debug_gnu_pubnames")
-    Err = DWARFYAML::emitDebugGNUPubnames(*OS, DWARF);
-  else if (Name == ".debug_gnu_pubtypes")
-    Err = DWARFYAML::emitDebugGNUPubtypes(*OS, DWARF);
-  else if (Name == ".debug_str_offsets")
-    Err = DWARFYAML::emitDebugStrOffsets(*OS, DWARF);
-  else if (Name == ".debug_rnglists")
-    Err = DWARFYAML::emitDebugRnglists(*OS, DWARF);
-  else
-    llvm_unreachable("unexpected emitDWARF() call");
 
-  if (Err)
+  auto EmitFunc = DWARFYAML::getDWARFEmitterByName(Name.substr(1));
+  if (Error Err = EmitFunc(*OS, DWARF))
     return std::move(Err);
 
   return CBA.tell() - BeginOffset;

From 74b02d73e34278e081dcb4946d66b9562f0986fe Mon Sep 17 00:00:00 2001
From: Xing GUO <higuoxing@gmail.com>
Date: Fri, 31 Jul 2020 12:56:10 +0800
Subject: [PATCH 0730/1035] [DWARFYAML] Make the debug_aranges entry optional.

This patch makes the 'debug_aranges' entry optional. If the entry is
empty, yaml2obj will only emit the header for it.

Reviewed By: jhenderson

Differential Revision: https://reviews.llvm.org/D84921
---
 llvm/include/llvm/ObjectYAML/DWARFYAML.h      |  2 +-
 llvm/lib/ObjectYAML/DWARFEmitter.cpp          |  3 +-
 llvm/lib/ObjectYAML/DWARFYAML.cpp             |  7 +-
 llvm/lib/ObjectYAML/MachOEmitter.cpp          |  7 +-
 .../ObjectYAML/MachO/DWARF-debug_aranges.yaml | 67 ++++++++++++++++++-
 .../yaml2obj/ELF/DWARF/debug-aranges.yaml     | 19 ++++++
 llvm/tools/obj2yaml/dwarf2yaml.cpp            |  6 +-
 7 files changed, 99 insertions(+), 12 deletions(-)

diff --git a/llvm/include/llvm/ObjectYAML/DWARFYAML.h b/llvm/include/llvm/ObjectYAML/DWARFYAML.h
index 32382da6de504..5737ceccc0a40 100644
--- a/llvm/include/llvm/ObjectYAML/DWARFYAML.h
+++ b/llvm/include/llvm/ObjectYAML/DWARFYAML.h
@@ -211,7 +211,7 @@ struct Data {
   std::vector<Abbrev> AbbrevDecls;
   std::vector<StringRef> DebugStrings;
   Optional<std::vector<StringOffsetsTable>> DebugStrOffsets;
-  std::vector<ARange> ARanges;
+  Optional<std::vector<ARange>> DebugAranges;
   std::vector<Ranges> DebugRanges;
   std::vector<AddrTableEntry> DebugAddr;
   Optional<PubSection> PubNames;
diff --git a/llvm/lib/ObjectYAML/DWARFEmitter.cpp b/llvm/lib/ObjectYAML/DWARFEmitter.cpp
index dbf417780cfc7..ab3cd05a6495d 100644
--- a/llvm/lib/ObjectYAML/DWARFEmitter.cpp
+++ b/llvm/lib/ObjectYAML/DWARFEmitter.cpp
@@ -126,7 +126,8 @@ Error DWARFYAML::emitDebugAbbrev(raw_ostream &OS, const DWARFYAML::Data &DI) {
 }
 
 Error DWARFYAML::emitDebugAranges(raw_ostream &OS, const DWARFYAML::Data &DI) {
-  for (auto Range : DI.ARanges) {
+  assert(DI.DebugAranges && "unexpected emitDebugAranges() call");
+  for (auto Range : *DI.DebugAranges) {
     uint8_t AddrSize;
     if (Range.AddrSize)
       AddrSize = *Range.AddrSize;
diff --git a/llvm/lib/ObjectYAML/DWARFYAML.cpp b/llvm/lib/ObjectYAML/DWARFYAML.cpp
index 7dd289ff6973b..04dd185ad954e 100644
--- a/llvm/lib/ObjectYAML/DWARFYAML.cpp
+++ b/llvm/lib/ObjectYAML/DWARFYAML.cpp
@@ -17,7 +17,7 @@
 namespace llvm {
 
 bool DWARFYAML::Data::isEmpty() const {
-  return DebugStrings.empty() && AbbrevDecls.empty() && ARanges.empty() &&
+  return DebugStrings.empty() && AbbrevDecls.empty() && DebugAranges &&
          DebugRanges.empty() && !PubNames && !PubTypes && !GNUPubNames &&
          !GNUPubTypes && CompileUnits.empty() && DebugLines.empty();
 }
@@ -26,7 +26,7 @@ SetVector<StringRef> DWARFYAML::Data::getNonEmptySectionNames() const {
   SetVector<StringRef> SecNames;
   if (!DebugStrings.empty())
     SecNames.insert("debug_str");
-  if (!ARanges.empty())
+  if (DebugAranges)
     SecNames.insert("debug_aranges");
   if (!DebugRanges.empty())
     SecNames.insert("debug_ranges");
@@ -61,8 +61,7 @@ void MappingTraits<DWARFYAML::Data>::mapping(IO &IO, DWARFYAML::Data &DWARF) {
   IO.setContext(&DWARFCtx);
   IO.mapOptional("debug_str", DWARF.DebugStrings);
   IO.mapOptional("debug_abbrev", DWARF.AbbrevDecls);
-  if (!DWARF.ARanges.empty() || !IO.outputting())
-    IO.mapOptional("debug_aranges", DWARF.ARanges);
+  IO.mapOptional("debug_aranges", DWARF.DebugAranges);
   if (!DWARF.DebugRanges.empty() || !IO.outputting())
     IO.mapOptional("debug_ranges", DWARF.DebugRanges);
   IO.mapOptional("debug_pubnames", DWARF.PubNames);
diff --git a/llvm/lib/ObjectYAML/MachOEmitter.cpp b/llvm/lib/ObjectYAML/MachOEmitter.cpp
index 619572a7532c4..9b454c528a7e2 100644
--- a/llvm/lib/ObjectYAML/MachOEmitter.cpp
+++ b/llvm/lib/ObjectYAML/MachOEmitter.cpp
@@ -293,9 +293,10 @@ Error MachOWriter::writeSectionData(raw_ostream &OS) {
             Err = DWARFYAML::emitDebugStr(OS, Obj.DWARF);
           else if (0 == strncmp(&Sec.sectname[0], "__debug_abbrev", 16))
             Err = DWARFYAML::emitDebugAbbrev(OS, Obj.DWARF);
-          else if (0 == strncmp(&Sec.sectname[0], "__debug_aranges", 16))
-            Err = DWARFYAML::emitDebugAranges(OS, Obj.DWARF);
-          else if (0 == strncmp(&Sec.sectname[0], "__debug_ranges", 16))
+          else if (0 == strncmp(&Sec.sectname[0], "__debug_aranges", 16)) {
+            if (Obj.DWARF.DebugAranges)
+              Err = DWARFYAML::emitDebugAranges(OS, Obj.DWARF);
+          } else if (0 == strncmp(&Sec.sectname[0], "__debug_ranges", 16))
             Err = DWARFYAML::emitDebugRanges(OS, Obj.DWARF);
           else if (0 == strncmp(&Sec.sectname[0], "__debug_pubnames", 16)) {
             if (Obj.DWARF.PubNames)
diff --git a/llvm/test/ObjectYAML/MachO/DWARF-debug_aranges.yaml b/llvm/test/ObjectYAML/MachO/DWARF-debug_aranges.yaml
index ecdd88a68dd0c..1e9b880c3cd3d 100644
--- a/llvm/test/ObjectYAML/MachO/DWARF-debug_aranges.yaml
+++ b/llvm/test/ObjectYAML/MachO/DWARF-debug_aranges.yaml
@@ -1,4 +1,7 @@
-# RUN: yaml2obj %s | obj2yaml | FileCheck %s
+## a) Test that yaml2macho is able to emit the .debug_aranges section and obj2yaml
+## is able to convert it back.
+
+# RUN: yaml2obj --docnum=1 %s | obj2yaml | FileCheck %s
 
 --- !mach-o
 FileHeader:      
@@ -331,3 +334,65 @@ DWARF:
 # CHECK-NEXT:       Descriptors:
 # CHECK-NEXT:         - Address: 0x0000000100000F50
 # CHECK-NEXT:           Length:  52
+
+## b) Test that if the "debug_aranges" entry is empty, yaml2macho will only emit the
+## section header.
+
+# RUN: yaml2obj --docnum=2 %s -o %t2.o
+# RUN: llvm-readobj --sections --section-data %t2.o | FileCheck %s --check-prefix=EMPTY-SECTION
+
+#      EMPTY-SECTION: Index: 0
+# EMPTY-SECTION-NEXT: Name: __debug_aranges (5F 5F 64 65 62 75 67 5F 61 72 61 6E 67 65 73 00)
+# EMPTY-SECTION-NEXT: Segment: __DWARF (5F 5F 44 57 41 52 46 00 00 00 00 00 00 00 00 00)
+# EMPTY-SECTION-NEXT: Address: 0x0
+# EMPTY-SECTION-NEXT: Size: 0xC
+# EMPTY-SECTION-NEXT: Offset: 528
+# EMPTY-SECTION-NEXT: Alignment: 0
+# EMPTY-SECTION-NEXT: RelocationOffset: 0x0
+# EMPTY-SECTION-NEXT: RelocationCount: 0
+# EMPTY-SECTION-NEXT: Type: Regular (0x0)
+# EMPTY-SECTION-NEXT: Attributes [ (0x0)
+# EMPTY-SECTION-NEXT: ]
+# EMPTY-SECTION-NEXT: Reserved1: 0x0
+# EMPTY-SECTION-NEXT: Reserved2: 0x0
+# EMPTY-SECTION-NEXT: Reserved3: 0x0
+# EMPTY-SECTION-NEXT: SectionData (
+# EMPTY-SECTION-NEXT: )
+
+--- !mach-o
+FileHeader:
+  magic:      0xFEEDFACF
+  cputype:    0x01000007
+  cpusubtype: 0x00000003
+  filetype:   0x0000000A
+  ncmds:      1
+  sizeofcmds: 232
+  flags:      0x00000000
+  reserved:   0x00000000
+LoadCommands:
+  - cmd:      LC_SEGMENT_64
+    cmdsize:  152
+    segname:  __DWARF
+    vmaddr:   0x00
+    vmsize:   0x00
+    fileoff:  0x00
+    filesize: 0x00
+    maxprot:  0
+    initprot: 0
+    nsects:   1
+    flags:    0
+    Sections:
+      - sectname:  __debug_aranges
+        segname:   __DWARF
+        addr:      0x00
+        size:      12
+        offset:    528
+        align:     0
+        reloff:    0x00000000
+        nreloc:    0
+        flags:     0x00000000
+        reserved1: 0x00000000
+        reserved2: 0x00000000
+        reserved3: 0x00000000
+DWARF:
+  debug_aranges: []
diff --git a/llvm/test/tools/yaml2obj/ELF/DWARF/debug-aranges.yaml b/llvm/test/tools/yaml2obj/ELF/DWARF/debug-aranges.yaml
index 9fa86449053d0..4fa924c33ad13 100644
--- a/llvm/test/tools/yaml2obj/ELF/DWARF/debug-aranges.yaml
+++ b/llvm/test/tools/yaml2obj/ELF/DWARF/debug-aranges.yaml
@@ -595,3 +595,22 @@ DWARF:
       Descriptors:
         - Address: 0x1234
           Length:  0x4321
+
+## l) Test that the .debug_aranges section header is emitted if the "debug_aranges"
+## entry is empty.
+
+# RUN: yaml2obj --docnum=12 %s -o %t12.o
+# RUN: llvm-readobj --sections --section-data %t12.o | \
+# RUN:   FileCheck -DSIZE=0 -DADDRALIGN=1 %s --check-prefixes=DWARF-HEADER,EMPTY-CONTENT
+
+# EMPTY-CONTENT-NEXT: SectionData (
+# EMPTY-CONTENT-NEXT: )
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+DWARF:
+  debug_aranges: []
diff --git a/llvm/tools/obj2yaml/dwarf2yaml.cpp b/llvm/tools/obj2yaml/dwarf2yaml.cpp
index 66264a46068a6..4c1742cf922be 100644
--- a/llvm/tools/obj2yaml/dwarf2yaml.cpp
+++ b/llvm/tools/obj2yaml/dwarf2yaml.cpp
@@ -64,7 +64,7 @@ Error dumpDebugARanges(DWARFContext &DCtx, DWARFYAML::Data &Y) {
                                  DCtx.isLittleEndian(), 0);
   uint64_t Offset = 0;
   DWARFDebugArangeSet Set;
-
+  std::vector<DWARFYAML::ARange> DebugAranges;
   while (ArangesData.isValidOffset(Offset)) {
     if (Error E = Set.extract(ArangesData, &Offset))
       return E;
@@ -81,8 +81,10 @@ Error dumpDebugARanges(DWARFContext &DCtx, DWARFYAML::Data &Y) {
       Desc.Length = Descriptor.Length;
       Range.Descriptors.push_back(Desc);
     }
-    Y.ARanges.push_back(Range);
+    DebugAranges.push_back(Range);
   }
+
+  Y.DebugAranges = DebugAranges;
   return ErrorSuccess();
 }
 

From 0d25d3b7e3e3acb86d93acb2291c1d26e056746b Mon Sep 17 00:00:00 2001
From: Bjorn Pettersson <bjorn.a.pettersson@ericsson.com>
Date: Fri, 31 Jul 2020 14:28:49 +0200
Subject: [PATCH 0731/1035] [clang-tidy] Fix build problem after commit
 45a720a864320bbbeb596a

When building with LLVM8.0 on RHEL7.8 I got failures like this
after commit 45a720a864320bbbe:

/app/llvm/8.0/bin/../lib/gcc/x86_64-unknown-linux-gnu/
5.4.0/../../../../include/c++/5.4.0/ext/new_allocator.h:120:23:
error: no matching constructor for initialization of
'std::pair<std::__cxx11::basic_string<char>,
std::__cxx11::basic_string<char> >'
{ ::new((void *)__p) _Up(std::forward<_Args>(__args)...); }

...

../../clang-tools-extra/clang-tidy/ClangTidyOptions.cpp:73:15:
note: in instantiation of function template specialization
'std::vector<std::pair<std::__cxx11::basic_string<char>,
std::__cxx11::basic_string<char> >,
std::allocator<std::pair<std::__cxx11::basic_string<char>,
std::__cxx11::basic_string<char> > > >::emplace_back<llvm::StringRef,
const std::__cxx11::basic_string<char> &>' requested here
Options.emplace_back(KeyValue.getKey(), KeyValue.getValue().Value);

This is an attempt to avoid such build problems.
---
 clang-tools-extra/clang-tidy/ClangTidyOptions.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang-tools-extra/clang-tidy/ClangTidyOptions.cpp b/clang-tools-extra/clang-tidy/ClangTidyOptions.cpp
index 19ba47f005dc8..6b28cb2bdd13d 100644
--- a/clang-tools-extra/clang-tidy/ClangTidyOptions.cpp
+++ b/clang-tools-extra/clang-tidy/ClangTidyOptions.cpp
@@ -70,7 +70,7 @@ struct NOptionMap {
   NOptionMap(IO &, const ClangTidyOptions::OptionMap &OptionMap) {
     Options.reserve(OptionMap.size());
     for (const auto &KeyValue : OptionMap)
-      Options.emplace_back(KeyValue.getKey(), KeyValue.getValue().Value);
+      Options.emplace_back(std::string(KeyValue.getKey()), KeyValue.getValue().Value);
   }
   ClangTidyOptions::OptionMap denormalize(IO &) {
     ClangTidyOptions::OptionMap Map;

From 638f0cf565f2121151c32d7eb52a1de0e333d5f6 Mon Sep 17 00:00:00 2001
From: Haojian Wu <hokein.wu@gmail.com>
Date: Fri, 31 Jul 2020 14:32:18 +0200
Subject: [PATCH 0732/1035] [clangd] Be more explicit on testing the optional
 DefLoc in LocatedSymbol.

And also fix a bug where we may return a meaningless location.

Differential Revision: https://reviews.llvm.org/D84919
---
 clang-tools-extra/clangd/XRefs.cpp            | 13 ++--
 .../clangd/unittests/XRefsTests.cpp           | 74 ++++++++++++-------
 2 files changed, 51 insertions(+), 36 deletions(-)

diff --git a/clang-tools-extra/clangd/XRefs.cpp b/clang-tools-extra/clangd/XRefs.cpp
index cf747b607f4a1..1fc89f3e08472 100644
--- a/clang-tools-extra/clangd/XRefs.cpp
+++ b/clang-tools-extra/clangd/XRefs.cpp
@@ -405,15 +405,17 @@ locateSymbolTextually(const SpelledWord &Word, ParsedAST &AST,
       log("locateSymbolNamedTextuallyAt: {0}", MaybeDeclLoc.takeError());
       return;
     }
-    Location DeclLoc = *MaybeDeclLoc;
-    Location DefLoc;
+    LocatedSymbol Located;
+    Located.PreferredDeclaration = *MaybeDeclLoc;
+    Located.Name = (Sym.Name + Sym.TemplateSpecializationArgs).str();
     if (Sym.Definition) {
       auto MaybeDefLoc = indexToLSPLocation(Sym.Definition, MainFilePath);
       if (!MaybeDefLoc) {
         log("locateSymbolNamedTextuallyAt: {0}", MaybeDefLoc.takeError());
         return;
       }
-      DefLoc = *MaybeDefLoc;
+      Located.PreferredDeclaration = *MaybeDefLoc;
+      Located.Definition = *MaybeDefLoc;
     }
 
     if (ScoredResults.size() >= 3) {
@@ -424,11 +426,6 @@ locateSymbolTextually(const SpelledWord &Word, ParsedAST &AST,
       return;
     }
 
-    LocatedSymbol Located;
-    Located.Name = (Sym.Name + Sym.TemplateSpecializationArgs).str();
-    Located.PreferredDeclaration = bool(Sym.Definition) ? DefLoc : DeclLoc;
-    Located.Definition = DefLoc;
-
     SymbolQualitySignals Quality;
     Quality.merge(Sym);
     SymbolRelevanceSignals Relevance;
diff --git a/clang-tools-extra/clangd/unittests/XRefsTests.cpp b/clang-tools-extra/clangd/unittests/XRefsTests.cpp
index 0a8f85ed53176..c9c115fd19d83 100644
--- a/clang-tools-extra/clangd/unittests/XRefsTests.cpp
+++ b/clang-tools-extra/clangd/unittests/XRefsTests.cpp
@@ -41,6 +41,7 @@ using ::testing::ElementsAre;
 using ::testing::Eq;
 using ::testing::IsEmpty;
 using ::testing::Matcher;
+using ::testing::UnorderedElementsAre;
 using ::testing::UnorderedElementsAreArray;
 
 MATCHER_P2(FileRange, File, Range, "") {
@@ -264,19 +265,23 @@ MATCHER_P3(Sym, Name, Decl, DefOrNone, "") {
                      << llvm::to_string(arg.PreferredDeclaration);
     return false;
   }
+  if (!Def && !arg.Definition)
+    return true;
   if (Def && !arg.Definition) {
     *result_listener << "Has no definition";
     return false;
   }
-  if (Def && arg.Definition->range != *Def) {
+  if (!Def && arg.Definition) {
+    *result_listener << "Definition is " << llvm::to_string(arg.Definition);
+    return false;
+  }
+  if (arg.Definition->range != *Def) {
     *result_listener << "Definition is " << llvm::to_string(arg.Definition);
     return false;
   }
   return true;
 }
-::testing::Matcher<LocatedSymbol> Sym(std::string Name, Range Decl) {
-  return Sym(Name, Decl, llvm::None);
-}
+
 MATCHER_P(Sym, Name, "") { return arg.Name == Name; }
 
 MATCHER_P(RangeIs, R, "") { return arg.range == R; }
@@ -771,7 +776,7 @@ TEST(LocateSymbol, TextualSmoke) {
   auto AST = TU.build();
   auto Index = TU.index();
   EXPECT_THAT(locateSymbolAt(AST, T.point(), Index.get()),
-              ElementsAre(Sym("MyClass", T.range())));
+              ElementsAre(Sym("MyClass", T.range(), T.range())));
 }
 
 TEST(LocateSymbol, Textual) {
@@ -891,18 +896,20 @@ TEST(LocateSymbol, Ambiguous) {
   // FIXME: Target the constructor as well.
   EXPECT_THAT(locateSymbolAt(AST, T.point("9")), ElementsAre(Sym("Foo")));
   EXPECT_THAT(locateSymbolAt(AST, T.point("10")),
-              ElementsAre(Sym("Foo", T.range("ConstructorLoc"))));
+              ElementsAre(Sym("Foo", T.range("ConstructorLoc"), llvm::None)));
   EXPECT_THAT(locateSymbolAt(AST, T.point("11")),
-              ElementsAre(Sym("Foo", T.range("ConstructorLoc"))));
+              ElementsAre(Sym("Foo", T.range("ConstructorLoc"), llvm::None)));
   // These assertions are unordered because the order comes from
   // CXXRecordDecl::lookupDependentName() which doesn't appear to provide
   // an order guarantee.
   EXPECT_THAT(locateSymbolAt(AST, T.point("12")),
-              UnorderedElementsAre(Sym("bar", T.range("NonstaticOverload1")),
-                                   Sym("bar", T.range("NonstaticOverload2"))));
-  EXPECT_THAT(locateSymbolAt(AST, T.point("13")),
-              UnorderedElementsAre(Sym("baz", T.range("StaticOverload1")),
-                                   Sym("baz", T.range("StaticOverload2"))));
+              UnorderedElementsAre(
+                  Sym("bar", T.range("NonstaticOverload1"), llvm::None),
+                  Sym("bar", T.range("NonstaticOverload2"), llvm::None)));
+  EXPECT_THAT(
+      locateSymbolAt(AST, T.point("13")),
+      UnorderedElementsAre(Sym("baz", T.range("StaticOverload1"), llvm::None),
+                           Sym("baz", T.range("StaticOverload2"), llvm::None)));
 }
 
 TEST(LocateSymbol, TextualDependent) {
@@ -932,9 +939,10 @@ TEST(LocateSymbol, TextualDependent) {
   // interaction between locateASTReferent() and
   // locateSymbolNamedTextuallyAt().
   auto Results = locateSymbolAt(AST, Source.point(), Index.get());
-  EXPECT_THAT(Results, UnorderedElementsAre(
-                           Sym("uniqueMethodName", Header.range("FooLoc")),
-                           Sym("uniqueMethodName", Header.range("BarLoc"))));
+  EXPECT_THAT(Results,
+              UnorderedElementsAre(
+                  Sym("uniqueMethodName", Header.range("FooLoc"), llvm::None),
+                  Sym("uniqueMethodName", Header.range("BarLoc"), llvm::None)));
 }
 
 TEST(LocateSymbol, TemplateTypedefs) {
@@ -992,20 +1000,23 @@ int [[bar_not_preamble]];
   auto Locations =
       runLocateSymbolAt(Server, FooCpp, SourceAnnotations.point("p1"));
   EXPECT_TRUE(bool(Locations)) << "findDefinitions returned an error";
-  EXPECT_THAT(*Locations, ElementsAre(Sym("foo", SourceAnnotations.range())));
+  EXPECT_THAT(*Locations, ElementsAre(Sym("foo", SourceAnnotations.range(),
+                                          SourceAnnotations.range())));
 
   // Go to a definition in header_in_preamble.h.
   Locations = runLocateSymbolAt(Server, FooCpp, SourceAnnotations.point("p2"));
   EXPECT_TRUE(bool(Locations)) << "findDefinitions returned an error";
   EXPECT_THAT(
       *Locations,
-      ElementsAre(Sym("bar_preamble", HeaderInPreambleAnnotations.range())));
+      ElementsAre(Sym("bar_preamble", HeaderInPreambleAnnotations.range(),
+                      HeaderInPreambleAnnotations.range())));
 
   // Go to a definition in header_not_in_preamble.h.
   Locations = runLocateSymbolAt(Server, FooCpp, SourceAnnotations.point("p3"));
   EXPECT_TRUE(bool(Locations)) << "findDefinitions returned an error";
   EXPECT_THAT(*Locations,
               ElementsAre(Sym("bar_not_preamble",
+                              HeaderNotInPreambleAnnotations.range(),
                               HeaderNotInPreambleAnnotations.range())));
 }
 
@@ -1039,21 +1050,25 @@ TEST(GoToInclude, All) {
   // Test include in preamble.
   auto Locations = runLocateSymbolAt(Server, FooCpp, SourceAnnotations.point());
   ASSERT_TRUE(bool(Locations)) << "locateSymbolAt returned an error";
-  EXPECT_THAT(*Locations, ElementsAre(Sym("foo.h", HeaderAnnotations.range())));
+  EXPECT_THAT(*Locations, ElementsAre(Sym("foo.h", HeaderAnnotations.range(),
+                                          HeaderAnnotations.range())));
 
   // Test include in preamble, last char.
   Locations = runLocateSymbolAt(Server, FooCpp, SourceAnnotations.point("2"));
   ASSERT_TRUE(bool(Locations)) << "locateSymbolAt returned an error";
-  EXPECT_THAT(*Locations, ElementsAre(Sym("foo.h", HeaderAnnotations.range())));
+  EXPECT_THAT(*Locations, ElementsAre(Sym("foo.h", HeaderAnnotations.range(),
+                                          HeaderAnnotations.range())));
 
   Locations = runLocateSymbolAt(Server, FooCpp, SourceAnnotations.point("3"));
   ASSERT_TRUE(bool(Locations)) << "locateSymbolAt returned an error";
-  EXPECT_THAT(*Locations, ElementsAre(Sym("foo.h", HeaderAnnotations.range())));
+  EXPECT_THAT(*Locations, ElementsAre(Sym("foo.h", HeaderAnnotations.range(),
+                                          HeaderAnnotations.range())));
 
   // Test include outside of preamble.
   Locations = runLocateSymbolAt(Server, FooCpp, SourceAnnotations.point("6"));
   ASSERT_TRUE(bool(Locations)) << "locateSymbolAt returned an error";
-  EXPECT_THAT(*Locations, ElementsAre(Sym("foo.h", HeaderAnnotations.range())));
+  EXPECT_THAT(*Locations, ElementsAre(Sym("foo.h", HeaderAnnotations.range(),
+                                          HeaderAnnotations.range())));
 
   // Test a few positions that do not result in Locations.
   Locations = runLocateSymbolAt(Server, FooCpp, SourceAnnotations.point("4"));
@@ -1062,11 +1077,13 @@ TEST(GoToInclude, All) {
 
   Locations = runLocateSymbolAt(Server, FooCpp, SourceAnnotations.point("5"));
   ASSERT_TRUE(bool(Locations)) << "locateSymbolAt returned an error";
-  EXPECT_THAT(*Locations, ElementsAre(Sym("foo.h", HeaderAnnotations.range())));
+  EXPECT_THAT(*Locations, ElementsAre(Sym("foo.h", HeaderAnnotations.range(),
+                                          HeaderAnnotations.range())));
 
   Locations = runLocateSymbolAt(Server, FooCpp, SourceAnnotations.point("7"));
   ASSERT_TRUE(bool(Locations)) << "locateSymbolAt returned an error";
-  EXPECT_THAT(*Locations, ElementsAre(Sym("foo.h", HeaderAnnotations.range())));
+  EXPECT_THAT(*Locations, ElementsAre(Sym("foo.h", HeaderAnnotations.range(),
+                                          HeaderAnnotations.range())));
 
   // Objective C #import directive.
   Annotations ObjC(R"objc(
@@ -1078,7 +1095,8 @@ TEST(GoToInclude, All) {
   Server.addDocument(FooM, ObjC.code());
   Locations = runLocateSymbolAt(Server, FooM, ObjC.point());
   ASSERT_TRUE(bool(Locations)) << "locateSymbolAt returned an error";
-  EXPECT_THAT(*Locations, ElementsAre(Sym("foo.h", HeaderAnnotations.range())));
+  EXPECT_THAT(*Locations, ElementsAre(Sym("foo.h", HeaderAnnotations.range(),
+                                          HeaderAnnotations.range())));
 }
 
 TEST(LocateSymbol, WithPreamble) {
@@ -1103,7 +1121,7 @@ TEST(LocateSymbol, WithPreamble) {
   // LocateSymbol goes to a #include file: the result comes from the preamble.
   EXPECT_THAT(
       cantFail(runLocateSymbolAt(Server, FooCpp, FooWithHeader.point())),
-      ElementsAre(Sym("foo.h", FooHeader.range())));
+      ElementsAre(Sym("foo.h", FooHeader.range(), FooHeader.range())));
 
   // Only preamble is built, and no AST is built in this request.
   Server.addDocument(FooCpp, FooWithoutHeader.code(), "null",
@@ -1112,7 +1130,7 @@ TEST(LocateSymbol, WithPreamble) {
   // stale one.
   EXPECT_THAT(
       cantFail(runLocateSymbolAt(Server, FooCpp, FooWithoutHeader.point())),
-      ElementsAre(Sym("foo", FooWithoutHeader.range())));
+      ElementsAre(Sym("foo", FooWithoutHeader.range(), llvm::None)));
 
   // Reset test environment.
   runAddDocument(Server, FooCpp, FooWithHeader.code());
@@ -1122,7 +1140,7 @@ TEST(LocateSymbol, WithPreamble) {
   // Use the AST being built in above request.
   EXPECT_THAT(
       cantFail(runLocateSymbolAt(Server, FooCpp, FooWithoutHeader.point())),
-      ElementsAre(Sym("foo", FooWithoutHeader.range())));
+      ElementsAre(Sym("foo", FooWithoutHeader.range(), llvm::None)));
 }
 
 TEST(LocateSymbol, NearbyTokenSmoke) {
@@ -1133,7 +1151,7 @@ TEST(LocateSymbol, NearbyTokenSmoke) {
   auto AST = TestTU::withCode(T.code()).build();
   // We don't pass an index, so can't hit index-based fallback.
   EXPECT_THAT(locateSymbolAt(AST, T.point()),
-              ElementsAre(Sym("err", T.range())));
+              ElementsAre(Sym("err", T.range(), T.range())));
 }
 
 TEST(LocateSymbol, NearbyIdentifier) {

From e704aa4f254a26505d4bb9dc38bdee0ff4efa4ba Mon Sep 17 00:00:00 2001
From: Erich Keane <erich.keane@intel.com>
Date: Fri, 17 Jul 2020 09:48:01 -0700
Subject: [PATCH 0733/1035] DR2303: Prefer 'nearer' base classes during
 template deduction.

DR2303 fixes the case where the derived-base match for template
deduction is ambiguous if a base-of-base ALSO matches. The canonical
example (as shown in the test) is just like the MSVC implementation of
std::tuple.

This fixes a fairly sizable issue, where if a user inherits from
std::tuple on Windows (with the MS STL), they cannot use that type to
call a function that takes std::tuple.

Differential Revision: https://reviews.llvm.org/D84048
---
 clang/lib/Sema/SemaTemplateDeduction.cpp | 187 ++++++++++++++---------
 clang/test/CXX/drs/dr23xx.cpp            |  32 ++++
 clang/www/cxx_dr_status.html             |   2 +-
 3 files changed, 152 insertions(+), 69 deletions(-)

diff --git a/clang/lib/Sema/SemaTemplateDeduction.cpp b/clang/lib/Sema/SemaTemplateDeduction.cpp
index 1f7d0f0e8d973..7aa94502fa846 100644
--- a/clang/lib/Sema/SemaTemplateDeduction.cpp
+++ b/clang/lib/Sema/SemaTemplateDeduction.cpp
@@ -1201,6 +1201,120 @@ static bool isForwardingReference(QualType Param, unsigned FirstInnerIndex) {
   return false;
 }
 
+///  Attempt to deduce the template arguments by checking the base types
+///  according to (C++20 [temp.deduct.call] p4b3.
+///
+/// \param S the semantic analysis object within which we are deducing.
+///
+/// \param RecordT the top level record object we are deducing against.
+///
+/// \param TemplateParams the template parameters that we are deducing.
+///
+/// \param SpecParam the template specialization parameter type.
+///
+/// \param Info information about the template argument deduction itself.
+///
+/// \param Deduced the deduced template arguments.
+///
+/// \returns the result of template argument deduction with the bases. "invalid"
+/// means no matches, "success" found a single item, and the
+/// "MiscellaneousDeductionFailure" result happens when the match is ambiguous.
+static Sema::TemplateDeductionResult DeduceTemplateBases(
+    Sema &S, const RecordType *RecordT, TemplateParameterList *TemplateParams,
+    const TemplateSpecializationType *SpecParam, TemplateDeductionInfo &Info,
+    SmallVectorImpl<DeducedTemplateArgument> &Deduced) {
+  // C++14 [temp.deduct.call] p4b3:
+  //   If P is a class and P has the form simple-template-id, then the
+  //   transformed A can be a derived class of the deduced A. Likewise if
+  //   P is a pointer to a class of the form simple-template-id, the
+  //   transformed A can be a pointer to a derived class pointed to by the
+  //   deduced A. However, if there is a class C that is a (direct or
+  //   indirect) base class of D and derived (directly or indirectly) from a
+  //   class B and that would be a valid deduced A, the deduced A cannot be
+  //   B or pointer to B, respectively.
+  //
+  //   These alternatives are considered only if type deduction would
+  //   otherwise fail. If they yield more than one possible deduced A, the
+  //   type deduction fails.
+
+  // Use a breadth-first search through the bases to collect the set of
+  // successful matches. Visited contains the set of nodes we have already
+  // visited, while ToVisit is our stack of records that we still need to
+  // visit.  Matches contains a list of matches that have yet to be
+  // disqualified.
+  llvm::SmallPtrSet<const RecordType *, 8> Visited;
+  SmallVector<const RecordType *, 8> ToVisit;
+  // We iterate over this later, so we have to use MapVector to ensure
+  // determinism.
+  llvm::MapVector<const RecordType *, SmallVector<DeducedTemplateArgument, 8>>
+      Matches;
+
+  auto AddBases = [&Visited, &ToVisit](const RecordType *RT) {
+    CXXRecordDecl *RD = cast<CXXRecordDecl>(RT->getDecl());
+    for (const auto &Base : RD->bases()) {
+      assert(Base.getType()->isRecordType() &&
+             "Base class that isn't a record?");
+      const RecordType *RT = Base.getType()->getAs<RecordType>();
+      if (Visited.insert(RT).second)
+        ToVisit.push_back(Base.getType()->getAs<RecordType>());
+    }
+  };
+
+  // Set up the loop by adding all the bases.
+  AddBases(RecordT);
+
+  // Search each path of bases until we either run into a successful match
+  // (where all bases of it are invalid), or we run out of bases.
+  while (!ToVisit.empty()) {
+    const RecordType *NextT = ToVisit.pop_back_val();
+
+    SmallVector<DeducedTemplateArgument, 8> DeducedCopy(Deduced.begin(),
+                                                        Deduced.end());
+    TemplateDeductionInfo BaseInfo(TemplateDeductionInfo::ForBase, Info);
+    Sema::TemplateDeductionResult BaseResult =
+        DeduceTemplateArguments(S, TemplateParams, SpecParam,
+                                QualType(NextT, 0), BaseInfo, DeducedCopy);
+
+    // If this was a successful deduction, add it to the list of matches,
+    // otherwise we need to continue searching its bases.
+    if (BaseResult == Sema::TDK_Success)
+      Matches.insert({NextT, DeducedCopy});
+    else
+      AddBases(NextT);
+  }
+
+  // At this point, 'Matches' contains a list of seemingly valid bases, however
+  // in the event that we have more than 1 match, it is possible that the base
+  // of one of the matches might be disqualified for being a base of another
+  // valid match. We can count on cyclical instantiations being invalid to
+  // simplify the disqualifications.  That is, if A & B are both matches, and B
+  // inherits from A (disqualifying A), we know that A cannot inherit from B.
+  if (Matches.size() > 1) {
+    Visited.clear();
+    for (const auto &Match : Matches)
+      AddBases(Match.first);
+
+    // We can give up once we have a single item (or have run out of things to
+    // search) since cyclical inheritence isn't valid.
+    while (Matches.size() > 1 && !ToVisit.empty()) {
+      const RecordType *NextT = ToVisit.pop_back_val();
+      Matches.erase(NextT);
+
+      // Always add all bases, since the inheritence tree can contain
+      // disqualifications for multiple matches.
+      AddBases(NextT);
+    }
+  }
+
+  if (Matches.empty())
+    return Sema::TDK_Invalid;
+  if (Matches.size() > 1)
+    return Sema::TDK_MiscellaneousDeductionFailure;
+
+  std::swap(Matches.front().second, Deduced);
+  return Sema::TDK_Success;
+}
+
 /// Deduce the template arguments by comparing the parameter type and
 /// the argument type (C++ [temp.deduct.type]).
 ///
@@ -1787,78 +1901,15 @@ DeduceTemplateArgumentsByTypeMatch(Sema &S,
       if (!S.isCompleteType(Info.getLocation(), Arg))
         return Result;
 
-      // C++14 [temp.deduct.call] p4b3:
-      //   If P is a class and P has the form simple-template-id, then the
-      //   transformed A can be a derived class of the deduced A. Likewise if
-      //   P is a pointer to a class of the form simple-template-id, the
-      //   transformed A can be a pointer to a derived class pointed to by the
-      //   deduced A.
-      //
-      //   These alternatives are considered only if type deduction would
-      //   otherwise fail. If they yield more than one possible deduced A, the
-      //   type deduction fails.
-
       // Reset the incorrectly deduced argument from above.
       Deduced = DeducedOrig;
 
-      // Use data recursion to crawl through the list of base classes.
-      // Visited contains the set of nodes we have already visited, while
-      // ToVisit is our stack of records that we still need to visit.
-      llvm::SmallPtrSet<const RecordType *, 8> Visited;
-      SmallVector<const RecordType *, 8> ToVisit;
-      ToVisit.push_back(RecordT);
-      bool Successful = false;
-      SmallVector<DeducedTemplateArgument, 8> SuccessfulDeduced;
-      while (!ToVisit.empty()) {
-        // Retrieve the next class in the inheritance hierarchy.
-        const RecordType *NextT = ToVisit.pop_back_val();
-
-        // If we have already seen this type, skip it.
-        if (!Visited.insert(NextT).second)
-          continue;
-
-        // If this is a base class, try to perform template argument
-        // deduction from it.
-        if (NextT != RecordT) {
-          TemplateDeductionInfo BaseInfo(TemplateDeductionInfo::ForBase, Info);
-          Sema::TemplateDeductionResult BaseResult =
-              DeduceTemplateArguments(S, TemplateParams, SpecParam,
-                                      QualType(NextT, 0), BaseInfo, Deduced);
-
-          // If template argument deduction for this base was successful,
-          // note that we had some success. Otherwise, ignore any deductions
-          // from this base class.
-          if (BaseResult == Sema::TDK_Success) {
-            // If we've already seen some success, then deduction fails due to
-            // an ambiguity (temp.deduct.call p5).
-            if (Successful)
-              return Sema::TDK_MiscellaneousDeductionFailure;
-
-            Successful = true;
-            std::swap(SuccessfulDeduced, Deduced);
-
-            Info.Param = BaseInfo.Param;
-            Info.FirstArg = BaseInfo.FirstArg;
-            Info.SecondArg = BaseInfo.SecondArg;
-          }
-
-          Deduced = DeducedOrig;
-        }
-
-        // Visit base classes
-        CXXRecordDecl *Next = cast<CXXRecordDecl>(NextT->getDecl());
-        for (const auto &Base : Next->bases()) {
-          assert(Base.getType()->isRecordType() &&
-                 "Base class that isn't a record?");
-          ToVisit.push_back(Base.getType()->getAs<RecordType>());
-        }
-      }
-
-      if (Successful) {
-        std::swap(SuccessfulDeduced, Deduced);
-        return Sema::TDK_Success;
-      }
+      // Check bases according to C++14 [temp.deduct.call] p4b3:
+      Sema::TemplateDeductionResult BaseResult = DeduceTemplateBases(
+          S, RecordT, TemplateParams, SpecParam, Info, Deduced);
 
+      if (BaseResult != Sema::TDK_Invalid)
+        return BaseResult;
       return Result;
     }
 
diff --git a/clang/test/CXX/drs/dr23xx.cpp b/clang/test/CXX/drs/dr23xx.cpp
index 3268838ac6c85..c265ebbe359cb 100644
--- a/clang/test/CXX/drs/dr23xx.cpp
+++ b/clang/test/CXX/drs/dr23xx.cpp
@@ -113,3 +113,35 @@ namespace dr2387 { // dr2387: 9
   extern template const int d<const int>;
 #endif
 }
+
+#if __cplusplus >= 201103L
+namespace dr2303 { // dr2303: 12
+template <typename... T>
+struct A;
+template <>
+struct A<> {};
+template <typename T, typename... Ts>
+struct A<T, Ts...> : A<Ts...> {};
+struct B : A<int, int> {};
+struct C : A<int, int>, A<int> {}; // expected-warning {{direct base 'A<int>' is inaccessible}}
+struct D : A<int>, A<int, int> {}; // expected-warning {{direct base 'A<int>' is inaccessible}}
+struct E : A<int, int> {};
+struct F : B, E {};
+
+template <typename... T>
+void f(const A<T...> &) {
+  static_assert(sizeof...(T) == 2, "Should only match A<int,int>");
+}
+template <typename... T>
+void f2(const A<T...> *);
+
+void g() {
+  f(B{}); // This is no longer ambiguous.
+  B b;
+  f2(&b);
+  f(C{});
+  f(D{});
+  f(F{}); // expected-error {{ambiguous conversion from derived class}}
+}
+} //namespace dr2303
+#endif
diff --git a/clang/www/cxx_dr_status.html b/clang/www/cxx_dr_status.html
index c7369525c36fb..74319b1389437 100755
--- a/clang/www/cxx_dr_status.html
+++ b/clang/www/cxx_dr_status.html
@@ -13633,7 +13633,7 @@ <h2 id="cxxdr">C++ defect report implementation status</h2>
     <td><a href="https://wg21.link/cwg2303">2303</a></td>
     <td>DRWP</td>
     <td>Partial ordering and recursive variadic inheritance</td>
-    <td class="none" align="center">Unknown</td>
+    <td class="unreleased" align="center">Clang 12</td>
   </tr>
   <tr id="2304">
     <td><a href="https://wg21.link/cwg2304">2304</a></td>

From 793c29a267ca85da84403e3bb032ea49eed9e5f7 Mon Sep 17 00:00:00 2001
From: Sourabh Singh Tomar <SourabhSingh.Tomar@amd.com>
Date: Fri, 31 Jul 2020 18:52:12 +0530
Subject: [PATCH 0734/1035] [MLIR,OpenMP][NFCI] Removed loop for accessing
 regions of ParallelOp

`ParallelOp` has only one region associated with it.

Reviewed By: kiranchandramohan, ftynse

Differential Revision: https://reviews.llvm.org/D85008
---
 mlir/lib/Target/LLVMIR/ModuleTranslation.cpp | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
index 2c61f2a4ac11b..6b068660d98ff 100644
--- a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
@@ -399,13 +399,13 @@ ModuleTranslation::convertOmpParallel(Operation &opInst,
     llvm::Instruction *codeGenIPBBTI = codeGenIPBB->getTerminator();
 
     builder.SetInsertPoint(codeGenIPBB);
-
-    for (auto &region : opInst.getRegions()) {
-      for (auto &bb : region) {
-        auto *llvmBB = llvm::BasicBlock::Create(
-            llvmContext, "omp.par.region", codeGenIP.getBlock()->getParent());
-        blockMapping[&bb] = llvmBB;
-      }
+    // ParallelOp has only `1` region associated with it.
+    auto &region = cast<omp::ParallelOp>(opInst).getRegion();
+    for (auto &bb : region) {
+      auto *llvmBB = llvm::BasicBlock::Create(
+          llvmContext, "omp.par.region", codeGenIP.getBlock()->getParent());
+      blockMapping[&bb] = llvmBB;
+    }
 
       // Then, convert blocks one by one in topological order to ensure
       // defs are converted before uses.
@@ -433,7 +433,6 @@ ModuleTranslation::convertOmpParallel(Operation &opInst,
       // Finally, after all blocks have been traversed and values mapped,
       // connect the PHI nodes to the results of preceding blocks.
       connectPHINodes(region, valueMapping, blockMapping);
-    }
   };
 
   // TODO: Perform appropriate actions according to the data-sharing

From 2da9b44415ce7958d09da53746ad46be631dcf1f Mon Sep 17 00:00:00 2001
From: Nico Weber <thakis@chromium.org>
Date: Fri, 31 Jul 2020 09:54:39 -0400
Subject: [PATCH 0735/1035] [gn build] (manually) merge 63d3aeb529

---
 llvm/utils/gn/secondary/llvm/include/llvm/Config/BUILD.gn | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/llvm/utils/gn/secondary/llvm/include/llvm/Config/BUILD.gn b/llvm/utils/gn/secondary/llvm/include/llvm/Config/BUILD.gn
index 94f0a66ecb182..c9e7c45fc118b 100644
--- a/llvm/utils/gn/secondary/llvm/include/llvm/Config/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/include/llvm/Config/BUILD.gn
@@ -125,7 +125,6 @@ write_cmake_config("config") {
     "RETSIGTYPE=void",
     "LLVM_GISEL_COV_ENABLED=",
     "LLVM_GISEL_COV_PREFIX=",
-    "LLVM_WITH_Z3=",
 
     # FIXME: Set to 1 on mac once the 10.14 SDK is in common use.
     "LLVM_SUPPORT_XCODE_SIGNPOSTS=",
@@ -315,6 +314,7 @@ write_cmake_config("llvm-config") {
   values = [
     "LLVM_ENABLE_DUMP=",
     "LLVM_DEFAULT_TARGET_TRIPLE=$llvm_target_triple",
+    "LLVM_FORCE_ENABLE_STATS=",
     "LLVM_HAS_ATOMICS=1",
     "LLVM_HAVE_TF_AOT=",
     "LLVM_HAVE_TF_API=",
@@ -332,8 +332,8 @@ write_cmake_config("llvm-config") {
     "LLVM_VERSION_MAJOR=$llvm_version_major",
     "LLVM_VERSION_MINOR=$llvm_version_minor",
     "LLVM_VERSION_PATCH=$llvm_version_patch",
+    "LLVM_WITH_Z3=",
     "PACKAGE_VERSION=${llvm_version}git",
-    "LLVM_FORCE_ENABLE_STATS=",
   ]
 
   if (current_os == "win") {

From 6983cf3a57aa6d8619eb39e1625eed5340ba05c7 Mon Sep 17 00:00:00 2001
From: Frederik Gossen <frgossen@google.com>
Date: Fri, 31 Jul 2020 14:17:31 +0000
Subject: [PATCH 0736/1035] [MLIR][Shape] Allow unsafe `shape.broadcast`

In a context in which `shape.broadcast` is known not to produce an error value,
we want it to operate solely on extent tensors. The operation's behavior is
then undefined in the error case as the result type cannot hold this value.

Differential Revision: https://reviews.llvm.org/D84933
---
 .../include/mlir/Dialect/Shape/IR/ShapeOps.td | 42 ++++++++++---------
 mlir/test/Dialect/Shape/canonicalize.mlir     | 25 +++++++++++
 mlir/test/Dialect/Shape/invalid.mlir          | 10 +++--
 3 files changed, 53 insertions(+), 24 deletions(-)

diff --git a/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td b/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td
index 72e392b256db3..bc7b6048e28f4 100644
--- a/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td
+++ b/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td
@@ -49,25 +49,24 @@ def Shape_AddOp : Shape_Op<"add", [Commutative, NoSideEffect]> {
 def Shape_BroadcastOp : Shape_Op<"broadcast", [Commutative]> {
   let summary = "Returns the broadcasted output shape of two inputs";
   let description = [{
-    Computes the broadcasted output shape following:
-    1. If any inputs are unranked, output is unranked;
-    2. Else the input array with number of dimensions smaller than the max
-       input dimension, has 1’s prepended to its shapes and the output shape is
-       calculated as follows:
-
-           output[i] = lhs[i] if lhs[i] == rhs[i] or rhs[i] is unknown/undefined
-                     = rhs[i] if lhs[i] is unknown/undefined
-                     = lhs[i] if rhs[i] == 1
-                     = rhs[i] if lhs[i] == 1
-                     = error  if lhs[i] != rhs[i]
-
-    Op has an optional string attribute for the error case where there is no
-    broadcastable output shape possible for the given inputs.
-
-    Op may also return an ExtentTensor, but this should only be done when this
-    is statically guaranteed to never fail, either because of a dependency on a
-    cstr_broadcastable operation or other details of the construction of the
-    program.
+    Returns the broadcasted shape for two input shapes or extent tensors. Both
+    operands can be of type `shape.shape` or `tensor<?xindex>`. The result is of
+    type `shape.shape` and, if both operands are tensors, may be of type
+    `tensor<?xindex>`.
+
+    If the two operand shapes are of different rank the smaller one is padded
+    with 1's from the left. The resulting broadcasted shape is then defined as
+
+        result[i] = lhs[i] if lhs[i] == rhs[i]
+                  = lhs[i] if rhs[i] == 1
+                  = rhs[i] if lhs[i] == 1.
+
+    In case the resulting shape is undefined, i.e. if corresponding extents are
+    different from each other but none is 1, the result is an error shape.
+    Likewise error values are propagated if any of the operands holds an error
+    value. If the result type is an extent tensor (and can therefore not hold
+    the error value) the behavior may be undefined. The optional string
+    attribute can be used to describe the error case.
   }];
 
   let arguments = (ins Shape_ShapeOrExtentTensorType:$lhs,
@@ -75,8 +74,11 @@ def Shape_BroadcastOp : Shape_Op<"broadcast", [Commutative]> {
                        OptionalAttr<StrAttr>:$error);
   let results = (outs Shape_ShapeOrExtentTensorType:$result);
 
-  let assemblyFormat = "$lhs `,` $rhs attr-dict `:` type($lhs) `,` type($rhs) `->` type($result)";
+  let assemblyFormat = [{
+    $lhs `,` $rhs attr-dict `:` type($lhs) `,` type($rhs) `->` type($result)
+  }];
 
+  let verifier = [{ return ::verifyShapeOrExtentTensorOp(*this); }];
   let hasFolder = 1;
 
   let verifier = [{ return ::verifyShapeOrExtentTensorOp(*this); }];
diff --git a/mlir/test/Dialect/Shape/canonicalize.mlir b/mlir/test/Dialect/Shape/canonicalize.mlir
index e18ff14df304a..21c5a68c3adc4 100644
--- a/mlir/test/Dialect/Shape/canonicalize.mlir
+++ b/mlir/test/Dialect/Shape/canonicalize.mlir
@@ -60,6 +60,31 @@ func @f() -> !shape.shape {
 
 // -----
 
+// Basic case including extent tensors.
+// CHECK-LABEL: @broadcast
+func @broadcast() -> tensor<?xindex> {
+  // CHECK: shape.const_shape [7, 2] : tensor<?xindex>
+  %0 = shape.const_shape [1, 2] : tensor<?xindex>
+  %1 = shape.const_shape [7, 1] : tensor<?xindex>
+  %2 = shape.broadcast %0, %1
+      : tensor<?xindex>, tensor<?xindex> -> tensor<?xindex>
+  return %2 : tensor<?xindex>
+}
+
+// -----
+
+// Basic case including extent tensors.
+// CHECK-LABEL: @broadcast
+func @broadcast() -> !shape.shape {
+  // CHECK: shape.const_shape [7, 2] : !shape.shape
+  %0 = shape.const_shape [1, 2] : tensor<?xindex>
+  %1 = shape.const_shape [7, 1] : tensor<?xindex>
+  %2 = shape.broadcast %0, %1 : tensor<?xindex>, tensor<?xindex> -> !shape.shape
+  return %2 : !shape.shape
+}
+
+// -----
+
 // Rhs is a scalar.
 // CHECK-LABEL: func @f
 func @f(%arg0 : !shape.shape) -> !shape.shape {
diff --git a/mlir/test/Dialect/Shape/invalid.mlir b/mlir/test/Dialect/Shape/invalid.mlir
index 448bd84e754ee..eb0ae5ae05a9b 100644
--- a/mlir/test/Dialect/Shape/invalid.mlir
+++ b/mlir/test/Dialect/Shape/invalid.mlir
@@ -138,17 +138,19 @@ func @add(%lhs : !shape.size, %rhs : index) -> index {
 
 // -----
 
-func @broadcast_error_possible(%arg0 : !shape.shape, %arg1 : !shape.shape) -> tensor<?xindex> {
+func @broadcast(%arg0 : !shape.shape, %arg1 : !shape.shape) -> tensor<?xindex> {
   // expected-error@+1 {{if at least one of the operands can hold error values then the result must be of type `shape` to propagate them}}
-  %result = shape.broadcast %arg0, %arg1 : !shape.shape, !shape.shape -> tensor<?xindex>
+  %result = shape.broadcast %arg0, %arg1
+      : !shape.shape, !shape.shape -> tensor<?xindex>
   return %result : tensor<?xindex>
 }
 
 
 // -----
 
-func @broadcast_error_possible(%arg0 : !shape.shape, %arg1 : tensor<?xindex>) -> tensor<?xindex> {
+func @broadcast(%arg0 : !shape.shape, %arg1 : tensor<?xindex>) -> tensor<?xindex> {
   // expected-error@+1 {{if at least one of the operands can hold error values then the result must be of type `shape` to propagate them}}
-  %result = shape.broadcast %arg0, %arg1 : !shape.shape, tensor<?xindex> -> tensor<?xindex>
+  %result = shape.broadcast %arg0, %arg1
+      : !shape.shape, tensor<?xindex> -> tensor<?xindex>
   return %result : tensor<?xindex>
 }

From 57bd64ff8434aa55aeb1c7a1035f4b5b7468b809 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Fri, 27 May 2016 15:50:12 -0700
Subject: [PATCH 0737/1035] Support addrspacecast initializers with
 isNoopAddrSpaceCast

Moves isNoopAddrSpaceCast to the TargetMachine. It logically belongs
with the DataLayout.
---
 llvm/include/llvm/CodeGen/BasicTTIImpl.h      |  2 +-
 llvm/include/llvm/CodeGen/TargetLowering.h    |  9 +------
 llvm/include/llvm/Target/TargetMachine.h      |  5 ++++
 llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp    | 10 +++++++
 llvm/lib/CodeGen/CodeGenPrepare.cpp           |  2 +-
 .../lib/CodeGen/SelectionDAG/SelectionDAG.cpp |  2 +-
 .../SelectionDAG/SelectionDAGBuilder.cpp      |  2 +-
 llvm/lib/CodeGen/TargetLoweringBase.cpp       |  5 ++++
 llvm/lib/Target/AArch64/AArch64ISelLowering.h |  6 -----
 .../lib/Target/AArch64/AArch64TargetMachine.h |  6 +++++
 llvm/lib/Target/AMDGPU/AMDGPU.h               | 15 +++++++++--
 .../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp |  6 ++---
 .../Target/AMDGPU/AMDGPURegisterBankInfo.cpp  |  5 ++--
 .../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp |  6 +++++
 llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h  |  2 ++
 .../AMDGPU/AMDGPUTargetTransformInfo.cpp      |  5 +++-
 llvm/lib/Target/AMDGPU/SIISelLowering.cpp     | 17 ++++++------
 llvm/lib/Target/AMDGPU/SIISelLowering.h       |  9 -------
 llvm/lib/Target/ARM/ARMISelLowering.h         |  6 -----
 llvm/lib/Target/ARM/ARMTargetMachine.h        |  6 +++++
 llvm/lib/Target/Mips/MipsISelLowering.h       |  8 ------
 llvm/lib/Target/Mips/MipsTargetMachine.h      |  8 ++++++
 llvm/lib/Target/PowerPC/PPCISelLowering.h     |  5 ----
 llvm/lib/Target/PowerPC/PPCTargetMachine.h    |  5 ++++
 llvm/lib/Target/X86/X86ISelLowering.cpp       | 11 --------
 llvm/lib/Target/X86/X86ISelLowering.h         |  2 --
 llvm/lib/Target/X86/X86TargetMachine.cpp      |  8 ++++++
 llvm/lib/Target/X86/X86TargetMachine.h        |  2 ++
 .../addrspacecast-initializer-unsupported.ll  |  7 +++++
 .../AMDGPU/addrspacecast-initializer.ll       | 27 +++++++++++++++++++
 30 files changed, 131 insertions(+), 78 deletions(-)
 create mode 100644 llvm/test/CodeGen/AMDGPU/addrspacecast-initializer-unsupported.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/addrspacecast-initializer.ll

diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index d8af891a4b762..9e5c45084c599 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -222,7 +222,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
   }
 
   bool isNoopAddrSpaceCast(unsigned FromAS, unsigned ToAS) const {
-    return getTLI()->isNoopAddrSpaceCast(FromAS, ToAS);
+    return getTLI()->getTargetMachine().isNoopAddrSpaceCast(FromAS, ToAS);
   }
 
   Value *rewriteIntrinsicWithAddressSpace(IntrinsicInst *II, Value *OldV,
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index a92761abd2f82..79ef2d06d38f4 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -1759,17 +1759,10 @@ class TargetLoweringBase {
     return "";
   }
 
-  /// Returns true if a cast between SrcAS and DestAS is a noop.
-  virtual bool isNoopAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const {
-    return false;
-  }
-
   /// Returns true if a cast from SrcAS to DestAS is "cheap", such that e.g. we
   /// are happy to sink it into basic blocks. A cast may be free, but not
   /// necessarily a no-op. e.g. a free truncate from a 64-bit to 32-bit pointer.
-  virtual bool isFreeAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const {
-    return isNoopAddrSpaceCast(SrcAS, DestAS);
-  }
+  virtual bool isFreeAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const;
 
   /// Return true if the pointer arguments to CI should be aligned by aligning
   /// the object whose address is being passed. If so then MinSize is set to the
diff --git a/llvm/include/llvm/Target/TargetMachine.h b/llvm/include/llvm/Target/TargetMachine.h
index f59bc5e5bae59..2a422341fdc84 100644
--- a/llvm/include/llvm/Target/TargetMachine.h
+++ b/llvm/include/llvm/Target/TargetMachine.h
@@ -271,6 +271,11 @@ class TargetMachine {
     return Options.BBSectionsFuncListBuf.get();
   }
 
+  /// Returns true if a cast between SrcAS and DestAS is a noop.
+  virtual bool isNoopAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const {
+    return false;
+  }
+
   /// Get a \c TargetIRAnalysis appropriate for the target.
   ///
   /// This is used to construct the new pass manager's target IR analysis pass,
diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
index b50cae89b3494..78f18ab8aff6e 100644
--- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
@@ -2295,6 +2295,16 @@ const MCExpr *AsmPrinter::lowerConstant(const Constant *CV) {
   }
 
   switch (CE->getOpcode()) {
+  case Instruction::AddrSpaceCast: {
+    const Constant *Op = CE->getOperand(0);
+    unsigned DstAS = CE->getType()->getPointerAddressSpace();
+    unsigned SrcAS = Op->getType()->getPointerAddressSpace();
+    if (TM.isNoopAddrSpaceCast(SrcAS, DstAS))
+      return lowerConstant(Op);
+
+    // Fallthrough to error.
+    LLVM_FALLTHROUGH;
+  }
   default: {
     // If the code isn't optimized, there may be outstanding folding
     // opportunities. Attempt to fold the expression using DataLayout as a
diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp
index 42cffafbb1ce9..a85ac80ef3652 100644
--- a/llvm/lib/CodeGen/CodeGenPrepare.cpp
+++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp
@@ -4322,7 +4322,7 @@ bool AddressingModeMatcher::matchOperationAddr(User *AddrInst, unsigned Opcode,
     unsigned SrcAS
       = AddrInst->getOperand(0)->getType()->getPointerAddressSpace();
     unsigned DestAS = AddrInst->getType()->getPointerAddressSpace();
-    if (TLI.isNoopAddrSpaceCast(SrcAS, DestAS))
+    if (TLI.getTargetMachine().isNoopAddrSpaceCast(SrcAS, DestAS))
       return matchAddr(AddrInst->getOperand(0), Depth);
     return false;
   }
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 87d2fa15d0377..7fdf8a82bae85 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -6394,7 +6394,7 @@ static void checkAddrSpaceIsValidForLibcall(const TargetLowering *TLI,
                                             unsigned AS) {
   // Lowering memcpy / memset / memmove intrinsics to calls is only valid if all
   // pointer operands can be losslessly bitcasted to pointers of address space 0
-  if (AS != 0 && !TLI->isNoopAddrSpaceCast(AS, 0)) {
+  if (AS != 0 && !TLI->getTargetMachine().isNoopAddrSpaceCast(AS, 0)) {
     report_fatal_error("cannot lower memory intrinsic in address space " +
                        Twine(AS));
   }
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index d152cf8c4792a..c2a284af592d7 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -3425,7 +3425,7 @@ void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {
   unsigned SrcAS = SV->getType()->getPointerAddressSpace();
   unsigned DestAS = I.getType()->getPointerAddressSpace();
 
-  if (!TLI.isNoopAddrSpaceCast(SrcAS, DestAS))
+  if (!TM.isNoopAddrSpaceCast(SrcAS, DestAS))
     N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS);
 
   setValue(&I, N);
diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index db4fcf7494c7f..4562e1e018c0b 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -801,6 +801,11 @@ bool TargetLoweringBase::canOpTrap(unsigned Op, EVT VT) const {
   }
 }
 
+bool TargetLoweringBase::isFreeAddrSpaceCast(unsigned SrcAS,
+                                             unsigned DestAS) const {
+  return TM.isNoopAddrSpaceCast(SrcAS, DestAS);
+}
+
 void TargetLoweringBase::setJumpIsExpensive(bool isExpensive) {
   // If the command-line option was specified, ignore this request.
   if (!JumpIsExpensiveOverride.getNumOccurrences())
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index 344ed96d79609..a793fb6bb4625 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -462,12 +462,6 @@ class AArch64TargetLowering : public TargetLowering {
 
   SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override;
 
-  /// Returns true if a cast between SrcAS and DestAS is a noop.
-  bool isNoopAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const override {
-    // Addrspacecasts are always noops.
-    return true;
-  }
-
   /// This method returns a target specific FastISel object, or null if the
   /// target does not support "fast" ISel.
   FastISel *createFastISel(FunctionLoweringInfo &funcInfo,
diff --git a/llvm/lib/Target/AArch64/AArch64TargetMachine.h b/llvm/lib/Target/AArch64/AArch64TargetMachine.h
index 7738a42293919..25e6261343179 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetMachine.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetMachine.h
@@ -57,6 +57,12 @@ class AArch64TargetMachine : public LLVMTargetMachine {
                                 SMDiagnostic &Error,
                                 SMRange &SourceRange) const override;
 
+  /// Returns true if a cast between SrcAS and DestAS is a noop.
+  bool isNoopAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const override {
+    // Addrspacecasts are always noops.
+    return true;
+  }
+
 private:
   bool isLittle;
 };
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h
index 88c79665be60d..251e12ee09f25 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.h
@@ -281,8 +281,6 @@ enum TargetIndex {
 };
 }
 
-} // End namespace llvm
-
 /// OpenCL uses address spaces to differentiate between
 /// various memory regions on the hardware. On the CPU
 /// all of the address spaces point to the same memory,
@@ -339,4 +337,17 @@ namespace AMDGPUAS {
   };
 }
 
+namespace AMDGPU {
+
+// FIXME: Missing constant_32bit
+inline bool isFlatGlobalAddrSpace(unsigned AS) {
+  return AS == AMDGPUAS::GLOBAL_ADDRESS ||
+         AS == AMDGPUAS::FLAT_ADDRESS ||
+         AS == AMDGPUAS::CONSTANT_ADDRESS ||
+         AS > AMDGPUAS::MAX_AMDGPU_ADDRESS;
+}
+}
+
+} // End namespace llvm
+
 #endif
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 4320151d5758e..c5d5f1675bc8d 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -1677,8 +1677,7 @@ bool AMDGPULegalizerInfo::legalizeAddrSpaceCast(
   const AMDGPUTargetMachine &TM
     = static_cast<const AMDGPUTargetMachine &>(MF.getTarget());
 
-  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
-  if (ST.getTargetLowering()->isNoopAddrSpaceCast(SrcAS, DestAS)) {
+  if (TM.isNoopAddrSpaceCast(SrcAS, DestAS)) {
     MI.setDesc(B.getTII().get(TargetOpcode::G_BITCAST));
     return true;
   }
@@ -2251,8 +2250,7 @@ bool AMDGPULegalizerInfo::legalizeAtomicCmpXChg(
   Register CmpVal = MI.getOperand(2).getReg();
   Register NewVal = MI.getOperand(3).getReg();
 
-  assert(SITargetLowering::isFlatGlobalAddrSpace(
-           MRI.getType(PtrReg).getAddressSpace()) &&
+  assert(AMDGPU::isFlatGlobalAddrSpace(MRI.getType(PtrReg).getAddressSpace()) &&
          "this should not have been custom lowered");
 
   LLT ValTy = MRI.getType(CmpVal);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
index 3fed45e84b8f7..9674474cd3cf9 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
@@ -3232,7 +3232,7 @@ AMDGPURegisterBankInfo::getValueMappingForPtr(const MachineRegisterInfo &MRI,
   LLT PtrTy = MRI.getType(PtrReg);
   unsigned Size = PtrTy.getSizeInBits();
   if (Subtarget.useFlatForGlobal() ||
-      !SITargetLowering::isFlatGlobalAddrSpace(PtrTy.getAddressSpace()))
+      !AMDGPU::isFlatGlobalAddrSpace(PtrTy.getAddressSpace()))
     return AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size);
 
   // If we're using MUBUF instructions for global memory, an SGPR base register
@@ -3258,8 +3258,7 @@ AMDGPURegisterBankInfo::getInstrMappingForLoad(const MachineInstr &MI) const {
 
   const RegisterBank *PtrBank = getRegBank(PtrReg, MRI, *TRI);
 
-  if (PtrBank == &AMDGPU::SGPRRegBank &&
-      SITargetLowering::isFlatGlobalAddrSpace(AS)) {
+  if (PtrBank == &AMDGPU::SGPRRegBank && AMDGPU::isFlatGlobalAddrSpace(AS)) {
     if (isScalarLoadLegal(MI)) {
       // We have a uniform instruction so we want to use an SMRD load
       ValMapping = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index b4b10835837cd..b49a417dfb09c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -526,6 +526,12 @@ const R600Subtarget *R600TargetMachine::getSubtargetImpl(
   return I.get();
 }
 
+bool AMDGPUTargetMachine::isNoopAddrSpaceCast(unsigned SrcAS,
+                                              unsigned DestAS) const {
+  return AMDGPU::isFlatGlobalAddrSpace(SrcAS) &&
+         AMDGPU::isFlatGlobalAddrSpace(DestAS);
+}
+
 TargetTransformInfo
 R600TargetMachine::getTargetTransformInfo(const Function &F) {
   return TargetTransformInfo(R600TTIImpl(this, F));
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h
index e223fecc88195..aedcaf3fe4149 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h
@@ -62,6 +62,8 @@ class AMDGPUTargetMachine : public LLVMTargetMachine {
             AddrSpace == AMDGPUAS::PRIVATE_ADDRESS ||
             AddrSpace == AMDGPUAS::REGION_ADDRESS) ? -1 : 0;
   }
+
+  bool isNoopAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const override;
 };
 
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index ad15763e1a010..da00a993bd649 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -934,7 +934,10 @@ Value *GCNTTIImpl::rewriteIntrinsicWithAddressSpace(IntrinsicInst *II,
     Type *MaskTy = MaskOp->getType();
 
     bool DoTruncate = false;
-    if (!getTLI()->isNoopAddrSpaceCast(OldAS, NewAS)) {
+
+    const GCNTargetMachine &TM =
+        static_cast<const GCNTargetMachine &>(getTLI()->getTargetMachine());
+    if (!TM.isNoopAddrSpaceCast(OldAS, NewAS)) {
       // All valid 64-bit to 32-bit casts work by chopping off the high
       // bits. Any masking only clearing the low bits will also apply in the new
       // address space.
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 4ea44373b976a..59f45f9daf428 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -1478,11 +1478,6 @@ EVT SITargetLowering::getOptimalMemOpType(
   return MVT::Other;
 }
 
-bool SITargetLowering::isNoopAddrSpaceCast(unsigned SrcAS,
-                                           unsigned DestAS) const {
-  return isFlatGlobalAddrSpace(SrcAS) && isFlatGlobalAddrSpace(DestAS);
-}
-
 bool SITargetLowering::isMemOpHasNoClobberedMemOperand(const SDNode *N) const {
   const MemSDNode *MemNode = cast<MemSDNode>(N);
   const Value *Ptr = MemNode->getMemOperand()->getValue();
@@ -1497,7 +1492,9 @@ bool SITargetLowering::isFreeAddrSpaceCast(unsigned SrcAS,
   if (SrcAS == AMDGPUAS::FLAT_ADDRESS)
     return true;
 
-  return isNoopAddrSpaceCast(SrcAS, DestAS);
+  const GCNTargetMachine &TM =
+      static_cast<const GCNTargetMachine &>(getTargetMachine());
+  return TM.isNoopAddrSpaceCast(SrcAS, DestAS);
 }
 
 bool SITargetLowering::isMemOpUniform(const SDNode *N) const {
@@ -2285,8 +2282,10 @@ SDValue SITargetLowering::LowerFormalArguments(
       if (Arg.Flags.isByRef()) {
         SDValue Ptr = lowerKernArgParameterPtr(DAG, DL, Chain, Offset);
 
-        if (!isNoopAddrSpaceCast(AMDGPUAS::CONSTANT_ADDRESS,
-                                 Arg.Flags.getPointerAddrSpace())) {
+        const GCNTargetMachine &TM =
+            static_cast<const GCNTargetMachine &>(getTargetMachine());
+        if (!TM.isNoopAddrSpaceCast(AMDGPUAS::CONSTANT_ADDRESS,
+                                    Arg.Flags.getPointerAddrSpace())) {
           Ptr = DAG.getAddrSpaceCast(DL, VT, Ptr, AMDGPUAS::CONSTANT_ADDRESS,
                                      Arg.Flags.getPointerAddrSpace());
         }
@@ -8506,7 +8505,7 @@ SDValue SITargetLowering::LowerATOMIC_CMP_SWAP(SDValue Op, SelectionDAG &DAG) co
   unsigned AS = AtomicNode->getAddressSpace();
 
   // No custom lowering required for local address space
-  if (!isFlatGlobalAddrSpace(AS))
+  if (!AMDGPU::isFlatGlobalAddrSpace(AS))
     return Op;
 
   // Non-local address space requires custom lowering for atomic compare
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h
index f4c0764640575..19dea37ad410d 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h
@@ -275,15 +275,6 @@ class SITargetLowering final : public AMDGPUTargetLowering {
            AS == AMDGPUAS::PRIVATE_ADDRESS;
   }
 
-  // FIXME: Missing constant_32bit
-  static bool isFlatGlobalAddrSpace(unsigned AS) {
-    return AS == AMDGPUAS::GLOBAL_ADDRESS ||
-           AS == AMDGPUAS::FLAT_ADDRESS ||
-           AS == AMDGPUAS::CONSTANT_ADDRESS ||
-           AS > AMDGPUAS::MAX_AMDGPU_ADDRESS;
-  }
-
-  bool isNoopAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const override;
   bool isFreeAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const override;
 
   TargetLoweringBase::LegalizeTypeAction
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.h b/llvm/lib/Target/ARM/ARMISelLowering.h
index 1428600ca5240..f5bb097062aff 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.h
+++ b/llvm/lib/Target/ARM/ARMISelLowering.h
@@ -528,12 +528,6 @@ class VectorType;
     const TargetRegisterClass *
     getRegClassFor(MVT VT, bool isDivergent = false) const override;
 
-    /// Returns true if a cast between SrcAS and DestAS is a noop.
-    bool isNoopAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const override {
-      // Addrspacecasts are always noops.
-      return true;
-    }
-
     bool shouldAlignPointerArgs(CallInst *CI, unsigned &MinSize,
                                 unsigned &PrefAlign) const override;
 
diff --git a/llvm/lib/Target/ARM/ARMTargetMachine.h b/llvm/lib/Target/ARM/ARMTargetMachine.h
index ac55d2bdcc2b3..8428092bf1794 100644
--- a/llvm/lib/Target/ARM/ARMTargetMachine.h
+++ b/llvm/lib/Target/ARM/ARMTargetMachine.h
@@ -72,6 +72,12 @@ class ARMBaseTargetMachine : public LLVMTargetMachine {
   }
 
   bool targetSchedulesPostRAScheduling() const override { return true; };
+
+  /// Returns true if a cast between SrcAS and DestAS is a noop.
+  bool isNoopAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const override {
+    // Addrspacecasts are always noops.
+    return true;
+  }
 };
 
 /// ARM/Thumb little endian target machine.
diff --git a/llvm/lib/Target/Mips/MipsISelLowering.h b/llvm/lib/Target/Mips/MipsISelLowering.h
index 16b4d51d3ca63..0c5df4ba1bade 100644
--- a/llvm/lib/Target/Mips/MipsISelLowering.h
+++ b/llvm/lib/Target/Mips/MipsISelLowering.h
@@ -365,14 +365,6 @@ class TargetRegisterClass;
       return ABI.IsN64() ? Mips::A1_64 : Mips::A1;
     }
 
-    /// Returns true if a cast between SrcAS and DestAS is a noop.
-    bool isNoopAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const override {
-      // Mips doesn't have any special address spaces so we just reserve
-      // the first 256 for software use (e.g. OpenCL) and treat casts
-      // between them as noops.
-      return SrcAS < 256 && DestAS < 256;
-    }
-
     bool isJumpTableRelative() const override {
       return getTargetMachine().isPositionIndependent();
     }
diff --git a/llvm/lib/Target/Mips/MipsTargetMachine.h b/llvm/lib/Target/Mips/MipsTargetMachine.h
index 25300504a02dc..e0de924be4fd1 100644
--- a/llvm/lib/Target/Mips/MipsTargetMachine.h
+++ b/llvm/lib/Target/Mips/MipsTargetMachine.h
@@ -63,6 +63,14 @@ class MipsTargetMachine : public LLVMTargetMachine {
     return TLOF.get();
   }
 
+  /// Returns true if a cast between SrcAS and DestAS is a noop.
+  bool isNoopAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const override {
+    // Mips doesn't have any special address spaces so we just reserve
+    // the first 256 for software use (e.g. OpenCL) and treat casts
+    // between them as noops.
+    return SrcAS < 256 && DestAS < 256;
+  }
+
   bool isLittleEndian() const { return isLittle; }
   const MipsABIInfo &getABI() const { return ABI; }
 };
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.h b/llvm/lib/Target/PowerPC/PPCISelLowering.h
index 8cc42226d7f0b..80588a1bd4019 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.h
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.h
@@ -1022,11 +1022,6 @@ namespace llvm {
       }
     };
 
-    bool isNoopAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const override {
-      // Addrspacecasts are always noops.
-      return true;
-    }
-
     bool canReuseLoadAddress(SDValue Op, EVT MemVT, ReuseLoadInfo &RLI,
                              SelectionDAG &DAG,
                              ISD::LoadExtType ET = ISD::NON_EXTLOAD) const;
diff --git a/llvm/lib/Target/PowerPC/PPCTargetMachine.h b/llvm/lib/Target/PowerPC/PPCTargetMachine.h
index fd1d14ae32d4a..21faa4e710e3e 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetMachine.h
+++ b/llvm/lib/Target/PowerPC/PPCTargetMachine.h
@@ -58,6 +58,11 @@ class PPCTargetMachine final : public LLVMTargetMachine {
     const Triple &TT = getTargetTriple();
     return (TT.getArch() == Triple::ppc64 || TT.getArch() == Triple::ppc64le);
   };
+
+  bool isNoopAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const override {
+    // Addrspacecasts are always noops.
+    return true;
+  }
 };
 } // end namespace llvm
 
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 493d934e5381a..cb1067a06239e 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -2537,17 +2537,6 @@ Value *X86TargetLowering::getSafeStackPointerLocation(IRBuilder<> &IRB) const {
   return TargetLowering::getSafeStackPointerLocation(IRB);
 }
 
-bool X86TargetLowering::isNoopAddrSpaceCast(unsigned SrcAS,
-                                            unsigned DestAS) const {
-  assert(SrcAS != DestAS && "Expected different address spaces!");
-
-  const TargetMachine &TM = getTargetMachine();
-  if (TM.getPointerSize(SrcAS) != TM.getPointerSize(DestAS))
-    return false;
-
-  return SrcAS < 256 && DestAS < 256;
-}
-
 //===----------------------------------------------------------------------===//
 //               Return Value Calling Convention Implementation
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/X86/X86ISelLowering.h b/llvm/lib/Target/X86/X86ISelLowering.h
index 7f3dc90a2d735..2c22a62fb506f 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.h
+++ b/llvm/lib/Target/X86/X86ISelLowering.h
@@ -1349,8 +1349,6 @@ namespace llvm {
                                           Align Alignment,
                                           SelectionDAG &DAG) const;
 
-    bool isNoopAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const override;
-
     /// Customize the preferred legalization strategy for certain types.
     LegalizeTypeAction getPreferredVectorAction(MVT VT) const override;
 
diff --git a/llvm/lib/Target/X86/X86TargetMachine.cpp b/llvm/lib/Target/X86/X86TargetMachine.cpp
index f660b99a4511d..685a8e8fa8774 100644
--- a/llvm/lib/Target/X86/X86TargetMachine.cpp
+++ b/llvm/lib/Target/X86/X86TargetMachine.cpp
@@ -311,6 +311,14 @@ X86TargetMachine::getSubtargetImpl(const Function &F) const {
   return I.get();
 }
 
+bool X86TargetMachine::isNoopAddrSpaceCast(unsigned SrcAS,
+                                           unsigned DestAS) const {
+  assert(SrcAS != DestAS && "Expected different address spaces!");
+  if (getPointerSize(SrcAS) != getPointerSize(DestAS))
+    return false;
+  return SrcAS < 256 && DestAS < 256;
+}
+
 //===----------------------------------------------------------------------===//
 // X86 TTI query.
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/X86/X86TargetMachine.h b/llvm/lib/Target/X86/X86TargetMachine.h
index 8d98474a39c06..69d7e48b89778 100644
--- a/llvm/lib/Target/X86/X86TargetMachine.h
+++ b/llvm/lib/Target/X86/X86TargetMachine.h
@@ -54,6 +54,8 @@ class X86TargetMachine final : public LLVMTargetMachine {
   }
 
   bool isJIT() const { return IsJIT; }
+
+  bool isNoopAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const override;
 };
 
 } // end namespace llvm
diff --git a/llvm/test/CodeGen/AMDGPU/addrspacecast-initializer-unsupported.ll b/llvm/test/CodeGen/AMDGPU/addrspacecast-initializer-unsupported.ll
new file mode 100644
index 0000000000000..223efcc738188
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/addrspacecast-initializer-unsupported.ll
@@ -0,0 +1,7 @@
+; RUN: not --crash llc -march=amdgcn -verify-machineinstrs < %s 2>&1 | FileCheck -check-prefix=ERROR %s
+
+; ERROR: LLVM ERROR: Unsupported expression in static initializer: addrspacecast ([256 x i32] addrspace(3)* @lds.arr to [256 x i32] addrspace(4)*)
+
+@lds.arr = unnamed_addr addrspace(3) global [256 x i32] undef, align 4
+
+@gv_flatptr_from_lds = unnamed_addr addrspace(2) global i32 addrspace(4)* getelementptr ([256 x i32], [256 x i32] addrspace(4)* addrspacecast ([256 x i32] addrspace(3)* @lds.arr to [256 x i32] addrspace(4)*), i64 0, i64 8), align 4
diff --git a/llvm/test/CodeGen/AMDGPU/addrspacecast-initializer.ll b/llvm/test/CodeGen/AMDGPU/addrspacecast-initializer.ll
new file mode 100644
index 0000000000000..4f5082f9bd08a
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/addrspacecast-initializer.ll
@@ -0,0 +1,27 @@
+; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck %s
+
+; CHECK: global.arr:
+; CHECK: .zero 1024
+; CHECK: .size global.arr, 1024
+
+; CHECK: gv_flatptr_from_global:
+; CHECK: .quad global.arr+32
+; CHECK: .size gv_flatptr_from_global, 8
+
+; CHECK: gv_global_ptr:
+; CHECK: .quad global.arr+32
+; CHECK: .size gv_global_ptr, 8
+
+; CHECK: gv_flatptr_from_constant:
+; CHECK: .quad constant.arr+32
+; CHECK: .size	gv_flatptr_from_constant, 8
+
+@global.arr = unnamed_addr addrspace(1) global [256 x i32] undef, align 4
+@constant.arr = external unnamed_addr addrspace(4) global [256 x i32], align 4
+
+@gv_flatptr_from_global = unnamed_addr addrspace(4) global i32 addrspace(0)* getelementptr ([256 x i32], [256 x i32] addrspace(0)* addrspacecast ([256 x i32] addrspace(1)* @global.arr to [256 x i32] addrspace(0)*), i64 0, i64 8), align 4
+
+
+@gv_global_ptr = unnamed_addr addrspace(4) global i32 addrspace(1)* getelementptr ([256 x i32], [256 x i32] addrspace(1)* @global.arr, i64 0, i64 8), align 4
+
+@gv_flatptr_from_constant = unnamed_addr addrspace(4) global i32 addrspace(0)* getelementptr ([256 x i32], [256 x i32] addrspace(0)* addrspacecast ([256 x i32] addrspace(4)* @constant.arr to [256 x i32] addrspace(0)*), i64 0, i64 8), align 4

From dd5ea5674b86bade4904fab4c66a1156b3df033e Mon Sep 17 00:00:00 2001
From: Richard Barton <richard.barton@arm.com>
Date: Fri, 17 Jul 2020 09:15:21 +0100
Subject: [PATCH 0738/1035] [flang] Make interactive behaviour more obvious

When flang is invoked with no files it waits for input on stdin. Make it
print a message saying this to prevent the user being surprised.

Differential Revision: https://reviews.llvm.org/D84855
---
 flang/test/Driver/Inputs/hello.f90 |  3 +++
 flang/test/Driver/no_files.f90     | 10 ++++++++++
 flang/tools/f18/f18.cpp            |  2 ++
 3 files changed, 15 insertions(+)
 create mode 100644 flang/test/Driver/Inputs/hello.f90
 create mode 100644 flang/test/Driver/no_files.f90

diff --git a/flang/test/Driver/Inputs/hello.f90 b/flang/test/Driver/Inputs/hello.f90
new file mode 100644
index 0000000000000..d0c7eb94f53c8
--- /dev/null
+++ b/flang/test/Driver/Inputs/hello.f90
@@ -0,0 +1,3 @@
+program hello
+  write (*,*), "hello world" 
+end program hello
diff --git a/flang/test/Driver/no_files.f90 b/flang/test/Driver/no_files.f90
new file mode 100644
index 0000000000000..718985dce4ca5
--- /dev/null
+++ b/flang/test/Driver/no_files.f90
@@ -0,0 +1,10 @@
+! RUN: %f18 < %S/Inputs/hello.f90 | FileCheck %s
+
+
+! CHECK: Enter Fortran source
+! CHECK: Use EOF character (^D) to end file
+
+! CHECK: Parse tree comprises {{.*}} objects and occupies {{.*}} total bytes
+! CHECK: PROGRAM hello
+! CHECK:  WRITE (*, *) "hello world"
+! CHECK: END PROGRAM hello
diff --git a/flang/tools/f18/f18.cpp b/flang/tools/f18/f18.cpp
index dc5ddd12295f6..5df78a467f2b1 100644
--- a/flang/tools/f18/f18.cpp
+++ b/flang/tools/f18/f18.cpp
@@ -686,6 +686,8 @@ int main(int argc, char *const argv[]) {
   if (!anyFiles) {
     driver.measureTree = true;
     driver.dumpUnparse = true;
+    llvm::outs() << "Enter Fortran source\n"
+                 << "Use EOF character (^D) to end file\n";
     CompileFortran("-", options, driver, defaultKinds);
     return exitStatus;
   }

From b068d19a151d9d3a73b0265df27836d9fd0ad1e3 Mon Sep 17 00:00:00 2001
From: Richard Barton <richard.barton@arm.com>
Date: Thu, 25 Jun 2020 16:01:56 +0100
Subject: [PATCH 0739/1035] [flang] Add details to --help screen on default
 behaviour

Add a usage string and a defaults section that clarifies:
 * If no input files are given, f18 reads from stdin
 * If no input files are given, f18 dumps the parse tree.
 * The default behaviour is to exec F18_FC.
 * The fefault F18_FC setting is 'gfortran'

Adds a simple regression test which tests the top and tail of the help
screen and the exit status.

Depends on D84855

Differential Revision: https://reviews.llvm.org/D84856
---
 flang/test/Driver/help.f90 |  9 +++++++++
 flang/tools/f18/f18.cpp    | 16 +++++++++++++++-
 2 files changed, 24 insertions(+), 1 deletion(-)
 create mode 100644 flang/test/Driver/help.f90

diff --git a/flang/test/Driver/help.f90 b/flang/test/Driver/help.f90
new file mode 100644
index 0000000000000..66dd14aa5a86a
--- /dev/null
+++ b/flang/test/Driver/help.f90
@@ -0,0 +1,9 @@
+! RUN: %f18 -help 2>&1 | FileCheck %s
+! RUN: %f18 --help 2>&1 | FileCheck %s
+! RUN: %f18 -? 2>&1 | FileCheck %s
+
+! CHECK: f18: LLVM Fortran compiler
+
+! CHECK:   -help                print this again
+! CHECK: Unrecognised options are passed through to the external compiler
+! CHECK: set by F18_FC (see defaults).
diff --git a/flang/tools/f18/f18.cpp b/flang/tools/f18/f18.cpp
index 5df78a467f2b1..02bd5b26805ab 100644
--- a/flang/tools/f18/f18.cpp
+++ b/flang/tools/f18/f18.cpp
@@ -605,6 +605,19 @@ int main(int argc, char *const argv[]) {
       driver.byteswapio = true; // TODO: Pass to lowering, generate call
     } else if (arg == "-help" || arg == "--help" || arg == "-?") {
       llvm::errs()
+          << "f18: LLVM Fortran compiler\n"
+          << "\n"
+          << "Usage: f18 [options] <input files>\n"
+          << "\n"
+          << "Defaults:\n"
+          << "  When invoked with input files, and no options to tell\n"
+          << "  it otherwise, f18 will unparse its input and pass that on to an\n"
+          << "  external compiler to continue the compilation.\n"
+          << "  The external compiler is specified by the F18_FC environment\n"
+          << "  variable. The default is 'gfortran'.\n"
+          << "  If invoked with no input files, f18 reads source code from\n"
+          << "  stdin and runs with -fdebug-measure-parse-tree -funparse.\n"
+          << "\n"
           << "f18 options:\n"
           << "  -Mfixed | -Mfree | -ffixed-form | -ffree-form   force the "
              "source form\n"
@@ -638,7 +651,8 @@ int main(int argc, char *const argv[]) {
           << "  -fget-symbols-sources\n"
           << "  -v -c -o -I -D -U    have their usual meanings\n"
           << "  -help                print this again\n"
-          << "Other options are passed through to the compiler.\n";
+          << "Unrecognised options are passed through to the external compiler\n"
+          << "set by F18_FC (see defaults).\n";
       return exitStatus;
     } else if (arg == "-V") {
       llvm::errs() << "\nf18 compiler (under development)\n";

From 30e45f339eb0841dc7fe27fad119cc5db0c052f3 Mon Sep 17 00:00:00 2001
From: Richard Barton <richard.barton@arm.com>
Date: Sat, 18 Jul 2020 14:22:18 +0100
Subject: [PATCH 0740/1035] [flang] Add -h as a synonym for help

As expected by user in http://lists.llvm.org/pipermail/flang-dev/2020-June/000404.html

Depends on D84856

Differential Revision: https://reviews.llvm.org/D84857
---
 flang/test/Driver/help.f90 | 1 +
 flang/tools/f18/f18.cpp    | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/flang/test/Driver/help.f90 b/flang/test/Driver/help.f90
index 66dd14aa5a86a..d6162954a8723 100644
--- a/flang/test/Driver/help.f90
+++ b/flang/test/Driver/help.f90
@@ -1,3 +1,4 @@
+! RUN: %f18 -h 2>&1 | FileCheck %s
 ! RUN: %f18 -help 2>&1 | FileCheck %s
 ! RUN: %f18 --help 2>&1 | FileCheck %s
 ! RUN: %f18 -? 2>&1 | FileCheck %s
diff --git a/flang/tools/f18/f18.cpp b/flang/tools/f18/f18.cpp
index 02bd5b26805ab..338d04e7e8f5c 100644
--- a/flang/tools/f18/f18.cpp
+++ b/flang/tools/f18/f18.cpp
@@ -603,7 +603,7 @@ int main(int argc, char *const argv[]) {
       driver.getSymbolsSources = true;
     } else if (arg == "-byteswapio") {
       driver.byteswapio = true; // TODO: Pass to lowering, generate call
-    } else if (arg == "-help" || arg == "--help" || arg == "-?") {
+    } else if (arg == "-h" || arg == "-help" || arg == "--help" || arg == "-?") {
       llvm::errs()
           << "f18: LLVM Fortran compiler\n"
           << "\n"

From cfb955ac370cb724c51423a05694aaf5b70903a4 Mon Sep 17 00:00:00 2001
From: Thomas Raoux <thomasraoux@google.com>
Date: Fri, 31 Jul 2020 08:02:21 -0700
Subject: [PATCH 0741/1035] [mlir][spirv] Relax restriction on pointer type for
 CooperativeMatrix load/store

This change allow CooperativeMatrix Load/Store operations to use pointer type
that may not match the matrix element type. This allow us to declare buffer
with a larger type size than the matrix element type. This follows SPIR-V spec
and this is needed to be able to use cooperative matrix in combination with
shared local memory efficiently.

Differential Revision: https://reviews.llvm.org/D84993
---
 .../SPIRV/SPIRVCooperativeMatrixOps.td        | 16 ++---
 mlir/lib/Dialect/SPIRV/SPIRVOps.cpp           | 61 ++++++++-----------
 .../Serialization/cooperative-matrix.mlir     | 16 ++---
 .../Dialect/SPIRV/cooperative-matrix.mlir     | 38 +++++++++---
 4 files changed, 74 insertions(+), 57 deletions(-)

diff --git a/mlir/include/mlir/Dialect/SPIRV/SPIRVCooperativeMatrixOps.td b/mlir/include/mlir/Dialect/SPIRV/SPIRVCooperativeMatrixOps.td
index 9c3462a2e5bf1..720cfd697c24e 100644
--- a/mlir/include/mlir/Dialect/SPIRV/SPIRVCooperativeMatrixOps.td
+++ b/mlir/include/mlir/Dialect/SPIRV/SPIRVCooperativeMatrixOps.td
@@ -101,16 +101,17 @@ def SPV_CooperativeMatrixLoadNVOp : SPV_Op<"CooperativeMatrixLoadNV", []> {
 
     ``` {.ebnf}
     cooperative-matrixload-op ::= ssa-id `=` `spv.CooperativeMatrixLoadNV`
-                              storage-class ssa-use `,` ssa-use `,` ssa-use
+                              ssa-use `,` ssa-use `,` ssa-use
                               (`[` memory-access `]`)? ` : `
+                              pointer-type `as`
                               cooperative-matrix-type
     ```
 
     For example:
 
     ```
-    %0 = spv.CooperativeMatrixLoadNV "StorageBuffer" %ptr, %stride, %colMajor
-         : !spv.coopmatrix<i32, Workgroup, 16, 8>
+    %0 = spv.CooperativeMatrixLoadNV %ptr, %stride, %colMajor
+         : !spv.ptr<i32, StorageBuffer> as !spv.coopmatrix<i32, Workgroup, 16, 8>
     ```
   }];
 
@@ -243,16 +244,17 @@ def SPV_CooperativeMatrixStoreNVOp : SPV_Op<"CooperativeMatrixStoreNV", []> {
 
     ``` {.ebnf}
     coop-matrix-store-op ::= `spv.CooperativeMatrixStoreNV `
-                              storage-class ssa-use `, ` ssa-use `, `
                               ssa-use `, ` ssa-use `, `
-                  (`[` memory-access `]`)? `:` spirv-element-type
+                              ssa-use `, ` ssa-use `, `
+                              (`[` memory-access `]`)? `:`
+                              pointer-type `,` spirv-element-type
     ```
 
     For example:
 
     ```
-      spv.CooperativeMatrixStoreNV "StorageBuffer" %arg0, %arg2, %arg1, %arg3 :
-        !spv.coopmatrix<Workgroup, i32, 16, 8>
+      spv.CooperativeMatrixStoreNV %arg0, %arg2, %arg1, %arg3 :
+        !spv.ptr<i32, StorageBuffer>, !spv.coopmatrix<Workgroup, i32, 16, 8>
     ```
   }];
 
diff --git a/mlir/lib/Dialect/SPIRV/SPIRVOps.cpp b/mlir/lib/Dialect/SPIRV/SPIRVOps.cpp
index b0235d419ebe4..bac65a02f63de 100644
--- a/mlir/lib/Dialect/SPIRV/SPIRVOps.cpp
+++ b/mlir/lib/Dialect/SPIRV/SPIRVOps.cpp
@@ -2793,21 +2793,16 @@ static LogicalResult verify(spirv::VariableOp varOp) {
 
 static ParseResult parseCooperativeMatrixLoadNVOp(OpAsmParser &parser,
                                                   OperationState &state) {
-  spirv::StorageClass storageClass;
   SmallVector<OpAsmParser::OperandType, 3> operandInfo;
   Type strideType = parser.getBuilder().getIntegerType(32);
   Type columnMajorType = parser.getBuilder().getIntegerType(1);
+  Type ptrType;
   Type elementType;
-  if (parseEnumStrAttr(storageClass, parser) ||
-      parser.parseOperandList(operandInfo, 3) ||
+  if (parser.parseOperandList(operandInfo, 3) ||
       parseMemoryAccessAttributes(parser, state) || parser.parseColon() ||
-      parser.parseType(elementType)) {
+      parser.parseType(ptrType) || parser.parseKeywordType("as", elementType)) {
     return failure();
   }
-
-  auto ptrType = spirv::PointerType::get(
-      elementType.cast<spirv::CooperativeMatrixNVType>().getElementType(),
-      storageClass);
   SmallVector<Type, 3> OperandType = {ptrType, strideType, columnMajorType};
   if (parser.resolveOperands(operandInfo, OperandType, parser.getNameLoc(),
                              state.operands)) {
@@ -2819,25 +2814,30 @@ static ParseResult parseCooperativeMatrixLoadNVOp(OpAsmParser &parser,
 }
 
 static void print(spirv::CooperativeMatrixLoadNVOp M, OpAsmPrinter &printer) {
-  StringRef sc = stringifyStorageClass(
-      M.pointer().getType().cast<spirv::PointerType>().getStorageClass());
-  printer << spirv::CooperativeMatrixLoadNVOp::getOperationName() << " \"" << sc
-          << "\" " << M.pointer() << ", " << M.stride() << ", "
-          << M.columnmajor();
+  printer << spirv::CooperativeMatrixLoadNVOp::getOperationName() << " "
+          << M.pointer() << ", " << M.stride() << ", " << M.columnmajor();
   // Print optional memory access attribute.
   if (auto memAccess = M.memory_access())
     printer << " [\"" << stringifyMemoryAccess(*memAccess) << "\"]";
-  printer << " : " << M.getType();
+  printer << " : " << M.pointer().getType() << " as " << M.getType();
 }
 
 static LogicalResult verifyPointerAndCoopMatrixType(Operation *op, Type pointer,
                                                     Type coopMatrix) {
-  if (pointer.cast<spirv::PointerType>().getPointeeType() !=
-      coopMatrix.cast<spirv::CooperativeMatrixNVType>().getElementType())
+  Type pointeeType = pointer.cast<spirv::PointerType>().getPointeeType();
+  if (!pointeeType.isa<spirv::ScalarType>() && !pointeeType.isa<VectorType>())
     return op->emitError(
-               "expected the same type for pointer and the cooperative matrix"
-               "element, bu provided ")
-           << pointer << " and " << coopMatrix;
+               "Pointer must point to a scalar or vector type but provided ")
+           << pointeeType;
+  spirv::StorageClass storage =
+      pointer.cast<spirv::PointerType>().getStorageClass();
+  if (storage != spirv::StorageClass::Workgroup &&
+      storage != spirv::StorageClass::StorageBuffer &&
+      storage != spirv::StorageClass::PhysicalStorageBuffer)
+    return op->emitError(
+               "Pointer storage class must be Workgroup, StorageBuffer or "
+               "PhysicalStorageBufferEXT but provided ")
+           << stringifyStorageClass(storage);
   return success();
 }
 
@@ -2847,21 +2847,17 @@ static LogicalResult verifyPointerAndCoopMatrixType(Operation *op, Type pointer,
 
 static ParseResult parseCooperativeMatrixStoreNVOp(OpAsmParser &parser,
                                                    OperationState &state) {
-  spirv::StorageClass storageClass;
   SmallVector<OpAsmParser::OperandType, 4> operandInfo;
   Type strideType = parser.getBuilder().getIntegerType(32);
   Type columnMajorType = parser.getBuilder().getIntegerType(1);
+  Type ptrType;
   Type elementType;
-  if (parseEnumStrAttr(storageClass, parser) ||
-      parser.parseOperandList(operandInfo, 4) ||
+  if (parser.parseOperandList(operandInfo, 4) ||
       parseMemoryAccessAttributes(parser, state) || parser.parseColon() ||
+      parser.parseType(ptrType) || parser.parseComma() ||
       parser.parseType(elementType)) {
     return failure();
   }
-
-  auto ptrType = spirv::PointerType::get(
-      elementType.cast<spirv::CooperativeMatrixNVType>().getElementType(),
-      storageClass);
   SmallVector<Type, 4> OperandType = {ptrType, elementType, strideType,
                                       columnMajorType};
   if (parser.resolveOperands(operandInfo, OperandType, parser.getNameLoc(),
@@ -2874,17 +2870,14 @@ static ParseResult parseCooperativeMatrixStoreNVOp(OpAsmParser &parser,
 
 static void print(spirv::CooperativeMatrixStoreNVOp coopMatrix,
                   OpAsmPrinter &printer) {
-  StringRef sc = stringifyStorageClass(coopMatrix.pointer()
-                                           .getType()
-                                           .cast<spirv::PointerType>()
-                                           .getStorageClass());
-  printer << spirv::CooperativeMatrixStoreNVOp::getOperationName() << " \""
-          << sc << "\" " << coopMatrix.pointer() << ", " << coopMatrix.object()
-          << ", " << coopMatrix.stride() << ", " << coopMatrix.columnmajor();
+  printer << spirv::CooperativeMatrixStoreNVOp::getOperationName() << " "
+          << coopMatrix.pointer() << ", " << coopMatrix.object() << ", "
+          << coopMatrix.stride() << ", " << coopMatrix.columnmajor();
   // Print optional memory access attribute.
   if (auto memAccess = coopMatrix.memory_access())
     printer << " [\"" << stringifyMemoryAccess(*memAccess) << "\"]";
-  printer << " : " << coopMatrix.getOperand(1).getType();
+  printer << " : " << coopMatrix.pointer().getType() << ", "
+          << coopMatrix.getOperand(1).getType();
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/test/Dialect/SPIRV/Serialization/cooperative-matrix.mlir b/mlir/test/Dialect/SPIRV/Serialization/cooperative-matrix.mlir
index ad913dfb1624c..3f8c4bff47387 100644
--- a/mlir/test/Dialect/SPIRV/Serialization/cooperative-matrix.mlir
+++ b/mlir/test/Dialect/SPIRV/Serialization/cooperative-matrix.mlir
@@ -3,29 +3,29 @@
 spv.module Logical GLSL450 requires #spv.vce<v1.0, [CooperativeMatrixNV], [SPV_NV_cooperative_matrix]> {
   // CHECK-LABEL: @cooperative_matrix_load
   spv.func @cooperative_matrix_load(%ptr : !spv.ptr<i32, StorageBuffer>, %stride : i32, %b : i1) "None" {
-    // CHECK: {{%.*}} = spv.CooperativeMatrixLoadNV "StorageBuffer" {{%.*}}, {{%.*}}, {{%.*}} : !spv.coopmatrix<16x8xi32, Workgroup>
-    %0 = spv.CooperativeMatrixLoadNV "StorageBuffer" %ptr, %stride, %b : !spv.coopmatrix<16x8xi32, Workgroup>
+    // CHECK: {{%.*}} = spv.CooperativeMatrixLoadNV {{%.*}}, {{%.*}}, {{%.*}} : !spv.ptr<i32, StorageBuffer> as !spv.coopmatrix<16x8xi32, Workgroup>
+    %0 = spv.CooperativeMatrixLoadNV %ptr, %stride, %b : !spv.ptr<i32, StorageBuffer> as !spv.coopmatrix<16x8xi32, Workgroup>
     spv.Return
   }
 
   // CHECK-LABEL: @cooperative_matrix_load_memaccess
   spv.func @cooperative_matrix_load_memaccess(%ptr : !spv.ptr<i32, StorageBuffer>, %stride : i32, %b : i1) "None" {
-    // CHECK: {{%.*}} = spv.CooperativeMatrixLoadNV "StorageBuffer" {{%.*}}, {{%.*}}, {{%.*}} ["Volatile"] : !spv.coopmatrix<8x16xi32, Subgroup>
-    %0 = spv.CooperativeMatrixLoadNV "StorageBuffer" %ptr, %stride, %b ["Volatile"] : !spv.coopmatrix<8x16xi32, Subgroup>
+    // CHECK: {{%.*}} = spv.CooperativeMatrixLoadNV {{%.*}}, {{%.*}}, {{%.*}} ["Volatile"] : !spv.ptr<i32, StorageBuffer> as !spv.coopmatrix<8x16xi32, Subgroup>
+    %0 = spv.CooperativeMatrixLoadNV %ptr, %stride, %b ["Volatile"] : !spv.ptr<i32, StorageBuffer> as !spv.coopmatrix<8x16xi32, Subgroup>
     spv.Return
   }
 
   // CHECK-LABEL: @cooperative_matrix_store
   spv.func @cooperative_matrix_store(%ptr : !spv.ptr<i32, StorageBuffer>, %stride : i32, %m : !spv.coopmatrix<16x8xi32, Workgroup>, %b : i1) "None" {
-    // CHECK: spv.CooperativeMatrixStoreNV "StorageBuffer" {{%.*}}, {{%.*}}, {{%.*}} : !spv.coopmatrix<16x8xi32, Workgroup>
-    spv.CooperativeMatrixStoreNV "StorageBuffer" %ptr, %m, %stride, %b : !spv.coopmatrix<16x8xi32, Workgroup>
+    // CHECK: spv.CooperativeMatrixStoreNV {{%.*}}, {{%.*}}, {{%.*}} : !spv.ptr<i32, StorageBuffer>, !spv.coopmatrix<16x8xi32, Workgroup>
+    spv.CooperativeMatrixStoreNV %ptr, %m, %stride, %b : !spv.ptr<i32, StorageBuffer>, !spv.coopmatrix<16x8xi32, Workgroup>
     spv.Return
   }
 
   // CHECK-LABEL: @cooperative_matrix_store_memaccess
   spv.func @cooperative_matrix_store_memaccess(%ptr : !spv.ptr<i32, StorageBuffer>, %m : !spv.coopmatrix<8x16xi32, Subgroup>, %stride : i32, %b : i1) "None" {
-    // CHECK: spv.CooperativeMatrixStoreNV "StorageBuffer" {{%.*}}, {{%.*}}, {{%.*}} ["Volatile"] : !spv.coopmatrix<8x16xi32, Subgroup>
-    spv.CooperativeMatrixStoreNV "StorageBuffer" %ptr, %m, %stride, %b ["Volatile"] : !spv.coopmatrix<8x16xi32, Subgroup>
+    // CHECK: spv.CooperativeMatrixStoreNV {{%.*}}, {{%.*}}, {{%.*}} ["Volatile"] : !spv.ptr<i32, StorageBuffer>, !spv.coopmatrix<8x16xi32, Subgroup>
+    spv.CooperativeMatrixStoreNV %ptr, %m, %stride, %b ["Volatile"] : !spv.ptr<i32, StorageBuffer>, !spv.coopmatrix<8x16xi32, Subgroup>
     spv.Return
   }
 
diff --git a/mlir/test/Dialect/SPIRV/cooperative-matrix.mlir b/mlir/test/Dialect/SPIRV/cooperative-matrix.mlir
index 523bd6bfb0309..f0bb50d10f58e 100644
--- a/mlir/test/Dialect/SPIRV/cooperative-matrix.mlir
+++ b/mlir/test/Dialect/SPIRV/cooperative-matrix.mlir
@@ -2,30 +2,37 @@
 
 // CHECK-LABEL: @cooperative_matrix_load
 spv.func @cooperative_matrix_load(%ptr : !spv.ptr<i32, StorageBuffer>, %stride : i32, %b : i1) "None" {
-  // CHECK: {{%.*}} = spv.CooperativeMatrixLoadNV "StorageBuffer" {{%.*}}, {{%.*}}, {{%.*}} : !spv.coopmatrix<16x8xi32, Workgroup>
-  %0 = spv.CooperativeMatrixLoadNV "StorageBuffer" %ptr, %stride, %b : !spv.coopmatrix<16x8xi32, Workgroup>
+  // CHECK: {{%.*}} = spv.CooperativeMatrixLoadNV {{%.*}}, {{%.*}}, {{%.*}} : !spv.ptr<i32, StorageBuffer> as !spv.coopmatrix<16x8xi32, Workgroup>
+  %0 = spv.CooperativeMatrixLoadNV %ptr, %stride, %b : !spv.ptr<i32, StorageBuffer> as !spv.coopmatrix<16x8xi32, Workgroup>
   spv.Return
 }
 
 // -----
 // CHECK-LABEL: @cooperative_matrix_load_memaccess
 spv.func @cooperative_matrix_load_memaccess(%ptr : !spv.ptr<i32, StorageBuffer>, %stride : i32, %b : i1) "None" {
-  // CHECK: {{%.*}} = spv.CooperativeMatrixLoadNV "StorageBuffer" {{%.*}}, {{%.*}}, {{%.*}} ["Volatile"] : !spv.coopmatrix<8x16xi32, Subgroup>
-  %0 = spv.CooperativeMatrixLoadNV "StorageBuffer" %ptr, %stride, %b ["Volatile"] : !spv.coopmatrix<8x16xi32, Subgroup>
+  // CHECK: {{%.*}} = spv.CooperativeMatrixLoadNV {{%.*}}, {{%.*}}, {{%.*}} ["Volatile"] : !spv.ptr<i32, StorageBuffer> as !spv.coopmatrix<8x16xi32, Subgroup>
+  %0 = spv.CooperativeMatrixLoadNV %ptr, %stride, %b ["Volatile"] : !spv.ptr<i32, StorageBuffer> as !spv.coopmatrix<8x16xi32, Subgroup>
+  spv.Return
+}
+
+// CHECK-LABEL: @cooperative_matrix_load_diff_ptr_type
+spv.func @cooperative_matrix_load_diff_ptr_type(%ptr : !spv.ptr<vector<4xi32>, StorageBuffer>, %stride : i32, %b : i1) "None" {
+  // CHECK: {{%.*}} = spv.CooperativeMatrixLoadNV {{%.*}}, {{%.*}}, {{%.*}} ["Volatile"] : !spv.ptr<vector<4xi32>, StorageBuffer> as !spv.coopmatrix<8x16xi32, Subgroup>
+  %0 = spv.CooperativeMatrixLoadNV %ptr, %stride, %b ["Volatile"] : !spv.ptr<vector<4xi32>, StorageBuffer> as !spv.coopmatrix<8x16xi32, Subgroup>
   spv.Return
 }
 
 // CHECK-LABEL: @cooperative_matrix_store
 spv.func @cooperative_matrix_store(%ptr : !spv.ptr<i32, StorageBuffer>, %stride : i32, %m : !spv.coopmatrix<8x16xi32, Workgroup>, %b : i1) "None" {
-  // CHECK: spv.CooperativeMatrixStoreNV "StorageBuffer" {{%.*}}, {{%.*}}, {{%.*}} : !spv.coopmatrix<8x16xi32, Workgroup>
-  spv.CooperativeMatrixStoreNV "StorageBuffer" %ptr, %m, %stride, %b : !spv.coopmatrix<8x16xi32, Workgroup>
+  // CHECK: spv.CooperativeMatrixStoreNV {{%.*}}, {{%.*}}, {{%.*}} : !spv.ptr<i32, StorageBuffer>, !spv.coopmatrix<8x16xi32, Workgroup>
+  spv.CooperativeMatrixStoreNV %ptr, %m, %stride, %b : !spv.ptr<i32, StorageBuffer>, !spv.coopmatrix<8x16xi32, Workgroup>
   spv.Return
 }
 
 // CHECK-LABEL: @cooperative_matrix_store_memaccess
 spv.func @cooperative_matrix_store_memaccess(%ptr : !spv.ptr<i32, StorageBuffer>, %m : !spv.coopmatrix<8x16xi32, Subgroup>, %stride : i32, %b : i1) "None" {
-  // CHECK: spv.CooperativeMatrixStoreNV "StorageBuffer" {{%.*}}, {{%.*}}, {{%.*}} ["Volatile"] : !spv.coopmatrix<8x16xi32, Subgroup>
-  spv.CooperativeMatrixStoreNV "StorageBuffer" %ptr, %m, %stride, %b ["Volatile"] : !spv.coopmatrix<8x16xi32, Subgroup>
+  // CHECK: spv.CooperativeMatrixStoreNV {{%.*}}, {{%.*}}, {{%.*}} ["Volatile"] : !spv.ptr<i32, StorageBuffer>, !spv.coopmatrix<8x16xi32, Subgroup>
+  spv.CooperativeMatrixStoreNV %ptr, %m, %stride, %b ["Volatile"] : !spv.ptr<i32, StorageBuffer>, !spv.coopmatrix<8x16xi32, Subgroup>
   spv.Return
 }
 
@@ -134,3 +141,18 @@ spv.func @cooperative_matrix_muladd(%a : !spv.coopmatrix<8x16xf32, Subgroup>, %b
   spv.Return
 }
 
+// -----
+
+spv.func @cooperative_matrix_load_memaccess(%ptr : !spv.ptr<!spv.struct<f32 [0]>, StorageBuffer>, %stride : i32, %b : i1) "None" {
+  // expected-error @+1 {{Pointer must point to a scalar or vector type}}
+  %0 = spv.CooperativeMatrixLoadNV %ptr, %stride, %b : !spv.ptr<!spv.struct<f32 [0]>, StorageBuffer> as !spv.coopmatrix<8x16xi32, Subgroup>
+  spv.Return
+}
+
+// -----
+
+spv.func @cooperative_matrix_load_memaccess(%ptr : !spv.ptr<i32, Function>, %stride : i32, %b : i1) "None" {
+  // expected-error @+1 {{Pointer storage class must be Workgroup, StorageBuffer or PhysicalStorageBufferEXT}}
+  %0 = spv.CooperativeMatrixLoadNV %ptr, %stride, %b : !spv.ptr<i32, Function> as !spv.coopmatrix<8x16xi32, Subgroup>
+  spv.Return
+}

From d275da17e4f0a17615b24c352aab0d34f647bfa7 Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne@apple.com>
Date: Fri, 31 Jul 2020 11:18:01 -0400
Subject: [PATCH 0742/1035] [libc++] Fix eager generator expression in
 DefineLinkerScript

As explained in https://gitlab.kitware.com/cmake/cmake/-/issues/21045,
both branches of an $<IF> generator expression are evaluated eagerly
by CMake. As a result, if the non-selected branch contains an invalid
generator expression (such as getting the OUTPUT_NAME property of a
non-existent target), a hard error will occur.

This failed builds using the cxxrt ABI library, which doesn't create
a CMake target currently.
---
 libcxx/cmake/Modules/DefineLinkerScript.cmake | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/libcxx/cmake/Modules/DefineLinkerScript.cmake b/libcxx/cmake/Modules/DefineLinkerScript.cmake
index 41426bf067149..be7f026af7e87 100644
--- a/libcxx/cmake/Modules/DefineLinkerScript.cmake
+++ b/libcxx/cmake/Modules/DefineLinkerScript.cmake
@@ -34,7 +34,13 @@ function(define_linker_script target)
       if ("${lib}" STREQUAL "cxx-headers")
         continue()
       endif()
-      set(libname "$<IF:$<TARGET_EXISTS:${lib}>,$<TARGET_PROPERTY:${lib},OUTPUT_NAME>,${lib}>")
+      # If ${lib} is not a target, we use a dummy target which we know will
+      # have an OUTPUT_NAME property so that CMake doesn't fail when evaluating
+      # the non-selected branch of the `IF`. It doesn't matter what it evaluates
+      # to because it's not selected, but it must not cause an error.
+      # See https://gitlab.kitware.com/cmake/cmake/-/issues/21045.
+      set(output_name_tgt "$<IF:$<TARGET_EXISTS:${lib}>,${lib},${target}>")
+      set(libname "$<IF:$<TARGET_EXISTS:${lib}>,$<TARGET_PROPERTY:${output_name_tgt},OUTPUT_NAME>,${lib}>")
       list(APPEND link_libraries "${CMAKE_LINK_LIBRARY_FLAG}${libname}")
     endforeach()
   endif()

From 9853786ce39b9510eeb2688baaef7a364d58e113 Mon Sep 17 00:00:00 2001
From: Hans Wennborg <hans@chromium.org>
Date: Fri, 31 Jul 2020 17:22:49 +0200
Subject: [PATCH 0743/1035] Add flang to export.sh to it gets source tarballs
 in releases

---
 llvm/utils/release/export.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/utils/release/export.sh b/llvm/utils/release/export.sh
index 02a77afd0533f..c3277de38b53a 100755
--- a/llvm/utils/release/export.sh
+++ b/llvm/utils/release/export.sh
@@ -13,7 +13,7 @@
 
 set -e
 
-projects="llvm clang test-suite compiler-rt libcxx libcxxabi clang-tools-extra polly lldb lld openmp libunwind"
+projects="llvm clang test-suite compiler-rt libcxx libcxxabi clang-tools-extra polly lldb lld openmp libunwind flang"
 
 release=""
 rc=""

From c6f08b14d4895928232fac38d266bb53aafa6b29 Mon Sep 17 00:00:00 2001
From: Benjamin Kramer <benny.kra@googlemail.com>
Date: Fri, 31 Jul 2020 17:27:44 +0200
Subject: [PATCH 0744/1035] Hide some internal symbols. NFC.

---
 llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp     | 8 ++++----
 llvm/lib/Target/ARM/MVEGatherScatterLowering.cpp          | 7 +++----
 llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp | 2 ++
 3 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
index 58f251fec5e98..372c5154acef0 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
@@ -723,10 +723,10 @@ GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
 ///
 /// Note: This only supports non-TFE/LWE image intrinsic calls; those have
 ///       struct returns.
-Value *simplifyAMDGCNMemoryIntrinsicDemanded(InstCombiner &IC,
-                                             IntrinsicInst &II,
-                                             APInt DemandedElts,
-                                             int DMaskIdx = -1) {
+static Value *simplifyAMDGCNMemoryIntrinsicDemanded(InstCombiner &IC,
+                                                    IntrinsicInst &II,
+                                                    APInt DemandedElts,
+                                                    int DMaskIdx = -1) {
 
   auto *IIVTy = cast<VectorType>(II.getType());
   unsigned VWidth = IIVTy->getNumElements();
diff --git a/llvm/lib/Target/ARM/MVEGatherScatterLowering.cpp b/llvm/lib/Target/ARM/MVEGatherScatterLowering.cpp
index 2746d4d456e4f..d037fe7537d20 100644
--- a/llvm/lib/Target/ARM/MVEGatherScatterLowering.cpp
+++ b/llvm/lib/Target/ARM/MVEGatherScatterLowering.cpp
@@ -172,7 +172,7 @@ bool MVEGatherScatterLowering::isLegalTypeAndAlignment(unsigned NumElements,
   return false;
 }
 
-bool checkOffsetSize(Value *Offsets, unsigned TargetElemCount) {
+static bool checkOffsetSize(Value *Offsets, unsigned TargetElemCount) {
   // Offsets that are not of type <N x i32> are sign extended by the
   // getelementptr instruction, and MVE gathers/scatters treat the offset as
   // unsigned. Thus, if the element size is smaller than 32, we can only allow
@@ -1030,9 +1030,8 @@ bool MVEGatherScatterLowering::optimiseOffsets(Value *Offsets, BasicBlock *BB,
   return true;
 }
 
-Value *CheckAndCreateOffsetAdd(Value *X, Value *Y, Value *GEP,
-                               IRBuilder<> &Builder) {
-
+static Value *CheckAndCreateOffsetAdd(Value *X, Value *Y, Value *GEP,
+                                      IRBuilder<> &Builder) {
   // Splat the non-vector value to a vector of the given type - if the value is
   // a constant (and its value isn't too big), we can even use this opportunity
   // to scale it to the size of the vector elements
diff --git a/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
index 0af71e9b72b83..523e7b19ecb1d 100644
--- a/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
@@ -1859,6 +1859,7 @@ void DFSanVisitor::visitPHINode(PHINode &PN) {
   DFSF.setShadow(&PN, ShadowPN);
 }
 
+namespace {
 class DataFlowSanitizerLegacyPass : public ModulePass {
 private:
   std::vector<std::string> ABIListFiles;
@@ -1874,6 +1875,7 @@ class DataFlowSanitizerLegacyPass : public ModulePass {
     return DataFlowSanitizer(ABIListFiles).runImpl(M);
   }
 };
+} // namespace
 
 char DataFlowSanitizerLegacyPass::ID;
 

From 7ad6ea520fe49e9320bd15a4daf88e36259efedc Mon Sep 17 00:00:00 2001
From: Xing GUO <higuoxing@gmail.com>
Date: Fri, 31 Jul 2020 23:20:44 +0800
Subject: [PATCH 0745/1035] [DWARFYAML][debug_aranges] Use yaml::Hex64 rather
 than uint64_t as length. NFC.

It's better to use yaml::Hex64 as length in the tuples of the address
range table.
---
 llvm/include/llvm/ObjectYAML/DWARFYAML.h            | 2 +-
 llvm/test/ObjectYAML/MachO/DWARF-debug_aranges.yaml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/llvm/include/llvm/ObjectYAML/DWARFYAML.h b/llvm/include/llvm/ObjectYAML/DWARFYAML.h
index 5737ceccc0a40..127a529139786 100644
--- a/llvm/include/llvm/ObjectYAML/DWARFYAML.h
+++ b/llvm/include/llvm/ObjectYAML/DWARFYAML.h
@@ -61,7 +61,7 @@ struct Abbrev {
 
 struct ARangeDescriptor {
   llvm::yaml::Hex64 Address;
-  uint64_t Length;
+  yaml::Hex64 Length;
 };
 
 struct ARange {
diff --git a/llvm/test/ObjectYAML/MachO/DWARF-debug_aranges.yaml b/llvm/test/ObjectYAML/MachO/DWARF-debug_aranges.yaml
index 1e9b880c3cd3d..762fcbfa2d22c 100644
--- a/llvm/test/ObjectYAML/MachO/DWARF-debug_aranges.yaml
+++ b/llvm/test/ObjectYAML/MachO/DWARF-debug_aranges.yaml
@@ -333,7 +333,7 @@ DWARF:
 # CHECK-NEXT:       AddressSize:         0x08
 # CHECK-NEXT:       Descriptors:
 # CHECK-NEXT:         - Address: 0x0000000100000F50
-# CHECK-NEXT:           Length:  52
+# CHECK-NEXT:           Length:  0x0000000000000034
 
 ## b) Test that if the "debug_aranges" entry is empty, yaml2macho will only emit the
 ## section header.

From c4e574323210feda1a3988e85fdd93b90a63d1b1 Mon Sep 17 00:00:00 2001
From: Amy Kwan <amy.kwan1@ibm.com>
Date: Mon, 27 Jul 2020 23:45:54 -0500
Subject: [PATCH 0746/1035] [PowerPC] Implement low-order Vector Modulus
 Builtins, and add Vector Multiply/Divide/Modulus Builtins Tests

Power10 introduces new instructions for vector multiply, divide and modulus.
These instructions can be exploited by the builtin functions: vec_mul, vec_div,
and vec_mod, respectively.

This patch aims adds the function prototype, vec_mod, as vec_mul and vec_div
been previously implemented in altivec.h.

This patch also adds the following front end tests:
vec_mul for v2i64
vec_div for v4i32 and v2i64
vec_mod for v4i32 and v2i64

Differential Revision: https://reviews.llvm.org/D82576
---
 clang/lib/Headers/altivec.h                 | 22 ++++++++
 clang/test/CodeGen/builtins-ppc-p10vector.c | 60 +++++++++++++++++++++
 2 files changed, 82 insertions(+)

diff --git a/clang/lib/Headers/altivec.h b/clang/lib/Headers/altivec.h
index 4e25ec118072b..f42200f5bd4e7 100644
--- a/clang/lib/Headers/altivec.h
+++ b/clang/lib/Headers/altivec.h
@@ -16933,6 +16933,28 @@ vec_cnttzm(vector unsigned long long __a, vector unsigned long long __b) {
   return __builtin_altivec_vctzdm(__a, __b);
 }
 
+/* vec_mod */
+
+static __inline__ vector signed int __ATTRS_o_ai
+vec_mod(vector signed int __a, vector signed int __b) {
+  return __a % __b;
+}
+
+static __inline__ vector unsigned int __ATTRS_o_ai
+vec_mod(vector unsigned int __a, vector unsigned int __b) {
+  return __a % __b;
+}
+
+static __inline__ vector signed long long __ATTRS_o_ai
+vec_mod(vector signed long long __a, vector signed long long __b) {
+  return __a % __b;
+}
+
+static __inline__ vector unsigned long long __ATTRS_o_ai
+vec_mod(vector unsigned long long __a, vector unsigned long long __b) {
+  return __a % __b;
+}
+
 /* vec_sldbi */
 
 #define vec_sldb(__a, __b, __c) __builtin_altivec_vsldbi(__a, __b, (__c & 0x7))
diff --git a/clang/test/CodeGen/builtins-ppc-p10vector.c b/clang/test/CodeGen/builtins-ppc-p10vector.c
index e67018b062141..571d33d34a220 100644
--- a/clang/test/CodeGen/builtins-ppc-p10vector.c
+++ b/clang/test/CodeGen/builtins-ppc-p10vector.c
@@ -25,6 +25,66 @@ unsigned char uca;
 unsigned short usa;
 unsigned long long ulla;
 
+vector signed long long test_vec_mul_sll(void) {
+  // CHECK: mul <2 x i64>
+  // CHECK-NEXT: ret <2 x i64>
+  return vec_mul(vslla, vsllb);
+}
+
+vector unsigned long long test_vec_mul_ull(void) {
+  // CHECK: mul <2 x i64>
+  // CHECK-NEXT: ret <2 x i64>
+  return vec_mul(vulla, vullb);
+}
+
+vector signed int test_vec_div_si(void) {
+  // CHECK: sdiv <4 x i32>
+  // CHECK-NEXT: ret <4 x i32>
+  return vec_div(vsia, vsib);
+}
+
+vector unsigned int test_vec_div_ui(void) {
+  // CHECK: udiv <4 x i32>
+  // CHECK-NEXT: ret <4 x i32>
+  return vec_div(vuia, vuib);
+}
+
+vector signed long long test_vec_div_sll(void) {
+  // CHECK: sdiv <2 x i64>
+  // CHECK-NEXT: ret <2 x i64>
+  return vec_div(vslla, vsllb);
+}
+
+vector unsigned long long test_vec_div_ull(void) {
+  // CHECK: udiv <2 x i64>
+  // CHECK-NEXT: ret <2 x i64>
+  return vec_div(vulla, vullb);
+}
+
+vector signed int test_vec_mod_si(void) {
+  // CHECK: srem <4 x i32>
+  // CHECK-NEXT: ret <4 x i32>
+  return vec_mod(vsia, vsib);
+}
+
+vector unsigned int test_vec_mod_ui(void) {
+  // CHECK: urem <4 x i32>
+  // CHECK-NEXT: ret <4 x i32>
+  return vec_mod(vuia, vuib);
+}
+
+vector signed long long test_vec_mod_sll(void) {
+  // CHECK: srem <2 x i64>
+  // CHECK-NEXT: ret <2 x i64>
+  return vec_mod(vslla, vsllb);
+}
+
+vector unsigned long long test_vec_mod_ull(void) {
+  // CHECK: urem <2 x i64>
+  // CHECK-NEXT: ret <2 x i64>
+  return vec_mod(vulla, vullb);
+}
+
 vector unsigned long long test_vpdepd(void) {
   // CHECK: @llvm.ppc.altivec.vpdepd(<2 x i64>
   // CHECK-NEXT: ret <2 x i64>

From df69492cdfa82ff6453c887cd45b3a5903b79afb Mon Sep 17 00:00:00 2001
From: Sameer Arora <sameerarora101@fb.com>
Date: Thu, 23 Jul 2020 16:26:42 -0700
Subject: [PATCH 0747/1035] [llvm-libtool-darwin] Refactor Slice and
 writeUniversalBinary

Refactoring `Slice` class and function `createUniversalBinary` from
`llvm-lipo` into  MachOUniversalWriter. This refactoring is necessary so
as to use the refactored code for creating universal binaries under
llvm-libtool-darwin.

Reviewed by alexshap, smeenai

Differential Revision: https://reviews.llvm.org/D84662
---
 .../llvm/Object/MachOUniversalWriter.h        |  84 ++++++
 llvm/lib/Object/CMakeLists.txt                |   1 +
 llvm/lib/Object/MachOUniversalWriter.cpp      | 220 ++++++++++++++
 llvm/tools/llvm-lipo/llvm-lipo.cpp            | 274 +++---------------
 4 files changed, 343 insertions(+), 236 deletions(-)
 create mode 100644 llvm/include/llvm/Object/MachOUniversalWriter.h
 create mode 100644 llvm/lib/Object/MachOUniversalWriter.cpp

diff --git a/llvm/include/llvm/Object/MachOUniversalWriter.h b/llvm/include/llvm/Object/MachOUniversalWriter.h
new file mode 100644
index 0000000000000..c860495ddd6f0
--- /dev/null
+++ b/llvm/include/llvm/Object/MachOUniversalWriter.h
@@ -0,0 +1,84 @@
+//===- MachOUniversalWriter.h - MachO universal binary writer----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Declares the Slice class and writeUniversalBinary function for writing a
+// MachO universal binary file.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_OBJECT_MACHOUNIVERSALWRITER_H
+#define LLVM_OBJECT_MACHOUNIVERSALWRITER_H
+
+#include "llvm/Object/Archive.h"
+#include "llvm/Object/Binary.h"
+#include "llvm/Object/MachO.h"
+
+namespace llvm {
+namespace object {
+
+class Slice {
+  const Binary *B;
+  uint32_t CPUType;
+  uint32_t CPUSubType;
+  std::string ArchName;
+
+  // P2Alignment field stores slice alignment values from universal
+  // binaries. This is also needed to order the slices so the total
+  // file size can be calculated before creating the output buffer.
+  uint32_t P2Alignment;
+
+public:
+  explicit Slice(const MachOObjectFile &O);
+
+  Slice(const MachOObjectFile &O, uint32_t Align);
+
+  static Expected<Slice> create(const Archive *A);
+
+  void setP2Alignment(uint32_t Align) { P2Alignment = Align; }
+
+  const Binary *getBinary() const { return B; }
+
+  uint32_t getCPUType() const { return CPUType; }
+
+  uint32_t getCPUSubType() const { return CPUSubType; }
+
+  uint32_t getP2Alignment() const { return P2Alignment; }
+
+  uint64_t getCPUID() const {
+    return static_cast<uint64_t>(CPUType) << 32 | CPUSubType;
+  }
+
+  std::string getArchString() const {
+    if (!ArchName.empty())
+      return ArchName;
+    return ("unknown(" + Twine(CPUType) + "," +
+            Twine(CPUSubType & ~MachO::CPU_SUBTYPE_MASK) + ")")
+        .str();
+  }
+
+  friend bool operator<(const Slice &Lhs, const Slice &Rhs) {
+    if (Lhs.CPUType == Rhs.CPUType)
+      return Lhs.CPUSubType < Rhs.CPUSubType;
+    // force arm64-family to follow after all other slices for
+    // compatibility with cctools lipo
+    if (Lhs.CPUType == MachO::CPU_TYPE_ARM64)
+      return false;
+    if (Rhs.CPUType == MachO::CPU_TYPE_ARM64)
+      return true;
+    // Sort by alignment to minimize file size
+    return Lhs.P2Alignment < Rhs.P2Alignment;
+  }
+};
+
+Error writeUniversalBinary(ArrayRef<Slice> Slices, StringRef OutputFileName);
+
+} // end namespace object
+
+} // end namespace llvm
+
+#endif // LLVM_OBJECT_MACHOUNIVERSALWRITER_H
diff --git a/llvm/lib/Object/CMakeLists.txt b/llvm/lib/Object/CMakeLists.txt
index 61888cbe46f73..9f912ccdff1f1 100644
--- a/llvm/lib/Object/CMakeLists.txt
+++ b/llvm/lib/Object/CMakeLists.txt
@@ -23,6 +23,7 @@ add_llvm_component_library(LLVMObject
   SymbolSize.cpp
   TapiFile.cpp
   TapiUniversal.cpp
+  MachOUniversalWriter.cpp
   WasmObjectFile.cpp
   WindowsMachineFlag.cpp
   WindowsResource.cpp
diff --git a/llvm/lib/Object/MachOUniversalWriter.cpp b/llvm/lib/Object/MachOUniversalWriter.cpp
new file mode 100644
index 0000000000000..169d64430284e
--- /dev/null
+++ b/llvm/lib/Object/MachOUniversalWriter.cpp
@@ -0,0 +1,220 @@
+//===- MachOUniversalWriter.cpp - MachO universal binary writer---*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Defines the Slice class and writeUniversalBinary function for writing a MachO
+// universal binary file.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Object/MachOUniversalWriter.h"
+#include "llvm/Object/Archive.h"
+#include "llvm/Object/Binary.h"
+#include "llvm/Object/Error.h"
+#include "llvm/Object/MachO.h"
+#include "llvm/Object/MachOUniversal.h"
+#include "llvm/Support/FileOutputBuffer.h"
+
+using namespace llvm;
+using namespace object;
+
+// For compatibility with cctools lipo, a file's alignment is calculated as the
+// minimum aligment of all segments. For object files, the file's alignment is
+// the maximum alignment of its sections.
+static uint32_t calculateFileAlignment(const MachOObjectFile &O) {
+  uint32_t P2CurrentAlignment;
+  uint32_t P2MinAlignment = MachOUniversalBinary::MaxSectionAlignment;
+  const bool Is64Bit = O.is64Bit();
+
+  for (const auto &LC : O.load_commands()) {
+    if (LC.C.cmd != (Is64Bit ? MachO::LC_SEGMENT_64 : MachO::LC_SEGMENT))
+      continue;
+    if (O.getHeader().filetype == MachO::MH_OBJECT) {
+      unsigned NumberOfSections =
+          (Is64Bit ? O.getSegment64LoadCommand(LC).nsects
+                   : O.getSegmentLoadCommand(LC).nsects);
+      P2CurrentAlignment = NumberOfSections ? 2 : P2MinAlignment;
+      for (unsigned SI = 0; SI < NumberOfSections; ++SI) {
+        P2CurrentAlignment = std::max(P2CurrentAlignment,
+                                      (Is64Bit ? O.getSection64(LC, SI).align
+                                               : O.getSection(LC, SI).align));
+      }
+    } else {
+      P2CurrentAlignment =
+          countTrailingZeros(Is64Bit ? O.getSegment64LoadCommand(LC).vmaddr
+                                     : O.getSegmentLoadCommand(LC).vmaddr);
+    }
+    P2MinAlignment = std::min(P2MinAlignment, P2CurrentAlignment);
+  }
+  // return a value >= 4 byte aligned, and less than MachO MaxSectionAlignment
+  return std::max(
+      static_cast<uint32_t>(2),
+      std::min(P2MinAlignment, static_cast<uint32_t>(
+                                   MachOUniversalBinary::MaxSectionAlignment)));
+}
+
+static uint32_t calculateAlignment(const MachOObjectFile &ObjectFile) {
+  switch (ObjectFile.getHeader().cputype) {
+  case MachO::CPU_TYPE_I386:
+  case MachO::CPU_TYPE_X86_64:
+  case MachO::CPU_TYPE_POWERPC:
+  case MachO::CPU_TYPE_POWERPC64:
+    return 12; // log2 value of page size(4k) for x86 and PPC
+  case MachO::CPU_TYPE_ARM:
+  case MachO::CPU_TYPE_ARM64:
+  case MachO::CPU_TYPE_ARM64_32:
+    return 14; // log2 value of page size(16k) for Darwin ARM
+  default:
+    return calculateFileAlignment(ObjectFile);
+  }
+}
+
+Slice::Slice(const MachOObjectFile &O, uint32_t Align)
+    : B(&O), CPUType(O.getHeader().cputype),
+      CPUSubType(O.getHeader().cpusubtype),
+      ArchName(std::string(O.getArchTriple().getArchName())),
+      P2Alignment(Align) {}
+
+Slice::Slice(const MachOObjectFile &O) : Slice(O, calculateAlignment(O)) {}
+
+Expected<Slice> Slice::create(const Archive *A) {
+  Error Err = Error::success();
+  std::unique_ptr<MachOObjectFile> FO = nullptr;
+  for (const Archive::Child &Child : A->children(Err)) {
+    Expected<std::unique_ptr<Binary>> ChildOrErr = Child.getAsBinary();
+    if (!ChildOrErr)
+      return createFileError(A->getFileName(), ChildOrErr.takeError());
+    Binary *Bin = ChildOrErr.get().get();
+    if (Bin->isMachOUniversalBinary())
+      return createStringError(std::errc::invalid_argument,
+                               ("archive member " + Bin->getFileName() +
+                                " is a fat file (not allowed in an archive)")
+                                   .str()
+                                   .c_str());
+    if (!Bin->isMachO())
+      return createStringError(
+          std::errc::invalid_argument,
+          ("archive member " + Bin->getFileName() +
+           " is not a MachO file (not allowed in an archive)")
+              .str()
+              .c_str());
+    MachOObjectFile *O = cast<MachOObjectFile>(Bin);
+    if (FO && std::tie(FO->getHeader().cputype, FO->getHeader().cpusubtype) !=
+                  std::tie(O->getHeader().cputype, O->getHeader().cpusubtype)) {
+      return createStringError(
+          std::errc::invalid_argument,
+          ("archive member " + O->getFileName() + " cputype (" +
+           Twine(O->getHeader().cputype) + ") and cpusubtype(" +
+           Twine(O->getHeader().cpusubtype) +
+           ") does not match previous archive members cputype (" +
+           Twine(FO->getHeader().cputype) + ") and cpusubtype(" +
+           Twine(FO->getHeader().cpusubtype) + ") (all members must match) " +
+           FO->getFileName())
+              .str()
+              .c_str());
+    }
+    if (!FO) {
+      ChildOrErr.get().release();
+      FO.reset(O);
+    }
+  }
+  if (Err)
+    return createFileError(A->getFileName(), std::move(Err));
+  if (!FO)
+    return createStringError(
+        std::errc::invalid_argument,
+        ("empty archive with no architecture specification: " +
+         A->getFileName() + " (can't determine architecture for it)")
+            .str()
+            .c_str());
+
+  Slice ArchiveSlice = Slice(*(FO.get()), FO->is64Bit() ? 3 : 2);
+  ArchiveSlice.B = A;
+  return ArchiveSlice;
+}
+
+static Expected<SmallVector<MachO::fat_arch, 2>>
+buildFatArchList(ArrayRef<Slice> Slices) {
+  SmallVector<MachO::fat_arch, 2> FatArchList;
+  uint64_t Offset =
+      sizeof(MachO::fat_header) + Slices.size() * sizeof(MachO::fat_arch);
+
+  for (const auto &S : Slices) {
+    Offset = alignTo(Offset, 1ull << S.getP2Alignment());
+    if (Offset > UINT32_MAX)
+      return createStringError(
+          std::errc::invalid_argument,
+          ("fat file too large to be created because the offset "
+           "field in struct fat_arch is only 32-bits and the offset " +
+           Twine(Offset) + " for " + S.getBinary()->getFileName() +
+           " for architecture " + S.getArchString() + "exceeds that.")
+              .str()
+              .c_str());
+
+    MachO::fat_arch FatArch;
+    FatArch.cputype = S.getCPUType();
+    FatArch.cpusubtype = S.getCPUSubType();
+    FatArch.offset = Offset;
+    FatArch.size = S.getBinary()->getMemoryBufferRef().getBufferSize();
+    FatArch.align = S.getP2Alignment();
+    Offset += FatArch.size;
+    FatArchList.push_back(FatArch);
+  }
+  return FatArchList;
+}
+
+Error object::writeUniversalBinary(ArrayRef<Slice> Slices,
+                                   StringRef OutputFileName) {
+  MachO::fat_header FatHeader;
+  FatHeader.magic = MachO::FAT_MAGIC;
+  FatHeader.nfat_arch = Slices.size();
+
+  Expected<SmallVector<MachO::fat_arch, 2>> FatArchListOrErr =
+      buildFatArchList(Slices);
+  if (!FatArchListOrErr)
+    return FatArchListOrErr.takeError();
+  SmallVector<MachO::fat_arch, 2> FatArchList = *FatArchListOrErr;
+
+  const bool IsExecutable = any_of(Slices, [](Slice S) {
+    return sys::fs::can_execute(S.getBinary()->getFileName());
+  });
+  const uint64_t OutputFileSize =
+      static_cast<uint64_t>(FatArchList.back().offset) +
+      FatArchList.back().size;
+  Expected<std::unique_ptr<FileOutputBuffer>> OutFileOrError =
+      FileOutputBuffer::create(OutputFileName, OutputFileSize,
+                               IsExecutable ? FileOutputBuffer::F_executable
+                                            : 0);
+  if (!OutFileOrError)
+    return createFileError(OutputFileName, OutFileOrError.takeError());
+  std::unique_ptr<FileOutputBuffer> OutFile = std::move(OutFileOrError.get());
+  std::memset(OutFile->getBufferStart(), 0, OutputFileSize);
+
+  if (sys::IsLittleEndianHost)
+    MachO::swapStruct(FatHeader);
+  std::memcpy(OutFile->getBufferStart(), &FatHeader, sizeof(MachO::fat_header));
+
+  for (size_t Index = 0, Size = Slices.size(); Index < Size; ++Index) {
+    MemoryBufferRef BufferRef = Slices[Index].getBinary()->getMemoryBufferRef();
+    std::copy(BufferRef.getBufferStart(), BufferRef.getBufferEnd(),
+              OutFile->getBufferStart() + FatArchList[Index].offset);
+  }
+
+  // FatArchs written after Slices in order to reduce the number of swaps for
+  // the LittleEndian case
+  if (sys::IsLittleEndianHost)
+    for (MachO::fat_arch &FA : FatArchList)
+      MachO::swapStruct(FA);
+  std::memcpy(OutFile->getBufferStart() + sizeof(MachO::fat_header),
+              FatArchList.begin(),
+              sizeof(MachO::fat_arch) * FatArchList.size());
+
+  if (Error E = OutFile->commit())
+    return createFileError(OutputFileName, std::move(E));
+
+  return Error::success();
+}
diff --git a/llvm/tools/llvm-lipo/llvm-lipo.cpp b/llvm/tools/llvm-lipo/llvm-lipo.cpp
index 8c2740d8c94d1..f6d2202579024 100644
--- a/llvm/tools/llvm-lipo/llvm-lipo.cpp
+++ b/llvm/tools/llvm-lipo/llvm-lipo.cpp
@@ -15,6 +15,7 @@
 #include "llvm/Object/Binary.h"
 #include "llvm/Object/MachO.h"
 #include "llvm/Object/MachOUniversal.h"
+#include "llvm/Object/MachOUniversalWriter.h"
 #include "llvm/Object/ObjectFile.h"
 #include "llvm/Option/Arg.h"
 #include "llvm/Option/ArgList.h"
@@ -36,6 +37,15 @@ LLVM_ATTRIBUTE_NORETURN static void reportError(Twine Message) {
   exit(EXIT_FAILURE);
 }
 
+LLVM_ATTRIBUTE_NORETURN static void reportError(Error E) {
+  assert(E);
+  std::string Buf;
+  raw_string_ostream OS(Buf);
+  logAllUnhandledErrors(std::move(E), OS);
+  OS.flush();
+  reportError(Buf);
+}
+
 LLVM_ATTRIBUTE_NORETURN static void reportError(StringRef File, Error E) {
   assert(E);
   std::string Buf;
@@ -103,159 +113,13 @@ struct Config {
   LipoAction ActionToPerform;
 };
 
-// For compatibility with cctools lipo, a file's alignment is calculated as the
-// minimum aligment of all segments. For object files, the file's alignment is
-// the maximum alignment of its sections.
-static uint32_t calculateFileAlignment(const MachOObjectFile &O) {
-  uint32_t P2CurrentAlignment;
-  uint32_t P2MinAlignment = MachOUniversalBinary::MaxSectionAlignment;
-  const bool Is64Bit = O.is64Bit();
-
-  for (const auto &LC : O.load_commands()) {
-    if (LC.C.cmd != (Is64Bit ? MachO::LC_SEGMENT_64 : MachO::LC_SEGMENT))
-      continue;
-    if (O.getHeader().filetype == MachO::MH_OBJECT) {
-      unsigned NumberOfSections =
-          (Is64Bit ? O.getSegment64LoadCommand(LC).nsects
-                   : O.getSegmentLoadCommand(LC).nsects);
-      P2CurrentAlignment = NumberOfSections ? 2 : P2MinAlignment;
-      for (unsigned SI = 0; SI < NumberOfSections; ++SI) {
-        P2CurrentAlignment = std::max(P2CurrentAlignment,
-                                      (Is64Bit ? O.getSection64(LC, SI).align
-                                               : O.getSection(LC, SI).align));
-      }
-    } else {
-      P2CurrentAlignment =
-          countTrailingZeros(Is64Bit ? O.getSegment64LoadCommand(LC).vmaddr
-                                     : O.getSegmentLoadCommand(LC).vmaddr);
-    }
-    P2MinAlignment = std::min(P2MinAlignment, P2CurrentAlignment);
-  }
-  // return a value >= 4 byte aligned, and less than MachO MaxSectionAlignment
-  return std::max(
-      static_cast<uint32_t>(2),
-      std::min(P2MinAlignment, static_cast<uint32_t>(
-                                   MachOUniversalBinary::MaxSectionAlignment)));
-}
-
-static uint32_t calculateAlignment(const MachOObjectFile *ObjectFile) {
-  switch (ObjectFile->getHeader().cputype) {
-  case MachO::CPU_TYPE_I386:
-  case MachO::CPU_TYPE_X86_64:
-  case MachO::CPU_TYPE_POWERPC:
-  case MachO::CPU_TYPE_POWERPC64:
-    return 12; // log2 value of page size(4k) for x86 and PPC
-  case MachO::CPU_TYPE_ARM:
-  case MachO::CPU_TYPE_ARM64:
-  case MachO::CPU_TYPE_ARM64_32:
-    return 14; // log2 value of page size(16k) for Darwin ARM
-  default:
-    return calculateFileAlignment(*ObjectFile);
-  }
+static Slice archiveSlice(const Archive *A, StringRef File) {
+  Expected<Slice> ArchiveOrSlice = Slice::create(A);
+  if (!ArchiveOrSlice)
+    reportError(File, ArchiveOrSlice.takeError());
+  return *ArchiveOrSlice;
 }
 
-class Slice {
-  const Binary *B;
-  uint32_t CPUType;
-  uint32_t CPUSubType;
-  std::string ArchName;
-
-  // P2Alignment field stores slice alignment values from universal
-  // binaries. This is also needed to order the slices so the total
-  // file size can be calculated before creating the output buffer.
-  uint32_t P2Alignment;
-
-public:
-  Slice(const MachOObjectFile *O, uint32_t Align)
-      : B(O), CPUType(O->getHeader().cputype),
-        CPUSubType(O->getHeader().cpusubtype),
-        ArchName(std::string(O->getArchTriple().getArchName())),
-        P2Alignment(Align) {}
-
-  explicit Slice(const MachOObjectFile *O) : Slice(O, calculateAlignment(O)){};
-
-  explicit Slice(const Archive *A) : B(A) {
-    Error Err = Error::success();
-    std::unique_ptr<MachOObjectFile> FO = nullptr;
-    for (const Archive::Child &Child : A->children(Err)) {
-      Expected<std::unique_ptr<Binary>> ChildOrErr = Child.getAsBinary();
-      if (!ChildOrErr)
-        reportError(A->getFileName(), ChildOrErr.takeError());
-      Binary *Bin = ChildOrErr.get().get();
-      if (Bin->isMachOUniversalBinary())
-        reportError(("archive member " + Bin->getFileName() +
-                     " is a fat file (not allowed in an archive)")
-                        .str());
-      if (!Bin->isMachO())
-        reportError(("archive member " + Bin->getFileName() +
-                     " is not a MachO file (not allowed in an archive)"));
-      MachOObjectFile *O = cast<MachOObjectFile>(Bin);
-      if (FO &&
-          std::tie(FO->getHeader().cputype, FO->getHeader().cpusubtype) !=
-              std::tie(O->getHeader().cputype, O->getHeader().cpusubtype)) {
-        reportError(("archive member " + O->getFileName() + " cputype (" +
-                     Twine(O->getHeader().cputype) + ") and cpusubtype(" +
-                     Twine(O->getHeader().cpusubtype) +
-                     ") does not match previous archive members cputype (" +
-                     Twine(FO->getHeader().cputype) + ") and cpusubtype(" +
-                     Twine(FO->getHeader().cpusubtype) +
-                     ") (all members must match) " + FO->getFileName())
-                        .str());
-      }
-      if (!FO) {
-        ChildOrErr.get().release();
-        FO.reset(O);
-      }
-    }
-    if (Err)
-      reportError(A->getFileName(), std::move(Err));
-    if (!FO)
-      reportError(("empty archive with no architecture specification: " +
-                   A->getFileName() + " (can't determine architecture for it)")
-                      .str());
-    CPUType = FO->getHeader().cputype;
-    CPUSubType = FO->getHeader().cpusubtype;
-    ArchName = std::string(FO->getArchTriple().getArchName());
-    // Replicate the behavior of cctools lipo.
-    P2Alignment = FO->is64Bit() ? 3 : 2;
-  }
-
-  void setP2Alignment(uint32_t Align) { P2Alignment = Align; }
-
-  const Binary *getBinary() const { return B; }
-
-  uint32_t getCPUType() const { return CPUType; }
-
-  uint32_t getCPUSubType() const { return CPUSubType; }
-
-  uint32_t getP2Alignment() const { return P2Alignment; }
-
-  uint64_t getCPUID() const {
-    return static_cast<uint64_t>(CPUType) << 32 | CPUSubType;
-  }
-
-  std::string getArchString() const {
-    if (!ArchName.empty())
-      return ArchName;
-    return ("unknown(" + Twine(CPUType) + "," +
-            Twine(CPUSubType & ~MachO::CPU_SUBTYPE_MASK) + ")")
-        .str();
-  }
-
-  friend bool operator<(const Slice &Lhs, const Slice &Rhs) {
-    if (Lhs.CPUType == Rhs.CPUType)
-      return Lhs.CPUSubType < Rhs.CPUSubType;
-    // force arm64-family to follow after all other slices for
-    // compatibility with cctools lipo
-    if (Lhs.CPUType == MachO::CPU_TYPE_ARM64)
-      return false;
-    if (Rhs.CPUType == MachO::CPU_TYPE_ARM64)
-      return true;
-    // Sort by alignment to minimize file size
-    return Lhs.P2Alignment < Rhs.P2Alignment;
-  }
-};
-
 } // end namespace
 
 static void validateArchitectureName(StringRef ArchitectureName) {
@@ -450,8 +314,8 @@ readInputBinaries(ArrayRef<InputFile> InputFiles) {
     if (!B->isArchive() && !B->isMachO() && !B->isMachOUniversalBinary())
       reportError("File " + IF.FileName + " has unsupported binary format");
     if (IF.ArchType && (B->isMachO() || B->isArchive())) {
-      const auto S = B->isMachO() ? Slice(cast<MachOObjectFile>(B))
-                                  : Slice(cast<Archive>(B));
+      const auto S = B->isMachO() ? Slice(*cast<MachOObjectFile>(B))
+                                  : archiveSlice(cast<Archive>(B), IF.FileName);
       const auto SpecifiedCPUType = MachO::getCPUTypeFromArchitecture(
                                         MachO::getArchitectureFromName(
                                             Triple(*IF.ArchType).getArchName()))
@@ -506,13 +370,15 @@ static void printBinaryArchs(const Binary *Binary, raw_ostream &OS) {
       Expected<std::unique_ptr<MachOObjectFile>> MachOObjOrError =
           O.getAsObjectFile();
       if (MachOObjOrError) {
-        OS << Slice(MachOObjOrError->get()).getArchString() << " ";
+        OS << Slice(*(MachOObjOrError->get())).getArchString() << " ";
         continue;
       }
       Expected<std::unique_ptr<Archive>> ArchiveOrError = O.getAsArchive();
       if (ArchiveOrError) {
         consumeError(MachOObjOrError.takeError());
-        OS << Slice(ArchiveOrError->get()).getArchString() << " ";
+        OS << archiveSlice(ArchiveOrError->get(), Binary->getFileName())
+                  .getArchString()
+           << " ";
         continue;
       }
       consumeError(ArchiveOrError.takeError());
@@ -521,7 +387,7 @@ static void printBinaryArchs(const Binary *Binary, raw_ostream &OS) {
     OS << "\n";
     return;
   }
-  OS << Slice(cast<MachOObjectFile>(Binary)).getArchString() << " \n";
+  OS << Slice(*cast<MachOObjectFile>(Binary)).getArchString() << " \n";
 }
 
 LLVM_ATTRIBUTE_NORETURN
@@ -646,12 +512,12 @@ static SmallVector<Slice, 2> buildSlices(
         if (!BinaryOrError)
           reportError(InputBinary->getFileName(), BinaryOrError.takeError());
         ExtractedObjects.push_back(std::move(BinaryOrError.get()));
-        Slices.emplace_back(ExtractedObjects.back().get(), O.getAlign());
+        Slices.emplace_back(*(ExtractedObjects.back().get()), O.getAlign());
       }
     } else if (auto O = dyn_cast<MachOObjectFile>(InputBinary)) {
-      Slices.emplace_back(O);
+      Slices.emplace_back(*O);
     } else if (auto A = dyn_cast<Archive>(InputBinary)) {
-      Slices.emplace_back(A);
+      Slices.push_back(archiveSlice(A, InputBinary->getFileName()));
     } else {
       llvm_unreachable("Unexpected binary format");
     }
@@ -660,79 +526,6 @@ static SmallVector<Slice, 2> buildSlices(
   return Slices;
 }
 
-static SmallVector<MachO::fat_arch, 2>
-buildFatArchList(ArrayRef<Slice> Slices) {
-  SmallVector<MachO::fat_arch, 2> FatArchList;
-  uint64_t Offset =
-      sizeof(MachO::fat_header) + Slices.size() * sizeof(MachO::fat_arch);
-
-  for (const auto &S : Slices) {
-    Offset = alignTo(Offset, 1ull << S.getP2Alignment());
-    if (Offset > UINT32_MAX)
-      reportError("fat file too large to be created because the offset "
-                  "field in struct fat_arch is only 32-bits and the offset " +
-                  Twine(Offset) + " for " + S.getBinary()->getFileName() +
-                  " for architecture " + S.getArchString() + "exceeds that.");
-
-    MachO::fat_arch FatArch;
-    FatArch.cputype = S.getCPUType();
-    FatArch.cpusubtype = S.getCPUSubType();
-    FatArch.offset = Offset;
-    FatArch.size = S.getBinary()->getMemoryBufferRef().getBufferSize();
-    FatArch.align = S.getP2Alignment();
-    Offset += FatArch.size;
-    FatArchList.push_back(FatArch);
-  }
-  return FatArchList;
-}
-
-static void createUniversalBinary(SmallVectorImpl<Slice> &Slices,
-                                  StringRef OutputFileName) {
-  MachO::fat_header FatHeader;
-  FatHeader.magic = MachO::FAT_MAGIC;
-  FatHeader.nfat_arch = Slices.size();
-
-  stable_sort(Slices);
-  SmallVector<MachO::fat_arch, 2> FatArchList = buildFatArchList(Slices);
-
-  const bool IsExecutable = any_of(Slices, [](Slice S) {
-    return sys::fs::can_execute(S.getBinary()->getFileName());
-  });
-  const uint64_t OutputFileSize =
-      static_cast<uint64_t>(FatArchList.back().offset) +
-      FatArchList.back().size;
-  Expected<std::unique_ptr<FileOutputBuffer>> OutFileOrError =
-      FileOutputBuffer::create(OutputFileName, OutputFileSize,
-                               IsExecutable ? FileOutputBuffer::F_executable
-                                            : 0);
-  if (!OutFileOrError)
-    reportError(OutputFileName, OutFileOrError.takeError());
-  std::unique_ptr<FileOutputBuffer> OutFile = std::move(OutFileOrError.get());
-  std::memset(OutFile->getBufferStart(), 0, OutputFileSize);
-
-  if (sys::IsLittleEndianHost)
-    MachO::swapStruct(FatHeader);
-  std::memcpy(OutFile->getBufferStart(), &FatHeader, sizeof(MachO::fat_header));
-
-  for (size_t Index = 0, Size = Slices.size(); Index < Size; ++Index) {
-    MemoryBufferRef BufferRef = Slices[Index].getBinary()->getMemoryBufferRef();
-    std::copy(BufferRef.getBufferStart(), BufferRef.getBufferEnd(),
-              OutFile->getBufferStart() + FatArchList[Index].offset);
-  }
-
-  // FatArchs written after Slices in order to reduce the number of swaps for
-  // the LittleEndian case
-  if (sys::IsLittleEndianHost)
-    for (MachO::fat_arch &FA : FatArchList)
-      MachO::swapStruct(FA);
-  std::memcpy(OutFile->getBufferStart() + sizeof(MachO::fat_header),
-              FatArchList.begin(),
-              sizeof(MachO::fat_arch) * FatArchList.size());
-
-  if (Error E = OutFile->commit())
-    reportError(OutputFileName, std::move(E));
-}
-
 LLVM_ATTRIBUTE_NORETURN
 static void createUniversalBinary(ArrayRef<OwningBinary<Binary>> InputBinaries,
                                   const StringMap<const uint32_t> &Alignments,
@@ -745,7 +538,10 @@ static void createUniversalBinary(ArrayRef<OwningBinary<Binary>> InputBinaries,
       buildSlices(InputBinaries, Alignments, ExtractedObjects);
   checkArchDuplicates(Slices);
   checkUnusedAlignments(Slices, Alignments);
-  createUniversalBinary(Slices, OutputFileName);
+
+  llvm::stable_sort(Slices);
+  if (Error E = writeUniversalBinary(Slices, OutputFileName))
+    reportError(std::move(E));
 
   exit(EXIT_SUCCESS);
 }
@@ -776,7 +572,10 @@ static void extractSlice(ArrayRef<OwningBinary<Binary>> InputBinaries,
     reportError(
         "fat input file " + InputBinaries.front().getBinary()->getFileName() +
         " does not contain the specified architecture " + ArchType);
-  createUniversalBinary(Slices, OutputFileName);
+
+  llvm::stable_sort(Slices);
+  if (Error E = writeUniversalBinary(Slices, OutputFileName))
+    reportError(std::move(E));
   exit(EXIT_SUCCESS);
 }
 
@@ -792,7 +591,7 @@ buildReplacementSlices(ArrayRef<OwningBinary<Binary>> ReplacementBinaries,
     if (!O)
       reportError("replacement file: " + ReplacementBinary->getFileName() +
                   " is a fat file (must be a thin file)");
-    Slice S(O);
+    Slice S(*O);
     auto Entry = Slices.try_emplace(S.getArchString(), S);
     if (!Entry.second)
       reportError("-replace " + S.getArchString() +
@@ -843,7 +642,10 @@ static void replaceSlices(ArrayRef<OwningBinary<Binary>> InputBinaries,
                 " does not contain that architecture");
 
   checkUnusedAlignments(Slices, Alignments);
-  createUniversalBinary(Slices, OutputFileName);
+
+  llvm::stable_sort(Slices);
+  if (Error E = writeUniversalBinary(Slices, OutputFileName))
+    reportError(std::move(E));
   exit(EXIT_SUCCESS);
 }
 

From c75c7d51968d397e5521c8b6c1e906bde1245af6 Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
Date: Fri, 31 Jul 2020 16:23:24 +0000
Subject: [PATCH 0748/1035] [gn build] Port df69492cdfa

---
 llvm/utils/gn/secondary/llvm/lib/Object/BUILD.gn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/utils/gn/secondary/llvm/lib/Object/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Object/BUILD.gn
index 4a3679900fe9f..42dde29a8f8a0 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Object/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Object/BUILD.gn
@@ -27,6 +27,7 @@ static_library("Object") {
     "IRSymtab.cpp",
     "MachOObjectFile.cpp",
     "MachOUniversal.cpp",
+    "MachOUniversalWriter.cpp",
     "Minidump.cpp",
     "ModuleSymbolTable.cpp",
     "Object.cpp",

From d23c1d6a8dddf0e1b9b9fa64726941e402ede8af Mon Sep 17 00:00:00 2001
From: Hongtao Yu <hoy@fb.com>
Date: Thu, 30 Jul 2020 18:22:50 -0700
Subject: [PATCH 0749/1035] [AutoFDO] Avoid merging inlinee samples multiple
 times

A function call can be replicated by optimizations like loop unroll and jump threading and the replicates end up sharing the sample nested callee profile. Therefore when it comes to merging samples for uninlined callees in the sample profile inliner, a callee profile can be merged multiple times which will cause an assert to fire.

This change avoids merging same callee profile for duplicate callsites by filtering out callee profiles with a non-zero head sample count.

Reviewed By: wenlei, wmi

Differential Revision: https://reviews.llvm.org/D84997
---
 llvm/lib/Transforms/IPO/SampleProfile.cpp     | 30 ++++---
 .../SampleProfile/inline-mergeprof-dup.ll     | 80 +++++++++++++++++++
 2 files changed, 100 insertions(+), 10 deletions(-)
 create mode 100644 llvm/test/Transforms/SampleProfile/inline-mergeprof-dup.ll

diff --git a/llvm/lib/Transforms/IPO/SampleProfile.cpp b/llvm/lib/Transforms/IPO/SampleProfile.cpp
index ca60d35b8aafb..5978f2dc6d1dd 100644
--- a/llvm/lib/Transforms/IPO/SampleProfile.cpp
+++ b/llvm/lib/Transforms/IPO/SampleProfile.cpp
@@ -1106,16 +1106,26 @@ bool SampleProfileLoader::inlineHotFunctions(
     }
 
     if (ProfileMergeInlinee) {
-      // Use entry samples as head samples during the merge, as inlinees
-      // don't have head samples.
-      assert(FS->getHeadSamples() == 0 && "Expect 0 head sample for inlinee");
-      const_cast<FunctionSamples *>(FS)->addHeadSamples(FS->getEntrySamples());
-
-      // Note that we have to do the merge right after processing function.
-      // This allows OutlineFS's profile to be used for annotation during
-      // top-down processing of functions' annotation.
-      FunctionSamples *OutlineFS = Reader->getOrCreateSamplesFor(*Callee);
-      OutlineFS->merge(*FS);
+      // A function call can be replicated by optimizations like callsite
+      // splitting or jump threading and the replicates end up sharing the
+      // sample nested callee profile instead of slicing the original inlinee's
+      // profile. We want to do merge exactly once by filtering out callee
+      // profiles with a non-zero head sample count.
+      if (FS->getHeadSamples() == 0) {
+        // Use entry samples as head samples during the merge, as inlinees
+        // don't have head samples.
+        const_cast<FunctionSamples *>(FS)->addHeadSamples(
+            FS->getEntrySamples());
+
+        // Note that we have to do the merge right after processing function.
+        // This allows OutlineFS's profile to be used for annotation during
+        // top-down processing of functions' annotation.
+        FunctionSamples *OutlineFS = Reader->getOrCreateSamplesFor(*Callee);
+        OutlineFS->merge(*FS);
+      } else
+        assert(FS->getHeadSamples() == FS->getEntrySamples() &&
+               "Expect same head and entry sample counts for profiles already "
+               "merged.");
     } else {
       auto pair =
           notInlinedCallInfo.try_emplace(Callee, NotInlinedProfileInfo{0});
diff --git a/llvm/test/Transforms/SampleProfile/inline-mergeprof-dup.ll b/llvm/test/Transforms/SampleProfile/inline-mergeprof-dup.ll
new file mode 100644
index 0000000000000..8d1379a582291
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/inline-mergeprof-dup.ll
@@ -0,0 +1,80 @@
+;; Test we merge non-inlined profile only once with '-sample-profile-merge-inlinee'
+; RUN: opt < %s -passes='function(callsite-splitting),sample-profile' -sample-profile-file=%S/Inputs/inline-mergeprof.prof -sample-profile-merge-inlinee=true -S | FileCheck %s
+
+%struct.bitmap = type { i32, %struct.bitmap* }
+
+; CHECK-LABEL: @main
+define void @main(i1 %c, %struct.bitmap* %a_elt, %struct.bitmap* %b_elt) #0 !dbg !6 {
+entry:
+  br label %Top
+
+Top:
+  %tobool1 = icmp eq %struct.bitmap* %a_elt, null
+  br i1 %tobool1, label %CallSiteBB, label %NextCond
+
+NextCond:
+  %cmp = icmp ne %struct.bitmap* %b_elt, null
+  br i1 %cmp, label %CallSiteBB, label %End
+
+CallSiteBB:
+  %p = phi i1 [0, %Top], [%c, %NextCond]
+;; The call site is replicated by callsite-splitting pass and they end up share the same sample profile
+; CHECK: call void @_Z3sumii(%struct.bitmap* null, %struct.bitmap* null, %struct.bitmap* %b_elt, i1 false)
+; CHECK: call void @_Z3sumii(%struct.bitmap* nonnull %a_elt, %struct.bitmap* nonnull %a_elt, %struct.bitmap* nonnull %b_elt, i1 %c)
+  call void @_Z3sumii(%struct.bitmap* %a_elt, %struct.bitmap* %a_elt, %struct.bitmap* %b_elt, i1 %p), !dbg !8
+  br label %End
+
+End:
+  ret void
+}
+
+define void @_Z3sumii(%struct.bitmap* %dst_elt, %struct.bitmap* %a_elt, %struct.bitmap* %b_elt, i1 %c)  #0 !dbg !12 {
+entry:
+  %tobool = icmp ne %struct.bitmap* %a_elt, null
+  %tobool1 = icmp ne %struct.bitmap* %b_elt, null
+  %or.cond = and i1 %tobool, %tobool1, !dbg !13
+  br i1 %or.cond, label %Cond, label %Big
+
+Cond:
+  %cmp = icmp eq %struct.bitmap*  %dst_elt, %a_elt, !dbg !14
+  br i1 %cmp, label %Small, label %Big, !dbg !15
+
+Small:
+  br label %End
+
+Big:
+  br label %End
+
+End:
+  ret void
+}
+
+attributes #0 = { "use-sample-profile" }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4}
+!llvm.ident = !{!5}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 3.5 ", isOptimized: false, runtimeVersion: 0, emissionKind: NoDebug, enums: !2, retainedTypes: !2, globals: !2, imports: !2)
+!1 = !DIFile(filename: "calls.cc", directory: ".")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 1, !"Debug Info Version", i32 3}
+!5 = !{!"clang version 3.5 "}
+!6 = distinct !DISubprogram(name: "main", scope: !1, file: !1, line: 7, type: !7, scopeLine: 7, virtualIndex: 6, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !2)
+!7 = !DISubroutineType(types: !2)
+!8 = !DILocation(line: 10, scope: !9)
+!9 = !DILexicalBlockFile(scope: !10, file: !1, discriminator: 2)
+!10 = distinct !DILexicalBlock(scope: !6, file: !1, line: 10)
+!11 = !DILocation(line: 12, scope: !6)
+!12 = distinct !DISubprogram(name: "sum", scope: !1, file: !1, line: 3, type: !7, scopeLine: 3, virtualIndex: 6, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !2)
+!13 = !DILocation(line: 4, scope: !12)
+!14 = !DILocation(line: 5, scope: !12)
+!15 = !DILocation(line: 6, scope: !12)
+
+
+;; Check the profile of funciton sum is only merged once though the original callsite is replicted.
+; CHECK: name: "sum"
+; CHECK-NEXT: {!"function_entry_count", i64 46}
+; CHECK: !{!"branch_weights", i32 11, i32 37}
+; CHECK: !{!"branch_weights", i32 11, i32 1}

From dcdc77619e1af61181e7214d292b7e5b59946186 Mon Sep 17 00:00:00 2001
From: Hiroshi Yamauchi <yamauchi@google.com>
Date: Thu, 30 Jul 2020 11:22:25 -0700
Subject: [PATCH 0750/1035] [PGO][test] Add test to check memops changes
 function hash

Following up D84782.

Differential Revision: https://reviews.llvm.org/D84953
---
 llvm/test/Transforms/PGOProfile/memop_hash.ll | 25 +++++++++++++++++++
 1 file changed, 25 insertions(+)
 create mode 100644 llvm/test/Transforms/PGOProfile/memop_hash.ll

diff --git a/llvm/test/Transforms/PGOProfile/memop_hash.ll b/llvm/test/Transforms/PGOProfile/memop_hash.ll
new file mode 100644
index 0000000000000..aca3b0c4dbeb3
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/memop_hash.ll
@@ -0,0 +1,25 @@
+; RUN: opt < %s -pgo-instr-gen -S | FileCheck %s
+; RUN: opt < %s -passes=pgo-instr-gen -S | FileCheck %s
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture writeonly, i8* nocapture readonly, i32, i1)
+
+define i64 @foo1(i8* %a, i8* %b, i32 %s) {
+entry:
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %a, i8* %b, i32 %s, i1 false);
+  ret i64 0
+}
+
+define i64 @foo2(i8* %a, i8* %b, i32 %s) {
+entry:
+  ret i64 0
+}
+
+; The two hashes should not be equal as the existence of the memcpy should change the hash.
+;
+; CHECK: @foo1
+; CHECK: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @__profn_foo1, i32 0, i32 0), i64 [[FOO1_HASH:[0-9]+]], i32 1, i32 0)
+; CHECK: @foo2
+; CHECK-NOT: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @__profn_foo2, i32 0, i32 0), i64 [[FOO1_HASH]], i32 1, i32 0)

From 86609b7af79fc9c71371ca255b271105f546c9c4 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Fri, 31 Jul 2020 12:51:46 +0100
Subject: [PATCH 0751/1035] [X86][SSE] Cleanup bitwise reduction check
 prefixes. NFC

Add AVX512BW/AVX512BWVL prefixes for a future patch
---
 llvm/test/CodeGen/X86/vector-reduce-and-cmp.ll |  4 ++--
 llvm/test/CodeGen/X86/vector-reduce-and.ll     | 12 ++++++------
 llvm/test/CodeGen/X86/vector-reduce-or.ll      | 12 ++++++------
 llvm/test/CodeGen/X86/vector-reduce-xor.ll     | 12 ++++++------
 4 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/llvm/test/CodeGen/X86/vector-reduce-and-cmp.ll b/llvm/test/CodeGen/X86/vector-reduce-and-cmp.ll
index 91668148a9ac4..0df3238e6a2dc 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-and-cmp.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-and-cmp.ll
@@ -3,8 +3,8 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=SSE,SSE41
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=AVX,AVX1
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX,AVX2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX,AVX512
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512vl | FileCheck %s --check-prefixes=AVX,AVX512
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX,AVX512,AVX512BW
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512vl | FileCheck %s --check-prefixes=AVX,AVX512,AVX512BWVL
 
 ;
 ; vXi64
diff --git a/llvm/test/CodeGen/X86/vector-reduce-and.ll b/llvm/test/CodeGen/X86/vector-reduce-and.ll
index 088913ed69521..9545d29cee588 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-and.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-and.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE41
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX512
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512vl | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX512
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=SSE,SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=SSE,SSE41
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=AVX,AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX,AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX,AVX512,AVX512BW
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512vl | FileCheck %s --check-prefixes=AVX,AVX512,AVX512BWVL
 
 ;
 ; vXi64
diff --git a/llvm/test/CodeGen/X86/vector-reduce-or.ll b/llvm/test/CodeGen/X86/vector-reduce-or.ll
index 622f503ef6cbf..35193e9feadcd 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-or.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-or.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE41
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX512
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512vl | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX512
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=SSE,SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=SSE,SSE41
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=AVX,AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX,AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX,AVX512,AVX512BW
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512vl | FileCheck %s --check-prefixes=AVX,AVX512,AVX512BWVL
 
 ;
 ; vXi64
diff --git a/llvm/test/CodeGen/X86/vector-reduce-xor.ll b/llvm/test/CodeGen/X86/vector-reduce-xor.ll
index 3c09e25341ad3..4fd84897445c4 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-xor.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-xor.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE41
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX512
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512vl | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX512
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=SSE,SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=SSE,SSE41
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=AVX,AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX,AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX,AVX512,AVX512BW
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512vl | FileCheck %s --check-prefixes=AVX,AVX512,AVX512BWVL
 
 ;
 ; vXi64

From 6c75db8b4bc59eace18143ce086419d37da24746 Mon Sep 17 00:00:00 2001
From: Keith Randall <khr@golang.org>
Date: Fri, 31 Jul 2020 10:04:04 -0700
Subject: [PATCH 0752/1035] Disable getauxval for Go

We want the Go build to not use getauxval, as we must support glibc < 2.16 platforms.

Reviewed By: dvyukov

Differential Revision: https://reviews.llvm.org/D84859
---
 compiler-rt/lib/sanitizer_common/sanitizer_getauxval.h | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_getauxval.h b/compiler-rt/lib/sanitizer_common/sanitizer_getauxval.h
index 86ad3a5e2c2aa..38439e44f611e 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_getauxval.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_getauxval.h
@@ -21,8 +21,9 @@
 
 #if SANITIZER_LINUX || SANITIZER_FUCHSIA
 
-# if __GLIBC_PREREQ(2, 16) || (SANITIZER_ANDROID && __ANDROID_API__ >= 21) || \
-     SANITIZER_FUCHSIA
+# if (__GLIBC_PREREQ(2, 16) || (SANITIZER_ANDROID && __ANDROID_API__ >= 21) || \
+      SANITIZER_FUCHSIA) &&                                                    \
+     !SANITIZER_GO
 #  define SANITIZER_USE_GETAUXVAL 1
 # else
 #  define SANITIZER_USE_GETAUXVAL 0

From 2144a3bdbba40f0a78d30921088e2c8407ff9dd2 Mon Sep 17 00:00:00 2001
From: Aditya Nandakumar <aditya_nandakumar@apple.com>
Date: Fri, 31 Jul 2020 09:41:06 -0700
Subject: [PATCH 0753/1035] [GISel] Add combiners for G_INTTOPTR and G_PTRTOINT

https://reviews.llvm.org/D84909

Patch adds two new GICombinerRules, one for G_INTTOPTR and one for
G_PTRTOINT. The G_INTTOPTR elides ptr2int(int2ptr(x)) to a copy of x, if
the cast is within the same address space. The G_PTRTOINT elides
int2ptr(ptr2int(x)) to a copy of x. Patch additionally adds new combiner
tests for the AArch64 target to test these new combiner rules.

Patch by mkitzan
---
 .../llvm/CodeGen/GlobalISel/CombinerHelper.h  |  8 +++++
 .../include/llvm/Target/GlobalISel/Combine.td | 21 +++++++++++-
 .../lib/CodeGen/GlobalISel/CombinerHelper.cpp | 33 +++++++++++++++++++
 .../GlobalISel/combine-inttoptr-ptrtoint.mir  | 33 +++++++++++++++++++
 .../AArch64/GlobalISel/combine-ptrtoint.mir   | 17 ++++++++++
 5 files changed, 111 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/combine-inttoptr-ptrtoint.mir
 create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/combine-ptrtoint.mir

diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
index e694e7ad2c834..3847112d51836 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
@@ -200,6 +200,14 @@ class CombinerHelper {
   bool applyCombineShiftToUnmerge(MachineInstr &MI, const unsigned &ShiftVal);
   bool tryCombineShiftToUnmerge(MachineInstr &MI, unsigned TargetShiftAmount);
 
+  /// Transform IntToPtr(PtrToInt(x)) to x if cast is in the same address space.
+  bool matchCombineI2PToP2I(MachineInstr &MI, Register &Reg);
+  bool applyCombineI2PToP2I(MachineInstr &MI, Register &Reg);
+
+  /// Transform PtrToInt(IntToPtr(x)) to x.
+  bool matchCombineP2IToI2P(MachineInstr &MI, Register &Reg);
+  bool applyCombineP2IToI2P(MachineInstr &MI, Register &Reg);
+
   /// Return true if any explicit use operand on \p MI is defined by a
   /// G_IMPLICIT_DEF.
   bool matchAnyExplicitUseIsUndef(MachineInstr &MI);
diff --git a/llvm/include/llvm/Target/GlobalISel/Combine.td b/llvm/include/llvm/Target/GlobalISel/Combine.td
index 40ed6be089ac8..ba19a4635570a 100644
--- a/llvm/include/llvm/Target/GlobalISel/Combine.td
+++ b/llvm/include/llvm/Target/GlobalISel/Combine.td
@@ -257,6 +257,24 @@ def simplify_add_to_sub: GICombineRule <
   (apply [{ return Helper.applySimplifyAddToSub(*${root}, ${info});}])
 >;
 
+// Fold int2ptr(ptr2int(x)) -> x
+def p2i_to_i2p_matchinfo: GIDefMatchData<"Register">;
+def p2i_to_i2p: GICombineRule<
+  (defs root:$root, p2i_to_i2p_matchinfo:$info),
+  (match (wip_match_opcode G_INTTOPTR):$root,
+    [{ return Helper.matchCombineI2PToP2I(*${root}, ${info}); }]),
+  (apply [{ return Helper.applyCombineI2PToP2I(*${root}, ${info}); }])
+>;
+
+// Fold ptr2int(int2ptr(x)) -> x
+def i2p_to_p2i_matchinfo: GIDefMatchData<"Register">;
+def i2p_to_p2i: GICombineRule<
+  (defs root:$root, i2p_to_p2i_matchinfo:$info),
+  (match (wip_match_opcode G_PTRTOINT):$root,
+    [{ return Helper.matchCombineP2IToI2P(*${root}, ${info}); }]),
+  (apply [{ return Helper.applyCombineP2IToI2P(*${root}, ${info}); }])
+>;
+
 // FIXME: These should use the custom predicate feature once it lands.
 def undef_combines : GICombineGroup<[undef_to_fp_zero, undef_to_int_zero,
                                      undef_to_negative_one,
@@ -267,7 +285,8 @@ def undef_combines : GICombineGroup<[undef_to_fp_zero, undef_to_int_zero,
 
 def identity_combines : GICombineGroup<[select_same_val, right_identity_zero,
                                         binop_same_val, binop_left_to_zero,
-                                        binop_right_to_zero]>;
+                                        binop_right_to_zero, p2i_to_i2p, 
+                                        i2p_to_p2i]>;
 
 def trivial_combines : GICombineGroup<[copy_prop, mul_to_shl]>;
 def all_combines : GICombineGroup<[trivial_combines, ptr_add_immed_chain,
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
index 422b71b28b6fe..a9ec1d4365a34 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -1545,6 +1545,39 @@ bool CombinerHelper::tryCombineShiftToUnmerge(MachineInstr &MI,
   return false;
 }
 
+bool CombinerHelper::matchCombineI2PToP2I(MachineInstr &MI, Register &Reg) {
+  assert(MI.getOpcode() == TargetOpcode::G_INTTOPTR && "Expected a G_INTTOPTR");
+  Register DstReg = MI.getOperand(0).getReg();
+  LLT DstTy = MRI.getType(DstReg);
+  Register SrcReg = MI.getOperand(1).getReg();
+  return mi_match(SrcReg, MRI,
+                  m_GPtrToInt(m_all_of(m_SpecificType(DstTy), m_Reg(Reg))));
+}
+
+bool CombinerHelper::applyCombineI2PToP2I(MachineInstr &MI, Register &Reg) {
+  assert(MI.getOpcode() == TargetOpcode::G_INTTOPTR && "Expected a G_INTTOPTR");
+  Register DstReg = MI.getOperand(0).getReg();
+  Builder.setInstr(MI);
+  Builder.buildCopy(DstReg, Reg);
+  MI.eraseFromParent();
+  return true;
+}
+
+bool CombinerHelper::matchCombineP2IToI2P(MachineInstr &MI, Register &Reg) {
+  assert(MI.getOpcode() == TargetOpcode::G_PTRTOINT && "Expected a G_PTRTOINT");
+  Register SrcReg = MI.getOperand(1).getReg();
+  return mi_match(SrcReg, MRI, m_GIntToPtr(m_Reg(Reg)));
+}
+
+bool CombinerHelper::applyCombineP2IToI2P(MachineInstr &MI, Register &Reg) {
+  assert(MI.getOpcode() == TargetOpcode::G_PTRTOINT && "Expected a G_PTRTOINT");
+  Register DstReg = MI.getOperand(0).getReg();
+  Builder.setInstr(MI);
+  Builder.buildZExtOrTrunc(DstReg, Reg);
+  MI.eraseFromParent();
+  return true;
+}
+
 bool CombinerHelper::matchAnyExplicitUseIsUndef(MachineInstr &MI) {
   return any_of(MI.explicit_uses(), [this](const MachineOperand &MO) {
     return MO.isReg() &&
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-inttoptr-ptrtoint.mir b/llvm/test/CodeGen/AArch64/GlobalISel/combine-inttoptr-ptrtoint.mir
new file mode 100644
index 0000000000000..d99b38900383b
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-inttoptr-ptrtoint.mir
@@ -0,0 +1,33 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -run-pass=aarch64-prelegalizer-combiner -verify-machineinstrs -mtriple aarch64-unknown-unknown %s -o - | FileCheck %s
+# RUN: llc -debugify-and-strip-all-safe -run-pass=aarch64-prelegalizer-combiner -verify-machineinstrs -mtriple aarch64-unknown-unknown %s -o - | FileCheck %s
+---
+name:            test_combine_inttoptr_same_addressspace
+body:             |
+  bb.1:
+  liveins: $x0, $x1
+
+    ; CHECK-LABEL: name: test_combine_inttoptr_same_addressspace
+    ; CHECK: [[COPY:%[0-9]+]]:_(p64) = COPY $x0
+    ; CHECK: $x1 = COPY [[COPY]](p64)
+    %0:_(p64) = COPY $x0
+    %1:_(s64) = G_PTRTOINT %0
+    %2:_(p64) = G_INTTOPTR %1
+    $x1 = COPY %2
+...
+---
+name:            test_combine_inttoptr_diff_addressspace
+body:             |
+  bb.1:
+  liveins: $x0, $x1
+
+    ; CHECK-LABEL: name: test_combine_inttoptr_diff_addressspace
+    ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+    ; CHECK: [[PTRTOINT:%[0-9]+]]:_(s64) = G_PTRTOINT [[COPY]](p0)
+    ; CHECK: [[INTTOPTR:%[0-9]+]]:_(p64) = G_INTTOPTR [[PTRTOINT]](s64)
+    ; CHECK: $x1 = COPY [[INTTOPTR]](p64)
+    %0:_(p0) = COPY $x0
+    %1:_(s64) = G_PTRTOINT %0
+    %2:_(p64) = G_INTTOPTR %1
+    $x1 = COPY %2
+...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-ptrtoint.mir b/llvm/test/CodeGen/AArch64/GlobalISel/combine-ptrtoint.mir
new file mode 100644
index 0000000000000..8164b1bf872fc
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-ptrtoint.mir
@@ -0,0 +1,17 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -run-pass=aarch64-prelegalizer-combiner -verify-machineinstrs -mtriple aarch64-unknown-unknown %s -o - | FileCheck %s
+# RUN: llc -debugify-and-strip-all-safe -run-pass=aarch64-prelegalizer-combiner -verify-machineinstrs -mtriple aarch64-unknown-unknown %s -o - | FileCheck %s
+---
+name:            test_combine_ptrtoint
+body:             |
+  bb.1:
+  liveins: $x0, $x1
+
+    ; CHECK-LABEL: name: test_combine_ptrtoint
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $x0
+    ; CHECK: $x1 = COPY [[COPY]](s64)
+    %0:_(s64) = COPY $x0
+    %1:_(p64) = G_INTTOPTR %0
+    %2:_(s64) = G_PTRTOINT %1
+    $x1 = COPY %2
+...

From 3b0d30ffd30a704d09a34031d8797e22b708caab Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Fri, 31 Jul 2020 18:16:01 +0100
Subject: [PATCH 0754/1035] [SCEVExpander] Name temporary instructions for
 LCSSA insertion (NFC).

---
 llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
index 1a10e580c68c0..d31bf6791075a 100644
--- a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
+++ b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
@@ -1771,10 +1771,12 @@ Value *SCEVExpander::expandCodeForImpl(const SCEV *SH, Type *Ty, bool Root) {
       // instruction.
       Instruction *Tmp;
       if (Inst->getType()->isIntegerTy())
-        Tmp = cast<Instruction>(Builder.CreateAdd(Inst, Inst));
+        Tmp =
+            cast<Instruction>(Builder.CreateAdd(Inst, Inst, "tmp.lcssa.user"));
       else {
         assert(Inst->getType()->isPointerTy());
-        Tmp = cast<Instruction>(Builder.CreateGEP(Inst, Builder.getInt32(1)));
+        Tmp = cast<Instruction>(
+            Builder.CreateGEP(Inst, Builder.getInt32(1), "tmp.lcssa.user"));
       }
       V = fixupLCSSAFormFor(Tmp, 0);
 

From 93fd8dbdc250330b84eeca3387e895407663d750 Mon Sep 17 00:00:00 2001
From: Albion Fung <albion.fung@ibm.com>
Date: Fri, 31 Jul 2020 11:57:59 -0500
Subject: [PATCH 0755/1035] [PowerPC] Add Vector String Isolate instruction
 definitions and MC Tests

This patch implements the instruction definition and MC tests for the vector
string isolate instructions.

Differential Revision: https://reviews.llvm.org/D84197
---
 llvm/lib/Target/PowerPC/PPCInstrPrefix.td     | 41 +++++++++++++++++++
 .../PowerPC/ppc64-encoding-ISA31.txt          | 23 +++++++++++
 llvm/test/MC/PowerPC/ppc64-encoding-ISA31.s   | 21 ++++++++++
 3 files changed, 85 insertions(+)

diff --git a/llvm/lib/Target/PowerPC/PPCInstrPrefix.td b/llvm/lib/Target/PowerPC/PPCInstrPrefix.td
index 8ff0d460f47fb..a92168789bbe0 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrPrefix.td
+++ b/llvm/lib/Target/PowerPC/PPCInstrPrefix.td
@@ -59,6 +59,39 @@ class PI<bits<6> pref, bits<6> opcode, dag OOL, dag IOL, string asmstr,
   string BaseName = "";
 }
 
+// VX-Form: [ PO VT R VB RC XO ]
+class VXForm_VTB5_RC<bits<10> xo, bits<5> R, dag OOL, dag IOL, string asmstr,
+                      InstrItinClass itin, list<dag> pattern>
+  : I<4, OOL, IOL, asmstr, itin> {
+  bits<5> VT;
+  bits<5> VB;
+  bit RC = 0;
+
+  let Pattern = pattern;
+
+  let Inst{6-10} = VT;
+  let Inst{11-15} = R;
+  let Inst{16-20} = VB;
+  let Inst{21} = RC;
+  let Inst{22-31} = xo;
+}
+
+// Multiclass definition to account for record and non-record form
+// instructions of VXRForm.
+multiclass VXForm_VTB5_RCr<bits<10> xo, bits<5> R, dag OOL, dag IOL,
+                            string asmbase, string asmstr,
+                            InstrItinClass itin, list<dag> pattern> {
+  let BaseName = asmbase in {
+    def NAME : VXForm_VTB5_RC<xo, R, OOL, IOL,
+                               !strconcat(asmbase, !strconcat(" ", asmstr)),
+                               itin, pattern>, RecFormRel;
+    let Defs = [CR6] in
+    def _rec : VXForm_VTB5_RC<xo, R, OOL, IOL,
+                               !strconcat(asmbase, !strconcat(". ", asmstr)),
+                               itin, []>, isRecordForm, RecFormRel;
+  }
+}
+
 class MLS_DForm_R_SI34_RTA5_MEM<bits<6> opcode, dag OOL, dag IOL, string asmstr,
                                 InstrItinClass itin, list<dag> pattern>
   : PI<1, opcode, OOL, IOL, asmstr, itin> {
@@ -822,6 +855,14 @@ let Predicates = [IsISA3_1] in {
                                       (int_ppc_altivec_vsrdbi v16i8:$VRA,
                                                               v16i8:$VRB, 
                                                               i32:$SH))]>;
+  defm VSTRIBR : VXForm_VTB5_RCr<13, 1, (outs vrrc:$vT), (ins vrrc:$vB),
+                                 "vstribr", "$vT, $vB", IIC_VecGeneral, []>;
+  defm VSTRIBL : VXForm_VTB5_RCr<13, 0, (outs vrrc:$vT), (ins vrrc:$vB),
+                                 "vstribl", "$vT, $vB", IIC_VecGeneral, []>;
+  defm VSTRIHR : VXForm_VTB5_RCr<13, 3, (outs vrrc:$vT), (ins vrrc:$vB),
+                                 "vstrihr", "$vT, $vB", IIC_VecGeneral, []>;
+  defm VSTRIHL : VXForm_VTB5_RCr<13, 2, (outs vrrc:$vT), (ins vrrc:$vB),
+                                 "vstrihl", "$vT, $vB", IIC_VecGeneral, []>;
   def VINSW :
     VXForm_1<207, (outs vrrc:$vD), (ins vrrc:$vDi, u4imm:$UIM, gprc:$rB),
              "vinsw $vD, $rB, $UIM", IIC_VecGeneral,
diff --git a/llvm/test/MC/Disassembler/PowerPC/ppc64-encoding-ISA31.txt b/llvm/test/MC/Disassembler/PowerPC/ppc64-encoding-ISA31.txt
index c8dae6a160a5b..569068ab21a7a 100644
--- a/llvm/test/MC/Disassembler/PowerPC/ppc64-encoding-ISA31.txt
+++ b/llvm/test/MC/Disassembler/PowerPC/ppc64-encoding-ISA31.txt
@@ -459,3 +459,26 @@
 # CHECK: xscvsqqp 8, 28
 0xfd 0xb 0xe6 0x88
 
+# CHECK: vstribr 2, 2
+0x10 0x41 0x10 0x0d
+
+# CHECK: vstribl 2, 2
+0x10 0x40 0x10 0x0d
+
+# CHECK: vstrihr 2, 2
+0x10 0x43 0x10 0x0d
+
+# CHECK: vstrihl 2, 2
+0x10 0x42 0x10 0x0d
+
+# CHECK: vstribr. 2, 2
+0x10 0x41 0x14 0x0d
+
+# CHECK: vstribl. 2, 2
+0x10 0x40 0x14 0x0d
+
+# CHECK: vstrihr. 2, 2
+0x10 0x43 0x14 0x0d
+
+# CHECK: vstrihl. 2, 2
+0x10 0x42 0x14 0x0d
diff --git a/llvm/test/MC/PowerPC/ppc64-encoding-ISA31.s b/llvm/test/MC/PowerPC/ppc64-encoding-ISA31.s
index bd1187f18ed8f..6968821a3c09b 100644
--- a/llvm/test/MC/PowerPC/ppc64-encoding-ISA31.s
+++ b/llvm/test/MC/PowerPC/ppc64-encoding-ISA31.s
@@ -585,3 +585,24 @@
 # CHECK-BE: xscvsqqp 8, 28                        # encoding: [0xfd,0x0b,0xe6,0x88]
 # CHECK-LE: xscvsqqp 8, 28                        # encoding: [0x88,0xe6,0x0b,0xfd]
             xscvsqqp 8, 28
+# CHECK-BE: vstribr 2, 2                          # encoding: [0x10,0x41,0x10,0x0d]
+# CHECK-LE: vstribr 2, 2                          # encoding: [0x0d,0x10,0x41,0x10]
+            vstribr 2, 2
+# CHECK-BE: vstribl 2, 2                          # encoding: [0x10,0x40,0x10,0x0d]
+# CHECK-LE: vstribl 2, 2                          # encoding: [0x0d,0x10,0x40,0x10]
+            vstribl 2, 2
+# CHECK-BE: vstrihr 2, 2                          # encoding: [0x10,0x43,0x10,0x0d]
+# CHECK-LE: vstrihr 2, 2                          # encoding: [0x0d,0x10,0x43,0x10]
+            vstrihr 2, 2
+# CHECK-BE: vstribr. 2, 2                         # encoding: [0x10,0x41,0x14,0x0d]
+# CHECK-LE: vstribr. 2, 2                         # encoding: [0x0d,0x14,0x41,0x10]
+            vstribr. 2, 2
+# CHECK-BE: vstribl. 2, 2                         # encoding: [0x10,0x40,0x14,0x0d]
+# CHECK-LE: vstribl. 2, 2                         # encoding: [0x0d,0x14,0x40,0x10]
+            vstribl. 2, 2
+# CHECK-BE: vstrihr. 2, 2                         # encoding: [0x10,0x43,0x14,0x0d]
+# CHECK-LE: vstrihr. 2, 2                         # encoding: [0x0d,0x14,0x43,0x10]
+            vstrihr. 2, 2
+# CHECK-BE: vstrihl. 2, 2                         # encoding: [0x10,0x42,0x14,0x0d]
+# CHECK-LE: vstrihl. 2, 2                         # encoding: [0x0d,0x14,0x42,0x10]
+            vstrihl. 2, 2

From c068e9c8c123e7f8c8f3feb57245a012ccd09ccf Mon Sep 17 00:00:00 2001
From: Fangrui Song <maskray@google.com>
Date: Fri, 31 Jul 2020 10:46:27 -0700
Subject: [PATCH 0756/1035] [Support][CommandLine] Delete unused
 llvm::cl::ParseEnvrironmentOptions

The function was added in 2003. It is not used and can be emulated with ParseCommandLineOptions.
---
 llvm/docs/CommandLine.rst                  | 23 ----------------
 llvm/include/llvm/Support/CommandLine.h    |  7 -----
 llvm/lib/Support/CommandLine.cpp           | 30 --------------------
 llvm/unittests/Support/CommandLineTest.cpp | 32 ----------------------
 4 files changed, 92 deletions(-)

diff --git a/llvm/docs/CommandLine.rst b/llvm/docs/CommandLine.rst
index ab2826d789f2f..431ebc0e67e67 100644
--- a/llvm/docs/CommandLine.rst
+++ b/llvm/docs/CommandLine.rst
@@ -1369,29 +1369,6 @@ The ``cl::ParseCommandLineOptions`` function requires two parameters (``argc``
 and ``argv``), but may also take an optional third parameter which holds
 `additional extra text`_ to emit when the ``-help`` option is invoked.
 
-.. _cl::ParseEnvironmentOptions:
-
-The ``cl::ParseEnvironmentOptions`` function
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-The ``cl::ParseEnvironmentOptions`` function has mostly the same effects as
-`cl::ParseCommandLineOptions`_, except that it is designed to take values for
-options from an environment variable, for those cases in which reading the
-command line is not convenient or desired. It fills in the values of all the
-command line option variables just like `cl::ParseCommandLineOptions`_ does.
-
-It takes four parameters: the name of the program (since ``argv`` may not be
-available, it can't just look in ``argv[0]``), the name of the environment
-variable to examine, and the optional `additional extra text`_ to emit when the
-``-help`` option is invoked.
-
-``cl::ParseEnvironmentOptions`` will break the environment variable's value up
-into words and then process them using `cl::ParseCommandLineOptions`_.
-**Note:** Currently ``cl::ParseEnvironmentOptions`` does not support quoting, so
-an environment variable containing ``-option "foo bar"`` will be parsed as three
-words, ``-option``, ``"foo``, and ``bar"``, which is different from what you
-would get from the shell with the same input.
-
 The ``cl::SetVersionPrinter`` function
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
diff --git a/llvm/include/llvm/Support/CommandLine.h b/llvm/include/llvm/Support/CommandLine.h
index 466945e40a9ce..62e44aeefe9cf 100644
--- a/llvm/include/llvm/Support/CommandLine.h
+++ b/llvm/include/llvm/Support/CommandLine.h
@@ -71,13 +71,6 @@ bool ParseCommandLineOptions(int argc, const char *const *argv,
                              const char *EnvVar = nullptr,
                              bool LongOptionsUseDoubleDash = false);
 
-//===----------------------------------------------------------------------===//
-// ParseEnvironmentOptions - Environment variable option processing alternate
-//                           entry point.
-//
-void ParseEnvironmentOptions(const char *progName, const char *envvar,
-                             const char *Overview = "");
-
 // Function pointer type for printing version information.
 using VersionPrinterTy = std::function<void(raw_ostream &)>;
 
diff --git a/llvm/lib/Support/CommandLine.cpp b/llvm/lib/Support/CommandLine.cpp
index 12ef0d511b147..4fba6a9ada2c0 100644
--- a/llvm/lib/Support/CommandLine.cpp
+++ b/llvm/lib/Support/CommandLine.cpp
@@ -1271,36 +1271,6 @@ bool cl::readConfigFile(StringRef CfgFile, StringSaver &Saver,
                              /*MarkEOLs*/ false, /*RelativeNames*/ true);
 }
 
-/// ParseEnvironmentOptions - An alternative entry point to the
-/// CommandLine library, which allows you to read the program's name
-/// from the caller (as PROGNAME) and its command-line arguments from
-/// an environment variable (whose name is given in ENVVAR).
-///
-void cl::ParseEnvironmentOptions(const char *progName, const char *envVar,
-                                 const char *Overview) {
-  // Check args.
-  assert(progName && "Program name not specified");
-  assert(envVar && "Environment variable name missing");
-
-  // Get the environment variable they want us to parse options out of.
-  llvm::Optional<std::string> envValue = sys::Process::GetEnv(StringRef(envVar));
-  if (!envValue)
-    return;
-
-  // Get program's "name", which we wouldn't know without the caller
-  // telling us.
-  SmallVector<const char *, 20> newArgv;
-  BumpPtrAllocator A;
-  StringSaver Saver(A);
-  newArgv.push_back(Saver.save(progName).data());
-
-  // Parse the value of the environment variable into a "command line"
-  // and hand it off to ParseCommandLineOptions().
-  TokenizeGNUCommandLine(*envValue, Saver, newArgv);
-  int newArgc = static_cast<int>(newArgv.size());
-  ParseCommandLineOptions(newArgc, &newArgv[0], StringRef(Overview));
-}
-
 bool cl::ParseCommandLineOptions(int argc, const char *const *argv,
                                  StringRef Overview, raw_ostream *Errs,
                                  const char *EnvVar,
diff --git a/llvm/unittests/Support/CommandLineTest.cpp b/llvm/unittests/Support/CommandLineTest.cpp
index e8c2cef18e367..be8217b109627 100644
--- a/llvm/unittests/Support/CommandLineTest.cpp
+++ b/llvm/unittests/Support/CommandLineTest.cpp
@@ -45,8 +45,6 @@ class TempEnvVar {
     EXPECT_EQ(nullptr, old_value) << old_value;
 #if HAVE_SETENV
     setenv(name, value, true);
-#else
-#   define SKIP_ENVIRONMENT_TESTS
 #endif
   }
 
@@ -137,36 +135,6 @@ TEST(CommandLineTest, ModifyExisitingOption) {
   ASSERT_EQ(cl::Hidden, TestOption.getOptionHiddenFlag()) <<
     "Failed to modify option's hidden flag.";
 }
-#ifndef SKIP_ENVIRONMENT_TESTS
-
-const char test_env_var[] = "LLVM_TEST_COMMAND_LINE_FLAGS";
-
-cl::opt<std::string> EnvironmentTestOption("env-test-opt");
-TEST(CommandLineTest, ParseEnvironment) {
-  TempEnvVar TEV(test_env_var, "-env-test-opt=hello");
-  EXPECT_EQ("", EnvironmentTestOption);
-  cl::ParseEnvironmentOptions("CommandLineTest", test_env_var);
-  EXPECT_EQ("hello", EnvironmentTestOption);
-}
-
-// This test used to make valgrind complain
-// ("Conditional jump or move depends on uninitialised value(s)")
-//
-// Warning: Do not run any tests after this one that try to gain access to
-// registered command line options because this will likely result in a
-// SEGFAULT. This can occur because the cl::opt in the test below is declared
-// on the stack which will be destroyed after the test completes but the
-// command line system will still hold a pointer to a deallocated cl::Option.
-TEST(CommandLineTest, ParseEnvironmentToLocalVar) {
-  // Put cl::opt on stack to check for proper initialization of fields.
-  StackOption<std::string> EnvironmentTestOptionLocal("env-test-opt-local");
-  TempEnvVar TEV(test_env_var, "-env-test-opt-local=hello-local");
-  EXPECT_EQ("", EnvironmentTestOptionLocal);
-  cl::ParseEnvironmentOptions("CommandLineTest", test_env_var);
-  EXPECT_EQ("hello-local", EnvironmentTestOptionLocal);
-}
-
-#endif  // SKIP_ENVIRONMENT_TESTS
 
 TEST(CommandLineTest, UseOptionCategory) {
   StackOption<int> TestOption2("test-option", cl::cat(TestCategory));

From 1479cdfe4ff603e7b0140dab3ca08ff095473cbd Mon Sep 17 00:00:00 2001
From: Teresa Johnson <tejohnson@google.com>
Date: Thu, 30 Jul 2020 13:49:49 -0700
Subject: [PATCH 0757/1035] [ThinLTO] Compile time improvement to
 propagateAttributes

I found that propagateAttributes was ~23% of a thin link's run time
(almost 4x higher than the second hottest function). The main reason is
that it re-examines a global var each time it is referenced. This
becomes unnecessary once it is marked both non read only and non write
only. I added a set to avoid doing redundant work, which dropped the
runtime of that thin link by almost 15%.

I made a smaller efficiency improvement (no measurable impact) to skip
all summaries for a VI if the first copy is dead. I added an assert to
ensure that all copies are dead if any is. The code in
computeDeadSymbols marks all summaries for a VI as live. There is one
corner case where it was skipping marking an alias as live, that I
fixed. However, since the code earlier marked all copies of a preserved
GUID's VI as live, and each 'visit' marks all copies live, the only case
where this could make a difference is summaries that were marked live
when they were built initially, and that is only a few special compiler
generated symbols and inline assembly symbols, so it likely is never
provoked in practice.

Differential Revision: https://reviews.llvm.org/D84985
---
 llvm/lib/IR/ModuleSummaryIndex.cpp         | 28 ++++++++++++++++++----
 llvm/lib/Transforms/IPO/FunctionImport.cpp |  3 +--
 2 files changed, 25 insertions(+), 6 deletions(-)

diff --git a/llvm/lib/IR/ModuleSummaryIndex.cpp b/llvm/lib/IR/ModuleSummaryIndex.cpp
index 91612eafada73..5346323ceabba 100644
--- a/llvm/lib/IR/ModuleSummaryIndex.cpp
+++ b/llvm/lib/IR/ModuleSummaryIndex.cpp
@@ -163,7 +163,9 @@ bool ModuleSummaryIndex::isGUIDLive(GlobalValue::GUID GUID) const {
   return false;
 }
 
-static void propagateAttributesToRefs(GlobalValueSummary *S) {
+static void
+propagateAttributesToRefs(GlobalValueSummary *S,
+                          DenseSet<ValueInfo> &MarkedNonReadWriteOnly) {
   // If reference is not readonly or writeonly then referenced summary is not
   // read/writeonly either. Note that:
   // - All references from GlobalVarSummary are conservatively considered as
@@ -174,6 +176,11 @@ static void propagateAttributesToRefs(GlobalValueSummary *S) {
   //   for them.
   for (auto &VI : S->refs()) {
     assert(VI.getAccessSpecifier() == 0 || isa<FunctionSummary>(S));
+    if (!VI.getAccessSpecifier()) {
+      if (!MarkedNonReadWriteOnly.insert(VI).second)
+        continue;
+    } else if (MarkedNonReadWriteOnly.find(VI) != MarkedNonReadWriteOnly.end())
+      continue;
     for (auto &Ref : VI.getSummaryList())
       // If references to alias is not read/writeonly then aliasee
       // is not read/writeonly
@@ -216,11 +223,24 @@ void ModuleSummaryIndex::propagateAttributes(
     const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) {
   if (!PropagateAttrs)
     return;
+  DenseSet<ValueInfo> MarkedNonReadWriteOnly;
   for (auto &P : *this)
     for (auto &S : P.second.SummaryList) {
-      if (!isGlobalValueLive(S.get()))
+      if (!isGlobalValueLive(S.get())) {
+        // computeDeadSymbols should have marked all copies live. Note that
+        // it is possible that there is a GUID collision between internal
+        // symbols with the same name in different files of the same name but
+        // not enough distinguishing path. Because computeDeadSymbols should
+        // conservatively mark all copies live we can assert here that all are
+        // dead if any copy is dead.
+        assert(llvm::none_of(
+            P.second.SummaryList,
+            [&](const std::unique_ptr<GlobalValueSummary> &Summary) {
+              return isGlobalValueLive(Summary.get());
+            }));
         // We don't examine references from dead objects
-        continue;
+        break;
+      }
 
       // Global variable can't be marked read/writeonly if it is not eligible
       // to import since we need to ensure that all external references get
@@ -240,7 +260,7 @@ void ModuleSummaryIndex::propagateAttributes(
           GVS->setReadOnly(false);
           GVS->setWriteOnly(false);
         }
-      propagateAttributesToRefs(S.get());
+      propagateAttributesToRefs(S.get(), MarkedNonReadWriteOnly);
     }
   setWithAttributePropagation();
   if (llvm::AreStatisticsEnabled())
diff --git a/llvm/lib/Transforms/IPO/FunctionImport.cpp b/llvm/lib/Transforms/IPO/FunctionImport.cpp
index 468bf19f2e48a..e02f8d62da7a0 100644
--- a/llvm/lib/Transforms/IPO/FunctionImport.cpp
+++ b/llvm/lib/Transforms/IPO/FunctionImport.cpp
@@ -884,6 +884,7 @@ void llvm::computeDeadSymbols(
   while (!Worklist.empty()) {
     auto VI = Worklist.pop_back_val();
     for (auto &Summary : VI.getSummaryList()) {
+      Summary->setLive(true);
       if (auto *AS = dyn_cast<AliasSummary>(Summary.get())) {
         // If this is an alias, visit the aliasee VI to ensure that all copies
         // are marked live and it is added to the worklist for further
@@ -891,8 +892,6 @@ void llvm::computeDeadSymbols(
         visit(AS->getAliaseeVI(), true);
         continue;
       }
-
-      Summary->setLive(true);
       for (auto Ref : Summary->refs())
         visit(Ref, false);
       if (auto *FS = dyn_cast<FunctionSummary>(Summary.get()))

From 162e9f72ac57342a0ea12036576d6212720cc93f Mon Sep 17 00:00:00 2001
From: Jonas Devlieghere <jonas@devlieghere.com>
Date: Fri, 31 Jul 2020 10:52:49 -0700
Subject: [PATCH 0758/1035] [lldb/Test] Reduce code duplication by importing
 subprocess globally

Import the subprocess module once instead of doing it inline which is
error prone and leads to needless code duplication.
---
 lldb/test/API/lit.cfg.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/lldb/test/API/lit.cfg.py b/lldb/test/API/lit.cfg.py
index a211abe6cc41c..e083e2fd9bebb 100644
--- a/lldb/test/API/lit.cfg.py
+++ b/lldb/test/API/lit.cfg.py
@@ -6,6 +6,7 @@
 import platform
 import shlex
 import shutil
+import subprocess
 
 import lit.formats
 
@@ -33,7 +34,6 @@ def mkdir_p(path):
 
 
 def find_sanitizer_runtime(name):
-  import subprocess
   resource_dir = subprocess.check_output(
       [config.cmake_cxx_compiler,
        '-print-resource-dir']).decode('utf-8').strip()
@@ -60,7 +60,6 @@ def find_python_interpreter():
     return copied_python
 
   # Find the "real" python binary.
-  import shutil, subprocess
   real_python = subprocess.check_output([
       config.python_executable,
       os.path.join(os.path.dirname(os.path.realpath(__file__)),

From 49660234db94faf54c2d0ffa150841b9b52671c9 Mon Sep 17 00:00:00 2001
From: sameeran joshi <sameeranjayant.joshi@amd.com>
Date: Fri, 31 Jul 2020 01:02:04 +0530
Subject: [PATCH 0759/1035] [Flang] Checks for constraint C7110-C7115.

Added more tests.
	 Annotate sources and tests.
	 Improve error message.

Reviewed By: PeteSteinfeld

Differential Revision: https://reviews.llvm.org/D85014
---
 flang/lib/Semantics/expression.cpp           | 21 ++++---
 flang/test/Semantics/array-constr-values.f90 | 60 ++++++++++++++++++++
 2 files changed, 73 insertions(+), 8 deletions(-)
 create mode 100644 flang/test/Semantics/array-constr-values.f90

diff --git a/flang/lib/Semantics/expression.cpp b/flang/lib/Semantics/expression.cpp
index e78c2d20edcf7..3ec2af5d70fa8 100644
--- a/flang/lib/Semantics/expression.cpp
+++ b/flang/lib/Semantics/expression.cpp
@@ -1202,6 +1202,7 @@ class ArrayConstructorContext {
   bool explicitType_{type_.has_value()};
   std::optional<std::int64_t> constantLength_;
   ArrayConstructorValues<SomeType> values_;
+  bool messageDisplayedOnce{false};
 };
 
 void ArrayConstructorContext::Push(MaybeExpr &&x) {
@@ -1252,17 +1253,21 @@ void ArrayConstructorContext::Push(MaybeExpr &&x) {
           }
         }
       } else {
-        exprAnalyzer_.Say(
-            "Values in array constructor must have the same declared type "
-            "when no explicit type appears"_err_en_US);
+        if (!messageDisplayedOnce) {
+          exprAnalyzer_.Say(
+              "Values in array constructor must have the same declared type "
+              "when no explicit type appears"_err_en_US); // C7110
+          messageDisplayedOnce = true;
+        }
       }
     } else {
       if (auto cast{ConvertToType(*type_, std::move(*x))}) {
         values_.Push(std::move(*cast));
       } else {
         exprAnalyzer_.Say(
-            "Value in array constructor could not be converted to the type "
-            "of the array"_err_en_US);
+            "Value in array constructor of type '%s' could not "
+            "be converted to the type of the array '%s'"_err_en_US,
+            x->GetType()->AsFortran(), type_->AsFortran()); // C7111, C7112
       }
     }
   }
@@ -1304,7 +1309,7 @@ void ArrayConstructorContext::Add(const parser::AcValue &x) {
                 if (exprType->IsUnlimitedPolymorphic()) {
                   exprAnalyzer_.Say(
                       "Cannot have an unlimited polymorphic value in an "
-                      "array constructor"_err_en_US);
+                      "array constructor"_err_en_US); // C7113
                 }
               }
               Push(std::move(*v));
@@ -1346,7 +1351,7 @@ void ArrayConstructorContext::Add(const parser::AcValue &x) {
             } else {
               exprAnalyzer_.SayAt(name,
                   "Implied DO index is active in surrounding implied DO loop "
-                  "and may not have the same name"_err_en_US);
+                  "and may not have the same name"_err_en_US); // C7115
             }
           },
       },
@@ -1386,7 +1391,7 @@ MaybeExpr ExpressionAnalyzer::Analyze(
                           "ABSTRACT derived type '%s' may not be used in a "
                           "structure constructor"_err_en_US,
                           typeName),
-        typeSymbol);
+        typeSymbol); // C7114
   }
 
   // This iterator traverses all of the components in the derived type and its
diff --git a/flang/test/Semantics/array-constr-values.f90 b/flang/test/Semantics/array-constr-values.f90
new file mode 100644
index 0000000000000..30739f8c095bc
--- /dev/null
+++ b/flang/test/Semantics/array-constr-values.f90
@@ -0,0 +1,60 @@
+! RUN: %S/test_errors.sh %s %t %f18
+! Confirm enforcement of constraints and restrictions in 7.8
+! C7110, C7111, C7112, C7113, C7114, C7115
+
+subroutine arrayconstructorvalues()
+  integer :: intarray(5)
+  integer(KIND=8) :: k8 = 20
+
+  TYPE EMPLOYEE
+    INTEGER AGE
+    CHARACTER (LEN = 30) NAME
+  END TYPE EMPLOYEE
+  TYPE EMPLOYEER
+    CHARACTER (LEN = 30) NAME
+  END TYPE EMPLOYEER
+
+  TYPE(EMPLOYEE) :: emparray(3)
+  class(*), pointer :: unlim_polymorphic
+  TYPE, ABSTRACT :: base_type
+    INTEGER :: CARPRIZE
+  END TYPE
+  ! Different declared type
+  !ERROR: Values in array constructor must have the same declared type when no explicit type appears
+  intarray = (/ 1, 2, 3, 4., 5/)  ! C7110
+  ! Different kind type parameter
+  !ERROR: Values in array constructor must have the same declared type when no explicit type appears
+  intarray = (/ 1,2,3,4, k8 /)    ! C7110
+
+  ! C7111
+  !ERROR: Value in array constructor of type 'LOGICAL(4)' could not be converted to the type of the array 'INTEGER(4)'
+  intarray = [integer:: .true., 2, 3, 4, 5]
+  !ERROR: Value in array constructor of type 'CHARACTER(1)' could not be converted to the type of the array 'INTEGER(4)'
+  intarray = [integer:: "RAM stores information", 2, 3, 4, 5]
+  !ERROR: Value in array constructor of type 'employee' could not be converted to the type of the array 'INTEGER(4)'
+  intarray = [integer:: EMPLOYEE (19, "Jack"), 2, 3, 4, 5]
+
+  ! C7112
+  !ERROR: Value in array constructor of type 'INTEGER(4)' could not be converted to the type of the array 'employee'
+  emparray = (/ EMPLOYEE:: EMPLOYEE(19, "Ganesh"), EMPLOYEE(22, "Omkar"), 19 /)
+  !ERROR: Value in array constructor of type 'employeer' could not be converted to the type of the array 'employee'
+  emparray = (/ EMPLOYEE:: EMPLOYEE(19, "Ganesh"), EMPLOYEE(22, "Ram"),EMPLOYEER("ShriniwasPvtLtd") /)
+
+  ! C7113
+  !ERROR: Cannot have an unlimited polymorphic value in an array constructor
+  !ERROR: Values in array constructor must have the same declared type when no explicit type appears
+  intarray = (/ unlim_polymorphic, 2, 3, 4, 5/)
+
+  ! C7114
+  !ERROR: No intrinsic or user-defined ASSIGNMENT(=) matches operand types INTEGER(4) and TYPE(base_type)
+  !ERROR: ABSTRACT derived type 'base_type' may not be used in a structure constructor
+  !ERROR: Values in array constructor must have the same declared type when no explicit type appears
+  intarray = (/ base_type(10), 2, 3, 4, 5 /)
+end subroutine arrayconstructorvalues
+subroutine checkC7115()
+  real, dimension(10), parameter :: good1 = [(99.9, i = 1, 10)]
+  real, dimension(100), parameter :: good2 = [((88.8, i = 1, 10), j = 1, 10)]
+  !ERROR: Implied DO index is active in surrounding implied DO loop and may not have the same name
+  !ERROR: 'i' is already declared in this scoping unit
+  real, dimension(100), parameter :: bad = [((88.8, i = 1, 10), i = 1, 10)]
+end subroutine checkC7115

From 0e0aebc5273c6d676584ecb50ed49e888029c2de Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@intel.com>
Date: Fri, 31 Jul 2020 10:43:00 -0700
Subject: [PATCH 0760/1035] [ValueTracking] Add ComputeNumSignBits support for
 llvm.abs intrinsic

If absolute value needs turn a negative number into a positive number it reduces the number of sign bits by at most 1.

Differential Revision: https://reviews.llvm.org/D84971
---
 llvm/lib/Analysis/ValueTracking.cpp               | 13 +++++++++++++
 llvm/test/Transforms/InstCombine/abs-intrinsic.ll | 15 +++++++++++++++
 2 files changed, 28 insertions(+)

diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index 52132072a7d1e..4cd2d07bf4b0c 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -3001,6 +3001,19 @@ static unsigned ComputeNumSignBitsImpl(const Value *V,
              "Failed to determine minimum sign bits");
       return Tmp;
     }
+    case Instruction::Call: {
+      if (const auto *II = dyn_cast<IntrinsicInst>(U)) {
+        switch (II->getIntrinsicID()) {
+        default: break;
+        case Intrinsic::abs:
+          Tmp = ComputeNumSignBits(U->getOperand(0), Depth + 1, Q);
+          if (Tmp == 1) break;
+
+          // Absolute value reduces number of sign bits by at most 1.
+          return Tmp - 1;
+        }
+      }
+    }
     }
   }
 
diff --git a/llvm/test/Transforms/InstCombine/abs-intrinsic.ll b/llvm/test/Transforms/InstCombine/abs-intrinsic.ll
index 780f28238bf60..8b965b3d1e440 100644
--- a/llvm/test/Transforms/InstCombine/abs-intrinsic.ll
+++ b/llvm/test/Transforms/InstCombine/abs-intrinsic.ll
@@ -51,3 +51,18 @@ define i32 @abs_trailing_zeros_negative(i32 %x) {
   %and2 = and i32 %abs, -4
   ret i32 %and2
 }
+
+; Make sure we infer this add doesn't overflow. The input to the abs has 3
+; sign bits, the abs reduces this to 2 sign bits.
+define i32 @abs_signbits(i30 %x) {
+; CHECK-LABEL: @abs_signbits(
+; CHECK-NEXT:    [[AND:%.*]] = sext i30 [[X:%.*]] to i32
+; CHECK-NEXT:    [[ABS:%.*]] = call i32 @llvm.abs.i32(i32 [[AND]], i1 false)
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[ABS]], 1
+; CHECK-NEXT:    ret i32 [[ADD]]
+;
+  %ext = sext i30 %x to i32
+  %abs = call i32 @llvm.abs.i32(i32 %ext, i1 false)
+  %add = add i32 %abs, 1
+  ret i32 %add
+}

From 18eba165e7ba80328a910cad3407599d8ff60f4f Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project@meinersbur.de>
Date: Fri, 31 Jul 2020 13:00:06 -0500
Subject: [PATCH 0761/1035] [OpenMP][docs] Update loop tiling status.

---
 clang/docs/OpenMPSupport.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/docs/OpenMPSupport.rst b/clang/docs/OpenMPSupport.rst
index 28fbd7baebb24..af5e538b1435a 100644
--- a/clang/docs/OpenMPSupport.rst
+++ b/clang/docs/OpenMPSupport.rst
@@ -266,7 +266,7 @@ want to help with the implementation.
 +------------------------------+--------------------------------------------------------------+--------------------------+-----------------------------------------------------------------------+
 | misc extension               | default(firstprivate) & default(private)                     | :part:`partial`          | firstprivate done: D75591                                             |
 +------------------------------+--------------------------------------------------------------+--------------------------+-----------------------------------------------------------------------+
-| loop extension               | Loop tiling transformation                                   | :part:`claimed`          |                                                                       |
+| loop extension               | Loop tiling transformation                                   | :part:`worked on`        | D76342                                                                |
 +------------------------------+--------------------------------------------------------------+--------------------------+-----------------------------------------------------------------------+
 | device extension             | 'present' map type modifier                                  | :part:`mostly done`      | D83061, D83062, D84422                                                |
 +------------------------------+--------------------------------------------------------------+--------------------------+-----------------------------------------------------------------------+

From 46591b95362325d262ca29ce13e7b5ddda624bc8 Mon Sep 17 00:00:00 2001
From: kristina <notstina@gmail.com>
Date: Fri, 31 Jul 2020 18:44:02 +0100
Subject: [PATCH 0762/1035] [libunwind] Add -Wno-suggest-override to
 CMakeLists.txt.

Set -Wno-suggest-override where such warning is provided
by the compiler when building libunwind, alongside libcxx
and libcxxabi, using recent Clang. This extends behavior
introduced in 77e0e9e17daf0865620abcd41f692ab0642367c4
to libunwind, avoiding a large amount of warnings during
builds. See D84126 for the original patch.
---
 libunwind/CMakeLists.txt | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/libunwind/CMakeLists.txt b/libunwind/CMakeLists.txt
index 4606360f07ab7..8419d851ab7f4 100644
--- a/libunwind/CMakeLists.txt
+++ b/libunwind/CMakeLists.txt
@@ -271,6 +271,8 @@ add_compile_flags_if_supported(-Wunused-variable)
 add_compile_flags_if_supported(-Wwrite-strings)
 add_compile_flags_if_supported(-Wundef)
 
+add_compile_flags_if_supported(-Wno-suggest-override)
+
 if (LIBUNWIND_ENABLE_WERROR)
   add_compile_flags_if_supported(-Werror)
   add_compile_flags_if_supported(-WX)

From e2d4bf6ceca84c2ff515d6bc89da7d40d1c971fb Mon Sep 17 00:00:00 2001
From: cgyurgyik <gyurgyikcp@gmail.com>
Date: Fri, 31 Jul 2020 14:04:36 -0400
Subject: [PATCH 0763/1035] [libc] Add islower and isupper implementation.

Reviewed By: sivachandra

Differential Revision: https://reviews.llvm.org/D84960
---
 libc/config/linux/aarch64/entrypoints.txt |  2 ++
 libc/config/linux/api.td                  |  2 ++
 libc/config/linux/x86_64/entrypoints.txt  |  2 ++
 libc/spec/stdc.td                         | 10 ++++++++++
 libc/src/ctype/CMakeLists.txt             | 16 ++++++++++++++++
 libc/src/ctype/islower.cpp                | 22 ++++++++++++++++++++++
 libc/src/ctype/islower.h                  | 18 ++++++++++++++++++
 libc/src/ctype/isupper.cpp                | 22 ++++++++++++++++++++++
 libc/src/ctype/isupper.h                  | 18 ++++++++++++++++++
 libc/test/src/ctype/CMakeLists.txt        | 20 ++++++++++++++++++++
 libc/test/src/ctype/isalnum_test.cpp      | 12 ++++--------
 libc/test/src/ctype/isalpha_test.cpp      | 14 +++++---------
 libc/test/src/ctype/isdigit_test.cpp      | 14 +++++---------
 libc/test/src/ctype/islower_test.cpp      | 21 +++++++++++++++++++++
 libc/test/src/ctype/isupper_test.cpp      | 21 +++++++++++++++++++++
 15 files changed, 188 insertions(+), 26 deletions(-)
 create mode 100644 libc/src/ctype/islower.cpp
 create mode 100644 libc/src/ctype/islower.h
 create mode 100644 libc/src/ctype/isupper.cpp
 create mode 100644 libc/src/ctype/isupper.h
 create mode 100644 libc/test/src/ctype/islower_test.cpp
 create mode 100644 libc/test/src/ctype/isupper_test.cpp

diff --git a/libc/config/linux/aarch64/entrypoints.txt b/libc/config/linux/aarch64/entrypoints.txt
index 8d2cc16189465..5f058e6116001 100644
--- a/libc/config/linux/aarch64/entrypoints.txt
+++ b/libc/config/linux/aarch64/entrypoints.txt
@@ -3,6 +3,8 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.ctype.isalnum
     libc.src.ctype.isalpha
     libc.src.ctype.isdigit
+    libc.src.ctype.islower
+    libc.src.ctype.isupper
     
     # errno.h entrypoints
     libc.src.errno.__errno_location
diff --git a/libc/config/linux/api.td b/libc/config/linux/api.td
index 01e6eab92f016..03d5d66e41e79 100644
--- a/libc/config/linux/api.td
+++ b/libc/config/linux/api.td
@@ -91,6 +91,8 @@ def CTypeAPI : PublicAPI<"ctype.h"> {
     "isalnum",
     "isalpha",
     "isdigit",
+    "islower",
+    "isupper",
   ];
 }
 
diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt
index 4484f6cbf1a79..5dc1d38455dcf 100644
--- a/libc/config/linux/x86_64/entrypoints.txt
+++ b/libc/config/linux/x86_64/entrypoints.txt
@@ -6,6 +6,8 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.ctype.isalnum
     libc.src.ctype.isalpha
     libc.src.ctype.isdigit
+    libc.src.ctype.islower
+    libc.src.ctype.isupper
 
     # errno.h entrypoints
     libc.src.errno.__errno_location
diff --git a/libc/spec/stdc.td b/libc/spec/stdc.td
index fed24cb2dbdd3..4fffc5cdc8579 100644
--- a/libc/spec/stdc.td
+++ b/libc/spec/stdc.td
@@ -61,6 +61,16 @@ def StdC : StandardSpec<"stdc"> {
               RetValSpec<IntType>,
               [ArgSpec<IntType>]
           >,
+          FunctionSpec<
+              "islower",
+              RetValSpec<IntType>,
+              [ArgSpec<IntType>]
+          >,
+          FunctionSpec<
+              "isupper",
+              RetValSpec<IntType>,
+              [ArgSpec<IntType>]
+          >,
       ]
   >;
      
diff --git a/libc/src/ctype/CMakeLists.txt b/libc/src/ctype/CMakeLists.txt
index 53161b8b179fd..30995ab2f714c 100644
--- a/libc/src/ctype/CMakeLists.txt
+++ b/libc/src/ctype/CMakeLists.txt
@@ -33,3 +33,19 @@ add_entrypoint_object(
   DEPENDS
     .ctype_utils
 )
+
+add_entrypoint_object(
+  islower
+  SRCS
+    islower.cpp
+  HDRS
+    islower.h
+)
+
+add_entrypoint_object(
+  isupper
+  SRCS
+    isupper.cpp
+  HDRS
+    isupper.h
+)
diff --git a/libc/src/ctype/islower.cpp b/libc/src/ctype/islower.cpp
new file mode 100644
index 0000000000000..df21355f31ac6
--- /dev/null
+++ b/libc/src/ctype/islower.cpp
@@ -0,0 +1,22 @@
+//===-- Implementation of islower------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/ctype/islower.h"
+
+#include "src/__support/common.h"
+
+namespace __llvm_libc {
+
+// TODO: Currently restricted to default locale.
+// These should be extended using locale information.
+int LLVM_LIBC_ENTRYPOINT(islower)(int c) {
+  const unsigned ch = c;
+  return (ch - 'a') < 26;
+}
+
+} // namespace __llvm_libc
diff --git a/libc/src/ctype/islower.h b/libc/src/ctype/islower.h
new file mode 100644
index 0000000000000..7643542fb7a99
--- /dev/null
+++ b/libc/src/ctype/islower.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for islower -------------------------*-C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_CTYPE_ISLOWER_H
+#define LLVM_LIBC_SRC_CTYPE_ISLOWER_H
+
+namespace __llvm_libc {
+
+int islower(int c);
+
+} // namespace __llvm_libc
+
+#endif //  LLVM_LIBC_SRC_CTYPE_ISLOWER_H
diff --git a/libc/src/ctype/isupper.cpp b/libc/src/ctype/isupper.cpp
new file mode 100644
index 0000000000000..57aed961d1e57
--- /dev/null
+++ b/libc/src/ctype/isupper.cpp
@@ -0,0 +1,22 @@
+//===-- Implementation of isupper------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/ctype/isupper.h"
+
+#include "src/__support/common.h"
+
+namespace __llvm_libc {
+
+// TODO: Currently restricted to default locale.
+// These should be extended using locale information.
+int LLVM_LIBC_ENTRYPOINT(isupper)(int c) {
+  const unsigned ch = c;
+  return (ch - 'A') < 26;
+}
+
+} // namespace __llvm_libc
diff --git a/libc/src/ctype/isupper.h b/libc/src/ctype/isupper.h
new file mode 100644
index 0000000000000..7a1f2270943a9
--- /dev/null
+++ b/libc/src/ctype/isupper.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for isupper -------------------------*-C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_CTYPE_ISUPPER_H
+#define LLVM_LIBC_SRC_CTYPE_ISUPPER_H
+
+namespace __llvm_libc {
+
+int isupper(int c);
+
+} // namespace __llvm_libc
+
+#endif //  LLVM_LIBC_SRC_CTYPE_ISUPPER_H
diff --git a/libc/test/src/ctype/CMakeLists.txt b/libc/test/src/ctype/CMakeLists.txt
index 7834746ab1d73..c9959465c697c 100644
--- a/libc/test/src/ctype/CMakeLists.txt
+++ b/libc/test/src/ctype/CMakeLists.txt
@@ -29,3 +29,23 @@ add_libc_unittest(
   DEPENDS
     libc.src.ctype.isdigit
 )
+
+add_libc_unittest(
+  islower
+  SUITE
+    libc_ctype_unittests
+  SRCS
+    islower_test.cpp
+  DEPENDS
+    libc.src.ctype.islower
+)
+
+add_libc_unittest(
+  isupper
+  SUITE
+    libc_ctype_unittests
+  SRCS
+    isupper_test.cpp
+  DEPENDS
+    libc.src.ctype.isupper
+)
diff --git a/libc/test/src/ctype/isalnum_test.cpp b/libc/test/src/ctype/isalnum_test.cpp
index 1c4ad7d3ff8ac..ca77285c5614e 100644
--- a/libc/test/src/ctype/isalnum_test.cpp
+++ b/libc/test/src/ctype/isalnum_test.cpp
@@ -7,21 +7,17 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/ctype/isalnum.h"
-#include "utils/UnitTest/Test.h"
 
-// Helper function that makes a call to isalnum a bit cleaner
-// for use with testing utilities, since it explicitly requires
-// a boolean value for EXPECT_TRUE and EXPECT_FALSE.
-bool call_isalnum(int c) { return __llvm_libc::isalnum(c); }
+#include "utils/UnitTest/Test.h"
 
 TEST(IsAlNum, DefaultLocale) {
   // Loops through all characters, verifying that numbers and letters
-  // return true and everything else returns false.
+  // return non-zero integer and everything else returns a zero.
   for (int c = 0; c < 255; ++c) {
     if (('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') ||
         ('0' <= c && c <= '9'))
-      EXPECT_TRUE(call_isalnum(c));
+      EXPECT_NE(__llvm_libc::isalnum(c), 0);
     else
-      EXPECT_FALSE(call_isalnum(c));
+      EXPECT_EQ(__llvm_libc::isalnum(c), 0);
   }
 }
diff --git a/libc/test/src/ctype/isalpha_test.cpp b/libc/test/src/ctype/isalpha_test.cpp
index 81fc7248f8714..d91219b504063 100644
--- a/libc/test/src/ctype/isalpha_test.cpp
+++ b/libc/test/src/ctype/isalpha_test.cpp
@@ -7,20 +7,16 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/ctype/isalpha.h"
-#include "utils/UnitTest/Test.h"
 
-// Helper function that makes a call to isalpha a bit cleaner
-// for use with testing utilities, since it explicitly requires
-// a boolean value for EXPECT_TRUE and EXPECT_FALSE.
-bool call_isalpha(int c) { return __llvm_libc::isalpha(c); }
+#include "utils/UnitTest/Test.h"
 
 TEST(IsAlpha, DefaultLocale) {
-  // Loops through all characters, verifying that letters return true
-  // and everything else returns false.
+  // Loops through all characters, verifying that letters return a
+  // non-zero integer and everything else returns zero.
   for (int ch = 0; ch < 255; ++ch) {
     if (('a' <= ch && ch <= 'z') || ('A' <= ch && ch <= 'Z'))
-      EXPECT_TRUE(call_isalpha(ch));
+      EXPECT_NE(__llvm_libc::isalpha(ch), 0);
     else
-      EXPECT_FALSE(call_isalpha(ch));
+      EXPECT_EQ(__llvm_libc::isalpha(ch), 0);
   }
 }
diff --git a/libc/test/src/ctype/isdigit_test.cpp b/libc/test/src/ctype/isdigit_test.cpp
index 6fea9564db674..2430a92425c4a 100644
--- a/libc/test/src/ctype/isdigit_test.cpp
+++ b/libc/test/src/ctype/isdigit_test.cpp
@@ -7,20 +7,16 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/ctype/isdigit.h"
-#include "utils/UnitTest/Test.h"
 
-// Helper function that makes a call to isdigit a bit cleaner
-// for use with testing utilities, since it explicitly requires
-// a boolean value for EXPECT_TRUE and EXPECT_FALSE.
-bool call_isdigit(int c) { return __llvm_libc::isdigit(c); }
+#include "utils/UnitTest/Test.h"
 
 TEST(IsDigit, DefaultLocale) {
-  // Loops through all characters, verifying that numbers return true
-  // and everything else returns false.
+  // Loops through all characters, verifying that numbers return a
+  // non-zero integer and everything else returns zero.
   for (int ch = 0; ch < 255; ++ch) {
     if ('0' <= ch && ch <= '9')
-      EXPECT_TRUE(call_isdigit(ch));
+      EXPECT_NE(__llvm_libc::isdigit(ch), 0);
     else
-      EXPECT_FALSE(call_isdigit(ch));
+      EXPECT_EQ(__llvm_libc::isdigit(ch), 0);
   }
 }
diff --git a/libc/test/src/ctype/islower_test.cpp b/libc/test/src/ctype/islower_test.cpp
new file mode 100644
index 0000000000000..9b38cabc67aa7
--- /dev/null
+++ b/libc/test/src/ctype/islower_test.cpp
@@ -0,0 +1,21 @@
+//===-- Unittests for islower----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/ctype/islower.h"
+#include "utils/UnitTest/Test.h"
+
+TEST(IsLower, DefaultLocale) {
+  // Loops through all characters, verifying that lowercase letters
+  // return a non-zero integer and everything else returns zero.
+  for (int ch = 0; ch < 255; ++ch) {
+    if ('a' <= ch && ch <= 'z')
+      EXPECT_NE(__llvm_libc::islower(ch), 0);
+    else
+      EXPECT_EQ(__llvm_libc::islower(ch), 0);
+  }
+}
diff --git a/libc/test/src/ctype/isupper_test.cpp b/libc/test/src/ctype/isupper_test.cpp
new file mode 100644
index 0000000000000..0a13f4e11b0e1
--- /dev/null
+++ b/libc/test/src/ctype/isupper_test.cpp
@@ -0,0 +1,21 @@
+//===-- Unittests for isupper----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/ctype/isupper.h"
+#include "utils/UnitTest/Test.h"
+
+TEST(IsUpper, DefaultLocale) {
+  // Loops through all characters, verifying that uppercase letters
+  // return a non-zero integer and everything else returns zero.
+  for (int ch = 0; ch < 255; ++ch) {
+    if ('A' <= ch && ch <= 'Z')
+      EXPECT_NE(__llvm_libc::isupper(ch), 0);
+    else
+      EXPECT_EQ(__llvm_libc::isupper(ch), 0);
+  }
+}

From 6a3b07a4bf14be32569550f2e9814d8797d27d31 Mon Sep 17 00:00:00 2001
From: Hans Wennborg <hans@chromium.org>
Date: Thu, 23 Jul 2020 11:17:16 +0200
Subject: [PATCH 0764/1035] RuntimeDyldELF: report_fatal_error instead of
 asserting for unimplemented relocations (PR46816)

This fixes the ExecutionEngine/MCJIT/stubs-sm-pic.ll test in no-asserts
builds which is set to XFAIL on some platforms like 32-bit x86. More
importantly, we probably don't want to silently error in these cases.

Differential revision: https://reviews.llvm.org/D84390
---
 .../ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp b/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp
index 7c39ddc8b1da0..7ed8a718ed3c1 100644
--- a/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp
+++ b/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp
@@ -269,7 +269,7 @@ void RuntimeDyldELF::resolveX86_64Relocation(const SectionEntry &Section,
                                              uint64_t SymOffset) {
   switch (Type) {
   default:
-    llvm_unreachable("Relocation type not implemented yet!");
+    report_fatal_error("Relocation type not implemented yet!");
     break;
   case ELF::R_X86_64_NONE:
     break;
@@ -359,7 +359,7 @@ void RuntimeDyldELF::resolveX86Relocation(const SectionEntry &Section,
   default:
     // There are other relocation types, but it appears these are the
     // only ones currently used by the LLVM ELF object writer
-    llvm_unreachable("Relocation type not implemented yet!");
+    report_fatal_error("Relocation type not implemented yet!");
     break;
   }
 }
@@ -382,7 +382,7 @@ void RuntimeDyldELF::resolveAArch64Relocation(const SectionEntry &Section,
 
   switch (Type) {
   default:
-    llvm_unreachable("Relocation type not implemented yet!");
+    report_fatal_error("Relocation type not implemented yet!");
     break;
   case ELF::R_AARCH64_ABS16: {
     uint64_t Result = Value + Addend;
@@ -721,7 +721,7 @@ void RuntimeDyldELF::resolvePPC32Relocation(const SectionEntry &Section,
   uint8_t *LocalAddress = Section.getAddressWithOffset(Offset);
   switch (Type) {
   default:
-    llvm_unreachable("Relocation type not implemented yet!");
+    report_fatal_error("Relocation type not implemented yet!");
     break;
   case ELF::R_PPC_ADDR16_LO:
     writeInt16BE(LocalAddress, applyPPClo(Value + Addend));
@@ -741,7 +741,7 @@ void RuntimeDyldELF::resolvePPC64Relocation(const SectionEntry &Section,
   uint8_t *LocalAddress = Section.getAddressWithOffset(Offset);
   switch (Type) {
   default:
-    llvm_unreachable("Relocation type not implemented yet!");
+    report_fatal_error("Relocation type not implemented yet!");
     break;
   case ELF::R_PPC64_ADDR16:
     writeInt16BE(LocalAddress, applyPPClo(Value + Addend));
@@ -835,7 +835,7 @@ void RuntimeDyldELF::resolveSystemZRelocation(const SectionEntry &Section,
   uint8_t *LocalAddress = Section.getAddressWithOffset(Offset);
   switch (Type) {
   default:
-    llvm_unreachable("Relocation type not implemented yet!");
+    report_fatal_error("Relocation type not implemented yet!");
     break;
   case ELF::R_390_PC16DBL:
   case ELF::R_390_PLT16DBL: {
@@ -890,7 +890,7 @@ void RuntimeDyldELF::resolveBPFRelocation(const SectionEntry &Section,
 
   switch (Type) {
   default:
-    llvm_unreachable("Relocation type not implemented yet!");
+    report_fatal_error("Relocation type not implemented yet!");
     break;
   case ELF::R_BPF_NONE:
     break;

From e591713bff1fdec8fe2b09d084b5cf665794cb65 Mon Sep 17 00:00:00 2001
From: Sanjay Patel <spatel@rotateright.com>
Date: Fri, 31 Jul 2020 11:50:39 -0400
Subject: [PATCH 0765/1035] [ConstantFolding] fold abs intrinsic

The handling for minimum value is similar to cttz/ctlz with 0 just above this case.

Differential Revision: https://reviews.llvm.org/D84942
---
 llvm/lib/Analysis/ConstantFolding.cpp     | 13 +++++++++++++
 llvm/test/Analysis/ConstantFolding/abs.ll | 15 +++++----------
 2 files changed, 18 insertions(+), 10 deletions(-)

diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp
index b55ed6647d355..28a30090a0093 100644
--- a/llvm/lib/Analysis/ConstantFolding.cpp
+++ b/llvm/lib/Analysis/ConstantFolding.cpp
@@ -1436,6 +1436,7 @@ bool llvm::canConstantFoldCallTo(const CallBase *Call, const Function *F) {
   case Intrinsic::launder_invariant_group:
   case Intrinsic::strip_invariant_group:
   case Intrinsic::masked_load:
+  case Intrinsic::abs:
   case Intrinsic::smax:
   case Intrinsic::smin:
   case Intrinsic::umax:
@@ -2505,6 +2506,18 @@ static Constant *ConstantFoldScalarCall2(StringRef Name,
         return ConstantInt::get(Ty, C0->countTrailingZeros());
       else
         return ConstantInt::get(Ty, C0->countLeadingZeros());
+
+    case Intrinsic::abs:
+      // Undef or minimum val operand with poison min --> undef
+      assert(C1 && "Must be constant int");
+      if (C1->isOneValue() && (!C0 || C0->isMinSignedValue()))
+        return UndefValue::get(Ty);
+
+      // Undef operand with no poison min --> 0 (sign bit must be clear)
+      if (C1->isNullValue() && !C0)
+        return Constant::getNullValue(Ty);
+
+      return ConstantInt::get(Ty, C0->abs());
     }
 
     return nullptr;
diff --git a/llvm/test/Analysis/ConstantFolding/abs.ll b/llvm/test/Analysis/ConstantFolding/abs.ll
index 24171b1d4c6e4..7b3a146e86143 100644
--- a/llvm/test/Analysis/ConstantFolding/abs.ll
+++ b/llvm/test/Analysis/ConstantFolding/abs.ll
@@ -6,8 +6,7 @@ declare <8 x i8> @llvm.abs.v8i8(<8 x i8>, i1)
 
 define i8 @undef_val_min_poison() {
 ; CHECK-LABEL: @undef_val_min_poison(
-; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.abs.i8(i8 undef, i1 true)
-; CHECK-NEXT:    ret i8 [[R]]
+; CHECK-NEXT:    ret i8 undef
 ;
   %r = call i8 @llvm.abs.i8(i8 undef, i1 true)
   ret i8 %r
@@ -15,8 +14,7 @@ define i8 @undef_val_min_poison() {
 
 define i8 @undef_val_min_not_poison() {
 ; CHECK-LABEL: @undef_val_min_not_poison(
-; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.abs.i8(i8 undef, i1 false)
-; CHECK-NEXT:    ret i8 [[R]]
+; CHECK-NEXT:    ret i8 0
 ;
   %r = call i8 @llvm.abs.i8(i8 undef, i1 false)
   ret i8 %r
@@ -24,8 +22,7 @@ define i8 @undef_val_min_not_poison() {
 
 define i8 @min_val_min_poison() {
 ; CHECK-LABEL: @min_val_min_poison(
-; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.abs.i8(i8 -128, i1 true)
-; CHECK-NEXT:    ret i8 [[R]]
+; CHECK-NEXT:    ret i8 undef
 ;
   %r = call i8 @llvm.abs.i8(i8 -128, i1 true)
   ret i8 %r
@@ -33,8 +30,7 @@ define i8 @min_val_min_poison() {
 
 define i8 @min_val_min_not_poison() {
 ; CHECK-LABEL: @min_val_min_not_poison(
-; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.abs.i8(i8 -128, i1 false)
-; CHECK-NEXT:    ret i8 [[R]]
+; CHECK-NEXT:    ret i8 -128
 ;
   %r = call i8 @llvm.abs.i8(i8 -128, i1 false)
   ret i8 %r
@@ -42,8 +38,7 @@ define i8 @min_val_min_not_poison() {
 
 define <8 x i8> @vec_const() {
 ; CHECK-LABEL: @vec_const(
-; CHECK-NEXT:    [[R:%.*]] = call <8 x i8> @llvm.abs.v8i8(<8 x i8> <i8 -127, i8 -126, i8 -42, i8 -1, i8 0, i8 1, i8 42, i8 127>, i1 true)
-; CHECK-NEXT:    ret <8 x i8> [[R]]
+; CHECK-NEXT:    ret <8 x i8> <i8 127, i8 126, i8 42, i8 1, i8 0, i8 1, i8 42, i8 127>
 ;
   %r = call <8 x i8> @llvm.abs.v8i8(<8 x i8> <i8 -127, i8 -126, i8 -42, i8 -1, i8 0, i8 1, i8 42, i8 127>, i1 1)
   ret <8 x i8> %r

From 765b81f6b93f747bbca57e9042bdd3ce9fea5c5b Mon Sep 17 00:00:00 2001
From: Tim Keith <tkeith@nvidia.com>
Date: Fri, 31 Jul 2020 11:10:44 -0700
Subject: [PATCH 0766/1035] Revert "[flang] Fix multi-config generator builds."

This reverts commit 332170356e35ea9cdc2c1d612b61a50ec5ea322e.

The change breaks out-of-tree builds.
Discussion in https://reviews.llvm.org/D84022
---
 flang/CMakeLists.txt           |  4 +++-
 flang/test/lit.cfg.py          |  8 +++++++-
 flang/test/lit.site.cfg.py.in  |  1 +
 flang/tools/f18/CMakeLists.txt | 10 ++--------
 4 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/flang/CMakeLists.txt b/flang/CMakeLists.txt
index 337311fc0a63d..f1aaa5c6473fe 100644
--- a/flang/CMakeLists.txt
+++ b/flang/CMakeLists.txt
@@ -180,7 +180,7 @@ else()
          ${LLVM_INCLUDE_TESTS})
   set(FLANG_GTEST_AVAIL 1)
 
-  set(FLANG_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR})
+  set(FLANG_BINARY_DIR ${CMAKE_BINARY_DIR}/tools/flang)
   set(BACKEND_PACKAGE_STRING "${PACKAGE_STRING}")
   if (LINK_WITH_FIR)
     set(MLIR_MAIN_SRC_DIR ${LLVM_MAIN_SRC_DIR}/../mlir/include ) # --src-root
@@ -194,6 +194,8 @@ endif()
 
 if(LINK_WITH_FIR)
   # tco tool and FIR lib output directories
+  set(LLVM_RUNTIME_OUTPUT_INTDIR ${CMAKE_BINARY_DIR}/bin)
+  set(LLVM_LIBRARY_OUTPUT_INTDIR ${CMAKE_BINARY_DIR}/lib)
   # Always build tco tool
   set(LLVM_BUILD_TOOLS ON)
   message(STATUS "Linking driver with FIR and LLVM")
diff --git a/flang/test/lit.cfg.py b/flang/test/lit.cfg.py
index 521ae968d1cb9..8ad5a9b6357f9 100644
--- a/flang/test/lit.cfg.py
+++ b/flang/test/lit.cfg.py
@@ -48,6 +48,12 @@
 llvm_config.with_environment('PATH', config.flang_tools_dir, append_path=True)
 llvm_config.with_environment('PATH', config.llvm_tools_dir, append_path=True)
 
+# For builds with FIR, set path for tco and enable related tests
+if config.flang_llvm_tools_dir != "" :
+  config.available_features.add('fir')
+  if config.llvm_tools_dir != config.flang_llvm_tools_dir :
+    llvm_config.with_environment('PATH', config.flang_llvm_tools_dir, append_path=True)
+
 # For each occurrence of a flang tool name, replace it with the full path to
 # the build directory holding that tool.
 tools = [
@@ -55,7 +61,7 @@
     extra_args=["-intrinsic-module-directory "+config.flang_intrinsic_modules_dir],
     unresolved='fatal')
 ]
-llvm_config.add_tool_substitutions(tools, config.llvm_tools_dir)
+llvm_config.add_tool_substitutions(tools, [config.flang_llvm_tools_dir])
 
 # Enable libpgmath testing
 result = lit_config.params.get("LIBPGMATH")
diff --git a/flang/test/lit.site.cfg.py.in b/flang/test/lit.site.cfg.py.in
index 34bcdab11b55b..e8e2945a2cbf0 100644
--- a/flang/test/lit.site.cfg.py.in
+++ b/flang/test/lit.site.cfg.py.in
@@ -7,6 +7,7 @@ config.flang_obj_root = "@FLANG_BINARY_DIR@"
 config.flang_src_dir = "@FLANG_SOURCE_DIR@"
 config.flang_tools_dir = "@FLANG_TOOLS_DIR@"
 config.flang_intrinsic_modules_dir = "@FLANG_INTRINSIC_MODULES_DIR@"
+config.flang_llvm_tools_dir = "@CMAKE_BINARY_DIR@/bin"
 config.python_executable = "@PYTHON_EXECUTABLE@"
 
 # Support substitution of the tools_dir with user parameters. This is
diff --git a/flang/tools/f18/CMakeLists.txt b/flang/tools/f18/CMakeLists.txt
index 8e8b3d6b6ab11..46c38fa43a2e5 100644
--- a/flang/tools/f18/CMakeLists.txt
+++ b/flang/tools/f18/CMakeLists.txt
@@ -59,14 +59,8 @@ add_custom_target(module_files ALL DEPENDS ${MODULE_FILES})
 install(TARGETS f18 DESTINATION bin)
 
 set(FLANG_INTRINSIC_MODULES_DIR ${FLANG_BINARY_DIR}/include/flang)
-# This flang shell script will only work in a POSIX shell.
-if (NOT WIN32)
-  configure_file(${CMAKE_CURRENT_SOURCE_DIR}/flang.sh.in ${CMAKE_CURRENT_BINARY_DIR}/tools/flang/bin/flang @ONLY)
-add_custom_command(TARGET f18
-  POST_BUILD
-  COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_BINARY_DIR}/tools/flang/bin/flang ${LLVM_RUNTIME_OUTPUT_INTDIR}/flang
-  COMMAND chmod +x ${LLVM_RUNTIME_OUTPUT_INTDIR}/flang)
-endif()
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/flang.sh.in ${CMAKE_BINARY_DIR}/tools/flang/bin/flang @ONLY)
+file(COPY ${CMAKE_BINARY_DIR}/tools/flang/bin/flang DESTINATION ${CMAKE_BINARY_DIR}/bin FILE_PERMISSIONS OWNER_EXECUTE OWNER_READ OWNER_WRITE)
 # The flang script to be installed needs a different path to the headers.
 set(FLANG_INTRINSIC_MODULES_DIR ${CMAKE_INSTALL_PREFIX}/include/flang)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/flang.sh.in ${FLANG_BINARY_DIR}/bin/flang-install.sh @ONLY)

From 8830f1170dae898d7a0d6a95897e1c23e316ec1e Mon Sep 17 00:00:00 2001
From: sameeran joshi <sameeranjayant.joshi@amd.com>
Date: Fri, 24 Jul 2020 13:49:29 +0530
Subject: [PATCH 0767/1035] [flang]Verify C7107, C7108, C7109 from - Clause 7
 constraint checks for f18.

   1.  Annotate the sources with constraint numbers.
   2.  Add tests for

    *C7107 (R765) digit shall have one of the values 0 or 1.
    *C7108 (R766) digit shall have one of the values 0 through 7.
    *C7109 (R764) A boz-literal-constant shall appear only as a data-stmt-constant in a DATA statement, or where explicitly allowed in 16.9 as an actual argument of an intrinsic procedure.

Reviewed By: PeteSteinfeld

Differential Revision: https://reviews.llvm.org/D84504
---
 flang/lib/Evaluate/intrinsics.cpp             |  5 +-
 flang/lib/Semantics/expression.cpp            |  3 +-
 .../test/Semantics/boz-literal-constants.f90  | 79 +++++++++++++++++++
 3 files changed, 84 insertions(+), 3 deletions(-)
 create mode 100644 flang/test/Semantics/boz-literal-constants.f90

diff --git a/flang/lib/Evaluate/intrinsics.cpp b/flang/lib/Evaluate/intrinsics.cpp
index 35a69e4e9b937..ca9a91c98dafe 100644
--- a/flang/lib/Evaluate/intrinsics.cpp
+++ b/flang/lib/Evaluate/intrinsics.cpp
@@ -1129,9 +1129,10 @@ std::optional<SpecificCall> IntrinsicInterface::Match(
             d.rank == Rank::elementalOrBOZ) {
           continue;
         } else {
+          const IntrinsicDummyArgument &nextParam{dummy[j + 1]};
           messages.Say(
-              "Typeless (BOZ) not allowed for '%s=' argument"_err_en_US,
-              d.keyword);
+              "Typeless (BOZ) not allowed for both '%s=' & '%s=' arguments"_err_en_US, // C7109
+              d.keyword, nextParam.keyword);
         }
       } else {
         // NULL(), procedure, or procedure pointer
diff --git a/flang/lib/Semantics/expression.cpp b/flang/lib/Semantics/expression.cpp
index 3ec2af5d70fa8..0a6e448e98a22 100644
--- a/flang/lib/Semantics/expression.cpp
+++ b/flang/lib/Semantics/expression.cpp
@@ -633,7 +633,8 @@ MaybeExpr ExpressionAnalyzer::Analyze(const parser::BOZLiteralConstant &x) {
   ++p;
   auto value{BOZLiteralConstant::Read(p, base, false /*unsigned*/)};
   if (*p != '"') {
-    Say("Invalid digit ('%c') in BOZ literal '%s'"_err_en_US, *p, x.v);
+    Say("Invalid digit ('%c') in BOZ literal '%s'"_err_en_US, *p,
+        x.v); // C7107, C7108
     return std::nullopt;
   }
   if (value.overflow) {
diff --git a/flang/test/Semantics/boz-literal-constants.f90 b/flang/test/Semantics/boz-literal-constants.f90
new file mode 100644
index 0000000000000..23155cc29cb7d
--- /dev/null
+++ b/flang/test/Semantics/boz-literal-constants.f90
@@ -0,0 +1,79 @@
+! RUN: %S/test_errors.sh %s %t %f18
+! Confirm enforcement of constraints and restrictions in 7.7
+! C7107, C7108, C7109
+
+subroutine bozchecks
+  ! Type declaration statements
+  integer :: f, realpart = B"0101", img = B"1111", resint
+  logical :: resbit
+  complex :: rescmplx
+  real :: dbl, e
+  ! C7107
+  !ERROR: Invalid digit ('a') in BOZ literal 'b"110a"'
+  integer, parameter :: a = B"110A"
+  !ERROR: Invalid digit ('2') in BOZ literal 'b"1232"'
+  integer, parameter :: b = B"1232"
+  !ERROR: BOZ literal 'b"010101010101010101010101011111111111111111111111111111111111111111111111111111111111111111111111111111111111000000000000000000000000000000000000"' too large
+  integer, parameter :: b1 = B"010101010101010101010101011111111111111111111&
+                              &111111111111111111111111111111111111111111111&
+                              &111111111111111111000000000000000000000000000&
+                              &000000000"
+  ! C7108
+  !ERROR: Invalid digit ('8') in BOZ literal 'o"8"'
+  integer :: c = O"8"
+  !ERROR: Invalid digit ('a') in BOZ literal 'o"a"'
+  integer :: d = O"A"
+
+  ! C7109
+  !    A) can appear only in data statement
+  !    B) Argument to intrinsics listed from 16.9 below
+  !       BGE, BGT, BLE, BLT, CMPLX, DBLE, DSHIFTL,
+  !       DSHIFTR, IAND, IEOR, INT, IOR, MERGE_BITS, REAL
+
+  ! part A
+  data f / Z"AA" / ! OK
+  !ERROR: DATA statement value could not be converted to the type 'COMPLEX(4)' of the object 'rescmplx'
+  data rescmplx / B"010101" /
+  ! part B
+  resbit = BGE(B"0101", B"1111")
+  resbit = BGT(Z"0101", B"1111")
+  resbit = BLE(B"0101", B"1111")
+  resbit = BLT(B"0101", B"1111")
+
+  res = CMPLX (realpart, img, 4)
+  res = CMPLX (B"0101", B"1111", 4)
+
+  dbl = DBLE(B"1111")
+  dbl = DBLE(realpart)
+
+  !ERROR: Typeless (BOZ) not allowed for both 'i=' & 'j=' arguments
+  dbl = DSHIFTL(B"0101",B"0101",2)
+  !ERROR: Typeless (BOZ) not allowed for both 'i=' & 'j=' arguments
+  dbl = DSHIFTR(B"1010",B"1010",2)
+  dbl = DSHIFTL(B"0101",5,2) ! OK
+  dbl = DSHIFTR(B"1010",5,2) ! OK
+
+  !ERROR: Typeless (BOZ) not allowed for both 'i=' & 'j=' arguments
+  resint = IAND(B"0001", B"0011")
+  resint = IAND(B"0001", 3)
+
+  !ERROR: Typeless (BOZ) not allowed for both 'i=' & 'j=' arguments
+  resint = IEOR(B"0001", B"0011")
+  resint = IEOR(B"0001", 3)
+
+  resint = INT(B"1010")
+
+  !ERROR: Typeless (BOZ) not allowed for both 'i=' & 'j=' arguments
+  res = IOR(B"0101", B"0011")
+  res = IOR(B"0101", 3)
+
+  res = MERGE_BITS(13,3,11)
+  res = MERGE_BITS(B"1101",3,11)
+  !ERROR: Typeless (BOZ) not allowed for both 'i=' & 'j=' arguments
+  res = MERGE_BITS(B"1101",B"0011",11)
+  !ERROR: Typeless (BOZ) not allowed for both 'i=' & 'j=' arguments
+  res = MERGE_BITS(B"1101",B"0011",B"1011")
+  res = MERGE_BITS(B"1101",3,B"1011")
+
+  res = REAL(B"1101")
+end subroutine

From cd53ded557c3487b8dae2f9de894fdb5b75cb8c8 Mon Sep 17 00:00:00 2001
From: Fangrui Song <maskray@google.com>
Date: Fri, 31 Jul 2020 11:20:15 -0700
Subject: [PATCH 0768/1035] [Support] Fix computeHostNumPhysicalCores() to
 respect affinity

computeHostNumPhysicalCores() is designed to respect CPU affinity.
D84764 used sysconf(_SC_NPROCESSORS_ONLN) which does not respect
affinity.
SupportTests Threading.PhysicalConcurrency may fail if taskset -c is specified.
---
 llvm/lib/Support/Host.cpp | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/llvm/lib/Support/Host.cpp b/llvm/lib/Support/Host.cpp
index 01ede62c755a6..d3b255ae0f2ec 100644
--- a/llvm/lib/Support/Host.cpp
+++ b/llvm/lib/Support/Host.cpp
@@ -1271,11 +1271,14 @@ int computeHostNumPhysicalCores() {
   }
   return CPU_COUNT(&Enabled);
 }
-#elif (defined(__linux__) &&                                                   \
-       (defined(__ppc__) || defined(__powerpc__) || defined(__s390x__)))
-#include <unistd.h>
-
-// Gets the number of *physical cores* on the machine.
+#elif defined(__linux__) && defined(__powerpc__)
+int computeHostNumPhysicalCores() {
+  cpu_set_t Affinity;
+  if (sched_getaffinity(0, sizeof(Affinity), &Affinity) != 0)
+    return -1;
+  return CPU_COUNT(&Affinity);
+}
+#elif defined(__linux__) && defined(__s390x__)
 int computeHostNumPhysicalCores() { return sysconf(_SC_NPROCESSORS_ONLN); }
 #elif defined(__APPLE__) && defined(__x86_64__)
 #include <sys/param.h>

From 19bc9ea480b60b607a3e303f20c7a3a2ea553369 Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne@apple.com>
Date: Fri, 31 Jul 2020 12:56:36 -0400
Subject: [PATCH 0769/1035] [libc++] Avoid including <Block.h> from
 <functional>

Block.h is a pretty common name, which can lead to nasty collisions with
user provided headers. Since we're only getting a few simple declarations
from the header, it's better to declare them manually than to include the
header.

rdar://66384326

Differential Revision: https://reviews.llvm.org/D85035
---
 libcxx/include/functional                           | 13 ++++++-------
 .../utilities/function.objects/func.blocks.sh.cpp   |  2 ++
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/libcxx/include/functional b/libcxx/include/functional
index 3e9425320fc32..9a0ca96c4611b 100644
--- a/libcxx/include/functional
+++ b/libcxx/include/functional
@@ -508,10 +508,6 @@ POLICY:  For non-variadic implementations, the number of arguments is limited
 
 #include <__functional_base>
 
-#if defined(_LIBCPP_HAS_BLOCKS_RUNTIME) && !defined(_LIBCPP_HAS_OBJC_ARC)
-#include <Block.h>
-#endif
-
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #pragma GCC system_header
 #endif
@@ -2257,6 +2253,9 @@ template <class _Rp, class... _ArgTypes> class __policy_func<_Rp(_ArgTypes...)>
 
 #if defined(_LIBCPP_HAS_BLOCKS_RUNTIME) && !defined(_LIBCPP_HAS_OBJC_ARC)
 
+extern "C" void *_Block_copy(const void *);
+extern "C" void _Block_release(const void *);
+
 template<class _Rp1, class ..._ArgTypes1, class _Alloc, class _Rp, class ..._ArgTypes>
 class __func<_Rp1(^)(_ArgTypes1...), _Alloc, _Rp(_ArgTypes...)>
     : public  __base<_Rp(_ArgTypes...)>
@@ -2267,14 +2266,14 @@ class __func<_Rp1(^)(_ArgTypes1...), _Alloc, _Rp(_ArgTypes...)>
 public:
     _LIBCPP_INLINE_VISIBILITY
     explicit __func(__block_type const& __f)
-        : __f_(__f ? Block_copy(__f) : (__block_type)0)
+        : __f_(reinterpret_cast<__block_type>(__f ? _Block_copy(__f) : nullptr))
     { }
 
     // [TODO] add && to save on a retain
 
     _LIBCPP_INLINE_VISIBILITY
     explicit __func(__block_type __f, const _Alloc& /* unused */)
-        : __f_(__f ? Block_copy(__f) : (__block_type)0)
+        : __f_(reinterpret_cast<__block_type>(__f ? _Block_copy(__f) : nullptr))
     { }
 
     virtual __base<_Rp(_ArgTypes...)>* __clone() const {
@@ -2291,7 +2290,7 @@ public:
 
     virtual void destroy() _NOEXCEPT {
         if (__f_)
-            Block_release(__f_);
+            _Block_release(__f_);
         __f_ = 0;
     }
 
diff --git a/libcxx/test/libcxx/utilities/function.objects/func.blocks.sh.cpp b/libcxx/test/libcxx/utilities/function.objects/func.blocks.sh.cpp
index 33c11651f12f5..9a8e9389426a4 100644
--- a/libcxx/test/libcxx/utilities/function.objects/func.blocks.sh.cpp
+++ b/libcxx/test/libcxx/utilities/function.objects/func.blocks.sh.cpp
@@ -21,6 +21,8 @@
 #include <cstdlib>
 #include <cassert>
 
+#include <Block.h>
+
 #include "test_macros.h"
 #include "count_new.h"
 

From b752a8ca499b84d46c5b710f7a3b475fdf0d752e Mon Sep 17 00:00:00 2001
From: sameeran joshi <sameeranjayant.joshi@amd.com>
Date: Fri, 24 Jul 2020 12:25:32 +0530
Subject: [PATCH 0770/1035] [flang][NFC] Verify C781 from - Clause 7 constraint
 checks for f18.

Reviewed By: PeteSteinfeld

Differential Revision: https://reviews.llvm.org/D84503
---
 flang/lib/Semantics/resolve-names.cpp | 2 +-
 flang/test/Semantics/bindings01.f90   | 2 +-
 flang/test/Semantics/resolve80.f90    | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp
index eb7dd697b2748..e85dfa9c91ef5 100644
--- a/flang/lib/Semantics/resolve-names.cpp
+++ b/flang/lib/Semantics/resolve-names.cpp
@@ -1547,7 +1547,7 @@ bool AttrsVisitor::IsConflictingAttr(Attr attrName) {
   return HaveAttrConflict(attrName, Attr::INTENT_IN, Attr::INTENT_INOUT) ||
       HaveAttrConflict(attrName, Attr::INTENT_IN, Attr::INTENT_OUT) ||
       HaveAttrConflict(attrName, Attr::INTENT_INOUT, Attr::INTENT_OUT) ||
-      HaveAttrConflict(attrName, Attr::PASS, Attr::NOPASS) ||
+      HaveAttrConflict(attrName, Attr::PASS, Attr::NOPASS) || // C781
       HaveAttrConflict(attrName, Attr::PURE, Attr::IMPURE) ||
       HaveAttrConflict(attrName, Attr::PUBLIC, Attr::PRIVATE) ||
       HaveAttrConflict(attrName, Attr::RECURSIVE, Attr::NON_RECURSIVE);
diff --git a/flang/test/Semantics/bindings01.f90 b/flang/test/Semantics/bindings01.f90
index 26dd25a24650c..f2d5876167c25 100644
--- a/flang/test/Semantics/bindings01.f90
+++ b/flang/test/Semantics/bindings01.f90
@@ -1,6 +1,6 @@
 ! RUN: %S/test_errors.sh %s %t %f18
 ! Confirm enforcement of constraints and restrictions in 7.5.7.3
-! and C733, C734 and C779, C780, C781, C782, C783, C784, and C785.
+! and C733, C734 and C779, C780, C782, C783, C784, and C785.
 
 module m
   !ERROR: An ABSTRACT derived type must be extensible
diff --git a/flang/test/Semantics/resolve80.f90 b/flang/test/Semantics/resolve80.f90
index 7e9df344a1611..689187d4e15ed 100644
--- a/flang/test/Semantics/resolve80.f90
+++ b/flang/test/Semantics/resolve80.f90
@@ -26,7 +26,7 @@ module m
     !WARNING: Attribute 'PASS' cannot be used more than once
     procedure(subPass), pass, deferred, pass :: passBinding
     !ERROR: Attributes 'PASS' and 'NOPASS' conflict with each other
-    procedure(subPassNopass), pass, deferred, nopass :: passNopassBinding
+    procedure(subPassNopass), pass, deferred, nopass :: passNopassBinding  ! C781
   end type boundProcType
 
 contains

From bf812c145ca2edc4fb76133ec8104267d66f8ee6 Mon Sep 17 00:00:00 2001
From: Sourabh Singh Tomar <SourabhSingh.Tomar@amd.com>
Date: Thu, 30 Jul 2020 23:47:51 +0530
Subject: [PATCH 0771/1035] [flang][OpenMP] Added initial support for lowering
 OpenMP  parallel construct

This patch lower `!OMP PARALLEL` construct from PFT to OpenMPDialect operations.
This is first patch in this direction(lowering parallel construct).

OpenMP parallel construct can have multiple clauses and parameters. This patch
only implements lowering of an empty(contains no code in body) parallel construct
without any clauses or parameters.

Patch is carved out of following approved PR:
https://github.com/flang-compiler/f18-llvm-project/pull/322

Reviewed By: kiranchandramohan, DavidTruby

Differential Revision: https://reviews.llvm.org/D84965
---
 flang/lib/Lower/OpenMP.cpp                   | 32 +++++++++++++++++++-
 flang/unittests/Lower/OpenMPLoweringTest.cpp | 27 +++++++++++++++++
 2 files changed, 58 insertions(+), 1 deletion(-)

diff --git a/flang/lib/Lower/OpenMP.cpp b/flang/lib/Lower/OpenMP.cpp
index bc7cd27df320e..017187069dcbb 100644
--- a/flang/lib/Lower/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP.cpp
@@ -75,6 +75,36 @@ genOMP(Fortran::lower::AbstractConverter &absConv,
       standaloneConstruct.u);
 }
 
+static void
+genOMP(Fortran::lower::AbstractConverter &absConv,
+       Fortran::lower::pft::Evaluation &eval,
+       const Fortran::parser::OpenMPBlockConstruct &blockConstruct) {
+  const auto &blockDirective =
+      std::get<Fortran::parser::OmpBeginBlockDirective>(blockConstruct.t);
+  const auto &parallelDirective =
+      std::get<Fortran::parser::OmpBlockDirective>(blockDirective.t);
+  if (parallelDirective.v == llvm::omp::OMPD_parallel) {
+    auto &firOpBuilder = absConv.getFirOpBuilder();
+    auto currentLocation = absConv.getCurrentLocation();
+    auto insertPt = firOpBuilder.saveInsertionPoint();
+    llvm::ArrayRef<mlir::Type> argTy;
+    mlir::ValueRange range;
+    llvm::SmallVector<int32_t, 6> operandSegmentSizes(6 /*Size=*/,
+                                                      0 /*Value=*/);
+    // create and insert the operation.
+    auto parallelOp = firOpBuilder.create<mlir::omp::ParallelOp>(
+        currentLocation, argTy, range);
+    parallelOp.setAttr(mlir::omp::ParallelOp::getOperandSegmentSizeAttr(),
+                       firOpBuilder.getI32VectorAttr(operandSegmentSizes));
+    parallelOp.getRegion().push_back(new Block{});
+    auto &block = parallelOp.getRegion().back();
+    firOpBuilder.setInsertionPointToStart(&block);
+    // ensure the block is well-formed.
+    firOpBuilder.create<mlir::omp::TerminatorOp>(currentLocation);
+    firOpBuilder.restoreInsertionPoint(insertPt);
+  }
+}
+
 void Fortran::lower::genOpenMPConstruct(
     Fortran::lower::AbstractConverter &absConv,
     Fortran::lower::pft::Evaluation &eval,
@@ -92,7 +122,7 @@ void Fortran::lower::genOpenMPConstruct(
             TODO();
           },
           [&](const Fortran::parser::OpenMPBlockConstruct &blockConstruct) {
-            TODO();
+            genOMP(absConv, eval, blockConstruct);
           },
           [&](const Fortran::parser::OpenMPAtomicConstruct &atomicConstruct) {
             TODO();
diff --git a/flang/unittests/Lower/OpenMPLoweringTest.cpp b/flang/unittests/Lower/OpenMPLoweringTest.cpp
index fd580d71fab3c..ad6fe739d16b4 100644
--- a/flang/unittests/Lower/OpenMPLoweringTest.cpp
+++ b/flang/unittests/Lower/OpenMPLoweringTest.cpp
@@ -71,4 +71,31 @@ TEST_F(OpenMPLoweringTest, TaskYield) {
   EXPECT_EQ(succeeded(taskYieldOp.verify()), true);
 }
 
+TEST_F(OpenMPLoweringTest, EmptyParallel) {
+  // Construct a dummy parse tree node for `!OMP parallel`.
+  struct Fortran::parser::OmpSimpleStandaloneDirective parallelDirective(
+      llvm::omp::Directive::OMPD_parallel);
+
+  // Check and lower the `!OMP parallel` node to `ParallelOp` operation of
+  // OpenMPDialect.
+  EXPECT_EQ(parallelDirective.v, llvm::omp::Directive::OMPD_parallel);
+  auto insertPt = mlirOpBuilder->saveInsertionPoint();
+  llvm::ArrayRef<mlir::Type> argTy;
+  mlir::ValueRange range;
+  llvm::SmallVector<int32_t, 6> operandSegmentSizes(6 /*Size=*/, 0 /*Value=*/);
+  // create and insert the operation.
+  auto parallelOp = mlirOpBuilder->create<mlir::omp::ParallelOp>(
+      mlirOpBuilder->getUnknownLoc(), argTy, range);
+  parallelOp.setAttr(mlir::omp::ParallelOp::getOperandSegmentSizeAttr(),
+      mlirOpBuilder->getI32VectorAttr(operandSegmentSizes));
+  parallelOp.getRegion().push_back(new mlir::Block{});
+  auto &block = parallelOp.getRegion().back();
+  mlirOpBuilder->setInsertionPointToStart(&block);
+  // ensure the block is well-formed.
+  mlirOpBuilder->create<mlir::omp::TerminatorOp>(
+      mlirOpBuilder->getUnknownLoc());
+  mlirOpBuilder->restoreInsertionPoint(insertPt);
+  EXPECT_EQ(succeeded(parallelOp.verify()), true);
+}
+
 // main() from gtest_main

From b7cfa6ca92830b3c331cb44706bb279996663439 Mon Sep 17 00:00:00 2001
From: Sidharth Baveja <sidharth.baveja@ibm.com>
Date: Fri, 31 Jul 2020 18:31:58 +0000
Subject: [PATCH 0772/1035] [Loop Peeling] Separate the Loop Peeling Utilities
 from the Loop Unrolling Utilities

Summary: This patch separates the Loop Peeling Utilities from Loop Unrolling.
The reason for this change is that Loop Peeling is no longer only being used by
loop unrolling; Patch D82927 introduces loop peeling with fusion, such that
loops can be modified to have to same trip count, making them legal to be
peeled.

Reviewed By: Meinersbur

Differential Revision: https://reviews.llvm.org/D83056
---
 llvm/include/llvm/Transforms/Utils/LoopPeel.h | 40 ++++++++++
 .../llvm/Transforms/Utils/UnrollLoop.h        | 17 -----
 .../Hexagon/HexagonTargetTransformInfo.cpp    |  1 +
 llvm/lib/Transforms/Scalar/LoopFuse.cpp       |  2 +-
 .../Scalar/LoopUnrollAndJamPass.cpp           |  1 +
 llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp | 51 +------------
 llvm/lib/Transforms/Utils/CMakeLists.txt      |  2 +-
 .../{LoopUnrollPeel.cpp => LoopPeel.cpp}      | 76 +++++++++++++++----
 llvm/lib/Transforms/Utils/LoopUnroll.cpp      |  1 +
 9 files changed, 109 insertions(+), 82 deletions(-)
 create mode 100644 llvm/include/llvm/Transforms/Utils/LoopPeel.h
 rename llvm/lib/Transforms/Utils/{LoopUnrollPeel.cpp => LoopPeel.cpp} (92%)

diff --git a/llvm/include/llvm/Transforms/Utils/LoopPeel.h b/llvm/include/llvm/Transforms/Utils/LoopPeel.h
new file mode 100644
index 0000000000000..8f857e1e5c215
--- /dev/null
+++ b/llvm/include/llvm/Transforms/Utils/LoopPeel.h
@@ -0,0 +1,40 @@
+//===- llvm/Transforms/Utils/LoopPeel.h ----- Peeling utilities -*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines some loop peeling utilities. It does not define any
+// actual pass or policy.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TRANSFORMS_UTILS_LOOPPEEL_H
+#define LLVM_TRANSFORMS_UTILS_LOOPPEEL_H
+
+#include "llvm/Analysis/TargetTransformInfo.h"
+
+namespace llvm {
+
+bool canPeel(Loop *L);
+
+bool peelLoop(Loop *L, unsigned PeelCount, LoopInfo *LI, ScalarEvolution *SE,
+              DominatorTree *DT, AssumptionCache *AC, bool PreserveLCSSA);
+
+TargetTransformInfo::PeelingPreferences
+gatherPeelingPreferences(Loop *L, ScalarEvolution &SE,
+                         const TargetTransformInfo &TTI,
+                         Optional<bool> UserAllowPeeling,
+                         Optional<bool> UserAllowProfileBasedPeeling,
+                         bool UnrollingSpecficValues = false);
+
+void computePeelCount(Loop *L, unsigned LoopSize,
+                      TargetTransformInfo::PeelingPreferences &PP,
+                      unsigned &TripCount, ScalarEvolution &SE,
+                      unsigned Threshold = UINT_MAX);
+
+} // end namespace llvm
+
+#endif // LLVM_TRANSFORMS_UTILS_LOOPPEEL_H
diff --git a/llvm/include/llvm/Transforms/Utils/UnrollLoop.h b/llvm/include/llvm/Transforms/Utils/UnrollLoop.h
index bb3d02b959564..4254bd71a41c0 100644
--- a/llvm/include/llvm/Transforms/Utils/UnrollLoop.h
+++ b/llvm/include/llvm/Transforms/Utils/UnrollLoop.h
@@ -92,16 +92,6 @@ bool UnrollRuntimeLoopRemainder(
     const TargetTransformInfo *TTI, bool PreserveLCSSA,
     Loop **ResultLoop = nullptr);
 
-void computePeelCount(Loop *L, unsigned LoopSize,
-                      TargetTransformInfo::UnrollingPreferences &UP,
-                      TargetTransformInfo::PeelingPreferences &PP,
-                      unsigned &TripCount, ScalarEvolution &SE);
-
-bool canPeel(Loop *L);
-
-bool peelLoop(Loop *L, unsigned PeelCount, LoopInfo *LI, ScalarEvolution *SE,
-              DominatorTree *DT, AssumptionCache *AC, bool PreserveLCSSA);
-
 LoopUnrollResult UnrollAndJamLoop(Loop *L, unsigned Count, unsigned TripCount,
                                   unsigned TripMultiple, bool UnrollRemainder,
                                   LoopInfo *LI, ScalarEvolution *SE,
@@ -121,7 +111,6 @@ bool computeUnrollCount(Loop *L, const TargetTransformInfo &TTI,
                         unsigned &TripMultiple, unsigned LoopSize,
                         TargetTransformInfo::UnrollingPreferences &UP,
                         TargetTransformInfo::PeelingPreferences &PP,
-
                         bool &UseUpperBound);
 
 void simplifyLoopAfterUnroll(Loop *L, bool SimplifyIVs, LoopInfo *LI,
@@ -138,12 +127,6 @@ TargetTransformInfo::UnrollingPreferences gatherUnrollingPreferences(
     Optional<bool> UserAllowPartial, Optional<bool> UserRuntime,
     Optional<bool> UserUpperBound, Optional<unsigned> UserFullUnrollMaxCount);
 
-TargetTransformInfo::PeelingPreferences
-gatherPeelingPreferences(Loop *L, ScalarEvolution &SE,
-                         const TargetTransformInfo &TTI,
-                         Optional<bool> UserAllowPeeling,
-                         Optional<bool> UserAllowProfileBasedPeeling);
-
 unsigned ApproximateLoopSize(const Loop *L, unsigned &NumCalls,
                              bool &NotDuplicatable, bool &Convergent,
                              const TargetTransformInfo &TTI,
diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
index 68efaf767502c..0cdb383eb9241 100644
--- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
@@ -21,6 +21,7 @@
 #include "llvm/IR/User.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Transforms/Utils/LoopPeel.h"
 #include "llvm/Transforms/Utils/UnrollLoop.h"
 
 using namespace llvm;
diff --git a/llvm/lib/Transforms/Scalar/LoopFuse.cpp b/llvm/lib/Transforms/Scalar/LoopFuse.cpp
index 5573af834c597..90356ed0f2a75 100644
--- a/llvm/lib/Transforms/Scalar/LoopFuse.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopFuse.cpp
@@ -66,7 +66,7 @@
 #include "llvm/Transforms/Utils.h"
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
 #include "llvm/Transforms/Utils/CodeMoverUtils.h"
-#include "llvm/Transforms/Utils/UnrollLoop.h"
+#include "llvm/Transforms/Utils/LoopPeel.h"
 
 using namespace llvm;
 
diff --git a/llvm/lib/Transforms/Scalar/LoopUnrollAndJamPass.cpp b/llvm/lib/Transforms/Scalar/LoopUnrollAndJamPass.cpp
index 285cba6ee2054..bd62419323065 100644
--- a/llvm/lib/Transforms/Scalar/LoopUnrollAndJamPass.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopUnrollAndJamPass.cpp
@@ -41,6 +41,7 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Transforms/Scalar.h"
+#include "llvm/Transforms/Utils/LoopPeel.h"
 #include "llvm/Transforms/Utils/LoopSimplify.h"
 #include "llvm/Transforms/Utils/LoopUtils.h"
 #include "llvm/Transforms/Utils/UnrollLoop.h"
diff --git a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
index 87f40bb7ba852..2b610392dcfd5 100644
--- a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
@@ -56,6 +56,7 @@
 #include "llvm/Transforms/Scalar.h"
 #include "llvm/Transforms/Scalar/LoopPassManager.h"
 #include "llvm/Transforms/Utils.h"
+#include "llvm/Transforms/Utils/LoopPeel.h"
 #include "llvm/Transforms/Utils/LoopSimplify.h"
 #include "llvm/Transforms/Utils/LoopUtils.h"
 #include "llvm/Transforms/Utils/SizeOpts.h"
@@ -115,10 +116,6 @@ static cl::opt<unsigned> UnrollFullMaxCount(
     cl::desc(
         "Set the max unroll count for full unrolling, for testing purposes"));
 
-static cl::opt<unsigned> UnrollPeelCount(
-    "unroll-peel-count", cl::Hidden,
-    cl::desc("Set the unroll peeling count, for testing purposes"));
-
 static cl::opt<bool>
     UnrollAllowPartial("unroll-allow-partial", cl::Hidden,
                        cl::desc("Allows loops to be partially unrolled until "
@@ -149,15 +146,6 @@ static cl::opt<unsigned> FlatLoopTripCountThreshold(
              "threshold, the loop is considered as flat and will be less "
              "aggressively unrolled."));
 
-static cl::opt<bool>
-    UnrollAllowPeeling("unroll-allow-peeling", cl::init(true), cl::Hidden,
-                       cl::desc("Allows loops to be peeled when the dynamic "
-                                "trip count is known to be low."));
-
-static cl::opt<bool> UnrollAllowLoopNestsPeeling(
-    "unroll-allow-loop-nests-peeling", cl::init(false), cl::Hidden,
-    cl::desc("Allows loop nests to be peeled."));
-
 static cl::opt<bool> UnrollUnrollRemainder(
   "unroll-remainder", cl::Hidden,
   cl::desc("Allow the loop remainder to be unrolled."));
@@ -275,39 +263,6 @@ TargetTransformInfo::UnrollingPreferences llvm::gatherUnrollingPreferences(
   return UP;
 }
 
-TargetTransformInfo::PeelingPreferences
-llvm::gatherPeelingPreferences(Loop *L, ScalarEvolution &SE,
-                               const TargetTransformInfo &TTI,
-                               Optional<bool> UserAllowPeeling,
-                               Optional<bool> UserAllowProfileBasedPeeling) {
-  TargetTransformInfo::PeelingPreferences PP;
-
-  // Default values
-  PP.PeelCount = 0;
-  PP.AllowPeeling = true;
-  PP.AllowLoopNestsPeeling = false;
-  PP.PeelProfiledIterations = true;
-
-  // Get Target Specifc Values
-  TTI.getPeelingPreferences(L, SE, PP);
-
-  // User Specified Values using cl::opt
-  if (UnrollPeelCount.getNumOccurrences() > 0)
-    PP.PeelCount = UnrollPeelCount;
-  if (UnrollAllowPeeling.getNumOccurrences() > 0)
-    PP.AllowPeeling = UnrollAllowPeeling;
-  if (UnrollAllowLoopNestsPeeling.getNumOccurrences() > 0)
-    PP.AllowLoopNestsPeeling = UnrollAllowLoopNestsPeeling;
-
-  // User Specifed values provided by argument
-  if (UserAllowPeeling.hasValue())
-    PP.AllowPeeling = *UserAllowPeeling;
-  if (UserAllowProfileBasedPeeling.hasValue())
-    PP.PeelProfiledIterations = *UserAllowProfileBasedPeeling;
-
-  return PP;
-}
-
 namespace {
 
 /// A struct to densely store the state of an instruction after unrolling at
@@ -881,7 +836,7 @@ bool llvm::computeUnrollCount(
   }
 
   // 4th priority is loop peeling.
-  computePeelCount(L, LoopSize, UP, PP, TripCount, SE);
+  computePeelCount(L, LoopSize, PP, TripCount, SE, UP.Threshold);
   if (PP.PeelCount) {
     UP.Runtime = false;
     UP.Count = 1;
@@ -1087,7 +1042,7 @@ static LoopUnrollResult tryToUnrollLoop(
       ProvidedAllowPartial, ProvidedRuntime, ProvidedUpperBound,
       ProvidedFullUnrollMaxCount);
   TargetTransformInfo::PeelingPreferences PP = gatherPeelingPreferences(
-      L, SE, TTI, ProvidedAllowPeeling, ProvidedAllowProfileBasedPeeling);
+      L, SE, TTI, ProvidedAllowPeeling, ProvidedAllowProfileBasedPeeling, true);
 
   // Exit early if unrolling is disabled. For OptForSize, we pick the loop size
   // as threshold later on.
diff --git a/llvm/lib/Transforms/Utils/CMakeLists.txt b/llvm/lib/Transforms/Utils/CMakeLists.txt
index 19f655c3a78fb..2ef17dc595fa8 100644
--- a/llvm/lib/Transforms/Utils/CMakeLists.txt
+++ b/llvm/lib/Transforms/Utils/CMakeLists.txt
@@ -35,11 +35,11 @@ add_llvm_component_library(LLVMTransformUtils
   LCSSA.cpp
   LibCallsShrinkWrap.cpp
   Local.cpp
+  LoopPeel.cpp
   LoopRotationUtils.cpp
   LoopSimplify.cpp
   LoopUnroll.cpp
   LoopUnrollAndJam.cpp
-  LoopUnrollPeel.cpp
   LoopUnrollRuntime.cpp
   LoopUtils.cpp
   LoopVersioning.cpp
diff --git a/llvm/lib/Transforms/Utils/LoopUnrollPeel.cpp b/llvm/lib/Transforms/Utils/LoopPeel.cpp
similarity index 92%
rename from llvm/lib/Transforms/Utils/LoopUnrollPeel.cpp
rename to llvm/lib/Transforms/Utils/LoopPeel.cpp
index c653aacbee6cc..ff9b9eca8fdf7 100644
--- a/llvm/lib/Transforms/Utils/LoopUnrollPeel.cpp
+++ b/llvm/lib/Transforms/Utils/LoopPeel.cpp
@@ -1,4 +1,4 @@
-//===- UnrollLoopPeel.cpp - Loop peeling utilities ------------------------===//
+//===- LoopPeel.cpp -------------------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,12 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// This file implements some loop unrolling utilities for peeling loops
-// with dynamically inferred (from PGO) trip counts. See LoopUnroll.cpp for
-// unrolling loops with compile-time constant trip counts.
-//
+// Loop Peeling Utilities.
 //===----------------------------------------------------------------------===//
 
+#include "llvm/Transforms/Utils/LoopPeel.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/SmallVector.h"
@@ -49,10 +47,24 @@
 using namespace llvm;
 using namespace llvm::PatternMatch;
 
-#define DEBUG_TYPE "loop-unroll"
+#define DEBUG_TYPE "loop-peel"
 
 STATISTIC(NumPeeled, "Number of loops peeled");
 
+static cl::opt<unsigned> UnrollPeelCount(
+    "unroll-peel-count", cl::Hidden,
+    cl::desc("Set the unroll peeling count, for testing purposes"));
+
+static cl::opt<bool>
+    UnrollAllowPeeling("unroll-allow-peeling", cl::init(true), cl::Hidden,
+                       cl::desc("Allows loops to be peeled when the dynamic "
+                                "trip count is known to be low."));
+
+static cl::opt<bool>
+    UnrollAllowLoopNestsPeeling("unroll-allow-loop-nests-peeling",
+                                cl::init(false), cl::Hidden,
+                                cl::desc("Allows loop nests to be peeled."));
+
 static cl::opt<unsigned> UnrollPeelMaxCount(
     "unroll-peel-max-count", cl::init(7), cl::Hidden,
     cl::desc("Max average trip count which will cause loop peeling."));
@@ -278,9 +290,9 @@ static unsigned countToEliminateCompares(Loop &L, unsigned MaxPeelCount,
 
 // Return the number of iterations we want to peel off.
 void llvm::computePeelCount(Loop *L, unsigned LoopSize,
-                            TargetTransformInfo::UnrollingPreferences &UP,
                             TargetTransformInfo::PeelingPreferences &PP,
-                            unsigned &TripCount, ScalarEvolution &SE) {
+                            unsigned &TripCount, ScalarEvolution &SE,
+                            unsigned Threshold) {
   assert(LoopSize > 0 && "Zero loop size is not allowed!");
   // Save the PP.PeelCount value set by the target in
   // TTI.getPeelingPreferences or by the flag -unroll-peel-count.
@@ -322,7 +334,7 @@ void llvm::computePeelCount(Loop *L, unsigned LoopSize,
   // maximum number of iterations among these values, thus turning all those
   // Phis into invariants.
   // First, check that we can peel at least one iteration.
-  if (2 * LoopSize <= UP.Threshold && UnrollPeelMaxCount > 0) {
+  if (2 * LoopSize <= Threshold && UnrollPeelMaxCount > 0) {
     // Store the pre-calculated values here.
     SmallDenseMap<PHINode *, unsigned> IterationsToInvariance;
     // Now go through all Phis to calculate their the number of iterations they
@@ -342,7 +354,7 @@ void llvm::computePeelCount(Loop *L, unsigned LoopSize,
 
     // Pay respect to limitations implied by loop size and the max peel count.
     unsigned MaxPeelCount = UnrollPeelMaxCount;
-    MaxPeelCount = std::min(MaxPeelCount, UP.Threshold / LoopSize - 1);
+    MaxPeelCount = std::min(MaxPeelCount, Threshold / LoopSize - 1);
 
     DesiredPeelCount = std::max(DesiredPeelCount,
                                 countToEliminateCompares(*L, MaxPeelCount, SE));
@@ -385,7 +397,7 @@ void llvm::computePeelCount(Loop *L, unsigned LoopSize,
 
     if (*PeelCount) {
       if ((*PeelCount + AlreadyPeeled <= UnrollPeelMaxCount) &&
-          (LoopSize * (*PeelCount + 1) <= UP.Threshold)) {
+          (LoopSize * (*PeelCount + 1) <= Threshold)) {
         LLVM_DEBUG(dbgs() << "Peeling first " << *PeelCount
                           << " iterations.\n");
         PP.PeelCount = *PeelCount;
@@ -396,7 +408,7 @@ void llvm::computePeelCount(Loop *L, unsigned LoopSize,
       LLVM_DEBUG(dbgs() << "Max peel count: " << UnrollPeelMaxCount << "\n");
       LLVM_DEBUG(dbgs() << "Peel cost: " << LoopSize * (*PeelCount + 1)
                         << "\n");
-      LLVM_DEBUG(dbgs() << "Max peel cost: " << UP.Threshold << "\n");
+      LLVM_DEBUG(dbgs() << "Max peel cost: " << Threshold << "\n");
     }
   }
 }
@@ -491,7 +503,7 @@ static void fixupBranchWeights(BasicBlock *Header, BranchInst *LatchBR,
 /// instructions in the last peeled-off iteration.
 static void cloneLoopBlocks(
     Loop *L, unsigned IterNumber, BasicBlock *InsertTop, BasicBlock *InsertBot,
-    SmallVectorImpl<std::pair<BasicBlock *, BasicBlock *> > &ExitEdges,
+    SmallVectorImpl<std::pair<BasicBlock *, BasicBlock *>> &ExitEdges,
     SmallVectorImpl<BasicBlock *> &NewBlocks, LoopBlocksDFS &LoopBlocks,
     ValueToValueMapTy &VMap, ValueToValueMapTy &LVMap, DominatorTree *DT,
     LoopInfo *LI) {
@@ -599,6 +611,40 @@ static void cloneLoopBlocks(
     LVMap[KV.first] = KV.second;
 }
 
+TargetTransformInfo::PeelingPreferences llvm::gatherPeelingPreferences(
+    Loop *L, ScalarEvolution &SE, const TargetTransformInfo &TTI,
+    Optional<bool> UserAllowPeeling,
+    Optional<bool> UserAllowProfileBasedPeeling, bool UnrollingSpecficValues) {
+  TargetTransformInfo::PeelingPreferences PP;
+
+  // Set the default values.
+  PP.PeelCount = 0;
+  PP.AllowPeeling = true;
+  PP.AllowLoopNestsPeeling = false;
+  PP.PeelProfiledIterations = true;
+
+  // Get the target specifc values.
+  TTI.getPeelingPreferences(L, SE, PP);
+
+  // User specified values using cl::opt.
+  if (UnrollingSpecficValues) {
+    if (UnrollPeelCount.getNumOccurrences() > 0)
+      PP.PeelCount = UnrollPeelCount;
+    if (UnrollAllowPeeling.getNumOccurrences() > 0)
+      PP.AllowPeeling = UnrollAllowPeeling;
+    if (UnrollAllowLoopNestsPeeling.getNumOccurrences() > 0)
+      PP.AllowLoopNestsPeeling = UnrollAllowLoopNestsPeeling;
+  }
+
+  // User specifed values provided by argument.
+  if (UserAllowPeeling.hasValue())
+    PP.AllowPeeling = *UserAllowPeeling;
+  if (UserAllowProfileBasedPeeling.hasValue())
+    PP.PeelProfiledIterations = *UserAllowProfileBasedPeeling;
+
+  return PP;
+}
+
 /// Peel off the first \p PeelCount iterations of loop \p L.
 ///
 /// Note that this does not peel them off as a single straight-line block.
@@ -609,8 +655,8 @@ static void cloneLoopBlocks(
 /// for the bulk of dynamic execution, can be further simplified by scalar
 /// optimizations.
 bool llvm::peelLoop(Loop *L, unsigned PeelCount, LoopInfo *LI,
-                    ScalarEvolution *SE, DominatorTree *DT,
-                    AssumptionCache *AC, bool PreserveLCSSA) {
+                    ScalarEvolution *SE, DominatorTree *DT, AssumptionCache *AC,
+                    bool PreserveLCSSA) {
   assert(PeelCount > 0 && "Attempt to peel out zero iterations?");
   assert(canPeel(L) && "Attempt to peel a loop which is not peelable?");
 
diff --git a/llvm/lib/Transforms/Utils/LoopUnroll.cpp b/llvm/lib/Transforms/Utils/LoopUnroll.cpp
index 8e8aeea15dbf9..7bea696a853a9 100644
--- a/llvm/lib/Transforms/Utils/LoopUnroll.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUnroll.cpp
@@ -59,6 +59,7 @@
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
 #include "llvm/Transforms/Utils/Cloning.h"
 #include "llvm/Transforms/Utils/Local.h"
+#include "llvm/Transforms/Utils/LoopPeel.h"
 #include "llvm/Transforms/Utils/LoopSimplify.h"
 #include "llvm/Transforms/Utils/LoopUtils.h"
 #include "llvm/Transforms/Utils/SimplifyIndVar.h"

From 09746fbfb9cd80a06280e9ca34eeadea93bcb3b2 Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
Date: Fri, 31 Jul 2020 18:32:54 +0000
Subject: [PATCH 0773/1035] [gn build] Port b7cfa6ca928

---
 llvm/utils/gn/secondary/llvm/lib/Transforms/Utils/BUILD.gn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/utils/gn/secondary/llvm/lib/Transforms/Utils/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Transforms/Utils/BUILD.gn
index 5fae2e450b481..a0de4e4980b12 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Transforms/Utils/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Transforms/Utils/BUILD.gn
@@ -42,11 +42,11 @@ static_library("Utils") {
     "LCSSA.cpp",
     "LibCallsShrinkWrap.cpp",
     "Local.cpp",
+    "LoopPeel.cpp",
     "LoopRotationUtils.cpp",
     "LoopSimplify.cpp",
     "LoopUnroll.cpp",
     "LoopUnrollAndJam.cpp",
-    "LoopUnrollPeel.cpp",
     "LoopUnrollRuntime.cpp",
     "LoopUtils.cpp",
     "LoopVersioning.cpp",

From e8a2af28afd5b4370fb66d8810ff979d1cb4d45b Mon Sep 17 00:00:00 2001
From: cgyurgyik <gyurgyikcp@gmail.com>
Date: Fri, 31 Jul 2020 14:36:23 -0400
Subject: [PATCH 0774/1035] [libc] Adds strrchr implementation.

Reviewed By: sivachandra

Differential Revision: https://reviews.llvm.org/D84875
---
 libc/config/linux/aarch64/entrypoints.txt |  1 +
 libc/config/linux/x86_64/entrypoints.txt  |  1 +
 libc/src/string/CMakeLists.txt            |  8 +++
 libc/src/string/strrchr.cpp               | 28 ++++++++
 libc/src/string/strrchr.h                 | 18 +++++
 libc/test/src/string/CMakeLists.txt       | 10 +++
 libc/test/src/string/strrchr_test.cpp     | 81 +++++++++++++++++++++++
 7 files changed, 147 insertions(+)
 create mode 100644 libc/src/string/strrchr.cpp
 create mode 100644 libc/src/string/strrchr.h
 create mode 100644 libc/test/src/string/strrchr_test.cpp

diff --git a/libc/config/linux/aarch64/entrypoints.txt b/libc/config/linux/aarch64/entrypoints.txt
index 5f058e6116001..8314df89b0636 100644
--- a/libc/config/linux/aarch64/entrypoints.txt
+++ b/libc/config/linux/aarch64/entrypoints.txt
@@ -20,6 +20,7 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.string.strcpy
     libc.src.string.strlen
     libc.src.string.strnlen
+    libc.src.string.strrchr
     libc.src.string.strstr
 )
 
diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt
index 5dc1d38455dcf..56a99d00d7847 100644
--- a/libc/config/linux/x86_64/entrypoints.txt
+++ b/libc/config/linux/x86_64/entrypoints.txt
@@ -38,6 +38,7 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.string.strcpy
     libc.src.string.strlen
     libc.src.string.strnlen
+    libc.src.string.strrchr
     libc.src.string.strstr
 
     # sys/mman.h entrypoints
diff --git a/libc/src/string/CMakeLists.txt b/libc/src/string/CMakeLists.txt
index 99450d5564593..d0eab632e9d79 100644
--- a/libc/src/string/CMakeLists.txt
+++ b/libc/src/string/CMakeLists.txt
@@ -86,6 +86,14 @@ add_entrypoint_object(
     memrchr.h
 )
 
+add_entrypoint_object(
+  strrchr
+  SRCS
+    strrchr.cpp
+  HDRS
+    strrchr.h
+)
+
 # Helper to define a function with multiple implementations
 # - Computes flags to satisfy required/rejected features and arch,
 # - Declares an entry point,
diff --git a/libc/src/string/strrchr.cpp b/libc/src/string/strrchr.cpp
new file mode 100644
index 0000000000000..28716c28a2664
--- /dev/null
+++ b/libc/src/string/strrchr.cpp
@@ -0,0 +1,28 @@
+//===-- Implementation of strrchr------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/string/strrchr.h"
+
+#include "src/__support/common.h"
+
+namespace __llvm_libc {
+
+char *LLVM_LIBC_ENTRYPOINT(strrchr)(const char *src, int c) {
+  unsigned char *str =
+      const_cast<unsigned char *>(reinterpret_cast<const unsigned char *>(src));
+  const unsigned char ch = c;
+
+  unsigned char *last_occurrence = nullptr;
+  do {
+    if (*str == ch)
+      last_occurrence = str;
+  } while (*str++);
+  return reinterpret_cast<char *>(last_occurrence);
+}
+
+} // namespace __llvm_libc
diff --git a/libc/src/string/strrchr.h b/libc/src/string/strrchr.h
new file mode 100644
index 0000000000000..7b85929b290b7
--- /dev/null
+++ b/libc/src/string/strrchr.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for strrchr -----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_STRING_STRRCHR_H
+#define LLVM_LIBC_SRC_STRING_STRRCHR_H
+
+namespace __llvm_libc {
+
+char *strrchr(const char *src, int c);
+
+} // namespace __llvm_libc
+
+#endif // LLVM_LIBC_SRC_STRING_STRRCHR_H
diff --git a/libc/test/src/string/CMakeLists.txt b/libc/test/src/string/CMakeLists.txt
index a116effef2719..0fff250fabd15 100644
--- a/libc/test/src/string/CMakeLists.txt
+++ b/libc/test/src/string/CMakeLists.txt
@@ -92,6 +92,16 @@ add_libc_unittest(
     libc.src.string.memrchr
 )
 
+add_libc_unittest(
+  strrchr_test
+  SUITE
+    libc_string_unittests
+  SRCS
+    strrchr_test.cpp
+  DEPENDS
+    libc.src.string.strrchr
+)
+
 # Tests all implementations that can run on the host.
 function(add_libc_multi_impl_test name)
   get_property(fq_implementations GLOBAL PROPERTY ${name}_implementations)
diff --git a/libc/test/src/string/strrchr_test.cpp b/libc/test/src/string/strrchr_test.cpp
new file mode 100644
index 0000000000000..18fddda600870
--- /dev/null
+++ b/libc/test/src/string/strrchr_test.cpp
@@ -0,0 +1,81 @@
+//===-- Unittests for strrchr ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/string/strrchr.h"
+#include "utils/UnitTest/Test.h"
+
+TEST(StrRChrTest, FindsFirstCharacter) {
+  const char *src = "abcde";
+  const char *src_copy = src;
+
+  // Should return original string since 'a' is the first character.
+  ASSERT_STREQ(__llvm_libc::strrchr(src, 'a'), "abcde");
+  // Source string should not change.
+  ASSERT_STREQ(src, src_copy);
+}
+
+TEST(StrRChrTest, FindsMiddleCharacter) {
+  const char *src = "abcde";
+  const char *src_copy = src;
+
+  // Should return characters after (and including) 'c'.
+  ASSERT_STREQ(__llvm_libc::strrchr(src, 'c'), "cde");
+  // Source string should not change.
+  ASSERT_STREQ(src, src_copy);
+}
+
+TEST(StrRChrTest, FindsLastCharacterThatIsNotNullTerminator) {
+  const char *src = "abcde";
+  const char *src_copy = src;
+
+  // Should return 'e' and null-terminator.
+  ASSERT_STREQ(__llvm_libc::strrchr(src, 'e'), "e");
+  // Source string should not change.
+  ASSERT_STREQ(src, src_copy);
+}
+
+TEST(StrRChrTest, FindsNullTerminator) {
+  const char *src = "abcde";
+  const char *src_copy = src;
+
+  // Should return null terminator.
+  ASSERT_STREQ(__llvm_libc::strrchr(src, '\0'), "");
+  // Source string should not change.
+  ASSERT_STREQ(src, src_copy);
+}
+
+TEST(StrRChrTest, FindsLastNullTerminator) {
+  const char src[5] = {'a', '\0', 'b', '\0', 'c'};
+  // 'b' is behind a null terminator, so should not be found.
+  ASSERT_STREQ(__llvm_libc::strrchr(src, 'b'), nullptr);
+  // Same goes for 'c'.
+  ASSERT_STREQ(__llvm_libc::strrchr(src, 'c'), nullptr);
+}
+
+TEST(StrRChrTest, CharacterNotWithinStringShouldReturnNullptr) {
+  // Since 'z' is not within the string, should return nullptr.
+  ASSERT_STREQ(__llvm_libc::strrchr("123?", 'z'), nullptr);
+}
+
+TEST(StrRChrTest, ShouldFindLastOfDuplicates) {
+  // '1' is duplicated in the string, but it should find the last copy.
+  ASSERT_STREQ(__llvm_libc::strrchr("abc1def1ghi", '1'), "1ghi");
+
+  const char *dups = "XXXXX";
+  // Should return the last occurrence of 'X'.
+  ASSERT_STREQ(__llvm_libc::strrchr(dups, 'X'), "X");
+}
+
+TEST(StrRChrTest, EmptyStringShouldOnlyMatchNullTerminator) {
+  // Null terminator should match.
+  ASSERT_STREQ(__llvm_libc::strrchr("", '\0'), "");
+  // All other characters should not match.
+  ASSERT_STREQ(__llvm_libc::strrchr("", 'A'), nullptr);
+  ASSERT_STREQ(__llvm_libc::strrchr("", '2'), nullptr);
+  ASSERT_STREQ(__llvm_libc::strrchr("", '*'), nullptr);
+}

From 43bf902c2e3416179cf41eba9307fc74bcba0ecd Mon Sep 17 00:00:00 2001
From: Julian Lettner <julian.lettner@apple.com>
Date: Fri, 31 Jul 2020 11:38:10 -0700
Subject: [PATCH 0775/1035] [compiler-rt][Darwin] Fix GetOSMajorKernelOffset()
 on watchOS

`TARGET_OS_IOS` and `TARGET_OS_WATCH` are not mutually exclusive.
`SANITIZER_IOS` is defined for all embedded platforms.  So the branch
for watchOS is never taken.  We could fix this by switching the order
of the branches (but the reason for doing so is non-obvious).  Instead,
lets use the Darwin-specific `TARGET_OS_*` macros which are mutually
exclusive.
---
 compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp
index a10ba774b9552..21a9c01bf2a9a 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp
@@ -610,8 +610,8 @@ HandleSignalMode GetHandleSignalMode(int signum) {
 // XNU 17 -- macOS 10.13 -- iOS 11 -- tvOS 11 -- watchOS 4
 constexpr u16 GetOSMajorKernelOffset() {
   if (TARGET_OS_OSX) return 4;
-  if (SANITIZER_IOS || SANITIZER_TVOS) return 6;
-  if (SANITIZER_WATCHOS) return 13;
+  if (TARGET_OS_IOS || TARGET_OS_TV) return 6;
+  if (TARGET_OS_WATCH) return 13;
 }
 
 using VersStr = char[64];
@@ -661,9 +661,9 @@ static void MapToMacos(u16 *major, u16 *minor) {
   if (TARGET_OS_OSX)
     return;
 
-  if (SANITIZER_IOS || SANITIZER_TVOS)
+  if (TARGET_OS_IOS || TARGET_OS_TV)
     *major += 2;
-  else if (SANITIZER_WATCHOS)
+  else if (TARGET_OS_WATCH)
     *major += 9;
   else
     UNREACHABLE("unsupported platform");

From ca6b6d40ffba27fe231f55f7edc533f0a1815d31 Mon Sep 17 00:00:00 2001
From: Sriraman Tallam <tmsriram@google.com>
Date: Fri, 31 Jul 2020 11:14:49 -0700
Subject: [PATCH 0776/1035] Rename basic block sections options to be
 consistent.

D68049 created options for basic block sections: -fbasic-block-sections=,
-funique-basic-block-section-names. Rename options in llc and lld (--lto-)
to be consistent. Specifically,

+ Rename basicblock-sections to basic-block-sections
+ Rename unique-bb-section-names to unique-basic-block-section-names

Differential Revision: https://reviews.llvm.org/D84462
---
 lld/ELF/Driver.cpp                                        | 6 +++---
 lld/ELF/LTO.cpp                                           | 2 +-
 lld/ELF/Options.td                                        | 4 ++--
 ...-sections-and-icf.s => basic-block-sections-and-icf.s} | 2 +-
 ...-fallthru.s => basic-block-sections-delete-fallthru.s} | 2 +-
 ...tions-pc32reloc.s => basic-block-sections-pc32reloc.s} | 2 +-
 llvm/lib/CodeGen/CommandFlags.cpp                         | 4 ++--
 ...nches.ll => basic-block-sections-clusters-branches.ll} | 4 ++--
 ...clusters-eh.ll => basic-block-sections-clusters-eh.ll} | 4 ++--
 ...rs-error.ll => basic-block-sections-clusters-error.ll} | 8 ++++----
 ...tions-clusters.ll => basic-block-sections-clusters.ll} | 4 ++--
 ...lock-sections-cold.ll => basic-block-sections-cold.ll} | 2 +-
 ...directjumps.ll => basic-block-sections-directjumps.ll} | 4 ++--
 ...sicblock-sections-eh.ll => basic-block-sections-eh.ll} | 2 +-
 ...-sections-labels.ll => basic-block-sections-labels.ll} | 2 +-
 ...lock-sections-list.ll => basic-block-sections-list.ll} | 2 +-
 ...-sections-listbb.ll => basic-block-sections-listbb.ll} | 2 +-
 ...s-mir-parse.mir => basic-block-sections-mir-parse.mir} | 2 +-
 ...ons-mir-print.ll => basic-block-sections-mir-print.ll} | 2 +-
 .../{basicblock-sections.ll => basic-block-sections.ll}   | 4 ++--
 llvm/test/CodeGen/X86/cfi-basic-block-sections-1.ll       | 6 +++---
 ...inserter-basic-block-sections-callee-save-registers.ll | 2 +-
 ...basicblock-sections_1.ll => basic-block-sections_1.ll} | 6 +++---
 23 files changed, 39 insertions(+), 39 deletions(-)
 rename lld/test/ELF/{bb-sections-and-icf.s => basic-block-sections-and-icf.s} (96%)
 rename lld/test/ELF/{bb-sections-delete-fallthru.s => basic-block-sections-delete-fallthru.s} (98%)
 rename lld/test/ELF/{bb-sections-pc32reloc.s => basic-block-sections-pc32reloc.s} (96%)
 rename llvm/test/CodeGen/X86/{basicblock-sections-clusters-branches.ll => basic-block-sections-clusters-branches.ll} (93%)
 rename llvm/test/CodeGen/X86/{basicblock-sections-clusters-eh.ll => basic-block-sections-clusters-eh.ll} (95%)
 rename llvm/test/CodeGen/X86/{basicblock-sections-clusters-error.ll => basic-block-sections-clusters-error.ll} (75%)
 rename llvm/test/CodeGen/X86/{basicblock-sections-clusters.ll => basic-block-sections-clusters.ll} (94%)
 rename llvm/test/CodeGen/X86/{basicblock-sections-cold.ll => basic-block-sections-cold.ll} (93%)
 rename llvm/test/CodeGen/X86/{basicblock-sections-directjumps.ll => basic-block-sections-directjumps.ll} (84%)
 rename llvm/test/CodeGen/X86/{basicblock-sections-eh.ll => basic-block-sections-eh.ll} (96%)
 rename llvm/test/CodeGen/X86/{basicblock-sections-labels.ll => basic-block-sections-labels.ll} (92%)
 rename llvm/test/CodeGen/X86/{basicblock-sections-list.ll => basic-block-sections-list.ll} (95%)
 rename llvm/test/CodeGen/X86/{basicblock-sections-listbb.ll => basic-block-sections-listbb.ll} (93%)
 rename llvm/test/CodeGen/X86/{basicblock-sections-mir-parse.mir => basic-block-sections-mir-parse.mir} (98%)
 rename llvm/test/CodeGen/X86/{basicblock-sections-mir-print.ll => basic-block-sections-mir-print.ll} (91%)
 rename llvm/test/CodeGen/X86/{basicblock-sections.ll => basic-block-sections.ll} (84%)
 rename llvm/test/DebugInfo/X86/{basicblock-sections_1.ll => basic-block-sections_1.ll} (88%)

diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp
index 64a41ba77ba2f..cdb7355968377 100644
--- a/lld/ELF/Driver.cpp
+++ b/lld/ELF/Driver.cpp
@@ -965,10 +965,10 @@ static void readConfigs(opt::InputArgList &args) {
   config->ltoPartitions = args::getInteger(args, OPT_lto_partitions, 1);
   config->ltoSampleProfile = args.getLastArgValue(OPT_lto_sample_profile);
   config->ltoBasicBlockSections =
-      args.getLastArgValue(OPT_lto_basicblock_sections);
+      args.getLastArgValue(OPT_lto_basic_block_sections);
   config->ltoUniqueBasicBlockSectionNames =
-      args.hasFlag(OPT_lto_unique_bb_section_names,
-                   OPT_no_lto_unique_bb_section_names, false);
+      args.hasFlag(OPT_lto_unique_basic_block_section_names,
+                   OPT_no_lto_unique_basic_block_section_names, false);
   config->mapFile = args.getLastArgValue(OPT_Map);
   config->mipsGotSize = args::getInteger(args, OPT_mips_got_size, 0xfff0);
   config->mergeArmExidx =
diff --git a/lld/ELF/LTO.cpp b/lld/ELF/LTO.cpp
index b8041afed6c96..ae77fadcc78d3 100644
--- a/lld/ELF/LTO.cpp
+++ b/lld/ELF/LTO.cpp
@@ -76,7 +76,7 @@ static lto::Config createConfig() {
   c.Options.DataSections = true;
 
   // Check if basic block sections must be used.
-  // Allowed values for --lto-basicblock-sections are "all", "labels",
+  // Allowed values for --lto-basic-block-sections are "all", "labels",
   // "<file name specifying basic block ids>", or none.  This is the equivalent
   // of -fbasic-block-sections= flag in clang.
   if (!config->ltoBasicBlockSections.empty()) {
diff --git a/lld/ELF/Options.td b/lld/ELF/Options.td
index c3c1309aca1a7..18bc612f6af45 100644
--- a/lld/ELF/Options.td
+++ b/lld/ELF/Options.td
@@ -545,9 +545,9 @@ def opt_remarks_with_hotness: FF<"opt-remarks-with-hotness">,
 def opt_remarks_format: Separate<["--"], "opt-remarks-format">,
   HelpText<"The format used for serializing remarks (default: YAML)">;
 def save_temps: F<"save-temps">;
-def lto_basicblock_sections: JJ<"lto-basicblock-sections=">,
+def lto_basic_block_sections: JJ<"lto-basic-block-sections=">,
   HelpText<"Enable basic block sections for LTO">;
-defm lto_unique_bb_section_names: BB<"lto-unique-bb-section-names",
+defm lto_unique_basic_block_section_names: BB<"lto-unique-basic-block-section-names",
     "Give unique names to every basic block section for LTO",
     "Do not give unique names to every basic block section for LTO (default)">;
 def shuffle_sections: JJ<"shuffle-sections=">, MetaVarName<"<seed>">,
diff --git a/lld/test/ELF/bb-sections-and-icf.s b/lld/test/ELF/basic-block-sections-and-icf.s
similarity index 96%
rename from lld/test/ELF/bb-sections-and-icf.s
rename to lld/test/ELF/basic-block-sections-and-icf.s
index bcc9193c6ed8f..998d63b47b01c 100644
--- a/lld/test/ELF/bb-sections-and-icf.s
+++ b/lld/test/ELF/basic-block-sections-and-icf.s
@@ -1,5 +1,5 @@
 # REQUIRES: x86
-## basicblock-sections tests.
+## basic-block-sections tests.
 ## This simple test checks foo is folded into bar with bb sections
 ## and the jumps are deleted.
 
diff --git a/lld/test/ELF/bb-sections-delete-fallthru.s b/lld/test/ELF/basic-block-sections-delete-fallthru.s
similarity index 98%
rename from lld/test/ELF/bb-sections-delete-fallthru.s
rename to lld/test/ELF/basic-block-sections-delete-fallthru.s
index c8a0e93534242..b029e664a1e12 100644
--- a/lld/test/ELF/bb-sections-delete-fallthru.s
+++ b/lld/test/ELF/basic-block-sections-delete-fallthru.s
@@ -1,5 +1,5 @@
 # REQUIRES: x86
-## basicblock-sections tests.
+## basic-block-sections tests.
 ## This simple test checks if redundant direct jumps are converted to
 ## implicit fallthrus.  The jcc's must be converted to their inverted
 ## opcode, for instance jne to je and jmp must be deleted.
diff --git a/lld/test/ELF/bb-sections-pc32reloc.s b/lld/test/ELF/basic-block-sections-pc32reloc.s
similarity index 96%
rename from lld/test/ELF/bb-sections-pc32reloc.s
rename to lld/test/ELF/basic-block-sections-pc32reloc.s
index 9631a3cfe6c3a..f276cd9fac731 100644
--- a/lld/test/ELF/bb-sections-pc32reloc.s
+++ b/lld/test/ELF/basic-block-sections-pc32reloc.s
@@ -1,5 +1,5 @@
 # REQUIRES: x86
-## basicblock-sections tests.
+## basic-block-sections tests.
 ## This simple test checks if redundant direct jumps are converted to
 ## implicit fallthrus when PC32 reloc is present.  The jcc's must be converted
 ## to their inverted opcode, for instance jne to je and jmp must be deleted.
diff --git a/llvm/lib/CodeGen/CommandFlags.cpp b/llvm/lib/CodeGen/CommandFlags.cpp
index 12dadf97e02c6..0ada09f469b76 100644
--- a/llvm/lib/CodeGen/CommandFlags.cpp
+++ b/llvm/lib/CodeGen/CommandFlags.cpp
@@ -332,7 +332,7 @@ codegen::RegisterCodeGenFlags::RegisterCodeGenFlags() {
   CGBINDOPT(FunctionSections);
 
   static cl::opt<std::string> BBSections(
-      "basicblock-sections",
+      "basic-block-sections",
       cl::desc("Emit basic blocks into separate sections"),
       cl::value_desc("all | <function list (file)> | labels | none"),
       cl::init("none"));
@@ -352,7 +352,7 @@ codegen::RegisterCodeGenFlags::RegisterCodeGenFlags() {
   CGBINDOPT(UniqueSectionNames);
 
   static cl::opt<bool> UniqueBasicBlockSectionNames(
-      "unique-bb-section-names",
+      "unique-basic-block-section-names",
       cl::desc("Give unique names to every basic block section"),
       cl::init(false));
   CGBINDOPT(UniqueBasicBlockSectionNames);
diff --git a/llvm/test/CodeGen/X86/basicblock-sections-clusters-branches.ll b/llvm/test/CodeGen/X86/basic-block-sections-clusters-branches.ll
similarity index 93%
rename from llvm/test/CodeGen/X86/basicblock-sections-clusters-branches.ll
rename to llvm/test/CodeGen/X86/basic-block-sections-clusters-branches.ll
index a6f297392d94d..ec90c279e6ffa 100644
--- a/llvm/test/CodeGen/X86/basicblock-sections-clusters-branches.ll
+++ b/llvm/test/CodeGen/X86/basic-block-sections-clusters-branches.ll
@@ -7,7 +7,7 @@
 ; RUN: echo '!foo' > %t1
 ; RUN: echo '!!0 2' >> %t1
 ; RUN: echo '!!1' >> %t1
-; RUN: llc < %s -O0 -mtriple=x86_64-pc-linux -function-sections -basicblock-sections=%t1 | FileCheck %s -check-prefix=LINUX-SECTIONS1
+; RUN: llc < %s -O0 -mtriple=x86_64-pc-linux -function-sections -basic-block-sections=%t1 | FileCheck %s -check-prefix=LINUX-SECTIONS1
 ;
 ; Test2: Basic blocks #1 and #3 will be placed in the same section.
 ; The rest (#0 and #2) go into the function's section.
@@ -15,7 +15,7 @@
 ; #2 must have an explicit jump to #3.
 ; RUN: echo '!foo' > %t2
 ; RUN: echo '!!1 3' >> %t2
-; RUN: llc < %s -O0 -mtriple=x86_64-pc-linux -function-sections -basicblock-sections=%t2 | FileCheck %s -check-prefix=LINUX-SECTIONS2
+; RUN: llc < %s -O0 -mtriple=x86_64-pc-linux -function-sections -basic-block-sections=%t2 | FileCheck %s -check-prefix=LINUX-SECTIONS2
 
 define void @foo(i1 zeroext) nounwind {
   %2 = alloca i8, align 1
diff --git a/llvm/test/CodeGen/X86/basicblock-sections-clusters-eh.ll b/llvm/test/CodeGen/X86/basic-block-sections-clusters-eh.ll
similarity index 95%
rename from llvm/test/CodeGen/X86/basicblock-sections-clusters-eh.ll
rename to llvm/test/CodeGen/X86/basic-block-sections-clusters-eh.ll
index 60f62d4c152ff..4e80c72f11036 100644
--- a/llvm/test/CodeGen/X86/basicblock-sections-clusters-eh.ll
+++ b/llvm/test/CodeGen/X86/basic-block-sections-clusters-eh.ll
@@ -5,14 +5,14 @@
 ; The rest will be placed in a section along with the entry basic block.
 ; RUN: echo '!main' > %t1
 ; RUN: echo '!!1 2' >> %t1
-; RUN: llc < %s -O0 -mtriple=x86_64-pc-linux -function-sections -basicblock-sections=%t1 | FileCheck %s -check-prefix=LINUX-SECTIONS1
+; RUN: llc < %s -O0 -mtriple=x86_64-pc-linux -function-sections -basic-block-sections=%t1 | FileCheck %s -check-prefix=LINUX-SECTIONS1
 ;
 ; Test2: Basic blocks #1, #2, and #3 go into a separate section.
 ; No separate exception section will be created as #1 and #3 are already in one section.
 ; The rest will be placed in a section along with the entry basic block.
 ; RUN: echo '!main' > %t2
 ; RUN: echo '!!1 2 3' >> %t2
-; RUN: llc < %s -O0 -mtriple=x86_64-pc-linux -function-sections -basicblock-sections=%t2 | FileCheck %s -check-prefix=LINUX-SECTIONS2
+; RUN: llc < %s -O0 -mtriple=x86_64-pc-linux -function-sections -basic-block-sections=%t2 | FileCheck %s -check-prefix=LINUX-SECTIONS2
 
 @_ZTIi = external constant i8*
 
diff --git a/llvm/test/CodeGen/X86/basicblock-sections-clusters-error.ll b/llvm/test/CodeGen/X86/basic-block-sections-clusters-error.ll
similarity index 75%
rename from llvm/test/CodeGen/X86/basicblock-sections-clusters-error.ll
rename to llvm/test/CodeGen/X86/basic-block-sections-clusters-error.ll
index 29946d0f5b4bf..28c79e28fc038 100644
--- a/llvm/test/CodeGen/X86/basicblock-sections-clusters-error.ll
+++ b/llvm/test/CodeGen/X86/basic-block-sections-clusters-error.ll
@@ -2,19 +2,19 @@
 ; RUN: echo '!f' > %t1
 ; RUN: echo '!!1 4' >> %t1
 ; RUN: echo '!!1' >> %t1
-; RUN: not --crash llc < %s -O0 -mtriple=x86_64-pc-linux -function-sections -basicblock-sections=%t1 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR1
+; RUN: not --crash llc < %s -O0 -mtriple=x86_64-pc-linux -function-sections -basic-block-sections=%t1 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR1
 ; CHECK-ERROR1: LLVM ERROR: Invalid profile {{.*}} at line 3: Duplicate basic block id found '1'.
 ; RUN: echo '!f' > %t2
 ; RUN: echo '!!4 0' >> %t2
-; RUN: not --crash llc < %s -O0 -mtriple=x86_64-pc-linux -function-sections -basicblock-sections=%t2 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR2
+; RUN: not --crash llc < %s -O0 -mtriple=x86_64-pc-linux -function-sections -basic-block-sections=%t2 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR2
 ; CHECK-ERROR2: LLVM ERROR: Invalid profile {{.*}} at line 2: Entry BB (0) does not begin a cluster.
 ; RUN: echo '!f' > %t3
 ; RUN: echo '!!-1' >> %t3
-; RUN: not --crash llc < %s -O0 -mtriple=x86_64-pc-linux -function-sections -basicblock-sections=%t3 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR3
+; RUN: not --crash llc < %s -O0 -mtriple=x86_64-pc-linux -function-sections -basic-block-sections=%t3 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR3
 ; CHECK-ERROR3: LLVM ERROR: Invalid profile {{.*}} at line 2: Unsigned integer expected: '-1'.
 ; RUN: echo '!!1' > %t4
 ; RUN: echo '!f' >> %t4
-; RUN: not --crash llc < %s -O0 -mtriple=x86_64-pc-linux -function-sections -basicblock-sections=%t4 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR4
+; RUN: not --crash llc < %s -O0 -mtriple=x86_64-pc-linux -function-sections -basic-block-sections=%t4 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR4
 ; CHECK-ERROR4: LLVM ERROR: Invalid profile {{.*}} at line 1: Cluster list does not follow a function name specifier.
 
 define i32 @dummy(i32 %x, i32 %y, i32 %z) {
diff --git a/llvm/test/CodeGen/X86/basicblock-sections-clusters.ll b/llvm/test/CodeGen/X86/basic-block-sections-clusters.ll
similarity index 94%
rename from llvm/test/CodeGen/X86/basicblock-sections-clusters.ll
rename to llvm/test/CodeGen/X86/basic-block-sections-clusters.ll
index 2c40542f6808f..12759496fddd3 100644
--- a/llvm/test/CodeGen/X86/basicblock-sections-clusters.ll
+++ b/llvm/test/CodeGen/X86/basic-block-sections-clusters.ll
@@ -6,13 +6,13 @@
 ; RUN: echo '!foo' > %t1
 ; RUN: echo '!!0 2' >> %t1
 ; RUN: echo '!!1' >> %t1
-; RUN: llc < %s -O0 -mtriple=x86_64-pc-linux -function-sections -basicblock-sections=%t1 | FileCheck %s -check-prefix=LINUX-SECTIONS1
+; RUN: llc < %s -O0 -mtriple=x86_64-pc-linux -function-sections -basic-block-sections=%t1 | FileCheck %s -check-prefix=LINUX-SECTIONS1
 ;
 ; Test2: Basic blocks #1 and #3 will be placed in the same section.
 ; All other BBs (including the entry block) go into the function's section.
 ; RUN: echo '!foo' > %t2
 ; RUN: echo '!!1 3' >> %t2
-; RUN: llc < %s -O0 -mtriple=x86_64-pc-linux -function-sections -basicblock-sections=%t2 | FileCheck %s -check-prefix=LINUX-SECTIONS2
+; RUN: llc < %s -O0 -mtriple=x86_64-pc-linux -function-sections -basic-block-sections=%t2 | FileCheck %s -check-prefix=LINUX-SECTIONS2
 
 define void @foo(i1 zeroext) nounwind {
   %2 = alloca i8, align 1
diff --git a/llvm/test/CodeGen/X86/basicblock-sections-cold.ll b/llvm/test/CodeGen/X86/basic-block-sections-cold.ll
similarity index 93%
rename from llvm/test/CodeGen/X86/basicblock-sections-cold.ll
rename to llvm/test/CodeGen/X86/basic-block-sections-cold.ll
index c7282a1e57368..432c0952a4a08 100644
--- a/llvm/test/CodeGen/X86/basicblock-sections-cold.ll
+++ b/llvm/test/CodeGen/X86/basic-block-sections-cold.ll
@@ -2,7 +2,7 @@
 ; Basic block with id 1 and 2 must be in the cold section.
 ; RUN: echo '!_Z3bazb' > %t
 ; RUN: echo '!!0' >> %t
-; RUN: llc < %s -mtriple=x86_64-pc-linux -function-sections -basicblock-sections=%t -unique-bb-section-names | FileCheck %s -check-prefix=LINUX-SECTIONS
+; RUN: llc < %s -mtriple=x86_64-pc-linux -function-sections -basic-block-sections=%t -unique-basic-block-section-names | FileCheck %s -check-prefix=LINUX-SECTIONS
 
 define void @_Z3bazb(i1 zeroext) nounwind {
   %2 = alloca i8, align 1
diff --git a/llvm/test/CodeGen/X86/basicblock-sections-directjumps.ll b/llvm/test/CodeGen/X86/basic-block-sections-directjumps.ll
similarity index 84%
rename from llvm/test/CodeGen/X86/basicblock-sections-directjumps.ll
rename to llvm/test/CodeGen/X86/basic-block-sections-directjumps.ll
index 8604b129b54cf..99a64ef130319 100644
--- a/llvm/test/CodeGen/X86/basicblock-sections-directjumps.ll
+++ b/llvm/test/CodeGen/X86/basic-block-sections-directjumps.ll
@@ -1,5 +1,5 @@
-; RUN: llc < %s -mtriple=x86_64-pc-linux -function-sections -basicblock-sections=all -unique-bb-section-names | FileCheck %s -check-prefix=LINUX-SECTIONS
-; RUN: llc < %s -mtriple=i386-unknown-linux-gnu  -function-sections -basicblock-sections=all -unique-bb-section-names | FileCheck %s -check-prefix=LINUX-SECTIONS
+; RUN: llc < %s -mtriple=x86_64-pc-linux -function-sections -basic-block-sections=all -unique-basic-block-section-names | FileCheck %s -check-prefix=LINUX-SECTIONS
+; RUN: llc < %s -mtriple=i386-unknown-linux-gnu  -function-sections -basic-block-sections=all -unique-basic-block-section-names | FileCheck %s -check-prefix=LINUX-SECTIONS
 
 define void @_Z3bazb(i1 zeroext) {
   %2 = alloca i8, align 1
diff --git a/llvm/test/CodeGen/X86/basicblock-sections-eh.ll b/llvm/test/CodeGen/X86/basic-block-sections-eh.ll
similarity index 96%
rename from llvm/test/CodeGen/X86/basicblock-sections-eh.ll
rename to llvm/test/CodeGen/X86/basic-block-sections-eh.ll
index 7e5f4a2fe3927..3cc5979d7c0c0 100644
--- a/llvm/test/CodeGen/X86/basicblock-sections-eh.ll
+++ b/llvm/test/CodeGen/X86/basic-block-sections-eh.ll
@@ -1,5 +1,5 @@
 ; Check if landing pads are kept in a separate eh section
-; RUN: llc < %s -mtriple=i386-unknown-linux-gnu  -function-sections -basicblock-sections=all -unique-bb-section-names | FileCheck %s -check-prefix=LINUX-SECTIONS
+; RUN: llc < %s -mtriple=i386-unknown-linux-gnu  -function-sections -basic-block-sections=all -unique-basic-block-section-names | FileCheck %s -check-prefix=LINUX-SECTIONS
 
 @_ZTIb = external constant i8*
 define i32 @_Z3foob(i1 zeroext %0) #0 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
diff --git a/llvm/test/CodeGen/X86/basicblock-sections-labels.ll b/llvm/test/CodeGen/X86/basic-block-sections-labels.ll
similarity index 92%
rename from llvm/test/CodeGen/X86/basicblock-sections-labels.ll
rename to llvm/test/CodeGen/X86/basic-block-sections-labels.ll
index 2f077e6866924..80aaf79c115a4 100644
--- a/llvm/test/CodeGen/X86/basicblock-sections-labels.ll
+++ b/llvm/test/CodeGen/X86/basic-block-sections-labels.ll
@@ -1,5 +1,5 @@
 ; Check the basic block sections labels option
-; RUN: llc < %s -mtriple=x86_64-pc-linux -function-sections -basicblock-sections=labels | FileCheck %s -check-prefix=LINUX-LABELS
+; RUN: llc < %s -mtriple=x86_64-pc-linux -function-sections -basic-block-sections=labels | FileCheck %s -check-prefix=LINUX-LABELS
 
 define void @_Z3bazb(i1 zeroext) {
   %2 = alloca i8, align 1
diff --git a/llvm/test/CodeGen/X86/basicblock-sections-list.ll b/llvm/test/CodeGen/X86/basic-block-sections-list.ll
similarity index 95%
rename from llvm/test/CodeGen/X86/basicblock-sections-list.ll
rename to llvm/test/CodeGen/X86/basic-block-sections-list.ll
index 9a5056af39ed0..7c48628f73a8c 100644
--- a/llvm/test/CodeGen/X86/basicblock-sections-list.ll
+++ b/llvm/test/CodeGen/X86/basic-block-sections-list.ll
@@ -1,6 +1,6 @@
 ; Check the basic block sections list option.
 ; RUN: echo '!_Z3foob' > %t
-; RUN: llc < %s -mtriple=x86_64-pc-linux -function-sections -basicblock-sections=%t -unique-bb-section-names | FileCheck %s -check-prefix=LINUX-SECTIONS
+; RUN: llc < %s -mtriple=x86_64-pc-linux -function-sections -basic-block-sections=%t -unique-basic-block-section-names | FileCheck %s -check-prefix=LINUX-SECTIONS
 
 define i32 @_Z3foob(i1 zeroext %0) nounwind {
   %2 = alloca i32, align 4
diff --git a/llvm/test/CodeGen/X86/basicblock-sections-listbb.ll b/llvm/test/CodeGen/X86/basic-block-sections-listbb.ll
similarity index 93%
rename from llvm/test/CodeGen/X86/basicblock-sections-listbb.ll
rename to llvm/test/CodeGen/X86/basic-block-sections-listbb.ll
index ac17a461d7af9..ab729ecf9eb24 100644
--- a/llvm/test/CodeGen/X86/basicblock-sections-listbb.ll
+++ b/llvm/test/CodeGen/X86/basic-block-sections-listbb.ll
@@ -2,7 +2,7 @@
 ; Only basic block with id 2 must get a section.
 ; RUN: echo '!_Z3bazb' > %t
 ; RUN: echo '!!2' >> %t
-; RUN: llc < %s -mtriple=x86_64-pc-linux -function-sections -basicblock-sections=%t -unique-bb-section-names | FileCheck %s -check-prefix=LINUX-SECTIONS
+; RUN: llc < %s -mtriple=x86_64-pc-linux -function-sections -basic-block-sections=%t -unique-basic-block-section-names | FileCheck %s -check-prefix=LINUX-SECTIONS
 
 define void @_Z3bazb(i1 zeroext) nounwind {
   %2 = alloca i8, align 1
diff --git a/llvm/test/CodeGen/X86/basicblock-sections-mir-parse.mir b/llvm/test/CodeGen/X86/basic-block-sections-mir-parse.mir
similarity index 98%
rename from llvm/test/CodeGen/X86/basicblock-sections-mir-parse.mir
rename to llvm/test/CodeGen/X86/basic-block-sections-mir-parse.mir
index 6011342a6f003..a4219fa1509d5 100644
--- a/llvm/test/CodeGen/X86/basicblock-sections-mir-parse.mir
+++ b/llvm/test/CodeGen/X86/basic-block-sections-mir-parse.mir
@@ -10,7 +10,7 @@
 # }
 #
 # clang -O0 -S -emit-llvm foo.cc
-# llc < foo.ll -stop-after=bbsections-prepare -basicblock-sections=all
+# llc < foo.ll -stop-after=bbsections-prepare -basic-block-sections=all
 
 --- |
   ; Function Attrs: noinline nounwind optnone uwtable
diff --git a/llvm/test/CodeGen/X86/basicblock-sections-mir-print.ll b/llvm/test/CodeGen/X86/basic-block-sections-mir-print.ll
similarity index 91%
rename from llvm/test/CodeGen/X86/basicblock-sections-mir-print.ll
rename to llvm/test/CodeGen/X86/basic-block-sections-mir-print.ll
index bd66f44ecc6db..7b3c7246971da 100644
--- a/llvm/test/CodeGen/X86/basicblock-sections-mir-print.ll
+++ b/llvm/test/CodeGen/X86/basic-block-sections-mir-print.ll
@@ -2,7 +2,7 @@
 ; RUN: echo '!_Z3foob' > %t
 ; RUN: echo '!!1' >> %t
 ; RUN: echo '!!2' >> %t
-; RUN: llc < %s -O0 -mtriple=x86_64-pc-linux -function-sections -basicblock-sections=%t -stop-after=bbsections-prepare | FileCheck %s -check-prefix=CHECK
+; RUN: llc < %s -O0 -mtriple=x86_64-pc-linux -function-sections -basic-block-sections=%t -stop-after=bbsections-prepare | FileCheck %s -check-prefix=CHECK
 
 @_ZTIb = external constant i8*
 define dso_local i32 @_Z3foob(i1 zeroext %0) {
diff --git a/llvm/test/CodeGen/X86/basicblock-sections.ll b/llvm/test/CodeGen/X86/basic-block-sections.ll
similarity index 84%
rename from llvm/test/CodeGen/X86/basicblock-sections.ll
rename to llvm/test/CodeGen/X86/basic-block-sections.ll
index 5c17b755fa70b..d996f5e9f5397 100644
--- a/llvm/test/CodeGen/X86/basicblock-sections.ll
+++ b/llvm/test/CodeGen/X86/basic-block-sections.ll
@@ -1,5 +1,5 @@
-; RUN: llc < %s -mtriple=x86_64-pc-linux -function-sections -basicblock-sections=all -unique-bb-section-names | FileCheck %s -check-prefix=LINUX-SECTIONS
-; RUN: llc < %s -mtriple=i386-unknown-linux-gnu  -function-sections -basicblock-sections=all -unique-bb-section-names | FileCheck %s -check-prefix=LINUX-SECTIONS
+; RUN: llc < %s -mtriple=x86_64-pc-linux -function-sections -basic-block-sections=all -unique-basic-block-section-names | FileCheck %s -check-prefix=LINUX-SECTIONS
+; RUN: llc < %s -mtriple=i386-unknown-linux-gnu  -function-sections -basic-block-sections=all -unique-basic-block-section-names | FileCheck %s -check-prefix=LINUX-SECTIONS
 
 define void @_Z3bazb(i1 zeroext) nounwind {
   %2 = alloca i8, align 1
diff --git a/llvm/test/CodeGen/X86/cfi-basic-block-sections-1.ll b/llvm/test/CodeGen/X86/cfi-basic-block-sections-1.ll
index 62e669eff9e4e..a5a6d697451fc 100644
--- a/llvm/test/CodeGen/X86/cfi-basic-block-sections-1.ll
+++ b/llvm/test/CodeGen/X86/cfi-basic-block-sections-1.ll
@@ -1,6 +1,6 @@
-; RUN: llc -O0 %s --basicblock-sections=all -mtriple=x86_64 -filetype=asm --frame-pointer=all -o - | FileCheck --check-prefix=SECTIONS_CFI %s
-; RUN: llc -O0 %s --basicblock-sections=all -mtriple=x86_64 -filetype=asm --frame-pointer=none -o - | FileCheck --check-prefix=SECTIONS_NOFP_CFI %s
-; RUN: llc -O0 %s --basicblock-sections=all -mtriple=x86_64 -filetype=obj --frame-pointer=all -o - | llvm-dwarfdump --eh-frame  - | FileCheck --check-prefix=EH_FRAME %s
+; RUN: llc -O0 %s --basic-block-sections=all -mtriple=x86_64 -filetype=asm --frame-pointer=all -o - | FileCheck --check-prefix=SECTIONS_CFI %s
+; RUN: llc -O0 %s --basic-block-sections=all -mtriple=x86_64 -filetype=asm --frame-pointer=none -o - | FileCheck --check-prefix=SECTIONS_NOFP_CFI %s
+; RUN: llc -O0 %s --basic-block-sections=all -mtriple=x86_64 -filetype=obj --frame-pointer=all -o - | llvm-dwarfdump --eh-frame  - | FileCheck --check-prefix=EH_FRAME %s
 
 ;; void f1();
 ;; void f3(bool b) {
diff --git a/llvm/test/CodeGen/X86/cfi-inserter-basic-block-sections-callee-save-registers.ll b/llvm/test/CodeGen/X86/cfi-inserter-basic-block-sections-callee-save-registers.ll
index 19725138f6ed9..d87ead0e864cf 100644
--- a/llvm/test/CodeGen/X86/cfi-inserter-basic-block-sections-callee-save-registers.ll
+++ b/llvm/test/CodeGen/X86/cfi-inserter-basic-block-sections-callee-save-registers.ll
@@ -1,6 +1,6 @@
 ;; This test checks if CFI instructions for all callee saved registers are emitted
 ;; correctly with basic block sections.
-; RUN: llc  %s -mtriple=x86_64 -filetype=asm --basicblock-sections=all --frame-pointer=all -o - | FileCheck --check-prefix=SECTIONS_CFI %s
+; RUN: llc  %s -mtriple=x86_64 -filetype=asm --basic-block-sections=all --frame-pointer=all -o - | FileCheck --check-prefix=SECTIONS_CFI %s
 
 ; SECTIONS_CFI:       _Z3foob:
 ; SECTIONS_CFI:      .cfi_offset %rbp, -16
diff --git a/llvm/test/DebugInfo/X86/basicblock-sections_1.ll b/llvm/test/DebugInfo/X86/basic-block-sections_1.ll
similarity index 88%
rename from llvm/test/DebugInfo/X86/basicblock-sections_1.ll
rename to llvm/test/DebugInfo/X86/basic-block-sections_1.ll
index f3bedd977693f..655e84731cba1 100644
--- a/llvm/test/DebugInfo/X86/basicblock-sections_1.ll
+++ b/llvm/test/DebugInfo/X86/basic-block-sections_1.ll
@@ -1,7 +1,7 @@
 ; RUN: llc -O0 %s -mtriple=x86_64-* -filetype=obj -o %t && llvm-dwarfdump  -debug-info -v %t | FileCheck --check-prefix=NO-SECTIONS %s
-; RUN: llc -O0 %s --basicblock-sections=all --unique-bb-section-names -mtriple=x86_64-* -filetype=obj -o %t && llvm-dwarfdump  -debug-info -v %t | FileCheck --check-prefix=BB-SECTIONS %s
-; RUN: llc -O0 %s --basicblock-sections=all --unique-bb-section-names -mtriple=x86_64-* -filetype=obj -split-dwarf-file=%t.dwo -o %t && llvm-dwarfdump  -debug-info -v %t | FileCheck --check-prefix=BB-SECTIONS %s
-; RUN: llc -O0 %s --basicblock-sections=all -mtriple=x86_64-* -filetype=asm -o - | FileCheck --check-prefix=BB-SECTIONS-ASM %s
+; RUN: llc -O0 %s --basic-block-sections=all --unique-basic-block-section-names -mtriple=x86_64-* -filetype=obj -o %t && llvm-dwarfdump  -debug-info -v %t | FileCheck --check-prefix=BB-SECTIONS %s
+; RUN: llc -O0 %s --basic-block-sections=all --unique-basic-block-section-names -mtriple=x86_64-* -filetype=obj -split-dwarf-file=%t.dwo -o %t && llvm-dwarfdump  -debug-info -v %t | FileCheck --check-prefix=BB-SECTIONS %s
+; RUN: llc -O0 %s --basic-block-sections=all -mtriple=x86_64-* -filetype=asm -o - | FileCheck --check-prefix=BB-SECTIONS-ASM %s
 
 ; From:
 ; int foo(int a) {

From 7212ad067e6efcd8431a9e38f26de45ae21eeafb Mon Sep 17 00:00:00 2001
From: Chris Gyurgyik <37983775+cgyurgyik@users.noreply.github.com>
Date: Fri, 31 Jul 2020 14:57:46 -0400
Subject: [PATCH 0777/1035] [libc] [obvious] Add rest of strrchr test.

---
 libc/test/src/string/strrchr_test.cpp | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/libc/test/src/string/strrchr_test.cpp b/libc/test/src/string/strrchr_test.cpp
index 18fddda600870..cf29de220d498 100644
--- a/libc/test/src/string/strrchr_test.cpp
+++ b/libc/test/src/string/strrchr_test.cpp
@@ -49,12 +49,15 @@ TEST(StrRChrTest, FindsNullTerminator) {
   ASSERT_STREQ(src, src_copy);
 }
 
-TEST(StrRChrTest, FindsLastNullTerminator) {
-  const char src[5] = {'a', '\0', 'b', '\0', 'c'};
+TEST(StrRChrTest, FindsLastBehindFirstNullTerminator) {
+  const char src[6] = {'a', 'a', '\0', 'b', '\0', 'c'};
   // 'b' is behind a null terminator, so should not be found.
   ASSERT_STREQ(__llvm_libc::strrchr(src, 'b'), nullptr);
   // Same goes for 'c'.
   ASSERT_STREQ(__llvm_libc::strrchr(src, 'c'), nullptr);
+  
+  // Should find the second of the two a's.
+  ASSERT_STREQ(__llvm_libc::strrchr(src, 'a'), "a");
 }
 
 TEST(StrRChrTest, CharacterNotWithinStringShouldReturnNullptr) {

From a77afc62d9e48c334e5a7a03fe6ffe3b614742d5 Mon Sep 17 00:00:00 2001
From: Sriraman Tallam <tmsriram@google.com>
Date: Fri, 31 Jul 2020 12:00:59 -0700
Subject: [PATCH 0778/1035] New test for basic block sections options.

This tests lld basic block sections options:
+ --lto-basic-block-sections=
+ --lto-unique-basic-block-section-names

Differential Revision: https://reviews.llvm.org/D84462
---
 lld/test/ELF/lto/basic-block-sections.ll | 41 ++++++++++++++++++++++++
 1 file changed, 41 insertions(+)
 create mode 100644 lld/test/ELF/lto/basic-block-sections.ll

diff --git a/lld/test/ELF/lto/basic-block-sections.ll b/lld/test/ELF/lto/basic-block-sections.ll
new file mode 100644
index 0000000000000..04cd296262b1e
--- /dev/null
+++ b/lld/test/ELF/lto/basic-block-sections.ll
@@ -0,0 +1,41 @@
+; REQUIRES: x86
+; RUN: llvm-as %s -o %t.o
+; RUN: ld.lld %t.o -o %t --lto-basic-block-sections=all --lto-O0 --save-temps
+; RUN: llvm-readobj -s %t.lto.o | FileCheck --check-prefix=SECNAMES %s
+; RUN: ld.lld %t.o -o %t --lto-basic-block-sections=all --lto-unique-basic-block-section-names --lto-O0 --save-temps
+; RUN: llvm-readobj -s %t.lto.o | FileCheck --check-prefix=SECNAMES-FULL %s
+; RUN: llvm-nm %t.out | FileCheck --check-prefix=SYMS %s
+
+; SECNAMES: Name: .text.foo {{.*}}
+; SECNAMES: Name: .text.foo {{.*}}
+; SECNAMES: Name: .text.foo {{.*}}
+
+; SECNAMES-FULL: Name: .text.foo {{.*}}
+; SECNAMES-FULL: Name: .text.foo.foo.1 {{.*}}
+; SECNAMES-FULL: Name: .text.foo.foo.2 {{.*}}
+
+; SYMS: foo
+; SYMS: foo.1
+; SYMS: foo.2
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function Attrs: nounwind uwtable
+define dso_local void @foo(i32 %b) local_unnamed_addr {
+entry:
+  %tobool.not = icmp eq i32 %b, 0
+  br i1 %tobool.not, label %if.end, label %if.then
+
+if.then:                                          ; preds = %entry
+  tail call void @foo(i32 0)
+  br label %if.end
+
+if.end:                                           ; preds = %entry, %if.then
+  ret void
+}
+
+define void @_start() {
+  call void @foo(i32 1)
+  ret void
+}

From 938adf42e6b07414d4194ce8b76bbdcdc9df6459 Mon Sep 17 00:00:00 2001
From: Sriraman Tallam <tmsriram@google.com>
Date: Fri, 31 Jul 2020 12:26:53 -0700
Subject: [PATCH 0779/1035] Fix a test typo which caused a breakage.

---
 lld/test/ELF/lto/basic-block-sections.ll | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lld/test/ELF/lto/basic-block-sections.ll b/lld/test/ELF/lto/basic-block-sections.ll
index 04cd296262b1e..1f932ac50a879 100644
--- a/lld/test/ELF/lto/basic-block-sections.ll
+++ b/lld/test/ELF/lto/basic-block-sections.ll
@@ -4,7 +4,7 @@
 ; RUN: llvm-readobj -s %t.lto.o | FileCheck --check-prefix=SECNAMES %s
 ; RUN: ld.lld %t.o -o %t --lto-basic-block-sections=all --lto-unique-basic-block-section-names --lto-O0 --save-temps
 ; RUN: llvm-readobj -s %t.lto.o | FileCheck --check-prefix=SECNAMES-FULL %s
-; RUN: llvm-nm %t.out | FileCheck --check-prefix=SYMS %s
+; RUN: llvm-nm %t | FileCheck --check-prefix=SYMS %s
 
 ; SECNAMES: Name: .text.foo {{.*}}
 ; SECNAMES: Name: .text.foo {{.*}}

From 38d3e7533279fd4bfefcd88eac7d3b64f804c53a Mon Sep 17 00:00:00 2001
From: Jaydeep Chauhan <jaydeepchauhan1494@gmail.com>
Date: Fri, 31 Jul 2020 20:22:22 +0100
Subject: [PATCH 0780/1035] [clang] Use the location of the void parameters
 when complaining that only a single void parameter should be present.

Fixes PR46417.

Differential Revision: https://reviews.llvm.org/D84678

Reviewed By: aaron.ballman
---
 clang/lib/Sema/SemaType.cpp          |  2 +-
 clang/test/SemaCXX/void-argument.cpp | 25 +++++++++++++++++++++++++
 2 files changed, 26 insertions(+), 1 deletion(-)
 create mode 100644 clang/test/SemaCXX/void-argument.cpp

diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index abf1d6450036f..ff5223c0795e5 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -5114,7 +5114,7 @@ static TypeSourceInfo *GetFullTypeForDeclarator(TypeProcessingState &state,
             // is an incomplete type (C99 6.2.5p19) and function decls cannot
             // have parameters of incomplete type.
             if (FTI.NumParams != 1 || FTI.isVariadic) {
-              S.Diag(DeclType.Loc, diag::err_void_only_param);
+              S.Diag(FTI.Params[i].IdentLoc, diag::err_void_only_param);
               ParamTy = Context.IntTy;
               Param->setType(ParamTy);
             } else if (FTI.Params[i].Ident) {
diff --git a/clang/test/SemaCXX/void-argument.cpp b/clang/test/SemaCXX/void-argument.cpp
new file mode 100644
index 0000000000000..8354347f5559e
--- /dev/null
+++ b/clang/test/SemaCXX/void-argument.cpp
@@ -0,0 +1,25 @@
+// RUN: %clang_cc1 -fsyntax-only -verify %s
+
+void fun(
+    void a, // expected-error{{'void' must be the first and only parameter if specified}}
+    double b,
+    int c,
+    void d, // expected-error{{'void' must be the first and only parameter if specified}}
+    int e,
+    void f) // expected-error{{'void' must be the first and only parameter if specified}}
+{}
+
+void foo(
+    int a,
+    void, // expected-error{{'void' must be the first and only parameter if specified}}
+    int b);
+
+void bar(
+    void, // expected-error{{'void' must be the first and only parameter if specified}}
+    ...);
+
+struct S {
+  S(
+      void,  // expected-error{{'void' must be the first and only parameter if specified}}
+      void); // expected-error{{'void' must be the first and only parameter if specified}}
+};

From 8739445e3221ea05bc71a592f4114e10510b0b34 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Lubo=C5=A1=20Lu=C5=88=C3=A1k?= <l.lunak@centrum.cz>
Date: Thu, 30 Jul 2020 21:25:38 +0200
Subject: [PATCH 0781/1035] [lldb] force full gui redraw on Ctrl+L

As is common with curses apps, this allows to redraw everything
in case something corrupts the screen. Apparently key modifiers
are difficult with curses (curses FAQ it "doesn't do that"),
thankfully Ctrl+key are simply control characters, so it's
(ascii & 037) => 12.

Differential Revision: https://reviews.llvm.org/D84972
---
 lldb/source/Core/IOHandlerCursesGUI.cpp | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/lldb/source/Core/IOHandlerCursesGUI.cpp b/lldb/source/Core/IOHandlerCursesGUI.cpp
index fda3aa1886795..262a19dc04b4c 100644
--- a/lldb/source/Core/IOHandlerCursesGUI.cpp
+++ b/lldb/source/Core/IOHandlerCursesGUI.cpp
@@ -1287,6 +1287,10 @@ class Application {
           update = true;
           break;
         case eKeyNotHandled:
+          if (ch == 12) { // Ctrl+L, force full redraw
+            redrawwin(m_window_sp->get());
+            update = true;
+          }
           break;
         case eQuitApplication:
           done = true;

From 77d5a63c191ca791f081ff153276170bbfb10cee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Lubo=C5=A1=20Lu=C5=88=C3=A1k?= <l.lunak@centrum.cz>
Date: Thu, 30 Jul 2020 18:08:13 +0200
Subject: [PATCH 0782/1035] [lldb] report an error if a CLI option lacks an
 argument

Differential Revision: https://reviews.llvm.org/D84955
---
 lldb/test/Shell/Driver/TestError.test |  2 ++
 lldb/tools/driver/Driver.cpp          | 15 ++++++++++++---
 2 files changed, 14 insertions(+), 3 deletions(-)
 create mode 100644 lldb/test/Shell/Driver/TestError.test

diff --git a/lldb/test/Shell/Driver/TestError.test b/lldb/test/Shell/Driver/TestError.test
new file mode 100644
index 0000000000000..3d34a72b14aba
--- /dev/null
+++ b/lldb/test/Shell/Driver/TestError.test
@@ -0,0 +1,2 @@
+RUN: not %lldb --arch 2>&1 | FileCheck %s
+CHECK: error: argument to '--arch' is missing
diff --git a/lldb/tools/driver/Driver.cpp b/lldb/tools/driver/Driver.cpp
index cea9e5a44aa80..0cd1ffc57aa7a 100644
--- a/lldb/tools/driver/Driver.cpp
+++ b/lldb/tools/driver/Driver.cpp
@@ -853,10 +853,11 @@ int main(int argc, char const *argv[]) {
 
   // Parse arguments.
   LLDBOptTable T;
-  unsigned MAI;
-  unsigned MAC;
+  unsigned MissingArgIndex;
+  unsigned MissingArgCount;
   ArrayRef<const char *> arg_arr = makeArrayRef(argv + 1, argc - 1);
-  opt::InputArgList input_args = T.ParseArgs(arg_arr, MAI, MAC);
+  opt::InputArgList input_args =
+      T.ParseArgs(arg_arr, MissingArgIndex, MissingArgCount);
   llvm::StringRef argv0 = llvm::sys::path::filename(argv[0]);
 
   if (input_args.hasArg(OPT_help)) {
@@ -864,11 +865,19 @@ int main(int argc, char const *argv[]) {
     return 0;
   }
 
+  // Check for missing argument error.
+  if (MissingArgCount) {
+    WithColor::error() << "argument to '"
+                       << input_args.getArgString(MissingArgIndex)
+                       << "' is missing\n";
+  }
   // Error out on unknown options.
   if (input_args.hasArg(OPT_UNKNOWN)) {
     for (auto *arg : input_args.filtered(OPT_UNKNOWN)) {
       WithColor::error() << "unknown option: " << arg->getSpelling() << '\n';
     }
+  }
+  if (MissingArgCount || input_args.hasArg(OPT_UNKNOWN)) {
     llvm::errs() << "Use '" << argv0
                  << " --help' for a complete list of options.\n";
     return 1;

From eb8c72cb0d8949114350d2f24a1741898b63dae5 Mon Sep 17 00:00:00 2001
From: Rahul Joshi <jurahul@google.com>
Date: Fri, 31 Jul 2020 11:39:49 -0700
Subject: [PATCH 0783/1035] [MLIR][NFC] Add FuncOp::getArgumentTypes()

Differential Revision: https://reviews.llvm.org/D85038
---
 mlir/include/mlir/IR/FunctionSupport.h | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/mlir/include/mlir/IR/FunctionSupport.h b/mlir/include/mlir/IR/FunctionSupport.h
index b358215ca9624..7e281f393af94 100644
--- a/mlir/include/mlir/IR/FunctionSupport.h
+++ b/mlir/include/mlir/IR/FunctionSupport.h
@@ -206,6 +206,7 @@ class FunctionLike : public OpTrait::TraitBase<ConcreteType, FunctionLike> {
   //===--------------------------------------------------------------------===//
   // Argument Handling
   //===--------------------------------------------------------------------===//
+  using BlockArgListType = Region::BlockArgListType;
 
   unsigned getNumArguments() {
     return static_cast<ConcreteType *>(this)->getNumFuncArguments();
@@ -224,6 +225,10 @@ class FunctionLike : public OpTrait::TraitBase<ConcreteType, FunctionLike> {
   args_iterator args_end() { return getBody().args_end(); }
   Block::BlockArgListType getArguments() { return getBody().getArguments(); }
 
+  ValueTypeRange<BlockArgListType> getArgumentTypes() {
+    return getBody().getArgumentTypes();
+  }
+
   //===--------------------------------------------------------------------===//
   // Argument Attributes
   //===--------------------------------------------------------------------===//

From 2a6c8b2e9581ebca4b05d1e64458f2dccf3db61f Mon Sep 17 00:00:00 2001
From: River Riddle <riddleriver@gmail.com>
Date: Fri, 31 Jul 2020 13:18:13 -0700
Subject: [PATCH 0784/1035] [mlir][PassIncGen] Refactor how pass registration
 is generated

The current output is a bit clunky and requires including files+macros everywhere, or manually wrapping the file inclusion in a registration function. This revision refactors the pass backend to automatically generate `registerFooPass`/`registerFooPasses` functions that wrap the pass registration. `gen-pass-decls` now takes a `-name` input that specifies a tag name for the group of passes that are being generated. For each pass, the generator now produces a `registerFooPass` where `Foo` is the name of the definition specified in tablegen. It also generates a `registerGroupPasses`, where `Group` is the tag provided via the `-name` input parameter, that registers all of the passes present.

Differential Revision: https://reviews.llvm.org/D84983
---
 .../flang/Optimizer/CodeGen/CMakeLists.txt    |  2 +-
 .../include/flang/Optimizer/CodeGen/CodeGen.h |  3 -
 .../flang/Optimizer/Transforms/CMakeLists.txt |  2 +-
 .../flang/Optimizer/Transforms/Passes.h       |  3 -
 mlir/docs/PassManagement.md                   | 30 +++++--
 .../AVX512ToLLVM/ConvertAVX512ToLLVM.h        |  6 +-
 .../AffineToStandard/AffineToStandard.h       |  7 ++
 mlir/include/mlir/Conversion/CMakeLists.txt   |  2 +-
 mlir/include/mlir/Conversion/Passes.h         | 41 ++++++++++
 .../mlir/Dialect/Affine/CMakeLists.txt        |  2 +-
 mlir/include/mlir/Dialect/Affine/Passes.h     | 15 ++--
 mlir/include/mlir/Dialect/GPU/CMakeLists.txt  |  2 +-
 mlir/include/mlir/Dialect/GPU/Passes.h        | 16 ++--
 .../Dialect/LLVMIR/Transforms/CMakeLists.txt  |  2 +-
 .../mlir/Dialect/LLVMIR/Transforms/Passes.h   | 26 ++++++
 .../mlir/Dialect/Linalg/CMakeLists.txt        |  2 +-
 mlir/include/mlir/Dialect/Linalg/Passes.h     | 18 ++---
 .../include/mlir/Dialect/Quant/CMakeLists.txt |  2 +-
 mlir/include/mlir/Dialect/Quant/Passes.h      | 13 ++-
 mlir/include/mlir/Dialect/SCF/CMakeLists.txt  |  2 +-
 mlir/include/mlir/Dialect/SCF/Passes.h        | 13 ++-
 .../include/mlir/Dialect/SPIRV/CMakeLists.txt |  2 +-
 mlir/include/mlir/Dialect/SPIRV/Passes.h      |  8 ++
 .../Dialect/Shape/Transforms/CMakeLists.txt   |  2 +-
 .../mlir/Dialect/Shape/Transforms/Passes.h    | 16 ++--
 .../StandardOps/Transforms/CMakeLists.txt     |  2 +-
 .../Dialect/StandardOps/Transforms/Passes.h   | 12 ++-
 mlir/include/mlir/InitAllPasses.h             | 80 ++++---------------
 mlir/include/mlir/Transforms/CMakeLists.txt   |  2 +-
 mlir/include/mlir/Transforms/Passes.h         | 23 ++++--
 mlir/tools/mlir-tblgen/PassGen.cpp            | 59 +++++++++-----
 31 files changed, 257 insertions(+), 158 deletions(-)
 create mode 100644 mlir/include/mlir/Conversion/Passes.h
 create mode 100644 mlir/include/mlir/Dialect/LLVMIR/Transforms/Passes.h

diff --git a/flang/include/flang/Optimizer/CodeGen/CMakeLists.txt b/flang/include/flang/Optimizer/CodeGen/CMakeLists.txt
index ab6526ee18330..9acf6f89e12f5 100644
--- a/flang/include/flang/Optimizer/CodeGen/CMakeLists.txt
+++ b/flang/include/flang/Optimizer/CodeGen/CMakeLists.txt
@@ -1,6 +1,6 @@
 
 set(LLVM_TARGET_DEFINITIONS CGPasses.td)
-mlir_tablegen(CGPasses.h.inc -gen-pass-decls)
+mlir_tablegen(CGPasses.h.inc -gen-pass-decls -name OptCodeGen)
 add_public_tablegen_target(FIROptCodeGenPassIncGen)
 
 add_mlir_doc(Passes -gen-pass-doc OptimizerCodeGenPasses ./)
diff --git a/flang/include/flang/Optimizer/CodeGen/CodeGen.h b/flang/include/flang/Optimizer/CodeGen/CodeGen.h
index 9b968172f3486..a90d0a50dac64 100644
--- a/flang/include/flang/Optimizer/CodeGen/CodeGen.h
+++ b/flang/include/flang/Optimizer/CodeGen/CodeGen.h
@@ -28,12 +28,9 @@ std::unique_ptr<mlir::Pass> createFIRToLLVMPass(NameUniquer &uniquer);
 std::unique_ptr<mlir::Pass>
 createLLVMDialectToLLVMPass(llvm::raw_ostream &output);
 
-inline void registerOptCodeGenPasses() {
-  using mlir::Pass;
 // declarative passes
 #define GEN_PASS_REGISTRATION
 #include "flang/Optimizer/CodeGen/CGPasses.h.inc"
-}
 
 } // namespace fir
 
diff --git a/flang/include/flang/Optimizer/Transforms/CMakeLists.txt b/flang/include/flang/Optimizer/Transforms/CMakeLists.txt
index fde17eb88622e..b928991e0a37e 100644
--- a/flang/include/flang/Optimizer/Transforms/CMakeLists.txt
+++ b/flang/include/flang/Optimizer/Transforms/CMakeLists.txt
@@ -1,6 +1,6 @@
 
 set(LLVM_TARGET_DEFINITIONS Passes.td)
-mlir_tablegen(Passes.h.inc -gen-pass-decls)
+mlir_tablegen(Passes.h.inc -gen-pass-decls -name OptTransform)
 add_public_tablegen_target(FIROptTransformsPassIncGen)
 
 add_mlir_doc(Passes -gen-pass-doc OptimizerTransformPasses ./)
diff --git a/flang/include/flang/Optimizer/Transforms/Passes.h b/flang/include/flang/Optimizer/Transforms/Passes.h
index 9377c2dc61cc3..5e71995736e6a 100644
--- a/flang/include/flang/Optimizer/Transforms/Passes.h
+++ b/flang/include/flang/Optimizer/Transforms/Passes.h
@@ -46,12 +46,9 @@ std::unique_ptr<mlir::Pass> createMemToRegPass();
 bool canLegallyInline(mlir::Operation *op, mlir::Region *reg,
                       mlir::BlockAndValueMapping &map);
 
-inline void registerOptTransformPasses() {
-using mlir::Pass;
 // declarative passes
 #define GEN_PASS_REGISTRATION
 #include "flang/Optimizer/Transforms/Passes.h.inc"
-}
 
 } // namespace fir
 
diff --git a/mlir/docs/PassManagement.md b/mlir/docs/PassManagement.md
index 04a4ca0a7b3c5..92ca92218219c 100644
--- a/mlir/docs/PassManagement.md
+++ b/mlir/docs/PassManagement.md
@@ -622,18 +622,34 @@ def MyPass : Pass<"my-pass", "ModuleOp"> {
 }
 ```
 
-We can include the generated registration calls via:
+Using the `gen-pass-decls` generator, we can generate the much of the
+boilerplater above automatically. This generator takes as an input a `-name`
+parameter, that provides a tag for the group of passes that are being generated.
+This generator produces two chunks of output:
+
+The first is the code for registering the declarative passes with the global
+registry. For each pass, the generator produces a `registerFooPass` where `Foo`
+is the name of the definition specified in tablegen. It also generates a
+`registerGroupPasses`, where `Group` is the tag provided via the `-name` input
+parameter, that registers all of the passes present.
 
 ```c++
-void registerMyPasses() {
-  // The generated registration is not static, so we need to include this in
-  // a location that we can call into.
 #define GEN_PASS_REGISTRATION
 #include "Passes.h.inc"
+
+void registerMyPasses() {
+  // Register all of our passes.
+  registerMyPasses();
+
+  // Register `MyPass` specifically.
+  registerMyPassPass();
 }
 ```
 
-We can then update the original C++ pass definition:
+The second is a base class for each of the passes, with each containing most of
+the boiler plate related to pass definition. These classes are named in the form
+of `MyPassBase`, where `MyPass` is the name of the definition in tablegen. We
+can update the original C++ pass definition as so:
 
 ```c++
 /// Include the generated base pass class definitions.
@@ -651,6 +667,10 @@ std::unique_ptr<Pass> foo::createMyPass() {
 }
 ```
 
+Using the `gen-pass-doc` generator, we can generate markdown documentation for
+each of our passes. See [Passes.md](Passes.md) for example output of real MLIR
+passes.
+
 ### Tablegen Specification
 
 The `Pass` class is used to begin a new pass definition. This class takes as an
diff --git a/mlir/include/mlir/Conversion/AVX512ToLLVM/ConvertAVX512ToLLVM.h b/mlir/include/mlir/Conversion/AVX512ToLLVM/ConvertAVX512ToLLVM.h
index fdd203a6f6ef1..aff5c4ca2c70d 100644
--- a/mlir/include/mlir/Conversion/AVX512ToLLVM/ConvertAVX512ToLLVM.h
+++ b/mlir/include/mlir/Conversion/AVX512ToLLVM/ConvertAVX512ToLLVM.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef MLIR_EDGE_CONVERSION_AVX512TOLLVM_CONVERTAVX512TOLLVM_H_
-#define MLIR_EDGE_CONVERSION_AVX512TOLLVM_CONVERTAVX512TOLLVM_H_
+#ifndef MLIR_CONVERSION_AVX512TOLLVM_CONVERTAVX512TOLLVM_H_
+#define MLIR_CONVERSION_AVX512TOLLVM_CONVERTAVX512TOLLVM_H_
 
 #include <memory>
 
@@ -26,4 +26,4 @@ std::unique_ptr<OperationPass<ModuleOp>> createConvertAVX512ToLLVMPass();
 
 } // namespace mlir
 
-#endif // MLIR_EDGE_CONVERSION_AVX512TOLLVM_CONVERTAVX512TOLLVM_H_
+#endif // MLIR_CONVERSION_AVX512TOLLVM_CONVERTAVX512TOLLVM_H_
diff --git a/mlir/include/mlir/Conversion/AffineToStandard/AffineToStandard.h b/mlir/include/mlir/Conversion/AffineToStandard/AffineToStandard.h
index 4deffafe0ec60..4647cacdd9cd8 100644
--- a/mlir/include/mlir/Conversion/AffineToStandard/AffineToStandard.h
+++ b/mlir/include/mlir/Conversion/AffineToStandard/AffineToStandard.h
@@ -20,6 +20,7 @@ class Location;
 struct LogicalResult;
 class MLIRContext;
 class OpBuilder;
+class Pass;
 class RewritePattern;
 class Value;
 class ValueRange;
@@ -57,6 +58,12 @@ Value lowerAffineLowerBound(AffineForOp op, OpBuilder &builder);
 /// Emit code that computes the upper bound of the given affine loop using
 /// standard arithmetic operations.
 Value lowerAffineUpperBound(AffineForOp op, OpBuilder &builder);
+
+/// Lowers affine control flow operations (ForStmt, IfStmt and AffineApplyOp)
+/// to equivalent lower-level constructs (flow of basic blocks and arithmetic
+/// primitives).
+std::unique_ptr<Pass> createLowerAffinePass();
+
 } // namespace mlir
 
 #endif // MLIR_CONVERSION_AFFINETOSTANDARD_AFFINETOSTANDARD_H
diff --git a/mlir/include/mlir/Conversion/CMakeLists.txt b/mlir/include/mlir/Conversion/CMakeLists.txt
index d4ce2634f4505..ae0afc97dc639 100644
--- a/mlir/include/mlir/Conversion/CMakeLists.txt
+++ b/mlir/include/mlir/Conversion/CMakeLists.txt
@@ -1,6 +1,6 @@
 
 set(LLVM_TARGET_DEFINITIONS Passes.td)
-mlir_tablegen(Passes.h.inc -gen-pass-decls)
+mlir_tablegen(Passes.h.inc -gen-pass-decls -name Conversion)
 add_public_tablegen_target(MLIRConversionPassIncGen)
 
 add_mlir_doc(Passes -gen-pass-doc ConversionPasses ./)
diff --git a/mlir/include/mlir/Conversion/Passes.h b/mlir/include/mlir/Conversion/Passes.h
new file mode 100644
index 0000000000000..87f2c97e766df
--- /dev/null
+++ b/mlir/include/mlir/Conversion/Passes.h
@@ -0,0 +1,41 @@
+//===- Passes.h - Conversion Pass Construction and Registration -----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_CONVERSION_PASSES_H
+#define MLIR_CONVERSION_PASSES_H
+
+#include "mlir/Conversion/AVX512ToLLVM/ConvertAVX512ToLLVM.h"
+#include "mlir/Conversion/AffineToStandard/AffineToStandard.h"
+#include "mlir/Conversion/GPUCommon/GPUCommonPass.h"
+#include "mlir/Conversion/GPUToNVVM/GPUToNVVMPass.h"
+#include "mlir/Conversion/GPUToROCDL/GPUToROCDLPass.h"
+#include "mlir/Conversion/GPUToSPIRV/ConvertGPUToSPIRVPass.h"
+#include "mlir/Conversion/GPUToVulkan/ConvertGPUToVulkanPass.h"
+#include "mlir/Conversion/LinalgToLLVM/LinalgToLLVM.h"
+#include "mlir/Conversion/LinalgToSPIRV/LinalgToSPIRVPass.h"
+#include "mlir/Conversion/LinalgToStandard/LinalgToStandard.h"
+#include "mlir/Conversion/SCFToGPU/SCFToGPUPass.h"
+#include "mlir/Conversion/SCFToStandard/SCFToStandard.h"
+#include "mlir/Conversion/SPIRVToLLVM/ConvertSPIRVToLLVMPass.h"
+#include "mlir/Conversion/ShapeToSCF/ShapeToSCF.h"
+#include "mlir/Conversion/ShapeToStandard/ShapeToStandard.h"
+#include "mlir/Conversion/StandardToLLVM/ConvertStandardToLLVMPass.h"
+#include "mlir/Conversion/StandardToSPIRV/ConvertStandardToSPIRVPass.h"
+#include "mlir/Conversion/VectorToLLVM/ConvertVectorToLLVM.h"
+#include "mlir/Conversion/VectorToROCDL/VectorToROCDL.h"
+#include "mlir/Conversion/VectorToSCF/VectorToSCF.h"
+
+namespace mlir {
+
+/// Generate the code for registering conversion passes.
+#define GEN_PASS_REGISTRATION
+#include "mlir/Conversion/Passes.h.inc"
+
+} // namespace mlir
+
+#endif // MLIR_CONVERSION_PASSES_H
diff --git a/mlir/include/mlir/Dialect/Affine/CMakeLists.txt b/mlir/include/mlir/Dialect/Affine/CMakeLists.txt
index 404c926f60eda..96d951dedf4c6 100644
--- a/mlir/include/mlir/Dialect/Affine/CMakeLists.txt
+++ b/mlir/include/mlir/Dialect/Affine/CMakeLists.txt
@@ -1,7 +1,7 @@
 add_subdirectory(IR)
 
 set(LLVM_TARGET_DEFINITIONS Passes.td)
-mlir_tablegen(Passes.h.inc -gen-pass-decls)
+mlir_tablegen(Passes.h.inc -gen-pass-decls -name Affine)
 add_public_tablegen_target(MLIRAffinePassIncGen)
 
 add_mlir_doc(Passes -gen-pass-doc AffinePasses ./)
diff --git a/mlir/include/mlir/Dialect/Affine/Passes.h b/mlir/include/mlir/Dialect/Affine/Passes.h
index 18b3b790338d8..f2cef42a43561 100644
--- a/mlir/include/mlir/Dialect/Affine/Passes.h
+++ b/mlir/include/mlir/Dialect/Affine/Passes.h
@@ -14,17 +14,12 @@
 #ifndef MLIR_DIALECT_AFFINE_TRANSFORMS_PASSES_H
 #define MLIR_DIALECT_AFFINE_TRANSFORMS_PASSES_H
 
-#include "mlir/Support/LLVM.h"
-#include <functional>
+#include "mlir/Pass/Pass.h"
 #include <limits>
 
 namespace mlir {
 
 class AffineForOp;
-class FuncOp;
-class ModuleOp;
-class Pass;
-template <typename T> class OperationPass;
 
 /// Creates a simplification pass for affine structures (maps and sets). In
 /// addition, this pass also normalizes memrefs to have the trivial (identity)
@@ -79,6 +74,14 @@ createSuperVectorizePass(ArrayRef<int64_t> virtualVectorSize);
 /// Overload relying on pass options for initialization.
 std::unique_ptr<OperationPass<FuncOp>> createSuperVectorizePass();
 
+//===----------------------------------------------------------------------===//
+// Registration
+//===----------------------------------------------------------------------===//
+
+/// Generate the code for registering passes.
+#define GEN_PASS_REGISTRATION
+#include "mlir/Dialect/Affine/Passes.h.inc"
+
 } // end namespace mlir
 
 #endif // MLIR_DIALECT_AFFINE_RANSFORMS_PASSES_H
diff --git a/mlir/include/mlir/Dialect/GPU/CMakeLists.txt b/mlir/include/mlir/Dialect/GPU/CMakeLists.txt
index 6c80b4c8e3b98..68313c9788422 100644
--- a/mlir/include/mlir/Dialect/GPU/CMakeLists.txt
+++ b/mlir/include/mlir/Dialect/GPU/CMakeLists.txt
@@ -12,7 +12,7 @@ mlir_tablegen(ParallelLoopMapperEnums.cpp.inc -gen-enum-defs)
 add_public_tablegen_target(MLIRParallelLoopMapperEnumsGen)
 
 set(LLVM_TARGET_DEFINITIONS Passes.td)
-mlir_tablegen(Passes.h.inc -gen-pass-decls)
+mlir_tablegen(Passes.h.inc -gen-pass-decls -name GPU)
 add_public_tablegen_target(MLIRGPUPassIncGen)
 
 add_mlir_doc(Passes -gen-pass-doc GPUPasses ./)
diff --git a/mlir/include/mlir/Dialect/GPU/Passes.h b/mlir/include/mlir/Dialect/GPU/Passes.h
index bc349061f39f0..64b744b6b172c 100644
--- a/mlir/include/mlir/Dialect/GPU/Passes.h
+++ b/mlir/include/mlir/Dialect/GPU/Passes.h
@@ -13,21 +13,23 @@
 #ifndef MLIR_DIALECT_GPU_PASSES_H_
 #define MLIR_DIALECT_GPU_PASSES_H_
 
-#include <memory>
+#include "mlir/Pass/Pass.h"
 
 namespace mlir {
-
-class MLIRContext;
-class ModuleOp;
-template <typename T> class OperationPass;
-class OwningRewritePatternList;
-
 std::unique_ptr<OperationPass<ModuleOp>> createGpuKernelOutliningPass();
 
 /// Collect a set of patterns to rewrite ops within the GPU dialect.
 void populateGpuRewritePatterns(MLIRContext *context,
                                 OwningRewritePatternList &patterns);
 
+//===----------------------------------------------------------------------===//
+// Registration
+//===----------------------------------------------------------------------===//
+
+/// Generate the code for registering passes.
+#define GEN_PASS_REGISTRATION
+#include "mlir/Dialect/GPU/Passes.h.inc"
+
 } // namespace mlir
 
 #endif // MLIR_DIALECT_GPU_PASSES_H_
diff --git a/mlir/include/mlir/Dialect/LLVMIR/Transforms/CMakeLists.txt b/mlir/include/mlir/Dialect/LLVMIR/Transforms/CMakeLists.txt
index a2fd81c23e11b..a744b0706ffd4 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/Transforms/CMakeLists.txt
+++ b/mlir/include/mlir/Dialect/LLVMIR/Transforms/CMakeLists.txt
@@ -1,5 +1,5 @@
 set(LLVM_TARGET_DEFINITIONS Passes.td)
-mlir_tablegen(Passes.h.inc -gen-pass-decls)
+mlir_tablegen(Passes.h.inc -gen-pass-decls -name LLVM)
 add_public_tablegen_target(MLIRLLVMPassIncGen)
 
 add_mlir_doc(Passes -gen-pass-doc LLVMPasses ./)
diff --git a/mlir/include/mlir/Dialect/LLVMIR/Transforms/Passes.h b/mlir/include/mlir/Dialect/LLVMIR/Transforms/Passes.h
new file mode 100644
index 0000000000000..868a0e5635105
--- /dev/null
+++ b/mlir/include/mlir/Dialect/LLVMIR/Transforms/Passes.h
@@ -0,0 +1,26 @@
+//===- Passes.h - LLVM Pass Construction and Registration -----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_LLVMIR_TRANSFORMS_PASSES_H
+#define MLIR_DIALECT_LLVMIR_TRANSFORMS_PASSES_H
+
+#include "mlir/Dialect/LLVMIR/Transforms/LegalizeForExport.h"
+#include "mlir/Pass/Pass.h"
+
+namespace mlir {
+
+namespace LLVM {
+
+/// Generate the code for registering conversion passes.
+#define GEN_PASS_REGISTRATION
+#include "mlir/Dialect/LLVMIR/Transforms/Passes.h.inc"
+
+} // namespace LLVM
+} // namespace mlir
+
+#endif // MLIR_DIALECT_LLVMIR_TRANSFORMS_PASSES_H
diff --git a/mlir/include/mlir/Dialect/Linalg/CMakeLists.txt b/mlir/include/mlir/Dialect/Linalg/CMakeLists.txt
index 66ac74515dddd..d0edae3979e04 100644
--- a/mlir/include/mlir/Dialect/Linalg/CMakeLists.txt
+++ b/mlir/include/mlir/Dialect/Linalg/CMakeLists.txt
@@ -1,7 +1,7 @@
 add_subdirectory(IR)
 
 set(LLVM_TARGET_DEFINITIONS Passes.td)
-mlir_tablegen(Passes.h.inc -gen-pass-decls)
+mlir_tablegen(Passes.h.inc -gen-pass-decls -name Linalg)
 add_public_tablegen_target(MLIRLinalgPassIncGen)
 
 add_mlir_doc(Passes -gen-pass-doc LinalgPasses ./)
diff --git a/mlir/include/mlir/Dialect/Linalg/Passes.h b/mlir/include/mlir/Dialect/Linalg/Passes.h
index a5c09b3f75b7c..d74714cdaa56b 100644
--- a/mlir/include/mlir/Dialect/Linalg/Passes.h
+++ b/mlir/include/mlir/Dialect/Linalg/Passes.h
@@ -13,17 +13,9 @@
 #ifndef MLIR_DIALECT_LINALG_PASSES_H_
 #define MLIR_DIALECT_LINALG_PASSES_H_
 
-#include "mlir/Support/LLVM.h"
-#include "llvm/ADT/ArrayRef.h"
+#include "mlir/Pass/Pass.h"
 
 namespace mlir {
-class FuncOp;
-class MLIRContext;
-class ModuleOp;
-template <typename T> class OperationPass;
-class OwningRewritePatternList;
-class Pass;
-
 std::unique_ptr<OperationPass<FuncOp>> createLinalgFoldUnitExtentDimsPass();
 
 std::unique_ptr<OperationPass<FuncOp>> createLinalgFusionPass();
@@ -66,6 +58,14 @@ void populateLinalgTensorOpsFusionPatterns(MLIRContext *context,
 void populateLinalgFoldUnitExtentDimsPatterns(
     MLIRContext *context, OwningRewritePatternList &patterns);
 
+//===----------------------------------------------------------------------===//
+// Registration
+//===----------------------------------------------------------------------===//
+
+/// Generate the code for registering passes.
+#define GEN_PASS_REGISTRATION
+#include "mlir/Dialect/Linalg/Passes.h.inc"
+
 } // namespace mlir
 
 #endif // MLIR_DIALECT_LINALG_PASSES_H_
diff --git a/mlir/include/mlir/Dialect/Quant/CMakeLists.txt b/mlir/include/mlir/Dialect/Quant/CMakeLists.txt
index 1a48e4928b334..177d129a805af 100644
--- a/mlir/include/mlir/Dialect/Quant/CMakeLists.txt
+++ b/mlir/include/mlir/Dialect/Quant/CMakeLists.txt
@@ -2,7 +2,7 @@ add_mlir_dialect(QuantOps quant)
 add_mlir_doc(QuantOps -gen-dialect-doc QuantDialect Dialects/)
 
 set(LLVM_TARGET_DEFINITIONS Passes.td)
-mlir_tablegen(Passes.h.inc -gen-pass-decls)
+mlir_tablegen(Passes.h.inc -gen-pass-decls -name Quant)
 add_public_tablegen_target(MLIRQuantPassIncGen)
 
 add_mlir_doc(Passes -gen-pass-doc QuantPasses ./)
diff --git a/mlir/include/mlir/Dialect/Quant/Passes.h b/mlir/include/mlir/Dialect/Quant/Passes.h
index b938c9a86b722..090653eabe3ff 100644
--- a/mlir/include/mlir/Dialect/Quant/Passes.h
+++ b/mlir/include/mlir/Dialect/Quant/Passes.h
@@ -16,12 +16,9 @@
 #ifndef MLIR_DIALECT_QUANT_PASSES_H
 #define MLIR_DIALECT_QUANT_PASSES_H
 
-#include <memory>
+#include "mlir/Pass/Pass.h"
 
 namespace mlir {
-class FuncOp;
-template <typename T> class OperationPass;
-
 namespace quant {
 
 /// Creates a pass that converts quantization simulation operations (i.e.
@@ -35,6 +32,14 @@ std::unique_ptr<OperationPass<FuncOp>> createConvertSimulatedQuantPass();
 /// destructive and cannot be undone.
 std::unique_ptr<OperationPass<FuncOp>> createConvertConstPass();
 
+//===----------------------------------------------------------------------===//
+// Registration
+//===----------------------------------------------------------------------===//
+
+/// Generate the code for registering passes.
+#define GEN_PASS_REGISTRATION
+#include "mlir/Dialect/Quant/Passes.h.inc"
+
 } // namespace quant
 } // namespace mlir
 
diff --git a/mlir/include/mlir/Dialect/SCF/CMakeLists.txt b/mlir/include/mlir/Dialect/SCF/CMakeLists.txt
index 9467b97b384bd..546ada0224cf1 100644
--- a/mlir/include/mlir/Dialect/SCF/CMakeLists.txt
+++ b/mlir/include/mlir/Dialect/SCF/CMakeLists.txt
@@ -2,7 +2,7 @@ add_mlir_dialect(SCFOps scf Ops)
 add_mlir_doc(SCFOps -gen-dialect-doc SCFDialect Dialects/)
 
 set(LLVM_TARGET_DEFINITIONS Passes.td)
-mlir_tablegen(Passes.h.inc -gen-pass-decls)
+mlir_tablegen(Passes.h.inc -gen-pass-decls -name SCF)
 add_public_tablegen_target(MLIRSCFPassIncGen)
 add_dependencies(mlir-headers MLIRSCFPassIncGen)
 
diff --git a/mlir/include/mlir/Dialect/SCF/Passes.h b/mlir/include/mlir/Dialect/SCF/Passes.h
index df6037874f2b4..7edb2444e87c0 100644
--- a/mlir/include/mlir/Dialect/SCF/Passes.h
+++ b/mlir/include/mlir/Dialect/SCF/Passes.h
@@ -13,13 +13,10 @@
 #ifndef MLIR_DIALECT_SCF_PASSES_H_
 #define MLIR_DIALECT_SCF_PASSES_H_
 
-#include "llvm/ADT/ArrayRef.h"
-#include <memory>
+#include "mlir/Pass/Pass.h"
 
 namespace mlir {
 
-class Pass;
-
 /// Creates a pass that specializes for loop for unrolling and
 /// vectorization.
 std::unique_ptr<Pass> createForLoopSpecializationPass();
@@ -35,6 +32,14 @@ std::unique_ptr<Pass> createParallelLoopSpecializationPass();
 std::unique_ptr<Pass>
 createParallelLoopTilingPass(llvm::ArrayRef<int64_t> tileSize = {});
 
+//===----------------------------------------------------------------------===//
+// Registration
+//===----------------------------------------------------------------------===//
+
+/// Generate the code for registering passes.
+#define GEN_PASS_REGISTRATION
+#include "mlir/Dialect/SCF/Passes.h.inc"
+
 } // namespace mlir
 
 #endif // MLIR_DIALECT_SCF_PASSES_H_
diff --git a/mlir/include/mlir/Dialect/SPIRV/CMakeLists.txt b/mlir/include/mlir/Dialect/SPIRV/CMakeLists.txt
index 1e0901f07e91d..ff078ef9d9464 100644
--- a/mlir/include/mlir/Dialect/SPIRV/CMakeLists.txt
+++ b/mlir/include/mlir/Dialect/SPIRV/CMakeLists.txt
@@ -38,7 +38,7 @@ add_public_tablegen_target(MLIRSPIRVTargetAndABIIncGen)
 add_dependencies(mlir-headers MLIRSPIRVTargetAndABIIncGen)
 
 set(LLVM_TARGET_DEFINITIONS Passes.td)
-mlir_tablegen(Passes.h.inc -gen-pass-decls)
+mlir_tablegen(Passes.h.inc -gen-pass-decls -name SPIRV)
 add_public_tablegen_target(MLIRSPIRVPassIncGen)
 add_dependencies(mlir-headers MLIRSPIRVPassIncGen)
 
diff --git a/mlir/include/mlir/Dialect/SPIRV/Passes.h b/mlir/include/mlir/Dialect/SPIRV/Passes.h
index df516430be52c..dbd2c93a53a57 100644
--- a/mlir/include/mlir/Dialect/SPIRV/Passes.h
+++ b/mlir/include/mlir/Dialect/SPIRV/Passes.h
@@ -50,6 +50,14 @@ std::unique_ptr<OperationPass<spirv::ModuleOp>> createLowerABIAttributesPass();
 /// spv.CompositeInsert into spv.CompositeConstruct.
 std::unique_ptr<OperationPass<spirv::ModuleOp>> createRewriteInsertsPass();
 
+//===----------------------------------------------------------------------===//
+// Registration
+//===----------------------------------------------------------------------===//
+
+/// Generate the code for registering passes.
+#define GEN_PASS_REGISTRATION
+#include "mlir/Dialect/SPIRV/Passes.h.inc"
+
 } // namespace spirv
 } // namespace mlir
 
diff --git a/mlir/include/mlir/Dialect/Shape/Transforms/CMakeLists.txt b/mlir/include/mlir/Dialect/Shape/Transforms/CMakeLists.txt
index 629b8c0db2947..8bbe1cb3fbc63 100644
--- a/mlir/include/mlir/Dialect/Shape/Transforms/CMakeLists.txt
+++ b/mlir/include/mlir/Dialect/Shape/Transforms/CMakeLists.txt
@@ -1,5 +1,5 @@
 set(LLVM_TARGET_DEFINITIONS Passes.td)
-mlir_tablegen(Passes.h.inc -gen-pass-decls)
+mlir_tablegen(Passes.h.inc -gen-pass-decls -name Shape)
 add_public_tablegen_target(MLIRShapeTransformsIncGen)
 
 add_mlir_doc(Passes -gen-pass-doc ShapePasses ./)
diff --git a/mlir/include/mlir/Dialect/Shape/Transforms/Passes.h b/mlir/include/mlir/Dialect/Shape/Transforms/Passes.h
index e8d2167916d06..543ffc617a5cb 100644
--- a/mlir/include/mlir/Dialect/Shape/Transforms/Passes.h
+++ b/mlir/include/mlir/Dialect/Shape/Transforms/Passes.h
@@ -14,15 +14,9 @@
 #ifndef MLIR_DIALECT_SHAPE_TRANSFORMS_PASSES_H_
 #define MLIR_DIALECT_SHAPE_TRANSFORMS_PASSES_H_
 
-#include <memory>
+#include "mlir/Pass/Pass.h"
 
 namespace mlir {
-
-class FunctionPass;
-class MLIRContext;
-class OwningRewritePatternList;
-class Pass;
-
 /// Creates an instance of the ShapeToShapeLowering pass that legalizes Shape
 /// dialect to be convertible to Standard. For example, `shape.num_elements` get
 /// transformed to `shape.reduce`, which can be lowered to SCF and Standard.
@@ -42,6 +36,14 @@ void populateRemoveShapeConstraintsPatterns(OwningRewritePatternList &patterns,
                                             MLIRContext *ctx);
 std::unique_ptr<FunctionPass> createRemoveShapeConstraintsPass();
 
+//===----------------------------------------------------------------------===//
+// Registration
+//===----------------------------------------------------------------------===//
+
+/// Generate the code for registering passes.
+#define GEN_PASS_REGISTRATION
+#include "mlir/Dialect/Shape/Transforms/Passes.h.inc"
+
 } // end namespace mlir
 
 #endif // MLIR_DIALECT_SHAPE_TRANSFORMS_PASSES_H_
diff --git a/mlir/include/mlir/Dialect/StandardOps/Transforms/CMakeLists.txt b/mlir/include/mlir/Dialect/StandardOps/Transforms/CMakeLists.txt
index 413c6523a7564..f1cc5d81e0fe4 100644
--- a/mlir/include/mlir/Dialect/StandardOps/Transforms/CMakeLists.txt
+++ b/mlir/include/mlir/Dialect/StandardOps/Transforms/CMakeLists.txt
@@ -1,5 +1,5 @@
 set(LLVM_TARGET_DEFINITIONS Passes.td)
-mlir_tablegen(Passes.h.inc -gen-pass-decls)
+mlir_tablegen(Passes.h.inc -gen-pass-decls -name Standard)
 add_public_tablegen_target(MLIRStandardTransformsIncGen)
 
 add_mlir_doc(Passes -gen-pass-doc StandardPasses ./)
diff --git a/mlir/include/mlir/Dialect/StandardOps/Transforms/Passes.h b/mlir/include/mlir/Dialect/StandardOps/Transforms/Passes.h
index aadc41d2790dd..fba5f4b320430 100644
--- a/mlir/include/mlir/Dialect/StandardOps/Transforms/Passes.h
+++ b/mlir/include/mlir/Dialect/StandardOps/Transforms/Passes.h
@@ -15,12 +15,10 @@
 #ifndef MLIR_DIALECT_STANDARD_TRANSFORMS_PASSES_H_
 #define MLIR_DIALECT_STANDARD_TRANSFORMS_PASSES_H_
 
-#include <memory>
+#include "mlir/Pass/Pass.h"
 
 namespace mlir {
 
-class Pass;
-class MLIRContext;
 class OwningRewritePatternList;
 
 /// Creates an instance of the ExpandAtomic pass.
@@ -29,6 +27,14 @@ std::unique_ptr<Pass> createExpandAtomicPass();
 void populateExpandTanhPattern(OwningRewritePatternList &patterns,
                                MLIRContext *ctx);
 
+//===----------------------------------------------------------------------===//
+// Registration
+//===----------------------------------------------------------------------===//
+
+/// Generate the code for registering passes.
+#define GEN_PASS_REGISTRATION
+#include "mlir/Dialect/StandardOps/Transforms/Passes.h.inc"
+
 } // end namespace mlir
 
 #endif // MLIR_DIALECT_STANDARD_TRANSFORMS_PASSES_H_
diff --git a/mlir/include/mlir/InitAllPasses.h b/mlir/include/mlir/InitAllPasses.h
index a2810f3b270b2..7d0a7726ea6cc 100644
--- a/mlir/include/mlir/InitAllPasses.h
+++ b/mlir/include/mlir/InitAllPasses.h
@@ -14,38 +14,17 @@
 #ifndef MLIR_INITALLPASSES_H_
 #define MLIR_INITALLPASSES_H_
 
-#include "mlir/Conversion/AVX512ToLLVM/ConvertAVX512ToLLVM.h"
-#include "mlir/Conversion/GPUCommon/GPUCommonPass.h"
-#include "mlir/Conversion/GPUToNVVM/GPUToNVVMPass.h"
-#include "mlir/Conversion/GPUToROCDL/GPUToROCDLPass.h"
-#include "mlir/Conversion/GPUToSPIRV/ConvertGPUToSPIRVPass.h"
-#include "mlir/Conversion/GPUToVulkan/ConvertGPUToVulkanPass.h"
-#include "mlir/Conversion/LinalgToLLVM/LinalgToLLVM.h"
-#include "mlir/Conversion/LinalgToSPIRV/LinalgToSPIRVPass.h"
-#include "mlir/Conversion/LinalgToStandard/LinalgToStandard.h"
-#include "mlir/Conversion/SCFToGPU/SCFToGPUPass.h"
-#include "mlir/Conversion/SCFToStandard/SCFToStandard.h"
-#include "mlir/Conversion/SPIRVToLLVM/ConvertSPIRVToLLVMPass.h"
-#include "mlir/Conversion/ShapeToSCF/ShapeToSCF.h"
-#include "mlir/Conversion/ShapeToStandard/ShapeToStandard.h"
-#include "mlir/Conversion/StandardToLLVM/ConvertStandardToLLVMPass.h"
-#include "mlir/Conversion/StandardToSPIRV/ConvertStandardToSPIRVPass.h"
-#include "mlir/Conversion/VectorToLLVM/ConvertVectorToLLVM.h"
-#include "mlir/Conversion/VectorToROCDL/VectorToROCDL.h"
-#include "mlir/Conversion/VectorToSCF/VectorToSCF.h"
+#include "mlir/Conversion/Passes.h"
 #include "mlir/Dialect/Affine/Passes.h"
 #include "mlir/Dialect/GPU/Passes.h"
-#include "mlir/Dialect/LLVMIR/Transforms/LegalizeForExport.h"
+#include "mlir/Dialect/LLVMIR/Transforms/Passes.h"
 #include "mlir/Dialect/Linalg/Passes.h"
 #include "mlir/Dialect/Quant/Passes.h"
 #include "mlir/Dialect/SCF/Passes.h"
 #include "mlir/Dialect/SPIRV/Passes.h"
 #include "mlir/Dialect/Shape/Transforms/Passes.h"
 #include "mlir/Dialect/StandardOps/Transforms/Passes.h"
-#include "mlir/Transforms/LocationSnapshot.h"
 #include "mlir/Transforms/Passes.h"
-#include "mlir/Transforms/ViewOpGraph.h"
-#include "mlir/Transforms/ViewRegionGraph.h"
 
 #include <cstdlib>
 
@@ -59,49 +38,22 @@ namespace mlir {
 // individual passes.
 // The global registry is interesting to interact with the command-line tools.
 inline void registerAllPasses() {
-  // Init general passes
-#define GEN_PASS_REGISTRATION
-#include "mlir/Transforms/Passes.h.inc"
+  // General passes
+  registerTransformsPasses();
 
   // Conversion passes
-#define GEN_PASS_REGISTRATION
-#include "mlir/Conversion/Passes.h.inc"
-
-  // Affine
-#define GEN_PASS_REGISTRATION
-#include "mlir/Dialect/Affine/Passes.h.inc"
-
-  // GPU
-#define GEN_PASS_REGISTRATION
-#include "mlir/Dialect/GPU/Passes.h.inc"
-
-  // Linalg
-#define GEN_PASS_REGISTRATION
-#include "mlir/Dialect/Linalg/Passes.h.inc"
-
-  // LLVM
-#define GEN_PASS_REGISTRATION
-#include "mlir/Dialect/LLVMIR/Transforms/Passes.h.inc"
-
-  // Loop
-#define GEN_PASS_REGISTRATION
-#include "mlir/Dialect/SCF/Passes.h.inc"
-
-  // Quant
-#define GEN_PASS_REGISTRATION
-#include "mlir/Dialect/Quant/Passes.h.inc"
-
-  // SPIR-V
-#define GEN_PASS_REGISTRATION
-#include "mlir/Dialect/SPIRV/Passes.h.inc"
-
-  // Standard
-#define GEN_PASS_REGISTRATION
-#include "mlir/Dialect/StandardOps/Transforms/Passes.h.inc"
-
-  // Shape
-#define GEN_PASS_REGISTRATION
-#include "mlir/Dialect/Shape/Transforms/Passes.h.inc"
+  registerConversionPasses();
+
+  // Dialect passes
+  registerAffinePasses();
+  registerGPUPasses();
+  registerLinalgPasses();
+  LLVM::registerLLVMPasses();
+  quant::registerQuantPasses();
+  registerSCFPasses();
+  registerShapePasses();
+  spirv::registerSPIRVPasses();
+  registerStandardPasses();
 }
 
 } // namespace mlir
diff --git a/mlir/include/mlir/Transforms/CMakeLists.txt b/mlir/include/mlir/Transforms/CMakeLists.txt
index 706193188eddd..f1006e06757b2 100644
--- a/mlir/include/mlir/Transforms/CMakeLists.txt
+++ b/mlir/include/mlir/Transforms/CMakeLists.txt
@@ -1,6 +1,6 @@
 
 set(LLVM_TARGET_DEFINITIONS Passes.td)
-mlir_tablegen(Passes.h.inc -gen-pass-decls)
+mlir_tablegen(Passes.h.inc -gen-pass-decls -name Transforms)
 add_public_tablegen_target(MLIRTransformsPassIncGen)
 
 add_mlir_doc(Passes -gen-pass-doc GeneralPasses ./)
diff --git a/mlir/include/mlir/Transforms/Passes.h b/mlir/include/mlir/Transforms/Passes.h
index 1ffff1a25a6d5..ef8524ebd28a2 100644
--- a/mlir/include/mlir/Transforms/Passes.h
+++ b/mlir/include/mlir/Transforms/Passes.h
@@ -14,19 +14,19 @@
 #ifndef MLIR_TRANSFORMS_PASSES_H
 #define MLIR_TRANSFORMS_PASSES_H
 
-#include "mlir/Support/LLVM.h"
-#include <functional>
+#include "mlir/Pass/Pass.h"
+#include "mlir/Transforms/LocationSnapshot.h"
+#include "mlir/Transforms/ViewOpGraph.h"
+#include "mlir/Transforms/ViewRegionGraph.h"
 #include <limits>
 
 namespace mlir {
 
 class AffineForOp;
-class FuncOp;
-class ModuleOp;
-class Pass;
 
-template <typename T>
-class OperationPass;
+//===----------------------------------------------------------------------===//
+// Passes
+//===----------------------------------------------------------------------===//
 
 /// Creates an instance of the BufferPlacement pass.
 std::unique_ptr<Pass> createBufferPlacementPass();
@@ -95,6 +95,15 @@ std::unique_ptr<Pass> createSymbolDCEPass();
 /// Creates an interprocedural pass to normalize memrefs to have a trivial
 /// (identity) layout map.
 std::unique_ptr<OperationPass<ModuleOp>> createNormalizeMemRefsPass();
+
+//===----------------------------------------------------------------------===//
+// Registration
+//===----------------------------------------------------------------------===//
+
+/// Generate the code for registering passes.
+#define GEN_PASS_REGISTRATION
+#include "mlir/Transforms/Passes.h.inc"
+
 } // end namespace mlir
 
 #endif // MLIR_TRANSFORMS_PASSES_H
diff --git a/mlir/tools/mlir-tblgen/PassGen.cpp b/mlir/tools/mlir-tblgen/PassGen.cpp
index f8998f09a4368..c2dcdb8e4ac9b 100644
--- a/mlir/tools/mlir-tblgen/PassGen.cpp
+++ b/mlir/tools/mlir-tblgen/PassGen.cpp
@@ -14,6 +14,7 @@
 #include "mlir/TableGen/GenInfo.h"
 #include "mlir/TableGen/Pass.h"
 #include "llvm/ADT/StringExtras.h"
+#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/FormatVariadic.h"
 #include "llvm/TableGen/Error.h"
 #include "llvm/TableGen/Record.h"
@@ -21,6 +22,11 @@
 using namespace mlir;
 using namespace mlir::tblgen;
 
+static llvm::cl::OptionCategory passGenCat("Options for -gen-pass-decls");
+static llvm::cl::opt<std::string>
+    groupName("name", llvm::cl::desc("The name of this group of passes"),
+              llvm::cl::cat(passGenCat));
+
 //===----------------------------------------------------------------------===//
 // GEN: Pass base class generation
 //===----------------------------------------------------------------------===//
@@ -109,36 +115,49 @@ static void emitPassDecls(ArrayRef<Pass> passes, raw_ostream &os) {
 // GEN: Pass registration generation
 //===----------------------------------------------------------------------===//
 
+/// The code snippet used to generate the start of a pass base class.
+///
+/// {0}: The def name of the pass record.
+/// {1}: The argument of the pass.
+/// {2): The summary of the pass.
+/// {3}: The code for constructing the pass.
+const char *const passRegistrationCode = R"(
+//===----------------------------------------------------------------------===//
+// {0} Registration
+//===----------------------------------------------------------------------===//
+
+inline void register{0}Pass() {{
+  ::mlir::registerPass("{1}", "{2}", []() -> std::unique_ptr<::mlir::Pass> {{
+    return {3};
+  });
+}
+)";
+
+/// {0}: The name of the pass group.
+const char *const passGroupRegistrationCode = R"(
+//===----------------------------------------------------------------------===//
+// {0} Registration
+//===----------------------------------------------------------------------===//
+
+inline void register{0}Passes() {{
+)";
+
 /// Emit the code for registering each of the given passes with the global
 /// PassRegistry.
 static void emitRegistration(ArrayRef<Pass> passes, raw_ostream &os) {
   os << "#ifdef GEN_PASS_REGISTRATION\n";
   for (const Pass &pass : passes) {
-    os << llvm::formatv("#define GEN_PASS_REGISTRATION_{0}\n",
-                        pass.getDef()->getName());
-  }
-  os << "#endif // GEN_PASS_REGISTRATION\n";
-
-  for (const Pass &pass : passes) {
-    os << llvm::formatv("#ifdef GEN_PASS_REGISTRATION_{0}\n",
-                        pass.getDef()->getName());
-    os << llvm::formatv("::mlir::registerPass(\"{0}\", \"{1}\", []() -> "
-                        "std::unique_ptr<::mlir::Pass> {{ return {2}; });\n",
+    os << llvm::formatv(passRegistrationCode, pass.getDef()->getName(),
                         pass.getArgument(), pass.getSummary(),
                         pass.getConstructor());
-    os << llvm::formatv("#endif // GEN_PASS_REGISTRATION_{0}\n",
-                        pass.getDef()->getName());
-    os << llvm::formatv("#undef GEN_PASS_REGISTRATION_{0}\n",
-                        pass.getDef()->getName());
   }
 
-  os << "#ifdef GEN_PASS_REGISTRATION\n";
-  for (const Pass &pass : passes) {
-    os << llvm::formatv("#undef GEN_PASS_REGISTRATION_{0}\n",
-                        pass.getDef()->getName());
-  }
-  os << "#endif // GEN_PASS_REGISTRATION\n";
+  os << llvm::formatv(passGroupRegistrationCode, groupName);
+  for (const Pass &pass : passes)
+    os << "  register" << pass.getDef()->getName() << "Pass();\n";
+  os << "}\n";
   os << "#undef GEN_PASS_REGISTRATION\n";
+  os << "#endif // GEN_PASS_REGISTRATION\n";
 }
 
 //===----------------------------------------------------------------------===//

From 86a78546b97950dfacd44ab77f17f4ce055d16e5 Mon Sep 17 00:00:00 2001
From: Jacques Pienaar <jpienaar@google.com>
Date: Fri, 31 Jul 2020 14:46:48 -0700
Subject: [PATCH 0785/1035] [mlir] Add shape.with_shape op

This is an operation that can returns a new ValueShape with a different shape. Useful for composing shape function calls and reusing existing shape transfer functions.

Just adding the op in this change.

Differential Revision: https://reviews.llvm.org/D84217
---
 .../mlir/Dialect/Shape/IR/ShapeBase.td        |  6 ++-
 .../include/mlir/Dialect/Shape/IR/ShapeOps.td | 43 +++++++++++++++++++
 mlir/test/Dialect/Shape/ops.mlir              | 15 ++++++-
 3 files changed, 62 insertions(+), 2 deletions(-)

diff --git a/mlir/include/mlir/Dialect/Shape/IR/ShapeBase.td b/mlir/include/mlir/Dialect/Shape/IR/ShapeBase.td
index 8f64e3c081e6b..3e0177bca50e9 100644
--- a/mlir/include/mlir/Dialect/Shape/IR/ShapeBase.td
+++ b/mlir/include/mlir/Dialect/Shape/IR/ShapeBase.td
@@ -100,7 +100,11 @@ def Shape_ValueShapeType : DialectType<ShapeDialect,
     corresponds to `Value` in the compiler) and a shape. Conceptually this is a
     tuple of a value (potentially unknown) and `shape.type`. The value and shape
     can either or both be unknown. If both the `value` and `shape` are known,
-    then the shape of `value` is conformant with `shape`.
+    then the shape of `value` is conformant with `shape`. That is, the shape of
+    the value conforms to the shape of the ValueShape, so that if we have
+    ValueShape `(value, shape)` then `join(shape_of(value), shape)` would be
+    error free and in particular it means that if both are statically known,
+    then they are equal.
   }];
 }
 
diff --git a/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td b/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td
index bc7b6048e28f4..ac077439be3cf 100644
--- a/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td
+++ b/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td
@@ -432,6 +432,49 @@ def Shape_SizeToIndexOp : Shape_Op<"size_to_index", [NoSideEffect]> {
   let hasCanonicalizer = 1;
 }
 
+def Shape_WithOp : Shape_Op<"with_shape", [NoSideEffect]> {
+  let summary = "Returns ValueShape with given shape";
+  let description = [{
+    Returns ValueShape with the shape updated to match the shape operand. That
+    is a new ValueShape tuple is created with value equal to `operand`'s
+    value and shape equal to `shape`. If the ValueShape and given `shape` are
+    non-conformant, then the returned ValueShape will represent an error of
+    this mismatch. Similarly if either inputs are in an error state, then an
+    error is popagated.
+
+    Usage:
+      %0 = shape.with_shape %1, %2 : tensor<...>, !shape.shape
+
+    This is used, for example, where one combines shape function calculations
+    and/or call one shape function from another. E.g.,
+
+    ```mlir
+    func @shape_foobah(%a: !shape.value_shape,
+                       %b: !shape.value_shape,
+                       %c: !shape.value_shape) -> !shape.shape {
+      %0 = call @shape_foo(%a, %b) :
+        (!shape.value_shape, !shape.value_shape) -> !shape.shape
+      %1 = shape.with_shape %b, %0 : !shape.value_shape, !shape.shape
+      %2 = call @shape_bah(%c, %1) :
+        (!shape.value_shape, !shape.value_shape) -> !shape.shape
+      return %2 : !shape.shape
+    }
+    ```
+
+    This op need not be a refinement of the shape. In non-error cases the input
+    ValueShape's value and shape are conformant and so too for the output, but
+    the result may be less specified than `operand`'s shape as `shape` is
+    merely used to construct the new ValueShape. If join behavior is desired
+    then a join op should be used.
+  }];
+
+  let arguments = (ins AnyTypeOf<[AnyShaped, Shape_ValueShapeType]>:$operand,
+                       Shape_ShapeType:$shape);
+  let results = (outs Shape_ValueShapeType:$result);
+
+  let assemblyFormat = "operands attr-dict `:` type($operand) `,` type($shape)";
+}
+
 def Shape_YieldOp : Shape_Op<"yield",
     [HasParent<"ReduceOp">,
      NoSideEffect,
diff --git a/mlir/test/Dialect/Shape/ops.mlir b/mlir/test/Dialect/Shape/ops.mlir
index 48b3805d0a3b7..172835a2c6d55 100644
--- a/mlir/test/Dialect/Shape/ops.mlir
+++ b/mlir/test/Dialect/Shape/ops.mlir
@@ -221,4 +221,17 @@ func @num_elements_shape(%arg : !shape.shape) -> !shape.size {
   return %result : !shape.size
 }
 
-
+// Testing nvoking shape function from another. shape_equal_shapes is merely
+// a trivial helper function to invoke elsewhere.
+func @shape_equal_shapes(%a : !shape.value_shape, %b : !shape.value_shape) -> !shape.shape {
+  %0 = shape.shape_of %a : !shape.value_shape -> !shape.shape
+  %1 = shape.shape_of %b : !shape.value_shape -> !shape.shape
+  %2 = "shape.join"(%0, %1) : (!shape.shape, !shape.shape) -> !shape.shape
+  return %2 : !shape.shape
+}
+func @shape_with_shape(%a : !shape.value_shape, %b : !shape.value_shape) -> !shape.shape {
+  %0 = shape.shape_of %a : !shape.value_shape -> !shape.shape
+  %1 = shape.with_shape %b, %0 : !shape.value_shape, !shape.shape
+  %2 = call @shape_equal_shapes(%a, %1) : (!shape.value_shape, !shape.value_shape) -> !shape.shape
+  return %2 : !shape.shape
+}

From 5110fd0343c2d06c8ae538741fbef13ece5e68de Mon Sep 17 00:00:00 2001
From: Adrian Prantl <aprantl@apple.com>
Date: Fri, 31 Jul 2020 13:30:59 -0700
Subject: [PATCH 0786/1035] Convert to early exit (NFC)

---
 lldb/source/Target/TargetList.cpp | 55 ++++++++++++++++---------------
 1 file changed, 28 insertions(+), 27 deletions(-)

diff --git a/lldb/source/Target/TargetList.cpp b/lldb/source/Target/TargetList.cpp
index 75a022d2c4636..7e243e5ed3380 100644
--- a/lldb/source/Target/TargetList.cpp
+++ b/lldb/source/Target/TargetList.cpp
@@ -394,37 +394,38 @@ Status TargetList::CreateTargetInternal(Debugger &debugger,
     target_sp.reset(new Target(debugger, arch, platform_sp, is_dummy_target));
   }
 
-  if (target_sp) {
-    // Set argv0 with what the user typed, unless the user specified a
-    // directory. If the user specified a directory, then it is probably a
-    // bundle that was resolved and we need to use the resolved bundle path
-    if (!user_exe_path.empty()) {
-      // Use exactly what the user typed as the first argument when we exec or
-      // posix_spawn
-      if (user_exe_path_is_bundle && resolved_bundle_exe_path[0]) {
-        target_sp->SetArg0(resolved_bundle_exe_path);
-      } else {
-        // Use resolved path
-        target_sp->SetArg0(file.GetPath().c_str());
-      }
-    }
-    if (file.GetDirectory()) {
-      FileSpec file_dir;
-      file_dir.GetDirectory() = file.GetDirectory();
-      target_sp->AppendExecutableSearchPaths(file_dir);
-    }
+  if (!target_sp)
+    return error;
 
-    // Don't put the dummy target in the target list, it's held separately.
-    if (!is_dummy_target) {
-      std::lock_guard<std::recursive_mutex> guard(m_target_list_mutex);
-      m_selected_target_idx = m_target_list.size();
-      m_target_list.push_back(target_sp);
-      // Now prime this from the dummy target:
-      target_sp->PrimeFromDummyTarget(debugger.GetDummyTarget());
+  // Set argv0 with what the user typed, unless the user specified a
+  // directory. If the user specified a directory, then it is probably a
+  // bundle that was resolved and we need to use the resolved bundle path
+  if (!user_exe_path.empty()) {
+    // Use exactly what the user typed as the first argument when we exec or
+    // posix_spawn
+    if (user_exe_path_is_bundle && resolved_bundle_exe_path[0]) {
+      target_sp->SetArg0(resolved_bundle_exe_path);
     } else {
-      m_dummy_target_sp = target_sp;
+      // Use resolved path
+      target_sp->SetArg0(file.GetPath().c_str());
     }
   }
+  if (file.GetDirectory()) {
+    FileSpec file_dir;
+    file_dir.GetDirectory() = file.GetDirectory();
+    target_sp->AppendExecutableSearchPaths(file_dir);
+  }
+
+  // Don't put the dummy target in the target list, it's held separately.
+  if (!is_dummy_target) {
+    std::lock_guard<std::recursive_mutex> guard(m_target_list_mutex);
+    m_selected_target_idx = m_target_list.size();
+    m_target_list.push_back(target_sp);
+    // Now prime this from the dummy target:
+    target_sp->PrimeFromDummyTarget(debugger.GetDummyTarget());
+  } else {
+    m_dummy_target_sp = target_sp;
+  }
 
   return error;
 }

From 731292e5f30074c282d5ea1ebb86bb7adbc9e90e Mon Sep 17 00:00:00 2001
From: Andrei Lebedev <alebedev@accesssoftek.com>
Date: Fri, 31 Jul 2020 15:39:12 -0700
Subject: [PATCH 0787/1035] Updated the -I option description.

---
 clang/docs/ClangCommandLineReference.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/clang/docs/ClangCommandLineReference.rst b/clang/docs/ClangCommandLineReference.rst
index 1613c8e453184..699a0be720368 100644
--- a/clang/docs/ClangCommandLineReference.rst
+++ b/clang/docs/ClangCommandLineReference.rst
@@ -1014,9 +1014,9 @@ Include path management
 
 Flags controlling how ``#include``\s are resolved to files.
 
-.. option:: -I<dir>, --include-directory <arg>, --include-directory=<arg>
+.. option:: -I<dir>, --include-directory <dir>, --include-directory=<dir>
 
-Add directory to include search path
+Add directory <dir> to the list of include files search paths. If there are multiple -I options, these directories are searched in the order they are given before the standard system directories are searched. If the same directory is in the SYSTEM include search paths, for example if also specified with -isystem, the -I option will be ignored
 
 .. option:: -I-, --include-barrier
 

From 77a02527dc392121b31221cef73c4f933083a58e Mon Sep 17 00:00:00 2001
From: Sanjay Patel <spatel@rotateright.com>
Date: Fri, 31 Jul 2020 16:35:41 -0400
Subject: [PATCH 0788/1035] [InstSimplify] add tests for abs intrinsic; NFC

---
 llvm/test/Transforms/InstSimplify/call.ll | 49 +++++++++++++++++++++++
 1 file changed, 49 insertions(+)

diff --git a/llvm/test/Transforms/InstSimplify/call.ll b/llvm/test/Transforms/InstSimplify/call.ll
index 6579bda52795b..e8c39dbda8f98 100644
--- a/llvm/test/Transforms/InstSimplify/call.ll
+++ b/llvm/test/Transforms/InstSimplify/call.ll
@@ -2,6 +2,55 @@
 ; RUN: opt < %s -instsimplify -S | FileCheck %s
 ; RUN: opt < %s -passes=instsimplify -S | FileCheck %s
 
+declare i32 @llvm.abs.i32(i32, i1)
+declare <3 x i82> @llvm.abs.v3i82(<3 x i82>, i1)
+
+; TODO: If the sign bit is known zero, the abs is not needed.
+
+define i32 @zext_abs(i31 %x) {
+; CHECK-LABEL: @zext_abs(
+; CHECK-NEXT:    [[ZEXT:%.*]] = zext i31 [[X:%.*]] to i32
+; CHECK-NEXT:    [[ABS:%.*]] = call i32 @llvm.abs.i32(i32 [[ZEXT]], i1 false)
+; CHECK-NEXT:    ret i32 [[ABS]]
+;
+  %zext = zext i31 %x to i32
+  %abs = call i32 @llvm.abs.i32(i32 %zext, i1 false)
+  ret i32 %abs
+}
+
+define <3 x i82> @lshr_abs(<3 x i82> %x) {
+; CHECK-LABEL: @lshr_abs(
+; CHECK-NEXT:    [[LSHR:%.*]] = lshr <3 x i82> [[X:%.*]], <i82 1, i82 1, i82 1>
+; CHECK-NEXT:    [[ABS:%.*]] = call <3 x i82> @llvm.abs.v3i82(<3 x i82> [[LSHR]], i1 true)
+; CHECK-NEXT:    ret <3 x i82> [[ABS]]
+;
+  %lshr = lshr <3 x i82> %x, <i82 1, i82 1, i82 1>
+  %abs = call <3 x i82> @llvm.abs.v3i82(<3 x i82> %lshr, i1 true)
+  ret <3 x i82> %abs
+}
+
+define i32 @and_abs(i32 %x) {
+; CHECK-LABEL: @and_abs(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 2147483644
+; CHECK-NEXT:    [[ABS:%.*]] = call i32 @llvm.abs.i32(i32 [[AND]], i1 true)
+; CHECK-NEXT:    ret i32 [[ABS]]
+;
+  %and = and i32 %x, 2147483644
+  %abs = call i32 @llvm.abs.i32(i32 %and, i1 true)
+  ret i32 %abs
+}
+
+define <3 x i82> @select_abs(<3 x i1> %cond) {
+; CHECK-LABEL: @select_abs(
+; CHECK-NEXT:    [[SEL:%.*]] = select <3 x i1> [[COND:%.*]], <3 x i82> zeroinitializer, <3 x i82> <i82 2147483647, i82 42, i82 1>
+; CHECK-NEXT:    [[ABS:%.*]] = call <3 x i82> @llvm.abs.v3i82(<3 x i82> [[SEL]], i1 false)
+; CHECK-NEXT:    ret <3 x i82> [[ABS]]
+;
+  %sel = select <3 x i1> %cond, <3 x i82> zeroinitializer, <3 x i82> <i82 2147483647, i82 42, i82 1>
+  %abs = call <3 x i82> @llvm.abs.v3i82(<3 x i82> %sel, i1 false)
+  ret <3 x i82> %abs
+}
+
 declare {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a, i8 %b)
 declare {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a, i8 %b)
 declare {i8, i1} @llvm.usub.with.overflow.i8(i8 %a, i8 %b)

From f2400f024d323bc9000a4c126f2008a8b58fb4a0 Mon Sep 17 00:00:00 2001
From: Shilei Tian <tianshilei1992@gmail.com>
Date: Fri, 31 Jul 2020 18:54:09 -0400
Subject: [PATCH 0789/1035] [OpenMP] Fixed the issue that target memory
 deallocation might be called when they're being used

This patch fixed the issue that target memory might be deallocated when
they're still being used or before they're used.

Reviewed By: ye-luo

Differential Revision: https://reviews.llvm.org/D84996
---
 openmp/libomptarget/src/omptarget.cpp | 60 ++++++++++++++++++++++-----
 1 file changed, 50 insertions(+), 10 deletions(-)

diff --git a/openmp/libomptarget/src/omptarget.cpp b/openmp/libomptarget/src/omptarget.cpp
index 6704a6fd6e6b0..f4d79d8064b94 100644
--- a/openmp/libomptarget/src/omptarget.cpp
+++ b/openmp/libomptarget/src/omptarget.cpp
@@ -421,11 +421,32 @@ int targetDataBegin(DeviceTy &Device, int32_t arg_num, void **args_base,
   return OFFLOAD_SUCCESS;
 }
 
+namespace {
+/// This structure contains information to deallocate a target pointer, aka.
+/// used to call the function \p DeviceTy::deallocTgtPtr.
+struct DeallocTgtPtrInfo {
+  /// Host pointer used to look up into the map table
+  void *HstPtrBegin;
+  /// Size of the data
+  int64_t DataSize;
+  /// Whether it is forced to be removed from the map table
+  bool ForceDelete;
+  /// Whether it has \p close modifier
+  bool HasCloseModifier;
+
+  DeallocTgtPtrInfo(void *HstPtr, int64_t Size, bool ForceDelete,
+                    bool HasCloseModifier)
+      : HstPtrBegin(HstPtr), DataSize(Size), ForceDelete(ForceDelete),
+        HasCloseModifier(HasCloseModifier) {}
+};
+} // namespace
+
 /// Internal function to undo the mapping and retrieve the data from the device.
 int targetDataEnd(DeviceTy &Device, int32_t ArgNum, void **ArgBases,
                   void **Args, int64_t *ArgSizes, int64_t *ArgTypes,
                   void **ArgMappers, __tgt_async_info *AsyncInfo) {
   int Ret;
+  std::vector<DeallocTgtPtrInfo> DeallocTgtPtrs;
   // process each input.
   for (int32_t I = ArgNum - 1; I >= 0; --I) {
     // Ignore private variables and arrays - there is no mapping for them.
@@ -574,15 +595,34 @@ int targetDataEnd(DeviceTy &Device, int32_t ArgNum, void **ArgBases,
       }
       Device.ShadowMtx.unlock();
 
-      // Deallocate map
-      if (DelEntry) {
-        Ret = Device.deallocTgtPtr(HstPtrBegin, DataSize, ForceDelete,
-                                   HasCloseModifier);
-        if (Ret != OFFLOAD_SUCCESS) {
-          DP("Deallocating data from device failed.\n");
-          return OFFLOAD_FAIL;
-        }
-      }
+      // Add pointer to the buffer for later deallocation
+      if (DelEntry)
+        DeallocTgtPtrs.emplace_back(HstPtrBegin, DataSize, ForceDelete,
+                                    HasCloseModifier);
+    }
+  }
+
+  // We need to synchronize before deallocating data.
+  // If AsyncInfo is nullptr, the previous data transfer (if has) will be
+  // synchronous, so we don't need to synchronize again. If AsyncInfo->Queue is
+  // nullptr, there is no data transfer happened because once there is,
+  // AsyncInfo->Queue will not be nullptr, so again, we don't need to
+  // synchronize.
+  if (AsyncInfo && AsyncInfo->Queue) {
+    Ret = Device.synchronize(AsyncInfo);
+    if (Ret != OFFLOAD_SUCCESS) {
+      DP("Failed to synchronize device.\n");
+      return OFFLOAD_FAIL;
+    }
+  }
+
+  // Deallocate target pointer
+  for (DeallocTgtPtrInfo &Info : DeallocTgtPtrs) {
+    Ret = Device.deallocTgtPtr(Info.HstPtrBegin, Info.DataSize,
+                               Info.ForceDelete, Info.HasCloseModifier);
+    if (Ret != OFFLOAD_SUCCESS) {
+      DP("Deallocating data from device failed.\n");
+      return OFFLOAD_FAIL;
     }
   }
 
@@ -1006,5 +1046,5 @@ int target(int64_t DeviceId, void *HostPtr, int32_t ArgNum, void **ArgBases,
     return OFFLOAD_FAIL;
   }
 
-  return Device.synchronize(&AsyncInfo);
+  return OFFLOAD_SUCCESS;
 }

From 86dea1f39bd127776b999e10dff212003068d30a Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@intel.com>
Date: Fri, 31 Jul 2020 15:46:12 -0700
Subject: [PATCH 0790/1035] [ValueTracking] Improve llvm.abs handling in
 computeKnownBits.

Add the optimizations we have in the SelectionDAG version.
Known non-negative copies all known bits. Any known one other than
the sign bit makes result non-negative.

Differential Revision: https://reviews.llvm.org/D85000
---
 llvm/lib/Analysis/ValueTracking.cpp           |  21 +++-
 .../Transforms/InstCombine/abs-intrinsic.ll   | 103 +++++++++++++++++-
 2 files changed, 117 insertions(+), 7 deletions(-)

diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index 4cd2d07bf4b0c..af55c96f3a531 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -1644,13 +1644,24 @@ static void computeKnownBitsFromOperator(const Operator *I,
       default: break;
       case Intrinsic::abs:
         computeKnownBits(I->getOperand(0), Known2, Depth + 1, Q);
-        // Otherwise, if this call is undefined for INT_MIN, the result is
-        // positive.
-        if (match(II->getArgOperand(1), m_One()))
-          Known.Zero.setSignBit();
+
+        // If the source's MSB is zero then we know the rest of the bits.
+        if (Known2.isNonNegative()) {
+          Known.Zero |= Known2.Zero;
+          Known.One |= Known2.One;
+          break;
+        }
+
         // Absolute value preserves trailing zero count.
         Known.Zero.setLowBits(Known2.Zero.countTrailingOnes());
-        // FIXME: Handle known negative/non-negative input?
+
+        // If this call is undefined for INT_MIN, the result is positive. We
+        // also know it can't be INT_MIN if there is a set bit that isn't the
+        // sign bit.
+        Known2.One.clearSignBit();
+        if (match(II->getArgOperand(1), m_One()) || Known2.One.getBoolValue())
+          Known.Zero.setSignBit();
+        // FIXME: Handle known negative input?
         // FIXME: Calculate the negated Known bits and combine them?
         break;
       case Intrinsic::bitreverse:
diff --git a/llvm/test/Transforms/InstCombine/abs-intrinsic.ll b/llvm/test/Transforms/InstCombine/abs-intrinsic.ll
index 8b965b3d1e440..ed845cc842677 100644
--- a/llvm/test/Transforms/InstCombine/abs-intrinsic.ll
+++ b/llvm/test/Transforms/InstCombine/abs-intrinsic.ll
@@ -2,6 +2,7 @@
 ; RUN: opt < %s -instcombine -S | FileCheck %s
 
 declare i32 @llvm.abs.i32(i32, i1)
+declare <4 x i32> @llvm.abs.v4i32(<4 x i32>, i1)
 
 define i1 @abs_nsw_must_be_positive(i32 %x) {
 ; CHECK-LABEL: @abs_nsw_must_be_positive(
@@ -12,6 +13,15 @@ define i1 @abs_nsw_must_be_positive(i32 %x) {
   ret i1 %c2
 }
 
+define <4 x i1> @abs_nsw_must_be_positive_vec(<4 x i32> %x) {
+; CHECK-LABEL: @abs_nsw_must_be_positive_vec(
+; CHECK-NEXT:    ret <4 x i1> <i1 true, i1 true, i1 true, i1 true>
+;
+  %abs = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %x, i1 true)
+  %c2 = icmp sge <4 x i32> %abs, zeroinitializer
+  ret <4 x i1> %c2
+}
+
 ; Negative test, no nsw provides no information about the sign bit of the result.
 define i1 @abs_nonsw(i32 %x) {
 ; CHECK-LABEL: @abs_nonsw(
@@ -24,6 +34,17 @@ define i1 @abs_nonsw(i32 %x) {
   ret i1 %c2
 }
 
+define <4 x i1> @abs_nonsw_vec(<4 x i32> %x) {
+; CHECK-LABEL: @abs_nonsw_vec(
+; CHECK-NEXT:    [[ABS:%.*]] = call <4 x i32> @llvm.abs.v4i32(<4 x i32> [[X:%.*]], i1 false)
+; CHECK-NEXT:    [[C2:%.*]] = icmp sgt <4 x i32> [[ABS]], <i32 -1, i32 -1, i32 -1, i32 -1>
+; CHECK-NEXT:    ret <4 x i1> [[C2]]
+;
+  %abs = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %x, i1 false)
+  %c2 = icmp sge <4 x i32> %abs, zeroinitializer
+  ret <4 x i1> %c2
+}
+
 ; abs preserves trailing zeros so the second and is unneeded
 define i32 @abs_trailing_zeros(i32 %x) {
 ; CHECK-LABEL: @abs_trailing_zeros(
@@ -37,6 +58,18 @@ define i32 @abs_trailing_zeros(i32 %x) {
   ret i32 %and2
 }
 
+define <4 x i32> @abs_trailing_zeros_vec(<4 x i32> %x) {
+; CHECK-LABEL: @abs_trailing_zeros_vec(
+; CHECK-NEXT:    [[AND:%.*]] = and <4 x i32> [[X:%.*]], <i32 -4, i32 -8, i32 -16, i32 -32>
+; CHECK-NEXT:    [[ABS:%.*]] = call <4 x i32> @llvm.abs.v4i32(<4 x i32> [[AND]], i1 false)
+; CHECK-NEXT:    ret <4 x i32> [[ABS]]
+;
+  %and = and <4 x i32> %x, <i32 -4, i32 -8, i32 -16, i32 -32>
+  %abs = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %and, i1 false)
+  %and2 = and <4 x i32> %abs, <i32 -2, i32 -2, i32 -2, i32 -2>
+  ret <4 x i32> %and2
+}
+
 ; negative test, can't remove the second and based on trailing zeroes.
 ; FIXME: Could remove the first and using demanded bits.
 define i32 @abs_trailing_zeros_negative(i32 %x) {
@@ -52,12 +85,25 @@ define i32 @abs_trailing_zeros_negative(i32 %x) {
   ret i32 %and2
 }
 
+define <4 x i32> @abs_trailing_zeros_negative_vec(<4 x i32> %x) {
+; CHECK-LABEL: @abs_trailing_zeros_negative_vec(
+; CHECK-NEXT:    [[AND:%.*]] = and <4 x i32> [[X:%.*]], <i32 -2, i32 -2, i32 -2, i32 -2>
+; CHECK-NEXT:    [[ABS:%.*]] = call <4 x i32> @llvm.abs.v4i32(<4 x i32> [[AND]], i1 false)
+; CHECK-NEXT:    [[AND2:%.*]] = and <4 x i32> [[ABS]], <i32 -4, i32 -4, i32 -4, i32 -4>
+; CHECK-NEXT:    ret <4 x i32> [[AND2]]
+;
+  %and = and <4 x i32> %x, <i32 -2, i32 -2, i32 -2, i32 -2>
+  %abs = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %and, i1 false)
+  %and2 = and <4 x i32> %abs, <i32 -4, i32 -4, i32 -4, i32 -4>
+  ret <4 x i32> %and2
+}
+
 ; Make sure we infer this add doesn't overflow. The input to the abs has 3
 ; sign bits, the abs reduces this to 2 sign bits.
 define i32 @abs_signbits(i30 %x) {
 ; CHECK-LABEL: @abs_signbits(
-; CHECK-NEXT:    [[AND:%.*]] = sext i30 [[X:%.*]] to i32
-; CHECK-NEXT:    [[ABS:%.*]] = call i32 @llvm.abs.i32(i32 [[AND]], i1 false)
+; CHECK-NEXT:    [[EXT:%.*]] = sext i30 [[X:%.*]] to i32
+; CHECK-NEXT:    [[ABS:%.*]] = call i32 @llvm.abs.i32(i32 [[EXT]], i1 false)
 ; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[ABS]], 1
 ; CHECK-NEXT:    ret i32 [[ADD]]
 ;
@@ -66,3 +112,56 @@ define i32 @abs_signbits(i30 %x) {
   %add = add i32 %abs, 1
   ret i32 %add
 }
+
+define <4 x i32> @abs_signbits_vec(<4 x i30> %x) {
+; CHECK-LABEL: @abs_signbits_vec(
+; CHECK-NEXT:    [[EXT:%.*]] = sext <4 x i30> [[X:%.*]] to <4 x i32>
+; CHECK-NEXT:    [[ABS:%.*]] = call <4 x i32> @llvm.abs.v4i32(<4 x i32> [[EXT]], i1 false)
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw <4 x i32> [[ABS]], <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    ret <4 x i32> [[ADD]]
+;
+  %ext = sext <4 x i30> %x to <4 x i32>
+  %abs = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %ext, i1 false)
+  %add = add <4 x i32> %abs, <i32 1, i32 1, i32 1, i32 1>
+  ret <4 x i32> %add
+}
+
+define i1 @abs_known_positive_input_compare(i31 %x) {
+; CHECK-LABEL: @abs_known_positive_input_compare(
+; CHECK-NEXT:    ret i1 true
+;
+  %zext = zext i31 %x to i32
+  %abs = call i32 @llvm.abs.i32(i32 %zext, i1 false)
+  %c2 = icmp sge i32 %abs, 0
+  ret i1 %c2
+}
+
+define <4 x i1> @abs_known_positive_input_compare_vec(<4 x i31> %x) {
+; CHECK-LABEL: @abs_known_positive_input_compare_vec(
+; CHECK-NEXT:    ret <4 x i1> <i1 true, i1 true, i1 true, i1 true>
+;
+  %zext = zext <4 x i31> %x to <4 x i32>
+  %abs = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %zext, i1 false)
+  %c2 = icmp sge <4 x i32> %abs, zeroinitializer
+  ret <4 x i1> %c2
+}
+
+define i1 @abs_known_not_int_min(i32 %x) {
+; CHECK-LABEL: @abs_known_not_int_min(
+; CHECK-NEXT:    ret i1 true
+;
+  %or = or i32 %x, 1
+  %abs = call i32 @llvm.abs.i32(i32 %or, i1 false)
+  %c2 = icmp sge i32 %abs, 0
+  ret i1 %c2
+}
+
+define <4 x i1> @abs_known_not_int_min_vec(<4 x i32> %x) {
+; CHECK-LABEL: @abs_known_not_int_min_vec(
+; CHECK-NEXT:    ret <4 x i1> <i1 true, i1 true, i1 true, i1 true>
+;
+  %or = or <4 x i32> %x, <i32 1, i32 1, i32 1, i32 1>
+  %abs = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %or, i1 false)
+  %c2 = icmp sge <4 x i32> %abs, zeroinitializer
+  ret <4 x i1> %c2
+}

From 47a4a27f47203055a4700b65533262409f83c491 Mon Sep 17 00:00:00 2001
From: Hsiangkai Wang <kai.wang@sifive.com>
Date: Thu, 21 May 2020 17:30:20 +0800
Subject: [PATCH 0791/1035] Upgrade MC to v0.9.

Differential revision: https://reviews.llvm.org/D80802
---
 .../Target/RISCV/AsmParser/RISCVAsmParser.cpp | 185 ++++----
 .../RISCV/MCTargetDesc/RISCVInstPrinter.cpp   |  22 +-
 llvm/lib/Target/RISCV/RISCVInstrFormats.td    |  40 +-
 llvm/lib/Target/RISCV/RISCVInstrFormatsV.td   |  93 ++--
 llvm/lib/Target/RISCV/RISCVInstrInfo.h        |  55 ++-
 llvm/lib/Target/RISCV/RISCVInstrInfoV.td      | 242 ++++++-----
 llvm/test/MC/RISCV/rvv/compare.s              |   6 +
 llvm/test/MC/RISCV/rvv/convert.s              | 192 ++++++---
 llvm/test/MC/RISCV/rvv/ext.s                  |  81 ++++
 llvm/test/MC/RISCV/rvv/fcompare.s             |   6 +
 llvm/test/MC/RISCV/rvv/fothers.s              |  40 +-
 llvm/test/MC/RISCV/rvv/freduction.s           |   6 +
 llvm/test/MC/RISCV/rvv/invalid.s              | 272 ++----------
 llvm/test/MC/RISCV/rvv/load.s                 | 400 ++++++++++--------
 llvm/test/MC/RISCV/rvv/mask.s                 |   4 +-
 llvm/test/MC/RISCV/rvv/reduction.s            |   6 +
 llvm/test/MC/RISCV/rvv/snippet.s              |  30 +-
 llvm/test/MC/RISCV/rvv/store.s                | 228 +++++++---
 llvm/test/MC/RISCV/rvv/vsetvl.s               |  68 ++-
 19 files changed, 1187 insertions(+), 789 deletions(-)
 create mode 100644 llvm/test/MC/RISCV/rvv/ext.s

diff --git a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
index 407f980bd35e8..c094163c2f9d7 100644
--- a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
+++ b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
@@ -288,11 +288,21 @@ struct RISCVOperand : public MCParsedAsmOperand {
     SEW_1024,
   };
 
-  enum class VLMUL { LMUL_1 = 0, LMUL_2, LMUL_4, LMUL_8 };
+  enum class VLMUL {
+    LMUL_1 = 0,
+    LMUL_2,
+    LMUL_4,
+    LMUL_8,
+    LMUL_F8 = 5,
+    LMUL_F4,
+    LMUL_F2
+  };
 
   struct VTypeOp {
     VSEW Sew;
     VLMUL Lmul;
+    bool TailAgnostic;
+    bool MaskedoffAgnostic;
     unsigned Encoding;
   };
 
@@ -763,7 +773,7 @@ struct RISCVOperand : public MCParsedAsmOperand {
     case VSEW::SEW_1024:
       return "e1024";
     }
-    return "";
+    llvm_unreachable("Unknown SEW.");
   }
 
   static StringRef getLMULStr(VLMUL Lmul) {
@@ -776,8 +786,14 @@ struct RISCVOperand : public MCParsedAsmOperand {
       return "m4";
     case VLMUL::LMUL_8:
       return "m8";
+    case VLMUL::LMUL_F2:
+      return "mf2";
+    case VLMUL::LMUL_F4:
+      return "mf4";
+    case VLMUL::LMUL_F8:
+      return "mf8";
     }
-    return "";
+    llvm_unreachable("Unknown LMUL.");
   }
 
   StringRef getVType(SmallString<32> &Buf) const {
@@ -852,15 +868,31 @@ struct RISCVOperand : public MCParsedAsmOperand {
     return Op;
   }
 
-  static std::unique_ptr<RISCVOperand> createVType(APInt Sew, APInt Lmul,
-                                                   SMLoc S, bool IsRV64) {
+  static std::unique_ptr<RISCVOperand>
+  createVType(APInt Sew, APInt Lmul, bool Fractional, bool TailAgnostic,
+              bool MaskedoffAgnostic, SMLoc S, bool IsRV64) {
     auto Op = std::make_unique<RISCVOperand>(KindTy::VType);
     Sew.ashrInPlace(3);
     unsigned SewLog2 = Sew.logBase2();
     unsigned LmulLog2 = Lmul.logBase2();
     Op->VType.Sew = static_cast<VSEW>(SewLog2);
-    Op->VType.Lmul = static_cast<VLMUL>(LmulLog2);
-    Op->VType.Encoding = (SewLog2 << 2) | LmulLog2;
+    if (Fractional) {
+      unsigned Flmul = 8 - LmulLog2;
+      Op->VType.Lmul = static_cast<VLMUL>(Flmul);
+      Op->VType.Encoding =
+          ((Flmul & 0x4) << 3) | ((SewLog2 & 0x7) << 2) | (Flmul & 0x3);
+    } else {
+      Op->VType.Lmul = static_cast<VLMUL>(LmulLog2);
+      Op->VType.Encoding = (SewLog2 << 2) | LmulLog2;
+    }
+    if (TailAgnostic) {
+      Op->VType.Encoding |= 0x40;
+    }
+    if (MaskedoffAgnostic) {
+      Op->VType.Encoding |= 0x80;
+    }
+    Op->VType.TailAgnostic = TailAgnostic;
+    Op->VType.MaskedoffAgnostic = MaskedoffAgnostic;
     Op->StartLoc = S;
     Op->IsRV64 = IsRV64;
     return Op;
@@ -1181,8 +1213,10 @@ bool RISCVAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
   }
   case Match_InvalidVTypeI: {
     SMLoc ErrorLoc = ((RISCVOperand &)*Operands[ErrorInfo]).getStartLoc();
-    return Error(ErrorLoc,
-                 "operand must be e[8|16|32|64|128|256|512|1024],m[1|2|4|8]");
+    return Error(
+        ErrorLoc,
+        "operand must be "
+        "e[8|16|32|64|128|256|512|1024],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]");
   }
   case Match_InvalidVMaskRegister: {
     SMLoc ErrorLoc = ((RISCVOperand &)*Operands[ErrorInfo]).getStartLoc();
@@ -1549,7 +1583,7 @@ OperandMatchResultTy RISCVAsmParser::parseVTypeI(OperandVector &Operands) {
   if (getLexer().getKind() != AsmToken::Identifier)
     return MatchOperand_NoMatch;
 
-  // Parse "e8,m1"
+  // Parse "e8,m1,t[a|u],m[a|u]"
   StringRef Name = getLexer().getTok().getIdentifier();
   if (!Name.consume_front("e"))
     return MatchOperand_NoMatch;
@@ -1559,13 +1593,6 @@ OperandMatchResultTy RISCVAsmParser::parseVTypeI(OperandVector &Operands) {
     return MatchOperand_NoMatch;
   getLexer().Lex();
 
-  if (getLexer().getKind() == AsmToken::EndOfStatement) {
-    Operands.push_back(
-        RISCVOperand::createVType(Sew, APInt(16, 1), S, isRV64()));
-
-    return MatchOperand_Success;
-  }
-
   if (!getLexer().is(AsmToken::Comma))
     return MatchOperand_NoMatch;
   getLexer().Lex();
@@ -1573,15 +1600,51 @@ OperandMatchResultTy RISCVAsmParser::parseVTypeI(OperandVector &Operands) {
   Name = getLexer().getTok().getIdentifier();
   if (!Name.consume_front("m"))
     return MatchOperand_NoMatch;
+  // "m" or "mf"
+  bool Fractional = false;
+  if (Name.consume_front("f")) {
+    Fractional = true;
+  }
   APInt Lmul(16, Name, 10);
   if (Lmul != 1 && Lmul != 2 && Lmul != 4 && Lmul != 8)
     return MatchOperand_NoMatch;
   getLexer().Lex();
 
+  if (!getLexer().is(AsmToken::Comma))
+    return MatchOperand_NoMatch;
+  getLexer().Lex();
+
+  Name = getLexer().getTok().getIdentifier();
+  // ta or tu
+  bool TailAgnostic;
+  if (Name.consume_front("ta"))
+    TailAgnostic = true;
+  else if (Name.consume_front("tu"))
+    TailAgnostic = false;
+  else
+    return MatchOperand_NoMatch;
+  getLexer().Lex();
+
+  if (!getLexer().is(AsmToken::Comma))
+    return MatchOperand_NoMatch;
+  getLexer().Lex();
+
+  Name = getLexer().getTok().getIdentifier();
+  // ma or mu
+  bool MaskedoffAgnostic;
+  if (Name.consume_front("ma"))
+    MaskedoffAgnostic = true;
+  else if (Name.consume_front("mu"))
+    MaskedoffAgnostic = false;
+  else
+    return MatchOperand_NoMatch;
+  getLexer().Lex();
+
   if (getLexer().getKind() != AsmToken::EndOfStatement)
     return MatchOperand_NoMatch;
 
-  Operands.push_back(RISCVOperand::createVType(Sew, Lmul, S, isRV64()));
+  Operands.push_back(RISCVOperand::createVType(
+      Sew, Lmul, Fractional, TailAgnostic, MaskedoffAgnostic, S, isRV64()));
 
   return MatchOperand_Success;
 }
@@ -2281,71 +2344,41 @@ bool RISCVAsmParser::validateInstruction(MCInst &Inst,
     return false;
 
   unsigned DestReg = Inst.getOperand(0).getReg();
+  unsigned CheckReg;
   // Operands[1] will be the first operand, DestReg.
   SMLoc Loc = Operands[1]->getStartLoc();
-  if ((TargetFlags == RISCV::WidenV) || (TargetFlags == RISCV::WidenW) ||
-      (TargetFlags == RISCV::SlideUp) || (TargetFlags == RISCV::Vrgather) ||
-      (TargetFlags == RISCV::Vcompress)) {
-    if (TargetFlags != RISCV::WidenW) {
-      unsigned Src2Reg = Inst.getOperand(1).getReg();
-      if (DestReg == Src2Reg)
-        return Error(Loc, "The destination vector register group cannot overlap"
-                          " the source vector register group.");
-      if (TargetFlags == RISCV::WidenV) {
-        // Assume DestReg LMUL is 2 at least for widening/narrowing operations.
-        if (DestReg + 1 == Src2Reg)
-          return Error(Loc,
-                       "The destination vector register group cannot overlap"
-                       " the source vector register group.");
-      }
-    }
-    if (Inst.getOperand(2).isReg()) {
-      unsigned Src1Reg = Inst.getOperand(2).getReg();
-      if (DestReg == Src1Reg)
-        return Error(Loc, "The destination vector register group cannot overlap"
-                          " the source vector register group.");
-      if (TargetFlags == RISCV::WidenV || TargetFlags == RISCV::WidenW) {
-        // Assume DestReg LMUL is 2 at least for widening/narrowing operations.
-        if (DestReg + 1 == Src1Reg)
-          return Error(Loc,
-                       "The destination vector register group cannot overlap"
-                       " the source vector register group.");
-      }
-    }
-    if (Inst.getNumOperands() == 4) {
-      unsigned MaskReg = Inst.getOperand(3).getReg();
-
-      if (DestReg == MaskReg)
-        return Error(Loc, "The destination vector register group cannot overlap"
-                          " the mask register.");
-    }
-  } else if (TargetFlags == RISCV::Narrow) {
-    unsigned Src2Reg = Inst.getOperand(1).getReg();
-    if (DestReg == Src2Reg)
+  if (TargetFlags & RISCV::VS2Constraint) {
+    CheckReg = Inst.getOperand(1).getReg();
+    if (DestReg == CheckReg)
       return Error(Loc, "The destination vector register group cannot overlap"
                         " the source vector register group.");
-    // Assume Src2Reg LMUL is 2 at least for widening/narrowing operations.
-    if (DestReg == Src2Reg + 1)
+  }
+  if ((TargetFlags & RISCV::VS1Constraint) && (Inst.getOperand(2).isReg())) {
+    CheckReg = Inst.getOperand(2).getReg();
+    if (DestReg == CheckReg)
       return Error(Loc, "The destination vector register group cannot overlap"
                         " the source vector register group.");
-  } else if (TargetFlags == RISCV::WidenCvt || TargetFlags == RISCV::Iota) {
-    unsigned Src2Reg = Inst.getOperand(1).getReg();
-    if (DestReg == Src2Reg)
+  }
+  if ((TargetFlags & RISCV::VMConstraint) && (DestReg == RISCV::V0)) {
+    // vadc, vsbc are special cases. These instructions have no mask register.
+    // The destination register could not be V0.
+    unsigned Opcode = Inst.getOpcode();
+    if (Opcode == RISCV::VADC_VVM || Opcode == RISCV::VADC_VXM ||
+        Opcode == RISCV::VADC_VIM || Opcode == RISCV::VSBC_VVM ||
+        Opcode == RISCV::VSBC_VXM)
+      return Error(Loc, "The destination vector register group cannot be V0.");
+
+    // Regardless masked or unmasked version, the number of operands is the
+    // same. For example, "viota.m v0, v2" is "viota.m v0, v2, NoRegister"
+    // actually. We need to check the last operand to ensure whether it is
+    // masked or not.
+    if ((TargetFlags & RISCV::OneInput) && (Inst.getNumOperands() == 3))
+      CheckReg = Inst.getOperand(2).getReg();
+    else if (Inst.getNumOperands() == 4)
+      CheckReg = Inst.getOperand(3).getReg();
+    if (DestReg == CheckReg)
       return Error(Loc, "The destination vector register group cannot overlap"
-                        " the source vector register group.");
-    if (TargetFlags == RISCV::WidenCvt) {
-      // Assume DestReg LMUL is 2 at least for widening/narrowing operations.
-      if (DestReg + 1 == Src2Reg)
-        return Error(Loc, "The destination vector register group cannot overlap"
-                          " the source vector register group.");
-    }
-    if (Inst.getNumOperands() == 3) {
-      unsigned MaskReg = Inst.getOperand(2).getReg();
-
-      if (DestReg == MaskReg)
-        return Error(Loc, "The destination vector register group cannot overlap"
-                          " the mask register.");
-    }
+                        " the mask register.");
   }
   return false;
 }
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp
index eae3e13dbe409..2c7038a0a8ace 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp
@@ -155,10 +155,28 @@ void RISCVInstPrinter::printVTypeI(const MCInst *MI, unsigned OpNo,
   unsigned Imm = MI->getOperand(OpNo).getImm();
   unsigned Sew = (Imm >> 2) & 0x7;
   unsigned Lmul = Imm & 0x3;
+  bool Fractional = (Imm >> 5) & 0x1;
 
-  Lmul = 0x1 << Lmul;
   Sew = 0x1 << (Sew + 3);
-  O << "e" << Sew << ",m" << Lmul;
+  O << "e" << Sew;
+  if (Fractional) {
+    Lmul = 4 - Lmul;
+    Lmul = 0x1 << Lmul;
+    O << ",mf" << Lmul;
+  } else {
+    Lmul = 0x1 << Lmul;
+    O << ",m" << Lmul;
+  }
+  bool TailAgnostic = Imm & 0x40;
+  bool MaskedoffAgnostic = Imm & 0x80;
+  if (TailAgnostic)
+    O << ",ta";
+  else
+    O << ",tu";
+  if (MaskedoffAgnostic)
+    O << ",ma";
+  else
+    O << ",mu";
 }
 
 void RISCVInstPrinter::printVMaskReg(const MCInst *MI, unsigned OpNo,
diff --git a/llvm/lib/Target/RISCV/RISCVInstrFormats.td b/llvm/lib/Target/RISCV/RISCVInstrFormats.td
index a47945a6a5154..21f0514146353 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrFormats.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrFormats.td
@@ -52,15 +52,37 @@ def InstFormatOther  : InstFormat<17>;
 class RISCVVConstraint<bits<4> val> {
   bits<4> Value = val;
 }
-def NoConstraint : RISCVVConstraint<0>;
-def WidenV       : RISCVVConstraint<1>;
-def WidenW       : RISCVVConstraint<2>;
-def WidenCvt     : RISCVVConstraint<3>;
-def Narrow       : RISCVVConstraint<4>;
-def Iota         : RISCVVConstraint<5>;
-def SlideUp      : RISCVVConstraint<6>;
-def Vrgather     : RISCVVConstraint<7>;
-def Vcompress    : RISCVVConstraint<8>;
+def NoConstraint  : RISCVVConstraint<0b0000>;
+def VS2Constraint : RISCVVConstraint<0b0001>;
+def VS1Constraint : RISCVVConstraint<0b0010>;
+def VMConstraint  : RISCVVConstraint<0b0100>;
+def OneInput      : RISCVVConstraint<0b1000>;
+
+def WidenV       : RISCVVConstraint<!or(VS2Constraint.Value,
+                                        VS1Constraint.Value,
+                                        VMConstraint.Value)>;
+def WidenW       : RISCVVConstraint<!or(VS1Constraint.Value,
+                                        VMConstraint.Value)>;
+def WidenCvt     : RISCVVConstraint<!or(VS2Constraint.Value,
+                                        VMConstraint.Value,
+                                        OneInput.Value)>;
+def Narrow       : RISCVVConstraint<!or(VS2Constraint.Value,
+                                        VMConstraint.Value)>;
+def NarrowCvt    : RISCVVConstraint<!or(VS2Constraint.Value,
+                                        VMConstraint.Value,
+                                        OneInput.Value)>;
+def Vmadc        : RISCVVConstraint<!or(VS2Constraint.Value,
+                                        VS1Constraint.Value)>;
+def Iota         : RISCVVConstraint<!or(VS2Constraint.Value,
+                                        VMConstraint.Value,
+                                        OneInput.Value)>;
+def SlideUp      : RISCVVConstraint<!or(VS2Constraint.Value,
+                                        VMConstraint.Value)>;
+def Vrgather     : RISCVVConstraint<!or(VS2Constraint.Value,
+                                        VS1Constraint.Value,
+                                        VMConstraint.Value)>;
+def Vcompress    : RISCVVConstraint<!or(VS2Constraint.Value,
+                                        VS1Constraint.Value)>;
 
 // The following opcode names match those given in Table 19.1 in the
 // RISC-V User-level ISA specification ("RISC-V base opcode map").
diff --git a/llvm/lib/Target/RISCV/RISCVInstrFormatsV.td b/llvm/lib/Target/RISCV/RISCVInstrFormatsV.td
index e5f154966ba61..8ca010d033c39 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrFormatsV.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrFormatsV.td
@@ -21,20 +21,17 @@ def OPIVX : RISCVVFormat<0b100>;
 def OPFVF : RISCVVFormat<0b101>;
 def OPMVX : RISCVVFormat<0b110>;
 
-class RISCVMOP<bits<3> val> {
-  bits<3> Value = val;
+class RISCVMOP<bits<2> val> {
+  bits<2> Value = val;
 }
-def MOPLDUnitStrideU : RISCVMOP<0b000>;
-def MOPLDStridedU    : RISCVMOP<0b010>;
-def MOPLDIndexedU    : RISCVMOP<0b011>;
-def MOPLDUnitStrideS : RISCVMOP<0b100>;
-def MOPLDStridedS    : RISCVMOP<0b110>;
-def MOPLDIndexedS    : RISCVMOP<0b111>;
-
-def MOPSTUnitStride  : RISCVMOP<0b000>;
-def MOPSTStrided     : RISCVMOP<0b010>;
-def MOPSTIndexedOrder: RISCVMOP<0b011>;
-def MOPSTIndexedUnOrd: RISCVMOP<0b111>;
+def MOPLDUnitStride : RISCVMOP<0b00>;
+def MOPLDStrided    : RISCVMOP<0b10>;
+def MOPLDIndexed    : RISCVMOP<0b11>;
+
+def MOPSTUnitStride   : RISCVMOP<0b00>;
+def MOPSTIndexedUnord : RISCVMOP<0b01>;
+def MOPSTStrided      : RISCVMOP<0b10>;
+def MOPSTIndexedOrder : RISCVMOP<0b11>;
 
 class RISCVLSUMOP<bits<5> val> {
   bits<5> Value = val;
@@ -45,13 +42,17 @@ def LUMOPUnitStrideFF: RISCVLSUMOP<0b10000>;
 def SUMOPUnitStride  : RISCVLSUMOP<0b00000>;
 def SUMOPUnitStrideWholeReg : RISCVLSUMOP<0b01000>;
 
-class RISCVWidth<bits<3> val> {
-  bits<3> Value = val;
+class RISCVWidth<bits<4> val> {
+  bits<4> Value = val;
 }
-def LSWidthVByte : RISCVWidth<0b000>;
-def LSWidthVHalf : RISCVWidth<0b101>;
-def LSWidthVWord : RISCVWidth<0b110>;
-def LSWidthVSEW  : RISCVWidth<0b111>;
+def LSWidth8     : RISCVWidth<0b0000>;
+def LSWidth16    : RISCVWidth<0b0101>;
+def LSWidth32    : RISCVWidth<0b0110>;
+def LSWidth64    : RISCVWidth<0b0111>;
+def LSWidth128   : RISCVWidth<0b1000>;
+def LSWidth256   : RISCVWidth<0b1101>;
+def LSWidth512   : RISCVWidth<0b1110>;
+def LSWidth1024  : RISCVWidth<0b1111>;
 
 class RVInstSetVLi<dag outs, dag ins, string opcodestr, string argstr>
     : RVInst<outs, ins, opcodestr, argstr, [], InstFormatI> {
@@ -103,6 +104,7 @@ class RVInstVV<bits<6> funct6, RISCVVFormat opv, dag outs, dag ins,
   let Opcode = OPC_OP_V.Value;
 
   let Uses = [VTYPE, VL];
+  let RVVConstraint = VMConstraint;
 }
 
 class RVInstVX<bits<6> funct6, RISCVVFormat opv, dag outs, dag ins,
@@ -122,6 +124,7 @@ class RVInstVX<bits<6> funct6, RISCVVFormat opv, dag outs, dag ins,
   let Opcode = OPC_OP_V.Value;
 
   let Uses = [VTYPE, VL];
+  let RVVConstraint = VMConstraint;
 }
 
 class RVInstV2<bits<6> funct6, bits<5> vs2, RISCVVFormat opv, dag outs, dag ins,
@@ -140,6 +143,7 @@ class RVInstV2<bits<6> funct6, bits<5> vs2, RISCVVFormat opv, dag outs, dag ins,
   let Opcode = OPC_OP_V.Value;
 
   let Uses = [VTYPE, VL];
+  let RVVConstraint = VMConstraint;
 }
 
 class RVInstIVI<bits<6> funct6, dag outs, dag ins, string opcodestr,
@@ -159,6 +163,7 @@ class RVInstIVI<bits<6> funct6, dag outs, dag ins, string opcodestr,
   let Opcode = OPC_OP_V.Value;
 
   let Uses = [VTYPE, VL];
+  let RVVConstraint = VMConstraint;
 }
 
 class RVInstV<bits<6> funct6, bits<5> vs1, RISCVVFormat opv, dag outs,
@@ -177,10 +182,11 @@ class RVInstV<bits<6> funct6, bits<5> vs1, RISCVVFormat opv, dag outs,
   let Opcode = OPC_OP_V.Value;
 
   let Uses = [VTYPE, VL];
+  let RVVConstraint = VMConstraint;
 }
 
-class RVInstVLU<bits<3> nf, RISCVMOP mop, RISCVLSUMOP lumop,
-                RISCVWidth width, dag outs, dag ins, string opcodestr,
+class RVInstVLU<bits<3> nf, bit mew, RISCVLSUMOP lumop,
+                bits<3> width, dag outs, dag ins, string opcodestr,
                 string argstr>
     : RVInst<outs, ins, opcodestr, argstr, [], InstFormatR> {
   bits<5> rs1;
@@ -188,18 +194,20 @@ class RVInstVLU<bits<3> nf, RISCVMOP mop, RISCVLSUMOP lumop,
   bit vm;
 
   let Inst{31-29} = nf;
-  let Inst{28-26} = mop.Value;
+  let Inst{28} = mew;
+  let Inst{27-26} = MOPLDUnitStride.Value;
   let Inst{25} = vm;
   let Inst{24-20} = lumop.Value;
   let Inst{19-15} = rs1;
-  let Inst{14-12} = width.Value;
+  let Inst{14-12} = width;
   let Inst{11-7} = vd;
   let Opcode = OPC_LOAD_FP.Value;
 
   let Uses = [VTYPE, VL];
+  let RVVConstraint = VMConstraint;
 }
 
-class RVInstVLS<bits<3> nf, RISCVMOP mop, RISCVWidth width,
+class RVInstVLS<bits<3> nf, bit mew, bits<3> width,
                 dag outs, dag ins, string opcodestr, string argstr>
     : RVInst<outs, ins, opcodestr, argstr, [], InstFormatR> {
   bits<5> rs2;
@@ -208,18 +216,20 @@ class RVInstVLS<bits<3> nf, RISCVMOP mop, RISCVWidth width,
   bit vm;
 
   let Inst{31-29} = nf;
-  let Inst{28-26} = mop.Value;
+  let Inst{28} = mew;
+  let Inst{27-26} = MOPLDStrided.Value;
   let Inst{25} = vm;
   let Inst{24-20} = rs2;
   let Inst{19-15} = rs1;
-  let Inst{14-12} = width.Value;
+  let Inst{14-12} = width;
   let Inst{11-7} = vd;
   let Opcode = OPC_LOAD_FP.Value;
 
   let Uses = [VTYPE, VL];
+  let RVVConstraint = VMConstraint;
 }
 
-class RVInstVLX<bits<3> nf, RISCVMOP mop, RISCVWidth width,
+class RVInstVLX<bits<3> nf, bit mew, bits<3> width,
                 dag outs, dag ins, string opcodestr, string argstr>
     : RVInst<outs, ins, opcodestr, argstr, [], InstFormatR> {
   bits<5> vs2;
@@ -228,19 +238,21 @@ class RVInstVLX<bits<3> nf, RISCVMOP mop, RISCVWidth width,
   bit vm;
 
   let Inst{31-29} = nf;
-  let Inst{28-26} = mop.Value;
+  let Inst{28} = mew;
+  let Inst{27-26} = MOPLDIndexed.Value;
   let Inst{25} = vm;
   let Inst{24-20} = vs2;
   let Inst{19-15} = rs1;
-  let Inst{14-12} = width.Value;
+  let Inst{14-12} = width;
   let Inst{11-7} = vd;
   let Opcode = OPC_LOAD_FP.Value;
 
   let Uses = [VTYPE, VL];
+  let RVVConstraint = VMConstraint;
 }
 
-class RVInstVSU<bits<3> nf, RISCVMOP mop, RISCVLSUMOP sumop,
-                RISCVWidth width, dag outs, dag ins, string opcodestr,
+class RVInstVSU<bits<3> nf, bit mew, RISCVLSUMOP sumop,
+                bits<3> width, dag outs, dag ins, string opcodestr,
                 string argstr>
     : RVInst<outs, ins, opcodestr, argstr, [], InstFormatR> {
   bits<5> rs1;
@@ -248,18 +260,19 @@ class RVInstVSU<bits<3> nf, RISCVMOP mop, RISCVLSUMOP sumop,
   bit vm;
 
   let Inst{31-29} = nf;
-  let Inst{28-26} = mop.Value;
+  let Inst{28} = mew;
+  let Inst{27-26} = MOPSTUnitStride.Value;
   let Inst{25} = vm;
   let Inst{24-20} = sumop.Value;
   let Inst{19-15} = rs1;
-  let Inst{14-12} = width.Value;
+  let Inst{14-12} = width;
   let Inst{11-7} = vs3;
   let Opcode = OPC_STORE_FP.Value;
 
   let Uses = [VTYPE, VL];
 }
 
-class RVInstVSS<bits<3> nf, RISCVMOP mop, RISCVWidth width,
+class RVInstVSS<bits<3> nf, bit mew, bits<3> width,
                 dag outs, dag ins, string opcodestr, string argstr>
     : RVInst<outs, ins, opcodestr, argstr, [], InstFormatR> {
   bits<5> rs2;
@@ -268,18 +281,19 @@ class RVInstVSS<bits<3> nf, RISCVMOP mop, RISCVWidth width,
   bit vm;
 
   let Inst{31-29} = nf;
-  let Inst{28-26} = mop.Value;
+  let Inst{28} = mew;
+  let Inst{27-26} = MOPSTStrided.Value;
   let Inst{25} = vm;
   let Inst{24-20} = rs2;
   let Inst{19-15} = rs1;
-  let Inst{14-12} = width.Value;
+  let Inst{14-12} = width;
   let Inst{11-7} = vs3;
   let Opcode = OPC_STORE_FP.Value;
 
   let Uses = [VTYPE, VL];
 }
 
-class RVInstVSX<bits<3> nf, RISCVMOP mop, RISCVWidth width,
+class RVInstVSX<bits<3> nf, bit mew, RISCVMOP mop, bits<3> width,
                 dag outs, dag ins, string opcodestr, string argstr>
     : RVInst<outs, ins, opcodestr, argstr, [], InstFormatR> {
   bits<5> vs2;
@@ -288,11 +302,12 @@ class RVInstVSX<bits<3> nf, RISCVMOP mop, RISCVWidth width,
   bit vm;
 
   let Inst{31-29} = nf;
-  let Inst{28-26} = mop.Value;
+  let Inst{28} = mew;
+  let Inst{27-26} = mop.Value;
   let Inst{25} = vm;
   let Inst{24-20} = vs2;
   let Inst{19-15} = rs1;
-  let Inst{14-12} = width.Value;
+  let Inst{14-12} = width;
   let Inst{11-7} = vs3;
   let Opcode = OPC_STORE_FP.Value;
 
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.h b/llvm/lib/Target/RISCV/RISCVInstrInfo.h
index 21bc508cdc9ca..cd8b6d5fba59c 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.h
@@ -138,14 +138,53 @@ namespace RISCV {
 // Match with the definitions in RISCVInstrFormatsV.td
 enum RVVConstraintType {
   NoConstraint = 0,
-  WidenV = 1,
-  WidenW = 2,
-  WidenCvt = 3,
-  Narrow = 4,
-  Iota = 5,
-  SlideUp = 6,
-  Vrgather = 7,
-  Vcompress = 8,
+  VS2Constraint = 0b0001,
+  VS1Constraint = 0b0010,
+  VMConstraint = 0b0100,
+  OneInput = 0b1000,
+
+  // Illegal instructions:
+  //
+  // * The destination vector register group for a masked vector instruction
+  // cannot overlap the source mask register (v0), unless the destination vector
+  // register is being written with a mask value (e.g., comparisons) or the
+  // scalar result of a reduction.
+  //
+  // * Widening: The destination vector register group cannot overlap a source
+  // vector register group of a different EEW
+  //
+  // * Narrowing: The destination vector register group cannot overlap the
+  // first source vector register group
+  //
+  // * For vadc and vsbc, an illegal instruction exception is raised if the
+  // destination vector register is v0.
+  //
+  // * For vmadc and vmsbc, an illegal instruction exception is raised if the
+  // destination vector register overlaps a source vector register group.
+  //
+  // * viota: An illegal instruction exception is raised if the destination
+  // vector register group overlaps the source vector mask register. If the
+  // instruction is masked, an illegal instruction exception is issued if the
+  // destination vector register group overlaps v0.
+  //
+  // * v[f]slide[1]up: The destination vector register group for vslideup cannot
+  // overlap the source vector register group.
+  //
+  // * vrgather: The destination vector register group cannot overlap with the
+  // source vector register groups.
+  //
+  // * vcompress: The destination vector register group cannot overlap the
+  // source vector register group or the source mask register
+  WidenV = VS2Constraint | VS1Constraint | VMConstraint,
+  WidenW = VS1Constraint | VMConstraint,
+  WidenCvt = VS2Constraint | VMConstraint | OneInput,
+  Narrow = VS2Constraint | VMConstraint,
+  NarrowCvt = VS2Constraint | VMConstraint | OneInput,
+  Vmadc = VS2Constraint | VS1Constraint,
+  Iota = VS2Constraint | VMConstraint | OneInput,
+  SlideUp = VS2Constraint | VMConstraint,
+  Vrgather = VS2Constraint | VS1Constraint | VMConstraint,
+  Vcompress = VS2Constraint | VS1Constraint,
 
   ConstraintOffset = 5,
   ConstraintMask = 0b1111
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoV.td b/llvm/lib/Target/RISCV/RISCVInstrInfoV.td
index 1c7f53fecb8c9..ae335ddce882a 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoV.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoV.td
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 ///
 /// This file describes the RISC-V instructions from the standard 'V' Vector
-/// extension, version 0.8.
+/// extension, version 0.9.
 /// This version is still experimental as the 'V' extension hasn't been
 /// ratified yet.
 ///
@@ -96,27 +96,30 @@ def simm5_plus1 : Operand<XLenVT>, ImmLeaf<XLenVT,
 
 let hasSideEffects = 0, mayLoad = 1, mayStore = 0 in {
 // load vd, (rs1), vm
-class VUnitStrideLoad<RISCVMOP mop, RISCVLSUMOP lumop, RISCVWidth width,
-                        string opcodestr>
-    : RVInstVLU<0b000, mop, lumop, width, (outs VRegOp:$vd),
+class VUnitStrideLoad<RISCVLSUMOP lumop, RISCVWidth width,
+                      string opcodestr>
+    : RVInstVLU<0b000, width.Value{3}, lumop, width.Value{2-0},
+                (outs VRegOp:$vd),
                 (ins GPR:$rs1, VMaskOp:$vm), opcodestr, "$vd, (${rs1})$vm">;
 
 // load vd, (rs1), rs2, vm
-class VStridedLoad<RISCVMOP mop, RISCVWidth width, string opcodestr>
-    : RVInstVLS<0b000, mop, width, (outs VRegOp:$vd),
+class VStridedLoad<RISCVWidth width, string opcodestr>
+    : RVInstVLS<0b000, width.Value{3}, width.Value{2-0},
+                (outs VRegOp:$vd),
                 (ins GPR:$rs1, GPR:$rs2, VMaskOp:$vm), opcodestr,
                 "$vd, (${rs1}), $rs2$vm">;
 
 // load vd, (rs1), vs2, vm
-class VIndexedLoad<RISCVMOP mop, RISCVWidth width, string opcodestr>
-    : RVInstVLX<0b000, mop, width, (outs VRegOp:$vd),
+class VIndexedLoad<RISCVWidth width, string opcodestr>
+    : RVInstVLX<0b000, width.Value{3}, width.Value{2-0},
+                (outs VRegOp:$vd),
                 (ins GPR:$rs1, VRegOp:$vs2, VMaskOp:$vm), opcodestr,
                 "$vd, (${rs1}), $vs2$vm">;
 
 // vl<nf>r.v vd, (rs1)
 class VWholeLoad<bits<3> nf, string opcodestr>
-    : RVInstVLU<nf, MOPLDUnitStrideU, LUMOPUnitStrideWholeReg,
-                LSWidthVSEW, (outs VRegOp:$vd), (ins GPR:$rs1),
+    : RVInstVLU<nf, 0b0, LUMOPUnitStrideWholeReg,
+                0b000, (outs VRegOp:$vd), (ins GPR:$rs1),
                 opcodestr, "$vd, (${rs1})"> {
   let vm = 1;
   let Uses = [];
@@ -125,28 +128,28 @@ class VWholeLoad<bits<3> nf, string opcodestr>
 
 let hasSideEffects = 0, mayLoad = 0, mayStore = 1 in {
 // store vd, vs3, (rs1), vm
-class VUnitStrideStore<RISCVMOP mop, RISCVLSUMOP sumop, RISCVWidth width,
+class VUnitStrideStore<RISCVLSUMOP sumop, RISCVWidth width,
                          string opcodestr>
-    : RVInstVSU<0b000, mop, sumop, width, (outs),
-                (ins VRegOp:$vs3, GPR:$rs1, VMaskOp:$vm), opcodestr,
+    : RVInstVSU<0b000, width.Value{3}, sumop, width.Value{2-0},
+                (outs), (ins VRegOp:$vs3, GPR:$rs1, VMaskOp:$vm), opcodestr,
                 "$vs3, (${rs1})$vm">;
 
 // store vd, vs3, (rs1), rs2, vm
-class VStridedStore<RISCVMOP mop, RISCVWidth width, string opcodestr>
-    : RVInstVSS<0b000, mop, width, (outs),
+class VStridedStore<RISCVWidth width, string opcodestr>
+    : RVInstVSS<0b000, width.Value{3}, width.Value{2-0}, (outs),
                 (ins VRegOp:$vs3, GPR:$rs1, GPR:$rs2, VMaskOp:$vm),
                 opcodestr, "$vs3, (${rs1}), $rs2$vm">;
 
 // store vd, vs3, (rs1), vs2, vm
 class VIndexedStore<RISCVMOP mop, RISCVWidth width, string opcodestr>
-    : RVInstVSX<0b000, mop, width, (outs),
+    : RVInstVSX<0b000, width.Value{3}, mop, width.Value{2-0}, (outs),
                 (ins VRegOp:$vs3, GPR:$rs1, VRegOp:$vs2, VMaskOp:$vm),
                 opcodestr, "$vs3, (${rs1}), $vs2$vm">;
 
 // vs<nf>r.v vd, (rs1)
 class VWholeStore<bits<3> nf, string opcodestr>
-    : RVInstVSU<nf, MOPSTUnitStride, SUMOPUnitStrideWholeReg,
-                LSWidthVSEW, (outs), (ins VRegOp:$vs3, GPR:$rs1),
+    : RVInstVSU<nf, 0b0, SUMOPUnitStrideWholeReg,
+                0b000, (outs), (ins VRegOp:$vs3, GPR:$rs1),
                 opcodestr, "$vs3, (${rs1})"> {
   let vm = 1;
   let Uses = [];
@@ -372,68 +375,79 @@ def VSETVL : RVInstSetVL<(outs GPR:$rd), (ins GPR:$rs1, GPR:$rs2),
 } // hasSideEffects = 1, mayLoad = 0, mayStore = 0
 
 // Vector Unit-Stride Instructions
-def VLB_V : VUnitStrideLoad<MOPLDUnitStrideS, LUMOPUnitStride, LSWidthVByte, "vlb.v">;
-def VLH_V : VUnitStrideLoad<MOPLDUnitStrideS, LUMOPUnitStride, LSWidthVHalf, "vlh.v">;
-def VLW_V : VUnitStrideLoad<MOPLDUnitStrideS, LUMOPUnitStride, LSWidthVWord, "vlw.v">;
-
-def VLBU_V : VUnitStrideLoad<MOPLDUnitStrideU, LUMOPUnitStride, LSWidthVByte, "vlbu.v">;
-def VLHU_V : VUnitStrideLoad<MOPLDUnitStrideU, LUMOPUnitStride, LSWidthVHalf, "vlhu.v">;
-def VLWU_V : VUnitStrideLoad<MOPLDUnitStrideU, LUMOPUnitStride, LSWidthVWord, "vlwu.v">;
-
-def VLE_V : VUnitStrideLoad<MOPLDUnitStrideU, LUMOPUnitStride, LSWidthVSEW, "vle.v">;
-
-def VLBFF_V : VUnitStrideLoad<MOPLDUnitStrideS, LUMOPUnitStrideFF, LSWidthVByte, "vlbff.v">;
-def VLHFF_V : VUnitStrideLoad<MOPLDUnitStrideS, LUMOPUnitStrideFF, LSWidthVHalf, "vlhff.v">;
-def VLWFF_V : VUnitStrideLoad<MOPLDUnitStrideS, LUMOPUnitStrideFF, LSWidthVWord, "vlwff.v">;
-
-def VLBUFF_V : VUnitStrideLoad<MOPLDUnitStrideU, LUMOPUnitStrideFF, LSWidthVByte, "vlbuff.v">;
-def VLHUFF_V : VUnitStrideLoad<MOPLDUnitStrideU, LUMOPUnitStrideFF, LSWidthVHalf, "vlhuff.v">;
-def VLWUFF_V : VUnitStrideLoad<MOPLDUnitStrideU, LUMOPUnitStrideFF, LSWidthVWord, "vlwuff.v">;
-
-def VLEFF_V : VUnitStrideLoad<MOPLDUnitStrideU, LUMOPUnitStrideFF, LSWidthVSEW, "vleff.v">;
-
-def VSB_V : VUnitStrideStore<MOPSTUnitStride, SUMOPUnitStride, LSWidthVByte, "vsb.v">;
-def VSH_V : VUnitStrideStore<MOPSTUnitStride, SUMOPUnitStride, LSWidthVHalf, "vsh.v">;
-def VSW_V : VUnitStrideStore<MOPSTUnitStride, SUMOPUnitStride, LSWidthVWord, "vsw.v">;
-
-def VSE_V : VUnitStrideStore<MOPSTUnitStride, SUMOPUnitStride, LSWidthVSEW, "vse.v">;
+def VLE8_V : VUnitStrideLoad<LUMOPUnitStride, LSWidth8, "vle8.v">;
+def VLE16_V : VUnitStrideLoad<LUMOPUnitStride, LSWidth16, "vle16.v">;
+def VLE32_V : VUnitStrideLoad<LUMOPUnitStride, LSWidth32, "vle32.v">;
+def VLE64_V : VUnitStrideLoad<LUMOPUnitStride, LSWidth64, "vle64.v">;
+def VLE128_V : VUnitStrideLoad<LUMOPUnitStride, LSWidth128, "vle128.v">;
+def VLE256_V : VUnitStrideLoad<LUMOPUnitStride, LSWidth256, "vle256.v">;
+def VLE512_V : VUnitStrideLoad<LUMOPUnitStride, LSWidth512, "vle512.v">;
+def VLE1024_V : VUnitStrideLoad<LUMOPUnitStride, LSWidth1024, "vle1024.v">;
+
+def VLE8FF_V : VUnitStrideLoad<LUMOPUnitStrideFF, LSWidth8, "vle8ff.v">;
+def VLE16FF_V : VUnitStrideLoad<LUMOPUnitStrideFF, LSWidth16, "vle16ff.v">;
+def VLE32FF_V : VUnitStrideLoad<LUMOPUnitStrideFF, LSWidth32, "vle32ff.v">;
+def VLE64FF_V : VUnitStrideLoad<LUMOPUnitStrideFF, LSWidth64, "vle64ff.v">;
+def VLE128FF_V : VUnitStrideLoad<LUMOPUnitStrideFF, LSWidth128, "vle128ff.v">;
+def VLE256FF_V : VUnitStrideLoad<LUMOPUnitStrideFF, LSWidth256, "vle256ff.v">;
+def VLE512FF_V : VUnitStrideLoad<LUMOPUnitStrideFF, LSWidth512, "vle512ff.v">;
+def VLE1024FF_V : VUnitStrideLoad<LUMOPUnitStrideFF, LSWidth1024, "vle1024ff.v">;
+
+def VSE8_V : VUnitStrideStore<SUMOPUnitStride, LSWidth8, "vse8.v">;
+def VSE16_V : VUnitStrideStore<SUMOPUnitStride, LSWidth16, "vse16.v">;
+def VSE32_V : VUnitStrideStore<SUMOPUnitStride, LSWidth32, "vse32.v">;
+def VSE64_V : VUnitStrideStore<SUMOPUnitStride, LSWidth64, "vse64.v">;
+def VSE128_V : VUnitStrideStore<SUMOPUnitStride, LSWidth128, "vse128.v">;
+def VSE256_V : VUnitStrideStore<SUMOPUnitStride, LSWidth256, "vse256.v">;
+def VSE512_V : VUnitStrideStore<SUMOPUnitStride, LSWidth512, "vse512.v">;
+def VSE1024_V : VUnitStrideStore<SUMOPUnitStride, LSWidth1024, "vse1024.v">;
 
 // Vector Strided Instructions
-def VLSB_V : VStridedLoad<MOPLDStridedS, LSWidthVByte, "vlsb.v">;
-def VLSH_V : VStridedLoad<MOPLDStridedS, LSWidthVHalf, "vlsh.v">;
-def VLSW_V : VStridedLoad<MOPLDStridedS, LSWidthVWord, "vlsw.v">;
-
-def VLSBU_V : VStridedLoad<MOPLDStridedU, LSWidthVByte, "vlsbu.v">;
-def VLSHU_V : VStridedLoad<MOPLDStridedU, LSWidthVHalf, "vlshu.v">;
-def VLSWU_V : VStridedLoad<MOPLDStridedU, LSWidthVWord, "vlswu.v">;
-
-def VLSE_V : VStridedLoad<MOPLDStridedU, LSWidthVSEW, "vlse.v">;
-
-def VSSB_V : VStridedStore<MOPSTStrided, LSWidthVByte, "vssb.v">;
-def VSSH_V : VStridedStore<MOPSTStrided, LSWidthVHalf, "vssh.v">;
-def VSSW_V : VStridedStore<MOPSTStrided, LSWidthVWord, "vssw.v">;
-def VSSE_V : VStridedStore<MOPSTStrided, LSWidthVSEW, "vsse.v">;
+def VLSE8_V : VStridedLoad<LSWidth8, "vlse8.v">;
+def VLSE16_V : VStridedLoad<LSWidth16, "vlse16.v">;
+def VLSE32_V : VStridedLoad<LSWidth32, "vlse32.v">;
+def VLSE64_V : VStridedLoad<LSWidth64, "vlse64.v">;
+def VLSE128_V : VStridedLoad<LSWidth128, "vlse128.v">;
+def VLSE256_V : VStridedLoad<LSWidth256, "vlse256.v">;
+def VLSE512_V : VStridedLoad<LSWidth512, "vlse512.v">;
+def VLSE1024_V : VStridedLoad<LSWidth1024, "vlse1024.v">;
+
+def VSSE8_V : VStridedStore<LSWidth8, "vsse8.v">;
+def VSSE16_V : VStridedStore<LSWidth16, "vsse16.v">;
+def VSSE32_V : VStridedStore<LSWidth32, "vsse32.v">;
+def VSSE64_V : VStridedStore<LSWidth64, "vsse64.v">;
+def VSSE128_V : VStridedStore<LSWidth128, "vsse128.v">;
+def VSSE256_V : VStridedStore<LSWidth256, "vsse256.v">;
+def VSSE512_V : VStridedStore<LSWidth512, "vsse512.v">;
+def VSSE1024_V : VStridedStore<LSWidth1024, "vsse1024.v">;
 
 // Vector Indexed Instructions
-def VLXB_V : VIndexedLoad<MOPLDIndexedS, LSWidthVByte, "vlxb.v">;
-def VLXH_V : VIndexedLoad<MOPLDIndexedS, LSWidthVHalf, "vlxh.v">;
-def VLXW_V : VIndexedLoad<MOPLDIndexedS, LSWidthVWord, "vlxw.v">;
-
-def VLXBU_V : VIndexedLoad<MOPLDIndexedU, LSWidthVByte, "vlxbu.v">;
-def VLXHU_V : VIndexedLoad<MOPLDIndexedU, LSWidthVHalf, "vlxhu.v">;
-def VLXWU_V : VIndexedLoad<MOPLDIndexedU, LSWidthVWord, "vlxwu.v">;
-
-def VLXE_V : VIndexedLoad<MOPLDIndexedU, LSWidthVSEW, "vlxe.v">;
-
-def VSXB_V : VIndexedStore<MOPSTIndexedOrder, LSWidthVByte, "vsxb.v">;
-def VSXH_V : VIndexedStore<MOPSTIndexedOrder, LSWidthVHalf, "vsxh.v">;
-def VSXW_V : VIndexedStore<MOPSTIndexedOrder, LSWidthVWord, "vsxw.v">;
-def VSXE_V : VIndexedStore<MOPSTIndexedOrder, LSWidthVSEW, "vsxe.v">;
-
-def VSUXB_V : VIndexedStore<MOPSTIndexedUnOrd, LSWidthVByte, "vsuxb.v">;
-def VSUXH_V : VIndexedStore<MOPSTIndexedUnOrd, LSWidthVHalf, "vsuxh.v">;
-def VSUXW_V : VIndexedStore<MOPSTIndexedUnOrd, LSWidthVWord, "vsuxw.v">;
-def VSUXE_V : VIndexedStore<MOPSTIndexedUnOrd, LSWidthVSEW, "vsuxe.v">;
+def VLXEI8_V : VIndexedLoad<LSWidth8, "vlxei8.v">;
+def VLXEI16_V : VIndexedLoad<LSWidth16, "vlxei16.v">;
+def VLXEI32_V : VIndexedLoad<LSWidth32, "vlxei32.v">;
+def VLXEI64_V : VIndexedLoad<LSWidth64, "vlxei64.v">;
+def VLXEI128_V : VIndexedLoad<LSWidth128, "vlxei128.v">;
+def VLXEI256_V : VIndexedLoad<LSWidth256, "vlxei256.v">;
+def VLXEI512_V : VIndexedLoad<LSWidth512, "vlxei512.v">;
+def VLXEI1024_V : VIndexedLoad<LSWidth1024, "vlxei1024.v">;
+
+def VSXEI8_V : VIndexedStore<MOPSTIndexedOrder, LSWidth8, "vsxei8.v">;
+def VSXEI16_V : VIndexedStore<MOPSTIndexedOrder, LSWidth16, "vsxei16.v">;
+def VSXEI32_V : VIndexedStore<MOPSTIndexedOrder, LSWidth32, "vsxei32.v">;
+def VSXEI64_V : VIndexedStore<MOPSTIndexedOrder, LSWidth64, "vsxei64.v">;
+def VSXEI128_V : VIndexedStore<MOPSTIndexedOrder, LSWidth128, "vsxei128.v">;
+def VSXEI256_V : VIndexedStore<MOPSTIndexedOrder, LSWidth256, "vsxei256.v">;
+def VSXEI512_V : VIndexedStore<MOPSTIndexedOrder, LSWidth512, "vsxei512.v">;
+def VSXEI1024_V : VIndexedStore<MOPSTIndexedOrder, LSWidth1024, "vsxei1024.v">;
+
+def VSUXEI8_V : VIndexedStore<MOPSTIndexedUnord, LSWidth8, "vsuxei8.v">;
+def VSUXEI16_V : VIndexedStore<MOPSTIndexedUnord, LSWidth16, "vsuxei16.v">;
+def VSUXEI32_V : VIndexedStore<MOPSTIndexedUnord, LSWidth32, "vsuxei32.v">;
+def VSUXEI64_V : VIndexedStore<MOPSTIndexedUnord, LSWidth64, "vsuxei64.v">;
+def VSUXEI128_V : VIndexedStore<MOPSTIndexedUnord, LSWidth128, "vsuxei128.v">;
+def VSUXEI256_V : VIndexedStore<MOPSTIndexedUnord, LSWidth256, "vsuxei256.v">;
+def VSUXEI512_V : VIndexedStore<MOPSTIndexedUnord, LSWidth512, "vsuxei512.v">;
+def VSUXEI1024_V : VIndexedStore<MOPSTIndexedUnord, LSWidth1024, "vsuxei1024.v">;
 
 def VL1R_V : VWholeLoad<0, "vl1r.v">;
 def VS1R_V : VWholeStore<0, "vs1r.v">;
@@ -472,13 +486,25 @@ def : InstAlias<"vwcvt.x.x.v $vd, $vs$vm",
 def : InstAlias<"vwcvtu.x.x.v $vd, $vs$vm",
                 (VWADDU_VX VRegOp:$vd, VRegOp:$vs, X0, VMaskOp:$vm)>;
 
+// Vector Integer Extension
+defm VZEXT_VF8 : VALU_MV_VS2<"vzext.vf8", 0b010010, 0b00010>;
+defm VSEXT_VF8 : VALU_MV_VS2<"vsext.vf8", 0b010010, 0b00011>;
+defm VZEXT_VF4 : VALU_MV_VS2<"vzext.vf4", 0b010010, 0b00100>;
+defm VSEXT_VF4 : VALU_MV_VS2<"vsext.vf4", 0b010010, 0b00101>;
+defm VZEXT_VF2 : VALU_MV_VS2<"vzext.vf2", 0b010010, 0b00110>;
+defm VSEXT_VF2 : VALU_MV_VS2<"vsext.vf2", 0b010010, 0b00111>;
+
 // Vector Integer Add-with-Carry / Subtract-with-Borrow Instructions
 defm VADC_V : VALUm_IV_V_X_I<"vadc", 0b010000>;
+let Constraints = "@earlyclobber $vd", RVVConstraint = Vmadc in {
 defm VMADC_V : VALUm_IV_V_X_I<"vmadc", 0b010001>;
 defm VMADC_V : VALUNoVm_IV_V_X_I<"vmadc", 0b010001>;
+} // Constraints = "@earlyclobber $vd", RVVConstraint = Vmadc
 defm VSBC_V : VALUm_IV_V_X<"vsbc", 0b010010>;
+let Constraints = "@earlyclobber $vd", RVVConstraint = Vmadc in {
 defm VMSBC_V : VALUm_IV_V_X<"vmsbc", 0b010011>;
 defm VMSBC_V : VALUNoVm_IV_V_X<"vmsbc", 0b010011>;
+} // Constraints = "@earlyclobber $vd", RVVConstraint = Vmadc
 
 // Vector Bitwise Logical Instructions
 defm VAND_V : VALU_IV_V_X_I<"vand", 0b001001>;
@@ -504,6 +530,7 @@ defm VNSRA_W : VALU_IV_V_X_I<"vnsra", 0b101101, uimm5, "w">;
 } // Constraints = "@earlyclobber $vd", RVVConstraint = Narrow
 
 // Vector Integer Comparison Instructions
+let RVVConstraint = NoConstraint in {
 defm VMSEQ_V : VALU_IV_V_X_I<"vmseq", 0b011000>;
 defm VMSNE_V : VALU_IV_V_X_I<"vmsne", 0b011001>;
 defm VMSLTU_V : VALU_IV_V_X<"vmsltu", 0b011010>;
@@ -512,6 +539,7 @@ defm VMSLEU_V : VALU_IV_V_X_I<"vmsleu", 0b011100>;
 defm VMSLE_V : VALU_IV_V_X_I<"vmsle", 0b011101>;
 defm VMSGTU_V : VALU_IV_X_I<"vmsgtu", 0b011110>;
 defm VMSGT_V : VALU_IV_X_I<"vmsgt", 0b011111>;
+} // RVVConstraint = NoConstraint
 
 def : InstAlias<"vmsgtu.vv $vd, $va, $vb$vm",
                 (VMSLTU_VV VRegOp:$vd, VRegOp:$vb, VRegOp:$va, VMaskOp:$vm), 0>;
@@ -664,7 +692,7 @@ defm VFWNMSAC_V : VALUr_FV_V_F<"vfwnmsac", 0b111111>;
 } // Constraints = "@earlyclobber $vd", RVVConstraint = WidenV
 
 // Vector Floating-Point Square-Root Instruction
-defm VFSQRT_V : VALU_FV_VS2<"vfsqrt.v", 0b100011, 0b00000>;
+defm VFSQRT_V : VALU_FV_VS2<"vfsqrt.v", 0b010011, 0b00000>;
 
 // Vector Floating-Point MIN/MAX Instructions
 defm VFMIN_V : VALU_FV_V_F<"vfmin", 0b000100>;
@@ -676,12 +704,14 @@ defm VFSGNJN_V : VALU_FV_V_F<"vfsgnjn", 0b001001>;
 defm VFSGNJX_V : VALU_FV_V_F<"vfsgnjx", 0b001010>;
 
 // Vector Floating-Point Compare Instructions
+let RVVConstraint = NoConstraint in {
 defm VMFEQ_V : VALU_FV_V_F<"vmfeq", 0b011000>;
 defm VMFNE_V : VALU_FV_V_F<"vmfne", 0b011100>;
 defm VMFLT_V : VALU_FV_V_F<"vmflt", 0b011011>;
 defm VMFLE_V : VALU_FV_V_F<"vmfle", 0b011001>;
 defm VMFGT_V : VALU_FV_F<"vmfgt", 0b011101>;
 defm VMFGE_V : VALU_FV_F<"vmfge", 0b011111>;
+} // RVVConstraint = NoConstraint
 
 def : InstAlias<"vmfgt.vv $vd, $va, $vb$vm",
                 (VMFLT_VV VRegOp:$vd, VRegOp:$vb, VRegOp:$va, VMaskOp:$vm), 0>;
@@ -689,7 +719,7 @@ def : InstAlias<"vmfge.vv $vd, $va, $vb$vm",
                 (VMFLE_VV VRegOp:$vd, VRegOp:$vb, VRegOp:$va, VMaskOp:$vm), 0>;
 
 // Vector Floating-Point Classify Instruction
-defm VFCLASS_V : VALU_FV_VS2<"vfclass.v", 0b100011, 0b10000>;
+defm VFCLASS_V : VALU_FV_VS2<"vfclass.v", 0b010011, 0b10000>;
 
 let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in {
 // Vector Floating-Point Merge Instruction
@@ -708,31 +738,38 @@ def VFMV_V_F : RVInstVX<0b010111, OPFVF, (outs VRegOp:$vd),
 } // hasSideEffects = 0, mayLoad = 0, mayStore = 0
 
 // Single-Width Floating-Point/Integer Type-Convert Instructions
-defm VFCVT_XU_F_V : VALU_FV_VS2<"vfcvt.xu.f.v", 0b100010, 0b00000>;
-defm VFCVT_X_F_V : VALU_FV_VS2<"vfcvt.x.f.v", 0b100010, 0b00001>;
-defm VFCVT_F_XU_V : VALU_FV_VS2<"vfcvt.f.xu.v", 0b100010, 0b00010>;
-defm VFCVT_F_X_V : VALU_FV_VS2<"vfcvt.f.x.v", 0b100010, 0b00011>;
+defm VFCVT_XU_F_V : VALU_FV_VS2<"vfcvt.xu.f.v", 0b010010, 0b00000>;
+defm VFCVT_X_F_V : VALU_FV_VS2<"vfcvt.x.f.v", 0b010010, 0b00001>;
+defm VFCVT_RTZ_XU_F_V : VALU_FV_VS2<"vfcvt.rtz.xu.f.v", 0b010010, 0b00110>;
+defm VFCVT_RTZ_X_F_V : VALU_FV_VS2<"vfcvt.rtz.x.f.v", 0b010010, 0b00111>;
+defm VFCVT_F_XU_V : VALU_FV_VS2<"vfcvt.f.xu.v", 0b010010, 0b00010>;
+defm VFCVT_F_X_V : VALU_FV_VS2<"vfcvt.f.x.v", 0b010010, 0b00011>;
 
 // Widening Floating-Point/Integer Type-Convert Instructions
 let Constraints = "@earlyclobber $vd", RVVConstraint = WidenCvt in {
-defm VFWCVT_XU_F_V : VALU_FV_VS2<"vfwcvt.xu.f.v", 0b100010, 0b01000>;
-defm VFWCVT_X_F_V : VALU_FV_VS2<"vfwcvt.x.f.v", 0b100010, 0b01001>;
-defm VFWCVT_F_XU_V : VALU_FV_VS2<"vfwcvt.f.xu.v", 0b100010, 0b01010>;
-defm VFWCVT_F_X_V : VALU_FV_VS2<"vfwcvt.f.x.v", 0b100010, 0b01011>;
-defm VFWCVT_F_F_V : VALU_FV_VS2<"vfwcvt.f.f.v", 0b100010, 0b01100>;
+defm VFWCVT_XU_F_V : VALU_FV_VS2<"vfwcvt.xu.f.v", 0b010010, 0b01000>;
+defm VFWCVT_X_F_V : VALU_FV_VS2<"vfwcvt.x.f.v", 0b010010, 0b01001>;
+defm VFWCVT_RTZ_XU_F_V : VALU_FV_VS2<"vfwcvt.rtz.xu.f.v", 0b010010, 0b01110>;
+defm VFWCVT_RTZ_X_F_V : VALU_FV_VS2<"vfwcvt.rtz.x.f.v", 0b010010, 0b01111>;
+defm VFWCVT_F_XU_V : VALU_FV_VS2<"vfwcvt.f.xu.v", 0b010010, 0b01010>;
+defm VFWCVT_F_X_V : VALU_FV_VS2<"vfwcvt.f.x.v", 0b010010, 0b01011>;
+defm VFWCVT_F_F_V : VALU_FV_VS2<"vfwcvt.f.f.v", 0b010010, 0b01100>;
 } // Constraints = "@earlyclobber $vd", RVVConstraint = WidenCvt
 
 // Narrowing Floating-Point/Integer Type-Convert Instructions
-let Constraints = "@earlyclobber $vd", RVVConstraint = Narrow in {
-defm VFNCVT_XU_F_W : VALU_FV_VS2<"vfncvt.xu.f.w", 0b100010, 0b10000>;
-defm VFNCVT_X_F_W : VALU_FV_VS2<"vfncvt.x.f.w", 0b100010, 0b10001>;
-defm VFNCVT_F_XU_W : VALU_FV_VS2<"vfncvt.f.xu.w", 0b100010, 0b10010>;
-defm VFNCVT_F_X_W : VALU_FV_VS2<"vfncvt.f.x.w", 0b100010, 0b10011>;
-defm VFNCVT_F_F_W : VALU_FV_VS2<"vfncvt.f.f.w", 0b100010, 0b10100>;
-defm VFNCVT_ROD_F_F_W : VALU_FV_VS2<"vfncvt.rod.f.f.w", 0b100010, 0b10101>;
-} // Constraints = "@earlyclobber $vd", RVVConstraint = Narrow
+let Constraints = "@earlyclobber $vd", RVVConstraint = NarrowCvt in {
+defm VFNCVT_XU_F_W : VALU_FV_VS2<"vfncvt.xu.f.w", 0b010010, 0b10000>;
+defm VFNCVT_X_F_W : VALU_FV_VS2<"vfncvt.x.f.w", 0b010010, 0b10001>;
+defm VFNCVT_RTZ_XU_F_W : VALU_FV_VS2<"vfncvt.rtz.xu.f.w", 0b010010, 0b10110>;
+defm VFNCVT_RTZ_X_F_W : VALU_FV_VS2<"vfncvt.rtz.x.f.w", 0b010010, 0b10111>;
+defm VFNCVT_F_XU_W : VALU_FV_VS2<"vfncvt.f.xu.w", 0b010010, 0b10010>;
+defm VFNCVT_F_X_W : VALU_FV_VS2<"vfncvt.f.x.w", 0b010010, 0b10011>;
+defm VFNCVT_F_F_W : VALU_FV_VS2<"vfncvt.f.f.w", 0b010010, 0b10100>;
+defm VFNCVT_ROD_F_F_W : VALU_FV_VS2<"vfncvt.rod.f.f.w", 0b010010, 0b10101>;
+} // Constraints = "@earlyclobber $vd", RVVConstraint = NarrowCvt
 
 // Vector Single-Width Integer Reduction Instructions
+let RVVConstraint = NoConstraint in {
 defm VREDSUM : VALU_MV_V<"vredsum", 0b000000>;
 defm VREDMAXU : VALU_MV_V<"vredmaxu", 0b000110>;
 defm VREDMAX : VALU_MV_V<"vredmax", 0b000111>;
@@ -741,32 +778,35 @@ defm VREDMIN : VALU_MV_V<"vredmin", 0b000101>;
 defm VREDAND : VALU_MV_V<"vredand", 0b000001>;
 defm VREDOR : VALU_MV_V<"vredor", 0b000010>;
 defm VREDXOR : VALU_MV_V<"vredxor", 0b000011>;
+} // RVVConstraint = NoConstraint
 
 // Vector Widening Integer Reduction Instructions
-let Constraints = "@earlyclobber $vd" in {
+let Constraints = "@earlyclobber $vd", RVVConstraint = NoConstraint in {
 // Set earlyclobber for following instructions for second and mask operands.
 // This has the downside that the earlyclobber constraint is too coarse and
 // will impose unnecessary restrictions by not allowing the destination to
 // overlap with the first (wide) operand.
 defm VWREDSUMU : VALU_IV_V<"vwredsumu", 0b110000>;
 defm VWREDSUM : VALU_IV_V<"vwredsum", 0b110001>;
-} // Constraints = "@earlyclobber $vd"
+} // Constraints = "@earlyclobber $vd", RVVConstraint = NoConstraint
 
 // Vector Single-Width Floating-Point Reduction Instructions
+let RVVConstraint = NoConstraint in {
 defm VFREDOSUM : VALU_FV_V<"vfredosum", 0b000011>;
 defm VFREDSUM : VALU_FV_V<"vfredsum", 0b000001>;
 defm VFREDMAX : VALU_FV_V<"vfredmax", 0b000111>;
 defm VFREDMIN : VALU_FV_V<"vfredmin", 0b000101>;
+} // RVVConstraint = NoConstraint
 
 // Vector Widening Floating-Point Reduction Instructions
-let Constraints = "@earlyclobber $vd" in {
+let Constraints = "@earlyclobber $vd", RVVConstraint = NoConstraint in {
 // Set earlyclobber for following instructions for second and mask operands.
 // This has the downside that the earlyclobber constraint is too coarse and
 // will impose unnecessary restrictions by not allowing the destination to
 // overlap with the first (wide) operand.
 defm VFWREDOSUM : VALU_FV_V<"vfwredosum", 0b110011>;
 defm VFWREDSUM : VALU_FV_V<"vfwredsum", 0b110001>;
-} // Constraints = "@earlyclobber $vd"
+} // Constraints = "@earlyclobber $vd", RVVConstraint = NoConstraint
 
 // Vector Mask-Register Logical Instructions
 defm VMAND_M : VALU_MV_Mask<"vmand", 0b011001, "m">;
@@ -778,7 +818,7 @@ defm VMNOR_M : VALU_MV_Mask<"vmnor", 0b011110, "m">;
 defm VMORNOT_M : VALU_MV_Mask<"vmornot", 0b011100, "m">;
 defm VMXNOR_M : VALU_MV_Mask<"vmxnor", 0b011111, "m">;
 
-def : InstAlias<"vmcpy.m $vd, $vs",
+def : InstAlias<"vmmv.m $vd, $vs",
                 (VMAND_MM VRegOp:$vd, VRegOp:$vs, VRegOp:$vs)>;
 def : InstAlias<"vmclr.m $vd",
                 (VMXOR_MM VRegOp:$vd, VRegOp:$vd, VRegOp:$vd)>;
@@ -847,8 +887,10 @@ defm VSLIDEDOWN_V : VALU_IV_X_I<"vslidedown", 0b001111, uimm5>;
 
 let Constraints = "@earlyclobber $vd", RVVConstraint = SlideUp in {
 defm VSLIDE1UP_V : VALU_MV_X<"vslide1up", 0b001110>;
+defm VFSLIDE1UP_V : VALU_FV_F<"vfslide1up", 0b001110>;
 } // Constraints = "@earlyclobber $vd", RVVConstraint = SlideUp
 defm VSLIDE1DOWN_V : VALU_MV_X<"vslide1down", 0b001111>;
+defm VFSLIDE1DOWN_V : VALU_FV_F<"vfslide1down", 0b001111>;
 
 // Vector Register Gather Instruction
 let Constraints = "@earlyclobber $vd", RVVConstraint = Vrgather in {
diff --git a/llvm/test/MC/RISCV/rvv/compare.s b/llvm/test/MC/RISCV/rvv/compare.s
index 7bf452f771a7e..f93aeac1796a7 100644
--- a/llvm/test/MC/RISCV/rvv/compare.s
+++ b/llvm/test/MC/RISCV/rvv/compare.s
@@ -8,6 +8,12 @@
 # RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
 # RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
 
+vmslt.vv v0, v4, v20, v0.t
+# CHECK-INST: vmslt.vv v0, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x00,0x4a,0x6c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 00 4a 6c <unknown>
+
 vmseq.vv v8, v4, v20, v0.t
 # CHECK-INST: vmseq.vv v8, v4, v20, v0.t
 # CHECK-ENCODING: [0x57,0x04,0x4a,0x60]
diff --git a/llvm/test/MC/RISCV/rvv/convert.s b/llvm/test/MC/RISCV/rvv/convert.s
index e9f8860e97b59..bc818523e3e9c 100644
--- a/llvm/test/MC/RISCV/rvv/convert.s
+++ b/llvm/test/MC/RISCV/rvv/convert.s
@@ -10,180 +10,252 @@
 
 vfcvt.xu.f.v v8, v4, v0.t
 # CHECK-INST: vfcvt.xu.f.v v8, v4, v0.t
-# CHECK-ENCODING: [0x57,0x14,0x40,0x88]
+# CHECK-ENCODING: [0x57,0x14,0x40,0x48]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
-# CHECK-UNKNOWN: 57 14 40 88 <unknown>
+# CHECK-UNKNOWN: 57 14 40 48 <unknown>
 
 vfcvt.xu.f.v v8, v4
 # CHECK-INST: vfcvt.xu.f.v v8, v4
-# CHECK-ENCODING: [0x57,0x14,0x40,0x8a]
+# CHECK-ENCODING: [0x57,0x14,0x40,0x4a]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
-# CHECK-UNKNOWN: 57 14 40 8a <unknown>
+# CHECK-UNKNOWN: 57 14 40 4a <unknown>
 
 vfcvt.x.f.v v8, v4, v0.t
 # CHECK-INST: vfcvt.x.f.v v8, v4, v0.t
-# CHECK-ENCODING: [0x57,0x94,0x40,0x88]
+# CHECK-ENCODING: [0x57,0x94,0x40,0x48]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
-# CHECK-UNKNOWN: 57 94 40 88 <unknown>
+# CHECK-UNKNOWN: 57 94 40 48 <unknown>
 
 vfcvt.x.f.v v8, v4
 # CHECK-INST: vfcvt.x.f.v v8, v4
-# CHECK-ENCODING: [0x57,0x94,0x40,0x8a]
+# CHECK-ENCODING: [0x57,0x94,0x40,0x4a]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
-# CHECK-UNKNOWN: 57 94 40 8a <unknown>
+# CHECK-UNKNOWN: 57 94 40 4a <unknown>
 
 vfcvt.f.xu.v v8, v4, v0.t
 # CHECK-INST: vfcvt.f.xu.v v8, v4, v0.t
-# CHECK-ENCODING: [0x57,0x14,0x41,0x88]
+# CHECK-ENCODING: [0x57,0x14,0x41,0x48]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
-# CHECK-UNKNOWN: 57 14 41 88 <unknown>
+# CHECK-UNKNOWN: 57 14 41 48 <unknown>
 
 vfcvt.f.xu.v v8, v4
 # CHECK-INST: vfcvt.f.xu.v v8, v4
-# CHECK-ENCODING: [0x57,0x14,0x41,0x8a]
+# CHECK-ENCODING: [0x57,0x14,0x41,0x4a]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
-# CHECK-UNKNOWN: 57 14 41 8a <unknown>
+# CHECK-UNKNOWN: 57 14 41 4a <unknown>
 
 vfcvt.f.x.v v8, v4, v0.t
 # CHECK-INST: vfcvt.f.x.v v8, v4, v0.t
-# CHECK-ENCODING: [0x57,0x94,0x41,0x88]
+# CHECK-ENCODING: [0x57,0x94,0x41,0x48]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
-# CHECK-UNKNOWN: 57 94 41 88 <unknown>
+# CHECK-UNKNOWN: 57 94 41 48 <unknown>
 
 vfcvt.f.x.v v8, v4
 # CHECK-INST: vfcvt.f.x.v v8, v4
-# CHECK-ENCODING: [0x57,0x94,0x41,0x8a]
+# CHECK-ENCODING: [0x57,0x94,0x41,0x4a]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
-# CHECK-UNKNOWN: 57 94 41 8a <unknown>
+# CHECK-UNKNOWN: 57 94 41 4a <unknown>
+
+vfcvt.rtz.xu.f.v v8, v4, v0.t
+# CHECK-INST: vfcvt.rtz.xu.f.v v8, v4, v0.t
+# CHECK-ENCODING: [0x57,0x14,0x43,0x48]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 43 48 <unknown>
+
+vfcvt.rtz.xu.f.v v8, v4
+# CHECK-INST: vfcvt.rtz.xu.f.v v8, v4
+# CHECK-ENCODING: [0x57,0x14,0x43,0x4a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 43 4a <unknown>
+
+vfcvt.rtz.x.f.v v8, v4, v0.t
+# CHECK-INST: vfcvt.rtz.x.f.v v8, v4, v0.t
+# CHECK-ENCODING: [0x57,0x94,0x43,0x48]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 94 43 48 <unknown>
+
+vfcvt.rtz.x.f.v v8, v4
+# CHECK-INST: vfcvt.rtz.x.f.v v8, v4
+# CHECK-ENCODING: [0x57,0x94,0x43,0x4a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 94 43 4a <unknown>
 
 vfwcvt.xu.f.v v8, v4, v0.t
 # CHECK-INST: vfwcvt.xu.f.v v8, v4, v0.t
-# CHECK-ENCODING: [0x57,0x14,0x44,0x88]
+# CHECK-ENCODING: [0x57,0x14,0x44,0x48]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
-# CHECK-UNKNOWN: 57 14 44 88 <unknown>
+# CHECK-UNKNOWN: 57 14 44 48 <unknown>
 
 vfwcvt.xu.f.v v8, v4
 # CHECK-INST: vfwcvt.xu.f.v v8, v4
-# CHECK-ENCODING: [0x57,0x14,0x44,0x8a]
+# CHECK-ENCODING: [0x57,0x14,0x44,0x4a]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
-# CHECK-UNKNOWN: 57 14 44 8a <unknown>
+# CHECK-UNKNOWN: 57 14 44 4a <unknown>
 
 vfwcvt.x.f.v v8, v4, v0.t
 # CHECK-INST: vfwcvt.x.f.v v8, v4, v0.t
-# CHECK-ENCODING: [0x57,0x94,0x44,0x88]
+# CHECK-ENCODING: [0x57,0x94,0x44,0x48]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
-# CHECK-UNKNOWN: 57 94 44 88 <unknown>
+# CHECK-UNKNOWN: 57 94 44 48 <unknown>
 
 vfwcvt.x.f.v v8, v4
 # CHECK-INST: vfwcvt.x.f.v v8, v4
-# CHECK-ENCODING: [0x57,0x94,0x44,0x8a]
+# CHECK-ENCODING: [0x57,0x94,0x44,0x4a]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
-# CHECK-UNKNOWN: 57 94 44 8a <unknown>
+# CHECK-UNKNOWN: 57 94 44 4a <unknown>
 
 vfwcvt.f.xu.v v8, v4, v0.t
 # CHECK-INST: vfwcvt.f.xu.v v8, v4, v0.t
-# CHECK-ENCODING: [0x57,0x14,0x45,0x88]
+# CHECK-ENCODING: [0x57,0x14,0x45,0x48]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
-# CHECK-UNKNOWN: 57 14 45 88 <unknown>
+# CHECK-UNKNOWN: 57 14 45 48 <unknown>
 
 vfwcvt.f.xu.v v8, v4
 # CHECK-INST: vfwcvt.f.xu.v v8, v4
-# CHECK-ENCODING: [0x57,0x14,0x45,0x8a]
+# CHECK-ENCODING: [0x57,0x14,0x45,0x4a]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
-# CHECK-UNKNOWN: 57 14 45 8a <unknown>
+# CHECK-UNKNOWN: 57 14 45 4a <unknown>
 
 vfwcvt.f.x.v v8, v4, v0.t
 # CHECK-INST: vfwcvt.f.x.v v8, v4, v0.t
-# CHECK-ENCODING: [0x57,0x94,0x45,0x88]
+# CHECK-ENCODING: [0x57,0x94,0x45,0x48]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
-# CHECK-UNKNOWN: 57 94 45 88 <unknown>
+# CHECK-UNKNOWN: 57 94 45 48 <unknown>
 
 vfwcvt.f.x.v v8, v4
 # CHECK-INST: vfwcvt.f.x.v v8, v4
-# CHECK-ENCODING: [0x57,0x94,0x45,0x8a]
+# CHECK-ENCODING: [0x57,0x94,0x45,0x4a]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
-# CHECK-UNKNOWN: 57 94 45 8a <unknown>
+# CHECK-UNKNOWN: 57 94 45 4a <unknown>
 
 vfwcvt.f.f.v v8, v4, v0.t
 # CHECK-INST: vfwcvt.f.f.v v8, v4, v0.t
-# CHECK-ENCODING: [0x57,0x14,0x46,0x88]
+# CHECK-ENCODING: [0x57,0x14,0x46,0x48]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
-# CHECK-UNKNOWN: 57 14 46 88 <unknown>
+# CHECK-UNKNOWN: 57 14 46 48 <unknown>
 
 vfwcvt.f.f.v v8, v4
 # CHECK-INST: vfwcvt.f.f.v v8, v4
-# CHECK-ENCODING: [0x57,0x14,0x46,0x8a]
+# CHECK-ENCODING: [0x57,0x14,0x46,0x4a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 46 4a <unknown>
+
+vfwcvt.rtz.xu.f.v v8, v4, v0.t
+# CHECK-INST: vfwcvt.rtz.xu.f.v v8, v4, v0.t
+# CHECK-ENCODING: [0x57,0x14,0x47,0x48]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 47 48 <unknown>
+
+vfwcvt.rtz.xu.f.v v8, v4
+# CHECK-INST: vfwcvt.rtz.xu.f.v v8, v4
+# CHECK-ENCODING: [0x57,0x14,0x47,0x4a]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
-# CHECK-UNKNOWN: 57 14 46 8a <unknown>
+# CHECK-UNKNOWN: 57 14 47 4a <unknown>
+
+vfwcvt.rtz.x.f.v v8, v4, v0.t
+# CHECK-INST: vfwcvt.rtz.x.f.v v8, v4, v0.t
+# CHECK-ENCODING: [0x57,0x94,0x47,0x48]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 94 47 48 <unknown>
+
+vfwcvt.rtz.x.f.v v8, v4
+# CHECK-INST: vfwcvt.rtz.x.f.v v8, v4
+# CHECK-ENCODING: [0x57,0x94,0x47,0x4a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 94 47 4a <unknown>
 
 vfncvt.xu.f.w v8, v4, v0.t
 # CHECK-INST: vfncvt.xu.f.w v8, v4, v0.t
-# CHECK-ENCODING: [0x57,0x14,0x48,0x88]
+# CHECK-ENCODING: [0x57,0x14,0x48,0x48]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
-# CHECK-UNKNOWN: 57 14 48 88 <unknown>
+# CHECK-UNKNOWN: 57 14 48 48 <unknown>
 
 vfncvt.xu.f.w v8, v4
 # CHECK-INST: vfncvt.xu.f.w v8, v4
-# CHECK-ENCODING: [0x57,0x14,0x48,0x8a]
+# CHECK-ENCODING: [0x57,0x14,0x48,0x4a]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
-# CHECK-UNKNOWN: 57 14 48 8a <unknown>
+# CHECK-UNKNOWN: 57 14 48 4a <unknown>
 
 vfncvt.x.f.w v8, v4, v0.t
 # CHECK-INST: vfncvt.x.f.w v8, v4, v0.t
-# CHECK-ENCODING: [0x57,0x94,0x48,0x88]
+# CHECK-ENCODING: [0x57,0x94,0x48,0x48]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
-# CHECK-UNKNOWN: 57 94 48 88 <unknown>
+# CHECK-UNKNOWN: 57 94 48 48 <unknown>
 
 vfncvt.x.f.w v8, v4
 # CHECK-INST: vfncvt.x.f.w v8, v4
-# CHECK-ENCODING: [0x57,0x94,0x48,0x8a]
+# CHECK-ENCODING: [0x57,0x94,0x48,0x4a]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
-# CHECK-UNKNOWN: 57 94 48 8a <unknown>
+# CHECK-UNKNOWN: 57 94 48 4a <unknown>
 
 vfncvt.f.xu.w v8, v4, v0.t
 # CHECK-INST: vfncvt.f.xu.w v8, v4, v0.t
-# CHECK-ENCODING: [0x57,0x14,0x49,0x88]
+# CHECK-ENCODING: [0x57,0x14,0x49,0x48]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
-# CHECK-UNKNOWN: 57 14 49 88 <unknown>
+# CHECK-UNKNOWN: 57 14 49 48 <unknown>
 
 vfncvt.f.xu.w v8, v4
 # CHECK-INST: vfncvt.f.xu.w v8, v4
-# CHECK-ENCODING: [0x57,0x14,0x49,0x8a]
+# CHECK-ENCODING: [0x57,0x14,0x49,0x4a]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
-# CHECK-UNKNOWN: 57 14 49 8a <unknown>
+# CHECK-UNKNOWN: 57 14 49 4a <unknown>
 
 vfncvt.f.x.w v8, v4, v0.t
 # CHECK-INST: vfncvt.f.x.w v8, v4, v0.t
-# CHECK-ENCODING: [0x57,0x94,0x49,0x88]
+# CHECK-ENCODING: [0x57,0x94,0x49,0x48]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
-# CHECK-UNKNOWN: 57 94 49 88 <unknown>
+# CHECK-UNKNOWN: 57 94 49 48 <unknown>
 
 vfncvt.f.x.w v8, v4
 # CHECK-INST: vfncvt.f.x.w v8, v4
-# CHECK-ENCODING: [0x57,0x94,0x49,0x8a]
+# CHECK-ENCODING: [0x57,0x94,0x49,0x4a]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
-# CHECK-UNKNOWN: 57 94 49 8a <unknown>
+# CHECK-UNKNOWN: 57 94 49 4a <unknown>
 
 vfncvt.f.f.w v8, v4, v0.t
 # CHECK-INST: vfncvt.f.f.w v8, v4, v0.t
-# CHECK-ENCODING: [0x57,0x14,0x4a,0x88]
+# CHECK-ENCODING: [0x57,0x14,0x4a,0x48]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
-# CHECK-UNKNOWN: 57 14 4a 88 <unknown>
+# CHECK-UNKNOWN: 57 14 4a 48 <unknown>
 
 vfncvt.f.f.w v8, v4
 # CHECK-INST: vfncvt.f.f.w v8, v4
-# CHECK-ENCODING: [0x57,0x14,0x4a,0x8a]
+# CHECK-ENCODING: [0x57,0x14,0x4a,0x4a]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
-# CHECK-UNKNOWN: 57 14 4a 8a <unknown>
+# CHECK-UNKNOWN: 57 14 4a 4a <unknown>
 
 vfncvt.rod.f.f.w v8, v4, v0.t
 # CHECK-INST: vfncvt.rod.f.f.w v8, v4, v0.t
-# CHECK-ENCODING: [0x57,0x94,0x4a,0x88]
+# CHECK-ENCODING: [0x57,0x94,0x4a,0x48]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
-# CHECK-UNKNOWN: 57 94 4a 88 <unknown>
+# CHECK-UNKNOWN: 57 94 4a 48 <unknown>
 
 vfncvt.rod.f.f.w v8, v4
 # CHECK-INST: vfncvt.rod.f.f.w v8, v4
-# CHECK-ENCODING: [0x57,0x94,0x4a,0x8a]
+# CHECK-ENCODING: [0x57,0x94,0x4a,0x4a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 94 4a 4a <unknown>
+
+vfncvt.rtz.xu.f.w v8, v4, v0.t
+# CHECK-INST: vfncvt.rtz.xu.f.w v8, v4, v0.t
+# CHECK-ENCODING: [0x57,0x14,0x4b,0x48]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4b 48 <unknown>
+
+vfncvt.rtz.xu.f.w v8, v4
+# CHECK-INST: vfncvt.rtz.xu.f.w v8, v4
+# CHECK-ENCODING: [0x57,0x14,0x4b,0x4a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 14 4b 4a <unknown>
+
+vfncvt.rtz.x.f.w v8, v4, v0.t
+# CHECK-INST: vfncvt.rtz.x.f.w v8, v4, v0.t
+# CHECK-ENCODING: [0x57,0x94,0x4b,0x48]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 94 4b 48 <unknown>
+
+vfncvt.rtz.x.f.w v8, v4
+# CHECK-INST: vfncvt.rtz.x.f.w v8, v4
+# CHECK-ENCODING: [0x57,0x94,0x4b,0x4a]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
-# CHECK-UNKNOWN: 57 94 4a 8a <unknown>
+# CHECK-UNKNOWN: 57 94 4b 4a <unknown>
diff --git a/llvm/test/MC/RISCV/rvv/ext.s b/llvm/test/MC/RISCV/rvv/ext.s
new file mode 100644
index 0000000000000..45ac980b47b65
--- /dev/null
+++ b/llvm/test/MC/RISCV/rvv/ext.s
@@ -0,0 +1,81 @@
+# RUN: llvm-mc -triple=riscv64 -show-encoding --mattr=+experimental-v %s \
+# RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+# RUN: not llvm-mc -triple=riscv64 -show-encoding %s 2>&1 \
+# RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d --mattr=+experimental-v - \
+# RUN:        | FileCheck %s --check-prefix=CHECK-INST
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
+# RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+
+vzext.vf2 v8, v4, v0.t
+# CHECK-INST: vzext.vf2 v8, v4, v0.t
+# CHECK-ENCODING: [0x57,0x24,0x43,0x48]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 43 48 <unknown>
+
+vzext.vf2 v8, v4
+# CHECK-INST: vzext.vf2 v8, v4
+# CHECK-ENCODING: [0x57,0x24,0x43,0x4a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 43 4a <unknown>
+
+vsext.vf2 v8, v4, v0.t
+# CHECK-INST: vsext.vf2 v8, v4, v0.t
+# CHECK-ENCODING: [0x57,0xa4,0x43,0x48]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 a4 43 48 <unknown>
+
+vsext.vf2 v8, v4
+# CHECK-INST: vsext.vf2 v8, v4
+# CHECK-ENCODING: [0x57,0xa4,0x43,0x4a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 a4 43 4a <unknown>
+
+vzext.vf4 v8, v4, v0.t
+# CHECK-INST: vzext.vf4 v8, v4, v0.t
+# CHECK-ENCODING: [0x57,0x24,0x42,0x48]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 42 48 <unknown>
+
+vzext.vf4 v8, v4
+# CHECK-INST: vzext.vf4 v8, v4
+# CHECK-ENCODING: [0x57,0x24,0x42,0x4a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 42 4a <unknown>
+
+vsext.vf4 v8, v4, v0.t
+# CHECK-INST: vsext.vf4 v8, v4, v0.t
+# CHECK-ENCODING: [0x57,0xa4,0x42,0x48]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 a4 42 48 <unknown>
+
+vsext.vf4 v8, v4
+# CHECK-INST: vsext.vf4 v8, v4
+# CHECK-ENCODING: [0x57,0xa4,0x42,0x4a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 a4 42 4a <unknown>
+
+vzext.vf8 v8, v4, v0.t
+# CHECK-INST: vzext.vf8 v8, v4, v0.t
+# CHECK-ENCODING: [0x57,0x24,0x41,0x48]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 41 48 <unknown>
+
+vzext.vf8 v8, v4
+# CHECK-INST: vzext.vf8 v8, v4
+# CHECK-ENCODING: [0x57,0x24,0x41,0x4a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 24 41 4a <unknown>
+
+vsext.vf8 v8, v4, v0.t
+# CHECK-INST: vsext.vf8 v8, v4, v0.t
+# CHECK-ENCODING: [0x57,0xa4,0x41,0x48]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 a4 41 48 <unknown>
+
+vsext.vf8 v8, v4
+# CHECK-INST: vsext.vf8 v8, v4
+# CHECK-ENCODING: [0x57,0xa4,0x41,0x4a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 a4 41 4a <unknown>
diff --git a/llvm/test/MC/RISCV/rvv/fcompare.s b/llvm/test/MC/RISCV/rvv/fcompare.s
index 9ad55dc54a07e..4bb4e2d966f28 100644
--- a/llvm/test/MC/RISCV/rvv/fcompare.s
+++ b/llvm/test/MC/RISCV/rvv/fcompare.s
@@ -151,3 +151,9 @@ vmfge.vv v8, v20, v4
 # CHECK-ENCODING: [0x57,0x14,0x4a,0x66]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 57 14 4a 66 <unknown>
+
+vmfeq.vv v0, v4, v20, v0.t
+# CHECK-INST: vmfeq.vv v0, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x10,0x4a,0x60]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 10 4a 60 <unknown>
diff --git a/llvm/test/MC/RISCV/rvv/fothers.s b/llvm/test/MC/RISCV/rvv/fothers.s
index 768800def4248..b9e03842fa2de 100644
--- a/llvm/test/MC/RISCV/rvv/fothers.s
+++ b/llvm/test/MC/RISCV/rvv/fothers.s
@@ -10,30 +10,54 @@
 
 vfsqrt.v v8, v4, v0.t
 # CHECK-INST: vfsqrt.v v8, v4, v0.t
-# CHECK-ENCODING: [0x57,0x14,0x40,0x8c]
+# CHECK-ENCODING: [0x57,0x14,0x40,0x4c]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
-# CHECK-UNKNOWN: 57 14 40 8c <unknown>
+# CHECK-UNKNOWN: 57 14 40 4c <unknown>
 
 vfsqrt.v v8, v4
 # CHECK-INST: vfsqrt.v v8, v4
-# CHECK-ENCODING: [0x57,0x14,0x40,0x8e]
+# CHECK-ENCODING: [0x57,0x14,0x40,0x4e]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
-# CHECK-UNKNOWN: 57 14 40 8e <unknown>
+# CHECK-UNKNOWN: 57 14 40 4e <unknown>
 
 vfclass.v v8, v4, v0.t
 # CHECK-INST: vfclass.v v8, v4, v0.t
-# CHECK-ENCODING: [0x57,0x14,0x48,0x8c]
+# CHECK-ENCODING: [0x57,0x14,0x48,0x4c]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
-# CHECK-UNKNOWN: 57 14 48 8c <unknown>
+# CHECK-UNKNOWN: 57 14 48 4c <unknown>
 
 vfclass.v v8, v4
 # CHECK-INST: vfclass.v v8, v4
-# CHECK-ENCODING: [0x57,0x14,0x48,0x8e]
+# CHECK-ENCODING: [0x57,0x14,0x48,0x4e]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
-# CHECK-UNKNOWN: 57 14 48 8e <unknown>
+# CHECK-UNKNOWN: 57 14 48 4e <unknown>
 
 vfmerge.vfm v8, v4, fa0, v0
 # CHECK-INST: vfmerge.vfm v8, v4, fa0, v0
 # CHECK-ENCODING: [0x57,0x54,0x45,0x5c]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 57 54 45 5c <unknown>
+
+vfslide1up.vf v8, v4, fa0, v0.t
+# CHECK-INST: vfslide1up.vf v8, v4, fa0, v0.t
+# CHECK-ENCODING: [0x57,0x54,0x45,0x38]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 38 <unknown>
+
+vfslide1up.vf v8, v4, fa0
+# CHECK-INST: vfslide1up.vf v8, v4, fa0
+# CHECK-ENCODING: [0x57,0x54,0x45,0x3a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 3a <unknown>
+
+vfslide1down.vf v8, v4, fa0, v0.t
+# CHECK-INST: vfslide1down.vf v8, v4, fa0, v0.t
+# CHECK-ENCODING: [0x57,0x54,0x45,0x3c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 3c <unknown>
+
+vfslide1down.vf v8, v4, fa0
+# CHECK-INST: vfslide1down.vf v8, v4, fa0
+# CHECK-ENCODING: [0x57,0x54,0x45,0x3e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 54 45 3e <unknown>
diff --git a/llvm/test/MC/RISCV/rvv/freduction.s b/llvm/test/MC/RISCV/rvv/freduction.s
index 2131dadff8ca8..a85e676f9a8ca 100644
--- a/llvm/test/MC/RISCV/rvv/freduction.s
+++ b/llvm/test/MC/RISCV/rvv/freduction.s
@@ -79,3 +79,9 @@ vfwredsum.vs v8, v4, v20
 # CHECK-ENCODING: [0x57,0x14,0x4a,0xc6]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 57 14 4a c6 <unknown>
+
+vfredosum.vs v0, v4, v20, v0.t
+# CHECK-INST: vfredosum.vs v0, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x10,0x4a,0x0c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 10 4a 0c <unknown>
diff --git a/llvm/test/MC/RISCV/rvv/invalid.s b/llvm/test/MC/RISCV/rvv/invalid.s
index 814093eba5059..615dc08ad67ce 100644
--- a/llvm/test/MC/RISCV/rvv/invalid.s
+++ b/llvm/test/MC/RISCV/rvv/invalid.s
@@ -2,22 +2,34 @@
 # RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
 
 vsetvli a2, a0, e31
-# CHECK-ERROR: operand must be e[8|16|32|64|128|256|512|1024],m[1|2|4|8]
+# CHECK-ERROR: operand must be e[8|16|32|64|128|256|512|1024],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
 
 vsetvli a2, a0, e32,m3
-# CHECK-ERROR: operand must be e[8|16|32|64|128|256|512|1024],m[1|2|4|8]
+# CHECK-ERROR: operand must be e[8|16|32|64|128|256|512|1024],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
 
 vsetvli a2, a0, m1,e32
-# CHECK-ERROR: operand must be e[8|16|32|64|128|256|512|1024],m[1|2|4|8]
+# CHECK-ERROR: operand must be e[8|16|32|64|128|256|512|1024],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
 
 vsetvli a2, a0, e32,m16
-# CHECK-ERROR: operand must be e[8|16|32|64|128|256|512|1024],m[1|2|4|8]
+# CHECK-ERROR: operand must be e[8|16|32|64|128|256|512|1024],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
 
 vsetvli a2, a0, e2048,m8
-# CHECK-ERROR: operand must be e[8|16|32|64|128|256|512|1024],m[1|2|4|8]
+# CHECK-ERROR: operand must be e[8|16|32|64|128|256|512|1024],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
 
 vsetvli a2, a0, e1,m8
-# CHECK-ERROR: operand must be e[8|16|32|64|128|256|512|1024],m[1|2|4|8]
+# CHECK-ERROR: operand must be e[8|16|32|64|128|256|512|1024],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
+
+vsetvli a2, a0, e8,m1,tx
+# CHECK-ERROR: operand must be e[8|16|32|64|128|256|512|1024],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
+
+vsetvli a2, a0, e8,m1,ta,mx
+# CHECK-ERROR: operand must be e[8|16|32|64|128|256|512|1024],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
+
+vsetvli a2, a0, e8,m1,ma
+# CHECK-ERROR: operand must be e[8|16|32|64|128|256|512|1024],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
+
+vsetvli a2, a0, e8,m1,mu
+# CHECK-ERROR: operand must be e[8|16|32|64|128|256|512|1024],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
 
 vadd.vv v1, v3, v2, v4.t
 # CHECK-ERROR: operand must be v0.t
@@ -47,10 +59,6 @@ vfwcvt.xu.f.v v2, v2
 # CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
 # CHECK-ERROR-LABEL: vfwcvt.xu.f.v v2, v2
 
-vfwcvt.xu.f.v v2, v3
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
-# CHECK-ERROR-LABEL: vfwcvt.xu.f.v v2, v3
-
 vfwcvt.x.f.v v0, v2, v0.t
 # CHECK-ERROR: The destination vector register group cannot overlap the mask register.
 # CHECK-ERROR-LABEL: vfwcvt.x.f.v v0, v2, v0.t
@@ -59,10 +67,6 @@ vfwcvt.x.f.v v2, v2
 # CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
 # CHECK-ERROR-LABEL: vfwcvt.x.f.v v2, v2
 
-vfwcvt.x.f.v v2, v3
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
-# CHECK-ERROR-LABEL: vfwcvt.x.f.v v2, v3
-
 vfwcvt.f.xu.v v0, v2, v0.t
 # CHECK-ERROR: The destination vector register group cannot overlap the mask register.
 # CHECK-ERROR-LABEL: vfwcvt.f.xu.v v0, v2, v0.t
@@ -71,10 +75,6 @@ vfwcvt.f.xu.v v2, v2
 # CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
 # CHECK-ERROR-LABEL: vfwcvt.f.xu.v v2, v2
 
-vfwcvt.f.xu.v v2, v3
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
-# CHECK-ERROR-LABEL: vfwcvt.f.xu.v v2, v3
-
 vfwcvt.f.x.v v0, v2, v0.t
 # CHECK-ERROR: The destination vector register group cannot overlap the mask register.
 # CHECK-ERROR-LABEL: vfwcvt.f.x.v v0, v2, v0.t
@@ -83,10 +83,6 @@ vfwcvt.f.x.v v2, v2
 # CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
 # CHECK-ERROR-LABEL: vfwcvt.f.x.v v2, v2
 
-vfwcvt.f.x.v v2, v3
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
-# CHECK-ERROR-LABEL: vfwcvt.f.x.v v2, v3
-
 vfwcvt.f.f.v v0, v2, v0.t
 # CHECK-ERROR: The destination vector register group cannot overlap the mask register.
 # CHECK-ERROR-LABEL: vfwcvt.f.f.v v0, v2, v0.t
@@ -95,10 +91,6 @@ vfwcvt.f.f.v v2, v2
 # CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
 # CHECK-ERROR-LABEL: vfwcvt.f.f.v v2, v2
 
-vfwcvt.f.f.v v2, v3
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
-# CHECK-ERROR-LABEL: vfwcvt.f.f.v v2, v3
-
 vslideup.vx v0, v2, a0, v0.t
 # CHECK-ERROR: The destination vector register group cannot overlap the mask register.
 # CHECK-ERROR-LABEL: vslideup.vx v0, v2, a0, v0.t
@@ -127,10 +119,6 @@ vnsrl.wv v2, v2, v4
 # CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
 # CHECK-ERROR-LABEL: vnsrl.wv v2, v2, v4
 
-vnsrl.wv v3, v2, v4
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
-# CHECK-ERROR-LABEL: vnsrl.wv v3, v2, v4
-
 vnsrl.wx v2, v2, a0
 # CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
 # CHECK-ERROR-LABEL: vnsrl.wx v2, v2, a0
@@ -143,10 +131,6 @@ vnsra.wv v2, v2, v4
 # CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
 # CHECK-ERROR-LABEL: vnsra.wv v2, v2, v4
 
-vnsra.wv v3, v2, v4
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
-# CHECK-ERROR-LABEL: vnsra.wv v3, v2, v4
-
 vnsra.wx v2, v2, a0
 # CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
 # CHECK-ERROR-LABEL: vnsra.wx v2, v2, a0
@@ -159,10 +143,6 @@ vnclipu.wv v2, v2, v4
 # CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
 # CHECK-ERROR-LABEL: vnclipu.wv v2, v2, v4
 
-vnclipu.wv v3, v2, v4
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
-# CHECK-ERROR-LABEL: vnclipu.wv v3, v2, v4
-
 vnclipu.wx v2, v2, a0
 # CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
 # CHECK-ERROR-LABEL: vnclipu.wx v2, v2, a0
@@ -175,10 +155,6 @@ vnclip.wv v2, v2, v4
 # CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
 # CHECK-ERROR-LABEL: vnclip.wv v2, v2, v4
 
-vnclip.wv v3, v2, v4
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
-# CHECK-ERROR-LABEL: vnclip.wv v3, v2, v4
-
 vnclip.wx v2, v2, a0
 # CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
 # CHECK-ERROR-LABEL: vnclip.wx v2, v2, a0
@@ -191,50 +167,26 @@ vfncvt.xu.f.w v2, v2
 # CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
 # CHECK-ERROR-LABEL: vfncvt.xu.f.w v2, v2
 
-vfncvt.xu.f.w v3, v2
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
-# CHECK-ERROR-LABEL: vfncvt.xu.f.w v3, v2
-
 vfncvt.x.f.w v2, v2
 # CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
 # CHECK-ERROR-LABEL: vfncvt.x.f.w v2, v2
 
-vfncvt.x.f.w v3, v2
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
-# CHECK-ERROR-LABEL: vfncvt.x.f.w v3, v2
-
 vfncvt.f.xu.w v2, v2
 # CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
 # CHECK-ERROR-LABEL: vfncvt.f.xu.w v2, v2
 
-vfncvt.f.xu.w v3, v2
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
-# CHECK-ERROR-LABEL: vfncvt.f.xu.w v3, v2
-
 vfncvt.f.x.w v2, v2
 # CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
 # CHECK-ERROR-LABEL: vfncvt.f.x.w v2, v2
 
-vfncvt.f.x.w v3, v2
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
-# CHECK-ERROR-LABEL: vfncvt.f.x.w v3, v2
-
 vfncvt.f.f.w v2, v2
 # CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
 # CHECK-ERROR-LABEL: vfncvt.f.f.w v2, v2
 
-vfncvt.f.f.w v3, v2
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
-# CHECK-ERROR-LABEL: vfncvt.f.f.w v3, v2
-
 vfncvt.rod.f.f.w v2, v2
 # CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
 # CHECK-ERROR-LABEL: vfncvt.rod.f.f.w v2, v2
 
-vfncvt.rod.f.f.w v3, v2
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
-# CHECK-ERROR-LABEL: vfncvt.rod.f.f.w v3, v2
-
 vrgather.vv v0, v2, v4, v0.t
 # CHECK-ERROR: The destination vector register group cannot overlap the mask register.
 # CHECK-ERROR-LABEL: vrgather.vv v0, v2, v4, v0.t
@@ -267,10 +219,6 @@ vwaddu.vv v2, v2, v4
 # CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
 # CHECK-ERROR-LABEL: vwaddu.vv v2, v2, v4
 
-vwaddu.vv v2, v3, v4
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
-# CHECK-ERROR-LABEL: vwaddu.vv v2, v3, v4
-
 vwsubu.vv v0, v2, v4, v0.t
 # CHECK-ERROR: The destination vector register group cannot overlap the mask register.
 # CHECK-ERROR-LABEL: vwsubu.vv v0, v2, v4, v0.t
@@ -279,10 +227,6 @@ vwsubu.vv v2, v2, v4
 # CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
 # CHECK-ERROR-LABEL: vwsubu.vv v2, v2, v4
 
-vwsubu.vv v2, v3, v4
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
-# CHECK-ERROR-LABEL: vwsubu.vv v2, v3, v4
-
 vwadd.vv v0, v2, v4, v0.t
 # CHECK-ERROR: The destination vector register group cannot overlap the mask register.
 # CHECK-ERROR-LABEL: vwadd.vv v0, v2, v4, v0.t
@@ -291,10 +235,6 @@ vwadd.vv v2, v2, v4
 # CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
 # CHECK-ERROR-LABEL: vwadd.vv v2, v2, v4
 
-vwadd.vv v2, v3, v4
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
-# CHECK-ERROR-LABEL: vwadd.vv v2, v3, v4
-
 vwsub.vv v0, v2, v4, v0.t
 # CHECK-ERROR: The destination vector register group cannot overlap the mask register.
 # CHECK-ERROR-LABEL: vwsub.vv v0, v2, v4, v0.t
@@ -303,10 +243,6 @@ vwsub.vv v2, v2, v4
 # CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
 # CHECK-ERROR-LABEL: vwsub.vv v2, v2, v4
 
-vwsub.vv v2, v3, v4
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
-# CHECK-ERROR-LABEL: vwsub.vv v2, v3, v4
-
 vwmul.vv v0, v2, v4, v0.t
 # CHECK-ERROR: The destination vector register group cannot overlap the mask register.
 # CHECK-ERROR-LABEL: vwmul.vv v0, v2, v4, v0.t
@@ -315,10 +251,6 @@ vwmul.vv v2, v2, v4
 # CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
 # CHECK-ERROR-LABEL: vwmul.vv v2, v2, v4
 
-vwmul.vv v2, v3, v4
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
-# CHECK-ERROR-LABEL: vwmul.vv v2, v3, v4
-
 vwmulu.vv v0, v2, v4, v0.t
 # CHECK-ERROR: The destination vector register group cannot overlap the mask register.
 # CHECK-ERROR-LABEL: vwmulu.vv v0, v2, v4, v0.t
@@ -327,10 +259,6 @@ vwmulu.vv v2, v2, v4
 # CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
 # CHECK-ERROR-LABEL: vwmulu.vv v2, v2, v4
 
-vwmulu.vv v2, v3, v4
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
-# CHECK-ERROR-LABEL: vwmulu.vv v2, v3, v4
-
 vwmulsu.vv v0, v2, v4, v0.t
 # CHECK-ERROR: The destination vector register group cannot overlap the mask register.
 # CHECK-ERROR-LABEL: vwmulsu.vv v0, v2, v4, v0.t
@@ -339,10 +267,6 @@ vwmulsu.vv v2, v2, v4
 # CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
 # CHECK-ERROR-LABEL: vwmulsu.vv v2, v2, v4
 
-vwmulsu.vv v2, v3, v4
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
-# CHECK-ERROR-LABEL: vwmulsu.vv v2, v3, v4
-
 vwmaccu.vv v0, v4, v2, v0.t
 # CHECK-ERROR: The destination vector register group cannot overlap the mask register.
 # CHECK-ERROR-LABEL: vwmaccu.vv v0, v4, v2, v0.t
@@ -351,10 +275,6 @@ vwmaccu.vv v2, v4, v2
 # CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
 # CHECK-ERROR-LABEL: vwmaccu.vv v2, v4, v2
 
-vwmaccu.vv v2, v4, v3
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
-# CHECK-ERROR-LABEL: vwmaccu.vv v2, v4, v3
-
 vwmacc.vv v0, v4, v2, v0.t
 # CHECK-ERROR: The destination vector register group cannot overlap the mask register.
 # CHECK-ERROR-LABEL: vwmacc.vv v0, v4, v2, v0.t
@@ -363,10 +283,6 @@ vwmacc.vv v2, v4, v2
 # CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
 # CHECK-ERROR-LABEL: vwmacc.vv v2, v4, v2
 
-vwmacc.vv v2, v4, v3
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
-# CHECK-ERROR-LABEL: vwmacc.vv v2, v4, v3
-
 vwmaccsu.vv v0, v4, v2, v0.t
 # CHECK-ERROR: The destination vector register group cannot overlap the mask register.
 # CHECK-ERROR-LABEL: vwmaccsu.vv v0, v4, v2, v0.t
@@ -375,10 +291,6 @@ vwmaccsu.vv v2, v4, v2
 # CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
 # CHECK-ERROR-LABEL: vwmaccsu.vv v2, v4, v2
 
-vwmaccsu.vv v2, v4, v3
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
-# CHECK-ERROR-LABEL: vwmaccsu.vv v2, v4, v3
-
 vfwadd.vv v0, v2, v4, v0.t
 # CHECK-ERROR: The destination vector register group cannot overlap the mask register.
 # CHECK-ERROR-LABEL: vfwadd.vv v0, v2, v4, v0.t
@@ -387,10 +299,6 @@ vfwadd.vv v2, v2, v4
 # CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
 # CHECK-ERROR-LABEL: vfwadd.vv v2, v2, v4
 
-vfwadd.vv v2, v3, v4
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
-# CHECK-ERROR-LABEL: vfwadd.vv v2, v3, v4
-
 vfwsub.vv v0, v2, v4, v0.t
 # CHECK-ERROR: The destination vector register group cannot overlap the mask register.
 # CHECK-ERROR-LABEL: vfwsub.vv v0, v2, v4, v0.t
@@ -399,10 +307,6 @@ vfwsub.vv v2, v2, v4
 # CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
 # CHECK-ERROR-LABEL: vfwsub.vv v2, v2, v4
 
-vfwsub.vv v2, v3, v4
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
-# CHECK-ERROR-LABEL: vfwsub.vv v2, v3, v4
-
 vfwmul.vv v0, v2, v4, v0.t
 # CHECK-ERROR: The destination vector register group cannot overlap the mask register.
 # CHECK-ERROR-LABEL: vfwmul.vv v0, v2, v4, v0.t
@@ -411,10 +315,6 @@ vfwmul.vv v2, v2, v4
 # CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
 # CHECK-ERROR-LABEL: vfwmul.vv v2, v2, v4
 
-vfwmul.vv v2, v3, v4
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
-# CHECK-ERROR-LABEL: vfwmul.vv v2, v3, v4
-
 vfwmacc.vv v0, v4, v2, v0.t
 # CHECK-ERROR: The destination vector register group cannot overlap the mask register.
 # CHECK-ERROR-LABEL: vfwmacc.vv v0, v4, v2, v0.t
@@ -423,10 +323,6 @@ vfwmacc.vv v2, v4, v2
 # CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
 # CHECK-ERROR-LABEL: vfwmacc.vv v2, v4, v2
 
-vfwmacc.vv v2, v4, v3
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
-# CHECK-ERROR-LABEL: vfwmacc.vv v2, v4, v3
-
 vfwnmacc.vv v0, v4, v2, v0.t
 # CHECK-ERROR: The destination vector register group cannot overlap the mask register.
 # CHECK-ERROR-LABEL: vfwnmacc.vv v0, v4, v2, v0.t
@@ -435,10 +331,6 @@ vfwnmacc.vv v2, v4, v2
 # CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
 # CHECK-ERROR-LABEL: vfwnmacc.vv v2, v4, v2
 
-vfwnmacc.vv v2, v4, v3
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
-# CHECK-ERROR-LABEL: vfwnmacc.vv v2, v4, v3
-
 vfwmsac.vv v0, v4, v2, v0.t
 # CHECK-ERROR: The destination vector register group cannot overlap the mask register.
 # CHECK-ERROR-LABEL: vfwmsac.vv v0, v4, v2, v0.t
@@ -447,10 +339,6 @@ vfwmsac.vv v2, v4, v2
 # CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
 # CHECK-ERROR-LABEL: vfwmsac.vv v2, v4, v2
 
-vfwmsac.vv v2, v4, v3
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
-# CHECK-ERROR-LABEL: vfwmsac.vv v2, v4, v3
-
 vfwnmsac.vv v0, v4, v2, v0.t
 # CHECK-ERROR: The destination vector register group cannot overlap the mask register.
 # CHECK-ERROR-LABEL: vfwnmsac.vv v0, v4, v2, v0.t
@@ -459,10 +347,6 @@ vfwnmsac.vv v2, v4, v2
 # CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
 # CHECK-ERROR-LABEL: vfwnmsac.vv v2, v4, v2
 
-vfwnmsac.vv v2, v4, v3
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
-# CHECK-ERROR-LABEL: vfwnmsac.vv v2, v4, v3
-
 vwaddu.vx v0, v2, a0, v0.t
 # CHECK-ERROR: The destination vector register group cannot overlap the mask register.
 # CHECK-ERROR-LABEL: vwaddu.vx v0, v2, a0, v0.t
@@ -471,10 +355,6 @@ vwaddu.vx v2, v2, a0
 # CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
 # CHECK-ERROR-LABEL: vwaddu.vx v2, v2, a0
 
-vwaddu.vx v2, v3, a0
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
-# CHECK-ERROR-LABEL: vwaddu.vx v2, v3, a0
-
 vwsubu.vx v0, v2, a0, v0.t
 # CHECK-ERROR: The destination vector register group cannot overlap the mask register.
 # CHECK-ERROR-LABEL: vwsubu.vx v0, v2, a0, v0.t
@@ -483,10 +363,6 @@ vwsubu.vx v2, v2, a0
 # CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
 # CHECK-ERROR-LABEL: vwsubu.vx v2, v2, a0
 
-vwsubu.vx v2, v3, a0
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
-# CHECK-ERROR-LABEL: vwsubu.vx v2, v3, a0
-
 vwadd.vx v0, v2, a0, v0.t
 # CHECK-ERROR: The destination vector register group cannot overlap the mask register.
 # CHECK-ERROR-LABEL: vwadd.vx v0, v2, a0, v0.t
@@ -495,10 +371,6 @@ vwadd.vx v2, v2, a0
 # CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
 # CHECK-ERROR-LABEL: vwadd.vx v2, v2, a0
 
-vwadd.vx v2, v3, a0
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
-# CHECK-ERROR-LABEL: vwadd.vx v2, v3, a0
-
 vwsub.vx v0, v2, a0, v0.t
 # CHECK-ERROR: The destination vector register group cannot overlap the mask register.
 # CHECK-ERROR-LABEL: vwsub.vx v0, v2, a0, v0.t
@@ -507,10 +379,6 @@ vwsub.vx v2, v2, a0
 # CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
 # CHECK-ERROR-LABEL: vwsub.vx v2, v2, a0
 
-vwsub.vx v2, v3, a0
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
-# CHECK-ERROR-LABEL: vwsub.vx v2, v3, a0
-
 vwmul.vx v0, v2, a0, v0.t
 # CHECK-ERROR: The destination vector register group cannot overlap the mask register.
 # CHECK-ERROR-LABEL: vwmul.vx v0, v2, a0, v0.t
@@ -519,10 +387,6 @@ vwmul.vx v2, v2, a0
 # CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
 # CHECK-ERROR-LABEL: vwmul.vx v2, v2, a0
 
-vwmul.vx v2, v3, a0
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
-# CHECK-ERROR-LABEL: vwmul.vx v2, v3, a0
-
 vwmulu.vx v0, v2, a0, v0.t
 # CHECK-ERROR: The destination vector register group cannot overlap the mask register.
 # CHECK-ERROR-LABEL: vwmulu.vx v0, v2, a0, v0.t
@@ -531,10 +395,6 @@ vwmulu.vx v2, v2, a0
 # CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
 # CHECK-ERROR-LABEL: vwmulu.vx v2, v2, a0
 
-vwmulu.vx v2, v3, a0
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
-# CHECK-ERROR-LABEL: vwmulu.vx v2, v3, a0
-
 vwmulsu.vx v0, v2, a0, v0.t
 # CHECK-ERROR: The destination vector register group cannot overlap the mask register.
 # CHECK-ERROR-LABEL: vwmulsu.vx v0, v2, a0, v0.t
@@ -543,10 +403,6 @@ vwmulsu.vx v2, v2, a0
 # CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
 # CHECK-ERROR-LABEL: vwmulsu.vx v2, v2, a0
 
-vwmulsu.vx v2, v3, a0
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
-# CHECK-ERROR-LABEL: vwmulsu.vx v2, v3, a0
-
 vwmaccu.vx v0, a0, v2, v0.t
 # CHECK-ERROR: The destination vector register group cannot overlap the mask register.
 # CHECK-ERROR-LABEL: vwmaccu.vx v0, a0, v2, v0.t
@@ -555,10 +411,6 @@ vwmaccu.vx v2, a0, v2
 # CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
 # CHECK-ERROR-LABEL: vwmaccu.vx v2, a0, v2
 
-vwmaccu.vx v2, a0, v3
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
-# CHECK-ERROR-LABEL: vwmaccu.vx v2, a0, v3
-
 vwmacc.vx v0, a0, v2, v0.t
 # CHECK-ERROR: The destination vector register group cannot overlap the mask register.
 # CHECK-ERROR-LABEL: vwmacc.vx v0, a0, v2, v0.t
@@ -567,10 +419,6 @@ vwmacc.vx v2, a0, v2
 # CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
 # CHECK-ERROR-LABEL: vwmacc.vx v2, a0, v2
 
-vwmacc.vx v2, a0, v3
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
-# CHECK-ERROR-LABEL: vwmacc.vx v2, a0, v3
-
 vwmaccsu.vx v0, a0, v2, v0.t
 # CHECK-ERROR: The destination vector register group cannot overlap the mask register.
 # CHECK-ERROR-LABEL: vwmaccsu.vx v0, a0, v2, v0.t
@@ -579,10 +427,6 @@ vwmaccsu.vx v2, a0, v2
 # CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
 # CHECK-ERROR-LABEL: vwmaccsu.vx v2, a0, v2
 
-vwmaccsu.vx v2, a0, v3
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
-# CHECK-ERROR-LABEL: vwmaccsu.vx v2, a0, v3
-
 vwmaccus.vx v0, a0, v2, v0.t
 # CHECK-ERROR: The destination vector register group cannot overlap the mask register.
 # CHECK-ERROR-LABEL: vwmaccus.vx v0, a0, v2, v0.t
@@ -591,10 +435,6 @@ vwmaccus.vx v2, a0, v2
 # CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
 # CHECK-ERROR-LABEL: vwmaccus.vx v2, a0, v2
 
-vwmaccus.vx v2, a0, v3
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
-# CHECK-ERROR-LABEL: vwmaccus.vx v2, a0, v3
-
 vfwadd.vf v0, v2, fa0, v0.t
 # CHECK-ERROR: The destination vector register group cannot overlap the mask register.
 # CHECK-ERROR-LABEL: vfwadd.vf v0, v2, fa0, v0.t
@@ -603,10 +443,6 @@ vfwadd.vf v2, v2, fa0
 # CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
 # CHECK-ERROR-LABEL: vfwadd.vf v2, v2, fa0
 
-vfwadd.vf v2, v3, fa0
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
-# CHECK-ERROR-LABEL: vfwadd.vf v2, v3, fa0
-
 vfwsub.vf v0, v2, fa0, v0.t
 # CHECK-ERROR: The destination vector register group cannot overlap the mask register.
 # CHECK-ERROR-LABEL: vfwsub.vf v0, v2, fa0, v0.t
@@ -615,10 +451,6 @@ vfwsub.vf v2, v2, fa0
 # CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
 # CHECK-ERROR-LABEL: vfwsub.vf v2, v2, fa0
 
-vfwsub.vf v2, v3, fa0
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
-# CHECK-ERROR-LABEL: vfwsub.vf v2, v3, fa0
-
 vfwmul.vf v0, v2, fa0, v0.t
 # CHECK-ERROR: The destination vector register group cannot overlap the mask register.
 # CHECK-ERROR-LABEL: vfwmul.vf v0, v2, fa0, v0.t
@@ -627,10 +459,6 @@ vfwmul.vf v2, v2, fa0
 # CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
 # CHECK-ERROR-LABEL: vfwmul.vf v2, v2, fa0
 
-vfwmul.vf v2, v3, fa0
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
-# CHECK-ERROR-LABEL: vfwmul.vf v2, v3, fa0
-
 vfwmacc.vf v0, fa0, v2, v0.t
 # CHECK-ERROR: The destination vector register group cannot overlap the mask register.
 # CHECK-ERROR-LABEL: vfwmacc.vf v0, fa0, v2, v0.t
@@ -639,10 +467,6 @@ vfwmacc.vf v2, fa0, v2
 # CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
 # CHECK-ERROR-LABEL: vfwmacc.vf v2, fa0, v2
 
-vfwmacc.vf v2, fa0, v3
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
-# CHECK-ERROR-LABEL: vfwmacc.vf v2, fa0, v3
-
 vfwnmacc.vf v0, fa0, v2, v0.t
 # CHECK-ERROR: The destination vector register group cannot overlap the mask register.
 # CHECK-ERROR-LABEL: vfwnmacc.vf v0, fa0, v2, v0.t
@@ -651,10 +475,6 @@ vfwnmacc.vf v2, fa0, v2
 # CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
 # CHECK-ERROR-LABEL: vfwnmacc.vf v2, fa0, v2
 
-vfwnmacc.vf v2, fa0, v3
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
-# CHECK-ERROR-LABEL: vfwnmacc.vf v2, fa0, v3
-
 vfwmsac.vf v0, fa0, v2, v0.t
 # CHECK-ERROR: The destination vector register group cannot overlap the mask register.
 # CHECK-ERROR-LABEL: vfwmsac.vf v0, fa0, v2, v0.t
@@ -663,10 +483,6 @@ vfwmsac.vf v2, fa0, v2
 # CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
 # CHECK-ERROR-LABEL: vfwmsac.vf v2, fa0, v2
 
-vfwmsac.vf v2, fa0, v3
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
-# CHECK-ERROR-LABEL: vfwmsac.vf v2, fa0, v3
-
 vfwnmsac.vf v0, fa0, v2, v0.t
 # CHECK-ERROR: The destination vector register group cannot overlap the mask register.
 # CHECK-ERROR-LABEL: vfwnmsac.vf v0, fa0, v2, v0.t
@@ -675,10 +491,6 @@ vfwnmsac.vf v2, fa0, v2
 # CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
 # CHECK-ERROR-LABEL: vfwnmsac.vf v2, fa0, v2
 
-vfwnmsac.vf v2, fa0, v3
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
-# CHECK-ERROR-LABEL: vfwnmsac.vf v2, fa0, v3
-
 vcompress.vm v2, v2, v4
 # CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
 # CHECK-ERROR-LABEL: vcompress.vm v2, v2, v4
@@ -691,10 +503,6 @@ vwaddu.wv v2, v4, v2
 # CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
 # CHECK-ERROR-LABEL: vwaddu.wv v2, v4, v2
 
-vwaddu.wv v2, v4, v3
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
-# CHECK-ERROR-LABEL: vwaddu.wv v2, v4, v3
-
 vwsubu.wv v0, v2, v4, v0.t
 # CHECK-ERROR: The destination vector register group cannot overlap the mask register.
 # CHECK-ERROR-LABEL: vwsubu.wv v0, v2, v4, v0.t
@@ -703,10 +511,6 @@ vwsubu.wv v2, v4, v2
 # CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
 # CHECK-ERROR-LABEL: vwsubu.wv v2, v4, v2
 
-vwsubu.wv v2, v4, v3
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
-# CHECK-ERROR-LABEL: vwsubu.wv v2, v4, v3
-
 vwadd.wv v0, v2, v4, v0.t
 # CHECK-ERROR: The destination vector register group cannot overlap the mask register.
 # CHECK-ERROR-LABEL: vwadd.wv v0, v2, v4, v0.t
@@ -715,10 +519,6 @@ vwadd.wv v2, v4, v2
 # CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
 # CHECK-ERROR-LABEL: vwadd.wv v2, v4, v2
 
-vwadd.wv v2, v4, v3
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
-# CHECK-ERROR-LABEL: vwadd.wv v2, v4, v3
-
 vwsub.wv v0, v2, v4, v0.t
 # CHECK-ERROR: The destination vector register group cannot overlap the mask register.
 # CHECK-ERROR-LABEL: vwsub.wv v0, v2, v4, v0.t
@@ -727,10 +527,6 @@ vwsub.wv v2, v4, v2
 # CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
 # CHECK-ERROR-LABEL: vwsub.wv v2, v4, v2
 
-vwsub.wv v2, v4, v3
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
-# CHECK-ERROR-LABEL: vwsub.wv v2, v4, v3
-
 vfwadd.wv v0, v2, v4, v0.t
 # CHECK-ERROR: The destination vector register group cannot overlap the mask register.
 # CHECK-ERROR-LABEL: vfwadd.wv v0, v2, v4, v0.t
@@ -739,10 +535,6 @@ vfwadd.wv v2, v4, v2
 # CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
 # CHECK-ERROR-LABEL: vfwadd.wv v2, v4, v2
 
-vfwadd.wv v2, v4, v3
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
-# CHECK-ERROR-LABEL: vfwadd.wv v2, v4, v3
-
 vfwsub.wv v0, v2, v4, v0.t
 # CHECK-ERROR: The destination vector register group cannot overlap the mask register.
 # CHECK-ERROR-LABEL: vfwsub.wv v0, v2, v4, v0.t
@@ -751,10 +543,6 @@ vfwsub.wv v2, v4, v2
 # CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
 # CHECK-ERROR-LABEL: vfwsub.wv v2, v4, v2
 
-vfwsub.wv v2, v4, v3
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
-# CHECK-ERROR-LABEL: vfwsub.wv v2, v4, v3
-
 vwaddu.wx v0, v2, a0, v0.t
 # CHECK-ERROR: The destination vector register group cannot overlap the mask register.
 # CHECK-ERROR-LABEL: vwaddu.wx v0, v2, a0, v0.t
@@ -778,3 +566,27 @@ vfwadd.wf v0, v2, fa0, v0.t
 vfwsub.wf v0, v2, fa0, v0.t
 # CHECK-ERROR: The destination vector register group cannot overlap the mask register.
 # CHECK-ERROR-LABEL: vfwsub.wf v0, v2, fa0, v0.t
+
+vadc.vvm v0, v2, v4, v0
+# CHECK-ERROR: The destination vector register group cannot be V0.
+# CHECK-ERROR-LABEL: vadc.vvm v0, v2, v4, v0
+
+vmadc.vvm v2, v2, v4, v0
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vmadc.vvm v2, v2, v4, v0
+
+vmadc.vvm v4, v2, v4, v0
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vmadc.vvm v4, v2, v4, v0
+
+vadd.vv v0, v2, v4, v0.t
+# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR-LABEL: vadd.vv v0, v2, v4, v0.t
+
+vadd.vx v0, v2, a0, v0.t
+# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR-LABEL: vadd.vx v0, v2, a0, v0.t
+
+vadd.vi v0, v2, 1, v0.t
+# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR-LABEL: vadd.vi v0, v2, 1, v0.t
diff --git a/llvm/test/MC/RISCV/rvv/load.s b/llvm/test/MC/RISCV/rvv/load.s
index c8284b561f4af..eac3191c2d6b1 100644
--- a/llvm/test/MC/RISCV/rvv/load.s
+++ b/llvm/test/MC/RISCV/rvv/load.s
@@ -8,332 +8,392 @@
 # RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
 # RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
 
-vlb.v v8, (a0), v0.t
-# CHECK-INST: vlb.v v8, (a0), v0.t
-# CHECK-ENCODING: [0x07,0x04,0x05,0x10]
-# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
-# CHECK-UNKNOWN: 07 04 05 10 <unknown>
-
-vlb.v v8, (a0)
-# CHECK-INST: vlb.v v8, (a0)
-# CHECK-ENCODING: [0x07,0x04,0x05,0x12]
-# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
-# CHECK-UNKNOWN: 07 04 05 12 <unknown>
-
-vlh.v v8, (a0), v0.t
-# CHECK-INST: vlh.v v8, (a0), v0.t
-# CHECK-ENCODING: [0x07,0x54,0x05,0x10]
-# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
-# CHECK-UNKNOWN: 07 54 05 10 <unknown>
-
-vlh.v v8, (a0)
-# CHECK-INST: vlh.v v8, (a0)
-# CHECK-ENCODING: [0x07,0x54,0x05,0x12]
-# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
-# CHECK-UNKNOWN: 07 54 05 12 <unknown>
-
-vlw.v v8, (a0), v0.t
-# CHECK-INST: vlw.v v8, (a0), v0.t
-# CHECK-ENCODING: [0x07,0x64,0x05,0x10]
-# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
-# CHECK-UNKNOWN: 07 64 05 10 <unknown>
-
-vlw.v v8, (a0)
-# CHECK-INST: vlw.v v8, (a0)
-# CHECK-ENCODING: [0x07,0x64,0x05,0x12]
-# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
-# CHECK-UNKNOWN: 07 64 05 12 <unknown>
-
-vlbu.v v8, (a0), v0.t
-# CHECK-INST: vlbu.v v8, (a0), v0.t
+vle8.v v8, (a0), v0.t
+# CHECK-INST: vle8.v v8, (a0), v0.t
 # CHECK-ENCODING: [0x07,0x04,0x05,0x00]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 07 04 05 00 <unknown>
 
-vlbu.v v8, (a0)
-# CHECK-INST: vlbu.v v8, (a0)
+vle8.v v8, (a0)
+# CHECK-INST: vle8.v v8, (a0)
 # CHECK-ENCODING: [0x07,0x04,0x05,0x02]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 07 04 05 02 <unknown>
 
-vlhu.v v8, (a0), v0.t
-# CHECK-INST: vlhu.v v8, (a0), v0.t
+vle16.v v8, (a0), v0.t
+# CHECK-INST: vle16.v v8, (a0), v0.t
 # CHECK-ENCODING: [0x07,0x54,0x05,0x00]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 07 54 05 00 <unknown>
 
-vlhu.v v8, (a0)
-# CHECK-INST: vlhu.v v8, (a0)
+vle16.v v8, (a0)
+# CHECK-INST: vle16.v v8, (a0)
 # CHECK-ENCODING: [0x07,0x54,0x05,0x02]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 07 54 05 02 <unknown>
 
-vlwu.v v8, (a0), v0.t
-# CHECK-INST: vlwu.v v8, (a0), v0.t
+vle32.v v8, (a0), v0.t
+# CHECK-INST: vle32.v v8, (a0), v0.t
 # CHECK-ENCODING: [0x07,0x64,0x05,0x00]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 07 64 05 00 <unknown>
 
-vlwu.v v8, (a0)
-# CHECK-INST: vlwu.v v8, (a0)
+vle32.v v8, (a0)
+# CHECK-INST: vle32.v v8, (a0)
 # CHECK-ENCODING: [0x07,0x64,0x05,0x02]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 07 64 05 02 <unknown>
 
-vlbff.v v8, (a0), v0.t
-# CHECK-INST: vlbff.v v8, (a0), v0.t
-# CHECK-ENCODING: [0x07,0x04,0x05,0x11]
+vle64.v v8, (a0), v0.t
+# CHECK-INST: vle64.v v8, (a0), v0.t
+# CHECK-ENCODING: [0x07,0x74,0x05,0x00]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
-# CHECK-UNKNOWN: 07 04 05 11 <unknown>
+# CHECK-UNKNOWN: 07 74 05 00 <unknown>
 
-vlbff.v v8, (a0)
-# CHECK-INST: vlbff.v v8, (a0)
-# CHECK-ENCODING: [0x07,0x04,0x05,0x13]
+vle64.v v8, (a0)
+# CHECK-INST: vle64.v v8, (a0)
+# CHECK-ENCODING: [0x07,0x74,0x05,0x02]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
-# CHECK-UNKNOWN: 07 04 05 13 <unknown>
+# CHECK-UNKNOWN: 07 74 05 02 <unknown>
 
-vlhff.v v8, (a0), v0.t
-# CHECK-INST: vlhff.v v8, (a0), v0.t
-# CHECK-ENCODING: [0x07,0x54,0x05,0x11]
+vle128.v v8, (a0), v0.t
+# CHECK-INST: vle128.v v8, (a0), v0.t
+# CHECK-ENCODING: [0x07,0x04,0x05,0x10]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
-# CHECK-UNKNOWN: 07 54 05 11 <unknown>
+# CHECK-UNKNOWN: 07 04 05 10 <unknown>
 
-vlhff.v v8, (a0)
-# CHECK-INST: vlhff.v v8, (a0)
-# CHECK-ENCODING: [0x07,0x54,0x05,0x13]
+vle128.v v8, (a0)
+# CHECK-INST: vle128.v v8, (a0)
+# CHECK-ENCODING: [0x07,0x04,0x05,0x12]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
-# CHECK-UNKNOWN: 07 54 05 13 <unknown>
+# CHECK-UNKNOWN: 07 04 05 12 <unknown>
 
-vlwff.v v8, (a0), v0.t
-# CHECK-INST: vlwff.v v8, (a0), v0.t
-# CHECK-ENCODING: [0x07,0x64,0x05,0x11]
+vle256.v v8, (a0), v0.t
+# CHECK-INST: vle256.v v8, (a0), v0.t
+# CHECK-ENCODING: [0x07,0x54,0x05,0x10]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
-# CHECK-UNKNOWN: 07 64 05 11 <unknown>
+# CHECK-UNKNOWN: 07 54 05 10 <unknown>
 
-vlwff.v v8, (a0)
-# CHECK-INST: vlwff.v v8, (a0)
-# CHECK-ENCODING: [0x07,0x64,0x05,0x13]
+vle256.v v8, (a0)
+# CHECK-INST: vle256.v v8, (a0)
+# CHECK-ENCODING: [0x07,0x54,0x05,0x12]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
-# CHECK-UNKNOWN: 07 64 05 13 <unknown>
+# CHECK-UNKNOWN: 07 54 05 12 <unknown>
 
-vlbuff.v v8, (a0), v0.t
-# CHECK-INST: vlbuff.v v8, (a0), v0.t
+vle512.v v8, (a0), v0.t
+# CHECK-INST: vle512.v v8, (a0), v0.t
+# CHECK-ENCODING: [0x07,0x64,0x05,0x10]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 64 05 10 <unknown>
+
+vle512.v v8, (a0)
+# CHECK-INST: vle512.v v8, (a0)
+# CHECK-ENCODING: [0x07,0x64,0x05,0x12]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 64 05 12 <unknown>
+
+vle1024.v v8, (a0), v0.t
+# CHECK-INST: vle1024.v v8, (a0), v0.t
+# CHECK-ENCODING: [0x07,0x74,0x05,0x10]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 74 05 10 <unknown>
+
+vle1024.v v8, (a0)
+# CHECK-INST: vle1024.v v8, (a0)
+# CHECK-ENCODING: [0x07,0x74,0x05,0x12]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 74 05 12 <unknown>
+
+vle8ff.v v8, (a0), v0.t
+# CHECK-INST: vle8ff.v v8, (a0), v0.t
 # CHECK-ENCODING: [0x07,0x04,0x05,0x01]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 07 04 05 01 <unknown>
 
-vlbuff.v v8, (a0)
-# CHECK-INST: vlbuff.v v8, (a0)
+vle8ff.v v8, (a0)
+# CHECK-INST: vle8ff.v v8, (a0)
 # CHECK-ENCODING: [0x07,0x04,0x05,0x03]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 07 04 05 03 <unknown>
 
-vlhuff.v v8, (a0), v0.t
-# CHECK-INST: vlhuff.v v8, (a0), v0.t
+vle16ff.v v8, (a0), v0.t
+# CHECK-INST: vle16ff.v v8, (a0), v0.t
 # CHECK-ENCODING: [0x07,0x54,0x05,0x01]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 07 54 05 01 <unknown>
 
-vlhuff.v v8, (a0)
-# CHECK-INST: vlhuff.v v8, (a0)
+vle16ff.v v8, (a0)
+# CHECK-INST: vle16ff.v v8, (a0)
 # CHECK-ENCODING: [0x07,0x54,0x05,0x03]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 07 54 05 03 <unknown>
 
-vlwuff.v v8, (a0), v0.t
-# CHECK-INST: vlwuff.v v8, (a0), v0.t
+vle32ff.v v8, (a0), v0.t
+# CHECK-INST: vle32ff.v v8, (a0), v0.t
 # CHECK-ENCODING: [0x07,0x64,0x05,0x01]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 07 64 05 01 <unknown>
 
-vlwuff.v v8, (a0)
-# CHECK-INST: vlwuff.v v8, (a0)
+vle32ff.v v8, (a0)
+# CHECK-INST: vle32ff.v v8, (a0)
 # CHECK-ENCODING: [0x07,0x64,0x05,0x03]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 07 64 05 03 <unknown>
 
-vleff.v v8, (a0), v0.t
-# CHECK-INST: vleff.v v8, (a0), v0.t
+vle64ff.v v8, (a0), v0.t
+# CHECK-INST: vle64ff.v v8, (a0), v0.t
 # CHECK-ENCODING: [0x07,0x74,0x05,0x01]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 07 74 05 01 <unknown>
 
-vleff.v v8, (a0)
-# CHECK-INST: vleff.v v8, (a0)
+vle64ff.v v8, (a0)
+# CHECK-INST: vle64ff.v v8, (a0)
 # CHECK-ENCODING: [0x07,0x74,0x05,0x03]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 07 74 05 03 <unknown>
 
-vlsb.v v8, (a0), a1, v0.t
-# CHECK-INST: vlsb.v v8, (a0), a1, v0.t
-# CHECK-ENCODING: [0x07,0x04,0xb5,0x18]
+vle128ff.v v8, (a0), v0.t
+# CHECK-INST: vle128ff.v v8, (a0), v0.t
+# CHECK-ENCODING: [0x07,0x04,0x05,0x11]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
-# CHECK-UNKNOWN: 07 04 b5 18 <unknown>
+# CHECK-UNKNOWN: 07 04 05 11 <unknown>
 
-vlsb.v v8, (a0), a1
-# CHECK-INST: vlsb.v v8, (a0), a1
-# CHECK-ENCODING: [0x07,0x04,0xb5,0x1a]
+vle128ff.v v8, (a0)
+# CHECK-INST: vle128ff.v v8, (a0)
+# CHECK-ENCODING: [0x07,0x04,0x05,0x13]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
-# CHECK-UNKNOWN: 07 04 b5 1a <unknown>
+# CHECK-UNKNOWN: 07 04 05 13 <unknown>
 
-vlsh.v v8, (a0), a1, v0.t
-# CHECK-INST: vlsh.v v8, (a0), a1, v0.t
-# CHECK-ENCODING: [0x07,0x54,0xb5,0x18]
+vle256ff.v v8, (a0), v0.t
+# CHECK-INST: vle256ff.v v8, (a0), v0.t
+# CHECK-ENCODING: [0x07,0x54,0x05,0x11]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
-# CHECK-UNKNOWN: 07 54 b5 18 <unknown>
+# CHECK-UNKNOWN: 07 54 05 11 <unknown>
 
-vlsh.v v8, (a0), a1
-# CHECK-INST: vlsh.v v8, (a0), a1
-# CHECK-ENCODING: [0x07,0x54,0xb5,0x1a]
+vle256ff.v v8, (a0)
+# CHECK-INST: vle256ff.v v8, (a0)
+# CHECK-ENCODING: [0x07,0x54,0x05,0x13]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
-# CHECK-UNKNOWN: 07 54 b5 1a <unknown>
+# CHECK-UNKNOWN: 07 54 05 13 <unknown>
 
-vlsw.v v8, (a0), a1, v0.t
-# CHECK-INST: vlsw.v v8, (a0), a1, v0.t
-# CHECK-ENCODING: [0x07,0x64,0xb5,0x18]
+vle512ff.v v8, (a0), v0.t
+# CHECK-INST: vle512ff.v v8, (a0), v0.t
+# CHECK-ENCODING: [0x07,0x64,0x05,0x11]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
-# CHECK-UNKNOWN: 07 64 b5 18 <unknown>
+# CHECK-UNKNOWN: 07 64 05 11 <unknown>
 
-vlsw.v v8, (a0), a1
-# CHECK-INST: vlsw.v v8, (a0), a1
-# CHECK-ENCODING: [0x07,0x64,0xb5,0x1a]
+vle512ff.v v8, (a0)
+# CHECK-INST: vle512ff.v v8, (a0)
+# CHECK-ENCODING: [0x07,0x64,0x05,0x13]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
-# CHECK-UNKNOWN: 07 64 b5 1a <unknown>
+# CHECK-UNKNOWN: 07 64 05 13 <unknown>
+
+vle1024ff.v v8, (a0), v0.t
+# CHECK-INST: vle1024ff.v v8, (a0), v0.t
+# CHECK-ENCODING: [0x07,0x74,0x05,0x11]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 74 05 11 <unknown>
+
+vle1024ff.v v8, (a0)
+# CHECK-INST: vle1024ff.v v8, (a0)
+# CHECK-ENCODING: [0x07,0x74,0x05,0x13]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 74 05 13 <unknown>
 
-vlsbu.v v8, (a0), a1, v0.t
-# CHECK-INST: vlsbu.v v8, (a0), a1, v0.t
+vlse8.v v8, (a0), a1, v0.t
+# CHECK-INST: vlse8.v v8, (a0), a1, v0.t
 # CHECK-ENCODING: [0x07,0x04,0xb5,0x08]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 07 04 b5 08 <unknown>
 
-vlsbu.v v8, (a0), a1
-# CHECK-INST: vlsbu.v v8, (a0), a1
+vlse8.v v8, (a0), a1
+# CHECK-INST: vlse8.v v8, (a0), a1
 # CHECK-ENCODING: [0x07,0x04,0xb5,0x0a]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 07 04 b5 0a <unknown>
 
-vlshu.v v8, (a0), a1, v0.t
-# CHECK-INST: vlshu.v v8, (a0), a1, v0.t
+vlse16.v v8, (a0), a1, v0.t
+# CHECK-INST: vlse16.v v8, (a0), a1, v0.t
 # CHECK-ENCODING: [0x07,0x54,0xb5,0x08]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 07 54 b5 08 <unknown>
 
-vlshu.v v8, (a0), a1
-# CHECK-INST: vlshu.v v8, (a0), a1
+vlse16.v v8, (a0), a1
+# CHECK-INST: vlse16.v v8, (a0), a1
 # CHECK-ENCODING: [0x07,0x54,0xb5,0x0a]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 07 54 b5 0a <unknown>
 
-vlswu.v v8, (a0), a1, v0.t
-# CHECK-INST: vlswu.v v8, (a0), a1, v0.t
+vlse32.v v8, (a0), a1, v0.t
+# CHECK-INST: vlse32.v v8, (a0), a1, v0.t
 # CHECK-ENCODING: [0x07,0x64,0xb5,0x08]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 07 64 b5 08 <unknown>
 
-vlswu.v v8, (a0), a1
-# CHECK-INST: vlswu.v v8, (a0), a1
+vlse32.v v8, (a0), a1
+# CHECK-INST: vlse32.v v8, (a0), a1
 # CHECK-ENCODING: [0x07,0x64,0xb5,0x0a]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 07 64 b5 0a <unknown>
 
-vlse.v v8, (a0), a1, v0.t
-# CHECK-INST: vlse.v v8, (a0), a1, v0.t
+vlse64.v v8, (a0), a1, v0.t
+# CHECK-INST: vlse64.v v8, (a0), a1, v0.t
 # CHECK-ENCODING: [0x07,0x74,0xb5,0x08]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 07 74 b5 08 <unknown>
 
-vlse.v v8, (a0), a1
-# CHECK-INST: vlse.v v8, (a0), a1
+vlse64.v v8, (a0), a1
+# CHECK-INST: vlse64.v v8, (a0), a1
 # CHECK-ENCODING: [0x07,0x74,0xb5,0x0a]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 07 74 b5 0a <unknown>
 
-vlxb.v v8, (a0), v4, v0.t
-# CHECK-INST: vlxb.v v8, (a0), v4, v0.t
-# CHECK-ENCODING: [0x07,0x04,0x45,0x1c]
+vlse128.v v8, (a0), a1, v0.t
+# CHECK-INST: vlse128.v v8, (a0), a1, v0.t
+# CHECK-ENCODING: [0x07,0x04,0xb5,0x18]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
-# CHECK-UNKNOWN: 07 04 45 1c <unknown>
+# CHECK-UNKNOWN: 07 04 b5 18 <unknown>
 
-vlxb.v v8, (a0), v4
-# CHECK-INST: vlxb.v v8, (a0), v4
-# CHECK-ENCODING: [0x07,0x04,0x45,0x1e]
+vlse128.v v8, (a0), a1
+# CHECK-INST: vlse128.v v8, (a0), a1
+# CHECK-ENCODING: [0x07,0x04,0xb5,0x1a]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
-# CHECK-UNKNOWN: 07 04 45 1e <unknown>
+# CHECK-UNKNOWN: 07 04 b5 1a <unknown>
 
-vlxh.v v8, (a0), v4, v0.t
-# CHECK-INST: vlxh.v v8, (a0), v4, v0.t
-# CHECK-ENCODING: [0x07,0x54,0x45,0x1c]
+vlse256.v v8, (a0), a1, v0.t
+# CHECK-INST: vlse256.v v8, (a0), a1, v0.t
+# CHECK-ENCODING: [0x07,0x54,0xb5,0x18]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
-# CHECK-UNKNOWN: 07 54 45 1c <unknown>
+# CHECK-UNKNOWN: 07 54 b5 18 <unknown>
 
-vlxh.v v8, (a0), v4
-# CHECK-INST: vlxh.v v8, (a0), v4
-# CHECK-ENCODING: [0x07,0x54,0x45,0x1e]
+vlse256.v v8, (a0), a1
+# CHECK-INST: vlse256.v v8, (a0), a1
+# CHECK-ENCODING: [0x07,0x54,0xb5,0x1a]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
-# CHECK-UNKNOWN: 07 54 45 1e <unknown>
+# CHECK-UNKNOWN: 07 54 b5 1a <unknown>
 
-vlxw.v v8, (a0), v4, v0.t
-# CHECK-INST: vlxw.v v8, (a0), v4, v0.t
-# CHECK-ENCODING: [0x07,0x64,0x45,0x1c]
+vlse512.v v8, (a0), a1, v0.t
+# CHECK-INST: vlse512.v v8, (a0), a1, v0.t
+# CHECK-ENCODING: [0x07,0x64,0xb5,0x18]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
-# CHECK-UNKNOWN: 07 64 45 1c <unknown>
+# CHECK-UNKNOWN: 07 64 b5 18 <unknown>
 
-vlxw.v v8, (a0), v4
-# CHECK-INST: vlxw.v v8, (a0), v4
-# CHECK-ENCODING: [0x07,0x64,0x45,0x1e]
+vlse512.v v8, (a0), a1
+# CHECK-INST: vlse512.v v8, (a0), a1
+# CHECK-ENCODING: [0x07,0x64,0xb5,0x1a]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
-# CHECK-UNKNOWN: 07 64 45 1e <unknown>
+# CHECK-UNKNOWN: 07 64 b5 1a <unknown>
+
+vlse1024.v v8, (a0), a1, v0.t
+# CHECK-INST: vlse1024.v v8, (a0), a1, v0.t
+# CHECK-ENCODING: [0x07,0x74,0xb5,0x18]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 74 b5 18 <unknown>
 
-vlxbu.v v8, (a0), v4, v0.t
-# CHECK-INST: vlxbu.v v8, (a0), v4, v0.t
+vlse1024.v v8, (a0), a1
+# CHECK-INST: vlse1024.v v8, (a0), a1
+# CHECK-ENCODING: [0x07,0x74,0xb5,0x1a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 74 b5 1a <unknown>
+
+vlxei8.v v8, (a0), v4, v0.t
+# CHECK-INST: vlxei8.v v8, (a0), v4, v0.t
 # CHECK-ENCODING: [0x07,0x04,0x45,0x0c]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 07 04 45 0c <unknown>
 
-vlxbu.v v8, (a0), v4
-# CHECK-INST: vlxbu.v v8, (a0), v4
+vlxei8.v v8, (a0), v4
+# CHECK-INST: vlxei8.v v8, (a0), v4
 # CHECK-ENCODING: [0x07,0x04,0x45,0x0e]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 07 04 45 0e <unknown>
 
-vlxhu.v v8, (a0), v4, v0.t
-# CHECK-INST: vlxhu.v v8, (a0), v4, v0.t
+vlxei16.v v8, (a0), v4, v0.t
+# CHECK-INST: vlxei16.v v8, (a0), v4, v0.t
 # CHECK-ENCODING: [0x07,0x54,0x45,0x0c]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 07 54 45 0c <unknown>
 
-vlxhu.v v8, (a0), v4
-# CHECK-INST: vlxhu.v v8, (a0), v4
+vlxei16.v v8, (a0), v4
+# CHECK-INST: vlxei16.v v8, (a0), v4
 # CHECK-ENCODING: [0x07,0x54,0x45,0x0e]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 07 54 45 0e <unknown>
 
-vlxwu.v v8, (a0), v4, v0.t
-# CHECK-INST: vlxwu.v v8, (a0), v4, v0.t
+vlxei32.v v8, (a0), v4, v0.t
+# CHECK-INST: vlxei32.v v8, (a0), v4, v0.t
 # CHECK-ENCODING: [0x07,0x64,0x45,0x0c]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 07 64 45 0c <unknown>
 
-vlxwu.v v8, (a0), v4
-# CHECK-INST: vlxwu.v v8, (a0), v4
+vlxei32.v v8, (a0), v4
+# CHECK-INST: vlxei32.v v8, (a0), v4
 # CHECK-ENCODING: [0x07,0x64,0x45,0x0e]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 07 64 45 0e <unknown>
 
-vlxe.v v8, (a0), v4, v0.t
-# CHECK-INST: vlxe.v v8, (a0), v4, v0.t
+vlxei64.v v8, (a0), v4, v0.t
+# CHECK-INST: vlxei64.v v8, (a0), v4, v0.t
 # CHECK-ENCODING: [0x07,0x74,0x45,0x0c]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 07 74 45 0c <unknown>
 
-vlxe.v v8, (a0), v4
-# CHECK-INST: vlxe.v v8, (a0), v4
+vlxei64.v v8, (a0), v4
+# CHECK-INST: vlxei64.v v8, (a0), v4
 # CHECK-ENCODING: [0x07,0x74,0x45,0x0e]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 07 74 45 0e <unknown>
 
+vlxei128.v v8, (a0), v4, v0.t
+# CHECK-INST: vlxei128.v v8, (a0), v4, v0.t
+# CHECK-ENCODING: [0x07,0x04,0x45,0x1c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 04 45 1c <unknown>
+
+vlxei128.v v8, (a0), v4
+# CHECK-INST: vlxei128.v v8, (a0), v4
+# CHECK-ENCODING: [0x07,0x04,0x45,0x1e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 04 45 1e <unknown>
+
+vlxei256.v v8, (a0), v4, v0.t
+# CHECK-INST: vlxei256.v v8, (a0), v4, v0.t
+# CHECK-ENCODING: [0x07,0x54,0x45,0x1c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 54 45 1c <unknown>
+
+vlxei256.v v8, (a0), v4
+# CHECK-INST: vlxei256.v v8, (a0), v4
+# CHECK-ENCODING: [0x07,0x54,0x45,0x1e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 54 45 1e <unknown>
+
+vlxei512.v v8, (a0), v4, v0.t
+# CHECK-INST: vlxei512.v v8, (a0), v4, v0.t
+# CHECK-ENCODING: [0x07,0x64,0x45,0x1c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 64 45 1c <unknown>
+
+vlxei512.v v8, (a0), v4
+# CHECK-INST: vlxei512.v v8, (a0), v4
+# CHECK-ENCODING: [0x07,0x64,0x45,0x1e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 64 45 1e <unknown>
+
+vlxei1024.v v8, (a0), v4, v0.t
+# CHECK-INST: vlxei1024.v v8, (a0), v4, v0.t
+# CHECK-ENCODING: [0x07,0x74,0x45,0x1c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 74 45 1c <unknown>
+
+vlxei1024.v v8, (a0), v4
+# CHECK-INST: vlxei1024.v v8, (a0), v4
+# CHECK-ENCODING: [0x07,0x74,0x45,0x1e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 07 74 45 1e <unknown>
+
 vl1r.v v8, (a0)
 # CHECK-INST: vl1r.v v8, (a0)
-# CHECK-ENCODING: [0x07,0x74,0x85,0x02]
+# CHECK-ENCODING: [0x07,0x04,0x85,0x02]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
-# CHECK-UNKNOWN: 07 74 85 02 <unknown>
+# CHECK-UNKNOWN: 07 04 85 02 <unknown>
diff --git a/llvm/test/MC/RISCV/rvv/mask.s b/llvm/test/MC/RISCV/rvv/mask.s
index d2157bc215bf3..ef029388da9c3 100644
--- a/llvm/test/MC/RISCV/rvv/mask.s
+++ b/llvm/test/MC/RISCV/rvv/mask.s
@@ -140,8 +140,8 @@ vid.v v8
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 57 a4 08 52 <unknown>
 
-vmcpy.m v8, v4
-# CHECK-INST: vmcpy.m v8, v4
+vmmv.m v8, v4
+# CHECK-INST: vmmv.m v8, v4
 # CHECK-ENCODING: [0x57,0x24,0x42,0x66]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 57 24 42 66 <unknown>
diff --git a/llvm/test/MC/RISCV/rvv/reduction.s b/llvm/test/MC/RISCV/rvv/reduction.s
index dc5adec0a5668..7599c603363f4 100644
--- a/llvm/test/MC/RISCV/rvv/reduction.s
+++ b/llvm/test/MC/RISCV/rvv/reduction.s
@@ -127,3 +127,9 @@ vwredsum.vs v8, v4, v20
 # CHECK-ENCODING: [0x57,0x04,0x4a,0xc6]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 57 04 4a c6 <unknown>
+
+vredsum.vs v0, v4, v20, v0.t
+# CHECK-INST: vredsum.vs v0, v4, v20, v0.t
+# CHECK-ENCODING: [0x57,0x20,0x4a,0x00]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 20 4a 00 <unknown>
diff --git a/llvm/test/MC/RISCV/rvv/snippet.s b/llvm/test/MC/RISCV/rvv/snippet.s
index fd38631c4ba7e..94f6556ea10c5 100644
--- a/llvm/test/MC/RISCV/rvv/snippet.s
+++ b/llvm/test/MC/RISCV/rvv/snippet.s
@@ -5,28 +5,28 @@
 # RUN: | FileCheck %s --check-prefix=CHECK-INST
 
 loop:
-    vsetvli a3, a0, e16,m4  # vtype = 16-bit integer vectors
-# CHECK-INST: d7 76 65 00    vsetvli a3, a0, e16,m4
-    vlh.v v4, (a1)          # Get 16b vector
-# CHECK-INST: 07 d2 05 12    vlh.v   v4, (a1)
-    slli t1, a3, 1          # Multiply length by two bytes/element
+    vsetvli a3, a0, e16,m4,ta,ma  # vtype = 16-bit integer vectors
+# CHECK-INST: d7 76 65 0c    vsetvli a3, a0, e16,m4,ta,ma
+    vle16.v v4, (a1)              # Get 16b vector
+# CHECK-INST: 07 d2 05 02    vle16.v   v4, (a1)
+    slli t1, a3, 1                # Multiply length by two bytes/element
 # CHECK-INST: 13 93 16 00    slli    t1, a3, 1
-    add a1, a1, t1          # Bump pointer
+    add a1, a1, t1                # Bump pointer
 # CHECK-INST: b3 85 65 00    add     a1, a1, t1
-    vwmul.vx v8, v4, x10    # 32b in <v8--v15>
+    vwmul.vx v8, v4, x10          # 32b in <v8--v15>
 # CHECK-INST: 57 64 45 ee    vwmul.vx        v8, v4, a0
 
-    vsetvli x0, a0, e32,m8  # Operate on 32b values
-# CHECK-INST: 57 70 b5 00    vsetvli zero, a0, e32,m8
+    vsetvli x0, a0, e32,m8,ta,ma  # Operate on 32b values
+# CHECK-INST: 57 70 b5 0c    vsetvli zero, a0, e32,m8,ta,ma
     vsrl.vi v8, v8, 3
 # CHECK-INST: 57 b4 81 a2    vsrl.vi v8, v8, 3
-    vsw.v v8, (a2)          # Store vector of 32b
-# CHECK-INST: 27 64 06 02    vsw.v   v8, (a2)
-    slli t1, a3, 2          # Multiply length by four bytes/element
+    vse32.v v8, (a2)              # Store vector of 32b
+# CHECK-INST: 27 64 06 02    vse32.v   v8, (a2)
+    slli t1, a3, 2                # Multiply length by four bytes/element
 # CHECK-INST: 13 93 26 00    slli    t1, a3, 2
-    add a2, a2, t1          # Bump pointer
+    add a2, a2, t1                # Bump pointer
 # CHECK-INST: 33 06 66 00    add     a2, a2, t1
-    sub a0, a0, a3          # Decrement count
+    sub a0, a0, a3                # Decrement count
 # CHECK-INST: 33 05 d5 40    sub     a0, a0, a3
-    bnez a0, loop           # Any more?
+    bnez a0, loop                 # Any more?
 # CHECK-INST: e3 1a 05 fc    bnez    a0, -44
diff --git a/llvm/test/MC/RISCV/rvv/store.s b/llvm/test/MC/RISCV/rvv/store.s
index 9d25b593ef824..75dacb9d19e5d 100644
--- a/llvm/test/MC/RISCV/rvv/store.s
+++ b/llvm/test/MC/RISCV/rvv/store.s
@@ -8,200 +8,296 @@
 # RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
 # RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
 
-vsb.v v24, (a0), v0.t
-# CHECK-INST: vsb.v v24, (a0), v0.t
+vse8.v v24, (a0), v0.t
+# CHECK-INST: vse8.v v24, (a0), v0.t
 # CHECK-ENCODING: [0x27,0x0c,0x05,0x00]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 27 0c 05 00 <unknown>
 
-vsb.v v24, (a0)
-# CHECK-INST: vsb.v v24, (a0)
+vse8.v v24, (a0)
+# CHECK-INST: vse8.v v24, (a0)
 # CHECK-ENCODING: [0x27,0x0c,0x05,0x02]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 27 0c 05 02 <unknown>
 
-vsh.v v24, (a0), v0.t
-# CHECK-INST: vsh.v v24, (a0), v0.t
+vse16.v v24, (a0), v0.t
+# CHECK-INST: vse16.v v24, (a0), v0.t
 # CHECK-ENCODING: [0x27,0x5c,0x05,0x00]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 27 5c 05 00 <unknown>
 
-vsh.v v24, (a0)
-# CHECK-INST: vsh.v v24, (a0)
+vse16.v v24, (a0)
+# CHECK-INST: vse16.v v24, (a0)
 # CHECK-ENCODING: [0x27,0x5c,0x05,0x02]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 27 5c 05 02 <unknown>
 
-vsw.v v24, (a0), v0.t
-# CHECK-INST: vsw.v v24, (a0), v0.t
+vse32.v v24, (a0), v0.t
+# CHECK-INST: vse32.v v24, (a0), v0.t
 # CHECK-ENCODING: [0x27,0x6c,0x05,0x00]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 27 6c 05 00 <unknown>
 
-vsw.v v24, (a0)
-# CHECK-INST: vsw.v v24, (a0)
+vse32.v v24, (a0)
+# CHECK-INST: vse32.v v24, (a0)
 # CHECK-ENCODING: [0x27,0x6c,0x05,0x02]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 27 6c 05 02 <unknown>
 
-vse.v v24, (a0), v0.t
-# CHECK-INST: vse.v v24, (a0), v0.t
+vse64.v v24, (a0), v0.t
+# CHECK-INST: vse64.v v24, (a0), v0.t
 # CHECK-ENCODING: [0x27,0x7c,0x05,0x00]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 27 7c 05 00 <unknown>
 
-vse.v v24, (a0)
-# CHECK-INST: vse.v v24, (a0)
+vse64.v v24, (a0)
+# CHECK-INST: vse64.v v24, (a0)
 # CHECK-ENCODING: [0x27,0x7c,0x05,0x02]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 27 7c 05 02 <unknown>
 
-vssb.v v24, (a0), a1, v0.t
-# CHECK-INST: vssb.v v24, (a0), a1, v0.t
+vse128.v v24, (a0), v0.t
+# CHECK-INST: vse128.v v24, (a0), v0.t
+# CHECK-ENCODING: [0x27,0x0c,0x05,0x10]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 27 0c 05 10 <unknown>
+
+vse128.v v24, (a0)
+# CHECK-INST: vse128.v v24, (a0)
+# CHECK-ENCODING: [0x27,0x0c,0x05,0x12]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 27 0c 05 12 <unknown>
+
+vse256.v v24, (a0), v0.t
+# CHECK-INST: vse256.v v24, (a0), v0.t
+# CHECK-ENCODING: [0x27,0x5c,0x05,0x10]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 27 5c 05 10 <unknown>
+
+vse256.v v24, (a0)
+# CHECK-INST: vse256.v v24, (a0)
+# CHECK-ENCODING: [0x27,0x5c,0x05,0x12]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 27 5c 05 12 <unknown>
+
+vse512.v v24, (a0), v0.t
+# CHECK-INST: vse512.v v24, (a0), v0.t
+# CHECK-ENCODING: [0x27,0x6c,0x05,0x10]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 27 6c 05 10 <unknown>
+
+vse512.v v24, (a0)
+# CHECK-INST: vse512.v v24, (a0)
+# CHECK-ENCODING: [0x27,0x6c,0x05,0x12]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 27 6c 05 12 <unknown>
+
+vse1024.v v24, (a0), v0.t
+# CHECK-INST: vse1024.v v24, (a0), v0.t
+# CHECK-ENCODING: [0x27,0x7c,0x05,0x10]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 27 7c 05 10 <unknown>
+
+vse1024.v v24, (a0)
+# CHECK-INST: vse1024.v v24, (a0)
+# CHECK-ENCODING: [0x27,0x7c,0x05,0x12]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 27 7c 05 12 <unknown>
+
+vsse8.v v24, (a0), a1, v0.t
+# CHECK-INST: vsse8.v v24, (a0), a1, v0.t
 # CHECK-ENCODING: [0x27,0x0c,0xb5,0x08]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 27 0c b5 08 <unknown>
 
-vssb.v v24, (a0), a1
-# CHECK-INST: vssb.v v24, (a0), a1
+vsse8.v v24, (a0), a1
+# CHECK-INST: vsse8.v v24, (a0), a1
 # CHECK-ENCODING: [0x27,0x0c,0xb5,0x0a]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 27 0c b5 0a <unknown>
 
-vssh.v v24, (a0), a1, v0.t
-# CHECK-INST: vssh.v v24, (a0), a1, v0.t
+vsse16.v v24, (a0), a1, v0.t
+# CHECK-INST: vsse16.v v24, (a0), a1, v0.t
 # CHECK-ENCODING: [0x27,0x5c,0xb5,0x08]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 27 5c b5 08 <unknown>
 
-vssh.v v24, (a0), a1
-# CHECK-INST: vssh.v v24, (a0), a1
+vsse16.v v24, (a0), a1
+# CHECK-INST: vsse16.v v24, (a0), a1
 # CHECK-ENCODING: [0x27,0x5c,0xb5,0x0a]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 27 5c b5 0a <unknown>
 
-vssw.v v24, (a0), a1, v0.t
-# CHECK-INST: vssw.v v24, (a0), a1, v0.t
+vsse32.v v24, (a0), a1, v0.t
+# CHECK-INST: vsse32.v v24, (a0), a1, v0.t
 # CHECK-ENCODING: [0x27,0x6c,0xb5,0x08]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 27 6c b5 08 <unknown>
 
-vssw.v v24, (a0), a1
-# CHECK-INST: vssw.v v24, (a0), a1
+vsse32.v v24, (a0), a1
+# CHECK-INST: vsse32.v v24, (a0), a1
 # CHECK-ENCODING: [0x27,0x6c,0xb5,0x0a]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 27 6c b5 0a <unknown>
 
-vsse.v v24, (a0), a1, v0.t
-# CHECK-INST: vsse.v v24, (a0), a1, v0.t
+vsse64.v v24, (a0), a1, v0.t
+# CHECK-INST: vsse64.v v24, (a0), a1, v0.t
 # CHECK-ENCODING: [0x27,0x7c,0xb5,0x08]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 27 7c b5 08 <unknown>
 
-vsse.v v24, (a0), a1
-# CHECK-INST: vsse.v v24, (a0), a1
+vsse64.v v24, (a0), a1
+# CHECK-INST: vsse64.v v24, (a0), a1
 # CHECK-ENCODING: [0x27,0x7c,0xb5,0x0a]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 27 7c b5 0a <unknown>
 
-vsxb.v v24, (a0), v4, v0.t
-# CHECK-INST: vsxb.v v24, (a0), v4, v0.t
+vsse128.v v24, (a0), a1, v0.t
+# CHECK-INST: vsse128.v v24, (a0), a1, v0.t
+# CHECK-ENCODING: [0x27,0x0c,0xb5,0x18]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 27 0c b5 18 <unknown>
+
+vsse128.v v24, (a0), a1
+# CHECK-INST: vsse128.v v24, (a0), a1
+# CHECK-ENCODING: [0x27,0x0c,0xb5,0x1a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 27 0c b5 1a <unknown>
+
+vsse256.v v24, (a0), a1, v0.t
+# CHECK-INST: vsse256.v v24, (a0), a1, v0.t
+# CHECK-ENCODING: [0x27,0x5c,0xb5,0x18]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 27 5c b5 18 <unknown>
+
+vsse256.v v24, (a0), a1
+# CHECK-INST: vsse256.v v24, (a0), a1
+# CHECK-ENCODING: [0x27,0x5c,0xb5,0x1a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 27 5c b5 1a <unknown>
+
+vsse512.v v24, (a0), a1, v0.t
+# CHECK-INST: vsse512.v v24, (a0), a1, v0.t
+# CHECK-ENCODING: [0x27,0x6c,0xb5,0x18]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 27 6c b5 18 <unknown>
+
+vsse512.v v24, (a0), a1
+# CHECK-INST: vsse512.v v24, (a0), a1
+# CHECK-ENCODING: [0x27,0x6c,0xb5,0x1a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 27 6c b5 1a <unknown>
+
+vsse1024.v v24, (a0), a1, v0.t
+# CHECK-INST: vsse1024.v v24, (a0), a1, v0.t
+# CHECK-ENCODING: [0x27,0x7c,0xb5,0x18]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 27 7c b5 18 <unknown>
+
+vsse1024.v v24, (a0), a1
+# CHECK-INST: vsse1024.v v24, (a0), a1
+# CHECK-ENCODING: [0x27,0x7c,0xb5,0x1a]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 27 7c b5 1a <unknown>
+
+vsxei8.v v24, (a0), v4, v0.t
+# CHECK-INST: vsxei8.v v24, (a0), v4, v0.t
 # CHECK-ENCODING: [0x27,0x0c,0x45,0x0c]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 27 0c 45 0c <unknown>
 
-vsxb.v v24, (a0), v4
-# CHECK-INST: vsxb.v v24, (a0), v4
+vsxei8.v v24, (a0), v4
+# CHECK-INST: vsxei8.v v24, (a0), v4
 # CHECK-ENCODING: [0x27,0x0c,0x45,0x0e]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 27 0c 45 0e <unknown>
 
-vsxh.v v24, (a0), v4, v0.t
-# CHECK-INST: vsxh.v v24, (a0), v4, v0.t
+vsxei16.v v24, (a0), v4, v0.t
+# CHECK-INST: vsxei16.v v24, (a0), v4, v0.t
 # CHECK-ENCODING: [0x27,0x5c,0x45,0x0c]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 27 5c 45 0c <unknown>
 
-vsxh.v v24, (a0), v4
-# CHECK-INST: vsxh.v v24, (a0), v4
+vsxei16.v v24, (a0), v4
+# CHECK-INST: vsxei16.v v24, (a0), v4
 # CHECK-ENCODING: [0x27,0x5c,0x45,0x0e]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 27 5c 45 0e <unknown>
 
-vsxw.v v24, (a0), v4, v0.t
-# CHECK-INST: vsxw.v v24, (a0), v4, v0.t
+vsxei32.v v24, (a0), v4, v0.t
+# CHECK-INST: vsxei32.v v24, (a0), v4, v0.t
 # CHECK-ENCODING: [0x27,0x6c,0x45,0x0c]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 27 6c 45 0c <unknown>
 
-vsxw.v v24, (a0), v4
-# CHECK-INST: vsxw.v v24, (a0), v4
+vsxei32.v v24, (a0), v4
+# CHECK-INST: vsxei32.v v24, (a0), v4
 # CHECK-ENCODING: [0x27,0x6c,0x45,0x0e]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 27 6c 45 0e <unknown>
 
-vsxe.v v24, (a0), v4, v0.t
-# CHECK-INST: vsxe.v v24, (a0), v4, v0.t
+vsxei64.v v24, (a0), v4, v0.t
+# CHECK-INST: vsxei64.v v24, (a0), v4, v0.t
 # CHECK-ENCODING: [0x27,0x7c,0x45,0x0c]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 27 7c 45 0c <unknown>
 
-vsxe.v v24, (a0), v4
-# CHECK-INST: vsxe.v v24, (a0), v4
+vsxei64.v v24, (a0), v4
+# CHECK-INST: vsxei64.v v24, (a0), v4
 # CHECK-ENCODING: [0x27,0x7c,0x45,0x0e]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 27 7c 45 0e <unknown>
 
-vsuxb.v v24, (a0), v4, v0.t
-# CHECK-INST: vsuxb.v v24, (a0), v4, v0.t
+vsxei128.v v24, (a0), v4, v0.t
+# CHECK-INST: vsxei128.v v24, (a0), v4, v0.t
 # CHECK-ENCODING: [0x27,0x0c,0x45,0x1c]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 27 0c 45 1c <unknown>
 
-vsuxb.v v24, (a0), v4
-# CHECK-INST: vsuxb.v v24, (a0), v4
+vsxei128.v v24, (a0), v4
+# CHECK-INST: vsxei128.v v24, (a0), v4
 # CHECK-ENCODING: [0x27,0x0c,0x45,0x1e]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 27 0c 45 1e <unknown>
 
-vsuxh.v v24, (a0), v4, v0.t
-# CHECK-INST: vsuxh.v v24, (a0), v4, v0.t
+vsxei256.v v24, (a0), v4, v0.t
+# CHECK-INST: vsxei256.v v24, (a0), v4, v0.t
 # CHECK-ENCODING: [0x27,0x5c,0x45,0x1c]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 27 5c 45 1c <unknown>
 
-vsuxh.v v24, (a0), v4
-# CHECK-INST: vsuxh.v v24, (a0), v4
+vsxei256.v v24, (a0), v4
+# CHECK-INST: vsxei256.v v24, (a0), v4
 # CHECK-ENCODING: [0x27,0x5c,0x45,0x1e]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 27 5c 45 1e <unknown>
 
-vsuxw.v v24, (a0), v4, v0.t
-# CHECK-INST: vsuxw.v v24, (a0), v4, v0.t
+vsxei512.v v24, (a0), v4, v0.t
+# CHECK-INST: vsxei512.v v24, (a0), v4, v0.t
 # CHECK-ENCODING: [0x27,0x6c,0x45,0x1c]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 27 6c 45 1c <unknown>
 
-vsuxw.v v24, (a0), v4
-# CHECK-INST: vsuxw.v v24, (a0), v4
+vsxei512.v v24, (a0), v4
+# CHECK-INST: vsxei512.v v24, (a0), v4
 # CHECK-ENCODING: [0x27,0x6c,0x45,0x1e]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 27 6c 45 1e <unknown>
 
-vsuxe.v v24, (a0), v4, v0.t
-# CHECK-INST: vsuxe.v v24, (a0), v4, v0.t
+vsxei1024.v v24, (a0), v4, v0.t
+# CHECK-INST: vsxei1024.v v24, (a0), v4, v0.t
 # CHECK-ENCODING: [0x27,0x7c,0x45,0x1c]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 27 7c 45 1c <unknown>
 
-vsuxe.v v24, (a0), v4
-# CHECK-INST: vsuxe.v v24, (a0), v4
+vsxei1024.v v24, (a0), v4
+# CHECK-INST: vsxei1024.v v24, (a0), v4
 # CHECK-ENCODING: [0x27,0x7c,0x45,0x1e]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
 # CHECK-UNKNOWN: 27 7c 45 1e <unknown>
 
 vs1r.v v24, (a0)
 # CHECK-INST: vs1r.v v24, (a0)
-# CHECK-ENCODING: [0x27,0x7c,0x85,0x02]
+# CHECK-ENCODING: [0x27,0x0c,0x85,0x02]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
-# CHECK-UNKNOWN: 27 7c 85 02 <unknown>
+# CHECK-UNKNOWN: 27 0c 85 02 <unknown>
diff --git a/llvm/test/MC/RISCV/rvv/vsetvl.s b/llvm/test/MC/RISCV/rvv/vsetvl.s
index 784533445f0c6..351d3febdf039 100644
--- a/llvm/test/MC/RISCV/rvv/vsetvl.s
+++ b/llvm/test/MC/RISCV/rvv/vsetvl.s
@@ -8,11 +8,71 @@
 # RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-v %s \
 # RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
 
-vsetvli a2, a0, e32,m4
-# CHECK-INST: vsetvli a2, a0, e32,m4
-# CHECK-ENCODING: [0x57,0x76,0xa5,0x00]
+vsetvli a2, a0, e32,m1,ta,ma
+# CHECK-INST: vsetvli a2, a0, e32,m1,ta,ma
+# CHECK-ENCODING: [0x57,0x76,0x85,0x0c]
 # CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
-# CHECK-UNKNOWN: 57 76 a5 00 <unknown>
+# CHECK-UNKNOWN: 57 76 85 0c <unknown>
+
+vsetvli a2, a0, e32,m2,ta,ma
+# CHECK-INST: vsetvli a2, a0, e32,m2,ta,ma
+# CHECK-ENCODING: [0x57,0x76,0x95,0x0c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 76 95 0c <unknown>
+
+vsetvli a2, a0, e32,m4,ta,ma
+# CHECK-INST: vsetvli a2, a0, e32,m4,ta,ma
+# CHECK-ENCODING: [0x57,0x76,0xa5,0x0c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 76 a5 0c <unknown>
+
+vsetvli a2, a0, e32,m8,ta,ma
+# CHECK-INST: vsetvli a2, a0, e32,m8,ta,ma
+# CHECK-ENCODING: [0x57,0x76,0xb5,0x0c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 76 b5 0c <unknown>
+
+vsetvli a2, a0, e32,mf2,ta,ma
+# CHECK-INST: vsetvli a2, a0, e32,mf2,ta,ma
+# CHECK-ENCODING: [0x57,0x76,0xb5,0x0e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 76 b5 0e <unknown>
+
+vsetvli a2, a0, e32,mf4,ta,ma
+# CHECK-INST: vsetvli a2, a0, e32,mf4,ta,ma
+# CHECK-ENCODING: [0x57,0x76,0xa5,0x0e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 76 a5 0e <unknown>
+
+vsetvli a2, a0, e32,mf8,ta,ma
+# CHECK-INST: vsetvli a2, a0, e32,mf8,ta,ma
+# CHECK-ENCODING: [0x57,0x76,0x95,0x0e]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 76 95 0e <unknown>
+
+vsetvli a2, a0, e32,m1,ta,ma
+# CHECK-INST: vsetvli a2, a0, e32,m1,ta,ma
+# CHECK-ENCODING: [0x57,0x76,0x85,0x0c]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 76 85 0c <unknown>
+
+vsetvli a2, a0, e32,m1,tu,ma
+# CHECK-INST: vsetvli a2, a0, e32,m1,tu,ma
+# CHECK-ENCODING: [0x57,0x76,0x85,0x08]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 76 85 08 <unknown>
+
+vsetvli a2, a0, e32,m1,ta,mu
+# CHECK-INST: vsetvli a2, a0, e32,m1,ta,mu
+# CHECK-ENCODING: [0x57,0x76,0x85,0x04]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 76 85 04 <unknown>
+
+vsetvli a2, a0, e32,m1,tu,mu
+# CHECK-INST: vsetvli a2, a0, e32,m1
+# CHECK-ENCODING: [0x57,0x76,0x85,0x00]
+# CHECK-ERROR: instruction requires the following: 'V' (Vector Instructions)
+# CHECK-UNKNOWN: 57 76 85 00 <unknown>
 
 vsetvl a2, a0, a1
 # CHECK-INST: vsetvl a2, a0, a1

From 721d93fc5aa8c9f9fc9b86a9d3d1a58c6790213e Mon Sep 17 00:00:00 2001
From: Hsiangkai Wang <kai.wang@sifive.com>
Date: Fri, 5 Jun 2020 03:34:01 +0800
Subject: [PATCH 0792/1035] Support experimental v extension v0.9.

Differential revision: https://reviews.llvm.org/D81213
---
 clang/lib/Driver/ToolChains/Arch/RISCV.cpp | 2 +-
 clang/test/Driver/riscv-arch.c             | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/clang/lib/Driver/ToolChains/Arch/RISCV.cpp b/clang/lib/Driver/ToolChains/Arch/RISCV.cpp
index 09ae4538b3acc..7ca05a1f3a395 100644
--- a/clang/lib/Driver/ToolChains/Arch/RISCV.cpp
+++ b/clang/lib/Driver/ToolChains/Arch/RISCV.cpp
@@ -63,7 +63,7 @@ isExperimentalExtension(StringRef Ext) {
       Ext == "zbs" || Ext == "zbt" || Ext == "zbproposedc")
     return RISCVExtensionVersion{"0", "92"};
   if (Ext == "v")
-    return RISCVExtensionVersion{"0", "8"};
+    return RISCVExtensionVersion{"0", "9"};
   return None;
 }
 
diff --git a/clang/test/Driver/riscv-arch.c b/clang/test/Driver/riscv-arch.c
index 725201a77ba7c..8b630b1846c97 100644
--- a/clang/test/Driver/riscv-arch.c
+++ b/clang/test/Driver/riscv-arch.c
@@ -380,6 +380,6 @@
 // RV32-EXPERIMENTAL-V-BADVERS: error: invalid arch name 'rv32iv0p1'
 // RV32-EXPERIMENTAL-V-BADVERS: unsupported version number 0.1 for experimental extension
 
-// RUN: %clang -target riscv32-unknown-elf -march=rv32iv0p8 -menable-experimental-extensions -### %s -c 2>&1 | \
+// RUN: %clang -target riscv32-unknown-elf -march=rv32iv0p9 -menable-experimental-extensions -### %s -c 2>&1 | \
 // RUN:   FileCheck -check-prefix=RV32-EXPERIMENTAL-V-GOODVERS %s
 // RV32-EXPERIMENTAL-V-GOODVERS: "-target-feature" "+experimental-v"

From ff756f5231cc2ee9457129404e78420fa2791c7b Mon Sep 17 00:00:00 2001
From: Julian Lettner <julian.lettner@apple.com>
Date: Fri, 31 Jul 2020 16:09:13 -0700
Subject: [PATCH 0793/1035] [compiler-rt][Darwin] Fix linker errors for
 check-asan

A recent change broke `ninja check-asan` on Darwin by causing an error
during linking of ASan unit tests [1].

Move the addition of `-ObjC` compiler flag outside of the new
`if(COMPILER_RT_STANDALONE_BUILD)` block.  It doesn't add any global
flags (e.g, `${CMAKE_CXX_FLAGS}`) and the decision to add is based
solely on source paths (`${source_rpath}`).

[1] 8b2fcc42b895, https://reviews.llvm.org/D84466

Differential Revision: https://reviews.llvm.org/D85057
---
 compiler-rt/cmake/Modules/CompilerRTCompile.cmake | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/compiler-rt/cmake/Modules/CompilerRTCompile.cmake b/compiler-rt/cmake/Modules/CompilerRTCompile.cmake
index 3330038f80688..0b679dbf68fae 100644
--- a/compiler-rt/cmake/Modules/CompilerRTCompile.cmake
+++ b/compiler-rt/cmake/Modules/CompilerRTCompile.cmake
@@ -73,7 +73,6 @@ function(clang_compile object_file source)
   if(COMPILER_RT_STANDALONE_BUILD)
     # Only add global flags in standalone build.
     string(REGEX MATCH "[.](cc|cpp)$" is_cxx ${source_rpath})
-    string(REGEX MATCH "[.](m|mm)$" is_objc ${source_rpath})
     if(is_cxx)
       string(REPLACE " " ";" global_flags "${CMAKE_CXX_FLAGS}")
     else()
@@ -87,9 +86,6 @@ function(clang_compile object_file source)
     if (APPLE)
       set(global_flags ${OSX_SYSROOT_FLAG} ${global_flags})
     endif()
-    if (is_objc)
-      list(APPEND global_flags -ObjC)
-    endif()
 
     # Ignore unknown warnings. CMAKE_CXX_FLAGS may contain GCC-specific options
     # which are not supported by Clang.
@@ -98,6 +94,12 @@ function(clang_compile object_file source)
   else()
     set(compile_flags ${SOURCE_CFLAGS})
   endif()
+
+  string(REGEX MATCH "[.](m|mm)$" is_objc ${source_rpath})
+  if (is_objc)
+    list(APPEND compile_flags "-ObjC")
+  endif()
+
   add_custom_command(
     OUTPUT ${object_file}
     COMMAND ${COMPILER_RT_TEST_COMPILER} ${compile_flags} -c

From 5954755939febabcf1edb52b53214f25f06ce584 Mon Sep 17 00:00:00 2001
From: cgyurgyik <gyurgyikcp@gmail.com>
Date: Fri, 31 Jul 2020 19:59:29 -0400
Subject: [PATCH 0794/1035] [libc] [obvious] Fix strchr and strrchr tests so
 that constness is actually verified.

---
 libc/test/src/string/strchr_test.cpp  | 19 +++++++------------
 libc/test/src/string/strrchr_test.cpp | 12 ++++--------
 2 files changed, 11 insertions(+), 20 deletions(-)

diff --git a/libc/test/src/string/strchr_test.cpp b/libc/test/src/string/strchr_test.cpp
index 37b3733857997..dda930c0c1552 100644
--- a/libc/test/src/string/strchr_test.cpp
+++ b/libc/test/src/string/strchr_test.cpp
@@ -11,42 +11,38 @@
 
 TEST(StrChrTest, FindsFirstCharacter) {
   const char *src = "abcde";
-  const char *src_copy = src;
 
   // Should return original string since 'a' is the first character.
   ASSERT_STREQ(__llvm_libc::strchr(src, 'a'), "abcde");
   // Source string should not change.
-  ASSERT_STREQ(src, src_copy);
+  ASSERT_STREQ(src, "abcde");
 }
 
 TEST(StrChrTest, FindsMiddleCharacter) {
   const char *src = "abcde";
-  const char *src_copy = src;
 
   // Should return characters after (and including) 'c'.
   ASSERT_STREQ(__llvm_libc::strchr(src, 'c'), "cde");
   // Source string should not change.
-  ASSERT_STREQ(src, src_copy);
+  ASSERT_STREQ(src, "abcde");
 }
 
 TEST(StrChrTest, FindsLastCharacterThatIsNotNullTerminator) {
   const char *src = "abcde";
-  const char *src_copy = src;
 
   // Should return 'e' and null-terminator.
   ASSERT_STREQ(__llvm_libc::strchr(src, 'e'), "e");
   // Source string should not change.
-  ASSERT_STREQ(src, src_copy);
+  ASSERT_STREQ(src, "abcde");
 }
 
 TEST(StrChrTest, FindsNullTerminator) {
   const char *src = "abcde";
-  const char *src_copy = src;
 
   // Should return null terminator.
   ASSERT_STREQ(__llvm_libc::strchr(src, '\0'), "");
   // Source string should not change.
-  ASSERT_STREQ(src, src_copy);
+  ASSERT_STREQ(src, "abcde");
 }
 
 TEST(StrChrTest, CharacterNotWithinStringShouldReturnNullptr) {
@@ -56,16 +52,15 @@ TEST(StrChrTest, CharacterNotWithinStringShouldReturnNullptr) {
 
 TEST(StrChrTest, TheSourceShouldNotChange) {
   const char *src = "abcde";
-  const char *src_copy = src;
   // When the character is found, the source string should not change.
   __llvm_libc::strchr(src, 'd');
-  ASSERT_STREQ(src, src_copy);
+  ASSERT_STREQ(src, "abcde");
   // Same case for when the character is not found.
   __llvm_libc::strchr(src, 'z');
-  ASSERT_STREQ(src, src_copy);
+  ASSERT_STREQ(src, "abcde");
   // Same case for when looking for nullptr.
   __llvm_libc::strchr(src, '\0');
-  ASSERT_STREQ(src, src_copy);
+  ASSERT_STREQ(src, "abcde");
 }
 
 TEST(StrChrTest, ShouldFindFirstOfDuplicates) {
diff --git a/libc/test/src/string/strrchr_test.cpp b/libc/test/src/string/strrchr_test.cpp
index cf29de220d498..5ed83aa64bfbc 100644
--- a/libc/test/src/string/strrchr_test.cpp
+++ b/libc/test/src/string/strrchr_test.cpp
@@ -11,42 +11,38 @@
 
 TEST(StrRChrTest, FindsFirstCharacter) {
   const char *src = "abcde";
-  const char *src_copy = src;
 
   // Should return original string since 'a' is the first character.
   ASSERT_STREQ(__llvm_libc::strrchr(src, 'a'), "abcde");
   // Source string should not change.
-  ASSERT_STREQ(src, src_copy);
+  ASSERT_STREQ(src, "abcde");
 }
 
 TEST(StrRChrTest, FindsMiddleCharacter) {
   const char *src = "abcde";
-  const char *src_copy = src;
 
   // Should return characters after (and including) 'c'.
   ASSERT_STREQ(__llvm_libc::strrchr(src, 'c'), "cde");
   // Source string should not change.
-  ASSERT_STREQ(src, src_copy);
+  ASSERT_STREQ(src, "abcde");
 }
 
 TEST(StrRChrTest, FindsLastCharacterThatIsNotNullTerminator) {
   const char *src = "abcde";
-  const char *src_copy = src;
 
   // Should return 'e' and null-terminator.
   ASSERT_STREQ(__llvm_libc::strrchr(src, 'e'), "e");
   // Source string should not change.
-  ASSERT_STREQ(src, src_copy);
+  ASSERT_STREQ(src, "abcde");
 }
 
 TEST(StrRChrTest, FindsNullTerminator) {
   const char *src = "abcde";
-  const char *src_copy = src;
 
   // Should return null terminator.
   ASSERT_STREQ(__llvm_libc::strrchr(src, '\0'), "");
   // Source string should not change.
-  ASSERT_STREQ(src, src_copy);
+  ASSERT_STREQ(src, "abcde");
 }
 
 TEST(StrRChrTest, FindsLastBehindFirstNullTerminator) {

From 1b35c4fed29d6136ce241a692ce0a7165e59bf81 Mon Sep 17 00:00:00 2001
From: cgyurgyik <gyurgyikcp@gmail.com>
Date: Fri, 31 Jul 2020 20:14:34 -0400
Subject: [PATCH 0795/1035] [libc] [obvious] In strrchr, remove cast to
 unsigned char before comparison.

---
 libc/src/string/strrchr.cpp | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)

diff --git a/libc/src/string/strrchr.cpp b/libc/src/string/strrchr.cpp
index 28716c28a2664..374a802fbb9e2 100644
--- a/libc/src/string/strrchr.cpp
+++ b/libc/src/string/strrchr.cpp
@@ -13,16 +13,13 @@
 namespace __llvm_libc {
 
 char *LLVM_LIBC_ENTRYPOINT(strrchr)(const char *src, int c) {
-  unsigned char *str =
-      const_cast<unsigned char *>(reinterpret_cast<const unsigned char *>(src));
-  const unsigned char ch = c;
-
-  unsigned char *last_occurrence = nullptr;
+  const char ch = c;
+  char *last_occurrence = nullptr;
   do {
-    if (*str == ch)
-      last_occurrence = str;
-  } while (*str++);
-  return reinterpret_cast<char *>(last_occurrence);
+    if (*src == ch)
+      last_occurrence = const_cast<char *>(src);
+  } while (*src++);
+  return last_occurrence;
 }
 
 } // namespace __llvm_libc

From 93c678a79b0a8914954d77973593ea36706db5d5 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@intel.com>
Date: Fri, 31 Jul 2020 17:07:20 -0700
Subject: [PATCH 0796/1035] [X86] Simplify vpternlog immediate selection.

Rather than hardcoding immediate values for 12 different combinations
in a nested pair of switches, we can perform the matched logic
operation on 3 magic constants to calculate the immediate.

Special thanks to this tweet https://twitter.com/rygorous/status/1187034321992871936
for making me realize I could do this.
---
 llvm/lib/Target/X86/X86ISelDAGToDAG.cpp | 49 +++++++++----------------
 1 file changed, 18 insertions(+), 31 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
index 697c160f5bffd..3b333496dd748 100644
--- a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
@@ -3974,39 +3974,26 @@ bool X86DAGToDAGISel::tryVPTERNLOG(SDNode *N) {
   SDValue B = FoldableOp.getOperand(0);
   SDValue C = FoldableOp.getOperand(1);
 
-  unsigned Opc1 = N->getOpcode();
-  unsigned Opc2 = FoldableOp.getOpcode();
+  // We can build the appropriate control immediate by performing the logic
+  // operation we're matching using these constants for A, B, and C.
+  const uint8_t TernlogMagicA = 0xf0;
+  const uint8_t TernlogMagicB = 0xcc;
+  const uint8_t TernlogMagicC = 0xaa;
+
+  uint8_t Imm;
+  switch (FoldableOp.getOpcode()) {
+  default: llvm_unreachable("Unexpected opcode!");
+  case ISD::AND:      Imm = TernlogMagicB & TernlogMagicC; break;
+  case ISD::OR:       Imm = TernlogMagicB | TernlogMagicC; break;
+  case ISD::XOR:      Imm = TernlogMagicB ^ TernlogMagicC; break;
+  case X86ISD::ANDNP: Imm = ~(TernlogMagicB) & TernlogMagicC; break;
+  }
 
-  uint64_t Imm;
-  switch (Opc1) {
+  switch (N->getOpcode()) {
   default: llvm_unreachable("Unexpected opcode!");
-  case ISD::AND:
-    switch (Opc2) {
-    default: llvm_unreachable("Unexpected opcode!");
-    case ISD::AND:      Imm = 0x80; break;
-    case ISD::OR:       Imm = 0xe0; break;
-    case ISD::XOR:      Imm = 0x60; break;
-    case X86ISD::ANDNP: Imm = 0x20; break;
-    }
-    break;
-  case ISD::OR:
-    switch (Opc2) {
-    default: llvm_unreachable("Unexpected opcode!");
-    case ISD::AND:      Imm = 0xf8; break;
-    case ISD::OR:       Imm = 0xfe; break;
-    case ISD::XOR:      Imm = 0xf6; break;
-    case X86ISD::ANDNP: Imm = 0xf2; break;
-    }
-    break;
-  case ISD::XOR:
-    switch (Opc2) {
-    default: llvm_unreachable("Unexpected opcode!");
-    case ISD::AND:      Imm = 0x78; break;
-    case ISD::OR:       Imm = 0x1e; break;
-    case ISD::XOR:      Imm = 0x96; break;
-    case X86ISD::ANDNP: Imm = 0xd2; break;
-    }
-    break;
+  case ISD::AND: Imm &= TernlogMagicA; break;
+  case ISD::OR:  Imm |= TernlogMagicA; break;
+  case ISD::XOR: Imm ^= TernlogMagicA; break;
   }
 
   auto tryFoldLoadOrBCast =

From 234f51a65a45b79402996ac6f0abcbb5793814bf Mon Sep 17 00:00:00 2001
From: Richard Smith <richard@metafoo.co.uk>
Date: Fri, 31 Jul 2020 17:07:36 -0700
Subject: [PATCH 0797/1035] Don't crash if we deserialize a pack expansion type
 whose pattern contains no packs.

Fixes a regression from 740a164dec483225cbd02ab6c82199e2747ffacb.
---
 clang/include/clang/AST/TypeProperties.td | 3 ++-
 clang/lib/AST/ASTImporter.cpp             | 3 ++-
 clang/test/PCH/cxx-variadic-templates.cpp | 5 +++++
 clang/test/PCH/cxx-variadic-templates.h   | 5 +++++
 clang/test/PCH/cxx1y-lambdas.mm           | 4 ++++
 clang/test/PCH/cxx2a-constraints.cpp      | 5 ++++-
 6 files changed, 22 insertions(+), 3 deletions(-)

diff --git a/clang/include/clang/AST/TypeProperties.td b/clang/include/clang/AST/TypeProperties.td
index 4540ea0e1952a..ed91670829b8b 100644
--- a/clang/include/clang/AST/TypeProperties.td
+++ b/clang/include/clang/AST/TypeProperties.td
@@ -722,7 +722,8 @@ let Class = PackExpansionType in {
   }
 
   def : Creator<[{
-    return ctx.getPackExpansionType(pattern, numExpansions);
+    return ctx.getPackExpansionType(pattern, numExpansions,
+                                    /*ExpectPackInType*/false);
   }]>;
 }
 
diff --git a/clang/lib/AST/ASTImporter.cpp b/clang/lib/AST/ASTImporter.cpp
index 12dcd14c06bfa..ee6daf45b7c30 100644
--- a/clang/lib/AST/ASTImporter.cpp
+++ b/clang/lib/AST/ASTImporter.cpp
@@ -1498,7 +1498,8 @@ ASTNodeImporter::VisitPackExpansionType(const PackExpansionType *T) {
     return ToPatternOrErr.takeError();
 
   return Importer.getToContext().getPackExpansionType(*ToPatternOrErr,
-                                                      T->getNumExpansions());
+                                                      T->getNumExpansions(),
+                                                      /*ExpactPack=*/false);
 }
 
 ExpectedType ASTNodeImporter::VisitDependentTemplateSpecializationType(
diff --git a/clang/test/PCH/cxx-variadic-templates.cpp b/clang/test/PCH/cxx-variadic-templates.cpp
index 87b101d73c142..b1eed5adb647b 100644
--- a/clang/test/PCH/cxx-variadic-templates.cpp
+++ b/clang/test/PCH/cxx-variadic-templates.cpp
@@ -19,3 +19,8 @@ shared_ptr<int> spi = shared_ptr<int>::allocate_shared(1, 2);
 template<int> struct A {};
 template<int> struct B {};
 outer<int, int>::inner<1, 2, A, B> i(A<1>{}, B<2>{});
+
+void test_nondependent_pack() {
+  take_nondependent_pack<int, int>(nullptr, nullptr);
+  take_nondependent_pack_2<int, int>({});
+}
diff --git a/clang/test/PCH/cxx-variadic-templates.h b/clang/test/PCH/cxx-variadic-templates.h
index 50596cdf5dbf9..45395e9ae84aa 100644
--- a/clang/test/PCH/cxx-variadic-templates.h
+++ b/clang/test/PCH/cxx-variadic-templates.h
@@ -23,3 +23,8 @@ template<typename...Ts> struct outer {
   };
 };
 template struct outer<int, int>;
+
+template<typename ...T> void take_nondependent_pack(int (...arr)[sizeof(sizeof(T))]);
+
+template<typename T> using hide = int;
+template<typename ...T> void take_nondependent_pack_2(outer<hide<T>...>);
diff --git a/clang/test/PCH/cxx1y-lambdas.mm b/clang/test/PCH/cxx1y-lambdas.mm
index f140a15215b8f..9c4c11970473b 100644
--- a/clang/test/PCH/cxx1y-lambdas.mm
+++ b/clang/test/PCH/cxx1y-lambdas.mm
@@ -39,6 +39,8 @@ int init_capture(T t) {
   return [&, x(t)] { return sizeof(x); };
 }
 
+auto with_pack = [](auto ...xs){};
+
 #else
 
 // CHECK-PRINT: T add_slowly
@@ -55,4 +57,6 @@ int add(int x, int y) {
 // CHECK-PRINT: init_capture
 // CHECK-PRINT: [&, x(t)]
 
+void use_with_pack() { with_pack(1, 2, 3); }
+
 #endif
diff --git a/clang/test/PCH/cxx2a-constraints.cpp b/clang/test/PCH/cxx2a-constraints.cpp
index d8b79337c8f18..3f3b5e536cc93 100644
--- a/clang/test/PCH/cxx2a-constraints.cpp
+++ b/clang/test/PCH/cxx2a-constraints.cpp
@@ -24,6 +24,8 @@ template <SizedLike<char> T> void h(T) {}
 template <SizedLike<int> T> void i(T) {}
 template <SizedLike T> void i(T) {}
 
+void j(SizedLike<int> auto ...ints) {}
+
 #else /*included pch*/
 
 int main() {
@@ -35,6 +37,7 @@ int main() {
   (void)h(1);
   (void)i('1');
   (void)i(1);
+  (void)j(1, 2, 3);
 }
 
-#endif // HEADER
\ No newline at end of file
+#endif // HEADER

From 01bfe2e494027e473ba920ef324b1929af16936e Mon Sep 17 00:00:00 2001
From: Huihui Zhang <huihuiz@quicinc.com>
Date: Fri, 31 Jul 2020 17:08:17 -0700
Subject: [PATCH 0798/1035] [AArch64][SVE] Allow vector of pointers as legal
 type for masked load/store.
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Refer to LangRef http://llvm.org/docs/LangRef.html#llvm-masked-load-intrinsics
'llvm.masked.load/store.*’ intrinsics are overloaded intrinsic, which allow the
load/store data to be a vector of any integer, floating-point or pointer data type.

Therefore, allow pointer data type when checking 'isLegalMaskedLoadStore()'.

Reviewed By: paulwalker-arm

Differential Revision: https://reviews.llvm.org/D85045
---
 .../AArch64/AArch64TargetTransformInfo.h      |   3 +
 .../CodeGen/AArch64/sve-masked-ldst-nonext.ll | 102 ++++++++++++++++++
 2 files changed, 105 insertions(+)

diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index 5d1371f13fb3e..05b7f70f2335c 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -166,6 +166,9 @@ class AArch64TTIImpl : public BasicTTIImplBase<AArch64TTIImpl> {
       return false;
 
     Type *Ty = cast<ScalableVectorType>(DataType)->getElementType();
+    if (Ty->isPointerTy())
+      return true;
+
     if (Ty->isBFloatTy() || Ty->isHalfTy() ||
         Ty->isFloatTy() || Ty->isDoubleTy())
       return true;
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-ldst-nonext.ll b/llvm/test/CodeGen/AArch64/sve-masked-ldst-nonext.ll
index f5047a7bcbaff..5e48ad8b628e0 100644
--- a/llvm/test/CodeGen/AArch64/sve-masked-ldst-nonext.ll
+++ b/llvm/test/CodeGen/AArch64/sve-masked-ldst-nonext.ll
@@ -188,6 +188,94 @@ define void @masked_store_nxv8bf16(<vscale x 8 x bfloat> *%a, <vscale x 8 x bflo
   ret void
 }
 
+;
+; Masked load store of pointer data type
+;
+
+; Pointer of integer type
+
+define <vscale x 2 x i8*> @masked.load.nxv2p0i8(<vscale x 2 x i8*>* %vector_ptr, <vscale x 2 x i1> %mask) nounwind {
+; CHECK-LABEL: masked.load.nxv2p0i8:
+; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT:    ret
+  %v = call <vscale x 2 x i8*> @llvm.masked.load.nxv2p0i8.p0nxv2p0i8(<vscale x 2 x i8*>* %vector_ptr, i32 8, <vscale x 2 x i1> %mask, <vscale x 2 x i8*> undef)
+  ret <vscale x 2 x i8*> %v
+}
+define <vscale x 2 x i16*> @masked.load.nxv2p0i16(<vscale x 2 x i16*>* %vector_ptr, <vscale x 2 x i1> %mask) nounwind {
+; CHECK-LABEL: masked.load.nxv2p0i16:
+; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT:    ret
+  %v = call <vscale x 2 x i16*> @llvm.masked.load.nxv2p0i16.p0nxv2p0i16(<vscale x 2 x i16*>* %vector_ptr, i32 8, <vscale x 2 x i1> %mask, <vscale x 2 x i16*> undef)
+  ret <vscale x 2 x i16*> %v
+}
+define <vscale x 2 x i32*> @masked.load.nxv2p0i32(<vscale x 2 x i32*>* %vector_ptr, <vscale x 2 x i1> %mask) nounwind {
+; CHECK-LABEL: masked.load.nxv2p0i32:
+; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT:    ret
+  %v = call <vscale x 2 x i32*> @llvm.masked.load.nxv2p0i32.p0nxv2p0i32(<vscale x 2 x i32*>* %vector_ptr, i32 8, <vscale x 2 x i1> %mask, <vscale x 2 x i32*> undef)
+  ret <vscale x 2 x i32*> %v
+}
+define <vscale x 2 x i64*> @masked.load.nxv2p0i64(<vscale x 2 x i64*>* %vector_ptr, <vscale x 2 x i1> %mask) nounwind {
+; CHECK-LABEL: masked.load.nxv2p0i64:
+; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT:    ret
+  %v = call <vscale x 2 x i64*> @llvm.masked.load.nxv2p0i64.p0nxv2p0i64(<vscale x 2 x i64*>* %vector_ptr, i32 8, <vscale x 2 x i1> %mask, <vscale x 2 x i64*> undef)
+  ret <vscale x 2 x i64*> %v
+}
+
+; Pointer of floating-point type
+
+define <vscale x 2 x bfloat*> @masked.load.nxv2p0bf16(<vscale x 2 x bfloat*>* %vector_ptr, <vscale x 2 x i1> %mask) nounwind #0 {
+; CHECK-LABEL: masked.load.nxv2p0bf16:
+; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT:    ret
+  %v = call <vscale x 2 x bfloat*> @llvm.masked.load.nxv2p0bf16.p0nxv2p0bf16(<vscale x 2 x bfloat*>* %vector_ptr, i32 8, <vscale x 2 x i1> %mask, <vscale x 2 x bfloat*> undef)
+  ret <vscale x 2 x bfloat*> %v
+}
+define <vscale x 2 x half*> @masked.load.nxv2p0f16(<vscale x 2 x half*>* %vector_ptr, <vscale x 2 x i1> %mask) nounwind {
+; CHECK-LABEL: masked.load.nxv2p0f16:
+; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT:    ret
+  %v = call <vscale x 2 x half*> @llvm.masked.load.nxv2p0f16.p0nxv2p0f16(<vscale x 2 x half*>* %vector_ptr, i32 8, <vscale x 2 x i1> %mask, <vscale x 2 x half*> undef)
+  ret <vscale x 2 x half*> %v
+}
+define <vscale x 2 x float*> @masked.load.nxv2p0f32(<vscale x 2 x float*>* %vector_ptr, <vscale x 2 x i1> %mask) nounwind {
+; CHECK-LABEL: masked.load.nxv2p0f32:
+; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT:    ret
+  %v = call <vscale x 2 x float*> @llvm.masked.load.nxv2p0f32.p0nxv2p0f32(<vscale x 2 x float*>* %vector_ptr, i32 8, <vscale x 2 x i1> %mask, <vscale x 2 x float*> undef)
+  ret <vscale x 2 x float*> %v
+}
+define <vscale x 2 x double*> @masked.load.nxv2p0f64(<vscale x 2 x double*>* %vector_ptr, <vscale x 2 x i1> %mask) nounwind {
+; CHECK-LABEL: masked.load.nxv2p0f64:
+; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT:    ret
+  %v = call <vscale x 2 x double*> @llvm.masked.load.nxv2p0f64.p0nxv2p0f64(<vscale x 2 x double*>* %vector_ptr, i32 8, <vscale x 2 x i1> %mask, <vscale x 2 x double*> undef)
+  ret <vscale x 2 x double*> %v
+}
+
+; Pointer of array type
+
+define void @masked.store.nxv2p0a64i16(<vscale x 2 x [64 x i16]*> %data, <vscale x 2 x [64 x i16]*>* %vector_ptr, <vscale x 2 x i1> %mask) nounwind {
+; CHECK-LABEL: masked.store.nxv2p0a64i16:
+; CHECK-NEXT:    st1d { z0.d }, p0, [x0]
+; CHECK-NEXT:    ret
+  call void @llvm.masked.store.nxv2p0a64i16.p0nxv2p0a64i16(<vscale x 2 x [64 x i16]*> %data, <vscale x 2 x [64 x i16]*>* %vector_ptr, i32 8, <vscale x 2 x i1> %mask)
+  ret void
+}
+
+; Pointer of struct type
+
+%struct = type { i8*, i32 }
+define void @masked.store.nxv2p0s_struct(<vscale x 2 x %struct*> %data, <vscale x 2 x %struct*>* %vector_ptr, <vscale x 2 x i1> %mask) nounwind {
+; CHECK-LABEL: masked.store.nxv2p0s_struct:
+; CHECK-NEXT:    st1d { z0.d }, p0, [x0]
+; CHECK-NEXT:    ret
+  call void @llvm.masked.store.nxv2p0s_struct.p0nxv2p0s_struct(<vscale x 2 x %struct*> %data, <vscale x 2 x %struct*>* %vector_ptr, i32 8, <vscale x 2 x i1> %mask)
+  ret void
+}
+
+
 declare <vscale x 2 x i64> @llvm.masked.load.nxv2i64(<vscale x 2 x i64>*, i32, <vscale x 2 x i1>, <vscale x 2 x i64>)
 declare <vscale x 4 x i32> @llvm.masked.load.nxv4i32(<vscale x 4 x i32>*, i32, <vscale x 4 x i1>, <vscale x 4 x i32>)
 declare <vscale x 8 x i16> @llvm.masked.load.nxv8i16(<vscale x 8 x i16>*, i32, <vscale x 8 x i1>, <vscale x 8 x i16>)
@@ -214,5 +302,19 @@ declare void @llvm.masked.store.nxv4f16(<vscale x 4 x half>, <vscale x 4 x half>
 declare void @llvm.masked.store.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>*, i32, <vscale x 8 x i1>)
 declare void @llvm.masked.store.nxv8bf16(<vscale x 8 x bfloat>, <vscale x 8 x bfloat>*, i32, <vscale x 8 x i1>)
 
+declare <vscale x 2 x i8*> @llvm.masked.load.nxv2p0i8.p0nxv2p0i8(<vscale x 2 x i8*>*, i32 immarg, <vscale x 2 x i1>, <vscale x 2 x i8*>)
+declare <vscale x 2 x i16*> @llvm.masked.load.nxv2p0i16.p0nxv2p0i16(<vscale x 2 x i16*>*, i32 immarg, <vscale x 2 x i1>, <vscale x 2 x i16*>)
+declare <vscale x 2 x i32*> @llvm.masked.load.nxv2p0i32.p0nxv2p0i32(<vscale x 2 x i32*>*, i32 immarg, <vscale x 2 x i1>, <vscale x 2 x i32*>)
+declare <vscale x 2 x i64*> @llvm.masked.load.nxv2p0i64.p0nxv2p0i64(<vscale x 2 x i64*>*, i32 immarg, <vscale x 2 x i1>, <vscale x 2 x i64*>)
+
+declare <vscale x 2 x bfloat*> @llvm.masked.load.nxv2p0bf16.p0nxv2p0bf16(<vscale x 2 x bfloat*>*, i32 immarg, <vscale x 2 x i1>, <vscale x 2 x bfloat*>)
+declare <vscale x 2 x half*> @llvm.masked.load.nxv2p0f16.p0nxv2p0f16(<vscale x 2 x half*>*, i32 immarg, <vscale x 2 x i1>, <vscale x 2 x half*>)
+declare <vscale x 2 x float*> @llvm.masked.load.nxv2p0f32.p0nxv2p0f32(<vscale x 2 x float*>*, i32 immarg, <vscale x 2 x i1>, <vscale x 2 x float*>)
+declare <vscale x 2 x double*> @llvm.masked.load.nxv2p0f64.p0nxv2p0f64(<vscale x 2 x double*>*, i32 immarg, <vscale x 2 x i1>, <vscale x 2 x double*>)
+
+declare void @llvm.masked.store.nxv2p0a64i16.p0nxv2p0a64i16(<vscale x 2 x [64 x i16]*>, <vscale x 2 x [64 x i16]*>*, i32 immarg, <vscale x 2 x i1>)
+
+declare void @llvm.masked.store.nxv2p0s_struct.p0nxv2p0s_struct(<vscale x 2 x %struct*>, <vscale x 2 x %struct*>*, i32 immarg, <vscale x 2 x i1>)
+
 ; +bf16 is required for the bfloat version.
 attributes #0 = { "target-features"="+sve,+bf16" }

From 1fd2049e38daf0992f63883d68609b85dfb9cb26 Mon Sep 17 00:00:00 2001
From: Nathan James <n.james93@hotmail.co.uk>
Date: Sat, 1 Aug 2020 01:45:33 +0100
Subject: [PATCH 0799/1035] [clang-tidy][NFC] Added convienence methods for
 getting optional options

These methods abstract away Error handling when trying to read options that can't be parsed by logging the error automatically and returning None.

Reviewed By: gribozavr2

Differential Revision: https://reviews.llvm.org/D84812
---
 .../clang-tidy/ClangTidyCheck.cpp             | 39 +++++++++++----
 clang-tools-extra/clang-tidy/ClangTidyCheck.h | 49 +++++++++++++++++--
 2 files changed, 74 insertions(+), 14 deletions(-)

diff --git a/clang-tools-extra/clang-tidy/ClangTidyCheck.cpp b/clang-tools-extra/clang-tidy/ClangTidyCheck.cpp
index 737d85e092d97..c99931e0aa3a2 100644
--- a/clang-tools-extra/clang-tidy/ClangTidyCheck.cpp
+++ b/clang-tools-extra/clang-tidy/ClangTidyCheck.cpp
@@ -10,6 +10,7 @@
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Error.h"
+#include "llvm/Support/WithColor.h"
 #include "llvm/Support/raw_ostream.h"
 
 namespace clang {
@@ -126,7 +127,7 @@ bool ClangTidyCheck::OptionsView::get<bool>(StringRef LocalName,
   llvm::Expected<bool> ValueOr = get<bool>(LocalName);
   if (ValueOr)
     return *ValueOr;
-  logErrToStdErr(ValueOr.takeError());
+  logIfOptionParsingError(ValueOr.takeError());
   return Default;
 }
 
@@ -145,7 +146,7 @@ bool ClangTidyCheck::OptionsView::getLocalOrGlobal<bool>(StringRef LocalName,
   llvm::Expected<bool> ValueOr = getLocalOrGlobal<bool>(LocalName);
   if (ValueOr)
     return *ValueOr;
-  logErrToStdErr(ValueOr.takeError());
+  logIfOptionParsingError(ValueOr.takeError());
   return Default;
 }
 
@@ -204,13 +205,33 @@ llvm::Expected<int64_t> ClangTidyCheck::OptionsView::getEnumInt(
                                                       Iter->getValue().Value);
 }
 
-void ClangTidyCheck::OptionsView::logErrToStdErr(llvm::Error &&Err) {
-  llvm::logAllUnhandledErrors(
-      llvm::handleErrors(std::move(Err),
-                         [](const MissingOptionError &) -> llvm::Error {
-                           return llvm::Error::success();
-                         }),
-      llvm::errs(), "warning: ");
+void ClangTidyCheck::OptionsView::logIfOptionParsingError(llvm::Error &&Err) {
+  if (auto RemainingErrors =
+          llvm::handleErrors(std::move(Err), [](const MissingOptionError &) {}))
+    llvm::logAllUnhandledErrors(std::move(RemainingErrors),
+                                llvm::WithColor::warning());
 }
+
+template <>
+Optional<std::string> ClangTidyCheck::OptionsView::getOptional<std::string>(
+    StringRef LocalName) const {
+  if (auto ValueOr = get(LocalName))
+    return *ValueOr;
+  else
+    consumeError(ValueOr.takeError());
+  return llvm::None;
+}
+
+template <>
+Optional<std::string>
+ClangTidyCheck::OptionsView::getOptionalLocalOrGlobal<std::string>(
+    StringRef LocalName) const {
+  if (auto ValueOr = getLocalOrGlobal(LocalName))
+    return *ValueOr;
+  else
+    consumeError(ValueOr.takeError());
+  return llvm::None;
+}
+
 } // namespace tidy
 } // namespace clang
diff --git a/clang-tools-extra/clang-tidy/ClangTidyCheck.h b/clang-tools-extra/clang-tidy/ClangTidyCheck.h
index 4df8071c841e0..6237e216656be 100644
--- a/clang-tools-extra/clang-tidy/ClangTidyCheck.h
+++ b/clang-tools-extra/clang-tidy/ClangTidyCheck.h
@@ -268,7 +268,7 @@ class ClangTidyCheck : public ast_matchers::MatchFinder::MatchCallback {
       if (llvm::Expected<T> ValueOr = get<T>(LocalName))
         return *ValueOr;
       else
-        logErrToStdErr(ValueOr.takeError());
+        logIfOptionParsingError(ValueOr.takeError());
       return Default;
     }
 
@@ -314,7 +314,7 @@ class ClangTidyCheck : public ast_matchers::MatchFinder::MatchCallback {
       if (llvm::Expected<T> ValueOr = getLocalOrGlobal<T>(LocalName))
         return *ValueOr;
       else
-        logErrToStdErr(ValueOr.takeError());
+        logIfOptionParsingError(ValueOr.takeError());
       return Default;
     }
 
@@ -353,7 +353,7 @@ class ClangTidyCheck : public ast_matchers::MatchFinder::MatchCallback {
       if (auto ValueOr = get<T>(LocalName, IgnoreCase))
         return *ValueOr;
       else
-        logErrToStdErr(ValueOr.takeError());
+        logIfOptionParsingError(ValueOr.takeError());
       return Default;
     }
 
@@ -395,10 +395,35 @@ class ClangTidyCheck : public ast_matchers::MatchFinder::MatchCallback {
       if (auto ValueOr = getLocalOrGlobal<T>(LocalName, IgnoreCase))
         return *ValueOr;
       else
-        logErrToStdErr(ValueOr.takeError());
+        logIfOptionParsingError(ValueOr.takeError());
       return Default;
     }
 
+    /// Returns the value for the option \p LocalName represented as a ``T``.
+    /// If the option is missing returns None, if the option can't be parsed
+    /// as a ``T``, log that to stderr and return None.
+    template <typename T = std::string>
+    llvm::Optional<T> getOptional(StringRef LocalName) const {
+      if (auto ValueOr = get<T>(LocalName))
+        return *ValueOr;
+      else
+        logIfOptionParsingError(ValueOr.takeError());
+      return llvm::None;
+    }
+
+    /// Returns the value for the local or global option \p LocalName
+    /// represented as a ``T``.
+    /// If the option is missing returns None, if the
+    /// option can't be parsed as a ``T``, log that to stderr and return None.
+    template <typename T = std::string>
+    llvm::Optional<T> getOptionalLocalOrGlobal(StringRef LocalName) const {
+      if (auto ValueOr = getLocalOrGlobal<T>(LocalName))
+        return *ValueOr;
+      else
+        logIfOptionParsingError(ValueOr.takeError());
+      return llvm::None;
+    }
+
     /// Stores an option with the check-local name \p LocalName with
     /// string value \p Value to \p Options.
     void store(ClangTidyOptions::OptionMap &Options, StringRef LocalName,
@@ -456,7 +481,8 @@ class ClangTidyCheck : public ast_matchers::MatchFinder::MatchCallback {
     void storeInt(ClangTidyOptions::OptionMap &Options, StringRef LocalName,
                   int64_t Value) const;
 
-    static void logErrToStdErr(llvm::Error &&Err);
+    /// Logs an Error to stderr if a \p Err is not a MissingOptionError.
+    static void logIfOptionParsingError(llvm::Error &&Err);
 
     std::string NamePrefix;
     const ClangTidyOptions::OptionMap &CheckOptions;
@@ -524,6 +550,19 @@ void ClangTidyCheck::OptionsView::store<bool>(
     ClangTidyOptions::OptionMap &Options, StringRef LocalName,
     bool Value) const;
 
+/// Returns the value for the option \p LocalName.
+/// If the option is missing returns None.
+template <>
+Optional<std::string> ClangTidyCheck::OptionsView::getOptional<std::string>(
+    StringRef LocalName) const;
+
+/// Returns the value for the local or global option \p LocalName.
+/// If the option is missing returns None.
+template <>
+Optional<std::string>
+ClangTidyCheck::OptionsView::getOptionalLocalOrGlobal<std::string>(
+    StringRef LocalName) const;
+
 } // namespace tidy
 } // namespace clang
 

From 605fd4d77ce19c4d8c331732b490ef436ab093c2 Mon Sep 17 00:00:00 2001
From: "Kazushi (Jam) Marukawa" <marukawa@nec.com>
Date: Sat, 11 Jul 2020 16:01:13 +0900
Subject: [PATCH 0800/1035] [VE] Change calling convention to follow ABI

Change to expand all arguments and return values to i64 to follow ABI.
Update regression tests also.

Reviewed By: simoll

Differential Revision: https://reviews.llvm.org/D84581
---
 llvm/lib/Target/VE/VECallingConv.td           |  62 +++----
 llvm/lib/Target/VE/VEISelLowering.cpp         |  96 +++++++---
 llvm/test/CodeGen/VE/addition.ll              |  34 +++-
 llvm/test/CodeGen/VE/bitcast.ll               |   5 +-
 llvm/test/CodeGen/VE/bitreverse.ll            |  12 --
 llvm/test/CodeGen/VE/branch1.ll               | 126 +++++++------
 llvm/test/CodeGen/VE/bswap.ll                 |  17 +-
 llvm/test/CodeGen/VE/call.ll                  |  34 ++--
 llvm/test/CodeGen/VE/cast.ll                  |  98 +++++-----
 llvm/test/CodeGen/VE/constants.ll             |   3 -
 llvm/test/CodeGen/VE/ctlz.ll                  |   3 +-
 llvm/test/CodeGen/VE/ctpop.ll                 |   7 +-
 llvm/test/CodeGen/VE/cttz.ll                  |   6 +-
 llvm/test/CodeGen/VE/div.ll                   |  30 +++-
 llvm/test/CodeGen/VE/fp_to_int.ll             |  10 +-
 llvm/test/CodeGen/VE/int_to_fp.ll             |  14 +-
 llvm/test/CodeGen/VE/left_shift.ll            |  40 +++--
 llvm/test/CodeGen/VE/load_off.ll              |   2 +-
 llvm/test/CodeGen/VE/max.ll                   |  26 ++-
 llvm/test/CodeGen/VE/min.ll                   |  31 ++--
 llvm/test/CodeGen/VE/multiply.ll              |  35 +++-
 llvm/test/CodeGen/VE/nnd.ll                   |  29 +++
 llvm/test/CodeGen/VE/or.ll                    |  10 ++
 .../test/CodeGen/VE/pic_access_static_data.ll |   2 +-
 llvm/test/CodeGen/VE/rem.ll                   |  34 +++-
 llvm/test/CodeGen/VE/right_shift.ll           |  38 ++--
 llvm/test/CodeGen/VE/rotl.ll                  |   4 +-
 llvm/test/CodeGen/VE/rotr.ll                  |   4 +-
 llvm/test/CodeGen/VE/select.ll                |  20 ++-
 llvm/test/CodeGen/VE/selectccf32.ll           |  28 ---
 llvm/test/CodeGen/VE/selectccf32c.ll          |  32 ++--
 llvm/test/CodeGen/VE/selectccf32i.ll          |  28 ---
 llvm/test/CodeGen/VE/selectccf64c.ll          |  18 +-
 llvm/test/CodeGen/VE/selectcci32.ll           | 154 ++++++++--------
 llvm/test/CodeGen/VE/selectcci32c.ll          |  85 +++++----
 llvm/test/CodeGen/VE/selectcci32i.ll          | 168 +++++++++---------
 llvm/test/CodeGen/VE/selectcci64c.ll          |  18 +-
 llvm/test/CodeGen/VE/setccf32.ll              | 112 ++++++------
 llvm/test/CodeGen/VE/setccf32i.ll             | 112 ++++++------
 llvm/test/CodeGen/VE/setccf64.ll              | 112 ++++++------
 llvm/test/CodeGen/VE/setccf64i.ll             | 112 ++++++------
 llvm/test/CodeGen/VE/setcci32.ll              | 100 ++++++-----
 llvm/test/CodeGen/VE/setcci32i.ll             |  90 +++++-----
 llvm/test/CodeGen/VE/setcci64.ll              |  80 ++++-----
 llvm/test/CodeGen/VE/setcci64i.ll             |  80 ++++-----
 llvm/test/CodeGen/VE/sext_zext_load.ll        |  12 +-
 llvm/test/CodeGen/VE/subtraction.ll           |  34 +++-
 llvm/test/CodeGen/VE/truncstore.ll            |   2 -
 llvm/test/CodeGen/VE/va_caller.ll             |  28 ++-
 llvm/test/CodeGen/VE/xor.ll                   |  13 ++
 50 files changed, 1221 insertions(+), 1029 deletions(-)

diff --git a/llvm/lib/Target/VE/VECallingConv.td b/llvm/lib/Target/VE/VECallingConv.td
index 4f04dae884ab5..acdae68323fdd 100644
--- a/llvm/lib/Target/VE/VECallingConv.td
+++ b/llvm/lib/Target/VE/VECallingConv.td
@@ -14,13 +14,6 @@
 // Aurora VE
 //===----------------------------------------------------------------------===//
 def CC_VE_C_Stack: CallingConv<[
-  // float --> need special handling like below.
-  //    0      4
-  //    +------+------+
-  //    | empty| float|
-  //    +------+------+
-  CCIfType<[f32], CCCustom<"allocateFloat">>,
-
   // All of the rest are assigned to the stack in 8-byte aligned units.
   CCAssignToStack<0, 8>
 ]>;
@@ -28,20 +21,18 @@ def CC_VE_C_Stack: CallingConv<[
 def CC_VE : CallingConv<[
   // All arguments get passed in generic registers if there is space.
 
-  // Promote i1/i8/i16 arguments to i32.
-  CCIfType<[i1, i8, i16], CCPromoteToType<i32>>,
+  // Promote i1/i8/i16/i32 arguments to i64.
+  CCIfType<[i1, i8, i16, i32], CCPromoteToType<i64>>,
 
-  // bool, char, int, enum, long --> generic integer 32 bit registers
-  CCIfType<[i32], CCAssignToRegWithShadow<
-    [SW0, SW1, SW2, SW3, SW4, SW5, SW6, SW7],
-    [SX0, SX1, SX2, SX3, SX4, SX5, SX6, SX7]>>,
-
-  // float --> generic floating point 32 bit registers
-  CCIfType<[f32], CCAssignToRegWithShadow<
-    [SF0, SF1, SF2, SF3, SF4, SF5, SF6, SF7],
-    [SX0, SX1, SX2, SX3, SX4, SX5, SX6, SX7]>>,
+  // Convert float arguments to i64 with padding.
+  //     63     31   0
+  //    +------+------+
+  //    | float|   0  |
+  //    +------+------+
+  CCIfType<[f32], CCBitConvertToType<i64>>,
 
-  // long long/double --> generic 64 bit registers
+  // bool, char, int, enum, long, long long, float, double
+  //     --> generic 64 bit registers
   CCIfType<[i64, f64],
            CCAssignToReg<[SX0, SX1, SX2, SX3, SX4, SX5, SX6, SX7]>>,
 
@@ -52,31 +43,32 @@ def CC_VE : CallingConv<[
 // All arguments get passed in stack for varargs function or non-prototyped
 // function.
 def CC_VE2 : CallingConv<[
-  // float --> need special handling like below.
-  //    0      4
+  // Promote i1/i8/i16/i32 arguments to i64.
+  CCIfType<[i1, i8, i16, i32], CCPromoteToType<i64>>,
+
+  // Convert float arguments to i64 with padding.
+  //     63     31   0
   //    +------+------+
-  //    | empty| float|
+  //    | float|   0  |
   //    +------+------+
-  CCIfType<[f32], CCCustom<"allocateFloat">>,
+  CCIfType<[f32], CCBitConvertToType<i64>>,
 
   CCAssignToStack<0, 8>
 ]>;
 
 def RetCC_VE : CallingConv<[
-  // Promote i1/i8/i16 arguments to i32.
-  CCIfType<[i1, i8, i16], CCPromoteToType<i32>>,
-
-  // bool, char, int, enum, long --> generic integer 32 bit registers
-  CCIfType<[i32], CCAssignToRegWithShadow<
-    [SW0, SW1, SW2, SW3, SW4, SW5, SW6, SW7],
-    [SX0, SX1, SX2, SX3, SX4, SX5, SX6, SX7]>>,
+  // Promote i1/i8/i16/i32 return values to i64.
+  CCIfType<[i1, i8, i16, i32], CCPromoteToType<i64>>,
 
-  // float --> generic floating point 32 bit registers
-  CCIfType<[f32], CCAssignToRegWithShadow<
-    [SF0, SF1, SF2, SF3, SF4, SF5, SF6, SF7],
-    [SX0, SX1, SX2, SX3, SX4, SX5, SX6, SX7]>>,
+  // Convert float return values to i64 with padding.
+  //     63     31   0
+  //    +------+------+
+  //    | float|   0  |
+  //    +------+------+
+  CCIfType<[f32], CCBitConvertToType<i64>>,
 
-  // long long/double --> generic 64 bit registers
+  // bool, char, int, enum, long, long long, float, double
+  //     --> generic 64 bit registers
   CCIfType<[i64, f64],
            CCAssignToReg<[SX0, SX1, SX2, SX3, SX4, SX5, SX6, SX7]>>,
 ]>;
diff --git a/llvm/lib/Target/VE/VEISelLowering.cpp b/llvm/lib/Target/VE/VEISelLowering.cpp
index ab720545dd831..e2232f4500e35 100644
--- a/llvm/lib/Target/VE/VEISelLowering.cpp
+++ b/llvm/lib/Target/VE/VEISelLowering.cpp
@@ -38,28 +38,6 @@ using namespace llvm;
 // Calling Convention Implementation
 //===----------------------------------------------------------------------===//
 
-static bool allocateFloat(unsigned ValNo, MVT ValVT, MVT LocVT,
-                          CCValAssign::LocInfo LocInfo,
-                          ISD::ArgFlagsTy ArgFlags, CCState &State) {
-  switch (LocVT.SimpleTy) {
-  case MVT::f32: {
-    // Allocate stack like below
-    //    0      4
-    //    +------+------+
-    //    | empty| float|
-    //    +------+------+
-    // Use align=8 for dummy area to align the beginning of these 2 area.
-    State.AllocateStack(4, Align(8)); // for empty area
-    // Use align=4 for value to place it at just after the dummy area.
-    unsigned Offset = State.AllocateStack(4, Align(4)); // for float value area
-    State.addLoc(CCValAssign::getMem(ValNo, ValVT, Offset, LocVT, LocInfo));
-    return true;
-  }
-  default:
-    return false;
-  }
-}
-
 #include "VEGenCallingConv.inc"
 
 bool VETargetLowering::CanLowerReturn(
@@ -109,6 +87,22 @@ VETargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
     case CCValAssign::AExt:
       OutVal = DAG.getNode(ISD::ANY_EXTEND, DL, VA.getLocVT(), OutVal);
       break;
+    case CCValAssign::BCvt: {
+      // Convert a float return value to i64 with padding.
+      //     63     31   0
+      //    +------+------+
+      //    | float|   0  |
+      //    +------+------+
+      assert(VA.getLocVT() == MVT::i64);
+      assert(VA.getValVT() == MVT::f32);
+      SDValue Undef = SDValue(
+          DAG.getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, MVT::i64), 0);
+      SDValue Sub_f32 = DAG.getTargetConstant(VE::sub_f32, DL, MVT::i32);
+      OutVal = SDValue(DAG.getMachineNode(TargetOpcode::INSERT_SUBREG, DL,
+                                          MVT::i64, Undef, OutVal, Sub_f32),
+                       0);
+      break;
+    }
     default:
       llvm_unreachable("Unknown loc info!");
     }
@@ -179,6 +173,20 @@ SDValue VETargetLowering::LowerFormalArguments(
         Arg = DAG.getNode(ISD::AssertZext, DL, VA.getLocVT(), Arg,
                           DAG.getValueType(VA.getValVT()));
         break;
+      case CCValAssign::BCvt: {
+        // Extract a float argument from i64 with padding.
+        //     63     31   0
+        //    +------+------+
+        //    | float|   0  |
+        //    +------+------+
+        assert(VA.getLocVT() == MVT::i64);
+        assert(VA.getValVT() == MVT::f32);
+        SDValue Sub_f32 = DAG.getTargetConstant(VE::sub_f32, DL, MVT::i32);
+        Arg = SDValue(DAG.getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL,
+                                         MVT::f32, Arg, Sub_f32),
+                      0);
+        break;
+      }
       default:
         break;
       }
@@ -197,6 +205,20 @@ SDValue VETargetLowering::LowerFormalArguments(
     // beginning of the arguments area at %fp+176.
     unsigned Offset = VA.getLocMemOffset() + ArgsBaseOffset;
     unsigned ValSize = VA.getValVT().getSizeInBits() / 8;
+
+    // Adjust offset for a float argument by adding 4 since the argument is
+    // stored in 8 bytes buffer with offset like below.  LLVM generates
+    // 4 bytes load instruction, so need to adjust offset here.  This
+    // adjustment is required in only LowerFormalArguments.  In LowerCall,
+    // a float argument is converted to i64 first, and stored as 8 bytes
+    // data, which is required by ABI, so no need for adjustment.
+    //    0      4
+    //    +------+------+
+    //    | empty| float|
+    //    +------+------+
+    if (VA.getValVT() == MVT::f32)
+      Offset += 4;
+
     int FI = MF.getFrameInfo().CreateFixedObject(ValSize, Offset, true);
     InVals.push_back(
         DAG.getLoad(VA.getValVT(), DL, Chain,
@@ -371,6 +393,22 @@ SDValue VETargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
     case CCValAssign::AExt:
       Arg = DAG.getNode(ISD::ANY_EXTEND, DL, VA.getLocVT(), Arg);
       break;
+    case CCValAssign::BCvt: {
+      // Convert a float argument to i64 with padding.
+      //     63     31   0
+      //    +------+------+
+      //    | float|   0  |
+      //    +------+------+
+      assert(VA.getLocVT() == MVT::i64);
+      assert(VA.getValVT() == MVT::f32);
+      SDValue Undef = SDValue(
+          DAG.getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, MVT::i64), 0);
+      SDValue Sub_f32 = DAG.getTargetConstant(VE::sub_f32, DL, MVT::i32);
+      Arg = SDValue(DAG.getMachineNode(TargetOpcode::INSERT_SUBREG, DL,
+                                       MVT::i64, Undef, Arg, Sub_f32),
+                    0);
+      break;
+    }
     }
 
     if (VA.isRegLoc()) {
@@ -488,6 +526,20 @@ SDValue VETargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
       RV = DAG.getNode(ISD::AssertZext, DL, VA.getLocVT(), RV,
                        DAG.getValueType(VA.getValVT()));
       break;
+    case CCValAssign::BCvt: {
+      // Extract a float return value from i64 with padding.
+      //     63     31   0
+      //    +------+------+
+      //    | float|   0  |
+      //    +------+------+
+      assert(VA.getLocVT() == MVT::i64);
+      assert(VA.getValVT() == MVT::f32);
+      SDValue Sub_f32 = DAG.getTargetConstant(VE::sub_f32, DL, MVT::i32);
+      RV = SDValue(DAG.getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL,
+                                      MVT::f32, RV, Sub_f32),
+                   0);
+      break;
+    }
     default:
       break;
     }
diff --git a/llvm/test/CodeGen/VE/addition.ll b/llvm/test/CodeGen/VE/addition.ll
index 730776ec534b8..54275e9e0e267 100644
--- a/llvm/test/CodeGen/VE/addition.ll
+++ b/llvm/test/CodeGen/VE/addition.ll
@@ -3,9 +3,11 @@
 define signext i8 @func1(i8 signext %0, i8 signext %1) {
 ; CHECK-LABEL: func1:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
 ; CHECK-NEXT:    adds.w.sx %s0, %s1, %s0
-; CHECK-NEXT:    sla.w.sx %s0, %s0, 24
-; CHECK-NEXT:    sra.w.sx %s0, %s0, 24
+; CHECK-NEXT:    sll %s0, %s0, 56
+; CHECK-NEXT:    sra.l %s0, %s0, 56
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = add i8 %1, %0
   ret i8 %3
@@ -14,9 +16,11 @@ define signext i8 @func1(i8 signext %0, i8 signext %1) {
 define signext i16 @func2(i16 signext %0, i16 signext %1) {
 ; CHECK-LABEL: func2:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
 ; CHECK-NEXT:    adds.w.sx %s0, %s1, %s0
-; CHECK-NEXT:    sla.w.sx %s0, %s0, 16
-; CHECK-NEXT:    sra.w.sx %s0, %s0, 16
+; CHECK-NEXT:    sll %s0, %s0, 48
+; CHECK-NEXT:    sra.l %s0, %s0, 48
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = add i16 %1, %0
   ret i16 %3
@@ -25,6 +29,8 @@ define signext i16 @func2(i16 signext %0, i16 signext %1) {
 define i32 @func3(i32 %0, i32 %1) {
 ; CHECK-LABEL: func3:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
 ; CHECK-NEXT:    adds.w.sx %s0, %s1, %s0
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = add nsw i32 %1, %0
@@ -43,6 +49,8 @@ define i64 @func4(i64 %0, i64 %1) {
 define zeroext i8 @func6(i8 zeroext %0, i8 zeroext %1) {
 ; CHECK-LABEL: func6:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
 ; CHECK-NEXT:    adds.w.sx %s0, %s1, %s0
 ; CHECK-NEXT:    and %s0, %s0, (56)0
 ; CHECK-NEXT:    or %s11, 0, %s9
@@ -53,6 +61,8 @@ define zeroext i8 @func6(i8 zeroext %0, i8 zeroext %1) {
 define zeroext i16 @func7(i16 zeroext %0, i16 zeroext %1) {
 ; CHECK-LABEL: func7:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
 ; CHECK-NEXT:    adds.w.sx %s0, %s1, %s0
 ; CHECK-NEXT:    and %s0, %s0, (48)0
 ; CHECK-NEXT:    or %s11, 0, %s9
@@ -63,6 +73,8 @@ define zeroext i16 @func7(i16 zeroext %0, i16 zeroext %1) {
 define i32 @func8(i32 %0, i32 %1) {
 ; CHECK-LABEL: func8:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
 ; CHECK-NEXT:    adds.w.sx %s0, %s1, %s0
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = add i32 %1, %0
@@ -81,9 +93,10 @@ define i64 @func9(i64 %0, i64 %1) {
 define signext i8 @func13(i8 signext %0) {
 ; CHECK-LABEL: func13:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    adds.w.sx %s0, 5, %s0
-; CHECK-NEXT:    sla.w.sx %s0, %s0, 24
-; CHECK-NEXT:    sra.w.sx %s0, %s0, 24
+; CHECK-NEXT:    sll %s0, %s0, 56
+; CHECK-NEXT:    sra.l %s0, %s0, 56
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = add i8 %0, 5
   ret i8 %2
@@ -92,9 +105,10 @@ define signext i8 @func13(i8 signext %0) {
 define signext i16 @func14(i16 signext %0) {
 ; CHECK-LABEL: func14:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    adds.w.sx %s0, 5, %s0
-; CHECK-NEXT:    sla.w.sx %s0, %s0, 16
-; CHECK-NEXT:    sra.w.sx %s0, %s0, 16
+; CHECK-NEXT:    sll %s0, %s0, 48
+; CHECK-NEXT:    sra.l %s0, %s0, 48
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = add i16 %0, 5
   ret i16 %2
@@ -103,6 +117,7 @@ define signext i16 @func14(i16 signext %0) {
 define i32 @func15(i32 %0) {
 ; CHECK-LABEL: func15:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    adds.w.sx %s0, 5, %s0
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = add nsw i32 %0, 5
@@ -121,6 +136,7 @@ define i64 @func16(i64 %0) {
 define zeroext i8 @func18(i8 zeroext %0) {
 ; CHECK-LABEL: func18:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    adds.w.sx %s0, 5, %s0
 ; CHECK-NEXT:    and %s0, %s0, (56)0
 ; CHECK-NEXT:    or %s11, 0, %s9
@@ -131,6 +147,7 @@ define zeroext i8 @func18(i8 zeroext %0) {
 define zeroext i16 @func19(i16 zeroext %0) {
 ; CHECK-LABEL: func19:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    adds.w.sx %s0, 5, %s0
 ; CHECK-NEXT:    and %s0, %s0, (48)0
 ; CHECK-NEXT:    or %s11, 0, %s9
@@ -141,6 +158,7 @@ define zeroext i16 @func19(i16 zeroext %0) {
 define i32 @func20(i32 %0) {
 ; CHECK-LABEL: func20:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    adds.w.sx %s0, 5, %s0
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = add i32 %0, 5
diff --git a/llvm/test/CodeGen/VE/bitcast.ll b/llvm/test/CodeGen/VE/bitcast.ll
index dacc8f189e966..d7c09cd46b613 100644
--- a/llvm/test/CodeGen/VE/bitcast.ll
+++ b/llvm/test/CodeGen/VE/bitcast.ll
@@ -22,9 +22,8 @@ define dso_local double @bitcastl2d(i64 %x) {
 define dso_local float @bitcastw2f(i32 %x) {
 ; CHECK-LABEL: bitcastw2f:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:      # kill: def $sw0 killed $sw0 def $sx0
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    sll %s0, %s0, 32
-; CHECK-NEXT:      # kill: def $sf0 killed $sf0 killed $sx0
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = bitcast i32 %x to float
   ret float %r
@@ -34,9 +33,7 @@ define dso_local float @bitcastw2f(i32 %x) {
 define dso_local i32 @bitcastf2w(float %x) {
 ; CHECK-LABEL: bitcastf2w:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:      # kill: def $sf0 killed $sf0 def $sx0
 ; CHECK-NEXT:    sra.l %s0, %s0, 32
-; CHECK-NEXT:      # kill: def $sw0 killed $sw0 killed $sx0
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = bitcast float %x to i32
   ret i32 %r
diff --git a/llvm/test/CodeGen/VE/bitreverse.ll b/llvm/test/CodeGen/VE/bitreverse.ll
index fce969af657e2..af58afe38fd97 100644
--- a/llvm/test/CodeGen/VE/bitreverse.ll
+++ b/llvm/test/CodeGen/VE/bitreverse.ll
@@ -14,10 +14,8 @@ declare i64 @llvm.bitreverse.i64(i64)
 define i32 @func2(i32 %p) {
 ; CHECK-LABEL: func2:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 def $sx0
 ; CHECK-NEXT:    brv %s0, %s0
 ; CHECK-NEXT:    srl %s0, %s0, 32
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = tail call i32 @llvm.bitreverse.i32(i32 %p)
   ret i32 %r
@@ -28,10 +26,8 @@ declare i32 @llvm.bitreverse.i32(i32)
 define signext i16 @func3(i16 signext %p) {
 ; CHECK-LABEL: func3:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 def $sx0
 ; CHECK-NEXT:    brv %s0, %s0
 ; CHECK-NEXT:    sra.l %s0, %s0, 48
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = tail call i16 @llvm.bitreverse.i16(i16 %p)
   ret i16 %r
@@ -42,10 +38,8 @@ declare i16 @llvm.bitreverse.i16(i16)
 define signext i8 @func4(i8 signext %p) {
 ; CHECK-LABEL: func4:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 def $sx0
 ; CHECK-NEXT:    brv %s0, %s0
 ; CHECK-NEXT:    sra.l %s0, %s0, 56
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = tail call i8 @llvm.bitreverse.i8(i8 %p)
   ret i8 %r
@@ -65,10 +59,8 @@ define i64 @func5(i64 %p) {
 define i32 @func6(i32 %p) {
 ; CHECK-LABEL: func6:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 def $sx0
 ; CHECK-NEXT:    brv %s0, %s0
 ; CHECK-NEXT:    srl %s0, %s0, 32
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = tail call i32 @llvm.bitreverse.i32(i32 %p)
   ret i32 %r
@@ -77,10 +69,8 @@ define i32 @func6(i32 %p) {
 define zeroext i16 @func7(i16 zeroext %p) {
 ; CHECK-LABEL: func7:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 def $sx0
 ; CHECK-NEXT:    brv %s0, %s0
 ; CHECK-NEXT:    srl %s0, %s0, 48
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = tail call i16 @llvm.bitreverse.i16(i16 %p)
   ret i16 %r
@@ -89,10 +79,8 @@ define zeroext i16 @func7(i16 zeroext %p) {
 define zeroext i8 @func8(i8 zeroext %p) {
 ; CHECK-LABEL: func8:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 def $sx0
 ; CHECK-NEXT:    brv %s0, %s0
 ; CHECK-NEXT:    srl %s0, %s0, 56
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = tail call i8 @llvm.bitreverse.i8(i8 %p)
   ret i8 %r
diff --git a/llvm/test/CodeGen/VE/branch1.ll b/llvm/test/CodeGen/VE/branch1.ll
index c9f0a22c4c0a0..5561284c992e6 100644
--- a/llvm/test/CodeGen/VE/branch1.ll
+++ b/llvm/test/CodeGen/VE/branch1.ll
@@ -3,19 +3,22 @@
 define signext i8 @func1(i8 signext %a, i8 signext %b) {
 ; CHECK-LABEL: func1:
 ; CHECK:       .LBB{{[0-9]+}}_5:
-; CHECK-NEXT:    brle.w %s0, %s1, .LBB0_1
-; CHECK-NEXT:  # %bb.2:
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    brle.w %s0, %s1, .LBB{{[0-9]+}}_1
+; CHECK-NEXT:  # %bb.2: # %on.true
 ; CHECK-NEXT:    lea %s0, ret@lo
 ; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    lea.sl %s12, ret@hi(, %s0)
 ; CHECK-NEXT:    or %s0, 2, (0)1
 ; CHECK-NEXT:    bsic %s10, (, %s12)
-; CHECK-NEXT:    br.l.t .LBB0_3
-; CHECK:       .LBB{{[0-9]+}}_1:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    br.l.t .LBB{{[0-9]+}}_3
+; CHECK-NEXT:  .LBB{{[0-9]+}}_1:
 ; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK:       .LBB{{[0-9]+}}_3:
-; CHECK-NEXT:    sla.w.sx %s0, %s0, 24
-; CHECK-NEXT:    sra.w.sx %s0, %s0, 24
+; CHECK-NEXT:  .LBB{{[0-9]+}}_3: # %join
+; CHECK-NEXT:    sll %s0, %s0, 56
+; CHECK-NEXT:    sra.l %s0, %s0, 56
 ; CHECK-NEXT:    or %s11, 0, %s9
 entry:
   %cmp = icmp sgt i8 %a, %b
@@ -36,17 +39,20 @@ declare i32 @ret(i32)
 define i32 @func2(i16 signext %a, i16 signext %b) {
 ; CHECK-LABEL: func2:
 ; CHECK:       .LBB{{[0-9]+}}_5:
-; CHECK-NEXT:    brle.w %s0, %s1, .LBB1_1
-; CHECK-NEXT:  # %bb.2:
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    brle.w %s0, %s1, .LBB{{[0-9]+}}_1
+; CHECK-NEXT:  # %bb.2: # %on.true
 ; CHECK-NEXT:    lea %s0, ret@lo
 ; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    lea.sl %s12, ret@hi(, %s0)
 ; CHECK-NEXT:    or %s0, 2, (0)1
 ; CHECK-NEXT:    bsic %s10, (, %s12)
-; CHECK-NEXT:    br.l.t .LBB1_3
-; CHECK:       .LBB{{[0-9]+}}_1:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    br.l.t .LBB{{[0-9]+}}_3
+; CHECK-NEXT:  .LBB{{[0-9]+}}_1:
 ; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK:       .LBB{{[0-9]+}}_3:
+; CHECK-NEXT:  .LBB{{[0-9]+}}_3: # %join
 ; CHECK-NEXT:    or %s11, 0, %s9
 entry:
   %cmp = icmp sgt i16 %a, %b
@@ -64,17 +70,20 @@ join:
 define i32 @func3(i32 %a, i32 %b) {
 ; CHECK-LABEL: func3:
 ; CHECK:       .LBB{{[0-9]+}}_5:
-; CHECK-NEXT:    brle.w %s0, %s1, .LBB2_1
-; CHECK-NEXT:  # %bb.2:
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    brle.w %s0, %s1, .LBB{{[0-9]+}}_1
+; CHECK-NEXT:  # %bb.2: # %on.true
 ; CHECK-NEXT:    lea %s0, ret@lo
 ; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    lea.sl %s12, ret@hi(, %s0)
 ; CHECK-NEXT:    or %s0, 2, (0)1
 ; CHECK-NEXT:    bsic %s10, (, %s12)
-; CHECK-NEXT:    br.l.t .LBB2_3
-; CHECK:       .LBB{{[0-9]+}}_1:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    br.l.t .LBB{{[0-9]+}}_3
+; CHECK-NEXT:  .LBB{{[0-9]+}}_1:
 ; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK:       .LBB{{[0-9]+}}_3:
+; CHECK-NEXT:  .LBB{{[0-9]+}}_3: # %join
 ; CHECK-NEXT:    or %s11, 0, %s9
 entry:
   %cmp = icmp sgt i32 %a, %b
@@ -92,17 +101,18 @@ join:
 define i32 @func4(i64 %a, i64 %b) {
 ; CHECK-LABEL: func4:
 ; CHECK:       .LBB{{[0-9]+}}_5:
-; CHECK-NEXT:    brle.l %s0, %s1, .LBB3_1
-; CHECK-NEXT:  # %bb.2:
+; CHECK-NEXT:    brle.l %s0, %s1, .LBB{{[0-9]+}}_1
+; CHECK-NEXT:  # %bb.2: # %on.true
 ; CHECK-NEXT:    lea %s0, ret@lo
 ; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    lea.sl %s12, ret@hi(, %s0)
 ; CHECK-NEXT:    or %s0, 2, (0)1
 ; CHECK-NEXT:    bsic %s10, (, %s12)
-; CHECK-NEXT:    br.l.t .LBB3_3
-; CHECK:       .LBB{{[0-9]+}}_1:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    br.l.t .LBB{{[0-9]+}}_3
+; CHECK-NEXT:  .LBB{{[0-9]+}}_1:
 ; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK:       .LBB{{[0-9]+}}_3:
+; CHECK-NEXT:  .LBB{{[0-9]+}}_3: # %join
 ; CHECK-NEXT:    or %s11, 0, %s9
 entry:
   %cmp = icmp sgt i64 %a, %b
@@ -120,18 +130,21 @@ join:
 define i32 @func5(i8 zeroext %a, i8 zeroext %b) {
 ; CHECK-LABEL: func5:
 ; CHECK:       .LBB{{[0-9]+}}_5:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
 ; CHECK-NEXT:    cmpu.w %s0, %s1, %s0
-; CHECK-NEXT:    brle.w 0, %s0, .LBB4_1
-; CHECK-NEXT:  # %bb.2:
+; CHECK-NEXT:    brle.w 0, %s0, .LBB{{[0-9]+}}_1
+; CHECK-NEXT:  # %bb.2: # %on.true
 ; CHECK-NEXT:    lea %s0, ret@lo
 ; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    lea.sl %s12, ret@hi(, %s0)
 ; CHECK-NEXT:    or %s0, 2, (0)1
 ; CHECK-NEXT:    bsic %s10, (, %s12)
-; CHECK-NEXT:    br.l.t .LBB4_3
-; CHECK:       .LBB{{[0-9]+}}_1:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    br.l.t .LBB{{[0-9]+}}_3
+; CHECK-NEXT:  .LBB{{[0-9]+}}_1:
 ; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK:       .LBB{{[0-9]+}}_3:
+; CHECK-NEXT:  .LBB{{[0-9]+}}_3: # %join
 ; CHECK-NEXT:    or %s11, 0, %s9
 entry:
   %cmp = icmp ugt i8 %a, %b
@@ -149,18 +162,21 @@ join:
 define i32 @func6(i16 zeroext %a, i16 zeroext %b) {
 ; CHECK-LABEL: func6:
 ; CHECK:       .LBB{{[0-9]+}}_5:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
 ; CHECK-NEXT:    cmpu.w %s0, %s1, %s0
-; CHECK-NEXT:    brle.w 0, %s0, .LBB5_1
-; CHECK-NEXT:  # %bb.2:
+; CHECK-NEXT:    brle.w 0, %s0, .LBB{{[0-9]+}}_1
+; CHECK-NEXT:  # %bb.2: # %on.true
 ; CHECK-NEXT:    lea %s0, ret@lo
 ; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    lea.sl %s12, ret@hi(, %s0)
 ; CHECK-NEXT:    or %s0, 2, (0)1
 ; CHECK-NEXT:    bsic %s10, (, %s12)
-; CHECK-NEXT:    br.l.t .LBB5_3
-; CHECK:       .LBB{{[0-9]+}}_1:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    br.l.t .LBB{{[0-9]+}}_3
+; CHECK-NEXT:  .LBB{{[0-9]+}}_1:
 ; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK:       .LBB{{[0-9]+}}_3:
+; CHECK-NEXT:  .LBB{{[0-9]+}}_3: # %join
 ; CHECK-NEXT:    or %s11, 0, %s9
 entry:
   %cmp = icmp ugt i16 %a, %b
@@ -178,18 +194,21 @@ join:
 define i32 @func7(i32 %a, i32 %b) {
 ; CHECK-LABEL: func7:
 ; CHECK:       .LBB{{[0-9]+}}_5:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
 ; CHECK-NEXT:    cmpu.w %s0, %s1, %s0
-; CHECK-NEXT:    brle.w 0, %s0, .LBB6_1
-; CHECK-NEXT:  # %bb.2:
+; CHECK-NEXT:    brle.w 0, %s0, .LBB{{[0-9]+}}_1
+; CHECK-NEXT:  # %bb.2: # %on.true
 ; CHECK-NEXT:    lea %s0, ret@lo
 ; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    lea.sl %s12, ret@hi(, %s0)
 ; CHECK-NEXT:    or %s0, 2, (0)1
 ; CHECK-NEXT:    bsic %s10, (, %s12)
-; CHECK-NEXT:    br.l.t .LBB6_3
-; CHECK:       .LBB{{[0-9]+}}_1:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    br.l.t .LBB{{[0-9]+}}_3
+; CHECK-NEXT:  .LBB{{[0-9]+}}_1:
 ; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK:       .LBB{{[0-9]+}}_3:
+; CHECK-NEXT:  .LBB{{[0-9]+}}_3: # %join
 ; CHECK-NEXT:    or %s11, 0, %s9
 entry:
   %cmp = icmp ugt i32 %a, %b
@@ -207,17 +226,18 @@ join:
 define i32 @func8(float %a, float %b) {
 ; CHECK-LABEL: func8:
 ; CHECK:       .LBB{{[0-9]+}}_5:
-; CHECK-NEXT:    brlenan.s %s0, %s1, .LBB7_1
-; CHECK-NEXT:  # %bb.2:
+; CHECK-NEXT:    brlenan.s %s0, %s1, .LBB{{[0-9]+}}_1
+; CHECK-NEXT:  # %bb.2: # %on.true
 ; CHECK-NEXT:    lea %s0, ret@lo
 ; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    lea.sl %s12, ret@hi(, %s0)
 ; CHECK-NEXT:    or %s0, 2, (0)1
 ; CHECK-NEXT:    bsic %s10, (, %s12)
-; CHECK-NEXT:    br.l.t .LBB7_3
-; CHECK:       .LBB{{[0-9]+}}_1:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    br.l.t .LBB{{[0-9]+}}_3
+; CHECK-NEXT:  .LBB{{[0-9]+}}_1:
 ; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK:       .LBB{{[0-9]+}}_3:
+; CHECK-NEXT:  .LBB{{[0-9]+}}_3: # %join
 ; CHECK-NEXT:    or %s11, 0, %s9
 entry:
   %cmp = fcmp ogt float %a, %b
@@ -235,17 +255,18 @@ join:
 define i32 @func9(double %a, double %b) {
 ; CHECK-LABEL: func9:
 ; CHECK:       .LBB{{[0-9]+}}_5:
-; CHECK-NEXT:    brlenan.d %s0, %s1, .LBB8_1
-; CHECK-NEXT:  # %bb.2:
+; CHECK-NEXT:    brlenan.d %s0, %s1, .LBB{{[0-9]+}}_1
+; CHECK-NEXT:  # %bb.2: # %on.true
 ; CHECK-NEXT:    lea %s0, ret@lo
 ; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    lea.sl %s12, ret@hi(, %s0)
 ; CHECK-NEXT:    or %s0, 2, (0)1
 ; CHECK-NEXT:    bsic %s10, (, %s12)
-; CHECK-NEXT:    br.l.t .LBB8_3
-; CHECK:       .LBB{{[0-9]+}}_1:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    br.l.t .LBB{{[0-9]+}}_3
+; CHECK-NEXT:  .LBB{{[0-9]+}}_1:
 ; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK:       .LBB{{[0-9]+}}_3:
+; CHECK-NEXT:  .LBB{{[0-9]+}}_3: # %join
 ; CHECK-NEXT:    or %s11, 0, %s9
 entry:
   %cmp = fcmp ogt double %a, %b
@@ -264,17 +285,18 @@ define i32 @func10(double %a, double %b) {
 ; CHECK-LABEL: func10:
 ; CHECK:       .LBB{{[0-9]+}}_5:
 ; CHECK-NEXT:    lea.sl %s1, 1075052544
-; CHECK-NEXT:    brlenan.d %s0, %s1, .LBB9_1
-; CHECK-NEXT:  # %bb.2:
+; CHECK-NEXT:    brlenan.d %s0, %s1, .LBB{{[0-9]+}}_1
+; CHECK-NEXT:  # %bb.2: # %on.true
 ; CHECK-NEXT:    lea %s0, ret@lo
 ; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    lea.sl %s12, ret@hi(, %s0)
 ; CHECK-NEXT:    or %s0, 2, (0)1
 ; CHECK-NEXT:    bsic %s10, (, %s12)
-; CHECK-NEXT:    br.l.t .LBB9_3
-; CHECK:       .LBB{{[0-9]+}}_1:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    br.l.t .LBB{{[0-9]+}}_3
+; CHECK-NEXT:  .LBB{{[0-9]+}}_1:
 ; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK:       .LBB{{[0-9]+}}_3:
+; CHECK-NEXT:  .LBB{{[0-9]+}}_3: # %join
 ; CHECK-NEXT:    or %s11, 0, %s9
 entry:
   %cmp = fcmp ogt double %a, 5.000000e+00
diff --git a/llvm/test/CodeGen/VE/bswap.ll b/llvm/test/CodeGen/VE/bswap.ll
index 274085462856f..39569d8889c5a 100644
--- a/llvm/test/CodeGen/VE/bswap.ll
+++ b/llvm/test/CodeGen/VE/bswap.ll
@@ -14,9 +14,8 @@ declare i64 @llvm.bswap.i64(i64)
 define i32 @func2(i32 %p) {
 ; CHECK-LABEL: func2:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 def $sx0
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    bswp %s0, %s0, 1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = tail call i32 @llvm.bswap.i32(i32 %p)
   ret i32 %r
@@ -27,9 +26,12 @@ declare i32 @llvm.bswap.i32(i32)
 define signext i16 @func3(i16 signext %p) {
 ; CHECK-LABEL: func3:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 def $sx0
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    bswp %s0, %s0, 1
-; CHECK-NEXT:    sra.w.sx %s0, %s0, 16
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    srl %s0, %s0, 16
+; CHECK-NEXT:    sll %s0, %s0, 48
+; CHECK-NEXT:    sra.l %s0, %s0, 48
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = tail call i16 @llvm.bswap.i16(i16 %p)
   ret i16 %r
@@ -49,9 +51,8 @@ define i64 @func4(i64 %p) {
 define i32 @func5(i32 %p) {
 ; CHECK-LABEL: func5:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 def $sx0
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    bswp %s0, %s0, 1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = tail call i32 @llvm.bswap.i32(i32 %p)
   ret i32 %r
@@ -60,11 +61,11 @@ define i32 @func5(i32 %p) {
 define zeroext i16 @func6(i16 zeroext %p) {
 ; CHECK-LABEL: func6:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 def $sx0
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    bswp %s0, %s0, 1
 ; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    srl %s0, %s0, 16
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    adds.w.zx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = tail call i16 @llvm.bswap.i16(i16 %p)
   ret i16 %r
diff --git a/llvm/test/CodeGen/VE/call.ll b/llvm/test/CodeGen/VE/call.ll
index 9e9f22b6d8233..386a5fd74bf97 100644
--- a/llvm/test/CodeGen/VE/call.ll
+++ b/llvm/test/CodeGen/VE/call.ll
@@ -20,7 +20,7 @@ define i32 @stack_call_int() {
 ; CHECK-LABEL: stack_call_int:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    or %s0, 10, (0)1
-; CHECK-NEXT:    stl %s0, 248(, %s11)
+; CHECK-NEXT:    st %s0, 248(, %s11)
 ; CHECK-NEXT:    or %s34, 9, (0)1
 ; CHECK-NEXT:    lea %s0, stack_callee_int@lo
 ; CHECK-NEXT:    and %s0, %s0, (32)0
@@ -33,7 +33,7 @@ define i32 @stack_call_int() {
 ; CHECK-NEXT:    or %s5, 6, (0)1
 ; CHECK-NEXT:    or %s6, 7, (0)1
 ; CHECK-NEXT:    or %s7, 8, (0)1
-; CHECK-NEXT:    stl %s34, 240(, %s11)
+; CHECK-NEXT:    st %s34, 240(, %s11)
 ; CHECK-NEXT:    bsic %s10, (, %s12)
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = tail call i32 @stack_callee_int(i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10)
@@ -46,7 +46,7 @@ define i32 @stack_call_int_szext() {
 ; CHECK-LABEL: stack_call_int_szext:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    or %s0, -1, (0)1
-; CHECK-NEXT:    stl %s0, 248(, %s11)
+; CHECK-NEXT:    st %s0, 248(, %s11)
 ; CHECK-NEXT:    lea %s34, 65535
 ; CHECK-NEXT:    lea %s1, stack_callee_int_szext@lo
 ; CHECK-NEXT:    and %s1, %s1, (32)0
@@ -58,7 +58,7 @@ define i32 @stack_call_int_szext() {
 ; CHECK-NEXT:    or %s5, 6, (0)1
 ; CHECK-NEXT:    or %s6, 7, (0)1
 ; CHECK-NEXT:    or %s7, 8, (0)1
-; CHECK-NEXT:    stl %s34, 240(, %s11)
+; CHECK-NEXT:    st %s34, 240(, %s11)
 ; CHECK-NEXT:    bsic %s10, (, %s12)
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = tail call i32 @stack_callee_int_szext(i1 -1, i8 -1, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i16 -1, i8 -1)
@@ -70,12 +70,9 @@ declare i32 @stack_callee_int_szext(i1 signext, i8 zeroext, i32, i32, i32, i32,
 define float @stack_call_float() {
 ; CHECK-LABEL: stack_call_float:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    lea %s0, 1092616192
-; CHECK-NEXT:    stl %s0, 252(, %s11)
-; CHECK-NEXT:    lea %s34, 1091567616
-; CHECK-NEXT:    lea %s0, stack_callee_float@lo
-; CHECK-NEXT:    and %s0, %s0, (32)0
-; CHECK-NEXT:    lea.sl %s12, stack_callee_float@hi(, %s0)
+; CHECK-NEXT:    lea.sl %s0, 1092616192
+; CHECK-NEXT:    st %s0, 248(, %s11)
+; CHECK-NEXT:    lea.sl %s34, 1091567616
 ; CHECK-NEXT:    lea.sl %s0, 1065353216
 ; CHECK-NEXT:    lea.sl %s1, 1073741824
 ; CHECK-NEXT:    lea.sl %s2, 1077936128
@@ -84,15 +81,10 @@ define float @stack_call_float() {
 ; CHECK-NEXT:    lea.sl %s5, 1086324736
 ; CHECK-NEXT:    lea.sl %s6, 1088421888
 ; CHECK-NEXT:    lea.sl %s7, 1090519040
-; CHECK-NEXT:    stl %s34, 244(, %s11)
-; CHECK-NEXT:    # kill: def $sf0 killed $sf0 killed $sx0
-; CHECK-NEXT:    # kill: def $sf1 killed $sf1 killed $sx1
-; CHECK-NEXT:    # kill: def $sf2 killed $sf2 killed $sx2
-; CHECK-NEXT:    # kill: def $sf3 killed $sf3 killed $sx3
-; CHECK-NEXT:    # kill: def $sf4 killed $sf4 killed $sx4
-; CHECK-NEXT:    # kill: def $sf5 killed $sf5 killed $sx5
-; CHECK-NEXT:    # kill: def $sf6 killed $sf6 killed $sx6
-; CHECK-NEXT:    # kill: def $sf7 killed $sf7 killed $sx7
+; CHECK-NEXT:    lea %s35, stack_callee_float@lo
+; CHECK-NEXT:    and %s35, %s35, (32)0
+; CHECK-NEXT:    lea.sl %s12, stack_callee_float@hi(, %s35)
+; CHECK-NEXT:    st %s34, 240(, %s11)
 ; CHECK-NEXT:    bsic %s10, (, %s12)
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = tail call float @stack_callee_float(float 1.0, float 2.0, float 3.0, float 4.0, float 5.0, float 6.0, float 7.0, float 8.0, float 9.0, float 10.0)
@@ -104,11 +96,11 @@ declare float @stack_callee_float(float, float, float, float, float, float, floa
 define float @stack_call_float2(float %p0) {
 ; CHECK-LABEL: stack_call_float2:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    stu %s0, 252(, %s11)
+; CHECK-NEXT:    st %s0, 248(, %s11)
 ; CHECK-NEXT:    lea %s1, stack_callee_float@lo
 ; CHECK-NEXT:    and %s1, %s1, (32)0
 ; CHECK-NEXT:    lea.sl %s12, stack_callee_float@hi(, %s1)
-; CHECK-NEXT:    stu %s0, 244(, %s11)
+; CHECK-NEXT:    st %s0, 240(, %s11)
 ; CHECK-NEXT:    or %s1, 0, %s0
 ; CHECK-NEXT:    or %s2, 0, %s0
 ; CHECK-NEXT:    or %s3, 0, %s0
diff --git a/llvm/test/CodeGen/VE/cast.ll b/llvm/test/CodeGen/VE/cast.ll
index 51126e123ac67..07ad969a1bd3e 100644
--- a/llvm/test/CodeGen/VE/cast.ll
+++ b/llvm/test/CodeGen/VE/cast.ll
@@ -4,6 +4,7 @@ define i32 @i() {
 ; CHECK-LABEL: i:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    lea %s0, -2147483648
+; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    or %s11, 0, %s9
   ret i32 -2147483648
 }
@@ -12,6 +13,7 @@ define i32 @ui() {
 ; CHECK-LABEL: ui:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    lea %s0, -2147483648
+; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    or %s11, 0, %s9
   ret i32 -2147483648
 }
@@ -37,6 +39,7 @@ define signext i8 @d2c(double %x) {
 ; CHECK-LABEL: d2c:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    cvt.w.d.sx.rz %s0, %s0
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = fptosi double %x to i8
   ret i8 %r
@@ -46,6 +49,7 @@ define zeroext i8 @d2uc(double %x) {
 ; CHECK-LABEL: d2uc:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    cvt.w.d.sx.rz %s0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = fptoui double %x to i8
   ret i8 %r
@@ -55,6 +59,7 @@ define signext i16 @d2s(double %x) {
 ; CHECK-LABEL: d2s:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    cvt.w.d.sx.rz %s0, %s0
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = fptosi double %x to i16
   ret i16 %r
@@ -64,6 +69,7 @@ define zeroext i16 @d2us(double %x) {
 ; CHECK-LABEL: d2us:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    cvt.w.d.sx.rz %s0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = fptoui double %x to i16
   ret i16 %r
@@ -82,7 +88,6 @@ define i32 @d2ui(double %x) {
 ; CHECK-LABEL: d2ui:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    cvt.l.d.rz %s0, %s0
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = fptoui double %x to i32
   ret i32 %r
@@ -133,6 +138,7 @@ define signext i8 @f2c(float %x) {
 ; CHECK-LABEL: f2c:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    cvt.w.s.sx.rz %s0, %s0
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = fptosi float %x to i8
   ret i8 %r
@@ -142,6 +148,7 @@ define zeroext i8 @f2uc(float %x) {
 ; CHECK-LABEL: f2uc:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    cvt.w.s.sx.rz %s0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = fptoui float %x to i8
   ret i8 %r
@@ -151,6 +158,7 @@ define signext i16 @f2s(float %x) {
 ; CHECK-LABEL: f2s:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    cvt.w.s.sx.rz %s0, %s0
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = fptosi float %x to i16
   ret i16 %r
@@ -160,6 +168,7 @@ define zeroext i16 @f2us(float %x) {
 ; CHECK-LABEL: f2us:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    cvt.w.s.sx.rz %s0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = fptoui float %x to i16
   ret i16 %r
@@ -179,7 +188,6 @@ define i32 @f2ui(float %x) {
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    cvt.d.s %s0, %s0
 ; CHECK-NEXT:    cvt.l.d.rz %s0, %s0
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = fptoui float %x to i32
   ret i32 %r
@@ -234,7 +242,6 @@ define signext i8 @ll2c(i64 %0) {
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    sll %s0, %s0, 56
 ; CHECK-NEXT:    sra.l %s0, %s0, 56
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = trunc i64 %0 to i8
   ret i8 %2
@@ -254,7 +261,6 @@ define signext i16 @ll2s(i64 %0) {
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    sll %s0, %s0, 48
 ; CHECK-NEXT:    sra.l %s0, %s0, 48
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = trunc i64 %0 to i16
   ret i16 %2
@@ -272,7 +278,6 @@ define zeroext i16 @ll2us(i64 %0) {
 define i32 @ll2i(i64 %0) {
 ; CHECK-LABEL: ll2i:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = trunc i64 %0 to i32
   ret i32 %2
@@ -281,7 +286,6 @@ define i32 @ll2i(i64 %0) {
 define i32 @ll2ui(i64 %0) {
 ; CHECK-LABEL: ll2ui:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = trunc i64 %0 to i32
   ret i32 %2
@@ -325,7 +329,6 @@ define signext i8 @ull2c(i64 %0) {
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    sll %s0, %s0, 56
 ; CHECK-NEXT:    sra.l %s0, %s0, 56
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = trunc i64 %0 to i8
   ret i8 %2
@@ -345,7 +348,6 @@ define signext i16 @ull2s(i64 %0) {
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    sll %s0, %s0, 48
 ; CHECK-NEXT:    sra.l %s0, %s0, 48
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = trunc i64 %0 to i16
   ret i16 %2
@@ -363,7 +365,6 @@ define zeroext i16 @ull2us(i64 %0) {
 define i32 @ull2i(i64 %0) {
 ; CHECK-LABEL: ull2i:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = trunc i64 %0 to i32
   ret i32 %2
@@ -372,7 +373,6 @@ define i32 @ull2i(i64 %0) {
 define i32 @ull2ui(i64 %0) {
 ; CHECK-LABEL: ull2ui:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = trunc i64 %0 to i32
   ret i32 %2
@@ -433,8 +433,8 @@ define double @ull2d(i64 %x) {
 define signext i8 @i2c(i32 %0) {
 ; CHECK-LABEL: i2c:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    sla.w.sx %s0, %s0, 24
-; CHECK-NEXT:    sra.w.sx %s0, %s0, 24
+; CHECK-NEXT:    sll %s0, %s0, 56
+; CHECK-NEXT:    sra.l %s0, %s0, 56
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = trunc i32 %0 to i8
   ret i8 %2
@@ -452,8 +452,8 @@ define zeroext i8 @i2uc(i32 %0) {
 define signext i16 @i2s(i32 %0) {
 ; CHECK-LABEL: i2s:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    sla.w.sx %s0, %s0, 16
-; CHECK-NEXT:    sra.w.sx %s0, %s0, 16
+; CHECK-NEXT:    sll %s0, %s0, 48
+; CHECK-NEXT:    sra.l %s0, %s0, 48
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = trunc i32 %0 to i16
   ret i16 %2
@@ -503,6 +503,7 @@ define i64 @i2ull(i32 %0) {
 define float @i2f(i32 %x) {
 ; CHECK-LABEL: i2f:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    cvt.s.w %s0, %s0
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = sitofp i32 %x to float
@@ -512,6 +513,7 @@ define float @i2f(i32 %x) {
 define double @i2d(i32 %x) {
 ; CHECK-LABEL: i2d:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    cvt.d.w %s0, %s0
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = sitofp i32 %x to double
@@ -521,8 +523,8 @@ define double @i2d(i32 %x) {
 define signext i8 @ui2c(i32 %0) {
 ; CHECK-LABEL: ui2c:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    sla.w.sx %s0, %s0, 24
-; CHECK-NEXT:    sra.w.sx %s0, %s0, 24
+; CHECK-NEXT:    sll %s0, %s0, 56
+; CHECK-NEXT:    sra.l %s0, %s0, 56
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = trunc i32 %0 to i8
   ret i8 %2
@@ -540,8 +542,8 @@ define zeroext i8 @ui2uc(i32 %0) {
 define signext i16 @ui2s(i32 %0) {
 ; CHECK-LABEL: ui2s:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    sla.w.sx %s0, %s0, 16
-; CHECK-NEXT:    sra.w.sx %s0, %s0, 16
+; CHECK-NEXT:    sll %s0, %s0, 48
+; CHECK-NEXT:    sra.l %s0, %s0, 48
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = trunc i32 %0 to i16
   ret i16 %2
@@ -573,7 +575,7 @@ define i32 @ui2ui(i32 returned %0) {
 define i64 @ui2ll(i32 %0) {
 ; CHECK-LABEL: ui2ll:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    adds.w.zx %s0, %s0, (0)1
+; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = zext i32 %0 to i64
   ret i64 %2
@@ -582,7 +584,7 @@ define i64 @ui2ll(i32 %0) {
 define i64 @ui2ull(i32 %0) {
 ; CHECK-LABEL: ui2ull:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    adds.w.zx %s0, %s0, (0)1
+; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = zext i32 %0 to i64
   ret i64 %2
@@ -591,7 +593,7 @@ define i64 @ui2ull(i32 %0) {
 define float @ui2f(i32 %x) {
 ; CHECK-LABEL: ui2f:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    adds.w.zx %s0, %s0, (0)1
+; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    cvt.d.l %s0, %s0
 ; CHECK-NEXT:    cvt.s.d %s0, %s0
 ; CHECK-NEXT:    or %s11, 0, %s9
@@ -602,7 +604,7 @@ define float @ui2f(i32 %x) {
 define double @ui2d(i32 %x) {
 ; CHECK-LABEL: ui2d:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    adds.w.zx %s0, %s0, (0)1
+; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    cvt.d.l %s0, %s0
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = uitofp i32 %x to double
@@ -612,8 +614,8 @@ define double @ui2d(i32 %x) {
 define signext i8 @s2c(i16 signext %0) {
 ; CHECK-LABEL: s2c:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    sla.w.sx %s0, %s0, 24
-; CHECK-NEXT:    sra.w.sx %s0, %s0, 24
+; CHECK-NEXT:    sll %s0, %s0, 56
+; CHECK-NEXT:    sra.l %s0, %s0, 56
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = trunc i16 %0 to i8
   ret i8 %2
@@ -662,7 +664,6 @@ define i32 @s2ui(i16 signext %0) {
 define i64 @s2ll(i16 signext %0) {
 ; CHECK-LABEL: s2ll:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = sext i16 %0 to i64
   ret i64 %2
@@ -671,7 +672,6 @@ define i64 @s2ll(i16 signext %0) {
 define i64 @s2ull(i16 signext %0) {
 ; CHECK-LABEL: s2ull:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = sext i16 %0 to i64
   ret i64 %2
@@ -680,6 +680,7 @@ define i64 @s2ull(i16 signext %0) {
 define float @s2f(i16 signext %x) {
 ; CHECK-LABEL: s2f:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    cvt.s.w %s0, %s0
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = sitofp i16 %x to float
@@ -689,6 +690,7 @@ define float @s2f(i16 signext %x) {
 define double @s2d(i16 signext %x) {
 ; CHECK-LABEL: s2d:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    cvt.d.w %s0, %s0
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = sitofp i16 %x to double
@@ -698,8 +700,8 @@ define double @s2d(i16 signext %x) {
 define signext i8 @us2c(i16 zeroext %0) {
 ; CHECK-LABEL: us2c:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    sla.w.sx %s0, %s0, 24
-; CHECK-NEXT:    sra.w.sx %s0, %s0, 24
+; CHECK-NEXT:    sll %s0, %s0, 56
+; CHECK-NEXT:    sra.l %s0, %s0, 56
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = trunc i16 %0 to i8
   ret i8 %2
@@ -717,8 +719,8 @@ define zeroext i8 @us2uc(i16 zeroext %0) {
 define signext i16 @us2s(i16 returned zeroext %0) {
 ; CHECK-LABEL: us2s:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    sla.w.sx %s0, %s0, 16
-; CHECK-NEXT:    sra.w.sx %s0, %s0, 16
+; CHECK-NEXT:    sll %s0, %s0, 48
+; CHECK-NEXT:    sra.l %s0, %s0, 48
 ; CHECK-NEXT:    or %s11, 0, %s9
   ret i16 %0
 }
@@ -749,7 +751,6 @@ define i32 @us2ui(i16 zeroext %0) {
 define i64 @us2ll(i16 zeroext %0) {
 ; CHECK-LABEL: us2ll:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    adds.w.zx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = zext i16 %0 to i64
   ret i64 %2
@@ -758,7 +759,6 @@ define i64 @us2ll(i16 zeroext %0) {
 define i64 @us2ull(i16 zeroext %0) {
 ; CHECK-LABEL: us2ull:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    adds.w.zx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = zext i16 %0 to i64
   ret i64 %2
@@ -767,6 +767,7 @@ define i64 @us2ull(i16 zeroext %0) {
 define float @us2f(i16 zeroext %x) {
 ; CHECK-LABEL: us2f:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    cvt.s.w %s0, %s0
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = uitofp i16 %x to float
@@ -776,6 +777,7 @@ define float @us2f(i16 zeroext %x) {
 define double @us2d(i16 zeroext %x) {
 ; CHECK-LABEL: us2d:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    cvt.d.w %s0, %s0
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = uitofp i16 %x to double
@@ -833,7 +835,6 @@ define i32 @c2ui(i8 signext %0) {
 define i64 @c2ll(i8 signext %0) {
 ; CHECK-LABEL: c2ll:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = sext i8 %0 to i64
   ret i64 %2
@@ -842,7 +843,6 @@ define i64 @c2ll(i8 signext %0) {
 define i64 @c2ull(i8 signext %0) {
 ; CHECK-LABEL: c2ull:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = sext i8 %0 to i64
   ret i64 %2
@@ -851,6 +851,7 @@ define i64 @c2ull(i8 signext %0) {
 define float @c2f(i8 signext %x) {
 ; CHECK-LABEL: c2f:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    cvt.s.w %s0, %s0
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = sitofp i8 %x to float
@@ -860,6 +861,7 @@ define float @c2f(i8 signext %x) {
 define double @c2d(i8 signext %x) {
 ; CHECK-LABEL: c2d:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    cvt.d.w %s0, %s0
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = sitofp i8 %x to double
@@ -869,8 +871,8 @@ define double @c2d(i8 signext %x) {
 define signext i8 @uc2c(i8 returned zeroext %0) {
 ; CHECK-LABEL: uc2c:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    sla.w.sx %s0, %s0, 24
-; CHECK-NEXT:    sra.w.sx %s0, %s0, 24
+; CHECK-NEXT:    sll %s0, %s0, 56
+; CHECK-NEXT:    sra.l %s0, %s0, 56
 ; CHECK-NEXT:    or %s11, 0, %s9
   ret i8 %0
 }
@@ -917,7 +919,6 @@ define i32 @uc2ui(i8 zeroext %0) {
 define i64 @uc2ll(i8 zeroext %0) {
 ; CHECK-LABEL: uc2ll:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    adds.w.zx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = zext i8 %0 to i64
   ret i64 %2
@@ -926,7 +927,6 @@ define i64 @uc2ll(i8 zeroext %0) {
 define i64 @uc2ull(i8 zeroext %0) {
 ; CHECK-LABEL: uc2ull:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    adds.w.zx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = zext i8 %0 to i64
   ret i64 %2
@@ -935,6 +935,7 @@ define i64 @uc2ull(i8 zeroext %0) {
 define float @uc2f(i8 zeroext %x) {
 ; CHECK-LABEL: uc2f:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    cvt.s.w %s0, %s0
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = uitofp i8 %x to float
@@ -944,6 +945,7 @@ define float @uc2f(i8 zeroext %x) {
 define double @uc2d(i8 zeroext %x) {
 ; CHECK-LABEL: uc2d:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    cvt.d.w %s0, %s0
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = uitofp i8 %x to double
@@ -976,7 +978,6 @@ define signext i8 @i1282c(i128 %0) {
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    sll %s0, %s0, 56
 ; CHECK-NEXT:    sra.l %s0, %s0, 56
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = trunc i128 %0 to i8
   ret i8 %2
@@ -988,7 +989,6 @@ define signext i8 @ui1282c(i128 %0) {
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    sll %s0, %s0, 56
 ; CHECK-NEXT:    sra.l %s0, %s0, 56
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = trunc i128 %0 to i8
   ret i8 %2
@@ -1020,7 +1020,6 @@ define signext i16 @i1282s(i128 %0) {
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    sll %s0, %s0, 48
 ; CHECK-NEXT:    sra.l %s0, %s0, 48
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = trunc i128 %0 to i16
   ret i16 %2
@@ -1032,7 +1031,6 @@ define signext i16 @ui1282s(i128 %0) {
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    sll %s0, %s0, 48
 ; CHECK-NEXT:    sra.l %s0, %s0, 48
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = trunc i128 %0 to i16
   ret i16 %2
@@ -1062,7 +1060,6 @@ define zeroext i16 @ui1282us(i128 %0) {
 define i32 @i1282i(i128 %0) {
 ; CHECK-LABEL: i1282i:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = trunc i128 %0 to i32
   ret i32 %2
@@ -1072,7 +1069,6 @@ define i32 @i1282i(i128 %0) {
 define i32 @ui1282i(i128 %0) {
 ; CHECK-LABEL: ui1282i:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = trunc i128 %0 to i32
   ret i32 %2
@@ -1082,7 +1078,6 @@ define i32 @ui1282i(i128 %0) {
 define i32 @i1282ui(i128 %0) {
 ; CHECK-LABEL: i1282ui:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = trunc i128 %0 to i32
   ret i32 %2
@@ -1092,7 +1087,6 @@ define i32 @i1282ui(i128 %0) {
 define i32 @ui1282ui(i128 %0) {
 ; CHECK-LABEL: ui1282ui:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = trunc i128 %0 to i32
   ret i32 %2
@@ -1216,7 +1210,7 @@ define i128 @i2ui128(i32 %0) {
 define i128 @ui2i128(i32 %0) {
 ; CHECK-LABEL: ui2i128:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    adds.w.zx %s0, %s0, (0)1
+; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    or %s1, 0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = zext i32 %0 to i128
@@ -1227,7 +1221,7 @@ define i128 @ui2i128(i32 %0) {
 define i128 @ui2ui128(i32 %0) {
 ; CHECK-LABEL: ui2ui128:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    adds.w.zx %s0, %s0, (0)1
+; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    or %s1, 0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = zext i32 %0 to i128
@@ -1238,7 +1232,6 @@ define i128 @ui2ui128(i32 %0) {
 define i128 @s2i128(i16 signext %0) {
 ; CHECK-LABEL: s2i128:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    sra.l %s1, %s0, 63
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = sext i16 %0 to i128
@@ -1249,7 +1242,6 @@ define i128 @s2i128(i16 signext %0) {
 define i128 @s2ui128(i16 signext %0) {
 ; CHECK-LABEL: s2ui128:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    sra.l %s1, %s0, 63
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = sext i16 %0 to i128
@@ -1260,7 +1252,6 @@ define i128 @s2ui128(i16 signext %0) {
 define i128 @us2i128(i16 zeroext %0) {
 ; CHECK-LABEL: us2i128:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    adds.w.zx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s1, 0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = zext i16 %0 to i128
@@ -1271,7 +1262,6 @@ define i128 @us2i128(i16 zeroext %0) {
 define i128 @us2ui128(i16 zeroext %0) {
 ; CHECK-LABEL: us2ui128:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    adds.w.zx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s1, 0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = zext i16 %0 to i128
@@ -1282,7 +1272,6 @@ define i128 @us2ui128(i16 zeroext %0) {
 define i128 @c2i128(i8 signext %0) {
 ; CHECK-LABEL: c2i128:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    sra.l %s1, %s0, 63
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = sext i8 %0 to i128
@@ -1293,7 +1282,6 @@ define i128 @c2i128(i8 signext %0) {
 define i128 @char2ui128(i8 signext %0) {
 ; CHECK-LABEL: char2ui128:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    sra.l %s1, %s0, 63
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = sext i8 %0 to i128
@@ -1304,7 +1292,6 @@ define i128 @char2ui128(i8 signext %0) {
 define i128 @uc2i128(i8 zeroext %0) {
 ; CHECK-LABEL: uc2i128:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    adds.w.zx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s1, 0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = zext i8 %0 to i128
@@ -1315,7 +1302,6 @@ define i128 @uc2i128(i8 zeroext %0) {
 define i128 @uc2ui128(i8 zeroext %0) {
 ; CHECK-LABEL: uc2ui128:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    adds.w.zx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s1, 0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = zext i8 %0 to i128
diff --git a/llvm/test/CodeGen/VE/constants.ll b/llvm/test/CodeGen/VE/constants.ll
index b7a43605ae347..baebf5ef3621e 100644
--- a/llvm/test/CodeGen/VE/constants.ll
+++ b/llvm/test/CodeGen/VE/constants.ll
@@ -304,7 +304,6 @@ define float @m5f32() {
 ; CHECK-LABEL: m5f32:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    lea.sl %s0, -1063256064
-; CHECK-NEXT:    # kill: def $sf0 killed $sf0 killed $sx0
 ; CHECK-NEXT:    or %s11, 0, %s9
   ret float -5.000000e+00
 }
@@ -321,7 +320,6 @@ define float @p2p3f32() {
 ; CHECK-LABEL: p2p3f32:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    lea.sl %s0, 1075000115
-; CHECK-NEXT:    # kill: def $sf0 killed $sf0 killed $sx0
 ; CHECK-NEXT:    or %s11, 0, %s9
   ret float 0x4002666660000000 ; 2.3
 }
@@ -339,7 +337,6 @@ define float @p128p3f32() {
 ; CHECK-LABEL: p128p3f32:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    lea.sl %s0, 1124093133
-; CHECK-NEXT:    # kill: def $sf0 killed $sf0 killed $sx0
 ; CHECK-NEXT:    or %s11, 0, %s9
   ret float 0x40600999A0000000 ; 128.3
 }
diff --git a/llvm/test/CodeGen/VE/ctlz.ll b/llvm/test/CodeGen/VE/ctlz.ll
index de44790014a07..5853851ac9c97 100644
--- a/llvm/test/CodeGen/VE/ctlz.ll
+++ b/llvm/test/CodeGen/VE/ctlz.ll
@@ -14,10 +14,9 @@ declare i64 @llvm.ctlz.i64(i64, i1)
 define i32 @func2(i32 %p) {
 ; CHECK-LABEL: func2:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 def $sx0
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    sll %s0, %s0, 32
 ; CHECK-NEXT:    ldz %s0, %s0
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = tail call i32 @llvm.ctlz.i32(i32 %p, i1 true)
   ret i32 %r
diff --git a/llvm/test/CodeGen/VE/ctpop.ll b/llvm/test/CodeGen/VE/ctpop.ll
index 3d25909ab25cb..8fee9104ed077 100644
--- a/llvm/test/CodeGen/VE/ctpop.ll
+++ b/llvm/test/CodeGen/VE/ctpop.ll
@@ -14,10 +14,9 @@ declare i64 @llvm.ctpop.i64(i64 %p)
 define i32 @func2(i32 %p) {
 ; CHECK-LABEL: func2:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 def $sx0
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    pcnt %s0, %s0
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = tail call i32 @llvm.ctpop.i32(i32 %p)
   ret i32 %r
@@ -29,9 +28,7 @@ define i16 @func3(i16 %p) {
 ; CHECK-LABEL: func3:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    and %s0, %s0, (48)0
-; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    pcnt %s0, %s0
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = tail call i16 @llvm.ctpop.i16(i16 %p)
   ret i16 %r
@@ -43,9 +40,7 @@ define i8 @func4(i8 %p) {
 ; CHECK-LABEL: func4:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    and %s0, %s0, (56)0
-; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    pcnt %s0, %s0
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = tail call i8 @llvm.ctpop.i8(i8 %p)
   ret i8 %r
diff --git a/llvm/test/CodeGen/VE/cttz.ll b/llvm/test/CodeGen/VE/cttz.ll
index 4b79a0f988e86..46bb52d29102b 100644
--- a/llvm/test/CodeGen/VE/cttz.ll
+++ b/llvm/test/CodeGen/VE/cttz.ll
@@ -16,12 +16,12 @@ declare i64 @llvm.cttz.i64(i64, i1)
 define i32 @func2(i32 %p) {
 ; CHECK-LABEL: func2:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    adds.w.sx %s1, -1, %s0
 ; CHECK-NEXT:    xor %s0, -1, %s0
 ; CHECK-NEXT:    and %s0, %s0, %s1
 ; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    pcnt %s0, %s0
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = tail call i32 @llvm.cttz.i32(i32 %p, i1 true)
   ret i32 %r
@@ -32,12 +32,12 @@ declare i32 @llvm.cttz.i32(i32, i1)
 define i16 @func3(i16 %p) {
 ; CHECK-LABEL: func3:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    adds.w.sx %s1, -1, %s0
 ; CHECK-NEXT:    xor %s0, -1, %s0
 ; CHECK-NEXT:    and %s0, %s0, %s1
 ; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    pcnt %s0, %s0
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = tail call i16 @llvm.cttz.i16(i16 %p, i1 true)
   ret i16 %r
@@ -48,12 +48,12 @@ declare i16 @llvm.cttz.i16(i16, i1)
 define i8 @func4(i8 %p) {
 ; CHECK-LABEL: func4:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    adds.w.sx %s1, -1, %s0
 ; CHECK-NEXT:    xor %s0, -1, %s0
 ; CHECK-NEXT:    and %s0, %s0, %s1
 ; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    pcnt %s0, %s0
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = tail call i8 @llvm.cttz.i8(i8 %p, i1 true)
   ret i8 %r
diff --git a/llvm/test/CodeGen/VE/div.ll b/llvm/test/CodeGen/VE/div.ll
index 8d4a0ddd2c156..ed434a9a3c7b5 100644
--- a/llvm/test/CodeGen/VE/div.ll
+++ b/llvm/test/CodeGen/VE/div.ll
@@ -14,6 +14,8 @@ define i64 @divi64(i64 %a, i64 %b) {
 define i32 @divi32(i32 %a, i32 %b) {
 ; CHECK-LABEL: divi32:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    divs.w.sx %s0, %s0, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = sdiv i32 %a, %b
@@ -34,6 +36,8 @@ define i64 @divu64(i64 %a, i64 %b) {
 define i32 @divu32(i32 %a, i32 %b) {
 ; CHECK-LABEL: divu32:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    divu.w %s0, %s0, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = udiv i32 %a, %b
@@ -44,9 +48,11 @@ define i32 @divu32(i32 %a, i32 %b) {
 define signext i16 @divi16(i16 signext %a, i16 signext %b) {
 ; CHECK-LABEL: divi16:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    divs.w.sx %s0, %s0, %s1
-; CHECK-NEXT:    sla.w.sx %s0, %s0, 16
-; CHECK-NEXT:    sra.w.sx %s0, %s0, 16
+; CHECK-NEXT:    sll %s0, %s0, 48
+; CHECK-NEXT:    sra.l %s0, %s0, 48
 ; CHECK-NEXT:    or %s11, 0, %s9
   %a32 = sext i16 %a to i32
   %b32 = sext i16 %b to i32
@@ -59,7 +65,10 @@ define signext i16 @divi16(i16 signext %a, i16 signext %b) {
 define zeroext i16 @divu16(i16 zeroext %a, i16 zeroext %b) {
 ; CHECK-LABEL: divu16:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    divu.w %s0, %s0, %s1
+; CHECK-NEXT:    adds.w.zx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = udiv i16 %a, %b
   ret i16 %r
@@ -69,9 +78,11 @@ define zeroext i16 @divu16(i16 zeroext %a, i16 zeroext %b) {
 define signext i8 @divi8(i8 signext %a, i8 signext %b) {
 ; CHECK-LABEL: divi8:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    divs.w.sx %s0, %s0, %s1
-; CHECK-NEXT:    sla.w.sx %s0, %s0, 24
-; CHECK-NEXT:    sra.w.sx %s0, %s0, 24
+; CHECK-NEXT:    sll %s0, %s0, 56
+; CHECK-NEXT:    sra.l %s0, %s0, 56
 ; CHECK-NEXT:    or %s11, 0, %s9
   %a32 = sext i8 %a to i32
   %b32 = sext i8 %b to i32
@@ -84,7 +95,10 @@ define signext i8 @divi8(i8 signext %a, i8 signext %b) {
 define zeroext i8 @divu8(i8 zeroext %a, i8 zeroext %b) {
 ; CHECK-LABEL: divu8:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    divu.w %s0, %s0, %s1
+; CHECK-NEXT:    adds.w.zx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = udiv i8 %a, %b
   ret i8 %r
@@ -104,6 +118,7 @@ define i64 @divi64ri(i64 %a, i64 %b) {
 define i32 @divi32ri(i32 %a, i32 %b) {
 ; CHECK-LABEL: divi32ri:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    divs.w.sx %s0, %s0, (62)0
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = sdiv i32 %a, 3
@@ -124,6 +139,7 @@ define i64 @divu64ri(i64 %a, i64 %b) {
 define i32 @divu32ri(i32 %a, i32 %b) {
 ; CHECK-LABEL: divu32ri:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    divu.w %s0, %s0, (62)0
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = udiv i32 %a, 3
@@ -144,7 +160,8 @@ define i64 @divi64li(i64 %a, i64 %b) {
 define i32 @divi32li(i32 %a, i32 %b) {
 ; CHECK-LABEL: divi32li:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    divs.w.sx %s0, 3, %s1
+; CHECK-NEXT:    adds.w.sx %s0, %s1, (0)1
+; CHECK-NEXT:    divs.w.sx %s0, 3, %s0
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = sdiv i32 3, %b
   ret i32 %r
@@ -164,7 +181,8 @@ define i64 @divu64li(i64 %a, i64 %b) {
 define i32 @divu32li(i32 %a, i32 %b) {
 ; CHECK-LABEL: divu32li:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    divu.w %s0, 3, %s1
+; CHECK-NEXT:    adds.w.sx %s0, %s1, (0)1
+; CHECK-NEXT:    divu.w %s0, 3, %s0
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = udiv i32 3, %b
   ret i32 %r
diff --git a/llvm/test/CodeGen/VE/fp_to_int.ll b/llvm/test/CodeGen/VE/fp_to_int.ll
index 9a1a7e35c1190..d9b1926ea9d5d 100644
--- a/llvm/test/CodeGen/VE/fp_to_int.ll
+++ b/llvm/test/CodeGen/VE/fp_to_int.ll
@@ -5,6 +5,7 @@ define signext i8 @f2c(float %a) {
 ; CHECK-LABEL: f2c:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    cvt.w.s.sx.rz %s0, %s0
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
 entry:
   %conv = fptosi float %a to i8
@@ -16,6 +17,7 @@ define signext i16 @f2s(float %a) {
 ; CHECK-LABEL: f2s:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    cvt.w.s.sx.rz %s0, %s0
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
 entry:
   %conv = fptosi float %a to i16
@@ -50,6 +52,7 @@ define zeroext i8 @f2uc(float %a) {
 ; CHECK-LABEL: f2uc:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    cvt.w.s.sx.rz %s0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
 entry:
   %conv = fptoui float %a to i8
@@ -61,6 +64,7 @@ define zeroext i16 @f2us(float %a) {
 ; CHECK-LABEL: f2us:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    cvt.w.s.sx.rz %s0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
 entry:
   %conv = fptoui float %a to i16
@@ -73,7 +77,6 @@ define i32 @f2ui(float %a) {
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    cvt.d.s %s0, %s0
 ; CHECK-NEXT:    cvt.l.d.rz %s0, %s0
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
 entry:
   %conv = fptoui float %a to i32
@@ -105,6 +108,7 @@ define signext i8 @d2c(double %a) {
 ; CHECK-LABEL: d2c:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    cvt.w.d.sx.rz %s0, %s0
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
 entry:
   %conv = fptosi double %a to i8
@@ -116,6 +120,7 @@ define signext i16 @d2s(double %a) {
 ; CHECK-LABEL: d2s:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    cvt.w.d.sx.rz %s0, %s0
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
 entry:
   %conv = fptosi double %a to i16
@@ -149,6 +154,7 @@ define zeroext i8 @d2uc(double %a) {
 ; CHECK-LABEL: d2uc:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    cvt.w.d.sx.rz %s0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
 entry:
   %conv = fptoui double %a to i8
@@ -160,6 +166,7 @@ define zeroext i16 @d2us(double %a) {
 ; CHECK-LABEL: d2us:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    cvt.w.d.sx.rz %s0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
 entry:
   %conv = fptoui double %a to i16
@@ -171,7 +178,6 @@ define i32 @d2ui(double %a) {
 ; CHECK-LABEL: d2ui:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    cvt.l.d.rz %s0, %s0
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
 entry:
   %conv = fptoui double %a to i32
diff --git a/llvm/test/CodeGen/VE/int_to_fp.ll b/llvm/test/CodeGen/VE/int_to_fp.ll
index 5069a0ca0d7d5..2e850142e2e9f 100644
--- a/llvm/test/CodeGen/VE/int_to_fp.ll
+++ b/llvm/test/CodeGen/VE/int_to_fp.ll
@@ -4,6 +4,7 @@
 define float @c2f(i8 signext %a) {
 ; CHECK-LABEL: c2f:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    cvt.s.w %s0, %s0
 ; CHECK-NEXT:    or %s11, 0, %s9
 entry:
@@ -15,6 +16,7 @@ entry:
 define float @s2f(i16 signext %a) {
 ; CHECK-LABEL: s2f:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    cvt.s.w %s0, %s0
 ; CHECK-NEXT:    or %s11, 0, %s9
 entry:
@@ -26,6 +28,7 @@ entry:
 define float @i2f(i32 %a) {
 ; CHECK-LABEL: i2f:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    cvt.s.w %s0, %s0
 ; CHECK-NEXT:    or %s11, 0, %s9
 entry:
@@ -49,6 +52,7 @@ entry:
 define float @uc2f(i8 zeroext %a) {
 ; CHECK-LABEL: uc2f:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    cvt.s.w %s0, %s0
 ; CHECK-NEXT:    or %s11, 0, %s9
 entry:
@@ -60,6 +64,7 @@ entry:
 define float @us2f(i16 zeroext %a) {
 ; CHECK-LABEL: us2f:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    cvt.s.w %s0, %s0
 ; CHECK-NEXT:    or %s11, 0, %s9
 entry:
@@ -71,7 +76,7 @@ entry:
 define float @ui2f(i32 %a) {
 ; CHECK-LABEL: ui2f:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    adds.w.zx %s0, %s0, (0)1
+; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    cvt.d.l %s0, %s0
 ; CHECK-NEXT:    cvt.s.d %s0, %s0
 ; CHECK-NEXT:    or %s11, 0, %s9
@@ -106,6 +111,7 @@ entry:
 define double @c2d(i8 signext %a) {
 ; CHECK-LABEL: c2d:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    cvt.d.w %s0, %s0
 ; CHECK-NEXT:    or %s11, 0, %s9
 entry:
@@ -117,6 +123,7 @@ entry:
 define double @s2d(i16 signext %a) {
 ; CHECK-LABEL: s2d:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    cvt.d.w %s0, %s0
 ; CHECK-NEXT:    or %s11, 0, %s9
 entry:
@@ -128,6 +135,7 @@ entry:
 define double @i2d(i32 %a) {
 ; CHECK-LABEL: i2d:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    cvt.d.w %s0, %s0
 ; CHECK-NEXT:    or %s11, 0, %s9
 entry:
@@ -150,6 +158,7 @@ entry:
 define double @uc2d(i8 zeroext %a) {
 ; CHECK-LABEL: uc2d:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    cvt.d.w %s0, %s0
 ; CHECK-NEXT:    or %s11, 0, %s9
 entry:
@@ -161,6 +170,7 @@ entry:
 define double @us2d(i16 zeroext %a) {
 ; CHECK-LABEL: us2d:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    cvt.d.w %s0, %s0
 ; CHECK-NEXT:    or %s11, 0, %s9
 entry:
@@ -172,7 +182,7 @@ entry:
 define double @ui2d(i32 %a) {
 ; CHECK-LABEL: ui2d:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    adds.w.zx %s0, %s0, (0)1
+; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    cvt.d.l %s0, %s0
 ; CHECK-NEXT:    or %s11, 0, %s9
 entry:
diff --git a/llvm/test/CodeGen/VE/left_shift.ll b/llvm/test/CodeGen/VE/left_shift.ll
index d568846974dd9..fa595a916e529 100644
--- a/llvm/test/CodeGen/VE/left_shift.ll
+++ b/llvm/test/CodeGen/VE/left_shift.ll
@@ -3,9 +3,11 @@
 define signext i8 @func1(i8 signext %0, i8 signext %1) {
 ; CHECK-LABEL: func1:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    sla.w.sx %s0, %s0, %s1
-; CHECK-NEXT:    sla.w.sx %s0, %s0, 24
-; CHECK-NEXT:    sra.w.sx %s0, %s0, 24
+; CHECK-NEXT:    sll %s0, %s0, 56
+; CHECK-NEXT:    sra.l %s0, %s0, 56
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = sext i8 %0 to i32
   %4 = sext i8 %1 to i32
@@ -17,9 +19,11 @@ define signext i8 @func1(i8 signext %0, i8 signext %1) {
 define signext i16 @func2(i16 signext %0, i16 signext %1) {
 ; CHECK-LABEL: func2:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    sla.w.sx %s0, %s0, %s1
-; CHECK-NEXT:    sla.w.sx %s0, %s0, 16
-; CHECK-NEXT:    sra.w.sx %s0, %s0, 16
+; CHECK-NEXT:    sll %s0, %s0, 48
+; CHECK-NEXT:    sra.l %s0, %s0, 48
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = sext i16 %0 to i32
   %4 = sext i16 %1 to i32
@@ -31,6 +35,8 @@ define signext i16 @func2(i16 signext %0, i16 signext %1) {
 define i32 @func3(i32 %0, i32 %1) {
 ; CHECK-LABEL: func3:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    sla.w.sx %s0, %s0, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = shl i32 %0, %1
@@ -50,6 +56,8 @@ define i64 @func4(i64 %0, i64 %1) {
 define zeroext i8 @func6(i8 zeroext %0, i8 zeroext %1) {
 ; CHECK-LABEL: func6:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    sla.w.sx %s0, %s0, %s1
 ; CHECK-NEXT:    and %s0, %s0, (56)0
 ; CHECK-NEXT:    or %s11, 0, %s9
@@ -63,6 +71,8 @@ define zeroext i8 @func6(i8 zeroext %0, i8 zeroext %1) {
 define zeroext i16 @func7(i16 zeroext %0, i16 zeroext %1) {
 ; CHECK-LABEL: func7:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    sla.w.sx %s0, %s0, %s1
 ; CHECK-NEXT:    and %s0, %s0, (48)0
 ; CHECK-NEXT:    or %s11, 0, %s9
@@ -76,6 +86,8 @@ define zeroext i16 @func7(i16 zeroext %0, i16 zeroext %1) {
 define i32 @func8(i32 %0, i32 %1) {
 ; CHECK-LABEL: func8:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    sla.w.sx %s0, %s0, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = shl i32 %0, %1
@@ -95,9 +107,10 @@ define i64 @func9(i64 %0, i64 %1) {
 define signext i8 @func11(i8 signext %0) {
 ; CHECK-LABEL: func11:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    sla.w.sx %s0, %s0, 5
-; CHECK-NEXT:    sla.w.sx %s0, %s0, 24
-; CHECK-NEXT:    sra.w.sx %s0, %s0, 24
+; CHECK-NEXT:    sll %s0, %s0, 56
+; CHECK-NEXT:    sra.l %s0, %s0, 56
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = shl i8 %0, 5
   ret i8 %2
@@ -106,9 +119,10 @@ define signext i8 @func11(i8 signext %0) {
 define signext i16 @func12(i16 signext %0) {
 ; CHECK-LABEL: func12:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    sla.w.sx %s0, %s0, 5
-; CHECK-NEXT:    sla.w.sx %s0, %s0, 16
-; CHECK-NEXT:    sra.w.sx %s0, %s0, 16
+; CHECK-NEXT:    sll %s0, %s0, 48
+; CHECK-NEXT:    sra.l %s0, %s0, 48
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = shl i16 %0, 5
   ret i16 %2
@@ -117,6 +131,7 @@ define signext i16 @func12(i16 signext %0) {
 define i32 @func13(i32 %0) {
 ; CHECK-LABEL: func13:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    sla.w.sx %s0, %s0, 5
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = shl i32 %0, 5
@@ -135,8 +150,10 @@ define i64 @func14(i64 %0) {
 define zeroext i8 @func16(i8 zeroext %0) {
 ; CHECK-LABEL: func16:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    sla.w.sx %s0, %s0, 5
-; CHECK-NEXT:    and %s0, %s0, (56)0
+; CHECK-NEXT:    lea %s1, 224
+; CHECK-NEXT:    and %s0, %s0, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = shl i8 %0, 5
   ret i8 %2
@@ -145,8 +162,10 @@ define zeroext i8 @func16(i8 zeroext %0) {
 define zeroext i16 @func17(i16 zeroext %0) {
 ; CHECK-LABEL: func17:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    sla.w.sx %s0, %s0, 5
-; CHECK-NEXT:    and %s0, %s0, (48)0
+; CHECK-NEXT:    lea %s1, 65504
+; CHECK-NEXT:    and %s0, %s0, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = shl i16 %0, 5
   ret i16 %2
@@ -155,6 +174,7 @@ define zeroext i16 @func17(i16 zeroext %0) {
 define i32 @func18(i32 %0) {
 ; CHECK-LABEL: func18:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    sla.w.sx %s0, %s0, 5
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = shl i32 %0, 5
diff --git a/llvm/test/CodeGen/VE/load_off.ll b/llvm/test/CodeGen/VE/load_off.ll
index cc3da7a3a1cd5..2ebd616e533cc 100644
--- a/llvm/test/CodeGen/VE/load_off.ll
+++ b/llvm/test/CodeGen/VE/load_off.ll
@@ -118,7 +118,7 @@ define zeroext i32 @loadi32z() {
 ; CHECK-NEXT:    lea %s0, bufi32+8@lo
 ; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    lea.sl %s0, bufi32+8@hi(, %s0)
-; CHECK-NEXT:    ldl.sx %s0, (, %s0)
+; CHECK-NEXT:    ldl.zx %s0, (, %s0)
 ; CHECK-NEXT:    or %s11, 0, %s9
 entry:
   %0 = load i32, i32* getelementptr inbounds ([3 x i32], [3 x i32]* @bufi32, i64 0, i64 2), align 4
diff --git a/llvm/test/CodeGen/VE/max.ll b/llvm/test/CodeGen/VE/max.ll
index 67f68b02b4510..2c342faa1f052 100644
--- a/llvm/test/CodeGen/VE/max.ll
+++ b/llvm/test/CodeGen/VE/max.ll
@@ -69,8 +69,6 @@ define float @max2f32(float, float) {
 define float @maxuf32(float, float) {
 ; CHECK-LABEL: maxuf32:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sf1 killed $sf1 def $sx1
-; CHECK-NEXT:    # kill: def $sf0 killed $sf0 def $sx0
 ; CHECK-NEXT:    fcmp.s %s2, %s0, %s1
 ; CHECK-NEXT:    cmov.s.gtnan %s1, %s0, %s2
 ; CHECK-NEXT:    or %s0, 0, %s1
@@ -83,8 +81,6 @@ define float @maxuf32(float, float) {
 define float @max2uf32(float, float) {
 ; CHECK-LABEL: max2uf32:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sf1 killed $sf1 def $sx1
-; CHECK-NEXT:    # kill: def $sf0 killed $sf0 def $sx0
 ; CHECK-NEXT:    fcmp.s %s2, %s0, %s1
 ; CHECK-NEXT:    cmov.s.genan %s1, %s0, %s2
 ; CHECK-NEXT:    or %s0, 0, %s1
@@ -141,6 +137,8 @@ define i64 @max2u64(i64, i64) {
 define i32 @maxi32(i32, i32) {
 ; CHECK-LABEL: maxi32:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    maxs.w.sx %s0, %s0, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = icmp sgt i32 %0, %1
@@ -151,6 +149,8 @@ define i32 @maxi32(i32, i32) {
 define i32 @max2i32(i32, i32) {
 ; CHECK-LABEL: max2i32:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    maxs.w.sx %s0, %s0, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = icmp sge i32 %0, %1
@@ -161,11 +161,10 @@ define i32 @max2i32(i32, i32) {
 define i32 @maxu32(i32, i32) {
 ; CHECK-LABEL: maxu32:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sw1 killed $sw1 def $sx1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 def $sx0
-; CHECK-NEXT:    cmpu.w %s2, %s0, %s1
-; CHECK-NEXT:    cmov.w.gt %s1, %s0, %s2
-; CHECK-NEXT:    or %s0, 0, %s1
+; CHECK-NEXT:    adds.w.sx %s2, %s0, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s1, (0)1
+; CHECK-NEXT:    cmpu.w %s1, %s2, %s0
+; CHECK-NEXT:    cmov.w.gt %s0, %s2, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = icmp ugt i32 %0, %1
   %4 = select i1 %3, i32 %0, i32 %1
@@ -175,11 +174,10 @@ define i32 @maxu32(i32, i32) {
 define i32 @max2u32(i32, i32) {
 ; CHECK-LABEL: max2u32:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sw1 killed $sw1 def $sx1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 def $sx0
-; CHECK-NEXT:    cmpu.w %s2, %s0, %s1
-; CHECK-NEXT:    cmov.w.ge %s1, %s0, %s2
-; CHECK-NEXT:    or %s0, 0, %s1
+; CHECK-NEXT:    adds.w.sx %s2, %s0, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s1, (0)1
+; CHECK-NEXT:    cmpu.w %s1, %s2, %s0
+; CHECK-NEXT:    cmov.w.ge %s0, %s2, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = icmp uge i32 %0, %1
   %4 = select i1 %3, i32 %0, i32 %1
diff --git a/llvm/test/CodeGen/VE/min.ll b/llvm/test/CodeGen/VE/min.ll
index 3e28f757d9e98..dd6ad8460c806 100644
--- a/llvm/test/CodeGen/VE/min.ll
+++ b/llvm/test/CodeGen/VE/min.ll
@@ -67,8 +67,6 @@ define float @min2f32(float, float) {
 define float @minuf32(float, float) {
 ; CHECK-LABEL: minuf32:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sf1 killed $sf1 def $sx1
-; CHECK-NEXT:    # kill: def $sf0 killed $sf0 def $sx0
 ; CHECK-NEXT:    fcmp.s %s2, %s0, %s1
 ; CHECK-NEXT:    cmov.s.ltnan %s1, %s0, %s2
 ; CHECK-NEXT:    or %s0, 0, %s1
@@ -81,8 +79,6 @@ define float @minuf32(float, float) {
 define float @min2uf32(float, float) {
 ; CHECK-LABEL: min2uf32:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sf1 killed $sf1 def $sx1
-; CHECK-NEXT:    # kill: def $sf0 killed $sf0 def $sx0
 ; CHECK-NEXT:    fcmp.s %s2, %s0, %s1
 ; CHECK-NEXT:    cmov.s.lenan %s1, %s0, %s2
 ; CHECK-NEXT:    or %s0, 0, %s1
@@ -139,6 +135,8 @@ define i64 @min2u64(i64, i64) {
 define i32 @mini32(i32, i32) {
 ; CHECK-LABEL: mini32:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    mins.w.sx %s0, %s0, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = icmp slt i32 %0, %1
@@ -149,6 +147,8 @@ define i32 @mini32(i32, i32) {
 define i32 @min2i32(i32, i32) {
 ; CHECK-LABEL: min2i32:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    mins.w.sx %s0, %s0, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = icmp sle i32 %0, %1
@@ -159,11 +159,10 @@ define i32 @min2i32(i32, i32) {
 define i32 @minu32(i32, i32) {
 ; CHECK-LABEL: minu32:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sw1 killed $sw1 def $sx1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 def $sx0
-; CHECK-NEXT:    cmpu.w %s2, %s0, %s1
-; CHECK-NEXT:    cmov.w.lt %s1, %s0, %s2
-; CHECK-NEXT:    or %s0, 0, %s1
+; CHECK-NEXT:    adds.w.sx %s2, %s0, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s1, (0)1
+; CHECK-NEXT:    cmpu.w %s1, %s2, %s0
+; CHECK-NEXT:    cmov.w.lt %s0, %s2, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = icmp ult i32 %0, %1
   %4 = select i1 %3, i32 %0, i32 %1
@@ -173,11 +172,10 @@ define i32 @minu32(i32, i32) {
 define i32 @min2u32(i32, i32) {
 ; CHECK-LABEL: min2u32:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sw1 killed $sw1 def $sx1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 def $sx0
-; CHECK-NEXT:    cmpu.w %s2, %s0, %s1
-; CHECK-NEXT:    cmov.w.le %s1, %s0, %s2
-; CHECK-NEXT:    or %s0, 0, %s1
+; CHECK-NEXT:    adds.w.sx %s2, %s0, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s1, (0)1
+; CHECK-NEXT:    cmpu.w %s1, %s2, %s0
+; CHECK-NEXT:    cmov.w.le %s0, %s2, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = icmp ule i32 %0, %1
   %4 = select i1 %3, i32 %0, i32 %1
@@ -187,10 +185,11 @@ define i32 @min2u32(i32, i32) {
 define zeroext i1 @mini1(i1 zeroext, i1 zeroext) {
 ; CHECK-LABEL: mini1:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sw1 killed $sw1 def $sx1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
 ; CHECK-NEXT:    and %s2, %s1, %s0
 ; CHECK-NEXT:    cmov.w.ne %s2, %s1, %s0
-; CHECK-NEXT:    or %s0, 0, %s2
+; CHECK-NEXT:    adds.w.zx %s0, %s2, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = xor i1 %0, true
   %4 = and i1 %3, %1
diff --git a/llvm/test/CodeGen/VE/multiply.ll b/llvm/test/CodeGen/VE/multiply.ll
index dabb6cf85d12f..83b7a67ff453b 100644
--- a/llvm/test/CodeGen/VE/multiply.ll
+++ b/llvm/test/CodeGen/VE/multiply.ll
@@ -3,9 +3,11 @@
 define signext i8 @func1(i8 signext %a, i8 signext %b) {
 ; CHECK-LABEL: func1:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
 ; CHECK-NEXT:    muls.w.sx %s0, %s1, %s0
-; CHECK-NEXT:    sla.w.sx %s0, %s0, 24
-; CHECK-NEXT:    sra.w.sx %s0, %s0, 24
+; CHECK-NEXT:    sll %s0, %s0, 56
+; CHECK-NEXT:    sra.l %s0, %s0, 56
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = mul i8 %b, %a
   ret i8 %r
@@ -14,9 +16,11 @@ define signext i8 @func1(i8 signext %a, i8 signext %b) {
 define signext i16 @func2(i16 signext %a, i16 signext %b) {
 ; CHECK-LABEL: func2:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
 ; CHECK-NEXT:    muls.w.sx %s0, %s1, %s0
-; CHECK-NEXT:    sla.w.sx %s0, %s0, 16
-; CHECK-NEXT:    sra.w.sx %s0, %s0, 16
+; CHECK-NEXT:    sll %s0, %s0, 48
+; CHECK-NEXT:    sra.l %s0, %s0, 48
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = mul i16 %b, %a
   ret i16 %r
@@ -25,6 +29,8 @@ define signext i16 @func2(i16 signext %a, i16 signext %b) {
 define i32 @func3(i32 %a, i32 %b) {
 ; CHECK-LABEL: func3:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
 ; CHECK-NEXT:    muls.w.sx %s0, %s1, %s0
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = mul nsw i32 %b, %a
@@ -43,6 +49,8 @@ define i64 @func4(i64 %a, i64 %b) {
 define zeroext i8 @func5(i8 zeroext %a, i8 zeroext %b) {
 ; CHECK-LABEL: func5:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
 ; CHECK-NEXT:    muls.w.sx %s0, %s1, %s0
 ; CHECK-NEXT:    and %s0, %s0, (56)0
 ; CHECK-NEXT:    or %s11, 0, %s9
@@ -53,6 +61,8 @@ define zeroext i8 @func5(i8 zeroext %a, i8 zeroext %b) {
 define zeroext i16 @func6(i16 zeroext %a, i16 zeroext %b) {
 ; CHECK-LABEL: func6:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
 ; CHECK-NEXT:    muls.w.sx %s0, %s1, %s0
 ; CHECK-NEXT:    and %s0, %s0, (48)0
 ; CHECK-NEXT:    or %s11, 0, %s9
@@ -63,6 +73,8 @@ define zeroext i16 @func6(i16 zeroext %a, i16 zeroext %b) {
 define i32 @func7(i32 %a, i32 %b) {
 ; CHECK-LABEL: func7:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
 ; CHECK-NEXT:    muls.w.sx %s0, %s1, %s0
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = mul i32 %b, %a
@@ -81,9 +93,10 @@ define i64 @func8(i64 %a, i64 %b) {
 define signext i8 @func9(i8 signext %a) {
 ; CHECK-LABEL: func9:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    muls.w.sx %s0, 5, %s0
-; CHECK-NEXT:    sla.w.sx %s0, %s0, 24
-; CHECK-NEXT:    sra.w.sx %s0, %s0, 24
+; CHECK-NEXT:    sll %s0, %s0, 56
+; CHECK-NEXT:    sra.l %s0, %s0, 56
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = mul i8 %a, 5
   ret i8 %r
@@ -92,9 +105,10 @@ define signext i8 @func9(i8 signext %a) {
 define signext i16 @func10(i16 signext %a) {
 ; CHECK-LABEL: func10:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    muls.w.sx %s0, 5, %s0
-; CHECK-NEXT:    sla.w.sx %s0, %s0, 16
-; CHECK-NEXT:    sra.w.sx %s0, %s0, 16
+; CHECK-NEXT:    sll %s0, %s0, 48
+; CHECK-NEXT:    sra.l %s0, %s0, 48
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = mul i16 %a, 5
   ret i16 %r
@@ -103,6 +117,7 @@ define signext i16 @func10(i16 signext %a) {
 define i32 @func11(i32 %a) {
 ; CHECK-LABEL: func11:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    muls.w.sx %s0, 5, %s0
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = mul nsw i32 %a, 5
@@ -121,6 +136,7 @@ define i64 @func12(i64 %a) {
 define zeroext i8 @func13(i8 zeroext %a) {
 ; CHECK-LABEL: func13:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    muls.w.sx %s0, 5, %s0
 ; CHECK-NEXT:    and %s0, %s0, (56)0
 ; CHECK-NEXT:    or %s11, 0, %s9
@@ -131,6 +147,7 @@ define zeroext i8 @func13(i8 zeroext %a) {
 define zeroext i16 @func14(i16 zeroext %a) {
 ; CHECK-LABEL: func14:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    muls.w.sx %s0, 5, %s0
 ; CHECK-NEXT:    and %s0, %s0, (48)0
 ; CHECK-NEXT:    or %s11, 0, %s9
@@ -141,6 +158,7 @@ define zeroext i16 @func14(i16 zeroext %a) {
 define i32 @func15(i32 %a) {
 ; CHECK-LABEL: func15:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    muls.w.sx %s0, 5, %s0
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = mul i32 %a, 5
@@ -159,6 +177,7 @@ define i64 @func16(i64 %a) {
 define i32 @func17(i32 %a) {
 ; CHECK-LABEL: func17:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    sla.w.sx %s0, %s0, 31
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = shl i32 %a, 31
diff --git a/llvm/test/CodeGen/VE/nnd.ll b/llvm/test/CodeGen/VE/nnd.ll
index aea10d4834cdd..aedb85050f301 100644
--- a/llvm/test/CodeGen/VE/nnd.ll
+++ b/llvm/test/CodeGen/VE/nnd.ll
@@ -3,8 +3,11 @@
 define signext i8 @func8s(i8 signext %a, i8 signext %b) {
 ; CHECK-LABEL: func8s:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    xor %s0, -1, %s0
 ; CHECK-NEXT:    and %s0, %s0, %s1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %not = xor i8 %a, -1
   %res = and i8 %not, %b
@@ -14,8 +17,11 @@ define signext i8 @func8s(i8 signext %a, i8 signext %b) {
 define zeroext i8 @func8z(i8 zeroext %a, i8 zeroext %b) {
 ; CHECK-LABEL: func8z:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    xor %s0, -1, %s0
 ; CHECK-NEXT:    and %s0, %s1, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %not = xor i8 %a, -1
   %res = and i8 %b, %not
@@ -25,8 +31,10 @@ define zeroext i8 @func8z(i8 zeroext %a, i8 zeroext %b) {
 define signext i8 @funci8s(i8 signext %a) {
 ; CHECK-LABEL: funci8s:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    xor %s0, -1, %s0
 ; CHECK-NEXT:    and %s0, 5, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %not = xor i8 %a, -1
   %res = and i8 %not, 5
@@ -36,6 +44,7 @@ define signext i8 @funci8s(i8 signext %a) {
 define zeroext i8 @funci8z(i8 zeroext %a) {
 ; CHECK-LABEL: funci8z:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    xor %s0, -1, %s0
 ; CHECK-NEXT:    lea %s1, 251
 ; CHECK-NEXT:    and %s0, %s0, %s1
@@ -48,8 +57,11 @@ define zeroext i8 @funci8z(i8 zeroext %a) {
 define signext i16 @func16s(i16 signext %a, i16 signext %b) {
 ; CHECK-LABEL: func16s:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    xor %s0, -1, %s0
 ; CHECK-NEXT:    and %s0, %s0, %s1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %not = xor i16 %a, -1
   %res = and i16 %not, %b
@@ -59,8 +71,11 @@ define signext i16 @func16s(i16 signext %a, i16 signext %b) {
 define zeroext i16 @func16z(i16 zeroext %a, i16 zeroext %b) {
 ; CHECK-LABEL: func16z:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    xor %s0, -1, %s0
 ; CHECK-NEXT:    and %s0, %s1, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %not = xor i16 %a, -1
   %res = and i16 %b, %not
@@ -70,7 +85,9 @@ define zeroext i16 @func16z(i16 zeroext %a, i16 zeroext %b) {
 define signext i16 @funci16s(i16 signext %a) {
 ; CHECK-LABEL: funci16s:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    xor %s0, -1, %s0
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %not = xor i16 %a, -1
   %res = and i16 %not, 65535
@@ -80,8 +97,10 @@ define signext i16 @funci16s(i16 signext %a) {
 define zeroext i16 @funci16z(i16 zeroext %a) {
 ; CHECK-LABEL: funci16z:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    xor %s0, -1, %s0
 ; CHECK-NEXT:    and %s0, %s0, (52)0
+; CHECK-NEXT:    adds.w.zx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %not = xor i16 %a, -1
   %res = and i16 4095, %not
@@ -91,8 +110,11 @@ define zeroext i16 @funci16z(i16 zeroext %a) {
 define signext i32 @func32s(i32 signext %a, i32 signext %b) {
 ; CHECK-LABEL: func32s:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    xor %s0, -1, %s0
 ; CHECK-NEXT:    and %s0, %s0, %s1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %not = xor i32 %a, -1
   %res = and i32 %not, %b
@@ -102,8 +124,11 @@ define signext i32 @func32s(i32 signext %a, i32 signext %b) {
 define zeroext i32 @func32z(i32 zeroext %a, i32 zeroext %b) {
 ; CHECK-LABEL: func32z:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    xor %s0, -1, %s0
 ; CHECK-NEXT:    and %s0, %s0, %s1
+; CHECK-NEXT:    adds.w.zx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %not = xor i32 %a, -1
   %res = and i32 %not, %b
@@ -113,8 +138,10 @@ define zeroext i32 @func32z(i32 zeroext %a, i32 zeroext %b) {
 define signext i32 @funci32s(i32 signext %a) {
 ; CHECK-LABEL: funci32s:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    xor %s0, -1, %s0
 ; CHECK-NEXT:    and %s0, %s0, (36)0
+; CHECK-NEXT:    adds.w.zx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %not = xor i32 %a, -1
   %res = and i32 %not, 268435455
@@ -124,8 +151,10 @@ define signext i32 @funci32s(i32 signext %a) {
 define zeroext i32 @funci32z(i32 zeroext %a) {
 ; CHECK-LABEL: funci32z:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    xor %s0, -1, %s0
 ; CHECK-NEXT:    and %s0, %s0, (36)0
+; CHECK-NEXT:    adds.w.zx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %not = xor i32 %a, -1
   %res = and i32 %not, 268435455
diff --git a/llvm/test/CodeGen/VE/or.ll b/llvm/test/CodeGen/VE/or.ll
index 1f8c35012f811..8ddb1b5fbf80a 100644
--- a/llvm/test/CodeGen/VE/or.ll
+++ b/llvm/test/CodeGen/VE/or.ll
@@ -21,7 +21,9 @@ define zeroext i8 @func8z(i8 zeroext %a, i8 zeroext %b) {
 define signext i8 @funci8s(i8 signext %a) {
 ; CHECK-LABEL: funci8s:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s0, 5, %s0
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %res = or i8 %a, 5
   ret i8 %res
@@ -30,8 +32,10 @@ define signext i8 @funci8s(i8 signext %a) {
 define zeroext i8 @funci8z(i8 zeroext %a) {
 ; CHECK-LABEL: funci8z:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    lea %s1, 251
 ; CHECK-NEXT:    or %s0, %s0, %s1
+; CHECK-NEXT:    adds.w.zx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %res = or i8 -5, %a
   ret i8 %res
@@ -67,7 +71,9 @@ define signext i16 @funci16s(i16 signext %a) {
 define zeroext i16 @funci16z(i16 zeroext %a) {
 ; CHECK-LABEL: funci16z:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s0, %s0, (52)0
+; CHECK-NEXT:    adds.w.zx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %res = or i16 4095, %a
   ret i16 %res
@@ -94,7 +100,9 @@ define zeroext i32 @func32z(i32 zeroext %a, i32 zeroext %b) {
 define signext i32 @funci32s(i32 signext %a) {
 ; CHECK-LABEL: funci32s:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s0, %s0, (36)0
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %res = or i32 %a, 268435455
   ret i32 %res
@@ -103,7 +111,9 @@ define signext i32 @funci32s(i32 signext %a) {
 define zeroext i32 @funci32z(i32 zeroext %a) {
 ; CHECK-LABEL: funci32z:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s0, %s0, (36)0
+; CHECK-NEXT:    adds.w.zx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %res = or i32 %a, 268435455
   ret i32 %res
diff --git a/llvm/test/CodeGen/VE/pic_access_static_data.ll b/llvm/test/CodeGen/VE/pic_access_static_data.ll
index e0741724172e9..892aa8465d99f 100644
--- a/llvm/test/CodeGen/VE/pic_access_static_data.ll
+++ b/llvm/test/CodeGen/VE/pic_access_static_data.ll
@@ -52,7 +52,7 @@ define i32 @main() {
 ; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    lea.sl %s0, dst@gotoff_hi(, %s0)
 ; CHECK-NEXT:    ldl.sx %s1, (%s0, %s15)
-; CHECK-NEXT:    stl %s1, 184(, %s11)
+; CHECK-NEXT:    st %s1, 184(, %s11)
 ; CHECK-NEXT:    lea %s0, .L.str@gotoff_lo
 ; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    lea.sl %s0, .L.str@gotoff_hi(, %s0)
diff --git a/llvm/test/CodeGen/VE/rem.ll b/llvm/test/CodeGen/VE/rem.ll
index 9fa558f5ca3cd..52ac3c3a3c9e7 100644
--- a/llvm/test/CodeGen/VE/rem.ll
+++ b/llvm/test/CodeGen/VE/rem.ll
@@ -16,6 +16,8 @@ define i64 @remi64(i64 %a, i64 %b) {
 define i32 @remi32(i32 %a, i32 %b) {
 ; CHECK-LABEL: remi32:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    divs.w.sx %s2, %s0, %s1
 ; CHECK-NEXT:    muls.w.sx %s1, %s2, %s1
 ; CHECK-NEXT:    subs.w.sx %s0, %s0, %s1
@@ -40,6 +42,8 @@ define i64 @remu64(i64 %a, i64 %b) {
 define i32 @remu32(i32 %a, i32 %b) {
 ; CHECK-LABEL: remu32:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    divu.w %s2, %s0, %s1
 ; CHECK-NEXT:    muls.w.sx %s1, %s2, %s1
 ; CHECK-NEXT:    subs.w.sx %s0, %s0, %s1
@@ -52,11 +56,13 @@ define i32 @remu32(i32 %a, i32 %b) {
 define signext i16 @remi16(i16 signext %a, i16 signext %b) {
 ; CHECK-LABEL: remi16:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    divs.w.sx %s2, %s0, %s1
 ; CHECK-NEXT:    muls.w.sx %s1, %s2, %s1
 ; CHECK-NEXT:    subs.w.sx %s0, %s0, %s1
-; CHECK-NEXT:    sla.w.sx %s0, %s0, 16
-; CHECK-NEXT:    sra.w.sx %s0, %s0, 16
+; CHECK-NEXT:    sll %s0, %s0, 48
+; CHECK-NEXT:    sra.l %s0, %s0, 48
 ; CHECK-NEXT:    or %s11, 0, %s9
   %a32 = sext i16 %a to i32
   %b32 = sext i16 %b to i32
@@ -69,9 +75,12 @@ define signext i16 @remi16(i16 signext %a, i16 signext %b) {
 define zeroext i16 @remu16(i16 zeroext %a, i16 zeroext %b) {
 ; CHECK-LABEL: remu16:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    divu.w %s2, %s0, %s1
 ; CHECK-NEXT:    muls.w.sx %s1, %s2, %s1
 ; CHECK-NEXT:    subs.w.sx %s0, %s0, %s1
+; CHECK-NEXT:    adds.w.zx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = urem i16 %a, %b
   ret i16 %r
@@ -81,11 +90,13 @@ define zeroext i16 @remu16(i16 zeroext %a, i16 zeroext %b) {
 define signext i8 @remi8(i8 signext %a, i8 signext %b) {
 ; CHECK-LABEL: remi8:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    divs.w.sx %s2, %s0, %s1
 ; CHECK-NEXT:    muls.w.sx %s1, %s2, %s1
 ; CHECK-NEXT:    subs.w.sx %s0, %s0, %s1
-; CHECK-NEXT:    sla.w.sx %s0, %s0, 24
-; CHECK-NEXT:    sra.w.sx %s0, %s0, 24
+; CHECK-NEXT:    sll %s0, %s0, 56
+; CHECK-NEXT:    sra.l %s0, %s0, 56
 ; CHECK-NEXT:    or %s11, 0, %s9
   %a32 = sext i8 %a to i32
   %b32 = sext i8 %b to i32
@@ -98,9 +109,12 @@ define signext i8 @remi8(i8 signext %a, i8 signext %b) {
 define zeroext i8 @remu8(i8 zeroext %a, i8 zeroext %b) {
 ; CHECK-LABEL: remu8:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    divu.w %s2, %s0, %s1
 ; CHECK-NEXT:    muls.w.sx %s1, %s2, %s1
 ; CHECK-NEXT:    subs.w.sx %s0, %s0, %s1
+; CHECK-NEXT:    adds.w.zx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = urem i8 %a, %b
   ret i8 %r
@@ -122,6 +136,7 @@ define i64 @remi64ri(i64 %a, i64 %b) {
 define i32 @remi32ri(i32 %a, i32 %b) {
 ; CHECK-LABEL: remi32ri:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    divs.w.sx %s1, %s0, (62)0
 ; CHECK-NEXT:    muls.w.sx %s1, 3, %s1
 ; CHECK-NEXT:    subs.w.sx %s0, %s0, %s1
@@ -146,6 +161,7 @@ define i64 @remu64ri(i64 %a, i64 %b) {
 define i32 @remu32ri(i32 %a, i32 %b) {
 ; CHECK-LABEL: remu32ri:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    divu.w %s1, %s0, (62)0
 ; CHECK-NEXT:    muls.w.sx %s1, 3, %s1
 ; CHECK-NEXT:    subs.w.sx %s0, %s0, %s1
@@ -170,8 +186,9 @@ define i64 @remi64li(i64 %a, i64 %b) {
 define i32 @remi32li(i32 %a, i32 %b) {
 ; CHECK-LABEL: remi32li:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    divs.w.sx %s0, 3, %s1
-; CHECK-NEXT:    muls.w.sx %s0, %s0, %s1
+; CHECK-NEXT:    adds.w.sx %s0, %s1, (0)1
+; CHECK-NEXT:    divs.w.sx %s1, 3, %s0
+; CHECK-NEXT:    muls.w.sx %s0, %s1, %s0
 ; CHECK-NEXT:    subs.w.sx %s0, 3, %s0
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = srem i32 3, %b
@@ -194,8 +211,9 @@ define i64 @remu64li(i64 %a, i64 %b) {
 define i32 @remu32li(i32 %a, i32 %b) {
 ; CHECK-LABEL: remu32li:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    divu.w %s0, 3, %s1
-; CHECK-NEXT:    muls.w.sx %s0, %s0, %s1
+; CHECK-NEXT:    adds.w.sx %s0, %s1, (0)1
+; CHECK-NEXT:    divu.w %s1, 3, %s0
+; CHECK-NEXT:    muls.w.sx %s0, %s1, %s0
 ; CHECK-NEXT:    subs.w.sx %s0, 3, %s0
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = urem i32 3, %b
diff --git a/llvm/test/CodeGen/VE/right_shift.ll b/llvm/test/CodeGen/VE/right_shift.ll
index faad722a30531..87ac6df7e62ed 100644
--- a/llvm/test/CodeGen/VE/right_shift.ll
+++ b/llvm/test/CodeGen/VE/right_shift.ll
@@ -3,7 +3,10 @@
 define signext i8 @func1(i8 signext %0, i8 signext %1) {
 ; CHECK-LABEL: func1:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    sra.w.sx %s0, %s0, %s1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = sext i8 %0 to i32
   %4 = sext i8 %1 to i32
@@ -15,7 +18,10 @@ define signext i8 @func1(i8 signext %0, i8 signext %1) {
 define signext i16 @func2(i16 signext %0, i16 signext %1) {
 ; CHECK-LABEL: func2:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    sra.w.sx %s0, %s0, %s1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = sext i16 %0 to i32
   %4 = sext i16 %1 to i32
@@ -27,6 +33,8 @@ define signext i16 @func2(i16 signext %0, i16 signext %1) {
 define i32 @func3(i32 %0, i32 %1) {
 ; CHECK-LABEL: func3:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    sra.w.sx %s0, %s0, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = ashr i32 %0, %1
@@ -46,10 +54,11 @@ define i64 @func4(i64 %0, i64 %1) {
 define zeroext i8 @func7(i8 zeroext %0, i8 zeroext %1) {
 ; CHECK-LABEL: func7:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 def $sx0
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    srl %s0, %s0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    adds.w.zx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = zext i8 %0 to i32
   %4 = zext i8 %1 to i32
@@ -61,10 +70,11 @@ define zeroext i8 @func7(i8 zeroext %0, i8 zeroext %1) {
 define zeroext i16 @func8(i16 zeroext %0, i16 zeroext %1) {
 ; CHECK-LABEL: func8:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 def $sx0
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    srl %s0, %s0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    adds.w.zx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = zext i16 %0 to i32
   %4 = zext i16 %1 to i32
@@ -76,10 +86,10 @@ define zeroext i16 @func8(i16 zeroext %0, i16 zeroext %1) {
 define i32 @func9(i32 %0, i32 %1) {
 ; CHECK-LABEL: func9:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 def $sx0
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    srl %s0, %s0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = lshr i32 %0, %1
   ret i32 %3
@@ -98,7 +108,9 @@ define i64 @func10(i64 %0, i64 %1) {
 define signext i8 @func12(i8 signext %0) {
 ; CHECK-LABEL: func12:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    sra.w.sx %s0, %s0, 5
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = ashr i8 %0, 5
   ret i8 %2
@@ -107,7 +119,9 @@ define signext i8 @func12(i8 signext %0) {
 define signext i16 @func13(i16 signext %0) {
 ; CHECK-LABEL: func13:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    sra.w.sx %s0, %s0, 5
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = ashr i16 %0, 5
   ret i16 %2
@@ -116,6 +130,7 @@ define signext i16 @func13(i16 signext %0) {
 define i32 @func14(i32 %0) {
 ; CHECK-LABEL: func14:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    sra.w.sx %s0, %s0, 5
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = ashr i32 %0, 5
@@ -134,10 +149,10 @@ define i64 @func15(i64 %0) {
 define zeroext i8 @func17(i8 zeroext %0) {
 ; CHECK-LABEL: func17:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 def $sx0
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    srl %s0, %s0, 5
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    adds.w.zx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = lshr i8 %0, 5
   ret i8 %2
@@ -146,10 +161,10 @@ define zeroext i8 @func17(i8 zeroext %0) {
 define zeroext i16 @func18(i16 zeroext %0) {
 ; CHECK-LABEL: func18:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 def $sx0
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    srl %s0, %s0, 5
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    adds.w.zx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = lshr i16 %0, 5
   ret i16 %2
@@ -158,10 +173,9 @@ define zeroext i16 @func18(i16 zeroext %0) {
 define i32 @func19(i32 %0) {
 ; CHECK-LABEL: func19:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 def $sx0
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    srl %s0, %s0, 5
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = lshr i32 %0, 5
   ret i32 %2
diff --git a/llvm/test/CodeGen/VE/rotl.ll b/llvm/test/CodeGen/VE/rotl.ll
index e7c498f1d34d2..cc5e004478ab5 100644
--- a/llvm/test/CodeGen/VE/rotl.ll
+++ b/llvm/test/CodeGen/VE/rotl.ll
@@ -3,6 +3,7 @@
 define i64 @func1(i64 %a, i32 %b) {
 ; CHECK-LABEL: func1:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
 ; CHECK-NEXT:    sll %s2, %s0, %s1
 ; CHECK-NEXT:    lea %s3, 64
 ; CHECK-NEXT:    subs.w.sx %s1, %s3, %s1
@@ -21,7 +22,8 @@ define i64 @func1(i64 %a, i32 %b) {
 define i32 @func2(i32 %a, i32 %b) {
 ; CHECK-LABEL: func2:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 def $sx0
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    sla.w.sx %s2, %s0, %s1
 ; CHECK-NEXT:    subs.w.sx %s1, 32, %s1
 ; CHECK-NEXT:    and %s0, %s0, (32)0
diff --git a/llvm/test/CodeGen/VE/rotr.ll b/llvm/test/CodeGen/VE/rotr.ll
index 40734a3d5178a..93dcbbc7e0a82 100644
--- a/llvm/test/CodeGen/VE/rotr.ll
+++ b/llvm/test/CodeGen/VE/rotr.ll
@@ -3,6 +3,7 @@
 define i64 @func1(i64 %a, i32 %b) {
 ; CHECK-LABEL: func1:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
 ; CHECK-NEXT:    srl %s2, %s0, %s1
 ; CHECK-NEXT:    lea %s3, 64
 ; CHECK-NEXT:    subs.w.sx %s1, %s3, %s1
@@ -21,7 +22,8 @@ define i64 @func1(i64 %a, i32 %b) {
 define i32 @func2(i32 %a, i32 %b) {
 ; CHECK-LABEL: func2:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 def $sx0
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    and %s2, %s0, (32)0
 ; CHECK-NEXT:    srl %s2, %s2, %s1
 ; CHECK-NEXT:    subs.w.sx %s1, 32, %s1
diff --git a/llvm/test/CodeGen/VE/select.ll b/llvm/test/CodeGen/VE/select.ll
index 6efe073381d51..81234d3d955cc 100644
--- a/llvm/test/CodeGen/VE/select.ll
+++ b/llvm/test/CodeGen/VE/select.ll
@@ -3,6 +3,7 @@
 define double @selectf64(i1 zeroext, double, double) {
 ; CHECK-LABEL: selectf64:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    cmov.w.ne %s2, %s1, %s0
 ; CHECK-NEXT:    or %s0, 0, %s2
 ; CHECK-NEXT:    or %s11, 0, %s9
@@ -13,8 +14,7 @@ define double @selectf64(i1 zeroext, double, double) {
 define float @selectf32(i1 zeroext, float, float) {
 ; CHECK-LABEL: selectf32:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sf2 killed $sf2 def $sx2
-; CHECK-NEXT:    # kill: def $sf1 killed $sf1 def $sx1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    cmov.w.ne %s2, %s1, %s0
 ; CHECK-NEXT:    or %s0, 0, %s2
 ; CHECK-NEXT:    or %s11, 0, %s9
@@ -25,6 +25,7 @@ define float @selectf32(i1 zeroext, float, float) {
 define i64 @selecti64(i1 zeroext, i64, i64) {
 ; CHECK-LABEL: selecti64:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    cmov.w.ne %s2, %s1, %s0
 ; CHECK-NEXT:    or %s0, 0, %s2
 ; CHECK-NEXT:    or %s11, 0, %s9
@@ -35,10 +36,10 @@ define i64 @selecti64(i1 zeroext, i64, i64) {
 define i32 @selecti32(i1 zeroext, i32, i32) {
 ; CHECK-LABEL: selecti32:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sw2 killed $sw2 def $sx2
-; CHECK-NEXT:    # kill: def $sw1 killed $sw1 def $sx1
-; CHECK-NEXT:    cmov.w.ne %s2, %s1, %s0
-; CHECK-NEXT:    or %s0, 0, %s2
+; CHECK-NEXT:    adds.w.sx %s3, %s0, (0)1
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s2, (0)1
+; CHECK-NEXT:    cmov.w.ne %s0, %s1, %s3
 ; CHECK-NEXT:    or %s11, 0, %s9
   %4 = select i1 %0, i32 %1, i32 %2
   ret i32 %4
@@ -47,10 +48,11 @@ define i32 @selecti32(i1 zeroext, i32, i32) {
 define zeroext i1 @selecti1(i1 zeroext, i1 zeroext, i1 zeroext) {
 ; CHECK-LABEL: selecti1:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sw2 killed $sw2 def $sx2
-; CHECK-NEXT:    # kill: def $sw1 killed $sw1 def $sx1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s2, %s2, (0)1
 ; CHECK-NEXT:    cmov.w.ne %s2, %s1, %s0
-; CHECK-NEXT:    or %s0, 0, %s2
+; CHECK-NEXT:    adds.w.zx %s0, %s2, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %4 = select i1 %0, i1 %1, i1 %2
   ret i1 %4
diff --git a/llvm/test/CodeGen/VE/selectccf32.ll b/llvm/test/CodeGen/VE/selectccf32.ll
index 748a3e9275571..2832be6bc12a6 100644
--- a/llvm/test/CodeGen/VE/selectccf32.ll
+++ b/llvm/test/CodeGen/VE/selectccf32.ll
@@ -23,8 +23,6 @@ define float @selectccat(float, float, float, float) {
 define float @selectccoeq(float, float, float, float) {
 ; CHECK-LABEL: selectccoeq:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sf3 killed $sf3 def $sx3
-; CHECK-NEXT:    # kill: def $sf2 killed $sf2 def $sx2
 ; CHECK-NEXT:    fcmp.s %s0, %s0, %s1
 ; CHECK-NEXT:    cmov.s.eq %s3, %s2, %s0
 ; CHECK-NEXT:    or %s0, 0, %s3
@@ -37,8 +35,6 @@ define float @selectccoeq(float, float, float, float) {
 define float @selectccone(float, float, float, float) {
 ; CHECK-LABEL: selectccone:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sf3 killed $sf3 def $sx3
-; CHECK-NEXT:    # kill: def $sf2 killed $sf2 def $sx2
 ; CHECK-NEXT:    fcmp.s %s0, %s0, %s1
 ; CHECK-NEXT:    cmov.s.ne %s3, %s2, %s0
 ; CHECK-NEXT:    or %s0, 0, %s3
@@ -51,8 +47,6 @@ define float @selectccone(float, float, float, float) {
 define float @selectccogt(float, float, float, float) {
 ; CHECK-LABEL: selectccogt:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sf3 killed $sf3 def $sx3
-; CHECK-NEXT:    # kill: def $sf2 killed $sf2 def $sx2
 ; CHECK-NEXT:    fcmp.s %s0, %s0, %s1
 ; CHECK-NEXT:    cmov.s.gt %s3, %s2, %s0
 ; CHECK-NEXT:    or %s0, 0, %s3
@@ -65,8 +59,6 @@ define float @selectccogt(float, float, float, float) {
 define float @selectccoge(float, float, float, float) {
 ; CHECK-LABEL: selectccoge:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sf3 killed $sf3 def $sx3
-; CHECK-NEXT:    # kill: def $sf2 killed $sf2 def $sx2
 ; CHECK-NEXT:    fcmp.s %s0, %s0, %s1
 ; CHECK-NEXT:    cmov.s.ge %s3, %s2, %s0
 ; CHECK-NEXT:    or %s0, 0, %s3
@@ -79,8 +71,6 @@ define float @selectccoge(float, float, float, float) {
 define float @selectccolt(float, float, float, float) {
 ; CHECK-LABEL: selectccolt:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sf3 killed $sf3 def $sx3
-; CHECK-NEXT:    # kill: def $sf2 killed $sf2 def $sx2
 ; CHECK-NEXT:    fcmp.s %s0, %s0, %s1
 ; CHECK-NEXT:    cmov.s.lt %s3, %s2, %s0
 ; CHECK-NEXT:    or %s0, 0, %s3
@@ -93,8 +83,6 @@ define float @selectccolt(float, float, float, float) {
 define float @selectccole(float, float, float, float) {
 ; CHECK-LABEL: selectccole:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sf3 killed $sf3 def $sx3
-; CHECK-NEXT:    # kill: def $sf2 killed $sf2 def $sx2
 ; CHECK-NEXT:    fcmp.s %s0, %s0, %s1
 ; CHECK-NEXT:    cmov.s.le %s3, %s2, %s0
 ; CHECK-NEXT:    or %s0, 0, %s3
@@ -107,8 +95,6 @@ define float @selectccole(float, float, float, float) {
 define float @selectccord(float, float, float, float) {
 ; CHECK-LABEL: selectccord:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sf3 killed $sf3 def $sx3
-; CHECK-NEXT:    # kill: def $sf2 killed $sf2 def $sx2
 ; CHECK-NEXT:    fcmp.s %s0, %s0, %s1
 ; CHECK-NEXT:    cmov.s.num %s3, %s2, %s0
 ; CHECK-NEXT:    or %s0, 0, %s3
@@ -121,8 +107,6 @@ define float @selectccord(float, float, float, float) {
 define float @selectccuno(float, float, float, float) {
 ; CHECK-LABEL: selectccuno:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sf3 killed $sf3 def $sx3
-; CHECK-NEXT:    # kill: def $sf2 killed $sf2 def $sx2
 ; CHECK-NEXT:    fcmp.s %s0, %s0, %s1
 ; CHECK-NEXT:    cmov.s.nan %s3, %s2, %s0
 ; CHECK-NEXT:    or %s0, 0, %s3
@@ -135,8 +119,6 @@ define float @selectccuno(float, float, float, float) {
 define float @selectccueq(float, float, float, float) {
 ; CHECK-LABEL: selectccueq:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sf3 killed $sf3 def $sx3
-; CHECK-NEXT:    # kill: def $sf2 killed $sf2 def $sx2
 ; CHECK-NEXT:    fcmp.s %s0, %s0, %s1
 ; CHECK-NEXT:    cmov.s.eqnan %s3, %s2, %s0
 ; CHECK-NEXT:    or %s0, 0, %s3
@@ -149,8 +131,6 @@ define float @selectccueq(float, float, float, float) {
 define float @selectccune(float, float, float, float) {
 ; CHECK-LABEL: selectccune:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sf3 killed $sf3 def $sx3
-; CHECK-NEXT:    # kill: def $sf2 killed $sf2 def $sx2
 ; CHECK-NEXT:    fcmp.s %s0, %s0, %s1
 ; CHECK-NEXT:    cmov.s.nenan %s3, %s2, %s0
 ; CHECK-NEXT:    or %s0, 0, %s3
@@ -163,8 +143,6 @@ define float @selectccune(float, float, float, float) {
 define float @selectccugt(float, float, float, float) {
 ; CHECK-LABEL: selectccugt:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sf3 killed $sf3 def $sx3
-; CHECK-NEXT:    # kill: def $sf2 killed $sf2 def $sx2
 ; CHECK-NEXT:    fcmp.s %s0, %s0, %s1
 ; CHECK-NEXT:    cmov.s.gtnan %s3, %s2, %s0
 ; CHECK-NEXT:    or %s0, 0, %s3
@@ -177,8 +155,6 @@ define float @selectccugt(float, float, float, float) {
 define float @selectccuge(float, float, float, float) {
 ; CHECK-LABEL: selectccuge:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sf3 killed $sf3 def $sx3
-; CHECK-NEXT:    # kill: def $sf2 killed $sf2 def $sx2
 ; CHECK-NEXT:    fcmp.s %s0, %s0, %s1
 ; CHECK-NEXT:    cmov.s.genan %s3, %s2, %s0
 ; CHECK-NEXT:    or %s0, 0, %s3
@@ -191,8 +167,6 @@ define float @selectccuge(float, float, float, float) {
 define float @selectccult(float, float, float, float) {
 ; CHECK-LABEL: selectccult:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sf3 killed $sf3 def $sx3
-; CHECK-NEXT:    # kill: def $sf2 killed $sf2 def $sx2
 ; CHECK-NEXT:    fcmp.s %s0, %s0, %s1
 ; CHECK-NEXT:    cmov.s.ltnan %s3, %s2, %s0
 ; CHECK-NEXT:    or %s0, 0, %s3
@@ -205,8 +179,6 @@ define float @selectccult(float, float, float, float) {
 define float @selectccule(float, float, float, float) {
 ; CHECK-LABEL: selectccule:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sf3 killed $sf3 def $sx3
-; CHECK-NEXT:    # kill: def $sf2 killed $sf2 def $sx2
 ; CHECK-NEXT:    fcmp.s %s0, %s0, %s1
 ; CHECK-NEXT:    cmov.s.lenan %s3, %s2, %s0
 ; CHECK-NEXT:    or %s0, 0, %s3
diff --git a/llvm/test/CodeGen/VE/selectccf32c.ll b/llvm/test/CodeGen/VE/selectccf32c.ll
index 78a9aaf96729c..54a9da4c8e460 100644
--- a/llvm/test/CodeGen/VE/selectccf32c.ll
+++ b/llvm/test/CodeGen/VE/selectccf32c.ll
@@ -3,12 +3,10 @@
 define float @selectccsgti8(i8, i8, float, float) {
 ; CHECK-LABEL: selectccsgti8:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sf3 killed $sf3 def $sx3
-; CHECK-NEXT:    # kill: def $sf2 killed $sf2 def $sx2
-; CHECK-NEXT:    sla.w.sx %s1, %s1, 24
-; CHECK-NEXT:    sra.w.sx %s1, %s1, 24
-; CHECK-NEXT:    sla.w.sx %s0, %s0, 24
-; CHECK-NEXT:    sra.w.sx %s0, %s0, 24
+; CHECK-NEXT:    sll %s1, %s1, 56
+; CHECK-NEXT:    sra.l %s1, %s1, 56
+; CHECK-NEXT:    sll %s0, %s0, 56
+; CHECK-NEXT:    sra.l %s0, %s0, 56
 ; CHECK-NEXT:    cmps.w.sx %s0, %s0, %s1
 ; CHECK-NEXT:    cmov.w.gt %s3, %s2, %s0
 ; CHECK-NEXT:    or %s0, 0, %s3
@@ -21,12 +19,10 @@ define float @selectccsgti8(i8, i8, float, float) {
 define float @selectccsgti16(i16, i16, float, float) {
 ; CHECK-LABEL: selectccsgti16:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sf3 killed $sf3 def $sx3
-; CHECK-NEXT:    # kill: def $sf2 killed $sf2 def $sx2
-; CHECK-NEXT:    sla.w.sx %s1, %s1, 16
-; CHECK-NEXT:    sra.w.sx %s1, %s1, 16
-; CHECK-NEXT:    sla.w.sx %s0, %s0, 16
-; CHECK-NEXT:    sra.w.sx %s0, %s0, 16
+; CHECK-NEXT:    sll %s1, %s1, 48
+; CHECK-NEXT:    sra.l %s1, %s1, 48
+; CHECK-NEXT:    sll %s0, %s0, 48
+; CHECK-NEXT:    sra.l %s0, %s0, 48
 ; CHECK-NEXT:    cmps.w.sx %s0, %s0, %s1
 ; CHECK-NEXT:    cmov.w.gt %s3, %s2, %s0
 ; CHECK-NEXT:    or %s0, 0, %s3
@@ -39,8 +35,8 @@ define float @selectccsgti16(i16, i16, float, float) {
 define float @selectccsgti32(i32, i32, float, float) {
 ; CHECK-LABEL: selectccsgti32:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sf3 killed $sf3 def $sx3
-; CHECK-NEXT:    # kill: def $sf2 killed $sf2 def $sx2
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    cmps.w.sx %s0, %s0, %s1
 ; CHECK-NEXT:    cmov.w.gt %s3, %s2, %s0
 ; CHECK-NEXT:    or %s0, 0, %s3
@@ -53,8 +49,6 @@ define float @selectccsgti32(i32, i32, float, float) {
 define float @selectccsgti64(i64, i64, float, float) {
 ; CHECK-LABEL: selectccsgti64:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sf3 killed $sf3 def $sx3
-; CHECK-NEXT:    # kill: def $sf2 killed $sf2 def $sx2
 ; CHECK-NEXT:    cmps.l %s0, %s0, %s1
 ; CHECK-NEXT:    cmov.l.gt %s3, %s2, %s0
 ; CHECK-NEXT:    or %s0, 0, %s3
@@ -67,8 +61,6 @@ define float @selectccsgti64(i64, i64, float, float) {
 define float @selectccsgti128(i128, i128, float, float) {
 ; CHECK-LABEL: selectccsgti128:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sf5 killed $sf5 def $sx5
-; CHECK-NEXT:    # kill: def $sf4 killed $sf4 def $sx4
 ; CHECK-NEXT:    or %s6, 0, (0)1
 ; CHECK-NEXT:    cmps.l %s1, %s1, %s3
 ; CHECK-NEXT:    or %s3, 0, %s6
@@ -89,8 +81,6 @@ define float @selectccsgti128(i128, i128, float, float) {
 define float @selectccogtf32(float, float, float, float) {
 ; CHECK-LABEL: selectccogtf32:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sf3 killed $sf3 def $sx3
-; CHECK-NEXT:    # kill: def $sf2 killed $sf2 def $sx2
 ; CHECK-NEXT:    fcmp.s %s0, %s0, %s1
 ; CHECK-NEXT:    cmov.s.gt %s3, %s2, %s0
 ; CHECK-NEXT:    or %s0, 0, %s3
@@ -103,8 +93,6 @@ define float @selectccogtf32(float, float, float, float) {
 define float @selectccogtf64(double, double, float, float) {
 ; CHECK-LABEL: selectccogtf64:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sf3 killed $sf3 def $sx3
-; CHECK-NEXT:    # kill: def $sf2 killed $sf2 def $sx2
 ; CHECK-NEXT:    fcmp.d %s0, %s0, %s1
 ; CHECK-NEXT:    cmov.d.gt %s3, %s2, %s0
 ; CHECK-NEXT:    or %s0, 0, %s3
diff --git a/llvm/test/CodeGen/VE/selectccf32i.ll b/llvm/test/CodeGen/VE/selectccf32i.ll
index e8285c818c54d..5e2698b4feb17 100644
--- a/llvm/test/CodeGen/VE/selectccf32i.ll
+++ b/llvm/test/CodeGen/VE/selectccf32i.ll
@@ -23,8 +23,6 @@ define float @selectccat(float, float, float, float) {
 define float @selectccoeq(float, float, float, float) {
 ; CHECK-LABEL: selectccoeq:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sf3 killed $sf3 def $sx3
-; CHECK-NEXT:    # kill: def $sf2 killed $sf2 def $sx2
 ; CHECK-NEXT:    lea.sl %s1, 0
 ; CHECK-NEXT:    fcmp.s %s0, %s0, %s1
 ; CHECK-NEXT:    cmov.s.eq %s3, %s2, %s0
@@ -38,8 +36,6 @@ define float @selectccoeq(float, float, float, float) {
 define float @selectccone(float, float, float, float) {
 ; CHECK-LABEL: selectccone:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sf3 killed $sf3 def $sx3
-; CHECK-NEXT:    # kill: def $sf2 killed $sf2 def $sx2
 ; CHECK-NEXT:    lea.sl %s1, 0
 ; CHECK-NEXT:    fcmp.s %s0, %s0, %s1
 ; CHECK-NEXT:    cmov.s.ne %s3, %s2, %s0
@@ -53,8 +49,6 @@ define float @selectccone(float, float, float, float) {
 define float @selectccogt(float, float, float, float) {
 ; CHECK-LABEL: selectccogt:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sf3 killed $sf3 def $sx3
-; CHECK-NEXT:    # kill: def $sf2 killed $sf2 def $sx2
 ; CHECK-NEXT:    lea.sl %s1, 0
 ; CHECK-NEXT:    fcmp.s %s0, %s0, %s1
 ; CHECK-NEXT:    cmov.s.gt %s3, %s2, %s0
@@ -68,8 +62,6 @@ define float @selectccogt(float, float, float, float) {
 define float @selectccoge(float, float, float, float) {
 ; CHECK-LABEL: selectccoge:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sf3 killed $sf3 def $sx3
-; CHECK-NEXT:    # kill: def $sf2 killed $sf2 def $sx2
 ; CHECK-NEXT:    lea.sl %s1, 0
 ; CHECK-NEXT:    fcmp.s %s0, %s0, %s1
 ; CHECK-NEXT:    cmov.s.ge %s3, %s2, %s0
@@ -83,8 +75,6 @@ define float @selectccoge(float, float, float, float) {
 define float @selectccolt(float, float, float, float) {
 ; CHECK-LABEL: selectccolt:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sf3 killed $sf3 def $sx3
-; CHECK-NEXT:    # kill: def $sf2 killed $sf2 def $sx2
 ; CHECK-NEXT:    lea.sl %s1, 0
 ; CHECK-NEXT:    fcmp.s %s0, %s0, %s1
 ; CHECK-NEXT:    cmov.s.lt %s3, %s2, %s0
@@ -98,8 +88,6 @@ define float @selectccolt(float, float, float, float) {
 define float @selectccole(float, float, float, float) {
 ; CHECK-LABEL: selectccole:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sf3 killed $sf3 def $sx3
-; CHECK-NEXT:    # kill: def $sf2 killed $sf2 def $sx2
 ; CHECK-NEXT:    lea.sl %s1, 0
 ; CHECK-NEXT:    fcmp.s %s0, %s0, %s1
 ; CHECK-NEXT:    cmov.s.le %s3, %s2, %s0
@@ -113,8 +101,6 @@ define float @selectccole(float, float, float, float) {
 define float @selectccord(float, float, float, float) {
 ; CHECK-LABEL: selectccord:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sf3 killed $sf3 def $sx3
-; CHECK-NEXT:    # kill: def $sf2 killed $sf2 def $sx2
 ; CHECK-NEXT:    fcmp.s %s0, %s0, %s0
 ; CHECK-NEXT:    cmov.s.num %s3, %s2, %s0
 ; CHECK-NEXT:    or %s0, 0, %s3
@@ -127,8 +113,6 @@ define float @selectccord(float, float, float, float) {
 define float @selectccuno(float, float, float, float) {
 ; CHECK-LABEL: selectccuno:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sf3 killed $sf3 def $sx3
-; CHECK-NEXT:    # kill: def $sf2 killed $sf2 def $sx2
 ; CHECK-NEXT:    fcmp.s %s0, %s0, %s0
 ; CHECK-NEXT:    cmov.s.nan %s3, %s2, %s0
 ; CHECK-NEXT:    or %s0, 0, %s3
@@ -141,8 +125,6 @@ define float @selectccuno(float, float, float, float) {
 define float @selectccueq(float, float, float, float) {
 ; CHECK-LABEL: selectccueq:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sf3 killed $sf3 def $sx3
-; CHECK-NEXT:    # kill: def $sf2 killed $sf2 def $sx2
 ; CHECK-NEXT:    lea.sl %s1, 0
 ; CHECK-NEXT:    fcmp.s %s0, %s0, %s1
 ; CHECK-NEXT:    cmov.s.eqnan %s3, %s2, %s0
@@ -156,8 +138,6 @@ define float @selectccueq(float, float, float, float) {
 define float @selectccune(float, float, float, float) {
 ; CHECK-LABEL: selectccune:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sf3 killed $sf3 def $sx3
-; CHECK-NEXT:    # kill: def $sf2 killed $sf2 def $sx2
 ; CHECK-NEXT:    lea.sl %s1, 0
 ; CHECK-NEXT:    fcmp.s %s0, %s0, %s1
 ; CHECK-NEXT:    cmov.s.nenan %s3, %s2, %s0
@@ -171,8 +151,6 @@ define float @selectccune(float, float, float, float) {
 define float @selectccugt(float, float, float, float) {
 ; CHECK-LABEL: selectccugt:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sf3 killed $sf3 def $sx3
-; CHECK-NEXT:    # kill: def $sf2 killed $sf2 def $sx2
 ; CHECK-NEXT:    lea.sl %s1, 0
 ; CHECK-NEXT:    fcmp.s %s0, %s0, %s1
 ; CHECK-NEXT:    cmov.s.gtnan %s3, %s2, %s0
@@ -186,8 +164,6 @@ define float @selectccugt(float, float, float, float) {
 define float @selectccuge(float, float, float, float) {
 ; CHECK-LABEL: selectccuge:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sf3 killed $sf3 def $sx3
-; CHECK-NEXT:    # kill: def $sf2 killed $sf2 def $sx2
 ; CHECK-NEXT:    lea.sl %s1, 0
 ; CHECK-NEXT:    fcmp.s %s0, %s0, %s1
 ; CHECK-NEXT:    cmov.s.genan %s3, %s2, %s0
@@ -201,8 +177,6 @@ define float @selectccuge(float, float, float, float) {
 define float @selectccult(float, float, float, float) {
 ; CHECK-LABEL: selectccult:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sf3 killed $sf3 def $sx3
-; CHECK-NEXT:    # kill: def $sf2 killed $sf2 def $sx2
 ; CHECK-NEXT:    lea.sl %s1, 0
 ; CHECK-NEXT:    fcmp.s %s0, %s0, %s1
 ; CHECK-NEXT:    cmov.s.ltnan %s3, %s2, %s0
@@ -216,8 +190,6 @@ define float @selectccult(float, float, float, float) {
 define float @selectccule(float, float, float, float) {
 ; CHECK-LABEL: selectccule:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sf3 killed $sf3 def $sx3
-; CHECK-NEXT:    # kill: def $sf2 killed $sf2 def $sx2
 ; CHECK-NEXT:    lea.sl %s1, 0
 ; CHECK-NEXT:    fcmp.s %s0, %s0, %s1
 ; CHECK-NEXT:    cmov.s.lenan %s3, %s2, %s0
diff --git a/llvm/test/CodeGen/VE/selectccf64c.ll b/llvm/test/CodeGen/VE/selectccf64c.ll
index 4481d1d6197f5..24b61ece8d545 100644
--- a/llvm/test/CodeGen/VE/selectccf64c.ll
+++ b/llvm/test/CodeGen/VE/selectccf64c.ll
@@ -3,10 +3,10 @@
 define double @selectccsgti8(i8, i8, double, double) {
 ; CHECK-LABEL: selectccsgti8:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    sla.w.sx %s1, %s1, 24
-; CHECK-NEXT:    sra.w.sx %s1, %s1, 24
-; CHECK-NEXT:    sla.w.sx %s0, %s0, 24
-; CHECK-NEXT:    sra.w.sx %s0, %s0, 24
+; CHECK-NEXT:    sll %s1, %s1, 56
+; CHECK-NEXT:    sra.l %s1, %s1, 56
+; CHECK-NEXT:    sll %s0, %s0, 56
+; CHECK-NEXT:    sra.l %s0, %s0, 56
 ; CHECK-NEXT:    cmps.w.sx %s0, %s0, %s1
 ; CHECK-NEXT:    cmov.w.gt %s3, %s2, %s0
 ; CHECK-NEXT:    or %s0, 0, %s3
@@ -19,10 +19,10 @@ define double @selectccsgti8(i8, i8, double, double) {
 define double @selectccsgti16(i16, i16, double, double) {
 ; CHECK-LABEL: selectccsgti16:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    sla.w.sx %s1, %s1, 16
-; CHECK-NEXT:    sra.w.sx %s1, %s1, 16
-; CHECK-NEXT:    sla.w.sx %s0, %s0, 16
-; CHECK-NEXT:    sra.w.sx %s0, %s0, 16
+; CHECK-NEXT:    sll %s1, %s1, 48
+; CHECK-NEXT:    sra.l %s1, %s1, 48
+; CHECK-NEXT:    sll %s0, %s0, 48
+; CHECK-NEXT:    sra.l %s0, %s0, 48
 ; CHECK-NEXT:    cmps.w.sx %s0, %s0, %s1
 ; CHECK-NEXT:    cmov.w.gt %s3, %s2, %s0
 ; CHECK-NEXT:    or %s0, 0, %s3
@@ -35,6 +35,8 @@ define double @selectccsgti16(i16, i16, double, double) {
 define double @selectccsgti32(i32, i32, double, double) {
 ; CHECK-LABEL: selectccsgti32:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    cmps.w.sx %s0, %s0, %s1
 ; CHECK-NEXT:    cmov.w.gt %s3, %s2, %s0
 ; CHECK-NEXT:    or %s0, 0, %s3
diff --git a/llvm/test/CodeGen/VE/selectcci32.ll b/llvm/test/CodeGen/VE/selectcci32.ll
index eea115c8b936b..af1861487b892 100644
--- a/llvm/test/CodeGen/VE/selectcci32.ll
+++ b/llvm/test/CodeGen/VE/selectcci32.ll
@@ -3,11 +3,12 @@
 define i32 @selectcceq(i32, i32, i32, i32) {
 ; CHECK-LABEL: selectcceq:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sw3 killed $sw3 def $sx3
-; CHECK-NEXT:    # kill: def $sw2 killed $sw2 def $sx2
-; CHECK-NEXT:    cmps.w.sx %s0, %s0, %s1
-; CHECK-NEXT:    cmov.w.eq %s3, %s2, %s0
-; CHECK-NEXT:    or %s0, 0, %s3
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s4, %s0, (0)1
+; CHECK-NEXT:    adds.w.sx %s2, %s2, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s3, (0)1
+; CHECK-NEXT:    cmps.w.sx %s1, %s4, %s1
+; CHECK-NEXT:    cmov.w.eq %s0, %s2, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %5 = icmp eq i32 %0, %1
   %6 = select i1 %5, i32 %2, i32 %3
@@ -17,11 +18,12 @@ define i32 @selectcceq(i32, i32, i32, i32) {
 define i32 @selectccne(i32, i32, i32, i32) {
 ; CHECK-LABEL: selectccne:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sw3 killed $sw3 def $sx3
-; CHECK-NEXT:    # kill: def $sw2 killed $sw2 def $sx2
-; CHECK-NEXT:    cmps.w.sx %s0, %s0, %s1
-; CHECK-NEXT:    cmov.w.ne %s3, %s2, %s0
-; CHECK-NEXT:    or %s0, 0, %s3
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s4, %s0, (0)1
+; CHECK-NEXT:    adds.w.sx %s2, %s2, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s3, (0)1
+; CHECK-NEXT:    cmps.w.sx %s1, %s4, %s1
+; CHECK-NEXT:    cmov.w.ne %s0, %s2, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %5 = icmp ne i32 %0, %1
   %6 = select i1 %5, i32 %2, i32 %3
@@ -31,11 +33,12 @@ define i32 @selectccne(i32, i32, i32, i32) {
 define i32 @selectccsgt(i32, i32, i32, i32) {
 ; CHECK-LABEL: selectccsgt:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sw3 killed $sw3 def $sx3
-; CHECK-NEXT:    # kill: def $sw2 killed $sw2 def $sx2
-; CHECK-NEXT:    cmps.w.sx %s0, %s0, %s1
-; CHECK-NEXT:    cmov.w.gt %s3, %s2, %s0
-; CHECK-NEXT:    or %s0, 0, %s3
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s4, %s0, (0)1
+; CHECK-NEXT:    adds.w.sx %s2, %s2, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s3, (0)1
+; CHECK-NEXT:    cmps.w.sx %s1, %s4, %s1
+; CHECK-NEXT:    cmov.w.gt %s0, %s2, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %5 = icmp sgt i32 %0, %1
   %6 = select i1 %5, i32 %2, i32 %3
@@ -45,11 +48,12 @@ define i32 @selectccsgt(i32, i32, i32, i32) {
 define i32 @selectccsge(i32, i32, i32, i32) {
 ; CHECK-LABEL: selectccsge:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sw3 killed $sw3 def $sx3
-; CHECK-NEXT:    # kill: def $sw2 killed $sw2 def $sx2
-; CHECK-NEXT:    cmps.w.sx %s0, %s0, %s1
-; CHECK-NEXT:    cmov.w.ge %s3, %s2, %s0
-; CHECK-NEXT:    or %s0, 0, %s3
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s4, %s0, (0)1
+; CHECK-NEXT:    adds.w.sx %s2, %s2, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s3, (0)1
+; CHECK-NEXT:    cmps.w.sx %s1, %s4, %s1
+; CHECK-NEXT:    cmov.w.ge %s0, %s2, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %5 = icmp sge i32 %0, %1
   %6 = select i1 %5, i32 %2, i32 %3
@@ -59,11 +63,12 @@ define i32 @selectccsge(i32, i32, i32, i32) {
 define i32 @selectccslt(i32, i32, i32, i32) {
 ; CHECK-LABEL: selectccslt:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sw3 killed $sw3 def $sx3
-; CHECK-NEXT:    # kill: def $sw2 killed $sw2 def $sx2
-; CHECK-NEXT:    cmps.w.sx %s0, %s0, %s1
-; CHECK-NEXT:    cmov.w.lt %s3, %s2, %s0
-; CHECK-NEXT:    or %s0, 0, %s3
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s4, %s0, (0)1
+; CHECK-NEXT:    adds.w.sx %s2, %s2, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s3, (0)1
+; CHECK-NEXT:    cmps.w.sx %s1, %s4, %s1
+; CHECK-NEXT:    cmov.w.lt %s0, %s2, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %5 = icmp slt i32 %0, %1
   %6 = select i1 %5, i32 %2, i32 %3
@@ -73,11 +78,12 @@ define i32 @selectccslt(i32, i32, i32, i32) {
 define i32 @selectccsle(i32, i32, i32, i32) {
 ; CHECK-LABEL: selectccsle:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sw3 killed $sw3 def $sx3
-; CHECK-NEXT:    # kill: def $sw2 killed $sw2 def $sx2
-; CHECK-NEXT:    cmps.w.sx %s0, %s0, %s1
-; CHECK-NEXT:    cmov.w.le %s3, %s2, %s0
-; CHECK-NEXT:    or %s0, 0, %s3
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s4, %s0, (0)1
+; CHECK-NEXT:    adds.w.sx %s2, %s2, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s3, (0)1
+; CHECK-NEXT:    cmps.w.sx %s1, %s4, %s1
+; CHECK-NEXT:    cmov.w.le %s0, %s2, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %5 = icmp sle i32 %0, %1
   %6 = select i1 %5, i32 %2, i32 %3
@@ -87,11 +93,12 @@ define i32 @selectccsle(i32, i32, i32, i32) {
 define i32 @selectccugt(i32, i32, i32, i32) {
 ; CHECK-LABEL: selectccugt:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sw3 killed $sw3 def $sx3
-; CHECK-NEXT:    # kill: def $sw2 killed $sw2 def $sx2
-; CHECK-NEXT:    cmpu.w %s0, %s0, %s1
-; CHECK-NEXT:    cmov.w.gt %s3, %s2, %s0
-; CHECK-NEXT:    or %s0, 0, %s3
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s4, %s0, (0)1
+; CHECK-NEXT:    adds.w.sx %s2, %s2, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s3, (0)1
+; CHECK-NEXT:    cmpu.w %s1, %s4, %s1
+; CHECK-NEXT:    cmov.w.gt %s0, %s2, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %5 = icmp ugt i32 %0, %1
   %6 = select i1 %5, i32 %2, i32 %3
@@ -101,11 +108,12 @@ define i32 @selectccugt(i32, i32, i32, i32) {
 define i32 @selectccuge(i32, i32, i32, i32) {
 ; CHECK-LABEL: selectccuge:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sw3 killed $sw3 def $sx3
-; CHECK-NEXT:    # kill: def $sw2 killed $sw2 def $sx2
-; CHECK-NEXT:    cmpu.w %s0, %s0, %s1
-; CHECK-NEXT:    cmov.w.ge %s3, %s2, %s0
-; CHECK-NEXT:    or %s0, 0, %s3
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s4, %s0, (0)1
+; CHECK-NEXT:    adds.w.sx %s2, %s2, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s3, (0)1
+; CHECK-NEXT:    cmpu.w %s1, %s4, %s1
+; CHECK-NEXT:    cmov.w.ge %s0, %s2, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %5 = icmp uge i32 %0, %1
   %6 = select i1 %5, i32 %2, i32 %3
@@ -115,11 +123,12 @@ define i32 @selectccuge(i32, i32, i32, i32) {
 define i32 @selectccult(i32, i32, i32, i32) {
 ; CHECK-LABEL: selectccult:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sw3 killed $sw3 def $sx3
-; CHECK-NEXT:    # kill: def $sw2 killed $sw2 def $sx2
-; CHECK-NEXT:    cmpu.w %s0, %s0, %s1
-; CHECK-NEXT:    cmov.w.lt %s3, %s2, %s0
-; CHECK-NEXT:    or %s0, 0, %s3
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s4, %s0, (0)1
+; CHECK-NEXT:    adds.w.sx %s2, %s2, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s3, (0)1
+; CHECK-NEXT:    cmpu.w %s1, %s4, %s1
+; CHECK-NEXT:    cmov.w.lt %s0, %s2, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %5 = icmp ult i32 %0, %1
   %6 = select i1 %5, i32 %2, i32 %3
@@ -129,11 +138,12 @@ define i32 @selectccult(i32, i32, i32, i32) {
 define i32 @selectccule(i32, i32, i32, i32) {
 ; CHECK-LABEL: selectccule:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sw3 killed $sw3 def $sx3
-; CHECK-NEXT:    # kill: def $sw2 killed $sw2 def $sx2
-; CHECK-NEXT:    cmpu.w %s0, %s0, %s1
-; CHECK-NEXT:    cmov.w.le %s3, %s2, %s0
-; CHECK-NEXT:    or %s0, 0, %s3
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s4, %s0, (0)1
+; CHECK-NEXT:    adds.w.sx %s2, %s2, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s3, (0)1
+; CHECK-NEXT:    cmpu.w %s1, %s4, %s1
+; CHECK-NEXT:    cmov.w.le %s0, %s2, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %5 = icmp ule i32 %0, %1
   %6 = select i1 %5, i32 %2, i32 %3
@@ -143,11 +153,12 @@ define i32 @selectccule(i32, i32, i32, i32) {
 define i32 @selectccugt2(i32, i32, i32, i32) {
 ; CHECK-LABEL: selectccugt2:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sw3 killed $sw3 def $sx3
-; CHECK-NEXT:    # kill: def $sw2 killed $sw2 def $sx2
-; CHECK-NEXT:    cmpu.w %s0, %s0, %s1
-; CHECK-NEXT:    cmov.w.gt %s3, %s2, %s0
-; CHECK-NEXT:    or %s0, 0, %s3
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s4, %s0, (0)1
+; CHECK-NEXT:    adds.w.sx %s2, %s2, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s3, (0)1
+; CHECK-NEXT:    cmpu.w %s1, %s4, %s1
+; CHECK-NEXT:    cmov.w.gt %s0, %s2, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %5 = icmp ugt i32 %0, %1
   %6 = select i1 %5, i32 %2, i32 %3
@@ -157,11 +168,12 @@ define i32 @selectccugt2(i32, i32, i32, i32) {
 define i32 @selectccuge2(i32, i32, i32, i32) {
 ; CHECK-LABEL: selectccuge2:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sw3 killed $sw3 def $sx3
-; CHECK-NEXT:    # kill: def $sw2 killed $sw2 def $sx2
-; CHECK-NEXT:    cmpu.w %s0, %s0, %s1
-; CHECK-NEXT:    cmov.w.ge %s3, %s2, %s0
-; CHECK-NEXT:    or %s0, 0, %s3
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s4, %s0, (0)1
+; CHECK-NEXT:    adds.w.sx %s2, %s2, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s3, (0)1
+; CHECK-NEXT:    cmpu.w %s1, %s4, %s1
+; CHECK-NEXT:    cmov.w.ge %s0, %s2, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %5 = icmp uge i32 %0, %1
   %6 = select i1 %5, i32 %2, i32 %3
@@ -171,11 +183,12 @@ define i32 @selectccuge2(i32, i32, i32, i32) {
 define i32 @selectccult2(i32, i32, i32, i32) {
 ; CHECK-LABEL: selectccult2:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sw3 killed $sw3 def $sx3
-; CHECK-NEXT:    # kill: def $sw2 killed $sw2 def $sx2
-; CHECK-NEXT:    cmpu.w %s0, %s0, %s1
-; CHECK-NEXT:    cmov.w.lt %s3, %s2, %s0
-; CHECK-NEXT:    or %s0, 0, %s3
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s4, %s0, (0)1
+; CHECK-NEXT:    adds.w.sx %s2, %s2, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s3, (0)1
+; CHECK-NEXT:    cmpu.w %s1, %s4, %s1
+; CHECK-NEXT:    cmov.w.lt %s0, %s2, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %5 = icmp ult i32 %0, %1
   %6 = select i1 %5, i32 %2, i32 %3
@@ -185,11 +198,12 @@ define i32 @selectccult2(i32, i32, i32, i32) {
 define i32 @selectccule2(i32, i32, i32, i32) {
 ; CHECK-LABEL: selectccule2:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sw3 killed $sw3 def $sx3
-; CHECK-NEXT:    # kill: def $sw2 killed $sw2 def $sx2
-; CHECK-NEXT:    cmpu.w %s0, %s0, %s1
-; CHECK-NEXT:    cmov.w.le %s3, %s2, %s0
-; CHECK-NEXT:    or %s0, 0, %s3
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s4, %s0, (0)1
+; CHECK-NEXT:    adds.w.sx %s2, %s2, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s3, (0)1
+; CHECK-NEXT:    cmpu.w %s1, %s4, %s1
+; CHECK-NEXT:    cmov.w.le %s0, %s2, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %5 = icmp ule i32 %0, %1
   %6 = select i1 %5, i32 %2, i32 %3
diff --git a/llvm/test/CodeGen/VE/selectcci32c.ll b/llvm/test/CodeGen/VE/selectcci32c.ll
index 474e9e0dcd4dd..e4017c46c5790 100644
--- a/llvm/test/CodeGen/VE/selectcci32c.ll
+++ b/llvm/test/CodeGen/VE/selectcci32c.ll
@@ -3,15 +3,14 @@
 define i32 @selectccsgti8(i8, i8, i32, i32) {
 ; CHECK-LABEL: selectccsgti8:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sw3 killed $sw3 def $sx3
-; CHECK-NEXT:    # kill: def $sw2 killed $sw2 def $sx2
-; CHECK-NEXT:    sla.w.sx %s1, %s1, 24
-; CHECK-NEXT:    sra.w.sx %s1, %s1, 24
-; CHECK-NEXT:    sla.w.sx %s0, %s0, 24
-; CHECK-NEXT:    sra.w.sx %s0, %s0, 24
-; CHECK-NEXT:    cmps.w.sx %s0, %s0, %s1
-; CHECK-NEXT:    cmov.w.gt %s3, %s2, %s0
-; CHECK-NEXT:    or %s0, 0, %s3
+; CHECK-NEXT:    sll %s1, %s1, 56
+; CHECK-NEXT:    sra.l %s1, %s1, 56
+; CHECK-NEXT:    sll %s0, %s0, 56
+; CHECK-NEXT:    sra.l %s4, %s0, 56
+; CHECK-NEXT:    adds.w.sx %s2, %s2, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s3, (0)1
+; CHECK-NEXT:    cmps.w.sx %s1, %s4, %s1
+; CHECK-NEXT:    cmov.w.gt %s0, %s2, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %5 = icmp sgt i8 %0, %1
   %6 = select i1 %5, i32 %2, i32 %3
@@ -21,15 +20,14 @@ define i32 @selectccsgti8(i8, i8, i32, i32) {
 define i32 @selectccsgti16(i16, i16, i32, i32) {
 ; CHECK-LABEL: selectccsgti16:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sw3 killed $sw3 def $sx3
-; CHECK-NEXT:    # kill: def $sw2 killed $sw2 def $sx2
-; CHECK-NEXT:    sla.w.sx %s1, %s1, 16
-; CHECK-NEXT:    sra.w.sx %s1, %s1, 16
-; CHECK-NEXT:    sla.w.sx %s0, %s0, 16
-; CHECK-NEXT:    sra.w.sx %s0, %s0, 16
-; CHECK-NEXT:    cmps.w.sx %s0, %s0, %s1
-; CHECK-NEXT:    cmov.w.gt %s3, %s2, %s0
-; CHECK-NEXT:    or %s0, 0, %s3
+; CHECK-NEXT:    sll %s1, %s1, 48
+; CHECK-NEXT:    sra.l %s1, %s1, 48
+; CHECK-NEXT:    sll %s0, %s0, 48
+; CHECK-NEXT:    sra.l %s4, %s0, 48
+; CHECK-NEXT:    adds.w.sx %s2, %s2, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s3, (0)1
+; CHECK-NEXT:    cmps.w.sx %s1, %s4, %s1
+; CHECK-NEXT:    cmov.w.gt %s0, %s2, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %5 = icmp sgt i16 %0, %1
   %6 = select i1 %5, i32 %2, i32 %3
@@ -39,11 +37,12 @@ define i32 @selectccsgti16(i16, i16, i32, i32) {
 define i32 @selectccsgti32(i32, i32, i32, i32) {
 ; CHECK-LABEL: selectccsgti32:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sw3 killed $sw3 def $sx3
-; CHECK-NEXT:    # kill: def $sw2 killed $sw2 def $sx2
-; CHECK-NEXT:    cmps.w.sx %s0, %s0, %s1
-; CHECK-NEXT:    cmov.w.gt %s3, %s2, %s0
-; CHECK-NEXT:    or %s0, 0, %s3
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s4, %s0, (0)1
+; CHECK-NEXT:    adds.w.sx %s2, %s2, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s3, (0)1
+; CHECK-NEXT:    cmps.w.sx %s1, %s4, %s1
+; CHECK-NEXT:    cmov.w.gt %s0, %s2, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %5 = icmp sgt i32 %0, %1
   %6 = select i1 %5, i32 %2, i32 %3
@@ -53,11 +52,11 @@ define i32 @selectccsgti32(i32, i32, i32, i32) {
 define i32 @selectccsgti64(i64, i64, i32, i32) {
 ; CHECK-LABEL: selectccsgti64:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sw3 killed $sw3 def $sx3
-; CHECK-NEXT:    # kill: def $sw2 killed $sw2 def $sx2
+; CHECK-NEXT:    adds.w.sx %s4, %s2, (0)1
+; CHECK-NEXT:    adds.w.sx %s2, %s3, (0)1
 ; CHECK-NEXT:    cmps.l %s0, %s0, %s1
-; CHECK-NEXT:    cmov.l.gt %s3, %s2, %s0
-; CHECK-NEXT:    or %s0, 0, %s3
+; CHECK-NEXT:    cmov.l.gt %s2, %s4, %s0
+; CHECK-NEXT:    or %s0, 0, %s2
 ; CHECK-NEXT:    or %s11, 0, %s9
   %5 = icmp sgt i64 %0, %1
   %6 = select i1 %5, i32 %2, i32 %3
@@ -67,19 +66,19 @@ define i32 @selectccsgti64(i64, i64, i32, i32) {
 define i32 @selectccsgti128(i128, i128, i32, i32) {
 ; CHECK-LABEL: selectccsgti128:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sw5 killed $sw5 def $sx5
-; CHECK-NEXT:    # kill: def $sw4 killed $sw4 def $sx4
-; CHECK-NEXT:    or %s6, 0, (0)1
+; CHECK-NEXT:    adds.w.sx %s6, %s4, (0)1
+; CHECK-NEXT:    adds.w.sx %s4, %s5, (0)1
+; CHECK-NEXT:    or %s5, 0, (0)1
 ; CHECK-NEXT:    cmps.l %s1, %s1, %s3
-; CHECK-NEXT:    or %s3, 0, %s6
+; CHECK-NEXT:    or %s3, 0, %s5
 ; CHECK-NEXT:    cmov.l.gt %s3, (63)0, %s1
 ; CHECK-NEXT:    cmpu.l %s0, %s0, %s2
-; CHECK-NEXT:    cmov.l.gt %s6, (63)0, %s0
-; CHECK-NEXT:    cmov.l.eq %s3, %s6, %s1
+; CHECK-NEXT:    cmov.l.gt %s5, (63)0, %s0
+; CHECK-NEXT:    cmov.l.eq %s3, %s5, %s1
 ; CHECK-NEXT:    or %s0, 0, (0)1
 ; CHECK-NEXT:    cmps.w.sx %s0, %s3, %s0
-; CHECK-NEXT:    cmov.w.ne %s5, %s4, %s0
-; CHECK-NEXT:    or %s0, 0, %s5
+; CHECK-NEXT:    cmov.w.ne %s4, %s6, %s0
+; CHECK-NEXT:    or %s0, 0, %s4
 ; CHECK-NEXT:    or %s11, 0, %s9
   %5 = icmp sgt i128 %0, %1
   %6 = select i1 %5, i32 %2, i32 %3
@@ -89,11 +88,11 @@ define i32 @selectccsgti128(i128, i128, i32, i32) {
 define i32 @selectccogtf32(float, float, i32, i32) {
 ; CHECK-LABEL: selectccogtf32:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sw3 killed $sw3 def $sx3
-; CHECK-NEXT:    # kill: def $sw2 killed $sw2 def $sx2
+; CHECK-NEXT:    adds.w.sx %s4, %s2, (0)1
+; CHECK-NEXT:    adds.w.sx %s2, %s3, (0)1
 ; CHECK-NEXT:    fcmp.s %s0, %s0, %s1
-; CHECK-NEXT:    cmov.s.gt %s3, %s2, %s0
-; CHECK-NEXT:    or %s0, 0, %s3
+; CHECK-NEXT:    cmov.s.gt %s2, %s4, %s0
+; CHECK-NEXT:    or %s0, 0, %s2
 ; CHECK-NEXT:    or %s11, 0, %s9
   %5 = fcmp ogt float %0, %1
   %6 = select i1 %5, i32 %2, i32 %3
@@ -103,11 +102,11 @@ define i32 @selectccogtf32(float, float, i32, i32) {
 define i32 @selectccogtf64(double, double, i32, i32) {
 ; CHECK-LABEL: selectccogtf64:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sw3 killed $sw3 def $sx3
-; CHECK-NEXT:    # kill: def $sw2 killed $sw2 def $sx2
+; CHECK-NEXT:    adds.w.sx %s4, %s2, (0)1
+; CHECK-NEXT:    adds.w.sx %s2, %s3, (0)1
 ; CHECK-NEXT:    fcmp.d %s0, %s0, %s1
-; CHECK-NEXT:    cmov.d.gt %s3, %s2, %s0
-; CHECK-NEXT:    or %s0, 0, %s3
+; CHECK-NEXT:    cmov.d.gt %s2, %s4, %s0
+; CHECK-NEXT:    or %s0, 0, %s2
 ; CHECK-NEXT:    or %s11, 0, %s9
   %5 = fcmp ogt double %0, %1
   %6 = select i1 %5, i32 %2, i32 %3
diff --git a/llvm/test/CodeGen/VE/selectcci32i.ll b/llvm/test/CodeGen/VE/selectcci32i.ll
index 0e17f83ad1781..a4cccd0ebf935 100644
--- a/llvm/test/CodeGen/VE/selectcci32i.ll
+++ b/llvm/test/CodeGen/VE/selectcci32i.ll
@@ -3,12 +3,12 @@
 define i32 @selectcceq(i32, i32, i32, i32) {
 ; CHECK-LABEL: selectcceq:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sw3 killed $sw3 def $sx3
-; CHECK-NEXT:    # kill: def $sw2 killed $sw2 def $sx2
-; CHECK-NEXT:    or %s1, 12, (0)1
-; CHECK-NEXT:    cmps.w.sx %s0, %s0, %s1
-; CHECK-NEXT:    cmov.w.eq %s3, %s2, %s0
-; CHECK-NEXT:    or %s0, 0, %s3
+; CHECK-NEXT:    adds.w.sx %s1, %s0, (0)1
+; CHECK-NEXT:    adds.w.sx %s2, %s2, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s3, (0)1
+; CHECK-NEXT:    or %s3, 12, (0)1
+; CHECK-NEXT:    cmps.w.sx %s1, %s1, %s3
+; CHECK-NEXT:    cmov.w.eq %s0, %s2, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %5 = icmp eq i32 %0, 12
   %6 = select i1 %5, i32 %2, i32 %3
@@ -18,12 +18,12 @@ define i32 @selectcceq(i32, i32, i32, i32) {
 define i32 @selectccne(i32, i32, i32, i32) {
 ; CHECK-LABEL: selectccne:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sw3 killed $sw3 def $sx3
-; CHECK-NEXT:    # kill: def $sw2 killed $sw2 def $sx2
-; CHECK-NEXT:    or %s1, 12, (0)1
-; CHECK-NEXT:    cmps.w.sx %s0, %s0, %s1
-; CHECK-NEXT:    cmov.w.ne %s3, %s2, %s0
-; CHECK-NEXT:    or %s0, 0, %s3
+; CHECK-NEXT:    adds.w.sx %s1, %s0, (0)1
+; CHECK-NEXT:    adds.w.sx %s2, %s2, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s3, (0)1
+; CHECK-NEXT:    or %s3, 12, (0)1
+; CHECK-NEXT:    cmps.w.sx %s1, %s1, %s3
+; CHECK-NEXT:    cmov.w.ne %s0, %s2, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %5 = icmp ne i32 %0, 12
   %6 = select i1 %5, i32 %2, i32 %3
@@ -33,12 +33,12 @@ define i32 @selectccne(i32, i32, i32, i32) {
 define i32 @selectccsgt(i32, i32, i32, i32) {
 ; CHECK-LABEL: selectccsgt:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sw3 killed $sw3 def $sx3
-; CHECK-NEXT:    # kill: def $sw2 killed $sw2 def $sx2
-; CHECK-NEXT:    or %s1, 12, (0)1
-; CHECK-NEXT:    cmps.w.sx %s0, %s0, %s1
-; CHECK-NEXT:    cmov.w.gt %s3, %s2, %s0
-; CHECK-NEXT:    or %s0, 0, %s3
+; CHECK-NEXT:    adds.w.sx %s1, %s0, (0)1
+; CHECK-NEXT:    adds.w.sx %s2, %s2, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s3, (0)1
+; CHECK-NEXT:    or %s3, 12, (0)1
+; CHECK-NEXT:    cmps.w.sx %s1, %s1, %s3
+; CHECK-NEXT:    cmov.w.gt %s0, %s2, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %5 = icmp sgt i32 %0, 12
   %6 = select i1 %5, i32 %2, i32 %3
@@ -48,12 +48,12 @@ define i32 @selectccsgt(i32, i32, i32, i32) {
 define i32 @selectccsge(i32, i32, i32, i32) {
 ; CHECK-LABEL: selectccsge:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sw3 killed $sw3 def $sx3
-; CHECK-NEXT:    # kill: def $sw2 killed $sw2 def $sx2
-; CHECK-NEXT:    or %s1, 11, (0)1
-; CHECK-NEXT:    cmps.w.sx %s0, %s0, %s1
-; CHECK-NEXT:    cmov.w.gt %s3, %s2, %s0
-; CHECK-NEXT:    or %s0, 0, %s3
+; CHECK-NEXT:    adds.w.sx %s1, %s0, (0)1
+; CHECK-NEXT:    adds.w.sx %s2, %s2, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s3, (0)1
+; CHECK-NEXT:    or %s3, 11, (0)1
+; CHECK-NEXT:    cmps.w.sx %s1, %s1, %s3
+; CHECK-NEXT:    cmov.w.gt %s0, %s2, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %5 = icmp sge i32 %0, 12
   %6 = select i1 %5, i32 %2, i32 %3
@@ -63,12 +63,12 @@ define i32 @selectccsge(i32, i32, i32, i32) {
 define i32 @selectccslt(i32, i32, i32, i32) {
 ; CHECK-LABEL: selectccslt:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sw3 killed $sw3 def $sx3
-; CHECK-NEXT:    # kill: def $sw2 killed $sw2 def $sx2
-; CHECK-NEXT:    or %s1, 12, (0)1
-; CHECK-NEXT:    cmps.w.sx %s0, %s0, %s1
-; CHECK-NEXT:    cmov.w.lt %s3, %s2, %s0
-; CHECK-NEXT:    or %s0, 0, %s3
+; CHECK-NEXT:    adds.w.sx %s1, %s0, (0)1
+; CHECK-NEXT:    adds.w.sx %s2, %s2, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s3, (0)1
+; CHECK-NEXT:    or %s3, 12, (0)1
+; CHECK-NEXT:    cmps.w.sx %s1, %s1, %s3
+; CHECK-NEXT:    cmov.w.lt %s0, %s2, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %5 = icmp slt i32 %0, 12
   %6 = select i1 %5, i32 %2, i32 %3
@@ -78,12 +78,12 @@ define i32 @selectccslt(i32, i32, i32, i32) {
 define i32 @selectccsle(i32, i32, i32, i32) {
 ; CHECK-LABEL: selectccsle:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sw3 killed $sw3 def $sx3
-; CHECK-NEXT:    # kill: def $sw2 killed $sw2 def $sx2
-; CHECK-NEXT:    or %s1, 13, (0)1
-; CHECK-NEXT:    cmps.w.sx %s0, %s0, %s1
-; CHECK-NEXT:    cmov.w.lt %s3, %s2, %s0
-; CHECK-NEXT:    or %s0, 0, %s3
+; CHECK-NEXT:    adds.w.sx %s1, %s0, (0)1
+; CHECK-NEXT:    adds.w.sx %s2, %s2, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s3, (0)1
+; CHECK-NEXT:    or %s3, 13, (0)1
+; CHECK-NEXT:    cmps.w.sx %s1, %s1, %s3
+; CHECK-NEXT:    cmov.w.lt %s0, %s2, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %5 = icmp sle i32 %0, 12
   %6 = select i1 %5, i32 %2, i32 %3
@@ -93,12 +93,12 @@ define i32 @selectccsle(i32, i32, i32, i32) {
 define i32 @selectccugt(i32, i32, i32, i32) {
 ; CHECK-LABEL: selectccugt:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sw3 killed $sw3 def $sx3
-; CHECK-NEXT:    # kill: def $sw2 killed $sw2 def $sx2
-; CHECK-NEXT:    or %s1, 12, (0)1
-; CHECK-NEXT:    cmpu.w %s0, %s0, %s1
-; CHECK-NEXT:    cmov.w.gt %s3, %s2, %s0
-; CHECK-NEXT:    or %s0, 0, %s3
+; CHECK-NEXT:    adds.w.sx %s1, %s0, (0)1
+; CHECK-NEXT:    adds.w.sx %s2, %s2, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s3, (0)1
+; CHECK-NEXT:    or %s3, 12, (0)1
+; CHECK-NEXT:    cmpu.w %s1, %s1, %s3
+; CHECK-NEXT:    cmov.w.gt %s0, %s2, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %5 = icmp ugt i32 %0, 12
   %6 = select i1 %5, i32 %2, i32 %3
@@ -108,12 +108,12 @@ define i32 @selectccugt(i32, i32, i32, i32) {
 define i32 @selectccuge(i32, i32, i32, i32) {
 ; CHECK-LABEL: selectccuge:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sw3 killed $sw3 def $sx3
-; CHECK-NEXT:    # kill: def $sw2 killed $sw2 def $sx2
-; CHECK-NEXT:    or %s1, 11, (0)1
-; CHECK-NEXT:    cmpu.w %s0, %s0, %s1
-; CHECK-NEXT:    cmov.w.gt %s3, %s2, %s0
-; CHECK-NEXT:    or %s0, 0, %s3
+; CHECK-NEXT:    adds.w.sx %s1, %s0, (0)1
+; CHECK-NEXT:    adds.w.sx %s2, %s2, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s3, (0)1
+; CHECK-NEXT:    or %s3, 11, (0)1
+; CHECK-NEXT:    cmpu.w %s1, %s1, %s3
+; CHECK-NEXT:    cmov.w.gt %s0, %s2, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %5 = icmp uge i32 %0, 12
   %6 = select i1 %5, i32 %2, i32 %3
@@ -123,12 +123,12 @@ define i32 @selectccuge(i32, i32, i32, i32) {
 define i32 @selectccult(i32, i32, i32, i32) {
 ; CHECK-LABEL: selectccult:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sw3 killed $sw3 def $sx3
-; CHECK-NEXT:    # kill: def $sw2 killed $sw2 def $sx2
-; CHECK-NEXT:    or %s1, 12, (0)1
-; CHECK-NEXT:    cmpu.w %s0, %s0, %s1
-; CHECK-NEXT:    cmov.w.lt %s3, %s2, %s0
-; CHECK-NEXT:    or %s0, 0, %s3
+; CHECK-NEXT:    adds.w.sx %s1, %s0, (0)1
+; CHECK-NEXT:    adds.w.sx %s2, %s2, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s3, (0)1
+; CHECK-NEXT:    or %s3, 12, (0)1
+; CHECK-NEXT:    cmpu.w %s1, %s1, %s3
+; CHECK-NEXT:    cmov.w.lt %s0, %s2, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %5 = icmp ult i32 %0, 12
   %6 = select i1 %5, i32 %2, i32 %3
@@ -138,12 +138,12 @@ define i32 @selectccult(i32, i32, i32, i32) {
 define i32 @selectccule(i32, i32, i32, i32) {
 ; CHECK-LABEL: selectccule:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sw3 killed $sw3 def $sx3
-; CHECK-NEXT:    # kill: def $sw2 killed $sw2 def $sx2
-; CHECK-NEXT:    or %s1, 13, (0)1
-; CHECK-NEXT:    cmpu.w %s0, %s0, %s1
-; CHECK-NEXT:    cmov.w.lt %s3, %s2, %s0
-; CHECK-NEXT:    or %s0, 0, %s3
+; CHECK-NEXT:    adds.w.sx %s1, %s0, (0)1
+; CHECK-NEXT:    adds.w.sx %s2, %s2, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s3, (0)1
+; CHECK-NEXT:    or %s3, 13, (0)1
+; CHECK-NEXT:    cmpu.w %s1, %s1, %s3
+; CHECK-NEXT:    cmov.w.lt %s0, %s2, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %5 = icmp ule i32 %0, 12
   %6 = select i1 %5, i32 %2, i32 %3
@@ -153,12 +153,12 @@ define i32 @selectccule(i32, i32, i32, i32) {
 define i32 @selectccugt2(i32, i32, i32, i32) {
 ; CHECK-LABEL: selectccugt2:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sw3 killed $sw3 def $sx3
-; CHECK-NEXT:    # kill: def $sw2 killed $sw2 def $sx2
-; CHECK-NEXT:    or %s1, 12, (0)1
-; CHECK-NEXT:    cmpu.w %s0, %s0, %s1
-; CHECK-NEXT:    cmov.w.gt %s3, %s2, %s0
-; CHECK-NEXT:    or %s0, 0, %s3
+; CHECK-NEXT:    adds.w.sx %s1, %s0, (0)1
+; CHECK-NEXT:    adds.w.sx %s2, %s2, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s3, (0)1
+; CHECK-NEXT:    or %s3, 12, (0)1
+; CHECK-NEXT:    cmpu.w %s1, %s1, %s3
+; CHECK-NEXT:    cmov.w.gt %s0, %s2, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %5 = icmp ugt i32 %0, 12
   %6 = select i1 %5, i32 %2, i32 %3
@@ -168,12 +168,12 @@ define i32 @selectccugt2(i32, i32, i32, i32) {
 define i32 @selectccuge2(i32, i32, i32, i32) {
 ; CHECK-LABEL: selectccuge2:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sw3 killed $sw3 def $sx3
-; CHECK-NEXT:    # kill: def $sw2 killed $sw2 def $sx2
-; CHECK-NEXT:    or %s1, 11, (0)1
-; CHECK-NEXT:    cmpu.w %s0, %s0, %s1
-; CHECK-NEXT:    cmov.w.gt %s3, %s2, %s0
-; CHECK-NEXT:    or %s0, 0, %s3
+; CHECK-NEXT:    adds.w.sx %s1, %s0, (0)1
+; CHECK-NEXT:    adds.w.sx %s2, %s2, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s3, (0)1
+; CHECK-NEXT:    or %s3, 11, (0)1
+; CHECK-NEXT:    cmpu.w %s1, %s1, %s3
+; CHECK-NEXT:    cmov.w.gt %s0, %s2, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %5 = icmp uge i32 %0, 12
   %6 = select i1 %5, i32 %2, i32 %3
@@ -183,12 +183,12 @@ define i32 @selectccuge2(i32, i32, i32, i32) {
 define i32 @selectccult2(i32, i32, i32, i32) {
 ; CHECK-LABEL: selectccult2:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sw3 killed $sw3 def $sx3
-; CHECK-NEXT:    # kill: def $sw2 killed $sw2 def $sx2
-; CHECK-NEXT:    or %s1, 12, (0)1
-; CHECK-NEXT:    cmpu.w %s0, %s0, %s1
-; CHECK-NEXT:    cmov.w.lt %s3, %s2, %s0
-; CHECK-NEXT:    or %s0, 0, %s3
+; CHECK-NEXT:    adds.w.sx %s1, %s0, (0)1
+; CHECK-NEXT:    adds.w.sx %s2, %s2, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s3, (0)1
+; CHECK-NEXT:    or %s3, 12, (0)1
+; CHECK-NEXT:    cmpu.w %s1, %s1, %s3
+; CHECK-NEXT:    cmov.w.lt %s0, %s2, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %5 = icmp ult i32 %0, 12
   %6 = select i1 %5, i32 %2, i32 %3
@@ -198,12 +198,12 @@ define i32 @selectccult2(i32, i32, i32, i32) {
 define i32 @selectccule2(i32, i32, i32, i32) {
 ; CHECK-LABEL: selectccule2:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    # kill: def $sw3 killed $sw3 def $sx3
-; CHECK-NEXT:    # kill: def $sw2 killed $sw2 def $sx2
-; CHECK-NEXT:    or %s1, 13, (0)1
-; CHECK-NEXT:    cmpu.w %s0, %s0, %s1
-; CHECK-NEXT:    cmov.w.lt %s3, %s2, %s0
-; CHECK-NEXT:    or %s0, 0, %s3
+; CHECK-NEXT:    adds.w.sx %s1, %s0, (0)1
+; CHECK-NEXT:    adds.w.sx %s2, %s2, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s3, (0)1
+; CHECK-NEXT:    or %s3, 13, (0)1
+; CHECK-NEXT:    cmpu.w %s1, %s1, %s3
+; CHECK-NEXT:    cmov.w.lt %s0, %s2, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %5 = icmp ule i32 %0, 12
   %6 = select i1 %5, i32 %2, i32 %3
diff --git a/llvm/test/CodeGen/VE/selectcci64c.ll b/llvm/test/CodeGen/VE/selectcci64c.ll
index 7bb7d7fee1c31..276f23d9a5ffd 100644
--- a/llvm/test/CodeGen/VE/selectcci64c.ll
+++ b/llvm/test/CodeGen/VE/selectcci64c.ll
@@ -3,10 +3,10 @@
 define i64 @selectccsgti8(i8, i8, i64, i64) {
 ; CHECK-LABEL: selectccsgti8:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    sla.w.sx %s1, %s1, 24
-; CHECK-NEXT:    sra.w.sx %s1, %s1, 24
-; CHECK-NEXT:    sla.w.sx %s0, %s0, 24
-; CHECK-NEXT:    sra.w.sx %s0, %s0, 24
+; CHECK-NEXT:    sll %s1, %s1, 56
+; CHECK-NEXT:    sra.l %s1, %s1, 56
+; CHECK-NEXT:    sll %s0, %s0, 56
+; CHECK-NEXT:    sra.l %s0, %s0, 56
 ; CHECK-NEXT:    cmps.w.sx %s0, %s0, %s1
 ; CHECK-NEXT:    cmov.w.gt %s3, %s2, %s0
 ; CHECK-NEXT:    or %s0, 0, %s3
@@ -19,10 +19,10 @@ define i64 @selectccsgti8(i8, i8, i64, i64) {
 define i64 @selectccsgti16(i16, i16, i64, i64) {
 ; CHECK-LABEL: selectccsgti16:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    sla.w.sx %s1, %s1, 16
-; CHECK-NEXT:    sra.w.sx %s1, %s1, 16
-; CHECK-NEXT:    sla.w.sx %s0, %s0, 16
-; CHECK-NEXT:    sra.w.sx %s0, %s0, 16
+; CHECK-NEXT:    sll %s1, %s1, 48
+; CHECK-NEXT:    sra.l %s1, %s1, 48
+; CHECK-NEXT:    sll %s0, %s0, 48
+; CHECK-NEXT:    sra.l %s0, %s0, 48
 ; CHECK-NEXT:    cmps.w.sx %s0, %s0, %s1
 ; CHECK-NEXT:    cmov.w.gt %s3, %s2, %s0
 ; CHECK-NEXT:    or %s0, 0, %s3
@@ -35,6 +35,8 @@ define i64 @selectccsgti16(i16, i16, i64, i64) {
 define i64 @selectccsgti32(i32, i32, i64, i64) {
 ; CHECK-LABEL: selectccsgti32:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    cmps.w.sx %s0, %s0, %s1
 ; CHECK-NEXT:    cmov.w.gt %s3, %s2, %s0
 ; CHECK-NEXT:    or %s0, 0, %s3
diff --git a/llvm/test/CodeGen/VE/setccf32.ll b/llvm/test/CodeGen/VE/setccf32.ll
index 6ced8ce53b9c4..f2e9062fcf624 100644
--- a/llvm/test/CodeGen/VE/setccf32.ll
+++ b/llvm/test/CodeGen/VE/setccf32.ll
@@ -21,10 +21,10 @@ define zeroext i1 @setccat(float, float) {
 define zeroext i1 @setccoeq(float, float) {
 ; CHECK-LABEL: setccoeq:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    fcmp.s %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.s.eq %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    fcmp.s %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.s.eq %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = fcmp oeq float %0, %1
   ret i1 %3
@@ -33,10 +33,10 @@ define zeroext i1 @setccoeq(float, float) {
 define zeroext i1 @setccone(float, float) {
 ; CHECK-LABEL: setccone:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    fcmp.s %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.s.ne %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    fcmp.s %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.s.ne %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = fcmp one float %0, %1
   ret i1 %3
@@ -45,10 +45,10 @@ define zeroext i1 @setccone(float, float) {
 define zeroext i1 @setccogt(float, float) {
 ; CHECK-LABEL: setccogt:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    fcmp.s %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.s.gt %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    fcmp.s %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.s.gt %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = fcmp ogt float %0, %1
   ret i1 %3
@@ -57,10 +57,10 @@ define zeroext i1 @setccogt(float, float) {
 define zeroext i1 @setccoge(float, float) {
 ; CHECK-LABEL: setccoge:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    fcmp.s %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.s.ge %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    fcmp.s %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.s.ge %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = fcmp oge float %0, %1
   ret i1 %3
@@ -69,10 +69,10 @@ define zeroext i1 @setccoge(float, float) {
 define zeroext i1 @setccolt(float, float) {
 ; CHECK-LABEL: setccolt:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    fcmp.s %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.s.lt %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    fcmp.s %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.s.lt %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = fcmp olt float %0, %1
   ret i1 %3
@@ -81,10 +81,10 @@ define zeroext i1 @setccolt(float, float) {
 define zeroext i1 @setccole(float, float) {
 ; CHECK-LABEL: setccole:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    fcmp.s %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.s.le %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    fcmp.s %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.s.le %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = fcmp ole float %0, %1
   ret i1 %3
@@ -93,10 +93,10 @@ define zeroext i1 @setccole(float, float) {
 define zeroext i1 @setccord(float, float) {
 ; CHECK-LABEL: setccord:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    fcmp.s %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.s.num %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    fcmp.s %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.s.num %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = fcmp ord float %0, %1
   ret i1 %3
@@ -105,10 +105,10 @@ define zeroext i1 @setccord(float, float) {
 define zeroext i1 @setccuno(float, float) {
 ; CHECK-LABEL: setccuno:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    fcmp.s %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.s.nan %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    fcmp.s %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.s.nan %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = fcmp uno float %0, %1
   ret i1 %3
@@ -117,10 +117,10 @@ define zeroext i1 @setccuno(float, float) {
 define zeroext i1 @setccueq(float, float) {
 ; CHECK-LABEL: setccueq:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    fcmp.s %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.s.eqnan %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    fcmp.s %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.s.eqnan %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = fcmp ueq float %0, %1
   ret i1 %3
@@ -129,10 +129,10 @@ define zeroext i1 @setccueq(float, float) {
 define zeroext i1 @setccune(float, float) {
 ; CHECK-LABEL: setccune:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    fcmp.s %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.s.nenan %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    fcmp.s %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.s.nenan %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = fcmp une float %0, %1
   ret i1 %3
@@ -141,10 +141,10 @@ define zeroext i1 @setccune(float, float) {
 define zeroext i1 @setccugt(float, float) {
 ; CHECK-LABEL: setccugt:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    fcmp.s %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.s.gtnan %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    fcmp.s %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.s.gtnan %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = fcmp ugt float %0, %1
   ret i1 %3
@@ -153,10 +153,10 @@ define zeroext i1 @setccugt(float, float) {
 define zeroext i1 @setccuge(float, float) {
 ; CHECK-LABEL: setccuge:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    fcmp.s %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.s.genan %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    fcmp.s %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.s.genan %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = fcmp uge float %0, %1
   ret i1 %3
@@ -165,10 +165,10 @@ define zeroext i1 @setccuge(float, float) {
 define zeroext i1 @setccult(float, float) {
 ; CHECK-LABEL: setccult:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    fcmp.s %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.s.ltnan %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    fcmp.s %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.s.ltnan %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = fcmp ult float %0, %1
   ret i1 %3
@@ -177,10 +177,10 @@ define zeroext i1 @setccult(float, float) {
 define zeroext i1 @setccule(float, float) {
 ; CHECK-LABEL: setccule:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    fcmp.s %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.s.lenan %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    fcmp.s %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.s.lenan %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = fcmp ule float %0, %1
   ret i1 %3
diff --git a/llvm/test/CodeGen/VE/setccf32i.ll b/llvm/test/CodeGen/VE/setccf32i.ll
index 8f79219359f3c..3f90a103fec53 100644
--- a/llvm/test/CodeGen/VE/setccf32i.ll
+++ b/llvm/test/CodeGen/VE/setccf32i.ll
@@ -22,10 +22,10 @@ define zeroext i1 @setccoeq(float, float) {
 ; CHECK-LABEL: setccoeq:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    lea.sl %s1, 0
-; CHECK-NEXT:    fcmp.s %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.s.eq %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    fcmp.s %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.s.eq %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = fcmp oeq float %0, 0.0
   ret i1 %3
@@ -35,10 +35,10 @@ define zeroext i1 @setccone(float, float) {
 ; CHECK-LABEL: setccone:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    lea.sl %s1, 0
-; CHECK-NEXT:    fcmp.s %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.s.ne %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    fcmp.s %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.s.ne %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = fcmp one float %0, 0.0
   ret i1 %3
@@ -48,10 +48,10 @@ define zeroext i1 @setccogt(float, float) {
 ; CHECK-LABEL: setccogt:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    lea.sl %s1, 0
-; CHECK-NEXT:    fcmp.s %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.s.gt %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    fcmp.s %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.s.gt %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = fcmp ogt float %0, 0.0
   ret i1 %3
@@ -61,10 +61,10 @@ define zeroext i1 @setccoge(float, float) {
 ; CHECK-LABEL: setccoge:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    lea.sl %s1, 0
-; CHECK-NEXT:    fcmp.s %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.s.ge %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    fcmp.s %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.s.ge %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = fcmp oge float %0, 0.0
   ret i1 %3
@@ -74,10 +74,10 @@ define zeroext i1 @setccolt(float, float) {
 ; CHECK-LABEL: setccolt:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    lea.sl %s1, 0
-; CHECK-NEXT:    fcmp.s %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.s.lt %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    fcmp.s %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.s.lt %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = fcmp olt float %0, 0.0
   ret i1 %3
@@ -87,10 +87,10 @@ define zeroext i1 @setccole(float, float) {
 ; CHECK-LABEL: setccole:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    lea.sl %s1, 0
-; CHECK-NEXT:    fcmp.s %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.s.le %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    fcmp.s %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.s.le %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = fcmp ole float %0, 0.0
   ret i1 %3
@@ -99,10 +99,10 @@ define zeroext i1 @setccole(float, float) {
 define zeroext i1 @setccord(float, float) {
 ; CHECK-LABEL: setccord:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    fcmp.s %s1, %s0, %s0
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.s.num %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    fcmp.s %s0, %s0, %s0
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.s.num %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = fcmp ord float %0, 0.0
   ret i1 %3
@@ -111,10 +111,10 @@ define zeroext i1 @setccord(float, float) {
 define zeroext i1 @setccuno(float, float) {
 ; CHECK-LABEL: setccuno:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    fcmp.s %s1, %s0, %s0
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.s.nan %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    fcmp.s %s0, %s0, %s0
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.s.nan %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = fcmp uno float %0, 0.0
   ret i1 %3
@@ -124,10 +124,10 @@ define zeroext i1 @setccueq(float, float) {
 ; CHECK-LABEL: setccueq:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    lea.sl %s1, 0
-; CHECK-NEXT:    fcmp.s %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.s.eqnan %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    fcmp.s %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.s.eqnan %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = fcmp ueq float %0, 0.0
   ret i1 %3
@@ -137,10 +137,10 @@ define zeroext i1 @setccune(float, float) {
 ; CHECK-LABEL: setccune:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    lea.sl %s1, 0
-; CHECK-NEXT:    fcmp.s %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.s.nenan %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    fcmp.s %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.s.nenan %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = fcmp une float %0, 0.0
   ret i1 %3
@@ -150,10 +150,10 @@ define zeroext i1 @setccugt(float, float) {
 ; CHECK-LABEL: setccugt:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    lea.sl %s1, 0
-; CHECK-NEXT:    fcmp.s %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.s.gtnan %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    fcmp.s %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.s.gtnan %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = fcmp ugt float %0, 0.0
   ret i1 %3
@@ -163,10 +163,10 @@ define zeroext i1 @setccuge(float, float) {
 ; CHECK-LABEL: setccuge:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    lea.sl %s1, 0
-; CHECK-NEXT:    fcmp.s %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.s.genan %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    fcmp.s %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.s.genan %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = fcmp uge float %0, 0.0
   ret i1 %3
@@ -176,10 +176,10 @@ define zeroext i1 @setccult(float, float) {
 ; CHECK-LABEL: setccult:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    lea.sl %s1, 0
-; CHECK-NEXT:    fcmp.s %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.s.ltnan %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    fcmp.s %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.s.ltnan %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = fcmp ult float %0, 0.0
   ret i1 %3
@@ -189,10 +189,10 @@ define zeroext i1 @setccule(float, float) {
 ; CHECK-LABEL: setccule:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    lea.sl %s1, 0
-; CHECK-NEXT:    fcmp.s %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.s.lenan %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    fcmp.s %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.s.lenan %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = fcmp ule float %0, 0.0
   ret i1 %3
diff --git a/llvm/test/CodeGen/VE/setccf64.ll b/llvm/test/CodeGen/VE/setccf64.ll
index dca40e8231fa9..98c0e6c56bf41 100644
--- a/llvm/test/CodeGen/VE/setccf64.ll
+++ b/llvm/test/CodeGen/VE/setccf64.ll
@@ -21,10 +21,10 @@ define zeroext i1 @setccat(double, double) {
 define zeroext i1 @setccoeq(double, double) {
 ; CHECK-LABEL: setccoeq:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    fcmp.d %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.d.eq %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    fcmp.d %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.d.eq %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = fcmp oeq double %0, %1
   ret i1 %3
@@ -33,10 +33,10 @@ define zeroext i1 @setccoeq(double, double) {
 define zeroext i1 @setccone(double, double) {
 ; CHECK-LABEL: setccone:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    fcmp.d %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.d.ne %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    fcmp.d %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.d.ne %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = fcmp one double %0, %1
   ret i1 %3
@@ -45,10 +45,10 @@ define zeroext i1 @setccone(double, double) {
 define zeroext i1 @setccogt(double, double) {
 ; CHECK-LABEL: setccogt:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    fcmp.d %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.d.gt %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    fcmp.d %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.d.gt %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = fcmp ogt double %0, %1
   ret i1 %3
@@ -57,10 +57,10 @@ define zeroext i1 @setccogt(double, double) {
 define zeroext i1 @setccoge(double, double) {
 ; CHECK-LABEL: setccoge:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    fcmp.d %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.d.ge %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    fcmp.d %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.d.ge %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = fcmp oge double %0, %1
   ret i1 %3
@@ -69,10 +69,10 @@ define zeroext i1 @setccoge(double, double) {
 define zeroext i1 @setccolt(double, double) {
 ; CHECK-LABEL: setccolt:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    fcmp.d %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.d.lt %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    fcmp.d %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.d.lt %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = fcmp olt double %0, %1
   ret i1 %3
@@ -81,10 +81,10 @@ define zeroext i1 @setccolt(double, double) {
 define zeroext i1 @setccole(double, double) {
 ; CHECK-LABEL: setccole:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    fcmp.d %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.d.le %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    fcmp.d %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.d.le %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = fcmp ole double %0, %1
   ret i1 %3
@@ -93,10 +93,10 @@ define zeroext i1 @setccole(double, double) {
 define zeroext i1 @setccord(double, double) {
 ; CHECK-LABEL: setccord:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    fcmp.d %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.d.num %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    fcmp.d %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.d.num %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = fcmp ord double %0, %1
   ret i1 %3
@@ -105,10 +105,10 @@ define zeroext i1 @setccord(double, double) {
 define zeroext i1 @setccuno(double, double) {
 ; CHECK-LABEL: setccuno:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    fcmp.d %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.d.nan %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    fcmp.d %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.d.nan %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = fcmp uno double %0, %1
   ret i1 %3
@@ -117,10 +117,10 @@ define zeroext i1 @setccuno(double, double) {
 define zeroext i1 @setccueq(double, double) {
 ; CHECK-LABEL: setccueq:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    fcmp.d %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.d.eqnan %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    fcmp.d %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.d.eqnan %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = fcmp ueq double %0, %1
   ret i1 %3
@@ -129,10 +129,10 @@ define zeroext i1 @setccueq(double, double) {
 define zeroext i1 @setccune(double, double) {
 ; CHECK-LABEL: setccune:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    fcmp.d %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.d.nenan %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    fcmp.d %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.d.nenan %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = fcmp une double %0, %1
   ret i1 %3
@@ -141,10 +141,10 @@ define zeroext i1 @setccune(double, double) {
 define zeroext i1 @setccugt(double, double) {
 ; CHECK-LABEL: setccugt:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    fcmp.d %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.d.gtnan %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    fcmp.d %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.d.gtnan %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = fcmp ugt double %0, %1
   ret i1 %3
@@ -153,10 +153,10 @@ define zeroext i1 @setccugt(double, double) {
 define zeroext i1 @setccuge(double, double) {
 ; CHECK-LABEL: setccuge:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    fcmp.d %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.d.genan %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    fcmp.d %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.d.genan %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = fcmp uge double %0, %1
   ret i1 %3
@@ -165,10 +165,10 @@ define zeroext i1 @setccuge(double, double) {
 define zeroext i1 @setccult(double, double) {
 ; CHECK-LABEL: setccult:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    fcmp.d %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.d.ltnan %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    fcmp.d %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.d.ltnan %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = fcmp ult double %0, %1
   ret i1 %3
@@ -177,10 +177,10 @@ define zeroext i1 @setccult(double, double) {
 define zeroext i1 @setccule(double, double) {
 ; CHECK-LABEL: setccule:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    fcmp.d %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.d.lenan %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    fcmp.d %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.d.lenan %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = fcmp ule double %0, %1
   ret i1 %3
diff --git a/llvm/test/CodeGen/VE/setccf64i.ll b/llvm/test/CodeGen/VE/setccf64i.ll
index 59af1b4103f6b..f3a8d2f35f6f7 100644
--- a/llvm/test/CodeGen/VE/setccf64i.ll
+++ b/llvm/test/CodeGen/VE/setccf64i.ll
@@ -22,10 +22,10 @@ define zeroext i1 @setccoeq(double, double) {
 ; CHECK-LABEL: setccoeq:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    lea.sl %s1, 0
-; CHECK-NEXT:    fcmp.d %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.d.eq %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    fcmp.d %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.d.eq %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = fcmp oeq double %0, 0.0
   ret i1 %3
@@ -35,10 +35,10 @@ define zeroext i1 @setccone(double, double) {
 ; CHECK-LABEL: setccone:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    lea.sl %s1, 0
-; CHECK-NEXT:    fcmp.d %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.d.ne %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    fcmp.d %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.d.ne %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = fcmp one double %0, 0.0
   ret i1 %3
@@ -48,10 +48,10 @@ define zeroext i1 @setccogt(double, double) {
 ; CHECK-LABEL: setccogt:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    lea.sl %s1, 0
-; CHECK-NEXT:    fcmp.d %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.d.gt %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    fcmp.d %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.d.gt %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = fcmp ogt double %0, 0.0
   ret i1 %3
@@ -61,10 +61,10 @@ define zeroext i1 @setccoge(double, double) {
 ; CHECK-LABEL: setccoge:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    lea.sl %s1, 0
-; CHECK-NEXT:    fcmp.d %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.d.ge %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    fcmp.d %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.d.ge %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = fcmp oge double %0, 0.0
   ret i1 %3
@@ -74,10 +74,10 @@ define zeroext i1 @setccolt(double, double) {
 ; CHECK-LABEL: setccolt:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    lea.sl %s1, 0
-; CHECK-NEXT:    fcmp.d %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.d.lt %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    fcmp.d %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.d.lt %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = fcmp olt double %0, 0.0
   ret i1 %3
@@ -87,10 +87,10 @@ define zeroext i1 @setccole(double, double) {
 ; CHECK-LABEL: setccole:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    lea.sl %s1, 0
-; CHECK-NEXT:    fcmp.d %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.d.le %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    fcmp.d %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.d.le %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = fcmp ole double %0, 0.0
   ret i1 %3
@@ -99,10 +99,10 @@ define zeroext i1 @setccole(double, double) {
 define zeroext i1 @setccord(double, double) {
 ; CHECK-LABEL: setccord:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    fcmp.d %s1, %s0, %s0
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.d.num %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    fcmp.d %s0, %s0, %s0
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.d.num %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = fcmp ord double %0, 0.0
   ret i1 %3
@@ -111,10 +111,10 @@ define zeroext i1 @setccord(double, double) {
 define zeroext i1 @setccuno(double, double) {
 ; CHECK-LABEL: setccuno:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    fcmp.d %s1, %s0, %s0
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.d.nan %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    fcmp.d %s0, %s0, %s0
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.d.nan %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = fcmp uno double %0, 0.0
   ret i1 %3
@@ -124,10 +124,10 @@ define zeroext i1 @setccueq(double, double) {
 ; CHECK-LABEL: setccueq:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    lea.sl %s1, 0
-; CHECK-NEXT:    fcmp.d %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.d.eqnan %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    fcmp.d %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.d.eqnan %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = fcmp ueq double %0, 0.0
   ret i1 %3
@@ -137,10 +137,10 @@ define zeroext i1 @setccune(double, double) {
 ; CHECK-LABEL: setccune:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    lea.sl %s1, 0
-; CHECK-NEXT:    fcmp.d %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.d.nenan %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    fcmp.d %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.d.nenan %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = fcmp une double %0, 0.0
   ret i1 %3
@@ -150,10 +150,10 @@ define zeroext i1 @setccugt(double, double) {
 ; CHECK-LABEL: setccugt:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    lea.sl %s1, 0
-; CHECK-NEXT:    fcmp.d %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.d.gtnan %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    fcmp.d %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.d.gtnan %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = fcmp ugt double %0, 0.0
   ret i1 %3
@@ -163,10 +163,10 @@ define zeroext i1 @setccuge(double, double) {
 ; CHECK-LABEL: setccuge:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    lea.sl %s1, 0
-; CHECK-NEXT:    fcmp.d %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.d.genan %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    fcmp.d %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.d.genan %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = fcmp uge double %0, 0.0
   ret i1 %3
@@ -176,10 +176,10 @@ define zeroext i1 @setccult(double, double) {
 ; CHECK-LABEL: setccult:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    lea.sl %s1, 0
-; CHECK-NEXT:    fcmp.d %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.d.ltnan %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    fcmp.d %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.d.ltnan %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = fcmp ult double %0, 0.0
   ret i1 %3
@@ -189,10 +189,10 @@ define zeroext i1 @setccule(double, double) {
 ; CHECK-LABEL: setccule:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    lea.sl %s1, 0
-; CHECK-NEXT:    fcmp.d %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.d.lenan %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    fcmp.d %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.d.lenan %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = fcmp ule double %0, 0.0
   ret i1 %3
diff --git a/llvm/test/CodeGen/VE/setcci32.ll b/llvm/test/CodeGen/VE/setcci32.ll
index 7e92a2c7f5ede..15bf130fec8b4 100644
--- a/llvm/test/CodeGen/VE/setcci32.ll
+++ b/llvm/test/CodeGen/VE/setcci32.ll
@@ -3,10 +3,12 @@
 define zeroext i1 @setcceq(i32, i32) {
 ; CHECK-LABEL: setcceq:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    cmps.w.sx %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.w.eq %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    cmps.w.sx %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.w.eq %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = icmp eq i32 %0, %1
   ret i1 %3
@@ -15,10 +17,12 @@ define zeroext i1 @setcceq(i32, i32) {
 define zeroext i1 @setccne(i32, i32) {
 ; CHECK-LABEL: setccne:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    cmps.w.sx %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.w.ne %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    cmps.w.sx %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.w.ne %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = icmp ne i32 %0, %1
   ret i1 %3
@@ -27,10 +31,12 @@ define zeroext i1 @setccne(i32, i32) {
 define zeroext i1 @setccugt(i32, i32) {
 ; CHECK-LABEL: setccugt:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    cmpu.w %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.w.gt %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    cmpu.w %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.w.gt %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = icmp ugt i32 %0, %1
   ret i1 %3
@@ -39,10 +45,12 @@ define zeroext i1 @setccugt(i32, i32) {
 define zeroext i1 @setccuge(i32, i32) {
 ; CHECK-LABEL: setccuge:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    cmpu.w %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.w.ge %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    cmpu.w %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.w.ge %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = icmp uge i32 %0, %1
   ret i1 %3
@@ -51,10 +59,12 @@ define zeroext i1 @setccuge(i32, i32) {
 define zeroext i1 @setccult(i32, i32) {
 ; CHECK-LABEL: setccult:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    cmpu.w %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.w.lt %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    cmpu.w %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.w.lt %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = icmp ult i32 %0, %1
   ret i1 %3
@@ -63,10 +73,12 @@ define zeroext i1 @setccult(i32, i32) {
 define zeroext i1 @setccule(i32, i32) {
 ; CHECK-LABEL: setccule:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    cmpu.w %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.w.le %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    cmpu.w %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.w.le %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = icmp ule i32 %0, %1
   ret i1 %3
@@ -75,10 +87,12 @@ define zeroext i1 @setccule(i32, i32) {
 define zeroext i1 @setccsgt(i32, i32) {
 ; CHECK-LABEL: setccsgt:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    cmps.w.sx %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.w.gt %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    cmps.w.sx %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.w.gt %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = icmp sgt i32 %0, %1
   ret i1 %3
@@ -87,10 +101,12 @@ define zeroext i1 @setccsgt(i32, i32) {
 define zeroext i1 @setccsge(i32, i32) {
 ; CHECK-LABEL: setccsge:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    cmps.w.sx %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.w.ge %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    cmps.w.sx %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.w.ge %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = icmp sge i32 %0, %1
   ret i1 %3
@@ -99,10 +115,12 @@ define zeroext i1 @setccsge(i32, i32) {
 define zeroext i1 @setccslt(i32, i32) {
 ; CHECK-LABEL: setccslt:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    cmps.w.sx %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.w.lt %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    cmps.w.sx %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.w.lt %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = icmp slt i32 %0, %1
   ret i1 %3
@@ -111,10 +129,12 @@ define zeroext i1 @setccslt(i32, i32) {
 define zeroext i1 @setccsle(i32, i32) {
 ; CHECK-LABEL: setccsle:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    cmps.w.sx %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.w.le %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    cmps.w.sx %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.w.le %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = icmp sle i32 %0, %1
   ret i1 %3
diff --git a/llvm/test/CodeGen/VE/setcci32i.ll b/llvm/test/CodeGen/VE/setcci32i.ll
index c0d1fb0a67e4a..ac226190ae44f 100644
--- a/llvm/test/CodeGen/VE/setcci32i.ll
+++ b/llvm/test/CodeGen/VE/setcci32i.ll
@@ -3,11 +3,12 @@
 define zeroext i1 @setcceq(i32, i32) {
 ; CHECK-LABEL: setcceq:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s1, 12, (0)1
-; CHECK-NEXT:    cmps.w.sx %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.w.eq %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    cmps.w.sx %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.w.eq %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = icmp eq i32 %0, 12
   ret i1 %3
@@ -16,11 +17,12 @@ define zeroext i1 @setcceq(i32, i32) {
 define zeroext i1 @setccne(i32, i32) {
 ; CHECK-LABEL: setccne:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s1, 12, (0)1
-; CHECK-NEXT:    cmps.w.sx %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.w.ne %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    cmps.w.sx %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.w.ne %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = icmp ne i32 %0, 12
   ret i1 %3
@@ -29,11 +31,12 @@ define zeroext i1 @setccne(i32, i32) {
 define zeroext i1 @setccugt(i32, i32) {
 ; CHECK-LABEL: setccugt:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s1, 12, (0)1
-; CHECK-NEXT:    cmpu.w %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.w.gt %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    cmpu.w %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.w.gt %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = icmp ugt i32 %0, 12
   ret i1 %3
@@ -42,11 +45,12 @@ define zeroext i1 @setccugt(i32, i32) {
 define zeroext i1 @setccuge(i32, i32) {
 ; CHECK-LABEL: setccuge:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s1, 11, (0)1
-; CHECK-NEXT:    cmpu.w %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.w.gt %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    cmpu.w %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.w.gt %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = icmp uge i32 %0, 12
   ret i1 %3
@@ -55,11 +59,12 @@ define zeroext i1 @setccuge(i32, i32) {
 define zeroext i1 @setccult(i32, i32) {
 ; CHECK-LABEL: setccult:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s1, 12, (0)1
-; CHECK-NEXT:    cmpu.w %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.w.lt %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    cmpu.w %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.w.lt %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = icmp ult i32 %0, 12
   ret i1 %3
@@ -68,11 +73,12 @@ define zeroext i1 @setccult(i32, i32) {
 define zeroext i1 @setccule(i32, i32) {
 ; CHECK-LABEL: setccule:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s1, 13, (0)1
-; CHECK-NEXT:    cmpu.w %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.w.lt %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    cmpu.w %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.w.lt %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = icmp ule i32 %0, 12
   ret i1 %3
@@ -81,11 +87,12 @@ define zeroext i1 @setccule(i32, i32) {
 define zeroext i1 @setccsgt(i32, i32) {
 ; CHECK-LABEL: setccsgt:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s1, 12, (0)1
-; CHECK-NEXT:    cmps.w.sx %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.w.gt %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    cmps.w.sx %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.w.gt %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = icmp sgt i32 %0, 12
   ret i1 %3
@@ -94,11 +101,12 @@ define zeroext i1 @setccsgt(i32, i32) {
 define zeroext i1 @setccsge(i32, i32) {
 ; CHECK-LABEL: setccsge:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s1, 11, (0)1
-; CHECK-NEXT:    cmps.w.sx %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.w.gt %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    cmps.w.sx %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.w.gt %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = icmp sge i32 %0, 12
   ret i1 %3
@@ -107,11 +115,12 @@ define zeroext i1 @setccsge(i32, i32) {
 define zeroext i1 @setccslt(i32, i32) {
 ; CHECK-LABEL: setccslt:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s1, 12, (0)1
-; CHECK-NEXT:    cmps.w.sx %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.w.lt %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    cmps.w.sx %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.w.lt %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = icmp slt i32 %0, 12
   ret i1 %3
@@ -120,11 +129,12 @@ define zeroext i1 @setccslt(i32, i32) {
 define zeroext i1 @setccsle(i32, i32) {
 ; CHECK-LABEL: setccsle:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s1, 13, (0)1
-; CHECK-NEXT:    cmps.w.sx %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.w.lt %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    cmps.w.sx %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.w.lt %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = icmp sle i32 %0, 12
   ret i1 %3
diff --git a/llvm/test/CodeGen/VE/setcci64.ll b/llvm/test/CodeGen/VE/setcci64.ll
index 8b86601594da9..5cae80a60f06d 100644
--- a/llvm/test/CodeGen/VE/setcci64.ll
+++ b/llvm/test/CodeGen/VE/setcci64.ll
@@ -3,10 +3,10 @@
 define zeroext i1 @setcceq(i64, i64) {
 ; CHECK-LABEL: setcceq:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    cmps.l %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.l.eq %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    cmps.l %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.l.eq %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = icmp eq i64 %0, %1
   ret i1 %3
@@ -15,10 +15,10 @@ define zeroext i1 @setcceq(i64, i64) {
 define zeroext i1 @setccne(i64, i64) {
 ; CHECK-LABEL: setccne:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    cmps.l %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.l.ne %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    cmps.l %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.l.ne %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = icmp ne i64 %0, %1
   ret i1 %3
@@ -27,10 +27,10 @@ define zeroext i1 @setccne(i64, i64) {
 define zeroext i1 @setccugt(i64, i64) {
 ; CHECK-LABEL: setccugt:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    cmpu.l %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.l.gt %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    cmpu.l %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.l.gt %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = icmp ugt i64 %0, %1
   ret i1 %3
@@ -39,10 +39,10 @@ define zeroext i1 @setccugt(i64, i64) {
 define zeroext i1 @setccuge(i64, i64) {
 ; CHECK-LABEL: setccuge:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    cmpu.l %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.l.ge %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    cmpu.l %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.l.ge %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = icmp uge i64 %0, %1
   ret i1 %3
@@ -51,10 +51,10 @@ define zeroext i1 @setccuge(i64, i64) {
 define zeroext i1 @setccult(i64, i64) {
 ; CHECK-LABEL: setccult:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    cmpu.l %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.l.lt %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    cmpu.l %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.l.lt %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = icmp ult i64 %0, %1
   ret i1 %3
@@ -63,10 +63,10 @@ define zeroext i1 @setccult(i64, i64) {
 define zeroext i1 @setccule(i64, i64) {
 ; CHECK-LABEL: setccule:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    cmpu.l %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.l.le %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    cmpu.l %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.l.le %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = icmp ule i64 %0, %1
   ret i1 %3
@@ -75,10 +75,10 @@ define zeroext i1 @setccule(i64, i64) {
 define zeroext i1 @setccsgt(i64, i64) {
 ; CHECK-LABEL: setccsgt:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    cmps.l %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.l.gt %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    cmps.l %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.l.gt %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = icmp sgt i64 %0, %1
   ret i1 %3
@@ -87,10 +87,10 @@ define zeroext i1 @setccsgt(i64, i64) {
 define zeroext i1 @setccsge(i64, i64) {
 ; CHECK-LABEL: setccsge:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    cmps.l %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.l.ge %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    cmps.l %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.l.ge %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = icmp sge i64 %0, %1
   ret i1 %3
@@ -99,10 +99,10 @@ define zeroext i1 @setccsge(i64, i64) {
 define zeroext i1 @setccslt(i64, i64) {
 ; CHECK-LABEL: setccslt:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    cmps.l %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.l.lt %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    cmps.l %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.l.lt %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = icmp slt i64 %0, %1
   ret i1 %3
@@ -111,10 +111,10 @@ define zeroext i1 @setccslt(i64, i64) {
 define zeroext i1 @setccsle(i64, i64) {
 ; CHECK-LABEL: setccsle:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    cmps.l %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.l.le %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    cmps.l %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.l.le %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = icmp sle i64 %0, %1
   ret i1 %3
diff --git a/llvm/test/CodeGen/VE/setcci64i.ll b/llvm/test/CodeGen/VE/setcci64i.ll
index aecbe40b0a34d..c73db4c72276d 100644
--- a/llvm/test/CodeGen/VE/setcci64i.ll
+++ b/llvm/test/CodeGen/VE/setcci64i.ll
@@ -4,10 +4,10 @@ define zeroext i1 @setcceq(i64, i64) {
 ; CHECK-LABEL: setcceq:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    or %s1, 12, (0)1
-; CHECK-NEXT:    cmps.l %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.l.eq %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    cmps.l %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.l.eq %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = icmp eq i64 %0, 12
   ret i1 %3
@@ -17,10 +17,10 @@ define zeroext i1 @setccne(i64, i64) {
 ; CHECK-LABEL: setccne:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    or %s1, 12, (0)1
-; CHECK-NEXT:    cmps.l %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.l.ne %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    cmps.l %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.l.ne %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = icmp ne i64 %0, 12
   ret i1 %3
@@ -30,10 +30,10 @@ define zeroext i1 @setccugt(i64, i64) {
 ; CHECK-LABEL: setccugt:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    or %s1, 12, (0)1
-; CHECK-NEXT:    cmpu.l %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.l.gt %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    cmpu.l %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.l.gt %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = icmp ugt i64 %0, 12
   ret i1 %3
@@ -43,10 +43,10 @@ define zeroext i1 @setccuge(i64, i64) {
 ; CHECK-LABEL: setccuge:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    or %s1, 11, (0)1
-; CHECK-NEXT:    cmpu.l %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.l.gt %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    cmpu.l %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.l.gt %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = icmp uge i64 %0, 12
   ret i1 %3
@@ -56,10 +56,10 @@ define zeroext i1 @setccult(i64, i64) {
 ; CHECK-LABEL: setccult:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    or %s1, 12, (0)1
-; CHECK-NEXT:    cmpu.l %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.l.lt %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    cmpu.l %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.l.lt %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = icmp ult i64 %0, 12
   ret i1 %3
@@ -69,10 +69,10 @@ define zeroext i1 @setccule(i64, i64) {
 ; CHECK-LABEL: setccule:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    or %s1, 13, (0)1
-; CHECK-NEXT:    cmpu.l %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.l.lt %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    cmpu.l %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.l.lt %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = icmp ule i64 %0, 12
   ret i1 %3
@@ -82,10 +82,10 @@ define zeroext i1 @setccsgt(i64, i64) {
 ; CHECK-LABEL: setccsgt:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    or %s1, 12, (0)1
-; CHECK-NEXT:    cmps.l %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.l.gt %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    cmps.l %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.l.gt %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = icmp sgt i64 %0, 12
   ret i1 %3
@@ -95,10 +95,10 @@ define zeroext i1 @setccsge(i64, i64) {
 ; CHECK-LABEL: setccsge:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    or %s1, 11, (0)1
-; CHECK-NEXT:    cmps.l %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.l.gt %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    cmps.l %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.l.gt %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = icmp sge i64 %0, 12
   ret i1 %3
@@ -108,10 +108,10 @@ define zeroext i1 @setccslt(i64, i64) {
 ; CHECK-LABEL: setccslt:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    or %s1, 12, (0)1
-; CHECK-NEXT:    cmps.l %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.l.lt %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    cmps.l %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.l.lt %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = icmp slt i64 %0, 12
   ret i1 %3
@@ -121,10 +121,10 @@ define zeroext i1 @setccsle(i64, i64) {
 ; CHECK-LABEL: setccsle:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    or %s1, 13, (0)1
-; CHECK-NEXT:    cmps.l %s1, %s0, %s1
-; CHECK-NEXT:    or %s0, 0, (0)1
-; CHECK-NEXT:    cmov.l.lt %s0, (63)0, %s1
-; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    cmps.l %s0, %s0, %s1
+; CHECK-NEXT:    or %s1, 0, (0)1
+; CHECK-NEXT:    cmov.l.lt %s1, (63)0, %s0
+; CHECK-NEXT:    adds.w.zx %s0, %s1, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = icmp sle i64 %0, 12
   ret i1 %3
diff --git a/llvm/test/CodeGen/VE/sext_zext_load.ll b/llvm/test/CodeGen/VE/sext_zext_load.ll
index b9fc6bc4daf74..600a02a5b1303 100644
--- a/llvm/test/CodeGen/VE/sext_zext_load.ll
+++ b/llvm/test/CodeGen/VE/sext_zext_load.ll
@@ -267,8 +267,8 @@ define signext i8 @func37() {
 ; CHECK-LABEL: func37:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    ld1b.zx %s0, 191(, %s11)
-; CHECK-NEXT:    sla.w.sx %s0, %s0, 31
-; CHECK-NEXT:    sra.w.sx %s0, %s0, 31
+; CHECK-NEXT:    sll %s0, %s0, 63
+; CHECK-NEXT:    sra.l %s0, %s0, 63
 ; CHECK-NEXT:    or %s11, 0, %s9
   %a = alloca i1, align 1
   %a.val = load i1, i1* %a, align 1
@@ -280,8 +280,8 @@ define signext i16 @func38() {
 ; CHECK-LABEL: func38:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    ld1b.zx %s0, 191(, %s11)
-; CHECK-NEXT:    sla.w.sx %s0, %s0, 31
-; CHECK-NEXT:    sra.w.sx %s0, %s0, 31
+; CHECK-NEXT:    sll %s0, %s0, 63
+; CHECK-NEXT:    sra.l %s0, %s0, 63
 ; CHECK-NEXT:    or %s11, 0, %s9
   %a = alloca i1, align 1
   %a.val = load i1, i1* %a, align 1
@@ -293,8 +293,8 @@ define signext i32 @func39() {
 ; CHECK-LABEL: func39:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    ld1b.zx %s0, 191(, %s11)
-; CHECK-NEXT:    sla.w.sx %s0, %s0, 31
-; CHECK-NEXT:    sra.w.sx %s0, %s0, 31
+; CHECK-NEXT:    sll %s0, %s0, 63
+; CHECK-NEXT:    sra.l %s0, %s0, 63
 ; CHECK-NEXT:    or %s11, 0, %s9
   %a = alloca i1, align 1
   %a.val = load i1, i1* %a, align 1
diff --git a/llvm/test/CodeGen/VE/subtraction.ll b/llvm/test/CodeGen/VE/subtraction.ll
index 1bd85d4290929..43a30bfe1e1b7 100644
--- a/llvm/test/CodeGen/VE/subtraction.ll
+++ b/llvm/test/CodeGen/VE/subtraction.ll
@@ -3,9 +3,11 @@
 define signext i8 @func1(i8 signext %0, i8 signext %1) {
 ; CHECK-LABEL: func1:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    subs.w.sx %s0, %s0, %s1
-; CHECK-NEXT:    sla.w.sx %s0, %s0, 24
-; CHECK-NEXT:    sra.w.sx %s0, %s0, 24
+; CHECK-NEXT:    sll %s0, %s0, 56
+; CHECK-NEXT:    sra.l %s0, %s0, 56
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = sub i8 %0, %1
   ret i8 %3
@@ -14,9 +16,11 @@ define signext i8 @func1(i8 signext %0, i8 signext %1) {
 define signext i16 @func2(i16 signext %0, i16 signext %1) {
 ; CHECK-LABEL: func2:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    subs.w.sx %s0, %s0, %s1
-; CHECK-NEXT:    sla.w.sx %s0, %s0, 16
-; CHECK-NEXT:    sra.w.sx %s0, %s0, 16
+; CHECK-NEXT:    sll %s0, %s0, 48
+; CHECK-NEXT:    sra.l %s0, %s0, 48
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = sub i16 %0, %1
   ret i16 %3
@@ -25,6 +29,8 @@ define signext i16 @func2(i16 signext %0, i16 signext %1) {
 define i32 @func3(i32 %0, i32 %1) {
 ; CHECK-LABEL: func3:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    subs.w.sx %s0, %s0, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = sub nsw i32 %0, %1
@@ -43,6 +49,8 @@ define i64 @func4(i64 %0, i64 %1) {
 define zeroext i8 @func6(i8 zeroext %0, i8 zeroext %1) {
 ; CHECK-LABEL: func6:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    subs.w.sx %s0, %s0, %s1
 ; CHECK-NEXT:    and %s0, %s0, (56)0
 ; CHECK-NEXT:    or %s11, 0, %s9
@@ -53,6 +61,8 @@ define zeroext i8 @func6(i8 zeroext %0, i8 zeroext %1) {
 define zeroext i16 @func7(i16 zeroext %0, i16 zeroext %1) {
 ; CHECK-LABEL: func7:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    subs.w.sx %s0, %s0, %s1
 ; CHECK-NEXT:    and %s0, %s0, (48)0
 ; CHECK-NEXT:    or %s11, 0, %s9
@@ -63,6 +73,8 @@ define zeroext i16 @func7(i16 zeroext %0, i16 zeroext %1) {
 define i32 @func8(i32 %0, i32 %1) {
 ; CHECK-LABEL: func8:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    subs.w.sx %s0, %s0, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = sub i32 %0, %1
@@ -81,9 +93,10 @@ define i64 @func9(i64 %0, i64 %1) {
 define signext i8 @func13(i8 signext %0, i8 signext %1) {
 ; CHECK-LABEL: func13:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    adds.w.sx %s0, -5, %s0
-; CHECK-NEXT:    sla.w.sx %s0, %s0, 24
-; CHECK-NEXT:    sra.w.sx %s0, %s0, 24
+; CHECK-NEXT:    sll %s0, %s0, 56
+; CHECK-NEXT:    sra.l %s0, %s0, 56
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = add i8 %0, -5
   ret i8 %3
@@ -92,9 +105,10 @@ define signext i8 @func13(i8 signext %0, i8 signext %1) {
 define signext i16 @func14(i16 signext %0, i16 signext %1) {
 ; CHECK-LABEL: func14:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    adds.w.sx %s0, -5, %s0
-; CHECK-NEXT:    sla.w.sx %s0, %s0, 16
-; CHECK-NEXT:    sra.w.sx %s0, %s0, 16
+; CHECK-NEXT:    sll %s0, %s0, 48
+; CHECK-NEXT:    sra.l %s0, %s0, 48
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = add i16 %0, -5
   ret i16 %3
@@ -103,6 +117,7 @@ define signext i16 @func14(i16 signext %0, i16 signext %1) {
 define i32 @func15(i32 %0, i32 %1) {
 ; CHECK-LABEL: func15:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    adds.w.sx %s0, -5, %s0
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = add nsw i32 %0, -5
@@ -121,6 +136,7 @@ define i64 @func16(i64 %0, i64 %1) {
 define zeroext i8 @func18(i8 zeroext %0, i8 zeroext %1) {
 ; CHECK-LABEL: func18:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    adds.w.sx %s0, -5, %s0
 ; CHECK-NEXT:    and %s0, %s0, (56)0
 ; CHECK-NEXT:    or %s11, 0, %s9
@@ -131,6 +147,7 @@ define zeroext i8 @func18(i8 zeroext %0, i8 zeroext %1) {
 define zeroext i16 @func19(i16 zeroext %0, i16 zeroext %1) {
 ; CHECK-LABEL: func19:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    adds.w.sx %s0, -5, %s0
 ; CHECK-NEXT:    and %s0, %s0, (48)0
 ; CHECK-NEXT:    or %s11, 0, %s9
@@ -141,6 +158,7 @@ define zeroext i16 @func19(i16 zeroext %0, i16 zeroext %1) {
 define i32 @func20(i32 %0, i32 %1) {
 ; CHECK-LABEL: func20:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    adds.w.sx %s0, -5, %s0
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = add i32 %0, -5
diff --git a/llvm/test/CodeGen/VE/truncstore.ll b/llvm/test/CodeGen/VE/truncstore.ll
index 357cc6b117914..97a4da4cd93be 100644
--- a/llvm/test/CodeGen/VE/truncstore.ll
+++ b/llvm/test/CodeGen/VE/truncstore.ll
@@ -33,7 +33,6 @@ define void @func2(i8 signext %p, i32* %a) {
 define void @func3(i8 signext %p, i64* %a) {
 ; CHECK-LABEL: func3:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    st %s0, (, %s1)
 ; CHECK-NEXT:    or %s11, 0, %s9
   %p.conv = sext i8 %p to i64
@@ -54,7 +53,6 @@ define void @func5(i16 signext %p, i32* %a) {
 define void @func6(i16 signext %p, i64* %a) {
 ; CHECK-LABEL: func6:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    st %s0, (, %s1)
 ; CHECK-NEXT:    or %s11, 0, %s9
   %p.conv = sext i16 %p to i64
diff --git a/llvm/test/CodeGen/VE/va_caller.ll b/llvm/test/CodeGen/VE/va_caller.ll
index 345ab80867f1f..b43ce999c589d 100644
--- a/llvm/test/CodeGen/VE/va_caller.ll
+++ b/llvm/test/CodeGen/VE/va_caller.ll
@@ -6,38 +6,36 @@ define i32 @caller() {
 ; CHECK-LABEL: caller:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    st %s18, 48(, %s9) # 8-byte Folded Spill
-; CHECK-NEXT:    or %s7, 0, (0)1
-; CHECK-NEXT:    st %s7, 280(, %s11)
+; CHECK-NEXT:    or %s18, 0, (0)1
+; CHECK-NEXT:    st %s18, 280(, %s11)
 ; CHECK-NEXT:    or %s0, 11, (0)1
 ; CHECK-NEXT:    st %s0, 272(, %s11)
-; CHECK-NEXT:    st %s7, 264(, %s11)
+; CHECK-NEXT:    st %s18, 264(, %s11)
 ; CHECK-NEXT:    or %s0, 10, (0)1
 ; CHECK-NEXT:    st %s0, 256(, %s11)
 ; CHECK-NEXT:    lea.sl %s0, 1075970048
 ; CHECK-NEXT:    st %s0, 248(, %s11)
 ; CHECK-NEXT:    or %s0, 8, (0)1
 ; CHECK-NEXT:    st %s0, 240(, %s11)
-; CHECK-NEXT:    st %s7, 232(, %s11)
-; CHECK-NEXT:    lea %s0, 1086324736
-; CHECK-NEXT:    stl %s0, 228(, %s11)
+; CHECK-NEXT:    st %s18, 232(, %s11)
 ; CHECK-NEXT:    or %s5, 5, (0)1
-; CHECK-NEXT:    stl %s5, 216(, %s11)
+; CHECK-NEXT:    st %s5, 216(, %s11)
 ; CHECK-NEXT:    or %s4, 4, (0)1
-; CHECK-NEXT:    stl %s4, 208(, %s11)
+; CHECK-NEXT:    st %s4, 208(, %s11)
 ; CHECK-NEXT:    or %s3, 3, (0)1
-; CHECK-NEXT:    stl %s3, 200(, %s11)
+; CHECK-NEXT:    st %s3, 200(, %s11)
 ; CHECK-NEXT:    or %s2, 2, (0)1
-; CHECK-NEXT:    stl %s2, 192(, %s11)
+; CHECK-NEXT:    st %s2, 192(, %s11)
 ; CHECK-NEXT:    or %s1, 1, (0)1
-; CHECK-NEXT:    stl %s1, 184(, %s11)
-; CHECK-NEXT:    or %s18, 0, (0)1
+; CHECK-NEXT:    st %s1, 184(, %s11)
+; CHECK-NEXT:    st %s18, 176(, %s11)
+; CHECK-NEXT:    lea.sl %s6, 1086324736
 ; CHECK-NEXT:    lea %s0, func@lo
 ; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    lea.sl %s12, func@hi(, %s0)
-; CHECK-NEXT:    lea.sl %s6, 1086324736
-; CHECK-NEXT:    stl %s18, 176(, %s11)
+; CHECK-NEXT:    st %s6, 224(, %s11)
 ; CHECK-NEXT:    or %s0, 0, %s18
-; CHECK-NEXT:    # kill: def $sf6 killed $sf6 killed $sx6
+; CHECK-NEXT:    or %s7, 0, %s18
 ; CHECK-NEXT:    bsic %s10, (, %s12)
 ; CHECK-NEXT:    or %s0, 0, %s18
 ; CHECK-NEXT:    ld %s18, 48(, %s9) # 8-byte Folded Reload
diff --git a/llvm/test/CodeGen/VE/xor.ll b/llvm/test/CodeGen/VE/xor.ll
index d1701f6ef5017..b3336bb72ff5a 100644
--- a/llvm/test/CodeGen/VE/xor.ll
+++ b/llvm/test/CodeGen/VE/xor.ll
@@ -21,7 +21,9 @@ define zeroext i8 @func8z(i8 zeroext %a, i8 zeroext %b) {
 define signext i8 @funci8s(i8 signext %a) {
 ; CHECK-LABEL: funci8s:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    xor %s0, 5, %s0
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %res = xor i8 %a, 5
   ret i8 %res
@@ -30,8 +32,10 @@ define signext i8 @funci8s(i8 signext %a) {
 define zeroext i8 @funci8z(i8 zeroext %a) {
 ; CHECK-LABEL: funci8z:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    lea %s1, 251
 ; CHECK-NEXT:    xor %s0, %s0, %s1
+; CHECK-NEXT:    adds.w.zx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %res = xor i8 -5, %a
   ret i8 %res
@@ -58,7 +62,9 @@ define zeroext i16 @func16z(i16 zeroext %a, i16 zeroext %b) {
 define signext i16 @funci16s(i16 signext %a) {
 ; CHECK-LABEL: funci16s:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    xor %s0, -1, %s0
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %res = xor i16 %a, 65535
   ret i16 %res
@@ -67,7 +73,9 @@ define signext i16 @funci16s(i16 signext %a) {
 define zeroext i16 @funci16z(i16 zeroext %a) {
 ; CHECK-LABEL: funci16z:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    xor %s0, %s0, (52)0
+; CHECK-NEXT:    adds.w.zx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %res = xor i16 4095, %a
   ret i16 %res
@@ -94,7 +102,9 @@ define zeroext i32 @func32z(i32 zeroext %a, i32 zeroext %b) {
 define signext i32 @funci32s(i32 signext %a) {
 ; CHECK-LABEL: funci32s:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    xor %s0, %s0, (36)0
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %res = xor i32 %a, 268435455
   ret i32 %res
@@ -103,7 +113,9 @@ define signext i32 @funci32s(i32 signext %a) {
 define zeroext i32 @funci32z(i32 zeroext %a) {
 ; CHECK-LABEL: funci32z:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    xor %s0, %s0, (36)0
+; CHECK-NEXT:    adds.w.zx %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %res = xor i32 %a, 268435455
   ret i32 %res
@@ -112,6 +124,7 @@ define zeroext i32 @funci32z(i32 zeroext %a) {
 define i32 @funci32_another(i32 %0) {
 ; CHECK-LABEL: funci32_another:
 ; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    xor %s0, %s0, (33)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = xor i32 %0, -2147483648

From 914dbf4808d46632cc7b8dda861a11f978083416 Mon Sep 17 00:00:00 2001
From: Justin Hibbits <jrh29@alumni.cwru.edu>
Date: Thu, 16 Apr 2020 09:42:39 -0500
Subject: [PATCH 0801/1035] PowerPC: Fix SPE extloadf32 handling.

The patterns were incorrect copies from the FPU code, and are
unnecessary, since there's no extended load for SPE.  Just let LLVM
itself do the work by marking it expand.

Reviewed By: #powerpc, lkail
Differential Revision: https://reviews.llvm.org/D78670
---
 llvm/lib/Target/PowerPC/PPCISelLowering.cpp |  3 +
 llvm/lib/Target/PowerPC/PPCInstrSPE.td      | 10 ----
 llvm/test/CodeGen/PowerPC/spe.ll            | 61 +++++++++++++++++++++
 3 files changed, 64 insertions(+), 10 deletions(-)

diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index 13dbcbe10d337..69afed8fb41a7 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -339,6 +339,9 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM,
     setOperationAction(ISD::FMA  , MVT::f32, Legal);
   }
 
+  if (Subtarget.hasSPE())
+    setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand);
+
   setOperationAction(ISD::FLT_ROUNDS_, MVT::i32, Custom);
 
   // If we're enabling GP optimizations, use hardware square root
diff --git a/llvm/lib/Target/PowerPC/PPCInstrSPE.td b/llvm/lib/Target/PowerPC/PPCInstrSPE.td
index 858eb0c9fe500..299b34ca82836 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrSPE.td
+++ b/llvm/lib/Target/PowerPC/PPCInstrSPE.td
@@ -819,16 +819,6 @@ def SPESTWX       : XForm_8<31, 151, (outs), (ins spe4rc:$rS, memrr:$dst),
 
 } // HasSPE
 
-let Predicates = [HasSPE] in {
-def : Pat<(f64 (extloadf32 iaddr:$src)),
-          (COPY_TO_REGCLASS (SPELWZ iaddr:$src), SPERC)>;
-def : Pat<(f64 (extloadf32 xaddr:$src)),
-          (COPY_TO_REGCLASS (SPELWZX xaddr:$src), SPERC)>;
-
-def : Pat<(f64 (fpextend f32:$src)),
-          (COPY_TO_REGCLASS $src, SPERC)>;
-}
-
 let Predicates = [HasSPE] in {
 def SELECT_CC_SPE4 : PPCCustomInserterPseudo<(outs spe4rc:$dst),
                             (ins crrc:$cond, spe4rc:$T, spe4rc:$F,
diff --git a/llvm/test/CodeGen/PowerPC/spe.ll b/llvm/test/CodeGen/PowerPC/spe.ll
index 1c4c7a3398177..1e4cecb1d136d 100644
--- a/llvm/test/CodeGen/PowerPC/spe.ll
+++ b/llvm/test/CodeGen/PowerPC/spe.ll
@@ -1422,3 +1422,64 @@ for.body:                                         ; preds = %for.body, %entry
 declare float @llvm.fma.f32(float, float, float) #1
 
 attributes #1 = { nounwind readnone speculatable willreturn }
+
+%struct.a = type { float, float }
+
+define void @d(%struct.a* %e, %struct.a* %f) {
+; CHECK-LABEL: d:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    mflr 0
+; CHECK-NEXT:    stw 0, 4(1)
+; CHECK-NEXT:    stwu 1, -48(1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    .cfi_offset lr, 4
+; CHECK-NEXT:    .cfi_offset r29, -12
+; CHECK-NEXT:    .cfi_offset r30, -8
+; CHECK-NEXT:    .cfi_offset r29, -40
+; CHECK-NEXT:    .cfi_offset r30, -32
+; CHECK-NEXT:    lwz 4, 0(4)
+; CHECK-NEXT:    lwz 3, 0(3)
+; CHECK-NEXT:    stw 29, 36(1) # 4-byte Folded Spill
+; CHECK-NEXT:    evstdd 29, 8(1) # 8-byte Folded Spill
+; CHECK-NEXT:    efdcfs 29, 4
+; CHECK-NEXT:    stw 30, 40(1) # 4-byte Folded Spill
+; CHECK-NEXT:    mr 4, 29
+; CHECK-NEXT:    evstdd 30, 16(1) # 8-byte Folded Spill
+; CHECK-NEXT:    efdcfs 30, 3
+; CHECK-NEXT:    evmergehi 3, 29, 29
+; CHECK-NEXT:    mtctr 3
+; CHECK-NEXT:    # kill: def $r3 killed $r3 killed $s3
+; CHECK-NEXT:    bctrl
+; CHECK-NEXT:    evmergehi 3, 30, 30
+; CHECK-NEXT:    mr 4, 30
+; CHECK-NEXT:    mtctr 3
+; CHECK-NEXT:    # kill: def $r3 killed $r3 killed $s3
+; CHECK-NEXT:    bctrl
+; CHECK-NEXT:    li 3, .LCPI58_0@l
+; CHECK-NEXT:    lis 4, .LCPI58_0@ha
+; CHECK-NEXT:    evlddx 3, 4, 3
+; CHECK-NEXT:    evldd 30, 16(1) # 8-byte Folded Reload
+; CHECK-NEXT:    efdmul 3, 29, 3
+; CHECK-NEXT:    evldd 29, 8(1) # 8-byte Folded Reload
+; CHECK-NEXT:    efscfd 3, 3
+; CHECK-NEXT:    stw 3, 0(3)
+; CHECK-NEXT:    lwz 30, 40(1) # 4-byte Folded Reload
+; CHECK-NEXT:    lwz 29, 36(1) # 4-byte Folded Reload
+; CHECK-NEXT:    lwz 0, 52(1)
+; CHECK-NEXT:    addi 1, 1, 48
+; CHECK-NEXT:    mtlr 0
+; CHECK-NEXT:    blr
+entry:
+  %0 = getelementptr %struct.a, %struct.a* %f, i32 0, i32 0
+  %1 = load float, float* undef
+  %conv = fpext float %1 to double
+  %2 = load float, float* %0
+  %g = fpext float %2 to double
+  %3 = call i32 undef(double %g)
+  %h = call i32 undef(double %conv)
+  %n = sitofp i32 %3 to double
+  %k = fmul double %g, %n
+  %l = fptrunc double %k to float
+  store float %l, float* undef
+  ret void
+}

From 7e9153e940e21a937ff3a0e7425eb1b24bd1bb76 Mon Sep 17 00:00:00 2001
From: Justin Hibbits <jrh29@alumni.cwru.edu>
Date: Wed, 8 Apr 2020 21:29:40 -0500
Subject: [PATCH 0802/1035] PowerPC: Don't lower SELECT_CC to PPCISD::FSEL on
 SPE

SPE doesn't have a fsel instruction, so don't try to lower to it.

This fixes a "Cannot select: tN: f64 = PPCISD::FSEL tX, tY, tZ" error.

Reviewed By: #powerpc, lkail
Differential Revision: https://reviews.llvm.org/D77773
---
 llvm/lib/Target/PowerPC/PPCISelLowering.cpp |  4 +--
 llvm/test/CodeGen/PowerPC/spe-fastmath.ll   | 31 +++++++++++++++++++++
 2 files changed, 33 insertions(+), 2 deletions(-)
 create mode 100644 llvm/test/CodeGen/PowerPC/spe-fastmath.ll

diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index 69afed8fb41a7..85f1630d8e223 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -7797,9 +7797,9 @@ SDValue PPCTargetLowering::LowerTRUNCATEVector(SDValue Op,
 /// LowerSELECT_CC - Lower floating point select_cc's into fsel instruction when
 /// possible.
 SDValue PPCTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
-  // Not FP? Not a fsel.
+  // Not FP, or using SPE? Not a fsel.
   if (!Op.getOperand(0).getValueType().isFloatingPoint() ||
-      !Op.getOperand(2).getValueType().isFloatingPoint())
+      !Op.getOperand(2).getValueType().isFloatingPoint() || Subtarget.hasSPE())
     return Op;
 
   ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get();
diff --git a/llvm/test/CodeGen/PowerPC/spe-fastmath.ll b/llvm/test/CodeGen/PowerPC/spe-fastmath.ll
new file mode 100644
index 0000000000000..d2b83f7ee1da2
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/spe-fastmath.ll
@@ -0,0 +1,31 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc-unknown-linux-gnu \
+; RUN:   -mattr=+spe |  FileCheck %s
+
+define void @no_fsel(i32 %e) #0 {
+; CHECK-LABEL: no_fsel:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li 4, .LCPI0_0@l
+; CHECK-NEXT:    lis 5, .LCPI0_0@ha
+; CHECK-NEXT:    evlddx 4, 5, 4
+; CHECK-NEXT:    efdcfui 3, 3
+; CHECK-NEXT:    efdmul 5, 3, 3
+; CHECK-NEXT:    efdcmpeq 0, 5, 4
+; CHECK-NEXT:    ble 0, .LBB0_2
+; CHECK-NEXT:  # %bb.1: # %entry
+; CHECK-NEXT:    evor 3, 4, 4
+; CHECK-NEXT:  .LBB0_2: # %entry
+; CHECK-NEXT:    efdctsiz 3, 3
+; CHECK-NEXT:    sth 3, 0(3)
+; CHECK-NEXT:    blr
+entry:
+  %conv = uitofp i32 %e to double
+  %mul = fmul double %conv, %conv
+  %tobool = fcmp une double %mul, 0.000000e+00
+  %cond = select i1 %tobool, double %conv, double 0.000000e+00
+  %conv3 = fptosi double %cond to i16
+  store i16 %conv3, i16* undef
+  ret void
+}
+
+attributes #0 = { "no-infs-fp-math"="true" "no-nans-fp-math"="true" }

From 1bd7046e4ce0102adef6096a12a289d7f94b8c73 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@intel.com>
Date: Fri, 31 Jul 2020 21:38:59 -0700
Subject: [PATCH 0803/1035] [X86] Use TargetLowering::getRegClassFor to
 simplify some code in tryVPTESTM. NFCI

---
 llvm/lib/Target/X86/X86ISelDAGToDAG.cpp | 16 ++--------------
 1 file changed, 2 insertions(+), 14 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
index 3b333496dd748..bb04690e04d16 100644
--- a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
@@ -4336,18 +4336,6 @@ bool X86DAGToDAGISel::tryVPTESTM(SDNode *Root, SDValue Setcc,
     }
   }
 
-  auto getMaskRC = [](MVT MaskVT) {
-    switch (MaskVT.SimpleTy) {
-    default: llvm_unreachable("Unexpected VT!");
-    case MVT::v2i1:  return X86::VK2RegClassID;
-    case MVT::v4i1:  return X86::VK4RegClassID;
-    case MVT::v8i1:  return X86::VK8RegClassID;
-    case MVT::v16i1: return X86::VK16RegClassID;
-    case MVT::v32i1: return X86::VK32RegClassID;
-    case MVT::v64i1: return X86::VK64RegClassID;
-    }
-  };
-
   bool IsMasked = InMask.getNode() != nullptr;
 
   SDLoc dl(Root);
@@ -4371,7 +4359,7 @@ bool X86DAGToDAGISel::tryVPTESTM(SDNode *Root, SDValue Setcc,
 
     if (IsMasked) {
       // Widen the mask.
-      unsigned RegClass = getMaskRC(MaskVT);
+      unsigned RegClass = TLI->getRegClassFor(MaskVT)->getID();
       SDValue RC = CurDAG->getTargetConstant(RegClass, dl, MVT::i32);
       InMask = SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
                                               dl, MaskVT, InMask, RC), 0);
@@ -4409,7 +4397,7 @@ bool X86DAGToDAGISel::tryVPTESTM(SDNode *Root, SDValue Setcc,
 
   // If we widened, we need to shrink the mask VT.
   if (Widen) {
-    unsigned RegClass = getMaskRC(ResVT);
+    unsigned RegClass = TLI->getRegClassFor(ResVT)->getID();
     SDValue RC = CurDAG->getTargetConstant(RegClass, dl, MVT::i32);
     CNode = CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
                                    dl, ResVT, SDValue(CNode, 0), RC);

From 8c5edf50234f52e0de37df3e4e7cec92bdb70e12 Mon Sep 17 00:00:00 2001
From: Chen Zheng <czhengsz@cn.ibm.com>
Date: Sat, 1 Aug 2020 01:28:15 -0400
Subject: [PATCH 0804/1035] [SCEV] don't query getSCEV() for incomplete phis

querying getSCEV() for incomplete phis leads to wrong cache value in `ExprToIVMap`,
because incomplete phis may be simplified to same value before get SCEV expression.

Reviewed By: lebedev.ri, mkazantsev

Differential Revision: https://reviews.llvm.org/D77560
---
 llvm/include/llvm/IR/Instructions.h           | 10 +++
 .../Utils/ScalarEvolutionExpander.cpp         | 10 +++
 .../Power/incomplete-phi.ll                   | 66 +++++++++++++++++++
 .../LoopStrengthReduce/Power/lit.local.cfg    |  2 +
 4 files changed, 88 insertions(+)
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/Power/incomplete-phi.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/Power/lit.local.cfg

diff --git a/llvm/include/llvm/IR/Instructions.h b/llvm/include/llvm/IR/Instructions.h
index 0afc585dfbe5a..63194fa93cbca 100644
--- a/llvm/include/llvm/IR/Instructions.h
+++ b/llvm/include/llvm/IR/Instructions.h
@@ -27,6 +27,7 @@
 #include "llvm/IR/Attributes.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/CallingConv.h"
+#include "llvm/IR/CFG.h"
 #include "llvm/IR/Constant.h"
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/Function.h"
@@ -2743,6 +2744,15 @@ class PHINode : public Instruction {
   /// non-undef value.
   bool hasConstantOrUndefValue() const;
 
+  /// If the PHI node is complete which means all of its parent's predecessors
+  /// have incoming value in this PHI, return true, otherwise return false.
+  bool isComplete() const {
+    return llvm::all_of(predecessors(getParent()),
+                        [this](const BasicBlock *Pred) {
+                          return getBasicBlockIndex(Pred) >= 0;
+                        });
+  }
+
   /// Methods for support type inquiry through isa, cast, and dyn_cast:
   static bool classof(const Instruction *I) {
     return I->getOpcode() == Instruction::PHI;
diff --git a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
index d31bf6791075a..555da5df65e15 100644
--- a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
+++ b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
@@ -1187,6 +1187,14 @@ SCEVExpander::getAddRecExprPHILiterally(const SCEVAddRecExpr *Normalized,
       if (!SE.isSCEVable(PN.getType()))
         continue;
 
+      // We should not look for a incomplete PHI. Getting SCEV for a incomplete
+      // PHI has no meaning at all.
+      if (!PN.isComplete()) {
+        DEBUG_WITH_TYPE(
+            DebugType, dbgs() << "One incomplete PHI is found: " << PN << "\n");
+        continue;
+      }
+
       const SCEVAddRecExpr *PhiSCEV = dyn_cast<SCEVAddRecExpr>(SE.getSCEV(&PN));
       if (!PhiSCEV)
         continue;
@@ -2102,6 +2110,8 @@ SCEVExpander::replaceCongruentIVs(Loop *L, const DominatorTree *DT,
     }
     DEBUG_WITH_TYPE(DebugType, dbgs() << "INDVARS: Eliminated congruent iv: "
                                       << *Phi << '\n');
+    DEBUG_WITH_TYPE(DebugType, dbgs() << "INDVARS: Original iv: "
+                                      << *OrigPhiRef << '\n');
     ++NumElim;
     Value *NewIV = OrigPhiRef;
     if (OrigPhiRef->getType() != Phi->getType()) {
diff --git a/llvm/test/Transforms/LoopStrengthReduce/Power/incomplete-phi.ll b/llvm/test/Transforms/LoopStrengthReduce/Power/incomplete-phi.ll
new file mode 100644
index 0000000000000..2abdc54bbad71
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/Power/incomplete-phi.ll
@@ -0,0 +1,66 @@
+; REQUIRES: asserts
+; RUN: opt -loop-reduce -debug-only=loop-reduce -S  < %s 2>&1 | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-n32:64"
+target triple = "powerpc64le-unknown-linux-gnu"
+;
+; %lsr.iv2 and %lsr.iv10 are in same bb, but they are not equal since start
+; value are different.
+;
+; %scevgep = getelementptr [0 x %0], [0 x %0]* %arg, i64 0, i64 99
+; %scevgep1 = bitcast %0* %scevgep to [0 x %0]*
+; %lsr.iv2 = phi [0 x %0]* [ %1, %bb18 ], [ %scevgep1, %bb ]
+;
+; %lsr.iv10 = phi [0 x %0]* [ %2, %bb18 ], [ %arg, %bb ]
+;
+; Make sure two incomplete phis will not be marked as congruent.
+;
+; CHECK: One incomplete PHI is found:   %[[IV:.*]] = phi [0 x %0]*
+; CHECK: One incomplete PHI is found:   %[[IV2:.*]] = phi [0 x %0]*
+; CHECK-NOT: Eliminated congruent iv:  %[[IV]]
+; CHECK-NOT: Original iv: %[[IV2]]
+; CHECK-NOT: Eliminated congruent iv:  %[[IV2]]
+; CHECK-NOT: Original iv: %[[IV]]
+
+%0 = type <{ float }>
+
+define void @foo([0 x %0]* %arg) {
+bb:
+  %i = getelementptr inbounds [0 x %0], [0 x %0]* %arg, i64 0, i64 -1
+  %i1 = bitcast %0* %i to i8*
+  %i2 = getelementptr i8, i8* %i1, i64 4
+  br label %bb3
+
+bb3:                                              ; preds = %bb18, %bb
+  %i4 = phi i64 [ %i20, %bb18 ], [ 0, %bb ]
+  %i5 = phi i64 [ %i21, %bb18 ], [ 1, %bb ]
+  br i1 undef, label %bb22, label %bb9
+
+bb9:                                              ; preds = %bb9, %bb3
+  %i10 = phi i64 [ 0, %bb3 ], [ %i16, %bb9 ]
+  %i11 = add i64 %i10, %i4
+  %i12 = shl i64 %i11, 2
+  %i13 = getelementptr i8, i8* %i2, i64 %i12
+  %i14 = bitcast i8* %i13 to float*
+  %i15 = bitcast float* %i14 to <4 x float>*
+  store <4 x float> undef, <4 x float>* %i15, align 4
+  %i16 = add i64 %i10, 32
+  br i1 true, label %bb17, label %bb9
+
+bb17:                                             ; preds = %bb9
+  br i1 undef, label %bb18, label %bb22
+
+bb18:                                             ; preds = %bb17
+  %i19 = add i64 undef, %i4
+  %i20 = add i64 %i19, %i5
+  %i21 = add nuw nsw i64 %i5, 1
+  br label %bb3
+
+bb22:                                             ; preds = %bb22, %bb17, %bb3
+  %i23 = phi i64 [ %i26, %bb22 ], [ undef, %bb17 ], [ 100, %bb3 ]
+  %i24 = add nsw i64 %i23, %i4
+  %i25 = getelementptr %0, %0* %i, i64 %i24, i32 0
+  store float undef, float* %i25, align 4
+  %i26 = add nuw nsw i64 %i23, 1
+  br label %bb22
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/Power/lit.local.cfg b/llvm/test/Transforms/LoopStrengthReduce/Power/lit.local.cfg
new file mode 100644
index 0000000000000..091332439b186
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/Power/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'PowerPC' in config.root.targets:
+    config.unsupported = True

From 75f134eec1ca30a4080e38d6d364ed932c0322ab Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@intel.com>
Date: Fri, 31 Jul 2020 23:10:47 -0700
Subject: [PATCH 0805/1035] [X86] Refactor the broadcast and load folding in
 tryVPTESTM to reduce some code.

Now we try to load and broadcast together for operand 1. Followed
by load and broadcast for operand 1. Previously we tried load
operand 1, load operand 1, broadcast operand 0, broadcast operand 1.

Now we have a single helper that tries load and broadcast for
one operand that we can just call twice.
---
 llvm/lib/Target/X86/X86ISelDAGToDAG.cpp | 104 +++++++++++-------------
 1 file changed, 46 insertions(+), 58 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
index bb04690e04d16..58424892535a7 100644
--- a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
@@ -4207,15 +4207,15 @@ VPTESTM_CASE(v16i16, WZ256##SUFFIX) \
 VPTESTM_CASE(v64i8, BZ##SUFFIX) \
 VPTESTM_CASE(v32i16, WZ##SUFFIX)
 
-  if (FoldedLoad) {
+  if (FoldedBCast) {
     switch (TestVT.SimpleTy) {
-    VPTESTM_FULL_CASES(rm)
+    VPTESTM_BROADCAST_CASES(rmb)
     }
   }
 
-  if (FoldedBCast) {
+  if (FoldedLoad) {
     switch (TestVT.SimpleTy) {
-    VPTESTM_BROADCAST_CASES(rmb)
+    VPTESTM_FULL_CASES(rm)
     }
   }
 
@@ -4274,68 +4274,57 @@ bool X86DAGToDAGISel::tryVPTESTM(SDNode *Root, SDValue Setcc,
     }
   }
 
-  // Without VLX we need to widen the load.
+  // Without VLX we need to widen the operation.
   bool Widen = !Subtarget->hasVLX() && !CmpVT.is512BitVector();
 
-  // We can only fold loads if the sources are unique.
-  bool CanFoldLoads = Src0 != Src1;
+  auto tryFoldLoadOrBCast = [&](SDNode *Root, SDNode *P, SDValue &L,
+                                SDValue &Base, SDValue &Scale, SDValue &Index,
+                                SDValue &Disp, SDValue &Segment) {
+    // If we need to widen, we can't fold the load.
+    if (!Widen)
+      if (tryFoldLoad(Root, P, L, Base, Scale, Index, Disp, Segment))
+        return true;
 
-  // Try to fold loads unless we need to widen.
-  bool FoldedLoad = false;
-  SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Load;
-  if (!Widen && CanFoldLoads) {
-    Load = Src1;
-    FoldedLoad = tryFoldLoad(Root, N0.getNode(), Load, Tmp0, Tmp1, Tmp2, Tmp3,
-                             Tmp4);
-    if (!FoldedLoad) {
-      // And is computative.
-      Load = Src0;
-      FoldedLoad = tryFoldLoad(Root, N0.getNode(), Load, Tmp0, Tmp1, Tmp2,
-                               Tmp3, Tmp4);
-      if (FoldedLoad)
-        std::swap(Src0, Src1);
-    }
-  }
+    // If we didn't fold a load, try to match broadcast. No widening limitation
+    // for this. But only 32 and 64 bit types are supported.
+    if (CmpSVT != MVT::i32 && CmpSVT != MVT::i64)
+      return false;
 
-  auto findBroadcastedOp = [](SDValue Src, MVT CmpSVT, SDNode *&Parent) {
     // Look through single use bitcasts.
-    if (Src.getOpcode() == ISD::BITCAST && Src.hasOneUse()) {
-      Parent = Src.getNode();
-      Src = Src.getOperand(0);
+    if (L.getOpcode() == ISD::BITCAST && L.hasOneUse()) {
+      P = L.getNode();
+      L = L.getOperand(0);
     }
 
-    if (Src.getOpcode() == X86ISD::VBROADCAST_LOAD && Src.hasOneUse()) {
-      auto *MemIntr = cast<MemIntrinsicSDNode>(Src);
-      if (MemIntr->getMemoryVT().getSizeInBits() == CmpSVT.getSizeInBits())
-        return Src;
-    }
+    if (L.getOpcode() != X86ISD::VBROADCAST_LOAD)
+      return false;
 
-    return SDValue();
+    auto *MemIntr = cast<MemIntrinsicSDNode>(L);
+    if (MemIntr->getMemoryVT().getSizeInBits() != CmpSVT.getSizeInBits())
+      return false;
+
+    return tryFoldBroadcast(Root, P, L, Base, Scale, Index, Disp, Segment);
   };
 
-  // If we didn't fold a load, try to match broadcast. No widening limitation
-  // for this. But only 32 and 64 bit types are supported.
-  bool FoldedBCast = false;
-  if (!FoldedLoad && CanFoldLoads &&
-      (CmpSVT == MVT::i32 || CmpSVT == MVT::i64)) {
-    SDNode *ParentNode = N0.getNode();
-    if ((Load = findBroadcastedOp(Src1, CmpSVT, ParentNode))) {
-      FoldedBCast = tryFoldBroadcast(Root, ParentNode, Load, Tmp0,
-                                     Tmp1, Tmp2, Tmp3, Tmp4);
-    }
+  // We can only fold loads if the sources are unique.
+  bool CanFoldLoads = Src0 != Src1;
 
-    // Try the other operand.
-    if (!FoldedBCast) {
-      SDNode *ParentNode = N0.getNode();
-      if ((Load = findBroadcastedOp(Src0, CmpSVT, ParentNode))) {
-        FoldedBCast = tryFoldBroadcast(Root, ParentNode, Load, Tmp0,
-                                       Tmp1, Tmp2, Tmp3, Tmp4);
-        if (FoldedBCast)
-          std::swap(Src0, Src1);
-      }
+  bool FoldedLoad = false;
+  SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
+  if (CanFoldLoads) {
+    FoldedLoad = tryFoldLoadOrBCast(Root, N0.getNode(), Src1, Tmp0, Tmp1, Tmp2,
+                                    Tmp3, Tmp4);
+    if (!FoldedLoad) {
+      // And is commutative.
+      FoldedLoad = tryFoldLoadOrBCast(Root, N0.getNode(), Src0, Tmp0, Tmp1,
+                                      Tmp2, Tmp3, Tmp4);
+      if (FoldedLoad)
+        std::swap(Src0, Src1);
     }
   }
 
+  bool FoldedBCast = FoldedLoad && Src1.getOpcode() == X86ISD::VBROADCAST_LOAD;
+
   bool IsMasked = InMask.getNode() != nullptr;
 
   SDLoc dl(Root);
@@ -4353,7 +4342,6 @@ bool X86DAGToDAGISel::tryVPTESTM(SDNode *Root, SDValue Setcc,
                                                      CmpVT), 0);
     Src0 = CurDAG->getTargetInsertSubreg(SubReg, dl, CmpVT, ImplDef, Src0);
 
-    assert(!FoldedLoad && "Shouldn't have folded the load");
     if (!FoldedBCast)
       Src1 = CurDAG->getTargetInsertSubreg(SubReg, dl, CmpVT, ImplDef, Src1);
 
@@ -4371,23 +4359,23 @@ bool X86DAGToDAGISel::tryVPTESTM(SDNode *Root, SDValue Setcc,
                                IsMasked);
 
   MachineSDNode *CNode;
-  if (FoldedLoad || FoldedBCast) {
+  if (FoldedLoad) {
     SDVTList VTs = CurDAG->getVTList(MaskVT, MVT::Other);
 
     if (IsMasked) {
       SDValue Ops[] = { InMask, Src0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4,
-                        Load.getOperand(0) };
+                        Src1.getOperand(0) };
       CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops);
     } else {
       SDValue Ops[] = { Src0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4,
-                        Load.getOperand(0) };
+                        Src1.getOperand(0) };
       CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops);
     }
 
     // Update the chain.
-    ReplaceUses(Load.getValue(1), SDValue(CNode, 1));
+    ReplaceUses(Src1.getValue(1), SDValue(CNode, 1));
     // Record the mem-refs
-    CurDAG->setNodeMemRefs(CNode, {cast<MemSDNode>(Load)->getMemOperand()});
+    CurDAG->setNodeMemRefs(CNode, {cast<MemSDNode>(Src1)->getMemOperand()});
   } else {
     if (IsMasked)
       CNode = CurDAG->getMachineNode(Opc, dl, MaskVT, InMask, Src0, Src1);

From 4888c9ce97d8c20d988212b10f1045e3c4022b8e Mon Sep 17 00:00:00 2001
From: Nathan James <n.james93@hotmail.co.uk>
Date: Sat, 1 Aug 2020 10:35:13 +0100
Subject: [PATCH 0806/1035] [clang-tidy] readability-identifier-naming checks
 configs for included files

When checking for the style of a decl that isn't in the main file, the check will now search for the configuration that the included files uses to gather the style for its decls.

This can be useful to silence warnings in header files that follow a different naming convention without using header-filter to silence all warnings(even from other checks) in the header file.

Reviewed By: aaron.ballman, gribozavr2

Differential Revision: https://reviews.llvm.org/D84814
---
 .../readability/IdentifierNamingCheck.cpp     | 147 ++++++++++--------
 .../readability/IdentifierNamingCheck.h       |  13 +-
 clang-tools-extra/docs/ReleaseNotes.rst       |   9 +-
 .../checks/readability-identifier-naming.rst  |   8 +
 .../global-style-disabled/.clang-tidy         |   5 +
 .../global-style-disabled/header.h            |   3 +
 .../global-style1/.clang-tidy                 |   5 +
 .../global-style1/header.h                    |   5 +
 .../global-style2/.clang-tidy                 |   5 +
 .../global-style2/header.h                    |   5 +
 ...lity-identifier-naming-multiple-styles.cpp |  64 ++++++++
 11 files changed, 201 insertions(+), 68 deletions(-)
 create mode 100644 clang-tools-extra/test/clang-tidy/checkers/Inputs/readability-identifier-naming/global-style-disabled/.clang-tidy
 create mode 100644 clang-tools-extra/test/clang-tidy/checkers/Inputs/readability-identifier-naming/global-style-disabled/header.h
 create mode 100644 clang-tools-extra/test/clang-tidy/checkers/Inputs/readability-identifier-naming/global-style1/.clang-tidy
 create mode 100644 clang-tools-extra/test/clang-tidy/checkers/Inputs/readability-identifier-naming/global-style1/header.h
 create mode 100644 clang-tools-extra/test/clang-tidy/checkers/Inputs/readability-identifier-naming/global-style2/.clang-tidy
 create mode 100644 clang-tools-extra/test/clang-tidy/checkers/Inputs/readability-identifier-naming/global-style2/header.h
 create mode 100644 clang-tools-extra/test/clang-tidy/checkers/readability-identifier-naming-multiple-styles.cpp

diff --git a/clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.cpp b/clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.cpp
index c885aac89072a..e004ce6fbd208 100644
--- a/clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.cpp
@@ -8,6 +8,7 @@
 
 #include "IdentifierNamingCheck.h"
 
+#include "../GlobList.h"
 #include "clang/AST/CXXInheritance.h"
 #include "clang/Lex/PPCallbacks.h"
 #include "clang/Lex/Preprocessor.h"
@@ -15,7 +16,8 @@
 #include "llvm/ADT/DenseMapInfo.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/Error.h"
-#include "llvm/Support/Format.h"
+#include "llvm/Support/FormatVariadic.h"
+#include "llvm/Support/Path.h"
 #include "llvm/Support/Regex.h"
 
 #define DEBUG_TYPE "clang-tidy"
@@ -119,41 +121,47 @@ static StringRef const StyleNames[] = {
 #undef NAMING_KEYS
 // clang-format on
 
+static std::vector<llvm::Optional<IdentifierNamingCheck::NamingStyle>>
+getNamingStyles(const ClangTidyCheck::OptionsView &Options) {
+  std::vector<llvm::Optional<IdentifierNamingCheck::NamingStyle>> Styles;
+  Styles.reserve(StyleNames->size());
+  for (auto const &StyleName : StyleNames) {
+    auto CaseOptional = Options.getOptional<IdentifierNamingCheck::CaseType>(
+        (StyleName + "Case").str());
+    auto Prefix = Options.get((StyleName + "Prefix").str(), "");
+    auto Postfix = Options.get((StyleName + "Suffix").str(), "");
+
+    if (CaseOptional || !Prefix.empty() || !Postfix.empty())
+      Styles.emplace_back(IdentifierNamingCheck::NamingStyle{
+          std::move(CaseOptional), std::move(Prefix), std::move(Postfix)});
+    else
+      Styles.emplace_back(llvm::None);
+  }
+  return Styles;
+}
+
 IdentifierNamingCheck::IdentifierNamingCheck(StringRef Name,
                                              ClangTidyContext *Context)
-    : RenamerClangTidyCheck(Name, Context),
+    : RenamerClangTidyCheck(Name, Context), Context(Context), CheckName(Name),
+      GetConfigPerFile(Options.get("GetConfigPerFile", true)),
       IgnoreFailedSplit(Options.get("IgnoreFailedSplit", false)),
       IgnoreMainLikeFunctions(Options.get("IgnoreMainLikeFunctions", false)) {
 
-  for (auto const &Name : StyleNames) {
-    auto CaseOptional = [&]() -> llvm::Optional<CaseType> {
-      auto ValueOr = Options.get<CaseType>((Name + "Case").str());
-      if (ValueOr)
-        return *ValueOr;
-      llvm::logAllUnhandledErrors(
-          llvm::handleErrors(ValueOr.takeError(),
-                             [](const MissingOptionError &) -> llvm::Error {
-                               return llvm::Error::success();
-                             }),
-          llvm::errs(), "warning: ");
-      return llvm::None;
-    }();
-
-    auto prefix = Options.get((Name + "Prefix").str(), "");
-    auto postfix = Options.get((Name + "Suffix").str(), "");
-
-    if (CaseOptional || !prefix.empty() || !postfix.empty()) {
-      NamingStyles.push_back(NamingStyle(CaseOptional, prefix, postfix));
-    } else {
-      NamingStyles.push_back(llvm::None);
-    }
-  }
+  auto IterAndInserted = NamingStylesCache.try_emplace(
+      llvm::sys::path::parent_path(Context->getCurrentFile()),
+      getNamingStyles(Options));
+  assert(IterAndInserted.second && "Couldn't insert Style");
+  // Holding a reference to the data in the vector is safe as it should never
+  // move.
+  MainFileStyle = IterAndInserted.first->getValue();
 }
 
 IdentifierNamingCheck::~IdentifierNamingCheck() = default;
 
 void IdentifierNamingCheck::storeOptions(ClangTidyOptions::OptionMap &Opts) {
   RenamerClangTidyCheck::storeOptions(Opts);
+  ArrayRef<llvm::Optional<NamingStyle>> NamingStyles =
+      getStyleForFile(Context->getCurrentFile());
   for (size_t i = 0; i < SK_Count; ++i) {
     if (NamingStyles[i]) {
       if (NamingStyles[i]->Case) {
@@ -166,7 +174,7 @@ void IdentifierNamingCheck::storeOptions(ClangTidyOptions::OptionMap &Opts) {
                     NamingStyles[i]->Suffix);
     }
   }
-
+  Options.store(Opts, "GetConfigPerFile", GetConfigPerFile);
   Options.store(Opts, "IgnoreFailedSplit", IgnoreFailedSplit);
   Options.store(Opts, "IgnoreMainLikeFunctions", IgnoreMainLikeFunctions);
 }
@@ -374,8 +382,7 @@ fixupWithStyle(StringRef Name,
 
 static StyleKind findStyleKind(
     const NamedDecl *D,
-    const std::vector<llvm::Optional<IdentifierNamingCheck::NamingStyle>>
-        &NamingStyles,
+    ArrayRef<llvm::Optional<IdentifierNamingCheck::NamingStyle>> NamingStyles,
     bool IgnoreMainLikeFunctions) {
   assert(D && D->getIdentifier() && !D->getName().empty() && !D->isImplicit() &&
          "Decl must be an explicit identifier with a name.");
@@ -652,63 +659,56 @@ static StyleKind findStyleKind(
   return SK_Invalid;
 }
 
-llvm::Optional<RenamerClangTidyCheck::FailureInfo>
-IdentifierNamingCheck::GetDeclFailureInfo(const NamedDecl *Decl,
-                                          const SourceManager &SM) const {
-  StyleKind SK = findStyleKind(Decl, NamingStyles, IgnoreMainLikeFunctions);
-  if (SK == SK_Invalid)
+static llvm::Optional<RenamerClangTidyCheck::FailureInfo> getFailureInfo(
+    StringRef Name, SourceLocation Location,
+    ArrayRef<llvm::Optional<IdentifierNamingCheck::NamingStyle>> NamingStyles,
+    StyleKind SK, const SourceManager &SM, bool IgnoreFailedSplit) {
+  if (SK == SK_Invalid || !NamingStyles[SK])
     return None;
 
-  if (!NamingStyles[SK])
-    return None;
-
-  const NamingStyle &Style = *NamingStyles[SK];
-  StringRef Name = Decl->getName();
+  const IdentifierNamingCheck::NamingStyle &Style = *NamingStyles[SK];
   if (matchesStyle(Name, Style))
     return None;
 
-  std::string KindName = fixupWithCase(StyleNames[SK], CT_LowerCase);
+  std::string KindName =
+      fixupWithCase(StyleNames[SK], IdentifierNamingCheck::CT_LowerCase);
   std::replace(KindName.begin(), KindName.end(), '_', ' ');
 
   std::string Fixup = fixupWithStyle(Name, Style);
   if (StringRef(Fixup).equals(Name)) {
     if (!IgnoreFailedSplit) {
-      LLVM_DEBUG(llvm::dbgs()
-                 << Decl->getBeginLoc().printToString(SM)
-                 << llvm::format(": unable to split words for %s '%s'\n",
-                                 KindName.c_str(), Name.str().c_str()));
+      LLVM_DEBUG(Location.print(llvm::dbgs(), SM);
+                 llvm::dbgs()
+                 << llvm::formatv(": unable to split words for {0} '{1}'\n",
+                                  KindName, Name));
     }
     return None;
   }
-  return FailureInfo{std::move(KindName), std::move(Fixup)};
+  return RenamerClangTidyCheck::FailureInfo{std::move(KindName),
+                                            std::move(Fixup)};
+}
+
+llvm::Optional<RenamerClangTidyCheck::FailureInfo>
+IdentifierNamingCheck::GetDeclFailureInfo(const NamedDecl *Decl,
+                                          const SourceManager &SM) const {
+  SourceLocation Loc = Decl->getLocation();
+  ArrayRef<llvm::Optional<NamingStyle>> NamingStyles =
+      getStyleForFile(SM.getFilename(Loc));
+
+  return getFailureInfo(
+      Decl->getName(), Loc, NamingStyles,
+      findStyleKind(Decl, NamingStyles, IgnoreMainLikeFunctions), SM,
+      IgnoreFailedSplit);
 }
 
 llvm::Optional<RenamerClangTidyCheck::FailureInfo>
 IdentifierNamingCheck::GetMacroFailureInfo(const Token &MacroNameTok,
                                            const SourceManager &SM) const {
-  if (!NamingStyles[SK_MacroDefinition])
-    return None;
-
-  StringRef Name = MacroNameTok.getIdentifierInfo()->getName();
-  const NamingStyle &Style = *NamingStyles[SK_MacroDefinition];
-  if (matchesStyle(Name, Style))
-    return None;
+  SourceLocation Loc = MacroNameTok.getLocation();
 
-  std::string KindName =
-      fixupWithCase(StyleNames[SK_MacroDefinition], CT_LowerCase);
-  std::replace(KindName.begin(), KindName.end(), '_', ' ');
-
-  std::string Fixup = fixupWithStyle(Name, Style);
-  if (StringRef(Fixup).equals(Name)) {
-    if (!IgnoreFailedSplit) {
-      LLVM_DEBUG(llvm::dbgs()
-                 << MacroNameTok.getLocation().printToString(SM)
-                 << llvm::format(": unable to split words for %s '%s'\n",
-                                 KindName.c_str(), Name.str().c_str()));
-    }
-    return None;
-  }
-  return FailureInfo{std::move(KindName), std::move(Fixup)};
+  return getFailureInfo(MacroNameTok.getIdentifierInfo()->getName(), Loc,
+                        getStyleForFile(SM.getFilename(Loc)),
+                        SK_MacroDefinition, SM, IgnoreFailedSplit);
 }
 
 RenamerClangTidyCheck::DiagInfo
@@ -720,6 +720,21 @@ IdentifierNamingCheck::GetDiagInfo(const NamingCheckId &ID,
                   }};
 }
 
+ArrayRef<llvm::Optional<IdentifierNamingCheck::NamingStyle>>
+IdentifierNamingCheck::getStyleForFile(StringRef FileName) const {
+  if (!GetConfigPerFile)
+    return MainFileStyle;
+  auto &Styles = NamingStylesCache[llvm::sys::path::parent_path(FileName)];
+  if (Styles.empty()) {
+    ClangTidyOptions Options = Context->getOptionsForFile(FileName);
+    if (Options.Checks && GlobList(*Options.Checks).contains(CheckName))
+      Styles = getNamingStyles({CheckName, Options.CheckOptions});
+    else
+      Styles.resize(SK_Count, None);
+  }
+  return Styles;
+}
+
 } // namespace readability
 } // namespace tidy
 } // namespace clang
diff --git a/clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.h b/clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.h
index 0f6c77b2c9a86..ad1c582d100bc 100644
--- a/clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.h
+++ b/clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.h
@@ -10,6 +10,7 @@
 #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_IDENTIFIERNAMINGCHECK_H
 
 #include "../utils/RenamerClangTidyCheck.h"
+#include "llvm/ADT/Optional.h"
 namespace clang {
 
 class MacroInfo;
@@ -69,7 +70,17 @@ class IdentifierNamingCheck final : public RenamerClangTidyCheck {
   DiagInfo GetDiagInfo(const NamingCheckId &ID,
                        const NamingCheckFailure &Failure) const override;
 
-  std::vector<llvm::Optional<NamingStyle>> NamingStyles;
+  ArrayRef<llvm::Optional<NamingStyle>>
+  getStyleForFile(StringRef FileName) const;
+
+  /// Stores the style options as a vector, indexed by the specified \ref
+  /// StyleKind, for a given directory.
+  mutable llvm::StringMap<std::vector<llvm::Optional<NamingStyle>>>
+      NamingStylesCache;
+  ArrayRef<llvm::Optional<NamingStyle>> MainFileStyle;
+  ClangTidyContext *const Context;
+  const std::string CheckName;
+  const bool GetConfigPerFile;
   const bool IgnoreFailedSplit;
   const bool IgnoreMainLikeFunctions;
 };
diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst
index 1d447938eae0c..89f6a50888ccc 100644
--- a/clang-tools-extra/docs/ReleaseNotes.rst
+++ b/clang-tools-extra/docs/ReleaseNotes.rst
@@ -67,7 +67,14 @@ The improvements are...
 Improvements to clang-tidy
 --------------------------
 
-The improvements are...
+Changes in existing checks
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+- Improved :doc:`readability-identifier-naming
+  <clang-tidy/checks/readability-identifier-naming>` check.
+
+  Added an option `GetConfigPerFile` to support including files which use
+  different naming styles.
 
 Improvements to include-fixer
 -----------------------------
diff --git a/clang-tools-extra/docs/clang-tidy/checks/readability-identifier-naming.rst b/clang-tools-extra/docs/clang-tidy/checks/readability-identifier-naming.rst
index eefa5234fb211..9eec3c03f7d7d 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/readability-identifier-naming.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/readability-identifier-naming.rst
@@ -51,6 +51,7 @@ The following options are describe below:
  - :option:`EnumCase`, :option:`EnumPrefix`, :option:`EnumSuffix`
  - :option:`EnumConstantCase`, :option:`EnumConstantPrefix`, :option:`EnumConstantSuffix`
  - :option:`FunctionCase`, :option:`FunctionPrefix`, :option:`FunctionSuffix`
+ - :option:`GetConfigPerFile`
  - :option:`GlobalConstantCase`, :option:`GlobalConstantPrefix`, :option:`GlobalConstantSuffix`
  - :option:`GlobalConstantPointerCase`, :option:`GlobalConstantPointerPrefix`, :option:`GlobalConstantPointerSuffix`
  - :option:`GlobalFunctionCase`, :option:`GlobalFunctionPrefix`, :option:`GlobalFunctionSuffix`
@@ -713,6 +714,13 @@ After:
 
     char pre_my_function_string_post();
 
+.. option:: GetConfigPerFile
+
+    When `true` the check will look for the configuration for where an
+    identifier is declared. Useful for when included header files use a 
+    different style. 
+    Default value is `true`.
+
 .. option:: GlobalConstantCase
 
     When defined, the check will ensure global constant names conform to the
diff --git a/clang-tools-extra/test/clang-tidy/checkers/Inputs/readability-identifier-naming/global-style-disabled/.clang-tidy b/clang-tools-extra/test/clang-tidy/checkers/Inputs/readability-identifier-naming/global-style-disabled/.clang-tidy
new file mode 100644
index 0000000000000..6a704df8b7b19
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/Inputs/readability-identifier-naming/global-style-disabled/.clang-tidy
@@ -0,0 +1,5 @@
+Checks: -readability-identifier-naming
+CheckOptions:
+  - key:             readability-identifier-naming.GlobalFunctionCase
+    value:           lower_case
+
diff --git a/clang-tools-extra/test/clang-tidy/checkers/Inputs/readability-identifier-naming/global-style-disabled/header.h b/clang-tools-extra/test/clang-tidy/checkers/Inputs/readability-identifier-naming/global-style-disabled/header.h
new file mode 100644
index 0000000000000..e863f70f7fcb2
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/Inputs/readability-identifier-naming/global-style-disabled/header.h
@@ -0,0 +1,3 @@
+void disabled_style_1();
+void disabledStyle2();
+void DISABLED_STYLE_3();
diff --git a/clang-tools-extra/test/clang-tidy/checkers/Inputs/readability-identifier-naming/global-style1/.clang-tidy b/clang-tools-extra/test/clang-tidy/checkers/Inputs/readability-identifier-naming/global-style1/.clang-tidy
new file mode 100644
index 0000000000000..85af9672b61d3
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/Inputs/readability-identifier-naming/global-style1/.clang-tidy
@@ -0,0 +1,5 @@
+Checks: readability-identifier-naming
+CheckOptions:
+  - key:             readability-identifier-naming.GlobalFunctionCase
+    value:           lower_case
+
diff --git a/clang-tools-extra/test/clang-tidy/checkers/Inputs/readability-identifier-naming/global-style1/header.h b/clang-tools-extra/test/clang-tidy/checkers/Inputs/readability-identifier-naming/global-style1/header.h
new file mode 100644
index 0000000000000..b170bed7c3f60
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/Inputs/readability-identifier-naming/global-style1/header.h
@@ -0,0 +1,5 @@
+
+
+void style_first_good();
+
+void styleFirstBad();
diff --git a/clang-tools-extra/test/clang-tidy/checkers/Inputs/readability-identifier-naming/global-style2/.clang-tidy b/clang-tools-extra/test/clang-tidy/checkers/Inputs/readability-identifier-naming/global-style2/.clang-tidy
new file mode 100644
index 0000000000000..b2e67ea9c87b5
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/Inputs/readability-identifier-naming/global-style2/.clang-tidy
@@ -0,0 +1,5 @@
+Checks: readability-identifier-naming
+CheckOptions:
+  - key:             readability-identifier-naming.GlobalFunctionCase
+    value:           UPPER_CASE
+
diff --git a/clang-tools-extra/test/clang-tidy/checkers/Inputs/readability-identifier-naming/global-style2/header.h b/clang-tools-extra/test/clang-tidy/checkers/Inputs/readability-identifier-naming/global-style2/header.h
new file mode 100644
index 0000000000000..6b78ad82a1fdd
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/Inputs/readability-identifier-naming/global-style2/header.h
@@ -0,0 +1,5 @@
+
+
+void STYLE_SECOND_GOOD();
+
+void styleSecondBad();
diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability-identifier-naming-multiple-styles.cpp b/clang-tools-extra/test/clang-tidy/checkers/readability-identifier-naming-multiple-styles.cpp
new file mode 100644
index 0000000000000..54880d2ca3d0a
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/readability-identifier-naming-multiple-styles.cpp
@@ -0,0 +1,64 @@
+// Setup header directory
+
+// RUN: rm -rf %theaders
+// RUN: mkdir %theaders
+// RUN: cp -R %S/Inputs/readability-identifier-naming/. %theaders
+
+// C++11 isn't explicitly required, but failing to specify a standard means the
+// check will run multiple times for different standards. This will cause the
+// second test to fail as the header file will be changed during the first run.
+// InheritParentConfig is needed to look for the clang-tidy configuration files.
+
+// RUN: %check_clang_tidy -check-suffixes=ENABLED,SHARED -std=c++11 %s \
+// RUN: readability-identifier-naming %t -- \
+// RUN:  -config='{ InheritParentConfig: true, CheckOptions: [ \
+// RUN:   {key: readability-identifier-naming.FunctionCase, value: camelBack}, \
+// RUN:   {key: readability-identifier-naming.GetConfigPerFile, value: true} \
+// RUN:  ]}' -header-filter='.*' -- -I%theaders
+
+// On DISABLED run, everything should be made 'camelBack'.
+
+// RUN: cp -R %S/Inputs/readability-identifier-naming/. %theaders
+// RUN: %check_clang_tidy -check-suffixes=DISABLED,SHARED -std=c++11 %s \
+// RUN: readability-identifier-naming %t -- \
+// RUN:  -config='{ InheritParentConfig: true, CheckOptions: [ \
+// RUN:   {key: readability-identifier-naming.FunctionCase, value: camelBack}, \
+// RUN:   {key: readability-identifier-naming.GetConfigPerFile, value: false} \
+// RUN:  ]}' -header-filter='.*' -- -I%theaders
+
+#include "global-style-disabled/header.h"
+#include "global-style1/header.h"
+#include "global-style2/header.h"
+// CHECK-MESSAGES-ENABLED-DAG: global-style1/header.h:5:6: warning: invalid case style for global function 'styleFirstBad'
+// CHECK-MESSAGES-ENABLED-DAG: global-style2/header.h:5:6: warning: invalid case style for global function 'styleSecondBad'
+// CHECK-MESSAGES-DISABLED-DAG: global-style1/header.h:3:6: warning: invalid case style for function 'style_first_good'
+// CHECK-MESSAGES-DISABLED-DAG: global-style2/header.h:3:6: warning: invalid case style for function 'STYLE_SECOND_GOOD'
+// CHECK-MESSAGES-DISABLED-DAG: global-style-disabled/header.h:1:6: warning: invalid case style for function 'disabled_style_1'
+// CHECK-MESSAGES-DISABLED-DAG: global-style-disabled/header.h:3:6: warning: invalid case style for function 'DISABLED_STYLE_3'
+
+void goodStyle() {
+  style_first_good();
+  STYLE_SECOND_GOOD();
+  //      CHECK-FIXES-DISABLED: styleFirstGood();
+  // CHECK-FIXES-DISABLED-NEXT: styleSecondGood();
+}
+// CHECK-MESSAGES-SHARED-DAG: :[[@LINE+1]]:6: warning: invalid case style for function 'bad_style'
+void bad_style() {
+  styleFirstBad();
+  styleSecondBad();
+}
+//        CHECK-FIXES-SHARED: void badStyle() {
+// CHECK-FIXES-DISABLED-NEXT:   styleFirstBad();
+//  CHECK-FIXES-ENABLED-NEXT:   style_first_bad();
+// CHECK-FIXES-DISABLED-NEXT:   styleSecondBad();
+//  CHECK-FIXES-ENABLED-NEXT:   STYLE_SECOND_BAD();
+//   CHECK-FIXES-SHARED-NEXT: }
+
+void expectNoStyle() {
+  disabled_style_1();
+  disabledStyle2();
+  DISABLED_STYLE_3();
+  //      CHECK-FIXES-DISABLED: disabledStyle1();
+  // CHECK-FIXES-DISABLED-NEXT: disabledStyle2();
+  // CHECK-FIXES-DISABLED-NEXT: disabledStyle3();
+}

From e73f5d86f179644b8d66d4141d8d359cd6f0435b Mon Sep 17 00:00:00 2001
From: Evgeny Leviant <eleviant@accesssoftek.com>
Date: Sat, 1 Aug 2020 12:58:52 +0300
Subject: [PATCH 0807/1035] [MachineVerifier] Refactor calcRegsPassed. NFC

Patch improves performance of verify-machineinstrs pass up to 10x.
Differential revision: https://reviews.llvm.org/D84105
---
 llvm/lib/CodeGen/MachineVerifier.cpp | 69 +++++++---------------------
 1 file changed, 17 insertions(+), 52 deletions(-)

diff --git a/llvm/lib/CodeGen/MachineVerifier.cpp b/llvm/lib/CodeGen/MachineVerifier.cpp
index e45e6963edb11..238df43da9f5c 100644
--- a/llvm/lib/CodeGen/MachineVerifier.cpp
+++ b/llvm/lib/CodeGen/MachineVerifier.cpp
@@ -2230,63 +2230,28 @@ class FilteringVRegSet {
 // can pass through an MBB live, but may not be live every time. It is assumed
 // that all vregsPassed sets are empty before the call.
 void MachineVerifier::calcRegsPassed() {
-  // This is a forward dataflow, doing it in RPO. A standard map serves as a
-  // priority (sorting by RPO number) queue, deduplicating worklist, and an RPO
-  // number to MBB mapping all at once.
-  std::map<unsigned, const MachineBasicBlock *> RPOWorklist;
-  DenseMap<const MachineBasicBlock *, unsigned> RPONumbers;
-  if (MF->empty()) {
+  if (MF->empty())
     // ReversePostOrderTraversal doesn't handle empty functions.
     return;
-  }
-  std::vector<FilteringVRegSet> VRegsPassedSets(MF->size());
-  for (const MachineBasicBlock *MBB :
-       ReversePostOrderTraversal<const MachineFunction *>(MF)) {
-    // Careful with the evaluation order, fetch next number before allocating.
-    unsigned Number = RPONumbers.size();
-    RPONumbers[MBB] = Number;
-    // Set-up the transfer functions for all blocks.
-    const BBInfo &MInfo = MBBInfoMap[MBB];
-    VRegsPassedSets[Number].addToFilter(MInfo.regsKilled);
-    VRegsPassedSets[Number].addToFilter(MInfo.regsLiveOut);
-  }
-  // First push live-out regs to successors' vregsPassed. Remember the MBBs that
-  // have any vregsPassed.
-  for (const MachineBasicBlock &MBB : *MF) {
-    const BBInfo &MInfo = MBBInfoMap[&MBB];
-    if (!MInfo.reachable)
-      continue;
-    for (const MachineBasicBlock *Succ : MBB.successors()) {
-      unsigned SuccNumber = RPONumbers[Succ];
-      FilteringVRegSet &SuccSet = VRegsPassedSets[SuccNumber];
-      if (SuccSet.add(MInfo.regsLiveOut))
-        RPOWorklist.emplace(SuccNumber, Succ);
-    }
-  }
 
-  // Iteratively push vregsPassed to successors.
-  while (!RPOWorklist.empty()) {
-    auto Next = RPOWorklist.begin();
-    const MachineBasicBlock *MBB = Next->second;
-    RPOWorklist.erase(Next);
-    FilteringVRegSet &MSet = VRegsPassedSets[RPONumbers[MBB]];
-    for (const MachineBasicBlock *Succ : MBB->successors()) {
-      if (Succ == MBB)
+  for (const MachineBasicBlock *MB :
+       ReversePostOrderTraversal<const MachineFunction *>(MF)) {
+    FilteringVRegSet VRegs;
+    BBInfo &Info = MBBInfoMap[MB];
+    assert(Info.reachable);
+
+    VRegs.addToFilter(Info.regsKilled);
+    VRegs.addToFilter(Info.regsLiveOut);
+    for (const MachineBasicBlock *Pred : MB->predecessors()) {
+      const BBInfo &PredInfo = MBBInfoMap[Pred];
+      if (!PredInfo.reachable)
         continue;
-      unsigned SuccNumber = RPONumbers[Succ];
-      FilteringVRegSet &SuccSet = VRegsPassedSets[SuccNumber];
-      if (SuccSet.add(MSet))
-        RPOWorklist.emplace(SuccNumber, Succ);
+
+      VRegs.add(PredInfo.regsLiveOut);
+      VRegs.add(PredInfo.vregsPassed);
     }
-  }
-  // Copy the results back to BBInfos.
-  for (const MachineBasicBlock &MBB : *MF) {
-    BBInfo &MInfo = MBBInfoMap[&MBB];
-    if (!MInfo.reachable)
-      continue;
-    const FilteringVRegSet &MSet = VRegsPassedSets[RPONumbers[&MBB]];
-    MInfo.vregsPassed.reserve(MSet.size());
-    MInfo.vregsPassed.insert(MSet.begin(), MSet.end());
+    Info.vregsPassed.reserve(VRegs.size());
+    Info.vregsPassed.insert(VRegs.begin(), VRegs.end());
   }
 }
 

From 9f21947a331203ee2579db87f1d1ec22a949e20a Mon Sep 17 00:00:00 2001
From: Nathan James <n.james93@hotmail.co.uk>
Date: Sat, 1 Aug 2020 11:04:29 +0100
Subject: [PATCH 0808/1035] [clang-tidy][NFC] Small refactor

---
 .../readability/IdentifierNamingCheck.cpp     | 55 +++++++++----------
 1 file changed, 25 insertions(+), 30 deletions(-)

diff --git a/clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.cpp b/clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.cpp
index e004ce6fbd208..e7fe25d8e2214 100644
--- a/clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.cpp
@@ -162,17 +162,16 @@ void IdentifierNamingCheck::storeOptions(ClangTidyOptions::OptionMap &Opts) {
   RenamerClangTidyCheck::storeOptions(Opts);
   ArrayRef<llvm::Optional<NamingStyle>> NamingStyles =
       getStyleForFile(Context->getCurrentFile());
-  for (size_t i = 0; i < SK_Count; ++i) {
-    if (NamingStyles[i]) {
-      if (NamingStyles[i]->Case) {
-        Options.store(Opts, (StyleNames[i] + "Case").str(),
-                      *NamingStyles[i]->Case);
-      }
-      Options.store(Opts, (StyleNames[i] + "Prefix").str(),
-                    NamingStyles[i]->Prefix);
-      Options.store(Opts, (StyleNames[i] + "Suffix").str(),
-                    NamingStyles[i]->Suffix);
-    }
+  for (size_t I = 0; I < SK_Count; ++I) {
+    if (!NamingStyles[I])
+      continue;
+    if (NamingStyles[I]->Case)
+      Options.store(Opts, (StyleNames[I] + "Case").str(),
+                    *NamingStyles[I]->Case);
+    Options.store(Opts, (StyleNames[I] + "Prefix").str(),
+                  NamingStyles[I]->Prefix);
+    Options.store(Opts, (StyleNames[I] + "Suffix").str(),
+                  NamingStyles[I]->Suffix);
   }
   Options.store(Opts, "GetConfigPerFile", GetConfigPerFile);
   Options.store(Opts, "IgnoreFailedSplit", IgnoreFailedSplit);
@@ -191,14 +190,9 @@ static bool matchesStyle(StringRef Name,
       llvm::Regex("^[a-z]([a-z0-9]*(_[A-Z])?)*"),
   };
 
-  if (Name.startswith(Style.Prefix))
-    Name = Name.drop_front(Style.Prefix.size());
-  else
+  if (!Name.consume_front(Style.Prefix))
     return false;
-
-  if (Name.endswith(Style.Suffix))
-    Name = Name.drop_back(Style.Suffix.size());
-  else
+  if (!Name.consume_back(Style.Suffix))
     return false;
 
   // Ensure the name doesn't have any extra underscores beyond those specified
@@ -221,9 +215,10 @@ static std::string fixupWithCase(StringRef Name,
   Name.split(Substrs, "_", -1, false);
 
   SmallVector<StringRef, 8> Words;
+  SmallVector<StringRef, 8> Groups;
   for (auto Substr : Substrs) {
     while (!Substr.empty()) {
-      SmallVector<StringRef, 8> Groups;
+      Groups.clear();
       if (!Splitter.match(Substr, &Groups))
         break;
 
@@ -241,12 +236,12 @@ static std::string fixupWithCase(StringRef Name,
   }
 
   if (Words.empty())
-    return std::string(Name);
+    return Name.str();
 
-  std::string Fixup;
+  SmallString<128> Fixup;
   switch (Case) {
   case IdentifierNamingCheck::CT_AnyCase:
-    Fixup += Name;
+    return Name.str();
     break;
 
   case IdentifierNamingCheck::CT_LowerCase:
@@ -267,7 +262,7 @@ static std::string fixupWithCase(StringRef Name,
 
   case IdentifierNamingCheck::CT_CamelCase:
     for (auto const &Word : Words) {
-      Fixup += Word.substr(0, 1).upper();
+      Fixup += toupper(Word.front());
       Fixup += Word.substr(1).lower();
     }
     break;
@@ -277,7 +272,7 @@ static std::string fixupWithCase(StringRef Name,
       if (&Word == &Words.front()) {
         Fixup += Word.lower();
       } else {
-        Fixup += Word.substr(0, 1).upper();
+        Fixup += toupper(Word.front());
         Fixup += Word.substr(1).lower();
       }
     }
@@ -287,7 +282,7 @@ static std::string fixupWithCase(StringRef Name,
     for (auto const &Word : Words) {
       if (&Word != &Words.front())
         Fixup += "_";
-      Fixup += Word.substr(0, 1).upper();
+      Fixup += toupper(Word.front());
       Fixup += Word.substr(1).lower();
     }
     break;
@@ -296,16 +291,16 @@ static std::string fixupWithCase(StringRef Name,
     for (auto const &Word : Words) {
       if (&Word != &Words.front()) {
         Fixup += "_";
-        Fixup += Word.substr(0, 1).upper();
+        Fixup += toupper(Word.front());
       } else {
-        Fixup += Word.substr(0, 1).lower();
+        Fixup += tolower(Word.front());
       }
       Fixup += Word.substr(1).lower();
     }
     break;
   }
 
-  return Fixup;
+  return Fixup.str().str();
 }
 
 static bool isParamInMainLikeFunction(const ParmVarDecl &ParmDecl,
@@ -715,8 +710,8 @@ RenamerClangTidyCheck::DiagInfo
 IdentifierNamingCheck::GetDiagInfo(const NamingCheckId &ID,
                                    const NamingCheckFailure &Failure) const {
   return DiagInfo{"invalid case style for %0 '%1'",
-                  [&](DiagnosticBuilder &diag) {
-                    diag << Failure.Info.KindName << ID.second;
+                  [&](DiagnosticBuilder &Diag) {
+                    Diag << Failure.Info.KindName << ID.second;
                   }};
 }
 

From 1b1901536a7db8a074e794f004f9d94b941f7068 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Sat, 1 Aug 2020 12:28:10 +0100
Subject: [PATCH 0809/1035] [X86][AVX] Extend v2f64 BROADCAST(LOAD) ->
 BROADCAST_LOAD to v2i64/v4f32/v4i32

Minor precursor fix for D66004, but helps the SSE41 tests as well as they run with -disable-peephole
---
 llvm/lib/Target/X86/X86ISelLowering.cpp | 9 +++++----
 llvm/test/CodeGen/X86/sse41.ll          | 6 ++----
 2 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index cb1067a06239e..d628cdfc1fdf1 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -36144,15 +36144,16 @@ static SDValue combineTargetShuffle(SDValue N, SelectionDAG &DAG,
     }
 
     // vbroadcast(vector load X) -> vbroadcast_load
-    if (SrcVT == MVT::v2f64 && Src.hasOneUse() &&
-        ISD::isNormalLoad(Src.getNode())) {
+    if ((SrcVT == MVT::v2f64 || SrcVT == MVT::v4f32 || SrcVT == MVT::v2i64 ||
+         SrcVT == MVT::v4i32) &&
+        Src.hasOneUse() && ISD::isNormalLoad(Src.getNode())) {
       LoadSDNode *LN = cast<LoadSDNode>(Src);
       // Unless the load is volatile or atomic.
       if (LN->isSimple()) {
         SDVTList Tys = DAG.getVTList(VT, MVT::Other);
-        SDValue Ops[] = { LN->getChain(), LN->getBasePtr() };
+        SDValue Ops[] = {LN->getChain(), LN->getBasePtr()};
         SDValue BcastLd = DAG.getMemIntrinsicNode(
-            X86ISD::VBROADCAST_LOAD, DL, Tys, Ops, MVT::f64,
+            X86ISD::VBROADCAST_LOAD, DL, Tys, Ops, SrcVT.getScalarType(),
             LN->getPointerInfo(), LN->getOriginalAlign(),
             LN->getMemOperand()->getFlags());
         DCI.CombineTo(N.getNode(), BcastLd);
diff --git a/llvm/test/CodeGen/X86/sse41.ll b/llvm/test/CodeGen/X86/sse41.ll
index 99cd686f50bd5..ef503760890f7 100644
--- a/llvm/test/CodeGen/X86/sse41.ll
+++ b/llvm/test/CodeGen/X86/sse41.ll
@@ -654,8 +654,7 @@ define <4 x i32> @pinsrd_from_shufflevector_i32(<4 x i32> %a, <4 x i32>* nocaptu
 ; X86-AVX512-LABEL: pinsrd_from_shufflevector_i32:
 ; X86-AVX512:       ## %bb.0: ## %entry
 ; X86-AVX512-NEXT:    movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x04]
-; X86-AVX512-NEXT:    vmovaps (%eax), %xmm1 ## EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0x08]
-; X86-AVX512-NEXT:    vbroadcastss %xmm1, %xmm1 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x18,0xc9]
+; X86-AVX512-NEXT:    vbroadcastss (%eax), %xmm1 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x18,0x08]
 ; X86-AVX512-NEXT:    vblendps $8, %xmm1, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x0c,0xc1,0x08]
 ; X86-AVX512-NEXT:    ## xmm0 = xmm0[0,1,2],xmm1[3]
 ; X86-AVX512-NEXT:    retl ## encoding: [0xc3]
@@ -678,8 +677,7 @@ define <4 x i32> @pinsrd_from_shufflevector_i32(<4 x i32> %a, <4 x i32>* nocaptu
 ;
 ; X64-AVX512-LABEL: pinsrd_from_shufflevector_i32:
 ; X64-AVX512:       ## %bb.0: ## %entry
-; X64-AVX512-NEXT:    vmovaps (%rdi), %xmm1 ## EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0x0f]
-; X64-AVX512-NEXT:    vbroadcastss %xmm1, %xmm1 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x18,0xc9]
+; X64-AVX512-NEXT:    vbroadcastss (%rdi), %xmm1 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x18,0x0f]
 ; X64-AVX512-NEXT:    vblendps $8, %xmm1, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x0c,0xc1,0x08]
 ; X64-AVX512-NEXT:    ## xmm0 = xmm0[0,1,2],xmm1[3]
 ; X64-AVX512-NEXT:    retq ## encoding: [0xc3]

From 1aa52d67d1c1a8eb174d4a85391a17c5f0731116 Mon Sep 17 00:00:00 2001
From: Sanjay Patel <spatel@rotateright.com>
Date: Sat, 1 Aug 2020 07:44:33 -0400
Subject: [PATCH 0810/1035] [InstSimplify] add abs test with assume; NFC

---
 llvm/test/Transforms/InstSimplify/call.ll | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/llvm/test/Transforms/InstSimplify/call.ll b/llvm/test/Transforms/InstSimplify/call.ll
index e8c39dbda8f98..e4fcf72327cc4 100644
--- a/llvm/test/Transforms/InstSimplify/call.ll
+++ b/llvm/test/Transforms/InstSimplify/call.ll
@@ -51,6 +51,21 @@ define <3 x i82> @select_abs(<3 x i1> %cond) {
   ret <3 x i82> %abs
 }
 
+declare void @llvm.assume(i1)
+
+define i32 @assume_abs(i32 %x) {
+; CHECK-LABEL: @assume_abs(
+; CHECK-NEXT:    [[ASSUME:%.*]] = icmp sge i32 [[X:%.*]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[ASSUME]])
+; CHECK-NEXT:    [[ABS:%.*]] = call i32 @llvm.abs.i32(i32 [[X]], i1 true)
+; CHECK-NEXT:    ret i32 [[ABS]]
+;
+  %assume = icmp sge i32 %x, 0
+  call void @llvm.assume(i1 %assume)
+  %abs = call i32 @llvm.abs.i32(i32 %x, i1 true)
+  ret i32 %abs
+}
+
 declare {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a, i8 %b)
 declare {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a, i8 %b)
 declare {i8, i1} @llvm.usub.with.overflow.i8(i8 %a, i8 %b)
@@ -1092,7 +1107,7 @@ define i32 @call_undef_musttail() {
 
 define float @nobuiltin_fmax() {
 ; CHECK-LABEL: @nobuiltin_fmax(
-; CHECK-NEXT:    [[M:%.*]] = call float @fmaxf(float 0.000000e+00, float 1.000000e+00) #3
+; CHECK-NEXT:    [[M:%.*]] = call float @fmaxf(float 0.000000e+00, float 1.000000e+00) #4
 ; CHECK-NEXT:    [[R:%.*]] = call float @llvm.fabs.f32(float [[M]])
 ; CHECK-NEXT:    ret float [[R]]
 ;

From 04b99a4d18cf13c13f1d76c5698696bcaef4e4b6 Mon Sep 17 00:00:00 2001
From: Sanjay Patel <spatel@rotateright.com>
Date: Sat, 1 Aug 2020 07:46:23 -0400
Subject: [PATCH 0811/1035] [InstSimplify] simplify abs if operand is known
 non-negative

abs() should be rare enough that using value tracking is not going
to be a compile-time cost burden, so use it to reduce a variety of
potential patterns. We do this in DAGCombiner too.

Differential Revision: https://reviews.llvm.org/D85043
---
 llvm/lib/Analysis/InstructionSimplify.cpp |  6 ++++++
 llvm/test/Transforms/InstSimplify/call.ll | 17 ++++++-----------
 2 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp
index 38f3bbf4c6f53..d3928a502965b 100644
--- a/llvm/lib/Analysis/InstructionSimplify.cpp
+++ b/llvm/lib/Analysis/InstructionSimplify.cpp
@@ -5255,6 +5255,12 @@ static Value *simplifyBinaryIntrinsic(Function *F, Value *Op0, Value *Op1,
   Type *ReturnType = F->getReturnType();
   unsigned BitWidth = ReturnType->getScalarSizeInBits();
   switch (IID) {
+  case Intrinsic::abs:
+    // If the sign bit is clear already, then abs does not do anything.
+    if (isKnownNonNegative(Op0, Q.DL, 0, Q.AC, Q.CxtI, Q.DT))
+      return Op0;
+    break;
+
   case Intrinsic::smax:
   case Intrinsic::smin:
   case Intrinsic::umax:
diff --git a/llvm/test/Transforms/InstSimplify/call.ll b/llvm/test/Transforms/InstSimplify/call.ll
index e4fcf72327cc4..2325dccd17a85 100644
--- a/llvm/test/Transforms/InstSimplify/call.ll
+++ b/llvm/test/Transforms/InstSimplify/call.ll
@@ -5,13 +5,12 @@
 declare i32 @llvm.abs.i32(i32, i1)
 declare <3 x i82> @llvm.abs.v3i82(<3 x i82>, i1)
 
-; TODO: If the sign bit is known zero, the abs is not needed.
+; If the sign bit is known zero, the abs is not needed.
 
 define i32 @zext_abs(i31 %x) {
 ; CHECK-LABEL: @zext_abs(
 ; CHECK-NEXT:    [[ZEXT:%.*]] = zext i31 [[X:%.*]] to i32
-; CHECK-NEXT:    [[ABS:%.*]] = call i32 @llvm.abs.i32(i32 [[ZEXT]], i1 false)
-; CHECK-NEXT:    ret i32 [[ABS]]
+; CHECK-NEXT:    ret i32 [[ZEXT]]
 ;
   %zext = zext i31 %x to i32
   %abs = call i32 @llvm.abs.i32(i32 %zext, i1 false)
@@ -21,8 +20,7 @@ define i32 @zext_abs(i31 %x) {
 define <3 x i82> @lshr_abs(<3 x i82> %x) {
 ; CHECK-LABEL: @lshr_abs(
 ; CHECK-NEXT:    [[LSHR:%.*]] = lshr <3 x i82> [[X:%.*]], <i82 1, i82 1, i82 1>
-; CHECK-NEXT:    [[ABS:%.*]] = call <3 x i82> @llvm.abs.v3i82(<3 x i82> [[LSHR]], i1 true)
-; CHECK-NEXT:    ret <3 x i82> [[ABS]]
+; CHECK-NEXT:    ret <3 x i82> [[LSHR]]
 ;
   %lshr = lshr <3 x i82> %x, <i82 1, i82 1, i82 1>
   %abs = call <3 x i82> @llvm.abs.v3i82(<3 x i82> %lshr, i1 true)
@@ -32,8 +30,7 @@ define <3 x i82> @lshr_abs(<3 x i82> %x) {
 define i32 @and_abs(i32 %x) {
 ; CHECK-LABEL: @and_abs(
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 2147483644
-; CHECK-NEXT:    [[ABS:%.*]] = call i32 @llvm.abs.i32(i32 [[AND]], i1 true)
-; CHECK-NEXT:    ret i32 [[ABS]]
+; CHECK-NEXT:    ret i32 [[AND]]
 ;
   %and = and i32 %x, 2147483644
   %abs = call i32 @llvm.abs.i32(i32 %and, i1 true)
@@ -43,8 +40,7 @@ define i32 @and_abs(i32 %x) {
 define <3 x i82> @select_abs(<3 x i1> %cond) {
 ; CHECK-LABEL: @select_abs(
 ; CHECK-NEXT:    [[SEL:%.*]] = select <3 x i1> [[COND:%.*]], <3 x i82> zeroinitializer, <3 x i82> <i82 2147483647, i82 42, i82 1>
-; CHECK-NEXT:    [[ABS:%.*]] = call <3 x i82> @llvm.abs.v3i82(<3 x i82> [[SEL]], i1 false)
-; CHECK-NEXT:    ret <3 x i82> [[ABS]]
+; CHECK-NEXT:    ret <3 x i82> [[SEL]]
 ;
   %sel = select <3 x i1> %cond, <3 x i82> zeroinitializer, <3 x i82> <i82 2147483647, i82 42, i82 1>
   %abs = call <3 x i82> @llvm.abs.v3i82(<3 x i82> %sel, i1 false)
@@ -57,8 +53,7 @@ define i32 @assume_abs(i32 %x) {
 ; CHECK-LABEL: @assume_abs(
 ; CHECK-NEXT:    [[ASSUME:%.*]] = icmp sge i32 [[X:%.*]], 0
 ; CHECK-NEXT:    call void @llvm.assume(i1 [[ASSUME]])
-; CHECK-NEXT:    [[ABS:%.*]] = call i32 @llvm.abs.i32(i32 [[X]], i1 true)
-; CHECK-NEXT:    ret i32 [[ABS]]
+; CHECK-NEXT:    ret i32 [[X]]
 ;
   %assume = icmp sge i32 %x, 0
   call void @llvm.assume(i1 %assume)

From eb41f9edde1070d68fce4a4eb31118e0ec1ca36d Mon Sep 17 00:00:00 2001
From: Benjamin Kramer <benny.kra@googlemail.com>
Date: Sat, 1 Aug 2020 14:48:42 +0200
Subject: [PATCH 0812/1035] [mlir][Vector] Simplify code a bit. NFCI.

---
 mlir/lib/Dialect/Vector/VectorOps.cpp        | 26 ++++++++------------
 mlir/lib/Dialect/Vector/VectorTransforms.cpp | 18 +++++++-------
 2 files changed, 19 insertions(+), 25 deletions(-)

diff --git a/mlir/lib/Dialect/Vector/VectorOps.cpp b/mlir/lib/Dialect/Vector/VectorOps.cpp
index d16c7c3d6fdbe..c788d4ccb4a08 100644
--- a/mlir/lib/Dialect/Vector/VectorOps.cpp
+++ b/mlir/lib/Dialect/Vector/VectorOps.cpp
@@ -184,9 +184,9 @@ static ParseResult parseContractionOp(OpAsmParser &parser,
   auto lhsType = types[0].cast<VectorType>();
   auto rhsType = types[1].cast<VectorType>();
   auto maskElementType = parser.getBuilder().getI1Type();
-  SmallVector<Type, 2> maskTypes;
-  maskTypes.push_back(VectorType::get(lhsType.getShape(), maskElementType));
-  maskTypes.push_back(VectorType::get(rhsType.getShape(), maskElementType));
+  std::array<Type, 2> maskTypes = {
+      VectorType::get(lhsType.getShape(), maskElementType),
+      VectorType::get(rhsType.getShape(), maskElementType)};
   if (parser.resolveOperands(masksInfo, maskTypes, loc, result.operands))
     return failure();
   return success();
@@ -462,12 +462,10 @@ std::vector<std::pair<int64_t, int64_t>> ContractionOp::getBatchDimMap() {
 }
 
 SmallVector<AffineMap, 4> ContractionOp::getIndexingMaps() {
-  SmallVector<AffineMap, 4> res;
-  auto mapAttrs = indexing_maps().getValue();
-  res.reserve(mapAttrs.size());
-  for (auto mapAttr : mapAttrs)
-    res.push_back(mapAttr.cast<AffineMapAttr>().getValue());
-  return res;
+  return llvm::to_vector<4>(
+      llvm::map_range(indexing_maps().getValue(), [](Attribute mapAttr) {
+        return mapAttr.cast<AffineMapAttr>().getValue();
+      }));
 }
 
 Optional<SmallVector<int64_t, 4>> ContractionOp::getShapeForUnroll() {
@@ -1854,8 +1852,7 @@ LogicalResult TransferWriteOp::fold(ArrayRef<Attribute>,
 }
 
 Optional<SmallVector<int64_t, 4>> TransferWriteOp::getShapeForUnroll() {
-  auto s = getVectorType().getShape();
-  return SmallVector<int64_t, 4>{s.begin(), s.end()};
+  return llvm::to_vector<4>(getVectorType().getShape());
 }
 
 //===----------------------------------------------------------------------===//
@@ -2014,11 +2011,8 @@ static SmallVector<int64_t, 8> extractShape(MemRefType memRefType) {
   auto vectorType = memRefType.getElementType().dyn_cast<VectorType>();
   SmallVector<int64_t, 8> res(memRefType.getShape().begin(),
                               memRefType.getShape().end());
-  if (vectorType) {
-    res.reserve(memRefType.getRank() + vectorType.getRank());
-    for (auto s : vectorType.getShape())
-      res.push_back(s);
-  }
+  if (vectorType)
+    res.append(vectorType.getShape().begin(), vectorType.getShape().end());
   return res;
 }
 
diff --git a/mlir/lib/Dialect/Vector/VectorTransforms.cpp b/mlir/lib/Dialect/Vector/VectorTransforms.cpp
index ab93ef406024e..197b1c62274b2 100644
--- a/mlir/lib/Dialect/Vector/VectorTransforms.cpp
+++ b/mlir/lib/Dialect/Vector/VectorTransforms.cpp
@@ -1707,7 +1707,7 @@ void ContractionOpToOuterProductOpLowering::rewrite(
   auto infer = [](MapList m) { return AffineMap::inferFromExprList(m); };
   AffineExpr m, n, k;
   bindDims(rewriter.getContext(), m, n, k);
-  SmallVector<int64_t, 2> perm{1, 0};
+  static constexpr std::array<int64_t, 2> perm = {1, 0};
   auto iteratorTypes = op.iterator_types().getValue();
   SmallVector<AffineMap, 4> maps = op.getIndexingMaps();
   if (isParallelIterator(iteratorTypes[0]) &&
@@ -1911,10 +1911,10 @@ Value ContractionOpLowering::lowerParallel(vector::ContractionOp op,
   assert(lookup.hasValue() && "parallel index not listed in reduction");
   int64_t resIndex = lookup.getValue();
   // Construct new iterator types and affine map array attribute.
-  SmallVector<AffineMap, 4> lowIndexingMaps;
-  lowIndexingMaps.push_back(adjustMap(iMap[0], iterIndex, rewriter));
-  lowIndexingMaps.push_back(adjustMap(iMap[1], iterIndex, rewriter));
-  lowIndexingMaps.push_back(adjustMap(iMap[2], iterIndex, rewriter));
+  std::array<AffineMap, 3> lowIndexingMaps = {
+      adjustMap(iMap[0], iterIndex, rewriter),
+      adjustMap(iMap[1], iterIndex, rewriter),
+      adjustMap(iMap[2], iterIndex, rewriter)};
   auto lowAffine = rewriter.getAffineMapArrayAttr(lowIndexingMaps);
   auto lowIter =
       rewriter.getArrayAttr(adjustIter(op.iterator_types(), iterIndex));
@@ -1962,10 +1962,10 @@ Value ContractionOpLowering::lowerReduction(vector::ContractionOp op,
                                                 op.acc());
   }
   // Construct new iterator types and affine map array attribute.
-  SmallVector<AffineMap, 4> lowIndexingMaps;
-  lowIndexingMaps.push_back(adjustMap(iMap[0], iterIndex, rewriter));
-  lowIndexingMaps.push_back(adjustMap(iMap[1], iterIndex, rewriter));
-  lowIndexingMaps.push_back(adjustMap(iMap[2], iterIndex, rewriter));
+  std::array<AffineMap, 3> lowIndexingMaps = {
+      adjustMap(iMap[0], iterIndex, rewriter),
+      adjustMap(iMap[1], iterIndex, rewriter),
+      adjustMap(iMap[2], iterIndex, rewriter)};
   auto lowAffine = rewriter.getAffineMapArrayAttr(lowIndexingMaps);
   auto lowIter =
       rewriter.getArrayAttr(adjustIter(op.iterator_types(), iterIndex));

From fd69df62ed1091f25ba4749cc5152e9ce2fe3af4 Mon Sep 17 00:00:00 2001
From: David Green <david.green@arm.com>
Date: Sat, 1 Aug 2020 14:01:18 +0100
Subject: [PATCH 0813/1035] [ARM] Distribute post-inc for Thumb2 sign/zero
 extending loads/stores

This adds sign/zero extending scalar loads/stores to the MVE
instructions added in D77813, allowing us to create up more post-inc
instructions. These are comparatively simple, compared to LDR/STR (which
may be better turned into an LDRD/LDM), but still require some additions
over MVE instructions. Because there are i12 and i8 variants of the
offset loads/stores dealing with different signs, we may need to convert
an i12 address to a i8 negative instruction. t2LDRBi12 can also be
shrunk to a tLDRi under the right conditions, so we need to be careful
with codesize too.

Differential Revision: https://reviews.llvm.org/D78625
---
 llvm/lib/Target/ARM/ARMBaseInstrInfo.h        |   4 +
 llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp | 169 ++++++++++++++++--
 .../LowOverheadLoops/mve-float-loops.ll       |  15 +-
 .../LowOverheadLoops/mve-tail-data-types.ll   |  20 +--
 .../CodeGen/Thumb2/mve-float16regloops.ll     |  17 +-
 .../CodeGen/Thumb2/mve-postinc-distribute.ll  |  12 +-
 .../CodeGen/Thumb2/postinc-distribute.mir     |  94 ++++++++--
 7 files changed, 261 insertions(+), 70 deletions(-)

diff --git a/llvm/lib/Target/ARM/ARMBaseInstrInfo.h b/llvm/lib/Target/ARM/ARMBaseInstrInfo.h
index 1a75b011ca59d..f71445cf59c35 100644
--- a/llvm/lib/Target/ARM/ARMBaseInstrInfo.h
+++ b/llvm/lib/Target/ARM/ARMBaseInstrInfo.h
@@ -829,6 +829,10 @@ inline bool isLegalAddressImm(unsigned Opcode, int Imm,
     return std::abs(Imm) < (((1 << 7) * 2) - 1) && Imm % 2 == 0;
   case ARMII::AddrModeT2_i7s4:
     return std::abs(Imm) < (((1 << 7) * 4) - 1) && Imm % 4 == 0;
+  case ARMII::AddrModeT2_i8:
+    return std::abs(Imm) < (((1 << 8) * 1) - 1);
+  case ARMII::AddrModeT2_i12:
+    return Imm >= 0 && Imm < (((1 << 12) * 1) - 1);
   default:
     llvm_unreachable("Unhandled Addressing mode");
   }
diff --git a/llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp b/llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp
index a84d23d3bb96a..09bb3b3c6f728 100644
--- a/llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp
+++ b/llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp
@@ -1382,9 +1382,27 @@ static unsigned getPostIndexedLoadStoreOpcode(unsigned Opc,
   case ARM::t2LDRi8:
   case ARM::t2LDRi12:
     return ARM::t2LDR_POST;
+  case ARM::t2LDRBi8:
+  case ARM::t2LDRBi12:
+    return ARM::t2LDRB_POST;
+  case ARM::t2LDRSBi8:
+  case ARM::t2LDRSBi12:
+    return ARM::t2LDRSB_POST;
+  case ARM::t2LDRHi8:
+  case ARM::t2LDRHi12:
+    return ARM::t2LDRH_POST;
+  case ARM::t2LDRSHi8:
+  case ARM::t2LDRSHi12:
+    return ARM::t2LDRSH_POST;
   case ARM::t2STRi8:
   case ARM::t2STRi12:
     return ARM::t2STR_POST;
+  case ARM::t2STRBi8:
+  case ARM::t2STRBi12:
+    return ARM::t2STRB_POST;
+  case ARM::t2STRHi8:
+  case ARM::t2STRHi12:
+    return ARM::t2STRH_POST;
 
   case ARM::MVE_VLDRBS16:
     return ARM::MVE_VLDRBS16_post;
@@ -2539,11 +2557,94 @@ static int getBaseOperandIndex(MachineInstr &MI) {
   case ARM::MVE_VSTRBU8:
   case ARM::MVE_VSTRHU16:
   case ARM::MVE_VSTRWU32:
+  case ARM::t2LDRHi8:
+  case ARM::t2LDRHi12:
+  case ARM::t2LDRSHi8:
+  case ARM::t2LDRSHi12:
+  case ARM::t2LDRBi8:
+  case ARM::t2LDRBi12:
+  case ARM::t2LDRSBi8:
+  case ARM::t2LDRSBi12:
+  case ARM::t2STRBi8:
+  case ARM::t2STRBi12:
+  case ARM::t2STRHi8:
+  case ARM::t2STRHi12:
     return 1;
   }
   return -1;
 }
 
+// Given a memory access Opcode, check that the give Imm would be a valid Offset
+// for this instruction (same as isLegalAddressImm), Or if the instruction
+// could be easily converted to one where that was valid. For example converting
+// t2LDRi12 to t2LDRi8 for negative offsets. Works in conjunction with
+// AdjustBaseAndOffset below.
+static bool isLegalOrConvertableAddressImm(unsigned Opcode, int Imm,
+                                           const TargetInstrInfo *TII,
+                                           int &CodesizeEstimate) {
+  if (isLegalAddressImm(Opcode, Imm, TII))
+    return true;
+
+  // We can convert AddrModeT2_i12 to AddrModeT2_i8.
+  const MCInstrDesc &Desc = TII->get(Opcode);
+  unsigned AddrMode = (Desc.TSFlags & ARMII::AddrModeMask);
+  switch (AddrMode) {
+  case ARMII::AddrModeT2_i12:
+    CodesizeEstimate += 1;
+    return std::abs(Imm) < (((1 << 8) * 1) - 1);
+  }
+  return false;
+}
+
+// Given an MI adjust its address BaseReg to use NewBaseReg and address offset
+// by -Offset. This can either happen in-place or be a replacement as MI is
+// converted to another instruction type.
+static void AdjustBaseAndOffset(MachineInstr *MI, Register NewBaseReg,
+                                int Offset, const TargetInstrInfo *TII) {
+  unsigned BaseOp = getBaseOperandIndex(*MI);
+  MI->getOperand(BaseOp).setReg(NewBaseReg);
+  int OldOffset = MI->getOperand(BaseOp + 1).getImm();
+  if (isLegalAddressImm(MI->getOpcode(), OldOffset - Offset, TII))
+    MI->getOperand(BaseOp + 1).setImm(OldOffset - Offset);
+  else {
+    unsigned ConvOpcode;
+    switch (MI->getOpcode()) {
+    case ARM::t2LDRHi12:
+      ConvOpcode = ARM::t2LDRHi8;
+      break;
+    case ARM::t2LDRSHi12:
+      ConvOpcode = ARM::t2LDRSHi8;
+      break;
+    case ARM::t2LDRBi12:
+      ConvOpcode = ARM::t2LDRBi8;
+      break;
+    case ARM::t2LDRSBi12:
+      ConvOpcode = ARM::t2LDRSBi8;
+      break;
+    case ARM::t2STRHi12:
+      ConvOpcode = ARM::t2STRHi8;
+      break;
+    case ARM::t2STRBi12:
+      ConvOpcode = ARM::t2STRBi8;
+      break;
+    default:
+      llvm_unreachable("Unhandled convertable opcode");
+    }
+    assert(isLegalAddressImm(ConvOpcode, OldOffset - Offset, TII) &&
+           "Illegal Address Immediate after convert!");
+
+    const MCInstrDesc &MCID = TII->get(ConvOpcode);
+    BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), MCID)
+        .add(MI->getOperand(0))
+        .add(MI->getOperand(1))
+        .addImm(OldOffset - Offset)
+        .add(MI->getOperand(3))
+        .add(MI->getOperand(4))
+        .cloneMemRefs(*MI);
+    MI->eraseFromParent();
+  }
+}
+
 static MachineInstr *createPostIncLoadStore(MachineInstr *MI, int Offset,
                                             Register NewReg,
                                             const TargetInstrInfo *TII,
@@ -2562,14 +2663,43 @@ static MachineInstr *createPostIncLoadStore(MachineInstr *MI, int Offset,
   TRC = TII->getRegClass(MCID, 2, TRI, *MF);
   MRI.constrainRegClass(MI->getOperand(1).getReg(), TRC);
 
-  return BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), MCID)
-      .addReg(NewReg, RegState::Define)
-      .add(MI->getOperand(0))
-      .add(MI->getOperand(1))
-      .addImm(Offset)
-      .add(MI->getOperand(3))
-      .add(MI->getOperand(4))
-      .cloneMemRefs(*MI);
+  unsigned AddrMode = (MCID.TSFlags & ARMII::AddrModeMask);
+  switch (AddrMode) {
+  case ARMII::AddrModeT2_i7:
+  case ARMII::AddrModeT2_i7s2:
+  case ARMII::AddrModeT2_i7s4:
+    // Any MVE load/store
+    return BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), MCID)
+        .addReg(NewReg, RegState::Define)
+        .add(MI->getOperand(0))
+        .add(MI->getOperand(1))
+        .addImm(Offset)
+        .add(MI->getOperand(3))
+        .add(MI->getOperand(4))
+        .cloneMemRefs(*MI);
+  case ARMII::AddrModeT2_i8:
+    if (MI->mayLoad()) {
+      return BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), MCID)
+          .add(MI->getOperand(0))
+          .addReg(NewReg, RegState::Define)
+          .add(MI->getOperand(1))
+          .addImm(Offset)
+          .add(MI->getOperand(3))
+          .add(MI->getOperand(4))
+          .cloneMemRefs(*MI);
+    } else {
+      return BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), MCID)
+          .addReg(NewReg, RegState::Define)
+          .add(MI->getOperand(0))
+          .add(MI->getOperand(1))
+          .addImm(Offset)
+          .add(MI->getOperand(3))
+          .add(MI->getOperand(4))
+          .cloneMemRefs(*MI);
+    }
+  default:
+    llvm_unreachable("Unhandled createPostIncLoadStore");
+  }
 }
 
 // Given a Base Register, optimise the load/store uses to attempt to create more
@@ -2589,7 +2719,7 @@ bool ARMPreAllocLoadStoreOpt::DistributeIncrements(Register Base) {
   // An increment that can be folded in
   MachineInstr *Increment = nullptr;
   // Other accesses after BaseAccess that will need to be updated to use the
-  // postinc value
+  // postinc value.
   SmallPtrSet<MachineInstr *, 8> OtherAccesses;
   for (auto &Use : MRI->use_nodbg_instructions(Base)) {
     if (!Increment && getAddSubImmediate(Use) != 0) {
@@ -2643,14 +2773,20 @@ bool ARMPreAllocLoadStoreOpt::DistributeIncrements(Register Base) {
   // other offsets after the BaseAccess. We rely on either
   // dominates(BaseAccess, OtherAccess) or dominates(OtherAccess, BaseAccess)
   // to keep things simple.
+  // This also adds a simple codesize metric, to detect if an instruction (like
+  // t2LDRBi12) which can often be shrunk to a thumb1 instruction (tLDRBi)
+  // cannot because it is converted to something else (t2LDRBi8). We start this
+  // at -1 for the gain from removing the increment.
   SmallPtrSet<MachineInstr *, 4> SuccessorAccesses;
+  int CodesizeEstimate = -1;
   for (auto *Use : OtherAccesses) {
     if (DT->dominates(BaseAccess, Use)) {
       SuccessorAccesses.insert(Use);
       unsigned BaseOp = getBaseOperandIndex(*Use);
-      if (!isLegalAddressImm(
-              Use->getOpcode(),
-              Use->getOperand(BaseOp + 1).getImm() - IncrementOffset, TII)) {
+      if (!isLegalOrConvertableAddressImm(Use->getOpcode(),
+                                          Use->getOperand(BaseOp + 1).getImm() -
+                                              IncrementOffset,
+                                          TII, CodesizeEstimate)) {
         LLVM_DEBUG(dbgs() << "  Illegal addressing mode immediate on use\n");
         return false;
       }
@@ -2660,6 +2796,10 @@ bool ARMPreAllocLoadStoreOpt::DistributeIncrements(Register Base) {
       return false;
     }
   }
+  if (STI->hasMinSize() && CodesizeEstimate > 0) {
+    LLVM_DEBUG(dbgs() << "  Expected to grow instructions under minsize\n");
+    return false;
+  }
 
   // Replace BaseAccess with a post inc
   LLVM_DEBUG(dbgs() << "Changing: "; BaseAccess->dump());
@@ -2674,10 +2814,7 @@ bool ARMPreAllocLoadStoreOpt::DistributeIncrements(Register Base) {
 
   for (auto *Use : SuccessorAccesses) {
     LLVM_DEBUG(dbgs() << "Changing: "; Use->dump());
-    unsigned BaseOp = getBaseOperandIndex(*Use);
-    Use->getOperand(BaseOp).setReg(NewBaseReg);
-    int OldOffset = Use->getOperand(BaseOp + 1).getImm();
-    Use->getOperand(BaseOp + 1).setImm(OldOffset - IncrementOffset);
+    AdjustBaseAndOffset(Use, NewBaseReg, IncrementOffset, TII);
     LLVM_DEBUG(dbgs() << "  To    : "; Use->dump());
   }
 
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-float-loops.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-float-loops.ll
index e69610f9df2a9..261222f60f17a 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-float-loops.ll
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-float-loops.ll
@@ -1798,20 +1798,20 @@ define arm_aapcs_vfpcc float @half_short_mac(half* nocapture readonly %a, i16* n
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    ldrsh.w r4, [r3, #2]
 ; CHECK-NEXT:    vldr.16 s2, [r2, #2]
-; CHECK-NEXT:    ldrsh r5, [r3, #-2]
 ; CHECK-NEXT:    add.w r12, r12, #4
 ; CHECK-NEXT:    vmov s4, r4
-; CHECK-NEXT:    ldrsh.w r4, [r3]
+; CHECK-NEXT:    ldrsh r4, [r3], #8
 ; CHECK-NEXT:    vcvt.f16.s32 s4, s4
-; CHECK-NEXT:    vmov s8, r5
+; CHECK-NEXT:    ldrsh r5, [r3, #-10]
 ; CHECK-NEXT:    vmul.f16 s2, s2, s4
-; CHECK-NEXT:    vldr.16 s4, [r2]
 ; CHECK-NEXT:    vmov s6, r4
-; CHECK-NEXT:    ldrsh r4, [r3, #-4]
+; CHECK-NEXT:    vldr.16 s4, [r2]
 ; CHECK-NEXT:    vcvt.f16.s32 s6, s6
-; CHECK-NEXT:    vcvt.f16.s32 s8, s8
+; CHECK-NEXT:    ldrsh r4, [r3, #-12]
 ; CHECK-NEXT:    vmul.f16 s4, s4, s6
+; CHECK-NEXT:    vmov s8, r5
 ; CHECK-NEXT:    vldr.16 s6, [r2, #-2]
+; CHECK-NEXT:    vcvt.f16.s32 s8, s8
 ; CHECK-NEXT:    vmov s10, r4
 ; CHECK-NEXT:    vcvtb.f32.f16 s4, s4
 ; CHECK-NEXT:    vmul.f16 s6, s6, s8
@@ -1821,9 +1821,8 @@ define arm_aapcs_vfpcc float @half_short_mac(half* nocapture readonly %a, i16* n
 ; CHECK-NEXT:    vmul.f16 s8, s8, s10
 ; CHECK-NEXT:    vcvtb.f32.f16 s2, s2
 ; CHECK-NEXT:    vcvtb.f32.f16 s8, s8
-; CHECK-NEXT:    adds r3, #8
-; CHECK-NEXT:    vadd.f32 s0, s0, s8
 ; CHECK-NEXT:    adds r2, #8
+; CHECK-NEXT:    vadd.f32 s0, s0, s8
 ; CHECK-NEXT:    vadd.f32 s0, s0, s6
 ; CHECK-NEXT:    vadd.f32 s0, s0, s4
 ; CHECK-NEXT:    vadd.f32 s0, s0, s2
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-tail-data-types.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-tail-data-types.ll
index 428c703dd341e..69039f9a4eaa2 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-tail-data-types.ll
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-tail-data-types.ll
@@ -437,17 +437,15 @@ define arm_aapcs_vfpcc void @test_vec_mul_scalar_add_char(i8* nocapture readonly
 ; CHECK-NEXT:    smlabb r7, r7, r8, r2
 ; CHECK-NEXT:    str r7, [r4, #-8]
 ; CHECK-NEXT:    ldrb r8, [r5, #-2]
-; CHECK-NEXT:    ldrb r7, [r6]
+; CHECK-NEXT:    ldrb r7, [r6], #4
 ; CHECK-NEXT:    smlabb r7, r7, r8, r2
 ; CHECK-NEXT:    str r7, [r4, #-4]
 ; CHECK-NEXT:    ldrb r8, [r5, #-1]
-; CHECK-NEXT:    ldrb r7, [r6, #1]
+; CHECK-NEXT:    ldrb r7, [r6, #-3]
 ; CHECK-NEXT:    smlabb r7, r7, r8, r2
 ; CHECK-NEXT:    str r7, [r4]
-; CHECK-NEXT:    ldrb.w r8, [r5]
-; CHECK-NEXT:    adds r5, #4
-; CHECK-NEXT:    ldrb r7, [r6, #2]
-; CHECK-NEXT:    adds r6, #4
+; CHECK-NEXT:    ldrb r8, [r5], #4
+; CHECK-NEXT:    ldrb r7, [r6, #-2]
 ; CHECK-NEXT:    smlabb r7, r7, r8, r2
 ; CHECK-NEXT:    str r7, [r4, #4]
 ; CHECK-NEXT:    adds r4, #16
@@ -740,17 +738,15 @@ define arm_aapcs_vfpcc void @test_vec_mul_scalar_add_uchar(i8* nocapture readonl
 ; CHECK-NEXT:    smlabb r7, r7, r8, r2
 ; CHECK-NEXT:    str r7, [r4, #-8]
 ; CHECK-NEXT:    ldrb r8, [r5, #-2]
-; CHECK-NEXT:    ldrb r7, [r6]
+; CHECK-NEXT:    ldrb r7, [r6], #4
 ; CHECK-NEXT:    smlabb r7, r7, r8, r2
 ; CHECK-NEXT:    str r7, [r4, #-4]
 ; CHECK-NEXT:    ldrb r8, [r5, #-1]
-; CHECK-NEXT:    ldrb r7, [r6, #1]
+; CHECK-NEXT:    ldrb r7, [r6, #-3]
 ; CHECK-NEXT:    smlabb r7, r7, r8, r2
 ; CHECK-NEXT:    str r7, [r4]
-; CHECK-NEXT:    ldrb.w r8, [r5]
-; CHECK-NEXT:    adds r5, #4
-; CHECK-NEXT:    ldrb r7, [r6, #2]
-; CHECK-NEXT:    adds r6, #4
+; CHECK-NEXT:    ldrb r8, [r5], #4
+; CHECK-NEXT:    ldrb r7, [r6, #-2]
 ; CHECK-NEXT:    smlabb r7, r7, r8, r2
 ; CHECK-NEXT:    str r7, [r4, #4]
 ; CHECK-NEXT:    adds r4, #16
diff --git a/llvm/test/CodeGen/Thumb2/mve-float16regloops.ll b/llvm/test/CodeGen/Thumb2/mve-float16regloops.ll
index e8ab7792b6dfc..1a53561388138 100644
--- a/llvm/test/CodeGen/Thumb2/mve-float16regloops.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-float16regloops.ll
@@ -1147,31 +1147,30 @@ define void @fir(%struct.arm_fir_instance_f32* nocapture readonly %S, half* noca
 ; CHECK-NEXT:  .LBB16_6: @ %for.body
 ; CHECK-NEXT:    @ Parent Loop BB16_4 Depth=1
 ; CHECK-NEXT:    @ => This Inner Loop Header: Depth=2
-; CHECK-NEXT:    ldrh r0, [r6]
+; CHECK-NEXT:    ldrh r0, [r6], #16
 ; CHECK-NEXT:    vldrw.u32 q1, [r5]
 ; CHECK-NEXT:    adds r1, r5, #2
 ; CHECK-NEXT:    vfma.f16 q0, q1, r0
 ; CHECK-NEXT:    vldrw.u32 q1, [r1]
-; CHECK-NEXT:    ldrh r0, [r6, #2]
+; CHECK-NEXT:    ldrh r0, [r6, #-14]
 ; CHECK-NEXT:    adds r1, r5, #6
 ; CHECK-NEXT:    vfma.f16 q0, q1, r0
-; CHECK-NEXT:    ldrh r0, [r6, #4]
+; CHECK-NEXT:    ldrh r0, [r6, #-12]
 ; CHECK-NEXT:    vldrw.u32 q1, [r5, #4]
 ; CHECK-NEXT:    vfma.f16 q0, q1, r0
 ; CHECK-NEXT:    vldrw.u32 q1, [r1]
-; CHECK-NEXT:    ldrh r0, [r6, #6]
+; CHECK-NEXT:    ldrh r0, [r6, #-10]
 ; CHECK-NEXT:    add.w r1, r5, #10
 ; CHECK-NEXT:    vfma.f16 q0, q1, r0
-; CHECK-NEXT:    ldrh r0, [r6, #8]
+; CHECK-NEXT:    ldrh r0, [r6, #-8]
 ; CHECK-NEXT:    vldrw.u32 q1, [r5, #8]
 ; CHECK-NEXT:    vfma.f16 q0, q1, r0
 ; CHECK-NEXT:    vldrw.u32 q1, [r1]
-; CHECK-NEXT:    ldrh r0, [r6, #10]
-; CHECK-NEXT:    ldrh r1, [r6, #14]
+; CHECK-NEXT:    ldrh r0, [r6, #-6]
+; CHECK-NEXT:    ldrh r1, [r6, #-2]
 ; CHECK-NEXT:    vfma.f16 q0, q1, r0
-; CHECK-NEXT:    ldrh r0, [r6, #12]
+; CHECK-NEXT:    ldrh r0, [r6, #-4]
 ; CHECK-NEXT:    vldrw.u32 q1, [r5, #12]
-; CHECK-NEXT:    adds r6, #16
 ; CHECK-NEXT:    vfma.f16 q0, q1, r0
 ; CHECK-NEXT:    add.w r0, r5, #14
 ; CHECK-NEXT:    vldrw.u32 q1, [r0]
diff --git a/llvm/test/CodeGen/Thumb2/mve-postinc-distribute.ll b/llvm/test/CodeGen/Thumb2/mve-postinc-distribute.ll
index fe9e7d197ca26..2db5bf59ecfae 100644
--- a/llvm/test/CodeGen/Thumb2/mve-postinc-distribute.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-postinc-distribute.ll
@@ -106,14 +106,12 @@ define void @arm_cmplx_dot_prod_q15(i16* nocapture readonly %pSrcA, i16* nocaptu
 ; CHECK-NEXT:    wls lr, lr, .LBB1_7
 ; CHECK-NEXT:  .LBB1_5: @ %while.body11
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    ldrsh.w r5, [r0, #2]
-; CHECK-NEXT:    ldrsh.w r6, [r1]
-; CHECK-NEXT:    ldrsh.w r9, [r0]
-; CHECK-NEXT:    adds r0, #4
-; CHECK-NEXT:    ldrsh.w r2, [r1, #2]
-; CHECK-NEXT:    adds r1, #4
-; CHECK-NEXT:    smlalbb r4, r11, r6, r5
+; CHECK-NEXT:    ldrsh r9, [r0], #4
+; CHECK-NEXT:    ldrsh r6, [r1], #4
+; CHECK-NEXT:    ldrsh r5, [r0, #-2]
+; CHECK-NEXT:    ldrsh r2, [r1, #-2]
 ; CHECK-NEXT:    smlalbb r12, r7, r6, r9
+; CHECK-NEXT:    smlalbb r4, r11, r6, r5
 ; CHECK-NEXT:    muls r5, r2, r5
 ; CHECK-NEXT:    smlalbb r4, r11, r2, r9
 ; CHECK-NEXT:    subs.w r12, r12, r5
diff --git a/llvm/test/CodeGen/Thumb2/postinc-distribute.mir b/llvm/test/CodeGen/Thumb2/postinc-distribute.mir
index af39cced110a0..d08ac4754c703 100644
--- a/llvm/test/CodeGen/Thumb2/postinc-distribute.mir
+++ b/llvm/test/CodeGen/Thumb2/postinc-distribute.mir
@@ -12,6 +12,8 @@
   define i32* @t2STRBi12(i32* %x, i32 %y) { unreachable }
 
   define i32* @storedadd(i32* %x, i32 %y) { unreachable }
+  define i32* @minsize2(i32* %x, i32 %y) minsize optsize { unreachable }
+  define i32* @minsize3(i32* %x, i32 %y) minsize optsize { unreachable }
 
 ...
 ---
@@ -57,9 +59,8 @@ body:             |
     ; CHECK-LABEL: name: t2LDRHi12
     ; CHECK: liveins: $r0
     ; CHECK: [[COPY:%[0-9]+]]:gprnopc = COPY $r0
-    ; CHECK: [[t2LDRHi12_:%[0-9]+]]:rgpr = t2LDRHi12 [[COPY]], 0, 14 /* CC::al */, $noreg :: (load 4)
-    ; CHECK: [[t2ADDri:%[0-9]+]]:rgpr = nuw t2ADDri [[COPY]], 32, 14 /* CC::al */, $noreg, $noreg
-    ; CHECK: $r0 = COPY [[t2ADDri]]
+    ; CHECK: [[t2LDRH_POST:%[0-9]+]]:rgpr, [[t2LDRH_POST1:%[0-9]+]]:rgpr = t2LDRH_POST [[COPY]], 32, 14 /* CC::al */, $noreg :: (load 4)
+    ; CHECK: $r0 = COPY [[t2LDRH_POST1]]
     ; CHECK: tBX_RET 14 /* CC::al */, $noreg, implicit $r0
     %0:gprnopc = COPY $r0
     %1:rgpr = t2LDRHi12 %0, 0, 14, $noreg :: (load 4, align 4)
@@ -84,9 +85,8 @@ body:             |
     ; CHECK-LABEL: name: t2LDRSHi12
     ; CHECK: liveins: $r0
     ; CHECK: [[COPY:%[0-9]+]]:gprnopc = COPY $r0
-    ; CHECK: [[t2LDRSHi12_:%[0-9]+]]:rgpr = t2LDRSHi12 [[COPY]], 0, 14 /* CC::al */, $noreg :: (load 4)
-    ; CHECK: [[t2ADDri:%[0-9]+]]:rgpr = nuw t2ADDri [[COPY]], 32, 14 /* CC::al */, $noreg, $noreg
-    ; CHECK: $r0 = COPY [[t2ADDri]]
+    ; CHECK: [[t2LDRSH_POST:%[0-9]+]]:rgpr, [[t2LDRSH_POST1:%[0-9]+]]:rgpr = t2LDRSH_POST [[COPY]], 32, 14 /* CC::al */, $noreg :: (load 4)
+    ; CHECK: $r0 = COPY [[t2LDRSH_POST1]]
     ; CHECK: tBX_RET 14 /* CC::al */, $noreg, implicit $r0
     %0:gprnopc = COPY $r0
     %1:rgpr = t2LDRSHi12 %0, 0, 14, $noreg :: (load 4, align 4)
@@ -111,9 +111,8 @@ body:             |
     ; CHECK-LABEL: name: t2LDRBi12
     ; CHECK: liveins: $r0
     ; CHECK: [[COPY:%[0-9]+]]:gprnopc = COPY $r0
-    ; CHECK: [[t2LDRBi12_:%[0-9]+]]:rgpr = t2LDRBi12 [[COPY]], 0, 14 /* CC::al */, $noreg :: (load 4)
-    ; CHECK: [[t2ADDri:%[0-9]+]]:rgpr = nuw t2ADDri [[COPY]], 32, 14 /* CC::al */, $noreg, $noreg
-    ; CHECK: $r0 = COPY [[t2ADDri]]
+    ; CHECK: [[t2LDRB_POST:%[0-9]+]]:rgpr, [[t2LDRB_POST1:%[0-9]+]]:rgpr = t2LDRB_POST [[COPY]], 32, 14 /* CC::al */, $noreg :: (load 4)
+    ; CHECK: $r0 = COPY [[t2LDRB_POST1]]
     ; CHECK: tBX_RET 14 /* CC::al */, $noreg, implicit $r0
     %0:gprnopc = COPY $r0
     %1:rgpr = t2LDRBi12 %0, 0, 14, $noreg :: (load 4, align 4)
@@ -138,9 +137,8 @@ body:             |
     ; CHECK-LABEL: name: t2LDRSBi12
     ; CHECK: liveins: $r0
     ; CHECK: [[COPY:%[0-9]+]]:gprnopc = COPY $r0
-    ; CHECK: [[t2LDRSBi12_:%[0-9]+]]:rgpr = t2LDRSBi12 [[COPY]], 0, 14 /* CC::al */, $noreg :: (load 4)
-    ; CHECK: [[t2ADDri:%[0-9]+]]:rgpr = nuw t2ADDri [[COPY]], 32, 14 /* CC::al */, $noreg, $noreg
-    ; CHECK: $r0 = COPY [[t2ADDri]]
+    ; CHECK: [[t2LDRSB_POST:%[0-9]+]]:rgpr, [[t2LDRSB_POST1:%[0-9]+]]:rgpr = t2LDRSB_POST [[COPY]], 32, 14 /* CC::al */, $noreg :: (load 4)
+    ; CHECK: $r0 = COPY [[t2LDRSB_POST1]]
     ; CHECK: tBX_RET 14 /* CC::al */, $noreg, implicit $r0
     %0:gprnopc = COPY $r0
     %1:rgpr = t2LDRSBi12 %0, 0, 14, $noreg :: (load 4, align 4)
@@ -197,9 +195,8 @@ body:             |
     ; CHECK: liveins: $r0, $r1
     ; CHECK: [[COPY:%[0-9]+]]:gprnopc = COPY $r0
     ; CHECK: [[COPY1:%[0-9]+]]:rgpr = COPY $r1
-    ; CHECK: t2STRHi12 [[COPY1]], [[COPY]], 0, 14 /* CC::al */, $noreg :: (store 4)
-    ; CHECK: [[t2ADDri:%[0-9]+]]:rgpr = nuw t2ADDri [[COPY]], 32, 14 /* CC::al */, $noreg, $noreg
-    ; CHECK: $r0 = COPY [[t2ADDri]]
+    ; CHECK: early-clobber %2:rgpr = t2STRH_POST [[COPY1]], [[COPY]], 32, 14 /* CC::al */, $noreg :: (store 4)
+    ; CHECK: $r0 = COPY %2
     ; CHECK: tBX_RET 14 /* CC::al */, $noreg, implicit $r0
     %0:gprnopc = COPY $r0
     %1:rgpr = COPY $r1
@@ -227,9 +224,8 @@ body:             |
     ; CHECK: liveins: $r0, $r1
     ; CHECK: [[COPY:%[0-9]+]]:gprnopc = COPY $r0
     ; CHECK: [[COPY1:%[0-9]+]]:rgpr = COPY $r1
-    ; CHECK: t2STRBi12 [[COPY1]], [[COPY]], 0, 14 /* CC::al */, $noreg :: (store 4)
-    ; CHECK: [[t2ADDri:%[0-9]+]]:rgpr = nuw t2ADDri [[COPY]], 32, 14 /* CC::al */, $noreg, $noreg
-    ; CHECK: $r0 = COPY [[t2ADDri]]
+    ; CHECK: early-clobber %2:rgpr = t2STRB_POST [[COPY1]], [[COPY]], 32, 14 /* CC::al */, $noreg :: (store 4)
+    ; CHECK: $r0 = COPY %2
     ; CHECK: tBX_RET 14 /* CC::al */, $noreg, implicit $r0
     %0:gprnopc = COPY $r0
     %1:rgpr = COPY $r1
@@ -265,3 +261,65 @@ body:             |
     tBX_RET 14, $noreg, implicit $r0
 
 ...
+---
+name:            minsize2
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: gprnopc, preferred-register: '' }
+  - { id: 1, class: rgpr, preferred-register: '' }
+  - { id: 2, class: rgpr, preferred-register: '' }
+  - { id: 3, class: rgpr, preferred-register: '' }
+liveins:
+  - { reg: '$r0', virtual-reg: '%0' }
+body:             |
+  bb.0:
+    liveins: $r0
+
+    ; CHECK-LABEL: name: minsize2
+    ; CHECK: liveins: $r0
+    ; CHECK: [[COPY:%[0-9]+]]:gprnopc = COPY $r0
+    ; CHECK: [[t2LDRB_POST:%[0-9]+]]:rgpr, [[t2LDRB_POST1:%[0-9]+]]:rgpr = t2LDRB_POST [[COPY]], 32, 14 /* CC::al */, $noreg :: (load 4)
+    ; CHECK: [[t2LDRBi8_:%[0-9]+]]:rgpr = t2LDRBi8 [[t2LDRB_POST1]], -30, 14 /* CC::al */, $noreg :: (load 4)
+    ; CHECK: $r0 = COPY [[t2LDRB_POST1]]
+    ; CHECK: tBX_RET 14 /* CC::al */, $noreg, implicit $r0
+    %0:gprnopc = COPY $r0
+    %1:rgpr = t2LDRBi12 %0, 0, 14, $noreg :: (load 4, align 4)
+    %3:rgpr = t2LDRBi12 %0, 2, 14, $noreg :: (load 4, align 4)
+    %2:rgpr = nuw t2ADDri %0, 32, 14, $noreg, $noreg
+    $r0 = COPY %2
+    tBX_RET 14, $noreg, implicit $r0
+
+...
+---
+name:            minsize3
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: gprnopc, preferred-register: '' }
+  - { id: 1, class: rgpr, preferred-register: '' }
+  - { id: 2, class: rgpr, preferred-register: '' }
+  - { id: 3, class: rgpr, preferred-register: '' }
+  - { id: 4, class: rgpr, preferred-register: '' }
+liveins:
+  - { reg: '$r0', virtual-reg: '%0' }
+body:             |
+  bb.0:
+    liveins: $r0
+
+    ; CHECK-LABEL: name: minsize3
+    ; CHECK: liveins: $r0
+    ; CHECK: [[COPY:%[0-9]+]]:gprnopc = COPY $r0
+    ; CHECK: [[t2LDRBi12_:%[0-9]+]]:rgpr = t2LDRBi12 [[COPY]], 0, 14 /* CC::al */, $noreg :: (load 4)
+    ; CHECK: [[t2LDRBi12_1:%[0-9]+]]:rgpr = t2LDRBi12 [[COPY]], 2, 14 /* CC::al */, $noreg :: (load 4)
+    ; CHECK: [[t2LDRBi12_2:%[0-9]+]]:rgpr = t2LDRBi12 [[COPY]], 4, 14 /* CC::al */, $noreg :: (load 4)
+    ; CHECK: [[t2ADDri:%[0-9]+]]:rgpr = nuw t2ADDri [[COPY]], 32, 14 /* CC::al */, $noreg, $noreg
+    ; CHECK: $r0 = COPY [[t2ADDri]]
+    ; CHECK: tBX_RET 14 /* CC::al */, $noreg, implicit $r0
+    %0:gprnopc = COPY $r0
+    %1:rgpr = t2LDRBi12 %0, 0, 14, $noreg :: (load 4, align 4)
+    %3:rgpr = t2LDRBi12 %0, 2, 14, $noreg :: (load 4, align 4)
+    %4:rgpr = t2LDRBi12 %0, 4, 14, $noreg :: (load 4, align 4)
+    %2:rgpr = nuw t2ADDri %0, 32, 14, $noreg, $noreg
+    $r0 = COPY %2
+    tBX_RET 14, $noreg, implicit $r0
+
+...

From fa30adecc766eb627a85f746b0f6f22b0eadbda8 Mon Sep 17 00:00:00 2001
From: Luofan Chen <clfbbn@gmail.com>
Date: Sat, 1 Aug 2020 21:27:16 +0800
Subject: [PATCH 0814/1035] [Attributor][NFC] Update description for the
 dependency graph

The word "dependency graph" is a bit misleading. When there is an
edge from node A to B (A -> B), it actually mean that B depends on
A and when the state of A is updated, B should also be updated. So
I update the comment to make the description clearer.

Reviewed By: jdoerfert

Differential Revision: https://reviews.llvm.org/D85065
---
 llvm/include/llvm/Transforms/IPO/Attributor.h | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/llvm/include/llvm/Transforms/IPO/Attributor.h b/llvm/include/llvm/Transforms/IPO/Attributor.h
index bd6ff03a31fad..418a007319679 100644
--- a/llvm/include/llvm/Transforms/IPO/Attributor.h
+++ b/llvm/include/llvm/Transforms/IPO/Attributor.h
@@ -155,8 +155,8 @@ struct AADepGraphNode {
   using DepTy = PointerIntPair<AADepGraphNode *, 1>;
 
 protected:
-  /// Set of dependency graph nodes which this one depends on.
-  /// The bit encodes if it is optional.
+  /// Set of dependency graph nodes which should be updated if this one
+  /// is updated. The bit encodes if it is optional.
   TinyPtrVector<DepTy> Deps;
 
   static AADepGraphNode *DepGetVal(DepTy &DT) { return DT.getPointer(); }
@@ -184,6 +184,11 @@ struct AADepGraphNode {
   friend struct AADepGraph;
 };
 
+/// The data structure for the dependency graph
+///
+/// Note that in this graph if there is an edge from A to B (A -> B),
+/// then it means that B depends on A, and when the state of A is
+/// updated, node B should also be updated
 struct AADepGraph {
   AADepGraph() {}
   ~AADepGraph() {}
@@ -197,7 +202,6 @@ struct AADepGraph {
   /// requires a single entry point, so we maintain a fake("synthetic") root
   /// node that depends on every node.
   AADepGraphNode SyntheticRoot;
-
   AADepGraphNode *GetEntryNode() { return &SyntheticRoot; }
 
   iterator begin() { return SyntheticRoot.child_begin(); }

From d620a6fe98f74d9b305a0d45d4c6804b0e46bf6c Mon Sep 17 00:00:00 2001
From: Sanjay Patel <spatel@rotateright.com>
Date: Sat, 1 Aug 2020 08:38:28 -0400
Subject: [PATCH 0815/1035] [VectorCombine] add tests for non-zero gep offsets;
 NFC

---
 .../test/Transforms/VectorCombine/X86/load.ll | 48 ++++++++++++++++++-
 1 file changed, 46 insertions(+), 2 deletions(-)

diff --git a/llvm/test/Transforms/VectorCombine/X86/load.ll b/llvm/test/Transforms/VectorCombine/X86/load.ll
index d560f671cfc03..4d8f3e5cd0ee5 100644
--- a/llvm/test/Transforms/VectorCombine/X86/load.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/load.ll
@@ -55,8 +55,8 @@ define float @matching_fp_vector(<4 x float>* align 16 dereferenceable(16) %p) {
   ret float %r
 }
 
-define float @matching_fp_vector_gep0(<4 x float>* align 16 dereferenceable(16) %p) {
-; CHECK-LABEL: @matching_fp_vector_gep0(
+define float @matching_fp_vector_gep00(<4 x float>* align 16 dereferenceable(16) %p) {
+; CHECK-LABEL: @matching_fp_vector_gep00(
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds <4 x float>, <4 x float>* [[P:%.*]], i64 0, i64 0
 ; CHECK-NEXT:    [[R:%.*]] = load float, float* [[GEP]], align 16
 ; CHECK-NEXT:    ret float [[R]]
@@ -66,6 +66,50 @@ define float @matching_fp_vector_gep0(<4 x float>* align 16 dereferenceable(16)
   ret float %r
 }
 
+define float @matching_fp_vector_gep01(<4 x float>* align 16 dereferenceable(20) %p) {
+; CHECK-LABEL: @matching_fp_vector_gep01(
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds <4 x float>, <4 x float>* [[P:%.*]], i64 0, i64 1
+; CHECK-NEXT:    [[R:%.*]] = load float, float* [[GEP]], align 4
+; CHECK-NEXT:    ret float [[R]]
+;
+  %gep = getelementptr inbounds <4 x float>, <4 x float>* %p, i64 0, i64 1
+  %r = load float, float* %gep, align 4
+  ret float %r
+}
+
+define float @matching_fp_vector_gep01_deref(<4 x float>* align 16 dereferenceable(19) %p) {
+; CHECK-LABEL: @matching_fp_vector_gep01_deref(
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds <4 x float>, <4 x float>* [[P:%.*]], i64 0, i64 1
+; CHECK-NEXT:    [[R:%.*]] = load float, float* [[GEP]], align 4
+; CHECK-NEXT:    ret float [[R]]
+;
+  %gep = getelementptr inbounds <4 x float>, <4 x float>* %p, i64 0, i64 1
+  %r = load float, float* %gep, align 4
+  ret float %r
+}
+
+define float @matching_fp_vector_gep10(<4 x float>* align 16 dereferenceable(32) %p) {
+; CHECK-LABEL: @matching_fp_vector_gep10(
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds <4 x float>, <4 x float>* [[P:%.*]], i64 1, i64 0
+; CHECK-NEXT:    [[R:%.*]] = load float, float* [[GEP]], align 16
+; CHECK-NEXT:    ret float [[R]]
+;
+  %gep = getelementptr inbounds <4 x float>, <4 x float>* %p, i64 1, i64 0
+  %r = load float, float* %gep, align 16
+  ret float %r
+}
+
+define float @matching_fp_vector_gep10_deref(<4 x float>* align 16 dereferenceable(31) %p) {
+; CHECK-LABEL: @matching_fp_vector_gep10_deref(
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds <4 x float>, <4 x float>* [[P:%.*]], i64 1, i64 0
+; CHECK-NEXT:    [[R:%.*]] = load float, float* [[GEP]], align 16
+; CHECK-NEXT:    ret float [[R]]
+;
+  %gep = getelementptr inbounds <4 x float>, <4 x float>* %p, i64 1, i64 0
+  %r = load float, float* %gep, align 16
+  ret float %r
+}
+
 define float @nonmatching_int_vector(<2 x i64>* align 16 dereferenceable(16) %p) {
 ; CHECK-LABEL: @nonmatching_int_vector(
 ; CHECK-NEXT:    [[BC:%.*]] = bitcast <2 x i64>* [[P:%.*]] to float*

From a9b06a2c14f9a38ba16165f0343faaa9ae713fec Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Sat, 1 Aug 2020 17:54:23 +0100
Subject: [PATCH 0816/1035] [LCSSA] Use IRBuilder for PHI creation.

Use IRBuilder instead PHINode::Create. This should not impact the
generated code, but IRBuilder provides a way to register callbacks for
inserted instructions, which is convenient for some users.

Reviewed By: lebedev.ri

Differential Revision: https://reviews.llvm.org/D85037
---
 llvm/include/llvm/Transforms/Utils/LoopUtils.h   |  2 +-
 llvm/lib/Transforms/Utils/LCSSA.cpp              | 16 +++++++++++-----
 .../Transforms/Utils/ScalarEvolutionExpander.cpp |  9 +++------
 3 files changed, 15 insertions(+), 12 deletions(-)

diff --git a/llvm/include/llvm/Transforms/Utils/LoopUtils.h b/llvm/include/llvm/Transforms/Utils/LoopUtils.h
index 60446bca53174..c6a8b27811ed1 100644
--- a/llvm/include/llvm/Transforms/Utils/LoopUtils.h
+++ b/llvm/include/llvm/Transforms/Utils/LoopUtils.h
@@ -76,7 +76,7 @@ bool formDedicatedExitBlocks(Loop *L, DominatorTree *DT, LoopInfo *LI,
 /// Returns true if any modifications are made.
 bool formLCSSAForInstructions(SmallVectorImpl<Instruction *> &Worklist,
                               const DominatorTree &DT, const LoopInfo &LI,
-                              ScalarEvolution *SE);
+                              ScalarEvolution *SE, IRBuilderBase &Builder);
 
 /// Put loop into LCSSA form.
 ///
diff --git a/llvm/lib/Transforms/Utils/LCSSA.cpp b/llvm/lib/Transforms/Utils/LCSSA.cpp
index b1a1c564d2171..9c606251ae0f8 100644
--- a/llvm/lib/Transforms/Utils/LCSSA.cpp
+++ b/llvm/lib/Transforms/Utils/LCSSA.cpp
@@ -40,6 +40,7 @@
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/Function.h"
+#include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/PredIteratorCache.h"
@@ -77,12 +78,15 @@ static bool isExitBlock(BasicBlock *BB,
 /// rewrite the uses.
 bool llvm::formLCSSAForInstructions(SmallVectorImpl<Instruction *> &Worklist,
                                     const DominatorTree &DT, const LoopInfo &LI,
-                                    ScalarEvolution *SE) {
+                                    ScalarEvolution *SE,
+                                    IRBuilderBase &Builder) {
   SmallVector<Use *, 16> UsesToRewrite;
   SmallSetVector<PHINode *, 16> PHIsToRemove;
   PredIteratorCache PredCache;
   bool Changed = false;
 
+  IRBuilderBase::InsertPointGuard InsertPtGuard(Builder);
+
   // Cache the Loop ExitBlocks across this loop.  We expect to get a lot of
   // instructions within the same loops, computing the exit blocks is
   // expensive, and we're not mutating the loop structure.
@@ -151,9 +155,9 @@ bool llvm::formLCSSAForInstructions(SmallVectorImpl<Instruction *> &Worklist,
       // If we already inserted something for this BB, don't reprocess it.
       if (SSAUpdate.HasValueForBlock(ExitBB))
         continue;
-
-      PHINode *PN = PHINode::Create(I->getType(), PredCache.size(ExitBB),
-                                    I->getName() + ".lcssa", &ExitBB->front());
+      Builder.SetInsertPoint(&ExitBB->front());
+      PHINode *PN = Builder.CreatePHI(I->getType(), PredCache.size(ExitBB),
+                                      I->getName() + ".lcssa");
       // Get the debug location from the original instruction.
       PN->setDebugLoc(I->getDebugLoc());
       // Add inputs from inside the loop for this PHI.
@@ -369,7 +373,9 @@ bool llvm::formLCSSA(Loop &L, const DominatorTree &DT, const LoopInfo *LI,
       Worklist.push_back(&I);
     }
   }
-  Changed = formLCSSAForInstructions(Worklist, DT, *LI, SE);
+
+  IRBuilder<> Builder(L.getHeader()->getContext());
+  Changed = formLCSSAForInstructions(Worklist, DT, *LI, SE, Builder);
 
   // If we modified the code, remove any caches about the loop from SCEV to
   // avoid dangling entries.
diff --git a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
index 555da5df65e15..a8302b7ccfc1a 100644
--- a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
+++ b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
@@ -1951,11 +1951,8 @@ void SCEVExpander::rememberInstruction(Value *I) {
     // a defining loop. Fix LCSSA from for each operand of the new instruction,
     // if required.
     for (unsigned OpIdx = 0, OpEnd = Inst->getNumOperands(); OpIdx != OpEnd;
-         OpIdx++) {
-      auto *V = fixupLCSSAFormFor(Inst, OpIdx);
-      if (V != I)
-        DoInsert(V);
-    }
+         OpIdx++)
+      fixupLCSSAFormFor(Inst, OpIdx);
   }
 }
 
@@ -2540,7 +2537,7 @@ Value *SCEVExpander::fixupLCSSAFormFor(Instruction *User, unsigned OpIdx) {
     return OpV;
 
   ToUpdate.push_back(OpI);
-  formLCSSAForInstructions(ToUpdate, SE.DT, SE.LI, &SE);
+  formLCSSAForInstructions(ToUpdate, SE.DT, SE.LI, &SE, Builder);
   return User->getOperand(OpIdx);
 }
 

From bb13c34c3aa100006461c972319abfef0af70603 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Sat, 1 Aug 2020 14:54:21 +0100
Subject: [PATCH 0817/1035] [X86][AVX] Ensure we only combine to
 PSHUFLW/PSHUFHW on supporting targets

Noticed while investigating combining from concatenated shuffle vectors, we weren't checking that PSHUFLW/PSHUFHW was legal - we were depending on lowering splitting to subvectors.
---
 llvm/lib/Target/X86/X86ISelLowering.cpp | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index d628cdfc1fdf1..b89502dc70204 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -34146,7 +34146,10 @@ static bool matchUnaryPermuteShuffle(MVT MaskVT, ArrayRef<int> Mask,
   }
 
   // Handle PSHUFLW/PSHUFHW vXi16 repeated patterns.
-  if (!ContainsZeros && AllowIntDomain && MaskScalarSizeInBits == 16) {
+  if (!ContainsZeros && AllowIntDomain && MaskScalarSizeInBits == 16 &&
+      ((MaskVT.is128BitVector() && Subtarget.hasSSE2()) ||
+       (MaskVT.is256BitVector() && Subtarget.hasAVX2()) ||
+       (MaskVT.is512BitVector() && Subtarget.hasBWI()))) {
     SmallVector<int, 4> RepeatedMask;
     if (is128BitLaneRepeatedShuffleMask(MaskEltVT, Mask, RepeatedMask)) {
       ArrayRef<int> LoMask(RepeatedMask.data() + 0, 4);

From 8697d6cfa8947c4033dfe1d2ac708482d75a28d4 Mon Sep 17 00:00:00 2001
From: zoecarver <z.zoelec2@gmail.com>
Date: Sat, 1 Aug 2020 12:06:31 -0700
Subject: [PATCH 0818/1035] [libcxx] Add compatible with constraint tests for
 some shared_ptr constructors.

Add shared_ptr tests where the element type and pointer type aren't 'convertible' but are 'compatible'.

Responding to a comment from D81414.

Differential Revision: https://reviews.llvm.org/D81532
---
 .../pointer_deleter.pass.cpp                           |  9 +++++++++
 .../pointer_deleter_allocator.pass.cpp                 | 10 ++++++++++
 2 files changed, 19 insertions(+)

diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/util.smartptr.shared.const/pointer_deleter.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/util.smartptr.shared.const/pointer_deleter.pass.cpp
index bb41fa89be372..e736eb57e0569 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/util.smartptr.shared.const/pointer_deleter.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/util.smartptr.shared.const/pointer_deleter.pass.cpp
@@ -28,6 +28,9 @@ struct A
 
 int A::count = 0;
 
+struct Base { };
+struct Derived : Base { };
+
 int main(int, char**)
 {
     {
@@ -46,5 +49,11 @@ int main(int, char**)
     assert(test_deleter<A>::count == 0);
     assert(test_deleter<A>::dealloc_count == 1);
 
+    {
+        // Make sure that we can construct a shared_ptr where the element type and pointer type
+        // aren't "convertible" but are "compatible".
+        static_assert(!std::is_constructible<std::shared_ptr<Derived[4]>, Base[4], test_deleter<Derived[4]> >::value, "");
+    }
+
   return 0;
 }
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/util.smartptr.shared.const/pointer_deleter_allocator.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/util.smartptr.shared.const/pointer_deleter_allocator.pass.cpp
index e17ae6f34312d..af9bfe9ab057b 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/util.smartptr.shared.const/pointer_deleter_allocator.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/util.smartptr.shared.const/pointer_deleter_allocator.pass.cpp
@@ -28,6 +28,8 @@ struct A
 
 int A::count = 0;
 
+struct Base { };
+struct Derived : Base { };
 
 int main(int, char**)
 {
@@ -87,5 +89,13 @@ int main(int, char**)
     assert(test_deleter<A>::dealloc_count == 1);
 #endif
 
+    {
+        // Make sure that we can construct a shared_ptr where the element type and pointer type
+        // aren't "convertible" but are "compatible".
+        static_assert(!std::is_constructible<std::shared_ptr<Derived[4]>,
+                                             Base[4], test_deleter<Derived[4]>,
+                                             test_allocator<Derived[4]> >::value, "");
+    }
+
   return 0;
 }

From 82a5c848e7f531ee636f643450072059397ac90c Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Sat, 1 Aug 2020 20:34:39 +0100
Subject: [PATCH 0819/1035] [X86][AVX512] Fold concat(and(x,y),and(z,w)) ->
 and(concat(x,z),concat(y,w)) for 512-bit vectors

Helps vpternlog folding on non-AVX512BW targets
---
 llvm/lib/Target/X86/X86ISelLowering.cpp       |  19 ++
 .../test/CodeGen/X86/avx512-insert-extract.ll | 246 +++++++++---------
 llvm/test/CodeGen/X86/vector-bitreverse.ll    |  56 ++--
 llvm/test/CodeGen/X86/vector-fshl-512.ll      | 106 ++++----
 llvm/test/CodeGen/X86/vector-fshl-rot-512.ll  |  70 ++---
 llvm/test/CodeGen/X86/vector-fshr-512.ll      | 134 +++++-----
 llvm/test/CodeGen/X86/vector-fshr-rot-512.ll  |  74 +++---
 llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll |  54 ++--
 llvm/test/CodeGen/X86/vector-rotate-512.ll    | 134 ++++------
 .../test/CodeGen/X86/vector-shift-lshr-512.ll |  22 +-
 llvm/test/CodeGen/X86/vector-shift-shl-512.ll |  20 +-
 11 files changed, 431 insertions(+), 504 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index b89502dc70204..c135b91620724 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -48090,6 +48090,25 @@ static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT,
                            Op0.getOperand(1));
       }
       break;
+    case ISD::AND:
+    case ISD::OR:
+    case ISD::XOR:
+    case X86ISD::ANDNP:
+      // TODO: Add 256-bit support.
+      if (!IsSplat && VT.is512BitVector()) {
+        SmallVector<SDValue, 2> LHS, RHS;
+        for (unsigned i = 0; i != NumOps; ++i) {
+          LHS.push_back(Ops[i].getOperand(0));
+          RHS.push_back(Ops[i].getOperand(1));
+        }
+        MVT SrcVT = Op0.getOperand(0).getSimpleValueType();
+        SrcVT = MVT::getVectorVT(SrcVT.getScalarType(),
+                                 NumOps * SrcVT.getVectorNumElements());
+        return DAG.getNode(Op0.getOpcode(), DL, VT,
+                           DAG.getNode(ISD::CONCAT_VECTORS, DL, SrcVT, LHS),
+                           DAG.getNode(ISD::CONCAT_VECTORS, DL, SrcVT, RHS));
+      }
+      break;
     case X86ISD::PACKSS:
     case X86ISD::PACKUS:
       if (!IsSplat && NumOps == 2 && VT.is256BitVector() &&
diff --git a/llvm/test/CodeGen/X86/avx512-insert-extract.ll b/llvm/test/CodeGen/X86/avx512-insert-extract.ll
index f6ffd6419c13a..41bdaf21baa38 100644
--- a/llvm/test/CodeGen/X86/avx512-insert-extract.ll
+++ b/llvm/test/CodeGen/X86/avx512-insert-extract.ll
@@ -1692,16 +1692,15 @@ define i64 @test_insertelement_variable_v64i1(<64 x i8> %a, i8 %b, i32 %index) {
 ; KNL-NEXT:    andq $-64, %rsp
 ; KNL-NEXT:    subq $128, %rsp
 ; KNL-NEXT:    ## kill: def $esi killed $esi def $rsi
-; KNL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; KNL-NEXT:    vpcmpeqb %ymm1, %ymm0, %ymm2
-; KNL-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
-; KNL-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
-; KNL-NEXT:    vpcmpeqb %ymm1, %ymm0, %ymm0
+; KNL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; KNL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; KNL-NEXT:    vpcmpeqb %ymm2, %ymm1, %ymm1
+; KNL-NEXT:    vpcmpeqb %ymm2, %ymm0, %ymm0
+; KNL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; KNL-NEXT:    vpternlogq $15, %zmm0, %zmm0, %zmm0
 ; KNL-NEXT:    andl $63, %esi
 ; KNL-NEXT:    testb %dil, %dil
-; KNL-NEXT:    vmovdqa %ymm0, {{[0-9]+}}(%rsp)
-; KNL-NEXT:    vmovdqa %ymm2, (%rsp)
+; KNL-NEXT:    vmovdqa64 %zmm0, (%rsp)
 ; KNL-NEXT:    setne (%rsp,%rsi)
 ; KNL-NEXT:    vpmovsxbd (%rsp), %zmm0
 ; KNL-NEXT:    vpslld $31, %zmm0, %zmm0
@@ -1772,116 +1771,115 @@ define i96 @test_insertelement_variable_v96i1(<96 x i8> %a, i8 %b, i32 %index) {
 ; KNL-NEXT:    subq $192, %rsp
 ; KNL-NEXT:    movl 744(%rbp), %eax
 ; KNL-NEXT:    andl $127, %eax
-; KNL-NEXT:    vmovd %edi, %xmm0
-; KNL-NEXT:    vpinsrb $1, %esi, %xmm0, %xmm0
-; KNL-NEXT:    vpinsrb $2, %edx, %xmm0, %xmm0
-; KNL-NEXT:    vpinsrb $3, %ecx, %xmm0, %xmm0
-; KNL-NEXT:    vpinsrb $4, %r8d, %xmm0, %xmm0
-; KNL-NEXT:    vpinsrb $5, %r9d, %xmm0, %xmm0
-; KNL-NEXT:    vpinsrb $6, 16(%rbp), %xmm0, %xmm0
-; KNL-NEXT:    vpinsrb $7, 24(%rbp), %xmm0, %xmm0
-; KNL-NEXT:    vpinsrb $8, 32(%rbp), %xmm0, %xmm0
-; KNL-NEXT:    vpinsrb $9, 40(%rbp), %xmm0, %xmm0
-; KNL-NEXT:    vpinsrb $10, 48(%rbp), %xmm0, %xmm0
-; KNL-NEXT:    vpinsrb $11, 56(%rbp), %xmm0, %xmm0
-; KNL-NEXT:    vpinsrb $12, 64(%rbp), %xmm0, %xmm0
-; KNL-NEXT:    vpinsrb $13, 72(%rbp), %xmm0, %xmm0
-; KNL-NEXT:    vpinsrb $14, 80(%rbp), %xmm0, %xmm0
-; KNL-NEXT:    vpinsrb $15, 88(%rbp), %xmm0, %xmm0
+; KNL-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; KNL-NEXT:    vpinsrb $1, 232(%rbp), %xmm0, %xmm0
+; KNL-NEXT:    vpinsrb $2, 240(%rbp), %xmm0, %xmm0
+; KNL-NEXT:    vpinsrb $3, 248(%rbp), %xmm0, %xmm0
+; KNL-NEXT:    vpinsrb $4, 256(%rbp), %xmm0, %xmm0
+; KNL-NEXT:    vpinsrb $5, 264(%rbp), %xmm0, %xmm0
+; KNL-NEXT:    vpinsrb $6, 272(%rbp), %xmm0, %xmm0
+; KNL-NEXT:    vpinsrb $7, 280(%rbp), %xmm0, %xmm0
+; KNL-NEXT:    vpinsrb $8, 288(%rbp), %xmm0, %xmm0
+; KNL-NEXT:    vpinsrb $9, 296(%rbp), %xmm0, %xmm0
+; KNL-NEXT:    vpinsrb $10, 304(%rbp), %xmm0, %xmm0
+; KNL-NEXT:    vpinsrb $11, 312(%rbp), %xmm0, %xmm0
+; KNL-NEXT:    vpinsrb $12, 320(%rbp), %xmm0, %xmm0
+; KNL-NEXT:    vpinsrb $13, 328(%rbp), %xmm0, %xmm0
+; KNL-NEXT:    vpinsrb $14, 336(%rbp), %xmm0, %xmm0
+; KNL-NEXT:    vpinsrb $15, 344(%rbp), %xmm0, %xmm0
 ; KNL-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; KNL-NEXT:    vpinsrb $1, 104(%rbp), %xmm1, %xmm1
-; KNL-NEXT:    vpinsrb $2, 112(%rbp), %xmm1, %xmm1
-; KNL-NEXT:    vpinsrb $3, 120(%rbp), %xmm1, %xmm1
-; KNL-NEXT:    vpinsrb $4, 128(%rbp), %xmm1, %xmm1
-; KNL-NEXT:    vpinsrb $5, 136(%rbp), %xmm1, %xmm1
-; KNL-NEXT:    vpinsrb $6, 144(%rbp), %xmm1, %xmm1
-; KNL-NEXT:    vpinsrb $7, 152(%rbp), %xmm1, %xmm1
-; KNL-NEXT:    vpinsrb $8, 160(%rbp), %xmm1, %xmm1
-; KNL-NEXT:    vpinsrb $9, 168(%rbp), %xmm1, %xmm1
-; KNL-NEXT:    vpinsrb $10, 176(%rbp), %xmm1, %xmm1
-; KNL-NEXT:    vpinsrb $11, 184(%rbp), %xmm1, %xmm1
-; KNL-NEXT:    vpinsrb $12, 192(%rbp), %xmm1, %xmm1
-; KNL-NEXT:    vpinsrb $13, 200(%rbp), %xmm1, %xmm1
-; KNL-NEXT:    vpinsrb $14, 208(%rbp), %xmm1, %xmm1
-; KNL-NEXT:    vpinsrb $15, 216(%rbp), %xmm1, %xmm1
-; KNL-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; KNL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; KNL-NEXT:    vpcmpeqb %ymm1, %ymm0, %ymm0
-; KNL-NEXT:    vpternlogq $15, %zmm0, %zmm0, %zmm0
-; KNL-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; KNL-NEXT:    vpinsrb $1, 232(%rbp), %xmm2, %xmm2
-; KNL-NEXT:    vpinsrb $2, 240(%rbp), %xmm2, %xmm2
-; KNL-NEXT:    vpinsrb $3, 248(%rbp), %xmm2, %xmm2
-; KNL-NEXT:    vpinsrb $4, 256(%rbp), %xmm2, %xmm2
-; KNL-NEXT:    vpinsrb $5, 264(%rbp), %xmm2, %xmm2
-; KNL-NEXT:    vpinsrb $6, 272(%rbp), %xmm2, %xmm2
-; KNL-NEXT:    vpinsrb $7, 280(%rbp), %xmm2, %xmm2
-; KNL-NEXT:    vpinsrb $8, 288(%rbp), %xmm2, %xmm2
-; KNL-NEXT:    vpinsrb $9, 296(%rbp), %xmm2, %xmm2
-; KNL-NEXT:    vpinsrb $10, 304(%rbp), %xmm2, %xmm2
-; KNL-NEXT:    vpinsrb $11, 312(%rbp), %xmm2, %xmm2
-; KNL-NEXT:    vpinsrb $12, 320(%rbp), %xmm2, %xmm2
-; KNL-NEXT:    vpinsrb $13, 328(%rbp), %xmm2, %xmm2
-; KNL-NEXT:    vpinsrb $14, 336(%rbp), %xmm2, %xmm2
-; KNL-NEXT:    vpinsrb $15, 344(%rbp), %xmm2, %xmm2
+; KNL-NEXT:    vpinsrb $1, 360(%rbp), %xmm1, %xmm1
+; KNL-NEXT:    vpinsrb $2, 368(%rbp), %xmm1, %xmm1
+; KNL-NEXT:    vpinsrb $3, 376(%rbp), %xmm1, %xmm1
+; KNL-NEXT:    vpinsrb $4, 384(%rbp), %xmm1, %xmm1
+; KNL-NEXT:    vpinsrb $5, 392(%rbp), %xmm1, %xmm1
+; KNL-NEXT:    vpinsrb $6, 400(%rbp), %xmm1, %xmm1
+; KNL-NEXT:    vpinsrb $7, 408(%rbp), %xmm1, %xmm1
+; KNL-NEXT:    vpinsrb $8, 416(%rbp), %xmm1, %xmm1
+; KNL-NEXT:    vpinsrb $9, 424(%rbp), %xmm1, %xmm1
+; KNL-NEXT:    vpinsrb $10, 432(%rbp), %xmm1, %xmm1
+; KNL-NEXT:    vpinsrb $11, 440(%rbp), %xmm1, %xmm1
+; KNL-NEXT:    vpinsrb $12, 448(%rbp), %xmm1, %xmm1
+; KNL-NEXT:    vpinsrb $13, 456(%rbp), %xmm1, %xmm1
+; KNL-NEXT:    vpinsrb $14, 464(%rbp), %xmm1, %xmm1
+; KNL-NEXT:    vpinsrb $15, 472(%rbp), %xmm1, %xmm1
+; KNL-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
+; KNL-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; KNL-NEXT:    vpcmpeqb %ymm0, %ymm1, %ymm1
+; KNL-NEXT:    vmovd %edi, %xmm2
+; KNL-NEXT:    vpinsrb $1, %esi, %xmm2, %xmm2
+; KNL-NEXT:    vpinsrb $2, %edx, %xmm2, %xmm2
+; KNL-NEXT:    vpinsrb $3, %ecx, %xmm2, %xmm2
+; KNL-NEXT:    vpinsrb $4, %r8d, %xmm2, %xmm2
+; KNL-NEXT:    vpinsrb $5, %r9d, %xmm2, %xmm2
+; KNL-NEXT:    vpinsrb $6, 16(%rbp), %xmm2, %xmm2
+; KNL-NEXT:    vpinsrb $7, 24(%rbp), %xmm2, %xmm2
+; KNL-NEXT:    vpinsrb $8, 32(%rbp), %xmm2, %xmm2
+; KNL-NEXT:    vpinsrb $9, 40(%rbp), %xmm2, %xmm2
+; KNL-NEXT:    vpinsrb $10, 48(%rbp), %xmm2, %xmm2
+; KNL-NEXT:    vpinsrb $11, 56(%rbp), %xmm2, %xmm2
+; KNL-NEXT:    vpinsrb $12, 64(%rbp), %xmm2, %xmm2
+; KNL-NEXT:    vpinsrb $13, 72(%rbp), %xmm2, %xmm2
+; KNL-NEXT:    vpinsrb $14, 80(%rbp), %xmm2, %xmm2
+; KNL-NEXT:    vpinsrb $15, 88(%rbp), %xmm2, %xmm2
 ; KNL-NEXT:    vmovd {{.*#+}} xmm3 = mem[0],zero,zero,zero
-; KNL-NEXT:    vpinsrb $1, 360(%rbp), %xmm3, %xmm3
-; KNL-NEXT:    vpinsrb $2, 368(%rbp), %xmm3, %xmm3
-; KNL-NEXT:    vpinsrb $3, 376(%rbp), %xmm3, %xmm3
-; KNL-NEXT:    vpinsrb $4, 384(%rbp), %xmm3, %xmm3
-; KNL-NEXT:    vpinsrb $5, 392(%rbp), %xmm3, %xmm3
-; KNL-NEXT:    vpinsrb $6, 400(%rbp), %xmm3, %xmm3
-; KNL-NEXT:    vpinsrb $7, 408(%rbp), %xmm3, %xmm3
-; KNL-NEXT:    vpinsrb $8, 416(%rbp), %xmm3, %xmm3
-; KNL-NEXT:    vpinsrb $9, 424(%rbp), %xmm3, %xmm3
-; KNL-NEXT:    vpinsrb $10, 432(%rbp), %xmm3, %xmm3
-; KNL-NEXT:    vpinsrb $11, 440(%rbp), %xmm3, %xmm3
-; KNL-NEXT:    vpinsrb $12, 448(%rbp), %xmm3, %xmm3
-; KNL-NEXT:    vpinsrb $13, 456(%rbp), %xmm3, %xmm3
-; KNL-NEXT:    vpinsrb $14, 464(%rbp), %xmm3, %xmm3
-; KNL-NEXT:    vpinsrb $15, 472(%rbp), %xmm3, %xmm3
+; KNL-NEXT:    vpinsrb $1, 104(%rbp), %xmm3, %xmm3
+; KNL-NEXT:    vpinsrb $2, 112(%rbp), %xmm3, %xmm3
+; KNL-NEXT:    vpinsrb $3, 120(%rbp), %xmm3, %xmm3
+; KNL-NEXT:    vpinsrb $4, 128(%rbp), %xmm3, %xmm3
+; KNL-NEXT:    vpinsrb $5, 136(%rbp), %xmm3, %xmm3
+; KNL-NEXT:    vpinsrb $6, 144(%rbp), %xmm3, %xmm3
+; KNL-NEXT:    vpinsrb $7, 152(%rbp), %xmm3, %xmm3
+; KNL-NEXT:    vpinsrb $8, 160(%rbp), %xmm3, %xmm3
+; KNL-NEXT:    vpinsrb $9, 168(%rbp), %xmm3, %xmm3
+; KNL-NEXT:    vpinsrb $10, 176(%rbp), %xmm3, %xmm3
+; KNL-NEXT:    vpinsrb $11, 184(%rbp), %xmm3, %xmm3
+; KNL-NEXT:    vpinsrb $12, 192(%rbp), %xmm3, %xmm3
+; KNL-NEXT:    vpinsrb $13, 200(%rbp), %xmm3, %xmm3
+; KNL-NEXT:    vpinsrb $14, 208(%rbp), %xmm3, %xmm3
+; KNL-NEXT:    vpinsrb $15, 216(%rbp), %xmm3, %xmm3
 ; KNL-NEXT:    vinserti128 $1, %xmm3, %ymm2, %ymm2
-; KNL-NEXT:    vpcmpeqb %ymm1, %ymm2, %ymm2
-; KNL-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
-; KNL-NEXT:    vmovd {{.*#+}} xmm3 = mem[0],zero,zero,zero
-; KNL-NEXT:    vpinsrb $1, 488(%rbp), %xmm3, %xmm3
-; KNL-NEXT:    vpinsrb $2, 496(%rbp), %xmm3, %xmm3
-; KNL-NEXT:    vpinsrb $3, 504(%rbp), %xmm3, %xmm3
-; KNL-NEXT:    vpinsrb $4, 512(%rbp), %xmm3, %xmm3
-; KNL-NEXT:    vpinsrb $5, 520(%rbp), %xmm3, %xmm3
-; KNL-NEXT:    vpinsrb $6, 528(%rbp), %xmm3, %xmm3
-; KNL-NEXT:    vpinsrb $7, 536(%rbp), %xmm3, %xmm3
-; KNL-NEXT:    vpinsrb $8, 544(%rbp), %xmm3, %xmm3
-; KNL-NEXT:    vpinsrb $9, 552(%rbp), %xmm3, %xmm3
-; KNL-NEXT:    vpinsrb $10, 560(%rbp), %xmm3, %xmm3
-; KNL-NEXT:    vpinsrb $11, 568(%rbp), %xmm3, %xmm3
-; KNL-NEXT:    vpinsrb $12, 576(%rbp), %xmm3, %xmm3
-; KNL-NEXT:    vpinsrb $13, 584(%rbp), %xmm3, %xmm3
-; KNL-NEXT:    vpinsrb $14, 592(%rbp), %xmm3, %xmm3
-; KNL-NEXT:    vpinsrb $15, 600(%rbp), %xmm3, %xmm3
-; KNL-NEXT:    vmovd {{.*#+}} xmm4 = mem[0],zero,zero,zero
-; KNL-NEXT:    vpinsrb $1, 616(%rbp), %xmm4, %xmm4
-; KNL-NEXT:    vpinsrb $2, 624(%rbp), %xmm4, %xmm4
-; KNL-NEXT:    vpinsrb $3, 632(%rbp), %xmm4, %xmm4
-; KNL-NEXT:    vpinsrb $4, 640(%rbp), %xmm4, %xmm4
-; KNL-NEXT:    vpinsrb $5, 648(%rbp), %xmm4, %xmm4
-; KNL-NEXT:    vpinsrb $6, 656(%rbp), %xmm4, %xmm4
-; KNL-NEXT:    vpinsrb $7, 664(%rbp), %xmm4, %xmm4
-; KNL-NEXT:    vpinsrb $8, 672(%rbp), %xmm4, %xmm4
-; KNL-NEXT:    vpinsrb $9, 680(%rbp), %xmm4, %xmm4
-; KNL-NEXT:    vpinsrb $10, 688(%rbp), %xmm4, %xmm4
-; KNL-NEXT:    vpinsrb $11, 696(%rbp), %xmm4, %xmm4
-; KNL-NEXT:    vpinsrb $12, 704(%rbp), %xmm4, %xmm4
-; KNL-NEXT:    vpinsrb $13, 712(%rbp), %xmm4, %xmm4
-; KNL-NEXT:    vpinsrb $14, 720(%rbp), %xmm4, %xmm4
-; KNL-NEXT:    vpinsrb $15, 728(%rbp), %xmm4, %xmm4
-; KNL-NEXT:    vinserti128 $1, %xmm4, %ymm3, %ymm3
-; KNL-NEXT:    vpcmpeqb %ymm1, %ymm3, %ymm1
+; KNL-NEXT:    vpcmpeqb %ymm0, %ymm2, %ymm2
+; KNL-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm1
 ; KNL-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
+; KNL-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; KNL-NEXT:    vpinsrb $1, 488(%rbp), %xmm2, %xmm2
+; KNL-NEXT:    vpinsrb $2, 496(%rbp), %xmm2, %xmm2
+; KNL-NEXT:    vpinsrb $3, 504(%rbp), %xmm2, %xmm2
+; KNL-NEXT:    vpinsrb $4, 512(%rbp), %xmm2, %xmm2
+; KNL-NEXT:    vpinsrb $5, 520(%rbp), %xmm2, %xmm2
+; KNL-NEXT:    vpinsrb $6, 528(%rbp), %xmm2, %xmm2
+; KNL-NEXT:    vpinsrb $7, 536(%rbp), %xmm2, %xmm2
+; KNL-NEXT:    vpinsrb $8, 544(%rbp), %xmm2, %xmm2
+; KNL-NEXT:    vpinsrb $9, 552(%rbp), %xmm2, %xmm2
+; KNL-NEXT:    vpinsrb $10, 560(%rbp), %xmm2, %xmm2
+; KNL-NEXT:    vpinsrb $11, 568(%rbp), %xmm2, %xmm2
+; KNL-NEXT:    vpinsrb $12, 576(%rbp), %xmm2, %xmm2
+; KNL-NEXT:    vpinsrb $13, 584(%rbp), %xmm2, %xmm2
+; KNL-NEXT:    vpinsrb $14, 592(%rbp), %xmm2, %xmm2
+; KNL-NEXT:    vpinsrb $15, 600(%rbp), %xmm2, %xmm2
+; KNL-NEXT:    vmovd {{.*#+}} xmm3 = mem[0],zero,zero,zero
+; KNL-NEXT:    vpinsrb $1, 616(%rbp), %xmm3, %xmm3
+; KNL-NEXT:    vpinsrb $2, 624(%rbp), %xmm3, %xmm3
+; KNL-NEXT:    vpinsrb $3, 632(%rbp), %xmm3, %xmm3
+; KNL-NEXT:    vpinsrb $4, 640(%rbp), %xmm3, %xmm3
+; KNL-NEXT:    vpinsrb $5, 648(%rbp), %xmm3, %xmm3
+; KNL-NEXT:    vpinsrb $6, 656(%rbp), %xmm3, %xmm3
+; KNL-NEXT:    vpinsrb $7, 664(%rbp), %xmm3, %xmm3
+; KNL-NEXT:    vpinsrb $8, 672(%rbp), %xmm3, %xmm3
+; KNL-NEXT:    vpinsrb $9, 680(%rbp), %xmm3, %xmm3
+; KNL-NEXT:    vpinsrb $10, 688(%rbp), %xmm3, %xmm3
+; KNL-NEXT:    vpinsrb $11, 696(%rbp), %xmm3, %xmm3
+; KNL-NEXT:    vpinsrb $12, 704(%rbp), %xmm3, %xmm3
+; KNL-NEXT:    vpinsrb $13, 712(%rbp), %xmm3, %xmm3
+; KNL-NEXT:    vpinsrb $14, 720(%rbp), %xmm3, %xmm3
+; KNL-NEXT:    vpinsrb $15, 728(%rbp), %xmm3, %xmm3
+; KNL-NEXT:    vinserti128 $1, %xmm3, %ymm2, %ymm2
+; KNL-NEXT:    vpcmpeqb %ymm0, %ymm2, %ymm0
+; KNL-NEXT:    vpternlogq $15, %zmm0, %zmm0, %zmm0
 ; KNL-NEXT:    cmpb $0, 736(%rbp)
-; KNL-NEXT:    vmovdqa %ymm1, {{[0-9]+}}(%rsp)
-; KNL-NEXT:    vmovdqa %ymm2, {{[0-9]+}}(%rsp)
-; KNL-NEXT:    vmovdqa %ymm0, (%rsp)
+; KNL-NEXT:    vmovdqa %ymm0, {{[0-9]+}}(%rsp)
+; KNL-NEXT:    vmovdqa64 %zmm1, (%rsp)
 ; KNL-NEXT:    setne (%rsp,%rax)
 ; KNL-NEXT:    vpmovsxbd (%rsp), %zmm0
 ; KNL-NEXT:    vpslld $31, %zmm0, %zmm0
@@ -2079,23 +2077,21 @@ define i128 @test_insertelement_variable_v128i1(<128 x i8> %a, i8 %b, i32 %index
 ; KNL-NEXT:    andq $-64, %rsp
 ; KNL-NEXT:    subq $192, %rsp
 ; KNL-NEXT:    ## kill: def $esi killed $esi def $rsi
-; KNL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; KNL-NEXT:    vpcmpeqb %ymm2, %ymm0, %ymm3
-; KNL-NEXT:    vpternlogq $15, %zmm3, %zmm3, %zmm3
-; KNL-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
-; KNL-NEXT:    vpcmpeqb %ymm2, %ymm0, %ymm0
+; KNL-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; KNL-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; KNL-NEXT:    vpcmpeqb %ymm3, %ymm2, %ymm2
+; KNL-NEXT:    vpcmpeqb %ymm3, %ymm0, %ymm0
+; KNL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; KNL-NEXT:    vpternlogq $15, %zmm0, %zmm0, %zmm0
-; KNL-NEXT:    vpcmpeqb %ymm2, %ymm1, %ymm4
-; KNL-NEXT:    vpternlogq $15, %zmm4, %zmm4, %zmm4
-; KNL-NEXT:    vextracti64x4 $1, %zmm1, %ymm1
-; KNL-NEXT:    vpcmpeqb %ymm2, %ymm1, %ymm1
+; KNL-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
+; KNL-NEXT:    vpcmpeqb %ymm3, %ymm2, %ymm2
+; KNL-NEXT:    vpcmpeqb %ymm3, %ymm1, %ymm1
+; KNL-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm1
 ; KNL-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
 ; KNL-NEXT:    andl $127, %esi
 ; KNL-NEXT:    testb %dil, %dil
-; KNL-NEXT:    vmovdqa %ymm1, {{[0-9]+}}(%rsp)
-; KNL-NEXT:    vmovdqa %ymm4, {{[0-9]+}}(%rsp)
-; KNL-NEXT:    vmovdqa %ymm0, {{[0-9]+}}(%rsp)
-; KNL-NEXT:    vmovdqa %ymm3, (%rsp)
+; KNL-NEXT:    vmovdqa64 %zmm1, {{[0-9]+}}(%rsp)
+; KNL-NEXT:    vmovdqa64 %zmm0, (%rsp)
 ; KNL-NEXT:    setne (%rsp,%rsi)
 ; KNL-NEXT:    vpmovsxbd (%rsp), %zmm0
 ; KNL-NEXT:    vpslld $31, %zmm0, %zmm0
diff --git a/llvm/test/CodeGen/X86/vector-bitreverse.ll b/llvm/test/CodeGen/X86/vector-bitreverse.ll
index 5d9cd1643aece..f99fc38c6625e 100644
--- a/llvm/test/CodeGen/X86/vector-bitreverse.ll
+++ b/llvm/test/CodeGen/X86/vector-bitreverse.ll
@@ -1314,18 +1314,18 @@ define <64 x i8> @test_bitreverse_v64i8(<64 x i8> %a) nounwind {
 ; AVX512F-NEXT:    vpand %ymm2, %ymm1, %ymm3
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm4 = [0,128,64,192,32,160,96,224,16,144,80,208,48,176,112,240,0,128,64,192,32,160,96,224,16,144,80,208,48,176,112,240]
 ; AVX512F-NEXT:    vpshufb %ymm3, %ymm4, %ymm3
+; AVX512F-NEXT:    vpand %ymm2, %ymm0, %ymm5
+; AVX512F-NEXT:    vpshufb %ymm5, %ymm4, %ymm4
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm3, %zmm4, %zmm3
 ; AVX512F-NEXT:    vpsrlw $4, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpand %ymm2, %ymm1, %ymm1
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm5 = [0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15,0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15]
-; AVX512F-NEXT:    vpshufb %ymm1, %ymm5, %ymm1
-; AVX512F-NEXT:    vpor %ymm1, %ymm3, %ymm1
-; AVX512F-NEXT:    vpand %ymm2, %ymm0, %ymm3
-; AVX512F-NEXT:    vpshufb %ymm3, %ymm4, %ymm3
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm4 = [0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15,0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15]
+; AVX512F-NEXT:    vpshufb %ymm1, %ymm4, %ymm1
 ; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpand %ymm2, %ymm0, %ymm0
-; AVX512F-NEXT:    vpshufb %ymm0, %ymm5, %ymm0
-; AVX512F-NEXT:    vpor %ymm0, %ymm3, %ymm0
+; AVX512F-NEXT:    vpshufb %ymm0, %ymm4, %ymm0
 ; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vporq %zmm0, %zmm3, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: test_bitreverse_v64i8:
@@ -1591,19 +1591,19 @@ define <32 x i16> @test_bitreverse_v32i16(<32 x i16> %a) nounwind {
 ; AVX512F-NEXT:    vpand %ymm3, %ymm1, %ymm4
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm5 = [0,128,64,192,32,160,96,224,16,144,80,208,48,176,112,240,0,128,64,192,32,160,96,224,16,144,80,208,48,176,112,240]
 ; AVX512F-NEXT:    vpshufb %ymm4, %ymm5, %ymm4
-; AVX512F-NEXT:    vpsrlw $4, %ymm1, %ymm1
-; AVX512F-NEXT:    vpand %ymm3, %ymm1, %ymm1
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm6 = [0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15,0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15]
-; AVX512F-NEXT:    vpshufb %ymm1, %ymm6, %ymm1
-; AVX512F-NEXT:    vpor %ymm1, %ymm4, %ymm1
 ; AVX512F-NEXT:    vpshufb %ymm2, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpand %ymm3, %ymm0, %ymm2
 ; AVX512F-NEXT:    vpshufb %ymm2, %ymm5, %ymm2
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm4, %zmm2, %zmm2
+; AVX512F-NEXT:    vpsrlw $4, %ymm1, %ymm1
+; AVX512F-NEXT:    vpand %ymm3, %ymm1, %ymm1
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm4 = [0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15,0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15]
+; AVX512F-NEXT:    vpshufb %ymm1, %ymm4, %ymm1
 ; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpand %ymm3, %ymm0, %ymm0
-; AVX512F-NEXT:    vpshufb %ymm0, %ymm6, %ymm0
-; AVX512F-NEXT:    vpor %ymm0, %ymm2, %ymm0
+; AVX512F-NEXT:    vpshufb %ymm0, %ymm4, %ymm0
 ; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vporq %zmm0, %zmm2, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: test_bitreverse_v32i16:
@@ -1887,19 +1887,19 @@ define <16 x i32> @test_bitreverse_v16i32(<16 x i32> %a) nounwind {
 ; AVX512F-NEXT:    vpand %ymm3, %ymm1, %ymm4
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm5 = [0,128,64,192,32,160,96,224,16,144,80,208,48,176,112,240,0,128,64,192,32,160,96,224,16,144,80,208,48,176,112,240]
 ; AVX512F-NEXT:    vpshufb %ymm4, %ymm5, %ymm4
-; AVX512F-NEXT:    vpsrlw $4, %ymm1, %ymm1
-; AVX512F-NEXT:    vpand %ymm3, %ymm1, %ymm1
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm6 = [0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15,0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15]
-; AVX512F-NEXT:    vpshufb %ymm1, %ymm6, %ymm1
-; AVX512F-NEXT:    vpor %ymm1, %ymm4, %ymm1
 ; AVX512F-NEXT:    vpshufb %ymm2, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpand %ymm3, %ymm0, %ymm2
 ; AVX512F-NEXT:    vpshufb %ymm2, %ymm5, %ymm2
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm4, %zmm2, %zmm2
+; AVX512F-NEXT:    vpsrlw $4, %ymm1, %ymm1
+; AVX512F-NEXT:    vpand %ymm3, %ymm1, %ymm1
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm4 = [0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15,0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15]
+; AVX512F-NEXT:    vpshufb %ymm1, %ymm4, %ymm1
 ; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpand %ymm3, %ymm0, %ymm0
-; AVX512F-NEXT:    vpshufb %ymm0, %ymm6, %ymm0
-; AVX512F-NEXT:    vpor %ymm0, %ymm2, %ymm0
+; AVX512F-NEXT:    vpshufb %ymm0, %ymm4, %ymm0
 ; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vporq %zmm0, %zmm2, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: test_bitreverse_v16i32:
@@ -2191,19 +2191,19 @@ define <8 x i64> @test_bitreverse_v8i64(<8 x i64> %a) nounwind {
 ; AVX512F-NEXT:    vpand %ymm3, %ymm1, %ymm4
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm5 = [0,128,64,192,32,160,96,224,16,144,80,208,48,176,112,240,0,128,64,192,32,160,96,224,16,144,80,208,48,176,112,240]
 ; AVX512F-NEXT:    vpshufb %ymm4, %ymm5, %ymm4
-; AVX512F-NEXT:    vpsrlw $4, %ymm1, %ymm1
-; AVX512F-NEXT:    vpand %ymm3, %ymm1, %ymm1
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm6 = [0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15,0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15]
-; AVX512F-NEXT:    vpshufb %ymm1, %ymm6, %ymm1
-; AVX512F-NEXT:    vpor %ymm1, %ymm4, %ymm1
 ; AVX512F-NEXT:    vpshufb %ymm2, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpand %ymm3, %ymm0, %ymm2
 ; AVX512F-NEXT:    vpshufb %ymm2, %ymm5, %ymm2
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm4, %zmm2, %zmm2
+; AVX512F-NEXT:    vpsrlw $4, %ymm1, %ymm1
+; AVX512F-NEXT:    vpand %ymm3, %ymm1, %ymm1
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm4 = [0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15,0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15]
+; AVX512F-NEXT:    vpshufb %ymm1, %ymm4, %ymm1
 ; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpand %ymm3, %ymm0, %ymm0
-; AVX512F-NEXT:    vpshufb %ymm0, %ymm6, %ymm0
-; AVX512F-NEXT:    vpor %ymm0, %ymm2, %ymm0
+; AVX512F-NEXT:    vpshufb %ymm0, %ymm4, %ymm0
 ; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vporq %zmm0, %zmm2, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: test_bitreverse_v8i64:
diff --git a/llvm/test/CodeGen/X86/vector-fshl-512.ll b/llvm/test/CodeGen/X86/vector-fshl-512.ll
index 088a590a2e07f..60406c45ba899 100644
--- a/llvm/test/CodeGen/X86/vector-fshl-512.ll
+++ b/llvm/test/CodeGen/X86/vector-fshl-512.ll
@@ -821,31 +821,30 @@ define <64 x i8> @splatvar_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> %
 ; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
 ; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512F-NEXT:    vpsllw %xmm4, %ymm3, %ymm3
+; AVX512F-NEXT:    vpsllw %xmm4, %ymm0, %ymm5
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm3, %zmm5, %zmm3
 ; AVX512F-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
-; AVX512F-NEXT:    vpsllw %xmm4, %xmm5, %xmm6
-; AVX512F-NEXT:    vpbroadcastb %xmm6, %ymm6
-; AVX512F-NEXT:    vpand %ymm6, %ymm3, %ymm3
-; AVX512F-NEXT:    vpsllw %xmm4, %ymm0, %ymm4
-; AVX512F-NEXT:    vpand %ymm6, %ymm4, %ymm4
-; AVX512F-NEXT:    vinserti64x4 $1, %ymm3, %zmm4, %zmm3
+; AVX512F-NEXT:    vpsllw %xmm4, %xmm5, %xmm4
+; AVX512F-NEXT:    vpbroadcastb %xmm4, %ymm4
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm4, %zmm4, %zmm4
+; AVX512F-NEXT:    vpandq %zmm4, %zmm3, %zmm3
 ; AVX512F-NEXT:    vpbroadcastb %xmm2, %ymm2
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
 ; AVX512F-NEXT:    vpsubb %xmm2, %xmm4, %xmm4
 ; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
 ; AVX512F-NEXT:    vpsrlw %xmm4, %ymm6, %ymm6
-; AVX512F-NEXT:    vpsrlw %xmm4, %xmm5, %xmm5
-; AVX512F-NEXT:    vpsrlw $8, %xmm5, %xmm5
-; AVX512F-NEXT:    vpbroadcastb %xmm5, %ymm5
-; AVX512F-NEXT:    vpand %ymm5, %ymm6, %ymm6
 ; AVX512F-NEXT:    vpsrlw %xmm4, %ymm1, %ymm1
-; AVX512F-NEXT:    vpand %ymm5, %ymm1, %ymm1
 ; AVX512F-NEXT:    vinserti64x4 $1, %ymm6, %zmm1, %zmm1
-; AVX512F-NEXT:    vporq %zmm1, %zmm3, %zmm1
-; AVX512F-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX512F-NEXT:    vpcmpeqb %ymm3, %ymm2, %ymm2
-; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm2, %zmm2
-; AVX512F-NEXT:    vpternlogq $226, %zmm1, %zmm2, %zmm0
+; AVX512F-NEXT:    vpsrlw %xmm4, %xmm5, %xmm4
+; AVX512F-NEXT:    vpsrlw $8, %xmm4, %xmm4
+; AVX512F-NEXT:    vpbroadcastb %xmm4, %ymm4
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm4, %zmm4, %zmm4
+; AVX512F-NEXT:    vpternlogq $236, %zmm1, %zmm3, %zmm4
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpcmpeqb %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm1, %zmm1
+; AVX512F-NEXT:    vpternlogq $226, %zmm4, %zmm1, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatvar_funnnel_v64i8:
@@ -854,31 +853,30 @@ define <64 x i8> @splatvar_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> %
 ; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
 ; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512VL-NEXT:    vpsllw %xmm4, %ymm3, %ymm3
+; AVX512VL-NEXT:    vpsllw %xmm4, %ymm0, %ymm5
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm3, %zmm5, %zmm3
 ; AVX512VL-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
-; AVX512VL-NEXT:    vpsllw %xmm4, %xmm5, %xmm6
-; AVX512VL-NEXT:    vpbroadcastb %xmm6, %ymm6
-; AVX512VL-NEXT:    vpand %ymm6, %ymm3, %ymm3
-; AVX512VL-NEXT:    vpsllw %xmm4, %ymm0, %ymm4
-; AVX512VL-NEXT:    vpand %ymm6, %ymm4, %ymm4
-; AVX512VL-NEXT:    vinserti64x4 $1, %ymm3, %zmm4, %zmm3
+; AVX512VL-NEXT:    vpsllw %xmm4, %xmm5, %xmm4
+; AVX512VL-NEXT:    vpbroadcastb %xmm4, %ymm4
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm4, %zmm4, %zmm4
+; AVX512VL-NEXT:    vpandq %zmm4, %zmm3, %zmm3
 ; AVX512VL-NEXT:    vpbroadcastb %xmm2, %ymm2
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
 ; AVX512VL-NEXT:    vpsubb %xmm2, %xmm4, %xmm4
 ; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512VL-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
 ; AVX512VL-NEXT:    vpsrlw %xmm4, %ymm6, %ymm6
-; AVX512VL-NEXT:    vpsrlw %xmm4, %xmm5, %xmm5
-; AVX512VL-NEXT:    vpsrlw $8, %xmm5, %xmm5
-; AVX512VL-NEXT:    vpbroadcastb %xmm5, %ymm5
-; AVX512VL-NEXT:    vpand %ymm5, %ymm6, %ymm6
 ; AVX512VL-NEXT:    vpsrlw %xmm4, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpand %ymm5, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vinserti64x4 $1, %ymm6, %zmm1, %zmm1
-; AVX512VL-NEXT:    vporq %zmm1, %zmm3, %zmm1
-; AVX512VL-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX512VL-NEXT:    vpcmpeqb %ymm3, %ymm2, %ymm2
-; AVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm2, %zmm2
-; AVX512VL-NEXT:    vpternlogq $226, %zmm1, %zmm2, %zmm0
+; AVX512VL-NEXT:    vpsrlw %xmm4, %xmm5, %xmm4
+; AVX512VL-NEXT:    vpsrlw $8, %xmm4, %xmm4
+; AVX512VL-NEXT:    vpbroadcastb %xmm4, %ymm4
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm4, %zmm4, %zmm4
+; AVX512VL-NEXT:    vpternlogq $236, %zmm1, %zmm3, %zmm4
+; AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512VL-NEXT:    vpcmpeqb %ymm1, %ymm2, %ymm1
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm1, %zmm1
+; AVX512VL-NEXT:    vpternlogq $226, %zmm4, %zmm1, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatvar_funnnel_v64i8:
@@ -1510,40 +1508,28 @@ define <32 x i16> @splatconstant_funnnel_v32i16(<32 x i16> %x, <32 x i16> %y) no
 define <64 x i8> @splatconstant_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y) nounwind {
 ; AVX512F-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
-; AVX512F-NEXT:    vpsrlw $4, %ymm2, %ymm2
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512F-NEXT:    vpand %ymm3, %ymm2, %ymm2
-; AVX512F-NEXT:    vpsrlw $4, %ymm1, %ymm1
-; AVX512F-NEXT:    vpand %ymm3, %ymm1, %ymm1
-; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm1
-; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
-; AVX512F-NEXT:    vpsllw $4, %ymm2, %ymm2
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX512F-NEXT:    vpand %ymm3, %ymm2, %ymm2
+; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
 ; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm0
-; AVX512F-NEXT:    vpand %ymm3, %ymm0, %ymm0
-; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; AVX512F-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm0, %zmm2, %zmm2
+; AVX512F-NEXT:    vpsrlw $4, %ymm1, %ymm0
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm1
+; AVX512F-NEXT:    vpsrlw $4, %ymm1, %ymm1
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vpternlogq $216, {{.*}}(%rip), %zmm2, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
-; AVX512VL-NEXT:    vpsrlw $4, %ymm2, %ymm2
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512VL-NEXT:    vpand %ymm3, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpsrlw $4, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpand %ymm3, %ymm1, %ymm1
-; AVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm1
-; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
-; AVX512VL-NEXT:    vpsllw $4, %ymm2, %ymm2
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX512VL-NEXT:    vpand %ymm3, %ymm2, %ymm2
+; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm2
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
 ; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpand %ymm3, %ymm0, %ymm0
-; AVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; AVX512VL-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm0, %zmm2, %zmm2
+; AVX512VL-NEXT:    vpsrlw $4, %ymm1, %ymm0
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm1, %ymm1
+; AVX512VL-NEXT:    vpsrlw $4, %ymm1, %ymm1
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512VL-NEXT:    vpternlogq $216, {{.*}}(%rip), %zmm2, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatconstant_funnnel_v64i8:
diff --git a/llvm/test/CodeGen/X86/vector-fshl-rot-512.ll b/llvm/test/CodeGen/X86/vector-fshl-rot-512.ll
index 2481b8ebfe25d..6671f3ec4c0f8 100644
--- a/llvm/test/CodeGen/X86/vector-fshl-rot-512.ll
+++ b/llvm/test/CodeGen/X86/vector-fshl-rot-512.ll
@@ -483,14 +483,13 @@ define <64 x i8> @splatvar_funnnel_v64i8(<64 x i8> %x, <64 x i8> %amt) nounwind
 ; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
 ; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512F-NEXT:    vpsllw %xmm2, %ymm1, %ymm1
-; AVX512F-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX512F-NEXT:    vpsllw %xmm2, %xmm4, %xmm4
-; AVX512F-NEXT:    vpbroadcastb %xmm4, %ymm4
-; AVX512F-NEXT:    vpand %ymm4, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpsllw %xmm2, %ymm0, %ymm0
-; AVX512F-NEXT:    vpand %ymm4, %ymm0, %ymm0
-; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512F-NEXT:    vporq %zmm3, %zmm0, %zmm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm1
+; AVX512F-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX512F-NEXT:    vpsllw %xmm2, %xmm0, %xmm0
+; AVX512F-NEXT:    vpbroadcastb %xmm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm0
+; AVX512F-NEXT:    vpternlogq $236, %zmm1, %zmm3, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatvar_funnnel_v64i8:
@@ -529,14 +528,13 @@ define <64 x i8> @splatvar_funnnel_v64i8(<64 x i8> %x, <64 x i8> %amt) nounwind
 ; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
 ; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512VL-NEXT:    vpsllw %xmm2, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX512VL-NEXT:    vpsllw %xmm2, %xmm4, %xmm4
-; AVX512VL-NEXT:    vpbroadcastb %xmm4, %ymm4
-; AVX512VL-NEXT:    vpand %ymm4, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vpsllw %xmm2, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpand %ymm4, %ymm0, %ymm0
-; AVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512VL-NEXT:    vporq %zmm3, %zmm0, %zmm0
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm1
+; AVX512VL-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpsllw %xmm2, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpbroadcastb %xmm0, %ymm0
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm0
+; AVX512VL-NEXT:    vpternlogq $236, %zmm1, %zmm3, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatvar_funnnel_v64i8:
@@ -886,38 +884,26 @@ define <32 x i16> @splatconstant_funnnel_v32i16(<32 x i16> %x) nounwind {
 define <64 x i8> @splatconstant_funnnel_v64i8(<64 x i8> %x) nounwind {
 ; AVX512F-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
-; AVX512F-NEXT:    vpsrlw $4, %ymm1, %ymm2
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512F-NEXT:    vpand %ymm3, %ymm2, %ymm2
-; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm4
-; AVX512F-NEXT:    vpand %ymm3, %ymm4, %ymm3
-; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm2
-; AVX512F-NEXT:    vpsllw $4, %ymm1, %ymm1
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX512F-NEXT:    vpand %ymm3, %ymm1, %ymm1
-; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm0
-; AVX512F-NEXT:    vpand %ymm3, %ymm0, %ymm0
-; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512F-NEXT:    vporq %zmm2, %zmm0, %zmm0
+; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm1
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512F-NEXT:    vpsllw $4, %ymm2, %ymm3
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm1
+; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; AVX512F-NEXT:    vpsrlw $4, %ymm2, %ymm2
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512F-NEXT:    vpternlogq $216, {{.*}}(%rip), %zmm1, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
-; AVX512VL-NEXT:    vpsrlw $4, %ymm1, %ymm2
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512VL-NEXT:    vpand %ymm3, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm4
-; AVX512VL-NEXT:    vpand %ymm3, %ymm4, %ymm3
-; AVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm2
-; AVX512VL-NEXT:    vpsllw $4, %ymm1, %ymm1
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX512VL-NEXT:    vpand %ymm3, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpand %ymm3, %ymm0, %ymm0
-; AVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512VL-NEXT:    vporq %zmm2, %zmm0, %zmm0
+; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm1
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512VL-NEXT:    vpsllw $4, %ymm2, %ymm3
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm1
+; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpsrlw $4, %ymm2, %ymm2
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512VL-NEXT:    vpternlogq $216, {{.*}}(%rip), %zmm1, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatconstant_funnnel_v64i8:
diff --git a/llvm/test/CodeGen/X86/vector-fshr-512.ll b/llvm/test/CodeGen/X86/vector-fshr-512.ll
index fa70e840081f5..9aa74f165bddb 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-512.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-512.ll
@@ -805,68 +805,66 @@ define <32 x i16> @splatvar_funnnel_v32i16(<32 x i16> %x, <32 x i16> %y, <32 x i
 define <64 x i8> @splatvar_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> %amt) nounwind {
 ; AVX512F-LABEL: splatvar_funnnel_v64i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
 ; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
-; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512F-NEXT:    vpsrlw %xmm4, %ymm3, %ymm3
-; AVX512F-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
-; AVX512F-NEXT:    vpsrlw %xmm4, %xmm5, %xmm6
-; AVX512F-NEXT:    vpsrlw $8, %xmm6, %xmm6
-; AVX512F-NEXT:    vpbroadcastb %xmm6, %ymm6
-; AVX512F-NEXT:    vpand %ymm6, %ymm3, %ymm3
-; AVX512F-NEXT:    vpsrlw %xmm4, %ymm1, %ymm4
-; AVX512F-NEXT:    vpand %ymm6, %ymm4, %ymm4
-; AVX512F-NEXT:    vinserti64x4 $1, %ymm3, %zmm4, %zmm3
-; AVX512F-NEXT:    vpbroadcastb %xmm2, %ymm2
+; AVX512F-NEXT:    vpbroadcastb %xmm2, %ymm3
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512F-NEXT:    vpsubb %xmm2, %xmm4, %xmm4
+; AVX512F-NEXT:    vpsubb %xmm3, %xmm4, %xmm4
 ; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm6
-; AVX512F-NEXT:    vpsllw %xmm4, %ymm6, %ymm6
-; AVX512F-NEXT:    vpsllw %xmm4, %xmm5, %xmm5
-; AVX512F-NEXT:    vpbroadcastb %xmm5, %ymm5
-; AVX512F-NEXT:    vpand %ymm5, %ymm6, %ymm6
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm5
+; AVX512F-NEXT:    vpsllw %xmm4, %ymm5, %ymm5
 ; AVX512F-NEXT:    vpsllw %xmm4, %ymm0, %ymm0
-; AVX512F-NEXT:    vpand %ymm5, %ymm0, %ymm0
-; AVX512F-NEXT:    vinserti64x4 $1, %ymm6, %zmm0, %zmm0
-; AVX512F-NEXT:    vporq %zmm3, %zmm0, %zmm3
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm5, %zmm0, %zmm0
+; AVX512F-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
+; AVX512F-NEXT:    vpsllw %xmm4, %xmm5, %xmm4
+; AVX512F-NEXT:    vpbroadcastb %xmm4, %ymm4
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm4, %zmm4, %zmm4
+; AVX512F-NEXT:    vpandq %zmm4, %zmm0, %zmm0
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm4
+; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512F-NEXT:    vpsrlw %xmm2, %ymm4, %ymm4
+; AVX512F-NEXT:    vpsrlw %xmm2, %ymm1, %ymm6
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm4, %zmm6, %zmm4
+; AVX512F-NEXT:    vpsrlw %xmm2, %xmm5, %xmm2
+; AVX512F-NEXT:    vpsrlw $8, %xmm2, %xmm2
+; AVX512F-NEXT:    vpbroadcastb %xmm2, %ymm2
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm2, %zmm2
+; AVX512F-NEXT:    vpternlogq $236, %zmm4, %zmm0, %zmm2
 ; AVX512F-NEXT:    vpxor %xmm0, %xmm0, %xmm0
-; AVX512F-NEXT:    vpcmpeqb %ymm0, %ymm2, %ymm0
+; AVX512F-NEXT:    vpcmpeqb %ymm0, %ymm3, %ymm0
 ; AVX512F-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm0
-; AVX512F-NEXT:    vpternlogq $202, %zmm3, %zmm1, %zmm0
+; AVX512F-NEXT:    vpternlogq $202, %zmm2, %zmm1, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatvar_funnnel_v64i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
 ; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
-; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512VL-NEXT:    vpsrlw %xmm4, %ymm3, %ymm3
-; AVX512VL-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
-; AVX512VL-NEXT:    vpsrlw %xmm4, %xmm5, %xmm6
-; AVX512VL-NEXT:    vpsrlw $8, %xmm6, %xmm6
-; AVX512VL-NEXT:    vpbroadcastb %xmm6, %ymm6
-; AVX512VL-NEXT:    vpand %ymm6, %ymm3, %ymm3
-; AVX512VL-NEXT:    vpsrlw %xmm4, %ymm1, %ymm4
-; AVX512VL-NEXT:    vpand %ymm6, %ymm4, %ymm4
-; AVX512VL-NEXT:    vinserti64x4 $1, %ymm3, %zmm4, %zmm3
-; AVX512VL-NEXT:    vpbroadcastb %xmm2, %ymm2
+; AVX512VL-NEXT:    vpbroadcastb %xmm2, %ymm3
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512VL-NEXT:    vpsubb %xmm2, %xmm4, %xmm4
+; AVX512VL-NEXT:    vpsubb %xmm3, %xmm4, %xmm4
 ; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm6
-; AVX512VL-NEXT:    vpsllw %xmm4, %ymm6, %ymm6
-; AVX512VL-NEXT:    vpsllw %xmm4, %xmm5, %xmm5
-; AVX512VL-NEXT:    vpbroadcastb %xmm5, %ymm5
-; AVX512VL-NEXT:    vpand %ymm5, %ymm6, %ymm6
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm5
+; AVX512VL-NEXT:    vpsllw %xmm4, %ymm5, %ymm5
 ; AVX512VL-NEXT:    vpsllw %xmm4, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpand %ymm5, %ymm0, %ymm0
-; AVX512VL-NEXT:    vinserti64x4 $1, %ymm6, %zmm0, %zmm0
-; AVX512VL-NEXT:    vporq %zmm3, %zmm0, %zmm3
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm5, %zmm0, %zmm0
+; AVX512VL-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
+; AVX512VL-NEXT:    vpsllw %xmm4, %xmm5, %xmm4
+; AVX512VL-NEXT:    vpbroadcastb %xmm4, %ymm4
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm4, %zmm4, %zmm4
+; AVX512VL-NEXT:    vpandq %zmm4, %zmm0, %zmm0
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm1, %ymm4
+; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VL-NEXT:    vpsrlw %xmm2, %ymm4, %ymm4
+; AVX512VL-NEXT:    vpsrlw %xmm2, %ymm1, %ymm6
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm4, %zmm6, %zmm4
+; AVX512VL-NEXT:    vpsrlw %xmm2, %xmm5, %xmm2
+; AVX512VL-NEXT:    vpsrlw $8, %xmm2, %xmm2
+; AVX512VL-NEXT:    vpbroadcastb %xmm2, %ymm2
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm2, %zmm2
+; AVX512VL-NEXT:    vpternlogq $236, %zmm4, %zmm0, %zmm2
 ; AVX512VL-NEXT:    vpxor %xmm0, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpcmpeqb %ymm0, %ymm2, %ymm0
+; AVX512VL-NEXT:    vpcmpeqb %ymm0, %ymm3, %ymm0
 ; AVX512VL-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm0
-; AVX512VL-NEXT:    vpternlogq $202, %zmm3, %zmm1, %zmm0
+; AVX512VL-NEXT:    vpternlogq $202, %zmm2, %zmm1, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatvar_funnnel_v64i8:
@@ -1494,40 +1492,28 @@ define <32 x i16> @splatconstant_funnnel_v32i16(<32 x i16> %x, <32 x i16> %y) no
 define <64 x i8> @splatconstant_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y) nounwind {
 ; AVX512F-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
-; AVX512F-NEXT:    vpsrlw $4, %ymm2, %ymm2
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512F-NEXT:    vpand %ymm3, %ymm2, %ymm2
-; AVX512F-NEXT:    vpsrlw $4, %ymm1, %ymm1
-; AVX512F-NEXT:    vpand %ymm3, %ymm1, %ymm1
-; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm1
-; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
-; AVX512F-NEXT:    vpsllw $4, %ymm2, %ymm2
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX512F-NEXT:    vpand %ymm3, %ymm2, %ymm2
+; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
 ; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm0
-; AVX512F-NEXT:    vpand %ymm3, %ymm0, %ymm0
-; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; AVX512F-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm0, %zmm2, %zmm2
+; AVX512F-NEXT:    vpsrlw $4, %ymm1, %ymm0
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm1
+; AVX512F-NEXT:    vpsrlw $4, %ymm1, %ymm1
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vpternlogq $216, {{.*}}(%rip), %zmm2, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
-; AVX512VL-NEXT:    vpsrlw $4, %ymm2, %ymm2
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512VL-NEXT:    vpand %ymm3, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpsrlw $4, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpand %ymm3, %ymm1, %ymm1
-; AVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm1
-; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
-; AVX512VL-NEXT:    vpsllw $4, %ymm2, %ymm2
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX512VL-NEXT:    vpand %ymm3, %ymm2, %ymm2
+; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm2
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
 ; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpand %ymm3, %ymm0, %ymm0
-; AVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; AVX512VL-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm0, %zmm2, %zmm2
+; AVX512VL-NEXT:    vpsrlw $4, %ymm1, %ymm0
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm1, %ymm1
+; AVX512VL-NEXT:    vpsrlw $4, %ymm1, %ymm1
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512VL-NEXT:    vpternlogq $216, {{.*}}(%rip), %zmm2, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatconstant_funnnel_v64i8:
diff --git a/llvm/test/CodeGen/X86/vector-fshr-rot-512.ll b/llvm/test/CodeGen/X86/vector-fshr-rot-512.ll
index 04a883171a7ce..0084702b7fd76 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-rot-512.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-rot-512.ll
@@ -480,15 +480,14 @@ define <64 x i8> @splatvar_funnnel_v64i8(<64 x i8> %x, <64 x i8> %amt) nounwind
 ; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
 ; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512F-NEXT:    vpsrlw %xmm1, %ymm2, %ymm2
-; AVX512F-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX512F-NEXT:    vpsrlw %xmm1, %xmm4, %xmm4
-; AVX512F-NEXT:    vpsrlw $8, %xmm4, %xmm4
-; AVX512F-NEXT:    vpbroadcastb %xmm4, %ymm4
-; AVX512F-NEXT:    vpand %ymm4, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
-; AVX512F-NEXT:    vpand %ymm4, %ymm0, %ymm0
-; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; AVX512F-NEXT:    vporq %zmm0, %zmm3, %zmm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm2
+; AVX512F-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX512F-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    vpsrlw $8, %xmm0, %xmm0
+; AVX512F-NEXT:    vpbroadcastb %xmm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm0
+; AVX512F-NEXT:    vpternlogq $236, %zmm2, %zmm3, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatvar_funnnel_v64i8:
@@ -524,15 +523,14 @@ define <64 x i8> @splatvar_funnnel_v64i8(<64 x i8> %x, <64 x i8> %amt) nounwind
 ; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
 ; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512VL-NEXT:    vpsrlw %xmm1, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX512VL-NEXT:    vpsrlw %xmm1, %xmm4, %xmm4
-; AVX512VL-NEXT:    vpsrlw $8, %xmm4, %xmm4
-; AVX512VL-NEXT:    vpbroadcastb %xmm4, %ymm4
-; AVX512VL-NEXT:    vpand %ymm4, %ymm2, %ymm2
 ; AVX512VL-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpand %ymm4, %ymm0, %ymm0
-; AVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; AVX512VL-NEXT:    vporq %zmm0, %zmm3, %zmm0
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm2
+; AVX512VL-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpsrlw $8, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpbroadcastb %xmm0, %ymm0
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm0
+; AVX512VL-NEXT:    vpternlogq $236, %zmm2, %zmm3, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatvar_funnnel_v64i8:
@@ -882,38 +880,26 @@ define <32 x i16> @splatconstant_funnnel_v32i16(<32 x i16> %x) nounwind {
 define <64 x i8> @splatconstant_funnnel_v64i8(<64 x i8> %x) nounwind {
 ; AVX512F-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
-; AVX512F-NEXT:    vpsrlw $4, %ymm1, %ymm2
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512F-NEXT:    vpand %ymm3, %ymm2, %ymm2
-; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm4
-; AVX512F-NEXT:    vpand %ymm3, %ymm4, %ymm3
-; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm2
-; AVX512F-NEXT:    vpsllw $4, %ymm1, %ymm1
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX512F-NEXT:    vpand %ymm3, %ymm1, %ymm1
-; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm0
-; AVX512F-NEXT:    vpand %ymm3, %ymm0, %ymm0
-; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512F-NEXT:    vporq %zmm2, %zmm0, %zmm0
+; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm1
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512F-NEXT:    vpsllw $4, %ymm2, %ymm3
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm1
+; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; AVX512F-NEXT:    vpsrlw $4, %ymm2, %ymm2
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512F-NEXT:    vpternlogq $216, {{.*}}(%rip), %zmm1, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
-; AVX512VL-NEXT:    vpsrlw $4, %ymm1, %ymm2
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512VL-NEXT:    vpand %ymm3, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm4
-; AVX512VL-NEXT:    vpand %ymm3, %ymm4, %ymm3
-; AVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm2
-; AVX512VL-NEXT:    vpsllw $4, %ymm1, %ymm1
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX512VL-NEXT:    vpand %ymm3, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpand %ymm3, %ymm0, %ymm0
-; AVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512VL-NEXT:    vporq %zmm2, %zmm0, %zmm0
+; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm1
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512VL-NEXT:    vpsllw $4, %ymm2, %ymm3
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm1
+; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpsrlw $4, %ymm2, %ymm2
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512VL-NEXT:    vpternlogq $216, {{.*}}(%rip), %zmm1, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatconstant_funnnel_v64i8:
diff --git a/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll b/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll
index 5b88eaec05969..e756f3ecc3538 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll
@@ -133,38 +133,36 @@ define <32 x i16> @test_div7_32i16(<32 x i16> %a) nounwind {
 define <64 x i8> @test_div7_64i8(<64 x i8> %a) nounwind {
 ; AVX512F-LABEL: test_div7_64i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15],ymm1[24],ymm2[24],ymm1[25],ymm2[25],ymm1[26],ymm2[26],ymm1[27],ymm2[27],ymm1[28],ymm2[28],ymm1[29],ymm2[29],ymm1[30],ymm2[30],ymm1[31],ymm2[31]
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm4 = [37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37]
-; AVX512F-NEXT:    vpmullw %ymm4, %ymm3, %ymm3
-; AVX512F-NEXT:    vpsrlw $8, %ymm3, %ymm3
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[16],ymm2[16],ymm1[17],ymm2[17],ymm1[18],ymm2[18],ymm1[19],ymm2[19],ymm1[20],ymm2[20],ymm1[21],ymm2[21],ymm1[22],ymm2[22],ymm1[23],ymm2[23]
-; AVX512F-NEXT:    vpmullw %ymm4, %ymm5, %ymm5
-; AVX512F-NEXT:    vpsrlw $8, %ymm5, %ymm5
-; AVX512F-NEXT:    vpackuswb %ymm3, %ymm5, %ymm3
-; AVX512F-NEXT:    vpsubb %ymm3, %ymm1, %ymm1
-; AVX512F-NEXT:    vpsrlw $1, %ymm1, %ymm1
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm5 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; AVX512F-NEXT:    vpand %ymm5, %ymm1, %ymm1
-; AVX512F-NEXT:    vpaddb %ymm3, %ymm1, %ymm1
-; AVX512F-NEXT:    vpsrlw $2, %ymm1, %ymm1
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
-; AVX512F-NEXT:    vpand %ymm3, %ymm1, %ymm1
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm6 = ymm0[8],ymm2[8],ymm0[9],ymm2[9],ymm0[10],ymm2[10],ymm0[11],ymm2[11],ymm0[12],ymm2[12],ymm0[13],ymm2[13],ymm0[14],ymm2[14],ymm0[15],ymm2[15],ymm0[24],ymm2[24],ymm0[25],ymm2[25],ymm0[26],ymm2[26],ymm0[27],ymm2[27],ymm0[28],ymm2[28],ymm0[29],ymm2[29],ymm0[30],ymm2[30],ymm0[31],ymm2[31]
-; AVX512F-NEXT:    vpmullw %ymm4, %ymm6, %ymm6
-; AVX512F-NEXT:    vpsrlw $8, %ymm6, %ymm6
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm0[0],ymm2[0],ymm0[1],ymm2[1],ymm0[2],ymm2[2],ymm0[3],ymm2[3],ymm0[4],ymm2[4],ymm0[5],ymm2[5],ymm0[6],ymm2[6],ymm0[7],ymm2[7],ymm0[16],ymm2[16],ymm0[17],ymm2[17],ymm0[18],ymm2[18],ymm0[19],ymm2[19],ymm0[20],ymm2[20],ymm0[21],ymm2[21],ymm0[22],ymm2[22],ymm0[23],ymm2[23]
-; AVX512F-NEXT:    vpmullw %ymm4, %ymm2, %ymm2
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15],ymm0[24],ymm1[24],ymm0[25],ymm1[25],ymm0[26],ymm1[26],ymm0[27],ymm1[27],ymm0[28],ymm1[28],ymm0[29],ymm1[29],ymm0[30],ymm1[30],ymm0[31],ymm1[31]
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = [37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37]
+; AVX512F-NEXT:    vpmullw %ymm3, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpsrlw $8, %ymm2, %ymm2
-; AVX512F-NEXT:    vpackuswb %ymm6, %ymm2, %ymm2
-; AVX512F-NEXT:    vpsubb %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[16],ymm1[16],ymm0[17],ymm1[17],ymm0[18],ymm1[18],ymm0[19],ymm1[19],ymm0[20],ymm1[20],ymm0[21],ymm1[21],ymm0[22],ymm1[22],ymm0[23],ymm1[23]
+; AVX512F-NEXT:    vpmullw %ymm3, %ymm4, %ymm4
+; AVX512F-NEXT:    vpsrlw $8, %ymm4, %ymm4
+; AVX512F-NEXT:    vpackuswb %ymm2, %ymm4, %ymm2
+; AVX512F-NEXT:    vpsubb %ymm2, %ymm0, %ymm4
+; AVX512F-NEXT:    vpsrlw $1, %ymm4, %ymm4
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm5 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
+; AVX512F-NEXT:    vpand %ymm5, %ymm4, %ymm4
+; AVX512F-NEXT:    vpaddb %ymm2, %ymm4, %ymm2
+; AVX512F-NEXT:    vpsrlw $2, %ymm2, %ymm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15],ymm0[24],ymm1[24],ymm0[25],ymm1[25],ymm0[26],ymm1[26],ymm0[27],ymm1[27],ymm0[28],ymm1[28],ymm0[29],ymm1[29],ymm0[30],ymm1[30],ymm0[31],ymm1[31]
+; AVX512F-NEXT:    vpmullw %ymm3, %ymm4, %ymm4
+; AVX512F-NEXT:    vpsrlw $8, %ymm4, %ymm4
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[16],ymm1[16],ymm0[17],ymm1[17],ymm0[18],ymm1[18],ymm0[19],ymm1[19],ymm0[20],ymm1[20],ymm0[21],ymm1[21],ymm0[22],ymm1[22],ymm0[23],ymm1[23]
+; AVX512F-NEXT:    vpmullw %ymm3, %ymm1, %ymm1
+; AVX512F-NEXT:    vpsrlw $8, %ymm1, %ymm1
+; AVX512F-NEXT:    vpackuswb %ymm4, %ymm1, %ymm1
+; AVX512F-NEXT:    vpsubb %ymm1, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpsrlw $1, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpand %ymm5, %ymm0, %ymm0
-; AVX512F-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vpaddb %ymm1, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpsrlw $2, %ymm0, %ymm0
-; AVX512F-NEXT:    vpand %ymm3, %ymm0, %ymm0
-; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm0, %zmm2, %zmm0
+; AVX512F-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: test_div7_64i8:
diff --git a/llvm/test/CodeGen/X86/vector-rotate-512.ll b/llvm/test/CodeGen/X86/vector-rotate-512.ll
index 831c03f03825a..690d9f721bb20 100644
--- a/llvm/test/CodeGen/X86/vector-rotate-512.ll
+++ b/llvm/test/CodeGen/X86/vector-rotate-512.ll
@@ -418,22 +418,21 @@ define <64 x i8> @splatvar_rotate_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
 ; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
 ; AVX512F-NEXT:    vpsllw %xmm2, %ymm3, %ymm4
+; AVX512F-NEXT:    vpsllw %xmm2, %ymm0, %ymm5
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm4, %zmm5, %zmm4
 ; AVX512F-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
-; AVX512F-NEXT:    vpsllw %xmm2, %xmm5, %xmm6
-; AVX512F-NEXT:    vpbroadcastb %xmm6, %ymm6
-; AVX512F-NEXT:    vpand %ymm6, %ymm4, %ymm4
-; AVX512F-NEXT:    vpsllw %xmm2, %ymm0, %ymm2
-; AVX512F-NEXT:    vpand %ymm6, %ymm2, %ymm2
-; AVX512F-NEXT:    vinserti64x4 $1, %ymm4, %zmm2, %zmm2
+; AVX512F-NEXT:    vpsllw %xmm2, %xmm5, %xmm2
+; AVX512F-NEXT:    vpbroadcastb %xmm2, %ymm2
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm2, %zmm2
+; AVX512F-NEXT:    vpandq %zmm2, %zmm4, %zmm2
 ; AVX512F-NEXT:    vpsrlw %xmm1, %ymm3, %ymm3
-; AVX512F-NEXT:    vpsrlw %xmm1, %xmm5, %xmm4
-; AVX512F-NEXT:    vpsrlw $8, %xmm4, %xmm4
-; AVX512F-NEXT:    vpbroadcastb %xmm4, %ymm4
-; AVX512F-NEXT:    vpand %ymm4, %ymm3, %ymm3
 ; AVX512F-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
-; AVX512F-NEXT:    vpand %ymm4, %ymm0, %ymm0
-; AVX512F-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm0
-; AVX512F-NEXT:    vporq %zmm0, %zmm2, %zmm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm3
+; AVX512F-NEXT:    vpsrlw %xmm1, %xmm5, %xmm0
+; AVX512F-NEXT:    vpsrlw $8, %xmm0, %xmm0
+; AVX512F-NEXT:    vpbroadcastb %xmm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm0
+; AVX512F-NEXT:    vpternlogq $236, %zmm3, %zmm2, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatvar_rotate_v64i8:
@@ -445,22 +444,21 @@ define <64 x i8> @splatvar_rotate_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
 ; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
 ; AVX512VL-NEXT:    vpsllw %xmm2, %ymm3, %ymm4
+; AVX512VL-NEXT:    vpsllw %xmm2, %ymm0, %ymm5
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm4, %zmm5, %zmm4
 ; AVX512VL-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
-; AVX512VL-NEXT:    vpsllw %xmm2, %xmm5, %xmm6
-; AVX512VL-NEXT:    vpbroadcastb %xmm6, %ymm6
-; AVX512VL-NEXT:    vpand %ymm6, %ymm4, %ymm4
-; AVX512VL-NEXT:    vpsllw %xmm2, %ymm0, %ymm2
-; AVX512VL-NEXT:    vpand %ymm6, %ymm2, %ymm2
-; AVX512VL-NEXT:    vinserti64x4 $1, %ymm4, %zmm2, %zmm2
+; AVX512VL-NEXT:    vpsllw %xmm2, %xmm5, %xmm2
+; AVX512VL-NEXT:    vpbroadcastb %xmm2, %ymm2
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm2, %zmm2
+; AVX512VL-NEXT:    vpandq %zmm2, %zmm4, %zmm2
 ; AVX512VL-NEXT:    vpsrlw %xmm1, %ymm3, %ymm3
-; AVX512VL-NEXT:    vpsrlw %xmm1, %xmm5, %xmm4
-; AVX512VL-NEXT:    vpsrlw $8, %xmm4, %xmm4
-; AVX512VL-NEXT:    vpbroadcastb %xmm4, %ymm4
-; AVX512VL-NEXT:    vpand %ymm4, %ymm3, %ymm3
 ; AVX512VL-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpand %ymm4, %ymm0, %ymm0
-; AVX512VL-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm0
-; AVX512VL-NEXT:    vporq %zmm0, %zmm2, %zmm0
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm3
+; AVX512VL-NEXT:    vpsrlw %xmm1, %xmm5, %xmm0
+; AVX512VL-NEXT:    vpsrlw $8, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpbroadcastb %xmm0, %ymm0
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm0
+; AVX512VL-NEXT:    vpternlogq $236, %zmm3, %zmm2, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatvar_rotate_v64i8:
@@ -809,38 +807,26 @@ define <32 x i16> @splatconstant_rotate_v32i16(<32 x i16> %a) nounwind {
 define <64 x i8> @splatconstant_rotate_v64i8(<64 x i8> %a) nounwind {
 ; AVX512F-LABEL: splatconstant_rotate_v64i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
-; AVX512F-NEXT:    vpsllw $4, %ymm1, %ymm2
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX512F-NEXT:    vpand %ymm3, %ymm2, %ymm2
-; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm4
-; AVX512F-NEXT:    vpand %ymm3, %ymm4, %ymm3
-; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm2
-; AVX512F-NEXT:    vpsrlw $4, %ymm1, %ymm1
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512F-NEXT:    vpand %ymm3, %ymm1, %ymm1
+; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm1
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512F-NEXT:    vpsllw $4, %ymm2, %ymm3
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm1
 ; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm0
-; AVX512F-NEXT:    vpand %ymm3, %ymm0, %ymm0
-; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512F-NEXT:    vporq %zmm0, %zmm2, %zmm0
+; AVX512F-NEXT:    vpsrlw $4, %ymm2, %ymm2
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512F-NEXT:    vpternlogq $216, {{.*}}(%rip), %zmm1, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatconstant_rotate_v64i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
-; AVX512VL-NEXT:    vpsllw $4, %ymm1, %ymm2
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX512VL-NEXT:    vpand %ymm3, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm4
-; AVX512VL-NEXT:    vpand %ymm3, %ymm4, %ymm3
-; AVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm2
-; AVX512VL-NEXT:    vpsrlw $4, %ymm1, %ymm1
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512VL-NEXT:    vpand %ymm3, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm1
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512VL-NEXT:    vpsllw $4, %ymm2, %ymm3
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm1
 ; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpand %ymm3, %ymm0, %ymm0
-; AVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512VL-NEXT:    vporq %zmm0, %zmm2, %zmm0
+; AVX512VL-NEXT:    vpsrlw $4, %ymm2, %ymm2
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512VL-NEXT:    vpternlogq $216, {{.*}}(%rip), %zmm1, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatconstant_rotate_v64i8:
@@ -947,40 +933,28 @@ define <32 x i16> @splatconstant_rotate_mask_v32i16(<32 x i16> %a) nounwind {
 define <64 x i8> @splatconstant_rotate_mask_v64i8(<64 x i8> %a) nounwind {
 ; AVX512F-LABEL: splatconstant_rotate_mask_v64i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
-; AVX512F-NEXT:    vpsllw $4, %ymm1, %ymm2
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX512F-NEXT:    vpand %ymm3, %ymm2, %ymm2
-; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm4
-; AVX512F-NEXT:    vpand %ymm3, %ymm4, %ymm3
-; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm2
-; AVX512F-NEXT:    vpsrlw $4, %ymm1, %ymm1
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512F-NEXT:    vpand %ymm3, %ymm1, %ymm1
+; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm1
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512F-NEXT:    vpsllw $4, %ymm2, %ymm3
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm1
 ; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm0
-; AVX512F-NEXT:    vpand %ymm3, %ymm0, %ymm0
-; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm1
-; AVX512F-NEXT:    vpandq {{.*}}(%rip), %zmm2, %zmm0
-; AVX512F-NEXT:    vpternlogq $248, {{.*}}(%rip), %zmm1, %zmm0
+; AVX512F-NEXT:    vpsrlw $4, %ymm2, %ymm2
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm2
+; AVX512F-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm0
+; AVX512F-NEXT:    vpternlogq $248, {{.*}}(%rip), %zmm2, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatconstant_rotate_mask_v64i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
-; AVX512VL-NEXT:    vpsllw $4, %ymm1, %ymm2
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX512VL-NEXT:    vpand %ymm3, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm4
-; AVX512VL-NEXT:    vpand %ymm3, %ymm4, %ymm3
-; AVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm2
-; AVX512VL-NEXT:    vpsrlw $4, %ymm1, %ymm1
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512VL-NEXT:    vpand %ymm3, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm1
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512VL-NEXT:    vpsllw $4, %ymm2, %ymm3
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm1
 ; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpand %ymm3, %ymm0, %ymm0
-; AVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm1
-; AVX512VL-NEXT:    vpandq {{.*}}(%rip), %zmm2, %zmm0
-; AVX512VL-NEXT:    vpternlogq $248, {{.*}}(%rip), %zmm1, %zmm0
+; AVX512VL-NEXT:    vpsrlw $4, %ymm2, %ymm2
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm2
+; AVX512VL-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm0
+; AVX512VL-NEXT:    vpternlogq $248, {{.*}}(%rip), %zmm2, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatconstant_rotate_mask_v64i8:
diff --git a/llvm/test/CodeGen/X86/vector-shift-lshr-512.ll b/llvm/test/CodeGen/X86/vector-shift-lshr-512.ll
index 7cdeb29d35842..c311f138e789b 100644
--- a/llvm/test/CodeGen/X86/vector-shift-lshr-512.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-lshr-512.ll
@@ -156,14 +156,14 @@ define <64 x i8> @splatvar_shift_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
 ; AVX512DQ-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
 ; AVX512DQ-NEXT:    vpsrlw %xmm1, %ymm2, %ymm2
-; AVX512DQ-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX512DQ-NEXT:    vpsrlw %xmm1, %xmm3, %xmm3
-; AVX512DQ-NEXT:    vpsrlw $8, %xmm3, %xmm3
-; AVX512DQ-NEXT:    vpbroadcastb %xmm3, %ymm3
-; AVX512DQ-NEXT:    vpand %ymm3, %ymm2, %ymm2
 ; AVX512DQ-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpand %ymm3, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX512DQ-NEXT:    vpsrlw %xmm1, %xmm2, %xmm1
+; AVX512DQ-NEXT:    vpsrlw $8, %xmm1, %xmm1
+; AVX512DQ-NEXT:    vpbroadcastb %xmm1, %ymm1
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm1, %zmm1
+; AVX512DQ-NEXT:    vpandq %zmm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatvar_shift_v64i8:
@@ -308,13 +308,11 @@ define <32 x i16> @splatconstant_shift_v32i16(<32 x i16> %a) nounwind {
 define <64 x i8> @splatconstant_shift_v64i8(<64 x i8> %a) nounwind {
 ; AVX512DQ-LABEL: splatconstant_shift_v64i8:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
-; AVX512DQ-NEXT:    vpsrlw $3, %ymm1, %ymm1
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm2 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31]
-; AVX512DQ-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpsrlw $3, %ymm0, %ymm1
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
 ; AVX512DQ-NEXT:    vpsrlw $3, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpand %ymm2, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512DQ-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatconstant_shift_v64i8:
diff --git a/llvm/test/CodeGen/X86/vector-shift-shl-512.ll b/llvm/test/CodeGen/X86/vector-shift-shl-512.ll
index 91146cfdf1d42..1bf878739bf8b 100644
--- a/llvm/test/CodeGen/X86/vector-shift-shl-512.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-shl-512.ll
@@ -151,13 +151,13 @@ define <64 x i8> @splatvar_shift_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
 ; AVX512DQ-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
 ; AVX512DQ-NEXT:    vpsllw %xmm1, %ymm2, %ymm2
-; AVX512DQ-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX512DQ-NEXT:    vpsllw %xmm1, %xmm3, %xmm3
-; AVX512DQ-NEXT:    vpbroadcastb %xmm3, %ymm3
-; AVX512DQ-NEXT:    vpand %ymm3, %ymm2, %ymm2
 ; AVX512DQ-NEXT:    vpsllw %xmm1, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpand %ymm3, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX512DQ-NEXT:    vpsllw %xmm1, %xmm2, %xmm1
+; AVX512DQ-NEXT:    vpbroadcastb %xmm1, %ymm1
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm1, %zmm1
+; AVX512DQ-NEXT:    vpandq %zmm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatvar_shift_v64i8:
@@ -306,13 +306,11 @@ define <32 x i16> @splatconstant_shift_v32i16(<32 x i16> %a) nounwind {
 define <64 x i8> @splatconstant_shift_v64i8(<64 x i8> %a) nounwind {
 ; AVX512DQ-LABEL: splatconstant_shift_v64i8:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
-; AVX512DQ-NEXT:    vpsllw $3, %ymm1, %ymm1
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm2 = [248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248]
-; AVX512DQ-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpsllw $3, %ymm0, %ymm1
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
 ; AVX512DQ-NEXT:    vpsllw $3, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpand %ymm2, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512DQ-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatconstant_shift_v64i8:

From 05b44f7eaebfbca19999fde149c4c586fc965015 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Sat, 1 Aug 2020 20:38:38 +0100
Subject: [PATCH 0820/1035] [LCSSA] Provide option for caller to clean up
 unused PHIs.

formLCSSAForInstructions is used by SCEVExpander, which tracks all
inserted instructions including LCSSA phis using asserting value
handles. This means cleanup needs to happen in the caller.

Extend formLCSSAForInstructions  to take an optional pointer to a
vector. If this argument is non-nullptr, instead of directly deleting
the phis, add them to the vector, so the caller can process them.

This should address various PPC buildbot failures, including
http://lab.llvm.org:8011/builders/clang-ppc64be-linux-lnt/builds/40567
---
 .../include/llvm/Transforms/Utils/LoopUtils.h |  11 +-
 llvm/lib/Transforms/Utils/LCSSA.cpp           |  34 +++---
 .../Utils/ScalarEvolutionExpander.cpp         |  11 +-
 .../CodeGen/PowerPC/hardware-loops-crash.ll   | 101 ++++++++++++++++++
 4 files changed, 139 insertions(+), 18 deletions(-)
 create mode 100644 llvm/test/CodeGen/PowerPC/hardware-loops-crash.ll

diff --git a/llvm/include/llvm/Transforms/Utils/LoopUtils.h b/llvm/include/llvm/Transforms/Utils/LoopUtils.h
index c6a8b27811ed1..70c8c84c857bf 100644
--- a/llvm/include/llvm/Transforms/Utils/LoopUtils.h
+++ b/llvm/include/llvm/Transforms/Utils/LoopUtils.h
@@ -74,9 +74,14 @@ bool formDedicatedExitBlocks(Loop *L, DominatorTree *DT, LoopInfo *LI,
 /// changes to CFG, preserved.
 ///
 /// Returns true if any modifications are made.
-bool formLCSSAForInstructions(SmallVectorImpl<Instruction *> &Worklist,
-                              const DominatorTree &DT, const LoopInfo &LI,
-                              ScalarEvolution *SE, IRBuilderBase &Builder);
+///
+/// This function may introduce unused PHI nodes. If \p PHIsToRemove is not
+/// nullptr, those are added to it (before removing, the caller has to check if
+/// they still do not have any uses). Otherwise the PHIs are directly removed.
+bool formLCSSAForInstructions(
+    SmallVectorImpl<Instruction *> &Worklist, const DominatorTree &DT,
+    const LoopInfo &LI, ScalarEvolution *SE, IRBuilderBase &Builder,
+    SmallVectorImpl<PHINode *> *PHIsToRemove = nullptr);
 
 /// Put loop into LCSSA form.
 ///
diff --git a/llvm/lib/Transforms/Utils/LCSSA.cpp b/llvm/lib/Transforms/Utils/LCSSA.cpp
index 9c606251ae0f8..630aadadbbce6 100644
--- a/llvm/lib/Transforms/Utils/LCSSA.cpp
+++ b/llvm/lib/Transforms/Utils/LCSSA.cpp
@@ -78,10 +78,10 @@ static bool isExitBlock(BasicBlock *BB,
 /// rewrite the uses.
 bool llvm::formLCSSAForInstructions(SmallVectorImpl<Instruction *> &Worklist,
                                     const DominatorTree &DT, const LoopInfo &LI,
-                                    ScalarEvolution *SE,
-                                    IRBuilderBase &Builder) {
+                                    ScalarEvolution *SE, IRBuilderBase &Builder,
+                                    SmallVectorImpl<PHINode *> *PHIsToRemove) {
   SmallVector<Use *, 16> UsesToRewrite;
-  SmallSetVector<PHINode *, 16> PHIsToRemove;
+  SmallSetVector<PHINode *, 16> LocalPHIsToRemove;
   PredIteratorCache PredCache;
   bool Changed = false;
 
@@ -257,22 +257,28 @@ bool llvm::formLCSSAForInstructions(SmallVectorImpl<Instruction *> &Worklist,
     SmallVector<PHINode *, 2> NeedDbgValues;
     for (PHINode *PN : AddedPHIs)
       if (PN->use_empty())
-        PHIsToRemove.insert(PN);
+        LocalPHIsToRemove.insert(PN);
       else
         NeedDbgValues.push_back(PN);
     insertDebugValuesForPHIs(InstBB, NeedDbgValues);
     Changed = true;
   }
-  // Remove PHI nodes that did not have any uses rewritten. We need to redo the
-  // use_empty() check here, because even if the PHI node wasn't used when added
-  // to PHIsToRemove, later added PHI nodes can be using it.  This cleanup is
-  // not guaranteed to handle trees/cycles of PHI nodes that only are used by
-  // each other. Such situations has only been noticed when the input IR
-  // contains unreachable code, and leaving some extra redundant PHI nodes in
-  // such situations is considered a minor problem.
-  for (PHINode *PN : PHIsToRemove)
-    if (PN->use_empty())
-      PN->eraseFromParent();
+
+  // Remove PHI nodes that did not have any uses rewritten or add them to
+  // PHIsToRemove, so the caller can remove them after some additional cleanup.
+  // We need to redo the use_empty() check here, because even if the PHI node
+  // wasn't used when added to LocalPHIsToRemove, later added PHI nodes can be
+  // using it.  This cleanup is not guaranteed to handle trees/cycles of PHI
+  // nodes that only are used by each other. Such situations has only been
+  // noticed when the input IR contains unreachable code, and leaving some extra
+  // redundant PHI nodes in such situations is considered a minor problem.
+  if (PHIsToRemove) {
+    PHIsToRemove->append(LocalPHIsToRemove.begin(), LocalPHIsToRemove.end());
+  } else {
+    for (PHINode *PN : LocalPHIsToRemove)
+      if (PN->use_empty())
+        PN->eraseFromParent();
+  }
   return Changed;
 }
 
diff --git a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
index a8302b7ccfc1a..aaa28feb32b86 100644
--- a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
+++ b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
@@ -2537,7 +2537,16 @@ Value *SCEVExpander::fixupLCSSAFormFor(Instruction *User, unsigned OpIdx) {
     return OpV;
 
   ToUpdate.push_back(OpI);
-  formLCSSAForInstructions(ToUpdate, SE.DT, SE.LI, &SE, Builder);
+  SmallVector<PHINode *, 16> PHIsToRemove;
+  formLCSSAForInstructions(ToUpdate, SE.DT, SE.LI, &SE, Builder, &PHIsToRemove);
+  for (PHINode *PN : PHIsToRemove) {
+    if (!PN->use_empty())
+      continue;
+    InsertedValues.erase(PN);
+    InsertedPostIncValues.erase(PN);
+    PN->eraseFromParent();
+  }
+
   return User->getOperand(OpIdx);
 }
 
diff --git a/llvm/test/CodeGen/PowerPC/hardware-loops-crash.ll b/llvm/test/CodeGen/PowerPC/hardware-loops-crash.ll
new file mode 100644
index 0000000000000..24e9592a6c2be
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/hardware-loops-crash.ll
@@ -0,0 +1,101 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -hardware-loops -S -verify-loop-lcssa | FileCheck %s
+
+target datalayout = "E-m:e-i64:64-n32:64"
+target triple = "ppc64-unknown-linux-elf"
+
+declare i1 @cond() readnone
+
+; Make sure we do not crash on the test.
+
+define void @test() {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[WHILE_COND:%.*]]
+; CHECK:       while.cond:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    br label [[FOR_INC:%.*]]
+; CHECK:       for.inc:
+; CHECK-NEXT:    [[C_0:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[C_0]], label [[WHILE_COND25:%.*]], label [[FOR_BODY]]
+; CHECK:       while.cond25:
+; CHECK-NEXT:    [[INDVAR:%.*]] = phi i64 [ [[INDVAR_NEXT:%.*]], [[LAND_RHS:%.*]] ], [ 0, [[FOR_INC]] ]
+; CHECK-NEXT:    [[INDVARS_IV349:%.*]] = phi i64 [ [[INDVARS_IV_NEXT350:%.*]], [[LAND_RHS]] ], [ 50, [[FOR_INC]] ]
+; CHECK-NEXT:    [[CMP26_NOT:%.*]] = icmp eq i64 [[INDVARS_IV349]], 0
+; CHECK-NEXT:    br i1 [[CMP26_NOT]], label [[WHILE_END187:%.*]], label [[LAND_RHS]]
+; CHECK:       land.rhs:
+; CHECK-NEXT:    [[INDVARS_IV_NEXT350]] = add nsw i64 [[INDVARS_IV349]], -1
+; CHECK-NEXT:    [[C_1:%.*]] = call i1 @cond()
+; CHECK-NEXT:    [[INDVAR_NEXT]] = add i64 [[INDVAR]], 1
+; CHECK-NEXT:    br i1 [[C_1]], label [[WHILE_COND25]], label [[WHILE_END:%.*]]
+; CHECK:       while.end:
+; CHECK-NEXT:    [[INDVAR_LCSSA1:%.*]] = phi i64 [ [[INDVAR]], [[LAND_RHS]] ]
+; CHECK-NEXT:    [[C_2:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[C_2]], label [[WHILE_END187]], label [[WHILE_COND35_PREHEADER:%.*]]
+; CHECK:       while.cond35.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = mul nsw i64 [[INDVAR_LCSSA1]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[TMP0]], 51
+; CHECK-NEXT:    call void @llvm.set.loop.iterations.i64(i64 [[TMP1]])
+; CHECK-NEXT:    br label [[WHILE_COND35:%.*]]
+; CHECK:       while.cond35:
+; CHECK-NEXT:    [[TMP2:%.*]] = call i1 @llvm.loop.decrement.i64(i64 1)
+; CHECK-NEXT:    br i1 [[TMP2]], label [[LAND_RHS37:%.*]], label [[IF_END51:%.*]]
+; CHECK:       land.rhs37:
+; CHECK-NEXT:    br label [[WHILE_COND35]]
+; CHECK:       if.end51:
+; CHECK-NEXT:    br label [[WHILE_COND_BACKEDGE:%.*]]
+; CHECK:       while.cond.backedge:
+; CHECK-NEXT:    br label [[WHILE_COND]]
+; CHECK:       while.end187:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %while.cond
+
+while.cond:                                       ; preds = %while.cond.backedge, %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.inc, %while.cond
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body
+  %c.0 = call i1 @cond()
+  br i1 %c.0, label %while.cond25, label %for.body
+
+while.cond25:                                     ; preds = %land.rhs, %for.inc
+  %indvars.iv349 = phi i64 [ %indvars.iv.next350, %land.rhs ], [ 50, %for.inc ]
+  %cmp26.not = icmp eq i64 %indvars.iv349, 0
+  br i1 %cmp26.not, label %while.end187, label %land.rhs
+
+land.rhs:                                         ; preds = %while.cond25
+  %indvars.iv.next350 = add nsw i64 %indvars.iv349, -1
+  %c.1 = call i1 @cond()
+  br i1 %c.1, label %while.cond25, label %while.end
+
+while.end:                                        ; preds = %land.rhs
+  %c.2 = call i1 @cond()
+  br i1 %c.2, label %while.end187, label %while.cond35.preheader
+
+while.cond35.preheader:                           ; preds = %while.end
+  %0 = and i64 %indvars.iv349, 4294967295
+  br label %while.cond35
+
+while.cond35:                                     ; preds = %land.rhs37, %while.cond35.preheader
+  %indvars.iv351 = phi i64 [ %0, %while.cond35.preheader ], [ %indvars.iv.next352, %land.rhs37 ]
+  %cmp36 = icmp sgt i64 %indvars.iv351, 0
+  br i1 %cmp36, label %land.rhs37, label %if.end51
+
+land.rhs37:                                       ; preds = %while.cond35
+  %indvars.iv.next352 = add nsw i64 %indvars.iv351, -1
+  br label %while.cond35
+
+if.end51:                                         ; preds = %while.cond35
+  br label %while.cond.backedge
+
+while.cond.backedge:                              ; preds = %if.end51
+  br label %while.cond
+
+while.end187:                                     ; preds = %while.end, %while.cond25
+  ret void
+}

From 95ddb9ff673001b2745c871c5751d165c2a27546 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Sat, 1 Aug 2020 20:58:05 +0100
Subject: [PATCH 0821/1035] [PPC] Adjust run line for hardware-loops-crash.ll

Looks like %s was accidentally dropped.
---
 llvm/test/CodeGen/PowerPC/hardware-loops-crash.ll | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/test/CodeGen/PowerPC/hardware-loops-crash.ll b/llvm/test/CodeGen/PowerPC/hardware-loops-crash.ll
index 24e9592a6c2be..23e5b44fa87d1 100644
--- a/llvm/test/CodeGen/PowerPC/hardware-loops-crash.ll
+++ b/llvm/test/CodeGen/PowerPC/hardware-loops-crash.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -hardware-loops -S -verify-loop-lcssa | FileCheck %s
+; RUN: opt -hardware-loops -S -verify-loop-lcssa %s | FileCheck %s
 
 target datalayout = "E-m:e-i64:64-n32:64"
 target triple = "ppc64-unknown-linux-elf"

From 4a19e6156ed5b6e87d708e6de29b675be69c574f Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@intel.com>
Date: Sat, 1 Aug 2020 09:59:09 -0700
Subject: [PATCH 0822/1035] [InstCombine] Fold abs(-x) -> abs(x)

Negating the input doesn't matter. I left a FIXME to copy the nsw flag if its present on the neg but not on the abs.

Reviewed By: nikic

Differential Revision: https://reviews.llvm.org/D85055
---
 .../InstCombine/InstCombineCalls.cpp          | 10 ++++++++++
 .../Transforms/InstCombine/abs-intrinsic.ll   | 20 +++++++++++++++++++
 2 files changed, 30 insertions(+)

diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
index ed93c33c1a597..4eb3e2e4434fd 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -769,6 +769,16 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
     if (Value *V = lowerObjectSizeCall(II, DL, &TLI, /*MustSucceed=*/false))
       return replaceInstUsesWith(CI, V);
     return nullptr;
+  case Intrinsic::abs: {
+    Value *IIOperand = II->getArgOperand(0);
+    // abs(-x) -> abs(x)
+    // TODO: Copy nsw if it was present on the neg?
+    Value *X;
+    if (match(IIOperand, m_Neg(m_Value(X))))
+      return replaceOperand(*II, 0, X);
+
+    break;
+  }
   case Intrinsic::bswap: {
     Value *IIOperand = II->getArgOperand(0);
     Value *X = nullptr;
diff --git a/llvm/test/Transforms/InstCombine/abs-intrinsic.ll b/llvm/test/Transforms/InstCombine/abs-intrinsic.ll
index ed845cc842677..9e64aea9dabe5 100644
--- a/llvm/test/Transforms/InstCombine/abs-intrinsic.ll
+++ b/llvm/test/Transforms/InstCombine/abs-intrinsic.ll
@@ -165,3 +165,23 @@ define <4 x i1> @abs_known_not_int_min_vec(<4 x i32> %x) {
   %c2 = icmp sge <4 x i32> %abs, zeroinitializer
   ret <4 x i1> %c2
 }
+
+define i32 @abs_of_neg(i32 %x) {
+; CHECK-LABEL: @abs_of_neg(
+; CHECK-NEXT:    [[B:%.*]] = call i32 @llvm.abs.i32(i32 [[X:%.*]], i1 false)
+; CHECK-NEXT:    ret i32 [[B]]
+;
+  %a = sub i32 0, %x
+  %b = call i32 @llvm.abs.i32(i32 %a, i1 false)
+  ret i32 %b
+}
+
+define <4 x i32> @abs_of_neg_vec(<4 x i32> %x) {
+; CHECK-LABEL: @abs_of_neg_vec(
+; CHECK-NEXT:    [[B:%.*]] = call <4 x i32> @llvm.abs.v4i32(<4 x i32> [[X:%.*]], i1 false)
+; CHECK-NEXT:    ret <4 x i32> [[B]]
+;
+  %a = sub nsw <4 x i32> zeroinitializer, %x
+  %b = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %a, i1 false)
+  ret <4 x i32> %b
+}

From 85b5315dbe9d52766ab326e702d638fcf58579b5 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@intel.com>
Date: Sat, 1 Aug 2020 10:01:58 -0700
Subject: [PATCH 0823/1035] [InstSimplify] Fold abs(abs(x)) -> abs(x)

It's always safe to pick the earlier abs regardless of the nsw flag. We'll just lose it if it is on the outer abs but not the inner abs.

Differential Revision: https://reviews.llvm.org/D85053
---
 llvm/lib/Analysis/InstructionSimplify.cpp |  5 +++
 llvm/test/Transforms/InstSimplify/call.ll | 40 +++++++++++++++++++++++
 2 files changed, 45 insertions(+)

diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp
index d3928a502965b..b1438b416d56b 100644
--- a/llvm/lib/Analysis/InstructionSimplify.cpp
+++ b/llvm/lib/Analysis/InstructionSimplify.cpp
@@ -5256,6 +5256,11 @@ static Value *simplifyBinaryIntrinsic(Function *F, Value *Op0, Value *Op1,
   unsigned BitWidth = ReturnType->getScalarSizeInBits();
   switch (IID) {
   case Intrinsic::abs:
+    // abs(abs(x)) -> abs(x). We don't need to worry about the nsw arg here.
+    // It is always ok to pick the earlier abs. We'll just lose nsw if its only
+    // on the outer abs.
+    if (match(Op0, m_Intrinsic<Intrinsic::abs>(m_Value(), m_Value())))
+      return Op0;
     // If the sign bit is clear already, then abs does not do anything.
     if (isKnownNonNegative(Op0, Q.DL, 0, Q.AC, Q.CxtI, Q.DT))
       return Op0;
diff --git a/llvm/test/Transforms/InstSimplify/call.ll b/llvm/test/Transforms/InstSimplify/call.ll
index 2325dccd17a85..344cefd5b2c3a 100644
--- a/llvm/test/Transforms/InstSimplify/call.ll
+++ b/llvm/test/Transforms/InstSimplify/call.ll
@@ -5,6 +5,46 @@
 declare i32 @llvm.abs.i32(i32, i1)
 declare <3 x i82> @llvm.abs.v3i82(<3 x i82>, i1)
 
+define i32 @test_abs_abs_0(i32 %x) {
+; CHECK-LABEL: @test_abs_abs_0(
+; CHECK-NEXT:    [[A:%.*]] = call i32 @llvm.abs.i32(i32 [[X:%.*]], i1 false)
+; CHECK-NEXT:    ret i32 [[A]]
+;
+  %a = call i32 @llvm.abs.i32(i32 %x, i1 false)
+  %b = call i32 @llvm.abs.i32(i32 %a, i1 false)
+  ret i32 %b
+}
+
+define i32 @test_abs_abs_1(i32 %x) {
+; CHECK-LABEL: @test_abs_abs_1(
+; CHECK-NEXT:    [[A:%.*]] = call i32 @llvm.abs.i32(i32 [[X:%.*]], i1 true)
+; CHECK-NEXT:    ret i32 [[A]]
+;
+  %a = call i32 @llvm.abs.i32(i32 %x, i1 true)
+  %b = call i32 @llvm.abs.i32(i32 %a, i1 false)
+  ret i32 %b
+}
+
+define i32 @test_abs_abs_2(i32 %x) {
+; CHECK-LABEL: @test_abs_abs_2(
+; CHECK-NEXT:    [[A:%.*]] = call i32 @llvm.abs.i32(i32 [[X:%.*]], i1 false)
+; CHECK-NEXT:    ret i32 [[A]]
+;
+  %a = call i32 @llvm.abs.i32(i32 %x, i1 false)
+  %b = call i32 @llvm.abs.i32(i32 %a, i1 true)
+  ret i32 %b
+}
+
+define i32 @test_abs_abs_3(i32 %x) {
+; CHECK-LABEL: @test_abs_abs_3(
+; CHECK-NEXT:    [[A:%.*]] = call i32 @llvm.abs.i32(i32 [[X:%.*]], i1 true)
+; CHECK-NEXT:    ret i32 [[A]]
+;
+  %a = call i32 @llvm.abs.i32(i32 %x, i1 true)
+  %b = call i32 @llvm.abs.i32(i32 %a, i1 true)
+  ret i32 %b
+}
+
 ; If the sign bit is known zero, the abs is not needed.
 
 define i32 @zext_abs(i31 %x) {

From e297d928dcde31ac92eff72532095f4f657f2ebd Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@intel.com>
Date: Sat, 1 Aug 2020 13:25:18 -0700
Subject: [PATCH 0824/1035] [X86] Add assembler support for {disp8} and
 {disp32} to control the size of displacement used for memory operands.

These prefixes should override the default behavior and force a larger immediate size. I don't believe gas issues any warning if you use {disp8} when a 32-bit displacement is already required. And this patch doesn't either.

This completes the {disp8} and {disp32} support from PR46650.

Reviewed By: RKSimon

Differential Revision: https://reviews.llvm.org/D84793
---
 .../lib/Target/X86/AsmParser/X86AsmParser.cpp | 12 +++
 .../lib/Target/X86/MCTargetDesc/X86BaseInfo.h |  2 +
 .../X86/MCTargetDesc/X86MCCodeEmitter.cpp     | 59 ++++++++-----
 llvm/test/MC/X86/x86-64.s                     | 88 +++++++++++++++++++
 4 files changed, 137 insertions(+), 24 deletions(-)

diff --git a/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp b/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
index 6b4f2e33f67de..49c01d7b9ef0b 100644
--- a/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
+++ b/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
@@ -3572,6 +3572,12 @@ bool X86AsmParser::MatchAndEmitATTInstruction(SMLoc IDLoc, unsigned &Opcode,
   if (ForcedVEXEncoding == VEXEncoding_VEX3)
     Prefixes |= X86::IP_USE_VEX3;
 
+  // Set encoded flags for {disp8} and {disp32}.
+  if (ForcedDispEncoding == DispEncoding_Disp8)
+    Prefixes |= X86::IP_USE_DISP8;
+  else if (ForcedDispEncoding == DispEncoding_Disp32)
+    Prefixes |= X86::IP_USE_DISP32;
+
   if (Prefixes)
     Inst.setFlags(Prefixes);
 
@@ -3806,6 +3812,12 @@ bool X86AsmParser::MatchAndEmitIntelInstruction(SMLoc IDLoc, unsigned &Opcode,
   if (ForcedVEXEncoding == VEXEncoding_VEX3)
     Prefixes |= X86::IP_USE_VEX3;
 
+  // Set encoded flags for {disp8} and {disp32}.
+  if (ForcedDispEncoding == DispEncoding_Disp8)
+    Prefixes |= X86::IP_USE_DISP8;
+  else if (ForcedDispEncoding == DispEncoding_Disp32)
+    Prefixes |= X86::IP_USE_DISP32;
+
   if (Prefixes)
     Inst.setFlags(Prefixes);
 
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86BaseInfo.h b/llvm/lib/Target/X86/MCTargetDesc/X86BaseInfo.h
index 79f07d3c7792a..b6e8d4813d4ca 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86BaseInfo.h
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86BaseInfo.h
@@ -62,6 +62,8 @@ namespace X86 {
     IP_HAS_LOCK = 16,
     IP_HAS_NOTRACK = 32,
     IP_USE_VEX3 = 64,
+    IP_USE_DISP8 = 128,
+    IP_USE_DISP32 = 256,
   };
 
   enum OperandType : unsigned {
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp
index abdc0f156b9f9..0de94cda2d739 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp
@@ -505,12 +505,18 @@ void X86MCCodeEmitter::emitMemModRMByte(const MCInst &MI, unsigned Op,
     return;
   }
 
-  // Determine whether a SIB byte is needed.
-  // If no BaseReg, issue a RIP relative instruction only if the MCE can
-  // resolve addresses on-the-fly, otherwise use SIB (Intel Manual 2A, table
-  // 2-7) and absolute references.
+  // Check for presence of {disp8} or {disp32} pseudo prefixes.
+  bool UseDisp8 = MI.getFlags() & X86::IP_USE_DISP8;
+  bool UseDisp32 = MI.getFlags() & X86::IP_USE_DISP32;
+
+  // We only allow no displacement if no pseudo prefix is present.
+  bool AllowNoDisp = !UseDisp8 && !UseDisp32;
+  // Disp8 is allowed unless the {disp32} prefix is present.
+  bool AllowDisp8 = !UseDisp32;
 
-  if ( // The SIB byte must be used if there is an index register.
+  // Determine whether a SIB byte is needed.
+  if (// The SIB byte must be used if there is an index register or the
+      // encoding requires a SIB byte.
       !ForceSIB && IndexReg.getReg() == 0 &&
       // The SIB byte must be used if the base is ESP/RSP/R12, all of which
       // encode to an R/M value of 4, which indicates that a SIB byte is
@@ -526,12 +532,12 @@ void X86MCCodeEmitter::emitMemModRMByte(const MCInst &MI, unsigned Op,
       return;
     }
 
-    // If the base is not EBP/ESP and there is no displacement, use simple
-    // indirect register encoding, this handles addresses like [EAX].  The
-    // encoding for [EBP] with no displacement means [disp32] so we handle it
-    // by emitting a displacement of 0 below.
+    // If the base is not EBP/ESP/R12/R13 and there is no displacement, use
+    // simple indirect register encoding, this handles addresses like [EAX].
+    // The encoding for [EBP] or[R13] with no displacement means [disp32] so we
+    // handle it by emitting a displacement of 0 later.
     if (BaseRegNo != N86::EBP) {
-      if (Disp.isImm() && Disp.getImm() == 0) {
+      if (Disp.isImm() && Disp.getImm() == 0 && AllowNoDisp) {
         emitByte(modRMByte(0, RegOpcodeField, BaseRegNo), OS);
         return;
       }
@@ -550,7 +556,10 @@ void X86MCCodeEmitter::emitMemModRMByte(const MCInst &MI, unsigned Op,
     }
 
     // Otherwise, if the displacement fits in a byte, encode as [REG+disp8].
-    if (Disp.isImm()) {
+    // Including a compressed disp8 for EVEX instructions that support it.
+    // This also handles the 0 displacement for [EBP] or [R13]. We can't use
+    // disp8 if the {disp32} pseudo prefix is present.
+    if (Disp.isImm() && AllowDisp8) {
       int ImmOffset = 0;
       if (isDispOrCDisp8(TSFlags, Disp.getImm(), ImmOffset)) {
         emitByte(modRMByte(1, RegOpcodeField, BaseRegNo), OS);
@@ -560,7 +569,9 @@ void X86MCCodeEmitter::emitMemModRMByte(const MCInst &MI, unsigned Op,
       }
     }
 
-    // Otherwise, emit the most general non-SIB encoding: [REG+disp32]
+    // Otherwise, emit the most general non-SIB encoding: [REG+disp32].
+    // Displacement may be 0 for [EBP] or [R13] case if {disp32} pseudo prefix
+    // prevented using disp8 above.
     emitByte(modRMByte(2, RegOpcodeField, BaseRegNo), OS);
     unsigned Opcode = MI.getOpcode();
     unsigned FixupKind = Opcode == X86::MOV32rm ? X86::reloc_signed_4byte_relax
@@ -580,21 +591,26 @@ void X86MCCodeEmitter::emitMemModRMByte(const MCInst &MI, unsigned Op,
   if (BaseReg == 0) {
     // If there is no base register, we emit the special case SIB byte with
     // MOD=0, BASE=5, to JUST get the index, scale, and displacement.
+    BaseRegNo = 5;
     emitByte(modRMByte(0, RegOpcodeField, 4), OS);
     ForceDisp32 = true;
-  } else if (Disp.isImm() && Disp.getImm() == 0 &&
-             // Base reg can't be anything that ends up with '5' as the base
-             // reg, it is the magic [*] nomenclature that indicates no base.
+  } else if (Disp.isImm() && Disp.getImm() == 0 && AllowNoDisp &&
+             // Base reg can't be EBP/RBP/R13 as that would end up with '5' as
+             // the base field, but that is the magic [*] nomenclature that
+             // indicates no base when mod=0. For these cases we'll emit a 0
+             // displacement instead.
              BaseRegNo != N86::EBP) {
     // Emit no displacement ModR/M byte
     emitByte(modRMByte(0, RegOpcodeField, 4), OS);
-  } else if (Disp.isImm() &&
+  } else if (Disp.isImm() && AllowDisp8 &&
              isDispOrCDisp8(TSFlags, Disp.getImm(), ImmOffset)) {
-    // Emit the disp8 encoding.
+    // Displacement fits in a byte or matches an EVEX compressed disp8, use
+    // disp8 encoding. This also handles EBP/R13 base with 0 displacement unless
+    // {disp32} pseudo prefix was used.
     emitByte(modRMByte(1, RegOpcodeField, 4), OS);
-    ForceDisp8 = true; // Make sure to force 8 bit disp if Base=EBP
+    ForceDisp8 = true;
   } else {
-    // Emit the normal disp32 encoding.
+    // Otherwise, emit the normal disp32 encoding.
     emitByte(modRMByte(2, RegOpcodeField, 4), OS);
     ForceDisp32 = true;
   }
@@ -605,11 +621,6 @@ void X86MCCodeEmitter::emitMemModRMByte(const MCInst &MI, unsigned Op,
 
   unsigned IndexRegNo = IndexReg.getReg() ? getX86RegNum(IndexReg) : 4;
 
-  // Handle the SIB byte for the case where there is no base, see Intel
-  // Manual 2A, table 2-7. The displacement has already been output.
-  if (BaseReg == 0)
-    BaseRegNo = 5;
-
   emitSIBByte(SS, IndexRegNo, BaseRegNo, OS);
 
   // Do we need to output a displacement?
diff --git a/llvm/test/MC/X86/x86-64.s b/llvm/test/MC/X86/x86-64.s
index 38d5c6173f3b4..c61cae69c3ffe 100644
--- a/llvm/test/MC/X86/x86-64.s
+++ b/llvm/test/MC/X86/x86-64.s
@@ -1904,3 +1904,91 @@ ud1 %rdx, %rdi
 // CHECK: ud1q (%rbx), %rcx
 // CHECK:  encoding: [0x48,0x0f,0xb9,0x0b]
 ud2b (%rbx), %rcx
+
+// Requires no displacement by default
+// CHECK: movl $1, (%rax)
+// CHECK: encoding: [0xc7,0x00,0x01,0x00,0x00,0x00]
+// CHECK: movl $1, (%rax)
+// CHECK: encoding: [0xc7,0x40,0x00,0x01,0x00,0x00,0x00]
+// CHECK: movl $1, (%rax)
+// CHECK: encoding: [0xc7,0x80,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00]
+movl $1, (%rax)
+{disp8} movl $1, (%rax)
+{disp32} movl $1, (%rax)
+
+// Requires disp8 by default
+// CHECK: movl $1, (%rbp)
+// CHECK: encoding: [0xc7,0x45,0x00,0x01,0x00,0x00,0x00]
+// CHECK: movl $1, (%rbp)
+// CHECK: encoding: [0xc7,0x45,0x00,0x01,0x00,0x00,0x00]
+// CHECK: movl $1, (%rbp)
+// CHECK: encoding: [0xc7,0x85,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00]
+movl $1, (%rbp)
+{disp8} movl $1, (%rbp)
+{disp32} movl $1, (%rbp)
+
+// Requires disp8 by default
+// CHECK: movl $1, (%r13)
+// CHECK: encoding: [0x41,0xc7,0x45,0x00,0x01,0x00,0x00,0x00]
+// CHECK: movl $1, (%r13)
+// CHECK: encoding: [0x41,0xc7,0x45,0x00,0x01,0x00,0x00,0x00]
+// CHECK: movl $1, (%r13)
+// CHECK: encoding: [0x41,0xc7,0x85,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00]
+movl $1, (%r13)
+{disp8} movl $1, (%r13)
+{disp32} movl $1, (%r13)
+
+// Requires disp8 by default
+// CHECK: movl $1, 8(%rax)
+// CHECK: encoding: [0xc7,0x40,0x08,0x01,0x00,0x00,0x00]
+// CHECK: movl $1, 8(%rax)
+// CHECK: encoding: [0xc7,0x40,0x08,0x01,0x00,0x00,0x00]
+// CHECK: movl $1, 8(%rax)
+// CHECK: encoding: [0xc7,0x80,0x08,0x00,0x00,0x00,0x01,0x00,0x00,0x00]
+movl $1, 8(%rax)
+{disp8} movl $1, 8(%rax)
+{disp32} movl $1, 8(%rax)
+
+// Requires no displacement by default
+// CHECK: movl $1, (%rax,%rbx,4)
+// CHECK: encoding: [0xc7,0x04,0x98,0x01,0x00,0x00,0x00]
+// CHECK: movl $1, (%rax,%rbx,4)
+// CHECK: encoding: [0xc7,0x44,0x98,0x00,0x01,0x00,0x00,0x00]
+// CHECK: movl $1, (%rax,%rbx,4)
+// CHECK: encoding: [0xc7,0x84,0x98,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00]
+movl $1, (%rax,%rbx,4)
+{disp8} movl $1, (%rax,%rbx,4)
+{disp32} movl $1, (%rax,%rbx,4)
+
+// Requires disp8 by default.
+// CHECK: movl $1, 8(%rax,%rbx,4)
+// CHECK: encoding: [0xc7,0x44,0x98,0x08,0x01,0x00,0x00,0x00]
+// CHECK: movl $1, 8(%rax,%rbx,4)
+// CHECK: encoding: [0xc7,0x44,0x98,0x08,0x01,0x00,0x00,0x00]
+// CHECK: movl $1, 8(%rax,%rbx,4)
+// CHECK: encoding: [0xc7,0x84,0x98,0x08,0x00,0x00,0x00,0x01,0x00,0x00,0x00]
+movl $1, 8(%rax,%rbx,4)
+{disp8} movl $1, 8(%rax,%rbx,4)
+{disp32} movl $1, 8(%rax,%rbx,4)
+
+// Requires disp8 by default.
+// CHECK: movl $1, (%rbp,%rbx,4)
+// CHECK: encoding: [0xc7,0x44,0x9d,0x00,0x01,0x00,0x00,0x00]
+// CHECK: movl $1, (%rbp,%rbx,4)
+// CHECK: encoding: [0xc7,0x44,0x9d,0x00,0x01,0x00,0x00,0x00]
+// CHECK: movl $1, (%rbp,%rbx,4)
+// CHECK: encoding: [0xc7,0x84,0x9d,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00]
+movl $1, (%rbp,%rbx,4)
+{disp8} movl $1, (%rbp,%rbx,4)
+{disp32} movl $1, (%rbp,%rbx,4)
+
+// Requires disp8 by default.
+// CHECK: movl $1, (%r13,%rbx,4)
+// CHECK: encoding: [0x41,0xc7,0x44,0x9d,0x00,0x01,0x00,0x00,0x00]
+// CHECK: movl $1, (%r13,%rbx,4)
+// CHECK: encoding: [0x41,0xc7,0x44,0x9d,0x00,0x01,0x00,0x00,0x00]
+// CHECK: movl $1, (%r13,%rbx,4)
+// CHECK: encoding: [0x41,0xc7,0x84,0x9d,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00]
+movl $1, (%r13,%rbx,4)
+{disp8} movl $1, (%r13,%rbx,4)
+{disp32} movl $1, (%r13,%rbx,4)

From 25af353b0e74907d5d50c8616b885bd1f73a68b3 Mon Sep 17 00:00:00 2001
From: Nikita Popov <nikita.ppv@gmail.com>
Date: Thu, 30 Jul 2020 19:14:02 +0200
Subject: [PATCH 0825/1035] [NewPM][LVI] Abandon LVI after CVP

As mentioned on D70376, LVI can currently cause performance issues
when running under NewPM. The problem is that, unlike the legacy
pass manager, NewPM will not immediately discard the LVI analysis
if the following pass does not need it. This is a problem, because
LVI has a high memory requirement, and mass invalidation of LVI
values is very inefficient. LVI should only be alive during passes
that actively interact with it.

This patch addresses the issue by explicitly abandoning LVI after CVP,
which gets us back to the LegacyPM behavior.

Differential Revision: https://reviews.llvm.org/D84959
---
 .../test/CodeGen/thinlto-distributed-newpm.ll |  2 +-
 .../Scalar/CorrelatedValuePropagation.cpp     | 18 +++++++++++++-----
 llvm/test/Other/new-pm-defaults.ll            |  3 +++
 llvm/test/Other/new-pm-thinlto-defaults.ll    |  3 +++
 .../new-pm-thinlto-postlink-pgo-defaults.ll   |  3 +++
 ...-pm-thinlto-postlink-samplepgo-defaults.ll |  3 +++
 .../new-pm-thinlto-prelink-pgo-defaults.ll    | 19 +++++++++++++++++++
 ...w-pm-thinlto-prelink-samplepgo-defaults.ll |  3 +++
 8 files changed, 48 insertions(+), 6 deletions(-)

diff --git a/clang/test/CodeGen/thinlto-distributed-newpm.ll b/clang/test/CodeGen/thinlto-distributed-newpm.ll
index caf294df8eb85..9f9a8bec4ef5d 100644
--- a/clang/test/CodeGen/thinlto-distributed-newpm.ll
+++ b/clang/test/CodeGen/thinlto-distributed-newpm.ll
@@ -97,6 +97,7 @@
 ; CHECK-O: Running pass: JumpThreadingPass on main
 ; CHECK-O: Running analysis: LazyValueAnalysis on main
 ; CHECK-O: Running pass: CorrelatedValuePropagationPass on main
+; CHECK-O: Invalidating analysis: LazyValueAnalysis on main
 ; CHECK-O: Running pass: SimplifyCFGPass on main
 ; CHECK-O3: Running pass: AggressiveInstCombinePass on main
 ; CHECK-O: Running pass: InstCombinePass on main
@@ -144,7 +145,6 @@
 ; CHECK-O: Invalidating analysis: BasicAA on main
 ; CHECK-O: Invalidating analysis: AAManager on main
 ; CHECK-O: Invalidating analysis: MemorySSAAnalysis on main
-; CHECK-O: Invalidating analysis: LazyValueAnalysis on main
 ; CHECK-O: Invalidating analysis: LoopAnalysis on main
 ; CHECK-O: Invalidating analysis: PhiValuesAnalysis on main
 ; CHECK-O: Invalidating analysis: MemoryDependenceAnalysis on main
diff --git a/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp b/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp
index 48968166c605f..397d62a5d21d3 100644
--- a/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp
+++ b/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp
@@ -973,11 +973,19 @@ CorrelatedValuePropagationPass::run(Function &F, FunctionAnalysisManager &AM) {
 
   bool Changed = runImpl(F, LVI, DT, getBestSimplifyQuery(AM, F));
 
-  if (!Changed)
-    return PreservedAnalyses::all();
   PreservedAnalyses PA;
-  PA.preserve<GlobalsAA>();
-  PA.preserve<DominatorTreeAnalysis>();
-  PA.preserve<LazyValueAnalysis>();
+  if (!Changed) {
+    PA = PreservedAnalyses::all();
+  } else {
+    PA.preserve<GlobalsAA>();
+    PA.preserve<DominatorTreeAnalysis>();
+    PA.preserve<LazyValueAnalysis>();
+  }
+
+  // Keeping LVI alive is expensive, both because it uses a lot of memory, and
+  // because invalidating values in LVI is expensive. While CVP does preserve
+  // LVI, we know that passes after JumpThreading+CVP will not need the result
+  // of this analysis, so we forcefully discard it early.
+  PA.abandon<LazyValueAnalysis>();
   return PA;
 }
diff --git a/llvm/test/Other/new-pm-defaults.ll b/llvm/test/Other/new-pm-defaults.ll
index 8f75e3ce0bf36..59c24acb17f04 100644
--- a/llvm/test/Other/new-pm-defaults.ll
+++ b/llvm/test/Other/new-pm-defaults.ll
@@ -147,6 +147,7 @@
 ; CHECK-O23SZ-NEXT: Running pass: JumpThreadingPass
 ; CHECK-O23SZ-NEXT: Running analysis: LazyValueAnalysis
 ; CHECK-O23SZ-NEXT: Running pass: CorrelatedValuePropagationPass
+; CHECK-O23SZ-NEXT: Invalidating analysis: LazyValueAnalysis
 ; CHECK-O-NEXT: Running pass: SimplifyCFGPass
 ; CHECK-O3-NEXT: AggressiveInstCombinePass
 ; CHECK-O-NEXT: Running pass: InstCombinePass
@@ -200,7 +201,9 @@
 ; CHECK-O-NEXT: Running pass: InstCombinePass
 ; CHECK-EP-PEEPHOLE-NEXT: Running pass: NoOpFunctionPass
 ; CHECK-O23SZ-NEXT: Running pass: JumpThreadingPass
+; CHECK-O23SZ-NEXT: Running analysis: LazyValueAnalysis
 ; CHECK-O23SZ-NEXT: Running pass: CorrelatedValuePropagationPass
+; CHECK-O23SZ-NEXT: Invalidating analysis: LazyValueAnalysis
 ; CHECK-O23SZ-NEXT: Running pass: DSEPass
 ; CHECK-O23SZ-NEXT: Starting llvm::Function pass manager run.
 ; CHECK-O23SZ-NEXT: Running pass: LoopSimplifyPass
diff --git a/llvm/test/Other/new-pm-thinlto-defaults.ll b/llvm/test/Other/new-pm-thinlto-defaults.ll
index a39656cd26f66..0b9b52a57e2a5 100644
--- a/llvm/test/Other/new-pm-thinlto-defaults.ll
+++ b/llvm/test/Other/new-pm-thinlto-defaults.ll
@@ -112,6 +112,7 @@
 ; CHECK-O23SZ-NEXT: Running pass: JumpThreadingPass
 ; CHECK-O23SZ-NEXT: Running analysis: LazyValueAnalysis
 ; CHECK-O23SZ-NEXT: Running pass: CorrelatedValuePropagationPass
+; CHECK-O23SZ-NEXT: Invalidating analysis: LazyValueAnalysis
 ; CHECK-O-NEXT: Running pass: SimplifyCFGPass
 ; CHECK-O3-NEXT: Running pass: AggressiveInstCombinePass
 ; CHECK-O-NEXT: Running pass: InstCombinePass
@@ -173,7 +174,9 @@
 ; CHECK-O-NEXT: Running analysis: DemandedBitsAnalysis
 ; CHECK-O-NEXT: Running pass: InstCombinePass
 ; CHECK-O23SZ-NEXT: Running pass: JumpThreadingPass
+; CHECK-O23SZ-NEXT: Running analysis: LazyValueAnalysis
 ; CHECK-O23SZ-NEXT: Running pass: CorrelatedValuePropagationPass
+; CHECK-O23SZ-NEXT: Invalidating analysis: LazyValueAnalysis
 ; CHECK-O23SZ-NEXT: Running pass: DSEPass
 ; CHECK-O23SZ-NEXT: Starting llvm::Function pass manager run
 ; CHECK-O23SZ-NEXT: Running pass: LoopSimplifyPass
diff --git a/llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll b/llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll
index 333f4c0d413ed..7efc5357253e8 100644
--- a/llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll
+++ b/llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll
@@ -86,6 +86,7 @@
 ; CHECK-O23SZ-NEXT: Running pass: JumpThreadingPass
 ; CHECK-O23SZ-NEXT: Running analysis: LazyValueAnalysis
 ; CHECK-O23SZ-NEXT: Running pass: CorrelatedValuePropagationPass
+; CHECK-O23SZ-NEXT: Invalidating analysis: LazyValueAnalysis
 ; CHECK-O-NEXT: Running pass: SimplifyCFGPass
 ; CHECK-O3-NEXT: Running pass: AggressiveInstCombinePass
 ; CHECK-O-NEXT: Running pass: InstCombinePass
@@ -146,7 +147,9 @@
 ; CHECK-O-NEXT: Running analysis: DemandedBitsAnalysis
 ; CHECK-O-NEXT: Running pass: InstCombinePass
 ; CHECK-O23SZ-NEXT: Running pass: JumpThreadingPass
+; CHECK-O23SZ-NEXT: Running analysis: LazyValueAnalysis
 ; CHECK-O23SZ-NEXT: Running pass: CorrelatedValuePropagationPass
+; CHECK-O23SZ-NEXT: Invalidating analysis: LazyValueAnalysis
 ; CHECK-O23SZ-NEXT: Running pass: DSEPass
 ; CHECK-O23SZ-NEXT: Starting {{.*}}Function pass manager run
 ; CHECK-O23SZ-NEXT: Running pass: LoopSimplifyPass
diff --git a/llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll b/llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll
index 360f7ee07037f..9c5e36c5886ac 100644
--- a/llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll
+++ b/llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll
@@ -94,6 +94,7 @@
 ; CHECK-O23SZ-NEXT: Running pass: JumpThreadingPass
 ; CHECK-O23SZ-NEXT: Running analysis: LazyValueAnalysis
 ; CHECK-O23SZ-NEXT: Running pass: CorrelatedValuePropagationPass
+; CHECK-O23SZ-NEXT: Invalidating analysis: LazyValueAnalysis
 ; CHECK-O-NEXT: Running pass: SimplifyCFGPass
 ; CHECK-O3-NEXT: Running pass: AggressiveInstCombinePass
 ; CHECK-O-NEXT: Running pass: InstCombinePass
@@ -154,7 +155,9 @@
 ; CHECK-O-NEXT: Running analysis: DemandedBitsAnalysis
 ; CHECK-O-NEXT: Running pass: InstCombinePass
 ; CHECK-O23SZ-NEXT: Running pass: JumpThreadingPass
+; CHECK-O23SZ-NEXT: Running analysis: LazyValueAnalysis
 ; CHECK-O23SZ-NEXT: Running pass: CorrelatedValuePropagationPass
+; CHECK-O23SZ-NEXT: Invalidating analysis: LazyValueAnalysis
 ; CHECK-O23SZ-NEXT: Running pass: DSEPass
 ; CHECK-O23SZ-NEXT: Starting {{.*}}Function pass manager run
 ; CHECK-O23SZ-NEXT: Running pass: LoopSimplifyPass
diff --git a/llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll b/llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll
index f94b0992431fd..45bb71a6d304e 100644
--- a/llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll
+++ b/llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll
@@ -127,6 +127,7 @@
 ; CHECK-O23SZ-NEXT: Running pass: JumpThreadingPass
 ; CHECK-O23SZ-NEXT: Running analysis: LazyValueAnalysis
 ; CHECK-O23SZ-NEXT: Running pass: CorrelatedValuePropagationPass
+; CHECK-O23SZ-NEXT: Invalidating analysis: LazyValueAnalysis
 ; CHECK-O-NEXT: Running pass: SimplifyCFGPass
 ; CHECK-O3-NEXT: Running pass: AggressiveInstCombinePass
 ; CHECK-O-NEXT: Running pass: InstCombinePass
@@ -196,7 +197,9 @@
 ; CHECK-O-NEXT: Running analysis: DemandedBitsAnalysis
 ; CHECK-O-NEXT: Running pass: InstCombinePass
 ; CHECK-O23SZ-NEXT: Running pass: JumpThreadingPass
+; CHECK-O23SZ-NEXT: Running analysis: LazyValueAnalysis
 ; CHECK-O23SZ-NEXT: Running pass: CorrelatedValuePropagationPass
+; CHECK-O23SZ-NEXT: Invalidating analysis: LazyValueAnalysis
 ; CHECK-O23SZ-NEXT: Running pass: DSEPass
 ; CHECK-O23SZ-NEXT: Starting {{.*}}Function pass manager run
 ; CHECK-O23SZ-NEXT: Running pass: LoopSimplifyPass
@@ -213,10 +216,26 @@
 ; CHECK-O-NEXT: Finished CGSCC pass manager run.
 ; CHECK-O-NEXT: Finished {{.*}}Module pass manager run.
 ; CHECK-O-NEXT: Finished {{.*}}Module pass manager run.
+; CHECK-O23SZ-NEXT: Clearing all analysis results for: <possibly invalidated loop>
+; CHECK-O23SZ-NEXT: Invalidating analysis: DominatorTreeAnalysis
+; CHECK-O23SZ-NEXT: Invalidating analysis: MemorySSAAnalysis
+; CHECK-O23SZ-NEXT: Invalidating analysis: LoopAnalysis
+; CHECK-O23SZ-NEXT: Invalidating analysis: PostDominatorTreeAnalysis
+; CHECK-O23SZ-NEXT: Invalidating analysis: BranchProbabilityAnalysis
+; CHECK-O23SZ-NEXT: Invalidating analysis: BlockFrequencyAnalysis
+; CHECK-O23SZ-NEXT: Invalidating analysis: ScalarEvolutionAnalysis
+; CHECK-O23SZ-NEXT: Invalidating analysis: InnerAnalysisManagerProxy
+; CHECK-O23SZ-NEXT: Invalidating analysis: PhiValuesAnalysis
+; CHECK-O23SZ-NEXT: Invalidating analysis: MemoryDependenceAnalysis
+; CHECK-O23SZ-NEXT: Invalidating analysis: DemandedBitsAnalysis
+; CHECK-O3-NEXT: Invalidating analysis: DominanceFrontierAnalysis
+; CHECK-O3-NEXT: Invalidating analysis: RegionInfoAnalysis
+; CHECK-O23SZ-NEXT: Clearing all analysis results for: foo
 ; CHECK-O-NEXT: Running pass: GlobalOptPass
 ; CHECK-O-NEXT: Running analysis: TargetLibraryAnalysis on bar
 ; CHECK-EXT: Running pass: {{.*}}::Bye
 ; CHECK-O-NEXT: Finished {{.*}}Module pass manager run.
+; CHECK-O23SZ-NEXT: Clearing all analysis results for: foo
 ; CHECK-O-NEXT: Running pass: NameAnonGlobalPass
 ; CHECK-O-NEXT: Running pass: PrintModulePass
 
diff --git a/llvm/test/Other/new-pm-thinlto-prelink-samplepgo-defaults.ll b/llvm/test/Other/new-pm-thinlto-prelink-samplepgo-defaults.ll
index b062afdfb49cd..d97fe18524db8 100644
--- a/llvm/test/Other/new-pm-thinlto-prelink-samplepgo-defaults.ll
+++ b/llvm/test/Other/new-pm-thinlto-prelink-samplepgo-defaults.ll
@@ -93,6 +93,7 @@
 ; CHECK-O23SZ-NEXT: Running pass: JumpThreadingPass
 ; CHECK-O23SZ-NEXT: Running analysis: LazyValueAnalysis
 ; CHECK-O23SZ-NEXT: Running pass: CorrelatedValuePropagationPass
+; CHECK-O23SZ-NEXT: Invalidating analysis: LazyValueAnalysis
 ; CHECK-O-NEXT: Running pass: SimplifyCFGPass
 ; CHECK-O3-NEXT: Running pass: AggressiveInstCombinePass
 ; CHECK-O-NEXT: Running pass: InstCombinePass
@@ -152,7 +153,9 @@
 ; CHECK-O-NEXT: Running analysis: DemandedBitsAnalysis
 ; CHECK-O-NEXT: Running pass: InstCombinePass
 ; CHECK-O23SZ-NEXT: Running pass: JumpThreadingPass
+; CHECK-O23SZ-NEXT: Running analysis: LazyValueAnalysis
 ; CHECK-O23SZ-NEXT: Running pass: CorrelatedValuePropagationPass
+; CHECK-O23SZ-NEXT: Invalidating analysis: LazyValueAnalysis
 ; CHECK-O23SZ-NEXT: Running pass: DSEPass
 ; CHECK-O23SZ-NEXT: Starting {{.*}}Function pass manager run
 ; CHECK-O23SZ-NEXT: Running pass: LoopSimplifyPass

From 8dd4e3ceb804a58bcf25e6856fc6fde5e1995a66 Mon Sep 17 00:00:00 2001
From: Andrei Lebedev <alebedev@accesssoftek.com>
Date: Sat, 1 Aug 2020 15:54:11 -0700
Subject: [PATCH 0826/1035] Updated the -I option description.

---
 clang/docs/ClangCommandLineReference.rst | 4 ++--
 clang/include/clang/Driver/Options.td    | 9 +++++++--
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/clang/docs/ClangCommandLineReference.rst b/clang/docs/ClangCommandLineReference.rst
index 699a0be720368..8eb010eae2659 100644
--- a/clang/docs/ClangCommandLineReference.rst
+++ b/clang/docs/ClangCommandLineReference.rst
@@ -1014,9 +1014,9 @@ Include path management
 
 Flags controlling how ``#include``\s are resolved to files.
 
-.. option:: -I<dir>, --include-directory <dir>, --include-directory=<dir>
+.. option:: -I<dir>, --include-directory <arg>, --include-directory=<arg>
 
-Add directory <dir> to the list of include files search paths. If there are multiple -I options, these directories are searched in the order they are given before the standard system directories are searched. If the same directory is in the SYSTEM include search paths, for example if also specified with -isystem, the -I option will be ignored
+Add directory to include search path. If there are multiple -I options, these directories are searched in the order they are given before the standard system directories are searched. If the same directory is in the SYSTEM include search paths, for example if also specified with -isystem, the -I option will be ignored
 
 .. option:: -I-, --include-barrier
 
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 889035a0815e0..16051934c1e0b 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -395,7 +395,12 @@ def I_ : Flag<["-"], "I-">, Group<I_Group>,
              "remove current directory from include path">;
 def I : JoinedOrSeparate<["-"], "I">, Group<I_Group>,
     Flags<[CC1Option,CC1AsOption]>, MetaVarName<"<dir>">,
-    HelpText<"Add directory to include search path">;
+    HelpText<"Add directory to include search path. If there are multiple -I "
+             "options, these directories are searched in the order they are "
+             "given before the standard system directories are searched. "
+             "If the same directory is in the SYSTEM include search paths, for "
+             "example if also specified with -isystem, the -I option will be "
+             "ignored">;
 def L : JoinedOrSeparate<["-"], "L">, Flags<[RenderJoined]>, Group<Link_Group>,
     MetaVarName<"<dir>">, HelpText<"Add directory to library search path">;
 def MD : Flag<["-"], "MD">, Group<M_Group>,
@@ -1244,7 +1249,7 @@ def finline_functions : Flag<["-"], "finline-functions">, Group<f_clang_Group>,
 def finline_hint_functions: Flag<["-"], "finline-hint-functions">, Group<f_clang_Group>, Flags<[CC1Option]>,
   HelpText<"Inline functions which are (explicitly or implicitly) marked inline">;
 def finline : Flag<["-"], "finline">, Group<clang_ignored_f_Group>;
-def fglobal_isel : Flag<["-"], "fglobal-isel">, Group<f_clang_Group>, 
+def fglobal_isel : Flag<["-"], "fglobal-isel">, Group<f_clang_Group>,
   HelpText<"Enables the global instruction selector">;
 def fexperimental_isel : Flag<["-"], "fexperimental-isel">, Group<f_clang_Group>,
   Alias<fglobal_isel>;

From dc3388b0209d17f7ee2f4dc3e4f072dc397dd75d Mon Sep 17 00:00:00 2001
From: Evgenii Stepanov <eugenis@google.com>
Date: Fri, 31 Jul 2020 17:18:06 -0700
Subject: [PATCH 0827/1035] [msan] Respect no_huge_pages_for_shadow.

Disable huge pages in the MSan shadow region when
no_huge_pages_for_shadow == true (default).

Differential Revision: https://reviews.llvm.org/D85061
---
 compiler-rt/lib/msan/msan_linux.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/compiler-rt/lib/msan/msan_linux.cpp b/compiler-rt/lib/msan/msan_linux.cpp
index d61e9dee30654..bfdae0b920c9b 100644
--- a/compiler-rt/lib/msan/msan_linux.cpp
+++ b/compiler-rt/lib/msan/msan_linux.cpp
@@ -142,7 +142,7 @@ bool InitShadow(bool init_origins) {
     if (map) {
       if (!CheckMemoryRangeAvailability(start, size))
         return false;
-      if (!MmapFixedNoReserve(start, size, kMemoryLayout[i].name))
+      if (!MmapFixedSuperNoReserve(start, size, kMemoryLayout[i].name))
         return false;
       if (common_flags()->use_madv_dontdump)
         DontDumpShadowMemory(start, size);

From e281376e996e37fb6411363510e917b5b2c53c89 Mon Sep 17 00:00:00 2001
From: Fangrui Song <maskray@google.com>
Date: Sat, 1 Aug 2020 18:19:14 -0700
Subject: [PATCH 0828/1035] [ELF] --wrap: set isUsedInRegularObj of __wrap_
 only if it is defined

Fixes PR46169
---
 lld/ELF/Driver.cpp                  | 5 +++--
 lld/test/ELF/wrap-dynamic-undef.s   | 3 +--
 lld/test/ELF/wrap-shlib-undefined.s | 5 ++---
 3 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp
index cdb7355968377..3e60ffdb1dc1d 100644
--- a/lld/ELF/Driver.cpp
+++ b/lld/ELF/Driver.cpp
@@ -1782,7 +1782,7 @@ static std::vector<WrappedSymbol> addWrappedSymbols(opt::InputArgList &args) {
       continue;
 
     Symbol *real = addUndefined(saver.save("__real_" + name));
-    Symbol *wrap = addUndefined(saver.save("__wrap_" + name));
+    Symbol *wrap = addUnusedUndefined(saver.save("__wrap_" + name));
     v.push_back({sym, real, wrap});
 
     // We want to tell LTO not to inline symbols to be overwritten
@@ -1792,7 +1792,8 @@ static std::vector<WrappedSymbol> addWrappedSymbols(opt::InputArgList &args) {
 
     // Tell LTO not to eliminate these symbols.
     sym->isUsedInRegularObj = true;
-    wrap->isUsedInRegularObj = true;
+    if (wrap->isDefined())
+      wrap->isUsedInRegularObj = true;
   }
   return v;
 }
diff --git a/lld/test/ELF/wrap-dynamic-undef.s b/lld/test/ELF/wrap-dynamic-undef.s
index 2abb826411b39..af2871cfe6eae 100644
--- a/lld/test/ELF/wrap-dynamic-undef.s
+++ b/lld/test/ELF/wrap-dynamic-undef.s
@@ -8,10 +8,9 @@
 # Test that the dynamic relocation uses foo. We used to produce a
 # relocation with __real_foo.
 
-# CHECK:      Symbol table '.dynsym' contains 3 entries:
+# CHECK:      Symbol table '.dynsym' contains 2 entries:
 # CHECK:      NOTYPE  LOCAL  DEFAULT  UND
 # CHECK-NEXT: NOTYPE  GLOBAL DEFAULT  UND foo
-# CHECK-NEXT: NOTYPE  GLOBAL DEFAULT  UND __wrap_foo
 
 .global _start
 _start:
diff --git a/lld/test/ELF/wrap-shlib-undefined.s b/lld/test/ELF/wrap-shlib-undefined.s
index 8bbda963f0702..b0451b2865abe 100644
--- a/lld/test/ELF/wrap-shlib-undefined.s
+++ b/lld/test/ELF/wrap-shlib-undefined.s
@@ -12,12 +12,11 @@
 # RUN: ld.lld %t.o %t.so --wrap=foo -o %t
 # RUN: llvm-readelf --dyn-syms %t | FileCheck %s
 
-## FIXME GNU ld does not export __wrap_foo
 ## The reference __real_foo from %t.so causes foo to be exported.
-# CHECK:      Symbol table '.dynsym' contains 4 entries:
+## __wrap_foo is not used, thus not exported.
+# CHECK:      Symbol table '.dynsym' contains 3 entries:
 # CHECK:      NOTYPE  LOCAL  DEFAULT  UND
 # CHECK-NEXT: NOTYPE  GLOBAL DEFAULT  UND bar
-# CHECK-NEXT: NOTYPE  GLOBAL DEFAULT  UND __wrap_foo
 # CHECK-NEXT: NOTYPE  GLOBAL DEFAULT    6 foo
 
 .globl _start, foo

From e12a028ed3ed79fba1e1aa5212c7e22d83949192 Mon Sep 17 00:00:00 2001
From: Lang Hames <lhames@gmail.com>
Date: Fri, 31 Jul 2020 21:32:27 -0700
Subject: [PATCH 0829/1035] [llvm-jitlink] Support promotion of ODR weak
 symbols in -harness mode.

This prevents weak symbols from being immediately dead-stripped when not
directly referenced from the test harneess, enabling use of weak symbols
from the code under test.
---
 .../X86/Inputs/MachO_test_harness_test.s      |  7 +++
 llvm/tools/llvm-jitlink/llvm-jitlink.cpp      | 43 ++++++++++++++++---
 llvm/tools/llvm-jitlink/llvm-jitlink.h        |  1 +
 3 files changed, 44 insertions(+), 7 deletions(-)

diff --git a/llvm/test/ExecutionEngine/JITLink/X86/Inputs/MachO_test_harness_test.s b/llvm/test/ExecutionEngine/JITLink/X86/Inputs/MachO_test_harness_test.s
index e0764a93fa435..3ca616a4a7294 100644
--- a/llvm/test/ExecutionEngine/JITLink/X86/Inputs/MachO_test_harness_test.s
+++ b/llvm/test/ExecutionEngine/JITLink/X86/Inputs/MachO_test_harness_test.s
@@ -17,9 +17,16 @@ _public_func_to_interpose:
 _private_func_to_interpose:
 	retq
 
+	.globl	_used_weak
+	.weak_definition	_used_weak
+	.p2align	4, 0x90
+_used_weak:
+	retq
+
 	.globl	_public_func_to_test
 	.p2align	4, 0x90
 _public_func_to_test:
+  callq _used_weak
 	jmp	_public_func_to_interpose
 
 	.p2align	4, 0x90
diff --git a/llvm/tools/llvm-jitlink/llvm-jitlink.cpp b/llvm/tools/llvm-jitlink/llvm-jitlink.cpp
index 6828944ced238..22d29c12e1923 100644
--- a/llvm/tools/llvm-jitlink/llvm-jitlink.cpp
+++ b/llvm/tools/llvm-jitlink/llvm-jitlink.cpp
@@ -188,10 +188,28 @@ static Error applyHarnessPromotions(Session &S, LinkGraph &G) {
     if (!Sym->hasName())
       continue;
 
-    if (S.HarnessExternals.count(Sym->getName())) {
+    if (Sym->getLinkage() == Linkage::Weak) {
+      if (!S.CanonicalWeakDefs.count(Sym->getName()) ||
+          S.CanonicalWeakDefs[Sym->getName()] != G.getName()) {
+        LLVM_DEBUG({
+          dbgs() << "  Externalizing weak symbol " << Sym->getName() << "\n";
+        });
+        DefinitionsToRemove.push_back(Sym);
+      } else {
+        LLVM_DEBUG({
+          dbgs() << "  Making weak symbol " << Sym->getName() << " strong\n";
+        });
+        if (S.HarnessExternals.count(Sym->getName()))
+          Sym->setScope(Scope::Default);
+        else
+          Sym->setScope(Scope::Hidden);
+        Sym->setLinkage(Linkage::Strong);
+      }
+    } else if (S.HarnessExternals.count(Sym->getName())) {
       LLVM_DEBUG(dbgs() << "  Promoting " << Sym->getName() << "\n");
       Sym->setScope(Scope::Default);
       Sym->setLive(true);
+      continue;
     } else if (S.HarnessDefinitions.count(Sym->getName())) {
       LLVM_DEBUG(dbgs() << "  Externalizing " << Sym->getName() << "\n");
       DefinitionsToRemove.push_back(Sym);
@@ -504,10 +522,6 @@ Error LLVMJITLinkObjectLinkingLayer::add(JITDylib &JD,
     if (!Name)
       return Name.takeError();
 
-    // Skip symbols that aren't in the HarnessExternals set.
-    if (!S.HarnessExternals.count(*Name))
-      continue;
-
     // Skip symbols that have type SF_File.
     if (auto SymType = Sym.getType()) {
       if (*SymType == object::SymbolRef::ST_File)
@@ -515,13 +529,28 @@ Error LLVMJITLinkObjectLinkingLayer::add(JITDylib &JD,
     } else
       return SymType.takeError();
 
-    auto InternedName = S.ES.intern(*Name);
     auto SymFlags = JITSymbolFlags::fromObjectSymbol(Sym);
     if (!SymFlags)
       return SymFlags.takeError();
 
-    *SymFlags |= JITSymbolFlags::Exported;
+    if (SymFlags->isWeak()) {
+      // If this is a weak symbol that's not defined in the harness then we
+      // need to either mark it as strong (if this is the first definition
+      // that we've seen) or discard it.
+      if (S.HarnessDefinitions.count(*Name) || S.CanonicalWeakDefs.count(*Name))
+        continue;
+      S.CanonicalWeakDefs[*Name] = O->getBufferIdentifier();
+      *SymFlags &= ~JITSymbolFlags::Weak;
+      if (!S.HarnessExternals.count(*Name))
+        *SymFlags &= ~JITSymbolFlags::Exported;
+    } else if (S.HarnessExternals.count(*Name)) {
+      *SymFlags |= JITSymbolFlags::Exported;
+    } else {
+      // Skip symbols that aren't in the HarnessExternals set.
+      continue;
+    }
 
+    auto InternedName = S.ES.intern(*Name);
     SymbolFlags[InternedName] = std::move(*SymFlags);
   }
 
diff --git a/llvm/tools/llvm-jitlink/llvm-jitlink.h b/llvm/tools/llvm-jitlink/llvm-jitlink.h
index c16aed9f2b504..227d3a9225c83 100644
--- a/llvm/tools/llvm-jitlink/llvm-jitlink.h
+++ b/llvm/tools/llvm-jitlink/llvm-jitlink.h
@@ -86,6 +86,7 @@ struct Session {
   StringSet<> HarnessFiles;
   StringSet<> HarnessExternals;
   StringSet<> HarnessDefinitions;
+  DenseMap<StringRef, StringRef> CanonicalWeakDefs;
 
 private:
   Session(Triple TT, Error &Err);

From 0f5b70769d15d8cc728dcba353a33fb459450381 Mon Sep 17 00:00:00 2001
From: Lang Hames <lhames@gmail.com>
Date: Sat, 1 Aug 2020 17:44:34 -0700
Subject: [PATCH 0830/1035] [llvm-jitlink] Add -phony-externals option to
 suppress unresolved externals.

The -phony-externals option adds a generator which explicitly defines any
otherwise unresolved externals as null. This transforms link-time
unresolved-symbol errors into potential runtime null pointer accesses
(if an unresolved external is actually accessed during execution).

This option can be useful in -harness mode to avoid having to mock a
large number of symbols that are not reachable at runtime (e.g. unused
methods referenced by a class vtable).
---
 .../JITLink/JITLinkGeneric.cpp                |  6 ----
 .../X86/Inputs/MachO_test_harness_test.s      |  1 +
 .../JITLink/X86/MachO_test_harness_harness.s  |  4 ++-
 llvm/tools/llvm-jitlink/llvm-jitlink.cpp      | 30 +++++++++++++++++--
 4 files changed, 31 insertions(+), 10 deletions(-)

diff --git a/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp b/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp
index e0901186347f5..7c60d51677188 100644
--- a/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp
@@ -337,12 +337,6 @@ void JITLinkerBase::applyLookupResult(AsyncLookupResult Result) {
       dbgs() << "  " << Sym->getName() << ": "
              << formatv("{0:x16}", Sym->getAddress()) << "\n";
   });
-  assert(llvm::all_of(G->external_symbols(),
-                      [](Symbol *Sym) {
-                        return Sym->getAddress() != 0 ||
-                               Sym->getLinkage() == Linkage::Weak;
-                      }) &&
-         "All strong external symbols should have been resolved by now");
 }
 
 void JITLinkerBase::copyBlockContentToWorkingMemory(
diff --git a/llvm/test/ExecutionEngine/JITLink/X86/Inputs/MachO_test_harness_test.s b/llvm/test/ExecutionEngine/JITLink/X86/Inputs/MachO_test_harness_test.s
index 3ca616a4a7294..337f467d09677 100644
--- a/llvm/test/ExecutionEngine/JITLink/X86/Inputs/MachO_test_harness_test.s
+++ b/llvm/test/ExecutionEngine/JITLink/X86/Inputs/MachO_test_harness_test.s
@@ -27,6 +27,7 @@ _used_weak:
 	.p2align	4, 0x90
 _public_func_to_test:
   callq _used_weak
+  callq _used_unresolved_external
 	jmp	_public_func_to_interpose
 
 	.p2align	4, 0x90
diff --git a/llvm/test/ExecutionEngine/JITLink/X86/MachO_test_harness_harness.s b/llvm/test/ExecutionEngine/JITLink/X86/MachO_test_harness_harness.s
index 7fdddf2a64ca7..ee510387b35f2 100644
--- a/llvm/test/ExecutionEngine/JITLink/X86/MachO_test_harness_harness.s
+++ b/llvm/test/ExecutionEngine/JITLink/X86/MachO_test_harness_harness.s
@@ -3,7 +3,9 @@
 # RUN:   -o %t/file_to_test.o %S/Inputs/MachO_test_harness_test.s
 # RUN: llvm-mc -triple=x86_64-apple-macosx10.9 -filetype=obj \
 # RUN:   -o %t/test_harness.o %s
-# RUN: llvm-jitlink -noexec -check %s %t/file_to_test.o \
+# RUN: not llvm-jitlink -noexec -check %s %t/file_to_test.o \
+# RUN:    -harness %t/test_harness.o
+# RUN: llvm-jitlink -noexec -phony-externals -check %s %t/file_to_test.o \
 # RUN:    -harness %t/test_harness.o
 #
 # Check that we
diff --git a/llvm/tools/llvm-jitlink/llvm-jitlink.cpp b/llvm/tools/llvm-jitlink/llvm-jitlink.cpp
index 22d29c12e1923..798087d8cae7b 100644
--- a/llvm/tools/llvm-jitlink/llvm-jitlink.cpp
+++ b/llvm/tools/llvm-jitlink/llvm-jitlink.cpp
@@ -132,6 +132,11 @@ static cl::opt<bool> ShowRelocatedSectionContents(
     cl::desc("show section contents after fixups have been applied"),
     cl::init(false));
 
+static cl::opt<bool> PhonyExternals(
+    "phony-externals",
+    cl::desc("resolve all otherwise unresolved externals to null"),
+    cl::init(false));
+
 ExitOnError ExitOnErr;
 
 namespace llvm {
@@ -179,9 +184,9 @@ static Error applyHarnessPromotions(Session &S, LinkGraph &G) {
 
   LLVM_DEBUG(dbgs() << "Appling promotions to graph " << G.getName() << "\n");
 
-  // If it isn't then promote any symbols referenced by the harness to default
-  // scope, remove all symbols that clash with harness definitions, and demote
-  // all others.
+  // If this graph is part of the test then promote any symbols referenced by
+  // the harness to default scope, remove all symbols that clash with harness
+  // definitions, demote all other definitions.
   std::vector<Symbol *> DefinitionsToRemove;
   for (auto *Sym : G.defined_symbols()) {
 
@@ -560,6 +565,18 @@ Error LLVMJITLinkObjectLinkingLayer::add(JITDylib &JD,
   return JD.define(std::move(MU));
 }
 
+class PhonyExternalsGenerator : public JITDylib::DefinitionGenerator {
+public:
+  Error tryToGenerate(LookupKind K, JITDylib &JD,
+                      JITDylibLookupFlags JDLookupFlags,
+                      const SymbolLookupSet &LookupSet) override {
+    SymbolMap PhonySymbols;
+    for (auto &KV : LookupSet)
+      PhonySymbols[KV.first] = JITEvaluatedSymbol(0, JITSymbolFlags::Exported);
+    return JD.define(absoluteSymbols(std::move(PhonySymbols)));
+  }
+};
+
 Expected<std::unique_ptr<Session>> Session::Create(Triple TT) {
   Error Err = Error::success();
   std::unique_ptr<Session> S(new Session(std::move(TT), Err));
@@ -813,6 +830,10 @@ Error loadDylibs() {
   return Error::success();
 }
 
+void addPhonyExternalsGenerator(Session &S) {
+  S.MainJD->addGenerator(std::make_unique<PhonyExternalsGenerator>());
+}
+
 Error loadObjects(Session &S) {
 
   std::map<unsigned, JITDylib *> IdxToJLD;
@@ -1039,6 +1060,9 @@ int main(int argc, char *argv[]) {
     ExitOnErr(loadProcessSymbols(*S));
   ExitOnErr(loadDylibs());
 
+  if (PhonyExternals)
+    addPhonyExternalsGenerator(*S);
+
   {
     TimeRegion TR(Timers ? &Timers->LoadObjectsTimer : nullptr);
     ExitOnErr(loadObjects(*S));

From 60434989e5cd718e0f84c7601f648aecd1e8e1eb Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Sat, 1 Aug 2020 21:49:38 -0700
Subject: [PATCH 0831/1035] Use llvm::is_contained where appropriate (NFC)

Use llvm::is_contained where appropriate (NFC)

Reviewed By: kazu

Differential Revision: https://reviews.llvm.org/D85083
---
 llvm/include/llvm/IR/PassManager.h            |  5 +-
 llvm/lib/Analysis/BranchProbabilityInfo.cpp   |  3 +-
 llvm/lib/Analysis/MemorySSAUpdater.cpp        |  3 +-
 llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp |  3 +-
 llvm/lib/Support/FileCheck.cpp                |  5 +-
 llvm/lib/Support/Unix/Signals.inc             |  3 +-
 .../AArch64/AsmParser/AArch64AsmParser.cpp    |  8 +---
 llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp   |  4 +-
 .../Target/PowerPC/PPCLowerMASSVEntries.cpp   |  4 +-
 .../lib/Transforms/Scalar/LoopInterchange.cpp |  3 +-
 llvm/tools/dsymutil/dsymutil.cpp              |  5 +-
 llvm/unittests/Support/TargetParserTest.cpp   | 46 ++++++++-----------
 llvm/utils/TableGen/CodeGenSchedule.h         |  4 +-
 13 files changed, 39 insertions(+), 57 deletions(-)

diff --git a/llvm/include/llvm/IR/PassManager.h b/llvm/include/llvm/IR/PassManager.h
index c410818c470ca..f16696d7c2e38 100644
--- a/llvm/include/llvm/IR/PassManager.h
+++ b/llvm/include/llvm/IR/PassManager.h
@@ -38,6 +38,7 @@
 #define LLVM_IR_PASSMANAGER_H
 
 #include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/TinyPtrVector.h"
@@ -1137,9 +1138,7 @@ class OuterAnalysisManagerProxy
       // analyses that all trigger invalidation on the same outer analysis,
       // this entire system should be changed to some other deterministic
       // data structure such as a `SetVector` of a pair of pointers.
-      auto InvalidatedIt = std::find(InvalidatedIDList.begin(),
-                                     InvalidatedIDList.end(), InvalidatedID);
-      if (InvalidatedIt == InvalidatedIDList.end())
+      if (!llvm::is_contained(InvalidatedIDList, InvalidatedID))
         InvalidatedIDList.push_back(InvalidatedID);
     }
 
diff --git a/llvm/lib/Analysis/BranchProbabilityInfo.cpp b/llvm/lib/Analysis/BranchProbabilityInfo.cpp
index 7e34f3b6c869b..7b24fe9d56c25 100644
--- a/llvm/lib/Analysis/BranchProbabilityInfo.cpp
+++ b/llvm/lib/Analysis/BranchProbabilityInfo.cpp
@@ -692,8 +692,7 @@ computeUnlikelySuccessors(const BasicBlock *BB, Loop *L,
       // we can constant-evaluate the compare to see if it makes the branch be
       // taken or not.
       Constant *CmpLHSConst = dyn_cast<Constant>(V);
-      if (!CmpLHSConst ||
-          std::find(succ_begin(BB), succ_end(BB), B) == succ_end(BB))
+      if (!CmpLHSConst || !llvm::is_contained(successors(BB), B))
         continue;
       // First collapse InstChain
       for (Instruction *I : llvm::reverse(InstChain)) {
diff --git a/llvm/lib/Analysis/MemorySSAUpdater.cpp b/llvm/lib/Analysis/MemorySSAUpdater.cpp
index 21cbdcd67147a..81582413a6042 100644
--- a/llvm/lib/Analysis/MemorySSAUpdater.cpp
+++ b/llvm/lib/Analysis/MemorySSAUpdater.cpp
@@ -319,8 +319,7 @@ void MemorySSAUpdater::insertDef(MemoryDef *MD, bool RenameUses) {
   bool DefBeforeSameBlock = false;
   if (DefBefore->getBlock() == MD->getBlock() &&
       !(isa<MemoryPhi>(DefBefore) &&
-        std::find(InsertedPHIs.begin(), InsertedPHIs.end(), DefBefore) !=
-            InsertedPHIs.end()))
+        llvm::is_contained(InsertedPHIs, DefBefore)))
     DefBeforeSameBlock = true;
 
   // There is a def before us, which means we can replace any store/phi uses
diff --git a/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp b/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp
index 5d52e7fb80920..b388e43447835 100644
--- a/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp
@@ -494,8 +494,7 @@ void CodeViewDebug::recordLocalVariable(LocalVariable &&Var,
 
 static void addLocIfNotPresent(SmallVectorImpl<const DILocation *> &Locs,
                                const DILocation *Loc) {
-  auto B = Locs.begin(), E = Locs.end();
-  if (std::find(B, E, Loc) == E)
+  if (!llvm::is_contained(Locs, Loc))
     Locs.push_back(Loc);
 }
 
diff --git a/llvm/lib/Support/FileCheck.cpp b/llvm/lib/Support/FileCheck.cpp
index 29a5a1345a58a..137eea0a65a64 100644
--- a/llvm/lib/Support/FileCheck.cpp
+++ b/llvm/lib/Support/FileCheck.cpp
@@ -15,6 +15,7 @@
 
 #include "llvm/Support/FileCheck.h"
 #include "FileCheckImpl.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringSet.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/Support/CheckedArithmetic.h"
@@ -1578,9 +1579,7 @@ FindCheckType(const FileCheckRequest &Req, StringRef Buffer, StringRef Prefix) {
   StringRef Rest = Buffer.drop_front(Prefix.size() + 1);
 
   // Check for comment.
-  if (Req.CommentPrefixes.end() != std::find(Req.CommentPrefixes.begin(),
-                                             Req.CommentPrefixes.end(),
-                                             Prefix)) {
+  if (llvm::is_contained(Req.CommentPrefixes, Prefix)) {
     if (NextChar == ':')
       return {Check::CheckComment, Rest};
     // Ignore a comment prefix if it has a suffix like "-NOT".
diff --git a/llvm/lib/Support/Unix/Signals.inc b/llvm/lib/Support/Unix/Signals.inc
index f68374d29f023..ce1fccf0b4271 100644
--- a/llvm/lib/Support/Unix/Signals.inc
+++ b/llvm/lib/Support/Unix/Signals.inc
@@ -382,8 +382,7 @@ static RETSIGTYPE SignalHandler(int Sig) {
               OneShotPipeSignalFunction.exchange(nullptr))
         return OldOneShotPipeFunction();
 
-    if (std::find(std::begin(IntSigs), std::end(IntSigs), Sig)
-        != std::end(IntSigs)) {
+    if (llvm::is_contained(IntSigs, Sig)) {
       if (auto OldInterruptFunction = InterruptFunction.exchange(nullptr))
         return OldInterruptFunction();
 
diff --git a/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp b/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
index 0ac09c4f96f04..beae2b059bec5 100644
--- a/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
+++ b/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
@@ -5089,12 +5089,8 @@ bool AArch64AsmParser::ParseDirective(AsmToken DirectiveID) {
 
 static void ExpandCryptoAEK(AArch64::ArchKind ArchKind,
                             SmallVector<StringRef, 4> &RequestedExtensions) {
-  const bool NoCrypto =
-      (std::find(RequestedExtensions.begin(), RequestedExtensions.end(),
-                 "nocrypto") != std::end(RequestedExtensions));
-  const bool Crypto =
-      (std::find(RequestedExtensions.begin(), RequestedExtensions.end(),
-                 "crypto") != std::end(RequestedExtensions));
+  const bool NoCrypto = llvm::is_contained(RequestedExtensions, "nocrypto");
+  const bool Crypto = llvm::is_contained(RequestedExtensions, "crypto");
 
   if (!NoCrypto && Crypto) {
     switch (ArchKind) {
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index f45cc06e0a0a3..916d515af5dac 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -19,6 +19,7 @@
 #include "NVPTXTargetObjectFile.h"
 #include "NVPTXUtilities.h"
 #include "llvm/ADT/APInt.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/CodeGen/Analysis.h"
@@ -2438,8 +2439,7 @@ static bool isImageOrSamplerVal(const Value *arg, const Module *context) {
   if (!STy || STy->isLiteral())
     return false;
 
-  return std::find(std::begin(specialTypes), std::end(specialTypes),
-                   STy->getName()) != std::end(specialTypes);
+  return llvm::is_contained(specialTypes, STy->getName());
 }
 
 SDValue NVPTXTargetLowering::LowerFormalArguments(
diff --git a/llvm/lib/Target/PowerPC/PPCLowerMASSVEntries.cpp b/llvm/lib/Target/PowerPC/PPCLowerMASSVEntries.cpp
index 2b0e604e0ccde..a61e1f83705e6 100644
--- a/llvm/lib/Target/PowerPC/PPCLowerMASSVEntries.cpp
+++ b/llvm/lib/Target/PowerPC/PPCLowerMASSVEntries.cpp
@@ -16,6 +16,7 @@
 #include "PPC.h"
 #include "PPCSubtarget.h"
 #include "PPCTargetMachine.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/CodeGen/TargetPassConfig.h"
 #include "llvm/IR/Instructions.h"
@@ -64,8 +65,7 @@ class PPCLowerMASSVEntries : public ModulePass {
 /// Checks if the specified function name represents an entry in the MASSV
 /// library.
 bool PPCLowerMASSVEntries::isMASSVFunc(StringRef Name) {
-  auto Iter = std::find(std::begin(MASSVFuncs), std::end(MASSVFuncs), Name);
-  return Iter != std::end(MASSVFuncs);
+  return llvm::is_contained(MASSVFuncs, Name);
 }
 
 // FIXME:
diff --git a/llvm/lib/Transforms/Scalar/LoopInterchange.cpp b/llvm/lib/Transforms/Scalar/LoopInterchange.cpp
index 7787c0bccd4ce..3cfe1b595571e 100644
--- a/llvm/lib/Transforms/Scalar/LoopInterchange.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopInterchange.cpp
@@ -1521,8 +1521,7 @@ bool LoopInterchangeTransform::adjustLoopBranches() {
                   InnerLoopPreHeader, DTUpdates, /*MustUpdateOnce=*/false);
   // The outer loop header might or might not branch to the outer latch.
   // We are guaranteed to branch to the inner loop preheader.
-  if (std::find(succ_begin(OuterLoopHeaderBI), succ_end(OuterLoopHeaderBI),
-                OuterLoopLatch) != succ_end(OuterLoopHeaderBI))
+  if (llvm::is_contained(OuterLoopHeaderBI->successors(), OuterLoopLatch))
     updateSuccessor(OuterLoopHeaderBI, OuterLoopLatch, LoopExit, DTUpdates,
                     /*MustUpdateOnce=*/false);
   updateSuccessor(OuterLoopHeaderBI, InnerLoopPreHeader,
diff --git a/llvm/tools/dsymutil/dsymutil.cpp b/llvm/tools/dsymutil/dsymutil.cpp
index 3a32acbec06f9..a7d5e7afca039 100644
--- a/llvm/tools/dsymutil/dsymutil.cpp
+++ b/llvm/tools/dsymutil/dsymutil.cpp
@@ -17,6 +17,7 @@
 #include "LinkUtils.h"
 #include "MachOUtils.h"
 #include "Reproducer.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringExtras.h"
@@ -157,9 +158,7 @@ static Error verifyOptions(const DsymutilOptions &Options) {
                                    errc::invalid_argument);
   }
 
-  if (Options.LinkOpts.Update &&
-      std::find(Options.InputFiles.begin(), Options.InputFiles.end(), "-") !=
-          Options.InputFiles.end()) {
+  if (Options.LinkOpts.Update && llvm::is_contained(Options.InputFiles, "-")) {
     // FIXME: We cannot use stdin for an update because stdin will be
     // consumed by the BinaryHolder during the debugmap parsing, and
     // then we will want to consume it again in DwarfLinker. If we
diff --git a/llvm/unittests/Support/TargetParserTest.cpp b/llvm/unittests/Support/TargetParserTest.cpp
index 9f923e1358dde..f9392751de4e4 100644
--- a/llvm/unittests/Support/TargetParserTest.cpp
+++ b/llvm/unittests/Support/TargetParserTest.cpp
@@ -610,16 +610,13 @@ TEST(TargetParserTest, ARMExtensionFeatures) {
     // test +extension
     Features.clear();
     ARM::getExtensionFeatures(E.first, Features);
-    auto Found =
-        std::find(std::begin(Features), std::end(Features), E.second.at(0));
-    EXPECT_TRUE(Found != std::end(Features));
+    EXPECT_TRUE(llvm::is_contained(Features, E.second.at(0)));
     EXPECT_TRUE(Extensions.size() == Features.size());
 
     // test -extension
     Features.clear();
     ARM::getExtensionFeatures(~E.first, Features);
-    Found = std::find(std::begin(Features), std::end(Features), E.second.at(1));
-    EXPECT_TRUE(Found != std::end(Features));
+    EXPECT_TRUE(llvm::is_contained(Features, E.second.at(1)));
     EXPECT_TRUE(Extensions.size() == Features.size());
   }
 }
@@ -1227,27 +1224,24 @@ TEST(TargetParserTest, AArch64ExtensionFeatures) {
   AArch64::getExtensionFeatures(ExtVal, Features);
   EXPECT_TRUE(Extensions.size() == Features.size());
 
-  auto B = std::begin(Features);
-  auto E = std::end(Features);
-
-  EXPECT_TRUE(std::find(B, E, "+crc") != E);
-  EXPECT_TRUE(std::find(B, E, "+crypto") != E);
-  EXPECT_TRUE(std::find(B, E, "+fp-armv8") != E);
-  EXPECT_TRUE(std::find(B, E, "+neon") != E);
-  EXPECT_TRUE(std::find(B, E, "+fullfp16") != E);
-  EXPECT_TRUE(std::find(B, E, "+spe") != E);
-  EXPECT_TRUE(std::find(B, E, "+ras") != E);
-  EXPECT_TRUE(std::find(B, E, "+lse") != E);
-  EXPECT_TRUE(std::find(B, E, "+rdm") != E);
-  EXPECT_TRUE(std::find(B, E, "+dotprod") != E);
-  EXPECT_TRUE(std::find(B, E, "+rcpc") != E);
-  EXPECT_TRUE(std::find(B, E, "+fp16fml") != E);
-  EXPECT_TRUE(std::find(B, E, "+sve") != E);
-  EXPECT_TRUE(std::find(B, E, "+sve2") != E);
-  EXPECT_TRUE(std::find(B, E, "+sve2-aes") != E);
-  EXPECT_TRUE(std::find(B, E, "+sve2-sm4") != E);
-  EXPECT_TRUE(std::find(B, E, "+sve2-sha3") != E);
-  EXPECT_TRUE(std::find(B, E, "+sve2-bitperm") != E);
+  EXPECT_TRUE(llvm::is_contained(Features, "+crc"));
+  EXPECT_TRUE(llvm::is_contained(Features, "+crypto"));
+  EXPECT_TRUE(llvm::is_contained(Features, "+fp-armv8"));
+  EXPECT_TRUE(llvm::is_contained(Features, "+neon"));
+  EXPECT_TRUE(llvm::is_contained(Features, "+fullfp16"));
+  EXPECT_TRUE(llvm::is_contained(Features, "+spe"));
+  EXPECT_TRUE(llvm::is_contained(Features, "+ras"));
+  EXPECT_TRUE(llvm::is_contained(Features, "+lse"));
+  EXPECT_TRUE(llvm::is_contained(Features, "+rdm"));
+  EXPECT_TRUE(llvm::is_contained(Features, "+dotprod"));
+  EXPECT_TRUE(llvm::is_contained(Features, "+rcpc"));
+  EXPECT_TRUE(llvm::is_contained(Features, "+fp16fml"));
+  EXPECT_TRUE(llvm::is_contained(Features, "+sve"));
+  EXPECT_TRUE(llvm::is_contained(Features, "+sve2"));
+  EXPECT_TRUE(llvm::is_contained(Features, "+sve2-aes"));
+  EXPECT_TRUE(llvm::is_contained(Features, "+sve2-sm4"));
+  EXPECT_TRUE(llvm::is_contained(Features, "+sve2-sha3"));
+  EXPECT_TRUE(llvm::is_contained(Features, "+sve2-bitperm"));
 }
 
 TEST(TargetParserTest, AArch64ArchFeatures) {
diff --git a/llvm/utils/TableGen/CodeGenSchedule.h b/llvm/utils/TableGen/CodeGenSchedule.h
index c487d142d46c4..b60a1b4fa242c 100644
--- a/llvm/utils/TableGen/CodeGenSchedule.h
+++ b/llvm/utils/TableGen/CodeGenSchedule.h
@@ -16,6 +16,7 @@
 
 #include "llvm/ADT/APInt.h"
 #include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/TableGen/Record.h"
@@ -358,8 +359,7 @@ class OpcodeGroup {
   OpcodeGroup(OpcodeGroup &&Other) = default;
 
   void addOpcode(const Record *Opcode) {
-    assert(std::find(Opcodes.begin(), Opcodes.end(), Opcode) == Opcodes.end() &&
-           "Opcode already in set!");
+    assert(!llvm::is_contained(Opcodes, Opcode) && "Opcode already in set!");
     Opcodes.push_back(Opcode);
   }
 

From 20797989ea190f2ef22d13c5a7a0535fe9afa58b Mon Sep 17 00:00:00 2001
From: AK <1894981+hiraditya@users.noreply.github.com>
Date: Sat, 1 Aug 2020 19:15:05 -0700
Subject: [PATCH 0832/1035] Outline non returning functions unless a longjmp

__assert_fail, abort, exit etc. are cold.
TODO: outline throw

Authored by: rjf (Ruijie Fang)
Reviewed by: hiraditya,tejohnson,fhahn

Differential Revision: https://reviews.llvm.org/D69257
---
 llvm/lib/Transforms/IPO/HotColdSplitting.cpp  |  18 ++-
 .../HotColdSplit/longjmp-nosplit.ll           |  97 +++++++++++++
 .../Transforms/HotColdSplit/longjmp-split.ll  | 132 +++++++++++++++++
 .../Transforms/HotColdSplit/sjlj-nosplit.ll   | 103 +++++++++++++
 .../Transforms/HotColdSplit/sjlj-split.ll     | 136 ++++++++++++++++++
 .../HotColdSplit/split-assert-fail.ll         |  47 ++++++
 6 files changed, 528 insertions(+), 5 deletions(-)
 create mode 100644 llvm/test/Transforms/HotColdSplit/longjmp-nosplit.ll
 create mode 100644 llvm/test/Transforms/HotColdSplit/longjmp-split.ll
 create mode 100644 llvm/test/Transforms/HotColdSplit/sjlj-nosplit.ll
 create mode 100644 llvm/test/Transforms/HotColdSplit/sjlj-split.ll
 create mode 100644 llvm/test/Transforms/HotColdSplit/split-assert-fail.ll

diff --git a/llvm/lib/Transforms/IPO/HotColdSplitting.cpp b/llvm/lib/Transforms/IPO/HotColdSplitting.cpp
index d0bd0166534a7..cdfd9879c030a 100644
--- a/llvm/lib/Transforms/IPO/HotColdSplitting.cpp
+++ b/llvm/lib/Transforms/IPO/HotColdSplitting.cpp
@@ -101,7 +101,8 @@ bool blockEndsInUnreachable(const BasicBlock &BB) {
   return !(isa<ReturnInst>(I) || isa<IndirectBrInst>(I));
 }
 
-bool unlikelyExecuted(BasicBlock &BB) {
+bool unlikelyExecuted(BasicBlock &BB, ProfileSummaryInfo *PSI,
+                      BlockFrequencyInfo *BFI) {
   // Exception handling blocks are unlikely executed.
   if (BB.isEHPad() || isa<ResumeInst>(BB.getTerminator()))
     return true;
@@ -114,12 +115,19 @@ bool unlikelyExecuted(BasicBlock &BB) {
         return true;
 
   // The block is cold if it has an unreachable terminator, unless it's
-  // preceded by a call to a (possibly warm) noreturn call (e.g. longjmp).
+  // preceded by a call to a (possibly warm) noreturn call (e.g. longjmp);
+  // in the case of a longjmp, if the block is cold according to
+  // profile information, we mark it as unlikely to be executed as well.
   if (blockEndsInUnreachable(BB)) {
     if (auto *CI =
             dyn_cast_or_null<CallInst>(BB.getTerminator()->getPrevNode()))
-      if (CI->hasFnAttr(Attribute::NoReturn))
-        return false;
+      if (CI->hasFnAttr(Attribute::NoReturn)) {
+        if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(CI))
+          return (II->getIntrinsicID() != Intrinsic::eh_sjlj_longjmp) ||
+                 (BFI && PSI->isColdBlock(&BB, BFI));
+        return !CI->getCalledFunction()->getName().contains("longjmp") ||
+               (BFI && PSI->isColdBlock(&BB, BFI));
+      }
     return true;
   }
 
@@ -575,7 +583,7 @@ bool HotColdSplitting::outlineColdRegions(Function &F, bool HasProfileSummary) {
       continue;
 
     bool Cold = (BFI && PSI->isColdBlock(BB, BFI)) ||
-                (EnableStaticAnalyis && unlikelyExecuted(*BB));
+                (EnableStaticAnalyis && unlikelyExecuted(*BB, PSI, BFI));
     if (!Cold)
       continue;
 
diff --git a/llvm/test/Transforms/HotColdSplit/longjmp-nosplit.ll b/llvm/test/Transforms/HotColdSplit/longjmp-nosplit.ll
new file mode 100644
index 0000000000000..d207e8dae3850
--- /dev/null
+++ b/llvm/test/Transforms/HotColdSplit/longjmp-nosplit.ll
@@ -0,0 +1,97 @@
+; RUN:  opt -hotcoldsplit -S < %s | FileCheck %s
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+%struct.__jmp_buf_tag = type { [8 x i64], i32, %struct.__sigset_t }
+%struct.__sigset_t = type { [16 x i64] }
+
+@c = dso_local global i32 1, align 4
+@buf = dso_local global [20 x i8*] zeroinitializer, align 16
+
+; CHECK-LABEL: @f
+; CHECK-NOT: f.cold.1
+define dso_local void @f() #0 {
+entry:
+  %i = alloca i32, align 4
+  %j = alloca i32, align 4
+  %k = alloca i32, align 4
+  %0 = load i32, i32* @c, align 4
+  %tobool = icmp ne i32 %0, 0
+  br i1 %tobool, label %if.then, label %if.else
+
+if.then:                                          ; preds = %entry
+  ret void
+
+if.else:                                          ; preds = %entry
+  %1 = load i32, i32* @c, align 4
+  %inc = add  i32 %1, 1
+  store i32 %inc, i32* @c, align 4
+  %2 = load i32, i32* @c, align 4
+  %inc1 = add  i32 %2, 1
+  store i32 %inc1, i32* @c, align 4
+  %3 = load i32, i32* @c, align 4
+  %inc2 = add  i32 %3, 1
+  store i32 %inc2, i32* @c, align 4
+  %4 = load i32, i32* @c, align 4
+  %inc3 = add  i32 %4, 1
+  store i32 %inc3, i32* @c, align 4
+  %5 = load i32, i32* @c, align 4
+  %dec = add  i32 %5, -1
+  store i32 %dec, i32* @c, align 4
+  %6 = load i32, i32* @c, align 4
+  %dec4 = add  i32 %6, -1
+  store i32 %dec4, i32* @c, align 4
+  %7 = load i32, i32* @c, align 4
+  %inc5 = add  i32 %7, 1
+  store i32 %inc5, i32* @c, align 4
+  %8 = load i32, i32* @c, align 4
+  %inc6 = add  i32 %8, 1
+  store i32 %inc6, i32* @c, align 4
+  %9 = load i32, i32* @c, align 4
+  %add = add  i32 %9, 1
+  store i32 %add, i32* %i, align 4
+  %10 = load i32, i32* %i, align 4
+  %sub = sub  i32 %10, 1
+  store i32 %sub, i32* %j, align 4
+  %11 = load i32, i32* %i, align 4
+  %add7 = add  i32 %11, 2
+  store i32 %add7, i32* %k, align 4
+  call void @longjmp(%struct.__jmp_buf_tag* bitcast ([20 x i8*]* @buf to %struct.__jmp_buf_tag*), i32 1) #3
+  unreachable
+}
+
+declare dso_local void @longjmp(%struct.__jmp_buf_tag*, i32) #1
+
+; CHECK-LABEL: @main
+; CHECK-NOT: main.cold.1
+define dso_local i32 @main() #0 {
+entry:
+  %retval = alloca i32, align 4
+  %i = alloca i32, align 4
+  store i32 0, i32* %retval, align 4
+  store i32 0, i32* %i, align 4
+  %call = call i32 @_setjmp(%struct.__jmp_buf_tag* bitcast ([20 x i8*]* @buf to %struct.__jmp_buf_tag*)) #4
+  %tobool = icmp ne i32 %call, 0
+  br i1 %tobool, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  store i32 1, i32* %retval, align 4
+  br label %return
+
+if.end:                                           ; preds = %entry
+  call void @f()
+  store i32 0, i32* %retval, align 4
+  br label %return
+
+return:                                           ; preds = %if.end, %if.then
+  %0 = load i32, i32* %retval, align 4
+  ret i32 %0
+}
+
+declare dso_local i32 @_setjmp(%struct.__jmp_buf_tag*) #2
+
+attributes #0 = { nounwind uwtable }
+attributes #1 = { noreturn nounwind }
+attributes #2 = { nounwind returns_twice }
+attributes #3 = { noreturn nounwind }
+attributes #4 = { nounwind returns_twice }
diff --git a/llvm/test/Transforms/HotColdSplit/longjmp-split.ll b/llvm/test/Transforms/HotColdSplit/longjmp-split.ll
new file mode 100644
index 0000000000000..905f146a79e18
--- /dev/null
+++ b/llvm/test/Transforms/HotColdSplit/longjmp-split.ll
@@ -0,0 +1,132 @@
+; RUN:  opt -profile-summary-cold-count=0 -hotcoldsplit -S < %s | FileCheck %s
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+%struct.__jmp_buf_tag = type { [8 x i64], i32, %struct.__sigset_t }
+%struct.__sigset_t = type { [16 x i64] }
+
+@c = dso_local global i32 1, align 4
+@buf = dso_local global [20 x i8*] zeroinitializer, align 16
+
+; CHECK-LABEL: @f
+; CHECK: f.cold.1
+define dso_local void @f() #0 !prof !31 {
+entry:
+  %i = alloca i32, align 4
+  %j = alloca i32, align 4
+  %k = alloca i32, align 4
+  %0 = load i32, i32* @c, align 4
+  %tobool = icmp ne i32 %0, 0
+  br i1 %tobool, label %if.then, label %if.else, !prof !32
+
+if.then:                                          ; preds = %entry
+  ret void
+
+if.else:                                          ; preds = %entry
+  %1 = load i32, i32* @c, align 4
+  %inc = add  i32 %1, 1
+  store i32 %inc, i32* @c, align 4
+  %2 = load i32, i32* @c, align 4
+  %inc1 = add  i32 %2, 1
+  store i32 %inc1, i32* @c, align 4
+  %3 = load i32, i32* @c, align 4
+  %inc2 = add  i32 %3, 1
+  store i32 %inc2, i32* @c, align 4
+  %4 = load i32, i32* @c, align 4
+  %inc3 = add  i32 %4, 1
+  store i32 %inc3, i32* @c, align 4
+  %5 = load i32, i32* @c, align 4
+  %dec = add  i32 %5, -1
+  store i32 %dec, i32* @c, align 4
+  %6 = load i32, i32* @c, align 4
+  %dec4 = add  i32 %6, -1
+  store i32 %dec4, i32* @c, align 4
+  %7 = load i32, i32* @c, align 4
+  %inc5 = add  i32 %7, 1
+  store i32 %inc5, i32* @c, align 4
+  %8 = load i32, i32* @c, align 4
+  %inc6 = add  i32 %8, 1
+  store i32 %inc6, i32* @c, align 4
+  %9 = load i32, i32* @c, align 4
+  %add = add  i32 %9, 1
+  store i32 %add, i32* %i, align 4
+  %10 = load i32, i32* %i, align 4
+  %sub = sub  i32 %10, 1
+  store i32 %sub, i32* %j, align 4
+  %11 = load i32, i32* %i, align 4
+  %add7 = add  i32 %11, 2
+  store i32 %add7, i32* %k, align 4
+  call void @longjmp(%struct.__jmp_buf_tag* bitcast ([20 x i8*]* @buf to %struct.__jmp_buf_tag*), i32 1) #3
+  unreachable
+}
+
+declare dso_local void @longjmp(%struct.__jmp_buf_tag*, i32) #1
+
+define dso_local i32 @main() #0 !prof !31 {
+entry:
+  %retval = alloca i32, align 4
+  %i = alloca i32, align 4
+  store i32 0, i32* %retval, align 4
+  store i32 0, i32* %i, align 4
+  %call = call i32 @_setjmp(%struct.__jmp_buf_tag* bitcast ([20 x i8*]* @buf to %struct.__jmp_buf_tag*)) #4
+  %tobool = icmp ne i32 %call, 0
+  br i1 %tobool, label %if.then, label %if.end, !prof !33
+
+if.then:                                          ; preds = %entry
+  store i32 1, i32* %retval, align 4
+  br label %return
+
+if.end:                                           ; preds = %entry
+  call void @f()
+  store i32 0, i32* %retval, align 4
+  br label %return
+
+return:                                           ; preds = %if.end, %if.then
+  %0 = load i32, i32* %retval, align 4
+  ret i32 %0
+}
+
+declare dso_local i32 @_setjmp(%struct.__jmp_buf_tag*) #2
+
+attributes #0 = { inlinehint nounwind uwtable }
+attributes #1 = { noreturn nounwind }
+attributes #2 = { nounwind returns_twice }
+attributes #3 = { noreturn nounwind }
+attributes #4 = { nounwind returns_twice }
+
+!llvm.module.flags = !{!0, !1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 1, !"ProfileSummary", !2}
+!2 = !{!3, !4, !5, !6, !7, !8, !9, !10, !11, !12}
+!3 = !{!"ProfileFormat", !"InstrProf"}
+!4 = !{!"TotalCount", i64 2}
+!5 = !{!"MaxCount", i64 1}
+!6 = !{!"MaxInternalCount", i64 1}
+!7 = !{!"MaxFunctionCount", i64 1}
+!8 = !{!"NumCounts", i64 4}
+!9 = !{!"NumFunctions", i64 2}
+!10 = !{!"IsPartialProfile", i64 0}
+!11 = !{!"PartialProfileRatio", double 0.000000e+00}
+!12 = !{!"DetailedSummary", !13}
+!13 = !{!14, !15, !16, !17, !18, !19, !20, !21, !22, !23, !24, !25, !26, !27, !28, !29}
+!14 = !{i32 10000, i64 0, i32 0}
+!15 = !{i32 100000, i64 0, i32 0}
+!16 = !{i32 200000, i64 0, i32 0}
+!17 = !{i32 300000, i64 0, i32 0}
+!18 = !{i32 400000, i64 0, i32 0}
+!19 = !{i32 500000, i64 1, i32 2}
+!20 = !{i32 600000, i64 1, i32 2}
+!21 = !{i32 700000, i64 1, i32 2}
+!22 = !{i32 800000, i64 1, i32 2}
+!23 = !{i32 900000, i64 1, i32 2}
+!24 = !{i32 950000, i64 1, i32 2}
+!25 = !{i32 990000, i64 1, i32 2}
+!26 = !{i32 999000, i64 1, i32 2}
+!27 = !{i32 999900, i64 1, i32 2}
+!28 = !{i32 999990, i64 1, i32 2}
+!29 = !{i32 999999, i64 1, i32 2}
+!31 = !{!"function_entry_count", i64 1}
+!32 = !{!"branch_weights", i32 1, i32 0}
+!33 = !{!"branch_weights", i32 0, i32 1}
diff --git a/llvm/test/Transforms/HotColdSplit/sjlj-nosplit.ll b/llvm/test/Transforms/HotColdSplit/sjlj-nosplit.ll
new file mode 100644
index 0000000000000..f02a1b376ecb2
--- /dev/null
+++ b/llvm/test/Transforms/HotColdSplit/sjlj-nosplit.ll
@@ -0,0 +1,103 @@
+; RUN:  opt -hotcoldsplit -S < %s | FileCheck %s
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@c = dso_local global i32 1, align 4
+@buf = dso_local global [20 x i8*] zeroinitializer, align 16
+
+; CHECK-LABEL: @f
+; CHECK-NOT: f.cold.1
+define dso_local void @f() #0 {
+entry:
+  %i = alloca i32, align 4
+  %j = alloca i32, align 4
+  %k = alloca i32, align 4
+  %0 = load i32, i32* @c, align 4
+  %tobool = icmp ne i32 %0, 0
+  br i1 %tobool, label %if.then, label %if.else
+
+if.then:                                          ; preds = %entry
+  ret void
+
+if.else:                                          ; preds = %entry
+  %1 = load i32, i32* @c, align 4
+  %inc = add nsw i32 %1, 1
+  store i32 %inc, i32* @c, align 4
+  %2 = load i32, i32* @c, align 4
+  %inc1 = add nsw i32 %2, 1
+  store i32 %inc1, i32* @c, align 4
+  %3 = load i32, i32* @c, align 4
+  %inc2 = add nsw i32 %3, 1
+  store i32 %inc2, i32* @c, align 4
+  %4 = load i32, i32* @c, align 4
+  %inc3 = add nsw i32 %4, 1
+  store i32 %inc3, i32* @c, align 4
+  %5 = load i32, i32* @c, align 4
+  %dec = add nsw i32 %5, -1
+  store i32 %dec, i32* @c, align 4
+  %6 = load i32, i32* @c, align 4
+  %dec4 = add nsw i32 %6, -1
+  store i32 %dec4, i32* @c, align 4
+  %7 = load i32, i32* @c, align 4
+  %inc5 = add nsw i32 %7, 1
+  store i32 %inc5, i32* @c, align 4
+  %8 = load i32, i32* @c, align 4
+  %inc6 = add nsw i32 %8, 1
+  store i32 %inc6, i32* @c, align 4
+  %9 = load i32, i32* @c, align 4
+  %add = add nsw i32 %9, 1
+  store i32 %add, i32* %i, align 4
+  %10 = load i32, i32* %i, align 4
+  %sub = sub nsw i32 %10, 1
+  store i32 %sub, i32* %j, align 4
+  %11 = load i32, i32* %i, align 4
+  %add7 = add nsw i32 %11, 2
+  store i32 %add7, i32* %k, align 4
+  call void @llvm.eh.sjlj.longjmp(i8* bitcast ([20 x i8*]* @buf to i8*))
+  unreachable
+}
+
+declare void @llvm.eh.sjlj.longjmp(i8*) #1
+
+; CHECK-LABEL: @main
+; CHECK-NOT: main.cold.1
+define dso_local i32 @main() #0 {
+entry:
+  %retval = alloca i32, align 4
+  %i = alloca i32, align 4
+  store i32 0, i32* %retval, align 4
+  store i32 0, i32* %i, align 4
+  %0 = call i8* @llvm.frameaddress.p0i8(i32 0)
+  store i8* %0, i8** getelementptr inbounds ([20 x i8*], [20 x i8*]* @buf, i64 0, i64 0), align 16
+  %1 = call i8* @llvm.stacksave()
+  store i8* %1, i8** getelementptr inbounds ([20 x i8*], [20 x i8*]* @buf, i64 0, i64 2), align 16
+  %2 = call i32 @llvm.eh.sjlj.setjmp(i8* bitcast ([20 x i8*]* @buf to i8*))
+  %tobool = icmp ne i32 %2, 0
+  br i1 %tobool, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  store i32 1, i32* %retval, align 4
+  br label %return
+
+if.end:                                           ; preds = %entry
+  call void @f()
+  store i32 0, i32* %retval, align 4
+  br label %return
+
+return:                                           ; preds = %if.end, %if.then
+  %3 = load i32, i32* %retval, align 4
+  ret i32 %3
+}
+
+declare i8* @llvm.frameaddress.p0i8(i32 immarg) #2
+
+declare i8* @llvm.stacksave() #3
+
+declare i32 @llvm.eh.sjlj.setjmp(i8*) #3
+
+attributes #0 = { nounwind uwtable }
+attributes #1 = { noreturn nounwind }
+attributes #2 = { nounwind readnone }
+attributes #3 = { nounwind }
+
+
diff --git a/llvm/test/Transforms/HotColdSplit/sjlj-split.ll b/llvm/test/Transforms/HotColdSplit/sjlj-split.ll
new file mode 100644
index 0000000000000..3a12677ecf519
--- /dev/null
+++ b/llvm/test/Transforms/HotColdSplit/sjlj-split.ll
@@ -0,0 +1,136 @@
+; RUN:  opt -profile-summary-cold-count=0 -hotcoldsplit -S < %s | FileCheck %s
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@c = dso_local global i32 1, align 4
+@buf = dso_local global [20 x i8*] zeroinitializer, align 16
+
+; CHECK-LABEL: @f
+; CHECK: f.cold.1
+define dso_local void @f() #0 !prof !31 {
+entry:
+  %i = alloca i32, align 4
+  %j = alloca i32, align 4
+  %k = alloca i32, align 4
+  %0 = load i32, i32* @c, align 4
+  %tobool = icmp ne i32 %0, 0
+  br i1 %tobool, label %if.then, label %if.else, !prof !32
+
+if.then:                                          ; preds = %entry
+  ret void
+
+if.else:                                          ; preds = %entry
+  %1 = load i32, i32* @c, align 4
+  %inc = add  i32 %1, 1
+  store i32 %inc, i32* @c, align 4
+  %2 = load i32, i32* @c, align 4
+  %inc1 = add  i32 %2, 1
+  store i32 %inc1, i32* @c, align 4
+  %3 = load i32, i32* @c, align 4
+  %inc2 = add  i32 %3, 1
+  store i32 %inc2, i32* @c, align 4
+  %4 = load i32, i32* @c, align 4
+  %inc3 = add  i32 %4, 1
+  store i32 %inc3, i32* @c, align 4
+  %5 = load i32, i32* @c, align 4
+  %dec = add  i32 %5, -1
+  store i32 %dec, i32* @c, align 4
+  %6 = load i32, i32* @c, align 4
+  %dec4 = add  i32 %6, -1
+  store i32 %dec4, i32* @c, align 4
+  %7 = load i32, i32* @c, align 4
+  %inc5 = add  i32 %7, 1
+  store i32 %inc5, i32* @c, align 4
+  %8 = load i32, i32* @c, align 4
+  %inc6 = add  i32 %8, 1
+  store i32 %inc6, i32* @c, align 4
+  %9 = load i32, i32* @c, align 4
+  %add = add  i32 %9, 1
+  store i32 %add, i32* %i, align 4
+  %10 = load i32, i32* %i, align 4
+  %sub = sub  i32 %10, 1
+  store i32 %sub, i32* %j, align 4
+  %11 = load i32, i32* %i, align 4
+  %add7 = add  i32 %11, 2
+  store i32 %add7, i32* %k, align 4
+  call void @llvm.eh.sjlj.longjmp(i8* bitcast ([20 x i8*]* @buf to i8*))
+  unreachable
+}
+
+declare void @llvm.eh.sjlj.longjmp(i8*) #1
+
+define dso_local i32 @main() #0 !prof !31 {
+entry:
+  %retval = alloca i32, align 4
+  %i = alloca i32, align 4
+  store i32 0, i32* %retval, align 4
+  store i32 0, i32* %i, align 4
+  %0 = call i8* @llvm.frameaddress.p0i8(i32 0)
+  store i8* %0, i8** getelementptr inbounds ([20 x i8*], [20 x i8*]* @buf, i64 0, i64 0), align 16
+  %1 = call i8* @llvm.stacksave()
+  store i8* %1, i8** getelementptr inbounds ([20 x i8*], [20 x i8*]* @buf, i64 0, i64 2), align 16
+  %2 = call i32 @llvm.eh.sjlj.setjmp(i8* bitcast ([20 x i8*]* @buf to i8*))
+  %tobool = icmp ne i32 %2, 0
+  br i1 %tobool, label %if.then, label %if.end, !prof !33
+
+if.then:                                          ; preds = %entry
+  store i32 1, i32* %retval, align 4
+  br label %return
+
+if.end:                                           ; preds = %entry
+  call void @f()
+  store i32 0, i32* %retval, align 4
+  br label %return
+
+return:                                           ; preds = %if.end, %if.then
+  %3 = load i32, i32* %retval, align 4
+  ret i32 %3
+}
+
+declare i8* @llvm.frameaddress.p0i8(i32 immarg) #2
+
+declare i8* @llvm.stacksave() #3
+
+declare i32 @llvm.eh.sjlj.setjmp(i8*) #3
+
+attributes #0 = { inlinehint nounwind uwtable }
+attributes #1 = { noreturn nounwind }
+attributes #2 = { nounwind readnone }
+attributes #3 = { nounwind }
+
+!llvm.module.flags = !{!0, !1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 1, !"ProfileSummary", !2}
+!2 = !{!3, !4, !5, !6, !7, !8, !9, !10, !11, !12}
+!3 = !{!"ProfileFormat", !"InstrProf"}
+!4 = !{!"TotalCount", i64 2}
+!5 = !{!"MaxCount", i64 1}
+!6 = !{!"MaxInternalCount", i64 1}
+!7 = !{!"MaxFunctionCount", i64 1}
+!8 = !{!"NumCounts", i64 4}
+!9 = !{!"NumFunctions", i64 2}
+!10 = !{!"IsPartialProfile", i64 0}
+!11 = !{!"PartialProfileRatio", double 0.000000e+00}
+!12 = !{!"DetailedSummary", !13}
+!13 = !{!14, !15, !16, !17, !18, !19, !20, !21, !22, !23, !24, !25, !26, !27, !28, !29}
+!14 = !{i32 10000, i64 0, i32 0}
+!15 = !{i32 100000, i64 0, i32 0}
+!16 = !{i32 200000, i64 0, i32 0}
+!17 = !{i32 300000, i64 0, i32 0}
+!18 = !{i32 400000, i64 0, i32 0}
+!19 = !{i32 500000, i64 1, i32 2}
+!20 = !{i32 600000, i64 1, i32 2}
+!21 = !{i32 700000, i64 1, i32 2}
+!22 = !{i32 800000, i64 1, i32 2}
+!23 = !{i32 900000, i64 1, i32 2}
+!24 = !{i32 950000, i64 1, i32 2}
+!25 = !{i32 990000, i64 1, i32 2}
+!26 = !{i32 999000, i64 1, i32 2}
+!27 = !{i32 999900, i64 1, i32 2}
+!28 = !{i32 999990, i64 1, i32 2}
+!29 = !{i32 999999, i64 1, i32 2}
+!31 = !{!"function_entry_count", i64 1}
+!32 = !{!"branch_weights", i32 1, i32 0}
+!33 = !{!"branch_weights", i32 0, i32 1}
+
diff --git a/llvm/test/Transforms/HotColdSplit/split-assert-fail.ll b/llvm/test/Transforms/HotColdSplit/split-assert-fail.ll
new file mode 100644
index 0000000000000..ea5f4b9114917
--- /dev/null
+++ b/llvm/test/Transforms/HotColdSplit/split-assert-fail.ll
@@ -0,0 +1,47 @@
+; REQUIRES: asserts
+; RUN: opt -S -instsimplify -hotcoldsplit -debug < %s 2>&1 | FileCheck %s
+; RUN: opt -instcombine -hotcoldsplit -instsimplify %s -o /dev/null
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@.str = private unnamed_addr constant [2 x i8] c"0\00", align 1
+@.str.1 = private unnamed_addr constant [14 x i8] c"assert-fail.c\00", align 1
+@__PRETTY_FUNCTION__.main = private unnamed_addr constant [15 x i8] c"int main(void)\00", align 1
+
+; CHECK: @f
+; CHECK-LABEL: codeRepl:
+; CHECK }
+; CHECK: define {{.*}}@f.cold.1()
+; CHECK-LABEL: newFuncRoot:
+; CHECK:   br label %if.then
+
+; Function Attrs: nounwind willreturn
+define i32 @f() #0 {
+entry:
+  %retval = alloca i32, align 4
+  %i = alloca i32, align 4
+  store i32 0, i32* %retval, align 4
+  store i32 0, i32* %i, align 4
+  %0 = load i32, i32* %i, align 4
+  %cmp = icmp eq i32 %0, 2
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  call void @__assert_fail(i8* getelementptr inbounds ([2 x i8], [2 x i8]* @.str, i64 0, i64 0), i8* getelementptr inbounds ([14 x i8], [14 x i8]* @.str.1, i64 0, i64 0), i32 10, i8* getelementptr inbounds ([15 x i8], [15 x i8]* @__PRETTY_FUNCTION__.main, i64 0, i64 0)) #1
+  unreachable
+
+if.end:                                           ; preds = %entry
+  %1 = load i32, i32* %i, align 4
+  %add = add nsw i32 %1, 1
+  store i32 %add, i32* %i, align 4
+  %2 = load i32, i32* %i, align 4
+  ret i32 %2
+}
+
+; Function Attrs: noreturn nounwind
+declare dso_local void @__assert_fail(i8*, i8*, i32, i8*) #1
+
+attributes #0 = { nounwind willreturn }
+attributes #1 = { noreturn nounwind }
+

From aa1f905890fbbfedf396530f1e14409875ece13c Mon Sep 17 00:00:00 2001
From: AK <1894981+hiraditya@users.noreply.github.com>
Date: Sat, 1 Aug 2020 22:34:44 -0700
Subject: [PATCH 0833/1035] [HotColdSplit] Add test case for unlikely attribute
 in outlined function

Authored by: rjf (Ruijie Fang)
Reviewed by: hiraditya,tejohnson,fhahn

Differential Revision: https://reviews.llvm.org/D69384
---
 .../test/Transforms/HotColdSplit/coldentrycount.ll | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/llvm/test/Transforms/HotColdSplit/coldentrycount.ll b/llvm/test/Transforms/HotColdSplit/coldentrycount.ll
index d63acc188f544..7b196bf4c1048 100644
--- a/llvm/test/Transforms/HotColdSplit/coldentrycount.ll
+++ b/llvm/test/Transforms/HotColdSplit/coldentrycount.ll
@@ -1,13 +1,14 @@
 ; Test to ensure that split cold function gets 0 entry count profile
 ; metadata when compiling with pgo.
 
-; RUN: opt -hotcoldsplit -hotcoldsplit-threshold=0 -S < %s | FileCheck %s
+; RUN: opt -hotcoldsplit -hotcoldsplit-threshold=0 -codegenprepare -S < %s | FileCheck %s
 
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.14.0"
 
-; CHECK-LABEL: @fun
+; CHECK: define {{.*}} @fun{{.*}} ![[HOTPROF:[0-9]+]] {{.*}}section_prefix ![[LIKELY:[0-9]+]]
 ; CHECK: call void @fun.cold.1
+
 define void @fun() !prof !14 {
 entry:
   br i1 undef, label %if.then, label %if.else
@@ -22,8 +23,12 @@ if.else:
 
 declare void @sink() cold
 
-; CHECK: define {{.*}} @fun.cold.1{{.*}} ![[PROF:[0-9]+]]
+; CHECK: define {{.*}} @fun.cold.1{{.*}} ![[PROF:[0-9]+]] {{.*}}section_prefix ![[UNLIKELY:[0-9]+]]
+
+; CHECK: ![[HOTPROF]] = !{!"function_entry_count", i64 100}
+; CHECK: ![[LIKELY]] = !{!"function_section_prefix", !".hot"}
 ; CHECK: ![[PROF]] = !{!"function_entry_count", i64 0}
+; CHECK: ![[UNLIKELY]] = !{!"function_section_prefix", !".unlikely"}
 
 !llvm.module.flags = !{!0}
 !0 = !{i32 1, !"ProfileSummary", !1}
@@ -41,3 +46,6 @@ declare void @sink() cold
 !12 = !{i32 999000, i64 100, i32 1}
 !13 = !{i32 999999, i64 1, i32 2}
 !14 = !{!"function_entry_count", i64 100}
+!15 = !{!"function_section_prefix", !".hot"}
+!16 = !{!"function_entry_count", i64 0}
+!17 = !{!"function_section_prefix", !".unlikely"}

From f89d59a0853f90c54dea60500831d02901623223 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@intel.com>
Date: Sat, 1 Aug 2020 21:57:24 -0700
Subject: [PATCH 0834/1035] [X86] Add parity test cases for PR46954.

This adds tests cases where the parity idiom of (and (ctpop X, 1))
has a truncate in the middle.
---
 llvm/test/CodeGen/X86/parity.ll | 147 ++++++++++++++++++++++++++++++++
 1 file changed, 147 insertions(+)

diff --git a/llvm/test/CodeGen/X86/parity.ll b/llvm/test/CodeGen/X86/parity.ll
index 9ad04db8c280f..404d7a68a6e74 100644
--- a/llvm/test/CodeGen/X86/parity.ll
+++ b/llvm/test/CodeGen/X86/parity.ll
@@ -92,5 +92,152 @@ define i64 @parity_64(i64 %x) {
   ret i64 %2
 }
 
+define i32 @parity_64_trunc(i64 %x) {
+; X86-NOPOPCNT-LABEL: parity_64_trunc:
+; X86-NOPOPCNT:       # %bb.0:
+; X86-NOPOPCNT-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOPOPCNT-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NOPOPCNT-NEXT:    movl %ecx, %edx
+; X86-NOPOPCNT-NEXT:    shrl %edx
+; X86-NOPOPCNT-NEXT:    andl $1431655765, %edx # imm = 0x55555555
+; X86-NOPOPCNT-NEXT:    subl %edx, %ecx
+; X86-NOPOPCNT-NEXT:    movl %ecx, %edx
+; X86-NOPOPCNT-NEXT:    andl $858993459, %edx # imm = 0x33333333
+; X86-NOPOPCNT-NEXT:    shrl $2, %ecx
+; X86-NOPOPCNT-NEXT:    andl $858993459, %ecx # imm = 0x33333333
+; X86-NOPOPCNT-NEXT:    addl %edx, %ecx
+; X86-NOPOPCNT-NEXT:    movl %ecx, %edx
+; X86-NOPOPCNT-NEXT:    shrl $4, %edx
+; X86-NOPOPCNT-NEXT:    addl %ecx, %edx
+; X86-NOPOPCNT-NEXT:    andl $17764111, %edx # imm = 0x10F0F0F
+; X86-NOPOPCNT-NEXT:    imull $16843009, %edx, %ecx # imm = 0x1010101
+; X86-NOPOPCNT-NEXT:    shrl $24, %ecx
+; X86-NOPOPCNT-NEXT:    movl %eax, %edx
+; X86-NOPOPCNT-NEXT:    shrl %edx
+; X86-NOPOPCNT-NEXT:    andl $1431655765, %edx # imm = 0x55555555
+; X86-NOPOPCNT-NEXT:    subl %edx, %eax
+; X86-NOPOPCNT-NEXT:    movl %eax, %edx
+; X86-NOPOPCNT-NEXT:    andl $858993459, %edx # imm = 0x33333333
+; X86-NOPOPCNT-NEXT:    shrl $2, %eax
+; X86-NOPOPCNT-NEXT:    andl $858993459, %eax # imm = 0x33333333
+; X86-NOPOPCNT-NEXT:    addl %edx, %eax
+; X86-NOPOPCNT-NEXT:    movl %eax, %edx
+; X86-NOPOPCNT-NEXT:    shrl $4, %edx
+; X86-NOPOPCNT-NEXT:    addl %eax, %edx
+; X86-NOPOPCNT-NEXT:    andl $17764111, %edx # imm = 0x10F0F0F
+; X86-NOPOPCNT-NEXT:    imull $16843009, %edx, %eax # imm = 0x1010101
+; X86-NOPOPCNT-NEXT:    shrl $24, %eax
+; X86-NOPOPCNT-NEXT:    addl %ecx, %eax
+; X86-NOPOPCNT-NEXT:    andl $1, %eax
+; X86-NOPOPCNT-NEXT:    retl
+;
+; X64-NOPOPCNT-LABEL: parity_64_trunc:
+; X64-NOPOPCNT:       # %bb.0:
+; X64-NOPOPCNT-NEXT:    movq %rdi, %rax
+; X64-NOPOPCNT-NEXT:    shrq %rax
+; X64-NOPOPCNT-NEXT:    movabsq $6148914691236517205, %rcx # imm = 0x5555555555555555
+; X64-NOPOPCNT-NEXT:    andq %rax, %rcx
+; X64-NOPOPCNT-NEXT:    subq %rcx, %rdi
+; X64-NOPOPCNT-NEXT:    movabsq $3689348814741910323, %rax # imm = 0x3333333333333333
+; X64-NOPOPCNT-NEXT:    movq %rdi, %rcx
+; X64-NOPOPCNT-NEXT:    andq %rax, %rcx
+; X64-NOPOPCNT-NEXT:    shrq $2, %rdi
+; X64-NOPOPCNT-NEXT:    andq %rax, %rdi
+; X64-NOPOPCNT-NEXT:    addq %rcx, %rdi
+; X64-NOPOPCNT-NEXT:    movq %rdi, %rax
+; X64-NOPOPCNT-NEXT:    shrq $4, %rax
+; X64-NOPOPCNT-NEXT:    addq %rdi, %rax
+; X64-NOPOPCNT-NEXT:    movabsq $76296276040158991, %rcx # imm = 0x10F0F0F0F0F0F0F
+; X64-NOPOPCNT-NEXT:    andq %rax, %rcx
+; X64-NOPOPCNT-NEXT:    movabsq $72340172838076673, %rax # imm = 0x101010101010101
+; X64-NOPOPCNT-NEXT:    imulq %rcx, %rax
+; X64-NOPOPCNT-NEXT:    shrq $56, %rax
+; X64-NOPOPCNT-NEXT:    andl $1, %eax
+; X64-NOPOPCNT-NEXT:    # kill: def $eax killed $eax killed $rax
+; X64-NOPOPCNT-NEXT:    retq
+;
+; X86-POPCNT-LABEL: parity_64_trunc:
+; X86-POPCNT:       # %bb.0:
+; X86-POPCNT-NEXT:    popcntl {{[0-9]+}}(%esp), %ecx
+; X86-POPCNT-NEXT:    popcntl {{[0-9]+}}(%esp), %eax
+; X86-POPCNT-NEXT:    addl %ecx, %eax
+; X86-POPCNT-NEXT:    andl $1, %eax
+; X86-POPCNT-NEXT:    retl
+;
+; X64-POPCNT-LABEL: parity_64_trunc:
+; X64-POPCNT:       # %bb.0:
+; X64-POPCNT-NEXT:    popcntq %rdi, %rax
+; X64-POPCNT-NEXT:    andl $1, %eax
+; X64-POPCNT-NEXT:    # kill: def $eax killed $eax killed $rax
+; X64-POPCNT-NEXT:    retq
+  %1 = tail call i64 @llvm.ctpop.i64(i64 %x)
+  %2 = trunc i64 %1 to i32
+  %3 = and i32 %2, 1
+  ret i32 %3
+}
+
+define i8 @parity_32_trunc(i32 %x) {
+; X86-NOPOPCNT-LABEL: parity_32_trunc:
+; X86-NOPOPCNT:       # %bb.0:
+; X86-NOPOPCNT-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOPOPCNT-NEXT:    movl %eax, %ecx
+; X86-NOPOPCNT-NEXT:    shrl %ecx
+; X86-NOPOPCNT-NEXT:    andl $1431655765, %ecx # imm = 0x55555555
+; X86-NOPOPCNT-NEXT:    subl %ecx, %eax
+; X86-NOPOPCNT-NEXT:    movl %eax, %ecx
+; X86-NOPOPCNT-NEXT:    andl $858993459, %ecx # imm = 0x33333333
+; X86-NOPOPCNT-NEXT:    shrl $2, %eax
+; X86-NOPOPCNT-NEXT:    andl $858993459, %eax # imm = 0x33333333
+; X86-NOPOPCNT-NEXT:    addl %ecx, %eax
+; X86-NOPOPCNT-NEXT:    movl %eax, %ecx
+; X86-NOPOPCNT-NEXT:    shrl $4, %ecx
+; X86-NOPOPCNT-NEXT:    addl %eax, %ecx
+; X86-NOPOPCNT-NEXT:    andl $17764111, %ecx # imm = 0x10F0F0F
+; X86-NOPOPCNT-NEXT:    imull $16843009, %ecx, %eax # imm = 0x1010101
+; X86-NOPOPCNT-NEXT:    shrl $24, %eax
+; X86-NOPOPCNT-NEXT:    andb $1, %al
+; X86-NOPOPCNT-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NOPOPCNT-NEXT:    retl
+;
+; X64-NOPOPCNT-LABEL: parity_32_trunc:
+; X64-NOPOPCNT:       # %bb.0:
+; X64-NOPOPCNT-NEXT:    movl %edi, %eax
+; X64-NOPOPCNT-NEXT:    shrl %eax
+; X64-NOPOPCNT-NEXT:    andl $1431655765, %eax # imm = 0x55555555
+; X64-NOPOPCNT-NEXT:    subl %eax, %edi
+; X64-NOPOPCNT-NEXT:    movl %edi, %eax
+; X64-NOPOPCNT-NEXT:    andl $858993459, %eax # imm = 0x33333333
+; X64-NOPOPCNT-NEXT:    shrl $2, %edi
+; X64-NOPOPCNT-NEXT:    andl $858993459, %edi # imm = 0x33333333
+; X64-NOPOPCNT-NEXT:    addl %eax, %edi
+; X64-NOPOPCNT-NEXT:    movl %edi, %eax
+; X64-NOPOPCNT-NEXT:    shrl $4, %eax
+; X64-NOPOPCNT-NEXT:    addl %edi, %eax
+; X64-NOPOPCNT-NEXT:    andl $17764111, %eax # imm = 0x10F0F0F
+; X64-NOPOPCNT-NEXT:    imull $16843009, %eax, %eax # imm = 0x1010101
+; X64-NOPOPCNT-NEXT:    shrl $24, %eax
+; X64-NOPOPCNT-NEXT:    andb $1, %al
+; X64-NOPOPCNT-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NOPOPCNT-NEXT:    retq
+;
+; X86-POPCNT-LABEL: parity_32_trunc:
+; X86-POPCNT:       # %bb.0:
+; X86-POPCNT-NEXT:    popcntl {{[0-9]+}}(%esp), %eax
+; X86-POPCNT-NEXT:    andb $1, %al
+; X86-POPCNT-NEXT:    # kill: def $al killed $al killed $eax
+; X86-POPCNT-NEXT:    retl
+;
+; X64-POPCNT-LABEL: parity_32_trunc:
+; X64-POPCNT:       # %bb.0:
+; X64-POPCNT-NEXT:    popcntl %edi, %eax
+; X64-POPCNT-NEXT:    andb $1, %al
+; X64-POPCNT-NEXT:    # kill: def $al killed $al killed $eax
+; X64-POPCNT-NEXT:    retq
+  %1 = tail call i32 @llvm.ctpop.i32(i32 %x)
+  %2 = trunc i32 %1 to i8
+  %3 = and i8 %2, 1
+  ret i8 %3
+}
+
 declare i32 @llvm.ctpop.i32(i32 %x)
 declare i64 @llvm.ctpop.i64(i64 %x)

From 56166a3a5229722e442e7dfd861b6d83ebf6c8b5 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@intel.com>
Date: Sat, 1 Aug 2020 22:05:10 -0700
Subject: [PATCH 0835/1035] [X86] Improve parity idiom recognition to handle
 (and (truncate (ctpop X)), 1).

Fixes part of PR46954
---
 llvm/lib/Target/X86/X86ISelLowering.cpp |  34 ++++---
 llvm/test/CodeGen/X86/parity.ll         | 118 ++++++------------------
 2 files changed, 49 insertions(+), 103 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index c135b91620724..752b65fc03310 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -42744,26 +42744,30 @@ static SDValue combineAndLoadToBZHI(SDNode *Node, SelectionDAG &DAG,
 // Turn it into series of XORs and a setnp.
 static SDValue combineParity(SDNode *N, SelectionDAG &DAG,
                              const X86Subtarget &Subtarget) {
-  EVT VT = N->getValueType(0);
-
-  // We only support 64-bit and 32-bit. 64-bit requires special handling
-  // unless the 64-bit popcnt instruction is legal.
-  if (VT != MVT::i32 && VT != MVT::i64)
-    return SDValue();
+  SDValue N0 = N->getOperand(0);
+  SDValue N1 = N->getOperand(1);
 
-  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
-  if (TLI.isTypeLegal(VT) && TLI.isOperationLegal(ISD::CTPOP, VT))
+  // RHS needs to be 1.
+  if (!isOneConstant(N1))
     return SDValue();
 
-  SDValue N0 = N->getOperand(0);
-  SDValue N1 = N->getOperand(1);
+  // Popcnt may be truncated.
+  if (N0.getOpcode() == ISD::TRUNCATE && N0.hasOneUse())
+    N0 = N0.getOperand(0);
 
   // LHS needs to be a single use CTPOP.
   if (N0.getOpcode() != ISD::CTPOP || !N0.hasOneUse())
     return SDValue();
 
-  // RHS needs to be 1.
-  if (!isOneConstant(N1))
+  EVT VT = N0.getValueType();
+
+  // We only support 64-bit and 32-bit. 64-bit requires special handling
+  // unless the 64-bit popcnt instruction is legal.
+  if (VT != MVT::i32 && VT != MVT::i64)
+    return SDValue();
+
+  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+  if (TLI.isTypeLegal(VT) && TLI.isOperationLegal(ISD::CTPOP, VT))
     return SDValue();
 
   SDLoc DL(N);
@@ -42782,7 +42786,7 @@ static SDValue combineParity(SDNode *N, SelectionDAG &DAG,
     SDValue Parity = DAG.getNode(ISD::AND, DL, MVT::i32,
                                  DAG.getNode(ISD::CTPOP, DL, MVT::i32, X),
                                  DAG.getConstant(1, DL, MVT::i32));
-    return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Parity);
+    return DAG.getZExtOrTrunc(Parity, DL, N->getValueType(0));
   }
   assert(VT == MVT::i32 && "Unexpected VT!");
 
@@ -42803,8 +42807,8 @@ static SDValue combineParity(SDNode *N, SelectionDAG &DAG,
 
   // Copy the inverse of the parity flag into a register with setcc.
   SDValue Setnp = getSETCC(X86::COND_NP, Flags, DL, DAG);
-  // Zero extend to original type.
-  return DAG.getNode(ISD::ZERO_EXTEND, DL, N->getValueType(0), Setnp);
+  // Extend or truncate to the original type.
+  return DAG.getZExtOrTrunc(Setnp, DL, N->getValueType(0));
 }
 
 
diff --git a/llvm/test/CodeGen/X86/parity.ll b/llvm/test/CodeGen/X86/parity.ll
index 404d7a68a6e74..8637058e06807 100644
--- a/llvm/test/CodeGen/X86/parity.ll
+++ b/llvm/test/CodeGen/X86/parity.ll
@@ -96,71 +96,35 @@ define i32 @parity_64_trunc(i64 %x) {
 ; X86-NOPOPCNT-LABEL: parity_64_trunc:
 ; X86-NOPOPCNT:       # %bb.0:
 ; X86-NOPOPCNT-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NOPOPCNT-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NOPOPCNT-NEXT:    movl %ecx, %edx
-; X86-NOPOPCNT-NEXT:    shrl %edx
-; X86-NOPOPCNT-NEXT:    andl $1431655765, %edx # imm = 0x55555555
-; X86-NOPOPCNT-NEXT:    subl %edx, %ecx
-; X86-NOPOPCNT-NEXT:    movl %ecx, %edx
-; X86-NOPOPCNT-NEXT:    andl $858993459, %edx # imm = 0x33333333
-; X86-NOPOPCNT-NEXT:    shrl $2, %ecx
-; X86-NOPOPCNT-NEXT:    andl $858993459, %ecx # imm = 0x33333333
-; X86-NOPOPCNT-NEXT:    addl %edx, %ecx
-; X86-NOPOPCNT-NEXT:    movl %ecx, %edx
-; X86-NOPOPCNT-NEXT:    shrl $4, %edx
-; X86-NOPOPCNT-NEXT:    addl %ecx, %edx
-; X86-NOPOPCNT-NEXT:    andl $17764111, %edx # imm = 0x10F0F0F
-; X86-NOPOPCNT-NEXT:    imull $16843009, %edx, %ecx # imm = 0x1010101
-; X86-NOPOPCNT-NEXT:    shrl $24, %ecx
-; X86-NOPOPCNT-NEXT:    movl %eax, %edx
-; X86-NOPOPCNT-NEXT:    shrl %edx
-; X86-NOPOPCNT-NEXT:    andl $1431655765, %edx # imm = 0x55555555
-; X86-NOPOPCNT-NEXT:    subl %edx, %eax
-; X86-NOPOPCNT-NEXT:    movl %eax, %edx
-; X86-NOPOPCNT-NEXT:    andl $858993459, %edx # imm = 0x33333333
-; X86-NOPOPCNT-NEXT:    shrl $2, %eax
-; X86-NOPOPCNT-NEXT:    andl $858993459, %eax # imm = 0x33333333
-; X86-NOPOPCNT-NEXT:    addl %edx, %eax
-; X86-NOPOPCNT-NEXT:    movl %eax, %edx
-; X86-NOPOPCNT-NEXT:    shrl $4, %edx
-; X86-NOPOPCNT-NEXT:    addl %eax, %edx
-; X86-NOPOPCNT-NEXT:    andl $17764111, %edx # imm = 0x10F0F0F
-; X86-NOPOPCNT-NEXT:    imull $16843009, %edx, %eax # imm = 0x1010101
-; X86-NOPOPCNT-NEXT:    shrl $24, %eax
-; X86-NOPOPCNT-NEXT:    addl %ecx, %eax
-; X86-NOPOPCNT-NEXT:    andl $1, %eax
+; X86-NOPOPCNT-NEXT:    xorl {{[0-9]+}}(%esp), %eax
+; X86-NOPOPCNT-NEXT:    movl %eax, %ecx
+; X86-NOPOPCNT-NEXT:    shrl $16, %ecx
+; X86-NOPOPCNT-NEXT:    xorl %eax, %ecx
+; X86-NOPOPCNT-NEXT:    xorl %eax, %eax
+; X86-NOPOPCNT-NEXT:    xorb %ch, %cl
+; X86-NOPOPCNT-NEXT:    setnp %al
 ; X86-NOPOPCNT-NEXT:    retl
 ;
 ; X64-NOPOPCNT-LABEL: parity_64_trunc:
 ; X64-NOPOPCNT:       # %bb.0:
 ; X64-NOPOPCNT-NEXT:    movq %rdi, %rax
-; X64-NOPOPCNT-NEXT:    shrq %rax
-; X64-NOPOPCNT-NEXT:    movabsq $6148914691236517205, %rcx # imm = 0x5555555555555555
-; X64-NOPOPCNT-NEXT:    andq %rax, %rcx
-; X64-NOPOPCNT-NEXT:    subq %rcx, %rdi
-; X64-NOPOPCNT-NEXT:    movabsq $3689348814741910323, %rax # imm = 0x3333333333333333
-; X64-NOPOPCNT-NEXT:    movq %rdi, %rcx
-; X64-NOPOPCNT-NEXT:    andq %rax, %rcx
-; X64-NOPOPCNT-NEXT:    shrq $2, %rdi
-; X64-NOPOPCNT-NEXT:    andq %rax, %rdi
-; X64-NOPOPCNT-NEXT:    addq %rcx, %rdi
-; X64-NOPOPCNT-NEXT:    movq %rdi, %rax
-; X64-NOPOPCNT-NEXT:    shrq $4, %rax
-; X64-NOPOPCNT-NEXT:    addq %rdi, %rax
-; X64-NOPOPCNT-NEXT:    movabsq $76296276040158991, %rcx # imm = 0x10F0F0F0F0F0F0F
-; X64-NOPOPCNT-NEXT:    andq %rax, %rcx
-; X64-NOPOPCNT-NEXT:    movabsq $72340172838076673, %rax # imm = 0x101010101010101
-; X64-NOPOPCNT-NEXT:    imulq %rcx, %rax
-; X64-NOPOPCNT-NEXT:    shrq $56, %rax
-; X64-NOPOPCNT-NEXT:    andl $1, %eax
-; X64-NOPOPCNT-NEXT:    # kill: def $eax killed $eax killed $rax
+; X64-NOPOPCNT-NEXT:    shrq $32, %rax
+; X64-NOPOPCNT-NEXT:    xorl %edi, %eax
+; X64-NOPOPCNT-NEXT:    movl %eax, %ecx
+; X64-NOPOPCNT-NEXT:    shrl $16, %ecx
+; X64-NOPOPCNT-NEXT:    xorl %eax, %ecx
+; X64-NOPOPCNT-NEXT:    movl %ecx, %edx
+; X64-NOPOPCNT-NEXT:    shrl $8, %edx
+; X64-NOPOPCNT-NEXT:    xorl %eax, %eax
+; X64-NOPOPCNT-NEXT:    xorb %cl, %dl
+; X64-NOPOPCNT-NEXT:    setnp %al
 ; X64-NOPOPCNT-NEXT:    retq
 ;
 ; X86-POPCNT-LABEL: parity_64_trunc:
 ; X86-POPCNT:       # %bb.0:
-; X86-POPCNT-NEXT:    popcntl {{[0-9]+}}(%esp), %ecx
-; X86-POPCNT-NEXT:    popcntl {{[0-9]+}}(%esp), %eax
-; X86-POPCNT-NEXT:    addl %ecx, %eax
+; X86-POPCNT-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-POPCNT-NEXT:    xorl {{[0-9]+}}(%esp), %eax
+; X86-POPCNT-NEXT:    popcntl %eax, %eax
 ; X86-POPCNT-NEXT:    andl $1, %eax
 ; X86-POPCNT-NEXT:    retl
 ;
@@ -181,43 +145,21 @@ define i8 @parity_32_trunc(i32 %x) {
 ; X86-NOPOPCNT:       # %bb.0:
 ; X86-NOPOPCNT-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NOPOPCNT-NEXT:    movl %eax, %ecx
-; X86-NOPOPCNT-NEXT:    shrl %ecx
-; X86-NOPOPCNT-NEXT:    andl $1431655765, %ecx # imm = 0x55555555
-; X86-NOPOPCNT-NEXT:    subl %ecx, %eax
-; X86-NOPOPCNT-NEXT:    movl %eax, %ecx
-; X86-NOPOPCNT-NEXT:    andl $858993459, %ecx # imm = 0x33333333
-; X86-NOPOPCNT-NEXT:    shrl $2, %eax
-; X86-NOPOPCNT-NEXT:    andl $858993459, %eax # imm = 0x33333333
-; X86-NOPOPCNT-NEXT:    addl %ecx, %eax
-; X86-NOPOPCNT-NEXT:    movl %eax, %ecx
-; X86-NOPOPCNT-NEXT:    shrl $4, %ecx
-; X86-NOPOPCNT-NEXT:    addl %eax, %ecx
-; X86-NOPOPCNT-NEXT:    andl $17764111, %ecx # imm = 0x10F0F0F
-; X86-NOPOPCNT-NEXT:    imull $16843009, %ecx, %eax # imm = 0x1010101
-; X86-NOPOPCNT-NEXT:    shrl $24, %eax
-; X86-NOPOPCNT-NEXT:    andb $1, %al
-; X86-NOPOPCNT-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NOPOPCNT-NEXT:    shrl $16, %ecx
+; X86-NOPOPCNT-NEXT:    xorl %eax, %ecx
+; X86-NOPOPCNT-NEXT:    xorb %ch, %cl
+; X86-NOPOPCNT-NEXT:    setnp %al
 ; X86-NOPOPCNT-NEXT:    retl
 ;
 ; X64-NOPOPCNT-LABEL: parity_32_trunc:
 ; X64-NOPOPCNT:       # %bb.0:
 ; X64-NOPOPCNT-NEXT:    movl %edi, %eax
-; X64-NOPOPCNT-NEXT:    shrl %eax
-; X64-NOPOPCNT-NEXT:    andl $1431655765, %eax # imm = 0x55555555
-; X64-NOPOPCNT-NEXT:    subl %eax, %edi
-; X64-NOPOPCNT-NEXT:    movl %edi, %eax
-; X64-NOPOPCNT-NEXT:    andl $858993459, %eax # imm = 0x33333333
-; X64-NOPOPCNT-NEXT:    shrl $2, %edi
-; X64-NOPOPCNT-NEXT:    andl $858993459, %edi # imm = 0x33333333
-; X64-NOPOPCNT-NEXT:    addl %eax, %edi
-; X64-NOPOPCNT-NEXT:    movl %edi, %eax
-; X64-NOPOPCNT-NEXT:    shrl $4, %eax
-; X64-NOPOPCNT-NEXT:    addl %edi, %eax
-; X64-NOPOPCNT-NEXT:    andl $17764111, %eax # imm = 0x10F0F0F
-; X64-NOPOPCNT-NEXT:    imull $16843009, %eax, %eax # imm = 0x1010101
-; X64-NOPOPCNT-NEXT:    shrl $24, %eax
-; X64-NOPOPCNT-NEXT:    andb $1, %al
-; X64-NOPOPCNT-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NOPOPCNT-NEXT:    shrl $16, %eax
+; X64-NOPOPCNT-NEXT:    xorl %edi, %eax
+; X64-NOPOPCNT-NEXT:    movl %eax, %ecx
+; X64-NOPOPCNT-NEXT:    shrl $8, %ecx
+; X64-NOPOPCNT-NEXT:    xorb %al, %cl
+; X64-NOPOPCNT-NEXT:    setnp %al
 ; X64-NOPOPCNT-NEXT:    retq
 ;
 ; X86-POPCNT-LABEL: parity_32_trunc:

From 21c165de2a1bcca9dceb452f637d9e8959fba113 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= <mgorny@gentoo.org>
Date: Fri, 31 Jul 2020 00:11:40 +0200
Subject: [PATCH 0836/1035] [CMake] Pass bugreport URL to standalone clang
 build

BUG_REPORT_URL is currently used both in LLVM and in Clang but declared
only in the latter.  This means that it's missing in standalone clang
builds and the driver ends up outputting:

  PLEASE submit a bug report to  and include [...]

(note the missing URL)

To fix this, include LLVM_PACKAGE_BUGREPORT in LLVMConfig.cmake
(similarly to how we pass PACKAGE_VERSION) and use it to fill
BUG_REPORT_URL when building clang standalone.

Differential Revision: https://reviews.llvm.org/D84987
---
 clang/CMakeLists.txt                   | 2 ++
 llvm/cmake/modules/LLVMConfig.cmake.in | 1 +
 2 files changed, 3 insertions(+)

diff --git a/clang/CMakeLists.txt b/clang/CMakeLists.txt
index 0f08538495fca..c487e506cae11 100644
--- a/clang/CMakeLists.txt
+++ b/clang/CMakeLists.txt
@@ -121,6 +121,8 @@ if( CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR )
   include(LLVMDistributionSupport)
 
   set(PACKAGE_VERSION "${LLVM_PACKAGE_VERSION}")
+  set(BUG_REPORT_URL "${LLVM_PACKAGE_BUGREPORT}" CACHE STRING
+    "Default URL where bug reports are to be submitted.")
 
   if (NOT DEFINED LLVM_INCLUDE_TESTS)
     set(LLVM_INCLUDE_TESTS ON)
diff --git a/llvm/cmake/modules/LLVMConfig.cmake.in b/llvm/cmake/modules/LLVMConfig.cmake.in
index 17cc5eacc57b7..a5c370bbc25e4 100644
--- a/llvm/cmake/modules/LLVMConfig.cmake.in
+++ b/llvm/cmake/modules/LLVMConfig.cmake.in
@@ -7,6 +7,7 @@ set(LLVM_VERSION_MINOR @LLVM_VERSION_MINOR@)
 set(LLVM_VERSION_PATCH @LLVM_VERSION_PATCH@)
 set(LLVM_VERSION_SUFFIX @LLVM_VERSION_SUFFIX@)
 set(LLVM_PACKAGE_VERSION @PACKAGE_VERSION@)
+set(LLVM_PACKAGE_BUGREPORT @PACKAGE_BUGREPORT@)
 
 set(LLVM_BUILD_TYPE @CMAKE_BUILD_TYPE@)
 

From ffb47352005f2006851a3dcc2083329f434b8730 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Fri, 31 Jul 2020 22:10:39 +0100
Subject: [PATCH 0837/1035] [SCEV] Precommit tests with signed counting down
 loop.

From PR46939.
---
 .../pr46939-trip-count-count-down.ll          | 38 +++++++++++++++++++
 1 file changed, 38 insertions(+)
 create mode 100644 llvm/test/Analysis/ScalarEvolution/pr46939-trip-count-count-down.ll

diff --git a/llvm/test/Analysis/ScalarEvolution/pr46939-trip-count-count-down.ll b/llvm/test/Analysis/ScalarEvolution/pr46939-trip-count-count-down.ll
new file mode 100644
index 0000000000000..f1001ecec0fde
--- /dev/null
+++ b/llvm/test/Analysis/ScalarEvolution/pr46939-trip-count-count-down.ll
@@ -0,0 +1,38 @@
+; RUN: opt -analyze -scalar-evolution %s 2>&1 | FileCheck %s
+
+declare void @iteration()
+
+define void @reverse_loop(i32 %n) {
+; CHECK-LABEL: 'reverse_loop'
+; CHECK-NEXT:  Classifying expressions for: @reverse_loop
+; CHECK-NEXT:    %i.011 = phi i32 [ %n, %for.body.lr.ph ], [ %dec, %for.body ]
+; CHECK-NEXT:    --> {%n,+,-1}<nsw><%for.body> U: full-set S: full-set Exits: (0 smin %n) LoopDispositions: { %for.body: Computable }
+; CHECK-NEXT:    %dec = add nsw i32 %i.011, -1
+; CHECK-NEXT:    --> {(-1 + %n),+,-1}<nw><%for.body> U: full-set S: full-set Exits: (-1 + (0 smin %n)) LoopDispositions: { %for.body: Computable }
+; CHECK-NEXT:  Determining loop execution counts for: @reverse_loop
+; CHECK-NEXT:  Loop %for.body: backedge-taken count is ((-1 * (0 smin %n)) + %n)
+; CHECK-NEXT:  Loop %for.body: max backedge-taken count is 2147483647
+; CHECK-NEXT:  Loop %for.body: Predicated backedge-taken count is ((-1 * (0 smin %n)) + %n)
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %for.body: Trip multiple is 1
+;
+entry:
+  %cmp10 = icmp sgt i32 %n, -1
+  br i1 %cmp10, label %for.body.lr.ph, label %for.cond.cleanup
+
+for.body.lr.ph:
+  br label %for.body
+
+for.body:
+  %i.011 = phi i32 [ %n, %for.body.lr.ph ], [ %dec, %for.body ]
+  call void @iteration()
+  %dec = add nsw i32 %i.011, -1
+  %cmp = icmp sgt i32 %i.011, 0
+  br i1 %cmp, label %for.body, label %for.cond.cleanup.loopexit
+
+for.cond.cleanup.loopexit:
+  br label %for.cond.cleanup
+
+for.cond.cleanup:
+  ret void
+}

From d3f01b6681bf73b382521db812341e0041787cc9 Mon Sep 17 00:00:00 2001
From: Shinji Okumura <okuraofvegetable@gmail.com>
Date: Sun, 2 Aug 2020 18:37:07 +0900
Subject: [PATCH 0838/1035] [Attributor] AAPotentialValues Interface

This is a split patch of D80991.
This patch introduces AAPotentialValues and its interface only.
For more detail of AAPotentialValues abstract attribute, see the original patch.

Reviewed By: jdoerfert

Differential Revision: https://reviews.llvm.org/D83283
---
 llvm/include/llvm/ADT/APInt.h                 |  26 +-
 llvm/include/llvm/Transforms/IPO/Attributor.h | 187 +++++++++
 llvm/lib/IR/LLVMContextImpl.h                 |  22 +-
 llvm/lib/Transforms/IPO/Attributor.cpp        |  13 +
 .../Transforms/IPO/AttributorAttributes.cpp   | 162 +++++++
 llvm/test/Transforms/Attributor/potential.ll  | 394 ++++++++++++++++++
 6 files changed, 782 insertions(+), 22 deletions(-)
 create mode 100644 llvm/test/Transforms/Attributor/potential.ll

diff --git a/llvm/include/llvm/ADT/APInt.h b/llvm/include/llvm/ADT/APInt.h
index f7df648d27ed6..14643a14a2f30 100644
--- a/llvm/include/llvm/ADT/APInt.h
+++ b/llvm/include/llvm/ADT/APInt.h
@@ -15,6 +15,7 @@
 #ifndef LLVM_ADT_APINT_H
 #define LLVM_ADT_APINT_H
 
+#include "llvm/ADT/DenseMapInfo.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/MathExtras.h"
 #include <cassert>
@@ -96,7 +97,7 @@ class LLVM_NODISCARD APInt {
 
   unsigned BitWidth; ///< The number of bits in this APInt.
 
-  friend struct DenseMapAPIntKeyInfo;
+  friend struct DenseMapInfo<APInt>;
 
   friend class APSInt;
 
@@ -2288,6 +2289,29 @@ void StoreIntToMemory(const APInt &IntVal, uint8_t *Dst, unsigned StoreBytes);
 /// from Src into IntVal, which is assumed to be wide enough and to hold zero.
 void LoadIntFromMemory(APInt &IntVal, const uint8_t *Src, unsigned LoadBytes);
 
+/// Provide DenseMapInfo for APInt.
+template <> struct DenseMapInfo<APInt> {
+  static inline APInt getEmptyKey() {
+    APInt V(nullptr, 0);
+    V.U.VAL = 0;
+    return V;
+  }
+
+  static inline APInt getTombstoneKey() {
+    APInt V(nullptr, 0);
+    V.U.VAL = 1;
+    return V;
+  }
+
+  static unsigned getHashValue(const APInt &Key) {
+    return static_cast<unsigned>(hash_value(Key));
+  }
+
+  static bool isEqual(const APInt &LHS, const APInt &RHS) {
+    return LHS.getBitWidth() == RHS.getBitWidth() && LHS == RHS;
+  }
+};
+
 } // namespace llvm
 
 #endif
diff --git a/llvm/include/llvm/Transforms/IPO/Attributor.h b/llvm/include/llvm/Transforms/IPO/Attributor.h
index 418a007319679..b15c8f0dd6c35 100644
--- a/llvm/include/llvm/Transforms/IPO/Attributor.h
+++ b/llvm/include/llvm/Transforms/IPO/Attributor.h
@@ -97,6 +97,7 @@
 #ifndef LLVM_TRANSFORMS_IPO_ATTRIBUTOR_H
 #define LLVM_TRANSFORMS_IPO_ATTRIBUTOR_H
 
+#include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/GraphTraits.h"
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/STLExtras.h"
@@ -115,6 +116,7 @@
 #include "llvm/IR/PassManager.h"
 #include "llvm/Support/Allocator.h"
 #include "llvm/Support/Casting.h"
+#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/DOTGraphTraits.h"
 #include "llvm/Support/GraphWriter.h"
 #include "llvm/Support/TimeProfiler.h"
@@ -3348,6 +3350,191 @@ struct AAValueConstantRange
   static const char ID;
 };
 
+/// A class for a set state.
+/// The assumed boolean state indicates whether the corresponding set is full
+/// set or not. If the assumed state is false, this is the worst state. The
+/// worst state (invalid state) of set of potential values is when the set
+/// contains every possible value (i.e. we cannot in any way limit the value
+/// that the target position can take). That never happens naturally, we only
+/// force it. As for the conditions under which we force it, see
+/// AAPotentialValues.
+template <typename MemberTy, typename KeyInfo = DenseMapInfo<MemberTy>>
+struct PotentialValuesState : BooleanState {
+  using SetTy = DenseSet<MemberTy, KeyInfo>;
+
+  PotentialValuesState() : BooleanState(true) {}
+
+  PotentialValuesState(bool IsValid) : BooleanState(IsValid) {}
+
+  /// Return this set. We should check whether this set is valid or not by
+  /// isValidState() before calling this function.
+  const SetTy &getAssumedSet() const {
+    assert(isValidState() && "This set shoud not be used when it is invalid!");
+    return Set;
+  }
+
+  bool operator==(const PotentialValuesState &RHS) const {
+    if (isValidState() != RHS.isValidState())
+      return false;
+    if (!isValidState() && !RHS.isValidState())
+      return true;
+    return Set == RHS.getAssumedSet();
+  }
+
+  /// Maximum number of potential values to be tracked.
+  /// This is set by -attributor-max-potential-values command line option
+  static unsigned MaxPotentialValues;
+
+  /// Return empty set as the best state of potential values.
+  static PotentialValuesState getBestState() {
+    return PotentialValuesState(true);
+  }
+
+  static PotentialValuesState getBestState(PotentialValuesState &PVS) {
+    return getBestState();
+  }
+
+  /// Return full set as the worst state of potential values.
+  static PotentialValuesState getWorstState() {
+    return PotentialValuesState(false);
+  }
+
+  /// Union assumed set with the passed value.
+  void unionAssumed(const MemberTy &C) { insert(C); }
+
+  /// Union assumed set with assumed set of the passed state \p PVS.
+  void unionAssumed(const PotentialValuesState &PVS) { unionWith(PVS); }
+
+  /// "Clamp" this state with \p PVS.
+  PotentialValuesState operator^=(const PotentialValuesState &PVS) {
+    unionAssumed(PVS);
+    return *this;
+  }
+
+  PotentialValuesState operator&=(const PotentialValuesState &PVS) {
+    unionAssumed(PVS);
+    return *this;
+  }
+
+private:
+  /// Check the size of this set, and invalidate when the size is no
+  /// less than \p MaxPotentialValues threshold.
+  void checkAndInvalidate() {
+    if (Set.size() >= MaxPotentialValues)
+      indicatePessimisticFixpoint();
+  }
+
+  /// Insert an element into this set.
+  void insert(const MemberTy &C) {
+    if (!isValidState())
+      return;
+    Set.insert(C);
+    checkAndInvalidate();
+  }
+
+  /// Take union with R.
+  void unionWith(const PotentialValuesState &R) {
+    /// If this is a full set, do nothing.;
+    if (!isValidState())
+      return;
+    /// If R is full set, change L to a full set.
+    if (!R.isValidState()) {
+      indicatePessimisticFixpoint();
+      return;
+    }
+    for (const MemberTy &C : R.Set)
+      Set.insert(C);
+    checkAndInvalidate();
+  }
+
+  /// Take intersection with R.
+  void intersectWith(const PotentialValuesState &R) {
+    /// If R is a full set, do nothing.
+    if (!R.isValidState())
+      return;
+    /// If this is a full set, change this to R.
+    if (!isValidState()) {
+      *this = R;
+      return;
+    }
+    SetTy IntersectSet;
+    for (const MemberTy &C : Set) {
+      if (R.Set.count(C))
+        IntersectSet.insert(C);
+    }
+    Set = IntersectSet;
+  }
+
+  /// Container for potential values
+  SetTy Set;
+};
+
+using PotentialConstantIntValuesState = PotentialValuesState<APInt>;
+
+raw_ostream &operator<<(raw_ostream &OS,
+                        const PotentialConstantIntValuesState &R);
+
+/// An abstract interface for potential values analysis.
+///
+/// This AA collects potential values for each IR position.
+/// An assumed set of potential values is initialized with the empty set (the
+/// best state) and it will grow monotonically as we find more potential values
+/// for this position.
+/// The set might be forced to the worst state, that is, to contain every
+/// possible value for this position in 2 cases.
+///   1. We surpassed the \p MaxPotentialValues threshold. This includes the
+///      case that this position is affected (e.g. because of an operation) by a
+///      Value that is in the worst state.
+///   2. We tried to initialize on a Value that we cannot handle (e.g. an
+///      operator we do not currently handle).
+///
+/// TODO: Support values other than constant integers.
+struct AAPotentialValues
+    : public StateWrapper<PotentialConstantIntValuesState, AbstractAttribute> {
+  using Base = StateWrapper<PotentialConstantIntValuesState, AbstractAttribute>;
+  AAPotentialValues(const IRPosition &IRP, Attributor &A) : Base(IRP) {}
+
+  /// See AbstractAttribute::getState(...).
+  PotentialConstantIntValuesState &getState() override { return *this; }
+  const PotentialConstantIntValuesState &getState() const override {
+    return *this;
+  }
+
+  /// Create an abstract attribute view for the position \p IRP.
+  static AAPotentialValues &createForPosition(const IRPosition &IRP,
+                                              Attributor &A);
+
+  /// Return assumed constant for the associated value
+  Optional<ConstantInt *>
+  getAssumedConstantInt(Attributor &A,
+                        const Instruction *CtxI = nullptr) const {
+    if (!isValidState())
+      return nullptr;
+    if (getAssumedSet().size() == 1)
+      return cast<ConstantInt>(ConstantInt::get(getAssociatedValue().getType(),
+                                                *(getAssumedSet().begin())));
+    if (getAssumedSet().size() == 0)
+      return llvm::None;
+
+    return nullptr;
+  }
+
+  /// See AbstractAttribute::getName()
+  const std::string getName() const override { return "AAPotentialValues"; }
+
+  /// See AbstractAttribute::getIdAddr()
+  const char *getIdAddr() const override { return &ID; }
+
+  /// This function should return true if the type of the \p AA is
+  /// AAPotentialValues
+  static bool classof(const AbstractAttribute *AA) {
+    return (AA->getIdAddr() == &ID);
+  }
+
+  /// Unique ID (due to the unique address)
+  static const char ID;
+};
+
 /// Run options, used by the pass manager.
 enum AttributorRunOption {
   NONE = 0,
diff --git a/llvm/lib/IR/LLVMContextImpl.h b/llvm/lib/IR/LLVMContextImpl.h
index b97ac37c5fcfd..e8fdaa23761c3 100644
--- a/llvm/lib/IR/LLVMContextImpl.h
+++ b/llvm/lib/IR/LLVMContextImpl.h
@@ -57,27 +57,7 @@ class Type;
 class Value;
 class ValueHandleBase;
 
-struct DenseMapAPIntKeyInfo {
-  static inline APInt getEmptyKey() {
-    APInt V(nullptr, 0);
-    V.U.VAL = 0;
-    return V;
-  }
-
-  static inline APInt getTombstoneKey() {
-    APInt V(nullptr, 0);
-    V.U.VAL = 1;
-    return V;
-  }
-
-  static unsigned getHashValue(const APInt &Key) {
-    return static_cast<unsigned>(hash_value(Key));
-  }
-
-  static bool isEqual(const APInt &LHS, const APInt &RHS) {
-    return LHS.getBitWidth() == RHS.getBitWidth() && LHS == RHS;
-  }
-};
+using DenseMapAPIntKeyInfo = DenseMapInfo<APInt>;
 
 struct DenseMapAPFloatKeyInfo {
   static inline APFloat getEmptyKey() { return APFloat(APFloat::Bogus(), 1); }
diff --git a/llvm/lib/Transforms/IPO/Attributor.cpp b/llvm/lib/Transforms/IPO/Attributor.cpp
index 89971cc9c2946..764d71b98e063 100644
--- a/llvm/lib/Transforms/IPO/Attributor.cpp
+++ b/llvm/lib/Transforms/IPO/Attributor.cpp
@@ -2079,6 +2079,19 @@ raw_ostream &llvm::operator<<(raw_ostream &OS, const AbstractAttribute &AA) {
   return OS;
 }
 
+raw_ostream &llvm::operator<<(raw_ostream &OS,
+                              const PotentialConstantIntValuesState &S) {
+  OS << "set-state(< {";
+  if (!S.isValidState())
+    OS << "full-set";
+  else
+    for (auto &it : S.getAssumedSet())
+      OS << it << ", ";
+  OS << "} >)";
+
+  return OS;
+}
+
 void AbstractAttribute::print(raw_ostream &OS) const {
   OS << "[";
   OS << getName();
diff --git a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
index aa6bc94a3668e..a220bc001a4ce 100644
--- a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
+++ b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
@@ -45,6 +45,16 @@ static cl::opt<bool> ManifestInternal(
 static cl::opt<int> MaxHeapToStackSize("max-heap-to-stack-size", cl::init(128),
                                        cl::Hidden);
 
+static cl::opt<unsigned, true> MaxPotentialValues(
+    "attributor-max-potential-values", cl::Hidden,
+    cl::desc("Maximum number of potential values to be "
+             "tracked for each position."),
+    cl::location(llvm::PotentialConstantIntValuesState::MaxPotentialValues),
+    cl::init(7));
+
+template <>
+unsigned llvm::PotentialConstantIntValuesState::MaxPotentialValues = 0;
+
 STATISTIC(NumAAs, "Number of abstract attributes created");
 
 // Some helper macros to deal with statistics tracking.
@@ -120,6 +130,7 @@ PIPE_OPERATOR(AAMemoryLocation)
 PIPE_OPERATOR(AAValueConstantRange)
 PIPE_OPERATOR(AAPrivatizablePtr)
 PIPE_OPERATOR(AAUndefinedBehavior)
+PIPE_OPERATOR(AAPotentialValues)
 
 #undef PIPE_OPERATOR
 } // namespace llvm
@@ -7075,6 +7086,155 @@ struct AAValueConstantRangeCallSiteArgument : AAValueConstantRangeFloating {
     STATS_DECLTRACK_CSARG_ATTR(value_range)
   }
 };
+
+/// ------------------ Potential Values Attribute -------------------------
+
+struct AAPotentialValuesImpl : AAPotentialValues {
+  using StateType = PotentialConstantIntValuesState;
+
+  AAPotentialValuesImpl(const IRPosition &IRP, Attributor &A)
+      : AAPotentialValues(IRP, A) {}
+
+  /// See AbstractAttribute::getAsStr().
+  const std::string getAsStr() const override {
+    std::string Str;
+    llvm::raw_string_ostream OS(Str);
+    OS << getState();
+    return OS.str();
+  }
+
+  /// See AbstractAttribute::updateImpl(...).
+  ChangeStatus updateImpl(Attributor &A) override {
+    return indicatePessimisticFixpoint();
+  }
+};
+
+struct AAPotentialValuesArgument final
+    : AAArgumentFromCallSiteArguments<AAPotentialValues, AAPotentialValuesImpl,
+                                      PotentialConstantIntValuesState> {
+  using Base =
+      AAArgumentFromCallSiteArguments<AAPotentialValues, AAPotentialValuesImpl,
+                                      PotentialConstantIntValuesState>;
+  AAPotentialValuesArgument(const IRPosition &IRP, Attributor &A)
+      : Base(IRP, A) {}
+
+  /// See AbstractAttribute::initialize(..).
+  void initialize(Attributor &A) override {
+    if (!getAnchorScope() || getAnchorScope()->isDeclaration()) {
+      indicatePessimisticFixpoint();
+    } else {
+      Base::initialize(A);
+    }
+  }
+
+  /// See AbstractAttribute::trackStatistics()
+  void trackStatistics() const override {
+    STATS_DECLTRACK_ARG_ATTR(potential_values)
+  }
+};
+
+struct AAPotentialValuesReturned
+    : AAReturnedFromReturnedValues<AAPotentialValues, AAPotentialValuesImpl> {
+  using Base =
+      AAReturnedFromReturnedValues<AAPotentialValues, AAPotentialValuesImpl>;
+  AAPotentialValuesReturned(const IRPosition &IRP, Attributor &A)
+      : Base(IRP, A) {}
+
+  /// See AbstractAttribute::trackStatistics()
+  void trackStatistics() const override {
+    STATS_DECLTRACK_FNRET_ATTR(potential_values)
+  }
+};
+
+struct AAPotentialValuesFloating : AAPotentialValuesImpl {
+  AAPotentialValuesFloating(const IRPosition &IRP, Attributor &A)
+      : AAPotentialValuesImpl(IRP, A) {}
+
+  /// See AbstractAttribute::initialize(..).
+  void initialize(Attributor &A) override {
+    Value &V = getAssociatedValue();
+
+    if (auto *C = dyn_cast<ConstantInt>(&V)) {
+      unionAssumed(C->getValue());
+      indicateOptimisticFixpoint();
+      return;
+    }
+
+    if (isa<UndefValue>(&V)) {
+      // Collapse the undef state to 0.
+      unionAssumed(
+          APInt(/* numBits */ getAssociatedType()->getIntegerBitWidth(),
+                /* val */ 0));
+      indicateOptimisticFixpoint();
+      return;
+    }
+
+    if (isa<BinaryOperator>(&V) || isa<ICmpInst>(&V) || isa<CastInst>(&V))
+      return;
+
+    if (isa<SelectInst>(V) || isa<PHINode>(V))
+      return;
+
+    indicatePessimisticFixpoint();
+
+    LLVM_DEBUG(dbgs() << "[AAPotentialValues] We give up: "
+                      << getAssociatedValue() << "\n");
+  }
+
+  /// See AbstractAttribute::trackStatistics()
+  void trackStatistics() const override {
+    STATS_DECLTRACK_FLOATING_ATTR(potential_values)
+  }
+};
+
+struct AAPotentialValuesFunction : AAPotentialValuesImpl {
+  AAPotentialValuesFunction(const IRPosition &IRP, Attributor &A)
+      : AAPotentialValuesImpl(IRP, A) {}
+
+  /// See AbstractAttribute::initialize(...).
+  ChangeStatus updateImpl(Attributor &A) override {
+    llvm_unreachable("AAPotentialValues(Function|CallSite)::updateImpl will "
+                     "not be called");
+  }
+
+  /// See AbstractAttribute::trackStatistics()
+  void trackStatistics() const override {
+    STATS_DECLTRACK_FN_ATTR(potential_values)
+  }
+};
+
+struct AAPotentialValuesCallSite : AAPotentialValuesFunction {
+  AAPotentialValuesCallSite(const IRPosition &IRP, Attributor &A)
+      : AAPotentialValuesFunction(IRP, A) {}
+
+  /// See AbstractAttribute::trackStatistics()
+  void trackStatistics() const override {
+    STATS_DECLTRACK_CS_ATTR(potential_values)
+  }
+};
+
+struct AAPotentialValuesCallSiteReturned
+    : AACallSiteReturnedFromReturned<AAPotentialValues, AAPotentialValuesImpl> {
+  AAPotentialValuesCallSiteReturned(const IRPosition &IRP, Attributor &A)
+      : AACallSiteReturnedFromReturned<AAPotentialValues,
+                                       AAPotentialValuesImpl>(IRP, A) {}
+
+  /// See AbstractAttribute::trackStatistics()
+  void trackStatistics() const override {
+    STATS_DECLTRACK_CSRET_ATTR(potential_values)
+  }
+};
+
+struct AAPotentialValuesCallSiteArgument : AAPotentialValuesFloating {
+  AAPotentialValuesCallSiteArgument(const IRPosition &IRP, Attributor &A)
+      : AAPotentialValuesFloating(IRP, A) {}
+
+  /// See AbstractAttribute::trackStatistics()
+  void trackStatistics() const override {
+    STATS_DECLTRACK_CSARG_ATTR(potential_values)
+  }
+};
+
 } // namespace
 
 const char AAReturnedValues::ID = 0;
@@ -7098,6 +7258,7 @@ const char AAPrivatizablePtr::ID = 0;
 const char AAMemoryBehavior::ID = 0;
 const char AAMemoryLocation::ID = 0;
 const char AAValueConstantRange::ID = 0;
+const char AAPotentialValues::ID = 0;
 
 // Macro magic to create the static generator function for attributes that
 // follow the naming scheme.
@@ -7207,6 +7368,7 @@ CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AADereferenceable)
 CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAAlign)
 CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoCapture)
 CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAValueConstantRange)
+CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAPotentialValues)
 
 CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAValueSimplify)
 CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAIsDead)
diff --git a/llvm/test/Transforms/Attributor/potential.ll b/llvm/test/Transforms/Attributor/potential.ll
new file mode 100644
index 0000000000000..a4fc23c8b700b
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/potential.ll
@@ -0,0 +1,394 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=9 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=7 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
+; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
+;
+; Test for multiple potential values
+;
+; potential-test 1
+; bool iszero(int c) { return c == 0; }
+; bool potential_test1(bool c) { return iszero(c ? 1 : -1); }
+
+define internal i1 @iszero1(i32 %c) {
+; CHECK-LABEL: define {{[^@]+}}@iszero1
+; CHECK-SAME: (i32 [[C:%.*]])
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[C]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %cmp = icmp eq i32 %c, 0
+  ret i1 %cmp
+}
+
+define i1 @potential_test1(i1 %c) {
+; CHECK-LABEL: define {{[^@]+}}@potential_test1
+; CHECK-SAME: (i1 [[C:%.*]])
+; CHECK-NEXT:    [[ARG:%.*]] = select i1 [[C]], i32 -1, i32 1
+; CHECK-NEXT:    [[RET:%.*]] = call i1 @iszero1(i32 [[ARG]])
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %arg = select i1 %c, i32 -1, i32 1
+  %ret = call i1 @iszero1(i32 %arg)
+  ret i1 %ret
+}
+
+
+; potential-test 2
+;
+; potential values of argument of iszero are {1,-1}
+; potential value of returned value of iszero is 0
+;
+; int call_with_two_values(int x) { return iszero(x) + iszero(-x); }
+; int potential_test2(int x) { return call_with_two_values(1) + call_with_two_values(-1); }
+
+define internal i32 @iszero2(i32 %c) {
+; CHECK-LABEL: define {{[^@]+}}@iszero2
+; CHECK-SAME: (i32 [[C:%.*]])
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[C]], 0
+; CHECK-NEXT:    [[RET:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %cmp = icmp eq i32 %c, 0
+  %ret = zext i1 %cmp to i32
+  ret i32 %ret
+}
+
+define internal i32 @call_with_two_values(i32 %c) {
+; IS__TUNIT____-LABEL: define {{[^@]+}}@call_with_two_values
+; IS__TUNIT____-SAME: (i32 [[C:%.*]])
+; IS__TUNIT____-NEXT:    [[CSRET1:%.*]] = call i32 @iszero2(i32 [[C]]) #0, !range !0
+; IS__TUNIT____-NEXT:    [[MINUSC:%.*]] = sub i32 0, [[C]]
+; IS__TUNIT____-NEXT:    [[CSRET2:%.*]] = call i32 @iszero2(i32 [[MINUSC]]) #0, !range !0
+; IS__TUNIT____-NEXT:    [[RET:%.*]] = add i32 [[CSRET1]], [[CSRET2]]
+; IS__TUNIT____-NEXT:    ret i32 [[RET]]
+;
+; IS__CGSCC____-LABEL: define {{[^@]+}}@call_with_two_values
+; IS__CGSCC____-SAME: (i32 [[C:%.*]])
+; IS__CGSCC____-NEXT:    [[CSRET1:%.*]] = call i32 @iszero2(i32 [[C]])
+; IS__CGSCC____-NEXT:    [[MINUSC:%.*]] = sub i32 0, [[C]]
+; IS__CGSCC____-NEXT:    [[CSRET2:%.*]] = call i32 @iszero2(i32 [[MINUSC]])
+; IS__CGSCC____-NEXT:    [[RET:%.*]] = add i32 [[CSRET1]], [[CSRET2]]
+; IS__CGSCC____-NEXT:    ret i32 [[RET]]
+;
+  %csret1 = call i32 @iszero2(i32 %c)
+  %minusc = sub i32 0, %c
+  %csret2 = call i32 @iszero2(i32 %minusc)
+  %ret = add i32 %csret1, %csret2
+  ret i32 %ret
+}
+
+define i32 @potential_test2(i1 %c) {
+; IS__TUNIT____-LABEL: define {{[^@]+}}@potential_test2
+; IS__TUNIT____-SAME: (i1 [[C:%.*]])
+; IS__TUNIT____-NEXT:    [[CSRET1:%.*]] = call i32 @call_with_two_values(i32 1) #0, !range !1
+; IS__TUNIT____-NEXT:    [[CSRET2:%.*]] = call i32 @call_with_two_values(i32 -1) #1, !range !1
+; IS__TUNIT____-NEXT:    [[RET:%.*]] = add i32 [[CSRET1]], [[CSRET2]]
+; IS__TUNIT____-NEXT:    ret i32 [[RET]]
+;
+; IS__CGSCC____-LABEL: define {{[^@]+}}@potential_test2
+; IS__CGSCC____-SAME: (i1 [[C:%.*]])
+; IS__CGSCC____-NEXT:    [[CSRET1:%.*]] = call i32 @call_with_two_values(i32 1)
+; IS__CGSCC____-NEXT:    [[CSRET2:%.*]] = call i32 @call_with_two_values(i32 -1)
+; IS__CGSCC____-NEXT:    [[RET:%.*]] = add i32 [[CSRET1]], [[CSRET2]]
+; IS__CGSCC____-NEXT:    ret i32 [[RET]]
+;
+  %csret1 = call i32 @call_with_two_values(i32 1)
+  %csret2 = call i32 @call_with_two_values(i32 -1)
+  %ret = add i32 %csret1, %csret2
+  ret i32 %ret
+}
+
+
+; potential-test 3
+;
+; potential values of returned value of f are {0,1}
+; potential values of argument of g are {0,1}
+; potential value of returned value of g is 1
+; then returned value of g can be simplified
+;
+; int zero_or_one(int c) { return c < 2; }
+; int potential_test3() { return zero_or_one(iszero(0))+zero_or_one(iszero(1)); }
+
+define internal i32 @iszero3(i32 %c) {
+; CHECK-LABEL: define {{[^@]+}}@iszero3
+; CHECK-SAME: (i32 [[C:%.*]])
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[C]], 0
+; CHECK-NEXT:    [[RET:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %cmp = icmp eq i32 %c, 0
+  %ret = zext i1 %cmp to i32
+  ret i32 %ret
+}
+
+define internal i32 @less_than_two(i32 %c) {
+; CHECK-LABEL: define {{[^@]+}}@less_than_two
+; CHECK-SAME: (i32 [[C:%.*]])
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[C]], 2
+; CHECK-NEXT:    [[RET:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %cmp = icmp slt i32 %c, 2
+  %ret = zext i1 %cmp to i32
+  ret i32 %ret
+}
+
+define i32 @potential_test3() {
+; NOT_TUNIT_NPM-LABEL: define {{[^@]+}}@potential_test3()
+; NOT_TUNIT_NPM-NEXT:    [[CMP1:%.*]] = call i32 @iszero3(i32 0)
+; NOT_TUNIT_NPM-NEXT:    [[TRUE1:%.*]] = call i32 @less_than_two(i32 [[CMP1]])
+; NOT_TUNIT_NPM-NEXT:    [[CMP2:%.*]] = call i32 @iszero3(i32 1)
+; NOT_TUNIT_NPM-NEXT:    [[TRUE2:%.*]] = call i32 @less_than_two(i32 [[CMP2]])
+; NOT_TUNIT_NPM-NEXT:    [[RET:%.*]] = add i32 [[TRUE1]], [[TRUE2]]
+; NOT_TUNIT_NPM-NEXT:    ret i32 [[RET]]
+;
+; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@potential_test3()
+; IS__TUNIT_NPM-NEXT:    [[CMP1:%.*]] = call i32 @iszero3(i32 0) #0, !range !0
+; IS__TUNIT_NPM-NEXT:    [[TRUE1:%.*]] = call i32 @less_than_two(i32 [[CMP1]]) #0, !range !0
+; IS__TUNIT_NPM-NEXT:    [[CMP2:%.*]] = call i32 @iszero3(i32 1) #0, !range !0
+; IS__TUNIT_NPM-NEXT:    [[TRUE2:%.*]] = call i32 @less_than_two(i32 [[CMP2]]) #0, !range !0
+; IS__TUNIT_NPM-NEXT:    [[RET:%.*]] = add i32 [[TRUE1]], [[TRUE2]]
+; IS__TUNIT_NPM-NEXT:    ret i32 [[RET]]
+;
+  %cmp1 = call i32 @iszero3(i32 0)
+  %true1 = call i32 @less_than_two(i32 %cmp1)
+  %cmp2 = call i32 @iszero3(i32 1)
+  %true2 = call i32 @less_than_two(i32 %cmp2)
+  %ret = add i32 %true1, %true2
+  ret i32 %ret
+}
+
+
+; potential-test 4,5
+;
+; simplified
+; int potential_test4(int c) { return return1or3(c) == 2; }
+; int potential_test5(int c) { return return1or3(c) == return2or4(c); }
+;
+; not simplified
+; int potential_test6(int c) { return return1or3(c) == 3; }
+; int potential_test7(int c) { return return1or3(c) == return3or4(c); }
+
+define i32 @potential_test4(i32 %c) {
+; IS__TUNIT____-LABEL: define {{[^@]+}}@potential_test4
+; IS__TUNIT____-SAME: (i32 [[C:%.*]])
+; IS__TUNIT____-NEXT:    [[CSRET:%.*]] = call i32 @return1or3(i32 [[C]]) #0, !range !2
+; IS__TUNIT____-NEXT:    [[FALSE:%.*]] = icmp eq i32 [[CSRET]], 2
+; IS__TUNIT____-NEXT:    [[RET:%.*]] = zext i1 [[FALSE]] to i32
+; IS__TUNIT____-NEXT:    ret i32 [[RET]]
+;
+; IS__CGSCC____-LABEL: define {{[^@]+}}@potential_test4
+; IS__CGSCC____-SAME: (i32 [[C:%.*]])
+; IS__CGSCC____-NEXT:    [[CSRET:%.*]] = call i32 @return1or3(i32 [[C]])
+; IS__CGSCC____-NEXT:    [[FALSE:%.*]] = icmp eq i32 [[CSRET]], 2
+; IS__CGSCC____-NEXT:    [[RET:%.*]] = zext i1 [[FALSE]] to i32
+; IS__CGSCC____-NEXT:    ret i32 [[RET]]
+;
+  %csret = call i32 @return1or3(i32 %c)
+  %false = icmp eq i32 %csret, 2
+  %ret = zext i1 %false to i32
+  ret i32 %ret
+}
+
+define i32 @potential_test5(i32 %c) {
+; IS__TUNIT____-LABEL: define {{[^@]+}}@potential_test5
+; IS__TUNIT____-SAME: (i32 [[C:%.*]])
+; IS__TUNIT____-NEXT:    [[CSRET1:%.*]] = call i32 @return1or3(i32 [[C]]) #0, !range !2
+; IS__TUNIT____-NEXT:    [[CSRET2:%.*]] = call i32 @return2or4(i32 [[C]]) #0, !range !3
+; IS__TUNIT____-NEXT:    [[FALSE:%.*]] = icmp eq i32 [[CSRET1]], [[CSRET2]]
+; IS__TUNIT____-NEXT:    [[RET:%.*]] = zext i1 [[FALSE]] to i32
+; IS__TUNIT____-NEXT:    ret i32 [[RET]]
+;
+; IS__CGSCC____-LABEL: define {{[^@]+}}@potential_test5
+; IS__CGSCC____-SAME: (i32 [[C:%.*]])
+; IS__CGSCC____-NEXT:    [[CSRET1:%.*]] = call i32 @return1or3(i32 [[C]])
+; IS__CGSCC____-NEXT:    [[CSRET2:%.*]] = call i32 @return2or4(i32 [[C]])
+; IS__CGSCC____-NEXT:    [[FALSE:%.*]] = icmp eq i32 [[CSRET1]], [[CSRET2]]
+; IS__CGSCC____-NEXT:    [[RET:%.*]] = zext i1 [[FALSE]] to i32
+; IS__CGSCC____-NEXT:    ret i32 [[RET]]
+;
+  %csret1 = call i32 @return1or3(i32 %c)
+  %csret2 = call i32 @return2or4(i32 %c)
+  %false = icmp eq i32 %csret1, %csret2
+  %ret = zext i1 %false to i32
+  ret i32 %ret
+}
+
+define i1 @potential_test6(i32 %c) {
+; IS__TUNIT____-LABEL: define {{[^@]+}}@potential_test6
+; IS__TUNIT____-SAME: (i32 [[C:%.*]])
+; IS__TUNIT____-NEXT:    [[CSRET1:%.*]] = call i32 @return1or3(i32 [[C]]) #0, !range !2
+; IS__TUNIT____-NEXT:    [[RET:%.*]] = icmp eq i32 [[CSRET1]], 3
+; IS__TUNIT____-NEXT:    ret i1 [[RET]]
+;
+; IS__CGSCC____-LABEL: define {{[^@]+}}@potential_test6
+; IS__CGSCC____-SAME: (i32 [[C:%.*]])
+; IS__CGSCC____-NEXT:    [[CSRET1:%.*]] = call i32 @return1or3(i32 [[C]])
+; IS__CGSCC____-NEXT:    [[RET:%.*]] = icmp eq i32 [[CSRET1]], 3
+; IS__CGSCC____-NEXT:    ret i1 [[RET]]
+;
+  %csret1 = call i32 @return1or3(i32 %c)
+  %ret = icmp eq i32 %csret1, 3
+  ret i1 %ret
+}
+
+define i1 @potential_test7(i32 %c) {
+; IS__TUNIT____-LABEL: define {{[^@]+}}@potential_test7
+; IS__TUNIT____-SAME: (i32 [[C:%.*]])
+; IS__TUNIT____-NEXT:    [[CSRET1:%.*]] = call i32 @return1or3(i32 [[C]]) #0, !range !2
+; IS__TUNIT____-NEXT:    [[CSRET2:%.*]] = call i32 @return3or4(i32 [[C]]) #0, !range !4
+; IS__TUNIT____-NEXT:    [[RET:%.*]] = icmp eq i32 [[CSRET1]], [[CSRET2]]
+; IS__TUNIT____-NEXT:    ret i1 [[RET]]
+;
+; IS__CGSCC____-LABEL: define {{[^@]+}}@potential_test7
+; IS__CGSCC____-SAME: (i32 [[C:%.*]])
+; IS__CGSCC____-NEXT:    [[CSRET1:%.*]] = call i32 @return1or3(i32 [[C]])
+; IS__CGSCC____-NEXT:    [[CSRET2:%.*]] = call i32 @return3or4(i32 [[C]])
+; IS__CGSCC____-NEXT:    [[RET:%.*]] = icmp eq i32 [[CSRET1]], [[CSRET2]]
+; IS__CGSCC____-NEXT:    ret i1 [[RET]]
+;
+  %csret1 = call i32 @return1or3(i32 %c)
+  %csret2 = call i32 @return3or4(i32 %c)
+  %ret = icmp eq i32 %csret1, %csret2
+  ret i1 %ret
+}
+
+define internal i32 @return1or3(i32 %c) {
+; CHECK-LABEL: define {{[^@]+}}@return1or3
+; CHECK-SAME: (i32 [[C:%.*]])
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[C]], 0
+; CHECK-NEXT:    [[RET:%.*]] = select i1 [[CMP]], i32 1, i32 3
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %cmp = icmp eq i32 %c, 0
+  %ret = select i1 %cmp, i32 1, i32 3
+  ret i32 %ret
+}
+
+define internal i32 @return2or4(i32 %c) {
+; CHECK-LABEL: define {{[^@]+}}@return2or4
+; CHECK-SAME: (i32 [[C:%.*]])
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[C]], 0
+; CHECK-NEXT:    [[RET:%.*]] = select i1 [[CMP]], i32 2, i32 4
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %cmp = icmp eq i32 %c, 0
+  %ret = select i1 %cmp, i32 2, i32 4
+  ret i32 %ret
+}
+
+define internal i32 @return3or4(i32 %c) {
+; CHECK-LABEL: define {{[^@]+}}@return3or4
+; CHECK-SAME: (i32 [[C:%.*]])
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[C]], 0
+; CHECK-NEXT:    [[RET:%.*]] = select i1 [[CMP]], i32 3, i32 4
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %cmp = icmp eq i32 %c, 0
+  %ret = select i1 %cmp, i32 3, i32 4
+  ret i32 %ret
+}
+
+; potential-test 8
+;
+; propagate argument to callsite argument
+
+define internal i1 @cmp_with_four(i32 %c) {
+; CHECK-LABEL: define {{[^@]+}}@cmp_with_four
+; CHECK-SAME: (i32 [[C:%.*]])
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[C]], 4
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %cmp = icmp eq i32 %c, 4
+  ret i1 %cmp
+}
+
+define internal i1 @wrapper(i32 %c) {
+; CHECK-LABEL: define {{[^@]+}}@wrapper
+; CHECK-SAME: (i32 [[C:%.*]])
+; CHECK-NEXT:    [[RET:%.*]] = call i1 @cmp_with_four(i32 [[C]])
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %ret = call i1 @cmp_with_four(i32 %c)
+  ret i1 %ret
+}
+
+define i1 @potential_test8() {
+; CHECK-LABEL: define {{[^@]+}}@potential_test8()
+; CHECK-NEXT:    [[RES1:%.*]] = call i1 @wrapper(i32 1)
+; CHECK-NEXT:    [[RES3:%.*]] = call i1 @wrapper(i32 3)
+; CHECK-NEXT:    [[RES5:%.*]] = call i1 @wrapper(i32 5)
+; CHECK-NEXT:    [[RES13:%.*]] = or i1 [[RES1]], [[RES3]]
+; CHECK-NEXT:    [[RES135:%.*]] = or i1 [[RES13]], [[RES5]]
+; CHECK-NEXT:    ret i1 [[RES135]]
+;
+  %res1 = call i1 @wrapper(i32 1)
+  %res3 = call i1 @wrapper(i32 3)
+  %res5 = call i1 @wrapper(i32 5)
+  %res13 = or i1 %res1, %res3
+  %res135 =  or i1 %res13, %res5
+  ret i1 %res135
+}
+
+define i1 @potential_test9() {
+; IS________OPM-LABEL: define {{[^@]+}}@potential_test9()
+; IS________OPM-NEXT:  entry:
+; IS________OPM-NEXT:    br label [[COND:%.*]]
+; IS________OPM:       cond:
+; IS________OPM-NEXT:    [[I_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[I_1:%.*]], [[INC:%.*]] ]
+; IS________OPM-NEXT:    [[C_0:%.*]] = phi i32 [ 1, [[ENTRY]] ], [ [[C_1:%.*]], [[INC]] ]
+; IS________OPM-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I_0]], 10
+; IS________OPM-NEXT:    br i1 [[CMP]], label [[BODY:%.*]], label [[END:%.*]]
+; IS________OPM:       body:
+; IS________OPM-NEXT:    [[C_1]] = mul i32 [[C_0]], -1
+; IS________OPM-NEXT:    br label [[INC]]
+; IS________OPM:       inc:
+; IS________OPM-NEXT:    [[I_1]] = add i32 [[I_0]], 1
+; IS________OPM-NEXT:    br label [[COND]]
+; IS________OPM:       end:
+; IS________OPM-NEXT:    [[RET:%.*]] = icmp eq i32 [[C_0]], 0
+; IS________OPM-NEXT:    ret i1 [[RET]]
+;
+; IS________NPM-LABEL: define {{[^@]+}}@potential_test9()
+; IS________NPM-NEXT:  entry:
+; IS________NPM-NEXT:    br label [[COND:%.*]]
+; IS________NPM:       cond:
+; IS________NPM-NEXT:    [[I_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[I_1:%.*]], [[INC:%.*]] ]
+; IS________NPM-NEXT:    [[C_0:%.*]] = phi i32 [ 1, [[ENTRY]] ], [ [[C_1:%.*]], [[INC]] ]
+; IS________NPM-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I_0]], 10
+; IS________NPM-NEXT:    br i1 [[CMP]], label [[BODY:%.*]], label [[END:%.*]]
+; IS________NPM:       body:
+; IS________NPM-NEXT:    [[C_1]] = mul i32 [[C_0]], -1
+; IS________NPM-NEXT:    br label [[INC]]
+; IS________NPM:       inc:
+; IS________NPM-NEXT:    [[I_1]] = add i32 [[I_0]], 1
+; IS________NPM-NEXT:    br label [[COND]]
+; IS________NPM:       end:
+; IS________NPM-NEXT:    ret i1 false
+;
+entry:
+  br label %cond
+cond:
+  %i.0 = phi i32 [0, %entry], [%i.1, %inc]
+  %c.0 = phi i32 [1, %entry], [%c.1, %inc]
+  %cmp = icmp slt i32 %i.0, 10
+  br i1 %cmp, label %body, label %end
+body:
+  %c.1 = mul i32 %c.0, -1
+  br label %inc
+inc:
+  %i.1 = add i32 %i.0, 1
+  br label %cond
+end:
+  %ret = icmp eq i32 %c.0, 0
+  ret i1 %ret
+}
+
+; IS__TUNIT____: !0 = !{i32 0, i32 2}
+; IS__TUNIT____: !1 = !{i32 0, i32 3}
+; IS__TUNIT____: !2 = !{i32 1, i32 4}
+; IS__TUNIT____: !3 = !{i32 2, i32 5}
+; IS__TUNIT____: !4 = !{i32 3, i32 5}
+; IS__TUNIT____-NOT: !5
+
+; NOT_TUNIT____-NOT: !0

From 2700311cce99d2a3ef45002e32b8832b88214f7d Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Sun, 2 Aug 2020 14:13:13 +0100
Subject: [PATCH 0839/1035] [X86] combineX86ShuffleChain - pull out repeated
 RootVT.getSizeInBits() calls. NFCI.

---
 llvm/lib/Target/X86/X86ISelLowering.cpp | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 752b65fc03310..a16c3bf7ac51f 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -34464,6 +34464,10 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root,
   assert((Inputs.size() == 1 || Inputs.size() == 2) &&
          "Unexpected number of shuffle inputs!");
 
+  MVT RootVT = Root.getSimpleValueType();
+  unsigned RootSizeInBits = RootVT.getSizeInBits();
+  unsigned NumRootElts = RootVT.getVectorNumElements();
+
   // Find the inputs that enter the chain. Note that multiple uses are OK
   // here, we're not going to remove the operands we find.
   bool UnaryShuffle = (Inputs.size() == 1);
@@ -34473,10 +34477,8 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root,
 
   MVT VT1 = V1.getSimpleValueType();
   MVT VT2 = V2.getSimpleValueType();
-  MVT RootVT = Root.getSimpleValueType();
-  assert(VT1.getSizeInBits() == RootVT.getSizeInBits() &&
-         VT2.getSizeInBits() == RootVT.getSizeInBits() &&
-         "Vector size mismatch");
+  assert(VT1.getSizeInBits() == RootSizeInBits &&
+         VT2.getSizeInBits() == RootSizeInBits && "Vector size mismatch");
 
   SDLoc DL(Root);
   SDValue Res;
@@ -34488,8 +34490,6 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root,
   }
 
   bool OptForSize = DAG.shouldOptForSize();
-  unsigned RootSizeInBits = RootVT.getSizeInBits();
-  unsigned NumRootElts = RootVT.getVectorNumElements();
   unsigned BaseMaskEltSizeInBits = RootSizeInBits / NumBaseMaskElts;
   bool FloatDomain = VT1.isFloatingPoint() || VT2.isFloatingPoint() ||
                      (RootVT.isFloatingPoint() && Depth >= 1) ||

From 3f276840b6f8b2624f0bbeb6097d8049d27d5ca0 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Sun, 2 Aug 2020 14:15:15 +0100
Subject: [PATCH 0840/1035] [X86] Use const APInt& in for-range loop to avoid
 unnecessary copies. NFCI.

Fixes clang-tidy warning.
---
 llvm/lib/Target/X86/X86ISelLowering.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index a16c3bf7ac51f..ec34dcaaf95b8 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -6686,7 +6686,7 @@ static bool getTargetShuffleMaskIndices(SDValue MaskNode,
     return false;
 
   // Insert the extracted elements into the mask.
-  for (APInt Elt : EltBits)
+  for (const APInt &Elt : EltBits)
     RawMask.push_back(Elt.getZExtValue());
 
   return true;

From d7e261674141ce42557c57f01869d27f0aecf6ee Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Sun, 2 Aug 2020 14:23:00 +0100
Subject: [PATCH 0841/1035] [X86] Pass SDLoc by const reference. NFCI.

---
 llvm/lib/Target/X86/X86ISelLowering.cpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index ec34dcaaf95b8..26904f8945ee3 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -28119,8 +28119,8 @@ bool X86TargetLowering::lowerAtomicLoadAsLoadSDNode(const LoadInst &LI) const {
 /// a) very likely accessed only by a single thread to minimize cache traffic,
 /// and b) definitely dereferenceable.  Returns the new Chain result.
 static SDValue emitLockedStackOp(SelectionDAG &DAG,
-                                 const X86Subtarget &Subtarget,
-                                 SDValue Chain, SDLoc DL) {
+                                 const X86Subtarget &Subtarget, SDValue Chain,
+                                 const SDLoc &DL) {
   // Implementation notes:
   // 1) LOCK prefix creates a full read/write reordering barrier for memory
   // operations issued by the current processor.  As such, the location
@@ -38298,7 +38298,7 @@ static SDValue createMMXBuildVector(BuildVectorSDNode *BV, SelectionDAG &DAG,
 // a vector/float/double that got truncated/extended/bitcast to/from a scalar
 // integer. If so, replace the scalar ops with bool vector equivalents back down
 // the chain.
-static SDValue combineBitcastToBoolVector(EVT VT, SDValue V, SDLoc DL,
+static SDValue combineBitcastToBoolVector(EVT VT, SDValue V, const SDLoc &DL,
                                           SelectionDAG &DAG,
                                           const X86Subtarget &Subtarget) {
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();

From 20fbbbc583f4d7a35c380b0b88ea96cff0237856 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Sun, 2 Aug 2020 14:32:05 +0100
Subject: [PATCH 0842/1035] [X86] Use const APInt& in for-range loop to avoid
 unnecessary copies. NFCI.

Fixes clang-tidy warning.
---
 llvm/lib/Target/X86/X86ISelLowering.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 26904f8945ee3..15f283b1bc42a 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -42976,7 +42976,7 @@ static SDValue combineAnd(SDNode *N, SelectionDAG &DAG,
     if (VT == SrcVecVT.getScalarType() &&
         N->getOperand(0)->isOnlyUserOf(SrcVec.getNode()) &&
         getTargetConstantBitsFromNode(BitMask, 8, UndefElts, EltBits) &&
-        llvm::all_of(EltBits, [](APInt M) {
+        llvm::all_of(EltBits, [](const APInt &M) {
           return M.isNullValue() || M.isAllOnesValue();
         })) {
       unsigned NumElts = SrcVecVT.getVectorNumElements();

From 8d1b9505f24aad4015d435cc9f23f77f7ff703f8 Mon Sep 17 00:00:00 2001
From: Xing GUO <higuoxing@gmail.com>
Date: Sun, 2 Aug 2020 21:37:11 +0800
Subject: [PATCH 0843/1035] [DWARFYAML][debug_aranges] Make the 'Descriptors'
 field optional.

---
 llvm/lib/ObjectYAML/DWARFYAML.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/ObjectYAML/DWARFYAML.cpp b/llvm/lib/ObjectYAML/DWARFYAML.cpp
index 04dd185ad954e..37d45996786d3 100644
--- a/llvm/lib/ObjectYAML/DWARFYAML.cpp
+++ b/llvm/lib/ObjectYAML/DWARFYAML.cpp
@@ -107,7 +107,7 @@ void MappingTraits<DWARFYAML::ARange>::mapping(IO &IO,
   IO.mapRequired("CuOffset", ARange.CuOffset);
   IO.mapOptional("AddressSize", ARange.AddrSize);
   IO.mapOptional("SegmentSelectorSize", ARange.SegSize, 0);
-  IO.mapRequired("Descriptors", ARange.Descriptors);
+  IO.mapOptional("Descriptors", ARange.Descriptors);
 }
 
 void MappingTraits<DWARFYAML::RangeEntry>::mapping(

From a0addbb4ec8c7bf791139699d46b08413c46eed7 Mon Sep 17 00:00:00 2001
From: Nikita Popov <nikita.ppv@gmail.com>
Date: Sun, 2 Aug 2020 15:45:24 +0200
Subject: [PATCH 0844/1035] [InstSimplify] Reduce code duplication in icmp of
 binop folds (NFC)

For folds where we check for the binop on both the LHS and RHS,
extract a function that expects it on the LHS and call it with
swapped order.
---
 llvm/lib/Analysis/InstructionSimplify.cpp | 215 +++++++++-------------
 1 file changed, 82 insertions(+), 133 deletions(-)

diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp
index b1438b416d56b..4f09ea1896643 100644
--- a/llvm/lib/Analysis/InstructionSimplify.cpp
+++ b/llvm/lib/Analysis/InstructionSimplify.cpp
@@ -2753,14 +2753,87 @@ static Value *simplifyICmpWithConstant(CmpInst::Predicate Pred, Value *LHS,
   return nullptr;
 }
 
+static Value *simplifyICmpWithBinOpOnLHS(
+    CmpInst::Predicate Pred, BinaryOperator *LBO, Value *RHS,
+    const SimplifyQuery &Q, unsigned MaxRecurse) {
+  Type *ITy = GetCompareTy(RHS); // The return type.
+
+  Value *Y = nullptr;
+  // icmp pred (or X, Y), X
+  if (match(LBO, m_c_Or(m_Value(Y), m_Specific(RHS)))) {
+    if (Pred == ICmpInst::ICMP_ULT)
+      return getFalse(ITy);
+    if (Pred == ICmpInst::ICMP_UGE)
+      return getTrue(ITy);
+
+    if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SGE) {
+      KnownBits RHSKnown = computeKnownBits(RHS, Q.DL, 0, Q.AC, Q.CxtI, Q.DT);
+      KnownBits YKnown = computeKnownBits(Y, Q.DL, 0, Q.AC, Q.CxtI, Q.DT);
+      if (RHSKnown.isNonNegative() && YKnown.isNegative())
+        return Pred == ICmpInst::ICMP_SLT ? getTrue(ITy) : getFalse(ITy);
+      if (RHSKnown.isNegative() || YKnown.isNonNegative())
+        return Pred == ICmpInst::ICMP_SLT ? getFalse(ITy) : getTrue(ITy);
+    }
+  }
+
+  // icmp pred (and X, Y), X
+  if (match(LBO, m_c_And(m_Value(), m_Specific(RHS)))) {
+    if (Pred == ICmpInst::ICMP_UGT)
+      return getFalse(ITy);
+    if (Pred == ICmpInst::ICMP_ULE)
+      return getTrue(ITy);
+  }
+
+  // icmp pred (urem X, Y), Y
+  if (match(LBO, m_URem(m_Value(), m_Specific(RHS)))) {
+    switch (Pred) {
+    default:
+      break;
+    case ICmpInst::ICMP_SGT:
+    case ICmpInst::ICMP_SGE: {
+      KnownBits Known = computeKnownBits(RHS, Q.DL, 0, Q.AC, Q.CxtI, Q.DT);
+      if (!Known.isNonNegative())
+        break;
+      LLVM_FALLTHROUGH;
+    }
+    case ICmpInst::ICMP_EQ:
+    case ICmpInst::ICMP_UGT:
+    case ICmpInst::ICMP_UGE:
+      return getFalse(ITy);
+    case ICmpInst::ICMP_SLT:
+    case ICmpInst::ICMP_SLE: {
+      KnownBits Known = computeKnownBits(RHS, Q.DL, 0, Q.AC, Q.CxtI, Q.DT);
+      if (!Known.isNonNegative())
+        break;
+      LLVM_FALLTHROUGH;
+    }
+    case ICmpInst::ICMP_NE:
+    case ICmpInst::ICMP_ULT:
+    case ICmpInst::ICMP_ULE:
+      return getTrue(ITy);
+    }
+  }
+
+  // x >> y <=u x
+  // x udiv y <=u x.
+  if (match(LBO, m_LShr(m_Specific(RHS), m_Value())) ||
+      match(LBO, m_UDiv(m_Specific(RHS), m_Value()))) {
+    // icmp pred (X op Y), X
+    if (Pred == ICmpInst::ICMP_UGT)
+      return getFalse(ITy);
+    if (Pred == ICmpInst::ICMP_ULE)
+      return getTrue(ITy);
+  }
+
+  return nullptr;
+}
+
 /// TODO: A large part of this logic is duplicated in InstCombine's
 /// foldICmpBinOp(). We should be able to share that and avoid the code
 /// duplication.
 static Value *simplifyICmpWithBinOp(CmpInst::Predicate Pred, Value *LHS,
                                     Value *RHS, const SimplifyQuery &Q,
                                     unsigned MaxRecurse) {
-  Type *ITy = GetCompareTy(LHS); // The return type.
-
   BinaryOperator *LBO = dyn_cast<BinaryOperator>(LHS);
   BinaryOperator *RBO = dyn_cast<BinaryOperator>(RHS);
   if (MaxRecurse && (LBO || RBO)) {
@@ -2831,56 +2904,14 @@ static Value *simplifyICmpWithBinOp(CmpInst::Predicate Pred, Value *LHS,
     }
   }
 
-  {
-    Value *Y = nullptr;
-    // icmp pred (or X, Y), X
-    if (LBO && match(LBO, m_c_Or(m_Value(Y), m_Specific(RHS)))) {
-      if (Pred == ICmpInst::ICMP_ULT)
-        return getFalse(ITy);
-      if (Pred == ICmpInst::ICMP_UGE)
-        return getTrue(ITy);
-
-      if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SGE) {
-        KnownBits RHSKnown = computeKnownBits(RHS, Q.DL, 0, Q.AC, Q.CxtI, Q.DT);
-        KnownBits YKnown = computeKnownBits(Y, Q.DL, 0, Q.AC, Q.CxtI, Q.DT);
-        if (RHSKnown.isNonNegative() && YKnown.isNegative())
-          return Pred == ICmpInst::ICMP_SLT ? getTrue(ITy) : getFalse(ITy);
-        if (RHSKnown.isNegative() || YKnown.isNonNegative())
-          return Pred == ICmpInst::ICMP_SLT ? getFalse(ITy) : getTrue(ITy);
-      }
-    }
-    // icmp pred X, (or X, Y)
-    if (RBO && match(RBO, m_c_Or(m_Value(Y), m_Specific(LHS)))) {
-      if (Pred == ICmpInst::ICMP_ULE)
-        return getTrue(ITy);
-      if (Pred == ICmpInst::ICMP_UGT)
-        return getFalse(ITy);
-
-      if (Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SLE) {
-        KnownBits LHSKnown = computeKnownBits(LHS, Q.DL, 0, Q.AC, Q.CxtI, Q.DT);
-        KnownBits YKnown = computeKnownBits(Y, Q.DL, 0, Q.AC, Q.CxtI, Q.DT);
-        if (LHSKnown.isNonNegative() && YKnown.isNegative())
-          return Pred == ICmpInst::ICMP_SGT ? getTrue(ITy) : getFalse(ITy);
-        if (LHSKnown.isNegative() || YKnown.isNonNegative())
-          return Pred == ICmpInst::ICMP_SGT ? getFalse(ITy) : getTrue(ITy);
-      }
-    }
-  }
+  if (LBO)
+    if (Value *V = simplifyICmpWithBinOpOnLHS(Pred, LBO, RHS, Q, MaxRecurse))
+      return V;
 
-  // icmp pred (and X, Y), X
-  if (LBO && match(LBO, m_c_And(m_Value(), m_Specific(RHS)))) {
-    if (Pred == ICmpInst::ICMP_UGT)
-      return getFalse(ITy);
-    if (Pred == ICmpInst::ICMP_ULE)
-      return getTrue(ITy);
-  }
-  // icmp pred X, (and X, Y)
-  if (RBO && match(RBO, m_c_And(m_Value(), m_Specific(LHS)))) {
-    if (Pred == ICmpInst::ICMP_UGE)
-      return getTrue(ITy);
-    if (Pred == ICmpInst::ICMP_ULT)
-      return getFalse(ITy);
-  }
+  if (RBO)
+    if (Value *V = simplifyICmpWithBinOpOnLHS(
+            ICmpInst::getSwappedPredicate(Pred), RBO, LHS, Q, MaxRecurse))
+      return V;
 
   // 0 - (zext X) pred C
   if (!CmpInst::isUnsigned(Pred) && match(LHS, m_Neg(m_ZExt(m_Value())))) {
@@ -2904,88 +2935,6 @@ static Value *simplifyICmpWithBinOp(CmpInst::Predicate Pred, Value *LHS,
     }
   }
 
-  // icmp pred (urem X, Y), Y
-  if (LBO && match(LBO, m_URem(m_Value(), m_Specific(RHS)))) {
-    switch (Pred) {
-    default:
-      break;
-    case ICmpInst::ICMP_SGT:
-    case ICmpInst::ICMP_SGE: {
-      KnownBits Known = computeKnownBits(RHS, Q.DL, 0, Q.AC, Q.CxtI, Q.DT);
-      if (!Known.isNonNegative())
-        break;
-      LLVM_FALLTHROUGH;
-    }
-    case ICmpInst::ICMP_EQ:
-    case ICmpInst::ICMP_UGT:
-    case ICmpInst::ICMP_UGE:
-      return getFalse(ITy);
-    case ICmpInst::ICMP_SLT:
-    case ICmpInst::ICMP_SLE: {
-      KnownBits Known = computeKnownBits(RHS, Q.DL, 0, Q.AC, Q.CxtI, Q.DT);
-      if (!Known.isNonNegative())
-        break;
-      LLVM_FALLTHROUGH;
-    }
-    case ICmpInst::ICMP_NE:
-    case ICmpInst::ICMP_ULT:
-    case ICmpInst::ICMP_ULE:
-      return getTrue(ITy);
-    }
-  }
-
-  // icmp pred X, (urem Y, X)
-  if (RBO && match(RBO, m_URem(m_Value(), m_Specific(LHS)))) {
-    switch (Pred) {
-    default:
-      break;
-    case ICmpInst::ICMP_SGT:
-    case ICmpInst::ICMP_SGE: {
-      KnownBits Known = computeKnownBits(LHS, Q.DL, 0, Q.AC, Q.CxtI, Q.DT);
-      if (!Known.isNonNegative())
-        break;
-      LLVM_FALLTHROUGH;
-    }
-    case ICmpInst::ICMP_NE:
-    case ICmpInst::ICMP_UGT:
-    case ICmpInst::ICMP_UGE:
-      return getTrue(ITy);
-    case ICmpInst::ICMP_SLT:
-    case ICmpInst::ICMP_SLE: {
-      KnownBits Known = computeKnownBits(LHS, Q.DL, 0, Q.AC, Q.CxtI, Q.DT);
-      if (!Known.isNonNegative())
-        break;
-      LLVM_FALLTHROUGH;
-    }
-    case ICmpInst::ICMP_EQ:
-    case ICmpInst::ICMP_ULT:
-    case ICmpInst::ICMP_ULE:
-      return getFalse(ITy);
-    }
-  }
-
-  // x >> y <=u x
-  // x udiv y <=u x.
-  if (LBO && (match(LBO, m_LShr(m_Specific(RHS), m_Value())) ||
-              match(LBO, m_UDiv(m_Specific(RHS), m_Value())))) {
-    // icmp pred (X op Y), X
-    if (Pred == ICmpInst::ICMP_UGT)
-      return getFalse(ITy);
-    if (Pred == ICmpInst::ICMP_ULE)
-      return getTrue(ITy);
-  }
-
-  // x >=u x >> y
-  // x >=u x udiv y.
-  if (RBO && (match(RBO, m_LShr(m_Specific(LHS), m_Value())) ||
-              match(RBO, m_UDiv(m_Specific(LHS), m_Value())))) {
-    // icmp pred X, (X op Y)
-    if (Pred == ICmpInst::ICMP_ULT)
-      return getFalse(ITy);
-    if (Pred == ICmpInst::ICMP_UGE)
-      return getTrue(ITy);
-  }
-
   // handle:
   //   CI2 << X == CI
   //   CI2 << X != CI

From 376b64926b70c8b146caaf397616fb681ae329ca Mon Sep 17 00:00:00 2001
From: Shinji Okumura <okuraofvegetable@gmail.com>
Date: Sun, 2 Aug 2020 22:48:28 +0900
Subject: [PATCH 0845/1035] Revert "[Attributor] AAPotentialValues Interface"

The commit cause build failure.
---
 llvm/include/llvm/ADT/APInt.h                 |  26 +-
 llvm/include/llvm/Transforms/IPO/Attributor.h | 187 ---------
 llvm/lib/IR/LLVMContextImpl.h                 |  22 +-
 llvm/lib/Transforms/IPO/Attributor.cpp        |  13 -
 .../Transforms/IPO/AttributorAttributes.cpp   | 162 -------
 llvm/test/Transforms/Attributor/potential.ll  | 394 ------------------
 6 files changed, 22 insertions(+), 782 deletions(-)
 delete mode 100644 llvm/test/Transforms/Attributor/potential.ll

diff --git a/llvm/include/llvm/ADT/APInt.h b/llvm/include/llvm/ADT/APInt.h
index 14643a14a2f30..f7df648d27ed6 100644
--- a/llvm/include/llvm/ADT/APInt.h
+++ b/llvm/include/llvm/ADT/APInt.h
@@ -15,7 +15,6 @@
 #ifndef LLVM_ADT_APINT_H
 #define LLVM_ADT_APINT_H
 
-#include "llvm/ADT/DenseMapInfo.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/MathExtras.h"
 #include <cassert>
@@ -97,7 +96,7 @@ class LLVM_NODISCARD APInt {
 
   unsigned BitWidth; ///< The number of bits in this APInt.
 
-  friend struct DenseMapInfo<APInt>;
+  friend struct DenseMapAPIntKeyInfo;
 
   friend class APSInt;
 
@@ -2289,29 +2288,6 @@ void StoreIntToMemory(const APInt &IntVal, uint8_t *Dst, unsigned StoreBytes);
 /// from Src into IntVal, which is assumed to be wide enough and to hold zero.
 void LoadIntFromMemory(APInt &IntVal, const uint8_t *Src, unsigned LoadBytes);
 
-/// Provide DenseMapInfo for APInt.
-template <> struct DenseMapInfo<APInt> {
-  static inline APInt getEmptyKey() {
-    APInt V(nullptr, 0);
-    V.U.VAL = 0;
-    return V;
-  }
-
-  static inline APInt getTombstoneKey() {
-    APInt V(nullptr, 0);
-    V.U.VAL = 1;
-    return V;
-  }
-
-  static unsigned getHashValue(const APInt &Key) {
-    return static_cast<unsigned>(hash_value(Key));
-  }
-
-  static bool isEqual(const APInt &LHS, const APInt &RHS) {
-    return LHS.getBitWidth() == RHS.getBitWidth() && LHS == RHS;
-  }
-};
-
 } // namespace llvm
 
 #endif
diff --git a/llvm/include/llvm/Transforms/IPO/Attributor.h b/llvm/include/llvm/Transforms/IPO/Attributor.h
index b15c8f0dd6c35..418a007319679 100644
--- a/llvm/include/llvm/Transforms/IPO/Attributor.h
+++ b/llvm/include/llvm/Transforms/IPO/Attributor.h
@@ -97,7 +97,6 @@
 #ifndef LLVM_TRANSFORMS_IPO_ATTRIBUTOR_H
 #define LLVM_TRANSFORMS_IPO_ATTRIBUTOR_H
 
-#include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/GraphTraits.h"
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/STLExtras.h"
@@ -116,7 +115,6 @@
 #include "llvm/IR/PassManager.h"
 #include "llvm/Support/Allocator.h"
 #include "llvm/Support/Casting.h"
-#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/DOTGraphTraits.h"
 #include "llvm/Support/GraphWriter.h"
 #include "llvm/Support/TimeProfiler.h"
@@ -3350,191 +3348,6 @@ struct AAValueConstantRange
   static const char ID;
 };
 
-/// A class for a set state.
-/// The assumed boolean state indicates whether the corresponding set is full
-/// set or not. If the assumed state is false, this is the worst state. The
-/// worst state (invalid state) of set of potential values is when the set
-/// contains every possible value (i.e. we cannot in any way limit the value
-/// that the target position can take). That never happens naturally, we only
-/// force it. As for the conditions under which we force it, see
-/// AAPotentialValues.
-template <typename MemberTy, typename KeyInfo = DenseMapInfo<MemberTy>>
-struct PotentialValuesState : BooleanState {
-  using SetTy = DenseSet<MemberTy, KeyInfo>;
-
-  PotentialValuesState() : BooleanState(true) {}
-
-  PotentialValuesState(bool IsValid) : BooleanState(IsValid) {}
-
-  /// Return this set. We should check whether this set is valid or not by
-  /// isValidState() before calling this function.
-  const SetTy &getAssumedSet() const {
-    assert(isValidState() && "This set shoud not be used when it is invalid!");
-    return Set;
-  }
-
-  bool operator==(const PotentialValuesState &RHS) const {
-    if (isValidState() != RHS.isValidState())
-      return false;
-    if (!isValidState() && !RHS.isValidState())
-      return true;
-    return Set == RHS.getAssumedSet();
-  }
-
-  /// Maximum number of potential values to be tracked.
-  /// This is set by -attributor-max-potential-values command line option
-  static unsigned MaxPotentialValues;
-
-  /// Return empty set as the best state of potential values.
-  static PotentialValuesState getBestState() {
-    return PotentialValuesState(true);
-  }
-
-  static PotentialValuesState getBestState(PotentialValuesState &PVS) {
-    return getBestState();
-  }
-
-  /// Return full set as the worst state of potential values.
-  static PotentialValuesState getWorstState() {
-    return PotentialValuesState(false);
-  }
-
-  /// Union assumed set with the passed value.
-  void unionAssumed(const MemberTy &C) { insert(C); }
-
-  /// Union assumed set with assumed set of the passed state \p PVS.
-  void unionAssumed(const PotentialValuesState &PVS) { unionWith(PVS); }
-
-  /// "Clamp" this state with \p PVS.
-  PotentialValuesState operator^=(const PotentialValuesState &PVS) {
-    unionAssumed(PVS);
-    return *this;
-  }
-
-  PotentialValuesState operator&=(const PotentialValuesState &PVS) {
-    unionAssumed(PVS);
-    return *this;
-  }
-
-private:
-  /// Check the size of this set, and invalidate when the size is no
-  /// less than \p MaxPotentialValues threshold.
-  void checkAndInvalidate() {
-    if (Set.size() >= MaxPotentialValues)
-      indicatePessimisticFixpoint();
-  }
-
-  /// Insert an element into this set.
-  void insert(const MemberTy &C) {
-    if (!isValidState())
-      return;
-    Set.insert(C);
-    checkAndInvalidate();
-  }
-
-  /// Take union with R.
-  void unionWith(const PotentialValuesState &R) {
-    /// If this is a full set, do nothing.;
-    if (!isValidState())
-      return;
-    /// If R is full set, change L to a full set.
-    if (!R.isValidState()) {
-      indicatePessimisticFixpoint();
-      return;
-    }
-    for (const MemberTy &C : R.Set)
-      Set.insert(C);
-    checkAndInvalidate();
-  }
-
-  /// Take intersection with R.
-  void intersectWith(const PotentialValuesState &R) {
-    /// If R is a full set, do nothing.
-    if (!R.isValidState())
-      return;
-    /// If this is a full set, change this to R.
-    if (!isValidState()) {
-      *this = R;
-      return;
-    }
-    SetTy IntersectSet;
-    for (const MemberTy &C : Set) {
-      if (R.Set.count(C))
-        IntersectSet.insert(C);
-    }
-    Set = IntersectSet;
-  }
-
-  /// Container for potential values
-  SetTy Set;
-};
-
-using PotentialConstantIntValuesState = PotentialValuesState<APInt>;
-
-raw_ostream &operator<<(raw_ostream &OS,
-                        const PotentialConstantIntValuesState &R);
-
-/// An abstract interface for potential values analysis.
-///
-/// This AA collects potential values for each IR position.
-/// An assumed set of potential values is initialized with the empty set (the
-/// best state) and it will grow monotonically as we find more potential values
-/// for this position.
-/// The set might be forced to the worst state, that is, to contain every
-/// possible value for this position in 2 cases.
-///   1. We surpassed the \p MaxPotentialValues threshold. This includes the
-///      case that this position is affected (e.g. because of an operation) by a
-///      Value that is in the worst state.
-///   2. We tried to initialize on a Value that we cannot handle (e.g. an
-///      operator we do not currently handle).
-///
-/// TODO: Support values other than constant integers.
-struct AAPotentialValues
-    : public StateWrapper<PotentialConstantIntValuesState, AbstractAttribute> {
-  using Base = StateWrapper<PotentialConstantIntValuesState, AbstractAttribute>;
-  AAPotentialValues(const IRPosition &IRP, Attributor &A) : Base(IRP) {}
-
-  /// See AbstractAttribute::getState(...).
-  PotentialConstantIntValuesState &getState() override { return *this; }
-  const PotentialConstantIntValuesState &getState() const override {
-    return *this;
-  }
-
-  /// Create an abstract attribute view for the position \p IRP.
-  static AAPotentialValues &createForPosition(const IRPosition &IRP,
-                                              Attributor &A);
-
-  /// Return assumed constant for the associated value
-  Optional<ConstantInt *>
-  getAssumedConstantInt(Attributor &A,
-                        const Instruction *CtxI = nullptr) const {
-    if (!isValidState())
-      return nullptr;
-    if (getAssumedSet().size() == 1)
-      return cast<ConstantInt>(ConstantInt::get(getAssociatedValue().getType(),
-                                                *(getAssumedSet().begin())));
-    if (getAssumedSet().size() == 0)
-      return llvm::None;
-
-    return nullptr;
-  }
-
-  /// See AbstractAttribute::getName()
-  const std::string getName() const override { return "AAPotentialValues"; }
-
-  /// See AbstractAttribute::getIdAddr()
-  const char *getIdAddr() const override { return &ID; }
-
-  /// This function should return true if the type of the \p AA is
-  /// AAPotentialValues
-  static bool classof(const AbstractAttribute *AA) {
-    return (AA->getIdAddr() == &ID);
-  }
-
-  /// Unique ID (due to the unique address)
-  static const char ID;
-};
-
 /// Run options, used by the pass manager.
 enum AttributorRunOption {
   NONE = 0,
diff --git a/llvm/lib/IR/LLVMContextImpl.h b/llvm/lib/IR/LLVMContextImpl.h
index e8fdaa23761c3..b97ac37c5fcfd 100644
--- a/llvm/lib/IR/LLVMContextImpl.h
+++ b/llvm/lib/IR/LLVMContextImpl.h
@@ -57,7 +57,27 @@ class Type;
 class Value;
 class ValueHandleBase;
 
-using DenseMapAPIntKeyInfo = DenseMapInfo<APInt>;
+struct DenseMapAPIntKeyInfo {
+  static inline APInt getEmptyKey() {
+    APInt V(nullptr, 0);
+    V.U.VAL = 0;
+    return V;
+  }
+
+  static inline APInt getTombstoneKey() {
+    APInt V(nullptr, 0);
+    V.U.VAL = 1;
+    return V;
+  }
+
+  static unsigned getHashValue(const APInt &Key) {
+    return static_cast<unsigned>(hash_value(Key));
+  }
+
+  static bool isEqual(const APInt &LHS, const APInt &RHS) {
+    return LHS.getBitWidth() == RHS.getBitWidth() && LHS == RHS;
+  }
+};
 
 struct DenseMapAPFloatKeyInfo {
   static inline APFloat getEmptyKey() { return APFloat(APFloat::Bogus(), 1); }
diff --git a/llvm/lib/Transforms/IPO/Attributor.cpp b/llvm/lib/Transforms/IPO/Attributor.cpp
index 764d71b98e063..89971cc9c2946 100644
--- a/llvm/lib/Transforms/IPO/Attributor.cpp
+++ b/llvm/lib/Transforms/IPO/Attributor.cpp
@@ -2079,19 +2079,6 @@ raw_ostream &llvm::operator<<(raw_ostream &OS, const AbstractAttribute &AA) {
   return OS;
 }
 
-raw_ostream &llvm::operator<<(raw_ostream &OS,
-                              const PotentialConstantIntValuesState &S) {
-  OS << "set-state(< {";
-  if (!S.isValidState())
-    OS << "full-set";
-  else
-    for (auto &it : S.getAssumedSet())
-      OS << it << ", ";
-  OS << "} >)";
-
-  return OS;
-}
-
 void AbstractAttribute::print(raw_ostream &OS) const {
   OS << "[";
   OS << getName();
diff --git a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
index a220bc001a4ce..aa6bc94a3668e 100644
--- a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
+++ b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
@@ -45,16 +45,6 @@ static cl::opt<bool> ManifestInternal(
 static cl::opt<int> MaxHeapToStackSize("max-heap-to-stack-size", cl::init(128),
                                        cl::Hidden);
 
-static cl::opt<unsigned, true> MaxPotentialValues(
-    "attributor-max-potential-values", cl::Hidden,
-    cl::desc("Maximum number of potential values to be "
-             "tracked for each position."),
-    cl::location(llvm::PotentialConstantIntValuesState::MaxPotentialValues),
-    cl::init(7));
-
-template <>
-unsigned llvm::PotentialConstantIntValuesState::MaxPotentialValues = 0;
-
 STATISTIC(NumAAs, "Number of abstract attributes created");
 
 // Some helper macros to deal with statistics tracking.
@@ -130,7 +120,6 @@ PIPE_OPERATOR(AAMemoryLocation)
 PIPE_OPERATOR(AAValueConstantRange)
 PIPE_OPERATOR(AAPrivatizablePtr)
 PIPE_OPERATOR(AAUndefinedBehavior)
-PIPE_OPERATOR(AAPotentialValues)
 
 #undef PIPE_OPERATOR
 } // namespace llvm
@@ -7086,155 +7075,6 @@ struct AAValueConstantRangeCallSiteArgument : AAValueConstantRangeFloating {
     STATS_DECLTRACK_CSARG_ATTR(value_range)
   }
 };
-
-/// ------------------ Potential Values Attribute -------------------------
-
-struct AAPotentialValuesImpl : AAPotentialValues {
-  using StateType = PotentialConstantIntValuesState;
-
-  AAPotentialValuesImpl(const IRPosition &IRP, Attributor &A)
-      : AAPotentialValues(IRP, A) {}
-
-  /// See AbstractAttribute::getAsStr().
-  const std::string getAsStr() const override {
-    std::string Str;
-    llvm::raw_string_ostream OS(Str);
-    OS << getState();
-    return OS.str();
-  }
-
-  /// See AbstractAttribute::updateImpl(...).
-  ChangeStatus updateImpl(Attributor &A) override {
-    return indicatePessimisticFixpoint();
-  }
-};
-
-struct AAPotentialValuesArgument final
-    : AAArgumentFromCallSiteArguments<AAPotentialValues, AAPotentialValuesImpl,
-                                      PotentialConstantIntValuesState> {
-  using Base =
-      AAArgumentFromCallSiteArguments<AAPotentialValues, AAPotentialValuesImpl,
-                                      PotentialConstantIntValuesState>;
-  AAPotentialValuesArgument(const IRPosition &IRP, Attributor &A)
-      : Base(IRP, A) {}
-
-  /// See AbstractAttribute::initialize(..).
-  void initialize(Attributor &A) override {
-    if (!getAnchorScope() || getAnchorScope()->isDeclaration()) {
-      indicatePessimisticFixpoint();
-    } else {
-      Base::initialize(A);
-    }
-  }
-
-  /// See AbstractAttribute::trackStatistics()
-  void trackStatistics() const override {
-    STATS_DECLTRACK_ARG_ATTR(potential_values)
-  }
-};
-
-struct AAPotentialValuesReturned
-    : AAReturnedFromReturnedValues<AAPotentialValues, AAPotentialValuesImpl> {
-  using Base =
-      AAReturnedFromReturnedValues<AAPotentialValues, AAPotentialValuesImpl>;
-  AAPotentialValuesReturned(const IRPosition &IRP, Attributor &A)
-      : Base(IRP, A) {}
-
-  /// See AbstractAttribute::trackStatistics()
-  void trackStatistics() const override {
-    STATS_DECLTRACK_FNRET_ATTR(potential_values)
-  }
-};
-
-struct AAPotentialValuesFloating : AAPotentialValuesImpl {
-  AAPotentialValuesFloating(const IRPosition &IRP, Attributor &A)
-      : AAPotentialValuesImpl(IRP, A) {}
-
-  /// See AbstractAttribute::initialize(..).
-  void initialize(Attributor &A) override {
-    Value &V = getAssociatedValue();
-
-    if (auto *C = dyn_cast<ConstantInt>(&V)) {
-      unionAssumed(C->getValue());
-      indicateOptimisticFixpoint();
-      return;
-    }
-
-    if (isa<UndefValue>(&V)) {
-      // Collapse the undef state to 0.
-      unionAssumed(
-          APInt(/* numBits */ getAssociatedType()->getIntegerBitWidth(),
-                /* val */ 0));
-      indicateOptimisticFixpoint();
-      return;
-    }
-
-    if (isa<BinaryOperator>(&V) || isa<ICmpInst>(&V) || isa<CastInst>(&V))
-      return;
-
-    if (isa<SelectInst>(V) || isa<PHINode>(V))
-      return;
-
-    indicatePessimisticFixpoint();
-
-    LLVM_DEBUG(dbgs() << "[AAPotentialValues] We give up: "
-                      << getAssociatedValue() << "\n");
-  }
-
-  /// See AbstractAttribute::trackStatistics()
-  void trackStatistics() const override {
-    STATS_DECLTRACK_FLOATING_ATTR(potential_values)
-  }
-};
-
-struct AAPotentialValuesFunction : AAPotentialValuesImpl {
-  AAPotentialValuesFunction(const IRPosition &IRP, Attributor &A)
-      : AAPotentialValuesImpl(IRP, A) {}
-
-  /// See AbstractAttribute::initialize(...).
-  ChangeStatus updateImpl(Attributor &A) override {
-    llvm_unreachable("AAPotentialValues(Function|CallSite)::updateImpl will "
-                     "not be called");
-  }
-
-  /// See AbstractAttribute::trackStatistics()
-  void trackStatistics() const override {
-    STATS_DECLTRACK_FN_ATTR(potential_values)
-  }
-};
-
-struct AAPotentialValuesCallSite : AAPotentialValuesFunction {
-  AAPotentialValuesCallSite(const IRPosition &IRP, Attributor &A)
-      : AAPotentialValuesFunction(IRP, A) {}
-
-  /// See AbstractAttribute::trackStatistics()
-  void trackStatistics() const override {
-    STATS_DECLTRACK_CS_ATTR(potential_values)
-  }
-};
-
-struct AAPotentialValuesCallSiteReturned
-    : AACallSiteReturnedFromReturned<AAPotentialValues, AAPotentialValuesImpl> {
-  AAPotentialValuesCallSiteReturned(const IRPosition &IRP, Attributor &A)
-      : AACallSiteReturnedFromReturned<AAPotentialValues,
-                                       AAPotentialValuesImpl>(IRP, A) {}
-
-  /// See AbstractAttribute::trackStatistics()
-  void trackStatistics() const override {
-    STATS_DECLTRACK_CSRET_ATTR(potential_values)
-  }
-};
-
-struct AAPotentialValuesCallSiteArgument : AAPotentialValuesFloating {
-  AAPotentialValuesCallSiteArgument(const IRPosition &IRP, Attributor &A)
-      : AAPotentialValuesFloating(IRP, A) {}
-
-  /// See AbstractAttribute::trackStatistics()
-  void trackStatistics() const override {
-    STATS_DECLTRACK_CSARG_ATTR(potential_values)
-  }
-};
-
 } // namespace
 
 const char AAReturnedValues::ID = 0;
@@ -7258,7 +7098,6 @@ const char AAPrivatizablePtr::ID = 0;
 const char AAMemoryBehavior::ID = 0;
 const char AAMemoryLocation::ID = 0;
 const char AAValueConstantRange::ID = 0;
-const char AAPotentialValues::ID = 0;
 
 // Macro magic to create the static generator function for attributes that
 // follow the naming scheme.
@@ -7368,7 +7207,6 @@ CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AADereferenceable)
 CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAAlign)
 CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoCapture)
 CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAValueConstantRange)
-CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAPotentialValues)
 
 CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAValueSimplify)
 CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAIsDead)
diff --git a/llvm/test/Transforms/Attributor/potential.ll b/llvm/test/Transforms/Attributor/potential.ll
deleted file mode 100644
index a4fc23c8b700b..0000000000000
--- a/llvm/test/Transforms/Attributor/potential.ll
+++ /dev/null
@@ -1,394 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
-; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=9 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
-; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=7 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
-; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
-; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
-;
-; Test for multiple potential values
-;
-; potential-test 1
-; bool iszero(int c) { return c == 0; }
-; bool potential_test1(bool c) { return iszero(c ? 1 : -1); }
-
-define internal i1 @iszero1(i32 %c) {
-; CHECK-LABEL: define {{[^@]+}}@iszero1
-; CHECK-SAME: (i32 [[C:%.*]])
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[C]], 0
-; CHECK-NEXT:    ret i1 [[CMP]]
-;
-  %cmp = icmp eq i32 %c, 0
-  ret i1 %cmp
-}
-
-define i1 @potential_test1(i1 %c) {
-; CHECK-LABEL: define {{[^@]+}}@potential_test1
-; CHECK-SAME: (i1 [[C:%.*]])
-; CHECK-NEXT:    [[ARG:%.*]] = select i1 [[C]], i32 -1, i32 1
-; CHECK-NEXT:    [[RET:%.*]] = call i1 @iszero1(i32 [[ARG]])
-; CHECK-NEXT:    ret i1 [[RET]]
-;
-  %arg = select i1 %c, i32 -1, i32 1
-  %ret = call i1 @iszero1(i32 %arg)
-  ret i1 %ret
-}
-
-
-; potential-test 2
-;
-; potential values of argument of iszero are {1,-1}
-; potential value of returned value of iszero is 0
-;
-; int call_with_two_values(int x) { return iszero(x) + iszero(-x); }
-; int potential_test2(int x) { return call_with_two_values(1) + call_with_two_values(-1); }
-
-define internal i32 @iszero2(i32 %c) {
-; CHECK-LABEL: define {{[^@]+}}@iszero2
-; CHECK-SAME: (i32 [[C:%.*]])
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[C]], 0
-; CHECK-NEXT:    [[RET:%.*]] = zext i1 [[CMP]] to i32
-; CHECK-NEXT:    ret i32 [[RET]]
-;
-  %cmp = icmp eq i32 %c, 0
-  %ret = zext i1 %cmp to i32
-  ret i32 %ret
-}
-
-define internal i32 @call_with_two_values(i32 %c) {
-; IS__TUNIT____-LABEL: define {{[^@]+}}@call_with_two_values
-; IS__TUNIT____-SAME: (i32 [[C:%.*]])
-; IS__TUNIT____-NEXT:    [[CSRET1:%.*]] = call i32 @iszero2(i32 [[C]]) #0, !range !0
-; IS__TUNIT____-NEXT:    [[MINUSC:%.*]] = sub i32 0, [[C]]
-; IS__TUNIT____-NEXT:    [[CSRET2:%.*]] = call i32 @iszero2(i32 [[MINUSC]]) #0, !range !0
-; IS__TUNIT____-NEXT:    [[RET:%.*]] = add i32 [[CSRET1]], [[CSRET2]]
-; IS__TUNIT____-NEXT:    ret i32 [[RET]]
-;
-; IS__CGSCC____-LABEL: define {{[^@]+}}@call_with_two_values
-; IS__CGSCC____-SAME: (i32 [[C:%.*]])
-; IS__CGSCC____-NEXT:    [[CSRET1:%.*]] = call i32 @iszero2(i32 [[C]])
-; IS__CGSCC____-NEXT:    [[MINUSC:%.*]] = sub i32 0, [[C]]
-; IS__CGSCC____-NEXT:    [[CSRET2:%.*]] = call i32 @iszero2(i32 [[MINUSC]])
-; IS__CGSCC____-NEXT:    [[RET:%.*]] = add i32 [[CSRET1]], [[CSRET2]]
-; IS__CGSCC____-NEXT:    ret i32 [[RET]]
-;
-  %csret1 = call i32 @iszero2(i32 %c)
-  %minusc = sub i32 0, %c
-  %csret2 = call i32 @iszero2(i32 %minusc)
-  %ret = add i32 %csret1, %csret2
-  ret i32 %ret
-}
-
-define i32 @potential_test2(i1 %c) {
-; IS__TUNIT____-LABEL: define {{[^@]+}}@potential_test2
-; IS__TUNIT____-SAME: (i1 [[C:%.*]])
-; IS__TUNIT____-NEXT:    [[CSRET1:%.*]] = call i32 @call_with_two_values(i32 1) #0, !range !1
-; IS__TUNIT____-NEXT:    [[CSRET2:%.*]] = call i32 @call_with_two_values(i32 -1) #1, !range !1
-; IS__TUNIT____-NEXT:    [[RET:%.*]] = add i32 [[CSRET1]], [[CSRET2]]
-; IS__TUNIT____-NEXT:    ret i32 [[RET]]
-;
-; IS__CGSCC____-LABEL: define {{[^@]+}}@potential_test2
-; IS__CGSCC____-SAME: (i1 [[C:%.*]])
-; IS__CGSCC____-NEXT:    [[CSRET1:%.*]] = call i32 @call_with_two_values(i32 1)
-; IS__CGSCC____-NEXT:    [[CSRET2:%.*]] = call i32 @call_with_two_values(i32 -1)
-; IS__CGSCC____-NEXT:    [[RET:%.*]] = add i32 [[CSRET1]], [[CSRET2]]
-; IS__CGSCC____-NEXT:    ret i32 [[RET]]
-;
-  %csret1 = call i32 @call_with_two_values(i32 1)
-  %csret2 = call i32 @call_with_two_values(i32 -1)
-  %ret = add i32 %csret1, %csret2
-  ret i32 %ret
-}
-
-
-; potential-test 3
-;
-; potential values of returned value of f are {0,1}
-; potential values of argument of g are {0,1}
-; potential value of returned value of g is 1
-; then returned value of g can be simplified
-;
-; int zero_or_one(int c) { return c < 2; }
-; int potential_test3() { return zero_or_one(iszero(0))+zero_or_one(iszero(1)); }
-
-define internal i32 @iszero3(i32 %c) {
-; CHECK-LABEL: define {{[^@]+}}@iszero3
-; CHECK-SAME: (i32 [[C:%.*]])
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[C]], 0
-; CHECK-NEXT:    [[RET:%.*]] = zext i1 [[CMP]] to i32
-; CHECK-NEXT:    ret i32 [[RET]]
-;
-  %cmp = icmp eq i32 %c, 0
-  %ret = zext i1 %cmp to i32
-  ret i32 %ret
-}
-
-define internal i32 @less_than_two(i32 %c) {
-; CHECK-LABEL: define {{[^@]+}}@less_than_two
-; CHECK-SAME: (i32 [[C:%.*]])
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[C]], 2
-; CHECK-NEXT:    [[RET:%.*]] = zext i1 [[CMP]] to i32
-; CHECK-NEXT:    ret i32 [[RET]]
-;
-  %cmp = icmp slt i32 %c, 2
-  %ret = zext i1 %cmp to i32
-  ret i32 %ret
-}
-
-define i32 @potential_test3() {
-; NOT_TUNIT_NPM-LABEL: define {{[^@]+}}@potential_test3()
-; NOT_TUNIT_NPM-NEXT:    [[CMP1:%.*]] = call i32 @iszero3(i32 0)
-; NOT_TUNIT_NPM-NEXT:    [[TRUE1:%.*]] = call i32 @less_than_two(i32 [[CMP1]])
-; NOT_TUNIT_NPM-NEXT:    [[CMP2:%.*]] = call i32 @iszero3(i32 1)
-; NOT_TUNIT_NPM-NEXT:    [[TRUE2:%.*]] = call i32 @less_than_two(i32 [[CMP2]])
-; NOT_TUNIT_NPM-NEXT:    [[RET:%.*]] = add i32 [[TRUE1]], [[TRUE2]]
-; NOT_TUNIT_NPM-NEXT:    ret i32 [[RET]]
-;
-; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@potential_test3()
-; IS__TUNIT_NPM-NEXT:    [[CMP1:%.*]] = call i32 @iszero3(i32 0) #0, !range !0
-; IS__TUNIT_NPM-NEXT:    [[TRUE1:%.*]] = call i32 @less_than_two(i32 [[CMP1]]) #0, !range !0
-; IS__TUNIT_NPM-NEXT:    [[CMP2:%.*]] = call i32 @iszero3(i32 1) #0, !range !0
-; IS__TUNIT_NPM-NEXT:    [[TRUE2:%.*]] = call i32 @less_than_two(i32 [[CMP2]]) #0, !range !0
-; IS__TUNIT_NPM-NEXT:    [[RET:%.*]] = add i32 [[TRUE1]], [[TRUE2]]
-; IS__TUNIT_NPM-NEXT:    ret i32 [[RET]]
-;
-  %cmp1 = call i32 @iszero3(i32 0)
-  %true1 = call i32 @less_than_two(i32 %cmp1)
-  %cmp2 = call i32 @iszero3(i32 1)
-  %true2 = call i32 @less_than_two(i32 %cmp2)
-  %ret = add i32 %true1, %true2
-  ret i32 %ret
-}
-
-
-; potential-test 4,5
-;
-; simplified
-; int potential_test4(int c) { return return1or3(c) == 2; }
-; int potential_test5(int c) { return return1or3(c) == return2or4(c); }
-;
-; not simplified
-; int potential_test6(int c) { return return1or3(c) == 3; }
-; int potential_test7(int c) { return return1or3(c) == return3or4(c); }
-
-define i32 @potential_test4(i32 %c) {
-; IS__TUNIT____-LABEL: define {{[^@]+}}@potential_test4
-; IS__TUNIT____-SAME: (i32 [[C:%.*]])
-; IS__TUNIT____-NEXT:    [[CSRET:%.*]] = call i32 @return1or3(i32 [[C]]) #0, !range !2
-; IS__TUNIT____-NEXT:    [[FALSE:%.*]] = icmp eq i32 [[CSRET]], 2
-; IS__TUNIT____-NEXT:    [[RET:%.*]] = zext i1 [[FALSE]] to i32
-; IS__TUNIT____-NEXT:    ret i32 [[RET]]
-;
-; IS__CGSCC____-LABEL: define {{[^@]+}}@potential_test4
-; IS__CGSCC____-SAME: (i32 [[C:%.*]])
-; IS__CGSCC____-NEXT:    [[CSRET:%.*]] = call i32 @return1or3(i32 [[C]])
-; IS__CGSCC____-NEXT:    [[FALSE:%.*]] = icmp eq i32 [[CSRET]], 2
-; IS__CGSCC____-NEXT:    [[RET:%.*]] = zext i1 [[FALSE]] to i32
-; IS__CGSCC____-NEXT:    ret i32 [[RET]]
-;
-  %csret = call i32 @return1or3(i32 %c)
-  %false = icmp eq i32 %csret, 2
-  %ret = zext i1 %false to i32
-  ret i32 %ret
-}
-
-define i32 @potential_test5(i32 %c) {
-; IS__TUNIT____-LABEL: define {{[^@]+}}@potential_test5
-; IS__TUNIT____-SAME: (i32 [[C:%.*]])
-; IS__TUNIT____-NEXT:    [[CSRET1:%.*]] = call i32 @return1or3(i32 [[C]]) #0, !range !2
-; IS__TUNIT____-NEXT:    [[CSRET2:%.*]] = call i32 @return2or4(i32 [[C]]) #0, !range !3
-; IS__TUNIT____-NEXT:    [[FALSE:%.*]] = icmp eq i32 [[CSRET1]], [[CSRET2]]
-; IS__TUNIT____-NEXT:    [[RET:%.*]] = zext i1 [[FALSE]] to i32
-; IS__TUNIT____-NEXT:    ret i32 [[RET]]
-;
-; IS__CGSCC____-LABEL: define {{[^@]+}}@potential_test5
-; IS__CGSCC____-SAME: (i32 [[C:%.*]])
-; IS__CGSCC____-NEXT:    [[CSRET1:%.*]] = call i32 @return1or3(i32 [[C]])
-; IS__CGSCC____-NEXT:    [[CSRET2:%.*]] = call i32 @return2or4(i32 [[C]])
-; IS__CGSCC____-NEXT:    [[FALSE:%.*]] = icmp eq i32 [[CSRET1]], [[CSRET2]]
-; IS__CGSCC____-NEXT:    [[RET:%.*]] = zext i1 [[FALSE]] to i32
-; IS__CGSCC____-NEXT:    ret i32 [[RET]]
-;
-  %csret1 = call i32 @return1or3(i32 %c)
-  %csret2 = call i32 @return2or4(i32 %c)
-  %false = icmp eq i32 %csret1, %csret2
-  %ret = zext i1 %false to i32
-  ret i32 %ret
-}
-
-define i1 @potential_test6(i32 %c) {
-; IS__TUNIT____-LABEL: define {{[^@]+}}@potential_test6
-; IS__TUNIT____-SAME: (i32 [[C:%.*]])
-; IS__TUNIT____-NEXT:    [[CSRET1:%.*]] = call i32 @return1or3(i32 [[C]]) #0, !range !2
-; IS__TUNIT____-NEXT:    [[RET:%.*]] = icmp eq i32 [[CSRET1]], 3
-; IS__TUNIT____-NEXT:    ret i1 [[RET]]
-;
-; IS__CGSCC____-LABEL: define {{[^@]+}}@potential_test6
-; IS__CGSCC____-SAME: (i32 [[C:%.*]])
-; IS__CGSCC____-NEXT:    [[CSRET1:%.*]] = call i32 @return1or3(i32 [[C]])
-; IS__CGSCC____-NEXT:    [[RET:%.*]] = icmp eq i32 [[CSRET1]], 3
-; IS__CGSCC____-NEXT:    ret i1 [[RET]]
-;
-  %csret1 = call i32 @return1or3(i32 %c)
-  %ret = icmp eq i32 %csret1, 3
-  ret i1 %ret
-}
-
-define i1 @potential_test7(i32 %c) {
-; IS__TUNIT____-LABEL: define {{[^@]+}}@potential_test7
-; IS__TUNIT____-SAME: (i32 [[C:%.*]])
-; IS__TUNIT____-NEXT:    [[CSRET1:%.*]] = call i32 @return1or3(i32 [[C]]) #0, !range !2
-; IS__TUNIT____-NEXT:    [[CSRET2:%.*]] = call i32 @return3or4(i32 [[C]]) #0, !range !4
-; IS__TUNIT____-NEXT:    [[RET:%.*]] = icmp eq i32 [[CSRET1]], [[CSRET2]]
-; IS__TUNIT____-NEXT:    ret i1 [[RET]]
-;
-; IS__CGSCC____-LABEL: define {{[^@]+}}@potential_test7
-; IS__CGSCC____-SAME: (i32 [[C:%.*]])
-; IS__CGSCC____-NEXT:    [[CSRET1:%.*]] = call i32 @return1or3(i32 [[C]])
-; IS__CGSCC____-NEXT:    [[CSRET2:%.*]] = call i32 @return3or4(i32 [[C]])
-; IS__CGSCC____-NEXT:    [[RET:%.*]] = icmp eq i32 [[CSRET1]], [[CSRET2]]
-; IS__CGSCC____-NEXT:    ret i1 [[RET]]
-;
-  %csret1 = call i32 @return1or3(i32 %c)
-  %csret2 = call i32 @return3or4(i32 %c)
-  %ret = icmp eq i32 %csret1, %csret2
-  ret i1 %ret
-}
-
-define internal i32 @return1or3(i32 %c) {
-; CHECK-LABEL: define {{[^@]+}}@return1or3
-; CHECK-SAME: (i32 [[C:%.*]])
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[C]], 0
-; CHECK-NEXT:    [[RET:%.*]] = select i1 [[CMP]], i32 1, i32 3
-; CHECK-NEXT:    ret i32 [[RET]]
-;
-  %cmp = icmp eq i32 %c, 0
-  %ret = select i1 %cmp, i32 1, i32 3
-  ret i32 %ret
-}
-
-define internal i32 @return2or4(i32 %c) {
-; CHECK-LABEL: define {{[^@]+}}@return2or4
-; CHECK-SAME: (i32 [[C:%.*]])
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[C]], 0
-; CHECK-NEXT:    [[RET:%.*]] = select i1 [[CMP]], i32 2, i32 4
-; CHECK-NEXT:    ret i32 [[RET]]
-;
-  %cmp = icmp eq i32 %c, 0
-  %ret = select i1 %cmp, i32 2, i32 4
-  ret i32 %ret
-}
-
-define internal i32 @return3or4(i32 %c) {
-; CHECK-LABEL: define {{[^@]+}}@return3or4
-; CHECK-SAME: (i32 [[C:%.*]])
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[C]], 0
-; CHECK-NEXT:    [[RET:%.*]] = select i1 [[CMP]], i32 3, i32 4
-; CHECK-NEXT:    ret i32 [[RET]]
-;
-  %cmp = icmp eq i32 %c, 0
-  %ret = select i1 %cmp, i32 3, i32 4
-  ret i32 %ret
-}
-
-; potential-test 8
-;
-; propagate argument to callsite argument
-
-define internal i1 @cmp_with_four(i32 %c) {
-; CHECK-LABEL: define {{[^@]+}}@cmp_with_four
-; CHECK-SAME: (i32 [[C:%.*]])
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[C]], 4
-; CHECK-NEXT:    ret i1 [[CMP]]
-;
-  %cmp = icmp eq i32 %c, 4
-  ret i1 %cmp
-}
-
-define internal i1 @wrapper(i32 %c) {
-; CHECK-LABEL: define {{[^@]+}}@wrapper
-; CHECK-SAME: (i32 [[C:%.*]])
-; CHECK-NEXT:    [[RET:%.*]] = call i1 @cmp_with_four(i32 [[C]])
-; CHECK-NEXT:    ret i1 [[RET]]
-;
-  %ret = call i1 @cmp_with_four(i32 %c)
-  ret i1 %ret
-}
-
-define i1 @potential_test8() {
-; CHECK-LABEL: define {{[^@]+}}@potential_test8()
-; CHECK-NEXT:    [[RES1:%.*]] = call i1 @wrapper(i32 1)
-; CHECK-NEXT:    [[RES3:%.*]] = call i1 @wrapper(i32 3)
-; CHECK-NEXT:    [[RES5:%.*]] = call i1 @wrapper(i32 5)
-; CHECK-NEXT:    [[RES13:%.*]] = or i1 [[RES1]], [[RES3]]
-; CHECK-NEXT:    [[RES135:%.*]] = or i1 [[RES13]], [[RES5]]
-; CHECK-NEXT:    ret i1 [[RES135]]
-;
-  %res1 = call i1 @wrapper(i32 1)
-  %res3 = call i1 @wrapper(i32 3)
-  %res5 = call i1 @wrapper(i32 5)
-  %res13 = or i1 %res1, %res3
-  %res135 =  or i1 %res13, %res5
-  ret i1 %res135
-}
-
-define i1 @potential_test9() {
-; IS________OPM-LABEL: define {{[^@]+}}@potential_test9()
-; IS________OPM-NEXT:  entry:
-; IS________OPM-NEXT:    br label [[COND:%.*]]
-; IS________OPM:       cond:
-; IS________OPM-NEXT:    [[I_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[I_1:%.*]], [[INC:%.*]] ]
-; IS________OPM-NEXT:    [[C_0:%.*]] = phi i32 [ 1, [[ENTRY]] ], [ [[C_1:%.*]], [[INC]] ]
-; IS________OPM-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I_0]], 10
-; IS________OPM-NEXT:    br i1 [[CMP]], label [[BODY:%.*]], label [[END:%.*]]
-; IS________OPM:       body:
-; IS________OPM-NEXT:    [[C_1]] = mul i32 [[C_0]], -1
-; IS________OPM-NEXT:    br label [[INC]]
-; IS________OPM:       inc:
-; IS________OPM-NEXT:    [[I_1]] = add i32 [[I_0]], 1
-; IS________OPM-NEXT:    br label [[COND]]
-; IS________OPM:       end:
-; IS________OPM-NEXT:    [[RET:%.*]] = icmp eq i32 [[C_0]], 0
-; IS________OPM-NEXT:    ret i1 [[RET]]
-;
-; IS________NPM-LABEL: define {{[^@]+}}@potential_test9()
-; IS________NPM-NEXT:  entry:
-; IS________NPM-NEXT:    br label [[COND:%.*]]
-; IS________NPM:       cond:
-; IS________NPM-NEXT:    [[I_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[I_1:%.*]], [[INC:%.*]] ]
-; IS________NPM-NEXT:    [[C_0:%.*]] = phi i32 [ 1, [[ENTRY]] ], [ [[C_1:%.*]], [[INC]] ]
-; IS________NPM-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I_0]], 10
-; IS________NPM-NEXT:    br i1 [[CMP]], label [[BODY:%.*]], label [[END:%.*]]
-; IS________NPM:       body:
-; IS________NPM-NEXT:    [[C_1]] = mul i32 [[C_0]], -1
-; IS________NPM-NEXT:    br label [[INC]]
-; IS________NPM:       inc:
-; IS________NPM-NEXT:    [[I_1]] = add i32 [[I_0]], 1
-; IS________NPM-NEXT:    br label [[COND]]
-; IS________NPM:       end:
-; IS________NPM-NEXT:    ret i1 false
-;
-entry:
-  br label %cond
-cond:
-  %i.0 = phi i32 [0, %entry], [%i.1, %inc]
-  %c.0 = phi i32 [1, %entry], [%c.1, %inc]
-  %cmp = icmp slt i32 %i.0, 10
-  br i1 %cmp, label %body, label %end
-body:
-  %c.1 = mul i32 %c.0, -1
-  br label %inc
-inc:
-  %i.1 = add i32 %i.0, 1
-  br label %cond
-end:
-  %ret = icmp eq i32 %c.0, 0
-  ret i1 %ret
-}
-
-; IS__TUNIT____: !0 = !{i32 0, i32 2}
-; IS__TUNIT____: !1 = !{i32 0, i32 3}
-; IS__TUNIT____: !2 = !{i32 1, i32 4}
-; IS__TUNIT____: !3 = !{i32 2, i32 5}
-; IS__TUNIT____: !4 = !{i32 3, i32 5}
-; IS__TUNIT____-NOT: !5
-
-; NOT_TUNIT____-NOT: !0

From 90dab1aece7100ace855321162c0d2a09b31c1b3 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Sun, 2 Aug 2020 14:57:39 +0100
Subject: [PATCH 0846/1035] Remove unused param tag to fix Wdocumentation
 warning. NFC.

---
 clang/lib/Parse/ParseOpenMP.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/clang/lib/Parse/ParseOpenMP.cpp b/clang/lib/Parse/ParseOpenMP.cpp
index 8a92a8259c444..f24cb3d3f0b7d 100644
--- a/clang/lib/Parse/ParseOpenMP.cpp
+++ b/clang/lib/Parse/ParseOpenMP.cpp
@@ -3191,7 +3191,6 @@ static void parseMapType(Parser &P, Parser::OpenMPVarListDataTy &Data) {
 
 /// Parses simple expression in parens for single-expression clauses of OpenMP
 /// constructs.
-/// \param RLoc Returned location of right paren.
 ExprResult Parser::ParseOpenMPIteratorsExpr() {
   assert(Tok.is(tok::identifier) && PP.getSpelling(Tok) == "iterator" &&
          "Expected 'iterator' token.");

From d14a22da5e437dfbf5fe96a6586cec2153f36861 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Sun, 2 Aug 2020 15:03:04 +0100
Subject: [PATCH 0847/1035] [DAG] TargetLowering::LowerAsmOutputForConstraint -
 pass SDLoc as const&

Try to be more consistent with the SDLoc param in the TargetLowering methods.
---
 llvm/include/llvm/CodeGen/TargetLowering.h       | 2 +-
 llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp | 4 ++--
 llvm/lib/Target/X86/X86ISelLowering.cpp          | 4 ++--
 llvm/lib/Target/X86/X86ISelLowering.h            | 3 ++-
 4 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 79ef2d06d38f4..eae39f319d89f 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -4182,7 +4182,7 @@ class TargetLowering : public TargetLoweringBase {
 
   // Lower custom output constraints. If invalid, return SDValue().
   virtual SDValue LowerAsmOutputForConstraint(SDValue &Chain, SDValue &Flag,
-                                              SDLoc DL,
+                                              const SDLoc &DL,
                                               const AsmOperandInfo &OpInfo,
                                               SelectionDAG &DAG) const;
 
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 4c7c46218c785..5e26ff354b674 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -4322,8 +4322,8 @@ const char *TargetLowering::LowerXConstraint(EVT ConstraintVT) const {
 }
 
 SDValue TargetLowering::LowerAsmOutputForConstraint(
-    SDValue &Chain, SDValue &Flag, SDLoc DL, const AsmOperandInfo &OpInfo,
-    SelectionDAG &DAG) const {
+    SDValue &Chain, SDValue &Flag, const SDLoc &DL,
+    const AsmOperandInfo &OpInfo, SelectionDAG &DAG) const {
   return SDValue();
 }
 
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 15f283b1bc42a..93852e13b7d08 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -49594,8 +49594,8 @@ LowerXConstraint(EVT ConstraintVT) const {
 
 // Lower @cc targets via setcc.
 SDValue X86TargetLowering::LowerAsmOutputForConstraint(
-    SDValue &Chain, SDValue &Flag, SDLoc DL, const AsmOperandInfo &OpInfo,
-    SelectionDAG &DAG) const {
+    SDValue &Chain, SDValue &Flag, const SDLoc &DL,
+    const AsmOperandInfo &OpInfo, SelectionDAG &DAG) const {
   X86::CondCode Cond = parseConstraintCode(OpInfo.ConstraintCode);
   if (Cond == X86::COND_INVALID)
     return SDValue();
diff --git a/llvm/lib/Target/X86/X86ISelLowering.h b/llvm/lib/Target/X86/X86ISelLowering.h
index 2c22a62fb506f..55611b6bfc9f8 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.h
+++ b/llvm/lib/Target/X86/X86ISelLowering.h
@@ -1116,7 +1116,8 @@ namespace llvm {
     }
 
     /// Handle Lowering flag assembly outputs.
-    SDValue LowerAsmOutputForConstraint(SDValue &Chain, SDValue &Flag, SDLoc DL,
+    SDValue LowerAsmOutputForConstraint(SDValue &Chain, SDValue &Flag,
+                                        const SDLoc &DL,
                                         const AsmOperandInfo &Constraint,
                                         SelectionDAG &DAG) const override;
 

From b8ffbf0e02e24d7be3017c8ee5f17dab9e39719f Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Sun, 2 Aug 2020 15:31:36 +0100
Subject: [PATCH 0848/1035] [DAG] TargetLowering::expandMUL_LOHI - pass SDLoc
 as const&

Try to be more consistent with the SDLoc param in the TargetLowering methods.

This also exposes an issue where we were passing a SDNode as a SDLoc, relying on the implicit SDLoc(SDNode) constructor.
---
 llvm/include/llvm/CodeGen/TargetLowering.h       | 2 +-
 llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp    | 2 +-
 llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp | 4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index eae39f319d89f..f93b11bd1c2cb 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -4263,7 +4263,7 @@ class TargetLowering : public TargetLoweringBase {
   /// \param RL Low bits of the RHS of the MUL.  See LL for meaning
   /// \param RH High bits of the RHS of the MUL.  See LL for meaning.
   /// \returns true if the node has been expanded, false if it has not
-  bool expandMUL_LOHI(unsigned Opcode, EVT VT, SDLoc dl, SDValue LHS,
+  bool expandMUL_LOHI(unsigned Opcode, EVT VT, const SDLoc &dl, SDValue LHS,
                       SDValue RHS, SmallVectorImpl<SDValue> &Result, EVT HiLoVT,
                       SelectionDAG &DAG, MulExpansionKind Kind,
                       SDValue LL = SDValue(), SDValue LH = SDValue(),
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index ce185be162bcd..87e9a9e280d5c 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -3398,7 +3398,7 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
     SmallVector<SDValue, 4> Halves;
     EVT HalfType = EVT(VT).getHalfSizedIntegerVT(*DAG.getContext());
     assert(TLI.isTypeLegal(HalfType));
-    if (TLI.expandMUL_LOHI(Node->getOpcode(), VT, Node, LHS, RHS, Halves,
+    if (TLI.expandMUL_LOHI(Node->getOpcode(), VT, dl, LHS, RHS, Halves,
                            HalfType, DAG,
                            TargetLowering::MulExpansionKind::Always)) {
       for (unsigned i = 0; i < 2; ++i) {
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 5e26ff354b674..c7829cdd9cdbd 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -5928,7 +5928,7 @@ SDValue TargetLowering::getNegatedExpression(SDValue Op, SelectionDAG &DAG,
 // Legalization Utilities
 //===----------------------------------------------------------------------===//
 
-bool TargetLowering::expandMUL_LOHI(unsigned Opcode, EVT VT, SDLoc dl,
+bool TargetLowering::expandMUL_LOHI(unsigned Opcode, EVT VT, const SDLoc &dl,
                                     SDValue LHS, SDValue RHS,
                                     SmallVectorImpl<SDValue> &Result,
                                     EVT HiLoVT, SelectionDAG &DAG,
@@ -6116,7 +6116,7 @@ bool TargetLowering::expandMUL(SDNode *N, SDValue &Lo, SDValue &Hi, EVT HiLoVT,
                                SDValue LL, SDValue LH, SDValue RL,
                                SDValue RH) const {
   SmallVector<SDValue, 2> Result;
-  bool Ok = expandMUL_LOHI(N->getOpcode(), N->getValueType(0), N,
+  bool Ok = expandMUL_LOHI(N->getOpcode(), N->getValueType(0), SDLoc(N),
                            N->getOperand(0), N->getOperand(1), Result, HiLoVT,
                            DAG, Kind, LL, LH, RL, RH);
   if (Ok) {

From 7dd4f03595d3687cab555d538c82a41f1c1043ce Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Sun, 2 Aug 2020 15:16:00 +0100
Subject: [PATCH 0849/1035] Use merge null and isa<> tests into
 isa_and_nonnull<>. NFCI.

---
 llvm/lib/Target/X86/X86InstCombineIntrinsic.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Target/X86/X86InstCombineIntrinsic.cpp b/llvm/lib/Target/X86/X86InstCombineIntrinsic.cpp
index aea355a297318..c9e092e5deca7 100644
--- a/llvm/lib/Target/X86/X86InstCombineIntrinsic.cpp
+++ b/llvm/lib/Target/X86/X86InstCombineIntrinsic.cpp
@@ -377,7 +377,7 @@ static Value *simplifyX86varShift(const IntrinsicInst &II,
   SmallVector<int, 8> ShiftAmts;
   for (int I = 0; I < NumElts; ++I) {
     auto *CElt = CShift->getAggregateElement(I);
-    if (CElt && isa<UndefValue>(CElt)) {
+    if (isa_and_nonnull<UndefValue>(CElt)) {
       ShiftAmts.push_back(-1);
       continue;
     }

From 00d0f354f26dc725ee1ce756df383557eeb44c65 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Sun, 2 Aug 2020 15:21:52 +0100
Subject: [PATCH 0850/1035] X86InstrInfo.cpp - fix include ordering. NFCI.

---
 llvm/lib/Target/X86/X86InstrInfo.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp
index 42c1111735701..ae838274f2e68 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.cpp
+++ b/llvm/lib/Target/X86/X86InstrInfo.cpp
@@ -28,9 +28,9 @@
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/StackMaps.h"
+#include "llvm/IR/DebugInfoMetadata.h"
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/Function.h"
-#include "llvm/IR/DebugInfoMetadata.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCInst.h"

From 212570abcf755b8577a7aec80777503232d36d77 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Mon, 15 Jun 2020 11:54:49 -0400
Subject: [PATCH 0851/1035] GlobalISel: Implement bitcast action for
 G_EXTRACT_VECTOR_ELEMENT

For AMDGPU, vectors with elements < 32 bits should be indexed in
32-bit elements and the desired bits extracted from there. For
elements > 64-bits, these should be reduce to 64/32 elements to enable
the normal dynamic indexing paths.

In the dynamic index cases, this produces shorter code most of the
time. This does immediately regress the constant index cases, but this
should be fixed once we have the most basic of shift combines.

The element size > 64 case is pretty much ported from the exisiting
DAG implementation for extract element promote. The increasing element
size case is new.
---
 .../llvm/CodeGen/GlobalISel/LegalizerHelper.h |    4 +
 .../CodeGen/GlobalISel/LegalizerHelper.cpp    |  118 +
 .../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp |   44 +-
 .../AMDGPU/GlobalISel/extractelement.i128.ll  |  769 ++++
 .../AMDGPU/GlobalISel/extractelement.i16.ll   |  802 +++++
 .../AMDGPU/GlobalISel/extractelement.i8.ll    | 3135 +++++++++++++++++
 .../legalize-extract-vector-elt.mir           | 1114 +++++-
 .../legalize-llvm.amdgcn.image.dim.a16.ll     |  952 +++--
 .../GlobalISel/legalize-shuffle-vector.mir    |   34 +-
 .../legalize-shuffle-vector.s16.mir           |  371 +-
 10 files changed, 6776 insertions(+), 567 deletions(-)
 create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.i128.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.i16.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.i8.ll

diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
index d925c53a57502..e819dca5bdf0f 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
@@ -307,6 +307,10 @@ class LegalizerHelper {
   LegalizeResult narrowScalarCTTZ(MachineInstr &MI, unsigned TypeIdx, LLT Ty);
   LegalizeResult narrowScalarCTPOP(MachineInstr &MI, unsigned TypeIdx, LLT Ty);
 
+  /// Perform Bitcast legalize action on G_EXTRACT_VECTOR_ELT.
+  LegalizeResult bitcastExtractVectorElt(MachineInstr &MI, unsigned TypeIdx,
+                                         LLT CastTy);
+
   LegalizeResult lowerBitcast(MachineInstr &MI);
   LegalizeResult lowerBitCount(MachineInstr &MI, unsigned TypeIdx, LLT Ty);
 
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index abb983dac6bdc..920c9e008012e 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -2330,6 +2330,122 @@ LegalizerHelper::lowerBitcast(MachineInstr &MI) {
   return UnableToLegalize;
 }
 
+/// Perform a G_EXTRACT_VECTOR_ELT in a different sized vector element. If this
+/// is casting to a vector with a smaller element size, perform multiple element
+/// extracts and merge the results. If this is coercing to a vector with larger
+/// elements, index the bitcasted vector and extract the target element with bit
+/// operations. This is intended to force the indexing in the native register
+/// size for architectures that can dynamically index the register file.
+LegalizerHelper::LegalizeResult
+LegalizerHelper::bitcastExtractVectorElt(MachineInstr &MI, unsigned TypeIdx,
+                                         LLT CastTy) {
+  if (TypeIdx != 1)
+    return UnableToLegalize;
+
+  Register Dst = MI.getOperand(0).getReg();
+  Register SrcVec = MI.getOperand(1).getReg();
+  Register Idx = MI.getOperand(2).getReg();
+  LLT SrcVecTy = MRI.getType(SrcVec);
+  LLT IdxTy = MRI.getType(Idx);
+
+  LLT SrcEltTy = SrcVecTy.getElementType();
+  unsigned NewNumElts = CastTy.isVector() ? CastTy.getNumElements() : 1;
+  unsigned OldNumElts = SrcVecTy.getNumElements();
+
+  LLT NewEltTy = CastTy.isVector() ? CastTy.getElementType() : CastTy;
+  Register CastVec = MIRBuilder.buildBitcast(CastTy, SrcVec).getReg(0);
+
+  const unsigned NewEltSize = NewEltTy.getSizeInBits();
+  const unsigned OldEltSize = SrcEltTy.getSizeInBits();
+  if (NewNumElts > OldNumElts) {
+    // Decreasing the vector element size
+    //
+    // e.g. i64 = extract_vector_elt x:v2i64, y:i32
+    //  =>
+    //  v4i32:castx = bitcast x:v2i64
+    //
+    // i64 = bitcast
+    //   (v2i32 build_vector (i32 (extract_vector_elt castx, (2 * y))),
+    //                       (i32 (extract_vector_elt castx, (2 * y + 1)))
+    //
+    if (NewNumElts % OldNumElts != 0)
+      return UnableToLegalize;
+
+    // Type of the intermediate result vector.
+    const unsigned NewEltsPerOldElt = NewNumElts / OldNumElts;
+    LLT MidTy = LLT::scalarOrVector(NewEltsPerOldElt, NewEltTy);
+
+    auto NewEltsPerOldEltK = MIRBuilder.buildConstant(IdxTy, NewEltsPerOldElt);
+
+    SmallVector<Register, 8> NewOps(NewEltsPerOldElt);
+    auto NewBaseIdx = MIRBuilder.buildMul(IdxTy, Idx, NewEltsPerOldEltK);
+
+    for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
+      auto IdxOffset = MIRBuilder.buildConstant(IdxTy, I);
+      auto TmpIdx = MIRBuilder.buildAdd(IdxTy, NewBaseIdx, IdxOffset);
+      auto Elt = MIRBuilder.buildExtractVectorElement(NewEltTy, CastVec, TmpIdx);
+      NewOps[I] = Elt.getReg(0);
+    }
+
+    auto NewVec = MIRBuilder.buildBuildVector(MidTy, NewOps);
+    MIRBuilder.buildBitcast(Dst, NewVec);
+    MI.eraseFromParent();
+    return Legalized;
+  }
+
+  if (NewNumElts < OldNumElts) {
+    if (NewEltSize % OldEltSize != 0)
+      return UnableToLegalize;
+
+    // This only depends on powers of 2 because we use bit tricks to figure out
+    // the bit offset we need to shift to get the target element. A general
+    // expansion could emit division/multiply.
+    if (!isPowerOf2_32(NewEltSize / OldEltSize))
+      return UnableToLegalize;
+
+    // Increasing the vector element size.
+    // %elt:_(small_elt) = G_EXTRACT_VECTOR_ELT %vec:_(<N x small_elt>), %idx
+    //
+    //   =>
+    //
+    // %cast = G_BITCAST %vec
+    // %scaled_idx = G_LSHR %idx, Log2(DstEltSize / SrcEltSize)
+    // %wide_elt  = G_EXTRACT_VECTOR_ELT %cast, %scaled_idx
+    // %offset_idx = G_AND %idx, ~(-1 << Log2(DstEltSize / SrcEltSize))
+    // %offset_bits = G_SHL %offset_idx, Log2(SrcEltSize)
+    // %elt_bits = G_LSHR %wide_elt, %offset_bits
+    // %elt = G_TRUNC %elt_bits
+
+    const unsigned Log2EltRatio = Log2_32(NewEltSize / OldEltSize);
+    auto Log2Ratio = MIRBuilder.buildConstant(IdxTy, Log2EltRatio);
+
+    // Divide to get the index in the wider element type.
+    auto ScaledIdx = MIRBuilder.buildLShr(IdxTy, Idx, Log2Ratio);
+
+    Register WideElt = CastVec;
+    if (CastTy.isVector()) {
+      WideElt = MIRBuilder.buildExtractVectorElement(NewEltTy, CastVec,
+                                                     ScaledIdx).getReg(0);
+    }
+
+    // Now figure out the amount we need to shift to get the target bits.
+    auto OffsetMask = MIRBuilder.buildConstant(
+      IdxTy, ~(APInt::getAllOnesValue(IdxTy.getSizeInBits()) << Log2EltRatio));
+    auto OffsetIdx = MIRBuilder.buildAnd(IdxTy, Idx, OffsetMask);
+    auto OffsetBits = MIRBuilder.buildShl(
+      IdxTy, OffsetIdx,
+      MIRBuilder.buildConstant(IdxTy, Log2_32(OldEltSize)));
+
+    // Shift the wide element to get the target element.
+    auto ExtractedBits = MIRBuilder.buildLShr(NewEltTy, WideElt, OffsetBits);
+    MIRBuilder.buildTrunc(Dst, ExtractedBits);
+    MI.eraseFromParent();
+    return Legalized;
+  }
+
+  return UnableToLegalize;
+}
+
 LegalizerHelper::LegalizeResult
 LegalizerHelper::bitcast(MachineInstr &MI, unsigned TypeIdx, LLT CastTy) {
   switch (MI.getOpcode()) {
@@ -2378,6 +2494,8 @@ LegalizerHelper::bitcast(MachineInstr &MI, unsigned TypeIdx, LLT CastTy) {
     Observer.changedInstr(MI);
     return Legalized;
   }
+  case TargetOpcode::G_EXTRACT_VECTOR_ELT:
+    return bitcastExtractVectorElt(MI, TypeIdx, CastTy);
   default:
     return UnableToLegalize;
   }
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index c5d5f1675bc8d..cc97e11707ab1 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -70,6 +70,13 @@ static LegalityPredicate isSmallOddVector(unsigned TypeIdx) {
   };
 }
 
+static LegalityPredicate sizeIsMultipleOf32(unsigned TypeIdx) {
+  return [=](const LegalityQuery &Query) {
+    const LLT Ty = Query.Types[TypeIdx];
+    return Ty.getSizeInBits() % 32 == 0;
+  };
+}
+
 static LegalityPredicate isWideVec16(unsigned TypeIdx) {
   return [=](const LegalityQuery &Query) {
     const LLT Ty = Query.Types[TypeIdx];
@@ -132,6 +139,15 @@ static LegalizeMutation bitcastToRegisterType(unsigned TypeIdx) {
   };
 }
 
+static LegalizeMutation bitcastToVectorElement32(unsigned TypeIdx) {
+  return [=](const LegalityQuery &Query) {
+    const LLT Ty = Query.Types[TypeIdx];
+    unsigned Size = Ty.getSizeInBits();
+    assert(Size % 32 == 0);
+    return std::make_pair(TypeIdx, LLT::scalarOrVector(Size / 32, 32));
+  };
+}
+
 static LegalityPredicate vectorSmallerThan(unsigned TypeIdx, unsigned Size) {
   return [=](const LegalityQuery &Query) {
     const LLT QueryTy = Query.Types[TypeIdx];
@@ -1279,11 +1295,29 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
           const LLT EltTy = Query.Types[EltTypeIdx];
           const LLT VecTy = Query.Types[VecTypeIdx];
           const LLT IdxTy = Query.Types[IdxTypeIdx];
-          return (EltTy.getSizeInBits() == 16 ||
-                  EltTy.getSizeInBits() % 32 == 0) &&
-                 VecTy.getSizeInBits() % 32 == 0 &&
-                 VecTy.getSizeInBits() <= MaxRegisterSize &&
-                 IdxTy.getSizeInBits() == 32;
+          const unsigned EltSize = EltTy.getSizeInBits();
+          return (EltSize == 32 || EltSize == 64) &&
+                  VecTy.getSizeInBits() % 32 == 0 &&
+                  VecTy.getSizeInBits() <= MaxRegisterSize &&
+                  IdxTy.getSizeInBits() == 32;
+        })
+      .bitcastIf(all(sizeIsMultipleOf32(1), scalarOrEltNarrowerThan(1, 32)),
+                 bitcastToVectorElement32(1))
+      //.bitcastIf(vectorSmallerThan(1, 32), bitcastToScalar(1))
+      .bitcastIf(
+        all(sizeIsMultipleOf32(1), scalarOrEltWiderThan(1, 64)),
+        [=](const LegalityQuery &Query) {
+          // For > 64-bit element types, try to turn this into a 64-bit
+          // element vector since we may be able to do better indexing
+          // if this is scalar. If not, fall back to 32.
+          const LLT EltTy = Query.Types[EltTypeIdx];
+          const LLT VecTy = Query.Types[VecTypeIdx];
+          const unsigned DstEltSize = EltTy.getSizeInBits();
+          const unsigned VecSize = VecTy.getSizeInBits();
+
+          const unsigned TargetEltSize = DstEltSize % 64 == 0 ? 64 : 32;
+          return std::make_pair(
+            VecTypeIdx, LLT::vector(VecSize / TargetEltSize, TargetEltSize));
         })
       .clampScalar(EltTypeIdx, S32, S64)
       .clampScalar(VecTypeIdx, S32, S64)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.i128.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.i128.ll
new file mode 100644
index 0000000000000..28c0651b10fd2
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.i128.ll
@@ -0,0 +1,769 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX8 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=hawaii -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX7 %s
+
+define amdgpu_ps i128 @extractelement_sgpr_v4i128_sgpr_idx(<4 x i128> addrspace(4)* inreg %ptr, i32 inreg %idx) {
+; GFX9-LABEL: extractelement_sgpr_v4i128_sgpr_idx:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx16 s[8:23], s[2:3], 0x0
+; GFX9-NEXT:    s_lshl_b32 m0, s4, 1
+; GFX9-NEXT:    s_nop 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_movrels_b64 s[0:1], s[8:9]
+; GFX9-NEXT:    s_movrels_b64 s[2:3], s[10:11]
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: extractelement_sgpr_v4i128_sgpr_idx:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_load_dwordx16 s[8:23], s[2:3], 0x0
+; GFX8-NEXT:    s_lshl_b32 m0, s4, 1
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_movrels_b64 s[0:1], s[8:9]
+; GFX8-NEXT:    s_movrels_b64 s[2:3], s[10:11]
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX7-LABEL: extractelement_sgpr_v4i128_sgpr_idx:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_load_dwordx16 s[8:23], s[2:3], 0x0
+; GFX7-NEXT:    s_lshl_b32 m0, s4, 1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_movrels_b64 s[0:1], s[8:9]
+; GFX7-NEXT:    s_movrels_b64 s[2:3], s[10:11]
+; GFX7-NEXT:    ; return to shader part epilog
+  %vector = load <4 x i128>, <4 x i128> addrspace(4)* %ptr
+  %element = extractelement <4 x i128> %vector, i32 %idx
+  ret i128 %element
+}
+
+define amdgpu_ps i128 @extractelement_vgpr_v4i128_sgpr_idx(<4 x i128> addrspace(1)* %ptr, i32 inreg %idx) {
+; GFX9-LABEL: extractelement_vgpr_v4i128_sgpr_idx:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    global_load_dwordx4 v[2:5], v[0:1], off
+; GFX9-NEXT:    global_load_dwordx4 v[6:9], v[0:1], off offset:16
+; GFX9-NEXT:    global_load_dwordx4 v[10:13], v[0:1], off offset:32
+; GFX9-NEXT:    global_load_dwordx4 v[14:17], v[0:1], off offset:48
+; GFX9-NEXT:    s_lshl_b32 s0, s2, 1
+; GFX9-NEXT:    s_lshl_b32 s0, s0, 1
+; GFX9-NEXT:    s_set_gpr_idx_on s0, gpr_idx(SRC0)
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, v2
+; GFX9-NEXT:    v_mov_b32_e32 v1, v3
+; GFX9-NEXT:    v_mov_b32_e32 v18, v2
+; GFX9-NEXT:    v_mov_b32_e32 v3, v3
+; GFX9-NEXT:    s_set_gpr_idx_off
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v18
+; GFX9-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: extractelement_vgpr_v4i128_sgpr_idx:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    v_add_u32_e32 v6, vcc, 16, v0
+; GFX8-NEXT:    v_addc_u32_e32 v7, vcc, 0, v1, vcc
+; GFX8-NEXT:    v_add_u32_e32 v10, vcc, 32, v0
+; GFX8-NEXT:    v_addc_u32_e32 v11, vcc, 0, v1, vcc
+; GFX8-NEXT:    flat_load_dwordx4 v[2:5], v[0:1]
+; GFX8-NEXT:    flat_load_dwordx4 v[6:9], v[6:7]
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 48, v0
+; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT:    flat_load_dwordx4 v[10:13], v[10:11]
+; GFX8-NEXT:    flat_load_dwordx4 v[14:17], v[0:1]
+; GFX8-NEXT:    s_lshl_b32 s0, s2, 1
+; GFX8-NEXT:    s_lshl_b32 m0, s0, 1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_movrels_b32_e32 v1, v3
+; GFX8-NEXT:    v_movrels_b32_e32 v0, v2
+; GFX8-NEXT:    v_mov_b32_e32 v3, v1
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX8-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX8-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX7-LABEL: extractelement_vgpr_v4i128_sgpr_idx:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_mov_b32 s6, 0
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:    buffer_load_dwordx4 v[2:5], v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT:    buffer_load_dwordx4 v[6:9], v[0:1], s[4:7], 0 addr64 offset:16
+; GFX7-NEXT:    buffer_load_dwordx4 v[10:13], v[0:1], s[4:7], 0 addr64 offset:32
+; GFX7-NEXT:    buffer_load_dwordx4 v[14:17], v[0:1], s[4:7], 0 addr64 offset:48
+; GFX7-NEXT:    s_lshl_b32 s0, s2, 1
+; GFX7-NEXT:    s_lshl_b32 m0, s0, 1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_movrels_b32_e32 v1, v3
+; GFX7-NEXT:    v_movrels_b32_e32 v0, v2
+; GFX7-NEXT:    v_mov_b32_e32 v3, v1
+; GFX7-NEXT:    v_mov_b32_e32 v2, v0
+; GFX7-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX7-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX7-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX7-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX7-NEXT:    ; return to shader part epilog
+  %vector = load <4 x i128>, <4 x i128> addrspace(1)* %ptr
+  %element = extractelement <4 x i128> %vector, i32 %idx
+  ret i128 %element
+}
+
+define i128 @extractelement_vgpr_v4i128_vgpr_idx(<4 x i128> addrspace(1)* %ptr, i32 %idx) {
+; GFX9-LABEL: extractelement_vgpr_v4i128_vgpr_idx:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_lshlrev_b32_e32 v16, 1, v2
+; GFX9-NEXT:    global_load_dwordx4 v[2:5], v[0:1], off
+; GFX9-NEXT:    global_load_dwordx4 v[6:9], v[0:1], off offset:16
+; GFX9-NEXT:    v_add_u32_e32 v17, 1, v16
+; GFX9-NEXT:    v_cmp_eq_u32_e64 s[4:5], 1, v16
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v17
+; GFX9-NEXT:    v_cmp_eq_u32_e64 s[6:7], 6, v16
+; GFX9-NEXT:    v_cmp_eq_u32_e64 s[8:9], 7, v16
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_cndmask_b32_e64 v10, v2, v4, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v11, v3, v5, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 2, v16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, v10, v6, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v5, v11, v7, vcc
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 2, v17
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 3, v16
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, v4, v8, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v5, v5, v9, vcc
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 3, v17
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v8, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v9, vcc
+; GFX9-NEXT:    global_load_dwordx4 v[8:11], v[0:1], off offset:32
+; GFX9-NEXT:    global_load_dwordx4 v[12:15], v[0:1], off offset:48
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 4, v16
+; GFX9-NEXT:    v_cmp_eq_u32_e64 s[4:5], 7, v17
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v8, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v9, vcc
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 4, v17
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v8, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v9, vcc
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 5, v16
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v10, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v11, vcc
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 5, v17
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v10, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v11, vcc
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 6, v17
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v12, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v0, v12, s[6:7]
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, v13, s[6:7]
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v13, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v0, v14, s[8:9]
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, v15, s[8:9]
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, v2, v14, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v3, v3, v15, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: extractelement_vgpr_v4i128_vgpr_idx:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_add_u32_e32 v3, vcc, 16, v0
+; GFX8-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
+; GFX8-NEXT:    flat_load_dwordx4 v[8:11], v[0:1]
+; GFX8-NEXT:    flat_load_dwordx4 v[4:7], v[3:4]
+; GFX8-NEXT:    v_lshlrev_b32_e32 v16, 1, v2
+; GFX8-NEXT:    v_add_u32_e32 v17, vcc, 1, v16
+; GFX8-NEXT:    v_cmp_eq_u32_e64 s[4:5], 1, v16
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v17
+; GFX8-NEXT:    v_cmp_eq_u32_e64 s[6:7], 6, v16
+; GFX8-NEXT:    v_cmp_eq_u32_e64 s[8:9], 7, v16
+; GFX8-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, v8, v10, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v3, v9, v11, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v8, v8, v10, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v9, v9, v11, vcc
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 2, v16
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 2, v17
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v8, v4, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v5, v9, v5, vcc
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 3, v16
+; GFX8-NEXT:    v_cndmask_b32_e32 v18, v2, v6, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v19, v3, v7, vcc
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 3, v17
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v5, v5, v7, vcc
+; GFX8-NEXT:    v_add_u32_e32 v2, vcc, 32, v0
+; GFX8-NEXT:    v_addc_u32_e32 v3, vcc, 0, v1, vcc
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 48, v0
+; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT:    flat_load_dwordx4 v[8:11], v[2:3]
+; GFX8-NEXT:    flat_load_dwordx4 v[12:15], v[0:1]
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 4, v16
+; GFX8-NEXT:    v_cmp_eq_u32_e64 s[4:5], 7, v17
+; GFX8-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v18, v8, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v19, v9, vcc
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 4, v17
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v4, v8, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v5, v9, vcc
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 5, v16
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v10, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v11, vcc
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 5, v17
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v10, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v3, v11, vcc
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 6, v17
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v12, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v0, v12, s[6:7]
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, v1, v13, s[6:7]
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v3, v13, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v0, v14, s[8:9]
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, v1, v15, s[8:9]
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, v2, v14, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v3, v3, v15, s[4:5]
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: extractelement_vgpr_v4i128_vgpr_idx:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    s_mov_b32 s10, 0
+; GFX7-NEXT:    s_mov_b32 s11, 0xf000
+; GFX7-NEXT:    s_mov_b64 s[8:9], 0
+; GFX7-NEXT:    v_lshlrev_b32_e32 v16, 1, v2
+; GFX7-NEXT:    buffer_load_dwordx4 v[2:5], v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT:    buffer_load_dwordx4 v[6:9], v[0:1], s[8:11], 0 addr64 offset:16
+; GFX7-NEXT:    v_add_i32_e32 v17, vcc, 1, v16
+; GFX7-NEXT:    v_cmp_eq_u32_e64 s[4:5], 1, v16
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v17
+; GFX7-NEXT:    v_cmp_eq_u32_e64 s[6:7], 6, v16
+; GFX7-NEXT:    s_waitcnt vmcnt(1)
+; GFX7-NEXT:    v_cndmask_b32_e64 v10, v2, v4, s[4:5]
+; GFX7-NEXT:    v_cndmask_b32_e64 v11, v3, v5, s[4:5]
+; GFX7-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 2, v16
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_cndmask_b32_e32 v4, v10, v6, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v5, v11, v7, vcc
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 2, v17
+; GFX7-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 3, v16
+; GFX7-NEXT:    v_cndmask_b32_e32 v4, v4, v8, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v5, v5, v9, vcc
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 3, v17
+; GFX7-NEXT:    v_cndmask_b32_e32 v2, v2, v8, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v3, v3, v9, vcc
+; GFX7-NEXT:    buffer_load_dwordx4 v[8:11], v[0:1], s[8:11], 0 addr64 offset:32
+; GFX7-NEXT:    buffer_load_dwordx4 v[12:15], v[0:1], s[8:11], 0 addr64 offset:48
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 4, v16
+; GFX7-NEXT:    v_cmp_eq_u32_e64 s[4:5], 7, v17
+; GFX7-NEXT:    v_cmp_eq_u32_e64 s[8:9], 7, v16
+; GFX7-NEXT:    s_waitcnt vmcnt(1)
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v4, v8, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v5, v9, vcc
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 4, v17
+; GFX7-NEXT:    v_cndmask_b32_e32 v2, v2, v8, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v3, v3, v9, vcc
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 5, v16
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v10, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v1, v11, vcc
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 5, v17
+; GFX7-NEXT:    v_cndmask_b32_e32 v2, v2, v10, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v3, v3, v11, vcc
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 6, v17
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_cndmask_b32_e32 v2, v2, v12, vcc
+; GFX7-NEXT:    v_cndmask_b32_e64 v0, v0, v12, s[6:7]
+; GFX7-NEXT:    v_cndmask_b32_e64 v1, v1, v13, s[6:7]
+; GFX7-NEXT:    v_cndmask_b32_e32 v3, v3, v13, vcc
+; GFX7-NEXT:    v_cndmask_b32_e64 v0, v0, v14, s[8:9]
+; GFX7-NEXT:    v_cndmask_b32_e64 v1, v1, v15, s[8:9]
+; GFX7-NEXT:    v_cndmask_b32_e64 v2, v2, v14, s[4:5]
+; GFX7-NEXT:    v_cndmask_b32_e64 v3, v3, v15, s[4:5]
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+  %vector = load <4 x i128>, <4 x i128> addrspace(1)* %ptr
+  %element = extractelement <4 x i128> %vector, i32 %idx
+  ret i128 %element
+}
+
+define amdgpu_ps i128 @extractelement_sgpr_v4i128_vgpr_idx(<4 x i128> addrspace(4)* inreg %ptr, i32 %idx) {
+; GFX9-LABEL: extractelement_sgpr_v4i128_vgpr_idx:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx16 s[0:15], s[2:3], 0x0
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 1, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v3, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s2
+; GFX9-NEXT:    v_mov_b32_e32 v0, s1
+; GFX9-NEXT:    v_mov_b32_e32 v4, s3
+; GFX9-NEXT:    v_mov_b32_e32 v5, s4
+; GFX9-NEXT:    v_mov_b32_e32 v6, s5
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 2, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v6, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v7, s6
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 3, v2
+; GFX9-NEXT:    v_mov_b32_e32 v8, s7
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v7, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v9, s8
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 4, v2
+; GFX9-NEXT:    v_mov_b32_e32 v10, s9
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v10, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v11, s10
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 5, v2
+; GFX9-NEXT:    v_mov_b32_e32 v12, s11
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v11, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v12, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v13, s12
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 6, v2
+; GFX9-NEXT:    v_mov_b32_e32 v14, s13
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v13, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, v0, v14, vcc
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 7, v2
+; GFX9-NEXT:    v_mov_b32_e32 v15, s14
+; GFX9-NEXT:    v_mov_b32_e32 v16, s15
+; GFX9-NEXT:    v_add_u32_e32 v2, 1, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v1, v15, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v4, v16, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v4, s1
+; GFX9-NEXT:    v_mov_b32_e32 v5, s2
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
+; GFX9-NEXT:    v_mov_b32_e32 v6, s3
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v7, s4
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 2, v2
+; GFX9-NEXT:    v_mov_b32_e32 v8, s5
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, v4, v8, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v9, s6
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 3, v2
+; GFX9-NEXT:    v_mov_b32_e32 v10, s7
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v9, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, v4, v10, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v11, s8
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 4, v2
+; GFX9-NEXT:    v_mov_b32_e32 v12, s9
+; GFX9-NEXT:    v_mov_b32_e32 v13, s10
+; GFX9-NEXT:    v_mov_b32_e32 v5, s11
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v11, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, v4, v12, vcc
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 5, v2
+; GFX9-NEXT:    v_mov_b32_e32 v6, s12
+; GFX9-NEXT:    v_mov_b32_e32 v7, s13
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v13, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 6, v2
+; GFX9-NEXT:    v_mov_b32_e32 v8, s14
+; GFX9-NEXT:    v_mov_b32_e32 v9, s15
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v6, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, v4, v7, vcc
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 7, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v8, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v4, v9, vcc
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX9-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: extractelement_sgpr_v4i128_vgpr_idx:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_load_dwordx16 s[0:15], s[2:3], 0x0
+; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 1, v0
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v3, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s2
+; GFX8-NEXT:    v_mov_b32_e32 v0, s1
+; GFX8-NEXT:    v_mov_b32_e32 v4, s3
+; GFX8-NEXT:    v_mov_b32_e32 v5, s4
+; GFX8-NEXT:    v_mov_b32_e32 v6, s5
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 2, v2
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v6, vcc
+; GFX8-NEXT:    v_mov_b32_e32 v7, s6
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 3, v2
+; GFX8-NEXT:    v_mov_b32_e32 v8, s7
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v7, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc
+; GFX8-NEXT:    v_mov_b32_e32 v9, s8
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 4, v2
+; GFX8-NEXT:    v_mov_b32_e32 v10, s9
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v10, vcc
+; GFX8-NEXT:    v_mov_b32_e32 v11, s10
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 5, v2
+; GFX8-NEXT:    v_mov_b32_e32 v12, s11
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v11, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v12, vcc
+; GFX8-NEXT:    v_mov_b32_e32 v13, s12
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 6, v2
+; GFX8-NEXT:    v_mov_b32_e32 v14, s13
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v13, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v0, v14, vcc
+; GFX8-NEXT:    v_mov_b32_e32 v15, s14
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 7, v2
+; GFX8-NEXT:    v_mov_b32_e32 v16, s15
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v1, v15, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v4, v16, vcc
+; GFX8-NEXT:    v_add_u32_e32 v2, vcc, 1, v2
+; GFX8-NEXT:    v_mov_b32_e32 v4, s1
+; GFX8-NEXT:    v_mov_b32_e32 v5, s2
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
+; GFX8-NEXT:    v_mov_b32_e32 v6, s3
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
+; GFX8-NEXT:    v_mov_b32_e32 v7, s4
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 2, v2
+; GFX8-NEXT:    v_mov_b32_e32 v8, s5
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v4, v8, vcc
+; GFX8-NEXT:    v_mov_b32_e32 v9, s6
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 3, v2
+; GFX8-NEXT:    v_mov_b32_e32 v10, s7
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v3, v9, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v4, v10, vcc
+; GFX8-NEXT:    v_mov_b32_e32 v11, s8
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 4, v2
+; GFX8-NEXT:    v_mov_b32_e32 v12, s9
+; GFX8-NEXT:    v_mov_b32_e32 v13, s10
+; GFX8-NEXT:    v_mov_b32_e32 v5, s11
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v3, v11, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v4, v12, vcc
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 5, v2
+; GFX8-NEXT:    v_mov_b32_e32 v6, s12
+; GFX8-NEXT:    v_mov_b32_e32 v7, s13
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v3, v13, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 6, v2
+; GFX8-NEXT:    v_mov_b32_e32 v8, s14
+; GFX8-NEXT:    v_mov_b32_e32 v9, s15
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v3, v6, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v4, v7, vcc
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 7, v2
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v3, v8, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v4, v9, vcc
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX8-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX8-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX7-LABEL: extractelement_sgpr_v4i128_vgpr_idx:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_load_dwordx16 s[0:15], s[2:3], 0x0
+; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 1, v0
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v3, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s1
+; GFX7-NEXT:    v_mov_b32_e32 v4, s3
+; GFX7-NEXT:    v_mov_b32_e32 v5, s4
+; GFX7-NEXT:    v_mov_b32_e32 v6, s5
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 2, v2
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v6, vcc
+; GFX7-NEXT:    v_mov_b32_e32 v7, s6
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 3, v2
+; GFX7-NEXT:    v_mov_b32_e32 v8, s7
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v1, v7, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc
+; GFX7-NEXT:    v_mov_b32_e32 v9, s8
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 4, v2
+; GFX7-NEXT:    v_mov_b32_e32 v10, s9
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v10, vcc
+; GFX7-NEXT:    v_mov_b32_e32 v11, s10
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 5, v2
+; GFX7-NEXT:    v_mov_b32_e32 v12, s11
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v1, v11, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v12, vcc
+; GFX7-NEXT:    v_mov_b32_e32 v13, s12
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 6, v2
+; GFX7-NEXT:    v_mov_b32_e32 v14, s13
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v1, v13, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v4, v0, v14, vcc
+; GFX7-NEXT:    v_mov_b32_e32 v15, s14
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 7, v2
+; GFX7-NEXT:    v_mov_b32_e32 v16, s15
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v1, v15, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v4, v16, vcc
+; GFX7-NEXT:    v_add_i32_e32 v2, vcc, 1, v2
+; GFX7-NEXT:    v_mov_b32_e32 v4, s1
+; GFX7-NEXT:    v_mov_b32_e32 v5, s2
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
+; GFX7-NEXT:    v_mov_b32_e32 v6, s3
+; GFX7-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
+; GFX7-NEXT:    v_mov_b32_e32 v7, s4
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 2, v2
+; GFX7-NEXT:    v_mov_b32_e32 v8, s5
+; GFX7-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v4, v4, v8, vcc
+; GFX7-NEXT:    v_mov_b32_e32 v9, s6
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 3, v2
+; GFX7-NEXT:    v_mov_b32_e32 v10, s7
+; GFX7-NEXT:    v_cndmask_b32_e32 v3, v3, v9, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v4, v4, v10, vcc
+; GFX7-NEXT:    v_mov_b32_e32 v11, s8
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 4, v2
+; GFX7-NEXT:    v_mov_b32_e32 v12, s9
+; GFX7-NEXT:    v_mov_b32_e32 v13, s10
+; GFX7-NEXT:    v_mov_b32_e32 v5, s11
+; GFX7-NEXT:    v_cndmask_b32_e32 v3, v3, v11, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v4, v4, v12, vcc
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 5, v2
+; GFX7-NEXT:    v_mov_b32_e32 v6, s12
+; GFX7-NEXT:    v_mov_b32_e32 v7, s13
+; GFX7-NEXT:    v_cndmask_b32_e32 v3, v3, v13, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 6, v2
+; GFX7-NEXT:    v_mov_b32_e32 v8, s14
+; GFX7-NEXT:    v_mov_b32_e32 v9, s15
+; GFX7-NEXT:    v_cndmask_b32_e32 v3, v3, v6, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v4, v4, v7, vcc
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 7, v2
+; GFX7-NEXT:    v_cndmask_b32_e32 v2, v3, v8, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v3, v4, v9, vcc
+; GFX7-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX7-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX7-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX7-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX7-NEXT:    ; return to shader part epilog
+  %vector = load <4 x i128>, <4 x i128> addrspace(4)* %ptr
+  %element = extractelement <4 x i128> %vector, i32 %idx
+  ret i128 %element
+}
+
+define amdgpu_ps i128 @extractelement_sgpr_v4i128_idx0(<4 x i128> addrspace(4)* inreg %ptr) {
+; GCN-LABEL: extractelement_sgpr_v4i128_idx0:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx16 s[0:15], s[2:3], 0x0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    ; return to shader part epilog
+  %vector = load <4 x i128>, <4 x i128> addrspace(4)* %ptr
+  %element = extractelement <4 x i128> %vector, i32 0
+  ret i128 %element
+}
+
+define amdgpu_ps i128 @extractelement_sgpr_v4i128_idx1(<4 x i128> addrspace(4)* inreg %ptr) {
+; GCN-LABEL: extractelement_sgpr_v4i128_idx1:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx16 s[0:15], s[2:3], 0x0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_mov_b32 s0, s4
+; GCN-NEXT:    s_mov_b32 s1, s5
+; GCN-NEXT:    s_mov_b32 s2, s6
+; GCN-NEXT:    s_mov_b32 s3, s7
+; GCN-NEXT:    ; return to shader part epilog
+  %vector = load <4 x i128>, <4 x i128> addrspace(4)* %ptr
+  %element = extractelement <4 x i128> %vector, i32 1
+  ret i128 %element
+}
+
+define amdgpu_ps i128 @extractelement_sgpr_v4i128_idx2(<4 x i128> addrspace(4)* inreg %ptr) {
+; GCN-LABEL: extractelement_sgpr_v4i128_idx2:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx16 s[0:15], s[2:3], 0x0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_mov_b32 s0, s8
+; GCN-NEXT:    s_mov_b32 s1, s9
+; GCN-NEXT:    s_mov_b32 s2, s10
+; GCN-NEXT:    s_mov_b32 s3, s11
+; GCN-NEXT:    ; return to shader part epilog
+  %vector = load <4 x i128>, <4 x i128> addrspace(4)* %ptr
+  %element = extractelement <4 x i128> %vector, i32 2
+  ret i128 %element
+}
+
+define amdgpu_ps i128 @extractelement_sgpr_v4i128_idx3(<4 x i128> addrspace(4)* inreg %ptr) {
+; GCN-LABEL: extractelement_sgpr_v4i128_idx3:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx16 s[0:15], s[2:3], 0x0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_mov_b32 s0, s12
+; GCN-NEXT:    s_mov_b32 s1, s13
+; GCN-NEXT:    s_mov_b32 s2, s14
+; GCN-NEXT:    s_mov_b32 s3, s15
+; GCN-NEXT:    ; return to shader part epilog
+  %vector = load <4 x i128>, <4 x i128> addrspace(4)* %ptr
+  %element = extractelement <4 x i128> %vector, i32 3
+  ret i128 %element
+}
+
+define i128 @extractelement_vgpr_v4i128_idx0(<4 x i128> addrspace(1)* %ptr) {
+; GFX9-LABEL: extractelement_vgpr_v4i128_idx0:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dwordx4 v[0:3], v[0:1], off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: extractelement_vgpr_v4i128_idx0:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: extractelement_vgpr_v4i128_idx0:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    s_mov_b32 s6, 0
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:    buffer_load_dwordx4 v[0:3], v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+  %vector = load <4 x i128>, <4 x i128> addrspace(1)* %ptr
+  %element = extractelement <4 x i128> %vector, i32 0
+  ret i128 %element
+}
+
+define i128 @extractelement_vgpr_v4i128_idx1(<4 x i128> addrspace(1)* %ptr) {
+; GFX9-LABEL: extractelement_vgpr_v4i128_idx1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dwordx4 v[4:7], v[0:1], off offset:16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, v4
+; GFX9-NEXT:    v_mov_b32_e32 v1, v5
+; GFX9-NEXT:    v_mov_b32_e32 v2, v6
+; GFX9-NEXT:    v_mov_b32_e32 v3, v7
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: extractelement_vgpr_v4i128_idx1:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
+; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT:    flat_load_dwordx4 v[4:7], v[0:1]
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v0, v4
+; GFX8-NEXT:    v_mov_b32_e32 v1, v5
+; GFX8-NEXT:    v_mov_b32_e32 v2, v6
+; GFX8-NEXT:    v_mov_b32_e32 v3, v7
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: extractelement_vgpr_v4i128_idx1:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    s_mov_b32 s6, 0
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:    buffer_load_dwordx4 v[4:7], v[0:1], s[4:7], 0 addr64 offset:16
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, v4
+; GFX7-NEXT:    v_mov_b32_e32 v1, v5
+; GFX7-NEXT:    v_mov_b32_e32 v2, v6
+; GFX7-NEXT:    v_mov_b32_e32 v3, v7
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+  %vector = load <4 x i128>, <4 x i128> addrspace(1)* %ptr
+  %element = extractelement <4 x i128> %vector, i32 1
+  ret i128 %element
+}
+
+define i128 @extractelement_vgpr_v4i128_idx2(<4 x i128> addrspace(1)* %ptr) {
+; GFX9-LABEL: extractelement_vgpr_v4i128_idx2:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dwordx4 v[8:11], v[0:1], off offset:32
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, v8
+; GFX9-NEXT:    v_mov_b32_e32 v1, v9
+; GFX9-NEXT:    v_mov_b32_e32 v2, v10
+; GFX9-NEXT:    v_mov_b32_e32 v3, v11
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: extractelement_vgpr_v4i128_idx2:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 32, v0
+; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT:    flat_load_dwordx4 v[8:11], v[0:1]
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v0, v8
+; GFX8-NEXT:    v_mov_b32_e32 v1, v9
+; GFX8-NEXT:    v_mov_b32_e32 v2, v10
+; GFX8-NEXT:    v_mov_b32_e32 v3, v11
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: extractelement_vgpr_v4i128_idx2:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    s_mov_b32 s6, 0
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:    buffer_load_dwordx4 v[8:11], v[0:1], s[4:7], 0 addr64 offset:32
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, v8
+; GFX7-NEXT:    v_mov_b32_e32 v1, v9
+; GFX7-NEXT:    v_mov_b32_e32 v2, v10
+; GFX7-NEXT:    v_mov_b32_e32 v3, v11
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+  %vector = load <4 x i128>, <4 x i128> addrspace(1)* %ptr
+  %element = extractelement <4 x i128> %vector, i32 2
+  ret i128 %element
+}
+
+define i128 @extractelement_vgpr_v4i128_idx3(<4 x i128> addrspace(1)* %ptr) {
+; GFX9-LABEL: extractelement_vgpr_v4i128_idx3:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dwordx4 v[12:15], v[0:1], off offset:48
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, v12
+; GFX9-NEXT:    v_mov_b32_e32 v1, v13
+; GFX9-NEXT:    v_mov_b32_e32 v2, v14
+; GFX9-NEXT:    v_mov_b32_e32 v3, v15
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: extractelement_vgpr_v4i128_idx3:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 48, v0
+; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT:    flat_load_dwordx4 v[12:15], v[0:1]
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v0, v12
+; GFX8-NEXT:    v_mov_b32_e32 v1, v13
+; GFX8-NEXT:    v_mov_b32_e32 v2, v14
+; GFX8-NEXT:    v_mov_b32_e32 v3, v15
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: extractelement_vgpr_v4i128_idx3:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    s_mov_b32 s6, 0
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:    buffer_load_dwordx4 v[12:15], v[0:1], s[4:7], 0 addr64 offset:48
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, v12
+; GFX7-NEXT:    v_mov_b32_e32 v1, v13
+; GFX7-NEXT:    v_mov_b32_e32 v2, v14
+; GFX7-NEXT:    v_mov_b32_e32 v3, v15
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+  %vector = load <4 x i128>, <4 x i128> addrspace(1)* %ptr
+  %element = extractelement <4 x i128> %vector, i32 3
+  ret i128 %element
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.i16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.i16.ll
new file mode 100644
index 0000000000000..13d7fbeda0f6d
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.i16.ll
@@ -0,0 +1,802 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX8 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=hawaii -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX7 %s
+
+define amdgpu_ps i16 @extractelement_sgpr_v4i16_sgpr_idx(<4 x i16> addrspace(4)* inreg %ptr, i32 inreg %idx) {
+; GCN-LABEL: extractelement_sgpr_v4i16_sgpr_idx:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[2:3], 0x0
+; GCN-NEXT:    s_lshr_b32 s2, s4, 1
+; GCN-NEXT:    s_cmp_eq_u32 s2, 1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_cselect_b32 s0, s1, s0
+; GCN-NEXT:    s_and_b32 s1, s4, 1
+; GCN-NEXT:    s_lshl_b32 s1, s1, 4
+; GCN-NEXT:    s_lshr_b32 s0, s0, s1
+; GCN-NEXT:    ; return to shader part epilog
+  %vector = load <4 x i16>, <4 x i16> addrspace(4)* %ptr
+  %element = extractelement <4 x i16> %vector, i32 %idx
+  ret i16 %element
+}
+
+define amdgpu_ps i16 @extractelement_vgpr_v4i16_sgpr_idx(<4 x i16> addrspace(1)* %ptr, i32 inreg %idx) {
+; GFX9-LABEL: extractelement_vgpr_v4i16_sgpr_idx:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off
+; GFX9-NEXT:    s_lshr_b32 s0, s2, 1
+; GFX9-NEXT:    v_cmp_eq_u32_e64 vcc, s0, 1
+; GFX9-NEXT:    s_and_b32 s1, s2, 1
+; GFX9-NEXT:    s_lshl_b32 s0, s1, 4
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, s0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: extractelement_vgpr_v4i16_sgpr_idx:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; GFX8-NEXT:    s_lshr_b32 s0, s2, 1
+; GFX8-NEXT:    v_cmp_eq_u32_e64 vcc, s0, 1
+; GFX8-NEXT:    s_and_b32 s1, s2, 1
+; GFX8-NEXT:    s_lshl_b32 s0, s1, 4
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX8-NEXT:    v_lshrrev_b32_e32 v0, s0, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX7-LABEL: extractelement_vgpr_v4i16_sgpr_idx:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; GFX7-NEXT:    s_lshr_b32 s0, s2, 1
+; GFX7-NEXT:    v_cmp_eq_u32_e64 vcc, s0, 1
+; GFX7-NEXT:    s_and_b32 s1, s2, 1
+; GFX7-NEXT:    s_lshl_b32 s0, s1, 4
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX7-NEXT:    v_lshrrev_b32_e32 v0, s0, v0
+; GFX7-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX7-NEXT:    ; return to shader part epilog
+  %vector = load <4 x i16>, <4 x i16> addrspace(1)* %ptr
+  %element = extractelement <4 x i16> %vector, i32 %idx
+  ret i16 %element
+}
+
+define i16 @extractelement_vgpr_v4i16_vgpr_idx(<4 x i16> addrspace(1)* %ptr, i32 %idx) {
+; GFX9-LABEL: extractelement_vgpr_v4i16_vgpr_idx:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off
+; GFX9-NEXT:    v_lshrrev_b32_e32 v3, 1, v2
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v3
+; GFX9-NEXT:    v_and_b32_e32 v2, 1, v2
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 4, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, v1, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: extractelement_vgpr_v4i16_vgpr_idx:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; GFX8-NEXT:    v_lshrrev_b32_e32 v3, 1, v2
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v3
+; GFX8-NEXT:    v_and_b32_e32 v2, 1, v2
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX8-NEXT:    v_lshlrev_b32_e32 v1, 4, v2
+; GFX8-NEXT:    v_lshrrev_b32_e32 v0, v1, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: extractelement_vgpr_v4i16_vgpr_idx:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; GFX7-NEXT:    v_lshrrev_b32_e32 v3, 1, v2
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v3
+; GFX7-NEXT:    v_and_b32_e32 v2, 1, v2
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 4, v2
+; GFX7-NEXT:    v_lshrrev_b32_e32 v0, v1, v0
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+  %vector = load <4 x i16>, <4 x i16> addrspace(1)* %ptr
+  %element = extractelement <4 x i16> %vector, i32 %idx
+  ret i16 %element
+}
+
+define amdgpu_ps i16 @extractelement_sgpr_v4i16_vgpr_idx(<4 x i16> addrspace(4)* inreg %ptr, i32 %idx) {
+; GCN-LABEL: extractelement_sgpr_v4i16_vgpr_idx:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[2:3], 0x0
+; GCN-NEXT:    v_lshrrev_b32_e32 v1, 1, v0
+; GCN-NEXT:    v_and_b32_e32 v0, 1, v0
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 4, v0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v2, s0
+; GCN-NEXT:    v_mov_b32_e32 v3, s1
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v2, v3, vcc
+; GCN-NEXT:    v_lshrrev_b32_e32 v0, v0, v1
+; GCN-NEXT:    v_readfirstlane_b32 s0, v0
+; GCN-NEXT:    ; return to shader part epilog
+  %vector = load <4 x i16>, <4 x i16> addrspace(4)* %ptr
+  %element = extractelement <4 x i16> %vector, i32 %idx
+  ret i16 %element
+}
+
+define amdgpu_ps i16 @extractelement_sgpr_v4i16_idx0(<4 x i16> addrspace(4)* inreg %ptr) {
+; GCN-LABEL: extractelement_sgpr_v4i16_idx0:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[2:3], 0x0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    ; return to shader part epilog
+  %vector = load <4 x i16>, <4 x i16> addrspace(4)* %ptr
+  %element = extractelement <4 x i16> %vector, i32 0
+  ret i16 %element
+}
+
+define amdgpu_ps i16 @extractelement_sgpr_v4i16_idx1(<4 x i16> addrspace(4)* inreg %ptr) {
+; GCN-LABEL: extractelement_sgpr_v4i16_idx1:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[2:3], 0x0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_lshr_b32 s0, s0, 16
+; GCN-NEXT:    ; return to shader part epilog
+  %vector = load <4 x i16>, <4 x i16> addrspace(4)* %ptr
+  %element = extractelement <4 x i16> %vector, i32 1
+  ret i16 %element
+}
+
+define amdgpu_ps i16 @extractelement_sgpr_v4i16_idx2(<4 x i16> addrspace(4)* inreg %ptr) {
+; GCN-LABEL: extractelement_sgpr_v4i16_idx2:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[2:3], 0x0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_mov_b32 s0, s1
+; GCN-NEXT:    ; return to shader part epilog
+  %vector = load <4 x i16>, <4 x i16> addrspace(4)* %ptr
+  %element = extractelement <4 x i16> %vector, i32 2
+  ret i16 %element
+}
+
+define amdgpu_ps i16 @extractelement_sgpr_v4i16_idx3(<4 x i16> addrspace(4)* inreg %ptr) {
+; GCN-LABEL: extractelement_sgpr_v4i16_idx3:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[2:3], 0x0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_lshr_b32 s0, s1, 16
+; GCN-NEXT:    ; return to shader part epilog
+  %vector = load <4 x i16>, <4 x i16> addrspace(4)* %ptr
+  %element = extractelement <4 x i16> %vector, i32 3
+  ret i16 %element
+}
+
+define i16 @extractelement_vgpr_v4i16_idx0(<4 x i16> addrspace(1)* %ptr) {
+; GFX9-LABEL: extractelement_vgpr_v4i16_idx0:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: extractelement_vgpr_v4i16_idx0:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: extractelement_vgpr_v4i16_idx0:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+  %vector = load <4 x i16>, <4 x i16> addrspace(1)* %ptr
+  %element = extractelement <4 x i16> %vector, i32 0
+  ret i16 %element
+}
+
+define i16 @extractelement_vgpr_v4i16_idx1(<4 x i16> addrspace(1)* %ptr) {
+; GFX9-LABEL: extractelement_vgpr_v4i16_idx1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: extractelement_vgpr_v4i16_idx1:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: extractelement_vgpr_v4i16_idx1:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+  %vector = load <4 x i16>, <4 x i16> addrspace(1)* %ptr
+  %element = extractelement <4 x i16> %vector, i32 1
+  ret i16 %element
+}
+
+define i16 @extractelement_vgpr_v4i16_idx2(<4 x i16> addrspace(1)* %ptr) {
+; GFX9-LABEL: extractelement_vgpr_v4i16_idx2:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: extractelement_vgpr_v4i16_idx2:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v0, v1
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: extractelement_vgpr_v4i16_idx2:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, v1
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+  %vector = load <4 x i16>, <4 x i16> addrspace(1)* %ptr
+  %element = extractelement <4 x i16> %vector, i32 2
+  ret i16 %element
+}
+
+define i16 @extractelement_vgpr_v4i16_idx3(<4 x i16> addrspace(1)* %ptr) {
+; GFX9-LABEL: extractelement_vgpr_v4i16_idx3:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: extractelement_vgpr_v4i16_idx3:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_lshrrev_b32_e32 v0, 16, v1
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: extractelement_vgpr_v4i16_idx3:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_lshrrev_b32_e32 v0, 16, v1
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+  %vector = load <4 x i16>, <4 x i16> addrspace(1)* %ptr
+  %element = extractelement <4 x i16> %vector, i32 3
+  ret i16 %element
+}
+
+define amdgpu_ps i16 @extractelement_sgpr_v8i16_sgpr_idx(<8 x i16> addrspace(4)* inreg %ptr, i32 inreg %idx) {
+; GCN-LABEL: extractelement_sgpr_v8i16_sgpr_idx:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[2:3], 0x0
+; GCN-NEXT:    s_lshr_b32 s5, s4, 1
+; GCN-NEXT:    s_cmp_eq_u32 s5, 1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_cselect_b32 s0, s1, s0
+; GCN-NEXT:    s_cmp_eq_u32 s5, 2
+; GCN-NEXT:    s_cselect_b32 s0, s2, s0
+; GCN-NEXT:    s_cmp_eq_u32 s5, 3
+; GCN-NEXT:    s_cselect_b32 s0, s3, s0
+; GCN-NEXT:    s_and_b32 s1, s4, 1
+; GCN-NEXT:    s_lshl_b32 s1, s1, 4
+; GCN-NEXT:    s_lshr_b32 s0, s0, s1
+; GCN-NEXT:    ; return to shader part epilog
+  %vector = load <8 x i16>, <8 x i16> addrspace(4)* %ptr
+  %element = extractelement <8 x i16> %vector, i32 %idx
+  ret i16 %element
+}
+
+define amdgpu_ps i16 @extractelement_vgpr_v8i16_sgpr_idx(<8 x i16> addrspace(1)* %ptr, i32 inreg %idx) {
+; GFX9-LABEL: extractelement_vgpr_v8i16_sgpr_idx:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    global_load_dwordx4 v[0:3], v[0:1], off
+; GFX9-NEXT:    s_lshr_b32 s0, s2, 1
+; GFX9-NEXT:    v_cmp_eq_u32_e64 vcc, s0, 1
+; GFX9-NEXT:    s_and_b32 s1, s2, 1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX9-NEXT:    v_cmp_eq_u32_e64 vcc, s0, 2
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX9-NEXT:    v_cmp_eq_u32_e64 vcc, s0, 3
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; GFX9-NEXT:    s_lshl_b32 s0, s1, 4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, s0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: extractelement_vgpr_v8i16_sgpr_idx:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
+; GFX8-NEXT:    s_lshr_b32 s0, s2, 1
+; GFX8-NEXT:    v_cmp_eq_u32_e64 vcc, s0, 1
+; GFX8-NEXT:    s_and_b32 s1, s2, 1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX8-NEXT:    v_cmp_eq_u32_e64 vcc, s0, 2
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX8-NEXT:    v_cmp_eq_u32_e64 vcc, s0, 3
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; GFX8-NEXT:    s_lshl_b32 s0, s1, 4
+; GFX8-NEXT:    v_lshrrev_b32_e32 v0, s0, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX7-LABEL: extractelement_vgpr_v8i16_sgpr_idx:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_mov_b32 s6, 0
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:    buffer_load_dwordx4 v[0:3], v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT:    s_lshr_b32 s0, s2, 1
+; GFX7-NEXT:    v_cmp_eq_u32_e64 vcc, s0, 1
+; GFX7-NEXT:    s_and_b32 s1, s2, 1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX7-NEXT:    v_cmp_eq_u32_e64 vcc, s0, 2
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX7-NEXT:    v_cmp_eq_u32_e64 vcc, s0, 3
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; GFX7-NEXT:    s_lshl_b32 s0, s1, 4
+; GFX7-NEXT:    v_lshrrev_b32_e32 v0, s0, v0
+; GFX7-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX7-NEXT:    ; return to shader part epilog
+  %vector = load <8 x i16>, <8 x i16> addrspace(1)* %ptr
+  %element = extractelement <8 x i16> %vector, i32 %idx
+  ret i16 %element
+}
+
+define i16 @extractelement_vgpr_v8i16_vgpr_idx(<8 x i16> addrspace(1)* %ptr, i32 %idx) {
+; GFX9-LABEL: extractelement_vgpr_v8i16_vgpr_idx:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dwordx4 v[3:6], v[0:1], off
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 1, v2
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
+; GFX9-NEXT:    v_and_b32_e32 v1, 1, v2
+; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 4, v1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 2, v0
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 3, v0
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v6, vcc
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, v1, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: extractelement_vgpr_v8i16_vgpr_idx:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    flat_load_dwordx4 v[3:6], v[0:1]
+; GFX8-NEXT:    v_lshrrev_b32_e32 v0, 1, v2
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
+; GFX8-NEXT:    v_and_b32_e32 v1, 1, v2
+; GFX8-NEXT:    v_lshlrev_b32_e32 v1, 4, v1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 2, v0
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 3, v0
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v6, vcc
+; GFX8-NEXT:    v_lshrrev_b32_e32 v0, v1, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: extractelement_vgpr_v8i16_vgpr_idx:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    s_mov_b32 s6, 0
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:    buffer_load_dwordx4 v[3:6], v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT:    v_lshrrev_b32_e32 v0, 1, v2
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
+; GFX7-NEXT:    v_and_b32_e32 v1, 1, v2
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 4, v1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 2, v0
+; GFX7-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 3, v0
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v2, v6, vcc
+; GFX7-NEXT:    v_lshrrev_b32_e32 v0, v1, v0
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+  %vector = load <8 x i16>, <8 x i16> addrspace(1)* %ptr
+  %element = extractelement <8 x i16> %vector, i32 %idx
+  ret i16 %element
+}
+
+define amdgpu_ps i16 @extractelement_sgpr_v8i16_vgpr_idx(<8 x i16> addrspace(4)* inreg %ptr, i32 %idx) {
+; GCN-LABEL: extractelement_sgpr_v8i16_vgpr_idx:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[2:3], 0x0
+; GCN-NEXT:    v_lshrrev_b32_e32 v1, 1, v0
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v1
+; GCN-NEXT:    v_and_b32_e32 v0, 1, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 4, v0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v2, s0
+; GCN-NEXT:    v_mov_b32_e32 v3, s1
+; GCN-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; GCN-NEXT:    v_mov_b32_e32 v4, s2
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 2, v1
+; GCN-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; GCN-NEXT:    v_mov_b32_e32 v5, s3
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 3, v1
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v2, v5, vcc
+; GCN-NEXT:    v_lshrrev_b32_e32 v0, v0, v1
+; GCN-NEXT:    v_readfirstlane_b32 s0, v0
+; GCN-NEXT:    ; return to shader part epilog
+  %vector = load <8 x i16>, <8 x i16> addrspace(4)* %ptr
+  %element = extractelement <8 x i16> %vector, i32 %idx
+  ret i16 %element
+}
+
+define amdgpu_ps i16 @extractelement_sgpr_v8i16_idx0(<8 x i16> addrspace(4)* inreg %ptr) {
+; GCN-LABEL: extractelement_sgpr_v8i16_idx0:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[2:3], 0x0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    ; return to shader part epilog
+  %vector = load <8 x i16>, <8 x i16> addrspace(4)* %ptr
+  %element = extractelement <8 x i16> %vector, i32 0
+  ret i16 %element
+}
+
+define amdgpu_ps i16 @extractelement_sgpr_v8i16_idx1(<8 x i16> addrspace(4)* inreg %ptr) {
+; GCN-LABEL: extractelement_sgpr_v8i16_idx1:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[2:3], 0x0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_lshr_b32 s0, s0, 16
+; GCN-NEXT:    ; return to shader part epilog
+  %vector = load <8 x i16>, <8 x i16> addrspace(4)* %ptr
+  %element = extractelement <8 x i16> %vector, i32 1
+  ret i16 %element
+}
+
+define amdgpu_ps i16 @extractelement_sgpr_v8i16_idx2(<8 x i16> addrspace(4)* inreg %ptr) {
+; GCN-LABEL: extractelement_sgpr_v8i16_idx2:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[2:3], 0x0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_mov_b32 s0, s1
+; GCN-NEXT:    ; return to shader part epilog
+  %vector = load <8 x i16>, <8 x i16> addrspace(4)* %ptr
+  %element = extractelement <8 x i16> %vector, i32 2
+  ret i16 %element
+}
+
+define amdgpu_ps i16 @extractelement_sgpr_v8i16_idx3(<8 x i16> addrspace(4)* inreg %ptr) {
+; GCN-LABEL: extractelement_sgpr_v8i16_idx3:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[2:3], 0x0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_lshr_b32 s0, s1, 16
+; GCN-NEXT:    ; return to shader part epilog
+  %vector = load <8 x i16>, <8 x i16> addrspace(4)* %ptr
+  %element = extractelement <8 x i16> %vector, i32 3
+  ret i16 %element
+}
+
+define amdgpu_ps i16 @extractelement_sgpr_v8i16_idx4(<8 x i16> addrspace(4)* inreg %ptr) {
+; GCN-LABEL: extractelement_sgpr_v8i16_idx4:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[2:3], 0x0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_mov_b32 s0, s2
+; GCN-NEXT:    ; return to shader part epilog
+  %vector = load <8 x i16>, <8 x i16> addrspace(4)* %ptr
+  %element = extractelement <8 x i16> %vector, i32 4
+  ret i16 %element
+}
+
+define amdgpu_ps i16 @extractelement_sgpr_v8i16_idx5(<8 x i16> addrspace(4)* inreg %ptr) {
+; GCN-LABEL: extractelement_sgpr_v8i16_idx5:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[2:3], 0x0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_lshr_b32 s0, s2, 16
+; GCN-NEXT:    ; return to shader part epilog
+  %vector = load <8 x i16>, <8 x i16> addrspace(4)* %ptr
+  %element = extractelement <8 x i16> %vector, i32 5
+  ret i16 %element
+}
+
+define amdgpu_ps i16 @extractelement_sgpr_v8i16_idx6(<8 x i16> addrspace(4)* inreg %ptr) {
+; GCN-LABEL: extractelement_sgpr_v8i16_idx6:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[2:3], 0x0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_mov_b32 s0, s3
+; GCN-NEXT:    ; return to shader part epilog
+  %vector = load <8 x i16>, <8 x i16> addrspace(4)* %ptr
+  %element = extractelement <8 x i16> %vector, i32 6
+  ret i16 %element
+}
+
+define amdgpu_ps i16 @extractelement_sgpr_v8i16_idx7(<8 x i16> addrspace(4)* inreg %ptr) {
+; GCN-LABEL: extractelement_sgpr_v8i16_idx7:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[2:3], 0x0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_lshr_b32 s0, s3, 16
+; GCN-NEXT:    ; return to shader part epilog
+  %vector = load <8 x i16>, <8 x i16> addrspace(4)* %ptr
+  %element = extractelement <8 x i16> %vector, i32 7
+  ret i16 %element
+}
+
+define i16 @extractelement_vgpr_v8i16_idx0(<8 x i16> addrspace(1)* %ptr) {
+; GFX9-LABEL: extractelement_vgpr_v8i16_idx0:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dwordx4 v[0:3], v[0:1], off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: extractelement_vgpr_v8i16_idx0:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: extractelement_vgpr_v8i16_idx0:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    s_mov_b32 s6, 0
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:    buffer_load_dwordx4 v[0:3], v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+  %vector = load <8 x i16>, <8 x i16> addrspace(1)* %ptr
+  %element = extractelement <8 x i16> %vector, i32 0
+  ret i16 %element
+}
+
+define i16 @extractelement_vgpr_v8i16_idx1(<8 x i16> addrspace(1)* %ptr) {
+; GFX9-LABEL: extractelement_vgpr_v8i16_idx1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dwordx4 v[0:3], v[0:1], off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: extractelement_vgpr_v8i16_idx1:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: extractelement_vgpr_v8i16_idx1:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    s_mov_b32 s6, 0
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:    buffer_load_dwordx4 v[0:3], v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+  %vector = load <8 x i16>, <8 x i16> addrspace(1)* %ptr
+  %element = extractelement <8 x i16> %vector, i32 1
+  ret i16 %element
+}
+
+define i16 @extractelement_vgpr_v8i16_idx2(<8 x i16> addrspace(1)* %ptr) {
+; GFX9-LABEL: extractelement_vgpr_v8i16_idx2:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dwordx4 v[0:3], v[0:1], off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: extractelement_vgpr_v8i16_idx2:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v0, v1
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: extractelement_vgpr_v8i16_idx2:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    s_mov_b32 s6, 0
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:    buffer_load_dwordx4 v[0:3], v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, v1
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+  %vector = load <8 x i16>, <8 x i16> addrspace(1)* %ptr
+  %element = extractelement <8 x i16> %vector, i32 2
+  ret i16 %element
+}
+
+define i16 @extractelement_vgpr_v8i16_idx3(<8 x i16> addrspace(1)* %ptr) {
+; GFX9-LABEL: extractelement_vgpr_v8i16_idx3:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dwordx4 v[0:3], v[0:1], off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: extractelement_vgpr_v8i16_idx3:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_lshrrev_b32_e32 v0, 16, v1
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: extractelement_vgpr_v8i16_idx3:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    s_mov_b32 s6, 0
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:    buffer_load_dwordx4 v[0:3], v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_lshrrev_b32_e32 v0, 16, v1
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+  %vector = load <8 x i16>, <8 x i16> addrspace(1)* %ptr
+  %element = extractelement <8 x i16> %vector, i32 3
+  ret i16 %element
+}
+
+define i16 @extractelement_vgpr_v8i16_idx4(<8 x i16> addrspace(1)* %ptr) {
+; GFX9-LABEL: extractelement_vgpr_v8i16_idx4:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dwordx4 v[0:3], v[0:1], off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, v2
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: extractelement_vgpr_v8i16_idx4:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v0, v2
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: extractelement_vgpr_v8i16_idx4:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    s_mov_b32 s6, 0
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:    buffer_load_dwordx4 v[0:3], v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, v2
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+  %vector = load <8 x i16>, <8 x i16> addrspace(1)* %ptr
+  %element = extractelement <8 x i16> %vector, i32 4
+  ret i16 %element
+}
+
+define i16 @extractelement_vgpr_v8i16_idx5(<8 x i16> addrspace(1)* %ptr) {
+; GFX9-LABEL: extractelement_vgpr_v8i16_idx5:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dwordx4 v[0:3], v[0:1], off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v2
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: extractelement_vgpr_v8i16_idx5:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_lshrrev_b32_e32 v0, 16, v2
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: extractelement_vgpr_v8i16_idx5:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    s_mov_b32 s6, 0
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:    buffer_load_dwordx4 v[0:3], v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_lshrrev_b32_e32 v0, 16, v2
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+  %vector = load <8 x i16>, <8 x i16> addrspace(1)* %ptr
+  %element = extractelement <8 x i16> %vector, i32 5
+  ret i16 %element
+}
+
+define i16 @extractelement_vgpr_v8i16_idx6(<8 x i16> addrspace(1)* %ptr) {
+; GFX9-LABEL: extractelement_vgpr_v8i16_idx6:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dwordx4 v[0:3], v[0:1], off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, v3
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: extractelement_vgpr_v8i16_idx6:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v0, v3
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: extractelement_vgpr_v8i16_idx6:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    s_mov_b32 s6, 0
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:    buffer_load_dwordx4 v[0:3], v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, v3
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+  %vector = load <8 x i16>, <8 x i16> addrspace(1)* %ptr
+  %element = extractelement <8 x i16> %vector, i32 6
+  ret i16 %element
+}
+
+define i16 @extractelement_vgpr_v8i16_idx7(<8 x i16> addrspace(1)* %ptr) {
+; GFX9-LABEL: extractelement_vgpr_v8i16_idx7:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dwordx4 v[0:3], v[0:1], off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v3
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: extractelement_vgpr_v8i16_idx7:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_lshrrev_b32_e32 v0, 16, v3
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: extractelement_vgpr_v8i16_idx7:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    s_mov_b32 s6, 0
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:    buffer_load_dwordx4 v[0:3], v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_lshrrev_b32_e32 v0, 16, v3
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+  %vector = load <8 x i16>, <8 x i16> addrspace(1)* %ptr
+  %element = extractelement <8 x i16> %vector, i32 7
+  ret i16 %element
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.i8.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.i8.ll
new file mode 100644
index 0000000000000..95b4177abbcab
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.i8.ll
@@ -0,0 +1,3135 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX8 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=hawaii -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX7 %s
+
+define amdgpu_ps i8 @extractelement_sgpr_v4i8_sgpr_idx(<4 x i8> addrspace(4)* inreg %ptr, i32 inreg %idx) {
+; GCN-LABEL: extractelement_sgpr_v4i8_sgpr_idx:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dword s0, s[2:3], 0x0
+; GCN-NEXT:    s_movk_i32 s5, 0xff
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_lshr_b32 s1, s0, 8
+; GCN-NEXT:    s_and_b32 s1, s1, s5
+; GCN-NEXT:    s_lshr_b32 s2, s0, 16
+; GCN-NEXT:    s_lshr_b32 s3, s0, 24
+; GCN-NEXT:    s_and_b32 s0, s0, s5
+; GCN-NEXT:    s_lshl_b32 s1, s1, 8
+; GCN-NEXT:    s_or_b32 s0, s0, s1
+; GCN-NEXT:    s_and_b32 s1, s2, s5
+; GCN-NEXT:    s_lshl_b32 s1, s1, 16
+; GCN-NEXT:    s_or_b32 s0, s0, s1
+; GCN-NEXT:    s_and_b32 s1, s3, s5
+; GCN-NEXT:    s_lshl_b32 s1, s1, 24
+; GCN-NEXT:    s_or_b32 s0, s0, s1
+; GCN-NEXT:    s_and_b32 s1, s4, 3
+; GCN-NEXT:    s_lshl_b32 s1, s1, 3
+; GCN-NEXT:    s_lshr_b32 s0, s0, s1
+; GCN-NEXT:    ; return to shader part epilog
+  %vector = load <4 x i8>, <4 x i8> addrspace(4)* %ptr
+  %element = extractelement <4 x i8> %vector, i32 %idx
+  ret i8 %element
+}
+
+define amdgpu_ps i8 @extractelement_vgpr_v4i8_sgpr_idx(<4 x i8> addrspace(1)* %ptr, i32 inreg %idx) {
+; GFX9-LABEL: extractelement_vgpr_v4i8_sgpr_idx:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    global_load_dword v0, v[0:1], off
+; GFX9-NEXT:    s_mov_b32 s0, 8
+; GFX9-NEXT:    s_movk_i32 s1, 0xff
+; GFX9-NEXT:    s_and_b32 s2, s2, 3
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v1, 8, v0
+; GFX9-NEXT:    v_lshlrev_b32_sdwa v1, s0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_and_b32_sdwa v2, v0, s1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    v_and_b32_sdwa v3, v0, s1 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX9-NEXT:    v_and_or_b32 v0, v0, s1, v1
+; GFX9-NEXT:    v_or3_b32 v0, v0, v2, v3
+; GFX9-NEXT:    s_lshl_b32 s0, s2, 3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, s0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: extractelement_vgpr_v4i8_sgpr_idx:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    flat_load_dword v0, v[0:1]
+; GFX8-NEXT:    s_movk_i32 s0, 0xff
+; GFX8-NEXT:    v_mov_b32_e32 v1, 8
+; GFX8-NEXT:    v_mov_b32_e32 v2, s0
+; GFX8-NEXT:    s_and_b32 s0, s2, 3
+; GFX8-NEXT:    s_lshl_b32 s0, s0, 3
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_lshrrev_b32_e32 v3, 8, v0
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT:    v_and_b32_sdwa v4, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT:    v_and_b32_sdwa v2, v0, v2 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v4
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX8-NEXT:    v_lshrrev_b32_e32 v0, s0, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX7-LABEL: extractelement_vgpr_v4i8_sgpr_idx:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_mov_b32 s6, 0
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:    buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT:    s_movk_i32 s0, 0xff
+; GFX7-NEXT:    s_and_b32 s1, s2, 3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_lshrrev_b32_e32 v1, 8, v0
+; GFX7-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
+; GFX7-NEXT:    v_and_b32_e32 v1, s0, v1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v3, 24, v0
+; GFX7-NEXT:    v_and_b32_e32 v2, s0, v2
+; GFX7-NEXT:    v_and_b32_e32 v0, s0, v0
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX7-NEXT:    v_and_b32_e32 v3, s0, v3
+; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX7-NEXT:    v_lshlrev_b32_e32 v3, 24, v3
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v3
+; GFX7-NEXT:    s_lshl_b32 s0, s1, 3
+; GFX7-NEXT:    v_lshrrev_b32_e32 v0, s0, v0
+; GFX7-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX7-NEXT:    ; return to shader part epilog
+  %vector = load <4 x i8>, <4 x i8> addrspace(1)* %ptr
+  %element = extractelement <4 x i8> %vector, i32 %idx
+  ret i8 %element
+}
+
+define i8 @extractelement_vgpr_v4i8_vgpr_idx(<4 x i8> addrspace(1)* %ptr, i32 %idx) {
+; GFX9-LABEL: extractelement_vgpr_v4i8_vgpr_idx:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dword v0, v[0:1], off
+; GFX9-NEXT:    v_and_b32_e32 v1, 3, v2
+; GFX9-NEXT:    s_mov_b32 s4, 8
+; GFX9-NEXT:    s_movk_i32 s5, 0xff
+; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 3, v1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v2, 8, v0
+; GFX9-NEXT:    v_lshlrev_b32_sdwa v2, s4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_and_b32_sdwa v3, v0, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    v_and_b32_sdwa v4, v0, s5 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX9-NEXT:    v_and_or_b32 v0, v0, s5, v2
+; GFX9-NEXT:    v_or3_b32 v0, v0, v3, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, v1, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: extractelement_vgpr_v4i8_vgpr_idx:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    flat_load_dword v0, v[0:1]
+; GFX8-NEXT:    s_movk_i32 s4, 0xff
+; GFX8-NEXT:    v_mov_b32_e32 v1, 8
+; GFX8-NEXT:    v_mov_b32_e32 v3, s4
+; GFX8-NEXT:    v_and_b32_e32 v2, 3, v2
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_lshrrev_b32_e32 v4, 8, v0
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT:    v_and_b32_sdwa v5, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT:    v_and_b32_sdwa v3, v0, v3 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v5
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v3
+; GFX8-NEXT:    v_lshlrev_b32_e32 v1, 3, v2
+; GFX8-NEXT:    v_lshrrev_b32_e32 v0, v1, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: extractelement_vgpr_v4i8_vgpr_idx:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    s_mov_b32 s6, 0
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:    buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT:    v_and_b32_e32 v1, 3, v2
+; GFX7-NEXT:    s_movk_i32 s4, 0xff
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 3, v1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_lshrrev_b32_e32 v2, 8, v0
+; GFX7-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
+; GFX7-NEXT:    v_and_b32_e32 v2, s4, v2
+; GFX7-NEXT:    v_lshrrev_b32_e32 v4, 24, v0
+; GFX7-NEXT:    v_and_b32_e32 v3, s4, v3
+; GFX7-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
+; GFX7-NEXT:    v_and_b32_e32 v4, s4, v4
+; GFX7-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX7-NEXT:    v_lshlrev_b32_e32 v4, 24, v4
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v3
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v4
+; GFX7-NEXT:    v_lshrrev_b32_e32 v0, v1, v0
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+  %vector = load <4 x i8>, <4 x i8> addrspace(1)* %ptr
+  %element = extractelement <4 x i8> %vector, i32 %idx
+  ret i8 %element
+}
+
+define amdgpu_ps i8 @extractelement_sgpr_v4i8_vgpr_idx(<4 x i8> addrspace(4)* inreg %ptr, i32 %idx) {
+; GFX9-LABEL: extractelement_sgpr_v4i8_vgpr_idx:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dword s0, s[2:3], 0x0
+; GFX9-NEXT:    s_movk_i32 s4, 0xff
+; GFX9-NEXT:    v_and_b32_e32 v0, 3, v0
+; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_lshr_b32 s1, s0, 8
+; GFX9-NEXT:    s_and_b32 s1, s1, s4
+; GFX9-NEXT:    s_lshr_b32 s2, s0, 16
+; GFX9-NEXT:    s_lshr_b32 s3, s0, 24
+; GFX9-NEXT:    s_and_b32 s0, s0, s4
+; GFX9-NEXT:    s_lshl_b32 s1, s1, 8
+; GFX9-NEXT:    s_or_b32 s0, s0, s1
+; GFX9-NEXT:    s_and_b32 s1, s2, s4
+; GFX9-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX9-NEXT:    s_or_b32 s0, s0, s1
+; GFX9-NEXT:    s_and_b32 s1, s3, s4
+; GFX9-NEXT:    s_lshl_b32 s1, s1, 24
+; GFX9-NEXT:    s_or_b32 s0, s0, s1
+; GFX9-NEXT:    v_lshrrev_b32_e64 v0, v0, s0
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: extractelement_sgpr_v4i8_vgpr_idx:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_load_dword s0, s[2:3], 0x0
+; GFX8-NEXT:    s_movk_i32 s4, 0xff
+; GFX8-NEXT:    v_and_b32_e32 v0, 3, v0
+; GFX8-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_lshr_b32 s1, s0, 8
+; GFX8-NEXT:    s_and_b32 s1, s1, s4
+; GFX8-NEXT:    s_lshr_b32 s2, s0, 16
+; GFX8-NEXT:    s_lshr_b32 s3, s0, 24
+; GFX8-NEXT:    s_and_b32 s0, s0, s4
+; GFX8-NEXT:    s_lshl_b32 s1, s1, 8
+; GFX8-NEXT:    s_or_b32 s0, s0, s1
+; GFX8-NEXT:    s_and_b32 s1, s2, s4
+; GFX8-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX8-NEXT:    s_or_b32 s0, s0, s1
+; GFX8-NEXT:    s_and_b32 s1, s3, s4
+; GFX8-NEXT:    s_lshl_b32 s1, s1, 24
+; GFX8-NEXT:    s_or_b32 s0, s0, s1
+; GFX8-NEXT:    v_lshrrev_b32_e64 v0, v0, s0
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX7-LABEL: extractelement_sgpr_v4i8_vgpr_idx:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_load_dword s0, s[2:3], 0x0
+; GFX7-NEXT:    s_movk_i32 s4, 0xff
+; GFX7-NEXT:    v_and_b32_e32 v0, 3, v0
+; GFX7-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_lshr_b32 s1, s0, 8
+; GFX7-NEXT:    s_and_b32 s1, s1, s4
+; GFX7-NEXT:    s_lshr_b32 s2, s0, 16
+; GFX7-NEXT:    s_lshr_b32 s3, s0, 24
+; GFX7-NEXT:    s_and_b32 s0, s0, s4
+; GFX7-NEXT:    s_lshl_b32 s1, s1, 8
+; GFX7-NEXT:    s_or_b32 s0, s0, s1
+; GFX7-NEXT:    s_and_b32 s1, s2, s4
+; GFX7-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX7-NEXT:    s_or_b32 s0, s0, s1
+; GFX7-NEXT:    s_and_b32 s1, s3, s4
+; GFX7-NEXT:    s_lshl_b32 s1, s1, 24
+; GFX7-NEXT:    s_or_b32 s0, s0, s1
+; GFX7-NEXT:    v_lshr_b32_e32 v0, s0, v0
+; GFX7-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX7-NEXT:    ; return to shader part epilog
+  %vector = load <4 x i8>, <4 x i8> addrspace(4)* %ptr
+  %element = extractelement <4 x i8> %vector, i32 %idx
+  ret i8 %element
+}
+
+define amdgpu_ps i8 @extractelement_sgpr_v4i8_idx0(<4 x i8> addrspace(4)* inreg %ptr) {
+; GCN-LABEL: extractelement_sgpr_v4i8_idx0:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dword s1, s[2:3], 0x0
+; GCN-NEXT:    s_movk_i32 s0, 0xff
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_lshr_b32 s2, s1, 8
+; GCN-NEXT:    s_and_b32 s2, s2, s0
+; GCN-NEXT:    s_lshr_b32 s3, s1, 16
+; GCN-NEXT:    s_lshr_b32 s4, s1, 24
+; GCN-NEXT:    s_and_b32 s1, s1, s0
+; GCN-NEXT:    s_lshl_b32 s2, s2, 8
+; GCN-NEXT:    s_or_b32 s1, s1, s2
+; GCN-NEXT:    s_and_b32 s2, s3, s0
+; GCN-NEXT:    s_and_b32 s0, s4, s0
+; GCN-NEXT:    s_lshl_b32 s2, s2, 16
+; GCN-NEXT:    s_or_b32 s1, s1, s2
+; GCN-NEXT:    s_lshl_b32 s0, s0, 24
+; GCN-NEXT:    s_or_b32 s0, s1, s0
+; GCN-NEXT:    ; return to shader part epilog
+  %vector = load <4 x i8>, <4 x i8> addrspace(4)* %ptr
+  %element = extractelement <4 x i8> %vector, i32 0
+  ret i8 %element
+}
+
+define amdgpu_ps i8 @extractelement_sgpr_v4i8_idx1(<4 x i8> addrspace(4)* inreg %ptr) {
+; GCN-LABEL: extractelement_sgpr_v4i8_idx1:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dword s1, s[2:3], 0x0
+; GCN-NEXT:    s_movk_i32 s0, 0xff
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_lshr_b32 s2, s1, 8
+; GCN-NEXT:    s_and_b32 s2, s2, s0
+; GCN-NEXT:    s_lshr_b32 s3, s1, 16
+; GCN-NEXT:    s_lshr_b32 s4, s1, 24
+; GCN-NEXT:    s_and_b32 s1, s1, s0
+; GCN-NEXT:    s_lshl_b32 s2, s2, 8
+; GCN-NEXT:    s_or_b32 s1, s1, s2
+; GCN-NEXT:    s_and_b32 s2, s3, s0
+; GCN-NEXT:    s_and_b32 s0, s4, s0
+; GCN-NEXT:    s_lshl_b32 s2, s2, 16
+; GCN-NEXT:    s_or_b32 s1, s1, s2
+; GCN-NEXT:    s_lshl_b32 s0, s0, 24
+; GCN-NEXT:    s_or_b32 s0, s1, s0
+; GCN-NEXT:    s_lshr_b32 s0, s0, 8
+; GCN-NEXT:    ; return to shader part epilog
+  %vector = load <4 x i8>, <4 x i8> addrspace(4)* %ptr
+  %element = extractelement <4 x i8> %vector, i32 1
+  ret i8 %element
+}
+
+define amdgpu_ps i8 @extractelement_sgpr_v4i8_idx2(<4 x i8> addrspace(4)* inreg %ptr) {
+; GCN-LABEL: extractelement_sgpr_v4i8_idx2:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dword s1, s[2:3], 0x0
+; GCN-NEXT:    s_movk_i32 s0, 0xff
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_lshr_b32 s2, s1, 8
+; GCN-NEXT:    s_and_b32 s2, s2, s0
+; GCN-NEXT:    s_lshr_b32 s3, s1, 16
+; GCN-NEXT:    s_lshr_b32 s4, s1, 24
+; GCN-NEXT:    s_and_b32 s1, s1, s0
+; GCN-NEXT:    s_lshl_b32 s2, s2, 8
+; GCN-NEXT:    s_or_b32 s1, s1, s2
+; GCN-NEXT:    s_and_b32 s2, s3, s0
+; GCN-NEXT:    s_and_b32 s0, s4, s0
+; GCN-NEXT:    s_lshl_b32 s2, s2, 16
+; GCN-NEXT:    s_or_b32 s1, s1, s2
+; GCN-NEXT:    s_lshl_b32 s0, s0, 24
+; GCN-NEXT:    s_or_b32 s0, s1, s0
+; GCN-NEXT:    s_lshr_b32 s0, s0, 16
+; GCN-NEXT:    ; return to shader part epilog
+  %vector = load <4 x i8>, <4 x i8> addrspace(4)* %ptr
+  %element = extractelement <4 x i8> %vector, i32 2
+  ret i8 %element
+}
+
+define amdgpu_ps i8 @extractelement_sgpr_v4i8_idx3(<4 x i8> addrspace(4)* inreg %ptr) {
+; GCN-LABEL: extractelement_sgpr_v4i8_idx3:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dword s1, s[2:3], 0x0
+; GCN-NEXT:    s_movk_i32 s0, 0xff
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_lshr_b32 s2, s1, 8
+; GCN-NEXT:    s_and_b32 s2, s2, s0
+; GCN-NEXT:    s_lshr_b32 s3, s1, 16
+; GCN-NEXT:    s_lshr_b32 s4, s1, 24
+; GCN-NEXT:    s_and_b32 s1, s1, s0
+; GCN-NEXT:    s_lshl_b32 s2, s2, 8
+; GCN-NEXT:    s_or_b32 s1, s1, s2
+; GCN-NEXT:    s_and_b32 s2, s3, s0
+; GCN-NEXT:    s_and_b32 s0, s4, s0
+; GCN-NEXT:    s_lshl_b32 s2, s2, 16
+; GCN-NEXT:    s_or_b32 s1, s1, s2
+; GCN-NEXT:    s_lshl_b32 s0, s0, 24
+; GCN-NEXT:    s_or_b32 s0, s1, s0
+; GCN-NEXT:    s_lshr_b32 s0, s0, 24
+; GCN-NEXT:    ; return to shader part epilog
+  %vector = load <4 x i8>, <4 x i8> addrspace(4)* %ptr
+  %element = extractelement <4 x i8> %vector, i32 3
+  ret i8 %element
+}
+
+define i8 @extractelement_vgpr_v4i8_idx0(<4 x i8> addrspace(1)* %ptr) {
+; GFX9-LABEL: extractelement_vgpr_v4i8_idx0:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dword v0, v[0:1], off
+; GFX9-NEXT:    s_mov_b32 s4, 8
+; GFX9-NEXT:    s_movk_i32 s5, 0xff
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v1, 8, v0
+; GFX9-NEXT:    v_lshlrev_b32_sdwa v1, s4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_and_b32_sdwa v2, v0, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    v_and_or_b32 v1, v0, s5, v1
+; GFX9-NEXT:    v_and_b32_sdwa v0, v0, s5 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX9-NEXT:    v_or3_b32 v0, v1, v2, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: extractelement_vgpr_v4i8_idx0:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    flat_load_dword v0, v[0:1]
+; GFX8-NEXT:    s_movk_i32 s4, 0xff
+; GFX8-NEXT:    v_mov_b32_e32 v1, 8
+; GFX8-NEXT:    v_mov_b32_e32 v2, s4
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_lshrrev_b32_e32 v3, 8, v0
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT:    v_and_b32_sdwa v4, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v1, v1, v4
+; GFX8-NEXT:    v_and_b32_sdwa v0, v0, v2 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: extractelement_vgpr_v4i8_idx0:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    s_mov_b32 s6, 0
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:    buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT:    s_movk_i32 s4, 0xff
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_lshrrev_b32_e32 v1, 8, v0
+; GFX7-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
+; GFX7-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v3, 24, v0
+; GFX7-NEXT:    v_and_b32_e32 v2, s4, v2
+; GFX7-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX7-NEXT:    v_and_b32_e32 v3, s4, v3
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 24, v3
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+  %vector = load <4 x i8>, <4 x i8> addrspace(1)* %ptr
+  %element = extractelement <4 x i8> %vector, i32 0
+  ret i8 %element
+}
+
+define i8 @extractelement_vgpr_v4i8_idx1(<4 x i8> addrspace(1)* %ptr) {
+; GFX9-LABEL: extractelement_vgpr_v4i8_idx1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dword v0, v[0:1], off
+; GFX9-NEXT:    s_mov_b32 s4, 8
+; GFX9-NEXT:    s_movk_i32 s5, 0xff
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v1, 8, v0
+; GFX9-NEXT:    v_lshlrev_b32_sdwa v1, s4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_and_b32_sdwa v2, v0, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    v_and_or_b32 v1, v0, s5, v1
+; GFX9-NEXT:    v_and_b32_sdwa v0, v0, s5 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX9-NEXT:    v_or3_b32 v0, v1, v2, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 8, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: extractelement_vgpr_v4i8_idx1:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    flat_load_dword v0, v[0:1]
+; GFX8-NEXT:    s_movk_i32 s4, 0xff
+; GFX8-NEXT:    v_mov_b32_e32 v1, 8
+; GFX8-NEXT:    v_mov_b32_e32 v2, s4
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_lshrrev_b32_e32 v3, 8, v0
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT:    v_and_b32_sdwa v4, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v1, v1, v4
+; GFX8-NEXT:    v_and_b32_sdwa v0, v0, v2 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX8-NEXT:    v_lshrrev_b32_e32 v0, 8, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: extractelement_vgpr_v4i8_idx1:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    s_mov_b32 s6, 0
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:    buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT:    s_movk_i32 s4, 0xff
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_lshrrev_b32_e32 v1, 8, v0
+; GFX7-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
+; GFX7-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v3, 24, v0
+; GFX7-NEXT:    v_and_b32_e32 v2, s4, v2
+; GFX7-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX7-NEXT:    v_and_b32_e32 v3, s4, v3
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 24, v3
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v0, 8, v0
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+  %vector = load <4 x i8>, <4 x i8> addrspace(1)* %ptr
+  %element = extractelement <4 x i8> %vector, i32 1
+  ret i8 %element
+}
+
+define i8 @extractelement_vgpr_v4i8_idx2(<4 x i8> addrspace(1)* %ptr) {
+; GFX9-LABEL: extractelement_vgpr_v4i8_idx2:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dword v0, v[0:1], off
+; GFX9-NEXT:    s_mov_b32 s4, 8
+; GFX9-NEXT:    s_movk_i32 s5, 0xff
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v1, 8, v0
+; GFX9-NEXT:    v_lshlrev_b32_sdwa v1, s4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_and_b32_sdwa v2, v0, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    v_and_or_b32 v1, v0, s5, v1
+; GFX9-NEXT:    v_and_b32_sdwa v0, v0, s5 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX9-NEXT:    v_or3_b32 v0, v1, v2, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: extractelement_vgpr_v4i8_idx2:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    flat_load_dword v0, v[0:1]
+; GFX8-NEXT:    s_movk_i32 s4, 0xff
+; GFX8-NEXT:    v_mov_b32_e32 v1, 8
+; GFX8-NEXT:    v_mov_b32_e32 v2, s4
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_lshrrev_b32_e32 v3, 8, v0
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT:    v_and_b32_sdwa v4, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v1, v1, v4
+; GFX8-NEXT:    v_and_b32_sdwa v0, v0, v2 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX8-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: extractelement_vgpr_v4i8_idx2:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    s_mov_b32 s6, 0
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:    buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT:    s_movk_i32 s4, 0xff
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_lshrrev_b32_e32 v1, 8, v0
+; GFX7-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
+; GFX7-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v3, 24, v0
+; GFX7-NEXT:    v_and_b32_e32 v2, s4, v2
+; GFX7-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX7-NEXT:    v_and_b32_e32 v3, s4, v3
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 24, v3
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+  %vector = load <4 x i8>, <4 x i8> addrspace(1)* %ptr
+  %element = extractelement <4 x i8> %vector, i32 2
+  ret i8 %element
+}
+
+define i8 @extractelement_vgpr_v4i8_idx3(<4 x i8> addrspace(1)* %ptr) {
+; GFX9-LABEL: extractelement_vgpr_v4i8_idx3:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dword v0, v[0:1], off
+; GFX9-NEXT:    s_mov_b32 s4, 8
+; GFX9-NEXT:    s_movk_i32 s5, 0xff
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v1, 8, v0
+; GFX9-NEXT:    v_lshlrev_b32_sdwa v1, s4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_and_b32_sdwa v2, v0, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    v_and_or_b32 v1, v0, s5, v1
+; GFX9-NEXT:    v_and_b32_sdwa v0, v0, s5 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX9-NEXT:    v_or3_b32 v0, v1, v2, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 24, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: extractelement_vgpr_v4i8_idx3:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    flat_load_dword v0, v[0:1]
+; GFX8-NEXT:    s_movk_i32 s4, 0xff
+; GFX8-NEXT:    v_mov_b32_e32 v1, 8
+; GFX8-NEXT:    v_mov_b32_e32 v2, s4
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_lshrrev_b32_e32 v3, 8, v0
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT:    v_and_b32_sdwa v4, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v1, v1, v4
+; GFX8-NEXT:    v_and_b32_sdwa v0, v0, v2 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX8-NEXT:    v_lshrrev_b32_e32 v0, 24, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: extractelement_vgpr_v4i8_idx3:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    s_mov_b32 s6, 0
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:    buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT:    s_movk_i32 s4, 0xff
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_lshrrev_b32_e32 v1, 8, v0
+; GFX7-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
+; GFX7-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v3, 24, v0
+; GFX7-NEXT:    v_and_b32_e32 v2, s4, v2
+; GFX7-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX7-NEXT:    v_and_b32_e32 v3, s4, v3
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 24, v3
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v0, 24, v0
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+  %vector = load <4 x i8>, <4 x i8> addrspace(1)* %ptr
+  %element = extractelement <4 x i8> %vector, i32 3
+  ret i8 %element
+}
+
+define amdgpu_ps i8 @extractelement_sgpr_v8i8_sgpr_idx(<8 x i8> addrspace(4)* inreg %ptr, i32 inreg %idx) {
+; GCN-LABEL: extractelement_sgpr_v8i8_sgpr_idx:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[2:3], 0x0
+; GCN-NEXT:    s_movk_i32 s9, 0xff
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_lshr_b32 s2, s0, 8
+; GCN-NEXT:    s_and_b32 s2, s2, s9
+; GCN-NEXT:    s_lshr_b32 s3, s0, 16
+; GCN-NEXT:    s_lshr_b32 s5, s0, 24
+; GCN-NEXT:    s_and_b32 s0, s0, s9
+; GCN-NEXT:    s_lshl_b32 s2, s2, 8
+; GCN-NEXT:    s_or_b32 s0, s0, s2
+; GCN-NEXT:    s_and_b32 s2, s3, s9
+; GCN-NEXT:    s_lshl_b32 s2, s2, 16
+; GCN-NEXT:    s_or_b32 s0, s0, s2
+; GCN-NEXT:    s_and_b32 s2, s5, s9
+; GCN-NEXT:    s_lshl_b32 s2, s2, 24
+; GCN-NEXT:    s_lshr_b32 s6, s1, 8
+; GCN-NEXT:    s_or_b32 s0, s0, s2
+; GCN-NEXT:    s_and_b32 s2, s6, s9
+; GCN-NEXT:    s_lshr_b32 s7, s1, 16
+; GCN-NEXT:    s_lshr_b32 s8, s1, 24
+; GCN-NEXT:    s_and_b32 s1, s1, s9
+; GCN-NEXT:    s_lshl_b32 s2, s2, 8
+; GCN-NEXT:    s_or_b32 s1, s1, s2
+; GCN-NEXT:    s_and_b32 s2, s7, s9
+; GCN-NEXT:    s_lshl_b32 s2, s2, 16
+; GCN-NEXT:    s_or_b32 s1, s1, s2
+; GCN-NEXT:    s_and_b32 s2, s8, s9
+; GCN-NEXT:    s_lshl_b32 s2, s2, 24
+; GCN-NEXT:    s_or_b32 s1, s1, s2
+; GCN-NEXT:    s_lshr_b32 s2, s4, 2
+; GCN-NEXT:    s_cmp_eq_u32 s2, 1
+; GCN-NEXT:    s_cselect_b32 s0, s1, s0
+; GCN-NEXT:    s_and_b32 s1, s4, 3
+; GCN-NEXT:    s_lshl_b32 s1, s1, 3
+; GCN-NEXT:    s_lshr_b32 s0, s0, s1
+; GCN-NEXT:    ; return to shader part epilog
+  %vector = load <8 x i8>, <8 x i8> addrspace(4)* %ptr
+  %element = extractelement <8 x i8> %vector, i32 %idx
+  ret i8 %element
+}
+
+define amdgpu_ps i8 @extractelement_vgpr_v8i8_sgpr_idx(<8 x i8> addrspace(1)* %ptr, i32 inreg %idx) {
+; GFX9-LABEL: extractelement_vgpr_v8i8_sgpr_idx:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off
+; GFX9-NEXT:    s_mov_b32 s0, 8
+; GFX9-NEXT:    v_mov_b32_e32 v2, 8
+; GFX9-NEXT:    s_movk_i32 s1, 0xff
+; GFX9-NEXT:    s_lshr_b32 s3, s2, 2
+; GFX9-NEXT:    s_and_b32 s2, s2, 3
+; GFX9-NEXT:    v_cmp_eq_u32_e64 vcc, s3, 1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v3, 8, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v4, 8, v1
+; GFX9-NEXT:    v_lshlrev_b32_sdwa v3, s0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_lshlrev_b32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_and_b32_sdwa v5, v0, s1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    v_and_b32_sdwa v6, v0, s1 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX9-NEXT:    v_and_or_b32 v0, v0, s1, v3
+; GFX9-NEXT:    v_and_b32_sdwa v7, v1, s1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    v_and_b32_sdwa v8, v1, s1 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX9-NEXT:    v_and_or_b32 v1, v1, s1, v2
+; GFX9-NEXT:    v_or3_b32 v0, v0, v5, v6
+; GFX9-NEXT:    v_or3_b32 v1, v1, v7, v8
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX9-NEXT:    s_lshl_b32 s0, s2, 3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, s0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: extractelement_vgpr_v8i8_sgpr_idx:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; GFX8-NEXT:    s_movk_i32 s0, 0xff
+; GFX8-NEXT:    v_mov_b32_e32 v2, 8
+; GFX8-NEXT:    v_mov_b32_e32 v3, 8
+; GFX8-NEXT:    v_mov_b32_e32 v4, s0
+; GFX8-NEXT:    s_lshr_b32 s0, s2, 2
+; GFX8-NEXT:    s_and_b32 s1, s2, 3
+; GFX8-NEXT:    v_cmp_eq_u32_e64 vcc, s0, 1
+; GFX8-NEXT:    s_lshl_b32 s0, s1, 3
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_lshrrev_b32_e32 v5, 8, v0
+; GFX8-NEXT:    v_lshrrev_b32_e32 v6, 8, v1
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v3, v3, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT:    v_and_b32_sdwa v7, v0, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT:    v_and_b32_sdwa v8, v0, v4 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT:    v_and_b32_sdwa v9, v1, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT:    v_and_b32_sdwa v4, v1, v4 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v7
+; GFX8-NEXT:    v_or_b32_e32 v1, v1, v9
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v8
+; GFX8-NEXT:    v_or_b32_e32 v1, v1, v4
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX8-NEXT:    v_lshrrev_b32_e32 v0, s0, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX7-LABEL: extractelement_vgpr_v8i8_sgpr_idx:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_mov_b32 s6, 0
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:    buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT:    s_movk_i32 s0, 0xff
+; GFX7-NEXT:    s_lshr_b32 s1, s2, 2
+; GFX7-NEXT:    s_and_b32 s2, s2, 3
+; GFX7-NEXT:    v_cmp_eq_u32_e64 vcc, s1, 1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_lshrrev_b32_e32 v2, 8, v0
+; GFX7-NEXT:    v_lshrrev_b32_e32 v5, 8, v1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
+; GFX7-NEXT:    v_lshrrev_b32_e32 v6, 16, v1
+; GFX7-NEXT:    v_and_b32_e32 v2, s0, v2
+; GFX7-NEXT:    v_and_b32_e32 v5, s0, v5
+; GFX7-NEXT:    v_lshrrev_b32_e32 v4, 24, v0
+; GFX7-NEXT:    v_lshrrev_b32_e32 v7, 24, v1
+; GFX7-NEXT:    v_and_b32_e32 v3, s0, v3
+; GFX7-NEXT:    v_and_b32_e32 v6, s0, v6
+; GFX7-NEXT:    v_and_b32_e32 v0, s0, v0
+; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
+; GFX7-NEXT:    v_and_b32_e32 v1, s0, v1
+; GFX7-NEXT:    v_lshlrev_b32_e32 v5, 8, v5
+; GFX7-NEXT:    v_and_b32_e32 v4, s0, v4
+; GFX7-NEXT:    v_and_b32_e32 v7, s0, v7
+; GFX7-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX7-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GFX7-NEXT:    v_or_b32_e32 v1, v1, v5
+; GFX7-NEXT:    v_lshlrev_b32_e32 v4, 24, v4
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v3
+; GFX7-NEXT:    v_lshlrev_b32_e32 v7, 24, v7
+; GFX7-NEXT:    v_or_b32_e32 v1, v1, v6
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v4
+; GFX7-NEXT:    v_or_b32_e32 v1, v1, v7
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX7-NEXT:    s_lshl_b32 s0, s2, 3
+; GFX7-NEXT:    v_lshrrev_b32_e32 v0, s0, v0
+; GFX7-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX7-NEXT:    ; return to shader part epilog
+  %vector = load <8 x i8>, <8 x i8> addrspace(1)* %ptr
+  %element = extractelement <8 x i8> %vector, i32 %idx
+  ret i8 %element
+}
+
+define i8 @extractelement_vgpr_v8i8_vgpr_idx(<8 x i8> addrspace(1)* %ptr, i32 %idx) {
+; GFX9-LABEL: extractelement_vgpr_v8i8_vgpr_idx:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off
+; GFX9-NEXT:    s_mov_b32 s4, 8
+; GFX9-NEXT:    v_mov_b32_e32 v3, 8
+; GFX9-NEXT:    s_movk_i32 s5, 0xff
+; GFX9-NEXT:    v_lshrrev_b32_e32 v4, 2, v2
+; GFX9-NEXT:    v_and_b32_e32 v2, 3, v2
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v4
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v5, 8, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v6, 8, v1
+; GFX9-NEXT:    v_lshlrev_b32_sdwa v5, s4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_lshlrev_b32_sdwa v3, v3, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_and_b32_sdwa v7, v0, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    v_and_b32_sdwa v8, v0, s5 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX9-NEXT:    v_and_or_b32 v0, v0, s5, v5
+; GFX9-NEXT:    v_and_b32_sdwa v9, v1, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    v_and_b32_sdwa v10, v1, s5 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX9-NEXT:    v_and_or_b32 v1, v1, s5, v3
+; GFX9-NEXT:    v_or3_b32 v0, v0, v7, v8
+; GFX9-NEXT:    v_or3_b32 v1, v1, v9, v10
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 3, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, v1, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: extractelement_vgpr_v8i8_vgpr_idx:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; GFX8-NEXT:    s_movk_i32 s4, 0xff
+; GFX8-NEXT:    v_mov_b32_e32 v3, 8
+; GFX8-NEXT:    v_mov_b32_e32 v4, 8
+; GFX8-NEXT:    v_mov_b32_e32 v5, s4
+; GFX8-NEXT:    v_lshrrev_b32_e32 v6, 2, v2
+; GFX8-NEXT:    v_and_b32_e32 v2, 3, v2
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v6
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_lshrrev_b32_e32 v7, 8, v0
+; GFX8-NEXT:    v_lshrrev_b32_e32 v8, 8, v1
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v3, v3, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v4, v4, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT:    v_and_b32_sdwa v9, v0, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT:    v_and_b32_sdwa v10, v0, v5 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT:    v_and_b32_sdwa v11, v1, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT:    v_and_b32_sdwa v5, v1, v5 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v9
+; GFX8-NEXT:    v_or_b32_e32 v1, v1, v11
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v10
+; GFX8-NEXT:    v_or_b32_e32 v1, v1, v5
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX8-NEXT:    v_lshlrev_b32_e32 v1, 3, v2
+; GFX8-NEXT:    v_lshrrev_b32_e32 v0, v1, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: extractelement_vgpr_v8i8_vgpr_idx:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    s_mov_b32 s6, 0
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:    buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT:    s_movk_i32 s4, 0xff
+; GFX7-NEXT:    v_lshrrev_b32_e32 v3, 2, v2
+; GFX7-NEXT:    v_and_b32_e32 v2, 3, v2
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_lshrrev_b32_e32 v4, 8, v0
+; GFX7-NEXT:    v_lshrrev_b32_e32 v7, 8, v1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v5, 16, v0
+; GFX7-NEXT:    v_lshrrev_b32_e32 v8, 16, v1
+; GFX7-NEXT:    v_and_b32_e32 v4, s4, v4
+; GFX7-NEXT:    v_and_b32_e32 v7, s4, v7
+; GFX7-NEXT:    v_lshrrev_b32_e32 v6, 24, v0
+; GFX7-NEXT:    v_lshrrev_b32_e32 v9, 24, v1
+; GFX7-NEXT:    v_and_b32_e32 v5, s4, v5
+; GFX7-NEXT:    v_and_b32_e32 v8, s4, v8
+; GFX7-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX7-NEXT:    v_lshlrev_b32_e32 v4, 8, v4
+; GFX7-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX7-NEXT:    v_lshlrev_b32_e32 v7, 8, v7
+; GFX7-NEXT:    v_and_b32_e32 v6, s4, v6
+; GFX7-NEXT:    v_and_b32_e32 v9, s4, v9
+; GFX7-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v4
+; GFX7-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GFX7-NEXT:    v_or_b32_e32 v1, v1, v7
+; GFX7-NEXT:    v_lshlrev_b32_e32 v6, 24, v6
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v5
+; GFX7-NEXT:    v_lshlrev_b32_e32 v9, 24, v9
+; GFX7-NEXT:    v_or_b32_e32 v1, v1, v8
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v6
+; GFX7-NEXT:    v_or_b32_e32 v1, v1, v9
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 3, v2
+; GFX7-NEXT:    v_lshrrev_b32_e32 v0, v1, v0
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+  %vector = load <8 x i8>, <8 x i8> addrspace(1)* %ptr
+  %element = extractelement <8 x i8> %vector, i32 %idx
+  ret i8 %element
+}
+
+define amdgpu_ps i8 @extractelement_sgpr_v8i8_vgpr_idx(<8 x i8> addrspace(4)* inreg %ptr, i32 %idx) {
+; GCN-LABEL: extractelement_sgpr_v8i8_vgpr_idx:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[2:3], 0x0
+; GCN-NEXT:    s_movk_i32 s8, 0xff
+; GCN-NEXT:    v_lshrrev_b32_e32 v1, 2, v0
+; GCN-NEXT:    v_and_b32_e32 v0, 3, v0
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_lshr_b32 s2, s0, 8
+; GCN-NEXT:    s_and_b32 s2, s2, s8
+; GCN-NEXT:    s_lshr_b32 s3, s0, 16
+; GCN-NEXT:    s_lshr_b32 s4, s0, 24
+; GCN-NEXT:    s_and_b32 s0, s0, s8
+; GCN-NEXT:    s_lshl_b32 s2, s2, 8
+; GCN-NEXT:    s_or_b32 s0, s0, s2
+; GCN-NEXT:    s_and_b32 s2, s3, s8
+; GCN-NEXT:    s_lshl_b32 s2, s2, 16
+; GCN-NEXT:    s_or_b32 s0, s0, s2
+; GCN-NEXT:    s_and_b32 s2, s4, s8
+; GCN-NEXT:    s_lshl_b32 s2, s2, 24
+; GCN-NEXT:    s_lshr_b32 s5, s1, 8
+; GCN-NEXT:    s_or_b32 s0, s0, s2
+; GCN-NEXT:    s_and_b32 s2, s5, s8
+; GCN-NEXT:    s_lshr_b32 s6, s1, 16
+; GCN-NEXT:    s_lshr_b32 s7, s1, 24
+; GCN-NEXT:    s_and_b32 s1, s1, s8
+; GCN-NEXT:    s_lshl_b32 s2, s2, 8
+; GCN-NEXT:    s_or_b32 s1, s1, s2
+; GCN-NEXT:    s_and_b32 s2, s6, s8
+; GCN-NEXT:    s_lshl_b32 s2, s2, 16
+; GCN-NEXT:    s_or_b32 s1, s1, s2
+; GCN-NEXT:    s_and_b32 s2, s7, s8
+; GCN-NEXT:    s_lshl_b32 s2, s2, 24
+; GCN-NEXT:    s_or_b32 s1, s1, s2
+; GCN-NEXT:    v_mov_b32_e32 v2, s0
+; GCN-NEXT:    v_mov_b32_e32 v3, s1
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v2, v3, vcc
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
+; GCN-NEXT:    v_lshrrev_b32_e32 v0, v0, v1
+; GCN-NEXT:    v_readfirstlane_b32 s0, v0
+; GCN-NEXT:    ; return to shader part epilog
+  %vector = load <8 x i8>, <8 x i8> addrspace(4)* %ptr
+  %element = extractelement <8 x i8> %vector, i32 %idx
+  ret i8 %element
+}
+
+define amdgpu_ps i8 @extractelement_sgpr_v8i8_idx0(<8 x i8> addrspace(4)* inreg %ptr) {
+; GCN-LABEL: extractelement_sgpr_v8i8_idx0:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[2:3], 0x0
+; GCN-NEXT:    s_movk_i32 s4, 0xff
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_lshr_b32 s1, s0, 8
+; GCN-NEXT:    s_and_b32 s1, s1, s4
+; GCN-NEXT:    s_lshr_b32 s2, s0, 16
+; GCN-NEXT:    s_lshr_b32 s3, s0, 24
+; GCN-NEXT:    s_and_b32 s0, s0, s4
+; GCN-NEXT:    s_lshl_b32 s1, s1, 8
+; GCN-NEXT:    s_or_b32 s0, s0, s1
+; GCN-NEXT:    s_and_b32 s1, s2, s4
+; GCN-NEXT:    s_lshl_b32 s1, s1, 16
+; GCN-NEXT:    s_or_b32 s0, s0, s1
+; GCN-NEXT:    s_and_b32 s1, s3, s4
+; GCN-NEXT:    s_lshl_b32 s1, s1, 24
+; GCN-NEXT:    s_or_b32 s0, s0, s1
+; GCN-NEXT:    ; return to shader part epilog
+  %vector = load <8 x i8>, <8 x i8> addrspace(4)* %ptr
+  %element = extractelement <8 x i8> %vector, i32 0
+  ret i8 %element
+}
+
+define amdgpu_ps i8 @extractelement_sgpr_v8i8_idx1(<8 x i8> addrspace(4)* inreg %ptr) {
+; GCN-LABEL: extractelement_sgpr_v8i8_idx1:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[2:3], 0x0
+; GCN-NEXT:    s_movk_i32 s4, 0xff
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_lshr_b32 s1, s0, 8
+; GCN-NEXT:    s_and_b32 s1, s1, s4
+; GCN-NEXT:    s_lshr_b32 s2, s0, 16
+; GCN-NEXT:    s_lshr_b32 s3, s0, 24
+; GCN-NEXT:    s_and_b32 s0, s0, s4
+; GCN-NEXT:    s_lshl_b32 s1, s1, 8
+; GCN-NEXT:    s_or_b32 s0, s0, s1
+; GCN-NEXT:    s_and_b32 s1, s2, s4
+; GCN-NEXT:    s_lshl_b32 s1, s1, 16
+; GCN-NEXT:    s_or_b32 s0, s0, s1
+; GCN-NEXT:    s_and_b32 s1, s3, s4
+; GCN-NEXT:    s_lshl_b32 s1, s1, 24
+; GCN-NEXT:    s_or_b32 s0, s0, s1
+; GCN-NEXT:    s_lshr_b32 s0, s0, 8
+; GCN-NEXT:    ; return to shader part epilog
+  %vector = load <8 x i8>, <8 x i8> addrspace(4)* %ptr
+  %element = extractelement <8 x i8> %vector, i32 1
+  ret i8 %element
+}
+
+define amdgpu_ps i8 @extractelement_sgpr_v8i8_idx2(<8 x i8> addrspace(4)* inreg %ptr) {
+; GCN-LABEL: extractelement_sgpr_v8i8_idx2:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[2:3], 0x0
+; GCN-NEXT:    s_movk_i32 s4, 0xff
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_lshr_b32 s1, s0, 8
+; GCN-NEXT:    s_and_b32 s1, s1, s4
+; GCN-NEXT:    s_lshr_b32 s2, s0, 16
+; GCN-NEXT:    s_lshr_b32 s3, s0, 24
+; GCN-NEXT:    s_and_b32 s0, s0, s4
+; GCN-NEXT:    s_lshl_b32 s1, s1, 8
+; GCN-NEXT:    s_or_b32 s0, s0, s1
+; GCN-NEXT:    s_and_b32 s1, s2, s4
+; GCN-NEXT:    s_lshl_b32 s1, s1, 16
+; GCN-NEXT:    s_or_b32 s0, s0, s1
+; GCN-NEXT:    s_and_b32 s1, s3, s4
+; GCN-NEXT:    s_lshl_b32 s1, s1, 24
+; GCN-NEXT:    s_or_b32 s0, s0, s1
+; GCN-NEXT:    s_lshr_b32 s0, s0, 16
+; GCN-NEXT:    ; return to shader part epilog
+  %vector = load <8 x i8>, <8 x i8> addrspace(4)* %ptr
+  %element = extractelement <8 x i8> %vector, i32 2
+  ret i8 %element
+}
+
+define amdgpu_ps i8 @extractelement_sgpr_v8i8_idx3(<8 x i8> addrspace(4)* inreg %ptr) {
+; GCN-LABEL: extractelement_sgpr_v8i8_idx3:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[2:3], 0x0
+; GCN-NEXT:    s_movk_i32 s4, 0xff
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_lshr_b32 s1, s0, 8
+; GCN-NEXT:    s_and_b32 s1, s1, s4
+; GCN-NEXT:    s_lshr_b32 s2, s0, 16
+; GCN-NEXT:    s_lshr_b32 s3, s0, 24
+; GCN-NEXT:    s_and_b32 s0, s0, s4
+; GCN-NEXT:    s_lshl_b32 s1, s1, 8
+; GCN-NEXT:    s_or_b32 s0, s0, s1
+; GCN-NEXT:    s_and_b32 s1, s2, s4
+; GCN-NEXT:    s_lshl_b32 s1, s1, 16
+; GCN-NEXT:    s_or_b32 s0, s0, s1
+; GCN-NEXT:    s_and_b32 s1, s3, s4
+; GCN-NEXT:    s_lshl_b32 s1, s1, 24
+; GCN-NEXT:    s_or_b32 s0, s0, s1
+; GCN-NEXT:    s_lshr_b32 s0, s0, 24
+; GCN-NEXT:    ; return to shader part epilog
+  %vector = load <8 x i8>, <8 x i8> addrspace(4)* %ptr
+  %element = extractelement <8 x i8> %vector, i32 3
+  ret i8 %element
+}
+
+define amdgpu_ps i8 @extractelement_sgpr_v8i8_idx4(<8 x i8> addrspace(4)* inreg %ptr) {
+; GCN-LABEL: extractelement_sgpr_v8i8_idx4:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[2:3], 0x0
+; GCN-NEXT:    s_movk_i32 s4, 0xff
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_lshr_b32 s0, s1, 8
+; GCN-NEXT:    s_and_b32 s0, s0, s4
+; GCN-NEXT:    s_lshr_b32 s2, s1, 16
+; GCN-NEXT:    s_lshr_b32 s3, s1, 24
+; GCN-NEXT:    s_and_b32 s1, s1, s4
+; GCN-NEXT:    s_lshl_b32 s0, s0, 8
+; GCN-NEXT:    s_or_b32 s0, s1, s0
+; GCN-NEXT:    s_and_b32 s1, s2, s4
+; GCN-NEXT:    s_lshl_b32 s1, s1, 16
+; GCN-NEXT:    s_or_b32 s0, s0, s1
+; GCN-NEXT:    s_and_b32 s1, s3, s4
+; GCN-NEXT:    s_lshl_b32 s1, s1, 24
+; GCN-NEXT:    s_or_b32 s0, s0, s1
+; GCN-NEXT:    ; return to shader part epilog
+  %vector = load <8 x i8>, <8 x i8> addrspace(4)* %ptr
+  %element = extractelement <8 x i8> %vector, i32 4
+  ret i8 %element
+}
+
+define amdgpu_ps i8 @extractelement_sgpr_v8i8_idx5(<8 x i8> addrspace(4)* inreg %ptr) {
+; GCN-LABEL: extractelement_sgpr_v8i8_idx5:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[2:3], 0x0
+; GCN-NEXT:    s_movk_i32 s4, 0xff
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_lshr_b32 s0, s1, 8
+; GCN-NEXT:    s_and_b32 s0, s0, s4
+; GCN-NEXT:    s_lshr_b32 s2, s1, 16
+; GCN-NEXT:    s_lshr_b32 s3, s1, 24
+; GCN-NEXT:    s_and_b32 s1, s1, s4
+; GCN-NEXT:    s_lshl_b32 s0, s0, 8
+; GCN-NEXT:    s_or_b32 s0, s1, s0
+; GCN-NEXT:    s_and_b32 s1, s2, s4
+; GCN-NEXT:    s_lshl_b32 s1, s1, 16
+; GCN-NEXT:    s_or_b32 s0, s0, s1
+; GCN-NEXT:    s_and_b32 s1, s3, s4
+; GCN-NEXT:    s_lshl_b32 s1, s1, 24
+; GCN-NEXT:    s_or_b32 s0, s0, s1
+; GCN-NEXT:    s_lshr_b32 s0, s0, 8
+; GCN-NEXT:    ; return to shader part epilog
+  %vector = load <8 x i8>, <8 x i8> addrspace(4)* %ptr
+  %element = extractelement <8 x i8> %vector, i32 5
+  ret i8 %element
+}
+
+define amdgpu_ps i8 @extractelement_sgpr_v8i8_idx6(<8 x i8> addrspace(4)* inreg %ptr) {
+; GCN-LABEL: extractelement_sgpr_v8i8_idx6:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[2:3], 0x0
+; GCN-NEXT:    s_movk_i32 s4, 0xff
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_lshr_b32 s0, s1, 8
+; GCN-NEXT:    s_and_b32 s0, s0, s4
+; GCN-NEXT:    s_lshr_b32 s2, s1, 16
+; GCN-NEXT:    s_lshr_b32 s3, s1, 24
+; GCN-NEXT:    s_and_b32 s1, s1, s4
+; GCN-NEXT:    s_lshl_b32 s0, s0, 8
+; GCN-NEXT:    s_or_b32 s0, s1, s0
+; GCN-NEXT:    s_and_b32 s1, s2, s4
+; GCN-NEXT:    s_lshl_b32 s1, s1, 16
+; GCN-NEXT:    s_or_b32 s0, s0, s1
+; GCN-NEXT:    s_and_b32 s1, s3, s4
+; GCN-NEXT:    s_lshl_b32 s1, s1, 24
+; GCN-NEXT:    s_or_b32 s0, s0, s1
+; GCN-NEXT:    s_lshr_b32 s0, s0, 16
+; GCN-NEXT:    ; return to shader part epilog
+  %vector = load <8 x i8>, <8 x i8> addrspace(4)* %ptr
+  %element = extractelement <8 x i8> %vector, i32 6
+  ret i8 %element
+}
+
+define amdgpu_ps i8 @extractelement_sgpr_v8i8_idx7(<8 x i8> addrspace(4)* inreg %ptr) {
+; GCN-LABEL: extractelement_sgpr_v8i8_idx7:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[2:3], 0x0
+; GCN-NEXT:    s_movk_i32 s4, 0xff
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_lshr_b32 s0, s1, 8
+; GCN-NEXT:    s_and_b32 s0, s0, s4
+; GCN-NEXT:    s_lshr_b32 s2, s1, 16
+; GCN-NEXT:    s_lshr_b32 s3, s1, 24
+; GCN-NEXT:    s_and_b32 s1, s1, s4
+; GCN-NEXT:    s_lshl_b32 s0, s0, 8
+; GCN-NEXT:    s_or_b32 s0, s1, s0
+; GCN-NEXT:    s_and_b32 s1, s2, s4
+; GCN-NEXT:    s_lshl_b32 s1, s1, 16
+; GCN-NEXT:    s_or_b32 s0, s0, s1
+; GCN-NEXT:    s_and_b32 s1, s3, s4
+; GCN-NEXT:    s_lshl_b32 s1, s1, 24
+; GCN-NEXT:    s_or_b32 s0, s0, s1
+; GCN-NEXT:    s_lshr_b32 s0, s0, 24
+; GCN-NEXT:    ; return to shader part epilog
+  %vector = load <8 x i8>, <8 x i8> addrspace(4)* %ptr
+  %element = extractelement <8 x i8> %vector, i32 7
+  ret i8 %element
+}
+
+define i8 @extractelement_vgpr_v8i8_idx0(<8 x i8> addrspace(1)* %ptr) {
+; GFX9-LABEL: extractelement_vgpr_v8i8_idx0:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off
+; GFX9-NEXT:    s_mov_b32 s4, 8
+; GFX9-NEXT:    s_movk_i32 s5, 0xff
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v1, 8, v0
+; GFX9-NEXT:    v_lshlrev_b32_sdwa v1, s4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_and_b32_sdwa v2, v0, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    v_and_or_b32 v1, v0, s5, v1
+; GFX9-NEXT:    v_and_b32_sdwa v0, v0, s5 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX9-NEXT:    v_or3_b32 v0, v1, v2, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: extractelement_vgpr_v8i8_idx0:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; GFX8-NEXT:    s_movk_i32 s4, 0xff
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v1, 8
+; GFX8-NEXT:    v_mov_b32_e32 v2, s4
+; GFX8-NEXT:    v_lshrrev_b32_e32 v3, 8, v0
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT:    v_and_b32_sdwa v4, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v1, v1, v4
+; GFX8-NEXT:    v_and_b32_sdwa v0, v0, v2 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: extractelement_vgpr_v8i8_idx0:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    s_mov_b32 s6, 0
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:    buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT:    s_movk_i32 s4, 0xff
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_lshrrev_b32_e32 v1, 8, v0
+; GFX7-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
+; GFX7-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v3, 24, v0
+; GFX7-NEXT:    v_and_b32_e32 v2, s4, v2
+; GFX7-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX7-NEXT:    v_and_b32_e32 v3, s4, v3
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 24, v3
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+  %vector = load <8 x i8>, <8 x i8> addrspace(1)* %ptr
+  %element = extractelement <8 x i8> %vector, i32 0
+  ret i8 %element
+}
+
+define i8 @extractelement_vgpr_v8i8_idx1(<8 x i8> addrspace(1)* %ptr) {
+; GFX9-LABEL: extractelement_vgpr_v8i8_idx1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off
+; GFX9-NEXT:    s_mov_b32 s4, 8
+; GFX9-NEXT:    s_movk_i32 s5, 0xff
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v1, 8, v0
+; GFX9-NEXT:    v_lshlrev_b32_sdwa v1, s4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_and_b32_sdwa v2, v0, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    v_and_or_b32 v1, v0, s5, v1
+; GFX9-NEXT:    v_and_b32_sdwa v0, v0, s5 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX9-NEXT:    v_or3_b32 v0, v1, v2, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 8, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: extractelement_vgpr_v8i8_idx1:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; GFX8-NEXT:    s_movk_i32 s4, 0xff
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v1, 8
+; GFX8-NEXT:    v_mov_b32_e32 v2, s4
+; GFX8-NEXT:    v_lshrrev_b32_e32 v3, 8, v0
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT:    v_and_b32_sdwa v4, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v1, v1, v4
+; GFX8-NEXT:    v_and_b32_sdwa v0, v0, v2 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX8-NEXT:    v_lshrrev_b32_e32 v0, 8, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: extractelement_vgpr_v8i8_idx1:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    s_mov_b32 s6, 0
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:    buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT:    s_movk_i32 s4, 0xff
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_lshrrev_b32_e32 v1, 8, v0
+; GFX7-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
+; GFX7-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v3, 24, v0
+; GFX7-NEXT:    v_and_b32_e32 v2, s4, v2
+; GFX7-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX7-NEXT:    v_and_b32_e32 v3, s4, v3
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 24, v3
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v0, 8, v0
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+  %vector = load <8 x i8>, <8 x i8> addrspace(1)* %ptr
+  %element = extractelement <8 x i8> %vector, i32 1
+  ret i8 %element
+}
+
+define i8 @extractelement_vgpr_v8i8_idx2(<8 x i8> addrspace(1)* %ptr) {
+; GFX9-LABEL: extractelement_vgpr_v8i8_idx2:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off
+; GFX9-NEXT:    s_mov_b32 s4, 8
+; GFX9-NEXT:    s_movk_i32 s5, 0xff
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v1, 8, v0
+; GFX9-NEXT:    v_lshlrev_b32_sdwa v1, s4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_and_b32_sdwa v2, v0, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    v_and_or_b32 v1, v0, s5, v1
+; GFX9-NEXT:    v_and_b32_sdwa v0, v0, s5 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX9-NEXT:    v_or3_b32 v0, v1, v2, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: extractelement_vgpr_v8i8_idx2:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; GFX8-NEXT:    s_movk_i32 s4, 0xff
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v1, 8
+; GFX8-NEXT:    v_mov_b32_e32 v2, s4
+; GFX8-NEXT:    v_lshrrev_b32_e32 v3, 8, v0
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT:    v_and_b32_sdwa v4, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v1, v1, v4
+; GFX8-NEXT:    v_and_b32_sdwa v0, v0, v2 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX8-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: extractelement_vgpr_v8i8_idx2:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    s_mov_b32 s6, 0
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:    buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT:    s_movk_i32 s4, 0xff
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_lshrrev_b32_e32 v1, 8, v0
+; GFX7-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
+; GFX7-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v3, 24, v0
+; GFX7-NEXT:    v_and_b32_e32 v2, s4, v2
+; GFX7-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX7-NEXT:    v_and_b32_e32 v3, s4, v3
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 24, v3
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+  %vector = load <8 x i8>, <8 x i8> addrspace(1)* %ptr
+  %element = extractelement <8 x i8> %vector, i32 2
+  ret i8 %element
+}
+
+define i8 @extractelement_vgpr_v8i8_idx3(<8 x i8> addrspace(1)* %ptr) {
+; GFX9-LABEL: extractelement_vgpr_v8i8_idx3:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off
+; GFX9-NEXT:    s_mov_b32 s4, 8
+; GFX9-NEXT:    s_movk_i32 s5, 0xff
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v1, 8, v0
+; GFX9-NEXT:    v_lshlrev_b32_sdwa v1, s4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_and_b32_sdwa v2, v0, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    v_and_or_b32 v1, v0, s5, v1
+; GFX9-NEXT:    v_and_b32_sdwa v0, v0, s5 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX9-NEXT:    v_or3_b32 v0, v1, v2, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 24, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: extractelement_vgpr_v8i8_idx3:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; GFX8-NEXT:    s_movk_i32 s4, 0xff
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v1, 8
+; GFX8-NEXT:    v_mov_b32_e32 v2, s4
+; GFX8-NEXT:    v_lshrrev_b32_e32 v3, 8, v0
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT:    v_and_b32_sdwa v4, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v1, v1, v4
+; GFX8-NEXT:    v_and_b32_sdwa v0, v0, v2 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX8-NEXT:    v_lshrrev_b32_e32 v0, 24, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: extractelement_vgpr_v8i8_idx3:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    s_mov_b32 s6, 0
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:    buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT:    s_movk_i32 s4, 0xff
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_lshrrev_b32_e32 v1, 8, v0
+; GFX7-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
+; GFX7-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v3, 24, v0
+; GFX7-NEXT:    v_and_b32_e32 v2, s4, v2
+; GFX7-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX7-NEXT:    v_and_b32_e32 v3, s4, v3
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 24, v3
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v0, 24, v0
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+  %vector = load <8 x i8>, <8 x i8> addrspace(1)* %ptr
+  %element = extractelement <8 x i8> %vector, i32 3
+  ret i8 %element
+}
+
+define i8 @extractelement_vgpr_v8i8_idx4(<8 x i8> addrspace(1)* %ptr) {
+; GFX9-LABEL: extractelement_vgpr_v8i8_idx4:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off
+; GFX9-NEXT:    s_mov_b32 s4, 8
+; GFX9-NEXT:    s_movk_i32 s5, 0xff
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 8, v1
+; GFX9-NEXT:    v_lshlrev_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_and_b32_sdwa v2, v1, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    v_and_or_b32 v0, v1, s5, v0
+; GFX9-NEXT:    v_and_b32_sdwa v1, v1, s5 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX9-NEXT:    v_or3_b32 v0, v0, v2, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: extractelement_vgpr_v8i8_idx4:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; GFX8-NEXT:    s_movk_i32 s4, 0xff
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v0, 8
+; GFX8-NEXT:    v_mov_b32_e32 v2, s4
+; GFX8-NEXT:    v_lshrrev_b32_e32 v3, 8, v1
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT:    v_and_b32_sdwa v4, v1, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v4
+; GFX8-NEXT:    v_and_b32_sdwa v1, v1, v2 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: extractelement_vgpr_v8i8_idx4:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    s_mov_b32 s6, 0
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:    buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT:    s_movk_i32 s4, 0xff
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_lshrrev_b32_e32 v0, 8, v1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v2, 16, v1
+; GFX7-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX7-NEXT:    v_lshrrev_b32_e32 v3, 24, v1
+; GFX7-NEXT:    v_and_b32_e32 v2, s4, v2
+; GFX7-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX7-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
+; GFX7-NEXT:    v_and_b32_e32 v3, s4, v3
+; GFX7-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 24, v3
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+  %vector = load <8 x i8>, <8 x i8> addrspace(1)* %ptr
+  %element = extractelement <8 x i8> %vector, i32 4
+  ret i8 %element
+}
+
+define i8 @extractelement_vgpr_v8i8_idx5(<8 x i8> addrspace(1)* %ptr) {
+; GFX9-LABEL: extractelement_vgpr_v8i8_idx5:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off
+; GFX9-NEXT:    s_mov_b32 s4, 8
+; GFX9-NEXT:    s_movk_i32 s5, 0xff
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 8, v1
+; GFX9-NEXT:    v_lshlrev_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_and_b32_sdwa v2, v1, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    v_and_or_b32 v0, v1, s5, v0
+; GFX9-NEXT:    v_and_b32_sdwa v1, v1, s5 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX9-NEXT:    v_or3_b32 v0, v0, v2, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 8, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: extractelement_vgpr_v8i8_idx5:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; GFX8-NEXT:    s_movk_i32 s4, 0xff
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v0, 8
+; GFX8-NEXT:    v_mov_b32_e32 v2, s4
+; GFX8-NEXT:    v_lshrrev_b32_e32 v3, 8, v1
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT:    v_and_b32_sdwa v4, v1, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v4
+; GFX8-NEXT:    v_and_b32_sdwa v1, v1, v2 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    v_lshrrev_b32_e32 v0, 8, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: extractelement_vgpr_v8i8_idx5:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    s_mov_b32 s6, 0
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:    buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT:    s_movk_i32 s4, 0xff
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_lshrrev_b32_e32 v0, 8, v1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v2, 16, v1
+; GFX7-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX7-NEXT:    v_lshrrev_b32_e32 v3, 24, v1
+; GFX7-NEXT:    v_and_b32_e32 v2, s4, v2
+; GFX7-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX7-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
+; GFX7-NEXT:    v_and_b32_e32 v3, s4, v3
+; GFX7-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 24, v3
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v0, 8, v0
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+  %vector = load <8 x i8>, <8 x i8> addrspace(1)* %ptr
+  %element = extractelement <8 x i8> %vector, i32 5
+  ret i8 %element
+}
+
+define i8 @extractelement_vgpr_v8i8_idx6(<8 x i8> addrspace(1)* %ptr) {
+; GFX9-LABEL: extractelement_vgpr_v8i8_idx6:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off
+; GFX9-NEXT:    s_mov_b32 s4, 8
+; GFX9-NEXT:    s_movk_i32 s5, 0xff
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 8, v1
+; GFX9-NEXT:    v_lshlrev_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_and_b32_sdwa v2, v1, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    v_and_or_b32 v0, v1, s5, v0
+; GFX9-NEXT:    v_and_b32_sdwa v1, v1, s5 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX9-NEXT:    v_or3_b32 v0, v0, v2, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: extractelement_vgpr_v8i8_idx6:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; GFX8-NEXT:    s_movk_i32 s4, 0xff
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v0, 8
+; GFX8-NEXT:    v_mov_b32_e32 v2, s4
+; GFX8-NEXT:    v_lshrrev_b32_e32 v3, 8, v1
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT:    v_and_b32_sdwa v4, v1, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v4
+; GFX8-NEXT:    v_and_b32_sdwa v1, v1, v2 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: extractelement_vgpr_v8i8_idx6:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    s_mov_b32 s6, 0
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:    buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT:    s_movk_i32 s4, 0xff
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_lshrrev_b32_e32 v0, 8, v1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v2, 16, v1
+; GFX7-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX7-NEXT:    v_lshrrev_b32_e32 v3, 24, v1
+; GFX7-NEXT:    v_and_b32_e32 v2, s4, v2
+; GFX7-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX7-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
+; GFX7-NEXT:    v_and_b32_e32 v3, s4, v3
+; GFX7-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 24, v3
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+  %vector = load <8 x i8>, <8 x i8> addrspace(1)* %ptr
+  %element = extractelement <8 x i8> %vector, i32 6
+  ret i8 %element
+}
+
+define i8 @extractelement_vgpr_v8i8_idx7(<8 x i8> addrspace(1)* %ptr) {
+; GFX9-LABEL: extractelement_vgpr_v8i8_idx7:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off
+; GFX9-NEXT:    s_mov_b32 s4, 8
+; GFX9-NEXT:    s_movk_i32 s5, 0xff
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 8, v1
+; GFX9-NEXT:    v_lshlrev_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_and_b32_sdwa v2, v1, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    v_and_or_b32 v0, v1, s5, v0
+; GFX9-NEXT:    v_and_b32_sdwa v1, v1, s5 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX9-NEXT:    v_or3_b32 v0, v0, v2, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 24, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: extractelement_vgpr_v8i8_idx7:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; GFX8-NEXT:    s_movk_i32 s4, 0xff
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v0, 8
+; GFX8-NEXT:    v_mov_b32_e32 v2, s4
+; GFX8-NEXT:    v_lshrrev_b32_e32 v3, 8, v1
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT:    v_and_b32_sdwa v4, v1, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v4
+; GFX8-NEXT:    v_and_b32_sdwa v1, v1, v2 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    v_lshrrev_b32_e32 v0, 24, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: extractelement_vgpr_v8i8_idx7:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    s_mov_b32 s6, 0
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:    buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT:    s_movk_i32 s4, 0xff
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_lshrrev_b32_e32 v0, 8, v1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v2, 16, v1
+; GFX7-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX7-NEXT:    v_lshrrev_b32_e32 v3, 24, v1
+; GFX7-NEXT:    v_and_b32_e32 v2, s4, v2
+; GFX7-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX7-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
+; GFX7-NEXT:    v_and_b32_e32 v3, s4, v3
+; GFX7-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 24, v3
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v0, 24, v0
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+  %vector = load <8 x i8>, <8 x i8> addrspace(1)* %ptr
+  %element = extractelement <8 x i8> %vector, i32 7
+  ret i8 %element
+}
+
+define amdgpu_ps i8 @extractelement_sgpr_v16i8_sgpr_idx(<16 x i8> addrspace(4)* inreg %ptr, i32 inreg %idx) {
+; GCN-LABEL: extractelement_sgpr_v16i8_sgpr_idx:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[2:3], 0x0
+; GCN-NEXT:    s_movk_i32 s17, 0xff
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_lshr_b32 s5, s0, 8
+; GCN-NEXT:    s_and_b32 s5, s5, s17
+; GCN-NEXT:    s_lshr_b32 s6, s0, 16
+; GCN-NEXT:    s_lshr_b32 s7, s0, 24
+; GCN-NEXT:    s_and_b32 s0, s0, s17
+; GCN-NEXT:    s_lshl_b32 s5, s5, 8
+; GCN-NEXT:    s_or_b32 s0, s0, s5
+; GCN-NEXT:    s_and_b32 s5, s6, s17
+; GCN-NEXT:    s_lshl_b32 s5, s5, 16
+; GCN-NEXT:    s_or_b32 s0, s0, s5
+; GCN-NEXT:    s_and_b32 s5, s7, s17
+; GCN-NEXT:    s_lshl_b32 s5, s5, 24
+; GCN-NEXT:    s_lshr_b32 s8, s1, 8
+; GCN-NEXT:    s_or_b32 s0, s0, s5
+; GCN-NEXT:    s_and_b32 s5, s8, s17
+; GCN-NEXT:    s_lshr_b32 s9, s1, 16
+; GCN-NEXT:    s_lshr_b32 s10, s1, 24
+; GCN-NEXT:    s_and_b32 s1, s1, s17
+; GCN-NEXT:    s_lshl_b32 s5, s5, 8
+; GCN-NEXT:    s_or_b32 s1, s1, s5
+; GCN-NEXT:    s_and_b32 s5, s9, s17
+; GCN-NEXT:    s_lshl_b32 s5, s5, 16
+; GCN-NEXT:    s_or_b32 s1, s1, s5
+; GCN-NEXT:    s_and_b32 s5, s10, s17
+; GCN-NEXT:    s_lshl_b32 s5, s5, 24
+; GCN-NEXT:    s_lshr_b32 s11, s2, 8
+; GCN-NEXT:    s_or_b32 s1, s1, s5
+; GCN-NEXT:    s_and_b32 s5, s11, s17
+; GCN-NEXT:    s_lshr_b32 s12, s2, 16
+; GCN-NEXT:    s_lshr_b32 s13, s2, 24
+; GCN-NEXT:    s_and_b32 s2, s2, s17
+; GCN-NEXT:    s_lshl_b32 s5, s5, 8
+; GCN-NEXT:    s_or_b32 s2, s2, s5
+; GCN-NEXT:    s_and_b32 s5, s12, s17
+; GCN-NEXT:    s_lshl_b32 s5, s5, 16
+; GCN-NEXT:    s_or_b32 s2, s2, s5
+; GCN-NEXT:    s_and_b32 s5, s13, s17
+; GCN-NEXT:    s_lshl_b32 s5, s5, 24
+; GCN-NEXT:    s_lshr_b32 s14, s3, 8
+; GCN-NEXT:    s_or_b32 s2, s2, s5
+; GCN-NEXT:    s_and_b32 s5, s14, s17
+; GCN-NEXT:    s_lshr_b32 s15, s3, 16
+; GCN-NEXT:    s_lshr_b32 s16, s3, 24
+; GCN-NEXT:    s_and_b32 s3, s3, s17
+; GCN-NEXT:    s_lshl_b32 s5, s5, 8
+; GCN-NEXT:    s_or_b32 s3, s3, s5
+; GCN-NEXT:    s_and_b32 s5, s15, s17
+; GCN-NEXT:    s_lshl_b32 s5, s5, 16
+; GCN-NEXT:    s_or_b32 s3, s3, s5
+; GCN-NEXT:    s_and_b32 s5, s16, s17
+; GCN-NEXT:    s_lshl_b32 s5, s5, 24
+; GCN-NEXT:    s_or_b32 s3, s3, s5
+; GCN-NEXT:    s_lshr_b32 s5, s4, 2
+; GCN-NEXT:    s_cmp_eq_u32 s5, 1
+; GCN-NEXT:    s_cselect_b32 s0, s1, s0
+; GCN-NEXT:    s_cmp_eq_u32 s5, 2
+; GCN-NEXT:    s_cselect_b32 s0, s2, s0
+; GCN-NEXT:    s_cmp_eq_u32 s5, 3
+; GCN-NEXT:    s_cselect_b32 s0, s3, s0
+; GCN-NEXT:    s_and_b32 s1, s4, 3
+; GCN-NEXT:    s_lshl_b32 s1, s1, 3
+; GCN-NEXT:    s_lshr_b32 s0, s0, s1
+; GCN-NEXT:    ; return to shader part epilog
+  %vector = load <16 x i8>, <16 x i8> addrspace(4)* %ptr
+  %element = extractelement <16 x i8> %vector, i32 %idx
+  ret i8 %element
+}
+
+define amdgpu_ps i8 @extractelement_vgpr_v16i8_sgpr_idx(<16 x i8> addrspace(1)* %ptr, i32 inreg %idx) {
+; GFX9-LABEL: extractelement_vgpr_v16i8_sgpr_idx:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    global_load_dwordx4 v[0:3], v[0:1], off
+; GFX9-NEXT:    s_mov_b32 s0, 8
+; GFX9-NEXT:    v_mov_b32_e32 v5, 8
+; GFX9-NEXT:    s_movk_i32 s1, 0xff
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0xff
+; GFX9-NEXT:    s_lshr_b32 s3, s2, 2
+; GFX9-NEXT:    v_cmp_eq_u32_e64 vcc, s3, 1
+; GFX9-NEXT:    s_and_b32 s2, s2, 3
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v6, 8, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v7, 8, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v8, 8, v2
+; GFX9-NEXT:    v_lshlrev_b32_sdwa v6, s0, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_lshlrev_b32_sdwa v7, v5, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v9, 8, v3
+; GFX9-NEXT:    v_lshlrev_b32_sdwa v8, v5, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_and_b32_sdwa v10, v0, s1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    v_and_b32_sdwa v11, v0, s1 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX9-NEXT:    v_and_or_b32 v0, v0, s1, v6
+; GFX9-NEXT:    v_and_b32_sdwa v12, v1, s1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    v_and_b32_sdwa v13, v1, s1 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX9-NEXT:    v_and_or_b32 v1, v1, s1, v7
+; GFX9-NEXT:    v_and_b32_sdwa v14, v2, s1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    v_and_b32_sdwa v15, v2, v4 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX9-NEXT:    v_and_or_b32 v2, v2, s1, v8
+; GFX9-NEXT:    v_lshlrev_b32_sdwa v5, v5, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or3_b32 v0, v0, v10, v11
+; GFX9-NEXT:    v_or3_b32 v1, v1, v12, v13
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX9-NEXT:    v_and_b32_sdwa v16, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    v_and_b32_sdwa v17, v3, v4 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX9-NEXT:    v_and_or_b32 v3, v3, v4, v5
+; GFX9-NEXT:    v_or3_b32 v2, v2, v14, v15
+; GFX9-NEXT:    v_cmp_eq_u32_e64 vcc, s3, 2
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX9-NEXT:    v_or3_b32 v3, v3, v16, v17
+; GFX9-NEXT:    v_cmp_eq_u32_e64 vcc, s3, 3
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; GFX9-NEXT:    s_lshl_b32 s0, s2, 3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, s0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: extractelement_vgpr_v16i8_sgpr_idx:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
+; GFX8-NEXT:    s_movk_i32 s0, 0xff
+; GFX8-NEXT:    v_mov_b32_e32 v5, 8
+; GFX8-NEXT:    v_mov_b32_e32 v6, 8
+; GFX8-NEXT:    v_mov_b32_e32 v7, s0
+; GFX8-NEXT:    v_mov_b32_e32 v4, 0xff
+; GFX8-NEXT:    s_lshr_b32 s0, s2, 2
+; GFX8-NEXT:    v_cmp_eq_u32_e64 vcc, s0, 1
+; GFX8-NEXT:    s_and_b32 s1, s2, 3
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_lshrrev_b32_e32 v8, 8, v0
+; GFX8-NEXT:    v_lshrrev_b32_e32 v9, 8, v1
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v5, v5, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT:    v_lshrrev_b32_e32 v10, 8, v2
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v8, v6, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT:    v_lshrrev_b32_e32 v11, 8, v3
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v9, v6, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT:    v_and_b32_sdwa v12, v0, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT:    v_and_b32_sdwa v13, v0, v7 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT:    v_and_b32_sdwa v14, v1, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT:    v_and_b32_sdwa v7, v1, v7 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v1, v1, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v12
+; GFX8-NEXT:    v_or_b32_e32 v1, v1, v14
+; GFX8-NEXT:    v_and_b32_sdwa v15, v2, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT:    v_and_b32_sdwa v16, v2, v4 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v2, v2, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v6, v6, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT:    v_or_b32_e32 v2, v2, v15
+; GFX8-NEXT:    v_and_b32_sdwa v17, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT:    v_and_b32_sdwa v4, v3, v4 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v3, v3, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v13
+; GFX8-NEXT:    v_or_b32_e32 v1, v1, v7
+; GFX8-NEXT:    v_or_b32_e32 v3, v3, v17
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX8-NEXT:    v_or_b32_e32 v2, v2, v16
+; GFX8-NEXT:    v_cmp_eq_u32_e64 vcc, s0, 2
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX8-NEXT:    v_cmp_eq_u32_e64 vcc, s0, 3
+; GFX8-NEXT:    v_or_b32_e32 v3, v3, v4
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; GFX8-NEXT:    s_lshl_b32 s0, s1, 3
+; GFX8-NEXT:    v_lshrrev_b32_e32 v0, s0, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX7-LABEL: extractelement_vgpr_v16i8_sgpr_idx:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_mov_b32 s6, 0
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:    buffer_load_dwordx4 v[0:3], v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT:    s_movk_i32 s0, 0xff
+; GFX7-NEXT:    v_mov_b32_e32 v4, 0xff
+; GFX7-NEXT:    s_lshr_b32 s1, s2, 2
+; GFX7-NEXT:    v_cmp_eq_u32_e64 vcc, s1, 1
+; GFX7-NEXT:    s_and_b32 s2, s2, 3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_lshrrev_b32_e32 v5, 8, v0
+; GFX7-NEXT:    v_lshrrev_b32_e32 v8, 8, v1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v6, 16, v0
+; GFX7-NEXT:    v_lshrrev_b32_e32 v9, 16, v1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v11, 8, v2
+; GFX7-NEXT:    v_and_b32_e32 v5, s0, v5
+; GFX7-NEXT:    v_and_b32_e32 v8, s0, v8
+; GFX7-NEXT:    v_lshrrev_b32_e32 v7, 24, v0
+; GFX7-NEXT:    v_lshrrev_b32_e32 v10, 24, v1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v12, 16, v2
+; GFX7-NEXT:    v_lshrrev_b32_e32 v14, 8, v3
+; GFX7-NEXT:    v_and_b32_e32 v6, s0, v6
+; GFX7-NEXT:    v_and_b32_e32 v9, s0, v9
+; GFX7-NEXT:    v_and_b32_e32 v11, v11, v4
+; GFX7-NEXT:    v_and_b32_e32 v0, s0, v0
+; GFX7-NEXT:    v_lshlrev_b32_e32 v5, 8, v5
+; GFX7-NEXT:    v_and_b32_e32 v1, s0, v1
+; GFX7-NEXT:    v_lshlrev_b32_e32 v8, 8, v8
+; GFX7-NEXT:    v_lshrrev_b32_e32 v13, 24, v2
+; GFX7-NEXT:    v_lshrrev_b32_e32 v15, 16, v3
+; GFX7-NEXT:    v_and_b32_e32 v7, s0, v7
+; GFX7-NEXT:    v_and_b32_e32 v10, s0, v10
+; GFX7-NEXT:    v_and_b32_e32 v12, v12, v4
+; GFX7-NEXT:    v_and_b32_e32 v14, v14, v4
+; GFX7-NEXT:    v_and_b32_e32 v2, s0, v2
+; GFX7-NEXT:    v_lshlrev_b32_e32 v11, 8, v11
+; GFX7-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v5
+; GFX7-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GFX7-NEXT:    v_or_b32_e32 v1, v1, v8
+; GFX7-NEXT:    v_and_b32_e32 v13, v13, v4
+; GFX7-NEXT:    v_and_b32_e32 v15, v15, v4
+; GFX7-NEXT:    v_lshrrev_b32_e32 v16, 24, v3
+; GFX7-NEXT:    v_and_b32_e32 v3, v3, v4
+; GFX7-NEXT:    v_lshlrev_b32_e32 v14, 8, v14
+; GFX7-NEXT:    v_lshlrev_b32_e32 v7, 24, v7
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v6
+; GFX7-NEXT:    v_lshlrev_b32_e32 v10, 24, v10
+; GFX7-NEXT:    v_or_b32_e32 v1, v1, v9
+; GFX7-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; GFX7-NEXT:    v_or_b32_e32 v2, v2, v11
+; GFX7-NEXT:    v_and_b32_e32 v4, v16, v4
+; GFX7-NEXT:    v_lshlrev_b32_e32 v13, 24, v13
+; GFX7-NEXT:    v_or_b32_e32 v2, v2, v12
+; GFX7-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; GFX7-NEXT:    v_or_b32_e32 v3, v3, v14
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v7
+; GFX7-NEXT:    v_or_b32_e32 v1, v1, v10
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX7-NEXT:    v_lshlrev_b32_e32 v4, 24, v4
+; GFX7-NEXT:    v_or_b32_e32 v3, v3, v15
+; GFX7-NEXT:    v_or_b32_e32 v2, v2, v13
+; GFX7-NEXT:    v_cmp_eq_u32_e64 vcc, s1, 2
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX7-NEXT:    v_or_b32_e32 v3, v3, v4
+; GFX7-NEXT:    v_cmp_eq_u32_e64 vcc, s1, 3
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; GFX7-NEXT:    s_lshl_b32 s0, s2, 3
+; GFX7-NEXT:    v_lshrrev_b32_e32 v0, s0, v0
+; GFX7-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX7-NEXT:    ; return to shader part epilog
+  %vector = load <16 x i8>, <16 x i8> addrspace(1)* %ptr
+  %element = extractelement <16 x i8> %vector, i32 %idx
+  ret i8 %element
+}
+
+define i8 @extractelement_vgpr_v16i8_vgpr_idx(<16 x i8> addrspace(1)* %ptr, i32 %idx) {
+; GFX9-LABEL: extractelement_vgpr_v16i8_vgpr_idx:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dwordx4 v[3:6], v[0:1], off
+; GFX9-NEXT:    s_mov_b32 s4, 8
+; GFX9-NEXT:    v_mov_b32_e32 v1, 8
+; GFX9-NEXT:    s_movk_i32 s5, 0xff
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0xff
+; GFX9-NEXT:    v_lshrrev_b32_e32 v7, 2, v2
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v7
+; GFX9-NEXT:    v_and_b32_e32 v2, 3, v2
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v8, 8, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v9, 8, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v10, 8, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v11, 8, v6
+; GFX9-NEXT:    v_lshlrev_b32_sdwa v8, s4, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_lshlrev_b32_sdwa v9, v1, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_lshlrev_b32_sdwa v10, v1, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_and_b32_sdwa v12, v3, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    v_and_b32_sdwa v13, v3, s5 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX9-NEXT:    v_and_b32_sdwa v14, v4, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    v_and_b32_sdwa v15, v4, s5 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX9-NEXT:    v_lshlrev_b32_sdwa v1, v1, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_and_or_b32 v3, v3, s5, v8
+; GFX9-NEXT:    v_and_or_b32 v4, v4, s5, v9
+; GFX9-NEXT:    v_and_b32_sdwa v16, v5, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    v_and_b32_sdwa v17, v5, v0 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX9-NEXT:    v_and_b32_sdwa v18, v6, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    v_and_b32_sdwa v19, v6, v0 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX9-NEXT:    v_and_or_b32 v0, v6, v0, v1
+; GFX9-NEXT:    v_or3_b32 v1, v3, v12, v13
+; GFX9-NEXT:    v_or3_b32 v3, v4, v14, v15
+; GFX9-NEXT:    v_and_or_b32 v5, v5, s5, v10
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX9-NEXT:    v_or3_b32 v4, v5, v16, v17
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 2, v7
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
+; GFX9-NEXT:    v_or3_b32 v0, v0, v18, v19
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 3, v7
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 3, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, v1, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: extractelement_vgpr_v16i8_vgpr_idx:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    flat_load_dwordx4 v[3:6], v[0:1]
+; GFX8-NEXT:    s_movk_i32 s4, 0xff
+; GFX8-NEXT:    v_mov_b32_e32 v1, 8
+; GFX8-NEXT:    v_mov_b32_e32 v7, 8
+; GFX8-NEXT:    v_mov_b32_e32 v8, s4
+; GFX8-NEXT:    v_mov_b32_e32 v0, 0xff
+; GFX8-NEXT:    v_lshrrev_b32_e32 v9, 2, v2
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v9
+; GFX8-NEXT:    v_and_b32_e32 v2, 3, v2
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_lshrrev_b32_e32 v10, 8, v3
+; GFX8-NEXT:    v_lshrrev_b32_e32 v11, 8, v4
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v1, v1, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT:    v_lshrrev_b32_e32 v12, 8, v5
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v10, v7, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT:    v_lshrrev_b32_e32 v13, 8, v6
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v11, v7, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT:    v_and_b32_sdwa v14, v3, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT:    v_and_b32_sdwa v15, v3, v8 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX8-NEXT:    v_and_b32_sdwa v16, v4, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v3, v4, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v1, v1, v14
+; GFX8-NEXT:    v_and_b32_sdwa v8, v4, v8 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v3, v3, v16
+; GFX8-NEXT:    v_and_b32_sdwa v17, v5, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v4, v5, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v7, v7, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT:    v_and_b32_sdwa v18, v5, v0 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v4, v4, v17
+; GFX8-NEXT:    v_and_b32_sdwa v19, v6, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v5, v6, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v1, v1, v15
+; GFX8-NEXT:    v_or_b32_e32 v3, v3, v8
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX8-NEXT:    v_and_b32_sdwa v0, v6, v0 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v5, v5, v19
+; GFX8-NEXT:    v_or_b32_e32 v4, v4, v18
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 2, v9
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
+; GFX8-NEXT:    v_or_b32_e32 v0, v5, v0
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 3, v9
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX8-NEXT:    v_lshlrev_b32_e32 v1, 3, v2
+; GFX8-NEXT:    v_lshrrev_b32_e32 v0, v1, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: extractelement_vgpr_v16i8_vgpr_idx:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    s_mov_b32 s6, 0
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:    buffer_load_dwordx4 v[3:6], v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT:    s_movk_i32 s4, 0xff
+; GFX7-NEXT:    v_mov_b32_e32 v0, 0xff
+; GFX7-NEXT:    v_lshrrev_b32_e32 v18, 2, v2
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v18
+; GFX7-NEXT:    v_and_b32_e32 v2, 3, v2
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_lshrrev_b32_e32 v1, 8, v3
+; GFX7-NEXT:    v_lshrrev_b32_e32 v9, 8, v4
+; GFX7-NEXT:    v_lshrrev_b32_e32 v7, 16, v3
+; GFX7-NEXT:    v_lshrrev_b32_e32 v10, 16, v4
+; GFX7-NEXT:    v_lshrrev_b32_e32 v12, 8, v5
+; GFX7-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX7-NEXT:    v_and_b32_e32 v9, s4, v9
+; GFX7-NEXT:    v_lshrrev_b32_e32 v8, 24, v3
+; GFX7-NEXT:    v_lshrrev_b32_e32 v11, 24, v4
+; GFX7-NEXT:    v_lshrrev_b32_e32 v13, 16, v5
+; GFX7-NEXT:    v_lshrrev_b32_e32 v15, 8, v6
+; GFX7-NEXT:    v_and_b32_e32 v7, s4, v7
+; GFX7-NEXT:    v_and_b32_e32 v10, s4, v10
+; GFX7-NEXT:    v_and_b32_e32 v12, v12, v0
+; GFX7-NEXT:    v_and_b32_e32 v3, s4, v3
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX7-NEXT:    v_and_b32_e32 v4, s4, v4
+; GFX7-NEXT:    v_lshlrev_b32_e32 v9, 8, v9
+; GFX7-NEXT:    v_lshrrev_b32_e32 v14, 24, v5
+; GFX7-NEXT:    v_lshrrev_b32_e32 v16, 16, v6
+; GFX7-NEXT:    v_and_b32_e32 v8, s4, v8
+; GFX7-NEXT:    v_and_b32_e32 v11, s4, v11
+; GFX7-NEXT:    v_and_b32_e32 v13, v13, v0
+; GFX7-NEXT:    v_and_b32_e32 v15, v15, v0
+; GFX7-NEXT:    v_or_b32_e32 v1, v3, v1
+; GFX7-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GFX7-NEXT:    v_and_b32_e32 v5, s4, v5
+; GFX7-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; GFX7-NEXT:    v_or_b32_e32 v3, v4, v9
+; GFX7-NEXT:    v_lshlrev_b32_e32 v12, 8, v12
+; GFX7-NEXT:    v_and_b32_e32 v14, v14, v0
+; GFX7-NEXT:    v_and_b32_e32 v16, v16, v0
+; GFX7-NEXT:    v_lshrrev_b32_e32 v17, 24, v6
+; GFX7-NEXT:    v_and_b32_e32 v6, v6, v0
+; GFX7-NEXT:    v_lshlrev_b32_e32 v8, 24, v8
+; GFX7-NEXT:    v_or_b32_e32 v1, v1, v7
+; GFX7-NEXT:    v_lshlrev_b32_e32 v11, 24, v11
+; GFX7-NEXT:    v_or_b32_e32 v3, v3, v10
+; GFX7-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GFX7-NEXT:    v_or_b32_e32 v4, v5, v12
+; GFX7-NEXT:    v_lshlrev_b32_e32 v15, 8, v15
+; GFX7-NEXT:    v_and_b32_e32 v0, v17, v0
+; GFX7-NEXT:    v_lshlrev_b32_e32 v14, 24, v14
+; GFX7-NEXT:    v_or_b32_e32 v4, v4, v13
+; GFX7-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
+; GFX7-NEXT:    v_or_b32_e32 v5, v6, v15
+; GFX7-NEXT:    v_or_b32_e32 v1, v1, v8
+; GFX7-NEXT:    v_or_b32_e32 v3, v3, v11
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX7-NEXT:    v_lshlrev_b32_e32 v0, 24, v0
+; GFX7-NEXT:    v_or_b32_e32 v5, v5, v16
+; GFX7-NEXT:    v_or_b32_e32 v4, v4, v14
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 2, v18
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
+; GFX7-NEXT:    v_or_b32_e32 v0, v5, v0
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 3, v18
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 3, v2
+; GFX7-NEXT:    v_lshrrev_b32_e32 v0, v1, v0
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+  %vector = load <16 x i8>, <16 x i8> addrspace(1)* %ptr
+  %element = extractelement <16 x i8> %vector, i32 %idx
+  ret i8 %element
+}
+
+define amdgpu_ps i8 @extractelement_sgpr_v16i8_vgpr_idx(<16 x i8> addrspace(4)* inreg %ptr, i32 %idx) {
+; GCN-LABEL: extractelement_sgpr_v16i8_vgpr_idx:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[2:3], 0x0
+; GCN-NEXT:    s_movk_i32 s16, 0xff
+; GCN-NEXT:    v_lshrrev_b32_e32 v1, 2, v0
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v1
+; GCN-NEXT:    v_and_b32_e32 v0, 3, v0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_lshr_b32 s4, s0, 8
+; GCN-NEXT:    s_and_b32 s4, s4, s16
+; GCN-NEXT:    s_lshr_b32 s5, s0, 16
+; GCN-NEXT:    s_lshr_b32 s6, s0, 24
+; GCN-NEXT:    s_and_b32 s0, s0, s16
+; GCN-NEXT:    s_lshl_b32 s4, s4, 8
+; GCN-NEXT:    s_or_b32 s0, s0, s4
+; GCN-NEXT:    s_and_b32 s4, s5, s16
+; GCN-NEXT:    s_lshl_b32 s4, s4, 16
+; GCN-NEXT:    s_or_b32 s0, s0, s4
+; GCN-NEXT:    s_and_b32 s4, s6, s16
+; GCN-NEXT:    s_lshl_b32 s4, s4, 24
+; GCN-NEXT:    s_lshr_b32 s7, s1, 8
+; GCN-NEXT:    s_or_b32 s0, s0, s4
+; GCN-NEXT:    s_and_b32 s4, s7, s16
+; GCN-NEXT:    s_lshr_b32 s8, s1, 16
+; GCN-NEXT:    s_lshr_b32 s9, s1, 24
+; GCN-NEXT:    s_and_b32 s1, s1, s16
+; GCN-NEXT:    s_lshl_b32 s4, s4, 8
+; GCN-NEXT:    s_or_b32 s1, s1, s4
+; GCN-NEXT:    s_and_b32 s4, s8, s16
+; GCN-NEXT:    s_lshl_b32 s4, s4, 16
+; GCN-NEXT:    s_or_b32 s1, s1, s4
+; GCN-NEXT:    s_and_b32 s4, s9, s16
+; GCN-NEXT:    s_lshl_b32 s4, s4, 24
+; GCN-NEXT:    s_lshr_b32 s10, s2, 8
+; GCN-NEXT:    s_or_b32 s1, s1, s4
+; GCN-NEXT:    s_and_b32 s4, s10, s16
+; GCN-NEXT:    s_lshr_b32 s11, s2, 16
+; GCN-NEXT:    s_lshr_b32 s12, s2, 24
+; GCN-NEXT:    s_and_b32 s2, s2, s16
+; GCN-NEXT:    s_lshl_b32 s4, s4, 8
+; GCN-NEXT:    s_or_b32 s2, s2, s4
+; GCN-NEXT:    s_and_b32 s4, s11, s16
+; GCN-NEXT:    s_lshl_b32 s4, s4, 16
+; GCN-NEXT:    s_or_b32 s2, s2, s4
+; GCN-NEXT:    s_and_b32 s4, s12, s16
+; GCN-NEXT:    s_lshl_b32 s4, s4, 24
+; GCN-NEXT:    s_lshr_b32 s13, s3, 8
+; GCN-NEXT:    s_or_b32 s2, s2, s4
+; GCN-NEXT:    s_and_b32 s4, s13, s16
+; GCN-NEXT:    s_lshr_b32 s14, s3, 16
+; GCN-NEXT:    s_lshr_b32 s15, s3, 24
+; GCN-NEXT:    s_and_b32 s3, s3, s16
+; GCN-NEXT:    s_lshl_b32 s4, s4, 8
+; GCN-NEXT:    s_or_b32 s3, s3, s4
+; GCN-NEXT:    s_and_b32 s4, s14, s16
+; GCN-NEXT:    s_lshl_b32 s4, s4, 16
+; GCN-NEXT:    s_or_b32 s3, s3, s4
+; GCN-NEXT:    s_and_b32 s4, s15, s16
+; GCN-NEXT:    s_lshl_b32 s4, s4, 24
+; GCN-NEXT:    v_mov_b32_e32 v2, s0
+; GCN-NEXT:    v_mov_b32_e32 v3, s1
+; GCN-NEXT:    s_or_b32 s3, s3, s4
+; GCN-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; GCN-NEXT:    v_mov_b32_e32 v4, s2
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 2, v1
+; GCN-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; GCN-NEXT:    v_mov_b32_e32 v5, s3
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 3, v1
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v2, v5, vcc
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
+; GCN-NEXT:    v_lshrrev_b32_e32 v0, v0, v1
+; GCN-NEXT:    v_readfirstlane_b32 s0, v0
+; GCN-NEXT:    ; return to shader part epilog
+  %vector = load <16 x i8>, <16 x i8> addrspace(4)* %ptr
+  %element = extractelement <16 x i8> %vector, i32 %idx
+  ret i8 %element
+}
+
+define i8 @extractelement_vgpr_v16i8_idx0(<16 x i8> addrspace(1)* %ptr) {
+; GFX9-LABEL: extractelement_vgpr_v16i8_idx0:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dwordx4 v[0:3], v[0:1], off
+; GFX9-NEXT:    s_mov_b32 s4, 8
+; GFX9-NEXT:    s_movk_i32 s5, 0xff
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v1, 8, v0
+; GFX9-NEXT:    v_lshlrev_b32_sdwa v1, s4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_and_b32_sdwa v2, v0, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    v_and_or_b32 v1, v0, s5, v1
+; GFX9-NEXT:    v_and_b32_sdwa v0, v0, s5 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX9-NEXT:    v_or3_b32 v0, v1, v2, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: extractelement_vgpr_v16i8_idx0:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
+; GFX8-NEXT:    s_movk_i32 s4, 0xff
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v1, 8
+; GFX8-NEXT:    v_mov_b32_e32 v2, s4
+; GFX8-NEXT:    v_lshrrev_b32_e32 v3, 8, v0
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT:    v_and_b32_sdwa v4, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v1, v1, v4
+; GFX8-NEXT:    v_and_b32_sdwa v0, v0, v2 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: extractelement_vgpr_v16i8_idx0:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    s_mov_b32 s6, 0
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:    buffer_load_dwordx4 v[0:3], v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT:    s_movk_i32 s4, 0xff
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_lshrrev_b32_e32 v1, 8, v0
+; GFX7-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
+; GFX7-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v3, 24, v0
+; GFX7-NEXT:    v_and_b32_e32 v2, s4, v2
+; GFX7-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX7-NEXT:    v_and_b32_e32 v3, s4, v3
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 24, v3
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+  %vector = load <16 x i8>, <16 x i8> addrspace(1)* %ptr
+  %element = extractelement <16 x i8> %vector, i32 0
+  ret i8 %element
+}
+
+define i8 @extractelement_vgpr_v16i8_idx1(<16 x i8> addrspace(1)* %ptr) {
+; GFX9-LABEL: extractelement_vgpr_v16i8_idx1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dwordx4 v[0:3], v[0:1], off
+; GFX9-NEXT:    s_mov_b32 s4, 8
+; GFX9-NEXT:    s_movk_i32 s5, 0xff
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v1, 8, v0
+; GFX9-NEXT:    v_lshlrev_b32_sdwa v1, s4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_and_b32_sdwa v2, v0, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    v_and_or_b32 v1, v0, s5, v1
+; GFX9-NEXT:    v_and_b32_sdwa v0, v0, s5 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX9-NEXT:    v_or3_b32 v0, v1, v2, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 8, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: extractelement_vgpr_v16i8_idx1:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
+; GFX8-NEXT:    s_movk_i32 s4, 0xff
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v1, 8
+; GFX8-NEXT:    v_mov_b32_e32 v2, s4
+; GFX8-NEXT:    v_lshrrev_b32_e32 v3, 8, v0
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT:    v_and_b32_sdwa v4, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v1, v1, v4
+; GFX8-NEXT:    v_and_b32_sdwa v0, v0, v2 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX8-NEXT:    v_lshrrev_b32_e32 v0, 8, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: extractelement_vgpr_v16i8_idx1:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    s_mov_b32 s6, 0
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:    buffer_load_dwordx4 v[0:3], v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT:    s_movk_i32 s4, 0xff
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_lshrrev_b32_e32 v1, 8, v0
+; GFX7-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
+; GFX7-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v3, 24, v0
+; GFX7-NEXT:    v_and_b32_e32 v2, s4, v2
+; GFX7-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX7-NEXT:    v_and_b32_e32 v3, s4, v3
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 24, v3
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v0, 8, v0
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+  %vector = load <16 x i8>, <16 x i8> addrspace(1)* %ptr
+  %element = extractelement <16 x i8> %vector, i32 1
+  ret i8 %element
+}
+
+define i8 @extractelement_vgpr_v16i8_idx2(<16 x i8> addrspace(1)* %ptr) {
+; GFX9-LABEL: extractelement_vgpr_v16i8_idx2:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dwordx4 v[0:3], v[0:1], off
+; GFX9-NEXT:    s_mov_b32 s4, 8
+; GFX9-NEXT:    s_movk_i32 s5, 0xff
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v1, 8, v0
+; GFX9-NEXT:    v_lshlrev_b32_sdwa v1, s4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_and_b32_sdwa v2, v0, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    v_and_or_b32 v1, v0, s5, v1
+; GFX9-NEXT:    v_and_b32_sdwa v0, v0, s5 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX9-NEXT:    v_or3_b32 v0, v1, v2, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: extractelement_vgpr_v16i8_idx2:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
+; GFX8-NEXT:    s_movk_i32 s4, 0xff
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v1, 8
+; GFX8-NEXT:    v_mov_b32_e32 v2, s4
+; GFX8-NEXT:    v_lshrrev_b32_e32 v3, 8, v0
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT:    v_and_b32_sdwa v4, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v1, v1, v4
+; GFX8-NEXT:    v_and_b32_sdwa v0, v0, v2 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX8-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: extractelement_vgpr_v16i8_idx2:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    s_mov_b32 s6, 0
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:    buffer_load_dwordx4 v[0:3], v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT:    s_movk_i32 s4, 0xff
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_lshrrev_b32_e32 v1, 8, v0
+; GFX7-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
+; GFX7-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v3, 24, v0
+; GFX7-NEXT:    v_and_b32_e32 v2, s4, v2
+; GFX7-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX7-NEXT:    v_and_b32_e32 v3, s4, v3
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 24, v3
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+  %vector = load <16 x i8>, <16 x i8> addrspace(1)* %ptr
+  %element = extractelement <16 x i8> %vector, i32 2
+  ret i8 %element
+}
+
+define i8 @extractelement_vgpr_v16i8_idx3(<16 x i8> addrspace(1)* %ptr) {
+; GFX9-LABEL: extractelement_vgpr_v16i8_idx3:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dwordx4 v[0:3], v[0:1], off
+; GFX9-NEXT:    s_mov_b32 s4, 8
+; GFX9-NEXT:    s_movk_i32 s5, 0xff
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v1, 8, v0
+; GFX9-NEXT:    v_lshlrev_b32_sdwa v1, s4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_and_b32_sdwa v2, v0, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    v_and_or_b32 v1, v0, s5, v1
+; GFX9-NEXT:    v_and_b32_sdwa v0, v0, s5 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX9-NEXT:    v_or3_b32 v0, v1, v2, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 24, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: extractelement_vgpr_v16i8_idx3:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
+; GFX8-NEXT:    s_movk_i32 s4, 0xff
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v1, 8
+; GFX8-NEXT:    v_mov_b32_e32 v2, s4
+; GFX8-NEXT:    v_lshrrev_b32_e32 v3, 8, v0
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT:    v_and_b32_sdwa v4, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v1, v1, v4
+; GFX8-NEXT:    v_and_b32_sdwa v0, v0, v2 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX8-NEXT:    v_lshrrev_b32_e32 v0, 24, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: extractelement_vgpr_v16i8_idx3:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    s_mov_b32 s6, 0
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:    buffer_load_dwordx4 v[0:3], v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT:    s_movk_i32 s4, 0xff
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_lshrrev_b32_e32 v1, 8, v0
+; GFX7-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
+; GFX7-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v3, 24, v0
+; GFX7-NEXT:    v_and_b32_e32 v2, s4, v2
+; GFX7-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX7-NEXT:    v_and_b32_e32 v3, s4, v3
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 24, v3
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v0, 24, v0
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+  %vector = load <16 x i8>, <16 x i8> addrspace(1)* %ptr
+  %element = extractelement <16 x i8> %vector, i32 3
+  ret i8 %element
+}
+
+define i8 @extractelement_vgpr_v16i8_idx4(<16 x i8> addrspace(1)* %ptr) {
+; GFX9-LABEL: extractelement_vgpr_v16i8_idx4:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dwordx4 v[0:3], v[0:1], off
+; GFX9-NEXT:    s_mov_b32 s4, 8
+; GFX9-NEXT:    s_movk_i32 s5, 0xff
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 8, v1
+; GFX9-NEXT:    v_lshlrev_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_and_b32_sdwa v2, v1, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    v_and_or_b32 v0, v1, s5, v0
+; GFX9-NEXT:    v_and_b32_sdwa v1, v1, s5 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX9-NEXT:    v_or3_b32 v0, v0, v2, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: extractelement_vgpr_v16i8_idx4:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
+; GFX8-NEXT:    s_movk_i32 s4, 0xff
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v0, 8
+; GFX8-NEXT:    v_mov_b32_e32 v2, s4
+; GFX8-NEXT:    v_lshrrev_b32_e32 v3, 8, v1
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT:    v_and_b32_sdwa v4, v1, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v4
+; GFX8-NEXT:    v_and_b32_sdwa v1, v1, v2 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: extractelement_vgpr_v16i8_idx4:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    s_mov_b32 s6, 0
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:    buffer_load_dwordx4 v[0:3], v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT:    s_movk_i32 s4, 0xff
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_lshrrev_b32_e32 v0, 8, v1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v2, 16, v1
+; GFX7-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX7-NEXT:    v_lshrrev_b32_e32 v3, 24, v1
+; GFX7-NEXT:    v_and_b32_e32 v2, s4, v2
+; GFX7-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX7-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
+; GFX7-NEXT:    v_and_b32_e32 v3, s4, v3
+; GFX7-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 24, v3
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+  %vector = load <16 x i8>, <16 x i8> addrspace(1)* %ptr
+  %element = extractelement <16 x i8> %vector, i32 4
+  ret i8 %element
+}
+
+define i8 @extractelement_vgpr_v16i8_idx5(<16 x i8> addrspace(1)* %ptr) {
+; GFX9-LABEL: extractelement_vgpr_v16i8_idx5:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dwordx4 v[0:3], v[0:1], off
+; GFX9-NEXT:    s_mov_b32 s4, 8
+; GFX9-NEXT:    s_movk_i32 s5, 0xff
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 8, v1
+; GFX9-NEXT:    v_lshlrev_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_and_b32_sdwa v2, v1, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    v_and_or_b32 v0, v1, s5, v0
+; GFX9-NEXT:    v_and_b32_sdwa v1, v1, s5 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX9-NEXT:    v_or3_b32 v0, v0, v2, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 8, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: extractelement_vgpr_v16i8_idx5:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
+; GFX8-NEXT:    s_movk_i32 s4, 0xff
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v0, 8
+; GFX8-NEXT:    v_mov_b32_e32 v2, s4
+; GFX8-NEXT:    v_lshrrev_b32_e32 v3, 8, v1
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT:    v_and_b32_sdwa v4, v1, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v4
+; GFX8-NEXT:    v_and_b32_sdwa v1, v1, v2 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    v_lshrrev_b32_e32 v0, 8, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: extractelement_vgpr_v16i8_idx5:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    s_mov_b32 s6, 0
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:    buffer_load_dwordx4 v[0:3], v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT:    s_movk_i32 s4, 0xff
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_lshrrev_b32_e32 v0, 8, v1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v2, 16, v1
+; GFX7-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX7-NEXT:    v_lshrrev_b32_e32 v3, 24, v1
+; GFX7-NEXT:    v_and_b32_e32 v2, s4, v2
+; GFX7-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX7-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
+; GFX7-NEXT:    v_and_b32_e32 v3, s4, v3
+; GFX7-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 24, v3
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v0, 8, v0
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+  %vector = load <16 x i8>, <16 x i8> addrspace(1)* %ptr
+  %element = extractelement <16 x i8> %vector, i32 5
+  ret i8 %element
+}
+
+define i8 @extractelement_vgpr_v16i8_idx6(<16 x i8> addrspace(1)* %ptr) {
+; GFX9-LABEL: extractelement_vgpr_v16i8_idx6:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dwordx4 v[0:3], v[0:1], off
+; GFX9-NEXT:    s_mov_b32 s4, 8
+; GFX9-NEXT:    s_movk_i32 s5, 0xff
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 8, v1
+; GFX9-NEXT:    v_lshlrev_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_and_b32_sdwa v2, v1, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    v_and_or_b32 v0, v1, s5, v0
+; GFX9-NEXT:    v_and_b32_sdwa v1, v1, s5 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX9-NEXT:    v_or3_b32 v0, v0, v2, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: extractelement_vgpr_v16i8_idx6:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
+; GFX8-NEXT:    s_movk_i32 s4, 0xff
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v0, 8
+; GFX8-NEXT:    v_mov_b32_e32 v2, s4
+; GFX8-NEXT:    v_lshrrev_b32_e32 v3, 8, v1
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT:    v_and_b32_sdwa v4, v1, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v4
+; GFX8-NEXT:    v_and_b32_sdwa v1, v1, v2 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: extractelement_vgpr_v16i8_idx6:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    s_mov_b32 s6, 0
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:    buffer_load_dwordx4 v[0:3], v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT:    s_movk_i32 s4, 0xff
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_lshrrev_b32_e32 v0, 8, v1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v2, 16, v1
+; GFX7-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX7-NEXT:    v_lshrrev_b32_e32 v3, 24, v1
+; GFX7-NEXT:    v_and_b32_e32 v2, s4, v2
+; GFX7-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX7-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
+; GFX7-NEXT:    v_and_b32_e32 v3, s4, v3
+; GFX7-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 24, v3
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+  %vector = load <16 x i8>, <16 x i8> addrspace(1)* %ptr
+  %element = extractelement <16 x i8> %vector, i32 6
+  ret i8 %element
+}
+
+define i8 @extractelement_vgpr_v16i8_idx7(<16 x i8> addrspace(1)* %ptr) {
+; GFX9-LABEL: extractelement_vgpr_v16i8_idx7:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dwordx4 v[0:3], v[0:1], off
+; GFX9-NEXT:    s_mov_b32 s4, 8
+; GFX9-NEXT:    s_movk_i32 s5, 0xff
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 8, v1
+; GFX9-NEXT:    v_lshlrev_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_and_b32_sdwa v2, v1, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    v_and_or_b32 v0, v1, s5, v0
+; GFX9-NEXT:    v_and_b32_sdwa v1, v1, s5 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX9-NEXT:    v_or3_b32 v0, v0, v2, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 24, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: extractelement_vgpr_v16i8_idx7:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
+; GFX8-NEXT:    s_movk_i32 s4, 0xff
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v0, 8
+; GFX8-NEXT:    v_mov_b32_e32 v2, s4
+; GFX8-NEXT:    v_lshrrev_b32_e32 v3, 8, v1
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT:    v_and_b32_sdwa v4, v1, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v4
+; GFX8-NEXT:    v_and_b32_sdwa v1, v1, v2 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    v_lshrrev_b32_e32 v0, 24, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: extractelement_vgpr_v16i8_idx7:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    s_mov_b32 s6, 0
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:    buffer_load_dwordx4 v[0:3], v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT:    s_movk_i32 s4, 0xff
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_lshrrev_b32_e32 v0, 8, v1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v2, 16, v1
+; GFX7-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX7-NEXT:    v_lshrrev_b32_e32 v3, 24, v1
+; GFX7-NEXT:    v_and_b32_e32 v2, s4, v2
+; GFX7-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX7-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
+; GFX7-NEXT:    v_and_b32_e32 v3, s4, v3
+; GFX7-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 24, v3
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v0, 24, v0
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+  %vector = load <16 x i8>, <16 x i8> addrspace(1)* %ptr
+  %element = extractelement <16 x i8> %vector, i32 7
+  ret i8 %element
+}
+
+define i8 @extractelement_vgpr_v16i8_idx8(<16 x i8> addrspace(1)* %ptr) {
+; GFX9-LABEL: extractelement_vgpr_v16i8_idx8:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dwordx4 v[0:3], v[0:1], off
+; GFX9-NEXT:    s_mov_b32 s4, 8
+; GFX9-NEXT:    s_movk_i32 s5, 0xff
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 8, v2
+; GFX9-NEXT:    v_lshlrev_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_and_b32_sdwa v1, v2, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    v_and_or_b32 v0, v2, s5, v0
+; GFX9-NEXT:    v_and_b32_sdwa v2, v2, s5 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX9-NEXT:    v_or3_b32 v0, v0, v1, v2
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: extractelement_vgpr_v16i8_idx8:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
+; GFX8-NEXT:    s_movk_i32 s4, 0xff
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v0, 8
+; GFX8-NEXT:    v_mov_b32_e32 v1, s4
+; GFX8-NEXT:    v_lshrrev_b32_e32 v3, 8, v2
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT:    v_and_b32_sdwa v4, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v4
+; GFX8-NEXT:    v_and_b32_sdwa v1, v2, v1 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: extractelement_vgpr_v16i8_idx8:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    s_mov_b32 s6, 0
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:    buffer_load_dwordx4 v[0:3], v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT:    s_movk_i32 s4, 0xff
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_lshrrev_b32_e32 v0, 8, v2
+; GFX7-NEXT:    v_lshrrev_b32_e32 v1, 16, v2
+; GFX7-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX7-NEXT:    v_lshrrev_b32_e32 v3, 24, v2
+; GFX7-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX7-NEXT:    v_and_b32_e32 v2, s4, v2
+; GFX7-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
+; GFX7-NEXT:    v_and_b32_e32 v3, s4, v3
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-NEXT:    v_or_b32_e32 v0, v2, v0
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 24, v3
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+  %vector = load <16 x i8>, <16 x i8> addrspace(1)* %ptr
+  %element = extractelement <16 x i8> %vector, i32 8
+  ret i8 %element
+}
+
+define i8 @extractelement_vgpr_v16i8_idx9(<16 x i8> addrspace(1)* %ptr) {
+; GFX9-LABEL: extractelement_vgpr_v16i8_idx9:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dwordx4 v[0:3], v[0:1], off
+; GFX9-NEXT:    s_mov_b32 s4, 8
+; GFX9-NEXT:    s_movk_i32 s5, 0xff
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 8, v2
+; GFX9-NEXT:    v_lshlrev_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_and_b32_sdwa v1, v2, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    v_and_or_b32 v0, v2, s5, v0
+; GFX9-NEXT:    v_and_b32_sdwa v2, v2, s5 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX9-NEXT:    v_or3_b32 v0, v0, v1, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 8, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: extractelement_vgpr_v16i8_idx9:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
+; GFX8-NEXT:    s_movk_i32 s4, 0xff
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v0, 8
+; GFX8-NEXT:    v_mov_b32_e32 v1, s4
+; GFX8-NEXT:    v_lshrrev_b32_e32 v3, 8, v2
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT:    v_and_b32_sdwa v4, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v4
+; GFX8-NEXT:    v_and_b32_sdwa v1, v2, v1 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    v_lshrrev_b32_e32 v0, 8, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: extractelement_vgpr_v16i8_idx9:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    s_mov_b32 s6, 0
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:    buffer_load_dwordx4 v[0:3], v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT:    s_movk_i32 s4, 0xff
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_lshrrev_b32_e32 v0, 8, v2
+; GFX7-NEXT:    v_lshrrev_b32_e32 v1, 16, v2
+; GFX7-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX7-NEXT:    v_lshrrev_b32_e32 v3, 24, v2
+; GFX7-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX7-NEXT:    v_and_b32_e32 v2, s4, v2
+; GFX7-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
+; GFX7-NEXT:    v_and_b32_e32 v3, s4, v3
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-NEXT:    v_or_b32_e32 v0, v2, v0
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 24, v3
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v0, 8, v0
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+  %vector = load <16 x i8>, <16 x i8> addrspace(1)* %ptr
+  %element = extractelement <16 x i8> %vector, i32 9
+  ret i8 %element
+}
+
+define i8 @extractelement_vgpr_v16i8_idx10(<16 x i8> addrspace(1)* %ptr) {
+; GFX9-LABEL: extractelement_vgpr_v16i8_idx10:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dwordx4 v[0:3], v[0:1], off
+; GFX9-NEXT:    s_mov_b32 s4, 8
+; GFX9-NEXT:    s_movk_i32 s5, 0xff
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 8, v2
+; GFX9-NEXT:    v_lshlrev_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_and_b32_sdwa v1, v2, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    v_and_or_b32 v0, v2, s5, v0
+; GFX9-NEXT:    v_and_b32_sdwa v2, v2, s5 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX9-NEXT:    v_or3_b32 v0, v0, v1, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: extractelement_vgpr_v16i8_idx10:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
+; GFX8-NEXT:    s_movk_i32 s4, 0xff
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v0, 8
+; GFX8-NEXT:    v_mov_b32_e32 v1, s4
+; GFX8-NEXT:    v_lshrrev_b32_e32 v3, 8, v2
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT:    v_and_b32_sdwa v4, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v4
+; GFX8-NEXT:    v_and_b32_sdwa v1, v2, v1 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: extractelement_vgpr_v16i8_idx10:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    s_mov_b32 s6, 0
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:    buffer_load_dwordx4 v[0:3], v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT:    s_movk_i32 s4, 0xff
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_lshrrev_b32_e32 v0, 8, v2
+; GFX7-NEXT:    v_lshrrev_b32_e32 v1, 16, v2
+; GFX7-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX7-NEXT:    v_lshrrev_b32_e32 v3, 24, v2
+; GFX7-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX7-NEXT:    v_and_b32_e32 v2, s4, v2
+; GFX7-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
+; GFX7-NEXT:    v_and_b32_e32 v3, s4, v3
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-NEXT:    v_or_b32_e32 v0, v2, v0
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 24, v3
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+  %vector = load <16 x i8>, <16 x i8> addrspace(1)* %ptr
+  %element = extractelement <16 x i8> %vector, i32 10
+  ret i8 %element
+}
+
+define i8 @extractelement_vgpr_v16i8_idx11(<16 x i8> addrspace(1)* %ptr) {
+; GFX9-LABEL: extractelement_vgpr_v16i8_idx11:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dwordx4 v[0:3], v[0:1], off
+; GFX9-NEXT:    s_mov_b32 s4, 8
+; GFX9-NEXT:    s_movk_i32 s5, 0xff
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 8, v2
+; GFX9-NEXT:    v_lshlrev_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_and_b32_sdwa v1, v2, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    v_and_or_b32 v0, v2, s5, v0
+; GFX9-NEXT:    v_and_b32_sdwa v2, v2, s5 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX9-NEXT:    v_or3_b32 v0, v0, v1, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 24, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: extractelement_vgpr_v16i8_idx11:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
+; GFX8-NEXT:    s_movk_i32 s4, 0xff
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v0, 8
+; GFX8-NEXT:    v_mov_b32_e32 v1, s4
+; GFX8-NEXT:    v_lshrrev_b32_e32 v3, 8, v2
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT:    v_and_b32_sdwa v4, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v4
+; GFX8-NEXT:    v_and_b32_sdwa v1, v2, v1 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    v_lshrrev_b32_e32 v0, 24, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: extractelement_vgpr_v16i8_idx11:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    s_mov_b32 s6, 0
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:    buffer_load_dwordx4 v[0:3], v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT:    s_movk_i32 s4, 0xff
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_lshrrev_b32_e32 v0, 8, v2
+; GFX7-NEXT:    v_lshrrev_b32_e32 v1, 16, v2
+; GFX7-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX7-NEXT:    v_lshrrev_b32_e32 v3, 24, v2
+; GFX7-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX7-NEXT:    v_and_b32_e32 v2, s4, v2
+; GFX7-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
+; GFX7-NEXT:    v_and_b32_e32 v3, s4, v3
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-NEXT:    v_or_b32_e32 v0, v2, v0
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 24, v3
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v0, 24, v0
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+  %vector = load <16 x i8>, <16 x i8> addrspace(1)* %ptr
+  %element = extractelement <16 x i8> %vector, i32 11
+  ret i8 %element
+}
+
+define i8 @extractelement_vgpr_v16i8_idx12(<16 x i8> addrspace(1)* %ptr) {
+; GFX9-LABEL: extractelement_vgpr_v16i8_idx12:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dwordx4 v[0:3], v[0:1], off
+; GFX9-NEXT:    s_mov_b32 s4, 8
+; GFX9-NEXT:    s_movk_i32 s5, 0xff
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 8, v3
+; GFX9-NEXT:    v_lshlrev_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_and_b32_sdwa v1, v3, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    v_and_or_b32 v0, v3, s5, v0
+; GFX9-NEXT:    v_and_b32_sdwa v2, v3, s5 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX9-NEXT:    v_or3_b32 v0, v0, v1, v2
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: extractelement_vgpr_v16i8_idx12:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
+; GFX8-NEXT:    s_movk_i32 s4, 0xff
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v0, 8
+; GFX8-NEXT:    v_mov_b32_e32 v1, s4
+; GFX8-NEXT:    v_lshrrev_b32_e32 v2, 8, v3
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT:    v_and_b32_sdwa v4, v3, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v4
+; GFX8-NEXT:    v_and_b32_sdwa v1, v3, v1 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: extractelement_vgpr_v16i8_idx12:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    s_mov_b32 s6, 0
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:    buffer_load_dwordx4 v[0:3], v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT:    s_movk_i32 s4, 0xff
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_lshrrev_b32_e32 v0, 8, v3
+; GFX7-NEXT:    v_lshrrev_b32_e32 v1, 16, v3
+; GFX7-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX7-NEXT:    v_lshrrev_b32_e32 v2, 24, v3
+; GFX7-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX7-NEXT:    v_and_b32_e32 v3, s4, v3
+; GFX7-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
+; GFX7-NEXT:    v_and_b32_e32 v2, s4, v2
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-NEXT:    v_or_b32_e32 v0, v3, v0
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 24, v2
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+  %vector = load <16 x i8>, <16 x i8> addrspace(1)* %ptr
+  %element = extractelement <16 x i8> %vector, i32 12
+  ret i8 %element
+}
+
+define i8 @extractelement_vgpr_v16i8_idx13(<16 x i8> addrspace(1)* %ptr) {
+; GFX9-LABEL: extractelement_vgpr_v16i8_idx13:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dwordx4 v[0:3], v[0:1], off
+; GFX9-NEXT:    s_mov_b32 s4, 8
+; GFX9-NEXT:    s_movk_i32 s5, 0xff
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 8, v3
+; GFX9-NEXT:    v_lshlrev_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_and_b32_sdwa v1, v3, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    v_and_or_b32 v0, v3, s5, v0
+; GFX9-NEXT:    v_and_b32_sdwa v2, v3, s5 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX9-NEXT:    v_or3_b32 v0, v0, v1, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 8, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: extractelement_vgpr_v16i8_idx13:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
+; GFX8-NEXT:    s_movk_i32 s4, 0xff
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v0, 8
+; GFX8-NEXT:    v_mov_b32_e32 v1, s4
+; GFX8-NEXT:    v_lshrrev_b32_e32 v2, 8, v3
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT:    v_and_b32_sdwa v4, v3, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v4
+; GFX8-NEXT:    v_and_b32_sdwa v1, v3, v1 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    v_lshrrev_b32_e32 v0, 8, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: extractelement_vgpr_v16i8_idx13:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    s_mov_b32 s6, 0
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:    buffer_load_dwordx4 v[0:3], v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT:    s_movk_i32 s4, 0xff
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_lshrrev_b32_e32 v0, 8, v3
+; GFX7-NEXT:    v_lshrrev_b32_e32 v1, 16, v3
+; GFX7-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX7-NEXT:    v_lshrrev_b32_e32 v2, 24, v3
+; GFX7-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX7-NEXT:    v_and_b32_e32 v3, s4, v3
+; GFX7-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
+; GFX7-NEXT:    v_and_b32_e32 v2, s4, v2
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-NEXT:    v_or_b32_e32 v0, v3, v0
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 24, v2
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v0, 8, v0
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+  %vector = load <16 x i8>, <16 x i8> addrspace(1)* %ptr
+  %element = extractelement <16 x i8> %vector, i32 13
+  ret i8 %element
+}
+
+define i8 @extractelement_vgpr_v16i8_idx14(<16 x i8> addrspace(1)* %ptr) {
+; GFX9-LABEL: extractelement_vgpr_v16i8_idx14:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dwordx4 v[0:3], v[0:1], off
+; GFX9-NEXT:    s_mov_b32 s4, 8
+; GFX9-NEXT:    s_movk_i32 s5, 0xff
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 8, v3
+; GFX9-NEXT:    v_lshlrev_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_and_b32_sdwa v1, v3, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    v_and_or_b32 v0, v3, s5, v0
+; GFX9-NEXT:    v_and_b32_sdwa v2, v3, s5 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX9-NEXT:    v_or3_b32 v0, v0, v1, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: extractelement_vgpr_v16i8_idx14:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
+; GFX8-NEXT:    s_movk_i32 s4, 0xff
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v0, 8
+; GFX8-NEXT:    v_mov_b32_e32 v1, s4
+; GFX8-NEXT:    v_lshrrev_b32_e32 v2, 8, v3
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT:    v_and_b32_sdwa v4, v3, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v4
+; GFX8-NEXT:    v_and_b32_sdwa v1, v3, v1 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: extractelement_vgpr_v16i8_idx14:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    s_mov_b32 s6, 0
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:    buffer_load_dwordx4 v[0:3], v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT:    s_movk_i32 s4, 0xff
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_lshrrev_b32_e32 v0, 8, v3
+; GFX7-NEXT:    v_lshrrev_b32_e32 v1, 16, v3
+; GFX7-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX7-NEXT:    v_lshrrev_b32_e32 v2, 24, v3
+; GFX7-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX7-NEXT:    v_and_b32_e32 v3, s4, v3
+; GFX7-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
+; GFX7-NEXT:    v_and_b32_e32 v2, s4, v2
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-NEXT:    v_or_b32_e32 v0, v3, v0
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 24, v2
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+  %vector = load <16 x i8>, <16 x i8> addrspace(1)* %ptr
+  %element = extractelement <16 x i8> %vector, i32 14
+  ret i8 %element
+}
+
+define i8 @extractelement_vgpr_v16i8_idx15(<16 x i8> addrspace(1)* %ptr) {
+; GFX9-LABEL: extractelement_vgpr_v16i8_idx15:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dwordx4 v[0:3], v[0:1], off
+; GFX9-NEXT:    s_mov_b32 s4, 8
+; GFX9-NEXT:    s_movk_i32 s5, 0xff
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 8, v3
+; GFX9-NEXT:    v_lshlrev_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_and_b32_sdwa v1, v3, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    v_and_or_b32 v0, v3, s5, v0
+; GFX9-NEXT:    v_and_b32_sdwa v2, v3, s5 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX9-NEXT:    v_or3_b32 v0, v0, v1, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 24, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: extractelement_vgpr_v16i8_idx15:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
+; GFX8-NEXT:    s_movk_i32 s4, 0xff
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v0, 8
+; GFX8-NEXT:    v_mov_b32_e32 v1, s4
+; GFX8-NEXT:    v_lshrrev_b32_e32 v2, 8, v3
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT:    v_and_b32_sdwa v4, v3, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v4
+; GFX8-NEXT:    v_and_b32_sdwa v1, v3, v1 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    v_lshrrev_b32_e32 v0, 24, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-LABEL: extractelement_vgpr_v16i8_idx15:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    s_mov_b32 s6, 0
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:    buffer_load_dwordx4 v[0:3], v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT:    s_movk_i32 s4, 0xff
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_lshrrev_b32_e32 v0, 8, v3
+; GFX7-NEXT:    v_lshrrev_b32_e32 v1, 16, v3
+; GFX7-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX7-NEXT:    v_lshrrev_b32_e32 v2, 24, v3
+; GFX7-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX7-NEXT:    v_and_b32_e32 v3, s4, v3
+; GFX7-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
+; GFX7-NEXT:    v_and_b32_e32 v2, s4, v2
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-NEXT:    v_or_b32_e32 v0, v3, v0
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 24, v2
+; GFX7-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v0, 24, v0
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+  %vector = load <16 x i8>, <16 x i8> addrspace(1)* %ptr
+  %element = extractelement <16 x i8> %vector, i32 15
+  ret i8 %element
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-extract-vector-elt.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-extract-vector-elt.mir
index 1d3311673cf6c..b548ff5503435 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-extract-vector-elt.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-extract-vector-elt.mir
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -run-pass=legalizer %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -run-pass=legalizer -global-isel-abort=0 %s -o - | FileCheck %s
 
 ---
 name: extract_vector_elt_0_v2i32
@@ -228,8 +228,10 @@ body: |
 
     ; CHECK-LABEL: name: extract_vector_elt_0_v2i16_i32
     ; CHECK: [[DEF:%[0-9]+]]:_(<2 x s16>) = G_IMPLICIT_DEF
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; CHECK: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[DEF]](<2 x s16>)
-    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
+    ; CHECK: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
     ; CHECK: $vgpr0 = COPY [[COPY]](s32)
     %0:_(<2 x s16>) = G_IMPLICIT_DEF
     %1:_(s32) = G_CONSTANT i32 0
@@ -290,25 +292,155 @@ name: extract_vector_elt_v2s8_varidx_i32
 
 body: |
   bb.0:
-    liveins: $vgpr0_vgpr1, $vgpr2
+    liveins: $vgpr0, $vgpr1
 
     ; CHECK-LABEL: name: extract_vector_elt_v2s8_varidx_i32
-    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
-    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
-    ; CHECK: [[COPY2:%[0-9]+]]:_(<2 x s32>) = COPY [[COPY]](<2 x s32>)
-    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY2]](<2 x s32>)
-    ; CHECK: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[UV]], 8
-    ; CHECK: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[UV1]], 8
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; CHECK: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C]](s32)
+    ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CHECK: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C1]](s32)
+    ; CHECK: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; CHECK: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C2]](s32)
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; CHECK: [[COPY4:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+    ; CHECK: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY4]], 8
+    ; CHECK: [[COPY5:%[0-9]+]]:_(s32) = COPY [[COPY3]](s32)
+    ; CHECK: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY5]], 8
     ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SEXT_INREG]](s32), [[SEXT_INREG1]](s32)
     ; CHECK: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[BUILD_VECTOR]](<2 x s32>), [[COPY1]](s32)
-    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[EVEC]](s32)
-    ; CHECK: $vgpr0 = COPY [[COPY3]](s32)
-    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
-    %1:_(s32) = COPY $vgpr2
-    %2:_(<2 x s8>) = G_TRUNC %0
-    %3:_(s8) = G_EXTRACT_VECTOR_ELT %2, %1
-    %4:_(s32) = G_ANYEXT %3
-    $vgpr0 = COPY %4
+    ; CHECK: [[COPY6:%[0-9]+]]:_(s32) = COPY [[EVEC]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY6]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s16) = G_TRUNC %0
+    %3:_(<2 x s8>) = G_BITCAST %2
+    %4:_(s8) = G_EXTRACT_VECTOR_ELT %3, %1
+    %5:_(s32) = G_ANYEXT %4
+    $vgpr0 = COPY %5
+...
+
+---
+name: extract_vector_elt_v2s8_constidx_0_i32
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: extract_vector_elt_v2s8_constidx_0_i32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; CHECK: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C]](s32)
+    ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CHECK: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C1]](s32)
+    ; CHECK: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; CHECK: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C2]](s32)
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; CHECK: [[COPY4:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+    ; CHECK: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY4]], 8
+    ; CHECK: [[COPY5:%[0-9]+]]:_(s32) = COPY [[COPY3]](s32)
+    ; CHECK: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY5]], 8
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SEXT_INREG]](s32), [[SEXT_INREG1]](s32)
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(s32) = G_EXTRACT [[BUILD_VECTOR]](<2 x s32>), 0
+    ; CHECK: [[COPY6:%[0-9]+]]:_(s32) = COPY [[EXTRACT]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY6]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s16) = G_TRUNC %0
+    %3:_(<2 x s8>) = G_BITCAST %2
+    %4:_(s32) = G_CONSTANT i32 0
+    %5:_(s8) = G_EXTRACT_VECTOR_ELT %3, %4
+    %6:_(s32) = G_ANYEXT %5
+    $vgpr0 = COPY %6
+...
+
+---
+name: extract_vector_elt_v2s8_constidx_1_i32
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: extract_vector_elt_v2s8_constidx_1_i32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; CHECK: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C]](s32)
+    ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CHECK: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C1]](s32)
+    ; CHECK: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; CHECK: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C2]](s32)
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; CHECK: [[COPY4:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+    ; CHECK: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY4]], 8
+    ; CHECK: [[COPY5:%[0-9]+]]:_(s32) = COPY [[COPY3]](s32)
+    ; CHECK: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY5]], 8
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SEXT_INREG]](s32), [[SEXT_INREG1]](s32)
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(s32) = G_EXTRACT [[BUILD_VECTOR]](<2 x s32>), 32
+    ; CHECK: [[COPY6:%[0-9]+]]:_(s32) = COPY [[EXTRACT]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY6]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s16) = G_TRUNC %0
+    %3:_(<2 x s8>) = G_BITCAST %2
+    %4:_(s32) = G_CONSTANT i32 1
+    %5:_(s8) = G_EXTRACT_VECTOR_ELT %3, %4
+    %6:_(s32) = G_ANYEXT %5
+    $vgpr0 = COPY %6
+...
+
+---
+name: extract_vector_elt_v4s4_varidx_i32
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; CHECK-LABEL: name: extract_vector_elt_v4s4_varidx_i32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; CHECK: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C]](s32)
+    ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; CHECK: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C1]](s32)
+    ; CHECK: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+    ; CHECK: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C2]](s32)
+    ; CHECK: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CHECK: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C3]](s32)
+    ; CHECK: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+    ; CHECK: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C4]](s32)
+    ; CHECK: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; CHECK: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C5]](s32)
+    ; CHECK: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 28
+    ; CHECK: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C6]](s32)
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; CHECK: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+    ; CHECK: [[COPY5:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
+    ; CHECK: [[COPY6:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+    ; CHECK: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY6]], 4
+    ; CHECK: [[COPY7:%[0-9]+]]:_(s32) = COPY [[COPY3]](s32)
+    ; CHECK: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY7]], 4
+    ; CHECK: [[COPY8:%[0-9]+]]:_(s32) = COPY [[COPY4]](s32)
+    ; CHECK: [[SEXT_INREG2:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY8]], 4
+    ; CHECK: [[COPY9:%[0-9]+]]:_(s32) = COPY [[COPY5]](s32)
+    ; CHECK: [[SEXT_INREG3:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY9]], 4
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[SEXT_INREG]](s32), [[SEXT_INREG1]](s32), [[SEXT_INREG2]](s32), [[SEXT_INREG3]](s32)
+    ; CHECK: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[BUILD_VECTOR]](<4 x s32>), [[COPY1]](s32)
+    ; CHECK: [[COPY10:%[0-9]+]]:_(s32) = COPY [[EVEC]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY10]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s16) = G_TRUNC %0
+    %3:_(<4 x s4>) = G_BITCAST %2
+    %4:_(s4) = G_EXTRACT_VECTOR_ELT %3, %1
+    %5:_(s32) = G_ANYEXT %4
+    $vgpr0 = COPY %5
 ...
 
 ---
@@ -343,24 +475,559 @@ name: extract_vector_elt_v4s8_varidx_i32
 
 body: |
   bb.0:
-    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4
+    liveins: $vgpr0, $vgpr1
 
     ; CHECK-LABEL: name: extract_vector_elt_v4s8_varidx_i32
-    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
-    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr4
-    ; CHECK: [[COPY2:%[0-9]+]]:_(<4 x s32>) = COPY [[COPY]](<4 x s32>)
-    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY2]](<4 x s32>)
-    ; CHECK: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[UV]], 8
-    ; CHECK: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[UV1]], 8
-    ; CHECK: [[SEXT_INREG2:%[0-9]+]]:_(s32) = G_SEXT_INREG [[UV2]], 8
-    ; CHECK: [[SEXT_INREG3:%[0-9]+]]:_(s32) = G_SEXT_INREG [[UV3]], 8
-    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[SEXT_INREG]](s32), [[SEXT_INREG1]](s32), [[SEXT_INREG2]](s32), [[SEXT_INREG3]](s32)
-    ; CHECK: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[BUILD_VECTOR]](<4 x s32>), [[COPY1]](s32)
-    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[EVEC]](s32)
-    ; CHECK: $vgpr0 = COPY [[COPY3]](s32)
-    %0:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
-    %1:_(s32) = COPY $vgpr4
-    %2:_(<4 x s8>) = G_TRUNC %0
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; CHECK: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C]](s32)
+    ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CHECK: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C1]](s32)
+    ; CHECK: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; CHECK: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C2]](s32)
+    ; CHECK: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C3]]
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; CHECK: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C3]]
+    ; CHECK: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+    ; CHECK: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
+    ; CHECK: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+    ; CHECK: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C3]]
+    ; CHECK: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C1]](s32)
+    ; CHECK: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+    ; CHECK: [[COPY5:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
+    ; CHECK: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY5]], [[C3]]
+    ; CHECK: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C2]](s32)
+    ; CHECK: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[SHL2]]
+    ; CHECK: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
+    ; CHECK: [[AND4:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C4]]
+    ; CHECK: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[AND4]], [[C4]](s32)
+    ; CHECK: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[OR2]], [[SHL3]](s32)
+    ; CHECK: [[COPY6:%[0-9]+]]:_(s32) = COPY [[LSHR3]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY6]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(<4 x s8>) = G_BITCAST %0
+    %3:_(s8) = G_EXTRACT_VECTOR_ELT %2, %1
+    %4:_(s32) = G_ANYEXT %3
+    $vgpr0 = COPY %4
+...
+
+---
+name: extract_vector_elt_v4s8_constidx_0_i32
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: extract_vector_elt_v4s8_constidx_0_i32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; CHECK: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C]](s32)
+    ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CHECK: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C1]](s32)
+    ; CHECK: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; CHECK: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C2]](s32)
+    ; CHECK: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C4]]
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; CHECK: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C4]]
+    ; CHECK: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+    ; CHECK: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+    ; CHECK: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C4]]
+    ; CHECK: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C1]](s32)
+    ; CHECK: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+    ; CHECK: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
+    ; CHECK: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C4]]
+    ; CHECK: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C2]](s32)
+    ; CHECK: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[SHL2]]
+    ; CHECK: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[OR2]], [[C3]](s32)
+    ; CHECK: [[COPY5:%[0-9]+]]:_(s32) = COPY [[LSHR3]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY5]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(<4 x s8>) = G_BITCAST %0
+    %2:_(s32) = G_CONSTANT i32 0
+    %3:_(s8) = G_EXTRACT_VECTOR_ELT %1, %2
+    %4:_(s32) = G_ANYEXT %3
+    $vgpr0 = COPY %4
+...
+
+---
+name: extract_vector_elt_v4s8_constidx_1_i32
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: extract_vector_elt_v4s8_constidx_1_i32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; CHECK: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C]](s32)
+    ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CHECK: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C1]](s32)
+    ; CHECK: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; CHECK: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C2]](s32)
+    ; CHECK: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C3]]
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; CHECK: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C3]]
+    ; CHECK: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+    ; CHECK: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+    ; CHECK: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C3]]
+    ; CHECK: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C1]](s32)
+    ; CHECK: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+    ; CHECK: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
+    ; CHECK: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C3]]
+    ; CHECK: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C2]](s32)
+    ; CHECK: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[SHL2]]
+    ; CHECK: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[OR2]], [[C]](s32)
+    ; CHECK: [[COPY5:%[0-9]+]]:_(s32) = COPY [[LSHR3]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY5]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(<4 x s8>) = G_BITCAST %0
+    %2:_(s32) = G_CONSTANT i32 1
+    %3:_(s8) = G_EXTRACT_VECTOR_ELT %1, %2
+    %4:_(s32) = G_ANYEXT %3
+    $vgpr0 = COPY %4
+...
+
+---
+name: extract_vector_elt_v4s8_constidx_2_i32
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: extract_vector_elt_v4s8_constidx_2_i32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; CHECK: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C]](s32)
+    ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CHECK: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C1]](s32)
+    ; CHECK: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; CHECK: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C2]](s32)
+    ; CHECK: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C3]]
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; CHECK: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C3]]
+    ; CHECK: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+    ; CHECK: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+    ; CHECK: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C3]]
+    ; CHECK: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C1]](s32)
+    ; CHECK: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+    ; CHECK: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
+    ; CHECK: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C3]]
+    ; CHECK: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C2]](s32)
+    ; CHECK: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[SHL2]]
+    ; CHECK: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[OR2]], [[C1]](s32)
+    ; CHECK: [[COPY5:%[0-9]+]]:_(s32) = COPY [[LSHR3]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY5]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(<4 x s8>) = G_BITCAST %0
+    %2:_(s32) = G_CONSTANT i32 2
+    %3:_(s8) = G_EXTRACT_VECTOR_ELT %1, %2
+    %4:_(s32) = G_ANYEXT %3
+    $vgpr0 = COPY %4
+...
+
+---
+name: extract_vector_elt_v4s8_constidx_3_i32
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: extract_vector_elt_v4s8_constidx_3_i32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; CHECK: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C]](s32)
+    ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CHECK: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C1]](s32)
+    ; CHECK: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; CHECK: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C2]](s32)
+    ; CHECK: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C3]]
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; CHECK: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C3]]
+    ; CHECK: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+    ; CHECK: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+    ; CHECK: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C3]]
+    ; CHECK: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C1]](s32)
+    ; CHECK: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+    ; CHECK: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
+    ; CHECK: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C3]]
+    ; CHECK: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C2]](s32)
+    ; CHECK: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[SHL2]]
+    ; CHECK: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[OR2]], [[C2]](s32)
+    ; CHECK: [[COPY5:%[0-9]+]]:_(s32) = COPY [[LSHR3]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY5]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(<4 x s8>) = G_BITCAST %0
+    %2:_(s32) = G_CONSTANT i32 3
+    %3:_(s8) = G_EXTRACT_VECTOR_ELT %1, %2
+    %4:_(s32) = G_ANYEXT %3
+    $vgpr0 = COPY %4
+...
+
+
+
+---
+name: extract_vector_elt_v8s8_varidx_i32
+
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2
+
+    ; CHECK-LABEL: name: extract_vector_elt_v8s8_varidx_i32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[UV]](s32)
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CHECK: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[UV]], [[C]](s32)
+    ; CHECK: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; CHECK: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[UV1]](s32)
+    ; CHECK: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[UV1]], [[C]](s32)
+    ; CHECK: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+    ; CHECK: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; CHECK: [[LSHR2:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC]], [[C1]](s16)
+    ; CHECK: [[LSHR3:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC1]], [[C1]](s16)
+    ; CHECK: [[LSHR4:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC2]], [[C1]](s16)
+    ; CHECK: [[LSHR5:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC3]], [[C1]](s16)
+    ; CHECK: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[UV]](s32)
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C2]]
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR2]](s16)
+    ; CHECK: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C2]]
+    ; CHECK: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; CHECK: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C3]](s32)
+    ; CHECK: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; CHECK: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C2]]
+    ; CHECK: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C]](s32)
+    ; CHECK: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR3]](s16)
+    ; CHECK: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C2]]
+    ; CHECK: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; CHECK: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C4]](s32)
+    ; CHECK: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[SHL2]]
+    ; CHECK: [[COPY4:%[0-9]+]]:_(s32) = COPY [[UV1]](s32)
+    ; CHECK: [[AND4:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C2]]
+    ; CHECK: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR4]](s16)
+    ; CHECK: [[AND5:%[0-9]+]]:_(s32) = G_AND [[ANYEXT2]], [[C2]]
+    ; CHECK: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C3]](s32)
+    ; CHECK: [[OR3:%[0-9]+]]:_(s32) = G_OR [[AND4]], [[SHL3]]
+    ; CHECK: [[COPY5:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+    ; CHECK: [[AND6:%[0-9]+]]:_(s32) = G_AND [[COPY5]], [[C2]]
+    ; CHECK: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[AND6]], [[C]](s32)
+    ; CHECK: [[OR4:%[0-9]+]]:_(s32) = G_OR [[OR3]], [[SHL4]]
+    ; CHECK: [[ANYEXT3:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR5]](s16)
+    ; CHECK: [[AND7:%[0-9]+]]:_(s32) = G_AND [[ANYEXT3]], [[C2]]
+    ; CHECK: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[AND7]], [[C4]](s32)
+    ; CHECK: [[OR5:%[0-9]+]]:_(s32) = G_OR [[OR4]], [[SHL5]]
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32)
+    ; CHECK: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; CHECK: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C5]](s32)
+    ; CHECK: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[BUILD_VECTOR]](<2 x s32>), [[LSHR6]](s32)
+    ; CHECK: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
+    ; CHECK: [[AND8:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C6]]
+    ; CHECK: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[AND8]], [[C6]](s32)
+    ; CHECK: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[EVEC]], [[SHL6]](s32)
+    ; CHECK: [[COPY6:%[0-9]+]]:_(s32) = COPY [[LSHR7]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY6]](s32)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(s32) = COPY $vgpr2
+    %2:_(<8 x s8>) = G_BITCAST %0
+    %3:_(s8) = G_EXTRACT_VECTOR_ELT %2, %1
+    %4:_(s32) = G_ANYEXT %3
+    $vgpr0 = COPY %4
+...
+
+
+---
+name: extract_vector_elt_v8s8_constidx_0_i32
+
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: extract_vector_elt_v8s8_constidx_0_i32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[UV]](s32)
+    ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CHECK: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[UV]], [[C1]](s32)
+    ; CHECK: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; CHECK: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; CHECK: [[LSHR1:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC]], [[C2]](s16)
+    ; CHECK: [[LSHR2:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC1]], [[C2]](s16)
+    ; CHECK: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[UV]](s32)
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C3]]
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR1]](s16)
+    ; CHECK: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C3]]
+    ; CHECK: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; CHECK: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C4]](s32)
+    ; CHECK: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; CHECK: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C3]]
+    ; CHECK: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C1]](s32)
+    ; CHECK: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR2]](s16)
+    ; CHECK: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C3]]
+    ; CHECK: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; CHECK: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C5]](s32)
+    ; CHECK: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[SHL2]]
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[OR2]](s32)
+    ; CHECK: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[COPY3]], [[C]](s32)
+    ; CHECK: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LSHR3]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY4]](s32)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(s32) = G_CONSTANT i32 0
+    %2:_(<8 x s8>) = G_BITCAST %0
+    %3:_(s8) = G_EXTRACT_VECTOR_ELT %2, %1
+    %4:_(s32) = G_ANYEXT %3
+    $vgpr0 = COPY %4
+...
+
+---
+name: extract_vector_elt_v8s8_constidx_1_i32
+
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: extract_vector_elt_v8s8_constidx_1_i32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[UV]](s32)
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CHECK: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[UV]], [[C]](s32)
+    ; CHECK: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; CHECK: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; CHECK: [[LSHR1:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC]], [[C1]](s16)
+    ; CHECK: [[LSHR2:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC1]], [[C1]](s16)
+    ; CHECK: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[UV]](s32)
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C2]]
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR1]](s16)
+    ; CHECK: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C2]]
+    ; CHECK: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; CHECK: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C3]](s32)
+    ; CHECK: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; CHECK: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C2]]
+    ; CHECK: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C]](s32)
+    ; CHECK: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR2]](s16)
+    ; CHECK: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C2]]
+    ; CHECK: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; CHECK: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C4]](s32)
+    ; CHECK: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[SHL2]]
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[OR2]](s32)
+    ; CHECK: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[COPY3]], [[C3]](s32)
+    ; CHECK: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LSHR3]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY4]](s32)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(s32) = G_CONSTANT i32 1
+    %2:_(<8 x s8>) = G_BITCAST %0
+    %3:_(s8) = G_EXTRACT_VECTOR_ELT %2, %1
+    %4:_(s32) = G_ANYEXT %3
+    $vgpr0 = COPY %4
+...
+
+---
+name: extract_vector_elt_v8s8_constidx_3_i32
+
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: extract_vector_elt_v8s8_constidx_3_i32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[UV]](s32)
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CHECK: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[UV]], [[C]](s32)
+    ; CHECK: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; CHECK: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; CHECK: [[LSHR1:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC]], [[C1]](s16)
+    ; CHECK: [[LSHR2:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC1]], [[C1]](s16)
+    ; CHECK: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[UV]](s32)
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C2]]
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR1]](s16)
+    ; CHECK: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C2]]
+    ; CHECK: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; CHECK: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C3]](s32)
+    ; CHECK: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; CHECK: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C2]]
+    ; CHECK: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C]](s32)
+    ; CHECK: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR2]](s16)
+    ; CHECK: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C2]]
+    ; CHECK: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; CHECK: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C4]](s32)
+    ; CHECK: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[SHL2]]
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[OR2]](s32)
+    ; CHECK: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[COPY3]], [[C4]](s32)
+    ; CHECK: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LSHR3]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY4]](s32)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(s32) = G_CONSTANT i32 3
+    %2:_(<8 x s8>) = G_BITCAST %0
+    %3:_(s8) = G_EXTRACT_VECTOR_ELT %2, %1
+    %4:_(s32) = G_ANYEXT %3
+    $vgpr0 = COPY %4
+...
+
+---
+name: extract_vector_elt_v8s8_constidx_4_i32
+
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: extract_vector_elt_v8s8_constidx_4_i32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[UV1]](s32)
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CHECK: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[UV1]], [[C]](s32)
+    ; CHECK: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; CHECK: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; CHECK: [[LSHR1:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC]], [[C1]](s16)
+    ; CHECK: [[LSHR2:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC1]], [[C1]](s16)
+    ; CHECK: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[UV1]](s32)
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C2]]
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR1]](s16)
+    ; CHECK: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C2]]
+    ; CHECK: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; CHECK: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C3]](s32)
+    ; CHECK: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; CHECK: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C2]]
+    ; CHECK: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C]](s32)
+    ; CHECK: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR2]](s16)
+    ; CHECK: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C2]]
+    ; CHECK: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; CHECK: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C4]](s32)
+    ; CHECK: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[SHL2]]
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[OR2]](s32)
+    ; CHECK: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[COPY3]], [[C5]](s32)
+    ; CHECK: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LSHR3]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY4]](s32)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(s32) = G_CONSTANT i32 4
+    %2:_(<8 x s8>) = G_BITCAST %0
+    %3:_(s8) = G_EXTRACT_VECTOR_ELT %2, %1
+    %4:_(s32) = G_ANYEXT %3
+    $vgpr0 = COPY %4
+...
+
+---
+name: extract_vector_elt_v8s8_constidx_5_i32
+
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: extract_vector_elt_v8s8_constidx_5_i32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[UV1]](s32)
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CHECK: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[UV1]], [[C]](s32)
+    ; CHECK: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; CHECK: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; CHECK: [[LSHR1:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC]], [[C1]](s16)
+    ; CHECK: [[LSHR2:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC1]], [[C1]](s16)
+    ; CHECK: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[UV1]](s32)
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C2]]
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR1]](s16)
+    ; CHECK: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C2]]
+    ; CHECK: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; CHECK: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C3]](s32)
+    ; CHECK: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; CHECK: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C2]]
+    ; CHECK: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C]](s32)
+    ; CHECK: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR2]](s16)
+    ; CHECK: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C2]]
+    ; CHECK: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; CHECK: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C4]](s32)
+    ; CHECK: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[SHL2]]
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[OR2]](s32)
+    ; CHECK: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[COPY3]], [[C3]](s32)
+    ; CHECK: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LSHR3]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY4]](s32)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(s32) = G_CONSTANT i32 5
+    %2:_(<8 x s8>) = G_BITCAST %0
+    %3:_(s8) = G_EXTRACT_VECTOR_ELT %2, %1
+    %4:_(s32) = G_ANYEXT %3
+    $vgpr0 = COPY %4
+...
+
+---
+name: extract_vector_elt_v8s8_constidx_7_i32
+
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: extract_vector_elt_v8s8_constidx_7_i32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[UV1]](s32)
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CHECK: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[UV1]], [[C]](s32)
+    ; CHECK: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; CHECK: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; CHECK: [[LSHR1:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC]], [[C1]](s16)
+    ; CHECK: [[LSHR2:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC1]], [[C1]](s16)
+    ; CHECK: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[UV1]](s32)
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C2]]
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR1]](s16)
+    ; CHECK: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C2]]
+    ; CHECK: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; CHECK: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C3]](s32)
+    ; CHECK: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; CHECK: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C2]]
+    ; CHECK: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C]](s32)
+    ; CHECK: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR2]](s16)
+    ; CHECK: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C2]]
+    ; CHECK: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; CHECK: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C4]](s32)
+    ; CHECK: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[SHL2]]
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[OR2]](s32)
+    ; CHECK: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[COPY3]], [[C4]](s32)
+    ; CHECK: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LSHR3]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY4]](s32)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(s32) = G_CONSTANT i32 7
+    %2:_(<8 x s8>) = G_BITCAST %0
     %3:_(s8) = G_EXTRACT_VECTOR_ELT %2, %1
     %4:_(s32) = G_ANYEXT %3
     $vgpr0 = COPY %4
@@ -376,9 +1043,14 @@ body: |
     ; CHECK-LABEL: name: extract_vector_elt_v2s16_varidx_i32
     ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
     ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-    ; CHECK: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[COPY]](<2 x s16>), [[COPY1]](s32)
-    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[EVEC]](s16)
-    ; CHECK: $vgpr0 = COPY [[ANYEXT]](s32)
+    ; CHECK: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
+    ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; CHECK: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND]], [[C1]](s32)
+    ; CHECK: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[SHL]](s32)
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY2]](s32)
     %0:_(<2 x s16>) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
     %2:_(s16) = G_EXTRACT_VECTOR_ELT %0, %1
@@ -395,8 +1067,10 @@ body: |
 
     ; CHECK-LABEL: name: extract_vector_elt_v2s16_idx0_i32
     ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; CHECK: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
-    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
+    ; CHECK: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
     ; CHECK: $vgpr0 = COPY [[COPY1]](s32)
     %0:_(<2 x s16>) = COPY $vgpr0
     %1:_(s32) = G_CONSTANT i32 0
@@ -435,8 +1109,11 @@ body: |
 
     ; CHECK-LABEL: name: extract_vector_elt_v2s16_idx2_i32
     ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
-    ; CHECK: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-    ; CHECK: $vgpr0 = COPY [[DEF]](s32)
+    ; CHECK: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY1]](s32)
     %0:_(<2 x s16>) = COPY $vgpr0
     %1:_(s32) = G_CONSTANT i32 2
     %2:_(s16) = G_EXTRACT_VECTOR_ELT %0, %1
@@ -579,9 +1256,16 @@ body: |
     ; CHECK-LABEL: name: extract_vector_elt_v4s16_varidx_i32
     ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
     ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
-    ; CHECK: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[COPY]](<4 x s16>), [[COPY1]](s32)
-    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[EVEC]](s16)
-    ; CHECK: $vgpr0 = COPY [[ANYEXT]](s32)
+    ; CHECK: [[BITCAST:%[0-9]+]]:_(<2 x s32>) = G_BITCAST [[COPY]](<4 x s16>)
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C]](s32)
+    ; CHECK: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[BITCAST]](<2 x s32>), [[LSHR]](s32)
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
+    ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; CHECK: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND]], [[C1]](s32)
+    ; CHECK: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[EVEC]], [[SHL]](s32)
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY2]](s32)
     %0:_(<4 x s16>) = COPY $vgpr0_vgpr1
     %1:_(s32) = COPY $vgpr2
     %2:_(s16) = G_EXTRACT_VECTOR_ELT %0, %1
@@ -599,8 +1283,18 @@ body: |
     ; CHECK-LABEL: name: extract_vector_elt_v2s128_varidx_i32
     ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s128>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
     ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr8
-    ; CHECK: [[EVEC:%[0-9]+]]:_(s128) = G_EXTRACT_VECTOR_ELT [[COPY]](<2 x s128>), [[COPY1]](s32)
-    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[EVEC]](s128)
+    ; CHECK: [[BITCAST:%[0-9]+]]:_(<4 x s64>) = G_BITCAST [[COPY]](<2 x s128>)
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; CHECK: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[COPY1]], [[C]]
+    ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[MUL]], [[C1]]
+    ; CHECK: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[BITCAST]](<4 x s64>), [[ADD]](s32)
+    ; CHECK: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[MUL]], [[C2]]
+    ; CHECK: [[EVEC1:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[BITCAST]](<4 x s64>), [[ADD1]](s32)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[EVEC]](s64), [[EVEC1]](s64)
+    ; CHECK: [[BITCAST1:%[0-9]+]]:_(s128) = G_BITCAST [[BUILD_VECTOR]](<2 x s64>)
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BITCAST1]](s128)
     %0:_(<2 x s128>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
     %1:_(s32) = COPY $vgpr8
     %2:_(s128) = G_EXTRACT_VECTOR_ELT %0, %1
@@ -1146,3 +1840,335 @@ body: |
     %3:_(s32) = G_EXTRACT_VECTOR_ELT %2, %1
     S_ENDPGM 0, implicit %3
 ...
+
+---
+name: extract_vector_elt_v32s1_varidx_i32
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; CHECK-LABEL: name: extract_vector_elt_v32s1_varidx_i32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C]](s32)
+    ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; CHECK: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C1]](s32)
+    ; CHECK: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
+    ; CHECK: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C2]](s32)
+    ; CHECK: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; CHECK: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C3]](s32)
+    ; CHECK: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 5
+    ; CHECK: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C4]](s32)
+    ; CHECK: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 6
+    ; CHECK: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C5]](s32)
+    ; CHECK: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 7
+    ; CHECK: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C6]](s32)
+    ; CHECK: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; CHECK: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C7]](s32)
+    ; CHECK: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 9
+    ; CHECK: [[LSHR8:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C8]](s32)
+    ; CHECK: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+    ; CHECK: [[LSHR9:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C9]](s32)
+    ; CHECK: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 11
+    ; CHECK: [[LSHR10:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C10]](s32)
+    ; CHECK: [[C11:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+    ; CHECK: [[LSHR11:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C11]](s32)
+    ; CHECK: [[C12:%[0-9]+]]:_(s32) = G_CONSTANT i32 13
+    ; CHECK: [[LSHR12:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C12]](s32)
+    ; CHECK: [[C13:%[0-9]+]]:_(s32) = G_CONSTANT i32 14
+    ; CHECK: [[LSHR13:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C13]](s32)
+    ; CHECK: [[C14:%[0-9]+]]:_(s32) = G_CONSTANT i32 15
+    ; CHECK: [[LSHR14:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C14]](s32)
+    ; CHECK: [[C15:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CHECK: [[LSHR15:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C15]](s32)
+    ; CHECK: [[C16:%[0-9]+]]:_(s32) = G_CONSTANT i32 17
+    ; CHECK: [[LSHR16:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C16]](s32)
+    ; CHECK: [[C17:%[0-9]+]]:_(s32) = G_CONSTANT i32 18
+    ; CHECK: [[LSHR17:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C17]](s32)
+    ; CHECK: [[C18:%[0-9]+]]:_(s32) = G_CONSTANT i32 19
+    ; CHECK: [[LSHR18:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C18]](s32)
+    ; CHECK: [[C19:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+    ; CHECK: [[LSHR19:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C19]](s32)
+    ; CHECK: [[C20:%[0-9]+]]:_(s32) = G_CONSTANT i32 21
+    ; CHECK: [[LSHR20:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C20]](s32)
+    ; CHECK: [[C21:%[0-9]+]]:_(s32) = G_CONSTANT i32 22
+    ; CHECK: [[LSHR21:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C21]](s32)
+    ; CHECK: [[C22:%[0-9]+]]:_(s32) = G_CONSTANT i32 23
+    ; CHECK: [[LSHR22:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C22]](s32)
+    ; CHECK: [[C23:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; CHECK: [[LSHR23:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C23]](s32)
+    ; CHECK: [[C24:%[0-9]+]]:_(s32) = G_CONSTANT i32 25
+    ; CHECK: [[LSHR24:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C24]](s32)
+    ; CHECK: [[C25:%[0-9]+]]:_(s32) = G_CONSTANT i32 26
+    ; CHECK: [[LSHR25:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C25]](s32)
+    ; CHECK: [[C26:%[0-9]+]]:_(s32) = G_CONSTANT i32 27
+    ; CHECK: [[LSHR26:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C26]](s32)
+    ; CHECK: [[C27:%[0-9]+]]:_(s32) = G_CONSTANT i32 28
+    ; CHECK: [[LSHR27:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C27]](s32)
+    ; CHECK: [[C28:%[0-9]+]]:_(s32) = G_CONSTANT i32 29
+    ; CHECK: [[LSHR28:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C28]](s32)
+    ; CHECK: [[C29:%[0-9]+]]:_(s32) = G_CONSTANT i32 30
+    ; CHECK: [[LSHR29:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C29]](s32)
+    ; CHECK: [[C30:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
+    ; CHECK: [[LSHR30:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C30]](s32)
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; CHECK: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C]]
+    ; CHECK: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+    ; CHECK: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
+    ; CHECK: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+    ; CHECK: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C]]
+    ; CHECK: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C1]](s32)
+    ; CHECK: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+    ; CHECK: [[COPY5:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
+    ; CHECK: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY5]], [[C]]
+    ; CHECK: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C2]](s32)
+    ; CHECK: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[SHL2]]
+    ; CHECK: [[COPY6:%[0-9]+]]:_(s32) = COPY [[LSHR3]](s32)
+    ; CHECK: [[AND4:%[0-9]+]]:_(s32) = G_AND [[COPY6]], [[C]]
+    ; CHECK: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[AND4]], [[C3]](s32)
+    ; CHECK: [[OR3:%[0-9]+]]:_(s32) = G_OR [[OR2]], [[SHL3]]
+    ; CHECK: [[COPY7:%[0-9]+]]:_(s32) = COPY [[LSHR4]](s32)
+    ; CHECK: [[AND5:%[0-9]+]]:_(s32) = G_AND [[COPY7]], [[C]]
+    ; CHECK: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C4]](s32)
+    ; CHECK: [[OR4:%[0-9]+]]:_(s32) = G_OR [[OR3]], [[SHL4]]
+    ; CHECK: [[COPY8:%[0-9]+]]:_(s32) = COPY [[LSHR5]](s32)
+    ; CHECK: [[AND6:%[0-9]+]]:_(s32) = G_AND [[COPY8]], [[C]]
+    ; CHECK: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[AND6]], [[C5]](s32)
+    ; CHECK: [[OR5:%[0-9]+]]:_(s32) = G_OR [[OR4]], [[SHL5]]
+    ; CHECK: [[COPY9:%[0-9]+]]:_(s32) = COPY [[LSHR6]](s32)
+    ; CHECK: [[AND7:%[0-9]+]]:_(s32) = G_AND [[COPY9]], [[C]]
+    ; CHECK: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[AND7]], [[C6]](s32)
+    ; CHECK: [[OR6:%[0-9]+]]:_(s32) = G_OR [[OR5]], [[SHL6]]
+    ; CHECK: [[COPY10:%[0-9]+]]:_(s32) = COPY [[LSHR7]](s32)
+    ; CHECK: [[AND8:%[0-9]+]]:_(s32) = G_AND [[COPY10]], [[C]]
+    ; CHECK: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[AND8]], [[C7]](s32)
+    ; CHECK: [[OR7:%[0-9]+]]:_(s32) = G_OR [[OR6]], [[SHL7]]
+    ; CHECK: [[COPY11:%[0-9]+]]:_(s32) = COPY [[LSHR8]](s32)
+    ; CHECK: [[AND9:%[0-9]+]]:_(s32) = G_AND [[COPY11]], [[C]]
+    ; CHECK: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[AND9]], [[C8]](s32)
+    ; CHECK: [[OR8:%[0-9]+]]:_(s32) = G_OR [[OR7]], [[SHL8]]
+    ; CHECK: [[COPY12:%[0-9]+]]:_(s32) = COPY [[LSHR9]](s32)
+    ; CHECK: [[AND10:%[0-9]+]]:_(s32) = G_AND [[COPY12]], [[C]]
+    ; CHECK: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[AND10]], [[C9]](s32)
+    ; CHECK: [[OR9:%[0-9]+]]:_(s32) = G_OR [[OR8]], [[SHL9]]
+    ; CHECK: [[COPY13:%[0-9]+]]:_(s32) = COPY [[LSHR10]](s32)
+    ; CHECK: [[AND11:%[0-9]+]]:_(s32) = G_AND [[COPY13]], [[C]]
+    ; CHECK: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[AND11]], [[C10]](s32)
+    ; CHECK: [[OR10:%[0-9]+]]:_(s32) = G_OR [[OR9]], [[SHL10]]
+    ; CHECK: [[COPY14:%[0-9]+]]:_(s32) = COPY [[LSHR11]](s32)
+    ; CHECK: [[AND12:%[0-9]+]]:_(s32) = G_AND [[COPY14]], [[C]]
+    ; CHECK: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[AND12]], [[C11]](s32)
+    ; CHECK: [[OR11:%[0-9]+]]:_(s32) = G_OR [[OR10]], [[SHL11]]
+    ; CHECK: [[COPY15:%[0-9]+]]:_(s32) = COPY [[LSHR12]](s32)
+    ; CHECK: [[AND13:%[0-9]+]]:_(s32) = G_AND [[COPY15]], [[C]]
+    ; CHECK: [[SHL12:%[0-9]+]]:_(s32) = G_SHL [[AND13]], [[C12]](s32)
+    ; CHECK: [[OR12:%[0-9]+]]:_(s32) = G_OR [[OR11]], [[SHL12]]
+    ; CHECK: [[COPY16:%[0-9]+]]:_(s32) = COPY [[LSHR13]](s32)
+    ; CHECK: [[AND14:%[0-9]+]]:_(s32) = G_AND [[COPY16]], [[C]]
+    ; CHECK: [[SHL13:%[0-9]+]]:_(s32) = G_SHL [[AND14]], [[C13]](s32)
+    ; CHECK: [[OR13:%[0-9]+]]:_(s32) = G_OR [[OR12]], [[SHL13]]
+    ; CHECK: [[COPY17:%[0-9]+]]:_(s32) = COPY [[LSHR14]](s32)
+    ; CHECK: [[AND15:%[0-9]+]]:_(s32) = G_AND [[COPY17]], [[C]]
+    ; CHECK: [[SHL14:%[0-9]+]]:_(s32) = G_SHL [[AND15]], [[C14]](s32)
+    ; CHECK: [[OR14:%[0-9]+]]:_(s32) = G_OR [[OR13]], [[SHL14]]
+    ; CHECK: [[COPY18:%[0-9]+]]:_(s32) = COPY [[LSHR15]](s32)
+    ; CHECK: [[AND16:%[0-9]+]]:_(s32) = G_AND [[COPY18]], [[C]]
+    ; CHECK: [[SHL15:%[0-9]+]]:_(s32) = G_SHL [[AND16]], [[C15]](s32)
+    ; CHECK: [[OR15:%[0-9]+]]:_(s32) = G_OR [[OR14]], [[SHL15]]
+    ; CHECK: [[COPY19:%[0-9]+]]:_(s32) = COPY [[LSHR16]](s32)
+    ; CHECK: [[AND17:%[0-9]+]]:_(s32) = G_AND [[COPY19]], [[C]]
+    ; CHECK: [[SHL16:%[0-9]+]]:_(s32) = G_SHL [[AND17]], [[C16]](s32)
+    ; CHECK: [[OR16:%[0-9]+]]:_(s32) = G_OR [[OR15]], [[SHL16]]
+    ; CHECK: [[COPY20:%[0-9]+]]:_(s32) = COPY [[LSHR17]](s32)
+    ; CHECK: [[AND18:%[0-9]+]]:_(s32) = G_AND [[COPY20]], [[C]]
+    ; CHECK: [[SHL17:%[0-9]+]]:_(s32) = G_SHL [[AND18]], [[C17]](s32)
+    ; CHECK: [[OR17:%[0-9]+]]:_(s32) = G_OR [[OR16]], [[SHL17]]
+    ; CHECK: [[COPY21:%[0-9]+]]:_(s32) = COPY [[LSHR18]](s32)
+    ; CHECK: [[AND19:%[0-9]+]]:_(s32) = G_AND [[COPY21]], [[C]]
+    ; CHECK: [[SHL18:%[0-9]+]]:_(s32) = G_SHL [[AND19]], [[C18]](s32)
+    ; CHECK: [[OR18:%[0-9]+]]:_(s32) = G_OR [[OR17]], [[SHL18]]
+    ; CHECK: [[COPY22:%[0-9]+]]:_(s32) = COPY [[LSHR19]](s32)
+    ; CHECK: [[AND20:%[0-9]+]]:_(s32) = G_AND [[COPY22]], [[C]]
+    ; CHECK: [[SHL19:%[0-9]+]]:_(s32) = G_SHL [[AND20]], [[C19]](s32)
+    ; CHECK: [[OR19:%[0-9]+]]:_(s32) = G_OR [[OR18]], [[SHL19]]
+    ; CHECK: [[COPY23:%[0-9]+]]:_(s32) = COPY [[LSHR20]](s32)
+    ; CHECK: [[AND21:%[0-9]+]]:_(s32) = G_AND [[COPY23]], [[C]]
+    ; CHECK: [[SHL20:%[0-9]+]]:_(s32) = G_SHL [[AND21]], [[C20]](s32)
+    ; CHECK: [[OR20:%[0-9]+]]:_(s32) = G_OR [[OR19]], [[SHL20]]
+    ; CHECK: [[COPY24:%[0-9]+]]:_(s32) = COPY [[LSHR21]](s32)
+    ; CHECK: [[AND22:%[0-9]+]]:_(s32) = G_AND [[COPY24]], [[C]]
+    ; CHECK: [[SHL21:%[0-9]+]]:_(s32) = G_SHL [[AND22]], [[C21]](s32)
+    ; CHECK: [[OR21:%[0-9]+]]:_(s32) = G_OR [[OR20]], [[SHL21]]
+    ; CHECK: [[COPY25:%[0-9]+]]:_(s32) = COPY [[LSHR22]](s32)
+    ; CHECK: [[AND23:%[0-9]+]]:_(s32) = G_AND [[COPY25]], [[C]]
+    ; CHECK: [[SHL22:%[0-9]+]]:_(s32) = G_SHL [[AND23]], [[C22]](s32)
+    ; CHECK: [[OR22:%[0-9]+]]:_(s32) = G_OR [[OR21]], [[SHL22]]
+    ; CHECK: [[COPY26:%[0-9]+]]:_(s32) = COPY [[LSHR23]](s32)
+    ; CHECK: [[AND24:%[0-9]+]]:_(s32) = G_AND [[COPY26]], [[C]]
+    ; CHECK: [[SHL23:%[0-9]+]]:_(s32) = G_SHL [[AND24]], [[C23]](s32)
+    ; CHECK: [[OR23:%[0-9]+]]:_(s32) = G_OR [[OR22]], [[SHL23]]
+    ; CHECK: [[COPY27:%[0-9]+]]:_(s32) = COPY [[LSHR24]](s32)
+    ; CHECK: [[AND25:%[0-9]+]]:_(s32) = G_AND [[COPY27]], [[C]]
+    ; CHECK: [[SHL24:%[0-9]+]]:_(s32) = G_SHL [[AND25]], [[C24]](s32)
+    ; CHECK: [[OR24:%[0-9]+]]:_(s32) = G_OR [[OR23]], [[SHL24]]
+    ; CHECK: [[COPY28:%[0-9]+]]:_(s32) = COPY [[LSHR25]](s32)
+    ; CHECK: [[AND26:%[0-9]+]]:_(s32) = G_AND [[COPY28]], [[C]]
+    ; CHECK: [[SHL25:%[0-9]+]]:_(s32) = G_SHL [[AND26]], [[C25]](s32)
+    ; CHECK: [[OR25:%[0-9]+]]:_(s32) = G_OR [[OR24]], [[SHL25]]
+    ; CHECK: [[COPY29:%[0-9]+]]:_(s32) = COPY [[LSHR26]](s32)
+    ; CHECK: [[AND27:%[0-9]+]]:_(s32) = G_AND [[COPY29]], [[C]]
+    ; CHECK: [[SHL26:%[0-9]+]]:_(s32) = G_SHL [[AND27]], [[C26]](s32)
+    ; CHECK: [[OR26:%[0-9]+]]:_(s32) = G_OR [[OR25]], [[SHL26]]
+    ; CHECK: [[COPY30:%[0-9]+]]:_(s32) = COPY [[LSHR27]](s32)
+    ; CHECK: [[AND28:%[0-9]+]]:_(s32) = G_AND [[COPY30]], [[C]]
+    ; CHECK: [[SHL27:%[0-9]+]]:_(s32) = G_SHL [[AND28]], [[C27]](s32)
+    ; CHECK: [[OR27:%[0-9]+]]:_(s32) = G_OR [[OR26]], [[SHL27]]
+    ; CHECK: [[COPY31:%[0-9]+]]:_(s32) = COPY [[LSHR28]](s32)
+    ; CHECK: [[AND29:%[0-9]+]]:_(s32) = G_AND [[COPY31]], [[C]]
+    ; CHECK: [[SHL28:%[0-9]+]]:_(s32) = G_SHL [[AND29]], [[C28]](s32)
+    ; CHECK: [[OR28:%[0-9]+]]:_(s32) = G_OR [[OR27]], [[SHL28]]
+    ; CHECK: [[COPY32:%[0-9]+]]:_(s32) = COPY [[LSHR29]](s32)
+    ; CHECK: [[AND30:%[0-9]+]]:_(s32) = G_AND [[COPY32]], [[C]]
+    ; CHECK: [[SHL29:%[0-9]+]]:_(s32) = G_SHL [[AND30]], [[C29]](s32)
+    ; CHECK: [[OR29:%[0-9]+]]:_(s32) = G_OR [[OR28]], [[SHL29]]
+    ; CHECK: [[COPY33:%[0-9]+]]:_(s32) = COPY [[LSHR30]](s32)
+    ; CHECK: [[AND31:%[0-9]+]]:_(s32) = G_AND [[COPY33]], [[C]]
+    ; CHECK: [[SHL30:%[0-9]+]]:_(s32) = G_SHL [[AND31]], [[C30]](s32)
+    ; CHECK: [[OR30:%[0-9]+]]:_(s32) = G_OR [[OR29]], [[SHL30]]
+    ; CHECK: [[AND32:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C30]]
+    ; CHECK: [[C31:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[SHL31:%[0-9]+]]:_(s32) = G_SHL [[AND32]], [[C31]](s32)
+    ; CHECK: [[LSHR31:%[0-9]+]]:_(s32) = G_LSHR [[OR30]], [[SHL31]](s32)
+    ; CHECK: [[COPY34:%[0-9]+]]:_(s32) = COPY [[LSHR31]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY34]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(<32 x s1>) = G_BITCAST %0
+    %3:_(s1) = G_EXTRACT_VECTOR_ELT %2, %1
+    %4:_(s32) = G_ANYEXT %3
+    $vgpr0 = COPY %4
+...
+
+---
+name: extract_vector_elt_v12s8_varidx_s32
+
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2, $vgpr3
+    ; CHECK-LABEL: name: extract_vector_elt_v12s8_varidx_s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<3 x s32>)
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; CHECK: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[UV]], [[C]](s32)
+    ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CHECK: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[UV]], [[C1]](s32)
+    ; CHECK: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; CHECK: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[UV]], [[C2]](s32)
+    ; CHECK: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[UV1]], [[C]](s32)
+    ; CHECK: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[UV1]], [[C1]](s32)
+    ; CHECK: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[UV1]], [[C2]](s32)
+    ; CHECK: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[UV2]], [[C]](s32)
+    ; CHECK: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[UV2]], [[C1]](s32)
+    ; CHECK: [[LSHR8:%[0-9]+]]:_(s32) = G_LSHR [[UV2]], [[C2]](s32)
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr3
+    ; CHECK: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[UV]](s32)
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C3]]
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; CHECK: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C3]]
+    ; CHECK: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+    ; CHECK: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
+    ; CHECK: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+    ; CHECK: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C3]]
+    ; CHECK: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C1]](s32)
+    ; CHECK: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+    ; CHECK: [[COPY5:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
+    ; CHECK: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY5]], [[C3]]
+    ; CHECK: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C2]](s32)
+    ; CHECK: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[SHL2]]
+    ; CHECK: [[COPY6:%[0-9]+]]:_(s32) = COPY [[UV1]](s32)
+    ; CHECK: [[AND4:%[0-9]+]]:_(s32) = G_AND [[COPY6]], [[C3]]
+    ; CHECK: [[COPY7:%[0-9]+]]:_(s32) = COPY [[LSHR3]](s32)
+    ; CHECK: [[AND5:%[0-9]+]]:_(s32) = G_AND [[COPY7]], [[C3]]
+    ; CHECK: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C]](s32)
+    ; CHECK: [[OR3:%[0-9]+]]:_(s32) = G_OR [[AND4]], [[SHL3]]
+    ; CHECK: [[COPY8:%[0-9]+]]:_(s32) = COPY [[LSHR4]](s32)
+    ; CHECK: [[AND6:%[0-9]+]]:_(s32) = G_AND [[COPY8]], [[C3]]
+    ; CHECK: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[AND6]], [[C1]](s32)
+    ; CHECK: [[OR4:%[0-9]+]]:_(s32) = G_OR [[OR3]], [[SHL4]]
+    ; CHECK: [[COPY9:%[0-9]+]]:_(s32) = COPY [[LSHR5]](s32)
+    ; CHECK: [[AND7:%[0-9]+]]:_(s32) = G_AND [[COPY9]], [[C3]]
+    ; CHECK: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[AND7]], [[C2]](s32)
+    ; CHECK: [[OR5:%[0-9]+]]:_(s32) = G_OR [[OR4]], [[SHL5]]
+    ; CHECK: [[COPY10:%[0-9]+]]:_(s32) = COPY [[UV2]](s32)
+    ; CHECK: [[AND8:%[0-9]+]]:_(s32) = G_AND [[COPY10]], [[C3]]
+    ; CHECK: [[COPY11:%[0-9]+]]:_(s32) = COPY [[LSHR6]](s32)
+    ; CHECK: [[AND9:%[0-9]+]]:_(s32) = G_AND [[COPY11]], [[C3]]
+    ; CHECK: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[AND9]], [[C]](s32)
+    ; CHECK: [[OR6:%[0-9]+]]:_(s32) = G_OR [[AND8]], [[SHL6]]
+    ; CHECK: [[COPY12:%[0-9]+]]:_(s32) = COPY [[LSHR7]](s32)
+    ; CHECK: [[AND10:%[0-9]+]]:_(s32) = G_AND [[COPY12]], [[C3]]
+    ; CHECK: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[AND10]], [[C1]](s32)
+    ; CHECK: [[OR7:%[0-9]+]]:_(s32) = G_OR [[OR6]], [[SHL7]]
+    ; CHECK: [[COPY13:%[0-9]+]]:_(s32) = COPY [[LSHR8]](s32)
+    ; CHECK: [[AND11:%[0-9]+]]:_(s32) = G_AND [[COPY13]], [[C3]]
+    ; CHECK: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[AND11]], [[C2]](s32)
+    ; CHECK: [[OR8:%[0-9]+]]:_(s32) = G_OR [[OR7]], [[SHL8]]
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32), [[OR8]](s32)
+    ; CHECK: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; CHECK: [[LSHR9:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C4]](s32)
+    ; CHECK: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[BUILD_VECTOR]](<3 x s32>), [[LSHR9]](s32)
+    ; CHECK: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
+    ; CHECK: [[AND12:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C5]]
+    ; CHECK: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[AND12]], [[C5]](s32)
+    ; CHECK: [[LSHR10:%[0-9]+]]:_(s32) = G_LSHR [[EVEC]], [[SHL9]](s32)
+    ; CHECK: [[COPY14:%[0-9]+]]:_(s32) = COPY [[LSHR10]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY14]](s32)
+    %0:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    %1:_(<12 x s8>) = G_BITCAST %0
+    %2:_(s32) = COPY $vgpr3
+    %3:_(s8) = G_EXTRACT_VECTOR_ELT %1, %2
+    %4:_(s32) = G_ANYEXT %3
+    $vgpr0 = COPY %4
+...
+
+---
+name: extract_vector_elt_v3s8_varidx_s32
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; CHECK-LABEL: name: extract_vector_elt_v3s8_varidx_s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; CHECK: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C]](s32)
+    ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CHECK: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C1]](s32)
+    ; CHECK: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; CHECK: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C2]](s32)
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; CHECK: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+    ; CHECK: [[COPY5:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+    ; CHECK: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY5]], 8
+    ; CHECK: [[COPY6:%[0-9]+]]:_(s32) = COPY [[COPY3]](s32)
+    ; CHECK: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY6]], 8
+    ; CHECK: [[COPY7:%[0-9]+]]:_(s32) = COPY [[COPY4]](s32)
+    ; CHECK: [[SEXT_INREG2:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY7]], 8
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[SEXT_INREG]](s32), [[SEXT_INREG1]](s32), [[SEXT_INREG2]](s32)
+    ; CHECK: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[BUILD_VECTOR]](<3 x s32>), [[COPY1]](s32)
+    ; CHECK: [[COPY8:%[0-9]+]]:_(s32) = COPY [[EVEC]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY8]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s24) = G_TRUNC %0
+    %3:_(<3 x s8>) = G_BITCAST %2
+    %4:_(s8) = G_EXTRACT_VECTOR_ELT %3, %1
+    %5:_(s32) = G_ANYEXT %4
+    $vgpr0 = COPY %5
+...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.dim.a16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.dim.a16.ll
index 86d7a2f4e4dbd..e566572763e57 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.dim.a16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.dim.a16.ll
@@ -16,8 +16,10 @@ define amdgpu_ps <4 x float> @load_1d(<8 x i32> inreg %rsrc, <2 x i16> %coords)
   ; GFX9:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
   ; GFX9:   [[COPY8:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
   ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX9:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
-  ; GFX9:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+  ; GFX9:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
   ; GFX9:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable load 16 from custom "TargetCustom8")
   ; GFX9:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9:   $vgpr0 = COPY [[UV]](s32)
@@ -38,8 +40,10 @@ define amdgpu_ps <4 x float> @load_1d(<8 x i32> inreg %rsrc, <2 x i16> %coords)
   ; GFX10NSA:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
   ; GFX10NSA:   [[COPY8:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
   ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10NSA:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
-  ; GFX10NSA:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+  ; GFX10NSA:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
   ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable load 16 from custom "TargetCustom8")
   ; GFX10NSA:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10NSA:   $vgpr0 = COPY [[UV]](s32)
@@ -67,12 +71,14 @@ define amdgpu_ps <4 x float> @load_2d(<8 x i32> inreg %rsrc, <2 x i16> %coords)
   ; GFX9:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
   ; GFX9:   [[COPY8:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
   ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX9:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
+  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX9:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
-  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
-  ; GFX9:   [[COPY9:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-  ; GFX9:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX9:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; GFX9:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
+  ; GFX9:   [[COPY9:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX9:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
   ; GFX9:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY9]](s32), [[COPY10]](s32)
   ; GFX9:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 15, [[BUILD_VECTOR_TRUNC]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable load 16 from custom "TargetCustom8")
   ; GFX9:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
@@ -94,12 +100,14 @@ define amdgpu_ps <4 x float> @load_2d(<8 x i32> inreg %rsrc, <2 x i16> %coords)
   ; GFX10NSA:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
   ; GFX10NSA:   [[COPY8:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
   ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10NSA:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
+  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX10NSA:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
-  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
-  ; GFX10NSA:   [[COPY9:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX10NSA:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; GFX10NSA:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
+  ; GFX10NSA:   [[COPY9:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
   ; GFX10NSA:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY9]](s32), [[COPY10]](s32)
   ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 15, [[BUILD_VECTOR_TRUNC]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable load 16 from custom "TargetCustom8")
   ; GFX10NSA:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
@@ -130,15 +138,18 @@ define amdgpu_ps <4 x float> @load_3d(<8 x i32> inreg %rsrc, <2 x i16> %coords_l
   ; GFX9:   [[COPY8:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
   ; GFX9:   [[COPY9:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
   ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX9:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
+  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX9:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
-  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+  ; GFX9:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; GFX9:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
   ; GFX9:   [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x s16>)
-  ; GFX9:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-  ; GFX9:   [[COPY11:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX9:   [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
+  ; GFX9:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX9:   [[COPY11:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
   ; GFX9:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX9:   [[COPY12:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
+  ; GFX9:   [[COPY12:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
   ; GFX9:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
   ; GFX9:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY12]](s32), [[DEF]](s32)
   ; GFX9:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
@@ -163,15 +174,18 @@ define amdgpu_ps <4 x float> @load_3d(<8 x i32> inreg %rsrc, <2 x i16> %coords_l
   ; GFX10NSA:   [[COPY8:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
   ; GFX10NSA:   [[COPY9:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
   ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10NSA:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
+  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX10NSA:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
-  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+  ; GFX10NSA:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; GFX10NSA:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
   ; GFX10NSA:   [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x s16>)
-  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-  ; GFX10NSA:   [[COPY11:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX10NSA:   [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
+  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX10NSA:   [[COPY11:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
   ; GFX10NSA:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
+  ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
   ; GFX10NSA:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
   ; GFX10NSA:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY12]](s32), [[DEF]](s32)
   ; GFX10NSA:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
@@ -205,15 +219,18 @@ define amdgpu_ps <4 x float> @load_cube(<8 x i32> inreg %rsrc, <2 x i16> %coords
   ; GFX9:   [[COPY8:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
   ; GFX9:   [[COPY9:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
   ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX9:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
+  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX9:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
-  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+  ; GFX9:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; GFX9:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
   ; GFX9:   [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x s16>)
-  ; GFX9:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-  ; GFX9:   [[COPY11:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX9:   [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
+  ; GFX9:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX9:   [[COPY11:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
   ; GFX9:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX9:   [[COPY12:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
+  ; GFX9:   [[COPY12:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
   ; GFX9:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
   ; GFX9:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY12]](s32), [[DEF]](s32)
   ; GFX9:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
@@ -238,15 +255,18 @@ define amdgpu_ps <4 x float> @load_cube(<8 x i32> inreg %rsrc, <2 x i16> %coords
   ; GFX10NSA:   [[COPY8:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
   ; GFX10NSA:   [[COPY9:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
   ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10NSA:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
+  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX10NSA:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
-  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+  ; GFX10NSA:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; GFX10NSA:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
   ; GFX10NSA:   [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x s16>)
-  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-  ; GFX10NSA:   [[COPY11:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX10NSA:   [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
+  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX10NSA:   [[COPY11:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
   ; GFX10NSA:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
+  ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
   ; GFX10NSA:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
   ; GFX10NSA:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY12]](s32), [[DEF]](s32)
   ; GFX10NSA:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
@@ -279,12 +299,14 @@ define amdgpu_ps <4 x float> @load_1darray(<8 x i32> inreg %rsrc, <2 x i16> %coo
   ; GFX9:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
   ; GFX9:   [[COPY8:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
   ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX9:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
+  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX9:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
-  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
-  ; GFX9:   [[COPY9:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-  ; GFX9:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX9:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; GFX9:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
+  ; GFX9:   [[COPY9:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX9:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
   ; GFX9:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY9]](s32), [[COPY10]](s32)
   ; GFX9:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1darray), 15, [[BUILD_VECTOR_TRUNC]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable load 16 from custom "TargetCustom8")
   ; GFX9:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
@@ -306,12 +328,14 @@ define amdgpu_ps <4 x float> @load_1darray(<8 x i32> inreg %rsrc, <2 x i16> %coo
   ; GFX10NSA:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
   ; GFX10NSA:   [[COPY8:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
   ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10NSA:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
+  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX10NSA:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
-  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
-  ; GFX10NSA:   [[COPY9:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX10NSA:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; GFX10NSA:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
+  ; GFX10NSA:   [[COPY9:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
   ; GFX10NSA:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY9]](s32), [[COPY10]](s32)
   ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1darray), 15, [[BUILD_VECTOR_TRUNC]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable load 16 from custom "TargetCustom8")
   ; GFX10NSA:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
@@ -342,15 +366,18 @@ define amdgpu_ps <4 x float> @load_2darray(<8 x i32> inreg %rsrc, <2 x i16> %coo
   ; GFX9:   [[COPY8:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
   ; GFX9:   [[COPY9:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
   ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX9:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
+  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX9:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
-  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+  ; GFX9:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; GFX9:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
   ; GFX9:   [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x s16>)
-  ; GFX9:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-  ; GFX9:   [[COPY11:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX9:   [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
+  ; GFX9:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX9:   [[COPY11:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
   ; GFX9:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX9:   [[COPY12:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
+  ; GFX9:   [[COPY12:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
   ; GFX9:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
   ; GFX9:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY12]](s32), [[DEF]](s32)
   ; GFX9:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
@@ -375,15 +402,18 @@ define amdgpu_ps <4 x float> @load_2darray(<8 x i32> inreg %rsrc, <2 x i16> %coo
   ; GFX10NSA:   [[COPY8:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
   ; GFX10NSA:   [[COPY9:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
   ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10NSA:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
+  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX10NSA:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
-  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+  ; GFX10NSA:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; GFX10NSA:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
   ; GFX10NSA:   [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x s16>)
-  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-  ; GFX10NSA:   [[COPY11:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX10NSA:   [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
+  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX10NSA:   [[COPY11:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
   ; GFX10NSA:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
+  ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
   ; GFX10NSA:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
   ; GFX10NSA:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY12]](s32), [[DEF]](s32)
   ; GFX10NSA:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
@@ -417,15 +447,18 @@ define amdgpu_ps <4 x float> @load_2dmsaa(<8 x i32> inreg %rsrc, <2 x i16> %coor
   ; GFX9:   [[COPY8:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
   ; GFX9:   [[COPY9:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
   ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX9:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
+  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX9:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
-  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+  ; GFX9:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; GFX9:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
   ; GFX9:   [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x s16>)
-  ; GFX9:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-  ; GFX9:   [[COPY11:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX9:   [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
+  ; GFX9:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX9:   [[COPY11:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
   ; GFX9:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX9:   [[COPY12:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
+  ; GFX9:   [[COPY12:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
   ; GFX9:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
   ; GFX9:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY12]](s32), [[DEF]](s32)
   ; GFX9:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
@@ -450,15 +483,18 @@ define amdgpu_ps <4 x float> @load_2dmsaa(<8 x i32> inreg %rsrc, <2 x i16> %coor
   ; GFX10NSA:   [[COPY8:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
   ; GFX10NSA:   [[COPY9:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
   ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10NSA:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
+  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX10NSA:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
-  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+  ; GFX10NSA:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; GFX10NSA:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
   ; GFX10NSA:   [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x s16>)
-  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-  ; GFX10NSA:   [[COPY11:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX10NSA:   [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
+  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX10NSA:   [[COPY11:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
   ; GFX10NSA:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
+  ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
   ; GFX10NSA:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
   ; GFX10NSA:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY12]](s32), [[DEF]](s32)
   ; GFX10NSA:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
@@ -492,18 +528,21 @@ define amdgpu_ps <4 x float> @load_2darraymsaa(<8 x i32> inreg %rsrc, <2 x i16>
   ; GFX9:   [[COPY8:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
   ; GFX9:   [[COPY9:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
   ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX9:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
+  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX9:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
-  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+  ; GFX9:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; GFX9:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
   ; GFX9:   [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x s16>)
+  ; GFX9:   [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
   ; GFX9:   [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x s16>)
-  ; GFX9:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
-  ; GFX9:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-  ; GFX9:   [[COPY11:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX9:   [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C1]](s32)
+  ; GFX9:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX9:   [[COPY11:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
   ; GFX9:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX9:   [[COPY12:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
-  ; GFX9:   [[COPY13:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+  ; GFX9:   [[COPY12:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
+  ; GFX9:   [[COPY13:%[0-9]+]]:_(s32) = COPY [[LSHR3]](s32)
   ; GFX9:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY12]](s32), [[COPY13]](s32)
   ; GFX9:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
   ; GFX9:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2darraymsaa), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable load 16 from custom "TargetCustom8")
@@ -527,18 +566,21 @@ define amdgpu_ps <4 x float> @load_2darraymsaa(<8 x i32> inreg %rsrc, <2 x i16>
   ; GFX10NSA:   [[COPY8:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
   ; GFX10NSA:   [[COPY9:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
   ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10NSA:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
+  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX10NSA:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
-  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+  ; GFX10NSA:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; GFX10NSA:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
   ; GFX10NSA:   [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x s16>)
+  ; GFX10NSA:   [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
   ; GFX10NSA:   [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x s16>)
-  ; GFX10NSA:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
-  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-  ; GFX10NSA:   [[COPY11:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX10NSA:   [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C1]](s32)
+  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX10NSA:   [[COPY11:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
   ; GFX10NSA:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
-  ; GFX10NSA:   [[COPY13:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+  ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
+  ; GFX10NSA:   [[COPY13:%[0-9]+]]:_(s32) = COPY [[LSHR3]](s32)
   ; GFX10NSA:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY12]](s32), [[COPY13]](s32)
   ; GFX10NSA:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
   ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2darraymsaa), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable load 16 from custom "TargetCustom8")
@@ -571,12 +613,14 @@ define amdgpu_ps <4 x float> @load_mip_1d(<8 x i32> inreg %rsrc, <2 x i16> %coor
   ; GFX9:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
   ; GFX9:   [[COPY8:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
   ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX9:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
+  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX9:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
-  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
-  ; GFX9:   [[COPY9:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-  ; GFX9:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX9:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; GFX9:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
+  ; GFX9:   [[COPY9:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX9:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
   ; GFX9:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY9]](s32), [[COPY10]](s32)
   ; GFX9:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.mip.1d), 15, [[BUILD_VECTOR_TRUNC]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable load 16 from custom "TargetCustom8")
   ; GFX9:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
@@ -598,12 +642,14 @@ define amdgpu_ps <4 x float> @load_mip_1d(<8 x i32> inreg %rsrc, <2 x i16> %coor
   ; GFX10NSA:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
   ; GFX10NSA:   [[COPY8:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
   ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10NSA:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
+  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX10NSA:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
-  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
-  ; GFX10NSA:   [[COPY9:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX10NSA:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; GFX10NSA:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
+  ; GFX10NSA:   [[COPY9:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
   ; GFX10NSA:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY9]](s32), [[COPY10]](s32)
   ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.mip.1d), 15, [[BUILD_VECTOR_TRUNC]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable load 16 from custom "TargetCustom8")
   ; GFX10NSA:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
@@ -634,15 +680,18 @@ define amdgpu_ps <4 x float> @load_mip_2d(<8 x i32> inreg %rsrc, <2 x i16> %coor
   ; GFX9:   [[COPY8:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
   ; GFX9:   [[COPY9:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
   ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX9:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
+  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX9:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
-  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+  ; GFX9:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; GFX9:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
   ; GFX9:   [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x s16>)
-  ; GFX9:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-  ; GFX9:   [[COPY11:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX9:   [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
+  ; GFX9:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX9:   [[COPY11:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
   ; GFX9:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX9:   [[COPY12:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
+  ; GFX9:   [[COPY12:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
   ; GFX9:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
   ; GFX9:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY12]](s32), [[DEF]](s32)
   ; GFX9:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
@@ -667,15 +716,18 @@ define amdgpu_ps <4 x float> @load_mip_2d(<8 x i32> inreg %rsrc, <2 x i16> %coor
   ; GFX10NSA:   [[COPY8:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
   ; GFX10NSA:   [[COPY9:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
   ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10NSA:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
+  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX10NSA:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
-  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+  ; GFX10NSA:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; GFX10NSA:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
   ; GFX10NSA:   [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x s16>)
-  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-  ; GFX10NSA:   [[COPY11:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX10NSA:   [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
+  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX10NSA:   [[COPY11:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
   ; GFX10NSA:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
+  ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
   ; GFX10NSA:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
   ; GFX10NSA:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY12]](s32), [[DEF]](s32)
   ; GFX10NSA:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
@@ -709,18 +761,21 @@ define amdgpu_ps <4 x float> @load_mip_3d(<8 x i32> inreg %rsrc, <2 x i16> %coor
   ; GFX9:   [[COPY8:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
   ; GFX9:   [[COPY9:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
   ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX9:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
+  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX9:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
-  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+  ; GFX9:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; GFX9:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
   ; GFX9:   [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x s16>)
+  ; GFX9:   [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
   ; GFX9:   [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x s16>)
-  ; GFX9:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
-  ; GFX9:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-  ; GFX9:   [[COPY11:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX9:   [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C1]](s32)
+  ; GFX9:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX9:   [[COPY11:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
   ; GFX9:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX9:   [[COPY12:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
-  ; GFX9:   [[COPY13:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+  ; GFX9:   [[COPY12:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
+  ; GFX9:   [[COPY13:%[0-9]+]]:_(s32) = COPY [[LSHR3]](s32)
   ; GFX9:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY12]](s32), [[COPY13]](s32)
   ; GFX9:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
   ; GFX9:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.mip.3d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable load 16 from custom "TargetCustom8")
@@ -744,18 +799,21 @@ define amdgpu_ps <4 x float> @load_mip_3d(<8 x i32> inreg %rsrc, <2 x i16> %coor
   ; GFX10NSA:   [[COPY8:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
   ; GFX10NSA:   [[COPY9:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
   ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10NSA:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
+  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX10NSA:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
-  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+  ; GFX10NSA:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; GFX10NSA:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
   ; GFX10NSA:   [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x s16>)
+  ; GFX10NSA:   [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
   ; GFX10NSA:   [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x s16>)
-  ; GFX10NSA:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
-  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-  ; GFX10NSA:   [[COPY11:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX10NSA:   [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C1]](s32)
+  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX10NSA:   [[COPY11:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
   ; GFX10NSA:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
-  ; GFX10NSA:   [[COPY13:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+  ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
+  ; GFX10NSA:   [[COPY13:%[0-9]+]]:_(s32) = COPY [[LSHR3]](s32)
   ; GFX10NSA:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY12]](s32), [[COPY13]](s32)
   ; GFX10NSA:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
   ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.mip.3d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable load 16 from custom "TargetCustom8")
@@ -789,18 +847,21 @@ define amdgpu_ps <4 x float> @load_mip_cube(<8 x i32> inreg %rsrc, <2 x i16> %co
   ; GFX9:   [[COPY8:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
   ; GFX9:   [[COPY9:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
   ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX9:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
+  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX9:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
-  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+  ; GFX9:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; GFX9:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
   ; GFX9:   [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x s16>)
+  ; GFX9:   [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
   ; GFX9:   [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x s16>)
-  ; GFX9:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
-  ; GFX9:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-  ; GFX9:   [[COPY11:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX9:   [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C1]](s32)
+  ; GFX9:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX9:   [[COPY11:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
   ; GFX9:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX9:   [[COPY12:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
-  ; GFX9:   [[COPY13:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+  ; GFX9:   [[COPY12:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
+  ; GFX9:   [[COPY13:%[0-9]+]]:_(s32) = COPY [[LSHR3]](s32)
   ; GFX9:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY12]](s32), [[COPY13]](s32)
   ; GFX9:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
   ; GFX9:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.mip.cube), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable load 16 from custom "TargetCustom8")
@@ -824,18 +885,21 @@ define amdgpu_ps <4 x float> @load_mip_cube(<8 x i32> inreg %rsrc, <2 x i16> %co
   ; GFX10NSA:   [[COPY8:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
   ; GFX10NSA:   [[COPY9:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
   ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10NSA:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
+  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX10NSA:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
-  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+  ; GFX10NSA:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; GFX10NSA:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
   ; GFX10NSA:   [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x s16>)
+  ; GFX10NSA:   [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
   ; GFX10NSA:   [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x s16>)
-  ; GFX10NSA:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
-  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-  ; GFX10NSA:   [[COPY11:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX10NSA:   [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C1]](s32)
+  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX10NSA:   [[COPY11:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
   ; GFX10NSA:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
-  ; GFX10NSA:   [[COPY13:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+  ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
+  ; GFX10NSA:   [[COPY13:%[0-9]+]]:_(s32) = COPY [[LSHR3]](s32)
   ; GFX10NSA:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY12]](s32), [[COPY13]](s32)
   ; GFX10NSA:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
   ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.mip.cube), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable load 16 from custom "TargetCustom8")
@@ -869,15 +933,18 @@ define amdgpu_ps <4 x float> @load_mip_1darray(<8 x i32> inreg %rsrc, <2 x i16>
   ; GFX9:   [[COPY8:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
   ; GFX9:   [[COPY9:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
   ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX9:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
+  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX9:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
-  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+  ; GFX9:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; GFX9:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
   ; GFX9:   [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x s16>)
-  ; GFX9:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-  ; GFX9:   [[COPY11:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX9:   [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
+  ; GFX9:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX9:   [[COPY11:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
   ; GFX9:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX9:   [[COPY12:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
+  ; GFX9:   [[COPY12:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
   ; GFX9:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
   ; GFX9:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY12]](s32), [[DEF]](s32)
   ; GFX9:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
@@ -902,15 +969,18 @@ define amdgpu_ps <4 x float> @load_mip_1darray(<8 x i32> inreg %rsrc, <2 x i16>
   ; GFX10NSA:   [[COPY8:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
   ; GFX10NSA:   [[COPY9:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
   ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10NSA:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
+  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX10NSA:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
-  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+  ; GFX10NSA:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; GFX10NSA:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
   ; GFX10NSA:   [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x s16>)
-  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-  ; GFX10NSA:   [[COPY11:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX10NSA:   [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
+  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX10NSA:   [[COPY11:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
   ; GFX10NSA:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
+  ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
   ; GFX10NSA:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
   ; GFX10NSA:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY12]](s32), [[DEF]](s32)
   ; GFX10NSA:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
@@ -944,18 +1014,21 @@ define amdgpu_ps <4 x float> @load_mip_2darray(<8 x i32> inreg %rsrc, <2 x i16>
   ; GFX9:   [[COPY8:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
   ; GFX9:   [[COPY9:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
   ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX9:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
+  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX9:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
-  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+  ; GFX9:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; GFX9:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
   ; GFX9:   [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x s16>)
+  ; GFX9:   [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
   ; GFX9:   [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x s16>)
-  ; GFX9:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
-  ; GFX9:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-  ; GFX9:   [[COPY11:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX9:   [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C1]](s32)
+  ; GFX9:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX9:   [[COPY11:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
   ; GFX9:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX9:   [[COPY12:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
-  ; GFX9:   [[COPY13:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+  ; GFX9:   [[COPY12:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
+  ; GFX9:   [[COPY13:%[0-9]+]]:_(s32) = COPY [[LSHR3]](s32)
   ; GFX9:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY12]](s32), [[COPY13]](s32)
   ; GFX9:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
   ; GFX9:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.mip.2darray), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable load 16 from custom "TargetCustom8")
@@ -979,18 +1052,21 @@ define amdgpu_ps <4 x float> @load_mip_2darray(<8 x i32> inreg %rsrc, <2 x i16>
   ; GFX10NSA:   [[COPY8:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
   ; GFX10NSA:   [[COPY9:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
   ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10NSA:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
+  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX10NSA:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
-  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+  ; GFX10NSA:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; GFX10NSA:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
   ; GFX10NSA:   [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x s16>)
+  ; GFX10NSA:   [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
   ; GFX10NSA:   [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x s16>)
-  ; GFX10NSA:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
-  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-  ; GFX10NSA:   [[COPY11:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX10NSA:   [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C1]](s32)
+  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX10NSA:   [[COPY11:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
   ; GFX10NSA:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
-  ; GFX10NSA:   [[COPY13:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+  ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
+  ; GFX10NSA:   [[COPY13:%[0-9]+]]:_(s32) = COPY [[LSHR3]](s32)
   ; GFX10NSA:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY12]](s32), [[COPY13]](s32)
   ; GFX10NSA:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
   ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.mip.2darray), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable load 16 from custom "TargetCustom8")
@@ -1028,8 +1104,10 @@ define amdgpu_ps void @store_1d(<8 x i32> inreg %rsrc, <4 x float> %vdata, <2 x
   ; GFX9:   [[COPY12:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr4
   ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; GFX9:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
+  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX9:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
-  ; GFX9:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+  ; GFX9:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
   ; GFX9:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1d), [[BUILD_VECTOR1]](<4 x s32>), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable store 16 into custom "TargetCustom8")
   ; GFX9:   S_ENDPGM 0
   ; GFX10NSA-LABEL: name: store_1d
@@ -1050,8 +1128,10 @@ define amdgpu_ps void @store_1d(<8 x i32> inreg %rsrc, <4 x float> %vdata, <2 x
   ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr4
   ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; GFX10NSA:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
+  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10NSA:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
-  ; GFX10NSA:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+  ; GFX10NSA:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
   ; GFX10NSA:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1d), [[BUILD_VECTOR1]](<4 x s32>), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable store 16 into custom "TargetCustom8")
   ; GFX10NSA:   S_ENDPGM 0
 main_body:
@@ -1079,12 +1159,14 @@ define amdgpu_ps void @store_2d(<8 x i32> inreg %rsrc, <4 x float> %vdata, <2 x
   ; GFX9:   [[COPY12:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr4
   ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; GFX9:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
+  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX9:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
+  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX9:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
-  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
-  ; GFX9:   [[COPY13:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-  ; GFX9:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX9:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; GFX9:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
+  ; GFX9:   [[COPY13:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX9:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
   ; GFX9:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY13]](s32), [[COPY14]](s32)
   ; GFX9:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.2d), [[BUILD_VECTOR1]](<4 x s32>), 15, [[BUILD_VECTOR_TRUNC]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable store 16 into custom "TargetCustom8")
   ; GFX9:   S_ENDPGM 0
@@ -1106,12 +1188,14 @@ define amdgpu_ps void @store_2d(<8 x i32> inreg %rsrc, <4 x float> %vdata, <2 x
   ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr4
   ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; GFX10NSA:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
+  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10NSA:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
+  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX10NSA:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
-  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
-  ; GFX10NSA:   [[COPY13:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-  ; GFX10NSA:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX10NSA:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; GFX10NSA:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
+  ; GFX10NSA:   [[COPY13:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX10NSA:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
   ; GFX10NSA:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY13]](s32), [[COPY14]](s32)
   ; GFX10NSA:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.2d), [[BUILD_VECTOR1]](<4 x s32>), 15, [[BUILD_VECTOR_TRUNC]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable store 16 into custom "TargetCustom8")
   ; GFX10NSA:   S_ENDPGM 0
@@ -1142,15 +1226,18 @@ define amdgpu_ps void @store_3d(<8 x i32> inreg %rsrc, <4 x float> %vdata, <2 x
   ; GFX9:   [[COPY13:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr5
   ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; GFX9:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
+  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX9:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
+  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX9:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
-  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+  ; GFX9:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; GFX9:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
   ; GFX9:   [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x s16>)
-  ; GFX9:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-  ; GFX9:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX9:   [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
+  ; GFX9:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX9:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
   ; GFX9:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY14]](s32), [[COPY15]](s32)
-  ; GFX9:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
+  ; GFX9:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
   ; GFX9:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
   ; GFX9:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY16]](s32), [[DEF]](s32)
   ; GFX9:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
@@ -1175,15 +1262,18 @@ define amdgpu_ps void @store_3d(<8 x i32> inreg %rsrc, <4 x float> %vdata, <2 x
   ; GFX10NSA:   [[COPY13:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr5
   ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; GFX10NSA:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
+  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10NSA:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
+  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX10NSA:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
-  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+  ; GFX10NSA:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; GFX10NSA:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
   ; GFX10NSA:   [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x s16>)
-  ; GFX10NSA:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-  ; GFX10NSA:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX10NSA:   [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
+  ; GFX10NSA:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX10NSA:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
   ; GFX10NSA:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY14]](s32), [[COPY15]](s32)
-  ; GFX10NSA:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
+  ; GFX10NSA:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
   ; GFX10NSA:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
   ; GFX10NSA:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY16]](s32), [[DEF]](s32)
   ; GFX10NSA:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
@@ -1217,15 +1307,18 @@ define amdgpu_ps void @store_cube(<8 x i32> inreg %rsrc, <4 x float> %vdata, <2
   ; GFX9:   [[COPY13:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr5
   ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; GFX9:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
+  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX9:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
+  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX9:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
-  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+  ; GFX9:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; GFX9:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
   ; GFX9:   [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x s16>)
-  ; GFX9:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-  ; GFX9:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX9:   [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
+  ; GFX9:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX9:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
   ; GFX9:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY14]](s32), [[COPY15]](s32)
-  ; GFX9:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
+  ; GFX9:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
   ; GFX9:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
   ; GFX9:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY16]](s32), [[DEF]](s32)
   ; GFX9:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
@@ -1250,15 +1343,18 @@ define amdgpu_ps void @store_cube(<8 x i32> inreg %rsrc, <4 x float> %vdata, <2
   ; GFX10NSA:   [[COPY13:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr5
   ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; GFX10NSA:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
+  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10NSA:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
+  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX10NSA:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
-  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+  ; GFX10NSA:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; GFX10NSA:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
   ; GFX10NSA:   [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x s16>)
-  ; GFX10NSA:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-  ; GFX10NSA:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX10NSA:   [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
+  ; GFX10NSA:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX10NSA:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
   ; GFX10NSA:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY14]](s32), [[COPY15]](s32)
-  ; GFX10NSA:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
+  ; GFX10NSA:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
   ; GFX10NSA:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
   ; GFX10NSA:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY16]](s32), [[DEF]](s32)
   ; GFX10NSA:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
@@ -1291,12 +1387,14 @@ define amdgpu_ps void @store_1darray(<8 x i32> inreg %rsrc, <4 x float> %vdata,
   ; GFX9:   [[COPY12:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr4
   ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; GFX9:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
+  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX9:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
+  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX9:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
-  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
-  ; GFX9:   [[COPY13:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-  ; GFX9:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX9:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; GFX9:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
+  ; GFX9:   [[COPY13:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX9:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
   ; GFX9:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY13]](s32), [[COPY14]](s32)
   ; GFX9:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1darray), [[BUILD_VECTOR1]](<4 x s32>), 15, [[BUILD_VECTOR_TRUNC]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable store 16 into custom "TargetCustom8")
   ; GFX9:   S_ENDPGM 0
@@ -1318,12 +1416,14 @@ define amdgpu_ps void @store_1darray(<8 x i32> inreg %rsrc, <4 x float> %vdata,
   ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr4
   ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; GFX10NSA:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
+  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10NSA:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
+  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX10NSA:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
-  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
-  ; GFX10NSA:   [[COPY13:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-  ; GFX10NSA:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX10NSA:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; GFX10NSA:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
+  ; GFX10NSA:   [[COPY13:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX10NSA:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
   ; GFX10NSA:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY13]](s32), [[COPY14]](s32)
   ; GFX10NSA:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1darray), [[BUILD_VECTOR1]](<4 x s32>), 15, [[BUILD_VECTOR_TRUNC]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable store 16 into custom "TargetCustom8")
   ; GFX10NSA:   S_ENDPGM 0
@@ -1354,15 +1454,18 @@ define amdgpu_ps void @store_2darray(<8 x i32> inreg %rsrc, <4 x float> %vdata,
   ; GFX9:   [[COPY13:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr5
   ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; GFX9:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
+  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX9:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
+  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX9:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
-  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+  ; GFX9:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; GFX9:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
   ; GFX9:   [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x s16>)
-  ; GFX9:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-  ; GFX9:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX9:   [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
+  ; GFX9:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX9:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
   ; GFX9:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY14]](s32), [[COPY15]](s32)
-  ; GFX9:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
+  ; GFX9:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
   ; GFX9:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
   ; GFX9:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY16]](s32), [[DEF]](s32)
   ; GFX9:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
@@ -1387,15 +1490,18 @@ define amdgpu_ps void @store_2darray(<8 x i32> inreg %rsrc, <4 x float> %vdata,
   ; GFX10NSA:   [[COPY13:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr5
   ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; GFX10NSA:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
+  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10NSA:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
+  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX10NSA:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
-  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+  ; GFX10NSA:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; GFX10NSA:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
   ; GFX10NSA:   [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x s16>)
-  ; GFX10NSA:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-  ; GFX10NSA:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX10NSA:   [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
+  ; GFX10NSA:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX10NSA:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
   ; GFX10NSA:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY14]](s32), [[COPY15]](s32)
-  ; GFX10NSA:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
+  ; GFX10NSA:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
   ; GFX10NSA:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
   ; GFX10NSA:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY16]](s32), [[DEF]](s32)
   ; GFX10NSA:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
@@ -1429,15 +1535,18 @@ define amdgpu_ps void @store_2dmsaa(<8 x i32> inreg %rsrc, <4 x float> %vdata, <
   ; GFX9:   [[COPY13:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr5
   ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; GFX9:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
+  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX9:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
+  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX9:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
-  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+  ; GFX9:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; GFX9:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
   ; GFX9:   [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x s16>)
-  ; GFX9:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-  ; GFX9:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX9:   [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
+  ; GFX9:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX9:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
   ; GFX9:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY14]](s32), [[COPY15]](s32)
-  ; GFX9:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
+  ; GFX9:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
   ; GFX9:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
   ; GFX9:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY16]](s32), [[DEF]](s32)
   ; GFX9:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
@@ -1462,15 +1571,18 @@ define amdgpu_ps void @store_2dmsaa(<8 x i32> inreg %rsrc, <4 x float> %vdata, <
   ; GFX10NSA:   [[COPY13:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr5
   ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; GFX10NSA:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
+  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10NSA:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
+  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX10NSA:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
-  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+  ; GFX10NSA:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; GFX10NSA:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
   ; GFX10NSA:   [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x s16>)
-  ; GFX10NSA:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-  ; GFX10NSA:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX10NSA:   [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
+  ; GFX10NSA:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX10NSA:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
   ; GFX10NSA:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY14]](s32), [[COPY15]](s32)
-  ; GFX10NSA:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
+  ; GFX10NSA:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
   ; GFX10NSA:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
   ; GFX10NSA:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY16]](s32), [[DEF]](s32)
   ; GFX10NSA:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
@@ -1504,18 +1616,21 @@ define amdgpu_ps void @store_2darraymsaa(<8 x i32> inreg %rsrc, <4 x float> %vda
   ; GFX9:   [[COPY13:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr5
   ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; GFX9:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
+  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX9:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
+  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX9:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
-  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+  ; GFX9:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; GFX9:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
   ; GFX9:   [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x s16>)
+  ; GFX9:   [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
   ; GFX9:   [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x s16>)
-  ; GFX9:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
-  ; GFX9:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-  ; GFX9:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX9:   [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C1]](s32)
+  ; GFX9:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX9:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
   ; GFX9:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY14]](s32), [[COPY15]](s32)
-  ; GFX9:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
-  ; GFX9:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+  ; GFX9:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
+  ; GFX9:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[LSHR3]](s32)
   ; GFX9:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY16]](s32), [[COPY17]](s32)
   ; GFX9:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
   ; GFX9:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.2darraymsaa), [[BUILD_VECTOR1]](<4 x s32>), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable store 16 into custom "TargetCustom8")
@@ -1539,18 +1654,21 @@ define amdgpu_ps void @store_2darraymsaa(<8 x i32> inreg %rsrc, <4 x float> %vda
   ; GFX10NSA:   [[COPY13:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr5
   ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; GFX10NSA:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
+  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10NSA:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
+  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX10NSA:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
-  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+  ; GFX10NSA:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; GFX10NSA:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
   ; GFX10NSA:   [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x s16>)
+  ; GFX10NSA:   [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
   ; GFX10NSA:   [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x s16>)
-  ; GFX10NSA:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
-  ; GFX10NSA:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-  ; GFX10NSA:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX10NSA:   [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C1]](s32)
+  ; GFX10NSA:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX10NSA:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
   ; GFX10NSA:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY14]](s32), [[COPY15]](s32)
-  ; GFX10NSA:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
-  ; GFX10NSA:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+  ; GFX10NSA:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
+  ; GFX10NSA:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[LSHR3]](s32)
   ; GFX10NSA:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY16]](s32), [[COPY17]](s32)
   ; GFX10NSA:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
   ; GFX10NSA:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.2darraymsaa), [[BUILD_VECTOR1]](<4 x s32>), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable store 16 into custom "TargetCustom8")
@@ -1583,12 +1701,14 @@ define amdgpu_ps void @store_mip_1d(<8 x i32> inreg %rsrc, <4 x float> %vdata, <
   ; GFX9:   [[COPY12:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr4
   ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; GFX9:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
+  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX9:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
+  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX9:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
-  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
-  ; GFX9:   [[COPY13:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-  ; GFX9:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX9:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; GFX9:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
+  ; GFX9:   [[COPY13:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX9:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
   ; GFX9:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY13]](s32), [[COPY14]](s32)
   ; GFX9:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.mip.1d), [[BUILD_VECTOR1]](<4 x s32>), 15, [[BUILD_VECTOR_TRUNC]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable store 16 into custom "TargetCustom8")
   ; GFX9:   S_ENDPGM 0
@@ -1610,12 +1730,14 @@ define amdgpu_ps void @store_mip_1d(<8 x i32> inreg %rsrc, <4 x float> %vdata, <
   ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr4
   ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; GFX10NSA:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
+  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10NSA:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
+  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX10NSA:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
-  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
-  ; GFX10NSA:   [[COPY13:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-  ; GFX10NSA:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX10NSA:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; GFX10NSA:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
+  ; GFX10NSA:   [[COPY13:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX10NSA:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
   ; GFX10NSA:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY13]](s32), [[COPY14]](s32)
   ; GFX10NSA:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.mip.1d), [[BUILD_VECTOR1]](<4 x s32>), 15, [[BUILD_VECTOR_TRUNC]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable store 16 into custom "TargetCustom8")
   ; GFX10NSA:   S_ENDPGM 0
@@ -1646,15 +1768,18 @@ define amdgpu_ps void @store_mip_2d(<8 x i32> inreg %rsrc, <4 x float> %vdata, <
   ; GFX9:   [[COPY13:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr5
   ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; GFX9:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
+  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX9:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
+  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX9:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
-  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+  ; GFX9:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; GFX9:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
   ; GFX9:   [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x s16>)
-  ; GFX9:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-  ; GFX9:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX9:   [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
+  ; GFX9:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX9:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
   ; GFX9:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY14]](s32), [[COPY15]](s32)
-  ; GFX9:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
+  ; GFX9:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
   ; GFX9:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
   ; GFX9:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY16]](s32), [[DEF]](s32)
   ; GFX9:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
@@ -1679,15 +1804,18 @@ define amdgpu_ps void @store_mip_2d(<8 x i32> inreg %rsrc, <4 x float> %vdata, <
   ; GFX10NSA:   [[COPY13:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr5
   ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; GFX10NSA:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
+  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10NSA:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
+  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX10NSA:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
-  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+  ; GFX10NSA:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; GFX10NSA:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
   ; GFX10NSA:   [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x s16>)
-  ; GFX10NSA:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-  ; GFX10NSA:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX10NSA:   [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
+  ; GFX10NSA:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX10NSA:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
   ; GFX10NSA:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY14]](s32), [[COPY15]](s32)
-  ; GFX10NSA:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
+  ; GFX10NSA:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
   ; GFX10NSA:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
   ; GFX10NSA:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY16]](s32), [[DEF]](s32)
   ; GFX10NSA:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
@@ -1721,18 +1849,21 @@ define amdgpu_ps void @store_mip_3d(<8 x i32> inreg %rsrc, <4 x float> %vdata, <
   ; GFX9:   [[COPY13:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr5
   ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; GFX9:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
+  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX9:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
+  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX9:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
-  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+  ; GFX9:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; GFX9:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
   ; GFX9:   [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x s16>)
+  ; GFX9:   [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
   ; GFX9:   [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x s16>)
-  ; GFX9:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
-  ; GFX9:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-  ; GFX9:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX9:   [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C1]](s32)
+  ; GFX9:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX9:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
   ; GFX9:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY14]](s32), [[COPY15]](s32)
-  ; GFX9:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
-  ; GFX9:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+  ; GFX9:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
+  ; GFX9:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[LSHR3]](s32)
   ; GFX9:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY16]](s32), [[COPY17]](s32)
   ; GFX9:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
   ; GFX9:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.mip.3d), [[BUILD_VECTOR1]](<4 x s32>), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable store 16 into custom "TargetCustom8")
@@ -1756,18 +1887,21 @@ define amdgpu_ps void @store_mip_3d(<8 x i32> inreg %rsrc, <4 x float> %vdata, <
   ; GFX10NSA:   [[COPY13:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr5
   ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; GFX10NSA:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
+  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10NSA:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
+  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX10NSA:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
-  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+  ; GFX10NSA:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; GFX10NSA:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
   ; GFX10NSA:   [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x s16>)
+  ; GFX10NSA:   [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
   ; GFX10NSA:   [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x s16>)
-  ; GFX10NSA:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
-  ; GFX10NSA:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-  ; GFX10NSA:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX10NSA:   [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C1]](s32)
+  ; GFX10NSA:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX10NSA:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
   ; GFX10NSA:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY14]](s32), [[COPY15]](s32)
-  ; GFX10NSA:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
-  ; GFX10NSA:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+  ; GFX10NSA:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
+  ; GFX10NSA:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[LSHR3]](s32)
   ; GFX10NSA:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY16]](s32), [[COPY17]](s32)
   ; GFX10NSA:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
   ; GFX10NSA:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.mip.3d), [[BUILD_VECTOR1]](<4 x s32>), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable store 16 into custom "TargetCustom8")
@@ -1801,18 +1935,21 @@ define amdgpu_ps void @store_mip_cube(<8 x i32> inreg %rsrc, <4 x float> %vdata,
   ; GFX9:   [[COPY13:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr5
   ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; GFX9:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
+  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX9:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
+  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX9:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
-  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+  ; GFX9:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; GFX9:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
   ; GFX9:   [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x s16>)
+  ; GFX9:   [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
   ; GFX9:   [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x s16>)
-  ; GFX9:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
-  ; GFX9:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-  ; GFX9:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX9:   [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C1]](s32)
+  ; GFX9:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX9:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
   ; GFX9:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY14]](s32), [[COPY15]](s32)
-  ; GFX9:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
-  ; GFX9:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+  ; GFX9:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
+  ; GFX9:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[LSHR3]](s32)
   ; GFX9:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY16]](s32), [[COPY17]](s32)
   ; GFX9:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
   ; GFX9:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.mip.cube), [[BUILD_VECTOR1]](<4 x s32>), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable store 16 into custom "TargetCustom8")
@@ -1836,18 +1973,21 @@ define amdgpu_ps void @store_mip_cube(<8 x i32> inreg %rsrc, <4 x float> %vdata,
   ; GFX10NSA:   [[COPY13:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr5
   ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; GFX10NSA:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
+  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10NSA:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
+  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX10NSA:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
-  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+  ; GFX10NSA:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; GFX10NSA:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
   ; GFX10NSA:   [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x s16>)
+  ; GFX10NSA:   [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
   ; GFX10NSA:   [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x s16>)
-  ; GFX10NSA:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
-  ; GFX10NSA:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-  ; GFX10NSA:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX10NSA:   [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C1]](s32)
+  ; GFX10NSA:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX10NSA:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
   ; GFX10NSA:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY14]](s32), [[COPY15]](s32)
-  ; GFX10NSA:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
-  ; GFX10NSA:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+  ; GFX10NSA:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
+  ; GFX10NSA:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[LSHR3]](s32)
   ; GFX10NSA:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY16]](s32), [[COPY17]](s32)
   ; GFX10NSA:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
   ; GFX10NSA:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.mip.cube), [[BUILD_VECTOR1]](<4 x s32>), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable store 16 into custom "TargetCustom8")
@@ -1881,15 +2021,18 @@ define amdgpu_ps void @store_mip_1darray(<8 x i32> inreg %rsrc, <4 x float> %vda
   ; GFX9:   [[COPY13:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr5
   ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; GFX9:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
+  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX9:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
+  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX9:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
-  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+  ; GFX9:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; GFX9:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
   ; GFX9:   [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x s16>)
-  ; GFX9:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-  ; GFX9:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX9:   [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
+  ; GFX9:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX9:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
   ; GFX9:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY14]](s32), [[COPY15]](s32)
-  ; GFX9:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
+  ; GFX9:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
   ; GFX9:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
   ; GFX9:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY16]](s32), [[DEF]](s32)
   ; GFX9:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
@@ -1914,15 +2057,18 @@ define amdgpu_ps void @store_mip_1darray(<8 x i32> inreg %rsrc, <4 x float> %vda
   ; GFX10NSA:   [[COPY13:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr5
   ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; GFX10NSA:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
+  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10NSA:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
+  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX10NSA:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
-  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+  ; GFX10NSA:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; GFX10NSA:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
   ; GFX10NSA:   [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x s16>)
-  ; GFX10NSA:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-  ; GFX10NSA:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX10NSA:   [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
+  ; GFX10NSA:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX10NSA:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
   ; GFX10NSA:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY14]](s32), [[COPY15]](s32)
-  ; GFX10NSA:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
+  ; GFX10NSA:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
   ; GFX10NSA:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
   ; GFX10NSA:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY16]](s32), [[DEF]](s32)
   ; GFX10NSA:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
@@ -1956,18 +2102,21 @@ define amdgpu_ps void @store_mip_2darray(<8 x i32> inreg %rsrc, <4 x float> %vda
   ; GFX9:   [[COPY13:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr5
   ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; GFX9:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
+  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX9:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
+  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX9:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
-  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+  ; GFX9:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; GFX9:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
   ; GFX9:   [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x s16>)
+  ; GFX9:   [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
   ; GFX9:   [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x s16>)
-  ; GFX9:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
-  ; GFX9:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-  ; GFX9:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX9:   [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C1]](s32)
+  ; GFX9:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX9:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
   ; GFX9:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY14]](s32), [[COPY15]](s32)
-  ; GFX9:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
-  ; GFX9:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+  ; GFX9:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
+  ; GFX9:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[LSHR3]](s32)
   ; GFX9:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY16]](s32), [[COPY17]](s32)
   ; GFX9:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
   ; GFX9:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.mip.2darray), [[BUILD_VECTOR1]](<4 x s32>), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable store 16 into custom "TargetCustom8")
@@ -1991,18 +2140,21 @@ define amdgpu_ps void @store_mip_2darray(<8 x i32> inreg %rsrc, <4 x float> %vda
   ; GFX10NSA:   [[COPY13:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr5
   ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; GFX10NSA:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
+  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10NSA:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
+  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX10NSA:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
-  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+  ; GFX10NSA:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; GFX10NSA:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
   ; GFX10NSA:   [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x s16>)
+  ; GFX10NSA:   [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
   ; GFX10NSA:   [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x s16>)
-  ; GFX10NSA:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
-  ; GFX10NSA:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-  ; GFX10NSA:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX10NSA:   [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C1]](s32)
+  ; GFX10NSA:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX10NSA:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
   ; GFX10NSA:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY14]](s32), [[COPY15]](s32)
-  ; GFX10NSA:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
-  ; GFX10NSA:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+  ; GFX10NSA:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
+  ; GFX10NSA:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[LSHR3]](s32)
   ; GFX10NSA:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY16]](s32), [[COPY17]](s32)
   ; GFX10NSA:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
   ; GFX10NSA:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.mip.2darray), [[BUILD_VECTOR1]](<4 x s32>), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable store 16 into custom "TargetCustom8")
@@ -2030,8 +2182,10 @@ define amdgpu_ps <4 x float> @getresinfo_1d(<8 x i32> inreg %rsrc, <2 x i16> %co
   ; GFX9:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
   ; GFX9:   [[COPY8:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
   ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX9:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
-  ; GFX9:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+  ; GFX9:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
   ; GFX9:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.getresinfo.1d), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3
   ; GFX9:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9:   $vgpr0 = COPY [[UV]](s32)
@@ -2052,8 +2206,10 @@ define amdgpu_ps <4 x float> @getresinfo_1d(<8 x i32> inreg %rsrc, <2 x i16> %co
   ; GFX10NSA:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
   ; GFX10NSA:   [[COPY8:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
   ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10NSA:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
-  ; GFX10NSA:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+  ; GFX10NSA:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
   ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.getresinfo.1d), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3
   ; GFX10NSA:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10NSA:   $vgpr0 = COPY [[UV]](s32)
@@ -2081,8 +2237,10 @@ define amdgpu_ps <4 x float> @getresinfo_2d(<8 x i32> inreg %rsrc, <2 x i16> %co
   ; GFX9:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
   ; GFX9:   [[COPY8:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
   ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX9:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
-  ; GFX9:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+  ; GFX9:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
   ; GFX9:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.getresinfo.2d), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3
   ; GFX9:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9:   $vgpr0 = COPY [[UV]](s32)
@@ -2103,8 +2261,10 @@ define amdgpu_ps <4 x float> @getresinfo_2d(<8 x i32> inreg %rsrc, <2 x i16> %co
   ; GFX10NSA:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
   ; GFX10NSA:   [[COPY8:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
   ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10NSA:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
-  ; GFX10NSA:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+  ; GFX10NSA:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
   ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.getresinfo.2d), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3
   ; GFX10NSA:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10NSA:   $vgpr0 = COPY [[UV]](s32)
@@ -2132,8 +2292,10 @@ define amdgpu_ps <4 x float> @getresinfo_3d(<8 x i32> inreg %rsrc, <2 x i16> %co
   ; GFX9:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
   ; GFX9:   [[COPY8:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
   ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX9:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
-  ; GFX9:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+  ; GFX9:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
   ; GFX9:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.getresinfo.3d), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3
   ; GFX9:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9:   $vgpr0 = COPY [[UV]](s32)
@@ -2154,8 +2316,10 @@ define amdgpu_ps <4 x float> @getresinfo_3d(<8 x i32> inreg %rsrc, <2 x i16> %co
   ; GFX10NSA:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
   ; GFX10NSA:   [[COPY8:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
   ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10NSA:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
-  ; GFX10NSA:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+  ; GFX10NSA:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
   ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.getresinfo.3d), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3
   ; GFX10NSA:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10NSA:   $vgpr0 = COPY [[UV]](s32)
@@ -2183,8 +2347,10 @@ define amdgpu_ps <4 x float> @getresinfo_cube(<8 x i32> inreg %rsrc, <2 x i16> %
   ; GFX9:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
   ; GFX9:   [[COPY8:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
   ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX9:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
-  ; GFX9:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+  ; GFX9:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
   ; GFX9:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.getresinfo.cube), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3
   ; GFX9:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9:   $vgpr0 = COPY [[UV]](s32)
@@ -2205,8 +2371,10 @@ define amdgpu_ps <4 x float> @getresinfo_cube(<8 x i32> inreg %rsrc, <2 x i16> %
   ; GFX10NSA:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
   ; GFX10NSA:   [[COPY8:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
   ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10NSA:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
-  ; GFX10NSA:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+  ; GFX10NSA:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
   ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.getresinfo.cube), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3
   ; GFX10NSA:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10NSA:   $vgpr0 = COPY [[UV]](s32)
@@ -2234,8 +2402,10 @@ define amdgpu_ps <4 x float> @getresinfo_1darray(<8 x i32> inreg %rsrc, <2 x i16
   ; GFX9:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
   ; GFX9:   [[COPY8:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
   ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX9:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
-  ; GFX9:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+  ; GFX9:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
   ; GFX9:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.getresinfo.1darray), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3
   ; GFX9:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9:   $vgpr0 = COPY [[UV]](s32)
@@ -2256,8 +2426,10 @@ define amdgpu_ps <4 x float> @getresinfo_1darray(<8 x i32> inreg %rsrc, <2 x i16
   ; GFX10NSA:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
   ; GFX10NSA:   [[COPY8:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
   ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10NSA:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
-  ; GFX10NSA:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+  ; GFX10NSA:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
   ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.getresinfo.1darray), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3
   ; GFX10NSA:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10NSA:   $vgpr0 = COPY [[UV]](s32)
@@ -2285,8 +2457,10 @@ define amdgpu_ps <4 x float> @getresinfo_2darray(<8 x i32> inreg %rsrc, <2 x i16
   ; GFX9:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
   ; GFX9:   [[COPY8:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
   ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX9:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
-  ; GFX9:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+  ; GFX9:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
   ; GFX9:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.getresinfo.2darray), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3
   ; GFX9:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9:   $vgpr0 = COPY [[UV]](s32)
@@ -2307,8 +2481,10 @@ define amdgpu_ps <4 x float> @getresinfo_2darray(<8 x i32> inreg %rsrc, <2 x i16
   ; GFX10NSA:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
   ; GFX10NSA:   [[COPY8:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
   ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10NSA:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
-  ; GFX10NSA:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+  ; GFX10NSA:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
   ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.getresinfo.2darray), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3
   ; GFX10NSA:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10NSA:   $vgpr0 = COPY [[UV]](s32)
@@ -2336,8 +2512,10 @@ define amdgpu_ps <4 x float> @getresinfo_2dmsaa(<8 x i32> inreg %rsrc, <2 x i16>
   ; GFX9:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
   ; GFX9:   [[COPY8:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
   ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX9:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
-  ; GFX9:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+  ; GFX9:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
   ; GFX9:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.getresinfo.2dmsaa), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3
   ; GFX9:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9:   $vgpr0 = COPY [[UV]](s32)
@@ -2358,8 +2536,10 @@ define amdgpu_ps <4 x float> @getresinfo_2dmsaa(<8 x i32> inreg %rsrc, <2 x i16>
   ; GFX10NSA:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
   ; GFX10NSA:   [[COPY8:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
   ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10NSA:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
-  ; GFX10NSA:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+  ; GFX10NSA:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
   ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.getresinfo.2dmsaa), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3
   ; GFX10NSA:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10NSA:   $vgpr0 = COPY [[UV]](s32)
@@ -2387,8 +2567,10 @@ define amdgpu_ps <4 x float> @getresinfo_2darraymsaa(<8 x i32> inreg %rsrc, <2 x
   ; GFX9:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
   ; GFX9:   [[COPY8:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
   ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX9:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
-  ; GFX9:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+  ; GFX9:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
   ; GFX9:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.getresinfo.2darraymsaa), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3
   ; GFX9:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9:   $vgpr0 = COPY [[UV]](s32)
@@ -2409,8 +2591,10 @@ define amdgpu_ps <4 x float> @getresinfo_2darraymsaa(<8 x i32> inreg %rsrc, <2 x
   ; GFX10NSA:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
   ; GFX10NSA:   [[COPY8:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
   ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10NSA:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
-  ; GFX10NSA:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+  ; GFX10NSA:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
   ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.getresinfo.2darraymsaa), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3
   ; GFX10NSA:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10NSA:   $vgpr0 = COPY [[UV]](s32)
@@ -2438,8 +2622,10 @@ define amdgpu_ps float @load_1d_V1(<8 x i32> inreg %rsrc, <2 x i16> %coords) {
   ; GFX9:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
   ; GFX9:   [[COPY8:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
   ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX9:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
-  ; GFX9:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+  ; GFX9:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
   ; GFX9:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 8, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable load 4 from custom "TargetCustom8")
   ; GFX9:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX9:   SI_RETURN_TO_EPILOG implicit $vgpr0
@@ -2456,8 +2642,10 @@ define amdgpu_ps float @load_1d_V1(<8 x i32> inreg %rsrc, <2 x i16> %coords) {
   ; GFX10NSA:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
   ; GFX10NSA:   [[COPY8:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
   ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10NSA:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
-  ; GFX10NSA:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+  ; GFX10NSA:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
   ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 8, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable load 4 from custom "TargetCustom8")
   ; GFX10NSA:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX10NSA:   SI_RETURN_TO_EPILOG implicit $vgpr0
@@ -2481,8 +2669,10 @@ define amdgpu_ps <2 x float> @load_1d_V2(<8 x i32> inreg %rsrc, <2 x i16> %coord
   ; GFX9:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
   ; GFX9:   [[COPY8:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
   ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX9:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
-  ; GFX9:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+  ; GFX9:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
   ; GFX9:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 9, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable load 8 from custom "TargetCustom8")
   ; GFX9:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x s32>)
   ; GFX9:   $vgpr0 = COPY [[UV]](s32)
@@ -2501,8 +2691,10 @@ define amdgpu_ps <2 x float> @load_1d_V2(<8 x i32> inreg %rsrc, <2 x i16> %coord
   ; GFX10NSA:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
   ; GFX10NSA:   [[COPY8:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
   ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10NSA:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
-  ; GFX10NSA:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+  ; GFX10NSA:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
   ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 9, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable load 8 from custom "TargetCustom8")
   ; GFX10NSA:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x s32>)
   ; GFX10NSA:   $vgpr0 = COPY [[UV]](s32)
@@ -2529,8 +2721,10 @@ define amdgpu_ps void @store_1d_V1(<8 x i32> inreg %rsrc, float %vdata, <2 x i16
   ; GFX9:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; GFX9:   [[COPY9:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
   ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX9:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x s16>)
-  ; GFX9:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+  ; GFX9:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
   ; GFX9:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1d), [[COPY8]](s32), 2, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable store 4 into custom "TargetCustom8")
   ; GFX9:   S_ENDPGM 0
   ; GFX10NSA-LABEL: name: store_1d_V1
@@ -2547,8 +2741,10 @@ define amdgpu_ps void @store_1d_V1(<8 x i32> inreg %rsrc, float %vdata, <2 x i16
   ; GFX10NSA:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; GFX10NSA:   [[COPY9:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
   ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10NSA:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x s16>)
-  ; GFX10NSA:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+  ; GFX10NSA:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
   ; GFX10NSA:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1d), [[COPY8]](s32), 2, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable store 4 into custom "TargetCustom8")
   ; GFX10NSA:   S_ENDPGM 0
 main_body:
@@ -2574,8 +2770,10 @@ define amdgpu_ps void @store_1d_V2(<8 x i32> inreg %rsrc, <2 x float> %vdata, <2
   ; GFX9:   [[COPY10:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr2
   ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; GFX9:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
+  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX9:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY10]](<2 x s16>)
-  ; GFX9:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+  ; GFX9:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
   ; GFX9:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1d), [[BUILD_VECTOR1]](<2 x s32>), 12, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable store 8 into custom "TargetCustom8")
   ; GFX9:   S_ENDPGM 0
   ; GFX10NSA-LABEL: name: store_1d_V2
@@ -2594,8 +2792,10 @@ define amdgpu_ps void @store_1d_V2(<8 x i32> inreg %rsrc, <2 x float> %vdata, <2
   ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr2
   ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; GFX10NSA:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
+  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10NSA:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY10]](<2 x s16>)
-  ; GFX10NSA:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+  ; GFX10NSA:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
   ; GFX10NSA:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1d), [[BUILD_VECTOR1]](<2 x s32>), 12, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable store 8 into custom "TargetCustom8")
   ; GFX10NSA:   S_ENDPGM 0
 main_body:
@@ -2618,8 +2818,10 @@ define amdgpu_ps <4 x float> @load_1d_glc(<8 x i32> inreg %rsrc, <2 x i16> %coor
   ; GFX9:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
   ; GFX9:   [[COPY8:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
   ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX9:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
-  ; GFX9:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+  ; GFX9:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
   ; GFX9:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 1, 3 :: (dereferenceable load 16 from custom "TargetCustom8")
   ; GFX9:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9:   $vgpr0 = COPY [[UV]](s32)
@@ -2640,8 +2842,10 @@ define amdgpu_ps <4 x float> @load_1d_glc(<8 x i32> inreg %rsrc, <2 x i16> %coor
   ; GFX10NSA:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
   ; GFX10NSA:   [[COPY8:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
   ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10NSA:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
-  ; GFX10NSA:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+  ; GFX10NSA:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
   ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 1, 3 :: (dereferenceable load 16 from custom "TargetCustom8")
   ; GFX10NSA:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10NSA:   $vgpr0 = COPY [[UV]](s32)
@@ -2669,8 +2873,10 @@ define amdgpu_ps <4 x float> @load_1d_slc(<8 x i32> inreg %rsrc, <2 x i16> %coor
   ; GFX9:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
   ; GFX9:   [[COPY8:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
   ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX9:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
-  ; GFX9:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+  ; GFX9:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
   ; GFX9:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 2, 3 :: (dereferenceable load 16 from custom "TargetCustom8")
   ; GFX9:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9:   $vgpr0 = COPY [[UV]](s32)
@@ -2691,8 +2897,10 @@ define amdgpu_ps <4 x float> @load_1d_slc(<8 x i32> inreg %rsrc, <2 x i16> %coor
   ; GFX10NSA:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
   ; GFX10NSA:   [[COPY8:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
   ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10NSA:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
-  ; GFX10NSA:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+  ; GFX10NSA:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
   ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 2, 3 :: (dereferenceable load 16 from custom "TargetCustom8")
   ; GFX10NSA:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10NSA:   $vgpr0 = COPY [[UV]](s32)
@@ -2720,8 +2928,10 @@ define amdgpu_ps <4 x float> @load_1d_glc_slc(<8 x i32> inreg %rsrc, <2 x i16> %
   ; GFX9:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
   ; GFX9:   [[COPY8:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
   ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX9:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
-  ; GFX9:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+  ; GFX9:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
   ; GFX9:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 3, 3 :: (dereferenceable load 16 from custom "TargetCustom8")
   ; GFX9:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9:   $vgpr0 = COPY [[UV]](s32)
@@ -2742,8 +2952,10 @@ define amdgpu_ps <4 x float> @load_1d_glc_slc(<8 x i32> inreg %rsrc, <2 x i16> %
   ; GFX10NSA:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
   ; GFX10NSA:   [[COPY8:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
   ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10NSA:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
-  ; GFX10NSA:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+  ; GFX10NSA:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
   ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 3, 3 :: (dereferenceable load 16 from custom "TargetCustom8")
   ; GFX10NSA:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10NSA:   $vgpr0 = COPY [[UV]](s32)
@@ -2776,8 +2988,10 @@ define amdgpu_ps void @store_1d_glc(<8 x i32> inreg %rsrc, <4 x float> %vdata, <
   ; GFX9:   [[COPY12:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr4
   ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; GFX9:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
+  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX9:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
-  ; GFX9:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+  ; GFX9:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
   ; GFX9:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1d), [[BUILD_VECTOR1]](<4 x s32>), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 1, 3 :: (dereferenceable store 16 into custom "TargetCustom8")
   ; GFX9:   S_ENDPGM 0
   ; GFX10NSA-LABEL: name: store_1d_glc
@@ -2798,8 +3012,10 @@ define amdgpu_ps void @store_1d_glc(<8 x i32> inreg %rsrc, <4 x float> %vdata, <
   ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr4
   ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; GFX10NSA:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
+  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10NSA:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
-  ; GFX10NSA:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+  ; GFX10NSA:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
   ; GFX10NSA:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1d), [[BUILD_VECTOR1]](<4 x s32>), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 1, 3 :: (dereferenceable store 16 into custom "TargetCustom8")
   ; GFX10NSA:   S_ENDPGM 0
 main_body:
@@ -2827,8 +3043,10 @@ define amdgpu_ps void @store_1d_slc(<8 x i32> inreg %rsrc, <4 x float> %vdata, <
   ; GFX9:   [[COPY12:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr4
   ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; GFX9:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
+  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX9:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
-  ; GFX9:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+  ; GFX9:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
   ; GFX9:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1d), [[BUILD_VECTOR1]](<4 x s32>), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 2, 3 :: (dereferenceable store 16 into custom "TargetCustom8")
   ; GFX9:   S_ENDPGM 0
   ; GFX10NSA-LABEL: name: store_1d_slc
@@ -2849,8 +3067,10 @@ define amdgpu_ps void @store_1d_slc(<8 x i32> inreg %rsrc, <4 x float> %vdata, <
   ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr4
   ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; GFX10NSA:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
+  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10NSA:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
-  ; GFX10NSA:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+  ; GFX10NSA:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
   ; GFX10NSA:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1d), [[BUILD_VECTOR1]](<4 x s32>), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 2, 3 :: (dereferenceable store 16 into custom "TargetCustom8")
   ; GFX10NSA:   S_ENDPGM 0
 main_body:
@@ -2878,8 +3098,10 @@ define amdgpu_ps void @store_1d_glc_slc(<8 x i32> inreg %rsrc, <4 x float> %vdat
   ; GFX9:   [[COPY12:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr4
   ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; GFX9:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
+  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX9:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
-  ; GFX9:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+  ; GFX9:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
   ; GFX9:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1d), [[BUILD_VECTOR1]](<4 x s32>), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 3, 3 :: (dereferenceable store 16 into custom "TargetCustom8")
   ; GFX9:   S_ENDPGM 0
   ; GFX10NSA-LABEL: name: store_1d_glc_slc
@@ -2900,8 +3122,10 @@ define amdgpu_ps void @store_1d_glc_slc(<8 x i32> inreg %rsrc, <4 x float> %vdat
   ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr4
   ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; GFX10NSA:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
+  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10NSA:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
-  ; GFX10NSA:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+  ; GFX10NSA:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
   ; GFX10NSA:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1d), [[BUILD_VECTOR1]](<4 x s32>), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 3, 3 :: (dereferenceable store 16 into custom "TargetCustom8")
   ; GFX10NSA:   S_ENDPGM 0
 main_body:
@@ -2970,8 +3194,10 @@ define amdgpu_ps <4 x float> @load_1d_tfe(<8 x i32> inreg %rsrc, <2 x i16> %coor
   ; GFX9:   [[COPY8:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
   ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; GFX9:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX9:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
-  ; GFX9:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+  ; GFX9:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
   ; GFX9:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<5 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 1, 0, 3 :: (dereferenceable load 16 from custom "TargetCustom8")
   ; GFX9:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<5 x s32>)
   ; GFX9:   G_STORE [[UV4]](s32), [[DEF]](p1) :: (store 4 into `i32 addrspace(1)* undef`, addrspace 1)
@@ -2994,8 +3220,10 @@ define amdgpu_ps <4 x float> @load_1d_tfe(<8 x i32> inreg %rsrc, <2 x i16> %coor
   ; GFX10NSA:   [[COPY8:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
   ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; GFX10NSA:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10NSA:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
-  ; GFX10NSA:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+  ; GFX10NSA:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
   ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<5 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 1, 0, 3 :: (dereferenceable load 16 from custom "TargetCustom8")
   ; GFX10NSA:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<5 x s32>)
   ; GFX10NSA:   G_STORE [[UV4]](s32), [[DEF]](p1) :: (store 4 into `i32 addrspace(1)* undef`, addrspace 1)
@@ -3028,12 +3256,14 @@ define amdgpu_ps <4 x float> @load_2d_tfe(<8 x i32> inreg %rsrc, <2 x i16> %coor
   ; GFX9:   [[COPY8:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
   ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; GFX9:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX9:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
+  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX9:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
-  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
-  ; GFX9:   [[COPY9:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-  ; GFX9:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX9:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; GFX9:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
+  ; GFX9:   [[COPY9:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX9:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
   ; GFX9:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY9]](s32), [[COPY10]](s32)
   ; GFX9:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<5 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 15, [[BUILD_VECTOR_TRUNC]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 3 :: (dereferenceable load 16 from custom "TargetCustom8")
   ; GFX9:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<5 x s32>)
@@ -3057,12 +3287,14 @@ define amdgpu_ps <4 x float> @load_2d_tfe(<8 x i32> inreg %rsrc, <2 x i16> %coor
   ; GFX10NSA:   [[COPY8:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
   ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; GFX10NSA:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10NSA:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
+  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX10NSA:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
-  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
-  ; GFX10NSA:   [[COPY9:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX10NSA:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; GFX10NSA:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
+  ; GFX10NSA:   [[COPY9:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
   ; GFX10NSA:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY9]](s32), [[COPY10]](s32)
   ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<5 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 15, [[BUILD_VECTOR_TRUNC]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 3 :: (dereferenceable load 16 from custom "TargetCustom8")
   ; GFX10NSA:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<5 x s32>)
@@ -3098,15 +3330,18 @@ define amdgpu_ps <4 x float> @load_3d_tfe(<8 x i32> inreg %rsrc, <2 x i16> %coor
   ; GFX9:   [[COPY9:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
   ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; GFX9:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX9:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
+  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX9:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
-  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+  ; GFX9:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; GFX9:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
   ; GFX9:   [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x s16>)
-  ; GFX9:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-  ; GFX9:   [[COPY11:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX9:   [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
+  ; GFX9:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX9:   [[COPY11:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
   ; GFX9:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX9:   [[COPY12:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
+  ; GFX9:   [[COPY12:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
   ; GFX9:   [[DEF1:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
   ; GFX9:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY12]](s32), [[DEF1]](s32)
   ; GFX9:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
@@ -3133,15 +3368,18 @@ define amdgpu_ps <4 x float> @load_3d_tfe(<8 x i32> inreg %rsrc, <2 x i16> %coor
   ; GFX10NSA:   [[COPY9:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
   ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; GFX10NSA:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10NSA:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
+  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX10NSA:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
-  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+  ; GFX10NSA:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; GFX10NSA:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
   ; GFX10NSA:   [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x s16>)
-  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-  ; GFX10NSA:   [[COPY11:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX10NSA:   [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
+  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX10NSA:   [[COPY11:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
   ; GFX10NSA:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
+  ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
   ; GFX10NSA:   [[DEF1:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
   ; GFX10NSA:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY12]](s32), [[DEF1]](s32)
   ; GFX10NSA:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
@@ -3180,18 +3418,21 @@ define amdgpu_ps <4 x float> @load_2darraymsaa_tfe(<8 x i32> inreg %rsrc, <2 x i
   ; GFX9:   [[COPY9:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
   ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; GFX9:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX9:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
+  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX9:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
-  ; GFX9:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; GFX9:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+  ; GFX9:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; GFX9:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
   ; GFX9:   [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x s16>)
+  ; GFX9:   [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
   ; GFX9:   [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x s16>)
-  ; GFX9:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
-  ; GFX9:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-  ; GFX9:   [[COPY11:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX9:   [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C1]](s32)
+  ; GFX9:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX9:   [[COPY11:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
   ; GFX9:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX9:   [[COPY12:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
-  ; GFX9:   [[COPY13:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+  ; GFX9:   [[COPY12:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
+  ; GFX9:   [[COPY13:%[0-9]+]]:_(s32) = COPY [[LSHR3]](s32)
   ; GFX9:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY12]](s32), [[COPY13]](s32)
   ; GFX9:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
   ; GFX9:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<5 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2darraymsaa), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 3 :: (dereferenceable load 16 from custom "TargetCustom8")
@@ -3217,18 +3458,21 @@ define amdgpu_ps <4 x float> @load_2darraymsaa_tfe(<8 x i32> inreg %rsrc, <2 x i
   ; GFX10NSA:   [[COPY9:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
   ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; GFX10NSA:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10NSA:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
+  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX10NSA:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
-  ; GFX10NSA:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-  ; GFX10NSA:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+  ; GFX10NSA:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; GFX10NSA:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
   ; GFX10NSA:   [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x s16>)
+  ; GFX10NSA:   [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
   ; GFX10NSA:   [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x s16>)
-  ; GFX10NSA:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
-  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-  ; GFX10NSA:   [[COPY11:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX10NSA:   [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C1]](s32)
+  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+  ; GFX10NSA:   [[COPY11:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
   ; GFX10NSA:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
-  ; GFX10NSA:   [[COPY13:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+  ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
+  ; GFX10NSA:   [[COPY13:%[0-9]+]]:_(s32) = COPY [[LSHR3]](s32)
   ; GFX10NSA:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY12]](s32), [[COPY13]](s32)
   ; GFX10NSA:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
   ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<5 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2darraymsaa), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 3 :: (dereferenceable load 16 from custom "TargetCustom8")
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-shuffle-vector.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-shuffle-vector.mir
index 9f614bef378ed..692078edbe65f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-shuffle-vector.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-shuffle-vector.mir
@@ -217,16 +217,18 @@ body: |
     ; CHECK: liveins: $vgpr0, $vgpr1
     ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
     ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; CHECK: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
+    ; CHECK: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
     ; CHECK: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
-    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; CHECK: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
-    ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C1]]
-    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
-    ; CHECK: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C1]]
-    ; CHECK: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+    ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CHECK: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
+    ; CHECK: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C2]]
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+    ; CHECK: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C2]]
+    ; CHECK: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
     ; CHECK: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
     ; CHECK: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
     ; CHECK: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -250,15 +252,17 @@ body: |
     ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
     ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
     ; CHECK: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
-    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; CHECK: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CHECK: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
     ; CHECK: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
-    ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; CHECK: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+    ; CHECK: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
     ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
-    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C1]]
-    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[BITCAST1]](s32)
-    ; CHECK: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C1]]
-    ; CHECK: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C2]]
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+    ; CHECK: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C2]]
+    ; CHECK: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
     ; CHECK: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
     ; CHECK: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
     ; CHECK: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-shuffle-vector.s16.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-shuffle-vector.s16.mir
index d4de328b679a7..d4b80ce4a5721 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-shuffle-vector.s16.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-shuffle-vector.s16.mir
@@ -45,13 +45,15 @@ body: |
     ; GFX8: liveins: $vgpr0, $vgpr1
     ; GFX8: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
     ; GFX8: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; GFX8: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
-    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; GFX8: [[COPY2:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-    ; GFX8: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
-    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX8: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; GFX8: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; GFX8: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C1]]
+    ; GFX8: [[COPY3:%[0-9]+]]:_(s32) = COPY [[C]](s32)
     ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C2]](s32)
+    ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[C2]](s32)
     ; GFX8: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
     ; GFX8: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
     ; GFX8: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -80,14 +82,16 @@ body: |
     ; GFX8: liveins: $vgpr0, $vgpr1
     ; GFX8: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
     ; GFX8: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
-    ; GFX8: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
     ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX8: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
+    ; GFX8: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+    ; GFX8: [[COPY2:%[0-9]+]]:_(s32) = COPY [[C]](s32)
     ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; GFX8: [[COPY2:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-    ; GFX8: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C1]]
+    ; GFX8: [[COPY3:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; GFX8: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C1]]
     ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND]], [[C2]](s32)
-    ; GFX8: [[OR:%[0-9]+]]:_(s32) = G_OR [[C]], [[SHL]]
+    ; GFX8: [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY2]], [[SHL]]
     ; GFX8: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
     ; GFX8: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
     ; GFX9-LABEL: name: shufflevector_v2s16_v2s16_undef_0
@@ -115,16 +119,18 @@ body: |
     ; GFX8: liveins: $vgpr0, $vgpr1
     ; GFX8: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
     ; GFX8: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; GFX8: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
+    ; GFX8: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
     ; GFX8: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
-    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX8: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
-    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; GFX8: [[COPY2:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-    ; GFX8: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C1]]
-    ; GFX8: [[COPY3:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
-    ; GFX8: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C1]]
-    ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX8: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
+    ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; GFX8: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; GFX8: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C2]]
+    ; GFX8: [[COPY3:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+    ; GFX8: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C2]]
+    ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
     ; GFX8: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
     ; GFX8: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
     ; GFX8: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -154,15 +160,17 @@ body: |
     ; GFX8: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
     ; GFX8: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
     ; GFX8: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
-    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX8: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX8: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
     ; GFX8: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
-    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; GFX8: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+    ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
     ; GFX8: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
-    ; GFX8: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C1]]
-    ; GFX8: [[COPY3:%[0-9]+]]:_(s32) = COPY [[BITCAST1]](s32)
-    ; GFX8: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C1]]
-    ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+    ; GFX8: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C2]]
+    ; GFX8: [[COPY3:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+    ; GFX8: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C2]]
+    ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
     ; GFX8: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
     ; GFX8: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
     ; GFX8: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -191,15 +199,18 @@ body: |
     ; GFX8: liveins: $vgpr0, $vgpr1
     ; GFX8: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
     ; GFX8: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; GFX8: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
+    ; GFX8: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
     ; GFX8: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
-    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; GFX8: [[COPY2:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-    ; GFX8: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
-    ; GFX8: [[COPY3:%[0-9]+]]:_(s32) = COPY [[BITCAST1]](s32)
-    ; GFX8: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C]]
-    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+    ; GFX8: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; GFX8: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; GFX8: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C1]]
+    ; GFX8: [[COPY3:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+    ; GFX8: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C1]]
+    ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
     ; GFX8: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
     ; GFX8: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
     ; GFX8: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -267,15 +278,18 @@ body: |
     ; GFX8: liveins: $vgpr0, $vgpr1
     ; GFX8: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
     ; GFX8: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; GFX8: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
+    ; GFX8: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
     ; GFX8: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
-    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; GFX8: [[COPY2:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-    ; GFX8: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
-    ; GFX8: [[COPY3:%[0-9]+]]:_(s32) = COPY [[BITCAST1]](s32)
-    ; GFX8: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C]]
-    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+    ; GFX8: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; GFX8: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; GFX8: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C1]]
+    ; GFX8: [[COPY3:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+    ; GFX8: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C1]]
+    ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
     ; GFX8: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
     ; GFX8: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
     ; GFX8: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -304,13 +318,15 @@ body: |
     ; GFX8: liveins: $vgpr0, $vgpr1
     ; GFX8: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
     ; GFX8: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; GFX8: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
-    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; GFX8: [[COPY2:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-    ; GFX8: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
-    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX8: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; GFX8: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; GFX8: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C1]]
+    ; GFX8: [[COPY3:%[0-9]+]]:_(s32) = COPY [[C]](s32)
     ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C2]](s32)
+    ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[C2]](s32)
     ; GFX8: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
     ; GFX8: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
     ; GFX8: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -339,14 +355,16 @@ body: |
     ; GFX8: liveins: $vgpr0, $vgpr1
     ; GFX8: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
     ; GFX8: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
-    ; GFX8: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
     ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX8: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
+    ; GFX8: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+    ; GFX8: [[COPY2:%[0-9]+]]:_(s32) = COPY [[C]](s32)
     ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; GFX8: [[COPY2:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-    ; GFX8: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C1]]
+    ; GFX8: [[COPY3:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; GFX8: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C1]]
     ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND]], [[C2]](s32)
-    ; GFX8: [[OR:%[0-9]+]]:_(s32) = G_OR [[C]], [[SHL]]
+    ; GFX8: [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY2]], [[SHL]]
     ; GFX8: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
     ; GFX8: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
     ; GFX9-LABEL: name: shufflevector_v2s16_v2s16_undef_2
@@ -374,16 +392,18 @@ body: |
     ; GFX8: liveins: $vgpr0, $vgpr1
     ; GFX8: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
     ; GFX8: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; GFX8: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
+    ; GFX8: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
     ; GFX8: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
-    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX8: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
-    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; GFX8: [[COPY2:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-    ; GFX8: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C1]]
-    ; GFX8: [[COPY3:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
-    ; GFX8: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C1]]
-    ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX8: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
+    ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; GFX8: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; GFX8: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C2]]
+    ; GFX8: [[COPY3:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+    ; GFX8: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C2]]
+    ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
     ; GFX8: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
     ; GFX8: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
     ; GFX8: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -413,15 +433,17 @@ body: |
     ; GFX8: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
     ; GFX8: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
     ; GFX8: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
-    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX8: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX8: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
     ; GFX8: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
-    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; GFX8: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+    ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
     ; GFX8: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
-    ; GFX8: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C1]]
-    ; GFX8: [[COPY3:%[0-9]+]]:_(s32) = COPY [[BITCAST1]](s32)
-    ; GFX8: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C1]]
-    ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+    ; GFX8: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C2]]
+    ; GFX8: [[COPY3:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+    ; GFX8: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C2]]
+    ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
     ; GFX8: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
     ; GFX8: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
     ; GFX8: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -561,15 +583,18 @@ body: |
     ; GFX8: liveins: $vgpr0, $vgpr1
     ; GFX8: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
     ; GFX8: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; GFX8: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
+    ; GFX8: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
     ; GFX8: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
-    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; GFX8: [[COPY2:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-    ; GFX8: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
-    ; GFX8: [[COPY3:%[0-9]+]]:_(s32) = COPY [[BITCAST1]](s32)
-    ; GFX8: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C]]
-    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+    ; GFX8: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; GFX8: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; GFX8: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C1]]
+    ; GFX8: [[COPY3:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+    ; GFX8: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C1]]
+    ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
     ; GFX8: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
     ; GFX8: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
     ; GFX8: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -577,10 +602,13 @@ body: |
     ; GFX9: liveins: $vgpr0, $vgpr1
     ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
     ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; GFX9: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
     ; GFX9: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
-    ; GFX9: [[COPY2:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-    ; GFX9: [[COPY3:%[0-9]+]]:_(s32) = COPY [[BITCAST1]](s32)
+    ; GFX9: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+    ; GFX9: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; GFX9: [[COPY3:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
     ; GFX9: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY2]](s32), [[COPY3]](s32)
     ; GFX9: $vgpr0 = COPY [[BUILD_VECTOR_TRUNC]](<2 x s16>)
     %0:_(<2 x s16>) = COPY $vgpr0
@@ -602,15 +630,18 @@ body: |
     ; GFX8: liveins: $vgpr0, $vgpr1
     ; GFX8: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
     ; GFX8: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; GFX8: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
+    ; GFX8: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
     ; GFX8: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
-    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; GFX8: [[COPY2:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-    ; GFX8: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
-    ; GFX8: [[COPY3:%[0-9]+]]:_(s32) = COPY [[BITCAST1]](s32)
-    ; GFX8: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C]]
-    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+    ; GFX8: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; GFX8: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; GFX8: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C1]]
+    ; GFX8: [[COPY3:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+    ; GFX8: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C1]]
+    ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
     ; GFX8: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
     ; GFX8: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
     ; GFX8: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -640,15 +671,17 @@ body: |
     ; GFX8: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
     ; GFX8: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
     ; GFX8: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
-    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX8: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX8: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
     ; GFX8: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
-    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; GFX8: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+    ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
     ; GFX8: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
-    ; GFX8: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C1]]
-    ; GFX8: [[COPY3:%[0-9]+]]:_(s32) = COPY [[BITCAST1]](s32)
-    ; GFX8: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C1]]
-    ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+    ; GFX8: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C2]]
+    ; GFX8: [[COPY3:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+    ; GFX8: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C2]]
+    ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
     ; GFX8: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
     ; GFX8: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
     ; GFX8: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -657,11 +690,13 @@ body: |
     ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
     ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
     ; GFX9: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
-    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX9: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
     ; GFX9: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
+    ; GFX9: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
     ; GFX9: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
-    ; GFX9: [[COPY3:%[0-9]+]]:_(s32) = COPY [[BITCAST1]](s32)
+    ; GFX9: [[COPY3:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
     ; GFX9: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY2]](s32), [[COPY3]](s32)
     ; GFX9: $vgpr0 = COPY [[BUILD_VECTOR_TRUNC]](<2 x s16>)
     %0:_(<2 x s16>) = COPY $vgpr0
@@ -683,16 +718,18 @@ body: |
     ; GFX8: liveins: $vgpr0, $vgpr1
     ; GFX8: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
     ; GFX8: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; GFX8: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
+    ; GFX8: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
     ; GFX8: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
-    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX8: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
-    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; GFX8: [[COPY2:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-    ; GFX8: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C1]]
-    ; GFX8: [[COPY3:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
-    ; GFX8: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C1]]
-    ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX8: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
+    ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; GFX8: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; GFX8: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C2]]
+    ; GFX8: [[COPY3:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+    ; GFX8: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C2]]
+    ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
     ; GFX8: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
     ; GFX8: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
     ; GFX8: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -700,12 +737,14 @@ body: |
     ; GFX9: liveins: $vgpr0, $vgpr1
     ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
     ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; GFX9: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
     ; GFX9: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
-    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX9: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
-    ; GFX9: [[COPY2:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-    ; GFX9: [[COPY3:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX9: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
+    ; GFX9: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; GFX9: [[COPY3:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
     ; GFX9: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY2]](s32), [[COPY3]](s32)
     ; GFX9: $vgpr0 = COPY [[BUILD_VECTOR_TRUNC]](<2 x s16>)
     %0:_(<2 x s16>) = COPY $vgpr0
@@ -728,15 +767,17 @@ body: |
     ; GFX8: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
     ; GFX8: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
     ; GFX8: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
-    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX8: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX8: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
     ; GFX8: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
-    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; GFX8: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+    ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
     ; GFX8: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
-    ; GFX8: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C1]]
-    ; GFX8: [[COPY3:%[0-9]+]]:_(s32) = COPY [[BITCAST1]](s32)
-    ; GFX8: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C1]]
-    ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+    ; GFX8: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C2]]
+    ; GFX8: [[COPY3:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+    ; GFX8: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C2]]
+    ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
     ; GFX8: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
     ; GFX8: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
     ; GFX8: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -745,11 +786,13 @@ body: |
     ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
     ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
     ; GFX9: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
-    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX9: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
     ; GFX9: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
+    ; GFX9: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
     ; GFX9: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
-    ; GFX9: [[COPY3:%[0-9]+]]:_(s32) = COPY [[BITCAST1]](s32)
+    ; GFX9: [[COPY3:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
     ; GFX9: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY2]](s32), [[COPY3]](s32)
     ; GFX9: $vgpr0 = COPY [[BUILD_VECTOR_TRUNC]](<2 x s16>)
     %0:_(<2 x s16>) = COPY $vgpr0
@@ -771,16 +814,18 @@ body: |
     ; GFX8: liveins: $vgpr0, $vgpr1
     ; GFX8: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
     ; GFX8: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; GFX8: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
+    ; GFX8: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
     ; GFX8: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
-    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX8: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
-    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; GFX8: [[COPY2:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-    ; GFX8: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C1]]
-    ; GFX8: [[COPY3:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
-    ; GFX8: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C1]]
-    ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX8: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
+    ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; GFX8: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; GFX8: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C2]]
+    ; GFX8: [[COPY3:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+    ; GFX8: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C2]]
+    ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
     ; GFX8: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
     ; GFX8: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
     ; GFX8: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -788,12 +833,14 @@ body: |
     ; GFX9: liveins: $vgpr0, $vgpr1
     ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
     ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; GFX9: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
     ; GFX9: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
-    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX9: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
-    ; GFX9: [[COPY2:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
-    ; GFX9: [[COPY3:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX9: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
+    ; GFX9: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; GFX9: [[COPY3:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
     ; GFX9: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY2]](s32), [[COPY3]](s32)
     ; GFX9: $vgpr0 = COPY [[BUILD_VECTOR_TRUNC]](<2 x s16>)
     %0:_(<2 x s16>) = COPY $vgpr0
@@ -933,24 +980,38 @@ body: |
     ; GFX8: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
     ; GFX8: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
     ; GFX8: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
-    ; GFX8: [[EXTRACT:%[0-9]+]]:_(s16) = G_EXTRACT [[COPY]](<4 x s16>), 16
-    ; GFX8: [[EXTRACT1:%[0-9]+]]:_(s16) = G_EXTRACT [[COPY]](<4 x s16>), 0
-    ; GFX8: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[EXTRACT]](s16)
-    ; GFX8: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[EXTRACT1]](s16)
-    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
-    ; GFX8: [[OR:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL]]
-    ; GFX8: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
-    ; GFX8: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
+    ; GFX8: [[BITCAST:%[0-9]+]]:_(<2 x s32>) = G_BITCAST [[COPY]](<4 x s16>)
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX8: [[EXTRACT:%[0-9]+]]:_(s32) = G_EXTRACT [[BITCAST]](<2 x s32>), 0
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX8: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[EXTRACT]], [[C1]](s32)
+    ; GFX8: [[BITCAST1:%[0-9]+]]:_(<2 x s32>) = G_BITCAST [[COPY]](<4 x s16>)
+    ; GFX8: [[EXTRACT1:%[0-9]+]]:_(s32) = G_EXTRACT [[BITCAST1]](<2 x s32>), 0
+    ; GFX8: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[EXTRACT1]], [[C]](s32)
+    ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; GFX8: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; GFX8: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C2]]
+    ; GFX8: [[COPY3:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+    ; GFX8: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C2]]
+    ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+    ; GFX8: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
+    ; GFX8: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
+    ; GFX8: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
     ; GFX9-LABEL: name: shufflevector_v2s16_v4s16_1_0
     ; GFX9: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
     ; GFX9: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
     ; GFX9: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
-    ; GFX9: [[EXTRACT:%[0-9]+]]:_(s16) = G_EXTRACT [[COPY]](<4 x s16>), 16
-    ; GFX9: [[EXTRACT1:%[0-9]+]]:_(s16) = G_EXTRACT [[COPY]](<4 x s16>), 0
-    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[EXTRACT]](s16)
-    ; GFX9: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[EXTRACT1]](s16)
-    ; GFX9: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[ANYEXT]](s32), [[ANYEXT1]](s32)
+    ; GFX9: [[BITCAST:%[0-9]+]]:_(<2 x s32>) = G_BITCAST [[COPY]](<4 x s16>)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX9: [[EXTRACT:%[0-9]+]]:_(s32) = G_EXTRACT [[BITCAST]](<2 x s32>), 0
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[EXTRACT]], [[C1]](s32)
+    ; GFX9: [[BITCAST1:%[0-9]+]]:_(<2 x s32>) = G_BITCAST [[COPY]](<4 x s16>)
+    ; GFX9: [[EXTRACT1:%[0-9]+]]:_(s32) = G_EXTRACT [[BITCAST1]](<2 x s32>), 0
+    ; GFX9: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[EXTRACT1]], [[C]](s32)
+    ; GFX9: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; GFX9: [[COPY3:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY2]](s32), [[COPY3]](s32)
     ; GFX9: $vgpr0 = COPY [[BUILD_VECTOR_TRUNC]](<2 x s16>)
     %0:_(<4 x s16>) = COPY $vgpr0_vgpr1
     %1:_(<4 x s16>) = COPY $vgpr2_vgpr3
@@ -971,24 +1032,36 @@ body: |
     ; GFX8: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
     ; GFX8: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
     ; GFX8: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
-    ; GFX8: [[EXTRACT:%[0-9]+]]:_(s16) = G_EXTRACT [[COPY]](<4 x s16>), 16
-    ; GFX8: [[EXTRACT1:%[0-9]+]]:_(s16) = G_EXTRACT [[COPY]](<4 x s16>), 48
-    ; GFX8: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[EXTRACT]](s16)
-    ; GFX8: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[EXTRACT1]](s16)
+    ; GFX8: [[BITCAST:%[0-9]+]]:_(<2 x s32>) = G_BITCAST [[COPY]](<4 x s16>)
+    ; GFX8: [[EXTRACT:%[0-9]+]]:_(s32) = G_EXTRACT [[BITCAST]](<2 x s32>), 0
     ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
-    ; GFX8: [[OR:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL]]
-    ; GFX8: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
-    ; GFX8: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
+    ; GFX8: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[EXTRACT]], [[C]](s32)
+    ; GFX8: [[BITCAST1:%[0-9]+]]:_(<2 x s32>) = G_BITCAST [[COPY]](<4 x s16>)
+    ; GFX8: [[EXTRACT1:%[0-9]+]]:_(s32) = G_EXTRACT [[BITCAST1]](<2 x s32>), 32
+    ; GFX8: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[EXTRACT1]], [[C]](s32)
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; GFX8: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; GFX8: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C1]]
+    ; GFX8: [[COPY3:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+    ; GFX8: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C1]]
+    ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+    ; GFX8: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
+    ; GFX8: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
+    ; GFX8: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
     ; GFX9-LABEL: name: shufflevector_v2s16_v4s16_1_3
     ; GFX9: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
     ; GFX9: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
     ; GFX9: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
-    ; GFX9: [[EXTRACT:%[0-9]+]]:_(s16) = G_EXTRACT [[COPY]](<4 x s16>), 16
-    ; GFX9: [[EXTRACT1:%[0-9]+]]:_(s16) = G_EXTRACT [[COPY]](<4 x s16>), 48
-    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[EXTRACT]](s16)
-    ; GFX9: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[EXTRACT1]](s16)
-    ; GFX9: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[ANYEXT]](s32), [[ANYEXT1]](s32)
+    ; GFX9: [[BITCAST:%[0-9]+]]:_(<2 x s32>) = G_BITCAST [[COPY]](<4 x s16>)
+    ; GFX9: [[EXTRACT:%[0-9]+]]:_(s32) = G_EXTRACT [[BITCAST]](<2 x s32>), 0
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[EXTRACT]], [[C]](s32)
+    ; GFX9: [[BITCAST1:%[0-9]+]]:_(<2 x s32>) = G_BITCAST [[COPY]](<4 x s16>)
+    ; GFX9: [[EXTRACT1:%[0-9]+]]:_(s32) = G_EXTRACT [[BITCAST1]](<2 x s32>), 32
+    ; GFX9: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[EXTRACT1]], [[C]](s32)
+    ; GFX9: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; GFX9: [[COPY3:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY2]](s32), [[COPY3]](s32)
     ; GFX9: $vgpr0 = COPY [[BUILD_VECTOR_TRUNC]](<2 x s16>)
     %0:_(<4 x s16>) = COPY $vgpr0_vgpr1
     %1:_(<4 x s16>) = COPY $vgpr2_vgpr3

From e37987563ad194c41125ce836cc04df57737c698 Mon Sep 17 00:00:00 2001
From: Sanjay Patel <spatel@rotateright.com>
Date: Sun, 2 Aug 2020 10:53:51 -0400
Subject: [PATCH 0852/1035] [InstSimplify] add tests for max(max x,y), x) and
 variants; NFC

---
 .../InstSimplify/maxmin_intrinsics.ll         | 176 ++++++++++++++++++
 1 file changed, 176 insertions(+)

diff --git a/llvm/test/Transforms/InstSimplify/maxmin_intrinsics.ll b/llvm/test/Transforms/InstSimplify/maxmin_intrinsics.ll
index 40dacb2d07ba6..6fcc6133aca3b 100644
--- a/llvm/test/Transforms/InstSimplify/maxmin_intrinsics.ll
+++ b/llvm/test/Transforms/InstSimplify/maxmin_intrinsics.ll
@@ -267,3 +267,179 @@ define <2 x i8> @umin_maxval_partial_undef(<2 x i8> %x) {
   %r = call <2 x i8> @llvm.umin.v2i8(<2 x i8> %x, <2 x i8> <i8 255, i8 undef>)
   ret <2 x i8> %r
 }
+
+define i8 @umax_umax(i8 %x, i8 %y) {
+; CHECK-LABEL: @umax_umax(
+; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.umax.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.umax.i8(i8 [[X]], i8 [[M]])
+; CHECK-NEXT:    ret i8 [[M2]]
+;
+  %m = call i8 @llvm.umax.i8(i8 %x, i8 %y)
+  %m2 = call i8 @llvm.umax.i8(i8 %x, i8 %m)
+  ret i8 %m2
+}
+
+define i8 @umax_umax_commute1(i8 %x, i8 %y) {
+; CHECK-LABEL: @umax_umax_commute1(
+; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.umax.i8(i8 [[Y:%.*]], i8 [[X:%.*]])
+; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.umax.i8(i8 [[X]], i8 [[M]])
+; CHECK-NEXT:    ret i8 [[M2]]
+;
+  %m = call i8 @llvm.umax.i8(i8 %y, i8 %x)
+  %m2 = call i8 @llvm.umax.i8(i8 %x, i8 %m)
+  ret i8 %m2
+}
+
+define i8 @umax_umax_commute2(i8 %x, i8 %y) {
+; CHECK-LABEL: @umax_umax_commute2(
+; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.umax.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.umax.i8(i8 [[M]], i8 [[X]])
+; CHECK-NEXT:    ret i8 [[M2]]
+;
+  %m = call i8 @llvm.umax.i8(i8 %x, i8 %y)
+  %m2 = call i8 @llvm.umax.i8(i8 %m, i8 %x)
+  ret i8 %m2
+}
+
+define <2 x i8> @umax_umax_commute3(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @umax_umax_commute3(
+; CHECK-NEXT:    [[M:%.*]] = call <2 x i8> @llvm.umax.v2i8(<2 x i8> [[Y:%.*]], <2 x i8> [[X:%.*]])
+; CHECK-NEXT:    [[M2:%.*]] = call <2 x i8> @llvm.umax.v2i8(<2 x i8> [[M]], <2 x i8> [[X]])
+; CHECK-NEXT:    ret <2 x i8> [[M2]]
+;
+  %m = call <2 x i8> @llvm.umax.v2i8(<2 x i8> %y, <2 x i8> %x)
+  %m2 = call <2 x i8> @llvm.umax.v2i8(<2 x i8> %m, <2 x i8> %x)
+  ret <2 x i8> %m2
+}
+
+define i8 @umin_umin(i8 %x, i8 %y) {
+; CHECK-LABEL: @umin_umin(
+; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.umin.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.umin.i8(i8 [[X]], i8 [[M]])
+; CHECK-NEXT:    ret i8 [[M2]]
+;
+  %m = call i8 @llvm.umin.i8(i8 %x, i8 %y)
+  %m2 = call i8 @llvm.umin.i8(i8 %x, i8 %m)
+  ret i8 %m2
+}
+
+define i8 @umin_umin_commute1(i8 %x, i8 %y) {
+; CHECK-LABEL: @umin_umin_commute1(
+; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.umin.i8(i8 [[Y:%.*]], i8 [[X:%.*]])
+; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.umin.i8(i8 [[X]], i8 [[M]])
+; CHECK-NEXT:    ret i8 [[M2]]
+;
+  %m = call i8 @llvm.umin.i8(i8 %y, i8 %x)
+  %m2 = call i8 @llvm.umin.i8(i8 %x, i8 %m)
+  ret i8 %m2
+}
+
+define <2 x i8> @umin_umin_commute2(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @umin_umin_commute2(
+; CHECK-NEXT:    [[M:%.*]] = call <2 x i8> @llvm.umin.v2i8(<2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]])
+; CHECK-NEXT:    [[M2:%.*]] = call <2 x i8> @llvm.umin.v2i8(<2 x i8> [[M]], <2 x i8> [[X]])
+; CHECK-NEXT:    ret <2 x i8> [[M2]]
+;
+  %m = call <2 x i8> @llvm.umin.v2i8(<2 x i8> %x, <2 x i8> %y)
+  %m2 = call <2 x i8> @llvm.umin.v2i8(<2 x i8> %m, <2 x i8> %x)
+  ret <2 x i8> %m2
+}
+
+define i8 @umin_umin_commute3(i8 %x, i8 %y) {
+; CHECK-LABEL: @umin_umin_commute3(
+; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.umin.i8(i8 [[Y:%.*]], i8 [[X:%.*]])
+; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.umin.i8(i8 [[M]], i8 [[X]])
+; CHECK-NEXT:    ret i8 [[M2]]
+;
+  %m = call i8 @llvm.umin.i8(i8 %y, i8 %x)
+  %m2 = call i8 @llvm.umin.i8(i8 %m, i8 %x)
+  ret i8 %m2
+}
+
+define i8 @smax_smax(i8 %x, i8 %y) {
+; CHECK-LABEL: @smax_smax(
+; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.smax.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.smax.i8(i8 [[X]], i8 [[M]])
+; CHECK-NEXT:    ret i8 [[M2]]
+;
+  %m = call i8 @llvm.smax.i8(i8 %x, i8 %y)
+  %m2 = call i8 @llvm.smax.i8(i8 %x, i8 %m)
+  ret i8 %m2
+}
+
+define <2 x i8> @smax_smax_commute1(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @smax_smax_commute1(
+; CHECK-NEXT:    [[M:%.*]] = call <2 x i8> @llvm.smax.v2i8(<2 x i8> [[Y:%.*]], <2 x i8> [[X:%.*]])
+; CHECK-NEXT:    [[M2:%.*]] = call <2 x i8> @llvm.smax.v2i8(<2 x i8> [[X]], <2 x i8> [[M]])
+; CHECK-NEXT:    ret <2 x i8> [[M2]]
+;
+  %m = call <2 x i8> @llvm.smax.v2i8(<2 x i8> %y, <2 x i8> %x)
+  %m2 = call <2 x i8> @llvm.smax.v2i8(<2 x i8> %x, <2 x i8> %m)
+  ret <2 x i8> %m2
+}
+
+define i8 @smax_smax_commute2(i8 %x, i8 %y) {
+; CHECK-LABEL: @smax_smax_commute2(
+; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.smax.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.smax.i8(i8 [[M]], i8 [[X]])
+; CHECK-NEXT:    ret i8 [[M2]]
+;
+  %m = call i8 @llvm.smax.i8(i8 %x, i8 %y)
+  %m2 = call i8 @llvm.smax.i8(i8 %m, i8 %x)
+  ret i8 %m2
+}
+
+define i8 @smax_smax_commute3(i8 %x, i8 %y) {
+; CHECK-LABEL: @smax_smax_commute3(
+; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.smax.i8(i8 [[Y:%.*]], i8 [[X:%.*]])
+; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.smax.i8(i8 [[M]], i8 [[X]])
+; CHECK-NEXT:    ret i8 [[M2]]
+;
+  %m = call i8 @llvm.smax.i8(i8 %y, i8 %x)
+  %m2 = call i8 @llvm.smax.i8(i8 %m, i8 %x)
+  ret i8 %m2
+}
+
+define <2 x i8> @smin_smin(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @smin_smin(
+; CHECK-NEXT:    [[M:%.*]] = call <2 x i8> @llvm.smin.v2i8(<2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]])
+; CHECK-NEXT:    [[M2:%.*]] = call <2 x i8> @llvm.smin.v2i8(<2 x i8> [[X]], <2 x i8> [[M]])
+; CHECK-NEXT:    ret <2 x i8> [[M2]]
+;
+  %m = call <2 x i8> @llvm.smin.v2i8(<2 x i8> %x, <2 x i8> %y)
+  %m2 = call <2 x i8> @llvm.smin.v2i8(<2 x i8> %x, <2 x i8> %m)
+  ret <2 x i8> %m2
+}
+
+define i8 @smin_smin_commute1(i8 %x, i8 %y) {
+; CHECK-LABEL: @smin_smin_commute1(
+; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.smin.i8(i8 [[Y:%.*]], i8 [[X:%.*]])
+; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.smin.i8(i8 [[X]], i8 [[M]])
+; CHECK-NEXT:    ret i8 [[M2]]
+;
+  %m = call i8 @llvm.smin.i8(i8 %y, i8 %x)
+  %m2 = call i8 @llvm.smin.i8(i8 %x, i8 %m)
+  ret i8 %m2
+}
+
+define i8 @smin_smin_commute2(i8 %x, i8 %y) {
+; CHECK-LABEL: @smin_smin_commute2(
+; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.smin.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.smin.i8(i8 [[M]], i8 [[X]])
+; CHECK-NEXT:    ret i8 [[M2]]
+;
+  %m = call i8 @llvm.smin.i8(i8 %x, i8 %y)
+  %m2 = call i8 @llvm.smin.i8(i8 %m, i8 %x)
+  ret i8 %m2
+}
+
+define i8 @smin_smin_commute3(i8 %x, i8 %y) {
+; CHECK-LABEL: @smin_smin_commute3(
+; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.smin.i8(i8 [[Y:%.*]], i8 [[X:%.*]])
+; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.smin.i8(i8 [[M]], i8 [[X]])
+; CHECK-NEXT:    ret i8 [[M2]]
+;
+  %m = call i8 @llvm.smin.i8(i8 %y, i8 %x)
+  %m2 = call i8 @llvm.smin.i8(i8 %m, i8 %x)
+  ret i8 %m2
+}

From 4abc69c6f541e7726913c9b0940728b1e0024b4a Mon Sep 17 00:00:00 2001
From: Sanjay Patel <spatel@rotateright.com>
Date: Sun, 2 Aug 2020 11:11:05 -0400
Subject: [PATCH 0853/1035] [InstSimplify] fold max (max X, Y), X --> max X, Y

https://alive2.llvm.org/ce/z/VGgG3M
---
 llvm/lib/Analysis/InstructionSimplify.cpp     | 11 +++++
 .../InstSimplify/maxmin_intrinsics.ll         | 48 +++++++------------
 2 files changed, 27 insertions(+), 32 deletions(-)

diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp
index 4f09ea1896643..6e75478d52afd 100644
--- a/llvm/lib/Analysis/InstructionSimplify.cpp
+++ b/llvm/lib/Analysis/InstructionSimplify.cpp
@@ -5239,6 +5239,17 @@ static Value *simplifyBinaryIntrinsic(Function *F, Value *Op0, Value *Op1,
         return ConstantInt::get(ReturnType, APInt::getMinValue(BitWidth));
     }
 
+    // For 4 commuted variants of each intrinsic:
+    // max (max X, Y), X --> max X, Y
+    if (auto *MinMax0 = dyn_cast<IntrinsicInst>(Op0))
+      if (MinMax0->getIntrinsicID() == IID &&
+          (MinMax0->getOperand(0) == Op1 || MinMax0->getOperand(1) == Op1))
+        return MinMax0;
+    if (auto *MinMax1 = dyn_cast<IntrinsicInst>(Op1))
+      if (MinMax1->getIntrinsicID() == IID &&
+          (MinMax1->getOperand(0) == Op0 || MinMax1->getOperand(1) == Op0))
+        return MinMax1;
+
     const APInt *C;
     if (!match(Op1, m_APIntAllowUndef(C)))
       break;
diff --git a/llvm/test/Transforms/InstSimplify/maxmin_intrinsics.ll b/llvm/test/Transforms/InstSimplify/maxmin_intrinsics.ll
index 6fcc6133aca3b..c0064ab0a423a 100644
--- a/llvm/test/Transforms/InstSimplify/maxmin_intrinsics.ll
+++ b/llvm/test/Transforms/InstSimplify/maxmin_intrinsics.ll
@@ -271,8 +271,7 @@ define <2 x i8> @umin_maxval_partial_undef(<2 x i8> %x) {
 define i8 @umax_umax(i8 %x, i8 %y) {
 ; CHECK-LABEL: @umax_umax(
 ; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.umax.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
-; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.umax.i8(i8 [[X]], i8 [[M]])
-; CHECK-NEXT:    ret i8 [[M2]]
+; CHECK-NEXT:    ret i8 [[M]]
 ;
   %m = call i8 @llvm.umax.i8(i8 %x, i8 %y)
   %m2 = call i8 @llvm.umax.i8(i8 %x, i8 %m)
@@ -282,8 +281,7 @@ define i8 @umax_umax(i8 %x, i8 %y) {
 define i8 @umax_umax_commute1(i8 %x, i8 %y) {
 ; CHECK-LABEL: @umax_umax_commute1(
 ; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.umax.i8(i8 [[Y:%.*]], i8 [[X:%.*]])
-; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.umax.i8(i8 [[X]], i8 [[M]])
-; CHECK-NEXT:    ret i8 [[M2]]
+; CHECK-NEXT:    ret i8 [[M]]
 ;
   %m = call i8 @llvm.umax.i8(i8 %y, i8 %x)
   %m2 = call i8 @llvm.umax.i8(i8 %x, i8 %m)
@@ -293,8 +291,7 @@ define i8 @umax_umax_commute1(i8 %x, i8 %y) {
 define i8 @umax_umax_commute2(i8 %x, i8 %y) {
 ; CHECK-LABEL: @umax_umax_commute2(
 ; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.umax.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
-; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.umax.i8(i8 [[M]], i8 [[X]])
-; CHECK-NEXT:    ret i8 [[M2]]
+; CHECK-NEXT:    ret i8 [[M]]
 ;
   %m = call i8 @llvm.umax.i8(i8 %x, i8 %y)
   %m2 = call i8 @llvm.umax.i8(i8 %m, i8 %x)
@@ -304,8 +301,7 @@ define i8 @umax_umax_commute2(i8 %x, i8 %y) {
 define <2 x i8> @umax_umax_commute3(<2 x i8> %x, <2 x i8> %y) {
 ; CHECK-LABEL: @umax_umax_commute3(
 ; CHECK-NEXT:    [[M:%.*]] = call <2 x i8> @llvm.umax.v2i8(<2 x i8> [[Y:%.*]], <2 x i8> [[X:%.*]])
-; CHECK-NEXT:    [[M2:%.*]] = call <2 x i8> @llvm.umax.v2i8(<2 x i8> [[M]], <2 x i8> [[X]])
-; CHECK-NEXT:    ret <2 x i8> [[M2]]
+; CHECK-NEXT:    ret <2 x i8> [[M]]
 ;
   %m = call <2 x i8> @llvm.umax.v2i8(<2 x i8> %y, <2 x i8> %x)
   %m2 = call <2 x i8> @llvm.umax.v2i8(<2 x i8> %m, <2 x i8> %x)
@@ -315,8 +311,7 @@ define <2 x i8> @umax_umax_commute3(<2 x i8> %x, <2 x i8> %y) {
 define i8 @umin_umin(i8 %x, i8 %y) {
 ; CHECK-LABEL: @umin_umin(
 ; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.umin.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
-; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.umin.i8(i8 [[X]], i8 [[M]])
-; CHECK-NEXT:    ret i8 [[M2]]
+; CHECK-NEXT:    ret i8 [[M]]
 ;
   %m = call i8 @llvm.umin.i8(i8 %x, i8 %y)
   %m2 = call i8 @llvm.umin.i8(i8 %x, i8 %m)
@@ -326,8 +321,7 @@ define i8 @umin_umin(i8 %x, i8 %y) {
 define i8 @umin_umin_commute1(i8 %x, i8 %y) {
 ; CHECK-LABEL: @umin_umin_commute1(
 ; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.umin.i8(i8 [[Y:%.*]], i8 [[X:%.*]])
-; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.umin.i8(i8 [[X]], i8 [[M]])
-; CHECK-NEXT:    ret i8 [[M2]]
+; CHECK-NEXT:    ret i8 [[M]]
 ;
   %m = call i8 @llvm.umin.i8(i8 %y, i8 %x)
   %m2 = call i8 @llvm.umin.i8(i8 %x, i8 %m)
@@ -337,8 +331,7 @@ define i8 @umin_umin_commute1(i8 %x, i8 %y) {
 define <2 x i8> @umin_umin_commute2(<2 x i8> %x, <2 x i8> %y) {
 ; CHECK-LABEL: @umin_umin_commute2(
 ; CHECK-NEXT:    [[M:%.*]] = call <2 x i8> @llvm.umin.v2i8(<2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]])
-; CHECK-NEXT:    [[M2:%.*]] = call <2 x i8> @llvm.umin.v2i8(<2 x i8> [[M]], <2 x i8> [[X]])
-; CHECK-NEXT:    ret <2 x i8> [[M2]]
+; CHECK-NEXT:    ret <2 x i8> [[M]]
 ;
   %m = call <2 x i8> @llvm.umin.v2i8(<2 x i8> %x, <2 x i8> %y)
   %m2 = call <2 x i8> @llvm.umin.v2i8(<2 x i8> %m, <2 x i8> %x)
@@ -348,8 +341,7 @@ define <2 x i8> @umin_umin_commute2(<2 x i8> %x, <2 x i8> %y) {
 define i8 @umin_umin_commute3(i8 %x, i8 %y) {
 ; CHECK-LABEL: @umin_umin_commute3(
 ; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.umin.i8(i8 [[Y:%.*]], i8 [[X:%.*]])
-; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.umin.i8(i8 [[M]], i8 [[X]])
-; CHECK-NEXT:    ret i8 [[M2]]
+; CHECK-NEXT:    ret i8 [[M]]
 ;
   %m = call i8 @llvm.umin.i8(i8 %y, i8 %x)
   %m2 = call i8 @llvm.umin.i8(i8 %m, i8 %x)
@@ -359,8 +351,7 @@ define i8 @umin_umin_commute3(i8 %x, i8 %y) {
 define i8 @smax_smax(i8 %x, i8 %y) {
 ; CHECK-LABEL: @smax_smax(
 ; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.smax.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
-; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.smax.i8(i8 [[X]], i8 [[M]])
-; CHECK-NEXT:    ret i8 [[M2]]
+; CHECK-NEXT:    ret i8 [[M]]
 ;
   %m = call i8 @llvm.smax.i8(i8 %x, i8 %y)
   %m2 = call i8 @llvm.smax.i8(i8 %x, i8 %m)
@@ -370,8 +361,7 @@ define i8 @smax_smax(i8 %x, i8 %y) {
 define <2 x i8> @smax_smax_commute1(<2 x i8> %x, <2 x i8> %y) {
 ; CHECK-LABEL: @smax_smax_commute1(
 ; CHECK-NEXT:    [[M:%.*]] = call <2 x i8> @llvm.smax.v2i8(<2 x i8> [[Y:%.*]], <2 x i8> [[X:%.*]])
-; CHECK-NEXT:    [[M2:%.*]] = call <2 x i8> @llvm.smax.v2i8(<2 x i8> [[X]], <2 x i8> [[M]])
-; CHECK-NEXT:    ret <2 x i8> [[M2]]
+; CHECK-NEXT:    ret <2 x i8> [[M]]
 ;
   %m = call <2 x i8> @llvm.smax.v2i8(<2 x i8> %y, <2 x i8> %x)
   %m2 = call <2 x i8> @llvm.smax.v2i8(<2 x i8> %x, <2 x i8> %m)
@@ -381,8 +371,7 @@ define <2 x i8> @smax_smax_commute1(<2 x i8> %x, <2 x i8> %y) {
 define i8 @smax_smax_commute2(i8 %x, i8 %y) {
 ; CHECK-LABEL: @smax_smax_commute2(
 ; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.smax.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
-; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.smax.i8(i8 [[M]], i8 [[X]])
-; CHECK-NEXT:    ret i8 [[M2]]
+; CHECK-NEXT:    ret i8 [[M]]
 ;
   %m = call i8 @llvm.smax.i8(i8 %x, i8 %y)
   %m2 = call i8 @llvm.smax.i8(i8 %m, i8 %x)
@@ -392,8 +381,7 @@ define i8 @smax_smax_commute2(i8 %x, i8 %y) {
 define i8 @smax_smax_commute3(i8 %x, i8 %y) {
 ; CHECK-LABEL: @smax_smax_commute3(
 ; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.smax.i8(i8 [[Y:%.*]], i8 [[X:%.*]])
-; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.smax.i8(i8 [[M]], i8 [[X]])
-; CHECK-NEXT:    ret i8 [[M2]]
+; CHECK-NEXT:    ret i8 [[M]]
 ;
   %m = call i8 @llvm.smax.i8(i8 %y, i8 %x)
   %m2 = call i8 @llvm.smax.i8(i8 %m, i8 %x)
@@ -403,8 +391,7 @@ define i8 @smax_smax_commute3(i8 %x, i8 %y) {
 define <2 x i8> @smin_smin(<2 x i8> %x, <2 x i8> %y) {
 ; CHECK-LABEL: @smin_smin(
 ; CHECK-NEXT:    [[M:%.*]] = call <2 x i8> @llvm.smin.v2i8(<2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]])
-; CHECK-NEXT:    [[M2:%.*]] = call <2 x i8> @llvm.smin.v2i8(<2 x i8> [[X]], <2 x i8> [[M]])
-; CHECK-NEXT:    ret <2 x i8> [[M2]]
+; CHECK-NEXT:    ret <2 x i8> [[M]]
 ;
   %m = call <2 x i8> @llvm.smin.v2i8(<2 x i8> %x, <2 x i8> %y)
   %m2 = call <2 x i8> @llvm.smin.v2i8(<2 x i8> %x, <2 x i8> %m)
@@ -414,8 +401,7 @@ define <2 x i8> @smin_smin(<2 x i8> %x, <2 x i8> %y) {
 define i8 @smin_smin_commute1(i8 %x, i8 %y) {
 ; CHECK-LABEL: @smin_smin_commute1(
 ; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.smin.i8(i8 [[Y:%.*]], i8 [[X:%.*]])
-; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.smin.i8(i8 [[X]], i8 [[M]])
-; CHECK-NEXT:    ret i8 [[M2]]
+; CHECK-NEXT:    ret i8 [[M]]
 ;
   %m = call i8 @llvm.smin.i8(i8 %y, i8 %x)
   %m2 = call i8 @llvm.smin.i8(i8 %x, i8 %m)
@@ -425,8 +411,7 @@ define i8 @smin_smin_commute1(i8 %x, i8 %y) {
 define i8 @smin_smin_commute2(i8 %x, i8 %y) {
 ; CHECK-LABEL: @smin_smin_commute2(
 ; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.smin.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
-; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.smin.i8(i8 [[M]], i8 [[X]])
-; CHECK-NEXT:    ret i8 [[M2]]
+; CHECK-NEXT:    ret i8 [[M]]
 ;
   %m = call i8 @llvm.smin.i8(i8 %x, i8 %y)
   %m2 = call i8 @llvm.smin.i8(i8 %m, i8 %x)
@@ -436,8 +421,7 @@ define i8 @smin_smin_commute2(i8 %x, i8 %y) {
 define i8 @smin_smin_commute3(i8 %x, i8 %y) {
 ; CHECK-LABEL: @smin_smin_commute3(
 ; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.smin.i8(i8 [[Y:%.*]], i8 [[X:%.*]])
-; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.smin.i8(i8 [[M]], i8 [[X]])
-; CHECK-NEXT:    ret i8 [[M2]]
+; CHECK-NEXT:    ret i8 [[M]]
 ;
   %m = call i8 @llvm.smin.i8(i8 %y, i8 %x)
   %m2 = call i8 @llvm.smin.i8(i8 %m, i8 %x)

From e20223672100ed4826827412b80a605c759538da Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Sun, 2 Aug 2020 16:55:16 +0100
Subject: [PATCH 0854/1035] [IR] Add
 IRBuilderBase::CreateVectorSplat(ElementCount EC) variant

As discussed on D81500, this adds a more general ElementCount variant of the build helper and converts the (non-scalable) unsigned NumElts variant to use it internally.
---
 llvm/include/llvm/IR/IRBuilder.h            |  4 ++++
 llvm/lib/IR/IRBuilder.cpp                   | 13 +++++++++----
 llvm/unittests/Analysis/VectorUtilsTest.cpp |  3 +++
 3 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/llvm/include/llvm/IR/IRBuilder.h b/llvm/include/llvm/IR/IRBuilder.h
index ffec4ff64ca66..d467789132aca 100644
--- a/llvm/include/llvm/IR/IRBuilder.h
+++ b/llvm/include/llvm/IR/IRBuilder.h
@@ -2484,6 +2484,10 @@ class IRBuilderBase {
   /// NumElts elements.
   Value *CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name = "");
 
+  /// Return a vector value that contains \arg V broadcasted to \p
+  /// EC elements.
+  Value *CreateVectorSplat(ElementCount EC, Value *V, const Twine &Name = "");
+
   /// Return a value that has been extracted from a larger integer type.
   Value *CreateExtractInteger(const DataLayout &DL, Value *From,
                               IntegerType *ExtractedTy, uint64_t Offset,
diff --git a/llvm/lib/IR/IRBuilder.cpp b/llvm/lib/IR/IRBuilder.cpp
index 1fffce015f707..f223f2c6a2b3d 100644
--- a/llvm/lib/IR/IRBuilder.cpp
+++ b/llvm/lib/IR/IRBuilder.cpp
@@ -996,17 +996,22 @@ Value *IRBuilderBase::CreateStripInvariantGroup(Value *Ptr) {
 
 Value *IRBuilderBase::CreateVectorSplat(unsigned NumElts, Value *V,
                                         const Twine &Name) {
-  assert(NumElts > 0 && "Cannot splat to an empty vector!");
+  ElementCount EC(NumElts, false);
+  return CreateVectorSplat(EC, V, Name);
+}
+
+Value *IRBuilderBase::CreateVectorSplat(ElementCount EC, Value *V,
+                                        const Twine &Name) {
+  assert(EC.Min > 0 && "Cannot splat to an empty vector!");
 
   // First insert it into an undef vector so we can shuffle it.
   Type *I32Ty = getInt32Ty();
-  Value *Undef = UndefValue::get(FixedVectorType::get(V->getType(), NumElts));
+  Value *Undef = UndefValue::get(VectorType::get(V->getType(), EC));
   V = CreateInsertElement(Undef, V, ConstantInt::get(I32Ty, 0),
                           Name + ".splatinsert");
 
   // Shuffle the value across the desired number of elements.
-  Value *Zeros =
-      ConstantAggregateZero::get(FixedVectorType::get(I32Ty, NumElts));
+  Value *Zeros = ConstantAggregateZero::get(VectorType::get(I32Ty, EC));
   return CreateShuffleVector(V, Undef, Zeros, Name + ".splat");
 }
 
diff --git a/llvm/unittests/Analysis/VectorUtilsTest.cpp b/llvm/unittests/Analysis/VectorUtilsTest.cpp
index 69e5285e87312..731ebdfe16f52 100644
--- a/llvm/unittests/Analysis/VectorUtilsTest.cpp
+++ b/llvm/unittests/Analysis/VectorUtilsTest.cpp
@@ -93,6 +93,9 @@ TEST_F(BasicTest, isSplat) {
   Value *SplatC = IRB.CreateVectorSplat(5, ScalarC);
   EXPECT_TRUE(isSplatValue(SplatC));
 
+  Value *SplatC_SVE = IRB.CreateVectorSplat(ElementCount(5, true), ScalarC);
+  EXPECT_TRUE(isSplatValue(SplatC_SVE));
+
   // FIXME: Constant splat analysis does not allow undef elements.
   Constant *SplatWithUndefC = ConstantVector::get({ScalarC, UndefScalar});
   EXPECT_FALSE(isSplatValue(SplatWithUndefC));

From 4091413c0047b58853b3f62dd5f36c836f75330d Mon Sep 17 00:00:00 2001
From: Mehdi Amini <joker.eph@gmail.com>
Date: Sun, 2 Aug 2020 16:57:02 +0000
Subject: [PATCH 0855/1035] Remove debug flags from test (NFC)

---
 mlir/test/mlir-cuda-runner/two-modules.mlir | 2 +-
 mlir/test/mlir-rocm-runner/two-modules.mlir | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/mlir/test/mlir-cuda-runner/two-modules.mlir b/mlir/test/mlir-cuda-runner/two-modules.mlir
index ef4dd0c48b8df..9bdda2ae9c667 100644
--- a/mlir/test/mlir-cuda-runner/two-modules.mlir
+++ b/mlir/test/mlir-cuda-runner/two-modules.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-cuda-runner %s --print-ir-after-all --shared-libs=%cuda_wrapper_library_dir/libcuda-runtime-wrappers%shlibext,%linalg_test_lib_dir/libmlir_runner_utils%shlibext --entry-point-result=void | FileCheck %s --dump-input=always
+// RUN: mlir-cuda-runner %s --shared-libs=%cuda_wrapper_library_dir/libcuda-runtime-wrappers%shlibext,%linalg_test_lib_dir/libmlir_runner_utils%shlibext --entry-point-result=void | FileCheck %s
 
 // CHECK: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]
 func @main() {
diff --git a/mlir/test/mlir-rocm-runner/two-modules.mlir b/mlir/test/mlir-rocm-runner/two-modules.mlir
index 8aa62eda73992..d6b92229b5856 100644
--- a/mlir/test/mlir-rocm-runner/two-modules.mlir
+++ b/mlir/test/mlir-rocm-runner/two-modules.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-rocm-runner %s --print-ir-after-all --shared-libs=%rocm_wrapper_library_dir/librocm-runtime-wrappers%shlibext,%linalg_test_lib_dir/libmlir_runner_utils%shlibext --entry-point-result=void | FileCheck %s --dump-input=always
+// RUN: mlir-rocm-runner %s --shared-libs=%rocm_wrapper_library_dir/librocm-runtime-wrappers%shlibext,%linalg_test_lib_dir/libmlir_runner_utils%shlibext --entry-point-result=void | FileCheck %s
 
 // CHECK: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]
 func @main() {

From e7a8ee00e6c3b20fc04792db1acf9d5324a1b7bb Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Sun, 2 Aug 2020 18:11:01 +0100
Subject: [PATCH 0856/1035] [AMDGPU] Regenerate tests to fix whitespace
 indentations

Noticed while updating D77804
---
 llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll | 216 +++++++--------
 llvm/test/CodeGen/AMDGPU/fshr.ll          | 310 +++++++++++-----------
 2 files changed, 263 insertions(+), 263 deletions(-)

diff --git a/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll b/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
index 7ea072bffecb5..f520b4a8fd8f4 100644
--- a/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
+++ b/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
@@ -636,81 +636,81 @@ define amdgpu_kernel void @load_v4i8_to_v4f32_unaligned(<4 x float> addrspace(1)
 define amdgpu_kernel void @load_v4i8_to_v4f32_2_uses(<4 x float> addrspace(1)* noalias %out, <4 x i8> addrspace(1)* noalias %out2, <4 x i8> addrspace(1)* noalias %in) nounwind {
 ; SI-LABEL: load_v4i8_to_v4f32_2_uses:
 ; SI:       ; %bb.0:
-; SI-NEXT:        s_load_dwordx2 s[4:5], s[0:1], 0x9
-; SI-NEXT:        s_load_dwordx2 s[8:9], s[0:1], 0xb
-; SI-NEXT:        s_load_dwordx2 s[0:1], s[0:1], 0xd
-; SI-NEXT:        s_mov_b32 s11, 0xf000
-; SI-NEXT:        s_mov_b32 s2, 0
-; SI-NEXT:        s_mov_b32 s3, s11
-; SI-NEXT:        v_lshlrev_b32_e32 v0, 2, v0
-; SI-NEXT:        v_mov_b32_e32 v1, 0
-; SI-NEXT:        s_waitcnt lgkmcnt(0)
-; SI-NEXT:        buffer_load_dword v4, v[0:1], s[0:3], 0 addr64
-; SI-NEXT:        s_mov_b32 s10, -1
-; SI-NEXT:        s_movk_i32 s0, 0xff
-; SI-NEXT:        s_mov_b32 s6, s10
-; SI-NEXT:        s_mov_b32 s7, s11
-; SI-NEXT:        s_waitcnt vmcnt(0)
-; SI-NEXT:        v_lshrrev_b32_e32 v5, 16, v4
-; SI-NEXT:        v_lshrrev_b32_e32 v6, 24, v4
-; SI-NEXT:        v_and_b32_e32 v7, 0xff00, v4
-; SI-NEXT:        v_cvt_f32_ubyte3_e32 v3, v4
-; SI-NEXT:        v_cvt_f32_ubyte2_e32 v2, v4
-; SI-NEXT:        v_cvt_f32_ubyte1_e32 v1, v4
-; SI-NEXT:        v_cvt_f32_ubyte0_e32 v0, v4
-; SI-NEXT:        v_add_i32_e32 v4, vcc, 9, v4
-; SI-NEXT:        buffer_store_dwordx4 v[0:3], off, s[4:7], 0
-; SI-NEXT:        s_waitcnt expcnt(0)
-; SI-NEXT:        v_and_b32_e32 v0, s0, v4
-; SI-NEXT:        v_add_i32_e32 v2, vcc, 9, v5
-; SI-NEXT:        v_or_b32_e32 v0, v7, v0
-; SI-NEXT:        v_lshlrev_b32_e32 v1, 8, v6
-; SI-NEXT:        v_and_b32_e32 v2, s0, v2
-; SI-NEXT:        v_add_i32_e32 v0, vcc, 0x900, v0
-; SI-NEXT:        v_or_b32_e32 v1, v1, v2
-; SI-NEXT:        v_and_b32_e32 v0, 0xffff, v0
-; SI-NEXT:        v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT:        v_or_b32_e32 v0, v1, v0
-; SI-NEXT:        v_add_i32_e32 v0, vcc, 0x9000000, v0
-; SI-NEXT:        buffer_store_dword v0, off, s[8:11], 0
-; SI-NEXT:        s_endpgm
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SI-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xb
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
+; SI-NEXT:    s_mov_b32 s11, 0xf000
+; SI-NEXT:    s_mov_b32 s2, 0
+; SI-NEXT:    s_mov_b32 s3, s11
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-NEXT:    v_mov_b32_e32 v1, 0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    buffer_load_dword v4, v[0:1], s[0:3], 0 addr64
+; SI-NEXT:    s_mov_b32 s10, -1
+; SI-NEXT:    s_movk_i32 s0, 0xff
+; SI-NEXT:    s_mov_b32 s6, s10
+; SI-NEXT:    s_mov_b32 s7, s11
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_lshrrev_b32_e32 v5, 16, v4
+; SI-NEXT:    v_lshrrev_b32_e32 v6, 24, v4
+; SI-NEXT:    v_and_b32_e32 v7, 0xff00, v4
+; SI-NEXT:    v_cvt_f32_ubyte3_e32 v3, v4
+; SI-NEXT:    v_cvt_f32_ubyte2_e32 v2, v4
+; SI-NEXT:    v_cvt_f32_ubyte1_e32 v1, v4
+; SI-NEXT:    v_cvt_f32_ubyte0_e32 v0, v4
+; SI-NEXT:    v_add_i32_e32 v4, vcc, 9, v4
+; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_and_b32_e32 v0, s0, v4
+; SI-NEXT:    v_add_i32_e32 v2, vcc, 9, v5
+; SI-NEXT:    v_or_b32_e32 v0, v7, v0
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v6
+; SI-NEXT:    v_and_b32_e32 v2, s0, v2
+; SI-NEXT:    v_add_i32_e32 v0, vcc, 0x900, v0
+; SI-NEXT:    v_or_b32_e32 v1, v1, v2
+; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT:    v_or_b32_e32 v0, v1, v0
+; SI-NEXT:    v_add_i32_e32 v0, vcc, 0x9000000, v0
+; SI-NEXT:    buffer_store_dword v0, off, s[8:11], 0
+; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: load_v4i8_to_v4f32_2_uses:
 ; VI:       ; %bb.0:
-; VI-NEXT:        s_load_dwordx2 s[4:5], s[0:1], 0x24
-; VI-NEXT:        s_load_dwordx2 s[8:9], s[0:1], 0x2c
-; VI-NEXT:        s_load_dwordx2 s[0:1], s[0:1], 0x34
-; VI-NEXT:        v_lshlrev_b32_e32 v0, 2, v0
-; VI-NEXT:        s_mov_b32 s11, 0xf000
-; VI-NEXT:        s_mov_b32 s10, -1
-; VI-NEXT:        v_mov_b32_e32 v5, 9
-; VI-NEXT:        s_waitcnt lgkmcnt(0)
-; VI-NEXT:        v_add_u32_e32 v0, vcc, s0, v0
-; VI-NEXT:        v_mov_b32_e32 v1, s1
-; VI-NEXT:        v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT:        flat_load_dword v4, v[0:1]
-; VI-NEXT:        s_mov_b32 s6, s10
-; VI-NEXT:        s_mov_b32 s7, s11
-; VI-NEXT:        s_movk_i32 s0, 0x900
-; VI-NEXT:        s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT:        v_lshrrev_b32_e32 v6, 24, v4
-; VI-NEXT:        v_cvt_f32_ubyte3_e32 v3, v4
-; VI-NEXT:        v_cvt_f32_ubyte2_e32 v2, v4
-; VI-NEXT:        v_cvt_f32_ubyte1_e32 v1, v4
-; VI-NEXT:        v_cvt_f32_ubyte0_e32 v0, v4
-; VI-NEXT:        buffer_store_dwordx4 v[0:3], off, s[4:7], 0
-; VI-NEXT:        v_and_b32_e32 v7, 0xffffff00, v4
-; VI-NEXT:        v_add_u16_e32 v8, 9, v4
-; VI-NEXT:        v_add_u16_sdwa v4, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; VI-NEXT:        v_lshlrev_b16_e32 v1, 8, v6
-; VI-NEXT:        v_or_b32_sdwa v0, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:        v_or_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT:        v_mov_b32_e32 v2, s0
-; VI-NEXT:        v_add_u16_e32 v0, s0, v0
-; VI-NEXT:        v_add_u16_sdwa v1, v1, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT:        v_or_b32_e32 v0, v0, v1
-; VI-NEXT:        buffer_store_dword v0, off, s[8:11], 0
-; VI-NEXT:        s_endpgm
+; VI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
+; VI-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0x2c
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT:    s_mov_b32 s11, 0xf000
+; VI-NEXT:    s_mov_b32 s10, -1
+; VI-NEXT:    v_mov_b32_e32 v5, 9
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dword v4, v[0:1]
+; VI-NEXT:    s_mov_b32 s6, s10
+; VI-NEXT:    s_mov_b32 s7, s11
+; VI-NEXT:    s_movk_i32 s0, 0x900
+; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_lshrrev_b32_e32 v6, 24, v4
+; VI-NEXT:    v_cvt_f32_ubyte3_e32 v3, v4
+; VI-NEXT:    v_cvt_f32_ubyte2_e32 v2, v4
+; VI-NEXT:    v_cvt_f32_ubyte1_e32 v1, v4
+; VI-NEXT:    v_cvt_f32_ubyte0_e32 v0, v4
+; VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; VI-NEXT:    v_and_b32_e32 v7, 0xffffff00, v4
+; VI-NEXT:    v_add_u16_e32 v8, 9, v4
+; VI-NEXT:    v_add_u16_sdwa v4, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b16_e32 v1, 8, v6
+; VI-NEXT:    v_or_b32_sdwa v0, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    v_add_u16_e32 v0, s0, v0
+; VI-NEXT:    v_add_u16_sdwa v1, v1, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT:    v_or_b32_e32 v0, v0, v1
+; VI-NEXT:    buffer_store_dword v0, off, s[8:11], 0
+; VI-NEXT:    s_endpgm
   %tid.x = call i32 @llvm.amdgcn.workitem.id.x()
   %in.ptr = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %in, i32 %tid.x
   %load = load <4 x i8>, <4 x i8> addrspace(1)* %in.ptr, align 4
@@ -725,42 +725,42 @@ define amdgpu_kernel void @load_v4i8_to_v4f32_2_uses(<4 x float> addrspace(1)* n
 define amdgpu_kernel void @load_v7i8_to_v7f32(<7 x float> addrspace(1)* noalias %out, <7 x i8> addrspace(1)* noalias %in) nounwind {
 ; SI-LABEL: load_v7i8_to_v7f32:
 ; SI:       ; %bb.0:
-; SI-NEXT:        s_load_dwordx2 s[4:5], s[0:1], 0x9
-; SI-NEXT:        s_load_dwordx2 s[0:1], s[0:1], 0xb
-; SI-NEXT:        s_mov_b32 s7, 0xf000
-; SI-NEXT:        s_mov_b32 s2, 0
-; SI-NEXT:        s_mov_b32 s3, s7
-; SI-NEXT:        v_lshlrev_b32_e32 v0, 3, v0
-; SI-NEXT:        v_mov_b32_e32 v1, 0
-; SI-NEXT:        s_waitcnt lgkmcnt(0)
-; SI-NEXT:        buffer_load_ubyte v2, v[0:1], s[0:3], 0 addr64
-; SI-NEXT:        buffer_load_ubyte v3, v[0:1], s[0:3], 0 addr64 offset:1
-; SI-NEXT:        buffer_load_ubyte v6, v[0:1], s[0:3], 0 addr64 offset:2
-; SI-NEXT:        buffer_load_ubyte v4, v[0:1], s[0:3], 0 addr64 offset:3
-; SI-NEXT:        buffer_load_ubyte v7, v[0:1], s[0:3], 0 addr64 offset:4
-; SI-NEXT:        buffer_load_ubyte v5, v[0:1], s[0:3], 0 addr64 offset:5
-; SI-NEXT:        buffer_load_ubyte v8, v[0:1], s[0:3], 0 addr64 offset:6
-; SI-NEXT:        s_mov_b32 s6, -1
-; SI-NEXT:        s_waitcnt vmcnt(6)
-; SI-NEXT:        v_cvt_f32_ubyte0_e32 v0, v2
-; SI-NEXT:        s_waitcnt vmcnt(5)
-; SI-NEXT:        v_cvt_f32_ubyte2_e32 v1, v3
-; SI-NEXT:        s_waitcnt vmcnt(3)
-; SI-NEXT:        v_lshlrev_b32_e32 v9, 8, v4
-; SI-NEXT:        v_or_b32_e32 v3, v9, v6
-; SI-NEXT:        s_waitcnt vmcnt(1)
-; SI-NEXT:        v_cvt_f32_ubyte2_e32 v5, v5
-; SI-NEXT:        s_waitcnt vmcnt(0)
-; SI-NEXT:        v_cvt_f32_ubyte0_e32 v2, v8
-; SI-NEXT:        buffer_store_dword v2, off, s[4:7], 0 offset:24
-; SI-NEXT:        s_waitcnt expcnt(0)
-; SI-NEXT:        v_lshlrev_b32_e32 v2, 16, v3
-; SI-NEXT:        v_cvt_f32_ubyte0_e32 v4, v7
-; SI-NEXT:        v_cvt_f32_ubyte3_e32 v3, v2
-; SI-NEXT:        v_cvt_f32_ubyte2_e32 v2, v2
-; SI-NEXT:        buffer_store_dwordx2 v[4:5], off, s[4:7], 0 offset:16
-; SI-NEXT:        buffer_store_dwordx4 v[0:3], off, s[4:7], 0
-; SI-NEXT:        s_endpgm
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s2, 0
+; SI-NEXT:    s_mov_b32 s3, s7
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
+; SI-NEXT:    v_mov_b32_e32 v1, 0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    buffer_load_ubyte v2, v[0:1], s[0:3], 0 addr64
+; SI-NEXT:    buffer_load_ubyte v3, v[0:1], s[0:3], 0 addr64 offset:1
+; SI-NEXT:    buffer_load_ubyte v6, v[0:1], s[0:3], 0 addr64 offset:2
+; SI-NEXT:    buffer_load_ubyte v4, v[0:1], s[0:3], 0 addr64 offset:3
+; SI-NEXT:    buffer_load_ubyte v7, v[0:1], s[0:3], 0 addr64 offset:4
+; SI-NEXT:    buffer_load_ubyte v5, v[0:1], s[0:3], 0 addr64 offset:5
+; SI-NEXT:    buffer_load_ubyte v8, v[0:1], s[0:3], 0 addr64 offset:6
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_waitcnt vmcnt(6)
+; SI-NEXT:    v_cvt_f32_ubyte0_e32 v0, v2
+; SI-NEXT:    s_waitcnt vmcnt(5)
+; SI-NEXT:    v_cvt_f32_ubyte2_e32 v1, v3
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_lshlrev_b32_e32 v9, 8, v4
+; SI-NEXT:    v_or_b32_e32 v3, v9, v6
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_cvt_f32_ubyte2_e32 v5, v5
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cvt_f32_ubyte0_e32 v2, v8
+; SI-NEXT:    buffer_store_dword v2, off, s[4:7], 0 offset:24
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v3
+; SI-NEXT:    v_cvt_f32_ubyte0_e32 v4, v7
+; SI-NEXT:    v_cvt_f32_ubyte3_e32 v3, v2
+; SI-NEXT:    v_cvt_f32_ubyte2_e32 v2, v2
+; SI-NEXT:    buffer_store_dwordx2 v[4:5], off, s[4:7], 0 offset:16
+; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: load_v7i8_to_v7f32:
 ; VI:       ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/fshr.ll b/llvm/test/CodeGen/AMDGPU/fshr.ll
index 444421443b4f2..bacbfcb8f500d 100644
--- a/llvm/test/CodeGen/AMDGPU/fshr.ll
+++ b/llvm/test/CodeGen/AMDGPU/fshr.ll
@@ -1210,167 +1210,167 @@ define i24 @v_fshr_i24(i24 %src0, i24 %src1, i24 %src2) {
 define <2 x i24> @v_fshr_v2i24(<2 x i24> %src0, <2 x i24> %src1, <2 x i24> %src2) {
 ; SI-LABEL: v_fshr_v2i24:
 ; SI:       ; %bb.0:
-; SI-NEXT:	s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT:	buffer_load_dword v1, off, s[0:3], s32 offset:8
-; SI-NEXT:	buffer_load_dword v2, off, s[0:3], s32 offset:16
-; SI-NEXT:	buffer_load_dword v3, off, s[0:3], s32 offset:20
-; SI-NEXT:	buffer_load_dword v4, off, s[0:3], s32 offset:4
-; SI-NEXT:	buffer_load_dword v5, off, s[0:3], s32 offset:12
-; SI-NEXT:	buffer_load_dword v6, off, s[0:3], s32
-; SI-NEXT:	s_mov_b32 s4, 0xffffff
-; SI-NEXT:	s_mov_b32 s5, 0xaaaaaaab
-; SI-NEXT:	v_add_i32_e32 v7, vcc, 3, v0
-; SI-NEXT:	v_add_i32_e32 v8, vcc, 4, v0
-; SI-NEXT:	v_add_i32_e32 v9, vcc, 5, v0
-; SI-NEXT:	v_add_i32_e32 v10, vcc, 2, v0
-; SI-NEXT:	s_waitcnt vmcnt(5)
-; SI-NEXT:	v_and_b32_e32 v14, s4, v1
-; SI-NEXT:	s_waitcnt vmcnt(4)
-; SI-NEXT:	v_and_b32_e32 v2, s4, v2
-; SI-NEXT:	v_mul_hi_u32 v12, v2, s5
-; SI-NEXT:	s_waitcnt vmcnt(3)
-; SI-NEXT:	v_and_b32_e32 v3, s4, v3
-; SI-NEXT:	v_mul_hi_u32 v13, v3, s5
-; SI-NEXT:	s_waitcnt vmcnt(1)
-; SI-NEXT:	v_and_b32_e32 v11, s4, v5
-; SI-NEXT:	v_lshrrev_b32_e32 v12, 4, v12
-; SI-NEXT:	v_mul_lo_u32 v12, v12, 24
-; SI-NEXT:	v_lshrrev_b32_e32 v13, 4, v13
-; SI-NEXT:	v_mul_lo_u32 v13, v13, 24
-; SI-NEXT:	v_sub_i32_e32 v2, vcc, v2, v12
-; SI-NEXT:	v_lshr_b32_e32 v12, v14, v2
-; SI-NEXT:	v_sub_i32_e32 v3, vcc, v3, v13
-; SI-NEXT:	v_sub_i32_e32 v13, vcc, 24, v2
-; SI-NEXT:	v_sub_i32_e32 v14, vcc, 24, v3
-; SI-NEXT:	v_and_b32_e32 v13, s4, v13
-; SI-NEXT:	s_waitcnt vmcnt(0)
-; SI-NEXT:	v_lshl_b32_e32 v6, v6, v13
-; SI-NEXT:	v_and_b32_e32 v14, 0xffffff, v14
-; SI-NEXT:	v_lshr_b32_e32 v11, v11, v3
-; SI-NEXT:	v_lshl_b32_e32 v4, v4, v14
-; SI-NEXT:	v_or_b32_e32 v6, v6, v12
-; SI-NEXT:	v_cmp_eq_u32_e32 vcc, 0, v2
-; SI-NEXT:	v_cndmask_b32_e32 v1, v6, v1, vcc
-; SI-NEXT:	v_or_b32_e32 v4, v4, v11
-; SI-NEXT:	v_cmp_eq_u32_e32 vcc, 0, v3
-; SI-NEXT:	v_cndmask_b32_e32 v2, v4, v5, vcc
-; SI-NEXT:	buffer_store_byte v2, v7, s[0:3], 0 offen
-; SI-NEXT:	buffer_store_short v1, v0, s[0:3], 0 offen
-; SI-NEXT:	v_lshrrev_b32_e32 v0, 8, v2
-; SI-NEXT:	s_waitcnt expcnt(1)
-; SI-NEXT:	v_lshrrev_b32_e32 v2, 16, v2
-; SI-NEXT:	s_waitcnt expcnt(0)
-; SI-NEXT:	v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT:	buffer_store_byte v0, v8, s[0:3], 0 offen
-; SI-NEXT:	buffer_store_byte v2, v9, s[0:3], 0 offen
-; SI-NEXT:	buffer_store_byte v1, v10, s[0:3], 0 offen
-; SI-NEXT:	s_waitcnt vmcnt(0) expcnt(0)
-; SI-NEXT:	s_setpc_b64 s[30:31]
+; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:8
+; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:16
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:20
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:4
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:12
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32
+; SI-NEXT:    s_mov_b32 s4, 0xffffff
+; SI-NEXT:    s_mov_b32 s5, 0xaaaaaaab
+; SI-NEXT:    v_add_i32_e32 v7, vcc, 3, v0
+; SI-NEXT:    v_add_i32_e32 v8, vcc, 4, v0
+; SI-NEXT:    v_add_i32_e32 v9, vcc, 5, v0
+; SI-NEXT:    v_add_i32_e32 v10, vcc, 2, v0
+; SI-NEXT:    s_waitcnt vmcnt(5)
+; SI-NEXT:    v_and_b32_e32 v14, s4, v1
+; SI-NEXT:    s_waitcnt vmcnt(4)
+; SI-NEXT:    v_and_b32_e32 v2, s4, v2
+; SI-NEXT:    v_mul_hi_u32 v12, v2, s5
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_and_b32_e32 v3, s4, v3
+; SI-NEXT:    v_mul_hi_u32 v13, v3, s5
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_and_b32_e32 v11, s4, v5
+; SI-NEXT:    v_lshrrev_b32_e32 v12, 4, v12
+; SI-NEXT:    v_mul_lo_u32 v12, v12, 24
+; SI-NEXT:    v_lshrrev_b32_e32 v13, 4, v13
+; SI-NEXT:    v_mul_lo_u32 v13, v13, 24
+; SI-NEXT:    v_sub_i32_e32 v2, vcc, v2, v12
+; SI-NEXT:    v_lshr_b32_e32 v12, v14, v2
+; SI-NEXT:    v_sub_i32_e32 v3, vcc, v3, v13
+; SI-NEXT:    v_sub_i32_e32 v13, vcc, 24, v2
+; SI-NEXT:    v_sub_i32_e32 v14, vcc, 24, v3
+; SI-NEXT:    v_and_b32_e32 v13, s4, v13
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_lshl_b32_e32 v6, v6, v13
+; SI-NEXT:    v_and_b32_e32 v14, 0xffffff, v14
+; SI-NEXT:    v_lshr_b32_e32 v11, v11, v3
+; SI-NEXT:    v_lshl_b32_e32 v4, v4, v14
+; SI-NEXT:    v_or_b32_e32 v6, v6, v12
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
+; SI-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
+; SI-NEXT:    v_or_b32_e32 v4, v4, v11
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; SI-NEXT:    v_cndmask_b32_e32 v2, v4, v5, vcc
+; SI-NEXT:    buffer_store_byte v2, v7, s[0:3], 0 offen
+; SI-NEXT:    buffer_store_short v1, v0, s[0:3], 0 offen
+; SI-NEXT:    v_lshrrev_b32_e32 v0, 8, v2
+; SI-NEXT:    s_waitcnt expcnt(1)
+; SI-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; SI-NEXT:    buffer_store_byte v0, v8, s[0:3], 0 offen
+; SI-NEXT:    buffer_store_byte v2, v9, s[0:3], 0 offen
+; SI-NEXT:    buffer_store_byte v1, v10, s[0:3], 0 offen
+; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: v_fshr_v2i24:
 ; VI:       ; %bb.0:
-; VI-NEXT:	s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:	buffer_load_dword v1, off, s[0:3], s32 offset:8
-; VI-NEXT:	buffer_load_dword v2, off, s[0:3], s32 offset:16
-; VI-NEXT:	buffer_load_dword v3, off, s[0:3], s32 offset:20
-; VI-NEXT:	buffer_load_dword v4, off, s[0:3], s32 offset:4
-; VI-NEXT:	buffer_load_dword v5, off, s[0:3], s32 offset:12
-; VI-NEXT:	buffer_load_dword v6, off, s[0:3], s32
-; VI-NEXT:	s_mov_b32 s4, 0xffffff
-; VI-NEXT:	s_mov_b32 s5, 0xaaaaaaab
-; VI-NEXT:	v_add_u32_e32 v7, vcc, 3, v0
-; VI-NEXT:	v_add_u32_e32 v8, vcc, 4, v0
-; VI-NEXT:	v_add_u32_e32 v9, vcc, 5, v0
-; VI-NEXT:	v_add_u32_e32 v10, vcc, 2, v0
-; VI-NEXT:	s_waitcnt vmcnt(5)
-; VI-NEXT:	v_and_b32_e32 v14, s4, v1
-; VI-NEXT:	s_waitcnt vmcnt(4)
-; VI-NEXT:	v_and_b32_e32 v2, s4, v2
-; VI-NEXT:	v_mul_hi_u32 v12, v2, s5
-; VI-NEXT:	s_waitcnt vmcnt(3)
-; VI-NEXT:	v_and_b32_e32 v3, s4, v3
-; VI-NEXT:	v_mul_hi_u32 v13, v3, s5
-; VI-NEXT:	s_waitcnt vmcnt(1)
-; VI-NEXT:	v_and_b32_e32 v11, s4, v5
-; VI-NEXT:	v_lshrrev_b32_e32 v12, 4, v12
-; VI-NEXT:	v_mul_lo_u32 v12, v12, 24
-; VI-NEXT:	v_lshrrev_b32_e32 v13, 4, v13
-; VI-NEXT:	v_mul_lo_u32 v13, v13, 24
-; VI-NEXT:	v_sub_u32_e32 v2, vcc, v2, v12
-; VI-NEXT:	v_lshrrev_b32_e32 v12, v2, v14
-; VI-NEXT:	v_sub_u32_e32 v3, vcc, v3, v13
-; VI-NEXT:	v_sub_u32_e32 v13, vcc, 24, v2
-; VI-NEXT:	v_sub_u32_e32 v14, vcc, 24, v3
-; VI-NEXT:	v_and_b32_e32 v13, s4, v13
-; VI-NEXT:	s_waitcnt vmcnt(0)
-; VI-NEXT:	v_lshlrev_b32_e32 v6, v13, v6
-; VI-NEXT:	v_and_b32_e32 v14, 0xffffff, v14
-; VI-NEXT:	v_lshrrev_b32_e32 v11, v3, v11
-; VI-NEXT:	v_lshlrev_b32_e32 v4, v14, v4
-; VI-NEXT:	v_or_b32_e32 v6, v6, v12
-; VI-NEXT:	v_cmp_eq_u32_e32 vcc, 0, v2
-; VI-NEXT:	v_cndmask_b32_e32 v1, v6, v1, vcc
-; VI-NEXT:	v_or_b32_e32 v4, v4, v11
-; VI-NEXT:	v_cmp_eq_u32_e32 vcc, 0, v3
-; VI-NEXT:	v_cndmask_b32_e32 v2, v4, v5, vcc
-; VI-NEXT:	buffer_store_byte v2, v7, s[0:3], 0 offen
-; VI-NEXT:	buffer_store_short v1, v0, s[0:3], 0 offen
-; VI-NEXT:	v_lshrrev_b32_e32 v0, 8, v2
-; VI-NEXT:	v_lshrrev_b32_e32 v2, 16, v2
-; VI-NEXT:	v_lshrrev_b32_e32 v1, 16, v1
-; VI-NEXT:	buffer_store_byte v0, v8, s[0:3], 0 offen
-; VI-NEXT:	buffer_store_byte v2, v9, s[0:3], 0 offen
-; VI-NEXT:	buffer_store_byte v1, v10, s[0:3], 0 offen
-; VI-NEXT:	s_waitcnt vmcnt(0)
-; VI-NEXT:	s_setpc_b64 s[30:31]
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:8
+; VI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:16
+; VI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:20
+; VI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:4
+; VI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:12
+; VI-NEXT:    buffer_load_dword v6, off, s[0:3], s32
+; VI-NEXT:    s_mov_b32 s4, 0xffffff
+; VI-NEXT:    s_mov_b32 s5, 0xaaaaaaab
+; VI-NEXT:    v_add_u32_e32 v7, vcc, 3, v0
+; VI-NEXT:    v_add_u32_e32 v8, vcc, 4, v0
+; VI-NEXT:    v_add_u32_e32 v9, vcc, 5, v0
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 2, v0
+; VI-NEXT:    s_waitcnt vmcnt(5)
+; VI-NEXT:    v_and_b32_e32 v14, s4, v1
+; VI-NEXT:    s_waitcnt vmcnt(4)
+; VI-NEXT:    v_and_b32_e32 v2, s4, v2
+; VI-NEXT:    v_mul_hi_u32 v12, v2, s5
+; VI-NEXT:    s_waitcnt vmcnt(3)
+; VI-NEXT:    v_and_b32_e32 v3, s4, v3
+; VI-NEXT:    v_mul_hi_u32 v13, v3, s5
+; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_and_b32_e32 v11, s4, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v12, 4, v12
+; VI-NEXT:    v_mul_lo_u32 v12, v12, 24
+; VI-NEXT:    v_lshrrev_b32_e32 v13, 4, v13
+; VI-NEXT:    v_mul_lo_u32 v13, v13, 24
+; VI-NEXT:    v_sub_u32_e32 v2, vcc, v2, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v12, v2, v14
+; VI-NEXT:    v_sub_u32_e32 v3, vcc, v3, v13
+; VI-NEXT:    v_sub_u32_e32 v13, vcc, 24, v2
+; VI-NEXT:    v_sub_u32_e32 v14, vcc, 24, v3
+; VI-NEXT:    v_and_b32_e32 v13, s4, v13
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_lshlrev_b32_e32 v6, v13, v6
+; VI-NEXT:    v_and_b32_e32 v14, 0xffffff, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v11, v3, v11
+; VI-NEXT:    v_lshlrev_b32_e32 v4, v14, v4
+; VI-NEXT:    v_or_b32_e32 v6, v6, v12
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
+; VI-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
+; VI-NEXT:    v_or_b32_e32 v4, v4, v11
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; VI-NEXT:    v_cndmask_b32_e32 v2, v4, v5, vcc
+; VI-NEXT:    buffer_store_byte v2, v7, s[0:3], 0 offen
+; VI-NEXT:    buffer_store_short v1, v0, s[0:3], 0 offen
+; VI-NEXT:    v_lshrrev_b32_e32 v0, 8, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; VI-NEXT:    buffer_store_byte v0, v8, s[0:3], 0 offen
+; VI-NEXT:    buffer_store_byte v2, v9, s[0:3], 0 offen
+; VI-NEXT:    buffer_store_byte v1, v10, s[0:3], 0 offen
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_fshr_v2i24:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:        s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:        buffer_load_dword v1, off, s[0:3], s32 offset:16
-; GFX9-NEXT:        buffer_load_dword v2, off, s[0:3], s32 offset:20
-; GFX9-NEXT:        buffer_load_dword v3, off, s[0:3], s32 offset:4
-; GFX9-NEXT:        buffer_load_dword v4, off, s[0:3], s32 offset:12
-; GFX9-NEXT:        buffer_load_dword v5, off, s[0:3], s32
-; GFX9-NEXT:        buffer_load_dword v8, off, s[0:3], s32 offset:8
-; GFX9-NEXT:        s_mov_b32 s4, 0xffffff
-; GFX9-NEXT:        s_mov_b32 s5, 0xaaaaaaab
-; GFX9-NEXT:        s_waitcnt vmcnt(5)
-; GFX9-NEXT:        v_and_b32_e32 v1, s4, v1
-; GFX9-NEXT:        v_mul_hi_u32 v6, v1, s5
-; GFX9-NEXT:        s_waitcnt vmcnt(4)
-; GFX9-NEXT:        v_and_b32_e32 v2, s4, v2
-; GFX9-NEXT:        v_mul_hi_u32 v7, v2, s5
-; GFX9-NEXT:        s_waitcnt vmcnt(2)
-; GFX9-NEXT:        v_and_b32_e32 v9, s4, v4
-; GFX9-NEXT:        v_lshrrev_b32_e32 v6, 4, v6
-; GFX9-NEXT:        v_mul_lo_u32 v6, v6, 24
-; GFX9-NEXT:        v_lshrrev_b32_e32 v7, 4, v7
-; GFX9-NEXT:        v_mul_lo_u32 v7, v7, 24
-; GFX9-NEXT:        s_waitcnt vmcnt(0)
-; GFX9-NEXT:        v_and_b32_e32 v10, s4, v8
-; GFX9-NEXT:        v_sub_u32_e32 v1, v1, v6
-; GFX9-NEXT:        v_lshrrev_b32_e32 v6, v1, v10
-; GFX9-NEXT:        v_sub_u32_e32 v2, v2, v7
-; GFX9-NEXT:        v_sub_u32_e32 v7, 24, v1
-; GFX9-NEXT:        v_sub_u32_e32 v10, 24, v2
-; GFX9-NEXT:        v_and_b32_e32 v7, s4, v7
-; GFX9-NEXT:        v_lshrrev_b32_e32 v9, v2, v9
-; GFX9-NEXT:        v_and_b32_e32 v10, 0xffffff, v10
-; GFX9-NEXT:        v_lshl_or_b32 v5, v5, v7, v6
-; GFX9-NEXT:        v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX9-NEXT:        v_cndmask_b32_e32 v1, v5, v8, vcc
-; GFX9-NEXT:        v_lshl_or_b32 v3, v3, v10, v9
-; GFX9-NEXT:        v_cmp_eq_u32_e32 vcc, 0, v2
-; GFX9-NEXT:        v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX9-NEXT:        v_lshrrev_b32_e32 v3, 8, v2
-; GFX9-NEXT:        buffer_store_byte_d16_hi v2, v0, s[0:3], 0 offen offset:5
-; GFX9-NEXT:        buffer_store_byte v3, v0, s[0:3], 0 offen offset:4
-; GFX9-NEXT:        buffer_store_byte v2, v0, s[0:3], 0 offen offset:3
-; GFX9-NEXT:        buffer_store_byte_d16_hi v1, v0, s[0:3], 0 offen offset:2
-; GFX9-NEXT:        buffer_store_short v1, v0, s[0:3], 0 offen
-; GFX9-NEXT:        s_waitcnt vmcnt(0)
-; GFX9-NEXT:        s_setpc_b64 s[30:31]
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:16
+; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:20
+; GFX9-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:4
+; GFX9-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:12
+; GFX9-NEXT:    buffer_load_dword v5, off, s[0:3], s32
+; GFX9-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:8
+; GFX9-NEXT:    s_mov_b32 s4, 0xffffff
+; GFX9-NEXT:    s_mov_b32 s5, 0xaaaaaaab
+; GFX9-NEXT:    s_waitcnt vmcnt(5)
+; GFX9-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX9-NEXT:    v_mul_hi_u32 v6, v1, s5
+; GFX9-NEXT:    s_waitcnt vmcnt(4)
+; GFX9-NEXT:    v_and_b32_e32 v2, s4, v2
+; GFX9-NEXT:    v_mul_hi_u32 v7, v2, s5
+; GFX9-NEXT:    s_waitcnt vmcnt(2)
+; GFX9-NEXT:    v_and_b32_e32 v9, s4, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v6, 4, v6
+; GFX9-NEXT:    v_mul_lo_u32 v6, v6, 24
+; GFX9-NEXT:    v_lshrrev_b32_e32 v7, 4, v7
+; GFX9-NEXT:    v_mul_lo_u32 v7, v7, 24
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v10, s4, v8
+; GFX9-NEXT:    v_sub_u32_e32 v1, v1, v6
+; GFX9-NEXT:    v_lshrrev_b32_e32 v6, v1, v10
+; GFX9-NEXT:    v_sub_u32_e32 v2, v2, v7
+; GFX9-NEXT:    v_sub_u32_e32 v7, 24, v1
+; GFX9-NEXT:    v_sub_u32_e32 v10, 24, v2
+; GFX9-NEXT:    v_and_b32_e32 v7, s4, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v9, v2, v9
+; GFX9-NEXT:    v_and_b32_e32 v10, 0xffffff, v10
+; GFX9-NEXT:    v_lshl_or_b32 v5, v5, v7, v6
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v8, vcc
+; GFX9-NEXT:    v_lshl_or_b32 v3, v3, v10, v9
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT:    v_lshrrev_b32_e32 v3, 8, v2
+; GFX9-NEXT:    buffer_store_byte_d16_hi v2, v0, s[0:3], 0 offen offset:5
+; GFX9-NEXT:    buffer_store_byte v3, v0, s[0:3], 0 offen offset:4
+; GFX9-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:3
+; GFX9-NEXT:    buffer_store_byte_d16_hi v1, v0, s[0:3], 0 offen offset:2
+; GFX9-NEXT:    buffer_store_short v1, v0, s[0:3], 0 offen
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; R600-LABEL: v_fshr_v2i24:
 ; R600:       ; %bb.0:

From a258338d627170f204c40ebe93ea7fb18c7c1197 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@intel.com>
Date: Sun, 2 Aug 2020 00:20:11 -0700
Subject: [PATCH 0857/1035] [X86] Add test cases for missed opportunity to use
 a byte test instruction instead of an xor with 0 in parity patterns.

If the input to the ctpop fits in 8 bits, we can use the parity
flag from a TEST instruction, but we're currently XORing with 0.
---
 llvm/test/CodeGen/X86/parity.ll | 70 +++++++++++++++++++++++++++++++++
 1 file changed, 70 insertions(+)

diff --git a/llvm/test/CodeGen/X86/parity.ll b/llvm/test/CodeGen/X86/parity.ll
index 8637058e06807..68f2b8b1b5538 100644
--- a/llvm/test/CodeGen/X86/parity.ll
+++ b/llvm/test/CodeGen/X86/parity.ll
@@ -181,5 +181,75 @@ define i8 @parity_32_trunc(i32 %x) {
   ret i8 %3
 }
 
+define i32 @parity_8_zext(i8 %x) {
+; X86-NOPOPCNT-LABEL: parity_8_zext:
+; X86-NOPOPCNT:       # %bb.0:
+; X86-NOPOPCNT-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-NOPOPCNT-NEXT:    xorl %eax, %eax
+; X86-NOPOPCNT-NEXT:    xorb $0, %cl
+; X86-NOPOPCNT-NEXT:    setnp %al
+; X86-NOPOPCNT-NEXT:    retl
+;
+; X64-NOPOPCNT-LABEL: parity_8_zext:
+; X64-NOPOPCNT:       # %bb.0:
+; X64-NOPOPCNT-NEXT:    xorl %eax, %eax
+; X64-NOPOPCNT-NEXT:    xorb $0, %dil
+; X64-NOPOPCNT-NEXT:    setnp %al
+; X64-NOPOPCNT-NEXT:    retq
+;
+; X86-POPCNT-LABEL: parity_8_zext:
+; X86-POPCNT:       # %bb.0:
+; X86-POPCNT-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-POPCNT-NEXT:    popcntl %eax, %eax
+; X86-POPCNT-NEXT:    andl $1, %eax
+; X86-POPCNT-NEXT:    retl
+;
+; X64-POPCNT-LABEL: parity_8_zext:
+; X64-POPCNT:       # %bb.0:
+; X64-POPCNT-NEXT:    movzbl %dil, %eax
+; X64-POPCNT-NEXT:    popcntl %eax, %eax
+; X64-POPCNT-NEXT:    andl $1, %eax
+; X64-POPCNT-NEXT:    retq
+  %a = zext i8 %x to i32
+  %b = tail call i32 @llvm.ctpop.i32(i32 %a)
+  %c = and i32 %b, 1
+  ret i32 %c
+}
+
+define i32 @parity_8_mask(i32 %x) {
+; X86-NOPOPCNT-LABEL: parity_8_mask:
+; X86-NOPOPCNT:       # %bb.0:
+; X86-NOPOPCNT-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-NOPOPCNT-NEXT:    xorl %eax, %eax
+; X86-NOPOPCNT-NEXT:    xorb $0, %cl
+; X86-NOPOPCNT-NEXT:    setnp %al
+; X86-NOPOPCNT-NEXT:    retl
+;
+; X64-NOPOPCNT-LABEL: parity_8_mask:
+; X64-NOPOPCNT:       # %bb.0:
+; X64-NOPOPCNT-NEXT:    xorl %eax, %eax
+; X64-NOPOPCNT-NEXT:    xorb $0, %dil
+; X64-NOPOPCNT-NEXT:    setnp %al
+; X64-NOPOPCNT-NEXT:    retq
+;
+; X86-POPCNT-LABEL: parity_8_mask:
+; X86-POPCNT:       # %bb.0:
+; X86-POPCNT-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-POPCNT-NEXT:    popcntl %eax, %eax
+; X86-POPCNT-NEXT:    andl $1, %eax
+; X86-POPCNT-NEXT:    retl
+;
+; X64-POPCNT-LABEL: parity_8_mask:
+; X64-POPCNT:       # %bb.0:
+; X64-POPCNT-NEXT:    movzbl %dil, %eax
+; X64-POPCNT-NEXT:    popcntl %eax, %eax
+; X64-POPCNT-NEXT:    andl $1, %eax
+; X64-POPCNT-NEXT:    retq
+  %a = and i32 %x, 255
+  %b = tail call i32 @llvm.ctpop.i32(i32 %a)
+  %c = and i32 %b, 1
+  ret i32 %c
+}
+
 declare i32 @llvm.ctpop.i32(i32 %x)
 declare i64 @llvm.ctpop.i64(i64 %x)

From 64516ec7c1298a4cb16980db49c2f9466f0f3ab5 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@intel.com>
Date: Sun, 2 Aug 2020 09:58:55 -0700
Subject: [PATCH 0858/1035] [X86] Use parity flag from byte test/cmp
 instruction for __builtin_parity when input fits in 8 bits.

If the upper bits of the __builtin_parity idiom are known to be
0 we were previously emitting an xor with 0 to get the parity flag.
But we can use cmp/test instead which may expose opportunities for
load folding or combining an AND.
---
 llvm/lib/Target/X86/X86ISelLowering.cpp       |  11 ++
 llvm/test/CodeGen/X86/parity.ll               |  10 +-
 .../CodeGen/X86/vector-reduce-xor-bool.ll     | 162 ++++++++----------
 3 files changed, 87 insertions(+), 96 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 93852e13b7d08..ff59f28c8b6d2 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -42773,6 +42773,17 @@ static SDValue combineParity(SDNode *N, SelectionDAG &DAG,
   SDLoc DL(N);
   SDValue X = N0.getOperand(0);
 
+  // Special case. If the input fits in 8-bits we can use a single 8-bit TEST.
+  if (DAG.MaskedValueIsZero(X, APInt::getBitsSetFrom(VT.getSizeInBits(), 8))) {
+    X = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, X);
+    SDValue Flags = DAG.getNode(X86ISD::CMP, DL, MVT::i32, X,
+                                DAG.getConstant(0, DL, MVT::i8));
+    // Copy the inverse of the parity flag into a register with setcc.
+    SDValue Setnp = getSETCC(X86::COND_NP, Flags, DL, DAG);
+    // Extend or truncate to the original type.
+    return DAG.getZExtOrTrunc(Setnp, DL, N->getValueType(0));
+  }
+
   // If this is 64-bit, its always best to xor the two 32-bit pieces together
   // even if we have popcnt.
   if (VT == MVT::i64) {
diff --git a/llvm/test/CodeGen/X86/parity.ll b/llvm/test/CodeGen/X86/parity.ll
index 68f2b8b1b5538..869ee552c67da 100644
--- a/llvm/test/CodeGen/X86/parity.ll
+++ b/llvm/test/CodeGen/X86/parity.ll
@@ -184,16 +184,15 @@ define i8 @parity_32_trunc(i32 %x) {
 define i32 @parity_8_zext(i8 %x) {
 ; X86-NOPOPCNT-LABEL: parity_8_zext:
 ; X86-NOPOPCNT:       # %bb.0:
-; X86-NOPOPCNT-NEXT:    movb {{[0-9]+}}(%esp), %cl
 ; X86-NOPOPCNT-NEXT:    xorl %eax, %eax
-; X86-NOPOPCNT-NEXT:    xorb $0, %cl
+; X86-NOPOPCNT-NEXT:    cmpb $0, {{[0-9]+}}(%esp)
 ; X86-NOPOPCNT-NEXT:    setnp %al
 ; X86-NOPOPCNT-NEXT:    retl
 ;
 ; X64-NOPOPCNT-LABEL: parity_8_zext:
 ; X64-NOPOPCNT:       # %bb.0:
 ; X64-NOPOPCNT-NEXT:    xorl %eax, %eax
-; X64-NOPOPCNT-NEXT:    xorb $0, %dil
+; X64-NOPOPCNT-NEXT:    testb %dil, %dil
 ; X64-NOPOPCNT-NEXT:    setnp %al
 ; X64-NOPOPCNT-NEXT:    retq
 ;
@@ -219,16 +218,15 @@ define i32 @parity_8_zext(i8 %x) {
 define i32 @parity_8_mask(i32 %x) {
 ; X86-NOPOPCNT-LABEL: parity_8_mask:
 ; X86-NOPOPCNT:       # %bb.0:
-; X86-NOPOPCNT-NEXT:    movb {{[0-9]+}}(%esp), %cl
 ; X86-NOPOPCNT-NEXT:    xorl %eax, %eax
-; X86-NOPOPCNT-NEXT:    xorb $0, %cl
+; X86-NOPOPCNT-NEXT:    cmpb $0, {{[0-9]+}}(%esp)
 ; X86-NOPOPCNT-NEXT:    setnp %al
 ; X86-NOPOPCNT-NEXT:    retl
 ;
 ; X64-NOPOPCNT-LABEL: parity_8_mask:
 ; X64-NOPOPCNT:       # %bb.0:
 ; X64-NOPOPCNT-NEXT:    xorl %eax, %eax
-; X64-NOPOPCNT-NEXT:    xorb $0, %dil
+; X64-NOPOPCNT-NEXT:    testb %dil, %dil
 ; X64-NOPOPCNT-NEXT:    setnp %al
 ; X64-NOPOPCNT-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/vector-reduce-xor-bool.ll b/llvm/test/CodeGen/X86/vector-reduce-xor-bool.ll
index 627faa02d2b7d..8e50cfc4e4b29 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-xor-bool.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-xor-bool.ll
@@ -16,7 +16,7 @@ define i1 @trunc_v2i64_v2i1(<2 x i64>) {
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    psllq $63, %xmm0
 ; SSE-NEXT:    movmskpd %xmm0, %eax
-; SSE-NEXT:    xorb $0, %al
+; SSE-NEXT:    testb %al, %al
 ; SSE-NEXT:    setnp %al
 ; SSE-NEXT:    retq
 ;
@@ -24,7 +24,7 @@ define i1 @trunc_v2i64_v2i1(<2 x i64>) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpsllq $63, %xmm0, %xmm0
 ; AVX-NEXT:    vmovmskpd %xmm0, %eax
-; AVX-NEXT:    xorb $0, %al
+; AVX-NEXT:    testb %al, %al
 ; AVX-NEXT:    setnp %al
 ; AVX-NEXT:    retq
 ;
@@ -33,8 +33,7 @@ define i1 @trunc_v2i64_v2i1(<2 x i64>) {
 ; AVX512F-NEXT:    vpsllq $63, %xmm0, %xmm0
 ; AVX512F-NEXT:    vptestmq %zmm0, %zmm0, %k0
 ; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    andl $3, %eax
-; AVX512F-NEXT:    xorb $0, %al
+; AVX512F-NEXT:    testb $3, %al
 ; AVX512F-NEXT:    setnp %al
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
@@ -44,8 +43,7 @@ define i1 @trunc_v2i64_v2i1(<2 x i64>) {
 ; AVX512BW-NEXT:    vpsllq $63, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vptestmq %zmm0, %zmm0, %k0
 ; AVX512BW-NEXT:    kmovd %k0, %eax
-; AVX512BW-NEXT:    andl $3, %eax
-; AVX512BW-NEXT:    xorb $0, %al
+; AVX512BW-NEXT:    testb $3, %al
 ; AVX512BW-NEXT:    setnp %al
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -55,8 +53,7 @@ define i1 @trunc_v2i64_v2i1(<2 x i64>) {
 ; AVX512VL-NEXT:    vpsllq $63, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vptestmq %xmm0, %xmm0, %k0
 ; AVX512VL-NEXT:    kmovd %k0, %eax
-; AVX512VL-NEXT:    andl $3, %eax
-; AVX512VL-NEXT:    xorb $0, %al
+; AVX512VL-NEXT:    testb %al, %al
 ; AVX512VL-NEXT:    setnp %al
 ; AVX512VL-NEXT:    retq
   %a = trunc <2 x i64> %0 to <2 x i1>
@@ -69,7 +66,7 @@ define i1 @trunc_v4i32_v4i1(<4 x i32>) {
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pslld $31, %xmm0
 ; SSE-NEXT:    movmskps %xmm0, %eax
-; SSE-NEXT:    xorb $0, %al
+; SSE-NEXT:    testb %al, %al
 ; SSE-NEXT:    setnp %al
 ; SSE-NEXT:    retq
 ;
@@ -77,7 +74,7 @@ define i1 @trunc_v4i32_v4i1(<4 x i32>) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpslld $31, %xmm0, %xmm0
 ; AVX-NEXT:    vmovmskps %xmm0, %eax
-; AVX-NEXT:    xorb $0, %al
+; AVX-NEXT:    testb %al, %al
 ; AVX-NEXT:    setnp %al
 ; AVX-NEXT:    retq
 ;
@@ -86,8 +83,7 @@ define i1 @trunc_v4i32_v4i1(<4 x i32>) {
 ; AVX512F-NEXT:    vpslld $31, %xmm0, %xmm0
 ; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k0
 ; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    andl $15, %eax
-; AVX512F-NEXT:    xorb $0, %al
+; AVX512F-NEXT:    testb $15, %al
 ; AVX512F-NEXT:    setnp %al
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
@@ -97,8 +93,7 @@ define i1 @trunc_v4i32_v4i1(<4 x i32>) {
 ; AVX512BW-NEXT:    vpslld $31, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vptestmd %zmm0, %zmm0, %k0
 ; AVX512BW-NEXT:    kmovd %k0, %eax
-; AVX512BW-NEXT:    andl $15, %eax
-; AVX512BW-NEXT:    xorb $0, %al
+; AVX512BW-NEXT:    testb $15, %al
 ; AVX512BW-NEXT:    setnp %al
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -108,8 +103,7 @@ define i1 @trunc_v4i32_v4i1(<4 x i32>) {
 ; AVX512VL-NEXT:    vpslld $31, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vptestmd %xmm0, %xmm0, %k0
 ; AVX512VL-NEXT:    kmovd %k0, %eax
-; AVX512VL-NEXT:    andl $15, %eax
-; AVX512VL-NEXT:    xorb $0, %al
+; AVX512VL-NEXT:    testb %al, %al
 ; AVX512VL-NEXT:    setnp %al
 ; AVX512VL-NEXT:    retq
   %a = trunc <4 x i32> %0 to <4 x i1>
@@ -124,7 +118,7 @@ define i1 @trunc_v8i16_v8i1(<8 x i8>) {
 ; SSE2-NEXT:    psllw $15, %xmm0
 ; SSE2-NEXT:    packsswb %xmm0, %xmm0
 ; SSE2-NEXT:    pmovmskb %xmm0, %eax
-; SSE2-NEXT:    xorb $0, %al
+; SSE2-NEXT:    testb %al, %al
 ; SSE2-NEXT:    setnp %al
 ; SSE2-NEXT:    retq
 ;
@@ -134,7 +128,7 @@ define i1 @trunc_v8i16_v8i1(<8 x i8>) {
 ; SSE41-NEXT:    psllw $15, %xmm0
 ; SSE41-NEXT:    packsswb %xmm0, %xmm0
 ; SSE41-NEXT:    pmovmskb %xmm0, %eax
-; SSE41-NEXT:    xorb $0, %al
+; SSE41-NEXT:    testb %al, %al
 ; SSE41-NEXT:    setnp %al
 ; SSE41-NEXT:    retq
 ;
@@ -144,7 +138,7 @@ define i1 @trunc_v8i16_v8i1(<8 x i8>) {
 ; AVX-NEXT:    vpsllw $15, %xmm0, %xmm0
 ; AVX-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
 ; AVX-NEXT:    vpmovmskb %xmm0, %eax
-; AVX-NEXT:    xorb $0, %al
+; AVX-NEXT:    testb %al, %al
 ; AVX-NEXT:    setnp %al
 ; AVX-NEXT:    retq
 ;
@@ -154,7 +148,7 @@ define i1 @trunc_v8i16_v8i1(<8 x i8>) {
 ; AVX512F-NEXT:    vpslld $31, %zmm0, %zmm0
 ; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k0
 ; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    xorb $0, %al
+; AVX512F-NEXT:    testb %al, %al
 ; AVX512F-NEXT:    setnp %al
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
@@ -164,7 +158,7 @@ define i1 @trunc_v8i16_v8i1(<8 x i8>) {
 ; AVX512BW-NEXT:    vpsllw $7, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vpmovb2m %zmm0, %k0
 ; AVX512BW-NEXT:    kmovd %k0, %eax
-; AVX512BW-NEXT:    xorb $0, %al
+; AVX512BW-NEXT:    testb %al, %al
 ; AVX512BW-NEXT:    setnp %al
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -174,7 +168,7 @@ define i1 @trunc_v8i16_v8i1(<8 x i8>) {
 ; AVX512VL-NEXT:    vpsllw $7, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vpmovb2m %xmm0, %k0
 ; AVX512VL-NEXT:    kmovd %k0, %eax
-; AVX512VL-NEXT:    xorb $0, %al
+; AVX512VL-NEXT:    testb %al, %al
 ; AVX512VL-NEXT:    setnp %al
 ; AVX512VL-NEXT:    retq
   %a = trunc <8 x i8> %0 to <8 x i1>
@@ -223,7 +217,7 @@ define i1 @trunc_v4i64_v4i1(<4 x i64>) {
 ; SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
 ; SSE-NEXT:    pslld $31, %xmm0
 ; SSE-NEXT:    movmskps %xmm0, %eax
-; SSE-NEXT:    xorb $0, %al
+; SSE-NEXT:    testb %al, %al
 ; SSE-NEXT:    setnp %al
 ; SSE-NEXT:    retq
 ;
@@ -233,7 +227,7 @@ define i1 @trunc_v4i64_v4i1(<4 x i64>) {
 ; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
 ; AVX-NEXT:    vpslld $31, %xmm0, %xmm0
 ; AVX-NEXT:    vmovmskps %xmm0, %eax
-; AVX-NEXT:    xorb $0, %al
+; AVX-NEXT:    testb %al, %al
 ; AVX-NEXT:    setnp %al
 ; AVX-NEXT:    vzeroupper
 ; AVX-NEXT:    retq
@@ -243,8 +237,7 @@ define i1 @trunc_v4i64_v4i1(<4 x i64>) {
 ; AVX512F-NEXT:    vpsllq $63, %ymm0, %ymm0
 ; AVX512F-NEXT:    vptestmq %zmm0, %zmm0, %k0
 ; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    andl $15, %eax
-; AVX512F-NEXT:    xorb $0, %al
+; AVX512F-NEXT:    testb $15, %al
 ; AVX512F-NEXT:    setnp %al
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
@@ -254,8 +247,7 @@ define i1 @trunc_v4i64_v4i1(<4 x i64>) {
 ; AVX512BW-NEXT:    vpsllq $63, %ymm0, %ymm0
 ; AVX512BW-NEXT:    vptestmq %zmm0, %zmm0, %k0
 ; AVX512BW-NEXT:    kmovd %k0, %eax
-; AVX512BW-NEXT:    andl $15, %eax
-; AVX512BW-NEXT:    xorb $0, %al
+; AVX512BW-NEXT:    testb $15, %al
 ; AVX512BW-NEXT:    setnp %al
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -265,8 +257,7 @@ define i1 @trunc_v4i64_v4i1(<4 x i64>) {
 ; AVX512VL-NEXT:    vpsllq $63, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vptestmq %ymm0, %ymm0, %k0
 ; AVX512VL-NEXT:    kmovd %k0, %eax
-; AVX512VL-NEXT:    andl $15, %eax
-; AVX512VL-NEXT:    xorb $0, %al
+; AVX512VL-NEXT:    testb %al, %al
 ; AVX512VL-NEXT:    setnp %al
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
@@ -286,7 +277,7 @@ define i1 @trunc_v8i32_v8i1(<8 x i32>) {
 ; SSE2-NEXT:    psllw $15, %xmm0
 ; SSE2-NEXT:    packsswb %xmm0, %xmm0
 ; SSE2-NEXT:    pmovmskb %xmm0, %eax
-; SSE2-NEXT:    xorb $0, %al
+; SSE2-NEXT:    testb %al, %al
 ; SSE2-NEXT:    setnp %al
 ; SSE2-NEXT:    retq
 ;
@@ -299,7 +290,7 @@ define i1 @trunc_v8i32_v8i1(<8 x i32>) {
 ; SSE41-NEXT:    psllw $15, %xmm0
 ; SSE41-NEXT:    packsswb %xmm0, %xmm0
 ; SSE41-NEXT:    pmovmskb %xmm0, %eax
-; SSE41-NEXT:    xorb $0, %al
+; SSE41-NEXT:    testb %al, %al
 ; SSE41-NEXT:    setnp %al
 ; SSE41-NEXT:    retq
 ;
@@ -313,7 +304,7 @@ define i1 @trunc_v8i32_v8i1(<8 x i32>) {
 ; AVX1-NEXT:    vpsllw $15, %xmm0, %xmm0
 ; AVX1-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
 ; AVX1-NEXT:    vpmovmskb %xmm0, %eax
-; AVX1-NEXT:    xorb $0, %al
+; AVX1-NEXT:    testb %al, %al
 ; AVX1-NEXT:    setnp %al
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -325,7 +316,7 @@ define i1 @trunc_v8i32_v8i1(<8 x i32>) {
 ; AVX2-NEXT:    vpsllw $15, %xmm0, %xmm0
 ; AVX2-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
 ; AVX2-NEXT:    vpmovmskb %xmm0, %eax
-; AVX2-NEXT:    xorb $0, %al
+; AVX2-NEXT:    testb %al, %al
 ; AVX2-NEXT:    setnp %al
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -335,7 +326,7 @@ define i1 @trunc_v8i32_v8i1(<8 x i32>) {
 ; AVX512F-NEXT:    vpslld $31, %ymm0, %ymm0
 ; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k0
 ; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    xorb $0, %al
+; AVX512F-NEXT:    testb %al, %al
 ; AVX512F-NEXT:    setnp %al
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
@@ -345,7 +336,7 @@ define i1 @trunc_v8i32_v8i1(<8 x i32>) {
 ; AVX512BW-NEXT:    vpslld $31, %ymm0, %ymm0
 ; AVX512BW-NEXT:    vptestmd %zmm0, %zmm0, %k0
 ; AVX512BW-NEXT:    kmovd %k0, %eax
-; AVX512BW-NEXT:    xorb $0, %al
+; AVX512BW-NEXT:    testb %al, %al
 ; AVX512BW-NEXT:    setnp %al
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -355,7 +346,7 @@ define i1 @trunc_v8i32_v8i1(<8 x i32>) {
 ; AVX512VL-NEXT:    vpslld $31, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vptestmd %ymm0, %ymm0, %k0
 ; AVX512VL-NEXT:    kmovd %k0, %eax
-; AVX512VL-NEXT:    xorb $0, %al
+; AVX512VL-NEXT:    testb %al, %al
 ; AVX512VL-NEXT:    setnp %al
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
@@ -556,7 +547,7 @@ define i1 @trunc_v8i64_v8i1(<8 x i64>) {
 ; SSE2-NEXT:    psllw $15, %xmm2
 ; SSE2-NEXT:    packsswb %xmm2, %xmm2
 ; SSE2-NEXT:    pmovmskb %xmm2, %eax
-; SSE2-NEXT:    xorb $0, %al
+; SSE2-NEXT:    testb %al, %al
 ; SSE2-NEXT:    setnp %al
 ; SSE2-NEXT:    retq
 ;
@@ -573,7 +564,7 @@ define i1 @trunc_v8i64_v8i1(<8 x i64>) {
 ; SSE41-NEXT:    psllw $15, %xmm0
 ; SSE41-NEXT:    packsswb %xmm0, %xmm0
 ; SSE41-NEXT:    pmovmskb %xmm0, %eax
-; SSE41-NEXT:    xorb $0, %al
+; SSE41-NEXT:    testb %al, %al
 ; SSE41-NEXT:    setnp %al
 ; SSE41-NEXT:    retq
 ;
@@ -590,7 +581,7 @@ define i1 @trunc_v8i64_v8i1(<8 x i64>) {
 ; AVX1-NEXT:    vpsllw $15, %xmm0, %xmm0
 ; AVX1-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
 ; AVX1-NEXT:    vpmovmskb %xmm0, %eax
-; AVX1-NEXT:    xorb $0, %al
+; AVX1-NEXT:    testb %al, %al
 ; AVX1-NEXT:    setnp %al
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -605,7 +596,7 @@ define i1 @trunc_v8i64_v8i1(<8 x i64>) {
 ; AVX2-NEXT:    vpsllw $15, %xmm0, %xmm0
 ; AVX2-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
 ; AVX2-NEXT:    vpmovmskb %xmm0, %eax
-; AVX2-NEXT:    xorb $0, %al
+; AVX2-NEXT:    testb %al, %al
 ; AVX2-NEXT:    setnp %al
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -615,7 +606,7 @@ define i1 @trunc_v8i64_v8i1(<8 x i64>) {
 ; AVX512F-NEXT:    vpsllq $63, %zmm0, %zmm0
 ; AVX512F-NEXT:    vptestmq %zmm0, %zmm0, %k0
 ; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    xorb $0, %al
+; AVX512F-NEXT:    testb %al, %al
 ; AVX512F-NEXT:    setnp %al
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
@@ -625,7 +616,7 @@ define i1 @trunc_v8i64_v8i1(<8 x i64>) {
 ; AVX512BW-NEXT:    vpsllq $63, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vptestmq %zmm0, %zmm0, %k0
 ; AVX512BW-NEXT:    kmovd %k0, %eax
-; AVX512BW-NEXT:    xorb $0, %al
+; AVX512BW-NEXT:    testb %al, %al
 ; AVX512BW-NEXT:    setnp %al
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -635,7 +626,7 @@ define i1 @trunc_v8i64_v8i1(<8 x i64>) {
 ; AVX512VL-NEXT:    vpsllq $63, %zmm0, %zmm0
 ; AVX512VL-NEXT:    vptestmq %zmm0, %zmm0, %k0
 ; AVX512VL-NEXT:    kmovd %k0, %eax
-; AVX512VL-NEXT:    xorb $0, %al
+; AVX512VL-NEXT:    testb %al, %al
 ; AVX512VL-NEXT:    setnp %al
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
@@ -987,7 +978,7 @@ define i1 @icmp_v2i64_v2i1(<2 x i64>) {
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,0,3,2]
 ; SSE2-NEXT:    pand %xmm1, %xmm0
 ; SSE2-NEXT:    movmskpd %xmm0, %eax
-; SSE2-NEXT:    xorb $0, %al
+; SSE2-NEXT:    testb %al, %al
 ; SSE2-NEXT:    setnp %al
 ; SSE2-NEXT:    retq
 ;
@@ -996,7 +987,7 @@ define i1 @icmp_v2i64_v2i1(<2 x i64>) {
 ; SSE41-NEXT:    pxor %xmm1, %xmm1
 ; SSE41-NEXT:    pcmpeqq %xmm0, %xmm1
 ; SSE41-NEXT:    movmskpd %xmm1, %eax
-; SSE41-NEXT:    xorb $0, %al
+; SSE41-NEXT:    testb %al, %al
 ; SSE41-NEXT:    setnp %al
 ; SSE41-NEXT:    retq
 ;
@@ -1005,7 +996,7 @@ define i1 @icmp_v2i64_v2i1(<2 x i64>) {
 ; AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX-NEXT:    vpcmpeqq %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovmskpd %xmm0, %eax
-; AVX-NEXT:    xorb $0, %al
+; AVX-NEXT:    testb %al, %al
 ; AVX-NEXT:    setnp %al
 ; AVX-NEXT:    retq
 ;
@@ -1014,8 +1005,7 @@ define i1 @icmp_v2i64_v2i1(<2 x i64>) {
 ; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; AVX512F-NEXT:    vptestnmq %zmm0, %zmm0, %k0
 ; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    andl $3, %eax
-; AVX512F-NEXT:    xorb $0, %al
+; AVX512F-NEXT:    testb $3, %al
 ; AVX512F-NEXT:    setnp %al
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
@@ -1025,8 +1015,7 @@ define i1 @icmp_v2i64_v2i1(<2 x i64>) {
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; AVX512BW-NEXT:    vptestnmq %zmm0, %zmm0, %k0
 ; AVX512BW-NEXT:    kmovd %k0, %eax
-; AVX512BW-NEXT:    andl $3, %eax
-; AVX512BW-NEXT:    xorb $0, %al
+; AVX512BW-NEXT:    testb $3, %al
 ; AVX512BW-NEXT:    setnp %al
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -1035,8 +1024,7 @@ define i1 @icmp_v2i64_v2i1(<2 x i64>) {
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vptestnmq %xmm0, %xmm0, %k0
 ; AVX512VL-NEXT:    kmovd %k0, %eax
-; AVX512VL-NEXT:    andl $3, %eax
-; AVX512VL-NEXT:    xorb $0, %al
+; AVX512VL-NEXT:    testb %al, %al
 ; AVX512VL-NEXT:    setnp %al
 ; AVX512VL-NEXT:    retq
   %a = icmp eq <2 x i64> %0, zeroinitializer
@@ -1050,7 +1038,7 @@ define i1 @icmp_v4i32_v4i1(<4 x i32>) {
 ; SSE-NEXT:    pxor %xmm1, %xmm1
 ; SSE-NEXT:    pcmpeqd %xmm0, %xmm1
 ; SSE-NEXT:    movmskps %xmm1, %eax
-; SSE-NEXT:    xorb $0, %al
+; SSE-NEXT:    testb %al, %al
 ; SSE-NEXT:    setnp %al
 ; SSE-NEXT:    retq
 ;
@@ -1059,7 +1047,7 @@ define i1 @icmp_v4i32_v4i1(<4 x i32>) {
 ; AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovmskps %xmm0, %eax
-; AVX-NEXT:    xorb $0, %al
+; AVX-NEXT:    testb %al, %al
 ; AVX-NEXT:    setnp %al
 ; AVX-NEXT:    retq
 ;
@@ -1068,8 +1056,7 @@ define i1 @icmp_v4i32_v4i1(<4 x i32>) {
 ; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; AVX512F-NEXT:    vptestnmd %zmm0, %zmm0, %k0
 ; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    andl $15, %eax
-; AVX512F-NEXT:    xorb $0, %al
+; AVX512F-NEXT:    testb $15, %al
 ; AVX512F-NEXT:    setnp %al
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
@@ -1079,8 +1066,7 @@ define i1 @icmp_v4i32_v4i1(<4 x i32>) {
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; AVX512BW-NEXT:    vptestnmd %zmm0, %zmm0, %k0
 ; AVX512BW-NEXT:    kmovd %k0, %eax
-; AVX512BW-NEXT:    andl $15, %eax
-; AVX512BW-NEXT:    xorb $0, %al
+; AVX512BW-NEXT:    testb $15, %al
 ; AVX512BW-NEXT:    setnp %al
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -1089,8 +1075,7 @@ define i1 @icmp_v4i32_v4i1(<4 x i32>) {
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vptestnmd %xmm0, %xmm0, %k0
 ; AVX512VL-NEXT:    kmovd %k0, %eax
-; AVX512VL-NEXT:    andl $15, %eax
-; AVX512VL-NEXT:    xorb $0, %al
+; AVX512VL-NEXT:    testb %al, %al
 ; AVX512VL-NEXT:    setnp %al
 ; AVX512VL-NEXT:    retq
   %a = icmp eq <4 x i32> %0, zeroinitializer
@@ -1106,7 +1091,7 @@ define i1 @icmp_v8i16_v8i1(<8 x i8>) {
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
 ; SSE2-NEXT:    packsswb %xmm0, %xmm0
 ; SSE2-NEXT:    pmovmskb %xmm0, %eax
-; SSE2-NEXT:    xorb $0, %al
+; SSE2-NEXT:    testb %al, %al
 ; SSE2-NEXT:    setnp %al
 ; SSE2-NEXT:    retq
 ;
@@ -1117,7 +1102,7 @@ define i1 @icmp_v8i16_v8i1(<8 x i8>) {
 ; SSE41-NEXT:    pmovsxbw %xmm1, %xmm0
 ; SSE41-NEXT:    packsswb %xmm0, %xmm0
 ; SSE41-NEXT:    pmovmskb %xmm0, %eax
-; SSE41-NEXT:    xorb $0, %al
+; SSE41-NEXT:    testb %al, %al
 ; SSE41-NEXT:    setnp %al
 ; SSE41-NEXT:    retq
 ;
@@ -1128,7 +1113,7 @@ define i1 @icmp_v8i16_v8i1(<8 x i8>) {
 ; AVX-NEXT:    vpmovsxbw %xmm0, %xmm0
 ; AVX-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
 ; AVX-NEXT:    vpmovmskb %xmm0, %eax
-; AVX-NEXT:    xorb $0, %al
+; AVX-NEXT:    testb %al, %al
 ; AVX-NEXT:    setnp %al
 ; AVX-NEXT:    retq
 ;
@@ -1139,7 +1124,7 @@ define i1 @icmp_v8i16_v8i1(<8 x i8>) {
 ; AVX512F-NEXT:    vpmovsxbd %xmm0, %zmm0
 ; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k0
 ; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    xorb $0, %al
+; AVX512F-NEXT:    testb %al, %al
 ; AVX512F-NEXT:    setnp %al
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
@@ -1149,7 +1134,7 @@ define i1 @icmp_v8i16_v8i1(<8 x i8>) {
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; AVX512BW-NEXT:    vptestnmb %zmm0, %zmm0, %k0
 ; AVX512BW-NEXT:    kmovd %k0, %eax
-; AVX512BW-NEXT:    xorb $0, %al
+; AVX512BW-NEXT:    testb %al, %al
 ; AVX512BW-NEXT:    setnp %al
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -1158,7 +1143,7 @@ define i1 @icmp_v8i16_v8i1(<8 x i8>) {
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vptestnmb %xmm0, %xmm0, %k0
 ; AVX512VL-NEXT:    kmovd %k0, %eax
-; AVX512VL-NEXT:    xorb $0, %al
+; AVX512VL-NEXT:    testb %al, %al
 ; AVX512VL-NEXT:    setnp %al
 ; AVX512VL-NEXT:    retq
   %a = icmp eq <8 x i8> %0, zeroinitializer
@@ -1238,7 +1223,7 @@ define i1 @icmp_v4i64_v4i1(<4 x i64>) {
 ; SSE2-NEXT:    pand %xmm0, %xmm1
 ; SSE2-NEXT:    packssdw %xmm3, %xmm1
 ; SSE2-NEXT:    movmskps %xmm1, %eax
-; SSE2-NEXT:    xorb $0, %al
+; SSE2-NEXT:    testb %al, %al
 ; SSE2-NEXT:    setnp %al
 ; SSE2-NEXT:    retq
 ;
@@ -1249,7 +1234,7 @@ define i1 @icmp_v4i64_v4i1(<4 x i64>) {
 ; SSE41-NEXT:    pcmpeqq %xmm2, %xmm0
 ; SSE41-NEXT:    packssdw %xmm1, %xmm0
 ; SSE41-NEXT:    movmskps %xmm0, %eax
-; SSE41-NEXT:    xorb $0, %al
+; SSE41-NEXT:    testb %al, %al
 ; SSE41-NEXT:    setnp %al
 ; SSE41-NEXT:    retq
 ;
@@ -1261,7 +1246,7 @@ define i1 @icmp_v4i64_v4i1(<4 x i64>) {
 ; AVX1-NEXT:    vpcmpeqq %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vmovmskpd %ymm0, %eax
-; AVX1-NEXT:    xorb $0, %al
+; AVX1-NEXT:    testb %al, %al
 ; AVX1-NEXT:    setnp %al
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -1271,7 +1256,7 @@ define i1 @icmp_v4i64_v4i1(<4 x i64>) {
 ; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX2-NEXT:    vpcmpeqq %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vmovmskpd %ymm0, %eax
-; AVX2-NEXT:    xorb $0, %al
+; AVX2-NEXT:    testb %al, %al
 ; AVX2-NEXT:    setnp %al
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -1281,8 +1266,7 @@ define i1 @icmp_v4i64_v4i1(<4 x i64>) {
 ; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
 ; AVX512F-NEXT:    vptestnmq %zmm0, %zmm0, %k0
 ; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    andl $15, %eax
-; AVX512F-NEXT:    xorb $0, %al
+; AVX512F-NEXT:    testb $15, %al
 ; AVX512F-NEXT:    setnp %al
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
@@ -1292,8 +1276,7 @@ define i1 @icmp_v4i64_v4i1(<4 x i64>) {
 ; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
 ; AVX512BW-NEXT:    vptestnmq %zmm0, %zmm0, %k0
 ; AVX512BW-NEXT:    kmovd %k0, %eax
-; AVX512BW-NEXT:    andl $15, %eax
-; AVX512BW-NEXT:    xorb $0, %al
+; AVX512BW-NEXT:    testb $15, %al
 ; AVX512BW-NEXT:    setnp %al
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -1302,8 +1285,7 @@ define i1 @icmp_v4i64_v4i1(<4 x i64>) {
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vptestnmq %ymm0, %ymm0, %k0
 ; AVX512VL-NEXT:    kmovd %k0, %eax
-; AVX512VL-NEXT:    andl $15, %eax
-; AVX512VL-NEXT:    xorb $0, %al
+; AVX512VL-NEXT:    testb %al, %al
 ; AVX512VL-NEXT:    setnp %al
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
@@ -1321,7 +1303,7 @@ define i1 @icmp_v8i32_v8i1(<8 x i32>) {
 ; SSE-NEXT:    packssdw %xmm1, %xmm0
 ; SSE-NEXT:    packsswb %xmm0, %xmm0
 ; SSE-NEXT:    pmovmskb %xmm0, %eax
-; SSE-NEXT:    xorb $0, %al
+; SSE-NEXT:    testb %al, %al
 ; SSE-NEXT:    setnp %al
 ; SSE-NEXT:    retq
 ;
@@ -1333,7 +1315,7 @@ define i1 @icmp_v8i32_v8i1(<8 x i32>) {
 ; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vmovmskps %ymm0, %eax
-; AVX1-NEXT:    xorb $0, %al
+; AVX1-NEXT:    testb %al, %al
 ; AVX1-NEXT:    setnp %al
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -1343,7 +1325,7 @@ define i1 @icmp_v8i32_v8i1(<8 x i32>) {
 ; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX2-NEXT:    vpcmpeqd %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vmovmskps %ymm0, %eax
-; AVX2-NEXT:    xorb $0, %al
+; AVX2-NEXT:    testb %al, %al
 ; AVX2-NEXT:    setnp %al
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -1353,7 +1335,7 @@ define i1 @icmp_v8i32_v8i1(<8 x i32>) {
 ; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
 ; AVX512F-NEXT:    vptestnmd %zmm0, %zmm0, %k0
 ; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    xorb $0, %al
+; AVX512F-NEXT:    testb %al, %al
 ; AVX512F-NEXT:    setnp %al
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
@@ -1363,7 +1345,7 @@ define i1 @icmp_v8i32_v8i1(<8 x i32>) {
 ; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
 ; AVX512BW-NEXT:    vptestnmd %zmm0, %zmm0, %k0
 ; AVX512BW-NEXT:    kmovd %k0, %eax
-; AVX512BW-NEXT:    xorb $0, %al
+; AVX512BW-NEXT:    testb %al, %al
 ; AVX512BW-NEXT:    setnp %al
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -1372,7 +1354,7 @@ define i1 @icmp_v8i32_v8i1(<8 x i32>) {
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vptestnmd %ymm0, %ymm0, %k0
 ; AVX512VL-NEXT:    kmovd %k0, %eax
-; AVX512VL-NEXT:    xorb $0, %al
+; AVX512VL-NEXT:    testb %al, %al
 ; AVX512VL-NEXT:    setnp %al
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
@@ -1584,7 +1566,7 @@ define i1 @icmp_v8i64_v8i1(<8 x i64>) {
 ; SSE2-NEXT:    packssdw %xmm3, %xmm1
 ; SSE2-NEXT:    packsswb %xmm1, %xmm1
 ; SSE2-NEXT:    pmovmskb %xmm1, %eax
-; SSE2-NEXT:    xorb $0, %al
+; SSE2-NEXT:    testb %al, %al
 ; SSE2-NEXT:    setnp %al
 ; SSE2-NEXT:    retq
 ;
@@ -1600,7 +1582,7 @@ define i1 @icmp_v8i64_v8i1(<8 x i64>) {
 ; SSE41-NEXT:    packssdw %xmm2, %xmm0
 ; SSE41-NEXT:    packsswb %xmm0, %xmm0
 ; SSE41-NEXT:    pmovmskb %xmm0, %eax
-; SSE41-NEXT:    xorb $0, %al
+; SSE41-NEXT:    testb %al, %al
 ; SSE41-NEXT:    setnp %al
 ; SSE41-NEXT:    retq
 ;
@@ -1617,7 +1599,7 @@ define i1 @icmp_v8i64_v8i1(<8 x i64>) {
 ; AVX1-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vmovmskps %ymm0, %eax
-; AVX1-NEXT:    xorb $0, %al
+; AVX1-NEXT:    testb %al, %al
 ; AVX1-NEXT:    setnp %al
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -1630,7 +1612,7 @@ define i1 @icmp_v8i64_v8i1(<8 x i64>) {
 ; AVX2-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vmovmskps %ymm0, %eax
-; AVX2-NEXT:    xorb $0, %al
+; AVX2-NEXT:    testb %al, %al
 ; AVX2-NEXT:    setnp %al
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -1639,7 +1621,7 @@ define i1 @icmp_v8i64_v8i1(<8 x i64>) {
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vptestnmq %zmm0, %zmm0, %k0
 ; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    xorb $0, %al
+; AVX512F-NEXT:    testb %al, %al
 ; AVX512F-NEXT:    setnp %al
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
@@ -1648,7 +1630,7 @@ define i1 @icmp_v8i64_v8i1(<8 x i64>) {
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    vptestnmq %zmm0, %zmm0, %k0
 ; AVX512BW-NEXT:    kmovd %k0, %eax
-; AVX512BW-NEXT:    xorb $0, %al
+; AVX512BW-NEXT:    testb %al, %al
 ; AVX512BW-NEXT:    setnp %al
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -1657,7 +1639,7 @@ define i1 @icmp_v8i64_v8i1(<8 x i64>) {
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vptestnmq %zmm0, %zmm0, %k0
 ; AVX512VL-NEXT:    kmovd %k0, %eax
-; AVX512VL-NEXT:    xorb $0, %al
+; AVX512VL-NEXT:    testb %al, %al
 ; AVX512VL-NEXT:    setnp %al
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq

From e6c2c9a7d15171a57c98024511eaa8885f1bd5fd Mon Sep 17 00:00:00 2001
From: Jan Kratochvil <jan.kratochvil@redhat.com>
Date: Sun, 2 Aug 2020 22:41:02 +0200
Subject: [PATCH 0859/1035] [lldb] [test] Fix
 DW_TAG_GNU_call_site-DW_AT_low_pc.s relocation

I have made the DW_FORM_ref4 relative. One could also use relocated
DW_FORM_ref_addr instead.

Tested with:
        echo 'void f(){}'|clang -o 1.o -c -Wall -g -x c -;./bin/clang -o 1 1.o ../llvm-monorepo/lldb/test/Shell/SymbolFile/DWARF/DW_TAG_GNU_call_site-DW_AT_low_pc.s;./bin/lldb --no-lldbinit ./1 -o r -o 'p p' -o exit
---
 .../SymbolFile/DWARF/DW_TAG_GNU_call_site-DW_AT_low_pc.s    | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lldb/test/Shell/SymbolFile/DWARF/DW_TAG_GNU_call_site-DW_AT_low_pc.s b/lldb/test/Shell/SymbolFile/DWARF/DW_TAG_GNU_call_site-DW_AT_low_pc.s
index 29cafebaa4405..8cc1cc1ed2f95 100644
--- a/lldb/test/Shell/SymbolFile/DWARF/DW_TAG_GNU_call_site-DW_AT_low_pc.s
+++ b/lldb/test/Shell/SymbolFile/DWARF/DW_TAG_GNU_call_site-DW_AT_low_pc.s
@@ -79,7 +79,7 @@ main:
 	.uleb128 0x2	# (DIE (0x2d) DW_TAG_subprogram)
 			# DW_AT_external
 	.asciz "main"	# DW_AT_name: "main"
-	.long	.Ltype_int	# DW_AT_type
+	.long	.Ltype_int - .Ldebug_info0	# DW_AT_type
 	.quad	.LFB2	# DW_AT_low_pc
 	.quad	.LFE2-.LFB2	# DW_AT_high_pc
 	.uleb128 0x1	# DW_AT_frame_base
@@ -87,7 +87,7 @@ main:
 			# DW_AT_GNU_all_call_sites
 	.uleb128 0x3	# (DIE (0x4f) DW_TAG_GNU_call_site)
 	.quad	.LVL4	# DW_AT_low_pc
-	.long	.Lfunc_a	# DW_AT_abstract_origin
+	.long	.Lfunc_a - .Ldebug_info0	# DW_AT_abstract_origin
 	.uleb128 0x4	# (DIE (0x5c) DW_TAG_GNU_call_site_parameter)
 	.uleb128 0x1	# DW_AT_location
 	.byte	0x55	# DW_OP_reg5
@@ -111,7 +111,7 @@ main:
 			# DW_AT_GNU_all_call_sites
 	.uleb128 0x7	# (DIE (0x86) DW_TAG_formal_parameter)
 	.asciz "p"	# DW_AT_name
-	.long	.Ltype_int	# DW_AT_type
+	.long	.Ltype_int - .Ldebug_info0	# DW_AT_type
 	.long	.LLST0	# DW_AT_location
 	.byte	0	# end of children of DIE 0x6a
 	.byte	0	# end of children of DIE 0xb

From 00a0282ff8f9a790e93c19ef6fa3758e209cdbe6 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Sun, 2 Aug 2020 21:45:42 +0100
Subject: [PATCH 0860/1035] [Clang] Remove run-lines which use opt to run
 -ipconstprop.

ipconstprop is going to get removed and checking opt with specific
passes makes the tests more fragile.

The tests retain the important checks that !callback metadata is created
correctly.
---
 clang/test/CodeGen/callback_annotated.c      | 16 +---------------
 clang/test/CodeGen/callback_openmp.c         |  6 +-----
 clang/test/CodeGen/callback_pthread_create.c |  7 +------
 3 files changed, 3 insertions(+), 26 deletions(-)

diff --git a/clang/test/CodeGen/callback_annotated.c b/clang/test/CodeGen/callback_annotated.c
index c5b431d5ef845..83a79c3491daf 100644
--- a/clang/test/CodeGen/callback_annotated.c
+++ b/clang/test/CodeGen/callback_annotated.c
@@ -1,6 +1,4 @@
-// RUN: %clang_cc1 -triple i386-unknown-unknown -fopenmp -O1 -fno-experimental-new-pass-manager %s -emit-llvm -o - | FileCheck %s --check-prefix=RUN1
-// RUN: %clang_cc1 -triple i386-unknown-unknown -fopenmp -O1 -fno-experimental-new-pass-manager %s -emit-llvm -o - | FileCheck %s --check-prefix=RUN2
-// RUN: %clang_cc1 -triple i386-unknown-unknown -fopenmp -O1 -fno-experimental-new-pass-manager %s -emit-llvm -o - | opt -ipconstprop -S | FileCheck --check-prefix=IPCP %s
+// RUN: %clang_cc1 -triple i386-unknown-unknown -fopenmp -fno-experimental-new-pass-manager %s -emit-llvm -o - -disable-llvm-optzns | FileCheck %s --check-prefix=RUN1
 
 // RUN1-DAG: @broker0({{[^#]*#[0-9]+}} !callback ![[cid0:[0-9]+]]
 __attribute__((callback(1, 2))) void *broker0(void *(*callee)(void *), void *payload) {
@@ -29,22 +27,10 @@ __attribute__((callback(4, -1, a, __))) void *broker4(int a, int, int, int (*cal
 __attribute__((callback(4, d, 5, 2))) void *broker5(int, int, int, int (*callee)(int, int, int), int d);
 
 static void *VoidPtr2VoidPtr(void *payload) {
-  // RUN2: ret i8* %payload
-  // IPCP: ret i8* null
   return payload;
 }
 
 static int ThreeInt2Int(int a, int b, int c) {
-  // RUN2:   define internal i32 @ThreeInt2Int(i32 %a, i32 %b, i32 %c)
-  // RUN2:     %mul = mul nsw i32 %b, %a
-  // RUN2:     %add = add nsw i32 %mul, %c
-  // RUN2:     ret i32 %add
-
-  // IPCP:   define internal i32 @ThreeInt2Int(i32 %a, i32 %b, i32 %c)
-  // IPCP:     %mul = mul nsw i32 4, %a
-  // IPCP:     %add = add nsw i32 %mul, %c
-  // IPCP:     ret i32 %add
-
   return a * b + c;
 }
 
diff --git a/clang/test/CodeGen/callback_openmp.c b/clang/test/CodeGen/callback_openmp.c
index 2fc9dcd391f63..90e63fdb2e580 100644
--- a/clang/test/CodeGen/callback_openmp.c
+++ b/clang/test/CodeGen/callback_openmp.c
@@ -1,5 +1,4 @@
-// RUN: %clang_cc1 -triple i386-unknown-unknown -fopenmp -O1 %s -emit-llvm -o - | FileCheck %s
-// RUN: %clang_cc1 -triple i386-unknown-unknown -fopenmp -O1 %s -emit-llvm -o - | opt -ipconstprop -S | FileCheck --check-prefix=IPCP %s
+// RUN: %clang_cc1 -triple i386-unknown-unknown -fopenmp %s -emit-llvm -o - -disable-llvm-optzns | FileCheck %s
 
 // CHECK: declare !callback ![[cid:[0-9]+]] void @__kmpc_fork_call
 // CHECK: declare !callback ![[cid]] void @__kmpc_fork_teams
@@ -15,14 +14,11 @@ void foo(int q) {
 
   #pragma omp parallel firstprivate(q, p)
   work1(p, q);
-// IPCP: call void @work1(i32 2, i32 %{{[._a-zA-Z0-9]*}})
 
   #pragma omp parallel for firstprivate(p, q)
   for (int i = 0; i < q; i++)
     work2(i, p);
-// IPCP: call void @work2(i32 %{{[._a-zA-Z0-9]*}}, i32 2)
 
   #pragma omp target teams firstprivate(p)
   work12(p, p);
-// IPCP: call void @work12(i32 2, i32 2)
 }
diff --git a/clang/test/CodeGen/callback_pthread_create.c b/clang/test/CodeGen/callback_pthread_create.c
index 785440030b32e..d1b01b91eac3f 100644
--- a/clang/test/CodeGen/callback_pthread_create.c
+++ b/clang/test/CodeGen/callback_pthread_create.c
@@ -1,5 +1,4 @@
-// RUN: %clang_cc1 -O1 %s -S -emit-llvm -o - | FileCheck %s
-// RUN: %clang_cc1 -O1 %s -S -emit-llvm -o - | opt -ipconstprop -S | FileCheck --check-prefix=IPCP %s
+// RUN: %clang_cc1 %s -S -emit-llvm -o - -disable-llvm-optzns | FileCheck %s
 
 // CHECK: declare !callback ![[cid:[0-9]+]] {{.*}}i32 @pthread_create
 // CHECK: ![[cid]] = !{![[cidb:[0-9]+]]}
@@ -21,14 +20,10 @@ int pthread_create(pthread_t *, const pthread_attr_t *,
 const int GlobalVar = 0;
 
 static void *callee0(void *payload) {
-// IPCP:      define internal i8* @callee0
-// IPCP:        ret i8* null
   return payload;
 }
 
 static void *callee1(void *payload) {
-// IPCP:      define internal i8* @callee1
-// IPCP:        ret i8* bitcast (i32* @GlobalVar to i8*)
   return payload;
 }
 

From 08cf49658c1da891fb2b2cb577a89a4d6d1a7adc Mon Sep 17 00:00:00 2001
From: Vitaly Buka <vitalybuka@google.com>
Date: Sun, 2 Aug 2020 13:56:38 -0700
Subject: [PATCH 0861/1035] [StackSafety, NFC] Don't insert empty objects into
 the map

Result should be the same but it makes generateParamAccessSummary 5x
faster.
---
 llvm/lib/Analysis/StackSafetyAnalysis.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Analysis/StackSafetyAnalysis.cpp b/llvm/lib/Analysis/StackSafetyAnalysis.cpp
index bbfc303aefac4..bcbe9a63c182d 100644
--- a/llvm/lib/Analysis/StackSafetyAnalysis.cpp
+++ b/llvm/lib/Analysis/StackSafetyAnalysis.cpp
@@ -928,7 +928,7 @@ void llvm::generateParamAccessSummary(ModuleSummaryIndex &Index) {
   for (auto &GVS : Index) {
     for (auto &GV : GVS.second.SummaryList) {
       FunctionSummary *FS = dyn_cast<FunctionSummary>(GV.get());
-      if (!FS)
+      if (!FS || FS->paramAccesses().empty())
         continue;
       if (FS->isLive() && FS->isDSOLocal()) {
         FunctionInfo<FunctionSummary> FI;

From 599955eb56ebad50c12422cb6194a2da770902a0 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Sun, 2 Aug 2020 22:04:44 +0100
Subject: [PATCH 0862/1035] Recommit "[IPConstProp] Remove and move tests to
 SCCP."

This reverts commit 59d6e814ce0e7b40b7cc3ab136b9af2ffab9c6f8.

The cause for the revert (3 clang tests running opt -ipconstprop) was
fixed by removing those lines.
---
 llvm/bindings/go/llvm/transforms_ipo.go       |   1 -
 .../bindings/ocaml/transforms/ipo/ipo_ocaml.c |   6 -
 .../ocaml/transforms/ipo/llvm_ipo.mli         |   5 -
 llvm/docs/Passes.rst                          |   9 -
 llvm/include/llvm-c/Transforms/IPO.h          |   3 -
 llvm/include/llvm/InitializePasses.h          |   1 -
 llvm/include/llvm/LinkAllPasses.h             |   1 -
 llvm/include/llvm/Transforms/IPO.h            |   6 -
 llvm/lib/Transforms/IPO/CMakeLists.txt        |   1 -
 .../Transforms/IPO/IPConstantPropagation.cpp  | 308 ------------------
 llvm/lib/Transforms/IPO/IPO.cpp               |   5 -
 .../Transforms/IPConstantProp/comdat-ipo.ll   |  34 --
 .../2008-06-09-WeakProp.ll                    |   2 +-
 .../{IPConstantProp => SCCP}/PR43857.ll       |   4 +-
 .../arg-count-mismatch.ll                     |   4 +-
 .../arg-type-mismatch.ll                      |   2 +-
 llvm/test/Transforms/SCCP/comdat-ipo.ll       |  14 +-
 .../{IPConstantProp => SCCP}/deadarg.ll       |   2 +-
 .../multiple_callbacks.ll                     |   7 +-
 .../{IPConstantProp => SCCP}/naked-return.ll  |   1 -
 .../openmp_parallel_for.ll                    |   4 +-
 .../{IPConstantProp => SCCP}/pthreads.ll      |   6 +-
 .../{IPConstantProp => SCCP}/recursion.ll     |   2 +-
 .../return-argument.ll                        |   8 +-
 .../return-constant.ll                        |   6 +-
 .../return-constants.ll                       |  18 +-
 .../thread_local_acs.ll                       |   4 +-
 llvm/utils/findoptdiff                        |   2 +-
 .../llvm/lib/Transforms/IPO/BUILD.gn          |   1 -
 29 files changed, 43 insertions(+), 424 deletions(-)
 delete mode 100644 llvm/lib/Transforms/IPO/IPConstantPropagation.cpp
 delete mode 100644 llvm/test/Transforms/IPConstantProp/comdat-ipo.ll
 rename llvm/test/Transforms/{IPConstantProp => SCCP}/2008-06-09-WeakProp.ll (91%)
 rename llvm/test/Transforms/{IPConstantProp => SCCP}/PR43857.ll (86%)
 rename llvm/test/Transforms/{IPConstantProp => SCCP}/arg-count-mismatch.ll (96%)
 rename llvm/test/Transforms/{IPConstantProp => SCCP}/arg-type-mismatch.ll (92%)
 rename llvm/test/Transforms/{IPConstantProp => SCCP}/deadarg.ll (77%)
 rename llvm/test/Transforms/{IPConstantProp => SCCP}/multiple_callbacks.ll (96%)
 rename llvm/test/Transforms/{IPConstantProp => SCCP}/naked-return.ll (97%)
 rename llvm/test/Transforms/{IPConstantProp => SCCP}/openmp_parallel_for.ll (98%)
 rename llvm/test/Transforms/{IPConstantProp => SCCP}/pthreads.ll (93%)
 rename llvm/test/Transforms/{IPConstantProp => SCCP}/recursion.ll (90%)
 rename llvm/test/Transforms/{IPConstantProp => SCCP}/return-argument.ll (91%)
 rename llvm/test/Transforms/{IPConstantProp => SCCP}/return-constant.ll (91%)
 rename llvm/test/Transforms/{IPConstantProp => SCCP}/return-constants.ll (70%)
 rename llvm/test/Transforms/{IPConstantProp => SCCP}/thread_local_acs.ll (92%)

diff --git a/llvm/bindings/go/llvm/transforms_ipo.go b/llvm/bindings/go/llvm/transforms_ipo.go
index 1dcb2af8bf243..8a158f208bef4 100644
--- a/llvm/bindings/go/llvm/transforms_ipo.go
+++ b/llvm/bindings/go/llvm/transforms_ipo.go
@@ -32,7 +32,6 @@ func (pm PassManager) AddFunctionAttrsPass()         { C.LLVMAddFunctionAttrsPas
 func (pm PassManager) AddFunctionInliningPass()      { C.LLVMAddFunctionInliningPass(pm.C) }
 func (pm PassManager) AddGlobalDCEPass()             { C.LLVMAddGlobalDCEPass(pm.C) }
 func (pm PassManager) AddGlobalOptimizerPass()       { C.LLVMAddGlobalOptimizerPass(pm.C) }
-func (pm PassManager) AddIPConstantPropagationPass() { C.LLVMAddIPConstantPropagationPass(pm.C) }
 func (pm PassManager) AddPruneEHPass()               { C.LLVMAddPruneEHPass(pm.C) }
 func (pm PassManager) AddIPSCCPPass()                { C.LLVMAddIPSCCPPass(pm.C) }
 func (pm PassManager) AddInternalizePass(allButMain bool) {
diff --git a/llvm/bindings/ocaml/transforms/ipo/ipo_ocaml.c b/llvm/bindings/ocaml/transforms/ipo/ipo_ocaml.c
index 9fcaa10534f6b..c0e213714ed02 100644
--- a/llvm/bindings/ocaml/transforms/ipo/ipo_ocaml.c
+++ b/llvm/bindings/ocaml/transforms/ipo/ipo_ocaml.c
@@ -73,12 +73,6 @@ CAMLprim value llvm_add_global_optimizer(LLVMPassManagerRef PM) {
   return Val_unit;
 }
 
-/* [`Module] Llvm.PassManager.t -> unit */
-CAMLprim value llvm_add_ip_constant_propagation(LLVMPassManagerRef PM) {
-  LLVMAddIPConstantPropagationPass(PM);
-  return Val_unit;
-}
-
 /* [`Module] Llvm.PassManager.t -> unit */
 CAMLprim value llvm_add_prune_eh(LLVMPassManagerRef PM) {
   LLVMAddPruneEHPass(PM);
diff --git a/llvm/bindings/ocaml/transforms/ipo/llvm_ipo.mli b/llvm/bindings/ocaml/transforms/ipo/llvm_ipo.mli
index 6507c5d92c2b1..a581924c6d549 100644
--- a/llvm/bindings/ocaml/transforms/ipo/llvm_ipo.mli
+++ b/llvm/bindings/ocaml/transforms/ipo/llvm_ipo.mli
@@ -56,11 +56,6 @@ external add_global_optimizer
   : [ `Module ] Llvm.PassManager.t -> unit
   = "llvm_add_global_optimizer"
 
-(** See the [llvm::createIPConstantPropagationPass] function. *)
-external add_ipc_propagation
-  : [ `Module ] Llvm.PassManager.t -> unit
-  = "llvm_add_ip_constant_propagation"
-
 (** See the [llvm::createPruneEHPass] function. *)
 external add_prune_eh
   : [ `Module ] Llvm.PassManager.t -> unit
diff --git a/llvm/docs/Passes.rst b/llvm/docs/Passes.rst
index e45adad98c157..13317ecc966e7 100644
--- a/llvm/docs/Passes.rst
+++ b/llvm/docs/Passes.rst
@@ -676,15 +676,6 @@ This pass loops over all of the functions in the input module, looking for a
 main function.  If a main function is found, all other functions and all global
 variables with initializers are marked as internal.
 
-``-ipconstprop``: Interprocedural constant propagation
-------------------------------------------------------
-
-This pass implements an *extremely* simple interprocedural constant propagation
-pass.  It could certainly be improved in many different ways, like using a
-worklist.  This pass makes arguments dead, but does not remove them.  The
-existing dead argument elimination pass should be run after this to clean up
-the mess.
-
 ``-ipsccp``: Interprocedural Sparse Conditional Constant Propagation
 --------------------------------------------------------------------
 
diff --git a/llvm/include/llvm-c/Transforms/IPO.h b/llvm/include/llvm-c/Transforms/IPO.h
index cde3d24609208..3f2cadf32366b 100644
--- a/llvm/include/llvm-c/Transforms/IPO.h
+++ b/llvm/include/llvm-c/Transforms/IPO.h
@@ -57,9 +57,6 @@ void LLVMAddGlobalDCEPass(LLVMPassManagerRef PM);
 /** See llvm::createGlobalOptimizerPass function. */
 void LLVMAddGlobalOptimizerPass(LLVMPassManagerRef PM);
 
-/** See llvm::createIPConstantPropagationPass function. */
-void LLVMAddIPConstantPropagationPass(LLVMPassManagerRef PM);
-
 /** See llvm::createPruneEHPass function. */
 void LLVMAddPruneEHPass(LLVMPassManagerRef PM);
 
diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h
index cce6a43504021..d23ecfb98ce92 100644
--- a/llvm/include/llvm/InitializePasses.h
+++ b/llvm/include/llvm/InitializePasses.h
@@ -179,7 +179,6 @@ void initializeGuardWideningLegacyPassPass(PassRegistry&);
 void initializeHardwareLoopsPass(PassRegistry&);
 void initializeHotColdSplittingLegacyPassPass(PassRegistry&);
 void initializeHWAddressSanitizerLegacyPassPass(PassRegistry &);
-void initializeIPCPPass(PassRegistry&);
 void initializeIPSCCPLegacyPassPass(PassRegistry&);
 void initializeIRCELegacyPassPass(PassRegistry&);
 void initializeIRTranslatorPass(PassRegistry&);
diff --git a/llvm/include/llvm/LinkAllPasses.h b/llvm/include/llvm/LinkAllPasses.h
index 90e2e24294d40..9d7ac2b3f3b99 100644
--- a/llvm/include/llvm/LinkAllPasses.h
+++ b/llvm/include/llvm/LinkAllPasses.h
@@ -116,7 +116,6 @@ namespace {
       (void) llvm::createGlobalsAAWrapperPass();
       (void) llvm::createGuardWideningPass();
       (void) llvm::createLoopGuardWideningPass();
-      (void) llvm::createIPConstantPropagationPass();
       (void) llvm::createIPSCCPPass();
       (void) llvm::createInductiveRangeCheckEliminationPass();
       (void) llvm::createIndVarSimplifyPass();
diff --git a/llvm/include/llvm/Transforms/IPO.h b/llvm/include/llvm/Transforms/IPO.h
index 28e454d3b0fc7..7b73eeaf8e45b 100644
--- a/llvm/include/llvm/Transforms/IPO.h
+++ b/llvm/include/llvm/Transforms/IPO.h
@@ -155,12 +155,6 @@ Pass *createArgumentPromotionPass(unsigned maxElements = 3);
 /// createOpenMPOptLegacyPass - OpenMP specific optimizations.
 Pass *createOpenMPOptLegacyPass();
 
-//===----------------------------------------------------------------------===//
-/// createIPConstantPropagationPass - This pass propagates constants from call
-/// sites into the bodies of functions.
-///
-ModulePass *createIPConstantPropagationPass();
-
 //===----------------------------------------------------------------------===//
 /// createIPSCCPPass - This pass propagates constants from call sites into the
 /// bodies of functions, and keeps track of whether basic blocks are executable
diff --git a/llvm/lib/Transforms/IPO/CMakeLists.txt b/llvm/lib/Transforms/IPO/CMakeLists.txt
index 3f30c0289e8c3..a17ef63371063 100644
--- a/llvm/lib/Transforms/IPO/CMakeLists.txt
+++ b/llvm/lib/Transforms/IPO/CMakeLists.txt
@@ -18,7 +18,6 @@ add_llvm_component_library(LLVMipo
   GlobalOpt.cpp
   GlobalSplit.cpp
   HotColdSplitting.cpp
-  IPConstantPropagation.cpp
   IPO.cpp
   InferFunctionAttrs.cpp
   InlineSimple.cpp
diff --git a/llvm/lib/Transforms/IPO/IPConstantPropagation.cpp b/llvm/lib/Transforms/IPO/IPConstantPropagation.cpp
deleted file mode 100644
index 8d05a72d68dac..0000000000000
--- a/llvm/lib/Transforms/IPO/IPConstantPropagation.cpp
+++ /dev/null
@@ -1,308 +0,0 @@
-//===-- IPConstantPropagation.cpp - Propagate constants through calls -----===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This pass implements an _extremely_ simple interprocedural constant
-// propagation pass.  It could certainly be improved in many different ways,
-// like using a worklist.  This pass makes arguments dead, but does not remove
-// them.  The existing dead argument elimination pass should be run after this
-// to clean up the mess.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/Statistic.h"
-#include "llvm/Analysis/ValueTracking.h"
-#include "llvm/IR/AbstractCallSite.h"
-#include "llvm/IR/Constants.h"
-#include "llvm/IR/Instructions.h"
-#include "llvm/IR/Module.h"
-#include "llvm/InitializePasses.h"
-#include "llvm/Pass.h"
-#include "llvm/Transforms/IPO.h"
-using namespace llvm;
-
-#define DEBUG_TYPE "ipconstprop"
-
-STATISTIC(NumArgumentsProped, "Number of args turned into constants");
-STATISTIC(NumReturnValProped, "Number of return values turned into constants");
-
-namespace {
-  /// IPCP - The interprocedural constant propagation pass
-  ///
-  struct IPCP : public ModulePass {
-    static char ID; // Pass identification, replacement for typeid
-    IPCP() : ModulePass(ID) {
-      initializeIPCPPass(*PassRegistry::getPassRegistry());
-    }
-
-    bool runOnModule(Module &M) override;
-  };
-}
-
-/// PropagateConstantsIntoArguments - Look at all uses of the specified
-/// function.  If all uses are direct call sites, and all pass a particular
-/// constant in for an argument, propagate that constant in as the argument.
-///
-static bool PropagateConstantsIntoArguments(Function &F) {
-  if (F.arg_empty() || F.use_empty()) return false; // No arguments? Early exit.
-
-  // For each argument, keep track of its constant value and whether it is a
-  // constant or not.  The bool is driven to true when found to be non-constant.
-  SmallVector<PointerIntPair<Constant *, 1, bool>, 16> ArgumentConstants;
-  ArgumentConstants.resize(F.arg_size());
-
-  unsigned NumNonconstant = 0;
-  for (Use &U : F.uses()) {
-    User *UR = U.getUser();
-    // Ignore blockaddress uses.
-    if (isa<BlockAddress>(UR)) continue;
-
-    // If no abstract call site was created we did not understand the use, bail.
-    AbstractCallSite ACS(&U);
-    if (!ACS)
-      return false;
-
-    // Mismatched argument count is undefined behavior. Simply bail out to avoid
-    // handling of such situations below (avoiding asserts/crashes).
-    unsigned NumActualArgs = ACS.getNumArgOperands();
-    if (F.isVarArg() ? ArgumentConstants.size() > NumActualArgs
-                     : ArgumentConstants.size() != NumActualArgs)
-      return false;
-
-    // Check out all of the potentially constant arguments.  Note that we don't
-    // inspect varargs here.
-    Function::arg_iterator Arg = F.arg_begin();
-    for (unsigned i = 0, e = ArgumentConstants.size(); i != e; ++i, ++Arg) {
-
-      // If this argument is known non-constant, ignore it.
-      if (ArgumentConstants[i].getInt())
-        continue;
-
-      Value *V = ACS.getCallArgOperand(i);
-      Constant *C = dyn_cast_or_null<Constant>(V);
-
-      // Mismatched argument type is undefined behavior. Simply bail out to avoid
-      // handling of such situations below (avoiding asserts/crashes).
-      if (C && Arg->getType() != C->getType())
-        return false;
-
-      // We can only propagate thread independent values through callbacks.
-      // This is different to direct/indirect call sites because for them we
-      // know the thread executing the caller and callee is the same. For
-      // callbacks this is not guaranteed, thus a thread dependent value could
-      // be different for the caller and callee, making it invalid to propagate.
-      if (C && ACS.isCallbackCall() && C->isThreadDependent()) {
-        // Argument became non-constant. If all arguments are non-constant now,
-        // give up on this function.
-        if (++NumNonconstant == ArgumentConstants.size())
-          return false;
-
-        ArgumentConstants[i].setInt(true);
-        continue;
-      }
-
-      if (C && ArgumentConstants[i].getPointer() == nullptr) {
-        ArgumentConstants[i].setPointer(C); // First constant seen.
-      } else if (C && ArgumentConstants[i].getPointer() == C) {
-        // Still the constant value we think it is.
-      } else if (V == &*Arg) {
-        // Ignore recursive calls passing argument down.
-      } else {
-        // Argument became non-constant.  If all arguments are non-constant now,
-        // give up on this function.
-        if (++NumNonconstant == ArgumentConstants.size())
-          return false;
-        ArgumentConstants[i].setInt(true);
-      }
-    }
-  }
-
-  // If we got to this point, there is a constant argument!
-  assert(NumNonconstant != ArgumentConstants.size());
-  bool MadeChange = false;
-  Function::arg_iterator AI = F.arg_begin();
-  for (unsigned i = 0, e = ArgumentConstants.size(); i != e; ++i, ++AI) {
-    // Do we have a constant argument?
-    if (ArgumentConstants[i].getInt() || AI->use_empty() ||
-        (AI->hasByValAttr() && !F.onlyReadsMemory()))
-      continue;
-
-    Value *V = ArgumentConstants[i].getPointer();
-    if (!V) V = UndefValue::get(AI->getType());
-    AI->replaceAllUsesWith(V);
-    ++NumArgumentsProped;
-    MadeChange = true;
-  }
-  return MadeChange;
-}
-
-
-// Check to see if this function returns one or more constants. If so, replace
-// all callers that use those return values with the constant value. This will
-// leave in the actual return values and instructions, but deadargelim will
-// clean that up.
-//
-// Additionally if a function always returns one of its arguments directly,
-// callers will be updated to use the value they pass in directly instead of
-// using the return value.
-static bool PropagateConstantReturn(Function &F) {
-  if (F.getReturnType()->isVoidTy())
-    return false; // No return value.
-
-  // We can infer and propagate the return value only when we know that the
-  // definition we'll get at link time is *exactly* the definition we see now.
-  // For more details, see GlobalValue::mayBeDerefined.
-  if (!F.isDefinitionExact())
-    return false;
-
-  // Don't touch naked functions. The may contain asm returning
-  // value we don't see, so we may end up interprocedurally propagating
-  // the return value incorrectly.
-  if (F.hasFnAttribute(Attribute::Naked))
-    return false;
-
-  // Check to see if this function returns a constant.
-  SmallVector<Value *,4> RetVals;
-  StructType *STy = dyn_cast<StructType>(F.getReturnType());
-  if (STy)
-    for (unsigned i = 0, e = STy->getNumElements(); i < e; ++i)
-      RetVals.push_back(UndefValue::get(STy->getElementType(i)));
-  else
-    RetVals.push_back(UndefValue::get(F.getReturnType()));
-
-  unsigned NumNonConstant = 0;
-  for (BasicBlock &BB : F)
-    if (ReturnInst *RI = dyn_cast<ReturnInst>(BB.getTerminator())) {
-      for (unsigned i = 0, e = RetVals.size(); i != e; ++i) {
-        // Already found conflicting return values?
-        Value *RV = RetVals[i];
-        if (!RV)
-          continue;
-
-        // Find the returned value
-        Value *V;
-        if (!STy)
-          V = RI->getOperand(0);
-        else
-          V = FindInsertedValue(RI->getOperand(0), i);
-
-        if (V) {
-          // Ignore undefs, we can change them into anything
-          if (isa<UndefValue>(V))
-            continue;
-
-          // Try to see if all the rets return the same constant or argument.
-          if (isa<Constant>(V) || isa<Argument>(V)) {
-            if (isa<UndefValue>(RV)) {
-              // No value found yet? Try the current one.
-              RetVals[i] = V;
-              continue;
-            }
-            // Returning the same value? Good.
-            if (RV == V)
-              continue;
-          }
-        }
-        // Different or no known return value? Don't propagate this return
-        // value.
-        RetVals[i] = nullptr;
-        // All values non-constant? Stop looking.
-        if (++NumNonConstant == RetVals.size())
-          return false;
-      }
-    }
-
-  // If we got here, the function returns at least one constant value.  Loop
-  // over all users, replacing any uses of the return value with the returned
-  // constant.
-  bool MadeChange = false;
-  for (Use &U : F.uses()) {
-    CallBase *CB = dyn_cast<CallBase>(U.getUser());
-
-    // Not a call instruction or a call instruction that's not calling F
-    // directly?
-    if (!CB || !CB->isCallee(&U))
-      continue;
-
-    // Call result not used?
-    if (CB->use_empty())
-      continue;
-
-    MadeChange = true;
-
-    if (!STy) {
-      Value* New = RetVals[0];
-      if (Argument *A = dyn_cast<Argument>(New))
-        // Was an argument returned? Then find the corresponding argument in
-        // the call instruction and use that.
-        New = CB->getArgOperand(A->getArgNo());
-      CB->replaceAllUsesWith(New);
-      continue;
-    }
-
-    for (auto I = CB->user_begin(), E = CB->user_end(); I != E;) {
-      Instruction *Ins = cast<Instruction>(*I);
-
-      // Increment now, so we can remove the use
-      ++I;
-
-      // Find the index of the retval to replace with
-      int index = -1;
-      if (ExtractValueInst *EV = dyn_cast<ExtractValueInst>(Ins))
-        if (EV->getNumIndices() == 1)
-          index = *EV->idx_begin();
-
-      // If this use uses a specific return value, and we have a replacement,
-      // replace it.
-      if (index != -1) {
-        Value *New = RetVals[index];
-        if (New) {
-          if (Argument *A = dyn_cast<Argument>(New))
-            // Was an argument returned? Then find the corresponding argument in
-            // the call instruction and use that.
-            New = CB->getArgOperand(A->getArgNo());
-          Ins->replaceAllUsesWith(New);
-          Ins->eraseFromParent();
-        }
-      }
-    }
-  }
-
-  if (MadeChange) ++NumReturnValProped;
-  return MadeChange;
-}
-
-char IPCP::ID = 0;
-INITIALIZE_PASS(IPCP, "ipconstprop",
-                "Interprocedural constant propagation", false, false)
-
-ModulePass *llvm::createIPConstantPropagationPass() { return new IPCP(); }
-
-bool IPCP::runOnModule(Module &M) {
-  if (skipModule(M))
-    return false;
-
-  bool Changed = false;
-  bool LocalChange = true;
-
-  // FIXME: instead of using smart algorithms, we just iterate until we stop
-  // making changes.
-  while (LocalChange) {
-    LocalChange = false;
-    for (Function &F : M)
-      if (!F.isDeclaration()) {
-        // Delete any klingons.
-        F.removeDeadConstantUsers();
-        if (F.hasLocalLinkage())
-          LocalChange |= PropagateConstantsIntoArguments(F);
-        Changed |= PropagateConstantReturn(F);
-      }
-    Changed |= LocalChange;
-  }
-  return Changed;
-}
diff --git a/llvm/lib/Transforms/IPO/IPO.cpp b/llvm/lib/Transforms/IPO/IPO.cpp
index d37b9236380d4..45aca100086d4 100644
--- a/llvm/lib/Transforms/IPO/IPO.cpp
+++ b/llvm/lib/Transforms/IPO/IPO.cpp
@@ -35,7 +35,6 @@ void llvm::initializeIPO(PassRegistry &Registry) {
   initializeGlobalOptLegacyPassPass(Registry);
   initializeGlobalSplitPass(Registry);
   initializeHotColdSplittingLegacyPassPass(Registry);
-  initializeIPCPPass(Registry);
   initializeAlwaysInlinerLegacyPassPass(Registry);
   initializeSimpleInlinerPass(Registry);
   initializeInferFunctionAttrsLegacyPassPass(Registry);
@@ -104,10 +103,6 @@ void LLVMAddGlobalOptimizerPass(LLVMPassManagerRef PM) {
   unwrap(PM)->add(createGlobalOptimizerPass());
 }
 
-void LLVMAddIPConstantPropagationPass(LLVMPassManagerRef PM) {
-  unwrap(PM)->add(createIPConstantPropagationPass());
-}
-
 void LLVMAddPruneEHPass(LLVMPassManagerRef PM) {
   unwrap(PM)->add(createPruneEHPass());
 }
diff --git a/llvm/test/Transforms/IPConstantProp/comdat-ipo.ll b/llvm/test/Transforms/IPConstantProp/comdat-ipo.ll
deleted file mode 100644
index a19c89cb9bcf8..0000000000000
--- a/llvm/test/Transforms/IPConstantProp/comdat-ipo.ll
+++ /dev/null
@@ -1,34 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -ipconstprop -S | FileCheck %s
-
-; See PR26774
-
-define i32 @baz() {
-; CHECK-LABEL: @baz(
-; CHECK-NEXT:    ret i32 10
-;
-  ret i32 10
-}
-
-; We can const-prop @baz's return value *into* @foo, but cannot
-; constprop @foo's return value into bar.
-
-define linkonce_odr i32 @foo() {
-; CHECK-LABEL: @foo(
-; CHECK-NEXT:    [[VAL:%.*]] = call i32 @baz()
-; CHECK-NEXT:    ret i32 10
-;
-
-  %val = call i32 @baz()
-  ret i32 %val
-}
-
-define i32 @bar() {
-; CHECK-LABEL: @bar(
-; CHECK-NEXT:    [[VAL:%.*]] = call i32 @foo()
-; CHECK-NEXT:    ret i32 [[VAL]]
-;
-
-  %val = call i32 @foo()
-  ret i32 %val
-}
diff --git a/llvm/test/Transforms/IPConstantProp/2008-06-09-WeakProp.ll b/llvm/test/Transforms/SCCP/2008-06-09-WeakProp.ll
similarity index 91%
rename from llvm/test/Transforms/IPConstantProp/2008-06-09-WeakProp.ll
rename to llvm/test/Transforms/SCCP/2008-06-09-WeakProp.ll
index 270115cf5ddd1..b6c5299ae7dc4 100644
--- a/llvm/test/Transforms/IPConstantProp/2008-06-09-WeakProp.ll
+++ b/llvm/test/Transforms/SCCP/2008-06-09-WeakProp.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -ipconstprop -S | FileCheck %s
+; RUN: opt < %s -ipsccp -S | FileCheck %s
 ; Should not propagate the result of a weak function.
 ; PR2411
 
diff --git a/llvm/test/Transforms/IPConstantProp/PR43857.ll b/llvm/test/Transforms/SCCP/PR43857.ll
similarity index 86%
rename from llvm/test/Transforms/IPConstantProp/PR43857.ll
rename to llvm/test/Transforms/SCCP/PR43857.ll
index 0d0d14d398dea..efe4bca4f514c 100644
--- a/llvm/test/Transforms/IPConstantProp/PR43857.ll
+++ b/llvm/test/Transforms/SCCP/PR43857.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -S -ipconstprop | FileCheck %s
+; RUN: opt < %s -S -ipsccp | FileCheck %s
 
 %struct.wobble = type { i32 }
 %struct.zot = type { %struct.wobble, %struct.wobble, %struct.wobble }
@@ -19,7 +19,7 @@ define void @baz(<8 x i32> %arg) local_unnamed_addr {
 ; CHECK-LABEL: @baz(
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[TMP:%.*]] = call [[STRUCT_ZOT:%.*]] @widget(<8 x i32> [[ARG:%.*]])
-; CHECK-NEXT:    [[TMP1:%.*]] = extractvalue [[STRUCT_ZOT]] %tmp, 0, 0
+; CHECK-NEXT:    [[TMP1:%.*]] = extractvalue [[STRUCT_ZOT]] undef, 0, 0
 ; CHECK-NEXT:    ret void
 ;
 bb:
diff --git a/llvm/test/Transforms/IPConstantProp/arg-count-mismatch.ll b/llvm/test/Transforms/SCCP/arg-count-mismatch.ll
similarity index 96%
rename from llvm/test/Transforms/IPConstantProp/arg-count-mismatch.ll
rename to llvm/test/Transforms/SCCP/arg-count-mismatch.ll
index 7afe858e52d11..ba5f1a6d83f72 100644
--- a/llvm/test/Transforms/IPConstantProp/arg-count-mismatch.ll
+++ b/llvm/test/Transforms/SCCP/arg-count-mismatch.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -ipconstprop -S -o - | FileCheck %s
+; RUN: opt < %s -ipsccp -S -o - | FileCheck %s
 
 ; The original C source looked like this:
 ;
@@ -53,7 +53,7 @@ define internal i16 @bar(i16 %p1, i16 %p2) {
 
 define internal i16 @vararg_prop(i16 %p1, ...) {
 ; CHECK-LABEL: @vararg_prop(
-; CHECK-NEXT:    ret i16 7
+; CHECK-NEXT:    ret i16 undef
 ;
   ret i16 %p1
 }
diff --git a/llvm/test/Transforms/IPConstantProp/arg-type-mismatch.ll b/llvm/test/Transforms/SCCP/arg-type-mismatch.ll
similarity index 92%
rename from llvm/test/Transforms/IPConstantProp/arg-type-mismatch.ll
rename to llvm/test/Transforms/SCCP/arg-type-mismatch.ll
index ff924d73390b0..9a9da52174c2a 100644
--- a/llvm/test/Transforms/IPConstantProp/arg-type-mismatch.ll
+++ b/llvm/test/Transforms/SCCP/arg-type-mismatch.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -ipconstprop -S -o - | FileCheck %s
+; RUN: opt < %s -ipsccp -S -o - | FileCheck %s
 
 ; This test is just to verify that we do not crash/assert due to mismatch in
 ; argument type between the caller and callee.
diff --git a/llvm/test/Transforms/SCCP/comdat-ipo.ll b/llvm/test/Transforms/SCCP/comdat-ipo.ll
index 618075fd5e3fe..fc715f45406f3 100644
--- a/llvm/test/Transforms/SCCP/comdat-ipo.ll
+++ b/llvm/test/Transforms/SCCP/comdat-ipo.ll
@@ -1,8 +1,12 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -ipsccp -S | FileCheck %s
 
 ; See PR26774
 
 define i32 @baz() {
+; CHECK-LABEL: @baz(
+; CHECK-NEXT:    ret i32 10
+;
   ret i32 10
 }
 
@@ -11,8 +15,9 @@ define i32 @baz() {
 
 define linkonce_odr i32 @foo() {
 ; CHECK-LABEL: @foo(
-; CHECK-NEXT:  %val = call i32 @baz()
-; CHECK-NEXT:  ret i32 10
+; CHECK-NEXT:    [[VAL:%.*]] = call i32 @baz()
+; CHECK-NEXT:    ret i32 10
+;
 
   %val = call i32 @baz()
   ret i32 %val
@@ -20,8 +25,9 @@ define linkonce_odr i32 @foo() {
 
 define i32 @bar() {
 ; CHECK-LABEL: @bar(
-; CHECK-NEXT:  %val = call i32 @foo()
-; CHECK-NEXT:  ret i32 %val
+; CHECK-NEXT:    [[VAL:%.*]] = call i32 @foo()
+; CHECK-NEXT:    ret i32 [[VAL]]
+;
 
   %val = call i32 @foo()
   ret i32 %val
diff --git a/llvm/test/Transforms/IPConstantProp/deadarg.ll b/llvm/test/Transforms/SCCP/deadarg.ll
similarity index 77%
rename from llvm/test/Transforms/IPConstantProp/deadarg.ll
rename to llvm/test/Transforms/SCCP/deadarg.ll
index 25b9749b50790..1117acc7d0137 100644
--- a/llvm/test/Transforms/IPConstantProp/deadarg.ll
+++ b/llvm/test/Transforms/SCCP/deadarg.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -ipconstprop -disable-output
+; RUN: opt < %s -ipsccp -disable-output
 define internal void @foo(i32 %X) {
   call void @foo( i32 %X )
   ret void
diff --git a/llvm/test/Transforms/IPConstantProp/multiple_callbacks.ll b/llvm/test/Transforms/SCCP/multiple_callbacks.ll
similarity index 96%
rename from llvm/test/Transforms/IPConstantProp/multiple_callbacks.ll
rename to llvm/test/Transforms/SCCP/multiple_callbacks.ll
index 6684044e24ce0..3d196f86e4a1c 100644
--- a/llvm/test/Transforms/IPConstantProp/multiple_callbacks.ll
+++ b/llvm/test/Transforms/SCCP/multiple_callbacks.ll
@@ -1,5 +1,6 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -ipconstprop -S < %s | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-attributes
+; RUN: opt -ipsccp -S %s | FileCheck %s
+;
 ;
 ;                            /---------------------------------------|
 ;                            |                /----------------------|----|
@@ -38,7 +39,7 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 define internal i32 @cb0(i32 %zero) {
 ; CHECK-LABEL: @cb0(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    ret i32 0
+; CHECK-NEXT:    ret i32 [[ZERO:%.*]]
 ;
 entry:
   ret i32 %zero
diff --git a/llvm/test/Transforms/IPConstantProp/naked-return.ll b/llvm/test/Transforms/SCCP/naked-return.ll
similarity index 97%
rename from llvm/test/Transforms/IPConstantProp/naked-return.ll
rename to llvm/test/Transforms/SCCP/naked-return.ll
index 133662a211b41..daeb176b09972 100644
--- a/llvm/test/Transforms/IPConstantProp/naked-return.ll
+++ b/llvm/test/Transforms/SCCP/naked-return.ll
@@ -1,6 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -ipsccp -S %s | FileCheck %s
-; RUN: opt -ipconstprop -S %s | FileCheck %s
 
 target datalayout = "e-m:x-p:32:32-i64:64-f80:32-n8:16:32-a:0:32-S32"
 target triple = "i686-pc-windows-msvc19.0.24215"
diff --git a/llvm/test/Transforms/IPConstantProp/openmp_parallel_for.ll b/llvm/test/Transforms/SCCP/openmp_parallel_for.ll
similarity index 98%
rename from llvm/test/Transforms/IPConstantProp/openmp_parallel_for.ll
rename to llvm/test/Transforms/SCCP/openmp_parallel_for.ll
index 338cc8886e29d..27831c6e66196 100644
--- a/llvm/test/Transforms/IPConstantProp/openmp_parallel_for.ll
+++ b/llvm/test/Transforms/SCCP/openmp_parallel_for.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S -ipconstprop < %s | FileCheck %s
+; RUN: opt -S -ipsccp < %s | FileCheck %s
 ;
 ;    void bar(int, float, double);
 ;
@@ -53,7 +53,7 @@ define internal void @.omp_outlined.(i32* noalias %.global_tid., i32* noalias %.
 ; CHECK-NEXT:    [[DOTOMP_UB:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    store i64 4617315517961601024, i64* [[Q_ADDR]], align 8
+; CHECK-NEXT:    store i64 [[Q:%.*]], i64* [[Q_ADDR]], align 8
 ; CHECK-NEXT:    [[CONV:%.*]] = bitcast i64* [[Q_ADDR]] to double*
 ; CHECK-NEXT:    [[TMP:%.*]] = load i32, i32* [[N:%.*]], align 4
 ; CHECK-NEXT:    [[SUB3:%.*]] = add nsw i32 [[TMP]], -3
diff --git a/llvm/test/Transforms/IPConstantProp/pthreads.ll b/llvm/test/Transforms/SCCP/pthreads.ll
similarity index 93%
rename from llvm/test/Transforms/IPConstantProp/pthreads.ll
rename to llvm/test/Transforms/SCCP/pthreads.ll
index dcad3858da13a..a9d2b942c1a5a 100644
--- a/llvm/test/Transforms/IPConstantProp/pthreads.ll
+++ b/llvm/test/Transforms/SCCP/pthreads.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -ipconstprop -S < %s | FileCheck %s
+; RUN: opt -ipsccp -S < %s | FileCheck %s
 ;
 ;    #include <pthread.h>
 ;
@@ -44,7 +44,7 @@ declare !callback !0 dso_local i32 @pthread_create(i64*, %union.pthread_attr_t*,
 define internal i8* @foo(i8* %arg) {
 ; CHECK-LABEL: @foo(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    ret i8* null
+; CHECK-NEXT:    ret i8* [[ARG:%.*]]
 ;
 entry:
   ret i8* %arg
@@ -53,7 +53,7 @@ entry:
 define internal i8* @bar(i8* %arg) {
 ; CHECK-LABEL: @bar(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    ret i8* bitcast (i8** @GlobalVPtr to i8*)
+; CHECK-NEXT:    ret i8* [[ARG:%.*]]
 ;
 entry:
   ret i8* %arg
diff --git a/llvm/test/Transforms/IPConstantProp/recursion.ll b/llvm/test/Transforms/SCCP/recursion.ll
similarity index 90%
rename from llvm/test/Transforms/IPConstantProp/recursion.ll
rename to llvm/test/Transforms/SCCP/recursion.ll
index ac8ff9ca00e32..e4f5f1d240e7c 100644
--- a/llvm/test/Transforms/IPConstantProp/recursion.ll
+++ b/llvm/test/Transforms/SCCP/recursion.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -ipconstprop -deadargelim -S | FileCheck %s
+; RUN: opt < %s -ipsccp -deadargelim -S | FileCheck %s
 
 ; CHECK-NOT: %X
 
diff --git a/llvm/test/Transforms/IPConstantProp/return-argument.ll b/llvm/test/Transforms/SCCP/return-argument.ll
similarity index 91%
rename from llvm/test/Transforms/IPConstantProp/return-argument.ll
rename to llvm/test/Transforms/SCCP/return-argument.ll
index 6a3eac0c120d0..764b4898c9616 100644
--- a/llvm/test/Transforms/IPConstantProp/return-argument.ll
+++ b/llvm/test/Transforms/SCCP/return-argument.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -ipconstprop -S | FileCheck %s
+; RUN: opt < %s -ipsccp -S | FileCheck %s
 
 ;; This function returns its second argument on all return statements
 define internal i32* @incdec(i1 %C, i32* %V) {
@@ -49,11 +49,13 @@ define void @caller(i1 %C) personality i32 (...)* @__gxx_personality_v0 {
 ; CHECK-NEXT:    [[Q:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[W:%.*]] = call i32* @incdec(i1 [[C:%.*]], i32* [[Q]])
 ; CHECK-NEXT:    [[S1:%.*]] = call { i32, i32 } @foo(i32 1, i32 2)
+; CHECK-NEXT:    [[X1:%.*]] = extractvalue { i32, i32 } [[S1]], 0
 ; CHECK-NEXT:    [[S2:%.*]] = invoke { i32, i32 } @foo(i32 3, i32 4)
 ; CHECK-NEXT:    to label [[OK:%.*]] unwind label [[LPAD:%.*]]
 ; CHECK:       OK:
-; CHECK-NEXT:    [[Z:%.*]] = add i32 1, 3
-; CHECK-NEXT:    store i32 [[Z]], i32* [[Q]], align 4
+; CHECK-NEXT:    [[X2:%.*]] = extractvalue { i32, i32 } [[S2]], 0
+; CHECK-NEXT:    [[Z:%.*]] = add i32 [[X1]], [[X2]]
+; CHECK-NEXT:    store i32 [[Z]], i32* [[W]], align 4
 ; CHECK-NEXT:    br label [[RET:%.*]]
 ; CHECK:       LPAD:
 ; CHECK-NEXT:    [[EXN:%.*]] = landingpad { i8*, i32 }
diff --git a/llvm/test/Transforms/IPConstantProp/return-constant.ll b/llvm/test/Transforms/SCCP/return-constant.ll
similarity index 91%
rename from llvm/test/Transforms/IPConstantProp/return-constant.ll
rename to llvm/test/Transforms/SCCP/return-constant.ll
index d75aa9b969317..5cf53eea12b77 100644
--- a/llvm/test/Transforms/IPConstantProp/return-constant.ll
+++ b/llvm/test/Transforms/SCCP/return-constant.ll
@@ -1,13 +1,13 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -ipconstprop -instcombine -S | FileCheck %s
+; RUN: opt < %s -ipsccp -instcombine -S | FileCheck %s
 
 define internal i32 @foo(i1 %C) {
 ; CHECK-LABEL: @foo(
 ; CHECK-NEXT:    br i1 [[C:%.*]], label [[T:%.*]], label [[F:%.*]]
 ; CHECK:       T:
-; CHECK-NEXT:    ret i32 52
+; CHECK-NEXT:    ret i32 undef
 ; CHECK:       F:
-; CHECK-NEXT:    ret i32 52
+; CHECK-NEXT:    ret i32 undef
 ;
   br i1 %C, label %T, label %F
 
diff --git a/llvm/test/Transforms/IPConstantProp/return-constants.ll b/llvm/test/Transforms/SCCP/return-constants.ll
similarity index 70%
rename from llvm/test/Transforms/IPConstantProp/return-constants.ll
rename to llvm/test/Transforms/SCCP/return-constants.ll
index 4611067316141..cbf178d2efcaf 100644
--- a/llvm/test/Transforms/IPConstantProp/return-constants.ll
+++ b/llvm/test/Transforms/SCCP/return-constants.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -ipconstprop -S | FileCheck %s
+; RUN: opt < %s -ipsccp -S | FileCheck %s
 
 %0 = type { i32, i32 }
 
@@ -7,13 +7,9 @@ define internal %0 @foo(i1 %Q) {
 ; CHECK-LABEL: @foo(
 ; CHECK-NEXT:    br i1 [[Q:%.*]], label [[T:%.*]], label [[F:%.*]]
 ; CHECK:       T:
-; CHECK-NEXT:    [[MRV:%.*]] = insertvalue [[TMP0:%.*]] undef, i32 21, 0
-; CHECK-NEXT:    [[MRV1:%.*]] = insertvalue [[TMP0]] %mrv, i32 22, 1
-; CHECK-NEXT:    ret [[TMP0]] %mrv1
+; CHECK-NEXT:    ret [[TMP0:%.*]] { i32 21, i32 22 }
 ; CHECK:       F:
-; CHECK-NEXT:    [[MRV2:%.*]] = insertvalue [[TMP0]] undef, i32 21, 0
-; CHECK-NEXT:    [[MRV3:%.*]] = insertvalue [[TMP0]] %mrv2, i32 23, 1
-; CHECK-NEXT:    ret [[TMP0]] %mrv3
+; CHECK-NEXT:    ret [[TMP0]] { i32 21, i32 23 }
 ;
   br i1 %Q, label %T, label %F
 
@@ -30,14 +26,11 @@ F:                                                ; preds = %0
 
 define internal %0 @bar(i1 %Q) {
 ; CHECK-LABEL: @bar(
-; CHECK-NEXT:    [[A:%.*]] = insertvalue [[TMP0:%.*]] undef, i32 21, 0
 ; CHECK-NEXT:    br i1 [[Q:%.*]], label [[T:%.*]], label [[F:%.*]]
 ; CHECK:       T:
-; CHECK-NEXT:    [[B:%.*]] = insertvalue [[TMP0]] %A, i32 22, 1
-; CHECK-NEXT:    ret [[TMP0]] %B
+; CHECK-NEXT:    ret [[TMP0:%.*]] { i32 21, i32 22 }
 ; CHECK:       F:
-; CHECK-NEXT:    [[C:%.*]] = insertvalue [[TMP0]] %A, i32 23, 1
-; CHECK-NEXT:    ret [[TMP0]] %C
+; CHECK-NEXT:    ret [[TMP0]] { i32 21, i32 23 }
 ;
   %A = insertvalue %0 undef, i32 21, 0
   br i1 %Q, label %T, label %F
@@ -57,7 +50,6 @@ define %0 @caller(i1 %Q) {
 ; CHECK-NEXT:    [[B:%.*]] = extractvalue [[TMP0]] %X, 1
 ; CHECK-NEXT:    [[Y:%.*]] = call [[TMP0]] @bar(i1 [[Q]])
 ; CHECK-NEXT:    [[D:%.*]] = extractvalue [[TMP0]] %Y, 1
-; CHECK-NEXT:    [[M:%.*]] = add i32 21, 21
 ; CHECK-NEXT:    [[N:%.*]] = add i32 [[B]], [[D]]
 ; CHECK-NEXT:    ret [[TMP0]] %X
 ;
diff --git a/llvm/test/Transforms/IPConstantProp/thread_local_acs.ll b/llvm/test/Transforms/SCCP/thread_local_acs.ll
similarity index 92%
rename from llvm/test/Transforms/IPConstantProp/thread_local_acs.ll
rename to llvm/test/Transforms/SCCP/thread_local_acs.ll
index 3f843d26077ad..69c0cfeec163e 100644
--- a/llvm/test/Transforms/IPConstantProp/thread_local_acs.ll
+++ b/llvm/test/Transforms/SCCP/thread_local_acs.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -ipconstprop -S < %s | FileCheck %s
+; RUN: opt -ipsccp -S < %s | FileCheck %s
 ;
 ;    #include <threads.h>
 ;    thread_local int gtl = 0;
@@ -24,7 +24,7 @@ define internal i32 @callee(i32* %thread_local_ptr, i32* %shared_ptr) {
 ; CHECK-LABEL: @callee(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP:%.*]] = load i32, i32* [[THREAD_LOCAL_PTR:%.*]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* @gsh, align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[SHARED_PTR:%.*]], align 4
 ; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP]], [[TMP1]]
 ; CHECK-NEXT:    ret i32 [[ADD]]
 ;
diff --git a/llvm/utils/findoptdiff b/llvm/utils/findoptdiff
index 9a8803184384e..925e2dce4bf3f 100755
--- a/llvm/utils/findoptdiff
+++ b/llvm/utils/findoptdiff
@@ -70,7 +70,7 @@ dis2="$llvm2/Debug/bin/llvm-dis"
 opt1="$llvm1/Debug/bin/opt"
 opt2="$llvm2/Debug/bin/opt"
 
-all_switches="-verify -lowersetjmp -simplifycfg -mem2reg -globalopt -globaldce -ipconstprop -deadargelim -instcombine -simplifycfg -prune-eh -inline -simplify-libcalls -argpromotion -tailduplicate -simplifycfg -sroa -instcombine -predsimplify -condprop -tailcallelim -simplifycfg -reassociate -licm -loop-unswitch -instcombine -indvars -loop-unroll -instcombine -load-vn -gcse -sccp -instcombine -condprop -dse -dce -simplifycfg -deadtypeelim -constmerge -internalize -ipsccp -globalopt -constmerge -deadargelim -inline -prune-eh -globalopt -globaldce -argpromotion -instcombine -predsimplify -sroa -globalsmodref-aa -licm -load-vn -gcse -dse -instcombine -simplifycfg -verify"
+all_switches="-verify -lowersetjmp -simplifycfg -mem2reg -globalopt -globaldce -deadargelim -instcombine -simplifycfg -prune-eh -inline -simplify-libcalls -argpromotion -tailduplicate -simplifycfg -sroa -instcombine -predsimplify -condprop -tailcallelim -simplifycfg -reassociate -licm -loop-unswitch -instcombine -indvars -loop-unroll -instcombine -load-vn -gcse -sccp -instcombine -condprop -dse -dce -simplifycfg -deadtypeelim -constmerge -internalize -ipsccp -globalopt -constmerge -deadargelim -inline -prune-eh -globalopt -globaldce -argpromotion -instcombine -predsimplify -sroa -globalsmodref-aa -licm -load-vn -gcse -dse -instcombine -simplifycfg -verify"
 
 #counter=0
 function tryit {
diff --git a/llvm/utils/gn/secondary/llvm/lib/Transforms/IPO/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Transforms/IPO/BUILD.gn
index 7d48256b6cfa2..168bb499dcffd 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Transforms/IPO/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Transforms/IPO/BUILD.gn
@@ -38,7 +38,6 @@ static_library("IPO") {
     "GlobalOpt.cpp",
     "GlobalSplit.cpp",
     "HotColdSplitting.cpp",
-    "IPConstantPropagation.cpp",
     "IPO.cpp",
     "InferFunctionAttrs.cpp",
     "InlineSimple.cpp",

From 4ffa6a27aca17fe88fa6bdd605b198df6632a570 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Sun, 2 Aug 2020 22:36:53 +0100
Subject: [PATCH 0863/1035] [Bindings] Remove ipc_propagation.

IPConstantPropagation has been removed, also remove the bindings.
---
 llvm/bindings/ocaml/transforms/ipo/llvm_ipo.ml | 3 ---
 llvm/test/Bindings/OCaml/ipo.ml                | 1 -
 2 files changed, 4 deletions(-)

diff --git a/llvm/bindings/ocaml/transforms/ipo/llvm_ipo.ml b/llvm/bindings/ocaml/transforms/ipo/llvm_ipo.ml
index 1fb5594fcc7d6..d5d959288d332 100644
--- a/llvm/bindings/ocaml/transforms/ipo/llvm_ipo.ml
+++ b/llvm/bindings/ocaml/transforms/ipo/llvm_ipo.ml
@@ -33,9 +33,6 @@ external add_global_dce
 external add_global_optimizer
   : [ `Module ] Llvm.PassManager.t -> unit
   = "llvm_add_global_optimizer"
-external add_ipc_propagation
-  : [ `Module ] Llvm.PassManager.t -> unit
-  = "llvm_add_ip_constant_propagation"
 external add_prune_eh
   : [ `Module ] Llvm.PassManager.t -> unit
   = "llvm_add_prune_eh"
diff --git a/llvm/test/Bindings/OCaml/ipo.ml b/llvm/test/Bindings/OCaml/ipo.ml
index 6a67f37570edb..c3630ece7bd48 100644
--- a/llvm/test/Bindings/OCaml/ipo.ml
+++ b/llvm/test/Bindings/OCaml/ipo.ml
@@ -55,7 +55,6 @@ let test_transforms () =
            ++ add_always_inliner
            ++ add_global_dce
            ++ add_global_optimizer
-           ++ add_ipc_propagation
            ++ add_prune_eh
            ++ add_ipsccp
            ++ add_internalize ~all_but_main:true

From b497665d98ad5026b1d3d67d5793a28fefe27bea Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Sun, 2 Aug 2020 18:05:17 -0700
Subject: [PATCH 0864/1035] Reland D64327 [MC][ELF] Allow STT_SECTION
 referencing SHF_MERGE on REL targets

This drops a GNU gold workaround and reverts the revert commit rL366708.

  Before binutils 2.34, gold -O2 and above did not correctly handle R_386_GOTOFF to
  SHF_MERGE|SHF_STRINGS sections: https://sourceware.org/bugzilla/show_bug.cgi?id=16794

From the original review:

  ... it reduced the size of a big ARM-32 debug image by 33%. It contained ~68M
  of relocations symbols out of total ~71M symbols (96% of symbols table was
  generated for relocations with symbol).

-Wl,-O2 (and -Wl,-O3) is so rare that we should just lower the
optimization level for LLVM_LINKER_IS_GOLD rather than pessimizing all users.
---
 llvm/cmake/modules/AddLLVM.cmake  |  9 +++++++--
 llvm/lib/MC/ELFObjectWriter.cpp   |  5 -----
 llvm/test/MC/ELF/basic-elf-32.s   |  4 ++--
 llvm/test/MC/ELF/compression.s    |  4 ++--
 llvm/test/MC/ELF/relocation-386.s |  2 +-
 llvm/test/MC/Mips/elf-relsym.s    | 10 ++--------
 llvm/test/MC/Mips/xgot.s          |  4 ++--
 7 files changed, 16 insertions(+), 22 deletions(-)

diff --git a/llvm/cmake/modules/AddLLVM.cmake b/llvm/cmake/modules/AddLLVM.cmake
index 333167bfb6b0d..a25f3e87af61c 100644
--- a/llvm/cmake/modules/AddLLVM.cmake
+++ b/llvm/cmake/modules/AddLLVM.cmake
@@ -222,8 +222,13 @@ function(add_link_opts target_name)
     # Pass -O3 to the linker. This enabled different optimizations on different
     # linkers.
     if(NOT (${CMAKE_SYSTEM_NAME} MATCHES "Darwin|SunOS|AIX" OR WIN32))
-      set_property(TARGET ${target_name} APPEND_STRING PROPERTY
-                   LINK_FLAGS " -Wl,-O3")
+      # Before binutils 2.34, gold -O2 and above did not correctly handle R_386_GOTOFF to
+      # SHF_MERGE|SHF_STRINGS sections: https://sourceware.org/bugzilla/show_bug.cgi?id=16794
+      if(LLVM_LINKER_IS_GOLD)
+        set_property(TARGET ${target_name} APPEND_STRING PROPERTY LINK_FLAGS " -Wl,-O1")
+      else()
+        set_property(TARGET ${target_name} APPEND_STRING PROPERTY LINK_FLAGS " -Wl,-O3")
+      endif()
     endif()
 
     if(LLVM_LINKER_IS_GOLD)
diff --git a/llvm/lib/MC/ELFObjectWriter.cpp b/llvm/lib/MC/ELFObjectWriter.cpp
index 1ca9d0fe1e18c..5a5692c0cb636 100644
--- a/llvm/lib/MC/ELFObjectWriter.cpp
+++ b/llvm/lib/MC/ELFObjectWriter.cpp
@@ -1389,11 +1389,6 @@ bool ELFObjectWriter::shouldRelocateWithSymbol(const MCAssembler &Asm,
     if (Flags & ELF::SHF_MERGE) {
       if (C != 0)
         return true;
-
-      // It looks like gold has a bug (http://sourceware.org/PR16794) and can
-      // only handle section relocations to mergeable sections if using RELA.
-      if (!hasRelocationAddend())
-        return true;
     }
 
     // Most TLS relocations use a got, so they need the symbol. Even those that
diff --git a/llvm/test/MC/ELF/basic-elf-32.s b/llvm/test/MC/ELF/basic-elf-32.s
index effc363d1c6c9..1a923106036a1 100644
--- a/llvm/test/MC/ELF/basic-elf-32.s
+++ b/llvm/test/MC/ELF/basic-elf-32.s
@@ -46,9 +46,9 @@ main:                                   # @main
 
 // CHECK: Relocations [
 // CHECK:   Section {{.*}} .rel.text {
-// CHECK:     0x6  R_386_32   .L.str1
+// CHECK:     0x6  R_386_32   .rodata.str1.1 0x0
 // CHECK:     0xB  R_386_PC32 puts
-// CHECK:     0x12 R_386_32   .L.str2
+// CHECK:     0x12 R_386_32   .rodata.str1.1 0x0
 // CHECK:     0x17 R_386_PC32 puts
 // CHECK:   }
 // CHECK: ]
diff --git a/llvm/test/MC/ELF/compression.s b/llvm/test/MC/ELF/compression.s
index 93a77b3fcbdb1..9bc5c0ecc699c 100644
--- a/llvm/test/MC/ELF/compression.s
+++ b/llvm/test/MC/ELF/compression.s
@@ -38,7 +38,7 @@
 
 // In x86 32 bit named symbols are used for temporary symbols in merge
 // sections, so make sure we handle symbols inside compressed sections
-// 386-SYMBOLS-GNU: Name: .Linfo_string0
+// 386-SYMBOLS-GNU: Name: .zdebug_str
 // 386-SYMBOLS-GNU-NOT: }
 // 386-SYMBOLS-GNU: Section: .zdebug_str
 
@@ -73,7 +73,7 @@
 // ZLIB-STYLE-FLAGS32-NEXT: AddressAlignment: 4
 // ZLIB-STYLE-FLAGS64-NEXT: AddressAlignment: 8
 
-// 386-SYMBOLS-ZLIB: Name: .Linfo_string0
+// 386-SYMBOLS-ZLIB: Name: .debug_str
 // 386-SYMBOLS-ZLIB-NOT: }
 // 386-SYMBOLS-ZLIB: Section: .debug_str
 
diff --git a/llvm/test/MC/ELF/relocation-386.s b/llvm/test/MC/ELF/relocation-386.s
index 4273750a41926..b9bf9b6a00c1d 100644
--- a/llvm/test/MC/ELF/relocation-386.s
+++ b/llvm/test/MC/ELF/relocation-386.s
@@ -8,7 +8,7 @@
 // I386: Format: elf32-i386
 // CHECK:      Relocations [
 // CHECK-NEXT:   Section {{.*}} .rel.text {
-// CHECK-NEXT:     0x2          R_386_GOTOFF     .Lfoo 0x0
+// CHECK-NEXT:     0x2          R_386_GOTOFF     .rodata.str1.16 0x0
 // CHECK-NEXT:     0x{{[^ ]+}}  R_386_PLT32      bar2 0x0
 // CHECK-NEXT:     0x{{[^ ]+}}  R_386_GOTPC      _GLOBAL_OFFSET_TABLE_ 0x0
 // Relocation 3 (bar3@GOTOFF) is done with symbol 7 (bss)
diff --git a/llvm/test/MC/Mips/elf-relsym.s b/llvm/test/MC/Mips/elf-relsym.s
index b8c2f89e82e6d..d19b4e3c48208 100644
--- a/llvm/test/MC/Mips/elf-relsym.s
+++ b/llvm/test/MC/Mips/elf-relsym.s
@@ -4,16 +4,10 @@
 
 // CHECK: Symbols [
 // CHECK:   Symbol {
-// CHECK:     Name: $.str
+// CHECK:     Name: .rodata.cst8
 // CHECK:   }
 // CHECK:   Symbol {
-// CHECK:     Name: $.str1
-// CHECK:   }
-// CHECK:   Symbol {
-// CHECK:     Name: $CPI0_0
-// CHECK:   }
-// CHECK:   Symbol {
-// CHECK:     Name: $CPI0_1
+// CHECK:     Name: .rodata.str1.1
 // CHECK:   }
 // CHECK: ]
 
diff --git a/llvm/test/MC/Mips/xgot.s b/llvm/test/MC/Mips/xgot.s
index 100d25e67223b..76490f0dec931 100644
--- a/llvm/test/MC/Mips/xgot.s
+++ b/llvm/test/MC/Mips/xgot.s
@@ -10,8 +10,8 @@
 // CHECK:   0x1C R_MIPS_GOT_LO16 ext_1
 // CHECK:   0x24 R_MIPS_CALL_HI16 printf
 // CHECK:   0x30 R_MIPS_CALL_LO16 printf
-// CHECK:   0x2C R_MIPS_GOT16 $.str
-// CHECK:   0x38 R_MIPS_LO16 $.str
+// CHECK:   0x2C R_MIPS_GOT16 .rodata.str1.1 0x0
+// CHECK:   0x38 R_MIPS_LO16 .rodata.str1.1 0x0
 // CHECK: ]
 
 	.text

From a96921afa7024533cf451ef13708082876233eef Mon Sep 17 00:00:00 2001
From: StephenFan <luxufan981014@gmail.com>
Date: Fri, 31 Jul 2020 13:31:48 +0800
Subject: [PATCH 0865/1035] [RISCV] eliminate the repetition declare of SDLoc
 DL

Differential revision: https://reviews.llvm.org/D85002
---
 llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index cad5f8e21185f..773a098690330 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -1003,7 +1003,6 @@ void RISCVTargetLowering::ReplaceNodeResults(SDNode *N,
   case ISD::BITCAST: {
     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
            Subtarget.hasStdExtF() && "Unexpected custom legalisation");
-    SDLoc DL(N);
     SDValue Op0 = N->getOperand(0);
     if (Op0.getValueType() != MVT::f32)
       return;

From 62e4644616dc87dca73357b2a4bf1487ce74e90d Mon Sep 17 00:00:00 2001
From: QingShan Zhang <qshanz@cn.ibm.com>
Date: Mon, 3 Aug 2020 03:25:20 +0000
Subject: [PATCH 0866/1035] [NFC][PowerPC] Add a multiclass for fsetcc to
 define them in a uniform way

This is a refactor patch to prepare for adding the support for strict-fsetcc
in PowerPC backend. We want to move their definition into a uniform way so that,
we could add the strict node easier.

Reviewed By: shchenz

Differential Revision: https://reviews.llvm.org/D81712
---
 llvm/lib/Target/PowerPC/PPCInstrInfo.td | 126 +++++++-----------------
 1 file changed, 34 insertions(+), 92 deletions(-)

diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.td b/llvm/lib/Target/PowerPC/PPCInstrInfo.td
index be1e59d7fae71..771a715926fd7 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrInfo.td
+++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.td
@@ -3901,105 +3901,47 @@ defm : CRNotPat<(i1 (setcc i64:$s1, i64:$s2, SETLE)),
 defm : CRNotPat<(i1 (setcc i64:$s1, i64:$s2, SETNE)),
                 (EXTRACT_SUBREG (CMPD $s1, $s2), sub_eq)>;
 
-let Predicates = [HasFPU] in {
-// Instantiations of CRNotPat for f32.
-defm : CRNotPat<(i1 (setcc f32:$s1, f32:$s2, SETUGE)),
-                (EXTRACT_SUBREG (FCMPUS $s1, $s2), sub_lt)>;
-defm : CRNotPat<(i1 (setcc f32:$s1, f32:$s2, SETGE)),
-                (EXTRACT_SUBREG (FCMPUS $s1, $s2), sub_lt)>;
-defm : CRNotPat<(i1 (setcc f32:$s1, f32:$s2, SETULE)),
-                (EXTRACT_SUBREG (FCMPUS $s1, $s2), sub_gt)>;
-defm : CRNotPat<(i1 (setcc f32:$s1, f32:$s2, SETLE)),
-                (EXTRACT_SUBREG (FCMPUS $s1, $s2), sub_gt)>;
-defm : CRNotPat<(i1 (setcc f32:$s1, f32:$s2, SETUNE)),
-                (EXTRACT_SUBREG (FCMPUS $s1, $s2), sub_eq)>;
-defm : CRNotPat<(i1 (setcc f32:$s1, f32:$s2, SETNE)),
-                (EXTRACT_SUBREG (FCMPUS $s1, $s2), sub_eq)>;
-defm : CRNotPat<(i1 (setcc f32:$s1, f32:$s2, SETO)),
-                (EXTRACT_SUBREG (FCMPUS $s1, $s2), sub_un)>;
-
-// Instantiations of CRNotPat for f64.
-defm : CRNotPat<(i1 (setcc f64:$s1, f64:$s2, SETUGE)),
-                (EXTRACT_SUBREG (FCMPUD $s1, $s2), sub_lt)>;
-defm : CRNotPat<(i1 (setcc f64:$s1, f64:$s2, SETGE)),
-                (EXTRACT_SUBREG (FCMPUD $s1, $s2), sub_lt)>;
-defm : CRNotPat<(i1 (setcc f64:$s1, f64:$s2, SETULE)),
-                (EXTRACT_SUBREG (FCMPUD $s1, $s2), sub_gt)>;
-defm : CRNotPat<(i1 (setcc f64:$s1, f64:$s2, SETLE)),
-                (EXTRACT_SUBREG (FCMPUD $s1, $s2), sub_gt)>;
-defm : CRNotPat<(i1 (setcc f64:$s1, f64:$s2, SETUNE)),
-                (EXTRACT_SUBREG (FCMPUD $s1, $s2), sub_eq)>;
-defm : CRNotPat<(i1 (setcc f64:$s1, f64:$s2, SETNE)),
-                (EXTRACT_SUBREG (FCMPUD $s1, $s2), sub_eq)>;
-defm : CRNotPat<(i1 (setcc f64:$s1, f64:$s2, SETO)),
-                (EXTRACT_SUBREG (FCMPUD $s1, $s2), sub_un)>;
-
-// Instantiations of CRNotPat for f128.
-defm : CRNotPat<(i1 (setcc f128:$s1, f128:$s2, SETUGE)),
-                (EXTRACT_SUBREG (XSCMPUQP $s1, $s2), sub_lt)>;
-defm : CRNotPat<(i1 (setcc f128:$s1, f128:$s2, SETGE)),
-                (EXTRACT_SUBREG (XSCMPUQP $s1, $s2), sub_lt)>;
-defm : CRNotPat<(i1 (setcc f128:$s1, f128:$s2, SETULE)),
-                (EXTRACT_SUBREG (XSCMPUQP $s1, $s2), sub_gt)>;
-defm : CRNotPat<(i1 (setcc f128:$s1, f128:$s2, SETLE)),
-                (EXTRACT_SUBREG (XSCMPUQP $s1, $s2), sub_gt)>;
-defm : CRNotPat<(i1 (setcc f128:$s1, f128:$s2, SETUNE)),
-                (EXTRACT_SUBREG (XSCMPUQP $s1, $s2), sub_eq)>;
-defm : CRNotPat<(i1 (setcc f128:$s1, f128:$s2, SETNE)),
-                (EXTRACT_SUBREG (XSCMPUQP $s1, $s2), sub_eq)>;
-defm : CRNotPat<(i1 (setcc f128:$s1, f128:$s2, SETO)),
-                (EXTRACT_SUBREG (XSCMPUQP $s1, $s2), sub_un)>;
+multiclass FSetCCPat<SDNode SetCC, ValueType Ty, PatLeaf FCmp> {
+  defm : CRNotPat<(i1 (SetCC Ty:$s1, Ty:$s2, SETUGE)),
+                  (EXTRACT_SUBREG (FCmp $s1, $s2), sub_lt)>;
+  defm : CRNotPat<(i1 (SetCC Ty:$s1, Ty:$s2, SETGE)),
+                  (EXTRACT_SUBREG (FCmp $s1, $s2), sub_lt)>;
+  defm : CRNotPat<(i1 (SetCC Ty:$s1, Ty:$s2, SETULE)),
+                  (EXTRACT_SUBREG (FCmp $s1, $s2), sub_gt)>;
+  defm : CRNotPat<(i1 (SetCC Ty:$s1, Ty:$s2, SETLE)),
+                  (EXTRACT_SUBREG (FCmp $s1, $s2), sub_gt)>;
+  defm : CRNotPat<(i1 (SetCC Ty:$s1, Ty:$s2, SETUNE)),
+                  (EXTRACT_SUBREG (FCmp $s1, $s2), sub_eq)>;
+  defm : CRNotPat<(i1 (SetCC Ty:$s1, Ty:$s2, SETNE)),
+                  (EXTRACT_SUBREG (FCmp $s1, $s2), sub_eq)>;
+  defm : CRNotPat<(i1 (SetCC Ty:$s1, Ty:$s2, SETO)),
+                  (EXTRACT_SUBREG (FCmp $s1, $s2), sub_un)>;
+
+  def : Pat<(i1 (SetCC Ty:$s1, Ty:$s2, SETOLT)),
+            (EXTRACT_SUBREG (FCmp $s1, $s2), sub_lt)>;
+  def : Pat<(i1 (SetCC Ty:$s1, Ty:$s2, SETLT)),
+            (EXTRACT_SUBREG (FCmp $s1, $s2), sub_lt)>;
+  def : Pat<(i1 (SetCC Ty:$s1, Ty:$s2, SETOGT)),
+            (EXTRACT_SUBREG (FCmp $s1, $s2), sub_gt)>;
+  def : Pat<(i1 (SetCC Ty:$s1, Ty:$s2, SETGT)),
+            (EXTRACT_SUBREG (FCmp $s1, $s2), sub_gt)>;
+  def : Pat<(i1 (SetCC Ty:$s1, Ty:$s2, SETOEQ)),
+            (EXTRACT_SUBREG (FCmp $s1, $s2), sub_eq)>;
+  def : Pat<(i1 (SetCC Ty:$s1, Ty:$s2, SETEQ)),
+            (EXTRACT_SUBREG (FCmp $s1, $s2), sub_eq)>;
+  def : Pat<(i1 (SetCC Ty:$s1, Ty:$s2, SETUO)),
+            (EXTRACT_SUBREG (FCmp $s1, $s2), sub_un)>;
 }
 
-// SETCC for f32.
 let Predicates = [HasFPU] in {
-def : Pat<(i1 (setcc f32:$s1, f32:$s2, SETOLT)),
-          (EXTRACT_SUBREG (FCMPUS $s1, $s2), sub_lt)>;
-def : Pat<(i1 (setcc f32:$s1, f32:$s2, SETLT)),
-          (EXTRACT_SUBREG (FCMPUS $s1, $s2), sub_lt)>;
-def : Pat<(i1 (setcc f32:$s1, f32:$s2, SETOGT)),
-          (EXTRACT_SUBREG (FCMPUS $s1, $s2), sub_gt)>;
-def : Pat<(i1 (setcc f32:$s1, f32:$s2, SETGT)),
-          (EXTRACT_SUBREG (FCMPUS $s1, $s2), sub_gt)>;
-def : Pat<(i1 (setcc f32:$s1, f32:$s2, SETOEQ)),
-          (EXTRACT_SUBREG (FCMPUS $s1, $s2), sub_eq)>;
-def : Pat<(i1 (setcc f32:$s1, f32:$s2, SETEQ)),
-          (EXTRACT_SUBREG (FCMPUS $s1, $s2), sub_eq)>;
-def : Pat<(i1 (setcc f32:$s1, f32:$s2, SETUO)),
-          (EXTRACT_SUBREG (FCMPUS $s1, $s2), sub_un)>;
+// SETCC for f32.
+defm : FSetCCPat<setcc, f32, FCMPUS>;
 
 // SETCC for f64.
-def : Pat<(i1 (setcc f64:$s1, f64:$s2, SETOLT)),
-          (EXTRACT_SUBREG (FCMPUD $s1, $s2), sub_lt)>;
-def : Pat<(i1 (setcc f64:$s1, f64:$s2, SETLT)),
-          (EXTRACT_SUBREG (FCMPUD $s1, $s2), sub_lt)>;
-def : Pat<(i1 (setcc f64:$s1, f64:$s2, SETOGT)),
-          (EXTRACT_SUBREG (FCMPUD $s1, $s2), sub_gt)>;
-def : Pat<(i1 (setcc f64:$s1, f64:$s2, SETGT)),
-          (EXTRACT_SUBREG (FCMPUD $s1, $s2), sub_gt)>;
-def : Pat<(i1 (setcc f64:$s1, f64:$s2, SETOEQ)),
-          (EXTRACT_SUBREG (FCMPUD $s1, $s2), sub_eq)>;
-def : Pat<(i1 (setcc f64:$s1, f64:$s2, SETEQ)),
-          (EXTRACT_SUBREG (FCMPUD $s1, $s2), sub_eq)>;
-def : Pat<(i1 (setcc f64:$s1, f64:$s2, SETUO)),
-          (EXTRACT_SUBREG (FCMPUD $s1, $s2), sub_un)>;
+defm : FSetCCPat<setcc, f64, FCMPUD>;
 
 // SETCC for f128.
-def : Pat<(i1 (setcc f128:$s1, f128:$s2, SETOLT)),
-          (EXTRACT_SUBREG (XSCMPUQP $s1, $s2), sub_lt)>;
-def : Pat<(i1 (setcc f128:$s1, f128:$s2, SETLT)),
-          (EXTRACT_SUBREG (XSCMPUQP $s1, $s2), sub_lt)>;
-def : Pat<(i1 (setcc f128:$s1, f128:$s2, SETOGT)),
-          (EXTRACT_SUBREG (XSCMPUQP $s1, $s2), sub_gt)>;
-def : Pat<(i1 (setcc f128:$s1, f128:$s2, SETGT)),
-          (EXTRACT_SUBREG (XSCMPUQP $s1, $s2), sub_gt)>;
-def : Pat<(i1 (setcc f128:$s1, f128:$s2, SETOEQ)),
-          (EXTRACT_SUBREG (XSCMPUQP $s1, $s2), sub_eq)>;
-def : Pat<(i1 (setcc f128:$s1, f128:$s2, SETEQ)),
-          (EXTRACT_SUBREG (XSCMPUQP $s1, $s2), sub_eq)>;
-def : Pat<(i1 (setcc f128:$s1, f128:$s2, SETUO)),
-          (EXTRACT_SUBREG (XSCMPUQP $s1, $s2), sub_un)>;
-
+defm : FSetCCPat<setcc, f128, XSCMPUQP>;
 }
 
 // This must be in this file because it relies on patterns defined in this file

From 594dec2884a4814dc97ebdfa7c83ef15bdfb379e Mon Sep 17 00:00:00 2001
From: compinder <inderjeet_kalra@hcl.com>
Date: Mon, 3 Aug 2020 08:57:17 +0530
Subject: [PATCH 0867/1035] [FLANG] Fix issues in SELECT TYPE construct when
 intrinsic type specification is specified in TYPE GUARD statement.

Fix of PR46789 and PR46830.

Differential Revision: https://reviews.llvm.org/D84290
---
 flang/lib/Semantics/check-select-type.cpp | 43 ++++++++++++++---------
 flang/test/Semantics/selecttype01.f90     | 18 ++++++++++
 flang/test/Semantics/symbol11.f90         |  8 +++--
 3 files changed, 50 insertions(+), 19 deletions(-)

diff --git a/flang/lib/Semantics/check-select-type.cpp b/flang/lib/Semantics/check-select-type.cpp
index 5b430440dffb1..ce675fa2f1dbe 100644
--- a/flang/lib/Semantics/check-select-type.cpp
+++ b/flang/lib/Semantics/check-select-type.cpp
@@ -39,7 +39,7 @@ class TypeCaseValues {
     if (std::holds_alternative<parser::Default>(guard.u)) {
       typeCases_.emplace_back(stmt, std::nullopt);
     } else if (std::optional<evaluate::DynamicType> type{GetGuardType(guard)}) {
-      if (PassesChecksOnGuard(guard, *type)) {
+      if (PassesChecksOnGuard(stmt, *type)) {
         typeCases_.emplace_back(stmt, *type);
       } else {
         hasErrors_ = true;
@@ -71,35 +71,46 @@ class TypeCaseValues {
         guard.u);
   }
 
-  bool PassesChecksOnGuard(const parser::TypeGuardStmt::Guard &guard,
+  bool PassesChecksOnGuard(const parser::Statement<parser::TypeGuardStmt> &stmt,
       const evaluate::DynamicType &guardDynamicType) {
+    const parser::TypeGuardStmt &typeGuardStmt{stmt.statement};
+    const auto &guard{std::get<parser::TypeGuardStmt::Guard>(typeGuardStmt.t)};
     return std::visit(
         common::visitors{
             [](const parser::Default &) { return true; },
             [&](const parser::TypeSpec &typeSpec) {
-              if (const DeclTypeSpec * spec{typeSpec.declTypeSpec}) {
+              const DeclTypeSpec *spec{typeSpec.declTypeSpec};
+              CHECK(spec);
+              CHECK(spec->AsIntrinsic() || spec->AsDerived());
+              bool typeSpecRetVal{false};
+              if (spec->AsIntrinsic()) {
+                typeSpecRetVal = true;
+                if (!selectorType_.IsUnlimitedPolymorphic()) { // C1162
+                  context_.Say(stmt.source,
+                      "If selector is not unlimited polymorphic, "
+                      "an intrinsic type specification must not be specified "
+                      "in the type guard statement"_err_en_US);
+                  typeSpecRetVal = false;
+                }
                 if (spec->category() == DeclTypeSpec::Character &&
                     !guardDynamicType.IsAssumedLengthCharacter()) { // C1160
                   context_.Say(parser::FindSourceLocation(typeSpec),
                       "The type specification statement must have "
                       "LEN type parameter as assumed"_err_en_US);
-                  return false;
+                  typeSpecRetVal = false;
                 }
-                if (const DerivedTypeSpec * derived{spec->AsDerived()}) {
-                  return PassesDerivedTypeChecks(
-                      *derived, parser::FindSourceLocation(typeSpec));
-                }
-                return false;
+              } else {
+                const DerivedTypeSpec *derived{spec->AsDerived()};
+                typeSpecRetVal = PassesDerivedTypeChecks(
+                    *derived, parser::FindSourceLocation(typeSpec));
               }
-              return false;
+              return typeSpecRetVal;
             },
             [&](const parser::DerivedTypeSpec &x) {
-              if (const semantics::DerivedTypeSpec *
-                  derived{x.derivedTypeSpec}) {
-                return PassesDerivedTypeChecks(
-                    *derived, parser::FindSourceLocation(x));
-              }
-              return false;
+              CHECK(x.derivedTypeSpec);
+              const semantics::DerivedTypeSpec *derived{x.derivedTypeSpec};
+              return PassesDerivedTypeChecks(
+                  *derived, parser::FindSourceLocation(x));
             },
         },
         guard.u);
diff --git a/flang/test/Semantics/selecttype01.f90 b/flang/test/Semantics/selecttype01.f90
index fe9838ae2760f..c726c232e18de 100644
--- a/flang/test/Semantics/selecttype01.f90
+++ b/flang/test/Semantics/selecttype01.f90
@@ -119,6 +119,7 @@ subroutine CheckC1159b
   integer :: x
   !ERROR: Selector 'x' in SELECT TYPE statement must be polymorphic
   select type (a => x)
+  !ERROR: If selector is not unlimited polymorphic, an intrinsic type specification must not be specified in the type guard statement
   type is (integer)
     print *,'integer ',a
   end select
@@ -127,6 +128,7 @@ subroutine CheckC1159b
 subroutine CheckC1159c
   !ERROR: Selector 'x' in SELECT TYPE statement must be polymorphic
   select type (a => x)
+  !ERROR: If selector is not unlimited polymorphic, an intrinsic type specification must not be specified in the type guard statement
   type is (integer)
     print *,'integer ',a
   end select
@@ -164,6 +166,16 @@ subroutine CheckC1162
     type is (extsquare)
     !Handle same types
     type is (rectangle)
+    !ERROR: If selector is not unlimited polymorphic, an intrinsic type specification must not be specified in the type guard statement
+    type is(integer)
+    !ERROR: If selector is not unlimited polymorphic, an intrinsic type specification must not be specified in the type guard statement
+    type is(real)
+    !ERROR: If selector is not unlimited polymorphic, an intrinsic type specification must not be specified in the type guard statement
+    type is(logical)
+    !ERROR: If selector is not unlimited polymorphic, an intrinsic type specification must not be specified in the type guard statement
+    type is(character(len=*))
+    !ERROR: If selector is not unlimited polymorphic, an intrinsic type specification must not be specified in the type guard statement
+    type is(complex)
   end select
 
   !Unlimited polymorphic objects are allowed.
@@ -187,6 +199,12 @@ subroutine CheckC1163
     !ERROR: Type specification 'square' conflicts with previous type specification
     class is (square)
   end select
+  select type (unlim_polymorphic)
+    type is (INTEGER(4))
+    type is (shape)
+    !ERROR: Type specification 'INTEGER(4)' conflicts with previous type specification
+    type is (INTEGER(4))
+  end select
 end
 
 subroutine CheckC1164
diff --git a/flang/test/Semantics/symbol11.f90 b/flang/test/Semantics/symbol11.f90
index e6ae26c740e56..3d2be676967f6 100644
--- a/flang/test/Semantics/symbol11.f90
+++ b/flang/test/Semantics/symbol11.f90
@@ -71,10 +71,12 @@ subroutine s3
    !DEF: /s3/Block1/y TARGET AssocEntity TYPE(t2)
    !REF: /s3/t2/a2
    i = y%a2
-  type is (integer(kind=8))
+  !REF: /s3/t1
+  type is (t1)
    !REF: /s3/i
-   !DEF: /s3/Block2/y TARGET AssocEntity INTEGER(8)
-   i = y
+   !DEF: /s3/Block2/y TARGET AssocEntity TYPE(t1)
+   !REF: /s3/t1/a1
+   i = y%a1
   class default
    !DEF: /s3/Block3/y TARGET AssocEntity CLASS(t1)
    print *, y

From 40da58a04bea6879e1b52a4ba35559f9d26bee07 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Sun, 2 Aug 2020 22:13:59 -0700
Subject: [PATCH 0868/1035] [MC] Default MCAsmBackend::mayNeedRelaxation() to
 false

---
 llvm/include/llvm/MC/MCAsmBackend.h                      | 4 +++-
 .../Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp    | 7 -------
 llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.h         | 5 -----
 llvm/lib/Target/BPF/MCTargetDesc/BPFAsmBackend.cpp       | 5 -----
 llvm/lib/Target/Lanai/MCTargetDesc/LanaiAsmBackend.cpp   | 5 -----
 llvm/lib/Target/MSP430/MCTargetDesc/MSP430AsmBackend.cpp | 5 -----
 llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.h       | 9 ---------
 llvm/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp   | 6 ------
 llvm/lib/Target/Sparc/MCTargetDesc/SparcAsmBackend.cpp   | 6 ------
 .../Target/SystemZ/MCTargetDesc/SystemZMCAsmBackend.cpp  | 4 ----
 .../WebAssembly/MCTargetDesc/WebAssemblyAsmBackend.cpp   | 5 -----
 11 files changed, 3 insertions(+), 58 deletions(-)

diff --git a/llvm/include/llvm/MC/MCAsmBackend.h b/llvm/include/llvm/MC/MCAsmBackend.h
index cc9f42023bc29..8f95cfd55a3d7 100644
--- a/llvm/include/llvm/MC/MCAsmBackend.h
+++ b/llvm/include/llvm/MC/MCAsmBackend.h
@@ -144,7 +144,9 @@ class MCAsmBackend {
   /// \param STI - The MCSubtargetInfo in effect when the instruction was
   /// encoded.
   virtual bool mayNeedRelaxation(const MCInst &Inst,
-                                 const MCSubtargetInfo &STI) const = 0;
+                                 const MCSubtargetInfo &STI) const {
+    return false;
+  }
 
   /// Target specific predicate for whether a given fixup requires the
   /// associated instruction to be relaxed.
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp
index 9f7dfdf624829..dc44980ce218c 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp
@@ -88,8 +88,6 @@ class AArch64AsmBackend : public MCAsmBackend {
                   uint64_t Value, bool IsResolved,
                   const MCSubtargetInfo *STI) const override;
 
-  bool mayNeedRelaxation(const MCInst &Inst,
-                         const MCSubtargetInfo &STI) const override;
   bool fixupNeedsRelaxation(const MCFixup &Fixup, uint64_t Value,
                             const MCRelaxableFragment *DF,
                             const MCAsmLayout &Layout) const override;
@@ -463,11 +461,6 @@ void AArch64AsmBackend::applyFixup(const MCAssembler &Asm, const MCFixup &Fixup,
   }
 }
 
-bool AArch64AsmBackend::mayNeedRelaxation(const MCInst &Inst,
-                                          const MCSubtargetInfo &STI) const {
-  return false;
-}
-
 bool AArch64AsmBackend::fixupNeedsRelaxation(const MCFixup &Fixup,
                                              uint64_t Value,
                                              const MCRelaxableFragment *DF,
diff --git a/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.h b/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.h
index 43b26fd0c5653..46dc914adf784 100644
--- a/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.h
+++ b/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.h
@@ -48,11 +48,6 @@ class AVRAsmBackend : public MCAsmBackend {
     return AVR::NumTargetFixupKinds;
   }
 
-  bool mayNeedRelaxation(const MCInst &Inst,
-                         const MCSubtargetInfo &STI) const override {
-    return false;
-  }
-
   bool fixupNeedsRelaxation(const MCFixup &Fixup, uint64_t Value,
                             const MCRelaxableFragment *DF,
                             const MCAsmLayout &Layout) const override {
diff --git a/llvm/lib/Target/BPF/MCTargetDesc/BPFAsmBackend.cpp b/llvm/lib/Target/BPF/MCTargetDesc/BPFAsmBackend.cpp
index 9d829ac45a10b..29e9d5da08364 100644
--- a/llvm/lib/Target/BPF/MCTargetDesc/BPFAsmBackend.cpp
+++ b/llvm/lib/Target/BPF/MCTargetDesc/BPFAsmBackend.cpp
@@ -43,11 +43,6 @@ class BPFAsmBackend : public MCAsmBackend {
 
   unsigned getNumFixupKinds() const override { return 1; }
 
-  bool mayNeedRelaxation(const MCInst &Inst,
-                         const MCSubtargetInfo &STI) const override {
-    return false;
-  }
-
   bool writeNopData(raw_ostream &OS, uint64_t Count) const override;
 };
 
diff --git a/llvm/lib/Target/Lanai/MCTargetDesc/LanaiAsmBackend.cpp b/llvm/lib/Target/Lanai/MCTargetDesc/LanaiAsmBackend.cpp
index 0fb27a926003f..a17afe5e62f67 100644
--- a/llvm/lib/Target/Lanai/MCTargetDesc/LanaiAsmBackend.cpp
+++ b/llvm/lib/Target/Lanai/MCTargetDesc/LanaiAsmBackend.cpp
@@ -69,11 +69,6 @@ class LanaiAsmBackend : public MCAsmBackend {
     return Lanai::NumTargetFixupKinds;
   }
 
-  bool mayNeedRelaxation(const MCInst & /*Inst*/,
-                         const MCSubtargetInfo &STI) const override {
-    return false;
-  }
-
   bool writeNopData(raw_ostream &OS, uint64_t Count) const override;
 };
 
diff --git a/llvm/lib/Target/MSP430/MCTargetDesc/MSP430AsmBackend.cpp b/llvm/lib/Target/MSP430/MCTargetDesc/MSP430AsmBackend.cpp
index 958212dc77c9c..071e1484196bc 100644
--- a/llvm/lib/Target/MSP430/MCTargetDesc/MSP430AsmBackend.cpp
+++ b/llvm/lib/Target/MSP430/MCTargetDesc/MSP430AsmBackend.cpp
@@ -90,11 +90,6 @@ class MSP430AsmBackend : public MCAsmBackend {
     return Infos[Kind - FirstTargetFixupKind];
   }
 
-  bool mayNeedRelaxation(const MCInst &Inst,
-                         const MCSubtargetInfo &STI) const override {
-    return false;
-  }
-
   bool writeNopData(raw_ostream &OS, uint64_t Count) const override;
 };
 
diff --git a/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.h b/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.h
index 1126b871cb114..16c7befb2670a 100644
--- a/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.h
+++ b/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.h
@@ -53,15 +53,6 @@ class MipsAsmBackend : public MCAsmBackend {
   /// @name Target Relaxation Interfaces
   /// @{
 
-  /// MayNeedRelaxation - Check whether the given instruction may need
-  /// relaxation.
-  ///
-  /// \param Inst - The instruction to test.
-  bool mayNeedRelaxation(const MCInst &Inst,
-                         const MCSubtargetInfo &STI) const override {
-    return false;
-  }
-
   /// fixupNeedsRelaxation - Target specific predicate for whether a given
   /// fixup requires the associated instruction to be relaxed.
   bool fixupNeedsRelaxation(const MCFixup &Fixup, uint64_t Value,
diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp
index 59cb2b994a4b3..72401668c8d07 100644
--- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp
+++ b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp
@@ -182,12 +182,6 @@ class PPCAsmBackend : public MCAsmBackend {
     }
   }
 
-  bool mayNeedRelaxation(const MCInst &Inst,
-                         const MCSubtargetInfo &STI) const override {
-    // FIXME.
-    return false;
-  }
-
   bool fixupNeedsRelaxation(const MCFixup &Fixup,
                             uint64_t Value,
                             const MCRelaxableFragment *DF,
diff --git a/llvm/lib/Target/Sparc/MCTargetDesc/SparcAsmBackend.cpp b/llvm/lib/Target/Sparc/MCTargetDesc/SparcAsmBackend.cpp
index 83c44e0682cef..5a9ecfe74ecc8 100644
--- a/llvm/lib/Target/Sparc/MCTargetDesc/SparcAsmBackend.cpp
+++ b/llvm/lib/Target/Sparc/MCTargetDesc/SparcAsmBackend.cpp
@@ -255,12 +255,6 @@ namespace {
       }
     }
 
-    bool mayNeedRelaxation(const MCInst &Inst,
-                           const MCSubtargetInfo &STI) const override {
-      // FIXME.
-      return false;
-    }
-
     /// fixupNeedsRelaxation - Target specific predicate for whether a given
     /// fixup requires the associated instruction to be relaxed.
     bool fixupNeedsRelaxation(const MCFixup &Fixup,
diff --git a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmBackend.cpp b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmBackend.cpp
index e62f5040898f0..5f276f7935784 100644
--- a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmBackend.cpp
+++ b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmBackend.cpp
@@ -54,10 +54,6 @@ class SystemZMCAsmBackend : public MCAsmBackend {
                   const MCValue &Target, MutableArrayRef<char> Data,
                   uint64_t Value, bool IsResolved,
                   const MCSubtargetInfo *STI) const override;
-  bool mayNeedRelaxation(const MCInst &Inst,
-                         const MCSubtargetInfo &STI) const override {
-    return false;
-  }
   bool fixupNeedsRelaxation(const MCFixup &Fixup, uint64_t Value,
                             const MCRelaxableFragment *Fragment,
                             const MCAsmLayout &Layout) const override {
diff --git a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyAsmBackend.cpp b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyAsmBackend.cpp
index 8ecd7c53621df..d88311197c1ad 100644
--- a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyAsmBackend.cpp
+++ b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyAsmBackend.cpp
@@ -59,11 +59,6 @@ class WebAssemblyAsmBackend final : public MCAsmBackend {
     return false;
   }
 
-  bool mayNeedRelaxation(const MCInst &Inst,
-                         const MCSubtargetInfo &STI) const override {
-    return false;
-  }
-
   bool writeNopData(raw_ostream &OS, uint64_t Count) const override;
 };
 

From 160ff83765ac284f3c7dd7b25d4ef105b9952ac0 Mon Sep 17 00:00:00 2001
From: Saiyedul Islam <Saiyedul.Islam@amd.com>
Date: Mon, 3 Aug 2020 05:29:48 +0000
Subject: [PATCH 0869/1035] [OpenMP][AMDGCN] Support OpenMP offloading for
 AMDGCN architecture - Part 3

Provides AMDGCN and NVPTX specific specialization of getGPUWarpSize,
getGPUThreadID, and getGPUNumThreads methods. Adds tests for AMDGCN
codegen for these methods in generic and simd modes. Also changes the
precondition in InitTempAlloca to be slightly more permissive. Useful for
AMDGCN OpenMP codegen where allocas are created with a cast to an
address space.

Reviewed By: ABataev

Differential Revision: https://reviews.llvm.org/D84260
---
 clang/lib/CodeGen/CGExpr.cpp                  |  9 ++-
 clang/lib/CodeGen/CGOpenMPRuntimeAMDGCN.cpp   | 61 +++++++++++++++++++
 clang/lib/CodeGen/CGOpenMPRuntimeAMDGCN.h     | 43 +++++++++++++
 clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp      | 50 +++++++--------
 clang/lib/CodeGen/CGOpenMPRuntimeGPU.h        | 13 +++-
 clang/lib/CodeGen/CGOpenMPRuntimeNVPTX.cpp    | 17 +++++-
 clang/lib/CodeGen/CGOpenMPRuntimeNVPTX.h      | 10 ++-
 clang/lib/CodeGen/CMakeLists.txt              |  1 +
 clang/lib/CodeGen/CodeGenModule.cpp           |  6 ++
 clang/test/OpenMP/amdgcn_target_codegen.cpp   | 43 +++++++++++++
 .../OpenMP/amdgcn_target_init_temp_alloca.cpp | 24 ++++++++
 11 files changed, 242 insertions(+), 35 deletions(-)
 create mode 100644 clang/lib/CodeGen/CGOpenMPRuntimeAMDGCN.cpp
 create mode 100644 clang/lib/CodeGen/CGOpenMPRuntimeAMDGCN.h
 create mode 100644 clang/test/OpenMP/amdgcn_target_codegen.cpp
 create mode 100644 clang/test/OpenMP/amdgcn_target_init_temp_alloca.cpp

diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index ab29e32929ceb..5d74d91065f56 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -125,8 +125,13 @@ Address CodeGenFunction::CreateDefaultAlignTempAlloca(llvm::Type *Ty,
 }
 
 void CodeGenFunction::InitTempAlloca(Address Var, llvm::Value *Init) {
-  assert(isa<llvm::AllocaInst>(Var.getPointer()));
-  auto *Store = new llvm::StoreInst(Init, Var.getPointer(), /*volatile*/ false,
+  auto *Alloca = Var.getPointer();
+  assert(isa<llvm::AllocaInst>(Alloca) ||
+         (isa<llvm::AddrSpaceCastInst>(Alloca) &&
+          isa<llvm::AllocaInst>(
+              cast<llvm::AddrSpaceCastInst>(Alloca)->getPointerOperand())));
+
+  auto *Store = new llvm::StoreInst(Init, Alloca, /*volatile*/ false,
                                     Var.getAlignment().getAsAlign());
   llvm::BasicBlock *Block = AllocaInsertPt->getParent();
   Block->getInstList().insertAfter(AllocaInsertPt->getIterator(), Store);
diff --git a/clang/lib/CodeGen/CGOpenMPRuntimeAMDGCN.cpp b/clang/lib/CodeGen/CGOpenMPRuntimeAMDGCN.cpp
new file mode 100644
index 0000000000000..ccffdf43549fe
--- /dev/null
+++ b/clang/lib/CodeGen/CGOpenMPRuntimeAMDGCN.cpp
@@ -0,0 +1,61 @@
+//===-- CGOpenMPRuntimeAMDGCN.cpp - Interface to OpenMP AMDGCN Runtimes --===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This provides a class for OpenMP runtime code generation specialized to
+// AMDGCN targets from generalized CGOpenMPRuntimeGPU class.
+//
+//===----------------------------------------------------------------------===//
+
+#include "CGOpenMPRuntimeAMDGCN.h"
+#include "CGOpenMPRuntimeGPU.h"
+#include "CodeGenFunction.h"
+#include "clang/AST/Attr.h"
+#include "clang/AST/DeclOpenMP.h"
+#include "clang/AST/StmtOpenMP.h"
+#include "clang/AST/StmtVisitor.h"
+#include "clang/Basic/Cuda.h"
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/IR/IntrinsicsAMDGPU.h"
+
+using namespace clang;
+using namespace CodeGen;
+using namespace llvm::omp;
+
+CGOpenMPRuntimeAMDGCN::CGOpenMPRuntimeAMDGCN(CodeGenModule &CGM)
+    : CGOpenMPRuntimeGPU(CGM) {
+  if (!CGM.getLangOpts().OpenMPIsDevice)
+    llvm_unreachable("OpenMP AMDGCN can only handle device code.");
+}
+
+llvm::Value *CGOpenMPRuntimeAMDGCN::getGPUWarpSize(CodeGenFunction &CGF) {
+  CGBuilderTy &Bld = CGF.Builder;
+  // return constant compile-time target-specific warp size
+  unsigned WarpSize = CGF.getTarget().getGridValue(llvm::omp::GV_Warp_Size);
+  return Bld.getInt32(WarpSize);
+}
+
+llvm::Value *CGOpenMPRuntimeAMDGCN::getGPUThreadID(CodeGenFunction &CGF) {
+  CGBuilderTy &Bld = CGF.Builder;
+  llvm::Function *F =
+      CGF.CGM.getIntrinsic(llvm::Intrinsic::amdgcn_workitem_id_x);
+  return Bld.CreateCall(F, llvm::None, "nvptx_tid");
+}
+
+llvm::Value *CGOpenMPRuntimeAMDGCN::getGPUNumThreads(CodeGenFunction &CGF) {
+  CGBuilderTy &Bld = CGF.Builder;
+  llvm::Module *M = &CGF.CGM.getModule();
+  const char *LocSize = "__ockl_get_local_size";
+  llvm::Function *F = M->getFunction(LocSize);
+  if (!F) {
+    F = llvm::Function::Create(
+        llvm::FunctionType::get(CGF.Int64Ty, {CGF.Int32Ty}, false),
+        llvm::GlobalVariable::ExternalLinkage, LocSize, &CGF.CGM.getModule());
+  }
+  return Bld.CreateTrunc(
+      Bld.CreateCall(F, {Bld.getInt32(0)}, "nvptx_num_threads"), CGF.Int32Ty);
+}
diff --git a/clang/lib/CodeGen/CGOpenMPRuntimeAMDGCN.h b/clang/lib/CodeGen/CGOpenMPRuntimeAMDGCN.h
new file mode 100644
index 0000000000000..c1421261bfc19
--- /dev/null
+++ b/clang/lib/CodeGen/CGOpenMPRuntimeAMDGCN.h
@@ -0,0 +1,43 @@
+//===--- CGOpenMPRuntimeAMDGCN.h - Interface to OpenMP AMDGCN Runtimes ---===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This provides a class for OpenMP runtime code generation specialized to
+// AMDGCN targets from generalized CGOpenMPRuntimeGPU class.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_LIB_CODEGEN_CGOPENMPRUNTIMEAMDGCN_H
+#define LLVM_CLANG_LIB_CODEGEN_CGOPENMPRUNTIMEAMDGCN_H
+
+#include "CGOpenMPRuntime.h"
+#include "CGOpenMPRuntimeGPU.h"
+#include "CodeGenFunction.h"
+#include "clang/AST/StmtOpenMP.h"
+
+namespace clang {
+namespace CodeGen {
+
+class CGOpenMPRuntimeAMDGCN final : public CGOpenMPRuntimeGPU {
+
+public:
+  explicit CGOpenMPRuntimeAMDGCN(CodeGenModule &CGM);
+
+  /// Get the GPU warp size.
+  llvm::Value *getGPUWarpSize(CodeGenFunction &CGF) override;
+
+  /// Get the id of the current thread on the GPU.
+  llvm::Value *getGPUThreadID(CodeGenFunction &CGF) override;
+
+  /// Get the maximum number of threads in a block of the GPU.
+  llvm::Value *getGPUNumThreads(CodeGenFunction &CGF) override;
+};
+
+} // namespace CodeGen
+} // namespace clang
+
+#endif // LLVM_CLANG_LIB_CODEGEN_CGOPENMPRUNTIMEAMDGCN_H
diff --git a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
index 1cd89c540f478..452eb15eb8d16 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 //
 // This provides a generalized class for OpenMP runtime code generation
-// specialized by GPU target NVPTX.
+// specialized by GPU targets NVPTX and AMDGCN.
 //
 //===----------------------------------------------------------------------===//
 
@@ -621,14 +621,6 @@ class CheckVarsEscapingDeclContext final
 };
 } // anonymous namespace
 
-/// Get the id of the current thread on the GPU.
-static llvm::Value *getNVPTXThreadID(CodeGenFunction &CGF) {
-  return CGF.EmitRuntimeCall(
-      llvm::Intrinsic::getDeclaration(
-          &CGF.CGM.getModule(), llvm::Intrinsic::nvvm_read_ptx_sreg_tid_x),
-      "nvptx_tid");
-}
-
 /// Get the id of the warp in the block.
 /// We assume that the warp size is 32, which is always the case
 /// on the NVPTX device, to generate more efficient code.
@@ -636,7 +628,8 @@ static llvm::Value *getNVPTXWarpID(CodeGenFunction &CGF) {
   CGBuilderTy &Bld = CGF.Builder;
   unsigned LaneIDBits =
       CGF.getTarget().getGridValue(llvm::omp::GV_Warp_Size_Log2);
-  return Bld.CreateAShr(getNVPTXThreadID(CGF), LaneIDBits, "nvptx_warp_id");
+  auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime());
+  return Bld.CreateAShr(RT.getGPUThreadID(CGF), LaneIDBits, "nvptx_warp_id");
 }
 
 /// Get the id of the current lane in the Warp.
@@ -646,18 +639,11 @@ static llvm::Value *getNVPTXLaneID(CodeGenFunction &CGF) {
   CGBuilderTy &Bld = CGF.Builder;
   unsigned LaneIDMask = CGF.getContext().getTargetInfo().getGridValue(
       llvm::omp::GV_Warp_Size_Log2_Mask);
-  return Bld.CreateAnd(getNVPTXThreadID(CGF), Bld.getInt32(LaneIDMask),
+  auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime());
+  return Bld.CreateAnd(RT.getGPUThreadID(CGF), Bld.getInt32(LaneIDMask),
                        "nvptx_lane_id");
 }
 
-/// Get the maximum number of threads in a block of the GPU.
-static llvm::Value *getNVPTXNumThreads(CodeGenFunction &CGF) {
-  return CGF.EmitRuntimeCall(
-      llvm::Intrinsic::getDeclaration(
-          &CGF.CGM.getModule(), llvm::Intrinsic::nvvm_read_ptx_sreg_ntid_x),
-      "nvptx_num_threads");
-}
-
 /// Get the value of the thread_limit clause in the teams directive.
 /// For the 'generic' execution mode, the runtime encodes thread_limit in
 /// the launch parameters, always starting thread_limit+warpSize threads per
@@ -668,9 +654,9 @@ static llvm::Value *getThreadLimit(CodeGenFunction &CGF,
   CGBuilderTy &Bld = CGF.Builder;
   auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime());
   return IsInSPMDExecutionMode
-             ? getNVPTXNumThreads(CGF)
-             : Bld.CreateNUWSub(getNVPTXNumThreads(CGF), RT.getGPUWarpSize(CGF),
-                                "thread_limit");
+             ? RT.getGPUNumThreads(CGF)
+             : Bld.CreateNUWSub(RT.getGPUNumThreads(CGF),
+                                RT.getGPUWarpSize(CGF), "thread_limit");
 }
 
 /// Get the thread id of the OMP master thread.
@@ -682,8 +668,8 @@ static llvm::Value *getThreadLimit(CodeGenFunction &CGF,
 ///      If NumThreads is 1024, master id is 992.
 static llvm::Value *getMasterThreadID(CodeGenFunction &CGF) {
   CGBuilderTy &Bld = CGF.Builder;
-  llvm::Value *NumThreads = getNVPTXNumThreads(CGF);
   auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime());
+  llvm::Value *NumThreads = RT.getGPUNumThreads(CGF);
   // We assume that the warp size is a power of 2.
   llvm::Value *Mask = Bld.CreateNUWSub(RT.getGPUWarpSize(CGF), Bld.getInt32(1));
 
@@ -1235,8 +1221,9 @@ void CGOpenMPRuntimeGPU::emitNonSPMDEntryHeader(CodeGenFunction &CGF,
   llvm::BasicBlock *MasterBB = CGF.createBasicBlock(".master");
   EST.ExitBB = CGF.createBasicBlock(".exit");
 
+  auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime());
   llvm::Value *IsWorker =
-      Bld.CreateICmpULT(getNVPTXThreadID(CGF), getThreadLimit(CGF));
+      Bld.CreateICmpULT(RT.getGPUThreadID(CGF), getThreadLimit(CGF));
   Bld.CreateCondBr(IsWorker, WorkerBB, MasterCheckBB);
 
   CGF.EmitBlock(WorkerBB);
@@ -1245,7 +1232,7 @@ void CGOpenMPRuntimeGPU::emitNonSPMDEntryHeader(CodeGenFunction &CGF,
 
   CGF.EmitBlock(MasterCheckBB);
   llvm::Value *IsMaster =
-      Bld.CreateICmpEQ(getNVPTXThreadID(CGF), getMasterThreadID(CGF));
+      Bld.CreateICmpEQ(RT.getGPUThreadID(CGF), getMasterThreadID(CGF));
   Bld.CreateCondBr(IsMaster, MasterBB, EST.ExitBB);
 
   CGF.EmitBlock(MasterBB);
@@ -2780,14 +2767,16 @@ void CGOpenMPRuntimeGPU::emitCriticalRegion(
   llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.critical.body");
   llvm::BasicBlock *ExitBB = CGF.createBasicBlock("omp.critical.exit");
 
+  auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime());
+
   // Get the mask of active threads in the warp.
   llvm::Value *Mask = CGF.EmitRuntimeCall(
       createNVPTXRuntimeFunction(OMPRTL_NVPTX__kmpc_warp_active_thread_mask));
   // Fetch team-local id of the thread.
-  llvm::Value *ThreadID = getNVPTXThreadID(CGF);
+  llvm::Value *ThreadID = RT.getGPUThreadID(CGF);
 
   // Get the width of the team.
-  llvm::Value *TeamWidth = getNVPTXNumThreads(CGF);
+  llvm::Value *TeamWidth = RT.getGPUNumThreads(CGF);
 
   // Initialize the counter variable for the loop.
   QualType Int32Ty =
@@ -3250,8 +3239,9 @@ static llvm::Value *emitInterWarpCopyFunction(CodeGenModule &CGM,
     CGM.addCompilerUsedGlobal(TransferMedium);
   }
 
+  auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime());
   // Get the CUDA thread id of the current OpenMP thread on the GPU.
-  llvm::Value *ThreadID = getNVPTXThreadID(CGF);
+  llvm::Value *ThreadID = RT.getGPUThreadID(CGF);
   // nvptx_lane_id = nvptx_id % warpsize
   llvm::Value *LaneID = getNVPTXLaneID(CGF);
   // nvptx_warp_id = nvptx_id / warpsize
@@ -4844,9 +4834,11 @@ void CGOpenMPRuntimeGPU::getDefaultDistScheduleAndChunk(
     CodeGenFunction &CGF, const OMPLoopDirective &S,
     OpenMPDistScheduleClauseKind &ScheduleKind,
     llvm::Value *&Chunk) const {
+  auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime());
   if (getExecutionMode() == CGOpenMPRuntimeGPU::EM_SPMD) {
     ScheduleKind = OMPC_DIST_SCHEDULE_static;
-    Chunk = CGF.EmitScalarConversion(getNVPTXNumThreads(CGF),
+    Chunk = CGF.EmitScalarConversion(
+        RT.getGPUNumThreads(CGF),
         CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/0),
         S.getIterationVariable()->getType(), S.getBeginLoc());
     return;
diff --git a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.h b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.h
index 316333072c5bc..7267511ca672a 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.h
+++ b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.h
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 //
 // This provides a generalized class for OpenMP runtime code generation
-// specialized by GPU target NVPTX.
+// specialized by GPU targets NVPTX and AMDGCN.
 //
 //===----------------------------------------------------------------------===//
 
@@ -199,9 +199,18 @@ class CGOpenMPRuntimeGPU : public CGOpenMPRuntime {
   void clear() override;
 
   /// Declare generalized virtual functions which need to be defined
-  /// by all specializations of OpenMPGPURuntime Targets.
+  /// by all specializations of OpenMPGPURuntime Targets like AMDGCN
+  /// and NVPTX.
+
+  /// Get the GPU warp size.
   virtual llvm::Value *getGPUWarpSize(CodeGenFunction &CGF) = 0;
 
+  /// Get the id of the current thread on the GPU.
+  virtual llvm::Value *getGPUThreadID(CodeGenFunction &CGF) = 0;
+
+  /// Get the maximum number of threads in a block of the GPU.
+  virtual llvm::Value *getGPUNumThreads(CodeGenFunction &CGF) = 0;
+
   /// Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32
   /// global_tid, int proc_bind) to generate code for 'proc_bind' clause.
   virtual void emitProcBindClause(CodeGenFunction &CGF,
diff --git a/clang/lib/CodeGen/CGOpenMPRuntimeNVPTX.cpp b/clang/lib/CodeGen/CGOpenMPRuntimeNVPTX.cpp
index 5fefc95ee4130..1688d07b90b6e 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntimeNVPTX.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntimeNVPTX.cpp
@@ -32,10 +32,25 @@ CGOpenMPRuntimeNVPTX::CGOpenMPRuntimeNVPTX(CodeGenModule &CGM)
     llvm_unreachable("OpenMP NVPTX can only handle device code.");
 }
 
-/// Get the GPU warp size.
 llvm::Value *CGOpenMPRuntimeNVPTX::getGPUWarpSize(CodeGenFunction &CGF) {
   return CGF.EmitRuntimeCall(
       llvm::Intrinsic::getDeclaration(
           &CGF.CGM.getModule(), llvm::Intrinsic::nvvm_read_ptx_sreg_warpsize),
       "nvptx_warp_size");
 }
+
+llvm::Value *CGOpenMPRuntimeNVPTX::getGPUThreadID(CodeGenFunction &CGF) {
+  CGBuilderTy &Bld = CGF.Builder;
+  llvm::Function *F;
+  F = llvm::Intrinsic::getDeclaration(
+      &CGF.CGM.getModule(), llvm::Intrinsic::nvvm_read_ptx_sreg_tid_x);
+  return Bld.CreateCall(F, llvm::None, "nvptx_tid");
+}
+
+llvm::Value *CGOpenMPRuntimeNVPTX::getGPUNumThreads(CodeGenFunction &CGF) {
+  CGBuilderTy &Bld = CGF.Builder;
+  llvm::Function *F;
+  F = llvm::Intrinsic::getDeclaration(
+      &CGF.CGM.getModule(), llvm::Intrinsic::nvvm_read_ptx_sreg_ntid_x);
+  return Bld.CreateCall(F, llvm::None, "nvptx_num_threads");
+}
diff --git a/clang/lib/CodeGen/CGOpenMPRuntimeNVPTX.h b/clang/lib/CodeGen/CGOpenMPRuntimeNVPTX.h
index 6dab79e6e20ae..5f16029592665 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntimeNVPTX.h
+++ b/clang/lib/CodeGen/CGOpenMPRuntimeNVPTX.h
@@ -22,11 +22,19 @@
 namespace clang {
 namespace CodeGen {
 
-class CGOpenMPRuntimeNVPTX : public CGOpenMPRuntimeGPU {
+class CGOpenMPRuntimeNVPTX final : public CGOpenMPRuntimeGPU {
 
 public:
   explicit CGOpenMPRuntimeNVPTX(CodeGenModule &CGM);
+
+  /// Get the GPU warp size.
   llvm::Value *getGPUWarpSize(CodeGenFunction &CGF) override;
+
+  /// Get the id of the current thread on the GPU.
+  llvm::Value *getGPUThreadID(CodeGenFunction &CGF) override;
+
+  /// Get the maximum number of threads in a block of the GPU.
+  llvm::Value *getGPUNumThreads(CodeGenFunction &CGF) override;
 };
 
 } // CodeGen namespace.
diff --git a/clang/lib/CodeGen/CMakeLists.txt b/clang/lib/CodeGen/CMakeLists.txt
index 88647a2007fb0..f47ecd9bf8465 100644
--- a/clang/lib/CodeGen/CMakeLists.txt
+++ b/clang/lib/CodeGen/CMakeLists.txt
@@ -58,6 +58,7 @@ add_clang_library(clangCodeGen
   CGObjCRuntime.cpp
   CGOpenCLRuntime.cpp
   CGOpenMPRuntime.cpp
+  CGOpenMPRuntimeAMDGCN.cpp
   CGOpenMPRuntimeGPU.cpp
   CGOpenMPRuntimeNVPTX.cpp
   CGRecordLayoutBuilder.cpp
diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp
index 48a1dddfb3315..f3712ea1f541d 100644
--- a/clang/lib/CodeGen/CodeGenModule.cpp
+++ b/clang/lib/CodeGen/CodeGenModule.cpp
@@ -19,6 +19,7 @@
 #include "CGObjCRuntime.h"
 #include "CGOpenCLRuntime.h"
 #include "CGOpenMPRuntime.h"
+#include "CGOpenMPRuntimeAMDGCN.h"
 #include "CGOpenMPRuntimeNVPTX.h"
 #include "CodeGenFunction.h"
 #include "CodeGenPGO.h"
@@ -215,6 +216,11 @@ void CodeGenModule::createOpenMPRuntime() {
            "OpenMP NVPTX is only prepared to deal with device code.");
     OpenMPRuntime.reset(new CGOpenMPRuntimeNVPTX(*this));
     break;
+  case llvm::Triple::amdgcn:
+    assert(getLangOpts().OpenMPIsDevice &&
+           "OpenMP AMDGCN is only prepared to deal with device code.");
+    OpenMPRuntime.reset(new CGOpenMPRuntimeAMDGCN(*this));
+    break;
   default:
     if (LangOpts.OpenMPSimd)
       OpenMPRuntime.reset(new CGOpenMPSIMDRuntime(*this));
diff --git a/clang/test/OpenMP/amdgcn_target_codegen.cpp b/clang/test/OpenMP/amdgcn_target_codegen.cpp
new file mode 100644
index 0000000000000..0b6f2d40ffe87
--- /dev/null
+++ b/clang/test/OpenMP/amdgcn_target_codegen.cpp
@@ -0,0 +1,43 @@
+// REQUIRES: amdgpu-registered-target
+
+// RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple x86_64-unknown-unknown -fopenmp-targets=amdgcn-amd-amdhsa -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple amdgcn-amd-amdhsa -fopenmp-targets=amdgcn-amd-amdhsa -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s
+// expected-no-diagnostics
+#ifndef HEADER
+#define HEADER
+
+#define N 1000
+
+int test_amdgcn_target_tid_threads() {
+// CHECK-LABEL: define weak void @{{.*}}test_amdgcn_target_tid_threads
+
+  int arr[N];
+
+// CHECK: [[NUM_THREADS:%.+]] = call i64 @__ockl_get_local_size(i32 0)
+// CHECK-NEXT: [[VAR:%.+]] = trunc i64 [[NUM_THREADS]] to i32
+// CHECK-NEXT: sub nuw i32 [[VAR]], 64
+// CHECK: call i32 @llvm.amdgcn.workitem.id.x()
+#pragma omp target
+  for (int i = 0; i < N; i++) {
+    arr[i] = 1;
+  }
+
+  return arr[0];
+}
+
+int test_amdgcn_target_tid_threads_simd() {
+// CHECK-LABEL: define weak void @{{.*}}test_amdgcn_target_tid_threads_simd
+
+  int arr[N];
+
+// CHECK: [[NUM_THREADS:%.+]] = call i64 @__ockl_get_local_size(i32 0)
+// CHECK-NEXT: [[VAR:%.+]] = trunc i64 [[NUM_THREADS]] to i32
+// CHECK-NEXT: call void @__kmpc_spmd_kernel_init(i32 [[VAR]], i16 0, i16 0)
+#pragma omp target simd
+  for (int i = 0; i < N; i++) {
+    arr[i] = 1;
+  }
+  return arr[0];
+}
+
+#endif
diff --git a/clang/test/OpenMP/amdgcn_target_init_temp_alloca.cpp b/clang/test/OpenMP/amdgcn_target_init_temp_alloca.cpp
new file mode 100644
index 0000000000000..4ed953a9ebf7b
--- /dev/null
+++ b/clang/test/OpenMP/amdgcn_target_init_temp_alloca.cpp
@@ -0,0 +1,24 @@
+// REQUIRES: amdgpu-registered-target
+
+// RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple x86_64-unknown-unknown -fopenmp-targets=amdgcn-amd-amdhsa -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple amdgcn-amd-amdhsa -fopenmp-targets=amdgcn-amd-amdhsa -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s
+// expected-no-diagnostics
+
+#define N 100
+
+int test_amdgcn_target_temp_alloca() {
+  // CHECK-LABEL: test_amdgcn_target_temp_alloca
+
+  int arr[N];
+
+  // CHECK:      [[VAR_ADDR:%.+]] = alloca [100 x i32]*, align 8, addrspace(5)
+  // CHECK-NEXT: [[VAR_ADDR_CAST:%.+]] = addrspacecast [100 x i32]* addrspace(5)* [[VAR_ADDR]] to [100 x i32]**
+  // CHECK:  store [100 x i32]* [[VAR:%.+]], [100 x i32]** [[VAR_ADDR_CAST]], align 8
+
+#pragma omp target
+  for (int i = 0; i < N; i++) {
+    arr[i] = 1;
+  }
+
+  return arr[0];
+}

From 5a4cd55e5d1452db7043ef9e9f1211172a6a10e1 Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
Date: Mon, 3 Aug 2020 05:55:14 +0000
Subject: [PATCH 0870/1035] [gn build] Port 160ff83765a

---
 llvm/utils/gn/secondary/clang/lib/CodeGen/BUILD.gn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/utils/gn/secondary/clang/lib/CodeGen/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/CodeGen/BUILD.gn
index c290312bd67ea..b7ecb646429e3 100644
--- a/llvm/utils/gn/secondary/clang/lib/CodeGen/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/lib/CodeGen/BUILD.gn
@@ -63,6 +63,7 @@ static_library("CodeGen") {
     "CGObjCRuntime.cpp",
     "CGOpenCLRuntime.cpp",
     "CGOpenMPRuntime.cpp",
+    "CGOpenMPRuntimeAMDGCN.cpp",
     "CGOpenMPRuntimeGPU.cpp",
     "CGOpenMPRuntimeNVPTX.cpp",
     "CGRecordLayoutBuilder.cpp",

From c41a18cf61790fc898dcda1055c3efbf442c14c0 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Sun, 2 Aug 2020 23:05:50 -0700
Subject: [PATCH 0871/1035] [CMake] Default ENABLE_X86_RELAX_RELOCATIONS to ON

This makes clang default to -Wa,-mrelax-relocations=yes, which enables
R_386_GOT32X (GNU as enables it regardless of -mrelax-relocations=) and
R_X86_64_[REX_]GOTPCRELX in MC. The produced object files require GNU ld>=2.26
to link. binutils 2.26 is considered a very old release today.
---
 clang/CMakeLists.txt                                        | 2 +-
 llvm/utils/gn/secondary/clang/include/clang/Config/BUILD.gn | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/clang/CMakeLists.txt b/clang/CMakeLists.txt
index c487e506cae11..7baf93cbf7924 100644
--- a/clang/CMakeLists.txt
+++ b/clang/CMakeLists.txt
@@ -259,7 +259,7 @@ set(DEFAULT_SYSROOT "" CACHE STRING
 
 set(ENABLE_LINKER_BUILD_ID OFF CACHE BOOL "pass --build-id to ld")
 
-set(ENABLE_X86_RELAX_RELOCATIONS OFF CACHE BOOL
+set(ENABLE_X86_RELAX_RELOCATIONS ON CACHE BOOL
     "enable x86 relax relocations by default")
 
 set(ENABLE_EXPERIMENTAL_NEW_PASS_MANAGER FALSE CACHE BOOL
diff --git a/llvm/utils/gn/secondary/clang/include/clang/Config/BUILD.gn b/llvm/utils/gn/secondary/clang/include/clang/Config/BUILD.gn
index 7fbfb46a41c54..49fc477d35f67 100644
--- a/llvm/utils/gn/secondary/clang/include/clang/Config/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/include/clang/Config/BUILD.gn
@@ -33,7 +33,7 @@ write_cmake_config("Config") {
     "GCC_INSTALL_PREFIX=",
     "BACKEND_PACKAGE_STRING=LLVM ${llvm_version}git",
     "ENABLE_LINKER_BUILD_ID=",
-    "ENABLE_X86_RELAX_RELOCATIONS=",
+    "ENABLE_X86_RELAX_RELOCATIONS=1",
     "ENABLE_EXPERIMENTAL_NEW_PASS_MANAGER=",
     "CLANG_ENABLE_OBJC_REWRITER=1",  # FIXME: flag?
     "CLANG_SYSTEMZ_DEFAULT_ARCH=z10",

From 91f6a5f7854a542611ed76442acb1ec375a9feb2 Mon Sep 17 00:00:00 2001
From: George Mitenkov <georgemitenk0v@gmail.com>
Date: Mon, 3 Aug 2020 09:31:08 +0300
Subject: [PATCH 0872/1035] [MLIR][SPIRV] Control attributes support for loop
 and selection

This patch handles loopControl and selectionControl in parsing and
printing. In order to reuse the functionality, and avoid handling cases when
`{` of the region is parsed as a dictionary attribute, `control` keyword was
introduced.`None` is a default control attribute. This functionality can be
later extended to `spv.func`.
Also, loopControl and selectionControl can now be (de)serialized.

Reviewed By: antiagainst

Differential Revision: https://reviews.llvm.org/D84175
---
 mlir/lib/Dialect/SPIRV/SPIRVOps.cpp           | 42 +++++++---
 .../SPIRV/Serialization/Deserializer.cpp      | 78 +++++++++----------
 .../SPIRV/Serialization/Serializer.cpp        |  6 +-
 .../Dialect/SPIRV/Serialization/loop.mlir     |  8 +-
 .../SPIRV/Serialization/selection.mlir        |  4 +-
 mlir/test/Dialect/SPIRV/control-flow-ops.mlir | 20 +++++
 6 files changed, 97 insertions(+), 61 deletions(-)

diff --git a/mlir/lib/Dialect/SPIRV/SPIRVOps.cpp b/mlir/lib/Dialect/SPIRV/SPIRVOps.cpp
index bac65a02f63de..b7d36f4a94875 100644
--- a/mlir/lib/Dialect/SPIRV/SPIRVOps.cpp
+++ b/mlir/lib/Dialect/SPIRV/SPIRVOps.cpp
@@ -36,6 +36,7 @@ static constexpr const char kSourceAlignmentAttrName[] = "source_alignment";
 static constexpr const char kBranchWeightAttrName[] = "branch_weights";
 static constexpr const char kCallee[] = "callee";
 static constexpr const char kClusterSize[] = "cluster_size";
+static constexpr const char kControl[] = "control";
 static constexpr const char kDefaultValueAttrName[] = "default_value";
 static constexpr const char kExecutionScopeAttrName[] = "execution_scope";
 static constexpr const char kEqualSemanticsAttrName[] = "equal_semantics";
@@ -161,6 +162,25 @@ parseEnumKeywordAttr(EnumClass &value, OpAsmParser &parser,
   return success();
 }
 
+/// Parses Function, Selection and Loop control attributes. If no control is
+/// specified, "None" is used as a default.
+template <typename EnumClass>
+static ParseResult
+parseControlAttribute(OpAsmParser &parser, OperationState &state,
+                      StringRef attrName = spirv::attributeName<EnumClass>()) {
+  if (succeeded(parser.parseOptionalKeyword(kControl))) {
+    EnumClass control;
+    if (parser.parseLParen() || parseEnumKeywordAttr(control, parser, state) ||
+        parser.parseRParen())
+      return failure();
+    return success();
+  }
+  // Set control to "None" otherwise.
+  Builder builder = parser.getBuilder();
+  state.addAttribute(attrName, builder.getI32IntegerAttr(0));
+  return success();
+}
+
 /// Parses optional memory access attributes attached to a memory access
 /// operand/pointer. Specifically, parses the following syntax:
 ///     (`[` memory-access `]`)?
@@ -2082,12 +2102,8 @@ void spirv::LoopOp::build(OpBuilder &builder, OperationState &state) {
 }
 
 static ParseResult parseLoopOp(OpAsmParser &parser, OperationState &state) {
-  // TODO: support loop control properly
-  Builder builder = parser.getBuilder();
-  state.addAttribute("loop_control",
-                     builder.getI32IntegerAttr(
-                         static_cast<uint32_t>(spirv::LoopControl::None)));
-
+  if (parseControlAttribute<spirv::LoopControl>(parser, state))
+    return failure();
   return parser.parseRegion(*state.addRegion(), /*arguments=*/{},
                             /*argTypes=*/{});
 }
@@ -2096,6 +2112,9 @@ static void print(spirv::LoopOp loopOp, OpAsmPrinter &printer) {
   auto *op = loopOp.getOperation();
 
   printer << spirv::LoopOp::getOperationName();
+  auto control = loopOp.loop_control();
+  if (control != spirv::LoopControl::None)
+    printer << " control(" << spirv::stringifyLoopControl(control) << ")";
   printer.printRegion(op->getRegion(0), /*printEntryBlockArgs=*/false,
                       /*printBlockTerminators=*/true);
 }
@@ -2445,12 +2464,8 @@ static LogicalResult verify(spirv::SelectOp op) {
 
 static ParseResult parseSelectionOp(OpAsmParser &parser,
                                     OperationState &state) {
-  // TODO: support selection control properly
-  Builder builder = parser.getBuilder();
-  state.addAttribute("selection_control",
-                     builder.getI32IntegerAttr(
-                         static_cast<uint32_t>(spirv::SelectionControl::None)));
-
+  if (parseControlAttribute<spirv::SelectionControl>(parser, state))
+    return failure();
   return parser.parseRegion(*state.addRegion(), /*arguments=*/{},
                             /*argTypes=*/{});
 }
@@ -2459,6 +2474,9 @@ static void print(spirv::SelectionOp selectionOp, OpAsmPrinter &printer) {
   auto *op = selectionOp.getOperation();
 
   printer << spirv::SelectionOp::getOperationName();
+  auto control = selectionOp.selection_control();
+  if (control != spirv::SelectionControl::None)
+    printer << " control(" << spirv::stringifySelectionControl(control) << ")";
   printer.printRegion(op->getRegion(0), /*printEntryBlockArgs=*/false,
                       /*printBlockTerminators=*/true);
 }
diff --git a/mlir/lib/Dialect/SPIRV/Serialization/Deserializer.cpp b/mlir/lib/Dialect/SPIRV/Serialization/Deserializer.cpp
index 4ba3f16feef07..eaa8f4d94833d 100644
--- a/mlir/lib/Dialect/SPIRV/Serialization/Deserializer.cpp
+++ b/mlir/lib/Dialect/SPIRV/Serialization/Deserializer.cpp
@@ -64,11 +64,14 @@ struct BlockMergeInfo {
   Block *mergeBlock;
   Block *continueBlock; // nullptr for spv.selection
   Location loc;
-
-  BlockMergeInfo(Location location)
-      : mergeBlock(nullptr), continueBlock(nullptr), loc(location) {}
-  BlockMergeInfo(Location location, Block *m, Block *c = nullptr)
-      : mergeBlock(m), continueBlock(c), loc(location) {}
+  uint32_t control;
+
+  BlockMergeInfo(Location location, uint32_t control)
+      : mergeBlock(nullptr), continueBlock(nullptr), loc(location),
+        control(control) {}
+  BlockMergeInfo(Location location, uint32_t control, Block *m,
+                 Block *c = nullptr)
+      : mergeBlock(m), continueBlock(c), loc(location), control(control) {}
 };
 
 /// A struct for containing OpLine instruction information.
@@ -1681,16 +1684,12 @@ LogicalResult Deserializer::processSelectionMerge(ArrayRef<uint32_t> operands) {
         "OpSelectionMerge must specify merge target and selection control");
   }
 
-  if (static_cast<uint32_t>(spirv::SelectionControl::None) != operands[1]) {
-    return emitError(unknownLoc,
-                     "unimplmented OpSelectionMerge selection control: ")
-           << operands[2];
-  }
-
   auto *mergeBlock = getOrCreateBlock(operands[0]);
   auto loc = createFileLineColLoc(opBuilder);
+  auto selectionControl = operands[1];
 
-  if (!blockMergeInfo.try_emplace(curBlock, loc, mergeBlock).second) {
+  if (!blockMergeInfo.try_emplace(curBlock, loc, selectionControl, mergeBlock)
+           .second) {
     return emitError(
         unknownLoc,
         "a block cannot have more than one OpSelectionMerge instruction");
@@ -1709,16 +1708,13 @@ LogicalResult Deserializer::processLoopMerge(ArrayRef<uint32_t> operands) {
                                  "continue target and loop control");
   }
 
-  if (static_cast<uint32_t>(spirv::LoopControl::None) != operands[2]) {
-    return emitError(unknownLoc, "unimplmented OpLoopMerge loop control: ")
-           << operands[2];
-  }
-
   auto *mergeBlock = getOrCreateBlock(operands[0]);
   auto *continueBlock = getOrCreateBlock(operands[1]);
   auto loc = createFileLineColLoc(opBuilder);
+  uint32_t loopControl = operands[2];
 
-  if (!blockMergeInfo.try_emplace(curBlock, loc, mergeBlock, continueBlock)
+  if (!blockMergeInfo
+           .try_emplace(curBlock, loc, loopControl, mergeBlock, continueBlock)
            .second) {
     return emitError(
         unknownLoc,
@@ -1771,25 +1767,27 @@ class ControlFlowStructurizer {
   /// the `headerBlock` will be redirected to the `mergeBlock`.
   /// This method will also update `mergeInfo` by remapping all blocks inside to
   /// the newly cloned ones inside structured control flow op's regions.
-  static LogicalResult structurize(Location loc, BlockMergeInfoMap &mergeInfo,
+  static LogicalResult structurize(Location loc, uint32_t control,
+                                   BlockMergeInfoMap &mergeInfo,
                                    Block *headerBlock, Block *mergeBlock,
                                    Block *continueBlock) {
-    return ControlFlowStructurizer(loc, mergeInfo, headerBlock, mergeBlock,
-                                   continueBlock)
+    return ControlFlowStructurizer(loc, control, mergeInfo, headerBlock,
+                                   mergeBlock, continueBlock)
         .structurizeImpl();
   }
 
 private:
-  ControlFlowStructurizer(Location loc, BlockMergeInfoMap &mergeInfo,
-                          Block *header, Block *merge, Block *cont)
-      : location(loc), blockMergeInfo(mergeInfo), headerBlock(header),
-        mergeBlock(merge), continueBlock(cont) {}
+  ControlFlowStructurizer(Location loc, uint32_t control,
+                          BlockMergeInfoMap &mergeInfo, Block *header,
+                          Block *merge, Block *cont)
+      : location(loc), control(control), blockMergeInfo(mergeInfo),
+        headerBlock(header), mergeBlock(merge), continueBlock(cont) {}
 
   /// Creates a new spv.selection op at the beginning of the `mergeBlock`.
-  spirv::SelectionOp createSelectionOp();
+  spirv::SelectionOp createSelectionOp(uint32_t selectionControl);
 
   /// Creates a new spv.loop op at the beginning of the `mergeBlock`.
-  spirv::LoopOp createLoopOp();
+  spirv::LoopOp createLoopOp(uint32_t loopControl);
 
   /// Collects all blocks reachable from `headerBlock` except `mergeBlock`.
   void collectBlocksInConstruct();
@@ -1797,6 +1795,7 @@ class ControlFlowStructurizer {
   LogicalResult structurizeImpl();
 
   Location location;
+  uint32_t control;
 
   BlockMergeInfoMap &blockMergeInfo;
 
@@ -1808,26 +1807,26 @@ class ControlFlowStructurizer {
 };
 } // namespace
 
-spirv::SelectionOp ControlFlowStructurizer::createSelectionOp() {
+spirv::SelectionOp
+ControlFlowStructurizer::createSelectionOp(uint32_t selectionControl) {
   // Create a builder and set the insertion point to the beginning of the
   // merge block so that the newly created SelectionOp will be inserted there.
   OpBuilder builder(&mergeBlock->front());
 
-  auto control = builder.getI32IntegerAttr(
-      static_cast<uint32_t>(spirv::SelectionControl::None));
+  auto control = builder.getI32IntegerAttr(selectionControl);
   auto selectionOp = builder.create<spirv::SelectionOp>(location, control);
   selectionOp.addMergeBlock();
 
   return selectionOp;
 }
 
-spirv::LoopOp ControlFlowStructurizer::createLoopOp() {
+spirv::LoopOp ControlFlowStructurizer::createLoopOp(uint32_t loopControl) {
   // Create a builder and set the insertion point to the beginning of the
   // merge block so that the newly created LoopOp will be inserted there.
   OpBuilder builder(&mergeBlock->front());
 
-  // TODO: handle loop control properly
-  auto loopOp = builder.create<spirv::LoopOp>(location);
+  auto control = builder.getI32IntegerAttr(loopControl);
+  auto loopOp = builder.create<spirv::LoopOp>(location, control);
   loopOp.addEntryAndMergeBlock();
 
   return loopOp;
@@ -1852,10 +1851,10 @@ LogicalResult ControlFlowStructurizer::structurizeImpl() {
   Operation *op = nullptr;
   bool isLoop = continueBlock != nullptr;
   if (isLoop) {
-    if (auto loopOp = createLoopOp())
+    if (auto loopOp = createLoopOp(control))
       op = loopOp.getOperation();
   } else {
-    if (auto selectionOp = createSelectionOp())
+    if (auto selectionOp = createSelectionOp(control))
       op = selectionOp.getOperation();
   }
   if (!op)
@@ -1992,7 +1991,8 @@ LogicalResult ControlFlowStructurizer::structurizeImpl() {
       // The iterator should be erased before adding a new entry into
       // blockMergeInfo to avoid iterator invalidation.
       blockMergeInfo.erase(it);
-      blockMergeInfo.try_emplace(newHeader, loc, newMerge, newContinue);
+      blockMergeInfo.try_emplace(newHeader, loc, it->second.control, newMerge,
+                                 newContinue);
     }
 
     // The structured selection/loop's entry block does not have arguments.
@@ -2096,9 +2096,9 @@ LogicalResult Deserializer::structurizeControlFlow() {
     // Erase this case before calling into structurizer, who will update
     // blockMergeInfo.
     blockMergeInfo.erase(blockMergeInfo.begin());
-    if (failed(ControlFlowStructurizer::structurize(mergeInfo.loc,
-                                                    blockMergeInfo, headerBlock,
-                                                    mergeBlock, continueBlock)))
+    if (failed(ControlFlowStructurizer::structurize(
+            mergeInfo.loc, mergeInfo.control, blockMergeInfo, headerBlock,
+            mergeBlock, continueBlock)))
       return failure();
   }
 
diff --git a/mlir/lib/Dialect/SPIRV/Serialization/Serializer.cpp b/mlir/lib/Dialect/SPIRV/Serialization/Serializer.cpp
index 859ea556f39f2..223adf47ab2e0 100644
--- a/mlir/lib/Dialect/SPIRV/Serialization/Serializer.cpp
+++ b/mlir/lib/Dialect/SPIRV/Serialization/Serializer.cpp
@@ -1573,10 +1573,9 @@ LogicalResult Serializer::processSelectionOp(spirv::SelectionOp selectionOp) {
   auto emitSelectionMerge = [&]() {
     emitDebugLine(functionBody, loc);
     lastProcessedWasMergeInst = true;
-    // TODO: properly support selection control here
     encodeInstructionInto(
         functionBody, spirv::Opcode::OpSelectionMerge,
-        {mergeID, static_cast<uint32_t>(spirv::SelectionControl::None)});
+        {mergeID, static_cast<uint32_t>(selectionOp.selection_control())});
   };
   // For structured selection, we cannot have blocks in the selection construct
   // branching to the selection header block. Entering the selection (and
@@ -1636,10 +1635,9 @@ LogicalResult Serializer::processLoopOp(spirv::LoopOp loopOp) {
   auto emitLoopMerge = [&]() {
     emitDebugLine(functionBody, loc);
     lastProcessedWasMergeInst = true;
-    // TODO: properly support loop control here
     encodeInstructionInto(
         functionBody, spirv::Opcode::OpLoopMerge,
-        {mergeID, continueID, static_cast<uint32_t>(spirv::LoopControl::None)});
+        {mergeID, continueID, static_cast<uint32_t>(loopOp.loop_control())});
   };
   if (failed(processBlock(headerBlock, /*omitLabel=*/false, emitLoopMerge)))
     return failure();
diff --git a/mlir/test/Dialect/SPIRV/Serialization/loop.mlir b/mlir/test/Dialect/SPIRV/Serialization/loop.mlir
index d6e2090f02bbb..8f0b35ef6fc82 100644
--- a/mlir/test/Dialect/SPIRV/Serialization/loop.mlir
+++ b/mlir/test/Dialect/SPIRV/Serialization/loop.mlir
@@ -119,8 +119,8 @@ spv.module Logical GLSL450 requires #spv.vce<v1.0, [Shader], []> {
 
 // CHECK:        spv.Branch ^bb1
 // CHECK-NEXT: ^bb1:
-// CHECK-NEXT:   spv.loop
-    spv.loop {
+// CHECK-NEXT:   spv.loop control(Unroll)
+    spv.loop control(Unroll) {
 // CHECK-NEXT:     spv.Branch ^bb1
       spv.Branch ^header
 
@@ -140,8 +140,8 @@ spv.module Logical GLSL450 requires #spv.vce<v1.0, [Shader], []> {
       spv.Store "Function" %jvar, %zero : i32
 // CHECK-NEXT:     spv.Branch ^bb3
 // CHECK-NEXT:   ^bb3:
-// CHECK-NEXT:     spv.loop
-      spv.loop {
+// CHECK-NEXT:     spv.loop control(DontUnroll)
+      spv.loop control(DontUnroll) {
 // CHECK-NEXT:       spv.Branch ^bb1
         spv.Branch ^header
 
diff --git a/mlir/test/Dialect/SPIRV/Serialization/selection.mlir b/mlir/test/Dialect/SPIRV/Serialization/selection.mlir
index e391bae5b4861..9e6ff5698f8a6 100644
--- a/mlir/test/Dialect/SPIRV/Serialization/selection.mlir
+++ b/mlir/test/Dialect/SPIRV/Serialization/selection.mlir
@@ -11,10 +11,10 @@ spv.module Logical GLSL450 requires #spv.vce<v1.0, [Shader], []> {
     %two = spv.constant 2: i32
     %var = spv.Variable init(%zero) : !spv.ptr<i32, Function>
 
-// CHECK-NEXT:   spv.selection {
+// CHECK-NEXT:   spv.selection control(Flatten)
 // CHECK-NEXT:     spv.constant 0
 // CHECK-NEXT:     spv.Variable
-    spv.selection {
+    spv.selection control(Flatten) {
 // CHECK-NEXT: spv.BranchConditional %{{.*}} [5, 10], ^bb1, ^bb2
       spv.BranchConditional %cond [5, 10], ^then, ^else
 
diff --git a/mlir/test/Dialect/SPIRV/control-flow-ops.mlir b/mlir/test/Dialect/SPIRV/control-flow-ops.mlir
index 97ee02d45f24b..267c45b080fa2 100644
--- a/mlir/test/Dialect/SPIRV/control-flow-ops.mlir
+++ b/mlir/test/Dialect/SPIRV/control-flow-ops.mlir
@@ -317,6 +317,16 @@ func @empty_region() -> () {
 
 // -----
 
+// CHECK-LABEL: @loop_with_control
+func @loop_with_control() -> () {
+  // CHECK: spv.loop control(Unroll)
+  spv.loop control(Unroll) {
+  }
+  return
+}
+
+// -----
+
 func @wrong_merge_block() -> () {
   // expected-error @+1 {{last block must be the merge block with only one 'spv._merge' op}}
   spv.loop {
@@ -718,6 +728,16 @@ func @empty_region() -> () {
 
 // -----
 
+// CHECK-LABEL: @selection_with_control
+func @selection_with_control() -> () {
+  // CHECK: spv.selection control(Flatten)
+  spv.selection control(Flatten) {
+  }
+  return
+}
+
+// -----
+
 func @wrong_merge_block() -> () {
   // expected-error @+1 {{last block must be the merge block with only one 'spv._merge' op}}
   spv.selection {

From 4fdc4d892b988bb9f2e06c3440971d28d6361722 Mon Sep 17 00:00:00 2001
From: Djordje Todorovic <djordje.todorovic@syrmia.com>
Date: Mon, 3 Aug 2020 09:01:37 +0200
Subject: [PATCH 0873/1035] [NFC] [MIR] Document the reg state flags

This patch adds documentation for the RegState enumeration.

Differential Revision: https://reviews.llvm.org/D84634
---
 .../llvm/CodeGen/MachineInstrBuilder.h        | 38 ++++++++++++-------
 1 file changed, 24 insertions(+), 14 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/MachineInstrBuilder.h b/llvm/include/llvm/CodeGen/MachineInstrBuilder.h
index cabb9f1c97c96..b31e9cdb0e903 100644
--- a/llvm/include/llvm/CodeGen/MachineInstrBuilder.h
+++ b/llvm/include/llvm/CodeGen/MachineInstrBuilder.h
@@ -40,20 +40,30 @@ class MDNode;
 
 namespace RegState {
 
-  enum {
-    Define         = 0x2,
-    Implicit       = 0x4,
-    Kill           = 0x8,
-    Dead           = 0x10,
-    Undef          = 0x20,
-    EarlyClobber   = 0x40,
-    Debug          = 0x80,
-    InternalRead   = 0x100,
-    Renamable      = 0x200,
-    DefineNoRead   = Define | Undef,
-    ImplicitDefine = Implicit | Define,
-    ImplicitKill   = Implicit | Kill
-  };
+enum {
+  /// Register definition.
+  Define = 0x2,
+  /// Not emitted register (e.g. carry, or temporary result).
+  Implicit = 0x4,
+  /// The last use of a register.
+  Kill = 0x8,
+  /// Unused definition.
+  Dead = 0x10,
+  /// Value of the register doesn't matter.
+  Undef = 0x20,
+  /// Register definition happens before uses.
+  EarlyClobber = 0x40,
+  /// Register 'use' is for debugging purpose.
+  Debug = 0x80,
+  /// Register reads a value that is defined inside the same instruction or
+  /// bundle.
+  InternalRead = 0x100,
+  /// Register that may be renamed.
+  Renamable = 0x200,
+  DefineNoRead = Define | Undef,
+  ImplicitDefine = Implicit | Define,
+  ImplicitKill = Implicit | Kill
+};
 
 } // end namespace RegState
 

From 9c3f6fb68807c8100797b001c0621ae0c9a6d1fc Mon Sep 17 00:00:00 2001
From: Daniel Kiss <daniel.kiss@arm.com>
Date: Mon, 3 Aug 2020 09:40:35 +0200
Subject: [PATCH 0874/1035] [libunwind] Make the test depend on the libunwind
 explicitly.

Before this patch the `ninja check-unwind` won't rebuild the unwind library.

Reviewed By: jroelofs

Differential Revision: https://reviews.llvm.org/D85004
---
 libunwind/test/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libunwind/test/CMakeLists.txt b/libunwind/test/CMakeLists.txt
index 794a59f58f84a..2b945e6eff762 100644
--- a/libunwind/test/CMakeLists.txt
+++ b/libunwind/test/CMakeLists.txt
@@ -32,4 +32,4 @@ configure_lit_site_cfg(
 
 add_lit_testsuite(check-unwind "Running libunwind tests"
   ${CMAKE_CURRENT_BINARY_DIR}
-  DEPENDS ${LIBUNWIND_TEST_DEPS})
+  DEPENDS unwind ${LIBUNWIND_TEST_DEPS})

From 4e10a18972a4569fe6b13e60becb44514b35c52e Mon Sep 17 00:00:00 2001
From: Igor Kudrin <ikudrin@accesssoftek.com>
Date: Mon, 3 Aug 2020 15:03:37 +0700
Subject: [PATCH 0875/1035] [DebugInfo] Make DIELocList::SizeOf() more
 explicit. NFCI.

DIELocList is used with a limited number of DWARF forms, see the only
place where it is instantiated, DwarfCompileUnit::addLocationList().

The patch marks the unexpected execution path in DIELocList::SizeOf()
as unreachable, to reduce ambiguity.

Differential Revision: https://reviews.llvm.org/D84092
---
 llvm/lib/CodeGen/AsmPrinter/DIE.cpp | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/CodeGen/AsmPrinter/DIE.cpp b/llvm/lib/CodeGen/AsmPrinter/DIE.cpp
index 713a15dd09391..03219637f216e 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DIE.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DIE.cpp
@@ -809,13 +809,17 @@ void DIEBlock::print(raw_ostream &O) const {
 //===----------------------------------------------------------------------===//
 
 unsigned DIELocList::SizeOf(const AsmPrinter *AP, dwarf::Form Form) const {
-  if (Form == dwarf::DW_FORM_loclistx)
+  switch (Form) {
+  case dwarf::DW_FORM_loclistx:
     return getULEB128Size(Index);
-  if (Form == dwarf::DW_FORM_data4)
+  case dwarf::DW_FORM_data4:
     return 4;
-  if (Form == dwarf::DW_FORM_sec_offset)
+  case dwarf::DW_FORM_sec_offset:
+    // FIXME: add support for DWARF64
     return 4;
-  return AP->MAI->getCodePointerSize();
+  default:
+    llvm_unreachable("DIE Value form not supported yet");
+  }
 }
 
 /// EmitValue - Emit label value.

From 8feff8d14f75aafe4af77a6295d59d28e7a72829 Mon Sep 17 00:00:00 2001
From: Igor Kudrin <ikudrin@accesssoftek.com>
Date: Mon, 3 Aug 2020 15:04:00 +0700
Subject: [PATCH 0876/1035] [DebugInfo] Fix a comment and a variable name. NFC.

DebugLocListIndex keeps the index of an entry list, not the offset.

Differential Revision: https://reviews.llvm.org/D84093
---
 llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp | 6 +++---
 llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h         | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
index 2de6569767f69..871977df7a2ce 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
@@ -678,9 +678,9 @@ DIE *DwarfCompileUnit::constructVariableDIEImpl(const DbgVariable &DV,
 
   // Add variable address.
 
-  unsigned Offset = DV.getDebugLocListIndex();
-  if (Offset != ~0U) {
-    addLocationList(*VariableDie, dwarf::DW_AT_location, Offset);
+  unsigned Index = DV.getDebugLocListIndex();
+  if (Index != ~0U) {
+    addLocationList(*VariableDie, dwarf::DW_AT_location, Index);
     auto TagOffset = DV.getDebugLocListTagOffset();
     if (TagOffset)
       addUInt(*VariableDie, dwarf::DW_AT_LLVM_tag_offset, dwarf::DW_FORM_data1,
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h
index 2587ee61b0599..34364134d92ac 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h
@@ -114,7 +114,7 @@ class DbgEntity {
 ///
 /// Variables that have been optimized out use none of these fields.
 class DbgVariable : public DbgEntity {
-  /// Offset in DebugLocs.
+  /// Index of the entry list in DebugLocs.
   unsigned DebugLocListIndex = ~0u;
   /// DW_OP_LLVM_tag_offset value from DebugLocs.
   Optional<uint8_t> DebugLocListTagOffset;

From f98e03a35ded30893095f71be933ffc754d71d37 Mon Sep 17 00:00:00 2001
From: Igor Kudrin <ikudrin@accesssoftek.com>
Date: Mon, 3 Aug 2020 15:04:08 +0700
Subject: [PATCH 0877/1035] [DebugInfo] Fix misleading using of DWARF forms
 with DIELabel. NFCI.

DIELabel can emit only 32- or 64-bit values, while it was created in
some places with DW_FORM_udata, which implies emitting uleb128.
Nevertheless, these places also expected to emit U32 or U64, but just
used a misleading DWARF form. The patch updates those places to use more
appropriate DWARF forms and restricts DIELabel::SizeOf() to accept only
forms that are actually used in the LLVM codebase.

Differential Revision: https://reviews.llvm.org/D84094
---
 llvm/lib/CodeGen/AsmPrinter/DIE.cpp           | 22 ++++++++++++-------
 .../CodeGen/AsmPrinter/DwarfCompileUnit.cpp   |  4 ++--
 llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp     |  2 +-
 3 files changed, 17 insertions(+), 11 deletions(-)

diff --git a/llvm/lib/CodeGen/AsmPrinter/DIE.cpp b/llvm/lib/CodeGen/AsmPrinter/DIE.cpp
index 03219637f216e..b041f94de075d 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DIE.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DIE.cpp
@@ -495,19 +495,25 @@ void DIEExpr::print(raw_ostream &O) const { O << "Expr: " << *Expr; }
 /// EmitValue - Emit label value.
 ///
 void DIELabel::emitValue(const AsmPrinter *AP, dwarf::Form Form) const {
-  AP->emitLabelReference(
-      Label, SizeOf(AP, Form),
-      Form == dwarf::DW_FORM_strp || Form == dwarf::DW_FORM_sec_offset ||
-          Form == dwarf::DW_FORM_ref_addr || Form == dwarf::DW_FORM_data4);
+  bool IsSectionRelative = Form != dwarf::DW_FORM_addr;
+  AP->emitLabelReference(Label, SizeOf(AP, Form), IsSectionRelative);
 }
 
 /// SizeOf - Determine size of label value in bytes.
 ///
 unsigned DIELabel::SizeOf(const AsmPrinter *AP, dwarf::Form Form) const {
-  if (Form == dwarf::DW_FORM_data4) return 4;
-  if (Form == dwarf::DW_FORM_sec_offset) return 4;
-  if (Form == dwarf::DW_FORM_strp) return 4;
-  return AP->MAI->getCodePointerSize();
+  switch (Form) {
+  case dwarf::DW_FORM_data4:
+    return 4;
+  case dwarf::DW_FORM_sec_offset:
+  case dwarf::DW_FORM_strp:
+    // FIXME: add support for DWARF64
+    return 4;
+  case dwarf::DW_FORM_addr:
+    return AP->MAI->getCodePointerSize();
+  default:
+    llvm_unreachable("DIE Value form not supported yet");
+  }
 }
 
 LLVM_DUMP_METHOD
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
index 871977df7a2ce..704fff246b6a4 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
@@ -439,8 +439,8 @@ DIE &DwarfCompileUnit::updateSubprogramScopeDIE(const DISubprogram *SP) {
             true});
         DIELoc *Loc = new (DIEValueAllocator) DIELoc;
         addUInt(*Loc, dwarf::DW_FORM_data1, dwarf::DW_OP_WASM_location);
-        addSInt(*Loc, dwarf::DW_FORM_sdata, FrameBase.Location.WasmLoc.Kind);
-        addLabel(*Loc, dwarf::DW_FORM_udata, SPSym);
+        addSInt(*Loc, dwarf::DW_FORM_sdata, TI_GLOBAL_RELOC);
+        addLabel(*Loc, dwarf::DW_FORM_data4, SPSym);
         DD->addArangeLabel(SymbolCU(this, SPSym));
         addUInt(*Loc, dwarf::DW_FORM_data1, dwarf::DW_OP_stack_value);
         addBlock(*SPDie, dwarf::DW_AT_frame_base, Loc);
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp
index 1ba2afe48bf4b..2b45e50869edf 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp
@@ -335,7 +335,7 @@ void DwarfUnit::addOpAddress(DIELoc &Die, const MCSymbol *Sym) {
   }
 
   addUInt(Die, dwarf::DW_FORM_data1, dwarf::DW_OP_addr);
-  addLabel(Die, dwarf::DW_FORM_udata, Sym);
+  addLabel(Die, dwarf::DW_FORM_addr, Sym);
 }
 
 void DwarfUnit::addLabelDelta(DIE &Die, dwarf::Attribute Attribute,

From 414b9bec6deb542f7ca729585b1e592cac8ccb30 Mon Sep 17 00:00:00 2001
From: Igor Kudrin <ikudrin@accesssoftek.com>
Date: Mon, 3 Aug 2020 15:04:15 +0700
Subject: [PATCH 0878/1035] [DebugInfo] Make DIEDelta::SizeOf() more explicit.
 NFCI.

The patch restricts DIEDelta::SizeOf() to accept only DWARF forms that
are actually used in the LLVM codebase. This should make the use of the
class more explicit and help to avoid issues similar to fixed in D83958
and D84094.

Differential Revision: https://reviews.llvm.org/D84095
---
 llvm/lib/CodeGen/AsmPrinter/DIE.cpp | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/CodeGen/AsmPrinter/DIE.cpp b/llvm/lib/CodeGen/AsmPrinter/DIE.cpp
index b041f94de075d..0524d666810cb 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DIE.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DIE.cpp
@@ -549,10 +549,15 @@ void DIEDelta::emitValue(const AsmPrinter *AP, dwarf::Form Form) const {
 /// SizeOf - Determine size of delta value in bytes.
 ///
 unsigned DIEDelta::SizeOf(const AsmPrinter *AP, dwarf::Form Form) const {
-  if (Form == dwarf::DW_FORM_data4) return 4;
-  if (Form == dwarf::DW_FORM_sec_offset) return 4;
-  if (Form == dwarf::DW_FORM_strp) return 4;
-  return AP->MAI->getCodePointerSize();
+  switch (Form) {
+  case dwarf::DW_FORM_data4:
+    return 4;
+  case dwarf::DW_FORM_sec_offset:
+    // FIXME: add support for DWARF64
+    return 4;
+  default:
+    llvm_unreachable("DIE Value form not supported yet");
+  }
 }
 
 LLVM_DUMP_METHOD

From 434cf2ded3836075daa34bb4bd6286ff571d24e1 Mon Sep 17 00:00:00 2001
From: Shinji Okumura <okuraofvegetable@gmail.com>
Date: Mon, 3 Aug 2020 17:02:49 +0900
Subject: [PATCH 0879/1035] [Attributor] Check nonnull attribute violation in
 AAUndefinedBehavior

This patch makes it possible to handle nonnull attribute violation at callsites in AAUndefinedBehavior.
If null pointer is passed to callee at a callsite and the corresponding argument of callee has nonnull attribute, the behavior of the callee is undefined.
In this patch, violations of argument nonnull attributes is only handled.
But violations of returned nonnull attributes can be handled and I will implement that in a follow-up patch.

Reviewed By: jdoerfert

Differential Revision: https://reviews.llvm.org/D84733
---
 .../Transforms/IPO/AttributorAttributes.cpp   |  56 ++++
 .../Attributor/undefined_behavior.ll          | 295 ++++++++++++++++++
 2 files changed, 351 insertions(+)

diff --git a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
index aa6bc94a3668e..5cd0c711ddde1 100644
--- a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
+++ b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
@@ -1983,6 +1983,61 @@ struct AAUndefinedBehaviorImpl : public AAUndefinedBehavior {
       return true;
     };
 
+    auto InspectCallSiteForUB = [&](Instruction &I) {
+      // Check whether a callsite always cause UB or not
+
+      // Skip instructions that are already saved.
+      if (AssumedNoUBInsts.count(&I) || KnownUBInsts.count(&I))
+        return true;
+
+      // Check nonnull and noundef argument attribute violation for each
+      // callsite.
+      CallBase &CB = cast<CallBase>(I);
+      Function *Callee = CB.getCalledFunction();
+      if (!Callee)
+        return true;
+      for (unsigned idx = 0; idx < CB.getNumArgOperands(); idx++) {
+        // If current argument is known to be simplified to null pointer and the
+        // corresponding argument position is known to have nonnull attribute,
+        // the argument is poison. Furthermore, if the argument is poison and
+        // the position is known to have noundef attriubte, this callsite is
+        // considered UB.
+        // TODO: Check also nopoison attribute if it is introduced.
+        if (idx >= Callee->arg_size())
+          break;
+        Value *ArgVal = CB.getArgOperand(idx);
+        if(!ArgVal)
+          continue;
+        IRPosition CalleeArgumentIRP =
+            IRPosition::argument(*Callee->getArg(idx));
+        if (!CalleeArgumentIRP.hasAttr({Attribute::NoUndef}))
+          continue;
+        auto &NonNullAA = A.getAAFor<AANonNull>(*this, CalleeArgumentIRP);
+        if (!NonNullAA.isKnownNonNull())
+          continue;
+        const auto &ValueSimplifyAA =
+            A.getAAFor<AAValueSimplify>(*this, IRPosition::value(*ArgVal));
+        Optional<Value *> SimplifiedVal =
+            ValueSimplifyAA.getAssumedSimplifiedValue(A);
+
+        if (!ValueSimplifyAA.isKnown())
+          continue;
+        // Here, we handle three cases.
+        //   (1) Not having a value means it is dead. (we can replace the value
+        //       with undef)
+        //   (2) Simplified to null pointer. The argument is a poison value and
+        //       violate noundef attribute.
+        //   (3) Simplified to undef. The argument violate noundef attriubte.
+        if (!SimplifiedVal.hasValue() ||
+            isa<ConstantPointerNull>(*SimplifiedVal.getValue()) ||
+            isa<UndefValue>(*SimplifiedVal.getValue())) {
+          KnownUBInsts.insert(&I);
+          return true;
+        }
+      }
+      return true;
+    };
+
     A.checkForAllInstructions(InspectMemAccessInstForUB, *this,
                               {Instruction::Load, Instruction::Store,
                                Instruction::AtomicCmpXchg,
@@ -1990,6 +2045,7 @@ struct AAUndefinedBehaviorImpl : public AAUndefinedBehavior {
                               /* CheckBBLivenessOnly */ true);
     A.checkForAllInstructions(InspectBrInstForUB, *this, {Instruction::Br},
                               /* CheckBBLivenessOnly */ true);
+    A.checkForAllCallLikeInstructions(InspectCallSiteForUB, *this);
     if (NoUBPrevSize != AssumedNoUBInsts.size() ||
         UBPrevSize != KnownUBInsts.size())
       return ChangeStatus::CHANGED;
diff --git a/llvm/test/Transforms/Attributor/undefined_behavior.ll b/llvm/test/Transforms/Attributor/undefined_behavior.ll
index 52761da4b869d..b5bf4c4726564 100644
--- a/llvm/test/Transforms/Attributor/undefined_behavior.ll
+++ b/llvm/test/Transforms/Attributor/undefined_behavior.ll
@@ -579,3 +579,298 @@ define i32 @foo() {
   %X = call i32 @callee(i1 false, i32* null)
   ret i32 %X
 }
+
+; Tests for nonnull attribute violation.
+
+define void @arg_nonnull_1(i32* nonnull %a) {
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind willreturn writeonly
+; IS__TUNIT____-LABEL: define {{[^@]+}}@arg_nonnull_1
+; IS__TUNIT____-SAME: (i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[A:%.*]])
+; IS__TUNIT____-NEXT:    store i32 0, i32* [[A]], align 4
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind willreturn writeonly
+; IS__CGSCC____-LABEL: define {{[^@]+}}@arg_nonnull_1
+; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[A:%.*]])
+; IS__CGSCC____-NEXT:    store i32 0, i32* [[A]], align 4
+; IS__CGSCC____-NEXT:    ret void
+;
+  store i32 0, i32* %a
+  ret void
+}
+
+define void @arg_nonnull_1_noundef_1(i32* nonnull noundef %a) {
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind willreturn writeonly
+; IS__TUNIT____-LABEL: define {{[^@]+}}@arg_nonnull_1_noundef_1
+; IS__TUNIT____-SAME: (i32* nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[A:%.*]])
+; IS__TUNIT____-NEXT:    store i32 0, i32* [[A]], align 4
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind willreturn writeonly
+; IS__CGSCC____-LABEL: define {{[^@]+}}@arg_nonnull_1_noundef_1
+; IS__CGSCC____-SAME: (i32* nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[A:%.*]])
+; IS__CGSCC____-NEXT:    store i32 0, i32* [[A]], align 4
+; IS__CGSCC____-NEXT:    ret void
+;
+  store i32 0, i32* %a
+  ret void
+}
+
+define void @arg_nonnull_12(i32* nonnull %a, i32* nonnull %b, i32* %c) {
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind willreturn writeonly
+; IS__TUNIT____-LABEL: define {{[^@]+}}@arg_nonnull_12
+; IS__TUNIT____-SAME: (i32* nocapture nofree nonnull writeonly [[A:%.*]], i32* nocapture nofree nonnull writeonly [[B:%.*]], i32* nofree writeonly [[C:%.*]])
+; IS__TUNIT____-NEXT:    [[D:%.*]] = icmp eq i32* [[C]], null
+; IS__TUNIT____-NEXT:    br i1 [[D]], label [[T:%.*]], label [[F:%.*]]
+; IS__TUNIT____:       t:
+; IS__TUNIT____-NEXT:    store i32 0, i32* [[A]], align 4
+; IS__TUNIT____-NEXT:    br label [[RET:%.*]]
+; IS__TUNIT____:       f:
+; IS__TUNIT____-NEXT:    store i32 1, i32* [[B]], align 4
+; IS__TUNIT____-NEXT:    br label [[RET]]
+; IS__TUNIT____:       ret:
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind willreturn writeonly
+; IS__CGSCC____-LABEL: define {{[^@]+}}@arg_nonnull_12
+; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull writeonly [[A:%.*]], i32* nocapture nofree nonnull writeonly [[B:%.*]], i32* nofree writeonly [[C:%.*]])
+; IS__CGSCC____-NEXT:    [[D:%.*]] = icmp eq i32* [[C]], null
+; IS__CGSCC____-NEXT:    br i1 [[D]], label [[T:%.*]], label [[F:%.*]]
+; IS__CGSCC____:       t:
+; IS__CGSCC____-NEXT:    store i32 0, i32* [[A]], align 4
+; IS__CGSCC____-NEXT:    br label [[RET:%.*]]
+; IS__CGSCC____:       f:
+; IS__CGSCC____-NEXT:    store i32 1, i32* [[B]], align 4
+; IS__CGSCC____-NEXT:    br label [[RET]]
+; IS__CGSCC____:       ret:
+; IS__CGSCC____-NEXT:    ret void
+;
+  %d = icmp eq i32* %c, null
+  br i1 %d, label %t, label %f
+t:
+  store i32 0, i32* %a
+  br label %ret
+f:
+  store i32 1, i32* %b
+  br label %ret
+ret:
+  ret void
+}
+
+define void @arg_nonnull_12_noundef_2(i32* nonnull %a, i32* noundef nonnull %b, i32* %c) {
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind willreturn writeonly
+; IS__TUNIT____-LABEL: define {{[^@]+}}@arg_nonnull_12_noundef_2
+; IS__TUNIT____-SAME: (i32* nocapture nofree nonnull writeonly [[A:%.*]], i32* nocapture nofree noundef nonnull writeonly [[B:%.*]], i32* nofree writeonly [[C:%.*]])
+; IS__TUNIT____-NEXT:    [[D:%.*]] = icmp eq i32* [[C]], null
+; IS__TUNIT____-NEXT:    br i1 [[D]], label [[T:%.*]], label [[F:%.*]]
+; IS__TUNIT____:       t:
+; IS__TUNIT____-NEXT:    store i32 0, i32* [[A]], align 4
+; IS__TUNIT____-NEXT:    br label [[RET:%.*]]
+; IS__TUNIT____:       f:
+; IS__TUNIT____-NEXT:    store i32 1, i32* [[B]], align 4
+; IS__TUNIT____-NEXT:    br label [[RET]]
+; IS__TUNIT____:       ret:
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind willreturn writeonly
+; IS__CGSCC____-LABEL: define {{[^@]+}}@arg_nonnull_12_noundef_2
+; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull writeonly [[A:%.*]], i32* nocapture nofree noundef nonnull writeonly [[B:%.*]], i32* nofree writeonly [[C:%.*]])
+; IS__CGSCC____-NEXT:    [[D:%.*]] = icmp eq i32* [[C]], null
+; IS__CGSCC____-NEXT:    br i1 [[D]], label [[T:%.*]], label [[F:%.*]]
+; IS__CGSCC____:       t:
+; IS__CGSCC____-NEXT:    store i32 0, i32* [[A]], align 4
+; IS__CGSCC____-NEXT:    br label [[RET:%.*]]
+; IS__CGSCC____:       f:
+; IS__CGSCC____-NEXT:    store i32 1, i32* [[B]], align 4
+; IS__CGSCC____-NEXT:    br label [[RET]]
+; IS__CGSCC____:       ret:
+; IS__CGSCC____-NEXT:    ret void
+;
+  %d = icmp eq i32* %c, null
+  br i1 %d, label %t, label %f
+t:
+  store i32 0, i32* %a
+  br label %ret
+f:
+  store i32 1, i32* %b
+  br label %ret
+ret:
+  ret void
+}
+
+; Pass null directly to argument with nonnull attribute
+define void @arg_nonnull_violation1_1() {
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@arg_nonnull_violation1_1()
+; IS__TUNIT____-NEXT:    call void @arg_nonnull_1(i32* noalias nocapture nofree nonnull writeonly align 536870912 null)
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@arg_nonnull_violation1_1()
+; IS__CGSCC____-NEXT:    call void @arg_nonnull_1(i32* noalias nocapture nofree nonnull writeonly align 536870912 dereferenceable(4) null)
+; IS__CGSCC____-NEXT:    ret void
+;
+  call void @arg_nonnull_1(i32* null)
+  ret void
+}
+
+define void @arg_nonnull_violation1_2() {
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@arg_nonnull_violation1_2()
+; IS__TUNIT____-NEXT:    unreachable
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@arg_nonnull_violation1_2()
+; IS__CGSCC____-NEXT:    unreachable
+;
+  call void @arg_nonnull_1_noundef_1(i32* null)
+  ret void
+}
+
+; A case that depends on value simplification
+define void @arg_nonnull_violation2_1(i1 %c) {
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@arg_nonnull_violation2_1
+; IS__TUNIT____-SAME: (i1 [[C:%.*]])
+; IS__TUNIT____-NEXT:    call void @arg_nonnull_1(i32* nocapture nofree nonnull writeonly align 536870912 null)
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@arg_nonnull_violation2_1
+; IS__CGSCC____-SAME: (i1 [[C:%.*]])
+; IS__CGSCC____-NEXT:    call void @arg_nonnull_1(i32* nocapture nofree nonnull writeonly align 536870912 dereferenceable(4) null)
+; IS__CGSCC____-NEXT:    ret void
+;
+  %null = getelementptr i32, i32* null, i32 0
+  %mustnull = select i1 %c, i32* null, i32* %null
+  call void @arg_nonnull_1(i32* %mustnull)
+  ret void
+}
+
+define void @arg_nonnull_violation2_2(i1 %c) {
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@arg_nonnull_violation2_2
+; IS__TUNIT____-SAME: (i1 [[C:%.*]])
+; IS__TUNIT____-NEXT:    unreachable
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@arg_nonnull_violation2_2
+; IS__CGSCC____-SAME: (i1 [[C:%.*]])
+; IS__CGSCC____-NEXT:    unreachable
+;
+  %null = getelementptr i32, i32* null, i32 0
+  %mustnull = select i1 %c, i32* null, i32* %null
+  call void @arg_nonnull_1_noundef_1(i32* %mustnull)
+  ret void
+}
+
+; Cases for single and multiple violation at a callsite
+define void @arg_nonnull_violation3_1(i1 %c) {
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@arg_nonnull_violation3_1
+; IS__TUNIT____-SAME: (i1 [[C:%.*]])
+; IS__TUNIT____-NEXT:    [[PTR:%.*]] = alloca i32, align 4
+; IS__TUNIT____-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]]
+; IS__TUNIT____:       t:
+; IS__TUNIT____-NEXT:    call void @arg_nonnull_12(i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[PTR]], i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[PTR]], i32* nofree nonnull writeonly align 4 dereferenceable(4) [[PTR]])
+; IS__TUNIT____-NEXT:    call void @arg_nonnull_12(i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[PTR]], i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[PTR]], i32* noalias nocapture nofree writeonly align 536870912 null)
+; IS__TUNIT____-NEXT:    call void @arg_nonnull_12(i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[PTR]], i32* noalias nocapture nofree nonnull writeonly align 536870912 null, i32* nofree nonnull writeonly align 4 dereferenceable(4) [[PTR]])
+; IS__TUNIT____-NEXT:    call void @arg_nonnull_12(i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[PTR]], i32* noalias nocapture nofree nonnull writeonly align 536870912 null, i32* noalias nocapture nofree writeonly align 536870912 null)
+; IS__TUNIT____-NEXT:    br label [[RET:%.*]]
+; IS__TUNIT____:       f:
+; IS__TUNIT____-NEXT:    call void @arg_nonnull_12(i32* noalias nocapture nofree nonnull writeonly align 536870912 null, i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[PTR]], i32* nofree nonnull writeonly align 4 dereferenceable(4) [[PTR]])
+; IS__TUNIT____-NEXT:    call void @arg_nonnull_12(i32* noalias nocapture nofree nonnull writeonly align 536870912 null, i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[PTR]], i32* noalias nocapture nofree writeonly align 536870912 null)
+; IS__TUNIT____-NEXT:    call void @arg_nonnull_12(i32* noalias nocapture nofree nonnull writeonly align 536870912 null, i32* noalias nocapture nofree nonnull writeonly align 536870912 null, i32* nofree nonnull writeonly align 4 dereferenceable(4) [[PTR]])
+; IS__TUNIT____-NEXT:    call void @arg_nonnull_12(i32* noalias nocapture nofree nonnull writeonly align 536870912 null, i32* noalias nocapture nofree nonnull writeonly align 536870912 null, i32* noalias nocapture nofree writeonly align 536870912 null)
+; IS__TUNIT____-NEXT:    br label [[RET]]
+; IS__TUNIT____:       ret:
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@arg_nonnull_violation3_1
+; IS__CGSCC____-SAME: (i1 [[C:%.*]])
+; IS__CGSCC____-NEXT:    [[PTR:%.*]] = alloca i32, align 4
+; IS__CGSCC____-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]]
+; IS__CGSCC____:       t:
+; IS__CGSCC____-NEXT:    call void @arg_nonnull_12(i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[PTR]], i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[PTR]], i32* nofree nonnull writeonly align 4 dereferenceable(4) [[PTR]])
+; IS__CGSCC____-NEXT:    call void @arg_nonnull_12(i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[PTR]], i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[PTR]], i32* noalias nocapture nofree writeonly align 536870912 null)
+; IS__CGSCC____-NEXT:    call void @arg_nonnull_12(i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[PTR]], i32* noalias nocapture nofree nonnull writeonly align 536870912 null, i32* nofree nonnull writeonly align 4 dereferenceable(4) [[PTR]])
+; IS__CGSCC____-NEXT:    call void @arg_nonnull_12(i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[PTR]], i32* noalias nocapture nofree nonnull writeonly align 536870912 null, i32* noalias nocapture nofree writeonly align 536870912 null)
+; IS__CGSCC____-NEXT:    br label [[RET:%.*]]
+; IS__CGSCC____:       f:
+; IS__CGSCC____-NEXT:    call void @arg_nonnull_12(i32* noalias nocapture nofree nonnull writeonly align 536870912 null, i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[PTR]], i32* nofree nonnull writeonly align 4 dereferenceable(4) [[PTR]])
+; IS__CGSCC____-NEXT:    call void @arg_nonnull_12(i32* noalias nocapture nofree nonnull writeonly align 536870912 null, i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[PTR]], i32* noalias nocapture nofree writeonly align 536870912 null)
+; IS__CGSCC____-NEXT:    call void @arg_nonnull_12(i32* noalias nocapture nofree nonnull writeonly align 536870912 null, i32* noalias nocapture nofree nonnull writeonly align 536870912 null, i32* nofree nonnull writeonly align 4 dereferenceable(4) [[PTR]])
+; IS__CGSCC____-NEXT:    call void @arg_nonnull_12(i32* noalias nocapture nofree nonnull writeonly align 536870912 null, i32* noalias nocapture nofree nonnull writeonly align 536870912 null, i32* noalias nocapture nofree writeonly align 536870912 null)
+; IS__CGSCC____-NEXT:    br label [[RET]]
+; IS__CGSCC____:       ret:
+; IS__CGSCC____-NEXT:    ret void
+;
+  %ptr = alloca i32
+  br i1 %c, label %t, label %f
+t:
+  call void @arg_nonnull_12(i32* %ptr, i32* %ptr, i32* %ptr)
+  call void @arg_nonnull_12(i32* %ptr, i32* %ptr, i32* null)
+  call void @arg_nonnull_12(i32* %ptr, i32* null, i32* %ptr)
+  call void @arg_nonnull_12(i32* %ptr, i32* null, i32* null)
+  br label %ret
+f:
+  call void @arg_nonnull_12(i32* null, i32* %ptr, i32* %ptr)
+  call void @arg_nonnull_12(i32* null, i32* %ptr, i32* null)
+  call void @arg_nonnull_12(i32* null, i32* null, i32* %ptr)
+  call void @arg_nonnull_12(i32* null, i32* null, i32* null)
+  br label %ret
+ret:
+  ret void
+}
+
+define void @arg_nonnull_violation3_2(i1 %c) {
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@arg_nonnull_violation3_2
+; IS__TUNIT____-SAME: (i1 [[C:%.*]])
+; IS__TUNIT____-NEXT:    [[PTR:%.*]] = alloca i32, align 4
+; IS__TUNIT____-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]]
+; IS__TUNIT____:       t:
+; IS__TUNIT____-NEXT:    call void @arg_nonnull_12_noundef_2(i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[PTR]], i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[PTR]], i32* nofree nonnull writeonly align 4 dereferenceable(4) [[PTR]])
+; IS__TUNIT____-NEXT:    call void @arg_nonnull_12_noundef_2(i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[PTR]], i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[PTR]], i32* noalias nocapture nofree writeonly align 536870912 null)
+; IS__TUNIT____-NEXT:    unreachable
+; IS__TUNIT____:       f:
+; IS__TUNIT____-NEXT:    call void @arg_nonnull_12_noundef_2(i32* noalias nocapture nofree nonnull writeonly align 536870912 null, i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[PTR]], i32* nofree nonnull writeonly align 4 dereferenceable(4) [[PTR]])
+; IS__TUNIT____-NEXT:    call void @arg_nonnull_12_noundef_2(i32* noalias nocapture nofree nonnull writeonly align 536870912 null, i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[PTR]], i32* noalias nocapture nofree writeonly align 536870912 null)
+; IS__TUNIT____-NEXT:    unreachable
+; IS__TUNIT____:       ret:
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@arg_nonnull_violation3_2
+; IS__CGSCC____-SAME: (i1 [[C:%.*]])
+; IS__CGSCC____-NEXT:    [[PTR:%.*]] = alloca i32, align 4
+; IS__CGSCC____-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]]
+; IS__CGSCC____:       t:
+; IS__CGSCC____-NEXT:    call void @arg_nonnull_12_noundef_2(i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[PTR]], i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[PTR]], i32* nofree nonnull writeonly align 4 dereferenceable(4) [[PTR]])
+; IS__CGSCC____-NEXT:    call void @arg_nonnull_12_noundef_2(i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[PTR]], i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[PTR]], i32* noalias nocapture nofree writeonly align 536870912 null)
+; IS__CGSCC____-NEXT:    unreachable
+; IS__CGSCC____:       f:
+; IS__CGSCC____-NEXT:    call void @arg_nonnull_12_noundef_2(i32* noalias nocapture nofree nonnull writeonly align 536870912 null, i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[PTR]], i32* nofree nonnull writeonly align 4 dereferenceable(4) [[PTR]])
+; IS__CGSCC____-NEXT:    call void @arg_nonnull_12_noundef_2(i32* noalias nocapture nofree nonnull writeonly align 536870912 null, i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[PTR]], i32* noalias nocapture nofree writeonly align 536870912 null)
+; IS__CGSCC____-NEXT:    unreachable
+; IS__CGSCC____:       ret:
+; IS__CGSCC____-NEXT:    ret void
+;
+  %ptr = alloca i32
+  br i1 %c, label %t, label %f
+t:
+  call void @arg_nonnull_12_noundef_2(i32* %ptr, i32* %ptr, i32* %ptr)
+  call void @arg_nonnull_12_noundef_2(i32* %ptr, i32* %ptr, i32* null)
+  call void @arg_nonnull_12_noundef_2(i32* %ptr, i32* null, i32* %ptr)
+  call void @arg_nonnull_12_noundef_2(i32* %ptr, i32* null, i32* null)
+  br label %ret
+f:
+  call void @arg_nonnull_12_noundef_2(i32* null, i32* %ptr, i32* %ptr)
+  call void @arg_nonnull_12_noundef_2(i32* null, i32* %ptr, i32* null)
+  call void @arg_nonnull_12_noundef_2(i32* null, i32* null, i32* %ptr)
+  call void @arg_nonnull_12_noundef_2(i32* null, i32* null, i32* null)
+  br label %ret
+ret:
+  ret void
+}

From ef005f204b5d30a2bccfe2ab5431729dd644548c Mon Sep 17 00:00:00 2001
From: Xing GUO <higuoxing@gmail.com>
Date: Mon, 3 Aug 2020 16:14:47 +0800
Subject: [PATCH 0880/1035] [MachOYAML] Remove redundant variable
 initialization. NFC.

The value of `is64Bit` is initialized in the constructor body.
---
 llvm/lib/ObjectYAML/MachOEmitter.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/ObjectYAML/MachOEmitter.cpp b/llvm/lib/ObjectYAML/MachOEmitter.cpp
index 9b454c528a7e2..3b1421440cb99 100644
--- a/llvm/lib/ObjectYAML/MachOEmitter.cpp
+++ b/llvm/lib/ObjectYAML/MachOEmitter.cpp
@@ -29,7 +29,7 @@ namespace {
 
 class MachOWriter {
 public:
-  MachOWriter(MachOYAML::Object &Obj) : Obj(Obj), is64Bit(true), fileStart(0) {
+  MachOWriter(MachOYAML::Object &Obj) : Obj(Obj), fileStart(0) {
     is64Bit = Obj.Header.magic == MachO::MH_MAGIC_64 ||
               Obj.Header.magic == MachO::MH_CIGAM_64;
     memset(reinterpret_cast<void *>(&Header), 0, sizeof(MachO::mach_header_64));

From 11492be9d72d4215ac2f61626264da05fee35e78 Mon Sep 17 00:00:00 2001
From: Frederik Gossen <frgossen@google.com>
Date: Mon, 3 Aug 2020 08:18:48 +0000
Subject: [PATCH 0881/1035] [MLIR][Shape] Lower `shape.broadcast` to `scf`

Differential Revision: https://reviews.llvm.org/D85027
---
 mlir/lib/Conversion/ShapeToSCF/ShapeToSCF.cpp | 94 ++++++++++++++++++-
 .../Conversion/ShapeToSCF/shape-to-scf.mlir   | 50 ++++++++++
 2 files changed, 143 insertions(+), 1 deletion(-)

diff --git a/mlir/lib/Conversion/ShapeToSCF/ShapeToSCF.cpp b/mlir/lib/Conversion/ShapeToSCF/ShapeToSCF.cpp
index a6c667f5641c3..ae326c5c513e6 100644
--- a/mlir/lib/Conversion/ShapeToSCF/ShapeToSCF.cpp
+++ b/mlir/lib/Conversion/ShapeToSCF/ShapeToSCF.cpp
@@ -19,6 +19,98 @@ using namespace mlir;
 using namespace mlir::shape;
 using namespace mlir::scf;
 
+namespace {
+struct BroadcastOpConverter : public OpConversionPattern<BroadcastOp> {
+  using OpConversionPattern<BroadcastOp>::OpConversionPattern;
+
+  LogicalResult
+  matchAndRewrite(BroadcastOp op, ArrayRef<Value> operands,
+                  ConversionPatternRewriter &rewriter) const override;
+};
+} // namespace
+
+LogicalResult BroadcastOpConverter::matchAndRewrite(
+    BroadcastOp op, ArrayRef<Value> operands,
+    ConversionPatternRewriter &rewriter) const {
+  // For now, this lowering is only defined on `tensor<?xindex>` operands, not
+  // on shapes.
+  if (op.getType().isa<ShapeType>())
+    return failure();
+
+  assert(!op.lhs().getType().isa<ShapeType>() &&
+         !op.rhs().getType().isa<ShapeType>());
+  auto loc = op.getLoc();
+  BroadcastOp::Adaptor transformed(operands);
+  Value zero = rewriter.create<ConstantIndexOp>(loc, 0);
+  Value one = rewriter.create<ConstantIndexOp>(loc, 1);
+
+  // Find smaller and greater rank and extent tensor.
+  Value lhsRank = rewriter.create<DimOp>(loc, transformed.lhs(), zero);
+  Value rhsRank = rewriter.create<DimOp>(loc, transformed.rhs(), zero);
+  Value lhsSmaller =
+      rewriter.create<CmpIOp>(loc, CmpIPredicate::ule, lhsRank, rhsRank);
+  Type indexTy = rewriter.getIndexType();
+  Type extentTensorTy = op.getType();
+  auto ifOp = rewriter.create<IfOp>(
+      loc, TypeRange{indexTy, extentTensorTy, indexTy, extentTensorTy},
+      lhsSmaller,
+      [&](OpBuilder &b, Location loc) {
+        b.create<scf::YieldOp>(loc, ValueRange{lhsRank, transformed.lhs(),
+                                               rhsRank, transformed.rhs()});
+      },
+      [&](OpBuilder &b, Location loc) {
+        b.create<scf::YieldOp>(loc, ValueRange{rhsRank, transformed.rhs(),
+                                               lhsRank, transformed.lhs()});
+      });
+  Value smallerRank = ifOp.getResult(0);
+  Value smallerOperand = ifOp.getResult(1);
+  Value greaterRank = ifOp.getResult(2);
+  Value greaterOperand = ifOp.getResult(3);
+
+  // Allocate stack memory for the broadcasted extent tensor.
+  Type memTy = MemRefType::get({ShapedType::kDynamicSize}, indexTy);
+  Value mem = rewriter.create<AllocaOp>(loc, memTy, ValueRange{greaterRank});
+
+  // Copy extents from greater operand that are not challenged.
+  Value rankDiff =
+      rewriter.create<SubIOp>(loc, indexTy, greaterRank, smallerRank);
+  rewriter.create<ForOp>(loc, zero, rankDiff, one, llvm::None,
+                         [&](OpBuilder &b, Location loc, Value iv, ValueRange) {
+                           Value extent = b.create<ExtractElementOp>(
+                               loc, greaterOperand, ValueRange{iv});
+                           b.create<StoreOp>(loc, extent, mem, ValueRange{iv});
+                           b.create<scf::YieldOp>(loc);
+                         });
+
+  // Determine remaining broadcasted extents.
+  rewriter.create<ForOp>(
+      loc, rankDiff, greaterRank, one, llvm::None,
+      [&](OpBuilder &b, Location loc, Value iv, ValueRange) {
+        Value greaterOperandExtent =
+            b.create<ExtractElementOp>(loc, greaterOperand, ValueRange{iv});
+        Value greaterOperandExtentIsOne =
+            b.create<CmpIOp>(loc, CmpIPredicate::eq, greaterOperandExtent, one);
+        auto ifOp = b.create<IfOp>(
+            loc, TypeRange{indexTy}, greaterOperandExtentIsOne,
+            [&](OpBuilder &b, Location loc) {
+              Value ivShifted = b.create<SubIOp>(loc, indexTy, iv, rankDiff);
+              Value smallerOperandExtent = b.create<ExtractElementOp>(
+                  loc, smallerOperand, ValueRange{ivShifted});
+              b.create<scf::YieldOp>(loc, smallerOperandExtent);
+            },
+            [&](OpBuilder &b, Location loc) {
+              b.create<scf::YieldOp>(loc, greaterOperandExtent);
+            });
+        Value extent = ifOp.getResult(0);
+        b.create<StoreOp>(loc, extent, mem, ValueRange{iv});
+        b.create<scf::YieldOp>(loc);
+      });
+
+  // Load broadcasted shape as an extent tensor.
+  rewriter.replaceOpWithNewOp<TensorLoadOp>(op, mem);
+  return success();
+}
+
 namespace {
 /// Converts `shape.shape_eq` to an `scf.for` loop. For now, the lowering is
 /// only defined on `tensor<?xindex>` operands. The test for equality first
@@ -223,7 +315,6 @@ void ConvertShapeToSCFPass::runOnFunction() {
   // Setup target legality.
   ConversionTarget target(getContext());
   target.addLegalDialect<SCFDialect, StandardOpsDialect>();
-  target.addLegalOp<ModuleOp, FuncOp>();
 
   // Apply conversion.
   if (failed(applyPartialConversion(getFunction(), target, patterns)))
@@ -234,6 +325,7 @@ void mlir::populateShapeToSCFConversionPatterns(
     OwningRewritePatternList &patterns, MLIRContext *ctx) {
   // clang-format off
   patterns.insert<
+      BroadcastOpConverter,
       ShapeEqOpConverter,
       ReduceOpConverter,
       ShapeOfOpConverter>(ctx);
diff --git a/mlir/test/Conversion/ShapeToSCF/shape-to-scf.mlir b/mlir/test/Conversion/ShapeToSCF/shape-to-scf.mlir
index 768a627208b8e..cc384496dff05 100644
--- a/mlir/test/Conversion/ShapeToSCF/shape-to-scf.mlir
+++ b/mlir/test/Conversion/ShapeToSCF/shape-to-scf.mlir
@@ -80,3 +80,53 @@ func @shape_eq(%a : tensor<?xindex>, %b : tensor<?xindex>) -> i1 {
   %result = shape.shape_eq %a, %b : tensor<?xindex>, tensor<?xindex>
   return %result : i1
 }
+
+// -----
+
+// Don't lower `shape.broadcast` if a `shape.shape` type is involved.
+// CHECK-LABEL: @broadcast
+func @broadcast(%a : tensor<?xindex>, %b : !shape.shape) -> !shape.shape {
+  // CHECK: shape.broadcast
+  %c = shape.broadcast %a, %b : tensor<?xindex>, !shape.shape -> !shape.shape
+  return %c : !shape.shape
+}
+
+// -----
+
+// CHECK-LABEL: @broadcast
+// CHECK-SAME:  (%[[LHS:.*]]: tensor<?xindex>, %[[RHS:.*]]: tensor<?xindex>)
+func @broadcast(%a : tensor<?xindex>, %b : tensor<?xindex>) {
+  // CHECK: %[[C0:.*]] = constant 0 : index
+  // CHECK: %[[C1:.*]] = constant 1 : index
+  // CHECK: %[[LHS_RANK:.*]] = dim %[[LHS]], %[[C0]] : tensor<?xindex>
+  // CHECK: %[[RHS_RANK:.*]] = dim %[[RHS]], %[[C0]] : tensor<?xindex>
+  // CHECK: %[[LHS_SMALLER:.*]] = cmpi "ule", %[[LHS_RANK]], %[[RHS_RANK]]
+  // CHECK: %[[ARG:.*]]:4 = scf.if %[[LHS_SMALLER]] -> (index, tensor<?xindex>, index, tensor<?xindex>) {
+  // CHECK:   scf.yield %[[LHS_RANK]], %[[LHS]], %[[RHS_RANK]], %[[RHS]] : index, tensor<?xindex>, index, tensor<?xindex>
+  // CHECK: } else {
+  // CHECK:   scf.yield %[[RHS_RANK]], %[[RHS]], %[[LHS_RANK]], %[[LHS]] : index, tensor<?xindex>, index, tensor<?xindex>
+  // CHECK: }
+  // CHECK: %[[MEM:.*]] = alloca(%[[ARG]]#2) : memref<?xindex>
+  // CHECK: %[[RANK_DIFF:.*]] = subi %[[ARG]]#2, %[[ARG]]#0 : index
+  // CHECK: scf.for %[[IV:.*]] = %[[C0]] to %[[RANK_DIFF]] step %[[C1]] {
+  // CHECK:   %[[EXTENT:.*]] = extract_element %[[ARG]]#3[%[[IV]]] : tensor<?xindex>
+  // CHECK:   store %[[EXTENT]], %[[MEM]][%[[IV]]] : memref<?xindex>
+  // CHECK: }
+  // CHECK: scf.for %[[IV:.*]] = %[[RANK_DIFF]] to %[[ARG]]#2 step %[[C1]] {
+  // CHECK:   %[[GREATER_OPERAND_EXTENT:.*]] = extract_element %[[ARG]]#3[%[[IV]]] : tensor<?xindex>
+  // CHECK:   %[[GREATER_OPERAND_EXTENT_IS_ONE:.*]] = cmpi "eq", %[[GREATER_OPERAND_EXTENT]], %[[C1]] : index
+  // CHECK:   %[[EXTENT:.*]] = scf.if %[[GREATER_OPERAND_EXTENT_IS_ONE]] -> (index) {
+  // CHECK:     %[[IV_SHIFTED:.*]] = subi %[[IV]], %[[RANK_DIFF]] : index
+  // CHECK:     %[[SMALLER_OPERAND_EXTENT:.*]] = extract_element %[[ARG]]#1[%[[IV_SHIFTED]]] : tensor<?xindex>
+  // CHECK:     scf.yield %[[SMALLER_OPERAND_EXTENT]] : index
+  // CHECK:   } else {
+  // CHECK:     scf.yield %[[GREATER_OPERAND_EXTENT]] : index
+  // CHECK:   }
+  // CHECK:   store %[[EXTENT]], %[[MEM]][%[[IV]]] : memref<?xindex>
+  // CHECK: }
+  // CHECK: %[[BROADCASTED:.*]] = tensor_load %[[MEM]] : memref<?xindex>
+  %0 = shape.broadcast %a, %b
+      : tensor<?xindex>, tensor<?xindex> -> tensor<?xindex>
+  return
+}
+

From 8aeb212887024a615ca02437cd12fa055bd54b6f Mon Sep 17 00:00:00 2001
From: Raphael Isemann <teemperor@gmail.com>
Date: Mon, 3 Aug 2020 10:23:48 +0200
Subject: [PATCH 0882/1035] [debugserver] Fix that is_dot_app is producing
 unused warnings

Some build configurations don't use this static function.
---
 lldb/tools/debugserver/source/debugserver.cpp | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/lldb/tools/debugserver/source/debugserver.cpp b/lldb/tools/debugserver/source/debugserver.cpp
index 4e6aa39e52d37..04cbd2c8b503e 100644
--- a/lldb/tools/debugserver/source/debugserver.cpp
+++ b/lldb/tools/debugserver/source/debugserver.cpp
@@ -212,19 +212,21 @@ RNBRunLoopMode RNBRunLoopLaunchInferior(RNBRemote *remote,
     // Our default launch method is posix spawn
     launch_flavor = eLaunchFlavorPosixSpawn;
 
+    const bool dot_app = is_dot_app(inferior_argv[0]);
+    (void)dot_app;
 #if defined WITH_FBS
     // Check if we have an app bundle, if so launch using BackBoard Services.
-    if (is_dot_app(inferior_argv[0])) {
+    if (dot_app) {
       launch_flavor = eLaunchFlavorFBS;
     }
 #elif defined WITH_BKS
     // Check if we have an app bundle, if so launch using BackBoard Services.
-    if (is_dot_app(inferior_argv[0])) {
+    if (dot_app) {
       launch_flavor = eLaunchFlavorBKS;
     }
 #elif defined WITH_SPRINGBOARD
     // Check if we have an app bundle, if so launch using SpringBoard.
-    if (is_dot_app(inferior_argv[0])) {
+    if (dot_app) {
       launch_flavor = eLaunchFlavorSpringBoard;
     }
 #endif

From 35b65be041127db9fe23d3128a004c888893cbae Mon Sep 17 00:00:00 2001
From: Nicolas Vasilache <ntv@google.com>
Date: Mon, 3 Aug 2020 04:39:18 -0400
Subject: [PATCH 0883/1035] [mlir][Vector] Add transformation + pattern to
 split vector.transfer_read into full and partial copies.

This revision adds a transformation and a pattern that rewrites a "maybe masked" `vector.transfer_read %view[...], %pad `into a pattern resembling:

```
   %1:3 = scf.if (%inBounds) {
      scf.yield %view : memref<A...>, index, index
    } else {
      %2 = vector.transfer_read %view[...], %pad : memref<A...>, vector<...>
      %3 = vector.type_cast %extra_alloc : memref<...> to
      memref<vector<...>> store %2, %3[] : memref<vector<...>> %4 =
      memref_cast %extra_alloc: memref<B...> to memref<A...> scf.yield %4 :
      memref<A...>, index, index
   }
   %res= vector.transfer_read %1#0[%1#1, %1#2] {masked = [false ... false]}
```
where `extra_alloc` is a top of the function alloca'ed buffer of one vector.

This rewrite makes it possible to realize the "always full tile" abstraction where vector.transfer_read operations are guaranteed to read from a padded full buffer.
The extra work only occurs on the boundary tiles.

Differential Revision: https://reviews.llvm.org/D84631
---
 .../mlir/Dialect/Vector/VectorTransforms.h    |  64 +++++
 .../mlir/Interfaces/VectorInterfaces.td       |  13 +
 mlir/lib/Dialect/Vector/VectorTransforms.cpp  | 234 ++++++++++++++++++
 .../vector-transfer-full-partial-split.mlir   | 102 ++++++++
 .../lib/Transforms/TestVectorTransforms.cpp   |  16 ++
 5 files changed, 429 insertions(+)
 create mode 100644 mlir/test/Dialect/Vector/vector-transfer-full-partial-split.mlir

diff --git a/mlir/include/mlir/Dialect/Vector/VectorTransforms.h b/mlir/include/mlir/Dialect/Vector/VectorTransforms.h
index 0d18c5aa782d1..835ad18a79ad2 100644
--- a/mlir/include/mlir/Dialect/Vector/VectorTransforms.h
+++ b/mlir/include/mlir/Dialect/Vector/VectorTransforms.h
@@ -17,6 +17,11 @@
 namespace mlir {
 class MLIRContext;
 class OwningRewritePatternList;
+class VectorTransferOpInterface;
+
+namespace scf {
+class IfOp;
+} // namespace scf
 
 /// Collect a set of patterns to convert from the Vector dialect to itself.
 /// Should be merged with populateVectorToSCFLoweringPattern.
@@ -104,6 +109,65 @@ struct UnrollVectorPattern : public OpRewritePattern<OpTy> {
   FilterConstraintType filter;
 };
 
+/// Split a vector.transfer operation into an unmasked fastpath vector.transfer
+/// and a slowpath masked vector.transfer. If `ifOp` is not null and the result
+/// is `success, the `ifOp` points to the newly created conditional upon
+/// function return. To accomodate for the fact that the original
+/// vector.transfer indexing may be arbitrary and the slow path indexes @[0...0]
+/// in the temporary buffer, the scf.if op returns a view and values of type
+/// index. At this time, only vector.transfer_read is implemented.
+///
+/// Example (a 2-D vector.transfer_read):
+/// ```
+///    %1 = vector.transfer_read %0[...], %pad : memref<A...>, vector<...>
+/// ```
+/// is transformed into:
+/// ```
+///    %1:3 = scf.if (%inBounds) {
+///       scf.yield %0 : memref<A...>, index, index
+///     } else {
+///       %2 = vector.transfer_read %0[...], %pad : memref<A...>, vector<...>
+///       %3 = vector.type_cast %extra_alloc : memref<...> to
+///       memref<vector<...>> store %2, %3[] : memref<vector<...>> %4 =
+///       memref_cast %extra_alloc: memref<B...> to memref<A...> scf.yield %4 :
+///       memref<A...>, index, index
+//     }
+///    %0 = vector.transfer_read %1#0[%1#1, %1#2] {masked = [false ... false]}
+/// ```
+/// where `extra_alloc` is a top of the function alloca'ed buffer of one vector.
+///
+/// Preconditions:
+///  1. `xferOp.permutation_map()` must be a minor identity map
+///  2. the rank of the `xferOp.memref()` and the rank of the `xferOp.vector()`
+///  must be equal. This will be relaxed in the future but requires
+///  rank-reducing subviews.
+LogicalResult
+splitFullAndPartialTransferPrecondition(VectorTransferOpInterface xferOp);
+LogicalResult splitFullAndPartialTransfer(OpBuilder &b,
+                                          VectorTransferOpInterface xferOp,
+                                          scf::IfOp *ifOp = nullptr);
+
+/// Apply `splitFullAndPartialTransfer` selectively via a pattern. This pattern
+/// may take an extra filter to perform selection at a finer granularity.
+struct VectorTransferFullPartialRewriter : public RewritePattern {
+  using FilterConstraintType =
+      std::function<LogicalResult(VectorTransferOpInterface op)>;
+
+  explicit VectorTransferFullPartialRewriter(
+      MLIRContext *context,
+      FilterConstraintType filter =
+          [](VectorTransferOpInterface op) { return success(); },
+      PatternBenefit benefit = 1)
+      : RewritePattern(benefit, MatchAnyOpTypeTag()), filter(filter) {}
+
+  /// Performs the rewrite.
+  LogicalResult matchAndRewrite(Operation *op,
+                                PatternRewriter &rewriter) const override;
+
+private:
+  FilterConstraintType filter;
+};
+
 } // namespace vector
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Interfaces/VectorInterfaces.td b/mlir/include/mlir/Interfaces/VectorInterfaces.td
index aefbb7d471172..218715318a867 100644
--- a/mlir/include/mlir/Interfaces/VectorInterfaces.td
+++ b/mlir/include/mlir/Interfaces/VectorInterfaces.td
@@ -160,6 +160,19 @@ def VectorTransferOpInterface : OpInterface<"VectorTransferOpInterface"> {
       /*defaultImplementation=*/
         "return $_op.getMemRefType().getRank() - $_op.getTransferRank();"
     >,
+    InterfaceMethod<
+      /*desc=*/[{ Returns true if at least one of the dimensions is masked.}],
+      /*retTy=*/"bool",
+      /*methodName=*/"hasMaskedDim",
+      /*args=*/(ins),
+      /*methodBody=*/"",
+      /*defaultImplementation=*/[{
+        for (unsigned idx = 0, e = $_op.getTransferRank(); idx < e; ++idx)
+          if ($_op.isMaskedDim(idx))
+            return true;
+        return false;
+      }]
+    >,
     InterfaceMethod<
       /*desc=*/[{
       Helper function to account for the fact that `permutationMap` results and
diff --git a/mlir/lib/Dialect/Vector/VectorTransforms.cpp b/mlir/lib/Dialect/Vector/VectorTransforms.cpp
index 197b1c62274b2..573b822503f3a 100644
--- a/mlir/lib/Dialect/Vector/VectorTransforms.cpp
+++ b/mlir/lib/Dialect/Vector/VectorTransforms.cpp
@@ -12,9 +12,13 @@
 
 #include <type_traits>
 
+#include "mlir/Dialect/Affine/EDSC/Intrinsics.h"
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
+#include "mlir/Dialect/SCF/EDSC/Intrinsics.h"
+#include "mlir/Dialect/StandardOps/EDSC/Intrinsics.h"
 #include "mlir/Dialect/StandardOps/IR/Ops.h"
 #include "mlir/Dialect/Utils/StructuredOpsUtils.h"
+#include "mlir/Dialect/Vector/EDSC/Intrinsics.h"
 #include "mlir/Dialect/Vector/VectorOps.h"
 #include "mlir/Dialect/Vector/VectorTransforms.h"
 #include "mlir/Dialect/Vector/VectorUtils.h"
@@ -1985,6 +1989,236 @@ Value ContractionOpLowering::lowerReduction(vector::ContractionOp op,
 
 } // namespace mlir
 
+static Optional<int64_t> extractConstantIndex(Value v) {
+  if (auto cstOp = v.getDefiningOp<ConstantIndexOp>())
+    return cstOp.getValue();
+  if (auto affineApplyOp = v.getDefiningOp<AffineApplyOp>())
+    if (affineApplyOp.getAffineMap().isSingleConstant())
+      return affineApplyOp.getAffineMap().getSingleConstantResult();
+  return None;
+}
+
+// Missing foldings of scf.if make it necessary to perform poor man's folding
+// eagerly, especially in the case of unrolling. In the future, this should go
+// away once scf.if folds properly.
+static Value createScopedFoldedSLE(Value v, Value ub) {
+  using namespace edsc::op;
+  auto maybeCstV = extractConstantIndex(v);
+  auto maybeCstUb = extractConstantIndex(ub);
+  if (maybeCstV && maybeCstUb && *maybeCstV < *maybeCstUb)
+    return Value();
+  return sle(v, ub);
+}
+
+// Operates under a scoped context to build the condition to ensure that a
+// particular VectorTransferOpInterface is unmasked.
+static Value createScopedInBoundsCond(VectorTransferOpInterface xferOp) {
+  assert(xferOp.permutation_map().isMinorIdentity() &&
+         "Expected minor identity map");
+  Value inBoundsCond;
+  xferOp.zipResultAndIndexing([&](int64_t resultIdx, int64_t indicesIdx) {
+    // Zip over the resulting vector shape and memref indices.
+    // If the dimension is known to be unmasked, it does not participate in the
+    // construction of `inBoundsCond`.
+    if (!xferOp.isMaskedDim(resultIdx))
+      return;
+    int64_t vectorSize = xferOp.getVectorType().getDimSize(resultIdx);
+    using namespace edsc::op;
+    using namespace edsc::intrinsics;
+    // Fold or create the check that `index + vector_size` <= `memref_size`.
+    Value sum = xferOp.indices()[indicesIdx] + std_constant_index(vectorSize);
+    Value cond =
+        createScopedFoldedSLE(sum, std_dim(xferOp.memref(), indicesIdx));
+    if (!cond)
+      return;
+    // Conjunction over all dims for which we are in-bounds.
+    inBoundsCond = inBoundsCond ? inBoundsCond && cond : cond;
+  });
+  return inBoundsCond;
+}
+
+LogicalResult mlir::vector::splitFullAndPartialTransferPrecondition(
+    VectorTransferOpInterface xferOp) {
+  // TODO: expand support to these 2 cases.
+  if (!xferOp.permutation_map().isMinorIdentity())
+    return failure();
+  // TODO: relax this precondition. This will require rank-reducing subviews.
+  if (xferOp.getMemRefType().getRank() != xferOp.getTransferRank())
+    return failure();
+  // Must have some masked dimension to be a candidate for splitting.
+  if (!xferOp.hasMaskedDim())
+    return failure();
+  // Don't split transfer operations under IfOp, this avoids applying the
+  // pattern recursively.
+  // TODO: improve the condition to make it more applicable.
+  if (xferOp.getParentOfType<scf::IfOp>())
+    return failure();
+  return success();
+}
+
+MemRefType getCastCompatibleMemRefType(MemRefType aT, MemRefType bT) {
+  if (MemRefCastOp::areCastCompatible(aT, bT))
+    return aT;
+  if (aT.getRank() != bT.getRank())
+    return MemRefType();
+  int64_t aOffset, bOffset;
+  SmallVector<int64_t, 4> aStrides, bStrides;
+  if (failed(getStridesAndOffset(aT, aStrides, aOffset)) ||
+      failed(getStridesAndOffset(bT, bStrides, bOffset)) ||
+      aStrides.size() != bStrides.size())
+    return MemRefType();
+
+  ArrayRef<int64_t> aShape = aT.getShape(), bShape = bT.getShape();
+  int64_t resOffset;
+  SmallVector<int64_t, 4> resShape(aT.getRank(), 0),
+      resStrides(bT.getRank(), 0);
+  for (int64_t idx = 0, e = aT.getRank(); idx < e; ++idx) {
+    resShape[idx] =
+        (aShape[idx] == bShape[idx]) ? aShape[idx] : MemRefType::kDynamicSize;
+    resStrides[idx] = (aStrides[idx] == bStrides[idx])
+                          ? aStrides[idx]
+                          : MemRefType::kDynamicStrideOrOffset;
+  }
+  resOffset =
+      (aOffset == bOffset) ? aOffset : MemRefType::kDynamicStrideOrOffset;
+  return MemRefType::get(
+      resShape, aT.getElementType(),
+      makeStridedLinearLayoutMap(resStrides, resOffset, aT.getContext()));
+}
+
+/// Split a vector.transfer operation into an unmasked fastpath vector.transfer
+/// and a slowpath masked vector.transfer. If `ifOp` is not null and the result
+/// is `success, the `ifOp` points to the newly created conditional upon
+/// function return. To accomodate for the fact that the original
+/// vector.transfer indexing may be arbitrary and the slow path indexes @[0...0]
+/// in the temporary buffer, the scf.if op returns a view and values of type
+/// index. At this time, only vector.transfer_read is implemented.
+///
+/// Example (a 2-D vector.transfer_read):
+/// ```
+///    %1 = vector.transfer_read %0[...], %pad : memref<A...>, vector<...>
+/// ```
+/// is transformed into:
+/// ```
+///    %1:3 = scf.if (%inBounds) {
+///       scf.yield %0 : memref<A...>, index, index
+///     } else {
+///       %2 = vector.transfer_read %0[...], %pad : memref<A...>, vector<...>
+///       %3 = vector.type_cast %extra_alloc : memref<...> to
+///       memref<vector<...>> store %2, %3[] : memref<vector<...>> %4 =
+///       memref_cast %extra_alloc: memref<B...> to memref<A...> scf.yield %4 :
+///       memref<A...>, index, index
+//     }
+///    %0 = vector.transfer_read %1#0[%1#1, %1#2] {masked = [false ... false]}
+/// ```
+/// where `extra_alloc` is a top of the function alloca'ed buffer of one vector.
+///
+/// Preconditions:
+///  1. `xferOp.permutation_map()` must be a minor identity map
+///  2. the rank of the `xferOp.memref()` and the rank of the `xferOp.vector()`
+///  must be equal. This will be relaxed in the future but requires
+///  rank-reducing subviews.
+LogicalResult mlir::vector::splitFullAndPartialTransfer(
+    OpBuilder &b, VectorTransferOpInterface xferOp, scf::IfOp *ifOp) {
+  using namespace edsc;
+  using namespace edsc::intrinsics;
+
+  assert(succeeded(splitFullAndPartialTransferPrecondition(xferOp)) &&
+         "Expected splitFullAndPartialTransferPrecondition to hold");
+  auto xferReadOp = dyn_cast<vector::TransferReadOp>(xferOp.getOperation());
+
+  // TODO: add support for write case.
+  if (!xferReadOp)
+    return failure();
+
+  OpBuilder::InsertionGuard guard(b);
+  if (xferOp.memref().getDefiningOp())
+    b.setInsertionPointAfter(xferOp.memref().getDefiningOp());
+  else
+    b.setInsertionPoint(xferOp);
+  ScopedContext scope(b, xferOp.getLoc());
+  Value inBoundsCond = createScopedInBoundsCond(
+      cast<VectorTransferOpInterface>(xferOp.getOperation()));
+  if (!inBoundsCond)
+    return failure();
+
+  // Top of the function `alloc` for transient storage.
+  Value alloc;
+  {
+    FuncOp funcOp = xferOp.getParentOfType<FuncOp>();
+    OpBuilder::InsertionGuard guard(b);
+    b.setInsertionPointToStart(&funcOp.getRegion().front());
+    auto shape = xferOp.getVectorType().getShape();
+    Type elementType = xferOp.getVectorType().getElementType();
+    alloc = std_alloca(MemRefType::get(shape, elementType), ValueRange{},
+                       b.getI64IntegerAttr(32));
+  }
+
+  Value memref = xferOp.memref();
+  SmallVector<bool, 4> bools(xferOp.getTransferRank(), false);
+  auto unmaskedAttr = b.getBoolArrayAttr(bools);
+
+  MemRefType compatibleMemRefType = getCastCompatibleMemRefType(
+      xferOp.getMemRefType(), alloc.getType().cast<MemRefType>());
+
+  // Read case: full fill + partial copy -> unmasked vector.xfer_read.
+  Value zero = std_constant_index(0);
+  SmallVector<Type, 4> returnTypes(1 + xferOp.getTransferRank(),
+                                   b.getIndexType());
+  returnTypes[0] = compatibleMemRefType;
+  scf::IfOp fullPartialIfOp;
+  conditionBuilder(
+      returnTypes, inBoundsCond,
+      [&]() -> scf::ValueVector {
+        Value res = memref;
+        if (compatibleMemRefType != xferOp.getMemRefType())
+          res = std_memref_cast(memref, compatibleMemRefType);
+        scf::ValueVector viewAndIndices{res};
+        viewAndIndices.insert(viewAndIndices.end(), xferOp.indices().begin(),
+                              xferOp.indices().end());
+        return viewAndIndices;
+      },
+      [&]() -> scf::ValueVector {
+        Operation *newXfer =
+            ScopedContext::getBuilderRef().clone(*xferOp.getOperation());
+        Value vector = cast<VectorTransferOpInterface>(newXfer).vector();
+        std_store(vector, vector_type_cast(
+                              MemRefType::get({}, vector.getType()), alloc));
+
+        Value casted = std_memref_cast(alloc, compatibleMemRefType);
+        scf::ValueVector viewAndIndices{casted};
+        viewAndIndices.insert(viewAndIndices.end(), xferOp.getTransferRank(),
+                              zero);
+
+        return viewAndIndices;
+      },
+      &fullPartialIfOp);
+  if (ifOp)
+    *ifOp = fullPartialIfOp;
+
+  // Unmask the existing read op, it always reads from a full buffer.
+  for (unsigned i = 0, e = returnTypes.size(); i != e; ++i)
+    xferReadOp.setOperand(i, fullPartialIfOp.getResult(i));
+  xferOp.setAttr(vector::TransferReadOp::getMaskedAttrName(), unmaskedAttr);
+
+  return success();
+}
+
+LogicalResult mlir::vector::VectorTransferFullPartialRewriter::matchAndRewrite(
+    Operation *op, PatternRewriter &rewriter) const {
+  auto xferOp = dyn_cast<VectorTransferOpInterface>(op);
+  if (!xferOp || failed(splitFullAndPartialTransferPrecondition(xferOp)) ||
+      failed(filter(xferOp)))
+    return failure();
+  rewriter.startRootUpdate(xferOp);
+  if (succeeded(splitFullAndPartialTransfer(rewriter, xferOp))) {
+    rewriter.finalizeRootUpdate(xferOp);
+    return success();
+  }
+  rewriter.cancelRootUpdate(xferOp);
+  return failure();
+}
+
 // TODO: Add pattern to rewrite ExtractSlices(ConstantMaskOp).
 // TODO: Add this as DRR pattern.
 void mlir::vector::populateVectorToVectorTransformationPatterns(
diff --git a/mlir/test/Dialect/Vector/vector-transfer-full-partial-split.mlir b/mlir/test/Dialect/Vector/vector-transfer-full-partial-split.mlir
new file mode 100644
index 0000000000000..ef76247ee9d4b
--- /dev/null
+++ b/mlir/test/Dialect/Vector/vector-transfer-full-partial-split.mlir
@@ -0,0 +1,102 @@
+// RUN: mlir-opt %s -test-vector-transfer-full-partial-split | FileCheck %s
+
+// CHECK-DAG: #[[$map_p4:.*]] = affine_map<()[s0] -> (s0 + 4)>
+// CHECK-DAG: #[[$map_p8:.*]] = affine_map<()[s0] -> (s0 + 8)>
+// CHECK-DAG: #[[$map_2d_stride_1:.*]] = affine_map<(d0, d1)[s0, s1] -> (d0 * s1 + s0 + d1)>
+
+// CHECK-LABEL: split_vector_transfer_read_2d(
+//  CHECK-SAME: %[[A:[a-zA-Z0-9]*]]: memref
+//  CHECK-SAME: %[[i:[a-zA-Z0-9]*]]: index
+//  CHECK-SAME: %[[j:[a-zA-Z0-9]*]]: index
+func @split_vector_transfer_read_2d(%A: memref<?x8xf32>, %i: index, %j: index) -> vector<4x8xf32> {
+  %c0 = constant 0 : index
+  %f0 = constant 0.0 : f32
+
+  //  CHECK-DAG: %[[c0:.*]] = constant 0 : index
+  //  CHECK-DAG: %[[c8:.*]] = constant 8 : index
+  //  CHECK-DAG: %[[cst:.*]] = constant 0.000000e+00 : f32
+  // alloca for boundary full tile
+  //      CHECK: %[[alloc:.*]] = alloca() {alignment = 32 : i64} : memref<4x8xf32>
+  // %i + 4 <= dim(%A, 0)
+  //      CHECK: %[[idx0:.*]] = affine.apply #[[$map_p4]]()[%[[i]]]
+  //      CHECK: %[[d0:.*]] = dim %[[A]], %[[c0]] : memref<?x8xf32>
+  //      CHECK: %[[cmp0:.*]] = cmpi "sle", %[[idx0]], %[[d0]] : index
+  // %j + 8 <= dim(%A, 1)
+  //      CHECK: %[[idx1:.*]] = affine.apply #[[$map_p8]]()[%[[j]]]
+  //      CHECK: %[[cmp1:.*]] = cmpi "sle", %[[idx1]], %[[c8]] : index
+  // are both conds true
+  //      CHECK: %[[cond:.*]] = and %[[cmp0]], %[[cmp1]] : i1
+  //      CHECK: %[[ifres:.*]]:3 = scf.if %[[cond]] -> (memref<?x8xf32>, index, index) {
+  //               inBounds, just yield %A
+  //      CHECK:   scf.yield %[[A]], %[[i]], %[[j]] : memref<?x8xf32>, index, index
+  //      CHECK: } else {
+  //               slow path, fill tmp alloc and yield a memref_casted version of it
+  //      CHECK:   %[[slow:.*]] = vector.transfer_read %[[A]][%[[i]], %[[j]]], %cst :
+  // CHECK-SAME:     memref<?x8xf32>, vector<4x8xf32>
+  //      CHECK:   %[[cast_alloc:.*]] = vector.type_cast %[[alloc]] :
+  // CHECK-SAME:     memref<4x8xf32> to memref<vector<4x8xf32>>
+  //      CHECK:   store %[[slow]], %[[cast_alloc]][] : memref<vector<4x8xf32>>
+  //      CHECK:   %[[yielded:.*]] = memref_cast %[[alloc]] :
+  // CHECK-SAME:     memref<4x8xf32> to memref<?x8xf32>
+  //      CHECK:   scf.yield %[[yielded]], %[[c0]], %[[c0]] :
+  // CHECK-SAME:     memref<?x8xf32>, index, index
+  //      CHECK: }
+  //      CHECK: %[[res:.*]] = vector.transfer_read %[[ifres]]#0[%[[ifres]]#1, %[[ifres]]#2], %[[cst]]
+  // CHECK_SAME:   {masked = [false, false]} : memref<?x8xf32>, vector<4x8xf32>
+  %1 = vector.transfer_read %A[%i, %j], %f0 : memref<?x8xf32>, vector<4x8xf32>
+
+  // CHECK: return %[[res]] : vector<4x8xf32>
+  return %1: vector<4x8xf32>
+}
+
+// CHECK-LABEL: split_vector_transfer_read_strided_2d(
+//  CHECK-SAME: %[[A:[a-zA-Z0-9]*]]: memref
+//  CHECK-SAME: %[[i:[a-zA-Z0-9]*]]: index
+//  CHECK-SAME: %[[j:[a-zA-Z0-9]*]]: index
+func @split_vector_transfer_read_strided_2d(
+    %A: memref<7x8xf32, offset:?, strides:[?, 1]>,
+    %i: index, %j: index) -> vector<4x8xf32> {
+  %c0 = constant 0 : index
+  %f0 = constant 0.0 : f32
+
+  //  CHECK-DAG: %[[c0:.*]] = constant 0 : index
+  //  CHECK-DAG: %[[c7:.*]] = constant 7 : index
+  //  CHECK-DAG: %[[c8:.*]] = constant 8 : index
+  //  CHECK-DAG: %[[cst:.*]] = constant 0.000000e+00 : f32
+  // alloca for boundary full tile
+  //      CHECK: %[[alloc:.*]] = alloca() {alignment = 32 : i64} : memref<4x8xf32>
+  // %i + 4 <= dim(%A, 0)
+  //      CHECK: %[[idx0:.*]] = affine.apply #[[$map_p4]]()[%[[i]]]
+  //      CHECK: %[[cmp0:.*]] = cmpi "sle", %[[idx0]], %[[c7]] : index
+  // %j + 8 <= dim(%A, 1)
+  //      CHECK: %[[idx1:.*]] = affine.apply #[[$map_p8]]()[%[[j]]]
+  //      CHECK: %[[cmp1:.*]] = cmpi "sle", %[[idx1]], %[[c8]] : index
+  // are both conds true
+  //      CHECK: %[[cond:.*]] = and %[[cmp0]], %[[cmp1]] : i1
+  //      CHECK: %[[ifres:.*]]:3 = scf.if %[[cond]] -> (memref<?x8xf32, #[[$map_2d_stride_1]]>, index, index) {
+  //               inBounds but not cast-compatible: yield a memref_casted form of %A
+  //      CHECK:   %[[casted:.*]] = memref_cast %arg0 :
+  // CHECK-SAME:     memref<7x8xf32, #[[$map_2d_stride_1]]> to memref<?x8xf32, #[[$map_2d_stride_1]]>
+  //      CHECK:   scf.yield %[[casted]], %[[i]], %[[j]] :
+  // CHECK-SAME:     memref<?x8xf32, #[[$map_2d_stride_1]]>, index, index
+  //      CHECK: } else {
+  //               slow path, fill tmp alloc and yield a memref_casted version of it
+  //      CHECK:   %[[slow:.*]] = vector.transfer_read %[[A]][%[[i]], %[[j]]], %cst :
+  // CHECK-SAME:     memref<7x8xf32, #[[$map_2d_stride_1]]>, vector<4x8xf32>
+  //      CHECK:   %[[cast_alloc:.*]] = vector.type_cast %[[alloc]] :
+  // CHECK-SAME:     memref<4x8xf32> to memref<vector<4x8xf32>>
+  //      CHECK:   store %[[slow]], %[[cast_alloc]][] :
+  // CHECK-SAME:     memref<vector<4x8xf32>>
+  //      CHECK:   %[[yielded:.*]] = memref_cast %[[alloc]] :
+  // CHECK-SAME:     memref<4x8xf32> to memref<?x8xf32, #[[$map_2d_stride_1]]>
+  //      CHECK:   scf.yield %[[yielded]], %[[c0]], %[[c0]] :
+  // CHECK-SAME:     memref<?x8xf32, #[[$map_2d_stride_1]]>, index, index
+  //      CHECK: }
+  //      CHECK: %[[res:.*]] = vector.transfer_read {{.*}} {masked = [false, false]} :
+  // CHECK-SAME:   memref<?x8xf32, #[[$map_2d_stride_1]]>, vector<4x8xf32>
+  %1 = vector.transfer_read %A[%i, %j], %f0 :
+    memref<7x8xf32, offset:?, strides:[?, 1]>, vector<4x8xf32>
+
+  // CHECK: return %[[res]] : vector<4x8xf32>
+  return %1 : vector<4x8xf32>
+}
diff --git a/mlir/test/lib/Transforms/TestVectorTransforms.cpp b/mlir/test/lib/Transforms/TestVectorTransforms.cpp
index 2058706dcbdd3..0bba74e76385e 100644
--- a/mlir/test/lib/Transforms/TestVectorTransforms.cpp
+++ b/mlir/test/lib/Transforms/TestVectorTransforms.cpp
@@ -122,6 +122,17 @@ struct TestVectorUnrollingPatterns
   }
 };
 
+struct TestVectorTransferFullPartialSplitPatterns
+    : public PassWrapper<TestVectorTransferFullPartialSplitPatterns,
+                         FunctionPass> {
+  void runOnFunction() override {
+    MLIRContext *ctx = &getContext();
+    OwningRewritePatternList patterns;
+    patterns.insert<VectorTransferFullPartialRewriter>(ctx);
+    applyPatternsAndFoldGreedily(getFunction(), patterns);
+  }
+};
+
 } // end anonymous namespace
 
 namespace mlir {
@@ -141,5 +152,10 @@ void registerTestVectorConversions() {
   PassRegistration<TestVectorUnrollingPatterns> contractionUnrollingPass(
       "test-vector-unrolling-patterns",
       "Test conversion patterns to unroll contract ops in the vector dialect");
+
+  PassRegistration<TestVectorTransferFullPartialSplitPatterns>
+      vectorTransformFullPartialPass("test-vector-transfer-full-partial-split",
+                                     "Test conversion patterns to split "
+                                     "transfer ops via scf.if + linalg ops");
 }
 } // namespace mlir

From 98db27711d86d4085db4a4a8ff68f8baa1b094ef Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Mon, 3 Aug 2020 09:47:16 +0100
Subject: [PATCH 0884/1035] [LV] Do not check widening decision for instrs
 outside of loop.

No widening decisions will be computed for instructions outside the
loop. Do not try to get a widening decision. The load/store will be just
a scalar load, so treating at as normal should be fine I think.

Fixes PR46950.

Reviewed By: dmgreen

Differential Revision: https://reviews.llvm.org/D85087
---
 .../Transforms/Vectorize/LoopVectorize.cpp    |  2 +-
 .../pr46950-load-cast-context-crash.ll        | 25 +++++++++++++++++++
 2 files changed, 26 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/pr46950-load-cast-context-crash.ll

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 5fffcc8cf0f3a..33bd31f6b9833 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6463,7 +6463,7 @@ unsigned LoopVectorizationCostModel::getInstructionCost(Instruction *I,
       assert((isa<LoadInst>(I) || isa<StoreInst>(I)) &&
              "Expected a load or a store!");
 
-      if (VF == 1)
+      if (VF == 1 || !TheLoop->contains(I))
         return TTI::CastContextHint::Normal;
 
       switch (getWideningDecision(I, VF)) {
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/pr46950-load-cast-context-crash.ll b/llvm/test/Transforms/LoopVectorize/AArch64/pr46950-load-cast-context-crash.ll
new file mode 100644
index 0000000000000..e357acca3fbf5
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/pr46950-load-cast-context-crash.ll
@@ -0,0 +1,25 @@
+; RUN: opt -loop-vectorize %s -mtriple=arm64-apple-iphoneos -S | FileCheck %s
+
+; CHECK-LABEL: define void @test(
+; CHECK: vector.body
+
+define void @test(i64* %dst, i32* %src) {
+entry:
+  %l = load i32, i32* %src
+  br label %loop.ph
+
+loop.ph:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %loop.ph ], [ %iv.next, %loop ]
+  %l.cast = sext i32 %l to i64
+  %dst.idx = getelementptr i64, i64* %dst, i64 %iv
+  store i64 %l.cast, i64* %dst.idx
+  %iv.next = add nuw nsw i64 %iv, 1
+  %cmp9.us = icmp ult i64 %iv.next, 20
+  br i1 %cmp9.us, label %loop, label %exit
+
+exit:
+  ret void
+}

From 18d4069503e729442158476960a797df963cf293 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Lubo=C5=A1=20Lu=C5=88=C3=A1k?= <l.lunak@centrum.cz>
Date: Sun, 2 Aug 2020 17:44:24 +0200
Subject: [PATCH 0885/1035] fix lldb test on lib64 systems

Differential revision: https://reviews.llvm.org/D85096
---
 lldb/unittests/Expression/ClangParserTest.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lldb/unittests/Expression/ClangParserTest.cpp b/lldb/unittests/Expression/ClangParserTest.cpp
index 81f9ed839fcc1..4df557475314e 100644
--- a/lldb/unittests/Expression/ClangParserTest.cpp
+++ b/lldb/unittests/Expression/ClangParserTest.cpp
@@ -11,6 +11,7 @@
 #include "Plugins/ExpressionParser/Clang/ClangHost.h"
 #include "TestingSupport/SubsystemRAII.h"
 #include "TestingSupport/TestUtilities.h"
+#include "lldb/Host/Config.h"
 #include "lldb/Host/FileSystem.h"
 #include "lldb/Host/HostInfo.h"
 #include "lldb/Utility/FileSpec.h"
@@ -36,7 +37,7 @@ static std::string ComputeClangResourceDir(std::string lldb_shlib_path,
 TEST_F(ClangHostTest, ComputeClangResourceDirectory) {
 #if !defined(_WIN32)
   std::string path_to_liblldb = "/foo/bar/lib/";
-  std::string path_to_clang_dir = "/foo/bar/lib/clang/" CLANG_VERSION_STRING;
+  std::string path_to_clang_dir = "/foo/bar/lib" LLDB_LIBDIR_SUFFIX "/clang/" CLANG_VERSION_STRING;
 #else
   std::string path_to_liblldb = "C:\\foo\\bar\\lib";
   std::string path_to_clang_dir = "C:\\foo\\bar\\lib\\clang\\" CLANG_VERSION_STRING;

From 6d47431d7eeed44ae46dd1e58cf5d04e9210c048 Mon Sep 17 00:00:00 2001
From: Julian Gross <julian.gross@dfki.de>
Date: Fri, 31 Jul 2020 11:42:31 +0200
Subject: [PATCH 0886/1035] [mlir] Extended Buffer Assignment to support
 AllocaOps.

Added support for AllocaOps in Buffer Assignment.

Differential Revision: https://reviews.llvm.org/D85017
---
 mlir/lib/Transforms/BufferPlacement.cpp    |  15 +-
 mlir/test/Transforms/buffer-placement.mlir | 192 +++++++++++++++++++++
 2 files changed, 201 insertions(+), 6 deletions(-)

diff --git a/mlir/lib/Transforms/BufferPlacement.cpp b/mlir/lib/Transforms/BufferPlacement.cpp
index 0c24621c36668..66d175dc1f3e8 100644
--- a/mlir/lib/Transforms/BufferPlacement.cpp
+++ b/mlir/lib/Transforms/BufferPlacement.cpp
@@ -264,12 +264,15 @@ class BufferPlacement {
       opInterface.getEffects(effects);
 
       SmallVector<MemoryEffects::EffectInstance, 2> allocateResultEffects;
-      llvm::copy_if(effects, std::back_inserter(allocateResultEffects),
-                    [=](MemoryEffects::EffectInstance &it) {
-                      Value value = it.getValue();
-                      return isa<MemoryEffects::Allocate>(it.getEffect()) &&
-                             value && value.isa<OpResult>();
-                    });
+      llvm::copy_if(
+          effects, std::back_inserter(allocateResultEffects),
+          [=](MemoryEffects::EffectInstance &it) {
+            Value value = it.getValue();
+            return isa<MemoryEffects::Allocate>(it.getEffect()) && value &&
+                   value.isa<OpResult>() &&
+                   it.getResource() !=
+                       SideEffects::AutomaticAllocationScopeResource::get();
+          });
       // If there is one result only, we will be able to move the allocation and
       // (possibly existing) deallocation ops.
       if (allocateResultEffects.size() != 1)
diff --git a/mlir/test/Transforms/buffer-placement.mlir b/mlir/test/Transforms/buffer-placement.mlir
index c3bce4ea54583..2ac212c7d681f 100644
--- a/mlir/test/Transforms/buffer-placement.mlir
+++ b/mlir/test/Transforms/buffer-placement.mlir
@@ -933,3 +933,195 @@ func @subview(%arg0 : index, %arg1 : index, %arg2 : memref<?x?xf32>) {
 // CHECK-NEXT: linalg.copy
 // CHECK-NEXT: dealloc %[[ALLOC]]
 // CHECK-NEXT: return
+
+// -----
+
+#map0 = affine_map<(d0) -> (d0)>
+
+// CHECK-LABEL: func @condBranchAlloca
+func @condBranchAlloca(%arg0: i1, %arg1: memref<2xf32>, %arg2: memref<2xf32>) {
+  cond_br %arg0, ^bb1, ^bb2
+^bb1:
+  br ^bb3(%arg1 : memref<2xf32>)
+^bb2:
+  %0 = alloca() : memref<2xf32>
+  linalg.generic {
+    args_in = 1 : i64,
+    args_out = 1 : i64,
+    indexing_maps = [#map0, #map0],
+    iterator_types = ["parallel"]} %arg1, %0 {
+  ^bb0(%gen1_arg0: f32, %gen1_arg1: f32):
+    %tmp1 = exp %gen1_arg0 : f32
+    linalg.yield %tmp1 : f32
+  }: memref<2xf32>, memref<2xf32>
+  br ^bb3(%0 : memref<2xf32>)
+^bb3(%1: memref<2xf32>):
+  "linalg.copy"(%1, %arg2) : (memref<2xf32>, memref<2xf32>) -> ()
+  return
+}
+
+// CHECK-NEXT: cond_br
+//      CHECK: %[[ALLOCA:.*]] = alloca()
+//      CHECK: br ^bb3(%[[ALLOCA:.*]])
+// CHECK-NEXT: ^bb3
+// CHECK-NEXT: linalg.copy
+// CHECK-NEXT: return
+
+// -----
+
+#map0 = affine_map<(d0) -> (d0)>
+
+// CHECK-LABEL: func @ifElseAlloca
+func @ifElseAlloca(%arg0: i1, %arg1: memref<2xf32>, %arg2: memref<2xf32>) {
+  %0 = alloc() : memref<2xf32>
+  linalg.generic {
+    args_in = 1 : i64,
+    args_out = 1 : i64,
+    indexing_maps = [#map0, #map0],
+    iterator_types = ["parallel"]} %arg1, %0 {
+  ^bb0(%gen1_arg0: f32, %gen1_arg1: f32):
+    %tmp1 = exp %gen1_arg0 : f32
+    linalg.yield %tmp1 : f32
+  }: memref<2xf32>, memref<2xf32>
+  cond_br %arg0,
+    ^bb1(%arg1, %0 : memref<2xf32>, memref<2xf32>),
+    ^bb2(%0, %arg1 : memref<2xf32>, memref<2xf32>)
+^bb1(%1: memref<2xf32>, %2: memref<2xf32>):
+  br ^bb3(%1, %2 : memref<2xf32>, memref<2xf32>)
+^bb2(%3: memref<2xf32>, %4: memref<2xf32>):
+  br ^bb3(%3, %4 : memref<2xf32>, memref<2xf32>)
+^bb3(%5: memref<2xf32>, %6: memref<2xf32>):
+  %7 = alloca() : memref<2xf32>
+  linalg.generic {
+    args_in = 1 : i64,
+    args_out = 1 : i64,
+    indexing_maps = [#map0, #map0],
+    iterator_types = ["parallel"]} %5, %7 {
+  ^bb0(%gen2_arg0: f32, %gen2_arg1: f32):
+    %tmp2 = exp %gen2_arg0 : f32
+    linalg.yield %tmp2 : f32
+  }: memref<2xf32>, memref<2xf32>
+  "linalg.copy"(%7, %arg2) : (memref<2xf32>, memref<2xf32>) -> ()
+  return
+}
+
+// CHECK-NEXT: %[[ALLOC:.*]] = alloc()
+// CHECK-NEXT: linalg.generic
+//      CHECK: %[[ALLOCA:.*]] = alloca()
+// CHECK-NEXT: linalg.generic
+//      CHECK: dealloc %[[ALLOC]]
+//      CHECK: linalg.copy
+// CHECK-NEXT: return
+
+// -----
+
+#map0 = affine_map<(d0) -> (d0)>
+
+// CHECK-LABEL: func @ifElseNestedAlloca
+func @ifElseNestedAlloca(%arg0: i1, %arg1: memref<2xf32>, %arg2: memref<2xf32>) {
+  %0 = alloca() : memref<2xf32>
+  linalg.generic {
+    args_in = 1 : i64,
+    args_out = 1 : i64,
+    indexing_maps = [#map0, #map0],
+    iterator_types = ["parallel"]} %arg1, %0 {
+  ^bb0(%gen1_arg0: f32, %gen1_arg1: f32):
+    %tmp1 = exp %gen1_arg0 : f32
+    linalg.yield %tmp1 : f32
+  }: memref<2xf32>, memref<2xf32>
+  cond_br %arg0,
+    ^bb1(%arg1, %0 : memref<2xf32>, memref<2xf32>),
+    ^bb2(%0, %arg1 : memref<2xf32>, memref<2xf32>)
+^bb1(%1: memref<2xf32>, %2: memref<2xf32>):
+  br ^bb5(%1, %2 : memref<2xf32>, memref<2xf32>)
+^bb2(%3: memref<2xf32>, %4: memref<2xf32>):
+  cond_br %arg0, ^bb3(%3 : memref<2xf32>), ^bb4(%4 : memref<2xf32>)
+^bb3(%5: memref<2xf32>):
+  br ^bb5(%5, %3 : memref<2xf32>, memref<2xf32>)
+^bb4(%6: memref<2xf32>):
+  br ^bb5(%3, %6 : memref<2xf32>, memref<2xf32>)
+^bb5(%7: memref<2xf32>, %8: memref<2xf32>):
+  %9 = alloc() : memref<2xf32>
+  linalg.generic {
+    args_in = 1 : i64,
+    args_out = 1 : i64,
+    indexing_maps = [#map0, #map0],
+    iterator_types = ["parallel"]} %7, %9 {
+  ^bb0(%gen2_arg0: f32, %gen2_arg1: f32):
+    %tmp2 = exp %gen2_arg0 : f32
+    linalg.yield %tmp2 : f32
+  }: memref<2xf32>, memref<2xf32>
+  "linalg.copy"(%9, %arg2) : (memref<2xf32>, memref<2xf32>) -> ()
+  return
+}
+
+// CHECK-NEXT: %[[ALLOCA:.*]] = alloca()
+// CHECK-NEXT: linalg.generic
+//      CHECK: %[[ALLOC:.*]] = alloc()
+// CHECK-NEXT: linalg.generic
+//      CHECK: linalg.copy
+// CHECK-NEXT: dealloc %[[ALLOC]]
+// CHECK-NEXT: return
+
+// -----
+
+#map0 = affine_map<(d0) -> (d0)>
+
+// CHECK-LABEL: func @nestedRegionsAndCondBranchAlloca
+func @nestedRegionsAndCondBranchAlloca(%arg0: i1, %arg1: memref<2xf32>, %arg2: memref<2xf32>) {
+  cond_br %arg0, ^bb1, ^bb2
+^bb1:
+  br ^bb3(%arg1 : memref<2xf32>)
+^bb2:
+  %0 = alloc() : memref<2xf32>
+  linalg.generic {args_in = 1 : i64, args_out = 1 : i64, indexing_maps = [#map0, #map0], iterator_types = ["parallel"]} %arg1, %0 {
+  ^bb0(%gen1_arg0: f32, %gen1_arg1: f32):
+    %1 = alloca() : memref<2xf32>
+    linalg.generic {args_in = 1 : i64, args_out = 1 : i64, indexing_maps = [#map0, #map0], iterator_types = ["parallel"]} %arg1, %1 {
+    ^bb0(%gen2_arg0: f32, %gen2_arg1: f32):
+      %tmp2 = exp %gen2_arg0 : f32
+      linalg.yield %tmp2 : f32
+    }: memref<2xf32>, memref<2xf32>
+    %tmp1 = exp %gen1_arg0 : f32
+    linalg.yield %tmp1 : f32
+  }: memref<2xf32>, memref<2xf32>
+  br ^bb3(%0 : memref<2xf32>)
+^bb3(%1: memref<2xf32>):
+  "linalg.copy"(%1, %arg2) : (memref<2xf32>, memref<2xf32>) -> ()
+  return
+}
+//      CHECK: (%[[cond:.*]]: {{.*}}, %[[ARG1:.*]]: {{.*}}, %{{.*}}: {{.*}})
+// CHECK-NEXT:   %[[ALLOC:.*]] = alloc()
+// CHECK-NEXT:   cond_br %[[cond]], ^[[BB1:.*]], ^[[BB2:.*]]
+//      CHECK: ^[[BB2]]:
+// CHECK-NEXT:   linalg.generic {{{.*}}} %[[ARG1]], %[[ALLOC]]
+//      CHECK:     %[[ALLOCA:.*]] = alloca()
+// CHECK-NEXT:     linalg.generic {{{.*}}} %[[ARG1]], %[[ALLOCA]]
+//      CHECK:     %{{.*}} = exp
+//      CHECK:  ^[[BB3:.*]]({{.*}}):
+//      CHECK:  linalg.copy
+// CHECK-NEXT:  dealloc %[[ALLOC]]
+
+// -----
+
+// CHECK-LABEL: func @nestedRegionControlFlowAlloca
+func @nestedRegionControlFlowAlloca(
+  %arg0 : index,
+  %arg1 : index) -> memref<?x?xf32> {
+  %0 = cmpi "eq", %arg0, %arg1 : index
+  %1 = alloc(%arg0, %arg0) : memref<?x?xf32>
+  %2 = scf.if %0 -> (memref<?x?xf32>) {
+    scf.yield %1 : memref<?x?xf32>
+  } else {
+    %3 = alloca(%arg0, %arg1) : memref<?x?xf32>
+    scf.yield %1 : memref<?x?xf32>
+  }
+  return %2 : memref<?x?xf32>
+}
+
+//      CHECK: %[[ALLOC0:.*]] = alloc(%arg0, %arg0)
+// CHECK-NEXT: %[[ALLOC1:.*]] = scf.if
+//      CHECK: scf.yield %[[ALLOC0]]
+//      CHECK: %[[ALLOCA:.*]] = alloca(%arg0, %arg1)
+// CHECK-NEXT: scf.yield %[[ALLOC0]]
+//      CHECK: return %[[ALLOC1]]

From d919ae9df8721a56c8457fd5f9cfd50a71c87262 Mon Sep 17 00:00:00 2001
From: Georgii Rymar <grimar@accesssoftek.com>
Date: Thu, 23 Jul 2020 15:26:23 +0300
Subject: [PATCH 0887/1035] [yaml2obj] - Add a support for "<none>" value for
 all optional fields.

It implements an approach suggested in the D84398 thread.

With it the following:

```
Sections:
  - Name:   .bar
    Type:   SHT_PROGBITS
    Offset: [[MACRO=<none>]]
```

works just like the `Offset` key was not specified.
It is useful for tests that want to have a default value for a field and to
have a way to override it at the same time.

Differential revision: https://reviews.llvm.org/D84526
---
 llvm/include/llvm/Support/YAMLTraits.h       | 53 +++++++++++++-------
 llvm/test/tools/yaml2obj/ELF/none-value.yaml | 45 +++++++++++++++++
 2 files changed, 80 insertions(+), 18 deletions(-)
 create mode 100644 llvm/test/tools/yaml2obj/ELF/none-value.yaml

diff --git a/llvm/include/llvm/Support/YAMLTraits.h b/llvm/include/llvm/Support/YAMLTraits.h
index 44e34a4a09b46..e52bf7892d711 100644
--- a/llvm/include/llvm/Support/YAMLTraits.h
+++ b/llvm/include/llvm/Support/YAMLTraits.h
@@ -902,24 +902,7 @@ class IO {
   template <typename T, typename Context>
   void processKeyWithDefault(const char *Key, Optional<T> &Val,
                              const Optional<T> &DefaultValue, bool Required,
-                             Context &Ctx) {
-    assert(DefaultValue.hasValue() == false &&
-           "Optional<T> shouldn't have a value!");
-    void *SaveInfo;
-    bool UseDefault = true;
-    const bool sameAsDefault = outputting() && !Val.hasValue();
-    if (!outputting() && !Val.hasValue())
-      Val = T();
-    if (Val.hasValue() &&
-        this->preflightKey(Key, Required, sameAsDefault, UseDefault,
-                           SaveInfo)) {
-      yamlize(*this, Val.getValue(), Required, Ctx);
-      this->postflightKey(SaveInfo);
-    } else {
-      if (UseDefault)
-        Val = DefaultValue;
-    }
-  }
+                             Context &Ctx);
 
   template <typename T, typename Context>
   void processKeyWithDefault(const char *Key, T &Val, const T &DefaultValue,
@@ -1625,6 +1608,40 @@ class Output : public IO {
   StringRef PaddingBeforeContainer;
 };
 
+template <typename T, typename Context>
+void IO::processKeyWithDefault(const char *Key, Optional<T> &Val,
+                               const Optional<T> &DefaultValue, bool Required,
+                               Context &Ctx) {
+  assert(DefaultValue.hasValue() == false &&
+         "Optional<T> shouldn't have a value!");
+  void *SaveInfo;
+  bool UseDefault = true;
+  const bool sameAsDefault = outputting() && !Val.hasValue();
+  if (!outputting() && !Val.hasValue())
+    Val = T();
+  if (Val.hasValue() &&
+      this->preflightKey(Key, Required, sameAsDefault, UseDefault, SaveInfo)) {
+
+    // When reading an Optional<X> key from a YAML description, we allow the
+    // special "<none>" value, which can be used to specify that no value was
+    // requested, i.e. the DefaultValue will be assigned. The DefaultValue is
+    // usually None.
+    bool IsNone = false;
+    if (!outputting())
+      if (auto *Node = dyn_cast<ScalarNode>(((Input *)this)->getCurrentNode()))
+        IsNone = Node->getRawValue() == "<none>";
+
+    if (IsNone)
+      Val = DefaultValue;
+    else
+      yamlize(*this, Val.getValue(), Required, Ctx);
+    this->postflightKey(SaveInfo);
+  } else {
+    if (UseDefault)
+      Val = DefaultValue;
+  }
+}
+
 /// YAML I/O does conversion based on types. But often native data types
 /// are just a typedef of built in intergral types (e.g. int).  But the C++
 /// type matching system sees through the typedef and all the typedefed types
diff --git a/llvm/test/tools/yaml2obj/ELF/none-value.yaml b/llvm/test/tools/yaml2obj/ELF/none-value.yaml
new file mode 100644
index 0000000000000..786a9b53aba78
--- /dev/null
+++ b/llvm/test/tools/yaml2obj/ELF/none-value.yaml
@@ -0,0 +1,45 @@
+## We have a special "<none>" value for all keys that are implemented
+## as Optional<> in the code. Setting a key to "<none>" means no-op and
+## works in the same way as when a field was not specified at all.
+
+## Test a few keys for which the "<none>" value is supported.
+## We do not test all possible keys, because it would be too verbose.
+## It reasonable to test all keys for a section, because normally many
+## of them would conflict or intersect when specified together.
+# RUN: yaml2obj %s --docnum=1 -o %t-none
+# RUN: yaml2obj %s --docnum=2 -o %t-base
+# RUN: cmp %t-none %t-base
+
+## We do not use the TEST macro. It exists to
+## demonstrate the expected use case for the <none> word.
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_REL
+  Machine: EM_X86_64
+Sections:
+  - Name:         .bar
+    Type:         SHT_PROGBITS
+    Offset:       [[TEST=<none>]]
+    Address:      [[TEST=<none>]]
+    Content:      [[TEST=<none>]]
+    Size:         [[TEST=<none>]]
+    ContentArray: [[TEST=<none>]]
+    Info:         [[TEST=<none>]]
+    EntSize:      [[TEST=<none>]]
+    ShName:       [[TEST=<none>]]
+    ShOffset:     [[TEST=<none>]]
+    ShSize:       [[TEST=<none>]]
+    ShFlags:      [[TEST=<none>]]
+
+## The same document, but all fields that were set to <none> are removed.
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_REL
+  Machine: EM_X86_64
+Sections:
+  - Name: .bar
+    Type: SHT_PROGBITS

From d76057c1fe6a368e9e422cf586e09fba827c96e1 Mon Sep 17 00:00:00 2001
From: Ilya Golovenko <ilya.golovenko@huawei.com>
Date: Mon, 3 Aug 2020 11:34:14 +0200
Subject: [PATCH 0888/1035] Add document outline symbols from unnamed contexts,
 e.g. extern "C".

It is necessary to traverse children of unnamed declaration contexts
to get symbols which are currently missing in document outline, e.g.:

extern "C" {
void foo();
}

Reviewed By: kadircet

Differential Revision: https://reviews.llvm.org/D84839
---
 clang-tools-extra/clangd/FindSymbols.cpp      | 31 ++++++++++++-----
 .../clangd/unittests/FindSymbolsTests.cpp     | 34 +++++++++++++++++++
 2 files changed, 56 insertions(+), 9 deletions(-)

diff --git a/clang-tools-extra/clangd/FindSymbols.cpp b/clang-tools-extra/clangd/FindSymbols.cpp
index f5d6a95aa713d..2471656988250 100644
--- a/clang-tools-extra/clangd/FindSymbols.cpp
+++ b/clang-tools-extra/clangd/FindSymbols.cpp
@@ -188,7 +188,7 @@ class DocumentOutline {
   }
 
 private:
-  enum class VisitKind { No, OnlyDecl, DeclAndChildren };
+  enum class VisitKind { No, OnlyDecl, OnlyChildren, DeclAndChildren };
 
   void traverseDecl(Decl *D, std::vector<DocumentSymbol> &Results) {
     if (auto *Templ = llvm::dyn_cast<TemplateDecl>(D)) {
@@ -196,18 +196,25 @@ class DocumentOutline {
       if (auto *TD = Templ->getTemplatedDecl())
         D = TD;
     }
-    auto *ND = llvm::dyn_cast<NamedDecl>(D);
-    if (!ND)
-      return;
-    VisitKind Visit = shouldVisit(ND);
+
+    VisitKind Visit = shouldVisit(D);
     if (Visit == VisitKind::No)
       return;
-    llvm::Optional<DocumentSymbol> Sym = declToSym(AST.getASTContext(), *ND);
+
+    if (Visit == VisitKind::OnlyChildren)
+      return traverseChildren(D, Results);
+
+    auto *ND = llvm::cast<NamedDecl>(D);
+    auto Sym = declToSym(AST.getASTContext(), *ND);
     if (!Sym)
       return;
-    if (Visit == VisitKind::DeclAndChildren)
-      traverseChildren(D, Sym->children);
     Results.push_back(std::move(*Sym));
+
+    if (Visit == VisitKind::OnlyDecl)
+      return;
+
+    assert(Visit == VisitKind::DeclAndChildren && "Unexpected VisitKind");
+    traverseChildren(ND, Results.back().children);
   }
 
   void traverseChildren(Decl *D, std::vector<DocumentSymbol> &Results) {
@@ -218,10 +225,16 @@ class DocumentOutline {
       traverseDecl(C, Results);
   }
 
-  VisitKind shouldVisit(NamedDecl *D) {
+  VisitKind shouldVisit(Decl *D) {
     if (D->isImplicit())
       return VisitKind::No;
 
+    if (llvm::isa<LinkageSpecDecl>(D) || llvm::isa<ExportDecl>(D))
+      return VisitKind::OnlyChildren;
+
+    if (!llvm::isa<NamedDecl>(D))
+      return VisitKind::No;
+
     if (auto Func = llvm::dyn_cast<FunctionDecl>(D)) {
       // Some functions are implicit template instantiations, those should be
       // ignored.
diff --git a/clang-tools-extra/clangd/unittests/FindSymbolsTests.cpp b/clang-tools-extra/clangd/unittests/FindSymbolsTests.cpp
index 07c42fcf20304..8576e11a5f21a 100644
--- a/clang-tools-extra/clangd/unittests/FindSymbolsTests.cpp
+++ b/clang-tools-extra/clangd/unittests/FindSymbolsTests.cpp
@@ -429,6 +429,40 @@ TEST(DocumentSymbols, ExternSymbol) {
   EXPECT_THAT(getSymbols(TU.build()), IsEmpty());
 }
 
+TEST(DocumentSymbols, ExternContext) {
+  TestTU TU;
+  TU.Code = R"cpp(
+      extern "C" {
+      void foo();
+      class Foo {};
+      }
+      namespace ns {
+        extern "C" {
+        void bar();
+        class Bar {};
+        }
+      })cpp";
+
+  EXPECT_THAT(getSymbols(TU.build()),
+              ElementsAre(WithName("foo"), WithName("Foo"),
+                          AllOf(WithName("ns"),
+                                Children(WithName("bar"), WithName("Bar")))));
+}
+
+TEST(DocumentSymbols, ExportContext) {
+  TestTU TU;
+  TU.ExtraArgs = {"-std=c++20"};
+  TU.Code = R"cpp(
+      export module test;
+      export {
+      void foo();
+      class Foo {};
+      })cpp";
+
+  EXPECT_THAT(getSymbols(TU.build()),
+              ElementsAre(WithName("foo"), WithName("Foo")));
+}
+
 TEST(DocumentSymbols, NoLocals) {
   TestTU TU;
   TU.Code = R"cpp(

From 7e32797552c69155676e45ec7d39f948779daa92 Mon Sep 17 00:00:00 2001
From: Georgii Rymar <grimar@accesssoftek.com>
Date: Mon, 27 Jul 2020 16:03:03 +0300
Subject: [PATCH 0889/1035] [llvm-readobj] - Don't call `unwrapOrErr` in
 `findSectionByName`.

We have a `findSectionByName` helper that tries to find a section
by it name. It is used in a few places, but never tested.

I'd like to reuse this helper for a different place.
For this, I've changed it to return Expected<> and now it
doesn't use `unwrapOrErr` anymore. It also now a member of
Dumper class and might report warnings.

Differential revision: https://reviews.llvm.org/D84651
---
 .../tools/llvm-readobj/ELF/mips-abiflags.test |  28 ++++-
 .../test/tools/llvm-readobj/ELF/mips-got.test |  32 ++++++
 .../llvm-readobj/ELF/mips-options-sec.test    |  36 +++++++
 .../tools/llvm-readobj/ELF/mips-reginfo.test  |  35 ++++++
 llvm/tools/llvm-readobj/ELFDumper.cpp         | 101 +++++++++++-------
 5 files changed, 193 insertions(+), 39 deletions(-)

diff --git a/llvm/test/tools/llvm-readobj/ELF/mips-abiflags.test b/llvm/test/tools/llvm-readobj/ELF/mips-abiflags.test
index f4432efb6aeba..791e3e9cc6e69 100644
--- a/llvm/test/tools/llvm-readobj/ELF/mips-abiflags.test
+++ b/llvm/test/tools/llvm-readobj/ELF/mips-abiflags.test
@@ -338,7 +338,7 @@ Sections: []
 # RUN: llvm-readelf -A %t.err1 2>&1 | FileCheck %s -DFILE=%t.err1 --check-prefix=CONTENT-ERR
 # RUN: llvm-readobj -A %t.err1 2>&1 | FileCheck %s -DFILE=%t.err1 --check-prefix=CONTENT-ERR
 
-# CONTENT-ERR:      warning: '[[FILE]]': unable to read the .MIPS.abiflags section: section [index 1] has a sh_offset (0xffffffff) + sh_size (0x18) that is greater than the file size (0x240)
+# CONTENT-ERR:      warning: '[[FILE]]': unable to read the .MIPS.abiflags section: section [index 2] has a sh_offset (0xffffffff) + sh_size (0x18) that is greater than the file size (0x2c0)
 # CONTENT-ERR-NEXT: There is no .MIPS.options section in the file.
 # CONTENT-ERR-NEXT: There is no .reginfo section in the file.
 
@@ -349,12 +349,17 @@ FileHeader:
   Type:    ET_REL
   Machine: EM_MIPS
 Sections:
+  - Type:   SHT_PROGBITS
+    ShName: [[NAME=<none>]]
   - Name:     .MIPS.abiflags
     Type:     SHT_MIPS_ABIFLAGS
     ISA:      MIPS32
     Offset:   0x100
     ShOffset: [[SHOFFSET=0x100]]
     ShSize:   [[SHSIZE=24]]
+    ShName: [[ABIFLAGSNAME=<none>]]
+  - Type:   SHT_PROGBITS
+    ShName: [[NAME=<none>]]
 
 ## Check we report a warning when the .MIPS.abiflags section has an unexpected size.
 # RUN: yaml2obj --docnum=3 -DSHSIZE=23 %s -o %t.err2
@@ -364,3 +369,24 @@ Sections:
 # SIZE-ERR:      warning: '[[FILE]]': unable to read the .MIPS.abiflags section: it has a wrong size (23)
 # SIZE-ERR-NEXT: There is no .MIPS.options section in the file.
 # SIZE-ERR-NEXT: There is no .reginfo section in the file.
+
+## Check that we try to dump the .MIPS.abiflags section when we are able to locate it by name.
+# RUN: yaml2obj --docnum=3 -DNAME=0xffff %s -o %t.err3
+# RUN: llvm-readelf -A %t.err3 2>&1 | \
+# RUN:   FileCheck %s -DFILE=%t.err3 --check-prefixes=NAME-ERR-FOUND,NAME-ERR-FOUND-GNU --implicit-check-not=warning:
+# RUN: llvm-readobj -A %t.err3 2>&1 | \
+# RUN:   FileCheck %s -DFILE=%t.err3 --check-prefixes=NAME-ERR-FOUND,NAME-ERR-FOUND-LLVM --implicit-check-not=warning:
+
+# NAME-ERR-FOUND:           warning: '[[FILE]]': unable to read the name of SHT_PROGBITS section with index 1: a section [index 1] has an invalid sh_name (0xffff) offset which goes past the end of the section name string table
+# NAME-ERR-FOUND-GNU-NEXT:  MIPS ABI Flags Version: 0
+# NAME-ERR-FOUND-LLVM-NEXT: MIPS ABI Flags {
+# NAME-ERR-FOUND:           warning: '[[FILE]]': unable to read the name of SHT_PROGBITS section with index 3: a section [index 3] has an invalid sh_name (0xffff) offset which goes past the end of the section name string table
+
+## Check we report a warning when we are unable to find the .MIPS.abiflags section due to an error.
+# RUN: yaml2obj --docnum=3 -DABIFLAGSNAME=0xffff %s -o %t.err4
+# RUN: llvm-readelf -A %t.err4 2>&1 | \
+# RUN:   FileCheck %s -DFILE=%t.err4 --check-prefix=NAME-ERR-NOTFOUND --implicit-check-not=warning: --implicit-check-not="MIPS ABI Flags"
+# RUN: llvm-readobj -A %t.err4 2>&1 | \
+# RUN:   FileCheck %s -DFILE=%t.err4 --check-prefix=NAME-ERR-NOTFOUND --implicit-check-not=warning: --implicit-check-not="MIPS ABI Flags"
+
+# NAME-ERR-NOTFOUND: warning: '[[FILE]]': unable to read the name of SHT_MIPS_ABIFLAGS section with index 2: a section [index 2] has an invalid sh_name (0xffff) offset which goes past the end of the section name string table
diff --git a/llvm/test/tools/llvm-readobj/ELF/mips-got.test b/llvm/test/tools/llvm-readobj/ELF/mips-got.test
index cfbf1c4f37a3e..7475a6d57d578 100644
--- a/llvm/test/tools/llvm-readobj/ELF/mips-got.test
+++ b/llvm/test/tools/llvm-readobj/ELF/mips-got.test
@@ -547,3 +547,35 @@ Sections:
       - Tag:   DT_PLTGOT
         Value: [[VAL2=0]]
 DynamicSymbols: []
+
+## Check that we do not report a warning about the .got section when we are able to locate it by name.
+# RUN: yaml2obj --docnum=3 -DNAME=0xffff %s -o %t.err6.o
+# RUN: llvm-readobj -A %t.err6.o 2>&1 | \
+# RUN:   FileCheck %s -DFILE=%t.err6.o -check-prefix=NAME-ERR-FOUND --implicit-check-not=warning:
+# RUN: llvm-readelf -A %t.err6.o 2>&1 | \
+# RUN:   FileCheck %s -DFILE=%t.err6.o -check-prefix=NAME-ERR-FOUND --implicit-check-not=warning:
+
+# NAME-ERR-FOUND:      warning: '[[FILE]]': unable to read the name of SHT_PROGBITS section with index 1: a section [index 1] has an invalid sh_name (0xffff) offset which goes past the end of the section name string table
+# NAME-ERR-FOUND-NEXT: warning: '[[FILE]]': unable to read the name of SHT_PROGBITS section with index 3: a section [index 3] has an invalid sh_name (0xffff) offset which goes past the end of the section name string table
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_REL
+  Machine: EM_MIPS
+Sections:
+  - Type:   SHT_PROGBITS
+    ShName: [[NAME=<none>]]
+  - Name:   .got
+    Type:   SHT_PROGBITS
+    ShName: [[GOTNAME=<none>]]
+  - Type:   SHT_PROGBITS
+    ShName: [[NAME=<none>]]
+
+## Check we report a warning when we are unable to find the .got section due to an error.
+# RUN: yaml2obj --docnum=3 -DGOTNAME=0xffff %s -o %t.err7.o
+# RUN: llvm-readelf -A %t.err7.o 2>&1 | FileCheck %s -DFILE=%t.err7.o --check-prefix=NAME-ERR-NOTFOUND --implicit-check-not=warning:
+# RUN: llvm-readobj -A %t.err7.o 2>&1 | FileCheck %s -DFILE=%t.err7.o --check-prefix=NAME-ERR-NOTFOUND --implicit-check-not=warning:
+
+# NAME-ERR-NOTFOUND:      warning: '[[FILE]]': unable to read the name of SHT_PROGBITS section with index 2: a section [index 2] has an invalid sh_name (0xffff) offset which goes past the end of the section name string table
diff --git a/llvm/test/tools/llvm-readobj/ELF/mips-options-sec.test b/llvm/test/tools/llvm-readobj/ELF/mips-options-sec.test
index f53f04c55e9bf..ef5ece472821e 100644
--- a/llvm/test/tools/llvm-readobj/ELF/mips-options-sec.test
+++ b/llvm/test/tools/llvm-readobj/ELF/mips-options-sec.test
@@ -12,3 +12,39 @@
 # CHECK-NEXT:     Co-Proc Mask3: 0x0
 # CHECK-NEXT:   }
 # CHECK-NEXT: }
+
+## Check that we try to dump the .MIPS.options section when we are able to locate it by name.
+# RUN: yaml2obj --docnum=1 -DNAME=0xffff %s -o %t.err1
+# RUN: llvm-readelf -A %t.err1 2>&1 | \
+# RUN:   FileCheck %s -DFILE=%t.err1 --check-prefix=NAME-ERR-FOUND --implicit-check-not=warning:
+# RUN: llvm-readobj -A %t.err1 2>&1 | \
+# RUN:   FileCheck %s -DFILE=%t.err1 --check-prefix=NAME-ERR-FOUND --implicit-check-not=warning:
+
+# NAME-ERR-FOUND:      warning: '[[FILE]]': unable to read the name of SHT_PROGBITS section with index 1: a section [index 1] has an invalid sh_name (0xffff) offset which goes past the end of the section name string table
+# NAME-ERR-FOUND-NEXT: warning: '[[FILE]]': unable to read the name of SHT_PROGBITS section with index 3: a section [index 3] has an invalid sh_name (0xffff) offset which goes past the end of the section name string table
+# NAME-ERR-FOUND:      MIPS Options {
+# NAME-ERR-FOUND-NEXT: }
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_REL
+  Machine: EM_MIPS
+Sections:
+  - Type:   SHT_PROGBITS
+    ShName: [[NAME=<none>]]
+  - Name:   .MIPS.options
+    Type:   SHT_MIPS_OPTIONS
+    ShName: [[OPTNAME=<none>]]
+  - Type:   SHT_PROGBITS
+    ShName: [[NAME=<none>]]
+
+## Check we report a warning when we are unable to find the .MIPS.options section due to an error.
+# RUN: yaml2obj --docnum=1 -DOPTNAME=0xffff %s -o %t.err2
+# RUN: llvm-readelf -A %t.err2 2>&1 | \
+# RUN:   FileCheck %s -DFILE=%t.err2 --check-prefix=NAME-ERR-NOTFOUND --implicit-check-not=warning: --implicit-check-not="MIPS Options"
+# RUN: llvm-readobj -A %t.err2 2>&1 | \
+# RUN:   FileCheck %s -DFILE=%t.err2 --check-prefix=NAME-ERR-NOTFOUND --implicit-check-not=warning: --implicit-check-not="MIPS Options"
+
+# NAME-ERR-NOTFOUND: warning: '[[FILE]]': unable to read the name of SHT_MIPS_OPTIONS section with index 2: a section [index 2] has an invalid sh_name (0xffff) offset which goes past the end of the section name string table
diff --git a/llvm/test/tools/llvm-readobj/ELF/mips-reginfo.test b/llvm/test/tools/llvm-readobj/ELF/mips-reginfo.test
index 0074631843cd0..9f200e9d95259 100644
--- a/llvm/test/tools/llvm-readobj/ELF/mips-reginfo.test
+++ b/llvm/test/tools/llvm-readobj/ELF/mips-reginfo.test
@@ -10,3 +10,38 @@
 # CHECK-NEXT:   Co-Proc Mask2: 0x0
 # CHECK-NEXT:   Co-Proc Mask3: 0x0
 # CHECK-NEXT: }
+
+## Check that we try to dump the .reginfo section when we are able to locate it by name.
+# RUN: yaml2obj --docnum=1 -DNAME=0xffff %s -o %t.err1
+# RUN: llvm-readelf -A %t.err1 2>&1 | \
+# RUN:   FileCheck %s -DFILE=%t.err1 --check-prefix=NAME-ERR-FOUND --implicit-check-not=warning:
+# RUN: llvm-readobj -A %t.err1 2>&1 | \
+# RUN:   FileCheck %s -DFILE=%t.err1 --check-prefix=NAME-ERR-FOUND --implicit-check-not=warning:
+
+# NAME-ERR-FOUND:      warning: '[[FILE]]': unable to read the name of SHT_PROGBITS section with index 1: a section [index 1] has an invalid sh_name (0xffff) offset which goes past the end of the section name string table
+# NAME-ERR-FOUND-NEXT: warning: '[[FILE]]': unable to read the name of SHT_PROGBITS section with index 3: a section [index 3] has an invalid sh_name (0xffff) offset which goes past the end of the section name string table
+# NAME-ERR-FOUND:      The .reginfo section has a wrong size.
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_REL
+  Machine: EM_MIPS
+Sections:
+  - Type:   SHT_PROGBITS
+    ShName: [[NAME=<none>]]
+  - Name:   .reginfo
+    Type:   SHT_MIPS_REGINFO
+    ShName: [[REGINFONAME=<none>]]
+  - Type:   SHT_PROGBITS
+    ShName: [[NAME=<none>]]
+
+## Check we report a warning when we are unable to find the .reginfo section due to an error.
+# RUN: yaml2obj --docnum=1 -DREGINFONAME=0xffff %s -o %t.err2
+# RUN: llvm-readelf -A %t.err2 2>&1 | \
+# RUN:   FileCheck %s -DFILE=%t.err2 --check-prefix=NAME-ERR-NOTFOUND --implicit-check-not=warning:
+# RUN: llvm-readobj -A %t.err2 2>&1 | \
+# RUN:   FileCheck %s -DFILE=%t.err2 --check-prefix=NAME-ERR-NOTFOUND --implicit-check-not=warning:
+
+# NAME-ERR-NOTFOUND: warning: '[[FILE]]': unable to read the name of SHT_MIPS_REGINFO section with index 2: a section [index 2] has an invalid sh_name (0xffff) offset which goes past the end of the section name string table
diff --git a/llvm/tools/llvm-readobj/ELFDumper.cpp b/llvm/tools/llvm-readobj/ELFDumper.cpp
index 0e9551edfb1b7..35b5e2637b4d4 100644
--- a/llvm/tools/llvm-readobj/ELFDumper.cpp
+++ b/llvm/tools/llvm-readobj/ELFDumper.cpp
@@ -353,6 +353,8 @@ template <typename ELFT> class ELFDumper : public ObjDumper {
   void printSymbolsHelper(bool IsDynamic) const;
   std::string getDynamicEntry(uint64_t Type, uint64_t Value) const;
 
+  Expected<const typename ELFT::Shdr *> findSectionByName(StringRef Name) const;
+
   const Elf_Shdr *getDotSymtabSec() const { return DotSymtabSec; }
   const Elf_Shdr *getDotCGProfileSec() const { return DotCGProfileSec; }
   const Elf_Shdr *getDotAddrsigSec() const { return DotAddrsigSec; }
@@ -1286,15 +1288,6 @@ findNotEmptySectionByAddress(const ELFO *Obj, StringRef FileName,
   return nullptr;
 }
 
-template <class ELFO>
-static const typename ELFO::Elf_Shdr *
-findSectionByName(const ELFO &Obj, StringRef FileName, StringRef Name) {
-  for (const typename ELFO::Elf_Shdr &Shdr : cantFail(Obj.sections()))
-    if (Name == unwrapOrError(FileName, Obj.getSectionName(&Shdr)))
-      return &Shdr;
-  return nullptr;
-}
-
 static const EnumEntry<unsigned> ElfClass[] = {
   {"None",   "none",   ELF::ELFCLASSNONE},
   {"32-bit", "ELF32",  ELF::ELFCLASS32},
@@ -2457,6 +2450,23 @@ void printFlags(T Value, ArrayRef<EnumEntry<TFlag>> Flags, raw_ostream &OS) {
   }
 }
 
+template <class ELFT>
+Expected<const typename ELFT::Shdr *>
+ELFDumper<ELFT>::findSectionByName(StringRef Name) const {
+  const ELFFile<ELFT> *Obj = ObjF->getELFFile();
+  for (const Elf_Shdr &Shdr : cantFail(Obj->sections())) {
+    if (Expected<StringRef> NameOrErr = Obj->getSectionName(&Shdr)) {
+      if (*NameOrErr == Name)
+        return &Shdr;
+    } else {
+      reportUniqueWarning(createError("unable to read the name of " +
+                                      describe(Shdr) + ": " +
+                                      toString(NameOrErr.takeError())));
+    }
+  }
+  return nullptr;
+}
+
 template <class ELFT>
 std::string ELFDumper<ELFT>::getDynamicEntry(uint64_t Type,
                                              uint64_t Value) const {
@@ -2864,9 +2874,7 @@ template <class ELFT> void ELFDumper<ELFT>::printArchSpecificInfo() {
     ELFDumperStyle->printMipsABIFlags(ObjF);
     printMipsOptions();
     printMipsReginfo();
-
-    MipsGOTParser<ELFT> Parser(Obj, ObjF->getFileName(), dynamic_table(),
-                               dynamic_symbols());
+    MipsGOTParser<ELFT> Parser(*this);
     if (Error E = Parser.findGOT(dynamic_table(), dynamic_symbols()))
       reportError(std::move(E), ObjF->getFileName());
     else if (!Parser.isGotEmpty())
@@ -2933,9 +2941,9 @@ template <class ELFT> class MipsGOTParser {
 
   const bool IsStatic;
   const ELFO * const Obj;
+  const ELFDumper<ELFT> &Dumper;
 
-  MipsGOTParser(const ELFO *Obj, StringRef FileName, Elf_Dyn_Range DynTable,
-                Elf_Sym_Range DynSyms);
+  MipsGOTParser(const ELFDumper<ELFT> &D);
   Error findGOT(Elf_Dyn_Range DynTable, Elf_Sym_Range DynSyms);
   Error findPLT(Elf_Dyn_Range DynTable);
 
@@ -2983,12 +2991,11 @@ template <class ELFT> class MipsGOTParser {
 } // end anonymous namespace
 
 template <class ELFT>
-MipsGOTParser<ELFT>::MipsGOTParser(const ELFO *Obj, StringRef FileName,
-                                   Elf_Dyn_Range DynTable,
-                                   Elf_Sym_Range DynSyms)
-    : IsStatic(DynTable.empty()), Obj(Obj), GotSec(nullptr), LocalNum(0),
-      GlobalNum(0), PltSec(nullptr), PltRelSec(nullptr), PltSymTable(nullptr),
-      FileName(FileName) {}
+MipsGOTParser<ELFT>::MipsGOTParser(const ELFDumper<ELFT> &D)
+    : IsStatic(D.dynamic_table().empty()), Obj(D.getElfObject()->getELFFile()),
+      Dumper(D), GotSec(nullptr), LocalNum(0), GlobalNum(0), PltSec(nullptr),
+      PltRelSec(nullptr), PltSymTable(nullptr),
+      FileName(D.getElfObject()->getFileName()) {}
 
 template <class ELFT>
 Error MipsGOTParser<ELFT>::findGOT(Elf_Dyn_Range DynTable,
@@ -2999,7 +3006,12 @@ Error MipsGOTParser<ELFT>::findGOT(Elf_Dyn_Range DynTable,
 
   // Find static GOT secton.
   if (IsStatic) {
-    GotSec = findSectionByName(*Obj, FileName, ".got");
+    Expected<const Elf_Shdr *> GotOrErr = Dumper.findSectionByName(".got");
+    if (!GotOrErr)
+      return GotOrErr.takeError();
+    else
+      GotSec = *GotOrErr;
+
     if (!GotSec)
       return Error::success();
 
@@ -3318,13 +3330,19 @@ static void printMipsReginfoData(ScopedPrinter &W,
 
 template <class ELFT> void ELFDumper<ELFT>::printMipsReginfo() {
   const ELFFile<ELFT> *Obj = ObjF->getELFFile();
-  const Elf_Shdr *Shdr = findSectionByName(*Obj, ObjF->getFileName(), ".reginfo");
-  if (!Shdr) {
+  Expected<const Elf_Shdr *> RegInfoOrErr = findSectionByName(".reginfo");
+  if (!RegInfoOrErr) {
+    reportUniqueWarning(RegInfoOrErr.takeError());
+    return;
+  }
+
+  if ((*RegInfoOrErr) == nullptr) {
     W.startLine() << "There is no .reginfo section in the file.\n";
     return;
   }
-  ArrayRef<uint8_t> Sec =
-      unwrapOrError(ObjF->getFileName(), Obj->getSectionContents(Shdr));
+
+  ArrayRef<uint8_t> Sec = unwrapOrError(ObjF->getFileName(),
+                                        Obj->getSectionContents(*RegInfoOrErr));
   if (Sec.size() != sizeof(Elf_Mips_RegInfo<ELFT>)) {
     W.startLine() << "The .reginfo section has a wrong size.\n";
     return;
@@ -3337,17 +3355,21 @@ template <class ELFT> void ELFDumper<ELFT>::printMipsReginfo() {
 
 template <class ELFT> void ELFDumper<ELFT>::printMipsOptions() {
   const ELFFile<ELFT> *Obj = ObjF->getELFFile();
-  const Elf_Shdr *Shdr =
-      findSectionByName(*Obj, ObjF->getFileName(), ".MIPS.options");
-  if (!Shdr) {
+  Expected<const Elf_Shdr *> MipsOptOrErr = findSectionByName(".MIPS.options");
+  if (!MipsOptOrErr) {
+    reportUniqueWarning(MipsOptOrErr.takeError());
+    return;
+  }
+
+  if ((*MipsOptOrErr) == nullptr) {
     W.startLine() << "There is no .MIPS.options section in the file.\n";
     return;
   }
 
   DictScope GS(W, "MIPS Options");
 
-  ArrayRef<uint8_t> Sec =
-      unwrapOrError(ObjF->getFileName(), Obj->getSectionContents(Shdr));
+  ArrayRef<uint8_t> Sec = unwrapOrError(ObjF->getFileName(),
+                                        Obj->getSectionContents(*MipsOptOrErr));
   while (!Sec.empty()) {
     if (Sec.size() < sizeof(Elf_Mips_Options<ELFT>)) {
       W.startLine() << "The .MIPS.options section has a wrong size.\n";
@@ -5959,15 +5981,18 @@ void GNUStyle<ELFT>::printMipsPLT(const MipsGOTParser<ELFT> &Parser) {
 
 template <class ELFT>
 Expected<const Elf_Mips_ABIFlags<ELFT> *>
-getMipsAbiFlagsSection(const ELFObjectFile<ELFT> *ObjF) {
-  const ELFFile<ELFT> *Obj = ObjF->getELFFile();
-  const typename ELFT::Shdr *Shdr =
-      findSectionByName(*Obj, ObjF->getFileName(), ".MIPS.abiflags");
-  if (!Shdr)
+getMipsAbiFlagsSection(const ELFObjectFile<ELFT> *ObjF,
+                       const ELFDumper<ELFT> &Dumper) {
+  Expected<const typename ELFT::Shdr *> SecOrErr =
+      Dumper.findSectionByName(".MIPS.abiflags");
+  if (!SecOrErr)
+    return SecOrErr.takeError();
+  if (*SecOrErr == nullptr)
     return nullptr;
 
+  const ELFFile<ELFT> *Obj = ObjF->getELFFile();
   constexpr StringRef ErrPrefix = "unable to read the .MIPS.abiflags section: ";
-  Expected<ArrayRef<uint8_t>> DataOrErr = Obj->getSectionContents(Shdr);
+  Expected<ArrayRef<uint8_t>> DataOrErr = Obj->getSectionContents(*SecOrErr);
   if (!DataOrErr)
     return createError(ErrPrefix + toString(DataOrErr.takeError()));
 
@@ -5981,7 +6006,7 @@ template <class ELFT>
 void GNUStyle<ELFT>::printMipsABIFlags(const ELFObjectFile<ELFT> *ObjF) {
   const Elf_Mips_ABIFlags<ELFT> *Flags = nullptr;
   if (Expected<const Elf_Mips_ABIFlags<ELFT> *> SecOrErr =
-          getMipsAbiFlagsSection(ObjF))
+          getMipsAbiFlagsSection(ObjF, *this->dumper()))
     Flags = *SecOrErr;
   else
     this->reportUniqueWarning(SecOrErr.takeError());
@@ -6926,7 +6951,7 @@ template <class ELFT>
 void LLVMStyle<ELFT>::printMipsABIFlags(const ELFObjectFile<ELFT> *ObjF) {
   const Elf_Mips_ABIFlags<ELFT> *Flags;
   if (Expected<const Elf_Mips_ABIFlags<ELFT> *> SecOrErr =
-          getMipsAbiFlagsSection(ObjF)) {
+          getMipsAbiFlagsSection(ObjF, *this->dumper())) {
     Flags = *SecOrErr;
     if (!Flags) {
       W.startLine() << "There is no .MIPS.abiflags section in the file.\n";

From 87de54dbb6efa0fc5e304f94b350a39066bc2759 Mon Sep 17 00:00:00 2001
From: Kadir Cetinkaya <kadircet@google.com>
Date: Sat, 1 Aug 2020 18:55:30 +0200
Subject: [PATCH 0890/1035] [clang][Tooling] Fix addTargetAndModeForProgramName
 to use correct flag names

The logic was using incorrect flag versions. For example:
- `-target=` can't be a prefix, it must be `--target=`.
- `--driver-mode` can't appear on its own, value must be attached to it.

While fixing those, also changes the append logic to make use of new
`--target=X` format instead of the legacy `-target X` version.

In addition to that makes use of the OPTTable instead of hardcoded strings to
make sure helper also gets updated if clang's options are modified.

Differential Revision: https://reviews.llvm.org/D85076
---
 clang/lib/Tooling/Tooling.cpp           | 52 +++++++++++++++----------
 clang/unittests/Tooling/ToolingTest.cpp | 16 +++-----
 2 files changed, 36 insertions(+), 32 deletions(-)

diff --git a/clang/lib/Tooling/Tooling.cpp b/clang/lib/Tooling/Tooling.cpp
index 40b6cff0d627a..0593f0cc1d195 100644
--- a/clang/lib/Tooling/Tooling.cpp
+++ b/clang/lib/Tooling/Tooling.cpp
@@ -245,27 +245,37 @@ std::string getAbsolutePath(StringRef File) {
 
 void addTargetAndModeForProgramName(std::vector<std::string> &CommandLine,
                                     StringRef InvokedAs) {
-  if (!CommandLine.empty() && !InvokedAs.empty()) {
-    bool AlreadyHasTarget = false;
-    bool AlreadyHasMode = false;
-    // Skip CommandLine[0].
-    for (auto Token = ++CommandLine.begin(); Token != CommandLine.end();
-         ++Token) {
-      StringRef TokenRef(*Token);
-      AlreadyHasTarget |=
-          (TokenRef == "-target" || TokenRef.startswith("-target="));
-      AlreadyHasMode |= (TokenRef == "--driver-mode" ||
-                         TokenRef.startswith("--driver-mode="));
-    }
-    auto TargetMode =
-        driver::ToolChain::getTargetAndModeFromProgramName(InvokedAs);
-    if (!AlreadyHasMode && TargetMode.DriverMode) {
-      CommandLine.insert(++CommandLine.begin(), TargetMode.DriverMode);
-    }
-    if (!AlreadyHasTarget && TargetMode.TargetIsValid) {
-      CommandLine.insert(++CommandLine.begin(), {"-target",
-                                                 TargetMode.TargetPrefix});
-    }
+  if (CommandLine.empty() || InvokedAs.empty())
+    return;
+  const auto &Table = driver::getDriverOptTable();
+  // --target=X
+  const std::string TargetOPT =
+      Table.getOption(driver::options::OPT_target).getPrefixedName();
+  // -target X
+  const std::string TargetOPTLegacy =
+      Table.getOption(driver::options::OPT_target_legacy_spelling)
+          .getPrefixedName();
+  // --driver-mode=X
+  const std::string DriverModeOPT =
+      Table.getOption(driver::options::OPT_driver_mode).getPrefixedName();
+  bool AlreadyHasTarget = false;
+  bool AlreadyHasMode = false;
+  // Skip CommandLine[0].
+  for (auto Token = ++CommandLine.begin(); Token != CommandLine.end();
+       ++Token) {
+    StringRef TokenRef(*Token);
+    AlreadyHasTarget |=
+        TokenRef.startswith(TargetOPT) || TokenRef.equals(TargetOPTLegacy);
+    AlreadyHasMode |= TokenRef.startswith(DriverModeOPT);
+  }
+  auto TargetMode =
+      driver::ToolChain::getTargetAndModeFromProgramName(InvokedAs);
+  if (!AlreadyHasMode && TargetMode.DriverMode) {
+    CommandLine.insert(++CommandLine.begin(), TargetMode.DriverMode);
+  }
+  if (!AlreadyHasTarget && TargetMode.TargetIsValid) {
+    CommandLine.insert(++CommandLine.begin(),
+                       TargetOPT + TargetMode.TargetPrefix);
   }
 }
 
diff --git a/clang/unittests/Tooling/ToolingTest.cpp b/clang/unittests/Tooling/ToolingTest.cpp
index 5bd2864b5ba1e..cc6f453284d71 100644
--- a/clang/unittests/Tooling/ToolingTest.cpp
+++ b/clang/unittests/Tooling/ToolingTest.cpp
@@ -621,7 +621,7 @@ TEST(addTargetAndModeForProgramName, AddsTargetAndMode) {
   addTargetAndModeForProgramName(Args, "");
   EXPECT_EQ((std::vector<std::string>{"clang", "-foo"}), Args);
   addTargetAndModeForProgramName(Args, Target + "-g++");
-  EXPECT_EQ((std::vector<std::string>{"clang", "-target", Target,
+  EXPECT_EQ((std::vector<std::string>{"clang", "--target=" + Target,
                                       "--driver-mode=g++", "-foo"}),
             Args);
 }
@@ -635,7 +635,7 @@ TEST(addTargetAndModeForProgramName, PathIgnored) {
 
   std::vector<std::string> Args = {"clang", "-foo"};
   addTargetAndModeForProgramName(Args, ToolPath);
-  EXPECT_EQ((std::vector<std::string>{"clang", "-target", Target,
+  EXPECT_EQ((std::vector<std::string>{"clang", "--target=" + Target,
                                       "--driver-mode=g++", "-foo"}),
             Args);
 }
@@ -650,10 +650,10 @@ TEST(addTargetAndModeForProgramName, IgnoresExistingTarget) {
                                       "-target", "something"}),
             Args);
 
-  std::vector<std::string> ArgsAlt = {"clang", "-foo", "-target=something"};
+  std::vector<std::string> ArgsAlt = {"clang", "-foo", "--target=something"};
   addTargetAndModeForProgramName(ArgsAlt, Target + "-g++");
   EXPECT_EQ((std::vector<std::string>{"clang", "--driver-mode=g++", "-foo",
-                                      "-target=something"}),
+                                      "--target=something"}),
             ArgsAlt);
 }
 
@@ -663,15 +663,9 @@ TEST(addTargetAndModeForProgramName, IgnoresExistingMode) {
 
   std::vector<std::string> Args = {"clang", "-foo", "--driver-mode=abc"};
   addTargetAndModeForProgramName(Args, Target + "-g++");
-  EXPECT_EQ((std::vector<std::string>{"clang", "-target", Target, "-foo",
+  EXPECT_EQ((std::vector<std::string>{"clang", "--target=" + Target, "-foo",
                                       "--driver-mode=abc"}),
             Args);
-
-  std::vector<std::string> ArgsAlt = {"clang", "-foo", "--driver-mode", "abc"};
-  addTargetAndModeForProgramName(ArgsAlt, Target + "-g++");
-  EXPECT_EQ((std::vector<std::string>{"clang", "-target", Target, "-foo",
-                                      "--driver-mode", "abc"}),
-            ArgsAlt);
 }
 
 #ifndef _WIN32

From 895878f4568d3c5fe470bd811c2dfdbbe285e5aa Mon Sep 17 00:00:00 2001
From: Alex Richardson <Alexander.Richardson@cl.cam.ac.uk>
Date: Mon, 3 Aug 2020 10:04:01 +0100
Subject: [PATCH 0891/1035] [asan][tsan] Mark tests failing with debug checks
 as XFAIL

See https://llvm.org/PR46862. This does not fix the underlying issue but at
least it allows me to run check-all again without having to disable
building compiler-rt.

Reviewed By: #sanitizers, vitalybuka

Differential Revision: https://reviews.llvm.org/D84650
---
 compiler-rt/test/asan/TestCases/Linux/activation-options.cpp   | 2 ++
 compiler-rt/test/asan/TestCases/Linux/odr-violation.cpp        | 2 ++
 compiler-rt/test/asan/TestCases/Linux/odr-vtable.cpp           | 3 +++
 compiler-rt/test/asan/TestCases/Posix/start-deactivated.cpp    | 2 ++
 compiler-rt/test/asan/TestCases/handle_noreturn_bug.cpp        | 2 ++
 .../test/sanitizer_common/TestCases/Linux/allow_user_segv.cpp  | 2 ++
 .../test/sanitizer_common/TestCases/Linux/signal_line.cpp      | 2 ++
 .../TestCases/Posix/dedup_token_length_test.cpp                | 2 ++
 .../sanitizer_common/TestCases/Posix/illegal_read_test.cpp     | 2 ++
 .../sanitizer_common/TestCases/Posix/illegal_write_test.cpp    | 2 ++
 .../TestCases/Posix/sanitizer_set_report_fd_test.cpp           | 2 ++
 11 files changed, 23 insertions(+)

diff --git a/compiler-rt/test/asan/TestCases/Linux/activation-options.cpp b/compiler-rt/test/asan/TestCases/Linux/activation-options.cpp
index 0074a6699d39e..59c4d5c46e771 100644
--- a/compiler-rt/test/asan/TestCases/Linux/activation-options.cpp
+++ b/compiler-rt/test/asan/TestCases/Linux/activation-options.cpp
@@ -1,6 +1,8 @@
 // Test for ASAN_OPTIONS=start_deactivated=1 mode.
 // Main executable is uninstrumented, but linked to ASan runtime. The shared
 // library is instrumented.
+// Fails with debug checks: https://bugs.llvm.org/show_bug.cgi?id=46862
+// XFAIL: !compiler-rt-optimized
 
 // RUN: %clangxx_asan -O0 -DSHARED_LIB %s -fPIC -shared -o %t-so.so
 // RUN: %clangxx -O0 %s -c -o %t.o
diff --git a/compiler-rt/test/asan/TestCases/Linux/odr-violation.cpp b/compiler-rt/test/asan/TestCases/Linux/odr-violation.cpp
index 9cdafca91b1f3..a14b59b67f114 100644
--- a/compiler-rt/test/asan/TestCases/Linux/odr-violation.cpp
+++ b/compiler-rt/test/asan/TestCases/Linux/odr-violation.cpp
@@ -1,5 +1,7 @@
 // FIXME: https://code.google.com/p/address-sanitizer/issues/detail?id=316
 // XFAIL: android
+// Fails with debug checks: https://bugs.llvm.org/show_bug.cgi?id=46862
+// XFAIL: !compiler-rt-optimized
 //
 // We use fast_unwind_on_malloc=0 to have full unwinding even w/o frame
 // pointers. This setting is not on by default because it's too expensive.
diff --git a/compiler-rt/test/asan/TestCases/Linux/odr-vtable.cpp b/compiler-rt/test/asan/TestCases/Linux/odr-vtable.cpp
index fdbab4bb1fd82..eb0805784df1a 100644
--- a/compiler-rt/test/asan/TestCases/Linux/odr-vtable.cpp
+++ b/compiler-rt/test/asan/TestCases/Linux/odr-vtable.cpp
@@ -1,3 +1,6 @@
+// Fails with debug checks: https://bugs.llvm.org/show_bug.cgi?id=46862
+// XFAIL: !compiler-rt-optimized
+
 // RUN: %clangxx_asan -fno-rtti -DBUILD_SO1 -fPIC -shared %s -o %dynamiclib1
 // RUN: %clangxx_asan -fno-rtti -DBUILD_SO2 -fPIC -shared %s -o %dynamiclib2
 // RUN: %clangxx_asan -fno-rtti %s %ld_flags_rpath_exe1 %ld_flags_rpath_exe2 -o %t
diff --git a/compiler-rt/test/asan/TestCases/Posix/start-deactivated.cpp b/compiler-rt/test/asan/TestCases/Posix/start-deactivated.cpp
index 9c674ac57e46a..c4f5f8c74a024 100644
--- a/compiler-rt/test/asan/TestCases/Posix/start-deactivated.cpp
+++ b/compiler-rt/test/asan/TestCases/Posix/start-deactivated.cpp
@@ -1,6 +1,8 @@
 // Test for ASAN_OPTIONS=start_deactivated=1 mode.
 // Main executable is uninstrumented, but linked to ASan runtime. The shared
 // library is instrumented. Memory errors before dlopen are not detected.
+// Fails with debug checks: https://bugs.llvm.org/show_bug.cgi?id=46862
+// XFAIL: !compiler-rt-optimized
 
 // RUN: %clangxx_asan -O0 -DSHARED_LIB %s -std=c++11 -fPIC -shared -o %t-so.so
 // RUN: %clangxx -O0 %s -std=c++11 -c -o %t.o
diff --git a/compiler-rt/test/asan/TestCases/handle_noreturn_bug.cpp b/compiler-rt/test/asan/TestCases/handle_noreturn_bug.cpp
index 8c3c66a423d50..1639ad7d0859b 100644
--- a/compiler-rt/test/asan/TestCases/handle_noreturn_bug.cpp
+++ b/compiler-rt/test/asan/TestCases/handle_noreturn_bug.cpp
@@ -1,4 +1,6 @@
 // Regression test: __asan_handle_no_return should unpoison stack even with poison_heap=0.
+// Fails with debug checks: https://bugs.llvm.org/show_bug.cgi?id=46862
+// XFAIL: !compiler-rt-optimized
 // RUN: %clangxx_asan -O0 %s -o %t && \
 // RUN: %env_asan_opts=poison_heap=1 %run %t && \
 // RUN: %env_asan_opts=poison_heap=0 %run %t
diff --git a/compiler-rt/test/sanitizer_common/TestCases/Linux/allow_user_segv.cpp b/compiler-rt/test/sanitizer_common/TestCases/Linux/allow_user_segv.cpp
index bd58f4bd92654..03ddf06ae2d41 100644
--- a/compiler-rt/test/sanitizer_common/TestCases/Linux/allow_user_segv.cpp
+++ b/compiler-rt/test/sanitizer_common/TestCases/Linux/allow_user_segv.cpp
@@ -1,5 +1,7 @@
 // Regression test for
 // https://code.google.com/p/address-sanitizer/issues/detail?id=180
+// Fails with debug checks: https://bugs.llvm.org/show_bug.cgi?id=46860
+// XFAIL: !compiler-rt-optimized && tsan
 
 // RUN: %clangxx -O0 %s -o %t
 
diff --git a/compiler-rt/test/sanitizer_common/TestCases/Linux/signal_line.cpp b/compiler-rt/test/sanitizer_common/TestCases/Linux/signal_line.cpp
index 1c9ea982b95b2..208ece3e05af4 100644
--- a/compiler-rt/test/sanitizer_common/TestCases/Linux/signal_line.cpp
+++ b/compiler-rt/test/sanitizer_common/TestCases/Linux/signal_line.cpp
@@ -1,4 +1,6 @@
 // Test line numbers in signal handlers
+// Fails with debug checks: https://bugs.llvm.org/show_bug.cgi?id=46860
+// XFAIL: !compiler-rt-optimized && tsan
 
 // RUN: %clangxx %s -o %t -O0
 // RUN: %env_tool_opts=handle_segv=1:print_stacktrace=1 not %run %t 1 2>&1 | FileCheck --check-prefixes=CHECK1,CHECK %s
diff --git a/compiler-rt/test/sanitizer_common/TestCases/Posix/dedup_token_length_test.cpp b/compiler-rt/test/sanitizer_common/TestCases/Posix/dedup_token_length_test.cpp
index 94c50be169b48..1630003e519bf 100644
--- a/compiler-rt/test/sanitizer_common/TestCases/Posix/dedup_token_length_test.cpp
+++ b/compiler-rt/test/sanitizer_common/TestCases/Posix/dedup_token_length_test.cpp
@@ -1,4 +1,6 @@
 // Test dedup_token_length
+// Fails with debug checks: https://bugs.llvm.org/show_bug.cgi?id=46860
+// XFAIL: !compiler-rt-optimized && tsan
 // RUN: %clangxx -O0 %s -o %t
 // RUN: env %tool_options='abort_on_error=0'                       not %run %t 2>&1   | FileCheck %s --check-prefix=CHECK0 --match-full-lines
 // RUN: env %tool_options='abort_on_error=0, dedup_token_length=0' not %run %t 2>&1   | FileCheck %s --check-prefix=CHECK0 --match-full-lines
diff --git a/compiler-rt/test/sanitizer_common/TestCases/Posix/illegal_read_test.cpp b/compiler-rt/test/sanitizer_common/TestCases/Posix/illegal_read_test.cpp
index 9615d7132da5e..2addb9bd47508 100644
--- a/compiler-rt/test/sanitizer_common/TestCases/Posix/illegal_read_test.cpp
+++ b/compiler-rt/test/sanitizer_common/TestCases/Posix/illegal_read_test.cpp
@@ -1,4 +1,6 @@
 // Test that there was an illegal READ memory access.
+// Fails with debug checks: https://bugs.llvm.org/show_bug.cgi?id=46860
+// XFAIL: !compiler-rt-optimized && tsan
 // RUN: %clangxx -O0 %s -o %t && not %run %t 2>&1 | FileCheck %s
 
 // REQUIRES: stable-runtime
diff --git a/compiler-rt/test/sanitizer_common/TestCases/Posix/illegal_write_test.cpp b/compiler-rt/test/sanitizer_common/TestCases/Posix/illegal_write_test.cpp
index 13d1c6a06905a..531b4d294b8fb 100644
--- a/compiler-rt/test/sanitizer_common/TestCases/Posix/illegal_write_test.cpp
+++ b/compiler-rt/test/sanitizer_common/TestCases/Posix/illegal_write_test.cpp
@@ -1,4 +1,6 @@
 // Test that there was an illegal WRITE memory access.
+// Fails with debug checks: https://bugs.llvm.org/show_bug.cgi?id=46860
+// XFAIL: !compiler-rt-optimized && tsan
 // RUN: %clangxx -O0 %s -o %t && not %run %t 2>&1 | FileCheck %s
 
 // REQUIRES: stable-runtime
diff --git a/compiler-rt/test/sanitizer_common/TestCases/Posix/sanitizer_set_report_fd_test.cpp b/compiler-rt/test/sanitizer_common/TestCases/Posix/sanitizer_set_report_fd_test.cpp
index cc7de193f0a41..6ba7025bf7578 100644
--- a/compiler-rt/test/sanitizer_common/TestCases/Posix/sanitizer_set_report_fd_test.cpp
+++ b/compiler-rt/test/sanitizer_common/TestCases/Posix/sanitizer_set_report_fd_test.cpp
@@ -1,4 +1,6 @@
 // Test __sanitizer_set_report_fd:
+// Fails with debug checks: https://bugs.llvm.org/show_bug.cgi?id=46860
+// XFAIL: !compiler-rt-optimized && tsan
 // RUN: %clangxx -O2 %s -o %t
 // RUN: not %run %t 2>&1   | FileCheck %s
 // RUN: not %run %t stdout | FileCheck %s

From 03affa8099cba3876aca209daa213dcb880bc34a Mon Sep 17 00:00:00 2001
From: Alex Richardson <Alexander.Richardson@cl.cam.ac.uk>
Date: Mon, 3 Aug 2020 10:31:31 +0100
Subject: [PATCH 0892/1035] [msan] Compile the libatomic.c test with a C
 compiler

Otherwise we end up compiling in C++ mode and on FreeBSD
/usr/include/stdatomic.h is not compatible with C++ since it uses _Bool.

Reviewed By: guiand, eugenis, vitalybuka, emaste

Differential Revision: https://reviews.llvm.org/D84510
---
 compiler-rt/test/msan/libatomic.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/compiler-rt/test/msan/libatomic.c b/compiler-rt/test/msan/libatomic.c
index 5d8aa9ab6a60f..a8c030b7dbb26 100644
--- a/compiler-rt/test/msan/libatomic.c
+++ b/compiler-rt/test/msan/libatomic.c
@@ -1,6 +1,6 @@
-// RUN: %clangxx_msan -fsanitize-memory-track-origins=2 -latomic -DTEST_STORE -O0 %s -o %t && %run %t 2>&1
-// RUN: %clangxx_msan -fsanitize-memory-track-origins=0 -latomic -DTEST_LOAD -O0 %s -o %t && not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK
-// RUN: %clangxx_msan -fsanitize-memory-track-origins=2 -latomic -DTEST_LOAD -O0 %s -o %t && not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-SHADOW
+// RUN: %clang_msan -fsanitize-memory-track-origins=2 -latomic -DTEST_STORE -O0 %s -o %t && %run %t 2>&1
+// RUN: %clang_msan -fsanitize-memory-track-origins=0 -latomic -DTEST_LOAD -O0 %s -o %t && not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK
+// RUN: %clang_msan -fsanitize-memory-track-origins=2 -latomic -DTEST_LOAD -O0 %s -o %t && not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-SHADOW
 
 // PPC has no libatomic
 // UNSUPPORTED: powerpc64-target-arch

From d23b15cc160775dcc5b6e2c2d7706a67be1225f1 Mon Sep 17 00:00:00 2001
From: Alex Richardson <Alexander.Richardson@cl.cam.ac.uk>
Date: Mon, 3 Aug 2020 10:51:14 +0100
Subject: [PATCH 0893/1035] Execute llvm-lit with the python found by CMake by
 default

The check-* targets run ${Python3_EXECUTABLE} $BUILD/bin/llvm-lit, but
running `./bin/llvm-lit $ARGS` from the build directory currently always
uses "python" to run llvm-lit. On most systems this will be python2.7 even
if we found python3 at CMake time.

Reviewed By: compnerd

Differential Revision: https://reviews.llvm.org/D84625
---
 llvm/utils/gn/secondary/llvm/utils/llvm-lit/BUILD.gn | 1 +
 llvm/utils/llvm-lit/llvm-lit.in                      | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/llvm/utils/gn/secondary/llvm/utils/llvm-lit/BUILD.gn b/llvm/utils/gn/secondary/llvm/utils/llvm-lit/BUILD.gn
index 526b93b4f0b99..648646e4188d6 100644
--- a/llvm/utils/gn/secondary/llvm/utils/llvm-lit/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/utils/llvm-lit/BUILD.gn
@@ -86,6 +86,7 @@ write_cmake_config("llvm-lit") {
 
   values = [
     "LLVM_SOURCE_DIR=" + rebase_path("//llvm", dir),
+    "Python3_EXECUTABLE=$python_path",
     "BUILD_MODE=.",
     "LLVM_LIT_CONFIG_MAP=" + config_map,
   ]
diff --git a/llvm/utils/llvm-lit/llvm-lit.in b/llvm/utils/llvm-lit/llvm-lit.in
index bfa55c6abfac1..33ec8017cf05f 100755
--- a/llvm/utils/llvm-lit/llvm-lit.in
+++ b/llvm/utils/llvm-lit/llvm-lit.in
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!@Python3_EXECUTABLE@
 # -*- coding: utf-8 -*-
 
 import os

From c3339e3e921c2e064936c30f5eacba4e41c432da Mon Sep 17 00:00:00 2001
From: Tatyana Krasnukha <tatyana@synopsys.com>
Date: Fri, 17 Jul 2020 15:58:28 +0300
Subject: [PATCH 0894/1035] [cmake] Make MSVC generate appropriate __cplusplus
 macro definition

Differential Revision: https://reviews.llvm.org/D84023
---
 llvm/cmake/modules/AddLLVM.cmake           | 7 -------
 llvm/cmake/modules/HandleLLVMOptions.cmake | 6 ++++++
 llvm/utils/unittest/CMakeLists.txt         | 5 -----
 3 files changed, 6 insertions(+), 12 deletions(-)

diff --git a/llvm/cmake/modules/AddLLVM.cmake b/llvm/cmake/modules/AddLLVM.cmake
index a25f3e87af61c..5b5be953767b5 100644
--- a/llvm/cmake/modules/AddLLVM.cmake
+++ b/llvm/cmake/modules/AddLLVM.cmake
@@ -1405,13 +1405,6 @@ function(add_unittest test_suite test_name)
     set(EXCLUDE_FROM_ALL ON)
   endif()
 
-  # Our current version of gtest does not properly recognize C++11 support
-  # with MSVC, so it falls back to tr1 / experimental classes.  Since LLVM
-  # itself requires C++11, we can safely force it on unconditionally so that
-  # we don't have to fight with the buggy gtest check.
-  add_definitions(-DGTEST_LANG_CXX11=1)
-  add_definitions(-DGTEST_HAS_TR1_TUPLE=0)
-
   include_directories(${LLVM_MAIN_SRC_DIR}/utils/unittest/googletest/include)
   include_directories(${LLVM_MAIN_SRC_DIR}/utils/unittest/googlemock/include)
   if (NOT LLVM_ENABLE_THREADS)
diff --git a/llvm/cmake/modules/HandleLLVMOptions.cmake b/llvm/cmake/modules/HandleLLVMOptions.cmake
index 0df9334363b43..4feb4b7a7f300 100644
--- a/llvm/cmake/modules/HandleLLVMOptions.cmake
+++ b/llvm/cmake/modules/HandleLLVMOptions.cmake
@@ -421,6 +421,12 @@ if( MSVC )
 
   append("/Zc:inline" CMAKE_C_FLAGS CMAKE_CXX_FLAGS)
 
+  # Some projects use the __cplusplus preprocessor macro to check support for
+  # a particular version of the C++ standard. When this option is not specified
+  # explicitly, macro's value is "199711L" that implies C++98 Standard.
+  # https://devblogs.microsoft.com/cppblog/msvc-now-correctly-reports-__cplusplus/
+  append("/Zc:__cplusplus" CMAKE_CXX_FLAGS)
+
   # Allow users to request PDBs in release mode. CMake offeres the
   # RelWithDebInfo configuration, but it uses different optimization settings
   # (/Ob1 vs /Ob2 or -O2 vs -O3). LLVM provides this flag so that users can get
diff --git a/llvm/utils/unittest/CMakeLists.txt b/llvm/utils/unittest/CMakeLists.txt
index bcae36fa150d1..9127ebce7b4c3 100644
--- a/llvm/utils/unittest/CMakeLists.txt
+++ b/llvm/utils/unittest/CMakeLists.txt
@@ -19,11 +19,6 @@ include_directories(
   googlemock
   )
 
-# LLVM requires C++11 but gtest doesn't correctly detect the availability
-# of C++11 on MSVC, so we force it on.
-add_definitions(-DGTEST_LANG_CXX11=1)
-add_definitions(-DGTEST_HAS_TR1_TUPLE=0)
-
 if(WIN32)
   add_definitions(-DGTEST_OS_WINDOWS=1)
 endif()

From e97c693bb0ece2d9a2b0db75034927405fe3bfdf Mon Sep 17 00:00:00 2001
From: Tatyana Krasnukha <tatyana@synopsys.com>
Date: Thu, 30 Jul 2020 20:13:23 +0300
Subject: [PATCH 0895/1035] [lldb/Process/Windows] Attempting to kill
 exited/detached process in not an error

The lldb test-suite on Windows reports a 'CLEANUP ERROR' when attempting to kill
an exited/detached process. This change makes ProcessWindows consistent with
the other processes which only log the error. After this change a number of
'CLEANUP ERROR' messages are now removed.

Differential Revision: https://reviews.llvm.org/D84957
---
 .../Windows/Common/ProcessDebugger.cpp        | 28 +++++++++----------
 1 file changed, 13 insertions(+), 15 deletions(-)

diff --git a/lldb/source/Plugins/Process/Windows/Common/ProcessDebugger.cpp b/lldb/source/Plugins/Process/Windows/Common/ProcessDebugger.cpp
index 8a85c8ba6f4e4..07a81cdf69ccd 100644
--- a/lldb/source/Plugins/Process/Windows/Common/ProcessDebugger.cpp
+++ b/lldb/source/Plugins/Process/Windows/Common/ProcessDebugger.cpp
@@ -227,22 +227,20 @@ Status ProcessDebugger::DestroyProcess(const lldb::StateType state) {
     debugger_thread = m_session_data->m_debugger;
   }
 
-  Status error;
-  if (state != eStateExited && state != eStateDetached) {
-    LLDB_LOG(
-        log, "Shutting down process {0}.",
-        debugger_thread->GetProcess().GetNativeProcess().GetSystemHandle());
-    error = debugger_thread->StopDebugging(true);
-
-    // By the time StopDebugging returns, there is no more debugger thread, so
-    // we can be assured that no other thread will race for the session data.
-    m_session_data.reset();
-  } else {
-    error.SetErrorStringWithFormat("cannot destroy process %" PRIx64
-                                   " while state = %d",
-                                   GetDebuggedProcessId(), state);
-    LLDB_LOG(log, "error: {0}", error);
+  if (state == eStateExited || state == eStateDetached) {
+    LLDB_LOG(log, "warning: cannot destroy process {0} while state = {1}.",
+             GetDebuggedProcessId(), state);
+    return Status();
   }
+
+  LLDB_LOG(log, "Shutting down process {0}.",
+           debugger_thread->GetProcess().GetNativeProcess().GetSystemHandle());
+  auto error = debugger_thread->StopDebugging(true);
+
+  // By the time StopDebugging returns, there is no more debugger thread, so
+  // we can be assured that no other thread will race for the session data.
+  m_session_data.reset();
+
   return error;
 }
 

From d1007478f19d3ff19a2ecd5ecb04b467933041e6 Mon Sep 17 00:00:00 2001
From: Alex Richardson <Alexander.Richardson@cl.cam.ac.uk>
Date: Mon, 3 Aug 2020 11:18:01 +0100
Subject: [PATCH 0896/1035] Fix update_cc_test_checks.py --llvm-bin after
 D78478

Not passing --clang would result in a python exception after this change:
(TypeError: expected str, bytes or os.PathLike object, not NoneType)
because the --clang argument default was only being populated in the
initial argument parsing pass but not later on.
Fix this by adding an argparse callback to set the default values.

Reviewed By: vitalybuka, MaskRay

Differential Revision: https://reviews.llvm.org/D84511
---
 llvm/utils/UpdateTestChecks/common.py | 17 +++++++++++------
 llvm/utils/update_cc_test_checks.py   | 27 ++++++++++++++++-----------
 2 files changed, 27 insertions(+), 17 deletions(-)

diff --git a/llvm/utils/UpdateTestChecks/common.py b/llvm/utils/UpdateTestChecks/common.py
index 17f738601f61b..35b7ba648d36d 100644
--- a/llvm/utils/UpdateTestChecks/common.py
+++ b/llvm/utils/UpdateTestChecks/common.py
@@ -44,8 +44,9 @@ def __init__(self, line, line_number, args, argv):
 
 class TestInfo(object):
   def __init__(self, test, parser, script_name, input_lines, args, argv,
-               comment_prefix):
+               comment_prefix, argparse_callback):
     self.parser = parser
+    self.argparse_callback = argparse_callback
     self.path = test
     self.args = args
     self.argv = argv
@@ -68,14 +69,14 @@ def iterlines(self, output_lines):
       if input_line.startswith(self.autogenerated_note_prefix):
         continue
       self.args, self.argv = check_for_command(input_line, self.parser,
-                                               self.args, self.argv)
+                                               self.args, self.argv, self.argparse_callback)
       if not self.args.enabled:
         output_lines.append(input_line)
         continue
       yield InputLineInfo(input_line, line_num, self.args, self.argv)
 
 
-def itertests(test_patterns, parser, script_name, comment_prefix=None):
+def itertests(test_patterns, parser, script_name, comment_prefix=None, argparse_callback=None):
   for pattern in test_patterns:
     # On Windows we must expand the patterns ourselves.
     tests_list = glob.glob(pattern)
@@ -86,19 +87,21 @@ def itertests(test_patterns, parser, script_name, comment_prefix=None):
       with open(test) as f:
         input_lines = [l.rstrip() for l in f]
       args = parser.parse_args()
+      if argparse_callback is not None:
+        argparse_callback(args)
       argv = sys.argv[:]
       first_line = input_lines[0] if input_lines else ""
       if UTC_ADVERT in first_line:
         if script_name not in first_line and not args.force_update:
           warn("Skipping test which wasn't autogenerated by " + script_name, test)
           continue
-        args, argv = check_for_command(first_line, parser, args, argv)
+        args, argv = check_for_command(first_line, parser, args, argv, argparse_callback)
       elif args.update_only:
         assert UTC_ADVERT not in first_line
         warn("Skipping test which isn't autogenerated: " + test)
         continue
       yield TestInfo(test, parser, script_name, input_lines, args, argv,
-                     comment_prefix)
+                     comment_prefix, argparse_callback)
 
 
 def should_add_line_to_output(input_line, prefix_set):
@@ -510,10 +513,12 @@ def get_autogennote_suffix(parser, args):
   return autogenerated_note_args
 
 
-def check_for_command(line, parser, args, argv):
+def check_for_command(line, parser, args, argv, argparse_callback):
     cmd_m = UTC_ARGS_CMD.match(line)
     if cmd_m:
         cmd = cmd_m.group('cmd').strip().split(' ')
         argv = argv + cmd
         args = parser.parse_args(filter(lambda arg: arg not in args.tests, argv))
+        if argparse_callback is not None:
+          argparse_callback(args)
     return args, argv
diff --git a/llvm/utils/update_cc_test_checks.py b/llvm/utils/update_cc_test_checks.py
index fa7b4fa73b98c..ba8a68b8669f9 100755
--- a/llvm/utils/update_cc_test_checks.py
+++ b/llvm/utils/update_cc_test_checks.py
@@ -112,6 +112,20 @@ def str_to_commandline(value):
     return []
   return shlex.split(value)
 
+
+def infer_dependent_args(args):
+  if not args.clang:
+    if not args.llvm_bin:
+      args.clang = 'clang'
+    else:
+      args.clang = os.path.join(args.llvm_bin, 'clang')
+  if not args.opt:
+    if not args.llvm_bin:
+      args.opt = 'opt'
+    else:
+      args.opt = os.path.join(args.llvm_bin, 'opt')
+
+
 def config():
   parser = argparse.ArgumentParser(
       description=__doc__,
@@ -135,12 +149,8 @@ def config():
                       help='Check "Function Attributes" for functions')
   parser.add_argument('tests', nargs='+')
   args = common.parse_commandline_args(parser)
+  infer_dependent_args(args)
 
-  if args.clang is None:
-    if args.llvm_bin is None:
-      args.clang = 'clang'
-    else:
-      args.clang = os.path.join(args.llvm_bin, 'clang')
   if not distutils.spawn.find_executable(args.clang):
     print('Please specify --llvm-bin or --clang', file=sys.stderr)
     sys.exit(1)
@@ -157,11 +167,6 @@ def config():
     common.warn('Could not determine clang builtins directory, some tests '
                 'might not update correctly.')
 
-  if args.opt is None:
-    if args.llvm_bin is None:
-      args.opt = 'opt'
-    else:
-      args.opt = os.path.join(args.llvm_bin, 'opt')
   if not distutils.spawn.find_executable(args.opt):
     # Many uses of this tool will not need an opt binary, because it's only
     # needed for updating a test that runs clang | opt | FileCheck. So we
@@ -203,7 +208,7 @@ def main():
   script_name = os.path.basename(__file__)
 
   for ti in common.itertests(initial_args.tests, parser, 'utils/' + script_name,
-                             comment_prefix='//'):
+                             comment_prefix='//', argparse_callback=infer_dependent_args):
     # Build a list of clang command lines and check prefixes from RUN lines.
     run_list = []
     line2spell_and_mangled_list = collections.defaultdict(list)

From 2bca784ab840a9fe0f4e508cece9cbcce95838bb Mon Sep 17 00:00:00 2001
From: Georgii Rymar <grimar@accesssoftek.com>
Date: Thu, 30 Jul 2020 17:55:47 +0300
Subject: [PATCH 0897/1035] [llvm-readobj] - Massive test cases cleanup.

This patch does the following:
1) Starts using YAML macro to reduce the number of YAML documents in tests.
2) Adds `#` before 'RUN'/`CHECK` lines in a few tests where it is missing.
3) Removes unused YAML keys.
4) Starts using `ENTSIZE=<none>` to simplify tests (see D84526).
5) Removes trailing white spaces in a few places.

Differential revision: https://reviews.llvm.org/D85013
---
 .../ELF/amdgpu-elf-definitions.test           |  20 +-
 .../ELF/broken-dynamic-reloc.test             | 246 ++----------------
 .../llvm-readobj/ELF/broken-dynsym-link.test  |  18 +-
 .../llvm-readobj/ELF/call-graph-profile.test  |   3 +-
 .../llvm-readobj/ELF/dependent-libraries.test |  25 +-
 .../tools/llvm-readobj/ELF/dyn-symbols.test   |  51 ++--
 .../ELF/dynamic-reloc-no-section-headers.test |   6 +-
 .../llvm-readobj/ELF/dynamic-table-dtnull.s   |  50 ++--
 .../tools/llvm-readobj/ELF/dynamic-tags.test  | 204 ++-------------
 .../ELF/file-header-abi-version.test          |  38 +--
 .../llvm-readobj/ELF/file-header-os-abi.test  | 184 ++-----------
 .../tools/llvm-readobj/ELF/file-types.test    |  76 +-----
 .../tools/llvm-readobj/ELF/gnu-notes.test     |   4 +-
 .../tools/llvm-readobj/ELF/gnu-phdrs.test     |   5 +-
 .../ELF/gnu-section-mapping-no-phdrs.test     |   2 +-
 .../llvm-readobj/ELF/gnu-section-mapping.test |   2 +-
 .../tools/llvm-readobj/ELF/gnu-sections.test  |  93 +++----
 .../tools/llvm-readobj/ELF/hidden-versym.test |  52 ++--
 llvm/test/tools/llvm-readobj/ELF/merged.test  |  22 +-
 .../llvm-readobj/ELF/mips-got-overlapped.test |  76 +++---
 .../tools/llvm-readobj/ELF/mips-options.test  |  40 +--
 .../llvm-readobj/ELF/mips-rld-map-rel.test    |  44 ++--
 .../tools/llvm-readobj/ELF/needed-libs.test   |   2 +-
 .../tools/llvm-readobj/ELF/no-action.test     |  22 +-
 .../tools/llvm-readobj/ELF/packed-relocs.test |  90 +++----
 .../llvm-readobj/ELF/pt-gnu-property.test     |   2 +
 .../ELF/reloc-symbol-with-versioning.test     | 149 +++++------
 .../ELF/reloc-types-elf-aarch64.test          |  10 +-
 .../llvm-readobj/ELF/reloc-types-elf-arm.test |  11 +-
 .../ELF/reloc-types-elf-i386.test             |  23 +-
 .../ELF/reloc-types-elf-lanai.test            |  12 +-
 .../ELF/reloc-types-elf-mips.test             |  11 +-
 .../ELF/reloc-types-elf-mips64.test           |  11 +-
 .../llvm-readobj/ELF/reloc-types-elf-x64.test |  14 +-
 .../test/tools/llvm-readobj/ELF/sections.test |  25 +-
 llvm/test/tools/llvm-readobj/ELF/types.test   |  68 +----
 .../llvm-readobj/ELF/verdef-invalid.test      |  35 +--
 .../llvm-readobj/ELF/versym-invalid.test      |  34 +--
 38 files changed, 463 insertions(+), 1317 deletions(-)

diff --git a/llvm/test/tools/llvm-readobj/ELF/amdgpu-elf-definitions.test b/llvm/test/tools/llvm-readobj/ELF/amdgpu-elf-definitions.test
index cb0e6213cafd1..19627378094e3 100644
--- a/llvm/test/tools/llvm-readobj/ELF/amdgpu-elf-definitions.test
+++ b/llvm/test/tools/llvm-readobj/ELF/amdgpu-elf-definitions.test
@@ -1,11 +1,11 @@
-RUN: llvm-readobj --file-headers -l -S --symbols %p/Inputs/trivial.obj.elf-amdhsa-gfx803 | FileCheck %s
+# RUN: llvm-readobj --file-headers -l -S --symbols %p/Inputs/trivial.obj.elf-amdhsa-gfx803 | FileCheck %s
 
-CHECK: Format: elf64-amdgpu
-CHECK: Arch: unknown
-CHECK: ElfHeader {
-CHECK:   Ident {
-CHECK:     OS/ABI: AMDGPU_HSA (0x40)
-CHECK:     ABIVersion: 0
-CHECK:   }
-CHECK:   Machine: EM_AMDGPU (0xE0)
-CHECK: }
+# CHECK: Format: elf64-amdgpu
+# CHECK: Arch: unknown
+# CHECK: ElfHeader {
+# CHECK:   Ident {
+# CHECK:     OS/ABI: AMDGPU_HSA (0x40)
+# CHECK:     ABIVersion: 0
+# CHECK:   }
+# CHECK:   Machine: EM_AMDGPU (0xE0)
+# CHECK: }
diff --git a/llvm/test/tools/llvm-readobj/ELF/broken-dynamic-reloc.test b/llvm/test/tools/llvm-readobj/ELF/broken-dynamic-reloc.test
index 9142fc65a025f..4a079c29b83ec 100644
--- a/llvm/test/tools/llvm-readobj/ELF/broken-dynamic-reloc.test
+++ b/llvm/test/tools/llvm-readobj/ELF/broken-dynamic-reloc.test
@@ -50,7 +50,7 @@ ProgramHeaders:
       - Section: .dynamic
 
 ## Show we print a warning for an invalid relocation table size stored in a DT_RELASZ entry.
-# RUN: yaml2obj --docnum=2 %s -o %t2
+# RUN: yaml2obj --docnum=2 -DRELTYPE=RELA -DTAG1=DT_RELASZ -DTAG1VAL=0xFF -DTAG2=DT_RELAENT %s -o %t2
 # RUN: llvm-readobj --dyn-relocations %t2 2>&1 | FileCheck %s -DFILE=%t2 --check-prefix=INVALID-DT-RELASZ
 # RUN: llvm-readelf --dyn-relocations %t2 2>&1 | FileCheck %s -DFILE=%t2 --check-prefix=INVALID-DT-RELASZ
 
@@ -63,277 +63,73 @@ FileHeader:
   Type:    ET_DYN
   Machine: EM_X86_64
 Sections:
-  - Name:    .rela.dyn
-    Type:    SHT_RELA
-    Relocations:
-      - Type:   R_X86_64_NONE
+  - Name:  .relx.dyn
+    Type:  SHT_[[RELTYPE]]
   - Name: .dynamic
     Type: SHT_DYNAMIC
     Entries:
-      - Tag:   DT_RELA
+      - Tag:   DT_[[RELTYPE]]
         Value: 0x0
-      - Tag:   DT_RELASZ
-        Value: 0xFF
-      - Tag:   DT_RELAENT
-        Value: 0x18
+      - Tag:   [[TAG1]]
+        Value: [[TAG1VAL=0x18]]
+      - Tag:   [[TAG2]]
+        Value: [[TAG2VAL=0x18]]
       - Tag:   DT_NULL
         Value: 0x0
 DynamicSymbols: []
 ProgramHeaders:
   - Type:  PT_LOAD
     Sections:
-      - Section: .rela.dyn
+      - Section: .relx.dyn
       - Section: .dynamic
 
 ## Show we print a warning for an invalid relocation table entry size stored in a DT_RELAENT entry.
-# RUN: yaml2obj --docnum=3 %s -o %t3
+# RUN: yaml2obj --docnum=2 -DRELTYPE=RELA -DTAG1=DT_RELASZ -DTAG2=DT_RELAENT -DTAG2VAL=0xFF %s -o %t3
 # RUN: llvm-readobj --dyn-relocations %t3 2>&1 | FileCheck %s -DFILE=%t3 --check-prefix=INVALID-DT-RELAENT
 # RUN: llvm-readelf --dyn-relocations %t3 2>&1 | FileCheck %s -DFILE=%t3 --check-prefix=INVALID-DT-RELAENT
 
 ## INVALID-DT-RELAENT: warning: '[[FILE]]': invalid DT_RELASZ value (0x18) or DT_RELAENT value (0xff)
 
---- !ELF
-FileHeader:
-  Class:   ELFCLASS64
-  Data:    ELFDATA2LSB
-  Type:    ET_DYN
-  Machine: EM_X86_64
-Sections:
-  - Name:    .rela.dyn
-    Type:    SHT_RELA
-    Relocations:
-      - Type:   R_X86_64_NONE
-  - Name: .dynamic
-    Type: SHT_DYNAMIC
-    Entries:
-      - Tag:   DT_RELA
-        Value: 0x0
-      - Tag:   DT_RELASZ
-        Value: 0x18
-      - Tag:   DT_RELAENT
-        Value: 0xFF
-      - Tag:   DT_NULL
-        Value: 0x0
-DynamicSymbols: []
-ProgramHeaders:
-  - Type:  PT_LOAD
-    Sections:
-      - Section: .rela.dyn
-      - Section: .dynamic
-
 ## Show we print a warning for an invalid relocation table size stored in a DT_RELSZ entry.
-# RUN: yaml2obj --docnum=4 %s -o %t4
+# RUN: yaml2obj --docnum=2 -DRELTYPE=REL -DTAG1=DT_RELSZ -DTAG1VAL=0xFF -DTAG2=DT_RELENT %s -o %t4
 # RUN: llvm-readobj --dyn-relocations %t4 2>&1 | FileCheck %s -DFILE=%t4 --check-prefix=INVALID-DT-RELSZ
 # RUN: llvm-readelf --dyn-relocations %t4 2>&1 | FileCheck %s -DFILE=%t4 --check-prefix=INVALID-DT-RELSZ
 
 ## INVALID-DT-RELSZ: warning: '[[FILE]]': invalid DT_RELSZ value (0xff) or DT_RELENT value (0x18)
 
---- !ELF
-FileHeader:
-  Class:   ELFCLASS32
-  Data:    ELFDATA2LSB
-  Type:    ET_DYN
-  Machine: EM_386
-Sections:
-  - Name:    .rela.dyn
-    Type:    SHT_REL
-    Relocations:
-      - Type:   R_386_NONE
-  - Name: .dynamic
-    Type: SHT_DYNAMIC
-    Entries:
-      - Tag:   DT_REL
-        Value: 0x0
-      - Tag:   DT_RELSZ
-        Value: 0xFF
-      - Tag:   DT_RELENT
-        Value: 0x18
-      - Tag:   DT_NULL
-        Value: 0x0
-DynamicSymbols: []
-ProgramHeaders:
-  - Type:  PT_LOAD
-    Sections:
-      - Section: .rela.dyn
-      - Section: .dynamic
-
 ## Show we print a warning for an invalid relocation table entry size stored in a DT_RELENT entry.
-# RUN: yaml2obj --docnum=5 %s -o %t5
+# RUN: yaml2obj --docnum=2 -DRELTYPE=REL -DTAG1=DT_RELSZ -DTAG2=DT_RELENT -DTAG2VAL=0xFF %s -o %t5
 # RUN: llvm-readobj --dyn-relocations %t5 2>&1 | FileCheck %s -DFILE=%t5 --check-prefix=INVALID-DT-RELENT
 # RUN: llvm-readelf --dyn-relocations %t5 2>&1 | FileCheck %s -DFILE=%t5 --check-prefix=INVALID-DT-RELENT
 
 ## INVALID-DT-RELENT: warning: '[[FILE]]': invalid DT_RELSZ value (0x18) or DT_RELENT value (0xff)
 
---- !ELF
-FileHeader:
-  Class:   ELFCLASS32
-  Data:    ELFDATA2LSB
-  Type:    ET_DYN
-  Machine: EM_386
-Sections:
-  - Name:    .rela.dyn
-    Type:    SHT_REL
-    Relocations:
-      - Type:   R_386_NONE
-  - Name: .dynamic
-    Type: SHT_DYNAMIC
-    Entries:
-      - Tag:   DT_REL
-        Value: 0x0
-      - Tag:   DT_RELSZ
-        Value: 0x18
-      - Tag:   DT_RELENT
-        Value: 0xFF
-      - Tag:   DT_NULL
-        Value: 0x0
-DynamicSymbols: []
-ProgramHeaders:
-  - Type:  PT_LOAD
-    Sections:
-      - Section: .rela.dyn
-      - Section: .dynamic
-
 ## Show we print a warning for an invalid relocation table size stored in a DT_RELRSZ/DT_ANDROID_RELRSZ entry.
-# RUN: yaml2obj --docnum=6 %s -o %t6
+# RUN: yaml2obj --docnum=2 -DRELTYPE=RELR -DTAG1=DT_RELRSZ -DTAG1VAL=0xFF -DTAG2=DT_RELRENT %s -o %t6
 # RUN: llvm-readobj --dyn-relocations %t6 2>&1 | FileCheck %s -DFILE=%t6 --check-prefix=INVALID-DT-RELRSZ
 # RUN: llvm-readelf --dyn-relocations %t6 2>&1 | FileCheck %s -DFILE=%t6 --check-prefix=INVALID-DT-RELRSZ
-# RUN: yaml2obj --docnum=7 %s -o %t7
+
+# RUN: yaml2obj --docnum=2 -DRELTYPE=RELR -DTAG1=DT_ANDROID_RELRSZ -DTAG1VAL=0xFF -DTAG2=DT_ANDROID_RELRENT %s -o %t7
 # RUN: llvm-readobj --dyn-relocations %t7 2>&1 | FileCheck %s -DFILE=%t7 --check-prefix=INVALID-DT-ANDROID-RELRSZ
 # RUN: llvm-readelf --dyn-relocations %t7 2>&1 | FileCheck %s -DFILE=%t7 --check-prefix=INVALID-DT-ANDROID-RELRSZ
 
 ## INVALID-DT-RELRSZ:         warning: '[[FILE]]': invalid DT_RELRSZ value (0xff) or DT_RELRENT value (0x18)
 ## INVALID-DT-ANDROID-RELRSZ: warning: '[[FILE]]': invalid DT_ANDROID_RELRSZ value (0xff) or DT_ANDROID_RELRENT value (0x18)
 
---- !ELF
-FileHeader:
-  Class:   ELFCLASS64
-  Data:    ELFDATA2LSB
-  Type:    ET_DYN
-  Machine: EM_X86_64
-Sections:
-  - Name:    .relr.dyn
-    Type:    SHT_RELR
-    Flags:   [ SHF_ALLOC ]
-    Content: ""
-  - Name: .dynamic
-    Type: SHT_DYNAMIC
-    Entries:
-      - Tag:   DT_RELR
-        Value: 0x0
-      - Tag:   DT_RELRSZ
-        Value: 0xFF
-      - Tag:   DT_RELRENT
-        Value: 0x18
-      - Tag:   DT_NULL
-        Value: 0x0
-DynamicSymbols: []
-ProgramHeaders:
-  - Type:  PT_LOAD
-    Sections:
-      - Section: .relr.dyn
-      - Section: .dynamic
-
---- !ELF
-FileHeader:
-  Class:   ELFCLASS64
-  Data:    ELFDATA2LSB
-  Type:    ET_DYN
-  Machine: EM_X86_64
-Sections:
-  - Name:    .relr.dyn
-    Type:    SHT_RELR
-    Flags:   [ SHF_ALLOC ]
-    Content: ""
-  - Name: .dynamic
-    Type: SHT_DYNAMIC
-    Entries:
-      - Tag:   DT_RELR
-        Value: 0x0
-      - Tag:   DT_ANDROID_RELRSZ
-        Value: 0xFF
-      - Tag:   DT_ANDROID_RELRENT
-        Value: 0x18
-      - Tag:   DT_NULL
-        Value: 0x0
-DynamicSymbols: []
-ProgramHeaders:
-  - Type:  PT_LOAD
-    Sections:
-      - Section: .relr.dyn
-      - Section: .dynamic
-
 ## Show we print a warning for an invalid relocation table entry size stored in a DT_RELRENT/DT_ANDROID_RELRENT entry.
-# RUN: yaml2obj --docnum=8 %s -o %t8
+# RUN: yaml2obj --docnum=2 -DRELTYPE=RELR -DTAG1=DT_RELRSZ -DTAG2=DT_RELRENT -DTAG2VAL=0xFF %s -o %t8
 # RUN: llvm-readobj --dyn-relocations %t8 2>&1 | FileCheck %s -DFILE=%t8 --check-prefix=INVALID-DT-RELRENT
 # RUN: llvm-readelf --dyn-relocations %t8 2>&1 | FileCheck %s -DFILE=%t8 --check-prefix=INVALID-DT-RELRENT
-# RUN: yaml2obj --docnum=9 %s -o %t9
+# RUN: yaml2obj --docnum=2 -DRELTYPE=RELR -DTAG1=DT_ANDROID_RELRSZ -DTAG2=DT_ANDROID_RELRENT -DTAG2VAL=0xFF %s -o %t9
 # RUN: llvm-readobj --dyn-relocations %t9 2>&1 | FileCheck %s -DFILE=%t9 --check-prefix=INVALID-DT-ANDROID-RELRENT
 # RUN: llvm-readelf --dyn-relocations %t9 2>&1 | FileCheck %s -DFILE=%t9 --check-prefix=INVALID-DT-ANDROID-RELRENT
 
 ## INVALID-DT-RELRENT:         invalid DT_RELRSZ value (0x18) or DT_RELRENT value (0xff)
 ## INVALID-DT-ANDROID-RELRENT: invalid DT_ANDROID_RELRSZ value (0x18) or DT_ANDROID_RELRENT value (0xff)
 
---- !ELF
-FileHeader:
-  Class:   ELFCLASS64
-  Data:    ELFDATA2LSB
-  Type:    ET_DYN
-  Machine: EM_X86_64
-Sections:
-  - Name:    .relr.dyn
-    Type:    SHT_RELR
-    Flags:   [ SHF_ALLOC ]
-    Content: ""
-  - Name: .dynamic
-    Type: SHT_DYNAMIC
-    Entries:
-      - Tag:   DT_RELR
-        Value: 0x0
-      - Tag:   DT_RELRSZ
-        Value: 0x18
-      - Tag:   DT_RELRENT
-        Value: 0xFF
-      - Tag:   DT_NULL
-        Value: 0x0
-DynamicSymbols: []
-ProgramHeaders:
-  - Type:  PT_LOAD
-    Sections:
-      - Section: .relr.dyn
-      - Section: .dynamic
-
---- !ELF
-FileHeader:
-  Class:   ELFCLASS64
-  Data:    ELFDATA2LSB
-  Type:    ET_DYN
-  Machine: EM_X86_64
-Sections:
-  - Name:    .relr.dyn
-    Type:    SHT_RELR
-    Flags:   [ SHF_ALLOC ]
-    Content: ""
-  - Name: .dynamic
-    Type: SHT_DYNAMIC
-    Entries:
-      - Tag:   DT_RELR
-        Value: 0x0
-      - Tag:   DT_ANDROID_RELRSZ
-        Value: 0x18
-      - Tag:   DT_ANDROID_RELRENT
-        Value: 0xFF
-      - Tag:   DT_NULL
-        Value: 0x0
-DynamicSymbols: []
-ProgramHeaders:
-  - Type:  PT_LOAD
-    Sections:
-      - Section: .relr.dyn
-      - Section: .dynamic
-
 ## Show we print a warning for an invalid value of DT_PLTRELSZ, which describes the total size
 ## of the relocation entries associated with the procedure linkage table.
-# RUN: yaml2obj --docnum=10 %s -o %t10
+# RUN: yaml2obj --docnum=3 %s -o %t10
 # RUN: llvm-readobj --dyn-relocations %t10 2>&1 | FileCheck %s -DFILE=%t10 --check-prefix=INVALID-DT-PLTRELSZ
 # RUN: llvm-readelf --dyn-relocations %t10 2>&1 | FileCheck %s -DFILE=%t10 --check-prefix=INVALID-DT-PLTRELSZ
 
@@ -369,7 +165,7 @@ ProgramHeaders:
       - Section: .dynamic
 
 ## Show we print a warning when dumping dynamic relocations if there is no dynamic symbol table.
-# RUN: yaml2obj --docnum=11 %s -o %t11
+# RUN: yaml2obj --docnum=4 %s -o %t11
 # RUN: llvm-readobj --dyn-relocations %t11 2>&1 | FileCheck %s -DFILE=%t11 --check-prefix=LLVM-NO-DYNSYM
 # RUN: llvm-readelf --dyn-relocations %t11 2>&1 | FileCheck %s -DFILE=%t11 --check-prefix=GNU-NO-DYNSYM
 
@@ -419,7 +215,7 @@ ProgramHeaders:
 
 ## Show we print a warning when the symbol index of a dynamic relocation is too
 ## large (goes past the end of the dynamic symbol table).
-# RUN: yaml2obj --docnum=12 %s -o %t12
+# RUN: yaml2obj --docnum=5 %s -o %t12
 # RUN: llvm-readobj --dyn-relocations %t12 2>&1 | FileCheck %s -DFILE=%t12 --check-prefix=LLVM-INVALID-DYNSYM
 # RUN: llvm-readelf --dyn-relocations %t12 2>&1 | FileCheck %s -DFILE=%t12 --check-prefix=GNU-INVALID-DYNSYM
 
@@ -465,7 +261,7 @@ ProgramHeaders:
       - Section: .dynamic
 
 ## Show that when we have both REL and RELA relocations, we dump both sets.
-# RUN: yaml2obj --docnum=13 %s -o %t13
+# RUN: yaml2obj --docnum=6 %s -o %t13
 # RUN: llvm-readobj --dyn-relocations %t13 2>&1 | FileCheck %s -DFILE=%t13 --check-prefix=BOTH-RELA-REL-LLVM
 # RUN: llvm-readelf --dyn-relocations %t13 2>&1 | FileCheck %s -DFILE=%t13 --check-prefix=BOTH-RELA-REL-GNU
 
diff --git a/llvm/test/tools/llvm-readobj/ELF/broken-dynsym-link.test b/llvm/test/tools/llvm-readobj/ELF/broken-dynsym-link.test
index d64b1dfb11a87..a685c7eccc1f4 100644
--- a/llvm/test/tools/llvm-readobj/ELF/broken-dynsym-link.test
+++ b/llvm/test/tools/llvm-readobj/ELF/broken-dynsym-link.test
@@ -2,7 +2,7 @@
 ## .dynsym section's sh_link field is broken.
 
 ## Case 1: sh_link is set to 0.
-# RUN: yaml2obj --docnum=1 %s -o %t1
+# RUN: yaml2obj --docnum=1 -DLINK=0 %s -o %t1
 # RUN: llvm-readobj -S %t1 2>&1 | FileCheck %s -DFILE=%t1 --check-prefixes=LLVM,ERR
 # RUN: llvm-readelf -S %t1 2>&1 | FileCheck %s -DFILE=%t1 --check-prefixes=GNU,ERR
 
@@ -32,25 +32,15 @@ FileHeader:
 Sections:
   - Name: .dynsym
     Type: SHT_DYNSYM
+    Link: [[LINK]]
 
 ## Case 2: sh_link is set to 255, which is larger than the number of the sections.
-# RUN: yaml2obj --docnum=2 %s -o %t2
+# RUN: yaml2obj --docnum=1 -DLINK=255 %s -o %t2
 
 # RUN: llvm-readobj -S %t2 2>&1 | FileCheck -DFILE=%t2 %s --check-prefixes=LLVM2,ERR2
 # RUN: llvm-readelf -S %t2 2>&1 | FileCheck -DFILE=%t2 %s --check-prefixes=GNU2,ERR2
 
-# ERR2: warning: '[[FILE]]': invalid section index
-
---- !ELF
-FileHeader:
-  Class:   ELFCLASS64
-  Data:    ELFDATA2LSB
-  Type:    ET_DYN
-  Machine: EM_X86_64
-Sections:
-  - Name: .dynsym
-    Type: SHT_DYNSYM
-    Link: 255
+# ERR2: warning: '[[FILE]]': invalid section index: 255
 
 # LLVM2:      Name: .dynsym
 # LLVM2-NEXT:  Type: SHT_DYNSYM
diff --git a/llvm/test/tools/llvm-readobj/ELF/call-graph-profile.test b/llvm/test/tools/llvm-readobj/ELF/call-graph-profile.test
index 8ccc93cf426f0..65709cb5f6969 100644
--- a/llvm/test/tools/llvm-readobj/ELF/call-graph-profile.test
+++ b/llvm/test/tools/llvm-readobj/ELF/call-graph-profile.test
@@ -37,8 +37,7 @@ Sections:
       - From:   bar
         To:     foo
         Weight: 98
-## 0x10 is the normal entry size for the SHT_LLVM_CALL_GRAPH_PROFILE section.
-    EntSize: [[ENTSIZE=0x10]]
+    EntSize: [[ENTSIZE=<none>]]
 Symbols:
   - Name: foo
   - Name: bar
diff --git a/llvm/test/tools/llvm-readobj/ELF/dependent-libraries.test b/llvm/test/tools/llvm-readobj/ELF/dependent-libraries.test
index 97f8a75781398..440217f3253d3 100644
--- a/llvm/test/tools/llvm-readobj/ELF/dependent-libraries.test
+++ b/llvm/test/tools/llvm-readobj/ELF/dependent-libraries.test
@@ -30,8 +30,18 @@ FileHeader:
 Sections:
   - Name:      .deplibs
     Type:      SHT_LLVM_DEPENDENT_LIBRARIES
+    ShType:    [[TYPE=<none>]]
     Libraries: [ foo, bar, foo ]
 
+## Show the output when there are no dependent library sections. Check that we are
+## locating dependent library sections by type. To do this we change the type to an arbitrary one.
+# RUN: yaml2obj --docnum=1 -DTYPE=SHT_PROGBITS %s -o %t3
+# RUN: llvm-readobj --dependent-libraries %t3 2>&1 | FileCheck %s --check-prefix=NONE
+# RUN: llvm-readelf --dependent-libraries %t3 2>&1 | FileCheck %s --allow-empty --implicit-check-not={{.}}
+
+# NONE:      DependentLibs [
+# NONE-NEXT: ]
+
 ## Now, check how we dump a mix of valid, empty and invalid SHT_LLVM_DEPENDENT_LIBRARIES sections.
 
 # RUN: yaml2obj --docnum=2 %s -o %t2
@@ -98,18 +108,3 @@ Sections:
   - ShName:    0x10000
     Type:      SHT_LLVM_DEPENDENT_LIBRARIES
     Libraries: [ baz ]
-
-## Show the output when there are no dependent library sections.
-# RUN: yaml2obj --docnum=3 %s -o %t3
-# RUN: llvm-readobj --dependent-libraries %t3 2>&1 | FileCheck %s --check-prefix=NONE
-# RUN: llvm-readelf --dependent-libraries %t3 2>&1 | FileCheck %s --allow-empty --implicit-check-not={{.}}
-
-# NONE:      DependentLibs [
-# NONE-NEXT: ]
-
---- !ELF
-FileHeader:
-  Class:   ELFCLASS64
-  Data:    ELFDATA2LSB
-  Type:    ET_REL
-  Machine: EM_X86_64
diff --git a/llvm/test/tools/llvm-readobj/ELF/dyn-symbols.test b/llvm/test/tools/llvm-readobj/ELF/dyn-symbols.test
index 6169871034d5f..4a9daa2641295 100644
--- a/llvm/test/tools/llvm-readobj/ELF/dyn-symbols.test
+++ b/llvm/test/tools/llvm-readobj/ELF/dyn-symbols.test
@@ -134,41 +134,24 @@ Sections:
         Value: 0xffff1234
       - Tag:   DT_NULL
         Value: 0
+  - Name:   .dynsym
+    Type:   SHT_DYNSYM
+    ShName: [[DYNSYMNAME=<none>]]
 DynamicSymbols:
   - Name: foo
 
 ## Case 3.2: the same as 3.1, but the sh_name field of the SHT_DYNSYM section is invalid.
 ##           Check we are still able to dump symbols.
-# RUN: yaml2obj --docnum=3 %s -o %t2.broken.name
+# RUN: yaml2obj --docnum=2 -DDYNSYMNAME=0xffffffff %s -o %t2.broken.name
 # RUN: llvm-readobj %t2.broken.name --dyn-symbols 2>&1 | \
 # RUN:   FileCheck %s -DFILE=%t2.broken.name --check-prefix=NOPHDRS-LLVM --implicit-check-not=warning:
 # RUN: llvm-readelf %t2.broken.name --dyn-symbols 2>&1 | \
 # RUN:   FileCheck %s -DFILE=%t2.broken.name -DNAME="<?>" \
 # RUN:     --check-prefixes=NOPHDRS-GNU,NOPHDRS-NAMEWARN --implicit-check-not=warning:
 
---- !ELF
-FileHeader:
-  Class:   ELFCLASS64
-  Data:    ELFDATA2LSB
-  Type:    ET_DYN
-  Machine: EM_X86_64
-Sections:
-  - Name:    .dynamic
-    Type:    SHT_DYNAMIC
-    Entries:
-      - Tag:   DT_SYMTAB
-        Value: 0xffff1234
-      - Tag:   DT_NULL
-        Value: 0
-  - Name:   .dynsym
-    Type:   SHT_DYNSYM
-    ShName: 0xffffffff
-DynamicSymbols:
-  - Name: foo
-
 ## Case 4: Check we report a warning when there is no SHT_DYNSYM section and we can't map the DT_SYMTAB value
 ## to an address because of the absence of a corresponding PT_LOAD program header.
-# RUN: yaml2obj --docnum=4 %s -o %t3.so
+# RUN: yaml2obj --docnum=3 %s -o %t3.so
 # RUN: llvm-readobj %t3.so --dyn-symbols 2>&1 | FileCheck %s -DFILE=%t3.so --check-prefixes=NOSHT-DYNSYM,NOSHT-DYNSYM-LLVM
 # RUN: llvm-readelf %t3.so --dyn-symbols 2>&1 | FileCheck %s -DFILE=%t3.so --check-prefix=NOSHT-DYNSYM
 
@@ -197,7 +180,7 @@ DynamicSymbols:
 
 ## Case 5: Check that when we can't map the value of the DT_SYMTAB tag to an address, we report a warning and
 ## use the information in the section header table to locate the dynamic symbol table.
-# RUN: yaml2obj --docnum=5 %s -o %t4.so
+# RUN: yaml2obj --docnum=4 %s -o %t4.so
 # RUN: llvm-readobj %t4.so --dyn-symbols 2>&1 | FileCheck -DFILE=%t4.so %s --check-prefixes=BROKEN-DTSYMTAB,BROKEN-DTSYMTAB-LLVM
 # RUN: llvm-readelf %t4.so --dyn-symbols 2>&1 | FileCheck -DFILE=%t4.so %s --check-prefixes=BROKEN-DTSYMTAB,BROKEN-DTSYMTAB-GNU
 
@@ -229,7 +212,7 @@ ProgramHeaders:
 
 ## Case 6: Check that if we can get the location of the dynamic symbol table using both the DT_SYMTAB value
 ## and the section headers table then we prefer the former and report a warning.
-# RUN: yaml2obj --docnum=6 %s -o %t5.so
+# RUN: yaml2obj --docnum=5 %s -o %t5.so
 # RUN: llvm-readobj %t5.so --dyn-symbols 2>&1 | FileCheck -DFILE=%t5.so %s --check-prefixes=PREFER-DTSYMTAB,PREFER-DTSYMTAB-LLVM
 # RUN: llvm-readelf %t5.so --dyn-symbols 2>&1 | FileCheck -DFILE=%t5.so %s --check-prefixes=PREFER-DTSYMTAB,PREFER-DTSYMTAB-GNU
 
@@ -267,7 +250,7 @@ ProgramHeaders:
 
 ## Case 7: Check how we dump versioned symbols. Use both -V and --dyn-symbols
 ## to check that printed version is consistent.
-# RUN: yaml2obj %s --docnum=7 -o %t6
+# RUN: yaml2obj %s --docnum=6 -o %t6
 # RUN: llvm-readobj -V --dyn-symbols %t6 | FileCheck %s --check-prefix=VERSIONED-LLVM
 # RUN: llvm-readelf -V --dyn-symbols %t6 | FileCheck %s --check-prefix=VERSIONED-GNU
 
@@ -350,15 +333,15 @@ DynamicSymbols:
 
 ## Case 8: Check what we print when:
 ## a) The dynamic symbol table does not exist.
-# RUN: yaml2obj %s --docnum=8 -o %t7
+# RUN: yaml2obj %s --docnum=7 -o %t7
 # RUN: llvm-readobj --dyn-symbols %t7 | FileCheck %s --check-prefix=NO-DYNSYM-LLVM
 # RUN: llvm-readelf --dyn-symbols %t7 | count 0
 ## b) The dynamic symbol table has a size of 0.
-# RUN: yaml2obj %s --docnum=9 -o %t8
+# RUN: yaml2obj %s --docnum=8 -o %t8
 # RUN: llvm-readobj --dyn-symbols %t8 | FileCheck %s --check-prefix=NO-DYNSYM-LLVM
 # RUN: llvm-readelf --dyn-symbols %t8 | count 0
 ## c) The dynamic symbol table only contains the null symbol.
-# RUN: yaml2obj %s --docnum=10 -o %t9
+# RUN: yaml2obj %s --docnum=9 -o %t9
 # RUN: llvm-readobj --dyn-symbols %t9 | FileCheck %s --check-prefix=DYNSYM-EMPTY-LLVM
 # RUN: llvm-readelf --dyn-symbols %t9 | FileCheck %s --check-prefix=DYNSYM-EMPTY-GNU
 
@@ -409,7 +392,7 @@ DynamicSymbols: []
 
 ## Case 9: Check what we print when:
 ## a) The size of the dynamic symbol table is not a multiple of its entry size.
-# RUN: yaml2obj %s --docnum=11 -o %t10
+# RUN: yaml2obj %s --docnum=10 -o %t10
 # RUN: llvm-readobj --dyn-symbols %t10 2>&1 | FileCheck %s -DFILE=%t10 --check-prefix=DYNSYM-SIZE-INVALID1
 # RUN: llvm-readelf --dyn-symbols %t10 2>&1 | FileCheck %s -DFILE=%t10 --check-prefix=DYNSYM-SIZE-INVALID1
 
@@ -419,7 +402,7 @@ DynamicSymbols: []
 ##    information about a location and an entity size of the dynamic symbol table from the section header.
 ##    The code uses sizeof(Elf_Sym) for an entity size, so it can't be incorrect and
 ##    the message printed is a bit shorter.
-# RUN: yaml2obj %s --docnum=12 -o %t11
+# RUN: yaml2obj %s --docnum=11 -o %t11
 # RUN: llvm-readobj --dyn-symbols %t11 2>&1 | FileCheck %s -DFILE=%t11 --check-prefix=DYNSYM-SIZE-INVALID2
 # RUN: llvm-readelf --dyn-symbols %t11 2>&1 | FileCheck %s -DFILE=%t11 --check-prefix=DYNSYM-SIZE-INVALID2
 
@@ -427,10 +410,10 @@ DynamicSymbols: []
 
 ## c) In the case when the DT_SYMENT tag is present, we report when it's value does not match the
 #     value of the symbol size for the platform.
-# RUN: yaml2obj %s -D BITS=32 --docnum=13 -o %t12
+# RUN: yaml2obj %s -D BITS=32 --docnum=12 -o %t12
 # RUN: llvm-readobj --dyn-symbols %t12 2>&1 | FileCheck %s -DFILE=%t12 --check-prefix=DYNSYM-SIZE-INVALID3
 # RUN: llvm-readelf --dyn-symbols %t12 2>&1 | FileCheck %s -DFILE=%t12 --check-prefix=DYNSYM-SIZE-INVALID3
-# RUN: yaml2obj %s -D BITS=64 --docnum=13 -o %t13
+# RUN: yaml2obj %s -D BITS=64 --docnum=12 -o %t13
 # RUN: llvm-readobj --dyn-symbols %t13 2>&1 | FileCheck %s -DFILE=%t13 --check-prefix=DYNSYM-SIZE-INVALID4
 # RUN: llvm-readelf --dyn-symbols %t13 2>&1 | FileCheck %s -DFILE=%t13 --check-prefix=DYNSYM-SIZE-INVALID4
 
@@ -490,7 +473,7 @@ Sections:
 ## Check we report a warning when the DT_STRSZ value is broken so that the dynamic string
 ## table goes past the end of the file. Document we stop dumping symbols and report an error.
 
-# RUN: yaml2obj %s --docnum=14 -o %t14
+# RUN: yaml2obj %s --docnum=13 -o %t14
 # RUN: llvm-readobj --dyn-symbols %t14 2>&1 | \
 # RUN:   FileCheck %s -DFILE=%t14 --check-prefix=DYNSTR-INVALID-LLVM
 # RUN: llvm-readelf --dyn-symbols %t14 2>&1 | \
@@ -570,7 +553,7 @@ ProgramHeaders:
       - Section: .dynamic
 
 ## Check we report a warning when the entry size of the dynamic symbol table is zero.
-# RUN: yaml2obj %s --docnum=15 -o %t15
+# RUN: yaml2obj %s --docnum=14 -o %t15
 # RUN: llvm-readobj --dyn-symbols %t15 2>&1 | FileCheck %s -DFILE=%t15 --check-prefix=DYNSYM-ZERO-ENTSIZE-LLVM
 # RUN: llvm-readelf --dyn-symbols %t15 2>&1 | \
 # RUN:   FileCheck %s -DFILE=%t15 --check-prefix=DYNSYM-ZERO-ENTSIZE-GNU --implicit-check-not="Symbol table"
diff --git a/llvm/test/tools/llvm-readobj/ELF/dynamic-reloc-no-section-headers.test b/llvm/test/tools/llvm-readobj/ELF/dynamic-reloc-no-section-headers.test
index 1ec81578b9cc9..b6c6f102c9c1b 100644
--- a/llvm/test/tools/llvm-readobj/ELF/dynamic-reloc-no-section-headers.test
+++ b/llvm/test/tools/llvm-readobj/ELF/dynamic-reloc-no-section-headers.test
@@ -21,10 +21,6 @@ FileHeader:
   Data:    ELFDATA2LSB
   Type:    ET_DYN
   Machine: EM_X86_64
-## We simulate no section header table by
-## overriding the ELF header properties.
-  EShOff:  0x0
-  EShNum:  0x0
 Sections:
   - Name:    .rela.dyn
     Type:    SHT_RELA
@@ -66,3 +62,5 @@ ProgramHeaders:
   - Type:  PT_DYNAMIC
     Sections:
       - Section: .dynamic
+SectionHeaderTable:
+  NoHeaders: true
diff --git a/llvm/test/tools/llvm-readobj/ELF/dynamic-table-dtnull.s b/llvm/test/tools/llvm-readobj/ELF/dynamic-table-dtnull.s
index 5524bdde154c5..b199bc428f2f9 100644
--- a/llvm/test/tools/llvm-readobj/ELF/dynamic-table-dtnull.s
+++ b/llvm/test/tools/llvm-readobj/ELF/dynamic-table-dtnull.s
@@ -15,26 +15,21 @@
 
 --- !ELF
 FileHeader:
-  Class:           ELFCLASS64
-  Data:            ELFDATA2LSB
-  Type:            ET_EXEC
-  Machine:         EM_X86_64
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
 Sections:
-  - Name:            .dynamic
-    Type:            SHT_DYNAMIC
-    Address:         0x0000000000001010
-    AddressAlign:    0x0000000000000010
-    EntSize:         0x0000000000000010
+  - Name: .dynamic
+    Type: SHT_DYNAMIC
     Entries:
-      - Tag:             DT_DEBUG
-        Value:           0x0000000000000000
+      - Tag:   DT_DEBUG
+        Value: 0x0000000000000000
 ProgramHeaders:
   - Type: PT_LOAD
-    VAddr: 0x1000
     Sections:
       - Section: .dynamic
   - Type: PT_DYNAMIC
-    VAddr: 0x1010
     Sections:
       - Section: .dynamic
 
@@ -59,29 +54,24 @@ ProgramHeaders:
 
 --- !ELF
 FileHeader:
-  Class:           ELFCLASS64
-  Data:            ELFDATA2LSB
-  Type:            ET_EXEC
-  Machine:         EM_X86_64
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
 Sections:
-  - Name:            .dynamic
-    Type:            SHT_DYNAMIC
-    Address:         0x0000000000001010
-    AddressAlign:    0x0000000000000010
-    EntSize:         0x0000000000000010
+  - Name: .dynamic
+    Type: SHT_DYNAMIC
     Entries:
-      - Tag:             DT_DEBUG
-        Value:           0x0000000000000000
-      - Tag:             DT_NULL
-        Value:           0x0000000000000000
-      - Tag:             DT_NULL
-        Value:           0x0000000000000000
+      - Tag:   DT_DEBUG
+        Value: 0x0000000000000000
+      - Tag:   DT_NULL
+        Value: 0x0000000000000000
+      - Tag:   DT_NULL
+        Value: 0x0000000000000000
 ProgramHeaders:
   - Type: PT_LOAD
-    VAddr: 0x1000
     Sections:
       - Section: .dynamic
   - Type: PT_DYNAMIC
-    VAddr: 0x1010
     Sections:
       - Section: .dynamic
diff --git a/llvm/test/tools/llvm-readobj/ELF/dynamic-tags.test b/llvm/test/tools/llvm-readobj/ELF/dynamic-tags.test
index 5f8b709cef5b6..24cc439bdbdbb 100644
--- a/llvm/test/tools/llvm-readobj/ELF/dynamic-tags.test
+++ b/llvm/test/tools/llvm-readobj/ELF/dynamic-tags.test
@@ -140,11 +140,11 @@
 
 --- !ELF
 FileHeader:
-  Class:      ELFCLASS64
+  Class:      ELFCLASS[[BITS=64]]
   Data:       ELFDATA2LSB
   Type:       ET_EXEC
-  Machine:    EM_X86_64
-  EPhEntSize: [[PHENTSIZE=56]]
+  Machine:    EM_NONE
+  EPhEntSize: [[PHENTSIZE=<none>]]
 Sections:
   - Name:    .dynstr
     Type:    SHT_STRTAB
@@ -186,7 +186,7 @@ Sections:
       - Tag:   DT_RPATH
         Value: 0x5
       - Tag:   DT_SYMBOLIC
-        Value: 0x1234567890abcdef
+        Value: [[SYMBOLIC=0x1234567890abcdef]]
       - Tag:   DT_REL
         Value: 0x1000
       - Tag:   DT_RELSZ
@@ -196,13 +196,13 @@ Sections:
       - Tag:   DT_PLTREL
         Value: 0x7
       - Tag:   DT_DEBUG
-        Value: 0xfedcba0987654321
+        Value: [[DEBUG=0xfedcba0987654321]]
       - Tag:   DT_TEXTREL
-        Value: 0x1122334455667788
+        Value: [[TEXTREL=0x1122334455667788]]
       - Tag:   DT_JMPREL
         Value: 0x1000
       - Tag:   DT_BIND_NOW
-        Value: 0x8877665544332211
+        Value: [[BINDNOW=0x8877665544332211]]
       - Tag:   DT_INIT_ARRAY
         Value: 0x1000
       - Tag:   DT_FINI_ARRAY
@@ -214,7 +214,7 @@ Sections:
       - Tag:   DT_RUNPATH
         Value: 0x7
       - Tag:   DT_FLAGS
-        Value: 0xffffffffffffffff
+        Value: [[FLAGS=0xffffffffffffffff]]
       - Tag:   DT_PREINIT_ARRAY
         Value: 0x1000
       - Tag:   DT_PREINIT_ARRAYSZ
@@ -252,7 +252,7 @@ Sections:
       - Tag:   DT_RELCOUNT
         Value: 0x0
       - Tag:   DT_FLAGS_1
-        Value: 0xffffffffffffffff
+        Value: [[FLAGS=0xffffffffffffffff]]
       - Tag:   DT_VERSYM
         Value: 0x1000
       - Tag:   DT_VERDEF
@@ -272,13 +272,13 @@ Sections:
 ## Show behaviour for unknown values in special and un-marked ranges.
 ## An arbitrary unknown value outside of the special ranges.
       - Tag:   0x12345678
-        Value: 0x8765432187654321
+        Value: [[UNKNOWN=0x8765432187654321]]
 ## An OS specific tag.
       - Tag:   0x6abcdef0
-        Value: 0x9988776655443322
+        Value: [[OSSPEC=0x9988776655443322]]
 ## A processor specific tags.
       - Tag:   0x76543210
-        Value: 0x5555666677778888
+        Value: [[PROCSPEC=0x5555666677778888]]
       - Tag:   DT_NULL
         Value: 0x0
 ProgramHeaders:
@@ -292,7 +292,9 @@ ProgramHeaders:
     Sections:
       - Section: .dynamic
 
-# RUN: yaml2obj %s --docnum=2 -o %t2
+# RUN: yaml2obj %s --docnum=1 -DBITS=32 -DSYMBOLIC=0x12345678 -DDEBUG=0xfedcba09 \
+# RUN:   -DTEXTREL=0x11223344 -DBINDNOW=0x88776655 -DFLAGS=0xffffffff \
+# RUN:   -DUNKNOWN=0x87654321 -DOSSPEC=0x99887766 -DPROCSPEC=0x55556666 -o %t2
 # RUN: llvm-readobj --dynamic-table %t2 \
 # RUN:   | FileCheck %s --check-prefix=LLVM32 --strict-whitespace --match-full-lines
 # RUN: llvm-readobj -d %t2 | FileCheck %s --check-prefix=LLVM32 --strict-whitespace --match-full-lines
@@ -429,169 +431,16 @@ ProgramHeaders:
 # GNU32-NEXT:  0x76543210 (<unknown:>0x76543210) 0x55556666
 # GNU32-NEXT:  0x00000000 (NULL)                 0x0
 
---- !ELF
-FileHeader:
-  Class:   ELFCLASS32
-  Data:    ELFDATA2LSB
-  Type:    ET_EXEC
-  Machine: EM_386
-Sections:
-  - Name:    .dynstr
-    Type:    SHT_STRTAB
-    Address: 0x1000
-    Size:    0x10
-    Content: "004400550066007700"
-  - Name:    .dynamic
-    Type:    SHT_DYNAMIC
-    Address: 0x1010
-    Entries:
-      - Tag:   DT_NEEDED
-        Value: 0x1
-      - Tag:   DT_PLTRELSZ
-        Value: 0x10
-      - Tag:   DT_PLTGOT
-        Value: 0x1000
-      - Tag:   DT_HASH
-        Value: 0x1000
-      - Tag:   DT_STRTAB
-        Value: 0x1000
-      - Tag:   DT_SYMTAB
-        Value: 0x1000
-      - Tag:   DT_RELA
-        Value: 0x1000
-      - Tag:   DT_RELASZ
-        Value: 0x10
-      - Tag:   DT_RELAENT
-        Value: 0x789
-      - Tag:   DT_STRSZ
-        Value: 0x10
-      - Tag:   DT_SYMENT
-        Value: 0x987
-      - Tag:   DT_INIT
-        Value: 0x1000
-      - Tag:   DT_FINI
-        Value: 0x1000
-      - Tag:   DT_SONAME
-        Value: 0x3
-      - Tag:   DT_RPATH
-        Value: 0x5
-      - Tag:   DT_SYMBOLIC
-        Value: 0x12345678
-      - Tag:   DT_REL
-        Value: 0x1000
-      - Tag:   DT_RELSZ
-        Value: 0x10
-      - Tag:   DT_RELENT
-        Value: 0x123
-      - Tag:   DT_PLTREL
-        Value: 0x7
-      - Tag:   DT_DEBUG
-        Value: 0xfedcba09
-      - Tag:   DT_TEXTREL
-        Value: 0x11223344
-      - Tag:   DT_JMPREL
-        Value: 0x1000
-      - Tag:   DT_BIND_NOW
-        Value: 0x88776655
-      - Tag:   DT_INIT_ARRAY
-        Value: 0x1000
-      - Tag:   DT_FINI_ARRAY
-        Value: 0x1000
-      - Tag:   DT_INIT_ARRAYSZ
-        Value: 0x10
-      - Tag:   DT_FINI_ARRAYSZ
-        Value: 0x10
-      - Tag:   DT_RUNPATH
-        Value: 0x7
-      - Tag:   DT_FLAGS
-        Value: 0xffffffff
-      - Tag:   DT_PREINIT_ARRAY
-        Value: 0x1000
-      - Tag:   DT_PREINIT_ARRAYSZ
-        Value: 0x10
-      - Tag:   DT_SYMTAB_SHNDX
-        Value: 0x1000
-      - Tag:   DT_RELRSZ
-        Value: 0x10
-      - Tag:   DT_RELR
-        Value: 0x1000
-      - Tag:   DT_RELRENT
-        Value: 0x4321
-      - Tag:   DT_ANDROID_REL
-        Value: 0x1000
-      - Tag:   DT_ANDROID_RELSZ
-        Value: 0x10
-      - Tag:   DT_ANDROID_RELA
-        Value: 0x1000
-      - Tag:   DT_ANDROID_RELASZ
-        Value: 0x10
-      - Tag:   DT_ANDROID_RELR
-        Value: 0x1000
-      - Tag:   DT_ANDROID_RELRSZ
-        Value: 0x10
-      - Tag:   DT_ANDROID_RELRENT
-        Value: 0x1234
-      - Tag:   DT_GNU_HASH
-        Value: 0x1000
-      - Tag:   DT_TLSDESC_PLT
-        Value: 0x1000
-      - Tag:   DT_TLSDESC_GOT
-        Value: 0x1000
-      - Tag:   DT_RELACOUNT
-        Value: 0x0
-      - Tag:   DT_RELCOUNT
-        Value: 0x0
-      - Tag:   DT_FLAGS_1
-        Value: 0xffffffff
-      - Tag:   DT_VERSYM
-        Value: 0x1000
-      - Tag:   DT_VERDEF
-        Value: 0x1000
-      - Tag:   DT_VERDEFNUM
-        Value: 0x0
-      - Tag:   DT_VERNEED
-        Value: 0x1000
-      - Tag:   DT_VERNEEDNUM
-        Value: 0x0
-      - Tag:   DT_AUXILIARY
-        Value: 0x1
-      - Tag:   DT_USED
-        Value: 0x3
-      - Tag:   DT_FILTER
-        Value: 0x3
-## Show behaviour for unknown values in special and un-marked ranges.
-## An arbitrary unknown value outside of the special ranges.
-      - Tag:   0x12345678
-        Value: 0x87654321
-## An OS specific tag.
-      - Tag:   0x6abcdef0
-        Value: 0x99887766
-## A processor specific tags.
-      - Tag:   0x76543210
-        Value: 0x55556666
-      - Tag:   DT_NULL
-        Value: 0x0
-ProgramHeaders:
-  - Type: PT_LOAD
-    VAddr: 0x1000
-    Sections:
-      - Section: .dynstr
-      - Section: .dynamic
-  - Type: PT_DYNAMIC
-    VAddr: 0x1010
-    Sections:
-      - Section: .dynamic
-
 ## When printing the "Name/Value" column we want to have the minimal possible indentation.
 ## Use an arbitrary dynamic tag to demonstrate this.
 
-# RUN: yaml2obj %s --docnum=3 -o %t3
+# RUN: yaml2obj %s --docnum=2 -o %t3
 # RUN: llvm-readobj --dynamic-table %t3 \
 # RUN:   | FileCheck %s --check-prefix=INDENT-LLVM-64 --strict-whitespace
 # RUN: llvm-readelf --dynamic-table %t3 \
 # RUN:   | FileCheck %s --check-prefix=INDENT-GNU-64 --strict-whitespace
 
-# RUN: yaml2obj %s --docnum=4 -o %t4
+# RUN: yaml2obj %s --docnum=2 -DBITS=32 -o %t4
 # RUN: llvm-readobj --dynamic-table %t4 \
 # RUN:   | FileCheck %s --check-prefix=INDENT-LLVM-32 --strict-whitespace
 # RUN: llvm-readelf --dynamic-table %t4 \
@@ -611,25 +460,10 @@ ProgramHeaders:
 
 --- !ELF
 FileHeader:
-  Class:   ELFCLASS64
-  Data:    ELFDATA2LSB
-  Type:    ET_EXEC
-  Machine: EM_X86_64
-Sections:
-  - Name: .dynamic
-    Type: SHT_DYNAMIC
-    Entries:
-     - Tag:   DT_NEEDED
-       Value: 0x1
-     - Tag:   DT_NULL
-       Value: 0x0
-
---- !ELF
-FileHeader:
-  Class:   ELFCLASS32
+  Class:   ELFCLASS[[BITS=64]]
   Data:    ELFDATA2LSB
   Type:    ET_EXEC
-  Machine: EM_386
+  Machine: EM_NONE
 Sections:
   - Name: .dynamic
     Type: SHT_DYNAMIC
diff --git a/llvm/test/tools/llvm-readobj/ELF/file-header-abi-version.test b/llvm/test/tools/llvm-readobj/ELF/file-header-abi-version.test
index 95e9e2636f8b9..61c02ee412e85 100644
--- a/llvm/test/tools/llvm-readobj/ELF/file-header-abi-version.test
+++ b/llvm/test/tools/llvm-readobj/ELF/file-header-abi-version.test
@@ -1,24 +1,26 @@
 ## This is a test to test how the ABI version field (EI_ABIVERSION) of an ELF file header is dumped.
 
 ## EI_ABIVERSION is set to zero.
-# RUN: yaml2obj %s --docnum=1 -o %t.abiver.zero
+# RUN: yaml2obj %s -DABIVERSION=0 -o %t.abiver.zero
 # RUN: llvm-readobj --file-headers %t.abiver.zero | FileCheck %s --match-full-lines --check-prefix=ABIVER-ZERO-LLVM
 # RUN: llvm-readelf --file-headers %t.abiver.zero | FileCheck %s --match-full-lines --check-prefix=ABIVER-ZERO-GNU
-## EI_ABIVERSION is set to an arbitrary number.
-# RUN: yaml2obj %s --docnum=2 -o %t.abiver.any
-# RUN: llvm-readobj --file-headers %t.abiver.any | FileCheck %s --match-full-lines --check-prefix=ABIVER-ANY-LLVM
-# RUN: llvm-readelf --file-headers %t.abiver.any | FileCheck %s --match-full-lines --check-prefix=ABIVER-ANY-GNU
-## EI_ABIVERSION is set to the maximum possible value.
-# RUN: yaml2obj %s --docnum=3 -o %t.abiver.max
-# RUN: llvm-readobj --file-headers %t.abiver.max | FileCheck %s --match-full-lines --check-prefix=ABIVER-MAX-LLVM
-# RUN: llvm-readelf --file-headers %t.abiver.max | FileCheck %s --match-full-lines --check-prefix=ABIVER-MAX-GNU
 
 # ABIVER-ZERO-LLVM: ABIVersion: 0
 # ABIVER-ZERO-GNU:  ABI Version: 0
 
+## EI_ABIVERSION is set to an arbitrary number.
+# RUN: yaml2obj %s -DABIVERSION=52 -o %t.abiver.any
+# RUN: llvm-readobj --file-headers %t.abiver.any | FileCheck %s --match-full-lines --check-prefix=ABIVER-ANY-LLVM
+# RUN: llvm-readelf --file-headers %t.abiver.any | FileCheck %s --match-full-lines --check-prefix=ABIVER-ANY-GNU
+
 # ABIVER-ANY-LLVM: ABIVersion: 52
 # ABIVER-ANY-GNU:  ABI Version: 52
 
+## EI_ABIVERSION is set to the maximum possible value.
+# RUN: yaml2obj %s -DABIVERSION=255 -o %t.abiver.max
+# RUN: llvm-readobj --file-headers %t.abiver.max | FileCheck %s --match-full-lines --check-prefix=ABIVER-MAX-LLVM
+# RUN: llvm-readelf --file-headers %t.abiver.max | FileCheck %s --match-full-lines --check-prefix=ABIVER-MAX-GNU
+
 # ABIVER-MAX-LLVM: ABIVersion: 255
 # ABIVER-MAX-GNU:  ABI Version: 255
 
@@ -28,20 +30,4 @@ FileHeader:
   Data:       ELFDATA2LSB
   Type:       ET_REL
   Machine:    EM_X86_64
-  ABIVersion: 0x0
-
---- !ELF
-FileHeader:
-  Class:      ELFCLASS64
-  Data:       ELFDATA2LSB
-  Type:       ET_REL
-  Machine:    EM_X86_64
-  ABIVersion: 0x34
-
---- !ELF
-FileHeader:
-  Class:      ELFCLASS64
-  Data:       ELFDATA2LSB
-  Type:       ET_REL
-  Machine:    EM_X86_64
-  ABIVersion: 0xFF
+  ABIVersion: [[ABIVERSION]]
diff --git a/llvm/test/tools/llvm-readobj/ELF/file-header-os-abi.test b/llvm/test/tools/llvm-readobj/ELF/file-header-os-abi.test
index c169d29ff5baa..642976775808a 100644
--- a/llvm/test/tools/llvm-readobj/ELF/file-header-os-abi.test
+++ b/llvm/test/tools/llvm-readobj/ELF/file-header-os-abi.test
@@ -1,7 +1,7 @@
 ## This is a test to test how the OS/ABI identification field (EI_OSABI) of an ELF file header is dumped.
 
 ## EI_OSABI is set to ELFOSABI_NONE.
-# RUN: yaml2obj %s --docnum=1 -o %t.osabi.none
+# RUN: yaml2obj %s -DOSABI=ELFOSABI_NONE -o %t.osabi.none
 # RUN: llvm-readobj --file-headers %t.osabi.none | FileCheck %s --match-full-lines --check-prefix=OSABI-NONE-LLVM
 # RUN: llvm-readelf --file-headers %t.osabi.none | FileCheck %s --match-full-lines --check-prefix=OSABI-NONE-GNU
 
@@ -14,292 +14,148 @@ FileHeader:
   Data:    ELFDATA2LSB
   Type:    ET_REL
   Machine: EM_X86_64
-  OSABI:   ELFOSABI_NONE
+  OSABI:   [[OSABI]]
 
 ## EI_OSABI is set to ELFOSABI_HPUX.
-# RUN: yaml2obj %s --docnum=2 -o %t.osabi.hpux
+# RUN: yaml2obj %s -DOSABI=ELFOSABI_HPUX -o %t.osabi.hpux
 # RUN: llvm-readobj --file-headers %t.osabi.hpux | FileCheck %s --match-full-lines --check-prefix=OSABI-HPUX-LLVM
 # RUN: llvm-readelf --file-headers %t.osabi.hpux | FileCheck %s --match-full-lines --check-prefix=OSABI-HPUX-GNU
 
 # OSABI-HPUX-LLVM: OS/ABI: HPUX (0x1)
 # OSABI-HPUX-GNU:  OS/ABI: UNIX - HP-UX
 
---- !ELF
-FileHeader:
-  Class:   ELFCLASS64
-  Data:    ELFDATA2LSB
-  Type:    ET_REL
-  Machine: EM_X86_64
-  OSABI:   ELFOSABI_HPUX
-
 ## EI_OSABI is set to ELFOSABI_NETBSD.
-# RUN: yaml2obj %s --docnum=3 -o %t.osabi.netbsd
+# RUN: yaml2obj %s -DOSABI=ELFOSABI_NETBSD -o %t.osabi.netbsd
 # RUN: llvm-readobj --file-headers %t.osabi.netbsd | FileCheck %s --match-full-lines --check-prefix=OSABI-NETBSD-LLVM
 # RUN: llvm-readelf --file-headers %t.osabi.netbsd | FileCheck %s --match-full-lines --check-prefix=OSABI-NETBSD-GNU
 
 # OSABI-NETBSD-LLVM: OS/ABI: NetBSD (0x2)
 # OSABI-NETBSD-GNU:  OS/ABI: UNIX - NetBSD
 
---- !ELF
-FileHeader:
-  Class:   ELFCLASS64
-  Data:    ELFDATA2LSB
-  Type:    ET_REL
-  Machine: EM_X86_64
-  OSABI:   ELFOSABI_NETBSD
-
 ## EI_OSABI is set to ELFOSABI_LINUX.
-# RUN: yaml2obj %s --docnum=4 -o %t.osabi.linux
+# RUN: yaml2obj %s -DOSABI=ELFOSABI_LINUX -o %t.osabi.linux
 # RUN: llvm-readobj --file-headers %t.osabi.linux | FileCheck %s --match-full-lines --check-prefix=OSABI-LINUX-LLVM
 # RUN: llvm-readelf --file-headers %t.osabi.linux | FileCheck %s --match-full-lines --check-prefix=OSABI-LINUX-GNU
 
 # OSABI-LINUX-LLVM: OS/ABI: GNU/Linux (0x3)
 # OSABI-LINUX-GNU:  OS/ABI: UNIX - GNU
 
---- !ELF
-FileHeader:
-  Class:   ELFCLASS64
-  Data:    ELFDATA2LSB
-  Type:    ET_REL
-  Machine: EM_X86_64
-  OSABI:   ELFOSABI_LINUX
-
 ## EI_OSABI is set to ELFOSABI_HURD.
-# RUN: yaml2obj %s --docnum=5 -o %t.osabi.hurd
+# RUN: yaml2obj %s -DOSABI=ELFOSABI_HURD -o %t.osabi.hurd
 # RUN: llvm-readobj --file-headers %t.osabi.hurd | FileCheck %s --match-full-lines --check-prefix=OSABI-HURD-LLVM
 # RUN: llvm-readelf --file-headers %t.osabi.hurd | FileCheck %s --match-full-lines --check-prefix=OSABI-HURD-GNU
 
 # OSABI-HURD-LLVM: OS/ABI: GNU/Hurd (0x4)
 # OSABI-HURD-GNU:  OS/ABI: GNU/Hurd
 
---- !ELF
-FileHeader:
-  Class:   ELFCLASS64
-  Data:    ELFDATA2LSB
-  Type:    ET_REL
-  Machine: EM_X86_64
-  OSABI:   ELFOSABI_HURD
-
 ## EI_OSABI is set to ELFOSABI_SOLARIS.
-# RUN: yaml2obj %s --docnum=6 -o %t.osabi.solaris
+# RUN: yaml2obj %s -DOSABI=ELFOSABI_SOLARIS -o %t.osabi.solaris
 # RUN: llvm-readobj --file-headers %t.osabi.solaris | FileCheck %s --match-full-lines --check-prefix=OSABI-SOLARIS-LLVM
 # RUN: llvm-readelf --file-headers %t.osabi.solaris | FileCheck %s --match-full-lines --check-prefix=OSABI-SOLARIS-GNU
 
 # OSABI-SOLARIS-LLVM: OS/ABI: Solaris (0x6)
 # OSABI-SOLARIS-GNU:  OS/ABI: UNIX - Solaris
 
---- !ELF
-FileHeader:
-  Class:   ELFCLASS64
-  Data:    ELFDATA2LSB
-  Type:    ET_REL
-  Machine: EM_X86_64
-  OSABI:   ELFOSABI_SOLARIS
-
 ## EI_OSABI is set to ELFOSABI_AIX.
-# RUN: yaml2obj %s --docnum=7 -o %t.osabi.aix
+# RUN: yaml2obj %s -DOSABI=ELFOSABI_AIX -o %t.osabi.aix
 # RUN: llvm-readobj --file-headers %t.osabi.aix | FileCheck %s --match-full-lines --check-prefix=OSABI-AIX-LLVM
 # RUN: llvm-readelf --file-headers %t.osabi.aix | FileCheck %s --match-full-lines --check-prefix=OSABI-AIX-GNU
 
 # OSABI-AIX-LLVM: OS/ABI: AIX (0x7)
 # OSABI-AIX-GNU:  OS/ABI: UNIX - AIX
 
---- !ELF
-FileHeader:
-  Class:   ELFCLASS64
-  Data:    ELFDATA2LSB
-  Type:    ET_REL
-  Machine: EM_X86_64
-  OSABI:   ELFOSABI_AIX
-
 ## EI_OSABI is set to ELFOSABI_IRIX.
-# RUN: yaml2obj %s --docnum=8 -o %t.osabi.irix
+# RUN: yaml2obj %s -DOSABI=ELFOSABI_IRIX -o %t.osabi.irix
 # RUN: llvm-readobj --file-headers %t.osabi.irix | FileCheck %s --match-full-lines --check-prefix=OSABI-IRIX-LLVM
 # RUN: llvm-readelf --file-headers %t.osabi.irix | FileCheck %s --match-full-lines --check-prefix=OSABI-IRIX-GNU
 
 # OSABI-IRIX-LLVM: OS/ABI: IRIX (0x8)
 # OSABI-IRIX-GNU:  OS/ABI: UNIX - IRIX
 
---- !ELF
-FileHeader:
-  Class:   ELFCLASS64
-  Data:    ELFDATA2LSB
-  Type:    ET_REL
-  Machine: EM_X86_64
-  OSABI:   ELFOSABI_IRIX
-
 ## EI_OSABI is set to ELFOSABI_FREEBSD.
-# RUN: yaml2obj %s --docnum=9 -o %t.osabi.freebsd
+# RUN: yaml2obj %s -DOSABI=ELFOSABI_FREEBSD -o %t.osabi.freebsd
 # RUN: llvm-readobj --file-headers %t.osabi.freebsd | FileCheck %s --match-full-lines --check-prefix=OSABI-FREEBSD-LLVM
 # RUN: llvm-readelf --file-headers %t.osabi.freebsd | FileCheck %s --match-full-lines --check-prefix=OSABI-FREEBSD-GNU
 
 # OSABI-FREEBSD-LLVM: OS/ABI: FreeBSD (0x9)
 # OSABI-FREEBSD-GNU:  OS/ABI: UNIX - FreeBSD
 
---- !ELF
-FileHeader:
-  Class:   ELFCLASS64
-  Data:    ELFDATA2LSB
-  Type:    ET_REL
-  Machine: EM_X86_64
-  OSABI:   ELFOSABI_FREEBSD
-
 ## EI_OSABI is set to ELFOSABI_TRU64.
-# RUN: yaml2obj %s --docnum=10 -o %t.osabi.tru64
+# RUN: yaml2obj %s -DOSABI=ELFOSABI_TRU64 -o %t.osabi.tru64
 # RUN: llvm-readobj --file-headers %t.osabi.tru64 | FileCheck %s --match-full-lines --check-prefix=OSABI-TRU64-LLVM
 # RUN: llvm-readelf --file-headers %t.osabi.tru64 | FileCheck %s --match-full-lines --check-prefix=OSABI-TRU64-GNU
 
 # OSABI-TRU64-LLVM: OS/ABI: TRU64 (0xA)
 # OSABI-TRU64-GNU:  OS/ABI: UNIX - TRU64
 
---- !ELF
-FileHeader:
-  Class:   ELFCLASS64
-  Data:    ELFDATA2LSB
-  Type:    ET_REL
-  Machine: EM_X86_64
-  OSABI:   ELFOSABI_TRU64
-
 ## EI_OSABI is set to ELFOSABI_MODESTO.
-# RUN: yaml2obj %s --docnum=11 -o %t.osabi.modesto
+# RUN: yaml2obj %s -DOSABI=ELFOSABI_MODESTO -o %t.osabi.modesto
 # RUN: llvm-readobj --file-headers %t.osabi.modesto | FileCheck %s --match-full-lines --check-prefix=OSABI-MODESTO-LLVM
 # RUN: llvm-readelf --file-headers %t.osabi.modesto | FileCheck %s --match-full-lines --check-prefix=OSABI-MODESTO-GNU
 
 # OSABI-MODESTO-LLVM: OS/ABI: Modesto (0xB)
 # OSABI-MODESTO-GNU:  OS/ABI: Novell - Modesto
 
---- !ELF
-FileHeader:
-  Class:   ELFCLASS64
-  Data:    ELFDATA2LSB
-  Type:    ET_REL
-  Machine: EM_X86_64
-  OSABI:   ELFOSABI_MODESTO
-
 ## EI_OSABI is set to ELFOSABI_OPENBSD.
-# RUN: yaml2obj %s --docnum=12 -o %t.osabi.openbsd
+# RUN: yaml2obj %s -DOSABI=ELFOSABI_OPENBSD -o %t.osabi.openbsd
 # RUN: llvm-readobj --file-headers %t.osabi.openbsd | FileCheck %s --match-full-lines --check-prefix=OSABI-OPENBSD-LLVM
 # RUN: llvm-readelf --file-headers %t.osabi.openbsd | FileCheck %s --match-full-lines --check-prefix=OSABI-OPENBSD-GNU
 
 # OSABI-OPENBSD-LLVM: OS/ABI: OpenBSD (0xC)
 # OSABI-OPENBSD-GNU:  OS/ABI: UNIX - OpenBSD
 
---- !ELF
-FileHeader:
-  Class:   ELFCLASS64
-  Data:    ELFDATA2LSB
-  Type:    ET_REL
-  Machine: EM_X86_64
-  OSABI:   ELFOSABI_OPENBSD
-
 ## EI_OSABI is set to ELFOSABI_OPENVMS.
-# RUN: yaml2obj %s --docnum=13 -o %t.osabi.openvms
+# RUN: yaml2obj %s -DOSABI=ELFOSABI_OPENVMS -o %t.osabi.openvms
 # RUN: llvm-readobj --file-headers %t.osabi.openvms | FileCheck %s --match-full-lines --check-prefix=OSABI-OPENVMS-LLVM
 # RUN: llvm-readelf --file-headers %t.osabi.openvms | FileCheck %s --match-full-lines --check-prefix=OSABI-OPENVMS-GNU
 
 # OSABI-OPENVMS-LLVM: OS/ABI: OpenVMS (0xD)
 # OSABI-OPENVMS-GNU:  OS/ABI: VMS - OpenVMS
 
---- !ELF
-FileHeader:
-  Class:   ELFCLASS64
-  Data:    ELFDATA2LSB
-  Type:    ET_REL
-  Machine: EM_X86_64
-  OSABI:   ELFOSABI_OPENVMS
-
 ## EI_OSABI is set to ELFOSABI_NSK.
-# RUN: yaml2obj %s --docnum=14 -o %t.osabi.nsk
+# RUN: yaml2obj %s -DOSABI=ELFOSABI_NSK -o %t.osabi.nsk
 # RUN: llvm-readobj --file-headers %t.osabi.nsk | FileCheck %s --match-full-lines --check-prefix=OSABI-NSK-LLVM
 # RUN: llvm-readelf --file-headers %t.osabi.nsk | FileCheck %s --match-full-lines --check-prefix=OSABI-NSK-GNU
 
 # OSABI-NSK-LLVM: OS/ABI: NSK (0xE)
 # OSABI-NSK-GNU:  OS/ABI: HP - Non-Stop Kernel
 
---- !ELF
-FileHeader:
-  Class:   ELFCLASS64
-  Data:    ELFDATA2LSB
-  Type:    ET_REL
-  Machine: EM_X86_64
-  OSABI:   ELFOSABI_NSK
-
 ## EI_OSABI is set to ELFOSABI_AROS.
-# RUN: yaml2obj %s --docnum=15 -o %t.osabi.aros
+# RUN: yaml2obj %s -DOSABI=ELFOSABI_AROS -o %t.osabi.aros
 # RUN: llvm-readobj --file-headers %t.osabi.aros | FileCheck %s --match-full-lines --check-prefix=OSABI-AROS-LLVM
 # RUN: llvm-readelf --file-headers %t.osabi.aros | FileCheck %s --match-full-lines --check-prefix=OSABI-AROS-GNU
 
 # OSABI-AROS-LLVM: OS/ABI: AROS (0xF)
 # OSABI-AROS-GNU:  OS/ABI: AROS
 
---- !ELF
-FileHeader:
-  Class:   ELFCLASS64
-  Data:    ELFDATA2LSB
-  Type:    ET_REL
-  Machine: EM_X86_64
-  OSABI:   ELFOSABI_AROS
-
 ## EI_OSABI is set to ELFOSABI_FENIXOS.
-# RUN: yaml2obj %s --docnum=16 -o %t.osabi.fenixos
+# RUN: yaml2obj %s -DOSABI=ELFOSABI_FENIXOS -o %t.osabi.fenixos
 # RUN: llvm-readobj --file-headers %t.osabi.fenixos | FileCheck %s --match-full-lines --check-prefix=OSABI-FENIXOS-LLVM
 # RUN: llvm-readelf --file-headers %t.osabi.fenixos | FileCheck %s --match-full-lines --check-prefix=OSABI-FENIXOS-GNU
 
 # OSABI-FENIXOS-LLVM: OS/ABI: FenixOS (0x10)
 # OSABI-FENIXOS-GNU:  OS/ABI: FenixOS
 
---- !ELF
-FileHeader:
-  Class:   ELFCLASS64
-  Data:    ELFDATA2LSB
-  Type:    ET_REL
-  Machine: EM_X86_64
-  OSABI:   ELFOSABI_FENIXOS
-
 ## EI_OSABI is set to ELFOSABI_CLOUDABI.
-# RUN: yaml2obj %s --docnum=17 -o %t.osabi.cloudabi
+# RUN: yaml2obj %s -DOSABI=ELFOSABI_CLOUDABI -o %t.osabi.cloudabi
 # RUN: llvm-readobj --file-headers %t.osabi.cloudabi | FileCheck %s --match-full-lines --check-prefix=OSABI-CLOUDABI-LLVM
 # RUN: llvm-readelf --file-headers %t.osabi.cloudabi | FileCheck %s --match-full-lines --check-prefix=OSABI-CLOUDABI-GNU
 
 # OSABI-CLOUDABI-LLVM: OS/ABI: CloudABI (0x11)
 # OSABI-CLOUDABI-GNU:  OS/ABI: CloudABI
 
---- !ELF
-FileHeader:
-  Class:   ELFCLASS64
-  Data:    ELFDATA2LSB
-  Type:    ET_REL
-  Machine: EM_X86_64
-  OSABI:   ELFOSABI_CLOUDABI
-
 ## EI_OSABI is set to ELFOSABI_STANDALONE.
-# RUN: yaml2obj %s --docnum=18 -o %t.osabi.standalone
+# RUN: yaml2obj %s -DOSABI=ELFOSABI_STANDALONE -o %t.osabi.standalone
 # RUN: llvm-readobj --file-headers %t.osabi.standalone | FileCheck %s --match-full-lines --check-prefix=OSABI-STANDALONE-LLVM
 # RUN: llvm-readelf --file-headers %t.osabi.standalone | FileCheck %s --match-full-lines --check-prefix=OSABI-STANDALONE-GNU
 
 # OSABI-STANDALONE-LLVM: OS/ABI: Standalone (0xFF)
 # OSABI-STANDALONE-GNU:  OS/ABI: Standalone App
 
---- !ELF
-FileHeader:
-  Class:   ELFCLASS64
-  Data:    ELFDATA2LSB
-  Type:    ET_REL
-  Machine: EM_X86_64
-  OSABI:   ELFOSABI_STANDALONE
-
 ## EI_OSABI is set to an arbitrary value which is not supported by llvm-readobj/llvm-readelf.
-# RUN: yaml2obj %s --docnum=19 -o %t.osabi.unknown
+# RUN: yaml2obj %s -DOSABI=0xFE -o %t.osabi.unknown
 # RUN: llvm-readobj --file-headers %t.osabi.unknown | FileCheck %s --match-full-lines --check-prefix=OSABI-UNKNOWN-LLVM
 # RUN: llvm-readelf --file-headers %t.osabi.unknown | FileCheck %s --match-full-lines --check-prefix=OSABI-UNKNOWN-GNU
 
 # OSABI-UNKNOWN-LLVM: OS/ABI: 0xFE
 # OSABI-UNKNOWN-GNU:  OS/ABI: fe
-
---- !ELF
-FileHeader:
-  Class:   ELFCLASS64
-  Data:    ELFDATA2LSB
-  Type:    ET_REL
-  Machine: EM_X86_64
-  OSABI:   0xFE
diff --git a/llvm/test/tools/llvm-readobj/ELF/file-types.test b/llvm/test/tools/llvm-readobj/ELF/file-types.test
index 0765cff1040c5..0a8e6050df818 100644
--- a/llvm/test/tools/llvm-readobj/ELF/file-types.test
+++ b/llvm/test/tools/llvm-readobj/ELF/file-types.test
@@ -1,6 +1,6 @@
 ## Check llvm-readobj and llvm-readelf can dump files of the different ELF types.
 
-# RUN: yaml2obj %s --docnum=1 -o %t1
+# RUN: yaml2obj %s -DTYPE=ET_NONE -o %t1
 # RUN: llvm-readobj -h %t1 | FileCheck %s --match-full-lines --check-prefix LLVM-NONE
 # RUN: llvm-readelf -h %t1 | FileCheck %s --match-full-lines --check-prefix GNU-NONE
 
@@ -14,10 +14,10 @@
 FileHeader:
   Class:   ELFCLASS64
   Data:    ELFDATA2LSB
-  Type:    ET_NONE
+  Type:    [[TYPE]]
   Machine: EM_X86_64
 
-# RUN: yaml2obj %s --docnum=2 -o %t2
+# RUN: yaml2obj %s -DTYPE=ET_REL -o %t2
 # RUN: llvm-readobj -h %t2 | FileCheck %s --match-full-lines --check-prefix LLVM-REL
 # RUN: llvm-readelf -h %t2 | FileCheck %s --match-full-lines --check-prefix GNU-REL
 
@@ -27,14 +27,7 @@ FileHeader:
 # GNU-REL: ELF Header:
 # GNU-REL:   Type: REL (Relocatable file)
 
---- !ELF
-FileHeader:
-  Class:   ELFCLASS64
-  Data:    ELFDATA2LSB
-  Type:    ET_REL
-  Machine: EM_X86_64
-
-# RUN: yaml2obj %s --docnum=3 -o %t3
+# RUN: yaml2obj %s -DTYPE=ET_EXEC -o %t3
 # RUN: llvm-readobj -h %t3 | FileCheck %s --match-full-lines --check-prefix LLVM-EXEC
 # RUN: llvm-readelf -h %t3 | FileCheck %s --match-full-lines --check-prefix GNU-EXEC
 
@@ -44,14 +37,7 @@ FileHeader:
 # GNU-EXEC: ELF Header:
 # GNU-EXEC:   Type: EXEC (Executable file)
 
---- !ELF
-FileHeader:
-  Class:   ELFCLASS64
-  Data:    ELFDATA2LSB
-  Type:    ET_EXEC
-  Machine: EM_X86_64
-
-# RUN: yaml2obj %s --docnum=4 -o %t4
+# RUN: yaml2obj %s -DTYPE=ET_DYN -o %t4
 # RUN: llvm-readobj -h %t4 | FileCheck %s --match-full-lines --check-prefix LLVM-DYN
 # RUN: llvm-readelf -h %t4 | FileCheck %s --match-full-lines --check-prefix GNU-DYN
 
@@ -61,14 +47,7 @@ FileHeader:
 # GNU-DYN: ELF Header:
 # GNU-DYN:   Type: DYN (Shared object file)
 
---- !ELF
-FileHeader:
-  Class:   ELFCLASS64
-  Data:    ELFDATA2LSB
-  Type:    ET_DYN
-  Machine: EM_X86_64
-
-# RUN: yaml2obj %s --docnum=5 -o %t5
+# RUN: yaml2obj %s -DTYPE=ET_CORE -o %t5
 # RUN: llvm-readobj -h %t5 | FileCheck %s --match-full-lines --check-prefix LLVM-CORE
 # RUN: llvm-readelf -h %t5 | FileCheck %s --match-full-lines --check-prefix GNU-CORE
 
@@ -78,14 +57,7 @@ FileHeader:
 # GNU-CORE: ELF Header:
 # GNU-CORE:   Type: CORE (Core file)
 
---- !ELF
-FileHeader:
-  Class:   ELFCLASS64
-  Data:    ELFDATA2LSB
-  Type:    ET_CORE
-  Machine: EM_X86_64
-
-# RUN: yaml2obj %s --docnum=6 -o %t6
+# RUN: yaml2obj %s -DTYPE=0xfe00 -o %t6
 # RUN: llvm-readobj -h %t6 | FileCheck %s --match-full-lines --check-prefix LLVM-LOOS
 # RUN: llvm-readelf -h %t6 | FileCheck %s --match-full-lines --check-prefix GNU-LOOS
 
@@ -95,14 +67,7 @@ FileHeader:
 # GNU-LOOS: ELF Header:
 # GNU-LOOS:   Type: fe00
 
---- !ELF
-FileHeader:
-  Class:   ELFCLASS64
-  Data:    ELFDATA2LSB
-  Type:    0xfe00
-  Machine: EM_X86_64
-
-# RUN: yaml2obj %s --docnum=7 -o %t7
+# RUN: yaml2obj %s -DTYPE=0xfeff -o %t7
 # RUN: llvm-readobj -h %t7 | FileCheck %s --match-full-lines --check-prefix LLVM-HIOS
 # RUN: llvm-readelf -h %t7 | FileCheck %s --match-full-lines --check-prefix GNU-HIOS
 
@@ -112,14 +77,7 @@ FileHeader:
 # GNU-HIOS: ELF Header:
 # GNU-HIOS:   Type: feff
 
---- !ELF
-FileHeader:
-  Class:   ELFCLASS64
-  Data:    ELFDATA2LSB
-  Type:    0xfeff
-  Machine: EM_X86_64
-
-# RUN: yaml2obj %s --docnum=8 -o %t8
+# RUN: yaml2obj %s -DTYPE=0xff00 -o %t8
 # RUN: llvm-readobj -h %t8 | FileCheck %s --match-full-lines --check-prefix LLVM-LOPROC
 # RUN: llvm-readelf -h %t8 | FileCheck %s --match-full-lines --check-prefix GNU-LOPROC
 
@@ -129,14 +87,7 @@ FileHeader:
 # GNU-LOPROC: ELF Header:
 # GNU-LOPROC:   Type: ff00
 
---- !ELF
-FileHeader:
-  Class:   ELFCLASS64
-  Data:    ELFDATA2LSB
-  Type:    0xff00
-  Machine: EM_X86_64
-
-# RUN: yaml2obj %s --docnum=9 -o %t9
+# RUN: yaml2obj %s -DTYPE=0xffff -o %t9
 # RUN: llvm-readobj -h %t9 | FileCheck %s --match-full-lines --check-prefix LLVM-HIPROC
 # RUN: llvm-readelf -h %t9 | FileCheck %s --match-full-lines --check-prefix GNU-HIPROC
 
@@ -145,10 +96,3 @@ FileHeader:
 
 # GNU-HIPROC: ELF Header:
 # GNU-HIPROC:   Type: ffff
-
---- !ELF
-FileHeader:
-  Class:   ELFCLASS64
-  Data:    ELFDATA2LSB
-  Type:    0xffff
-  Machine: EM_X86_64
diff --git a/llvm/test/tools/llvm-readobj/ELF/gnu-notes.test b/llvm/test/tools/llvm-readobj/ELF/gnu-notes.test
index 5af6a56e68950..011e5db474126 100644
--- a/llvm/test/tools/llvm-readobj/ELF/gnu-notes.test
+++ b/llvm/test/tools/llvm-readobj/ELF/gnu-notes.test
@@ -87,8 +87,8 @@ FileHeader:
   Data:       ELFDATA2LSB
   Type:       ET_EXEC
   Machine:    EM_X86_64
-  EPhEntSize: [[PHENTSIZE=56]]
-  EShNum:     [[SHNUM=6]]
+  EPhEntSize: [[PHENTSIZE=<none>]]
+  EShNum:     [[SHNUM=<none>]]
 Sections:
   - Name:         .note.ABI-tag
     Type:         SHT_NOTE
diff --git a/llvm/test/tools/llvm-readobj/ELF/gnu-phdrs.test b/llvm/test/tools/llvm-readobj/ELF/gnu-phdrs.test
index 1b5bb2572b111..a767c95069ccf 100644
--- a/llvm/test/tools/llvm-readobj/ELF/gnu-phdrs.test
+++ b/llvm/test/tools/llvm-readobj/ELF/gnu-phdrs.test
@@ -109,7 +109,6 @@ FileHeader:
   Data:    ELFDATA2LSB
   Type:    ET_EXEC
   Machine: [[MACHINE]]
-  Entry:   0x12345678
 Sections:
   - Name:    .foo.begin
     Type:    SHT_PROGBITS
@@ -374,8 +373,8 @@ FileHeader:
   Data:       ELFDATA2LSB
   Type:       ET_EXEC
   Machine:    EM_X86_64
-  EPhEntSize: [[PHENTSIZE=56]]
-  EPhOff:     [[PHOFF=64]]
+  EPhEntSize: [[PHENTSIZE=<none>]]
+  EPhOff:     [[PHOFF=<none>]]
 Sections:
   - Name: .foo
     Type: SHT_PROGBITS
diff --git a/llvm/test/tools/llvm-readobj/ELF/gnu-section-mapping-no-phdrs.test b/llvm/test/tools/llvm-readobj/ELF/gnu-section-mapping-no-phdrs.test
index fb508529fe5bb..86fe989861ce6 100644
--- a/llvm/test/tools/llvm-readobj/ELF/gnu-section-mapping-no-phdrs.test
+++ b/llvm/test/tools/llvm-readobj/ELF/gnu-section-mapping-no-phdrs.test
@@ -1,4 +1,4 @@
-## Test the behaviour of --section-mapping when there are no section headers in an object.
+## Test the behaviour of --section-mapping when there are no program headers in an object.
 # RUN: yaml2obj %s -o %t
 # RUN: llvm-readelf --section-mapping %t | FileCheck %s --strict-whitespace --match-full-lines
 
diff --git a/llvm/test/tools/llvm-readobj/ELF/gnu-section-mapping.test b/llvm/test/tools/llvm-readobj/ELF/gnu-section-mapping.test
index e1c7181f15589..2172ba2d58c6c 100644
--- a/llvm/test/tools/llvm-readobj/ELF/gnu-section-mapping.test
+++ b/llvm/test/tools/llvm-readobj/ELF/gnu-section-mapping.test
@@ -18,7 +18,7 @@ FileHeader:
   Data:       ELFDATA2LSB
   Type:       ET_EXEC
   Machine:    EM_X86_64
-  EPhEntSize: [[PHENTSIZE=56]]
+  EPhEntSize: [[PHENTSIZE=<none>]]
 Sections:
   - Name:    .foo.begin
     Type:    SHT_PROGBITS
diff --git a/llvm/test/tools/llvm-readobj/ELF/gnu-sections.test b/llvm/test/tools/llvm-readobj/ELF/gnu-sections.test
index 532ca22d3195b..ab6ea13c3a66c 100644
--- a/llvm/test/tools/llvm-readobj/ELF/gnu-sections.test
+++ b/llvm/test/tools/llvm-readobj/ELF/gnu-sections.test
@@ -1,23 +1,24 @@
 ## In this test case we check how we print section and flag descriptions for different targets.
 
-## EM_386 is a target that does not have any processor and OS specific flags,
+## EM_NONE is a target that does not have any processor and OS specific flags,
 ## we use it to show how the default flag key is printed.
 
-# RUN: yaml2obj --docnum=1 %s -o %t-i386.o
-# RUN: llvm-readelf -S %t-i386.o | FileCheck %s --check-prefix=ELF32 --strict-whitespace --match-full-lines
+# RUN: yaml2obj -DBITS=32 %s -o %t-default.o
+# RUN: llvm-readelf -S %t-default.o | FileCheck %s --check-prefix=ELF32 --strict-whitespace --match-full-lines
 
-#       ELF32:There are 8 section headers, starting at offset 0x90:
+#       ELF32:There are 9 section headers, starting at offset 0x9c:
 # ELF32-EMPTY:
 # ELF32-NEXT:Section Headers:
 # ELF32-NEXT:  [Nr] Name              Type            Address  Off    Size   ES Flg Lk Inf Al
 # ELF32-NEXT:  [ 0]                   NULL            00000000 000000 000000 00      0   0  0
 # ELF32-NEXT:  [ 1] .text             PROGBITS        00000000 000034 000001 00  AX  0   0  4
-# ELF32-NEXT:  [ 2] .rel.text         REL             00000000 000038 000000 08      5   1  4
-# ELF32-NEXT:  [ 3] .data             PROGBITS        00000000 000038 000000 00  WA  0   0  4
-# ELF32-NEXT:  [ 4] .bss              NOBITS          00000000 000038 000000 00  WA  0   0  4
-# ELF32-NEXT:  [ 5] .symtab           SYMTAB          00000000 000038 000020 10      6   2  8
-# ELF32-NEXT:  [ 6] .strtab           STRTAB          00000000 000058 000007 00      0   0  1
-# ELF32-NEXT:  [ 7] .shstrtab         STRTAB          00000000 00005f 000030 00      0   0  1
+# ELF32-NEXT:  [ 2] .rel.text         REL             00000000 000038 000000 08      6   1  4
+# ELF32-NEXT:  [ 3] .rela.text        RELA            00000000 000038 000000 18      6   1  8
+# ELF32-NEXT:  [ 4] .data             PROGBITS        00000000 000038 000000 00  WA  0   0  4
+# ELF32-NEXT:  [ 5] .bss              NOBITS          00000000 000038 000000 00  WA  0   0  4
+# ELF32-NEXT:  [ 6] .symtab           SYMTAB          00000000 000038 000020 10      7   2  8
+# ELF32-NEXT:  [ 7] .strtab           STRTAB          00000000 000058 000007 00      0   0  1
+# ELF32-NEXT:  [ 8] .shstrtab         STRTAB          00000000 00005f 00003b 00      0   0  1
 # ELF32-NEXT:Key to Flags:
 # ELF32-NEXT:  W (write), A (alloc), X (execute), M (merge), S (strings), I (info),
 # ELF32-NEXT:  L (link order), O (extra OS processing required), G (group), T (TLS),
@@ -26,11 +27,11 @@
 
 --- !ELF
 FileHeader:
-  Class:   ELFCLASS32
+  Class:   ELFCLASS[[BITS=64]]
   Data:    ELFDATA2LSB
   OSABI:   ELFOSABI_GNU
   Type:    ET_REL
-  Machine: EM_386
+  Machine: [[MACHINE=EM_NONE]]
 Sections:
   - Name:         .text
     Type:         SHT_PROGBITS
@@ -44,6 +45,13 @@ Sections:
     EntSize:      0x0000000000000008
     Info:         .text
     Relocations:
+  - Name:         .rela.text
+    Type:         SHT_RELA
+    Link:         .symtab
+    AddressAlign: 0x0000000000000008
+    EntSize:      0x0000000000000018
+    Info:         .text
+    Relocations:
   - Name:         .data
     Type:         SHT_PROGBITS
     Flags:        [ SHF_WRITE, SHF_ALLOC ]
@@ -61,7 +69,7 @@ Symbols:
 ## For an EM_X86_64 target we print "l" for the SHF_X86_64_LARGE section flag.
 ## Check we mention it in the flag key.
 
-# RUN: yaml2obj --docnum=2 %s -o %t-x64.o
+# RUN: yaml2obj -DMACHINE=EM_X86_64 %s -o %t-x64.o
 # RUN: llvm-readelf -S %t-x64.o | FileCheck %s --check-prefix=ELF64 --strict-whitespace --match-full-lines
 
 ## Check that --wide is the same as -W and ignored and also
@@ -72,62 +80,29 @@ Symbols:
 # RUN:   | FileCheck %s --check-prefix=ELF64
 # RUN: llvm-readelf -W -S %t-x64.o | FileCheck %s --check-prefix=ELF64
 
-#       ELF64:There are 8 section headers, starting at offset 0xb0:
+#       ELF64:There are 9 section headers, starting at offset 0xc0:
 # ELF64-EMPTY:
 # ELF64-NEXT:Section Headers:
 # ELF64-NEXT:  [Nr] Name              Type            Address          Off    Size   ES Flg Lk Inf Al
 # ELF64-NEXT:  [ 0]                   NULL            0000000000000000 000000 000000 00      0   0  0
 # ELF64-NEXT:  [ 1] .text             PROGBITS        0000000000000000 000040 000001 00  AX  0   0  4
-# ELF64-NEXT:  [ 2] .rela.text        RELA            0000000000000000 000048 000000 18      5   1  8
-# ELF64-NEXT:  [ 3] .data             PROGBITS        0000000000000000 000048 000000 00  WA  0   0  4
-# ELF64-NEXT:  [ 4] .bss              NOBITS          0000000000000000 000048 000000 00  WA  0   0  4
-# ELF64-NEXT:  [ 5] .symtab           SYMTAB          0000000000000000 000048 000030 18      6   2  8
-# ELF64-NEXT:  [ 6] .strtab           STRTAB          0000000000000000 000078 000007 00      0   0  1
-# ELF64-NEXT:  [ 7] .shstrtab         STRTAB          0000000000000000 00007f 000031 00      0   0  1
+# ELF64-NEXT:  [ 2] .rel.text         REL             0000000000000000 000044 000000 08      6   1  4
+# ELF64-NEXT:  [ 3] .rela.text        RELA            0000000000000000 000048 000000 18      6   1  8
+# ELF64-NEXT:  [ 4] .data             PROGBITS        0000000000000000 000048 000000 00  WA  0   0  4
+# ELF64-NEXT:  [ 5] .bss              NOBITS          0000000000000000 000048 000000 00  WA  0   0  4
+# ELF64-NEXT:  [ 6] .symtab           SYMTAB          0000000000000000 000048 000030 18      7   2  8
+# ELF64-NEXT:  [ 7] .strtab           STRTAB          0000000000000000 000078 000007 00      0   0  1
+# ELF64-NEXT:  [ 8] .shstrtab         STRTAB          0000000000000000 00007f 00003b 00      0   0  1
 # ELF64-NEXT:Key to Flags:
 # ELF64-NEXT:  W (write), A (alloc), X (execute), M (merge), S (strings), I (info),
 # ELF64-NEXT:  L (link order), O (extra OS processing required), G (group), T (TLS),
 # ELF64-NEXT:  C (compressed), x (unknown), o (OS specific), E (exclude),
 # ELF64-NEXT:  l (large), p (processor specific)
 
---- !ELF
-FileHeader:
-  Class:   ELFCLASS64
-  Data:    ELFDATA2LSB
-  OSABI:   ELFOSABI_GNU
-  Type:    ET_REL
-  Machine: EM_X86_64
-Sections:
-  - Name:         .text
-    Type:         SHT_PROGBITS
-    Flags:        [ SHF_ALLOC, SHF_EXECINSTR ]
-    AddressAlign: 0x0000000000000004
-    Content:      00
-  - Name:         .rela.text
-    Type:         SHT_RELA
-    Link:         .symtab
-    AddressAlign: 0x0000000000000008
-    EntSize:      0x0000000000000018
-    Info:         .text
-    Relocations:
-  - Name:         .data
-    Type:         SHT_PROGBITS
-    Flags:        [ SHF_WRITE, SHF_ALLOC ]
-    AddressAlign: 0x0000000000000004
-    Content:      ''
-  - Name:         .bss
-    Type:         SHT_NOBITS
-    Flags:        [ SHF_WRITE, SHF_ALLOC ]
-    AddressAlign: 0x0000000000000004
-Symbols:
-  - Name:    .text
-    Type:    STT_SECTION
-    Section: .text
-
 ## For an EM_ARM target we print "y" for the SHF_ARM_PURECODE section flag.
 ## Check we mention it in the flag key.
 
-# RUN: yaml2obj --docnum=3 %s -o %t-arm.o
+# RUN: yaml2obj -DMACHINE=EM_ARM %s -o %t-arm.o
 # RUN: llvm-readelf -S %t-arm.o | FileCheck %s --check-prefix=ARM --strict-whitespace --match-full-lines
 
 #      ARM:Key to Flags:
@@ -135,11 +110,3 @@ Symbols:
 # ARM-NEXT:  L (link order), O (extra OS processing required), G (group), T (TLS),
 # ARM-NEXT:  C (compressed), x (unknown), o (OS specific), E (exclude),
 # ARM-NEXT:  y (purecode), p (processor specific)
-
---- !ELF
-FileHeader:
-  Class:   ELFCLASS64
-  Data:    ELFDATA2LSB
-  Type:    ET_REL
-  Machine: EM_ARM
-Sections: []
diff --git a/llvm/test/tools/llvm-readobj/ELF/hidden-versym.test b/llvm/test/tools/llvm-readobj/ELF/hidden-versym.test
index 871bab1767f2c..01a092425f81c 100644
--- a/llvm/test/tools/llvm-readobj/ELF/hidden-versym.test
+++ b/llvm/test/tools/llvm-readobj/ELF/hidden-versym.test
@@ -7,36 +7,30 @@
 
 --- !ELF
 FileHeader:
-  Class:             ELFCLASS64
-  Data:              ELFDATA2LSB
-  Type:              ET_EXEC
-  Machine:           EM_X86_64
-  Entry:             0x0000000000201000
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
 Sections:
-  - Name:            .gnu.version
-    Type:            SHT_GNU_versym
-    Flags:           [ SHF_ALLOC ]
-    Address:         0x0000000000200210
-    Link:            .dynsym
-    AddressAlign:    0x0000000000000002
-    EntSize:         0x0000000000000002
-    Entries:         [ 0, 0x8003 ]
-  - Name:            .gnu.version_r
-    Type:            SHT_GNU_verneed
-    Flags:           [ SHF_ALLOC ]
-    Address:         0x0000000000200250
-    Link:            .dynstr
-    AddressAlign:    0x0000000000000004
-    Info:            0x0000000000000001
+  - Name:    .gnu.version
+    Type:    SHT_GNU_versym
+    Flags:   [ SHF_ALLOC ]
+    Address: 0x0000000000200210
+    Link:    .dynsym
+    Entries: [ 0, 0x8003 ]
+  - Name:  .gnu.version_r
+    Type:  SHT_GNU_verneed
+    Flags: [ SHF_ALLOC ]
+    Link:  .dynstr
+    Info:  0x0000000000000001
     Dependencies:
-      - Version:     1
-        File:        somefile
+      - Version: 1
+        File:    somefile
         Entries:
-          - Name:    hiddensym
-            Hash:    1234
-            Flags:   0
-            Other:   3
+          - Name:  hiddensym
+            Hash:  1234
+            Flags: 0
+            Other: 3
 DynamicSymbols:
-  - Name:          h
-    Binding:       STB_GLOBAL
-...
+  - Name:    h
+    Binding: STB_GLOBAL
diff --git a/llvm/test/tools/llvm-readobj/ELF/merged.test b/llvm/test/tools/llvm-readobj/ELF/merged.test
index a6f567cda6cfe..454699b8a4342 100644
--- a/llvm/test/tools/llvm-readobj/ELF/merged.test
+++ b/llvm/test/tools/llvm-readobj/ELF/merged.test
@@ -1,13 +1,13 @@
-# Check merged args produce identical output to when not merged.
-RUN: llvm-readelf -aeWhSrnudlVgIs %p/Inputs/trivial.obj.elf-i386 > %t.merged
-RUN: llvm-readelf -a -e -W -h -S -r -n -u -d -l -V -g -I -s %p/Inputs/trivial.obj.elf-i386 > %t.not-merged
-RUN: cmp %t.merged %t.not-merged
-RUN: FileCheck %s --input-file %t.merged
+## Check merged args produce identical output to when not merged.
+# RUN: llvm-readelf -aeWhSrnudlVgIs %p/Inputs/trivial.obj.elf-i386 > %t.merged
+# RUN: llvm-readelf -a -e -W -h -S -r -n -u -d -l -V -g -I -s %p/Inputs/trivial.obj.elf-i386 > %t.not-merged
+# RUN: cmp %t.merged %t.not-merged
+# RUN: FileCheck %s --input-file %t.merged
 
-# llvm-readobj does not support merged args, because it also supports some old
-# flags (-st, -sd, etc.), and it would be confusing if only some merged args
-# were supported.
-RUN: not llvm-readobj -aeWhSrnudlVgIs %p/Inputs/trivial.obj.elf-i386 2>&1 | FileCheck %s --check-prefix=UNKNOWN
+## llvm-readobj does not support merged args, because it also supports some old
+## flags (-st, -sd, etc.), and it would be confusing if only some merged args
+## were supported.
+# RUN: not llvm-readobj -aeWhSrnudlVgIs %p/Inputs/trivial.obj.elf-i386 2>&1 | FileCheck %s --check-prefix=UNKNOWN
 
-CHECK-NOT: Unknown command line argument
-UNKNOWN:   for the --section-headers option: may only occur zero or one times!
+# CHECK-NOT: Unknown command line argument
+# UNKNOWN:   for the --section-headers option: may only occur zero or one times!
diff --git a/llvm/test/tools/llvm-readobj/ELF/mips-got-overlapped.test b/llvm/test/tools/llvm-readobj/ELF/mips-got-overlapped.test
index 881c63b79a4f9..c8f81ccf9d280 100644
--- a/llvm/test/tools/llvm-readobj/ELF/mips-got-overlapped.test
+++ b/llvm/test/tools/llvm-readobj/ELF/mips-got-overlapped.test
@@ -3,43 +3,43 @@
 # address as the .got. got-over.exe.elf-mips has zero-sized .data
 # section at the same offset .got section.
 
-RUN: llvm-readobj -A %p/Inputs/got-over.exe.elf-mips | FileCheck %s
+# RUN: llvm-readobj -A %p/Inputs/got-over.exe.elf-mips | FileCheck %s
 
-GOT-OBJ: Cannot find PLTGOT dynamic table tag.
+# GOT-OBJ: Cannot find PLTGOT dynamic table tag.
 
-CHECK:      Primary GOT {
-CHECK-NEXT:   Canonical gp value: 0x418270
-CHECK-NEXT:   Reserved entries [
-CHECK-NEXT:     Entry {
-CHECK-NEXT:       Address: 0x410280
-CHECK-NEXT:       Access: -32752
-CHECK-NEXT:       Initial: 0x0
-CHECK-NEXT:       Purpose: Lazy resolver
-CHECK-NEXT:     }
-CHECK-NEXT:     Entry {
-CHECK-NEXT:       Address: 0x410284
-CHECK-NEXT:       Access: -32748
-CHECK-NEXT:       Initial: 0x80000000
-CHECK-NEXT:       Purpose: Module pointer (GNU extension)
-CHECK-NEXT:     }
-CHECK-NEXT:   ]
-CHECK-NEXT:   Local entries [
-CHECK-NEXT:     Entry {
-CHECK-NEXT:       Address: 0x410288
-CHECK-NEXT:       Access: -32744
-CHECK-NEXT:       Initial: 0x4001B8
-CHECK-NEXT:     }
-CHECK-NEXT:   ]
-CHECK-NEXT:   Global entries [
-CHECK-NEXT:     Entry {
-CHECK-NEXT:       Address: 0x41028C
-CHECK-NEXT:       Access: -32740
-CHECK-NEXT:       Initial: 0x0
-CHECK-NEXT:       Value: 0x0
-CHECK-NEXT:       Type: None
-CHECK-NEXT:       Section: Undefined
-CHECK-NEXT:       Name: _foo
-CHECK-NEXT:     }
-CHECK-NEXT:   ]
-CHECK-NEXT:   Number of TLS and multi-GOT entries: 0
-CHECK-NEXT: }
+# CHECK:      Primary GOT {
+# CHECK-NEXT:   Canonical gp value: 0x418270
+# CHECK-NEXT:   Reserved entries [
+# CHECK-NEXT:     Entry {
+# CHECK-NEXT:       Address: 0x410280
+# CHECK-NEXT:       Access: -32752
+# CHECK-NEXT:       Initial: 0x0
+# CHECK-NEXT:       Purpose: Lazy resolver
+# CHECK-NEXT:     }
+# CHECK-NEXT:     Entry {
+# CHECK-NEXT:       Address: 0x410284
+# CHECK-NEXT:       Access: -32748
+# CHECK-NEXT:       Initial: 0x80000000
+# CHECK-NEXT:       Purpose: Module pointer (GNU extension)
+# CHECK-NEXT:     }
+# CHECK-NEXT:   ]
+# CHECK-NEXT:   Local entries [
+# CHECK-NEXT:     Entry {
+# CHECK-NEXT:       Address: 0x410288
+# CHECK-NEXT:       Access: -32744
+# CHECK-NEXT:       Initial: 0x4001B8
+# CHECK-NEXT:     }
+# CHECK-NEXT:   ]
+# CHECK-NEXT:   Global entries [
+# CHECK-NEXT:     Entry {
+# CHECK-NEXT:       Address: 0x41028C
+# CHECK-NEXT:       Access: -32740
+# CHECK-NEXT:       Initial: 0x0
+# CHECK-NEXT:       Value: 0x0
+# CHECK-NEXT:       Type: None
+# CHECK-NEXT:       Section: Undefined
+# CHECK-NEXT:       Name: _foo
+# CHECK-NEXT:     }
+# CHECK-NEXT:   ]
+# CHECK-NEXT:   Number of TLS and multi-GOT entries: 0
+# CHECK-NEXT: }
diff --git a/llvm/test/tools/llvm-readobj/ELF/mips-options.test b/llvm/test/tools/llvm-readobj/ELF/mips-options.test
index d44b3c0096a38..0d0ec7e554015 100644
--- a/llvm/test/tools/llvm-readobj/ELF/mips-options.test
+++ b/llvm/test/tools/llvm-readobj/ELF/mips-options.test
@@ -1,22 +1,22 @@
-# Check DT_MIPS_OPTIONS .dynamic section tag reading
+## Check DT_MIPS_OPTIONS .dynamic section tag reading
 
-RUN: llvm-readobj --dynamic-table %p/Inputs/mips-options.elf-mips64el | \
-RUN:   FileCheck %s
+# RUN: llvm-readobj --dynamic-table %p/Inputs/mips-options.elf-mips64el | \
+# RUN:   FileCheck %s
 
-CHECK:      DynamicSection [ (14 entries)
-CHECK-NEXT:   Tag                Type                 Name/Value
-CHECK-NEXT:   0x0000000000000004 HASH                 0x158
-CHECK-NEXT:   0x0000000000000005 STRTAB               0x1C0
-CHECK-NEXT:   0x0000000000000006 SYMTAB               0x178
-CHECK-NEXT:   0x000000000000000A STRSZ                7 (bytes)
-CHECK-NEXT:   0x000000000000000B SYMENT               24 (bytes)
-CHECK-NEXT:   0x0000000070000001 MIPS_RLD_VERSION     1
-CHECK-NEXT:   0x0000000070000005 MIPS_FLAGS           NOTPOT 
-CHECK-NEXT:   0x0000000070000006 MIPS_BASE_ADDRESS    0x0
-CHECK-NEXT:   0x000000007000000A MIPS_LOCAL_GOTNO     2
-CHECK-NEXT:   0x0000000070000011 MIPS_SYMTABNO        3
-CHECK-NEXT:   0x0000000070000013 MIPS_GOTSYM          0x3
-CHECK-NEXT:   0x0000000000000003 PLTGOT               0x2000
-CHECK-NEXT:   0x0000000070000029 MIPS_OPTIONS         0x1000
-CHECK-NEXT:   0x0000000000000000 NULL                 0x0
-CHECK-NEXT: ]
+# CHECK:      DynamicSection [ (14 entries)
+# CHECK-NEXT:   Tag                Type                 Name/Value
+# CHECK-NEXT:   0x0000000000000004 HASH                 0x158
+# CHECK-NEXT:   0x0000000000000005 STRTAB               0x1C0
+# CHECK-NEXT:   0x0000000000000006 SYMTAB               0x178
+# CHECK-NEXT:   0x000000000000000A STRSZ                7 (bytes)
+# CHECK-NEXT:   0x000000000000000B SYMENT               24 (bytes)
+# CHECK-NEXT:   0x0000000070000001 MIPS_RLD_VERSION     1
+# CHECK-NEXT:   0x0000000070000005 MIPS_FLAGS           NOTPOT
+# CHECK-NEXT:   0x0000000070000006 MIPS_BASE_ADDRESS    0x0
+# CHECK-NEXT:   0x000000007000000A MIPS_LOCAL_GOTNO     2
+# CHECK-NEXT:   0x0000000070000011 MIPS_SYMTABNO        3
+# CHECK-NEXT:   0x0000000070000013 MIPS_GOTSYM          0x3
+# CHECK-NEXT:   0x0000000000000003 PLTGOT               0x2000
+# CHECK-NEXT:   0x0000000070000029 MIPS_OPTIONS         0x1000
+# CHECK-NEXT:   0x0000000000000000 NULL                 0x0
+# CHECK-NEXT: ]
diff --git a/llvm/test/tools/llvm-readobj/ELF/mips-rld-map-rel.test b/llvm/test/tools/llvm-readobj/ELF/mips-rld-map-rel.test
index 05e541120a651..00198bbfe7571 100644
--- a/llvm/test/tools/llvm-readobj/ELF/mips-rld-map-rel.test
+++ b/llvm/test/tools/llvm-readobj/ELF/mips-rld-map-rel.test
@@ -1,24 +1,24 @@
-# Check DT_MIPS_RLD_MAP_REL .dynamic section tag reading
+## Check DT_MIPS_RLD_MAP_REL .dynamic section tag reading
 
-RUN: llvm-readobj --dynamic-table %p/Inputs/mips-rld-map-rel.elf-mipsel | \
-RUN:   FileCheck %s
+# RUN: llvm-readobj --dynamic-table %p/Inputs/mips-rld-map-rel.elf-mipsel | \
+# RUN:   FileCheck %s
 
-CHECK:      DynamicSection [ (16 entries)
-CHECK-NEXT:   Tag                Type                 Name/Value
-CHECK-NEXT:   0x00000004 HASH                 0x220
-CHECK-NEXT:   0x00000005 STRTAB               0x2FC
-CHECK-NEXT:   0x00000006 SYMTAB               0x25C
-CHECK-NEXT:   0x0000000A STRSZ                72 (bytes)
-CHECK-NEXT:   0x0000000B SYMENT               16 (bytes)
-CHECK-NEXT:   0x70000035 MIPS_RLD_MAP_REL     0x101E0
-CHECK-NEXT:   0x00000015 DEBUG                0x0
-CHECK-NEXT:   0x00000003 PLTGOT               0x10390
-CHECK-NEXT:   0x70000001 MIPS_RLD_VERSION     1
-CHECK-NEXT:   0x70000005 MIPS_FLAGS           NOTPOT 
-CHECK-NEXT:   0x70000006 MIPS_BASE_ADDRESS    0x0
-CHECK-NEXT:   0x7000000A MIPS_LOCAL_GOTNO     2
-CHECK-NEXT:   0x70000011 MIPS_SYMTABNO        10
-CHECK-NEXT:   0x70000012 MIPS_UNREFEXTNO      15
-CHECK-NEXT:   0x70000013 MIPS_GOTSYM          0xA
-CHECK-NEXT:   0x00000000 NULL                 0x0
-CHECK-NEXT: ]
+# CHECK:      DynamicSection [ (16 entries)
+# CHECK-NEXT:   Tag                Type                 Name/Value
+# CHECK-NEXT:   0x00000004 HASH                 0x220
+# CHECK-NEXT:   0x00000005 STRTAB               0x2FC
+# CHECK-NEXT:   0x00000006 SYMTAB               0x25C
+# CHECK-NEXT:   0x0000000A STRSZ                72 (bytes)
+# CHECK-NEXT:   0x0000000B SYMENT               16 (bytes)
+# CHECK-NEXT:   0x70000035 MIPS_RLD_MAP_REL     0x101E0
+# CHECK-NEXT:   0x00000015 DEBUG                0x0
+# CHECK-NEXT:   0x00000003 PLTGOT               0x10390
+# CHECK-NEXT:   0x70000001 MIPS_RLD_VERSION     1
+# CHECK-NEXT:   0x70000005 MIPS_FLAGS           NOTPOT
+# CHECK-NEXT:   0x70000006 MIPS_BASE_ADDRESS    0x0
+# CHECK-NEXT:   0x7000000A MIPS_LOCAL_GOTNO     2
+# CHECK-NEXT:   0x70000011 MIPS_SYMTABNO        10
+# CHECK-NEXT:   0x70000012 MIPS_UNREFEXTNO      15
+# CHECK-NEXT:   0x70000013 MIPS_GOTSYM          0xA
+# CHECK-NEXT:   0x00000000 NULL                 0x0
+# CHECK-NEXT: ]
diff --git a/llvm/test/tools/llvm-readobj/ELF/needed-libs.test b/llvm/test/tools/llvm-readobj/ELF/needed-libs.test
index 1c5f4978f093b..99bdca29a5160 100644
--- a/llvm/test/tools/llvm-readobj/ELF/needed-libs.test
+++ b/llvm/test/tools/llvm-readobj/ELF/needed-libs.test
@@ -46,7 +46,7 @@ Sections:
       - Tag:   DT_NEEDED
         Value: 0x1111111
       - Tag:   DT_STRSZ
-        Value: 0xD
+        Value: [[SIZE=0xD]]
       - Tag:   DT_NULL
         Value: 0x0
 ProgramHeaders:
diff --git a/llvm/test/tools/llvm-readobj/ELF/no-action.test b/llvm/test/tools/llvm-readobj/ELF/no-action.test
index b18c900438542..acfcb285f58bd 100644
--- a/llvm/test/tools/llvm-readobj/ELF/no-action.test
+++ b/llvm/test/tools/llvm-readobj/ELF/no-action.test
@@ -2,7 +2,7 @@
 
 ## Check the behavior on ET_EXEC input.
 
-# RUN: yaml2obj --docnum=1 %s -o %t.exe
+# RUN: yaml2obj -DTYPE=ET_EXEC %s -o %t.exe
 # RUN: llvm-readobj %t.exe | FileCheck %s -DFILE=%t.exe --check-prefix LLVM
 # RUN: llvm-readelf %t.exe | FileCheck %s -DFILE=%t.exe --check-prefix GNU --allow-empty
 
@@ -20,31 +20,17 @@
 FileHeader:
   Class:   ELFCLASS64
   Data:    ELFDATA2LSB
-  Type:    ET_EXEC
+  Type:    [[TYPE]]
   Machine: EM_X86_64
 
 ## Check the behavior on ET_REL input.
 
-# RUN: yaml2obj --docnum=2 %s -o %t.o
+# RUN: yaml2obj -DTYPE=ET_REL %s -o %t.o
 # RUN: llvm-readobj %t.o | FileCheck %s -DFILE=%t.o --check-prefix LLVM
 # RUN: llvm-readelf %t.o | FileCheck %s -DFILE=%t.o --check-prefix GNU --allow-empty
 
---- !ELF
-FileHeader:
-  Class:   ELFCLASS64
-  Data:    ELFDATA2LSB
-  Type:    ET_REL
-  Machine: EM_X86_64
-
 ## Check the behavior on ET_DYN input.
 
-# RUN: yaml2obj --docnum=3 %s -o %t.so
+# RUN: yaml2obj -DTYPE=ET_DYN %s -o %t.so
 # RUN: llvm-readobj %t.so | FileCheck %s -DFILE=%t.so --check-prefix LLVM
 # RUN: llvm-readelf %t.so | FileCheck %s -DFILE=%t.so --check-prefix GNU --allow-empty
-
---- !ELF
-FileHeader:
-  Class:   ELFCLASS64
-  Data:    ELFDATA2LSB
-  Type:    ET_DYN
-  Machine: EM_X86_64
diff --git a/llvm/test/tools/llvm-readobj/ELF/packed-relocs.test b/llvm/test/tools/llvm-readobj/ELF/packed-relocs.test
index 330426c710ecf..e3664d44ccaad 100644
--- a/llvm/test/tools/llvm-readobj/ELF/packed-relocs.test
+++ b/llvm/test/tools/llvm-readobj/ELF/packed-relocs.test
@@ -27,25 +27,21 @@
 # elf-packed-relocs1.s
 --- !ELF
 FileHeader:
-  Class:           ELFCLASS64
-  Data:            ELFDATA2LSB
-  Type:            ET_DYN
-  Machine:         EM_X86_64
-  Entry:           0x0000000000001000
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_DYN
+  Machine: EM_X86_64
 Sections:
-  - Name:            .rela.dyn
-    Type:            SHT_ANDROID_RELA
-    Flags:           [ SHF_ALLOC ]
-    Address:         0x00000000000001C8
-    Link:            .symtab
-    AddressAlign:    0x0000000000000001
-    Content:         41505332088020020108800280010202088180808010818080802002080181808080100802818080802004020C7E048180808010088180808020
+  - Name:    .rela.dyn
+    Type:    SHT_ANDROID_RELA
+    Flags:   [ SHF_ALLOC ]
+    Link:    .symtab
+    Content: 41505332088020020108800280010202088180808010818080802002080181808080100802818080802004020C7E048180808010088180808020
 Symbols:
-  - Name:            sym1
-    Binding:         STB_GLOBAL
-  - Name:            sym2
-    Binding:         STB_GLOBAL
-...
+  - Name:    sym1
+    Binding: STB_GLOBAL
+  - Name:    sym2
+    Binding: STB_GLOBAL
 
 # RUN: yaml2obj --docnum=2 %s | llvm-readobj --relocations - | FileCheck --check-prefix=LLVM2 %s
 # LLVM2:      Section (1) .rel.dyn {
@@ -77,25 +73,21 @@ Symbols:
 # elf-packed-relocs2.s
 --- !ELF
 FileHeader:
-  Class:           ELFCLASS32
-  Data:            ELFDATA2LSB
-  Type:            ET_DYN
-  Machine:         EM_386
-  Entry:           0x0000000000001000
+  Class:   ELFCLASS32
+  Data:    ELFDATA2LSB
+  Type:    ET_DYN
+  Machine: EM_386
 Sections:
-  - Name:            .rel.dyn
-    Type:            SHT_ANDROID_REL
-    Flags:           [ SHF_ALLOC ]
-    Address:         0x00000000000001C8
-    Link:            .symtab
-    AddressAlign:    0x0000000000000001
-    Content:         415053320A80200202088102830408037C08
+  - Name:    .rel.dyn
+    Type:    SHT_ANDROID_REL
+    Flags:   [ SHF_ALLOC ]
+    Link:    .symtab
+    Content: 415053320A80200202088102830408037C08
 Symbols:
-  - Name:            sym1
-    Binding:         STB_GLOBAL
-  - Name:            sym2
-    Binding:         STB_GLOBAL
-...
+  - Name:    sym1
+    Binding: STB_GLOBAL
+  - Name:    sym2
+    Binding: STB_GLOBAL
 
 # RUN: yaml2obj --docnum=3 %s | llvm-readobj --relocations - | FileCheck --check-prefix=LLVM3 %s
 #
@@ -120,22 +112,18 @@ Symbols:
 # elf-packed-relocs3.s
 --- !ELF
 FileHeader:
-  Class:           ELFCLASS64
-  Data:            ELFDATA2LSB
-  Type:            ET_DYN
-  Machine:         EM_X86_64
-  Entry:           0x0000000000001000
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_DYN
+  Machine: EM_X86_64
 Sections:
-  - Name:            .rela.dyn
-    Type:            SHT_ANDROID_RELA
-    Flags:           [ SHF_ALLOC ]
-    Address:         0x00000000000001C8
-    Link:            .symtab
-    AddressAlign:    0x0000000000000001
-    Content:         415053320680200208800208008001080802008001818080801008818080802002080881808080100008818080802008
+  - Name:    .rela.dyn
+    Type:    SHT_ANDROID_RELA
+    Flags:   [ SHF_ALLOC ]
+    Link:    .symtab
+    Content: 415053320680200208800208008001080802008001818080801008818080802002080881808080100008818080802008
 Symbols:
-  - Name:            sym1
-    Binding:         STB_GLOBAL
-  - Name:            sym2
-    Binding:         STB_GLOBAL
-...
+  - Name:    sym1
+    Binding: STB_GLOBAL
+  - Name:    sym2
+    Binding: STB_GLOBAL
diff --git a/llvm/test/tools/llvm-readobj/ELF/pt-gnu-property.test b/llvm/test/tools/llvm-readobj/ELF/pt-gnu-property.test
index f47341115b5ca..31e2304334124 100644
--- a/llvm/test/tools/llvm-readobj/ELF/pt-gnu-property.test
+++ b/llvm/test/tools/llvm-readobj/ELF/pt-gnu-property.test
@@ -2,6 +2,8 @@
 # RUN: llvm-readelf --program-headers %t | FileCheck %s --check-prefix=GNU
 # RUN: llvm-readobj --program-headers %t | FileCheck %s --check-prefix=LLVM
 
+## TODO: merge this test with program-headers.test.
+
 # GNU: {{ }}GNU_PROPERTY{{ }}
 
 # LLVM:        ProgramHeader {
diff --git a/llvm/test/tools/llvm-readobj/ELF/reloc-symbol-with-versioning.test b/llvm/test/tools/llvm-readobj/ELF/reloc-symbol-with-versioning.test
index f6314cce99f3d..933d53f31221c 100644
--- a/llvm/test/tools/llvm-readobj/ELF/reloc-symbol-with-versioning.test
+++ b/llvm/test/tools/llvm-readobj/ELF/reloc-symbol-with-versioning.test
@@ -21,88 +21,69 @@
 # LLVM-NEXT: ]
 
 --- !ELF
-FileHeader:      
-  Class:           ELFCLASS64
-  Data:            ELFDATA2LSB
-  Type:            ET_EXEC
-  Machine:         EM_X86_64
-  Entry:           0x0000000000000000
-Sections:        
-  - Name:            .gnu.version
-    Type:            SHT_GNU_versym
-    Flags:           [ SHF_ALLOC ]
-    Link:            .dynsym
-    AddressAlign:    0x0000000000000002
-    EntSize:         0x0000000000000002
-    Entries:         [ 0, 2, 3, 4, 2 ]
-  - Name:            .gnu.version_r
-    Type:            SHT_GNU_verneed
-    Flags:           [ SHF_ALLOC ]
-    Link:            .dynstr
-    AddressAlign:    0x0000000000000004
-    Info:            0x0000000000000002
-    Dependencies:    
-      - Version:         1
-        File:            verneed1.so.0
-        Entries:         
-          - Name:            v2
-            Hash:            1938
-            Flags:           0
-            Other:           3
-          - Name:            v3
-            Hash:            1939
-            Flags:           0
-            Other:           2
-      - Version:         1
-        File:            verneed2.so.0
-        Entries:         
-          - Name:            v1
-            Hash:            1937
-            Flags:           0
-            Other:           4
-  - Name:            .rela.plt
-    Type:            SHT_RELA
-    Flags:           [ SHF_ALLOC ]
-    Info:            0
-    Link:            .dynsym
-    AddressAlign:    0x0000000000000008
-    EntSize:         0x0000000000000018
-    Relocations:     
-      - Offset:          0x0000000000013018
-        Symbol:          f1
-        Type:            R_X86_64_JUMP_SLOT
-      - Offset:          0x0000000000013020
-        Symbol:          f2
-        Type:            R_X86_64_JUMP_SLOT
-      - Offset:          0x0000000000013028
-        Symbol:          g1
-        Type:            R_X86_64_JUMP_SLOT
-      - Offset:          0x0000000000013040
-        Symbol:          _Z2f1v
-        Type:            R_X86_64_JUMP_SLOT
-      - Offset:          0x0000000000013058
-        Symbol:          f3
-        Type:            R_X86_64_JUMP_SLOT
-Symbols:
-  - Name:            f1
-    Binding:         STB_GLOBAL
-  - Name:            f2
-    Binding:         STB_GLOBAL
-  - Name:            g1
-    Binding:         STB_GLOBAL
-  - Name:            _Z2f1v
-    Binding:         STB_GLOBAL
-  - Name:            f3
-    Binding:         STB_GLOBAL
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name:    .gnu.version
+    Type:    SHT_GNU_versym
+    Flags:   [ SHF_ALLOC ]
+    Entries: [ 0, 2, 3, 4, 2 ]
+  - Name:         .gnu.version_r
+    Type:         SHT_GNU_verneed
+    Flags:        [ SHF_ALLOC ]
+    Link:         .dynstr
+    AddressAlign: 0x0000000000000004
+    Info:         0x0000000000000002
+    Dependencies:
+      - Version: 1
+        File:    verneed1.so.0
+        Entries:
+          - Name:  v2
+            Hash:  1938
+            Flags: 0
+            Other: 3
+          - Name:  v3
+            Hash:  1939
+            Flags: 0
+            Other: 2
+      - Version: 1
+        File:    verneed2.so.0
+        Entries:
+          - Name:  v1
+            Hash:  1937
+            Flags: 0
+            Other: 4
+  - Name:  .rela.plt
+    Type:  SHT_RELA
+    Flags: [ SHF_ALLOC ]
+    Link:  .dynsym
+    Relocations:
+      - Offset: 0x0000000000013018
+        Symbol: f1
+        Type:   R_X86_64_JUMP_SLOT
+      - Offset: 0x0000000000013020
+        Symbol: f2
+        Type:   R_X86_64_JUMP_SLOT
+      - Offset: 0x0000000000013028
+        Symbol: g1
+        Type:   R_X86_64_JUMP_SLOT
+      - Offset: 0x0000000000013040
+        Symbol: _Z2f1v
+        Type:   R_X86_64_JUMP_SLOT
+      - Offset: 0x0000000000013058
+        Symbol: f3
+        Type:   R_X86_64_JUMP_SLOT
 DynamicSymbols:  
-  - Name:            f1
-    Binding:         STB_GLOBAL
-  - Name:            f2
-    Binding:         STB_GLOBAL
-  - Name:            g1
-    Binding:         STB_GLOBAL
-  - Name:            _Z2f1v
-    Binding:         STB_GLOBAL
-  - Name:            f3
-    Binding:         STB_GLOBAL
-...
+  - Name:    f1
+    Binding: STB_GLOBAL
+  - Name:    f2
+    Binding: STB_GLOBAL
+  - Name:    g1
+    Binding: STB_GLOBAL
+  - Name:    _Z2f1v
+    Binding: STB_GLOBAL
+  - Name:    f3
+    Binding: STB_GLOBAL
diff --git a/llvm/test/tools/llvm-readobj/ELF/reloc-types-elf-aarch64.test b/llvm/test/tools/llvm-readobj/ELF/reloc-types-elf-aarch64.test
index 2882d32fd92a6..7291282811048 100644
--- a/llvm/test/tools/llvm-readobj/ELF/reloc-types-elf-aarch64.test
+++ b/llvm/test/tools/llvm-readobj/ELF/reloc-types-elf-aarch64.test
@@ -136,14 +136,8 @@ FileHeader:
   Type:    ET_REL
   Machine: EM_AARCH64
 Sections:
-  - Name:    .text
-    Type:    SHT_PROGBITS
-    Content: 00
-  - Name:         .rela.text
-    Type:         SHT_RELA
-    AddressAlign: 0x0000000000000008
-    EntSize:      0x0000000000000018
-    Info:         .text
+  - Name: .rela.text
+    Type: SHT_RELA
     Relocations:
       - Type: R_AARCH64_NONE
       - Type: R_AARCH64_ABS64
diff --git a/llvm/test/tools/llvm-readobj/ELF/reloc-types-elf-arm.test b/llvm/test/tools/llvm-readobj/ELF/reloc-types-elf-arm.test
index f72d7cdd89138..96d6cfed4df3e 100644
--- a/llvm/test/tools/llvm-readobj/ELF/reloc-types-elf-arm.test
+++ b/llvm/test/tools/llvm-readobj/ELF/reloc-types-elf-arm.test
@@ -142,16 +142,9 @@ FileHeader:
   Data:    ELFDATA2LSB
   Type:    ET_REL
   Machine: EM_ARM
-  Flags:   [ EF_ARM_EABI_VER5 ]
 Sections:
-  - Name:    .text
-    Type:    SHT_PROGBITS
-    Content: 00
-  - Name:         .rel.text
-    Type:         SHT_REL
-    AddressAlign: 0x0000000000000004
-    EntSize:      0x0000000000000008
-    Info:         .text
+  - Name: .rel.text
+    Type: SHT_REL
     Relocations:
       - Type: R_ARM_NONE
       - Type: R_ARM_PC24
diff --git a/llvm/test/tools/llvm-readobj/ELF/reloc-types-elf-i386.test b/llvm/test/tools/llvm-readobj/ELF/reloc-types-elf-i386.test
index 8606f3fa8cba5..fb16185ecf48e 100644
--- a/llvm/test/tools/llvm-readobj/ELF/reloc-types-elf-i386.test
+++ b/llvm/test/tools/llvm-readobj/ELF/reloc-types-elf-i386.test
@@ -47,7 +47,7 @@
 # LLVM: Type: R_386_IRELATIVE (42)
 # LLVM: Type: R_386_GOT32X (43)
 
-# GNU:      Relocation section '.rel.text' at offset 0x38 contains 41 entries:
+# GNU:      Relocation section '.rel.text' at offset 0x34 contains 41 entries:
 # GNU-NEXT:  Offset     Info    Type                Sym. Value  Symbol's Name
 # GNU-NEXT: 00000002  00000100 R_386_NONE             00000000   foo
 # GNU-NEXT: 00000008  00000101 R_386_32               00000000   foo
@@ -93,21 +93,14 @@
 
 --- !ELF
 FileHeader:
-  Class:    ELFCLASS32
-  Data:     ELFDATA2LSB
-  OSABI:    ELFOSABI_GNU
-  Type:     ET_REL
-  Machine:  EM_386
+  Class:   ELFCLASS32
+  Data:    ELFDATA2LSB
+  Type:    ET_REL
+  Machine: EM_386
 Sections:
-  - Name:    .text
-    Type:    SHT_PROGBITS
-    Content: 00
-  - Name:         .rel.text
-    Type:         SHT_REL
-    Link:         .symtab
-    AddressAlign: 0x0000000000000004
-    EntSize:      0x0000000000000008
-    Info:         .text
+  - Name: .rel.text
+    Type: SHT_REL
+    Link: .symtab
     Relocations:
       - Offset: 0x0000000000000002
         Type:   R_386_NONE
diff --git a/llvm/test/tools/llvm-readobj/ELF/reloc-types-elf-lanai.test b/llvm/test/tools/llvm-readobj/ELF/reloc-types-elf-lanai.test
index 6bbe1a839b509..123ab1f5bd0e6 100644
--- a/llvm/test/tools/llvm-readobj/ELF/reloc-types-elf-lanai.test
+++ b/llvm/test/tools/llvm-readobj/ELF/reloc-types-elf-lanai.test
@@ -19,16 +19,8 @@ FileHeader:
   Type:    ET_REL
   Machine: EM_LANAI
 Sections:
-  - Name:         .text
-    Type:         SHT_PROGBITS
-    Flags:        [ SHF_ALLOC, SHF_EXECINSTR ]
-    AddressAlign: 0x0000000000000004
-    Content:      00
-  - Name:         .rela.text
-    Type:         SHT_RELA
-    AddressAlign: 0x0000000000000004
-    EntSize:      0x000000000000000C
-    Info:         .text
+  - Name: .rela.text
+    Type: SHT_RELA
     Relocations:
       - Type: R_LANAI_NONE
       - Type: R_LANAI_21
diff --git a/llvm/test/tools/llvm-readobj/ELF/reloc-types-elf-mips.test b/llvm/test/tools/llvm-readobj/ELF/reloc-types-elf-mips.test
index 59793c512b35a..3566d42a5d735 100644
--- a/llvm/test/tools/llvm-readobj/ELF/reloc-types-elf-mips.test
+++ b/llvm/test/tools/llvm-readobj/ELF/reloc-types-elf-mips.test
@@ -61,18 +61,11 @@
 FileHeader:
   Class:   ELFCLASS32
   Data:    ELFDATA2MSB
-  OSABI:   ELFOSABI_GNU
   Type:    ET_REL
   Machine: EM_MIPS
 Sections:
-  - Name:    .text
-    Type:    SHT_PROGBITS
-    Content: 00
-  - Name:         .rel.text
-    Type:         SHT_REL
-    AddressAlign: 0x0000000000000004
-    EntSize:      0x0000000000000008
-    Info:         .text
+  - Name: .rel.text
+    Type: SHT_REL
     Relocations:
       - Type: R_MIPS_NONE
       - Type: R_MIPS_16
diff --git a/llvm/test/tools/llvm-readobj/ELF/reloc-types-elf-mips64.test b/llvm/test/tools/llvm-readobj/ELF/reloc-types-elf-mips64.test
index fc72edd29d2c0..d96c922bd8339 100644
--- a/llvm/test/tools/llvm-readobj/ELF/reloc-types-elf-mips64.test
+++ b/llvm/test/tools/llvm-readobj/ELF/reloc-types-elf-mips64.test
@@ -61,18 +61,11 @@
 FileHeader:
   Class:   ELFCLASS64
   Data:    ELFDATA2LSB
-  OSABI:   ELFOSABI_GNU
   Type:    ET_REL
   Machine: EM_MIPS
 Sections:
-  - Name:    .text
-    Type:    SHT_PROGBITS
-    Content: 00
-  - Name:         .rela.text
-    Type:         SHT_RELA
-    AddressAlign: 0x0000000000000008
-    EntSize:      0x0000000000000018
-    Info:         .text
+  - Name: .rela.text
+    Type: SHT_RELA
     Relocations:
       - Type:  R_MIPS_NONE
       - Type:  R_MIPS_16
diff --git a/llvm/test/tools/llvm-readobj/ELF/reloc-types-elf-x64.test b/llvm/test/tools/llvm-readobj/ELF/reloc-types-elf-x64.test
index ad4e06fc955a3..0a48325d9c9d2 100644
--- a/llvm/test/tools/llvm-readobj/ELF/reloc-types-elf-x64.test
+++ b/llvm/test/tools/llvm-readobj/ELF/reloc-types-elf-x64.test
@@ -46,7 +46,7 @@
 # LLVM: Type: R_X86_64_GOTPCRELX (41)
 # LLVM: Type: R_X86_64_REX_GOTPCRELX (42)
 
-# GNU:      Relocation section '.rela.text' at offset 0x48 contains 40 entries:
+# GNU:      Relocation section '.rela.text' at offset 0x40 contains 40 entries:
 # GNU-NEXT:     Offset             Info             Type               Symbol's Value  Symbol's Name + Addend
 # GNU-NEXT: 0000000000000003  0000000100000000 R_X86_64_NONE          0000000000000000 foo + 0
 # GNU-NEXT: 000000000000000a  0000000100000001 R_X86_64_64            0000000000000000 foo + 0
@@ -93,19 +93,11 @@
 FileHeader:
   Class:   ELFCLASS64
   Data:    ELFDATA2LSB
-  OSABI:   ELFOSABI_GNU
   Type:    ET_REL
   Machine: EM_X86_64
 Sections:
-  - Name:    .text
-    Type:    SHT_PROGBITS
-    Content: 00
-  - Name:         .rela.text
-    Type:         SHT_RELA
-    Link:         .symtab
-    AddressAlign: 0x0000000000000008
-    EntSize:      0x0000000000000018
-    Info:         .text
+  - Name: .rela.text
+    Type: SHT_RELA
     Relocations:
       - Offset: 0x0000000000000003
         Type:   R_X86_64_NONE
diff --git a/llvm/test/tools/llvm-readobj/ELF/sections.test b/llvm/test/tools/llvm-readobj/ELF/sections.test
index d2865c041798a..1b087f710f18f 100644
--- a/llvm/test/tools/llvm-readobj/ELF/sections.test
+++ b/llvm/test/tools/llvm-readobj/ELF/sections.test
@@ -1,9 +1,9 @@
 ## Check how llvm-readobj prints sections with --sections.
 ## We test the --sections flag for llvm-readelf in the gnu-sections.test.
 
-# RUN: yaml2obj --docnum=1 %s -o %t64
+# RUN: yaml2obj %s -o %t64
 # RUN: llvm-readobj --sections %t64 | FileCheck %s --check-prefixes=ELF,ELF64
-# RUN: yaml2obj --docnum=2 %s -o %t32
+# RUN: yaml2obj -DBITS=32 %s -o %t32
 # RUN: llvm-readobj --sections %t32 | FileCheck %s --check-prefixes=ELF,ELF32
 
 ## Check flag aliases produce identical output.
@@ -89,27 +89,10 @@
 
 --- !ELF
 FileHeader:
-  Class:   ELFCLASS64
+  Class:   ELFCLASS[[BITS=64]]
   Data:    ELFDATA2LSB
   Type:    ET_REL
-  Machine: EM_X86_64
-Sections:
-  - Name:         .foo
-    Type:         SHT_PROGBITS
-    Info:         1
-    Address:      0x2
-    Size:         0x3
-    Flags:        [ SHF_WRITE, SHF_ALLOC ]
-    Link:         4
-    AddressAlign: 5
-    EntSize:      6
-
---- !ELF
-FileHeader:
-  Class:   ELFCLASS32
-  Data:    ELFDATA2LSB
-  Type:    ET_REL
-  Machine: EM_386
+  Machine: EM_NONE
 Sections:
   - Name:         .foo
     Type:         SHT_PROGBITS
diff --git a/llvm/test/tools/llvm-readobj/ELF/types.test b/llvm/test/tools/llvm-readobj/ELF/types.test
index aecbdeacde983..a9413959e3146 100644
--- a/llvm/test/tools/llvm-readobj/ELF/types.test
+++ b/llvm/test/tools/llvm-readobj/ELF/types.test
@@ -1,5 +1,5 @@
 # Show that llvm-readobj can handle all standard ELF types.
-# RUN: yaml2obj %s --docnum=1 -o %t.none
+# RUN: yaml2obj %s -DTYPE=ET_NONE -o %t.none
 # RUN: llvm-readobj --file-headers %t.none | FileCheck %s --check-prefix=LLVM-NONE
 # RUN: llvm-readelf --file-headers %t.none | FileCheck %s --check-prefix=GNU-NONE
 
@@ -10,107 +10,57 @@
 FileHeader:
   Class:   ELFCLASS64
   Data:    ELFDATA2LSB
-  Type:    ET_NONE
+  Type:    [[TYPE]]
   Machine: EM_X86_64
 
-# RUN: yaml2obj %s --docnum=2 -o %t.rel
+# RUN: yaml2obj %s -DTYPE=ET_REL -o %t.rel
 # RUN: llvm-readobj --file-headers %t.rel | FileCheck %s --check-prefix=LLVM-REL
 # RUN: llvm-readelf --file-headers %t.rel | FileCheck %s --check-prefix=GNU-REL
 
 # LLVM-REL: Type: Relocatable (0x1)
 # GNU-REL:  Type: REL (Relocatable file)
 
---- !ELF
-FileHeader:
-  Class:   ELFCLASS64
-  Data:    ELFDATA2LSB
-  Type:    ET_REL
-  Machine: EM_X86_64
-
-
-# RUN: yaml2obj %s --docnum=3 -o %t.exec
+# RUN: yaml2obj %s -DTYPE=ET_EXEC -o %t.exec
 # RUN: llvm-readobj --file-headers %t.exec | FileCheck %s --check-prefix=LLVM-EXEC
 # RUN: llvm-readelf --file-headers %t.exec | FileCheck %s --check-prefix=GNU-EXEC
 
 # LLVM-EXEC: Type: Executable (0x2)
 # GNU-EXEC:  Type: EXEC (Executable file)
 
---- !ELF
-FileHeader:
-  Class:   ELFCLASS64
-  Data:    ELFDATA2LSB
-  Type:    ET_EXEC
-  Machine: EM_X86_64
-
-# RUN: yaml2obj %s --docnum=4 -o %t.so
+# RUN: yaml2obj %s -DTYPE=ET_DYN -o %t.so
 # RUN: llvm-readobj --file-headers %t.so | FileCheck %s --check-prefix=LLVM-SO
 # RUN: llvm-readelf --file-headers %t.so | FileCheck %s --check-prefix=GNU-SO
 
 # LLVM-SO: Type: SharedObject (0x3)
 # GNU-SO:  Type: DYN (Shared object file)
 
---- !ELF
-FileHeader:
-  Class:   ELFCLASS64
-  Data:    ELFDATA2LSB
-  Type:    ET_DYN
-  Machine: EM_X86_64
-
-# RUN: yaml2obj %s --docnum=5 -o %t.core
+# RUN: yaml2obj %s -DTYPE=ET_CORE -o %t.core
 # RUN: llvm-readobj --file-headers %t.core | FileCheck %s --check-prefix=LLVM-CORE
 # RUN: llvm-readelf --file-headers %t.core | FileCheck %s --check-prefix=GNU-CORE
 
 # LLVM-CORE: Type: Core (0x4)
 # GNU-CORE:  Type: CORE (Core file)
 
---- !ELF
-FileHeader:
-  Class:   ELFCLASS64
-  Data:    ELFDATA2LSB
-  Type:    ET_CORE
-  Machine: EM_X86_64
-
 # Show that llvm-readobj can handle an unknown ELF type.
-# RUN: yaml2obj %s --docnum=6 -o %t.unknown
+# RUN: yaml2obj %s -DTYPE=0x42 -o %t.unknown
 # RUN: llvm-readobj --file-headers %t.unknown | FileCheck %s --check-prefix=LLVM-UNKNOWN
 # RUN: llvm-readelf --file-headers %t.unknown | FileCheck %s --check-prefix=GNU-UNKNOWN
 
 # LLVM-UNKNOWN: Type: 0x42
 # GNU-UNKNOWN:  Type: 42
 
---- !ELF
-FileHeader:
-  Class:   ELFCLASS64
-  Data:    ELFDATA2LSB
-  Type:    0x42
-  Machine: EM_X86_64
-
 # Show that llvm-readobj can handle an unknown OS-specific ELF type.
-# RUN: yaml2obj %s --docnum=7 -o %t.os
+# RUN: yaml2obj %s -DTYPE=0xfe00 -o %t.os
 # RUN: llvm-readobj --file-headers %t.os | FileCheck %s --check-prefix=LLVM-OS
 # RUN: llvm-readelf --file-headers %t.os | FileCheck %s --check-prefix=GNU-OS
 
 # LLVM-OS: Type: 0xFE00
 # GNU-OS:  Type: fe00
 
---- !ELF
-FileHeader:
-  Class:   ELFCLASS64
-  Data:    ELFDATA2LSB
-  Type:    0xfe00
-  Machine: EM_X86_64
-
 # Show that llvm-readobj can handle an unknown machine-specific ELF type.
-# RUN: yaml2obj %s --docnum=8 -o %t.proc
+# RUN: yaml2obj %s -DTYPE=0xff00 -o %t.proc
 # RUN: llvm-readobj --file-headers %t.proc | FileCheck %s --check-prefix=LLVM-PROC
 # RUN: llvm-readelf --file-headers %t.proc | FileCheck %s --check-prefix=GNU-PROC
 
 # LLVM-PROC: Type: 0xFF00
 # GNU-PROC:  Type: ff00
-
---- !ELF
-FileHeader:
-  Class:   ELFCLASS64
-  Data:    ELFDATA2LSB
-  Type:    0xff00
-  Machine: EM_X86_64
diff --git a/llvm/test/tools/llvm-readobj/ELF/verdef-invalid.test b/llvm/test/tools/llvm-readobj/ELF/verdef-invalid.test
index 991ed8b9cb5fd..f568bfe5ed00b 100644
--- a/llvm/test/tools/llvm-readobj/ELF/verdef-invalid.test
+++ b/llvm/test/tools/llvm-readobj/ELF/verdef-invalid.test
@@ -2,7 +2,7 @@
 
 ## Check that we report a warning when sh_link references a non-existent section.
 
-# RUN: yaml2obj %s --docnum=1 -o %t1
+# RUN: yaml2obj %s --docnum=1 -DLINK=0xFF -o %t1
 # RUN: llvm-readobj -V %t1 2>&1 | FileCheck %s --check-prefix=INVALID-LINK-LLVM --implicit-check-not="warning:" -DFILE=%t1
 # RUN: llvm-readelf -V %t1 2>&1 | FileCheck %s --check-prefix=INVALID-LINK-GNU --implicit-check-not="warning:" -DFILE=%t1
 
@@ -21,34 +21,21 @@ FileHeader:
 Sections:
   - Name:    .gnu.version_d
     Type:    SHT_GNU_verdef
-    Link:    0xFF
+    Link:    [[LINK]]
     Info:    0x0
     Entries: []
 
 ## Check that we report a warning when the sh_link field of a SHT_GNU_verdef section references a non-string table section.
 
-# RUN: yaml2obj %s --docnum=2 -o %t2
+# RUN: yaml2obj %s --docnum=1 -DLINK=0x0 -o %t2
 # RUN: llvm-readobj -V %t2 2>&1 | FileCheck %s --check-prefix=INVALID-STRING-TABLE -DFILE=%t2
 # RUN: llvm-readelf -V %t2 2>&1 | FileCheck %s --check-prefix=INVALID-STRING-TABLE -DFILE=%t2
 
 # INVALID-STRING-TABLE: warning: '[[FILE]]': invalid string table linked to SHT_GNU_verdef section with index 1: invalid sh_type for string table section [index 0]: expected SHT_STRTAB, but got SHT_NULL
 
---- !ELF
-FileHeader:
-  Class:   ELFCLASS64
-  Data:    ELFDATA2LSB
-  Type:    ET_DYN
-  Machine: EM_X86_64
-Sections:
-  - Name:    .gnu.version_d
-    Type:    SHT_GNU_verdef
-    Link:    0x0
-    Info:    0x0
-    Entries: []
-
 ## Check that we report a warning when we can't read the content of the SHT_GNU_verdef section.
 
-# RUN: yaml2obj %s --docnum=3 -o %t3
+# RUN: yaml2obj %s --docnum=2 -o %t3
 # RUN: llvm-readobj -V %t3 2>&1 | FileCheck %s --check-prefix=INVALID-DATA -DFILE=%t3
 # RUN: llvm-readelf -V %t3 2>&1 | FileCheck %s --check-prefix=INVALID-DATA -DFILE=%t3
 
@@ -73,7 +60,7 @@ DynamicSymbols:
 ## Check that we report a warning when a SHT_GNU_verdef section contains a version definition
 ## that goes past the end of the section.
 
-# RUN: yaml2obj %s --docnum=4 -o %t4
+# RUN: yaml2obj %s --docnum=3 -o %t4
 # RUN: llvm-readobj -V %t4 2>&1 | FileCheck %s --check-prefix=DEF-PAST-END -DFILE=%t4
 # RUN: llvm-readelf -V %t4 2>&1 | FileCheck %s --check-prefix=DEF-PAST-END -DFILE=%t4
 
@@ -104,7 +91,7 @@ DynamicSymbols:
 ## Check that we report a warning when a SHT_GNU_verdef section contains a version definition
 ## that refers to an auxiliary entry that goes past the end of the section.
 
-# RUN: yaml2obj %s --docnum=5 -o %t5
+# RUN: yaml2obj %s --docnum=4 -o %t5
 # RUN: llvm-readobj -V %t5 2>&1 | FileCheck %s --check-prefix=AUX-PAST-END -DFILE=%t5
 # RUN: llvm-readelf -V %t5 2>&1 | FileCheck %s --check-prefix=AUX-PAST-END -DFILE=%t5
 
@@ -135,7 +122,7 @@ DynamicSymbols:
 ## Check that we can dump a SHT_GNU_verdef section properly even if it contains version names strings
 ## that overrun the linked string table.
 
-# RUN: yaml2obj %s --docnum=6 -o %t6
+# RUN: yaml2obj %s --docnum=5 -o %t6
 # RUN: llvm-readobj -V %t6 2>&1 | FileCheck %s --check-prefix=PAST-STRTAB-END-LLVM --implicit-check-not="warning:" -DFILE=%t6
 # RUN: llvm-readelf -V %t6 2>&1 | FileCheck %s --check-prefix=PAST-STRTAB-END-GNU --implicit-check-not="warning:" -DFILE=%t6
 
@@ -180,7 +167,7 @@ DynamicSymbols:
 
 ## Check we report a warning when a version definition is not correctly aligned in memory.
 
-# RUN: yaml2obj %s --docnum=7 -o %t7
+# RUN: yaml2obj %s --docnum=6 -o %t7
 # RUN: llvm-readobj -V %t7 2>&1 | FileCheck %s --check-prefix=MISALIGNED-DEF -DFILE=%t7
 # RUN: llvm-readelf -V %t7 2>&1 | FileCheck %s --check-prefix=MISALIGNED-DEF -DFILE=%t7
 
@@ -211,7 +198,7 @@ DynamicSymbols:
 
 ## Check we report a warning when an auxiliary entry is not correctly aligned in memory.
 
-# RUN: yaml2obj %s --docnum=8 -o %t8
+# RUN: yaml2obj %s --docnum=7 -o %t8
 # RUN: llvm-readobj -V %t8 2>&1 | FileCheck %s --check-prefix=MISALIGNED-AUX -DFILE=%t8
 # RUN: llvm-readelf -V %t8 2>&1 | FileCheck %s --check-prefix=MISALIGNED-AUX -DFILE=%t8
 
@@ -237,7 +224,7 @@ DynamicSymbols:
 
 ## Check how we handle a version definition entry with an unsupported version.
 
-# RUN: yaml2obj %s --docnum=9 -o %t9
+# RUN: yaml2obj %s --docnum=8 -o %t9
 # RUN: llvm-readobj -V %t9 2>&1 | FileCheck %s --check-prefix=UNSUPPORTED-VERSION -DFILE=%t9
 # RUN: llvm-readelf -V %t9 2>&1 | FileCheck %s --check-prefix=UNSUPPORTED-VERSION -DFILE=%t9
 
@@ -266,7 +253,7 @@ DynamicSymbols:
 ## Check we report a warning when trying to print version symbols, but SHT_GNU_verdef
 ## is invalid due to any reason.
 
-# RUN: yaml2obj %s --docnum=10 -o %t10
+# RUN: yaml2obj %s --docnum=9 -o %t10
 # RUN: llvm-readobj -V %t10 2>&1 | FileCheck %s --check-prefix=INVALID-VERDEF-LLVM -DFILE=%t10
 # RUN: llvm-readelf -V %t10 2>&1 | FileCheck %s --check-prefix=INVALID-VERDEF-GNU -DFILE=%t10
 
diff --git a/llvm/test/tools/llvm-readobj/ELF/versym-invalid.test b/llvm/test/tools/llvm-readobj/ELF/versym-invalid.test
index d495b1cfd0630..ce32644462db7 100644
--- a/llvm/test/tools/llvm-readobj/ELF/versym-invalid.test
+++ b/llvm/test/tools/llvm-readobj/ELF/versym-invalid.test
@@ -2,11 +2,11 @@
 
 ## Check that we report a warning when sh_link references a non-existent section.
 
-# RUN: yaml2obj --docnum=1 %s -o %t1
+# RUN: yaml2obj --docnum=1 -DLINK=0xFF %s -o %t1
 # RUN: llvm-readelf -V %t1 2>&1 | FileCheck -DFILE=%t1 %s --check-prefix=GNU-INVALID-LINK
 # RUN: llvm-readobj -V %t1 2>&1 | FileCheck -DFILE=%t1 %s --check-prefix=LLVM-INVALID-LINK
 
-# GNU-INVALID-LINK:      Version symbols section '.gnu.version' contains 0 entries:
+# GNU-INVALID-LINK:      Version symbols section '.gnu.version' contains 1 entries:
 # GNU-INVALID-LINK-NEXT: warning: '[[FILE]]': invalid section linked to SHT_GNU_versym section with index 1: invalid section index: 255
 # GNU-INVALID-LINK-NEXT:  Addr: 0000000000000000  Offset: 0x000040  Link: 255 (<corrupt>)
 
@@ -23,13 +23,13 @@ FileHeader:
 Sections:
   - Name:    .gnu.version
     Type:    SHT_GNU_versym
-    Link:    0xFF
-    Entries: [ ]
+    Link:    [[LINK]]
+    Entries: [ 0 ]
 
 ## Check that we report a warning when the sh_link field of a SHT_GNU_versym section does not reference
 ## a dynamic symbol table section.
 
-# RUN: yaml2obj --docnum=2 %s -o %t2
+# RUN: yaml2obj --docnum=1 -DLINK=0x0 %s -o %t2
 # RUN: llvm-readelf -V %t2 2>&1 | FileCheck -DFILE=%t2 %s --check-prefix=INVALID-SYMBOL-TABLE-GNU
 # RUN: llvm-readobj -V %t2 2>&1 | FileCheck -DFILE=%t2 %s --check-prefix=INVALID-SYMBOL-TABLE-LLVM
 
@@ -42,23 +42,11 @@ Sections:
 # INVALID-SYMBOL-TABLE-LLVM-NEXT: warning: '[[FILE]]': invalid section linked to SHT_GNU_versym section with index 1: expected SHT_DYNSYM, but got SHT_NULL
 # INVALID-SYMBOL-TABLE-LLVM-NEXT: ]
 
---- !ELF
-FileHeader:
-  Class:   ELFCLASS64
-  Data:    ELFDATA2LSB
-  Type:    ET_EXEC
-  Machine: EM_X86_64
-Sections:
-  - Name:    .gnu.version
-    Type:    SHT_GNU_versym
-    Link:    0x0
-    Entries: [ 0 ]
-
 ## Check we report a warning when something is wrong with a string table linked to a symbol table that
 ## is linked with SHT_GNU_versym. In this case we are unable to produce LLVM style output,
 ## but GNU style is fine because it does not need that string table.
 
-# RUN: yaml2obj --docnum=3 %s -o %t3
+# RUN: yaml2obj --docnum=2 %s -o %t3
 # RUN: llvm-readelf -V %t3 2>&1 | FileCheck -DFILE=%t3 %s --check-prefix=INVALID-STRING-TABLE-GNU
 # RUN: llvm-readobj -V %t3 2>&1 | FileCheck -DFILE=%t3 %s --check-prefix=INVALID-STRING-TABLE-LLVM
 
@@ -88,7 +76,7 @@ DynamicSymbols: []
 
 ## Check we report a warning when a SHT_GNU_versym section is not correctly aligned in memory.
 
-# RUN: yaml2obj --docnum=4 %s -o %t4
+# RUN: yaml2obj --docnum=3 %s -o %t4
 # RUN: llvm-readelf -V %t4 2>&1 | FileCheck -DFILE=%t4 %s --check-prefix=MISALIGNED-GNU
 # RUN: llvm-readobj -V %t4 2>&1 | FileCheck -DFILE=%t4 %s --check-prefix=MISALIGNED-LLVM
 
@@ -114,7 +102,7 @@ Sections:
 
 ## Check we report a warning when a SHT_GNU_versym section has an invalid entry size.
 
-# RUN: yaml2obj --docnum=5 %s -o %t5
+# RUN: yaml2obj --docnum=4 %s -o %t5
 # RUN: llvm-readelf -V --dyn-syms %t5 2>&1 | FileCheck -DFILE=%t5 %s --check-prefix=INVALID-ENT-SIZE-GNU
 # RUN: llvm-readobj -V --dyn-syms %t5 2>&1 | FileCheck -DFILE=%t5 %s --check-prefix=INVALID-ENT-SIZE-LLVM
 
@@ -168,7 +156,7 @@ DynamicSymbols:
 
 ## Check we report a warning when the number of version entries does not match the number of symbols in the associated symbol table.
 
-# RUN: yaml2obj --docnum=6 %s -o %t6
+# RUN: yaml2obj --docnum=5 %s -o %t6
 # RUN: llvm-readelf -V %t6 2>&1 | FileCheck -DFILE=%t6 %s --check-prefix=SYMBOLS-NUM-MISMATCH-GNU
 # RUN: llvm-readobj -V %t6 2>&1 | FileCheck -DFILE=%t6 %s --check-prefix=SYMBOLS-NUM-MISMATCH-LLVM
 
@@ -199,7 +187,7 @@ DynamicSymbols:
 ## Check we can dump a SHT_GNU_versym section when it is linked to a custom dynamic symbol
 ## table that is not called ".dynsym".
 
-# RUN: yaml2obj --docnum=7 %s -o %t7
+# RUN: yaml2obj --docnum=6 %s -o %t7
 # RUN: llvm-readelf -V %t7 2>&1 | FileCheck -DFILE=%t7 %s --check-prefix=CUSTOM-SYMTAB-GNU
 # RUN: llvm-readobj -V %t7 2>&1 | FileCheck -DFILE=%t7 %s --check-prefix=CUSTOM-SYMTAB-LLVM
 
@@ -238,7 +226,7 @@ DynamicSymbols:
 ## Version index in a SHT_GNU_versym section overflows the version map.
 ## Check we report it when trying to dump dynamic symbols.
 
-# RUN: yaml2obj %s --docnum=8 -o %t8
+# RUN: yaml2obj %s --docnum=7 -o %t8
 # RUN: llvm-readobj --dyn-syms %t8 2>&1 \
 # RUN:   | FileCheck -DFILE=%t8 --implicit-check-not=warning --check-prefix=VERSION-OVERFLOW-LLVM %s
 # RUN: llvm-readelf --dyn-syms %t8 2>&1 \

From 3de93230010fc8fbb76d5cb6bde709f4ed296a1c Mon Sep 17 00:00:00 2001
From: Georgii Rymar <grimar@accesssoftek.com>
Date: Thu, 23 Jul 2020 17:10:31 +0300
Subject: [PATCH 0898/1035] [llvm-readobj] - Don't stop dumping when the name
 of a relocation section can't be read.

This removes undesired `unwrapOrError` calls from printRelocations() methods.

Differential revision: https://reviews.llvm.org/D84408
---
 llvm/test/Object/invalid.test                 |  2 +-
 .../llvm-readobj/ELF/dependent-libraries.test |  2 +-
 .../llvm-readobj/ELF/invalid-shstrndx.test    |  4 +-
 .../tools/llvm-readobj/ELF/relocations.test   | 47 ++++++++++++++--
 ...ctions-no-section-header-string-table.test |  6 +--
 llvm/tools/llvm-readobj/ELFDumper.cpp         | 54 ++++++++-----------
 6 files changed, 75 insertions(+), 40 deletions(-)

diff --git a/llvm/test/Object/invalid.test b/llvm/test/Object/invalid.test
index bc95c5bc6e905..5c04eca1e96ba 100644
--- a/llvm/test/Object/invalid.test
+++ b/llvm/test/Object/invalid.test
@@ -343,7 +343,7 @@ FileHeader:
 # RUN: yaml2obj %s --docnum=16 -o %t16
 # RUN: llvm-readobj --sections %t16 2>&1 | FileCheck -DFILE=%t16 --check-prefix=BROKEN-SECNAME %s
 
-## BROKEN-SECNAME: warning: '[[FILE]]': a section [index 2] has an invalid sh_name (0xb) offset which goes past the end of the section name string table
+## BROKEN-SECNAME: warning: '[[FILE]]': unable to get the name of SHT_STRTAB section with index 2: a section [index 2] has an invalid sh_name (0xb) offset which goes past the end of the section name string table
 
 --- !ELF
 FileHeader:
diff --git a/llvm/test/tools/llvm-readobj/ELF/dependent-libraries.test b/llvm/test/tools/llvm-readobj/ELF/dependent-libraries.test
index 440217f3253d3..7c6759c2390fb 100644
--- a/llvm/test/tools/llvm-readobj/ELF/dependent-libraries.test
+++ b/llvm/test/tools/llvm-readobj/ELF/dependent-libraries.test
@@ -72,7 +72,7 @@ Sections:
 # MIX-GNU-NEXT:   [     0]  bar
 # MIX-GNU-NEXT:   [     4]  xxx
 # MIX-GNU-EMPTY:
-# MIX-GNU-NEXT: warning: '[[FILE]]': cannot get section name of SHT_LLVM_DEPENDENT_LIBRARIES section: a section [index 6] has an invalid sh_name (0x10000) offset which goes past the end of the section name string table
+# MIX-GNU-NEXT: warning: '[[FILE]]': unable to get the name of SHT_LLVM_DEPENDENT_LIBRARIES section with index 6: a section [index 6] has an invalid sh_name (0x10000) offset which goes past the end of the section name string table
 # MIX-GNU-NEXT: Dependent libraries section <?> at offset 0x53 contains 1 entries:
 # MIX-GNU-NEXT:   [     0]  baz
 
diff --git a/llvm/test/tools/llvm-readobj/ELF/invalid-shstrndx.test b/llvm/test/tools/llvm-readobj/ELF/invalid-shstrndx.test
index 50bdd3fc1fb66..1bb9e90b1eade 100644
--- a/llvm/test/tools/llvm-readobj/ELF/invalid-shstrndx.test
+++ b/llvm/test/tools/llvm-readobj/ELF/invalid-shstrndx.test
@@ -20,9 +20,9 @@
 # LLVM:        StringTableSectionIndex: 255
 # LLVM-NEXT: }
 # LLVM-NEXT: Sections [
-# LLVM-NEXT: warning: '[[FILE]]': section header string table index 255 does not exist
 # LLVM-NEXT: Section {
 # LLVM-NEXT:   Index: 0
+# LLVM-NEXT: warning: '[[FILE]]': unable to get the name of SHT_NULL section with index 0: section header string table index 255 does not exist
 # LLVM-NEXT:   Name: <?> (0)
 # LLVM-NEXT:   Type: SHT_NULL (0x0)
 # LLVM-NEXT:   Flags [ (0x0)
@@ -36,8 +36,10 @@
 # LLVM-NEXT:   EntrySize: 0
 # LLVM-NEXT: }
 # LLVM:      Index: 1
+# LLVM-NEXT: warning: '[[FILE]]': unable to get the name of SHT_STRTAB section with index 1: section header string table index 255 does not exist
 # LLVM-NEXT: Name: <?> (11)
 # LLVM:      Index: 2
+# LLVM-NEXT: warning: '[[FILE]]': unable to get the name of SHT_STRTAB section with index 2: section header string table index 255 does not exist
 # LLVM-NEXT: Name: <?> (1)
 
 --- !ELF
diff --git a/llvm/test/tools/llvm-readobj/ELF/relocations.test b/llvm/test/tools/llvm-readobj/ELF/relocations.test
index 5c8e1086e98ad..5dd5762fe19b2 100644
--- a/llvm/test/tools/llvm-readobj/ELF/relocations.test
+++ b/llvm/test/tools/llvm-readobj/ELF/relocations.test
@@ -125,9 +125,10 @@ Sections:
   - Name: .text
     Type: SHT_PROGBITS
     Content: '00FFFFFFFF020000001234567890ABCDEF'
-  - Name: .rel.text
-    Type: SHT_REL
-    Info: .text
+  - Name:   .rel.text
+    Type:   SHT_REL
+    Info:   .text
+    ShName: [[SHNAME=<none>]]
     Relocations:
       - Symbol: rel_0
         Type:   R_X86_64_NONE
@@ -143,6 +144,7 @@ Sections:
   - Name: .rela.text
     Type: SHT_RELA
     Info: .text
+    ShName: [[SHNAME=<none>]]
     Relocations:
       - Symbol: rela_0
         Type:   R_X86_64_NONE
@@ -369,3 +371,42 @@ Symbols:
   - Name:    rela_maxpos
     Section: .text
     Value:   0xFFFFFFFF
+
+## Check we report a warning when the name of a relocation section can't be read. Check we continue dumping.
+# RUN: yaml2obj %s --docnum=1 -DSHNAME=0xffffffff -o %tshname
+# RUN: llvm-readobj --relocs %tshname 2>&1 | FileCheck %s -DFILE=%tshname --check-prefix=LLVM-SHNAME
+# RUN: llvm-readelf --relocs %tshname 2>&1 | FileCheck %s -DFILE=%tshname --check-prefix=GNU-SHNAME
+
+# LLVM-SHNAME:      Relocations [
+# LLVM-SHNAME-NEXT: warning: '[[FILE]]': unable to get the name of SHT_REL section with index 2: a section [index 2] has an invalid sh_name (0xffffffff) offset which goes past the end of the section name string table
+# LLVM-SHNAME-NEXT:   Section (2) <?> {
+# LLVM-SHNAME-NEXT:     0x0 R_X86_64_NONE rel_0 0x0
+# LLVM-SHNAME-NEXT:     0x1 R_X86_64_PC32 rel_neg 0x0
+# LLVM-SHNAME-NEXT:     0x5 R_X86_64_PLT32 rel_pos 0x0
+# LLVM-SHNAME-NEXT:     0x9 R_X86_64_64 rel_64 0x0
+# LLVM-SHNAME-NEXT:   }
+# LLVM-SHNAME-NEXT: warning: '[[FILE]]': unable to get the name of SHT_RELA section with index 3: a section [index 3] has an invalid sh_name (0xffffffff) offset which goes past the end of the section name string table
+# LLVM-SHNAME-NEXT:   Section (3) <?> {
+# LLVM-SHNAME-NEXT:     0x0 R_X86_64_NONE rela_0 0x0
+# LLVM-SHNAME-NEXT:     0x1 R_X86_64_PC32 rela_neg 0xFFFFFFFFFFFFFFFF
+# LLVM-SHNAME-NEXT:     0x5 R_X86_64_PLT32 rela_pos 0x2
+# LLVM-SHNAME-NEXT:     0xFFFFFFFFFFFFFFFF R_X86_64_64 rela_minneg 0x8000000000000000
+# LLVM-SHNAME-NEXT:     0x9 R_X86_64_32S rela_maxpos 0x7FFFFFFFFFFFFFFF
+# LLVM-SHNAME-NEXT:   }
+# LLVM-SHNAME-NEXT: ]
+
+# GNU-SHNAME:      warning: '[[FILE]]': unable to get the name of SHT_REL section with index 2: a section [index 2] has an invalid sh_name (0xffffffff) offset which goes past the end of the section name string table
+# GNU-SHNAME:      Relocation section '<?>' at offset 0x51 contains 4 entries:
+# GNU-SHNAME-NEXT:     Offset             Info             Type               Symbol's Value  Symbol's Name
+# GNU-SHNAME-NEXT: 0000000000000000  0000000100000000 R_X86_64_NONE          0000000000000000 rel_0
+# GNU-SHNAME-NEXT: 0000000000000001  0000000200000002 R_X86_64_PC32          0000000000000001 rel_neg
+# GNU-SHNAME-NEXT: 0000000000000005  0000000300000004 R_X86_64_PLT32         0000000000000002 rel_pos
+# GNU-SHNAME-NEXT: 0000000000000009  0000000400000001 R_X86_64_64            ffffffffffffffff rel_64
+# GNU-SHNAME:      warning: '[[FILE]]': unable to get the name of SHT_RELA section with index 3: a section [index 3] has an invalid sh_name (0xffffffff) offset which goes past the end of the section name string table
+# GNU-SHNAME:      Relocation section '<?>' at offset 0x91 contains 5 entries:
+# GNU-SHNAME-NEXT:     Offset             Info             Type       Symbol's Value  Symbol's Name + Addend
+# GNU-SHNAME-NEXT: 0000000000000000  0000000500000000 R_X86_64_NONE  0000000000000000 rela_0 + 0
+# GNU-SHNAME-NEXT: 0000000000000001  0000000600000002 R_X86_64_PC32  0000000000000001 rela_neg - 1
+# GNU-SHNAME-NEXT: 0000000000000005  0000000700000004 R_X86_64_PLT32 0000000000000002 rela_pos + 2
+# GNU-SHNAME-NEXT: ffffffffffffffff  0000000800000001 R_X86_64_64    0000000000000003 rela_minneg - 8000000000000000
+# GNU-SHNAME-NEXT: 0000000000000009  000000090000000b R_X86_64_32S   ffffffffffffffff rela_maxpos + 7fffffffffffffff
diff --git a/llvm/test/tools/llvm-readobj/ELF/sections-no-section-header-string-table.test b/llvm/test/tools/llvm-readobj/ELF/sections-no-section-header-string-table.test
index 95a72a4d05136..a708af795c819 100644
--- a/llvm/test/tools/llvm-readobj/ELF/sections-no-section-header-string-table.test
+++ b/llvm/test/tools/llvm-readobj/ELF/sections-no-section-header-string-table.test
@@ -43,9 +43,9 @@
 # LLVM-NEXT:     AddressAlignment: 0
 # LLVM-NEXT:     EntrySize: 0
 # LLVM-NEXT:   }
-# LLVM-NEXT: warning: '[[FILE]]': a section [index 2] has an invalid sh_name (0x1) offset which goes past the end of the section name string table
 # LLVM-NEXT:  Section {
 # LLVM-NEXT:    Index: 2
+# LLVM-NEXT: warning: '[[FILE]]': unable to get the name of SHT_PROGBITS section with index 2: a section [index 2] has an invalid sh_name (0x1) offset which goes past the end of the section name string table
 # LLVM-NEXT:    Name: <?> (1)
 # LLVM-NEXT:    Type: SHT_PROGBITS (0x1)
 # LLVM-NEXT:    Flags [ (0x0)
@@ -58,9 +58,9 @@
 # LLVM-NEXT:    AddressAlignment: 0
 # LLVM-NEXT:    EntrySize: 0
 # LLVM-NEXT:  }
-# LLVM-NEXT: warning: '[[FILE]]': a section [index 3] has an invalid sh_name (0x15) offset which goes past the end of the section name string table
 # LLVM-NEXT:   Section {
 # LLVM-NEXT:     Index: 3
+# LLVM-NEXT: warning: '[[FILE]]': unable to get the name of SHT_STRTAB section with index 3: a section [index 3] has an invalid sh_name (0x15) offset which goes past the end of the section name string table
 # LLVM-NEXT:     Name: <?> (21)
 # LLVM-NEXT:     Type: SHT_STRTAB (0x3)
 # LLVM-NEXT:     Flags [ (0x0)
@@ -73,9 +73,9 @@
 # LLVM-NEXT:     AddressAlignment: 1
 # LLVM-NEXT:     EntrySize: 0
 # LLVM-NEXT:   }
-# LLVM-NEXT: warning: '[[FILE]]': a section [index 4] has an invalid sh_name (0xb) offset which goes past the end of the section name string table
 # LLVM-NEXT:   Section {
 # LLVM-NEXT:     Index: 4
+# LLVM-NEXT: warning: '[[FILE]]': unable to get the name of SHT_STRTAB section with index 4: a section [index 4] has an invalid sh_name (0xb) offset which goes past the end of the section name string table
 # LLVM-NEXT:     Name: <?> (11)
 # LLVM-NEXT:     Type: SHT_STRTAB (0x3)
 # LLVM-NEXT:     Flags [ (0x0)
diff --git a/llvm/tools/llvm-readobj/ELFDumper.cpp b/llvm/tools/llvm-readobj/ELFDumper.cpp
index 35b5e2637b4d4..2edca5aaa0b9d 100644
--- a/llvm/tools/llvm-readobj/ELFDumper.cpp
+++ b/llvm/tools/llvm-readobj/ELFDumper.cpp
@@ -778,6 +778,9 @@ template <typename ELFT> class DumpStyle {
   virtual void printRelrReloc(const Elf_Relr &R) = 0;
   void printRelocationsHelper(const ELFFile<ELFT> *Obj, const Elf_Shdr &Sec);
 
+  StringRef getPrintableSectionName(const ELFFile<ELFT> *Obj,
+                                    const Elf_Shdr &Sec) const;
+
   void reportUniqueWarning(Error Err) const;
   StringRef FileName;
 
@@ -3732,7 +3735,6 @@ template <class ELFT> void GNUStyle<ELFT>::printRelocations(const ELFO *Obj) {
       continue;
     HasRelocSections = true;
 
-    StringRef Name = unwrapOrError(this->FileName, Obj->getSectionName(&Sec));
     unsigned Entries;
     // Android's packed relocation section needs to be unpacked first
     // to get the actual number of entries.
@@ -3748,6 +3750,7 @@ template <class ELFT> void GNUStyle<ELFT>::printRelocations(const ELFO *Obj) {
     }
 
     uintX_t Offset = Sec.sh_offset;
+    StringRef Name = this->getPrintableSectionName(Obj, Sec);
     OS << "\nRelocation section '" << Name << "' at offset 0x"
        << to_hexString(Offset, false) << " contains " << Entries
        << " entries:\n";
@@ -3877,17 +3880,8 @@ void GNUStyle<ELFT>::printSymtabMessage(const ELFO *Obj, const Elf_Shdr *Symtab,
                                         size_t Entries,
                                         bool NonVisibilityBitsUsed) {
   StringRef Name;
-  if (Symtab) {
-    if (Expected<StringRef> NameOrErr = Obj->getSectionName(Symtab)) {
-      Name = *NameOrErr;
-    } else {
-      this->reportUniqueWarning(createError("unable to get the name of " +
-                                            describe(Obj, *Symtab) + ": " +
-                                            toString(NameOrErr.takeError())));
-      Name = "<?>";
-    }
-  }
-
+  if (Symtab)
+    Name = this->getPrintableSectionName(Obj, *Symtab);
   if (!Name.empty())
     OS << "\nSymbol table '" << Name << "'";
   else
@@ -5519,6 +5513,20 @@ void DumpStyle<ELFT>::printRelocationsHelper(const ELFFile<ELFT> *Obj,
   }
 }
 
+template <class ELFT>
+StringRef DumpStyle<ELFT>::getPrintableSectionName(const ELFFile<ELFT> *Obj,
+                                                   const Elf_Shdr &Sec) const {
+  StringRef Name = "<?>";
+  if (Expected<StringRef> SecNameOrErr =
+          Obj->getSectionName(&Sec, this->dumper()->WarningHandler))
+    Name = *SecNameOrErr;
+  else
+    this->reportUniqueWarning(createError("unable to get the name of " +
+                                          describe(Obj, Sec) + ": " +
+                                          toString(SecNameOrErr.takeError())));
+  return Name;
+}
+
 template <class ELFT>
 void GNUStyle<ELFT>::printDependentLibs(const ELFFile<ELFT> *Obj) {
   bool SectionStarted = false;
@@ -5544,16 +5552,7 @@ void GNUStyle<ELFT>::printDependentLibs(const ELFFile<ELFT> *Obj) {
       PrintSection();
     SectionStarted = true;
     Current.Offset = Shdr.sh_offset;
-    Expected<StringRef> Name = Obj->getSectionName(&Shdr);
-    if (!Name) {
-      Current.Name = "<?>";
-      this->reportUniqueWarning(
-          createError("cannot get section name of "
-                      "SHT_LLVM_DEPENDENT_LIBRARIES section: " +
-                      toString(Name.takeError())));
-    } else {
-      Current.Name = *Name;
-    }
+    Current.Name = this->getPrintableSectionName(Obj, Shdr);
   };
   auto OnLibEntry = [&](StringRef Lib, uint64_t Offset) {
     SecEntries.push_back(NameOffset{Lib, Offset});
@@ -6135,7 +6134,7 @@ template <class ELFT> void LLVMStyle<ELFT>::printRelocations(const ELFO *Obj) {
     if (!isRelocationSec<ELFT>(Sec))
       continue;
 
-    StringRef Name = unwrapOrError(this->FileName, Obj->getSectionName(&Sec));
+    StringRef Name = this->getPrintableSectionName(Obj, Sec);
     unsigned SecNdx = &Sec - &cantFail(Obj->sections()).front();
     W.startLine() << "Section (" << SecNdx << ") " << Name << " {\n";
     W.indent();
@@ -6205,16 +6204,9 @@ void LLVMStyle<ELFT>::printSectionHeaders(const ELFO *Obj) {
   std::vector<EnumEntry<unsigned>> FlagsList =
       getSectionFlagsForTarget(Obj->getHeader()->e_machine);
   for (const Elf_Shdr &Sec : cantFail(Obj->sections())) {
-    StringRef Name = "<?>";
-    if (Expected<StringRef> SecNameOrErr =
-            Obj->getSectionName(&Sec, this->dumper()->WarningHandler))
-      Name = *SecNameOrErr;
-    else
-      this->reportUniqueWarning(SecNameOrErr.takeError());
-
     DictScope SectionD(W, "Section");
     W.printNumber("Index", ++SectionIndex);
-    W.printNumber("Name", Name, Sec.sh_name);
+    W.printNumber("Name", this->getPrintableSectionName(Obj, Sec), Sec.sh_name);
     W.printHex(
         "Type",
         object::getELFSectionTypeName(Obj->getHeader()->e_machine, Sec.sh_type),

From 76c3ec814dec8eef020490b5d3a640d873b5918b Mon Sep 17 00:00:00 2001
From: Kadir Cetinkaya <kadircet@google.com>
Date: Sat, 1 Aug 2020 19:03:40 +0200
Subject: [PATCH 0899/1035] [clang][Tooling] Optimize addTargetAndMode in case
 of invalid modes

This skips searching for `target` related flags in the existing args if
we don't have a valid target to insert.

Depends on D85076

Differential Revision: https://reviews.llvm.org/D85077
---
 clang/lib/Tooling/Tooling.cpp | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/clang/lib/Tooling/Tooling.cpp b/clang/lib/Tooling/Tooling.cpp
index 0593f0cc1d195..1ee8ce28c2efa 100644
--- a/clang/lib/Tooling/Tooling.cpp
+++ b/clang/lib/Tooling/Tooling.cpp
@@ -258,22 +258,23 @@ void addTargetAndModeForProgramName(std::vector<std::string> &CommandLine,
   // --driver-mode=X
   const std::string DriverModeOPT =
       Table.getOption(driver::options::OPT_driver_mode).getPrefixedName();
-  bool AlreadyHasTarget = false;
-  bool AlreadyHasMode = false;
+  auto TargetMode =
+      driver::ToolChain::getTargetAndModeFromProgramName(InvokedAs);
+  // No need to search for target args if we don't have a target/mode to insert.
+  bool ShouldAddTarget = TargetMode.TargetIsValid;
+  bool ShouldAddMode = TargetMode.DriverMode != nullptr;
   // Skip CommandLine[0].
   for (auto Token = ++CommandLine.begin(); Token != CommandLine.end();
        ++Token) {
     StringRef TokenRef(*Token);
-    AlreadyHasTarget |=
-        TokenRef.startswith(TargetOPT) || TokenRef.equals(TargetOPTLegacy);
-    AlreadyHasMode |= TokenRef.startswith(DriverModeOPT);
+    ShouldAddTarget = ShouldAddTarget && !TokenRef.startswith(TargetOPT) &&
+                      !TokenRef.equals(TargetOPTLegacy);
+    ShouldAddMode = ShouldAddMode && !TokenRef.startswith(DriverModeOPT);
   }
-  auto TargetMode =
-      driver::ToolChain::getTargetAndModeFromProgramName(InvokedAs);
-  if (!AlreadyHasMode && TargetMode.DriverMode) {
+  if (ShouldAddMode) {
     CommandLine.insert(++CommandLine.begin(), TargetMode.DriverMode);
   }
-  if (!AlreadyHasTarget && TargetMode.TargetIsValid) {
+  if (ShouldAddTarget) {
     CommandLine.insert(++CommandLine.begin(),
                        TargetOPT + TargetMode.TargetPrefix);
   }

From 86e1b73507f3738f10eefb580d7c5e9adf17c6c0 Mon Sep 17 00:00:00 2001
From: Denys Petrov <dpetrov@accesssoftek.com>
Date: Fri, 31 Jul 2020 18:57:04 +0300
Subject: [PATCH 0900/1035] [analyzer] Simplify function
 SVal::getAsSymbolicExpression and similar ones

Summary: Simplify functions SVal::getAsSymbolicExpression SVal::getAsSymExpr and SVal::getAsSymbol. After revision I concluded that `getAsSymbolicExpression` and `getAsSymExpr` repeat functionality of `getAsSymbol`, thus them can be removed.

Fix: Remove functions SVal::getAsSymbolicExpression and SVal::getAsSymExpr.

Differential Revision: https://reviews.llvm.org/D85034
---
 .../StaticAnalyzer/Core/PathSensitive/SVals.h  |  6 ------
 .../Checkers/CheckObjCDealloc.cpp              |  2 +-
 .../Checkers/MacOSKeychainAPIChecker.cpp       |  2 +-
 .../RetainCountDiagnostics.cpp                 |  2 +-
 clang/lib/StaticAnalyzer/Checkers/Taint.cpp    |  2 +-
 clang/lib/StaticAnalyzer/Core/ProgramState.cpp |  3 ---
 clang/lib/StaticAnalyzer/Core/SValBuilder.cpp  |  6 +++---
 clang/lib/StaticAnalyzer/Core/SVals.cpp        | 18 ------------------
 .../Core/SimpleConstraintManager.cpp           |  4 ++--
 .../StaticAnalyzer/Core/SimpleSValBuilder.cpp  |  2 +-
 10 files changed, 10 insertions(+), 37 deletions(-)

diff --git a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/SVals.h b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/SVals.h
index 1abe297820886..a640d815a5ce4 100644
--- a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/SVals.h
+++ b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/SVals.h
@@ -182,12 +182,6 @@ class SVal {
   /// should continue to the base regions if the region is not symbolic.
   SymbolRef getAsSymbol(bool IncludeBaseRegions = false) const;
 
-  /// getAsSymbolicExpression - If this Sval wraps a symbolic expression then
-  ///  return that expression.  Otherwise return NULL.
-  const SymExpr *getAsSymbolicExpression() const;
-
-  const SymExpr *getAsSymExpr() const;
-
   const MemRegion *getAsRegion() const;
 
   /// printJson - Pretty-prints in JSON format.
diff --git a/clang/lib/StaticAnalyzer/Checkers/CheckObjCDealloc.cpp b/clang/lib/StaticAnalyzer/Checkers/CheckObjCDealloc.cpp
index 13836f08a61ef..78b3c209ad6bc 100644
--- a/clang/lib/StaticAnalyzer/Checkers/CheckObjCDealloc.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/CheckObjCDealloc.cpp
@@ -406,7 +406,7 @@ ProgramStateRef ObjCDeallocChecker::evalAssume(ProgramStateRef State, SVal Cond,
   if (State->get<UnreleasedIvarMap>().isEmpty())
     return State;
 
-  auto *CondBSE = dyn_cast_or_null<BinarySymExpr>(Cond.getAsSymExpr());
+  auto *CondBSE = dyn_cast_or_null<BinarySymExpr>(Cond.getAsSymbol());
   if (!CondBSE)
     return State;
 
diff --git a/clang/lib/StaticAnalyzer/Checkers/MacOSKeychainAPIChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/MacOSKeychainAPIChecker.cpp
index 87477e96d2d16..a157ee2da5df4 100644
--- a/clang/lib/StaticAnalyzer/Checkers/MacOSKeychainAPIChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/MacOSKeychainAPIChecker.cpp
@@ -509,7 +509,7 @@ ProgramStateRef MacOSKeychainAPIChecker::evalAssume(ProgramStateRef State,
   if (AMap.isEmpty())
     return State;
 
-  auto *CondBSE = dyn_cast_or_null<BinarySymExpr>(Cond.getAsSymExpr());
+  auto *CondBSE = dyn_cast_or_null<BinarySymExpr>(Cond.getAsSymbol());
   if (!CondBSE)
     return State;
   BinaryOperator::Opcode OpCode = CondBSE->getOpcode();
diff --git a/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountDiagnostics.cpp b/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountDiagnostics.cpp
index 1d8ed90f7590c..854646a8779d7 100644
--- a/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountDiagnostics.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountDiagnostics.cpp
@@ -177,7 +177,7 @@ static Optional<unsigned> findArgIdxOfSymbol(ProgramStateRef CurrSt,
   for (unsigned Idx = 0; Idx < (*CE)->getNumArgs(); Idx++)
     if (const MemRegion *MR = (*CE)->getArgSVal(Idx).getAsRegion())
       if (const auto *TR = dyn_cast<TypedValueRegion>(MR))
-        if (CurrSt->getSVal(MR, TR->getValueType()).getAsSymExpr() == Sym)
+        if (CurrSt->getSVal(MR, TR->getValueType()).getAsSymbol() == Sym)
           return Idx;
 
   return None;
diff --git a/clang/lib/StaticAnalyzer/Checkers/Taint.cpp b/clang/lib/StaticAnalyzer/Checkers/Taint.cpp
index 5b46ffb656cf8..71b2ab834a07a 100644
--- a/clang/lib/StaticAnalyzer/Checkers/Taint.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/Taint.cpp
@@ -148,7 +148,7 @@ bool taint::isTainted(ProgramStateRef State, const Stmt *S,
 }
 
 bool taint::isTainted(ProgramStateRef State, SVal V, TaintTagType Kind) {
-  if (const SymExpr *Sym = V.getAsSymExpr())
+  if (SymbolRef Sym = V.getAsSymbol())
     return isTainted(State, Sym, Kind);
   if (const MemRegion *Reg = V.getAsRegion())
     return isTainted(State, Reg, Kind);
diff --git a/clang/lib/StaticAnalyzer/Core/ProgramState.cpp b/clang/lib/StaticAnalyzer/Core/ProgramState.cpp
index 006a4006b7fc9..1ccb0de92fba3 100644
--- a/clang/lib/StaticAnalyzer/Core/ProgramState.cpp
+++ b/clang/lib/StaticAnalyzer/Core/ProgramState.cpp
@@ -582,9 +582,6 @@ bool ScanReachableSymbols::scan(SVal val) {
   if (SymbolRef Sym = val.getAsSymbol())
     return scan(Sym);
 
-  if (const SymExpr *Sym = val.getAsSymbolicExpression())
-    return scan(Sym);
-
   if (Optional<nonloc::CompoundVal> X = val.getAs<nonloc::CompoundVal>())
     return scan(*X);
 
diff --git a/clang/lib/StaticAnalyzer/Core/SValBuilder.cpp b/clang/lib/StaticAnalyzer/Core/SValBuilder.cpp
index c00a2c8ba8a2c..5b6b6973b310c 100644
--- a/clang/lib/StaticAnalyzer/Core/SValBuilder.cpp
+++ b/clang/lib/StaticAnalyzer/Core/SValBuilder.cpp
@@ -377,8 +377,8 @@ Optional<SVal> SValBuilder::getConstantVal(const Expr *E) {
 SVal SValBuilder::makeSymExprValNN(BinaryOperator::Opcode Op,
                                    NonLoc LHS, NonLoc RHS,
                                    QualType ResultTy) {
-  const SymExpr *symLHS = LHS.getAsSymExpr();
-  const SymExpr *symRHS = RHS.getAsSymExpr();
+  SymbolRef symLHS = LHS.getAsSymbol();
+  SymbolRef symRHS = RHS.getAsSymbol();
 
   // TODO: When the Max Complexity is reached, we should conjure a symbol
   // instead of generating an Unknown value and propagate the taint info to it.
@@ -492,7 +492,7 @@ SVal SValBuilder::evalIntegralCast(ProgramStateRef state, SVal val,
   if (getContext().getTypeSize(castTy) >= getContext().getTypeSize(originalTy))
     return evalCast(val, castTy, originalTy);
 
-  const SymExpr *se = val.getAsSymbolicExpression();
+  SymbolRef se = val.getAsSymbol();
   if (!se) // Let evalCast handle non symbolic expressions.
     return evalCast(val, castTy, originalTy);
 
diff --git a/clang/lib/StaticAnalyzer/Core/SVals.cpp b/clang/lib/StaticAnalyzer/Core/SVals.cpp
index 9b5de6c3eb92b..465800fa67fce 100644
--- a/clang/lib/StaticAnalyzer/Core/SVals.cpp
+++ b/clang/lib/StaticAnalyzer/Core/SVals.cpp
@@ -116,8 +116,6 @@ SymbolRef SVal::getLocSymbolInBase() const {
   return nullptr;
 }
 
-// TODO: The next 3 functions have to be simplified.
-
 /// If this SVal wraps a symbol return that SymbolRef.
 /// Otherwise, return 0.
 ///
@@ -132,22 +130,6 @@ SymbolRef SVal::getAsSymbol(bool IncludeBaseRegions) const {
   return getAsLocSymbol(IncludeBaseRegions);
 }
 
-/// getAsSymbolicExpression - If this Sval wraps a symbolic expression then
-///  return that expression.  Otherwise return NULL.
-const SymExpr *SVal::getAsSymbolicExpression() const {
-  if (Optional<nonloc::SymbolVal> X = getAs<nonloc::SymbolVal>())
-    return X->getSymbol();
-
-  return getAsSymbol();
-}
-
-const SymExpr* SVal::getAsSymExpr() const {
-  const SymExpr* Sym = getAsSymbol();
-  if (!Sym)
-    Sym = getAsSymbolicExpression();
-  return Sym;
-}
-
 const MemRegion *SVal::getAsRegion() const {
   if (Optional<loc::MemRegionVal> X = getAs<loc::MemRegionVal>())
     return X->getRegion();
diff --git a/clang/lib/StaticAnalyzer/Core/SimpleConstraintManager.cpp b/clang/lib/StaticAnalyzer/Core/SimpleConstraintManager.cpp
index 3709106ad44ce..f96974f97dcc5 100644
--- a/clang/lib/StaticAnalyzer/Core/SimpleConstraintManager.cpp
+++ b/clang/lib/StaticAnalyzer/Core/SimpleConstraintManager.cpp
@@ -57,7 +57,7 @@ ProgramStateRef SimpleConstraintManager::assumeAux(ProgramStateRef State,
   // SymIntExprs.
   if (!canReasonAbout(Cond)) {
     // Just add the constraint to the expression without trying to simplify.
-    SymbolRef Sym = Cond.getAsSymExpr();
+    SymbolRef Sym = Cond.getAsSymbol();
     assert(Sym);
     return assumeSymUnsupported(State, Sym, Assumption);
   }
@@ -101,7 +101,7 @@ ProgramStateRef SimpleConstraintManager::assumeInclusiveRange(
 
   if (!canReasonAbout(Value)) {
     // Just add the constraint to the expression without trying to simplify.
-    SymbolRef Sym = Value.getAsSymExpr();
+    SymbolRef Sym = Value.getAsSymbol();
     assert(Sym);
     return assumeSymInclusiveRange(State, Sym, From, To, InRange);
   }
diff --git a/clang/lib/StaticAnalyzer/Core/SimpleSValBuilder.cpp b/clang/lib/StaticAnalyzer/Core/SimpleSValBuilder.cpp
index 2e269f6a596e8..a64ed78ac3458 100644
--- a/clang/lib/StaticAnalyzer/Core/SimpleSValBuilder.cpp
+++ b/clang/lib/StaticAnalyzer/Core/SimpleSValBuilder.cpp
@@ -86,7 +86,7 @@ SVal SimpleSValBuilder::evalCastFromNonLoc(NonLoc val, QualType castTy) {
     return makeLocAsInteger(LI->getLoc(), castSize);
   }
 
-  if (const SymExpr *se = val.getAsSymbolicExpression()) {
+  if (SymbolRef se = val.getAsSymbol()) {
     QualType T = Context.getCanonicalType(se->getType());
     // If types are the same or both are integers, ignore the cast.
     // FIXME: Remove this hack when we support symbolic truncation/extension.

From 21fa82d5c63c30c745d5181889329084ac6d2767 Mon Sep 17 00:00:00 2001
From: Denys Petrov <dpetrov@accesssoftek.com>
Date: Fri, 31 Jul 2020 15:54:46 +0300
Subject: [PATCH 0901/1035] [analyzer] Introduce minor refactoring of
 SVal::getSubKind function

Summary: `BaseMask` occupies the lowest bits. Effect of applying the mask is neutralized by right shift operation, thus making it useless.

Fix: Remove a redundant bitwise operation.

Differential Revision: https://reviews.llvm.org/D85026
---
 clang/include/clang/StaticAnalyzer/Core/PathSensitive/SVals.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/SVals.h b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/SVals.h
index a640d815a5ce4..a561ac67bf786 100644
--- a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/SVals.h
+++ b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/SVals.h
@@ -80,7 +80,7 @@ class SVal {
 #define ABSTRACT_SVAL_WITH_KIND(Id, Parent) Id ## Kind,
 #include "clang/StaticAnalyzer/Core/PathSensitive/SVals.def"
   };
-  enum { BaseBits = 2, BaseMask = 0x3 };
+  enum { BaseBits = 2, BaseMask = 0b11 };
 
 protected:
   const void *Data = nullptr;
@@ -116,7 +116,7 @@ class SVal {
 
   unsigned getRawKind() const { return Kind; }
   BaseKind getBaseKind() const { return (BaseKind) (Kind & BaseMask); }
-  unsigned getSubKind() const { return (Kind & ~BaseMask) >> BaseBits; }
+  unsigned getSubKind() const { return Kind >> BaseBits; }
 
   // This method is required for using SVal in a FoldingSetNode.  It
   // extracts a unique signature for this SVal object.

From 5191f70ab1f4b0b9225b2e9e11584e199172418c Mon Sep 17 00:00:00 2001
From: Haojian Wu <hokein.wu@gmail.com>
Date: Sat, 25 Jul 2020 21:52:33 +0200
Subject: [PATCH 0902/1035] [clangd] Support new/deleta operator in
 TargetFinder.

Differential Revision: https://reviews.llvm.org/D85028
---
 clang-tools-extra/clangd/FindTarget.cpp       |  6 ++++
 clang-tools-extra/clangd/XRefs.cpp            |  4 +++
 .../clangd/unittests/FindTargetTests.cpp      | 31 +++++++++++++++++++
 3 files changed, 41 insertions(+)

diff --git a/clang-tools-extra/clangd/FindTarget.cpp b/clang-tools-extra/clangd/FindTarget.cpp
index a346d6b662e9a..e4d2dddb4b5d3 100644
--- a/clang-tools-extra/clangd/FindTarget.cpp
+++ b/clang-tools-extra/clangd/FindTarget.cpp
@@ -460,6 +460,12 @@ struct TargetFinder {
       void VisitPseudoObjectExpr(const PseudoObjectExpr *POE) {
         Outer.add(POE->getSyntacticForm(), Flags);
       }
+      void VisitCXXNewExpr(const CXXNewExpr *CNE) {
+        Outer.add(CNE->getOperatorNew(), Flags);
+      }
+      void VisitCXXDeleteExpr(const CXXDeleteExpr *CDE) {
+        Outer.add(CDE->getOperatorDelete(), Flags);
+      }
     };
     Visitor(*this, Flags).Visit(S);
   }
diff --git a/clang-tools-extra/clangd/XRefs.cpp b/clang-tools-extra/clangd/XRefs.cpp
index 1fc89f3e08472..26653aa409d7d 100644
--- a/clang-tools-extra/clangd/XRefs.cpp
+++ b/clang-tools-extra/clangd/XRefs.cpp
@@ -238,6 +238,10 @@ locateASTReferent(SourceLocation CurLoc, const syntax::Token *TouchedIdentifier,
   llvm::DenseMap<SymbolID, size_t> ResultIndex;
 
   auto AddResultDecl = [&](const NamedDecl *D) {
+    // FIXME: Canonical declarations of some symbols might refer to built-in
+    // decls with possibly-invalid source locations (e.g. global new operator).
+    // In such cases we should pick up a redecl with valid source location
+    // instead of failing.
     D = llvm::cast<NamedDecl>(D->getCanonicalDecl());
     auto Loc =
         makeLocation(AST.getASTContext(), nameLocation(*D, SM), MainFilePath);
diff --git a/clang-tools-extra/clangd/unittests/FindTargetTests.cpp b/clang-tools-extra/clangd/unittests/FindTargetTests.cpp
index 92095e871e201..8b872d6314d45 100644
--- a/clang-tools-extra/clangd/unittests/FindTargetTests.cpp
+++ b/clang-tools-extra/clangd/unittests/FindTargetTests.cpp
@@ -535,6 +535,7 @@ TEST_F(TargetDeclTest, OverloadExpr) {
   // FIXME: Auto-completion in a template requires disabling delayed template
   // parsing.
   Flags = {"-fno-delayed-template-parsing"};
+  Flags.push_back("--target=x86_64-pc-linux-gnu");
 
   Code = R"cpp(
     void func(int*);
@@ -559,6 +560,36 @@ TEST_F(TargetDeclTest, OverloadExpr) {
     };
   )cpp";
   EXPECT_DECLS("UnresolvedMemberExpr", "void func(int *)", "void func(char *)");
+
+  Code = R"cpp(
+    struct X {
+      static void *operator new(unsigned long);
+    };
+    auto* k = [[new]] X();
+  )cpp";
+  EXPECT_DECLS("CXXNewExpr", "static void *operator new(unsigned long)");
+  Code = R"cpp(
+    void *operator new(unsigned long);
+    auto* k = [[new]] int();
+  )cpp";
+  EXPECT_DECLS("CXXNewExpr", "void *operator new(unsigned long)");
+
+  Code = R"cpp(
+    struct X {
+      static void operator delete(void *) noexcept;
+    };
+    void k(X* x) {
+      [[delete]] x;
+    }
+  )cpp";
+  EXPECT_DECLS("CXXDeleteExpr", "static void operator delete(void *) noexcept");
+  Code = R"cpp(
+    void operator delete(void *) noexcept;
+    void k(int* x) {
+      [[delete]] x;
+    }
+  )cpp";
+  EXPECT_DECLS("CXXDeleteExpr", "void operator delete(void *) noexcept");
 }
 
 TEST_F(TargetDeclTest, DependentExprs) {

From 18279a54b5d3382874924d6a3c7775b7e22598dc Mon Sep 17 00:00:00 2001
From: Nicholas Guy <nicholas.guy@arm.com>
Date: Wed, 1 Jul 2020 11:35:58 +0100
Subject: [PATCH 0903/1035] [ARM] Fix IT block generation after
 Thumb2SizeReduce with -Oz

Fixes a regression caused by D82439, in which IT blocks were no longer being
generated when -Oz is present. This was due to the CPSR register being marked as
dead, while this case was not accounted for.

Differential Revision: https://reviews.llvm.org/D83667
---
 llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp      |  7 +++++
 llvm/test/CodeGen/Thumb2/constant-hoisting.ll | 27 +++++++++----------
 2 files changed, 20 insertions(+), 14 deletions(-)

diff --git a/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp b/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
index b501dc06ca72f..0353cfd3d86f7 100644
--- a/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
@@ -587,6 +587,13 @@ bool ARMBaseInstrInfo::DefinesPredicate(
     const MachineOperand &MO = MI.getOperand(i);
     if ((MO.isRegMask() && MO.clobbersPhysReg(ARM::CPSR)) ||
         (MO.isReg() && MO.isDef() && MO.getReg() == ARM::CPSR)) {
+
+      // Filter out T1 instructions that have a dead CPSR,
+      // allowing IT blocks to be generated containing T1 instructions
+      const MCInstrDesc &MCID = MI.getDesc();
+      if (MCID.TSFlags & ARMII::ThumbArithFlagSetting && MO.isDead())
+        continue;
+
       Pred.push_back(MO);
       Found = true;
     }
diff --git a/llvm/test/CodeGen/Thumb2/constant-hoisting.ll b/llvm/test/CodeGen/Thumb2/constant-hoisting.ll
index 5c8f934ce61d3..a106900dc3e99 100644
--- a/llvm/test/CodeGen/Thumb2/constant-hoisting.ll
+++ b/llvm/test/CodeGen/Thumb2/constant-hoisting.ll
@@ -37,26 +37,25 @@ define i32 @test_values(i32 %a, i32 %b) minsize optsize {
 ; CHECK-V7M:         mov r2, r0
 ; CHECK-V7M-NEXT:    ldr r0, .LCPI0_0
 ; CHECK-V7M-NEXT:    cmp r2, #50
-; CHECK-V7M-NEXT:    beq .LBB0_5
+; CHECK-V7M-NEXT:    beq .LBB0_3
 ; CHECK-V7M-NEXT:    cmp r2, #1
-; CHECK-V7M-NEXT:    beq .LBB0_7
+; CHECK-V7M-NEXT:    ittt eq
+; CHECK-V7M-NEXT:    addeq r0, r1
+; CHECK-V7M-NEXT:    addeq r0, #1
+; CHECK-V7M-NEXT:    bxeq lr
 ; CHECK-V7M-NEXT:    cmp r2, #30
-; CHECK-V7M-NEXT:    beq .LBB0_8
-; CHECK-V7M-NEXT:    cbnz r2, .LBB0_6
+; CHECK-V7M-NEXT:    ittt eq
+; CHECK-V7M-NEXT:    addeq r0, r1
+; CHECK-V7M-NEXT:    addeq r0, #2
+; CHECK-V7M-NEXT:    bxeq lr
+; CHECK-V7M-NEXT:    cbnz r2, .LBB0_4
+; CHECK-V7M-NEXT:  .LBB0_2:
 ; CHECK-V7M-NEXT:    add r0, r1
 ; CHECK-V7M-NEXT:    bx lr
-; CHECK-V7M-NEXT:  .LBB0_5:
+; CHECK-V7M-NEXT:  .LBB0_3:
 ; CHECK-V7M-NEXT:    add r0, r1
 ; CHECK-V7M-NEXT:    adds r0, #4
-; CHECK-V7M-NEXT:  .LBB0_6:
-; CHECK-V7M-NEXT:    bx lr
-; CHECK-V7M-NEXT:  .LBB0_7:
-; CHECK-V7M-NEXT:    add r0, r1
-; CHECK-V7M-NEXT:    adds r0, #1
-; CHECK-V7M-NEXT:    bx lr
-; CHECK-V7M-NEXT:  .LBB0_8:
-; CHECK-V7M-NEXT:    add r0, r1
-; CHECK-V7M-NEXT:    adds r0, #2
+; CHECK-V7M-NEXT:  .LBB0_4:
 ; CHECK-V7M-NEXT:    bx lr
 ; CHECK-V7M-NEXT:    .p2align 2
 ; CHECK-V7M-NEXT:  .LCPI0_0:

From ed0e4c70c99d3afd87fb202ab03bda40512677e7 Mon Sep 17 00:00:00 2001
From: Simon Tatham <simon.tatham@arm.com>
Date: Mon, 3 Aug 2020 13:30:48 +0100
Subject: [PATCH 0904/1035] [clang][ARM] Add name-mangling test for direct
 __fp16 arguments.

`clang/test/CodeGenCXX/fp16-mangle.cpp` tests pointers to __fp16, but
if you give the `-fallow-half-arguments-and-returns` option, then
clang can also leave an __fp16 unmodified as a function argument or
return type. This regression test checks the name-mangling of that.

Reviewed By: miyuki

Differential Revision: https://reviews.llvm.org/D85010
---
 clang/test/CodeGenCXX/fp16-mangle-arg-return.cpp | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
 create mode 100644 clang/test/CodeGenCXX/fp16-mangle-arg-return.cpp

diff --git a/clang/test/CodeGenCXX/fp16-mangle-arg-return.cpp b/clang/test/CodeGenCXX/fp16-mangle-arg-return.cpp
new file mode 100644
index 0000000000000..15214e13ad8a7
--- /dev/null
+++ b/clang/test/CodeGenCXX/fp16-mangle-arg-return.cpp
@@ -0,0 +1,16 @@
+// RUN: %clang_cc1 -emit-llvm -o - -triple arm-arm-none-eabi -fallow-half-arguments-and-returns %s | FileCheck %s
+// RUN: %clang_cc1 -emit-llvm -o - -triple aarch64-arm-none-eabi -fallow-half-arguments-and-returns %s | FileCheck %s
+
+// Test name-mangling of __fp16 passed directly as a function argument
+// (when that is permitted).
+
+// CHECK: define {{.*}}void @_Z13fp16_argumentDh(half %{{.*}})
+void fp16_argument(__fp16 arg) {}
+
+// Test name-mangling of __fp16 as a return type. The return type of
+// fp16_return itself isn't mentioned in the mangled name, so to test
+// this, we have to pass it a function pointer and make __fp16 the
+// return type of that.
+
+// CHECK: define {{.*}}void @_Z11fp16_returnPFDhvE(half ()* %{{.*}})
+void fp16_return(__fp16 (*func)(void)) {}

From b57ea8ef2a8a07ffd2c05389da3f759caaa49f3e Mon Sep 17 00:00:00 2001
From: Sanjay Patel <spatel@rotateright.com>
Date: Sun, 2 Aug 2020 13:40:23 -0400
Subject: [PATCH 0905/1035] [InstCombine] add tests for xor-of-ors; NFC

---
 llvm/test/Transforms/InstCombine/xor.ll | 82 +++++++++++++++++++++++++
 1 file changed, 82 insertions(+)

diff --git a/llvm/test/Transforms/InstCombine/xor.ll b/llvm/test/Transforms/InstCombine/xor.ll
index 0fc39103d96fe..a133f2a0e009b 100644
--- a/llvm/test/Transforms/InstCombine/xor.ll
+++ b/llvm/test/Transforms/InstCombine/xor.ll
@@ -912,3 +912,85 @@ define <2 x i32> @test51vec(<2 x i32> %x, <2 x i32> %y) {
   %e = xor <2 x i32> %d, <i32 -1, i32 -1>
   ret <2 x i32> %e
 }
+
+define i4 @or_or_xor(i4 %x, i4 %y, i4 %z) {
+; CHECK-LABEL: @or_or_xor(
+; CHECK-NEXT:    [[O1:%.*]] = or i4 [[Z:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[O2:%.*]] = or i4 [[Z]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = xor i4 [[O1]], [[O2]]
+; CHECK-NEXT:    ret i4 [[R]]
+;
+  %o1 = or i4 %z, %x
+  %o2 = or i4 %z, %y
+  %r = xor i4 %o1, %o2
+  ret i4 %r
+}
+
+define i4 @or_or_xor_commute1(i4 %x, i4 %y, i4 %z) {
+; CHECK-LABEL: @or_or_xor_commute1(
+; CHECK-NEXT:    [[O1:%.*]] = or i4 [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    [[O2:%.*]] = or i4 [[Z]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = xor i4 [[O1]], [[O2]]
+; CHECK-NEXT:    ret i4 [[R]]
+;
+  %o1 = or i4 %x, %z
+  %o2 = or i4 %z, %y
+  %r = xor i4 %o1, %o2
+  ret i4 %r
+}
+
+define i4 @or_or_xor_commute2(i4 %x, i4 %y, i4 %z) {
+; CHECK-LABEL: @or_or_xor_commute2(
+; CHECK-NEXT:    [[O1:%.*]] = or i4 [[Z:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[O2:%.*]] = or i4 [[Y:%.*]], [[Z]]
+; CHECK-NEXT:    [[R:%.*]] = xor i4 [[O1]], [[O2]]
+; CHECK-NEXT:    ret i4 [[R]]
+;
+  %o1 = or i4 %z, %x
+  %o2 = or i4 %y, %z
+  %r = xor i4 %o1, %o2
+  ret i4 %r
+}
+
+define <2 x i4> @or_or_xor_commute3(<2 x i4> %x, <2 x i4> %y, <2 x i4> %z) {
+; CHECK-LABEL: @or_or_xor_commute3(
+; CHECK-NEXT:    [[O1:%.*]] = or <2 x i4> [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    [[O2:%.*]] = or <2 x i4> [[Y:%.*]], [[Z]]
+; CHECK-NEXT:    [[R:%.*]] = xor <2 x i4> [[O1]], [[O2]]
+; CHECK-NEXT:    ret <2 x i4> [[R]]
+;
+  %o1 = or <2 x i4> %x, %z
+  %o2 = or <2 x i4> %y, %z
+  %r = xor <2 x i4> %o1, %o2
+  ret <2 x i4> %r
+}
+
+define i4 @or_or_xor_use1(i4 %x, i4 %y, i4 %z, i4* %p) {
+; CHECK-LABEL: @or_or_xor_use1(
+; CHECK-NEXT:    [[O1:%.*]] = or i4 [[Z:%.*]], [[X:%.*]]
+; CHECK-NEXT:    store i4 [[O1]], i4* [[P:%.*]], align 1
+; CHECK-NEXT:    [[O2:%.*]] = or i4 [[Z]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = xor i4 [[O1]], [[O2]]
+; CHECK-NEXT:    ret i4 [[R]]
+;
+  %o1 = or i4 %z, %x
+  store i4 %o1, i4* %p
+  %o2 = or i4 %z, %y
+  %r = xor i4 %o1, %o2
+  ret i4 %r
+}
+
+define i4 @or_or_xor_use2(i4 %x, i4 %y, i4 %z, i4* %p) {
+; CHECK-LABEL: @or_or_xor_use2(
+; CHECK-NEXT:    [[O1:%.*]] = or i4 [[Z:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[O2:%.*]] = or i4 [[Z]], [[Y:%.*]]
+; CHECK-NEXT:    store i4 [[O2]], i4* [[P:%.*]], align 1
+; CHECK-NEXT:    [[R:%.*]] = xor i4 [[O1]], [[O2]]
+; CHECK-NEXT:    ret i4 [[R]]
+;
+  %o1 = or i4 %z, %x
+  %o2 = or i4 %z, %y
+  store i4 %o2, i4* %p
+  %r = xor i4 %o1, %o2
+  ret i4 %r
+}

From 2265d01f2a5bd153959701e22f5be2a40e1674a3 Mon Sep 17 00:00:00 2001
From: Sanjay Patel <spatel@rotateright.com>
Date: Mon, 3 Aug 2020 08:11:06 -0400
Subject: [PATCH 0906/1035] [InstCombine] reduce xor-of-or's bitwise logic
 (PR46955)

I tried to use m_Deferred() on this, but didn't find
a clean way to do that.

http://bugs.llvm.org/PR46955

https://alive2.llvm.org/ce/z/2h6QTq
---
 .../InstCombine/InstCombineAndOrXor.cpp       | 14 +++++++++++
 llvm/test/Transforms/InstCombine/xor.ll       | 24 +++++++++----------
 2 files changed, 26 insertions(+), 12 deletions(-)

diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
index 030d2f203ed6c..ef1e8dbe01558 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
@@ -3351,6 +3351,20 @@ Instruction *InstCombinerImpl::visitXor(BinaryOperator &I) {
       match(Op1, m_Not(m_Specific(A))))
     return BinaryOperator::CreateNot(Builder.CreateAnd(A, B));
 
+  // (A | B) ^ (A | C) --> (B ^ C) & ~A -- There are 4 commuted variants.
+  // TODO: Loosen one-use restriction if common operand is a constant.
+  Value *D;
+  if (match(Op0, m_OneUse(m_Or(m_Value(A), m_Value(B)))) &&
+      match(Op1, m_OneUse(m_Or(m_Value(C), m_Value(D))))) {
+    if (B == C || B == D)
+      std::swap(A, B);
+    if (A == C)
+      std::swap(C, D);
+    if (A == D)
+      return BinaryOperator::CreateAnd(Builder.CreateXor(B, C),
+                                       Builder.CreateNot(A));
+  }
+
   if (auto *LHS = dyn_cast<ICmpInst>(I.getOperand(0)))
     if (auto *RHS = dyn_cast<ICmpInst>(I.getOperand(1)))
       if (Value *V = foldXorOfICmps(LHS, RHS, I))
diff --git a/llvm/test/Transforms/InstCombine/xor.ll b/llvm/test/Transforms/InstCombine/xor.ll
index a133f2a0e009b..2bdb837bff041 100644
--- a/llvm/test/Transforms/InstCombine/xor.ll
+++ b/llvm/test/Transforms/InstCombine/xor.ll
@@ -915,9 +915,9 @@ define <2 x i32> @test51vec(<2 x i32> %x, <2 x i32> %y) {
 
 define i4 @or_or_xor(i4 %x, i4 %y, i4 %z) {
 ; CHECK-LABEL: @or_or_xor(
-; CHECK-NEXT:    [[O1:%.*]] = or i4 [[Z:%.*]], [[X:%.*]]
-; CHECK-NEXT:    [[O2:%.*]] = or i4 [[Z]], [[Y:%.*]]
-; CHECK-NEXT:    [[R:%.*]] = xor i4 [[O1]], [[O2]]
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i4 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = xor i4 [[Z:%.*]], -1
+; CHECK-NEXT:    [[R:%.*]] = and i4 [[TMP1]], [[TMP2]]
 ; CHECK-NEXT:    ret i4 [[R]]
 ;
   %o1 = or i4 %z, %x
@@ -928,9 +928,9 @@ define i4 @or_or_xor(i4 %x, i4 %y, i4 %z) {
 
 define i4 @or_or_xor_commute1(i4 %x, i4 %y, i4 %z) {
 ; CHECK-LABEL: @or_or_xor_commute1(
-; CHECK-NEXT:    [[O1:%.*]] = or i4 [[X:%.*]], [[Z:%.*]]
-; CHECK-NEXT:    [[O2:%.*]] = or i4 [[Z]], [[Y:%.*]]
-; CHECK-NEXT:    [[R:%.*]] = xor i4 [[O1]], [[O2]]
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i4 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = xor i4 [[Z:%.*]], -1
+; CHECK-NEXT:    [[R:%.*]] = and i4 [[TMP1]], [[TMP2]]
 ; CHECK-NEXT:    ret i4 [[R]]
 ;
   %o1 = or i4 %x, %z
@@ -941,9 +941,9 @@ define i4 @or_or_xor_commute1(i4 %x, i4 %y, i4 %z) {
 
 define i4 @or_or_xor_commute2(i4 %x, i4 %y, i4 %z) {
 ; CHECK-LABEL: @or_or_xor_commute2(
-; CHECK-NEXT:    [[O1:%.*]] = or i4 [[Z:%.*]], [[X:%.*]]
-; CHECK-NEXT:    [[O2:%.*]] = or i4 [[Y:%.*]], [[Z]]
-; CHECK-NEXT:    [[R:%.*]] = xor i4 [[O1]], [[O2]]
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i4 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = xor i4 [[Z:%.*]], -1
+; CHECK-NEXT:    [[R:%.*]] = and i4 [[TMP1]], [[TMP2]]
 ; CHECK-NEXT:    ret i4 [[R]]
 ;
   %o1 = or i4 %z, %x
@@ -954,9 +954,9 @@ define i4 @or_or_xor_commute2(i4 %x, i4 %y, i4 %z) {
 
 define <2 x i4> @or_or_xor_commute3(<2 x i4> %x, <2 x i4> %y, <2 x i4> %z) {
 ; CHECK-LABEL: @or_or_xor_commute3(
-; CHECK-NEXT:    [[O1:%.*]] = or <2 x i4> [[X:%.*]], [[Z:%.*]]
-; CHECK-NEXT:    [[O2:%.*]] = or <2 x i4> [[Y:%.*]], [[Z]]
-; CHECK-NEXT:    [[R:%.*]] = xor <2 x i4> [[O1]], [[O2]]
+; CHECK-NEXT:    [[TMP1:%.*]] = xor <2 x i4> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = xor <2 x i4> [[Z:%.*]], <i4 -1, i4 -1>
+; CHECK-NEXT:    [[R:%.*]] = and <2 x i4> [[TMP1]], [[TMP2]]
 ; CHECK-NEXT:    ret <2 x i4> [[R]]
 ;
   %o1 = or <2 x i4> %x, %z

From d8ef1d1251e3c0e11894ed82904dbab5e41c5711 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Wed, 22 Jul 2020 21:07:03 -0400
Subject: [PATCH 0907/1035] AMDGPU/GlobalISel: Fix selecting broken copies for
 s32->s64 anyext

These should probably not be legal in the first place, but that might
also be a pain.
---
 .../AMDGPU/AMDGPUInstructionSelector.cpp      | 27 ++++++-
 .../AMDGPU/GlobalISel/inst-select-anyext.mir  | 76 +++++++++++++++++--
 2 files changed, 95 insertions(+), 8 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
index 709329b4c0c6d..5aceb40933c37 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
@@ -1894,12 +1894,33 @@ bool AMDGPUInstructionSelector::selectG_SZA_EXT(MachineInstr &I) const {
   if (!DstTy.isScalar())
     return false;
 
-  if (I.getOpcode() == AMDGPU::G_ANYEXT)
-    return selectCOPY(I);
-
   // Artifact casts should never use vcc.
   const RegisterBank *SrcBank = getArtifactRegBank(SrcReg, *MRI, TRI);
 
+  // FIXME: This should probably be illegal and split earlier.
+  if (I.getOpcode() == AMDGPU::G_ANYEXT) {
+    if (DstSize <= 32)
+      return selectCOPY(I);
+
+    const TargetRegisterClass *SrcRC =
+        TRI.getRegClassForTypeOnBank(SrcTy, *SrcBank, *MRI);
+    const RegisterBank *DstBank = RBI.getRegBank(DstReg, *MRI, TRI);
+    const TargetRegisterClass *DstRC =
+        TRI.getRegClassForSizeOnBank(DstSize, *DstBank, *MRI);
+
+    Register UndefReg = MRI->createVirtualRegister(SrcRC);
+    BuildMI(MBB, I, DL, TII.get(AMDGPU::IMPLICIT_DEF), UndefReg);
+    BuildMI(MBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), DstReg)
+      .addReg(SrcReg)
+      .addImm(AMDGPU::sub0)
+      .addReg(UndefReg)
+      .addImm(AMDGPU::sub1);
+    I.eraseFromParent();
+
+    return RBI.constrainGenericRegister(DstReg, *DstRC, *MRI) &&
+           RBI.constrainGenericRegister(SrcReg, *SrcRC, *MRI);
+  }
+
   if (SrcBank->getID() == AMDGPU::VGPRRegBankID && DstSize <= 32) {
     // 64-bit should have been split up in RegBankSelect
 
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-anyext.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-anyext.mir
index 58d01774f745b..dcad0a85e8e0e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-anyext.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-anyext.mir
@@ -22,22 +22,88 @@ body: |
 ...
 
 ---
+name:  anyext_sgpr_s32_to_sgpr_s64
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $sgpr0
+
+    ; GCN-LABEL: name: anyext_sgpr_s32_to_sgpr_s64
+    ; GCN: liveins: $sgpr0
+    ; GCN: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+    ; GCN: [[DEF:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+    ; GCN: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[DEF]], %subreg.sub1
+    ; GCN: S_ENDPGM 0, implicit [[REG_SEQUENCE]]
+    %0:sgpr(s32) = COPY $sgpr0
+    %1:sgpr(s64) = G_ANYEXT %0
+    S_ENDPGM 0, implicit %1
+
+...
 
-name: anyext_sgpr_s16_to_sgpr_s64
+---
+name:  anyext_sgpr_s16_to_sgpr_s64
 legalized:       true
 regBankSelected: true
-body: |
+tracksRegLiveness: true
+body:             |
   bb.0:
     liveins: $sgpr0
 
     ; GCN-LABEL: name: anyext_sgpr_s16_to_sgpr_s64
+    ; GCN: liveins: $sgpr0
     ; GCN: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
-    ; GCN: [[COPY1:%[0-9]+]]:sreg_64 = COPY [[COPY]]
-    ; GCN: $sgpr0_sgpr1 = COPY [[COPY1]]
+    ; GCN: [[DEF:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+    ; GCN: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[DEF]], %subreg.sub1
+    ; GCN: S_ENDPGM 0, implicit [[REG_SEQUENCE]]
     %0:sgpr(s32) = COPY $sgpr0
     %1:sgpr(s16) = G_TRUNC %0
     %2:sgpr(s64) = G_ANYEXT %1
-    $sgpr0_sgpr1 = COPY %2
+    S_ENDPGM 0, implicit %2
+
+...
+
+---
+name:  anyext_vgpr_s32_to_vgpr_s64
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0
+
+    ; GCN-LABEL: name: anyext_vgpr_s32_to_vgpr_s64
+    ; GCN: liveins: $vgpr0
+    ; GCN: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GCN: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; GCN: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[DEF]], %subreg.sub1
+    ; GCN: S_ENDPGM 0, implicit [[REG_SEQUENCE]]
+    %0:vgpr(s32) = COPY $vgpr0
+    %1:vgpr(s64) = G_ANYEXT %0
+    S_ENDPGM 0, implicit %1
+
+...
+
+---
+name:  anyext_vgpr_s16_to_vgpr_s64
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0
+
+    ; GCN-LABEL: name: anyext_vgpr_s16_to_vgpr_s64
+    ; GCN: liveins: $vgpr0
+    ; GCN: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GCN: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; GCN: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[DEF]], %subreg.sub1
+    ; GCN: S_ENDPGM 0, implicit [[REG_SEQUENCE]]
+    %0:vgpr(s32) = COPY $vgpr0
+    %1:vgpr(s16) = G_TRUNC %0
+    %2:vgpr(s64) = G_ANYEXT %1
+    S_ENDPGM 0, implicit %2
 
 ...
 

From 99a971cadff7832a846394462c39a74aac64325d Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Mon, 3 Aug 2020 12:18:21 +0100
Subject: [PATCH 0908/1035] [X86][SSE] Start shuffle combining from
 ANY_EXTEND_VECTOR_INREG on SSE targets

We already do this on AVX (+ for ZERO_EXTEND_VECTOR_INREG), but this enables it for all SSE targets - we attempted something similar back at rL357057 but hit issues with the ZERO_EXTEND_VECTOR_INREG handling (PR41249).

I'm still looking at the vector-mul.ll regression - which is due to 32-bit targets performing the load as a f64, resulting in the shuffle combiner thinking it has to create a shuffle in the float domain.
---
 llvm/lib/Target/X86/X86ISelLowering.cpp       |  11 +-
 llvm/test/CodeGen/X86/combine-pmuldq.ll       |   4 +-
 llvm/test/CodeGen/X86/mulvi32.ll              |   8 +-
 llvm/test/CodeGen/X86/pmul.ll                 |  16 +-
 llvm/test/CodeGen/X86/promote-cmp.ll          |  23 ++-
 llvm/test/CodeGen/X86/vector-idiv-udiv-128.ll |  14 +-
 llvm/test/CodeGen/X86/vector-mul.ll           |   3 +-
 llvm/test/CodeGen/X86/vector-reduce-mul.ll    | 171 ++++++------------
 llvm/test/CodeGen/X86/vector-trunc-math.ll    |  12 +-
 9 files changed, 98 insertions(+), 164 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index ff59f28c8b6d2..e9bb50aacec0e 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -48671,6 +48671,7 @@ static SDValue combineExtInVec(SDNode *N, SelectionDAG &DAG,
                                const X86Subtarget &Subtarget) {
   EVT VT = N->getValueType(0);
   SDValue In = N->getOperand(0);
+  unsigned Opcode = N->getOpcode();
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
 
   // Try to merge vector loads and extend_inreg to an extload.
@@ -48679,7 +48680,7 @@ static SDValue combineExtInVec(SDNode *N, SelectionDAG &DAG,
     auto *Ld = cast<LoadSDNode>(In);
     if (Ld->isSimple()) {
       MVT SVT = In.getSimpleValueType().getVectorElementType();
-      ISD::LoadExtType Ext = N->getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG
+      ISD::LoadExtType Ext = Opcode == ISD::SIGN_EXTEND_VECTOR_INREG
                                  ? ISD::SEXTLOAD
                                  : ISD::ZEXTLOAD;
       EVT MemVT =
@@ -48687,8 +48688,7 @@ static SDValue combineExtInVec(SDNode *N, SelectionDAG &DAG,
       if (TLI.isLoadExtLegal(Ext, VT, MemVT)) {
         SDValue Load =
             DAG.getExtLoad(Ext, SDLoc(N), VT, Ld->getChain(), Ld->getBasePtr(),
-                           Ld->getPointerInfo(), MemVT,
-                           Ld->getOriginalAlign(),
+                           Ld->getPointerInfo(), MemVT, Ld->getOriginalAlign(),
                            Ld->getMemOperand()->getFlags());
         DAG.ReplaceAllUsesOfValueWith(SDValue(Ld, 1), Load.getValue(1));
         return Load;
@@ -48697,8 +48697,9 @@ static SDValue combineExtInVec(SDNode *N, SelectionDAG &DAG,
   }
 
   // Attempt to combine as a shuffle.
-  // TODO: SSE41 support
-  if (Subtarget.hasAVX() && N->getOpcode() != ISD::SIGN_EXTEND_VECTOR_INREG) {
+  // TODO: SSE ZERO_EXTEND_VECTOR_INREG support.
+  if (Opcode == ISD::ANY_EXTEND_VECTOR_INREG ||
+      (Opcode == ISD::ZERO_EXTEND_VECTOR_INREG && Subtarget.hasAVX())) {
     SDValue Op(N, 0);
     if (TLI.isTypeLegal(VT) && TLI.isTypeLegal(In.getValueType()))
       if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
diff --git a/llvm/test/CodeGen/X86/combine-pmuldq.ll b/llvm/test/CodeGen/X86/combine-pmuldq.ll
index 0e448f3f3be06..27823cf5fe8c5 100644
--- a/llvm/test/CodeGen/X86/combine-pmuldq.ll
+++ b/llvm/test/CodeGen/X86/combine-pmuldq.ll
@@ -91,9 +91,9 @@ define <4 x i64> @combine_shuffle_zero_pmuludq_256(<8 x i32> %a0, <8 x i32> %a1)
 define <8 x i64> @combine_zext_pmuludq_256(<8 x i32> %a) {
 ; SSE-LABEL: combine_zext_pmuludq_256:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,2,3,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,1,3,3]
 ; SSE-NEXT:    pmovzxdq {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,2,3,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,1,3,3]
 ; SSE-NEXT:    pmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
 ; SSE-NEXT:    movdqa {{.*#+}} xmm4 = [715827883,715827883]
 ; SSE-NEXT:    pmuludq %xmm4, %xmm0
diff --git a/llvm/test/CodeGen/X86/mulvi32.ll b/llvm/test/CodeGen/X86/mulvi32.ll
index d05c26e9842ac..388b5bc8a746c 100644
--- a/llvm/test/CodeGen/X86/mulvi32.ll
+++ b/llvm/test/CodeGen/X86/mulvi32.ll
@@ -137,8 +137,8 @@ define <4 x i64> @_mul4xi32toi64a(<4 x i32>, <4 x i32>) {
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,1,1,3]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,1,1,3]
 ; SSE2-NEXT:    pmuludq %xmm3, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,1,3,3]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,1,3,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,3,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,3,3]
 ; SSE2-NEXT:    pmuludq %xmm3, %xmm1
 ; SSE2-NEXT:    movdqa %xmm2, %xmm0
 ; SSE2-NEXT:    retq
@@ -148,8 +148,8 @@ define <4 x i64> @_mul4xi32toi64a(<4 x i32>, <4 x i32>) {
 ; SSE42-NEXT:    pmovzxdq {{.*#+}} xmm3 = xmm1[0],zero,xmm1[1],zero
 ; SSE42-NEXT:    pmovzxdq {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero
 ; SSE42-NEXT:    pmuludq %xmm3, %xmm2
-; SSE42-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,2,3,3]
-; SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,2,3,3]
+; SSE42-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,1,3,3]
+; SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,1,3,3]
 ; SSE42-NEXT:    pmuludq %xmm3, %xmm1
 ; SSE42-NEXT:    movdqa %xmm2, %xmm0
 ; SSE42-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/pmul.ll b/llvm/test/CodeGen/X86/pmul.ll
index 9aeb8292ac0ca..5a3101bc3efda 100644
--- a/llvm/test/CodeGen/X86/pmul.ll
+++ b/llvm/test/CodeGen/X86/pmul.ll
@@ -1033,10 +1033,10 @@ define <4 x i32> @mul_v4i64_zero_upper(<4 x i32> %val1, <4 x i32> %val2) {
 ; SSE41-LABEL: mul_v4i64_zero_upper:
 ; SSE41:       # %bb.0: # %entry
 ; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,2,3,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,1,3,3]
 ; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero
 ; SSE41-NEXT:    pmuludq %xmm2, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,1,3,3]
 ; SSE41-NEXT:    pmuludq %xmm3, %xmm1
 ; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,3],xmm1[1,3]
 ; SSE41-NEXT:    retq
@@ -1186,17 +1186,17 @@ define <8 x i32> @mul_v8i64_zero_upper(<8 x i32> %val1, <8 x i32> %val2) {
 ; SSE41-LABEL: mul_v8i64_zero_upper:
 ; SSE41:       # %bb.0: # %entry
 ; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[2,2,3,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[2,1,3,3]
 ; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm6 = xmm1[0],zero,xmm1[1],zero
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm1[2,2,3,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm1[2,1,3,3]
 ; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm0 = xmm2[0],zero,xmm2[1],zero
 ; SSE41-NEXT:    pmuludq %xmm4, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,2,3,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,1,3,3]
 ; SSE41-NEXT:    pmuludq %xmm5, %xmm1
 ; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,3],xmm1[1,3]
 ; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm1 = xmm3[0],zero,xmm3[1],zero
 ; SSE41-NEXT:    pmuludq %xmm6, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,2,3,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,1,3,3]
 ; SSE41-NEXT:    pmuludq %xmm7, %xmm2
 ; SSE41-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,3],xmm2[1,3]
 ; SSE41-NEXT:    retq
@@ -1311,11 +1311,11 @@ define <8 x i64> @mul_v8i64_sext(<8 x i16> %val1, <8 x i32> %val2) {
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,1,1]
 ; SSE41-NEXT:    pmovsxwq %xmm3, %xmm6
 ; SSE41-NEXT:    pmovsxwq %xmm0, %xmm7
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[2,2,3,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[2,1,3,3]
 ; SSE41-NEXT:    pmuldq %xmm4, %xmm3
 ; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero
 ; SSE41-NEXT:    pmuldq %xmm5, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[2,2,3,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[2,1,3,3]
 ; SSE41-NEXT:    pmuldq %xmm6, %xmm4
 ; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero
 ; SSE41-NEXT:    pmuldq %xmm7, %xmm0
diff --git a/llvm/test/CodeGen/X86/promote-cmp.ll b/llvm/test/CodeGen/X86/promote-cmp.ll
index f23900e8b8f8c..d6fcac28f62ad 100644
--- a/llvm/test/CodeGen/X86/promote-cmp.ll
+++ b/llvm/test/CodeGen/X86/promote-cmp.ll
@@ -30,20 +30,19 @@ define <4 x i64> @PR45808(<4 x i64> %0, <4 x i64> %1) {
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm7[1,1,3,3]
 ; SSE2-NEXT:    por %xmm4, %xmm5
 ; SSE2-NEXT:    shufps {{.*#+}} xmm5 = xmm5[0,2],xmm6[0,2]
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm4
-; SSE2-NEXT:    movaps %xmm5, %xmm6
-; SSE2-NEXT:    shufps {{.*#+}} xmm6 = xmm6[2,1],xmm5[3,3]
-; SSE2-NEXT:    psllq $63, %xmm6
-; SSE2-NEXT:    psrad $31, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm1
-; SSE2-NEXT:    pandn %xmm3, %xmm6
-; SSE2-NEXT:    por %xmm6, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm5 = xmm5[0,1,1,3]
-; SSE2-NEXT:    xorps %xmm4, %xmm5
+; SSE2-NEXT:    movaps {{.*#+}} xmm4 = <1,1,u,0>
+; SSE2-NEXT:    xorps %xmm5, %xmm4
+; SSE2-NEXT:    shufps {{.*#+}} xmm5 = xmm5[2,1,3,3]
 ; SSE2-NEXT:    psllq $63, %xmm5
 ; SSE2-NEXT:    psrad $31, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm5[1,1,3,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSE2-NEXT:    pand %xmm5, %xmm1
+; SSE2-NEXT:    pandn %xmm3, %xmm5
+; SSE2-NEXT:    por %xmm5, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[0,1,1,3]
+; SSE2-NEXT:    psllq $63, %xmm3
+; SSE2-NEXT:    psrad $31, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm3, %xmm0
 ; SSE2-NEXT:    pandn %xmm2, %xmm3
 ; SSE2-NEXT:    por %xmm3, %xmm0
diff --git a/llvm/test/CodeGen/X86/vector-idiv-udiv-128.ll b/llvm/test/CodeGen/X86/vector-idiv-udiv-128.ll
index 65131c0e3cb73..28f3e436efcd9 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-udiv-128.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-udiv-128.ll
@@ -835,17 +835,15 @@ define <16 x i8> @test_remconstant_16i8(<16 x i8> %a) nounwind {
 ; SSE41-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
 ; SSE41-NEXT:    pmullw {{.*}}(%rip), %xmm2
 ; SSE41-NEXT:    psrlw $8, %xmm2
+; SSE41-NEXT:    pmullw {{.*}}(%rip), %xmm2
+; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
+; SSE41-NEXT:    pand %xmm1, %xmm2
 ; SSE41-NEXT:    pmullw {{.*}}(%rip), %xmm3
 ; SSE41-NEXT:    psrlw $8, %xmm3
+; SSE41-NEXT:    pmullw {{.*}}(%rip), %xmm3
+; SSE41-NEXT:    pand %xmm1, %xmm3
 ; SSE41-NEXT:    packuswb %xmm2, %xmm3
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
-; SSE41-NEXT:    pmullw {{.*}}(%rip), %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
-; SSE41-NEXT:    pand %xmm3, %xmm1
-; SSE41-NEXT:    pmullw {{.*}}(%rip), %xmm2
-; SSE41-NEXT:    pand %xmm3, %xmm2
-; SSE41-NEXT:    packuswb %xmm2, %xmm1
-; SSE41-NEXT:    psubb %xmm1, %xmm0
+; SSE41-NEXT:    psubb %xmm3, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: test_remconstant_16i8:
diff --git a/llvm/test/CodeGen/X86/vector-mul.ll b/llvm/test/CodeGen/X86/vector-mul.ll
index 805ff9f69ed5e..d93ac61592f29 100644
--- a/llvm/test/CodeGen/X86/vector-mul.ll
+++ b/llvm/test/CodeGen/X86/vector-mul.ll
@@ -1904,7 +1904,8 @@ define <2 x i64> @mul_v2i64_zext_cross_bb(<2 x i32>* %in, <2 x i32>* %y) {
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    pmovzxdq {{.*#+}} xmm0 = mem[0],zero,mem[1],zero
-; X86-NEXT:    pmovzxdq {{.*#+}} xmm1 = mem[0],zero,mem[1],zero
+; X86-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
+; X86-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0,0,1,1]
 ; X86-NEXT:    pmuludq %xmm1, %xmm0
 ; X86-NEXT:    retl
 ;
diff --git a/llvm/test/CodeGen/X86/vector-reduce-mul.ll b/llvm/test/CodeGen/X86/vector-reduce-mul.ll
index a3cec079ab091..b44b66eb1d87a 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-mul.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-mul.ll
@@ -1567,14 +1567,14 @@ define i8 @test_v4i8(<4 x i8> %a0) {
 ; SSE2-LABEL: test_v4i8:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pmullw %xmm1, %xmm0
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; SSE2-NEXT:    psrld $16, %xmm1
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    pmullw %xmm0, %xmm1
-; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    psrld $16, %xmm0
+; SSE2-NEXT:    pmullw %xmm1, %xmm0
+; SSE2-NEXT:    movd %xmm0, %eax
 ; SSE2-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-NEXT:    retq
 ;
@@ -1618,16 +1618,18 @@ define i8 @test_v4i8(<4 x i8> %a0) {
 define i8 @test_v8i8(<8 x i8> %a0) {
 ; SSE2-LABEL: test_v8i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,2,3,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    pmullw %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,2,3,0]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE2-NEXT:    pmullw %xmm1, %xmm0
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    psrld $16, %xmm1
-; SSE2-NEXT:    pmullw %xmm0, %xmm1
-; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
+; SSE2-NEXT:    pand %xmm0, %xmm1
+; SSE2-NEXT:    packuswb %xmm1, %xmm1
+; SSE2-NEXT:    psrld $16, %xmm0
+; SSE2-NEXT:    pmullw %xmm1, %xmm0
+; SSE2-NEXT:    movd %xmm0, %eax
 ; SSE2-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-NEXT:    retq
 ;
@@ -1637,13 +1639,11 @@ define i8 @test_v8i8(<8 x i8> %a0) {
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmullw %xmm0, %xmm1
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
-; SSE41-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; SSE41-NEXT:    pmullw %xmm0, %xmm1
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    psrld $16, %xmm0
 ; SSE41-NEXT:    pmullw %xmm1, %xmm0
-; SSE41-NEXT:    movd %xmm0, %eax
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    psrld $16, %xmm1
+; SSE41-NEXT:    pmullw %xmm0, %xmm1
+; SSE41-NEXT:    movd %xmm1, %eax
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    retq
 ;
@@ -1710,23 +1710,15 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; SSE41-NEXT:    pmullw %xmm1, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
-; SSE41-NEXT:    pand %xmm1, %xmm0
-; SSE41-NEXT:    pxor %xmm3, %xmm3
-; SSE41-NEXT:    packuswb %xmm3, %xmm0
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; SSE41-NEXT:    pmullw %xmm2, %xmm0
-; SSE41-NEXT:    pand %xmm0, %xmm1
-; SSE41-NEXT:    packuswb %xmm3, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmullw %xmm0, %xmm1
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    psrld $8, %xmm0
-; SSE41-NEXT:    psrlw $8, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE41-NEXT:    pmullw %xmm1, %xmm0
-; SSE41-NEXT:    movd %xmm0, %eax
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    psrld $8, %xmm1
+; SSE41-NEXT:    psrlw $8, %xmm1
+; SSE41-NEXT:    pmullw %xmm0, %xmm1
+; SSE41-NEXT:    movd %xmm1, %eax
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    retq
 ;
@@ -1900,30 +1892,17 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; SSE41-NEXT:    pmullw %xmm1, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
-; SSE41-NEXT:    pand %xmm1, %xmm4
-; SSE41-NEXT:    pmullw %xmm2, %xmm3
-; SSE41-NEXT:    pand %xmm1, %xmm3
-; SSE41-NEXT:    packuswb %xmm4, %xmm3
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm2 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
-; SSE41-NEXT:    pmullw %xmm0, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pxor %xmm3, %xmm3
-; SSE41-NEXT:    packuswb %xmm3, %xmm2
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
-; SSE41-NEXT:    pmullw %xmm0, %xmm2
-; SSE41-NEXT:    pand %xmm2, %xmm1
-; SSE41-NEXT:    packuswb %xmm3, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; SSE41-NEXT:    pmullw %xmm2, %xmm0
+; SSE41-NEXT:    pmullw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmullw %xmm0, %xmm1
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    psrld $8, %xmm0
-; SSE41-NEXT:    psrlw $8, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE41-NEXT:    pmullw %xmm1, %xmm0
-; SSE41-NEXT:    movd %xmm0, %eax
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    psrld $8, %xmm1
+; SSE41-NEXT:    psrlw $8, %xmm1
+; SSE41-NEXT:    pmullw %xmm0, %xmm1
+; SSE41-NEXT:    movd %xmm1, %eax
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    retq
 ;
@@ -2139,31 +2118,13 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; SSE41-NEXT:    pmullw %xmm2, %xmm0
-; SSE41-NEXT:    pmullw %xmm3, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
-; SSE41-NEXT:    pshufb %xmm2, %xmm1
 ; SSE41-NEXT:    pmullw %xmm4, %xmm5
-; SSE41-NEXT:    pshufb %xmm2, %xmm5
-; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
-; SSE41-NEXT:    movdqa %xmm0, %xmm3
-; SSE41-NEXT:    pand %xmm2, %xmm3
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm4 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; SSE41-NEXT:    pmullw %xmm4, %xmm1
-; SSE41-NEXT:    pand %xmm2, %xmm1
-; SSE41-NEXT:    packuswb %xmm3, %xmm1
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; SSE41-NEXT:    pmullw %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
-; SSE41-NEXT:    pand %xmm2, %xmm1
-; SSE41-NEXT:    pxor %xmm3, %xmm3
-; SSE41-NEXT:    packuswb %xmm3, %xmm1
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; SSE41-NEXT:    pmullw %xmm0, %xmm1
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    packuswb %xmm3, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
+; SSE41-NEXT:    pmullw %xmm3, %xmm5
+; SSE41-NEXT:    pmullw %xmm0, %xmm5
+; SSE41-NEXT:    pmullw %xmm1, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[2,3,2,3]
+; SSE41-NEXT:    pmullw %xmm5, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE41-NEXT:    pmullw %xmm0, %xmm1
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
 ; SSE41-NEXT:    psrld $8, %xmm0
@@ -2446,14 +2407,14 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm9 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
 ; SSE41-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; SSE41-NEXT:    pmullw %xmm6, %xmm2
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm10 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm6 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
 ; SSE41-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; SSE41-NEXT:    pmullw %xmm2, %xmm4
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm10 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm11 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero,xmm7[4],zero,xmm7[5],zero,xmm7[6],zero,xmm7[7],zero
 ; SSE41-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm6 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm2 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
 ; SSE41-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; SSE41-NEXT:    pmullw %xmm7, %xmm3
 ; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm7 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
@@ -2464,43 +2425,17 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; SSE41-NEXT:    pmullw %xmm5, %xmm1
 ; SSE41-NEXT:    pmullw %xmm4, %xmm1
 ; SSE41-NEXT:    pmullw %xmm0, %xmm1
-; SSE41-NEXT:    pmullw %xmm7, %xmm3
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
-; SSE41-NEXT:    pshufb %xmm0, %xmm3
-; SSE41-NEXT:    pmullw %xmm11, %xmm6
-; SSE41-NEXT:    pshufb %xmm0, %xmm6
-; SSE41-NEXT:    pmullw %xmm10, %xmm2
-; SSE41-NEXT:    pshufb %xmm0, %xmm2
 ; SSE41-NEXT:    pmullw %xmm8, %xmm9
-; SSE41-NEXT:    pshufb %xmm0, %xmm9
-; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm5 = xmm9[0],zero,xmm9[1],zero,xmm9[2],zero,xmm9[3],zero,xmm9[4],zero,xmm9[5],zero,xmm9[6],zero,xmm9[7],zero
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
-; SSE41-NEXT:    pmullw %xmm5, %xmm2
-; SSE41-NEXT:    pshufb %xmm0, %xmm2
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm5 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
-; SSE41-NEXT:    pmullw %xmm5, %xmm3
-; SSE41-NEXT:    pshufb %xmm0, %xmm3
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    pand %xmm4, %xmm0
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
-; SSE41-NEXT:    pmullw %xmm3, %xmm2
-; SSE41-NEXT:    pand %xmm4, %xmm2
-; SSE41-NEXT:    packuswb %xmm0, %xmm2
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
-; SSE41-NEXT:    pmullw %xmm1, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; SSE41-NEXT:    pand %xmm4, %xmm0
-; SSE41-NEXT:    pxor %xmm2, %xmm2
-; SSE41-NEXT:    packuswb %xmm2, %xmm0
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; SSE41-NEXT:    pmullw %xmm1, %xmm0
-; SSE41-NEXT:    pand %xmm0, %xmm4
-; SSE41-NEXT:    packuswb %xmm2, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
+; SSE41-NEXT:    pmullw %xmm9, %xmm6
+; SSE41-NEXT:    pmullw %xmm11, %xmm2
+; SSE41-NEXT:    pmullw %xmm2, %xmm7
+; SSE41-NEXT:    pmullw %xmm7, %xmm3
+; SSE41-NEXT:    pmullw %xmm6, %xmm3
+; SSE41-NEXT:    pmullw %xmm1, %xmm3
+; SSE41-NEXT:    pmullw %xmm10, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]
+; SSE41-NEXT:    pmullw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE41-NEXT:    pmullw %xmm0, %xmm1
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
 ; SSE41-NEXT:    psrld $8, %xmm0
diff --git a/llvm/test/CodeGen/X86/vector-trunc-math.ll b/llvm/test/CodeGen/X86/vector-trunc-math.ll
index 4489fd51035aa..81a6f60f034f1 100644
--- a/llvm/test/CodeGen/X86/vector-trunc-math.ll
+++ b/llvm/test/CodeGen/X86/vector-trunc-math.ll
@@ -5058,10 +5058,10 @@ define <4 x i32> @mul_add_const_v4i64_v4i32(<4 x i32> %a0, <4 x i32> %a1) nounwi
 ; SSE-LABEL: mul_add_const_v4i64_v4i32:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,1,1,3]
-; SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,1,3,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,3,3]
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,1,1,3]
 ; SSE-NEXT:    pmuludq %xmm2, %xmm0
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,1,3,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,3,3]
 ; SSE-NEXT:    pmuludq %xmm3, %xmm1
 ; SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
 ; SSE-NEXT:    paddd {{.*}}(%rip), %xmm0
@@ -5084,10 +5084,10 @@ define <4 x i32> @mul_add_self_v4i64_v4i32(<4 x i32> %a0, <4 x i32> %a1) nounwin
 ; SSE-LABEL: mul_add_self_v4i64_v4i32:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,1,1,3]
-; SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,1,3,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,3,3]
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,1,1,3]
 ; SSE-NEXT:    pmuludq %xmm2, %xmm0
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,1,3,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,3,3]
 ; SSE-NEXT:    pmuludq %xmm3, %xmm1
 ; SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
 ; SSE-NEXT:    paddd %xmm0, %xmm0
@@ -5110,10 +5110,10 @@ define <4 x i32> @mul_add_multiuse_v4i64_v4i32(<4 x i32> %a0, <4 x i32> %a1) nou
 ; SSE-LABEL: mul_add_multiuse_v4i64_v4i32:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,1,1,3]
-; SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,1,3,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,3,3]
 ; SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[0,1,1,3]
 ; SSE-NEXT:    pmuludq %xmm2, %xmm4
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,1,3,3]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,3,3]
 ; SSE-NEXT:    pmuludq %xmm3, %xmm1
 ; SSE-NEXT:    shufps {{.*#+}} xmm4 = xmm4[0,2],xmm1[0,2]
 ; SSE-NEXT:    paddd %xmm4, %xmm0

From fd63e46941fc48d4cc777ef94e185637898d0adb Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Tue, 16 Jun 2020 14:52:14 -0400
Subject: [PATCH 0909/1035] AMDGPU/GlobalISel: Apply load bitcast to
 s.buffer.load intrinsic

Should also apply this to the non-scalar buffer loads.
---
 .../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 68 +++++++++------
 llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h  |  4 +-
 .../legalize-llvm.amdgcn.s.buffer.load.mir    | 85 +++++++++++++++++--
 3 files changed, 121 insertions(+), 36 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index cc97e11707ab1..b40870024cc49 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -122,20 +122,23 @@ static LegalizeMutation moreEltsToNext32Bit(unsigned TypeIdx) {
   };
 }
 
+static LLT getBitcastRegisterType(const LLT Ty) {
+  const unsigned Size = Ty.getSizeInBits();
+
+  LLT CoercedTy;
+  if (Size <= 32) {
+    // <2 x s8> -> s16
+    // <4 x s8> -> s32
+    return LLT::scalar(Size);
+  }
+
+  return LLT::scalarOrVector(Size / 32, 32);
+}
+
 static LegalizeMutation bitcastToRegisterType(unsigned TypeIdx) {
   return [=](const LegalityQuery &Query) {
     const LLT Ty = Query.Types[TypeIdx];
-    unsigned Size = Ty.getSizeInBits();
-
-    LLT CoercedTy;
-    if (Size <= 32) {
-      // <2 x s8> -> s16
-      // <4 x s8> -> s32
-      CoercedTy = LLT::scalar(Size);
-    } else
-      CoercedTy = LLT::scalarOrVector(Size / 32, 32);
-
-    return std::make_pair(TypeIdx, CoercedTy);
+    return std::make_pair(TypeIdx, getBitcastRegisterType(Ty));
   };
 }
 
@@ -335,6 +338,20 @@ static bool isLoadStoreLegal(const GCNSubtarget &ST, const LegalityQuery &Query,
          !loadStoreBitcastWorkaround(Ty);
 }
 
+/// Return true if a load or store of the type should be lowered with a bitcast
+/// to a different type.
+static bool shouldBitcastLoadStoreType(const GCNSubtarget &ST, const LLT Ty,
+                                       const unsigned MemSizeInBits) {
+  const unsigned Size = Ty.getSizeInBits();
+    if (Size != MemSizeInBits)
+      return Size <= 32 && Ty.isVector();
+
+  if (loadStoreBitcastWorkaround(Ty) && isRegisterType(Ty))
+    return true;
+  return Ty.isVector() && (Size <= 32 || isRegisterSize(Size)) &&
+         !isRegisterVectorElementType(Ty.getElementType());
+}
+
 AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
                                          const GCNTargetMachine &TM)
   :  ST(ST_) {
@@ -1048,16 +1065,8 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
     // 16-bit vector parts.
     Actions.bitcastIf(
       [=](const LegalityQuery &Query) -> bool {
-        const LLT Ty = Query.Types[0];
-        const unsigned Size = Ty.getSizeInBits();
-
-        if (Size != Query.MMODescrs[0].SizeInBits)
-          return Size <= 32 && Ty.isVector();
-
-        if (loadStoreBitcastWorkaround(Ty) && isRegisterType(Ty))
-          return true;
-        return Ty.isVector() && (Size <= 32 || isRegisterSize(Size)) &&
-               !isRegisterVectorElementType(Ty.getElementType());
+        return shouldBitcastLoadStoreType(ST, Query.Types[0],
+                                          Query.MMODescrs[0].SizeInBits);
       }, bitcastToRegisterType(0));
 
     Actions
@@ -4137,8 +4146,10 @@ bool AMDGPULegalizerInfo::legalizeImageIntrinsic(
 }
 
 bool AMDGPULegalizerInfo::legalizeSBufferLoad(
-  MachineInstr &MI, MachineIRBuilder &B,
-  GISelChangeObserver &Observer) const {
+  LegalizerHelper &Helper, MachineInstr &MI) const {
+  MachineIRBuilder &B = Helper.MIRBuilder;
+  GISelChangeObserver &Observer = Helper.Observer;
+
   Register Dst = MI.getOperand(0).getReg();
   LLT Ty = B.getMRI()->getType(Dst);
   unsigned Size = Ty.getSizeInBits();
@@ -4146,6 +4157,13 @@ bool AMDGPULegalizerInfo::legalizeSBufferLoad(
 
   Observer.changingInstr(MI);
 
+  if (shouldBitcastLoadStoreType(ST, Ty, Size)) {
+    Ty = getBitcastRegisterType(Ty);
+    Helper.bitcastDst(MI, Ty, 0);
+    Dst = MI.getOperand(0).getReg();
+    B.setInsertPt(B.getMBB(), MI);
+  }
+
   // FIXME: We don't really need this intermediate instruction. The intrinsic
   // should be fixed to have a memory operand. Since it's readnone, we're not
   // allowed to add one.
@@ -4167,8 +4185,6 @@ bool AMDGPULegalizerInfo::legalizeSBufferLoad(
   // always be legal. We may need to restore this to a 96-bit result if it turns
   // out this needs to be converted to a vector load during RegBankSelect.
   if (!isPowerOf2_32(Size)) {
-    LegalizerHelper Helper(MF, *this, Observer, B);
-
     if (Ty.isVector())
       Helper.moreElementsVectorDst(MI, getPow2VectorType(Ty), 0);
     else
@@ -4360,7 +4376,7 @@ bool AMDGPULegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
     return true;
   }
   case Intrinsic::amdgcn_s_buffer_load:
-    return legalizeSBufferLoad(MI, B, Helper.Observer);
+    return legalizeSBufferLoad(Helper, MI);
   case Intrinsic::amdgcn_raw_buffer_store:
   case Intrinsic::amdgcn_struct_buffer_store:
     return legalizeBufferStore(MI, MRI, B, false, false);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h
index fe4e17db48a6b..332d675c1a88e 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h
@@ -167,9 +167,7 @@ class AMDGPULegalizerInfo final : public LegalizerInfo {
       GISelChangeObserver &Observer,
       const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr) const;
 
-  bool legalizeSBufferLoad(
-    MachineInstr &MI, MachineIRBuilder &B,
-    GISelChangeObserver &Observer) const;
+  bool legalizeSBufferLoad(LegalizerHelper &Helper, MachineInstr &MI) const;
 
   bool legalizeAtomicIncDec(MachineInstr &MI,  MachineIRBuilder &B,
                             bool IsInc) const;
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.s.buffer.load.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.s.buffer.load.mir
index 8860ca6ba5e58..9aee145ec1d15 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.s.buffer.load.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.s.buffer.load.mir
@@ -67,9 +67,10 @@ body:             |
     ; GCN-LABEL: name: s_buffer_load_v6s16
     ; GCN: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
     ; GCN: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GCN: [[AMDGPU_S_BUFFER_LOAD:%[0-9]+]]:_(<8 x s16>) = G_AMDGPU_S_BUFFER_LOAD [[COPY]](<4 x s32>), [[C]](s32), 0 :: (dereferenceable invariant load 12, align 4)
-    ; GCN: [[EXTRACT:%[0-9]+]]:_(<6 x s16>) = G_EXTRACT [[AMDGPU_S_BUFFER_LOAD]](<8 x s16>), 0
-    ; GCN: S_ENDPGM 0, implicit [[EXTRACT]](<6 x s16>)
+    ; GCN: [[AMDGPU_S_BUFFER_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_S_BUFFER_LOAD [[COPY]](<4 x s32>), [[C]](s32), 0 :: (dereferenceable invariant load 12, align 4)
+    ; GCN: [[EXTRACT:%[0-9]+]]:_(<3 x s32>) = G_EXTRACT [[AMDGPU_S_BUFFER_LOAD]](<4 x s32>), 0
+    ; GCN: [[BITCAST:%[0-9]+]]:_(<6 x s16>) = G_BITCAST [[EXTRACT]](<3 x s32>)
+    ; GCN: S_ENDPGM 0, implicit [[BITCAST]](<6 x s16>)
     %0:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
     %1:_(s32) = G_CONSTANT i32 0
     %2:_(<6 x s16>) = G_INTRINSIC intrinsic(@llvm.amdgcn.s.buffer.load), %0, %1, 0
@@ -124,13 +125,83 @@ body:             |
     ; GCN-LABEL: name: s_buffer_load_v12s8
     ; GCN: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
     ; GCN: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; GCN: [[AMDGPU_S_BUFFER_LOAD:%[0-9]+]]:_(<16 x s8>) = G_AMDGPU_S_BUFFER_LOAD [[COPY]](<4 x s32>), [[C]](s32), 0 :: (dereferenceable invariant load 12, align 4)
-    ; GCN: [[EXTRACT:%[0-9]+]]:_(<12 x s8>) = G_EXTRACT [[AMDGPU_S_BUFFER_LOAD]](<16 x s8>), 0
-    ; GCN: S_ENDPGM 0, implicit [[EXTRACT]](<12 x s8>)
+    ; GCN: [[AMDGPU_S_BUFFER_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_S_BUFFER_LOAD [[COPY]](<4 x s32>), [[C]](s32), 0 :: (dereferenceable invariant load 12, align 4)
+    ; GCN: [[EXTRACT:%[0-9]+]]:_(<3 x s32>) = G_EXTRACT [[AMDGPU_S_BUFFER_LOAD]](<4 x s32>), 0
+    ; GCN: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[EXTRACT]](<3 x s32>)
+    ; GCN: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; GCN: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[UV]], [[C1]](s32)
+    ; GCN: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GCN: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[UV]], [[C2]](s32)
+    ; GCN: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; GCN: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[UV]], [[C3]](s32)
+    ; GCN: [[COPY1:%[0-9]+]]:_(s32) = COPY [[UV]](s32)
+    ; GCN: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; GCN: [[COPY3:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+    ; GCN: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
+    ; GCN: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[UV1]], [[C1]](s32)
+    ; GCN: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[UV1]], [[C2]](s32)
+    ; GCN: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[UV1]], [[C3]](s32)
+    ; GCN: [[COPY5:%[0-9]+]]:_(s32) = COPY [[UV1]](s32)
+    ; GCN: [[COPY6:%[0-9]+]]:_(s32) = COPY [[LSHR3]](s32)
+    ; GCN: [[COPY7:%[0-9]+]]:_(s32) = COPY [[LSHR4]](s32)
+    ; GCN: [[COPY8:%[0-9]+]]:_(s32) = COPY [[LSHR5]](s32)
+    ; GCN: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[UV2]], [[C1]](s32)
+    ; GCN: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[UV2]], [[C2]](s32)
+    ; GCN: [[LSHR8:%[0-9]+]]:_(s32) = G_LSHR [[UV2]], [[C3]](s32)
+    ; GCN: [[COPY9:%[0-9]+]]:_(s32) = COPY [[UV2]](s32)
+    ; GCN: [[COPY10:%[0-9]+]]:_(s32) = COPY [[LSHR6]](s32)
+    ; GCN: [[COPY11:%[0-9]+]]:_(s32) = COPY [[LSHR7]](s32)
+    ; GCN: [[COPY12:%[0-9]+]]:_(s32) = COPY [[LSHR8]](s32)
+    ; GCN: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; GCN: [[COPY13:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; GCN: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY13]], [[C4]]
+    ; GCN: [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+    ; GCN: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY14]], [[C4]]
+    ; GCN: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
+    ; GCN: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
+    ; GCN: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
+    ; GCN: [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY3]](s32)
+    ; GCN: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY15]], [[C4]]
+    ; GCN: [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY4]](s32)
+    ; GCN: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY16]], [[C4]]
+    ; GCN: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C2]](s32)
+    ; GCN: [[OR1:%[0-9]+]]:_(s32) = G_OR [[AND2]], [[SHL1]]
+    ; GCN: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
+    ; GCN: [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY5]](s32)
+    ; GCN: [[AND4:%[0-9]+]]:_(s32) = G_AND [[COPY17]], [[C4]]
+    ; GCN: [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY6]](s32)
+    ; GCN: [[AND5:%[0-9]+]]:_(s32) = G_AND [[COPY18]], [[C4]]
+    ; GCN: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C2]](s32)
+    ; GCN: [[OR2:%[0-9]+]]:_(s32) = G_OR [[AND4]], [[SHL2]]
+    ; GCN: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](s32)
+    ; GCN: [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY7]](s32)
+    ; GCN: [[AND6:%[0-9]+]]:_(s32) = G_AND [[COPY19]], [[C4]]
+    ; GCN: [[COPY20:%[0-9]+]]:_(s32) = COPY [[COPY8]](s32)
+    ; GCN: [[AND7:%[0-9]+]]:_(s32) = G_AND [[COPY20]], [[C4]]
+    ; GCN: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[AND7]], [[C2]](s32)
+    ; GCN: [[OR3:%[0-9]+]]:_(s32) = G_OR [[AND6]], [[SHL3]]
+    ; GCN: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](s32)
+    ; GCN: [[COPY21:%[0-9]+]]:_(s32) = COPY [[COPY9]](s32)
+    ; GCN: [[AND8:%[0-9]+]]:_(s32) = G_AND [[COPY21]], [[C4]]
+    ; GCN: [[COPY22:%[0-9]+]]:_(s32) = COPY [[COPY10]](s32)
+    ; GCN: [[AND9:%[0-9]+]]:_(s32) = G_AND [[COPY22]], [[C4]]
+    ; GCN: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[AND9]], [[C2]](s32)
+    ; GCN: [[OR4:%[0-9]+]]:_(s32) = G_OR [[AND8]], [[SHL4]]
+    ; GCN: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](s32)
+    ; GCN: [[COPY23:%[0-9]+]]:_(s32) = COPY [[COPY11]](s32)
+    ; GCN: [[AND10:%[0-9]+]]:_(s32) = G_AND [[COPY23]], [[C4]]
+    ; GCN: [[COPY24:%[0-9]+]]:_(s32) = COPY [[COPY12]](s32)
+    ; GCN: [[AND11:%[0-9]+]]:_(s32) = G_AND [[COPY24]], [[C4]]
+    ; GCN: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[AND11]], [[C2]](s32)
+    ; GCN: [[OR5:%[0-9]+]]:_(s32) = G_OR [[AND10]], [[SHL5]]
+    ; GCN: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](s32)
+    ; GCN: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>), [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
+    ; GCN: S_ENDPGM 0, implicit [[CONCAT_VECTORS]](<12 x s16>)
     %0:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
     %1:_(s32) = G_CONSTANT i32 0
     %2:_(<12 x s8>) = G_INTRINSIC intrinsic(@llvm.amdgcn.s.buffer.load), %0, %1, 0
-    S_ENDPGM 0, implicit %2
+    %3:_(<12 x s16>) = G_ANYEXT %2
+    S_ENDPGM 0, implicit %3
 
 ...
 

From f19a9be385ef782140d1e551e03553daa79d1bc1 Mon Sep 17 00:00:00 2001
From: Sanjay Patel <spatel@rotateright.com>
Date: Mon, 3 Aug 2020 08:58:41 -0400
Subject: [PATCH 0910/1035] Revert "[InstCombine] reduce xor-of-or's bitwise
 logic (PR46955)"

This reverts commit 2265d01f2a5bd153959701e22f5be2a40e1674a3.
Seeing bot failures after this change like:
http://lab.llvm.org:8011/builders/clang-cmake-x86_64-sde-avx512-linux/builds/42586
---
 .../InstCombine/InstCombineAndOrXor.cpp       | 14 -----------
 llvm/test/Transforms/InstCombine/xor.ll       | 24 +++++++++----------
 2 files changed, 12 insertions(+), 26 deletions(-)

diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
index ef1e8dbe01558..030d2f203ed6c 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
@@ -3351,20 +3351,6 @@ Instruction *InstCombinerImpl::visitXor(BinaryOperator &I) {
       match(Op1, m_Not(m_Specific(A))))
     return BinaryOperator::CreateNot(Builder.CreateAnd(A, B));
 
-  // (A | B) ^ (A | C) --> (B ^ C) & ~A -- There are 4 commuted variants.
-  // TODO: Loosen one-use restriction if common operand is a constant.
-  Value *D;
-  if (match(Op0, m_OneUse(m_Or(m_Value(A), m_Value(B)))) &&
-      match(Op1, m_OneUse(m_Or(m_Value(C), m_Value(D))))) {
-    if (B == C || B == D)
-      std::swap(A, B);
-    if (A == C)
-      std::swap(C, D);
-    if (A == D)
-      return BinaryOperator::CreateAnd(Builder.CreateXor(B, C),
-                                       Builder.CreateNot(A));
-  }
-
   if (auto *LHS = dyn_cast<ICmpInst>(I.getOperand(0)))
     if (auto *RHS = dyn_cast<ICmpInst>(I.getOperand(1)))
       if (Value *V = foldXorOfICmps(LHS, RHS, I))
diff --git a/llvm/test/Transforms/InstCombine/xor.ll b/llvm/test/Transforms/InstCombine/xor.ll
index 2bdb837bff041..a133f2a0e009b 100644
--- a/llvm/test/Transforms/InstCombine/xor.ll
+++ b/llvm/test/Transforms/InstCombine/xor.ll
@@ -915,9 +915,9 @@ define <2 x i32> @test51vec(<2 x i32> %x, <2 x i32> %y) {
 
 define i4 @or_or_xor(i4 %x, i4 %y, i4 %z) {
 ; CHECK-LABEL: @or_or_xor(
-; CHECK-NEXT:    [[TMP1:%.*]] = xor i4 [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[TMP2:%.*]] = xor i4 [[Z:%.*]], -1
-; CHECK-NEXT:    [[R:%.*]] = and i4 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[O1:%.*]] = or i4 [[Z:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[O2:%.*]] = or i4 [[Z]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = xor i4 [[O1]], [[O2]]
 ; CHECK-NEXT:    ret i4 [[R]]
 ;
   %o1 = or i4 %z, %x
@@ -928,9 +928,9 @@ define i4 @or_or_xor(i4 %x, i4 %y, i4 %z) {
 
 define i4 @or_or_xor_commute1(i4 %x, i4 %y, i4 %z) {
 ; CHECK-LABEL: @or_or_xor_commute1(
-; CHECK-NEXT:    [[TMP1:%.*]] = xor i4 [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[TMP2:%.*]] = xor i4 [[Z:%.*]], -1
-; CHECK-NEXT:    [[R:%.*]] = and i4 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[O1:%.*]] = or i4 [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    [[O2:%.*]] = or i4 [[Z]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = xor i4 [[O1]], [[O2]]
 ; CHECK-NEXT:    ret i4 [[R]]
 ;
   %o1 = or i4 %x, %z
@@ -941,9 +941,9 @@ define i4 @or_or_xor_commute1(i4 %x, i4 %y, i4 %z) {
 
 define i4 @or_or_xor_commute2(i4 %x, i4 %y, i4 %z) {
 ; CHECK-LABEL: @or_or_xor_commute2(
-; CHECK-NEXT:    [[TMP1:%.*]] = xor i4 [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[TMP2:%.*]] = xor i4 [[Z:%.*]], -1
-; CHECK-NEXT:    [[R:%.*]] = and i4 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[O1:%.*]] = or i4 [[Z:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[O2:%.*]] = or i4 [[Y:%.*]], [[Z]]
+; CHECK-NEXT:    [[R:%.*]] = xor i4 [[O1]], [[O2]]
 ; CHECK-NEXT:    ret i4 [[R]]
 ;
   %o1 = or i4 %z, %x
@@ -954,9 +954,9 @@ define i4 @or_or_xor_commute2(i4 %x, i4 %y, i4 %z) {
 
 define <2 x i4> @or_or_xor_commute3(<2 x i4> %x, <2 x i4> %y, <2 x i4> %z) {
 ; CHECK-LABEL: @or_or_xor_commute3(
-; CHECK-NEXT:    [[TMP1:%.*]] = xor <2 x i4> [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[TMP2:%.*]] = xor <2 x i4> [[Z:%.*]], <i4 -1, i4 -1>
-; CHECK-NEXT:    [[R:%.*]] = and <2 x i4> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[O1:%.*]] = or <2 x i4> [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    [[O2:%.*]] = or <2 x i4> [[Y:%.*]], [[Z]]
+; CHECK-NEXT:    [[R:%.*]] = xor <2 x i4> [[O1]], [[O2]]
 ; CHECK-NEXT:    ret <2 x i4> [[R]]
 ;
   %o1 = or <2 x i4> %x, %z

From 1782fbbc69482e76eee8af203694bb771a44c921 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Sun, 2 Feb 2020 17:42:02 -0500
Subject: [PATCH 0911/1035] GlobalISel: Reimplement moreElementsVectorDst

Use pad with undef and unmerge with unused results. This is annoyingly
similar to several other places in LegalizerHelper, but they're all
slightly different.
---
 .../llvm/CodeGen/GlobalISel/LegalizerHelper.h |   4 +
 .../CodeGen/GlobalISel/LegalizerHelper.cpp    |  38 +++-
 .../AMDGPU/GlobalISel/legalize-and.mir        |  55 +++--
 .../AMDGPU/GlobalISel/legalize-ashr.mir       |  15 +-
 .../AMDGPU/GlobalISel/legalize-extract.mir    |  84 ++++---
 .../AMDGPU/GlobalISel/legalize-fabs.mir       |  64 +++---
 .../AMDGPU/GlobalISel/legalize-fadd.mir       |  95 ++++----
 .../GlobalISel/legalize-fcanonicalize.mir     |  60 ++---
 .../AMDGPU/GlobalISel/legalize-fcos.mir       |  58 ++---
 .../AMDGPU/GlobalISel/legalize-fdiv.mir       | 157 +++++++------
 .../AMDGPU/GlobalISel/legalize-ffloor.mir     |  58 ++---
 .../AMDGPU/GlobalISel/legalize-fma.mir        | 130 ++++++-----
 .../AMDGPU/GlobalISel/legalize-fmaxnum.mir    |   5 +-
 .../AMDGPU/GlobalISel/legalize-fminnum.mir    |   5 +-
 .../AMDGPU/GlobalISel/legalize-fmul.mir       |  95 ++++----
 .../AMDGPU/GlobalISel/legalize-fneg.mir       |  64 +++---
 .../AMDGPU/GlobalISel/legalize-fpext.mir      |  12 +-
 .../AMDGPU/GlobalISel/legalize-freeze.mir     |  34 +--
 .../AMDGPU/GlobalISel/legalize-fsin.mir       |  58 ++---
 .../AMDGPU/GlobalISel/legalize-fsqrt.mir      |  58 ++---
 .../AMDGPU/GlobalISel/legalize-fsub.mir       |  91 ++++----
 .../GlobalISel/legalize-implicit-def.mir      |  39 ++--
 .../AMDGPU/GlobalISel/legalize-insert.mir     |  35 +--
 .../legalize-llvm.amdgcn.image.load.2d.d16.ll |  26 ++-
 .../legalize-llvm.amdgcn.s.buffer.load.mir    |  70 +++---
 .../GlobalISel/legalize-load-constant.mir     | 200 ++++++++++-------
 .../AMDGPU/GlobalISel/legalize-load-flat.mir  | 175 +++++++++------
 .../GlobalISel/legalize-load-global.mir       | 210 +++++++++++-------
 .../AMDGPU/GlobalISel/legalize-load-local.mir | 175 +++++++++------
 .../GlobalISel/legalize-load-private.mir      | 180 +++++++++------
 .../AMDGPU/GlobalISel/legalize-lshr.mir       |  15 +-
 .../CodeGen/AMDGPU/GlobalISel/legalize-or.mir |  55 +++--
 .../AMDGPU/GlobalISel/legalize-phi.mir        |   5 +-
 .../AMDGPU/GlobalISel/legalize-saddsat.mir    |  24 +-
 .../AMDGPU/GlobalISel/legalize-select.mir     |  10 +-
 .../AMDGPU/GlobalISel/legalize-shl.mir        |  15 +-
 .../AMDGPU/GlobalISel/legalize-smax.mir       |  83 ++++---
 .../AMDGPU/GlobalISel/legalize-smin.mir       |  83 ++++---
 .../AMDGPU/GlobalISel/legalize-ssubsat.mir    |  24 +-
 .../AMDGPU/GlobalISel/legalize-uaddsat.mir    |  24 +-
 .../AMDGPU/GlobalISel/legalize-umax.mir       |  83 ++++---
 .../AMDGPU/GlobalISel/legalize-umin.mir       |  83 ++++---
 .../GlobalISel/legalize-unmerge-values.mir    |  12 +-
 .../AMDGPU/GlobalISel/legalize-usubsat.mir    |  24 +-
 .../AMDGPU/GlobalISel/legalize-xor.mir        |  55 +++--
 .../GlobalISel/llvm.amdgcn.s.buffer.load.ll   | 144 +++++++-----
 .../regbankselect-amdgcn.s.buffer.load.ll     |  26 ++-
 .../GlobalISel/LegalizerHelperTest.cpp        |   3 +-
 48 files changed, 1816 insertions(+), 1302 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
index e819dca5bdf0f..dfd27bd5f7c5f 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
@@ -154,6 +154,10 @@ class LegalizerHelper {
   /// def by inserting a G_BITCAST from \p CastTy
   void bitcastDst(MachineInstr &MI, LLT CastTy, unsigned OpIdx);
 
+  /// Widen \p OrigReg to \p WideTy by merging to a wider type, padding with
+  /// G_IMPLICIT_DEF, and producing dead results.
+  Register widenWithUnmerge(LLT WideTy, Register OrigReg);
+
 private:
   LegalizeResult
   widenScalarMergeValues(MachineInstr &MI, unsigned TypeIdx, LLT WideTy);
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index 920c9e008012e..c11f91bc0a6ca 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -1317,10 +1317,8 @@ void LegalizerHelper::narrowScalarDst(MachineInstr &MI, LLT NarrowTy,
 void LegalizerHelper::moreElementsVectorDst(MachineInstr &MI, LLT WideTy,
                                             unsigned OpIdx) {
   MachineOperand &MO = MI.getOperand(OpIdx);
-  Register DstExt = MRI.createGenericVirtualRegister(WideTy);
   MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
-  MIRBuilder.buildExtract(MO, DstExt, 0);
-  MO.setReg(DstExt);
+  MO.setReg(widenWithUnmerge(WideTy, MO.getReg()));
 }
 
 void LegalizerHelper::moreElementsVectorSrc(MachineInstr &MI, LLT MoreTy,
@@ -1488,6 +1486,40 @@ LegalizerHelper::widenScalarMergeValues(MachineInstr &MI, unsigned TypeIdx,
   return Legalized;
 }
 
+Register LegalizerHelper::widenWithUnmerge(LLT WideTy, Register OrigReg) {
+  Register WideReg = MRI.createGenericVirtualRegister(WideTy);
+  LLT OrigTy = MRI.getType(OrigReg);
+  LLT LCMTy = getLCMType(WideTy, OrigTy);
+
+  const int NumMergeParts = LCMTy.getSizeInBits() / WideTy.getSizeInBits();
+  const int NumUnmergeParts = LCMTy.getSizeInBits() / OrigTy.getSizeInBits();
+
+  Register UnmergeSrc = WideReg;
+
+  // Create a merge to the LCM type, padding with undef
+  // %0:_(<3 x s32>) = G_FOO => <4 x s32>
+  // =>
+  // %1:_(<4 x s32>) = G_FOO
+  // %2:_(<4 x s32>) = G_IMPLICIT_DEF
+  // %3:_(<12 x s32>) = G_CONCAT_VECTORS %1, %2, %2
+  // %0:_(<3 x s32>), %4:_, %5:_, %6:_ = G_UNMERGE_VALUES %3
+  if (NumMergeParts > 1) {
+    Register Undef = MIRBuilder.buildUndef(WideTy).getReg(0);
+    SmallVector<Register, 8> MergeParts(NumMergeParts, Undef);
+    MergeParts[0] = WideReg;
+    UnmergeSrc = MIRBuilder.buildMerge(LCMTy, MergeParts).getReg(0);
+  }
+
+  // Unmerge to the original register and pad with dead defs.
+  SmallVector<Register, 8> UnmergeResults(NumUnmergeParts);
+  UnmergeResults[0] = OrigReg;
+  for (int I = 1; I != NumUnmergeParts; ++I)
+    UnmergeResults[I] = MRI.createGenericVirtualRegister(OrigTy);
+
+  MIRBuilder.buildUnmerge(UnmergeResults, UnmergeSrc);
+  return WideReg;
+}
+
 LegalizerHelper::LegalizeResult
 LegalizerHelper::widenScalarUnmergeValues(MachineInstr &MI, unsigned TypeIdx,
                                           LLT WideTy) {
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-and.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-and.mir
index 3d26cefc4f55d..ef7e4e2f15a2e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-and.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-and.mir
@@ -361,15 +361,18 @@ body: |
 
     ; CHECK-LABEL: name: test_and_v3s16
     ; CHECK: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; CHECK: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
     ; CHECK: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; CHECK: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF1]](<4 x s16>), 0
+    ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; CHECK: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; CHECK: [[DEF2:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; CHECK: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF2]], [[EXTRACT]](<3 x s16>), 0
-    ; CHECK: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF2]], [[EXTRACT1]](<3 x s16>), 0
+    ; CHECK: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF2]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; CHECK: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; CHECK: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
+    ; CHECK: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
     ; CHECK: [[AND:%[0-9]+]]:_(<4 x s16>) = G_AND [[INSERT]], [[INSERT1]]
-    ; CHECK: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[AND]](<4 x s16>), 0
-    ; CHECK: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF2]], [[EXTRACT2]](<3 x s16>), 0
+    ; CHECK: [[CONCAT_VECTORS2:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[AND]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; CHECK: [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>), [[UV10:%[0-9]+]]:_(<3 x s16>), [[UV11:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS2]](<12 x s16>)
+    ; CHECK: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV8]](<3 x s16>), 0
     ; CHECK: $vgpr0_vgpr1 = COPY [[INSERT2]](<4 x s16>)
     %0:_(<3 x s16>) = G_IMPLICIT_DEF
     %1:_(<3 x s16>) = G_IMPLICIT_DEF
@@ -403,27 +406,32 @@ body: |
 
     ; CHECK-LABEL: name: test_and_v5s16
     ; CHECK: [[DEF:%[0-9]+]]:_(<6 x s16>) = G_IMPLICIT_DEF
-    ; CHECK: [[EXTRACT:%[0-9]+]]:_(<5 x s16>) = G_EXTRACT [[DEF]](<6 x s16>), 0
     ; CHECK: [[DEF1:%[0-9]+]]:_(<6 x s16>) = G_IMPLICIT_DEF
-    ; CHECK: [[EXTRACT1:%[0-9]+]]:_(<5 x s16>) = G_EXTRACT [[DEF1]](<6 x s16>), 0
+    ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<30 x s16>) = G_CONCAT_VECTORS [[DEF]](<6 x s16>), [[DEF1]](<6 x s16>), [[DEF1]](<6 x s16>), [[DEF1]](<6 x s16>), [[DEF1]](<6 x s16>)
+    ; CHECK: [[UV:%[0-9]+]]:_(<5 x s16>), [[UV1:%[0-9]+]]:_(<5 x s16>), [[UV2:%[0-9]+]]:_(<5 x s16>), [[UV3:%[0-9]+]]:_(<5 x s16>), [[UV4:%[0-9]+]]:_(<5 x s16>), [[UV5:%[0-9]+]]:_(<5 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<30 x s16>)
     ; CHECK: [[DEF2:%[0-9]+]]:_(<6 x s16>) = G_IMPLICIT_DEF
-    ; CHECK: [[INSERT:%[0-9]+]]:_(<6 x s16>) = G_INSERT [[DEF2]], [[EXTRACT]](<5 x s16>), 0
-    ; CHECK: [[INSERT1:%[0-9]+]]:_(<6 x s16>) = G_INSERT [[DEF2]], [[EXTRACT1]](<5 x s16>), 0
-    ; CHECK: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[INSERT]](<6 x s16>)
-    ; CHECK: [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<6 x s16>)
+    ; CHECK: [[CONCAT_VECTORS1:%[0-9]+]]:_(<30 x s16>) = G_CONCAT_VECTORS [[DEF2]](<6 x s16>), [[DEF1]](<6 x s16>), [[DEF1]](<6 x s16>), [[DEF1]](<6 x s16>), [[DEF1]](<6 x s16>)
+    ; CHECK: [[UV6:%[0-9]+]]:_(<5 x s16>), [[UV7:%[0-9]+]]:_(<5 x s16>), [[UV8:%[0-9]+]]:_(<5 x s16>), [[UV9:%[0-9]+]]:_(<5 x s16>), [[UV10:%[0-9]+]]:_(<5 x s16>), [[UV11:%[0-9]+]]:_(<5 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<30 x s16>)
+    ; CHECK: [[INSERT:%[0-9]+]]:_(<6 x s16>) = G_INSERT [[DEF1]], [[UV]](<5 x s16>), 0
+    ; CHECK: [[INSERT1:%[0-9]+]]:_(<6 x s16>) = G_INSERT [[DEF1]], [[UV6]](<5 x s16>), 0
+    ; CHECK: [[UV12:%[0-9]+]]:_(<3 x s16>), [[UV13:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[INSERT]](<6 x s16>)
+    ; CHECK: [[UV14:%[0-9]+]]:_(<3 x s16>), [[UV15:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<6 x s16>)
     ; CHECK: [[DEF3:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; CHECK: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF3]], [[UV]](<3 x s16>), 0
-    ; CHECK: [[INSERT3:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF3]], [[UV2]](<3 x s16>), 0
+    ; CHECK: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF3]], [[UV12]](<3 x s16>), 0
+    ; CHECK: [[INSERT3:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF3]], [[UV14]](<3 x s16>), 0
     ; CHECK: [[AND:%[0-9]+]]:_(<4 x s16>) = G_AND [[INSERT2]], [[INSERT3]]
-    ; CHECK: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[AND]](<4 x s16>), 0
-    ; CHECK: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF3]], [[UV1]](<3 x s16>), 0
-    ; CHECK: [[INSERT5:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF3]], [[UV3]](<3 x s16>), 0
+    ; CHECK: [[CONCAT_VECTORS2:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[AND]](<4 x s16>), [[DEF3]](<4 x s16>), [[DEF3]](<4 x s16>)
+    ; CHECK: [[UV16:%[0-9]+]]:_(<3 x s16>), [[UV17:%[0-9]+]]:_(<3 x s16>), [[UV18:%[0-9]+]]:_(<3 x s16>), [[UV19:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS2]](<12 x s16>)
+    ; CHECK: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF3]], [[UV13]](<3 x s16>), 0
+    ; CHECK: [[INSERT5:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF3]], [[UV15]](<3 x s16>), 0
     ; CHECK: [[AND1:%[0-9]+]]:_(<4 x s16>) = G_AND [[INSERT4]], [[INSERT5]]
-    ; CHECK: [[EXTRACT3:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[AND1]](<4 x s16>), 0
-    ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[EXTRACT2]](<3 x s16>), [[EXTRACT3]](<3 x s16>)
-    ; CHECK: [[EXTRACT4:%[0-9]+]]:_(<5 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<6 x s16>), 0
+    ; CHECK: [[CONCAT_VECTORS3:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[AND1]](<4 x s16>), [[DEF3]](<4 x s16>), [[DEF3]](<4 x s16>)
+    ; CHECK: [[UV20:%[0-9]+]]:_(<3 x s16>), [[UV21:%[0-9]+]]:_(<3 x s16>), [[UV22:%[0-9]+]]:_(<3 x s16>), [[UV23:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS3]](<12 x s16>)
+    ; CHECK: [[CONCAT_VECTORS4:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[UV16]](<3 x s16>), [[UV20]](<3 x s16>)
+    ; CHECK: [[CONCAT_VECTORS5:%[0-9]+]]:_(<30 x s16>) = G_CONCAT_VECTORS [[CONCAT_VECTORS4]](<6 x s16>), [[DEF1]](<6 x s16>), [[DEF1]](<6 x s16>), [[DEF1]](<6 x s16>), [[DEF1]](<6 x s16>)
+    ; CHECK: [[UV24:%[0-9]+]]:_(<5 x s16>), [[UV25:%[0-9]+]]:_(<5 x s16>), [[UV26:%[0-9]+]]:_(<5 x s16>), [[UV27:%[0-9]+]]:_(<5 x s16>), [[UV28:%[0-9]+]]:_(<5 x s16>), [[UV29:%[0-9]+]]:_(<5 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS5]](<30 x s16>)
     ; CHECK: [[DEF4:%[0-9]+]]:_(<8 x s16>) = G_IMPLICIT_DEF
-    ; CHECK: [[INSERT6:%[0-9]+]]:_(<8 x s16>) = G_INSERT [[DEF4]], [[EXTRACT4]](<5 x s16>), 0
+    ; CHECK: [[INSERT6:%[0-9]+]]:_(<8 x s16>) = G_INSERT [[DEF4]], [[UV24]](<5 x s16>), 0
     ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INSERT6]](<8 x s16>)
     %0:_(<5 x s16>) = G_IMPLICIT_DEF
     %1:_(<5 x s16>) = G_IMPLICIT_DEF
@@ -463,8 +471,9 @@ body: |
     ; CHECK: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ANYEXT6]], [[ANYEXT7]]
     ; CHECK: [[TRUNC3:%[0-9]+]]:_(s8) = G_TRUNC [[AND3]](s32)
     ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s8>) = G_BUILD_VECTOR [[TRUNC]](s8), [[TRUNC1]](s8), [[TRUNC2]](s8), [[TRUNC3]](s8)
-    ; CHECK: [[EXTRACT:%[0-9]+]]:_(<3 x s8>) = G_EXTRACT [[BUILD_VECTOR]](<4 x s8>), 0
-    ; CHECK: [[ANYEXT8:%[0-9]+]]:_(<3 x s32>) = G_ANYEXT [[EXTRACT]](<3 x s8>)
+    ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s8>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<4 x s8>), [[DEF2]](<4 x s8>), [[DEF2]](<4 x s8>)
+    ; CHECK: [[UV8:%[0-9]+]]:_(<3 x s8>), [[UV9:%[0-9]+]]:_(<3 x s8>), [[UV10:%[0-9]+]]:_(<3 x s8>), [[UV11:%[0-9]+]]:_(<3 x s8>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s8>)
+    ; CHECK: [[ANYEXT8:%[0-9]+]]:_(<3 x s32>) = G_ANYEXT [[UV8]](<3 x s8>)
     ; CHECK: $vgpr0_vgpr1_vgpr2 = COPY [[ANYEXT8]](<3 x s32>)
     %0:_(<3 x s8>) = G_IMPLICIT_DEF
     %1:_(<3 x s8>) = G_IMPLICIT_DEF
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ashr.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ashr.mir
index c4d511ddda3ca..a39e97cf3309e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ashr.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ashr.mir
@@ -863,14 +863,17 @@ body: |
     ; GFX9: [[ASHR:%[0-9]+]]:_(<2 x s16>) = G_ASHR [[EXTRACT2]], [[EXTRACT4]](<2 x s16>)
     ; GFX9: [[ASHR1:%[0-9]+]]:_(s16) = G_ASHR [[EXTRACT3]], [[EXTRACT5]](s16)
     ; GFX9: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9: [[EXTRACT6:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF1]](<4 x s16>), 0
-    ; GFX9: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT6]](<3 x s16>), 0
+    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF1]](<4 x s16>), [[DEF]](<4 x s16>), [[DEF]](<4 x s16>)
+    ; GFX9: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
+    ; GFX9: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV]](<3 x s16>), 0
     ; GFX9: [[INSERT5:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT4]], [[ASHR]](<2 x s16>), 0
-    ; GFX9: [[EXTRACT7:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT5]](<4 x s16>), 0
-    ; GFX9: [[INSERT6:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT7]](<3 x s16>), 0
+    ; GFX9: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT5]](<4 x s16>), [[DEF]](<4 x s16>), [[DEF]](<4 x s16>)
+    ; GFX9: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; GFX9: [[INSERT6:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV4]](<3 x s16>), 0
     ; GFX9: [[INSERT7:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT6]], [[ASHR1]](s16), 32
-    ; GFX9: [[EXTRACT8:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT7]](<4 x s16>), 0
-    ; GFX9: [[INSERT8:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT8]](<3 x s16>), 0
+    ; GFX9: [[CONCAT_VECTORS2:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT7]](<4 x s16>), [[DEF]](<4 x s16>), [[DEF]](<4 x s16>)
+    ; GFX9: [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>), [[UV10:%[0-9]+]]:_(<3 x s16>), [[UV11:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS2]](<12 x s16>)
+    ; GFX9: [[INSERT8:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV8]](<3 x s16>), 0
     ; GFX9: $vgpr0_vgpr1 = COPY [[INSERT8]](<4 x s16>)
     %0:_(<4 x s16>) = COPY $vgpr0_vgpr1
     %1:_(<4 x s16>) = COPY $vgpr2_vgpr3
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-extract.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-extract.mir
index 9bb25356c950c..0e23fba6afaaa 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-extract.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-extract.mir
@@ -522,11 +522,14 @@ body: |
     ; CHECK-LABEL: name: extract_s8_v3s8_offset16
     ; CHECK: [[DEF:%[0-9]+]]:_(<4 x s32>) = G_IMPLICIT_DEF
     ; CHECK: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[DEF]](<4 x s32>)
-    ; CHECK: [[EXTRACT:%[0-9]+]]:_(<3 x s8>) = G_EXTRACT [[TRUNC]](<4 x s8>), 0
-    ; CHECK: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8), [[UV2:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[EXTRACT]](<3 x s8>)
-    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s16) = G_ANYEXT [[UV]](s8)
-    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(s16) = G_ANYEXT [[UV1]](s8)
-    ; CHECK: [[ANYEXT2:%[0-9]+]]:_(s16) = G_ANYEXT [[UV2]](s8)
+    ; CHECK: [[DEF1:%[0-9]+]]:_(<4 x s32>) = G_IMPLICIT_DEF
+    ; CHECK: [[TRUNC1:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[DEF1]](<4 x s32>)
+    ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s8>) = G_CONCAT_VECTORS [[TRUNC]](<4 x s8>), [[TRUNC1]](<4 x s8>), [[TRUNC1]](<4 x s8>)
+    ; CHECK: [[UV:%[0-9]+]]:_(<3 x s8>), [[UV1:%[0-9]+]]:_(<3 x s8>), [[UV2:%[0-9]+]]:_(<3 x s8>), [[UV3:%[0-9]+]]:_(<3 x s8>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s8>)
+    ; CHECK: [[UV4:%[0-9]+]]:_(s8), [[UV5:%[0-9]+]]:_(s8), [[UV6:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[UV]](<3 x s8>)
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s16) = G_ANYEXT [[UV4]](s8)
+    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(s16) = G_ANYEXT [[UV5]](s8)
+    ; CHECK: [[ANYEXT2:%[0-9]+]]:_(s16) = G_ANYEXT [[UV6]](s8)
     ; CHECK: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ANYEXT]](s16)
     ; CHECK: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[ANYEXT1]](s16)
     ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
@@ -538,12 +541,12 @@ body: |
     ; CHECK: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C]](s32)
     ; CHECK: [[OR1:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[SHL1]]
     ; CHECK: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
-    ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>)
-    ; CHECK: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<4 x s16>), 0
-    ; CHECK: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; CHECK: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
-    ; CHECK: [[EXTRACT2:%[0-9]+]]:_(s16) = G_EXTRACT [[INSERT]](<4 x s16>), 32
-    ; CHECK: [[ANYEXT3:%[0-9]+]]:_(s32) = G_ANYEXT [[EXTRACT2]](s16)
+    ; CHECK: [[CONCAT_VECTORS1:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>)
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS1]](<4 x s16>), 0
+    ; CHECK: [[DEF2:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; CHECK: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF2]], [[EXTRACT]](<3 x s16>), 0
+    ; CHECK: [[EXTRACT1:%[0-9]+]]:_(s16) = G_EXTRACT [[INSERT]](<4 x s16>), 32
+    ; CHECK: [[ANYEXT3:%[0-9]+]]:_(s32) = G_ANYEXT [[EXTRACT1]](s16)
     ; CHECK: $vgpr0 = COPY [[ANYEXT3]](s32)
     %0:_(<3 x s8>) = G_IMPLICIT_DEF
     %1:_(s8) = G_EXTRACT %0, 16
@@ -558,13 +561,16 @@ body: |
     ; CHECK-LABEL: name: extract_s8_v5s1_offset4
     ; CHECK: [[DEF:%[0-9]+]]:_(<6 x s32>) = G_IMPLICIT_DEF
     ; CHECK: [[TRUNC:%[0-9]+]]:_(<6 x s1>) = G_TRUNC [[DEF]](<6 x s32>)
-    ; CHECK: [[EXTRACT:%[0-9]+]]:_(<5 x s1>) = G_EXTRACT [[TRUNC]](<6 x s1>), 0
-    ; CHECK: [[UV:%[0-9]+]]:_(s1), [[UV1:%[0-9]+]]:_(s1), [[UV2:%[0-9]+]]:_(s1), [[UV3:%[0-9]+]]:_(s1), [[UV4:%[0-9]+]]:_(s1) = G_UNMERGE_VALUES [[EXTRACT]](<5 x s1>)
-    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s16) = G_ANYEXT [[UV]](s1)
-    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(s16) = G_ANYEXT [[UV1]](s1)
-    ; CHECK: [[ANYEXT2:%[0-9]+]]:_(s16) = G_ANYEXT [[UV2]](s1)
-    ; CHECK: [[ANYEXT3:%[0-9]+]]:_(s16) = G_ANYEXT [[UV3]](s1)
-    ; CHECK: [[ANYEXT4:%[0-9]+]]:_(s16) = G_ANYEXT [[UV4]](s1)
+    ; CHECK: [[DEF1:%[0-9]+]]:_(<6 x s32>) = G_IMPLICIT_DEF
+    ; CHECK: [[TRUNC1:%[0-9]+]]:_(<6 x s1>) = G_TRUNC [[DEF1]](<6 x s32>)
+    ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<30 x s1>) = G_CONCAT_VECTORS [[TRUNC]](<6 x s1>), [[TRUNC1]](<6 x s1>), [[TRUNC1]](<6 x s1>), [[TRUNC1]](<6 x s1>), [[TRUNC1]](<6 x s1>)
+    ; CHECK: [[UV:%[0-9]+]]:_(<5 x s1>), [[UV1:%[0-9]+]]:_(<5 x s1>), [[UV2:%[0-9]+]]:_(<5 x s1>), [[UV3:%[0-9]+]]:_(<5 x s1>), [[UV4:%[0-9]+]]:_(<5 x s1>), [[UV5:%[0-9]+]]:_(<5 x s1>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<30 x s1>)
+    ; CHECK: [[UV6:%[0-9]+]]:_(s1), [[UV7:%[0-9]+]]:_(s1), [[UV8:%[0-9]+]]:_(s1), [[UV9:%[0-9]+]]:_(s1), [[UV10:%[0-9]+]]:_(s1) = G_UNMERGE_VALUES [[UV]](<5 x s1>)
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s16) = G_ANYEXT [[UV6]](s1)
+    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(s16) = G_ANYEXT [[UV7]](s1)
+    ; CHECK: [[ANYEXT2:%[0-9]+]]:_(s16) = G_ANYEXT [[UV8]](s1)
+    ; CHECK: [[ANYEXT3:%[0-9]+]]:_(s16) = G_ANYEXT [[UV9]](s1)
+    ; CHECK: [[ANYEXT4:%[0-9]+]]:_(s16) = G_ANYEXT [[UV10]](s1)
     ; CHECK: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ANYEXT]](s16)
     ; CHECK: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[ANYEXT1]](s16)
     ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
@@ -581,12 +587,12 @@ body: |
     ; CHECK: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C]](s32)
     ; CHECK: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT4]], [[SHL2]]
     ; CHECK: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](s32)
-    ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>)
-    ; CHECK: [[EXTRACT1:%[0-9]+]]:_(<5 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<6 x s16>), 0
-    ; CHECK: [[DEF1:%[0-9]+]]:_(<6 x s16>) = G_IMPLICIT_DEF
-    ; CHECK: [[INSERT:%[0-9]+]]:_(<6 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<5 x s16>), 0
-    ; CHECK: [[EXTRACT2:%[0-9]+]]:_(s16) = G_EXTRACT [[INSERT]](<6 x s16>), 64
-    ; CHECK: [[ANYEXT5:%[0-9]+]]:_(s32) = G_ANYEXT [[EXTRACT2]](s16)
+    ; CHECK: [[CONCAT_VECTORS1:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>)
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(<5 x s16>) = G_EXTRACT [[CONCAT_VECTORS1]](<6 x s16>), 0
+    ; CHECK: [[DEF2:%[0-9]+]]:_(<6 x s16>) = G_IMPLICIT_DEF
+    ; CHECK: [[INSERT:%[0-9]+]]:_(<6 x s16>) = G_INSERT [[DEF2]], [[EXTRACT]](<5 x s16>), 0
+    ; CHECK: [[EXTRACT1:%[0-9]+]]:_(s16) = G_EXTRACT [[INSERT]](<6 x s16>), 64
+    ; CHECK: [[ANYEXT5:%[0-9]+]]:_(s32) = G_ANYEXT [[EXTRACT1]](s16)
     ; CHECK: $vgpr0 = COPY [[ANYEXT5]](s32)
     %0:_(<5 x s1>) = G_IMPLICIT_DEF
     %1:_(s1) = G_EXTRACT %0, 4
@@ -850,10 +856,12 @@ body: |
 
     ; CHECK-LABEL: name: extract_s16_v3s16_offset0
     ; CHECK: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; CHECK: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
-    ; CHECK: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s16>), 0
-    ; CHECK: [[EXTRACT1:%[0-9]+]]:_(s16) = G_EXTRACT [[INSERT]](<4 x s16>), 0
-    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[EXTRACT1]](s16)
+    ; CHECK: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; CHECK: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
+    ; CHECK: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(s16) = G_EXTRACT [[INSERT]](<4 x s16>), 0
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[EXTRACT]](s16)
     ; CHECK: $vgpr0 = COPY [[ANYEXT]](s32)
     %0:_(<3 x s16>) = G_IMPLICIT_DEF
     %1:_(s16) = G_EXTRACT %0, 0
@@ -933,10 +941,12 @@ body: |
 
     ; CHECK-LABEL: name: extract_v2s16_v3s16_offset0
     ; CHECK: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; CHECK: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
-    ; CHECK: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s16>), 0
-    ; CHECK: [[EXTRACT1:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<4 x s16>), 0
-    ; CHECK: $vgpr0 = COPY [[EXTRACT1]](<2 x s16>)
+    ; CHECK: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; CHECK: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
+    ; CHECK: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<4 x s16>), 0
+    ; CHECK: $vgpr0 = COPY [[EXTRACT]](<2 x s16>)
     %0:_(<3 x s16>) = G_IMPLICIT_DEF
     %1:_(<2 x s16>) = G_EXTRACT %0, 0
     $vgpr0 = COPY %1
@@ -949,10 +959,12 @@ body: |
 
     ; CHECK-LABEL: name: extract_v2s16_v5s16_offset0
     ; CHECK: [[DEF:%[0-9]+]]:_(<6 x s16>) = G_IMPLICIT_DEF
-    ; CHECK: [[EXTRACT:%[0-9]+]]:_(<5 x s16>) = G_EXTRACT [[DEF]](<6 x s16>), 0
-    ; CHECK: [[INSERT:%[0-9]+]]:_(<6 x s16>) = G_INSERT [[DEF]], [[EXTRACT]](<5 x s16>), 0
-    ; CHECK: [[EXTRACT1:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<6 x s16>), 0
-    ; CHECK: $vgpr0 = COPY [[EXTRACT1]](<2 x s16>)
+    ; CHECK: [[DEF1:%[0-9]+]]:_(<6 x s16>) = G_IMPLICIT_DEF
+    ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<30 x s16>) = G_CONCAT_VECTORS [[DEF]](<6 x s16>), [[DEF1]](<6 x s16>), [[DEF1]](<6 x s16>), [[DEF1]](<6 x s16>), [[DEF1]](<6 x s16>)
+    ; CHECK: [[UV:%[0-9]+]]:_(<5 x s16>), [[UV1:%[0-9]+]]:_(<5 x s16>), [[UV2:%[0-9]+]]:_(<5 x s16>), [[UV3:%[0-9]+]]:_(<5 x s16>), [[UV4:%[0-9]+]]:_(<5 x s16>), [[UV5:%[0-9]+]]:_(<5 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<30 x s16>)
+    ; CHECK: [[INSERT:%[0-9]+]]:_(<6 x s16>) = G_INSERT [[DEF1]], [[UV]](<5 x s16>), 0
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<6 x s16>), 0
+    ; CHECK: $vgpr0 = COPY [[EXTRACT]](<2 x s16>)
     %0:_(<5 x s16>) = G_IMPLICIT_DEF
     %1:_(<2 x s16>) = G_EXTRACT %0, 0
     $vgpr0 = COPY %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fabs.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fabs.mir
index e693766954c98..0857d286ff5c9 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fabs.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fabs.mir
@@ -209,13 +209,15 @@ body: |
 
     ; SI-LABEL: name: test_fabs_v3s16
     ; SI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; SI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
-    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s16>), 0
-    ; SI: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
-    ; SI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; SI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; SI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; SI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
+    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
+    ; SI: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; SI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
     ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; SI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
-    ; SI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; SI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
     ; SI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
     ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
     ; SI: [[COPY:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
@@ -231,21 +233,23 @@ body: |
     ; SI: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C]](s32)
     ; SI: [[OR1:%[0-9]+]]:_(s32) = G_OR [[AND2]], [[SHL1]]
     ; SI: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
-    ; SI: [[DEF1:%[0-9]+]]:_(<2 x s16>) = G_IMPLICIT_DEF
+    ; SI: [[DEF2:%[0-9]+]]:_(<2 x s16>) = G_IMPLICIT_DEF
     ; SI: [[FABS:%[0-9]+]]:_(<2 x s16>) = G_FABS [[BITCAST2]]
     ; SI: [[FABS1:%[0-9]+]]:_(<2 x s16>) = G_FABS [[BITCAST3]]
-    ; SI: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[FABS]](<2 x s16>), [[FABS1]](<2 x s16>), [[DEF1]](<2 x s16>)
-    ; SI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<6 x s16>), 0
-    ; SI: S_NOP 0, implicit [[EXTRACT1]](<3 x s16>)
+    ; SI: [[CONCAT_VECTORS1:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[FABS]](<2 x s16>), [[FABS1]](<2 x s16>), [[DEF2]](<2 x s16>)
+    ; SI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS1]](<6 x s16>), 0
+    ; SI: S_NOP 0, implicit [[EXTRACT]](<3 x s16>)
     ; VI-LABEL: name: test_fabs_v3s16
     ; VI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
-    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s16>), 0
-    ; VI: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
-    ; VI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; VI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; VI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; VI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
+    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
+    ; VI: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; VI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
     ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; VI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
-    ; VI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; VI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
     ; VI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
     ; VI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
     ; VI: [[COPY:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
@@ -261,34 +265,36 @@ body: |
     ; VI: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C]](s32)
     ; VI: [[OR1:%[0-9]+]]:_(s32) = G_OR [[AND2]], [[SHL1]]
     ; VI: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
-    ; VI: [[DEF1:%[0-9]+]]:_(<2 x s16>) = G_IMPLICIT_DEF
+    ; VI: [[DEF2:%[0-9]+]]:_(<2 x s16>) = G_IMPLICIT_DEF
     ; VI: [[FABS:%[0-9]+]]:_(<2 x s16>) = G_FABS [[BITCAST2]]
     ; VI: [[FABS1:%[0-9]+]]:_(<2 x s16>) = G_FABS [[BITCAST3]]
-    ; VI: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[FABS]](<2 x s16>), [[FABS1]](<2 x s16>), [[DEF1]](<2 x s16>)
-    ; VI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<6 x s16>), 0
-    ; VI: S_NOP 0, implicit [[EXTRACT1]](<3 x s16>)
+    ; VI: [[CONCAT_VECTORS1:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[FABS]](<2 x s16>), [[FABS1]](<2 x s16>), [[DEF2]](<2 x s16>)
+    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS1]](<6 x s16>), 0
+    ; VI: S_NOP 0, implicit [[EXTRACT]](<3 x s16>)
     ; GFX9-LABEL: name: test_fabs_v3s16
     ; GFX9: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
-    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s16>), 0
-    ; GFX9: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
-    ; GFX9: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; GFX9: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX9: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
+    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
+    ; GFX9: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; GFX9: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
     ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; GFX9: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
-    ; GFX9: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; GFX9: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
     ; GFX9: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
     ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
     ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
     ; GFX9: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY]](s32), [[COPY1]](s32)
     ; GFX9: [[COPY2:%[0-9]+]]:_(s32) = COPY [[BITCAST1]](s32)
-    ; GFX9: [[DEF1:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-    ; GFX9: [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY2]](s32), [[DEF1]](s32)
-    ; GFX9: [[DEF2:%[0-9]+]]:_(<2 x s16>) = G_IMPLICIT_DEF
+    ; GFX9: [[DEF2:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+    ; GFX9: [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY2]](s32), [[DEF2]](s32)
+    ; GFX9: [[DEF3:%[0-9]+]]:_(<2 x s16>) = G_IMPLICIT_DEF
     ; GFX9: [[FABS:%[0-9]+]]:_(<2 x s16>) = G_FABS [[BUILD_VECTOR_TRUNC]]
     ; GFX9: [[FABS1:%[0-9]+]]:_(<2 x s16>) = G_FABS [[BUILD_VECTOR_TRUNC1]]
-    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[FABS]](<2 x s16>), [[FABS1]](<2 x s16>), [[DEF2]](<2 x s16>)
-    ; GFX9: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<6 x s16>), 0
-    ; GFX9: S_NOP 0, implicit [[EXTRACT1]](<3 x s16>)
+    ; GFX9: [[CONCAT_VECTORS1:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[FABS]](<2 x s16>), [[FABS1]](<2 x s16>), [[DEF3]](<2 x s16>)
+    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS1]](<6 x s16>), 0
+    ; GFX9: S_NOP 0, implicit [[EXTRACT]](<3 x s16>)
     %0:_(<3 x s16>) = G_IMPLICIT_DEF
     %1:_(<3 x s16>) = G_FABS %0
     S_NOP 0, implicit %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fadd.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fadd.mir
index 50d846ca2367a..1235d999f2e9c 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fadd.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fadd.mir
@@ -330,26 +330,29 @@ body: |
 
     ; SI-LABEL: name: test_fadd_v3s16
     ; SI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; SI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
     ; SI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; SI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF1]](<4 x s16>), 0
-    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
-    ; SI: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
-    ; SI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; SI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; SI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
+    ; SI: [[DEF2:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; SI: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF2]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; SI: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
+    ; SI: [[UV8:%[0-9]+]]:_(<2 x s16>), [[UV9:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; SI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV8]](<2 x s16>)
     ; SI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
     ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; SI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
     ; SI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-    ; SI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; SI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV9]](<2 x s16>)
     ; SI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
     ; SI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
-    ; SI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
-    ; SI: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
-    ; SI: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
+    ; SI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
+    ; SI: [[UV10:%[0-9]+]]:_(<2 x s16>), [[UV11:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
+    ; SI: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV10]](<2 x s16>)
     ; SI: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
     ; SI: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
     ; SI: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
-    ; SI: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
+    ; SI: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV11]](<2 x s16>)
     ; SI: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
     ; SI: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
     ; SI: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
@@ -374,31 +377,34 @@ body: |
     ; SI: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C]](s32)
     ; SI: [[OR1:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[SHL1]]
     ; SI: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
-    ; SI: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
-    ; SI: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<4 x s16>), 0
-    ; SI: S_NOP 0, implicit [[EXTRACT2]](<3 x s16>)
+    ; SI: [[CONCAT_VECTORS2:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
+    ; SI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS2]](<4 x s16>), 0
+    ; SI: S_NOP 0, implicit [[EXTRACT]](<3 x s16>)
     ; VI-LABEL: name: test_fadd_v3s16
     ; VI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
     ; VI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; VI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF1]](<4 x s16>), 0
-    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
-    ; VI: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
-    ; VI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; VI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; VI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
+    ; VI: [[DEF2:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; VI: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF2]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; VI: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
+    ; VI: [[UV8:%[0-9]+]]:_(<2 x s16>), [[UV9:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; VI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV8]](<2 x s16>)
     ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
     ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; VI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
     ; VI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-    ; VI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; VI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV9]](<2 x s16>)
     ; VI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
     ; VI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
-    ; VI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
-    ; VI: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
-    ; VI: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
+    ; VI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
+    ; VI: [[UV10:%[0-9]+]]:_(<2 x s16>), [[UV11:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
+    ; VI: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV10]](<2 x s16>)
     ; VI: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
     ; VI: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
     ; VI: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
-    ; VI: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
+    ; VI: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV11]](<2 x s16>)
     ; VI: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
     ; VI: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
     ; VI: [[FADD:%[0-9]+]]:_(s16) = G_FADD [[TRUNC]], [[TRUNC3]]
@@ -414,44 +420,47 @@ body: |
     ; VI: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C]](s32)
     ; VI: [[OR1:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[SHL1]]
     ; VI: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
-    ; VI: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
-    ; VI: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<4 x s16>), 0
-    ; VI: S_NOP 0, implicit [[EXTRACT2]](<3 x s16>)
+    ; VI: [[CONCAT_VECTORS2:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
+    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS2]](<4 x s16>), 0
+    ; VI: S_NOP 0, implicit [[EXTRACT]](<3 x s16>)
     ; GFX9-LABEL: name: test_fadd_v3s16
     ; GFX9: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
     ; GFX9: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF1]](<4 x s16>), 0
-    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
-    ; GFX9: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
-    ; GFX9: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX9: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
+    ; GFX9: [[DEF2:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; GFX9: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF2]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX9: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
+    ; GFX9: [[UV8:%[0-9]+]]:_(<2 x s16>), [[UV9:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; GFX9: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV8]](<2 x s16>)
     ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; GFX9: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
-    ; GFX9: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; GFX9: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV9]](<2 x s16>)
     ; GFX9: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
     ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
     ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
     ; GFX9: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY]](s32), [[COPY1]](s32)
     ; GFX9: [[COPY2:%[0-9]+]]:_(s32) = COPY [[BITCAST1]](s32)
-    ; GFX9: [[DEF2:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-    ; GFX9: [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY2]](s32), [[DEF2]](s32)
-    ; GFX9: [[DEF3:%[0-9]+]]:_(<2 x s16>) = G_IMPLICIT_DEF
-    ; GFX9: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
-    ; GFX9: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
-    ; GFX9: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
+    ; GFX9: [[DEF3:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+    ; GFX9: [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY2]](s32), [[DEF3]](s32)
+    ; GFX9: [[DEF4:%[0-9]+]]:_(<2 x s16>) = G_IMPLICIT_DEF
+    ; GFX9: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
+    ; GFX9: [[UV10:%[0-9]+]]:_(<2 x s16>), [[UV11:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
+    ; GFX9: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV10]](<2 x s16>)
     ; GFX9: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
-    ; GFX9: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
+    ; GFX9: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV11]](<2 x s16>)
     ; GFX9: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
     ; GFX9: [[COPY3:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
     ; GFX9: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
     ; GFX9: [[BUILD_VECTOR_TRUNC2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY3]](s32), [[COPY4]](s32)
     ; GFX9: [[COPY5:%[0-9]+]]:_(s32) = COPY [[BITCAST3]](s32)
-    ; GFX9: [[BUILD_VECTOR_TRUNC3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY5]](s32), [[DEF2]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY5]](s32), [[DEF3]](s32)
     ; GFX9: [[FADD:%[0-9]+]]:_(<2 x s16>) = G_FADD [[BUILD_VECTOR_TRUNC]], [[BUILD_VECTOR_TRUNC2]]
     ; GFX9: [[FADD1:%[0-9]+]]:_(<2 x s16>) = G_FADD [[BUILD_VECTOR_TRUNC1]], [[BUILD_VECTOR_TRUNC3]]
-    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[FADD]](<2 x s16>), [[FADD1]](<2 x s16>), [[DEF3]](<2 x s16>)
-    ; GFX9: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<6 x s16>), 0
-    ; GFX9: S_NOP 0, implicit [[EXTRACT2]](<3 x s16>)
+    ; GFX9: [[CONCAT_VECTORS2:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[FADD]](<2 x s16>), [[FADD1]](<2 x s16>), [[DEF4]](<2 x s16>)
+    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS2]](<6 x s16>), 0
+    ; GFX9: S_NOP 0, implicit [[EXTRACT]](<3 x s16>)
     %0:_(<3 x s16>) = G_IMPLICIT_DEF
     %1:_(<3 x s16>) = G_IMPLICIT_DEF
     %2:_(<3 x s16>) = G_FADD %0, %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fcanonicalize.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fcanonicalize.mir
index 9efb04c35f6e9..e5b9fe0b86562 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fcanonicalize.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fcanonicalize.mir
@@ -222,15 +222,17 @@ body: |
 
     ; SI-LABEL: name: test_fcanonicalize_v3s16
     ; SI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; SI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
-    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s16>), 0
-    ; SI: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
-    ; SI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; SI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; SI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; SI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
+    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
+    ; SI: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; SI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
     ; SI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
     ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; SI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
     ; SI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-    ; SI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; SI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
     ; SI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
     ; SI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
     ; SI: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
@@ -252,20 +254,22 @@ body: |
     ; SI: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C]](s32)
     ; SI: [[OR1:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[SHL1]]
     ; SI: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
-    ; SI: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
-    ; SI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<4 x s16>), 0
-    ; SI: S_NOP 0, implicit [[EXTRACT1]](<3 x s16>)
+    ; SI: [[CONCAT_VECTORS1:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
+    ; SI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS1]](<4 x s16>), 0
+    ; SI: S_NOP 0, implicit [[EXTRACT]](<3 x s16>)
     ; VI-LABEL: name: test_fcanonicalize_v3s16
     ; VI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
-    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s16>), 0
-    ; VI: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
-    ; VI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; VI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; VI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; VI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
+    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
+    ; VI: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; VI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
     ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
     ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; VI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
     ; VI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-    ; VI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; VI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
     ; VI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
     ; VI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
     ; VI: [[FCANONICALIZE:%[0-9]+]]:_(s16) = G_FCANONICALIZE [[TRUNC]]
@@ -281,31 +285,33 @@ body: |
     ; VI: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C]](s32)
     ; VI: [[OR1:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[SHL1]]
     ; VI: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
-    ; VI: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
-    ; VI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<4 x s16>), 0
-    ; VI: S_NOP 0, implicit [[EXTRACT1]](<3 x s16>)
+    ; VI: [[CONCAT_VECTORS1:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
+    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS1]](<4 x s16>), 0
+    ; VI: S_NOP 0, implicit [[EXTRACT]](<3 x s16>)
     ; GFX9-LABEL: name: test_fcanonicalize_v3s16
     ; GFX9: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
-    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s16>), 0
-    ; GFX9: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
-    ; GFX9: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; GFX9: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX9: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
+    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
+    ; GFX9: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; GFX9: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
     ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; GFX9: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
-    ; GFX9: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; GFX9: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
     ; GFX9: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
     ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
     ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
     ; GFX9: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY]](s32), [[COPY1]](s32)
     ; GFX9: [[COPY2:%[0-9]+]]:_(s32) = COPY [[BITCAST1]](s32)
-    ; GFX9: [[DEF1:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-    ; GFX9: [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY2]](s32), [[DEF1]](s32)
-    ; GFX9: [[DEF2:%[0-9]+]]:_(<2 x s16>) = G_IMPLICIT_DEF
+    ; GFX9: [[DEF2:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+    ; GFX9: [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY2]](s32), [[DEF2]](s32)
+    ; GFX9: [[DEF3:%[0-9]+]]:_(<2 x s16>) = G_IMPLICIT_DEF
     ; GFX9: [[FCANONICALIZE:%[0-9]+]]:_(<2 x s16>) = G_FCANONICALIZE [[BUILD_VECTOR_TRUNC]]
     ; GFX9: [[FCANONICALIZE1:%[0-9]+]]:_(<2 x s16>) = G_FCANONICALIZE [[BUILD_VECTOR_TRUNC1]]
-    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[FCANONICALIZE]](<2 x s16>), [[FCANONICALIZE1]](<2 x s16>), [[DEF2]](<2 x s16>)
-    ; GFX9: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<6 x s16>), 0
-    ; GFX9: S_NOP 0, implicit [[EXTRACT1]](<3 x s16>)
+    ; GFX9: [[CONCAT_VECTORS1:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[FCANONICALIZE]](<2 x s16>), [[FCANONICALIZE1]](<2 x s16>), [[DEF3]](<2 x s16>)
+    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS1]](<6 x s16>), 0
+    ; GFX9: S_NOP 0, implicit [[EXTRACT]](<3 x s16>)
     %0:_(<3 x s16>) = G_IMPLICIT_DEF
     %1:_(<3 x s16>) = G_FCANONICALIZE %0
     S_NOP 0, implicit %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fcos.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fcos.mir
index a6b5fc876f7d6..f97896ba623d1 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fcos.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fcos.mir
@@ -326,15 +326,17 @@ body: |
 
     ; SI-LABEL: name: test_fcos_v3s16
     ; SI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; SI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
-    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s16>), 0
-    ; SI: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
-    ; SI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; SI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; SI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; SI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
+    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
+    ; SI: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; SI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
     ; SI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
     ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; SI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
     ; SI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-    ; SI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; SI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
     ; SI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
     ; SI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
     ; SI: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
@@ -363,20 +365,22 @@ body: |
     ; SI: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C]](s32)
     ; SI: [[OR1:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[SHL1]]
     ; SI: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
-    ; SI: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
-    ; SI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<4 x s16>), 0
-    ; SI: S_NOP 0, implicit [[EXTRACT1]](<3 x s16>)
+    ; SI: [[CONCAT_VECTORS1:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
+    ; SI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS1]](<4 x s16>), 0
+    ; SI: S_NOP 0, implicit [[EXTRACT]](<3 x s16>)
     ; VI-LABEL: name: test_fcos_v3s16
     ; VI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
-    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s16>), 0
-    ; VI: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
-    ; VI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; VI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; VI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; VI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
+    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
+    ; VI: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; VI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
     ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
     ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; VI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
     ; VI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-    ; VI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; VI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
     ; VI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
     ; VI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
     ; VI: [[C1:%[0-9]+]]:_(s16) = G_FCONSTANT half 0xH3118
@@ -399,20 +403,22 @@ body: |
     ; VI: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C]](s32)
     ; VI: [[OR1:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[SHL1]]
     ; VI: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
-    ; VI: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
-    ; VI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<4 x s16>), 0
-    ; VI: S_NOP 0, implicit [[EXTRACT1]](<3 x s16>)
+    ; VI: [[CONCAT_VECTORS1:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
+    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS1]](<4 x s16>), 0
+    ; VI: S_NOP 0, implicit [[EXTRACT]](<3 x s16>)
     ; GFX9-LABEL: name: test_fcos_v3s16
     ; GFX9: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
-    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s16>), 0
-    ; GFX9: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
-    ; GFX9: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; GFX9: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX9: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
+    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
+    ; GFX9: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; GFX9: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
     ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
     ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; GFX9: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
     ; GFX9: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-    ; GFX9: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; GFX9: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
     ; GFX9: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
     ; GFX9: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
     ; GFX9: [[C1:%[0-9]+]]:_(s16) = G_FCONSTANT half 0xH3118
@@ -426,11 +432,11 @@ body: |
     ; GFX9: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[INT1]](s16)
     ; GFX9: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[ANYEXT]](s32), [[ANYEXT1]](s32)
     ; GFX9: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[INT2]](s16)
-    ; GFX9: [[DEF1:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-    ; GFX9: [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[ANYEXT2]](s32), [[DEF1]](s32)
-    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
-    ; GFX9: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<4 x s16>), 0
-    ; GFX9: S_NOP 0, implicit [[EXTRACT1]](<3 x s16>)
+    ; GFX9: [[DEF2:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+    ; GFX9: [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[ANYEXT2]](s32), [[DEF2]](s32)
+    ; GFX9: [[CONCAT_VECTORS1:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
+    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS1]](<4 x s16>), 0
+    ; GFX9: S_NOP 0, implicit [[EXTRACT]](<3 x s16>)
     %0:_(<3 x s16>) = G_IMPLICIT_DEF
     %1:_(<3 x s16>) = G_FCOS %0
     S_NOP 0, implicit %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fdiv.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fdiv.mir
index 3ed48e39e57e3..3236b8c5a3d06 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fdiv.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fdiv.mir
@@ -1307,26 +1307,29 @@ body: |
 
     ; SI-LABEL: name: test_fdiv_v3s16
     ; SI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; SI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
     ; SI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; SI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF1]](<4 x s16>), 0
-    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
-    ; SI: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
-    ; SI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; SI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; SI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
+    ; SI: [[DEF2:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; SI: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF2]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; SI: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
+    ; SI: [[UV8:%[0-9]+]]:_(<2 x s16>), [[UV9:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; SI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV8]](<2 x s16>)
     ; SI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
     ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; SI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
     ; SI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-    ; SI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; SI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV9]](<2 x s16>)
     ; SI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
     ; SI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
-    ; SI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
-    ; SI: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
-    ; SI: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
+    ; SI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
+    ; SI: [[UV10:%[0-9]+]]:_(<2 x s16>), [[UV11:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
+    ; SI: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV10]](<2 x s16>)
     ; SI: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
     ; SI: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
     ; SI: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
-    ; SI: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
+    ; SI: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV11]](<2 x s16>)
     ; SI: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
     ; SI: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
     ; SI: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
@@ -1385,31 +1388,34 @@ body: |
     ; SI: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C]](s32)
     ; SI: [[OR1:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[SHL1]]
     ; SI: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
-    ; SI: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
-    ; SI: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<4 x s16>), 0
-    ; SI: S_NOP 0, implicit [[EXTRACT2]](<3 x s16>)
+    ; SI: [[CONCAT_VECTORS2:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
+    ; SI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS2]](<4 x s16>), 0
+    ; SI: S_NOP 0, implicit [[EXTRACT]](<3 x s16>)
     ; VI-LABEL: name: test_fdiv_v3s16
     ; VI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
     ; VI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; VI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF1]](<4 x s16>), 0
-    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
-    ; VI: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
-    ; VI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; VI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; VI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
+    ; VI: [[DEF2:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; VI: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF2]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; VI: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
+    ; VI: [[UV8:%[0-9]+]]:_(<2 x s16>), [[UV9:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; VI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV8]](<2 x s16>)
     ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
     ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; VI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
     ; VI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-    ; VI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; VI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV9]](<2 x s16>)
     ; VI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
     ; VI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
-    ; VI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
-    ; VI: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
-    ; VI: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
+    ; VI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
+    ; VI: [[UV10:%[0-9]+]]:_(<2 x s16>), [[UV11:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
+    ; VI: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV10]](<2 x s16>)
     ; VI: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
     ; VI: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
     ; VI: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
-    ; VI: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
+    ; VI: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV11]](<2 x s16>)
     ; VI: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
     ; VI: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
     ; VI: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
@@ -1440,31 +1446,34 @@ body: |
     ; VI: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C]](s32)
     ; VI: [[OR1:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[SHL1]]
     ; VI: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
-    ; VI: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
-    ; VI: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<4 x s16>), 0
-    ; VI: S_NOP 0, implicit [[EXTRACT2]](<3 x s16>)
+    ; VI: [[CONCAT_VECTORS2:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
+    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS2]](<4 x s16>), 0
+    ; VI: S_NOP 0, implicit [[EXTRACT]](<3 x s16>)
     ; GFX9-LABEL: name: test_fdiv_v3s16
     ; GFX9: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
     ; GFX9: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF1]](<4 x s16>), 0
-    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
-    ; GFX9: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
-    ; GFX9: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX9: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
+    ; GFX9: [[DEF2:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; GFX9: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF2]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX9: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
+    ; GFX9: [[UV8:%[0-9]+]]:_(<2 x s16>), [[UV9:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; GFX9: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV8]](<2 x s16>)
     ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
     ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; GFX9: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
     ; GFX9: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-    ; GFX9: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; GFX9: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV9]](<2 x s16>)
     ; GFX9: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
     ; GFX9: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
-    ; GFX9: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
-    ; GFX9: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
-    ; GFX9: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
+    ; GFX9: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
+    ; GFX9: [[UV10:%[0-9]+]]:_(<2 x s16>), [[UV11:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
+    ; GFX9: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV10]](<2 x s16>)
     ; GFX9: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
     ; GFX9: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
     ; GFX9: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
-    ; GFX9: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
+    ; GFX9: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV11]](<2 x s16>)
     ; GFX9: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
     ; GFX9: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
     ; GFX9: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
@@ -1489,33 +1498,36 @@ body: |
     ; GFX9: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[INT3]](s16)
     ; GFX9: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[ANYEXT]](s32), [[ANYEXT1]](s32)
     ; GFX9: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[INT5]](s16)
-    ; GFX9: [[DEF2:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-    ; GFX9: [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[ANYEXT2]](s32), [[DEF2]](s32)
-    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
-    ; GFX9: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<4 x s16>), 0
-    ; GFX9: S_NOP 0, implicit [[EXTRACT2]](<3 x s16>)
+    ; GFX9: [[DEF3:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+    ; GFX9: [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[ANYEXT2]](s32), [[DEF3]](s32)
+    ; GFX9: [[CONCAT_VECTORS2:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
+    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS2]](<4 x s16>), 0
+    ; GFX9: S_NOP 0, implicit [[EXTRACT]](<3 x s16>)
     ; GFX9-UNSAFE-LABEL: name: test_fdiv_v3s16
     ; GFX9-UNSAFE: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9-UNSAFE: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
     ; GFX9-UNSAFE: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9-UNSAFE: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF1]](<4 x s16>), 0
-    ; GFX9-UNSAFE: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
-    ; GFX9-UNSAFE: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
-    ; GFX9-UNSAFE: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; GFX9-UNSAFE: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX9-UNSAFE: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
+    ; GFX9-UNSAFE: [[DEF2:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; GFX9-UNSAFE: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF2]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX9-UNSAFE: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; GFX9-UNSAFE: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
+    ; GFX9-UNSAFE: [[UV8:%[0-9]+]]:_(<2 x s16>), [[UV9:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; GFX9-UNSAFE: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV8]](<2 x s16>)
     ; GFX9-UNSAFE: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
     ; GFX9-UNSAFE: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; GFX9-UNSAFE: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
     ; GFX9-UNSAFE: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-    ; GFX9-UNSAFE: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; GFX9-UNSAFE: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV9]](<2 x s16>)
     ; GFX9-UNSAFE: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
     ; GFX9-UNSAFE: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
-    ; GFX9-UNSAFE: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
-    ; GFX9-UNSAFE: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
-    ; GFX9-UNSAFE: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
+    ; GFX9-UNSAFE: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
+    ; GFX9-UNSAFE: [[UV10:%[0-9]+]]:_(<2 x s16>), [[UV11:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
+    ; GFX9-UNSAFE: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV10]](<2 x s16>)
     ; GFX9-UNSAFE: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
     ; GFX9-UNSAFE: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
     ; GFX9-UNSAFE: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
-    ; GFX9-UNSAFE: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
+    ; GFX9-UNSAFE: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV11]](<2 x s16>)
     ; GFX9-UNSAFE: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
     ; GFX9-UNSAFE: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
     ; GFX9-UNSAFE: [[INT:%[0-9]+]]:_(s16) = G_INTRINSIC intrinsic(@llvm.amdgcn.rcp), [[TRUNC3]](s16)
@@ -1528,33 +1540,36 @@ body: |
     ; GFX9-UNSAFE: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[FMUL1]](s16)
     ; GFX9-UNSAFE: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[ANYEXT]](s32), [[ANYEXT1]](s32)
     ; GFX9-UNSAFE: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[FMUL2]](s16)
-    ; GFX9-UNSAFE: [[DEF2:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-    ; GFX9-UNSAFE: [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[ANYEXT2]](s32), [[DEF2]](s32)
-    ; GFX9-UNSAFE: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
-    ; GFX9-UNSAFE: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<4 x s16>), 0
-    ; GFX9-UNSAFE: S_NOP 0, implicit [[EXTRACT2]](<3 x s16>)
+    ; GFX9-UNSAFE: [[DEF3:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+    ; GFX9-UNSAFE: [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[ANYEXT2]](s32), [[DEF3]](s32)
+    ; GFX9-UNSAFE: [[CONCAT_VECTORS2:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
+    ; GFX9-UNSAFE: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS2]](<4 x s16>), 0
+    ; GFX9-UNSAFE: S_NOP 0, implicit [[EXTRACT]](<3 x s16>)
     ; GFX10-LABEL: name: test_fdiv_v3s16
     ; GFX10: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX10: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
     ; GFX10: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX10: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF1]](<4 x s16>), 0
-    ; GFX10: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
-    ; GFX10: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
-    ; GFX10: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; GFX10: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX10: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
+    ; GFX10: [[DEF2:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; GFX10: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF2]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX10: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; GFX10: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
+    ; GFX10: [[UV8:%[0-9]+]]:_(<2 x s16>), [[UV9:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; GFX10: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV8]](<2 x s16>)
     ; GFX10: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
     ; GFX10: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; GFX10: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
     ; GFX10: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-    ; GFX10: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; GFX10: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV9]](<2 x s16>)
     ; GFX10: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
     ; GFX10: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
-    ; GFX10: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
-    ; GFX10: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
-    ; GFX10: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
+    ; GFX10: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
+    ; GFX10: [[UV10:%[0-9]+]]:_(<2 x s16>), [[UV11:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
+    ; GFX10: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV10]](<2 x s16>)
     ; GFX10: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
     ; GFX10: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
     ; GFX10: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
-    ; GFX10: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
+    ; GFX10: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV11]](<2 x s16>)
     ; GFX10: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
     ; GFX10: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
     ; GFX10: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
@@ -1579,11 +1594,11 @@ body: |
     ; GFX10: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[INT3]](s16)
     ; GFX10: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[ANYEXT]](s32), [[ANYEXT1]](s32)
     ; GFX10: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[INT5]](s16)
-    ; GFX10: [[DEF2:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-    ; GFX10: [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[ANYEXT2]](s32), [[DEF2]](s32)
-    ; GFX10: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
-    ; GFX10: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<4 x s16>), 0
-    ; GFX10: S_NOP 0, implicit [[EXTRACT2]](<3 x s16>)
+    ; GFX10: [[DEF3:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+    ; GFX10: [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[ANYEXT2]](s32), [[DEF3]](s32)
+    ; GFX10: [[CONCAT_VECTORS2:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
+    ; GFX10: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS2]](<4 x s16>), 0
+    ; GFX10: S_NOP 0, implicit [[EXTRACT]](<3 x s16>)
     %0:_(<3 x s16>) = G_IMPLICIT_DEF
     %1:_(<3 x s16>) = G_IMPLICIT_DEF
     %2:_(<3 x s16>) = G_FDIV %0, %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ffloor.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ffloor.mir
index b1eec4dfee3d3..4466b4c521c56 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ffloor.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ffloor.mir
@@ -322,15 +322,17 @@ body: |
 
     ; SI-LABEL: name: test_ffloor_v3s16
     ; SI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; SI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
-    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s16>), 0
-    ; SI: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
-    ; SI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; SI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; SI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; SI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
+    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
+    ; SI: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; SI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
     ; SI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
     ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; SI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
     ; SI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-    ; SI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; SI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
     ; SI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
     ; SI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
     ; SI: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
@@ -352,20 +354,22 @@ body: |
     ; SI: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C]](s32)
     ; SI: [[OR1:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[SHL1]]
     ; SI: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
-    ; SI: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
-    ; SI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<4 x s16>), 0
-    ; SI: S_NOP 0, implicit [[EXTRACT1]](<3 x s16>)
+    ; SI: [[CONCAT_VECTORS1:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
+    ; SI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS1]](<4 x s16>), 0
+    ; SI: S_NOP 0, implicit [[EXTRACT]](<3 x s16>)
     ; VI-LABEL: name: test_ffloor_v3s16
     ; VI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
-    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s16>), 0
-    ; VI: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
-    ; VI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; VI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; VI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; VI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
+    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
+    ; VI: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; VI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
     ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
     ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; VI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
     ; VI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-    ; VI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; VI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
     ; VI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
     ; VI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
     ; VI: [[FFLOOR:%[0-9]+]]:_(s16) = G_FFLOOR [[TRUNC]]
@@ -381,20 +385,22 @@ body: |
     ; VI: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C]](s32)
     ; VI: [[OR1:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[SHL1]]
     ; VI: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
-    ; VI: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
-    ; VI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<4 x s16>), 0
-    ; VI: S_NOP 0, implicit [[EXTRACT1]](<3 x s16>)
+    ; VI: [[CONCAT_VECTORS1:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
+    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS1]](<4 x s16>), 0
+    ; VI: S_NOP 0, implicit [[EXTRACT]](<3 x s16>)
     ; GFX9-LABEL: name: test_ffloor_v3s16
     ; GFX9: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
-    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s16>), 0
-    ; GFX9: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
-    ; GFX9: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; GFX9: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX9: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
+    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
+    ; GFX9: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; GFX9: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
     ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
     ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; GFX9: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
     ; GFX9: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-    ; GFX9: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; GFX9: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
     ; GFX9: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
     ; GFX9: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
     ; GFX9: [[FFLOOR:%[0-9]+]]:_(s16) = G_FFLOOR [[TRUNC]]
@@ -404,11 +410,11 @@ body: |
     ; GFX9: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[FFLOOR1]](s16)
     ; GFX9: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[ANYEXT]](s32), [[ANYEXT1]](s32)
     ; GFX9: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[FFLOOR2]](s16)
-    ; GFX9: [[DEF1:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-    ; GFX9: [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[ANYEXT2]](s32), [[DEF1]](s32)
-    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
-    ; GFX9: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<4 x s16>), 0
-    ; GFX9: S_NOP 0, implicit [[EXTRACT1]](<3 x s16>)
+    ; GFX9: [[DEF2:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+    ; GFX9: [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[ANYEXT2]](s32), [[DEF2]](s32)
+    ; GFX9: [[CONCAT_VECTORS1:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
+    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS1]](<4 x s16>), 0
+    ; GFX9: S_NOP 0, implicit [[EXTRACT]](<3 x s16>)
     %0:_(<3 x s16>) = G_IMPLICIT_DEF
     %1:_(<3 x s16>) = G_FFLOOR %0
     S_NOP 0, implicit %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fma.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fma.mir
index 06f237f7a3050..56c51dfa726e3 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fma.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fma.mir
@@ -393,37 +393,41 @@ body: |
 
     ; SI-LABEL: name: test_fma_v3s16
     ; SI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; SI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
     ; SI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; SI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF1]](<4 x s16>), 0
+    ; SI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; SI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; SI: [[DEF2:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; SI: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF2]](<4 x s16>), 0
-    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF2]], [[EXTRACT]](<3 x s16>), 0
-    ; SI: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
-    ; SI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; SI: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF2]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; SI: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; SI: [[DEF3:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; SI: [[CONCAT_VECTORS2:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF3]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; SI: [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>), [[UV10:%[0-9]+]]:_(<3 x s16>), [[UV11:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS2]](<12 x s16>)
+    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
+    ; SI: [[UV12:%[0-9]+]]:_(<2 x s16>), [[UV13:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; SI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV12]](<2 x s16>)
     ; SI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
     ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; SI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
     ; SI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-    ; SI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; SI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV13]](<2 x s16>)
     ; SI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
     ; SI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
-    ; SI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF2]], [[EXTRACT1]](<3 x s16>), 0
-    ; SI: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
-    ; SI: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
+    ; SI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
+    ; SI: [[UV14:%[0-9]+]]:_(<2 x s16>), [[UV15:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
+    ; SI: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV14]](<2 x s16>)
     ; SI: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
     ; SI: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
     ; SI: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
-    ; SI: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
+    ; SI: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV15]](<2 x s16>)
     ; SI: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
     ; SI: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
-    ; SI: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF2]], [[EXTRACT2]](<3 x s16>), 0
-    ; SI: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT2]](<4 x s16>)
-    ; SI: [[BITCAST4:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
+    ; SI: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV8]](<3 x s16>), 0
+    ; SI: [[UV16:%[0-9]+]]:_(<2 x s16>), [[UV17:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT2]](<4 x s16>)
+    ; SI: [[BITCAST4:%[0-9]+]]:_(s32) = G_BITCAST [[UV16]](<2 x s16>)
     ; SI: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST4]](s32)
     ; SI: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST4]], [[C]](s32)
     ; SI: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR4]](s32)
-    ; SI: [[BITCAST5:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
+    ; SI: [[BITCAST5:%[0-9]+]]:_(s32) = G_BITCAST [[UV17]](<2 x s16>)
     ; SI: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST5]](s32)
     ; SI: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST5]], [[C]](s32)
     ; SI: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
@@ -451,42 +455,46 @@ body: |
     ; SI: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C]](s32)
     ; SI: [[OR1:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[SHL1]]
     ; SI: [[BITCAST7:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
-    ; SI: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST6]](<2 x s16>), [[BITCAST7]](<2 x s16>)
-    ; SI: [[EXTRACT3:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<4 x s16>), 0
-    ; SI: S_NOP 0, implicit [[EXTRACT3]](<3 x s16>)
+    ; SI: [[CONCAT_VECTORS3:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST6]](<2 x s16>), [[BITCAST7]](<2 x s16>)
+    ; SI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS3]](<4 x s16>), 0
+    ; SI: S_NOP 0, implicit [[EXTRACT]](<3 x s16>)
     ; VI-LABEL: name: test_fma_v3s16
     ; VI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
     ; VI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; VI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF1]](<4 x s16>), 0
+    ; VI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; VI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; VI: [[DEF2:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; VI: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF2]](<4 x s16>), 0
-    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF2]], [[EXTRACT]](<3 x s16>), 0
-    ; VI: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
-    ; VI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; VI: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF2]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; VI: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; VI: [[DEF3:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; VI: [[CONCAT_VECTORS2:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF3]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; VI: [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>), [[UV10:%[0-9]+]]:_(<3 x s16>), [[UV11:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS2]](<12 x s16>)
+    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
+    ; VI: [[UV12:%[0-9]+]]:_(<2 x s16>), [[UV13:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; VI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV12]](<2 x s16>)
     ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
     ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; VI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
     ; VI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-    ; VI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; VI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV13]](<2 x s16>)
     ; VI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
     ; VI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
-    ; VI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF2]], [[EXTRACT1]](<3 x s16>), 0
-    ; VI: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
-    ; VI: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
+    ; VI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
+    ; VI: [[UV14:%[0-9]+]]:_(<2 x s16>), [[UV15:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
+    ; VI: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV14]](<2 x s16>)
     ; VI: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
     ; VI: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
     ; VI: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
-    ; VI: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
+    ; VI: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV15]](<2 x s16>)
     ; VI: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
     ; VI: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
-    ; VI: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF2]], [[EXTRACT2]](<3 x s16>), 0
-    ; VI: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT2]](<4 x s16>)
-    ; VI: [[BITCAST4:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
+    ; VI: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV8]](<3 x s16>), 0
+    ; VI: [[UV16:%[0-9]+]]:_(<2 x s16>), [[UV17:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT2]](<4 x s16>)
+    ; VI: [[BITCAST4:%[0-9]+]]:_(s32) = G_BITCAST [[UV16]](<2 x s16>)
     ; VI: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST4]](s32)
     ; VI: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST4]], [[C]](s32)
     ; VI: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR4]](s32)
-    ; VI: [[BITCAST5:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
+    ; VI: [[BITCAST5:%[0-9]+]]:_(s32) = G_BITCAST [[UV17]](<2 x s16>)
     ; VI: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST5]](s32)
     ; VI: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST5]], [[C]](s32)
     ; VI: [[FMA:%[0-9]+]]:_(s16) = G_FMA [[TRUNC]], [[TRUNC3]], [[TRUNC6]]
@@ -502,57 +510,61 @@ body: |
     ; VI: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C]](s32)
     ; VI: [[OR1:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[SHL1]]
     ; VI: [[BITCAST7:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
-    ; VI: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST6]](<2 x s16>), [[BITCAST7]](<2 x s16>)
-    ; VI: [[EXTRACT3:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<4 x s16>), 0
-    ; VI: S_NOP 0, implicit [[EXTRACT3]](<3 x s16>)
+    ; VI: [[CONCAT_VECTORS3:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST6]](<2 x s16>), [[BITCAST7]](<2 x s16>)
+    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS3]](<4 x s16>), 0
+    ; VI: S_NOP 0, implicit [[EXTRACT]](<3 x s16>)
     ; GFX9-LABEL: name: test_fma_v3s16
     ; GFX9: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
     ; GFX9: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF1]](<4 x s16>), 0
+    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX9: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; GFX9: [[DEF2:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF2]](<4 x s16>), 0
-    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF2]], [[EXTRACT]](<3 x s16>), 0
-    ; GFX9: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
-    ; GFX9: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; GFX9: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF2]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX9: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; GFX9: [[DEF3:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; GFX9: [[CONCAT_VECTORS2:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF3]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX9: [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>), [[UV10:%[0-9]+]]:_(<3 x s16>), [[UV11:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS2]](<12 x s16>)
+    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
+    ; GFX9: [[UV12:%[0-9]+]]:_(<2 x s16>), [[UV13:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; GFX9: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV12]](<2 x s16>)
     ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; GFX9: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
-    ; GFX9: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; GFX9: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV13]](<2 x s16>)
     ; GFX9: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
     ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
     ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
     ; GFX9: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY]](s32), [[COPY1]](s32)
     ; GFX9: [[COPY2:%[0-9]+]]:_(s32) = COPY [[BITCAST1]](s32)
-    ; GFX9: [[DEF3:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-    ; GFX9: [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY2]](s32), [[DEF3]](s32)
-    ; GFX9: [[DEF4:%[0-9]+]]:_(<2 x s16>) = G_IMPLICIT_DEF
-    ; GFX9: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF2]], [[EXTRACT1]](<3 x s16>), 0
-    ; GFX9: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
-    ; GFX9: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
+    ; GFX9: [[DEF4:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+    ; GFX9: [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY2]](s32), [[DEF4]](s32)
+    ; GFX9: [[DEF5:%[0-9]+]]:_(<2 x s16>) = G_IMPLICIT_DEF
+    ; GFX9: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
+    ; GFX9: [[UV14:%[0-9]+]]:_(<2 x s16>), [[UV15:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
+    ; GFX9: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV14]](<2 x s16>)
     ; GFX9: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
-    ; GFX9: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
+    ; GFX9: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV15]](<2 x s16>)
     ; GFX9: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
     ; GFX9: [[COPY3:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
     ; GFX9: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
     ; GFX9: [[BUILD_VECTOR_TRUNC2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY3]](s32), [[COPY4]](s32)
     ; GFX9: [[COPY5:%[0-9]+]]:_(s32) = COPY [[BITCAST3]](s32)
-    ; GFX9: [[BUILD_VECTOR_TRUNC3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY5]](s32), [[DEF3]](s32)
-    ; GFX9: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF2]], [[EXTRACT2]](<3 x s16>), 0
-    ; GFX9: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT2]](<4 x s16>)
-    ; GFX9: [[BITCAST4:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
+    ; GFX9: [[BUILD_VECTOR_TRUNC3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY5]](s32), [[DEF4]](s32)
+    ; GFX9: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV8]](<3 x s16>), 0
+    ; GFX9: [[UV16:%[0-9]+]]:_(<2 x s16>), [[UV17:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT2]](<4 x s16>)
+    ; GFX9: [[BITCAST4:%[0-9]+]]:_(s32) = G_BITCAST [[UV16]](<2 x s16>)
     ; GFX9: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST4]], [[C]](s32)
-    ; GFX9: [[BITCAST5:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
+    ; GFX9: [[BITCAST5:%[0-9]+]]:_(s32) = G_BITCAST [[UV17]](<2 x s16>)
     ; GFX9: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST5]], [[C]](s32)
     ; GFX9: [[COPY6:%[0-9]+]]:_(s32) = COPY [[BITCAST4]](s32)
     ; GFX9: [[COPY7:%[0-9]+]]:_(s32) = COPY [[LSHR4]](s32)
     ; GFX9: [[BUILD_VECTOR_TRUNC4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY6]](s32), [[COPY7]](s32)
     ; GFX9: [[COPY8:%[0-9]+]]:_(s32) = COPY [[BITCAST5]](s32)
-    ; GFX9: [[BUILD_VECTOR_TRUNC5:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY8]](s32), [[DEF3]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC5:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY8]](s32), [[DEF4]](s32)
     ; GFX9: [[FMA:%[0-9]+]]:_(<2 x s16>) = G_FMA [[BUILD_VECTOR_TRUNC]], [[BUILD_VECTOR_TRUNC2]], [[BUILD_VECTOR_TRUNC4]]
     ; GFX9: [[FMA1:%[0-9]+]]:_(<2 x s16>) = G_FMA [[BUILD_VECTOR_TRUNC1]], [[BUILD_VECTOR_TRUNC3]], [[BUILD_VECTOR_TRUNC5]]
-    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[FMA]](<2 x s16>), [[FMA1]](<2 x s16>), [[DEF4]](<2 x s16>)
-    ; GFX9: [[EXTRACT3:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<6 x s16>), 0
-    ; GFX9: S_NOP 0, implicit [[EXTRACT3]](<3 x s16>)
+    ; GFX9: [[CONCAT_VECTORS3:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[FMA]](<2 x s16>), [[FMA1]](<2 x s16>), [[DEF5]](<2 x s16>)
+    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS3]](<6 x s16>), 0
+    ; GFX9: S_NOP 0, implicit [[EXTRACT]](<3 x s16>)
     %0:_(<3 x s16>) = G_IMPLICIT_DEF
     %1:_(<3 x s16>) = G_IMPLICIT_DEF
     %2:_(<3 x s16>) = G_IMPLICIT_DEF
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmaxnum.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmaxnum.mir
index 8b9b0e972e6f4..24253aaeb4c38 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmaxnum.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmaxnum.mir
@@ -506,8 +506,9 @@ body: |
     ; GFX9: [[FCANONICALIZE3:%[0-9]+]]:_(<2 x s16>) = G_FCANONICALIZE [[UV3]]
     ; GFX9: [[FMAXNUM_IEEE1:%[0-9]+]]:_(<2 x s16>) = G_FMAXNUM_IEEE [[FCANONICALIZE2]], [[FCANONICALIZE3]]
     ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[FMAXNUM_IEEE]](<2 x s16>), [[FMAXNUM_IEEE1]](<2 x s16>)
-    ; GFX9: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<4 x s16>), 0
-    ; GFX9: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT2]](<3 x s16>), 0
+    ; GFX9: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[CONCAT_VECTORS]](<4 x s16>), [[DEF]](<4 x s16>), [[DEF]](<4 x s16>)
+    ; GFX9: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; GFX9: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV4]](<3 x s16>), 0
     ; GFX9: $vgpr0_vgpr1 = COPY [[INSERT2]](<4 x s16>)
     %0:_(<4 x s16>) = COPY $vgpr0_vgpr1
     %1:_(<4 x s16>) = COPY $vgpr2_vgpr3
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fminnum.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fminnum.mir
index f6456cd57f01e..723168a2a8ec3 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fminnum.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fminnum.mir
@@ -506,8 +506,9 @@ body: |
     ; GFX9: [[FCANONICALIZE3:%[0-9]+]]:_(<2 x s16>) = G_FCANONICALIZE [[UV3]]
     ; GFX9: [[FMINNUM_IEEE1:%[0-9]+]]:_(<2 x s16>) = G_FMINNUM_IEEE [[FCANONICALIZE2]], [[FCANONICALIZE3]]
     ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[FMINNUM_IEEE]](<2 x s16>), [[FMINNUM_IEEE1]](<2 x s16>)
-    ; GFX9: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<4 x s16>), 0
-    ; GFX9: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT2]](<3 x s16>), 0
+    ; GFX9: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[CONCAT_VECTORS]](<4 x s16>), [[DEF]](<4 x s16>), [[DEF]](<4 x s16>)
+    ; GFX9: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; GFX9: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV4]](<3 x s16>), 0
     ; GFX9: $vgpr0_vgpr1 = COPY [[INSERT2]](<4 x s16>)
     %0:_(<4 x s16>) = COPY $vgpr0_vgpr1
     %1:_(<4 x s16>) = COPY $vgpr2_vgpr3
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmul.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmul.mir
index 45130639f696f..ea9b1e66d7e13 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmul.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmul.mir
@@ -329,26 +329,29 @@ body: |
 
     ; SI-LABEL: name: test_fmul_v3s16
     ; SI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; SI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
     ; SI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; SI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF1]](<4 x s16>), 0
-    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
-    ; SI: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
-    ; SI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; SI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; SI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
+    ; SI: [[DEF2:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; SI: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF2]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; SI: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
+    ; SI: [[UV8:%[0-9]+]]:_(<2 x s16>), [[UV9:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; SI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV8]](<2 x s16>)
     ; SI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
     ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; SI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
     ; SI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-    ; SI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; SI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV9]](<2 x s16>)
     ; SI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
     ; SI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
-    ; SI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
-    ; SI: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
-    ; SI: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
+    ; SI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
+    ; SI: [[UV10:%[0-9]+]]:_(<2 x s16>), [[UV11:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
+    ; SI: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV10]](<2 x s16>)
     ; SI: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
     ; SI: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
     ; SI: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
-    ; SI: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
+    ; SI: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV11]](<2 x s16>)
     ; SI: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
     ; SI: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
     ; SI: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
@@ -373,31 +376,34 @@ body: |
     ; SI: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C]](s32)
     ; SI: [[OR1:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[SHL1]]
     ; SI: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
-    ; SI: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
-    ; SI: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<4 x s16>), 0
-    ; SI: S_NOP 0, implicit [[EXTRACT2]](<3 x s16>)
+    ; SI: [[CONCAT_VECTORS2:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
+    ; SI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS2]](<4 x s16>), 0
+    ; SI: S_NOP 0, implicit [[EXTRACT]](<3 x s16>)
     ; VI-LABEL: name: test_fmul_v3s16
     ; VI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
     ; VI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; VI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF1]](<4 x s16>), 0
-    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
-    ; VI: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
-    ; VI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; VI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; VI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
+    ; VI: [[DEF2:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; VI: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF2]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; VI: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
+    ; VI: [[UV8:%[0-9]+]]:_(<2 x s16>), [[UV9:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; VI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV8]](<2 x s16>)
     ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
     ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; VI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
     ; VI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-    ; VI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; VI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV9]](<2 x s16>)
     ; VI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
     ; VI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
-    ; VI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
-    ; VI: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
-    ; VI: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
+    ; VI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
+    ; VI: [[UV10:%[0-9]+]]:_(<2 x s16>), [[UV11:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
+    ; VI: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV10]](<2 x s16>)
     ; VI: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
     ; VI: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
     ; VI: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
-    ; VI: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
+    ; VI: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV11]](<2 x s16>)
     ; VI: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
     ; VI: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
     ; VI: [[FMUL:%[0-9]+]]:_(s16) = G_FMUL [[TRUNC]], [[TRUNC3]]
@@ -413,44 +419,47 @@ body: |
     ; VI: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C]](s32)
     ; VI: [[OR1:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[SHL1]]
     ; VI: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
-    ; VI: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
-    ; VI: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<4 x s16>), 0
-    ; VI: S_NOP 0, implicit [[EXTRACT2]](<3 x s16>)
+    ; VI: [[CONCAT_VECTORS2:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
+    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS2]](<4 x s16>), 0
+    ; VI: S_NOP 0, implicit [[EXTRACT]](<3 x s16>)
     ; GFX9-LABEL: name: test_fmul_v3s16
     ; GFX9: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
     ; GFX9: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF1]](<4 x s16>), 0
-    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
-    ; GFX9: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
-    ; GFX9: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX9: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
+    ; GFX9: [[DEF2:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; GFX9: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF2]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX9: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
+    ; GFX9: [[UV8:%[0-9]+]]:_(<2 x s16>), [[UV9:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; GFX9: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV8]](<2 x s16>)
     ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; GFX9: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
-    ; GFX9: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; GFX9: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV9]](<2 x s16>)
     ; GFX9: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
     ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
     ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
     ; GFX9: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY]](s32), [[COPY1]](s32)
     ; GFX9: [[COPY2:%[0-9]+]]:_(s32) = COPY [[BITCAST1]](s32)
-    ; GFX9: [[DEF2:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-    ; GFX9: [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY2]](s32), [[DEF2]](s32)
-    ; GFX9: [[DEF3:%[0-9]+]]:_(<2 x s16>) = G_IMPLICIT_DEF
-    ; GFX9: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
-    ; GFX9: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
-    ; GFX9: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
+    ; GFX9: [[DEF3:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+    ; GFX9: [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY2]](s32), [[DEF3]](s32)
+    ; GFX9: [[DEF4:%[0-9]+]]:_(<2 x s16>) = G_IMPLICIT_DEF
+    ; GFX9: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
+    ; GFX9: [[UV10:%[0-9]+]]:_(<2 x s16>), [[UV11:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
+    ; GFX9: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV10]](<2 x s16>)
     ; GFX9: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
-    ; GFX9: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
+    ; GFX9: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV11]](<2 x s16>)
     ; GFX9: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
     ; GFX9: [[COPY3:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
     ; GFX9: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
     ; GFX9: [[BUILD_VECTOR_TRUNC2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY3]](s32), [[COPY4]](s32)
     ; GFX9: [[COPY5:%[0-9]+]]:_(s32) = COPY [[BITCAST3]](s32)
-    ; GFX9: [[BUILD_VECTOR_TRUNC3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY5]](s32), [[DEF2]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY5]](s32), [[DEF3]](s32)
     ; GFX9: [[FMUL:%[0-9]+]]:_(<2 x s16>) = G_FMUL [[BUILD_VECTOR_TRUNC]], [[BUILD_VECTOR_TRUNC2]]
     ; GFX9: [[FMUL1:%[0-9]+]]:_(<2 x s16>) = G_FMUL [[BUILD_VECTOR_TRUNC1]], [[BUILD_VECTOR_TRUNC3]]
-    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[FMUL]](<2 x s16>), [[FMUL1]](<2 x s16>), [[DEF3]](<2 x s16>)
-    ; GFX9: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<6 x s16>), 0
-    ; GFX9: S_NOP 0, implicit [[EXTRACT2]](<3 x s16>)
+    ; GFX9: [[CONCAT_VECTORS2:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[FMUL]](<2 x s16>), [[FMUL1]](<2 x s16>), [[DEF4]](<2 x s16>)
+    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS2]](<6 x s16>), 0
+    ; GFX9: S_NOP 0, implicit [[EXTRACT]](<3 x s16>)
     %0:_(<3 x s16>) = G_IMPLICIT_DEF
     %1:_(<3 x s16>) = G_IMPLICIT_DEF
     %2:_(<3 x s16>) = G_FMUL %0, %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fneg.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fneg.mir
index 75e498b3a2e75..35f229088167c 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fneg.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fneg.mir
@@ -207,13 +207,15 @@ body: |
 
     ; SI-LABEL: name: test_fneg_v3s16
     ; SI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; SI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
-    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s16>), 0
-    ; SI: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
-    ; SI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; SI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; SI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; SI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
+    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
+    ; SI: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; SI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
     ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; SI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
-    ; SI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; SI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
     ; SI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
     ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
     ; SI: [[COPY:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
@@ -229,21 +231,23 @@ body: |
     ; SI: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C]](s32)
     ; SI: [[OR1:%[0-9]+]]:_(s32) = G_OR [[AND2]], [[SHL1]]
     ; SI: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
-    ; SI: [[DEF1:%[0-9]+]]:_(<2 x s16>) = G_IMPLICIT_DEF
+    ; SI: [[DEF2:%[0-9]+]]:_(<2 x s16>) = G_IMPLICIT_DEF
     ; SI: [[FNEG:%[0-9]+]]:_(<2 x s16>) = G_FNEG [[BITCAST2]]
     ; SI: [[FNEG1:%[0-9]+]]:_(<2 x s16>) = G_FNEG [[BITCAST3]]
-    ; SI: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[FNEG]](<2 x s16>), [[FNEG1]](<2 x s16>), [[DEF1]](<2 x s16>)
-    ; SI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<6 x s16>), 0
-    ; SI: S_NOP 0, implicit [[EXTRACT1]](<3 x s16>)
+    ; SI: [[CONCAT_VECTORS1:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[FNEG]](<2 x s16>), [[FNEG1]](<2 x s16>), [[DEF2]](<2 x s16>)
+    ; SI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS1]](<6 x s16>), 0
+    ; SI: S_NOP 0, implicit [[EXTRACT]](<3 x s16>)
     ; VI-LABEL: name: test_fneg_v3s16
     ; VI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
-    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s16>), 0
-    ; VI: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
-    ; VI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; VI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; VI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; VI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
+    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
+    ; VI: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; VI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
     ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; VI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
-    ; VI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; VI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
     ; VI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
     ; VI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
     ; VI: [[COPY:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
@@ -259,34 +263,36 @@ body: |
     ; VI: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C]](s32)
     ; VI: [[OR1:%[0-9]+]]:_(s32) = G_OR [[AND2]], [[SHL1]]
     ; VI: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
-    ; VI: [[DEF1:%[0-9]+]]:_(<2 x s16>) = G_IMPLICIT_DEF
+    ; VI: [[DEF2:%[0-9]+]]:_(<2 x s16>) = G_IMPLICIT_DEF
     ; VI: [[FNEG:%[0-9]+]]:_(<2 x s16>) = G_FNEG [[BITCAST2]]
     ; VI: [[FNEG1:%[0-9]+]]:_(<2 x s16>) = G_FNEG [[BITCAST3]]
-    ; VI: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[FNEG]](<2 x s16>), [[FNEG1]](<2 x s16>), [[DEF1]](<2 x s16>)
-    ; VI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<6 x s16>), 0
-    ; VI: S_NOP 0, implicit [[EXTRACT1]](<3 x s16>)
+    ; VI: [[CONCAT_VECTORS1:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[FNEG]](<2 x s16>), [[FNEG1]](<2 x s16>), [[DEF2]](<2 x s16>)
+    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS1]](<6 x s16>), 0
+    ; VI: S_NOP 0, implicit [[EXTRACT]](<3 x s16>)
     ; GFX9-LABEL: name: test_fneg_v3s16
     ; GFX9: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
-    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s16>), 0
-    ; GFX9: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
-    ; GFX9: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; GFX9: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX9: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
+    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
+    ; GFX9: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; GFX9: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
     ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; GFX9: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
-    ; GFX9: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; GFX9: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
     ; GFX9: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
     ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
     ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
     ; GFX9: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY]](s32), [[COPY1]](s32)
     ; GFX9: [[COPY2:%[0-9]+]]:_(s32) = COPY [[BITCAST1]](s32)
-    ; GFX9: [[DEF1:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-    ; GFX9: [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY2]](s32), [[DEF1]](s32)
-    ; GFX9: [[DEF2:%[0-9]+]]:_(<2 x s16>) = G_IMPLICIT_DEF
+    ; GFX9: [[DEF2:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+    ; GFX9: [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY2]](s32), [[DEF2]](s32)
+    ; GFX9: [[DEF3:%[0-9]+]]:_(<2 x s16>) = G_IMPLICIT_DEF
     ; GFX9: [[FNEG:%[0-9]+]]:_(<2 x s16>) = G_FNEG [[BUILD_VECTOR_TRUNC]]
     ; GFX9: [[FNEG1:%[0-9]+]]:_(<2 x s16>) = G_FNEG [[BUILD_VECTOR_TRUNC1]]
-    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[FNEG]](<2 x s16>), [[FNEG1]](<2 x s16>), [[DEF2]](<2 x s16>)
-    ; GFX9: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<6 x s16>), 0
-    ; GFX9: S_NOP 0, implicit [[EXTRACT1]](<3 x s16>)
+    ; GFX9: [[CONCAT_VECTORS1:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[FNEG]](<2 x s16>), [[FNEG1]](<2 x s16>), [[DEF3]](<2 x s16>)
+    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS1]](<6 x s16>), 0
+    ; GFX9: S_NOP 0, implicit [[EXTRACT]](<3 x s16>)
     %0:_(<3 x s16>) = G_IMPLICIT_DEF
     %1:_(<3 x s16>) = G_FNEG %0
     S_NOP 0, implicit %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fpext.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fpext.mir
index c6ba0b60d99dd..844c972a6dfab 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fpext.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fpext.mir
@@ -70,15 +70,17 @@ body: |
 
     ; CHECK-LABEL: name: test_fpext_v3f16_to_v3f32
     ; CHECK: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; CHECK: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
-    ; CHECK: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s16>), 0
-    ; CHECK: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
-    ; CHECK: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; CHECK: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; CHECK: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
+    ; CHECK: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
+    ; CHECK: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; CHECK: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
     ; CHECK: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
     ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; CHECK: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
     ; CHECK: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-    ; CHECK: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; CHECK: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
     ; CHECK: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
     ; CHECK: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
     ; CHECK: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-freeze.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-freeze.mir
index 460d23b441a0b..b2a525da27515 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-freeze.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-freeze.mir
@@ -490,8 +490,9 @@ body: |
     ; CHECK: [[ANYEXT:%[0-9]+]]:_(<4 x s32>) = G_ANYEXT [[INSERT]](<4 x s1>)
     ; CHECK: [[FREEZE:%[0-9]+]]:_(<4 x s32>) = G_FREEZE [[ANYEXT]]
     ; CHECK: [[TRUNC:%[0-9]+]]:_(<4 x s1>) = G_TRUNC [[FREEZE]](<4 x s32>)
-    ; CHECK: [[EXTRACT:%[0-9]+]]:_(<3 x s1>) = G_EXTRACT [[TRUNC]](<4 x s1>), 0
-    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(<3 x s32>) = G_ANYEXT [[EXTRACT]](<3 x s1>)
+    ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s1>) = G_CONCAT_VECTORS [[TRUNC]](<4 x s1>), [[DEF1]](<4 x s1>), [[DEF1]](<4 x s1>)
+    ; CHECK: [[UV:%[0-9]+]]:_(<3 x s1>), [[UV1:%[0-9]+]]:_(<3 x s1>), [[UV2:%[0-9]+]]:_(<3 x s1>), [[UV3:%[0-9]+]]:_(<3 x s1>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s1>)
+    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(<3 x s32>) = G_ANYEXT [[UV]](<3 x s1>)
     ; CHECK: $vgpr0_vgpr1_vgpr2 = COPY [[ANYEXT1]](<3 x s32>)
     %0:_(<3 x s1>) = G_IMPLICIT_DEF
     %1:_(<3 x s1>) = G_FREEZE %0
@@ -530,8 +531,9 @@ body: |
     ; CHECK: [[ANYEXT:%[0-9]+]]:_(<4 x s32>) = G_ANYEXT [[INSERT]](<4 x s8>)
     ; CHECK: [[FREEZE:%[0-9]+]]:_(<4 x s32>) = G_FREEZE [[ANYEXT]]
     ; CHECK: [[TRUNC1:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[FREEZE]](<4 x s32>)
-    ; CHECK: [[EXTRACT:%[0-9]+]]:_(<3 x s8>) = G_EXTRACT [[TRUNC1]](<4 x s8>), 0
-    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(<3 x s32>) = G_ANYEXT [[EXTRACT]](<3 x s8>)
+    ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s8>) = G_CONCAT_VECTORS [[TRUNC1]](<4 x s8>), [[DEF]](<4 x s8>), [[DEF]](<4 x s8>)
+    ; CHECK: [[UV:%[0-9]+]]:_(<3 x s8>), [[UV1:%[0-9]+]]:_(<3 x s8>), [[UV2:%[0-9]+]]:_(<3 x s8>), [[UV3:%[0-9]+]]:_(<3 x s8>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s8>)
+    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(<3 x s32>) = G_ANYEXT [[UV]](<3 x s8>)
     ; CHECK: $vgpr0_vgpr1_vgpr2 = COPY [[ANYEXT1]](<3 x s32>)
     %0:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
     %1:_(<3 x s8>) = G_TRUNC %0
@@ -573,13 +575,14 @@ body: |
     ; CHECK: [[DEF2:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
     ; CHECK: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF2]], [[EXTRACT]](<3 x s16>), 0
     ; CHECK: [[FREEZE:%[0-9]+]]:_(<4 x s16>) = G_FREEZE [[INSERT]]
-    ; CHECK: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[FREEZE]](<4 x s16>), 0
-    ; CHECK: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF2]], [[EXTRACT1]](<3 x s16>), 0
-    ; CHECK: [[UV3:%[0-9]+]]:_(<2 x s16>), [[UV4:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
-    ; CHECK: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
+    ; CHECK: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[FREEZE]](<4 x s16>), [[DEF2]](<4 x s16>), [[DEF2]](<4 x s16>)
+    ; CHECK: [[UV3:%[0-9]+]]:_(<3 x s16>), [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; CHECK: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF2]], [[UV3]](<3 x s16>), 0
+    ; CHECK: [[UV7:%[0-9]+]]:_(<2 x s16>), [[UV8:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
+    ; CHECK: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV7]](<2 x s16>)
     ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; CHECK: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
-    ; CHECK: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
+    ; CHECK: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV8]](<2 x s16>)
     ; CHECK: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
     ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
     ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
@@ -628,15 +631,16 @@ body: |
     ; CHECK: [[DEF2:%[0-9]+]]:_(<6 x s16>) = G_IMPLICIT_DEF
     ; CHECK: [[INSERT:%[0-9]+]]:_(<6 x s16>) = G_INSERT [[DEF2]], [[EXTRACT]](<5 x s16>), 0
     ; CHECK: [[FREEZE:%[0-9]+]]:_(<6 x s16>) = G_FREEZE [[INSERT]]
-    ; CHECK: [[EXTRACT1:%[0-9]+]]:_(<5 x s16>) = G_EXTRACT [[FREEZE]](<6 x s16>), 0
-    ; CHECK: [[INSERT1:%[0-9]+]]:_(<6 x s16>) = G_INSERT [[DEF2]], [[EXTRACT1]](<5 x s16>), 0
-    ; CHECK: [[UV5:%[0-9]+]]:_(<2 x s16>), [[UV6:%[0-9]+]]:_(<2 x s16>), [[UV7:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<6 x s16>)
-    ; CHECK: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
+    ; CHECK: [[CONCAT_VECTORS1:%[0-9]+]]:_(<30 x s16>) = G_CONCAT_VECTORS [[FREEZE]](<6 x s16>), [[DEF2]](<6 x s16>), [[DEF2]](<6 x s16>), [[DEF2]](<6 x s16>), [[DEF2]](<6 x s16>)
+    ; CHECK: [[UV5:%[0-9]+]]:_(<5 x s16>), [[UV6:%[0-9]+]]:_(<5 x s16>), [[UV7:%[0-9]+]]:_(<5 x s16>), [[UV8:%[0-9]+]]:_(<5 x s16>), [[UV9:%[0-9]+]]:_(<5 x s16>), [[UV10:%[0-9]+]]:_(<5 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<30 x s16>)
+    ; CHECK: [[INSERT1:%[0-9]+]]:_(<6 x s16>) = G_INSERT [[DEF2]], [[UV5]](<5 x s16>), 0
+    ; CHECK: [[UV11:%[0-9]+]]:_(<2 x s16>), [[UV12:%[0-9]+]]:_(<2 x s16>), [[UV13:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<6 x s16>)
+    ; CHECK: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV11]](<2 x s16>)
     ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; CHECK: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
-    ; CHECK: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV6]](<2 x s16>)
+    ; CHECK: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV12]](<2 x s16>)
     ; CHECK: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
-    ; CHECK: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV7]](<2 x s16>)
+    ; CHECK: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV13]](<2 x s16>)
     ; CHECK: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
     ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
     ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fsin.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fsin.mir
index 20ba990e8bdb1..740f023b93ab1 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fsin.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fsin.mir
@@ -326,15 +326,17 @@ body: |
 
     ; SI-LABEL: name: test_fsin_v3s16
     ; SI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; SI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
-    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s16>), 0
-    ; SI: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
-    ; SI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; SI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; SI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; SI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
+    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
+    ; SI: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; SI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
     ; SI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
     ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; SI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
     ; SI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-    ; SI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; SI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
     ; SI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
     ; SI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
     ; SI: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
@@ -363,20 +365,22 @@ body: |
     ; SI: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C]](s32)
     ; SI: [[OR1:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[SHL1]]
     ; SI: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
-    ; SI: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
-    ; SI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<4 x s16>), 0
-    ; SI: S_NOP 0, implicit [[EXTRACT1]](<3 x s16>)
+    ; SI: [[CONCAT_VECTORS1:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
+    ; SI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS1]](<4 x s16>), 0
+    ; SI: S_NOP 0, implicit [[EXTRACT]](<3 x s16>)
     ; VI-LABEL: name: test_fsin_v3s16
     ; VI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
-    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s16>), 0
-    ; VI: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
-    ; VI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; VI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; VI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; VI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
+    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
+    ; VI: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; VI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
     ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
     ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; VI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
     ; VI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-    ; VI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; VI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
     ; VI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
     ; VI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
     ; VI: [[C1:%[0-9]+]]:_(s16) = G_FCONSTANT half 0xH3118
@@ -399,20 +403,22 @@ body: |
     ; VI: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C]](s32)
     ; VI: [[OR1:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[SHL1]]
     ; VI: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
-    ; VI: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
-    ; VI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<4 x s16>), 0
-    ; VI: S_NOP 0, implicit [[EXTRACT1]](<3 x s16>)
+    ; VI: [[CONCAT_VECTORS1:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
+    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS1]](<4 x s16>), 0
+    ; VI: S_NOP 0, implicit [[EXTRACT]](<3 x s16>)
     ; GFX9-LABEL: name: test_fsin_v3s16
     ; GFX9: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
-    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s16>), 0
-    ; GFX9: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
-    ; GFX9: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; GFX9: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX9: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
+    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
+    ; GFX9: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; GFX9: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
     ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
     ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; GFX9: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
     ; GFX9: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-    ; GFX9: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; GFX9: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
     ; GFX9: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
     ; GFX9: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
     ; GFX9: [[C1:%[0-9]+]]:_(s16) = G_FCONSTANT half 0xH3118
@@ -426,11 +432,11 @@ body: |
     ; GFX9: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[INT1]](s16)
     ; GFX9: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[ANYEXT]](s32), [[ANYEXT1]](s32)
     ; GFX9: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[INT2]](s16)
-    ; GFX9: [[DEF1:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-    ; GFX9: [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[ANYEXT2]](s32), [[DEF1]](s32)
-    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
-    ; GFX9: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<4 x s16>), 0
-    ; GFX9: S_NOP 0, implicit [[EXTRACT1]](<3 x s16>)
+    ; GFX9: [[DEF2:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+    ; GFX9: [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[ANYEXT2]](s32), [[DEF2]](s32)
+    ; GFX9: [[CONCAT_VECTORS1:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
+    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS1]](<4 x s16>), 0
+    ; GFX9: S_NOP 0, implicit [[EXTRACT]](<3 x s16>)
     %0:_(<3 x s16>) = G_IMPLICIT_DEF
     %1:_(<3 x s16>) = G_FSIN %0
     S_NOP 0, implicit %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fsqrt.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fsqrt.mir
index 9508109707cc1..4be1a32f81148 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fsqrt.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fsqrt.mir
@@ -246,15 +246,17 @@ body: |
 
     ; SI-LABEL: name: test_fsqrt_v3s16
     ; SI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; SI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
-    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s16>), 0
-    ; SI: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
-    ; SI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; SI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; SI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; SI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
+    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
+    ; SI: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; SI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
     ; SI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
     ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; SI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
     ; SI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-    ; SI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; SI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
     ; SI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
     ; SI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
     ; SI: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
@@ -276,20 +278,22 @@ body: |
     ; SI: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C]](s32)
     ; SI: [[OR1:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[SHL1]]
     ; SI: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
-    ; SI: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
-    ; SI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<4 x s16>), 0
-    ; SI: S_NOP 0, implicit [[EXTRACT1]](<3 x s16>)
+    ; SI: [[CONCAT_VECTORS1:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
+    ; SI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS1]](<4 x s16>), 0
+    ; SI: S_NOP 0, implicit [[EXTRACT]](<3 x s16>)
     ; VI-LABEL: name: test_fsqrt_v3s16
     ; VI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
-    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s16>), 0
-    ; VI: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
-    ; VI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; VI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; VI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; VI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
+    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
+    ; VI: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; VI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
     ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
     ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; VI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
     ; VI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-    ; VI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; VI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
     ; VI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
     ; VI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
     ; VI: [[FSQRT:%[0-9]+]]:_(s16) = G_FSQRT [[TRUNC]]
@@ -305,20 +309,22 @@ body: |
     ; VI: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C]](s32)
     ; VI: [[OR1:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[SHL1]]
     ; VI: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
-    ; VI: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
-    ; VI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<4 x s16>), 0
-    ; VI: S_NOP 0, implicit [[EXTRACT1]](<3 x s16>)
+    ; VI: [[CONCAT_VECTORS1:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
+    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS1]](<4 x s16>), 0
+    ; VI: S_NOP 0, implicit [[EXTRACT]](<3 x s16>)
     ; GFX9-LABEL: name: test_fsqrt_v3s16
     ; GFX9: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
-    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s16>), 0
-    ; GFX9: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
-    ; GFX9: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; GFX9: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX9: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
+    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
+    ; GFX9: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; GFX9: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
     ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
     ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; GFX9: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
     ; GFX9: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-    ; GFX9: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; GFX9: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
     ; GFX9: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
     ; GFX9: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
     ; GFX9: [[FSQRT:%[0-9]+]]:_(s16) = G_FSQRT [[TRUNC]]
@@ -328,11 +334,11 @@ body: |
     ; GFX9: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[FSQRT1]](s16)
     ; GFX9: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[ANYEXT]](s32), [[ANYEXT1]](s32)
     ; GFX9: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[FSQRT2]](s16)
-    ; GFX9: [[DEF1:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-    ; GFX9: [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[ANYEXT2]](s32), [[DEF1]](s32)
-    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
-    ; GFX9: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<4 x s16>), 0
-    ; GFX9: S_NOP 0, implicit [[EXTRACT1]](<3 x s16>)
+    ; GFX9: [[DEF2:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+    ; GFX9: [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[ANYEXT2]](s32), [[DEF2]](s32)
+    ; GFX9: [[CONCAT_VECTORS1:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
+    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS1]](<4 x s16>), 0
+    ; GFX9: S_NOP 0, implicit [[EXTRACT]](<3 x s16>)
     %0:_(<3 x s16>) = G_IMPLICIT_DEF
     %1:_(<3 x s16>) = G_FSQRT %0
     S_NOP 0, implicit %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fsub.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fsub.mir
index 7df607de5ed84..710e0b405fc9c 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fsub.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fsub.mir
@@ -374,26 +374,29 @@ body: |
 
     ; SI-LABEL: name: test_fsub_v3s16
     ; SI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; SI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
     ; SI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; SI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF1]](<4 x s16>), 0
-    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
-    ; SI: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
-    ; SI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; SI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; SI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
+    ; SI: [[DEF2:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; SI: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF2]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; SI: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
+    ; SI: [[UV8:%[0-9]+]]:_(<2 x s16>), [[UV9:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; SI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV8]](<2 x s16>)
     ; SI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
     ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; SI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
     ; SI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-    ; SI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; SI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV9]](<2 x s16>)
     ; SI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
     ; SI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
-    ; SI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
-    ; SI: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
-    ; SI: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
+    ; SI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
+    ; SI: [[UV10:%[0-9]+]]:_(<2 x s16>), [[UV11:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
+    ; SI: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV10]](<2 x s16>)
     ; SI: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
     ; SI: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
     ; SI: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
-    ; SI: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
+    ; SI: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV11]](<2 x s16>)
     ; SI: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
     ; SI: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
     ; SI: [[FNEG:%[0-9]+]]:_(s16) = G_FNEG [[TRUNC3]]
@@ -421,31 +424,34 @@ body: |
     ; SI: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C]](s32)
     ; SI: [[OR1:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[SHL1]]
     ; SI: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
-    ; SI: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
-    ; SI: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<4 x s16>), 0
-    ; SI: S_NOP 0, implicit [[EXTRACT2]](<3 x s16>)
+    ; SI: [[CONCAT_VECTORS2:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
+    ; SI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS2]](<4 x s16>), 0
+    ; SI: S_NOP 0, implicit [[EXTRACT]](<3 x s16>)
     ; VI-LABEL: name: test_fsub_v3s16
     ; VI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
     ; VI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; VI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF1]](<4 x s16>), 0
-    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
-    ; VI: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
-    ; VI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; VI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; VI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
+    ; VI: [[DEF2:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; VI: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF2]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; VI: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
+    ; VI: [[UV8:%[0-9]+]]:_(<2 x s16>), [[UV9:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; VI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV8]](<2 x s16>)
     ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
     ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; VI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
     ; VI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-    ; VI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; VI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV9]](<2 x s16>)
     ; VI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
     ; VI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
-    ; VI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
-    ; VI: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
-    ; VI: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
+    ; VI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
+    ; VI: [[UV10:%[0-9]+]]:_(<2 x s16>), [[UV11:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
+    ; VI: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV10]](<2 x s16>)
     ; VI: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
     ; VI: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
     ; VI: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
-    ; VI: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
+    ; VI: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV11]](<2 x s16>)
     ; VI: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
     ; VI: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
     ; VI: [[FNEG:%[0-9]+]]:_(s16) = G_FNEG [[TRUNC3]]
@@ -464,31 +470,34 @@ body: |
     ; VI: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C]](s32)
     ; VI: [[OR1:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[SHL1]]
     ; VI: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
-    ; VI: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
-    ; VI: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<4 x s16>), 0
-    ; VI: S_NOP 0, implicit [[EXTRACT2]](<3 x s16>)
+    ; VI: [[CONCAT_VECTORS2:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
+    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS2]](<4 x s16>), 0
+    ; VI: S_NOP 0, implicit [[EXTRACT]](<3 x s16>)
     ; GFX9-LABEL: name: test_fsub_v3s16
     ; GFX9: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
     ; GFX9: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF1]](<4 x s16>), 0
-    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
-    ; GFX9: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
-    ; GFX9: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX9: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
+    ; GFX9: [[DEF2:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; GFX9: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF2]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX9: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
+    ; GFX9: [[UV8:%[0-9]+]]:_(<2 x s16>), [[UV9:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; GFX9: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV8]](<2 x s16>)
     ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
     ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; GFX9: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
     ; GFX9: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-    ; GFX9: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; GFX9: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV9]](<2 x s16>)
     ; GFX9: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
     ; GFX9: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
-    ; GFX9: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
-    ; GFX9: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
-    ; GFX9: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
+    ; GFX9: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
+    ; GFX9: [[UV10:%[0-9]+]]:_(<2 x s16>), [[UV11:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
+    ; GFX9: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV10]](<2 x s16>)
     ; GFX9: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
     ; GFX9: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
     ; GFX9: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
-    ; GFX9: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
+    ; GFX9: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV11]](<2 x s16>)
     ; GFX9: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
     ; GFX9: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
     ; GFX9: [[FNEG:%[0-9]+]]:_(s16) = G_FNEG [[TRUNC3]]
@@ -501,11 +510,11 @@ body: |
     ; GFX9: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[FADD1]](s16)
     ; GFX9: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[ANYEXT]](s32), [[ANYEXT1]](s32)
     ; GFX9: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[FADD2]](s16)
-    ; GFX9: [[DEF2:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-    ; GFX9: [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[ANYEXT2]](s32), [[DEF2]](s32)
-    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
-    ; GFX9: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<4 x s16>), 0
-    ; GFX9: S_NOP 0, implicit [[EXTRACT2]](<3 x s16>)
+    ; GFX9: [[DEF3:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+    ; GFX9: [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[ANYEXT2]](s32), [[DEF3]](s32)
+    ; GFX9: [[CONCAT_VECTORS2:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
+    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS2]](<4 x s16>), 0
+    ; GFX9: S_NOP 0, implicit [[EXTRACT]](<3 x s16>)
     %0:_(<3 x s16>) = G_IMPLICIT_DEF
     %1:_(<3 x s16>) = G_IMPLICIT_DEF
     %2:_(<3 x s16>) = G_FSUB %0, %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-implicit-def.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-implicit-def.mir
index 1ce85a95ef6db..993bb6a437561 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-implicit-def.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-implicit-def.mir
@@ -377,11 +377,14 @@ body: |
     ; CHECK-LABEL: name: test_implicit_def_v3s1
     ; CHECK: [[DEF:%[0-9]+]]:_(<4 x s32>) = G_IMPLICIT_DEF
     ; CHECK: [[TRUNC:%[0-9]+]]:_(<4 x s1>) = G_TRUNC [[DEF]](<4 x s32>)
-    ; CHECK: [[EXTRACT:%[0-9]+]]:_(<3 x s1>) = G_EXTRACT [[TRUNC]](<4 x s1>), 0
-    ; CHECK: [[UV:%[0-9]+]]:_(s1), [[UV1:%[0-9]+]]:_(s1), [[UV2:%[0-9]+]]:_(s1) = G_UNMERGE_VALUES [[EXTRACT]](<3 x s1>)
-    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[UV]](s1)
-    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[UV1]](s1)
-    ; CHECK: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[UV2]](s1)
+    ; CHECK: [[DEF1:%[0-9]+]]:_(<4 x s32>) = G_IMPLICIT_DEF
+    ; CHECK: [[TRUNC1:%[0-9]+]]:_(<4 x s1>) = G_TRUNC [[DEF1]](<4 x s32>)
+    ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s1>) = G_CONCAT_VECTORS [[TRUNC]](<4 x s1>), [[TRUNC1]](<4 x s1>), [[TRUNC1]](<4 x s1>)
+    ; CHECK: [[UV:%[0-9]+]]:_(<3 x s1>), [[UV1:%[0-9]+]]:_(<3 x s1>), [[UV2:%[0-9]+]]:_(<3 x s1>), [[UV3:%[0-9]+]]:_(<3 x s1>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s1>)
+    ; CHECK: [[UV4:%[0-9]+]]:_(s1), [[UV5:%[0-9]+]]:_(s1), [[UV6:%[0-9]+]]:_(s1) = G_UNMERGE_VALUES [[UV]](<3 x s1>)
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[UV4]](s1)
+    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[UV5]](s1)
+    ; CHECK: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[UV6]](s1)
     ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[ANYEXT]](s32), [[ANYEXT1]](s32), [[ANYEXT2]](s32)
     ; CHECK: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
     %0:_(<3 x s1>) = G_IMPLICIT_DEF
@@ -411,11 +414,14 @@ body: |
     ; CHECK-LABEL: name: test_implicit_def_v3s8
     ; CHECK: [[DEF:%[0-9]+]]:_(<4 x s32>) = G_IMPLICIT_DEF
     ; CHECK: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[DEF]](<4 x s32>)
-    ; CHECK: [[EXTRACT:%[0-9]+]]:_(<3 x s8>) = G_EXTRACT [[TRUNC]](<4 x s8>), 0
-    ; CHECK: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8), [[UV2:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[EXTRACT]](<3 x s8>)
-    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[UV]](s8)
-    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[UV1]](s8)
-    ; CHECK: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[UV2]](s8)
+    ; CHECK: [[DEF1:%[0-9]+]]:_(<4 x s32>) = G_IMPLICIT_DEF
+    ; CHECK: [[TRUNC1:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[DEF1]](<4 x s32>)
+    ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s8>) = G_CONCAT_VECTORS [[TRUNC]](<4 x s8>), [[TRUNC1]](<4 x s8>), [[TRUNC1]](<4 x s8>)
+    ; CHECK: [[UV:%[0-9]+]]:_(<3 x s8>), [[UV1:%[0-9]+]]:_(<3 x s8>), [[UV2:%[0-9]+]]:_(<3 x s8>), [[UV3:%[0-9]+]]:_(<3 x s8>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s8>)
+    ; CHECK: [[UV4:%[0-9]+]]:_(s8), [[UV5:%[0-9]+]]:_(s8), [[UV6:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[UV]](<3 x s8>)
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[UV4]](s8)
+    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[UV5]](s8)
+    ; CHECK: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[UV6]](s8)
     ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[ANYEXT]](s32), [[ANYEXT1]](s32), [[ANYEXT2]](s32)
     ; CHECK: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
     %0:_(<3 x s8>) = G_IMPLICIT_DEF
@@ -442,9 +448,10 @@ body: |
 
     ; CHECK-LABEL: name: test_implicit_def_v3s16
     ; CHECK: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; CHECK: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), %1(<4 x s16>), %1(<4 x s16>)
+    ; CHECK: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; CHECK: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; CHECK: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
+    ; CHECK: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
     ; CHECK: $vgpr0_vgpr1 = COPY [[INSERT]](<4 x s16>)
     %0:_(<3 x s16>) = G_IMPLICIT_DEF
     %1:_(<4 x s16>) = G_IMPLICIT_DEF
@@ -471,9 +478,11 @@ body: |
 
     ; CHECK-LABEL: name: test_implicit_def_v5s16
     ; CHECK: [[DEF:%[0-9]+]]:_(<6 x s16>) = G_IMPLICIT_DEF
-    ; CHECK: [[EXTRACT:%[0-9]+]]:_(<5 x s16>) = G_EXTRACT [[DEF]](<6 x s16>), 0
-    ; CHECK: [[DEF1:%[0-9]+]]:_(<8 x s16>) = G_IMPLICIT_DEF
-    ; CHECK: [[INSERT:%[0-9]+]]:_(<8 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<5 x s16>), 0
+    ; CHECK: [[DEF1:%[0-9]+]]:_(<6 x s16>) = G_IMPLICIT_DEF
+    ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<30 x s16>) = G_CONCAT_VECTORS [[DEF]](<6 x s16>), [[DEF1]](<6 x s16>), [[DEF1]](<6 x s16>), [[DEF1]](<6 x s16>), [[DEF1]](<6 x s16>)
+    ; CHECK: [[UV:%[0-9]+]]:_(<5 x s16>), [[UV1:%[0-9]+]]:_(<5 x s16>), [[UV2:%[0-9]+]]:_(<5 x s16>), [[UV3:%[0-9]+]]:_(<5 x s16>), [[UV4:%[0-9]+]]:_(<5 x s16>), [[UV5:%[0-9]+]]:_(<5 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<30 x s16>)
+    ; CHECK: [[DEF2:%[0-9]+]]:_(<8 x s16>) = G_IMPLICIT_DEF
+    ; CHECK: [[INSERT:%[0-9]+]]:_(<8 x s16>) = G_INSERT [[DEF2]], [[UV]](<5 x s16>), 0
     ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INSERT]](<8 x s16>)
     %0:_(<5 x s16>) = G_IMPLICIT_DEF
     %1:_(<8 x s16>) = G_IMPLICIT_DEF
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-insert.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-insert.mir
index 37971d389fe4e..99ff3fff27dda 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-insert.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-insert.mir
@@ -843,8 +843,9 @@ body: |
     ; CHECK: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
     ; CHECK: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s16>), 0
     ; CHECK: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT]], [[TRUNC]](s16), 0
-    ; CHECK: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT1]](<4 x s16>), 0
-    ; CHECK: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT1]](<3 x s16>), 0
+    ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT1]](<4 x s16>), [[DEF]](<4 x s16>), [[DEF]](<4 x s16>)
+    ; CHECK: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
+    ; CHECK: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV]](<3 x s16>), 0
     ; CHECK: $vgpr0_vgpr1 = COPY [[INSERT2]](<4 x s16>)
     %0:_(<4 x s16>) = COPY $vgpr0_vgpr1
     %1:_(<3 x s16>) = G_EXTRACT %0, 0
@@ -869,8 +870,9 @@ body: |
     ; CHECK: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
     ; CHECK: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s16>), 0
     ; CHECK: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT]], [[TRUNC]](s16), 16
-    ; CHECK: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT1]](<4 x s16>), 0
-    ; CHECK: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT1]](<3 x s16>), 0
+    ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT1]](<4 x s16>), [[DEF]](<4 x s16>), [[DEF]](<4 x s16>)
+    ; CHECK: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
+    ; CHECK: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV]](<3 x s16>), 0
     ; CHECK: $vgpr0_vgpr1 = COPY [[INSERT2]](<4 x s16>)
     %0:_(<4 x s16>) = COPY $vgpr0_vgpr1
     %1:_(<3 x s16>) = G_EXTRACT %0, 0
@@ -895,8 +897,9 @@ body: |
     ; CHECK: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
     ; CHECK: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s16>), 0
     ; CHECK: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT]], [[TRUNC]](s16), 32
-    ; CHECK: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT1]](<4 x s16>), 0
-    ; CHECK: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT1]](<3 x s16>), 0
+    ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT1]](<4 x s16>), [[DEF]](<4 x s16>), [[DEF]](<4 x s16>)
+    ; CHECK: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
+    ; CHECK: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV]](<3 x s16>), 0
     ; CHECK: $vgpr0_vgpr1 = COPY [[INSERT2]](<4 x s16>)
     %0:_(<4 x s16>) = COPY $vgpr0_vgpr1
     %1:_(<3 x s16>) = G_EXTRACT %0, 0
@@ -920,8 +923,9 @@ body: |
     ; CHECK: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
     ; CHECK: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s16>), 0
     ; CHECK: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT]], [[COPY1]](<2 x s16>), 0
-    ; CHECK: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT1]](<4 x s16>), 0
-    ; CHECK: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT1]](<3 x s16>), 0
+    ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT1]](<4 x s16>), [[DEF]](<4 x s16>), [[DEF]](<4 x s16>)
+    ; CHECK: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
+    ; CHECK: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV]](<3 x s16>), 0
     ; CHECK: $vgpr0_vgpr1 = COPY [[INSERT2]](<4 x s16>)
     %0:_(<4 x s16>) = COPY $vgpr0_vgpr1
     %1:_(<3 x s16>) = G_EXTRACT %0, 0
@@ -944,8 +948,9 @@ body: |
     ; CHECK: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
     ; CHECK: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s16>), 0
     ; CHECK: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT]], [[COPY1]](<2 x s16>), 16
-    ; CHECK: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT1]](<4 x s16>), 0
-    ; CHECK: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT1]](<3 x s16>), 0
+    ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT1]](<4 x s16>), [[DEF]](<4 x s16>), [[DEF]](<4 x s16>)
+    ; CHECK: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
+    ; CHECK: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV]](<3 x s16>), 0
     ; CHECK: $vgpr0_vgpr1 = COPY [[INSERT2]](<4 x s16>)
     %0:_(<4 x s16>) = COPY $vgpr0_vgpr1
     %1:_(<3 x s16>) = G_EXTRACT %0, 0
@@ -968,8 +973,9 @@ body: |
     ; CHECK: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
     ; CHECK: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s16>), 0
     ; CHECK: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT]], [[COPY1]](s32), 0
-    ; CHECK: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT1]](<4 x s16>), 0
-    ; CHECK: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT1]](<3 x s16>), 0
+    ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT1]](<4 x s16>), [[DEF]](<4 x s16>), [[DEF]](<4 x s16>)
+    ; CHECK: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
+    ; CHECK: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV]](<3 x s16>), 0
     ; CHECK: $vgpr0_vgpr1 = COPY [[INSERT2]](<4 x s16>)
     %0:_(<4 x s16>) = COPY $vgpr0_vgpr1
     %1:_(<3 x s16>) = G_EXTRACT %0, 0
@@ -992,8 +998,9 @@ body: |
     ; CHECK: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
     ; CHECK: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s16>), 0
     ; CHECK: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT]], [[COPY1]](s32), 16
-    ; CHECK: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT1]](<4 x s16>), 0
-    ; CHECK: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT1]](<3 x s16>), 0
+    ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT1]](<4 x s16>), [[DEF]](<4 x s16>), [[DEF]](<4 x s16>)
+    ; CHECK: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
+    ; CHECK: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV]](<3 x s16>), 0
     ; CHECK: $vgpr0_vgpr1 = COPY [[INSERT2]](<4 x s16>)
     %0:_(<4 x s16>) = COPY $vgpr0_vgpr1
     %1:_(<3 x s16>) = G_EXTRACT %0, 0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.load.2d.d16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.load.2d.d16.ll
index a8ab9c7582751..d2c89ab19927e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.load.2d.d16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.load.2d.d16.ll
@@ -768,13 +768,14 @@ define amdgpu_ps <3 x half> @image_load_v3f16_dmask_0000(<8 x i32> inreg %rsrc,
   ; UNPACKED:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; UNPACKED:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; UNPACKED:   [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-  ; UNPACKED:   [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+  ; UNPACKED:   [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), %16(<4 x s16>), %16(<4 x s16>)
+  ; UNPACKED:   [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
   ; UNPACKED:   [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-  ; UNPACKED:   [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
-  ; UNPACKED:   [[EXTRACT1:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<4 x s16>), 0
-  ; UNPACKED:   [[EXTRACT2:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<4 x s16>), 32
-  ; UNPACKED:   $vgpr0 = COPY [[EXTRACT1]](<2 x s16>)
-  ; UNPACKED:   $vgpr1 = COPY [[EXTRACT2]](<2 x s16>)
+  ; UNPACKED:   [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
+  ; UNPACKED:   [[EXTRACT:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<4 x s16>), 0
+  ; UNPACKED:   [[EXTRACT1:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<4 x s16>), 32
+  ; UNPACKED:   $vgpr0 = COPY [[EXTRACT]](<2 x s16>)
+  ; UNPACKED:   $vgpr1 = COPY [[EXTRACT1]](<2 x s16>)
   ; UNPACKED:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
   ; PACKED-LABEL: name: image_load_v3f16_dmask_0000
   ; PACKED: bb.1 (%ir-block.0):
@@ -790,13 +791,14 @@ define amdgpu_ps <3 x half> @image_load_v3f16_dmask_0000(<8 x i32> inreg %rsrc,
   ; PACKED:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; PACKED:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; PACKED:   [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-  ; PACKED:   [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+  ; PACKED:   [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), %16(<4 x s16>), %16(<4 x s16>)
+  ; PACKED:   [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
   ; PACKED:   [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-  ; PACKED:   [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
-  ; PACKED:   [[EXTRACT1:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<4 x s16>), 0
-  ; PACKED:   [[EXTRACT2:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<4 x s16>), 32
-  ; PACKED:   $vgpr0 = COPY [[EXTRACT1]](<2 x s16>)
-  ; PACKED:   $vgpr1 = COPY [[EXTRACT2]](<2 x s16>)
+  ; PACKED:   [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
+  ; PACKED:   [[EXTRACT:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<4 x s16>), 0
+  ; PACKED:   [[EXTRACT1:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<4 x s16>), 32
+  ; PACKED:   $vgpr0 = COPY [[EXTRACT]](<2 x s16>)
+  ; PACKED:   $vgpr1 = COPY [[EXTRACT1]](<2 x s16>)
   ; PACKED:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
   %tex = call <3 x half> @llvm.amdgcn.image.load.2d.v3f16.i32(i32 0, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
   ret <3 x half> %tex
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.s.buffer.load.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.s.buffer.load.mir
index 9aee145ec1d15..b75ec76d7ff44 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.s.buffer.load.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.s.buffer.load.mir
@@ -30,8 +30,10 @@ body:             |
     ; GCN: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
     ; GCN: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; GCN: [[AMDGPU_S_BUFFER_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_S_BUFFER_LOAD [[COPY]](<4 x s32>), [[C]](s32), 0 :: (dereferenceable invariant load 12, align 4)
-    ; GCN: [[EXTRACT:%[0-9]+]]:_(<3 x s32>) = G_EXTRACT [[AMDGPU_S_BUFFER_LOAD]](<4 x s32>), 0
-    ; GCN: S_ENDPGM 0, implicit [[EXTRACT]](<3 x s32>)
+    ; GCN: [[DEF:%[0-9]+]]:_(<4 x s32>) = G_IMPLICIT_DEF
+    ; GCN: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s32>) = G_CONCAT_VECTORS [[AMDGPU_S_BUFFER_LOAD]](<4 x s32>), [[DEF]](<4 x s32>), [[DEF]](<4 x s32>)
+    ; GCN: [[UV:%[0-9]+]]:_(<3 x s32>), [[UV1:%[0-9]+]]:_(<3 x s32>), [[UV2:%[0-9]+]]:_(<3 x s32>), [[UV3:%[0-9]+]]:_(<3 x s32>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s32>)
+    ; GCN: S_ENDPGM 0, implicit [[UV]](<3 x s32>)
     %0:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
     %1:_(s32) = G_CONSTANT i32 0
     %2:_(<3 x s32>) = G_INTRINSIC intrinsic(@llvm.amdgcn.s.buffer.load), %0, %1, 0
@@ -49,8 +51,10 @@ body:             |
     ; GCN: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
     ; GCN: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; GCN: [[AMDGPU_S_BUFFER_LOAD:%[0-9]+]]:_(<4 x p3>) = G_AMDGPU_S_BUFFER_LOAD [[COPY]](<4 x s32>), [[C]](s32), 0 :: (dereferenceable invariant load 12, align 4)
-    ; GCN: [[EXTRACT:%[0-9]+]]:_(<3 x p3>) = G_EXTRACT [[AMDGPU_S_BUFFER_LOAD]](<4 x p3>), 0
-    ; GCN: S_ENDPGM 0, implicit [[EXTRACT]](<3 x p3>)
+    ; GCN: [[DEF:%[0-9]+]]:_(<4 x p3>) = G_IMPLICIT_DEF
+    ; GCN: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x p3>) = G_CONCAT_VECTORS [[AMDGPU_S_BUFFER_LOAD]](<4 x p3>), [[DEF]](<4 x p3>), [[DEF]](<4 x p3>)
+    ; GCN: [[UV:%[0-9]+]]:_(<3 x p3>), [[UV1:%[0-9]+]]:_(<3 x p3>), [[UV2:%[0-9]+]]:_(<3 x p3>), [[UV3:%[0-9]+]]:_(<3 x p3>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x p3>)
+    ; GCN: S_ENDPGM 0, implicit [[UV]](<3 x p3>)
     %0:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
     %1:_(s32) = G_CONSTANT i32 0
     %2:_(<3 x p3>) = G_INTRINSIC intrinsic(@llvm.amdgcn.s.buffer.load), %0, %1, 0
@@ -68,8 +72,10 @@ body:             |
     ; GCN: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
     ; GCN: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; GCN: [[AMDGPU_S_BUFFER_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_S_BUFFER_LOAD [[COPY]](<4 x s32>), [[C]](s32), 0 :: (dereferenceable invariant load 12, align 4)
-    ; GCN: [[EXTRACT:%[0-9]+]]:_(<3 x s32>) = G_EXTRACT [[AMDGPU_S_BUFFER_LOAD]](<4 x s32>), 0
-    ; GCN: [[BITCAST:%[0-9]+]]:_(<6 x s16>) = G_BITCAST [[EXTRACT]](<3 x s32>)
+    ; GCN: [[DEF:%[0-9]+]]:_(<4 x s32>) = G_IMPLICIT_DEF
+    ; GCN: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s32>) = G_CONCAT_VECTORS [[AMDGPU_S_BUFFER_LOAD]](<4 x s32>), [[DEF]](<4 x s32>), [[DEF]](<4 x s32>)
+    ; GCN: [[UV:%[0-9]+]]:_(<3 x s32>), [[UV1:%[0-9]+]]:_(<3 x s32>), [[UV2:%[0-9]+]]:_(<3 x s32>), [[UV3:%[0-9]+]]:_(<3 x s32>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s32>)
+    ; GCN: [[BITCAST:%[0-9]+]]:_(<6 x s16>) = G_BITCAST [[UV]](<3 x s32>)
     ; GCN: S_ENDPGM 0, implicit [[BITCAST]](<6 x s16>)
     %0:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
     %1:_(s32) = G_CONSTANT i32 0
@@ -88,8 +94,10 @@ body:             |
     ; GCN: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
     ; GCN: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; GCN: [[AMDGPU_S_BUFFER_LOAD:%[0-9]+]]:_(<8 x s32>) = G_AMDGPU_S_BUFFER_LOAD [[COPY]](<4 x s32>), [[C]](s32), 0 :: (dereferenceable invariant load 24, align 4)
-    ; GCN: [[EXTRACT:%[0-9]+]]:_(<6 x s32>) = G_EXTRACT [[AMDGPU_S_BUFFER_LOAD]](<8 x s32>), 0
-    ; GCN: S_ENDPGM 0, implicit [[EXTRACT]](<6 x s32>)
+    ; GCN: [[DEF:%[0-9]+]]:_(<8 x s32>) = G_IMPLICIT_DEF
+    ; GCN: [[CONCAT_VECTORS:%[0-9]+]]:_(<24 x s32>) = G_CONCAT_VECTORS [[AMDGPU_S_BUFFER_LOAD]](<8 x s32>), [[DEF]](<8 x s32>), [[DEF]](<8 x s32>)
+    ; GCN: [[UV:%[0-9]+]]:_(<6 x s32>), [[UV1:%[0-9]+]]:_(<6 x s32>), [[UV2:%[0-9]+]]:_(<6 x s32>), [[UV3:%[0-9]+]]:_(<6 x s32>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<24 x s32>)
+    ; GCN: S_ENDPGM 0, implicit [[UV]](<6 x s32>)
     %0:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
     %1:_(s32) = G_CONSTANT i32 0
     %2:_(<6 x s32>) = G_INTRINSIC intrinsic(@llvm.amdgcn.s.buffer.load), %0, %1, 0
@@ -107,8 +115,10 @@ body:             |
     ; GCN: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
     ; GCN: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; GCN: [[AMDGPU_S_BUFFER_LOAD:%[0-9]+]]:_(<4 x s64>) = G_AMDGPU_S_BUFFER_LOAD [[COPY]](<4 x s32>), [[C]](s32), 0 :: (dereferenceable invariant load 24, align 4)
-    ; GCN: [[EXTRACT:%[0-9]+]]:_(<3 x s64>) = G_EXTRACT [[AMDGPU_S_BUFFER_LOAD]](<4 x s64>), 0
-    ; GCN: S_ENDPGM 0, implicit [[EXTRACT]](<3 x s64>)
+    ; GCN: [[DEF:%[0-9]+]]:_(<4 x s64>) = G_IMPLICIT_DEF
+    ; GCN: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s64>) = G_CONCAT_VECTORS [[AMDGPU_S_BUFFER_LOAD]](<4 x s64>), [[DEF]](<4 x s64>), [[DEF]](<4 x s64>)
+    ; GCN: [[UV:%[0-9]+]]:_(<3 x s64>), [[UV1:%[0-9]+]]:_(<3 x s64>), [[UV2:%[0-9]+]]:_(<3 x s64>), [[UV3:%[0-9]+]]:_(<3 x s64>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s64>)
+    ; GCN: S_ENDPGM 0, implicit [[UV]](<3 x s64>)
     %0:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
     %1:_(s32) = G_CONSTANT i32 0
     %2:_(<3 x s64>) = G_INTRINSIC intrinsic(@llvm.amdgcn.s.buffer.load), %0, %1, 0
@@ -126,29 +136,31 @@ body:             |
     ; GCN: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
     ; GCN: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; GCN: [[AMDGPU_S_BUFFER_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_S_BUFFER_LOAD [[COPY]](<4 x s32>), [[C]](s32), 0 :: (dereferenceable invariant load 12, align 4)
-    ; GCN: [[EXTRACT:%[0-9]+]]:_(<3 x s32>) = G_EXTRACT [[AMDGPU_S_BUFFER_LOAD]](<4 x s32>), 0
-    ; GCN: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[EXTRACT]](<3 x s32>)
+    ; GCN: [[DEF:%[0-9]+]]:_(<4 x s32>) = G_IMPLICIT_DEF
+    ; GCN: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s32>) = G_CONCAT_VECTORS [[AMDGPU_S_BUFFER_LOAD]](<4 x s32>), [[DEF]](<4 x s32>), [[DEF]](<4 x s32>)
+    ; GCN: [[UV:%[0-9]+]]:_(<3 x s32>), [[UV1:%[0-9]+]]:_(<3 x s32>), [[UV2:%[0-9]+]]:_(<3 x s32>), [[UV3:%[0-9]+]]:_(<3 x s32>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s32>)
+    ; GCN: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32), [[UV6:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV]](<3 x s32>)
     ; GCN: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GCN: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[UV]], [[C1]](s32)
+    ; GCN: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[UV4]], [[C1]](s32)
     ; GCN: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GCN: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[UV]], [[C2]](s32)
+    ; GCN: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[UV4]], [[C2]](s32)
     ; GCN: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
-    ; GCN: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[UV]], [[C3]](s32)
-    ; GCN: [[COPY1:%[0-9]+]]:_(s32) = COPY [[UV]](s32)
+    ; GCN: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[UV4]], [[C3]](s32)
+    ; GCN: [[COPY1:%[0-9]+]]:_(s32) = COPY [[UV4]](s32)
     ; GCN: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
     ; GCN: [[COPY3:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
     ; GCN: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
-    ; GCN: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[UV1]], [[C1]](s32)
-    ; GCN: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[UV1]], [[C2]](s32)
-    ; GCN: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[UV1]], [[C3]](s32)
-    ; GCN: [[COPY5:%[0-9]+]]:_(s32) = COPY [[UV1]](s32)
+    ; GCN: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[UV5]], [[C1]](s32)
+    ; GCN: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[UV5]], [[C2]](s32)
+    ; GCN: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[UV5]], [[C3]](s32)
+    ; GCN: [[COPY5:%[0-9]+]]:_(s32) = COPY [[UV5]](s32)
     ; GCN: [[COPY6:%[0-9]+]]:_(s32) = COPY [[LSHR3]](s32)
     ; GCN: [[COPY7:%[0-9]+]]:_(s32) = COPY [[LSHR4]](s32)
     ; GCN: [[COPY8:%[0-9]+]]:_(s32) = COPY [[LSHR5]](s32)
-    ; GCN: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[UV2]], [[C1]](s32)
-    ; GCN: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[UV2]], [[C2]](s32)
-    ; GCN: [[LSHR8:%[0-9]+]]:_(s32) = G_LSHR [[UV2]], [[C3]](s32)
-    ; GCN: [[COPY9:%[0-9]+]]:_(s32) = COPY [[UV2]](s32)
+    ; GCN: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[UV6]], [[C1]](s32)
+    ; GCN: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[UV6]], [[C2]](s32)
+    ; GCN: [[LSHR8:%[0-9]+]]:_(s32) = G_LSHR [[UV6]], [[C3]](s32)
+    ; GCN: [[COPY9:%[0-9]+]]:_(s32) = COPY [[UV6]](s32)
     ; GCN: [[COPY10:%[0-9]+]]:_(s32) = COPY [[LSHR6]](s32)
     ; GCN: [[COPY11:%[0-9]+]]:_(s32) = COPY [[LSHR7]](s32)
     ; GCN: [[COPY12:%[0-9]+]]:_(s32) = COPY [[LSHR8]](s32)
@@ -195,8 +207,8 @@ body:             |
     ; GCN: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[AND11]], [[C2]](s32)
     ; GCN: [[OR5:%[0-9]+]]:_(s32) = G_OR [[AND10]], [[SHL5]]
     ; GCN: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](s32)
-    ; GCN: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>), [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
-    ; GCN: S_ENDPGM 0, implicit [[CONCAT_VECTORS]](<12 x s16>)
+    ; GCN: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>), [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
+    ; GCN: S_ENDPGM 0, implicit [[CONCAT_VECTORS1]](<12 x s16>)
     %0:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
     %1:_(s32) = G_CONSTANT i32 0
     %2:_(<12 x s8>) = G_INTRINSIC intrinsic(@llvm.amdgcn.s.buffer.load), %0, %1, 0
@@ -215,8 +227,10 @@ body:             |
     ; GCN: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
     ; GCN: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; GCN: [[AMDGPU_S_BUFFER_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_S_BUFFER_LOAD [[COPY]](<4 x s32>), [[C]](s32), 0 :: (dereferenceable invariant load 12, align 4)
-    ; GCN: [[EXTRACT:%[0-9]+]]:_(<3 x s32>) = G_EXTRACT [[AMDGPU_S_BUFFER_LOAD]](<4 x s32>), 0
-    ; GCN: S_ENDPGM 0, implicit [[EXTRACT]](<3 x s32>)
+    ; GCN: [[DEF:%[0-9]+]]:_(<4 x s32>) = G_IMPLICIT_DEF
+    ; GCN: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s32>) = G_CONCAT_VECTORS [[AMDGPU_S_BUFFER_LOAD]](<4 x s32>), [[DEF]](<4 x s32>), [[DEF]](<4 x s32>)
+    ; GCN: [[UV:%[0-9]+]]:_(<3 x s32>), [[UV1:%[0-9]+]]:_(<3 x s32>), [[UV2:%[0-9]+]]:_(<3 x s32>), [[UV3:%[0-9]+]]:_(<3 x s32>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s32>)
+    ; GCN: S_ENDPGM 0, implicit [[UV]](<3 x s32>)
     %0:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
     %1:_(s32) = G_CONSTANT i32 0
     %2:_(<3 x s32>) = G_INTRINSIC intrinsic(@llvm.amdgcn.s.buffer.load), %0, %1, 0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-constant.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-constant.mir
index d91886d1626da..739b76cad892c 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-constant.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-constant.mir
@@ -5884,37 +5884,42 @@ body: |
     ; CI-LABEL: name: test_load_constant_v3s16_align8
     ; CI: [[COPY:%[0-9]+]]:_(p4) = COPY $vgpr0_vgpr1
     ; CI: [[LOAD:%[0-9]+]]:_(<4 x s16>) = G_LOAD [[COPY]](p4) :: (load 6, align 8, addrspace 4)
-    ; CI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[LOAD]](<4 x s16>), 0
+    ; CI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[LOAD]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; CI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; CI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; CI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s16>), 0
+    ; CI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV]](<3 x s16>), 0
     ; CI: $vgpr0_vgpr1 = COPY [[INSERT]](<4 x s16>)
     ; VI-LABEL: name: test_load_constant_v3s16_align8
     ; VI: [[COPY:%[0-9]+]]:_(p4) = COPY $vgpr0_vgpr1
     ; VI: [[LOAD:%[0-9]+]]:_(<4 x s16>) = G_LOAD [[COPY]](p4) :: (load 6, align 8, addrspace 4)
-    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[LOAD]](<4 x s16>), 0
+    ; VI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[LOAD]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; VI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; VI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s16>), 0
+    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV]](<3 x s16>), 0
     ; VI: $vgpr0_vgpr1 = COPY [[INSERT]](<4 x s16>)
     ; GFX9-LABEL: name: test_load_constant_v3s16_align8
     ; GFX9: [[COPY:%[0-9]+]]:_(p4) = COPY $vgpr0_vgpr1
     ; GFX9: [[LOAD:%[0-9]+]]:_(<4 x s16>) = G_LOAD [[COPY]](p4) :: (load 6, align 8, addrspace 4)
-    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[LOAD]](<4 x s16>), 0
+    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[LOAD]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; GFX9: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; GFX9: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s16>), 0
+    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV]](<3 x s16>), 0
     ; GFX9: $vgpr0_vgpr1 = COPY [[INSERT]](<4 x s16>)
     ; CI-MESA-LABEL: name: test_load_constant_v3s16_align8
     ; CI-MESA: [[COPY:%[0-9]+]]:_(p4) = COPY $vgpr0_vgpr1
     ; CI-MESA: [[LOAD:%[0-9]+]]:_(<4 x s16>) = G_LOAD [[COPY]](p4) :: (load 6, align 8, addrspace 4)
-    ; CI-MESA: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[LOAD]](<4 x s16>), 0
+    ; CI-MESA: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[LOAD]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; CI-MESA: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; CI-MESA: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; CI-MESA: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s16>), 0
+    ; CI-MESA: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV]](<3 x s16>), 0
     ; CI-MESA: $vgpr0_vgpr1 = COPY [[INSERT]](<4 x s16>)
     ; GFX9-MESA-LABEL: name: test_load_constant_v3s16_align8
     ; GFX9-MESA: [[COPY:%[0-9]+]]:_(p4) = COPY $vgpr0_vgpr1
     ; GFX9-MESA: [[LOAD:%[0-9]+]]:_(<4 x s16>) = G_LOAD [[COPY]](p4) :: (load 6, align 8, addrspace 4)
-    ; GFX9-MESA: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[LOAD]](<4 x s16>), 0
+    ; GFX9-MESA: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[LOAD]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; GFX9-MESA: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; GFX9-MESA: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9-MESA: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s16>), 0
+    ; GFX9-MESA: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV]](<3 x s16>), 0
     ; GFX9-MESA: $vgpr0_vgpr1 = COPY [[INSERT]](<4 x s16>)
     %0:_(p4) = COPY $vgpr0_vgpr1
     %1:_(<3 x s16>) = G_LOAD %0 :: (load 6, align 8, addrspace 4)
@@ -5992,15 +5997,18 @@ body: |
     ; CI: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p4) :: (load 2 + 4, addrspace 4)
     ; CI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD2]](s32)
     ; CI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; CI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; CI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; CI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; CI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; CI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
+    ; CI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
     ; CI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT]], [[BITCAST]](<2 x s16>), 0
-    ; CI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT1]](<4 x s16>), 0
-    ; CI: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
+    ; CI: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT1]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; CI: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; CI: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
     ; CI: [[INSERT3:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT2]], [[TRUNC]](s16), 32
-    ; CI: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT3]](<4 x s16>), 0
-    ; CI: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT2]](<3 x s16>), 0
+    ; CI: [[CONCAT_VECTORS2:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT3]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; CI: [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>), [[UV10:%[0-9]+]]:_(<3 x s16>), [[UV11:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS2]](<12 x s16>)
+    ; CI: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV8]](<3 x s16>), 0
     ; CI: $vgpr0_vgpr1 = COPY [[INSERT4]](<4 x s16>)
     ; VI-LABEL: name: test_load_constant_v3s16_align2
     ; VI: [[COPY:%[0-9]+]]:_(p4) = COPY $vgpr0_vgpr1
@@ -6022,15 +6030,18 @@ body: |
     ; VI: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p4) :: (load 2 + 4, addrspace 4)
     ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD2]](s32)
     ; VI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; VI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; VI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; VI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
+    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
     ; VI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT]], [[BITCAST]](<2 x s16>), 0
-    ; VI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT1]](<4 x s16>), 0
-    ; VI: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
+    ; VI: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT1]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; VI: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; VI: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
     ; VI: [[INSERT3:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT2]], [[TRUNC]](s16), 32
-    ; VI: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT3]](<4 x s16>), 0
-    ; VI: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT2]](<3 x s16>), 0
+    ; VI: [[CONCAT_VECTORS2:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT3]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; VI: [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>), [[UV10:%[0-9]+]]:_(<3 x s16>), [[UV11:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS2]](<12 x s16>)
+    ; VI: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV8]](<3 x s16>), 0
     ; VI: $vgpr0_vgpr1 = COPY [[INSERT4]](<4 x s16>)
     ; GFX9-LABEL: name: test_load_constant_v3s16_align2
     ; GFX9: [[COPY:%[0-9]+]]:_(p4) = COPY $vgpr0_vgpr1
@@ -6046,15 +6057,18 @@ body: |
     ; GFX9: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p4) :: (load 2 + 4, addrspace 4)
     ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD2]](s32)
     ; GFX9: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; GFX9: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; GFX9: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
+    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
     ; GFX9: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT]], [[BUILD_VECTOR_TRUNC]](<2 x s16>), 0
-    ; GFX9: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT1]](<4 x s16>), 0
-    ; GFX9: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
+    ; GFX9: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT1]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX9: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; GFX9: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
     ; GFX9: [[INSERT3:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT2]], [[TRUNC]](s16), 32
-    ; GFX9: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT3]](<4 x s16>), 0
-    ; GFX9: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT2]](<3 x s16>), 0
+    ; GFX9: [[CONCAT_VECTORS2:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT3]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX9: [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>), [[UV10:%[0-9]+]]:_(<3 x s16>), [[UV11:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS2]](<12 x s16>)
+    ; GFX9: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV8]](<3 x s16>), 0
     ; GFX9: $vgpr0_vgpr1 = COPY [[INSERT4]](<4 x s16>)
     ; CI-MESA-LABEL: name: test_load_constant_v3s16_align2
     ; CI-MESA: [[COPY:%[0-9]+]]:_(p4) = COPY $vgpr0_vgpr1
@@ -6076,15 +6090,18 @@ body: |
     ; CI-MESA: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p4) :: (load 2 + 4, addrspace 4)
     ; CI-MESA: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD2]](s32)
     ; CI-MESA: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; CI-MESA: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; CI-MESA: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; CI-MESA: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; CI-MESA: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; CI-MESA: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
+    ; CI-MESA: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
     ; CI-MESA: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT]], [[BITCAST]](<2 x s16>), 0
-    ; CI-MESA: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT1]](<4 x s16>), 0
-    ; CI-MESA: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
+    ; CI-MESA: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT1]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; CI-MESA: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; CI-MESA: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
     ; CI-MESA: [[INSERT3:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT2]], [[TRUNC]](s16), 32
-    ; CI-MESA: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT3]](<4 x s16>), 0
-    ; CI-MESA: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT2]](<3 x s16>), 0
+    ; CI-MESA: [[CONCAT_VECTORS2:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT3]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; CI-MESA: [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>), [[UV10:%[0-9]+]]:_(<3 x s16>), [[UV11:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS2]](<12 x s16>)
+    ; CI-MESA: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV8]](<3 x s16>), 0
     ; CI-MESA: $vgpr0_vgpr1 = COPY [[INSERT4]](<4 x s16>)
     ; GFX9-MESA-LABEL: name: test_load_constant_v3s16_align2
     ; GFX9-MESA: [[COPY:%[0-9]+]]:_(p4) = COPY $vgpr0_vgpr1
@@ -6100,15 +6117,18 @@ body: |
     ; GFX9-MESA: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p4) :: (load 2 + 4, addrspace 4)
     ; GFX9-MESA: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD2]](s32)
     ; GFX9-MESA: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9-MESA: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; GFX9-MESA: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; GFX9-MESA: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; GFX9-MESA: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9-MESA: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
+    ; GFX9-MESA: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
     ; GFX9-MESA: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT]], [[BUILD_VECTOR_TRUNC]](<2 x s16>), 0
-    ; GFX9-MESA: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT1]](<4 x s16>), 0
-    ; GFX9-MESA: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
+    ; GFX9-MESA: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT1]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX9-MESA: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; GFX9-MESA: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
     ; GFX9-MESA: [[INSERT3:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT2]], [[TRUNC]](s16), 32
-    ; GFX9-MESA: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT3]](<4 x s16>), 0
-    ; GFX9-MESA: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT2]](<3 x s16>), 0
+    ; GFX9-MESA: [[CONCAT_VECTORS2:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT3]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX9-MESA: [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>), [[UV10:%[0-9]+]]:_(<3 x s16>), [[UV11:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS2]](<12 x s16>)
+    ; GFX9-MESA: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV8]](<3 x s16>), 0
     ; GFX9-MESA: $vgpr0_vgpr1 = COPY [[INSERT4]](<4 x s16>)
     %0:_(p4) = COPY $vgpr0_vgpr1
     %1:_(<3 x s16>) = G_LOAD %0 :: (load 6, align 2, addrspace 4)
@@ -6172,15 +6192,18 @@ body: |
     ; CI: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[SHL3]](s32)
     ; CI: [[OR3:%[0-9]+]]:_(s16) = G_OR [[AND4]], [[TRUNC5]]
     ; CI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; CI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; CI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; CI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; CI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; CI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
+    ; CI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
     ; CI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT]], [[BITCAST]](<2 x s16>), 0
-    ; CI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT1]](<4 x s16>), 0
-    ; CI: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
+    ; CI: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT1]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; CI: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; CI: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
     ; CI: [[INSERT3:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT2]], [[OR3]](s16), 32
-    ; CI: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT3]](<4 x s16>), 0
-    ; CI: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT2]](<3 x s16>), 0
+    ; CI: [[CONCAT_VECTORS2:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT3]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; CI: [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>), [[UV10:%[0-9]+]]:_(<3 x s16>), [[UV11:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS2]](<12 x s16>)
+    ; CI: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV8]](<3 x s16>), 0
     ; CI: $vgpr0_vgpr1 = COPY [[INSERT4]](<4 x s16>)
     ; VI-LABEL: name: test_load_constant_v3s16_align1
     ; VI: [[COPY:%[0-9]+]]:_(p4) = COPY $vgpr0_vgpr1
@@ -6225,15 +6248,18 @@ body: |
     ; VI: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[AND5]], [[C2]](s16)
     ; VI: [[OR3:%[0-9]+]]:_(s16) = G_OR [[AND4]], [[SHL3]]
     ; VI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; VI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; VI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; VI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
+    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
     ; VI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT]], [[BITCAST]](<2 x s16>), 0
-    ; VI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT1]](<4 x s16>), 0
-    ; VI: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
+    ; VI: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT1]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; VI: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; VI: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
     ; VI: [[INSERT3:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT2]], [[OR3]](s16), 32
-    ; VI: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT3]](<4 x s16>), 0
-    ; VI: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT2]](<3 x s16>), 0
+    ; VI: [[CONCAT_VECTORS2:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT3]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; VI: [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>), [[UV10:%[0-9]+]]:_(<3 x s16>), [[UV11:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS2]](<12 x s16>)
+    ; VI: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV8]](<3 x s16>), 0
     ; VI: $vgpr0_vgpr1 = COPY [[INSERT4]](<4 x s16>)
     ; GFX9-LABEL: name: test_load_constant_v3s16_align1
     ; GFX9: [[COPY:%[0-9]+]]:_(p4) = COPY $vgpr0_vgpr1
@@ -6275,15 +6301,18 @@ body: |
     ; GFX9: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[AND5]], [[C2]](s16)
     ; GFX9: [[OR2:%[0-9]+]]:_(s16) = G_OR [[AND4]], [[SHL2]]
     ; GFX9: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; GFX9: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; GFX9: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
+    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
     ; GFX9: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT]], [[BUILD_VECTOR_TRUNC]](<2 x s16>), 0
-    ; GFX9: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT1]](<4 x s16>), 0
-    ; GFX9: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
+    ; GFX9: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT1]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX9: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; GFX9: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
     ; GFX9: [[INSERT3:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT2]], [[OR2]](s16), 32
-    ; GFX9: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT3]](<4 x s16>), 0
-    ; GFX9: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT2]](<3 x s16>), 0
+    ; GFX9: [[CONCAT_VECTORS2:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT3]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX9: [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>), [[UV10:%[0-9]+]]:_(<3 x s16>), [[UV11:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS2]](<12 x s16>)
+    ; GFX9: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV8]](<3 x s16>), 0
     ; GFX9: $vgpr0_vgpr1 = COPY [[INSERT4]](<4 x s16>)
     ; CI-MESA-LABEL: name: test_load_constant_v3s16_align1
     ; CI-MESA: [[COPY:%[0-9]+]]:_(p4) = COPY $vgpr0_vgpr1
@@ -6334,15 +6363,18 @@ body: |
     ; CI-MESA: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[SHL3]](s32)
     ; CI-MESA: [[OR3:%[0-9]+]]:_(s16) = G_OR [[AND4]], [[TRUNC5]]
     ; CI-MESA: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; CI-MESA: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; CI-MESA: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; CI-MESA: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; CI-MESA: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; CI-MESA: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
+    ; CI-MESA: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
     ; CI-MESA: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT]], [[BITCAST]](<2 x s16>), 0
-    ; CI-MESA: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT1]](<4 x s16>), 0
-    ; CI-MESA: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
+    ; CI-MESA: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT1]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; CI-MESA: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; CI-MESA: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
     ; CI-MESA: [[INSERT3:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT2]], [[OR3]](s16), 32
-    ; CI-MESA: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT3]](<4 x s16>), 0
-    ; CI-MESA: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT2]](<3 x s16>), 0
+    ; CI-MESA: [[CONCAT_VECTORS2:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT3]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; CI-MESA: [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>), [[UV10:%[0-9]+]]:_(<3 x s16>), [[UV11:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS2]](<12 x s16>)
+    ; CI-MESA: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV8]](<3 x s16>), 0
     ; CI-MESA: $vgpr0_vgpr1 = COPY [[INSERT4]](<4 x s16>)
     ; GFX9-MESA-LABEL: name: test_load_constant_v3s16_align1
     ; GFX9-MESA: [[COPY:%[0-9]+]]:_(p4) = COPY $vgpr0_vgpr1
@@ -6384,15 +6416,18 @@ body: |
     ; GFX9-MESA: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[AND5]], [[C2]](s16)
     ; GFX9-MESA: [[OR2:%[0-9]+]]:_(s16) = G_OR [[AND4]], [[SHL2]]
     ; GFX9-MESA: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9-MESA: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; GFX9-MESA: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; GFX9-MESA: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; GFX9-MESA: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9-MESA: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
+    ; GFX9-MESA: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
     ; GFX9-MESA: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT]], [[BUILD_VECTOR_TRUNC]](<2 x s16>), 0
-    ; GFX9-MESA: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT1]](<4 x s16>), 0
-    ; GFX9-MESA: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
+    ; GFX9-MESA: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT1]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX9-MESA: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; GFX9-MESA: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
     ; GFX9-MESA: [[INSERT3:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT2]], [[OR2]](s16), 32
-    ; GFX9-MESA: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT3]](<4 x s16>), 0
-    ; GFX9-MESA: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT2]](<3 x s16>), 0
+    ; GFX9-MESA: [[CONCAT_VECTORS2:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT3]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX9-MESA: [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>), [[UV10:%[0-9]+]]:_(<3 x s16>), [[UV11:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS2]](<12 x s16>)
+    ; GFX9-MESA: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV8]](<3 x s16>), 0
     ; GFX9-MESA: $vgpr0_vgpr1 = COPY [[INSERT4]](<4 x s16>)
     %0:_(p4) = COPY $vgpr0_vgpr1
     %1:_(<3 x s16>) = G_LOAD %0 :: (load 6, align 1, addrspace 4)
@@ -8262,37 +8297,42 @@ body: |
     ; CI-LABEL: name: test_load_constant_v3s64_align32
     ; CI: [[COPY:%[0-9]+]]:_(p4) = COPY $vgpr0_vgpr1
     ; CI: [[LOAD:%[0-9]+]]:_(<4 x s64>) = G_LOAD [[COPY]](p4) :: (load 24, align 32, addrspace 4)
-    ; CI: [[EXTRACT:%[0-9]+]]:_(<3 x s64>) = G_EXTRACT [[LOAD]](<4 x s64>), 0
+    ; CI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s64>) = G_CONCAT_VECTORS [[LOAD]](<4 x s64>), %2(<4 x s64>), %2(<4 x s64>)
+    ; CI: [[UV:%[0-9]+]]:_(<3 x s64>), [[UV1:%[0-9]+]]:_(<3 x s64>), [[UV2:%[0-9]+]]:_(<3 x s64>), [[UV3:%[0-9]+]]:_(<3 x s64>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s64>)
     ; CI: [[DEF:%[0-9]+]]:_(<4 x s64>) = G_IMPLICIT_DEF
-    ; CI: [[INSERT:%[0-9]+]]:_(<4 x s64>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s64>), 0
+    ; CI: [[INSERT:%[0-9]+]]:_(<4 x s64>) = G_INSERT [[DEF]], [[UV]](<3 x s64>), 0
     ; CI: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[INSERT]](<4 x s64>)
     ; VI-LABEL: name: test_load_constant_v3s64_align32
     ; VI: [[COPY:%[0-9]+]]:_(p4) = COPY $vgpr0_vgpr1
     ; VI: [[LOAD:%[0-9]+]]:_(<4 x s64>) = G_LOAD [[COPY]](p4) :: (load 24, align 32, addrspace 4)
-    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s64>) = G_EXTRACT [[LOAD]](<4 x s64>), 0
+    ; VI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s64>) = G_CONCAT_VECTORS [[LOAD]](<4 x s64>), %2(<4 x s64>), %2(<4 x s64>)
+    ; VI: [[UV:%[0-9]+]]:_(<3 x s64>), [[UV1:%[0-9]+]]:_(<3 x s64>), [[UV2:%[0-9]+]]:_(<3 x s64>), [[UV3:%[0-9]+]]:_(<3 x s64>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s64>)
     ; VI: [[DEF:%[0-9]+]]:_(<4 x s64>) = G_IMPLICIT_DEF
-    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s64>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s64>), 0
+    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s64>) = G_INSERT [[DEF]], [[UV]](<3 x s64>), 0
     ; VI: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[INSERT]](<4 x s64>)
     ; GFX9-LABEL: name: test_load_constant_v3s64_align32
     ; GFX9: [[COPY:%[0-9]+]]:_(p4) = COPY $vgpr0_vgpr1
     ; GFX9: [[LOAD:%[0-9]+]]:_(<4 x s64>) = G_LOAD [[COPY]](p4) :: (load 24, align 32, addrspace 4)
-    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s64>) = G_EXTRACT [[LOAD]](<4 x s64>), 0
+    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s64>) = G_CONCAT_VECTORS [[LOAD]](<4 x s64>), %2(<4 x s64>), %2(<4 x s64>)
+    ; GFX9: [[UV:%[0-9]+]]:_(<3 x s64>), [[UV1:%[0-9]+]]:_(<3 x s64>), [[UV2:%[0-9]+]]:_(<3 x s64>), [[UV3:%[0-9]+]]:_(<3 x s64>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s64>)
     ; GFX9: [[DEF:%[0-9]+]]:_(<4 x s64>) = G_IMPLICIT_DEF
-    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s64>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s64>), 0
+    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s64>) = G_INSERT [[DEF]], [[UV]](<3 x s64>), 0
     ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[INSERT]](<4 x s64>)
     ; CI-MESA-LABEL: name: test_load_constant_v3s64_align32
     ; CI-MESA: [[COPY:%[0-9]+]]:_(p4) = COPY $vgpr0_vgpr1
     ; CI-MESA: [[LOAD:%[0-9]+]]:_(<4 x s64>) = G_LOAD [[COPY]](p4) :: (load 24, align 32, addrspace 4)
-    ; CI-MESA: [[EXTRACT:%[0-9]+]]:_(<3 x s64>) = G_EXTRACT [[LOAD]](<4 x s64>), 0
+    ; CI-MESA: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s64>) = G_CONCAT_VECTORS [[LOAD]](<4 x s64>), %2(<4 x s64>), %2(<4 x s64>)
+    ; CI-MESA: [[UV:%[0-9]+]]:_(<3 x s64>), [[UV1:%[0-9]+]]:_(<3 x s64>), [[UV2:%[0-9]+]]:_(<3 x s64>), [[UV3:%[0-9]+]]:_(<3 x s64>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s64>)
     ; CI-MESA: [[DEF:%[0-9]+]]:_(<4 x s64>) = G_IMPLICIT_DEF
-    ; CI-MESA: [[INSERT:%[0-9]+]]:_(<4 x s64>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s64>), 0
+    ; CI-MESA: [[INSERT:%[0-9]+]]:_(<4 x s64>) = G_INSERT [[DEF]], [[UV]](<3 x s64>), 0
     ; CI-MESA: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[INSERT]](<4 x s64>)
     ; GFX9-MESA-LABEL: name: test_load_constant_v3s64_align32
     ; GFX9-MESA: [[COPY:%[0-9]+]]:_(p4) = COPY $vgpr0_vgpr1
     ; GFX9-MESA: [[LOAD:%[0-9]+]]:_(<4 x s64>) = G_LOAD [[COPY]](p4) :: (load 24, align 32, addrspace 4)
-    ; GFX9-MESA: [[EXTRACT:%[0-9]+]]:_(<3 x s64>) = G_EXTRACT [[LOAD]](<4 x s64>), 0
+    ; GFX9-MESA: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s64>) = G_CONCAT_VECTORS [[LOAD]](<4 x s64>), %2(<4 x s64>), %2(<4 x s64>)
+    ; GFX9-MESA: [[UV:%[0-9]+]]:_(<3 x s64>), [[UV1:%[0-9]+]]:_(<3 x s64>), [[UV2:%[0-9]+]]:_(<3 x s64>), [[UV3:%[0-9]+]]:_(<3 x s64>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s64>)
     ; GFX9-MESA: [[DEF:%[0-9]+]]:_(<4 x s64>) = G_IMPLICIT_DEF
-    ; GFX9-MESA: [[INSERT:%[0-9]+]]:_(<4 x s64>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s64>), 0
+    ; GFX9-MESA: [[INSERT:%[0-9]+]]:_(<4 x s64>) = G_INSERT [[DEF]], [[UV]](<3 x s64>), 0
     ; GFX9-MESA: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[INSERT]](<4 x s64>)
     %0:_(p4) = COPY $vgpr0_vgpr1
     %1:_(<3 x s64>) = G_LOAD %0 :: (load 24, align 32, addrspace 4)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-flat.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-flat.mir
index a3f43848e8feb..034f3903c6538 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-flat.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-flat.mir
@@ -5924,37 +5924,42 @@ body: |
     ; CI-LABEL: name: test_load_flat_v3s16_align8
     ; CI: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
     ; CI: [[LOAD:%[0-9]+]]:_(<4 x s16>) = G_LOAD [[COPY]](p0) :: (load 6, align 8)
-    ; CI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[LOAD]](<4 x s16>), 0
+    ; CI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[LOAD]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; CI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; CI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; CI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s16>), 0
+    ; CI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV]](<3 x s16>), 0
     ; CI: $vgpr0_vgpr1 = COPY [[INSERT]](<4 x s16>)
     ; VI-LABEL: name: test_load_flat_v3s16_align8
     ; VI: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
     ; VI: [[LOAD:%[0-9]+]]:_(<4 x s16>) = G_LOAD [[COPY]](p0) :: (load 6, align 8)
-    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[LOAD]](<4 x s16>), 0
+    ; VI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[LOAD]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; VI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; VI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s16>), 0
+    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV]](<3 x s16>), 0
     ; VI: $vgpr0_vgpr1 = COPY [[INSERT]](<4 x s16>)
     ; GFX9-LABEL: name: test_load_flat_v3s16_align8
     ; GFX9: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
     ; GFX9: [[LOAD:%[0-9]+]]:_(<4 x s16>) = G_LOAD [[COPY]](p0) :: (load 6, align 8)
-    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[LOAD]](<4 x s16>), 0
+    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[LOAD]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; GFX9: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; GFX9: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s16>), 0
+    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV]](<3 x s16>), 0
     ; GFX9: $vgpr0_vgpr1 = COPY [[INSERT]](<4 x s16>)
     ; CI-MESA-LABEL: name: test_load_flat_v3s16_align8
     ; CI-MESA: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
     ; CI-MESA: [[LOAD:%[0-9]+]]:_(<4 x s16>) = G_LOAD [[COPY]](p0) :: (load 6, align 8)
-    ; CI-MESA: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[LOAD]](<4 x s16>), 0
+    ; CI-MESA: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[LOAD]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; CI-MESA: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; CI-MESA: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; CI-MESA: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s16>), 0
+    ; CI-MESA: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV]](<3 x s16>), 0
     ; CI-MESA: $vgpr0_vgpr1 = COPY [[INSERT]](<4 x s16>)
     ; GFX9-MESA-LABEL: name: test_load_flat_v3s16_align8
     ; GFX9-MESA: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
     ; GFX9-MESA: [[LOAD:%[0-9]+]]:_(<4 x s16>) = G_LOAD [[COPY]](p0) :: (load 6, align 8)
-    ; GFX9-MESA: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[LOAD]](<4 x s16>), 0
+    ; GFX9-MESA: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[LOAD]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; GFX9-MESA: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; GFX9-MESA: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9-MESA: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s16>), 0
+    ; GFX9-MESA: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV]](<3 x s16>), 0
     ; GFX9-MESA: $vgpr0_vgpr1 = COPY [[INSERT]](<4 x s16>)
     %0:_(p0) = COPY $vgpr0_vgpr1
     %1:_(<3 x s16>) = G_LOAD %0 :: (load 6, align 8, addrspace 0)
@@ -6032,15 +6037,18 @@ body: |
     ; CI: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p0) :: (load 2 + 4)
     ; CI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD2]](s32)
     ; CI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; CI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; CI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; CI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; CI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; CI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
+    ; CI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
     ; CI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT]], [[BITCAST]](<2 x s16>), 0
-    ; CI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT1]](<4 x s16>), 0
-    ; CI: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
+    ; CI: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT1]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; CI: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; CI: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
     ; CI: [[INSERT3:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT2]], [[TRUNC]](s16), 32
-    ; CI: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT3]](<4 x s16>), 0
-    ; CI: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT2]](<3 x s16>), 0
+    ; CI: [[CONCAT_VECTORS2:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT3]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; CI: [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>), [[UV10:%[0-9]+]]:_(<3 x s16>), [[UV11:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS2]](<12 x s16>)
+    ; CI: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV8]](<3 x s16>), 0
     ; CI: $vgpr0_vgpr1 = COPY [[INSERT4]](<4 x s16>)
     ; VI-LABEL: name: test_load_flat_v3s16_align2
     ; VI: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
@@ -6062,15 +6070,18 @@ body: |
     ; VI: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p0) :: (load 2 + 4)
     ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD2]](s32)
     ; VI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; VI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; VI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; VI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
+    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
     ; VI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT]], [[BITCAST]](<2 x s16>), 0
-    ; VI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT1]](<4 x s16>), 0
-    ; VI: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
+    ; VI: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT1]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; VI: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; VI: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
     ; VI: [[INSERT3:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT2]], [[TRUNC]](s16), 32
-    ; VI: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT3]](<4 x s16>), 0
-    ; VI: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT2]](<3 x s16>), 0
+    ; VI: [[CONCAT_VECTORS2:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT3]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; VI: [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>), [[UV10:%[0-9]+]]:_(<3 x s16>), [[UV11:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS2]](<12 x s16>)
+    ; VI: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV8]](<3 x s16>), 0
     ; VI: $vgpr0_vgpr1 = COPY [[INSERT4]](<4 x s16>)
     ; GFX9-LABEL: name: test_load_flat_v3s16_align2
     ; GFX9: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
@@ -6086,15 +6097,18 @@ body: |
     ; GFX9: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p0) :: (load 2 + 4)
     ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD2]](s32)
     ; GFX9: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; GFX9: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; GFX9: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
+    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
     ; GFX9: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT]], [[BUILD_VECTOR_TRUNC]](<2 x s16>), 0
-    ; GFX9: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT1]](<4 x s16>), 0
-    ; GFX9: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
+    ; GFX9: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT1]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX9: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; GFX9: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
     ; GFX9: [[INSERT3:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT2]], [[TRUNC]](s16), 32
-    ; GFX9: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT3]](<4 x s16>), 0
-    ; GFX9: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT2]](<3 x s16>), 0
+    ; GFX9: [[CONCAT_VECTORS2:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT3]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX9: [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>), [[UV10:%[0-9]+]]:_(<3 x s16>), [[UV11:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS2]](<12 x s16>)
+    ; GFX9: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV8]](<3 x s16>), 0
     ; GFX9: $vgpr0_vgpr1 = COPY [[INSERT4]](<4 x s16>)
     ; CI-MESA-LABEL: name: test_load_flat_v3s16_align2
     ; CI-MESA: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
@@ -6116,15 +6130,18 @@ body: |
     ; CI-MESA: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p0) :: (load 2 + 4)
     ; CI-MESA: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD2]](s32)
     ; CI-MESA: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; CI-MESA: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; CI-MESA: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; CI-MESA: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; CI-MESA: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; CI-MESA: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
+    ; CI-MESA: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
     ; CI-MESA: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT]], [[BITCAST]](<2 x s16>), 0
-    ; CI-MESA: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT1]](<4 x s16>), 0
-    ; CI-MESA: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
+    ; CI-MESA: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT1]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; CI-MESA: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; CI-MESA: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
     ; CI-MESA: [[INSERT3:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT2]], [[TRUNC]](s16), 32
-    ; CI-MESA: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT3]](<4 x s16>), 0
-    ; CI-MESA: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT2]](<3 x s16>), 0
+    ; CI-MESA: [[CONCAT_VECTORS2:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT3]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; CI-MESA: [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>), [[UV10:%[0-9]+]]:_(<3 x s16>), [[UV11:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS2]](<12 x s16>)
+    ; CI-MESA: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV8]](<3 x s16>), 0
     ; CI-MESA: $vgpr0_vgpr1 = COPY [[INSERT4]](<4 x s16>)
     ; GFX9-MESA-LABEL: name: test_load_flat_v3s16_align2
     ; GFX9-MESA: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
@@ -6140,15 +6157,18 @@ body: |
     ; GFX9-MESA: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p0) :: (load 2 + 4)
     ; GFX9-MESA: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD2]](s32)
     ; GFX9-MESA: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9-MESA: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; GFX9-MESA: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; GFX9-MESA: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; GFX9-MESA: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9-MESA: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
+    ; GFX9-MESA: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
     ; GFX9-MESA: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT]], [[BUILD_VECTOR_TRUNC]](<2 x s16>), 0
-    ; GFX9-MESA: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT1]](<4 x s16>), 0
-    ; GFX9-MESA: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
+    ; GFX9-MESA: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT1]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX9-MESA: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; GFX9-MESA: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
     ; GFX9-MESA: [[INSERT3:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT2]], [[TRUNC]](s16), 32
-    ; GFX9-MESA: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT3]](<4 x s16>), 0
-    ; GFX9-MESA: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT2]](<3 x s16>), 0
+    ; GFX9-MESA: [[CONCAT_VECTORS2:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT3]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX9-MESA: [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>), [[UV10:%[0-9]+]]:_(<3 x s16>), [[UV11:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS2]](<12 x s16>)
+    ; GFX9-MESA: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV8]](<3 x s16>), 0
     ; GFX9-MESA: $vgpr0_vgpr1 = COPY [[INSERT4]](<4 x s16>)
     %0:_(p0) = COPY $vgpr0_vgpr1
     %1:_(<3 x s16>) = G_LOAD %0 :: (load 6, align 2, addrspace 0)
@@ -6212,15 +6232,18 @@ body: |
     ; CI: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[SHL3]](s32)
     ; CI: [[OR3:%[0-9]+]]:_(s16) = G_OR [[AND4]], [[TRUNC5]]
     ; CI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; CI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; CI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; CI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; CI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; CI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
+    ; CI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
     ; CI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT]], [[BITCAST]](<2 x s16>), 0
-    ; CI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT1]](<4 x s16>), 0
-    ; CI: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
+    ; CI: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT1]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; CI: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; CI: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
     ; CI: [[INSERT3:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT2]], [[OR3]](s16), 32
-    ; CI: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT3]](<4 x s16>), 0
-    ; CI: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT2]](<3 x s16>), 0
+    ; CI: [[CONCAT_VECTORS2:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT3]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; CI: [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>), [[UV10:%[0-9]+]]:_(<3 x s16>), [[UV11:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS2]](<12 x s16>)
+    ; CI: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV8]](<3 x s16>), 0
     ; CI: $vgpr0_vgpr1 = COPY [[INSERT4]](<4 x s16>)
     ; VI-LABEL: name: test_load_flat_v3s16_align1
     ; VI: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
@@ -6265,15 +6288,18 @@ body: |
     ; VI: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[AND5]], [[C2]](s16)
     ; VI: [[OR3:%[0-9]+]]:_(s16) = G_OR [[AND4]], [[SHL3]]
     ; VI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; VI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; VI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; VI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
+    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
     ; VI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT]], [[BITCAST]](<2 x s16>), 0
-    ; VI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT1]](<4 x s16>), 0
-    ; VI: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
+    ; VI: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT1]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; VI: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; VI: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
     ; VI: [[INSERT3:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT2]], [[OR3]](s16), 32
-    ; VI: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT3]](<4 x s16>), 0
-    ; VI: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT2]](<3 x s16>), 0
+    ; VI: [[CONCAT_VECTORS2:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT3]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; VI: [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>), [[UV10:%[0-9]+]]:_(<3 x s16>), [[UV11:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS2]](<12 x s16>)
+    ; VI: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV8]](<3 x s16>), 0
     ; VI: $vgpr0_vgpr1 = COPY [[INSERT4]](<4 x s16>)
     ; GFX9-LABEL: name: test_load_flat_v3s16_align1
     ; GFX9: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
@@ -6315,15 +6341,18 @@ body: |
     ; GFX9: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[AND5]], [[C2]](s16)
     ; GFX9: [[OR2:%[0-9]+]]:_(s16) = G_OR [[AND4]], [[SHL2]]
     ; GFX9: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; GFX9: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; GFX9: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
+    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
     ; GFX9: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT]], [[BUILD_VECTOR_TRUNC]](<2 x s16>), 0
-    ; GFX9: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT1]](<4 x s16>), 0
-    ; GFX9: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
+    ; GFX9: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT1]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX9: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; GFX9: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
     ; GFX9: [[INSERT3:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT2]], [[OR2]](s16), 32
-    ; GFX9: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT3]](<4 x s16>), 0
-    ; GFX9: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT2]](<3 x s16>), 0
+    ; GFX9: [[CONCAT_VECTORS2:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT3]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX9: [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>), [[UV10:%[0-9]+]]:_(<3 x s16>), [[UV11:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS2]](<12 x s16>)
+    ; GFX9: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV8]](<3 x s16>), 0
     ; GFX9: $vgpr0_vgpr1 = COPY [[INSERT4]](<4 x s16>)
     ; CI-MESA-LABEL: name: test_load_flat_v3s16_align1
     ; CI-MESA: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
@@ -6374,15 +6403,18 @@ body: |
     ; CI-MESA: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[SHL3]](s32)
     ; CI-MESA: [[OR3:%[0-9]+]]:_(s16) = G_OR [[AND4]], [[TRUNC5]]
     ; CI-MESA: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; CI-MESA: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; CI-MESA: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; CI-MESA: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; CI-MESA: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; CI-MESA: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
+    ; CI-MESA: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
     ; CI-MESA: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT]], [[BITCAST]](<2 x s16>), 0
-    ; CI-MESA: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT1]](<4 x s16>), 0
-    ; CI-MESA: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
+    ; CI-MESA: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT1]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; CI-MESA: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; CI-MESA: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
     ; CI-MESA: [[INSERT3:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT2]], [[OR3]](s16), 32
-    ; CI-MESA: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT3]](<4 x s16>), 0
-    ; CI-MESA: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT2]](<3 x s16>), 0
+    ; CI-MESA: [[CONCAT_VECTORS2:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT3]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; CI-MESA: [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>), [[UV10:%[0-9]+]]:_(<3 x s16>), [[UV11:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS2]](<12 x s16>)
+    ; CI-MESA: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV8]](<3 x s16>), 0
     ; CI-MESA: $vgpr0_vgpr1 = COPY [[INSERT4]](<4 x s16>)
     ; GFX9-MESA-LABEL: name: test_load_flat_v3s16_align1
     ; GFX9-MESA: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
@@ -6424,15 +6456,18 @@ body: |
     ; GFX9-MESA: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[AND5]], [[C2]](s16)
     ; GFX9-MESA: [[OR2:%[0-9]+]]:_(s16) = G_OR [[AND4]], [[SHL2]]
     ; GFX9-MESA: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9-MESA: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; GFX9-MESA: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; GFX9-MESA: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; GFX9-MESA: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9-MESA: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
+    ; GFX9-MESA: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
     ; GFX9-MESA: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT]], [[BUILD_VECTOR_TRUNC]](<2 x s16>), 0
-    ; GFX9-MESA: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT1]](<4 x s16>), 0
-    ; GFX9-MESA: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
+    ; GFX9-MESA: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT1]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX9-MESA: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; GFX9-MESA: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
     ; GFX9-MESA: [[INSERT3:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT2]], [[OR2]](s16), 32
-    ; GFX9-MESA: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT3]](<4 x s16>), 0
-    ; GFX9-MESA: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT2]](<3 x s16>), 0
+    ; GFX9-MESA: [[CONCAT_VECTORS2:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT3]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX9-MESA: [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>), [[UV10:%[0-9]+]]:_(<3 x s16>), [[UV11:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS2]](<12 x s16>)
+    ; GFX9-MESA: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV8]](<3 x s16>), 0
     ; GFX9-MESA: $vgpr0_vgpr1 = COPY [[INSERT4]](<4 x s16>)
     %0:_(p0) = COPY $vgpr0_vgpr1
     %1:_(<3 x s16>) = G_LOAD %0 :: (load 6, align 1, addrspace 0)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-global.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-global.mir
index a8e3aa5c8c7ce..8169aa76bfa6e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-global.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-global.mir
@@ -1203,8 +1203,10 @@ body: |
     ; SI-LABEL: name: test_load_global_s96_align16
     ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; SI: [[LOAD:%[0-9]+]]:_(<4 x s32>) = G_LOAD [[COPY]](p1) :: (load 12, align 16, addrspace 1)
-    ; SI: [[EXTRACT:%[0-9]+]]:_(<3 x s32>) = G_EXTRACT [[LOAD]](<4 x s32>), 0
-    ; SI: [[BITCAST:%[0-9]+]]:_(s96) = G_BITCAST [[EXTRACT]](<3 x s32>)
+    ; SI: [[DEF:%[0-9]+]]:_(<4 x s32>) = G_IMPLICIT_DEF
+    ; SI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s32>) = G_CONCAT_VECTORS [[LOAD]](<4 x s32>), [[DEF]](<4 x s32>), [[DEF]](<4 x s32>)
+    ; SI: [[UV:%[0-9]+]]:_(<3 x s32>), [[UV1:%[0-9]+]]:_(<3 x s32>), [[UV2:%[0-9]+]]:_(<3 x s32>), [[UV3:%[0-9]+]]:_(<3 x s32>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s32>)
+    ; SI: [[BITCAST:%[0-9]+]]:_(s96) = G_BITCAST [[UV]](<3 x s32>)
     ; SI: $vgpr0_vgpr1_vgpr2 = COPY [[BITCAST]](s96)
     ; CI-HSA-LABEL: name: test_load_global_s96_align16
     ; CI-HSA: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
@@ -5929,44 +5931,50 @@ body: |
     ; SI-LABEL: name: test_load_global_v3s16_align8
     ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; SI: [[LOAD:%[0-9]+]]:_(<4 x s16>) = G_LOAD [[COPY]](p1) :: (load 6, align 8, addrspace 1)
-    ; SI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[LOAD]](<4 x s16>), 0
+    ; SI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[LOAD]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; SI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; SI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s16>), 0
+    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV]](<3 x s16>), 0
     ; SI: $vgpr0_vgpr1 = COPY [[INSERT]](<4 x s16>)
     ; CI-HSA-LABEL: name: test_load_global_v3s16_align8
     ; CI-HSA: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; CI-HSA: [[LOAD:%[0-9]+]]:_(<4 x s16>) = G_LOAD [[COPY]](p1) :: (load 6, align 8, addrspace 1)
-    ; CI-HSA: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[LOAD]](<4 x s16>), 0
+    ; CI-HSA: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[LOAD]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; CI-HSA: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; CI-HSA: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; CI-HSA: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s16>), 0
+    ; CI-HSA: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV]](<3 x s16>), 0
     ; CI-HSA: $vgpr0_vgpr1 = COPY [[INSERT]](<4 x s16>)
     ; CI-MESA-LABEL: name: test_load_global_v3s16_align8
     ; CI-MESA: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; CI-MESA: [[LOAD:%[0-9]+]]:_(<4 x s16>) = G_LOAD [[COPY]](p1) :: (load 6, align 8, addrspace 1)
-    ; CI-MESA: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[LOAD]](<4 x s16>), 0
+    ; CI-MESA: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[LOAD]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; CI-MESA: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; CI-MESA: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; CI-MESA: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s16>), 0
+    ; CI-MESA: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV]](<3 x s16>), 0
     ; CI-MESA: $vgpr0_vgpr1 = COPY [[INSERT]](<4 x s16>)
     ; VI-LABEL: name: test_load_global_v3s16_align8
     ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; VI: [[LOAD:%[0-9]+]]:_(<4 x s16>) = G_LOAD [[COPY]](p1) :: (load 6, align 8, addrspace 1)
-    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[LOAD]](<4 x s16>), 0
+    ; VI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[LOAD]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; VI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; VI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s16>), 0
+    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV]](<3 x s16>), 0
     ; VI: $vgpr0_vgpr1 = COPY [[INSERT]](<4 x s16>)
     ; GFX9-HSA-LABEL: name: test_load_global_v3s16_align8
     ; GFX9-HSA: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; GFX9-HSA: [[LOAD:%[0-9]+]]:_(<4 x s16>) = G_LOAD [[COPY]](p1) :: (load 6, align 8, addrspace 1)
-    ; GFX9-HSA: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[LOAD]](<4 x s16>), 0
+    ; GFX9-HSA: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[LOAD]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; GFX9-HSA: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; GFX9-HSA: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9-HSA: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s16>), 0
+    ; GFX9-HSA: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV]](<3 x s16>), 0
     ; GFX9-HSA: $vgpr0_vgpr1 = COPY [[INSERT]](<4 x s16>)
     ; GFX9-MESA-LABEL: name: test_load_global_v3s16_align8
     ; GFX9-MESA: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; GFX9-MESA: [[LOAD:%[0-9]+]]:_(<4 x s16>) = G_LOAD [[COPY]](p1) :: (load 6, align 8, addrspace 1)
-    ; GFX9-MESA: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[LOAD]](<4 x s16>), 0
+    ; GFX9-MESA: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[LOAD]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; GFX9-MESA: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; GFX9-MESA: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9-MESA: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s16>), 0
+    ; GFX9-MESA: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV]](<3 x s16>), 0
     ; GFX9-MESA: $vgpr0_vgpr1 = COPY [[INSERT]](<4 x s16>)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(<3 x s16>) = G_LOAD %0 :: (load 6, align 8, addrspace 1)
@@ -6056,15 +6064,18 @@ body: |
     ; SI: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p1) :: (load 2 + 4, addrspace 1)
     ; SI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD2]](s32)
     ; SI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; SI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; SI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; SI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; SI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
+    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
     ; SI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT]], [[BITCAST]](<2 x s16>), 0
-    ; SI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT1]](<4 x s16>), 0
-    ; SI: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
+    ; SI: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT1]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; SI: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; SI: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
     ; SI: [[INSERT3:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT2]], [[TRUNC]](s16), 32
-    ; SI: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT3]](<4 x s16>), 0
-    ; SI: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT2]](<3 x s16>), 0
+    ; SI: [[CONCAT_VECTORS2:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT3]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; SI: [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>), [[UV10:%[0-9]+]]:_(<3 x s16>), [[UV11:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS2]](<12 x s16>)
+    ; SI: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV8]](<3 x s16>), 0
     ; SI: $vgpr0_vgpr1 = COPY [[INSERT4]](<4 x s16>)
     ; CI-HSA-LABEL: name: test_load_global_v3s16_align2
     ; CI-HSA: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
@@ -6092,15 +6103,18 @@ body: |
     ; CI-MESA: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p1) :: (load 2 + 4, addrspace 1)
     ; CI-MESA: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD2]](s32)
     ; CI-MESA: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; CI-MESA: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; CI-MESA: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; CI-MESA: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; CI-MESA: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; CI-MESA: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
+    ; CI-MESA: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
     ; CI-MESA: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT]], [[BITCAST]](<2 x s16>), 0
-    ; CI-MESA: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT1]](<4 x s16>), 0
-    ; CI-MESA: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
+    ; CI-MESA: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT1]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; CI-MESA: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; CI-MESA: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
     ; CI-MESA: [[INSERT3:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT2]], [[TRUNC]](s16), 32
-    ; CI-MESA: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT3]](<4 x s16>), 0
-    ; CI-MESA: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT2]](<3 x s16>), 0
+    ; CI-MESA: [[CONCAT_VECTORS2:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT3]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; CI-MESA: [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>), [[UV10:%[0-9]+]]:_(<3 x s16>), [[UV11:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS2]](<12 x s16>)
+    ; CI-MESA: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV8]](<3 x s16>), 0
     ; CI-MESA: $vgpr0_vgpr1 = COPY [[INSERT4]](<4 x s16>)
     ; VI-LABEL: name: test_load_global_v3s16_align2
     ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
@@ -6122,15 +6136,18 @@ body: |
     ; VI: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p1) :: (load 2 + 4, addrspace 1)
     ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD2]](s32)
     ; VI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; VI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; VI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; VI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
+    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
     ; VI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT]], [[BITCAST]](<2 x s16>), 0
-    ; VI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT1]](<4 x s16>), 0
-    ; VI: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
+    ; VI: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT1]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; VI: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; VI: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
     ; VI: [[INSERT3:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT2]], [[TRUNC]](s16), 32
-    ; VI: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT3]](<4 x s16>), 0
-    ; VI: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT2]](<3 x s16>), 0
+    ; VI: [[CONCAT_VECTORS2:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT3]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; VI: [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>), [[UV10:%[0-9]+]]:_(<3 x s16>), [[UV11:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS2]](<12 x s16>)
+    ; VI: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV8]](<3 x s16>), 0
     ; VI: $vgpr0_vgpr1 = COPY [[INSERT4]](<4 x s16>)
     ; GFX9-HSA-LABEL: name: test_load_global_v3s16_align2
     ; GFX9-HSA: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
@@ -6152,15 +6169,18 @@ body: |
     ; GFX9-MESA: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p1) :: (load 2 + 4, addrspace 1)
     ; GFX9-MESA: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD2]](s32)
     ; GFX9-MESA: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9-MESA: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; GFX9-MESA: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; GFX9-MESA: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; GFX9-MESA: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9-MESA: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
+    ; GFX9-MESA: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
     ; GFX9-MESA: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT]], [[BUILD_VECTOR_TRUNC]](<2 x s16>), 0
-    ; GFX9-MESA: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT1]](<4 x s16>), 0
-    ; GFX9-MESA: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
+    ; GFX9-MESA: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT1]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX9-MESA: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; GFX9-MESA: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
     ; GFX9-MESA: [[INSERT3:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT2]], [[TRUNC]](s16), 32
-    ; GFX9-MESA: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT3]](<4 x s16>), 0
-    ; GFX9-MESA: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT2]](<3 x s16>), 0
+    ; GFX9-MESA: [[CONCAT_VECTORS2:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT3]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX9-MESA: [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>), [[UV10:%[0-9]+]]:_(<3 x s16>), [[UV11:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS2]](<12 x s16>)
+    ; GFX9-MESA: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV8]](<3 x s16>), 0
     ; GFX9-MESA: $vgpr0_vgpr1 = COPY [[INSERT4]](<4 x s16>)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(<3 x s16>) = G_LOAD %0 :: (load 6, align 2, addrspace 1)
@@ -6224,15 +6244,18 @@ body: |
     ; SI: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[SHL3]](s32)
     ; SI: [[OR3:%[0-9]+]]:_(s16) = G_OR [[AND4]], [[TRUNC5]]
     ; SI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; SI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; SI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; SI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; SI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
+    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
     ; SI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT]], [[BITCAST]](<2 x s16>), 0
-    ; SI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT1]](<4 x s16>), 0
-    ; SI: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
+    ; SI: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT1]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; SI: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; SI: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
     ; SI: [[INSERT3:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT2]], [[OR3]](s16), 32
-    ; SI: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT3]](<4 x s16>), 0
-    ; SI: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT2]](<3 x s16>), 0
+    ; SI: [[CONCAT_VECTORS2:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT3]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; SI: [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>), [[UV10:%[0-9]+]]:_(<3 x s16>), [[UV11:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS2]](<12 x s16>)
+    ; SI: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV8]](<3 x s16>), 0
     ; SI: $vgpr0_vgpr1 = COPY [[INSERT4]](<4 x s16>)
     ; CI-HSA-LABEL: name: test_load_global_v3s16_align1
     ; CI-HSA: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
@@ -6289,15 +6312,18 @@ body: |
     ; CI-MESA: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[SHL3]](s32)
     ; CI-MESA: [[OR3:%[0-9]+]]:_(s16) = G_OR [[AND4]], [[TRUNC5]]
     ; CI-MESA: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; CI-MESA: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; CI-MESA: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; CI-MESA: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; CI-MESA: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; CI-MESA: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
+    ; CI-MESA: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
     ; CI-MESA: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT]], [[BITCAST]](<2 x s16>), 0
-    ; CI-MESA: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT1]](<4 x s16>), 0
-    ; CI-MESA: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
+    ; CI-MESA: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT1]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; CI-MESA: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; CI-MESA: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
     ; CI-MESA: [[INSERT3:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT2]], [[OR3]](s16), 32
-    ; CI-MESA: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT3]](<4 x s16>), 0
-    ; CI-MESA: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT2]](<3 x s16>), 0
+    ; CI-MESA: [[CONCAT_VECTORS2:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT3]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; CI-MESA: [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>), [[UV10:%[0-9]+]]:_(<3 x s16>), [[UV11:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS2]](<12 x s16>)
+    ; CI-MESA: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV8]](<3 x s16>), 0
     ; CI-MESA: $vgpr0_vgpr1 = COPY [[INSERT4]](<4 x s16>)
     ; VI-LABEL: name: test_load_global_v3s16_align1
     ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
@@ -6342,15 +6368,18 @@ body: |
     ; VI: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[AND5]], [[C2]](s16)
     ; VI: [[OR3:%[0-9]+]]:_(s16) = G_OR [[AND4]], [[SHL3]]
     ; VI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; VI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; VI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; VI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
+    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
     ; VI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT]], [[BITCAST]](<2 x s16>), 0
-    ; VI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT1]](<4 x s16>), 0
-    ; VI: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
+    ; VI: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT1]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; VI: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; VI: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
     ; VI: [[INSERT3:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT2]], [[OR3]](s16), 32
-    ; VI: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT3]](<4 x s16>), 0
-    ; VI: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT2]](<3 x s16>), 0
+    ; VI: [[CONCAT_VECTORS2:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT3]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; VI: [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>), [[UV10:%[0-9]+]]:_(<3 x s16>), [[UV11:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS2]](<12 x s16>)
+    ; VI: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV8]](<3 x s16>), 0
     ; VI: $vgpr0_vgpr1 = COPY [[INSERT4]](<4 x s16>)
     ; GFX9-HSA-LABEL: name: test_load_global_v3s16_align1
     ; GFX9-HSA: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
@@ -6398,15 +6427,18 @@ body: |
     ; GFX9-MESA: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[AND5]], [[C2]](s16)
     ; GFX9-MESA: [[OR2:%[0-9]+]]:_(s16) = G_OR [[AND4]], [[SHL2]]
     ; GFX9-MESA: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9-MESA: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; GFX9-MESA: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; GFX9-MESA: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; GFX9-MESA: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9-MESA: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
+    ; GFX9-MESA: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
     ; GFX9-MESA: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT]], [[BUILD_VECTOR_TRUNC]](<2 x s16>), 0
-    ; GFX9-MESA: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT1]](<4 x s16>), 0
-    ; GFX9-MESA: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
+    ; GFX9-MESA: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT1]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX9-MESA: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; GFX9-MESA: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
     ; GFX9-MESA: [[INSERT3:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT2]], [[OR2]](s16), 32
-    ; GFX9-MESA: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT3]](<4 x s16>), 0
-    ; GFX9-MESA: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT2]](<3 x s16>), 0
+    ; GFX9-MESA: [[CONCAT_VECTORS2:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT3]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX9-MESA: [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>), [[UV10:%[0-9]+]]:_(<3 x s16>), [[UV11:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS2]](<12 x s16>)
+    ; GFX9-MESA: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV8]](<3 x s16>), 0
     ; GFX9-MESA: $vgpr0_vgpr1 = COPY [[INSERT4]](<4 x s16>)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(<3 x s16>) = G_LOAD %0 :: (load 6, align 1, addrspace 1)
@@ -7036,8 +7068,10 @@ body: |
     ; SI-LABEL: name: test_load_global_v3s32_align16
     ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; SI: [[LOAD:%[0-9]+]]:_(<4 x s32>) = G_LOAD [[COPY]](p1) :: (load 12, align 16, addrspace 1)
-    ; SI: [[EXTRACT:%[0-9]+]]:_(<3 x s32>) = G_EXTRACT [[LOAD]](<4 x s32>), 0
-    ; SI: $vgpr0_vgpr1_vgpr2 = COPY [[EXTRACT]](<3 x s32>)
+    ; SI: [[DEF:%[0-9]+]]:_(<4 x s32>) = G_IMPLICIT_DEF
+    ; SI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s32>) = G_CONCAT_VECTORS [[LOAD]](<4 x s32>), [[DEF]](<4 x s32>), [[DEF]](<4 x s32>)
+    ; SI: [[UV:%[0-9]+]]:_(<3 x s32>), [[UV1:%[0-9]+]]:_(<3 x s32>), [[UV2:%[0-9]+]]:_(<3 x s32>), [[UV3:%[0-9]+]]:_(<3 x s32>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s32>)
+    ; SI: $vgpr0_vgpr1_vgpr2 = COPY [[UV]](<3 x s32>)
     ; CI-HSA-LABEL: name: test_load_global_v3s32_align16
     ; CI-HSA: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; CI-HSA: [[LOAD:%[0-9]+]]:_(<3 x s32>) = G_LOAD [[COPY]](p1) :: (load 12, align 16, addrspace 1)
@@ -8167,44 +8201,50 @@ body: |
     ; SI-LABEL: name: test_load_global_v3s64_align32
     ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; SI: [[LOAD:%[0-9]+]]:_(<4 x s64>) = G_LOAD [[COPY]](p1) :: (load 24, align 32, addrspace 1)
-    ; SI: [[EXTRACT:%[0-9]+]]:_(<3 x s64>) = G_EXTRACT [[LOAD]](<4 x s64>), 0
+    ; SI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s64>) = G_CONCAT_VECTORS [[LOAD]](<4 x s64>), %2(<4 x s64>), %2(<4 x s64>)
+    ; SI: [[UV:%[0-9]+]]:_(<3 x s64>), [[UV1:%[0-9]+]]:_(<3 x s64>), [[UV2:%[0-9]+]]:_(<3 x s64>), [[UV3:%[0-9]+]]:_(<3 x s64>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s64>)
     ; SI: [[DEF:%[0-9]+]]:_(<4 x s64>) = G_IMPLICIT_DEF
-    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s64>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s64>), 0
+    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s64>) = G_INSERT [[DEF]], [[UV]](<3 x s64>), 0
     ; SI: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[INSERT]](<4 x s64>)
     ; CI-HSA-LABEL: name: test_load_global_v3s64_align32
     ; CI-HSA: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; CI-HSA: [[LOAD:%[0-9]+]]:_(<4 x s64>) = G_LOAD [[COPY]](p1) :: (load 24, align 32, addrspace 1)
-    ; CI-HSA: [[EXTRACT:%[0-9]+]]:_(<3 x s64>) = G_EXTRACT [[LOAD]](<4 x s64>), 0
+    ; CI-HSA: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s64>) = G_CONCAT_VECTORS [[LOAD]](<4 x s64>), %2(<4 x s64>), %2(<4 x s64>)
+    ; CI-HSA: [[UV:%[0-9]+]]:_(<3 x s64>), [[UV1:%[0-9]+]]:_(<3 x s64>), [[UV2:%[0-9]+]]:_(<3 x s64>), [[UV3:%[0-9]+]]:_(<3 x s64>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s64>)
     ; CI-HSA: [[DEF:%[0-9]+]]:_(<4 x s64>) = G_IMPLICIT_DEF
-    ; CI-HSA: [[INSERT:%[0-9]+]]:_(<4 x s64>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s64>), 0
+    ; CI-HSA: [[INSERT:%[0-9]+]]:_(<4 x s64>) = G_INSERT [[DEF]], [[UV]](<3 x s64>), 0
     ; CI-HSA: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[INSERT]](<4 x s64>)
     ; CI-MESA-LABEL: name: test_load_global_v3s64_align32
     ; CI-MESA: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; CI-MESA: [[LOAD:%[0-9]+]]:_(<4 x s64>) = G_LOAD [[COPY]](p1) :: (load 24, align 32, addrspace 1)
-    ; CI-MESA: [[EXTRACT:%[0-9]+]]:_(<3 x s64>) = G_EXTRACT [[LOAD]](<4 x s64>), 0
+    ; CI-MESA: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s64>) = G_CONCAT_VECTORS [[LOAD]](<4 x s64>), %2(<4 x s64>), %2(<4 x s64>)
+    ; CI-MESA: [[UV:%[0-9]+]]:_(<3 x s64>), [[UV1:%[0-9]+]]:_(<3 x s64>), [[UV2:%[0-9]+]]:_(<3 x s64>), [[UV3:%[0-9]+]]:_(<3 x s64>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s64>)
     ; CI-MESA: [[DEF:%[0-9]+]]:_(<4 x s64>) = G_IMPLICIT_DEF
-    ; CI-MESA: [[INSERT:%[0-9]+]]:_(<4 x s64>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s64>), 0
+    ; CI-MESA: [[INSERT:%[0-9]+]]:_(<4 x s64>) = G_INSERT [[DEF]], [[UV]](<3 x s64>), 0
     ; CI-MESA: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[INSERT]](<4 x s64>)
     ; VI-LABEL: name: test_load_global_v3s64_align32
     ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; VI: [[LOAD:%[0-9]+]]:_(<4 x s64>) = G_LOAD [[COPY]](p1) :: (load 24, align 32, addrspace 1)
-    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s64>) = G_EXTRACT [[LOAD]](<4 x s64>), 0
+    ; VI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s64>) = G_CONCAT_VECTORS [[LOAD]](<4 x s64>), %2(<4 x s64>), %2(<4 x s64>)
+    ; VI: [[UV:%[0-9]+]]:_(<3 x s64>), [[UV1:%[0-9]+]]:_(<3 x s64>), [[UV2:%[0-9]+]]:_(<3 x s64>), [[UV3:%[0-9]+]]:_(<3 x s64>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s64>)
     ; VI: [[DEF:%[0-9]+]]:_(<4 x s64>) = G_IMPLICIT_DEF
-    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s64>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s64>), 0
+    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s64>) = G_INSERT [[DEF]], [[UV]](<3 x s64>), 0
     ; VI: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[INSERT]](<4 x s64>)
     ; GFX9-HSA-LABEL: name: test_load_global_v3s64_align32
     ; GFX9-HSA: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; GFX9-HSA: [[LOAD:%[0-9]+]]:_(<4 x s64>) = G_LOAD [[COPY]](p1) :: (load 24, align 32, addrspace 1)
-    ; GFX9-HSA: [[EXTRACT:%[0-9]+]]:_(<3 x s64>) = G_EXTRACT [[LOAD]](<4 x s64>), 0
+    ; GFX9-HSA: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s64>) = G_CONCAT_VECTORS [[LOAD]](<4 x s64>), %2(<4 x s64>), %2(<4 x s64>)
+    ; GFX9-HSA: [[UV:%[0-9]+]]:_(<3 x s64>), [[UV1:%[0-9]+]]:_(<3 x s64>), [[UV2:%[0-9]+]]:_(<3 x s64>), [[UV3:%[0-9]+]]:_(<3 x s64>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s64>)
     ; GFX9-HSA: [[DEF:%[0-9]+]]:_(<4 x s64>) = G_IMPLICIT_DEF
-    ; GFX9-HSA: [[INSERT:%[0-9]+]]:_(<4 x s64>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s64>), 0
+    ; GFX9-HSA: [[INSERT:%[0-9]+]]:_(<4 x s64>) = G_INSERT [[DEF]], [[UV]](<3 x s64>), 0
     ; GFX9-HSA: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[INSERT]](<4 x s64>)
     ; GFX9-MESA-LABEL: name: test_load_global_v3s64_align32
     ; GFX9-MESA: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; GFX9-MESA: [[LOAD:%[0-9]+]]:_(<4 x s64>) = G_LOAD [[COPY]](p1) :: (load 24, align 32, addrspace 1)
-    ; GFX9-MESA: [[EXTRACT:%[0-9]+]]:_(<3 x s64>) = G_EXTRACT [[LOAD]](<4 x s64>), 0
+    ; GFX9-MESA: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s64>) = G_CONCAT_VECTORS [[LOAD]](<4 x s64>), %2(<4 x s64>), %2(<4 x s64>)
+    ; GFX9-MESA: [[UV:%[0-9]+]]:_(<3 x s64>), [[UV1:%[0-9]+]]:_(<3 x s64>), [[UV2:%[0-9]+]]:_(<3 x s64>), [[UV3:%[0-9]+]]:_(<3 x s64>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s64>)
     ; GFX9-MESA: [[DEF:%[0-9]+]]:_(<4 x s64>) = G_IMPLICIT_DEF
-    ; GFX9-MESA: [[INSERT:%[0-9]+]]:_(<4 x s64>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s64>), 0
+    ; GFX9-MESA: [[INSERT:%[0-9]+]]:_(<4 x s64>) = G_INSERT [[DEF]], [[UV]](<3 x s64>), 0
     ; GFX9-MESA: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[INSERT]](<4 x s64>)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(<3 x s64>) = G_LOAD %0 :: (load 24, align 32, addrspace 1)
@@ -12643,23 +12683,25 @@ body: |
     ; SI-LABEL: name: test_extload_global_v2s96_from_24_align16
     ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; SI: [[LOAD:%[0-9]+]]:_(<4 x s32>) = G_LOAD [[COPY]](p1) :: (load 12, align 16, addrspace 1)
-    ; SI: [[EXTRACT:%[0-9]+]]:_(<3 x s32>) = G_EXTRACT [[LOAD]](<4 x s32>), 0
-    ; SI: [[BITCAST:%[0-9]+]]:_(s96) = G_BITCAST [[EXTRACT]](<3 x s32>)
+    ; SI: [[DEF:%[0-9]+]]:_(<4 x s32>) = G_IMPLICIT_DEF
+    ; SI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s32>) = G_CONCAT_VECTORS [[LOAD]](<4 x s32>), [[DEF]](<4 x s32>), [[DEF]](<4 x s32>)
+    ; SI: [[UV:%[0-9]+]]:_(<3 x s32>), [[UV1:%[0-9]+]]:_(<3 x s32>), [[UV2:%[0-9]+]]:_(<3 x s32>), [[UV3:%[0-9]+]]:_(<3 x s32>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s32>)
+    ; SI: [[BITCAST:%[0-9]+]]:_(s96) = G_BITCAST [[UV]](<3 x s32>)
     ; SI: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 12
     ; SI: [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C]](s64)
     ; SI: [[LOAD1:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[PTR_ADD]](p1) :: (load 8 + 12, align 4, addrspace 1)
     ; SI: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
     ; SI: [[PTR_ADD1:%[0-9]+]]:_(p1) = G_PTR_ADD [[PTR_ADD]], [[C1]](s64)
     ; SI: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p1) :: (load 4 + 20, addrspace 1)
-    ; SI: [[DEF:%[0-9]+]]:_(<3 x s32>) = G_IMPLICIT_DEF
-    ; SI: [[INSERT:%[0-9]+]]:_(<3 x s32>) = G_INSERT [[DEF]], [[LOAD1]](<2 x s32>), 0
+    ; SI: [[DEF1:%[0-9]+]]:_(<3 x s32>) = G_IMPLICIT_DEF
+    ; SI: [[INSERT:%[0-9]+]]:_(<3 x s32>) = G_INSERT [[DEF1]], [[LOAD1]](<2 x s32>), 0
     ; SI: [[INSERT1:%[0-9]+]]:_(<3 x s32>) = G_INSERT [[INSERT]], [[LOAD2]](s32), 64
     ; SI: [[BITCAST1:%[0-9]+]]:_(s96) = G_BITCAST [[INSERT1]](<3 x s32>)
     ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s96>) = G_BUILD_VECTOR [[BITCAST]](s96), [[BITCAST1]](s96)
-    ; SI: [[EXTRACT1:%[0-9]+]]:_(s96) = G_EXTRACT [[BUILD_VECTOR]](<2 x s96>), 0
-    ; SI: [[EXTRACT2:%[0-9]+]]:_(s96) = G_EXTRACT [[BUILD_VECTOR]](<2 x s96>), 96
-    ; SI: $vgpr0_vgpr1_vgpr2 = COPY [[EXTRACT1]](s96)
-    ; SI: $vgpr3_vgpr4_vgpr5 = COPY [[EXTRACT2]](s96)
+    ; SI: [[EXTRACT:%[0-9]+]]:_(s96) = G_EXTRACT [[BUILD_VECTOR]](<2 x s96>), 0
+    ; SI: [[EXTRACT1:%[0-9]+]]:_(s96) = G_EXTRACT [[BUILD_VECTOR]](<2 x s96>), 96
+    ; SI: $vgpr0_vgpr1_vgpr2 = COPY [[EXTRACT]](s96)
+    ; SI: $vgpr3_vgpr4_vgpr5 = COPY [[EXTRACT1]](s96)
     ; CI-HSA-LABEL: name: test_extload_global_v2s96_from_24_align16
     ; CI-HSA: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; CI-HSA: [[LOAD:%[0-9]+]]:_(<3 x s32>) = G_LOAD [[COPY]](p1) :: (load 12, align 16, addrspace 1)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-local.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-local.mir
index f5a852bd98f36..376339482f5a1 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-local.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-local.mir
@@ -6061,37 +6061,42 @@ body: |
     ; SI-LABEL: name: test_load_local_v3s16_align8
     ; SI: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
     ; SI: [[LOAD:%[0-9]+]]:_(<4 x s16>) = G_LOAD [[COPY]](p3) :: (load 6, align 8, addrspace 3)
-    ; SI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[LOAD]](<4 x s16>), 0
+    ; SI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[LOAD]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; SI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; SI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s16>), 0
+    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV]](<3 x s16>), 0
     ; SI: $vgpr0_vgpr1 = COPY [[INSERT]](<4 x s16>)
     ; CI-LABEL: name: test_load_local_v3s16_align8
     ; CI: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
     ; CI: [[LOAD:%[0-9]+]]:_(<4 x s16>) = G_LOAD [[COPY]](p3) :: (load 6, align 8, addrspace 3)
-    ; CI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[LOAD]](<4 x s16>), 0
+    ; CI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[LOAD]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; CI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; CI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; CI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s16>), 0
+    ; CI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV]](<3 x s16>), 0
     ; CI: $vgpr0_vgpr1 = COPY [[INSERT]](<4 x s16>)
     ; CI-DS128-LABEL: name: test_load_local_v3s16_align8
     ; CI-DS128: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
     ; CI-DS128: [[LOAD:%[0-9]+]]:_(<4 x s16>) = G_LOAD [[COPY]](p3) :: (load 6, align 8, addrspace 3)
-    ; CI-DS128: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[LOAD]](<4 x s16>), 0
+    ; CI-DS128: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[LOAD]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; CI-DS128: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; CI-DS128: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; CI-DS128: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s16>), 0
+    ; CI-DS128: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV]](<3 x s16>), 0
     ; CI-DS128: $vgpr0_vgpr1 = COPY [[INSERT]](<4 x s16>)
     ; VI-LABEL: name: test_load_local_v3s16_align8
     ; VI: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
     ; VI: [[LOAD:%[0-9]+]]:_(<4 x s16>) = G_LOAD [[COPY]](p3) :: (load 6, align 8, addrspace 3)
-    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[LOAD]](<4 x s16>), 0
+    ; VI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[LOAD]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; VI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; VI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s16>), 0
+    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV]](<3 x s16>), 0
     ; VI: $vgpr0_vgpr1 = COPY [[INSERT]](<4 x s16>)
     ; GFX9-LABEL: name: test_load_local_v3s16_align8
     ; GFX9: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
     ; GFX9: [[LOAD:%[0-9]+]]:_(<4 x s16>) = G_LOAD [[COPY]](p3) :: (load 6, align 8, addrspace 3)
-    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[LOAD]](<4 x s16>), 0
+    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[LOAD]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; GFX9: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; GFX9: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s16>), 0
+    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV]](<3 x s16>), 0
     ; GFX9: $vgpr0_vgpr1 = COPY [[INSERT]](<4 x s16>)
     %0:_(p3) = COPY $vgpr0
     %1:_(<3 x s16>) = G_LOAD %0 :: (load 6, align 8, addrspace 3)
@@ -6126,15 +6131,18 @@ body: |
     ; SI: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p3) :: (load 2 + 4, addrspace 3)
     ; SI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD2]](s32)
     ; SI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; SI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; SI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; SI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; SI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
+    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
     ; SI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT]], [[BITCAST]](<2 x s16>), 0
-    ; SI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT1]](<4 x s16>), 0
-    ; SI: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
+    ; SI: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT1]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; SI: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; SI: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
     ; SI: [[INSERT3:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT2]], [[TRUNC]](s16), 32
-    ; SI: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT3]](<4 x s16>), 0
-    ; SI: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT2]](<3 x s16>), 0
+    ; SI: [[CONCAT_VECTORS2:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT3]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; SI: [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>), [[UV10:%[0-9]+]]:_(<3 x s16>), [[UV11:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS2]](<12 x s16>)
+    ; SI: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV8]](<3 x s16>), 0
     ; SI: $vgpr0_vgpr1 = COPY [[INSERT4]](<4 x s16>)
     ; CI-LABEL: name: test_load_local_v3s16_align2
     ; CI: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
@@ -6156,15 +6164,18 @@ body: |
     ; CI: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p3) :: (load 2 + 4, addrspace 3)
     ; CI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD2]](s32)
     ; CI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; CI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; CI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; CI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; CI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; CI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
+    ; CI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
     ; CI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT]], [[BITCAST]](<2 x s16>), 0
-    ; CI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT1]](<4 x s16>), 0
-    ; CI: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
+    ; CI: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT1]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; CI: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; CI: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
     ; CI: [[INSERT3:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT2]], [[TRUNC]](s16), 32
-    ; CI: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT3]](<4 x s16>), 0
-    ; CI: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT2]](<3 x s16>), 0
+    ; CI: [[CONCAT_VECTORS2:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT3]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; CI: [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>), [[UV10:%[0-9]+]]:_(<3 x s16>), [[UV11:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS2]](<12 x s16>)
+    ; CI: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV8]](<3 x s16>), 0
     ; CI: $vgpr0_vgpr1 = COPY [[INSERT4]](<4 x s16>)
     ; CI-DS128-LABEL: name: test_load_local_v3s16_align2
     ; CI-DS128: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
@@ -6186,15 +6197,18 @@ body: |
     ; CI-DS128: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p3) :: (load 2 + 4, addrspace 3)
     ; CI-DS128: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD2]](s32)
     ; CI-DS128: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; CI-DS128: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; CI-DS128: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; CI-DS128: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; CI-DS128: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; CI-DS128: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
+    ; CI-DS128: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
     ; CI-DS128: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT]], [[BITCAST]](<2 x s16>), 0
-    ; CI-DS128: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT1]](<4 x s16>), 0
-    ; CI-DS128: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
+    ; CI-DS128: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT1]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; CI-DS128: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; CI-DS128: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
     ; CI-DS128: [[INSERT3:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT2]], [[TRUNC]](s16), 32
-    ; CI-DS128: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT3]](<4 x s16>), 0
-    ; CI-DS128: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT2]](<3 x s16>), 0
+    ; CI-DS128: [[CONCAT_VECTORS2:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT3]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; CI-DS128: [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>), [[UV10:%[0-9]+]]:_(<3 x s16>), [[UV11:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS2]](<12 x s16>)
+    ; CI-DS128: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV8]](<3 x s16>), 0
     ; CI-DS128: $vgpr0_vgpr1 = COPY [[INSERT4]](<4 x s16>)
     ; VI-LABEL: name: test_load_local_v3s16_align2
     ; VI: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
@@ -6216,15 +6230,18 @@ body: |
     ; VI: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p3) :: (load 2 + 4, addrspace 3)
     ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD2]](s32)
     ; VI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; VI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; VI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; VI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
+    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
     ; VI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT]], [[BITCAST]](<2 x s16>), 0
-    ; VI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT1]](<4 x s16>), 0
-    ; VI: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
+    ; VI: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT1]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; VI: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; VI: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
     ; VI: [[INSERT3:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT2]], [[TRUNC]](s16), 32
-    ; VI: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT3]](<4 x s16>), 0
-    ; VI: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT2]](<3 x s16>), 0
+    ; VI: [[CONCAT_VECTORS2:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT3]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; VI: [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>), [[UV10:%[0-9]+]]:_(<3 x s16>), [[UV11:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS2]](<12 x s16>)
+    ; VI: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV8]](<3 x s16>), 0
     ; VI: $vgpr0_vgpr1 = COPY [[INSERT4]](<4 x s16>)
     ; GFX9-LABEL: name: test_load_local_v3s16_align2
     ; GFX9: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
@@ -6240,15 +6257,18 @@ body: |
     ; GFX9: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p3) :: (load 2 + 4, addrspace 3)
     ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD2]](s32)
     ; GFX9: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; GFX9: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; GFX9: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
+    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
     ; GFX9: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT]], [[BUILD_VECTOR_TRUNC]](<2 x s16>), 0
-    ; GFX9: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT1]](<4 x s16>), 0
-    ; GFX9: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
+    ; GFX9: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT1]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX9: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; GFX9: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
     ; GFX9: [[INSERT3:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT2]], [[TRUNC]](s16), 32
-    ; GFX9: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT3]](<4 x s16>), 0
-    ; GFX9: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT2]](<3 x s16>), 0
+    ; GFX9: [[CONCAT_VECTORS2:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT3]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX9: [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>), [[UV10:%[0-9]+]]:_(<3 x s16>), [[UV11:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS2]](<12 x s16>)
+    ; GFX9: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV8]](<3 x s16>), 0
     ; GFX9: $vgpr0_vgpr1 = COPY [[INSERT4]](<4 x s16>)
     %0:_(p3) = COPY $vgpr0
     %1:_(<3 x s16>) = G_LOAD %0 :: (load 6, align 2, addrspace 3)
@@ -6312,15 +6332,18 @@ body: |
     ; SI: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[SHL3]](s32)
     ; SI: [[OR3:%[0-9]+]]:_(s16) = G_OR [[AND4]], [[TRUNC5]]
     ; SI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; SI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; SI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; SI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; SI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
+    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
     ; SI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT]], [[BITCAST]](<2 x s16>), 0
-    ; SI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT1]](<4 x s16>), 0
-    ; SI: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
+    ; SI: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT1]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; SI: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; SI: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
     ; SI: [[INSERT3:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT2]], [[OR3]](s16), 32
-    ; SI: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT3]](<4 x s16>), 0
-    ; SI: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT2]](<3 x s16>), 0
+    ; SI: [[CONCAT_VECTORS2:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT3]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; SI: [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>), [[UV10:%[0-9]+]]:_(<3 x s16>), [[UV11:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS2]](<12 x s16>)
+    ; SI: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV8]](<3 x s16>), 0
     ; SI: $vgpr0_vgpr1 = COPY [[INSERT4]](<4 x s16>)
     ; CI-LABEL: name: test_load_local_v3s16_align1
     ; CI: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
@@ -6371,15 +6394,18 @@ body: |
     ; CI: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[SHL3]](s32)
     ; CI: [[OR3:%[0-9]+]]:_(s16) = G_OR [[AND4]], [[TRUNC5]]
     ; CI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; CI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; CI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; CI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; CI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; CI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
+    ; CI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
     ; CI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT]], [[BITCAST]](<2 x s16>), 0
-    ; CI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT1]](<4 x s16>), 0
-    ; CI: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
+    ; CI: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT1]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; CI: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; CI: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
     ; CI: [[INSERT3:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT2]], [[OR3]](s16), 32
-    ; CI: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT3]](<4 x s16>), 0
-    ; CI: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT2]](<3 x s16>), 0
+    ; CI: [[CONCAT_VECTORS2:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT3]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; CI: [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>), [[UV10:%[0-9]+]]:_(<3 x s16>), [[UV11:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS2]](<12 x s16>)
+    ; CI: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV8]](<3 x s16>), 0
     ; CI: $vgpr0_vgpr1 = COPY [[INSERT4]](<4 x s16>)
     ; CI-DS128-LABEL: name: test_load_local_v3s16_align1
     ; CI-DS128: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
@@ -6430,15 +6456,18 @@ body: |
     ; CI-DS128: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[SHL3]](s32)
     ; CI-DS128: [[OR3:%[0-9]+]]:_(s16) = G_OR [[AND4]], [[TRUNC5]]
     ; CI-DS128: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; CI-DS128: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; CI-DS128: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; CI-DS128: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; CI-DS128: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; CI-DS128: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
+    ; CI-DS128: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
     ; CI-DS128: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT]], [[BITCAST]](<2 x s16>), 0
-    ; CI-DS128: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT1]](<4 x s16>), 0
-    ; CI-DS128: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
+    ; CI-DS128: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT1]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; CI-DS128: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; CI-DS128: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
     ; CI-DS128: [[INSERT3:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT2]], [[OR3]](s16), 32
-    ; CI-DS128: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT3]](<4 x s16>), 0
-    ; CI-DS128: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT2]](<3 x s16>), 0
+    ; CI-DS128: [[CONCAT_VECTORS2:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT3]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; CI-DS128: [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>), [[UV10:%[0-9]+]]:_(<3 x s16>), [[UV11:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS2]](<12 x s16>)
+    ; CI-DS128: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV8]](<3 x s16>), 0
     ; CI-DS128: $vgpr0_vgpr1 = COPY [[INSERT4]](<4 x s16>)
     ; VI-LABEL: name: test_load_local_v3s16_align1
     ; VI: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
@@ -6483,15 +6512,18 @@ body: |
     ; VI: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[AND5]], [[C2]](s16)
     ; VI: [[OR3:%[0-9]+]]:_(s16) = G_OR [[AND4]], [[SHL3]]
     ; VI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; VI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; VI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; VI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
+    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
     ; VI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT]], [[BITCAST]](<2 x s16>), 0
-    ; VI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT1]](<4 x s16>), 0
-    ; VI: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
+    ; VI: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT1]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; VI: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; VI: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
     ; VI: [[INSERT3:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT2]], [[OR3]](s16), 32
-    ; VI: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT3]](<4 x s16>), 0
-    ; VI: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT2]](<3 x s16>), 0
+    ; VI: [[CONCAT_VECTORS2:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT3]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; VI: [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>), [[UV10:%[0-9]+]]:_(<3 x s16>), [[UV11:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS2]](<12 x s16>)
+    ; VI: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV8]](<3 x s16>), 0
     ; VI: $vgpr0_vgpr1 = COPY [[INSERT4]](<4 x s16>)
     ; GFX9-LABEL: name: test_load_local_v3s16_align1
     ; GFX9: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
@@ -6533,15 +6565,18 @@ body: |
     ; GFX9: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[AND5]], [[C2]](s16)
     ; GFX9: [[OR2:%[0-9]+]]:_(s16) = G_OR [[AND4]], [[SHL2]]
     ; GFX9: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; GFX9: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; GFX9: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
+    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
     ; GFX9: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT]], [[BUILD_VECTOR_TRUNC]](<2 x s16>), 0
-    ; GFX9: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT1]](<4 x s16>), 0
-    ; GFX9: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
+    ; GFX9: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT1]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX9: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; GFX9: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
     ; GFX9: [[INSERT3:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT2]], [[OR2]](s16), 32
-    ; GFX9: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT3]](<4 x s16>), 0
-    ; GFX9: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT2]](<3 x s16>), 0
+    ; GFX9: [[CONCAT_VECTORS2:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT3]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX9: [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>), [[UV10:%[0-9]+]]:_(<3 x s16>), [[UV11:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS2]](<12 x s16>)
+    ; GFX9: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV8]](<3 x s16>), 0
     ; GFX9: $vgpr0_vgpr1 = COPY [[INSERT4]](<4 x s16>)
     %0:_(p3) = COPY $vgpr0
     %1:_(<3 x s16>) = G_LOAD %0 :: (load 6, align 1, addrspace 3)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-private.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-private.mir
index 58f13f172d9ab..085cb619570d2 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-private.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-private.mir
@@ -5027,15 +5027,18 @@ body: |
     ; SI: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load 2 + 4, align 4, addrspace 5)
     ; SI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD1]](s32)
     ; SI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; SI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; SI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; SI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; SI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
+    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
     ; SI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT]], [[LOAD]](<2 x s16>), 0
-    ; SI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT1]](<4 x s16>), 0
-    ; SI: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
+    ; SI: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT1]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; SI: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; SI: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
     ; SI: [[INSERT3:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT2]], [[TRUNC]](s16), 32
-    ; SI: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT3]](<4 x s16>), 0
-    ; SI: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT2]](<3 x s16>), 0
+    ; SI: [[CONCAT_VECTORS2:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT3]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; SI: [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>), [[UV10:%[0-9]+]]:_(<3 x s16>), [[UV11:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS2]](<12 x s16>)
+    ; SI: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV8]](<3 x s16>), 0
     ; SI: $vgpr0_vgpr1 = COPY [[INSERT4]](<4 x s16>)
     ; CI-LABEL: name: test_load_private_v3s16_align8
     ; CI: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
@@ -5045,15 +5048,18 @@ body: |
     ; CI: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load 2 + 4, align 4, addrspace 5)
     ; CI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD1]](s32)
     ; CI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; CI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; CI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; CI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; CI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; CI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
+    ; CI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
     ; CI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT]], [[LOAD]](<2 x s16>), 0
-    ; CI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT1]](<4 x s16>), 0
-    ; CI: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
+    ; CI: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT1]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; CI: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; CI: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
     ; CI: [[INSERT3:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT2]], [[TRUNC]](s16), 32
-    ; CI: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT3]](<4 x s16>), 0
-    ; CI: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT2]](<3 x s16>), 0
+    ; CI: [[CONCAT_VECTORS2:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT3]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; CI: [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>), [[UV10:%[0-9]+]]:_(<3 x s16>), [[UV11:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS2]](<12 x s16>)
+    ; CI: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV8]](<3 x s16>), 0
     ; CI: $vgpr0_vgpr1 = COPY [[INSERT4]](<4 x s16>)
     ; VI-LABEL: name: test_load_private_v3s16_align8
     ; VI: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
@@ -5063,15 +5069,18 @@ body: |
     ; VI: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load 2 + 4, align 4, addrspace 5)
     ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD1]](s32)
     ; VI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; VI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; VI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; VI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
+    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
     ; VI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT]], [[LOAD]](<2 x s16>), 0
-    ; VI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT1]](<4 x s16>), 0
-    ; VI: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
+    ; VI: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT1]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; VI: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; VI: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
     ; VI: [[INSERT3:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT2]], [[TRUNC]](s16), 32
-    ; VI: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT3]](<4 x s16>), 0
-    ; VI: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT2]](<3 x s16>), 0
+    ; VI: [[CONCAT_VECTORS2:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT3]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; VI: [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>), [[UV10:%[0-9]+]]:_(<3 x s16>), [[UV11:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS2]](<12 x s16>)
+    ; VI: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV8]](<3 x s16>), 0
     ; VI: $vgpr0_vgpr1 = COPY [[INSERT4]](<4 x s16>)
     ; GFX9-LABEL: name: test_load_private_v3s16_align8
     ; GFX9: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
@@ -5081,15 +5090,18 @@ body: |
     ; GFX9: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load 2 + 4, align 4, addrspace 5)
     ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD1]](s32)
     ; GFX9: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; GFX9: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; GFX9: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
+    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
     ; GFX9: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT]], [[LOAD]](<2 x s16>), 0
-    ; GFX9: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT1]](<4 x s16>), 0
-    ; GFX9: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
+    ; GFX9: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT1]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX9: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; GFX9: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
     ; GFX9: [[INSERT3:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT2]], [[TRUNC]](s16), 32
-    ; GFX9: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT3]](<4 x s16>), 0
-    ; GFX9: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT2]](<3 x s16>), 0
+    ; GFX9: [[CONCAT_VECTORS2:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT3]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX9: [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>), [[UV10:%[0-9]+]]:_(<3 x s16>), [[UV11:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS2]](<12 x s16>)
+    ; GFX9: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV8]](<3 x s16>), 0
     ; GFX9: $vgpr0_vgpr1 = COPY [[INSERT4]](<4 x s16>)
     %0:_(p5) = COPY $vgpr0
     %1:_(<3 x s16>) = G_LOAD %0 :: (load 6, align 8, addrspace 5)
@@ -5124,15 +5136,18 @@ body: |
     ; SI: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load 2 + 4, addrspace 5)
     ; SI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD2]](s32)
     ; SI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; SI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; SI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; SI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; SI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
+    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
     ; SI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT]], [[BITCAST]](<2 x s16>), 0
-    ; SI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT1]](<4 x s16>), 0
-    ; SI: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
+    ; SI: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT1]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; SI: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; SI: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
     ; SI: [[INSERT3:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT2]], [[TRUNC]](s16), 32
-    ; SI: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT3]](<4 x s16>), 0
-    ; SI: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT2]](<3 x s16>), 0
+    ; SI: [[CONCAT_VECTORS2:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT3]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; SI: [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>), [[UV10:%[0-9]+]]:_(<3 x s16>), [[UV11:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS2]](<12 x s16>)
+    ; SI: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV8]](<3 x s16>), 0
     ; SI: $vgpr0_vgpr1 = COPY [[INSERT4]](<4 x s16>)
     ; CI-LABEL: name: test_load_private_v3s16_align2
     ; CI: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
@@ -5154,15 +5169,18 @@ body: |
     ; CI: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load 2 + 4, addrspace 5)
     ; CI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD2]](s32)
     ; CI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; CI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; CI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; CI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; CI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; CI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
+    ; CI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
     ; CI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT]], [[BITCAST]](<2 x s16>), 0
-    ; CI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT1]](<4 x s16>), 0
-    ; CI: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
+    ; CI: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT1]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; CI: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; CI: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
     ; CI: [[INSERT3:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT2]], [[TRUNC]](s16), 32
-    ; CI: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT3]](<4 x s16>), 0
-    ; CI: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT2]](<3 x s16>), 0
+    ; CI: [[CONCAT_VECTORS2:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT3]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; CI: [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>), [[UV10:%[0-9]+]]:_(<3 x s16>), [[UV11:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS2]](<12 x s16>)
+    ; CI: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV8]](<3 x s16>), 0
     ; CI: $vgpr0_vgpr1 = COPY [[INSERT4]](<4 x s16>)
     ; VI-LABEL: name: test_load_private_v3s16_align2
     ; VI: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
@@ -5184,15 +5202,18 @@ body: |
     ; VI: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load 2 + 4, addrspace 5)
     ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD2]](s32)
     ; VI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; VI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; VI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; VI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
+    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
     ; VI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT]], [[BITCAST]](<2 x s16>), 0
-    ; VI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT1]](<4 x s16>), 0
-    ; VI: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
+    ; VI: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT1]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; VI: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; VI: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
     ; VI: [[INSERT3:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT2]], [[TRUNC]](s16), 32
-    ; VI: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT3]](<4 x s16>), 0
-    ; VI: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT2]](<3 x s16>), 0
+    ; VI: [[CONCAT_VECTORS2:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT3]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; VI: [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>), [[UV10:%[0-9]+]]:_(<3 x s16>), [[UV11:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS2]](<12 x s16>)
+    ; VI: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV8]](<3 x s16>), 0
     ; VI: $vgpr0_vgpr1 = COPY [[INSERT4]](<4 x s16>)
     ; GFX9-LABEL: name: test_load_private_v3s16_align2
     ; GFX9: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
@@ -5208,15 +5229,18 @@ body: |
     ; GFX9: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load 2 + 4, addrspace 5)
     ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD2]](s32)
     ; GFX9: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; GFX9: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; GFX9: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
+    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
     ; GFX9: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT]], [[BUILD_VECTOR_TRUNC]](<2 x s16>), 0
-    ; GFX9: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT1]](<4 x s16>), 0
-    ; GFX9: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
+    ; GFX9: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT1]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX9: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; GFX9: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
     ; GFX9: [[INSERT3:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT2]], [[TRUNC]](s16), 32
-    ; GFX9: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT3]](<4 x s16>), 0
-    ; GFX9: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT2]](<3 x s16>), 0
+    ; GFX9: [[CONCAT_VECTORS2:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT3]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX9: [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>), [[UV10:%[0-9]+]]:_(<3 x s16>), [[UV11:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS2]](<12 x s16>)
+    ; GFX9: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV8]](<3 x s16>), 0
     ; GFX9: $vgpr0_vgpr1 = COPY [[INSERT4]](<4 x s16>)
     %0:_(p5) = COPY $vgpr0
     %1:_(<3 x s16>) = G_LOAD %0 :: (load 6, align 2, addrspace 5)
@@ -5280,15 +5304,18 @@ body: |
     ; SI: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[SHL3]](s32)
     ; SI: [[OR3:%[0-9]+]]:_(s16) = G_OR [[AND4]], [[TRUNC5]]
     ; SI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; SI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; SI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; SI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; SI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
+    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
     ; SI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT]], [[BITCAST]](<2 x s16>), 0
-    ; SI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT1]](<4 x s16>), 0
-    ; SI: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
+    ; SI: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT1]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; SI: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; SI: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
     ; SI: [[INSERT3:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT2]], [[OR3]](s16), 32
-    ; SI: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT3]](<4 x s16>), 0
-    ; SI: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT2]](<3 x s16>), 0
+    ; SI: [[CONCAT_VECTORS2:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT3]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; SI: [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>), [[UV10:%[0-9]+]]:_(<3 x s16>), [[UV11:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS2]](<12 x s16>)
+    ; SI: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV8]](<3 x s16>), 0
     ; SI: $vgpr0_vgpr1 = COPY [[INSERT4]](<4 x s16>)
     ; CI-LABEL: name: test_load_private_v3s16_align1
     ; CI: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
@@ -5339,15 +5366,18 @@ body: |
     ; CI: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[SHL3]](s32)
     ; CI: [[OR3:%[0-9]+]]:_(s16) = G_OR [[AND4]], [[TRUNC5]]
     ; CI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; CI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; CI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; CI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; CI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; CI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
+    ; CI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
     ; CI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT]], [[BITCAST]](<2 x s16>), 0
-    ; CI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT1]](<4 x s16>), 0
-    ; CI: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
+    ; CI: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT1]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; CI: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; CI: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
     ; CI: [[INSERT3:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT2]], [[OR3]](s16), 32
-    ; CI: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT3]](<4 x s16>), 0
-    ; CI: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT2]](<3 x s16>), 0
+    ; CI: [[CONCAT_VECTORS2:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT3]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; CI: [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>), [[UV10:%[0-9]+]]:_(<3 x s16>), [[UV11:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS2]](<12 x s16>)
+    ; CI: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV8]](<3 x s16>), 0
     ; CI: $vgpr0_vgpr1 = COPY [[INSERT4]](<4 x s16>)
     ; VI-LABEL: name: test_load_private_v3s16_align1
     ; VI: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
@@ -5392,15 +5422,18 @@ body: |
     ; VI: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[AND5]], [[C2]](s16)
     ; VI: [[OR3:%[0-9]+]]:_(s16) = G_OR [[AND4]], [[SHL3]]
     ; VI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; VI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; VI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; VI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
+    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
     ; VI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT]], [[BITCAST]](<2 x s16>), 0
-    ; VI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT1]](<4 x s16>), 0
-    ; VI: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
+    ; VI: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT1]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; VI: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; VI: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
     ; VI: [[INSERT3:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT2]], [[OR3]](s16), 32
-    ; VI: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT3]](<4 x s16>), 0
-    ; VI: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT2]](<3 x s16>), 0
+    ; VI: [[CONCAT_VECTORS2:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT3]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; VI: [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>), [[UV10:%[0-9]+]]:_(<3 x s16>), [[UV11:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS2]](<12 x s16>)
+    ; VI: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV8]](<3 x s16>), 0
     ; VI: $vgpr0_vgpr1 = COPY [[INSERT4]](<4 x s16>)
     ; GFX9-LABEL: name: test_load_private_v3s16_align1
     ; GFX9: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
@@ -5442,15 +5475,18 @@ body: |
     ; GFX9: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[AND5]], [[C2]](s16)
     ; GFX9: [[OR2:%[0-9]+]]:_(s16) = G_OR [[AND4]], [[SHL2]]
     ; GFX9: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), %2(<4 x s16>), %2(<4 x s16>)
+    ; GFX9: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; GFX9: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
+    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
     ; GFX9: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT]], [[BUILD_VECTOR_TRUNC]](<2 x s16>), 0
-    ; GFX9: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT1]](<4 x s16>), 0
-    ; GFX9: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
+    ; GFX9: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT1]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX9: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; GFX9: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
     ; GFX9: [[INSERT3:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT2]], [[OR2]](s16), 32
-    ; GFX9: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT3]](<4 x s16>), 0
-    ; GFX9: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT2]](<3 x s16>), 0
+    ; GFX9: [[CONCAT_VECTORS2:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT3]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX9: [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>), [[UV10:%[0-9]+]]:_(<3 x s16>), [[UV11:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS2]](<12 x s16>)
+    ; GFX9: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV8]](<3 x s16>), 0
     ; GFX9: $vgpr0_vgpr1 = COPY [[INSERT4]](<4 x s16>)
     %0:_(p5) = COPY $vgpr0
     %1:_(<3 x s16>) = G_LOAD %0 :: (load 6, align 1, addrspace 5)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-lshr.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-lshr.mir
index f5f7baeba73a4..4c69ff2edfa48 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-lshr.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-lshr.mir
@@ -726,14 +726,17 @@ body: |
     ; GFX9: [[LSHR:%[0-9]+]]:_(<2 x s16>) = G_LSHR [[EXTRACT2]], [[EXTRACT4]](<2 x s16>)
     ; GFX9: [[LSHR1:%[0-9]+]]:_(s16) = G_LSHR [[EXTRACT3]], [[EXTRACT5]](s16)
     ; GFX9: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9: [[EXTRACT6:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF1]](<4 x s16>), 0
-    ; GFX9: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT6]](<3 x s16>), 0
+    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF1]](<4 x s16>), [[DEF]](<4 x s16>), [[DEF]](<4 x s16>)
+    ; GFX9: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
+    ; GFX9: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV]](<3 x s16>), 0
     ; GFX9: [[INSERT5:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT4]], [[LSHR]](<2 x s16>), 0
-    ; GFX9: [[EXTRACT7:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT5]](<4 x s16>), 0
-    ; GFX9: [[INSERT6:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT7]](<3 x s16>), 0
+    ; GFX9: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT5]](<4 x s16>), [[DEF]](<4 x s16>), [[DEF]](<4 x s16>)
+    ; GFX9: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; GFX9: [[INSERT6:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV4]](<3 x s16>), 0
     ; GFX9: [[INSERT7:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT6]], [[LSHR1]](s16), 32
-    ; GFX9: [[EXTRACT8:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT7]](<4 x s16>), 0
-    ; GFX9: [[INSERT8:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT8]](<3 x s16>), 0
+    ; GFX9: [[CONCAT_VECTORS2:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT7]](<4 x s16>), [[DEF]](<4 x s16>), [[DEF]](<4 x s16>)
+    ; GFX9: [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>), [[UV10:%[0-9]+]]:_(<3 x s16>), [[UV11:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS2]](<12 x s16>)
+    ; GFX9: [[INSERT8:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV8]](<3 x s16>), 0
     ; GFX9: $vgpr0_vgpr1 = COPY [[INSERT8]](<4 x s16>)
     %0:_(<4 x s16>) = COPY $vgpr0_vgpr1
     %1:_(<4 x s16>) = COPY $vgpr2_vgpr3
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-or.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-or.mir
index b4a5c48eb2e68..a91139fc7fd50 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-or.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-or.mir
@@ -361,15 +361,18 @@ body: |
 
     ; CHECK-LABEL: name: test_or_v3s16
     ; CHECK: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; CHECK: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
     ; CHECK: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; CHECK: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF1]](<4 x s16>), 0
+    ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; CHECK: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; CHECK: [[DEF2:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; CHECK: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF2]], [[EXTRACT]](<3 x s16>), 0
-    ; CHECK: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF2]], [[EXTRACT1]](<3 x s16>), 0
+    ; CHECK: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF2]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; CHECK: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; CHECK: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
+    ; CHECK: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
     ; CHECK: [[OR:%[0-9]+]]:_(<4 x s16>) = G_OR [[INSERT]], [[INSERT1]]
-    ; CHECK: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[OR]](<4 x s16>), 0
-    ; CHECK: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF2]], [[EXTRACT2]](<3 x s16>), 0
+    ; CHECK: [[CONCAT_VECTORS2:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[OR]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; CHECK: [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>), [[UV10:%[0-9]+]]:_(<3 x s16>), [[UV11:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS2]](<12 x s16>)
+    ; CHECK: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV8]](<3 x s16>), 0
     ; CHECK: $vgpr0_vgpr1 = COPY [[INSERT2]](<4 x s16>)
     %0:_(<3 x s16>) = G_IMPLICIT_DEF
     %1:_(<3 x s16>) = G_IMPLICIT_DEF
@@ -403,27 +406,32 @@ body: |
 
     ; CHECK-LABEL: name: test_or_v5s16
     ; CHECK: [[DEF:%[0-9]+]]:_(<6 x s16>) = G_IMPLICIT_DEF
-    ; CHECK: [[EXTRACT:%[0-9]+]]:_(<5 x s16>) = G_EXTRACT [[DEF]](<6 x s16>), 0
     ; CHECK: [[DEF1:%[0-9]+]]:_(<6 x s16>) = G_IMPLICIT_DEF
-    ; CHECK: [[EXTRACT1:%[0-9]+]]:_(<5 x s16>) = G_EXTRACT [[DEF1]](<6 x s16>), 0
+    ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<30 x s16>) = G_CONCAT_VECTORS [[DEF]](<6 x s16>), [[DEF1]](<6 x s16>), [[DEF1]](<6 x s16>), [[DEF1]](<6 x s16>), [[DEF1]](<6 x s16>)
+    ; CHECK: [[UV:%[0-9]+]]:_(<5 x s16>), [[UV1:%[0-9]+]]:_(<5 x s16>), [[UV2:%[0-9]+]]:_(<5 x s16>), [[UV3:%[0-9]+]]:_(<5 x s16>), [[UV4:%[0-9]+]]:_(<5 x s16>), [[UV5:%[0-9]+]]:_(<5 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<30 x s16>)
     ; CHECK: [[DEF2:%[0-9]+]]:_(<6 x s16>) = G_IMPLICIT_DEF
-    ; CHECK: [[INSERT:%[0-9]+]]:_(<6 x s16>) = G_INSERT [[DEF2]], [[EXTRACT]](<5 x s16>), 0
-    ; CHECK: [[INSERT1:%[0-9]+]]:_(<6 x s16>) = G_INSERT [[DEF2]], [[EXTRACT1]](<5 x s16>), 0
-    ; CHECK: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[INSERT]](<6 x s16>)
-    ; CHECK: [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<6 x s16>)
+    ; CHECK: [[CONCAT_VECTORS1:%[0-9]+]]:_(<30 x s16>) = G_CONCAT_VECTORS [[DEF2]](<6 x s16>), [[DEF1]](<6 x s16>), [[DEF1]](<6 x s16>), [[DEF1]](<6 x s16>), [[DEF1]](<6 x s16>)
+    ; CHECK: [[UV6:%[0-9]+]]:_(<5 x s16>), [[UV7:%[0-9]+]]:_(<5 x s16>), [[UV8:%[0-9]+]]:_(<5 x s16>), [[UV9:%[0-9]+]]:_(<5 x s16>), [[UV10:%[0-9]+]]:_(<5 x s16>), [[UV11:%[0-9]+]]:_(<5 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<30 x s16>)
+    ; CHECK: [[INSERT:%[0-9]+]]:_(<6 x s16>) = G_INSERT [[DEF1]], [[UV]](<5 x s16>), 0
+    ; CHECK: [[INSERT1:%[0-9]+]]:_(<6 x s16>) = G_INSERT [[DEF1]], [[UV6]](<5 x s16>), 0
+    ; CHECK: [[UV12:%[0-9]+]]:_(<3 x s16>), [[UV13:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[INSERT]](<6 x s16>)
+    ; CHECK: [[UV14:%[0-9]+]]:_(<3 x s16>), [[UV15:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<6 x s16>)
     ; CHECK: [[DEF3:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; CHECK: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF3]], [[UV]](<3 x s16>), 0
-    ; CHECK: [[INSERT3:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF3]], [[UV2]](<3 x s16>), 0
+    ; CHECK: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF3]], [[UV12]](<3 x s16>), 0
+    ; CHECK: [[INSERT3:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF3]], [[UV14]](<3 x s16>), 0
     ; CHECK: [[OR:%[0-9]+]]:_(<4 x s16>) = G_OR [[INSERT2]], [[INSERT3]]
-    ; CHECK: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[OR]](<4 x s16>), 0
-    ; CHECK: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF3]], [[UV1]](<3 x s16>), 0
-    ; CHECK: [[INSERT5:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF3]], [[UV3]](<3 x s16>), 0
+    ; CHECK: [[CONCAT_VECTORS2:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[OR]](<4 x s16>), [[DEF3]](<4 x s16>), [[DEF3]](<4 x s16>)
+    ; CHECK: [[UV16:%[0-9]+]]:_(<3 x s16>), [[UV17:%[0-9]+]]:_(<3 x s16>), [[UV18:%[0-9]+]]:_(<3 x s16>), [[UV19:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS2]](<12 x s16>)
+    ; CHECK: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF3]], [[UV13]](<3 x s16>), 0
+    ; CHECK: [[INSERT5:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF3]], [[UV15]](<3 x s16>), 0
     ; CHECK: [[OR1:%[0-9]+]]:_(<4 x s16>) = G_OR [[INSERT4]], [[INSERT5]]
-    ; CHECK: [[EXTRACT3:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[OR1]](<4 x s16>), 0
-    ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[EXTRACT2]](<3 x s16>), [[EXTRACT3]](<3 x s16>)
-    ; CHECK: [[EXTRACT4:%[0-9]+]]:_(<5 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<6 x s16>), 0
+    ; CHECK: [[CONCAT_VECTORS3:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[OR1]](<4 x s16>), [[DEF3]](<4 x s16>), [[DEF3]](<4 x s16>)
+    ; CHECK: [[UV20:%[0-9]+]]:_(<3 x s16>), [[UV21:%[0-9]+]]:_(<3 x s16>), [[UV22:%[0-9]+]]:_(<3 x s16>), [[UV23:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS3]](<12 x s16>)
+    ; CHECK: [[CONCAT_VECTORS4:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[UV16]](<3 x s16>), [[UV20]](<3 x s16>)
+    ; CHECK: [[CONCAT_VECTORS5:%[0-9]+]]:_(<30 x s16>) = G_CONCAT_VECTORS [[CONCAT_VECTORS4]](<6 x s16>), [[DEF1]](<6 x s16>), [[DEF1]](<6 x s16>), [[DEF1]](<6 x s16>), [[DEF1]](<6 x s16>)
+    ; CHECK: [[UV24:%[0-9]+]]:_(<5 x s16>), [[UV25:%[0-9]+]]:_(<5 x s16>), [[UV26:%[0-9]+]]:_(<5 x s16>), [[UV27:%[0-9]+]]:_(<5 x s16>), [[UV28:%[0-9]+]]:_(<5 x s16>), [[UV29:%[0-9]+]]:_(<5 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS5]](<30 x s16>)
     ; CHECK: [[DEF4:%[0-9]+]]:_(<8 x s16>) = G_IMPLICIT_DEF
-    ; CHECK: [[INSERT6:%[0-9]+]]:_(<8 x s16>) = G_INSERT [[DEF4]], [[EXTRACT4]](<5 x s16>), 0
+    ; CHECK: [[INSERT6:%[0-9]+]]:_(<8 x s16>) = G_INSERT [[DEF4]], [[UV24]](<5 x s16>), 0
     ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INSERT6]](<8 x s16>)
     %0:_(<5 x s16>) = G_IMPLICIT_DEF
     %1:_(<5 x s16>) = G_IMPLICIT_DEF
@@ -463,8 +471,9 @@ body: |
     ; CHECK: [[OR3:%[0-9]+]]:_(s32) = G_OR [[ANYEXT6]], [[ANYEXT7]]
     ; CHECK: [[TRUNC3:%[0-9]+]]:_(s8) = G_TRUNC [[OR3]](s32)
     ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s8>) = G_BUILD_VECTOR [[TRUNC]](s8), [[TRUNC1]](s8), [[TRUNC2]](s8), [[TRUNC3]](s8)
-    ; CHECK: [[EXTRACT:%[0-9]+]]:_(<3 x s8>) = G_EXTRACT [[BUILD_VECTOR]](<4 x s8>), 0
-    ; CHECK: [[ANYEXT8:%[0-9]+]]:_(<3 x s32>) = G_ANYEXT [[EXTRACT]](<3 x s8>)
+    ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s8>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<4 x s8>), [[DEF2]](<4 x s8>), [[DEF2]](<4 x s8>)
+    ; CHECK: [[UV8:%[0-9]+]]:_(<3 x s8>), [[UV9:%[0-9]+]]:_(<3 x s8>), [[UV10:%[0-9]+]]:_(<3 x s8>), [[UV11:%[0-9]+]]:_(<3 x s8>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s8>)
+    ; CHECK: [[ANYEXT8:%[0-9]+]]:_(<3 x s32>) = G_ANYEXT [[UV8]](<3 x s8>)
     ; CHECK: $vgpr0_vgpr1_vgpr2 = COPY [[ANYEXT8]](<3 x s32>)
     %0:_(<3 x s8>) = G_IMPLICIT_DEF
     %1:_(<3 x s8>) = G_IMPLICIT_DEF
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-phi.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-phi.mir
index 9a91d908bb7b0..997dc3a521201 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-phi.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-phi.mir
@@ -174,9 +174,10 @@ body: |
   ; CHECK:   G_BR %bb.2
   ; CHECK: bb.2:
   ; CHECK:   [[PHI:%[0-9]+]]:_(<4 x s16>) = G_PHI [[INSERT]](<4 x s16>), %bb.0, [[INSERT3]](<4 x s16>), %bb.1
-  ; CHECK:   [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[PHI]](<4 x s16>), 0
+  ; CHECK:   [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[PHI]](<4 x s16>), %7(<4 x s16>), %7(<4 x s16>)
+  ; CHECK:   [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
   ; CHECK:   [[DEF2:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-  ; CHECK:   [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF2]], [[EXTRACT2]](<3 x s16>), 0
+  ; CHECK:   [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF2]], [[UV4]](<3 x s16>), 0
   ; CHECK:   $vgpr0_vgpr1 = COPY [[INSERT4]](<4 x s16>)
   ; CHECK:   S_SETPC_B64 undef $sgpr30_sgpr31
   bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-saddsat.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-saddsat.mir
index 51b6e014a9376..d96c88b027e59 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-saddsat.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-saddsat.mir
@@ -528,9 +528,11 @@ body: |
     ; GFX6: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
     ; GFX6: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
     ; GFX6: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<4 x s16>), 0
-    ; GFX6: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
-    ; GFX6: [[CONCAT_VECTORS1:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[EXTRACT]](<3 x s16>), [[EXTRACT1]](<3 x s16>)
-    ; GFX6: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS1]](<6 x s16>)
+    ; GFX6: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; GFX6: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF1]](<4 x s16>), [[DEF]](<4 x s16>), [[DEF]](<4 x s16>)
+    ; GFX6: [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>), [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; GFX6: [[CONCAT_VECTORS2:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[EXTRACT]](<3 x s16>), [[UV6]](<3 x s16>)
+    ; GFX6: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS2]](<6 x s16>)
     ; GFX8-LABEL: name: saddsat_v3s16
     ; GFX8: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
     ; GFX8: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
@@ -590,9 +592,11 @@ body: |
     ; GFX8: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
     ; GFX8: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
     ; GFX8: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<4 x s16>), 0
-    ; GFX8: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
-    ; GFX8: [[CONCAT_VECTORS1:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[EXTRACT]](<3 x s16>), [[EXTRACT1]](<3 x s16>)
-    ; GFX8: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS1]](<6 x s16>)
+    ; GFX8: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; GFX8: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF1]](<4 x s16>), [[DEF]](<4 x s16>), [[DEF]](<4 x s16>)
+    ; GFX8: [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>), [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; GFX8: [[CONCAT_VECTORS2:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[EXTRACT]](<3 x s16>), [[UV6]](<3 x s16>)
+    ; GFX8: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS2]](<6 x s16>)
     ; GFX9-LABEL: name: saddsat_v3s16
     ; GFX9: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
     ; GFX9: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
@@ -626,9 +630,11 @@ body: |
     ; GFX9: [[SADDSAT1:%[0-9]+]]:_(<2 x s16>) = G_SADDSAT [[BUILD_VECTOR_TRUNC1]], [[BUILD_VECTOR_TRUNC3]]
     ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[SADDSAT]](<2 x s16>), [[SADDSAT1]](<2 x s16>), [[DEF2]](<2 x s16>)
     ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<6 x s16>), 0
-    ; GFX9: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
-    ; GFX9: [[CONCAT_VECTORS1:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[EXTRACT]](<3 x s16>), [[EXTRACT1]](<3 x s16>)
-    ; GFX9: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS1]](<6 x s16>)
+    ; GFX9: [[DEF3:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; GFX9: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF3]](<4 x s16>), [[DEF]](<4 x s16>), [[DEF]](<4 x s16>)
+    ; GFX9: [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>), [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; GFX9: [[CONCAT_VECTORS2:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[EXTRACT]](<3 x s16>), [[UV6]](<3 x s16>)
+    ; GFX9: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS2]](<6 x s16>)
     %0:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
     %1:_(<3 x s16>), %2:_(<3 x s16>) = G_UNMERGE_VALUES %0
     %3:_(<3 x s16>) = G_SADDSAT %1, %2
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-select.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-select.mir
index 88689ae03616f..6dae4e59373c6 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-select.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-select.mir
@@ -291,8 +291,9 @@ body: |
     ; CHECK: [[SELECT3:%[0-9]+]]:_(s16) = G_SELECT [[ICMP]](s1), [[ANYEXT6]], [[ANYEXT7]]
     ; CHECK: [[TRUNC5:%[0-9]+]]:_(s8) = G_TRUNC [[SELECT3]](s16)
     ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s8>) = G_BUILD_VECTOR [[TRUNC2]](s8), [[TRUNC3]](s8), [[TRUNC4]](s8), [[TRUNC5]](s8)
-    ; CHECK: [[EXTRACT:%[0-9]+]]:_(<3 x s8>) = G_EXTRACT [[BUILD_VECTOR]](<4 x s8>), 0
-    ; CHECK: [[ANYEXT8:%[0-9]+]]:_(<3 x s32>) = G_ANYEXT [[EXTRACT]](<3 x s8>)
+    ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s8>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<4 x s8>), [[DEF]](<4 x s8>), [[DEF]](<4 x s8>)
+    ; CHECK: [[UV8:%[0-9]+]]:_(<3 x s8>), [[UV9:%[0-9]+]]:_(<3 x s8>), [[UV10:%[0-9]+]]:_(<3 x s8>), [[UV11:%[0-9]+]]:_(<3 x s8>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s8>)
+    ; CHECK: [[ANYEXT8:%[0-9]+]]:_(<3 x s32>) = G_ANYEXT [[UV8]](<3 x s8>)
     ; CHECK: $vgpr0_vgpr1_vgpr2 = COPY [[ANYEXT8]](<3 x s32>)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = G_CONSTANT i32 0
@@ -399,8 +400,9 @@ body: |
     ; CHECK: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s16>), 0
     ; CHECK: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT1]](<3 x s16>), 0
     ; CHECK: [[SELECT:%[0-9]+]]:_(<4 x s16>) = G_SELECT [[ICMP]](s1), [[INSERT]], [[INSERT1]]
-    ; CHECK: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[SELECT]](<4 x s16>), 0
-    ; CHECK: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT2]](<3 x s16>), 0
+    ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[SELECT]](<4 x s16>), [[DEF]](<4 x s16>), [[DEF]](<4 x s16>)
+    ; CHECK: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
+    ; CHECK: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV]](<3 x s16>), 0
     ; CHECK: $vgpr0_vgpr1 = COPY [[INSERT2]](<4 x s16>)
     %0:_(s32) = COPY $vgpr0
     %1:_(<4 x s16>) = COPY $vgpr1_vgpr2
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-shl.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-shl.mir
index 10e32d7f87c90..2d8812d5e7e6e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-shl.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-shl.mir
@@ -710,14 +710,17 @@ body: |
     ; GFX9: [[SHL:%[0-9]+]]:_(<2 x s16>) = G_SHL [[EXTRACT2]], [[EXTRACT4]](<2 x s16>)
     ; GFX9: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[EXTRACT3]], [[EXTRACT5]](s16)
     ; GFX9: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9: [[EXTRACT6:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF1]](<4 x s16>), 0
-    ; GFX9: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT6]](<3 x s16>), 0
+    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF1]](<4 x s16>), [[DEF]](<4 x s16>), [[DEF]](<4 x s16>)
+    ; GFX9: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
+    ; GFX9: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV]](<3 x s16>), 0
     ; GFX9: [[INSERT5:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT4]], [[SHL]](<2 x s16>), 0
-    ; GFX9: [[EXTRACT7:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT5]](<4 x s16>), 0
-    ; GFX9: [[INSERT6:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT7]](<3 x s16>), 0
+    ; GFX9: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT5]](<4 x s16>), [[DEF]](<4 x s16>), [[DEF]](<4 x s16>)
+    ; GFX9: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; GFX9: [[INSERT6:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV4]](<3 x s16>), 0
     ; GFX9: [[INSERT7:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[INSERT6]], [[SHL1]](s16), 32
-    ; GFX9: [[EXTRACT8:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[INSERT7]](<4 x s16>), 0
-    ; GFX9: [[INSERT8:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT8]](<3 x s16>), 0
+    ; GFX9: [[CONCAT_VECTORS2:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[INSERT7]](<4 x s16>), [[DEF]](<4 x s16>), [[DEF]](<4 x s16>)
+    ; GFX9: [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>), [[UV10:%[0-9]+]]:_(<3 x s16>), [[UV11:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS2]](<12 x s16>)
+    ; GFX9: [[INSERT8:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV8]](<3 x s16>), 0
     ; GFX9: $vgpr0_vgpr1 = COPY [[INSERT8]](<4 x s16>)
     %0:_(<4 x s16>) = COPY $vgpr0_vgpr1
     %1:_(<4 x s16>) = COPY $vgpr2_vgpr3
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-smax.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-smax.mir
index a1087058ae5d6..0350c1cc53a01 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-smax.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-smax.mir
@@ -349,21 +349,24 @@ body: |
 
     ; SI-LABEL: name: test_smax_v3s16
     ; SI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; SI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
     ; SI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; SI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF1]](<4 x s16>), 0
-    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
-    ; SI: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
-    ; SI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; SI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; SI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
+    ; SI: [[DEF2:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; SI: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF2]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; SI: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
+    ; SI: [[UV8:%[0-9]+]]:_(<2 x s16>), [[UV9:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; SI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV8]](<2 x s16>)
     ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; SI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
-    ; SI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; SI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV9]](<2 x s16>)
     ; SI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
-    ; SI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
-    ; SI: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
-    ; SI: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
+    ; SI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
+    ; SI: [[UV10:%[0-9]+]]:_(<2 x s16>), [[UV11:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
+    ; SI: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV10]](<2 x s16>)
     ; SI: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
-    ; SI: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
+    ; SI: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV11]](<2 x s16>)
     ; SI: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
     ; SI: [[COPY:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
     ; SI: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY]], 16
@@ -394,31 +397,34 @@ body: |
     ; SI: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C]](s32)
     ; SI: [[OR1:%[0-9]+]]:_(s32) = G_OR [[AND2]], [[SHL1]]
     ; SI: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
-    ; SI: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
-    ; SI: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<4 x s16>), 0
-    ; SI: S_NOP 0, implicit [[EXTRACT2]](<3 x s16>)
+    ; SI: [[CONCAT_VECTORS2:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
+    ; SI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS2]](<4 x s16>), 0
+    ; SI: S_NOP 0, implicit [[EXTRACT]](<3 x s16>)
     ; VI-LABEL: name: test_smax_v3s16
     ; VI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
     ; VI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; VI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF1]](<4 x s16>), 0
-    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
-    ; VI: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
-    ; VI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; VI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; VI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
+    ; VI: [[DEF2:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; VI: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF2]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; VI: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
+    ; VI: [[UV8:%[0-9]+]]:_(<2 x s16>), [[UV9:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; VI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV8]](<2 x s16>)
     ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
     ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; VI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
     ; VI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-    ; VI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; VI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV9]](<2 x s16>)
     ; VI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
     ; VI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
-    ; VI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
-    ; VI: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
-    ; VI: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
+    ; VI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
+    ; VI: [[UV10:%[0-9]+]]:_(<2 x s16>), [[UV11:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
+    ; VI: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV10]](<2 x s16>)
     ; VI: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
     ; VI: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
     ; VI: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
-    ; VI: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
+    ; VI: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV11]](<2 x s16>)
     ; VI: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
     ; VI: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
     ; VI: [[SMAX:%[0-9]+]]:_(s16) = G_SMAX [[TRUNC]], [[TRUNC3]]
@@ -434,24 +440,27 @@ body: |
     ; VI: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C]](s32)
     ; VI: [[OR1:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[SHL1]]
     ; VI: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
-    ; VI: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
-    ; VI: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<4 x s16>), 0
-    ; VI: S_NOP 0, implicit [[EXTRACT2]](<3 x s16>)
+    ; VI: [[CONCAT_VECTORS2:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
+    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS2]](<4 x s16>), 0
+    ; VI: S_NOP 0, implicit [[EXTRACT]](<3 x s16>)
     ; GFX9-LABEL: name: test_smax_v3s16
     ; GFX9: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
     ; GFX9: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF1]](<4 x s16>), 0
+    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX9: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; GFX9: [[DEF2:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF2]], [[EXTRACT]](<3 x s16>), 0
-    ; GFX9: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF2]], [[EXTRACT1]](<3 x s16>), 0
-    ; GFX9: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
-    ; GFX9: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
-    ; GFX9: [[SMAX:%[0-9]+]]:_(<2 x s16>) = G_SMAX [[UV]], [[UV2]]
-    ; GFX9: [[SMAX1:%[0-9]+]]:_(<2 x s16>) = G_SMAX [[UV1]], [[UV3]]
-    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[SMAX]](<2 x s16>), [[SMAX1]](<2 x s16>)
-    ; GFX9: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<4 x s16>), 0
-    ; GFX9: S_NOP 0, implicit [[EXTRACT2]](<3 x s16>)
+    ; GFX9: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF2]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX9: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
+    ; GFX9: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
+    ; GFX9: [[UV8:%[0-9]+]]:_(<2 x s16>), [[UV9:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; GFX9: [[UV10:%[0-9]+]]:_(<2 x s16>), [[UV11:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
+    ; GFX9: [[SMAX:%[0-9]+]]:_(<2 x s16>) = G_SMAX [[UV8]], [[UV10]]
+    ; GFX9: [[SMAX1:%[0-9]+]]:_(<2 x s16>) = G_SMAX [[UV9]], [[UV11]]
+    ; GFX9: [[CONCAT_VECTORS2:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[SMAX]](<2 x s16>), [[SMAX1]](<2 x s16>)
+    ; GFX9: [[CONCAT_VECTORS3:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[CONCAT_VECTORS2]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX9: [[UV12:%[0-9]+]]:_(<3 x s16>), [[UV13:%[0-9]+]]:_(<3 x s16>), [[UV14:%[0-9]+]]:_(<3 x s16>), [[UV15:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS3]](<12 x s16>)
+    ; GFX9: S_NOP 0, implicit [[UV12]](<3 x s16>)
     %0:_(<3 x s16>) = G_IMPLICIT_DEF
     %1:_(<3 x s16>) = G_IMPLICIT_DEF
     %2:_(<3 x s16>) = G_SMAX %0, %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-smin.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-smin.mir
index 483681f18de84..0e2dbbc948bdb 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-smin.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-smin.mir
@@ -349,21 +349,24 @@ body: |
 
     ; SI-LABEL: name: test_smin_v3s16
     ; SI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; SI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
     ; SI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; SI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF1]](<4 x s16>), 0
-    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
-    ; SI: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
-    ; SI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; SI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; SI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
+    ; SI: [[DEF2:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; SI: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF2]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; SI: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
+    ; SI: [[UV8:%[0-9]+]]:_(<2 x s16>), [[UV9:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; SI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV8]](<2 x s16>)
     ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; SI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
-    ; SI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; SI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV9]](<2 x s16>)
     ; SI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
-    ; SI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
-    ; SI: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
-    ; SI: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
+    ; SI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
+    ; SI: [[UV10:%[0-9]+]]:_(<2 x s16>), [[UV11:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
+    ; SI: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV10]](<2 x s16>)
     ; SI: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
-    ; SI: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
+    ; SI: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV11]](<2 x s16>)
     ; SI: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
     ; SI: [[COPY:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
     ; SI: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY]], 16
@@ -394,31 +397,34 @@ body: |
     ; SI: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C]](s32)
     ; SI: [[OR1:%[0-9]+]]:_(s32) = G_OR [[AND2]], [[SHL1]]
     ; SI: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
-    ; SI: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
-    ; SI: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<4 x s16>), 0
-    ; SI: S_NOP 0, implicit [[EXTRACT2]](<3 x s16>)
+    ; SI: [[CONCAT_VECTORS2:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
+    ; SI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS2]](<4 x s16>), 0
+    ; SI: S_NOP 0, implicit [[EXTRACT]](<3 x s16>)
     ; VI-LABEL: name: test_smin_v3s16
     ; VI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
     ; VI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; VI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF1]](<4 x s16>), 0
-    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
-    ; VI: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
-    ; VI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; VI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; VI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
+    ; VI: [[DEF2:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; VI: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF2]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; VI: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
+    ; VI: [[UV8:%[0-9]+]]:_(<2 x s16>), [[UV9:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; VI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV8]](<2 x s16>)
     ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
     ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; VI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
     ; VI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-    ; VI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; VI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV9]](<2 x s16>)
     ; VI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
     ; VI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
-    ; VI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
-    ; VI: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
-    ; VI: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
+    ; VI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
+    ; VI: [[UV10:%[0-9]+]]:_(<2 x s16>), [[UV11:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
+    ; VI: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV10]](<2 x s16>)
     ; VI: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
     ; VI: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
     ; VI: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
-    ; VI: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
+    ; VI: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV11]](<2 x s16>)
     ; VI: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
     ; VI: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
     ; VI: [[SMIN:%[0-9]+]]:_(s16) = G_SMIN [[TRUNC]], [[TRUNC3]]
@@ -434,24 +440,27 @@ body: |
     ; VI: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C]](s32)
     ; VI: [[OR1:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[SHL1]]
     ; VI: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
-    ; VI: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
-    ; VI: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<4 x s16>), 0
-    ; VI: S_NOP 0, implicit [[EXTRACT2]](<3 x s16>)
+    ; VI: [[CONCAT_VECTORS2:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
+    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS2]](<4 x s16>), 0
+    ; VI: S_NOP 0, implicit [[EXTRACT]](<3 x s16>)
     ; GFX9-LABEL: name: test_smin_v3s16
     ; GFX9: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
     ; GFX9: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF1]](<4 x s16>), 0
+    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX9: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; GFX9: [[DEF2:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF2]], [[EXTRACT]](<3 x s16>), 0
-    ; GFX9: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF2]], [[EXTRACT1]](<3 x s16>), 0
-    ; GFX9: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
-    ; GFX9: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
-    ; GFX9: [[SMIN:%[0-9]+]]:_(<2 x s16>) = G_SMIN [[UV]], [[UV2]]
-    ; GFX9: [[SMIN1:%[0-9]+]]:_(<2 x s16>) = G_SMIN [[UV1]], [[UV3]]
-    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[SMIN]](<2 x s16>), [[SMIN1]](<2 x s16>)
-    ; GFX9: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<4 x s16>), 0
-    ; GFX9: S_NOP 0, implicit [[EXTRACT2]](<3 x s16>)
+    ; GFX9: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF2]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX9: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
+    ; GFX9: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
+    ; GFX9: [[UV8:%[0-9]+]]:_(<2 x s16>), [[UV9:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; GFX9: [[UV10:%[0-9]+]]:_(<2 x s16>), [[UV11:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
+    ; GFX9: [[SMIN:%[0-9]+]]:_(<2 x s16>) = G_SMIN [[UV8]], [[UV10]]
+    ; GFX9: [[SMIN1:%[0-9]+]]:_(<2 x s16>) = G_SMIN [[UV9]], [[UV11]]
+    ; GFX9: [[CONCAT_VECTORS2:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[SMIN]](<2 x s16>), [[SMIN1]](<2 x s16>)
+    ; GFX9: [[CONCAT_VECTORS3:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[CONCAT_VECTORS2]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX9: [[UV12:%[0-9]+]]:_(<3 x s16>), [[UV13:%[0-9]+]]:_(<3 x s16>), [[UV14:%[0-9]+]]:_(<3 x s16>), [[UV15:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS3]](<12 x s16>)
+    ; GFX9: S_NOP 0, implicit [[UV12]](<3 x s16>)
     %0:_(<3 x s16>) = G_IMPLICIT_DEF
     %1:_(<3 x s16>) = G_IMPLICIT_DEF
     %2:_(<3 x s16>) = G_SMIN %0, %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ssubsat.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ssubsat.mir
index f38da863cba90..fc1f5ab266ec0 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ssubsat.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ssubsat.mir
@@ -528,9 +528,11 @@ body: |
     ; GFX6: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
     ; GFX6: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
     ; GFX6: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<4 x s16>), 0
-    ; GFX6: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
-    ; GFX6: [[CONCAT_VECTORS1:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[EXTRACT]](<3 x s16>), [[EXTRACT1]](<3 x s16>)
-    ; GFX6: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS1]](<6 x s16>)
+    ; GFX6: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; GFX6: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF1]](<4 x s16>), [[DEF]](<4 x s16>), [[DEF]](<4 x s16>)
+    ; GFX6: [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>), [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; GFX6: [[CONCAT_VECTORS2:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[EXTRACT]](<3 x s16>), [[UV6]](<3 x s16>)
+    ; GFX6: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS2]](<6 x s16>)
     ; GFX8-LABEL: name: ssubsat_v3s16
     ; GFX8: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
     ; GFX8: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
@@ -590,9 +592,11 @@ body: |
     ; GFX8: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
     ; GFX8: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
     ; GFX8: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<4 x s16>), 0
-    ; GFX8: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
-    ; GFX8: [[CONCAT_VECTORS1:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[EXTRACT]](<3 x s16>), [[EXTRACT1]](<3 x s16>)
-    ; GFX8: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS1]](<6 x s16>)
+    ; GFX8: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; GFX8: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF1]](<4 x s16>), [[DEF]](<4 x s16>), [[DEF]](<4 x s16>)
+    ; GFX8: [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>), [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; GFX8: [[CONCAT_VECTORS2:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[EXTRACT]](<3 x s16>), [[UV6]](<3 x s16>)
+    ; GFX8: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS2]](<6 x s16>)
     ; GFX9-LABEL: name: ssubsat_v3s16
     ; GFX9: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
     ; GFX9: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
@@ -626,9 +630,11 @@ body: |
     ; GFX9: [[SSUBSAT1:%[0-9]+]]:_(<2 x s16>) = G_SSUBSAT [[BUILD_VECTOR_TRUNC1]], [[BUILD_VECTOR_TRUNC3]]
     ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[SSUBSAT]](<2 x s16>), [[SSUBSAT1]](<2 x s16>), [[DEF2]](<2 x s16>)
     ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<6 x s16>), 0
-    ; GFX9: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
-    ; GFX9: [[CONCAT_VECTORS1:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[EXTRACT]](<3 x s16>), [[EXTRACT1]](<3 x s16>)
-    ; GFX9: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS1]](<6 x s16>)
+    ; GFX9: [[DEF3:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; GFX9: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF3]](<4 x s16>), [[DEF]](<4 x s16>), [[DEF]](<4 x s16>)
+    ; GFX9: [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>), [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; GFX9: [[CONCAT_VECTORS2:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[EXTRACT]](<3 x s16>), [[UV6]](<3 x s16>)
+    ; GFX9: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS2]](<6 x s16>)
     %0:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
     %1:_(<3 x s16>), %2:_(<3 x s16>) = G_UNMERGE_VALUES %0
     %3:_(<3 x s16>) = G_SSUBSAT %1, %2
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-uaddsat.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-uaddsat.mir
index 9d51870b4fed2..e6a7193a3b0f5 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-uaddsat.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-uaddsat.mir
@@ -419,9 +419,11 @@ body: |
     ; GFX6: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
     ; GFX6: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
     ; GFX6: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<4 x s16>), 0
-    ; GFX6: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
-    ; GFX6: [[CONCAT_VECTORS1:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[EXTRACT]](<3 x s16>), [[EXTRACT1]](<3 x s16>)
-    ; GFX6: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS1]](<6 x s16>)
+    ; GFX6: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; GFX6: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF1]](<4 x s16>), [[DEF]](<4 x s16>), [[DEF]](<4 x s16>)
+    ; GFX6: [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>), [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; GFX6: [[CONCAT_VECTORS2:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[EXTRACT]](<3 x s16>), [[UV6]](<3 x s16>)
+    ; GFX6: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS2]](<6 x s16>)
     ; GFX8-LABEL: name: uaddsat_v3s16
     ; GFX8: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
     ; GFX8: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
@@ -460,9 +462,11 @@ body: |
     ; GFX8: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
     ; GFX8: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
     ; GFX8: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<4 x s16>), 0
-    ; GFX8: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
-    ; GFX8: [[CONCAT_VECTORS1:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[EXTRACT]](<3 x s16>), [[EXTRACT1]](<3 x s16>)
-    ; GFX8: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS1]](<6 x s16>)
+    ; GFX8: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; GFX8: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF1]](<4 x s16>), [[DEF]](<4 x s16>), [[DEF]](<4 x s16>)
+    ; GFX8: [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>), [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; GFX8: [[CONCAT_VECTORS2:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[EXTRACT]](<3 x s16>), [[UV6]](<3 x s16>)
+    ; GFX8: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS2]](<6 x s16>)
     ; GFX9-LABEL: name: uaddsat_v3s16
     ; GFX9: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
     ; GFX9: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
@@ -496,9 +500,11 @@ body: |
     ; GFX9: [[UADDSAT1:%[0-9]+]]:_(<2 x s16>) = G_UADDSAT [[BUILD_VECTOR_TRUNC1]], [[BUILD_VECTOR_TRUNC3]]
     ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[UADDSAT]](<2 x s16>), [[UADDSAT1]](<2 x s16>), [[DEF2]](<2 x s16>)
     ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<6 x s16>), 0
-    ; GFX9: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
-    ; GFX9: [[CONCAT_VECTORS1:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[EXTRACT]](<3 x s16>), [[EXTRACT1]](<3 x s16>)
-    ; GFX9: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS1]](<6 x s16>)
+    ; GFX9: [[DEF3:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; GFX9: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF3]](<4 x s16>), [[DEF]](<4 x s16>), [[DEF]](<4 x s16>)
+    ; GFX9: [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>), [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; GFX9: [[CONCAT_VECTORS2:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[EXTRACT]](<3 x s16>), [[UV6]](<3 x s16>)
+    ; GFX9: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS2]](<6 x s16>)
     %0:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
     %1:_(<3 x s16>), %2:_(<3 x s16>) = G_UNMERGE_VALUES %0
     %3:_(<3 x s16>) = G_UADDSAT %1, %2
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-umax.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-umax.mir
index 7a1bf1b4467b4..da102794a474c 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-umax.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-umax.mir
@@ -351,21 +351,24 @@ body: |
 
     ; SI-LABEL: name: test_umax_v3s16
     ; SI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; SI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
     ; SI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; SI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF1]](<4 x s16>), 0
-    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
-    ; SI: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
-    ; SI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; SI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; SI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
+    ; SI: [[DEF2:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; SI: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF2]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; SI: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
+    ; SI: [[UV8:%[0-9]+]]:_(<2 x s16>), [[UV9:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; SI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV8]](<2 x s16>)
     ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; SI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
-    ; SI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; SI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV9]](<2 x s16>)
     ; SI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
-    ; SI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
-    ; SI: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
-    ; SI: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
+    ; SI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
+    ; SI: [[UV10:%[0-9]+]]:_(<2 x s16>), [[UV11:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
+    ; SI: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV10]](<2 x s16>)
     ; SI: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
-    ; SI: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
+    ; SI: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV11]](<2 x s16>)
     ; SI: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
     ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
     ; SI: [[COPY:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
@@ -396,31 +399,34 @@ body: |
     ; SI: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C]](s32)
     ; SI: [[OR1:%[0-9]+]]:_(s32) = G_OR [[AND8]], [[SHL1]]
     ; SI: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
-    ; SI: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
-    ; SI: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<4 x s16>), 0
-    ; SI: S_NOP 0, implicit [[EXTRACT2]](<3 x s16>)
+    ; SI: [[CONCAT_VECTORS2:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
+    ; SI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS2]](<4 x s16>), 0
+    ; SI: S_NOP 0, implicit [[EXTRACT]](<3 x s16>)
     ; VI-LABEL: name: test_umax_v3s16
     ; VI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
     ; VI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; VI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF1]](<4 x s16>), 0
-    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
-    ; VI: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
-    ; VI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; VI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; VI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
+    ; VI: [[DEF2:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; VI: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF2]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; VI: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
+    ; VI: [[UV8:%[0-9]+]]:_(<2 x s16>), [[UV9:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; VI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV8]](<2 x s16>)
     ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
     ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; VI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
     ; VI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-    ; VI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; VI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV9]](<2 x s16>)
     ; VI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
     ; VI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
-    ; VI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
-    ; VI: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
-    ; VI: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
+    ; VI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
+    ; VI: [[UV10:%[0-9]+]]:_(<2 x s16>), [[UV11:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
+    ; VI: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV10]](<2 x s16>)
     ; VI: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
     ; VI: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
     ; VI: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
-    ; VI: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
+    ; VI: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV11]](<2 x s16>)
     ; VI: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
     ; VI: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
     ; VI: [[UMAX:%[0-9]+]]:_(s16) = G_UMAX [[TRUNC]], [[TRUNC3]]
@@ -436,24 +442,27 @@ body: |
     ; VI: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C]](s32)
     ; VI: [[OR1:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[SHL1]]
     ; VI: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
-    ; VI: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
-    ; VI: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<4 x s16>), 0
-    ; VI: S_NOP 0, implicit [[EXTRACT2]](<3 x s16>)
+    ; VI: [[CONCAT_VECTORS2:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
+    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS2]](<4 x s16>), 0
+    ; VI: S_NOP 0, implicit [[EXTRACT]](<3 x s16>)
     ; GFX9-LABEL: name: test_umax_v3s16
     ; GFX9: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
     ; GFX9: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF1]](<4 x s16>), 0
+    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX9: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; GFX9: [[DEF2:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF2]], [[EXTRACT]](<3 x s16>), 0
-    ; GFX9: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF2]], [[EXTRACT1]](<3 x s16>), 0
-    ; GFX9: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
-    ; GFX9: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
-    ; GFX9: [[UMAX:%[0-9]+]]:_(<2 x s16>) = G_UMAX [[UV]], [[UV2]]
-    ; GFX9: [[UMAX1:%[0-9]+]]:_(<2 x s16>) = G_UMAX [[UV1]], [[UV3]]
-    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[UMAX]](<2 x s16>), [[UMAX1]](<2 x s16>)
-    ; GFX9: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<4 x s16>), 0
-    ; GFX9: S_NOP 0, implicit [[EXTRACT2]](<3 x s16>)
+    ; GFX9: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF2]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX9: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
+    ; GFX9: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
+    ; GFX9: [[UV8:%[0-9]+]]:_(<2 x s16>), [[UV9:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; GFX9: [[UV10:%[0-9]+]]:_(<2 x s16>), [[UV11:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
+    ; GFX9: [[UMAX:%[0-9]+]]:_(<2 x s16>) = G_UMAX [[UV8]], [[UV10]]
+    ; GFX9: [[UMAX1:%[0-9]+]]:_(<2 x s16>) = G_UMAX [[UV9]], [[UV11]]
+    ; GFX9: [[CONCAT_VECTORS2:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[UMAX]](<2 x s16>), [[UMAX1]](<2 x s16>)
+    ; GFX9: [[CONCAT_VECTORS3:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[CONCAT_VECTORS2]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX9: [[UV12:%[0-9]+]]:_(<3 x s16>), [[UV13:%[0-9]+]]:_(<3 x s16>), [[UV14:%[0-9]+]]:_(<3 x s16>), [[UV15:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS3]](<12 x s16>)
+    ; GFX9: S_NOP 0, implicit [[UV12]](<3 x s16>)
     %0:_(<3 x s16>) = G_IMPLICIT_DEF
     %1:_(<3 x s16>) = G_IMPLICIT_DEF
     %2:_(<3 x s16>) = G_UMAX %0, %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-umin.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-umin.mir
index de480c00783b5..d5cddf28000c4 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-umin.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-umin.mir
@@ -351,21 +351,24 @@ body: |
 
     ; SI-LABEL: name: test_umin_v3s16
     ; SI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; SI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
     ; SI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; SI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF1]](<4 x s16>), 0
-    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
-    ; SI: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
-    ; SI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; SI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; SI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
+    ; SI: [[DEF2:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; SI: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF2]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; SI: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
+    ; SI: [[UV8:%[0-9]+]]:_(<2 x s16>), [[UV9:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; SI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV8]](<2 x s16>)
     ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; SI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
-    ; SI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; SI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV9]](<2 x s16>)
     ; SI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
-    ; SI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
-    ; SI: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
-    ; SI: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
+    ; SI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
+    ; SI: [[UV10:%[0-9]+]]:_(<2 x s16>), [[UV11:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
+    ; SI: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV10]](<2 x s16>)
     ; SI: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
-    ; SI: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
+    ; SI: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV11]](<2 x s16>)
     ; SI: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
     ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
     ; SI: [[COPY:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
@@ -396,31 +399,34 @@ body: |
     ; SI: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C]](s32)
     ; SI: [[OR1:%[0-9]+]]:_(s32) = G_OR [[AND8]], [[SHL1]]
     ; SI: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
-    ; SI: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
-    ; SI: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<4 x s16>), 0
-    ; SI: S_NOP 0, implicit [[EXTRACT2]](<3 x s16>)
+    ; SI: [[CONCAT_VECTORS2:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
+    ; SI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS2]](<4 x s16>), 0
+    ; SI: S_NOP 0, implicit [[EXTRACT]](<3 x s16>)
     ; VI-LABEL: name: test_umin_v3s16
     ; VI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
     ; VI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; VI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF1]](<4 x s16>), 0
-    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
-    ; VI: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
-    ; VI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; VI: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; VI: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
+    ; VI: [[DEF2:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; VI: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF2]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; VI: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
+    ; VI: [[UV8:%[0-9]+]]:_(<2 x s16>), [[UV9:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; VI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV8]](<2 x s16>)
     ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
     ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; VI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
     ; VI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-    ; VI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; VI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV9]](<2 x s16>)
     ; VI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
     ; VI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
-    ; VI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT1]](<3 x s16>), 0
-    ; VI: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
-    ; VI: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
+    ; VI: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
+    ; VI: [[UV10:%[0-9]+]]:_(<2 x s16>), [[UV11:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
+    ; VI: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV10]](<2 x s16>)
     ; VI: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
     ; VI: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
     ; VI: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
-    ; VI: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
+    ; VI: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV11]](<2 x s16>)
     ; VI: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
     ; VI: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
     ; VI: [[UMIN:%[0-9]+]]:_(s16) = G_UMIN [[TRUNC]], [[TRUNC3]]
@@ -436,24 +442,27 @@ body: |
     ; VI: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C]](s32)
     ; VI: [[OR1:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[SHL1]]
     ; VI: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
-    ; VI: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
-    ; VI: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<4 x s16>), 0
-    ; VI: S_NOP 0, implicit [[EXTRACT2]](<3 x s16>)
+    ; VI: [[CONCAT_VECTORS2:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
+    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS2]](<4 x s16>), 0
+    ; VI: S_NOP 0, implicit [[EXTRACT]](<3 x s16>)
     ; GFX9-LABEL: name: test_umin_v3s16
     ; GFX9: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
     ; GFX9: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF1]](<4 x s16>), 0
+    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX9: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; GFX9: [[DEF2:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF2]], [[EXTRACT]](<3 x s16>), 0
-    ; GFX9: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF2]], [[EXTRACT1]](<3 x s16>), 0
-    ; GFX9: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
-    ; GFX9: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
-    ; GFX9: [[UMIN:%[0-9]+]]:_(<2 x s16>) = G_UMIN [[UV]], [[UV2]]
-    ; GFX9: [[UMIN1:%[0-9]+]]:_(<2 x s16>) = G_UMIN [[UV1]], [[UV3]]
-    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[UMIN]](<2 x s16>), [[UMIN1]](<2 x s16>)
-    ; GFX9: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<4 x s16>), 0
-    ; GFX9: S_NOP 0, implicit [[EXTRACT2]](<3 x s16>)
+    ; GFX9: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF2]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX9: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
+    ; GFX9: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
+    ; GFX9: [[UV8:%[0-9]+]]:_(<2 x s16>), [[UV9:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; GFX9: [[UV10:%[0-9]+]]:_(<2 x s16>), [[UV11:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
+    ; GFX9: [[UMIN:%[0-9]+]]:_(<2 x s16>) = G_UMIN [[UV8]], [[UV10]]
+    ; GFX9: [[UMIN1:%[0-9]+]]:_(<2 x s16>) = G_UMIN [[UV9]], [[UV11]]
+    ; GFX9: [[CONCAT_VECTORS2:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[UMIN]](<2 x s16>), [[UMIN1]](<2 x s16>)
+    ; GFX9: [[CONCAT_VECTORS3:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[CONCAT_VECTORS2]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; GFX9: [[UV12:%[0-9]+]]:_(<3 x s16>), [[UV13:%[0-9]+]]:_(<3 x s16>), [[UV14:%[0-9]+]]:_(<3 x s16>), [[UV15:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS3]](<12 x s16>)
+    ; GFX9: S_NOP 0, implicit [[UV12]](<3 x s16>)
     %0:_(<3 x s16>) = G_IMPLICIT_DEF
     %1:_(<3 x s16>) = G_IMPLICIT_DEF
     %2:_(<3 x s16>) = G_UMIN %0, %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-unmerge-values.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-unmerge-values.mir
index f40ed9796923c..11b9da883008e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-unmerge-values.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-unmerge-values.mir
@@ -60,13 +60,15 @@ body: |
   bb.0:
     ; CHECK-LABEL: name: test_unmerge_s16_v3s16
     ; CHECK: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; CHECK: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
-    ; CHECK: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[EXTRACT]](<3 x s16>), 0
-    ; CHECK: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
-    ; CHECK: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; CHECK: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; CHECK: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
+    ; CHECK: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
+    ; CHECK: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; CHECK: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
     ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; CHECK: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
-    ; CHECK: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; CHECK: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
     ; CHECK: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
     ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
     ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-usubsat.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-usubsat.mir
index 5bb430cf4a062..6b5d9c8cb5c59 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-usubsat.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-usubsat.mir
@@ -403,9 +403,11 @@ body: |
     ; GFX6: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
     ; GFX6: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
     ; GFX6: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<4 x s16>), 0
-    ; GFX6: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
-    ; GFX6: [[CONCAT_VECTORS1:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[EXTRACT]](<3 x s16>), [[EXTRACT1]](<3 x s16>)
-    ; GFX6: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS1]](<6 x s16>)
+    ; GFX6: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; GFX6: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF1]](<4 x s16>), [[DEF]](<4 x s16>), [[DEF]](<4 x s16>)
+    ; GFX6: [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>), [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; GFX6: [[CONCAT_VECTORS2:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[EXTRACT]](<3 x s16>), [[UV6]](<3 x s16>)
+    ; GFX6: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS2]](<6 x s16>)
     ; GFX8-LABEL: name: usubsat_v3s16
     ; GFX8: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
     ; GFX8: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
@@ -444,9 +446,11 @@ body: |
     ; GFX8: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
     ; GFX8: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
     ; GFX8: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<4 x s16>), 0
-    ; GFX8: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
-    ; GFX8: [[CONCAT_VECTORS1:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[EXTRACT]](<3 x s16>), [[EXTRACT1]](<3 x s16>)
-    ; GFX8: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS1]](<6 x s16>)
+    ; GFX8: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; GFX8: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF1]](<4 x s16>), [[DEF]](<4 x s16>), [[DEF]](<4 x s16>)
+    ; GFX8: [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>), [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; GFX8: [[CONCAT_VECTORS2:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[EXTRACT]](<3 x s16>), [[UV6]](<3 x s16>)
+    ; GFX8: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS2]](<6 x s16>)
     ; GFX9-LABEL: name: usubsat_v3s16
     ; GFX9: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
     ; GFX9: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
@@ -480,9 +484,11 @@ body: |
     ; GFX9: [[USUBSAT1:%[0-9]+]]:_(<2 x s16>) = G_USUBSAT [[BUILD_VECTOR_TRUNC1]], [[BUILD_VECTOR_TRUNC3]]
     ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[USUBSAT]](<2 x s16>), [[USUBSAT1]](<2 x s16>), [[DEF2]](<2 x s16>)
     ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<6 x s16>), 0
-    ; GFX9: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
-    ; GFX9: [[CONCAT_VECTORS1:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[EXTRACT]](<3 x s16>), [[EXTRACT1]](<3 x s16>)
-    ; GFX9: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS1]](<6 x s16>)
+    ; GFX9: [[DEF3:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; GFX9: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF3]](<4 x s16>), [[DEF]](<4 x s16>), [[DEF]](<4 x s16>)
+    ; GFX9: [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>), [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; GFX9: [[CONCAT_VECTORS2:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[EXTRACT]](<3 x s16>), [[UV6]](<3 x s16>)
+    ; GFX9: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS2]](<6 x s16>)
     %0:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
     %1:_(<3 x s16>), %2:_(<3 x s16>) = G_UNMERGE_VALUES %0
     %3:_(<3 x s16>) = G_USUBSAT %1, %2
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-xor.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-xor.mir
index 110ed17400d26..3c3987a61cf4f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-xor.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-xor.mir
@@ -361,15 +361,18 @@ body: |
 
     ; CHECK-LABEL: name: test_xor_v3s16
     ; CHECK: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; CHECK: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
     ; CHECK: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; CHECK: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF1]](<4 x s16>), 0
+    ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; CHECK: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>), [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s16>)
     ; CHECK: [[DEF2:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; CHECK: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF2]], [[EXTRACT]](<3 x s16>), 0
-    ; CHECK: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF2]], [[EXTRACT1]](<3 x s16>), 0
+    ; CHECK: [[CONCAT_VECTORS1:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[DEF2]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; CHECK: [[UV4:%[0-9]+]]:_(<3 x s16>), [[UV5:%[0-9]+]]:_(<3 x s16>), [[UV6:%[0-9]+]]:_(<3 x s16>), [[UV7:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<12 x s16>)
+    ; CHECK: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV]](<3 x s16>), 0
+    ; CHECK: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV4]](<3 x s16>), 0
     ; CHECK: [[XOR:%[0-9]+]]:_(<4 x s16>) = G_XOR [[INSERT]], [[INSERT1]]
-    ; CHECK: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[XOR]](<4 x s16>), 0
-    ; CHECK: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF2]], [[EXTRACT2]](<3 x s16>), 0
+    ; CHECK: [[CONCAT_VECTORS2:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[XOR]](<4 x s16>), [[DEF1]](<4 x s16>), [[DEF1]](<4 x s16>)
+    ; CHECK: [[UV8:%[0-9]+]]:_(<3 x s16>), [[UV9:%[0-9]+]]:_(<3 x s16>), [[UV10:%[0-9]+]]:_(<3 x s16>), [[UV11:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS2]](<12 x s16>)
+    ; CHECK: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[UV8]](<3 x s16>), 0
     ; CHECK: $vgpr0_vgpr1 = COPY [[INSERT2]](<4 x s16>)
     %0:_(<3 x s16>) = G_IMPLICIT_DEF
     %1:_(<3 x s16>) = G_IMPLICIT_DEF
@@ -403,27 +406,32 @@ body: |
 
     ; CHECK-LABEL: name: test_xor_v5s16
     ; CHECK: [[DEF:%[0-9]+]]:_(<6 x s16>) = G_IMPLICIT_DEF
-    ; CHECK: [[EXTRACT:%[0-9]+]]:_(<5 x s16>) = G_EXTRACT [[DEF]](<6 x s16>), 0
     ; CHECK: [[DEF1:%[0-9]+]]:_(<6 x s16>) = G_IMPLICIT_DEF
-    ; CHECK: [[EXTRACT1:%[0-9]+]]:_(<5 x s16>) = G_EXTRACT [[DEF1]](<6 x s16>), 0
+    ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<30 x s16>) = G_CONCAT_VECTORS [[DEF]](<6 x s16>), [[DEF1]](<6 x s16>), [[DEF1]](<6 x s16>), [[DEF1]](<6 x s16>), [[DEF1]](<6 x s16>)
+    ; CHECK: [[UV:%[0-9]+]]:_(<5 x s16>), [[UV1:%[0-9]+]]:_(<5 x s16>), [[UV2:%[0-9]+]]:_(<5 x s16>), [[UV3:%[0-9]+]]:_(<5 x s16>), [[UV4:%[0-9]+]]:_(<5 x s16>), [[UV5:%[0-9]+]]:_(<5 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<30 x s16>)
     ; CHECK: [[DEF2:%[0-9]+]]:_(<6 x s16>) = G_IMPLICIT_DEF
-    ; CHECK: [[INSERT:%[0-9]+]]:_(<6 x s16>) = G_INSERT [[DEF2]], [[EXTRACT]](<5 x s16>), 0
-    ; CHECK: [[INSERT1:%[0-9]+]]:_(<6 x s16>) = G_INSERT [[DEF2]], [[EXTRACT1]](<5 x s16>), 0
-    ; CHECK: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[INSERT]](<6 x s16>)
-    ; CHECK: [[UV2:%[0-9]+]]:_(<3 x s16>), [[UV3:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<6 x s16>)
+    ; CHECK: [[CONCAT_VECTORS1:%[0-9]+]]:_(<30 x s16>) = G_CONCAT_VECTORS [[DEF2]](<6 x s16>), [[DEF1]](<6 x s16>), [[DEF1]](<6 x s16>), [[DEF1]](<6 x s16>), [[DEF1]](<6 x s16>)
+    ; CHECK: [[UV6:%[0-9]+]]:_(<5 x s16>), [[UV7:%[0-9]+]]:_(<5 x s16>), [[UV8:%[0-9]+]]:_(<5 x s16>), [[UV9:%[0-9]+]]:_(<5 x s16>), [[UV10:%[0-9]+]]:_(<5 x s16>), [[UV11:%[0-9]+]]:_(<5 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS1]](<30 x s16>)
+    ; CHECK: [[INSERT:%[0-9]+]]:_(<6 x s16>) = G_INSERT [[DEF1]], [[UV]](<5 x s16>), 0
+    ; CHECK: [[INSERT1:%[0-9]+]]:_(<6 x s16>) = G_INSERT [[DEF1]], [[UV6]](<5 x s16>), 0
+    ; CHECK: [[UV12:%[0-9]+]]:_(<3 x s16>), [[UV13:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[INSERT]](<6 x s16>)
+    ; CHECK: [[UV14:%[0-9]+]]:_(<3 x s16>), [[UV15:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<6 x s16>)
     ; CHECK: [[DEF3:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; CHECK: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF3]], [[UV]](<3 x s16>), 0
-    ; CHECK: [[INSERT3:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF3]], [[UV2]](<3 x s16>), 0
+    ; CHECK: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF3]], [[UV12]](<3 x s16>), 0
+    ; CHECK: [[INSERT3:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF3]], [[UV14]](<3 x s16>), 0
     ; CHECK: [[XOR:%[0-9]+]]:_(<4 x s16>) = G_XOR [[INSERT2]], [[INSERT3]]
-    ; CHECK: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[XOR]](<4 x s16>), 0
-    ; CHECK: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF3]], [[UV1]](<3 x s16>), 0
-    ; CHECK: [[INSERT5:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF3]], [[UV3]](<3 x s16>), 0
+    ; CHECK: [[CONCAT_VECTORS2:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[XOR]](<4 x s16>), [[DEF3]](<4 x s16>), [[DEF3]](<4 x s16>)
+    ; CHECK: [[UV16:%[0-9]+]]:_(<3 x s16>), [[UV17:%[0-9]+]]:_(<3 x s16>), [[UV18:%[0-9]+]]:_(<3 x s16>), [[UV19:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS2]](<12 x s16>)
+    ; CHECK: [[INSERT4:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF3]], [[UV13]](<3 x s16>), 0
+    ; CHECK: [[INSERT5:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF3]], [[UV15]](<3 x s16>), 0
     ; CHECK: [[XOR1:%[0-9]+]]:_(<4 x s16>) = G_XOR [[INSERT4]], [[INSERT5]]
-    ; CHECK: [[EXTRACT3:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[XOR1]](<4 x s16>), 0
-    ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[EXTRACT2]](<3 x s16>), [[EXTRACT3]](<3 x s16>)
-    ; CHECK: [[EXTRACT4:%[0-9]+]]:_(<5 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<6 x s16>), 0
+    ; CHECK: [[CONCAT_VECTORS3:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[XOR1]](<4 x s16>), [[DEF3]](<4 x s16>), [[DEF3]](<4 x s16>)
+    ; CHECK: [[UV20:%[0-9]+]]:_(<3 x s16>), [[UV21:%[0-9]+]]:_(<3 x s16>), [[UV22:%[0-9]+]]:_(<3 x s16>), [[UV23:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS3]](<12 x s16>)
+    ; CHECK: [[CONCAT_VECTORS4:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[UV16]](<3 x s16>), [[UV20]](<3 x s16>)
+    ; CHECK: [[CONCAT_VECTORS5:%[0-9]+]]:_(<30 x s16>) = G_CONCAT_VECTORS [[CONCAT_VECTORS4]](<6 x s16>), [[DEF1]](<6 x s16>), [[DEF1]](<6 x s16>), [[DEF1]](<6 x s16>), [[DEF1]](<6 x s16>)
+    ; CHECK: [[UV24:%[0-9]+]]:_(<5 x s16>), [[UV25:%[0-9]+]]:_(<5 x s16>), [[UV26:%[0-9]+]]:_(<5 x s16>), [[UV27:%[0-9]+]]:_(<5 x s16>), [[UV28:%[0-9]+]]:_(<5 x s16>), [[UV29:%[0-9]+]]:_(<5 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS5]](<30 x s16>)
     ; CHECK: [[DEF4:%[0-9]+]]:_(<8 x s16>) = G_IMPLICIT_DEF
-    ; CHECK: [[INSERT6:%[0-9]+]]:_(<8 x s16>) = G_INSERT [[DEF4]], [[EXTRACT4]](<5 x s16>), 0
+    ; CHECK: [[INSERT6:%[0-9]+]]:_(<8 x s16>) = G_INSERT [[DEF4]], [[UV24]](<5 x s16>), 0
     ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INSERT6]](<8 x s16>)
     %0:_(<5 x s16>) = G_IMPLICIT_DEF
     %1:_(<5 x s16>) = G_IMPLICIT_DEF
@@ -463,8 +471,9 @@ body: |
     ; CHECK: [[XOR3:%[0-9]+]]:_(s32) = G_XOR [[ANYEXT6]], [[ANYEXT7]]
     ; CHECK: [[TRUNC3:%[0-9]+]]:_(s8) = G_TRUNC [[XOR3]](s32)
     ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s8>) = G_BUILD_VECTOR [[TRUNC]](s8), [[TRUNC1]](s8), [[TRUNC2]](s8), [[TRUNC3]](s8)
-    ; CHECK: [[EXTRACT:%[0-9]+]]:_(<3 x s8>) = G_EXTRACT [[BUILD_VECTOR]](<4 x s8>), 0
-    ; CHECK: [[ANYEXT8:%[0-9]+]]:_(<3 x s32>) = G_ANYEXT [[EXTRACT]](<3 x s8>)
+    ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s8>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<4 x s8>), [[DEF2]](<4 x s8>), [[DEF2]](<4 x s8>)
+    ; CHECK: [[UV8:%[0-9]+]]:_(<3 x s8>), [[UV9:%[0-9]+]]:_(<3 x s8>), [[UV10:%[0-9]+]]:_(<3 x s8>), [[UV11:%[0-9]+]]:_(<3 x s8>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s8>)
+    ; CHECK: [[ANYEXT8:%[0-9]+]]:_(<3 x s32>) = G_ANYEXT [[UV8]](<3 x s8>)
     ; CHECK: $vgpr0_vgpr1_vgpr2 = COPY [[ANYEXT8]](<3 x s32>)
     %0:_(<3 x s8>) = G_IMPLICIT_DEF
     %1:_(<3 x s8>) = G_IMPLICIT_DEF
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.s.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.s.buffer.load.ll
index aba47890f61cb..805aa301f9383 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.s.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.s.buffer.load.ll
@@ -172,19 +172,24 @@ define amdgpu_ps <3 x i32> @s_buffer_load_v3i32(<4 x i32> inreg %rsrc, i32 inreg
   ; GFX6:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
   ; GFX6:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; GFX6:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX6:   [[S_BUFFER_LOAD_DWORDX4_SGPR:%[0-9]+]]:sgpr_128_with_sub0_sub1_sub2 = S_BUFFER_LOAD_DWORDX4_SGPR [[REG_SEQUENCE]], [[COPY4]], 0, 0 :: (dereferenceable invariant load 12, align 4)
-  ; GFX6:   [[COPY5:%[0-9]+]]:sgpr_96 = COPY [[S_BUFFER_LOAD_DWORDX4_SGPR]].sub0_sub1_sub2
-  ; GFX6:   [[COPY6:%[0-9]+]]:sreg_32 = COPY [[COPY5]].sub0
-  ; GFX6:   [[COPY7:%[0-9]+]]:sreg_32 = COPY [[COPY5]].sub1
-  ; GFX6:   [[COPY8:%[0-9]+]]:sreg_32 = COPY [[COPY5]].sub2
-  ; GFX6:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
-  ; GFX6:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
+  ; GFX6:   [[S_BUFFER_LOAD_DWORDX4_SGPR:%[0-9]+]]:sgpr_128 = S_BUFFER_LOAD_DWORDX4_SGPR [[REG_SEQUENCE]], [[COPY4]], 0, 0 :: (dereferenceable invariant load 12, align 4)
+  ; GFX6:   [[DEF:%[0-9]+]]:sgpr_128 = IMPLICIT_DEF
+  ; GFX6:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_512 = REG_SEQUENCE [[S_BUFFER_LOAD_DWORDX4_SGPR]], %subreg.sub0_sub1_sub2_sub3, [[DEF]], %subreg.sub4_sub5_sub6_sub7, [[DEF]], %subreg.sub8_sub9_sub10_sub11
+  ; GFX6:   [[COPY5:%[0-9]+]]:sgpr_96 = COPY [[REG_SEQUENCE1]].sub0_sub1_sub2
+  ; GFX6:   [[COPY6:%[0-9]+]]:sgpr_96 = COPY [[REG_SEQUENCE1]].sub3_sub4_sub5
+  ; GFX6:   [[COPY7:%[0-9]+]]:sgpr_96 = COPY [[REG_SEQUENCE1]].sub6_sub7_sub8
+  ; GFX6:   [[COPY8:%[0-9]+]]:sgpr_96 = COPY [[REG_SEQUENCE1]].sub9_sub10_sub11
+  ; GFX6:   [[COPY9:%[0-9]+]]:sreg_32 = COPY [[COPY5]].sub0
+  ; GFX6:   [[COPY10:%[0-9]+]]:sreg_32 = COPY [[COPY5]].sub1
+  ; GFX6:   [[COPY11:%[0-9]+]]:sreg_32 = COPY [[COPY5]].sub2
+  ; GFX6:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[COPY9]]
+  ; GFX6:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY12]], implicit $exec
   ; GFX6:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
-  ; GFX6:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[COPY7]]
-  ; GFX6:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
+  ; GFX6:   [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[COPY10]]
+  ; GFX6:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY13]], implicit $exec
   ; GFX6:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
-  ; GFX6:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[COPY8]]
-  ; GFX6:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY11]], implicit $exec
+  ; GFX6:   [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[COPY11]]
+  ; GFX6:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY14]], implicit $exec
   ; GFX6:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_2]]
   ; GFX6:   SI_RETURN_TO_EPILOG implicit $sgpr0, implicit $sgpr1, implicit $sgpr2
   ; GFX7-LABEL: name: s_buffer_load_v3i32
@@ -196,19 +201,24 @@ define amdgpu_ps <3 x i32> @s_buffer_load_v3i32(<4 x i32> inreg %rsrc, i32 inreg
   ; GFX7:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
   ; GFX7:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; GFX7:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX7:   [[S_BUFFER_LOAD_DWORDX4_SGPR:%[0-9]+]]:sgpr_128_with_sub0_sub1_sub2 = S_BUFFER_LOAD_DWORDX4_SGPR [[REG_SEQUENCE]], [[COPY4]], 0, 0 :: (dereferenceable invariant load 12, align 4)
-  ; GFX7:   [[COPY5:%[0-9]+]]:sgpr_96 = COPY [[S_BUFFER_LOAD_DWORDX4_SGPR]].sub0_sub1_sub2
-  ; GFX7:   [[COPY6:%[0-9]+]]:sreg_32 = COPY [[COPY5]].sub0
-  ; GFX7:   [[COPY7:%[0-9]+]]:sreg_32 = COPY [[COPY5]].sub1
-  ; GFX7:   [[COPY8:%[0-9]+]]:sreg_32 = COPY [[COPY5]].sub2
-  ; GFX7:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
-  ; GFX7:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
+  ; GFX7:   [[S_BUFFER_LOAD_DWORDX4_SGPR:%[0-9]+]]:sgpr_128 = S_BUFFER_LOAD_DWORDX4_SGPR [[REG_SEQUENCE]], [[COPY4]], 0, 0 :: (dereferenceable invariant load 12, align 4)
+  ; GFX7:   [[DEF:%[0-9]+]]:sgpr_128 = IMPLICIT_DEF
+  ; GFX7:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_512 = REG_SEQUENCE [[S_BUFFER_LOAD_DWORDX4_SGPR]], %subreg.sub0_sub1_sub2_sub3, [[DEF]], %subreg.sub4_sub5_sub6_sub7, [[DEF]], %subreg.sub8_sub9_sub10_sub11
+  ; GFX7:   [[COPY5:%[0-9]+]]:sgpr_96 = COPY [[REG_SEQUENCE1]].sub0_sub1_sub2
+  ; GFX7:   [[COPY6:%[0-9]+]]:sgpr_96 = COPY [[REG_SEQUENCE1]].sub3_sub4_sub5
+  ; GFX7:   [[COPY7:%[0-9]+]]:sgpr_96 = COPY [[REG_SEQUENCE1]].sub6_sub7_sub8
+  ; GFX7:   [[COPY8:%[0-9]+]]:sgpr_96 = COPY [[REG_SEQUENCE1]].sub9_sub10_sub11
+  ; GFX7:   [[COPY9:%[0-9]+]]:sreg_32 = COPY [[COPY5]].sub0
+  ; GFX7:   [[COPY10:%[0-9]+]]:sreg_32 = COPY [[COPY5]].sub1
+  ; GFX7:   [[COPY11:%[0-9]+]]:sreg_32 = COPY [[COPY5]].sub2
+  ; GFX7:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[COPY9]]
+  ; GFX7:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY12]], implicit $exec
   ; GFX7:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
-  ; GFX7:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[COPY7]]
-  ; GFX7:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
+  ; GFX7:   [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[COPY10]]
+  ; GFX7:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY13]], implicit $exec
   ; GFX7:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
-  ; GFX7:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[COPY8]]
-  ; GFX7:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY11]], implicit $exec
+  ; GFX7:   [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[COPY11]]
+  ; GFX7:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY14]], implicit $exec
   ; GFX7:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_2]]
   ; GFX7:   SI_RETURN_TO_EPILOG implicit $sgpr0, implicit $sgpr1, implicit $sgpr2
   ; GFX8-LABEL: name: s_buffer_load_v3i32
@@ -220,19 +230,24 @@ define amdgpu_ps <3 x i32> @s_buffer_load_v3i32(<4 x i32> inreg %rsrc, i32 inreg
   ; GFX8:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
   ; GFX8:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; GFX8:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX8:   [[S_BUFFER_LOAD_DWORDX4_SGPR:%[0-9]+]]:sgpr_128_with_sub0_sub1_sub2 = S_BUFFER_LOAD_DWORDX4_SGPR [[REG_SEQUENCE]], [[COPY4]], 0, 0 :: (dereferenceable invariant load 12, align 4)
-  ; GFX8:   [[COPY5:%[0-9]+]]:sgpr_96 = COPY [[S_BUFFER_LOAD_DWORDX4_SGPR]].sub0_sub1_sub2
-  ; GFX8:   [[COPY6:%[0-9]+]]:sreg_32 = COPY [[COPY5]].sub0
-  ; GFX8:   [[COPY7:%[0-9]+]]:sreg_32 = COPY [[COPY5]].sub1
-  ; GFX8:   [[COPY8:%[0-9]+]]:sreg_32 = COPY [[COPY5]].sub2
-  ; GFX8:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
-  ; GFX8:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
+  ; GFX8:   [[S_BUFFER_LOAD_DWORDX4_SGPR:%[0-9]+]]:sgpr_128 = S_BUFFER_LOAD_DWORDX4_SGPR [[REG_SEQUENCE]], [[COPY4]], 0, 0 :: (dereferenceable invariant load 12, align 4)
+  ; GFX8:   [[DEF:%[0-9]+]]:sgpr_128 = IMPLICIT_DEF
+  ; GFX8:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_512 = REG_SEQUENCE [[S_BUFFER_LOAD_DWORDX4_SGPR]], %subreg.sub0_sub1_sub2_sub3, [[DEF]], %subreg.sub4_sub5_sub6_sub7, [[DEF]], %subreg.sub8_sub9_sub10_sub11
+  ; GFX8:   [[COPY5:%[0-9]+]]:sgpr_96 = COPY [[REG_SEQUENCE1]].sub0_sub1_sub2
+  ; GFX8:   [[COPY6:%[0-9]+]]:sgpr_96 = COPY [[REG_SEQUENCE1]].sub3_sub4_sub5
+  ; GFX8:   [[COPY7:%[0-9]+]]:sgpr_96 = COPY [[REG_SEQUENCE1]].sub6_sub7_sub8
+  ; GFX8:   [[COPY8:%[0-9]+]]:sgpr_96 = COPY [[REG_SEQUENCE1]].sub9_sub10_sub11
+  ; GFX8:   [[COPY9:%[0-9]+]]:sreg_32 = COPY [[COPY5]].sub0
+  ; GFX8:   [[COPY10:%[0-9]+]]:sreg_32 = COPY [[COPY5]].sub1
+  ; GFX8:   [[COPY11:%[0-9]+]]:sreg_32 = COPY [[COPY5]].sub2
+  ; GFX8:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[COPY9]]
+  ; GFX8:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY12]], implicit $exec
   ; GFX8:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
-  ; GFX8:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[COPY7]]
-  ; GFX8:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
+  ; GFX8:   [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[COPY10]]
+  ; GFX8:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY13]], implicit $exec
   ; GFX8:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
-  ; GFX8:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[COPY8]]
-  ; GFX8:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY11]], implicit $exec
+  ; GFX8:   [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[COPY11]]
+  ; GFX8:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY14]], implicit $exec
   ; GFX8:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_2]]
   ; GFX8:   SI_RETURN_TO_EPILOG implicit $sgpr0, implicit $sgpr1, implicit $sgpr2
   %val = call <3 x i32> @llvm.amdgcn.s.buffer.load.v3i32(<4 x i32> %rsrc, i32 %soffset, i32 0)
@@ -1580,13 +1595,20 @@ define amdgpu_ps <3 x float> @s_buffer_load_v3f32_vgpr_offset(<4 x i32> inreg %r
   ; GFX6:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; GFX6:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
   ; GFX6:   [[BUFFER_LOAD_DWORDX4_OFFEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16, align 4)
-  ; GFX6:   [[COPY5:%[0-9]+]]:vreg_96 = COPY [[BUFFER_LOAD_DWORDX4_OFFEN]].sub0_sub1_sub2
-  ; GFX6:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[COPY5]].sub0
-  ; GFX6:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY5]].sub1
-  ; GFX6:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY5]].sub2
-  ; GFX6:   $vgpr0 = COPY [[COPY6]]
-  ; GFX6:   $vgpr1 = COPY [[COPY7]]
-  ; GFX6:   $vgpr2 = COPY [[COPY8]]
+  ; GFX6:   [[DEF:%[0-9]+]]:sgpr_128 = IMPLICIT_DEF
+  ; GFX6:   [[COPY5:%[0-9]+]]:vreg_128 = COPY [[DEF]]
+  ; GFX6:   [[COPY6:%[0-9]+]]:vreg_128 = COPY [[DEF]]
+  ; GFX6:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_512 = REG_SEQUENCE [[BUFFER_LOAD_DWORDX4_OFFEN]], %subreg.sub0_sub1_sub2_sub3, [[COPY5]], %subreg.sub4_sub5_sub6_sub7, [[COPY6]], %subreg.sub8_sub9_sub10_sub11
+  ; GFX6:   [[COPY7:%[0-9]+]]:vreg_96 = COPY [[REG_SEQUENCE1]].sub0_sub1_sub2
+  ; GFX6:   [[COPY8:%[0-9]+]]:vreg_96 = COPY [[REG_SEQUENCE1]].sub3_sub4_sub5
+  ; GFX6:   [[COPY9:%[0-9]+]]:vreg_96 = COPY [[REG_SEQUENCE1]].sub6_sub7_sub8
+  ; GFX6:   [[COPY10:%[0-9]+]]:vreg_96 = COPY [[REG_SEQUENCE1]].sub9_sub10_sub11
+  ; GFX6:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[COPY7]].sub0
+  ; GFX6:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[COPY7]].sub1
+  ; GFX6:   [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[COPY7]].sub2
+  ; GFX6:   $vgpr0 = COPY [[COPY11]]
+  ; GFX6:   $vgpr1 = COPY [[COPY12]]
+  ; GFX6:   $vgpr2 = COPY [[COPY13]]
   ; GFX6:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2
   ; GFX7-LABEL: name: s_buffer_load_v3f32_vgpr_offset
   ; GFX7: bb.1 (%ir-block.0):
@@ -1599,13 +1621,20 @@ define amdgpu_ps <3 x float> @s_buffer_load_v3f32_vgpr_offset(<4 x i32> inreg %r
   ; GFX7:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; GFX7:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
   ; GFX7:   [[BUFFER_LOAD_DWORDX4_OFFEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16, align 4)
-  ; GFX7:   [[COPY5:%[0-9]+]]:vreg_96 = COPY [[BUFFER_LOAD_DWORDX4_OFFEN]].sub0_sub1_sub2
-  ; GFX7:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[COPY5]].sub0
-  ; GFX7:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY5]].sub1
-  ; GFX7:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY5]].sub2
-  ; GFX7:   $vgpr0 = COPY [[COPY6]]
-  ; GFX7:   $vgpr1 = COPY [[COPY7]]
-  ; GFX7:   $vgpr2 = COPY [[COPY8]]
+  ; GFX7:   [[DEF:%[0-9]+]]:sgpr_128 = IMPLICIT_DEF
+  ; GFX7:   [[COPY5:%[0-9]+]]:vreg_128 = COPY [[DEF]]
+  ; GFX7:   [[COPY6:%[0-9]+]]:vreg_128 = COPY [[DEF]]
+  ; GFX7:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_512 = REG_SEQUENCE [[BUFFER_LOAD_DWORDX4_OFFEN]], %subreg.sub0_sub1_sub2_sub3, [[COPY5]], %subreg.sub4_sub5_sub6_sub7, [[COPY6]], %subreg.sub8_sub9_sub10_sub11
+  ; GFX7:   [[COPY7:%[0-9]+]]:vreg_96 = COPY [[REG_SEQUENCE1]].sub0_sub1_sub2
+  ; GFX7:   [[COPY8:%[0-9]+]]:vreg_96 = COPY [[REG_SEQUENCE1]].sub3_sub4_sub5
+  ; GFX7:   [[COPY9:%[0-9]+]]:vreg_96 = COPY [[REG_SEQUENCE1]].sub6_sub7_sub8
+  ; GFX7:   [[COPY10:%[0-9]+]]:vreg_96 = COPY [[REG_SEQUENCE1]].sub9_sub10_sub11
+  ; GFX7:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[COPY7]].sub0
+  ; GFX7:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[COPY7]].sub1
+  ; GFX7:   [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[COPY7]].sub2
+  ; GFX7:   $vgpr0 = COPY [[COPY11]]
+  ; GFX7:   $vgpr1 = COPY [[COPY12]]
+  ; GFX7:   $vgpr2 = COPY [[COPY13]]
   ; GFX7:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2
   ; GFX8-LABEL: name: s_buffer_load_v3f32_vgpr_offset
   ; GFX8: bb.1 (%ir-block.0):
@@ -1618,13 +1647,20 @@ define amdgpu_ps <3 x float> @s_buffer_load_v3f32_vgpr_offset(<4 x i32> inreg %r
   ; GFX8:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; GFX8:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
   ; GFX8:   [[BUFFER_LOAD_DWORDX4_OFFEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 16, align 4)
-  ; GFX8:   [[COPY5:%[0-9]+]]:vreg_96 = COPY [[BUFFER_LOAD_DWORDX4_OFFEN]].sub0_sub1_sub2
-  ; GFX8:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[COPY5]].sub0
-  ; GFX8:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY5]].sub1
-  ; GFX8:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY5]].sub2
-  ; GFX8:   $vgpr0 = COPY [[COPY6]]
-  ; GFX8:   $vgpr1 = COPY [[COPY7]]
-  ; GFX8:   $vgpr2 = COPY [[COPY8]]
+  ; GFX8:   [[DEF:%[0-9]+]]:sgpr_128 = IMPLICIT_DEF
+  ; GFX8:   [[COPY5:%[0-9]+]]:vreg_128 = COPY [[DEF]]
+  ; GFX8:   [[COPY6:%[0-9]+]]:vreg_128 = COPY [[DEF]]
+  ; GFX8:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_512 = REG_SEQUENCE [[BUFFER_LOAD_DWORDX4_OFFEN]], %subreg.sub0_sub1_sub2_sub3, [[COPY5]], %subreg.sub4_sub5_sub6_sub7, [[COPY6]], %subreg.sub8_sub9_sub10_sub11
+  ; GFX8:   [[COPY7:%[0-9]+]]:vreg_96 = COPY [[REG_SEQUENCE1]].sub0_sub1_sub2
+  ; GFX8:   [[COPY8:%[0-9]+]]:vreg_96 = COPY [[REG_SEQUENCE1]].sub3_sub4_sub5
+  ; GFX8:   [[COPY9:%[0-9]+]]:vreg_96 = COPY [[REG_SEQUENCE1]].sub6_sub7_sub8
+  ; GFX8:   [[COPY10:%[0-9]+]]:vreg_96 = COPY [[REG_SEQUENCE1]].sub9_sub10_sub11
+  ; GFX8:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[COPY7]].sub0
+  ; GFX8:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[COPY7]].sub1
+  ; GFX8:   [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[COPY7]].sub2
+  ; GFX8:   $vgpr0 = COPY [[COPY11]]
+  ; GFX8:   $vgpr1 = COPY [[COPY12]]
+  ; GFX8:   $vgpr2 = COPY [[COPY13]]
   ; GFX8:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2
   %val = call <3 x float> @llvm.amdgcn.s.buffer.load.v3f32(<4 x i32> %rsrc, i32 %soffset, i32 0)
   ret <3 x float> %val
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.s.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.s.buffer.load.ll
index b8e69433913ea..670c9898c2798 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.s.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.s.buffer.load.ll
@@ -56,15 +56,17 @@ define amdgpu_ps <3 x i32> @s_buffer_load_v3i32(<4 x i32> inreg %rsrc, i32 inreg
   ; CHECK:   [[COPY4:%[0-9]+]]:sgpr(s32) = COPY $sgpr6
   ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32)
   ; CHECK:   [[AMDGPU_S_BUFFER_LOAD:%[0-9]+]]:sgpr(<4 x s32>) = G_AMDGPU_S_BUFFER_LOAD [[BUILD_VECTOR]](<4 x s32>), [[COPY4]](s32), 0 :: (dereferenceable invariant load 12, align 4)
-  ; CHECK:   [[EXTRACT:%[0-9]+]]:sgpr(<3 x s32>) = G_EXTRACT [[AMDGPU_S_BUFFER_LOAD]](<4 x s32>), 0
-  ; CHECK:   [[UV:%[0-9]+]]:sgpr(s32), [[UV1:%[0-9]+]]:sgpr(s32), [[UV2:%[0-9]+]]:sgpr(s32) = G_UNMERGE_VALUES [[EXTRACT]](<3 x s32>)
-  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr(s32) = COPY [[UV]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:sgpr(<4 x s32>) = G_IMPLICIT_DEF
+  ; CHECK:   [[CONCAT_VECTORS:%[0-9]+]]:sgpr(<12 x s32>) = G_CONCAT_VECTORS [[AMDGPU_S_BUFFER_LOAD]](<4 x s32>), [[DEF]](<4 x s32>), [[DEF]](<4 x s32>)
+  ; CHECK:   [[UV:%[0-9]+]]:sgpr(<3 x s32>), [[UV1:%[0-9]+]]:sgpr(<3 x s32>), [[UV2:%[0-9]+]]:sgpr(<3 x s32>), [[UV3:%[0-9]+]]:sgpr(<3 x s32>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s32>)
+  ; CHECK:   [[UV4:%[0-9]+]]:sgpr(s32), [[UV5:%[0-9]+]]:sgpr(s32), [[UV6:%[0-9]+]]:sgpr(s32) = G_UNMERGE_VALUES [[UV]](<3 x s32>)
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr(s32) = COPY [[UV4]](s32)
   ; CHECK:   [[INT:%[0-9]+]]:sgpr(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.readfirstlane), [[COPY5]](s32)
   ; CHECK:   $sgpr0 = COPY [[INT]](s32)
-  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr(s32) = COPY [[UV1]](s32)
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr(s32) = COPY [[UV5]](s32)
   ; CHECK:   [[INT1:%[0-9]+]]:sgpr(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.readfirstlane), [[COPY6]](s32)
   ; CHECK:   $sgpr1 = COPY [[INT1]](s32)
-  ; CHECK:   [[COPY7:%[0-9]+]]:vgpr(s32) = COPY [[UV2]](s32)
+  ; CHECK:   [[COPY7:%[0-9]+]]:vgpr(s32) = COPY [[UV6]](s32)
   ; CHECK:   [[INT2:%[0-9]+]]:sgpr(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.readfirstlane), [[COPY7]](s32)
   ; CHECK:   $sgpr2 = COPY [[INT2]](s32)
   ; CHECK:   SI_RETURN_TO_EPILOG implicit $sgpr0, implicit $sgpr1, implicit $sgpr2
@@ -232,11 +234,15 @@ define amdgpu_ps <3 x float> @s_buffer_load_v3f32_vgpr_offset(<4 x i32> inreg %r
   ; CHECK:   [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
   ; CHECK:   [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
   ; CHECK:   [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(<4 x s32>) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x s32>), [[C1]](s32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load 16, align 4)
-  ; CHECK:   [[EXTRACT:%[0-9]+]]:vgpr(<3 x s32>) = G_EXTRACT [[AMDGPU_BUFFER_LOAD]](<4 x s32>), 0
-  ; CHECK:   [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32), [[UV2:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[EXTRACT]](<3 x s32>)
-  ; CHECK:   $vgpr0 = COPY [[UV]](s32)
-  ; CHECK:   $vgpr1 = COPY [[UV1]](s32)
-  ; CHECK:   $vgpr2 = COPY [[UV2]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:sgpr(<4 x s32>) = G_IMPLICIT_DEF
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr(<4 x s32>) = COPY [[DEF]](<4 x s32>)
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr(<4 x s32>) = COPY [[DEF]](<4 x s32>)
+  ; CHECK:   [[CONCAT_VECTORS:%[0-9]+]]:vgpr(<12 x s32>) = G_CONCAT_VECTORS [[AMDGPU_BUFFER_LOAD]](<4 x s32>), [[COPY5]](<4 x s32>), [[COPY6]](<4 x s32>)
+  ; CHECK:   [[UV:%[0-9]+]]:vgpr(<3 x s32>), [[UV1:%[0-9]+]]:vgpr(<3 x s32>), [[UV2:%[0-9]+]]:vgpr(<3 x s32>), [[UV3:%[0-9]+]]:vgpr(<3 x s32>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<12 x s32>)
+  ; CHECK:   [[UV4:%[0-9]+]]:vgpr(s32), [[UV5:%[0-9]+]]:vgpr(s32), [[UV6:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[UV]](<3 x s32>)
+  ; CHECK:   $vgpr0 = COPY [[UV4]](s32)
+  ; CHECK:   $vgpr1 = COPY [[UV5]](s32)
+  ; CHECK:   $vgpr2 = COPY [[UV6]](s32)
   ; CHECK:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2
   %val = call <3 x float> @llvm.amdgcn.s.buffer.load.v3f32(<4 x i32> %rsrc, i32 %soffset, i32 0)
   ret <3 x float> %val
diff --git a/llvm/unittests/CodeGen/GlobalISel/LegalizerHelperTest.cpp b/llvm/unittests/CodeGen/GlobalISel/LegalizerHelperTest.cpp
index 2cfab39d45622..3a9fb59cae6f0 100644
--- a/llvm/unittests/CodeGen/GlobalISel/LegalizerHelperTest.cpp
+++ b/llvm/unittests/CodeGen/GlobalISel/LegalizerHelperTest.cpp
@@ -3041,11 +3041,10 @@ TEST_F(AArch64GISelMITest, MoreElementsFreeze) {
   const auto *CheckStr = R"(
   CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY
   CHECK: [[BITCAST:%[0-9]+]]:_(<2 x s32>) = G_BITCAST [[COPY]]
-
   CHECK: [[UNDEF:%[0-9]+]]:_(<2 x s32>) = G_IMPLICIT_DEF
   CHECK: [[CV:%[0-9]+]]:_(<4 x s32>) = G_CONCAT_VECTORS [[BITCAST]]:_(<2 x s32>), [[UNDEF]]
   CHECK: [[FREEZE:%[0-9]+]]:_(<4 x s32>) = G_FREEZE [[CV]]
-  CHECK: [[EXTR:%[0-9]+]]:_(<2 x s32>) = G_EXTRACT [[FREEZE]]:_(<4 x s32>), 0
+  CHECK: [[EXTR0:%[0-9]+]]:_(<2 x s32>), [[EXTR1:%[0-9]+]]:_(<2 x s32>) = G_UNMERGE_VALUES [[FREEZE]]:_(<4 x s32>)
   )";
 
   // Check

From 2414bab5d7d6b5b247f3f3b97140a2673fa8414b Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Sun, 26 Jul 2020 10:47:08 -0400
Subject: [PATCH 0912/1035] AMDGPU/GlobalISel: Remove old hacks for boolean
 selection

There were various hacks used to try to avoid making s1 SGPR vs. s1
VCC ambiguous after constraining the register before we had a strategy
to deal with this. This also attempted to handle undef operands, which
are now illegal gMIR.
---
 .../AMDGPU/AMDGPUInstructionSelector.cpp      | 70 ++++---------------
 llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h      |  4 ++
 .../AMDGPU/GlobalISel/inst-select-brcond.mir  |  6 +-
 .../AMDGPU/GlobalISel/inst-select-phi.mir     | 18 ++---
 4 files changed, 31 insertions(+), 67 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
index 5aceb40933c37..16fc759f0cbf6 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
@@ -170,19 +170,6 @@ bool AMDGPUInstructionSelector::selectCOPY(MachineInstr &I) const {
     if (RC && !RBI.constrainGenericRegister(DstReg, *RC, *MRI))
       return false;
 
-    // Don't constrain the source register to a class so the def instruction
-    // handles it (unless it's undef).
-    //
-    // FIXME: This is a hack. When selecting the def, we neeed to know
-    // specifically know that the result is VCCRegBank, and not just an SGPR
-    // with size 1. An SReg_32 with size 1 is ambiguous with wave32.
-    if (Src.isUndef()) {
-      const TargetRegisterClass *SrcRC =
-        TRI.getConstrainedRegClassForOperand(Src, *MRI);
-      if (SrcRC && !RBI.constrainGenericRegister(SrcReg, *SrcRC, *MRI))
-        return false;
-    }
-
     return true;
   }
 
@@ -286,50 +273,24 @@ static unsigned getLogicalBitOpcode(unsigned Opc, bool Is64) {
 }
 
 bool AMDGPUInstructionSelector::selectG_AND_OR_XOR(MachineInstr &I) const {
-  MachineOperand &Dst = I.getOperand(0);
-  MachineOperand &Src0 = I.getOperand(1);
-  MachineOperand &Src1 = I.getOperand(2);
-  Register DstReg = Dst.getReg();
+  Register DstReg = I.getOperand(0).getReg();
   unsigned Size = RBI.getSizeInBits(DstReg, *MRI, TRI);
 
   const RegisterBank *DstRB = RBI.getRegBank(DstReg, *MRI, TRI);
-  if (DstRB->getID() == AMDGPU::VCCRegBankID) {
-    const TargetRegisterClass *RC = TRI.getBoolRC();
-    unsigned InstOpc = getLogicalBitOpcode(I.getOpcode(),
-                                           RC == &AMDGPU::SReg_64RegClass);
-    I.setDesc(TII.get(InstOpc));
-    // Dead implicit-def of scc
-    I.addOperand(MachineOperand::CreateReg(AMDGPU::SCC, true, // isDef
-                                           true, // isImp
-                                           false, // isKill
-                                           true)); // isDead
-
-    // FIXME: Hack to avoid turning the register bank into a register class.
-    // The selector for G_ICMP relies on seeing the register bank for the result
-    // is VCC. In wave32 if we constrain the registers to SReg_32 here, it will
-    // be ambiguous whether it's a scalar or vector bool.
-    if (Src0.isUndef() && !MRI->getRegClassOrNull(Src0.getReg()))
-      MRI->setRegClass(Src0.getReg(), RC);
-    if (Src1.isUndef() && !MRI->getRegClassOrNull(Src1.getReg()))
-      MRI->setRegClass(Src1.getReg(), RC);
-
-    return RBI.constrainGenericRegister(DstReg, *RC, *MRI);
-  }
-
-  // TODO: Should this allow an SCC bank result, and produce a copy from SCC for
-  // the result?
-  if (DstRB->getID() == AMDGPU::SGPRRegBankID) {
-    unsigned InstOpc = getLogicalBitOpcode(I.getOpcode(), Size > 32);
-    I.setDesc(TII.get(InstOpc));
-    // Dead implicit-def of scc
-    I.addOperand(MachineOperand::CreateReg(AMDGPU::SCC, true, // isDef
-                                           true, // isImp
-                                           false, // isKill
-                                           true)); // isDead
-    return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
-  }
+  if (DstRB->getID() != AMDGPU::SGPRRegBankID &&
+      DstRB->getID() != AMDGPU::VCCRegBankID)
+    return false;
 
-  return false;
+  bool Is64 = Size > 32 || (DstRB->getID() == AMDGPU::VCCRegBankID &&
+                            STI.isWave64());
+  I.setDesc(TII.get(getLogicalBitOpcode(I.getOpcode(), Is64)));
+
+  // Dead implicit-def of scc
+  I.addOperand(MachineOperand::CreateReg(AMDGPU::SCC, true, // isDef
+                                         true, // isImp
+                                         false, // isKill
+                                         true)); // isDead
+  return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
 }
 
 bool AMDGPUInstructionSelector::selectG_ADD_SUB(MachineInstr &I) const {
@@ -2338,8 +2299,7 @@ bool AMDGPUInstructionSelector::selectG_BRCOND(MachineInstr &I) const {
 
     CondPhysReg = AMDGPU::SCC;
     BrOpcode = AMDGPU::S_CBRANCH_SCC1;
-    // FIXME: Hack for isSCC tests
-    ConstrainRC = &AMDGPU::SGPR_32RegClass;
+    ConstrainRC = &AMDGPU::SReg_32RegClass;
   } else {
     // FIXME: Do we have to insert an and with exec here, like in SelectionDAG?
     // We sort of know that a VCC producer based on the register bank, that ands
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h
index 6848f762fc276..0f57d34ba6aaf 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h
@@ -1210,6 +1210,10 @@ class GCNSubtarget : public AMDGPUGenSubtargetInfo,
     return getWavefrontSize() == 32;
   }
 
+  bool isWave64() const {
+    return getWavefrontSize() == 64;
+  }
+
   const TargetRegisterClass *getBoolRC() const {
     return getRegisterInfo()->getBoolRC();
   }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-brcond.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-brcond.mir
index 496d496a42c58..6adb3549778c1 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-brcond.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-brcond.mir
@@ -20,7 +20,7 @@ body: |
   ; GCN:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
   ; GCN:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
   ; GCN:   S_CMP_EQ_U32 [[COPY]], [[COPY1]], implicit-def $scc
-  ; GCN:   [[COPY2:%[0-9]+]]:sgpr_32 = COPY $scc
+  ; GCN:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $scc
   ; GCN:   $scc = COPY [[COPY2]]
   ; GCN:   S_CBRANCH_SCC1 %bb.1, implicit $scc
   ; GCN: bb.1:
@@ -46,7 +46,7 @@ body: |
   ; GCN-LABEL: name: brcond_scc_impdef
   ; GCN: bb.0:
   ; GCN:   successors: %bb.1(0x80000000)
-  ; GCN:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GCN:   [[DEF:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
   ; GCN:   $scc = COPY [[DEF]]
   ; GCN:   S_CBRANCH_SCC1 %bb.1, implicit $scc
   ; GCN: bb.1:
@@ -73,7 +73,7 @@ body: |
   ; GCN:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
   ; GCN:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
   ; GCN:   S_CMP_EQ_U32 [[COPY]], [[COPY1]], implicit-def $scc
-  ; GCN:   [[COPY2:%[0-9]+]]:sgpr_32 = COPY $scc
+  ; GCN:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $scc
   ; GCN:   $scc = COPY [[COPY2]]
   ; GCN:   S_CBRANCH_SCC1 %bb.1, implicit $scc
   ; GCN:   S_BRANCH %bb.1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-phi.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-phi.mir
index b4ef0caebfc13..4e7c81f5c79ad 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-phi.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-phi.mir
@@ -17,7 +17,7 @@ body:             |
   ; GCN:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
   ; GCN:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
   ; GCN:   S_CMP_EQ_U32 [[COPY2]], [[S_MOV_B32_]], implicit-def $scc
-  ; GCN:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $scc
+  ; GCN:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $scc
   ; GCN:   $scc = COPY [[COPY3]]
   ; GCN:   S_CBRANCH_SCC1 %bb.1, implicit $scc
   ; GCN:   S_BRANCH %bb.2
@@ -66,7 +66,7 @@ body:             |
   ; GCN:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
   ; GCN:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
   ; GCN:   S_CMP_EQ_U32 [[COPY2]], [[S_MOV_B32_]], implicit-def $scc
-  ; GCN:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $scc
+  ; GCN:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $scc
   ; GCN:   $scc = COPY [[COPY3]]
   ; GCN:   S_CBRANCH_SCC1 %bb.1, implicit $scc
   ; GCN:   S_BRANCH %bb.2
@@ -116,7 +116,7 @@ body:             |
   ; GCN:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
   ; GCN:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
   ; GCN:   S_CMP_EQ_U32 [[COPY2]], [[S_MOV_B32_]], implicit-def $scc
-  ; GCN:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $scc
+  ; GCN:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $scc
   ; GCN:   $scc = COPY [[COPY3]]
   ; GCN:   S_CBRANCH_SCC1 %bb.1, implicit $scc
   ; GCN:   S_BRANCH %bb.2
@@ -165,7 +165,7 @@ body:             |
   ; GCN:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr1
   ; GCN:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
   ; GCN:   S_CMP_EQ_U32 [[COPY2]], [[S_MOV_B32_]], implicit-def $scc
-  ; GCN:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $scc
+  ; GCN:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $scc
   ; GCN:   $scc = COPY [[COPY3]]
   ; GCN:   S_CBRANCH_SCC1 %bb.1, implicit $scc
   ; GCN:   S_BRANCH %bb.2
@@ -215,7 +215,7 @@ body:             |
   ; GCN:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
   ; GCN:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
   ; GCN:   S_CMP_EQ_U32 [[COPY2]], [[S_MOV_B32_]], implicit-def $scc
-  ; GCN:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $scc
+  ; GCN:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $scc
   ; GCN:   $scc = COPY [[COPY3]]
   ; GCN:   S_CBRANCH_SCC1 %bb.1, implicit $scc
   ; GCN:   S_BRANCH %bb.2
@@ -263,7 +263,7 @@ body:             |
   ; GCN:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
   ; GCN:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
   ; GCN:   S_CMP_EQ_U32 [[COPY2]], [[S_MOV_B32_]], implicit-def $scc
-  ; GCN:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $scc
+  ; GCN:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $scc
   ; GCN:   $scc = COPY [[COPY3]]
   ; GCN:   S_CBRANCH_SCC1 %bb.1, implicit $scc
   ; GCN:   S_BRANCH %bb.2
@@ -314,7 +314,7 @@ body:             |
   ; GCN:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
   ; GCN:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_EQ_U32_e64 [[COPY]], [[S_MOV_B32_]], implicit $exec
   ; GCN:   S_CMP_EQ_U32 [[COPY2]], [[S_MOV_B32_]], implicit-def $scc
-  ; GCN:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $scc
+  ; GCN:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $scc
   ; GCN:   $scc = COPY [[COPY3]]
   ; GCN:   S_CBRANCH_SCC1 %bb.1, implicit $scc
   ; GCN:   S_BRANCH %bb.2
@@ -363,7 +363,7 @@ body:             |
   ; GCN:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
   ; GCN:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
   ; GCN:   S_CMP_EQ_U32 [[COPY2]], [[S_MOV_B32_]], implicit-def $scc
-  ; GCN:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $scc
+  ; GCN:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $scc
   ; GCN:   $scc = COPY [[COPY3]]
   ; GCN:   S_CBRANCH_SCC1 %bb.1, implicit $scc
   ; GCN:   S_BRANCH %bb.2
@@ -412,7 +412,7 @@ body:             |
   ; GCN:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
   ; GCN:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
   ; GCN:   S_CMP_EQ_U32 [[COPY2]], [[S_MOV_B32_]], implicit-def $scc
-  ; GCN:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $scc
+  ; GCN:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $scc
   ; GCN:   $scc = COPY [[COPY3]]
   ; GCN:   S_CBRANCH_SCC1 %bb.1, implicit $scc
   ; GCN:   S_BRANCH %bb.2

From 42a9f6c554e378f1c010375eca30f04296aa0052 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Tue, 28 Jul 2020 10:15:30 -0400
Subject: [PATCH 0913/1035] GlobalISel: Handle arbitrary FewerElementsVector
 for G_IMPLICIT_DEF

---
 .../CodeGen/GlobalISel/LegalizerHelper.cpp    | 24 +++++-------------
 .../AArch64/GlobalISel/legalize-freeze.mir    |  3 +--
 .../AArch64/GlobalISel/legalize-itofp.mir     |  6 ++---
 .../AArch64/GlobalISel/legalize-undef.mir     |  3 +--
 .../AMDGPU/GlobalISel/legalize-freeze.mir     | 22 ++++++++--------
 .../GlobalISel/legalize-implicit-def.mir      | 23 ++++++++++-------
 .../AMDGPU/GlobalISel/legalize-phi.mir        | 25 ++++++++-----------
 7 files changed, 45 insertions(+), 61 deletions(-)

diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index c11f91bc0a6ca..592f79aa7b712 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -3001,28 +3001,16 @@ Register LegalizerHelper::getVectorElementPointer(Register VecPtr, LLT VecTy,
 
 LegalizerHelper::LegalizeResult LegalizerHelper::fewerElementsVectorImplicitDef(
     MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy) {
-  SmallVector<Register, 2> DstRegs;
-
-  unsigned NarrowSize = NarrowTy.getSizeInBits();
   Register DstReg = MI.getOperand(0).getReg();
-  unsigned Size = MRI.getType(DstReg).getSizeInBits();
-  int NumParts = Size / NarrowSize;
-  // FIXME: Don't know how to handle the situation where the small vectors
-  // aren't all the same size yet.
-  if (Size % NarrowSize != 0)
-    return UnableToLegalize;
+  LLT DstTy = MRI.getType(DstReg);
+  LLT LCMTy = getLCMType(DstTy, NarrowTy);
 
-  for (int i = 0; i < NumParts; ++i) {
-    Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy);
-    MIRBuilder.buildUndef(TmpReg);
-    DstRegs.push_back(TmpReg);
-  }
+  unsigned NumParts = LCMTy.getSizeInBits() / NarrowTy.getSizeInBits();
 
-  if (NarrowTy.isVector())
-    MIRBuilder.buildConcatVectors(DstReg, DstRegs);
-  else
-    MIRBuilder.buildBuildVector(DstReg, DstRegs);
+  auto NewUndef = MIRBuilder.buildUndef(NarrowTy);
+  SmallVector<Register, 8> Parts(NumParts, NewUndef.getReg(0));
 
+  buildWidenedRemergeToDst(DstReg, LCMTy, Parts);
   MI.eraseFromParent();
   return Legalized;
 }
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-freeze.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-freeze.mir
index f96e2e65ce8f4..9417df066a46b 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-freeze.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-freeze.mir
@@ -39,9 +39,8 @@ body: |
 
     ; CHECK-LABEL: name: test_freeze_v4s64
     ; CHECK: [[DEF:%[0-9]+]]:_(<2 x s64>) = G_IMPLICIT_DEF
-    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY [[DEF]](<2 x s64>)
     ; CHECK: [[FREEZE:%[0-9]+]]:_(<2 x s64>) = G_FREEZE [[DEF]]
-    ; CHECK: [[FREEZE1:%[0-9]+]]:_(<2 x s64>) = G_FREEZE [[COPY]]
+    ; CHECK: [[FREEZE1:%[0-9]+]]:_(<2 x s64>) = G_FREEZE [[DEF]]
     ; CHECK: $q0 = COPY [[FREEZE]](<2 x s64>)
     ; CHECK: $q1 = COPY [[FREEZE1]](<2 x s64>)
     %undef:_(<4 x s64>) = G_IMPLICIT_DEF
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-itofp.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-itofp.mir
index e348c0e454b80..2af37dff5fd49 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-itofp.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-itofp.mir
@@ -249,8 +249,7 @@ body: |
     liveins: $q0
     ; CHECK-LABEL: name: test_uitofp_v2s64_v2i1
     ; CHECK: [[DEF:%[0-9]+]]:_(s1) = G_IMPLICIT_DEF
-    ; CHECK: [[COPY:%[0-9]+]]:_(s1) = COPY [[DEF]](s1)
-    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s1>) = G_BUILD_VECTOR [[DEF]](s1), [[COPY]](s1)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s1>) = G_BUILD_VECTOR [[DEF]](s1), [[DEF]](s1)
     ; CHECK: [[ZEXT:%[0-9]+]]:_(<2 x s64>) = G_ZEXT [[BUILD_VECTOR]](<2 x s1>)
     ; CHECK: [[UITOFP:%[0-9]+]]:_(<2 x s64>) = G_UITOFP [[ZEXT]](<2 x s64>)
     ; CHECK: $q0 = COPY [[UITOFP]](<2 x s64>)
@@ -266,8 +265,7 @@ body: |
     liveins: $q0
     ; CHECK-LABEL: name: test_sitofp_v2s64_v2i1
     ; CHECK: [[DEF:%[0-9]+]]:_(s1) = G_IMPLICIT_DEF
-    ; CHECK: [[COPY:%[0-9]+]]:_(s1) = COPY [[DEF]](s1)
-    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s1>) = G_BUILD_VECTOR [[DEF]](s1), [[COPY]](s1)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s1>) = G_BUILD_VECTOR [[DEF]](s1), [[DEF]](s1)
     ; CHECK: [[SEXT:%[0-9]+]]:_(<2 x s64>) = G_SEXT [[BUILD_VECTOR]](<2 x s1>)
     ; CHECK: [[SITOFP:%[0-9]+]]:_(<2 x s64>) = G_SITOFP [[SEXT]](<2 x s64>)
     ; CHECK: $q0 = COPY [[SITOFP]](<2 x s64>)
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-undef.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-undef.mir
index 984909d342072..228e3a44b0c50 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-undef.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-undef.mir
@@ -54,9 +54,8 @@ body: |
 
     ; CHECK-LABEL: name: test_implicit_def_v4s64
     ; CHECK: [[DEF:%[0-9]+]]:_(<2 x s64>) = G_IMPLICIT_DEF
-    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY [[DEF]](<2 x s64>)
     ; CHECK: $q0 = COPY [[DEF]](<2 x s64>)
-    ; CHECK: $q1 = COPY [[COPY]](<2 x s64>)
+    ; CHECK: $q1 = COPY [[DEF]](<2 x s64>)
     %0:_(<4 x s64>) = G_IMPLICIT_DEF
     %1:_(<2 x s64> ), %2:_(<2 x s64>) = G_UNMERGE_VALUES %0
     $q0 = COPY %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-freeze.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-freeze.mir
index b2a525da27515..8b7460d6a73a0 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-freeze.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-freeze.mir
@@ -422,19 +422,20 @@ body: |
   bb.0:
 
     ; CHECK-LABEL: name: test_freeze_v33s32
-    ; CHECK: [[DEF:%[0-9]+]]:_(<33 x s32>) = G_IMPLICIT_DEF
-    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32), [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32), [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32), [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32), [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32), [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32), [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32), [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32), [[UV20:%[0-9]+]]:_(s32), [[UV21:%[0-9]+]]:_(s32), [[UV22:%[0-9]+]]:_(s32), [[UV23:%[0-9]+]]:_(s32), [[UV24:%[0-9]+]]:_(s32), [[UV25:%[0-9]+]]:_(s32), [[UV26:%[0-9]+]]:_(s32), [[UV27:%[0-9]+]]:_(s32), [[UV28:%[0-9]+]]:_(s32), [[UV29:%[0-9]+]]:_(s32), [[UV30:%[0-9]+]]:_(s32), [[UV31:%[0-9]+]]:_(s32), [[UV32:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[DEF]](<33 x s32>)
+    ; CHECK: [[DEF:%[0-9]+]]:_(<16 x s32>) = G_IMPLICIT_DEF
+    ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<528 x s32>) = G_CONCAT_VECTORS [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>)
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(<33 x s32>) = G_EXTRACT [[CONCAT_VECTORS]](<528 x s32>), 0
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32), [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32), [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32), [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32), [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32), [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32), [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32), [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32), [[UV20:%[0-9]+]]:_(s32), [[UV21:%[0-9]+]]:_(s32), [[UV22:%[0-9]+]]:_(s32), [[UV23:%[0-9]+]]:_(s32), [[UV24:%[0-9]+]]:_(s32), [[UV25:%[0-9]+]]:_(s32), [[UV26:%[0-9]+]]:_(s32), [[UV27:%[0-9]+]]:_(s32), [[UV28:%[0-9]+]]:_(s32), [[UV29:%[0-9]+]]:_(s32), [[UV30:%[0-9]+]]:_(s32), [[UV31:%[0-9]+]]:_(s32), [[UV32:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[EXTRACT]](<33 x s32>)
     ; CHECK: [[DEF1:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
     ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<16 x s32>) = G_BUILD_VECTOR [[UV]](s32), [[UV1]](s32), [[UV2]](s32), [[UV3]](s32), [[UV4]](s32), [[UV5]](s32), [[UV6]](s32), [[UV7]](s32), [[UV8]](s32), [[UV9]](s32), [[UV10]](s32), [[UV11]](s32), [[UV12]](s32), [[UV13]](s32), [[UV14]](s32), [[UV15]](s32)
     ; CHECK: [[BUILD_VECTOR1:%[0-9]+]]:_(<16 x s32>) = G_BUILD_VECTOR [[UV16]](s32), [[UV17]](s32), [[UV18]](s32), [[UV19]](s32), [[UV20]](s32), [[UV21]](s32), [[UV22]](s32), [[UV23]](s32), [[UV24]](s32), [[UV25]](s32), [[UV26]](s32), [[UV27]](s32), [[UV28]](s32), [[UV29]](s32), [[UV30]](s32), [[UV31]](s32)
     ; CHECK: [[BUILD_VECTOR2:%[0-9]+]]:_(<16 x s32>) = G_BUILD_VECTOR [[UV32]](s32), [[DEF1]](s32), [[DEF1]](s32), [[DEF1]](s32), [[DEF1]](s32), [[DEF1]](s32), [[DEF1]](s32), [[DEF1]](s32), [[DEF1]](s32), [[DEF1]](s32), [[DEF1]](s32), [[DEF1]](s32), [[DEF1]](s32), [[DEF1]](s32), [[DEF1]](s32), [[DEF1]](s32)
-    ; CHECK: [[DEF2:%[0-9]+]]:_(<16 x s32>) = G_IMPLICIT_DEF
     ; CHECK: [[FREEZE:%[0-9]+]]:_(<16 x s32>) = G_FREEZE [[BUILD_VECTOR]]
     ; CHECK: [[FREEZE1:%[0-9]+]]:_(<16 x s32>) = G_FREEZE [[BUILD_VECTOR1]]
     ; CHECK: [[FREEZE2:%[0-9]+]]:_(<16 x s32>) = G_FREEZE [[BUILD_VECTOR2]]
-    ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<528 x s32>) = G_CONCAT_VECTORS [[FREEZE]](<16 x s32>), [[FREEZE1]](<16 x s32>), [[FREEZE2]](<16 x s32>), [[DEF2]](<16 x s32>), [[DEF2]](<16 x s32>), [[DEF2]](<16 x s32>), [[DEF2]](<16 x s32>), [[DEF2]](<16 x s32>), [[DEF2]](<16 x s32>), [[DEF2]](<16 x s32>), [[DEF2]](<16 x s32>), [[DEF2]](<16 x s32>), [[DEF2]](<16 x s32>), [[DEF2]](<16 x s32>), [[DEF2]](<16 x s32>), [[DEF2]](<16 x s32>), [[DEF2]](<16 x s32>), [[DEF2]](<16 x s32>), [[DEF2]](<16 x s32>), [[DEF2]](<16 x s32>), [[DEF2]](<16 x s32>), [[DEF2]](<16 x s32>), [[DEF2]](<16 x s32>), [[DEF2]](<16 x s32>), [[DEF2]](<16 x s32>), [[DEF2]](<16 x s32>), [[DEF2]](<16 x s32>), [[DEF2]](<16 x s32>), [[DEF2]](<16 x s32>), [[DEF2]](<16 x s32>), [[DEF2]](<16 x s32>), [[DEF2]](<16 x s32>), [[DEF2]](<16 x s32>)
-    ; CHECK: [[EXTRACT:%[0-9]+]]:_(<33 x s32>) = G_EXTRACT [[CONCAT_VECTORS]](<528 x s32>), 0
-    ; CHECK: S_NOP 0, implicit [[EXTRACT]](<33 x s32>)
+    ; CHECK: [[CONCAT_VECTORS1:%[0-9]+]]:_(<528 x s32>) = G_CONCAT_VECTORS [[FREEZE]](<16 x s32>), [[FREEZE1]](<16 x s32>), [[FREEZE2]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>)
+    ; CHECK: [[EXTRACT1:%[0-9]+]]:_(<33 x s32>) = G_EXTRACT [[CONCAT_VECTORS1]](<528 x s32>), 0
+    ; CHECK: S_NOP 0, implicit [[EXTRACT1]](<33 x s32>)
     %0:_(<33 x s32>) = G_IMPLICIT_DEF
     %1:_(<33 x s32>) = G_FREEZE %0
     S_NOP 0, implicit %1
@@ -447,13 +448,10 @@ body: |
 
     ; CHECK-LABEL: name: test_freeze_v64s32
     ; CHECK: [[DEF:%[0-9]+]]:_(<16 x s32>) = G_IMPLICIT_DEF
-    ; CHECK: [[COPY:%[0-9]+]]:_(<16 x s32>) = COPY [[DEF]](<16 x s32>)
-    ; CHECK: [[COPY1:%[0-9]+]]:_(<16 x s32>) = COPY [[DEF]](<16 x s32>)
-    ; CHECK: [[COPY2:%[0-9]+]]:_(<16 x s32>) = COPY [[DEF]](<16 x s32>)
     ; CHECK: [[FREEZE:%[0-9]+]]:_(<16 x s32>) = G_FREEZE [[DEF]]
-    ; CHECK: [[FREEZE1:%[0-9]+]]:_(<16 x s32>) = G_FREEZE [[COPY]]
-    ; CHECK: [[FREEZE2:%[0-9]+]]:_(<16 x s32>) = G_FREEZE [[COPY1]]
-    ; CHECK: [[FREEZE3:%[0-9]+]]:_(<16 x s32>) = G_FREEZE [[COPY2]]
+    ; CHECK: [[FREEZE1:%[0-9]+]]:_(<16 x s32>) = G_FREEZE [[DEF]]
+    ; CHECK: [[FREEZE2:%[0-9]+]]:_(<16 x s32>) = G_FREEZE [[DEF]]
+    ; CHECK: [[FREEZE3:%[0-9]+]]:_(<16 x s32>) = G_FREEZE [[DEF]]
     ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<64 x s32>) = G_CONCAT_VECTORS [[FREEZE]](<16 x s32>), [[FREEZE1]](<16 x s32>), [[FREEZE2]](<16 x s32>), [[FREEZE3]](<16 x s32>)
     ; CHECK: S_NOP 0, implicit [[CONCAT_VECTORS]](<64 x s32>)
     %0:_(<64 x s32>) = G_IMPLICIT_DEF
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-implicit-def.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-implicit-def.mir
index 993bb6a437561..5fe42c120db81 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-implicit-def.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-implicit-def.mir
@@ -327,14 +327,22 @@ body: |
 name: test_implicit_def_v33s32
 body: |
   bb.0:
+    liveins: $vgpr0_vgpr1
 
     ; CHECK-LABEL: name: test_implicit_def_v33s32
-    ; CHECK: [[DEF:%[0-9]+]]:_(<33 x s32>) = G_IMPLICIT_DEF
-    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32), [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32), [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32), [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32), [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32), [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32), [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32), [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32), [[UV20:%[0-9]+]]:_(s32), [[UV21:%[0-9]+]]:_(s32), [[UV22:%[0-9]+]]:_(s32), [[UV23:%[0-9]+]]:_(s32), [[UV24:%[0-9]+]]:_(s32), [[UV25:%[0-9]+]]:_(s32), [[UV26:%[0-9]+]]:_(s32), [[UV27:%[0-9]+]]:_(s32), [[UV28:%[0-9]+]]:_(s32), [[UV29:%[0-9]+]]:_(s32), [[UV30:%[0-9]+]]:_(s32), [[UV31:%[0-9]+]]:_(s32), [[UV32:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[DEF]](<33 x s32>)
-    ; CHECK: S_NOP 0, implicit [[UV]](s32), implicit [[UV32]](s32)
+    ; CHECK: [[DEF:%[0-9]+]]:_(<16 x s32>) = G_IMPLICIT_DEF
+    ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<528 x s32>) = G_CONCAT_VECTORS [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>)
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(<33 x s32>) = G_EXTRACT [[CONCAT_VECTORS]](<528 x s32>), 0
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32), [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32), [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32), [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32), [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32), [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32), [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32), [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32), [[UV20:%[0-9]+]]:_(s32), [[UV21:%[0-9]+]]:_(s32), [[UV22:%[0-9]+]]:_(s32), [[UV23:%[0-9]+]]:_(s32), [[UV24:%[0-9]+]]:_(s32), [[UV25:%[0-9]+]]:_(s32), [[UV26:%[0-9]+]]:_(s32), [[UV27:%[0-9]+]]:_(s32), [[UV28:%[0-9]+]]:_(s32), [[UV29:%[0-9]+]]:_(s32), [[UV30:%[0-9]+]]:_(s32), [[UV31:%[0-9]+]]:_(s32), [[UV32:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[EXTRACT]](<33 x s32>)
+    ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; CHECK: G_STORE [[UV]](s32), [[COPY]](p1) :: (volatile store 4, addrspace 1)
+    ; CHECK: G_STORE [[UV32]](s32), [[COPY]](p1) :: (volatile store 4, addrspace 1)
     %0:_(<33 x s32>) = G_IMPLICIT_DEF
     %1:_(s32), %2:_(s32), %3:_(s32), %4:_(s32), %5:_(s32), %6:_(s32), %7:_(s32), %8:_(s32), %9:_(s32), %10:_(s32), %11:_(s32), %12:_(s32), %13:_(s32), %14:_(s32), %15:_(s32), %16:_(s32), %17:_(s32), %18:_(s32), %19:_(s32), %20:_(s32), %21:_(s32), %22:_(s32), %23:_(s32), %24:_(s32), %25:_(s32), %26:_(s32), %27:_(s32), %28:_(s32), %29:_(s32), %30:_(s32), %31:_(s32), %32:_(s32), %33:_(s32) = G_UNMERGE_VALUES %0
-  S_NOP 0, implicit %1, implicit %33
+    %34:_(p1) = COPY $vgpr0_vgpr1
+    G_STORE %1, %34 :: (volatile store 4, align 4, addrspace 1)
+    G_STORE %33, %34 :: (volatile store 4, align 4, addrspace 1)
+
 ...
 
 ---
@@ -344,11 +352,8 @@ body: |
 
     ; CHECK-LABEL: name: test_implicit_def_v64s32
     ; CHECK: [[DEF:%[0-9]+]]:_(<16 x s32>) = G_IMPLICIT_DEF
-    ; CHECK: [[COPY:%[0-9]+]]:_(<16 x s32>) = COPY [[DEF]](<16 x s32>)
-    ; CHECK: [[COPY1:%[0-9]+]]:_(<16 x s32>) = COPY [[DEF]](<16 x s32>)
-    ; CHECK: [[COPY2:%[0-9]+]]:_(<16 x s32>) = COPY [[DEF]](<16 x s32>)
-    ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<64 x s32>) = G_CONCAT_VECTORS [[DEF]](<16 x s32>), [[COPY]](<16 x s32>), [[COPY1]](<16 x s32>), [[COPY2]](<16 x s32>)
-    ; CHECK: [[CONCAT_VECTORS1:%[0-9]+]]:_(<32 x s32>) = G_CONCAT_VECTORS [[DEF]](<16 x s32>), [[COPY]](<16 x s32>)
+    ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<64 x s32>) = G_CONCAT_VECTORS [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>), [[DEF]](<16 x s32>)
+    ; CHECK: [[CONCAT_VECTORS1:%[0-9]+]]:_(<32 x s32>) = G_CONCAT_VECTORS [[DEF]](<16 x s32>), [[DEF]](<16 x s32>)
     ; CHECK: S_NOP 0, implicit [[CONCAT_VECTORS]](<64 x s32>), implicit [[CONCAT_VECTORS1]](<32 x s32>)
     %0:_(<64 x s32>) = G_IMPLICIT_DEF
     %1:_(<32 x s32>), %2:_(<32 x s32>) = G_UNMERGE_VALUES %0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-phi.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-phi.mir
index 997dc3a521201..0b1f7be76a0fa 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-phi.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-phi.mir
@@ -652,24 +652,21 @@ body: |
   ; CHECK:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
   ; CHECK:   liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4
   ; CHECK:   [[DEF:%[0-9]+]]:_(<16 x s32>) = G_IMPLICIT_DEF
-  ; CHECK:   [[COPY:%[0-9]+]]:_(<16 x s32>) = COPY [[DEF]](<16 x s32>)
-  ; CHECK:   [[COPY1:%[0-9]+]]:_(<16 x s32>) = COPY [[DEF]](<16 x s32>)
-  ; CHECK:   [[COPY2:%[0-9]+]]:_(<16 x s32>) = COPY [[DEF]](<16 x s32>)
-  ; CHECK:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr4
   ; CHECK:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-  ; CHECK:   [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[COPY3]](s32), [[C]]
+  ; CHECK:   [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[COPY]](s32), [[C]]
   ; CHECK:   G_BRCOND [[ICMP]](s1), %bb.1
   ; CHECK:   G_BR %bb.2
   ; CHECK: bb.1:
   ; CHECK:   successors: %bb.2(0x80000000)
   ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32), [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32), [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32), [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32), [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32), [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[DEF]](<16 x s32>)
-  ; CHECK:   [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32), [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32), [[UV20:%[0-9]+]]:_(s32), [[UV21:%[0-9]+]]:_(s32), [[UV22:%[0-9]+]]:_(s32), [[UV23:%[0-9]+]]:_(s32), [[UV24:%[0-9]+]]:_(s32), [[UV25:%[0-9]+]]:_(s32), [[UV26:%[0-9]+]]:_(s32), [[UV27:%[0-9]+]]:_(s32), [[UV28:%[0-9]+]]:_(s32), [[UV29:%[0-9]+]]:_(s32), [[UV30:%[0-9]+]]:_(s32), [[UV31:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<16 x s32>)
-  ; CHECK:   [[UV32:%[0-9]+]]:_(s32), [[UV33:%[0-9]+]]:_(s32), [[UV34:%[0-9]+]]:_(s32), [[UV35:%[0-9]+]]:_(s32), [[UV36:%[0-9]+]]:_(s32), [[UV37:%[0-9]+]]:_(s32), [[UV38:%[0-9]+]]:_(s32), [[UV39:%[0-9]+]]:_(s32), [[UV40:%[0-9]+]]:_(s32), [[UV41:%[0-9]+]]:_(s32), [[UV42:%[0-9]+]]:_(s32), [[UV43:%[0-9]+]]:_(s32), [[UV44:%[0-9]+]]:_(s32), [[UV45:%[0-9]+]]:_(s32), [[UV46:%[0-9]+]]:_(s32), [[UV47:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<16 x s32>)
-  ; CHECK:   [[UV48:%[0-9]+]]:_(s32), [[UV49:%[0-9]+]]:_(s32), [[UV50:%[0-9]+]]:_(s32), [[UV51:%[0-9]+]]:_(s32), [[UV52:%[0-9]+]]:_(s32), [[UV53:%[0-9]+]]:_(s32), [[UV54:%[0-9]+]]:_(s32), [[UV55:%[0-9]+]]:_(s32), [[UV56:%[0-9]+]]:_(s32), [[UV57:%[0-9]+]]:_(s32), [[UV58:%[0-9]+]]:_(s32), [[UV59:%[0-9]+]]:_(s32), [[UV60:%[0-9]+]]:_(s32), [[UV61:%[0-9]+]]:_(s32), [[UV62:%[0-9]+]]:_(s32), [[UV63:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY2]](<16 x s32>)
+  ; CHECK:   [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32), [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32), [[UV20:%[0-9]+]]:_(s32), [[UV21:%[0-9]+]]:_(s32), [[UV22:%[0-9]+]]:_(s32), [[UV23:%[0-9]+]]:_(s32), [[UV24:%[0-9]+]]:_(s32), [[UV25:%[0-9]+]]:_(s32), [[UV26:%[0-9]+]]:_(s32), [[UV27:%[0-9]+]]:_(s32), [[UV28:%[0-9]+]]:_(s32), [[UV29:%[0-9]+]]:_(s32), [[UV30:%[0-9]+]]:_(s32), [[UV31:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[DEF]](<16 x s32>)
+  ; CHECK:   [[UV32:%[0-9]+]]:_(s32), [[UV33:%[0-9]+]]:_(s32), [[UV34:%[0-9]+]]:_(s32), [[UV35:%[0-9]+]]:_(s32), [[UV36:%[0-9]+]]:_(s32), [[UV37:%[0-9]+]]:_(s32), [[UV38:%[0-9]+]]:_(s32), [[UV39:%[0-9]+]]:_(s32), [[UV40:%[0-9]+]]:_(s32), [[UV41:%[0-9]+]]:_(s32), [[UV42:%[0-9]+]]:_(s32), [[UV43:%[0-9]+]]:_(s32), [[UV44:%[0-9]+]]:_(s32), [[UV45:%[0-9]+]]:_(s32), [[UV46:%[0-9]+]]:_(s32), [[UV47:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[DEF]](<16 x s32>)
+  ; CHECK:   [[UV48:%[0-9]+]]:_(s32), [[UV49:%[0-9]+]]:_(s32), [[UV50:%[0-9]+]]:_(s32), [[UV51:%[0-9]+]]:_(s32), [[UV52:%[0-9]+]]:_(s32), [[UV53:%[0-9]+]]:_(s32), [[UV54:%[0-9]+]]:_(s32), [[UV55:%[0-9]+]]:_(s32), [[UV56:%[0-9]+]]:_(s32), [[UV57:%[0-9]+]]:_(s32), [[UV58:%[0-9]+]]:_(s32), [[UV59:%[0-9]+]]:_(s32), [[UV60:%[0-9]+]]:_(s32), [[UV61:%[0-9]+]]:_(s32), [[UV62:%[0-9]+]]:_(s32), [[UV63:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[DEF]](<16 x s32>)
   ; CHECK:   [[UV64:%[0-9]+]]:_(s32), [[UV65:%[0-9]+]]:_(s32), [[UV66:%[0-9]+]]:_(s32), [[UV67:%[0-9]+]]:_(s32), [[UV68:%[0-9]+]]:_(s32), [[UV69:%[0-9]+]]:_(s32), [[UV70:%[0-9]+]]:_(s32), [[UV71:%[0-9]+]]:_(s32), [[UV72:%[0-9]+]]:_(s32), [[UV73:%[0-9]+]]:_(s32), [[UV74:%[0-9]+]]:_(s32), [[UV75:%[0-9]+]]:_(s32), [[UV76:%[0-9]+]]:_(s32), [[UV77:%[0-9]+]]:_(s32), [[UV78:%[0-9]+]]:_(s32), [[UV79:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[DEF]](<16 x s32>)
-  ; CHECK:   [[UV80:%[0-9]+]]:_(s32), [[UV81:%[0-9]+]]:_(s32), [[UV82:%[0-9]+]]:_(s32), [[UV83:%[0-9]+]]:_(s32), [[UV84:%[0-9]+]]:_(s32), [[UV85:%[0-9]+]]:_(s32), [[UV86:%[0-9]+]]:_(s32), [[UV87:%[0-9]+]]:_(s32), [[UV88:%[0-9]+]]:_(s32), [[UV89:%[0-9]+]]:_(s32), [[UV90:%[0-9]+]]:_(s32), [[UV91:%[0-9]+]]:_(s32), [[UV92:%[0-9]+]]:_(s32), [[UV93:%[0-9]+]]:_(s32), [[UV94:%[0-9]+]]:_(s32), [[UV95:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<16 x s32>)
-  ; CHECK:   [[UV96:%[0-9]+]]:_(s32), [[UV97:%[0-9]+]]:_(s32), [[UV98:%[0-9]+]]:_(s32), [[UV99:%[0-9]+]]:_(s32), [[UV100:%[0-9]+]]:_(s32), [[UV101:%[0-9]+]]:_(s32), [[UV102:%[0-9]+]]:_(s32), [[UV103:%[0-9]+]]:_(s32), [[UV104:%[0-9]+]]:_(s32), [[UV105:%[0-9]+]]:_(s32), [[UV106:%[0-9]+]]:_(s32), [[UV107:%[0-9]+]]:_(s32), [[UV108:%[0-9]+]]:_(s32), [[UV109:%[0-9]+]]:_(s32), [[UV110:%[0-9]+]]:_(s32), [[UV111:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<16 x s32>)
-  ; CHECK:   [[UV112:%[0-9]+]]:_(s32), [[UV113:%[0-9]+]]:_(s32), [[UV114:%[0-9]+]]:_(s32), [[UV115:%[0-9]+]]:_(s32), [[UV116:%[0-9]+]]:_(s32), [[UV117:%[0-9]+]]:_(s32), [[UV118:%[0-9]+]]:_(s32), [[UV119:%[0-9]+]]:_(s32), [[UV120:%[0-9]+]]:_(s32), [[UV121:%[0-9]+]]:_(s32), [[UV122:%[0-9]+]]:_(s32), [[UV123:%[0-9]+]]:_(s32), [[UV124:%[0-9]+]]:_(s32), [[UV125:%[0-9]+]]:_(s32), [[UV126:%[0-9]+]]:_(s32), [[UV127:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY2]](<16 x s32>)
+  ; CHECK:   [[UV80:%[0-9]+]]:_(s32), [[UV81:%[0-9]+]]:_(s32), [[UV82:%[0-9]+]]:_(s32), [[UV83:%[0-9]+]]:_(s32), [[UV84:%[0-9]+]]:_(s32), [[UV85:%[0-9]+]]:_(s32), [[UV86:%[0-9]+]]:_(s32), [[UV87:%[0-9]+]]:_(s32), [[UV88:%[0-9]+]]:_(s32), [[UV89:%[0-9]+]]:_(s32), [[UV90:%[0-9]+]]:_(s32), [[UV91:%[0-9]+]]:_(s32), [[UV92:%[0-9]+]]:_(s32), [[UV93:%[0-9]+]]:_(s32), [[UV94:%[0-9]+]]:_(s32), [[UV95:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[DEF]](<16 x s32>)
+  ; CHECK:   [[UV96:%[0-9]+]]:_(s32), [[UV97:%[0-9]+]]:_(s32), [[UV98:%[0-9]+]]:_(s32), [[UV99:%[0-9]+]]:_(s32), [[UV100:%[0-9]+]]:_(s32), [[UV101:%[0-9]+]]:_(s32), [[UV102:%[0-9]+]]:_(s32), [[UV103:%[0-9]+]]:_(s32), [[UV104:%[0-9]+]]:_(s32), [[UV105:%[0-9]+]]:_(s32), [[UV106:%[0-9]+]]:_(s32), [[UV107:%[0-9]+]]:_(s32), [[UV108:%[0-9]+]]:_(s32), [[UV109:%[0-9]+]]:_(s32), [[UV110:%[0-9]+]]:_(s32), [[UV111:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[DEF]](<16 x s32>)
+  ; CHECK:   [[UV112:%[0-9]+]]:_(s32), [[UV113:%[0-9]+]]:_(s32), [[UV114:%[0-9]+]]:_(s32), [[UV115:%[0-9]+]]:_(s32), [[UV116:%[0-9]+]]:_(s32), [[UV117:%[0-9]+]]:_(s32), [[UV118:%[0-9]+]]:_(s32), [[UV119:%[0-9]+]]:_(s32), [[UV120:%[0-9]+]]:_(s32), [[UV121:%[0-9]+]]:_(s32), [[UV122:%[0-9]+]]:_(s32), [[UV123:%[0-9]+]]:_(s32), [[UV124:%[0-9]+]]:_(s32), [[UV125:%[0-9]+]]:_(s32), [[UV126:%[0-9]+]]:_(s32), [[UV127:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[DEF]](<16 x s32>)
   ; CHECK:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[UV]], [[UV64]]
   ; CHECK:   [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[UV1]], [[UV65]]
   ; CHECK:   [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[UV2]], [[UV66]]
@@ -741,9 +738,9 @@ body: |
   ; CHECK:   G_BR %bb.2
   ; CHECK: bb.2:
   ; CHECK:   [[PHI:%[0-9]+]]:_(<16 x s32>) = G_PHI [[DEF]](<16 x s32>), %bb.0, [[BUILD_VECTOR]](<16 x s32>), %bb.1
-  ; CHECK:   [[PHI1:%[0-9]+]]:_(<16 x s32>) = G_PHI [[COPY]](<16 x s32>), %bb.0, [[BUILD_VECTOR1]](<16 x s32>), %bb.1
-  ; CHECK:   [[PHI2:%[0-9]+]]:_(<16 x s32>) = G_PHI [[COPY1]](<16 x s32>), %bb.0, [[BUILD_VECTOR2]](<16 x s32>), %bb.1
-  ; CHECK:   [[PHI3:%[0-9]+]]:_(<16 x s32>) = G_PHI [[COPY2]](<16 x s32>), %bb.0, [[BUILD_VECTOR3]](<16 x s32>), %bb.1
+  ; CHECK:   [[PHI1:%[0-9]+]]:_(<16 x s32>) = G_PHI [[DEF]](<16 x s32>), %bb.0, [[BUILD_VECTOR1]](<16 x s32>), %bb.1
+  ; CHECK:   [[PHI2:%[0-9]+]]:_(<16 x s32>) = G_PHI [[DEF]](<16 x s32>), %bb.0, [[BUILD_VECTOR2]](<16 x s32>), %bb.1
+  ; CHECK:   [[PHI3:%[0-9]+]]:_(<16 x s32>) = G_PHI [[DEF]](<16 x s32>), %bb.0, [[BUILD_VECTOR3]](<16 x s32>), %bb.1
   ; CHECK:   [[CONCAT_VECTORS:%[0-9]+]]:_(<64 x s32>) = G_CONCAT_VECTORS [[PHI]](<16 x s32>), [[PHI1]](<16 x s32>), [[PHI2]](<16 x s32>), [[PHI3]](<16 x s32>)
   ; CHECK:   S_SETPC_B64 undef $sgpr30_sgpr31, implicit [[CONCAT_VECTORS]](<64 x s32>)
   bb.0:

From 0c40af6b594f6eb2dcd43cdb2bc2f4584ec8ca15 Mon Sep 17 00:00:00 2001
From: Alex Zinenko <zinenko@google.com>
Date: Wed, 29 Jul 2020 14:58:09 +0200
Subject: [PATCH 0914/1035] [mlir] First-party modeling of LLVM types

The current modeling of LLVM IR types in MLIR is based on the LLVMType class
that wraps a raw `llvm::Type *` and delegates uniquing, printing and parsing to
LLVM itself. This model makes thread-safe type manipulation hard and is being
progressively replaced with a cleaner MLIR model that replicates the type
system.  Introduce a set of classes reflecting the LLVM IR type system in MLIR
instead of wrapping the existing types. These are currently introduced as
separate classes without affecting the dialect flow, and are exercised through
a test dialect. Once feature parity is reached, the old implementation will be
gradually substituted with the new one.

Depends On D84171

Reviewed By: rriddle

Differential Revision: https://reviews.llvm.org/D84339
---
 mlir/include/mlir/Dialect/LLVMIR/LLVMTypes.h  | 470 +++++++++++++++++
 mlir/include/mlir/IR/DialectImplementation.h  |   6 +
 mlir/lib/Dialect/LLVMIR/CMakeLists.txt        |   2 +
 mlir/lib/Dialect/LLVMIR/IR/LLVMTypeSyntax.cpp | 477 ++++++++++++++++++
 mlir/lib/Dialect/LLVMIR/IR/LLVMTypes.cpp      | 163 ++++++
 mlir/lib/Dialect/LLVMIR/IR/TypeDetail.h       | 458 +++++++++++++++++
 mlir/lib/Parser/DialectSymbolParser.cpp       |  15 +
 mlir/test/Dialect/LLVMIR/types-invalid.mlir   |  95 ++++
 mlir/test/Dialect/LLVMIR/types.mlir           | 184 +++++++
 mlir/test/lib/Dialect/CMakeLists.txt          |   1 +
 mlir/test/lib/Dialect/LLVMIR/CMakeLists.txt   |  14 +
 .../Dialect/LLVMIR/LLVMTypeTestDialect.cpp    |  52 ++
 mlir/tools/mlir-opt/CMakeLists.txt            |   1 +
 mlir/tools/mlir-opt/mlir-opt.cpp              |   8 +-
 14 files changed, 1943 insertions(+), 3 deletions(-)
 create mode 100644 mlir/include/mlir/Dialect/LLVMIR/LLVMTypes.h
 create mode 100644 mlir/lib/Dialect/LLVMIR/IR/LLVMTypeSyntax.cpp
 create mode 100644 mlir/lib/Dialect/LLVMIR/IR/LLVMTypes.cpp
 create mode 100644 mlir/lib/Dialect/LLVMIR/IR/TypeDetail.h
 create mode 100644 mlir/test/Dialect/LLVMIR/types-invalid.mlir
 create mode 100644 mlir/test/Dialect/LLVMIR/types.mlir
 create mode 100644 mlir/test/lib/Dialect/LLVMIR/CMakeLists.txt
 create mode 100644 mlir/test/lib/Dialect/LLVMIR/LLVMTypeTestDialect.cpp

diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMTypes.h b/mlir/include/mlir/Dialect/LLVMIR/LLVMTypes.h
new file mode 100644
index 0000000000000..6764f9815c3fb
--- /dev/null
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMTypes.h
@@ -0,0 +1,470 @@
+//===- LLVMDialect.h - MLIR LLVM dialect types ------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the types for the LLVM dialect in MLIR. These MLIR types
+// correspond to the LLVM IR type system.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_LLVMIR_LLVMTYPES_H_
+#define MLIR_DIALECT_LLVMIR_LLVMTYPES_H_
+
+#include "mlir/IR/Types.h"
+
+namespace llvm {
+class ElementCount;
+} // namespace llvm
+
+namespace mlir {
+
+class DialectAsmParser;
+class DialectAsmPrinter;
+
+namespace LLVM {
+namespace detail {
+struct LLVMFunctionTypeStorage;
+struct LLVMIntegerTypeStorage;
+struct LLVMPointerTypeStorage;
+struct LLVMStructTypeStorage;
+struct LLVMTypeAndSizeStorage;
+} // namespace detail
+
+//===----------------------------------------------------------------------===//
+// LLVMTypeNew.
+//===----------------------------------------------------------------------===//
+
+/// Base class for LLVM dialect types.
+///
+/// The LLVM dialect in MLIR fully reflects the LLVM IR type system, prodiving a
+/// sperate MLIR type for each LLVM IR type. All types are represted as separate
+/// subclasses and are compatible with the isa/cast infrastructure. For
+/// convenience, the base class provides most of the APIs available on
+/// llvm::Type in addition to MLIR-compatible APIs.
+///
+/// The LLVM dialect type system is closed: parametric types can only refer to
+/// other LLVM dialect types. This is consistent with LLVM IR and enables a more
+/// concise pretty-printing format.
+///
+/// Similarly to other MLIR types, LLVM dialect types are owned by the MLIR
+/// context, have an immutable identifier (for most types except identified
+/// structs, the entire type is the identifier) and are thread-safe.
+class LLVMTypeNew : public Type {
+public:
+  enum Kind {
+    // Keep non-parametric types contiguous in the enum.
+    VoidType = FIRST_LLVM_TYPE + 1,
+    HalfType,
+    BFloatType,
+    FloatType,
+    DoubleType,
+    FP128Type,
+    X86FP80Type,
+    PPCFP128Type,
+    X86MMXType,
+    LabelType,
+    TokenType,
+    MetadataType,
+    // End of non-parametric types.
+    FunctionType,
+    IntegerType,
+    PointerType,
+    FixedVectorType,
+    ScalableVectorType,
+    ArrayType,
+    StructType,
+    FIRST_NEW_LLVM_TYPE = VoidType,
+    LAST_NEW_LLVM_TYPE = StructType,
+    FIRST_TRIVIAL_TYPE = VoidType,
+    LAST_TRIVIAL_TYPE = MetadataType
+  };
+
+  /// Inherit base constructors.
+  using Type::Type;
+
+  /// Support for PointerLikeTypeTraits.
+  using Type::getAsOpaquePointer;
+  static LLVMTypeNew getFromOpaquePointer(const void *ptr) {
+    return LLVMTypeNew(static_cast<ImplType *>(const_cast<void *>(ptr)));
+  }
+
+  /// Support for isa/cast.
+  static bool kindof(unsigned kind) {
+    return FIRST_NEW_LLVM_TYPE <= kind && kind <= LAST_NEW_LLVM_TYPE;
+  }
+};
+
+//===----------------------------------------------------------------------===//
+// Trivial types.
+//===----------------------------------------------------------------------===//
+
+// Batch-define trivial types.
+#define DEFINE_TRIVIAL_LLVM_TYPE(ClassName, Kind)                              \
+  class ClassName                                                              \
+      : public Type::TypeBase<ClassName, LLVMTypeNew, TypeStorage> {           \
+  public:                                                                      \
+    using Base::Base;                                                          \
+    static bool kindof(unsigned kind) { return kind == Kind; }                 \
+    static ClassName get(MLIRContext *context) {                               \
+      return Base::get(context, Kind);                                         \
+    }                                                                          \
+  }
+
+DEFINE_TRIVIAL_LLVM_TYPE(LLVMVoidType, LLVMTypeNew::VoidType);
+DEFINE_TRIVIAL_LLVM_TYPE(LLVMHalfType, LLVMTypeNew::HalfType);
+DEFINE_TRIVIAL_LLVM_TYPE(LLVMBFloatType, LLVMTypeNew::BFloatType);
+DEFINE_TRIVIAL_LLVM_TYPE(LLVMFloatType, LLVMTypeNew::FloatType);
+DEFINE_TRIVIAL_LLVM_TYPE(LLVMDoubleType, LLVMTypeNew::DoubleType);
+DEFINE_TRIVIAL_LLVM_TYPE(LLVMFP128Type, LLVMTypeNew::FP128Type);
+DEFINE_TRIVIAL_LLVM_TYPE(LLVMX86FP80Type, LLVMTypeNew::X86FP80Type);
+DEFINE_TRIVIAL_LLVM_TYPE(LLVMPPCFP128Type, LLVMTypeNew::PPCFP128Type);
+DEFINE_TRIVIAL_LLVM_TYPE(LLVMX86MMXType, LLVMTypeNew::X86MMXType);
+DEFINE_TRIVIAL_LLVM_TYPE(LLVMTokenType, LLVMTypeNew::TokenType);
+DEFINE_TRIVIAL_LLVM_TYPE(LLVMLabelType, LLVMTypeNew::LabelType);
+DEFINE_TRIVIAL_LLVM_TYPE(LLVMMetadataType, LLVMTypeNew::MetadataType);
+
+#undef DEFINE_TRIVIAL_LLVM_TYPE
+
+//===----------------------------------------------------------------------===//
+// LLVMArrayType.
+//===----------------------------------------------------------------------===//
+
+/// LLVM dialect array type. It is an aggregate type representing consecutive
+/// elements in memory, parameterized by the number of elements and the element
+/// type.
+class LLVMArrayType : public Type::TypeBase<LLVMArrayType, LLVMTypeNew,
+                                            detail::LLVMTypeAndSizeStorage> {
+public:
+  /// Inherit base constructors.
+  using Base::Base;
+
+  /// Support for isa/cast.
+  static bool kindof(unsigned kind) { return kind == LLVMTypeNew::ArrayType; }
+
+  /// Gets or creates an instance of LLVM dialect array type containing
+  /// `numElements` of `elementType`, in the same context as `elementType`.
+  static LLVMArrayType get(LLVMTypeNew elementType, unsigned numElements);
+
+  /// Returns the element type of the array.
+  LLVMTypeNew getElementType();
+
+  /// Returns the number of elements in the array type.
+  unsigned getNumElements();
+};
+
+//===----------------------------------------------------------------------===//
+// LLVMFunctionType.
+//===----------------------------------------------------------------------===//
+
+/// LLVM dialect function type. It consists of a single return type (unlike MLIR
+/// which can have multiple), a list of parameter types and can optionally be
+/// variadic.
+class LLVMFunctionType
+    : public Type::TypeBase<LLVMFunctionType, LLVMTypeNew,
+                            detail::LLVMFunctionTypeStorage> {
+public:
+  /// Inherit base constructors.
+  using Base::Base;
+
+  /// Support for isa/cast.
+  static bool kindof(unsigned kind) {
+    return kind == LLVMTypeNew::FunctionType;
+  }
+
+  /// Gets or creates an instance of LLVM dialect function in the same context
+  /// as the `result` type.
+  static LLVMFunctionType get(LLVMTypeNew result,
+                              ArrayRef<LLVMTypeNew> arguments,
+                              bool isVarArg = false);
+
+  /// Returns the result type of the function.
+  LLVMTypeNew getReturnType();
+
+  /// Returns the number of arguments to the function.
+  unsigned getNumParams();
+
+  /// Returns `i`-th argument of the function. Asserts on out-of-bounds.
+  LLVMTypeNew getParamType(unsigned i);
+
+  /// Returns whether the function is variadic.
+  bool isVarArg();
+
+  /// Returns a list of argument types of the function.
+  ArrayRef<LLVMTypeNew> getParams();
+  ArrayRef<LLVMTypeNew> params() { return getParams(); }
+};
+
+//===----------------------------------------------------------------------===//
+// LLVMIntegerType.
+//===----------------------------------------------------------------------===//
+
+/// LLVM dialect signless integer type parameterized by bitwidth.
+class LLVMIntegerType : public Type::TypeBase<LLVMIntegerType, LLVMTypeNew,
+                                              detail::LLVMIntegerTypeStorage> {
+public:
+  /// Inherit base constructor.
+  using Base::Base;
+
+  /// Support for isa/cast.
+  static bool kindof(unsigned kind) { return kind == LLVMTypeNew::IntegerType; }
+
+  /// Gets or creates an instance of the integer of the specified `bitwidth` in
+  /// the given context.
+  static LLVMIntegerType get(MLIRContext *ctx, unsigned bitwidth);
+
+  /// Returns the bitwidth of this integer type.
+  unsigned getBitWidth();
+};
+
+//===----------------------------------------------------------------------===//
+// LLVMPointerType.
+//===----------------------------------------------------------------------===//
+
+/// LLVM dialect pointer type. This type typically represents a reference to an
+/// object in memory. It is parameterized by the element type and the address
+/// space.
+class LLVMPointerType : public Type::TypeBase<LLVMPointerType, LLVMTypeNew,
+                                              detail::LLVMPointerTypeStorage> {
+public:
+  /// Inherit base constructors.
+  using Base::Base;
+
+  /// Support for isa/cast.
+  static bool kindof(unsigned kind) { return kind == LLVMTypeNew::PointerType; }
+
+  /// Gets or creates an instance of LLVM dialect pointer type pointing to an
+  /// object of `pointee` type in the given address space. The pointer type is
+  /// created in the same context as `pointee`.
+  static LLVMPointerType get(LLVMTypeNew pointee, unsigned addressSpace = 0);
+
+  /// Returns the pointed-to type.
+  LLVMTypeNew getElementType();
+
+  /// Returns the address space of the pointer.
+  unsigned getAddressSpace();
+};
+
+//===----------------------------------------------------------------------===//
+// LLVMStructType.
+//===----------------------------------------------------------------------===//
+
+/// LLVM dialect structure type representing a collection of different-typed
+/// elements manipulated together. Structured can optionally be packed, meaning
+/// that their elements immediately follow each other in memory without
+/// accounting for potential alignment.
+///
+/// Structure types can be identified (named) or literal. Literal structures
+/// are uniquely represented by the list of types they contain and packedness.
+/// Literal structure types are immutable after construction.
+///
+/// Identified structures are uniquely represented by their name, a string. They
+/// have a mutable component, consisting of the list of types they contain,
+/// the packedness and the opacity bits. Identified structs can be created
+/// without providing the lists of element types, making them suitable to
+/// represent recursive, i.e. self-referring, structures. Identified structs
+/// without body are considered opaque. For such structs, one can set the body.
+/// Identified structs can be created as intentionally-opaque, implying that the
+/// caller does not intend to ever set the body (e.g. forward-declarations of
+/// structs from another module) and wants to disallow further modification of
+/// the body. For intentionally-opaque structs or non-opaque structs with the
+/// body, one is not allowed to set another body (however, one can set exactly
+/// the same body).
+///
+/// Note that the packedness of the struct takes place in uniquing of literal
+/// structs, but does not in uniquing of identified structs.
+class LLVMStructType : public Type::TypeBase<LLVMStructType, LLVMTypeNew,
+                                             detail::LLVMStructTypeStorage> {
+public:
+  /// Inherit base construtors.
+  using Base::Base;
+
+  /// Support for isa/cast.
+  static bool kindof(unsigned kind) { return kind == LLVMTypeNew::StructType; }
+
+  /// Gets or creates an identified struct with the given name in the provided
+  /// context. Note that unlike llvm::StructType::create, this function will
+  /// _NOT_ rename a struct in case a struct with the same name already exists
+  /// in the context. Instead, it will just return the existing struct,
+  /// similarly to the rest of MLIR type ::get methods.
+  static LLVMStructType getIdentified(MLIRContext *context, StringRef name);
+
+  /// Gets or creates a literal struct with the given body in the provided
+  /// context.
+  static LLVMStructType getLiteral(MLIRContext *context,
+                                   ArrayRef<LLVMTypeNew> types,
+                                   bool isPacked = false);
+
+  /// Gets or creates an intentionally-opaque identified struct. Such a struct
+  /// cannot have its body set. To create an opaque struct with a mutable body,
+  /// use `getIdentified`. Note that unlike llvm::StructType::create, this
+  /// function will _NOT_ rename a struct in case a struct with the same name
+  /// already exists in the context. Instead, it will just return the existing
+  /// struct, similarly to the rest of MLIR type ::get methods.
+  static LLVMStructType getOpaque(StringRef name, MLIRContext *context);
+
+  /// Set the body of an identified struct. Returns failure if the body could
+  /// not be set, e.g. if the struct already has a body or if it was marked as
+  /// intentionally opaque. This might happen in a multi-threaded context when a
+  /// different thread modified the struct after it was created. Most callers
+  /// are likely to assert this always succeeds, but it is possible to implement
+  /// a local renaming scheme based on the result of this call.
+  LogicalResult setBody(ArrayRef<LLVMTypeNew> types, bool isPacked);
+
+  /// Checks if a struct is packed.
+  bool isPacked();
+
+  /// Checks if a struct is identified.
+  bool isIdentified();
+
+  /// Checks if a struct is opaque.
+  bool isOpaque();
+
+  /// Returns the name of an identified struct.
+  StringRef getName();
+
+  /// Returns the list of element types contained in a non-opaque struct.
+  ArrayRef<LLVMTypeNew> getBody();
+};
+
+//===----------------------------------------------------------------------===//
+// LLVMVectorType.
+//===----------------------------------------------------------------------===//
+
+/// LLVM dialect vector type, represents a sequence of elements that can be
+/// processed as one, typically in SIMD context. This is a base class for fixed
+/// and scalable vectors.
+class LLVMVectorType : public LLVMTypeNew {
+public:
+  /// Inherit base constructor.
+  using LLVMTypeNew::LLVMTypeNew;
+
+  /// Support for isa/cast.
+  static bool kindof(unsigned kind) {
+    return kind == LLVMTypeNew::FixedVectorType ||
+           kind == LLVMTypeNew::ScalableVectorType;
+  }
+
+  /// Returns the element type of the vector.
+  LLVMTypeNew getElementType();
+
+  /// Returns the number of elements in the vector.
+  llvm::ElementCount getElementCount();
+};
+
+//===----------------------------------------------------------------------===//
+// LLVMFixedVectorType.
+//===----------------------------------------------------------------------===//
+
+/// LLVM dialect fixed vector type, represents a sequence of elements of known
+/// length that can be processed as one.
+class LLVMFixedVectorType
+    : public Type::TypeBase<LLVMFixedVectorType, LLVMVectorType,
+                            detail::LLVMTypeAndSizeStorage> {
+public:
+  /// Inherit base constructor.
+  using Base::Base;
+
+  /// Support for isa/cast.
+  static bool kindof(unsigned kind) {
+    return kind == LLVMTypeNew::FixedVectorType;
+  }
+
+  /// Gets or creates a fixed vector type containing `numElements` of
+  /// `elementType` in the same context as `elementType`.
+  static LLVMFixedVectorType get(LLVMTypeNew elementType, unsigned numElements);
+
+  /// Returns the number of elements in the fixed vector.
+  unsigned getNumElements();
+};
+
+//===----------------------------------------------------------------------===//
+// LLVMScalableVectorType.
+//===----------------------------------------------------------------------===//
+
+/// LLVM dialect scalable vector type, represents a sequence of elements of
+/// unknown length that is known to be divisible by some constant. These
+/// elements can be processed as one in SIMD context.
+class LLVMScalableVectorType
+    : public Type::TypeBase<LLVMScalableVectorType, LLVMVectorType,
+                            detail::LLVMTypeAndSizeStorage> {
+public:
+  /// Inherit base constructor.
+  using Base::Base;
+
+  /// Support for isa/cast.
+  static bool kindof(unsigned kind) {
+    return kind == LLVMTypeNew::ScalableVectorType;
+  }
+
+  /// Gets or creates a scalable vector type containing a non-zero multiple of
+  /// `minNumElements` of `elementType` in the same context as `elementType`.
+  static LLVMScalableVectorType get(LLVMTypeNew elementType,
+                                    unsigned minNumElements);
+
+  /// Returns the scaling factor of the number of elements in the vector. The
+  /// vector contains at least the resulting number of elements, or any non-zero
+  /// multiple of this number.
+  unsigned getMinNumElements();
+};
+
+//===----------------------------------------------------------------------===//
+// Printing and parsing.
+//===----------------------------------------------------------------------===//
+
+namespace detail {
+/// Parses an LLVM dialect type.
+LLVMTypeNew parseType(DialectAsmParser &parser);
+
+/// Prints an LLVM Dialect type.
+void printType(LLVMTypeNew type, DialectAsmPrinter &printer);
+} // namespace detail
+
+} // namespace LLVM
+} // namespace mlir
+
+//===----------------------------------------------------------------------===//
+// Support for hashing and containers.
+//===----------------------------------------------------------------------===//
+
+namespace llvm {
+
+// LLVMTypeNew instances hash just like pointers.
+template <> struct DenseMapInfo<mlir::LLVM::LLVMTypeNew> {
+  static mlir::LLVM::LLVMTypeNew getEmptyKey() {
+    void *pointer = llvm::DenseMapInfo<void *>::getEmptyKey();
+    return mlir::LLVM::LLVMTypeNew(
+        static_cast<mlir::LLVM::LLVMTypeNew::ImplType *>(pointer));
+  }
+  static mlir::LLVM::LLVMTypeNew getTombstoneKey() {
+    void *pointer = llvm::DenseMapInfo<void *>::getTombstoneKey();
+    return mlir::LLVM::LLVMTypeNew(
+        static_cast<mlir::LLVM::LLVMTypeNew::ImplType *>(pointer));
+  }
+  static unsigned getHashValue(mlir::LLVM::LLVMTypeNew val) {
+    return mlir::hash_value(val);
+  }
+  static bool isEqual(mlir::LLVM::LLVMTypeNew lhs,
+                      mlir::LLVM::LLVMTypeNew rhs) {
+    return lhs == rhs;
+  }
+};
+
+// LLVMTypeNew behaves like a pointer similarly to mlir::Type.
+template <> struct PointerLikeTypeTraits<mlir::LLVM::LLVMTypeNew> {
+  static inline void *getAsVoidPointer(mlir::LLVM::LLVMTypeNew type) {
+    return const_cast<void *>(type.getAsOpaquePointer());
+  }
+  static inline mlir::LLVM::LLVMTypeNew getFromVoidPointer(void *ptr) {
+    return mlir::LLVM::LLVMTypeNew::getFromOpaquePointer(ptr);
+  }
+  static constexpr int NumLowBitsAvailable =
+      PointerLikeTypeTraits<mlir::Type>::NumLowBitsAvailable;
+};
+
+} // namespace llvm
+
+#endif // MLIR_DIALECT_LLVMIR_LLVMTYPES_H_
diff --git a/mlir/include/mlir/IR/DialectImplementation.h b/mlir/include/mlir/IR/DialectImplementation.h
index e2d7e2c409c45..c478b200b5d91 100644
--- a/mlir/include/mlir/IR/DialectImplementation.h
+++ b/mlir/include/mlir/IR/DialectImplementation.h
@@ -203,6 +203,9 @@ class DialectAsmParser {
   /// Parse a `=` token if present.
   virtual ParseResult parseOptionalEqual() = 0;
 
+  /// Parse a quoted string token if present.
+  virtual ParseResult parseOptionalString(StringRef *string) = 0;
+
   /// Parse a given keyword.
   ParseResult parseKeyword(StringRef keyword, const Twine &msg = "") {
     auto loc = getCurrentLocation();
@@ -323,6 +326,9 @@ class DialectAsmParser {
     return success();
   }
 
+  /// Parse a type if present.
+  virtual OptionalParseResult parseOptionalType(Type &result) = 0;
+
   /// Parse a 'x' separated dimension list. This populates the dimension list,
   /// using -1 for the `?` dimensions if `allowDynamic` is set and errors out on
   /// `?` otherwise.
diff --git a/mlir/lib/Dialect/LLVMIR/CMakeLists.txt b/mlir/lib/Dialect/LLVMIR/CMakeLists.txt
index e858a0a70c730..ff6560305cb80 100644
--- a/mlir/lib/Dialect/LLVMIR/CMakeLists.txt
+++ b/mlir/lib/Dialect/LLVMIR/CMakeLists.txt
@@ -2,6 +2,8 @@ add_subdirectory(Transforms)
 
 add_mlir_dialect_library(MLIRLLVMIR
   IR/LLVMDialect.cpp
+  IR/LLVMTypes.cpp
+  IR/LLVMTypeSyntax.cpp
 
   ADDITIONAL_HEADER_DIRS
   ${MLIR_MAIN_INCLUDE_DIR}/mlir/Dialect/LLVMIR
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMTypeSyntax.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMTypeSyntax.cpp
new file mode 100644
index 0000000000000..d272297525c1e
--- /dev/null
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMTypeSyntax.cpp
@@ -0,0 +1,477 @@
+//===- LLVMTypeSyntax.cpp - Parsing/printing for MLIR LLVM Dialect types --===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/LLVMIR/LLVMTypes.h"
+#include "mlir/IR/Builders.h"
+#include "mlir/IR/DialectImplementation.h"
+#include "llvm/ADT/SetVector.h"
+
+using namespace mlir;
+using namespace mlir::LLVM;
+
+//===----------------------------------------------------------------------===//
+// Printing.
+//===----------------------------------------------------------------------===//
+
+static void printTypeImpl(llvm::raw_ostream &os, LLVMTypeNew type,
+                          llvm::SetVector<StringRef> &stack);
+
+/// Returns the keyword to use for the given type.
+static StringRef getTypeKeyword(LLVMTypeNew type) {
+  switch (type.getKind()) {
+  case LLVMTypeNew::VoidType:
+    return "void";
+  case LLVMTypeNew::HalfType:
+    return "half";
+  case LLVMTypeNew::BFloatType:
+    return "bfloat";
+  case LLVMTypeNew::FloatType:
+    return "float";
+  case LLVMTypeNew::DoubleType:
+    return "double";
+  case LLVMTypeNew::FP128Type:
+    return "fp128";
+  case LLVMTypeNew::X86FP80Type:
+    return "x86_fp80";
+  case LLVMTypeNew::PPCFP128Type:
+    return "ppc_fp128";
+  case LLVMTypeNew::X86MMXType:
+    return "x86_mmx";
+  case LLVMTypeNew::TokenType:
+    return "token";
+  case LLVMTypeNew::LabelType:
+    return "label";
+  case LLVMTypeNew::MetadataType:
+    return "metadata";
+  case LLVMTypeNew::FunctionType:
+    return "func";
+  case LLVMTypeNew::IntegerType:
+    return "i";
+  case LLVMTypeNew::PointerType:
+    return "ptr";
+  case LLVMTypeNew::FixedVectorType:
+  case LLVMTypeNew::ScalableVectorType:
+    return "vec";
+  case LLVMTypeNew::ArrayType:
+    return "array";
+  case LLVMTypeNew::StructType:
+    return "struct";
+  }
+  llvm_unreachable("unhandled type kind");
+}
+
+/// Prints the body of a structure type. Uses `stack` to avoid printing
+/// recursive structs indefinitely.
+static void printStructTypeBody(llvm::raw_ostream &os, LLVMStructType type,
+                                llvm::SetVector<StringRef> &stack) {
+  if (type.isIdentified() && type.isOpaque()) {
+    os << "opaque";
+    return;
+  }
+
+  if (type.isPacked())
+    os << "packed ";
+
+  // Put the current type on stack to avoid infinite recursion.
+  os << '(';
+  if (type.isIdentified())
+    stack.insert(type.getName());
+  llvm::interleaveComma(type.getBody(), os, [&](LLVMTypeNew subtype) {
+    printTypeImpl(os, subtype, stack);
+  });
+  if (type.isIdentified())
+    stack.pop_back();
+  os << ')';
+}
+
+/// Prints a structure type. Uses `stack` to keep track of the identifiers of
+/// the structs being printed. Checks if the identifier of a struct is contained
+/// in `stack`, i.e. whether a self-reference to a recursive stack is being
+/// printed, and only prints the name to avoid infinite recursion.
+static void printStructType(llvm::raw_ostream &os, LLVMStructType type,
+                            llvm::SetVector<StringRef> &stack) {
+  os << "<";
+  if (type.isIdentified()) {
+    os << '"' << type.getName() << '"';
+    // If we are printing a reference to one of the enclosing structs, just
+    // print the name and stop to avoid infinitely long output.
+    if (stack.count(type.getName())) {
+      os << '>';
+      return;
+    }
+    os << ", ";
+  }
+
+  printStructTypeBody(os, type, stack);
+  os << '>';
+}
+
+/// Prints a type containing a fixed number of elements.
+template <typename TypeTy>
+static void printArrayOrVectorType(llvm::raw_ostream &os, TypeTy type,
+                                   llvm::SetVector<StringRef> &stack) {
+  os << '<' << type.getNumElements() << " x ";
+  printTypeImpl(os, type.getElementType(), stack);
+  os << '>';
+}
+
+/// Prints a function type.
+static void printFunctionType(llvm::raw_ostream &os, LLVMFunctionType funcType,
+                              llvm::SetVector<StringRef> &stack) {
+  os << '<';
+  printTypeImpl(os, funcType.getReturnType(), stack);
+  os << " (";
+  llvm::interleaveComma(funcType.getParams(), os,
+                        [&os, &stack](LLVMTypeNew subtype) {
+                          printTypeImpl(os, subtype, stack);
+                        });
+  if (funcType.isVarArg()) {
+    if (funcType.getNumParams() != 0)
+      os << ", ";
+    os << "...";
+  }
+  os << ")>";
+}
+
+/// Prints the given LLVM dialect type recursively. This leverages closedness of
+/// the LLVM dialect type system to avoid printing the dialect prefix
+/// repeatedly. For recursive structures, only prints the name of the structure
+/// when printing a self-reference. Note that this does not apply to sibling
+/// references. For example,
+///   struct<"a", (ptr<struct<"a">>)>
+///   struct<"c", (ptr<struct<"b", (ptr<struct<"c">>)>>,
+///                ptr<struct<"b", (ptr<struct<"c">>)>>)>
+/// note that "b" is printed twice.
+static void printTypeImpl(llvm::raw_ostream &os, LLVMTypeNew type,
+                          llvm::SetVector<StringRef> &stack) {
+  if (!type) {
+    os << "<<NULL-TYPE>>";
+    return;
+  }
+
+  unsigned kind = type.getKind();
+  os << getTypeKeyword(type);
+
+  // Trivial types only consist of their keyword.
+  if (LLVMTypeNew::FIRST_TRIVIAL_TYPE <= kind &&
+      kind <= LLVMTypeNew::LAST_TRIVIAL_TYPE)
+    return;
+
+  if (auto intType = type.dyn_cast<LLVMIntegerType>()) {
+    os << intType.getBitWidth();
+    return;
+  }
+
+  if (auto ptrType = type.dyn_cast<LLVMPointerType>()) {
+    os << '<';
+    printTypeImpl(os, ptrType.getElementType(), stack);
+    if (ptrType.getAddressSpace() != 0)
+      os << ", " << ptrType.getAddressSpace();
+    os << '>';
+    return;
+  }
+
+  if (auto arrayType = type.dyn_cast<LLVMArrayType>())
+    return printArrayOrVectorType(os, arrayType, stack);
+  if (auto vectorType = type.dyn_cast<LLVMFixedVectorType>())
+    return printArrayOrVectorType(os, vectorType, stack);
+
+  if (auto vectorType = type.dyn_cast<LLVMScalableVectorType>()) {
+    os << "<? x " << vectorType.getMinNumElements() << " x ";
+    printTypeImpl(os, vectorType.getElementType(), stack);
+    os << '>';
+    return;
+  }
+
+  if (auto structType = type.dyn_cast<LLVMStructType>())
+    return printStructType(os, structType, stack);
+
+  printFunctionType(os, type.cast<LLVMFunctionType>(), stack);
+}
+
+void mlir::LLVM::detail::printType(LLVMTypeNew type,
+                                   DialectAsmPrinter &printer) {
+  llvm::SetVector<StringRef> stack;
+  return printTypeImpl(printer.getStream(), type, stack);
+}
+
+//===----------------------------------------------------------------------===//
+// Parsing.
+//===----------------------------------------------------------------------===//
+
+static LLVMTypeNew parseTypeImpl(DialectAsmParser &parser,
+                                 llvm::SetVector<StringRef> &stack);
+
+/// Helper to be chained with other parsing functions.
+static ParseResult parseTypeImpl(DialectAsmParser &parser,
+                                 llvm::SetVector<StringRef> &stack,
+                                 LLVMTypeNew &result) {
+  result = parseTypeImpl(parser, stack);
+  return success(result != nullptr);
+}
+
+/// Parses an LLVM dialect function type.
+///   llvm-type :: = `func<` llvm-type `(` llvm-type-list `...`? `)>`
+static LLVMFunctionType parseFunctionType(DialectAsmParser &parser,
+                                          llvm::SetVector<StringRef> &stack) {
+  LLVMTypeNew returnType;
+  if (parser.parseLess() || parseTypeImpl(parser, stack, returnType) ||
+      parser.parseLParen())
+    return LLVMFunctionType();
+
+  // Function type without arguments.
+  if (succeeded(parser.parseOptionalRParen())) {
+    if (succeeded(parser.parseGreater()))
+      return LLVMFunctionType::get(returnType, {}, /*isVarArg=*/false);
+    return LLVMFunctionType();
+  }
+
+  // Parse arguments.
+  SmallVector<LLVMTypeNew, 8> argTypes;
+  do {
+    if (succeeded(parser.parseOptionalEllipsis())) {
+      if (parser.parseOptionalRParen() || parser.parseOptionalGreater())
+        return LLVMFunctionType();
+      return LLVMFunctionType::get(returnType, argTypes, /*isVarArg=*/true);
+    }
+
+    argTypes.push_back(parseTypeImpl(parser, stack));
+    if (!argTypes.back())
+      return LLVMFunctionType();
+  } while (succeeded(parser.parseOptionalComma()));
+
+  if (parser.parseOptionalRParen() || parser.parseOptionalGreater())
+    return LLVMFunctionType();
+  return LLVMFunctionType::get(returnType, argTypes, /*isVarArg=*/false);
+}
+
+/// Parses an LLVM dialect pointer type.
+///   llvm-type ::= `ptr<` llvm-type (`,` integer)? `>`
+static LLVMPointerType parsePointerType(DialectAsmParser &parser,
+                                        llvm::SetVector<StringRef> &stack) {
+  LLVMTypeNew elementType;
+  if (parser.parseLess() || parseTypeImpl(parser, stack, elementType))
+    return LLVMPointerType();
+
+  unsigned addressSpace = 0;
+  if (succeeded(parser.parseOptionalComma()) &&
+      failed(parser.parseInteger(addressSpace)))
+    return LLVMPointerType();
+  if (failed(parser.parseGreater()))
+    return LLVMPointerType();
+  return LLVMPointerType::get(elementType, addressSpace);
+}
+
+/// Parses an LLVM dialect vector type.
+///   llvm-type ::= `vec<` `? x`? integer `x` llvm-type `>`
+/// Supports both fixed and scalable vectors.
+static LLVMVectorType parseVectorType(DialectAsmParser &parser,
+                                      llvm::SetVector<StringRef> &stack) {
+  SmallVector<int64_t, 2> dims;
+  llvm::SMLoc dimPos;
+  LLVMTypeNew elementType;
+  if (parser.parseLess() || parser.getCurrentLocation(&dimPos) ||
+      parser.parseDimensionList(dims, /*allowDynamic=*/true) ||
+      parseTypeImpl(parser, stack, elementType) || parser.parseGreater())
+    return LLVMVectorType();
+
+  // We parsed a generic dimension list, but vectors only support two forms:
+  //  - single non-dynamic entry in the list (fixed vector);
+  //  - two elements, the first dynamic (indicated by -1) and the second
+  //    non-dynamic (scalable vector).
+  if (dims.empty() || dims.size() > 2 ||
+      ((dims.size() == 2) ^ (dims[0] == -1)) ||
+      (dims.size() == 2 && dims[1] == -1)) {
+    parser.emitError(dimPos)
+        << "expected '? x <integer> x <type>' or '<integer> x <type>'";
+    return LLVMVectorType();
+  }
+
+  bool isScalable = dims.size() == 2;
+  return isScalable ? static_cast<LLVMVectorType>(
+                          LLVMScalableVectorType::get(elementType, dims[1]))
+                    : LLVMFixedVectorType::get(elementType, dims[0]);
+}
+
+/// Parses an LLVM dialect array type.
+///   llvm-type ::= `array<` integer `x` llvm-type `>`
+static LLVMArrayType parseArrayType(DialectAsmParser &parser,
+                                    llvm::SetVector<StringRef> &stack) {
+  SmallVector<int64_t, 1> dims;
+  llvm::SMLoc sizePos;
+  LLVMTypeNew elementType;
+  if (parser.parseLess() || parser.getCurrentLocation(&sizePos) ||
+      parser.parseDimensionList(dims, /*allowDynamic=*/false) ||
+      parseTypeImpl(parser, stack, elementType) || parser.parseGreater())
+    return LLVMArrayType();
+
+  if (dims.size() != 1) {
+    parser.emitError(sizePos) << "expected ? x <type>";
+    return LLVMArrayType();
+  }
+
+  return LLVMArrayType::get(elementType, dims[0]);
+}
+
+/// Attempts to set the body of an identified structure type. Reports a parsing
+/// error at `subtypesLoc` in case of failure, uses `stack` to make sure the
+/// types printed in the error message look like they did when parsed.
+static LLVMStructType trySetStructBody(LLVMStructType type,
+                                       ArrayRef<LLVMTypeNew> subtypes,
+                                       bool isPacked, DialectAsmParser &parser,
+                                       llvm::SMLoc subtypesLoc,
+                                       llvm::SetVector<StringRef> &stack) {
+  if (succeeded(type.setBody(subtypes, isPacked)))
+    return type;
+
+  std::string currentBody;
+  llvm::raw_string_ostream currentBodyStream(currentBody);
+  printStructTypeBody(currentBodyStream, type, stack);
+  auto diag = parser.emitError(subtypesLoc)
+              << "identified type already used with a different body";
+  diag.attachNote() << "existing body: " << currentBodyStream.str();
+  return LLVMStructType();
+}
+
+/// Parses an LLVM dialect structure type.
+///   llvm-type ::= `struct<` (string-literal `,`)? `packed`?
+///                 `(` llvm-type-list `)` `>`
+///               | `struct<` string-literal `>`
+///               | `struct<` string-literal `, opaque>`
+static LLVMStructType parseStructType(DialectAsmParser &parser,
+                                      llvm::SetVector<StringRef> &stack) {
+  MLIRContext *ctx = parser.getBuilder().getContext();
+
+  if (failed(parser.parseLess()))
+    return LLVMStructType();
+
+  // If we are parsing a self-reference to a recursive struct, i.e. the parsing
+  // stack already contains a struct with the same identifier, bail out after
+  // the name.
+  StringRef name;
+  bool isIdentified = succeeded(parser.parseOptionalString(&name));
+  if (isIdentified) {
+    if (stack.count(name)) {
+      if (failed(parser.parseGreater()))
+        return LLVMStructType();
+      return LLVMStructType::getIdentified(ctx, name);
+    }
+    if (failed(parser.parseComma()))
+      return LLVMStructType();
+  }
+
+  // Handle intentionally opaque structs.
+  llvm::SMLoc kwLoc = parser.getCurrentLocation();
+  if (succeeded(parser.parseOptionalKeyword("opaque"))) {
+    if (!isIdentified)
+      return parser.emitError(kwLoc, "only identified structs can be opaque"),
+             LLVMStructType();
+    if (failed(parser.parseGreater()))
+      return LLVMStructType();
+    auto type = LLVMStructType::getOpaque(name, ctx);
+    if (!type.isOpaque()) {
+      parser.emitError(kwLoc, "redeclaring defined struct as opaque");
+      return LLVMStructType();
+    }
+    return type;
+  }
+
+  // Check for packedness.
+  bool isPacked = succeeded(parser.parseOptionalKeyword("packed"));
+  if (failed(parser.parseLParen()))
+    return LLVMStructType();
+
+  // Fast pass for structs with zero subtypes.
+  if (succeeded(parser.parseOptionalRParen())) {
+    if (failed(parser.parseGreater()))
+      return LLVMStructType();
+    if (!isIdentified)
+      return LLVMStructType::getLiteral(ctx, {}, isPacked);
+    auto type = LLVMStructType::getIdentified(ctx, name);
+    return trySetStructBody(type, {}, isPacked, parser, kwLoc, stack);
+  }
+
+  // Parse subtypes. For identified structs, put the identifier of the struct on
+  // the stack to support self-references in the recursive calls.
+  SmallVector<LLVMTypeNew, 4> subtypes;
+  llvm::SMLoc subtypesLoc = parser.getCurrentLocation();
+  do {
+    if (isIdentified)
+      stack.insert(name);
+    LLVMTypeNew type = parseTypeImpl(parser, stack);
+    if (!type)
+      return LLVMStructType();
+    subtypes.push_back(type);
+    if (isIdentified)
+      stack.pop_back();
+  } while (succeeded(parser.parseOptionalComma()));
+
+  if (parser.parseRParen() || parser.parseGreater())
+    return LLVMStructType();
+
+  // Construct the struct with body.
+  if (!isIdentified)
+    return LLVMStructType::getLiteral(ctx, subtypes, isPacked);
+  auto type = LLVMStructType::getIdentified(ctx, name);
+  return trySetStructBody(type, subtypes, isPacked, parser, subtypesLoc, stack);
+}
+
+/// Parses one of the LLVM dialect types.
+static LLVMTypeNew parseTypeImpl(DialectAsmParser &parser,
+                                 llvm::SetVector<StringRef> &stack) {
+  // Special case for integers (i[1-9][0-9]*) that are literals rather than
+  // keywords for the parser, so they are not caught by the main dispatch below.
+  // Try parsing it a built-in integer type instead.
+  Type maybeIntegerType;
+  MLIRContext *ctx = parser.getBuilder().getContext();
+  llvm::SMLoc keyLoc = parser.getCurrentLocation();
+  OptionalParseResult result = parser.parseOptionalType(maybeIntegerType);
+  if (result.hasValue()) {
+    if (failed(*result))
+      return LLVMTypeNew();
+
+    if (!maybeIntegerType.isSignlessInteger()) {
+      parser.emitError(keyLoc) << "unexpected type, expected i* or keyword";
+      return LLVMTypeNew();
+    }
+    return LLVMIntegerType::get(ctx, maybeIntegerType.getIntOrFloatBitWidth());
+  }
+
+  // Dispatch to concrete functions.
+  StringRef key;
+  if (failed(parser.parseKeyword(&key)))
+    return LLVMTypeNew();
+
+  return llvm::StringSwitch<function_ref<LLVMTypeNew()>>(key)
+      .Case("void", [&] { return LLVMVoidType::get(ctx); })
+      .Case("half", [&] { return LLVMHalfType::get(ctx); })
+      .Case("bfloat", [&] { return LLVMBFloatType::get(ctx); })
+      .Case("float", [&] { return LLVMFloatType::get(ctx); })
+      .Case("double", [&] { return LLVMDoubleType::get(ctx); })
+      .Case("fp128", [&] { return LLVMFP128Type::get(ctx); })
+      .Case("x86_fp80", [&] { return LLVMX86FP80Type::get(ctx); })
+      .Case("ppc_fp128", [&] { return LLVMPPCFP128Type::get(ctx); })
+      .Case("x86_mmx", [&] { return LLVMX86MMXType::get(ctx); })
+      .Case("token", [&] { return LLVMTokenType::get(ctx); })
+      .Case("label", [&] { return LLVMLabelType::get(ctx); })
+      .Case("metadata", [&] { return LLVMMetadataType::get(ctx); })
+      .Case("func", [&] { return parseFunctionType(parser, stack); })
+      .Case("ptr", [&] { return parsePointerType(parser, stack); })
+      .Case("vec", [&] { return parseVectorType(parser, stack); })
+      .Case("array", [&] { return parseArrayType(parser, stack); })
+      .Case("struct", [&] { return parseStructType(parser, stack); })
+      .Default([&] {
+        parser.emitError(keyLoc) << "unknown LLVM type: " << key;
+        return LLVMTypeNew();
+      })();
+}
+
+LLVMTypeNew mlir::LLVM::detail::parseType(DialectAsmParser &parser) {
+  llvm::SetVector<StringRef> stack;
+  return parseTypeImpl(parser, stack);
+}
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMTypes.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMTypes.cpp
new file mode 100644
index 0000000000000..3540091e90a3e
--- /dev/null
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMTypes.cpp
@@ -0,0 +1,163 @@
+//===- LLVMTypes.cpp - MLIR LLVM Dialect types ----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the types for the LLVM dialect in MLIR. These MLIR types
+// correspond to the LLVM IR type system.
+//
+//===----------------------------------------------------------------------===//
+
+#include "TypeDetail.h"
+
+#include "mlir/Dialect/LLVMIR/LLVMTypes.h"
+#include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/TypeSupport.h"
+
+#include "llvm/Support/TypeSize.h"
+
+using namespace mlir;
+using namespace mlir::LLVM;
+
+//===----------------------------------------------------------------------===//
+// Array type.
+
+LLVMArrayType LLVMArrayType::get(LLVMTypeNew elementType,
+                                 unsigned numElements) {
+  assert(elementType && "expected non-null subtype");
+  return Base::get(elementType.getContext(), LLVMTypeNew::ArrayType,
+                   elementType, numElements);
+}
+
+LLVMTypeNew LLVMArrayType::getElementType() { return getImpl()->elementType; }
+
+unsigned LLVMArrayType::getNumElements() { return getImpl()->numElements; }
+
+//===----------------------------------------------------------------------===//
+// Function type.
+
+LLVMFunctionType LLVMFunctionType::get(LLVMTypeNew result,
+                                       ArrayRef<LLVMTypeNew> arguments,
+                                       bool isVarArg) {
+  assert(result && "expected non-null result");
+  return Base::get(result.getContext(), LLVMTypeNew::FunctionType, result,
+                   arguments, isVarArg);
+}
+
+LLVMTypeNew LLVMFunctionType::getReturnType() {
+  return getImpl()->getReturnType();
+}
+
+unsigned LLVMFunctionType::getNumParams() {
+  return getImpl()->getArgumentTypes().size();
+}
+
+LLVMTypeNew LLVMFunctionType::getParamType(unsigned i) {
+  return getImpl()->getArgumentTypes()[i];
+}
+
+bool LLVMFunctionType::isVarArg() { return getImpl()->isVariadic(); }
+
+ArrayRef<LLVMTypeNew> LLVMFunctionType::getParams() {
+  return getImpl()->getArgumentTypes();
+}
+
+//===----------------------------------------------------------------------===//
+// Integer type.
+
+LLVMIntegerType LLVMIntegerType::get(MLIRContext *ctx, unsigned bitwidth) {
+  return Base::get(ctx, LLVMTypeNew::IntegerType, bitwidth);
+}
+
+unsigned LLVMIntegerType::getBitWidth() { return getImpl()->bitwidth; }
+
+//===----------------------------------------------------------------------===//
+// Pointer type.
+
+LLVMPointerType LLVMPointerType::get(LLVMTypeNew pointee,
+                                     unsigned addressSpace) {
+  assert(pointee && "expected non-null subtype");
+  return Base::get(pointee.getContext(), LLVMTypeNew::PointerType, pointee,
+                   addressSpace);
+}
+
+LLVMTypeNew LLVMPointerType::getElementType() { return getImpl()->pointeeType; }
+
+unsigned LLVMPointerType::getAddressSpace() { return getImpl()->addressSpace; }
+
+//===----------------------------------------------------------------------===//
+// Struct type.
+
+LLVMStructType LLVMStructType::getIdentified(MLIRContext *context,
+                                             StringRef name) {
+  return Base::get(context, LLVMTypeNew::StructType, name, /*opaque=*/false);
+}
+
+LLVMStructType LLVMStructType::getLiteral(MLIRContext *context,
+                                          ArrayRef<LLVMTypeNew> types,
+                                          bool isPacked) {
+  return Base::get(context, LLVMTypeNew::StructType, types, isPacked);
+}
+
+LLVMStructType LLVMStructType::getOpaque(StringRef name, MLIRContext *context) {
+  return Base::get(context, LLVMTypeNew::StructType, name, /*opaque=*/true);
+}
+
+LogicalResult LLVMStructType::setBody(ArrayRef<LLVMTypeNew> types,
+                                      bool isPacked) {
+  assert(isIdentified() && "can only set bodies of identified structs");
+  return Base::mutate(types, isPacked);
+}
+
+bool LLVMStructType::isPacked() { return getImpl()->isPacked(); }
+bool LLVMStructType::isIdentified() { return getImpl()->isIdentified(); }
+bool LLVMStructType::isOpaque() {
+  return getImpl()->isOpaque() || !getImpl()->isInitialized();
+}
+StringRef LLVMStructType::getName() { return getImpl()->getIdentifier(); }
+ArrayRef<LLVMTypeNew> LLVMStructType::getBody() {
+  return isIdentified() ? getImpl()->getIdentifiedStructBody()
+                        : getImpl()->getTypeList();
+}
+
+//===----------------------------------------------------------------------===//
+// Vector types.
+
+LLVMTypeNew LLVMVectorType::getElementType() {
+  // Both derived classes share the implementation type.
+  return static_cast<detail::LLVMTypeAndSizeStorage *>(impl)->elementType;
+}
+
+llvm::ElementCount LLVMVectorType::getElementCount() {
+  // Both derived classes share the implementation type.
+  return llvm::ElementCount(
+      static_cast<detail::LLVMTypeAndSizeStorage *>(impl)->numElements,
+      this->isa<LLVMScalableVectorType>());
+}
+
+LLVMFixedVectorType LLVMFixedVectorType::get(LLVMTypeNew elementType,
+                                             unsigned numElements) {
+  assert(elementType && "expected non-null subtype");
+  return Base::get(elementType.getContext(), LLVMTypeNew::FixedVectorType,
+                   elementType, numElements)
+      .cast<LLVMFixedVectorType>();
+}
+
+unsigned LLVMFixedVectorType::getNumElements() {
+  return getImpl()->numElements;
+}
+
+LLVMScalableVectorType LLVMScalableVectorType::get(LLVMTypeNew elementType,
+                                                   unsigned minNumElements) {
+  assert(elementType && "expected non-null subtype");
+  return Base::get(elementType.getContext(), LLVMTypeNew::ScalableVectorType,
+                   elementType, minNumElements)
+      .cast<LLVMScalableVectorType>();
+}
+
+unsigned LLVMScalableVectorType::getMinNumElements() {
+  return getImpl()->numElements;
+}
diff --git a/mlir/lib/Dialect/LLVMIR/IR/TypeDetail.h b/mlir/lib/Dialect/LLVMIR/IR/TypeDetail.h
new file mode 100644
index 0000000000000..2b72e43e51648
--- /dev/null
+++ b/mlir/lib/Dialect/LLVMIR/IR/TypeDetail.h
@@ -0,0 +1,458 @@
+//===- TypeDetail.h - Details of MLIR LLVM dialect types --------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains implementation details, such as storage structures, of
+// MLIR LLVM dialect types.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef DIALECT_LLVMIR_IR_TYPEDETAIL_H
+#define DIALECT_LLVMIR_IR_TYPEDETAIL_H
+
+#include "mlir/Dialect/LLVMIR/LLVMTypes.h"
+#include "mlir/IR/TypeSupport.h"
+#include "mlir/IR/Types.h"
+
+#include "llvm/ADT/Bitfields.h"
+#include "llvm/ADT/PointerIntPair.h"
+
+namespace mlir {
+namespace LLVM {
+namespace detail {
+
+//===----------------------------------------------------------------------===//
+// LLVMStructTypeStorage.
+//===----------------------------------------------------------------------===//
+
+/// Type storage for LLVM structure types.
+///
+/// Structures are uniqued using:
+/// - a bit indicating whether a struct is literal or identified;
+/// - for identified structs, in addition to the bit:
+///   - a string identifier;
+/// - for literal structs, in addition to the bit:
+///   - a list of contained types;
+///   - a bit indicating whether the literal struct is packed.
+///
+/// Identified structures only have a mutable component consisting of:
+///   - a list of contained types;
+///   - a bit indicating whether the identified struct is packed;
+///   - a bit indicating whether the identified struct is intentionally opaque;
+///   - a bit indicating whether the identified struct has been initialized.
+/// Uninitialized structs are considered opaque by the user, and can be mutated.
+/// Initialized and still opaque structs cannot be mutated.
+///
+/// The struct storage consists of:
+///   - immutable part:
+///     - a pointer to the first element of the key (character for identified
+///       structs, type for literal structs);
+///     - the number of elements in the key packed together with bits indicating
+///       whether a type is literal or identified, and the packedness bit for
+///       literal structs only;
+///   - mutable part:
+///     - a pointer to the first contained type for identified structs only;
+///     - the number of contained types packed together with bits of the mutable
+///       component, for identified structs only.
+struct LLVMStructTypeStorage : public TypeStorage {
+public:
+  /// Construction/uniquing key class for LLVM dialect structure storage. Note
+  /// that this is a transient helper data structure that is NOT stored.
+  /// Therefore, it intentionally avoids bit manipulation and type erasure in
+  /// pointers to make manipulation more straightforward. Not all elements of
+  /// the key participate in uniquing, but all elements participate in
+  /// construction.
+  class Key {
+  public:
+    /// Constructs a key for an identified struct.
+    Key(StringRef name, bool opaque)
+        : name(name), identified(true), packed(false), opaque(opaque) {}
+    /// Constructs a key for a literal struct.
+    Key(ArrayRef<LLVMTypeNew> types, bool packed)
+        : types(types), identified(false), packed(packed), opaque(false) {}
+
+    /// Checks a specific property of the struct.
+    bool isIdentified() const { return identified; }
+    bool isPacked() const {
+      assert(!isIdentified() &&
+             "'packed' bit is not part of the key for identified stucts");
+      return packed;
+    }
+    bool isOpaque() const {
+      assert(isIdentified() &&
+             "'opaque' bit is meaningless on literal structs");
+      return opaque;
+    }
+
+    /// Returns the identifier of a key for identified structs.
+    StringRef getIdentifier() const {
+      assert(isIdentified() &&
+             "non-identified struct key canont have an identifier");
+      return name;
+    }
+
+    /// Returns the list of type contained in the key of a literal struct.
+    ArrayRef<LLVMTypeNew> getTypeList() const {
+      assert(!isIdentified() &&
+             "identified struct key cannot have a type list");
+      return types;
+    }
+
+    /// Returns the hash value of the key. This combines various flags into a
+    /// single value: the identified flag sets the first bit, and the packedness
+    /// flag sets the second bit. Opacity bit is only used for construction and
+    /// does not participate in uniquing.
+    llvm::hash_code hashValue() const {
+      constexpr static unsigned kIdentifiedHashFlag = 1;
+      constexpr static unsigned kPackedHashFlag = 2;
+
+      unsigned flags = 0;
+      if (isIdentified()) {
+        flags |= kIdentifiedHashFlag;
+        return llvm::hash_combine(flags, getIdentifier());
+      }
+      if (isPacked())
+        flags |= kPackedHashFlag;
+      return llvm::hash_combine(flags, getTypeList());
+    }
+
+    /// Compares two keys.
+    bool operator==(const Key &other) const {
+      if (isIdentified())
+        return other.isIdentified() &&
+               other.getIdentifier().equals(getIdentifier());
+
+      return !other.isIdentified() && other.isPacked() == isPacked() &&
+             other.getTypeList() == getTypeList();
+    }
+
+    /// Copies dynamically-sized components of the key into the given allocator.
+    Key copyIntoAllocator(TypeStorageAllocator &allocator) const {
+      if (isIdentified())
+        return Key(allocator.copyInto(name), opaque);
+      return Key(allocator.copyInto(types), packed);
+    }
+
+  private:
+    ArrayRef<LLVMTypeNew> types;
+    StringRef name;
+    bool identified;
+    bool packed;
+    bool opaque;
+  };
+  using KeyTy = Key;
+
+  /// Returns the string identifier of an identified struct.
+  StringRef getIdentifier() const {
+    assert(isIdentified() && "requested identifier on a non-identified struct");
+    return StringRef(static_cast<const char *>(keyPtr), keySize());
+  }
+
+  /// Returns the list of types (partially) identifying a literal struct.
+  ArrayRef<LLVMTypeNew> getTypeList() const {
+    // If this triggers, use getIdentifiedStructBody() instead.
+    assert(!isIdentified() && "requested typelist on an identified struct");
+    return ArrayRef<LLVMTypeNew>(static_cast<const LLVMTypeNew *>(keyPtr),
+                                 keySize());
+  }
+
+  /// Returns the list of types contained in an identified struct.
+  ArrayRef<LLVMTypeNew> getIdentifiedStructBody() const {
+    // If this triggers, use getTypeList() instead.
+    assert(isIdentified() &&
+           "requested struct body on a non-identified struct");
+    return ArrayRef<LLVMTypeNew>(identifiedBodyArray, identifiedBodySize());
+  }
+
+  /// Checks whether the struct is identified.
+  bool isIdentified() const {
+    return llvm::Bitfield::get<KeyFlagIdentified>(keySizeAndFlags);
+  }
+
+  /// Checks whether the struct is packed (both literal and identified structs).
+  bool isPacked() const {
+    return isIdentified() ? llvm::Bitfield::get<MutableFlagPacked>(
+                                identifiedBodySizeAndFlags)
+                          : llvm::Bitfield::get<KeyFlagPacked>(keySizeAndFlags);
+  }
+
+  /// Checks whether a struct is marked as intentionally opaque (an
+  /// uninitialized struct is also considered opaque by the user, call
+  /// isInitialized to check that).
+  bool isOpaque() const {
+    return llvm::Bitfield::get<MutableFlagOpaque>(identifiedBodySizeAndFlags);
+  }
+
+  /// Checks whether an identified struct has been explicitly initialized either
+  /// by setting its body or by marking it as intentionally opaque.
+  bool isInitialized() const {
+    return llvm::Bitfield::get<MutableFlagInitialized>(
+        identifiedBodySizeAndFlags);
+  }
+
+  /// Constructs the storage from the given key. This sets up the uniquing key
+  /// components and optionally the mutable component if they construction key
+  /// has the relevant information. In the latter case, the struct is considered
+  /// as initalized and can no longer be mutated.
+  LLVMStructTypeStorage(const KeyTy &key) {
+    if (!key.isIdentified()) {
+      ArrayRef<LLVMTypeNew> types = key.getTypeList();
+      keyPtr = static_cast<const void *>(types.data());
+      setKeySize(types.size());
+      llvm::Bitfield::set<KeyFlagPacked>(keySizeAndFlags, key.isPacked());
+      return;
+    }
+
+    StringRef name = key.getIdentifier();
+    keyPtr = static_cast<const void *>(name.data());
+    setKeySize(name.size());
+    llvm::Bitfield::set<KeyFlagIdentified>(keySizeAndFlags, true);
+
+    // If the struct is being constructed directly as opaque, mark it as
+    // initialized.
+    llvm::Bitfield::set<MutableFlagInitialized>(identifiedBodySizeAndFlags,
+                                                key.isOpaque());
+    llvm::Bitfield::set<MutableFlagOpaque>(identifiedBodySizeAndFlags,
+                                           key.isOpaque());
+  }
+
+  /// Hook into the type unquing infrastructure.
+  bool operator==(const KeyTy &other) const { return getKey() == other; };
+  static llvm::hash_code hashKey(const KeyTy &key) { return key.hashValue(); }
+  static LLVMStructTypeStorage *construct(TypeStorageAllocator &allocator,
+                                          const KeyTy &key) {
+    return new (allocator.allocate<LLVMStructTypeStorage>())
+        LLVMStructTypeStorage(key.copyIntoAllocator(allocator));
+  }
+
+  /// Sets the body of an identified struct. If the struct is already
+  /// initialized, succeeds only if the body is equal to the current body. Fails
+  /// if the struct is marked as intentionally opaque. The struct will be marked
+  /// as initialized as a result of this operation and can no longer be changed.
+  LogicalResult mutate(TypeStorageAllocator &allocator,
+                       ArrayRef<LLVMTypeNew> body, bool packed) {
+    if (!isIdentified())
+      return failure();
+    if (isInitialized())
+      return success(!isOpaque() && body == getIdentifiedStructBody() &&
+                     packed == isPacked());
+
+    llvm::Bitfield::set<MutableFlagInitialized>(identifiedBodySizeAndFlags,
+                                                true);
+    llvm::Bitfield::set<MutableFlagPacked>(identifiedBodySizeAndFlags, packed);
+
+    ArrayRef<LLVMTypeNew> typesInAllocator = allocator.copyInto(body);
+    identifiedBodyArray = typesInAllocator.data();
+    setIdentifiedBodySize(typesInAllocator.size());
+
+    return success();
+  }
+
+private:
+  /// Returns the number of elements in the key.
+  unsigned keySize() const {
+    return llvm::Bitfield::get<KeySize>(keySizeAndFlags);
+  }
+
+  /// Sets the number of elements in the key.
+  void setKeySize(unsigned value) {
+    llvm::Bitfield::set<KeySize>(keySizeAndFlags, value);
+  }
+
+  /// Returns the number of types contained in an identified struct.
+  unsigned identifiedBodySize() const {
+    return llvm::Bitfield::get<MutableSize>(identifiedBodySizeAndFlags);
+  }
+  /// Sets the number of types contained in an identified struct.
+  void setIdentifiedBodySize(unsigned value) {
+    llvm::Bitfield::set<MutableSize>(identifiedBodySizeAndFlags, value);
+  }
+
+  /// Returns the key for the current storage.
+  Key getKey() const {
+    if (isIdentified())
+      return Key(getIdentifier(), isOpaque());
+    return Key(getTypeList(), isPacked());
+  }
+
+  /// Bitfield elements for `keyAndSizeFlags`:
+  ///   - bit 0: identified key flag;
+  ///   - bit 1: packed key flag;
+  ///   - bits 2..bitwidth(unsigned): size of the key.
+  using KeyFlagIdentified =
+      llvm::Bitfield::Element<bool, /*Offset=*/0, /*Size=*/1>;
+  using KeyFlagPacked = llvm::Bitfield::Element<bool, /*Offset=*/1, /*Size=*/1>;
+  using KeySize =
+      llvm::Bitfield::Element<unsigned, /*Offset=*/2,
+                              std::numeric_limits<unsigned>::digits - 2>;
+
+  /// Bitfield elements for `identifiedBodySizeAndFlags`:
+  ///   - bit 0: opaque flag;
+  ///   - bit 1: packed mutable flag;
+  ///   - bit 2: initialized flag;
+  ///   - bits 3..bitwidth(unsigned): size of the identified body.
+  using MutableFlagOpaque =
+      llvm::Bitfield::Element<bool, /*Offset=*/0, /*Size=*/1>;
+  using MutableFlagPacked =
+      llvm::Bitfield::Element<bool, /*Offset=*/1, /*Size=*/1>;
+  using MutableFlagInitialized =
+      llvm::Bitfield::Element<bool, /*Offset=*/2, /*Size=*/1>;
+  using MutableSize =
+      llvm::Bitfield::Element<unsigned, /*Offset=*/3,
+                              std::numeric_limits<unsigned>::digits - 3>;
+
+  /// Pointer to the first element of the uniquing key.
+  // Note: cannot use PointerUnion because bump-ptr allocator does not guarantee
+  // address alignment.
+  const void *keyPtr = nullptr;
+
+  /// Pointer to the first type contained in an identified struct.
+  const LLVMTypeNew *identifiedBodyArray = nullptr;
+
+  /// Size of the uniquing key combined with identified/literal and
+  /// packedness bits. Must only be used through the Key* bitfields.
+  unsigned keySizeAndFlags = 0;
+
+  /// Number of the types contained in an identified struct combined with
+  /// mutable flags. Must only be used through the Mutable* bitfields.
+  unsigned identifiedBodySizeAndFlags = 0;
+};
+
+//===----------------------------------------------------------------------===//
+// LLVMFunctionTypeStorage.
+//===----------------------------------------------------------------------===//
+
+/// Type storage for LLVM dialect function types. These are uniqued using the
+/// list of types they contain and the vararg bit.
+struct LLVMFunctionTypeStorage : public TypeStorage {
+  using KeyTy = std::tuple<LLVMTypeNew, ArrayRef<LLVMTypeNew>, bool>;
+
+  /// Construct a storage from the given components. The list is expected to be
+  /// allocated in the context.
+  LLVMFunctionTypeStorage(LLVMTypeNew result, ArrayRef<LLVMTypeNew> arguments,
+                          bool variadic)
+      : argumentTypes(arguments) {
+    returnTypeAndVariadic.setPointerAndInt(result, variadic);
+  }
+
+  /// Hook into the type uniquing infrastructure.
+  static LLVMFunctionTypeStorage *construct(TypeStorageAllocator &allocator,
+                                            const KeyTy &key) {
+    return new (allocator.allocate<LLVMFunctionTypeStorage>())
+        LLVMFunctionTypeStorage(std::get<0>(key),
+                                allocator.copyInto(std::get<1>(key)),
+                                std::get<2>(key));
+  }
+
+  static unsigned hashKey(const KeyTy &key) {
+    // LLVM doesn't like hashing bools in tuples.
+    return llvm::hash_combine(std::get<0>(key), std::get<1>(key),
+                              static_cast<int>(std::get<2>(key)));
+  }
+
+  bool operator==(const KeyTy &key) const {
+    return std::make_tuple(getReturnType(), getArgumentTypes(), isVariadic()) ==
+           key;
+  }
+
+  /// Returns the list of function argument types.
+  ArrayRef<LLVMTypeNew> getArgumentTypes() const { return argumentTypes; }
+
+  /// Checks whether the function type is variadic.
+  bool isVariadic() const { return returnTypeAndVariadic.getInt(); }
+
+  /// Returns the function result type.
+  LLVMTypeNew getReturnType() const {
+    return returnTypeAndVariadic.getPointer();
+  }
+
+private:
+  /// Function result type packed with the variadic bit.
+  llvm::PointerIntPair<LLVMTypeNew, 1, bool> returnTypeAndVariadic;
+  /// Argument types.
+  ArrayRef<LLVMTypeNew> argumentTypes;
+};
+
+//===----------------------------------------------------------------------===//
+// LLVMIntegerTypeStorage.
+//===----------------------------------------------------------------------===//
+
+/// Storage type for LLVM dialect integer types. These are uniqued by bitwidth.
+struct LLVMIntegerTypeStorage : public TypeStorage {
+  using KeyTy = unsigned;
+
+  LLVMIntegerTypeStorage(unsigned width) : bitwidth(width) {}
+
+  static LLVMIntegerTypeStorage *construct(TypeStorageAllocator &allocator,
+                                           const KeyTy &key) {
+    return new (allocator.allocate<LLVMIntegerTypeStorage>())
+        LLVMIntegerTypeStorage(key);
+  }
+
+  bool operator==(const KeyTy &key) const { return key == bitwidth; }
+
+  unsigned bitwidth;
+};
+
+//===----------------------------------------------------------------------===//
+// LLVMPointerTypeStorage.
+//===----------------------------------------------------------------------===//
+
+/// Storage type for LLVM dialect pointer types. These are uniqued by a pair of
+/// element type and address space.
+struct LLVMPointerTypeStorage : public TypeStorage {
+  using KeyTy = std::tuple<LLVMTypeNew, unsigned>;
+
+  LLVMPointerTypeStorage(const KeyTy &key)
+      : pointeeType(std::get<0>(key)), addressSpace(std::get<1>(key)) {}
+
+  static LLVMPointerTypeStorage *construct(TypeStorageAllocator &allocator,
+                                           const KeyTy &key) {
+    return new (allocator.allocate<LLVMPointerTypeStorage>())
+        LLVMPointerTypeStorage(key);
+  }
+
+  bool operator==(const KeyTy &key) const {
+    return std::make_tuple(pointeeType, addressSpace) == key;
+  }
+
+  LLVMTypeNew pointeeType;
+  unsigned addressSpace;
+};
+
+//===----------------------------------------------------------------------===//
+// LLVMTypeAndSizeStorage.
+//===----------------------------------------------------------------------===//
+
+/// Common storage used for LLVM dialect types that need an element type and a
+/// number: arrays, fixed and scalable vectors. The actual semantics of the
+/// type is defined by its kind.
+struct LLVMTypeAndSizeStorage : public TypeStorage {
+  using KeyTy = std::tuple<LLVMTypeNew, unsigned>;
+
+  LLVMTypeAndSizeStorage(const KeyTy &key)
+      : elementType(std::get<0>(key)), numElements(std::get<1>(key)) {}
+
+  static LLVMTypeAndSizeStorage *construct(TypeStorageAllocator &allocator,
+                                           const KeyTy &key) {
+    return new (allocator.allocate<LLVMTypeAndSizeStorage>())
+        LLVMTypeAndSizeStorage(key);
+  }
+
+  bool operator==(const KeyTy &key) const {
+    return std::make_tuple(elementType, numElements) == key;
+  }
+
+  LLVMTypeNew elementType;
+  unsigned numElements;
+};
+
+} // end namespace detail
+} // end namespace LLVM
+} // end namespace mlir
+
+#endif // DIALECT_LLVMIR_IR_TYPEDETAIL_H
diff --git a/mlir/lib/Parser/DialectSymbolParser.cpp b/mlir/lib/Parser/DialectSymbolParser.cpp
index 1a7e2c5448c19..3b522a876f254 100644
--- a/mlir/lib/Parser/DialectSymbolParser.cpp
+++ b/mlir/lib/Parser/DialectSymbolParser.cpp
@@ -237,6 +237,17 @@ class CustomDialectAsmParser : public DialectAsmParser {
     return success(parser.consumeIf(Token::star));
   }
 
+  /// Parses a quoted string token if present.
+  ParseResult parseOptionalString(StringRef *string) override {
+    if (!parser.getToken().is(Token::string))
+      return failure();
+
+    if (string)
+      *string = parser.getTokenSpelling().drop_front().drop_back();
+    parser.consumeToken();
+    return success();
+  }
+
   /// Returns if the current token corresponds to a keyword.
   bool isCurrentTokenAKeyword() const {
     return parser.getToken().is(Token::bare_identifier) ||
@@ -297,6 +308,10 @@ class CustomDialectAsmParser : public DialectAsmParser {
     return parser.parseDimensionListRanked(dimensions, allowDynamic);
   }
 
+  OptionalParseResult parseOptionalType(Type &result) override {
+    return parser.parseOptionalType(result);
+  }
+
 private:
   /// The full symbol specification.
   StringRef fullSpec;
diff --git a/mlir/test/Dialect/LLVMIR/types-invalid.mlir b/mlir/test/Dialect/LLVMIR/types-invalid.mlir
new file mode 100644
index 0000000000000..bb281087412c9
--- /dev/null
+++ b/mlir/test/Dialect/LLVMIR/types-invalid.mlir
@@ -0,0 +1,95 @@
+// RUN: mlir-opt --allow-unregistered-dialect -split-input-file -verify-diagnostics %s
+
+func @repeated_struct_name() {
+  "some.op"() : () -> !llvm2.struct<"a", (ptr<struct<"a">>)>
+  // expected-error @+2 {{identified type already used with a different body}}
+  // expected-note @+1 {{existing body: (ptr<struct<"a">>)}}
+  "some.op"() : () -> !llvm2.struct<"a", (i32)>
+}
+
+// -----
+
+func @repeated_struct_name_packed() {
+  "some.op"() : () -> !llvm2.struct<"a", packed (i32)>
+  // expected-error @+2 {{identified type already used with a different body}}
+  // expected-note @+1 {{existing body: packed (i32)}}
+  "some.op"() : () -> !llvm2.struct<"a", (i32)>
+}
+
+// -----
+
+func @repeated_struct_opaque() {
+  "some.op"() : () -> !llvm2.struct<"a", opaque>
+  // expected-error @+2 {{identified type already used with a different body}}
+  // expected-note @+1 {{existing body: opaque}}
+  "some.op"() : () -> !llvm2.struct<"a", ()>
+}
+
+// -----
+
+func @repeated_struct_opaque_non_empty() {
+  "some.op"() : () -> !llvm2.struct<"a", opaque>
+  // expected-error @+2 {{identified type already used with a different body}}
+  // expected-note @+1 {{existing body: opaque}}
+  "some.op"() : () -> !llvm2.struct<"a", (i32, i32)>
+}
+
+// -----
+
+func @repeated_struct_opaque_redefinition() {
+  "some.op"() : () -> !llvm2.struct<"a", ()>
+  // expected-error @+1 {{redeclaring defined struct as opaque}}
+  "some.op"() : () -> !llvm2.struct<"a", opaque>
+}
+
+// -----
+
+func @struct_literal_opaque() {
+  // expected-error @+1 {{only identified structs can be opaque}}
+  "some.op"() : () -> !llvm2.struct<opaque>
+}
+
+// -----
+
+func @unexpected_type() {
+  // expected-error @+1 {{unexpected type, expected i* or keyword}}
+  "some.op"() : () -> !llvm2.f32
+}
+
+// -----
+
+func @unexpected_type() {
+  // expected-error @+1 {{unknown LLVM type}}
+  "some.op"() : () -> !llvm2.ifoo
+}
+
+// -----
+
+func @explicitly_opaque_struct() {
+  "some.op"() : () -> !llvm2.struct<"a", opaque>
+  // expected-error @+2 {{identified type already used with a different body}}
+  // expected-note @+1 {{existing body: opaque}}
+  "some.op"() : () -> !llvm2.struct<"a", ()>
+}
+
+// -----
+
+func @dynamic_vector() {
+  // expected-error @+1 {{expected '? x <integer> x <type>' or '<integer> x <type>'}}
+  "some.op"() : () -> !llvm2.vec<? x float>
+}
+
+// -----
+
+func @dynamic_scalable_vector() {
+  // expected-error @+1 {{expected '? x <integer> x <type>' or '<integer> x <type>'}}
+  "some.op"() : () -> !llvm2.vec<? x ? x float>
+}
+
+// -----
+
+func @unscalable_vector() {
+  // expected-error @+1 {{expected '? x <integer> x <type>' or '<integer> x <type>'}}
+  "some.op"() : () -> !llvm2.vec<4 x 4 x i32>
+}
+
diff --git a/mlir/test/Dialect/LLVMIR/types.mlir b/mlir/test/Dialect/LLVMIR/types.mlir
new file mode 100644
index 0000000000000..7ce606fe8c6a8
--- /dev/null
+++ b/mlir/test/Dialect/LLVMIR/types.mlir
@@ -0,0 +1,184 @@
+// RUN: mlir-opt -allow-unregistered-dialect %s -split-input-file | mlir-opt -allow-unregistered-dialect | FileCheck %s
+
+// CHECK-LABEL: @primitive
+func @primitive() {
+  // CHECK: !llvm2.void
+  "some.op"() : () -> !llvm2.void
+  // CHECK: !llvm2.half
+  "some.op"() : () -> !llvm2.half
+  // CHECK: !llvm2.bfloat
+  "some.op"() : () -> !llvm2.bfloat
+  // CHECK: !llvm2.float
+  "some.op"() : () -> !llvm2.float
+  // CHECK: !llvm2.double
+  "some.op"() : () -> !llvm2.double
+  // CHECK: !llvm2.fp128
+  "some.op"() : () -> !llvm2.fp128
+  // CHECK: !llvm2.x86_fp80
+  "some.op"() : () -> !llvm2.x86_fp80
+  // CHECK: !llvm2.ppc_fp128
+  "some.op"() : () -> !llvm2.ppc_fp128
+  // CHECK: !llvm2.x86_mmx
+  "some.op"() : () -> !llvm2.x86_mmx
+  // CHECK: !llvm2.token
+  "some.op"() : () -> !llvm2.token
+  // CHECK: !llvm2.label
+  "some.op"() : () -> !llvm2.label
+  // CHECK: !llvm2.metadata
+  "some.op"() : () -> !llvm2.metadata
+  return
+}
+
+// CHECK-LABEL: @func
+func @func() {
+  // CHECK: !llvm2.func<void ()>
+  "some.op"() : () -> !llvm2.func<void ()>
+  // CHECK: !llvm2.func<void (i32)>
+  "some.op"() : () -> !llvm2.func<void (i32)>
+  // CHECK: !llvm2.func<i32 ()>
+  "some.op"() : () -> !llvm2.func<i32 ()>
+  // CHECK: !llvm2.func<i32 (half, bfloat, float, double)>
+  "some.op"() : () -> !llvm2.func<i32 (half, bfloat, float, double)>
+  // CHECK: !llvm2.func<i32 (i32, i32)>
+  "some.op"() : () -> !llvm2.func<i32 (i32, i32)>
+  // CHECK: !llvm2.func<void (...)>
+  "some.op"() : () -> !llvm2.func<void (...)>
+  // CHECK: !llvm2.func<void (i32, i32, ...)>
+  "some.op"() : () -> !llvm2.func<void (i32, i32, ...)>
+  return
+}
+
+// CHECK-LABEL: @integer
+func @integer() {
+  // CHECK: !llvm2.i1
+  "some.op"() : () -> !llvm2.i1
+  // CHECK: !llvm2.i8
+  "some.op"() : () -> !llvm2.i8
+  // CHECK: !llvm2.i16
+  "some.op"() : () -> !llvm2.i16
+  // CHECK: !llvm2.i32
+  "some.op"() : () -> !llvm2.i32
+  // CHECK: !llvm2.i64
+  "some.op"() : () -> !llvm2.i64
+  // CHECK: !llvm2.i57
+  "some.op"() : () -> !llvm2.i57
+  // CHECK: !llvm2.i129
+  "some.op"() : () -> !llvm2.i129
+  return
+}
+
+// CHECK-LABEL: @ptr
+func @ptr() {
+  // CHECK: !llvm2.ptr<i8>
+  "some.op"() : () -> !llvm2.ptr<i8>
+  // CHECK: !llvm2.ptr<float>
+  "some.op"() : () -> !llvm2.ptr<float>
+  // CHECK: !llvm2.ptr<ptr<i8>>
+  "some.op"() : () -> !llvm2.ptr<ptr<i8>>
+  // CHECK: !llvm2.ptr<ptr<ptr<ptr<ptr<i8>>>>>
+  "some.op"() : () -> !llvm2.ptr<ptr<ptr<ptr<ptr<i8>>>>>
+  // CHECK: !llvm2.ptr<i8>
+  "some.op"() : () -> !llvm2.ptr<i8, 0>
+  // CHECK: !llvm2.ptr<i8, 1>
+  "some.op"() : () -> !llvm2.ptr<i8, 1>
+  // CHECK: !llvm2.ptr<i8, 42>
+  "some.op"() : () -> !llvm2.ptr<i8, 42>
+  // CHECK: !llvm2.ptr<ptr<i8, 42>, 9>
+  "some.op"() : () -> !llvm2.ptr<ptr<i8, 42>, 9>
+  return
+}
+
+// CHECK-LABEL: @vec
+func @vec() {
+  // CHECK: !llvm2.vec<4 x i32>
+  "some.op"() : () -> !llvm2.vec<4 x i32>
+  // CHECK: !llvm2.vec<4 x float>
+  "some.op"() : () -> !llvm2.vec<4 x float>
+  // CHECK: !llvm2.vec<? x 4 x i32>
+  "some.op"() : () -> !llvm2.vec<? x 4 x i32>
+  // CHECK: !llvm2.vec<? x 8 x half>
+  "some.op"() : () -> !llvm2.vec<? x 8 x half>
+  // CHECK: !llvm2.vec<4 x ptr<i8>>
+  "some.op"() : () -> !llvm2.vec<4 x ptr<i8>>
+  return
+}
+
+// CHECK-LABEL: @array
+func @array() {
+  // CHECK: !llvm2.array<10 x i32>
+  "some.op"() : () -> !llvm2.array<10 x i32>
+  // CHECK: !llvm2.array<8 x float>
+  "some.op"() : () -> !llvm2.array<8 x float>
+  // CHECK: !llvm2.array<10 x ptr<i32, 4>>
+  "some.op"() : () -> !llvm2.array<10 x ptr<i32, 4>>
+  // CHECK: !llvm2.array<10 x array<4 x float>>
+  "some.op"() : () -> !llvm2.array<10 x array<4 x float>>
+  return
+}
+
+// CHECK-LABEL: @literal_struct
+func @literal_struct() {
+  // CHECK: !llvm2.struct<()>
+  "some.op"() : () -> !llvm2.struct<()>
+  // CHECK: !llvm2.struct<(i32)>
+  "some.op"() : () -> !llvm2.struct<(i32)>
+  // CHECK: !llvm2.struct<(float, i32)>
+  "some.op"() : () -> !llvm2.struct<(float, i32)>
+  // CHECK: !llvm2.struct<(struct<(i32)>)>
+  "some.op"() : () -> !llvm2.struct<(struct<(i32)>)>
+  // CHECK: !llvm2.struct<(i32, struct<(i32)>, float)>
+  "some.op"() : () -> !llvm2.struct<(i32, struct<(i32)>, float)>
+
+  // CHECK: !llvm2.struct<packed ()>
+  "some.op"() : () -> !llvm2.struct<packed ()>
+  // CHECK: !llvm2.struct<packed (i32)>
+  "some.op"() : () -> !llvm2.struct<packed (i32)>
+  // CHECK: !llvm2.struct<packed (float, i32)>
+  "some.op"() : () -> !llvm2.struct<packed (float, i32)>
+  // CHECK: !llvm2.struct<packed (float, i32)>
+  "some.op"() : () -> !llvm2.struct<packed (float, i32)>
+  // CHECK: !llvm2.struct<packed (struct<(i32)>)>
+  "some.op"() : () -> !llvm2.struct<packed (struct<(i32)>)>
+  // CHECK: !llvm2.struct<packed (i32, struct<(i32, i1)>, float)>
+  "some.op"() : () -> !llvm2.struct<packed (i32, struct<(i32, i1)>, float)>
+
+  // CHECK: !llvm2.struct<(struct<packed (i32)>)>
+  "some.op"() : () -> !llvm2.struct<(struct<packed (i32)>)>
+  // CHECK: !llvm2.struct<packed (struct<(i32)>)>
+  "some.op"() : () -> !llvm2.struct<packed (struct<(i32)>)>
+  return
+}
+
+// CHECK-LABEL: @identified_struct
+func @identified_struct() {
+  // CHECK: !llvm2.struct<"empty", ()>
+  "some.op"() : () -> !llvm2.struct<"empty", ()>
+  // CHECK: !llvm2.struct<"opaque", opaque>
+  "some.op"() : () -> !llvm2.struct<"opaque", opaque>
+  // CHECK: !llvm2.struct<"long", (i32, struct<(i32, i1)>, float, ptr<func<void ()>>)>
+  "some.op"() : () -> !llvm2.struct<"long", (i32, struct<(i32, i1)>, float, ptr<func<void ()>>)>
+  // CHECK: !llvm2.struct<"self-recursive", (ptr<struct<"self-recursive">>)>
+  "some.op"() : () -> !llvm2.struct<"self-recursive", (ptr<struct<"self-recursive">>)>
+  // CHECK: !llvm2.struct<"unpacked", (i32)>
+  "some.op"() : () -> !llvm2.struct<"unpacked", (i32)>
+  // CHECK: !llvm2.struct<"packed", packed (i32)>
+  "some.op"() : () -> !llvm2.struct<"packed", packed (i32)>
+  // CHECK: !llvm2.struct<"name with spaces and !^$@$#", packed (i32)>
+  "some.op"() : () -> !llvm2.struct<"name with spaces and !^$@$#", packed (i32)>
+
+  // CHECK: !llvm2.struct<"mutually-a", (ptr<struct<"mutually-b", (ptr<struct<"mutually-a">, 3>)>>)>
+  "some.op"() : () -> !llvm2.struct<"mutually-a", (ptr<struct<"mutually-b", (ptr<struct<"mutually-a">, 3>)>>)>
+  // CHECK: !llvm2.struct<"mutually-b", (ptr<struct<"mutually-a", (ptr<struct<"mutually-b">>)>, 3>)>
+  "some.op"() : () -> !llvm2.struct<"mutually-b", (ptr<struct<"mutually-a", (ptr<struct<"mutually-b">>)>, 3>)>
+  // CHECK: !llvm2.struct<"referring-another", (ptr<struct<"unpacked", (i32)>>)>
+  "some.op"() : () -> !llvm2.struct<"referring-another", (ptr<struct<"unpacked", (i32)>>)>
+
+  // CHECK: !llvm2.struct<"struct-of-arrays", (array<10 x i32>)>
+  "some.op"() : () -> !llvm2.struct<"struct-of-arrays", (array<10 x i32>)>
+  // CHECK: !llvm2.array<10 x struct<"array-of-structs", (i32)>>
+  "some.op"() : () -> !llvm2.array<10 x struct<"array-of-structs", (i32)>>
+  // CHECK: !llvm2.ptr<struct<"ptr-to-struct", (i8)>>
+  "some.op"() : () -> !llvm2.ptr<struct<"ptr-to-struct", (i8)>>
+  return
+}
+
diff --git a/mlir/test/lib/Dialect/CMakeLists.txt b/mlir/test/lib/Dialect/CMakeLists.txt
index 9008b86314be0..36a18f79a8cbf 100644
--- a/mlir/test/lib/Dialect/CMakeLists.txt
+++ b/mlir/test/lib/Dialect/CMakeLists.txt
@@ -1,3 +1,4 @@
 add_subdirectory(Affine)
+add_subdirectory(LLVMIR)
 add_subdirectory(SPIRV)
 add_subdirectory(Test)
diff --git a/mlir/test/lib/Dialect/LLVMIR/CMakeLists.txt b/mlir/test/lib/Dialect/LLVMIR/CMakeLists.txt
new file mode 100644
index 0000000000000..2a42bc6974850
--- /dev/null
+++ b/mlir/test/lib/Dialect/LLVMIR/CMakeLists.txt
@@ -0,0 +1,14 @@
+
+add_mlir_library(MLIRLLVMTypeTestDialect
+  LLVMTypeTestDialect.cpp
+
+  EXCLUDE_FROM_LIBMLIR
+
+  LINK_COMPONENTS
+  Core
+
+  LINK_LIBS PUBLIC
+  MLIRDialect
+  MLIRIR
+  MLIRLLVMIR
+  )
diff --git a/mlir/test/lib/Dialect/LLVMIR/LLVMTypeTestDialect.cpp b/mlir/test/lib/Dialect/LLVMIR/LLVMTypeTestDialect.cpp
new file mode 100644
index 0000000000000..8ac1ef0a8c170
--- /dev/null
+++ b/mlir/test/lib/Dialect/LLVMIR/LLVMTypeTestDialect.cpp
@@ -0,0 +1,52 @@
+#ifndef DIALECT_LLVMIR_LLVMTYPETESTDIALECT_H_
+#define DIALECT_LLVMIR_LLVMTYPETESTDIALECT_H_
+
+#include "mlir/Dialect/LLVMIR/LLVMTypes.h"
+#include "mlir/IR/Dialect.h"
+
+namespace mlir {
+namespace LLVM {
+namespace {
+class LLVMDialectNewTypes : public Dialect {
+public:
+  LLVMDialectNewTypes(MLIRContext *ctx) : Dialect(getDialectNamespace(), ctx) {
+    // clang-format off
+    addTypes<LLVMVoidType,
+             LLVMHalfType,
+             LLVMBFloatType,
+             LLVMFloatType,
+             LLVMDoubleType,
+             LLVMFP128Type,
+             LLVMX86FP80Type,
+             LLVMPPCFP128Type,
+             LLVMX86MMXType,
+             LLVMTokenType,
+             LLVMLabelType,
+             LLVMMetadataType,
+             LLVMFunctionType,
+             LLVMIntegerType,
+             LLVMPointerType,
+             LLVMFixedVectorType,
+             LLVMScalableVectorType,
+             LLVMArrayType,
+             LLVMStructType>();
+    // clang-format on
+  }
+  static StringRef getDialectNamespace() { return "llvm2"; }
+
+  Type parseType(DialectAsmParser &parser) const override {
+    return detail::parseType(parser);
+  }
+  void printType(Type type, DialectAsmPrinter &printer) const override {
+    detail::printType(type.cast<LLVMTypeNew>(), printer);
+  }
+};
+} // namespace
+} // namespace LLVM
+
+void registerLLVMTypeTestDialect() {
+  mlir::registerDialect<LLVM::LLVMDialectNewTypes>();
+}
+} // namespace mlir
+
+#endif // DIALECT_LLVMIR_LLVMTYPETESTDIALECT_H_
diff --git a/mlir/tools/mlir-opt/CMakeLists.txt b/mlir/tools/mlir-opt/CMakeLists.txt
index 483dcfec0c0ff..f52c5f41b22b6 100644
--- a/mlir/tools/mlir-opt/CMakeLists.txt
+++ b/mlir/tools/mlir-opt/CMakeLists.txt
@@ -13,6 +13,7 @@ set(LLVM_LINK_COMPONENTS
 if(MLIR_INCLUDE_TESTS)
   set(test_libs
     MLIRAffineTransformsTestPasses
+    MLIRLLVMTypeTestDialect
     MLIRSPIRVTestPasses
     MLIRTestDialect
     MLIRTestIR
diff --git a/mlir/tools/mlir-opt/mlir-opt.cpp b/mlir/tools/mlir-opt/mlir-opt.cpp
index 620c5871a420c..05fba34092cba 100644
--- a/mlir/tools/mlir-opt/mlir-opt.cpp
+++ b/mlir/tools/mlir-opt/mlir-opt.cpp
@@ -31,6 +31,7 @@ namespace mlir {
 // Defined in the test directory, no public header.
 void registerConvertToTargetEnvPass();
 void registerInliner();
+void registerLLVMTypeTestDialect();
 void registerMemRefBoundCheck();
 void registerPassManagerTestPass();
 void registerPatternsTestPass();
@@ -39,10 +40,9 @@ void registerSideEffectTestPasses();
 void registerSimpleParametricTilingPass();
 void registerSymbolTestPasses();
 void registerTestAffineDataCopyPass();
-void registerTestAllReduceLoweringPass();
 void registerTestAffineLoopUnswitchingPass();
+void registerTestAllReduceLoweringPass();
 void registerTestBufferPlacementPreparationPass();
-void registerTestLoopPermutationPass();
 void registerTestCallGraphPass();
 void registerTestConstantFold();
 void registerTestConvertGPUKernelToCubinPass();
@@ -51,12 +51,14 @@ void registerTestDominancePass();
 void registerTestExpandTanhPass();
 void registerTestFunc();
 void registerTestGpuMemoryPromotionPass();
+void registerTestGpuParallelLoopMappingPass();
 void registerTestInterfaces();
 void registerTestLinalgHoisting();
 void registerTestLinalgTransforms();
 void registerTestLivenessPass();
 void registerTestLoopFusion();
 void registerTestLoopMappingPass();
+void registerTestLoopPermutationPass();
 void registerTestLoopUnrollingPass();
 void registerTestMatchers();
 void registerTestMemRefDependenceCheck();
@@ -65,7 +67,6 @@ void registerTestOpaqueLoc();
 void registerTestPreparationPassWithAllowedMemrefResults();
 void registerTestRecursiveTypesPass();
 void registerTestReducer();
-void registerTestGpuParallelLoopMappingPass();
 void registerTestSpirvEntryPointABIPass();
 void registerTestSCFUtilsPass();
 void registerTestVectorConversions();
@@ -104,6 +105,7 @@ static cl::opt<bool> allowUnregisteredDialects(
 void registerTestPasses() {
   registerConvertToTargetEnvPass();
   registerInliner();
+  registerLLVMTypeTestDialect();
   registerMemRefBoundCheck();
   registerPassManagerTestPass();
   registerPatternsTestPass();

From 2d8ca4ae2b1a512d31566e042a4bf4fa1043def9 Mon Sep 17 00:00:00 2001
From: Xing GUO <higuoxing@gmail.com>
Date: Mon, 3 Aug 2020 22:04:33 +0800
Subject: [PATCH 0915/1035] [DWARFYAML] Offsets should be omitted when the
 OffsetEntryCount is 0.

The offsets field should be omitted when the 'OffsetEntryCount' entry is
specified to be 0.

Reviewed By: jhenderson

Differential Revision: https://reviews.llvm.org/D85006
---
 llvm/lib/ObjectYAML/DWARFEmitter.cpp          |  2 +-
 .../yaml2obj/ELF/DWARF/debug-rnglists.yaml    | 64 +++++++++++++++++++
 2 files changed, 65 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/ObjectYAML/DWARFEmitter.cpp b/llvm/lib/ObjectYAML/DWARFEmitter.cpp
index ab3cd05a6495d..030b0e01b6f17 100644
--- a/llvm/lib/ObjectYAML/DWARFEmitter.cpp
+++ b/llvm/lib/ObjectYAML/DWARFEmitter.cpp
@@ -749,7 +749,7 @@ Error writeDWARFLists(raw_ostream &OS,
       EmitOffsets(ArrayRef<uint64_t>((const uint64_t *)Table.Offsets->data(),
                                      Table.Offsets->size()),
                   0);
-    else
+    else if (OffsetEntryCount != 0)
       EmitOffsets(Offsets, OffsetsSize);
 
     OS.write(ListBuffer.data(), ListBuffer.size());
diff --git a/llvm/test/tools/yaml2obj/ELF/DWARF/debug-rnglists.yaml b/llvm/test/tools/yaml2obj/ELF/DWARF/debug-rnglists.yaml
index 248cb190235b2..de35adec57a0b 100644
--- a/llvm/test/tools/yaml2obj/ELF/DWARF/debug-rnglists.yaml
+++ b/llvm/test/tools/yaml2obj/ELF/DWARF/debug-rnglists.yaml
@@ -669,3 +669,67 @@ DWARF:
     - Lists:
         - Entries: []
           Content: ''
+
+## u) Test that when the "OffsetEntryCount" is specified to be 0 and "Offsets" is not specified,
+## the offsets array is not emitted.
+
+# RUN: yaml2obj --docnum=19 -DENTRYCOUNT=0 %s -o %t19.o
+# RUN: llvm-readelf --hex-dump=.debug_rnglists %t19.o | \
+# RUN:   FileCheck %s --check-prefix=NO-OFFSETS
+
+#      NO-OFFSETS: Hex dump of section '.debug_rnglists':
+# NO-OFFSETS-NEXT: 0x00000000 0e000000 05000800 00000000 02010202 ................
+##                                              ^-------          offset_entry_count (4-byte)
+##                                                       ^-       DW_RLE_startx_endx
+##                                                         ^-     operands[0] (ULEB128) 0x01
+##                                                           ^-   operands[1] (ULEB128) 0x02
+##                                                             ^- DW_RLE_startx_endx
+# NO-OFFSETS-NEXT: 0x00000010 0102                                ..
+##                            ^-                                  operands[0] (ULEB128) 0x01
+##                              ^-                                operands[1] (ULEB128) 0x02
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+DWARF:
+  debug_rnglists:
+    - OffsetEntryCount: [[ENTRYCOUNT=<none>]]
+      Offsets:          [[OFFSETS=<none>]]
+      Lists:
+        - Entries:
+            - Operator: DW_RLE_startx_endx
+              Values:   [ 0x01, 0x02 ]
+        - Entries:
+            - Operator: DW_RLE_startx_endx
+              Values:   [ 0x01, 0x02 ]
+
+## v) Test that when the "Offsets" entry is specified to be empty and the "OffsetEntryCount" is not specified,
+## the offsets array will be omitted.
+
+# RUN: yaml2obj --docnum=19 -DOFFSETS=[] %s -o %t20.o
+# RUN: llvm-readelf --hex-dump=.debug_rnglists %t20.o | \
+# RUN:   FileCheck %s --check-prefix=NO-OFFSETS
+
+## w) Test that if "Offsets" is specified, the offsets array will be emitted accordingly, even when
+## the "OffsetEntryCount" is specified to be 0.
+
+# RUN: yaml2obj --docnum=19 -DOFFSETS=[0x01,0x02,0x03] -DENTRYCOUNT=0 %s -o %t21.o
+# RUN: llvm-readelf --hex-dump=.debug_rnglists %t21.o | \
+# RUN:   FileCheck %s --check-prefix=OFFSETS
+
+#      OFFSETS: Hex dump of section '.debug_rnglists':
+# OFFSETS-NEXT: 0x00000000 0e000000 05000800 00000000 01000000 ................
+##                                           ^-------          offset_entry_count (4-byte)
+##                                                    ^------- offsets[0] (4-byte)
+# OFFSETS-NEXT: 0x00000010 02000000 03000000 02010202 0102     ..............
+##                         ^-------                            offsets[1] (4-byte)
+##                                  ^-------                   offsets[2] (4-byte)
+##                                           ^-                DW_RLE_startx_endx
+##                                             ^-              operands[0] (ULEB128) 0x01
+##                                               ^-            operands[1] (ULEB128) 0x02
+##                                                 ^-          DW_RLE_startx_endx
+##                                                    ^-       operands[0] (ULEB128) 0x01
+##                                                      ^-     operands[1] (ULEB128) 0x02

From 23693ffc3ba6146a22cd1e9373e25dc1e1a41a17 Mon Sep 17 00:00:00 2001
From: Sanjay Patel <spatel@rotateright.com>
Date: Mon, 3 Aug 2020 09:19:11 -0400
Subject: [PATCH 0916/1035] [InstCombine] reduce xor-of-or's bitwise logic
 (PR46955); 2nd try

The 1st try at this (rG2265d01f2a5b) exposed what looks like
unspecified behavior in C/C++ resulting in test variations.

The arguments to BinaryOperator::CreateAnd() were both IRBuilder
function calls, and the order in which they execute determines
the order of the new instructions in the IR. But the order of
function arg evaluation is not fixed by the rules of C/C++, so
depending on compiler config, the test would fail because the
test expected a single fixed ordering of instructions.

Original commit message:
I tried to use m_Deferred() on this, but didn't find
a clean way to do that.

http://bugs.llvm.org/PR46955

https://alive2.llvm.org/ce/z/2h6QTq
---
 .../InstCombine/InstCombineAndOrXor.cpp       | 15 ++++++++++++
 llvm/test/Transforms/InstCombine/xor.ll       | 24 +++++++++----------
 2 files changed, 27 insertions(+), 12 deletions(-)

diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
index 030d2f203ed6c..9d7effc724beb 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
@@ -3351,6 +3351,21 @@ Instruction *InstCombinerImpl::visitXor(BinaryOperator &I) {
       match(Op1, m_Not(m_Specific(A))))
     return BinaryOperator::CreateNot(Builder.CreateAnd(A, B));
 
+  // (A | B) ^ (A | C) --> (B ^ C) & ~A -- There are 4 commuted variants.
+  // TODO: Loosen one-use restriction if common operand is a constant.
+  Value *D;
+  if (match(Op0, m_OneUse(m_Or(m_Value(A), m_Value(B)))) &&
+      match(Op1, m_OneUse(m_Or(m_Value(C), m_Value(D))))) {
+    if (B == C || B == D)
+      std::swap(A, B);
+    if (A == C)
+      std::swap(C, D);
+    if (A == D) {
+      Value *NotA = Builder.CreateNot(A);
+      return BinaryOperator::CreateAnd(Builder.CreateXor(B, C), NotA);
+    }
+  }
+
   if (auto *LHS = dyn_cast<ICmpInst>(I.getOperand(0)))
     if (auto *RHS = dyn_cast<ICmpInst>(I.getOperand(1)))
       if (Value *V = foldXorOfICmps(LHS, RHS, I))
diff --git a/llvm/test/Transforms/InstCombine/xor.ll b/llvm/test/Transforms/InstCombine/xor.ll
index a133f2a0e009b..363fa8ff8fdb5 100644
--- a/llvm/test/Transforms/InstCombine/xor.ll
+++ b/llvm/test/Transforms/InstCombine/xor.ll
@@ -915,9 +915,9 @@ define <2 x i32> @test51vec(<2 x i32> %x, <2 x i32> %y) {
 
 define i4 @or_or_xor(i4 %x, i4 %y, i4 %z) {
 ; CHECK-LABEL: @or_or_xor(
-; CHECK-NEXT:    [[O1:%.*]] = or i4 [[Z:%.*]], [[X:%.*]]
-; CHECK-NEXT:    [[O2:%.*]] = or i4 [[Z]], [[Y:%.*]]
-; CHECK-NEXT:    [[R:%.*]] = xor i4 [[O1]], [[O2]]
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i4 [[Z:%.*]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = xor i4 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = and i4 [[TMP2]], [[TMP1]]
 ; CHECK-NEXT:    ret i4 [[R]]
 ;
   %o1 = or i4 %z, %x
@@ -928,9 +928,9 @@ define i4 @or_or_xor(i4 %x, i4 %y, i4 %z) {
 
 define i4 @or_or_xor_commute1(i4 %x, i4 %y, i4 %z) {
 ; CHECK-LABEL: @or_or_xor_commute1(
-; CHECK-NEXT:    [[O1:%.*]] = or i4 [[X:%.*]], [[Z:%.*]]
-; CHECK-NEXT:    [[O2:%.*]] = or i4 [[Z]], [[Y:%.*]]
-; CHECK-NEXT:    [[R:%.*]] = xor i4 [[O1]], [[O2]]
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i4 [[Z:%.*]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = xor i4 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = and i4 [[TMP2]], [[TMP1]]
 ; CHECK-NEXT:    ret i4 [[R]]
 ;
   %o1 = or i4 %x, %z
@@ -941,9 +941,9 @@ define i4 @or_or_xor_commute1(i4 %x, i4 %y, i4 %z) {
 
 define i4 @or_or_xor_commute2(i4 %x, i4 %y, i4 %z) {
 ; CHECK-LABEL: @or_or_xor_commute2(
-; CHECK-NEXT:    [[O1:%.*]] = or i4 [[Z:%.*]], [[X:%.*]]
-; CHECK-NEXT:    [[O2:%.*]] = or i4 [[Y:%.*]], [[Z]]
-; CHECK-NEXT:    [[R:%.*]] = xor i4 [[O1]], [[O2]]
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i4 [[Z:%.*]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = xor i4 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = and i4 [[TMP2]], [[TMP1]]
 ; CHECK-NEXT:    ret i4 [[R]]
 ;
   %o1 = or i4 %z, %x
@@ -954,9 +954,9 @@ define i4 @or_or_xor_commute2(i4 %x, i4 %y, i4 %z) {
 
 define <2 x i4> @or_or_xor_commute3(<2 x i4> %x, <2 x i4> %y, <2 x i4> %z) {
 ; CHECK-LABEL: @or_or_xor_commute3(
-; CHECK-NEXT:    [[O1:%.*]] = or <2 x i4> [[X:%.*]], [[Z:%.*]]
-; CHECK-NEXT:    [[O2:%.*]] = or <2 x i4> [[Y:%.*]], [[Z]]
-; CHECK-NEXT:    [[R:%.*]] = xor <2 x i4> [[O1]], [[O2]]
+; CHECK-NEXT:    [[TMP1:%.*]] = xor <2 x i4> [[Z:%.*]], <i4 -1, i4 -1>
+; CHECK-NEXT:    [[TMP2:%.*]] = xor <2 x i4> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = and <2 x i4> [[TMP2]], [[TMP1]]
 ; CHECK-NEXT:    ret <2 x i4> [[R]]
 ;
   %o1 = or <2 x i4> %x, %z

From 1c2777f585fc0e5e8f853dab455c62ae50298f9a Mon Sep 17 00:00:00 2001
From: Shinji Okumura <okuraofvegetable@gmail.com>
Date: Mon, 3 Aug 2020 23:31:13 +0900
Subject: [PATCH 0917/1035] [NFC][APInt][DenseMapInfo] Move
 DenseMapAPIntKeyInfo into DenseMap.h as DenseMapInfo<APInt>

`DenseMapAPIntKeyInfo` is now located in `lib/IR/LLVMContextImpl.h`.
Moved it into `include/ADT/DenseMapInfo.h` to use it.

Reviewed By: jdoerfert

Differential Revision: https://reviews.llvm.org/D85131
---
 llvm/include/llvm/ADT/APInt.h        |  3 ++-
 llvm/include/llvm/ADT/DenseMapInfo.h | 24 ++++++++++++++++++++++++
 llvm/lib/IR/LLVMContextImpl.h        | 22 +---------------------
 3 files changed, 27 insertions(+), 22 deletions(-)

diff --git a/llvm/include/llvm/ADT/APInt.h b/llvm/include/llvm/ADT/APInt.h
index f7df648d27ed6..715f7cd4fdf3f 100644
--- a/llvm/include/llvm/ADT/APInt.h
+++ b/llvm/include/llvm/ADT/APInt.h
@@ -31,6 +31,7 @@ class raw_ostream;
 template <typename T> class SmallVectorImpl;
 template <typename T> class ArrayRef;
 template <typename T> class Optional;
+template <typename T> struct DenseMapInfo;
 
 class APInt;
 
@@ -96,7 +97,7 @@ class LLVM_NODISCARD APInt {
 
   unsigned BitWidth; ///< The number of bits in this APInt.
 
-  friend struct DenseMapAPIntKeyInfo;
+  friend struct DenseMapInfo<APInt>;
 
   friend class APSInt;
 
diff --git a/llvm/include/llvm/ADT/DenseMapInfo.h b/llvm/include/llvm/ADT/DenseMapInfo.h
index e465331ac6f7b..1cace4b3192fb 100644
--- a/llvm/include/llvm/ADT/DenseMapInfo.h
+++ b/llvm/include/llvm/ADT/DenseMapInfo.h
@@ -13,6 +13,7 @@
 #ifndef LLVM_ADT_DENSEMAPINFO_H
 #define LLVM_ADT_DENSEMAPINFO_H
 
+#include "llvm/ADT/APInt.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/Hashing.h"
 #include "llvm/ADT/StringRef.h"
@@ -347,6 +348,29 @@ template <> struct DenseMapInfo<hash_code> {
   static bool isEqual(hash_code LHS, hash_code RHS) { return LHS == RHS; }
 };
 
+/// Provide DenseMapInfo for APInt.
+template <> struct DenseMapInfo<APInt> {
+  static inline APInt getEmptyKey() {
+    APInt V(nullptr, 0);
+    V.U.VAL = 0;
+    return V;
+  }
+
+  static inline APInt getTombstoneKey() {
+    APInt V(nullptr, 0);
+    V.U.VAL = 1;
+    return V;
+  }
+
+  static unsigned getHashValue(const APInt &Key) {
+    return static_cast<unsigned>(hash_value(Key));
+  }
+
+  static bool isEqual(const APInt &LHS, const APInt &RHS) {
+    return LHS.getBitWidth() == RHS.getBitWidth() && LHS == RHS;
+  }
+};
+
 } // end namespace llvm
 
 #endif // LLVM_ADT_DENSEMAPINFO_H
diff --git a/llvm/lib/IR/LLVMContextImpl.h b/llvm/lib/IR/LLVMContextImpl.h
index b97ac37c5fcfd..e8fdaa23761c3 100644
--- a/llvm/lib/IR/LLVMContextImpl.h
+++ b/llvm/lib/IR/LLVMContextImpl.h
@@ -57,27 +57,7 @@ class Type;
 class Value;
 class ValueHandleBase;
 
-struct DenseMapAPIntKeyInfo {
-  static inline APInt getEmptyKey() {
-    APInt V(nullptr, 0);
-    V.U.VAL = 0;
-    return V;
-  }
-
-  static inline APInt getTombstoneKey() {
-    APInt V(nullptr, 0);
-    V.U.VAL = 1;
-    return V;
-  }
-
-  static unsigned getHashValue(const APInt &Key) {
-    return static_cast<unsigned>(hash_value(Key));
-  }
-
-  static bool isEqual(const APInt &LHS, const APInt &RHS) {
-    return LHS.getBitWidth() == RHS.getBitWidth() && LHS == RHS;
-  }
-};
+using DenseMapAPIntKeyInfo = DenseMapInfo<APInt>;
 
 struct DenseMapAPFloatKeyInfo {
   static inline APFloat getEmptyKey() { return APFloat(APFloat::Bogus(), 1); }

From 08649d4321bb73c888e03ac316f8ccab600a9533 Mon Sep 17 00:00:00 2001
From: Xing GUO <higuoxing@gmail.com>
Date: Mon, 3 Aug 2020 23:19:42 +0800
Subject: [PATCH 0918/1035] [DWARFYAML] Implement the .debug_loclists section.

This patch implements the .debug_loclists section. There are only two
DWARF expressions are implemented in this patch (DW_OP_consts,
DW_OP_stack_value). We will implement more in the future.

The YAML description of the .debug_loclists section is:

```
debug_loclists:
  - Format:              DWARF32 ## Optional
    Length:              0x1234  ## Optional
    Version:             5       ## Optional (5 by default)
    AddressSize:         8       ## Optional
    SegmentSelectorSize: 0       ## Optional (0 by default)
    OffsetEntryCount:    1       ## Optional
    Offsets:             [ 1 ]   ## Optional
    Lists:
      - Entries:
          - Operator:          DW_LLE_startx_endx
            Values:            [ 0x1234, 0x4321 ]
            DescriptorsLength: 0x1234             ## Optional
            Descriptors:
              - Operator: DW_OP_consts
                Values:   [ 0x1234 ]
```

Reviewed By: jhenderson

Differential Revision: https://reviews.llvm.org/D84234
---
 llvm/include/llvm/ObjectYAML/DWARFEmitter.h   |   1 +
 llvm/include/llvm/ObjectYAML/DWARFYAML.h      |  46 +
 llvm/lib/ObjectYAML/DWARFEmitter.cpp          | 137 +++
 llvm/lib/ObjectYAML/DWARFYAML.cpp             |  17 +
 .../yaml2obj/ELF/DWARF/debug-loclists.yaml    | 914 ++++++++++++++++++
 5 files changed, 1115 insertions(+)
 create mode 100644 llvm/test/tools/yaml2obj/ELF/DWARF/debug-loclists.yaml

diff --git a/llvm/include/llvm/ObjectYAML/DWARFEmitter.h b/llvm/include/llvm/ObjectYAML/DWARFEmitter.h
index 89d01cecb9b7b..c7c3070651504 100644
--- a/llvm/include/llvm/ObjectYAML/DWARFEmitter.h
+++ b/llvm/include/llvm/ObjectYAML/DWARFEmitter.h
@@ -42,6 +42,7 @@ Error emitDebugLine(raw_ostream &OS, const Data &DI);
 Error emitDebugAddr(raw_ostream &OS, const Data &DI);
 Error emitDebugStrOffsets(raw_ostream &OS, const Data &DI);
 Error emitDebugRnglists(raw_ostream &OS, const Data &DI);
+Error emitDebugLoclists(raw_ostream &OS, const Data &DI);
 
 std::function<Error(raw_ostream &, const Data &)>
 getDWARFEmitterByName(StringRef SecName);
diff --git a/llvm/include/llvm/ObjectYAML/DWARFYAML.h b/llvm/include/llvm/ObjectYAML/DWARFYAML.h
index 127a529139786..ae3eff1fe8564 100644
--- a/llvm/include/llvm/ObjectYAML/DWARFYAML.h
+++ b/llvm/include/llvm/ObjectYAML/DWARFYAML.h
@@ -184,11 +184,23 @@ struct StringOffsetsTable {
   std::vector<yaml::Hex64> Offsets;
 };
 
+struct DWARFOperation {
+  dwarf::LocationAtom Operator;
+  std::vector<yaml::Hex64> Values;
+};
+
 struct RnglistEntry {
   dwarf::RnglistEntries Operator;
   std::vector<yaml::Hex64> Values;
 };
 
+struct LoclistEntry {
+  dwarf::LoclistEntries Operator;
+  std::vector<yaml::Hex64> Values;
+  Optional<yaml::Hex64> DescriptionsLength;
+  std::vector<DWARFOperation> Descriptions;
+};
+
 template <typename EntryType> struct ListEntries {
   Optional<std::vector<EntryType>> Entries;
   Optional<yaml::BinaryRef> Content;
@@ -224,6 +236,7 @@ struct Data {
 
   std::vector<LineTable> DebugLines;
   Optional<std::vector<ListTable<RnglistEntry>>> DebugRnglists;
+  Optional<std::vector<ListTable<LoclistEntry>>> DebugLoclists;
 
   bool isEmpty() const;
 
@@ -254,6 +267,12 @@ LLVM_YAML_IS_SEQUENCE_VECTOR(
 LLVM_YAML_IS_SEQUENCE_VECTOR(
     llvm::DWARFYAML::ListEntries<DWARFYAML::RnglistEntry>)
 LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::DWARFYAML::RnglistEntry)
+LLVM_YAML_IS_SEQUENCE_VECTOR(
+    llvm::DWARFYAML::ListTable<DWARFYAML::LoclistEntry>)
+LLVM_YAML_IS_SEQUENCE_VECTOR(
+    llvm::DWARFYAML::ListEntries<DWARFYAML::LoclistEntry>)
+LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::DWARFYAML::LoclistEntry)
+LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::DWARFYAML::DWARFOperation)
 
 namespace llvm {
 namespace yaml {
@@ -322,6 +341,10 @@ template <> struct MappingTraits<DWARFYAML::SegAddrPair> {
   static void mapping(IO &IO, DWARFYAML::SegAddrPair &SegAddrPair);
 };
 
+template <> struct MappingTraits<DWARFYAML::DWARFOperation> {
+  static void mapping(IO &IO, DWARFYAML::DWARFOperation &DWARFOperation);
+};
+
 template <typename EntryType>
 struct MappingTraits<DWARFYAML::ListTable<EntryType>> {
   static void mapping(IO &IO, DWARFYAML::ListTable<EntryType> &ListTable);
@@ -338,6 +361,10 @@ template <> struct MappingTraits<DWARFYAML::RnglistEntry> {
   static void mapping(IO &IO, DWARFYAML::RnglistEntry &RnglistEntry);
 };
 
+template <> struct MappingTraits<DWARFYAML::LoclistEntry> {
+  static void mapping(IO &IO, DWARFYAML::LoclistEntry &LoclistEntry);
+};
+
 template <> struct MappingTraits<DWARFYAML::AddrTableEntry> {
   static void mapping(IO &IO, DWARFYAML::AddrTableEntry &AddrTable);
 };
@@ -434,6 +461,25 @@ template <> struct ScalarEnumerationTraits<dwarf::RnglistEntries> {
   }
 };
 
+#define HANDLE_DW_LLE(unused, name)                                            \
+  io.enumCase(value, "DW_LLE_" #name, dwarf::DW_LLE_##name);
+
+template <> struct ScalarEnumerationTraits<dwarf::LoclistEntries> {
+  static void enumeration(IO &io, dwarf::LoclistEntries &value) {
+#include "llvm/BinaryFormat/Dwarf.def"
+  }
+};
+
+#define HANDLE_DW_OP(id, name, version, vendor)                                \
+  io.enumCase(value, "DW_OP_" #name, dwarf::DW_OP_##name);
+
+template <> struct ScalarEnumerationTraits<dwarf::LocationAtom> {
+  static void enumeration(IO &io, dwarf::LocationAtom &value) {
+#include "llvm/BinaryFormat/Dwarf.def"
+    io.enumFallback<yaml::Hex8>(value);
+  }
+};
+
 } // end namespace yaml
 } // end namespace llvm
 
diff --git a/llvm/lib/ObjectYAML/DWARFEmitter.cpp b/llvm/lib/ObjectYAML/DWARFEmitter.cpp
index 030b0e01b6f17..deff6a68363b2 100644
--- a/llvm/lib/ObjectYAML/DWARFEmitter.cpp
+++ b/llvm/lib/ObjectYAML/DWARFEmitter.cpp
@@ -611,6 +611,39 @@ static Error writeListEntryAddress(StringRef EncodingName, raw_ostream &OS,
   return Error::success();
 }
 
+static Expected<uint64_t>
+writeDWARFExpression(raw_ostream &OS,
+                     const DWARFYAML::DWARFOperation &Operation,
+                     uint8_t AddrSize, bool IsLittleEndian) {
+  auto CheckOperands = [&](uint64_t ExpectedOperands) -> Error {
+    return checkOperandCount(dwarf::OperationEncodingString(Operation.Operator),
+                             Operation.Values, ExpectedOperands);
+  };
+
+  uint64_t ExpressionBegin = OS.tell();
+  writeInteger((uint8_t)Operation.Operator, OS, IsLittleEndian);
+  switch (Operation.Operator) {
+  case dwarf::DW_OP_consts:
+    if (Error Err = CheckOperands(1))
+      return std::move(Err);
+    encodeSLEB128(Operation.Values[0], OS);
+    break;
+  case dwarf::DW_OP_stack_value:
+    if (Error Err = CheckOperands(0))
+      return std::move(Err);
+    break;
+  default:
+    StringRef EncodingStr = dwarf::OperationEncodingString(Operation.Operator);
+    return createStringError(errc::not_supported,
+                             "DWARF expression: " +
+                                 (EncodingStr.empty()
+                                      ? "0x" + utohexstr(Operation.Operator)
+                                      : EncodingStr) +
+                                 " is not supported");
+  }
+  return OS.tell() - ExpressionBegin;
+}
+
 static Expected<uint64_t> writeListEntry(raw_ostream &OS,
                                          const DWARFYAML::RnglistEntry &Entry,
                                          uint8_t AddrSize,
@@ -672,6 +705,103 @@ static Expected<uint64_t> writeListEntry(raw_ostream &OS,
   return OS.tell() - BeginOffset;
 }
 
+static Expected<uint64_t> writeListEntry(raw_ostream &OS,
+                                         const DWARFYAML::LoclistEntry &Entry,
+                                         uint8_t AddrSize,
+                                         bool IsLittleEndian) {
+  uint64_t BeginOffset = OS.tell();
+  writeInteger((uint8_t)Entry.Operator, OS, IsLittleEndian);
+
+  StringRef EncodingName = dwarf::LocListEncodingString(Entry.Operator);
+
+  auto CheckOperands = [&](uint64_t ExpectedOperands) -> Error {
+    return checkOperandCount(EncodingName, Entry.Values, ExpectedOperands);
+  };
+
+  auto WriteAddress = [&](uint64_t Addr) -> Error {
+    return writeListEntryAddress(EncodingName, OS, Addr, AddrSize,
+                                 IsLittleEndian);
+  };
+
+  auto WriteDWARFOperations = [&]() -> Error {
+    std::string OpBuffer;
+    raw_string_ostream OpBufferOS(OpBuffer);
+    uint64_t DescriptionsLength = 0;
+
+    for (const DWARFYAML::DWARFOperation &Op : Entry.Descriptions) {
+      if (Expected<uint64_t> OpSize =
+              writeDWARFExpression(OpBufferOS, Op, AddrSize, IsLittleEndian))
+        DescriptionsLength += *OpSize;
+      else
+        return OpSize.takeError();
+    }
+
+    if (Entry.DescriptionsLength)
+      DescriptionsLength = *Entry.DescriptionsLength;
+    else
+      DescriptionsLength = OpBuffer.size();
+
+    encodeULEB128(DescriptionsLength, OS);
+    OS.write(OpBuffer.data(), OpBuffer.size());
+
+    return Error::success();
+  };
+
+  switch (Entry.Operator) {
+  case dwarf::DW_LLE_end_of_list:
+    if (Error Err = CheckOperands(0))
+      return std::move(Err);
+    break;
+  case dwarf::DW_LLE_base_addressx:
+    if (Error Err = CheckOperands(1))
+      return std::move(Err);
+    encodeULEB128(Entry.Values[0], OS);
+    break;
+  case dwarf::DW_LLE_startx_endx:
+  case dwarf::DW_LLE_startx_length:
+  case dwarf::DW_LLE_offset_pair:
+    if (Error Err = CheckOperands(2))
+      return std::move(Err);
+    encodeULEB128(Entry.Values[0], OS);
+    encodeULEB128(Entry.Values[1], OS);
+    if (Error Err = WriteDWARFOperations())
+      return std::move(Err);
+    break;
+  case dwarf::DW_LLE_default_location:
+    if (Error Err = CheckOperands(0))
+      return std::move(Err);
+    if (Error Err = WriteDWARFOperations())
+      return std::move(Err);
+    break;
+  case dwarf::DW_LLE_base_address:
+    if (Error Err = CheckOperands(1))
+      return std::move(Err);
+    if (Error Err = WriteAddress(Entry.Values[0]))
+      return std::move(Err);
+    break;
+  case dwarf::DW_LLE_start_end:
+    if (Error Err = CheckOperands(2))
+      return std::move(Err);
+    if (Error Err = WriteAddress(Entry.Values[0]))
+      return std::move(Err);
+    cantFail(WriteAddress(Entry.Values[1]));
+    if (Error Err = WriteDWARFOperations())
+      return std::move(Err);
+    break;
+  case dwarf::DW_LLE_start_length:
+    if (Error Err = CheckOperands(2))
+      return std::move(Err);
+    if (Error Err = WriteAddress(Entry.Values[0]))
+      return std::move(Err);
+    encodeULEB128(Entry.Values[1], OS);
+    if (Error Err = WriteDWARFOperations())
+      return std::move(Err);
+    break;
+  }
+
+  return OS.tell() - BeginOffset;
+}
+
 template <typename EntryType>
 Error writeDWARFLists(raw_ostream &OS,
                       ArrayRef<DWARFYAML::ListTable<EntryType>> Tables,
@@ -764,6 +894,12 @@ Error DWARFYAML::emitDebugRnglists(raw_ostream &OS, const Data &DI) {
       OS, *DI.DebugRnglists, DI.IsLittleEndian, DI.Is64BitAddrSize);
 }
 
+Error DWARFYAML::emitDebugLoclists(raw_ostream &OS, const Data &DI) {
+  assert(DI.DebugLoclists && "unexpected emitDebugRnglists() call");
+  return writeDWARFLists<DWARFYAML::LoclistEntry>(
+      OS, *DI.DebugLoclists, DI.IsLittleEndian, DI.Is64BitAddrSize);
+}
+
 std::function<Error(raw_ostream &, const DWARFYAML::Data &)>
 DWARFYAML::getDWARFEmitterByName(StringRef SecName) {
   auto EmitFunc =
@@ -776,6 +912,7 @@ DWARFYAML::getDWARFEmitterByName(StringRef SecName) {
           .Case("debug_gnu_pubtypes", DWARFYAML::emitDebugGNUPubtypes)
           .Case("debug_info", DWARFYAML::emitDebugInfo)
           .Case("debug_line", DWARFYAML::emitDebugLine)
+          .Case("debug_loclists", DWARFYAML::emitDebugLoclists)
           .Case("debug_pubnames", DWARFYAML::emitDebugPubnames)
           .Case("debug_pubtypes", DWARFYAML::emitDebugPubtypes)
           .Case("debug_ranges", DWARFYAML::emitDebugRanges)
diff --git a/llvm/lib/ObjectYAML/DWARFYAML.cpp b/llvm/lib/ObjectYAML/DWARFYAML.cpp
index 37d45996786d3..a0caad10a36c1 100644
--- a/llvm/lib/ObjectYAML/DWARFYAML.cpp
+++ b/llvm/lib/ObjectYAML/DWARFYAML.cpp
@@ -50,6 +50,8 @@ SetVector<StringRef> DWARFYAML::Data::getNonEmptySectionNames() const {
     SecNames.insert("debug_str_offsets");
   if (DebugRnglists)
     SecNames.insert("debug_rnglists");
+  if (DebugLoclists)
+    SecNames.insert("debug_loclists");
   return SecNames;
 }
 
@@ -74,6 +76,7 @@ void MappingTraits<DWARFYAML::Data>::mapping(IO &IO, DWARFYAML::Data &DWARF) {
   IO.mapOptional("debug_addr", DWARF.DebugAddr);
   IO.mapOptional("debug_str_offsets", DWARF.DebugStrOffsets);
   IO.mapOptional("debug_rnglists", DWARF.DebugRnglists);
+  IO.mapOptional("debug_loclists", DWARF.DebugLoclists);
   IO.setContext(OldContext);
 }
 
@@ -235,12 +238,26 @@ void MappingTraits<DWARFYAML::StringOffsetsTable>::mapping(
   IO.mapOptional("Offsets", StrOffsetsTable.Offsets);
 }
 
+void MappingTraits<DWARFYAML::DWARFOperation>::mapping(
+    IO &IO, DWARFYAML::DWARFOperation &DWARFOperation) {
+  IO.mapRequired("Operator", DWARFOperation.Operator);
+  IO.mapOptional("Values", DWARFOperation.Values);
+}
+
 void MappingTraits<DWARFYAML::RnglistEntry>::mapping(
     IO &IO, DWARFYAML::RnglistEntry &RnglistEntry) {
   IO.mapRequired("Operator", RnglistEntry.Operator);
   IO.mapOptional("Values", RnglistEntry.Values);
 }
 
+void MappingTraits<DWARFYAML::LoclistEntry>::mapping(
+    IO &IO, DWARFYAML::LoclistEntry &LoclistEntry) {
+  IO.mapRequired("Operator", LoclistEntry.Operator);
+  IO.mapOptional("Values", LoclistEntry.Values);
+  IO.mapOptional("DescriptionsLength", LoclistEntry.DescriptionsLength);
+  IO.mapOptional("Descriptions", LoclistEntry.Descriptions);
+}
+
 template <typename EntryType>
 void MappingTraits<DWARFYAML::ListEntries<EntryType>>::mapping(
     IO &IO, DWARFYAML::ListEntries<EntryType> &ListEntries) {
diff --git a/llvm/test/tools/yaml2obj/ELF/DWARF/debug-loclists.yaml b/llvm/test/tools/yaml2obj/ELF/DWARF/debug-loclists.yaml
new file mode 100644
index 0000000000000..4da5595827d1e
--- /dev/null
+++ b/llvm/test/tools/yaml2obj/ELF/DWARF/debug-loclists.yaml
@@ -0,0 +1,914 @@
+## Test that yaml2obj emits a .debug_loclists section when requested.
+
+## a) Generate and verify a little endian DWARF32 .debug_loclists section in a 64-bit object file.
+
+# RUN: yaml2obj --docnum=1 -DENDIAN=ELFDATA2LSB %s -o %t1.dwarf32.le.o
+# RUN: llvm-readobj --sections --section-data %t1.dwarf32.le.o | \
+# RUN:   FileCheck -DSIZE=133 -DADDRALIGN=1 %s --check-prefixes=SHDR,DWARF32-LE
+
+#            SHDR: Index: 1
+#       SHDR-NEXT: Name: .debug_loclists (1)
+#       SHDR-NEXT: Type: SHT_PROGBITS (0x1)
+#       SHDR-NEXT: Flags [ (0x0)
+#       SHDR-NEXT: ]
+#       SHDR-NEXT: Address: 0x0
+#       SHDR-NEXT: Offset: 0x40
+#       SHDR-NEXT: Size: [[SIZE]]
+#       SHDR-NEXT: Link: 0
+#       SHDR-NEXT: Info: 0
+#       SHDR-NEXT: AddressAlignment: [[ADDRALIGN]]
+#       SHDR-NEXT: EntrySize: 0
+# DWARF32-LE-NEXT: SectionData (
+# DWARF32-LE-NEXT:   0000: 3D000000 05000800 03000000 0C000000  |=...............|
+##                         ^-------                             unit_length (4-byte)
+##                                  ^---                        version (2-byte)
+##                                      ^-                      address_size (1-byte)
+##                                        ^-                    segment_selector_size (1-byte)
+##                                           ^-------           offset_entry_count (4-byte)
+##                                                    ^-------  offsets[0] (4-byte)
+# DWARF32-LE-NEXT:   0010: 1B000000 2F000000 01B42402 B424A186  |..../.....$..$..|
+##                         ^-------                             offsets[1] (4-byte)
+##                                  ^-------                    offsets[2] (4-byte)
+##                                           ^-                 DW_LLE_base_addressx
+##                                             ^---             operands[0] (ULEB128) 0x1234
+##                                                 ^-           DW_LLE_startx_endx
+##                                                    ^---      operands[0] (ULEB128) 0x1234
+##                                                        ^---  operands[1] (ULEB128) 0x4321
+# DWARF32-LE-NEXT:   0020: 010411B4 249F0003 B424A186 0102117D  |....$....$.....}|
+##                         --
+##                           ^-                                 location descriptions length (ULEB128) 0x04
+##                             ^-                               DW_OP_consts
+##                               ^----                          operands[0] (SLEB128) +0x1234
+##                                    ^-                        DW_OP_stack_value
+##                                      ^-                      DW_LLE_end_of_list
+##                                        ^-                    DW_LLE_startx_length
+##                                           ^---               operands[0] (ULEB128) 0x1234
+##                                               ^------        operands[1] (ULEB128) 0x4321
+##                                                      ^-      location descriptions length (ULEB128) 0x02
+##                                                        ^-    DW_OP_consts
+##                                                          ^-  operands[0] (SLEB128) -0x03
+# DWARF32-LE-NEXT:   0030: 04B424A1 86010311 B4240005 0311B424  |..$......$.....$|
+##                         ^-                                   DW_LLE_offset_pair
+##                           ^---                               operands[0] (ULEB128) 0x1234
+##                               ^------                        operands[1] (ULEB128) 0x4321
+##                                      ^-                      location descriptions length (ULEB128) 0x03
+##                                        ^-                    DW_OP_consts
+##                                           ^---               operands[0] (SLEB128) +0x1234
+##                                               ^-             DW_LLE_end_of_list
+##                                                 ^-           DW_LLE_default_location
+##                                                    ^-        location descriptions length (ULEB128) 0x03
+##                                                      ^-      DW_OP_consts
+##                                                        ^---  operands[0] (SLEB128) +0x1234
+# DWARF32-LE-NEXT:   0040: 00400000 00050008 00020000 00080000  |.@..............|
+##                         ^-                                   DW_LLE_end_of_list
+##                           ^--------                          unit_length (4-byte)
+##                                    ^---                      version (2-byte)
+##                                        ^-                    address_size (1-byte)
+##                                           ^-                 segment_selector_size (1-byte)
+##                                             ^--------        offset_entry_count (4-byte)
+##                                                      ^-----  offsets[0] (4-byte)
+# DWARF32-LE-NEXT:   0050: 00270000 00063412 00000000 00000734  |.'....4........4|
+##                         --
+##                           ^--------                          offsets[1] (4-byte)
+##                                    ^-                        DW_LLE_base_address
+##                                      ^-----------------      operands[0] (8-byte)
+##                                                        ^-    DW_LLE_start_end
+##                                                          ^-  operands[0] (8-byte)
+# DWARF32-LE-NEXT:   0060: 12000000 00000021 43000000 00000003  |.......!C.......|
+##                         ---------------
+##                                        ^-----------------    operands[1] (8-byte)
+##                                                          ^-  location descriptions length (ULEB128) 0x03
+# DWARF32-LE-NEXT:   0070: 11B42400 08341200 00000000 00A18601  |..$..4..........|
+##                         ^-                                   DW_OP_consts
+##                           ^---                               operands[0] (SLEB128) +0x1234
+##                               ^-                             DW_LLE_end_of_list
+##                                  ^-                          DW_LLE_start_length
+##                                    ^-----------------        operands[0] (8-byte)
+##                                                      ^-----  operands[1] (LEB128) 0x4321
+# DWARF32-LE-NEXT:   0080: 0311B424 00                          |...$.|
+##                         ^-                                   location descriptions length (ULEB128) 0x03
+##                           ^-                                 DW_OP_consts
+##                             ^---                             operands[0] (SLEB128) +0x1234
+##                                  ^-                          DW_LLE_end_of_list
+# DWARF32-LE-NEXT: )
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    [[ENDIAN]]
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+DWARF:
+  debug_loclists:
+    - Lists:
+        - Entries:
+            - Operator: DW_LLE_base_addressx
+              Values:   [ 0x1234 ]
+            - Operator: DW_LLE_startx_endx
+              Values:   [ 0x1234, 0x4321 ]
+              Descriptions:
+                - Operator: DW_OP_consts
+                  Values:   [ 0x1234 ]
+                - Operator: DW_OP_stack_value
+            - Operator: DW_LLE_end_of_list
+        - Entries:
+            - Operator: DW_LLE_startx_length
+              Values:   [ 0x1234, 0x4321 ]
+              Descriptions:
+                - Operator: DW_OP_consts
+                  ## Test a negative number (-3).
+                  Values:   [ 0xfffffffffffffffd ]
+            - Operator: DW_LLE_offset_pair
+              Values:   [ 0x1234, 0x4321 ]
+              Descriptions:
+                - Operator: DW_OP_consts
+                  Values:   [ 0x1234 ]
+            - Operator: DW_LLE_end_of_list
+        - Entries:
+            - Operator: DW_LLE_default_location
+              Descriptions:
+                - Operator: DW_OP_consts
+                  Values:   [ 0x1234 ]
+            - Operator: DW_LLE_end_of_list
+    - Lists:
+        - Entries:
+            - Operator: DW_LLE_base_address
+              Values:   [ 0x1234 ]
+            - Operator: DW_LLE_start_end
+              Values:   [ 0x1234, 0x4321 ]
+              Descriptions:
+                - Operator: DW_OP_consts
+                  Values:   [ 0x1234 ]
+            - Operator: DW_LLE_end_of_list
+        - Entries:
+            - Operator: DW_LLE_start_length
+              Values:   [ 0x1234, 0x4321 ]
+              Descriptions:
+                - Operator: DW_OP_consts
+                  Values:   [ 0x1234 ]
+            - Operator: DW_LLE_end_of_list
+
+## b) Generate and verify a big endian DWARF32 .debug_loclists section in a 64-bit object file.
+
+# RUN: yaml2obj --docnum=1 -DENDIAN=ELFDATA2MSB %s -o %t.dwarf32.be.o
+# RUN: llvm-readobj --sections --section-data %t.dwarf32.be.o | \
+# RUN:   FileCheck -DSIZE=133 -DADDRALIGN=1 %s --check-prefixes=SHDR,DWARF32-BE
+
+# DWARF32-BE-NEXT: SectionData (
+# DWARF32-BE-NEXT:   0000: 0000003D 00050800 00000003 0000000C  |...=............|
+##                         ^-------                             unit_length (4-byte)
+##                                  ^---                        version (2-byte)
+##                                      ^-                      address_size (1-byte)
+##                                        ^-                    segment_selector_size (1-byte)
+##                                           ^-------           offset_entry_count (4-byte)
+##                                                    ^-------  offsets[0] (4-byte)
+# DWARF32-BE-NEXT:   0010: 0000001B 0000002F 01B42402 B424A186  |......./..$..$..|
+##                         ^-------                             offsets[1] (4-byte)
+##                                  ^-------                    offsets[2] (4-byte)
+##                                           ^-                 DW_LLE_base_addressx
+##                                             ^---             operands[0] (ULEB128) 0x1234
+##                                                 ^-           DW_LLE_startx_endx
+##                                                    ^---      operands[0] (ULEB128) 0x1234
+##                                                        ^---  operands[1] (ULEB128) 0x4321
+# DWARF32-BE-NEXT:   0020: 010411B4 249F0003 B424A186 0102117D  |....$....$.....}|
+##                         --
+##                           ^-                                 location descriptions length (ULEB128) 0x04
+##                             ^-                               DW_OP_consts
+##                               ^----                          operands[0] (SLEB128) +0x1234
+##                                    ^-                        DW_OP_stack_value
+##                                      ^-                      DW_LLE_end_of_list
+##                                        ^-                    DW_LLE_startx_length
+##                                           ^---               operands[0] (ULEB128) 0x1234
+##                                               ^------        operands[1] (ULEB128) 0x4321
+##                                                      ^-      location descriptions length (ULEB128) 0x02
+##                                                        ^-    DW_OP_consts
+##                                                          ^-  operands[0] (SLEB128) -0x03
+# DWARF32-BE-NEXT:   0030: 04B424A1 86010311 B4240005 0311B424  |..$......$.....$|
+##                         ^-                                   DW_LLE_offset_pair
+##                           ^---                               operands[0] (ULEB128) 0x1234
+##                               ^------                        operands[1] (ULEB128) 0x4321
+##                                      ^-                      location descriptions length (ULEB128) 0x03
+##                                        ^-                    DW_OP_consts
+##                                           ^---               operands[0] (SLEB128) +0x1234
+##                                               ^-             DW_LLE_end_of_list
+##                                                 ^-           DW_LLE_default_location
+##                                                    ^-        location descriptions length (ULEB128) 0x03
+##                                                      ^-      DW_OP_consts
+##                                                        ^---  operands[0] (SLEB128) +0x1234
+# DWARF32-BE-NEXT:   0040: 00000000 40000508 00000000 02000000  |....@...........|
+##                         ^-                                   DW_LLE_end_of_list
+##                           ^--------                          unit_length (4-byte)
+##                                    ^---                      version (2-byte)
+##                                        ^-                    address_size (1-byte)
+##                                           ^-                 segment_selector_size (1-byte)
+##                                             ^--------        offset_entry_count (4-byte)
+##                                                      ^-----  offsets[0] (4-byte)
+# DWARF32-BE-NEXT:   0050: 08000000 27060000 00000000 12340700  |....'........4..|
+##                         --
+##                           ^--------                          offsets[1] (4-byte)
+##                                    ^-                        DW_LLE_base_address
+##                                      ^-----------------      operands[0] (8-byte)
+##                                                        ^-    DW_LLE_start_end
+##                                                          ^-  operands[0] (8-byte)
+# DWARF32-BE-NEXT:   0060: 00000000 00123400 00000000 00432103  |......4......C!.|
+##                         ---------------
+##                                        ^-----------------    operands[1] (8-byte)
+##                                                          ^-  location descriptions length (ULEB128) 0x03
+# DWARF32-BE-NEXT:   0070: 11B42400 08000000 00000012 34A18601  |..$.........4...|
+##                         ^-                                   DW_OP_consts
+##                           ^---                               operands[0] (SLEB128) +0x1234
+##                               ^-                             DW_LLE_end_of_list
+##                                  ^-                          DW_LLE_start_length
+##                                    ^-----------------        operands[0] (8-byte)
+##                                                      ^-----  operands[1] (LEB128) 0x4321
+# DWARF32-BE-NEXT:   0080: 0311B424 00                          |...$.|
+##                         ^-                                   location descriptions length (ULEB128) 0x03
+##                           ^-                                 DW_OP_consts
+##                             ^---                             operands[0] (SLEB128) +0x1234
+##                                  ^-                          DW_LLE_end_of_list
+# DWARF32-BE-NEXT: )
+
+## c) Generate and verify a little endian DWARF64 .debug_loclists section in a 64-bit object file.
+
+# RUN: yaml2obj --docnum=2 -DENDIAN=ELFDATA2LSB %s -o %t2.dwarf64.le.o
+# RUN: llvm-readobj --sections --section-data %t2.dwarf64.le.o | \
+# RUN:   FileCheck -DSIZE=47 -DADDRALIGN=1 %s --check-prefixes=SHDR,DWARF64-LE
+
+# DWARF64-LE-NEXT: SectionData (
+# DWARF64-LE-NEXT:   0000: FFFFFFFF 23000000 00000000 05000800  |....#...........|
+##                         ^-------------------------           unit_length (12-byte)
+##                                                    ^---      version (2-byte)
+##                                                        ^-    address_size (1-byte)
+##                                                          ^-  segment_selector_size (1-byte)
+# DWARF64-LE-NEXT:   0010: 02000000 10000000 00000000 1A000000  |................|
+##                         ^-------                             offset_entry_count (4-byte)
+##                                  ^----------------           offsets[0] (8-byte)
+##                                                    ^-------  offsets[1] (8-byte)
+# DWARF64-LE-NEXT:   0020: 00000000 02B424A1 86010311 B42400    |......$......$.|
+##                         --------
+##                                  ^-                          DW_LLE_startx_endx
+##                                    ^---                      operands[0] (ULEB128) 0x1234
+##                                        ^------               operands[1] (ULEB128) 0x4321
+##                                               ^-             location descriptions length (ULEB128) 0x03
+##                                                 ^-           DW_OP_consts
+##                                                    ^---      operands[0] (SLEB128) +0x1234
+##                                                        ^-    DW_LLE_end_of_list
+# DWARF64-LE-NEXT: )
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    [[ENDIAN]]
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+DWARF:
+  debug_loclists:
+    - Format:  DWARF64
+      Lists:
+        - Entries:
+            - Operator: DW_LLE_startx_endx
+              Values:   [ 0x1234, 0x4321 ]
+              Descriptions:
+                - Operator: DW_OP_consts
+                  Values:   [ 0x1234 ]
+        - Entries:
+            - Operator: DW_LLE_end_of_list
+
+## d) Generate and verify a big endian DWARF64 .debug_loclists section in a 64-bit object file.
+
+# RUN: yaml2obj --docnum=2 -DENDIAN=ELFDATA2MSB %s -o %t2.dwarf64.be.o
+# RUN: llvm-readobj --sections --section-data %t2.dwarf64.be.o | \
+# RUN:   FileCheck -DSIZE=47 -DADDRALIGN=1 %s --check-prefixes=SHDR,DWARF64-BE
+
+# DWARF64-BE-NEXT: SectionData (
+# DWARF64-BE-NEXT:   0000: FFFFFFFF 00000000 00000023 00050800  |...........#....|
+##                         ^-------------------------           unit_length (12-byte)
+##                                                    ^---      version (2-byte)
+##                                                        ^-    address_size (1-byte)
+##                                                          ^-  segment_selector_size (1-byte)
+# DWARF64-BE-NEXT:   0010: 00000002 00000000 00000010 00000000  |................|
+##                         ^-------                             offset_entry_count (4-byte)
+##                                  ^----------------           offsets[0] (8-byte)
+##                                                    ^-------  offsets[1] (8-byte)
+# DWARF64-BE-NEXT:   0020: 0000001A 02B424A1 86010311 B42400    |......$......$.|
+##                         --------
+##                                  ^-                          DW_LLE_startx_endx
+##                                    ^---                      operands[0] (ULEB128) 0x1234
+##                                        ^------               operands[1] (ULEB128) 0x4321
+##                                               ^-             location descriptions length (ULEB128) 0x03
+##                                                 ^-           DW_OP_consts
+##                                                    ^---      operands[0] (SLEB128) +0x1234
+##                                                        ^-    DW_LLE_end_of_list
+# DWARF64-BE-NEXT: )
+
+## e) Test that the length, version, segment_selector_size, address_size, offset_entry_count,
+## offsets and location descriptions length fields can be specified manually.
+
+# RUN: yaml2obj --docnum=3 %s -o %t3.o
+# RUN: llvm-readelf --hex-dump=.debug_loclists %t3.o | \
+# RUN:   FileCheck %s --check-prefix=OVERWRITE
+
+#      OVERWRITE: Hex dump of section '.debug_loclists':
+# OVERWRITE-NEXT: 0x00000000 34120000 06000303 04000000 01000000 4...............
+##                           ^-------                            unit_length (4-byte) 0x1234
+##                                    ^---                       version (2-byte) 0x06
+##                                        ^-                     address_size (1-byte) 0x03
+##                                          ^-                   segment_selector_size (1-byte) 0x03
+##                                             ^-------          offset_entry_count (4-byte) 0x04
+##                                                      ^------- offsets[0] (4-byte) 0x01
+# OVERWRITE-NEXT: 0x00000010 02b424a1 8601a186 019f00            ..$........
+##                           ^-                                  DW_LLE_startx_endx
+##                             ^---                              operands[0] (ULEB128) 0x1234
+##                                 ^------                       operands[1] (ULEB128) 0x4321
+##                                        ^------                location descriptions length (ULEB128) 0x4321
+##                                               ^-              DW_OP_stack_value
+##                                                 ^-            DW_LLE_end_of_list
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+DWARF:
+  debug_loclists:
+    - Length:              0x1234
+      Version:             6
+      AddressSize:         3
+      SegmentSelectorSize: 3
+      OffsetEntryCount:    4
+      Offsets:             [ 0x01 ]
+      Lists:
+        - Entries:
+            - Operator:           DW_LLE_startx_endx
+              Values:             [ 0x1234, 0x4321 ]
+              DescriptionsLength: 0x4321
+              Descriptions:
+                - Operator: DW_OP_stack_value
+        - Entries:
+            - Operator: DW_LLE_end_of_list
+
+## f) Test that location descriptions can be omitted from the YAML description.
+
+# RUN: yaml2obj --docnum=4 %s -o %t4.o
+# RUN: llvm-readelf --hex-dump=.debug_loclists %t4.o | \
+# RUN:   FileCheck %s --check-prefix=OMIT-DESCRIPTIONS
+
+#      OMIT-DESCRIPTIONS: Hex dump of section '.debug_loclists':
+# OMIT-DESCRIPTIONS-NEXT: 0x00000000 42000000 05000800 01000000 04000000 B...............
+# OMIT-DESCRIPTIONS-NEXT: 0x00000010 02b424a1 86010003 b424a186 010004b4 ..$......$......
+##                                  ^-                                  DW_LLE_startx_endx
+##                                    ^---                              operands[0] (ULEB128) 0x1234
+##                                        ^------                       operands[1] (ULEB128) 0x4321
+##                                               ^-                     location descriptions length (ULEB128) 0x00
+##                                                 ^-                   DW_LLE_startx_length
+##                                                    ^---              operands[0] (ULEB128) 0x1234
+##                                                        ^------       operands[1] (ULEB128) 0x4321
+##                                                               ^-     location descriptions length (ULEB128) 0x00
+##                                                                 ^-   DW_LLE_offset_pair
+##                                                                   ^- operands[0] (ULEB128) 0x1234
+# OMIT-DESCRIPTIONS-NEXT: 0x00000020 24a18601 00050007 34120000 00000000 $.......4.......
+##                                  --
+##                                    ^-----                            operands[1] (ULEB128) 0x4321
+##                                           ^-                         location descriptions length (ULEB128) 0x00
+##                                             ^-                       DW_LLE_default_location
+##                                               ^-                     location descriptions length (ULEB128) 0x00
+##                                                 ^-                   DW_LLE_start_end
+##                                                    ^---------------- operands[0] (8-byte)
+# OMIT-DESCRIPTIONS-NEXT: 0x00000030 21430000 00000000 00083412 00000000 !C........4.....
+##                                  ^----------------                   operands[1] (8-byte)
+##                                                    ^-                location descriptions length (ULEB128) 0x00
+##                                                      ^-              DW_LLE_start_length
+##                                                        ^------------ operands[0] (8-byte)
+# OMIT-DESCRIPTIONS-NEXT: 0x00000040 0000a186 0100                       ......
+##                                  ----
+##                                      ^------                         operands[1] (ULEB128) 0x4321
+##                                             ^-                       location descriptions length (ULEB128) 0x00
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+DWARF:
+  debug_loclists:
+    - Lists:
+        - Entries:
+            - Operator: DW_LLE_startx_endx
+              Values:   [ 0x1234, 0x4321 ]
+            - Operator: DW_LLE_startx_length
+              Values:   [ 0x1234, 0x4321 ]
+            - Operator: DW_LLE_offset_pair
+              Values:   [ 0x1234, 0x4321 ]
+            - Operator: DW_LLE_default_location
+            - Operator: DW_LLE_start_end
+              Values:   [ 0x1234, 0x4321 ]
+            - Operator: DW_LLE_start_length
+              Values:   [ 0x1234, 0x4321 ]
+
+## g) Test that the default value of the address_size field in a 32-bit object file is 4.
+
+# RUN: yaml2obj --docnum=5 %s -o %t5.o
+# RUN: llvm-readelf --hex-dump=.debug_loclists %t5.o | \
+# RUN:   FileCheck %s --check-prefix=ADDRSIZE32
+
+#      ADDRSIZE32: Hex dump of section '.debug_loclists':
+# ADDRSIZE32-NEXT: 0x00000000 24000000 05000400 01000000 04000000 $...............
+##                                         ^-                     address_size (1-byte) 0x04
+# ADDRSIZE32-NEXT: 0x00000010 06341200 00073412 00002143 00000008 .4....4...!C....
+##                            ^-                                  DW_LLE_base_address
+##                              ^--------                         operands[0] (4-byte)
+##                                       ^-                       DW_LLE_start_end
+##                                         ^--------              operands[0] (4-byte)
+##                                                  ^--------     operands[1] (4-byte)
+##                                                           ^-   counted location description
+##                                                             ^- DW_LLE_start_length
+# ADDRSIZE32-NEXT: 0x00000020 34120000 a1860100                   4.......
+##                            ^-------                            operands[0] (4-byte)
+##                                     ^-----                     operands[1] (ULEB128) 0x4321
+##                                           ^-                   counted location description
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS32
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+DWARF:
+  debug_loclists:
+    - Lists:
+        - Entries:
+            - Operator: DW_LLE_base_address
+              Values:   [ 0x1234 ]
+            - Operator: DW_LLE_start_end
+              Values:   [ 0x1234, 0x4321 ]
+            - Operator: DW_LLE_start_length
+              Values:   [ 0x1234, 0x4321 ]
+
+## h) Test that the address_size field can be specified manually and the size of
+## corresponding operands will be changed accordingly.
+
+# RUN: yaml2obj --docnum=6 %s -o %t6.o
+# RUN: llvm-readelf --hex-dump=.debug_loclists %t6.o | \
+# RUN:   FileCheck %s --check-prefix=ADDRSIZE32
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+DWARF:
+  debug_loclists:
+    - AddressSize: 4
+      Lists:
+        - Entries:
+            - Operator: DW_LLE_base_address
+              Values:   [ 0x1234 ]
+            - Operator: DW_LLE_start_end
+              Values:   [ 0x1234, 0x4321 ]
+            - Operator: DW_LLE_start_length
+              Values:   [ 0x1234, 0x4321 ]
+
+## i) Test that yaml2obj emits an error message if we try to assign an invalid value to
+## 'AddressSize' when there is an entry whose operands contain address.
+
+# RUN: not yaml2obj -DOPERATOR=base_address -DVALUES=[0x01] --docnum=7 %s 2>&1 | \
+# RUN:   FileCheck -DOPERATOR=base_address %s --check-prefix=INVALID-ADDRSIZE
+
+# RUN: not yaml2obj -DOPERATOR=start_end -DVALUES=[0x01,0x02] --docnum=7 %s 2>&1 | \
+# RUN:   FileCheck -DOPERATOR=start_end %s --check-prefix=INVALID-ADDRSIZE
+
+# RUN: not yaml2obj -DOPERATOR=start_length -DVALUES=[0x01,0x02] --docnum=7 %s 2>&1 | \
+# RUN:   FileCheck -DOPERATOR=start_length %s --check-prefix=INVALID-ADDRSIZE
+
+# INVALID-ADDRSIZE: yaml2obj: error: unable to write address for the operator DW_LLE_[[OPERATOR]]: invalid integer write size: 3
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+DWARF:
+  debug_loclists:
+    - AddressSize: 3
+      Lists:
+        - Entries:
+            - Operator: DW_LLE_[[OPERATOR]]
+              Values:   [[VALUES]]
+
+## j) Test that yaml2obj emits an error message if we specify invalid numbers of operands
+## for a location list encoding.
+
+# RUN: not yaml2obj -DOPERATOR=end_of_list -DVALUES=[0x01] --docnum=8 %s 2>&1 | \
+# RUN:   FileCheck -DOPERATOR=end_of_list -DACTUAL=1 -DEXPECTED=0 %s --check-prefix=INVALID-LLE-OPERANDS
+
+# RUN: not yaml2obj -DOPERATOR=base_addressx -DVALUES=[] --docnum=8 %s 2>&1 | \
+# RUN:   FileCheck -DOPERATOR=base_addressx -DACTUAL=0 -DEXPECTED=1 %s --check-prefix=INVALID-LLE-OPERANDS
+
+# RUN: not yaml2obj -DOPERATOR=startx_endx -DVALUES=[0x01] --docnum=8 %s 2>&1 | \
+# RUN:   FileCheck -DOPERATOR=startx_endx -DACTUAL=1 -DEXPECTED=2 %s --check-prefix=INVALID-LLE-OPERANDS
+
+# RUN: not yaml2obj -DOPERATOR=startx_length -DVALUES=[0x01] --docnum=8 %s 2>&1 | \
+# RUN:   FileCheck -DOPERATOR=startx_length -DACTUAL=1 -DEXPECTED=2 %s --check-prefix=INVALID-LLE-OPERANDS
+
+# RUN: not yaml2obj -DOPERATOR=offset_pair -DVALUES=[] --docnum=8 %s 2>&1 | \
+# RUN:   FileCheck -DOPERATOR=offset_pair -DACTUAL=0 -DEXPECTED=2 %s --check-prefix=INVALID-LLE-OPERANDS
+
+# RUN: not yaml2obj -DOPERATOR=default_location -DVALUES=[0x01] --docnum=8 %s 2>&1 | \
+# RUN:   FileCheck -DOPERATOR=default_location -DACTUAL=1 -DEXPECTED=0 %s --check-prefix=INVALID-LLE-OPERANDS
+
+# RUN: not yaml2obj -DOPERATOR=base_address -DVALUES=[0x01,0x02] --docnum=8 %s 2>&1 | \
+# RUN:   FileCheck -DOPERATOR=base_address -DACTUAL=2 -DEXPECTED=1 %s --check-prefix=INVALID-LLE-OPERANDS
+
+# RUN: not yaml2obj -DOPERATOR=start_end -DVALUES=[0x01,0x02,0x03] --docnum=8 %s 2>&1 | \
+# RUN:   FileCheck -DOPERATOR=start_end -DACTUAL=3 -DEXPECTED=2 %s --check-prefix=INVALID-LLE-OPERANDS
+
+# RUN: not yaml2obj -DOPERATOR=start_length -DVALUES=[0x01] --docnum=8 %s 2>&1 | \
+# RUN:   FileCheck -DOPERATOR=start_length -DACTUAL=1 -DEXPECTED=2 %s --check-prefix=INVALID-LLE-OPERANDS
+
+# INVALID-LLE-OPERANDS: yaml2obj: error: invalid number ([[ACTUAL]]) of operands for the operator: DW_LLE_[[OPERATOR]], [[EXPECTED]] expected
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+DWARF:
+  debug_loclists:
+    - Lists:
+        - Entries:
+            - Operator: DW_LLE_[[OPERATOR]]
+              Values:   [[VALUES]]
+
+## k) Test that yaml2obj emits an error message if we specify invalid numbers of operands
+## for a DWARF expression operator.
+
+# RUN: not yaml2obj --docnum=9 -DOPERATOR=consts -DVALUES=[0x01,0x02] %s 2>&1 | \
+# RUN:   FileCheck -DACTUAL=2 -DEXPECTED=1 -DOPERATOR=consts %s --check-prefix=INVALID-OP-OPERANDS
+
+# RUN: not yaml2obj --docnum=9 -DOPERATOR=stack_value -DVALUES=[0x01] %s 2>&1 | \
+# RUN:   FileCheck -DACTUAL=1 -DEXPECTED=0 -DOPERATOR=stack_value %s --check-prefix=INVALID-OP-OPERANDS
+
+# INVALID-OP-OPERANDS: yaml2obj: error: invalid number ([[ACTUAL]]) of operands for the operator: DW_OP_[[OPERATOR]], [[EXPECTED]] expected
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+DWARF:
+  debug_loclists:
+    - Lists:
+        - Entries:
+            - Operator: DW_LLE_startx_endx
+              Values:   [ 0x01, 0x02 ]
+              Descriptions:
+                - Operator: DW_OP_[[OPERATOR]]
+                  Values:   [[VALUES]]
+
+## l) Test that an empty list is allowed for a location list table.
+
+# RUN: yaml2obj --docnum=10 %s -o %t10.o
+# RUN: llvm-readelf --hex-dump=.debug_loclists %t10.o | \
+# RUN:   FileCheck %s --check-prefix=EMPTY-LIST
+
+#      EMPTY-LIST: Hex dump of section '.debug_loclists':
+# EMPTY-LIST-NEXT: 0x00000000 08000000 05000800 00000000 ............
+##                            ^-------                   unit_length (4-byte)
+##                                     ^---              version (2-byte)
+##                                         ^-            address_size (1-byte)
+##                                           ^-          segment_selector_size (1-byte)
+##                                              ^------- offset_entry_count (4-byte)
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+DWARF:
+  debug_loclists:
+    - Lists: []
+
+## m) Generate the .debug_loclists section from raw section content.
+
+# RUN: yaml2obj --docnum=11 %s -o %t11.o
+# RUN: llvm-readobj --sections --section-data %t11.o | \
+# RUN:   FileCheck %s -DSIZE=3 -DADDRALIGN=0 --check-prefixes=SHDR,ARBITRARY-CONTENT
+
+#      ARBITRARY-CONTENT: SectionData (
+# ARBITRARY-CONTENT-NEXT:   0000: 112233
+# ARBITRARY-CONTENT-NEXT: )
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name:    .debug_loclists
+    Type:    SHT_PROGBITS
+    Content: "112233"
+
+## n) Generate the .debug_loclists section when the "Size" is specified.
+
+# RUN: yaml2obj --docnum=12 %s -o %t12.o
+# RUN: llvm-readelf --hex-dump=.debug_loclists %t12.o | \
+# RUN:   FileCheck %s --check-prefix=SIZE
+
+#       SIZE: Hex dump of section '.debug_loclists':
+#  SIZE-NEXT: 0x00000000 00000000 00000000 00000000 00000000 ................
+# SIZE-EMPTY:
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name: .debug_loclists
+    Type: SHT_PROGBITS
+    Size: 0x10
+
+## o) Test that yaml2obj emits an error message when both the "Size" and the
+## "debug_loclists" entry are specified at the same time.
+
+# RUN: not yaml2obj --docnum=13 %s 2>&1 | FileCheck %s --check-prefix=ERROR
+
+# ERROR: yaml2obj: error: cannot specify section '.debug_loclists' contents in the 'DWARF' entry and the 'Content' or 'Size' in the 'Sections' entry at the same time
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name: .debug_loclists
+    Type: SHT_PROGBITS
+    Size: 0x10
+DWARF:
+  debug_loclists:
+    - Lists: []
+
+## p) Test that yaml2obj emits an error message when both the "Content" and the
+## "debug_loclists" entry are specified at the same time.
+
+# RUN: not yaml2obj --docnum=14 %s 2>&1 | FileCheck %s --check-prefix=ERROR
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name:    .debug_loclists
+    Type:    SHT_PROGBITS
+    Content: "00"
+DWARF:
+  debug_loclists:
+    - Lists: []
+
+## q) Test that all the properties can be overridden by the section header when
+## the "debug_loclists" entry doesn't exist.
+
+# RUN: yaml2obj --docnum=15 %s -o %t15.o
+# RUN: llvm-readelf --sections %t15.o | FileCheck %s --check-prefix=OVERRIDDEN
+
+#      OVERRIDDEN: [Nr] Name            Type   Address          Off    Size   ES Flg Lk Inf Al
+#      OVERRIDDEN: [ 1] .debug_loclists STRTAB 0000000000002020 000050 00000c 01   A  2   1  2
+# OVERRIDDEN-NEXT: [ 2] .sec            STRTAB 0000000000000000 00005c 000000 00      0   0  0
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name:         .debug_loclists
+    Type:         SHT_STRTAB  ## SHT_PROGBITS by default.
+    Flags:        [SHF_ALLOC] ## 0 by default.
+    Link:         .sec        ## 0 by default.
+    EntSize:      1           ## 0 by default.
+    Info:         1           ## 0 by default.
+    AddressAlign: 2           ## 0 by default.
+    Address:      0x2020      ## 0x00 by default.
+    Offset:       0x50        ## 0x40 for the first section.
+    Size:         0x0c        ## Set the "Size" so that we can reuse the check tag "OVERRIDDEN".
+  - Name:         .sec        ## Linked by .debug_loclists.
+    Type:         SHT_STRTAB
+
+## r) Test that all the properties can be overridden by the section header when
+## the "debug_loclists" entry exists.
+
+# RUN: yaml2obj --docnum=16 %s -o %t16.o
+# RUN: llvm-readelf --sections %t16.o | FileCheck %s --check-prefix=OVERRIDDEN
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name:         .debug_loclists
+    Type:         SHT_STRTAB  ## SHT_PROGBITS by default.
+    Flags:        [SHF_ALLOC] ## 0 by default.
+    Link:         .sec        ## 0 by default.
+    EntSize:      1           ## 0 by default.
+    Info:         1           ## 0 by default.
+    AddressAlign: 2           ## 1 by default.
+    Address:      0x2020      ## 0x00 by default.
+    Offset:       0x50        ## 0x40 for the first section.
+  - Name:         .sec        ## Linked by .debug_loclists.
+    Type:         SHT_STRTAB
+DWARF:
+  debug_loclists:
+    - Lists: []
+
+## s) Test that the .debug_loclists section header is emitted if the "debug_loclists"
+## entry is empty.
+
+# RUN: yaml2obj --docnum=17 %s -o %t17.o
+# RUN: llvm-readobj --sections --section-data %t17.o | \
+# RUN:   FileCheck -DSIZE=0 -DADDRALIGN=1 %s --check-prefixes=SHDR,EMPTY-CONTENT
+
+# EMPTY-CONTENT-NEXT: SectionData (
+# EMPTY-CONTENT-NEXT: )
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+DWARF:
+  debug_loclists: []
+
+## t) Test that yaml2obj emits an error message if we use an unimplemented DWARF expression
+## operator.
+
+# RUN: not yaml2obj --docnum=18 -DOP=0x01 %s 2>&1 | \
+# RUN:   FileCheck -DOP=0x1 %s --check-prefix=UNSUPPORTED-OP
+
+# UNSUPPORTED-OP: yaml2obj: error: DWARF expression: [[OP]] is not supported
+
+# RUN: not yaml2obj --docnum=18 -DOP=DW_OP_entry_value %s 2>&1 | \
+# RUN:   FileCheck -DOP=DW_OP_entry_value %s --check-prefix=UNSUPPORTED-OP
+
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+DWARF:
+  debug_loclists:
+    - Lists:
+        - Entries:
+            - Operator: DW_LLE_default_location
+              Descriptions:
+                - Operator: [[OP]]
+
+## u) Test that we are able to generate a location list via raw binary data.
+
+# RUN: yaml2obj --docnum=19 %s -o %t19.o
+# RUN: llvm-readelf --hex-dump=.debug_loclists %t19.o | \
+# RUN:   FileCheck %s --check-prefix=CUSTOM-LIST
+
+#      CUSTOM-LIST: Hex dump of section '.debug_loclists':
+# CUSTOM-LIST-NEXT: 0x00000000 2a000000 05000800 03000000 0c000000 *...............
+##                             ^-------                            unit_length (4-byte)
+##                                      ^---                       version (2-byte)
+##                                          ^-                     address_size (1-byte)
+##                                            ^-                   segment_selector_size (1-byte)
+##                                               ^-------          offset_entry_count (4-byte)
+##                                                        ^------- offsets[0] (4-byte)
+# CUSTOM-LIST-NEXT: 0x00000010 12000000 1a000000 02b424b4 24001234 ..........$.$..4
+##                             ^-------                            offsets[1] (4-byte)
+##                                      ^-------                   offsets[2] (4-byte)
+##                                               ^-                DW_LLE_startx_endx
+##                                                 ^---            operands[0] (ULEB128) 0x1234
+##                                                     ^----       operands[1] (ULEB128) 0x1234
+##                                                          ^-     location descriptions length (ULEB128) 0x00
+##                                                            ^--- custom list content
+# CUSTOM-LIST-NEXT: 0x00000020 567890ab cdefabcd ef123456 7890     Vx........4Vx.
+##                             -------------
+##                                          ^-----------------     custom list content
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+DWARF:
+  debug_loclists:
+    - Lists:
+        - Entries:
+            - Operator: DW_LLE_startx_endx
+              Values:   [ 0x1234, 0x1234 ]
+        - Content: '1234567890abcdef'
+        - Content: 'abcdef1234567890'
+
+## v) Test that yaml2obj emits an error message when 'Content' and 'Entries' are specified
+## at the same time.
+
+# RUN: not yaml2obj --docnum=20 %s 2>&1 | FileCheck %s --check-prefix=ERR
+
+#      ERR: YAML:{{.*}}: error: Entries and Content can't be used together
+# ERR-NEXT:         - Entries: []
+# ERR-NEXT:           ^
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+DWARF:
+  debug_loclists:
+    - Lists:
+        - Entries: []
+          Content: ''
+
+## w) Test that when the "OffsetEntryCount" is specified to be 0 and "Offsets" is not specified,
+## the offsets array is not emitted.
+
+# RUN: yaml2obj --docnum=21 -DOFFSETENTRIES=0 %s -o %t21.o
+# RUN: llvm-readelf --hex-dump=.debug_loclists %t21.o | \
+# RUN:   FileCheck %s --check-prefix=NO-OFFSETS
+
+#      NO-OFFSETS: Hex dump of section '.debug_loclists':
+# NO-OFFSETS-NEXT: 0x00000000 0e000000 05000800 00000000 01010201 ................
+##                            ^-------                            unit_length (4-byte)
+##                                     ^---                       version (2-byte)
+##                                         ^-                     address_size (1-byte)
+##                                           ^-                   segment_selector_size (1-byte)
+##                                              ^-------          offset_entry_count (4-byte)
+##                                                       ^-       DW_LLE_base_addressx
+##                                                         ^-     operands[0] (ULEB128) 0x01
+##                                                           ^-   DW_LLE_startx_endx
+##                                                             ^- operands[0] (ULEB128) 0x01
+# NO-OFFSETS-NEXT: 0x00000010 0200                                ..
+##                            ^-                                  operands[1] (ULEB128) 0x02
+##                              ^-                                location descriptions length (ULEB128) 0x00
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+DWARF:
+  debug_loclists:
+    - OffsetEntryCount: [[OFFSETENTRIES=<none>]]
+      Offsets:          [[OFFSETS=<none>]]
+      Lists:
+        - Entries:
+            - Operator: DW_LLE_base_addressx
+              Values:   [ 0x01 ]
+            - Operator: DW_LLE_startx_endx
+              Values:   [ 0x01, 0x02 ]
+
+## x) Test that when the "Offsets" entry is specified to be empty and the "OffsetEntryCount" is not specified,
+## the offsets array will be omitted.
+
+# RUN: yaml2obj --docnum=21 -DOFFSETS=[] %s -o %t22.o
+# RUN: llvm-readelf --hex-dump=.debug_loclists %t22.o | \
+# RUN:   FileCheck %s --check-prefix=NO-OFFSETS
+
+## y) Test that if "Offsets" is specified, the offsets array will be emitted accordingly, even when
+## the "OffsetEntryCount" is specified to be 0.
+
+# RUN: yaml2obj --docnum=21 -DOFFSETENTRIES=0 -DOFFSETS=[0x01,0x02,0x03] %s -o %t23.o
+# RUN: llvm-readelf --hex-dump=.debug_loclists %t23.o | \
+# RUN:   FileCheck %s --check-prefix=OFFSETS
+
+#      OFFSETS: Hex dump of section '.debug_loclists':
+# OFFSETS-NEXT: 0x00000000 0e000000 05000800 00000000 01000000 ................
+##                         ^-------                            unit_length (4-byte)
+##                                  ^---                       version (2-byte)
+##                                      ^-                     address_size (1-byte)
+##                                        ^-                   segment_selector_size (1-byte)
+##                                           ^-------          offset_entry_count (4-byte)
+##                                                    ^------- offsets[0] (4-byte)
+# OFFSETS-NEXT: 0x00000010 02000000 03000000 01010201 0200     ..............
+##                         ^-------                            offsets[1] (4-byte)
+##                                  ^-------                   offsets[2] (4-byte)
+##                                           ^-                DW_LLE_base_addressx
+##                                             ^-              operands[0] (ULEB128) 0x01
+##                                               ^-            DW_LLE_startx_endx
+##                                                 ^-          operands[0] (ULEB128) 0x01
+##                                                    ^-       operands[1] (ULEB128) 0x02
+##                                                      ^-     location descriptions length (ULEB128) 0x00

From 10851f9db5f7d163135374b8dfc945e1b4a9c7d6 Mon Sep 17 00:00:00 2001
From: Valeriy Savchenko <vsavchenko@apple.com>
Date: Wed, 22 Jul 2020 13:36:13 +0300
Subject: [PATCH 0919/1035] [analyzer][tests] Fix SATest update functionality

Summary:
Not all projects in the project map file might have newer results
for updating, we should handle this situation gracefully.

Additionally, not every user of the test system would want storing
reference results in git.  For this reason, git functionality is now
optional.

Differential Revision: https://reviews.llvm.org/D84303
---
 clang/utils/analyzer/SATest.py            |  5 ++--
 clang/utils/analyzer/SATestUpdateDiffs.py | 33 +++++++++--------------
 2 files changed, 15 insertions(+), 23 deletions(-)

diff --git a/clang/utils/analyzer/SATest.py b/clang/utils/analyzer/SATest.py
index 46e636ad2895e..86571902502f9 100755
--- a/clang/utils/analyzer/SATest.py
+++ b/clang/utils/analyzer/SATest.py
@@ -78,7 +78,7 @@ def update(parser, args):
 
     project_map = ProjectMap()
     for project in project_map.projects:
-        SATestUpdateDiffs.update_reference_results(project)
+        SATestUpdateDiffs.update_reference_results(project, args.git)
 
 
 def benchmark(parser, args):
@@ -277,7 +277,8 @@ def main():
         "update",
         help="Update static analyzer reference results based on the previous "
         "run of SATest build. Assumes that SATest build was just run.")
-    # TODO: add option to decide whether we should use git
+    upd_parser.add_argument("--git", action="store_true",
+                            help="Stage updated results using git.")
     upd_parser.set_defaults(func=update)
 
     # docker subcommand
diff --git a/clang/utils/analyzer/SATestUpdateDiffs.py b/clang/utils/analyzer/SATestUpdateDiffs.py
index 920fa15e4c6f5..69b3383beaf17 100644
--- a/clang/utils/analyzer/SATestUpdateDiffs.py
+++ b/clang/utils/analyzer/SATestUpdateDiffs.py
@@ -15,7 +15,7 @@
 Verbose = 0
 
 
-def update_reference_results(project: ProjectInfo):
+def update_reference_results(project: ProjectInfo, git: bool = False):
     test_info = SATestBuild.TestInfo(project)
     tester = SATestBuild.ProjectTester(test_info)
     project_dir = tester.get_project_dir()
@@ -27,9 +27,10 @@ def update_reference_results(project: ProjectInfo):
     created_results_path = tester.get_output_dir()
 
     if not os.path.exists(created_results_path):
-        print("New results not found, was SATestBuild.py previously run?",
+        print(f"Skipping project '{project.name}', "
+              f"it doesn't have newer results.",
               file=sys.stderr)
-        sys.exit(1)
+        return
 
     build_log_path = SATestBuild.get_build_log_path(ref_results_path)
     build_log_dir = os.path.dirname(os.path.abspath(build_log_path))
@@ -45,7 +46,8 @@ def run_cmd(command: str):
         # Remove reference results: in git, and then again for a good measure
         # with rm, as git might not remove things fully if there are empty
         # directories involved.
-        run_cmd(f"git rm -r -q '{ref_results_path}'")
+        if git:
+            run_cmd(f"git rm -r -q '{ref_results_path}'")
         shutil.rmtree(ref_results_path)
 
         # Replace reference results with a freshly computed once.
@@ -60,22 +62,11 @@ def run_cmd(command: str):
         # Clean up the generated difference results.
         SATestBuild.cleanup_reference_results(ref_results_path)
 
-        run_cmd(f"git add '{ref_results_path}'")
+        if git:
+            run_cmd(f"git add '{ref_results_path}'")
 
 
-# TODO: use argparse
-def main(argv):
-    if len(argv) == 2 and argv[1] in ("-h", "--help"):
-        print("Update static analyzer reference results based "
-              "\non the previous run of SATestBuild.py.\n"
-              "\nN.B.: Assumes that SATestBuild.py was just run",
-              file=sys.stderr)
-        sys.exit(1)
-
-    project_map = ProjectMap()
-    for project in project_map.projects:
-        update_reference_results(project)
-
-
-if __name__ == '__main__':
-    main(sys.argv)
+if __name__ == "__main__":
+    print("SATestUpdateDiffs.py should not be used on its own.")
+    print("Please use 'SATest.py update' instead")
+    sys.exit(1)

From 31c7a2fd5c9a5b980768bd06354517d99ac8fe87 Mon Sep 17 00:00:00 2001
From: Cameron McInally <mcinally@cray.com>
Date: Mon, 3 Aug 2020 10:19:33 -0500
Subject: [PATCH 0920/1035] [FPEnv] Don't transform FSUB(-0,X)->FNEG(X) in
 SelectionDAGBuilder.

This patch stops unconditionally transforming FSUB(-0,X) into an FNEG(X) while building the DAG. There is also one small change to handle the new FSUB(-0,X) similarly to FNEG(X) in the AMDGPU backend.

Differential Revision: https://reviews.llvm.org/D84056
---
 .../SelectionDAG/SelectionDAGBuilder.cpp      |  14 --
 .../SelectionDAG/SelectionDAGBuilder.h        |   2 +-
 llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp |  11 +-
 .../AMDGPU/fcanonicalize-elimination.ll       |   4 +-
 llvm/test/CodeGen/AMDGPU/fcanonicalize.f16.ll |  12 +-
 llvm/test/CodeGen/AMDGPU/fcanonicalize.ll     |   8 +-
 .../CodeGen/AMDGPU/fdiv32-to-rcp-folding.ll   |   2 +-
 llvm/test/CodeGen/AMDGPU/fmuladd.f16.ll       |  10 +-
 llvm/test/CodeGen/AMDGPU/fmuladd.f32.ll       |  10 +-
 llvm/test/CodeGen/AMDGPU/fneg-combines.ll     | 228 +++++++++---------
 llvm/test/CodeGen/AMDGPU/selectcc-opt.ll      |   4 +-
 llvm/test/CodeGen/AMDGPU/set-dx10.ll          |  12 +-
 12 files changed, 155 insertions(+), 162 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index c2a284af592d7..9c1517ea74140 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -3005,20 +3005,6 @@ void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {
   DAG.setRoot(DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));
 }
 
-void SelectionDAGBuilder::visitFSub(const User &I) {
-  // -0.0 - X --> fneg
-  Type *Ty = I.getType();
-  if (isa<Constant>(I.getOperand(0)) &&
-      I.getOperand(0) == ConstantFP::getZeroValueForNegation(Ty)) {
-    SDValue Op2 = getValue(I.getOperand(1));
-    setValue(&I, DAG.getNode(ISD::FNEG, getCurSDLoc(),
-                             Op2.getValueType(), Op2));
-    return;
-  }
-
-  visitBinary(I, ISD::FSUB);
-}
-
 void SelectionDAGBuilder::visitUnary(const User &I, unsigned Opcode) {
   SDNodeFlags Flags;
 
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
index f0b7fb0d52299..7bad055198140 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
@@ -692,7 +692,7 @@ class SelectionDAGBuilder {
   void visitAdd(const User &I)  { visitBinary(I, ISD::ADD); }
   void visitFAdd(const User &I) { visitBinary(I, ISD::FADD); }
   void visitSub(const User &I)  { visitBinary(I, ISD::SUB); }
-  void visitFSub(const User &I);
+  void visitFSub(const User &I) { visitBinary(I, ISD::FSUB); }
   void visitMul(const User &I)  { visitBinary(I, ISD::MUL); }
   void visitFMul(const User &I) { visitBinary(I, ISD::FMUL); }
   void visitURem(const User &I) { visitBinary(I, ISD::UREM); }
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index 3ffd9b79864a2..1f5d83d379495 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -3795,8 +3795,15 @@ SDValue AMDGPUTargetLowering::performFNegCombine(SDNode *N,
     SDValue Res = DAG.getNode(AMDGPUISD::FMED3, SL, VT, Ops, N0->getFlags());
     if (Res.getOpcode() != AMDGPUISD::FMED3)
       return SDValue(); // Op got folded away.
-    if (!N0.hasOneUse())
-      DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res));
+
+    if (!N0.hasOneUse()) {
+      SDValue Neg = DAG.getNode(ISD::FNEG, SL, VT, Res);
+      DAG.ReplaceAllUsesWith(N0, Neg);
+
+      for (SDNode *U : Neg->uses())
+        DCI.AddToWorklist(U);
+    }
+
     return Res;
   }
   case ISD::FP_EXTEND:
diff --git a/llvm/test/CodeGen/AMDGPU/fcanonicalize-elimination.ll b/llvm/test/CodeGen/AMDGPU/fcanonicalize-elimination.ll
index f7f075b095317..0ba4d8d82bdf8 100644
--- a/llvm/test/CodeGen/AMDGPU/fcanonicalize-elimination.ll
+++ b/llvm/test/CodeGen/AMDGPU/fcanonicalize-elimination.ll
@@ -311,7 +311,7 @@ define amdgpu_kernel void @test_no_fold_canonicalize_fneg_value_f32(float addrsp
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr inbounds float, float addrspace(1)* %arg, i32 %id
   %load = load float, float addrspace(1)* %gep, align 4
-  %v = fsub float -0.0, %load
+  %v = fneg float %load
   %canonicalized = tail call float @llvm.canonicalize.f32(float %v)
   store float %canonicalized, float addrspace(1)* %gep, align 4
   ret void
@@ -327,7 +327,7 @@ define amdgpu_kernel void @test_fold_canonicalize_fneg_value_f32(float addrspace
   %gep = getelementptr inbounds float, float addrspace(1)* %arg, i32 %id
   %load = load float, float addrspace(1)* %gep, align 4
   %v0 = fadd float %load, 0.0
-  %v = fsub float -0.0, %v0
+  %v = fneg float %v0
   %canonicalized = tail call float @llvm.canonicalize.f32(float %v)
   store float %canonicalized, float addrspace(1)* %gep, align 4
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/fcanonicalize.f16.ll b/llvm/test/CodeGen/AMDGPU/fcanonicalize.f16.ll
index 7a44d11ad091d..c6af4baee6509 100644
--- a/llvm/test/CodeGen/AMDGPU/fcanonicalize.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fcanonicalize.f16.ll
@@ -77,7 +77,7 @@ define amdgpu_kernel void @v_test_canonicalize_fabs_var_f16(half addrspace(1)* %
 define amdgpu_kernel void @v_test_canonicalize_fneg_fabs_var_f16(half addrspace(1)* %out) #1 {
   %val = load half, half addrspace(1)* %out
   %val.fabs = call half @llvm.fabs.f16(half %val)
-  %val.fabs.fneg = fsub half -0.0, %val.fabs
+  %val.fabs.fneg = fneg half %val.fabs
   %canonicalized = call half @llvm.canonicalize.f16(half %val.fabs.fneg)
   store half %canonicalized, half addrspace(1)* %out
   ret void
@@ -91,7 +91,7 @@ define amdgpu_kernel void @v_test_canonicalize_fneg_fabs_var_f16(half addrspace(
 ; CI: v_mul_f32_e32 {{v[0-9]+}}, 1.0, {{v[0-9]+}}
 define amdgpu_kernel void @v_test_canonicalize_fneg_var_f16(half addrspace(1)* %out) #1 {
   %val = load half, half addrspace(1)* %out
-  %val.fneg = fsub half -0.0, %val
+  %val.fneg = fneg half %val
   %canonicalized = call half @llvm.canonicalize.f16(half %val.fneg)
   store half %canonicalized, half addrspace(1)* %out
   ret void
@@ -103,7 +103,7 @@ define amdgpu_kernel void @v_test_canonicalize_fneg_var_f16(half addrspace(1)* %
 ; GFX89: {{flat|global}}_store_short v{{\[[0-9]+:[0-9]+\]}}, [[REG]]
 define amdgpu_kernel void @v_test_no_denormals_canonicalize_fneg_var_f16(half addrspace(1)* %out) #2 {
   %val = load half, half addrspace(1)* %out
-  %val.fneg = fsub half -0.0, %val
+  %val.fneg = fneg half %val
   %canonicalized = call half @llvm.canonicalize.f16(half %val.fneg)
   store half %canonicalized, half addrspace(1)* %out
   ret void
@@ -120,7 +120,7 @@ define amdgpu_kernel void @v_test_no_denormals_canonicalize_fneg_var_f16(half ad
 define amdgpu_kernel void @v_test_no_denormals_canonicalize_fneg_fabs_var_f16(half addrspace(1)* %out) #2 {
   %val = load half, half addrspace(1)* %out
   %val.fabs = call half @llvm.fabs.f16(half %val)
-  %val.fabs.fneg = fsub half -0.0, %val.fabs
+  %val.fabs.fneg = fneg half %val.fabs
   %canonicalized = call half @llvm.canonicalize.f16(half %val.fabs.fneg)
   store half %canonicalized, half addrspace(1)* %out
   ret void
@@ -323,7 +323,7 @@ define amdgpu_kernel void @v_test_canonicalize_fneg_fabs_var_v2f16(<2 x half> ad
   %gep = getelementptr <2 x half>, <2 x half> addrspace(1)* %out, i32 %tid
   %val = load <2 x half>, <2 x half> addrspace(1)* %gep
   %val.fabs = call <2 x half> @llvm.fabs.v2f16(<2 x half> %val)
-  %val.fabs.fneg = fsub <2 x half> <half -0.0, half -0.0>, %val.fabs
+  %val.fabs.fneg = fneg <2 x half> %val.fabs
   %canonicalized = call <2 x half> @llvm.canonicalize.v2f16(<2 x half> %val.fabs.fneg)
   store <2 x half> %canonicalized, <2 x half> addrspace(1)* %out
   ret void
@@ -340,7 +340,7 @@ define amdgpu_kernel void @v_test_canonicalize_fneg_var_v2f16(<2 x half> addrspa
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr <2 x half>, <2 x half> addrspace(1)* %out, i32 %tid
   %val = load <2 x half>, <2 x half> addrspace(1)* %gep
-  %fneg.val = fsub <2 x half> <half -0.0, half -0.0>, %val
+  %fneg.val = fneg <2 x half> %val
   %canonicalized = call <2 x half> @llvm.canonicalize.v2f16(<2 x half> %fneg.val)
   store <2 x half> %canonicalized, <2 x half> addrspace(1)* %out
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/fcanonicalize.ll b/llvm/test/CodeGen/AMDGPU/fcanonicalize.ll
index e302d59b1ac74..d95194a8716df 100644
--- a/llvm/test/CodeGen/AMDGPU/fcanonicalize.ll
+++ b/llvm/test/CodeGen/AMDGPU/fcanonicalize.ll
@@ -57,7 +57,7 @@ define amdgpu_kernel void @v_test_canonicalize_fabs_var_f32(float addrspace(1)*
 define amdgpu_kernel void @v_test_canonicalize_fneg_fabs_var_f32(float addrspace(1)* %out) #1 {
   %val = load float, float addrspace(1)* %out
   %val.fabs = call float @llvm.fabs.f32(float %val)
-  %val.fabs.fneg = fsub float -0.0, %val.fabs
+  %val.fabs.fneg = fneg float %val.fabs
   %canonicalized = call float @llvm.canonicalize.f32(float %val.fabs.fneg)
   store float %canonicalized, float addrspace(1)* %out
   ret void
@@ -69,7 +69,7 @@ define amdgpu_kernel void @v_test_canonicalize_fneg_fabs_var_f32(float addrspace
 ; GCN: {{flat|global}}_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[REG]]
 define amdgpu_kernel void @v_test_canonicalize_fneg_var_f32(float addrspace(1)* %out) #1 {
   %val = load float, float addrspace(1)* %out
-  %val.fneg = fsub float -0.0, %val
+  %val.fneg = fneg float %val
   %canonicalized = call float @llvm.canonicalize.f32(float %val.fneg)
   store float %canonicalized, float addrspace(1)* %out
   ret void
@@ -264,7 +264,7 @@ define amdgpu_kernel void @v_test_canonicalize_fabs_var_f64(double addrspace(1)*
 define amdgpu_kernel void @v_test_canonicalize_fneg_fabs_var_f64(double addrspace(1)* %out) #1 {
   %val = load double, double addrspace(1)* %out
   %val.fabs = call double @llvm.fabs.f64(double %val)
-  %val.fabs.fneg = fsub double -0.0, %val.fabs
+  %val.fabs.fneg = fneg double %val.fabs
   %canonicalized = call double @llvm.canonicalize.f64(double %val.fabs.fneg)
   store double %canonicalized, double addrspace(1)* %out
   ret void
@@ -275,7 +275,7 @@ define amdgpu_kernel void @v_test_canonicalize_fneg_fabs_var_f64(double addrspac
 ; GCN: {{flat|global}}_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, [[REG]]
 define amdgpu_kernel void @v_test_canonicalize_fneg_var_f64(double addrspace(1)* %out) #1 {
   %val = load double, double addrspace(1)* %out
-  %val.fneg = fsub double -0.0, %val
+  %val.fneg = fneg double %val
   %canonicalized = call double @llvm.canonicalize.f64(double %val.fneg)
   store double %canonicalized, double addrspace(1)* %out
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/fdiv32-to-rcp-folding.ll b/llvm/test/CodeGen/AMDGPU/fdiv32-to-rcp-folding.ll
index 197bdd77d2544..d62155c1220dc 100644
--- a/llvm/test/CodeGen/AMDGPU/fdiv32-to-rcp-folding.ll
+++ b/llvm/test/CodeGen/AMDGPU/fdiv32-to-rcp-folding.ll
@@ -311,7 +311,7 @@ define amdgpu_kernel void @div_v4_c_by_x_25ulp(<4 x float> addrspace(1)* %arg) {
 ; GCN:            global_store_dwordx4
 define amdgpu_kernel void @div_v4_c_by_minus_x_25ulp(<4 x float> addrspace(1)* %arg) {
   %load = load <4 x float>, <4 x float> addrspace(1)* %arg, align 16
-  %neg = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %load
+  %neg = fneg <4 x float> %load
   %div = fdiv <4 x float> <float 2.000000e+00, float 1.000000e+00, float -1.000000e+00, float -2.000000e+00>, %neg, !fpmath !0
   store <4 x float> %div, <4 x float> addrspace(1)* %arg, align 16
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/fmuladd.f16.ll b/llvm/test/CodeGen/AMDGPU/fmuladd.f16.ll
index cd19e237be709..293f59275372d 100644
--- a/llvm/test/CodeGen/AMDGPU/fmuladd.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmuladd.f16.ll
@@ -256,7 +256,7 @@ define amdgpu_kernel void @fmuladd_neg_2.0_neg_a_b_f16(half addrspace(1)* %out,
   %r1 = load volatile half, half addrspace(1)* %gep.0
   %r2 = load volatile half, half addrspace(1)* %gep.1
 
-  %r1.fneg = fsub half -0.000000e+00, %r1
+  %r1.fneg = fneg half %r1
 
   %r3 = tail call half @llvm.fmuladd.f16(half -2.0, half %r1.fneg, half %r2)
   store half %r3, half addrspace(1)* %gep.out
@@ -287,7 +287,7 @@ define amdgpu_kernel void @fmuladd_2.0_neg_a_b_f16(half addrspace(1)* %out, half
   %r1 = load volatile half, half addrspace(1)* %gep.0
   %r2 = load volatile half, half addrspace(1)* %gep.1
 
-  %r1.fneg = fsub half -0.000000e+00, %r1
+  %r1.fneg = fneg half %r1
 
   %r3 = tail call half @llvm.fmuladd.f16(half 2.0, half %r1.fneg, half %r2)
   store half %r3, half addrspace(1)* %gep.out
@@ -312,7 +312,7 @@ define amdgpu_kernel void @fmuladd_2.0_a_neg_b_f16(half addrspace(1)* %out, half
   %r1 = load volatile half, half addrspace(1)* %gep.0
   %r2 = load volatile half, half addrspace(1)* %gep.1
 
-  %r2.fneg = fsub half -0.000000e+00, %r2
+  %r2.fneg = fneg half %r2
 
   %r3 = tail call half @llvm.fmuladd.f16(half 2.0, half %r1, half %r2.fneg)
   store half %r3, half addrspace(1)* %gep.out
@@ -494,8 +494,8 @@ define amdgpu_kernel void @neg_neg_mad_f16(half addrspace(1)* noalias nocapture
   %a = load volatile half, half addrspace(1)* %gep0, align 2
   %b = load volatile half, half addrspace(1)* %gep1, align 2
   %c = load volatile half, half addrspace(1)* %gep2, align 2
-  %nega = fsub half -0.000000e+00, %a
-  %negb = fsub half -0.000000e+00, %b
+  %nega = fneg half %a
+  %negb = fneg half %b
   %mul = fmul half %nega, %negb
   %sub = fadd half %mul, %c
   store half %sub, half addrspace(1)* %outgep, align 2
diff --git a/llvm/test/CodeGen/AMDGPU/fmuladd.f32.ll b/llvm/test/CodeGen/AMDGPU/fmuladd.f32.ll
index 19862fbf13fe6..b97d36c046aff 100644
--- a/llvm/test/CodeGen/AMDGPU/fmuladd.f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmuladd.f32.ll
@@ -274,7 +274,7 @@ define amdgpu_kernel void @fmuladd_neg_2.0_neg_a_b_f32(float addrspace(1)* %out,
   %r1 = load volatile float, float addrspace(1)* %gep.0
   %r2 = load volatile float, float addrspace(1)* %gep.1
 
-  %r1.fneg = fsub float -0.000000e+00, %r1
+  %r1.fneg = fneg float %r1
 
   %r3 = tail call float @llvm.fmuladd.f32(float -2.0, float %r1.fneg, float %r2)
   store float %r3, float addrspace(1)* %gep.out
@@ -307,7 +307,7 @@ define amdgpu_kernel void @fmuladd_2.0_neg_a_b_f32(float addrspace(1)* %out, flo
   %r1 = load volatile float, float addrspace(1)* %gep.0
   %r2 = load volatile float, float addrspace(1)* %gep.1
 
-  %r1.fneg = fsub float -0.000000e+00, %r1
+  %r1.fneg = fneg float %r1
 
   %r3 = tail call float @llvm.fmuladd.f32(float 2.0, float %r1.fneg, float %r2)
   store float %r3, float addrspace(1)* %gep.out
@@ -339,7 +339,7 @@ define amdgpu_kernel void @fmuladd_2.0_a_neg_b_f32(float addrspace(1)* %out, flo
   %r1 = load volatile float, float addrspace(1)* %gep.0
   %r2 = load volatile float, float addrspace(1)* %gep.1
 
-  %r2.fneg = fsub float -0.000000e+00, %r2
+  %r2.fneg = fneg float %r2
 
   %r3 = tail call float @llvm.fmuladd.f32(float 2.0, float %r1, float %r2.fneg)
   store float %r3, float addrspace(1)* %gep.out
@@ -517,8 +517,8 @@ define amdgpu_kernel void @neg_neg_mad_f32(float addrspace(1)* noalias nocapture
   %a = load volatile float, float addrspace(1)* %gep0, align 4
   %b = load volatile float, float addrspace(1)* %gep1, align 4
   %c = load volatile float, float addrspace(1)* %gep2, align 4
-  %nega = fsub float -0.000000e+00, %a
-  %negb = fsub float -0.000000e+00, %b
+  %nega = fneg float %a
+  %negb = fneg float %b
   %mul = fmul float %nega, %negb
   %sub = fadd float %mul, %c
   store float %sub, float addrspace(1)* %outgep, align 4
diff --git a/llvm/test/CodeGen/AMDGPU/fneg-combines.ll b/llvm/test/CodeGen/AMDGPU/fneg-combines.ll
index 54ccc8fd870d4..01b0e6d17d1ac 100644
--- a/llvm/test/CodeGen/AMDGPU/fneg-combines.ll
+++ b/llvm/test/CodeGen/AMDGPU/fneg-combines.ll
@@ -26,7 +26,7 @@ define amdgpu_kernel void @v_fneg_add_f32(float addrspace(1)* %out, float addrsp
   %a = load volatile float, float addrspace(1)* %a.gep
   %b = load volatile float, float addrspace(1)* %b.gep
   %add = fadd float %a, %b
-  %fneg = fsub float -0.000000e+00, %add
+  %fneg = fneg float %add
   store float %fneg, float addrspace(1)* %out.gep
   ret void
 }
@@ -47,7 +47,7 @@ define amdgpu_kernel void @v_fneg_add_store_use_add_f32(float addrspace(1)* %out
   %a = load volatile float, float addrspace(1)* %a.gep
   %b = load volatile float, float addrspace(1)* %b.gep
   %add = fadd float %a, %b
-  %fneg = fsub float -0.000000e+00, %add
+  %fneg = fneg float %add
   store volatile float %fneg, float addrspace(1)* %out
   store volatile float %add, float addrspace(1)* %out
   ret void
@@ -75,7 +75,7 @@ define amdgpu_kernel void @v_fneg_add_multi_use_add_f32(float addrspace(1)* %out
   %a = load volatile float, float addrspace(1)* %a.gep
   %b = load volatile float, float addrspace(1)* %b.gep
   %add = fadd float %a, %b
-  %fneg = fsub float -0.000000e+00, %add
+  %fneg = fneg float %add
   %use1 = fmul float %add, 4.0
   store volatile float %fneg, float addrspace(1)* %out
   store volatile float %use1, float addrspace(1)* %out
@@ -100,9 +100,9 @@ define amdgpu_kernel void @v_fneg_add_fneg_x_f32(float addrspace(1)* %out, float
   %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i64 %tid.ext
   %a = load volatile float, float addrspace(1)* %a.gep
   %b = load volatile float, float addrspace(1)* %b.gep
-  %fneg.a = fsub float -0.000000e+00, %a
+  %fneg.a = fneg float %a
   %add = fadd float %fneg.a, %b
-  %fneg = fsub float -0.000000e+00, %add
+  %fneg = fneg float %add
   store volatile float %fneg, float addrspace(1)* %out
   ret void
 }
@@ -124,9 +124,9 @@ define amdgpu_kernel void @v_fneg_add_x_fneg_f32(float addrspace(1)* %out, float
   %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i64 %tid.ext
   %a = load volatile float, float addrspace(1)* %a.gep
   %b = load volatile float, float addrspace(1)* %b.gep
-  %fneg.b = fsub float -0.000000e+00, %b
+  %fneg.b = fneg float %b
   %add = fadd float %a, %fneg.b
-  %fneg = fsub float -0.000000e+00, %add
+  %fneg = fneg float %add
   store volatile float %fneg, float addrspace(1)* %out
   ret void
 }
@@ -148,10 +148,10 @@ define amdgpu_kernel void @v_fneg_add_fneg_fneg_f32(float addrspace(1)* %out, fl
   %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i64 %tid.ext
   %a = load volatile float, float addrspace(1)* %a.gep
   %b = load volatile float, float addrspace(1)* %b.gep
-  %fneg.a = fsub float -0.000000e+00, %a
-  %fneg.b = fsub float -0.000000e+00, %b
+  %fneg.a = fneg float %a
+  %fneg.b = fneg float %b
   %add = fadd float %fneg.a, %fneg.b
-  %fneg = fsub float -0.000000e+00, %add
+  %fneg = fneg float %add
   store volatile float %fneg, float addrspace(1)* %out
   ret void
 }
@@ -177,9 +177,9 @@ define amdgpu_kernel void @v_fneg_add_store_use_fneg_x_f32(float addrspace(1)* %
   %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i64 %tid.ext
   %a = load volatile float, float addrspace(1)* %a.gep
   %b = load volatile float, float addrspace(1)* %b.gep
-  %fneg.a = fsub float -0.000000e+00, %a
+  %fneg.a = fneg float %a
   %add = fadd float %fneg.a, %b
-  %fneg = fsub float -0.000000e+00, %add
+  %fneg = fneg float %add
   store volatile float %fneg, float addrspace(1)* %out
   store volatile float %fneg.a, float addrspace(1)* %out
   ret void
@@ -205,9 +205,9 @@ define amdgpu_kernel void @v_fneg_add_multi_use_fneg_x_f32(float addrspace(1)* %
   %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i64 %tid.ext
   %a = load volatile float, float addrspace(1)* %a.gep
   %b = load volatile float, float addrspace(1)* %b.gep
-  %fneg.a = fsub float -0.000000e+00, %a
+  %fneg.a = fneg float %a
   %add = fadd float %fneg.a, %b
-  %fneg = fsub float -0.000000e+00, %add
+  %fneg = fneg float %add
   %use1 = fmul float %fneg.a, %c
   store volatile float %fneg, float addrspace(1)* %out
   store volatile float %use1, float addrspace(1)* %out
@@ -226,7 +226,7 @@ define amdgpu_ps float @fneg_fadd_0(float inreg %tmp2, float inreg %tmp6, <4 x i
   %tmp9 = fmul reassoc nnan arcp contract float 0.000000e+00, %tmp8
   %.i188 = fadd float %tmp9, 0.000000e+00
   %tmp10 = fcmp uge float %.i188, %tmp2
-  %tmp11 = fsub float -0.000000e+00, %.i188
+  %tmp11 = fneg float %.i188
   %.i092 = select i1 %tmp10, float %tmp2, float %tmp11
   %tmp12 = fcmp ule float %.i092, 0.000000e+00
   %.i198 = select i1 %tmp12, float 0.000000e+00, float 0x7FF8000000000000
@@ -249,7 +249,7 @@ define amdgpu_ps float @fneg_fadd_0_nsz(float inreg %tmp2, float inreg %tmp6, <4
   %tmp9 = fmul reassoc nnan arcp contract float 0.000000e+00, %tmp8
   %.i188 = fadd float %tmp9, 0.000000e+00
   %tmp10 = fcmp uge float %.i188, %tmp2
-  %tmp11 = fsub float -0.000000e+00, %.i188
+  %tmp11 = fneg float %.i188
   %.i092 = select i1 %tmp10, float %tmp2, float %tmp11
   %tmp12 = fcmp ule float %.i092, 0.000000e+00
   %.i198 = select i1 %tmp12, float 0.000000e+00, float 0x7FF8000000000000
@@ -274,7 +274,7 @@ define amdgpu_kernel void @v_fneg_mul_f32(float addrspace(1)* %out, float addrsp
   %a = load volatile float, float addrspace(1)* %a.gep
   %b = load volatile float, float addrspace(1)* %b.gep
   %mul = fmul float %a, %b
-  %fneg = fsub float -0.000000e+00, %mul
+  %fneg = fneg float %mul
   store float %fneg, float addrspace(1)* %out.gep
   ret void
 }
@@ -295,7 +295,7 @@ define amdgpu_kernel void @v_fneg_mul_store_use_mul_f32(float addrspace(1)* %out
   %a = load volatile float, float addrspace(1)* %a.gep
   %b = load volatile float, float addrspace(1)* %b.gep
   %mul = fmul float %a, %b
-  %fneg = fsub float -0.000000e+00, %mul
+  %fneg = fneg float %mul
   store volatile float %fneg, float addrspace(1)* %out
   store volatile float %mul, float addrspace(1)* %out
   ret void
@@ -318,7 +318,7 @@ define amdgpu_kernel void @v_fneg_mul_multi_use_mul_f32(float addrspace(1)* %out
   %a = load volatile float, float addrspace(1)* %a.gep
   %b = load volatile float, float addrspace(1)* %b.gep
   %mul = fmul float %a, %b
-  %fneg = fsub float -0.000000e+00, %mul
+  %fneg = fneg float %mul
   %use1 = fmul float %mul, 4.0
   store volatile float %fneg, float addrspace(1)* %out
   store volatile float %use1, float addrspace(1)* %out
@@ -338,9 +338,9 @@ define amdgpu_kernel void @v_fneg_mul_fneg_x_f32(float addrspace(1)* %out, float
   %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i64 %tid.ext
   %a = load volatile float, float addrspace(1)* %a.gep
   %b = load volatile float, float addrspace(1)* %b.gep
-  %fneg.a = fsub float -0.000000e+00, %a
+  %fneg.a = fneg float %a
   %mul = fmul float %fneg.a, %b
-  %fneg = fsub float -0.000000e+00, %mul
+  %fneg = fneg float %mul
   store volatile float %fneg, float addrspace(1)* %out
   ret void
 }
@@ -358,9 +358,9 @@ define amdgpu_kernel void @v_fneg_mul_x_fneg_f32(float addrspace(1)* %out, float
   %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i64 %tid.ext
   %a = load volatile float, float addrspace(1)* %a.gep
   %b = load volatile float, float addrspace(1)* %b.gep
-  %fneg.b = fsub float -0.000000e+00, %b
+  %fneg.b = fneg float %b
   %mul = fmul float %a, %fneg.b
-  %fneg = fsub float -0.000000e+00, %mul
+  %fneg = fneg float %mul
   store volatile float %fneg, float addrspace(1)* %out
   ret void
 }
@@ -378,10 +378,10 @@ define amdgpu_kernel void @v_fneg_mul_fneg_fneg_f32(float addrspace(1)* %out, fl
   %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i64 %tid.ext
   %a = load volatile float, float addrspace(1)* %a.gep
   %b = load volatile float, float addrspace(1)* %b.gep
-  %fneg.a = fsub float -0.000000e+00, %a
-  %fneg.b = fsub float -0.000000e+00, %b
+  %fneg.a = fneg float %a
+  %fneg.b = fneg float %b
   %mul = fmul float %fneg.a, %fneg.b
-  %fneg = fsub float -0.000000e+00, %mul
+  %fneg = fneg float %mul
   store volatile float %fneg, float addrspace(1)* %out
   ret void
 }
@@ -402,9 +402,9 @@ define amdgpu_kernel void @v_fneg_mul_store_use_fneg_x_f32(float addrspace(1)* %
   %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i64 %tid.ext
   %a = load volatile float, float addrspace(1)* %a.gep
   %b = load volatile float, float addrspace(1)* %b.gep
-  %fneg.a = fsub float -0.000000e+00, %a
+  %fneg.a = fneg float %a
   %mul = fmul float %fneg.a, %b
-  %fneg = fsub float -0.000000e+00, %mul
+  %fneg = fneg float %mul
   store volatile float %fneg, float addrspace(1)* %out
   store volatile float %fneg.a, float addrspace(1)* %out
   ret void
@@ -425,9 +425,9 @@ define amdgpu_kernel void @v_fneg_mul_multi_use_fneg_x_f32(float addrspace(1)* %
   %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i64 %tid.ext
   %a = load volatile float, float addrspace(1)* %a.gep
   %b = load volatile float, float addrspace(1)* %b.gep
-  %fneg.a = fsub float -0.000000e+00, %a
+  %fneg.a = fneg float %a
   %mul = fmul float %fneg.a, %b
-  %fneg = fsub float -0.000000e+00, %mul
+  %fneg = fneg float %mul
   %use1 = fmul float %fneg.a, %c
   store volatile float %fneg, float addrspace(1)* %out
   store volatile float %use1, float addrspace(1)* %out
@@ -454,7 +454,7 @@ define amdgpu_kernel void @v_fneg_minnum_f32_ieee(float addrspace(1)* %out, floa
   %a = load volatile float, float addrspace(1)* %a.gep
   %b = load volatile float, float addrspace(1)* %b.gep
   %min = call float @llvm.minnum.f32(float %a, float %b)
-  %fneg = fsub float -0.000000e+00, %min
+  %fneg = fneg float %min
   store float %fneg, float addrspace(1)* %out.gep
   ret void
 }
@@ -466,7 +466,7 @@ define amdgpu_kernel void @v_fneg_minnum_f32_ieee(float addrspace(1)* %out, floa
 ; GCN-NEXT: ; return
 define amdgpu_ps float @v_fneg_minnum_f32_no_ieee(float %a, float %b) #0 {
   %min = call float @llvm.minnum.f32(float %a, float %b)
-  %fneg = fsub float -0.000000e+00, %min
+  %fneg = fneg float %min
   ret float %fneg
 }
 
@@ -509,7 +509,7 @@ define amdgpu_kernel void @v_fneg_posk_minnum_f32_ieee(float addrspace(1)* %out,
   %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i64 %tid.ext
   %a = load volatile float, float addrspace(1)* %a.gep
   %min = call float @llvm.minnum.f32(float 4.0, float %a)
-  %fneg = fsub float -0.000000e+00, %min
+  %fneg = fneg float %min
   store float %fneg, float addrspace(1)* %out.gep
   ret void
 }
@@ -520,7 +520,7 @@ define amdgpu_kernel void @v_fneg_posk_minnum_f32_ieee(float addrspace(1)* %out,
 ; GCN-NEXT: ; return
 define amdgpu_ps float @v_fneg_posk_minnum_f32_no_ieee(float %a) #0 {
   %min = call float @llvm.minnum.f32(float 4.0, float %a)
-  %fneg = fsub float -0.000000e+00, %min
+  %fneg = fneg float %min
   ret float %fneg
 }
 
@@ -536,7 +536,7 @@ define amdgpu_kernel void @v_fneg_negk_minnum_f32_ieee(float addrspace(1)* %out,
   %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i64 %tid.ext
   %a = load volatile float, float addrspace(1)* %a.gep
   %min = call float @llvm.minnum.f32(float -4.0, float %a)
-  %fneg = fsub float -0.000000e+00, %min
+  %fneg = fneg float %min
   store float %fneg, float addrspace(1)* %out.gep
   ret void
 }
@@ -547,7 +547,7 @@ define amdgpu_kernel void @v_fneg_negk_minnum_f32_ieee(float addrspace(1)* %out,
 ; GCN-NEXT: ; return
 define amdgpu_ps float @v_fneg_negk_minnum_f32_no_ieee(float %a) #0 {
   %min = call float @llvm.minnum.f32(float -4.0, float %a)
-  %fneg = fsub float -0.000000e+00, %min
+  %fneg = fneg float %min
   ret float %fneg
 }
 
@@ -562,7 +562,7 @@ define amdgpu_kernel void @v_fneg_0_minnum_f32(float addrspace(1)* %out, float a
   %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i64 %tid.ext
   %a = load volatile float, float addrspace(1)* %a.gep
   %min = call float @llvm.minnum.f32(float 0.0, float %a)
-  %fneg = fsub float -0.000000e+00, %min
+  %fneg = fneg float %min
   store float %fneg, float addrspace(1)* %out.gep
   ret void
 }
@@ -579,7 +579,7 @@ define amdgpu_kernel void @v_fneg_neg0_minnum_f32_ieee(float addrspace(1)* %out,
   %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i64 %tid.ext
   %a = load volatile float, float addrspace(1)* %a.gep
   %min = call float @llvm.minnum.f32(float -0.0, float %a)
-  %fneg = fsub float -0.000000e+00, %min
+  %fneg = fneg float %min
   store float %fneg, float addrspace(1)* %out.gep
   ret void
 }
@@ -602,7 +602,7 @@ define amdgpu_kernel void @v_fneg_inv2pi_minnum_f32(float addrspace(1)* %out, fl
   %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i64 %tid.ext
   %a = load volatile float, float addrspace(1)* %a.gep
   %min = call float @llvm.minnum.f32(float 0x3FC45F3060000000, float %a)
-  %fneg = fsub float -0.000000e+00, %min
+  %fneg = fneg float %min
   store float %fneg, float addrspace(1)* %out.gep
   ret void
 }
@@ -624,7 +624,7 @@ define amdgpu_kernel void @v_fneg_neg_inv2pi_minnum_f32(float addrspace(1)* %out
   %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i64 %tid.ext
   %a = load volatile float, float addrspace(1)* %a.gep
   %min = call float @llvm.minnum.f32(float 0xBFC45F3060000000, float %a)
-  %fneg = fsub float -0.000000e+00, %min
+  %fneg = fneg float %min
   store float %fneg, float addrspace(1)* %out.gep
   ret void
 }
@@ -730,7 +730,7 @@ define amdgpu_kernel void @v_fneg_neg_inv2pi_minnum_f64(double addrspace(1)* %ou
 ; GCN-NEXT: ; return
 define amdgpu_ps float @v_fneg_neg0_minnum_f32_no_ieee(float %a) #0 {
   %min = call float @llvm.minnum.f32(float -0.0, float %a)
-  %fneg = fsub float -0.000000e+00, %min
+  %fneg = fneg float %min
   ret float %fneg
 }
 
@@ -750,7 +750,7 @@ define amdgpu_kernel void @v_fneg_0_minnum_foldable_use_f32_ieee(float addrspace
   %a = load volatile float, float addrspace(1)* %a.gep
   %b = load volatile float, float addrspace(1)* %b.gep
   %min = call float @llvm.minnum.f32(float 0.0, float %a)
-  %fneg = fsub float -0.000000e+00, %min
+  %fneg = fneg float %min
   %mul = fmul float %fneg, %b
   store float %mul, float addrspace(1)* %out.gep
   ret void
@@ -779,7 +779,7 @@ define amdgpu_kernel void @v_fneg_inv2pi_minnum_foldable_use_f32(float addrspace
   %a = load volatile float, float addrspace(1)* %a.gep
   %b = load volatile float, float addrspace(1)* %b.gep
   %min = call float @llvm.minnum.f32(float 0x3FC45F3060000000, float %a)
-  %fneg = fsub float -0.000000e+00, %min
+  %fneg = fneg float %min
   %mul = fmul float %fneg, %b
   store float %mul, float addrspace(1)* %out.gep
   ret void
@@ -793,7 +793,7 @@ define amdgpu_kernel void @v_fneg_inv2pi_minnum_foldable_use_f32(float addrspace
 ; GCN-NEXT: ; return
 define amdgpu_ps float @v_fneg_0_minnum_foldable_use_f32_no_ieee(float %a, float %b) #0 {
   %min = call float @llvm.minnum.f32(float 0.0, float %a)
-  %fneg = fsub float -0.000000e+00, %min
+  %fneg = fneg float %min
   %mul = fmul float %fneg, %b
   ret float %mul
 }
@@ -816,7 +816,7 @@ define amdgpu_kernel void @v_fneg_minnum_multi_use_minnum_f32_ieee(float addrspa
   %a = load volatile float, float addrspace(1)* %a.gep
   %b = load volatile float, float addrspace(1)* %b.gep
   %min = call float @llvm.minnum.f32(float %a, float %b)
-  %fneg = fsub float -0.000000e+00, %min
+  %fneg = fneg float %min
   %use1 = fmul float %min, 4.0
   store volatile float %fneg, float addrspace(1)* %out
   store volatile float %use1, float addrspace(1)* %out
@@ -831,7 +831,7 @@ define amdgpu_kernel void @v_fneg_minnum_multi_use_minnum_f32_ieee(float addrspa
 ; GCN-NEXT: ; return
 define amdgpu_ps <2 x float> @v_fneg_minnum_multi_use_minnum_f32_no_ieee(float %a, float %b) #0 {
   %min = call float @llvm.minnum.f32(float %a, float %b)
-  %fneg = fsub float -0.000000e+00, %min
+  %fneg = fneg float %min
   %use1 = fmul float %min, 4.0
   %ins0 = insertelement <2 x float> undef, float %fneg, i32 0
   %ins1 = insertelement <2 x float> %ins0, float %use1, i32 1
@@ -859,7 +859,7 @@ define amdgpu_kernel void @v_fneg_maxnum_f32_ieee(float addrspace(1)* %out, floa
   %a = load volatile float, float addrspace(1)* %a.gep
   %b = load volatile float, float addrspace(1)* %b.gep
   %max = call float @llvm.maxnum.f32(float %a, float %b)
-  %fneg = fsub float -0.000000e+00, %max
+  %fneg = fneg float %max
   store float %fneg, float addrspace(1)* %out.gep
   ret void
 }
@@ -871,7 +871,7 @@ define amdgpu_kernel void @v_fneg_maxnum_f32_ieee(float addrspace(1)* %out, floa
 ; GCN-NEXT: ; return
 define amdgpu_ps float @v_fneg_maxnum_f32_no_ieee(float %a, float %b) #0 {
   %max = call float @llvm.maxnum.f32(float %a, float %b)
-  %fneg = fsub float -0.000000e+00, %max
+  %fneg = fneg float %max
   ret float %fneg
 }
 
@@ -914,7 +914,7 @@ define amdgpu_kernel void @v_fneg_posk_maxnum_f32_ieee(float addrspace(1)* %out,
   %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i64 %tid.ext
   %a = load volatile float, float addrspace(1)* %a.gep
   %max = call float @llvm.maxnum.f32(float 4.0, float %a)
-  %fneg = fsub float -0.000000e+00, %max
+  %fneg = fneg float %max
   store float %fneg, float addrspace(1)* %out.gep
   ret void
 }
@@ -925,7 +925,7 @@ define amdgpu_kernel void @v_fneg_posk_maxnum_f32_ieee(float addrspace(1)* %out,
 ; GCN-NEXT: ; return
 define amdgpu_ps float @v_fneg_posk_maxnum_f32_no_ieee(float %a) #0 {
   %max = call float @llvm.maxnum.f32(float 4.0, float %a)
-  %fneg = fsub float -0.000000e+00, %max
+  %fneg = fneg float %max
   ret float %fneg
 }
 
@@ -941,7 +941,7 @@ define amdgpu_kernel void @v_fneg_negk_maxnum_f32_ieee(float addrspace(1)* %out,
   %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i64 %tid.ext
   %a = load volatile float, float addrspace(1)* %a.gep
   %max = call float @llvm.maxnum.f32(float -4.0, float %a)
-  %fneg = fsub float -0.000000e+00, %max
+  %fneg = fneg float %max
   store float %fneg, float addrspace(1)* %out.gep
   ret void
 }
@@ -952,7 +952,7 @@ define amdgpu_kernel void @v_fneg_negk_maxnum_f32_ieee(float addrspace(1)* %out,
 ; GCN-NEXT: ; return
 define amdgpu_ps float @v_fneg_negk_maxnum_f32_no_ieee(float %a) #0 {
   %max = call float @llvm.maxnum.f32(float -4.0, float %a)
-  %fneg = fsub float -0.000000e+00, %max
+  %fneg = fneg float %max
   ret float %fneg
 }
 
@@ -967,7 +967,7 @@ define amdgpu_kernel void @v_fneg_0_maxnum_f32(float addrspace(1)* %out, float a
   %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i64 %tid.ext
   %a = load volatile float, float addrspace(1)* %a.gep
   %max = call float @llvm.maxnum.f32(float 0.0, float %a)
-  %fneg = fsub float -0.000000e+00, %max
+  %fneg = fneg float %max
   store float %fneg, float addrspace(1)* %out.gep
   ret void
 }
@@ -984,7 +984,7 @@ define amdgpu_kernel void @v_fneg_neg0_maxnum_f32_ieee(float addrspace(1)* %out,
   %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i64 %tid.ext
   %a = load volatile float, float addrspace(1)* %a.gep
   %max = call float @llvm.maxnum.f32(float -0.0, float %a)
-  %fneg = fsub float -0.000000e+00, %max
+  %fneg = fneg float %max
   store float %fneg, float addrspace(1)* %out.gep
   ret void
 }
@@ -995,7 +995,7 @@ define amdgpu_kernel void @v_fneg_neg0_maxnum_f32_ieee(float addrspace(1)* %out,
 ; GCN-NEXT: ; return
 define amdgpu_ps float @v_fneg_neg0_maxnum_f32_no_ieee(float %a) #0 {
   %max = call float @llvm.maxnum.f32(float -0.0, float %a)
-  %fneg = fsub float -0.000000e+00, %max
+  %fneg = fneg float %max
   ret float %fneg
 }
 
@@ -1015,7 +1015,7 @@ define amdgpu_kernel void @v_fneg_0_maxnum_foldable_use_f32_ieee(float addrspace
   %a = load volatile float, float addrspace(1)* %a.gep
   %b = load volatile float, float addrspace(1)* %b.gep
   %max = call float @llvm.maxnum.f32(float 0.0, float %a)
-  %fneg = fsub float -0.000000e+00, %max
+  %fneg = fneg float %max
   %mul = fmul float %fneg, %b
   store float %mul, float addrspace(1)* %out.gep
   ret void
@@ -1029,7 +1029,7 @@ define amdgpu_kernel void @v_fneg_0_maxnum_foldable_use_f32_ieee(float addrspace
 ; GCN-NEXT: ; return
 define amdgpu_ps float @v_fneg_0_maxnum_foldable_use_f32_no_ieee(float %a, float %b) #0 {
   %max = call float @llvm.maxnum.f32(float 0.0, float %a)
-  %fneg = fsub float -0.000000e+00, %max
+  %fneg = fneg float %max
   %mul = fmul float %fneg, %b
   ret float %mul
 }
@@ -1052,7 +1052,7 @@ define amdgpu_kernel void @v_fneg_maxnum_multi_use_maxnum_f32_ieee(float addrspa
   %a = load volatile float, float addrspace(1)* %a.gep
   %b = load volatile float, float addrspace(1)* %b.gep
   %max = call float @llvm.maxnum.f32(float %a, float %b)
-  %fneg = fsub float -0.000000e+00, %max
+  %fneg = fneg float %max
   %use1 = fmul float %max, 4.0
   store volatile float %fneg, float addrspace(1)* %out
   store volatile float %use1, float addrspace(1)* %out
@@ -1067,7 +1067,7 @@ define amdgpu_kernel void @v_fneg_maxnum_multi_use_maxnum_f32_ieee(float addrspa
 ; GCN-NEXT: ; return
 define amdgpu_ps <2 x float> @v_fneg_maxnum_multi_use_maxnum_f32_no_ieee(float %a, float %b) #0 {
   %max = call float @llvm.maxnum.f32(float %a, float %b)
-  %fneg = fsub float -0.000000e+00, %max
+  %fneg = fneg float %max
   %use1 = fmul float %max, 4.0
   %ins0 = insertelement <2 x float> undef, float %fneg, i32 0
   %ins1 = insertelement <2 x float> %ins0, float %use1, i32 1
@@ -1099,7 +1099,7 @@ define amdgpu_kernel void @v_fneg_fma_f32(float addrspace(1)* %out, float addrsp
   %b = load volatile float, float addrspace(1)* %b.gep
   %c = load volatile float, float addrspace(1)* %c.gep
   %fma = call float @llvm.fma.f32(float %a, float %b, float %c)
-  %fneg = fsub float -0.000000e+00, %fma
+  %fneg = fneg float %fma
   store float %fneg, float addrspace(1)* %out.gep
   ret void
 }
@@ -1123,7 +1123,7 @@ define amdgpu_kernel void @v_fneg_fma_store_use_fma_f32(float addrspace(1)* %out
   %b = load volatile float, float addrspace(1)* %b.gep
   %c = load volatile float, float addrspace(1)* %c.gep
   %fma = call float @llvm.fma.f32(float %a, float %b, float %c)
-  %fneg = fsub float -0.000000e+00, %fma
+  %fneg = fneg float %fma
   store volatile float %fneg, float addrspace(1)* %out
   store volatile float %fma, float addrspace(1)* %out
   ret void
@@ -1154,7 +1154,7 @@ define amdgpu_kernel void @v_fneg_fma_multi_use_fma_f32(float addrspace(1)* %out
   %b = load volatile float, float addrspace(1)* %b.gep
   %c = load volatile float, float addrspace(1)* %c.gep
   %fma = call float @llvm.fma.f32(float %a, float %b, float %c)
-  %fneg = fsub float -0.000000e+00, %fma
+  %fneg = fneg float %fma
   %use1 = fmul float %fma, 4.0
   store volatile float %fneg, float addrspace(1)* %out
   store volatile float %use1, float addrspace(1)* %out
@@ -1182,9 +1182,9 @@ define amdgpu_kernel void @v_fneg_fma_fneg_x_y_f32(float addrspace(1)* %out, flo
   %a = load volatile float, float addrspace(1)* %a.gep
   %b = load volatile float, float addrspace(1)* %b.gep
   %c = load volatile float, float addrspace(1)* %c.gep
-  %fneg.a = fsub float -0.000000e+00, %a
+  %fneg.a = fneg float %a
   %fma = call float @llvm.fma.f32(float %fneg.a, float %b, float %c)
-  %fneg = fsub float -0.000000e+00, %fma
+  %fneg = fneg float %fma
   store volatile float %fneg, float addrspace(1)* %out
   ret void
 }
@@ -1210,9 +1210,9 @@ define amdgpu_kernel void @v_fneg_fma_x_fneg_y_f32(float addrspace(1)* %out, flo
   %a = load volatile float, float addrspace(1)* %a.gep
   %b = load volatile float, float addrspace(1)* %b.gep
   %c = load volatile float, float addrspace(1)* %c.gep
-  %fneg.b = fsub float -0.000000e+00, %b
+  %fneg.b = fneg float %b
   %fma = call float @llvm.fma.f32(float %a, float %fneg.b, float %c)
-  %fneg = fsub float -0.000000e+00, %fma
+  %fneg = fneg float %fma
   store volatile float %fneg, float addrspace(1)* %out
   ret void
 }
@@ -1238,10 +1238,10 @@ define amdgpu_kernel void @v_fneg_fma_fneg_fneg_y_f32(float addrspace(1)* %out,
   %a = load volatile float, float addrspace(1)* %a.gep
   %b = load volatile float, float addrspace(1)* %b.gep
   %c = load volatile float, float addrspace(1)* %c.gep
-  %fneg.a = fsub float -0.000000e+00, %a
-  %fneg.b = fsub float -0.000000e+00, %b
+  %fneg.a = fneg float %a
+  %fneg.b = fneg float %b
   %fma = call float @llvm.fma.f32(float %fneg.a, float %fneg.b, float %c)
-  %fneg = fsub float -0.000000e+00, %fma
+  %fneg = fneg float %fma
   store volatile float %fneg, float addrspace(1)* %out
   ret void
 }
@@ -1267,10 +1267,10 @@ define amdgpu_kernel void @v_fneg_fma_fneg_x_fneg_f32(float addrspace(1)* %out,
   %a = load volatile float, float addrspace(1)* %a.gep
   %b = load volatile float, float addrspace(1)* %b.gep
   %c = load volatile float, float addrspace(1)* %c.gep
-  %fneg.a = fsub float -0.000000e+00, %a
-  %fneg.c = fsub float -0.000000e+00, %c
+  %fneg.a = fneg float %a
+  %fneg.c = fneg float %c
   %fma = call float @llvm.fma.f32(float %fneg.a, float %b, float %fneg.c)
-  %fneg = fsub float -0.000000e+00, %fma
+  %fneg = fneg float %fma
   store volatile float %fneg, float addrspace(1)* %out
   ret void
 }
@@ -1296,9 +1296,9 @@ define amdgpu_kernel void @v_fneg_fma_x_y_fneg_f32(float addrspace(1)* %out, flo
   %a = load volatile float, float addrspace(1)* %a.gep
   %b = load volatile float, float addrspace(1)* %b.gep
   %c = load volatile float, float addrspace(1)* %c.gep
-  %fneg.c = fsub float -0.000000e+00, %c
+  %fneg.c = fneg float %c
   %fma = call float @llvm.fma.f32(float %a, float %b, float %fneg.c)
-  %fneg = fsub float -0.000000e+00, %fma
+  %fneg = fneg float %fma
   store volatile float %fneg, float addrspace(1)* %out
   ret void
 }
@@ -1330,9 +1330,9 @@ define amdgpu_kernel void @v_fneg_fma_store_use_fneg_x_y_f32(float addrspace(1)*
   %a = load volatile float, float addrspace(1)* %a.gep
   %b = load volatile float, float addrspace(1)* %b.gep
   %c = load volatile float, float addrspace(1)* %c.gep
-  %fneg.a = fsub float -0.000000e+00, %a
+  %fneg.a = fneg float %a
   %fma = call float @llvm.fma.f32(float %fneg.a, float %b, float %c)
-  %fneg = fsub float -0.000000e+00, %fma
+  %fneg = fneg float %fma
   store volatile float %fneg, float addrspace(1)* %out
   store volatile float %fneg.a, float addrspace(1)* %out
   ret void
@@ -1360,9 +1360,9 @@ define amdgpu_kernel void @v_fneg_fma_multi_use_fneg_x_y_f32(float addrspace(1)*
   %a = load volatile float, float addrspace(1)* %a.gep
   %b = load volatile float, float addrspace(1)* %b.gep
   %c = load volatile float, float addrspace(1)* %c.gep
-  %fneg.a = fsub float -0.000000e+00, %a
+  %fneg.a = fneg float %a
   %fma = call float @llvm.fma.f32(float %fneg.a, float %b, float %c)
-  %fneg = fsub float -0.000000e+00, %fma
+  %fneg = fneg float %fma
   %use1 = fmul float %fneg.a, %d
   store volatile float %fneg, float addrspace(1)* %out
   store volatile float %use1, float addrspace(1)* %out
@@ -1394,7 +1394,7 @@ define amdgpu_kernel void @v_fneg_fmad_f32(float addrspace(1)* %out, float addrs
   %b = load volatile float, float addrspace(1)* %b.gep
   %c = load volatile float, float addrspace(1)* %c.gep
   %fma = call float @llvm.fmuladd.f32(float %a, float %b, float %c)
-  %fneg = fsub float -0.000000e+00, %fma
+  %fneg = fneg float %fma
   store float %fneg, float addrspace(1)* %out.gep
   ret void
 }
@@ -1446,7 +1446,7 @@ define amdgpu_kernel void @v_fneg_fmad_multi_use_fmad_f32(float addrspace(1)* %o
   %b = load volatile float, float addrspace(1)* %b.gep
   %c = load volatile float, float addrspace(1)* %c.gep
   %fma = call float @llvm.fmuladd.f32(float %a, float %b, float %c)
-  %fneg = fsub float -0.000000e+00, %fma
+  %fneg = fneg float %fma
   %use1 = fmul float %fma, 4.0
   store volatile float %fneg, float addrspace(1)* %out
   store volatile float %use1, float addrspace(1)* %out
@@ -1483,7 +1483,7 @@ define amdgpu_kernel void @v_fneg_fp_extend_fneg_f32_to_f64(double addrspace(1)*
   %a.gep = getelementptr inbounds float, float addrspace(1)* %a.ptr, i64 %tid.ext
   %out.gep = getelementptr inbounds double, double addrspace(1)* %out, i64 %tid.ext
   %a = load volatile float, float addrspace(1)* %a.gep
-  %fneg.a = fsub float -0.000000e+00, %a
+  %fneg.a = fneg float %a
   %fpext = fpext float %fneg.a to double
   %fneg = fsub double -0.000000e+00, %fpext
   store double %fneg, double addrspace(1)* %out.gep
@@ -1502,7 +1502,7 @@ define amdgpu_kernel void @v_fneg_fp_extend_store_use_fneg_f32_to_f64(double add
   %a.gep = getelementptr inbounds float, float addrspace(1)* %a.ptr, i64 %tid.ext
   %out.gep = getelementptr inbounds double, double addrspace(1)* %out, i64 %tid.ext
   %a = load volatile float, float addrspace(1)* %a.gep
-  %fneg.a = fsub float -0.000000e+00, %a
+  %fneg.a = fneg float %a
   %fpext = fpext float %fneg.a to double
   %fneg = fsub double -0.000000e+00, %fpext
   store volatile double %fneg, double addrspace(1)* %out.gep
@@ -1559,7 +1559,7 @@ define amdgpu_kernel void @v_fneg_multi_use_fp_extend_fneg_f16_to_f32(float addr
   %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i64 %tid.ext
   %a = load volatile half, half addrspace(1)* %a.gep
   %fpext = fpext half %a to float
-  %fneg = fsub float -0.000000e+00, %fpext
+  %fneg = fneg float %fpext
   store volatile float %fneg, float addrspace(1)* %out.gep
   store volatile float %fpext, float addrspace(1)* %out.gep
   ret void
@@ -1573,7 +1573,7 @@ define amdgpu_kernel void @v_fneg_multi_foldable_use_fp_extend_fneg_f16_to_f32(f
   %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i64 %tid.ext
   %a = load volatile half, half addrspace(1)* %a.gep
   %fpext = fpext half %a to float
-  %fneg = fsub float -0.000000e+00, %fpext
+  %fneg = fneg float %fpext
   %mul = fmul float %fpext, 4.0
   store volatile float %fneg, float addrspace(1)* %out.gep
   store volatile float %mul, float addrspace(1)* %out.gep
@@ -1595,7 +1595,7 @@ define amdgpu_kernel void @v_fneg_fp_round_f64_to_f32(float addrspace(1)* %out,
   %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i64 %tid.ext
   %a = load volatile double, double addrspace(1)* %a.gep
   %fpround = fptrunc double %a to float
-  %fneg = fsub float -0.000000e+00, %fpround
+  %fneg = fneg float %fpround
   store float %fneg, float addrspace(1)* %out.gep
   ret void
 }
@@ -1612,7 +1612,7 @@ define amdgpu_kernel void @v_fneg_fp_round_fneg_f64_to_f32(float addrspace(1)* %
   %a = load volatile double, double addrspace(1)* %a.gep
   %fneg.a = fsub double -0.000000e+00, %a
   %fpround = fptrunc double %fneg.a to float
-  %fneg = fsub float -0.000000e+00, %fpround
+  %fneg = fneg float %fpround
   store float %fneg, float addrspace(1)* %out.gep
   ret void
 }
@@ -1631,7 +1631,7 @@ define amdgpu_kernel void @v_fneg_fp_round_store_use_fneg_f64_to_f32(float addrs
   %a = load volatile double, double addrspace(1)* %a.gep
   %fneg.a = fsub double -0.000000e+00, %a
   %fpround = fptrunc double %fneg.a to float
-  %fneg = fsub float -0.000000e+00, %fpround
+  %fneg = fneg float %fpround
   store volatile float %fneg, float addrspace(1)* %out.gep
   store volatile double %fneg.a, double addrspace(1)* undef
   ret void
@@ -1652,7 +1652,7 @@ define amdgpu_kernel void @v_fneg_fp_round_multi_use_fneg_f64_to_f32(float addrs
   %a = load volatile double, double addrspace(1)* %a.gep
   %fneg.a = fsub double -0.000000e+00, %a
   %fpround = fptrunc double %fneg.a to float
-  %fneg = fsub float -0.000000e+00, %fpround
+  %fneg = fneg float %fpround
   %use1 = fmul double %fneg.a, %c
   store volatile float %fneg, float addrspace(1)* %out.gep
   store volatile double %use1, double addrspace(1)* undef
@@ -1685,7 +1685,7 @@ define amdgpu_kernel void @v_fneg_fp_round_fneg_f32_to_f16(half addrspace(1)* %o
   %a.gep = getelementptr inbounds float, float addrspace(1)* %a.ptr, i64 %tid.ext
   %out.gep = getelementptr inbounds half, half addrspace(1)* %out, i64 %tid.ext
   %a = load volatile float, float addrspace(1)* %a.gep
-  %fneg.a = fsub float -0.000000e+00, %a
+  %fneg.a = fneg float %a
   %fpround = fptrunc float %fneg.a to half
   %fneg = fsub half -0.000000e+00, %fpround
   store half %fneg, half addrspace(1)* %out.gep
@@ -1705,7 +1705,7 @@ define amdgpu_kernel void @v_fneg_multi_use_fp_round_fneg_f64_to_f32(float addrs
   %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i64 %tid.ext
   %a = load volatile double, double addrspace(1)* %a.gep
   %fpround = fptrunc double %a to float
-  %fneg = fsub float -0.000000e+00, %fpround
+  %fneg = fneg float %fpround
   store volatile float %fneg, float addrspace(1)* %out.gep
   store volatile float %fpround, float addrspace(1)* %out.gep
   ret void
@@ -1723,7 +1723,7 @@ define amdgpu_kernel void @v_fneg_fp_round_store_use_fneg_f32_to_f16(half addrsp
   %a.gep = getelementptr inbounds float, float addrspace(1)* %a.ptr, i64 %tid.ext
   %out.gep = getelementptr inbounds half, half addrspace(1)* %out, i64 %tid.ext
   %a = load volatile float, float addrspace(1)* %a.gep
-  %fneg.a = fsub float -0.000000e+00, %a
+  %fneg.a = fneg float %a
   %fpround = fptrunc float %fneg.a to half
   %fneg = fsub half -0.000000e+00, %fpround
   store volatile half %fneg, half addrspace(1)* %out.gep
@@ -1743,7 +1743,7 @@ define amdgpu_kernel void @v_fneg_fp_round_multi_use_fneg_f32_to_f16(half addrsp
   %a.gep = getelementptr inbounds float, float addrspace(1)* %a.ptr, i64 %tid.ext
   %out.gep = getelementptr inbounds half, half addrspace(1)* %out, i64 %tid.ext
   %a = load volatile float, float addrspace(1)* %a.gep
-  %fneg.a = fsub float -0.000000e+00, %a
+  %fneg.a = fneg float %a
   %fpround = fptrunc float %fneg.a to half
   %fneg = fsub half -0.000000e+00, %fpround
   %use1 = fmul float %fneg.a, %c
@@ -1767,7 +1767,7 @@ define amdgpu_kernel void @v_fneg_rcp_f32(float addrspace(1)* %out, float addrsp
   %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i64 %tid.ext
   %a = load volatile float, float addrspace(1)* %a.gep
   %rcp = call float @llvm.amdgcn.rcp.f32(float %a)
-  %fneg = fsub float -0.000000e+00, %rcp
+  %fneg = fneg float %rcp
   store float %fneg, float addrspace(1)* %out.gep
   ret void
 }
@@ -1782,9 +1782,9 @@ define amdgpu_kernel void @v_fneg_rcp_fneg_f32(float addrspace(1)* %out, float a
   %a.gep = getelementptr inbounds float, float addrspace(1)* %a.ptr, i64 %tid.ext
   %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i64 %tid.ext
   %a = load volatile float, float addrspace(1)* %a.gep
-  %fneg.a = fsub float -0.000000e+00, %a
+  %fneg.a = fneg float %a
   %rcp = call float @llvm.amdgcn.rcp.f32(float %fneg.a)
-  %fneg = fsub float -0.000000e+00, %rcp
+  %fneg = fneg float %rcp
   store float %fneg, float addrspace(1)* %out.gep
   ret void
 }
@@ -1801,9 +1801,9 @@ define amdgpu_kernel void @v_fneg_rcp_store_use_fneg_f32(float addrspace(1)* %ou
   %a.gep = getelementptr inbounds float, float addrspace(1)* %a.ptr, i64 %tid.ext
   %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i64 %tid.ext
   %a = load volatile float, float addrspace(1)* %a.gep
-  %fneg.a = fsub float -0.000000e+00, %a
+  %fneg.a = fneg float %a
   %rcp = call float @llvm.amdgcn.rcp.f32(float %fneg.a)
-  %fneg = fsub float -0.000000e+00, %rcp
+  %fneg = fneg float %rcp
   store volatile float %fneg, float addrspace(1)* %out.gep
   store volatile float %fneg.a, float addrspace(1)* undef
   ret void
@@ -1821,9 +1821,9 @@ define amdgpu_kernel void @v_fneg_rcp_multi_use_fneg_f32(float addrspace(1)* %ou
   %a.gep = getelementptr inbounds float, float addrspace(1)* %a.ptr, i64 %tid.ext
   %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i64 %tid.ext
   %a = load volatile float, float addrspace(1)* %a.gep
-  %fneg.a = fsub float -0.000000e+00, %a
+  %fneg.a = fneg float %a
   %rcp = call float @llvm.amdgcn.rcp.f32(float %fneg.a)
-  %fneg = fsub float -0.000000e+00, %rcp
+  %fneg = fneg float %rcp
   %use1 = fmul float %fneg.a, %c
   store volatile float %fneg, float addrspace(1)* %out.gep
   store volatile float %use1, float addrspace(1)* undef
@@ -1848,7 +1848,7 @@ define amdgpu_kernel void @v_fneg_mul_legacy_f32(float addrspace(1)* %out, float
   %a = load volatile float, float addrspace(1)* %a.gep
   %b = load volatile float, float addrspace(1)* %b.gep
   %mul = call float @llvm.amdgcn.fmul.legacy(float %a, float %b)
-  %fneg = fsub float -0.000000e+00, %mul
+  %fneg = fneg float %mul
   store float %fneg, float addrspace(1)* %out.gep
   ret void
 }
@@ -1869,7 +1869,7 @@ define amdgpu_kernel void @v_fneg_mul_legacy_store_use_mul_legacy_f32(float addr
   %a = load volatile float, float addrspace(1)* %a.gep
   %b = load volatile float, float addrspace(1)* %b.gep
   %mul = call float @llvm.amdgcn.fmul.legacy(float %a, float %b)
-  %fneg = fsub float -0.000000e+00, %mul
+  %fneg = fneg float %mul
   store volatile float %fneg, float addrspace(1)* %out
   store volatile float %mul, float addrspace(1)* %out
   ret void
@@ -1891,7 +1891,7 @@ define amdgpu_kernel void @v_fneg_mul_legacy_multi_use_mul_legacy_f32(float addr
   %a = load volatile float, float addrspace(1)* %a.gep
   %b = load volatile float, float addrspace(1)* %b.gep
   %mul = call float @llvm.amdgcn.fmul.legacy(float %a, float %b)
-  %fneg = fsub float -0.000000e+00, %mul
+  %fneg = fneg float %mul
   %use1 = call float @llvm.amdgcn.fmul.legacy(float %mul, float 4.0)
   store volatile float %fneg, float addrspace(1)* %out
   store volatile float %use1, float addrspace(1)* %out
@@ -1911,9 +1911,9 @@ define amdgpu_kernel void @v_fneg_mul_legacy_fneg_x_f32(float addrspace(1)* %out
   %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i64 %tid.ext
   %a = load volatile float, float addrspace(1)* %a.gep
   %b = load volatile float, float addrspace(1)* %b.gep
-  %fneg.a = fsub float -0.000000e+00, %a
+  %fneg.a = fneg float %a
   %mul = call float @llvm.amdgcn.fmul.legacy(float %fneg.a, float %b)
-  %fneg = fsub float -0.000000e+00, %mul
+  %fneg = fneg float %mul
   store volatile float %fneg, float addrspace(1)* %out
   ret void
 }
@@ -1931,9 +1931,9 @@ define amdgpu_kernel void @v_fneg_mul_legacy_x_fneg_f32(float addrspace(1)* %out
   %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i64 %tid.ext
   %a = load volatile float, float addrspace(1)* %a.gep
   %b = load volatile float, float addrspace(1)* %b.gep
-  %fneg.b = fsub float -0.000000e+00, %b
+  %fneg.b = fneg float %b
   %mul = call float @llvm.amdgcn.fmul.legacy(float %a, float %fneg.b)
-  %fneg = fsub float -0.000000e+00, %mul
+  %fneg = fneg float %mul
   store volatile float %fneg, float addrspace(1)* %out
   ret void
 }
@@ -1951,10 +1951,10 @@ define amdgpu_kernel void @v_fneg_mul_legacy_fneg_fneg_f32(float addrspace(1)* %
   %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i64 %tid.ext
   %a = load volatile float, float addrspace(1)* %a.gep
   %b = load volatile float, float addrspace(1)* %b.gep
-  %fneg.a = fsub float -0.000000e+00, %a
-  %fneg.b = fsub float -0.000000e+00, %b
+  %fneg.a = fneg float %a
+  %fneg.b = fneg float %b
   %mul = call float @llvm.amdgcn.fmul.legacy(float %fneg.a, float %fneg.b)
-  %fneg = fsub float -0.000000e+00, %mul
+  %fneg = fneg float %mul
   store volatile float %fneg, float addrspace(1)* %out
   ret void
 }
@@ -1974,9 +1974,9 @@ define amdgpu_kernel void @v_fneg_mul_legacy_store_use_fneg_x_f32(float addrspac
   %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i64 %tid.ext
   %a = load volatile float, float addrspace(1)* %a.gep
   %b = load volatile float, float addrspace(1)* %b.gep
-  %fneg.a = fsub float -0.000000e+00, %a
+  %fneg.a = fneg float %a
   %mul = call float @llvm.amdgcn.fmul.legacy(float %fneg.a, float %b)
-  %fneg = fsub float -0.000000e+00, %mul
+  %fneg = fneg float %mul
   store volatile float %fneg, float addrspace(1)* %out
   store volatile float %fneg.a, float addrspace(1)* %out
   ret void
@@ -1997,9 +1997,9 @@ define amdgpu_kernel void @v_fneg_mul_legacy_multi_use_fneg_x_f32(float addrspac
   %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i64 %tid.ext
   %a = load volatile float, float addrspace(1)* %a.gep
   %b = load volatile float, float addrspace(1)* %b.gep
-  %fneg.a = fsub float -0.000000e+00, %a
+  %fneg.a = fneg float %a
   %mul = call float @llvm.amdgcn.fmul.legacy(float %fneg.a, float %b)
-  %fneg = fsub float -0.000000e+00, %mul
+  %fneg = fneg float %mul
   %use1 = call float @llvm.amdgcn.fmul.legacy(float %fneg.a, float %c)
   store volatile float %fneg, float addrspace(1)* %out
   store volatile float %use1, float addrspace(1)* %out
@@ -2023,7 +2023,7 @@ define amdgpu_kernel void @v_fneg_sin_f32(float addrspace(1)* %out, float addrsp
   %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i64 %tid.ext
   %a = load volatile float, float addrspace(1)* %a.gep
   %sin = call float @llvm.sin.f32(float %a)
-  %fneg = fsub float -0.000000e+00, %sin
+  %fneg = fneg float %sin
   store float %fneg, float addrspace(1)* %out.gep
   ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/selectcc-opt.ll b/llvm/test/CodeGen/AMDGPU/selectcc-opt.ll
index fe7a350a1d9f3..733399dd09096 100644
--- a/llvm/test/CodeGen/AMDGPU/selectcc-opt.ll
+++ b/llvm/test/CodeGen/AMDGPU/selectcc-opt.ll
@@ -11,7 +11,7 @@ define amdgpu_kernel void @test_a(i32 addrspace(1)* %out, float %in) {
 entry:
   %0 = fcmp olt float %in, 0.000000e+00
   %1 = select i1 %0, float 1.000000e+00, float 0.000000e+00
-  %2 = fsub float -0.000000e+00, %1
+  %2 = fneg float %1
   %3 = fptosi float %2 to i32
   %4 = bitcast i32 %3 to float
   %5 = bitcast float %4 to i32
@@ -39,7 +39,7 @@ define amdgpu_kernel void @test_b(i32 addrspace(1)* %out, float %in) {
 entry:
   %0 = fcmp olt float %in, 0.0
   %1 = select i1 %0, float 1.000000e+00, float 0.000000e+00
-  %2 = fsub float -0.000000e+00, %1
+  %2 = fneg float %1
   %3 = fptosi float %2 to i32
   %4 = bitcast i32 %3 to float
   %5 = bitcast float %4 to i32
diff --git a/llvm/test/CodeGen/AMDGPU/set-dx10.ll b/llvm/test/CodeGen/AMDGPU/set-dx10.ll
index 6867c6394937c..9a317a87540f4 100644
--- a/llvm/test/CodeGen/AMDGPU/set-dx10.ll
+++ b/llvm/test/CodeGen/AMDGPU/set-dx10.ll
@@ -12,7 +12,7 @@ define amdgpu_kernel void @fcmp_une_select_fptosi(i32 addrspace(1)* %out, float
 entry:
   %0 = fcmp une float %in, 5.0
   %1 = select i1 %0, float 1.000000e+00, float 0.000000e+00
-  %2 = fsub float -0.000000e+00, %1
+  %2 = fneg float %1
   %3 = fptosi float %2 to i32
   store i32 %3, i32 addrspace(1)* %out
   ret void
@@ -38,7 +38,7 @@ define amdgpu_kernel void @fcmp_oeq_select_fptosi(i32 addrspace(1)* %out, float
 entry:
   %0 = fcmp oeq float %in, 5.0
   %1 = select i1 %0, float 1.000000e+00, float 0.000000e+00
-  %2 = fsub float -0.000000e+00, %1
+  %2 = fneg float %1
   %3 = fptosi float %2 to i32
   store i32 %3, i32 addrspace(1)* %out
   ret void
@@ -64,7 +64,7 @@ define amdgpu_kernel void @fcmp_ogt_select_fptosi(i32 addrspace(1)* %out, float
 entry:
   %0 = fcmp ogt float %in, 5.0
   %1 = select i1 %0, float 1.000000e+00, float 0.000000e+00
-  %2 = fsub float -0.000000e+00, %1
+  %2 = fneg float %1
   %3 = fptosi float %2 to i32
   store i32 %3, i32 addrspace(1)* %out
   ret void
@@ -90,7 +90,7 @@ define amdgpu_kernel void @fcmp_oge_select_fptosi(i32 addrspace(1)* %out, float
 entry:
   %0 = fcmp oge float %in, 5.0
   %1 = select i1 %0, float 1.000000e+00, float 0.000000e+00
-  %2 = fsub float -0.000000e+00, %1
+  %2 = fneg float %1
   %3 = fptosi float %2 to i32
   store i32 %3, i32 addrspace(1)* %out
   ret void
@@ -116,7 +116,7 @@ define amdgpu_kernel void @fcmp_ole_select_fptosi(i32 addrspace(1)* %out, float
 entry:
   %0 = fcmp ole float %in, 5.0
   %1 = select i1 %0, float 1.000000e+00, float 0.000000e+00
-  %2 = fsub float -0.000000e+00, %1
+  %2 = fneg float %1
   %3 = fptosi float %2 to i32
   store i32 %3, i32 addrspace(1)* %out
   ret void
@@ -142,7 +142,7 @@ define amdgpu_kernel void @fcmp_olt_select_fptosi(i32 addrspace(1)* %out, float
 entry:
   %0 = fcmp olt float %in, 5.0
   %1 = select i1 %0, float 1.000000e+00, float 0.000000e+00
-  %2 = fsub float -0.000000e+00, %1
+  %2 = fneg float %1
   %3 = fptosi float %2 to i32
   store i32 %3, i32 addrspace(1)* %out
   ret void

From 7ba82a7320df82d07d3d5679bce89b14526b536c Mon Sep 17 00:00:00 2001
From: Mehdi Amini <joker.eph@gmail.com>
Date: Mon, 3 Aug 2020 16:16:47 +0000
Subject: [PATCH 0921/1035] Revert "[mlir][Vector] Add transformation + pattern
 to split vector.transfer_read into full and partial copies."

This reverts commit 35b65be041127db9fe23d3128a004c888893cbae.

Build is broken with -DBUILD_SHARED_LIBS=ON with some undefined
references like:

VectorTransforms.cpp:(.text._ZN4llvm12function_refIFvllEE11callback_fnIZL24createScopedInBoundsCondN4mlir25VectorTransferOpInterfaceEE3$_8EEvlll+0xa5): undefined reference to `mlir::edsc::op::operator+(mlir::Value, mlir::Value)'
---
 .../mlir/Dialect/Vector/VectorTransforms.h    |  64 -----
 .../mlir/Interfaces/VectorInterfaces.td       |  13 -
 mlir/lib/Dialect/Vector/VectorTransforms.cpp  | 234 ------------------
 .../vector-transfer-full-partial-split.mlir   | 102 --------
 .../lib/Transforms/TestVectorTransforms.cpp   |  16 --
 5 files changed, 429 deletions(-)
 delete mode 100644 mlir/test/Dialect/Vector/vector-transfer-full-partial-split.mlir

diff --git a/mlir/include/mlir/Dialect/Vector/VectorTransforms.h b/mlir/include/mlir/Dialect/Vector/VectorTransforms.h
index 835ad18a79ad2..0d18c5aa782d1 100644
--- a/mlir/include/mlir/Dialect/Vector/VectorTransforms.h
+++ b/mlir/include/mlir/Dialect/Vector/VectorTransforms.h
@@ -17,11 +17,6 @@
 namespace mlir {
 class MLIRContext;
 class OwningRewritePatternList;
-class VectorTransferOpInterface;
-
-namespace scf {
-class IfOp;
-} // namespace scf
 
 /// Collect a set of patterns to convert from the Vector dialect to itself.
 /// Should be merged with populateVectorToSCFLoweringPattern.
@@ -109,65 +104,6 @@ struct UnrollVectorPattern : public OpRewritePattern<OpTy> {
   FilterConstraintType filter;
 };
 
-/// Split a vector.transfer operation into an unmasked fastpath vector.transfer
-/// and a slowpath masked vector.transfer. If `ifOp` is not null and the result
-/// is `success, the `ifOp` points to the newly created conditional upon
-/// function return. To accomodate for the fact that the original
-/// vector.transfer indexing may be arbitrary and the slow path indexes @[0...0]
-/// in the temporary buffer, the scf.if op returns a view and values of type
-/// index. At this time, only vector.transfer_read is implemented.
-///
-/// Example (a 2-D vector.transfer_read):
-/// ```
-///    %1 = vector.transfer_read %0[...], %pad : memref<A...>, vector<...>
-/// ```
-/// is transformed into:
-/// ```
-///    %1:3 = scf.if (%inBounds) {
-///       scf.yield %0 : memref<A...>, index, index
-///     } else {
-///       %2 = vector.transfer_read %0[...], %pad : memref<A...>, vector<...>
-///       %3 = vector.type_cast %extra_alloc : memref<...> to
-///       memref<vector<...>> store %2, %3[] : memref<vector<...>> %4 =
-///       memref_cast %extra_alloc: memref<B...> to memref<A...> scf.yield %4 :
-///       memref<A...>, index, index
-//     }
-///    %0 = vector.transfer_read %1#0[%1#1, %1#2] {masked = [false ... false]}
-/// ```
-/// where `extra_alloc` is a top of the function alloca'ed buffer of one vector.
-///
-/// Preconditions:
-///  1. `xferOp.permutation_map()` must be a minor identity map
-///  2. the rank of the `xferOp.memref()` and the rank of the `xferOp.vector()`
-///  must be equal. This will be relaxed in the future but requires
-///  rank-reducing subviews.
-LogicalResult
-splitFullAndPartialTransferPrecondition(VectorTransferOpInterface xferOp);
-LogicalResult splitFullAndPartialTransfer(OpBuilder &b,
-                                          VectorTransferOpInterface xferOp,
-                                          scf::IfOp *ifOp = nullptr);
-
-/// Apply `splitFullAndPartialTransfer` selectively via a pattern. This pattern
-/// may take an extra filter to perform selection at a finer granularity.
-struct VectorTransferFullPartialRewriter : public RewritePattern {
-  using FilterConstraintType =
-      std::function<LogicalResult(VectorTransferOpInterface op)>;
-
-  explicit VectorTransferFullPartialRewriter(
-      MLIRContext *context,
-      FilterConstraintType filter =
-          [](VectorTransferOpInterface op) { return success(); },
-      PatternBenefit benefit = 1)
-      : RewritePattern(benefit, MatchAnyOpTypeTag()), filter(filter) {}
-
-  /// Performs the rewrite.
-  LogicalResult matchAndRewrite(Operation *op,
-                                PatternRewriter &rewriter) const override;
-
-private:
-  FilterConstraintType filter;
-};
-
 } // namespace vector
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Interfaces/VectorInterfaces.td b/mlir/include/mlir/Interfaces/VectorInterfaces.td
index 218715318a867..aefbb7d471172 100644
--- a/mlir/include/mlir/Interfaces/VectorInterfaces.td
+++ b/mlir/include/mlir/Interfaces/VectorInterfaces.td
@@ -160,19 +160,6 @@ def VectorTransferOpInterface : OpInterface<"VectorTransferOpInterface"> {
       /*defaultImplementation=*/
         "return $_op.getMemRefType().getRank() - $_op.getTransferRank();"
     >,
-    InterfaceMethod<
-      /*desc=*/[{ Returns true if at least one of the dimensions is masked.}],
-      /*retTy=*/"bool",
-      /*methodName=*/"hasMaskedDim",
-      /*args=*/(ins),
-      /*methodBody=*/"",
-      /*defaultImplementation=*/[{
-        for (unsigned idx = 0, e = $_op.getTransferRank(); idx < e; ++idx)
-          if ($_op.isMaskedDim(idx))
-            return true;
-        return false;
-      }]
-    >,
     InterfaceMethod<
       /*desc=*/[{
       Helper function to account for the fact that `permutationMap` results and
diff --git a/mlir/lib/Dialect/Vector/VectorTransforms.cpp b/mlir/lib/Dialect/Vector/VectorTransforms.cpp
index 573b822503f3a..197b1c62274b2 100644
--- a/mlir/lib/Dialect/Vector/VectorTransforms.cpp
+++ b/mlir/lib/Dialect/Vector/VectorTransforms.cpp
@@ -12,13 +12,9 @@
 
 #include <type_traits>
 
-#include "mlir/Dialect/Affine/EDSC/Intrinsics.h"
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
-#include "mlir/Dialect/SCF/EDSC/Intrinsics.h"
-#include "mlir/Dialect/StandardOps/EDSC/Intrinsics.h"
 #include "mlir/Dialect/StandardOps/IR/Ops.h"
 #include "mlir/Dialect/Utils/StructuredOpsUtils.h"
-#include "mlir/Dialect/Vector/EDSC/Intrinsics.h"
 #include "mlir/Dialect/Vector/VectorOps.h"
 #include "mlir/Dialect/Vector/VectorTransforms.h"
 #include "mlir/Dialect/Vector/VectorUtils.h"
@@ -1989,236 +1985,6 @@ Value ContractionOpLowering::lowerReduction(vector::ContractionOp op,
 
 } // namespace mlir
 
-static Optional<int64_t> extractConstantIndex(Value v) {
-  if (auto cstOp = v.getDefiningOp<ConstantIndexOp>())
-    return cstOp.getValue();
-  if (auto affineApplyOp = v.getDefiningOp<AffineApplyOp>())
-    if (affineApplyOp.getAffineMap().isSingleConstant())
-      return affineApplyOp.getAffineMap().getSingleConstantResult();
-  return None;
-}
-
-// Missing foldings of scf.if make it necessary to perform poor man's folding
-// eagerly, especially in the case of unrolling. In the future, this should go
-// away once scf.if folds properly.
-static Value createScopedFoldedSLE(Value v, Value ub) {
-  using namespace edsc::op;
-  auto maybeCstV = extractConstantIndex(v);
-  auto maybeCstUb = extractConstantIndex(ub);
-  if (maybeCstV && maybeCstUb && *maybeCstV < *maybeCstUb)
-    return Value();
-  return sle(v, ub);
-}
-
-// Operates under a scoped context to build the condition to ensure that a
-// particular VectorTransferOpInterface is unmasked.
-static Value createScopedInBoundsCond(VectorTransferOpInterface xferOp) {
-  assert(xferOp.permutation_map().isMinorIdentity() &&
-         "Expected minor identity map");
-  Value inBoundsCond;
-  xferOp.zipResultAndIndexing([&](int64_t resultIdx, int64_t indicesIdx) {
-    // Zip over the resulting vector shape and memref indices.
-    // If the dimension is known to be unmasked, it does not participate in the
-    // construction of `inBoundsCond`.
-    if (!xferOp.isMaskedDim(resultIdx))
-      return;
-    int64_t vectorSize = xferOp.getVectorType().getDimSize(resultIdx);
-    using namespace edsc::op;
-    using namespace edsc::intrinsics;
-    // Fold or create the check that `index + vector_size` <= `memref_size`.
-    Value sum = xferOp.indices()[indicesIdx] + std_constant_index(vectorSize);
-    Value cond =
-        createScopedFoldedSLE(sum, std_dim(xferOp.memref(), indicesIdx));
-    if (!cond)
-      return;
-    // Conjunction over all dims for which we are in-bounds.
-    inBoundsCond = inBoundsCond ? inBoundsCond && cond : cond;
-  });
-  return inBoundsCond;
-}
-
-LogicalResult mlir::vector::splitFullAndPartialTransferPrecondition(
-    VectorTransferOpInterface xferOp) {
-  // TODO: expand support to these 2 cases.
-  if (!xferOp.permutation_map().isMinorIdentity())
-    return failure();
-  // TODO: relax this precondition. This will require rank-reducing subviews.
-  if (xferOp.getMemRefType().getRank() != xferOp.getTransferRank())
-    return failure();
-  // Must have some masked dimension to be a candidate for splitting.
-  if (!xferOp.hasMaskedDim())
-    return failure();
-  // Don't split transfer operations under IfOp, this avoids applying the
-  // pattern recursively.
-  // TODO: improve the condition to make it more applicable.
-  if (xferOp.getParentOfType<scf::IfOp>())
-    return failure();
-  return success();
-}
-
-MemRefType getCastCompatibleMemRefType(MemRefType aT, MemRefType bT) {
-  if (MemRefCastOp::areCastCompatible(aT, bT))
-    return aT;
-  if (aT.getRank() != bT.getRank())
-    return MemRefType();
-  int64_t aOffset, bOffset;
-  SmallVector<int64_t, 4> aStrides, bStrides;
-  if (failed(getStridesAndOffset(aT, aStrides, aOffset)) ||
-      failed(getStridesAndOffset(bT, bStrides, bOffset)) ||
-      aStrides.size() != bStrides.size())
-    return MemRefType();
-
-  ArrayRef<int64_t> aShape = aT.getShape(), bShape = bT.getShape();
-  int64_t resOffset;
-  SmallVector<int64_t, 4> resShape(aT.getRank(), 0),
-      resStrides(bT.getRank(), 0);
-  for (int64_t idx = 0, e = aT.getRank(); idx < e; ++idx) {
-    resShape[idx] =
-        (aShape[idx] == bShape[idx]) ? aShape[idx] : MemRefType::kDynamicSize;
-    resStrides[idx] = (aStrides[idx] == bStrides[idx])
-                          ? aStrides[idx]
-                          : MemRefType::kDynamicStrideOrOffset;
-  }
-  resOffset =
-      (aOffset == bOffset) ? aOffset : MemRefType::kDynamicStrideOrOffset;
-  return MemRefType::get(
-      resShape, aT.getElementType(),
-      makeStridedLinearLayoutMap(resStrides, resOffset, aT.getContext()));
-}
-
-/// Split a vector.transfer operation into an unmasked fastpath vector.transfer
-/// and a slowpath masked vector.transfer. If `ifOp` is not null and the result
-/// is `success, the `ifOp` points to the newly created conditional upon
-/// function return. To accomodate for the fact that the original
-/// vector.transfer indexing may be arbitrary and the slow path indexes @[0...0]
-/// in the temporary buffer, the scf.if op returns a view and values of type
-/// index. At this time, only vector.transfer_read is implemented.
-///
-/// Example (a 2-D vector.transfer_read):
-/// ```
-///    %1 = vector.transfer_read %0[...], %pad : memref<A...>, vector<...>
-/// ```
-/// is transformed into:
-/// ```
-///    %1:3 = scf.if (%inBounds) {
-///       scf.yield %0 : memref<A...>, index, index
-///     } else {
-///       %2 = vector.transfer_read %0[...], %pad : memref<A...>, vector<...>
-///       %3 = vector.type_cast %extra_alloc : memref<...> to
-///       memref<vector<...>> store %2, %3[] : memref<vector<...>> %4 =
-///       memref_cast %extra_alloc: memref<B...> to memref<A...> scf.yield %4 :
-///       memref<A...>, index, index
-//     }
-///    %0 = vector.transfer_read %1#0[%1#1, %1#2] {masked = [false ... false]}
-/// ```
-/// where `extra_alloc` is a top of the function alloca'ed buffer of one vector.
-///
-/// Preconditions:
-///  1. `xferOp.permutation_map()` must be a minor identity map
-///  2. the rank of the `xferOp.memref()` and the rank of the `xferOp.vector()`
-///  must be equal. This will be relaxed in the future but requires
-///  rank-reducing subviews.
-LogicalResult mlir::vector::splitFullAndPartialTransfer(
-    OpBuilder &b, VectorTransferOpInterface xferOp, scf::IfOp *ifOp) {
-  using namespace edsc;
-  using namespace edsc::intrinsics;
-
-  assert(succeeded(splitFullAndPartialTransferPrecondition(xferOp)) &&
-         "Expected splitFullAndPartialTransferPrecondition to hold");
-  auto xferReadOp = dyn_cast<vector::TransferReadOp>(xferOp.getOperation());
-
-  // TODO: add support for write case.
-  if (!xferReadOp)
-    return failure();
-
-  OpBuilder::InsertionGuard guard(b);
-  if (xferOp.memref().getDefiningOp())
-    b.setInsertionPointAfter(xferOp.memref().getDefiningOp());
-  else
-    b.setInsertionPoint(xferOp);
-  ScopedContext scope(b, xferOp.getLoc());
-  Value inBoundsCond = createScopedInBoundsCond(
-      cast<VectorTransferOpInterface>(xferOp.getOperation()));
-  if (!inBoundsCond)
-    return failure();
-
-  // Top of the function `alloc` for transient storage.
-  Value alloc;
-  {
-    FuncOp funcOp = xferOp.getParentOfType<FuncOp>();
-    OpBuilder::InsertionGuard guard(b);
-    b.setInsertionPointToStart(&funcOp.getRegion().front());
-    auto shape = xferOp.getVectorType().getShape();
-    Type elementType = xferOp.getVectorType().getElementType();
-    alloc = std_alloca(MemRefType::get(shape, elementType), ValueRange{},
-                       b.getI64IntegerAttr(32));
-  }
-
-  Value memref = xferOp.memref();
-  SmallVector<bool, 4> bools(xferOp.getTransferRank(), false);
-  auto unmaskedAttr = b.getBoolArrayAttr(bools);
-
-  MemRefType compatibleMemRefType = getCastCompatibleMemRefType(
-      xferOp.getMemRefType(), alloc.getType().cast<MemRefType>());
-
-  // Read case: full fill + partial copy -> unmasked vector.xfer_read.
-  Value zero = std_constant_index(0);
-  SmallVector<Type, 4> returnTypes(1 + xferOp.getTransferRank(),
-                                   b.getIndexType());
-  returnTypes[0] = compatibleMemRefType;
-  scf::IfOp fullPartialIfOp;
-  conditionBuilder(
-      returnTypes, inBoundsCond,
-      [&]() -> scf::ValueVector {
-        Value res = memref;
-        if (compatibleMemRefType != xferOp.getMemRefType())
-          res = std_memref_cast(memref, compatibleMemRefType);
-        scf::ValueVector viewAndIndices{res};
-        viewAndIndices.insert(viewAndIndices.end(), xferOp.indices().begin(),
-                              xferOp.indices().end());
-        return viewAndIndices;
-      },
-      [&]() -> scf::ValueVector {
-        Operation *newXfer =
-            ScopedContext::getBuilderRef().clone(*xferOp.getOperation());
-        Value vector = cast<VectorTransferOpInterface>(newXfer).vector();
-        std_store(vector, vector_type_cast(
-                              MemRefType::get({}, vector.getType()), alloc));
-
-        Value casted = std_memref_cast(alloc, compatibleMemRefType);
-        scf::ValueVector viewAndIndices{casted};
-        viewAndIndices.insert(viewAndIndices.end(), xferOp.getTransferRank(),
-                              zero);
-
-        return viewAndIndices;
-      },
-      &fullPartialIfOp);
-  if (ifOp)
-    *ifOp = fullPartialIfOp;
-
-  // Unmask the existing read op, it always reads from a full buffer.
-  for (unsigned i = 0, e = returnTypes.size(); i != e; ++i)
-    xferReadOp.setOperand(i, fullPartialIfOp.getResult(i));
-  xferOp.setAttr(vector::TransferReadOp::getMaskedAttrName(), unmaskedAttr);
-
-  return success();
-}
-
-LogicalResult mlir::vector::VectorTransferFullPartialRewriter::matchAndRewrite(
-    Operation *op, PatternRewriter &rewriter) const {
-  auto xferOp = dyn_cast<VectorTransferOpInterface>(op);
-  if (!xferOp || failed(splitFullAndPartialTransferPrecondition(xferOp)) ||
-      failed(filter(xferOp)))
-    return failure();
-  rewriter.startRootUpdate(xferOp);
-  if (succeeded(splitFullAndPartialTransfer(rewriter, xferOp))) {
-    rewriter.finalizeRootUpdate(xferOp);
-    return success();
-  }
-  rewriter.cancelRootUpdate(xferOp);
-  return failure();
-}
-
 // TODO: Add pattern to rewrite ExtractSlices(ConstantMaskOp).
 // TODO: Add this as DRR pattern.
 void mlir::vector::populateVectorToVectorTransformationPatterns(
diff --git a/mlir/test/Dialect/Vector/vector-transfer-full-partial-split.mlir b/mlir/test/Dialect/Vector/vector-transfer-full-partial-split.mlir
deleted file mode 100644
index ef76247ee9d4b..0000000000000
--- a/mlir/test/Dialect/Vector/vector-transfer-full-partial-split.mlir
+++ /dev/null
@@ -1,102 +0,0 @@
-// RUN: mlir-opt %s -test-vector-transfer-full-partial-split | FileCheck %s
-
-// CHECK-DAG: #[[$map_p4:.*]] = affine_map<()[s0] -> (s0 + 4)>
-// CHECK-DAG: #[[$map_p8:.*]] = affine_map<()[s0] -> (s0 + 8)>
-// CHECK-DAG: #[[$map_2d_stride_1:.*]] = affine_map<(d0, d1)[s0, s1] -> (d0 * s1 + s0 + d1)>
-
-// CHECK-LABEL: split_vector_transfer_read_2d(
-//  CHECK-SAME: %[[A:[a-zA-Z0-9]*]]: memref
-//  CHECK-SAME: %[[i:[a-zA-Z0-9]*]]: index
-//  CHECK-SAME: %[[j:[a-zA-Z0-9]*]]: index
-func @split_vector_transfer_read_2d(%A: memref<?x8xf32>, %i: index, %j: index) -> vector<4x8xf32> {
-  %c0 = constant 0 : index
-  %f0 = constant 0.0 : f32
-
-  //  CHECK-DAG: %[[c0:.*]] = constant 0 : index
-  //  CHECK-DAG: %[[c8:.*]] = constant 8 : index
-  //  CHECK-DAG: %[[cst:.*]] = constant 0.000000e+00 : f32
-  // alloca for boundary full tile
-  //      CHECK: %[[alloc:.*]] = alloca() {alignment = 32 : i64} : memref<4x8xf32>
-  // %i + 4 <= dim(%A, 0)
-  //      CHECK: %[[idx0:.*]] = affine.apply #[[$map_p4]]()[%[[i]]]
-  //      CHECK: %[[d0:.*]] = dim %[[A]], %[[c0]] : memref<?x8xf32>
-  //      CHECK: %[[cmp0:.*]] = cmpi "sle", %[[idx0]], %[[d0]] : index
-  // %j + 8 <= dim(%A, 1)
-  //      CHECK: %[[idx1:.*]] = affine.apply #[[$map_p8]]()[%[[j]]]
-  //      CHECK: %[[cmp1:.*]] = cmpi "sle", %[[idx1]], %[[c8]] : index
-  // are both conds true
-  //      CHECK: %[[cond:.*]] = and %[[cmp0]], %[[cmp1]] : i1
-  //      CHECK: %[[ifres:.*]]:3 = scf.if %[[cond]] -> (memref<?x8xf32>, index, index) {
-  //               inBounds, just yield %A
-  //      CHECK:   scf.yield %[[A]], %[[i]], %[[j]] : memref<?x8xf32>, index, index
-  //      CHECK: } else {
-  //               slow path, fill tmp alloc and yield a memref_casted version of it
-  //      CHECK:   %[[slow:.*]] = vector.transfer_read %[[A]][%[[i]], %[[j]]], %cst :
-  // CHECK-SAME:     memref<?x8xf32>, vector<4x8xf32>
-  //      CHECK:   %[[cast_alloc:.*]] = vector.type_cast %[[alloc]] :
-  // CHECK-SAME:     memref<4x8xf32> to memref<vector<4x8xf32>>
-  //      CHECK:   store %[[slow]], %[[cast_alloc]][] : memref<vector<4x8xf32>>
-  //      CHECK:   %[[yielded:.*]] = memref_cast %[[alloc]] :
-  // CHECK-SAME:     memref<4x8xf32> to memref<?x8xf32>
-  //      CHECK:   scf.yield %[[yielded]], %[[c0]], %[[c0]] :
-  // CHECK-SAME:     memref<?x8xf32>, index, index
-  //      CHECK: }
-  //      CHECK: %[[res:.*]] = vector.transfer_read %[[ifres]]#0[%[[ifres]]#1, %[[ifres]]#2], %[[cst]]
-  // CHECK_SAME:   {masked = [false, false]} : memref<?x8xf32>, vector<4x8xf32>
-  %1 = vector.transfer_read %A[%i, %j], %f0 : memref<?x8xf32>, vector<4x8xf32>
-
-  // CHECK: return %[[res]] : vector<4x8xf32>
-  return %1: vector<4x8xf32>
-}
-
-// CHECK-LABEL: split_vector_transfer_read_strided_2d(
-//  CHECK-SAME: %[[A:[a-zA-Z0-9]*]]: memref
-//  CHECK-SAME: %[[i:[a-zA-Z0-9]*]]: index
-//  CHECK-SAME: %[[j:[a-zA-Z0-9]*]]: index
-func @split_vector_transfer_read_strided_2d(
-    %A: memref<7x8xf32, offset:?, strides:[?, 1]>,
-    %i: index, %j: index) -> vector<4x8xf32> {
-  %c0 = constant 0 : index
-  %f0 = constant 0.0 : f32
-
-  //  CHECK-DAG: %[[c0:.*]] = constant 0 : index
-  //  CHECK-DAG: %[[c7:.*]] = constant 7 : index
-  //  CHECK-DAG: %[[c8:.*]] = constant 8 : index
-  //  CHECK-DAG: %[[cst:.*]] = constant 0.000000e+00 : f32
-  // alloca for boundary full tile
-  //      CHECK: %[[alloc:.*]] = alloca() {alignment = 32 : i64} : memref<4x8xf32>
-  // %i + 4 <= dim(%A, 0)
-  //      CHECK: %[[idx0:.*]] = affine.apply #[[$map_p4]]()[%[[i]]]
-  //      CHECK: %[[cmp0:.*]] = cmpi "sle", %[[idx0]], %[[c7]] : index
-  // %j + 8 <= dim(%A, 1)
-  //      CHECK: %[[idx1:.*]] = affine.apply #[[$map_p8]]()[%[[j]]]
-  //      CHECK: %[[cmp1:.*]] = cmpi "sle", %[[idx1]], %[[c8]] : index
-  // are both conds true
-  //      CHECK: %[[cond:.*]] = and %[[cmp0]], %[[cmp1]] : i1
-  //      CHECK: %[[ifres:.*]]:3 = scf.if %[[cond]] -> (memref<?x8xf32, #[[$map_2d_stride_1]]>, index, index) {
-  //               inBounds but not cast-compatible: yield a memref_casted form of %A
-  //      CHECK:   %[[casted:.*]] = memref_cast %arg0 :
-  // CHECK-SAME:     memref<7x8xf32, #[[$map_2d_stride_1]]> to memref<?x8xf32, #[[$map_2d_stride_1]]>
-  //      CHECK:   scf.yield %[[casted]], %[[i]], %[[j]] :
-  // CHECK-SAME:     memref<?x8xf32, #[[$map_2d_stride_1]]>, index, index
-  //      CHECK: } else {
-  //               slow path, fill tmp alloc and yield a memref_casted version of it
-  //      CHECK:   %[[slow:.*]] = vector.transfer_read %[[A]][%[[i]], %[[j]]], %cst :
-  // CHECK-SAME:     memref<7x8xf32, #[[$map_2d_stride_1]]>, vector<4x8xf32>
-  //      CHECK:   %[[cast_alloc:.*]] = vector.type_cast %[[alloc]] :
-  // CHECK-SAME:     memref<4x8xf32> to memref<vector<4x8xf32>>
-  //      CHECK:   store %[[slow]], %[[cast_alloc]][] :
-  // CHECK-SAME:     memref<vector<4x8xf32>>
-  //      CHECK:   %[[yielded:.*]] = memref_cast %[[alloc]] :
-  // CHECK-SAME:     memref<4x8xf32> to memref<?x8xf32, #[[$map_2d_stride_1]]>
-  //      CHECK:   scf.yield %[[yielded]], %[[c0]], %[[c0]] :
-  // CHECK-SAME:     memref<?x8xf32, #[[$map_2d_stride_1]]>, index, index
-  //      CHECK: }
-  //      CHECK: %[[res:.*]] = vector.transfer_read {{.*}} {masked = [false, false]} :
-  // CHECK-SAME:   memref<?x8xf32, #[[$map_2d_stride_1]]>, vector<4x8xf32>
-  %1 = vector.transfer_read %A[%i, %j], %f0 :
-    memref<7x8xf32, offset:?, strides:[?, 1]>, vector<4x8xf32>
-
-  // CHECK: return %[[res]] : vector<4x8xf32>
-  return %1 : vector<4x8xf32>
-}
diff --git a/mlir/test/lib/Transforms/TestVectorTransforms.cpp b/mlir/test/lib/Transforms/TestVectorTransforms.cpp
index 0bba74e76385e..2058706dcbdd3 100644
--- a/mlir/test/lib/Transforms/TestVectorTransforms.cpp
+++ b/mlir/test/lib/Transforms/TestVectorTransforms.cpp
@@ -122,17 +122,6 @@ struct TestVectorUnrollingPatterns
   }
 };
 
-struct TestVectorTransferFullPartialSplitPatterns
-    : public PassWrapper<TestVectorTransferFullPartialSplitPatterns,
-                         FunctionPass> {
-  void runOnFunction() override {
-    MLIRContext *ctx = &getContext();
-    OwningRewritePatternList patterns;
-    patterns.insert<VectorTransferFullPartialRewriter>(ctx);
-    applyPatternsAndFoldGreedily(getFunction(), patterns);
-  }
-};
-
 } // end anonymous namespace
 
 namespace mlir {
@@ -152,10 +141,5 @@ void registerTestVectorConversions() {
   PassRegistration<TestVectorUnrollingPatterns> contractionUnrollingPass(
       "test-vector-unrolling-patterns",
       "Test conversion patterns to unroll contract ops in the vector dialect");
-
-  PassRegistration<TestVectorTransferFullPartialSplitPatterns>
-      vectorTransformFullPartialPass("test-vector-transfer-full-partial-split",
-                                     "Test conversion patterns to split "
-                                     "transfer ops via scf.if + linalg ops");
 }
 } // namespace mlir

From ee1c12708a4519361729205168dedb2b61bc2638 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Mon, 3 Aug 2020 16:19:06 +0100
Subject: [PATCH 0922/1035] [SCEV] If Start>=RHS, simplify (Start smin RHS) =
 RHS for trip counts.

In some cases, it seems like we can get rid of unnecessary s/umins by
using information from the loop guards (unless I am missing something).

One place where this seems to be helpful in practice is when computing
loop trip counts. This patch just changes howManyGreaterThans for now.
Note that this requires a loop for which we can check 'is guarded'.

On SPEC2000/SPEC2006/MultiSource, there are some notable changes for
some programs in the number of loops unrolled and trip counts computed.

```
Same hash: 179 (filtered out)
Remaining: 58
Metric: scalar-evolution.NumTripCountsComputed

Program                                        base    patch   diff
 test-suite...langs-C/compiler/compiler.test    25.00   31.00  24.0%
 test-suite.../Applications/SPASS/SPASS.test   2020.00 2323.00 15.0%
 test-suite...langs-C/allroots/allroots.test    29.00   32.00  10.3%
 test-suite.../Prolangs-C/loader/loader.test    17.00   18.00   5.9%
 test-suite...fice-ispell/office-ispell.test   253.00  265.00   4.7%
 test-suite...006/450.soplex/450.soplex.test   3552.00 3692.00  3.9%
 test-suite...chmarks/MallocBench/gs/gs.test   453.00  470.00   3.8%
 test-suite...ngs-C/assembler/assembler.test    29.00   30.00   3.4%
 test-suite.../Benchmarks/Ptrdist/bc/bc.test   263.00  270.00   2.7%
 test-suite...rks/FreeBench/pifft/pifft.test   722.00  741.00   2.6%
 test-suite...count/automotive-bitcount.test    41.00   42.00   2.4%
 test-suite...0/253.perlbmk/253.perlbmk.test   1417.00 1451.00  2.4%
 test-suite...000/197.parser/197.parser.test   387.00  396.00   2.3%
 test-suite...lications/sqlite3/sqlite3.test   1168.00 1189.00  1.8%
 test-suite...000/255.vortex/255.vortex.test   173.00  176.00   1.7%

Metric: loop-unroll.NumUnrolled

Program                                        base   patch  diff
 test-suite...langs-C/compiler/compiler.test     1.00   3.00 200.0%
 test-suite.../Applications/SPASS/SPASS.test   134.00 234.00 74.6%
 test-suite...count/automotive-bitcount.test     3.00   4.00 33.3%
 test-suite.../Prolangs-C/loader/loader.test     3.00   4.00 33.3%
 test-suite...langs-C/allroots/allroots.test     3.00   4.00 33.3%
 test-suite...Source/Benchmarks/sim/sim.test    10.00  12.00 20.0%
 test-suite...fice-ispell/office-ispell.test    21.00  25.00 19.0%
 test-suite.../Benchmarks/Ptrdist/bc/bc.test    32.00  38.00 18.8%
 test-suite...006/450.soplex/450.soplex.test   300.00 352.00 17.3%
 test-suite...rks/FreeBench/pifft/pifft.test    60.00  69.00 15.0%
 test-suite...chmarks/MallocBench/gs/gs.test    57.00  63.00 10.5%
 test-suite...ngs-C/assembler/assembler.test    10.00  11.00 10.0%
 test-suite...0/253.perlbmk/253.perlbmk.test   145.00 157.00  8.3%
 test-suite...000/197.parser/197.parser.test    43.00  46.00  7.0%
 test-suite...TimberWolfMC/timberwolfmc.test   205.00 214.00  4.4%
 Geomean difference                                           7.6%
```

Fixes https://bugs.llvm.org/show_bug.cgi?id=46939
Fixes https://bugs.llvm.org/show_bug.cgi?id=46924 on X86.

Reviewed By: mkazantsev

Differential Revision: https://reviews.llvm.org/D85046
---
 llvm/lib/Analysis/ScalarEvolution.cpp                 | 11 +++++++++--
 .../ScalarEvolution/pr46939-trip-count-count-down.ll  |  8 ++++----
 llvm/test/Transforms/HardwareLoops/scalar-while.ll    |  8 ++------
 3 files changed, 15 insertions(+), 12 deletions(-)

diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp
index a961be8cc35e1..23ad77ff60540 100644
--- a/llvm/lib/Analysis/ScalarEvolution.cpp
+++ b/llvm/lib/Analysis/ScalarEvolution.cpp
@@ -10628,8 +10628,15 @@ ScalarEvolution::howManyGreaterThans(const SCEV *LHS, const SCEV *RHS,
 
   const SCEV *Start = IV->getStart();
   const SCEV *End = RHS;
-  if (!isLoopEntryGuardedByCond(L, Cond, getAddExpr(Start, Stride), RHS))
-    End = IsSigned ? getSMinExpr(RHS, Start) : getUMinExpr(RHS, Start);
+  if (!isLoopEntryGuardedByCond(L, Cond, getAddExpr(Start, Stride), RHS)) {
+    // If we know that Start >= RHS in the context of loop, then we know that
+    // min(RHS, Start) = RHS at this point.
+    if (isLoopEntryGuardedByCond(
+            L, IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE, Start, RHS))
+      End = RHS;
+    else
+      End = IsSigned ? getSMinExpr(RHS, Start) : getUMinExpr(RHS, Start);
+  }
 
   const SCEV *BECount = computeBECount(getMinusSCEV(Start, End), Stride, false);
 
diff --git a/llvm/test/Analysis/ScalarEvolution/pr46939-trip-count-count-down.ll b/llvm/test/Analysis/ScalarEvolution/pr46939-trip-count-count-down.ll
index f1001ecec0fde..803652e47cf62 100644
--- a/llvm/test/Analysis/ScalarEvolution/pr46939-trip-count-count-down.ll
+++ b/llvm/test/Analysis/ScalarEvolution/pr46939-trip-count-count-down.ll
@@ -6,13 +6,13 @@ define void @reverse_loop(i32 %n) {
 ; CHECK-LABEL: 'reverse_loop'
 ; CHECK-NEXT:  Classifying expressions for: @reverse_loop
 ; CHECK-NEXT:    %i.011 = phi i32 [ %n, %for.body.lr.ph ], [ %dec, %for.body ]
-; CHECK-NEXT:    --> {%n,+,-1}<nsw><%for.body> U: full-set S: full-set Exits: (0 smin %n) LoopDispositions: { %for.body: Computable }
+; CHECK-NEXT:    --> {%n,+,-1}<nsw><%for.body> U: full-set S: full-set Exits: 0 LoopDispositions: { %for.body: Computable }
 ; CHECK-NEXT:    %dec = add nsw i32 %i.011, -1
-; CHECK-NEXT:    --> {(-1 + %n),+,-1}<nw><%for.body> U: full-set S: full-set Exits: (-1 + (0 smin %n)) LoopDispositions: { %for.body: Computable }
+; CHECK-NEXT:    --> {(-1 + %n),+,-1}<nw><%for.body> U: full-set S: full-set Exits: -1 LoopDispositions: { %for.body: Computable }
 ; CHECK-NEXT:  Determining loop execution counts for: @reverse_loop
-; CHECK-NEXT:  Loop %for.body: backedge-taken count is ((-1 * (0 smin %n)) + %n)
+; CHECK-NEXT:  Loop %for.body: backedge-taken count is %n
 ; CHECK-NEXT:  Loop %for.body: max backedge-taken count is 2147483647
-; CHECK-NEXT:  Loop %for.body: Predicated backedge-taken count is ((-1 * (0 smin %n)) + %n)
+; CHECK-NEXT:  Loop %for.body: Predicated backedge-taken count is %n
 ; CHECK-NEXT:   Predicates:
 ; CHECK:       Loop %for.body: Trip multiple is 1
 ;
diff --git a/llvm/test/Transforms/HardwareLoops/scalar-while.ll b/llvm/test/Transforms/HardwareLoops/scalar-while.ll
index aac94ecb86264..0b9847b33c714 100644
--- a/llvm/test/Transforms/HardwareLoops/scalar-while.ll
+++ b/llvm/test/Transforms/HardwareLoops/scalar-while.ll
@@ -76,18 +76,14 @@ while.end:
 ; CHECK-GUARD:   br i1 %cmp4, label %while.end, label %while.body.preheader
 ; CHECK-GUARD: while.body.preheader:
 ; CHECK-GUARD:   [[ADD:%[^ ]+]] = add i32 %i, 1
-; CHECK-GUARD:   [[SEL:%[^ ]+]] = icmp slt i32 %N, %i
-; CHECK-GUARD:   [[MIN:%[^ ]+]] = select i1 [[SEL]], i32 %N, i32 %i
-; CHECK-GUARD:   [[COUNT:%[^ ]+]] = sub i32 [[ADD]], [[MIN]]
+; CHECK-GUARD:   [[COUNT:%[^ ]+]] = sub i32 [[ADD]], %N
 ; CHECK-GUARD:   call void @llvm.set.loop.iterations.i32(i32 [[COUNT]])
 ; CHECK-GUARD:   br label %while.body
 
 ; CHECK-LABEL: while_gte
 ; CHECK: while.body.preheader:
 ; CHECK: [[ADD:%[^ ]+]] = add i32 %i, 1
-; CHECK: [[SEL:%[^ ]+]] = icmp slt i32 %N, %i
-; CHECK: [[MIN:%[^ ]+]] = select i1 [[SEL]], i32 %N, i32 %i
-; CHECK: [[COUNT:%[^ ]+]] = sub i32 [[ADD]], [[MIN]]
+; CHECK: [[COUNT:%[^ ]+]] = sub i32 [[ADD]], %N
 ; CHECK: call void @llvm.set.loop.iterations.i32(i32 [[COUNT]])
 ; CHECK: br label %while.body
 

From 3ebd1ba64f3d6f1e75f43213c50f0d1bd3902228 Mon Sep 17 00:00:00 2001
From: Gui Andrade <guiand@google.com>
Date: Fri, 31 Jul 2020 18:53:15 +0000
Subject: [PATCH 0923/1035] [MSAN] Instrument freeze instruction by clearing
 shadow

Freeze always returns a defined value. This also prevents msan from
checking the input shadow, which happened because freeze wasn't
explicitly visited.

Differential Revision: https://reviews.llvm.org/D85040
---
 .../Instrumentation/MemorySanitizer.cpp       |  6 +++++
 .../Instrumentation/MemorySanitizer/freeze.ll | 23 +++++++++++++++++++
 2 files changed, 29 insertions(+)
 create mode 100644 llvm/test/Instrumentation/MemorySanitizer/freeze.ll

diff --git a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
index edde80ce0ee8f..0f354c1da490a 100644
--- a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
@@ -4068,6 +4068,12 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     setOrigin(&I, getCleanOrigin());
   }
 
+  void visitFreezeInst(FreezeInst &I) {
+    // Freeze always returns a fully defined value.
+    setShadow(&I, getCleanShadow(&I));
+    setOrigin(&I, getCleanOrigin());
+  }
+
   void visitInstruction(Instruction &I) {
     // Everything else: stop propagating and check for poisoned shadow.
     if (ClDumpStrictInstructions)
diff --git a/llvm/test/Instrumentation/MemorySanitizer/freeze.ll b/llvm/test/Instrumentation/MemorySanitizer/freeze.ll
new file mode 100644
index 0000000000000..4be8d9efd6319
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/freeze.ll
@@ -0,0 +1,23 @@
+; RUN: opt < %s -msan-check-access-address=0 -S -passes=msan 2>&1 | FileCheck %s
+; RUN: opt < %s -msan-check-access-address=0 -msan-track-origins=2 -S -passes=msan 2>&1 | FileCheck %s -check-prefixes=CHECK,CHECK-ORIGIN
+; RUN: opt < %s -msan -msan-check-access-address=0 -S | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i32 @nofreeze(i32* %ptr) sanitize_memory {
+    ; CHECK-LABEL: @nofreeze
+    %val = load i32, i32* %ptr
+    ; CHECK: [[SHADOW_PTR:%.*]] = inttoptr
+    ; CHECK: [[SHADOW:%.*]] = load i32, i32* [[SHADOW_PTR]]
+    ; CHECK: store i32 [[SHADOW]], {{.*}} @__msan_retval_tls
+    ret i32 %val
+}
+
+define i32 @freeze_inst(i32* %ptr) sanitize_memory {
+    ; CHECK-LABEL: @freeze_inst
+    %val = load i32, i32* %ptr
+    %freeze_val = freeze i32 %val
+    ; CHECK-NOT: __msan_warning
+    ; CHECK: store i32 0, {{.*}} @__msan_retval_tls
+    ret i32 %freeze_val
+}

From caf002c7be44cb6c54de5a1b19aa177f18b6b0c1 Mon Sep 17 00:00:00 2001
From: Gui Andrade <guiand@google.com>
Date: Mon, 3 Aug 2020 16:45:35 +0000
Subject: [PATCH 0924/1035] [Utils] Add noundef attribute to vim/emacs/vscode
 syntax scripts

Differential Revision: https://reviews.llvm.org/D84553
---
 llvm/utils/emacs/llvm-mode.el                      | 2 +-
 llvm/utils/vim/syntax/llvm.vim                     | 1 +
 llvm/utils/vscode/llvm/syntaxes/ll.tmLanguage.yaml | 1 +
 3 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/llvm/utils/emacs/llvm-mode.el b/llvm/utils/emacs/llvm-mode.el
index 73b02763d0166..1d0b97c5cd094 100644
--- a/llvm/utils/emacs/llvm-mode.el
+++ b/llvm/utils/emacs/llvm-mode.el
@@ -25,7 +25,7 @@
        '("alwaysinline" "argmemonly" "builtin" "cold" "convergent" "inaccessiblememonly"
          "inaccessiblemem_or_argmemonly" "inlinehint" "jumptable" "minsize" "naked" "nobuiltin"
          "noduplicate" "noimplicitfloat" "noinline" "nonlazybind" "noredzone" "noreturn"
-         "norecurse" "nounwind" "optnone" "optsize" "readnone" "readonly" "returns_twice"
+         "norecurse" "noundef" "nounwind" "optnone" "optsize" "readnone" "readonly" "returns_twice"
          "speculatable" "ssp" "sspreq" "sspstrong" "safestack" "sanitize_address" "sanitize_hwaddress" "sanitize_memtag"
          "sanitize_thread" "sanitize_memory" "strictfp" "uwtable" "writeonly" "immarg") 'symbols) . font-lock-constant-face)
    ;; Variables
diff --git a/llvm/utils/vim/syntax/llvm.vim b/llvm/utils/vim/syntax/llvm.vim
index ce36b761d5daa..ebdd8faae3ef9 100644
--- a/llvm/utils/vim/syntax/llvm.vim
+++ b/llvm/utils/vim/syntax/llvm.vim
@@ -119,6 +119,7 @@ syn keyword llvmKeyword
       \ norecurse
       \ noredzone
       \ noreturn
+      \ noundef
       \ nounwind
       \ optnone
       \ optsize
diff --git a/llvm/utils/vscode/llvm/syntaxes/ll.tmLanguage.yaml b/llvm/utils/vscode/llvm/syntaxes/ll.tmLanguage.yaml
index 117ec134d5738..e6c574444321c 100644
--- a/llvm/utils/vscode/llvm/syntaxes/ll.tmLanguage.yaml
+++ b/llvm/utils/vscode/llvm/syntaxes/ll.tmLanguage.yaml
@@ -216,6 +216,7 @@ patterns:
             \\bnorecurse\\b|\
             \\bnoredzone\\b|\
             \\bnoreturn\\b|\
+            \\bnoundef\\b|\
             \\bnounwind\\b|\
             \\boptnone\\b|\
             \\boptsize\\b|\

From 4b1b109c5126efc963cc19949df5201e40f1bcc1 Mon Sep 17 00:00:00 2001
From: Mircea Trofin <mtrofin@google.com>
Date: Thu, 30 Jul 2020 12:44:07 -0700
Subject: [PATCH 0925/1035] [llvm] Add a parser from JSON to TensorSpec

A JSON->TensorSpec utility we will use subsequently to specify
additional outputs needed for certain training scenarios.

Differential Revision: https://reviews.llvm.org/D84976
---
 llvm/include/llvm/Analysis/Utils/TFUtils.h | 44 ++++++++++++----
 llvm/lib/Analysis/TFUtils.cpp              | 59 +++++++++++++++-------
 llvm/unittests/Analysis/TFUtilsTest.cpp    | 29 +++++++++++
 3 files changed, 103 insertions(+), 29 deletions(-)

diff --git a/llvm/include/llvm/Analysis/Utils/TFUtils.h b/llvm/include/llvm/Analysis/Utils/TFUtils.h
index 512f45bb5671a..d4450276a22ee 100644
--- a/llvm/include/llvm/Analysis/Utils/TFUtils.h
+++ b/llvm/include/llvm/Analysis/Utils/TFUtils.h
@@ -13,6 +13,7 @@
 
 #ifdef LLVM_HAVE_TF_API
 #include "llvm/IR/LLVMContext.h"
+#include "llvm/Support/JSON.h"
 
 #include <memory>
 #include <vector>
@@ -58,6 +59,13 @@ class TensorSpec final {
   int typeIndex() const { return TypeIndex; }
   const std::vector<int64_t> &shape() const { return Shape; }
 
+  bool operator==(const TensorSpec &Other) const {
+    return Name == Other.Name && Port == Other.Port &&
+           TypeIndex == Other.TypeIndex && Shape == Other.Shape;
+  }
+
+  bool operator!=(const TensorSpec &Other) const { return !(*this == Other); }
+
 private:
   TensorSpec(const std::string &Name, int Port, int TypeIndex,
              const std::vector<int64_t> &Shape)
@@ -73,6 +81,9 @@ class TensorSpec final {
   std::vector<int64_t> Shape;
 };
 
+Optional<TensorSpec> getTensorSpecFromJSON(LLVMContext &Ctx,
+                                           const json::Value &Value);
+
 class TFModelEvaluator final {
 public:
   /// The result of a model evaluation. Handles the lifetime of the output
@@ -124,17 +135,28 @@ class TFModelEvaluator final {
   std::unique_ptr<TFModelEvaluatorImpl> Impl;
 };
 
-template <> int TensorSpec::getDataType<float>();
-template <> int TensorSpec::getDataType<double>();
-template <> int TensorSpec::getDataType<int8_t>();
-template <> int TensorSpec::getDataType<uint8_t>();
-template <> int TensorSpec::getDataType<int16_t>();
-template <> int TensorSpec::getDataType<uint16_t>();
-template <> int TensorSpec::getDataType<int32_t>();
-template <> int TensorSpec::getDataType<uint32_t>();
-template <> int TensorSpec::getDataType<int64_t>();
-template <> int TensorSpec::getDataType<uint64_t>();
-
+/// List of supported types, as a triple:
+/// C++ type
+/// short name (for strings, for instance)
+/// capitalized short name (for enums, for instance)
+#define TFUTILS_SUPPORTED_TYPES(M)                                             \
+  M(float, float, FLOAT)                                                       \
+  M(double, double, DOUBLE)                                                    \
+  M(int8_t, int8, INT8)                                                        \
+  M(uint8_t, uint8, UINT8)                                                     \
+  M(int16_t, int16, INT16)                                                     \
+  M(uint16_t, uint16, UINT16)                                                  \
+  M(int32_t, int32, INT32)                                                     \
+  M(uint32_t, uint32, UINT32)                                                  \
+  M(int64_t, int64, INT64)                                                     \
+  M(uint64_t, uint64, UINT64)
+
+#define TFUTILS_GETDATATYPE_DEF(T, S, C)                                       \
+  template <> int TensorSpec::getDataType<T>();
+
+TFUTILS_SUPPORTED_TYPES(TFUTILS_GETDATATYPE_DEF)
+
+#undef TFUTILS_GETDATATYPE_DEF
 } // namespace llvm
 
 #endif // LLVM_HAVE_TF_API
diff --git a/llvm/lib/Analysis/TFUtils.cpp b/llvm/lib/Analysis/TFUtils.cpp
index b0ff19857963b..8fd4011e6cd42 100644
--- a/llvm/lib/Analysis/TFUtils.cpp
+++ b/llvm/lib/Analysis/TFUtils.cpp
@@ -13,9 +13,10 @@
 #include "llvm/Config/config.h"
 #if defined(LLVM_HAVE_TF_API)
 
-#include "llvm/Analysis/Utils/TFUtils.h"
 #include "llvm/ADT/Twine.h"
+#include "llvm/Analysis/Utils/TFUtils.h"
 #include "llvm/Support/Debug.h"
+#include "llvm/Support/JSON.h"
 #include "llvm/Support/ManagedStatic.h"
 #include "llvm/Support/raw_ostream.h"
 
@@ -83,6 +84,41 @@ class EvaluationResultImpl {
   std::vector<TF_Tensor *> Output;
 };
 
+Optional<TensorSpec> getTensorSpecFromJSON(LLVMContext &Ctx,
+                                           const json::Value &Value) {
+  auto EmitError = [&](const llvm::Twine &Message) -> Optional<TensorSpec> {
+    std::string S;
+    llvm::raw_string_ostream OS(S);
+    OS << Value;
+    Ctx.emitError("Unable to parse JSON Value as spec (" + Message + "): " + S);
+    return None;
+  };
+  json::ObjectMapper Mapper(Value);
+  if (!Mapper)
+    return EmitError("Value is not a dict");
+
+  std::string TensorName;
+  int TensorPort = -1;
+  std::string TensorType;
+  std::vector<int64_t> TensorShape;
+
+  if (!Mapper.map<std::string>("name", TensorName))
+    return EmitError("'name' property not present or not a string");
+  if (!Mapper.map<std::string>("type", TensorType))
+    return EmitError("'type' property not present or not a string");
+  if (!Mapper.map<int>("port", TensorPort))
+    return EmitError("'port' property not present or not an int");
+  if (!Mapper.map<std::vector<int64_t>>("shape", TensorShape))
+    return EmitError("'shape' property not present or not an int array");
+
+#define PARSE_TYPE(T, S, E)                                                    \
+  if (TensorType == #S)                                                        \
+    return TensorSpec::createSpec<T>(TensorName, TensorShape, TensorPort);
+  TFUTILS_SUPPORTED_TYPES(PARSE_TYPE)
+#undef PARSE_TYPE
+  return None;
+}
+
 class TFModelEvaluatorImpl {
 public:
   TFModelEvaluatorImpl(StringRef SavedModelPath,
@@ -249,25 +285,12 @@ void *TFModelEvaluator::EvaluationResult::getUntypedTensorValue(size_t Index) {
   return TF_TensorData(Impl->getOutput()[Index]);
 }
 
-template <> int TensorSpec::getDataType<float>() { return TF_FLOAT; }
-
-template <> int TensorSpec::getDataType<double>() { return TF_DOUBLE; }
-
-template <> int TensorSpec::getDataType<int8_t>() { return TF_INT8; }
-
-template <> int TensorSpec::getDataType<uint8_t>() { return TF_UINT8; }
-
-template <> int TensorSpec::getDataType<int16_t>() { return TF_INT16; }
-
-template <> int TensorSpec::getDataType<uint16_t>() { return TF_UINT16; }
-
-template <> int TensorSpec::getDataType<int32_t>() { return TF_INT32; }
-
-template <> int TensorSpec::getDataType<uint32_t>() { return TF_UINT32; }
+#define TFUTILS_GETDATATYPE_IMPL(T, S, E)                                      \
+  template <> int TensorSpec::getDataType<T>() { return TF_##E; }
 
-template <> int TensorSpec::getDataType<int64_t>() { return TF_INT64; }
+TFUTILS_SUPPORTED_TYPES(TFUTILS_GETDATATYPE_IMPL)
 
-template <> int TensorSpec::getDataType<uint64_t>() { return TF_UINT64; }
+#undef TFUTILS_GETDATATYPE_IMPL
 
 TFModelEvaluator::EvaluationResult::~EvaluationResult() {}
 TFModelEvaluator::~TFModelEvaluator() {}
diff --git a/llvm/unittests/Analysis/TFUtilsTest.cpp b/llvm/unittests/Analysis/TFUtilsTest.cpp
index e96d34092c7e5..abdf2b2b97844 100644
--- a/llvm/unittests/Analysis/TFUtilsTest.cpp
+++ b/llvm/unittests/Analysis/TFUtilsTest.cpp
@@ -94,3 +94,32 @@ TEST(TFUtilsTest, EvalError) {
   EXPECT_FALSE(ER.hasValue());
   EXPECT_FALSE(Evaluator.isValid());
 }
+
+TEST(TFUtilsTest, JSONParsing) {
+  auto Value = json::parse(
+      R"({"name": "tensor_name", 
+        "port": 2, 
+        "type": "int32", 
+        "shape":[1,4]
+        })");
+  EXPECT_TRUE(!!Value);
+  LLVMContext Ctx;
+  Optional<TensorSpec> Spec = getTensorSpecFromJSON(Ctx, *Value);
+  EXPECT_TRUE(Spec.hasValue());
+  EXPECT_EQ(*Spec, TensorSpec::createSpec<int32_t>("tensor_name", {1, 4}, 2));
+}
+
+TEST(TFUtilsTest, JSONParsingInvalidTensorType) {
+  auto Value = json::parse(
+      R"(
+        {"name": "tensor_name", 
+        "port": 2, 
+        "type": "no such type", 
+        "shape":[1,4]
+        }
+      )");
+  EXPECT_TRUE(!!Value);
+  LLVMContext Ctx;
+  auto Spec = getTensorSpecFromJSON(Ctx, *Value);
+  EXPECT_FALSE(Spec.hasValue());
+}

From d313e9c12ed3541f63a36e3b0d59e9e1185603d2 Mon Sep 17 00:00:00 2001
From: Nicolas Vasilache <ntv@google.com>
Date: Mon, 3 Aug 2020 12:24:53 -0400
Subject: [PATCH 0926/1035] [mlir][Vector] Add transformation + pattern to
 split vector.transfer_read into full and partial copies.

This revision adds a transformation and a pattern that rewrites a "maybe masked" `vector.transfer_read %view[...], %pad `into a pattern resembling:

```
   %1:3 = scf.if (%inBounds) {
      scf.yield %view : memref<A...>, index, index
    } else {
      %2 = vector.transfer_read %view[...], %pad : memref<A...>, vector<...>
      %3 = vector.type_cast %extra_alloc : memref<...> to
      memref<vector<...>> store %2, %3[] : memref<vector<...>> %4 =
      memref_cast %extra_alloc: memref<B...> to memref<A...> scf.yield %4 :
      memref<A...>, index, index
   }
   %res= vector.transfer_read %1#0[%1#1, %1#2] {masked = [false ... false]}
```
where `extra_alloc` is a top of the function alloca'ed buffer of one vector.

This rewrite makes it possible to realize the "always full tile" abstraction where vector.transfer_read operations are guaranteed to read from a padded full buffer.
The extra work only occurs on the boundary tiles.

Differential Revision: https://reviews.llvm.org/D84631
---
 .../mlir/Dialect/Vector/VectorTransforms.h    |  64 +++++
 .../mlir/Interfaces/VectorInterfaces.td       |  13 +
 mlir/lib/Dialect/Vector/CMakeLists.txt        |   1 +
 mlir/lib/Dialect/Vector/VectorTransforms.cpp  | 234 ++++++++++++++++++
 .../vector-transfer-full-partial-split.mlir   | 102 ++++++++
 .../lib/Transforms/TestVectorTransforms.cpp   |  16 ++
 6 files changed, 430 insertions(+)
 create mode 100644 mlir/test/Dialect/Vector/vector-transfer-full-partial-split.mlir

diff --git a/mlir/include/mlir/Dialect/Vector/VectorTransforms.h b/mlir/include/mlir/Dialect/Vector/VectorTransforms.h
index 0d18c5aa782d1..835ad18a79ad2 100644
--- a/mlir/include/mlir/Dialect/Vector/VectorTransforms.h
+++ b/mlir/include/mlir/Dialect/Vector/VectorTransforms.h
@@ -17,6 +17,11 @@
 namespace mlir {
 class MLIRContext;
 class OwningRewritePatternList;
+class VectorTransferOpInterface;
+
+namespace scf {
+class IfOp;
+} // namespace scf
 
 /// Collect a set of patterns to convert from the Vector dialect to itself.
 /// Should be merged with populateVectorToSCFLoweringPattern.
@@ -104,6 +109,65 @@ struct UnrollVectorPattern : public OpRewritePattern<OpTy> {
   FilterConstraintType filter;
 };
 
+/// Split a vector.transfer operation into an unmasked fastpath vector.transfer
+/// and a slowpath masked vector.transfer. If `ifOp` is not null and the result
+/// is `success, the `ifOp` points to the newly created conditional upon
+/// function return. To accomodate for the fact that the original
+/// vector.transfer indexing may be arbitrary and the slow path indexes @[0...0]
+/// in the temporary buffer, the scf.if op returns a view and values of type
+/// index. At this time, only vector.transfer_read is implemented.
+///
+/// Example (a 2-D vector.transfer_read):
+/// ```
+///    %1 = vector.transfer_read %0[...], %pad : memref<A...>, vector<...>
+/// ```
+/// is transformed into:
+/// ```
+///    %1:3 = scf.if (%inBounds) {
+///       scf.yield %0 : memref<A...>, index, index
+///     } else {
+///       %2 = vector.transfer_read %0[...], %pad : memref<A...>, vector<...>
+///       %3 = vector.type_cast %extra_alloc : memref<...> to
+///       memref<vector<...>> store %2, %3[] : memref<vector<...>> %4 =
+///       memref_cast %extra_alloc: memref<B...> to memref<A...> scf.yield %4 :
+///       memref<A...>, index, index
+//     }
+///    %0 = vector.transfer_read %1#0[%1#1, %1#2] {masked = [false ... false]}
+/// ```
+/// where `extra_alloc` is a top of the function alloca'ed buffer of one vector.
+///
+/// Preconditions:
+///  1. `xferOp.permutation_map()` must be a minor identity map
+///  2. the rank of the `xferOp.memref()` and the rank of the `xferOp.vector()`
+///  must be equal. This will be relaxed in the future but requires
+///  rank-reducing subviews.
+LogicalResult
+splitFullAndPartialTransferPrecondition(VectorTransferOpInterface xferOp);
+LogicalResult splitFullAndPartialTransfer(OpBuilder &b,
+                                          VectorTransferOpInterface xferOp,
+                                          scf::IfOp *ifOp = nullptr);
+
+/// Apply `splitFullAndPartialTransfer` selectively via a pattern. This pattern
+/// may take an extra filter to perform selection at a finer granularity.
+struct VectorTransferFullPartialRewriter : public RewritePattern {
+  using FilterConstraintType =
+      std::function<LogicalResult(VectorTransferOpInterface op)>;
+
+  explicit VectorTransferFullPartialRewriter(
+      MLIRContext *context,
+      FilterConstraintType filter =
+          [](VectorTransferOpInterface op) { return success(); },
+      PatternBenefit benefit = 1)
+      : RewritePattern(benefit, MatchAnyOpTypeTag()), filter(filter) {}
+
+  /// Performs the rewrite.
+  LogicalResult matchAndRewrite(Operation *op,
+                                PatternRewriter &rewriter) const override;
+
+private:
+  FilterConstraintType filter;
+};
+
 } // namespace vector
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Interfaces/VectorInterfaces.td b/mlir/include/mlir/Interfaces/VectorInterfaces.td
index aefbb7d471172..218715318a867 100644
--- a/mlir/include/mlir/Interfaces/VectorInterfaces.td
+++ b/mlir/include/mlir/Interfaces/VectorInterfaces.td
@@ -160,6 +160,19 @@ def VectorTransferOpInterface : OpInterface<"VectorTransferOpInterface"> {
       /*defaultImplementation=*/
         "return $_op.getMemRefType().getRank() - $_op.getTransferRank();"
     >,
+    InterfaceMethod<
+      /*desc=*/[{ Returns true if at least one of the dimensions is masked.}],
+      /*retTy=*/"bool",
+      /*methodName=*/"hasMaskedDim",
+      /*args=*/(ins),
+      /*methodBody=*/"",
+      /*defaultImplementation=*/[{
+        for (unsigned idx = 0, e = $_op.getTransferRank(); idx < e; ++idx)
+          if ($_op.isMaskedDim(idx))
+            return true;
+        return false;
+      }]
+    >,
     InterfaceMethod<
       /*desc=*/[{
       Helper function to account for the fact that `permutationMap` results and
diff --git a/mlir/lib/Dialect/Vector/CMakeLists.txt b/mlir/lib/Dialect/Vector/CMakeLists.txt
index 7b34f1933c42c..13dbf6da73fa2 100644
--- a/mlir/lib/Dialect/Vector/CMakeLists.txt
+++ b/mlir/lib/Dialect/Vector/CMakeLists.txt
@@ -11,6 +11,7 @@ add_mlir_dialect_library(MLIRVector
   MLIRVectorOpsIncGen
 
   LINK_LIBS PUBLIC
+  MLIRAffineEDSC
   MLIREDSC
   MLIRIR
   MLIRStandardOps
diff --git a/mlir/lib/Dialect/Vector/VectorTransforms.cpp b/mlir/lib/Dialect/Vector/VectorTransforms.cpp
index 197b1c62274b2..573b822503f3a 100644
--- a/mlir/lib/Dialect/Vector/VectorTransforms.cpp
+++ b/mlir/lib/Dialect/Vector/VectorTransforms.cpp
@@ -12,9 +12,13 @@
 
 #include <type_traits>
 
+#include "mlir/Dialect/Affine/EDSC/Intrinsics.h"
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
+#include "mlir/Dialect/SCF/EDSC/Intrinsics.h"
+#include "mlir/Dialect/StandardOps/EDSC/Intrinsics.h"
 #include "mlir/Dialect/StandardOps/IR/Ops.h"
 #include "mlir/Dialect/Utils/StructuredOpsUtils.h"
+#include "mlir/Dialect/Vector/EDSC/Intrinsics.h"
 #include "mlir/Dialect/Vector/VectorOps.h"
 #include "mlir/Dialect/Vector/VectorTransforms.h"
 #include "mlir/Dialect/Vector/VectorUtils.h"
@@ -1985,6 +1989,236 @@ Value ContractionOpLowering::lowerReduction(vector::ContractionOp op,
 
 } // namespace mlir
 
+static Optional<int64_t> extractConstantIndex(Value v) {
+  if (auto cstOp = v.getDefiningOp<ConstantIndexOp>())
+    return cstOp.getValue();
+  if (auto affineApplyOp = v.getDefiningOp<AffineApplyOp>())
+    if (affineApplyOp.getAffineMap().isSingleConstant())
+      return affineApplyOp.getAffineMap().getSingleConstantResult();
+  return None;
+}
+
+// Missing foldings of scf.if make it necessary to perform poor man's folding
+// eagerly, especially in the case of unrolling. In the future, this should go
+// away once scf.if folds properly.
+static Value createScopedFoldedSLE(Value v, Value ub) {
+  using namespace edsc::op;
+  auto maybeCstV = extractConstantIndex(v);
+  auto maybeCstUb = extractConstantIndex(ub);
+  if (maybeCstV && maybeCstUb && *maybeCstV < *maybeCstUb)
+    return Value();
+  return sle(v, ub);
+}
+
+// Operates under a scoped context to build the condition to ensure that a
+// particular VectorTransferOpInterface is unmasked.
+static Value createScopedInBoundsCond(VectorTransferOpInterface xferOp) {
+  assert(xferOp.permutation_map().isMinorIdentity() &&
+         "Expected minor identity map");
+  Value inBoundsCond;
+  xferOp.zipResultAndIndexing([&](int64_t resultIdx, int64_t indicesIdx) {
+    // Zip over the resulting vector shape and memref indices.
+    // If the dimension is known to be unmasked, it does not participate in the
+    // construction of `inBoundsCond`.
+    if (!xferOp.isMaskedDim(resultIdx))
+      return;
+    int64_t vectorSize = xferOp.getVectorType().getDimSize(resultIdx);
+    using namespace edsc::op;
+    using namespace edsc::intrinsics;
+    // Fold or create the check that `index + vector_size` <= `memref_size`.
+    Value sum = xferOp.indices()[indicesIdx] + std_constant_index(vectorSize);
+    Value cond =
+        createScopedFoldedSLE(sum, std_dim(xferOp.memref(), indicesIdx));
+    if (!cond)
+      return;
+    // Conjunction over all dims for which we are in-bounds.
+    inBoundsCond = inBoundsCond ? inBoundsCond && cond : cond;
+  });
+  return inBoundsCond;
+}
+
+LogicalResult mlir::vector::splitFullAndPartialTransferPrecondition(
+    VectorTransferOpInterface xferOp) {
+  // TODO: expand support to these 2 cases.
+  if (!xferOp.permutation_map().isMinorIdentity())
+    return failure();
+  // TODO: relax this precondition. This will require rank-reducing subviews.
+  if (xferOp.getMemRefType().getRank() != xferOp.getTransferRank())
+    return failure();
+  // Must have some masked dimension to be a candidate for splitting.
+  if (!xferOp.hasMaskedDim())
+    return failure();
+  // Don't split transfer operations under IfOp, this avoids applying the
+  // pattern recursively.
+  // TODO: improve the condition to make it more applicable.
+  if (xferOp.getParentOfType<scf::IfOp>())
+    return failure();
+  return success();
+}
+
+MemRefType getCastCompatibleMemRefType(MemRefType aT, MemRefType bT) {
+  if (MemRefCastOp::areCastCompatible(aT, bT))
+    return aT;
+  if (aT.getRank() != bT.getRank())
+    return MemRefType();
+  int64_t aOffset, bOffset;
+  SmallVector<int64_t, 4> aStrides, bStrides;
+  if (failed(getStridesAndOffset(aT, aStrides, aOffset)) ||
+      failed(getStridesAndOffset(bT, bStrides, bOffset)) ||
+      aStrides.size() != bStrides.size())
+    return MemRefType();
+
+  ArrayRef<int64_t> aShape = aT.getShape(), bShape = bT.getShape();
+  int64_t resOffset;
+  SmallVector<int64_t, 4> resShape(aT.getRank(), 0),
+      resStrides(bT.getRank(), 0);
+  for (int64_t idx = 0, e = aT.getRank(); idx < e; ++idx) {
+    resShape[idx] =
+        (aShape[idx] == bShape[idx]) ? aShape[idx] : MemRefType::kDynamicSize;
+    resStrides[idx] = (aStrides[idx] == bStrides[idx])
+                          ? aStrides[idx]
+                          : MemRefType::kDynamicStrideOrOffset;
+  }
+  resOffset =
+      (aOffset == bOffset) ? aOffset : MemRefType::kDynamicStrideOrOffset;
+  return MemRefType::get(
+      resShape, aT.getElementType(),
+      makeStridedLinearLayoutMap(resStrides, resOffset, aT.getContext()));
+}
+
+/// Split a vector.transfer operation into an unmasked fastpath vector.transfer
+/// and a slowpath masked vector.transfer. If `ifOp` is not null and the result
+/// is `success, the `ifOp` points to the newly created conditional upon
+/// function return. To accomodate for the fact that the original
+/// vector.transfer indexing may be arbitrary and the slow path indexes @[0...0]
+/// in the temporary buffer, the scf.if op returns a view and values of type
+/// index. At this time, only vector.transfer_read is implemented.
+///
+/// Example (a 2-D vector.transfer_read):
+/// ```
+///    %1 = vector.transfer_read %0[...], %pad : memref<A...>, vector<...>
+/// ```
+/// is transformed into:
+/// ```
+///    %1:3 = scf.if (%inBounds) {
+///       scf.yield %0 : memref<A...>, index, index
+///     } else {
+///       %2 = vector.transfer_read %0[...], %pad : memref<A...>, vector<...>
+///       %3 = vector.type_cast %extra_alloc : memref<...> to
+///       memref<vector<...>> store %2, %3[] : memref<vector<...>> %4 =
+///       memref_cast %extra_alloc: memref<B...> to memref<A...> scf.yield %4 :
+///       memref<A...>, index, index
+//     }
+///    %0 = vector.transfer_read %1#0[%1#1, %1#2] {masked = [false ... false]}
+/// ```
+/// where `extra_alloc` is a top of the function alloca'ed buffer of one vector.
+///
+/// Preconditions:
+///  1. `xferOp.permutation_map()` must be a minor identity map
+///  2. the rank of the `xferOp.memref()` and the rank of the `xferOp.vector()`
+///  must be equal. This will be relaxed in the future but requires
+///  rank-reducing subviews.
+LogicalResult mlir::vector::splitFullAndPartialTransfer(
+    OpBuilder &b, VectorTransferOpInterface xferOp, scf::IfOp *ifOp) {
+  using namespace edsc;
+  using namespace edsc::intrinsics;
+
+  assert(succeeded(splitFullAndPartialTransferPrecondition(xferOp)) &&
+         "Expected splitFullAndPartialTransferPrecondition to hold");
+  auto xferReadOp = dyn_cast<vector::TransferReadOp>(xferOp.getOperation());
+
+  // TODO: add support for write case.
+  if (!xferReadOp)
+    return failure();
+
+  OpBuilder::InsertionGuard guard(b);
+  if (xferOp.memref().getDefiningOp())
+    b.setInsertionPointAfter(xferOp.memref().getDefiningOp());
+  else
+    b.setInsertionPoint(xferOp);
+  ScopedContext scope(b, xferOp.getLoc());
+  Value inBoundsCond = createScopedInBoundsCond(
+      cast<VectorTransferOpInterface>(xferOp.getOperation()));
+  if (!inBoundsCond)
+    return failure();
+
+  // Top of the function `alloc` for transient storage.
+  Value alloc;
+  {
+    FuncOp funcOp = xferOp.getParentOfType<FuncOp>();
+    OpBuilder::InsertionGuard guard(b);
+    b.setInsertionPointToStart(&funcOp.getRegion().front());
+    auto shape = xferOp.getVectorType().getShape();
+    Type elementType = xferOp.getVectorType().getElementType();
+    alloc = std_alloca(MemRefType::get(shape, elementType), ValueRange{},
+                       b.getI64IntegerAttr(32));
+  }
+
+  Value memref = xferOp.memref();
+  SmallVector<bool, 4> bools(xferOp.getTransferRank(), false);
+  auto unmaskedAttr = b.getBoolArrayAttr(bools);
+
+  MemRefType compatibleMemRefType = getCastCompatibleMemRefType(
+      xferOp.getMemRefType(), alloc.getType().cast<MemRefType>());
+
+  // Read case: full fill + partial copy -> unmasked vector.xfer_read.
+  Value zero = std_constant_index(0);
+  SmallVector<Type, 4> returnTypes(1 + xferOp.getTransferRank(),
+                                   b.getIndexType());
+  returnTypes[0] = compatibleMemRefType;
+  scf::IfOp fullPartialIfOp;
+  conditionBuilder(
+      returnTypes, inBoundsCond,
+      [&]() -> scf::ValueVector {
+        Value res = memref;
+        if (compatibleMemRefType != xferOp.getMemRefType())
+          res = std_memref_cast(memref, compatibleMemRefType);
+        scf::ValueVector viewAndIndices{res};
+        viewAndIndices.insert(viewAndIndices.end(), xferOp.indices().begin(),
+                              xferOp.indices().end());
+        return viewAndIndices;
+      },
+      [&]() -> scf::ValueVector {
+        Operation *newXfer =
+            ScopedContext::getBuilderRef().clone(*xferOp.getOperation());
+        Value vector = cast<VectorTransferOpInterface>(newXfer).vector();
+        std_store(vector, vector_type_cast(
+                              MemRefType::get({}, vector.getType()), alloc));
+
+        Value casted = std_memref_cast(alloc, compatibleMemRefType);
+        scf::ValueVector viewAndIndices{casted};
+        viewAndIndices.insert(viewAndIndices.end(), xferOp.getTransferRank(),
+                              zero);
+
+        return viewAndIndices;
+      },
+      &fullPartialIfOp);
+  if (ifOp)
+    *ifOp = fullPartialIfOp;
+
+  // Unmask the existing read op, it always reads from a full buffer.
+  for (unsigned i = 0, e = returnTypes.size(); i != e; ++i)
+    xferReadOp.setOperand(i, fullPartialIfOp.getResult(i));
+  xferOp.setAttr(vector::TransferReadOp::getMaskedAttrName(), unmaskedAttr);
+
+  return success();
+}
+
+LogicalResult mlir::vector::VectorTransferFullPartialRewriter::matchAndRewrite(
+    Operation *op, PatternRewriter &rewriter) const {
+  auto xferOp = dyn_cast<VectorTransferOpInterface>(op);
+  if (!xferOp || failed(splitFullAndPartialTransferPrecondition(xferOp)) ||
+      failed(filter(xferOp)))
+    return failure();
+  rewriter.startRootUpdate(xferOp);
+  if (succeeded(splitFullAndPartialTransfer(rewriter, xferOp))) {
+    rewriter.finalizeRootUpdate(xferOp);
+    return success();
+  }
+  rewriter.cancelRootUpdate(xferOp);
+  return failure();
+}
+
 // TODO: Add pattern to rewrite ExtractSlices(ConstantMaskOp).
 // TODO: Add this as DRR pattern.
 void mlir::vector::populateVectorToVectorTransformationPatterns(
diff --git a/mlir/test/Dialect/Vector/vector-transfer-full-partial-split.mlir b/mlir/test/Dialect/Vector/vector-transfer-full-partial-split.mlir
new file mode 100644
index 0000000000000..ef76247ee9d4b
--- /dev/null
+++ b/mlir/test/Dialect/Vector/vector-transfer-full-partial-split.mlir
@@ -0,0 +1,102 @@
+// RUN: mlir-opt %s -test-vector-transfer-full-partial-split | FileCheck %s
+
+// CHECK-DAG: #[[$map_p4:.*]] = affine_map<()[s0] -> (s0 + 4)>
+// CHECK-DAG: #[[$map_p8:.*]] = affine_map<()[s0] -> (s0 + 8)>
+// CHECK-DAG: #[[$map_2d_stride_1:.*]] = affine_map<(d0, d1)[s0, s1] -> (d0 * s1 + s0 + d1)>
+
+// CHECK-LABEL: split_vector_transfer_read_2d(
+//  CHECK-SAME: %[[A:[a-zA-Z0-9]*]]: memref
+//  CHECK-SAME: %[[i:[a-zA-Z0-9]*]]: index
+//  CHECK-SAME: %[[j:[a-zA-Z0-9]*]]: index
+func @split_vector_transfer_read_2d(%A: memref<?x8xf32>, %i: index, %j: index) -> vector<4x8xf32> {
+  %c0 = constant 0 : index
+  %f0 = constant 0.0 : f32
+
+  //  CHECK-DAG: %[[c0:.*]] = constant 0 : index
+  //  CHECK-DAG: %[[c8:.*]] = constant 8 : index
+  //  CHECK-DAG: %[[cst:.*]] = constant 0.000000e+00 : f32
+  // alloca for boundary full tile
+  //      CHECK: %[[alloc:.*]] = alloca() {alignment = 32 : i64} : memref<4x8xf32>
+  // %i + 4 <= dim(%A, 0)
+  //      CHECK: %[[idx0:.*]] = affine.apply #[[$map_p4]]()[%[[i]]]
+  //      CHECK: %[[d0:.*]] = dim %[[A]], %[[c0]] : memref<?x8xf32>
+  //      CHECK: %[[cmp0:.*]] = cmpi "sle", %[[idx0]], %[[d0]] : index
+  // %j + 8 <= dim(%A, 1)
+  //      CHECK: %[[idx1:.*]] = affine.apply #[[$map_p8]]()[%[[j]]]
+  //      CHECK: %[[cmp1:.*]] = cmpi "sle", %[[idx1]], %[[c8]] : index
+  // are both conds true
+  //      CHECK: %[[cond:.*]] = and %[[cmp0]], %[[cmp1]] : i1
+  //      CHECK: %[[ifres:.*]]:3 = scf.if %[[cond]] -> (memref<?x8xf32>, index, index) {
+  //               inBounds, just yield %A
+  //      CHECK:   scf.yield %[[A]], %[[i]], %[[j]] : memref<?x8xf32>, index, index
+  //      CHECK: } else {
+  //               slow path, fill tmp alloc and yield a memref_casted version of it
+  //      CHECK:   %[[slow:.*]] = vector.transfer_read %[[A]][%[[i]], %[[j]]], %cst :
+  // CHECK-SAME:     memref<?x8xf32>, vector<4x8xf32>
+  //      CHECK:   %[[cast_alloc:.*]] = vector.type_cast %[[alloc]] :
+  // CHECK-SAME:     memref<4x8xf32> to memref<vector<4x8xf32>>
+  //      CHECK:   store %[[slow]], %[[cast_alloc]][] : memref<vector<4x8xf32>>
+  //      CHECK:   %[[yielded:.*]] = memref_cast %[[alloc]] :
+  // CHECK-SAME:     memref<4x8xf32> to memref<?x8xf32>
+  //      CHECK:   scf.yield %[[yielded]], %[[c0]], %[[c0]] :
+  // CHECK-SAME:     memref<?x8xf32>, index, index
+  //      CHECK: }
+  //      CHECK: %[[res:.*]] = vector.transfer_read %[[ifres]]#0[%[[ifres]]#1, %[[ifres]]#2], %[[cst]]
+  // CHECK_SAME:   {masked = [false, false]} : memref<?x8xf32>, vector<4x8xf32>
+  %1 = vector.transfer_read %A[%i, %j], %f0 : memref<?x8xf32>, vector<4x8xf32>
+
+  // CHECK: return %[[res]] : vector<4x8xf32>
+  return %1: vector<4x8xf32>
+}
+
+// CHECK-LABEL: split_vector_transfer_read_strided_2d(
+//  CHECK-SAME: %[[A:[a-zA-Z0-9]*]]: memref
+//  CHECK-SAME: %[[i:[a-zA-Z0-9]*]]: index
+//  CHECK-SAME: %[[j:[a-zA-Z0-9]*]]: index
+func @split_vector_transfer_read_strided_2d(
+    %A: memref<7x8xf32, offset:?, strides:[?, 1]>,
+    %i: index, %j: index) -> vector<4x8xf32> {
+  %c0 = constant 0 : index
+  %f0 = constant 0.0 : f32
+
+  //  CHECK-DAG: %[[c0:.*]] = constant 0 : index
+  //  CHECK-DAG: %[[c7:.*]] = constant 7 : index
+  //  CHECK-DAG: %[[c8:.*]] = constant 8 : index
+  //  CHECK-DAG: %[[cst:.*]] = constant 0.000000e+00 : f32
+  // alloca for boundary full tile
+  //      CHECK: %[[alloc:.*]] = alloca() {alignment = 32 : i64} : memref<4x8xf32>
+  // %i + 4 <= dim(%A, 0)
+  //      CHECK: %[[idx0:.*]] = affine.apply #[[$map_p4]]()[%[[i]]]
+  //      CHECK: %[[cmp0:.*]] = cmpi "sle", %[[idx0]], %[[c7]] : index
+  // %j + 8 <= dim(%A, 1)
+  //      CHECK: %[[idx1:.*]] = affine.apply #[[$map_p8]]()[%[[j]]]
+  //      CHECK: %[[cmp1:.*]] = cmpi "sle", %[[idx1]], %[[c8]] : index
+  // are both conds true
+  //      CHECK: %[[cond:.*]] = and %[[cmp0]], %[[cmp1]] : i1
+  //      CHECK: %[[ifres:.*]]:3 = scf.if %[[cond]] -> (memref<?x8xf32, #[[$map_2d_stride_1]]>, index, index) {
+  //               inBounds but not cast-compatible: yield a memref_casted form of %A
+  //      CHECK:   %[[casted:.*]] = memref_cast %arg0 :
+  // CHECK-SAME:     memref<7x8xf32, #[[$map_2d_stride_1]]> to memref<?x8xf32, #[[$map_2d_stride_1]]>
+  //      CHECK:   scf.yield %[[casted]], %[[i]], %[[j]] :
+  // CHECK-SAME:     memref<?x8xf32, #[[$map_2d_stride_1]]>, index, index
+  //      CHECK: } else {
+  //               slow path, fill tmp alloc and yield a memref_casted version of it
+  //      CHECK:   %[[slow:.*]] = vector.transfer_read %[[A]][%[[i]], %[[j]]], %cst :
+  // CHECK-SAME:     memref<7x8xf32, #[[$map_2d_stride_1]]>, vector<4x8xf32>
+  //      CHECK:   %[[cast_alloc:.*]] = vector.type_cast %[[alloc]] :
+  // CHECK-SAME:     memref<4x8xf32> to memref<vector<4x8xf32>>
+  //      CHECK:   store %[[slow]], %[[cast_alloc]][] :
+  // CHECK-SAME:     memref<vector<4x8xf32>>
+  //      CHECK:   %[[yielded:.*]] = memref_cast %[[alloc]] :
+  // CHECK-SAME:     memref<4x8xf32> to memref<?x8xf32, #[[$map_2d_stride_1]]>
+  //      CHECK:   scf.yield %[[yielded]], %[[c0]], %[[c0]] :
+  // CHECK-SAME:     memref<?x8xf32, #[[$map_2d_stride_1]]>, index, index
+  //      CHECK: }
+  //      CHECK: %[[res:.*]] = vector.transfer_read {{.*}} {masked = [false, false]} :
+  // CHECK-SAME:   memref<?x8xf32, #[[$map_2d_stride_1]]>, vector<4x8xf32>
+  %1 = vector.transfer_read %A[%i, %j], %f0 :
+    memref<7x8xf32, offset:?, strides:[?, 1]>, vector<4x8xf32>
+
+  // CHECK: return %[[res]] : vector<4x8xf32>
+  return %1 : vector<4x8xf32>
+}
diff --git a/mlir/test/lib/Transforms/TestVectorTransforms.cpp b/mlir/test/lib/Transforms/TestVectorTransforms.cpp
index 2058706dcbdd3..0bba74e76385e 100644
--- a/mlir/test/lib/Transforms/TestVectorTransforms.cpp
+++ b/mlir/test/lib/Transforms/TestVectorTransforms.cpp
@@ -122,6 +122,17 @@ struct TestVectorUnrollingPatterns
   }
 };
 
+struct TestVectorTransferFullPartialSplitPatterns
+    : public PassWrapper<TestVectorTransferFullPartialSplitPatterns,
+                         FunctionPass> {
+  void runOnFunction() override {
+    MLIRContext *ctx = &getContext();
+    OwningRewritePatternList patterns;
+    patterns.insert<VectorTransferFullPartialRewriter>(ctx);
+    applyPatternsAndFoldGreedily(getFunction(), patterns);
+  }
+};
+
 } // end anonymous namespace
 
 namespace mlir {
@@ -141,5 +152,10 @@ void registerTestVectorConversions() {
   PassRegistration<TestVectorUnrollingPatterns> contractionUnrollingPass(
       "test-vector-unrolling-patterns",
       "Test conversion patterns to unroll contract ops in the vector dialect");
+
+  PassRegistration<TestVectorTransferFullPartialSplitPatterns>
+      vectorTransformFullPartialPass("test-vector-transfer-full-partial-split",
+                                     "Test conversion patterns to split "
+                                     "transfer ops via scf.if + linalg ops");
 }
 } // namespace mlir

From e888886cc3daf2c2d6c20cad51cd5ec2ffc24789 Mon Sep 17 00:00:00 2001
From: MaheshRavishankar <ravishankarm@google.com>
Date: Mon, 3 Aug 2020 10:04:16 -0700
Subject: [PATCH 0927/1035] [mlir][DialectConversion] Add support for
 mergeBlocks in ConversionPatternRewriter.

Differential Revision: https://reviews.llvm.org/D84795
---
 mlir/lib/Transforms/DialectConversion.cpp   |  61 +++++++++-
 mlir/test/Transforms/test-merge-blocks.mlir |  59 ++++++++++
 mlir/test/lib/Dialect/Test/TestOps.td       |  10 ++
 mlir/test/lib/Dialect/Test/TestPatterns.cpp | 116 ++++++++++++++++++++
 4 files changed, 242 insertions(+), 4 deletions(-)
 create mode 100644 mlir/test/Transforms/test-merge-blocks.mlir

diff --git a/mlir/lib/Transforms/DialectConversion.cpp b/mlir/lib/Transforms/DialectConversion.cpp
index 9778958a45885..713f0b73dfe04 100644
--- a/mlir/lib/Transforms/DialectConversion.cpp
+++ b/mlir/lib/Transforms/DialectConversion.cpp
@@ -602,7 +602,14 @@ struct OpReplacement {
 
 /// The kind of the block action performed during the rewrite.  Actions can be
 /// undone if the conversion fails.
-enum class BlockActionKind { Create, Erase, Move, Split, TypeConversion };
+enum class BlockActionKind {
+  Create,
+  Erase,
+  Merge,
+  Move,
+  Split,
+  TypeConversion
+};
 
 /// Original position of the given block in its parent region.  We cannot use
 /// a region iterator because it could have been invalidated by other region
@@ -612,6 +619,15 @@ struct BlockPosition {
   Region::iterator::difference_type position;
 };
 
+/// Information needed to undo the merge actions.
+/// - the source block, and
+/// - the Operation that was the last operation in the dest block before the
+///   merge (could be null if the dest block was empty).
+struct MergeInfo {
+  Block *sourceBlock;
+  Operation *destBlockLastInst;
+};
+
 /// The storage class for an undoable block action (one of BlockActionKind),
 /// contains the information necessary to undo this action.
 struct BlockAction {
@@ -621,6 +637,11 @@ struct BlockAction {
   static BlockAction getErase(Block *block, BlockPosition originalPos) {
     return {BlockActionKind::Erase, block, {originalPos}};
   }
+  static BlockAction getMerge(Block *block, Block *sourceBlock) {
+    BlockAction action{BlockActionKind::Merge, block, {}};
+    action.mergeInfo = {sourceBlock, block->empty() ? nullptr : &block->back()};
+    return action;
+  }
   static BlockAction getMove(Block *block, BlockPosition originalPos) {
     return {BlockActionKind::Move, block, {originalPos}};
   }
@@ -647,6 +668,9 @@ struct BlockAction {
     // In use if kind == BlockActionKind::Split and contains a pointer to the
     // block that was split into two parts.
     Block *originalBlock;
+    // In use if kind == BlockActionKind::Merge, and contains the information
+    // needed to undo the merge.
+    MergeInfo mergeInfo;
   };
 };
 } // end anonymous namespace
@@ -738,6 +762,9 @@ struct ConversionPatternRewriterImpl {
   /// Notifies that a block was split.
   void notifySplitBlock(Block *block, Block *continuation);
 
+  /// Notifies that `block` is being merged with `srcBlock`.
+  void notifyBlocksBeingMerged(Block *block, Block *srcBlock);
+
   /// Notifies that the blocks of a region are about to be moved.
   void notifyRegionIsBeingInlinedBefore(Region &region, Region &parent,
                                         Region::iterator before);
@@ -966,6 +993,20 @@ void ConversionPatternRewriterImpl::undoBlockActions(
           action.block);
       break;
     }
+    // Split the block at the position which was originally the end of the
+    // destination block (owned by action), and put the instructions back into
+    // the block used before the merge.
+    case BlockActionKind::Merge: {
+      Block *sourceBlock = action.mergeInfo.sourceBlock;
+      Block::iterator splitPoint =
+          (action.mergeInfo.destBlockLastInst
+               ? ++Block::iterator(action.mergeInfo.destBlockLastInst)
+               : action.block->begin());
+      sourceBlock->getOperations().splice(sourceBlock->begin(),
+                                          action.block->getOperations(),
+                                          splitPoint, action.block->end());
+      break;
+    }
     // Move the block back to its original position.
     case BlockActionKind::Move: {
       Region *originalRegion = action.originalPosition.region;
@@ -1161,6 +1202,11 @@ void ConversionPatternRewriterImpl::notifySplitBlock(Block *block,
   blockActions.push_back(BlockAction::getSplit(continuation, block));
 }
 
+void ConversionPatternRewriterImpl::notifyBlocksBeingMerged(Block *block,
+                                                            Block *srcBlock) {
+  blockActions.push_back(BlockAction::getMerge(block, srcBlock));
+}
+
 void ConversionPatternRewriterImpl::notifyRegionIsBeingInlinedBefore(
     Region &region, Region &parent, Region::iterator before) {
   for (auto &pair : llvm::enumerate(region)) {
@@ -1283,9 +1329,16 @@ Block *ConversionPatternRewriter::splitBlock(Block *block,
 /// PatternRewriter hook for merging a block into another.
 void ConversionPatternRewriter::mergeBlocks(Block *source, Block *dest,
                                             ValueRange argValues) {
-  // TODO: This requires fixing the implementation of
-  // 'replaceUsesOfBlockArgument', which currently isn't undoable.
-  llvm_unreachable("block merging updates are currently not supported");
+  impl->notifyBlocksBeingMerged(dest, source);
+  assert(llvm::all_of(source->getPredecessors(),
+                      [dest](Block *succ) { return succ == dest; }) &&
+         "expected 'source' to have no predecessors or only 'dest'");
+  assert(argValues.size() == source->getNumArguments() &&
+         "incorrect # of argument replacement values");
+  for (auto it : llvm::zip(source->getArguments(), argValues))
+    replaceUsesOfBlockArgument(std::get<0>(it), std::get<1>(it));
+  dest->getOperations().splice(dest->end(), source->getOperations());
+  eraseBlock(source);
 }
 
 /// PatternRewriter hook for moving blocks out of a region.
diff --git a/mlir/test/Transforms/test-merge-blocks.mlir b/mlir/test/Transforms/test-merge-blocks.mlir
new file mode 100644
index 0000000000000..65dd50569416e
--- /dev/null
+++ b/mlir/test/Transforms/test-merge-blocks.mlir
@@ -0,0 +1,59 @@
+// RUN: mlir-opt -allow-unregistered-dialect -split-input-file -test-merge-blocks -verify-diagnostics %s | FileCheck %s
+
+// CHECK-LABEL: @merge_blocks
+func @merge_blocks(%arg0: i32, %arg1 : i32) -> () {
+  //      CHECK: "test.merge_blocks"() ( {
+  // CHECK-NEXT:   "test.return"
+  // CHECK-NEXT: })
+  // CHECK-NEXT: "test.return"
+  %0:2 = "test.merge_blocks"() ({
+  ^bb0:
+     "test.br"(%arg0, %arg1)[^bb1] : (i32, i32) -> ()
+  ^bb1(%arg3 : i32, %arg4 : i32):
+     "test.return"(%arg3, %arg4) : (i32, i32) -> ()
+  }) : () -> (i32, i32)
+  "test.return"(%0#0, %0#1) : (i32, i32) -> ()
+}
+
+// -----
+
+// The op in this function is rewritten to itself (and thus remains
+// illegal) by a pattern that merges the second block with the first
+// after adding an operation into it.  Check that we can undo block
+// removal succesfully.
+// CHECK-LABEL: @undo_blocks_merge
+func @undo_blocks_merge(%arg0: i32) {
+  "test.undo_blocks_merge"() ({
+    // expected-remark@-1 {{op 'test.undo_blocks_merge' is not legalizable}}
+    // CHECK: "unregistered.return"(%{{.*}})[^[[BB:.*]]]
+    "unregistered.return"(%arg0)[^bb1] : (i32) -> ()
+    // expected-remark@-1 {{op 'unregistered.return' is not legalizable}}
+  // CHECK: ^[[BB]]
+  ^bb1(%arg1 : i32):
+    // CHECK: "unregistered.return"
+    "unregistered.return"(%arg1) : (i32) -> ()
+    // expected-remark@-1 {{op 'unregistered.return' is not legalizable}}
+  }) : () -> ()
+}
+
+// -----
+
+// CHECK-LABEL: @inline_regions()
+func @inline_regions() -> ()
+{
+  //      CHECK: test.SingleBlockImplicitTerminator
+  // CHECK-NEXT:   %[[T0:.*]] = "test.type_producer"
+  // CHECK-NEXT:   "test.type_consumer"(%[[T0]])
+  // CHECK-NEXT:   "test.finish"
+  "test.SingleBlockImplicitTerminator"() ({
+  ^bb0:
+    %0 = "test.type_producer"() : () -> i32
+    "test.SingleBlockImplicitTerminator"() ({
+    ^bb1:
+      "test.type_consumer"(%0) : (i32) -> ()
+      "test.finish"() : () -> ()
+    }) : () -> ()
+    "test.finish"() : () -> ()
+  }) : () -> ()
+  "test.return"() : () -> ()
+}
diff --git a/mlir/test/lib/Dialect/Test/TestOps.td b/mlir/test/lib/Dialect/Test/TestOps.td
index af20034abd9ba..0c26f8a719c09 100644
--- a/mlir/test/lib/Dialect/Test/TestOps.td
+++ b/mlir/test/lib/Dialect/Test/TestOps.td
@@ -1163,6 +1163,16 @@ def TestTypeConsumerOp : TEST_Op<"type_consumer">,
 def TestValidOp : TEST_Op<"valid", [Terminator]>,
   Arguments<(ins Variadic<AnyType>)>;
 
+def TestMergeBlocksOp : TEST_Op<"merge_blocks"> {
+  let summary = "merge_blocks operation";
+  let description = [{
+    Test op with multiple blocks that are merged with Dialect Conversion"
+  }];
+
+  let regions = (region AnyRegion:$body);
+  let results = (outs Variadic<AnyType>:$result);
+}
+
 //===----------------------------------------------------------------------===//
 // Test parser.
 //===----------------------------------------------------------------------===//
diff --git a/mlir/test/lib/Dialect/Test/TestPatterns.cpp b/mlir/test/lib/Dialect/Test/TestPatterns.cpp
index 5bc947fc8c916..f6607a5f55246 100644
--- a/mlir/test/lib/Dialect/Test/TestPatterns.cpp
+++ b/mlir/test/lib/Dialect/Test/TestPatterns.cpp
@@ -834,6 +834,118 @@ struct TestTypeConversionDriver
 };
 } // end anonymous namespace
 
+namespace {
+/// A rewriter pattern that tests that blocks can be merged.
+struct TestMergeBlock : public OpConversionPattern<TestMergeBlocksOp> {
+  using OpConversionPattern<TestMergeBlocksOp>::OpConversionPattern;
+
+  LogicalResult
+  matchAndRewrite(TestMergeBlocksOp op, ArrayRef<Value> operands,
+                  ConversionPatternRewriter &rewriter) const final {
+    Block &firstBlock = op.body().front();
+    Operation *branchOp = firstBlock.getTerminator();
+    Block *secondBlock = &*(std::next(op.body().begin()));
+    auto succOperands = branchOp->getOperands();
+    SmallVector<Value, 2> replacements(succOperands);
+    rewriter.eraseOp(branchOp);
+    rewriter.mergeBlocks(secondBlock, &firstBlock, replacements);
+    rewriter.updateRootInPlace(op, [] {});
+    return success();
+  }
+};
+
+/// A rewrite pattern to tests the undo mechanism of blocks being merged.
+struct TestUndoBlocksMerge : public ConversionPattern {
+  TestUndoBlocksMerge(MLIRContext *ctx)
+      : ConversionPattern("test.undo_blocks_merge", /*benefit=*/1, ctx) {}
+  LogicalResult
+  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
+                  ConversionPatternRewriter &rewriter) const final {
+    Block &firstBlock = op->getRegion(0).front();
+    Operation *branchOp = firstBlock.getTerminator();
+    Block *secondBlock = &*(std::next(op->getRegion(0).begin()));
+    rewriter.setInsertionPointToStart(secondBlock);
+    rewriter.create<ILLegalOpF>(op->getLoc(), rewriter.getF32Type());
+    auto succOperands = branchOp->getOperands();
+    SmallVector<Value, 2> replacements(succOperands);
+    rewriter.eraseOp(branchOp);
+    rewriter.mergeBlocks(secondBlock, &firstBlock, replacements);
+    rewriter.updateRootInPlace(op, [] {});
+    return success();
+  }
+};
+
+/// A rewrite mechanism to inline the body of the op into its parent, when both
+/// ops can have a single block.
+struct TestMergeSingleBlockOps
+    : public OpConversionPattern<SingleBlockImplicitTerminatorOp> {
+  using OpConversionPattern<
+      SingleBlockImplicitTerminatorOp>::OpConversionPattern;
+
+  LogicalResult
+  matchAndRewrite(SingleBlockImplicitTerminatorOp op, ArrayRef<Value> operands,
+                  ConversionPatternRewriter &rewriter) const final {
+    SingleBlockImplicitTerminatorOp parentOp =
+        op.getParentOfType<SingleBlockImplicitTerminatorOp>();
+    if (!parentOp)
+      return failure();
+    Block &parentBlock = parentOp.region().front();
+    Block &innerBlock = op.region().front();
+    TerminatorOp innerTerminator =
+        cast<TerminatorOp>(innerBlock.getTerminator());
+    Block *parentPrologue =
+        rewriter.splitBlock(&parentBlock, Block::iterator(op));
+    rewriter.eraseOp(innerTerminator);
+    rewriter.mergeBlocks(&innerBlock, &parentBlock, {});
+    rewriter.eraseOp(op);
+    rewriter.mergeBlocks(parentPrologue, &parentBlock, {});
+    rewriter.updateRootInPlace(op, [] {});
+    return success();
+  }
+};
+
+struct TestMergeBlocksPatternDriver
+    : public PassWrapper<TestMergeBlocksPatternDriver,
+                         OperationPass<ModuleOp>> {
+  void runOnOperation() override {
+    mlir::OwningRewritePatternList patterns;
+    MLIRContext *context = &getContext();
+    patterns
+        .insert<TestMergeBlock, TestUndoBlocksMerge, TestMergeSingleBlockOps>(
+            context);
+    ConversionTarget target(*context);
+    target.addLegalOp<FuncOp, ModuleOp, ModuleTerminatorOp, TerminatorOp,
+                      TestBranchOp, TestTypeConsumerOp, TestTypeProducerOp,
+                      TestReturnOp>();
+    target.addIllegalOp<ILLegalOpF>();
+
+    /// Expect the op to have a single block after legalization.
+    target.addDynamicallyLegalOp<TestMergeBlocksOp>(
+        [&](TestMergeBlocksOp op) -> bool {
+          return llvm::hasSingleElement(op.body());
+        });
+
+    /// Only allow `test.br` within test.merge_blocks op.
+    target.addDynamicallyLegalOp<TestBranchOp>([&](TestBranchOp op) -> bool {
+      return op.getParentOfType<TestMergeBlocksOp>();
+    });
+
+    /// Expect that all nested test.SingleBlockImplicitTerminator ops are
+    /// inlined.
+    target.addDynamicallyLegalOp<SingleBlockImplicitTerminatorOp>(
+        [&](SingleBlockImplicitTerminatorOp op) -> bool {
+          return !op.getParentOfType<SingleBlockImplicitTerminatorOp>();
+        });
+
+    DenseSet<Operation *> unlegalizedOps;
+    (void)applyPartialConversion(getOperation(), target, patterns,
+                                 &unlegalizedOps);
+    for (auto *op : unlegalizedOps)
+      op->emitRemark() << "op '" << op->getName() << "' is not legalizable";
+  }
+};
+} // namespace
+
 //===----------------------------------------------------------------------===//
 // PassRegistration
 //===----------------------------------------------------------------------===//
@@ -866,5 +978,9 @@ void registerPatternsTestPass() {
   PassRegistration<TestTypeConversionDriver>(
       "test-legalize-type-conversion",
       "Test various type conversion functionalities in DialectConversion");
+
+  PassRegistration<TestMergeBlocksPatternDriver>{
+      "test-merge-blocks",
+      "Test Merging operation in ConversionPatternRewriter"};
 }
 } // namespace mlir

From 32f3a9a9d68eea7d40a19767b591622b4b737990 Mon Sep 17 00:00:00 2001
From: MaheshRavishankar <ravishankarm@google.com>
Date: Mon, 3 Aug 2020 10:04:48 -0700
Subject: [PATCH 0928/1035] [mlir][DialectConversion] Remove usage of
 std::distance to track position.

Remove use of iterator::difference_type to know where to insert a
moved or erased block during undo actions.

Differential Revision: https://reviews.llvm.org/D85066
---
 mlir/lib/Transforms/DialectConversion.cpp | 37 +++++++++++++----------
 1 file changed, 21 insertions(+), 16 deletions(-)

diff --git a/mlir/lib/Transforms/DialectConversion.cpp b/mlir/lib/Transforms/DialectConversion.cpp
index 713f0b73dfe04..a8d7fe71f75fb 100644
--- a/mlir/lib/Transforms/DialectConversion.cpp
+++ b/mlir/lib/Transforms/DialectConversion.cpp
@@ -611,12 +611,11 @@ enum class BlockActionKind {
   TypeConversion
 };
 
-/// Original position of the given block in its parent region.  We cannot use
-/// a region iterator because it could have been invalidated by other region
-/// operations since the position was stored.
+/// Original position of the given block in its parent region. During undo
+/// actions, the block needs to be placed after `insertAfterBlock`.
 struct BlockPosition {
   Region *region;
-  Region::iterator::difference_type position;
+  Block *insertAfterBlock;
 };
 
 /// Information needed to undo the merge actions.
@@ -634,16 +633,16 @@ struct BlockAction {
   static BlockAction getCreate(Block *block) {
     return {BlockActionKind::Create, block, {}};
   }
-  static BlockAction getErase(Block *block, BlockPosition originalPos) {
-    return {BlockActionKind::Erase, block, {originalPos}};
+  static BlockAction getErase(Block *block, BlockPosition originalPosition) {
+    return {BlockActionKind::Erase, block, {originalPosition}};
   }
   static BlockAction getMerge(Block *block, Block *sourceBlock) {
     BlockAction action{BlockActionKind::Merge, block, {}};
     action.mergeInfo = {sourceBlock, block->empty() ? nullptr : &block->back()};
     return action;
   }
-  static BlockAction getMove(Block *block, BlockPosition originalPos) {
-    return {BlockActionKind::Move, block, {originalPos}};
+  static BlockAction getMove(Block *block, BlockPosition originalPosition) {
+    return {BlockActionKind::Move, block, {originalPosition}};
   }
   static BlockAction getSplit(Block *block, Block *originalBlock) {
     BlockAction action{BlockActionKind::Split, block, {}};
@@ -988,9 +987,11 @@ void ConversionPatternRewriterImpl::undoBlockActions(
     // Put the block (owned by action) back into its original position.
     case BlockActionKind::Erase: {
       auto &blockList = action.originalPosition.region->getBlocks();
-      blockList.insert(
-          std::next(blockList.begin(), action.originalPosition.position),
-          action.block);
+      Block *insertAfterBlock = action.originalPosition.insertAfterBlock;
+      blockList.insert((insertAfterBlock
+                            ? std::next(Region::iterator(insertAfterBlock))
+                            : blockList.end()),
+                       action.block);
       break;
     }
     // Split the block at the position which was originally the end of the
@@ -1010,8 +1011,10 @@ void ConversionPatternRewriterImpl::undoBlockActions(
     // Move the block back to its original position.
     case BlockActionKind::Move: {
       Region *originalRegion = action.originalPosition.region;
+      Block *insertAfterBlock = action.originalPosition.insertAfterBlock;
       originalRegion->getBlocks().splice(
-          std::next(originalRegion->begin(), action.originalPosition.position),
+          (insertAfterBlock ? std::next(Region::iterator(insertAfterBlock))
+                            : originalRegion->end()),
           action.block->getParent()->getBlocks(), action.block);
       break;
     }
@@ -1189,8 +1192,8 @@ void ConversionPatternRewriterImpl::notifyOpReplaced(Operation *op,
 
 void ConversionPatternRewriterImpl::notifyBlockIsBeingErased(Block *block) {
   Region *region = block->getParent();
-  auto position = std::distance(region->begin(), Region::iterator(block));
-  blockActions.push_back(BlockAction::getErase(block, {region, position}));
+  Block *origPrevBlock = block->getPrevNode();
+  blockActions.push_back(BlockAction::getErase(block, {region, origPrevBlock}));
 }
 
 void ConversionPatternRewriterImpl::notifyCreatedBlock(Block *block) {
@@ -1209,10 +1212,12 @@ void ConversionPatternRewriterImpl::notifyBlocksBeingMerged(Block *block,
 
 void ConversionPatternRewriterImpl::notifyRegionIsBeingInlinedBefore(
     Region &region, Region &parent, Region::iterator before) {
+  Block *origPrevBlock = nullptr;
   for (auto &pair : llvm::enumerate(region)) {
     Block &block = pair.value();
-    Region::iterator::difference_type position = pair.index();
-    blockActions.push_back(BlockAction::getMove(&block, {&region, position}));
+    blockActions.push_back(
+        BlockAction::getMove(&block, {&region, origPrevBlock}));
+    origPrevBlock = &block;
   }
 }
 

From ac82b918c74f3fab8d4a7c1905277bda6b9bccb4 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@intel.com>
Date: Mon, 3 Aug 2020 10:09:57 -0700
Subject: [PATCH 0929/1035] [X86] Use h-register for final XOR of
 __builtin_parity on 64-bit targets.

This adds an isel pattern and special XOR8rr_NOREX instruction
to enable the use of h-registers for __builtin_parity. This avoids
a copy and a shift instruction. The NOREX instruction is in case
register allocation doesn't use the matching l-register for some
reason. If a R8-R15 register gets picked instead, we won't be
able to encode the instruction since an h-register can't be used
with a REX prefix.

Fixes PR46954
---
 llvm/lib/Target/X86/X86InstrArithmetic.td     |   9 +
 llvm/lib/Target/X86/X86InstrCompiler.td       |  10 +
 llvm/test/CodeGen/X86/parity.ll               |  16 +-
 .../CodeGen/X86/vector-reduce-xor-bool.ll     | 196 +++++-------------
 4 files changed, 72 insertions(+), 159 deletions(-)

diff --git a/llvm/lib/Target/X86/X86InstrArithmetic.td b/llvm/lib/Target/X86/X86InstrArithmetic.td
index f7f22285bd15d..e83e1e74ff526 100644
--- a/llvm/lib/Target/X86/X86InstrArithmetic.td
+++ b/llvm/lib/Target/X86/X86InstrArithmetic.td
@@ -1182,6 +1182,15 @@ defm SUB : ArithBinOp_RF<0x28, 0x2A, 0x2C, "sub", MRM5r, MRM5m,
                          X86sub_flag, sub, 0, 1, 0>;
 }
 
+// Version of XOR8rr_NOREX that use GR8_NOREX. This is used by the handling of
+// __builtin_parity where the last step xors an h-register with an l-register.
+let isCodeGenOnly = 1, hasSideEffects = 0, Constraints = "$src1 = $dst",
+    Defs = [EFLAGS], isCommutable = 1 in
+def XOR8rr_NOREX : I<0x30, MRMDestReg, (outs GR8_NOREX:$dst),
+                     (ins GR8_NOREX:$src1, GR8_NOREX:$src2),
+                     "xor{b}\t{$src2, $dst|$dst, $src2}", []>,
+                     Sched<[WriteALU]>;
+
 // Arithmetic.
 defm ADC : ArithBinOp_RFF<0x10, 0x12, 0x14, "adc", MRM2r, MRM2m, X86adc_flag,
                           1, 0>;
diff --git a/llvm/lib/Target/X86/X86InstrCompiler.td b/llvm/lib/Target/X86/X86InstrCompiler.td
index 4df93fb2ed60b..d78d9f7c80c76 100644
--- a/llvm/lib/Target/X86/X86InstrCompiler.td
+++ b/llvm/lib/Target/X86/X86InstrCompiler.td
@@ -1698,6 +1698,16 @@ def : Pat<(store (i8 (trunc_su (srl_su GR16:$src, (i8 8)))), addr:$dst),
             (EXTRACT_SUBREG GR16:$src, sub_8bit_hi))>,
       Requires<[In64BitMode]>;
 
+// Special pattern to catch the last step of __builtin_parity handling. Our
+// goal is to use an xor of an h-register with the corresponding l-register.
+// The above patterns would handle this on non 64-bit targets, but for 64-bit
+// we need to be more careful. We're using a NOREX instruction here in case
+// register allocation fails to keep the two registers together. So we need to
+// make sure we can't accidentally mix R8-R15 with an h-register.
+def : Pat<(X86xor_flag (i8 (trunc GR32:$src)),
+                       (i8 (trunc (srl_su GR32:$src, (i8 8))))),
+          (XOR8rr_NOREX (EXTRACT_SUBREG GR32:$src, sub_8bit),
+                        (EXTRACT_SUBREG GR32:$src, sub_8bit_hi))>;
 
 // (shl x, 1) ==> (add x, x)
 // Note that if x is undef (immediate or otherwise), we could theoretically
diff --git a/llvm/test/CodeGen/X86/parity.ll b/llvm/test/CodeGen/X86/parity.ll
index 869ee552c67da..6289ab482426c 100644
--- a/llvm/test/CodeGen/X86/parity.ll
+++ b/llvm/test/CodeGen/X86/parity.ll
@@ -21,10 +21,8 @@ define i32 @parity_32(i32 %x) {
 ; X64-NOPOPCNT-NEXT:    movl %edi, %ecx
 ; X64-NOPOPCNT-NEXT:    shrl $16, %ecx
 ; X64-NOPOPCNT-NEXT:    xorl %edi, %ecx
-; X64-NOPOPCNT-NEXT:    movl %ecx, %edx
-; X64-NOPOPCNT-NEXT:    shrl $8, %edx
 ; X64-NOPOPCNT-NEXT:    xorl %eax, %eax
-; X64-NOPOPCNT-NEXT:    xorb %cl, %dl
+; X64-NOPOPCNT-NEXT:    xorb %ch, %cl
 ; X64-NOPOPCNT-NEXT:    setnp %al
 ; X64-NOPOPCNT-NEXT:    retq
 ;
@@ -66,10 +64,8 @@ define i64 @parity_64(i64 %x) {
 ; X64-NOPOPCNT-NEXT:    movl %eax, %ecx
 ; X64-NOPOPCNT-NEXT:    shrl $16, %ecx
 ; X64-NOPOPCNT-NEXT:    xorl %eax, %ecx
-; X64-NOPOPCNT-NEXT:    movl %ecx, %edx
-; X64-NOPOPCNT-NEXT:    shrl $8, %edx
 ; X64-NOPOPCNT-NEXT:    xorl %eax, %eax
-; X64-NOPOPCNT-NEXT:    xorb %cl, %dl
+; X64-NOPOPCNT-NEXT:    xorb %ch, %cl
 ; X64-NOPOPCNT-NEXT:    setnp %al
 ; X64-NOPOPCNT-NEXT:    retq
 ;
@@ -113,10 +109,8 @@ define i32 @parity_64_trunc(i64 %x) {
 ; X64-NOPOPCNT-NEXT:    movl %eax, %ecx
 ; X64-NOPOPCNT-NEXT:    shrl $16, %ecx
 ; X64-NOPOPCNT-NEXT:    xorl %eax, %ecx
-; X64-NOPOPCNT-NEXT:    movl %ecx, %edx
-; X64-NOPOPCNT-NEXT:    shrl $8, %edx
 ; X64-NOPOPCNT-NEXT:    xorl %eax, %eax
-; X64-NOPOPCNT-NEXT:    xorb %cl, %dl
+; X64-NOPOPCNT-NEXT:    xorb %ch, %cl
 ; X64-NOPOPCNT-NEXT:    setnp %al
 ; X64-NOPOPCNT-NEXT:    retq
 ;
@@ -156,9 +150,7 @@ define i8 @parity_32_trunc(i32 %x) {
 ; X64-NOPOPCNT-NEXT:    movl %edi, %eax
 ; X64-NOPOPCNT-NEXT:    shrl $16, %eax
 ; X64-NOPOPCNT-NEXT:    xorl %edi, %eax
-; X64-NOPOPCNT-NEXT:    movl %eax, %ecx
-; X64-NOPOPCNT-NEXT:    shrl $8, %ecx
-; X64-NOPOPCNT-NEXT:    xorb %al, %cl
+; X64-NOPOPCNT-NEXT:    xorb %ah, %al
 ; X64-NOPOPCNT-NEXT:    setnp %al
 ; X64-NOPOPCNT-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/vector-reduce-xor-bool.ll b/llvm/test/CodeGen/X86/vector-reduce-xor-bool.ll
index 8e50cfc4e4b29..b28aa43affb74 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-xor-bool.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-xor-bool.ll
@@ -181,9 +181,7 @@ define i1 @trunc_v16i8_v16i1(<16 x i8>) {
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    psllw $7, %xmm0
 ; SSE-NEXT:    pmovmskb %xmm0, %eax
-; SSE-NEXT:    movl %eax, %ecx
-; SSE-NEXT:    shrl $8, %ecx
-; SSE-NEXT:    xorb %al, %cl
+; SSE-NEXT:    xorb %ah, %al
 ; SSE-NEXT:    setnp %al
 ; SSE-NEXT:    retq
 ;
@@ -191,9 +189,7 @@ define i1 @trunc_v16i8_v16i1(<16 x i8>) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpsllw $7, %xmm0, %xmm0
 ; AVX-NEXT:    vpmovmskb %xmm0, %eax
-; AVX-NEXT:    movl %eax, %ecx
-; AVX-NEXT:    shrl $8, %ecx
-; AVX-NEXT:    xorb %al, %cl
+; AVX-NEXT:    xorb %ah, %al
 ; AVX-NEXT:    setnp %al
 ; AVX-NEXT:    retq
 ;
@@ -201,9 +197,7 @@ define i1 @trunc_v16i8_v16i1(<16 x i8>) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpsllw $7, %xmm0, %xmm0
 ; AVX512-NEXT:    vpmovmskb %xmm0, %eax
-; AVX512-NEXT:    movl %eax, %ecx
-; AVX512-NEXT:    shrl $8, %ecx
-; AVX512-NEXT:    xorb %al, %cl
+; AVX512-NEXT:    xorb %ah, %al
 ; AVX512-NEXT:    setnp %al
 ; AVX512-NEXT:    retq
   %a = trunc <16 x i8> %0 to <16 x i1>
@@ -364,9 +358,7 @@ define i1 @trunc_v16i16_v16i1(<16 x i16>) {
 ; SSE-NEXT:    packuswb %xmm1, %xmm0
 ; SSE-NEXT:    psllw $7, %xmm0
 ; SSE-NEXT:    pmovmskb %xmm0, %eax
-; SSE-NEXT:    movl %eax, %ecx
-; SSE-NEXT:    shrl $8, %ecx
-; SSE-NEXT:    xorb %al, %cl
+; SSE-NEXT:    xorb %ah, %al
 ; SSE-NEXT:    setnp %al
 ; SSE-NEXT:    retq
 ;
@@ -377,9 +369,7 @@ define i1 @trunc_v16i16_v16i1(<16 x i16>) {
 ; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsllw $7, %xmm0, %xmm0
 ; AVX1-NEXT:    vpmovmskb %xmm0, %eax
-; AVX1-NEXT:    movl %eax, %ecx
-; AVX1-NEXT:    shrl $8, %ecx
-; AVX1-NEXT:    xorb %al, %cl
+; AVX1-NEXT:    xorb %ah, %al
 ; AVX1-NEXT:    setnp %al
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -391,9 +381,7 @@ define i1 @trunc_v16i16_v16i1(<16 x i16>) {
 ; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsllw $7, %xmm0, %xmm0
 ; AVX2-NEXT:    vpmovmskb %xmm0, %eax
-; AVX2-NEXT:    movl %eax, %ecx
-; AVX2-NEXT:    shrl $8, %ecx
-; AVX2-NEXT:    xorb %al, %cl
+; AVX2-NEXT:    xorb %ah, %al
 ; AVX2-NEXT:    setnp %al
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -445,9 +433,7 @@ define i1 @trunc_v32i8_v32i1(<32 x i8>) {
 ; SSE-NEXT:    pxor %xmm1, %xmm0
 ; SSE-NEXT:    psllw $7, %xmm0
 ; SSE-NEXT:    pmovmskb %xmm0, %eax
-; SSE-NEXT:    movl %eax, %ecx
-; SSE-NEXT:    shrl $8, %ecx
-; SSE-NEXT:    xorb %al, %cl
+; SSE-NEXT:    xorb %ah, %al
 ; SSE-NEXT:    setnp %al
 ; SSE-NEXT:    retq
 ;
@@ -457,9 +443,7 @@ define i1 @trunc_v32i8_v32i1(<32 x i8>) {
 ; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsllw $7, %xmm0, %xmm0
 ; AVX1-NEXT:    vpmovmskb %xmm0, %eax
-; AVX1-NEXT:    movl %eax, %ecx
-; AVX1-NEXT:    shrl $8, %ecx
-; AVX1-NEXT:    xorb %al, %cl
+; AVX1-NEXT:    xorb %ah, %al
 ; AVX1-NEXT:    setnp %al
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -471,9 +455,7 @@ define i1 @trunc_v32i8_v32i1(<32 x i8>) {
 ; AVX2-NEXT:    movl %eax, %ecx
 ; AVX2-NEXT:    shrl $16, %ecx
 ; AVX2-NEXT:    xorl %eax, %ecx
-; AVX2-NEXT:    movl %ecx, %eax
-; AVX2-NEXT:    shrl $8, %eax
-; AVX2-NEXT:    xorb %cl, %al
+; AVX2-NEXT:    xorb %ch, %cl
 ; AVX2-NEXT:    setnp %al
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -505,9 +487,7 @@ define i1 @trunc_v32i8_v32i1(<32 x i8>) {
 ; AVX512BW-NEXT:    movl %eax, %ecx
 ; AVX512BW-NEXT:    shrl $16, %ecx
 ; AVX512BW-NEXT:    xorl %eax, %ecx
-; AVX512BW-NEXT:    movl %ecx, %eax
-; AVX512BW-NEXT:    shrl $8, %eax
-; AVX512BW-NEXT:    xorb %cl, %al
+; AVX512BW-NEXT:    xorb %ch, %cl
 ; AVX512BW-NEXT:    setnp %al
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -519,9 +499,7 @@ define i1 @trunc_v32i8_v32i1(<32 x i8>) {
 ; AVX512VL-NEXT:    movl %eax, %ecx
 ; AVX512VL-NEXT:    shrl $16, %ecx
 ; AVX512VL-NEXT:    xorl %eax, %ecx
-; AVX512VL-NEXT:    movl %ecx, %eax
-; AVX512VL-NEXT:    shrl $8, %eax
-; AVX512VL-NEXT:    xorb %cl, %al
+; AVX512VL-NEXT:    xorb %ch, %cl
 ; AVX512VL-NEXT:    setnp %al
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
@@ -648,9 +626,7 @@ define i1 @trunc_v16i32_v16i1(<16 x i32>) {
 ; SSE2-NEXT:    packuswb %xmm2, %xmm0
 ; SSE2-NEXT:    psllw $7, %xmm0
 ; SSE2-NEXT:    pmovmskb %xmm0, %eax
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    shrl $8, %ecx
-; SSE2-NEXT:    xorb %al, %cl
+; SSE2-NEXT:    xorb %ah, %al
 ; SSE2-NEXT:    setnp %al
 ; SSE2-NEXT:    retq
 ;
@@ -666,9 +642,7 @@ define i1 @trunc_v16i32_v16i1(<16 x i32>) {
 ; SSE41-NEXT:    packuswb %xmm2, %xmm0
 ; SSE41-NEXT:    psllw $7, %xmm0
 ; SSE41-NEXT:    pmovmskb %xmm0, %eax
-; SSE41-NEXT:    movl %eax, %ecx
-; SSE41-NEXT:    shrl $8, %ecx
-; SSE41-NEXT:    xorb %al, %cl
+; SSE41-NEXT:    xorb %ah, %al
 ; SSE41-NEXT:    setnp %al
 ; SSE41-NEXT:    retq
 ;
@@ -684,9 +658,7 @@ define i1 @trunc_v16i32_v16i1(<16 x i32>) {
 ; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsllw $7, %xmm0, %xmm0
 ; AVX1-NEXT:    vpmovmskb %xmm0, %eax
-; AVX1-NEXT:    movl %eax, %ecx
-; AVX1-NEXT:    shrl $8, %ecx
-; AVX1-NEXT:    xorb %al, %cl
+; AVX1-NEXT:    xorb %ah, %al
 ; AVX1-NEXT:    setnp %al
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -704,9 +676,7 @@ define i1 @trunc_v16i32_v16i1(<16 x i32>) {
 ; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsllw $7, %xmm0, %xmm0
 ; AVX2-NEXT:    vpmovmskb %xmm0, %eax
-; AVX2-NEXT:    movl %eax, %ecx
-; AVX2-NEXT:    shrl $8, %ecx
-; AVX2-NEXT:    xorb %al, %cl
+; AVX2-NEXT:    xorb %ah, %al
 ; AVX2-NEXT:    setnp %al
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -764,9 +734,7 @@ define i1 @trunc_v32i16_v32i1(<32 x i16>) {
 ; SSE-NEXT:    pxor %xmm2, %xmm0
 ; SSE-NEXT:    psllw $7, %xmm0
 ; SSE-NEXT:    pmovmskb %xmm0, %eax
-; SSE-NEXT:    movl %eax, %ecx
-; SSE-NEXT:    shrl $8, %ecx
-; SSE-NEXT:    xorb %al, %cl
+; SSE-NEXT:    xorb %ah, %al
 ; SSE-NEXT:    setnp %al
 ; SSE-NEXT:    retq
 ;
@@ -778,9 +746,7 @@ define i1 @trunc_v32i16_v32i1(<32 x i16>) {
 ; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsllw $7, %xmm0, %xmm0
 ; AVX1-NEXT:    vpmovmskb %xmm0, %eax
-; AVX1-NEXT:    movl %eax, %ecx
-; AVX1-NEXT:    shrl $8, %ecx
-; AVX1-NEXT:    xorb %al, %cl
+; AVX1-NEXT:    xorb %ah, %al
 ; AVX1-NEXT:    setnp %al
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -798,9 +764,7 @@ define i1 @trunc_v32i16_v32i1(<32 x i16>) {
 ; AVX2-NEXT:    movl %eax, %ecx
 ; AVX2-NEXT:    shrl $16, %ecx
 ; AVX2-NEXT:    xorl %eax, %ecx
-; AVX2-NEXT:    movl %ecx, %eax
-; AVX2-NEXT:    shrl $8, %eax
-; AVX2-NEXT:    xorb %cl, %al
+; AVX2-NEXT:    xorb %ch, %cl
 ; AVX2-NEXT:    setnp %al
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -833,9 +797,7 @@ define i1 @trunc_v32i16_v32i1(<32 x i16>) {
 ; AVX512BW-NEXT:    movl %eax, %ecx
 ; AVX512BW-NEXT:    shrl $16, %ecx
 ; AVX512BW-NEXT:    xorl %eax, %ecx
-; AVX512BW-NEXT:    movl %ecx, %eax
-; AVX512BW-NEXT:    shrl $8, %eax
-; AVX512BW-NEXT:    xorb %cl, %al
+; AVX512BW-NEXT:    xorb %ch, %cl
 ; AVX512BW-NEXT:    setnp %al
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -848,9 +810,7 @@ define i1 @trunc_v32i16_v32i1(<32 x i16>) {
 ; AVX512VL-NEXT:    movl %eax, %ecx
 ; AVX512VL-NEXT:    shrl $16, %ecx
 ; AVX512VL-NEXT:    xorl %eax, %ecx
-; AVX512VL-NEXT:    movl %ecx, %eax
-; AVX512VL-NEXT:    shrl $8, %eax
-; AVX512VL-NEXT:    xorb %cl, %al
+; AVX512VL-NEXT:    xorb %ch, %cl
 ; AVX512VL-NEXT:    setnp %al
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
@@ -867,9 +827,7 @@ define i1 @trunc_v64i8_v64i1(<64 x i8>) {
 ; SSE-NEXT:    pxor %xmm0, %xmm1
 ; SSE-NEXT:    psllw $7, %xmm1
 ; SSE-NEXT:    pmovmskb %xmm1, %eax
-; SSE-NEXT:    movl %eax, %ecx
-; SSE-NEXT:    shrl $8, %ecx
-; SSE-NEXT:    xorb %al, %cl
+; SSE-NEXT:    xorb %ah, %al
 ; SSE-NEXT:    setnp %al
 ; SSE-NEXT:    retq
 ;
@@ -880,9 +838,7 @@ define i1 @trunc_v64i8_v64i1(<64 x i8>) {
 ; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsllw $7, %xmm0, %xmm0
 ; AVX1-NEXT:    vpmovmskb %xmm0, %eax
-; AVX1-NEXT:    movl %eax, %ecx
-; AVX1-NEXT:    shrl $8, %ecx
-; AVX1-NEXT:    xorb %al, %cl
+; AVX1-NEXT:    xorb %ah, %al
 ; AVX1-NEXT:    setnp %al
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -895,9 +851,7 @@ define i1 @trunc_v64i8_v64i1(<64 x i8>) {
 ; AVX2-NEXT:    movl %eax, %ecx
 ; AVX2-NEXT:    shrl $16, %ecx
 ; AVX2-NEXT:    xorl %eax, %ecx
-; AVX2-NEXT:    movl %ecx, %eax
-; AVX2-NEXT:    shrl $8, %eax
-; AVX2-NEXT:    xorb %cl, %al
+; AVX2-NEXT:    xorb %ch, %cl
 ; AVX2-NEXT:    setnp %al
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -937,9 +891,7 @@ define i1 @trunc_v64i8_v64i1(<64 x i8>) {
 ; AVX512BW-NEXT:    movl %ecx, %eax
 ; AVX512BW-NEXT:    shrl $16, %eax
 ; AVX512BW-NEXT:    xorl %ecx, %eax
-; AVX512BW-NEXT:    movl %eax, %ecx
-; AVX512BW-NEXT:    shrl $8, %ecx
-; AVX512BW-NEXT:    xorb %al, %cl
+; AVX512BW-NEXT:    xorb %ah, %al
 ; AVX512BW-NEXT:    setnp %al
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -955,9 +907,7 @@ define i1 @trunc_v64i8_v64i1(<64 x i8>) {
 ; AVX512VL-NEXT:    movl %ecx, %eax
 ; AVX512VL-NEXT:    shrl $16, %eax
 ; AVX512VL-NEXT:    xorl %ecx, %eax
-; AVX512VL-NEXT:    movl %eax, %ecx
-; AVX512VL-NEXT:    shrl $8, %ecx
-; AVX512VL-NEXT:    xorb %al, %cl
+; AVX512VL-NEXT:    xorb %ah, %al
 ; AVX512VL-NEXT:    setnp %al
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
@@ -1157,9 +1107,7 @@ define i1 @icmp_v16i8_v16i1(<16 x i8>) {
 ; SSE-NEXT:    pxor %xmm1, %xmm1
 ; SSE-NEXT:    pcmpeqb %xmm0, %xmm1
 ; SSE-NEXT:    pmovmskb %xmm1, %eax
-; SSE-NEXT:    movl %eax, %ecx
-; SSE-NEXT:    shrl $8, %ecx
-; SSE-NEXT:    xorb %al, %cl
+; SSE-NEXT:    xorb %ah, %al
 ; SSE-NEXT:    setnp %al
 ; SSE-NEXT:    retq
 ;
@@ -1168,9 +1116,7 @@ define i1 @icmp_v16i8_v16i1(<16 x i8>) {
 ; AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX-NEXT:    vpcmpeqb %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpmovmskb %xmm0, %eax
-; AVX-NEXT:    movl %eax, %ecx
-; AVX-NEXT:    shrl $8, %ecx
-; AVX-NEXT:    xorb %al, %cl
+; AVX-NEXT:    xorb %ah, %al
 ; AVX-NEXT:    setnp %al
 ; AVX-NEXT:    retq
 ;
@@ -1179,9 +1125,7 @@ define i1 @icmp_v16i8_v16i1(<16 x i8>) {
 ; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX512F-NEXT:    vpcmpeqb %xmm1, %xmm0, %xmm0
 ; AVX512F-NEXT:    vpmovmskb %xmm0, %eax
-; AVX512F-NEXT:    movl %eax, %ecx
-; AVX512F-NEXT:    shrl $8, %ecx
-; AVX512F-NEXT:    xorb %al, %cl
+; AVX512F-NEXT:    xorb %ah, %al
 ; AVX512F-NEXT:    setnp %al
 ; AVX512F-NEXT:    retq
 ;
@@ -1371,9 +1315,7 @@ define i1 @icmp_v16i16_v16i1(<16 x i16>) {
 ; SSE-NEXT:    pcmpeqw %xmm2, %xmm0
 ; SSE-NEXT:    packsswb %xmm1, %xmm0
 ; SSE-NEXT:    pmovmskb %xmm0, %eax
-; SSE-NEXT:    movl %eax, %ecx
-; SSE-NEXT:    shrl $8, %ecx
-; SSE-NEXT:    xorb %al, %cl
+; SSE-NEXT:    xorb %ah, %al
 ; SSE-NEXT:    setnp %al
 ; SSE-NEXT:    retq
 ;
@@ -1385,9 +1327,7 @@ define i1 @icmp_v16i16_v16i1(<16 x i16>) {
 ; AVX1-NEXT:    vpcmpeqw %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpmovmskb %xmm0, %eax
-; AVX1-NEXT:    movl %eax, %ecx
-; AVX1-NEXT:    shrl $8, %ecx
-; AVX1-NEXT:    xorb %al, %cl
+; AVX1-NEXT:    xorb %ah, %al
 ; AVX1-NEXT:    setnp %al
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -1399,9 +1339,7 @@ define i1 @icmp_v16i16_v16i1(<16 x i16>) {
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpmovmskb %xmm0, %eax
-; AVX2-NEXT:    movl %eax, %ecx
-; AVX2-NEXT:    shrl $8, %ecx
-; AVX2-NEXT:    xorb %al, %cl
+; AVX2-NEXT:    xorb %ah, %al
 ; AVX2-NEXT:    setnp %al
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -1455,9 +1393,7 @@ define i1 @icmp_v32i8_v32i1(<32 x i8>) {
 ; SSE-NEXT:    pcmpeqb %xmm2, %xmm0
 ; SSE-NEXT:    pxor %xmm1, %xmm0
 ; SSE-NEXT:    pmovmskb %xmm0, %eax
-; SSE-NEXT:    movl %eax, %ecx
-; SSE-NEXT:    shrl $8, %ecx
-; SSE-NEXT:    xorb %al, %cl
+; SSE-NEXT:    xorb %ah, %al
 ; SSE-NEXT:    setnp %al
 ; SSE-NEXT:    retq
 ;
@@ -1469,9 +1405,7 @@ define i1 @icmp_v32i8_v32i1(<32 x i8>) {
 ; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpmovmskb %xmm0, %eax
-; AVX1-NEXT:    movl %eax, %ecx
-; AVX1-NEXT:    shrl $8, %ecx
-; AVX1-NEXT:    xorb %al, %cl
+; AVX1-NEXT:    xorb %ah, %al
 ; AVX1-NEXT:    setnp %al
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -1484,9 +1418,7 @@ define i1 @icmp_v32i8_v32i1(<32 x i8>) {
 ; AVX2-NEXT:    movl %eax, %ecx
 ; AVX2-NEXT:    shrl $16, %ecx
 ; AVX2-NEXT:    xorl %eax, %ecx
-; AVX2-NEXT:    movl %ecx, %eax
-; AVX2-NEXT:    shrl $8, %eax
-; AVX2-NEXT:    xorb %cl, %al
+; AVX2-NEXT:    xorb %ch, %cl
 ; AVX2-NEXT:    setnp %al
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -1520,9 +1452,7 @@ define i1 @icmp_v32i8_v32i1(<32 x i8>) {
 ; AVX512BW-NEXT:    movl %eax, %ecx
 ; AVX512BW-NEXT:    shrl $16, %ecx
 ; AVX512BW-NEXT:    xorl %eax, %ecx
-; AVX512BW-NEXT:    movl %ecx, %eax
-; AVX512BW-NEXT:    shrl $8, %eax
-; AVX512BW-NEXT:    xorb %cl, %al
+; AVX512BW-NEXT:    xorb %ch, %cl
 ; AVX512BW-NEXT:    setnp %al
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -1534,9 +1464,7 @@ define i1 @icmp_v32i8_v32i1(<32 x i8>) {
 ; AVX512VL-NEXT:    movl %eax, %ecx
 ; AVX512VL-NEXT:    shrl $16, %ecx
 ; AVX512VL-NEXT:    xorl %eax, %ecx
-; AVX512VL-NEXT:    movl %ecx, %eax
-; AVX512VL-NEXT:    shrl $8, %eax
-; AVX512VL-NEXT:    xorb %cl, %al
+; AVX512VL-NEXT:    xorb %ch, %cl
 ; AVX512VL-NEXT:    setnp %al
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
@@ -1660,9 +1588,7 @@ define i1 @icmp_v16i32_v16i1(<16 x i32>) {
 ; SSE-NEXT:    packssdw %xmm1, %xmm0
 ; SSE-NEXT:    packsswb %xmm2, %xmm0
 ; SSE-NEXT:    pmovmskb %xmm0, %eax
-; SSE-NEXT:    movl %eax, %ecx
-; SSE-NEXT:    shrl $8, %ecx
-; SSE-NEXT:    xorb %al, %cl
+; SSE-NEXT:    xorb %ah, %al
 ; SSE-NEXT:    setnp %al
 ; SSE-NEXT:    retq
 ;
@@ -1679,9 +1605,7 @@ define i1 @icmp_v16i32_v16i1(<16 x i32>) {
 ; AVX1-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpmovmskb %xmm0, %eax
-; AVX1-NEXT:    movl %eax, %ecx
-; AVX1-NEXT:    shrl $8, %ecx
-; AVX1-NEXT:    xorb %al, %cl
+; AVX1-NEXT:    xorb %ah, %al
 ; AVX1-NEXT:    setnp %al
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -1696,9 +1620,7 @@ define i1 @icmp_v16i32_v16i1(<16 x i32>) {
 ; AVX2-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX2-NEXT:    vpmovmskb %xmm0, %eax
-; AVX2-NEXT:    movl %eax, %ecx
-; AVX2-NEXT:    shrl $8, %ecx
-; AVX2-NEXT:    xorb %al, %cl
+; AVX2-NEXT:    xorb %ah, %al
 ; AVX2-NEXT:    setnp %al
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -1752,9 +1674,7 @@ define i1 @icmp_v32i16_v32i1(<32 x i16>) {
 ; SSE-NEXT:    packsswb %xmm3, %xmm2
 ; SSE-NEXT:    pxor %xmm0, %xmm2
 ; SSE-NEXT:    pmovmskb %xmm2, %eax
-; SSE-NEXT:    movl %eax, %ecx
-; SSE-NEXT:    shrl $8, %ecx
-; SSE-NEXT:    xorb %al, %cl
+; SSE-NEXT:    xorb %ah, %al
 ; SSE-NEXT:    setnp %al
 ; SSE-NEXT:    retq
 ;
@@ -1771,9 +1691,7 @@ define i1 @icmp_v32i16_v32i1(<32 x i16>) {
 ; AVX1-NEXT:    vpacksswb %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpmovmskb %xmm0, %eax
-; AVX1-NEXT:    movl %eax, %ecx
-; AVX1-NEXT:    shrl $8, %ecx
-; AVX1-NEXT:    xorb %al, %cl
+; AVX1-NEXT:    xorb %ah, %al
 ; AVX1-NEXT:    setnp %al
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -1789,9 +1707,7 @@ define i1 @icmp_v32i16_v32i1(<32 x i16>) {
 ; AVX2-NEXT:    movl %eax, %ecx
 ; AVX2-NEXT:    shrl $16, %ecx
 ; AVX2-NEXT:    xorl %eax, %ecx
-; AVX2-NEXT:    movl %ecx, %eax
-; AVX2-NEXT:    shrl $8, %eax
-; AVX2-NEXT:    xorb %cl, %al
+; AVX2-NEXT:    xorb %ch, %cl
 ; AVX2-NEXT:    setnp %al
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -1825,9 +1741,7 @@ define i1 @icmp_v32i16_v32i1(<32 x i16>) {
 ; AVX512BW-NEXT:    movl %eax, %ecx
 ; AVX512BW-NEXT:    shrl $16, %ecx
 ; AVX512BW-NEXT:    xorl %eax, %ecx
-; AVX512BW-NEXT:    movl %ecx, %eax
-; AVX512BW-NEXT:    shrl $8, %eax
-; AVX512BW-NEXT:    xorb %cl, %al
+; AVX512BW-NEXT:    xorb %ch, %cl
 ; AVX512BW-NEXT:    setnp %al
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -1839,9 +1753,7 @@ define i1 @icmp_v32i16_v32i1(<32 x i16>) {
 ; AVX512VL-NEXT:    movl %eax, %ecx
 ; AVX512VL-NEXT:    shrl $16, %ecx
 ; AVX512VL-NEXT:    xorl %eax, %ecx
-; AVX512VL-NEXT:    movl %ecx, %eax
-; AVX512VL-NEXT:    shrl $8, %eax
-; AVX512VL-NEXT:    xorb %cl, %al
+; AVX512VL-NEXT:    xorb %ch, %cl
 ; AVX512VL-NEXT:    setnp %al
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
@@ -1862,9 +1774,7 @@ define i1 @icmp_v64i8_v64i1(<64 x i8>) {
 ; SSE-NEXT:    pxor %xmm2, %xmm1
 ; SSE-NEXT:    pxor %xmm0, %xmm1
 ; SSE-NEXT:    pmovmskb %xmm1, %eax
-; SSE-NEXT:    movl %eax, %ecx
-; SSE-NEXT:    shrl $8, %ecx
-; SSE-NEXT:    xorb %al, %cl
+; SSE-NEXT:    xorb %ah, %al
 ; SSE-NEXT:    setnp %al
 ; SSE-NEXT:    retq
 ;
@@ -1881,9 +1791,7 @@ define i1 @icmp_v64i8_v64i1(<64 x i8>) {
 ; AVX1-NEXT:    vpxor %xmm0, %xmm3, %xmm0
 ; AVX1-NEXT:    vpxor %xmm0, %xmm4, %xmm0
 ; AVX1-NEXT:    vpmovmskb %xmm0, %eax
-; AVX1-NEXT:    movl %eax, %ecx
-; AVX1-NEXT:    shrl $8, %ecx
-; AVX1-NEXT:    xorb %al, %cl
+; AVX1-NEXT:    xorb %ah, %al
 ; AVX1-NEXT:    setnp %al
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -1898,9 +1806,7 @@ define i1 @icmp_v64i8_v64i1(<64 x i8>) {
 ; AVX2-NEXT:    movl %eax, %ecx
 ; AVX2-NEXT:    shrl $16, %ecx
 ; AVX2-NEXT:    xorl %eax, %ecx
-; AVX2-NEXT:    movl %ecx, %eax
-; AVX2-NEXT:    shrl $8, %eax
-; AVX2-NEXT:    xorb %cl, %al
+; AVX2-NEXT:    xorb %ch, %cl
 ; AVX2-NEXT:    setnp %al
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -1942,9 +1848,7 @@ define i1 @icmp_v64i8_v64i1(<64 x i8>) {
 ; AVX512BW-NEXT:    movl %ecx, %eax
 ; AVX512BW-NEXT:    shrl $16, %eax
 ; AVX512BW-NEXT:    xorl %ecx, %eax
-; AVX512BW-NEXT:    movl %eax, %ecx
-; AVX512BW-NEXT:    shrl $8, %ecx
-; AVX512BW-NEXT:    xorb %al, %cl
+; AVX512BW-NEXT:    xorb %ah, %al
 ; AVX512BW-NEXT:    setnp %al
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -1959,9 +1863,7 @@ define i1 @icmp_v64i8_v64i1(<64 x i8>) {
 ; AVX512VL-NEXT:    movl %ecx, %eax
 ; AVX512VL-NEXT:    shrl $16, %eax
 ; AVX512VL-NEXT:    xorl %ecx, %eax
-; AVX512VL-NEXT:    movl %eax, %ecx
-; AVX512VL-NEXT:    shrl $8, %ecx
-; AVX512VL-NEXT:    xorb %al, %cl
+; AVX512VL-NEXT:    xorb %ah, %al
 ; AVX512VL-NEXT:    setnp %al
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq

From 317e00dc54c74a2e0fd0c62bdc6a6d68b0d2ca7e Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Mon, 3 Aug 2020 10:09:51 -0700
Subject: [PATCH 0930/1035] [PGO] Change a `NumVSites == 0` workaround to
 assert

The root cause was fixed by 3d6f53018f845e893ad34f64ff2851a2e5c3ba1d.
The workaround added in 99ad956fdaee5398fdcf46fa49cb433cf52dc461 can be changed
to an assert now. (In case the fix regresses, there will be a heap-use-after-free.)
---
 compiler-rt/lib/profile/InstrProfilingValue.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/compiler-rt/lib/profile/InstrProfilingValue.c b/compiler-rt/lib/profile/InstrProfilingValue.c
index 29b9e628a9c95..4b4081bd21b70 100644
--- a/compiler-rt/lib/profile/InstrProfilingValue.c
+++ b/compiler-rt/lib/profile/InstrProfilingValue.c
@@ -6,6 +6,7 @@
 |*
 \*===----------------------------------------------------------------------===*/
 
+#include <assert.h>
 #include <limits.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -93,8 +94,8 @@ static int allocateValueProfileCounters(__llvm_profile_data *Data) {
   for (VKI = IPVK_First; VKI <= IPVK_Last; ++VKI)
     NumVSites += Data->NumValueSites[VKI];
 
-  if (NumVSites == 0)
-    return 0;
+  // If NumVSites = 0, calloc is allowed to return a non-null pointer.
+  assert(NumVSites > 0 && "NumVSites can't be zero");
   ValueProfNode **Mem =
       (ValueProfNode **)calloc(NumVSites, sizeof(ValueProfNode *));
   if (!Mem)

From d535a91d13b88b547ba24ec50337aa0715d74d4d Mon Sep 17 00:00:00 2001
From: "Kevin P. Neal" <kevin.neal@sas.com>
Date: Wed, 29 Jul 2020 10:33:01 -0400
Subject: [PATCH 0931/1035] [FPEnv] IRBuilder fails to add strictfp attribute

The strictfp attribute is required on all function calls in a function
that is itself marked with the strictfp attribute. The IRBuilder knows
this and has a method for adding the attribute to function call instructions.

If a function being called has the strictfp attribute itself then the
IRBuilder will refuse to add the attribute to the calling instruction
despite being asked to add it. Eliminate this error.

Differential Revision: https://reviews.llvm.org/D84878
---
 llvm/include/llvm/IR/IRBuilder.h    |  3 +--
 llvm/unittests/IR/IRBuilderTest.cpp | 27 +++++++++++++++++++++++++++
 2 files changed, 28 insertions(+), 2 deletions(-)

diff --git a/llvm/include/llvm/IR/IRBuilder.h b/llvm/include/llvm/IR/IRBuilder.h
index d467789132aca..aa0286ab11f94 100644
--- a/llvm/include/llvm/IR/IRBuilder.h
+++ b/llvm/include/llvm/IR/IRBuilder.h
@@ -294,8 +294,7 @@ class IRBuilderBase {
   }
 
   void setConstrainedFPCallAttr(CallInst *I) {
-    if (!I->hasFnAttr(Attribute::StrictFP))
-      I->addAttribute(AttributeList::FunctionIndex, Attribute::StrictFP);
+    I->addAttribute(AttributeList::FunctionIndex, Attribute::StrictFP);
   }
 
   void setDefaultOperandBundles(ArrayRef<OperandBundleDef> OpBundles) {
diff --git a/llvm/unittests/IR/IRBuilderTest.cpp b/llvm/unittests/IR/IRBuilderTest.cpp
index fa0e33a07bb40..204e476b8632f 100644
--- a/llvm/unittests/IR/IRBuilderTest.cpp
+++ b/llvm/unittests/IR/IRBuilderTest.cpp
@@ -332,6 +332,33 @@ TEST_F(IRBuilderTest, ConstrainedFPIntrinsics) {
   EXPECT_EQ(fp::ebStrict, CII->getExceptionBehavior());
 }
 
+TEST_F(IRBuilderTest, ConstrainedFPFunctionCall) {
+  IRBuilder<> Builder(BB);
+
+  // Create an empty constrained FP function.
+  FunctionType *FTy = FunctionType::get(Type::getVoidTy(Ctx),
+                                        /*isVarArg=*/false);
+  Function *Callee =
+      Function::Create(FTy, Function::ExternalLinkage, "", M.get());
+  BasicBlock *CalleeBB = BasicBlock::Create(Ctx, "", Callee);
+  IRBuilder<> CalleeBuilder(CalleeBB);
+  CalleeBuilder.setIsFPConstrained(true);
+  CalleeBuilder.setConstrainedFPFunctionAttr();
+  CalleeBuilder.CreateRetVoid();
+
+  // Now call the empty constrained FP function.
+  Builder.setIsFPConstrained(true);
+  Builder.setConstrainedFPFunctionAttr();
+  CallInst *FCall = Builder.CreateCall(Callee, None);
+
+  // Check the attributes to verify the strictfp attribute is on the call.
+  EXPECT_TRUE(FCall->getAttributes().getFnAttributes().hasAttribute(
+      Attribute::StrictFP));
+
+  Builder.CreateRetVoid();
+  EXPECT_FALSE(verifyModule(*M));
+}
+
 TEST_F(IRBuilderTest, Lifetime) {
   IRBuilder<> Builder(BB);
   AllocaInst *Var1 = Builder.CreateAlloca(Builder.getInt8Ty());

From 7c19c89dd5c532fef533e008fb5911d20992d2ac Mon Sep 17 00:00:00 2001
From: Arthur Eubanks <aeubanks@google.com>
Date: Fri, 31 Jul 2020 17:30:30 -0700
Subject: [PATCH 0932/1035] [NewPM][LoopVersioning] Port LoopVersioning to NPM

Reviewed By: ychen, fhahn

Differential Revision: https://reviews.llvm.org/D85063
---
 llvm/include/llvm/InitializePasses.h          |   2 +-
 .../llvm/Transforms/Utils/LoopVersioning.h    |   9 ++
 llvm/lib/Passes/PassBuilder.cpp               |   1 +
 llvm/lib/Passes/PassRegistry.def              |   1 +
 llvm/lib/Transforms/Scalar/Scalar.cpp         |   2 +-
 llvm/lib/Transforms/Utils/LoopVersioning.cpp  | 107 ++++++++++++------
 llvm/test/Transforms/LoopVersioning/basic.ll  |   1 +
 7 files changed, 86 insertions(+), 37 deletions(-)

diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h
index d23ecfb98ce92..cc8397a36651c 100644
--- a/llvm/include/llvm/InitializePasses.h
+++ b/llvm/include/llvm/InitializePasses.h
@@ -254,7 +254,7 @@ void initializeLoopUnrollPass(PassRegistry&);
 void initializeLoopUnswitchPass(PassRegistry&);
 void initializeLoopVectorizePass(PassRegistry&);
 void initializeLoopVersioningLICMPass(PassRegistry&);
-void initializeLoopVersioningPassPass(PassRegistry&);
+void initializeLoopVersioningLegacyPassPass(PassRegistry &);
 void initializeLowerAtomicLegacyPassPass(PassRegistry&);
 void initializeLowerConstantIntrinsicsPass(PassRegistry&);
 void initializeLowerEmuTLSPass(PassRegistry&);
diff --git a/llvm/include/llvm/Transforms/Utils/LoopVersioning.h b/llvm/include/llvm/Transforms/Utils/LoopVersioning.h
index 1efdcc65b39a8..ac6cee637a46d 100644
--- a/llvm/include/llvm/Transforms/Utils/LoopVersioning.h
+++ b/llvm/include/llvm/Transforms/Utils/LoopVersioning.h
@@ -16,6 +16,7 @@
 #define LLVM_TRANSFORMS_UTILS_LOOPVERSIONING_H
 
 #include "llvm/Analysis/ScalarEvolution.h"
+#include "llvm/IR/PassManager.h"
 #include "llvm/Transforms/Utils/LoopUtils.h"
 #include "llvm/Transforms/Utils/ValueMapper.h"
 
@@ -148,6 +149,14 @@ class LoopVersioning {
   DominatorTree *DT;
   ScalarEvolution *SE;
 };
+
+/// Expose LoopVersioning as a pass.  Currently this is only used for
+/// unit-testing.  It adds all memchecks necessary to remove all may-aliasing
+/// array accesses from the loop.
+class LoopVersioningPass : public PassInfoMixin<LoopVersioningPass> {
+public:
+  PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
+};
 }
 
 #endif
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index eb9041311e1bf..b9d1e76fcf198 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -186,6 +186,7 @@
 #include "llvm/Transforms/Utils/LCSSA.h"
 #include "llvm/Transforms/Utils/LibCallsShrinkWrap.h"
 #include "llvm/Transforms/Utils/LoopSimplify.h"
+#include "llvm/Transforms/Utils/LoopVersioning.h"
 #include "llvm/Transforms/Utils/LowerInvoke.h"
 #include "llvm/Transforms/Utils/Mem2Reg.h"
 #include "llvm/Transforms/Utils/NameAnonGlobals.h"
diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def
index e77182031f108..7490ebfb29d56 100644
--- a/llvm/lib/Passes/PassRegistry.def
+++ b/llvm/lib/Passes/PassRegistry.def
@@ -225,6 +225,7 @@ FUNCTION_PASS("loop-data-prefetch", LoopDataPrefetchPass())
 FUNCTION_PASS("loop-load-elim", LoopLoadEliminationPass())
 FUNCTION_PASS("loop-fusion", LoopFusePass())
 FUNCTION_PASS("loop-distribute", LoopDistributePass())
+FUNCTION_PASS("loop-versioning", LoopVersioningPass())
 FUNCTION_PASS("pgo-memop-opt", PGOMemOPSizeOpt())
 FUNCTION_PASS("print", PrintFunctionPass(dbgs()))
 FUNCTION_PASS("print<assumptions>", AssumptionPrinterPass(dbgs()))
diff --git a/llvm/lib/Transforms/Scalar/Scalar.cpp b/llvm/lib/Transforms/Scalar/Scalar.cpp
index a0598447030c0..8bc35d5228df8 100644
--- a/llvm/lib/Transforms/Scalar/Scalar.cpp
+++ b/llvm/lib/Transforms/Scalar/Scalar.cpp
@@ -110,7 +110,7 @@ void llvm::initializeScalarOpts(PassRegistry &Registry) {
   initializeLoopDistributeLegacyPass(Registry);
   initializeLoopLoadEliminationPass(Registry);
   initializeLoopSimplifyCFGLegacyPassPass(Registry);
-  initializeLoopVersioningPassPass(Registry);
+  initializeLoopVersioningLegacyPassPass(Registry);
   initializeEntryExitInstrumenterPass(Registry);
   initializePostInlineEntryExitInstrumenterPass(Registry);
 }
diff --git a/llvm/lib/Transforms/Utils/LoopVersioning.cpp b/llvm/lib/Transforms/Utils/LoopVersioning.cpp
index 16bd08c704eeb..c8fa337ed04f7 100644
--- a/llvm/lib/Transforms/Utils/LoopVersioning.cpp
+++ b/llvm/lib/Transforms/Utils/LoopVersioning.cpp
@@ -16,10 +16,13 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/Analysis/LoopAccessAnalysis.h"
 #include "llvm/Analysis/LoopInfo.h"
+#include "llvm/Analysis/ScalarEvolution.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/MDBuilder.h"
+#include "llvm/IR/PassManager.h"
 #include "llvm/InitializePasses.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Transforms/Scalar/LoopPassManager.h"
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
 #include "llvm/Transforms/Utils/Cloning.h"
 #include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"
@@ -253,47 +256,55 @@ void LoopVersioning::annotateInstWithNoAlias(Instruction *VersionedInst,
 }
 
 namespace {
+bool runImpl(LoopInfo *LI, function_ref<const LoopAccessInfo &(Loop &)> GetLAA,
+             DominatorTree *DT, ScalarEvolution *SE) {
+  // Build up a worklist of inner-loops to version. This is necessary as the
+  // act of versioning a loop creates new loops and can invalidate iterators
+  // across the loops.
+  SmallVector<Loop *, 8> Worklist;
+
+  for (Loop *TopLevelLoop : *LI)
+    for (Loop *L : depth_first(TopLevelLoop))
+      // We only handle inner-most loops.
+      if (L->empty())
+        Worklist.push_back(L);
+
+  // Now walk the identified inner loops.
+  bool Changed = false;
+  for (Loop *L : Worklist) {
+    const LoopAccessInfo &LAI = GetLAA(*L);
+    if (L->isLoopSimplifyForm() && !LAI.hasConvergentOp() &&
+        (LAI.getNumRuntimePointerChecks() ||
+         !LAI.getPSE().getUnionPredicate().isAlwaysTrue())) {
+      LoopVersioning LVer(LAI, L, LI, DT, SE);
+      LVer.versionLoop();
+      LVer.annotateLoopWithNoAlias();
+      Changed = true;
+    }
+  }
+
+  return Changed;
+}
+
 /// Also expose this is a pass.  Currently this is only used for
 /// unit-testing.  It adds all memchecks necessary to remove all may-aliasing
 /// array accesses from the loop.
-class LoopVersioningPass : public FunctionPass {
+class LoopVersioningLegacyPass : public FunctionPass {
 public:
-  LoopVersioningPass() : FunctionPass(ID) {
-    initializeLoopVersioningPassPass(*PassRegistry::getPassRegistry());
+  LoopVersioningLegacyPass() : FunctionPass(ID) {
+    initializeLoopVersioningLegacyPassPass(*PassRegistry::getPassRegistry());
   }
 
   bool runOnFunction(Function &F) override {
     auto *LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
-    auto *LAA = &getAnalysis<LoopAccessLegacyAnalysis>();
+    auto GetLAA = [&](Loop &L) -> const LoopAccessInfo & {
+      return getAnalysis<LoopAccessLegacyAnalysis>().getInfo(&L);
+    };
+
     auto *DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
     auto *SE = &getAnalysis<ScalarEvolutionWrapperPass>().getSE();
 
-    // Build up a worklist of inner-loops to version. This is necessary as the
-    // act of versioning a loop creates new loops and can invalidate iterators
-    // across the loops.
-    SmallVector<Loop *, 8> Worklist;
-
-    for (Loop *TopLevelLoop : *LI)
-      for (Loop *L : depth_first(TopLevelLoop))
-        // We only handle inner-most loops.
-        if (L->empty())
-          Worklist.push_back(L);
-
-    // Now walk the identified inner loops.
-    bool Changed = false;
-    for (Loop *L : Worklist) {
-      const LoopAccessInfo &LAI = LAA->getInfo(L);
-      if (L->isLoopSimplifyForm() && !LAI.hasConvergentOp() &&
-          (LAI.getNumRuntimePointerChecks() ||
-           !LAI.getPSE().getUnionPredicate().isAlwaysTrue())) {
-        LoopVersioning LVer(LAI, L, LI, DT, SE);
-        LVer.versionLoop();
-        LVer.annotateLoopWithNoAlias();
-        Changed = true;
-      }
-    }
-
-    return Changed;
+    return runImpl(LI, GetLAA, DT, SE);
   }
 
   void getAnalysisUsage(AnalysisUsage &AU) const override {
@@ -312,18 +323,44 @@ class LoopVersioningPass : public FunctionPass {
 #define LVER_OPTION "loop-versioning"
 #define DEBUG_TYPE LVER_OPTION
 
-char LoopVersioningPass::ID;
+char LoopVersioningLegacyPass::ID;
 static const char LVer_name[] = "Loop Versioning";
 
-INITIALIZE_PASS_BEGIN(LoopVersioningPass, LVER_OPTION, LVer_name, false, false)
+INITIALIZE_PASS_BEGIN(LoopVersioningLegacyPass, LVER_OPTION, LVer_name, false,
+                      false)
 INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(LoopAccessLegacyAnalysis)
 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(ScalarEvolutionWrapperPass)
-INITIALIZE_PASS_END(LoopVersioningPass, LVER_OPTION, LVer_name, false, false)
+INITIALIZE_PASS_END(LoopVersioningLegacyPass, LVER_OPTION, LVer_name, false,
+                    false)
 
 namespace llvm {
-FunctionPass *createLoopVersioningPass() {
-  return new LoopVersioningPass();
+FunctionPass *createLoopVersioningLegacyPass() {
+  return new LoopVersioningLegacyPass();
 }
+
+PreservedAnalyses LoopVersioningPass::run(Function &F,
+                                          FunctionAnalysisManager &AM) {
+  auto &SE = AM.getResult<ScalarEvolutionAnalysis>(F);
+  auto &LI = AM.getResult<LoopAnalysis>(F);
+  auto &TTI = AM.getResult<TargetIRAnalysis>(F);
+  auto &DT = AM.getResult<DominatorTreeAnalysis>(F);
+  auto &TLI = AM.getResult<TargetLibraryAnalysis>(F);
+  auto &AA = AM.getResult<AAManager>(F);
+  auto &AC = AM.getResult<AssumptionAnalysis>(F);
+  MemorySSA *MSSA = EnableMSSALoopDependency
+                        ? &AM.getResult<MemorySSAAnalysis>(F).getMSSA()
+                        : nullptr;
+
+  auto &LAM = AM.getResult<LoopAnalysisManagerFunctionProxy>(F).getManager();
+  auto GetLAA = [&](Loop &L) -> const LoopAccessInfo & {
+    LoopStandardAnalysisResults AR = {AA, AC, DT, LI, SE, TLI, TTI, MSSA};
+    return LAM.getResult<LoopAccessAnalysis>(L, AR);
+  };
+
+  if (runImpl(&LI, GetLAA, &DT, &SE))
+    return PreservedAnalyses::none();
+  return PreservedAnalyses::all();
 }
+} // namespace llvm
diff --git a/llvm/test/Transforms/LoopVersioning/basic.ll b/llvm/test/Transforms/LoopVersioning/basic.ll
index bcd5e8c875c69..bd8ea4d28e162 100644
--- a/llvm/test/Transforms/LoopVersioning/basic.ll
+++ b/llvm/test/Transforms/LoopVersioning/basic.ll
@@ -1,4 +1,5 @@
 ; RUN: opt -basic-aa -loop-versioning -S < %s | FileCheck %s
+; RUN: opt -aa-pipeline=basic-aa -passes=loop-versioning -S < %s | FileCheck %s
 
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 

From 219f32f4b68679563443cdaae7b8174c9976409a Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Mon, 3 Aug 2020 17:53:32 +0100
Subject: [PATCH 0933/1035] [X86][SSE] Shuffle combine blends to OR(X,Y) if the
 relevant elements are known zero.

This allows us to remove the (depth violating) code in getFauxShuffleMask where we were combining the OR(SHUFFLE,SHUFFLE) shuffle inputs as well, and not just the OR().

This is a minor step toward being able to shuffle combine from/to SELECT/BLENDV as a faux shuffle.
---
 llvm/lib/Target/X86/X86ISelLowering.cpp       | 65 ++++++++++++++-----
 llvm/test/CodeGen/X86/insertelement-ones.ll   | 12 ++--
 llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll |  8 +--
 .../test/CodeGen/X86/vector-shuffle-128-v8.ll |  5 +-
 .../CodeGen/X86/vector-shuffle-256-v32.ll     |  6 +-
 5 files changed, 61 insertions(+), 35 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index e9bb50aacec0e..b2bfcc2698f4d 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -7401,8 +7401,8 @@ static bool getFauxShuffleMask(SDValue N, const APInt &DemandedElts,
 
     // Handle OR(SHUFFLE,SHUFFLE) case where one source is zero and the other
     // is a valid shuffle index.
-    SDValue N0 = peekThroughOneUseBitcasts(N.getOperand(0));
-    SDValue N1 = peekThroughOneUseBitcasts(N.getOperand(1));
+    SDValue N0 = peekThroughBitcasts(N.getOperand(0));
+    SDValue N1 = peekThroughBitcasts(N.getOperand(1));
     if (!N0.getValueType().isVector() || !N1.getValueType().isVector())
       return false;
     SmallVector<int, 64> SrcMask0, SrcMask1;
@@ -7413,34 +7413,24 @@ static bool getFauxShuffleMask(SDValue N, const APInt &DemandedElts,
                                 true))
       return false;
 
-    // Shuffle inputs must be the same size as the result.
-    if (llvm::any_of(SrcInputs0, [VT](SDValue Op) {
-          return VT.getSizeInBits() != Op.getValueSizeInBits();
-        }))
-      return false;
-    if (llvm::any_of(SrcInputs1, [VT](SDValue Op) {
-          return VT.getSizeInBits() != Op.getValueSizeInBits();
-        }))
-      return false;
-
     size_t MaskSize = std::max(SrcMask0.size(), SrcMask1.size());
     SmallVector<int, 64> Mask0, Mask1;
     narrowShuffleMaskElts(MaskSize / SrcMask0.size(), SrcMask0, Mask0);
     narrowShuffleMaskElts(MaskSize / SrcMask1.size(), SrcMask1, Mask1);
-    for (size_t i = 0; i != MaskSize; ++i) {
+    for (int i = 0; i != (int)MaskSize; ++i) {
       if (Mask0[i] == SM_SentinelUndef && Mask1[i] == SM_SentinelUndef)
         Mask.push_back(SM_SentinelUndef);
       else if (Mask0[i] == SM_SentinelZero && Mask1[i] == SM_SentinelZero)
         Mask.push_back(SM_SentinelZero);
       else if (Mask1[i] == SM_SentinelZero)
-        Mask.push_back(Mask0[i]);
+        Mask.push_back(i);
       else if (Mask0[i] == SM_SentinelZero)
-        Mask.push_back(Mask1[i] + (int)(MaskSize * SrcInputs0.size()));
+        Mask.push_back(i + MaskSize);
       else
         return false;
     }
-    Ops.append(SrcInputs0.begin(), SrcInputs0.end());
-    Ops.append(SrcInputs1.begin(), SrcInputs1.end());
+    Ops.push_back(N0);
+    Ops.push_back(N1);
     return true;
   }
   case ISD::INSERT_SUBVECTOR: {
@@ -34219,6 +34209,7 @@ static bool matchBinaryShuffle(MVT MaskVT, ArrayRef<int> Mask,
                                SelectionDAG &DAG, const X86Subtarget &Subtarget,
                                unsigned &Shuffle, MVT &SrcVT, MVT &DstVT,
                                bool IsUnary) {
+  unsigned NumMaskElts = Mask.size();
   unsigned EltSizeInBits = MaskVT.getScalarSizeInBits();
 
   if (MaskVT.is128BitVector()) {
@@ -34276,6 +34267,46 @@ static bool matchBinaryShuffle(MVT MaskVT, ArrayRef<int> Mask,
     }
   }
 
+  // Attempt to match against a OR if we're performing a blend shuffle and the
+  // non-blended source element is zero in each case.
+  if ((EltSizeInBits % V1.getScalarValueSizeInBits()) == 0 &&
+      (EltSizeInBits % V2.getScalarValueSizeInBits()) == 0) {
+    bool IsBlend = true;
+    unsigned NumV1Elts = V1.getValueType().getVectorNumElements();
+    unsigned NumV2Elts = V2.getValueType().getVectorNumElements();
+    unsigned Scale1 = NumV1Elts / NumMaskElts;
+    unsigned Scale2 = NumV2Elts / NumMaskElts;
+    APInt DemandedZeroV1 = APInt::getNullValue(NumV1Elts);
+    APInt DemandedZeroV2 = APInt::getNullValue(NumV2Elts);
+    for (unsigned i = 0; i != NumMaskElts; ++i) {
+      int M = Mask[i];
+      if (M == SM_SentinelUndef)
+        continue;
+      if (M == SM_SentinelZero) {
+        DemandedZeroV1.setBits(i * Scale1, (i + 1) * Scale1);
+        DemandedZeroV2.setBits(i * Scale2, (i + 1) * Scale2);
+        continue;
+      }
+      if (M == i) {
+        DemandedZeroV2.setBits(i * Scale2, (i + 1) * Scale2);
+        continue;
+      }
+      if (M == (i + NumMaskElts)) {
+        DemandedZeroV1.setBits(i * Scale1, (i + 1) * Scale1);
+        continue;
+      }
+      IsBlend = false;
+      break;
+    }
+    if (IsBlend &&
+        DAG.computeKnownBits(V1, DemandedZeroV1).isZero() &&
+        DAG.computeKnownBits(V2, DemandedZeroV2).isZero()) {
+      Shuffle = ISD::OR;
+      SrcVT = DstVT = EVT(MaskVT).changeTypeToInteger().getSimpleVT();
+      return true;
+    }
+  }
+
   return false;
 }
 
diff --git a/llvm/test/CodeGen/X86/insertelement-ones.ll b/llvm/test/CodeGen/X86/insertelement-ones.ll
index 3d8e42b9c07d0..6a9a401264c56 100644
--- a/llvm/test/CodeGen/X86/insertelement-ones.ll
+++ b/llvm/test/CodeGen/X86/insertelement-ones.ll
@@ -389,11 +389,9 @@ define <32 x i8> @insert_v32i8_x123456789ABCDEzGHIJKLMNOPQRSTxx(<32 x i8> %a) {
 ; SSE2-NEXT:    movdqa %xmm3, %xmm4
 ; SSE2-NEXT:    pslldq {{.*#+}} xmm4 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm4[0]
 ; SSE2-NEXT:    por %xmm4, %xmm0
-; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255]
-; SSE2-NEXT:    pand %xmm5, %xmm1
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
 ; SSE2-NEXT:    pslldq {{.*#+}} xmm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm3[0,1]
-; SSE2-NEXT:    pandn %xmm3, %xmm5
-; SSE2-NEXT:    por %xmm5, %xmm1
+; SSE2-NEXT:    por %xmm3, %xmm1
 ; SSE2-NEXT:    pand %xmm2, %xmm1
 ; SSE2-NEXT:    por %xmm4, %xmm1
 ; SSE2-NEXT:    retq
@@ -411,11 +409,9 @@ define <32 x i8> @insert_v32i8_x123456789ABCDEzGHIJKLMNOPQRSTxx(<32 x i8> %a) {
 ; SSE3-NEXT:    movdqa %xmm3, %xmm4
 ; SSE3-NEXT:    pslldq {{.*#+}} xmm4 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm4[0]
 ; SSE3-NEXT:    por %xmm4, %xmm0
-; SSE3-NEXT:    movdqa {{.*#+}} xmm5 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255]
-; SSE3-NEXT:    pand %xmm5, %xmm1
+; SSE3-NEXT:    pand {{.*}}(%rip), %xmm1
 ; SSE3-NEXT:    pslldq {{.*#+}} xmm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm3[0,1]
-; SSE3-NEXT:    pandn %xmm3, %xmm5
-; SSE3-NEXT:    por %xmm5, %xmm1
+; SSE3-NEXT:    por %xmm3, %xmm1
 ; SSE3-NEXT:    pand %xmm2, %xmm1
 ; SSE3-NEXT:    por %xmm4, %xmm1
 ; SSE3-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll b/llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll
index 6b49f22f21f1f..9256a43f8e339 100644
--- a/llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll
+++ b/llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll
@@ -1314,10 +1314,10 @@ define void @trunc_v4i64_to_v4i8(<32 x i8>* %L, <4 x i8>* %S) nounwind {
 define <16 x i8> @negative(<32 x i8> %v, <32 x i8> %w) nounwind {
 ; AVX1-LABEL: negative:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpshufb {{.*#+}} xmm2 = xmm0[u,2,4,6,8,10,12,14],zero,zero,zero,zero,zero,zero,zero,zero
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u],zero,zero,zero,zero,zero,zero,zero,xmm0[0,2,4,6,8,10,12,14]
-; AVX1-NEXT:    vpor %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,u,u,u,u,u,u,u,0,2,4,6,8,10,12,14]
+; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4,5,6,7]
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
 ; AVX1-NEXT:    vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
 ; AVX1-NEXT:    vzeroupper
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-128-v8.ll b/llvm/test/CodeGen/X86/vector-shuffle-128-v8.ll
index f448f41cf522e..86423ce76065b 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-128-v8.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-128-v8.ll
@@ -1713,9 +1713,8 @@ define <8 x i16> @shuffle_v8i16_XX4X8acX(<8 x i16> %a, <8 x i16> %b) {
 ;
 ; SSSE3-LABEL: shuffle_v8i16_XX4X8acX:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,8,9,u,u],zero,zero,zero,zero,zero,zero,xmm0[u,u]
-; SSSE3-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[u,u,u,u],zero,zero,xmm1[u,u,0,1,4,5,8,9,u,u]
-; SSSE3-NEXT:    por %xmm1, %xmm0
+; SSSE3-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[u,u,u,u,u,u,u,u,0,1,4,5,8,9,u,u]
+; SSSE3-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[2,3]
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: shuffle_v8i16_XX4X8acX:
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-256-v32.ll b/llvm/test/CodeGen/X86/vector-shuffle-256-v32.ll
index 82df05e5ae068..e5285aebda69e 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-256-v32.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-256-v32.ll
@@ -3358,9 +3358,9 @@ define <32 x i8> @shuffle_v32i8_42_45_12_13_35_35_60_40_17_22_29_44_33_12_48_51_
 ; AVX1-NEXT:    vpshufb {{.*#+}} xmm2 = zero,zero,xmm2[u,u],zero,zero,xmm2[12],zero,xmm2[u,u,u],zero,zero,xmm2[u,0,3]
 ; AVX1-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[10,13,u,u,3,3],zero,xmm1[8,u,u,u,12,1,u],zero,zero
 ; AVX1-NEXT:    vpor %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpshufb {{.*#+}} xmm2 = xmm4[u,u],zero,zero,xmm4[u,u,u,u,1,6,13,u,u],zero,xmm4[u,u]
-; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,12,13,u,u,u,u],zero,zero,zero,xmm0[u,u,12,u,u]
-; AVX1-NEXT:    vpor %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufb {{.*#+}} xmm2 = xmm4[u,u,u,u,u,u,u,u,1,6,13,u,u,u,u,u]
+; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,12,13,u,u,u,u,u,u,u,u,u,12,u,u]
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4,5],xmm0[6,7]
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [255,255,0,0,255,255,255,255,0,0,0,255,255,0,255,255]
 ; AVX1-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0

From f208c659fb76b1ad8ae83dd10c4f0c30986d48ee Mon Sep 17 00:00:00 2001
From: Joao Moreira <joao.moreira@intel.com>
Date: Mon, 3 Aug 2020 10:26:31 -0700
Subject: [PATCH 0934/1035] [X86] Make ENDBR instruction a scheduling boundary

Instructions should not be scheduled across ENDBR instructions, as this would result in the ENDBR being displaced, breaking the parity needed for the Indirect Branch Tracking feature of CET.

Currently, the X86IndirectBranchTracking pass is later than the instruction scheduling in the pipeline, what causes the bug to be unnoticeable and very hard (if not unfeasible) to be triggered while compiling C files with the standard LLVM setup. Yet, for correctness and to prevent issues in future changes, the compiler should prevent the such scheduling.

Differential Revision: https://reviews.llvm.org/D84862
---
 llvm/lib/Target/X86/X86InstrInfo.cpp | 12 ++++++++++++
 llvm/lib/Target/X86/X86InstrInfo.h   |  7 +++++++
 2 files changed, 19 insertions(+)

diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp
index ae838274f2e68..c753880fc92cd 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.cpp
+++ b/llvm/lib/Target/X86/X86InstrInfo.cpp
@@ -6675,6 +6675,18 @@ bool X86InstrInfo::shouldScheduleLoadsNear(SDNode *Load1, SDNode *Load2,
   return true;
 }
 
+bool X86InstrInfo::isSchedulingBoundary(const MachineInstr &MI,
+                                        const MachineBasicBlock *MBB,
+                                        const MachineFunction &MF) const {
+
+  // ENDBR instructions should not be scheduled around.
+  unsigned Opcode = MI.getOpcode();
+  if (Opcode == X86::ENDBR64 || Opcode == X86::ENDBR32)
+    return true;
+
+  return TargetInstrInfo::isSchedulingBoundary(MI, MBB, MF);
+}
+
 bool X86InstrInfo::
 reverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const {
   assert(Cond.size() == 1 && "Invalid X86 branch condition!");
diff --git a/llvm/lib/Target/X86/X86InstrInfo.h b/llvm/lib/Target/X86/X86InstrInfo.h
index 89f2ff118c378..c345a8217168d 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.h
+++ b/llvm/lib/Target/X86/X86InstrInfo.h
@@ -409,6 +409,13 @@ class X86InstrInfo final : public X86GenInstrInfo {
   bool areLoadsFromSameBasePtr(SDNode *Load1, SDNode *Load2, int64_t &Offset1,
                                int64_t &Offset2) const override;
 
+  /// isSchedulingBoundary - Overrides the isSchedulingBoundary from
+  ///	Codegen/TargetInstrInfo.cpp to make it capable of identifying ENDBR
+  /// intructions and prevent it from being re-scheduled.
+  bool isSchedulingBoundary(const MachineInstr &MI,
+                            const MachineBasicBlock *MBB,
+                            const MachineFunction &MF) const override;
+
   /// shouldScheduleLoadsNear - This is a used by the pre-regalloc scheduler to
   /// determine (in conjunction with areLoadsFromSameBasePtr) if two loads
   /// should be scheduled togther. On some targets if two loads are loading from

From 39494d9c21bab3281e4af30578af10f37ea09470 Mon Sep 17 00:00:00 2001
From: Rainer Orth <ro@gcc.gnu.org>
Date: Mon, 3 Aug 2020 19:56:05 +0200
Subject: [PATCH 0935/1035] [compiler-rt][profile] Fix various InstrProf tests
 on Solaris

Currently, several InstrProf tests `FAIL` on Solaris (both sparc and x86):

  Profile-i386 :: Posix/instrprof-visibility.cpp
  Profile-i386 :: instrprof-merging.cpp
  Profile-i386 :: instrprof-set-file-object-merging.c
  Profile-i386 :: instrprof-set-file-object.c

On sparc there's also

  Profile-sparc :: coverage_comments.cpp

The failure mode is always the same:

  error: /var/llvm/local-amd64/projects/compiler-rt/test/profile/Profile-i386/Posix/Output/instrprof-visibility.cpp.tmp: Failed to load coverage: Malformed coverage data

The error is from `llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp`
(`loadBinaryFormat`), l.926:

  InstrProfSymtab ProfileNames;
  std::vector<SectionRef> NamesSectionRefs = *NamesSection;
  if (NamesSectionRefs.size() != 1)
    return make_error<CoverageMapError>(coveragemap_error::malformed);

where .size() is 2 instead.

Looking at the executable, I find (with `elfdump -c -N __llvm_prf_names`):

  Section Header[15]:  sh_name: __llvm_prf_names
      sh_addr:      0x8053ca5       sh_flags:   [ SHF_ALLOC ]
      sh_size:      0x86            sh_type:    [ SHT_PROGBITS ]
      sh_offset:    0x3ca5          sh_entsize: 0
      sh_link:      0               sh_info:    0
      sh_addralign: 0x1

  Section Header[31]:  sh_name: __llvm_prf_names
      sh_addr:      0x8069998       sh_flags:   [ SHF_WRITE SHF_ALLOC ]
      sh_size:      0               sh_type:    [ SHT_PROGBITS ]
      sh_offset:    0x9998          sh_entsize: 0
      sh_link:      0               sh_info:    0
      sh_addralign: 0x1

Unlike GNU `ld` (which primarily operates on section names) the Solaris
linker, following the ELF spirit, only merges input sections into an output
section if both section name and section flags match, so two separate
sections are maintained.

The read-write one comes from `lib/clang/12.0.0/lib/sunos/libclang_rt.profile-i386.a(InstrProfilingPlatformLinux.c.o)`
while the read-only one is generated by
`llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp` (`InstrProfiling::emitNameData`)
at l.1004 where `isConstant = true`.

The easiest way to avoid the mismatch is to change the definition in
`compiler-rt/lib/profile/InstrProfilingPlatformLinux.c` to `const`.

This fixes all failures observed.

Tested on `amd64-pc-solaris2.11`, `sparcv9-sun-solaris2.11`, and
`x86_64-pc-linux-gnu`.

Differential Revision: https://reviews.llvm.org/D85116
---
 compiler-rt/lib/profile/InstrProfilingPlatformLinux.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c b/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c
index becfe1fd9f5ad..c9fb481f8e90c 100644
--- a/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c
+++ b/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c
@@ -43,7 +43,7 @@ uint64_t
     __prof_cnts_sect_data[0] COMPILER_RT_SECTION(INSTR_PROF_CNTS_SECT_NAME);
 uint32_t
     __prof_orderfile_sect_data[0] COMPILER_RT_SECTION(INSTR_PROF_ORDERFILE_SECT_NAME);
-char __prof_nms_sect_data[0] COMPILER_RT_SECTION(INSTR_PROF_NAME_SECT_NAME);
+const char __prof_nms_sect_data[0] COMPILER_RT_SECTION(INSTR_PROF_NAME_SECT_NAME);
 ValueProfNode __prof_vnodes_sect_data[0] COMPILER_RT_SECTION(INSTR_PROF_VNODES_SECT_NAME);
 
 COMPILER_RT_VISIBILITY const __llvm_profile_data *

From f78f509c75861dc4e26f9a22ad12996bf8005a2e Mon Sep 17 00:00:00 2001
From: Hiroshi Yamauchi <yamauchi@google.com>
Date: Mon, 3 Aug 2020 10:35:47 -0700
Subject: [PATCH 0936/1035] [PGO] Extend the value profile buckets for mem op
 sizes.

Extend the memop value profile buckets to be more flexible (could accommodate a
mix of individual values and ranges) and to cover more value ranges (from 11 to
22 buckets).

Disabled behind a flag (to be enabled separately) and the existing code to be
removed later.

Differential Revision: https://reviews.llvm.org/D81682
---
 compiler-rt/include/profile/InstrProfData.inc | 125 ++++++++++++++++++
 compiler-rt/lib/profile/InstrProfilingValue.c |  17 ++-
 llvm/include/llvm/ProfileData/InstrProf.h     |   8 ++
 .../llvm/ProfileData/InstrProfData.inc        | 125 ++++++++++++++++++
 .../Instrumentation/InstrProfiling.h          |   2 +
 llvm/lib/ProfileData/InstrProf.cpp            |   2 +
 .../Instrumentation/InstrProfiling.cpp        |  55 ++++++--
 .../Instrumentation/PGOMemOPSizeOpt.cpp       |  21 ++-
 llvm/test/Transforms/PGOProfile/memcpy.ll     |   9 +-
 .../PGOProfile/memop_profile_funclet.ll       |   9 +-
 llvm/unittests/ProfileData/CMakeLists.txt     |   1 +
 .../ProfileData/InstrProfDataTest.cpp         |  68 ++++++++++
 12 files changed, 423 insertions(+), 19 deletions(-)
 create mode 100644 llvm/unittests/ProfileData/InstrProfDataTest.cpp

diff --git a/compiler-rt/include/profile/InstrProfData.inc b/compiler-rt/include/profile/InstrProfData.inc
index a6913527e67f0..6d0ffb12294b0 100644
--- a/compiler-rt/include/profile/InstrProfData.inc
+++ b/compiler-rt/include/profile/InstrProfData.inc
@@ -157,6 +157,8 @@ VALUE_PROF_FUNC_PARAM(void *, Data, Type::getInt8PtrTy(Ctx)) INSTR_PROF_COMMA
 #ifndef VALUE_RANGE_PROF
 VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx))
 #else /* VALUE_RANGE_PROF */
+/* FIXME: This is to be removed after switching to the new memop value
+ * profiling. */
 VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx)) \
                       INSTR_PROF_COMMA
 VALUE_PROF_FUNC_PARAM(uint64_t, PreciseRangeStart, Type::getInt64Ty(Ctx)) \
@@ -753,9 +755,14 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
 #define INSTR_PROF_VALUE_PROF_FUNC __llvm_profile_instrument_target
 #define INSTR_PROF_VALUE_PROF_FUNC_STR \
         INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_FUNC)
+/* FIXME: This is to be removed after switching to the new memop value
+ * profiling. */
 #define INSTR_PROF_VALUE_RANGE_PROF_FUNC __llvm_profile_instrument_range
 #define INSTR_PROF_VALUE_RANGE_PROF_FUNC_STR \
         INSTR_PROF_QUOTE(INSTR_PROF_VALUE_RANGE_PROF_FUNC)
+#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC __llvm_profile_instrument_memop
+#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC_STR                                   \
+  INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_MEMOP_FUNC)
 
 /* InstrProfile per-function control data alignment.  */
 #define INSTR_PROF_DATA_ALIGNMENT 8
@@ -783,3 +790,121 @@ typedef struct InstrProfValueData {
 #endif
 
 #undef COVMAP_V2_OR_V3
+
+#ifdef INSTR_PROF_VALUE_PROF_MEMOP_API
+
+#ifdef __cplusplus
+#define INSTR_PROF_INLINE inline
+#else
+#define INSTR_PROF_INLINE
+#endif
+
+/* The value range buckets (22 buckets) for the memop size value profiling looks
+ * like:
+ *
+ *   [0, 0]
+ *   [1, 1]
+ *   [2, 2]
+ *   [3, 3]
+ *   [4, 4]
+ *   [5, 5]
+ *   [6, 6]
+ *   [7, 7]
+ *   [8, 8]
+ *   [9, 15]
+ *   [16, 16]
+ *   [17, 31]
+ *   [32, 32]
+ *   [33, 63]
+ *   [64, 64]
+ *   [65, 127]
+ *   [128, 128]
+ *   [129, 255]
+ *   [256, 256]
+ *   [257, 511]
+ *   [512, 512]
+ *   [513, UINT64_MAX]
+ *
+ * Each range has a 'representative value' which is the lower end value of the
+ * range and used to store in the runtime profile data records and the VP
+ * metadata. For example, it's 2 for [2, 2] and 64 for [65, 127].
+ */
+
+/*
+ * Clz and Popcount. This code was copied from
+ * compiler-rt/lib/fuzzer/{FuzzerBuiltins.h,FuzzerBuiltinsMsvc.h} and
+ * llvm/include/llvm/Support/MathExtras.h.
+ */
+#if defined(_MSC_VER) && !defined(__clang__)
+
+#include <intrin.h>
+INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
+int InstProfClzll(unsigned long long X) {
+  unsigned long LeadZeroIdx = 0;
+#if !defined(_M_ARM64) && !defined(_M_X64)
+  // Scan the high 32 bits.
+  if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X >> 32)))
+    return (int)(63 - (LeadZeroIdx + 32)); // Create a bit offset
+                                                      // from the MSB.
+  // Scan the low 32 bits.
+  if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X)))
+    return (int)(63 - LeadZeroIdx);
+#else
+  if (_BitScanReverse64(&LeadZeroIdx, X)) return 63 - LeadZeroIdx;
+#endif
+  return 64;
+}
+INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
+int InstProfPopcountll(unsigned long long X) {
+  // This code originates from https://reviews.llvm.org/rG30626254510f.
+  unsigned long long v = X;
+  v = v - ((v >> 1) & 0x5555555555555555ULL);
+  v = (v & 0x3333333333333333ULL) + ((v >> 2) & 0x3333333333333333ULL);
+  v = (v + (v >> 4)) & 0x0F0F0F0F0F0F0F0FULL;
+  return (int)((unsigned long long)(v * 0x0101010101010101ULL) >> 56);
+}
+
+#else
+
+INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
+int InstProfClzll(unsigned long long X) { return __builtin_clzll(X); }
+INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
+int InstProfPopcountll(unsigned long long X) { return __builtin_popcountll(X); }
+
+#endif  /* defined(_MSC_VER) && !defined(__clang__) */
+
+/* Map an (observed) memop size value to the representative value of its range.
+ * For example, 5 -> 5, 22 -> 17, 99 -> 65, 256 -> 256, 1001 -> 513. */
+INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint64_t
+InstrProfGetRangeRepValue(uint64_t Value) {
+  if (Value <= 8)
+    // The first ranges are individually tracked. Use the value as is.
+    return Value;
+  else if (Value >= 513)
+    // The last range is mapped to its lowest value.
+    return 513;
+  else if (InstProfPopcountll(Value) == 1)
+    // If it's a power of two, use it as is.
+    return Value;
+  else
+    // Otherwise, take to the previous power of two + 1.
+    return (1 << (64 - InstProfClzll(Value) - 1)) + 1;
+}
+
+/* Return true if the range that an (observed) memop size value belongs to has
+ * only a single value in the range.  For example, 0 -> true, 8 -> true, 10 ->
+ * false, 64 -> true, 100 -> false, 513 -> false. */
+INSTR_PROF_VISIBILITY INSTR_PROF_INLINE unsigned
+InstrProfIsSingleValRange(uint64_t Value) {
+  if (Value <= 8)
+    // The first ranges are individually tracked.
+    return 1;
+  else if (InstProfPopcountll(Value) == 1)
+    // If it's a power of two, there's only one value.
+    return 1;
+  else
+    // Otherwise, there's more than one value in the range.
+    return 0;
+}
+
+#endif /* INSTR_PROF_VALUE_PROF_MEMOP_API */
diff --git a/compiler-rt/lib/profile/InstrProfilingValue.c b/compiler-rt/lib/profile/InstrProfilingValue.c
index 4b4081bd21b70..f6b28b48e8426 100644
--- a/compiler-rt/lib/profile/InstrProfilingValue.c
+++ b/compiler-rt/lib/profile/InstrProfilingValue.c
@@ -18,13 +18,14 @@
 
 #define INSTR_PROF_VALUE_PROF_DATA
 #define INSTR_PROF_COMMON_API_IMPL
+#define INSTR_PROF_VALUE_PROF_MEMOP_API
 #include "profile/InstrProfData.inc"
 
 static int hasStaticCounters = 1;
 static int OutOfNodesWarnings = 0;
 static int hasNonDefaultValsPerSite = 0;
 #define INSTR_PROF_MAX_VP_WARNS 10
-#define INSTR_PROF_DEFAULT_NUM_VAL_PER_SITE 16
+#define INSTR_PROF_DEFAULT_NUM_VAL_PER_SITE 24
 #define INSTR_PROF_VNODE_POOL_SIZE 1024
 
 #ifndef _MSC_VER
@@ -253,6 +254,8 @@ __llvm_profile_instrument_target_value(uint64_t TargetValue, void *Data,
  * The range for large values is optional. The default value of INT64_MIN
  * indicates it is not specified.
  */
+/* FIXME: This is to be removed after switching to the new memop value
+ * profiling. */
 COMPILER_RT_VISIBILITY void __llvm_profile_instrument_range(
     uint64_t TargetValue, void *Data, uint32_t CounterIndex,
     int64_t PreciseRangeStart, int64_t PreciseRangeLast, int64_t LargeValue) {
@@ -266,6 +269,18 @@ COMPILER_RT_VISIBILITY void __llvm_profile_instrument_range(
   __llvm_profile_instrument_target(TargetValue, Data, CounterIndex);
 }
 
+/*
+ * The target values are partitioned into multiple ranges. The range spec is
+ * defined in InstrProfData.inc.
+ */
+COMPILER_RT_VISIBILITY void
+__llvm_profile_instrument_memop(uint64_t TargetValue, void *Data,
+                                uint32_t CounterIndex) {
+  // Map the target value to the representative value of its range.
+  uint64_t RepValue = InstrProfGetRangeRepValue(TargetValue);
+  __llvm_profile_instrument_target(RepValue, Data, CounterIndex);
+}
+
 /*
  * A wrapper struct that represents value profile runtime data.
  * Like InstrProfRecord class which is used by profiling host tools,
diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h
index 50c4857537812..c767a362d6dcb 100644
--- a/llvm/include/llvm/ProfileData/InstrProf.h
+++ b/llvm/include/llvm/ProfileData/InstrProf.h
@@ -75,10 +75,18 @@ inline StringRef getInstrProfValueProfFuncName() {
 }
 
 /// Return the name profile runtime entry point to do value range profiling.
+// FIXME: This is to be removed after switching to the new memop value
+// profiling.
 inline StringRef getInstrProfValueRangeProfFuncName() {
   return INSTR_PROF_VALUE_RANGE_PROF_FUNC_STR;
 }
 
+/// Return the name profile runtime entry point to do memop size value
+/// profiling.
+inline StringRef getInstrProfValueProfMemOpFuncName() {
+  return INSTR_PROF_VALUE_PROF_MEMOP_FUNC_STR;
+}
+
 /// Return the name prefix of variables containing instrumented function names.
 inline StringRef getInstrProfNameVarPrefix() { return "__profn_"; }
 
diff --git a/llvm/include/llvm/ProfileData/InstrProfData.inc b/llvm/include/llvm/ProfileData/InstrProfData.inc
index e56623afad644..06927fb5652b9 100644
--- a/llvm/include/llvm/ProfileData/InstrProfData.inc
+++ b/llvm/include/llvm/ProfileData/InstrProfData.inc
@@ -157,6 +157,8 @@ VALUE_PROF_FUNC_PARAM(void *, Data, Type::getInt8PtrTy(Ctx)) INSTR_PROF_COMMA
 #ifndef VALUE_RANGE_PROF
 VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx))
 #else /* VALUE_RANGE_PROF */
+/* FIXME: This is to be removed after switching to the new memop value
+ * profiling. */
 VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx)) \
                       INSTR_PROF_COMMA
 VALUE_PROF_FUNC_PARAM(uint64_t, PreciseRangeStart, Type::getInt64Ty(Ctx)) \
@@ -754,9 +756,14 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
 #define INSTR_PROF_VALUE_PROF_FUNC __llvm_profile_instrument_target
 #define INSTR_PROF_VALUE_PROF_FUNC_STR \
         INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_FUNC)
+/* FIXME: This is to be removed after switching to the new memop value
+ * profiling. */
 #define INSTR_PROF_VALUE_RANGE_PROF_FUNC __llvm_profile_instrument_range
 #define INSTR_PROF_VALUE_RANGE_PROF_FUNC_STR \
         INSTR_PROF_QUOTE(INSTR_PROF_VALUE_RANGE_PROF_FUNC)
+#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC __llvm_profile_instrument_memop
+#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC_STR                                   \
+  INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_MEMOP_FUNC)
 
 /* InstrProfile per-function control data alignment.  */
 #define INSTR_PROF_DATA_ALIGNMENT 8
@@ -784,3 +791,121 @@ typedef struct InstrProfValueData {
 #endif
 
 #undef COVMAP_V2_OR_V3
+
+#ifdef INSTR_PROF_VALUE_PROF_MEMOP_API
+
+#ifdef __cplusplus
+#define INSTR_PROF_INLINE inline
+#else
+#define INSTR_PROF_INLINE
+#endif
+
+/* The value range buckets (22 buckets) for the memop size value profiling looks
+ * like:
+ *
+ *   [0, 0]
+ *   [1, 1]
+ *   [2, 2]
+ *   [3, 3]
+ *   [4, 4]
+ *   [5, 5]
+ *   [6, 6]
+ *   [7, 7]
+ *   [8, 8]
+ *   [9, 15]
+ *   [16, 16]
+ *   [17, 31]
+ *   [32, 32]
+ *   [33, 63]
+ *   [64, 64]
+ *   [65, 127]
+ *   [128, 128]
+ *   [129, 255]
+ *   [256, 256]
+ *   [257, 511]
+ *   [512, 512]
+ *   [513, UINT64_MAX]
+ *
+ * Each range has a 'representative value' which is the lower end value of the
+ * range and used to store in the runtime profile data records and the VP
+ * metadata. For example, it's 2 for [2, 2] and 64 for [65, 127].
+ */
+
+/*
+ * Clz and Popcount. This code was copied from
+ * compiler-rt/lib/fuzzer/{FuzzerBuiltins.h,FuzzerBuiltinsMsvc.h} and
+ * llvm/include/llvm/Support/MathExtras.h.
+ */
+#if defined(_MSC_VER) && !defined(__clang__)
+
+#include <intrin.h>
+INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
+int InstProfClzll(unsigned long long X) {
+  unsigned long LeadZeroIdx = 0;
+#if !defined(_M_ARM64) && !defined(_M_X64)
+  // Scan the high 32 bits.
+  if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X >> 32)))
+    return (int)(63 - (LeadZeroIdx + 32)); // Create a bit offset
+                                                      // from the MSB.
+  // Scan the low 32 bits.
+  if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X)))
+    return (int)(63 - LeadZeroIdx);
+#else
+  if (_BitScanReverse64(&LeadZeroIdx, X)) return 63 - LeadZeroIdx;
+#endif
+  return 64;
+}
+INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
+int InstProfPopcountll(unsigned long long X) {
+  // This code originates from https://reviews.llvm.org/rG30626254510f.
+  unsigned long long v = X;
+  v = v - ((v >> 1) & 0x5555555555555555ULL);
+  v = (v & 0x3333333333333333ULL) + ((v >> 2) & 0x3333333333333333ULL);
+  v = (v + (v >> 4)) & 0x0F0F0F0F0F0F0F0FULL;
+  return (int)((unsigned long long)(v * 0x0101010101010101ULL) >> 56);
+}
+
+#else
+
+INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
+int InstProfClzll(unsigned long long X) { return __builtin_clzll(X); }
+INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
+int InstProfPopcountll(unsigned long long X) { return __builtin_popcountll(X); }
+
+#endif  /* defined(_MSC_VER) && !defined(__clang__) */
+
+/* Map an (observed) memop size value to the representative value of its range.
+ * For example, 5 -> 5, 22 -> 17, 99 -> 65, 256 -> 256, 1001 -> 513. */
+INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint64_t
+InstrProfGetRangeRepValue(uint64_t Value) {
+  if (Value <= 8)
+    // The first ranges are individually tracked. Use the value as is.
+    return Value;
+  else if (Value >= 513)
+    // The last range is mapped to its lowest value.
+    return 513;
+  else if (InstProfPopcountll(Value) == 1)
+    // If it's a power of two, use it as is.
+    return Value;
+  else
+    // Otherwise, take to the previous power of two + 1.
+    return (1 << (64 - InstProfClzll(Value) - 1)) + 1;
+}
+
+/* Return true if the range that an (observed) memop size value belongs to has
+ * only a single value in the range.  For example, 0 -> true, 8 -> true, 10 ->
+ * false, 64 -> true, 100 -> false, 513 -> false. */
+INSTR_PROF_VISIBILITY INSTR_PROF_INLINE unsigned
+InstrProfIsSingleValRange(uint64_t Value) {
+  if (Value <= 8)
+    // The first ranges are individually tracked.
+    return 1;
+  else if (InstProfPopcountll(Value) == 1)
+    // If it's a power of two, there's only one value.
+    return 1;
+  else
+    // Otherwise, there's more than one value in the range.
+    return 0;
+}
+
+#endif /* INSTR_PROF_VALUE_PROF_MEMOP_API */
diff --git a/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h b/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h
index 263d3b629589c..a7052f7b6a2b1 100644
--- a/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h
+++ b/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h
@@ -68,6 +68,8 @@ class InstrProfiling : public PassInfoMixin<InstrProfiling> {
   // vector of counter load/store pairs to be register promoted.
   std::vector<LoadStorePair> PromotionCandidates;
 
+  // FIXME: These are to be removed after switching to the new memop value
+  // profiling.
   // The start value of precise value profile range for memory intrinsic sizes.
   int64_t MemOPSizeRangeStart;
   // The end value of precise value profile range for memory intrinsic sizes.
diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp
index fb788ef4c7655..a172f319b502c 100644
--- a/llvm/lib/ProfileData/InstrProf.cpp
+++ b/llvm/lib/ProfileData/InstrProf.cpp
@@ -1112,6 +1112,8 @@ bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken) {
   return true;
 }
 
+// FIXME: This is to be removed after switching to the new memop value
+// profiling.
 // Parse the value profile options.
 void getMemOPSizeRangeFromOption(StringRef MemOPSizeRange, int64_t &RangeStart,
                                  int64_t &RangeLast) {
diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
index 8279716002864..3ab697d6cc321 100644
--- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
+++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
@@ -57,6 +57,8 @@ using namespace llvm;
 
 #define DEBUG_TYPE "instrprof"
 
+// FIXME: These are to be removed after switching to the new memop value
+// profiling.
 // The start and end values of precise value profile range for memory
 // intrinsic sizes
 cl::opt<std::string> MemOPSizeRange(
@@ -72,6 +74,12 @@ cl::opt<unsigned> MemOPSizeLarge(
              "Value of 0 disables the large value profiling."),
     cl::init(8192));
 
+cl::opt<bool> UseOldMemOpValueProf(
+    "use-old-memop-value-prof",
+    cl::desc("Use the old memop value profiling buckets. This is "
+             "transitional and to be removed after switching. "),
+    cl::init(true));
+
 namespace {
 
 cl::opt<bool> DoHashBasedCounterSplit(
@@ -411,6 +419,19 @@ class PGOCounterPromoter {
   BlockFrequencyInfo *BFI;
 };
 
+enum class ValueProfilingCallType {
+  // Individual values are tracked. Currently used for indiret call target
+  // profiling.
+  Default,
+
+  // The old memop size value profiling. FIXME: To be removed after switching to
+  // the new one.
+  OldMemOp,
+
+  // MemOp: the (new) memop size value profiling with extended buckets.
+  MemOp
+};
+
 } // end anonymous namespace
 
 PreservedAnalyses InstrProfiling::run(Module &M, ModuleAnalysisManager &AM) {
@@ -595,9 +616,9 @@ bool InstrProfiling::run(
   return true;
 }
 
-static FunctionCallee
-getOrInsertValueProfilingCall(Module &M, const TargetLibraryInfo &TLI,
-                              bool IsRange = false) {
+static FunctionCallee getOrInsertValueProfilingCall(
+    Module &M, const TargetLibraryInfo &TLI,
+    ValueProfilingCallType CallType = ValueProfilingCallType::Default) {
   LLVMContext &Ctx = M.getContext();
   auto *ReturnTy = Type::getVoidTy(M.getContext());
 
@@ -605,16 +626,22 @@ getOrInsertValueProfilingCall(Module &M, const TargetLibraryInfo &TLI,
   if (auto AK = TLI.getExtAttrForI32Param(false))
     AL = AL.addParamAttribute(M.getContext(), 2, AK);
 
-  if (!IsRange) {
+  if (CallType == ValueProfilingCallType::Default ||
+      CallType == ValueProfilingCallType::MemOp) {
     Type *ParamTypes[] = {
 #define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType
 #include "llvm/ProfileData/InstrProfData.inc"
     };
     auto *ValueProfilingCallTy =
         FunctionType::get(ReturnTy, makeArrayRef(ParamTypes), false);
-    return M.getOrInsertFunction(getInstrProfValueProfFuncName(),
-                                 ValueProfilingCallTy, AL);
+    StringRef FuncName = CallType == ValueProfilingCallType::Default
+                             ? getInstrProfValueProfFuncName()
+                             : getInstrProfValueProfMemOpFuncName();
+    return M.getOrInsertFunction(FuncName, ValueProfilingCallTy, AL);
   } else {
+    // FIXME: This code is to be removed after switching to the new memop value
+    // profiling.
+    assert(CallType == ValueProfilingCallType::OldMemOp);
     Type *RangeParamTypes[] = {
 #define VALUE_RANGE_PROF 1
 #define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType
@@ -654,8 +681,8 @@ void InstrProfiling::lowerValueProfileInst(InstrProfValueProfileInst *Ind) {
     Index += It->second.NumValueSites[Kind];
 
   IRBuilder<> Builder(Ind);
-  bool IsRange = (Ind->getValueKind()->getZExtValue() ==
-                  llvm::InstrProfValueKind::IPVK_MemOPSize);
+  bool IsMemOpSize = (Ind->getValueKind()->getZExtValue() ==
+                      llvm::InstrProfValueKind::IPVK_MemOPSize);
   CallInst *Call = nullptr;
   auto *TLI = &GetTLI(*Ind->getFunction());
 
@@ -665,12 +692,19 @@ void InstrProfiling::lowerValueProfileInst(InstrProfValueProfileInst *Ind) {
   // WinEHPrepare pass.
   SmallVector<OperandBundleDef, 1> OpBundles;
   Ind->getOperandBundlesAsDefs(OpBundles);
-  if (!IsRange) {
+  if (!IsMemOpSize) {
     Value *Args[3] = {Ind->getTargetValue(),
                       Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()),
                       Builder.getInt32(Index)};
     Call = Builder.CreateCall(getOrInsertValueProfilingCall(*M, *TLI), Args,
                               OpBundles);
+  } else if (!UseOldMemOpValueProf) {
+    Value *Args[3] = {Ind->getTargetValue(),
+                      Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()),
+                      Builder.getInt32(Index)};
+    Call = Builder.CreateCall(
+        getOrInsertValueProfilingCall(*M, *TLI, ValueProfilingCallType::MemOp),
+        Args, OpBundles);
   } else {
     Value *Args[6] = {
         Ind->getTargetValue(),
@@ -679,7 +713,8 @@ void InstrProfiling::lowerValueProfileInst(InstrProfValueProfileInst *Ind) {
         Builder.getInt64(MemOPSizeRangeStart),
         Builder.getInt64(MemOPSizeRangeLast),
         Builder.getInt64(MemOPSizeLarge == 0 ? INT64_MIN : MemOPSizeLarge)};
-    Call = Builder.CreateCall(getOrInsertValueProfilingCall(*M, *TLI, true),
+    Call = Builder.CreateCall(getOrInsertValueProfilingCall(
+                                  *M, *TLI, ValueProfilingCallType::OldMemOp),
                               Args, OpBundles);
   }
   if (auto AK = TLI->getExtAttrForI32Param(false))
diff --git a/llvm/lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp b/llvm/lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp
index 2b7b859891dcd..43a1434ae2d37 100644
--- a/llvm/lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp
+++ b/llvm/lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp
@@ -38,6 +38,8 @@
 #include "llvm/Pass.h"
 #include "llvm/PassRegistry.h"
 #include "llvm/ProfileData/InstrProf.h"
+#define INSTR_PROF_VALUE_PROF_MEMOP_API
+#include "llvm/ProfileData/InstrProfData.inc"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
@@ -89,17 +91,25 @@ static cl::opt<bool>
                     cl::desc("Scale the memop size counts using the basic "
                              " block count value"));
 
+// FIXME: These are to be removed after switching to the new memop value
+// profiling.
 // This option sets the rangge of precise profile memop sizes.
 extern cl::opt<std::string> MemOPSizeRange;
 
 // This option sets the value that groups large memop sizes
 extern cl::opt<unsigned> MemOPSizeLarge;
 
+extern cl::opt<bool> UseOldMemOpValueProf;
+
 cl::opt<bool>
     MemOPOptMemcmpBcmp("pgo-memop-optimize-memcmp-bcmp", cl::init(true),
                        cl::Hidden,
                        cl::desc("Size-specialize memcmp and bcmp calls"));
 
+static cl::opt<unsigned>
+    MemOpMaxOptSize("memop-value-prof-max-opt-size", cl::Hidden, cl::init(128),
+                    cl::desc("Optimize the memop size <= this value"));
+
 namespace {
 class PGOMemOPSizeOptLegacyPass : public FunctionPass {
 public:
@@ -269,6 +279,8 @@ class MemOPSizeOpt : public InstVisitor<MemOPSizeOpt> {
   TargetLibraryInfo &TLI;
   bool Changed;
   std::vector<MemOp> WorkList;
+  // FIXME: These are to be removed after switching to the new memop value
+  // profiling.
   // Start of the previse range.
   int64_t PreciseRangeStart;
   // Last value of the previse range.
@@ -277,6 +289,8 @@ class MemOPSizeOpt : public InstVisitor<MemOPSizeOpt> {
   std::unique_ptr<InstrProfValueData[]> ValueDataArray;
   bool perform(MemOp MO);
 
+  // FIXME: This is to be removed after switching to the new memop value
+  // profiling.
   // This kind shows which group the value falls in. For PreciseValue, we have
   // the profile count for that value. LargeGroup groups the values that are in
   // range [LargeValue, +inf). NonLargeGroup groups the rest of values.
@@ -365,8 +379,11 @@ bool MemOPSizeOpt::perform(MemOp MO) {
     if (MemOPScaleCount)
       C = getScaledCount(C, ActualCount, SavedTotalCount);
 
-    // Only care precise value here.
-    if (getMemOPSizeKind(V) != PreciseValue)
+    if (UseOldMemOpValueProf) {
+      // Only care precise value here.
+      if (getMemOPSizeKind(V) != PreciseValue)
+        continue;
+    } else if (!InstrProfIsSingleValRange(V) || V > MemOpMaxOptSize)
       continue;
 
     // ValueCounts are sorted on the count. Break at the first un-profitable
diff --git a/llvm/test/Transforms/PGOProfile/memcpy.ll b/llvm/test/Transforms/PGOProfile/memcpy.ll
index 6047c95e7c084..e00e1d350871d 100644
--- a/llvm/test/Transforms/PGOProfile/memcpy.ll
+++ b/llvm/test/Transforms/PGOProfile/memcpy.ll
@@ -1,5 +1,7 @@
-; RUN: opt < %s -pgo-instr-gen -instrprof -S | FileCheck %s
-; RUN: opt <%s -passes=pgo-instr-gen,instrprof -S | FileCheck %s
+; RUN: opt < %s -pgo-instr-gen -instrprof -use-old-memop-value-prof=true -S | FileCheck %s --check-prefix=OLDMEMOPVP
+; RUN: opt < %s -pgo-instr-gen -instrprof -use-old-memop-value-prof=false -S | FileCheck %s --check-prefix=NEWMEMOPVP
+; RUN: opt <%s -passes=pgo-instr-gen,instrprof -use-old-memop-value-prof=true -S | FileCheck %s --check-prefix=OLDMEMOPVP
+; RUN: opt <%s -passes=pgo-instr-gen,instrprof -use-old-memop-value-prof=false -S | FileCheck %s --check-prefix=NEWMEMOPVP
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
@@ -23,7 +25,8 @@ for.cond1:
 
 for.body3:
   %conv = sext i32 %add to i64
-; CHECK: call void @__llvm_profile_instrument_range(i64 %conv, i8* bitcast ({ i64, i64, i64*, i8*, i8*, i32, [2 x i16] }* @__profd_foo to i8*), i32 0, i64 0, i64 8, i64 8192)
+; OLDMEMOPVP: call void @__llvm_profile_instrument_range(i64 %conv, i8* bitcast ({ i64, i64, i64*, i8*, i8*, i32, [2 x i16] }* @__profd_foo to i8*), i32 0, i64 0, i64 8, i64 8192)
+; NEWMEMOPVP: call void @__llvm_profile_instrument_memop(i64 %conv, i8* bitcast ({ i64, i64, i64*, i8*, i8*, i32, [2 x i16] }* @__profd_foo to i8*), i32 0)
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 %conv, i1 false)
   %inc = add nsw i32 %j.0, 1
   br label %for.cond1
diff --git a/llvm/test/Transforms/PGOProfile/memop_profile_funclet.ll b/llvm/test/Transforms/PGOProfile/memop_profile_funclet.ll
index b79431e3128e8..43c85ed25baae 100644
--- a/llvm/test/Transforms/PGOProfile/memop_profile_funclet.ll
+++ b/llvm/test/Transforms/PGOProfile/memop_profile_funclet.ll
@@ -1,8 +1,10 @@
 ; RUN: opt < %s -pgo-instr-gen -S | FileCheck %s --check-prefix=GEN
-; RUN: opt < %s -pgo-instr-gen -instrprof -S | FileCheck %s --check-prefix=LOWER
+; RUN: opt < %s -pgo-instr-gen -instrprof -use-old-memop-value-prof=true -S | FileCheck %s --check-prefixes=LOWER,LOWEROLDMEMOPVP
+; RUN: opt < %s -pgo-instr-gen -instrprof -use-old-memop-value-prof=false -S | FileCheck %s --check-prefixes=LOWER,LOWERNEWMEMOPVP
 
 ; RUN: opt < %s -passes=pgo-instr-gen -S | FileCheck %s --check-prefix=GEN
-; RUN: opt < %s -passes=pgo-instr-gen,instrprof -S | FileCheck %s --check-prefix=LOWER
+; RUN: opt < %s -passes=pgo-instr-gen,instrprof -use-old-memop-value-prof=true -S | FileCheck %s --check-prefixes=LOWER,LOWEROLDMEMOPVP
+; RUN: opt < %s -passes=pgo-instr-gen,instrprof -use-old-memop-value-prof=false -S | FileCheck %s --check-prefixes=LOWER,LOWERNEWMEMOPVP
 
 ; This test is to verify that PGO runtime library calls get created with the
 ; appropriate operand bundle funclet information when a memory intrinsic
@@ -63,7 +65,8 @@ try.cont:                                         ; preds = %entry
 ; GEN-SAME: [ "funclet"(token %tmp1) ]
 
 ; LOWER: catch:
-; LOWER: call void @__llvm_profile_instrument_range(
+; LOWEROLDMEMOPVP: call void @__llvm_profile_instrument_range(
+; LOWERNEWMEMOPVP: call void @__llvm_profile_instrument_memop(
 ; LOWER-SAME: [ "funclet"(token %tmp1) ]
 
 declare dso_local void @"?may_throw@@YAXH@Z"(i32)
diff --git a/llvm/unittests/ProfileData/CMakeLists.txt b/llvm/unittests/ProfileData/CMakeLists.txt
index 366ed5482bf2c..00a0079e675a8 100644
--- a/llvm/unittests/ProfileData/CMakeLists.txt
+++ b/llvm/unittests/ProfileData/CMakeLists.txt
@@ -7,6 +7,7 @@ set(LLVM_LINK_COMPONENTS
 
 add_llvm_unittest(ProfileDataTests
   CoverageMappingTest.cpp
+  InstrProfDataTest.cpp
   InstrProfTest.cpp
   SampleProfTest.cpp
   )
diff --git a/llvm/unittests/ProfileData/InstrProfDataTest.cpp b/llvm/unittests/ProfileData/InstrProfDataTest.cpp
new file mode 100644
index 0000000000000..af1a3de0657c2
--- /dev/null
+++ b/llvm/unittests/ProfileData/InstrProfDataTest.cpp
@@ -0,0 +1,68 @@
+//===- unittest/ProfileData/InstProfDataTest.cpp ----------------------------=//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "gtest/gtest.h"
+
+#include <cstdint>
+
+#define INSTR_PROF_VALUE_PROF_MEMOP_API
+#include "llvm/ProfileData/InstrProfData.inc"
+
+namespace {
+
+TEST(InstrProfDataTest, MapValueToRangeRepValue) {
+  EXPECT_EQ(0ULL, InstrProfGetRangeRepValue(0));
+  EXPECT_EQ(1ULL, InstrProfGetRangeRepValue(1));
+  EXPECT_EQ(2ULL, InstrProfGetRangeRepValue(2));
+  EXPECT_EQ(3ULL, InstrProfGetRangeRepValue(3));
+  EXPECT_EQ(4ULL, InstrProfGetRangeRepValue(4));
+  EXPECT_EQ(5ULL, InstrProfGetRangeRepValue(5));
+  EXPECT_EQ(6ULL, InstrProfGetRangeRepValue(6));
+  EXPECT_EQ(7ULL, InstrProfGetRangeRepValue(7));
+  EXPECT_EQ(8ULL, InstrProfGetRangeRepValue(8));
+  EXPECT_EQ(9ULL, InstrProfGetRangeRepValue(9));
+  EXPECT_EQ(16ULL, InstrProfGetRangeRepValue(16));
+  EXPECT_EQ(17ULL, InstrProfGetRangeRepValue(30));
+  EXPECT_EQ(32ULL, InstrProfGetRangeRepValue(32));
+  EXPECT_EQ(33ULL, InstrProfGetRangeRepValue(54));
+  EXPECT_EQ(64ULL, InstrProfGetRangeRepValue(64));
+  EXPECT_EQ(65ULL, InstrProfGetRangeRepValue(127));
+  EXPECT_EQ(128ULL, InstrProfGetRangeRepValue(128));
+  EXPECT_EQ(129ULL, InstrProfGetRangeRepValue(200));
+  EXPECT_EQ(256ULL, InstrProfGetRangeRepValue(256));
+  EXPECT_EQ(257ULL, InstrProfGetRangeRepValue(397));
+  EXPECT_EQ(512ULL, InstrProfGetRangeRepValue(512));
+  EXPECT_EQ(513ULL, InstrProfGetRangeRepValue(2832048023));
+}
+
+TEST(InstrProfDataTest, IsInOneValueRange) {
+  EXPECT_EQ(true, InstrProfIsSingleValRange(0));
+  EXPECT_EQ(true, InstrProfIsSingleValRange(1));
+  EXPECT_EQ(true, InstrProfIsSingleValRange(2));
+  EXPECT_EQ(true, InstrProfIsSingleValRange(3));
+  EXPECT_EQ(true, InstrProfIsSingleValRange(4));
+  EXPECT_EQ(true, InstrProfIsSingleValRange(5));
+  EXPECT_EQ(true, InstrProfIsSingleValRange(6));
+  EXPECT_EQ(true, InstrProfIsSingleValRange(7));
+  EXPECT_EQ(true, InstrProfIsSingleValRange(8));
+  EXPECT_EQ(false, InstrProfIsSingleValRange(9));
+  EXPECT_EQ(true, InstrProfIsSingleValRange(16));
+  EXPECT_EQ(false, InstrProfIsSingleValRange(30));
+  EXPECT_EQ(true, InstrProfIsSingleValRange(32));
+  EXPECT_EQ(false, InstrProfIsSingleValRange(54));
+  EXPECT_EQ(true, InstrProfIsSingleValRange(64));
+  EXPECT_EQ(false, InstrProfIsSingleValRange(127));
+  EXPECT_EQ(true, InstrProfIsSingleValRange(128));
+  EXPECT_EQ(false, InstrProfIsSingleValRange(200));
+  EXPECT_EQ(true, InstrProfIsSingleValRange(256));
+  EXPECT_EQ(false, InstrProfIsSingleValRange(397));
+  EXPECT_EQ(true, InstrProfIsSingleValRange(512));
+  EXPECT_EQ(false, InstrProfIsSingleValRange(2832048023344));
+}
+
+} // end anonymous namespace

From c12bd8dac91adac81cd9721fe34daf473ebd5e10 Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
Date: Mon, 3 Aug 2020 18:05:15 +0000
Subject: [PATCH 0937/1035] [gn build] Port f78f509c758

---
 llvm/utils/gn/secondary/llvm/unittests/ProfileData/BUILD.gn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/utils/gn/secondary/llvm/unittests/ProfileData/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/ProfileData/BUILD.gn
index e933b510003cd..3818b0caad04b 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/ProfileData/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/ProfileData/BUILD.gn
@@ -9,6 +9,7 @@ unittest("ProfileDataTests") {
   ]
   sources = [
     "CoverageMappingTest.cpp",
+    "InstrProfDataTest.cpp",
     "InstrProfTest.cpp",
     "SampleProfTest.cpp",
   ]

From 1e392fc44584a4909b4dced02b8386b48963002b Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Mon, 3 Aug 2020 19:18:13 +0100
Subject: [PATCH 0938/1035] [ArgPromotion] Replace all md uses of promoted
 values with undef.

Currently, ArgPromotion may leave metadata uses of promoted values,
which will end up in the wrong function, creating invalid IR.

PR33641 fixed this for dead arguments, but it can be also be triggered
arguments with users that are promoted (see the updated test case).

We also have to drop uses to them after promoting them. We need to do
this after dealing with the non-metadata uses, so I also moved the empty
use case to the loop that deals with updating the arguments of the new
function.

Reviewed By: aprantl

Differential Revision: https://reviews.llvm.org/D85127
---
 llvm/lib/Transforms/IPO/ArgumentPromotion.cpp | 11 ++--
 .../pr33641_remove_arg_dbgvalue.ll            | 50 +++++++++++++++++++
 2 files changed, 56 insertions(+), 5 deletions(-)

diff --git a/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp b/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp
index ad0d7eb51507a..d511ad2729abc 100644
--- a/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp
+++ b/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp
@@ -33,6 +33,7 @@
 #include "llvm/ADT/None.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/ScopeExit.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
@@ -153,10 +154,6 @@ doPromotion(Function *F, SmallPtrSetImpl<Argument *> &ArgsToPromote,
     } else if (I->use_empty()) {
       // Dead argument (which are always marked as promotable)
       ++NumArgumentsDead;
-
-      // There may be remaining metadata uses of the argument for things like
-      // llvm.dbg.value. Replace them with undef.
-      I->replaceAllUsesWith(UndefValue::get(I->getType()));
     } else {
       // Okay, this is being promoted. This means that the only uses are loads
       // or GEPs which are only used by loads
@@ -414,6 +411,11 @@ doPromotion(Function *F, SmallPtrSetImpl<Argument *> &ArgsToPromote,
       continue;
     }
 
+    // There potentially are metadata uses for things like llvm.dbg.value.
+    // Replace them with undef, after handling the other regular uses.
+    auto RauwUndefMetadata = make_scope_exit(
+        [&]() { I->replaceAllUsesWith(UndefValue::get(I->getType())); });
+
     if (I->use_empty())
       continue;
 
@@ -465,7 +467,6 @@ doPromotion(Function *F, SmallPtrSetImpl<Argument *> &ArgsToPromote,
         GEP->eraseFromParent();
       }
     }
-
     // Increment I2 past all of the arguments added for this promoted pointer.
     std::advance(I2, ArgIndices.size());
   }
diff --git a/llvm/test/Transforms/ArgumentPromotion/pr33641_remove_arg_dbgvalue.ll b/llvm/test/Transforms/ArgumentPromotion/pr33641_remove_arg_dbgvalue.ll
index 4c2503d8b0ccc..8dcc5b8c4f137 100644
--- a/llvm/test/Transforms/ArgumentPromotion/pr33641_remove_arg_dbgvalue.ll
+++ b/llvm/test/Transforms/ArgumentPromotion/pr33641_remove_arg_dbgvalue.ll
@@ -30,6 +30,52 @@ define internal void @bar(%p_t %p)  {
 
 declare void @llvm.dbg.value(metadata, metadata, metadata)
 
+
+; Test case where the promoted argument has uses in @callee and we need to
+; retain a reference to the original function, because it is stored in @storer.
+define void @storer({i32, i32}* %ptr) {
+; CHECK-LABEL: define {{[^@]+}}@storer
+; CHECK-SAME: ({ i32, i32 }* [[PTR:%.*]])
+; CHECK-NEXT:    ret void
+;
+  %tmp = alloca i32 ({i32, i32}*)*
+  store i32 ({i32, i32}*)* @callee,  i32 ({i32, i32}*)** %tmp
+  ret void
+}
+
+define i32 @caller() {
+; CHECK-LABEL: define {{[^@]+}}@caller()
+; CHECK-NEXT:    [[TMP:%.*]] = alloca { i32, i32 }, align 8
+; CHECK-NEXT:    [[F_1:%.*]] = getelementptr { i32, i32 }, { i32, i32 }* [[TMP]], i32 0, i32 1
+; CHECK-NEXT:    store i32 10, i32* [[F_1]], align 4
+; CHECK-NEXT:    [[TMP_IDX:%.*]] = getelementptr { i32, i32 }, { i32, i32 }* [[TMP]], i64 0, i32 1
+; CHECK-NEXT:    [[TMP_IDX_VAL:%.*]] = load i32, i32* [[TMP_IDX]], align 4
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @callee(i32 [[TMP_IDX_VAL]])
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %tmp = alloca {i32, i32}
+  %f.1 = getelementptr {i32, i32}, {i32, i32}* %tmp, i32 0, i32 1
+  store i32 10, i32* %f.1
+  %res = call i32 @callee({i32, i32}* %tmp)
+  ret i32 %res
+}
+
+define internal i32 @callee({i32, i32}* %ptr)  !dbg !7 {
+; CHECK-LABEL: define {{[^@]+}}@callee
+; CHECK-SAME: (i32 [[PTR_0_1_VAL:%.*]]) !dbg !6
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata { i32, i32 }* undef, metadata !7, metadata !DIExpression()), !dbg !8
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata i32 [[PTR_0_1_VAL]], metadata !7, metadata !DIExpression()), !dbg !8
+; CHECK-NEXT:    ret i32 [[PTR_0_1_VAL]]
+;
+  call void @llvm.dbg.value(metadata {i32, i32}* %ptr, metadata !8, metadata !9), !dbg !10
+  %f.1 = getelementptr {i32, i32}, {i32, i32}* %ptr, i32 0, i32 1
+  %l.1 = load i32, i32* %f.1
+  call void @llvm.dbg.value(metadata i32 %l.1, metadata !8, metadata !9), !dbg !10
+  ret i32 %l.1
+}
+
+
+
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!2}
 
@@ -40,3 +86,7 @@ declare void @llvm.dbg.value(metadata, metadata, metadata)
 !4 = !DILocalVariable(name: "p", scope: !3)
 !5 = !DIExpression()
 !6 = !DILocation(line: 1, column: 1, scope: !3)
+!7 = distinct !DISubprogram(name: "callee", unit: !0)
+!8 = !DILocalVariable(name: "c", scope: !7)
+!9 = !DIExpression()
+!10 = !DILocation(line: 2, column: 2, scope: !7)

From c6334db577e7049fe4868b1647c9f937f68ff1f5 Mon Sep 17 00:00:00 2001
From: Jian Cai <caij2003@gmail.com>
Date: Thu, 30 Jul 2020 18:33:33 -0700
Subject: [PATCH 0939/1035] [X86] support .nops directive

Add support of .nops on X86. This addresses llvm.org/PR45788.

Reviewed By: craig.topper

Differential Revision: https://reviews.llvm.org/D82826
---
 llvm/include/llvm/MC/MCAsmBackend.h           |  4 ++
 llvm/include/llvm/MC/MCFragment.h             | 26 +++++++++++
 llvm/include/llvm/MC/MCObjectStreamer.h       |  2 +
 llvm/include/llvm/MC/MCStreamer.h             |  3 ++
 llvm/lib/MC/MCAssembler.cpp                   | 46 ++++++++++++++++++-
 llvm/lib/MC/MCFragment.cpp                    | 12 +++++
 llvm/lib/MC/MCObjectStreamer.cpp              | 10 ++++
 llvm/lib/MC/MCStreamer.cpp                    |  3 ++
 .../lib/Target/X86/AsmParser/X86AsmParser.cpp | 41 ++++++++++++++++-
 .../Target/X86/MCTargetDesc/X86AsmBackend.cpp | 35 +++++++-------
 llvm/test/MC/X86/x86-directive-nops-errors.s  | 12 +++++
 llvm/test/MC/X86/x86-directive-nops.s         | 12 +++++
 llvm/test/MC/X86/x86_64-directive-nops.s      | 19 ++++++++
 13 files changed, 205 insertions(+), 20 deletions(-)
 create mode 100644 llvm/test/MC/X86/x86-directive-nops-errors.s
 create mode 100644 llvm/test/MC/X86/x86-directive-nops.s
 create mode 100644 llvm/test/MC/X86/x86_64-directive-nops.s

diff --git a/llvm/include/llvm/MC/MCAsmBackend.h b/llvm/include/llvm/MC/MCAsmBackend.h
index 8f95cfd55a3d7..94ed3d27e7859 100644
--- a/llvm/include/llvm/MC/MCAsmBackend.h
+++ b/llvm/include/llvm/MC/MCAsmBackend.h
@@ -177,6 +177,10 @@ class MCAsmBackend {
   ///
   virtual unsigned getMinimumNopSize() const { return 1; }
 
+  /// Returns the maximum size of a nop in bytes on this target.
+  ///
+  virtual unsigned getMaximumNopSize() const { return 0; }
+
   /// Write an (optimal) nop sequence of Count bytes to the given output. If the
   /// target cannot generate such a sequence, it should return an error.
   ///
diff --git a/llvm/include/llvm/MC/MCFragment.h b/llvm/include/llvm/MC/MCFragment.h
index fb7166e82c098..87338ab46cc2a 100644
--- a/llvm/include/llvm/MC/MCFragment.h
+++ b/llvm/include/llvm/MC/MCFragment.h
@@ -37,6 +37,7 @@ class MCFragment : public ilist_node_with_parent<MCFragment, MCSection> {
     FT_Data,
     FT_CompactEncodedInst,
     FT_Fill,
+    FT_Nops,
     FT_Relaxable,
     FT_Org,
     FT_Dwarf,
@@ -350,6 +351,31 @@ class MCFillFragment : public MCFragment {
   }
 };
 
+class MCNopsFragment : public MCFragment {
+  /// The number of bytes to insert.
+  int64_t Size;
+  /// Maximum number of bytes allowed in each NOP instruction.
+  int64_t ControlledNopLength;
+
+  /// Source location of the directive that this fragment was created for.
+  SMLoc Loc;
+
+public:
+  MCNopsFragment(int64_t NumBytes, int64_t ControlledNopLength, SMLoc L,
+                 MCSection *Sec = nullptr)
+      : MCFragment(FT_Nops, false, Sec), Size(NumBytes),
+        ControlledNopLength(ControlledNopLength), Loc(L) {}
+
+  int64_t getNumBytes() const { return Size; }
+  int64_t getControlledNopLength() const { return ControlledNopLength; }
+
+  SMLoc getLoc() const { return Loc; }
+
+  static bool classof(const MCFragment *F) {
+    return F->getKind() == MCFragment::FT_Nops;
+  }
+};
+
 class MCOrgFragment : public MCFragment {
   /// Value to use for filling bytes.
   int8_t Value;
diff --git a/llvm/include/llvm/MC/MCObjectStreamer.h b/llvm/include/llvm/MC/MCObjectStreamer.h
index c3f3ae5de921e..a00000bc11b60 100644
--- a/llvm/include/llvm/MC/MCObjectStreamer.h
+++ b/llvm/include/llvm/MC/MCObjectStreamer.h
@@ -179,6 +179,8 @@ class MCObjectStreamer : public MCStreamer {
                 SMLoc Loc = SMLoc()) override;
   void emitFill(const MCExpr &NumValues, int64_t Size, int64_t Expr,
                 SMLoc Loc = SMLoc()) override;
+  void emitNops(int64_t NumBytes, int64_t ControlledNopLength,
+                SMLoc Loc) override;
   void emitFileDirective(StringRef Filename) override;
 
   void emitAddrsig() override;
diff --git a/llvm/include/llvm/MC/MCStreamer.h b/llvm/include/llvm/MC/MCStreamer.h
index 484c62538366e..63a4c1d190aca 100644
--- a/llvm/include/llvm/MC/MCStreamer.h
+++ b/llvm/include/llvm/MC/MCStreamer.h
@@ -777,6 +777,9 @@ class MCStreamer {
   virtual void emitFill(const MCExpr &NumValues, int64_t Size, int64_t Expr,
                         SMLoc Loc = SMLoc());
 
+  virtual void emitNops(int64_t NumBytes, int64_t ControlledNopLength,
+                        SMLoc Loc);
+
   /// Emit NumBytes worth of zeros.
   /// This function properly handles data in virtual sections.
   void emitZeros(uint64_t NumBytes);
diff --git a/llvm/lib/MC/MCAssembler.cpp b/llvm/lib/MC/MCAssembler.cpp
index 3ca8714b7817c..9515b7e2642bc 100644
--- a/llvm/lib/MC/MCAssembler.cpp
+++ b/llvm/lib/MC/MCAssembler.cpp
@@ -62,8 +62,8 @@ STATISTIC(EmittedAlignFragments,
           "Number of emitted assembler fragments - align");
 STATISTIC(EmittedFillFragments,
           "Number of emitted assembler fragments - fill");
-STATISTIC(EmittedOrgFragments,
-          "Number of emitted assembler fragments - org");
+STATISTIC(EmittedNopsFragments, "Number of emitted assembler fragments - nops");
+STATISTIC(EmittedOrgFragments, "Number of emitted assembler fragments - org");
 STATISTIC(evaluateFixup, "Number of evaluated fixups");
 STATISTIC(FragmentLayouts, "Number of fragment layouts");
 STATISTIC(ObjectBytes, "Number of emitted object file bytes");
@@ -312,6 +312,9 @@ uint64_t MCAssembler::computeFragmentSize(const MCAsmLayout &Layout,
     return Size;
   }
 
+  case MCFragment::FT_Nops:
+    return cast<MCNopsFragment>(F).getNumBytes();
+
   case MCFragment::FT_LEB:
     return cast<MCLEBFragment>(F).getContents().size();
 
@@ -613,6 +616,45 @@ static void writeFragment(raw_ostream &OS, const MCAssembler &Asm,
     break;
   }
 
+  case MCFragment::FT_Nops: {
+    ++stats::EmittedNopsFragments;
+    const MCNopsFragment &NF = cast<MCNopsFragment>(F);
+    int64_t NumBytes = NF.getNumBytes();
+    int64_t ControlledNopLength = NF.getControlledNopLength();
+    int64_t MaximumNopLength = Asm.getBackend().getMaximumNopSize();
+
+    assert(NumBytes > 0 && "Expected positive NOPs fragment size");
+    assert(ControlledNopLength >= 0 && "Expected non-negative NOP size");
+
+    if (ControlledNopLength > MaximumNopLength) {
+      Asm.getContext().reportError(NF.getLoc(),
+                                   "illegal NOP size " +
+                                       std::to_string(ControlledNopLength) +
+                                       ". (expected within [0, " +
+                                       std::to_string(MaximumNopLength) + "])");
+      // Clamp the NOP length as reportError does not stop the execution
+      // immediately.
+      ControlledNopLength = MaximumNopLength;
+    }
+
+    // Use maximum value if the size of each NOP is not specified
+    if (!ControlledNopLength)
+      ControlledNopLength = MaximumNopLength;
+
+    while (NumBytes) {
+      uint64_t NumBytesToEmit =
+          (uint64_t)std::min(NumBytes, ControlledNopLength);
+      assert(NumBytesToEmit && "try to emit empty NOP instruction");
+      if (!Asm.getBackend().writeNopData(OS, NumBytesToEmit)) {
+        report_fatal_error("unable to write nop sequence of the remaining " +
+                           Twine(NumBytesToEmit) + " bytes");
+        break;
+      }
+      NumBytes -= NumBytesToEmit;
+    }
+    break;
+  }
+
   case MCFragment::FT_LEB: {
     const MCLEBFragment &LF = cast<MCLEBFragment>(F);
     OS << LF.getContents();
diff --git a/llvm/lib/MC/MCFragment.cpp b/llvm/lib/MC/MCFragment.cpp
index 8e90e07a4dbfd..e9cea9d18f2e1 100644
--- a/llvm/lib/MC/MCFragment.cpp
+++ b/llvm/lib/MC/MCFragment.cpp
@@ -279,6 +279,9 @@ void MCFragment::destroy() {
     case FT_Fill:
       delete cast<MCFillFragment>(this);
       return;
+    case FT_Nops:
+      delete cast<MCNopsFragment>(this);
+      return;
     case FT_Relaxable:
       delete cast<MCRelaxableFragment>(this);
       return;
@@ -336,6 +339,9 @@ LLVM_DUMP_METHOD void MCFragment::dump() const {
   case MCFragment::FT_CompactEncodedInst:
     OS << "MCCompactEncodedInstFragment"; break;
   case MCFragment::FT_Fill:  OS << "MCFillFragment"; break;
+  case MCFragment::FT_Nops:
+    OS << "MCFNopsFragment";
+    break;
   case MCFragment::FT_Relaxable:  OS << "MCRelaxableFragment"; break;
   case MCFragment::FT_Org:   OS << "MCOrgFragment"; break;
   case MCFragment::FT_Dwarf: OS << "MCDwarfFragment"; break;
@@ -408,6 +414,12 @@ LLVM_DUMP_METHOD void MCFragment::dump() const {
        << " NumValues:" << FF->getNumValues();
     break;
   }
+  case MCFragment::FT_Nops: {
+    const auto *NF = cast<MCNopsFragment>(this);
+    OS << " NumBytes:" << NF->getNumBytes()
+       << " ControlledNopLength:" << NF->getControlledNopLength();
+    break;
+  }
   case MCFragment::FT_Relaxable:  {
     const auto *F = cast<MCRelaxableFragment>(this);
     OS << "\n       ";
diff --git a/llvm/lib/MC/MCObjectStreamer.cpp b/llvm/lib/MC/MCObjectStreamer.cpp
index 78ee215b59aae..f9e0d858cf701 100644
--- a/llvm/lib/MC/MCObjectStreamer.cpp
+++ b/llvm/lib/MC/MCObjectStreamer.cpp
@@ -819,6 +819,16 @@ void MCObjectStreamer::emitFill(const MCExpr &NumValues, int64_t Size,
   insert(new MCFillFragment(Expr, Size, NumValues, Loc));
 }
 
+void MCObjectStreamer::emitNops(int64_t NumBytes, int64_t ControlledNopLength,
+                                SMLoc Loc) {
+  // Emit an NOP fragment.
+  MCDataFragment *DF = getOrCreateDataFragment();
+  flushPendingLabels(DF, DF->getContents().size());
+
+  assert(getCurrentSectionOnly() && "need a section");
+  insert(new MCNopsFragment(NumBytes, ControlledNopLength, Loc));
+}
+
 void MCObjectStreamer::emitFileDirective(StringRef Filename) {
   getAssembler().addFileName(Filename);
 }
diff --git a/llvm/lib/MC/MCStreamer.cpp b/llvm/lib/MC/MCStreamer.cpp
index 6d3a933c96a37..df08c343f69f7 100644
--- a/llvm/lib/MC/MCStreamer.cpp
+++ b/llvm/lib/MC/MCStreamer.cpp
@@ -202,6 +202,9 @@ void MCStreamer::emitFill(uint64_t NumBytes, uint8_t FillValue) {
   emitFill(*MCConstantExpr::create(NumBytes, getContext()), FillValue);
 }
 
+void llvm::MCStreamer::emitNops(int64_t NumBytes, int64_t ControlledNopLen,
+                                llvm::SMLoc) {}
+
 /// The implementation in this class just redirects to emitFill.
 void MCStreamer::emitZeros(uint64_t NumBytes) { emitFill(NumBytes, 0); }
 
diff --git a/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp b/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
index 49c01d7b9ef0b..ddb13e46e9305 100644
--- a/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
+++ b/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
@@ -934,6 +934,7 @@ class X86AsmParser : public MCTargetAsmParser {
                                OperandVector &Operands);
 
   bool parseDirectiveArch();
+  bool parseDirectiveNops(SMLoc L);
   bool parseDirectiveEven(SMLoc L);
   bool ParseDirectiveCode(StringRef IDVal, SMLoc L);
 
@@ -4037,7 +4038,9 @@ bool X86AsmParser::ParseDirective(AsmToken DirectiveID) {
                                            "a '%' prefix in .intel_syntax");
     }
     return false;
-  } else if (IDVal == ".even")
+  } else if (IDVal == ".nops")
+    return parseDirectiveNops(DirectiveID.getLoc());
+  else if (IDVal == ".even")
     return parseDirectiveEven(DirectiveID.getLoc());
   else if (IDVal == ".cv_fpo_proc")
     return parseDirectiveFPOProc(DirectiveID.getLoc());
@@ -4073,6 +4076,42 @@ bool X86AsmParser::parseDirectiveArch() {
   return false;
 }
 
+/// parseDirectiveNops
+///  ::= .nops size[, control]
+bool X86AsmParser::parseDirectiveNops(SMLoc L) {
+  int64_t NumBytes = 0, Control = 0;
+  SMLoc NumBytesLoc, ControlLoc;
+  const MCSubtargetInfo STI = getSTI();
+  NumBytesLoc = getTok().getLoc();
+  if (getParser().checkForValidSection() ||
+      getParser().parseAbsoluteExpression(NumBytes))
+    return true;
+
+  if (parseOptionalToken(AsmToken::Comma)) {
+    ControlLoc = getTok().getLoc();
+    if (getParser().parseAbsoluteExpression(Control))
+      return true;
+  }
+  if (getParser().parseToken(AsmToken::EndOfStatement,
+                             "unexpected token in '.nops' directive"))
+    return true;
+
+  if (NumBytes <= 0) {
+    Error(NumBytesLoc, "'.nops' directive with non-positive size");
+    return false;
+  }
+
+  if (Control < 0) {
+    Error(ControlLoc, "'.nops' directive with negative NOP size");
+    return false;
+  }
+
+  /// Emit nops
+  getParser().getStreamer().emitNops(NumBytes, Control, L);
+
+  return false;
+}
+
 /// parseDirectiveEven
 ///  ::= .even
 bool X86AsmParser::parseDirectiveEven(SMLoc L) {
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
index bf3b6bcb5463f..31bc54f53d027 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
@@ -207,6 +207,8 @@ class X86AsmBackend : public MCAsmBackend {
 
   void finishLayout(MCAssembler const &Asm, MCAsmLayout &Layout) const override;
 
+  unsigned getMaximumNopSize() const override;
+
   bool writeNopData(raw_ostream &OS, uint64_t Count) const override;
 };
 } // end anonymous namespace
@@ -1067,6 +1069,21 @@ void X86AsmBackend::finishLayout(MCAssembler const &Asm,
   }
 }
 
+unsigned X86AsmBackend::getMaximumNopSize() const {
+  if (!STI.hasFeature(X86::FeatureNOPL) && !STI.hasFeature(X86::Mode64Bit))
+    return 1;
+  if (STI.getFeatureBits()[X86::FeatureFast7ByteNOP])
+    return 7;
+  if (STI.getFeatureBits()[X86::FeatureFast15ByteNOP])
+    return 15;
+  if (STI.getFeatureBits()[X86::FeatureFast11ByteNOP])
+    return 11;
+  // FIXME: handle 32-bit mode
+  // 15-bytes is the longest single NOP instruction, but 10-bytes is
+  // commonly the longest that can be efficiently decoded.
+  return 10;
+}
+
 /// Write a sequence of optimal nops to the output, covering \p Count
 /// bytes.
 /// \return - true on success, false on failure
@@ -1094,23 +1111,7 @@ bool X86AsmBackend::writeNopData(raw_ostream &OS, uint64_t Count) const {
     "\x66\x2e\x0f\x1f\x84\x00\x00\x00\x00\x00",
   };
 
-  // This CPU doesn't support long nops. If needed add more.
-  // FIXME: We could generated something better than plain 0x90.
-  if (!STI.hasFeature(X86::FeatureNOPL) && !STI.hasFeature(X86::Mode64Bit)) {
-    for (uint64_t i = 0; i < Count; ++i)
-      OS << '\x90';
-    return true;
-  }
-
-  // 15-bytes is the longest single NOP instruction, but 10-bytes is
-  // commonly the longest that can be efficiently decoded.
-  uint64_t MaxNopLength = 10;
-  if (STI.getFeatureBits()[X86::FeatureFast7ByteNOP])
-    MaxNopLength = 7;
-  else if (STI.getFeatureBits()[X86::FeatureFast15ByteNOP])
-    MaxNopLength = 15;
-  else if (STI.getFeatureBits()[X86::FeatureFast11ByteNOP])
-    MaxNopLength = 11;
+  uint64_t MaxNopLength = (uint64_t)getMaximumNopSize();
 
   // Emit as many MaxNopLength NOPs as needed, then emit a NOP of the remaining
   // length.
diff --git a/llvm/test/MC/X86/x86-directive-nops-errors.s b/llvm/test/MC/X86/x86-directive-nops-errors.s
new file mode 100644
index 0000000000000..473cb509442cd
--- /dev/null
+++ b/llvm/test/MC/X86/x86-directive-nops-errors.s
@@ -0,0 +1,12 @@
+# RUN: not llvm-mc -triple i386 %s -filetype=obj -o /dev/null 2>&1 | FileCheck --check-prefix=X86 %s
+# RUN: not llvm-mc -triple=x86_64 %s -filetype=obj -o /dev/null 2>&1 | FileCheck --check-prefix=X64 %s
+
+.nops 4, 3
+# X86: :[[@LINE-1]]:1: error: illegal NOP size 3.
+.nops 4, 4
+# X86: :[[@LINE-1]]:1: error: illegal NOP size 4.
+.nops 4, 5
+# X86: :[[@LINE-1]]:1: error: illegal NOP size 5.
+.nops 16, 15
+# X86: :[[@LINE-1]]:1: error: illegal NOP size 15.
+# X64: :[[@LINE-2]]:1: error: illegal NOP size 15.
diff --git a/llvm/test/MC/X86/x86-directive-nops.s b/llvm/test/MC/X86/x86-directive-nops.s
new file mode 100644
index 0000000000000..786a029d503ae
--- /dev/null
+++ b/llvm/test/MC/X86/x86-directive-nops.s
@@ -0,0 +1,12 @@
+# RUN: llvm-mc -triple i386 %s -filetype=obj | llvm-objdump -d - | FileCheck %s
+
+.nops 4
+# CHECK:       0: 90 nop
+# CHECK-NEXT:  1: 90 nop
+# CHECK-NEXT:  2: 90 nop
+# CHECK-NEXT:  3: 90 nop
+.nops 4, 1
+# CHECK:       4: 90 nop
+# CHECK-NEXT:  5: 90 nop
+# CHECK-NEXT:  6: 90 nop
+# CHECK-NEXT:  7: 90 nop
diff --git a/llvm/test/MC/X86/x86_64-directive-nops.s b/llvm/test/MC/X86/x86_64-directive-nops.s
new file mode 100644
index 0000000000000..2255cd3a2efdc
--- /dev/null
+++ b/llvm/test/MC/X86/x86_64-directive-nops.s
@@ -0,0 +1,19 @@
+# RUN: llvm-mc -triple=x86_64 %s -filetype=obj | llvm-objdump -d - | FileCheck %s
+
+.nops 4, 1
+# CHECK:       0: 90 nop
+# CHECK-NEXT:  1: 90 nop
+# CHECK-NEXT:  2: 90 nop
+# CHECK-NEXT:  3: 90 nop
+.nops 4, 2
+# CHECK-NEXT:  4: 66 90 nop
+# CHECK-NEXT:  6: 66 90 nop
+.nops 4, 3
+# CHECK-NEXT:  8: 0f 1f 00 nopl (%rax)
+# CHECK-NEXT:  b: 90 nop
+.nops 4, 4
+# CHECK-NEXT:  c: 0f 1f 40 00 nopl (%rax)
+.nops 4, 5
+# CHECK-NEXT:  10: 0f 1f 40 00 nopl (%rax)
+.nops 4
+# CHECK-NEXT:  14: 0f 1f 40 00 nopl (%rax)

From 456f38a97199770d4ea563dec8c50eaaf20f0309 Mon Sep 17 00:00:00 2001
From: Arthur Eubanks <aeubanks@google.com>
Date: Mon, 3 Aug 2020 11:50:07 -0700
Subject: [PATCH 0940/1035] Fix layering violation Transforms/Utils -> Scalar

Introduced in D85063.
---
 llvm/lib/Transforms/Utils/LoopVersioning.cpp | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/Transforms/Utils/LoopVersioning.cpp b/llvm/lib/Transforms/Utils/LoopVersioning.cpp
index c8fa337ed04f7..b4925064bc6b9 100644
--- a/llvm/lib/Transforms/Utils/LoopVersioning.cpp
+++ b/llvm/lib/Transforms/Utils/LoopVersioning.cpp
@@ -14,15 +14,17 @@
 
 #include "llvm/Transforms/Utils/LoopVersioning.h"
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Analysis/LoopAccessAnalysis.h"
 #include "llvm/Analysis/LoopInfo.h"
+#include "llvm/Analysis/MemorySSA.h"
 #include "llvm/Analysis/ScalarEvolution.h"
+#include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/MDBuilder.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/InitializePasses.h"
 #include "llvm/Support/CommandLine.h"
-#include "llvm/Transforms/Scalar/LoopPassManager.h"
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
 #include "llvm/Transforms/Utils/Cloning.h"
 #include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"

From 7efd9ceb588b5e76e4ce9ae0b8ed45bfc90645cd Mon Sep 17 00:00:00 2001
From: Sanjay Patel <spatel@rotateright.com>
Date: Mon, 3 Aug 2020 13:50:54 -0400
Subject: [PATCH 0941/1035] [InstSimplify] add tests for min-of-max variants;
 NFC

---
 .../InstSimplify/maxmin_intrinsics.ll         | 220 ++++++++++++++++++
 1 file changed, 220 insertions(+)

diff --git a/llvm/test/Transforms/InstSimplify/maxmin_intrinsics.ll b/llvm/test/Transforms/InstSimplify/maxmin_intrinsics.ll
index c0064ab0a423a..7c79357a0bd04 100644
--- a/llvm/test/Transforms/InstSimplify/maxmin_intrinsics.ll
+++ b/llvm/test/Transforms/InstSimplify/maxmin_intrinsics.ll
@@ -427,3 +427,223 @@ define i8 @smin_smin_commute3(i8 %x, i8 %y) {
   %m2 = call i8 @llvm.smin.i8(i8 %m, i8 %x)
   ret i8 %m2
 }
+
+define i8 @umax_umin(i8 %x, i8 %y) {
+; CHECK-LABEL: @umax_umin(
+; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.umax.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.umin.i8(i8 [[X]], i8 [[M]])
+; CHECK-NEXT:    ret i8 [[M2]]
+;
+  %m = call i8 @llvm.umax.i8(i8 %x, i8 %y)
+  %m2 = call i8 @llvm.umin.i8(i8 %x, i8 %m)
+  ret i8 %m2
+}
+
+define i8 @umax_umin_commute1(i8 %x, i8 %y) {
+; CHECK-LABEL: @umax_umin_commute1(
+; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.umax.i8(i8 [[Y:%.*]], i8 [[X:%.*]])
+; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.umin.i8(i8 [[X]], i8 [[M]])
+; CHECK-NEXT:    ret i8 [[M2]]
+;
+  %m = call i8 @llvm.umax.i8(i8 %y, i8 %x)
+  %m2 = call i8 @llvm.umin.i8(i8 %x, i8 %m)
+  ret i8 %m2
+}
+
+define i8 @umax_umin_commute2(i8 %x, i8 %y) {
+; CHECK-LABEL: @umax_umin_commute2(
+; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.umax.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.umin.i8(i8 [[M]], i8 [[X]])
+; CHECK-NEXT:    ret i8 [[M2]]
+;
+  %m = call i8 @llvm.umax.i8(i8 %x, i8 %y)
+  %m2 = call i8 @llvm.umin.i8(i8 %m, i8 %x)
+  ret i8 %m2
+}
+
+define <2 x i8> @umax_umin_commute3(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @umax_umin_commute3(
+; CHECK-NEXT:    [[M:%.*]] = call <2 x i8> @llvm.umax.v2i8(<2 x i8> [[Y:%.*]], <2 x i8> [[X:%.*]])
+; CHECK-NEXT:    [[M2:%.*]] = call <2 x i8> @llvm.umin.v2i8(<2 x i8> [[M]], <2 x i8> [[X]])
+; CHECK-NEXT:    ret <2 x i8> [[M2]]
+;
+  %m = call <2 x i8> @llvm.umax.v2i8(<2 x i8> %y, <2 x i8> %x)
+  %m2 = call <2 x i8> @llvm.umin.v2i8(<2 x i8> %m, <2 x i8> %x)
+  ret <2 x i8> %m2
+}
+
+define i8 @umin_umax(i8 %x, i8 %y) {
+; CHECK-LABEL: @umin_umax(
+; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.umin.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.umax.i8(i8 [[X]], i8 [[M]])
+; CHECK-NEXT:    ret i8 [[M2]]
+;
+  %m = call i8 @llvm.umin.i8(i8 %x, i8 %y)
+  %m2 = call i8 @llvm.umax.i8(i8 %x, i8 %m)
+  ret i8 %m2
+}
+
+define i8 @umin_umax_commute1(i8 %x, i8 %y) {
+; CHECK-LABEL: @umin_umax_commute1(
+; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.umin.i8(i8 [[Y:%.*]], i8 [[X:%.*]])
+; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.umax.i8(i8 [[X]], i8 [[M]])
+; CHECK-NEXT:    ret i8 [[M2]]
+;
+  %m = call i8 @llvm.umin.i8(i8 %y, i8 %x)
+  %m2 = call i8 @llvm.umax.i8(i8 %x, i8 %m)
+  ret i8 %m2
+}
+
+define <2 x i8> @umin_umax_commute2(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @umin_umax_commute2(
+; CHECK-NEXT:    [[M:%.*]] = call <2 x i8> @llvm.umin.v2i8(<2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]])
+; CHECK-NEXT:    [[M2:%.*]] = call <2 x i8> @llvm.umax.v2i8(<2 x i8> [[M]], <2 x i8> [[X]])
+; CHECK-NEXT:    ret <2 x i8> [[M2]]
+;
+  %m = call <2 x i8> @llvm.umin.v2i8(<2 x i8> %x, <2 x i8> %y)
+  %m2 = call <2 x i8> @llvm.umax.v2i8(<2 x i8> %m, <2 x i8> %x)
+  ret <2 x i8> %m2
+}
+
+define i8 @umin_umax_commute3(i8 %x, i8 %y) {
+; CHECK-LABEL: @umin_umax_commute3(
+; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.umin.i8(i8 [[Y:%.*]], i8 [[X:%.*]])
+; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.umax.i8(i8 [[M]], i8 [[X]])
+; CHECK-NEXT:    ret i8 [[M2]]
+;
+  %m = call i8 @llvm.umin.i8(i8 %y, i8 %x)
+  %m2 = call i8 @llvm.umax.i8(i8 %m, i8 %x)
+  ret i8 %m2
+}
+
+define i8 @smax_smin(i8 %x, i8 %y) {
+; CHECK-LABEL: @smax_smin(
+; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.smax.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.smin.i8(i8 [[X]], i8 [[M]])
+; CHECK-NEXT:    ret i8 [[M2]]
+;
+  %m = call i8 @llvm.smax.i8(i8 %x, i8 %y)
+  %m2 = call i8 @llvm.smin.i8(i8 %x, i8 %m)
+  ret i8 %m2
+}
+
+define <2 x i8> @smax_smin_commute1(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @smax_smin_commute1(
+; CHECK-NEXT:    [[M:%.*]] = call <2 x i8> @llvm.smax.v2i8(<2 x i8> [[Y:%.*]], <2 x i8> [[X:%.*]])
+; CHECK-NEXT:    [[M2:%.*]] = call <2 x i8> @llvm.smin.v2i8(<2 x i8> [[X]], <2 x i8> [[M]])
+; CHECK-NEXT:    ret <2 x i8> [[M2]]
+;
+  %m = call <2 x i8> @llvm.smax.v2i8(<2 x i8> %y, <2 x i8> %x)
+  %m2 = call <2 x i8> @llvm.smin.v2i8(<2 x i8> %x, <2 x i8> %m)
+  ret <2 x i8> %m2
+}
+
+define i8 @smax_smin_commute2(i8 %x, i8 %y) {
+; CHECK-LABEL: @smax_smin_commute2(
+; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.smax.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.smin.i8(i8 [[M]], i8 [[X]])
+; CHECK-NEXT:    ret i8 [[M2]]
+;
+  %m = call i8 @llvm.smax.i8(i8 %x, i8 %y)
+  %m2 = call i8 @llvm.smin.i8(i8 %m, i8 %x)
+  ret i8 %m2
+}
+
+define i8 @smax_smin_commute3(i8 %x, i8 %y) {
+; CHECK-LABEL: @smax_smin_commute3(
+; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.smax.i8(i8 [[Y:%.*]], i8 [[X:%.*]])
+; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.smin.i8(i8 [[M]], i8 [[X]])
+; CHECK-NEXT:    ret i8 [[M2]]
+;
+  %m = call i8 @llvm.smax.i8(i8 %y, i8 %x)
+  %m2 = call i8 @llvm.smin.i8(i8 %m, i8 %x)
+  ret i8 %m2
+}
+
+define <2 x i8> @smin_smax(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @smin_smax(
+; CHECK-NEXT:    [[M:%.*]] = call <2 x i8> @llvm.smin.v2i8(<2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]])
+; CHECK-NEXT:    [[M2:%.*]] = call <2 x i8> @llvm.smax.v2i8(<2 x i8> [[X]], <2 x i8> [[M]])
+; CHECK-NEXT:    ret <2 x i8> [[M2]]
+;
+  %m = call <2 x i8> @llvm.smin.v2i8(<2 x i8> %x, <2 x i8> %y)
+  %m2 = call <2 x i8> @llvm.smax.v2i8(<2 x i8> %x, <2 x i8> %m)
+  ret <2 x i8> %m2
+}
+
+define i8 @smin_smax_commute1(i8 %x, i8 %y) {
+; CHECK-LABEL: @smin_smax_commute1(
+; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.smin.i8(i8 [[Y:%.*]], i8 [[X:%.*]])
+; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.smax.i8(i8 [[X]], i8 [[M]])
+; CHECK-NEXT:    ret i8 [[M2]]
+;
+  %m = call i8 @llvm.smin.i8(i8 %y, i8 %x)
+  %m2 = call i8 @llvm.smax.i8(i8 %x, i8 %m)
+  ret i8 %m2
+}
+
+define i8 @smin_smax_commute2(i8 %x, i8 %y) {
+; CHECK-LABEL: @smin_smax_commute2(
+; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.smin.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.smax.i8(i8 [[M]], i8 [[X]])
+; CHECK-NEXT:    ret i8 [[M2]]
+;
+  %m = call i8 @llvm.smin.i8(i8 %x, i8 %y)
+  %m2 = call i8 @llvm.smax.i8(i8 %m, i8 %x)
+  ret i8 %m2
+}
+
+define i8 @smin_smax_commute3(i8 %x, i8 %y) {
+; CHECK-LABEL: @smin_smax_commute3(
+; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.smin.i8(i8 [[Y:%.*]], i8 [[X:%.*]])
+; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.smax.i8(i8 [[M]], i8 [[X]])
+; CHECK-NEXT:    ret i8 [[M2]]
+;
+  %m = call i8 @llvm.smin.i8(i8 %y, i8 %x)
+  %m2 = call i8 @llvm.smax.i8(i8 %m, i8 %x)
+  ret i8 %m2
+}
+
+define i8 @smax_umin(i8 %x, i8 %y) {
+; CHECK-LABEL: @smax_umin(
+; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.smax.i8(i8 [[Y:%.*]], i8 [[X:%.*]])
+; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.umin.i8(i8 [[M]], i8 [[X]])
+; CHECK-NEXT:    ret i8 [[M2]]
+;
+  %m = call i8 @llvm.smax.i8(i8 %y, i8 %x)
+  %m2 = call i8 @llvm.umin.i8(i8 %m, i8 %x)
+  ret i8 %m2
+}
+
+define i8 @smax_umax(i8 %x, i8 %y) {
+; CHECK-LABEL: @smax_umax(
+; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.smax.i8(i8 [[Y:%.*]], i8 [[X:%.*]])
+; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.umax.i8(i8 [[M]], i8 [[X]])
+; CHECK-NEXT:    ret i8 [[M2]]
+;
+  %m = call i8 @llvm.smax.i8(i8 %y, i8 %x)
+  %m2 = call i8 @llvm.umax.i8(i8 %m, i8 %x)
+  ret i8 %m2
+}
+
+define i8 @umax_smin(i8 %x, i8 %y) {
+; CHECK-LABEL: @umax_smin(
+; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.umax.i8(i8 [[Y:%.*]], i8 [[X:%.*]])
+; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.smin.i8(i8 [[M]], i8 [[X]])
+; CHECK-NEXT:    ret i8 [[M2]]
+;
+  %m = call i8 @llvm.umax.i8(i8 %y, i8 %x)
+  %m2 = call i8 @llvm.smin.i8(i8 %m, i8 %x)
+  ret i8 %m2
+}
+
+define i8 @umin_smin(i8 %x, i8 %y) {
+; CHECK-LABEL: @umin_smin(
+; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.umin.i8(i8 [[Y:%.*]], i8 [[X:%.*]])
+; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.smin.i8(i8 [[M]], i8 [[X]])
+; CHECK-NEXT:    ret i8 [[M2]]
+;
+  %m = call i8 @llvm.umin.i8(i8 %y, i8 %x)
+  %m2 = call i8 @llvm.smin.i8(i8 %m, i8 %x)
+  ret i8 %m2
+}

From 9e5cf6bde5963f14a38117061c7a4df064453088 Mon Sep 17 00:00:00 2001
From: Sanjay Patel <spatel@rotateright.com>
Date: Mon, 3 Aug 2020 14:02:09 -0400
Subject: [PATCH 0942/1035] [InstSimplify] fold variations of max-of-min with
 common operand

https://alive2.llvm.org/ce/z/ZtxpZ3
---
 llvm/lib/Analysis/InstructionSimplify.cpp     | 33 ++++++++--
 .../InstSimplify/maxmin_intrinsics.ll         | 64 +++++--------------
 2 files changed, 43 insertions(+), 54 deletions(-)

diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp
index 6e75478d52afd..f827f0230a3e4 100644
--- a/llvm/lib/Analysis/InstructionSimplify.cpp
+++ b/llvm/lib/Analysis/InstructionSimplify.cpp
@@ -5198,6 +5198,16 @@ static Value *simplifyUnaryIntrinsic(Function *F, Value *Op0,
   return nullptr;
 }
 
+static Intrinsic::ID getMaxMinOpposite(Intrinsic::ID ID) {
+  switch (ID) {
+  case Intrinsic::smax: return Intrinsic::smin;
+  case Intrinsic::smin: return Intrinsic::smax;
+  case Intrinsic::umax: return Intrinsic::umin;
+  case Intrinsic::umin: return Intrinsic::umax;
+  default: llvm_unreachable("Unexpected intrinsic");
+  }
+}
+
 static Value *simplifyBinaryIntrinsic(Function *F, Value *Op0, Value *Op1,
                                       const SimplifyQuery &Q) {
   Intrinsic::ID IID = F->getIntrinsicID();
@@ -5239,16 +5249,27 @@ static Value *simplifyBinaryIntrinsic(Function *F, Value *Op0, Value *Op1,
         return ConstantInt::get(ReturnType, APInt::getMinValue(BitWidth));
     }
 
+    auto hasSpecificOperand = [](IntrinsicInst *II, Value *V) {
+      return II->getOperand(0) == V || II->getOperand(1) == V;
+    };
+
     // For 4 commuted variants of each intrinsic:
     // max (max X, Y), X --> max X, Y
-    if (auto *MinMax0 = dyn_cast<IntrinsicInst>(Op0))
-      if (MinMax0->getIntrinsicID() == IID &&
-          (MinMax0->getOperand(0) == Op1 || MinMax0->getOperand(1) == Op1))
+    // max (min X, Y), X --> X
+    if (auto *MinMax0 = dyn_cast<IntrinsicInst>(Op0)) {
+      Intrinsic::ID InnerID = MinMax0->getIntrinsicID();
+      if (InnerID == IID && hasSpecificOperand(MinMax0, Op1))
         return MinMax0;
-    if (auto *MinMax1 = dyn_cast<IntrinsicInst>(Op1))
-      if (MinMax1->getIntrinsicID() == IID &&
-          (MinMax1->getOperand(0) == Op0 || MinMax1->getOperand(1) == Op0))
+      if (InnerID == getMaxMinOpposite(IID) && hasSpecificOperand(MinMax0, Op1))
+        return Op1;
+    }
+    if (auto *MinMax1 = dyn_cast<IntrinsicInst>(Op1)) {
+      Intrinsic::ID InnerID = MinMax1->getIntrinsicID();
+      if (InnerID == IID && hasSpecificOperand(MinMax1, Op0))
         return MinMax1;
+      if (InnerID == getMaxMinOpposite(IID) && hasSpecificOperand(MinMax1, Op0))
+        return Op0;
+    }
 
     const APInt *C;
     if (!match(Op1, m_APIntAllowUndef(C)))
diff --git a/llvm/test/Transforms/InstSimplify/maxmin_intrinsics.ll b/llvm/test/Transforms/InstSimplify/maxmin_intrinsics.ll
index 7c79357a0bd04..7a31a4dcb9a3b 100644
--- a/llvm/test/Transforms/InstSimplify/maxmin_intrinsics.ll
+++ b/llvm/test/Transforms/InstSimplify/maxmin_intrinsics.ll
@@ -430,9 +430,7 @@ define i8 @smin_smin_commute3(i8 %x, i8 %y) {
 
 define i8 @umax_umin(i8 %x, i8 %y) {
 ; CHECK-LABEL: @umax_umin(
-; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.umax.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
-; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.umin.i8(i8 [[X]], i8 [[M]])
-; CHECK-NEXT:    ret i8 [[M2]]
+; CHECK-NEXT:    ret i8 [[X:%.*]]
 ;
   %m = call i8 @llvm.umax.i8(i8 %x, i8 %y)
   %m2 = call i8 @llvm.umin.i8(i8 %x, i8 %m)
@@ -441,9 +439,7 @@ define i8 @umax_umin(i8 %x, i8 %y) {
 
 define i8 @umax_umin_commute1(i8 %x, i8 %y) {
 ; CHECK-LABEL: @umax_umin_commute1(
-; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.umax.i8(i8 [[Y:%.*]], i8 [[X:%.*]])
-; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.umin.i8(i8 [[X]], i8 [[M]])
-; CHECK-NEXT:    ret i8 [[M2]]
+; CHECK-NEXT:    ret i8 [[X:%.*]]
 ;
   %m = call i8 @llvm.umax.i8(i8 %y, i8 %x)
   %m2 = call i8 @llvm.umin.i8(i8 %x, i8 %m)
@@ -452,9 +448,7 @@ define i8 @umax_umin_commute1(i8 %x, i8 %y) {
 
 define i8 @umax_umin_commute2(i8 %x, i8 %y) {
 ; CHECK-LABEL: @umax_umin_commute2(
-; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.umax.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
-; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.umin.i8(i8 [[M]], i8 [[X]])
-; CHECK-NEXT:    ret i8 [[M2]]
+; CHECK-NEXT:    ret i8 [[X:%.*]]
 ;
   %m = call i8 @llvm.umax.i8(i8 %x, i8 %y)
   %m2 = call i8 @llvm.umin.i8(i8 %m, i8 %x)
@@ -463,9 +457,7 @@ define i8 @umax_umin_commute2(i8 %x, i8 %y) {
 
 define <2 x i8> @umax_umin_commute3(<2 x i8> %x, <2 x i8> %y) {
 ; CHECK-LABEL: @umax_umin_commute3(
-; CHECK-NEXT:    [[M:%.*]] = call <2 x i8> @llvm.umax.v2i8(<2 x i8> [[Y:%.*]], <2 x i8> [[X:%.*]])
-; CHECK-NEXT:    [[M2:%.*]] = call <2 x i8> @llvm.umin.v2i8(<2 x i8> [[M]], <2 x i8> [[X]])
-; CHECK-NEXT:    ret <2 x i8> [[M2]]
+; CHECK-NEXT:    ret <2 x i8> [[X:%.*]]
 ;
   %m = call <2 x i8> @llvm.umax.v2i8(<2 x i8> %y, <2 x i8> %x)
   %m2 = call <2 x i8> @llvm.umin.v2i8(<2 x i8> %m, <2 x i8> %x)
@@ -474,9 +466,7 @@ define <2 x i8> @umax_umin_commute3(<2 x i8> %x, <2 x i8> %y) {
 
 define i8 @umin_umax(i8 %x, i8 %y) {
 ; CHECK-LABEL: @umin_umax(
-; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.umin.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
-; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.umax.i8(i8 [[X]], i8 [[M]])
-; CHECK-NEXT:    ret i8 [[M2]]
+; CHECK-NEXT:    ret i8 [[X:%.*]]
 ;
   %m = call i8 @llvm.umin.i8(i8 %x, i8 %y)
   %m2 = call i8 @llvm.umax.i8(i8 %x, i8 %m)
@@ -485,9 +475,7 @@ define i8 @umin_umax(i8 %x, i8 %y) {
 
 define i8 @umin_umax_commute1(i8 %x, i8 %y) {
 ; CHECK-LABEL: @umin_umax_commute1(
-; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.umin.i8(i8 [[Y:%.*]], i8 [[X:%.*]])
-; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.umax.i8(i8 [[X]], i8 [[M]])
-; CHECK-NEXT:    ret i8 [[M2]]
+; CHECK-NEXT:    ret i8 [[X:%.*]]
 ;
   %m = call i8 @llvm.umin.i8(i8 %y, i8 %x)
   %m2 = call i8 @llvm.umax.i8(i8 %x, i8 %m)
@@ -496,9 +484,7 @@ define i8 @umin_umax_commute1(i8 %x, i8 %y) {
 
 define <2 x i8> @umin_umax_commute2(<2 x i8> %x, <2 x i8> %y) {
 ; CHECK-LABEL: @umin_umax_commute2(
-; CHECK-NEXT:    [[M:%.*]] = call <2 x i8> @llvm.umin.v2i8(<2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]])
-; CHECK-NEXT:    [[M2:%.*]] = call <2 x i8> @llvm.umax.v2i8(<2 x i8> [[M]], <2 x i8> [[X]])
-; CHECK-NEXT:    ret <2 x i8> [[M2]]
+; CHECK-NEXT:    ret <2 x i8> [[X:%.*]]
 ;
   %m = call <2 x i8> @llvm.umin.v2i8(<2 x i8> %x, <2 x i8> %y)
   %m2 = call <2 x i8> @llvm.umax.v2i8(<2 x i8> %m, <2 x i8> %x)
@@ -507,9 +493,7 @@ define <2 x i8> @umin_umax_commute2(<2 x i8> %x, <2 x i8> %y) {
 
 define i8 @umin_umax_commute3(i8 %x, i8 %y) {
 ; CHECK-LABEL: @umin_umax_commute3(
-; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.umin.i8(i8 [[Y:%.*]], i8 [[X:%.*]])
-; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.umax.i8(i8 [[M]], i8 [[X]])
-; CHECK-NEXT:    ret i8 [[M2]]
+; CHECK-NEXT:    ret i8 [[X:%.*]]
 ;
   %m = call i8 @llvm.umin.i8(i8 %y, i8 %x)
   %m2 = call i8 @llvm.umax.i8(i8 %m, i8 %x)
@@ -518,9 +502,7 @@ define i8 @umin_umax_commute3(i8 %x, i8 %y) {
 
 define i8 @smax_smin(i8 %x, i8 %y) {
 ; CHECK-LABEL: @smax_smin(
-; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.smax.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
-; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.smin.i8(i8 [[X]], i8 [[M]])
-; CHECK-NEXT:    ret i8 [[M2]]
+; CHECK-NEXT:    ret i8 [[X:%.*]]
 ;
   %m = call i8 @llvm.smax.i8(i8 %x, i8 %y)
   %m2 = call i8 @llvm.smin.i8(i8 %x, i8 %m)
@@ -529,9 +511,7 @@ define i8 @smax_smin(i8 %x, i8 %y) {
 
 define <2 x i8> @smax_smin_commute1(<2 x i8> %x, <2 x i8> %y) {
 ; CHECK-LABEL: @smax_smin_commute1(
-; CHECK-NEXT:    [[M:%.*]] = call <2 x i8> @llvm.smax.v2i8(<2 x i8> [[Y:%.*]], <2 x i8> [[X:%.*]])
-; CHECK-NEXT:    [[M2:%.*]] = call <2 x i8> @llvm.smin.v2i8(<2 x i8> [[X]], <2 x i8> [[M]])
-; CHECK-NEXT:    ret <2 x i8> [[M2]]
+; CHECK-NEXT:    ret <2 x i8> [[X:%.*]]
 ;
   %m = call <2 x i8> @llvm.smax.v2i8(<2 x i8> %y, <2 x i8> %x)
   %m2 = call <2 x i8> @llvm.smin.v2i8(<2 x i8> %x, <2 x i8> %m)
@@ -540,9 +520,7 @@ define <2 x i8> @smax_smin_commute1(<2 x i8> %x, <2 x i8> %y) {
 
 define i8 @smax_smin_commute2(i8 %x, i8 %y) {
 ; CHECK-LABEL: @smax_smin_commute2(
-; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.smax.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
-; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.smin.i8(i8 [[M]], i8 [[X]])
-; CHECK-NEXT:    ret i8 [[M2]]
+; CHECK-NEXT:    ret i8 [[X:%.*]]
 ;
   %m = call i8 @llvm.smax.i8(i8 %x, i8 %y)
   %m2 = call i8 @llvm.smin.i8(i8 %m, i8 %x)
@@ -551,9 +529,7 @@ define i8 @smax_smin_commute2(i8 %x, i8 %y) {
 
 define i8 @smax_smin_commute3(i8 %x, i8 %y) {
 ; CHECK-LABEL: @smax_smin_commute3(
-; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.smax.i8(i8 [[Y:%.*]], i8 [[X:%.*]])
-; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.smin.i8(i8 [[M]], i8 [[X]])
-; CHECK-NEXT:    ret i8 [[M2]]
+; CHECK-NEXT:    ret i8 [[X:%.*]]
 ;
   %m = call i8 @llvm.smax.i8(i8 %y, i8 %x)
   %m2 = call i8 @llvm.smin.i8(i8 %m, i8 %x)
@@ -562,9 +538,7 @@ define i8 @smax_smin_commute3(i8 %x, i8 %y) {
 
 define <2 x i8> @smin_smax(<2 x i8> %x, <2 x i8> %y) {
 ; CHECK-LABEL: @smin_smax(
-; CHECK-NEXT:    [[M:%.*]] = call <2 x i8> @llvm.smin.v2i8(<2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]])
-; CHECK-NEXT:    [[M2:%.*]] = call <2 x i8> @llvm.smax.v2i8(<2 x i8> [[X]], <2 x i8> [[M]])
-; CHECK-NEXT:    ret <2 x i8> [[M2]]
+; CHECK-NEXT:    ret <2 x i8> [[X:%.*]]
 ;
   %m = call <2 x i8> @llvm.smin.v2i8(<2 x i8> %x, <2 x i8> %y)
   %m2 = call <2 x i8> @llvm.smax.v2i8(<2 x i8> %x, <2 x i8> %m)
@@ -573,9 +547,7 @@ define <2 x i8> @smin_smax(<2 x i8> %x, <2 x i8> %y) {
 
 define i8 @smin_smax_commute1(i8 %x, i8 %y) {
 ; CHECK-LABEL: @smin_smax_commute1(
-; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.smin.i8(i8 [[Y:%.*]], i8 [[X:%.*]])
-; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.smax.i8(i8 [[X]], i8 [[M]])
-; CHECK-NEXT:    ret i8 [[M2]]
+; CHECK-NEXT:    ret i8 [[X:%.*]]
 ;
   %m = call i8 @llvm.smin.i8(i8 %y, i8 %x)
   %m2 = call i8 @llvm.smax.i8(i8 %x, i8 %m)
@@ -584,9 +556,7 @@ define i8 @smin_smax_commute1(i8 %x, i8 %y) {
 
 define i8 @smin_smax_commute2(i8 %x, i8 %y) {
 ; CHECK-LABEL: @smin_smax_commute2(
-; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.smin.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
-; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.smax.i8(i8 [[M]], i8 [[X]])
-; CHECK-NEXT:    ret i8 [[M2]]
+; CHECK-NEXT:    ret i8 [[X:%.*]]
 ;
   %m = call i8 @llvm.smin.i8(i8 %x, i8 %y)
   %m2 = call i8 @llvm.smax.i8(i8 %m, i8 %x)
@@ -595,9 +565,7 @@ define i8 @smin_smax_commute2(i8 %x, i8 %y) {
 
 define i8 @smin_smax_commute3(i8 %x, i8 %y) {
 ; CHECK-LABEL: @smin_smax_commute3(
-; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.smin.i8(i8 [[Y:%.*]], i8 [[X:%.*]])
-; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.smax.i8(i8 [[M]], i8 [[X]])
-; CHECK-NEXT:    ret i8 [[M2]]
+; CHECK-NEXT:    ret i8 [[X:%.*]]
 ;
   %m = call i8 @llvm.smin.i8(i8 %y, i8 %x)
   %m2 = call i8 @llvm.smax.i8(i8 %m, i8 %x)

From 0d454e8e087049ae86283e73a25cf8eaad488651 Mon Sep 17 00:00:00 2001
From: Tim Keith <tkeith@nvidia.com>
Date: Mon, 3 Aug 2020 12:21:57 -0700
Subject: [PATCH 0943/1035] [flang] Fix bug detecting intrinsic function

Don't set the INTRINSIC attribute on a dummy procedure.

Differential Revision: https://reviews.llvm.org/D85136
---
 flang/lib/Semantics/resolve-names.cpp |  3 ++-
 flang/test/Semantics/symbol18.f90     | 10 ++++++++++
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp
index e85dfa9c91ef5..c5b42473d0113 100644
--- a/flang/lib/Semantics/resolve-names.cpp
+++ b/flang/lib/Semantics/resolve-names.cpp
@@ -5749,7 +5749,8 @@ void ResolveNamesVisitor::HandleProcedureName(
     // error was reported
   } else {
     symbol = &Resolve(name, symbol)->GetUltimate();
-    if (ConvertToProcEntity(*symbol) && IsIntrinsic(symbol->name())) {
+    if (ConvertToProcEntity(*symbol) && IsIntrinsic(symbol->name()) &&
+        !IsDummy(*symbol)) {
       symbol->attrs().set(Attr::INTRINSIC);
       // 8.2(3): ignore type from intrinsic in type-declaration-stmt
       symbol->get<ProcEntityDetails>().set_interface(ProcInterface{});
diff --git a/flang/test/Semantics/symbol18.f90 b/flang/test/Semantics/symbol18.f90
index b7269b70be0a9..93987f6741ed6 100644
--- a/flang/test/Semantics/symbol18.f90
+++ b/flang/test/Semantics/symbol18.f90
@@ -19,3 +19,13 @@ program p1
  !REF: /p1/x
  y = f(x)
 end program
+
+!DEF: /f2 (Function) Subprogram REAL(4)
+!DEF: /f2/cos EXTERNAL (Function, Implicit) ProcEntity REAL(4)
+!DEF: /f2/x (Implicit) ObjectEntity REAL(4)
+function f2(cos, x)
+ !DEF: /f2/f2 (Implicit) ObjectEntity REAL(4)
+ !REF: /f2/cos
+ !REF: /f2/x
+ f2 = cos(x)
+end function

From 3e89cbf38e76d0d0ac75fe77d318a5cfeac512f5 Mon Sep 17 00:00:00 2001
From: Hiroshi Yamauchi <yamauchi@google.com>
Date: Mon, 3 Aug 2020 11:37:22 -0700
Subject: [PATCH 0944/1035] [PGO] Enable the extended value profile buckets for
 mem op sizes.

Following up D81682 and enable the new, extended value profile buckets for mem
op sizes.

Differential Revision: https://reviews.llvm.org/D83903
---
 llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
index 3ab697d6cc321..75988893fdb85 100644
--- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
+++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
@@ -78,7 +78,7 @@ cl::opt<bool> UseOldMemOpValueProf(
     "use-old-memop-value-prof",
     cl::desc("Use the old memop value profiling buckets. This is "
              "transitional and to be removed after switching. "),
-    cl::init(true));
+    cl::init(false));
 
 namespace {
 

From 777824b49d5d9e1fbc93108107fa6d12a936a2e4 Mon Sep 17 00:00:00 2001
From: Lang Hames <lhames@gmail.com>
Date: Mon, 3 Aug 2020 11:55:57 -0700
Subject: [PATCH 0945/1035] [llvm-jitlink] Add support for static archives and
 MachO universal archives.

Archives can now be specified as input files the same way that object
files are. Archives will always be linked after all objects (regardless
of the relative order of the inputs) but before any dynamic libraries or
process symbols.

This patch also relaxes matching for slice triples in
StaticLibraryDefinitionGenerator in order to support this feature:
Vendors need not match if the source vendor is unknown.
---
 .../ExecutionEngine/Orc/ExecutionUtils.cpp    |  3 ++-
 llvm/tools/llvm-jitlink/llvm-jitlink.cpp      | 21 +++++++++++++------
 2 files changed, 17 insertions(+), 7 deletions(-)

diff --git a/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp b/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp
index 4d255cd66c1be..278f492f0ebe7 100644
--- a/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp
@@ -322,7 +322,8 @@ StaticLibraryDefinitionGenerator::Load(ObjectLayer &L, const char *FileName,
       auto ObjTT = Obj.getTriple();
       if (ObjTT.getArch() == TT.getArch() &&
           ObjTT.getSubArch() == TT.getSubArch() &&
-          ObjTT.getVendor() == TT.getVendor()) {
+          (TT.getVendor() == Triple::UnknownVendor ||
+           ObjTT.getVendor() == TT.getVendor())) {
         // We found a match. Create an instance from a buffer covering this
         // slice.
         auto SliceBuffer = MemoryBuffer::getFileSlice(FileName, Obj.getSize(),
diff --git a/llvm/tools/llvm-jitlink/llvm-jitlink.cpp b/llvm/tools/llvm-jitlink/llvm-jitlink.cpp
index 798087d8cae7b..d5dc661cc69f7 100644
--- a/llvm/tools/llvm-jitlink/llvm-jitlink.cpp
+++ b/llvm/tools/llvm-jitlink/llvm-jitlink.cpp
@@ -14,6 +14,7 @@
 
 #include "llvm-jitlink.h"
 
+#include "llvm/BinaryFormat/Magic.h"
 #include "llvm/ExecutionEngine/JITLink/EHFrameSupport.h"
 #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
 #include "llvm/MC/MCAsmInfo.h"
@@ -887,13 +888,20 @@ Error loadObjects(Session &S) {
        InputFileItr != InputFileEnd; ++InputFileItr) {
     unsigned InputFileArgIdx =
         InputFiles.getPosition(InputFileItr - InputFiles.begin());
-    StringRef InputFile = *InputFileItr;
+    const std::string &InputFile = *InputFileItr;
     auto &JD = *std::prev(IdxToJLD.lower_bound(InputFileArgIdx))->second;
     LLVM_DEBUG(dbgs() << "  " << InputFileArgIdx << ": \"" << InputFile
                       << "\" to " << JD.getName() << "\n";);
     auto ObjBuffer =
         ExitOnErr(errorOrToExpected(MemoryBuffer::getFile(InputFile)));
-    ExitOnErr(S.ObjLayer.add(JD, std::move(ObjBuffer)));
+
+    auto Magic = identify_magic(ObjBuffer->getBuffer());
+    if (Magic == file_magic::archive ||
+        Magic == file_magic::macho_universal_binary)
+      JD.addGenerator(ExitOnErr(StaticLibraryDefinitionGenerator::Load(
+          S.ObjLayer, InputFile.c_str(), S.TT)));
+    else
+      ExitOnErr(S.ObjLayer.add(JD, std::move(ObjBuffer)));
   }
 
   // Define absolute symbols.
@@ -1056,6 +1064,11 @@ int main(int argc, char *argv[]) {
 
   ExitOnErr(sanitizeArguments(*S));
 
+  {
+    TimeRegion TR(Timers ? &Timers->LoadObjectsTimer : nullptr);
+    ExitOnErr(loadObjects(*S));
+  }
+
   if (!NoProcessSymbols)
     ExitOnErr(loadProcessSymbols(*S));
   ExitOnErr(loadDylibs());
@@ -1063,10 +1076,6 @@ int main(int argc, char *argv[]) {
   if (PhonyExternals)
     addPhonyExternalsGenerator(*S);
 
-  {
-    TimeRegion TR(Timers ? &Timers->LoadObjectsTimer : nullptr);
-    ExitOnErr(loadObjects(*S));
-  }
 
   if (ShowInitialExecutionSessionState)
     S->ES.dump(outs());

From dca23ed8952383701a62b778104f4db6f5d4b799 Mon Sep 17 00:00:00 2001
From: Eli Friedman <efriedma@quicinc.com>
Date: Mon, 3 Aug 2020 12:29:40 -0700
Subject: [PATCH 0946/1035] [AArch64] Add missing isel patterns for fcvtzs/u
 intrinsic on v1f64.

Fixes test-suite compile failure caused by 8dfb5d7.

While I'm in the area, add some more test coverage to related
operations, to make sure we aren't missing any other patterns.
---
 llvm/lib/Target/AArch64/AArch64InstrInfo.td   |   4 +
 llvm/test/CodeGen/AArch64/arm64-vcvt.ll       | 183 ++++++++++++++++++
 .../AArch64/fp16_intrinsic_scalar_1op.ll      |  40 ++++
 3 files changed, 227 insertions(+)

diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index 61a43ed9df67f..39e1ee3ad8c18 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -4483,6 +4483,10 @@ def : Pat<(v1i64 (int_aarch64_neon_fcvtps (v1f64 FPR64:$Rn))),
           (FCVTPSv1i64 FPR64:$Rn)>;
 def : Pat<(v1i64 (int_aarch64_neon_fcvtpu (v1f64 FPR64:$Rn))),
           (FCVTPUv1i64 FPR64:$Rn)>;
+def : Pat<(v1i64 (int_aarch64_neon_fcvtzs (v1f64 FPR64:$Rn))),
+          (FCVTZSv1i64 FPR64:$Rn)>;
+def : Pat<(v1i64 (int_aarch64_neon_fcvtzu (v1f64 FPR64:$Rn))),
+          (FCVTZUv1i64 FPR64:$Rn)>;
 
 def : Pat<(f16 (int_aarch64_neon_frecpe (f16 FPR16:$Rn))),
           (FRECPEv1f16 FPR16:$Rn)>;
diff --git a/llvm/test/CodeGen/AArch64/arm64-vcvt.ll b/llvm/test/CodeGen/AArch64/arm64-vcvt.ll
index d236aeaf32a70..9ab7247677070 100644
--- a/llvm/test/CodeGen/AArch64/arm64-vcvt.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-vcvt.ll
@@ -30,9 +30,19 @@ define <2 x i64> @fcvtas_2d(<2 x double> %A) nounwind {
 	ret <2 x i64> %tmp3
 }
 
+define <1 x i64> @fcvtas_1d(<1 x double> %A) nounwind {
+;CHECK-LABEL: fcvtas_1d:
+;CHECK-NOT: ld1
+;CHECK: fcvtas d0, d0
+;CHECK-NEXT: ret
+	%tmp3 = call <1 x i64> @llvm.aarch64.neon.fcvtas.v1i64.v1f64(<1 x double> %A)
+	ret <1 x i64> %tmp3
+}
+
 declare <2 x i32> @llvm.aarch64.neon.fcvtas.v2i32.v2f32(<2 x float>) nounwind readnone
 declare <4 x i32> @llvm.aarch64.neon.fcvtas.v4i32.v4f32(<4 x float>) nounwind readnone
 declare <2 x i64> @llvm.aarch64.neon.fcvtas.v2i64.v2f64(<2 x double>) nounwind readnone
+declare <1 x i64> @llvm.aarch64.neon.fcvtas.v1i64.v1f64(<1 x double>) nounwind readnone
 
 define <2 x i32> @fcvtau_2s(<2 x float> %A) nounwind {
 ;CHECK-LABEL: fcvtau_2s:
@@ -61,9 +71,19 @@ define <2 x i64> @fcvtau_2d(<2 x double> %A) nounwind {
 	ret <2 x i64> %tmp3
 }
 
+define <1 x i64> @fcvtau_1d(<1 x double> %A) nounwind {
+;CHECK-LABEL: fcvtau_1d:
+;CHECK-NOT: ld1
+;CHECK: fcvtau d0, d0
+;CHECK-NEXT: ret
+	%tmp3 = call <1 x i64> @llvm.aarch64.neon.fcvtau.v1i64.v1f64(<1 x double> %A)
+	ret <1 x i64> %tmp3
+}
+
 declare <2 x i32> @llvm.aarch64.neon.fcvtau.v2i32.v2f32(<2 x float>) nounwind readnone
 declare <4 x i32> @llvm.aarch64.neon.fcvtau.v4i32.v4f32(<4 x float>) nounwind readnone
 declare <2 x i64> @llvm.aarch64.neon.fcvtau.v2i64.v2f64(<2 x double>) nounwind readnone
+declare <1 x i64> @llvm.aarch64.neon.fcvtau.v1i64.v1f64(<1 x double>) nounwind readnone
 
 define <2 x i32> @fcvtms_2s(<2 x float> %A) nounwind {
 ;CHECK-LABEL: fcvtms_2s:
@@ -92,9 +112,19 @@ define <2 x i64> @fcvtms_2d(<2 x double> %A) nounwind {
 	ret <2 x i64> %tmp3
 }
 
+define <1 x i64> @fcvtms_1d(<1 x double> %A) nounwind {
+;CHECK-LABEL: fcvtms_1d:
+;CHECK-NOT: ld1
+;CHECK: fcvtms d0, d0
+;CHECK-NEXT: ret
+       %tmp3 = call <1 x i64> @llvm.aarch64.neon.fcvtms.v1i64.v1f64(<1 x double> %A)
+       ret <1 x i64> %tmp3
+}
+
 declare <2 x i32> @llvm.aarch64.neon.fcvtms.v2i32.v2f32(<2 x float>) nounwind readnone
 declare <4 x i32> @llvm.aarch64.neon.fcvtms.v4i32.v4f32(<4 x float>) nounwind readnone
 declare <2 x i64> @llvm.aarch64.neon.fcvtms.v2i64.v2f64(<2 x double>) nounwind readnone
+declare <1 x i64> @llvm.aarch64.neon.fcvtms.v1i64.v1f64(<1 x double>) nounwind readnone
 
 define <2 x i32> @fcvtmu_2s(<2 x float> %A) nounwind {
 ;CHECK-LABEL: fcvtmu_2s:
@@ -123,9 +153,19 @@ define <2 x i64> @fcvtmu_2d(<2 x double> %A) nounwind {
 	ret <2 x i64> %tmp3
 }
 
+define <1 x i64> @fcvtmu_1d(<1 x double> %A) nounwind {
+;CHECK-LABEL: fcvtmu_1d:
+;CHECK-NOT: ld1
+;CHECK: fcvtmu d0, d0
+;CHECK-NEXT: ret
+       %tmp3 = call <1 x i64> @llvm.aarch64.neon.fcvtmu.v1i64.v1f64(<1 x double> %A)
+       ret <1 x i64> %tmp3
+}
+
 declare <2 x i32> @llvm.aarch64.neon.fcvtmu.v2i32.v2f32(<2 x float>) nounwind readnone
 declare <4 x i32> @llvm.aarch64.neon.fcvtmu.v4i32.v4f32(<4 x float>) nounwind readnone
 declare <2 x i64> @llvm.aarch64.neon.fcvtmu.v2i64.v2f64(<2 x double>) nounwind readnone
+declare <1 x i64> @llvm.aarch64.neon.fcvtmu.v1i64.v1f64(<1 x double>) nounwind readnone
 
 define <2 x i32> @fcvtps_2s(<2 x float> %A) nounwind {
 ;CHECK-LABEL: fcvtps_2s:
@@ -154,9 +194,19 @@ define <2 x i64> @fcvtps_2d(<2 x double> %A) nounwind {
 	ret <2 x i64> %tmp3
 }
 
+define <1 x i64> @fcvtps_1d(<1 x double> %A) nounwind {
+;CHECK-LABEL: fcvtps_1d:
+;CHECK-NOT: ld1
+;CHECK: fcvtps d0, d0
+;CHECK-NEXT: ret
+       %tmp3 = call <1 x i64> @llvm.aarch64.neon.fcvtps.v1i64.v1f64(<1 x double> %A)
+       ret <1 x i64> %tmp3
+}
+
 declare <2 x i32> @llvm.aarch64.neon.fcvtps.v2i32.v2f32(<2 x float>) nounwind readnone
 declare <4 x i32> @llvm.aarch64.neon.fcvtps.v4i32.v4f32(<4 x float>) nounwind readnone
 declare <2 x i64> @llvm.aarch64.neon.fcvtps.v2i64.v2f64(<2 x double>) nounwind readnone
+declare <1 x i64> @llvm.aarch64.neon.fcvtps.v1i64.v1f64(<1 x double>) nounwind readnone
 
 define <2 x i32> @fcvtpu_2s(<2 x float> %A) nounwind {
 ;CHECK-LABEL: fcvtpu_2s:
@@ -185,9 +235,19 @@ define <2 x i64> @fcvtpu_2d(<2 x double> %A) nounwind {
 	ret <2 x i64> %tmp3
 }
 
+define <1 x i64> @fcvtpu_1d(<1 x double> %A) nounwind {
+;CHECK-LABEL: fcvtpu_1d:
+;CHECK-NOT: ld1
+;CHECK: fcvtpu d0, d0
+;CHECK-NEXT: ret
+       %tmp3 = call <1 x i64> @llvm.aarch64.neon.fcvtpu.v1i64.v1f64(<1 x double> %A)
+       ret <1 x i64> %tmp3
+}
+
 declare <2 x i32> @llvm.aarch64.neon.fcvtpu.v2i32.v2f32(<2 x float>) nounwind readnone
 declare <4 x i32> @llvm.aarch64.neon.fcvtpu.v4i32.v4f32(<4 x float>) nounwind readnone
 declare <2 x i64> @llvm.aarch64.neon.fcvtpu.v2i64.v2f64(<2 x double>) nounwind readnone
+declare <1 x i64> @llvm.aarch64.neon.fcvtpu.v1i64.v1f64(<1 x double>) nounwind readnone
 
 define <2 x i32> @fcvtns_2s(<2 x float> %A) nounwind {
 ;CHECK-LABEL: fcvtns_2s:
@@ -216,9 +276,19 @@ define <2 x i64> @fcvtns_2d(<2 x double> %A) nounwind {
 	ret <2 x i64> %tmp3
 }
 
+define <1 x i64> @fcvtns_1d(<1 x double> %A) nounwind {
+;CHECK-LABEL: fcvtns_1d:
+;CHECK-NOT: ld1
+;CHECK: fcvtns d0, d0
+;CHECK-NEXT: ret
+       %tmp3 = call <1 x i64> @llvm.aarch64.neon.fcvtns.v1i64.v1f64(<1 x double> %A)
+       ret <1 x i64> %tmp3
+}
+
 declare <2 x i32> @llvm.aarch64.neon.fcvtns.v2i32.v2f32(<2 x float>) nounwind readnone
 declare <4 x i32> @llvm.aarch64.neon.fcvtns.v4i32.v4f32(<4 x float>) nounwind readnone
 declare <2 x i64> @llvm.aarch64.neon.fcvtns.v2i64.v2f64(<2 x double>) nounwind readnone
+declare <1 x i64> @llvm.aarch64.neon.fcvtns.v1i64.v1f64(<1 x double>) nounwind readnone
 
 define <2 x i32> @fcvtnu_2s(<2 x float> %A) nounwind {
 ;CHECK-LABEL: fcvtnu_2s:
@@ -247,9 +317,19 @@ define <2 x i64> @fcvtnu_2d(<2 x double> %A) nounwind {
 	ret <2 x i64> %tmp3
 }
 
+define <1 x i64> @fcvtnu_1d(<1 x double> %A) nounwind {
+;CHECK-LABEL: fcvtnu_1d:
+;CHECK-NOT: ld1
+;CHECK: fcvtnu d0, d0
+;CHECK-NEXT: ret
+       %tmp3 = call <1 x i64> @llvm.aarch64.neon.fcvtnu.v1i64.v1f64(<1 x double> %A)
+       ret <1 x i64> %tmp3
+}
+
 declare <2 x i32> @llvm.aarch64.neon.fcvtnu.v2i32.v2f32(<2 x float>) nounwind readnone
 declare <4 x i32> @llvm.aarch64.neon.fcvtnu.v4i32.v4f32(<4 x float>) nounwind readnone
 declare <2 x i64> @llvm.aarch64.neon.fcvtnu.v2i64.v2f64(<2 x double>) nounwind readnone
+declare <1 x i64> @llvm.aarch64.neon.fcvtnu.v1i64.v1f64(<1 x double>) nounwind readnone
 
 define <2 x i32> @fcvtzs_2s(<2 x float> %A) nounwind {
 ;CHECK-LABEL: fcvtzs_2s:
@@ -278,6 +358,57 @@ define <2 x i64> @fcvtzs_2d(<2 x double> %A) nounwind {
 	ret <2 x i64> %tmp3
 }
 
+; FIXME: Generate "fcvtzs d0, d0"?
+define <1 x i64> @fcvtzs_1d(<1 x double> %A) nounwind {
+;CHECK-LABEL: fcvtzs_1d:
+;CHECK-NOT: ld1
+;CHECK: fcvtzs x8, d0
+;CHECK-NEXT: mov d0, x8
+;CHECK-NEXT: ret
+	%tmp3 = fptosi <1 x double> %A to <1 x i64>
+	ret <1 x i64> %tmp3
+}
+
+define <2 x i32> @fcvtzs_2s_intrinsic(<2 x float> %A) nounwind {
+;CHECK-LABEL: fcvtzs_2s_intrinsic:
+;CHECK-NOT: ld1
+;CHECK: fcvtzs.2s v0, v0
+;CHECK-NEXT: ret
+	%tmp3 = call <2 x i32> @llvm.aarch64.neon.fcvtzs.v2i32.v2f32(<2 x float> %A)
+	ret <2 x i32> %tmp3
+}
+
+define <4 x i32> @fcvtzs_4s_intrinsic(<4 x float> %A) nounwind {
+;CHECK-LABEL: fcvtzs_4s_intrinsic:
+;CHECK-NOT: ld1
+;CHECK: fcvtzs.4s v0, v0
+;CHECK-NEXT: ret
+	%tmp3 = call <4 x i32> @llvm.aarch64.neon.fcvtzs.v4i32.v4f32(<4 x float> %A)
+	ret <4 x i32> %tmp3
+}
+
+define <2 x i64> @fcvtzs_2d_intrinsic(<2 x double> %A) nounwind {
+;CHECK-LABEL: fcvtzs_2d_intrinsic:
+;CHECK-NOT: ld1
+;CHECK: fcvtzs.2d v0, v0
+;CHECK-NEXT: ret
+	%tmp3 = call <2 x i64> @llvm.aarch64.neon.fcvtzs.v2i64.v2f64(<2 x double> %A)
+	ret <2 x i64> %tmp3
+}
+
+define <1 x i64> @fcvtzs_1d_intrinsic(<1 x double> %A) nounwind {
+;CHECK-LABEL: fcvtzs_1d_intrinsic:
+;CHECK-NOT: ld1
+;CHECK: fcvtzs d0, d0
+;CHECK-NEXT: ret
+	%tmp3 = call <1 x i64> @llvm.aarch64.neon.fcvtzs.v1i64.v1f64(<1 x double> %A)
+	ret <1 x i64> %tmp3
+}
+
+declare <2 x i32> @llvm.aarch64.neon.fcvtzs.v2i32.v2f32(<2 x float>) nounwind readnone
+declare <4 x i32> @llvm.aarch64.neon.fcvtzs.v4i32.v4f32(<4 x float>) nounwind readnone
+declare <2 x i64> @llvm.aarch64.neon.fcvtzs.v2i64.v2f64(<2 x double>) nounwind readnone
+declare <1 x i64> @llvm.aarch64.neon.fcvtzs.v1i64.v1f64(<1 x double>) nounwind readnone
 
 define <2 x i32> @fcvtzu_2s(<2 x float> %A) nounwind {
 ;CHECK-LABEL: fcvtzu_2s:
@@ -306,6 +437,58 @@ define <2 x i64> @fcvtzu_2d(<2 x double> %A) nounwind {
 	ret <2 x i64> %tmp3
 }
 
+; FIXME: Generate "fcvtzu d0, d0"?
+define <1 x i64> @fcvtzu_1d(<1 x double> %A) nounwind {
+;CHECK-LABEL: fcvtzu_1d:
+;CHECK-NOT: ld1
+;CHECK: fcvtzu x8, d0
+;CHECK-NEXT: mov d0, x8
+;CHECK-NEXT: ret
+	%tmp3 = fptoui <1 x double> %A to <1 x i64>
+	ret <1 x i64> %tmp3
+}
+
+define <2 x i32> @fcvtzu_2s_intrinsic(<2 x float> %A) nounwind {
+;CHECK-LABEL: fcvtzu_2s_intrinsic:
+;CHECK-NOT: ld1
+;CHECK: fcvtzu.2s v0, v0
+;CHECK-NEXT: ret
+	%tmp3 = call <2 x i32> @llvm.aarch64.neon.fcvtzu.v2i32.v2f32(<2 x float> %A)
+	ret <2 x i32> %tmp3
+}
+
+define <4 x i32> @fcvtzu_4s_intrinsic(<4 x float> %A) nounwind {
+;CHECK-LABEL: fcvtzu_4s_intrinsic:
+;CHECK-NOT: ld1
+;CHECK: fcvtzu.4s v0, v0
+;CHECK-NEXT: ret
+	%tmp3 = call <4 x i32> @llvm.aarch64.neon.fcvtzu.v4i32.v4f32(<4 x float> %A)
+	ret <4 x i32> %tmp3
+}
+
+define <2 x i64> @fcvtzu_2d_intrinsic(<2 x double> %A) nounwind {
+;CHECK-LABEL: fcvtzu_2d_intrinsic:
+;CHECK-NOT: ld1
+;CHECK: fcvtzu.2d v0, v0
+;CHECK-NEXT: ret
+	%tmp3 = call <2 x i64> @llvm.aarch64.neon.fcvtzu.v2i64.v2f64(<2 x double> %A)
+	ret <2 x i64> %tmp3
+}
+
+define <1 x i64> @fcvtzu_1d_intrinsic(<1 x double> %A) nounwind {
+;CHECK-LABEL: fcvtzu_1d_intrinsic:
+;CHECK-NOT: ld1
+;CHECK: fcvtzu d0, d0
+;CHECK-NEXT: ret
+	%tmp3 = call <1 x i64> @llvm.aarch64.neon.fcvtzu.v1i64.v1f64(<1 x double> %A)
+	ret <1 x i64> %tmp3
+}
+
+declare <2 x i32> @llvm.aarch64.neon.fcvtzu.v2i32.v2f32(<2 x float>) nounwind readnone
+declare <4 x i32> @llvm.aarch64.neon.fcvtzu.v4i32.v4f32(<4 x float>) nounwind readnone
+declare <2 x i64> @llvm.aarch64.neon.fcvtzu.v2i64.v2f64(<2 x double>) nounwind readnone
+declare <1 x i64> @llvm.aarch64.neon.fcvtzu.v1i64.v1f64(<1 x double>) nounwind readnone
+
 define <2 x float> @frinta_2s(<2 x float> %A) nounwind {
 ;CHECK-LABEL: frinta_2s:
 ;CHECK-NOT: ld1
diff --git a/llvm/test/CodeGen/AArch64/fp16_intrinsic_scalar_1op.ll b/llvm/test/CodeGen/AArch64/fp16_intrinsic_scalar_1op.ll
index c8333b253ec42..ff19e6ac91a73 100644
--- a/llvm/test/CodeGen/AArch64/fp16_intrinsic_scalar_1op.ll
+++ b/llvm/test/CodeGen/AArch64/fp16_intrinsic_scalar_1op.ll
@@ -16,6 +16,10 @@ declare i64 @llvm.aarch64.neon.fcvtau.i64.f16(half)
 declare i32 @llvm.aarch64.neon.fcvtau.i32.f16(half)
 declare i64 @llvm.aarch64.neon.fcvtas.i64.f16(half)
 declare i32 @llvm.aarch64.neon.fcvtas.i32.f16(half)
+declare i64 @llvm.aarch64.neon.fcvtzs.i64.f16(half)
+declare i32 @llvm.aarch64.neon.fcvtzs.i32.f16(half)
+declare i64 @llvm.aarch64.neon.fcvtzu.i64.f16(half)
+declare i32 @llvm.aarch64.neon.fcvtzu.i32.f16(half)
 declare half @llvm.aarch64.neon.frsqrte.f16(half)
 declare half @llvm.aarch64.neon.frecpx.f16(half)
 declare half @llvm.aarch64.neon.frecpe.f16(half)
@@ -138,6 +142,42 @@ entry:
   ret i64 %0
 }
 
+define i32 @fcvtzu_intrinsic_i32(half %a) {
+; CHECK-LABEL: fcvtzu_intrinsic_i32:
+; CHECK:         fcvtzu w0, h0
+; CHECK-NEXT:    ret
+entry:
+  %fcvt = tail call i32 @llvm.aarch64.neon.fcvtzu.i32.f16(half %a)
+  ret i32 %fcvt
+}
+
+define i64 @fcvtzu_intrinsic_i64(half %a) {
+; CHECK-LABEL: fcvtzu_intrinsic_i64:
+; CHECK:         fcvtzs x0, h0
+; CHECK-NEXT:    ret
+entry:
+  %fcvt = tail call i64 @llvm.aarch64.neon.fcvtzs.i64.f16(half %a)
+  ret i64 %fcvt
+}
+
+define i32 @fcvtzs_intrinsic_i32(half %a) {
+; CHECK-LABEL: fcvtzs_intrinsic_i32:
+; CHECK:         fcvtzs w0, h0
+; CHECK-NEXT:    ret
+entry:
+  %fcvt = tail call i32 @llvm.aarch64.neon.fcvtzs.i32.f16(half %a)
+  ret i32 %fcvt
+}
+
+define i64 @fcvtzs_intrinsic_i64(half %a) {
+; CHECK-LABEL: fcvtzs_intrinsic_i64:
+; CHECK:         fcvtzs x0, h0
+; CHECK-NEXT:    ret
+entry:
+  %fcvt = tail call i64 @llvm.aarch64.neon.fcvtzs.i64.f16(half %a)
+  ret i64 %fcvt
+}
+
 define dso_local i16 @t19(half %a) {
 ; CHECK-LABEL: t19:
 ; CHECK:         fcvtas w0, h0

From 7f1556f292ccfd80c4ffa986d5b849f915e5cd82 Mon Sep 17 00:00:00 2001
From: Jon Roelofs <jonathan_roelofs@apple.com>
Date: Mon, 3 Aug 2020 14:09:46 -0600
Subject: [PATCH 0947/1035] Fix typo: s/epomymous/eponymous/ NFC

---
 llvm/lib/CodeGen/MachineScheduler.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/CodeGen/MachineScheduler.cpp b/llvm/lib/CodeGen/MachineScheduler.cpp
index cec7a0c031eb5..7daaa3526aa3a 100644
--- a/llvm/lib/CodeGen/MachineScheduler.cpp
+++ b/llvm/lib/CodeGen/MachineScheduler.cpp
@@ -1298,7 +1298,7 @@ void ScheduleDAGMILive::computeDFSResult() {
 /// The cyclic path estimation identifies a def-use pair that crosses the back
 /// edge and considers the depth and height of the nodes. For example, consider
 /// the following instruction sequence where each instruction has unit latency
-/// and defines an epomymous virtual register:
+/// and defines an eponymous virtual register:
 ///
 /// a->b(a,c)->c(b)->d(c)->exit
 ///

From 7209f83112db4dbe15d8328705f9d2aff0624fbd Mon Sep 17 00:00:00 2001
From: Daniel Sanders <daniel_l_sanders@apple.com>
Date: Mon, 3 Aug 2020 12:46:49 -0700
Subject: [PATCH 0948/1035] Allow .dSYM's to be directly placed in an alternate
 directory

Once available in the relevant toolchains this will allow us to implement
LLVM_EXTERNALIZE_DEBUGINFO_OUTPUT_DIR after D84127 by directly placing the dSYM
in the desired location instead of emitting next to the output file and moving
it.

Reviewed By: JDevlieghere

Differential Revision: https://reviews.llvm.org/D84572
---
 clang/include/clang/Driver/Options.td |  3 +++
 clang/lib/Driver/Driver.cpp           | 12 +++++++++++-
 clang/test/Driver/darwin-dsymutil.c   | 24 ++++++++++++++----------
 3 files changed, 28 insertions(+), 11 deletions(-)

diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 16051934c1e0b..fcb5c030755ed 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -677,6 +677,9 @@ def dependency_dot : Separate<["-"], "dependency-dot">, Flags<[CC1Option]>,
   HelpText<"Filename to write DOT-formatted header dependencies to">;
 def module_dependency_dir : Separate<["-"], "module-dependency-dir">,
   Flags<[CC1Option]>, HelpText<"Directory to dump module dependencies to">;
+def dsym_dir : JoinedOrSeparate<["-"], "dsym-dir">,
+  Flags<[DriverOption, RenderAsInput]>,
+  HelpText<"Directory to output dSYM's (if any) to">, MetaVarName<"<dir>">;
 def dumpmachine : Flag<["-"], "dumpmachine">;
 def dumpspecs : Flag<["-"], "dumpspecs">, Flags<[Unsupported]>;
 def dumpversion : Flag<["-"], "dumpversion">;
diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index 317098e248233..35263fbe1b2d8 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -4604,7 +4604,17 @@ const char *Driver::GetNamedOutputPath(Compilation &C, const JobAction &JA,
   StringRef BaseName;
 
   // Dsymutil actions should use the full path.
-  if (isa<DsymutilJobAction>(JA) || isa<VerifyJobAction>(JA))
+  if (isa<DsymutilJobAction>(JA) && C.getArgs().hasArg(options::OPT_dsym_dir)) {
+    SmallString<128> ExternalPath(
+        C.getArgs().getLastArg(options::OPT_dsym_dir)->getValue());
+    // We use posix style here because the tests (specifically
+    // darwin-dsymutil.c) demonstrate that posix style paths are acceptable
+    // even on Windows and if we don't then the similar test covering this
+    // fails.
+    llvm::sys::path::append(ExternalPath, llvm::sys::path::Style::posix,
+                            llvm::sys::path::filename(BasePath));
+    BaseName = ExternalPath;
+  } else if (isa<DsymutilJobAction>(JA) || isa<VerifyJobAction>(JA))
     BaseName = BasePath;
   else
     BaseName = llvm::sys::path::filename(BasePath);
diff --git a/clang/test/Driver/darwin-dsymutil.c b/clang/test/Driver/darwin-dsymutil.c
index 09451a81b797d..8cdb2f3cbf644 100644
--- a/clang/test/Driver/darwin-dsymutil.c
+++ b/clang/test/Driver/darwin-dsymutil.c
@@ -26,10 +26,21 @@
 //
 // RUN: %clang -target x86_64-apple-darwin10 -ccc-print-bindings \
 // RUN:   -o foo %s -g 2> %t
-// RUN: FileCheck -check-prefix=CHECK-OUTPUT-NAME < %t %s
+// RUN: FileCheck -Doutfile=foo -Ddsymfile=foo.dSYM \
+// RUN:          -check-prefix=CHECK-OUTPUT-NAME < %t %s
 //
-// CHECK-OUTPUT-NAME: "x86_64-apple-darwin10" - "darwin::Linker", inputs: [{{.*}}], output: "foo"
-// CHECK-OUTPUT-NAME: "x86_64-apple-darwin10" - "darwin::Dsymutil", inputs: ["foo"], output: "foo.dSYM"
+// RUN: %clang -target x86_64-apple-darwin10 -ccc-print-bindings \
+// RUN:   -o bar/foo %s -g 2> %t
+// RUN: FileCheck -Doutfile=bar/foo -Ddsymfile=bar/foo.dSYM \
+// RUN:           -check-prefix=CHECK-OUTPUT-NAME < %t %s
+//
+// RUN: %clang -target x86_64-apple-darwin10 -ccc-print-bindings \
+// RUN:   -o bar/foo -dsym-dir external %s -g 2> %t
+// RUN: FileCheck -Doutfile=bar/foo -Ddsymfile=external/foo.dSYM \
+// RUN:           -check-prefix=CHECK-OUTPUT-NAME < %t %s
+//
+// CHECK-OUTPUT-NAME: "x86_64-apple-darwin10" - "darwin::Linker", inputs: [{{.*}}], output: "[[outfile]]"
+// CHECK-OUTPUT-NAME: "x86_64-apple-darwin10" - "darwin::Dsymutil", inputs: ["[[outfile]]"], output: "[[dsymfile]]"
 
 // Check that we only use dsymutil when needed.
 //
@@ -38,12 +49,5 @@
 // RUN:   -o foo %t.o -g 2> %t
 // RUN: not grep "Dsymutil" %t
 
-// Check that we put the .dSYM in the right place.
-// RUN: %clang -target x86_64-apple-darwin10 -ccc-print-bindings \
-// RUN:   -o bar/foo %s -g 2> %t
-// RUN: FileCheck -check-prefix=CHECK-LOCATION < %t %s
-
-// CHECK-LOCATION: "x86_64-apple-darwin10" - "darwin::Dsymutil", inputs: ["bar/foo"], output: "bar/foo.dSYM"
-
 // Check that we don't crash when translating arguments for dsymutil.
 // RUN: %clang -m32 -arch x86_64 -g %s -###

From 41b1e97b12c1407e40d8e5081bf1f9cf183934b0 Mon Sep 17 00:00:00 2001
From: Akira Hatanaka <ahatanaka@apple.com>
Date: Mon, 3 Aug 2020 13:25:25 -0700
Subject: [PATCH 0949/1035] [CodeGen][ObjC] Mark calls to
 objc_unsafeClaimAutoreleasedReturnValue as notail on x86-64

This is needed because the epilogue code inserted before tail calls on
x86-64 breaks the handshake between the caller and callee.

Calls to objc_retainAutoreleasedReturnValue used to have the same
problem, which was fixed in https://reviews.llvm.org/D59656.

rdar://problem/66029552

Differential Revision: https://reviews.llvm.org/D84540
---
 clang/lib/CodeGen/CGObjC.cpp             | 14 +++++++----
 clang/lib/CodeGen/TargetInfo.cpp         |  6 ++---
 clang/lib/CodeGen/TargetInfo.h           |  8 +++----
 clang/test/CodeGenObjC/arc-unsafeclaim.m | 30 ++++++++++++++----------
 4 files changed, 32 insertions(+), 26 deletions(-)

diff --git a/clang/lib/CodeGen/CGObjC.cpp b/clang/lib/CodeGen/CGObjC.cpp
index cd2b84f5dd203..26dfb6259a290 100644
--- a/clang/lib/CodeGen/CGObjC.cpp
+++ b/clang/lib/CodeGen/CGObjC.cpp
@@ -2250,8 +2250,7 @@ llvm::Value *
 CodeGenFunction::EmitARCRetainAutoreleasedReturnValue(llvm::Value *value) {
   emitAutoreleasedReturnValueMarker(*this);
   llvm::CallInst::TailCallKind tailKind =
-      CGM.getTargetCodeGenInfo()
-              .shouldSuppressTailCallsOfRetainAutoreleasedReturnValue()
+      CGM.getTargetCodeGenInfo().markARCOptimizedReturnCallsAsNoTail()
           ? llvm::CallInst::TCK_NoTail
           : llvm::CallInst::TCK_None;
   return emitARCValueOperation(
@@ -2270,9 +2269,14 @@ CodeGenFunction::EmitARCRetainAutoreleasedReturnValue(llvm::Value *value) {
 llvm::Value *
 CodeGenFunction::EmitARCUnsafeClaimAutoreleasedReturnValue(llvm::Value *value) {
   emitAutoreleasedReturnValueMarker(*this);
-  return emitARCValueOperation(*this, value, nullptr,
-              CGM.getObjCEntrypoints().objc_unsafeClaimAutoreleasedReturnValue,
-                     llvm::Intrinsic::objc_unsafeClaimAutoreleasedReturnValue);
+  llvm::CallInst::TailCallKind tailKind =
+      CGM.getTargetCodeGenInfo().markARCOptimizedReturnCallsAsNoTail()
+          ? llvm::CallInst::TCK_NoTail
+          : llvm::CallInst::TCK_None;
+  return emitARCValueOperation(
+      *this, value, nullptr,
+      CGM.getObjCEntrypoints().objc_unsafeClaimAutoreleasedReturnValue,
+      llvm::Intrinsic::objc_unsafeClaimAutoreleasedReturnValue, tailKind);
 }
 
 /// Release the given object.
diff --git a/clang/lib/CodeGen/TargetInfo.cpp b/clang/lib/CodeGen/TargetInfo.cpp
index 9cd63ebe29ee0..f31d432eb3171 100644
--- a/clang/lib/CodeGen/TargetInfo.cpp
+++ b/clang/lib/CodeGen/TargetInfo.cpp
@@ -2404,10 +2404,8 @@ class X86_64TargetCodeGenInfo : public TargetCodeGenInfo {
   }
 
   /// Disable tail call on x86-64. The epilogue code before the tail jump blocks
-  /// the autoreleaseRV/retainRV optimization.
-  bool shouldSuppressTailCallsOfRetainAutoreleasedReturnValue() const override {
-    return true;
-  }
+  /// autoreleaseRV/retainRV and autoreleaseRV/unsafeClaimRV optimizations.
+  bool markARCOptimizedReturnCallsAsNoTail() const override { return true; }
 
   int getDwarfEHStackPointer(CodeGen::CodeGenModule &CGM) const override {
     return 7;
diff --git a/clang/lib/CodeGen/TargetInfo.h b/clang/lib/CodeGen/TargetInfo.h
index 1152cabce4a0d..0df9667e91e16 100644
--- a/clang/lib/CodeGen/TargetInfo.h
+++ b/clang/lib/CodeGen/TargetInfo.h
@@ -163,11 +163,9 @@ class TargetCodeGenInfo {
     return "";
   }
 
-  /// Determine whether a call to objc_retainAutoreleasedReturnValue should be
-  /// marked as 'notail'.
-  virtual bool shouldSuppressTailCallsOfRetainAutoreleasedReturnValue() const {
-    return false;
-  }
+  /// Determine whether a call to objc_retainAutoreleasedReturnValue or
+  /// objc_unsafeClaimAutoreleasedReturnValue should be marked as 'notail'.
+  virtual bool markARCOptimizedReturnCallsAsNoTail() const { return false; }
 
   /// Return a constant used by UBSan as a signature to identify functions
   /// possessing type information, or 0 if the platform is unsupported.
diff --git a/clang/test/CodeGenObjC/arc-unsafeclaim.m b/clang/test/CodeGenObjC/arc-unsafeclaim.m
index a8011e024180d..40f1f164455a7 100644
--- a/clang/test/CodeGenObjC/arc-unsafeclaim.m
+++ b/clang/test/CodeGenObjC/arc-unsafeclaim.m
@@ -1,16 +1,16 @@
 //   Make sure it works on x86-64.
-// RUN: %clang_cc1 -triple x86_64-apple-darwin11 -fobjc-runtime=macosx-10.11 -fobjc-arc -emit-llvm -o - %s | FileCheck %s -check-prefix=CHECK -check-prefix=CHECK-UNOPTIMIZED
+// RUN: %clang_cc1 -triple x86_64-apple-darwin11 -fobjc-runtime=macosx-10.11 -fobjc-arc -emit-llvm -o - %s | FileCheck %s -check-prefix=CHECK -check-prefix=CHECK-UNOPTIMIZED -check-prefix=NOTAIL-CALL
 
 //   Make sure it works on x86-32.
-// RUN: %clang_cc1 -triple i386-apple-darwin11 -fobjc-runtime=macosx-fragile-10.11 -fobjc-arc -emit-llvm -o - %s | FileCheck %s -check-prefix=CHECK -check-prefix=CHECK-UNOPTIMIZED -check-prefix=CHECK-MARKED
+// RUN: %clang_cc1 -triple i386-apple-darwin11 -fobjc-runtime=macosx-fragile-10.11 -fobjc-arc -emit-llvm -o - %s | FileCheck %s -check-prefix=CHECK -check-prefix=CHECK-UNOPTIMIZED -check-prefix=CHECK-MARKED -check-prefix=CALL
 
 //   Make sure it works on ARM.
-// RUN: %clang_cc1 -triple arm64-apple-ios9 -fobjc-runtime=ios-9.0 -fobjc-arc -emit-llvm -o - %s | FileCheck %s -check-prefix=CHECK -check-prefix=CHECK-UNOPTIMIZED -check-prefix=CHECK-MARKED
-// RUN: %clang_cc1 -triple arm64-apple-ios9 -fobjc-runtime=ios-9.0 -fobjc-arc -O -disable-llvm-passes -emit-llvm -o - %s | FileCheck %s -check-prefix=CHECK -check-prefix=CHECK-OPTIMIZED
+// RUN: %clang_cc1 -triple arm64-apple-ios9 -fobjc-runtime=ios-9.0 -fobjc-arc -emit-llvm -o - %s | FileCheck %s -check-prefix=CHECK -check-prefix=CHECK-UNOPTIMIZED -check-prefix=CHECK-MARKED -check-prefix=CALL
+// RUN: %clang_cc1 -triple arm64-apple-ios9 -fobjc-runtime=ios-9.0 -fobjc-arc -O -disable-llvm-passes -emit-llvm -o - %s | FileCheck %s -check-prefix=CHECK -check-prefix=CHECK-OPTIMIZED -check-prefix=CALL
 
 //   Make sure it works on ARM64.
-// RUN: %clang_cc1 -triple armv7-apple-ios9 -fobjc-runtime=ios-9.0 -fobjc-arc -emit-llvm -o - %s | FileCheck %s -check-prefix=CHECK -check-prefix=CHECK-UNOPTIMIZED -check-prefix=CHECK-MARKED
-// RUN: %clang_cc1 -triple armv7-apple-ios9 -fobjc-runtime=ios-9.0 -fobjc-arc -O -disable-llvm-passes -emit-llvm -o - %s | FileCheck %s -check-prefix=CHECK -check-prefix=CHECK-OPTIMIZED
+// RUN: %clang_cc1 -triple armv7-apple-ios9 -fobjc-runtime=ios-9.0 -fobjc-arc -emit-llvm -o - %s | FileCheck %s -check-prefix=CHECK -check-prefix=CHECK-UNOPTIMIZED -check-prefix=CHECK-MARKED -check-prefix=CALL
+// RUN: %clang_cc1 -triple armv7-apple-ios9 -fobjc-runtime=ios-9.0 -fobjc-arc -O -disable-llvm-passes -emit-llvm -o - %s | FileCheck %s -check-prefix=CHECK -check-prefix=CHECK-OPTIMIZED -check-prefix=CALL
 
 //   Make sure that it's implicitly disabled if the runtime version isn't high enough.
 // RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fobjc-runtime=macosx-10.10 -fobjc-arc -emit-llvm -o - %s | FileCheck %s -check-prefix=DISABLED
@@ -29,7 +29,8 @@ void test_assign() {
 // CHECK:                [[T0:%.*]] = call [[A:.*]]* @makeA()
 // CHECK-MARKED-NEXT:    call void asm sideeffect
 // CHECK-NEXT:           [[T1:%.*]] = bitcast [[A]]* [[T0]] to i8*
-// CHECK-NEXT:           [[T2:%.*]] = call i8* @llvm.objc.unsafeClaimAutoreleasedReturnValue(i8* [[T1]])
+// NOTAIL-CALL-NEXT:     [[T2:%.*]] = notail call i8* @llvm.objc.unsafeClaimAutoreleasedReturnValue(i8* [[T1]])
+// CALL-NEXT:            [[T2:%.*]] = call i8* @llvm.objc.unsafeClaimAutoreleasedReturnValue(i8* [[T1]])
 // CHECK-NEXT:           [[T3:%.*]] = bitcast i8* [[T2]] to [[A]]*
 // CHECK-NEXT:           [[T4:%.*]] = bitcast [[A]]* [[T3]] to i8*
 // CHECK-NEXT:           store i8* [[T4]], i8** [[X]]
@@ -53,7 +54,8 @@ void test_assign_assign() {
 // CHECK:                [[T0:%.*]] = call [[A]]* @makeA()
 // CHECK-MARKED-NEXT:    call void asm sideeffect
 // CHECK-NEXT:           [[T1:%.*]] = bitcast [[A]]* [[T0]] to i8*
-// CHECK-NEXT:           [[T2:%.*]] = call i8* @llvm.objc.unsafeClaimAutoreleasedReturnValue(i8* [[T1]])
+// NOTAIL-CALL-NEXT:     [[T2:%.*]] = notail call i8* @llvm.objc.unsafeClaimAutoreleasedReturnValue(i8* [[T1]])
+// CALL-NEXT:            [[T2:%.*]] = call i8* @llvm.objc.unsafeClaimAutoreleasedReturnValue(i8* [[T1]])
 // CHECK-NEXT:           [[T3:%.*]] = bitcast i8* [[T2]] to [[A]]*
 // CHECK-NEXT:           [[T4:%.*]] = bitcast [[A]]* [[T3]] to i8*
 // CHECK-NEXT:           store i8* [[T4]], i8** [[Y]]
@@ -126,7 +128,8 @@ void test_init() {
 // CHECK:                [[T0:%.*]] = call [[A]]* @makeA()
 // CHECK-MARKED-NEXT:    call void asm sideeffect
 // CHECK-NEXT:           [[T1:%.*]] = bitcast [[A]]* [[T0]] to i8*
-// CHECK-NEXT:           [[T2:%.*]] = call i8* @llvm.objc.unsafeClaimAutoreleasedReturnValue(i8* [[T1]])
+// NOTAIL-CALL-NEXT:     [[T2:%.*]] = notail call i8* @llvm.objc.unsafeClaimAutoreleasedReturnValue(i8* [[T1]])
+// CALL-NEXT:            [[T2:%.*]] = call i8* @llvm.objc.unsafeClaimAutoreleasedReturnValue(i8* [[T1]])
 // CHECK-NEXT:           [[T3:%.*]] = bitcast i8* [[T2]] to [[A]]*
 // CHECK-NEXT:           [[T4:%.*]] = bitcast [[A]]* [[T3]] to i8*
 // CHECK-NEXT:           store i8* [[T4]], i8** [[X]]
@@ -144,7 +147,8 @@ void test_init_assignment() {
 // CHECK:                [[T0:%.*]] = call [[A]]* @makeA()
 // CHECK-MARKED-NEXT:    call void asm sideeffect
 // CHECK-NEXT:           [[T1:%.*]] = bitcast [[A]]* [[T0]] to i8*
-// CHECK-NEXT:           [[T2:%.*]] = call i8* @llvm.objc.unsafeClaimAutoreleasedReturnValue(i8* [[T1]])
+// NOTAIL-CALL-NEXT:     [[T2:%.*]] = notail call i8* @llvm.objc.unsafeClaimAutoreleasedReturnValue(i8* [[T1]])
+// CALL-NEXT:            [[T2:%.*]] = call i8* @llvm.objc.unsafeClaimAutoreleasedReturnValue(i8* [[T1]])
 // CHECK-NEXT:           [[T3:%.*]] = bitcast i8* [[T2]] to [[A]]*
 // CHECK-NEXT:           [[T4:%.*]] = bitcast [[A]]* [[T3]] to i8*
 // CHECK-NEXT:           store i8* [[T4]], i8** [[X]]
@@ -212,7 +216,8 @@ void test_ignored() {
 // CHECK:             [[T0:%.*]] = call [[A]]* @makeA()
 // CHECK-MARKED-NEXT: call void asm sideeffect
 // CHECK-NEXT:        [[T1:%.*]] = bitcast [[A]]* [[T0]] to i8*
-// CHECK-NEXT:        [[T2:%.*]] = call i8* @llvm.objc.unsafeClaimAutoreleasedReturnValue(i8* [[T1]])
+// NOTAIL-CALL-NEXT:  [[T2:%.*]] = notail call i8* @llvm.objc.unsafeClaimAutoreleasedReturnValue(i8* [[T1]])
+// CALL-NEXT:         [[T2:%.*]] = call i8* @llvm.objc.unsafeClaimAutoreleasedReturnValue(i8* [[T1]])
 // CHECK-NEXT:        bitcast i8* [[T2]] to [[A]]*
 // CHECK-NEXT:        ret void
 
@@ -223,7 +228,8 @@ void test_cast_to_void() {
 // CHECK:             [[T0:%.*]] = call [[A]]* @makeA()
 // CHECK-MARKED-NEXT: call void asm sideeffect
 // CHECK-NEXT:        [[T1:%.*]] = bitcast [[A]]* [[T0]] to i8*
-// CHECK-NEXT:        [[T2:%.*]] = call i8* @llvm.objc.unsafeClaimAutoreleasedReturnValue(i8* [[T1]])
+// NOTAIL-CALL-NEXT:  [[T2:%.*]] = notail call i8* @llvm.objc.unsafeClaimAutoreleasedReturnValue(i8* [[T1]])
+// CALL-NEXT:         [[T2:%.*]] = call i8* @llvm.objc.unsafeClaimAutoreleasedReturnValue(i8* [[T1]])
 // CHECK-NEXT:        bitcast i8* [[T2]] to [[A]]*
 // CHECK-NEXT:        ret void
 

From 11bb7c220ccdff1ffec4780ff92fb5acec8f6f0b Mon Sep 17 00:00:00 2001
From: Fangrui Song <maskray@google.com>
Date: Mon, 3 Aug 2020 13:35:59 -0700
Subject: [PATCH 0950/1035] [MC] Set sh_link to 0 if the associated symbol is
 undefined

Part of https://bugs.llvm.org/show_bug.cgi?id=41734

LTO can drop externally available definitions. Such AssociatedSymbol is
not associated with a symbol. ELFWriter::writeSection() will assert.

Allow a SHF_LINK_ORDER section to have sh_link=0.

We need to give sh_link a syntax, a literal zero in the linked-to symbol
position, e.g. `.section name,"ao",@progbits,0`

Reviewed By: pcc

Differential Revision: https://reviews.llvm.org/D72899
---
 .../CodeGen/TargetLoweringObjectFileImpl.cpp  |  2 +-
 llvm/lib/MC/ELFObjectWriter.cpp               |  8 +++++--
 llvm/lib/MC/MCParser/ELFAsmParser.cpp         |  8 ++++++-
 llvm/lib/MC/MCSectionELF.cpp                  |  6 +++--
 .../CodeGen/X86/elf-associated-discarded.ll   | 23 +++++++++++++++++++
 llvm/test/CodeGen/X86/elf-associated.ll       |  6 ++---
 llvm/test/MC/ELF/section-linkorder.s          |  8 +++++++
 7 files changed, 52 insertions(+), 9 deletions(-)
 create mode 100644 llvm/test/CodeGen/X86/elf-associated-discarded.ll
 create mode 100644 llvm/test/MC/ELF/section-linkorder.s

diff --git a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
index 48599199626ce..8ef91250423f3 100644
--- a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
@@ -680,7 +680,7 @@ MCSection *TargetLoweringObjectFileELF::getExplicitSectionGlobal(
   // MD_associated in a unique section.
   unsigned UniqueID = MCContext::GenericSectionID;
   const MCSymbolELF *LinkedToSym = getLinkedToSymbol(GO, TM);
-  if (LinkedToSym) {
+  if (GO->getMetadata(LLVMContext::MD_associated)) {
     UniqueID = NextUniqueID++;
     Flags |= ELF::SHF_LINK_ORDER;
   } else {
diff --git a/llvm/lib/MC/ELFObjectWriter.cpp b/llvm/lib/MC/ELFObjectWriter.cpp
index 5a5692c0cb636..b44a36b9713ef 100644
--- a/llvm/lib/MC/ELFObjectWriter.cpp
+++ b/llvm/lib/MC/ELFObjectWriter.cpp
@@ -1024,9 +1024,13 @@ void ELFWriter::writeSection(const SectionIndexMapTy &SectionIndexMap,
   }
 
   if (Section.getFlags() & ELF::SHF_LINK_ORDER) {
+    // If the value in the associated metadata is not a definition, Sym will be
+    // undefined. Represent this with sh_link=0.
     const MCSymbol *Sym = Section.getLinkedToSymbol();
-    const MCSectionELF *Sec = cast<MCSectionELF>(&Sym->getSection());
-    sh_link = SectionIndexMap.lookup(Sec);
+    if (Sym && Sym->isInSection()) {
+      const MCSectionELF *Sec = cast<MCSectionELF>(&Sym->getSection());
+      sh_link = SectionIndexMap.lookup(Sec);
+    }
   }
 
   WriteSecHdrEntry(StrTabBuilder.getOffset(Section.getName()),
diff --git a/llvm/lib/MC/MCParser/ELFAsmParser.cpp b/llvm/lib/MC/MCParser/ELFAsmParser.cpp
index e5ab13bc719d4..41779d023a5df 100644
--- a/llvm/lib/MC/MCParser/ELFAsmParser.cpp
+++ b/llvm/lib/MC/MCParser/ELFAsmParser.cpp
@@ -450,8 +450,14 @@ bool ELFAsmParser::parseLinkedToSym(MCSymbolELF *&LinkedToSym) {
   Lex();
   StringRef Name;
   SMLoc StartLoc = L.getLoc();
-  if (getParser().parseIdentifier(Name))
+  if (getParser().parseIdentifier(Name)) {
+    if (getParser().getTok().getString() == "0") {
+      getParser().Lex();
+      LinkedToSym = nullptr;
+      return false;
+    }
     return TokError("invalid linked-to symbol");
+  }
   LinkedToSym = dyn_cast_or_null<MCSymbolELF>(getContext().lookupSymbol(Name));
   if (!LinkedToSym || !LinkedToSym->isInSection())
     return Error(StartLoc, "linked-to symbol is not in a section: " + Name);
diff --git a/llvm/lib/MC/MCSectionELF.cpp b/llvm/lib/MC/MCSectionELF.cpp
index 77c259c27a04e..7a15556182658 100644
--- a/llvm/lib/MC/MCSectionELF.cpp
+++ b/llvm/lib/MC/MCSectionELF.cpp
@@ -172,9 +172,11 @@ void MCSectionELF::PrintSwitchToSection(const MCAsmInfo &MAI, const Triple &T,
   }
 
   if (Flags & ELF::SHF_LINK_ORDER) {
-    assert(LinkedToSym);
     OS << ",";
-    printName(OS, LinkedToSym->getName());
+    if (LinkedToSym)
+      printName(OS, LinkedToSym->getName());
+    else
+      OS << '0';
   }
 
   if (isUnique())
diff --git a/llvm/test/CodeGen/X86/elf-associated-discarded.ll b/llvm/test/CodeGen/X86/elf-associated-discarded.ll
new file mode 100644
index 0000000000000..5a4fad4ebb7d9
--- /dev/null
+++ b/llvm/test/CodeGen/X86/elf-associated-discarded.ll
@@ -0,0 +1,23 @@
+;; Test that we keep SHF_LINK_ORDER but reset sh_link to 0 if the associated
+;; symbol is not defined.
+; RUN: llc -mtriple=x86_64 -data-sections=1 < %s | FileCheck %s
+; RUN: llc -filetype=obj -mtriple=x86_64 -data-sections=1 < %s | llvm-readelf -S - | FileCheck --check-prefix=SEC %s
+
+;; FIXME The assembly output cannot be assembled because foo is not defined.
+;; This is difficult to fix because we allow loops (see elf-associated.ll
+;; .data.c and .data.d).
+; CHECK: .section .data.a,"awo",@progbits,foo
+; CHECK: .section .data.b,"awo",@progbits,foo
+
+;; No 'L' (SHF_LINK_ORDER). sh_link=0.
+; SEC; Name    {{.*}} Flg Lk Inf
+; SEC: .data.a {{.*}} WAL  0   0
+; SEC: .data.b {{.*}} WAL  0   0
+
+;; The definition may be discarded by LTO.
+declare void @foo()
+
+@a = global i32 1, !associated !0
+@b = global i32 1, !associated !0
+
+!0 = !{void ()* @foo}
diff --git a/llvm/test/CodeGen/X86/elf-associated.ll b/llvm/test/CodeGen/X86/elf-associated.ll
index e0e9e00582892..14a4b5b85b0cf 100644
--- a/llvm/test/CodeGen/X86/elf-associated.ll
+++ b/llvm/test/CodeGen/X86/elf-associated.ll
@@ -36,15 +36,15 @@
 ; Non-GlobalValue metadata.
 @l = global i32 1, section "ccc", !associated !5
 !5 = !{i32* null}
-; CHECK-DAG: .section	ccc,"aw",@progbits
+; CHECK-DAG: .section	ccc,"awo",@progbits,0,unique,3
 
 ; Null metadata.
 @m = global i32 1, section "ddd", !associated !6
 !6 = distinct !{null}
-; CHECK-DAG: .section	ddd,"aw",@progbits
+; CHECK-DAG: .section	ddd,"awo",@progbits,0,unique,4
 
 ; Aliases are OK.
 @n = alias i32, i32* inttoptr (i64 add (i64 ptrtoint (i32* @a to i64), i64 1297036692682702848) to i32*)
 @o = global i32 1, section "eee", !associated !7
 !7 = !{i32* @n}
-; CHECK-DAG: .section	eee,"awo",@progbits,n,unique,3
+; CHECK-DAG: .section	eee,"awo",@progbits,n,unique,5
diff --git a/llvm/test/MC/ELF/section-linkorder.s b/llvm/test/MC/ELF/section-linkorder.s
new file mode 100644
index 0000000000000..a0f6357e52cb1
--- /dev/null
+++ b/llvm/test/MC/ELF/section-linkorder.s
@@ -0,0 +1,8 @@
+# RUN: llvm-mc -triple x86_64 %s | FileCheck %s --check-prefix=ASM
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o %t
+# RUN: llvm-readelf -S %t | FileCheck %s
+
+# ASM:   .section .linkorder,"ao",@progbits,0
+# CHECK: Name       Type     {{.*}} Flg Lk
+# CHECK: .linkorder PROGBITS {{.*}}  AL  0
+.section .linkorder,"ao",@progbits,0

From 21de4e74acf603f02f886a9e6030945f077bca3f Mon Sep 17 00:00:00 2001
From: David Green <david.green@arm.com>
Date: Mon, 3 Aug 2020 21:46:07 +0100
Subject: [PATCH 0951/1035] [ARM] Test for converting VPSEL to VMOVT. NFC

---
 .../test/CodeGen/Thumb2/mve-pred-vctpvpsel.ll | 104 ++++++++++++++++++
 1 file changed, 104 insertions(+)
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-pred-vctpvpsel.ll

diff --git a/llvm/test/CodeGen/Thumb2/mve-pred-vctpvpsel.ll b/llvm/test/CodeGen/Thumb2/mve-pred-vctpvpsel.ll
new file mode 100644
index 0000000000000..054499795ff7a
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-pred-vctpvpsel.ll
@@ -0,0 +1,104 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve.fp %s -verify-machineinstrs -o - | FileCheck %s
+
+define void @arm_min_helium_f32(float* %pSrc, i32 %blockSize, float* nocapture %pResult, i32* nocapture %pIndex) {
+; CHECK-LABEL: arm_min_helium_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, r6, r7, lr}
+; CHECK-NEXT:    push {r4, r6, r7, lr}
+; CHECK-NEXT:    .vsave {d8, d9}
+; CHECK-NEXT:    vpush {d8, d9}
+; CHECK-NEXT:    movs r6, #0
+; CHECK-NEXT:    mov r12, r1
+; CHECK-NEXT:    vidup.u32 q2, r6, #1
+; CHECK-NEXT:    cmp r1, #4
+; CHECK-NEXT:    it ge
+; CHECK-NEXT:    movge.w r12, #4
+; CHECK-NEXT:    sub.w r6, r1, r12
+; CHECK-NEXT:    adds r6, #3
+; CHECK-NEXT:    mov.w lr, #1
+; CHECK-NEXT:    adr r4, .LCPI0_0
+; CHECK-NEXT:    vmov.i32 q0, #0x0
+; CHECK-NEXT:    add.w lr, lr, r6, lsr #2
+; CHECK-NEXT:    vldrw.u32 q1, [r4]
+; CHECK-NEXT:    vmov.i32 q3, #0x4
+; CHECK-NEXT:    mov r12, r1
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB0_1: @ %do.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vctp.32 r12
+; CHECK-NEXT:    sub.w r12, r12, #4
+; CHECK-NEXT:    vpstt
+; CHECK-NEXT:    vldrwt.u32 q4, [r0], #16
+; CHECK-NEXT:    vcmpt.f32 ge, q1, q4
+; CHECK-NEXT:    vpsel q0, q2, q0
+; CHECK-NEXT:    vpsel q1, q4, q1
+; CHECK-NEXT:    vadd.i32 q2, q2, q3
+; CHECK-NEXT:    le lr, .LBB0_1
+; CHECK-NEXT:  @ %bb.2: @ %do.end
+; CHECK-NEXT:    vldr s8, .LCPI0_1
+; CHECK-NEXT:    vdup.32 q3, r1
+; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    vminnmv.f32 r0, q1
+; CHECK-NEXT:    vcmp.f32 le, q1, r0
+; CHECK-NEXT:    vmov s8, r0
+; CHECK-NEXT:    vpsel q0, q0, q3
+; CHECK-NEXT:    vminv.u32 r1, q0
+; CHECK-NEXT:    str r1, [r3]
+; CHECK-NEXT:    vstr s8, [r2]
+; CHECK-NEXT:    vpop {d8, d9}
+; CHECK-NEXT:    pop {r4, r6, r7, pc}
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  @ %bb.3:
+; CHECK-NEXT:  .LCPI0_0:
+; CHECK-NEXT:    .long 0x5368d4a5 @ float 9.99999995E+11
+; CHECK-NEXT:    .long 0x5368d4a5 @ float 9.99999995E+11
+; CHECK-NEXT:    .long 0x5368d4a5 @ float 9.99999995E+11
+; CHECK-NEXT:    .long 0x5368d4a5 @ float 9.99999995E+11
+; CHECK-NEXT:  .LCPI0_1:
+; CHECK-NEXT:    .long 0x5368d4a5 @ float 9.99999995E+11
+entry:
+  %0 = tail call { <4 x i32>, i32 } @llvm.arm.mve.vidup.v4i32(i32 0, i32 1)
+  %1 = extractvalue { <4 x i32>, i32 } %0, 0
+  br label %do.body
+
+do.body:                                          ; preds = %do.body, %entry
+  %curExtremValVec.0 = phi <4 x float> [ <float 0x426D1A94A0000000, float 0x426D1A94A0000000, float 0x426D1A94A0000000, float 0x426D1A94A0000000>, %entry ], [ %8, %do.body ]
+  %indexVec.0 = phi <4 x i32> [ %1, %entry ], [ %11, %do.body ]
+  %2 = phi <4 x float> [ zeroinitializer, %entry ], [ %10, %do.body ]
+  %blkCnt.0 = phi i32 [ %blockSize, %entry ], [ %sub, %do.body ]
+  %pSrc.addr.0 = phi float* [ %pSrc, %entry ], [ %add.ptr, %do.body ]
+  %3 = tail call <4 x i1> @llvm.arm.mve.vctp32(i32 %blkCnt.0)
+  %4 = bitcast float* %pSrc.addr.0 to <4 x float>*
+  %5 = tail call fast <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %4, i32 4, <4 x i1> %3, <4 x float> zeroinitializer)
+  %6 = fcmp fast ole <4 x float> %5, %curExtremValVec.0
+  %7 = and <4 x i1> %6, %3
+  %8 = select fast <4 x i1> %7, <4 x float> %5, <4 x float> %curExtremValVec.0
+  %9 = bitcast <4 x i32> %indexVec.0 to <4 x float>
+  %10 = select fast <4 x i1> %7, <4 x float> %9, <4 x float> %2
+  %11 = add <4 x i32> %indexVec.0, <i32 4, i32 4, i32 4, i32 4>
+  %add.ptr = getelementptr inbounds float, float* %pSrc.addr.0, i32 4
+  %sub = add nsw i32 %blkCnt.0, -4
+  %cmp = icmp sgt i32 %blkCnt.0, 4
+  br i1 %cmp, label %do.body, label %do.end
+
+do.end:                                           ; preds = %do.body
+  %12 = bitcast <4 x float> %10 to <4 x i32>
+  %13 = tail call fast float @llvm.arm.mve.minnmv.f32.v4f32(float 0x426D1A94A0000000, <4 x float> %8)
+  %.splatinsert = insertelement <4 x float> undef, float %13, i32 0
+  %.splat = shufflevector <4 x float> %.splatinsert, <4 x float> undef, <4 x i32> zeroinitializer
+  %14 = fcmp fast ole <4 x float> %8, %.splat
+  %.splatinsert1 = insertelement <4 x i32> undef, i32 %blockSize, i32 0
+  %.splat2 = shufflevector <4 x i32> %.splatinsert1, <4 x i32> undef, <4 x i32> zeroinitializer
+  %15 = select <4 x i1> %14, <4 x i32> %12, <4 x i32> %.splat2
+  %16 = tail call i32 @llvm.arm.mve.minv.v4i32(i32 %blockSize, <4 x i32> %15, i32 1)
+  store i32 %16, i32* %pIndex, align 4
+  store float %13, float* %pResult, align 4
+  ret void
+}
+
+declare { <4 x i32>, i32 } @llvm.arm.mve.vidup.v4i32(i32, i32) #1
+declare <4 x i1> @llvm.arm.mve.vctp32(i32) #1
+declare <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>*, i32 immarg, <4 x i1>, <4 x float>) #2
+declare float @llvm.arm.mve.minnmv.f32.v4f32(float, <4 x float>) #1
+declare i32 @llvm.arm.mve.minv.v4i32(i32, <4 x i32>, i32) #1

From 66e7dce714fabd3ddb1aed635e4b826476d4f1a2 Mon Sep 17 00:00:00 2001
From: Mitch Phillips <31459023+hctim@users.noreply.github.com>
Date: Mon, 3 Aug 2020 13:48:30 -0700
Subject: [PATCH 0952/1035] Revert "[X86][SSE] Shuffle combine blends to
 OR(X,Y) if the relevant elements are known zero."

This reverts commit 219f32f4b68679563443cdaae7b8174c9976409a.

Commit contains unsigned compasions that break bots that build with
-Wsign-compare.
---
 llvm/lib/Target/X86/X86ISelLowering.cpp       | 65 +++++--------------
 llvm/test/CodeGen/X86/insertelement-ones.ll   | 12 ++--
 llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll |  8 +--
 .../test/CodeGen/X86/vector-shuffle-128-v8.ll |  5 +-
 .../CodeGen/X86/vector-shuffle-256-v32.ll     |  6 +-
 5 files changed, 35 insertions(+), 61 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index b2bfcc2698f4d..e9bb50aacec0e 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -7401,8 +7401,8 @@ static bool getFauxShuffleMask(SDValue N, const APInt &DemandedElts,
 
     // Handle OR(SHUFFLE,SHUFFLE) case where one source is zero and the other
     // is a valid shuffle index.
-    SDValue N0 = peekThroughBitcasts(N.getOperand(0));
-    SDValue N1 = peekThroughBitcasts(N.getOperand(1));
+    SDValue N0 = peekThroughOneUseBitcasts(N.getOperand(0));
+    SDValue N1 = peekThroughOneUseBitcasts(N.getOperand(1));
     if (!N0.getValueType().isVector() || !N1.getValueType().isVector())
       return false;
     SmallVector<int, 64> SrcMask0, SrcMask1;
@@ -7413,24 +7413,34 @@ static bool getFauxShuffleMask(SDValue N, const APInt &DemandedElts,
                                 true))
       return false;
 
+    // Shuffle inputs must be the same size as the result.
+    if (llvm::any_of(SrcInputs0, [VT](SDValue Op) {
+          return VT.getSizeInBits() != Op.getValueSizeInBits();
+        }))
+      return false;
+    if (llvm::any_of(SrcInputs1, [VT](SDValue Op) {
+          return VT.getSizeInBits() != Op.getValueSizeInBits();
+        }))
+      return false;
+
     size_t MaskSize = std::max(SrcMask0.size(), SrcMask1.size());
     SmallVector<int, 64> Mask0, Mask1;
     narrowShuffleMaskElts(MaskSize / SrcMask0.size(), SrcMask0, Mask0);
     narrowShuffleMaskElts(MaskSize / SrcMask1.size(), SrcMask1, Mask1);
-    for (int i = 0; i != (int)MaskSize; ++i) {
+    for (size_t i = 0; i != MaskSize; ++i) {
       if (Mask0[i] == SM_SentinelUndef && Mask1[i] == SM_SentinelUndef)
         Mask.push_back(SM_SentinelUndef);
       else if (Mask0[i] == SM_SentinelZero && Mask1[i] == SM_SentinelZero)
         Mask.push_back(SM_SentinelZero);
       else if (Mask1[i] == SM_SentinelZero)
-        Mask.push_back(i);
+        Mask.push_back(Mask0[i]);
       else if (Mask0[i] == SM_SentinelZero)
-        Mask.push_back(i + MaskSize);
+        Mask.push_back(Mask1[i] + (int)(MaskSize * SrcInputs0.size()));
       else
         return false;
     }
-    Ops.push_back(N0);
-    Ops.push_back(N1);
+    Ops.append(SrcInputs0.begin(), SrcInputs0.end());
+    Ops.append(SrcInputs1.begin(), SrcInputs1.end());
     return true;
   }
   case ISD::INSERT_SUBVECTOR: {
@@ -34209,7 +34219,6 @@ static bool matchBinaryShuffle(MVT MaskVT, ArrayRef<int> Mask,
                                SelectionDAG &DAG, const X86Subtarget &Subtarget,
                                unsigned &Shuffle, MVT &SrcVT, MVT &DstVT,
                                bool IsUnary) {
-  unsigned NumMaskElts = Mask.size();
   unsigned EltSizeInBits = MaskVT.getScalarSizeInBits();
 
   if (MaskVT.is128BitVector()) {
@@ -34267,46 +34276,6 @@ static bool matchBinaryShuffle(MVT MaskVT, ArrayRef<int> Mask,
     }
   }
 
-  // Attempt to match against a OR if we're performing a blend shuffle and the
-  // non-blended source element is zero in each case.
-  if ((EltSizeInBits % V1.getScalarValueSizeInBits()) == 0 &&
-      (EltSizeInBits % V2.getScalarValueSizeInBits()) == 0) {
-    bool IsBlend = true;
-    unsigned NumV1Elts = V1.getValueType().getVectorNumElements();
-    unsigned NumV2Elts = V2.getValueType().getVectorNumElements();
-    unsigned Scale1 = NumV1Elts / NumMaskElts;
-    unsigned Scale2 = NumV2Elts / NumMaskElts;
-    APInt DemandedZeroV1 = APInt::getNullValue(NumV1Elts);
-    APInt DemandedZeroV2 = APInt::getNullValue(NumV2Elts);
-    for (unsigned i = 0; i != NumMaskElts; ++i) {
-      int M = Mask[i];
-      if (M == SM_SentinelUndef)
-        continue;
-      if (M == SM_SentinelZero) {
-        DemandedZeroV1.setBits(i * Scale1, (i + 1) * Scale1);
-        DemandedZeroV2.setBits(i * Scale2, (i + 1) * Scale2);
-        continue;
-      }
-      if (M == i) {
-        DemandedZeroV2.setBits(i * Scale2, (i + 1) * Scale2);
-        continue;
-      }
-      if (M == (i + NumMaskElts)) {
-        DemandedZeroV1.setBits(i * Scale1, (i + 1) * Scale1);
-        continue;
-      }
-      IsBlend = false;
-      break;
-    }
-    if (IsBlend &&
-        DAG.computeKnownBits(V1, DemandedZeroV1).isZero() &&
-        DAG.computeKnownBits(V2, DemandedZeroV2).isZero()) {
-      Shuffle = ISD::OR;
-      SrcVT = DstVT = EVT(MaskVT).changeTypeToInteger().getSimpleVT();
-      return true;
-    }
-  }
-
   return false;
 }
 
diff --git a/llvm/test/CodeGen/X86/insertelement-ones.ll b/llvm/test/CodeGen/X86/insertelement-ones.ll
index 6a9a401264c56..3d8e42b9c07d0 100644
--- a/llvm/test/CodeGen/X86/insertelement-ones.ll
+++ b/llvm/test/CodeGen/X86/insertelement-ones.ll
@@ -389,9 +389,11 @@ define <32 x i8> @insert_v32i8_x123456789ABCDEzGHIJKLMNOPQRSTxx(<32 x i8> %a) {
 ; SSE2-NEXT:    movdqa %xmm3, %xmm4
 ; SSE2-NEXT:    pslldq {{.*#+}} xmm4 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm4[0]
 ; SSE2-NEXT:    por %xmm4, %xmm0
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255]
+; SSE2-NEXT:    pand %xmm5, %xmm1
 ; SSE2-NEXT:    pslldq {{.*#+}} xmm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm3[0,1]
-; SSE2-NEXT:    por %xmm3, %xmm1
+; SSE2-NEXT:    pandn %xmm3, %xmm5
+; SSE2-NEXT:    por %xmm5, %xmm1
 ; SSE2-NEXT:    pand %xmm2, %xmm1
 ; SSE2-NEXT:    por %xmm4, %xmm1
 ; SSE2-NEXT:    retq
@@ -409,9 +411,11 @@ define <32 x i8> @insert_v32i8_x123456789ABCDEzGHIJKLMNOPQRSTxx(<32 x i8> %a) {
 ; SSE3-NEXT:    movdqa %xmm3, %xmm4
 ; SSE3-NEXT:    pslldq {{.*#+}} xmm4 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm4[0]
 ; SSE3-NEXT:    por %xmm4, %xmm0
-; SSE3-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE3-NEXT:    movdqa {{.*#+}} xmm5 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255]
+; SSE3-NEXT:    pand %xmm5, %xmm1
 ; SSE3-NEXT:    pslldq {{.*#+}} xmm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm3[0,1]
-; SSE3-NEXT:    por %xmm3, %xmm1
+; SSE3-NEXT:    pandn %xmm3, %xmm5
+; SSE3-NEXT:    por %xmm5, %xmm1
 ; SSE3-NEXT:    pand %xmm2, %xmm1
 ; SSE3-NEXT:    por %xmm4, %xmm1
 ; SSE3-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll b/llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll
index 9256a43f8e339..6b49f22f21f1f 100644
--- a/llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll
+++ b/llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll
@@ -1314,10 +1314,10 @@ define void @trunc_v4i64_to_v4i8(<32 x i8>* %L, <4 x i8>* %S) nounwind {
 define <16 x i8> @negative(<32 x i8> %v, <32 x i8> %w) nounwind {
 ; AVX1-LABEL: negative:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX1-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,u,u,u,u,u,u,u,0,2,4,6,8,10,12,14]
-; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4,5,6,7]
+; AVX1-NEXT:    vpshufb {{.*#+}} xmm2 = xmm0[u,2,4,6,8,10,12,14],zero,zero,zero,zero,zero,zero,zero,zero
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u],zero,zero,zero,zero,zero,zero,zero,xmm0[0,2,4,6,8,10,12,14]
+; AVX1-NEXT:    vpor %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
 ; AVX1-NEXT:    vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
 ; AVX1-NEXT:    vzeroupper
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-128-v8.ll b/llvm/test/CodeGen/X86/vector-shuffle-128-v8.ll
index 86423ce76065b..f448f41cf522e 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-128-v8.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-128-v8.ll
@@ -1713,8 +1713,9 @@ define <8 x i16> @shuffle_v8i16_XX4X8acX(<8 x i16> %a, <8 x i16> %b) {
 ;
 ; SSSE3-LABEL: shuffle_v8i16_XX4X8acX:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[u,u,u,u,u,u,u,u,0,1,4,5,8,9,u,u]
-; SSSE3-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[2,3]
+; SSSE3-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,8,9,u,u],zero,zero,zero,zero,zero,zero,xmm0[u,u]
+; SSSE3-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[u,u,u,u],zero,zero,xmm1[u,u,0,1,4,5,8,9,u,u]
+; SSSE3-NEXT:    por %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: shuffle_v8i16_XX4X8acX:
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-256-v32.ll b/llvm/test/CodeGen/X86/vector-shuffle-256-v32.ll
index e5285aebda69e..82df05e5ae068 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-256-v32.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-256-v32.ll
@@ -3358,9 +3358,9 @@ define <32 x i8> @shuffle_v32i8_42_45_12_13_35_35_60_40_17_22_29_44_33_12_48_51_
 ; AVX1-NEXT:    vpshufb {{.*#+}} xmm2 = zero,zero,xmm2[u,u],zero,zero,xmm2[12],zero,xmm2[u,u,u],zero,zero,xmm2[u,0,3]
 ; AVX1-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[10,13,u,u,3,3],zero,xmm1[8,u,u,u,12,1,u],zero,zero
 ; AVX1-NEXT:    vpor %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpshufb {{.*#+}} xmm2 = xmm4[u,u,u,u,u,u,u,u,1,6,13,u,u,u,u,u]
-; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,12,13,u,u,u,u,u,u,u,u,u,12,u,u]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4,5],xmm0[6,7]
+; AVX1-NEXT:    vpshufb {{.*#+}} xmm2 = xmm4[u,u],zero,zero,xmm4[u,u,u,u,1,6,13,u,u],zero,xmm4[u,u]
+; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,12,13,u,u,u,u],zero,zero,zero,xmm0[u,u,12,u,u]
+; AVX1-NEXT:    vpor %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [255,255,0,0,255,255,255,255,0,0,0,255,255,0,255,255]
 ; AVX1-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0

From cb327922101b28ea70ec68d7f026da0e5e388eed Mon Sep 17 00:00:00 2001
From: Thomas Lively <tlively@google.com>
Date: Mon, 3 Aug 2020 13:54:00 -0700
Subject: [PATCH 0953/1035] [WebAssembly] Implement prototype
 v128.load{32,64}_zero instructions

Specified in https://github.com/WebAssembly/simd/pull/237, these
instructions load the first vector lane from memory and zero the other
lanes. Since these instructions are not officially part of the SIMD
proposal, they are only available on an opt-in basis via LLVM
intrinsics and clang builtin functions. If these instructions are
merged to the proposal, this implementation will change so that the
instructions will be generated from normal IR. At that point the
intrinsics and builtin functions would be removed.

This PR also changes the opcodes for the experimental f32x4.qfm{a,s}
instructions because their opcodes conflicted with those of the
v128.load{32,64}_zero instructions. The new opcodes were chosen to
match those used in V8.

Differential Revision: https://reviews.llvm.org/D84820
---
 .../clang/Basic/BuiltinsWebAssembly.def       |   3 +
 clang/lib/CodeGen/CGBuiltin.cpp               |  10 +
 clang/test/CodeGen/builtins-wasm.c            |  12 +
 llvm/include/llvm/IR/IntrinsicsWebAssembly.td |  14 ++
 .../MCTargetDesc/WebAssemblyMCTargetDesc.h    |   2 +
 .../WebAssembly/WebAssemblyISelLowering.cpp   |   9 +
 .../WebAssembly/WebAssemblyInstrMemory.td     |   2 +-
 .../WebAssembly/WebAssemblyInstrSIMD.td       |  50 +++-
 .../WebAssembly/simd-load-zero-offset.ll      | 228 ++++++++++++++++++
 llvm/test/MC/WebAssembly/simd-encodings.s     |  16 +-
 10 files changed, 334 insertions(+), 12 deletions(-)
 create mode 100644 llvm/test/CodeGen/WebAssembly/simd-load-zero-offset.ll

diff --git a/clang/include/clang/Basic/BuiltinsWebAssembly.def b/clang/include/clang/Basic/BuiltinsWebAssembly.def
index d0f40f991a4c7..39f29740cf56d 100644
--- a/clang/include/clang/Basic/BuiltinsWebAssembly.def
+++ b/clang/include/clang/Basic/BuiltinsWebAssembly.def
@@ -169,5 +169,8 @@ TARGET_BUILTIN(__builtin_wasm_narrow_u_i8x16_i16x8, "V16cV8sV8s", "nc", "simd128
 TARGET_BUILTIN(__builtin_wasm_narrow_s_i16x8_i32x4, "V8sV4iV4i", "nc", "simd128")
 TARGET_BUILTIN(__builtin_wasm_narrow_u_i16x8_i32x4, "V8sV4iV4i", "nc", "simd128")
 
+TARGET_BUILTIN(__builtin_wasm_load32_zero, "V4ii*", "nU", "simd128")
+TARGET_BUILTIN(__builtin_wasm_load64_zero, "V2LLiLLi*", "nU", "simd128")
+
 #undef BUILTIN
 #undef TARGET_BUILTIN
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index 042b41a09f198..2ef164b8b65ab 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -16497,6 +16497,16 @@ Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID,
         CGM.getIntrinsic(IntNo, {ConvertType(E->getType()), Low->getType()});
     return Builder.CreateCall(Callee, {Low, High});
   }
+  case WebAssembly::BI__builtin_wasm_load32_zero: {
+    Value *Ptr = EmitScalarExpr(E->getArg(0));
+    Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_load32_zero);
+    return Builder.CreateCall(Callee, {Ptr});
+  }
+  case WebAssembly::BI__builtin_wasm_load64_zero: {
+    Value *Ptr = EmitScalarExpr(E->getArg(0));
+    Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_load64_zero);
+    return Builder.CreateCall(Callee, {Ptr});
+  }
   case WebAssembly::BI__builtin_wasm_shuffle_v8x16: {
     Value *Ops[18];
     size_t OpIdx = 0;
diff --git a/clang/test/CodeGen/builtins-wasm.c b/clang/test/CodeGen/builtins-wasm.c
index 0f66fceef4cca..14e0d0ac65ed2 100644
--- a/clang/test/CodeGen/builtins-wasm.c
+++ b/clang/test/CodeGen/builtins-wasm.c
@@ -737,6 +737,18 @@ i16x8 narrow_u_i16x8_i32x4(i32x4 low, i32x4 high) {
   // WEBASSEMBLY: ret
 }
 
+i32x4 load32_zero(int *p) {
+  return __builtin_wasm_load32_zero(p);
+  // WEBASSEMBLY: call <4 x i32> @llvm.wasm.load32.zero(i32* %p)
+  // WEBASSEMBLY: ret
+}
+
+i64x2 load64_zero(long long *p) {
+  return __builtin_wasm_load64_zero(p);
+  // WEBASSEMBLY: call <2 x i64> @llvm.wasm.load64.zero(i64* %p)
+  // WEBASSEMBLY: ret
+}
+
 i8x16 swizzle_v8x16(i8x16 x, i8x16 y) {
   return __builtin_wasm_swizzle_v8x16(x, y);
   // WEBASSEMBLY: call <16 x i8> @llvm.wasm.swizzle(<16 x i8> %x, <16 x i8> %y)
diff --git a/llvm/include/llvm/IR/IntrinsicsWebAssembly.td b/llvm/include/llvm/IR/IntrinsicsWebAssembly.td
index 9cc9f9eb6f187..627a579ae5065 100644
--- a/llvm/include/llvm/IR/IntrinsicsWebAssembly.td
+++ b/llvm/include/llvm/IR/IntrinsicsWebAssembly.td
@@ -190,6 +190,20 @@ def int_wasm_nearest :
             [LLVMMatchType<0>],
             [IntrNoMem, IntrSpeculatable]>;
 
+// TODO: Replace these intrinsic with normal ISel patterns once the
+// load_zero instructions are merged to the proposal.
+def int_wasm_load32_zero :
+  Intrinsic<[llvm_v4i32_ty],
+            [LLVMPointerType<llvm_i32_ty>],
+            [IntrReadMem, IntrArgMemOnly],
+             "", [SDNPMemOperand]>;
+
+def int_wasm_load64_zero :
+  Intrinsic<[llvm_v2i64_ty],
+            [LLVMPointerType<llvm_i64_ty>],
+            [IntrReadMem, IntrArgMemOnly],
+             "", [SDNPMemOperand]>;
+
 //===----------------------------------------------------------------------===//
 // Thread-local storage intrinsics
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.h b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.h
index 02b310628ee17..631e96dd9246b 100644
--- a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.h
+++ b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.h
@@ -232,6 +232,7 @@ inline unsigned GetDefaultP2AlignAny(unsigned Opc) {
   WASM_LOAD_STORE(ATOMIC_NOTIFY)
   WASM_LOAD_STORE(ATOMIC_WAIT_I32)
   WASM_LOAD_STORE(LOAD_SPLAT_v32x4)
+  WASM_LOAD_STORE(LOAD_ZERO_v4i32)
     return 2;
   WASM_LOAD_STORE(LOAD_I64)
   WASM_LOAD_STORE(LOAD_F64)
@@ -254,6 +255,7 @@ inline unsigned GetDefaultP2AlignAny(unsigned Opc) {
   WASM_LOAD_STORE(LOAD_EXTEND_U_v4i32)
   WASM_LOAD_STORE(LOAD_EXTEND_S_v2i64)
   WASM_LOAD_STORE(LOAD_EXTEND_U_v2i64)
+  WASM_LOAD_STORE(LOAD_ZERO_v2i64)
     return 3;
   WASM_LOAD_STORE(LOAD_V128)
   WASM_LOAD_STORE(STORE_V128)
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
index cdfbfe388abbf..5d377e8aa9bd1 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
@@ -675,6 +675,15 @@ bool WebAssemblyTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
     Info.align = Align(8);
     Info.flags = MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad;
     return true;
+  case Intrinsic::wasm_load32_zero:
+  case Intrinsic::wasm_load64_zero:
+    Info.opc = ISD::INTRINSIC_W_CHAIN;
+    Info.memVT = Intrinsic == Intrinsic::wasm_load32_zero ? MVT::i32 : MVT::i64;
+    Info.ptrVal = I.getArgOperand(0);
+    Info.offset = 0;
+    Info.align = Info.memVT == MVT::i32 ? Align(4) : Align(8);
+    Info.flags = MachineMemOperand::MOLoad;
+    return true;
   default:
     return false;
   }
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyInstrMemory.td b/llvm/lib/Target/WebAssembly/WebAssemblyInstrMemory.td
index b3c63cc1f884b..48b934457267e 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyInstrMemory.td
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyInstrMemory.td
@@ -70,7 +70,7 @@ defm LOAD_F64 : WebAssemblyLoad<F64, "f64.load", 0x2b, []>;
 multiclass LoadPatNoOffset<ValueType ty, PatFrag kind, string inst> {
   def : Pat<(ty (kind I32:$addr)), (!cast<NI>(inst # "_A32") 0, 0, I32:$addr)>,
         Requires<[HasAddr32]>;
-  def : Pat<(ty (kind I64:$addr)), (!cast<NI>(inst # "_A64") 0, 0, I64:$addr)>,
+  def : Pat<(ty (kind (i64 I64:$addr))), (!cast<NI>(inst # "_A64") 0, 0, I64:$addr)>,
         Requires<[HasAddr64]>;
 }
 
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td b/llvm/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td
index 9bbccecffaa09..cd088751bc8a9 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td
@@ -163,6 +163,43 @@ defm : LoadPatGlobalAddrOffOnly<types[0], !cast<PatFrag>(exts[0]#types[1]),
                                 "LOAD_EXTEND"#exts[1]#"_"#types[0]>;
 }
 
+// Load lane into zero vector
+multiclass SIMDLoadZero<ValueType vec_t, string name, bits<32> simdop> {
+  let mayLoad = 1, UseNamedOperandTable = 1 in {
+  defm LOAD_ZERO_#vec_t#_A32 :
+    SIMD_I<(outs V128:$dst),
+           (ins P2Align:$p2align, offset32_op:$off, I32:$addr),
+           (outs), (ins P2Align:$p2align, offset32_op:$off), [],
+           name#"\t$dst, ${off}(${addr})$p2align",
+           name#"\t$off$p2align", simdop>;
+  defm LOAD_ZERO_#vec_t#_A64 :
+    SIMD_I<(outs V128:$dst),
+           (ins P2Align:$p2align, offset64_op:$off, I64:$addr),
+           (outs), (ins P2Align:$p2align, offset64_op:$off), [],
+           name#"\t$dst, ${off}(${addr})$p2align",
+           name#"\t$off$p2align", simdop>;
+  } // mayLoad = 1, UseNamedOperandTable = 1
+}
+
+// TODO: Also support v4f32 and v2f64 once the instructions are merged
+// to the proposal
+defm "" : SIMDLoadZero<v4i32, "v128.load32_zero", 252>;
+defm "" : SIMDLoadZero<v2i64, "v128.load64_zero", 253>;
+
+defm : LoadPatNoOffset<v4i32, int_wasm_load32_zero, "LOAD_ZERO_v4i32">;
+defm : LoadPatNoOffset<v2i64, int_wasm_load64_zero, "LOAD_ZERO_v2i64">;
+
+defm : LoadPatImmOff<v4i32, int_wasm_load32_zero, regPlusImm, "LOAD_ZERO_v4i32">;
+defm : LoadPatImmOff<v2i64, int_wasm_load64_zero, regPlusImm, "LOAD_ZERO_v2i64">;
+
+defm : LoadPatImmOff<v4i32, int_wasm_load32_zero, or_is_add, "LOAD_ZERO_v4i32">;
+defm : LoadPatImmOff<v2i64, int_wasm_load64_zero, or_is_add, "LOAD_ZERO_v2i64">;
+
+defm : LoadPatOffsetOnly<v4i32, int_wasm_load32_zero, "LOAD_ZERO_v4i32">;
+defm : LoadPatOffsetOnly<v2i64, int_wasm_load64_zero, "LOAD_ZERO_v2i64">;
+
+defm : LoadPatGlobalAddrOffOnly<v4i32, int_wasm_load32_zero, "LOAD_ZERO_v4i32">;
+defm : LoadPatGlobalAddrOffOnly<v2i64, int_wasm_load64_zero, "LOAD_ZERO_v2i64">;
 
 // Store: v128.store
 let mayStore = 1, UseNamedOperandTable = 1 in {
@@ -800,7 +837,7 @@ let isCommutable = 1 in
 defm DOT : SIMD_I<(outs V128:$dst), (ins V128:$lhs, V128:$rhs), (outs), (ins),
                   [(set V128:$dst, (int_wasm_dot V128:$lhs, V128:$rhs))],
                   "i32x4.dot_i16x8_s\t$dst, $lhs, $rhs", "i32x4.dot_i16x8_s",
-                  180>;
+                  186>;
 
 //===----------------------------------------------------------------------===//
 // Floating-point unary arithmetic
@@ -1038,20 +1075,21 @@ def : Pat<(t1 (bitconvert (t2 V128:$v))), (t1 V128:$v)>;
 // Quasi-Fused Multiply- Add and Subtract (QFMA/QFMS)
 //===----------------------------------------------------------------------===//
 
-multiclass SIMDQFM<ValueType vec_t, string vec, bits<32> baseInst> {
+multiclass SIMDQFM<ValueType vec_t, string vec, bits<32> simdopA,
+                   bits<32> simdopS> {
   defm QFMA_#vec_t :
     SIMD_I<(outs V128:$dst), (ins V128:$a, V128:$b, V128:$c),
            (outs), (ins),
            [(set (vec_t V128:$dst),
              (int_wasm_qfma (vec_t V128:$a), (vec_t V128:$b), (vec_t V128:$c)))],
-           vec#".qfma\t$dst, $a, $b, $c", vec#".qfma", baseInst>;
+           vec#".qfma\t$dst, $a, $b, $c", vec#".qfma", simdopA>;
   defm QFMS_#vec_t :
     SIMD_I<(outs V128:$dst), (ins V128:$a, V128:$b, V128:$c),
            (outs), (ins),
            [(set (vec_t V128:$dst),
              (int_wasm_qfms (vec_t V128:$a), (vec_t V128:$b), (vec_t V128:$c)))],
-           vec#".qfms\t$dst, $a, $b, $c", vec#".qfms", !add(baseInst, 1)>;
+           vec#".qfms\t$dst, $a, $b, $c", vec#".qfms", simdopS>;
 }
 
-defm "" : SIMDQFM<v4f32, "f32x4", 252>;
-defm "" : SIMDQFM<v2f64, "f64x2", 254>;
+defm "" : SIMDQFM<v4f32, "f32x4", 180, 212>;
+defm "" : SIMDQFM<v2f64, "f64x2", 254, 255>;
diff --git a/llvm/test/CodeGen/WebAssembly/simd-load-zero-offset.ll b/llvm/test/CodeGen/WebAssembly/simd-load-zero-offset.ll
new file mode 100644
index 0000000000000..ab3643653deb4
--- /dev/null
+++ b/llvm/test/CodeGen/WebAssembly/simd-load-zero-offset.ll
@@ -0,0 +1,228 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -verify-machineinstrs -mattr=+simd128 | FileCheck %s
+
+; Test SIMD v128.load{32,64}_zero instructions
+
+target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
+target triple = "wasm32-unknown-unknown"
+
+declare <4 x i32> @llvm.wasm.load32.zero(i32*)
+declare <2 x i64> @llvm.wasm.load64.zero(i64*)
+
+;===----------------------------------------------------------------------------
+; v128.load32_zero
+;===----------------------------------------------------------------------------
+
+define <4 x i32> @load_zero_i32_no_offset(i32* %p) {
+; CHECK-LABEL: load_zero_i32_no_offset:
+; CHECK:         .functype load_zero_i32_no_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.load32_zero 0
+; CHECK-NEXT:    # fallthrough-return
+  %v = tail call <4 x i32> @llvm.wasm.load32.zero(i32* %p)
+  ret <4 x i32> %v
+}
+
+define <4 x i32> @load_zero_i32_with_folded_offset(i32* %p) {
+; CHECK-LABEL: load_zero_i32_with_folded_offset:
+; CHECK:         .functype load_zero_i32_with_folded_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.load32_zero 24
+; CHECK-NEXT:    # fallthrough-return
+  %q = ptrtoint i32* %p to i32
+  %r = add nuw i32 %q, 24
+  %s = inttoptr i32 %r to i32*
+  %t = tail call <4 x i32> @llvm.wasm.load32.zero(i32* %s)
+  ret <4 x i32> %t
+}
+
+define <4 x i32> @load_zero_i32_with_folded_gep_offset(i32* %p) {
+; CHECK-LABEL: load_zero_i32_with_folded_gep_offset:
+; CHECK:         .functype load_zero_i32_with_folded_gep_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.load32_zero 24
+; CHECK-NEXT:    # fallthrough-return
+  %s = getelementptr inbounds i32, i32* %p, i32 6
+  %t = tail call <4 x i32> @llvm.wasm.load32.zero(i32* %s)
+  ret <4 x i32> %t
+}
+
+define <4 x i32> @load_zero_i32_with_unfolded_gep_negative_offset(i32* %p) {
+; CHECK-LABEL: load_zero_i32_with_unfolded_gep_negative_offset:
+; CHECK:         .functype load_zero_i32_with_unfolded_gep_negative_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const -24
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    v128.load32_zero 0
+; CHECK-NEXT:    # fallthrough-return
+  %s = getelementptr inbounds i32, i32* %p, i32 -6
+  %t = tail call <4 x i32> @llvm.wasm.load32.zero(i32* %s)
+  ret <4 x i32> %t
+}
+
+define <4 x i32> @load_zero_i32_with_unfolded_offset(i32* %p) {
+; CHECK-LABEL: load_zero_i32_with_unfolded_offset:
+; CHECK:         .functype load_zero_i32_with_unfolded_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const 24
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    v128.load32_zero 0
+; CHECK-NEXT:    # fallthrough-return
+  %q = ptrtoint i32* %p to i32
+  %r = add nsw i32 %q, 24
+  %s = inttoptr i32 %r to i32*
+  %t = tail call <4 x i32> @llvm.wasm.load32.zero(i32* %s)
+  ret <4 x i32> %t
+}
+
+define <4 x i32> @load_zero_i32_with_unfolded_gep_offset(i32* %p) {
+; CHECK-LABEL: load_zero_i32_with_unfolded_gep_offset:
+; CHECK:         .functype load_zero_i32_with_unfolded_gep_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const 24
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    v128.load32_zero 0
+; CHECK-NEXT:    # fallthrough-return
+  %s = getelementptr i32, i32* %p, i32 6
+  %t = tail call <4 x i32> @llvm.wasm.load32.zero(i32* %s)
+  ret <4 x i32> %t
+}
+
+define <4 x i32> @load_zero_i32_from_numeric_address() {
+; CHECK-LABEL: load_zero_i32_from_numeric_address:
+; CHECK:         .functype load_zero_i32_from_numeric_address () -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    v128.load32_zero 42
+; CHECK-NEXT:    # fallthrough-return
+  %s = inttoptr i32 42 to i32*
+  %t = tail call <4 x i32> @llvm.wasm.load32.zero(i32* %s)
+  ret <4 x i32> %t
+}
+
+@gv_i32 = global i32 0
+define <4 x i32> @load_zero_i32_from_global_address() {
+; CHECK-LABEL: load_zero_i32_from_global_address:
+; CHECK:         .functype load_zero_i32_from_global_address () -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    v128.load32_zero gv_i32
+; CHECK-NEXT:    # fallthrough-return
+  %t = tail call <4 x i32> @llvm.wasm.load32.zero(i32* @gv_i32)
+  ret <4 x i32> %t
+}
+
+;===----------------------------------------------------------------------------
+; v128.load64_zero
+;===----------------------------------------------------------------------------
+
+define <2 x i64> @load_zero_i64_no_offset(i64* %p) {
+; CHECK-LABEL: load_zero_i64_no_offset:
+; CHECK:         .functype load_zero_i64_no_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.load64_zero 0
+; CHECK-NEXT:    # fallthrough-return
+  %v = tail call <2 x i64> @llvm.wasm.load64.zero(i64* %p)
+  ret <2 x i64> %v
+}
+
+define <2 x i64> @load_zero_i64_with_folded_offset(i64* %p) {
+; CHECK-LABEL: load_zero_i64_with_folded_offset:
+; CHECK:         .functype load_zero_i64_with_folded_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.load64_zero 24
+; CHECK-NEXT:    # fallthrough-return
+  %q = ptrtoint i64* %p to i32
+  %r = add nuw i32 %q, 24
+  %s = inttoptr i32 %r to i64*
+  %t = tail call <2 x i64> @llvm.wasm.load64.zero(i64* %s)
+  ret <2 x i64> %t
+}
+
+define <2 x i64> @load_zero_i64_with_folded_gep_offset(i64* %p) {
+; CHECK-LABEL: load_zero_i64_with_folded_gep_offset:
+; CHECK:         .functype load_zero_i64_with_folded_gep_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.load64_zero 48
+; CHECK-NEXT:    # fallthrough-return
+  %s = getelementptr inbounds i64, i64* %p, i64 6
+  %t = tail call <2 x i64> @llvm.wasm.load64.zero(i64* %s)
+  ret <2 x i64> %t
+}
+
+define <2 x i64> @load_zero_i64_with_unfolded_gep_negative_offset(i64* %p) {
+; CHECK-LABEL: load_zero_i64_with_unfolded_gep_negative_offset:
+; CHECK:         .functype load_zero_i64_with_unfolded_gep_negative_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const -48
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    v128.load64_zero 0
+; CHECK-NEXT:    # fallthrough-return
+  %s = getelementptr inbounds i64, i64* %p, i64 -6
+  %t = tail call <2 x i64> @llvm.wasm.load64.zero(i64* %s)
+  ret <2 x i64> %t
+}
+
+define <2 x i64> @load_zero_i64_with_unfolded_offset(i64* %p) {
+; CHECK-LABEL: load_zero_i64_with_unfolded_offset:
+; CHECK:         .functype load_zero_i64_with_unfolded_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const 24
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    v128.load64_zero 0
+; CHECK-NEXT:    # fallthrough-return
+  %q = ptrtoint i64* %p to i32
+  %r = add nsw i32 %q, 24
+  %s = inttoptr i32 %r to i64*
+  %t = tail call <2 x i64> @llvm.wasm.load64.zero(i64* %s)
+  ret <2 x i64> %t
+}
+
+define <2 x i64> @load_zero_i64_with_unfolded_gep_offset(i64* %p) {
+; CHECK-LABEL: load_zero_i64_with_unfolded_gep_offset:
+; CHECK:         .functype load_zero_i64_with_unfolded_gep_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const 48
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    v128.load64_zero 0
+; CHECK-NEXT:    # fallthrough-return
+  %s = getelementptr i64, i64* %p, i64 6
+  %t = tail call <2 x i64> @llvm.wasm.load64.zero(i64* %s)
+  ret <2 x i64> %t
+}
+
+define <2 x i64> @load_zero_i64_from_numeric_address() {
+; CHECK-LABEL: load_zero_i64_from_numeric_address:
+; CHECK:         .functype load_zero_i64_from_numeric_address () -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    v128.load64_zero 42
+; CHECK-NEXT:    # fallthrough-return
+  %s = inttoptr i32 42 to i64*
+  %t = tail call <2 x i64> @llvm.wasm.load64.zero(i64* %s)
+  ret <2 x i64> %t
+}
+
+@gv_i64 = global i64 0
+define <2 x i64> @load_zero_i64_from_global_address() {
+; CHECK-LABEL: load_zero_i64_from_global_address:
+; CHECK:         .functype load_zero_i64_from_global_address () -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    v128.load64_zero gv_i64
+; CHECK-NEXT:    # fallthrough-return
+  %t = tail call <2 x i64> @llvm.wasm.load64.zero(i64* @gv_i64)
+  ret <2 x i64> %t
+}
diff --git a/llvm/test/MC/WebAssembly/simd-encodings.s b/llvm/test/MC/WebAssembly/simd-encodings.s
index 5818588380a1f..bb101caefeb6f 100644
--- a/llvm/test/MC/WebAssembly/simd-encodings.s
+++ b/llvm/test/MC/WebAssembly/simd-encodings.s
@@ -463,9 +463,6 @@ main:
     # CHECK: i32x4.sub # encoding: [0xfd,0xb1,0x01]
     i32x4.sub
 
-    # CHECK: i32x4.dot_i16x8_s # encoding: [0xfd,0xb4,0x01]
-    i32x4.dot_i16x8_s
-
     # CHECK: i32x4.mul # encoding: [0xfd,0xb5,0x01]
     i32x4.mul
 
@@ -481,6 +478,9 @@ main:
     # CHECK: i32x4.max_u # encoding: [0xfd,0xb9,0x01]
     i32x4.max_u
 
+    # CHECK: i32x4.dot_i16x8_s # encoding: [0xfd,0xba,0x01]
+    i32x4.dot_i16x8_s
+
     # CHECK: i64x2.neg # encoding: [0xfd,0xc1,0x01]
     i64x2.neg
 
@@ -610,10 +610,16 @@ main:
     # CHECK: f32x4.convert_i32x4_u # encoding: [0xfd,0xfb,0x01]
     f32x4.convert_i32x4_u
 
-    # CHECK: f32x4.qfma # encoding: [0xfd,0xfc,0x01]
+    # CHECK: v128.load32_zero 32 # encoding: [0xfd,0xfc,0x01,0x02,0x20]
+    v128.load32_zero 32
+
+    # CHECK: v128.load64_zero 32 # encoding: [0xfd,0xfd,0x01,0x03,0x20]
+    v128.load64_zero 32
+
+    # CHECK: f32x4.qfma # encoding: [0xfd,0xb4,0x01]
     f32x4.qfma
 
-    # CHECK: f32x4.qfms # encoding: [0xfd,0xfd,0x01]
+    # CHECK: f32x4.qfms # encoding: [0xfd,0xd4,0x01]
     f32x4.qfms
 
     # CHECK: f64x2.qfma # encoding: [0xfd,0xfe,0x01]

From 22916481c11e1d46132752086290a668e62fc9ce Mon Sep 17 00:00:00 2001
From: David Green <david.green@arm.com>
Date: Mon, 3 Aug 2020 22:03:14 +0100
Subject: [PATCH 0954/1035] [ARM] Convert VPSEL to VMOV in tail predicated
 loops

VPSEL has slightly different semantics under tail predication (it can
end up selecting from Qn, Qm and Qd). We do not model that at the moment
so they block tail predicated loops from being formed.

This just converts them into a predicated VMOV instead (via a VORR),
allowing tail predication to happen whilst still modelling the original
behaviour of the input.

Differential Revision: https://reviews.llvm.org/D85110
---
 .../Target/ARM/MVEVPTOptimisationsPass.cpp    | 43 +++++++++-
 .../cond-vector-reduce-mve-codegen.ll         |  6 +-
 .../test/CodeGen/Thumb2/mve-pred-selectop3.ll | 80 +++++++++++--------
 .../test/CodeGen/Thumb2/mve-pred-vctpvpsel.ll | 22 ++---
 llvm/test/CodeGen/Thumb2/mve-vctp.ll          | 21 ++---
 5 files changed, 110 insertions(+), 62 deletions(-)

diff --git a/llvm/lib/Target/ARM/MVEVPTOptimisationsPass.cpp b/llvm/lib/Target/ARM/MVEVPTOptimisationsPass.cpp
index 382ddd4572c74..8dbb8b53c8905 100644
--- a/llvm/lib/Target/ARM/MVEVPTOptimisationsPass.cpp
+++ b/llvm/lib/Target/ARM/MVEVPTOptimisationsPass.cpp
@@ -57,6 +57,7 @@ class MVEVPTOptimisations : public MachineFunctionPass {
                                             Register Target);
   bool ReduceOldVCCRValueUses(MachineBasicBlock &MBB);
   bool ReplaceVCMPsByVPNOTs(MachineBasicBlock &MBB);
+  bool ConvertVPSEL(MachineBasicBlock &MBB);
 };
 
 char MVEVPTOptimisations::ID = 0;
@@ -356,7 +357,7 @@ bool MVEVPTOptimisations::ReduceOldVCCRValueUses(MachineBasicBlock &MBB) {
   }
 
   for (MachineInstr *DeadInstruction : DeadInstructions)
-    DeadInstruction->removeFromParent();
+    DeadInstruction->eraseFromParent();
 
   return Modified;
 }
@@ -430,7 +431,44 @@ bool MVEVPTOptimisations::ReplaceVCMPsByVPNOTs(MachineBasicBlock &MBB) {
   }
 
   for (MachineInstr *DeadInstruction : DeadInstructions)
-    DeadInstruction->removeFromParent();
+    DeadInstruction->eraseFromParent();
+
+  return !DeadInstructions.empty();
+}
+
+// Replace VPSEL with a predicated VMOV in blocks with a VCTP. This is a
+// somewhat blunt approximation to allow tail predicated with vpsel
+// instructions. We turn a vselect into a VPSEL in ISEL, but they have slightly
+// different semantics under tail predication. Until that is modelled we just
+// convert to a VMOVT (via a predicated VORR) instead.
+bool MVEVPTOptimisations::ConvertVPSEL(MachineBasicBlock &MBB) {
+  bool HasVCTP = false;
+  SmallVector<MachineInstr *, 4> DeadInstructions;
+
+  for (MachineInstr &MI : MBB.instrs()) {
+    if (isVCTP(&MI)) {
+      HasVCTP = true;
+      continue;
+    }
+
+    if (!HasVCTP || MI.getOpcode() != ARM::MVE_VPSEL)
+      continue;
+
+    MachineInstrBuilder MIBuilder =
+        BuildMI(MBB, &MI, MI.getDebugLoc(), TII->get(ARM::MVE_VORR))
+            .add(MI.getOperand(0))
+            .add(MI.getOperand(1))
+            .add(MI.getOperand(1))
+            .addImm(ARMVCC::Then)
+            .add(MI.getOperand(4))
+            .add(MI.getOperand(2));
+    LLVM_DEBUG(dbgs() << "Replacing VPSEL: "; MI.dump();
+               dbgs() << "     with VMOVT: "; MIBuilder.getInstr()->dump());
+    DeadInstructions.push_back(&MI);
+  }
+
+  for (MachineInstr *DeadInstruction : DeadInstructions)
+    DeadInstruction->eraseFromParent();
 
   return !DeadInstructions.empty();
 }
@@ -452,6 +490,7 @@ bool MVEVPTOptimisations::runOnMachineFunction(MachineFunction &Fn) {
   for (MachineBasicBlock &MBB : Fn) {
     Modified |= ReplaceVCMPsByVPNOTs(MBB);
     Modified |= ReduceOldVCCRValueUses(MBB);
+    Modified |= ConvertVPSEL(MBB);
   }
 
   LLVM_DEBUG(dbgs() << "**************************************\n");
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/cond-vector-reduce-mve-codegen.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/cond-vector-reduce-mve-codegen.ll
index eff56a041ee27..d1151f29a9b20 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/cond-vector-reduce-mve-codegen.ll
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/cond-vector-reduce-mve-codegen.ll
@@ -23,14 +23,14 @@ define dso_local i32 @vpsel_mul_reduce_add(i32* noalias nocapture readonly %a, i
 ; CHECK-NEXT:    vctp.32 r3
 ; CHECK-NEXT:    and r4, r12, #15
 ; CHECK-NEXT:    vstr p0, [sp] @ 4-byte Spill
-; CHECK-NEXT:    vdup.32 q3, r4
 ; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    vpstt
 ; CHECK-NEXT:    vldrwt.u32 q1, [r2], #16
 ; CHECK-NEXT:    vldrwt.u32 q2, [r1], #16
-; CHECK-NEXT:    vcmp.i32 eq, q3, zr
+; CHECK-NEXT:    vdup.32 q3, r4
+; CHECK-NEXT:    vpt.i32 eq, q3, zr
+; CHECK-NEXT:    vmovt q1, q2
 ; CHECK-NEXT:    add.w r12, r12, #4
-; CHECK-NEXT:    vpsel q1, q2, q1
 ; CHECK-NEXT:    vldr p0, [sp] @ 4-byte Reload
 ; CHECK-NEXT:    vpst
 ; CHECK-NEXT:    vldrwt.u32 q2, [r0], #16
diff --git a/llvm/test/CodeGen/Thumb2/mve-pred-selectop3.ll b/llvm/test/CodeGen/Thumb2/mve-pred-selectop3.ll
index df92d30da6af4..bf43a4956a539 100644
--- a/llvm/test/CodeGen/Thumb2/mve-pred-selectop3.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-pred-selectop3.ll
@@ -1739,9 +1739,10 @@ define arm_aapcs_vfpcc <4 x i32> @icmp_slt_v4i32_y(<4 x i32> %x, <4 x i32> %y, i
 ; CHECK-LABEL: icmp_slt_v4i32_y:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vctp.32 r0
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpstt
 ; CHECK-NEXT:    vcmpt.s32 gt, q1, q0
-; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    vmovt q1, q0
+; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
@@ -1755,9 +1756,10 @@ define arm_aapcs_vfpcc <8 x i16> @icmp_slt_v8i16_y(<8 x i16> %x, <8 x i16> %y, i
 ; CHECK-LABEL: icmp_slt_v8i16_y:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vctp.16 r0
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpstt
 ; CHECK-NEXT:    vcmpt.s16 gt, q1, q0
-; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    vmovt q1, q0
+; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
@@ -1771,9 +1773,10 @@ define arm_aapcs_vfpcc <16 x i8> @icmp_slt_v16i8_y(<16 x i8> %x, <16 x i8> %y, i
 ; CHECK-LABEL: icmp_slt_v16i8_y:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vctp.8 r0
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpstt
 ; CHECK-NEXT:    vcmpt.s8 gt, q1, q0
-; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    vmovt q1, q0
+; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
@@ -1787,9 +1790,10 @@ define arm_aapcs_vfpcc <4 x i32> @icmp_sgt_v4i32_y(<4 x i32> %x, <4 x i32> %y, i
 ; CHECK-LABEL: icmp_sgt_v4i32_y:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vctp.32 r0
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpstt
 ; CHECK-NEXT:    vcmpt.s32 gt, q0, q1
-; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    vmovt q1, q0
+; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
@@ -1803,9 +1807,10 @@ define arm_aapcs_vfpcc <8 x i16> @icmp_sgt_v8i16_y(<8 x i16> %x, <8 x i16> %y, i
 ; CHECK-LABEL: icmp_sgt_v8i16_y:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vctp.16 r0
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpstt
 ; CHECK-NEXT:    vcmpt.s16 gt, q0, q1
-; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    vmovt q1, q0
+; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
@@ -1819,9 +1824,10 @@ define arm_aapcs_vfpcc <16 x i8> @icmp_sgt_v16i8_y(<16 x i8> %x, <16 x i8> %y, i
 ; CHECK-LABEL: icmp_sgt_v16i8_y:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vctp.8 r0
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpstt
 ; CHECK-NEXT:    vcmpt.s8 gt, q0, q1
-; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    vmovt q1, q0
+; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
@@ -1835,9 +1841,10 @@ define arm_aapcs_vfpcc <4 x i32> @icmp_ult_v4i32_y(<4 x i32> %x, <4 x i32> %y, i
 ; CHECK-LABEL: icmp_ult_v4i32_y:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vctp.32 r0
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpstt
 ; CHECK-NEXT:    vcmpt.u32 hi, q1, q0
-; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    vmovt q1, q0
+; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
@@ -1851,9 +1858,10 @@ define arm_aapcs_vfpcc <8 x i16> @icmp_ult_v8i16_y(<8 x i16> %x, <8 x i16> %y, i
 ; CHECK-LABEL: icmp_ult_v8i16_y:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vctp.16 r0
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpstt
 ; CHECK-NEXT:    vcmpt.u16 hi, q1, q0
-; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    vmovt q1, q0
+; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
@@ -1867,9 +1875,10 @@ define arm_aapcs_vfpcc <16 x i8> @icmp_ult_v16i8_y(<16 x i8> %x, <16 x i8> %y, i
 ; CHECK-LABEL: icmp_ult_v16i8_y:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vctp.8 r0
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpstt
 ; CHECK-NEXT:    vcmpt.u8 hi, q1, q0
-; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    vmovt q1, q0
+; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
@@ -1883,9 +1892,10 @@ define arm_aapcs_vfpcc <4 x i32> @icmp_ugt_v4i32_y(<4 x i32> %x, <4 x i32> %y, i
 ; CHECK-LABEL: icmp_ugt_v4i32_y:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vctp.32 r0
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpstt
 ; CHECK-NEXT:    vcmpt.u32 hi, q0, q1
-; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    vmovt q1, q0
+; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
@@ -1899,9 +1909,10 @@ define arm_aapcs_vfpcc <8 x i16> @icmp_ugt_v8i16_y(<8 x i16> %x, <8 x i16> %y, i
 ; CHECK-LABEL: icmp_ugt_v8i16_y:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vctp.16 r0
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpstt
 ; CHECK-NEXT:    vcmpt.u16 hi, q0, q1
-; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    vmovt q1, q0
+; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
@@ -1915,9 +1926,10 @@ define arm_aapcs_vfpcc <16 x i8> @icmp_ugt_v16i8_y(<16 x i8> %x, <16 x i8> %y, i
 ; CHECK-LABEL: icmp_ugt_v16i8_y:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vctp.8 r0
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpstt
 ; CHECK-NEXT:    vcmpt.u8 hi, q0, q1
-; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    vmovt q1, q0
+; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
@@ -1931,9 +1943,10 @@ define arm_aapcs_vfpcc <4 x float> @fcmp_fast_olt_v4f32_y(<4 x float> %x, <4 x f
 ; CHECK-LABEL: fcmp_fast_olt_v4f32_y:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vctp.32 r0
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpstt
 ; CHECK-NEXT:    vcmpt.f32 gt, q1, q0
-; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    vmovt q1, q0
+; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
@@ -1947,9 +1960,10 @@ define arm_aapcs_vfpcc <8 x half> @fcmp_fast_olt_v8f16_y(<8 x half> %x, <8 x hal
 ; CHECK-LABEL: fcmp_fast_olt_v8f16_y:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vctp.16 r0
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpstt
 ; CHECK-NEXT:    vcmpt.f16 gt, q1, q0
-; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    vmovt q1, q0
+; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
@@ -1963,9 +1977,10 @@ define arm_aapcs_vfpcc <4 x float> @fcmp_fast_ogt_v4f32_y(<4 x float> %x, <4 x f
 ; CHECK-LABEL: fcmp_fast_ogt_v4f32_y:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vctp.32 r0
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpstt
 ; CHECK-NEXT:    vcmpt.f32 gt, q0, q1
-; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    vmovt q1, q0
+; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
@@ -1979,9 +1994,10 @@ define arm_aapcs_vfpcc <8 x half> @fcmp_fast_ogt_v8f16_y(<8 x half> %x, <8 x hal
 ; CHECK-LABEL: fcmp_fast_ogt_v8f16_y:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vctp.16 r0
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpstt
 ; CHECK-NEXT:    vcmpt.f16 gt, q0, q1
-; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    vmovt q1, q0
+; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
diff --git a/llvm/test/CodeGen/Thumb2/mve-pred-vctpvpsel.ll b/llvm/test/CodeGen/Thumb2/mve-pred-vctpvpsel.ll
index 054499795ff7a..311a06a675771 100644
--- a/llvm/test/CodeGen/Thumb2/mve-pred-vctpvpsel.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-pred-vctpvpsel.ll
@@ -9,32 +9,22 @@ define void @arm_min_helium_f32(float* %pSrc, i32 %blockSize, float* nocapture %
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
 ; CHECK-NEXT:    movs r6, #0
-; CHECK-NEXT:    mov r12, r1
 ; CHECK-NEXT:    vidup.u32 q2, r6, #1
-; CHECK-NEXT:    cmp r1, #4
-; CHECK-NEXT:    it ge
-; CHECK-NEXT:    movge.w r12, #4
-; CHECK-NEXT:    sub.w r6, r1, r12
-; CHECK-NEXT:    adds r6, #3
-; CHECK-NEXT:    mov.w lr, #1
 ; CHECK-NEXT:    adr r4, .LCPI0_0
 ; CHECK-NEXT:    vmov.i32 q0, #0x0
-; CHECK-NEXT:    add.w lr, lr, r6, lsr #2
 ; CHECK-NEXT:    vldrw.u32 q1, [r4]
 ; CHECK-NEXT:    vmov.i32 q3, #0x4
 ; CHECK-NEXT:    mov r12, r1
-; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:    dlstp.32 lr, r12
 ; CHECK-NEXT:  .LBB0_1: @ %do.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vctp.32 r12
-; CHECK-NEXT:    sub.w r12, r12, #4
+; CHECK-NEXT:    vldrw.u32 q4, [r0], #16
+; CHECK-NEXT:    vcmp.f32 ge, q1, q4
 ; CHECK-NEXT:    vpstt
-; CHECK-NEXT:    vldrwt.u32 q4, [r0], #16
-; CHECK-NEXT:    vcmpt.f32 ge, q1, q4
-; CHECK-NEXT:    vpsel q0, q2, q0
-; CHECK-NEXT:    vpsel q1, q4, q1
+; CHECK-NEXT:    vmovt q1, q4
+; CHECK-NEXT:    vmovt q0, q2
 ; CHECK-NEXT:    vadd.i32 q2, q2, q3
-; CHECK-NEXT:    le lr, .LBB0_1
+; CHECK-NEXT:    letp lr, .LBB0_1
 ; CHECK-NEXT:  @ %bb.2: @ %do.end
 ; CHECK-NEXT:    vldr s8, .LCPI0_1
 ; CHECK-NEXT:    vdup.32 q3, r1
diff --git a/llvm/test/CodeGen/Thumb2/mve-vctp.ll b/llvm/test/CodeGen/Thumb2/mve-vctp.ll
index 67bc161e02c64..8cddbc79e2e13 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vctp.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vctp.ll
@@ -4,10 +4,11 @@
 define void @vctp8(i32 %arg, <16 x i8> *%in, <16 x i8>* %out) {
 ; CHECK-LABEL: vctp8:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vldrw.u32 q1, [r1]
-; CHECK-NEXT:    vctp.8 r0
 ; CHECK-NEXT:    vmov.i32 q0, #0x0
-; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vldrw.u32 q1, [r1]
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmovt q0, q1
 ; CHECK-NEXT:    vstrw.32 q0, [r2]
 ; CHECK-NEXT:    bx lr
   %pred = call <16 x i1> @llvm.arm.mve.vctp8(i32 %arg)
@@ -20,10 +21,11 @@ define void @vctp8(i32 %arg, <16 x i8> *%in, <16 x i8>* %out) {
 define void @vctp16(i32 %arg, <8 x i16> *%in, <8 x i16>* %out) {
 ; CHECK-LABEL: vctp16:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vldrw.u32 q1, [r1]
-; CHECK-NEXT:    vctp.16 r0
 ; CHECK-NEXT:    vmov.i32 q0, #0x0
-; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vldrw.u32 q1, [r1]
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmovt q0, q1
 ; CHECK-NEXT:    vstrw.32 q0, [r2]
 ; CHECK-NEXT:    bx lr
   %pred = call <8 x i1> @llvm.arm.mve.vctp16(i32 %arg)
@@ -36,10 +38,11 @@ define void @vctp16(i32 %arg, <8 x i16> *%in, <8 x i16>* %out) {
 define void @vctp32(i32 %arg, <4 x i32> *%in, <4 x i32>* %out) {
 ; CHECK-LABEL: vctp32:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vldrw.u32 q1, [r1]
-; CHECK-NEXT:    vctp.32 r0
 ; CHECK-NEXT:    vmov.i32 q0, #0x0
-; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vldrw.u32 q1, [r1]
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmovt q0, q1
 ; CHECK-NEXT:    vstrw.32 q0, [r2]
 ; CHECK-NEXT:    bx lr
   %pred = call <4 x i1> @llvm.arm.mve.vctp32(i32 %arg)

From 9a05fa10bd05525adedb6117351333699a3d4ae2 Mon Sep 17 00:00:00 2001
From: Mitch Phillips <31459023+hctim@users.noreply.github.com>
Date: Mon, 3 Aug 2020 13:55:27 -0700
Subject: [PATCH 0955/1035] [HWASan] [GlobalISel] Add +tagged-globals backend
 feature for GlobalISel

GlobalISel is the default ISel for aarch64 at -O0. Prior to D78465, GlobalISel
didn't have support for dealing with address-of-global lowerings, so it fell
back to SelectionDAGISel.

HWASan Globals require special handling, as they contain the pointer tag in the
top 16-bits, and are thus outside the code model. We need to generate a `movk`
in the instruction sequence with a G3 relocation to ensure the bits are
relocated properly. This is implemented in SelectionDAGISel, this patch does
the same for GlobalISel.

GlobalISel and SelectionDAGISel differ in their lowering sequence, so there are
differences in the final instruction sequence, explained in
`tagged-globals.ll`. Both of these implementations are correct, but GlobalISel
is slightly larger code size / slightly slower (by a couple of arithmetic
instructions). I don't see this as a problem for now as GlobalISel is only on
by default at `-O0`.

Reviewed By: aemerson, arsenm

Differential Revision: https://reviews.llvm.org/D82615
---
 .../hwasan/TestCases/exported-tagged-global.c | 16 +++
 .../AArch64/GISel/AArch64LegalizerInfo.cpp    | 23 ++++-
 llvm/test/CodeGen/AArch64/tagged-globals.ll   | 98 ++++++++++++++++---
 3 files changed, 124 insertions(+), 13 deletions(-)
 create mode 100644 compiler-rt/test/hwasan/TestCases/exported-tagged-global.c

diff --git a/compiler-rt/test/hwasan/TestCases/exported-tagged-global.c b/compiler-rt/test/hwasan/TestCases/exported-tagged-global.c
new file mode 100644
index 0000000000000..198d8781f68af
--- /dev/null
+++ b/compiler-rt/test/hwasan/TestCases/exported-tagged-global.c
@@ -0,0 +1,16 @@
+// RUN: %clang_hwasan %s -o %t
+// RUN: %run %t
+// RUN: %clang_hwasan -O1 %s -o %t
+// RUN: %run %t
+// RUN: %clang_hwasan -O1 -mllvm --aarch64-enable-global-isel-at-O=1 %s -o %t
+// RUN: %run %t
+
+static int global;
+
+__attribute__((optnone)) int *address_of_global() { return &global; }
+
+int main(int argc, char **argv) {
+  int *global_address = address_of_global();
+  *global_address = 13;
+  return 0;
+}
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index 6e5563af43638..063c451440dc1 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -351,7 +351,7 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
 
     if (DstSize == 128 && !Query.Types[0].isVector())
       return false; // Extending to a scalar s128 needs narrowing.
-    
+
     // Make sure that we have something that will fit in a register, and
     // make sure it's a power of 2.
     if (DstSize < 8 || DstSize > 128 || !isPowerOf2_32(DstSize))
@@ -676,6 +676,27 @@ bool AArch64LegalizerInfo::legalizeSmallCMGlobalValue(MachineInstr &MI,
   // Set the regclass on the dest reg too.
   MRI.setRegClass(ADRP.getReg(0), &AArch64::GPR64RegClass);
 
+  // MO_TAGGED on the page indicates a tagged address. Set the tag now. We do so
+  // by creating a MOVK that sets bits 48-63 of the register to (global address
+  // + 0x100000000 - PC) >> 48. The additional 0x100000000 offset here is to
+  // prevent an incorrect tag being generated during relocation when the the
+  // global appears before the code section. Without the offset, a global at
+  // `0x0f00'0000'0000'1000` (i.e. at `0x1000` with tag `0xf`) that's referenced
+  // by code at `0x2000` would result in `0x0f00'0000'0000'1000 - 0x2000 =
+  // 0x0eff'ffff'ffff'f000`, meaning the tag would be incorrectly set to `0xe`
+  // instead of `0xf`.
+  // This assumes that we're in the small code model so we can assume a binary
+  // size of <= 4GB, which makes the untagged PC relative offset positive. The
+  // binary must also be loaded into address range [0, 2^48). Both of these
+  // properties need to be ensured at runtime when using tagged addresses.
+  if (OpFlags & AArch64II::MO_TAGGED) {
+    ADRP = MIRBuilder.buildInstr(AArch64::MOVKXi, {LLT::pointer(0, 64)}, {ADRP})
+               .addGlobalAddress(GV, 0x100000000,
+                                 AArch64II::MO_PREL | AArch64II::MO_G3)
+               .addImm(48);
+    MRI.setRegClass(ADRP.getReg(0), &AArch64::GPR64RegClass);
+  }
+
   MIRBuilder.buildInstr(AArch64::G_ADD_LOW, {DstReg}, {ADRP})
       .addGlobalAddress(GV, 0,
                         OpFlags | AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
diff --git a/llvm/test/CodeGen/AArch64/tagged-globals.ll b/llvm/test/CodeGen/AArch64/tagged-globals.ll
index b0cf882584c6f..cdca6c7a1fba1 100644
--- a/llvm/test/CodeGen/AArch64/tagged-globals.ll
+++ b/llvm/test/CodeGen/AArch64/tagged-globals.ll
@@ -1,31 +1,105 @@
-; RUN: llc < %s | FileCheck %s
+; RUN: llc --relocation-model=static < %s \
+; RUN:   | FileCheck %s --check-prefixes=CHECK-STATIC,CHECK-SELECTIONDAGISEL
+; RUN: llc --relocation-model=pic < %s \
+; RUN:   | FileCheck %s --check-prefix=CHECK-PIC
+
+; Ensure that GlobalISel lowers correctly. GlobalISel is the default ISel for
+; -O0 on aarch64. GlobalISel lowers the instruction sequence in the static
+; relocation model different to SelectionDAGISel. GlobalISel does the lowering
+; of AddLow *after* legalization, and thus doesn't differentiate between
+; address-taken-only vs. address-taken-for-loadstore. Hence, we generate a movk
+; instruction for load/store instructions as well with GlobalISel. GlobalISel
+; also doesn't have the scaffolding to correctly check the bounds of the global
+; offset, and cannot fold the lo12 bits into the load/store. Neither of these
+; things are a problem as GlobalISel is only used by default at -O0, so we don't
+; mind the code size and performance increase.
+
+; RUN: llc --aarch64-enable-global-isel-at-O=0 -O0 < %s \
+; RUN:   | FileCheck %s --check-prefixes=CHECK-STATIC,CHECK-GLOBALISEL
+; RUN: llc --aarch64-enable-global-isel-at-O=0 -O0 --relocation-model=pic < %s \
+; RUN:   | FileCheck %s --check-prefix=CHECK-PIC
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64-unknown-linux-android"
 
-@global = external hidden global i32
+@global = external global i32
 declare void @func()
 
 define i32* @global_addr() #0 {
-  ; CHECK: global_addr:
-  ; CHECK: adrp x0, :pg_hi21_nc:global
-  ; CHECK: movk x0, #:prel_g3:global+4294967296
-  ; CHECK: add x0, x0, :lo12:global
+  ; Static relocation model has common codegen between SelectionDAGISel and
+  ; GlobalISel when the address-taken of a global isn't folded into a load or
+  ; store instruction.
+  ; CHECK-STATIC: global_addr:
+  ; CHECK-STATIC: adrp [[REG:x[0-9]+]], :pg_hi21_nc:global
+  ; CHECK-STATIC: movk [[REG]], #:prel_g3:global+4294967296
+  ; CHECK-STATIC: add x0, [[REG]], :lo12:global
+  ; CHECK-STATIC: ret
+
+  ; CHECK-PIC: global_addr:
+  ; CHECK-PIC: adrp [[REG:x[0-9]+]], :got:global
+  ; CHECK-PIC: ldr x0, {{\[}}[[REG]], :got_lo12:global]
+  ; CHECK-PIC: ret
+
   ret i32* @global
 }
 
 define i32 @global_load() #0 {
-  ; CHECK: global_load:
-  ; CHECK: adrp x8, :pg_hi21_nc:global
-  ; CHECK: ldr w0, [x8, :lo12:global]
+  ; CHECK-SELECTIONDAGISEL: global_load:
+  ; CHECK-SELECTIONDAGISEL: adrp [[REG:x[0-9]+]], :pg_hi21_nc:global
+  ; CHECK-SELECTIONDAGISEL: ldr w0, {{\[}}[[REG]], :lo12:global{{\]}}
+  ; CHECK-SELECTIONDAGISEL: ret
+
+  ; CHECK-GLOBALISEL: global_load:
+  ; CHECK-GLOBALISEL: adrp [[REG:x[0-9]+]], :pg_hi21_nc:global
+  ; CHECK-GLOBALISEL: movk [[REG]], #:prel_g3:global+4294967296
+  ; CHECK-GLOBALISEL: add [[REG]], [[REG]], :lo12:global
+  ; CHECK-GLOBALISEL: ldr w0, {{\[}}[[REG]]{{\]}}
+  ; CHECK-GLOBALISEL: ret
+
+  ; CHECK-PIC: global_load:
+  ; CHECK-PIC: adrp [[REG:x[0-9]+]], :got:global
+  ; CHECK-PIC: ldr  [[REG]], {{\[}}[[REG]], :got_lo12:global]
+  ; CHECK-PIC: ldr w0, {{\[}}[[REG]]{{\]}}
+  ; CHECK-PIC: ret
+
   %load = load i32, i32* @global
   ret i32 %load
 }
 
+define void @global_store() #0 {
+  ; CHECK-SELECTIONDAGISEL: global_store:
+  ; CHECK-SELECTIONDAGISEL: adrp [[REG:x[0-9]+]], :pg_hi21_nc:global
+  ; CHECK-SELECTIONDAGISEL: str wzr, {{\[}}[[REG]], :lo12:global{{\]}}
+  ; CHECK-SELECTIONDAGISEL: ret
+
+  ; CHECK-GLOBALISEL: global_store:
+  ; CHECK-GLOBALISEL: adrp [[REG:x[0-9]+]], :pg_hi21_nc:global
+  ; CHECK-GLOBALISEL: movk [[REG]], #:prel_g3:global+4294967296
+  ; CHECK-GLOBALISEL: add [[REG]], [[REG]], :lo12:global
+  ; CHECK-GLOBALISEL: str wzr, {{\[}}[[REG]]{{\]}}
+  ; CHECK-GLOBALISEL: ret
+
+  ; CHECK-PIC: global_store:
+  ; CHECK-PIC: adrp [[REG:x[0-9]+]], :got:global
+  ; CHECK-PIC: ldr  [[REG]], {{\[}}[[REG]], :got_lo12:global]
+  ; CHECK-PIC: str wzr, {{\[}}[[REG]]{{\]}}
+  ; CHECK-PIC: ret
+
+  store i32 0, i32* @global
+  ret void
+}
+
 define void ()* @func_addr() #0 {
-  ; CHECK: func_addr:
-  ; CHECK: adrp x0, func
-  ; CHECK: add x0, x0, :lo12:func
+  ; CHECK-STATIC: func_addr:
+  ; CHECK-STATIC: adrp [[REG:x[0-9]+]], func
+  ; CHECK-STATIC: add x0, [[REG]], :lo12:func
+  ; CHECK-STATIC: ret
+
+  ; CHECK-PIC: func_addr:
+  ; CHECK-PIC: adrp [[REG:x[0-9]+]], :got:func
+  ; CHECK-PIC: ldr  x0, {{\[}}[[REG]], :got_lo12:func]
+  ; CHECK-PIC: ret
+
   ret void ()* @func
 }
 

From 8c39e70679e93da3af9f881d314940c570d5d822 Mon Sep 17 00:00:00 2001
From: River Riddle <riddleriver@gmail.com>
Date: Mon, 3 Aug 2020 14:20:50 -0700
Subject: [PATCH 0956/1035] [mlir][OpFormatGen] Add support for eliding
 UnitAttr when used to anchor an optional group

Unit attributes are given meaning by their existence, and thus have no meaningful value beyond "is it present". As such, in the format of an operation unit attributes are generally used to guard the printing of other elements and aren't generally printed themselves; as the presence of the group when parsing means that the unit attribute should be added. This revision adds support to the declarative format for eliding unit attributes in situations where they anchor an optional group, but aren't the first element.

For example,
```
let assemblyFormat = "(`is_optional` $unit_attr^)? attr-dict";
```

would print `foo.op is_optional` when $unit_attr is present, instead of the current `foo.op is_optional unit`.

Differential Revision: https://reviews.llvm.org/D84577
---
 mlir/docs/OpDefinitions.md             | 32 +++++++++++++++++++-
 mlir/test/lib/Dialect/Test/TestOps.td  | 11 +++++++
 mlir/test/mlir-tblgen/op-format.mlir   | 14 +++++++++
 mlir/tools/mlir-tblgen/OpFormatGen.cpp | 42 +++++++++++++++++++++++---
 4 files changed, 93 insertions(+), 6 deletions(-)

diff --git a/mlir/docs/OpDefinitions.md b/mlir/docs/OpDefinitions.md
index c068aac09babe..0997f29727433 100644
--- a/mlir/docs/OpDefinitions.md
+++ b/mlir/docs/OpDefinitions.md
@@ -733,7 +733,7 @@ information. An optional group is defined by wrapping a set of elements within
 An example of an operation with an optional group is `std.return`, which has a
 variadic number of operands.
 
-```
+```tablegen
 def ReturnOp : ... {
   let arguments = (ins Variadic<AnyType>:$operands);
 
@@ -743,6 +743,36 @@ def ReturnOp : ... {
 }
 ```
 
+##### Unit Attributes
+
+In MLIR, the [`unit` Attribute](LangRef.md#unit-attribute) is special in that it
+only has one possible value, i.e. it derives meaning from its existence. When a
+unit attribute is used to anchor an optional group and is not the first element
+of the group, the presence of the unit attribute can be directly correlated with
+the presence of the optional group itself. As such, in these situations the unit
+attribute will not be printed or present in the output and will be automatically
+inferred when parsing by the presence of the optional group itself.
+
+For example, the following operation:
+
+```tablegen
+def FooOp : ... {
+  let arguments = (ins UnitAttr:$is_read_only);
+
+  let assemblyFormat = "attr-dict (`is_read_only` $is_read_only^)?";
+}
+```
+
+would be formatted as such:
+
+```mlir
+// When the unit attribute is present:
+foo.op is_read_only
+
+// When the unit attribute is not present:
+foo.op
+```
+
 #### Requirements
 
 The format specification has a certain set of requirements that must be adhered
diff --git a/mlir/test/lib/Dialect/Test/TestOps.td b/mlir/test/lib/Dialect/Test/TestOps.td
index 0c26f8a719c09..742033b130bc3 100644
--- a/mlir/test/lib/Dialect/Test/TestOps.td
+++ b/mlir/test/lib/Dialect/Test/TestOps.td
@@ -1391,6 +1391,17 @@ def FormatInferVariadicTypeFromNonVariadic
   let assemblyFormat = "$operands attr-dict `:` type($result)";
 }
 
+def FormatOptionalUnitAttr : TEST_Op<"format_optional_unit_attribute"> {
+  let arguments = (ins UnitAttr:$is_optional);
+  let assemblyFormat = "(`is_optional` $is_optional^)? attr-dict";
+}
+
+def FormatOptionalUnitAttrNoElide
+    : TEST_Op<"format_optional_unit_attribute_no_elide"> {
+  let arguments = (ins UnitAttr:$is_optional);
+  let assemblyFormat = "($is_optional^)? attr-dict";
+}
+
 //===----------------------------------------------------------------------===//
 // AllTypesMatch type inference
 //===----------------------------------------------------------------------===//
diff --git a/mlir/test/mlir-tblgen/op-format.mlir b/mlir/test/mlir-tblgen/op-format.mlir
index af5976b227068..959bbdc5c6bb5 100644
--- a/mlir/test/mlir-tblgen/op-format.mlir
+++ b/mlir/test/mlir-tblgen/op-format.mlir
@@ -82,6 +82,20 @@ test.format_operand_e_op %i64, %memref : i64, memref<1xf64>
 
 }) { arg_names = ["i", "j", "k"] } : () -> ()
 
+//===----------------------------------------------------------------------===//
+// Format optional attributes
+//===----------------------------------------------------------------------===//
+
+// CHECK: test.format_optional_unit_attribute is_optional
+test.format_optional_unit_attribute is_optional
+
+// CHECK: test.format_optional_unit_attribute
+// CHECK-NOT: is_optional
+test.format_optional_unit_attribute
+
+// CHECK: test.format_optional_unit_attribute_no_elide unit
+test.format_optional_unit_attribute_no_elide unit
+
 //===----------------------------------------------------------------------===//
 // Format optional operands and results
 //===----------------------------------------------------------------------===//
diff --git a/mlir/tools/mlir-tblgen/OpFormatGen.cpp b/mlir/tools/mlir-tblgen/OpFormatGen.cpp
index 13f2a2fd96dc9..edf804578aa02 100644
--- a/mlir/tools/mlir-tblgen/OpFormatGen.cpp
+++ b/mlir/tools/mlir-tblgen/OpFormatGen.cpp
@@ -107,6 +107,11 @@ struct AttributeVariable
     Optional<Type> attrType = var->attr.getValueType();
     return attrType ? attrType->getBuilderCall() : llvm::None;
   }
+
+  /// Return if this attribute refers to a UnitAttr.
+  bool isUnitAttr() const {
+    return var->attr.getBaseAttr().getAttrDefName() == "UnitAttr";
+  }
 };
 
 /// This class represents a variable that refers to an operand argument.
@@ -645,9 +650,23 @@ static void genElementParser(Element *element, OpMethodBody &body,
       body << "  if (!" << opVar->getVar()->name << "Operands.empty()) {\n";
     }
 
+    // If the anchor is a unit attribute, we don't need to print it. When
+    // parsing, we will add this attribute if this group is present.
+    Element *elidedAnchorElement = nullptr;
+    auto *anchorAttr = dyn_cast<AttributeVariable>(optional->getAnchor());
+    if (anchorAttr && anchorAttr != firstElement && anchorAttr->isUnitAttr()) {
+      elidedAnchorElement = anchorAttr;
+
+      // Add the anchor unit attribute to the operation state.
+      body << "    result.addAttribute(\"" << anchorAttr->getVar()->name
+           << "\", parser.getBuilder().getUnitAttr());\n";
+    }
+
     // Generate the rest of the elements normally.
-    for (auto &childElement : llvm::drop_begin(elements, 1))
-      genElementParser(&childElement, body, attrTypeCtx);
+    for (Element &childElement : llvm::drop_begin(elements, 1)) {
+      if (&childElement != elidedAnchorElement)
+        genElementParser(&childElement, body, attrTypeCtx);
+    }
     body << "  }\n";
 
     /// Literals.
@@ -1058,10 +1077,23 @@ static void genElementPrinter(Element *element, OpMethodBody &body,
            << cast<AttributeVariable>(anchor)->getVar()->name << "\")) {\n";
     }
 
+    // If the anchor is a unit attribute, we don't need to print it. When
+    // parsing, we will add this attribute if this group is present.
+    auto elements = optional->getElements();
+    Element *elidedAnchorElement = nullptr;
+    auto *anchorAttr = dyn_cast<AttributeVariable>(anchor);
+    if (anchorAttr && anchorAttr != &*elements.begin() &&
+        anchorAttr->isUnitAttr()) {
+      elidedAnchorElement = anchorAttr;
+    }
+
     // Emit each of the elements.
-    for (Element &childElement : optional->getElements())
-      genElementPrinter(&childElement, body, fmt, op, shouldEmitSpace,
-                        lastWasPunctuation);
+    for (Element &childElement : elements) {
+      if (&childElement != elidedAnchorElement) {
+        genElementPrinter(&childElement, body, fmt, op, shouldEmitSpace,
+                          lastWasPunctuation);
+      }
+    }
     body << "  }\n";
     return;
   }

From 1ce82015f6d06f8026357e4faa925f900136b575 Mon Sep 17 00:00:00 2001
From: Alina Sbirlea <asbirlea@google.com>
Date: Thu, 25 Jun 2020 16:50:15 -0700
Subject: [PATCH 0957/1035] [MemorySSA] Restrict optimizations after a
 PhiTranslation.

Merging alias results from different paths, when a path did phi
translation is not necesarily correct. Conservatively terminate such paths.
Aimed to fix PR46156.

Differential Revision: https://reviews.llvm.org/D84905
---
 llvm/include/llvm/Analysis/MemorySSA.h        |  6 +-
 llvm/lib/Analysis/MemorySSA.cpp               | 27 ++++--
 .../Analysis/MemorySSA/phi-translation.ll     | 82 +++++++++++++++++++
 3 files changed, 109 insertions(+), 6 deletions(-)

diff --git a/llvm/include/llvm/Analysis/MemorySSA.h b/llvm/include/llvm/Analysis/MemorySSA.h
index 5ce2b3fd047f7..3ec09e8c0a45e 100644
--- a/llvm/include/llvm/Analysis/MemorySSA.h
+++ b/llvm/include/llvm/Analysis/MemorySSA.h
@@ -1214,6 +1214,8 @@ class upward_defs_iterator
 
   BasicBlock *getPhiArgBlock() const { return DefIterator.getPhiArgBlock(); }
 
+  bool performedPhiTranslation() const { return PerformedPhiTranslation; }
+
 private:
   void fillInCurrentPair() {
     CurrentPair.first = *DefIterator;
@@ -1226,6 +1228,7 @@ class upward_defs_iterator
                                         false)) {
         if (Translator.getAddr() != Location.Ptr) {
           CurrentPair.second = Location.getWithNewPtr(Translator.getAddr());
+          PerformedPhiTranslation = true;
           return;
         }
       } else {
@@ -1240,8 +1243,9 @@ class upward_defs_iterator
   memoryaccess_def_iterator DefIterator;
   MemoryLocation Location;
   MemoryAccess *OriginalAccess = nullptr;
-  bool WalkingPhi = false;
   DominatorTree *DT = nullptr;
+  bool WalkingPhi = false;
+  bool PerformedPhiTranslation = false;
 };
 
 inline upward_defs_iterator upward_defs_begin(const MemoryAccessPair &Pair,
diff --git a/llvm/lib/Analysis/MemorySSA.cpp b/llvm/lib/Analysis/MemorySSA.cpp
index f2f5fd70f4718..6fc827ae2b17c 100644
--- a/llvm/lib/Analysis/MemorySSA.cpp
+++ b/llvm/lib/Analysis/MemorySSA.cpp
@@ -519,9 +519,16 @@ template <class AliasAnalysisType> class ClobberWalker {
   UpwardsMemoryQuery *Query;
   unsigned *UpwardWalkLimit;
 
-  // Phi optimization bookkeeping
+  // Phi optimization bookkeeping:
+  // List of DefPath to process during the current phi optimization walk.
   SmallVector<DefPath, 32> Paths;
+  // List of visited <Access, Location> pairs; we can skip paths already
+  // visited with the same memory location.
   DenseSet<ConstMemoryAccessPair> VisitedPhis;
+  // Record if phi translation has been performed during the current phi
+  // optimization walk, as merging alias results after phi translation can
+  // yield incorrect results. Context in PR46156.
+  bool PerformedPhiTranslation = false;
 
   /// Find the nearest def or phi that `From` can legally be optimized to.
   const MemoryAccess *getWalkTarget(const MemoryPhi *From) const {
@@ -596,12 +603,13 @@ template <class AliasAnalysisType> class ClobberWalker {
 
   void addSearches(MemoryPhi *Phi, SmallVectorImpl<ListIndex> &PausedSearches,
                    ListIndex PriorNode) {
-    auto UpwardDefs = make_range(
-        upward_defs_begin({Phi, Paths[PriorNode].Loc}, DT), upward_defs_end());
+    auto UpwardDefsBegin = upward_defs_begin({Phi, Paths[PriorNode].Loc}, DT);
+    auto UpwardDefs = make_range(UpwardDefsBegin, upward_defs_end());
     for (const MemoryAccessPair &P : UpwardDefs) {
       PausedSearches.push_back(Paths.size());
       Paths.emplace_back(P.second, P.first, PriorNode);
     }
+    PerformedPhiTranslation |= UpwardDefsBegin.performedPhiTranslation();
   }
 
   /// Represents a search that terminated after finding a clobber. This clobber
@@ -651,8 +659,16 @@ template <class AliasAnalysisType> class ClobberWalker {
       //   - We still cache things for A, so C only needs to walk up a bit.
       // If this behavior becomes problematic, we can fix without a ton of extra
       // work.
-      if (!VisitedPhis.insert({Node.Last, Node.Loc}).second)
+      if (!VisitedPhis.insert({Node.Last, Node.Loc}).second) {
+        if (PerformedPhiTranslation) {
+          // If visiting this path performed Phi translation, don't continue,
+          // since it may not be correct to merge results from two paths if one
+          // relies on the phi translation.
+          TerminatedPath Term{Node.Last, PathIndex};
+          return Term;
+        }
         continue;
+      }
 
       const MemoryAccess *SkipStopWhere = nullptr;
       if (Query->SkipSelfAccess && Node.Loc == Query->StartingLoc) {
@@ -765,7 +781,7 @@ template <class AliasAnalysisType> class ClobberWalker {
   /// terminates when a MemoryAccess that clobbers said MemoryLocation is found.
   OptznResult tryOptimizePhi(MemoryPhi *Phi, MemoryAccess *Start,
                              const MemoryLocation &Loc) {
-    assert(Paths.empty() && VisitedPhis.empty() &&
+    assert(Paths.empty() && VisitedPhis.empty() && !PerformedPhiTranslation &&
            "Reset the optimization state.");
 
     Paths.emplace_back(Loc, Start, Phi, None);
@@ -921,6 +937,7 @@ template <class AliasAnalysisType> class ClobberWalker {
   void resetPhiOptznState() {
     Paths.clear();
     VisitedPhis.clear();
+    PerformedPhiTranslation = false;
   }
 
 public:
diff --git a/llvm/test/Analysis/MemorySSA/phi-translation.ll b/llvm/test/Analysis/MemorySSA/phi-translation.ll
index c39ccd31d24b3..93ebc86210d1f 100644
--- a/llvm/test/Analysis/MemorySSA/phi-translation.ll
+++ b/llvm/test/Analysis/MemorySSA/phi-translation.ll
@@ -287,3 +287,85 @@ the.end:                            ; preds = %for.main.body
   ret void
 
 }
+
+
+@c = local_unnamed_addr global [2 x i16] zeroinitializer, align 2
+
+define i32 @dont_merge_noalias_simple(i32* noalias %ptr) {
+; CHECK-LABEL: define i32 @dont_merge_noalias_simple
+; CHECK-LABEL: entry:
+; CHECK:       ; 1 = MemoryDef(liveOnEntry)
+; CHECK-NEXT:  store i16 1, i16* %s1.ptr, align 2
+
+; CHECK-LABEL: %for.body
+; CHECK:       ; MemoryUse(4) MayAlias
+; CHECK-NEXT:    %lv = load i16, i16* %arrayidx, align 2
+
+entry:
+  %s1.ptr = getelementptr inbounds [2 x i16], [2 x i16]* @c, i64 0, i64 0
+  store i16 1, i16* %s1.ptr, align 2
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %storemerge2 = phi i32 [ 1, %entry ], [ %dec, %for.body ]
+  %idxprom1 = zext i32 %storemerge2 to i64
+  %arrayidx = getelementptr inbounds [2 x i16], [2 x i16]* @c, i64 0, i64 %idxprom1
+  %lv = load i16, i16* %arrayidx, align 2
+  %conv = sext i16 %lv to i32
+  store i32 %conv, i32* %ptr, align 4
+  %dec = add nsw i32 %storemerge2, -1
+  %cmp = icmp sgt i32 %storemerge2, 0
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  %s2.ptr = getelementptr inbounds [2 x i16], [2 x i16]* @c, i64 0, i64 0
+  store i16 0, i16* %s2.ptr, align 2
+  ret i32 0
+}
+
+
+define i32 @dont_merge_noalias_complex(i32* noalias %ptr, i32* noalias %another) {
+; CHECK-LABEL: define i32 @dont_merge_noalias_complex
+; CHECK-LABEL: entry:
+; CHECK:       ; 1 = MemoryDef(liveOnEntry)
+; CHECK-NEXT:  store i16 1, i16* %s1.ptr, align 2
+
+; CHECK-LABEL: %for.body
+; CHECK:       ; MemoryUse(7) MayAlias
+; CHECK-NEXT:    %lv = load i16, i16* %arrayidx, align 2
+
+entry:
+  %s1.ptr = getelementptr inbounds [2 x i16], [2 x i16]* @c, i64 0, i64 0
+  store i16 1, i16* %s1.ptr, align 2
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %storemerge2 = phi i32 [ 1, %entry ], [ %dec, %merge.body ]
+  %idxprom1 = zext i32 %storemerge2 to i64
+  %arrayidx = getelementptr inbounds [2 x i16], [2 x i16]* @c, i64 0, i64 %idxprom1
+  %lv = load i16, i16* %arrayidx, align 2
+  %conv = sext i16 %lv to i32
+  store i32 %conv, i32* %ptr, align 4
+  %dec = add nsw i32 %storemerge2, -1
+
+  %cmpif = icmp sgt i32 %storemerge2, 1
+  br i1 %cmpif, label %if.body, label %else.body
+
+if.body:
+  store i32 %conv, i32* %another, align 4
+  br label %merge.body
+
+else.body:
+  store i32 %conv, i32* %another, align 4
+  br label %merge.body
+
+merge.body:
+  %cmp = icmp sgt i32 %storemerge2, 0
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  %s2.ptr = getelementptr inbounds [2 x i16], [2 x i16]* @c, i64 0, i64 0
+  store i16 0, i16* %s2.ptr, align 2
+  ret i32 0
+}
+

From 089adc339e7dd6924d7e107fa362d9d12642f6e0 Mon Sep 17 00:00:00 2001
From: peter klausler <pklausler@nvidia.com>
Date: Mon, 3 Aug 2020 10:50:42 -0700
Subject: [PATCH 0958/1035] [flang] Make preprocessing behavior tests runnable
 as regression tests

And fix a minor bug exposed by doing so.

Differential Revision: https://reviews.llvm.org/D85164
---
 flang/lib/Parser/preprocessor.cpp         |  2 +-
 flang/lib/Parser/preprocessor.h           |  2 +-
 flang/lib/Parser/prescan.cpp              | 30 +++++++++++++++--------
 flang/test/Preprocessing/lit.local.cfg.py |  7 ------
 flang/test/Preprocessing/pp001.F          |  6 +++--
 flang/test/Preprocessing/pp002.F          |  6 +++--
 flang/test/Preprocessing/pp003.F          |  6 +++--
 flang/test/Preprocessing/pp004.F          |  6 +++--
 flang/test/Preprocessing/pp005.F          |  6 +++--
 flang/test/Preprocessing/pp006.F          |  6 +++--
 flang/test/Preprocessing/pp007.F          |  6 +++--
 flang/test/Preprocessing/pp008.F          |  6 +++--
 flang/test/Preprocessing/pp009.F          |  6 +++--
 flang/test/Preprocessing/pp010.F          |  6 +++--
 flang/test/Preprocessing/pp011.F          |  6 +++--
 flang/test/Preprocessing/pp012.F          |  6 +++--
 flang/test/Preprocessing/pp013.F          |  6 +++--
 flang/test/Preprocessing/pp014.F          |  6 +++--
 flang/test/Preprocessing/pp015.F          |  6 +++--
 flang/test/Preprocessing/pp016.F          |  6 +++--
 flang/test/Preprocessing/pp017.F          |  6 +++--
 flang/test/Preprocessing/pp018.F          |  6 +++--
 flang/test/Preprocessing/pp019.F          |  6 +++--
 flang/test/Preprocessing/pp020.F          |  6 +++--
 flang/test/Preprocessing/pp021.F          |  7 ++++--
 flang/test/Preprocessing/pp022.F          |  7 ++++--
 flang/test/Preprocessing/pp023.F          |  7 ++++--
 flang/test/Preprocessing/pp024.F          |  7 ++++--
 flang/test/Preprocessing/pp025.F          |  6 +++--
 flang/test/Preprocessing/pp026.F          |  6 +++--
 flang/test/Preprocessing/pp027.F          |  7 ++++--
 flang/test/Preprocessing/pp028.F          |  6 +++--
 flang/test/Preprocessing/pp029.F          |  6 +++--
 flang/test/Preprocessing/pp030.F          |  6 +++--
 flang/test/Preprocessing/pp031.F          |  5 +++-
 flang/test/Preprocessing/pp032.F          |  7 ++++--
 flang/test/Preprocessing/pp033.F          |  7 ++++--
 flang/test/Preprocessing/pp034.F          |  7 ++++--
 flang/test/Preprocessing/pp035.F          |  7 ++++--
 flang/test/Preprocessing/pp036.F          |  7 ++++--
 flang/test/Preprocessing/pp037.F          |  7 ++++--
 flang/test/Preprocessing/pp038.F          |  6 +++--
 flang/test/Preprocessing/pp039.F          |  8 ++++--
 flang/test/Preprocessing/pp040.F          |  4 ++-
 flang/test/Preprocessing/pp041.F          |  6 +++--
 flang/test/Preprocessing/pp042.F          |  6 +++--
 flang/test/Preprocessing/pp043.F          |  6 +++--
 flang/test/Preprocessing/pp044.F          |  6 +++--
 flang/test/Preprocessing/pp101.F90        |  6 +++--
 flang/test/Preprocessing/pp102.F90        |  6 +++--
 flang/test/Preprocessing/pp103.F90        |  6 +++--
 flang/test/Preprocessing/pp104.F90        |  6 +++--
 flang/test/Preprocessing/pp105.F90        |  6 +++--
 flang/test/Preprocessing/pp106.F90        |  6 +++--
 flang/test/Preprocessing/pp107.F90        |  6 +++--
 flang/test/Preprocessing/pp108.F90        |  6 +++--
 flang/test/Preprocessing/pp109.F90        |  6 +++--
 flang/test/Preprocessing/pp110.F90        |  6 +++--
 flang/test/Preprocessing/pp111.F90        |  6 +++--
 flang/test/Preprocessing/pp112.F90        |  6 +++--
 flang/test/Preprocessing/pp113.F90        |  6 +++--
 flang/test/Preprocessing/pp114.F90        |  6 +++--
 flang/test/Preprocessing/pp115.F90        |  6 +++--
 flang/test/Preprocessing/pp116.F90        |  6 +++--
 flang/test/Preprocessing/pp117.F90        |  6 +++--
 flang/test/Preprocessing/pp118.F90        |  6 +++--
 flang/test/Preprocessing/pp119.F90        |  6 +++--
 flang/test/Preprocessing/pp120.F90        |  6 +++--
 flang/test/Preprocessing/pp121.F90        |  7 ++++--
 flang/test/Preprocessing/pp122.F90        |  6 +++--
 flang/test/Preprocessing/pp123.F90        |  6 +++--
 flang/test/Preprocessing/pp124.F90        |  6 +++--
 flang/test/Preprocessing/pp125.F90        |  6 +++--
 flang/test/Preprocessing/pp126.F90        |  6 +++--
 flang/test/Preprocessing/pp127.F90        |  6 +++--
 flang/test/Preprocessing/pp128.F90        |  6 +++--
 flang/test/Preprocessing/pp129.F90        |  4 ++-
 flang/test/Preprocessing/pp130.F90        |  6 +++--
 78 files changed, 330 insertions(+), 164 deletions(-)
 delete mode 100644 flang/test/Preprocessing/lit.local.cfg.py

diff --git a/flang/lib/Parser/preprocessor.cpp b/flang/lib/Parser/preprocessor.cpp
index a1f07967d9b08..9c10cedfd53bb 100644
--- a/flang/lib/Parser/preprocessor.cpp
+++ b/flang/lib/Parser/preprocessor.cpp
@@ -212,7 +212,7 @@ Preprocessor::Preprocessor(AllSources &allSources) : allSources_{allSources} {
 }
 
 void Preprocessor::Define(std::string macro, std::string value) {
-  definitions_.emplace(SaveTokenAsName(macro), Definition{value, allSources_});
+  definitions_.emplace(macro, Definition{value, allSources_});
 }
 
 void Preprocessor::Undefine(std::string macro) { definitions_.erase(macro); }
diff --git a/flang/lib/Parser/preprocessor.h b/flang/lib/Parser/preprocessor.h
index 977d480c2482e..b4374a9acf5c1 100644
--- a/flang/lib/Parser/preprocessor.h
+++ b/flang/lib/Parser/preprocessor.h
@@ -67,6 +67,7 @@ class Preprocessor {
 
   void Define(std::string macro, std::string value);
   void Undefine(std::string macro);
+  bool IsNameDefined(const CharBlock &);
 
   std::optional<TokenSequence> MacroReplacement(
       const TokenSequence &, const Prescanner &);
@@ -79,7 +80,6 @@ class Preprocessor {
   enum class CanDeadElseAppear { No, Yes };
 
   CharBlock SaveTokenAsName(const CharBlock &);
-  bool IsNameDefined(const CharBlock &);
   TokenSequence ReplaceMacros(const TokenSequence &, const Prescanner &);
   void SkipDisabledConditionalCode(
       const std::string &, IsElseActive, Prescanner *, ProvenanceRange);
diff --git a/flang/lib/Parser/prescan.cpp b/flang/lib/Parser/prescan.cpp
index c81d6cb508464..face0d20e4fc0 100644
--- a/flang/lib/Parser/prescan.cpp
+++ b/flang/lib/Parser/prescan.cpp
@@ -246,7 +246,7 @@ void Prescanner::NextLine() {
 }
 
 void Prescanner::LabelField(TokenSequence &token, int outCol) {
-  bool badLabel{false};
+  const char *bad{nullptr};
   for (; *at_ != '\n' && column_ <= 6; ++at_) {
     if (*at_ == '\t') {
       ++at_;
@@ -256,16 +256,18 @@ void Prescanner::LabelField(TokenSequence &token, int outCol) {
     if (*at_ != ' ' &&
         !(*at_ == '0' && column_ == 6)) { // '0' in column 6 becomes space
       EmitChar(token, *at_);
-      if (!IsDecimalDigit(*at_) && !badLabel) {
-        Say(GetProvenance(at_),
-            "Character in fixed-form label field must be a digit"_en_US);
-        badLabel = true;
+      if (!bad && !IsDecimalDigit(*at_)) {
+        bad = at_;
       }
       ++outCol;
     }
     ++column_;
   }
   if (outCol > 1) {
+    if (bad && !preprocessor_.IsNameDefined(token.CurrentOpenToken())) {
+      Say(GetProvenance(bad),
+          "Character in fixed-form label field must be a digit"_en_US);
+    }
     token.CloseToken();
   }
   SkipToNextSignificantCharacter();
@@ -1098,6 +1100,15 @@ const char *Prescanner::IsCompilerDirectiveSentinel(
   return iter == compilerDirectiveSentinels_.end() ? nullptr : iter->c_str();
 }
 
+constexpr bool IsDirective(const char *match, const char *dir) {
+  for (; *match; ++match) {
+    if (*match != ToLowerCaseLetter(*dir++)) {
+      return false;
+    }
+  }
+  return true;
+}
+
 Prescanner::LineClassification Prescanner::ClassifyLine(
     const char *start) const {
   if (inFixedForm_) {
@@ -1122,13 +1133,12 @@ Prescanner::LineClassification Prescanner::ClassifyLine(
     return {LineClassification::Kind::IncludeLine, *quoteOffset};
   }
   if (const char *dir{IsPreprocessorDirectiveLine(start)}) {
-    if (std::memcmp(dir, "if", 2) == 0 || std::memcmp(dir, "elif", 4) == 0 ||
-        std::memcmp(dir, "else", 4) == 0 || std::memcmp(dir, "endif", 5) == 0) {
+    if (IsDirective("if", dir) || IsDirective("elif", dir) ||
+        IsDirective("else", dir) || IsDirective("endif", dir)) {
       return {LineClassification::Kind::ConditionalCompilationDirective};
-    } else if (std::memcmp(dir, "include", 7) == 0) {
+    } else if (IsDirective("include", dir)) {
       return {LineClassification::Kind::IncludeDirective};
-    } else if (std::memcmp(dir, "define", 6) == 0 ||
-        std::memcmp(dir, "undef", 5) == 0) {
+    } else if (IsDirective("define", dir) || IsDirective("undef", dir)) {
       return {LineClassification::Kind::DefinitionDirective};
     } else {
       return {LineClassification::Kind::PreprocessorDirective};
diff --git a/flang/test/Preprocessing/lit.local.cfg.py b/flang/test/Preprocessing/lit.local.cfg.py
deleted file mode 100644
index a7cf401d8c66c..0000000000000
--- a/flang/test/Preprocessing/lit.local.cfg.py
+++ /dev/null
@@ -1,7 +0,0 @@
-# -*- Python -*-
-
-from lit.llvm import llvm_config
-
-# Added this line file to prevent lit from discovering these tests
-# See Issue #1052
-config.suffixes = []
diff --git a/flang/test/Preprocessing/pp001.F b/flang/test/Preprocessing/pp001.F
index ba131b4a24c6f..95198641655a5 100644
--- a/flang/test/Preprocessing/pp001.F
+++ b/flang/test/Preprocessing/pp001.F
@@ -1,9 +1,11 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: if(777.eq.777)then
 * keyword macros
       integer, parameter :: KWM = 666
 #define KWM 777
       if (KWM .eq. 777) then
-        print *, 'pp001.F pass'
+        print *, 'pp001.F yes'
       else
-        print *, 'pp001.F FAIL: ', KWM
+        print *, 'pp001.F no: ', KWM
       end if
       end
diff --git a/flang/test/Preprocessing/pp002.F b/flang/test/Preprocessing/pp002.F
index f46baf73e2894..b4cbd7545adf8 100644
--- a/flang/test/Preprocessing/pp002.F
+++ b/flang/test/Preprocessing/pp002.F
@@ -1,10 +1,12 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: if(kwm.eq.777)then
 * #undef
       integer, parameter :: KWM = 777
 #define KWM 666
 #undef KWM
       if (KWM .eq. 777) then
-        print *, 'pp002.F pass'
+        print *, 'pp002.F yes'
       else
-        print *, 'pp002.F FAIL: ', KWM
+        print *, 'pp002.F no: ', KWM
       end if
       end
diff --git a/flang/test/Preprocessing/pp003.F b/flang/test/Preprocessing/pp003.F
index 0470f1909a696..4df10c47d2359 100644
--- a/flang/test/Preprocessing/pp003.F
+++ b/flang/test/Preprocessing/pp003.F
@@ -1,3 +1,5 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: res=((666)+111)
 * function-like macros
       integer function IFLM(x)
         integer :: x
@@ -8,8 +10,8 @@ program main
       integer :: res
       res = IFLM(666)
       if (res .eq. 777) then
-        print *, 'pp003.F pass'
+        print *, 'pp003.F yes'
       else
-        print *, 'pp003.F FAIL: ', res
+        print *, 'pp003.F no: ', res
       end if
       end
diff --git a/flang/test/Preprocessing/pp004.F b/flang/test/Preprocessing/pp004.F
index 800a96fbedbaf..788d3682ab98a 100644
--- a/flang/test/Preprocessing/pp004.F
+++ b/flang/test/Preprocessing/pp004.F
@@ -1,9 +1,11 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: if(kwm.eq.777)then
 * KWMs case-sensitive
       integer, parameter :: KWM = 777
 #define KWM 666
       if (kwm .eq. 777) then
-        print *, 'pp004.F pass'
+        print *, 'pp004.F yes'
       else
-        print *, 'pp004.F FAIL: ', kwm
+        print *, 'pp004.F no: ', kwm
       end if
       end
diff --git a/flang/test/Preprocessing/pp005.F b/flang/test/Preprocessing/pp005.F
index 05fab7a92f1b6..390e662e35574 100644
--- a/flang/test/Preprocessing/pp005.F
+++ b/flang/test/Preprocessing/pp005.F
@@ -1,3 +1,5 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: res=777
 * KWM split across continuation, implicit padding
       integer, parameter :: KWM = 666
 #define KWM 777
@@ -5,8 +7,8 @@
       res = KW
      +M
       if (res .eq. 777) then
-        print *, 'pp005.F pass'
+        print *, 'pp005.F yes'
       else
-        print *, 'pp005.F FAIL: ', res
+        print *, 'pp005.F no: ', res
       end if
       end
diff --git a/flang/test/Preprocessing/pp006.F b/flang/test/Preprocessing/pp006.F
index 55b87df8d4304..3c44728f56b50 100644
--- a/flang/test/Preprocessing/pp006.F
+++ b/flang/test/Preprocessing/pp006.F
@@ -1,3 +1,5 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: res=777
 * ditto, but with intervening *comment line
       integer, parameter :: KWM = 666
 #define KWM 777
@@ -6,8 +8,8 @@
 *comment
      +M
       if (res .eq. 777) then
-        print *, 'pp006.F pass'
+        print *, 'pp006.F yes'
       else
-        print *, 'pp006.F FAIL: ', res
+        print *, 'pp006.F no: ', res
       end if
       end
diff --git a/flang/test/Preprocessing/pp007.F b/flang/test/Preprocessing/pp007.F
index 8be4396a24928..ca78f4f4c6c5c 100644
--- a/flang/test/Preprocessing/pp007.F
+++ b/flang/test/Preprocessing/pp007.F
@@ -1,3 +1,5 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: res=kwm
 * KWM split across continuation, clipped after column 72
       integer, parameter :: KWM = 666
 #define KWM 777
@@ -8,8 +10,8 @@
       res = KW                                                          comment
      +M
       if (res .eq. 777) then
-        print *, 'pp007.F pass'
+        print *, 'pp007.F yes'
       else
-        print *, 'pp007.F FAIL: ', res
+        print *, 'pp007.F no: ', res
       end if
       end
diff --git a/flang/test/Preprocessing/pp008.F b/flang/test/Preprocessing/pp008.F
index 38c5b6657a871..9dd7bd92ea50f 100644
--- a/flang/test/Preprocessing/pp008.F
+++ b/flang/test/Preprocessing/pp008.F
@@ -1,11 +1,13 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: res=kwm
 * KWM with spaces in name at invocation NOT replaced
       integer, parameter :: KWM = 777
 #define KWM 666
       integer :: res
       res = K W M
       if (res .eq. 777) then
-        print *, 'pp008.F pass'
+        print *, 'pp008.F yes'
       else
-        print *, 'pp008.F FAIL: ', res
+        print *, 'pp008.F no: ', res
       end if
       end
diff --git a/flang/test/Preprocessing/pp009.F b/flang/test/Preprocessing/pp009.F
index a53623ff03692..c29021da484aa 100644
--- a/flang/test/Preprocessing/pp009.F
+++ b/flang/test/Preprocessing/pp009.F
@@ -1,3 +1,5 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: res=((666)+111)
 * FLM call split across continuation, implicit padding
       integer function IFLM(x)
         integer :: x
@@ -9,8 +11,8 @@ program main
       res = IFL
      +M(666)
       if (res .eq. 777) then
-        print *, 'pp009.F pass'
+        print *, 'pp009.F yes'
       else
-        print *, 'pp009.F FAIL: ', res
+        print *, 'pp009.F no: ', res
       end if
       end
diff --git a/flang/test/Preprocessing/pp010.F b/flang/test/Preprocessing/pp010.F
index 0769c98274dd3..4a812cde86af0 100644
--- a/flang/test/Preprocessing/pp010.F
+++ b/flang/test/Preprocessing/pp010.F
@@ -1,3 +1,5 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: res=((666)+111)
 * ditto, but with intervening *comment line
       integer function IFLM(x)
         integer :: x
@@ -10,8 +12,8 @@ program main
 *comment
      +M(666)
       if (res .eq. 777) then
-        print *, 'pp010.F pass'
+        print *, 'pp010.F yes'
       else
-        print *, 'pp010.F FAIL: ', res
+        print *, 'pp010.F no: ', res
       end if
       end
diff --git a/flang/test/Preprocessing/pp011.F b/flang/test/Preprocessing/pp011.F
index 4ec376649422b..c106c8bc3f5c5 100644
--- a/flang/test/Preprocessing/pp011.F
+++ b/flang/test/Preprocessing/pp011.F
@@ -1,3 +1,5 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: res=iflm(666)
 * FLM call name split across continuation, clipped
       integer function IFLM(x)
         integer :: x
@@ -12,8 +14,8 @@ program main
       res = IFL                                                         comment
      +M(666)
       if (res .eq. 777) then
-        print *, 'pp011.F pass'
+        print *, 'pp011.F yes'
       else
-        print *, 'pp011.F FAIL: ', res
+        print *, 'pp011.F no: ', res
       end if
       end
diff --git a/flang/test/Preprocessing/pp012.F b/flang/test/Preprocessing/pp012.F
index 703fabf7d8db6..411cfb887bb1f 100644
--- a/flang/test/Preprocessing/pp012.F
+++ b/flang/test/Preprocessing/pp012.F
@@ -1,3 +1,5 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: res=((666)+111)
 * FLM call name split across continuation
       integer function IFLM(x)
         integer :: x
@@ -9,8 +11,8 @@ program main
       res = IFL
      +M(666)
       if (res .eq. 777) then
-        print *, 'pp012.F pass'
+        print *, 'pp012.F yes'
       else
-        print *, 'pp012.F FAIL: ', res
+        print *, 'pp012.F no: ', res
       end if
       end
diff --git a/flang/test/Preprocessing/pp013.F b/flang/test/Preprocessing/pp013.F
index 6fb8ca75b5c83..f05e2e30fee69 100644
--- a/flang/test/Preprocessing/pp013.F
+++ b/flang/test/Preprocessing/pp013.F
@@ -1,3 +1,5 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: res=((666)+111)
 * FLM call split between name and (
       integer function IFLM(x)
         integer :: x
@@ -9,8 +11,8 @@ program main
       res = IFLM
      +(666)
       if (res .eq. 777) then
-        print *, 'pp013.F pass'
+        print *, 'pp013.F yes'
       else
-        print *, 'pp013.F FAIL: ', res
+        print *, 'pp013.F no: ', res
       end if
       end
diff --git a/flang/test/Preprocessing/pp014.F b/flang/test/Preprocessing/pp014.F
index 397a31d1a8aaa..470966096e4ee 100644
--- a/flang/test/Preprocessing/pp014.F
+++ b/flang/test/Preprocessing/pp014.F
@@ -1,3 +1,5 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: res=((666)+111)
 * FLM call split between name and (, with intervening *comment
       integer function IFLM(x)
         integer :: x
@@ -10,8 +12,8 @@ program main
 *comment
      +(666)
       if (res .eq. 777) then
-        print *, 'pp014.F pass'
+        print *, 'pp014.F yes'
       else
-        print *, 'pp014.F FAIL: ', res
+        print *, 'pp014.F no: ', res
       end if
       end
diff --git a/flang/test/Preprocessing/pp015.F b/flang/test/Preprocessing/pp015.F
index 4c399a835567e..8320bd704fed9 100644
--- a/flang/test/Preprocessing/pp015.F
+++ b/flang/test/Preprocessing/pp015.F
@@ -1,3 +1,5 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: res=((666)+111)
 * FLM call split between name and (, clipped
       integer function IFLM(x)
         integer :: x
@@ -12,8 +14,8 @@ program main
       res = IFLM                                                        comment
      +(666)
       if (res .eq. 777) then
-        print *, 'pp015.F pass'
+        print *, 'pp015.F yes'
       else
-        print *, 'pp015.F FAIL: ', res
+        print *, 'pp015.F no: ', res
       end if
       end
diff --git a/flang/test/Preprocessing/pp016.F b/flang/test/Preprocessing/pp016.F
index 210ad0b3fddbf..95c89f8e1a03e 100644
--- a/flang/test/Preprocessing/pp016.F
+++ b/flang/test/Preprocessing/pp016.F
@@ -1,3 +1,5 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: res=((666)+111)
 * FLM call split between name and ( and in argument
       integer function IFLM(x)
         integer :: x
@@ -10,8 +12,8 @@ program main
      +(66
      +6)
       if (res .eq. 777) then
-        print *, 'pp016.F pass'
+        print *, 'pp016.F yes'
       else
-        print *, 'pp016.F FAIL: ', res
+        print *, 'pp016.F no: ', res
       end if
       end
diff --git a/flang/test/Preprocessing/pp017.F b/flang/test/Preprocessing/pp017.F
index e658fbd922cf4..a11f185147831 100644
--- a/flang/test/Preprocessing/pp017.F
+++ b/flang/test/Preprocessing/pp017.F
@@ -1,10 +1,12 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: if(777.eq.777)then
 * KLM rescan
       integer, parameter :: KWM = 666, KWM2 = 667
 #define KWM2 777
 #define KWM KWM2
       if (KWM .eq. 777) then
-        print *, 'pp017.F pass'
+        print *, 'pp017.F yes'
       else
-        print *, 'pp017.F FAIL: ', KWM
+        print *, 'pp017.F no: ', KWM
       end if
       end
diff --git a/flang/test/Preprocessing/pp018.F b/flang/test/Preprocessing/pp018.F
index 877c6545e1c82..69c24e59f53d6 100644
--- a/flang/test/Preprocessing/pp018.F
+++ b/flang/test/Preprocessing/pp018.F
@@ -1,11 +1,13 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: if(kwm2.eq.777)then
 * KLM rescan with #undef (so rescan is after expansion)
       integer, parameter :: KWM2 = 777, KWM = 667
 #define KWM2 666
 #define KWM KWM2
 #undef KWM2
       if (KWM .eq. 777) then
-        print *, 'pp018.F pass'
+        print *, 'pp018.F yes'
       else
-        print *, 'pp018.F FAIL: ', KWM
+        print *, 'pp018.F no: ', KWM
       end if
       end
diff --git a/flang/test/Preprocessing/pp019.F b/flang/test/Preprocessing/pp019.F
index a2c9a02848575..d607ad3484079 100644
--- a/flang/test/Preprocessing/pp019.F
+++ b/flang/test/Preprocessing/pp019.F
@@ -1,3 +1,5 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: res=((666)+111)
 * FLM rescan
       integer function IFLM(x)
         integer :: x
@@ -10,8 +12,8 @@ program main
       integer :: res
       res = IFLM(666)
       if (res .eq. 777) then
-        print *, 'pp019.F pass'
+        print *, 'pp019.F yes'
       else
-        print *, 'pp019.F FAIL: ', res
+        print *, 'pp019.F no: ', res
       end if
       end
diff --git a/flang/test/Preprocessing/pp020.F b/flang/test/Preprocessing/pp020.F
index f0d26357c5d2f..88525003ba893 100644
--- a/flang/test/Preprocessing/pp020.F
+++ b/flang/test/Preprocessing/pp020.F
@@ -1,3 +1,5 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: res=((111)+666)
 * FLM expansion of argument
       integer function IFLM(x)
         integer :: x
@@ -10,8 +12,8 @@ program main
       integer :: res
       res = IFLM(KWM)
       if (res .eq. 777) then
-        print *, 'pp020.F pass'
+        print *, 'pp020.F yes'
       else
-        print *, 'pp020.F FAIL: ', res
+        print *, 'pp020.F no: ', res
       end if
       end
diff --git a/flang/test/Preprocessing/pp021.F b/flang/test/Preprocessing/pp021.F
index 45073ab6f1e6c..1662a680f5aa9 100644
--- a/flang/test/Preprocessing/pp021.F
+++ b/flang/test/Preprocessing/pp021.F
@@ -1,10 +1,13 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: ch='KWM'
+! CHECK: if(ch.eq.'KWM')then
 * KWM NOT expanded in 'literal'
 #define KWM 666
       character(len=3) :: ch
       ch = 'KWM'
       if (ch .eq. 'KWM') then
-        print *, 'pp021.F pass'
+        print *, 'pp021.F yes'
       else
-        print *, 'pp021.F FAIL: ', ch
+        print *, 'pp021.F no: ', ch
       end if
       end
diff --git a/flang/test/Preprocessing/pp022.F b/flang/test/Preprocessing/pp022.F
index e9a1e8ba4b915..026c02e9d4f01 100644
--- a/flang/test/Preprocessing/pp022.F
+++ b/flang/test/Preprocessing/pp022.F
@@ -1,10 +1,13 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: ch="KWM"
+! CHECK: if(ch.eq.'KWM')then
 * KWM NOT expanded in "literal"
 #define KWM 666
       character(len=3) :: ch
       ch = "KWM"
       if (ch .eq. 'KWM') then
-        print *, 'pp022.F pass'
+        print *, 'pp022.F yes'
       else
-        print *, 'pp022.F FAIL: ', ch
+        print *, 'pp022.F no: ', ch
       end if
       end
diff --git a/flang/test/Preprocessing/pp023.F b/flang/test/Preprocessing/pp023.F
index fb63d63f4fc1e..a0d053f6addc2 100644
--- a/flang/test/Preprocessing/pp023.F
+++ b/flang/test/Preprocessing/pp023.F
@@ -1,11 +1,14 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: ch=3hKWM
+! CHECK: if(ch.eq.'KWM')then
 * KWM NOT expanded in 9HHOLLERITH literal
 #define KWM 666
 #define HKWM 667
       character(len=3) :: ch
       ch = 3HKWM
       if (ch .eq. 'KWM') then
-        print *, 'pp023.F pass'
+        print *, 'pp023.F yes'
       else
-        print *, 'pp023.F FAIL: ', ch
+        print *, 'pp023.F no: ', ch
       end if
       end
diff --git a/flang/test/Preprocessing/pp024.F b/flang/test/Preprocessing/pp024.F
index 9072f6e50cc89..6ea76cc9c24de 100644
--- a/flang/test/Preprocessing/pp024.F
+++ b/flang/test/Preprocessing/pp024.F
@@ -1,3 +1,6 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: 100format(3hKWM)
+! CHECK: if(ch.eq.'KWM')then
 * KWM NOT expanded in Hollerith in FORMAT
 #define KWM 666
 #define HKWM 667
@@ -5,8 +8,8 @@
  100  format(3HKWM)
       write(ch, 100)
       if (ch .eq. 'KWM') then
-        print *, 'pp024.F pass'
+        print *, 'pp024.F yes'
       else
-        print *, 'pp024.F FAIL: ', ch
+        print *, 'pp024.F no: ', ch
       end if
       end
diff --git a/flang/test/Preprocessing/pp025.F b/flang/test/Preprocessing/pp025.F
index 42ad011842ff7..49521d443bd3b 100644
--- a/flang/test/Preprocessing/pp025.F
+++ b/flang/test/Preprocessing/pp025.F
@@ -1,11 +1,13 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: res=ikwm2z
 * KWM expansion is before token pasting due to fixed-form space removal
       integer, parameter :: IKWM2Z = 777
 #define KWM KWM2
       integer :: res
       res = I KWM Z
       if (res .eq. 777) then
-        print *, 'pp025.F pass'
+        print *, 'pp025.F yes'
       else
-        print *, 'pp025.F FAIL: ', res
+        print *, 'pp025.F no: ', res
       end if
       end
diff --git a/flang/test/Preprocessing/pp026.F b/flang/test/Preprocessing/pp026.F
index e0ea032c383fc..b551f3b173ed3 100644
--- a/flang/test/Preprocessing/pp026.F
+++ b/flang/test/Preprocessing/pp026.F
@@ -1,3 +1,5 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: res=((111)+666)
 * ## token pasting works in FLM
       integer function IFLM(x)
         integer :: x
@@ -12,8 +14,8 @@ program main
       integer :: res
       res = IFLM(KWM)
       if (res .eq. 777) then
-        print *, 'pp026.F pass'
+        print *, 'pp026.F yes'
       else
-        print *, 'pp026.F FAIL: ', res
+        print *, 'pp026.F no: ', res
       end if
       end
diff --git a/flang/test/Preprocessing/pp027.F b/flang/test/Preprocessing/pp027.F
index e2663800c1ce3..c1b787f9366d3 100644
--- a/flang/test/Preprocessing/pp027.F
+++ b/flang/test/Preprocessing/pp027.F
@@ -1,9 +1,12 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: kwm=666
+! CHECK: if(777.eq.777)then
 * #DEFINE works in fixed form
       integer, parameter :: KWM = 666
 #DEFINE KWM 777
       if (KWM .eq. 777) then
-        print *, 'pp027.F pass'
+        print *, 'pp027.F yes'
       else
-        print *, 'pp027.F FAIL: ', KWM
+        print *, 'pp027.F no: ', KWM
       end if
       end
diff --git a/flang/test/Preprocessing/pp028.F b/flang/test/Preprocessing/pp028.F
index 2906d389b57d4..51eab84f36877 100644
--- a/flang/test/Preprocessing/pp028.F
+++ b/flang/test/Preprocessing/pp028.F
@@ -1,3 +1,5 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: res=kw
 * fixed-form clipping done before KWM expansion on source line
       integer, parameter :: KW = 777
 #define KWM 666
@@ -7,8 +9,8 @@
 *234567890123456789012345678901234567890123456789012345678901234567890123
       res =                                                           KWM
       if (res .eq. 777) then
-        print *, 'pp028.F pass'
+        print *, 'pp028.F yes'
       else
-        print *, 'pp028.F FAIL: ', res
+        print *, 'pp028.F no: ', res
       end if
       end
diff --git a/flang/test/Preprocessing/pp029.F b/flang/test/Preprocessing/pp029.F
index 4374ef84489b5..bb8efe6c1a2e0 100644
--- a/flang/test/Preprocessing/pp029.F
+++ b/flang/test/Preprocessing/pp029.F
@@ -1,10 +1,12 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: if(77 7.eq.777)then
 * \ newline allowed in #define
       integer, parameter :: KWM = 666
 #define KWM 77\
  7
       if (KWM .eq. 777) then
-        print *, 'pp029.F pass'
+        print *, 'pp029.F yes'
       else
-        print *, 'pp029.F FAIL: ', KWM
+        print *, 'pp029.F no: ', KWM
       end if
       end
diff --git a/flang/test/Preprocessing/pp030.F b/flang/test/Preprocessing/pp030.F
index 3022e0ddf3dff..c04cf949f4142 100644
--- a/flang/test/Preprocessing/pp030.F
+++ b/flang/test/Preprocessing/pp030.F
@@ -1,9 +1,11 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: if(777.eq.777)then
 * /* C comment */ erased from #define
       integer, parameter :: KWM = 666
 #define KWM 777 /* C comment */
       if (KWM .eq. 777) then
-        print *, 'pp030.F pass'
+        print *, 'pp030.F yes'
       else
-        print *, 'pp030.F FAIL: ', KWM
+        print *, 'pp030.F no: ', KWM
       end if
       end
diff --git a/flang/test/Preprocessing/pp031.F b/flang/test/Preprocessing/pp031.F
index 0f59921bcb820..90b14647c4c93 100644
--- a/flang/test/Preprocessing/pp031.F
+++ b/flang/test/Preprocessing/pp031.F
@@ -1,9 +1,12 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: if(777//ccomment.eq.777)then
+! CHECK: print*,'pp031.F no: ',777//ccomment
 *  // C++ comment NOT erased from #define
       integer, parameter :: KWM = 666
 #define KWM 777 // C comment
       if (KWM .eq. 777) then
         print *, 'pp031.F FAIL (should not have compiled)'
       else
-        print *, 'pp031.F FAIL: ', KWM
+        print *, 'pp031.F no: ', KWM
       end if
       end
diff --git a/flang/test/Preprocessing/pp032.F b/flang/test/Preprocessing/pp032.F
index 9d9f14238d86d..6b779141ecb56 100644
--- a/flang/test/Preprocessing/pp032.F
+++ b/flang/test/Preprocessing/pp032.F
@@ -1,10 +1,13 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: if(777.eq.777)then
+! CHECK: print*,'pp032.F no: ',777
 * /* C comment */ \ newline erased from #define
       integer, parameter :: KWM = 666
 #define KWM 77/* C comment */\
 7
       if (KWM .eq. 777) then
-        print *, 'pp032.F pass'
+        print *, 'pp032.F yes'
       else
-        print *, 'pp032.F FAIL: ', KWM
+        print *, 'pp032.F no: ', KWM
       end if
       end
diff --git a/flang/test/Preprocessing/pp033.F b/flang/test/Preprocessing/pp033.F
index 34cf1996cc74d..3364527cba6a3 100644
--- a/flang/test/Preprocessing/pp033.F
+++ b/flang/test/Preprocessing/pp033.F
@@ -1,10 +1,13 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: if(777.eq.777)then
+! CHECK: print*,'pp033.F no: ',777
 * /* C comment \ newline */ erased from #define
       integer, parameter :: KWM = 666
 #define KWM 77/* C comment \
 */7
       if (KWM .eq. 777) then
-        print *, 'pp033.F pass'
+        print *, 'pp033.F yes'
       else
-        print *, 'pp033.F FAIL: ', KWM
+        print *, 'pp033.F no: ', KWM
       end if
       end
diff --git a/flang/test/Preprocessing/pp034.F b/flang/test/Preprocessing/pp034.F
index a9ed984b3b6ec..0c64aca62b1db 100644
--- a/flang/test/Preprocessing/pp034.F
+++ b/flang/test/Preprocessing/pp034.F
@@ -1,10 +1,13 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: if(777.eq.777)then
+! CHECK: print*,'pp034.F no: ',777
 * \ newline allowed in name on KWM definition
       integer, parameter :: KWMC = 666
 #define KWM\
 C 777
       if (KWMC .eq. 777) then
-        print *, 'pp034.F pass'
+        print *, 'pp034.F yes'
       else
-        print *, 'pp034.F FAIL: ', KWMC
+        print *, 'pp034.F no: ', KWMC
       end if
       end
diff --git a/flang/test/Preprocessing/pp035.F b/flang/test/Preprocessing/pp035.F
index 0135c9c4551a7..808174129b897 100644
--- a/flang/test/Preprocessing/pp035.F
+++ b/flang/test/Preprocessing/pp035.F
@@ -1,3 +1,6 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: if(777.eq.777)then
+! CHECK: print*,'pp035.F no: ',777
 * #if 2 .LT. 3 works
       integer, parameter :: KWM = 666
 #if 2 .LT. 3
@@ -6,8 +9,8 @@
 #define KWM 667
 #endif
       if (KWM .eq. 777) then
-        print *, 'pp035.F pass'
+        print *, 'pp035.F yes'
       else
-        print *, 'pp035.F FAIL: ', KWM
+        print *, 'pp035.F no: ', KWM
       end if
       end
diff --git a/flang/test/Preprocessing/pp036.F b/flang/test/Preprocessing/pp036.F
index ac922ae42ceb5..b7024c41fa60f 100644
--- a/flang/test/Preprocessing/pp036.F
+++ b/flang/test/Preprocessing/pp036.F
@@ -1,8 +1,11 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: if(.true.)then
+! CHECK: print*,'pp036.F no: ',.true.
 * #define FALSE TRUE ...  .FALSE. -> .TRUE.
 #define FALSE TRUE
       if (.FALSE.) then
-        print *, 'pp036.F pass'
+        print *, 'pp036.F yes'
       else
-        print *, 'pp036.F FAIL: ', .FALSE.
+        print *, 'pp036.F no: ', .FALSE.
       end if
       end
diff --git a/flang/test/Preprocessing/pp037.F b/flang/test/Preprocessing/pp037.F
index 6c3edb09eb6f4..52bfa8ffdab3d 100644
--- a/flang/test/Preprocessing/pp037.F
+++ b/flang/test/Preprocessing/pp037.F
@@ -1,11 +1,14 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: if(7777.eq.777)then
+! CHECK: print*,'pp037.F no: ',7777
 * fixed-form clipping NOT applied to #define
       integer, parameter :: KWM = 666
 *        1         2         3         4         5         6         7
 *234567890123456789012345678901234567890123456789012345678901234567890123
 #define KWM                                                          7777
       if (KWM .eq. 777) then
-        print *, 'pp037.F pass'
+        print *, 'pp037.F yes'
       else
-        print *, 'pp037.F FAIL: ', KWM
+        print *, 'pp037.F no: ', KWM
       end if
       end
diff --git a/flang/test/Preprocessing/pp038.F b/flang/test/Preprocessing/pp038.F
index 3c83dda7d03b7..7386aeebddeef 100644
--- a/flang/test/Preprocessing/pp038.F
+++ b/flang/test/Preprocessing/pp038.F
@@ -1,3 +1,5 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: res=((666)+111)
 * FLM call with closing ')' on next line (not a continuation)
       integer function IFLM(x)
         integer :: x
@@ -9,8 +11,8 @@ program main
       res = IFLM(666
 )
       if (res .eq. 777) then
-        print *, 'pp038.F pass'
+        print *, 'pp038.F yes'
       else
-        print *, 'pp038.F FAIL: ', res
+        print *, 'pp038.F no: ', res
       end if
       end
diff --git a/flang/test/Preprocessing/pp039.F b/flang/test/Preprocessing/pp039.F
index 52e6dd78603a0..9124474b86ba3 100644
--- a/flang/test/Preprocessing/pp039.F
+++ b/flang/test/Preprocessing/pp039.F
@@ -1,3 +1,7 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: res=iflm
+! CHECK: (666)
+! CHECK-NOT: res=((666)+111)
 * FLM call with '(' on next line (not a continuation)
       integer function IFLM(x)
         integer :: x
@@ -9,8 +13,8 @@ program main
       res = IFLM
 (666)
       if (res .eq. 777) then
-        print *, 'pp039.F pass'
+        print *, 'pp039.F yes'
       else
-        print *, 'pp039.F FAIL: ', res
+        print *, 'pp039.F no: ', res
       end if
       end
diff --git a/flang/test/Preprocessing/pp040.F b/flang/test/Preprocessing/pp040.F
index 59e901ac3c6b6..d589c38489cfb 100644
--- a/flang/test/Preprocessing/pp040.F
+++ b/flang/test/Preprocessing/pp040.F
@@ -1,5 +1,7 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK-NOT: FAIL HARD!
 * #define KWM c, then KWM works as comment line initiator
 #define KWM c
 KWM   print *, 'pp040.F FAIL HARD!'; stop
-      print *, 'pp040.F pass'
+      print *, 'pp040.F yes'
       end
diff --git a/flang/test/Preprocessing/pp041.F b/flang/test/Preprocessing/pp041.F
index 33c5ced3924dc..dcf61536040ba 100644
--- a/flang/test/Preprocessing/pp041.F
+++ b/flang/test/Preprocessing/pp041.F
@@ -1,3 +1,5 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: j=666wmj=j+1wm211
 * use KWM expansion as continuation indicators
 #define KWM 0
 #define KWM2 1
@@ -6,8 +8,8 @@
      KWM j = j + 1
      KWM2 11
       if (j .eq. 777) then
-        print *, 'pp041.F pass'
+        print *, 'pp041.F yes'
       else
-        print *, 'pp041.F FAIL', j
+        print *, 'pp041.F no', j
       end if
       end
diff --git a/flang/test/Preprocessing/pp042.F b/flang/test/Preprocessing/pp042.F
index 439e1affbca22..d5cc46a7a4712 100644
--- a/flang/test/Preprocessing/pp042.F
+++ b/flang/test/Preprocessing/pp042.F
@@ -1,6 +1,8 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK-NOT: goto 2
 * #define c 1, then use c as label in fixed-form
 #define c 1
-c     print *, 'pp042.F pass'; goto 2
-      print *, 'pp042.F FAIL'
+c     print *, 'pp042.F yes'; goto 2
+      print *, 'pp042.F no'
 2     continue
       end
diff --git a/flang/test/Preprocessing/pp043.F b/flang/test/Preprocessing/pp043.F
index be0069cf85579..a079466e64f92 100644
--- a/flang/test/Preprocessing/pp043.F
+++ b/flang/test/Preprocessing/pp043.F
@@ -1,11 +1,13 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: if(kwm.eq.777)then
 * #define with # in column 6 is a continuation line in fixed-form
       integer, parameter :: defineKWM666 = 555
       integer, parameter :: KWM =
      #define KWM 666
      ++222
       if (KWM .eq. 777) then
-        print *, 'pp043.F pass'
+        print *, 'pp043.F yes'
       else
-        print *, 'pp043.F FAIL: ', KWM
+        print *, 'pp043.F no: ', KWM
       end if
       end
diff --git a/flang/test/Preprocessing/pp044.F b/flang/test/Preprocessing/pp044.F
index 72ce6cc411598..c14b29c3050cc 100644
--- a/flang/test/Preprocessing/pp044.F
+++ b/flang/test/Preprocessing/pp044.F
@@ -1,3 +1,5 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK-NOT:z=111
 * #define directive amid continuations
       integer, parameter :: KWM = 222, KWM111 = 333, KWM222 = 555
       integer, parameter :: KWMKWM = 333
@@ -5,8 +7,8 @@
 #define KWM 111
      +KWM+444
       if (z .EQ. 777) then
-        print *, 'pass'
+        print *, 'yes'
       else
-        print *, 'FAIL', z
+        print *, 'no', z
       end if
       end
diff --git a/flang/test/Preprocessing/pp101.F90 b/flang/test/Preprocessing/pp101.F90
index 694201a8f33e0..b0f860157c82e 100644
--- a/flang/test/Preprocessing/pp101.F90
+++ b/flang/test/Preprocessing/pp101.F90
@@ -1,9 +1,11 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK:  if(777 .eq. 777) then
 ! keyword macros
       integer, parameter :: KWM = 666
 #define KWM 777
       if (KWM .eq. 777) then
-        print *, 'pp101.F90 pass'
+        print *, 'pp101.F90 yes'
       else
-        print *, 'pp101.F90 FAIL: ', KWM
+        print *, 'pp101.F90 no: ', KWM
       end if
       end
diff --git a/flang/test/Preprocessing/pp102.F90 b/flang/test/Preprocessing/pp102.F90
index 22e4613b3b18a..e2c3207a4c248 100644
--- a/flang/test/Preprocessing/pp102.F90
+++ b/flang/test/Preprocessing/pp102.F90
@@ -1,10 +1,12 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: if(kwm .eq. 777) then
 ! #undef
       integer, parameter :: KWM = 777
 #define KWM 666
 #undef KWM
       if (KWM .eq. 777) then
-        print *, 'pp102.F90 pass'
+        print *, 'pp102.F90 yes'
       else
-        print *, 'pp102.F90 FAIL: ', KWM
+        print *, 'pp102.F90 no: ', KWM
       end if
       end
diff --git a/flang/test/Preprocessing/pp103.F90 b/flang/test/Preprocessing/pp103.F90
index 9df4c9dbdf7b0..3309d7f8205f0 100644
--- a/flang/test/Preprocessing/pp103.F90
+++ b/flang/test/Preprocessing/pp103.F90
@@ -1,3 +1,5 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: res = ((666)+111)
 ! function-like macros
       integer function IFLM(x)
         integer :: x
@@ -8,8 +10,8 @@ program main
       integer :: res
       res = IFLM(666)
       if (res .eq. 777) then
-        print *, 'pp103.F90 pass'
+        print *, 'pp103.F90 yes'
       else
-        print *, 'pp103.F90 FAIL: ', res
+        print *, 'pp103.F90 no: ', res
       end if
       end
diff --git a/flang/test/Preprocessing/pp104.F90 b/flang/test/Preprocessing/pp104.F90
index b15f0db7c5b97..51248c22154b9 100644
--- a/flang/test/Preprocessing/pp104.F90
+++ b/flang/test/Preprocessing/pp104.F90
@@ -1,9 +1,11 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: if(kwm .eq. 777) then
 ! KWMs case-sensitive
       integer, parameter :: KWM = 777
 #define KWM 666
       if (kwm .eq. 777) then
-        print *, 'pp104.F90 pass'
+        print *, 'pp104.F90 yes'
       else
-        print *, 'pp104.F90 FAIL: ', kwm
+        print *, 'pp104.F90 no: ', kwm
       end if
       end
diff --git a/flang/test/Preprocessing/pp105.F90 b/flang/test/Preprocessing/pp105.F90
index cd475db01c39e..898a4a1681377 100644
--- a/flang/test/Preprocessing/pp105.F90
+++ b/flang/test/Preprocessing/pp105.F90
@@ -1,3 +1,5 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: res = 777
 ! KWM call name split across continuation, with leading &
       integer, parameter :: KWM = 666
 #define KWM 777
@@ -5,8 +7,8 @@
       res = KW&
 &M
       if (res .eq. 777) then
-        print *, 'pp105.F90 pass'
+        print *, 'pp105.F90 yes'
       else
-        print *, 'pp105.F90 FAIL: ', res
+        print *, 'pp105.F90 no: ', res
       end if
       end
diff --git a/flang/test/Preprocessing/pp106.F90 b/flang/test/Preprocessing/pp106.F90
index e169ff70b2cec..d83085fc1d718 100644
--- a/flang/test/Preprocessing/pp106.F90
+++ b/flang/test/Preprocessing/pp106.F90
@@ -1,3 +1,5 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: res = 777
 ! ditto, with & ! comment
       integer, parameter :: KWM = 666
 #define KWM 777
@@ -5,8 +7,8 @@
       res = KW& ! comment
 &M
       if (res .eq. 777) then
-        print *, 'pp106.F90 pass'
+        print *, 'pp106.F90 yes'
       else
-        print *, 'pp106.F90 FAIL: ', res
+        print *, 'pp106.F90 no: ', res
       end if
       end
diff --git a/flang/test/Preprocessing/pp107.F90 b/flang/test/Preprocessing/pp107.F90
index bf6d427c04007..6973127a4cfab 100644
--- a/flang/test/Preprocessing/pp107.F90
+++ b/flang/test/Preprocessing/pp107.F90
@@ -1,3 +1,5 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: res = kwm
 ! KWM call name split across continuation, no leading &, with & ! comment
       integer, parameter :: KWM = 666
 #define KWM 777
@@ -5,8 +7,8 @@
       res = KW& ! comment
 M
       if (res .eq. 777) then
-        print *, 'pp107.F90 pass'
+        print *, 'pp107.F90 yes'
       else
-        print *, 'pp107.F90 FAIL: ', res
+        print *, 'pp107.F90 no: ', res
       end if
       end
diff --git a/flang/test/Preprocessing/pp108.F90 b/flang/test/Preprocessing/pp108.F90
index 7ce6ccbdedc17..b07ec984fd811 100644
--- a/flang/test/Preprocessing/pp108.F90
+++ b/flang/test/Preprocessing/pp108.F90
@@ -1,3 +1,5 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: res = kwm
 ! ditto, but without & ! comment
       integer, parameter :: KWM = 666
 #define KWM 777
@@ -5,8 +7,8 @@
       res = KW&
 M
       if (res .eq. 777) then
-        print *, 'pp108.F90 pass'
+        print *, 'pp108.F90 yes'
       else
-        print *, 'pp108.F90 FAIL: ', res
+        print *, 'pp108.F90 no: ', res
       end if
       end
diff --git a/flang/test/Preprocessing/pp109.F90 b/flang/test/Preprocessing/pp109.F90
index a80579d18b3ee..e75fd10c1cb88 100644
--- a/flang/test/Preprocessing/pp109.F90
+++ b/flang/test/Preprocessing/pp109.F90
@@ -1,3 +1,5 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: res = ((666)+111)
 ! FLM call name split with leading &
       integer function IFLM(x)
         integer :: x
@@ -9,8 +11,8 @@ program main
       res = IFL&
 &M(666)
       if (res .eq. 777) then
-        print *, 'pp109.F90 pass'
+        print *, 'pp109.F90 yes'
       else
-        print *, 'pp109.F90 FAIL: ', res
+        print *, 'pp109.F90 no: ', res
       end if
       end
diff --git a/flang/test/Preprocessing/pp110.F90 b/flang/test/Preprocessing/pp110.F90
index f5bf3b1867f48..681408ebfddf8 100644
--- a/flang/test/Preprocessing/pp110.F90
+++ b/flang/test/Preprocessing/pp110.F90
@@ -1,3 +1,5 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: res = ((666)+111)
 ! ditto, with & ! comment
       integer function IFLM(x)
         integer :: x
@@ -9,8 +11,8 @@ program main
       res = IFL& ! comment
 &M(666)
       if (res .eq. 777) then
-        print *, 'pp110.F90 pass'
+        print *, 'pp110.F90 yes'
       else
-        print *, 'pp110.F90 FAIL: ', res
+        print *, 'pp110.F90 no: ', res
       end if
       end
diff --git a/flang/test/Preprocessing/pp111.F90 b/flang/test/Preprocessing/pp111.F90
index 668fcdc5f8d39..4b49bf6ad6c92 100644
--- a/flang/test/Preprocessing/pp111.F90
+++ b/flang/test/Preprocessing/pp111.F90
@@ -1,3 +1,5 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: res = iflm (666)
 ! FLM call name split across continuation, no leading &, with & ! comment
       integer function IFLM(x)
         integer :: x
@@ -9,8 +11,8 @@ program main
       res = IFL& ! comment
 M(666)
       if (res .eq. 777) then
-        print *, 'pp111.F90 pass'
+        print *, 'pp111.F90 yes'
       else
-        print *, 'pp111.F90 FAIL: ', res
+        print *, 'pp111.F90 no: ', res
       end if
       end
diff --git a/flang/test/Preprocessing/pp112.F90 b/flang/test/Preprocessing/pp112.F90
index 0a3c7f8906dc7..9828366bef73a 100644
--- a/flang/test/Preprocessing/pp112.F90
+++ b/flang/test/Preprocessing/pp112.F90
@@ -1,3 +1,5 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: res = iflm (666)
 ! ditto, but without & ! comment
       integer function IFLM(x)
         integer :: x
@@ -9,8 +11,8 @@ program main
       res = IFL&
 M(666)
       if (res .eq. 777) then
-        print *, 'pp112.F90 pass'
+        print *, 'pp112.F90 yes'
       else
-        print *, 'pp112.F90 FAIL: ', res
+        print *, 'pp112.F90 no: ', res
       end if
       end
diff --git a/flang/test/Preprocessing/pp113.F90 b/flang/test/Preprocessing/pp113.F90
index 4c928033638f9..56e34389068f3 100644
--- a/flang/test/Preprocessing/pp113.F90
+++ b/flang/test/Preprocessing/pp113.F90
@@ -1,3 +1,5 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: res = ((666)+111)
 ! FLM call split across continuation between name and (, leading &
       integer function IFLM(x)
         integer :: x
@@ -9,8 +11,8 @@ program main
       res = IFLM&
 &(666)
       if (res .eq. 777) then
-        print *, 'pp113.F90 pass'
+        print *, 'pp113.F90 yes'
       else
-        print *, 'pp113.F90 FAIL: ', res
+        print *, 'pp113.F90 no: ', res
       end if
       end
diff --git a/flang/test/Preprocessing/pp114.F90 b/flang/test/Preprocessing/pp114.F90
index f6c0e0263a2f3..4c44759ba96eb 100644
--- a/flang/test/Preprocessing/pp114.F90
+++ b/flang/test/Preprocessing/pp114.F90
@@ -1,3 +1,5 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: res = ((666)+111)
 ! ditto, with & ! comment, leading &
       integer function IFLM(x)
         integer :: x
@@ -9,8 +11,8 @@ program main
       res = IFLM& ! comment
 &(666)
       if (res .eq. 777) then
-        print *, 'pp114.F90 pass'
+        print *, 'pp114.F90 yes'
       else
-        print *, 'pp114.F90 FAIL: ', res
+        print *, 'pp114.F90 no: ', res
       end if
       end
diff --git a/flang/test/Preprocessing/pp115.F90 b/flang/test/Preprocessing/pp115.F90
index 4a38aca533118..290f62e68948b 100644
--- a/flang/test/Preprocessing/pp115.F90
+++ b/flang/test/Preprocessing/pp115.F90
@@ -1,3 +1,5 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: res = iflm (666)
 ! ditto, with & ! comment, no leading &
       integer function IFLM(x)
         integer :: x
@@ -9,8 +11,8 @@ program main
       res = IFLM& ! comment
 (666)
       if (res .eq. 777) then
-        print *, 'pp115.F90 pass'
+        print *, 'pp115.F90 yes'
       else
-        print *, 'pp115.F90 FAIL: ', res
+        print *, 'pp115.F90 no: ', res
       end if
       end
diff --git a/flang/test/Preprocessing/pp116.F90 b/flang/test/Preprocessing/pp116.F90
index 8708f79347cc6..d41fab771f842 100644
--- a/flang/test/Preprocessing/pp116.F90
+++ b/flang/test/Preprocessing/pp116.F90
@@ -1,3 +1,5 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: res = iflm (666)
 ! FLM call split between name and (, no leading &
       integer function IFLM(x)
         integer :: x
@@ -9,8 +11,8 @@ program main
       res = IFLM&
 (666)
       if (res .eq. 777) then
-        print *, 'pp116.F90 pass'
+        print *, 'pp116.F90 yes'
       else
-        print *, 'pp116.F90 FAIL: ', res
+        print *, 'pp116.F90 no: ', res
       end if
       end
diff --git a/flang/test/Preprocessing/pp117.F90 b/flang/test/Preprocessing/pp117.F90
index 8b8687f03743d..c8d1c2ff31304 100644
--- a/flang/test/Preprocessing/pp117.F90
+++ b/flang/test/Preprocessing/pp117.F90
@@ -1,10 +1,12 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: if(777 .eq. 777) then
 ! KWM rescan
       integer, parameter :: KWM = 666, KWM2 = 667
 #define KWM2 777
 #define KWM KWM2
       if (KWM .eq. 777) then
-        print *, 'pp117.F90 pass'
+        print *, 'pp117.F90 yes'
       else
-        print *, 'pp117.F90 FAIL: ', KWM
+        print *, 'pp117.F90 no: ', KWM
       end if
       end
diff --git a/flang/test/Preprocessing/pp118.F90 b/flang/test/Preprocessing/pp118.F90
index 014d99791f1aa..e71cc410823ba 100644
--- a/flang/test/Preprocessing/pp118.F90
+++ b/flang/test/Preprocessing/pp118.F90
@@ -1,11 +1,13 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: if(kwm2 .eq. 777) then
 ! KWM rescan with #undef, proving rescan after expansion
       integer, parameter :: KWM2 = 777, KWM = 667
 #define KWM2 666
 #define KWM KWM2
 #undef KWM2
       if (KWM .eq. 777) then
-        print *, 'pp118.F90 pass'
+        print *, 'pp118.F90 yes'
       else
-        print *, 'pp118.F90 FAIL: ', KWM
+        print *, 'pp118.F90 no: ', KWM
       end if
       end
diff --git a/flang/test/Preprocessing/pp119.F90 b/flang/test/Preprocessing/pp119.F90
index 37470de411a4e..ea9a15cb02de4 100644
--- a/flang/test/Preprocessing/pp119.F90
+++ b/flang/test/Preprocessing/pp119.F90
@@ -1,3 +1,5 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: res = ((666)+111)
 ! FLM rescan
       integer function IFLM(x)
         integer :: x
@@ -10,8 +12,8 @@ program main
       integer :: res
       res = IFLM(666)
       if (res .eq. 777) then
-        print *, 'pp119.F90 pass'
+        print *, 'pp119.F90 yes'
       else
-        print *, 'pp119.F90 FAIL: ', res
+        print *, 'pp119.F90 no: ', res
       end if
       end
diff --git a/flang/test/Preprocessing/pp120.F90 b/flang/test/Preprocessing/pp120.F90
index f7e0ae1034908..9c4b88c8d15c9 100644
--- a/flang/test/Preprocessing/pp120.F90
+++ b/flang/test/Preprocessing/pp120.F90
@@ -1,3 +1,5 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: res = ((111)+666)
 ! FLM expansion of argument
       integer function IFLM(x)
         integer :: x
@@ -10,8 +12,8 @@ program main
       integer :: res
       res = IFLM(KWM)
       if (res .eq. 777) then
-        print *, 'pp120.F90 pass'
+        print *, 'pp120.F90 yes'
       else
-        print *, 'pp120.F90 FAIL: ', res
+        print *, 'pp120.F90 no: ', res
       end if
       end
diff --git a/flang/test/Preprocessing/pp121.F90 b/flang/test/Preprocessing/pp121.F90
index bd855fe2f6ab4..cdf25e15ee44e 100644
--- a/flang/test/Preprocessing/pp121.F90
+++ b/flang/test/Preprocessing/pp121.F90
@@ -1,10 +1,13 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: ch = 'KWM'
+! CHECK: if(ch .eq. 'KWM') then
 ! KWM NOT expanded in 'literal'
 #define KWM 666
       character(len=3) :: ch
       ch = 'KWM'
       if (ch .eq. 'KWM') then
-        print *, 'pp121.F90 pass'
+        print *, 'pp121.F90 yes'
       else
-        print *, 'pp121.F90 FAIL: ', ch
+        print *, 'pp121.F90 no: ', ch
       end if
       end
diff --git a/flang/test/Preprocessing/pp122.F90 b/flang/test/Preprocessing/pp122.F90
index dbad83a61c6ac..14c8762af1c7e 100644
--- a/flang/test/Preprocessing/pp122.F90
+++ b/flang/test/Preprocessing/pp122.F90
@@ -1,10 +1,12 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: ch = "KWM"
 ! KWM NOT expanded in "literal"
 #define KWM 666
       character(len=3) :: ch
       ch = "KWM"
       if (ch .eq. 'KWM') then
-        print *, 'pp122.F90 pass'
+        print *, 'pp122.F90 yes'
       else
-        print *, 'pp122.F90 FAIL: ', ch
+        print *, 'pp122.F90 no: ', ch
       end if
       end
diff --git a/flang/test/Preprocessing/pp123.F90 b/flang/test/Preprocessing/pp123.F90
index 6e6c45244b8a9..1768cec1c23ba 100644
--- a/flang/test/Preprocessing/pp123.F90
+++ b/flang/test/Preprocessing/pp123.F90
@@ -1,11 +1,13 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: ch = 3hKWM
 ! KWM NOT expanded in Hollerith literal
 #define KWM 666
 #define HKWM 667
       character(len=3) :: ch
       ch = 3HKWM
       if (ch .eq. 'KWM') then
-        print *, 'pp123.F90 pass'
+        print *, 'pp123.F90 yes'
       else
-        print *, 'pp123.F90 FAIL: ', ch
+        print *, 'pp123.F90 no: ', ch
       end if
       end
diff --git a/flang/test/Preprocessing/pp124.F90 b/flang/test/Preprocessing/pp124.F90
index 2cf4d56dba238..bb011515ce1c2 100644
--- a/flang/test/Preprocessing/pp124.F90
+++ b/flang/test/Preprocessing/pp124.F90
@@ -1,3 +1,5 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: 100 format(3hKWM)
 ! KWM NOT expanded in Hollerith in FORMAT
 #define KWM 666
 #define HKWM 667
@@ -5,8 +7,8 @@
  100  format(3HKWM)
       write(ch, 100)
       if (ch .eq. 'KWM') then
-        print *, 'pp124.F90 pass'
+        print *, 'pp124.F90 yes'
       else
-        print *, 'pp124.F90 FAIL: ', ch
+        print *, 'pp124.F90 no: ', ch
       end if
       end
diff --git a/flang/test/Preprocessing/pp125.F90 b/flang/test/Preprocessing/pp125.F90
index 5f3875d8e88ef..cf3909b888240 100644
--- a/flang/test/Preprocessing/pp125.F90
+++ b/flang/test/Preprocessing/pp125.F90
@@ -1,9 +1,11 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: if(777 .eq. 777) then
 ! #DEFINE works in free form
       integer, parameter :: KWM = 666
 #DEFINE KWM 777
       if (KWM .eq. 777) then
-        print *, 'pp125.F90 pass'
+        print *, 'pp125.F90 yes'
       else
-        print *, 'pp125.F90 FAIL: ', KWM
+        print *, 'pp125.F90 no: ', KWM
       end if
       end
diff --git a/flang/test/Preprocessing/pp126.F90 b/flang/test/Preprocessing/pp126.F90
index c2684c51413b7..fefb7fcf118f4 100644
--- a/flang/test/Preprocessing/pp126.F90
+++ b/flang/test/Preprocessing/pp126.F90
@@ -1,10 +1,12 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: if(777 .eq. 777) then
 ! \ newline works in #define
       integer, parameter :: KWM = 666
 #define KWM 77\
 7
       if (KWM .eq. 777) then
-        print *, 'pp126.F90 pass'
+        print *, 'pp126.F90 yes'
       else
-        print *, 'pp126.F90 FAIL: ', KWM
+        print *, 'pp126.F90 no: ', KWM
       end if
       end
diff --git a/flang/test/Preprocessing/pp127.F90 b/flang/test/Preprocessing/pp127.F90
index 19f83b6afbc9a..08feedf95305d 100644
--- a/flang/test/Preprocessing/pp127.F90
+++ b/flang/test/Preprocessing/pp127.F90
@@ -1,3 +1,5 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: res = iflm(666 )
 ! FLM call with closing ')' on next line (not a continuation)
       integer function IFLM(x)
         integer :: x
@@ -9,8 +11,8 @@ program main
       res = IFLM(666
 )
       if (res .eq. 777) then
-        print *, 'pp127.F90 pass'
+        print *, 'pp127.F90 yes'
       else
-        print *, 'pp127.F90 FAIL: ', res
+        print *, 'pp127.F90 no: ', res
       end if
       end
diff --git a/flang/test/Preprocessing/pp128.F90 b/flang/test/Preprocessing/pp128.F90
index 84b338bc7cc98..46918e05f5c39 100644
--- a/flang/test/Preprocessing/pp128.F90
+++ b/flang/test/Preprocessing/pp128.F90
@@ -1,3 +1,5 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: res = iflm
 ! FLM call with '(' on next line (not a continuation)
       integer function IFLM(x)
         integer :: x
@@ -9,8 +11,8 @@ program main
       res = IFLM
 (666)
       if (res .eq. 777) then
-        print *, 'pp128.F90 pass'
+        print *, 'pp128.F90 yes'
       else
-        print *, 'pp128.F90 FAIL: ', res
+        print *, 'pp128.F90 no: ', res
       end if
       end
diff --git a/flang/test/Preprocessing/pp129.F90 b/flang/test/Preprocessing/pp129.F90
index a8eea86996838..ab7e7f0baa8b8 100644
--- a/flang/test/Preprocessing/pp129.F90
+++ b/flang/test/Preprocessing/pp129.F90
@@ -1,5 +1,7 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK-NOT: stop
 ! #define KWM !, then KWM works as comment line initiator
 #define KWM !
 KWM   print *, 'pp129.F90 FAIL HARD!'; stop
-      print *, 'pp129.F90 pass'
+      print *, 'pp129.F90 yes'
       end
diff --git a/flang/test/Preprocessing/pp130.F90 b/flang/test/Preprocessing/pp130.F90
index c3d8079210c66..af4ad126e6fa4 100644
--- a/flang/test/Preprocessing/pp130.F90
+++ b/flang/test/Preprocessing/pp130.F90
@@ -1,3 +1,5 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: j = j + &
 ! #define KWM &, use for continuation w/o pasting (ifort and nag seem to continue #define)
 #define KWM &
 
@@ -6,8 +8,8 @@
       j = j + KWM
 111
       if (j .eq. 777) then
-        print *, 'pp130.F90 pass'
+        print *, 'pp130.F90 yes'
       else
-        print *, 'pp130.F90 FAIL', j
+        print *, 'pp130.F90 no', j
       end if
       end

From 7cf4603faee366a6e5860b6fdbedadd91872e231 Mon Sep 17 00:00:00 2001
From: Fangrui Song <maskray@google.com>
Date: Mon, 3 Aug 2020 15:11:28 -0700
Subject: [PATCH 0959/1035] Reland D61689 Change -gz and
 -Wa,--compress-debug-sections to use gABI compression (SHF_COMPRESSED) with
 integrated assembler

This fixes an inconsistency: clang -c -gz -fno-integrated-as means SHF_COMPRESSED
while clang -c -gz -fintegrated-as means zlib-gnu.

---

Since July 15, 2015 (binutils-gdb commit
19a7fe52ae3d0971e67a134bcb1648899e21ae1c, included in 2.26), gas
--compress-debug-sections=zlib (gcc -gz) means zlib-gabi:
SHF_COMPRESSED. Before that GCC/binutils used zlib-gnu (.zdebug).

clang's -gz was introduced in rC306115 (Jun 2017) to indicate zlib-gnu. It
is 2020 now and it is not unreasonable to assume users of the new
feature to have new linkers (ld.bfd/gold >= 2.26, lld >= rLLD273661).

Change clang's default accordingly to improve standard conformance.
zlib-gnu becomes out of fashion and gets poorer toolchain support.
Its mangled names confuse tools and are more likely to cause problems.

Reviewed By: compnerd

Differential Revision: https://reviews.llvm.org/D61689
---
 clang/docs/ReleaseNotes.rst               | 5 ++++-
 clang/lib/Frontend/CompilerInvocation.cpp | 3 +--
 clang/tools/driver/cc1as_main.cpp         | 3 +--
 3 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 9274081c4d62c..03eca8a26843e 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -100,7 +100,10 @@ future versions of Clang.
 Modified Compiler Flags
 -----------------------
 
-- ...
+- On ELF, ``-gz`` now defaults to ``-gz=zlib`` with the integrated assembler.
+  It produces ``SHF_COMPRESSED`` style compression of debug information. GNU
+  binutils 2.26 or newer, or lld is required to link produced object files. Use
+  ``-gz=zlib-gnu`` to get the old behavior.
 
 New Pragmas in Clang
 --------------------
diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp
index 0b5f33541060f..8e8bf9d9028eb 100644
--- a/clang/lib/Frontend/CompilerInvocation.cpp
+++ b/clang/lib/Frontend/CompilerInvocation.cpp
@@ -1155,8 +1155,7 @@ static bool ParseCodeGenArgs(CodeGenOptions &Opts, ArgList &Args, InputKind IK,
   if (const Arg *A = Args.getLastArg(OPT_compress_debug_sections,
                                      OPT_compress_debug_sections_EQ)) {
     if (A->getOption().getID() == OPT_compress_debug_sections) {
-      // TODO: be more clever about the compression type auto-detection
-      Opts.setCompressDebugSections(llvm::DebugCompressionType::GNU);
+      Opts.setCompressDebugSections(llvm::DebugCompressionType::Z);
     } else {
       auto DCT = llvm::StringSwitch<llvm::DebugCompressionType>(A->getValue())
                      .Case("none", llvm::DebugCompressionType::None)
diff --git a/clang/tools/driver/cc1as_main.cpp b/clang/tools/driver/cc1as_main.cpp
index 77b99b2013640..87047be3c2bc6 100644
--- a/clang/tools/driver/cc1as_main.cpp
+++ b/clang/tools/driver/cc1as_main.cpp
@@ -224,8 +224,7 @@ bool AssemblerInvocation::CreateFromArgs(AssemblerInvocation &Opts,
   if (const Arg *A = Args.getLastArg(OPT_compress_debug_sections,
                                      OPT_compress_debug_sections_EQ)) {
     if (A->getOption().getID() == OPT_compress_debug_sections) {
-      // TODO: be more clever about the compression type auto-detection
-      Opts.CompressDebugSections = llvm::DebugCompressionType::GNU;
+      Opts.CompressDebugSections = llvm::DebugCompressionType::Z;
     } else {
       Opts.CompressDebugSections =
           llvm::StringSwitch<llvm::DebugCompressionType>(A->getValue())

From b43791e7016d04c0dcc0d36bb43b8e96110608c9 Mon Sep 17 00:00:00 2001
From: Christopher Tetreault <ctetreau@quicinc.com>
Date: Mon, 3 Aug 2020 13:35:49 -0700
Subject: [PATCH 0960/1035] [SVE] Remove bad calls to
 VectorType::getNumElements() from PowerPC

Differential Revision: https://reviews.llvm.org/D85154
---
 llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
index 04b3dceae4754..8434fdebe1b07 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
+++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
@@ -113,7 +113,7 @@ PPCTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
     // the permutation mask with respect to 31 and reverse the order of
     // V1 and V2.
     if (Constant *Mask = dyn_cast<Constant>(II.getArgOperand(2))) {
-      assert(cast<VectorType>(Mask->getType())->getNumElements() == 16 &&
+      assert(cast<FixedVectorType>(Mask->getType())->getNumElements() == 16 &&
              "Bad type for intrinsic!");
 
       // Check that all of the elements are integer constants or undefs.

From af3ec731d54d8ca5e5752256bee5b93a5929fa14 Mon Sep 17 00:00:00 2001
From: Jordan Rupprecht <rupprecht@google.com>
Date: Mon, 3 Aug 2020 15:21:44 -0700
Subject: [PATCH 0961/1035] [NFC][ARM] Silence unused variable in release
 builds

---
 llvm/lib/Target/ARM/MVEVPTOptimisationsPass.cpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/llvm/lib/Target/ARM/MVEVPTOptimisationsPass.cpp b/llvm/lib/Target/ARM/MVEVPTOptimisationsPass.cpp
index 8dbb8b53c8905..0523ac9447c32 100644
--- a/llvm/lib/Target/ARM/MVEVPTOptimisationsPass.cpp
+++ b/llvm/lib/Target/ARM/MVEVPTOptimisationsPass.cpp
@@ -462,6 +462,8 @@ bool MVEVPTOptimisations::ConvertVPSEL(MachineBasicBlock &MBB) {
             .addImm(ARMVCC::Then)
             .add(MI.getOperand(4))
             .add(MI.getOperand(2));
+    // Silence unused variable warning in release builds.
+    (void)MIBuilder;
     LLVM_DEBUG(dbgs() << "Replacing VPSEL: "; MI.dump();
                dbgs() << "     with VMOVT: "; MIBuilder.getInstr()->dump());
     DeadInstructions.push_back(&MI);

From 1d7790604c0c4e189bf8d91583cae8f1e027b1fb Mon Sep 17 00:00:00 2001
From: Julian Lettner <julian.lettner@apple.com>
Date: Mon, 3 Aug 2020 13:21:08 -0700
Subject: [PATCH 0962/1035] [UBSan] Increase robustness of tests

These UBSan tests assert the absence of runtime errors via `count 0`,
which means "expect no output".  This fails the test unnecessarily in
some environments (e.g., iOS simulator in our case).  Alter the test to
be a bit more specific and "expect no error" instead of "expect no
output".

rdar://65503408

Differential Revision: https://reviews.llvm.org/D85155
---
 compiler-rt/test/ubsan/TestCases/Misc/nonnull.cpp   | 4 +++-
 compiler-rt/test/ubsan/TestCases/Misc/nullability.c | 4 +++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/compiler-rt/test/ubsan/TestCases/Misc/nonnull.cpp b/compiler-rt/test/ubsan/TestCases/Misc/nonnull.cpp
index d5cd2bf763b7c..c612cd8247156 100644
--- a/compiler-rt/test/ubsan/TestCases/Misc/nonnull.cpp
+++ b/compiler-rt/test/ubsan/TestCases/Misc/nonnull.cpp
@@ -1,5 +1,5 @@
 // RUN: %clangxx -fsanitize=returns-nonnull-attribute -w %s -O3 -o %t
-// RUN: %run %t foo 2>&1 | count 0
+// RUN: %run %t foo 2>&1 | FileCheck %s --check-prefix=NOERROR --allow-empty --implicit-check-not='runtime error'
 // RUN: %run %t 2>&1 | FileCheck %s
 // RUN: %clangxx -fsanitize=returns-nonnull-attribute -fno-sanitize-recover=returns-nonnull-attribute -w %s -O3 -o %t.abort
 // RUN: not %run %t.abort &> /dev/null
@@ -40,3 +40,5 @@ int main(int argc, char **argv) {
 
   return 0;
 }
+
+// NOERROR-NOT: runtime error
diff --git a/compiler-rt/test/ubsan/TestCases/Misc/nullability.c b/compiler-rt/test/ubsan/TestCases/Misc/nullability.c
index a68d0f8ea3628..42837983865d8 100644
--- a/compiler-rt/test/ubsan/TestCases/Misc/nullability.c
+++ b/compiler-rt/test/ubsan/TestCases/Misc/nullability.c
@@ -1,6 +1,6 @@
 // UNSUPPORTED: android
 // RUN: %clang -w -fsanitize=nullability-arg,nullability-assign,nullability-return %s -O3 -o %t
-// RUN: %run %t foo 2>&1 | count 0
+// RUN: %run %t foo 2>&1 | FileCheck %s --check-prefix=NOERROR --allow-empty --implicit-check-not='runtime error'
 // RUN: %run %t 2>&1 | FileCheck %s
 
 // RUN: echo "nullability-arg:nullability.c" > %t.supp
@@ -67,3 +67,5 @@ int main(int argc, char **argv) {
   nonnull_init2(p);
   return 0;
 }
+
+// NOERROR-NOT: runtime error

From b5059b7140232559ed123cb94d4e8f75ca9a44dc Mon Sep 17 00:00:00 2001
From: Christopher Tetreault <ctetreau@quicinc.com>
Date: Mon, 3 Aug 2020 15:16:16 -0700
Subject: [PATCH 0963/1035] [SVE] Remove bad call to
 VectorType::getNumElements() from ARM

Reviewed By: dmgreen

Differential Revision: https://reviews.llvm.org/D85152
---
 llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
index 9674a813a52af..68767398191aa 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
@@ -166,7 +166,7 @@ ARMTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
       if (auto *CI = dyn_cast<ConstantInt>(XorMask)) {
         if (CI->getValue().trunc(16).isAllOnesValue()) {
           auto TrueVector = IC.Builder.CreateVectorSplat(
-              cast<VectorType>(II.getType())->getNumElements(),
+              cast<FixedVectorType>(II.getType())->getNumElements(),
               IC.Builder.getTrue());
           return BinaryOperator::Create(Instruction::Xor, ArgArg, TrueVector);
         }

From 3b92db4c846ef3c7295444fa0b554905de0774b2 Mon Sep 17 00:00:00 2001
From: Christopher Tetreault <ctetreau@quicinc.com>
Date: Mon, 3 Aug 2020 15:42:13 -0700
Subject: [PATCH 0964/1035] [SVE] Remove bad call to
 VectorType::getNumElements() from AMDGPU

Differential Revision: https://reviews.llvm.org/D85151
---
 llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
index 372c5154acef0..e186f66a83536 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
@@ -728,7 +728,7 @@ static Value *simplifyAMDGCNMemoryIntrinsicDemanded(InstCombiner &IC,
                                                     APInt DemandedElts,
                                                     int DMaskIdx = -1) {
 
-  auto *IIVTy = cast<VectorType>(II.getType());
+  auto *IIVTy = cast<FixedVectorType>(II.getType());
   unsigned VWidth = IIVTy->getNumElements();
   if (VWidth == 1)
     return nullptr;

From 045e79e77c252f2c73c640e820e977ef52836d50 Mon Sep 17 00:00:00 2001
From: "Kazushi (Jam) Marukawa" <marukawa@nec.com>
Date: Sat, 25 Jul 2020 16:31:07 +0900
Subject: [PATCH 0965/1035] [VE] Extend integer arguments and return values
 smaller than 64 bits

In order to follow NEC Aurora SX VE ABI correctly, change to sign/zero
extend integer arguments and return values smaller than 64 bits in clang.
Also update regression test.

Reviewed By: simoll

Differential Revision: https://reviews.llvm.org/D85071
---
 clang/lib/CodeGen/TargetInfo.cpp | 13 +++--
 clang/test/CodeGen/ve-abi.c      | 92 ++++++++++++++++++++++++++++++--
 2 files changed, 95 insertions(+), 10 deletions(-)

diff --git a/clang/lib/CodeGen/TargetInfo.cpp b/clang/lib/CodeGen/TargetInfo.cpp
index f31d432eb3171..e011cfa811678 100644
--- a/clang/lib/CodeGen/TargetInfo.cpp
+++ b/clang/lib/CodeGen/TargetInfo.cpp
@@ -10743,21 +10743,24 @@ class VEABIInfo : public DefaultABIInfo {
 } // end anonymous namespace
 
 ABIArgInfo VEABIInfo::classifyReturnType(QualType Ty) const {
-  if (Ty->isAnyComplexType()) {
+  if (Ty->isAnyComplexType())
     return ABIArgInfo::getDirect();
-  }
+  uint64_t Size = getContext().getTypeSize(Ty);
+  if (Size < 64 && Ty->isIntegerType())
+    return ABIArgInfo::getExtend(Ty);
   return DefaultABIInfo::classifyReturnType(Ty);
 }
 
 ABIArgInfo VEABIInfo::classifyArgumentType(QualType Ty) const {
-  if (Ty->isAnyComplexType()) {
+  if (Ty->isAnyComplexType())
     return ABIArgInfo::getDirect();
-  }
+  uint64_t Size = getContext().getTypeSize(Ty);
+  if (Size < 64 && Ty->isIntegerType())
+    return ABIArgInfo::getExtend(Ty);
   return DefaultABIInfo::classifyArgumentType(Ty);
 }
 
 void VEABIInfo::computeInfo(CGFunctionInfo &FI) const {
-
   FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
   for (auto &Arg : FI.arguments())
     Arg.info = classifyArgumentType(Arg.type);
diff --git a/clang/test/CodeGen/ve-abi.c b/clang/test/CodeGen/ve-abi.c
index aa35095d5dea0..1c230cb616a38 100644
--- a/clang/test/CodeGen/ve-abi.c
+++ b/clang/test/CodeGen/ve-abi.c
@@ -1,14 +1,96 @@
+/// Check that ABI is correctly implemented.
+///
+///   1. Check that all integer arguments and return values less than 64 bits
+///      are sign/zero extended.
+///   2. Check that all complex arguments and return values are placed in
+///      registers if it is possible.  Not treat it as aggregate.
+///   3. Check that a function declared without argument type declarations is
+///      treated as VARARGS (in order to place arguments in both registers and
+///      memory locations in the back end)
+
 // RUN: %clang_cc1 -triple ve-linux-gnu -emit-llvm %s -o - | FileCheck %s
 
-// CHECK-LABEL: define { float, float } @p(float %a.coerce0, float %a.coerce1, float %b.coerce0, float %b.coerce1) #0 {
-float __complex__ p(float __complex__ a, float __complex__ b) {
+// CHECK-LABEL: define signext i8 @fun_si8(i8 signext %a, i8 signext %b) #0 {
+char fun_si8(char a, char b) {
+  return a;
+}
+
+// CHECK-LABEL: define zeroext i8 @fun_zi8(i8 zeroext %a, i8 zeroext %b) #0 {
+unsigned char fun_zi8(unsigned char a, unsigned char b) {
+  return a;
+}
+
+// CHECK-LABEL: define signext i16 @fun_si16(i16 signext %a, i16 signext %b) #0 {
+short fun_si16(short a, short b) {
+  return a;
+}
+
+// CHECK-LABEL: define zeroext i16 @fun_zi16(i16 zeroext %a, i16 zeroext %b) #0 {
+unsigned short fun_zi16(unsigned short a, unsigned short b) {
+  return a;
+}
+
+// CHECK-LABEL: define signext i32 @fun_si32(i32 signext %a, i32 signext %b) #0 {
+int fun_si32(int a, int b) {
+  return a;
+}
+
+// CHECK-LABEL: define zeroext i32 @fun_zi32(i32 zeroext %a, i32 zeroext %b) #0 {
+unsigned int fun_zi32(unsigned int a, unsigned int b) {
+  return a;
+}
+
+// CHECK-LABEL: define i64 @fun_si64(i64 %a, i64 %b) #0 {
+long fun_si64(long a, long b) {
+  return a;
+}
+
+// CHECK-LABEL: define i64 @fun_zi64(i64 %a, i64 %b) #0 {
+unsigned long fun_zi64(unsigned long a, unsigned long b) {
+  return a;
+}
+
+// CHECK-LABEL: define i128 @fun_si128(i128 %a, i128 %b) #0 {
+__int128 fun_si128(__int128 a, __int128 b) {
+}
+
+// CHECK-LABEL: define i128 @fun_zi128(i128 %a, i128 %b) #0 {
+unsigned __int128 fun_zi128(unsigned __int128 a, unsigned __int128 b) {
+  return a;
+}
+
+// CHECK-LABEL: define float @fun_float(float %a, float %b) #0 {
+float fun_float(float a, float b) {
+  return a;
+}
+
+// CHECK-LABEL: define double @fun_double(double %a, double %b) #0 {
+double fun_double(double a, double b) {
+  return a;
+}
+
+// CHECK-LABEL: define fp128 @fun_quad(fp128 %a, fp128 %b) #0 {
+long double fun_quad(long double a, long double b) {
+  return a;
+}
+
+// CHECK-LABEL: define { float, float } @fun_fcomplex(float %a.coerce0, float %a.coerce1, float %b.coerce0, float %b.coerce1) #0 {
+float __complex__ fun_fcomplex(float __complex__ a, float __complex__ b) {
+  return a;
+}
+
+// CHECK-LABEL: define { double, double } @fun_dcomplex(double %a.coerce0, double %a.coerce1, double %b.coerce0, double %b.coerce1) #0 {
+double __complex__ fun_dcomplex(double __complex__ a, double __complex__ b) {
+  return a;
 }
 
-// CHECK-LABEL: define { double, double } @q(double %a.coerce0, double %a.coerce1, double %b.coerce0, double %b.coerce1) #0 {
-double __complex__ q(double __complex__ a, double __complex__ b) {
+// CHECK-LABEL: define { fp128, fp128 } @fun_qcomplex(fp128 %a.coerce0, fp128 %a.coerce1, fp128 %b.coerce0, fp128 %b.coerce1) #0 {
+long double __complex__ fun_qcomplex(long double __complex__ a, long double __complex__ b) {
+  return a;
 }
 
+extern int hoge();
 void func() {
-  // CHECK-LABEL: %call = call i32 (i32, i32, i32, i32, i32, i32, i32, ...) bitcast (i32 (...)* @hoge to i32 (i32, i32, i32, i32, i32, i32, i32, ...)*)(i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7)
+  // CHECK: %call = call signext i32 (i32, i32, i32, i32, i32, i32, i32, ...) bitcast (i32 (...)* @hoge to i32 (i32, i32, i32, i32, i32, i32, i32, ...)*)(i32 signext 1, i32 signext 2, i32 signext 3, i32 signext 4, i32 signext 5, i32 signext 6, i32 signext 7)
   hoge(1, 2, 3, 4, 5, 6, 7);
 }

From 509f5c4ec2dbae99713ac6292c3a6bd7ba50027c Mon Sep 17 00:00:00 2001
From: hgreving <hgreving@hendrik-laptop.lan>
Date: Wed, 29 Jul 2020 16:48:03 -0700
Subject: [PATCH 0966/1035] [MC] Fix memory leak when allocating MCInst with
 bump allocator

Adds the function createMCInst() to MCContext that creates a MCInst using
a typed bump alloctor.

MCInst contains a SmallVector<MCOperand, 8>. The SmallVector is POD only
for <= 8 operands. The default untyped bump pointer allocator of MCContext
does not delete the MCInst, so if the SmallVector grows, it's a leak.

This fixes https://bugs.llvm.org/show_bug.cgi?id=46900.
---
 llvm/include/llvm/MC/MCContext.h               |  6 ++++++
 llvm/lib/MC/MCContext.cpp                      |  9 +++++++++
 .../Hexagon/AsmParser/HexagonAsmParser.cpp     |  2 +-
 .../Disassembler/HexagonDisassembler.cpp       |  6 +++---
 llvm/lib/Target/Hexagon/HexagonMCInstLower.cpp |  2 +-
 .../Hexagon/MCTargetDesc/HexagonAsmBackend.cpp |  4 ++--
 .../Hexagon/MCTargetDesc/HexagonMCCompound.cpp | 18 +++++++++---------
 7 files changed, 31 insertions(+), 16 deletions(-)

diff --git a/llvm/include/llvm/MC/MCContext.h b/llvm/include/llvm/MC/MCContext.h
index 45be9bb3d225b..d041b06c5568c 100644
--- a/llvm/include/llvm/MC/MCContext.h
+++ b/llvm/include/llvm/MC/MCContext.h
@@ -97,6 +97,7 @@ namespace llvm {
     SpecificBumpPtrAllocator<MCSectionMachO> MachOAllocator;
     SpecificBumpPtrAllocator<MCSectionWasm> WasmAllocator;
     SpecificBumpPtrAllocator<MCSectionXCOFF> XCOFFAllocator;
+    SpecificBumpPtrAllocator<MCInst> MCInstAllocator;
 
     /// Bindings of names to symbols.
     SymbolTable Symbols;
@@ -380,6 +381,11 @@ namespace llvm {
 
     /// @}
 
+    /// \name McInst Management
+
+    /// Create and return a new MC instruction.
+    MCInst *createMCInst();
+
     /// \name Symbol Management
     /// @{
 
diff --git a/llvm/lib/MC/MCContext.cpp b/llvm/lib/MC/MCContext.cpp
index a0f9212f3b142..5b0078974318a 100644
--- a/llvm/lib/MC/MCContext.cpp
+++ b/llvm/lib/MC/MCContext.cpp
@@ -90,6 +90,7 @@ void MCContext::reset() {
   ELFAllocator.DestroyAll();
   MachOAllocator.DestroyAll();
   XCOFFAllocator.DestroyAll();
+  MCInstAllocator.DestroyAll();
 
   MCSubtargetAllocator.DestroyAll();
   InlineAsmUsedLabelNames.clear();
@@ -126,6 +127,14 @@ void MCContext::reset() {
   HadError = false;
 }
 
+//===----------------------------------------------------------------------===//
+// MCInst Management
+//===----------------------------------------------------------------------===//
+
+MCInst *MCContext::createMCInst() {
+  return new (MCInstAllocator.Allocate()) MCInst;
+}
+
 //===----------------------------------------------------------------------===//
 // Symbol Manipulation
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp b/llvm/lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp
index 1e7862c36ea09..3759962c41563 100644
--- a/llvm/lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp
+++ b/llvm/lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp
@@ -641,7 +641,7 @@ bool HexagonAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
       return true;
     return finishBundle(IDLoc, Out);
   }
-  MCInst *SubInst = new (getParser().getContext()) MCInst;
+  MCInst *SubInst = getParser().getContext().createMCInst();
   if (matchOneInstruction(*SubInst, IDLoc, Operands, ErrorInfo,
                           MatchingInlineAsm)) {
     if (InBrackets)
diff --git a/llvm/lib/Target/Hexagon/Disassembler/HexagonDisassembler.cpp b/llvm/lib/Target/Hexagon/Disassembler/HexagonDisassembler.cpp
index f3a87ef20a608..aeaeac65de960 100644
--- a/llvm/lib/Target/Hexagon/Disassembler/HexagonDisassembler.cpp
+++ b/llvm/lib/Target/Hexagon/Disassembler/HexagonDisassembler.cpp
@@ -175,7 +175,7 @@ DecodeStatus HexagonDisassembler::getInstruction(MCInst &MI, uint64_t &Size,
   while (Result == Success && !Complete) {
     if (Bytes.size() < HEXAGON_INSTR_SIZE)
       return MCDisassembler::Fail;
-    MCInst *Inst = new (getContext()) MCInst;
+    MCInst *Inst = getContext().createMCInst();
     Result = getSingleInstruction(*Inst, MI, Bytes, Address, cs, Complete);
     MI.addOperand(MCOperand::createInst(Inst));
     Size += HEXAGON_INSTR_SIZE;
@@ -384,8 +384,8 @@ DecodeStatus HexagonDisassembler::getSingleInstruction(MCInst &MI, MCInst &MCB,
       break;
     }
     MI.setOpcode(Hexagon::DuplexIClass0 + duplexIClass);
-    MCInst *MILow = new (getContext()) MCInst;
-    MCInst *MIHigh = new (getContext()) MCInst;
+    MCInst *MILow = getContext().createMCInst();
+    MCInst *MIHigh = getContext().createMCInst();
     auto TmpExtender = CurrentExtender;
     CurrentExtender =
         nullptr; // constant extenders in duplex must always be in slot 1
diff --git a/llvm/lib/Target/Hexagon/HexagonMCInstLower.cpp b/llvm/lib/Target/Hexagon/HexagonMCInstLower.cpp
index 188d91355a35e..9507de95231fe 100644
--- a/llvm/lib/Target/Hexagon/HexagonMCInstLower.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonMCInstLower.cpp
@@ -104,7 +104,7 @@ void llvm::HexagonLowerToMC(const MCInstrInfo &MCII, const MachineInstr *MI,
     HexagonMCInstrInfo::setOuterLoop(MCB);
     return;
   }
-  MCInst *MCI = new (AP.OutContext) MCInst;
+  MCInst *MCI = AP.OutContext.createMCInst();
   MCI->setOpcode(MI->getOpcode());
   assert(MCI->getOpcode() == static_cast<unsigned>(MI->getOpcode()) &&
          "MCI opcode should have been set on construction");
diff --git a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonAsmBackend.cpp b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonAsmBackend.cpp
index e7069819fa575..627c53cadd84f 100644
--- a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonAsmBackend.cpp
+++ b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonAsmBackend.cpp
@@ -74,7 +74,7 @@ class HexagonAsmBackend : public MCAsmBackend {
 
   void setExtender(MCContext &Context) const {
     if (Extender == nullptr)
-      const_cast<HexagonAsmBackend *>(this)->Extender = new (Context) MCInst;
+      const_cast<HexagonAsmBackend *>(this)->Extender = Context.createMCInst();
   }
 
   MCInst *takeExtender() const {
@@ -736,7 +736,7 @@ class HexagonAsmBackend : public MCAsmBackend {
               auto &Inst = const_cast<MCInst &>(RF.getInst());
               while (Size > 0 &&
                      HexagonMCInstrInfo::bundleSize(Inst) < MaxPacketSize) {
-                MCInst *Nop = new (Context) MCInst;
+                MCInst *Nop = Context.createMCInst();
                 Nop->setOpcode(Hexagon::A2_nop);
                 Inst.addOperand(MCOperand::createInst(Nop));
                 Size -= 4;
diff --git a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCCompound.cpp b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCCompound.cpp
index 82b2074c5cd86..e7ade7834a9f4 100644
--- a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCCompound.cpp
+++ b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCCompound.cpp
@@ -210,7 +210,7 @@ static MCInst *getCompoundInsn(MCContext &Context, MCInst const &L,
   case Hexagon::A2_tfrsi:
     Rt = L.getOperand(0);
     compoundOpcode = J4_jumpseti;
-    CompoundInsn = new (Context) MCInst;
+    CompoundInsn = Context.createMCInst();
     CompoundInsn->setOpcode(compoundOpcode);
 
     CompoundInsn->addOperand(Rt);
@@ -223,7 +223,7 @@ static MCInst *getCompoundInsn(MCContext &Context, MCInst const &L,
     Rs = L.getOperand(1);
 
     compoundOpcode = J4_jumpsetr;
-    CompoundInsn = new (Context) MCInst;
+    CompoundInsn = Context.createMCInst();
     CompoundInsn->setOpcode(compoundOpcode);
     CompoundInsn->addOperand(Rt);
     CompoundInsn->addOperand(Rs);
@@ -237,7 +237,7 @@ static MCInst *getCompoundInsn(MCContext &Context, MCInst const &L,
     Rt = L.getOperand(2);
 
     compoundOpcode = cmpeqBitOpcode[getCompoundOp(R)];
-    CompoundInsn = new (Context) MCInst;
+    CompoundInsn = Context.createMCInst();
     CompoundInsn->setOpcode(compoundOpcode);
     CompoundInsn->addOperand(Rs);
     CompoundInsn->addOperand(Rt);
@@ -250,7 +250,7 @@ static MCInst *getCompoundInsn(MCContext &Context, MCInst const &L,
     Rt = L.getOperand(2);
 
     compoundOpcode = cmpgtBitOpcode[getCompoundOp(R)];
-    CompoundInsn = new (Context) MCInst;
+    CompoundInsn = Context.createMCInst();
     CompoundInsn->setOpcode(compoundOpcode);
     CompoundInsn->addOperand(Rs);
     CompoundInsn->addOperand(Rt);
@@ -263,7 +263,7 @@ static MCInst *getCompoundInsn(MCContext &Context, MCInst const &L,
     Rt = L.getOperand(2);
 
     compoundOpcode = cmpgtuBitOpcode[getCompoundOp(R)];
-    CompoundInsn = new (Context) MCInst;
+    CompoundInsn = Context.createMCInst();
     CompoundInsn->setOpcode(compoundOpcode);
     CompoundInsn->addOperand(Rs);
     CompoundInsn->addOperand(Rt);
@@ -281,7 +281,7 @@ static MCInst *getCompoundInsn(MCContext &Context, MCInst const &L,
       compoundOpcode = cmpeqiBitOpcode[getCompoundOp(R)];
 
     Rs = L.getOperand(1);
-    CompoundInsn = new (Context) MCInst;
+    CompoundInsn = Context.createMCInst();
     CompoundInsn->setOpcode(compoundOpcode);
     CompoundInsn->addOperand(Rs);
     CompoundInsn->addOperand(L.getOperand(2));
@@ -299,7 +299,7 @@ static MCInst *getCompoundInsn(MCContext &Context, MCInst const &L,
       compoundOpcode = cmpgtiBitOpcode[getCompoundOp(R)];
 
     Rs = L.getOperand(1);
-    CompoundInsn = new (Context) MCInst;
+    CompoundInsn = Context.createMCInst();
     CompoundInsn->setOpcode(compoundOpcode);
     CompoundInsn->addOperand(Rs);
     CompoundInsn->addOperand(L.getOperand(2));
@@ -310,7 +310,7 @@ static MCInst *getCompoundInsn(MCContext &Context, MCInst const &L,
     LLVM_DEBUG(dbgs() << "CX: C2_cmpgtui\n");
     Rs = L.getOperand(1);
     compoundOpcode = cmpgtuiBitOpcode[getCompoundOp(R)];
-    CompoundInsn = new (Context) MCInst;
+    CompoundInsn = Context.createMCInst();
     CompoundInsn->setOpcode(compoundOpcode);
     CompoundInsn->addOperand(Rs);
     CompoundInsn->addOperand(L.getOperand(2));
@@ -321,7 +321,7 @@ static MCInst *getCompoundInsn(MCContext &Context, MCInst const &L,
     LLVM_DEBUG(dbgs() << "CX: S2_tstbit_i\n");
     Rs = L.getOperand(1);
     compoundOpcode = tstBitOpcode[getCompoundOp(R)];
-    CompoundInsn = new (Context) MCInst;
+    CompoundInsn = Context.createMCInst();
     CompoundInsn->setOpcode(compoundOpcode);
     CompoundInsn->addOperand(Rs);
     CompoundInsn->addOperand(R.getOperand(1));

From 7c5630fe9908a8bf10be2e9d26054406fac8de87 Mon Sep 17 00:00:00 2001
From: peter klausler <pklausler@nvidia.com>
Date: Mon, 3 Aug 2020 11:20:35 -0700
Subject: [PATCH 0967/1035] [flang] Handle spaces (more) correctly in REAL
 input

Fixes problems in FCVS test fm110.f.
Add more comments, too.

Differential Revision: https://reviews.llvm.org/D85163
---
 flang/runtime/edit-input.cpp | 56 +++++++++++++++++++++---------------
 1 file changed, 33 insertions(+), 23 deletions(-)

diff --git a/flang/runtime/edit-input.cpp b/flang/runtime/edit-input.cpp
index ebbe41b49b2ce..998edc954ba75 100644
--- a/flang/runtime/edit-input.cpp
+++ b/flang/runtime/edit-input.cpp
@@ -13,6 +13,8 @@
 
 namespace Fortran::runtime::io {
 
+// For fixed-width fields, initialize the number of remaining characters.
+// Skip over leading blanks, then return the first non-blank character (if any).
 static std::optional<char32_t> PrepareInput(
     IoStatementState &io, const DataEdit &edit, std::optional<int> &remaining) {
   remaining.reset();
@@ -61,7 +63,8 @@ static bool EditBOZInput(IoStatementState &io, const DataEdit &edit, void *n,
   return true;
 }
 
-// Returns false if there's a '-' sign
+// Prepares input from a field, and consumes the sign, if any.
+// Returns true if there's a '-' sign.
 static bool ScanNumericPrefix(IoStatementState &io, const DataEdit &edit,
     std::optional<char32_t> &next, std::optional<int> &remaining) {
   next = PrepareInput(io, edit, remaining);
@@ -69,6 +72,7 @@ static bool ScanNumericPrefix(IoStatementState &io, const DataEdit &edit,
   if (next) {
     negative = *next == '-';
     if (negative || *next == '+') {
+      io.SkipSpaces(remaining);
       next = io.NextInField(remaining);
     }
   }
@@ -126,39 +130,44 @@ bool EditIntegerInput(
   return true;
 }
 
+// Parses a REAL input number from the input source as a normalized
+// fraction into a supplied buffer -- there's an optional '-', a
+// decimal point, and at least one digit.  The adjusted exponent value
+// is returned in a reference argument.  The returned value is the number
+// of characters that (should) have been written to the buffer -- this can
+// be larger than the buffer size and can indicate overflow.  Replaces
+// blanks with zeroes if appropriate.
 static int ScanRealInput(char *buffer, int bufferSize, IoStatementState &io,
     const DataEdit &edit, int &exponent) {
   std::optional<int> remaining;
   std::optional<char32_t> next;
   int got{0};
   std::optional<int> decimalPoint;
-  if (ScanNumericPrefix(io, edit, next, remaining) && next) {
+  auto Put{[&](char ch) -> void {
     if (got < bufferSize) {
-      buffer[got++] = '-';
+      buffer[got] = ch;
     }
+    ++got;
+  }};
+  if (ScanNumericPrefix(io, edit, next, remaining)) {
+    Put('-');
   }
   if (!next) { // empty field means zero
-    if (got < bufferSize) {
-      buffer[got++] = '0';
-    }
+    Put('0');
     return got;
   }
-  if (got < bufferSize) {
-    buffer[got++] = '.'; // input field is normalized to a fraction
-  }
   char32_t decimal = edit.modes.editingFlags & decimalComma ? ',' : '.';
-  auto start{got};
-  if ((*next >= 'a' && *next <= 'z') || (*next >= 'A' && *next <= 'Z')) {
+  char32_t first{*next >= 'a' && *next <= 'z' ? *next + 'A' - 'a' : *next};
+  if (first == 'N' || first == 'I') {
     // NaN or infinity - convert to upper case
+    // Subtle: a blank field of digits could be followed by 'E' or 'D',
     for (; next &&
          ((*next >= 'a' && *next <= 'z') || (*next >= 'A' && *next <= 'Z'));
          next = io.NextInField(remaining)) {
-      if (got < bufferSize) {
-        if (*next >= 'a' && *next <= 'z') {
-          buffer[got++] = *next - 'a' + 'A';
-        } else {
-          buffer[got++] = *next;
-        }
+      if (*next >= 'a' && *next <= 'z') {
+        Put(*next - 'a' + 'A');
+      } else {
+        Put(*next);
       }
     }
     if (next && *next == '(') { // NaN(...)
@@ -167,7 +176,10 @@ static int ScanRealInput(char *buffer, int bufferSize, IoStatementState &io,
       }
     }
     exponent = 0;
-  } else if (*next == decimal || (*next >= '0' && *next <= '9')) {
+  } else if (first == decimal || (first >= '0' && first <= '9') ||
+      first == 'E' || first == 'D' || first == 'Q') {
+    Put('.'); // input field is normalized to a fraction
+    auto start{got};
     for (; next; next = io.NextInField(remaining)) {
       char32_t ch{*next};
       if (ch == ' ' || ch == '\t') {
@@ -180,9 +192,7 @@ static int ScanRealInput(char *buffer, int bufferSize, IoStatementState &io,
       if (ch == '0' && got == start && !decimalPoint) {
         // omit leading zeroes before the decimal
       } else if (ch >= '0' && ch <= '9') {
-        if (got < bufferSize) {
-          buffer[got++] = ch;
-        }
+        Put(ch);
       } else if (ch == decimal && !decimalPoint) {
         // the decimal point is *not* copied to the buffer
         decimalPoint = got - start; // # of digits before the decimal point
@@ -190,8 +200,8 @@ static int ScanRealInput(char *buffer, int bufferSize, IoStatementState &io,
         break;
       }
     }
-    if (got == start && got < bufferSize) {
-      buffer[got++] = '0'; // all digits were zeroes
+    if (got == start) {
+      Put('0'); // emit at least one digit
     }
     if (next &&
         (*next == 'e' || *next == 'E' || *next == 'd' || *next == 'D' ||

From 49bbb8b60e451d173c7dd42993592e8aa4d95f24 Mon Sep 17 00:00:00 2001
From: peter klausler <pklausler@nvidia.com>
Date: Mon, 3 Aug 2020 11:11:39 -0700
Subject: [PATCH 0968/1035] Remove unneeded RoundDefault enumerator, and fix
 spelling in comments

---
 flang/include/flang/Decimal/decimal.h        | 3 +--
 flang/lib/Decimal/big-radix-floating-point.h | 6 +++---
 flang/lib/Decimal/binary-to-decimal.cpp      | 1 -
 flang/lib/Decimal/decimal-to-binary.cpp      | 2 --
 flang/lib/Evaluate/host.h                    | 6 +++---
 5 files changed, 7 insertions(+), 11 deletions(-)

diff --git a/flang/include/flang/Decimal/decimal.h b/flang/include/flang/Decimal/decimal.h
index fa687e92d35b4..214b1d15d44b3 100644
--- a/flang/include/flang/Decimal/decimal.h
+++ b/flang/include/flang/Decimal/decimal.h
@@ -44,12 +44,11 @@ struct ConversionToDecimalResult {
 };
 
 enum FortranRounding {
-  RoundNearest, /* RN */
+  RoundNearest, /* RN and RP */
   RoundUp, /* RU */
   RoundDown, /* RD */
   RoundToZero, /* RZ - no rounding */
   RoundCompatible, /* RC: like RN, but ties go away from 0 */
-  RoundDefault, /* RP: maps to one of the above */
 };
 
 /* The "minimize" flag causes the fewest number of output digits
diff --git a/flang/lib/Decimal/big-radix-floating-point.h b/flang/lib/Decimal/big-radix-floating-point.h
index 2fbb777104d6d..b0ee69ad5e426 100644
--- a/flang/lib/Decimal/big-radix-floating-point.h
+++ b/flang/lib/Decimal/big-radix-floating-point.h
@@ -66,12 +66,12 @@ template <int PREC, int LOG10RADIX = 16> class BigRadixFloatingPointNumber {
 
 public:
   explicit BigRadixFloatingPointNumber(
-      enum FortranRounding rounding = RoundDefault)
+      enum FortranRounding rounding = RoundNearest)
       : rounding_{rounding} {}
 
   // Converts a binary floating point value.
   explicit BigRadixFloatingPointNumber(
-      Real, enum FortranRounding = RoundDefault);
+      Real, enum FortranRounding = RoundNearest);
 
   BigRadixFloatingPointNumber &SetToZero() {
     isNegative_ = false;
@@ -355,7 +355,7 @@ template <int PREC, int LOG10RADIX = 16> class BigRadixFloatingPointNumber {
   int digitLimit_{maxDigits}; // precision clamp
   int exponent_{0}; // signed power of ten
   bool isNegative_{false};
-  enum FortranRounding rounding_ { RoundDefault };
+  enum FortranRounding rounding_ { RoundNearest };
 };
 } // namespace Fortran::decimal
 #endif
diff --git a/flang/lib/Decimal/binary-to-decimal.cpp b/flang/lib/Decimal/binary-to-decimal.cpp
index bcc0f08558aa4..c89bffc8ccd4c 100644
--- a/flang/lib/Decimal/binary-to-decimal.cpp
+++ b/flang/lib/Decimal/binary-to-decimal.cpp
@@ -143,7 +143,6 @@ BigRadixFloatingPointNumber<PREC, LOG10RADIX>::ConvertToDecimal(char *buffer,
     bool incr{false};
     switch (rounding_) {
     case RoundNearest:
-    case RoundDefault:
       incr = *end > '5' ||
           (*end == '5' && (p > end + 1 || ((end[-1] - '0') & 1) != 0));
       break;
diff --git a/flang/lib/Decimal/decimal-to-binary.cpp b/flang/lib/Decimal/decimal-to-binary.cpp
index 502f0a003d641..5e927e93b3bbe 100644
--- a/flang/lib/Decimal/decimal-to-binary.cpp
+++ b/flang/lib/Decimal/decimal-to-binary.cpp
@@ -150,7 +150,6 @@ void BigRadixFloatingPointNumber<PREC,
   bool incr{false};
   switch (rounding_) {
   case RoundNearest:
-  case RoundDefault:
     incr = LSD > radix / 2 || (LSD == radix / 2 && digit_[0] % 2 != 0);
     break;
   case RoundUp:
@@ -260,7 +259,6 @@ ConversionToBinaryResult<PREC> IntermediateFloat<PREC>::ToBinary(
   bool incr{false};
   switch (rounding) {
   case RoundNearest:
-  case RoundDefault:
     incr = guard > oneHalf || (guard == oneHalf && (fraction & 1));
     break;
   case RoundUp:
diff --git a/flang/lib/Evaluate/host.h b/flang/lib/Evaluate/host.h
index 2fac0424f17cc..1fc2423f4f0c3 100644
--- a/flang/lib/Evaluate/host.h
+++ b/flang/lib/Evaluate/host.h
@@ -140,14 +140,14 @@ template <> struct HostTypeHelper<Type<TypeCategory::Integer, 16>> {
 // It should be defined when gcc/clang have a better support for it.
 
 template <> struct HostTypeHelper<Type<TypeCategory::Real, 4>> {
-  // IEE 754 64bits
+  // IEEE 754 64bits
   using Type = std::conditional_t<sizeof(float) == 4 &&
           std::numeric_limits<float>::is_iec559,
       float, UnsupportedType>;
 };
 
 template <> struct HostTypeHelper<Type<TypeCategory::Real, 8>> {
-  // IEE 754 64bits
+  // IEEE 754 64bits
   using Type = std::conditional_t<sizeof(double) == 8 &&
           std::numeric_limits<double>::is_iec559,
       double, UnsupportedType>;
@@ -162,7 +162,7 @@ template <> struct HostTypeHelper<Type<TypeCategory::Real, 10>> {
 };
 
 template <> struct HostTypeHelper<Type<TypeCategory::Real, 16>> {
-  // IEE 754 128bits
+  // IEEE 754 128bits
   using Type = std::conditional_t<sizeof(long double) == 16 &&
           std::numeric_limits<long double>::digits == 113 &&
           std::numeric_limits<long double>::max_exponent == 16384,

From 3b44b6c900d1b71e6a6590e376d11dc303ac5159 Mon Sep 17 00:00:00 2001
From: Nathan James <n.james93@hotmail.co.uk>
Date: Tue, 4 Aug 2020 00:31:19 +0100
Subject: [PATCH 0969/1035] [clang-tidy][NFC] Use correct size call for reserve

---
 .../clang-tidy/readability/IdentifierNamingCheck.cpp            | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.cpp b/clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.cpp
index e7fe25d8e2214..c2a32474b2a8b 100644
--- a/clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.cpp
@@ -124,7 +124,7 @@ static StringRef const StyleNames[] = {
 static std::vector<llvm::Optional<IdentifierNamingCheck::NamingStyle>>
 getNamingStyles(const ClangTidyCheck::OptionsView &Options) {
   std::vector<llvm::Optional<IdentifierNamingCheck::NamingStyle>> Styles;
-  Styles.reserve(StyleNames->size());
+  Styles.reserve(array_lengthof(StyleNames));
   for (auto const &StyleName : StyleNames) {
     auto CaseOptional = Options.getOptional<IdentifierNamingCheck::CaseType>(
         (StyleName + "Case").str());

From adb5c23f8c0d60eeec41dcbe21d1b26184e1c97d Mon Sep 17 00:00:00 2001
From: Jordan Rupprecht <rupprecht@google.com>
Date: Mon, 3 Aug 2020 16:30:41 -0700
Subject: [PATCH 0970/1035] [test] Exit with an error if no tests are run.

If the test suite is misconfigured when it's run (a bad regexp, wrong test directory, etc.), the test suite may not discover any tests. When this happens, the test runner exits happily because no tests failed:

```
Ran 0 tests in 0.000s
RESULT: PASSED (0 passes, 0 failures, 0 errors, 0 skipped, 0 expected failures, 0 unexpected successes)
```

Change this to return an error so the misconfiguration can be more easily detected. Verified that `lldb-dotest -p TestDoesNotExist.py` successfully fails.

Reviewed By: JDevlieghere

Differential Revision: https://reviews.llvm.org/D85169
---
 lldb/packages/Python/lldbsuite/test/dotest.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/lldb/packages/Python/lldbsuite/test/dotest.py b/lldb/packages/Python/lldbsuite/test/dotest.py
index 3fb802f1c1aa5..6607f52c49dbd 100644
--- a/lldb/packages/Python/lldbsuite/test/dotest.py
+++ b/lldb/packages/Python/lldbsuite/test/dotest.py
@@ -1039,6 +1039,10 @@ def run_suite():
             (configuration.suite.countTestCases(),
              configuration.suite.countTestCases() != 1 and "s" or ""))
 
+    if configuration.suite.countTestCases() == 0:
+        logging.error("did not discover any matching tests")
+        exitTestSuite(1)
+
     # Invoke the test runner.
     if configuration.count == 1:
         result = unittest2.TextTestRunner(

From c9e6887f837933aa7e2bb2511a6f883739e30faa Mon Sep 17 00:00:00 2001
From: Christopher Tetreault <ctetreau@quicinc.com>
Date: Mon, 3 Aug 2020 16:15:45 -0700
Subject: [PATCH 0971/1035] [SVE] Remove bad calls to
 VectorType::getNumElements() from X86

Reviewed By: RKSimon

Differential Revision: https://reviews.llvm.org/D85156
---
 .../Target/X86/X86InstCombineIntrinsic.cpp    | 51 ++++++++++---------
 1 file changed, 26 insertions(+), 25 deletions(-)

diff --git a/llvm/lib/Target/X86/X86InstCombineIntrinsic.cpp b/llvm/lib/Target/X86/X86InstCombineIntrinsic.cpp
index c9e092e5deca7..3632305aa5cbf 100644
--- a/llvm/lib/Target/X86/X86InstCombineIntrinsic.cpp
+++ b/llvm/lib/Target/X86/X86InstCombineIntrinsic.cpp
@@ -202,7 +202,7 @@ static Value *simplifyX86immShift(const IntrinsicInst &II,
 
   auto Vec = II.getArgOperand(0);
   auto Amt = II.getArgOperand(1);
-  auto VT = cast<VectorType>(Vec->getType());
+  auto VT = cast<FixedVectorType>(Vec->getType());
   auto SVT = VT->getElementType();
   auto AmtVT = Amt->getType();
   unsigned VWidth = VT->getNumElements();
@@ -234,7 +234,7 @@ static Value *simplifyX86immShift(const IntrinsicInst &II,
     assert(AmtVT->isVectorTy() && AmtVT->getPrimitiveSizeInBits() == 128 &&
            cast<VectorType>(AmtVT)->getElementType() == SVT &&
            "Unexpected shift-by-scalar type");
-    unsigned NumAmtElts = cast<VectorType>(AmtVT)->getNumElements();
+    unsigned NumAmtElts = cast<FixedVectorType>(AmtVT)->getNumElements();
     APInt DemandedLower = APInt::getOneBitSet(NumAmtElts, 0);
     APInt DemandedUpper = APInt::getBitsSet(NumAmtElts, 1, NumAmtElts / 2);
     KnownBits KnownLowerBits = llvm::computeKnownBits(
@@ -350,7 +350,7 @@ static Value *simplifyX86varShift(const IntrinsicInst &II,
 
   auto Vec = II.getArgOperand(0);
   auto Amt = II.getArgOperand(1);
-  auto VT = cast<VectorType>(II.getType());
+  auto VT = cast<FixedVectorType>(II.getType());
   auto SVT = VT->getElementType();
   int NumElts = VT->getNumElements();
   int BitWidth = SVT->getIntegerBitWidth();
@@ -448,10 +448,10 @@ static Value *simplifyX86pack(IntrinsicInst &II,
   if (isa<UndefValue>(Arg0) && isa<UndefValue>(Arg1))
     return UndefValue::get(ResTy);
 
-  auto *ArgTy = cast<VectorType>(Arg0->getType());
+  auto *ArgTy = cast<FixedVectorType>(Arg0->getType());
   unsigned NumLanes = ResTy->getPrimitiveSizeInBits() / 128;
   unsigned NumSrcElts = ArgTy->getNumElements();
-  assert(cast<VectorType>(ResTy)->getNumElements() == (2 * NumSrcElts) &&
+  assert(cast<FixedVectorType>(ResTy)->getNumElements() == (2 * NumSrcElts) &&
          "Unexpected packing types");
 
   unsigned NumSrcEltsPerLane = NumSrcElts / NumLanes;
@@ -513,7 +513,7 @@ static Value *simplifyX86movmsk(const IntrinsicInst &II,
   if (isa<UndefValue>(Arg))
     return Constant::getNullValue(ResTy);
 
-  auto *ArgTy = dyn_cast<VectorType>(Arg->getType());
+  auto *ArgTy = dyn_cast<FixedVectorType>(Arg->getType());
   // We can't easily peek through x86_mmx types.
   if (!ArgTy)
     return nullptr;
@@ -567,7 +567,7 @@ static Value *simplifyX86insertps(const IntrinsicInst &II,
   if (!CInt)
     return nullptr;
 
-  VectorType *VecTy = cast<VectorType>(II.getType());
+  auto *VecTy = cast<FixedVectorType>(II.getType());
   assert(VecTy->getNumElements() == 4 && "insertps with wrong vector type");
 
   // The immediate permute control byte looks like this:
@@ -810,7 +810,7 @@ static Value *simplifyX86pshufb(const IntrinsicInst &II,
   if (!V)
     return nullptr;
 
-  auto *VecTy = cast<VectorType>(II.getType());
+  auto *VecTy = cast<FixedVectorType>(II.getType());
   unsigned NumElts = VecTy->getNumElements();
   assert((NumElts == 16 || NumElts == 32 || NumElts == 64) &&
          "Unexpected number of elements in shuffle mask!");
@@ -855,7 +855,7 @@ static Value *simplifyX86vpermilvar(const IntrinsicInst &II,
   if (!V)
     return nullptr;
 
-  auto *VecTy = cast<VectorType>(II.getType());
+  auto *VecTy = cast<FixedVectorType>(II.getType());
   unsigned NumElts = VecTy->getNumElements();
   bool IsPD = VecTy->getScalarType()->isDoubleTy();
   unsigned NumLaneElts = IsPD ? 2 : 4;
@@ -903,7 +903,7 @@ static Value *simplifyX86vpermv(const IntrinsicInst &II,
   if (!V)
     return nullptr;
 
-  auto *VecTy = cast<VectorType>(II.getType());
+  auto *VecTy = cast<FixedVectorType>(II.getType());
   unsigned Size = VecTy->getNumElements();
   assert((Size == 4 || Size == 8 || Size == 16 || Size == 32 || Size == 64) &&
          "Unexpected shuffle mask size");
@@ -1084,7 +1084,7 @@ X86TTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
     // These intrinsics only demand the 0th element of their input vectors. If
     // we can simplify the input based on that, do so now.
     Value *Arg = II.getArgOperand(0);
-    unsigned VWidth = cast<VectorType>(Arg->getType())->getNumElements();
+    unsigned VWidth = cast<FixedVectorType>(Arg->getType())->getNumElements();
     if (Value *V = SimplifyDemandedVectorEltsLow(Arg, VWidth, 1)) {
       return IC.replaceOperand(II, 0, V);
     }
@@ -1136,7 +1136,7 @@ X86TTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
     bool MadeChange = false;
     Value *Arg0 = II.getArgOperand(0);
     Value *Arg1 = II.getArgOperand(1);
-    unsigned VWidth = cast<VectorType>(Arg0->getType())->getNumElements();
+    unsigned VWidth = cast<FixedVectorType>(Arg0->getType())->getNumElements();
     if (Value *V = SimplifyDemandedVectorEltsLow(Arg0, VWidth, 1)) {
       IC.replaceOperand(II, 0, V);
       MadeChange = true;
@@ -1362,7 +1362,7 @@ X86TTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
     Value *Arg1 = II.getArgOperand(1);
     assert(Arg1->getType()->getPrimitiveSizeInBits() == 128 &&
            "Unexpected packed shift size");
-    unsigned VWidth = cast<VectorType>(Arg1->getType())->getNumElements();
+    unsigned VWidth = cast<FixedVectorType>(Arg1->getType())->getNumElements();
 
     if (Value *V = SimplifyDemandedVectorEltsLow(Arg1, VWidth, VWidth / 2)) {
       return IC.replaceOperand(II, 1, V);
@@ -1433,7 +1433,8 @@ X86TTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
       bool MadeChange = false;
       Value *Arg0 = II.getArgOperand(0);
       Value *Arg1 = II.getArgOperand(1);
-      unsigned VWidth = cast<VectorType>(Arg0->getType())->getNumElements();
+      unsigned VWidth =
+          cast<FixedVectorType>(Arg0->getType())->getNumElements();
 
       APInt UndefElts1(VWidth, 0);
       APInt DemandedElts1 =
@@ -1476,8 +1477,8 @@ X86TTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
   case Intrinsic::x86_sse4a_extrq: {
     Value *Op0 = II.getArgOperand(0);
     Value *Op1 = II.getArgOperand(1);
-    unsigned VWidth0 = cast<VectorType>(Op0->getType())->getNumElements();
-    unsigned VWidth1 = cast<VectorType>(Op1->getType())->getNumElements();
+    unsigned VWidth0 = cast<FixedVectorType>(Op0->getType())->getNumElements();
+    unsigned VWidth1 = cast<FixedVectorType>(Op1->getType())->getNumElements();
     assert(Op0->getType()->getPrimitiveSizeInBits() == 128 &&
            Op1->getType()->getPrimitiveSizeInBits() == 128 && VWidth0 == 2 &&
            VWidth1 == 16 && "Unexpected operand sizes");
@@ -1517,7 +1518,7 @@ X86TTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
     // EXTRQI: Extract Length bits starting from Index. Zero pad the remaining
     // bits of the lower 64-bits. The upper 64-bits are undefined.
     Value *Op0 = II.getArgOperand(0);
-    unsigned VWidth = cast<VectorType>(Op0->getType())->getNumElements();
+    unsigned VWidth = cast<FixedVectorType>(Op0->getType())->getNumElements();
     assert(Op0->getType()->getPrimitiveSizeInBits() == 128 && VWidth == 2 &&
            "Unexpected operand size");
 
@@ -1541,10 +1542,10 @@ X86TTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
   case Intrinsic::x86_sse4a_insertq: {
     Value *Op0 = II.getArgOperand(0);
     Value *Op1 = II.getArgOperand(1);
-    unsigned VWidth = cast<VectorType>(Op0->getType())->getNumElements();
+    unsigned VWidth = cast<FixedVectorType>(Op0->getType())->getNumElements();
     assert(Op0->getType()->getPrimitiveSizeInBits() == 128 &&
            Op1->getType()->getPrimitiveSizeInBits() == 128 && VWidth == 2 &&
-           cast<VectorType>(Op1->getType())->getNumElements() == 2 &&
+           cast<FixedVectorType>(Op1->getType())->getNumElements() == 2 &&
            "Unexpected operand size");
 
     // See if we're dealing with constant values.
@@ -1577,8 +1578,8 @@ X86TTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
     // undefined.
     Value *Op0 = II.getArgOperand(0);
     Value *Op1 = II.getArgOperand(1);
-    unsigned VWidth0 = cast<VectorType>(Op0->getType())->getNumElements();
-    unsigned VWidth1 = cast<VectorType>(Op1->getType())->getNumElements();
+    unsigned VWidth0 = cast<FixedVectorType>(Op0->getType())->getNumElements();
+    unsigned VWidth1 = cast<FixedVectorType>(Op1->getType())->getNumElements();
     assert(Op0->getType()->getPrimitiveSizeInBits() == 128 &&
            Op1->getType()->getPrimitiveSizeInBits() == 128 && VWidth0 == 2 &&
            VWidth1 == 2 && "Unexpected operand sizes");
@@ -1650,9 +1651,9 @@ X86TTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
              "Not expecting mask and operands with different sizes");
 
       unsigned NumMaskElts =
-          cast<VectorType>(Mask->getType())->getNumElements();
+          cast<FixedVectorType>(Mask->getType())->getNumElements();
       unsigned NumOperandElts =
-          cast<VectorType>(II.getType())->getNumElements();
+          cast<FixedVectorType>(II.getType())->getNumElements();
       if (NumMaskElts == NumOperandElts) {
         return SelectInst::Create(BoolVec, Op1, Op0);
       }
@@ -1768,7 +1769,7 @@ Optional<Value *> X86TTIImpl::simplifyDemandedUseBitsIntrinsic(
       ArgWidth = 8; // Arg is x86_mmx, but treated as <8 x i8>.
     } else {
       auto Arg = II.getArgOperand(0);
-      auto ArgType = cast<VectorType>(Arg->getType());
+      auto ArgType = cast<FixedVectorType>(Arg->getType());
       ArgWidth = ArgType->getNumElements();
     }
 
@@ -1934,7 +1935,7 @@ Optional<Value *> X86TTIImpl::simplifyDemandedVectorEltsIntrinsic(
   case Intrinsic::x86_avx512_packusdw_512:
   case Intrinsic::x86_avx512_packuswb_512: {
     auto *Ty0 = II.getArgOperand(0)->getType();
-    unsigned InnerVWidth = cast<VectorType>(Ty0)->getNumElements();
+    unsigned InnerVWidth = cast<FixedVectorType>(Ty0)->getNumElements();
     assert(VWidth == (InnerVWidth * 2) && "Unexpected input size");
 
     unsigned NumLanes = Ty0->getPrimitiveSizeInBits() / 128;

From d8334c43606a08dc13a69d0993dc7a52d5c0fe56 Mon Sep 17 00:00:00 2001
From: peter klausler <pklausler@nvidia.com>
Date: Mon, 3 Aug 2020 11:29:15 -0700
Subject: [PATCH 0972/1035] [flang] Acquire file accessibility, size,
 positioning

Extend the raw file wrapper to get accessibility, positioning,
and size information.  This is needed for INQUIRE (to follow).

Differential Revision: https://reviews.llvm.org/D85160
---
 flang/runtime/file.cpp | 21 +++++++++++++++++++--
 flang/runtime/file.h   |  4 ++++
 2 files changed, 23 insertions(+), 2 deletions(-)

diff --git a/flang/runtime/file.cpp b/flang/runtime/file.cpp
index 341702df995b8..6823b19e57912 100644
--- a/flang/runtime/file.cpp
+++ b/flang/runtime/file.cpp
@@ -18,6 +18,7 @@
 #include <io.h>
 #include <windows.h>
 #else
+#include <sys/stat.h>
 #include <unistd.h>
 #endif
 
@@ -84,8 +85,7 @@ void OpenFile::Open(OpenStatus status, std::optional<Action> action,
     fd_ = openfile_mkstemp(handler);
   } else {
     if (!path_.get()) {
-      handler.SignalError(
-          "FILE= is required unless STATUS='OLD' and unit is connected");
+      handler.SignalError("FILE= is required");
       return;
     }
     int flags{0};
@@ -134,8 +134,18 @@ void OpenFile::Open(OpenStatus status, std::optional<Action> action,
   mayWrite_ = *action != Action::Read;
   if (status == OpenStatus::Old || status == OpenStatus::Unknown) {
     knownSize_.reset();
+#ifndef _WIN32
+    struct stat buf;
+    if (::fstat(fd_, &buf) == 0) {
+      mayPosition_ = S_ISREG(buf.st_mode);
+      knownSize_ = buf.st_size;
+    }
+#else // TODO: _WIN32
+    mayPosition_ = true;
+#endif
   } else {
     knownSize_ = 0;
+    mayPosition_ = true;
   }
 }
 
@@ -385,4 +395,11 @@ int OpenFile::PendingResult(const Terminator &terminator, int iostat) {
 }
 
 bool IsATerminal(int fd) { return ::isatty(fd); }
+
+bool IsExtant(const char *path) { return ::access(path, F_OK) == 0; }
+bool MayRead(const char *path) { return ::access(path, R_OK) == 0; }
+bool MayWrite(const char *path) { return ::access(path, W_OK) == 0; }
+bool MayReadAndWrite(const char *path) {
+  return ::access(path, R_OK | W_OK) == 0;
+}
 } // namespace Fortran::runtime::io
diff --git a/flang/runtime/file.h b/flang/runtime/file.h
index 1d25a91558a4c..7e7b27c4be2a4 100644
--- a/flang/runtime/file.h
+++ b/flang/runtime/file.h
@@ -95,5 +95,9 @@ class OpenFile {
 };
 
 bool IsATerminal(int fd);
+bool IsExtant(const char *path);
+bool MayRead(const char *path);
+bool MayWrite(const char *path);
+bool MayReadAndWrite(const char *path);
 } // namespace Fortran::runtime::io
 #endif // FORTRAN_RUNTIME_FILE_H_

From d879ac8a6eef683392a02dd0aa62c69d61b894eb Mon Sep 17 00:00:00 2001
From: peter klausler <pklausler@nvidia.com>
Date: Mon, 3 Aug 2020 11:31:13 -0700
Subject: [PATCH 0973/1035] [flang] Defer "next input record" processing until
 handlers established

External input was detecting "end of file" conditions in
BeginExternal...Input() and BeginUnformattedInput() routines
before EnableHandlers() could have been called.  Defer the
"start next record" processing to the input data item
handlers (and EndIoStatement() for when there are no data
items).

Differential Revision: https://reviews.llvm.org/D85161
---
 flang/runtime/io-api.cpp  | 17 ++++++++---------
 flang/runtime/io-stmt.cpp | 19 ++++++++++++++++++-
 flang/runtime/io-stmt.h   |  6 ++++++
 3 files changed, 32 insertions(+), 10 deletions(-)

diff --git a/flang/runtime/io-api.cpp b/flang/runtime/io-api.cpp
index 708090ac68561..f36144d0c3c46 100644
--- a/flang/runtime/io-api.cpp
+++ b/flang/runtime/io-api.cpp
@@ -149,9 +149,6 @@ Cookie BeginExternalListIO(
   unit.SetDirection(DIR, handler);
   IoStatementState &io{unit.BeginIoStatement<ExternalListIoStatementState<DIR>>(
       unit, sourceFile, sourceLine)};
-  if constexpr (DIR == Direction::Input) {
-    unit.BeginReadingRecord(handler);
-  }
   return &io;
 }
 
@@ -185,9 +182,6 @@ Cookie BeginExternalFormattedIO(const char *format, std::size_t formatLength,
   IoStatementState &io{
       unit.BeginIoStatement<ExternalFormattedIoStatementState<DIR>>(
           unit, format, formatLength, sourceFile, sourceLine)};
-  if constexpr (DIR == Direction::Input) {
-    unit.BeginReadingRecord(handler);
-  }
   return &io;
 }
 
@@ -218,9 +212,7 @@ Cookie BeginUnformattedIO(
       unit, sourceFile, sourceLine)};
   IoErrorHandler handler{terminator};
   unit.SetDirection(DIR, handler);
-  if constexpr (DIR == Direction::Input) {
-    unit.BeginReadingRecord(handler);
-  } else {
+  if constexpr (DIR == Direction::Output) {
     if (unit.access == Access::Sequential && !unit.isFixedRecordLength) {
       // Create space for (sub)record header to be completed by
       // UnformattedIoStatementState<Direction::Output>::EndIoStatement()
@@ -838,6 +830,7 @@ bool IONAME(OutputDescriptor)(Cookie cookie, const Descriptor &) {
 
 bool IONAME(InputDescriptor)(Cookie cookie, const Descriptor &) {
   IoStatementState &io{*cookie};
+  io.BeginReadingRecord();
   io.GetIoErrorHandler().Crash("InputDescriptor: not yet implemented"); // TODO
 }
 
@@ -855,6 +848,7 @@ bool IONAME(OutputUnformattedBlock)(Cookie cookie, const char *x,
 bool IONAME(InputUnformattedBlock)(
     Cookie cookie, char *x, std::size_t length, std::size_t elementBytes) {
   IoStatementState &io{*cookie};
+  io.BeginReadingRecord();
   if (auto *unf{io.get_if<UnformattedIoStatementState<Direction::Input>>()}) {
     return unf->Receive(x, length, elementBytes);
   }
@@ -883,6 +877,7 @@ bool IONAME(InputInteger)(Cookie cookie, std::int64_t &n, int kind) {
         "InputInteger64() called for a non-input I/O statement");
     return false;
   }
+  io.BeginReadingRecord();
   if (auto edit{io.GetNextDataEdit()}) {
     if (edit->descriptor == DataEdit::ListDirectedNullValue) {
       return true;
@@ -922,6 +917,7 @@ static bool InputReal(Cookie cookie, REAL &x) {
         "InputReal() called for a non-input I/O statement");
     return false;
   }
+  io.BeginReadingRecord();
   if (auto edit{io.GetNextDataEdit()}) {
     if (edit->descriptor == DataEdit::ListDirectedNullValue) {
       return true;
@@ -968,6 +964,7 @@ static bool InputComplex(Cookie cookie, REAL x[2]) {
         "InputComplex() called for a non-input I/O statement");
     return false;
   }
+  io.BeginReadingRecord();
   for (int j{0}; j < 2; ++j) {
     if (auto edit{io.GetNextDataEdit()}) {
       if (edit->descriptor == DataEdit::ListDirectedNullValue) {
@@ -1012,6 +1009,7 @@ bool IONAME(InputAscii)(Cookie cookie, char *x, std::size_t length) {
         "InputAscii() called for a non-input I/O statement");
     return false;
   }
+  io.BeginReadingRecord();
   if (auto edit{io.GetNextDataEdit()}) {
     if (edit->descriptor == DataEdit::ListDirectedNullValue) {
       return true;
@@ -1044,6 +1042,7 @@ bool IONAME(InputLogical)(Cookie cookie, bool &truth) {
         "InputLogical() called for a non-input I/O statement");
     return false;
   }
+  io.BeginReadingRecord();
   if (auto edit{io.GetNextDataEdit()}) {
     if (edit->descriptor == DataEdit::ListDirectedNullValue) {
       return true;
diff --git a/flang/runtime/io-stmt.cpp b/flang/runtime/io-stmt.cpp
index b8e7781f235b0..a903f708bc627 100644
--- a/flang/runtime/io-stmt.cpp
+++ b/flang/runtime/io-stmt.cpp
@@ -185,7 +185,10 @@ int NoopCloseStatementState::EndIoStatement() {
 }
 
 template <Direction DIR> int ExternalIoStatementState<DIR>::EndIoStatement() {
-  if (!unit().nonAdvancing) {
+  if constexpr (DIR == Direction::Input) {
+    BeginReadingRecord(); // in case of READ with no data items
+  }
+  if (!unit().nonAdvancing && GetIoStat() != IostatEnd) {
     unit().AdvanceRecord(*this);
   }
   if constexpr (DIR == Direction::Output) {
@@ -260,6 +263,16 @@ void ExternalIoStatementState<DIR>::HandleRelativePosition(std::int64_t n) {
   return unit().HandleRelativePosition(n);
 }
 
+template <Direction DIR>
+void ExternalIoStatementState<DIR>::BeginReadingRecord() {
+  if constexpr (DIR == Direction::Input) {
+    if (!beganReading_) {
+      beganReading_ = true;
+      unit().BeginReadingRecord(*this);
+    }
+  }
+}
+
 template <Direction DIR, typename CHAR>
 ExternalFormattedIoStatementState<DIR, CHAR>::ExternalFormattedIoStatementState(
     ExternalFileUnit &unit, const CHAR *format, std::size_t formatLength,
@@ -315,6 +328,10 @@ MutableModes &IoStatementState::mutableModes() {
       [](auto &x) -> MutableModes & { return x.get().mutableModes(); }, u_);
 }
 
+void IoStatementState::BeginReadingRecord() {
+  std::visit([](auto &x) { return x.get().BeginReadingRecord(); }, u_);
+}
+
 IoErrorHandler &IoStatementState::GetIoErrorHandler() const {
   return std::visit(
       [](auto &x) -> IoErrorHandler & {
diff --git a/flang/runtime/io-stmt.h b/flang/runtime/io-stmt.h
index 755e5946ff3b9..ddc264aea3605 100644
--- a/flang/runtime/io-stmt.h
+++ b/flang/runtime/io-stmt.h
@@ -59,6 +59,7 @@ class IoStatementState {
   IoErrorHandler &GetIoErrorHandler() const;
   ExternalFileUnit *GetExternalFileUnit() const; // null if internal unit
   MutableModes &mutableModes();
+  void BeginReadingRecord();
 
   // N.B.: this also works with base classes
   template <typename A> A *get_if() const {
@@ -108,6 +109,7 @@ struct IoStatementBase : public DefaultFormatControlCallbacks {
   int EndIoStatement();
   std::optional<DataEdit> GetNextDataEdit(IoStatementState &, int = 1);
   ExternalFileUnit *GetExternalFileUnit() const { return nullptr; }
+  void BeginReadingRecord() {}
 };
 
 struct InputStatementState {};
@@ -247,6 +249,10 @@ class ExternalIoStatementState : public ExternalIoStatementBase,
   void BackspaceRecord();
   void HandleRelativePosition(std::int64_t);
   void HandleAbsolutePosition(std::int64_t);
+  void BeginReadingRecord();
+
+private:
+  bool beganReading_{false};
 };
 
 template <Direction DIR, typename CHAR>

From 1beb00db1f5197efb73f839da681b8e439f37628 Mon Sep 17 00:00:00 2001
From: Daniel Sanders <daniel_l_sanders@apple.com>
Date: Mon, 3 Aug 2020 16:53:41 -0700
Subject: [PATCH 0974/1035] Fix use-after-scope in 7209f83112db caught by the
 sanitizer bots

---
 clang/lib/Driver/Driver.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index 35263fbe1b2d8..e6a267621d8b4 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -4601,12 +4601,12 @@ const char *Driver::GetNamedOutputPath(Compilation &C, const JobAction &JA,
   }
 
   SmallString<128> BasePath(BaseInput);
+  SmallString<128> ExternalPath("");
   StringRef BaseName;
 
   // Dsymutil actions should use the full path.
   if (isa<DsymutilJobAction>(JA) && C.getArgs().hasArg(options::OPT_dsym_dir)) {
-    SmallString<128> ExternalPath(
-        C.getArgs().getLastArg(options::OPT_dsym_dir)->getValue());
+    ExternalPath += C.getArgs().getLastArg(options::OPT_dsym_dir)->getValue();
     // We use posix style here because the tests (specifically
     // darwin-dsymutil.c) demonstrate that posix style paths are acceptable
     // even on Windows and if we don't then the similar test covering this

From 81eeabbd97f32f7cd7dbe403e2b15db6fd23ad81 Mon Sep 17 00:00:00 2001
From: Petr Hosek <phosek@google.com>
Date: Tue, 23 Jun 2020 20:00:04 -0700
Subject: [PATCH 0975/1035] [ELF] Add --dependency-file option

Clang and GCC have a feature (-MD flag) to create a dependency file
in a format that build systems such as Make or Ninja can read, which
specifies all the additional inputs such .h files.

This change introduces the same functionality to lld bringing it to
feature parity with ld and gold which gained this feature recently.
See https://sourceware.org/bugzilla/show_bug.cgi?id=22843 for more
details and discussion.

The implementation corresponds to -MD -MP compiler flag where the
generated dependency file also includes phony targets which works
around the errors where the dependency is removed. This matches the
format used by ld and gold.

Fixes PR42806

Differential Revision: https://reviews.llvm.org/D82437
---
 lld/ELF/Config.h               |  6 ++-
 lld/ELF/Driver.cpp             | 75 ++++++++++++++++++++++++++++++++++
 lld/ELF/InputFiles.cpp         |  1 +
 lld/ELF/Options.td             |  3 ++
 lld/test/ELF/dependency-file.s | 21 ++++++++++
 5 files changed, 105 insertions(+), 1 deletion(-)
 create mode 100644 lld/test/ELF/dependency-file.s

diff --git a/lld/ELF/Config.h b/lld/ELF/Config.h
index e74a4a0c5b223..1afeee02efb30 100644
--- a/lld/ELF/Config.h
+++ b/lld/ELF/Config.h
@@ -10,7 +10,9 @@
 #define LLD_ELF_CONFIG_H
 
 #include "lld/Common/ErrorHandler.h"
+#include "llvm/ADT/CachedHashString.h"
 #include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/StringSet.h"
 #include "llvm/BinaryFormat/ELF.h"
@@ -90,11 +92,13 @@ struct Configuration {
   uint8_t osabi = 0;
   uint32_t andFeatures = 0;
   llvm::CachePruningPolicy thinLTOCachePolicy;
+  llvm::SetVector<llvm::CachedHashString> dependencyFiles; // for --dependency-file
   llvm::StringMap<uint64_t> sectionStartMap;
   llvm::StringRef bfdname;
   llvm::StringRef chroot;
-  llvm::StringRef dynamicLinker;
+  llvm::StringRef dependencyFile;
   llvm::StringRef dwoDir;
+  llvm::StringRef dynamicLinker;
   llvm::StringRef entry;
   llvm::StringRef emulation;
   llvm::StringRef fini;
diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp
index 3e60ffdb1dc1d..012da1485acb1 100644
--- a/lld/ELF/Driver.cpp
+++ b/lld/ELF/Driver.cpp
@@ -918,6 +918,7 @@ static void readConfigs(opt::InputArgList &args) {
   config->optimizeBBJumps =
       args.hasFlag(OPT_optimize_bb_jumps, OPT_no_optimize_bb_jumps, false);
   config->demangle = args.hasFlag(OPT_demangle, OPT_no_demangle, true);
+  config->dependencyFile = args.getLastArgValue(OPT_dependency_file);
   config->dependentLibraries = args.hasFlag(OPT_dependent_libraries, OPT_no_dependent_libraries, true);
   config->disableVerify = args.hasArg(OPT_disable_verify);
   config->discard = getDiscard(args);
@@ -1564,6 +1565,75 @@ static void handleLibcall(StringRef name) {
     sym->fetch();
 }
 
+// Handle --dependency-file=<path>. If that option is given, lld creates a
+// file at a given path with the following contents:
+//
+//   <output-file>: <input-file> ...
+//
+//   <input-file>:
+//
+// where <output-file> is a pathname of an output file and <input-file>
+// ... is a list of pathnames of all input files. `make` command can read a
+// file in the above format and interpret it as a dependency info. We write
+// phony targets for every <input-file> to avoid an error when that file is
+// removed.
+//
+// This option is useful if you want to make your final executable to depend
+// on all input files including system libraries. Here is why.
+//
+// When you write a Makefile, you usually write it so that the final
+// executable depends on all user-generated object files. Normally, you
+// don't make your executable to depend on system libraries (such as libc)
+// because you don't know the exact paths of libraries, even though system
+// libraries that are linked to your executable statically are technically a
+// part of your program. By using --dependency-file option, you can make
+// lld to dump dependency info so that you can maintain exact dependencies
+// easily.
+static void writeDependencyFile() {
+  std::error_code ec;
+  raw_fd_ostream os(config->dependencyFile, ec, sys::fs::F_None);
+  if (ec) {
+    error("cannot open " + config->dependencyFile + ": " + ec.message());
+    return;
+  }
+
+  // We use the same escape rules as Clang/GCC which are accepted by Make/Ninja:
+  // * A space is escaped by a backslash which itself must be escaped.
+  // * A hash sign is escaped by a single backslash.
+  // * $ is escapes as $$.
+  auto printFilename = [](raw_fd_ostream &os, StringRef filename) {
+    llvm::SmallString<256> nativePath;
+    llvm::sys::path::native(filename.str(), nativePath);
+    llvm::sys::path::remove_dots(nativePath, /*remove_dot_dot=*/true);
+    for (unsigned i = 0, e = nativePath.size(); i != e; ++i) {
+      if (nativePath[i] == '#') {
+        os << '\\';
+      } else if (nativePath[i] == ' ') {
+        os << '\\';
+        unsigned j = i;
+        while (j > 0 && nativePath[--j] == '\\')
+          os << '\\';
+      } else if (nativePath[i] == '$') {
+        os << '$';
+      }
+      os << nativePath[i];
+    }
+  };
+
+  os << config->outputFile << ":";
+  for (StringRef path : config->dependencyFiles) {
+    os << " \\\n ";
+    printFilename(os, path);
+  }
+  os << "\n";
+
+  for (StringRef path : config->dependencyFiles) {
+    os << "\n";
+    printFilename(os, path);
+    os << ":\n";
+  }
+}
+
 // Replaces common symbols with defined symbols reside in .bss sections.
 // This function is called after all symbol names are resolved. As a
 // result, the passes after the symbol resolution won't see any
@@ -2065,6 +2135,11 @@ template <class ELFT> void LinkerDriver::link(opt::InputArgList &args) {
     return false;
   });
 
+  // Since we now have a complete set of input files, we can create
+  // a .d file to record build dependencies.
+  if (!config->dependencyFile.empty())
+    writeDependencyFile();
+
   // Now that the number of partitions is fixed, save a pointer to the main
   // partition.
   mainPart = &partitions[0];
diff --git a/lld/ELF/InputFiles.cpp b/lld/ELF/InputFiles.cpp
index c142c00517ccf..6199f43b466a2 100644
--- a/lld/ELF/InputFiles.cpp
+++ b/lld/ELF/InputFiles.cpp
@@ -110,6 +110,7 @@ Optional<MemoryBufferRef> elf::readFile(StringRef path) {
     path = saver.save(config->chroot + path);
 
   log(path);
+  config->dependencyFiles.insert(llvm::CachedHashString(path));
 
   auto mbOrErr = MemoryBuffer::getFile(path, -1, false);
   if (auto ec = mbOrErr.getError()) {
diff --git a/lld/ELF/Options.td b/lld/ELF/Options.td
index 18bc612f6af45..c3cadafdccd28 100644
--- a/lld/ELF/Options.td
+++ b/lld/ELF/Options.td
@@ -132,6 +132,9 @@ defm demangle: B<"demangle",
     "Demangle symbol names (default)",
     "Do not demangle symbol names">;
 
+defm dependency_file: EEq<"dependency-file", "Write a dependency file">,
+  MetaVarName<"<file>">;
+
 def disable_new_dtags: F<"disable-new-dtags">,
   HelpText<"Disable new dynamic tags">;
 
diff --git a/lld/test/ELF/dependency-file.s b/lld/test/ELF/dependency-file.s
new file mode 100644
index 0000000000000..e7dbf9c7695f7
--- /dev/null
+++ b/lld/test/ELF/dependency-file.s
@@ -0,0 +1,21 @@
+# REQUIRES: x86
+# RUN: mkdir -p %t
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o %t/foo.o
+# RUN: llvm-mc -filetype=obj -triple=x86_64 /dev/null -o "%t/bar baz.o"
+# RUN: llvm-mc -filetype=obj -triple=x86_64 /dev/null -o "%t/#quux$.o"
+# RUN: ld.lld -o %t/foo.exe %t/foo.o %t/"bar baz.o" "%t/#quux$.o" --dependency-file=%t/foo.d
+# RUN: FileCheck --match-full-lines -DFILE=%t %s < %t/foo.d
+
+# CHECK:      [[FILE]]{{/|(\\)+}}foo.exe: \
+# CHECK-NEXT:   [[FILE]]{{/|(\\)+}}foo.o \
+# CHECK-NEXT:   [[FILE]]{{/|(\\)+}}bar\ baz.o \
+# CHECK-NEXT:   [[FILE]]{{/|(\\)+}}\#quux$$.o
+# CHECK-EMPTY:
+# CHECK-NEXT: [[FILE]]{{/|(\\)+}}foo.o:
+# CHECK-EMPTY:
+# CHECK-NEXT: [[FILE]]{{/|(\\)+}}bar\ baz.o:
+# CHECK-EMPTY:
+# CHECK-NEXT: [[FILE]]{{/|(\\)+}}\#quux$$.o:
+
+.global _start
+_start:

From 0c938a8dd80ad707ec1f20d936cc5c9d73df8de5 Mon Sep 17 00:00:00 2001
From: David Blaikie <dblaikie@gmail.com>
Date: Mon, 3 Aug 2020 16:37:25 -0700
Subject: [PATCH 0976/1035] OpenMP: Fix typo variabls -> variables

---
 openmp/runtime/tools/lib/Platform.pm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/openmp/runtime/tools/lib/Platform.pm b/openmp/runtime/tools/lib/Platform.pm
index a6b158d907535..38593a154d03f 100644
--- a/openmp/runtime/tools/lib/Platform.pm
+++ b/openmp/runtime/tools/lib/Platform.pm
@@ -450,7 +450,7 @@ C<--target-architecture=I<str>> and C<--target-os=I<str>> options. Typical usage
         Platform::target_options(),  # Let script recognize --target-os and --target-arch options.
         ...
     );
-    # Initialize variabls after parsing command line.
+    # Initialize variables after parsing command line.
     ( $os, $arch, $platform ) = ( Platform::target_os(), Platform::target_arch(), Platform::target_platform() );
 
 =back

From ffe0066b62e989ca3e59f1ed211324ca7ec37b5a Mon Sep 17 00:00:00 2001
From: Shinji Okumura <okuraofvegetable@gmail.com>
Date: Tue, 4 Aug 2020 08:59:23 +0900
Subject: [PATCH 0977/1035] [Attributor][NFC] Clang format

---
 llvm/lib/Transforms/IPO/AttributorAttributes.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
index 5cd0c711ddde1..89d359bcbe034 100644
--- a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
+++ b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
@@ -2006,7 +2006,7 @@ struct AAUndefinedBehaviorImpl : public AAUndefinedBehavior {
         if (idx >= Callee->arg_size())
           break;
         Value *ArgVal = CB.getArgOperand(idx);
-        if(!ArgVal)
+        if (!ArgVal)
           continue;
         IRPosition CalleeArgumentIRP =
             IRPosition::argument(*Callee->getArg(idx));

From 675ad1bc6a96d3c7ef1909c91695189cd818a143 Mon Sep 17 00:00:00 2001
From: peter klausler <pklausler@nvidia.com>
Date: Mon, 3 Aug 2020 11:35:29 -0700
Subject: [PATCH 0978/1035] [flang] Implement runtime support for INQUIRE
 statements

Differential Revision: https://reviews.llvm.org/D85166
---
 flang/runtime/io-api.cpp   | 126 ++++++---
 flang/runtime/io-api.h     |  38 ++-
 flang/runtime/io-error.h   |   2 +-
 flang/runtime/io-stmt.cpp  | 506 ++++++++++++++++++++++++++++++++++++-
 flang/runtime/io-stmt.h    |  88 ++++++-
 flang/runtime/memory.h     |   4 +-
 flang/runtime/tools.cpp    |   7 +
 flang/runtime/tools.h      |   2 +
 flang/runtime/unit-map.cpp |  14 +
 flang/runtime/unit-map.h   |   7 +
 flang/runtime/unit.cpp     |  33 ++-
 flang/runtime/unit.h       |   8 +-
 12 files changed, 760 insertions(+), 75 deletions(-)

diff --git a/flang/runtime/io-api.cpp b/flang/runtime/io-api.cpp
index f36144d0c3c46..f64fe97b2d233 100644
--- a/flang/runtime/io-api.cpp
+++ b/flang/runtime/io-api.cpp
@@ -23,6 +23,23 @@
 
 namespace Fortran::runtime::io {
 
+const char *InquiryKeywordHashDecode(
+    char *buffer, std::size_t n, InquiryKeywordHash hash) {
+  if (n < 1) {
+    return nullptr;
+  }
+  char *p{buffer + n};
+  *--p = '\0';
+  while (hash > 1) {
+    if (p < buffer) {
+      return nullptr;
+    }
+    *--p = 'A' + (hash % 26);
+    hash /= 26;
+  }
+  return hash == 1 ? p : nullptr;
+}
+
 template <Direction DIR>
 Cookie BeginInternalArrayListIO(const Descriptor &descriptor,
     void ** /*scratchArea*/, std::size_t /*scratchBytes*/,
@@ -289,8 +306,8 @@ Cookie IONAME(BeginBackspace)(
 Cookie IONAME(BeginEndfile)(
     ExternalUnit unitNumber, const char *sourceFile, int sourceLine) {
   Terminator terminator{sourceFile, sourceLine};
-  ExternalFileUnit &unit{
-      ExternalFileUnit::LookUpOrCrash(unitNumber, terminator)};
+  ExternalFileUnit &unit{ExternalFileUnit::LookUpOrCreateAnonymous(
+      unitNumber, Direction::Output, true /*formatted*/, terminator)};
   return &unit.BeginIoStatement<ExternalMiscIoStatementState>(
       unit, ExternalMiscIoStatementState::Endfile, sourceFile, sourceLine);
 }
@@ -298,12 +315,50 @@ Cookie IONAME(BeginEndfile)(
 Cookie IONAME(BeginRewind)(
     ExternalUnit unitNumber, const char *sourceFile, int sourceLine) {
   Terminator terminator{sourceFile, sourceLine};
-  ExternalFileUnit &unit{
-      ExternalFileUnit::LookUpOrCrash(unitNumber, terminator)};
+  ExternalFileUnit &unit{ExternalFileUnit::LookUpOrCreateAnonymous(
+      unitNumber, Direction::Input, true /*formatted*/, terminator)};
   return &unit.BeginIoStatement<ExternalMiscIoStatementState>(
       unit, ExternalMiscIoStatementState::Rewind, sourceFile, sourceLine);
 }
 
+Cookie IONAME(BeginInquireUnit)(
+    ExternalUnit unitNumber, const char *sourceFile, int sourceLine) {
+  if (ExternalFileUnit * unit{ExternalFileUnit::LookUp(unitNumber)}) {
+    return &unit->BeginIoStatement<InquireUnitState>(
+        *unit, sourceFile, sourceLine);
+  } else {
+    // INQUIRE(UNIT=unrecognized unit)
+    Terminator oom{sourceFile, sourceLine};
+    return &New<InquireNoUnitState>{oom}(sourceFile, sourceLine)
+                .release()
+                ->ioStatementState();
+  }
+}
+
+Cookie IONAME(BeginInquireFile)(const char *path, std::size_t pathLength,
+    const char *sourceFile, int sourceLine) {
+  Terminator oom{sourceFile, sourceLine};
+  auto trimmed{
+      SaveDefaultCharacter(path, TrimTrailingSpaces(path, pathLength), oom)};
+  if (ExternalFileUnit * unit{ExternalFileUnit::LookUp(trimmed.get())}) {
+    // INQUIRE(FILE=) to a connected unit
+    return &unit->BeginIoStatement<InquireUnitState>(
+        *unit, sourceFile, sourceLine);
+  } else {
+    return &New<InquireUnconnectedFileState>{oom}(
+        std::move(trimmed), sourceFile, sourceLine)
+                .release()
+                ->ioStatementState();
+  }
+}
+
+Cookie IONAME(BeginInquireIoLength)(const char *sourceFile, int sourceLine) {
+  Terminator oom{sourceFile, sourceLine};
+  return &New<InquireIOLengthState>{oom}(sourceFile, sourceLine)
+              .release()
+              ->ioStatementState();
+}
+
 // Control list items
 
 void IONAME(EnableHandlers)(Cookie cookie, bool hasIoStat, bool hasErr,
@@ -522,29 +577,21 @@ bool IONAME(SetAccess)(Cookie cookie, const char *keyword, std::size_t length) {
     io.GetIoErrorHandler().Crash(
         "SetAccess() called when not in an OPEN statement");
   }
-  ConnectionState &connection{open->GetConnectionState()};
-  Access access{connection.access};
   static const char *keywords[]{"SEQUENTIAL", "DIRECT", "STREAM", nullptr};
   switch (IdentifyValue(keyword, length, keywords)) {
   case 0:
-    access = Access::Sequential;
+    open->set_access(Access::Sequential);
     break;
   case 1:
-    access = Access::Direct;
+    open->set_access(Access::Direct);
     break;
   case 2:
-    access = Access::Stream;
+    open->set_access(Access::Stream);
     break;
   default:
     open->SignalError(IostatErrorInKeyword, "Invalid ACCESS='%.*s'",
         static_cast<int>(length), keyword);
   }
-  if (access != connection.access) {
-    if (open->wasExtant()) {
-      open->SignalError("ACCESS= may not be changed on an open unit");
-    }
-    connection.access = access;
-  }
   return true;
 }
 
@@ -661,25 +708,18 @@ bool IONAME(SetForm)(Cookie cookie, const char *keyword, std::size_t length) {
     io.GetIoErrorHandler().Crash(
         "SetEncoding() called when not in an OPEN statement");
   }
-  bool isUnformatted{false};
   static const char *keywords[]{"FORMATTED", "UNFORMATTED", nullptr};
   switch (IdentifyValue(keyword, length, keywords)) {
   case 0:
-    isUnformatted = false;
+    open->set_isUnformatted(false);
     break;
   case 1:
-    isUnformatted = true;
+    open->set_isUnformatted(true);
     break;
   default:
     open->SignalError(IostatErrorInKeyword, "Invalid FORM='%.*s'",
         static_cast<int>(length), keyword);
   }
-  if (isUnformatted != open->unit().isUnformatted) {
-    if (open->wasExtant()) {
-      open->SignalError("FORM= may not be changed on an open unit");
-    }
-    open->unit().isUnformatted = isUnformatted;
-  }
   return true;
 }
 
@@ -777,11 +817,10 @@ bool IONAME(SetStatus)(Cookie cookie, const char *keyword, std::size_t length) {
       "SetStatus() called when not in an OPEN or CLOSE statement");
 }
 
-bool IONAME(SetFile)(
-    Cookie cookie, const char *path, std::size_t chars, int kind) {
+bool IONAME(SetFile)(Cookie cookie, const char *path, std::size_t chars) {
   IoStatementState &io{*cookie};
   if (auto *open{io.get_if<OpenStatementState>()}) {
-    open->set_path(path, chars, kind);
+    open->set_path(path, chars);
     return true;
   }
   io.GetIoErrorHandler().Crash(
@@ -789,7 +828,8 @@ bool IONAME(SetFile)(
   return false;
 }
 
-static bool SetInteger(int &x, int kind, int value) {
+template <typename INT>
+static bool SetInteger(INT &x, int kind, std::int64_t value) {
   switch (kind) {
   case 1:
     reinterpret_cast<std::int8_t &>(x) = value;
@@ -798,7 +838,7 @@ static bool SetInteger(int &x, int kind, int value) {
     reinterpret_cast<std::int16_t &>(x) = value;
     return true;
   case 4:
-    x = value;
+    reinterpret_cast<std::int32_t &>(x) = value;
     return true;
   case 8:
     reinterpret_cast<std::int64_t &>(x) = value;
@@ -1059,6 +1099,34 @@ void IONAME(GetIoMsg)(Cookie cookie, char *msg, std::size_t length) {
   }
 }
 
+bool IONAME(InquireCharacter)(Cookie cookie, InquiryKeywordHash inquiry,
+    char *result, std::size_t length) {
+  IoStatementState &io{*cookie};
+  return io.Inquire(inquiry, result, length);
+}
+
+bool IONAME(InquireLogical)(
+    Cookie cookie, InquiryKeywordHash inquiry, bool &result) {
+  IoStatementState &io{*cookie};
+  return io.Inquire(inquiry, result);
+}
+
+bool IONAME(InquirePendingId)(Cookie cookie, std::int64_t id, bool &result) {
+  IoStatementState &io{*cookie};
+  return io.Inquire(HashInquiryKeyword("PENDING"), id, result);
+}
+
+bool IONAME(InquireInteger64)(
+    Cookie cookie, InquiryKeywordHash inquiry, std::int64_t &result, int kind) {
+  IoStatementState &io{*cookie};
+  std::int64_t n;
+  if (io.Inquire(inquiry, n)) {
+    SetInteger(result, kind, n);
+    return true;
+  }
+  return false;
+}
+
 enum Iostat IONAME(EndIoStatement)(Cookie cookie) {
   IoStatementState &io{*cookie};
   return static_cast<enum Iostat>(io.EndIoStatement());
diff --git a/flang/runtime/io-api.h b/flang/runtime/io-api.h
index f6ebc63e3f3d2..a38152d6ec1c1 100644
--- a/flang/runtime/io-api.h
+++ b/flang/runtime/io-api.h
@@ -29,6 +29,26 @@ using ExternalUnit = int;
 using AsynchronousId = int;
 static constexpr ExternalUnit DefaultUnit{-1}; // READ(*), WRITE(*), PRINT
 
+// INQUIRE specifiers are encoded as simple base-26 packings of
+// the spellings of their keywords.
+using InquiryKeywordHash = std::uint64_t;
+constexpr InquiryKeywordHash HashInquiryKeyword(const char *p) {
+  InquiryKeywordHash hash{1};
+  while (char ch{*p++}) {
+    std::uint64_t letter{0};
+    if (ch >= 'a' && ch <= 'z') {
+      letter = ch - 'a';
+    } else {
+      letter = ch - 'A';
+    }
+    hash = 26 * hash + letter;
+  }
+  return hash;
+}
+
+const char *InquiryKeywordHashDecode(
+    char *buffer, std::size_t, InquiryKeywordHash);
+
 extern "C" {
 
 #define IONAME(name) RTNAME(io##name)
@@ -150,7 +170,7 @@ Cookie IONAME(BeginOpenNewUnit)(
 // BeginInquireIoLength() is basically a no-op output statement.
 Cookie IONAME(BeginInquireUnit)(
     ExternalUnit, const char *sourceFile = nullptr, int sourceLine = 0);
-Cookie IONAME(BeginInquireFile)(const char *, std::size_t, int kind = 1,
+Cookie IONAME(BeginInquireFile)(const char *, std::size_t,
     const char *sourceFile = nullptr, int sourceLine = 0);
 Cookie IONAME(BeginInquireIoLength)(
     const char *sourceFile = nullptr, int sourceLine = 0);
@@ -255,10 +275,7 @@ bool IONAME(SetRecl)(Cookie, std::size_t); // RECL=
 // For CLOSE: STATUS=KEEP, DELETE
 bool IONAME(SetStatus)(Cookie, const char *, std::size_t);
 
-// SetFile() may pass a CHARACTER argument of non-default kind,
-// and such filenames are converted to UTF-8 before being
-// presented to the filesystem.
-bool IONAME(SetFile)(Cookie, const char *, std::size_t chars, int kind = 1);
+bool IONAME(SetFile)(Cookie, const char *, std::size_t chars);
 
 // Acquires the runtime-created unit number for OPEN(NEWUNIT=)
 bool IONAME(GetNewUnit)(Cookie, int &, int kind = 4);
@@ -275,18 +292,17 @@ void IONAME(GetIoMsg)(Cookie, char *, std::size_t); // IOMSG=
 
 // INQUIRE() specifiers are mostly identified by their NUL-terminated
 // case-insensitive names.
-// ACCESS, ACTION, ASYNCHRONOUS, BLANK, DECIMAL, DELIM, DIRECT, ENCODING,
-// FORM, FORMATTED, NAME, PAD, POSITION, READ, READWRITE, ROUND,
+// ACCESS, ACTION, ASYNCHRONOUS, BLANK, CONVERT, DECIMAL, DELIM, DIRECT,
+// ENCODING, FORM, FORMATTED, NAME, PAD, POSITION, READ, READWRITE, ROUND,
 // SEQUENTIAL, SIGN, STREAM, UNFORMATTED, WRITE:
-bool IONAME(InquireCharacter)(
-    Cookie, const char *specifier, char *, std::size_t);
+bool IONAME(InquireCharacter)(Cookie, InquiryKeywordHash, char *, std::size_t);
 // EXIST, NAMED, OPENED, and PENDING (without ID):
-bool IONAME(InquireLogical)(Cookie, const char *specifier, bool &);
+bool IONAME(InquireLogical)(Cookie, InquiryKeywordHash, bool &);
 // PENDING with ID
 bool IONAME(InquirePendingId)(Cookie, std::int64_t, bool &);
 // NEXTREC, NUMBER, POS, RECL, SIZE
 bool IONAME(InquireInteger64)(
-    Cookie, const char *specifier, std::int64_t &, int kind = 8);
+    Cookie, InquiryKeywordHash, std::int64_t &, int kind = 8);
 
 // This function must be called to end an I/O statement, and its
 // cookie value may not be used afterwards unless it is recycled
diff --git a/flang/runtime/io-error.h b/flang/runtime/io-error.h
index 8d43c40ef103c..5dd7f5e03d080 100644
--- a/flang/runtime/io-error.h
+++ b/flang/runtime/io-error.h
@@ -38,7 +38,7 @@ class IoErrorHandler : public Terminator {
 
   void SignalError(int iostatOrErrno, const char *msg, ...);
   void SignalError(int iostatOrErrno);
-  template <typename... X> void SignalError(const char *msg, X &&... xs) {
+  template <typename... X> void SignalError(const char *msg, X &&...xs) {
     SignalError(IostatGenericError, msg, std::forward<X>(xs)...);
   }
 
diff --git a/flang/runtime/io-stmt.cpp b/flang/runtime/io-stmt.cpp
index a903f708bc627..8300b1ea3c27b 100644
--- a/flang/runtime/io-stmt.cpp
+++ b/flang/runtime/io-stmt.cpp
@@ -26,6 +26,37 @@ std::optional<DataEdit> IoStatementBase::GetNextDataEdit(
   return std::nullopt;
 }
 
+bool IoStatementBase::Inquire(InquiryKeywordHash, char *, std::size_t) {
+  Crash(
+      "IoStatementBase::Inquire() called for I/O statement other than INQUIRE");
+  return false;
+}
+
+bool IoStatementBase::Inquire(InquiryKeywordHash, bool &) {
+  Crash(
+      "IoStatementBase::Inquire() called for I/O statement other than INQUIRE");
+  return false;
+}
+
+bool IoStatementBase::Inquire(InquiryKeywordHash, std::int64_t, bool &) {
+  Crash(
+      "IoStatementBase::Inquire() called for I/O statement other than INQUIRE");
+  return false;
+}
+
+bool IoStatementBase::Inquire(InquiryKeywordHash, std::int64_t &) {
+  Crash(
+      "IoStatementBase::Inquire() called for I/O statement other than INQUIRE");
+  return false;
+}
+
+void IoStatementBase::BadInquiryKeywordHashCrash(InquiryKeywordHash inquiry) {
+  char buffer[16];
+  const char *decode{InquiryKeywordHashDecode(buffer, sizeof buffer, inquiry)};
+  Crash("bad InquiryKeywordHash 0x%x (%s)", inquiry,
+      decode ? decode : "(cannot decode)");
+}
+
 template <Direction DIR, typename CHAR>
 InternalIoStatementState<DIR, CHAR>::InternalIoStatementState(
     Buffer scalar, std::size_t length, const char *sourceFile, int sourceLine)
@@ -151,14 +182,9 @@ int ExternalIoStatementBase::EndIoStatement() {
   return result;
 }
 
-void OpenStatementState::set_path(
-    const char *path, std::size_t length, int kind) {
-  if (kind != 1) { // TODO
-    Crash("OPEN: FILE= with unimplemented: CHARACTER(KIND=%d)", kind);
-  }
-  std::size_t bytes{length * kind}; // TODO: UTF-8 encoding of Unicode path
-  path_ = SaveDefaultCharacter(path, bytes, *this);
-  pathLength_ = length;
+void OpenStatementState::set_path(const char *path, std::size_t length) {
+  pathLength_ = TrimTrailingSpaces(path, length);
+  path_ = SaveDefaultCharacter(path, pathLength_, *this);
 }
 
 int OpenStatementState::EndIoStatement() {
@@ -166,8 +192,31 @@ int OpenStatementState::EndIoStatement() {
     SignalError("OPEN statement for connected unit may not have STATUS= other "
                 "than 'OLD'");
   }
-  unit().OpenUnit(status_.value_or(OpenStatus::Unknown), action_, position_,
-      std::move(path_), pathLength_, convert_, *this);
+  if (path_.get() || wasExtant_ ||
+      (status_ && *status_ == OpenStatus::Scratch)) {
+    unit().OpenUnit(status_.value_or(OpenStatus::Unknown), action_, position_,
+        std::move(path_), pathLength_, convert_, *this);
+  } else {
+    unit().OpenAnonymousUnit(status_.value_or(OpenStatus::Unknown), action_,
+        position_, convert_, *this);
+  }
+  if (access_) {
+    if (*access_ != unit().access) {
+      if (wasExtant_) {
+        SignalError("ACCESS= may not be changed on an open unit");
+      }
+    }
+    unit().access = *access_;
+  }
+  if (!isUnformatted_) {
+    isUnformatted_ = unit().access != Access::Sequential;
+  }
+  if (*isUnformatted_ != unit().isUnformatted) {
+    if (wasExtant_) {
+      SignalError("FORM= may not be changed on an open unit");
+    }
+    unit().isUnformatted = *isUnformatted_;
+  }
   return ExternalIoStatementBase::EndIoStatement();
 }
 
@@ -178,7 +227,7 @@ int CloseStatementState::EndIoStatement() {
   return result;
 }
 
-int NoopCloseStatementState::EndIoStatement() {
+int NoUnitIoStatementState::EndIoStatement() {
   auto result{IoStatementBase::EndIoStatement()};
   FreeMemory(this);
   return result;
@@ -454,6 +503,26 @@ bool ListDirectedStatementState<Direction::Output>::NeedAdvance(
       width > connection.RemainingSpaceInRecord();
 }
 
+bool IoStatementState::Inquire(
+    InquiryKeywordHash inquiry, char *out, std::size_t chars) {
+  return std::visit(
+      [&](auto &x) { return x.get().Inquire(inquiry, out, chars); }, u_);
+}
+
+bool IoStatementState::Inquire(InquiryKeywordHash inquiry, bool &out) {
+  return std::visit([&](auto &x) { return x.get().Inquire(inquiry, out); }, u_);
+}
+
+bool IoStatementState::Inquire(
+    InquiryKeywordHash inquiry, std::int64_t id, bool &out) {
+  return std::visit(
+      [&](auto &x) { return x.get().Inquire(inquiry, id, out); }, u_);
+}
+
+bool IoStatementState::Inquire(InquiryKeywordHash inquiry, std::int64_t &n) {
+  return std::visit([&](auto &x) { return x.get().Inquire(inquiry, n); }, u_);
+}
+
 bool ListDirectedStatementState<Direction::Output>::EmitLeadingSpaceOrAdvance(
     IoStatementState &io, std::size_t length, bool isCharacter) {
   if (length == 0) {
@@ -678,4 +747,419 @@ int ExternalMiscIoStatementState::EndIoStatement() {
   return ExternalIoStatementBase::EndIoStatement();
 }
 
+InquireUnitState::InquireUnitState(
+    ExternalFileUnit &unit, const char *sourceFile, int sourceLine)
+    : ExternalIoStatementBase{unit, sourceFile, sourceLine} {}
+
+bool InquireUnitState::Inquire(
+    InquiryKeywordHash inquiry, char *result, std::size_t length) {
+  const char *str{nullptr};
+  switch (inquiry) {
+  case HashInquiryKeyword("ACCESS"):
+    switch (unit().access) {
+    case Access::Sequential:
+      str = "SEQUENTIAL";
+      break;
+    case Access::Direct:
+      str = "DIRECT";
+      break;
+    case Access::Stream:
+      str = "STREAM";
+      break;
+    }
+    break;
+  case HashInquiryKeyword("ACTION"):
+    str = unit().mayWrite() ? unit().mayRead() ? "READWRITE" : "WRITE" : "READ";
+    break;
+  case HashInquiryKeyword("ASYNCHRONOUS"):
+    str = unit().mayAsynchronous() ? "YES" : "NO";
+    break;
+  case HashInquiryKeyword("BLANK"):
+    str = unit().isUnformatted                  ? "UNDEFINED"
+        : unit().modes.editingFlags & blankZero ? "ZERO"
+                                                : "NULL";
+    break;
+  case HashInquiryKeyword("CONVERT"):
+    str = unit().swapEndianness() ? "SWAP" : "NATIVE";
+    break;
+  case HashInquiryKeyword("DECIMAL"):
+    str = unit().isUnformatted                     ? "UNDEFINED"
+        : unit().modes.editingFlags & decimalComma ? "COMMA"
+                                                   : "POINT";
+    break;
+  case HashInquiryKeyword("DELIM"):
+    if (unit().isUnformatted) {
+      str = "UNDEFINED";
+    } else {
+      switch (unit().modes.delim) {
+      case '\'':
+        str = "APOSTROPHE";
+        break;
+      case '"':
+        str = "QUOTE";
+        break;
+      default:
+        str = "NONE";
+        break;
+      }
+    }
+    break;
+  case HashInquiryKeyword("DIRECT"):
+    str = unit().mayPosition() ? "YES" : "NO";
+    break;
+  case HashInquiryKeyword("ENCODING"):
+    str = unit().isUnformatted ? "UNDEFINED"
+        : unit().isUTF8        ? "UTF-8"
+                               : "ASCII";
+    break;
+  case HashInquiryKeyword("FORM"):
+    str = unit().isUnformatted ? "UNFORMATTED" : "FORMATTED";
+    break;
+  case HashInquiryKeyword("FORMATTED"):
+    str = "YES";
+    break;
+  case HashInquiryKeyword("NAME"):
+    str = unit().path();
+    if (!str) {
+      return true; // result is undefined
+    }
+    break;
+  case HashInquiryKeyword("PAD"):
+    str = unit().isUnformatted ? "UNDEFINED" : unit().modes.pad ? "YES" : "NO";
+    break;
+  case HashInquiryKeyword("POSITION"):
+    if (unit().access == Access::Direct) {
+      str = "UNDEFINED";
+    } else {
+      auto size{unit().knownSize()};
+      auto pos{unit().position()};
+      if (pos == size.value_or(pos + 1)) {
+        str = "APPEND";
+      } else if (pos == 0) {
+        str = "REWIND";
+      } else {
+        str = "ASIS"; // processor-dependent & no common behavior
+      }
+    }
+    break;
+  case HashInquiryKeyword("READ"):
+    str = unit().mayRead() ? "YES" : "NO";
+    break;
+  case HashInquiryKeyword("READWRITE"):
+    str = unit().mayRead() && unit().mayWrite() ? "YES" : "NO";
+    break;
+  case HashInquiryKeyword("ROUND"):
+    if (unit().isUnformatted) {
+      str = "UNDEFINED";
+    } else {
+      switch (unit().modes.round) {
+      case decimal::FortranRounding::RoundNearest:
+        str = "NEAREST";
+        break;
+      case decimal::FortranRounding::RoundUp:
+        str = "UP";
+        break;
+      case decimal::FortranRounding::RoundDown:
+        str = "DOWN";
+        break;
+      case decimal::FortranRounding::RoundToZero:
+        str = "ZERO";
+        break;
+      case decimal::FortranRounding::RoundCompatible:
+        str = "COMPATIBLE";
+        break;
+      }
+    }
+    break;
+  case HashInquiryKeyword("SEQUENTIAL"):
+    str = "YES";
+    break;
+  case HashInquiryKeyword("SIGN"):
+    str = unit().isUnformatted                 ? "UNDEFINED"
+        : unit().modes.editingFlags & signPlus ? "PLUS"
+                                               : "SUPPRESS";
+    break;
+  case HashInquiryKeyword("STREAM"):
+    str = "YES";
+    break;
+  case HashInquiryKeyword("WRITE"):
+    str = unit().mayWrite() ? "YES" : "NO";
+    break;
+  case HashInquiryKeyword("UNFORMATTED"):
+    str = "YES";
+    break;
+  }
+  if (str) {
+    ToFortranDefaultCharacter(result, length, str);
+    return true;
+  } else {
+    BadInquiryKeywordHashCrash(inquiry);
+    return false;
+  }
+}
+
+bool InquireUnitState::Inquire(InquiryKeywordHash inquiry, bool &result) {
+  switch (inquiry) {
+  case HashInquiryKeyword("EXIST"):
+    result = true;
+    return true;
+  case HashInquiryKeyword("NAMED"):
+    result = unit().path() != nullptr;
+    return true;
+  case HashInquiryKeyword("OPENED"):
+    result = true;
+    return true;
+  case HashInquiryKeyword("PENDING"):
+    result = false; // asynchronous I/O is not implemented
+    return true;
+  default:
+    BadInquiryKeywordHashCrash(inquiry);
+    return false;
+  }
+}
+
+bool InquireUnitState::Inquire(
+    InquiryKeywordHash inquiry, std::int64_t, bool &result) {
+  switch (inquiry) {
+  case HashInquiryKeyword("PENDING"):
+    result = false; // asynchronous I/O is not implemented
+    return true;
+  default:
+    BadInquiryKeywordHashCrash(inquiry);
+    return false;
+  }
+}
+
+bool InquireUnitState::Inquire(
+    InquiryKeywordHash inquiry, std::int64_t &result) {
+  switch (inquiry) {
+  case HashInquiryKeyword("NEXTREC"):
+    if (unit().access == Access::Direct) {
+      result = unit().currentRecordNumber;
+    }
+    return true;
+  case HashInquiryKeyword("NUMBER"):
+    result = unit().unitNumber();
+    return true;
+  case HashInquiryKeyword("POS"):
+    result = unit().position();
+    return true;
+  case HashInquiryKeyword("RECL"):
+    if (unit().access == Access::Stream) {
+      result = -2;
+    } else if (unit().isFixedRecordLength && unit().recordLength) {
+      result = *unit().recordLength;
+    } else {
+      result = std::numeric_limits<std::uint32_t>::max();
+    }
+    return true;
+  case HashInquiryKeyword("SIZE"):
+    if (auto size{unit().knownSize()}) {
+      result = *size;
+    } else {
+      result = -1;
+    }
+    return true;
+  default:
+    BadInquiryKeywordHashCrash(inquiry);
+    return false;
+  }
+}
+
+InquireNoUnitState::InquireNoUnitState(const char *sourceFile, int sourceLine)
+    : NoUnitIoStatementState{sourceFile, sourceLine, *this} {}
+
+bool InquireNoUnitState::Inquire(
+    InquiryKeywordHash inquiry, char *result, std::size_t length) {
+  switch (inquiry) {
+  case HashInquiryKeyword("ACCESS"):
+  case HashInquiryKeyword("ACTION"):
+  case HashInquiryKeyword("ASYNCHRONOUS"):
+  case HashInquiryKeyword("BLANK"):
+  case HashInquiryKeyword("CONVERT"):
+  case HashInquiryKeyword("DECIMAL"):
+  case HashInquiryKeyword("DELIM"):
+  case HashInquiryKeyword("FORM"):
+  case HashInquiryKeyword("NAME"):
+  case HashInquiryKeyword("PAD"):
+  case HashInquiryKeyword("POSITION"):
+  case HashInquiryKeyword("ROUND"):
+  case HashInquiryKeyword("SIGN"):
+    ToFortranDefaultCharacter(result, length, "UNDEFINED");
+    return true;
+  case HashInquiryKeyword("DIRECT"):
+  case HashInquiryKeyword("ENCODING"):
+  case HashInquiryKeyword("FORMATTED"):
+  case HashInquiryKeyword("READ"):
+  case HashInquiryKeyword("READWRITE"):
+  case HashInquiryKeyword("SEQUENTIAL"):
+  case HashInquiryKeyword("STREAM"):
+  case HashInquiryKeyword("WRITE"):
+  case HashInquiryKeyword("UNFORMATTED"):
+    ToFortranDefaultCharacter(result, length, "UNKNONN");
+    return true;
+  default:
+    BadInquiryKeywordHashCrash(inquiry);
+    return false;
+  }
+}
+
+bool InquireNoUnitState::Inquire(InquiryKeywordHash inquiry, bool &result) {
+  switch (inquiry) {
+  case HashInquiryKeyword("EXIST"):
+    result = true;
+    return true;
+  case HashInquiryKeyword("NAMED"):
+  case HashInquiryKeyword("OPENED"):
+  case HashInquiryKeyword("PENDING"):
+    result = false;
+    return true;
+  default:
+    BadInquiryKeywordHashCrash(inquiry);
+    return false;
+  }
+}
+
+bool InquireNoUnitState::Inquire(
+    InquiryKeywordHash inquiry, std::int64_t, bool &result) {
+  switch (inquiry) {
+  case HashInquiryKeyword("PENDING"):
+    result = false;
+    return true;
+  default:
+    BadInquiryKeywordHashCrash(inquiry);
+    return false;
+  }
+}
+
+bool InquireNoUnitState::Inquire(
+    InquiryKeywordHash inquiry, std::int64_t &result) {
+  switch (inquiry) {
+  case HashInquiryKeyword("NEXTREC"):
+  case HashInquiryKeyword("NUMBER"):
+  case HashInquiryKeyword("POS"):
+  case HashInquiryKeyword("RECL"):
+  case HashInquiryKeyword("SIZE"):
+    result = -1;
+    return true;
+  default:
+    BadInquiryKeywordHashCrash(inquiry);
+    return false;
+  }
+}
+
+InquireUnconnectedFileState::InquireUnconnectedFileState(
+    OwningPtr<char> &&path, const char *sourceFile, int sourceLine)
+    : NoUnitIoStatementState{sourceFile, sourceLine, *this}, path_{std::move(
+                                                                 path)} {}
+
+bool InquireUnconnectedFileState::Inquire(
+    InquiryKeywordHash inquiry, char *result, std::size_t length) {
+  const char *str{nullptr};
+  switch (inquiry) {
+  case HashInquiryKeyword("ACCESS"):
+  case HashInquiryKeyword("ACTION"):
+  case HashInquiryKeyword("ASYNCHRONOUS"):
+  case HashInquiryKeyword("BLANK"):
+  case HashInquiryKeyword("CONVERT"):
+  case HashInquiryKeyword("DECIMAL"):
+  case HashInquiryKeyword("DELIM"):
+  case HashInquiryKeyword("FORM"):
+  case HashInquiryKeyword("PAD"):
+  case HashInquiryKeyword("POSITION"):
+  case HashInquiryKeyword("ROUND"):
+  case HashInquiryKeyword("SIGN"):
+    str = "UNDEFINED";
+    break;
+  case HashInquiryKeyword("DIRECT"):
+  case HashInquiryKeyword("ENCODING"):
+    str = "UNKNONN";
+    break;
+  case HashInquiryKeyword("READ"):
+    str = MayRead(path_.get()) ? "YES" : "NO";
+    break;
+  case HashInquiryKeyword("READWRITE"):
+    str = MayReadAndWrite(path_.get()) ? "YES" : "NO";
+    break;
+  case HashInquiryKeyword("WRITE"):
+    str = MayWrite(path_.get()) ? "YES" : "NO";
+    break;
+  case HashInquiryKeyword("FORMATTED"):
+  case HashInquiryKeyword("SEQUENTIAL"):
+  case HashInquiryKeyword("STREAM"):
+  case HashInquiryKeyword("UNFORMATTED"):
+    str = "YES";
+    break;
+  case HashInquiryKeyword("NAME"):
+    str = path_.get();
+    return true;
+  }
+  if (str) {
+    ToFortranDefaultCharacter(result, length, str);
+    return true;
+  } else {
+    BadInquiryKeywordHashCrash(inquiry);
+    return false;
+  }
+}
+
+bool InquireUnconnectedFileState::Inquire(
+    InquiryKeywordHash inquiry, bool &result) {
+  switch (inquiry) {
+  case HashInquiryKeyword("EXIST"):
+    result = IsExtant(path_.get());
+    return true;
+  case HashInquiryKeyword("NAMED"):
+    result = true;
+    return true;
+  case HashInquiryKeyword("OPENED"):
+    result = false;
+    return true;
+  case HashInquiryKeyword("PENDING"):
+    result = false;
+    return true;
+  default:
+    BadInquiryKeywordHashCrash(inquiry);
+    return false;
+  }
+}
+
+bool InquireUnconnectedFileState::Inquire(
+    InquiryKeywordHash inquiry, std::int64_t, bool &result) {
+  switch (inquiry) {
+  case HashInquiryKeyword("PENDING"):
+    result = false;
+    return true;
+  default:
+    BadInquiryKeywordHashCrash(inquiry);
+    return false;
+  }
+}
+
+bool InquireUnconnectedFileState::Inquire(
+    InquiryKeywordHash inquiry, std::int64_t &result) {
+  switch (inquiry) {
+  case HashInquiryKeyword("NEXTREC"):
+  case HashInquiryKeyword("NUMBER"):
+  case HashInquiryKeyword("POS"):
+  case HashInquiryKeyword("RECL"):
+  case HashInquiryKeyword("SIZE"):
+    result = -1;
+    return true;
+  default:
+    BadInquiryKeywordHashCrash(inquiry);
+    return false;
+  }
+}
+
+InquireIOLengthState::InquireIOLengthState(
+    const char *sourceFile, int sourceLine)
+    : NoUnitIoStatementState{sourceFile, sourceLine, *this} {}
+
+bool InquireIOLengthState::Emit(
+    const char *, std::size_t n, std::size_t /*elementBytes*/) {
+  bytes_ += n;
+  return true;
+}
+
 } // namespace Fortran::runtime::io
diff --git a/flang/runtime/io-stmt.h b/flang/runtime/io-stmt.h
index ddc264aea3605..9e68deab2e641 100644
--- a/flang/runtime/io-stmt.h
+++ b/flang/runtime/io-stmt.h
@@ -16,6 +16,7 @@
 #include "file.h"
 #include "format.h"
 #include "internal-unit.h"
+#include "io-api.h"
 #include "io-error.h"
 #include <functional>
 #include <type_traits>
@@ -26,6 +27,11 @@ namespace Fortran::runtime::io {
 class ExternalFileUnit;
 
 class OpenStatementState;
+class InquireUnitState;
+class InquireNoUnitState;
+class InquireUnconnectedFileState;
+class InquireIOLengthState;
+class ExternalMiscIoStatementState;
 class CloseStatementState;
 class NoopCloseStatementState;
 
@@ -36,7 +42,6 @@ template <Direction, typename CHAR = char>
 class ExternalFormattedIoStatementState;
 template <Direction> class ExternalListIoStatementState;
 template <Direction> class UnformattedIoStatementState;
-class ExternalMiscIoStatementState;
 
 // The Cookie type in the I/O API is a pointer (for C) to this class.
 class IoStatementState {
@@ -60,6 +65,10 @@ class IoStatementState {
   ExternalFileUnit *GetExternalFileUnit() const; // null if internal unit
   MutableModes &mutableModes();
   void BeginReadingRecord();
+  bool Inquire(InquiryKeywordHash, char *, std::size_t);
+  bool Inquire(InquiryKeywordHash, bool &);
+  bool Inquire(InquiryKeywordHash, std::int64_t, bool &); // PENDING=
+  bool Inquire(InquiryKeywordHash, std::int64_t &);
 
   // N.B.: this also works with base classes
   template <typename A> A *get_if() const {
@@ -98,6 +107,10 @@ class IoStatementState {
       std::reference_wrapper<ExternalListIoStatementState<Direction::Input>>,
       std::reference_wrapper<UnformattedIoStatementState<Direction::Output>>,
       std::reference_wrapper<UnformattedIoStatementState<Direction::Input>>,
+      std::reference_wrapper<InquireUnitState>,
+      std::reference_wrapper<InquireNoUnitState>,
+      std::reference_wrapper<InquireUnconnectedFileState>,
+      std::reference_wrapper<InquireIOLengthState>,
       std::reference_wrapper<ExternalMiscIoStatementState>>
       u_;
 };
@@ -110,6 +123,12 @@ struct IoStatementBase : public DefaultFormatControlCallbacks {
   std::optional<DataEdit> GetNextDataEdit(IoStatementState &, int = 1);
   ExternalFileUnit *GetExternalFileUnit() const { return nullptr; }
   void BeginReadingRecord() {}
+
+  bool Inquire(InquiryKeywordHash, char *, std::size_t);
+  bool Inquire(InquiryKeywordHash, bool &);
+  bool Inquire(InquiryKeywordHash, std::int64_t, bool &);
+  bool Inquire(InquiryKeywordHash, std::int64_t &);
+  void BadInquiryKeywordHashCrash(InquiryKeywordHash);
 };
 
 struct InputStatementState {};
@@ -303,10 +322,12 @@ class OpenStatementState : public ExternalIoStatementBase {
                                                                    wasExtant} {}
   bool wasExtant() const { return wasExtant_; }
   void set_status(OpenStatus status) { status_ = status; } // STATUS=
-  void set_path(const char *, std::size_t, int kind); // FILE=
+  void set_path(const char *, std::size_t); // FILE=
   void set_position(Position position) { position_ = position; } // POSITION=
   void set_action(Action action) { action_ = action; } // ACTION=
   void set_convert(Convert convert) { convert_ = convert; } // CONVERT=
+  void set_access(Access access) { access_ = access; } // ACCESS=
+  void set_isUnformatted(bool yes = true) { isUnformatted_ = yes; } // FORM=
   int EndIoStatement();
 
 private:
@@ -317,6 +338,8 @@ class OpenStatementState : public ExternalIoStatementBase {
   Convert convert_{Convert::Native};
   OwningPtr<char> path_;
   std::size_t pathLength_;
+  std::optional<bool> isUnformatted_;
+  std::optional<Access> access_;
 };
 
 class CloseStatementState : public ExternalIoStatementBase {
@@ -331,21 +354,31 @@ class CloseStatementState : public ExternalIoStatementBase {
   CloseStatus status_{CloseStatus::Keep};
 };
 
-class NoopCloseStatementState : public IoStatementBase {
+// For CLOSE(bad unit) and INQUIRE(unconnected unit)
+class NoUnitIoStatementState : public IoStatementBase {
 public:
-  NoopCloseStatementState(const char *sourceFile, int sourceLine)
-      : IoStatementBase{sourceFile, sourceLine}, ioStatementState_{*this} {}
   IoStatementState &ioStatementState() { return ioStatementState_; }
-  void set_status(CloseStatus) {} // discards
   MutableModes &mutableModes() { return connection_.modes; }
   ConnectionState &GetConnectionState() { return connection_; }
   int EndIoStatement();
 
+protected:
+  template <typename A>
+  NoUnitIoStatementState(const char *sourceFile, int sourceLine, A &stmt)
+      : IoStatementBase{sourceFile, sourceLine}, ioStatementState_{stmt} {}
+
 private:
   IoStatementState ioStatementState_; // points to *this
   ConnectionState connection_;
 };
 
+class NoopCloseStatementState : public NoUnitIoStatementState {
+public:
+  NoopCloseStatementState(const char *sourceFile, int sourceLine)
+      : NoUnitIoStatementState{sourceFile, sourceLine, *this} {}
+  void set_status(CloseStatus) {} // discards
+};
+
 extern template class InternalIoStatementState<Direction::Output>;
 extern template class InternalIoStatementState<Direction::Input>;
 extern template class InternalFormattedIoStatementState<Direction::Output>;
@@ -369,6 +402,49 @@ extern template class FormatControl<
 extern template class FormatControl<
     ExternalFormattedIoStatementState<Direction::Input>>;
 
+class InquireUnitState : public ExternalIoStatementBase {
+public:
+  InquireUnitState(ExternalFileUnit &unit, const char *sourceFile = nullptr,
+      int sourceLine = 0);
+  bool Inquire(InquiryKeywordHash, char *, std::size_t);
+  bool Inquire(InquiryKeywordHash, bool &);
+  bool Inquire(InquiryKeywordHash, std::int64_t, bool &);
+  bool Inquire(InquiryKeywordHash, std::int64_t &);
+};
+
+class InquireNoUnitState : public NoUnitIoStatementState {
+public:
+  InquireNoUnitState(const char *sourceFile = nullptr, int sourceLine = 0);
+  bool Inquire(InquiryKeywordHash, char *, std::size_t);
+  bool Inquire(InquiryKeywordHash, bool &);
+  bool Inquire(InquiryKeywordHash, std::int64_t, bool &);
+  bool Inquire(InquiryKeywordHash, std::int64_t &);
+};
+
+class InquireUnconnectedFileState : public NoUnitIoStatementState {
+public:
+  InquireUnconnectedFileState(OwningPtr<char> &&path,
+      const char *sourceFile = nullptr, int sourceLine = 0);
+  bool Inquire(InquiryKeywordHash, char *, std::size_t);
+  bool Inquire(InquiryKeywordHash, bool &);
+  bool Inquire(InquiryKeywordHash, std::int64_t, bool &);
+  bool Inquire(InquiryKeywordHash, std::int64_t &);
+
+private:
+  OwningPtr<char> path_; // trimmed and NUL terminated
+};
+
+class InquireIOLengthState : public NoUnitIoStatementState,
+                             public OutputStatementState {
+public:
+  InquireIOLengthState(const char *sourceFile = nullptr, int sourceLine = 0);
+  std::size_t bytes() const { return bytes_; }
+  bool Emit(const char *, std::size_t, std::size_t elementBytes = 0);
+
+private:
+  std::size_t bytes_{0};
+};
+
 class ExternalMiscIoStatementState : public ExternalIoStatementBase {
 public:
   enum Which { Flush, Backspace, Endfile, Rewind };
diff --git a/flang/runtime/memory.h b/flang/runtime/memory.h
index f21b237f3905b..4b09fe80772ed 100644
--- a/flang/runtime/memory.h
+++ b/flang/runtime/memory.h
@@ -42,7 +42,7 @@ template <typename A> class SizedNew {
 public:
   explicit SizedNew(const Terminator &terminator) : terminator_{terminator} {}
   template <typename... X>
-  [[nodiscard]] OwningPtr<A> operator()(std::size_t bytes, X &&... x) {
+  [[nodiscard]] OwningPtr<A> operator()(std::size_t bytes, X &&...x) {
     return OwningPtr<A>{new (AllocateMemoryOrCrash(terminator_, bytes))
             A{std::forward<X>(x)...}};
   }
@@ -53,7 +53,7 @@ template <typename A> class SizedNew {
 
 template <typename A> struct New : public SizedNew<A> {
   using SizedNew<A>::SizedNew;
-  template <typename... X> [[nodiscard]] OwningPtr<A> operator()(X &&... x) {
+  template <typename... X> [[nodiscard]] OwningPtr<A> operator()(X &&...x) {
     return SizedNew<A>::operator()(sizeof(A), std::forward<X>(x)...);
   }
 };
diff --git a/flang/runtime/tools.cpp b/flang/runtime/tools.cpp
index ea9ad9063344b..219daaf2880b1 100644
--- a/flang/runtime/tools.cpp
+++ b/flang/runtime/tools.cpp
@@ -12,6 +12,13 @@
 
 namespace Fortran::runtime {
 
+std::size_t TrimTrailingSpaces(const char *s, std::size_t n) {
+  while (n > 0 && s[n - 1] == ' ') {
+    --n;
+  }
+  return n;
+}
+
 OwningPtr<char> SaveDefaultCharacter(
     const char *s, std::size_t length, const Terminator &terminator) {
   if (s) {
diff --git a/flang/runtime/tools.h b/flang/runtime/tools.h
index fad19f607c683..6c5eb63cc8c11 100644
--- a/flang/runtime/tools.h
+++ b/flang/runtime/tools.h
@@ -18,6 +18,8 @@ namespace Fortran::runtime {
 
 class Terminator;
 
+std::size_t TrimTrailingSpaces(const char *, std::size_t);
+
 OwningPtr<char> SaveDefaultCharacter(
     const char *, std::size_t, const Terminator &);
 
diff --git a/flang/runtime/unit-map.cpp b/flang/runtime/unit-map.cpp
index 905beb4d084fa..1cd2115f4aa1b 100644
--- a/flang/runtime/unit-map.cpp
+++ b/flang/runtime/unit-map.cpp
@@ -72,6 +72,20 @@ void UnitMap::FlushAll(IoErrorHandler &handler) {
   }
 }
 
+ExternalFileUnit *UnitMap::Find(const char *path) {
+  if (path) {
+    // TODO: Faster data structure
+    for (int j{0}; j < buckets_; ++j) {
+      for (Chain *p{bucket_[j].get()}; p; p = p->next.get()) {
+        if (p->unit.path() && std::strcmp(p->unit.path(), path) == 0) {
+          return &p->unit;
+        }
+      }
+    }
+  }
+  return nullptr;
+}
+
 ExternalFileUnit &UnitMap::Create(int n, const Terminator &terminator) {
   Chain &chain{*New<Chain>{terminator}(n).release()};
   chain.next.reset(&chain);
diff --git a/flang/runtime/unit-map.h b/flang/runtime/unit-map.h
index be244f5ae463a..961962a2d635c 100644
--- a/flang/runtime/unit-map.h
+++ b/flang/runtime/unit-map.h
@@ -34,6 +34,12 @@ class UnitMap {
     return p ? *p : Create(n, terminator);
   }
 
+  // Unit look-up by name is needed for INQUIRE(FILE="...")
+  ExternalFileUnit *LookUp(const char *path) {
+    CriticalSection critical{lock_};
+    return Find(path);
+  }
+
   ExternalFileUnit &NewUnit(const Terminator &terminator) {
     CriticalSection critical{lock_};
     return Create(nextNewUnit_--, terminator);
@@ -72,6 +78,7 @@ class UnitMap {
     }
     return nullptr;
   }
+  ExternalFileUnit *Find(const char *path);
 
   ExternalFileUnit &Create(int, const Terminator &);
 
diff --git a/flang/runtime/unit.cpp b/flang/runtime/unit.cpp
index a4c69df8d6a9a..be36666f66e46 100644
--- a/flang/runtime/unit.cpp
+++ b/flang/runtime/unit.cpp
@@ -59,20 +59,19 @@ ExternalFileUnit &ExternalFileUnit::LookUpOrCreateAnonymous(
   ExternalFileUnit &result{
       GetUnitMap().LookUpOrCreate(unit, terminator, exists)};
   if (!exists) {
-    // I/O to an unconnected unit reads/creates a local file, e.g. fort.7
-    std::size_t pathMaxLen{32};
-    auto path{SizedNew<char>{terminator}(pathMaxLen)};
-    std::snprintf(path.get(), pathMaxLen, "fort.%d", unit);
     IoErrorHandler handler{terminator};
-    result.OpenUnit(
-        dir == Direction::Input ? OpenStatus::Old : OpenStatus::Replace,
-        Action::ReadWrite, Position::Rewind, std::move(path),
-        std::strlen(path.get()), Convert::Native, handler);
+    result.OpenAnonymousUnit(
+        dir == Direction::Input ? OpenStatus::Unknown : OpenStatus::Replace,
+        Action::ReadWrite, Position::Rewind, Convert::Native, handler);
     result.isUnformatted = isUnformatted;
   }
   return result;
 }
 
+ExternalFileUnit *ExternalFileUnit::LookUp(const char *path) {
+  return GetUnitMap().LookUp(path);
+}
+
 ExternalFileUnit &ExternalFileUnit::CreateNew(
     int unit, const Terminator &terminator) {
   bool wasExtant{false};
@@ -125,10 +124,7 @@ void ExternalFileUnit::OpenUnit(OpenStatus status, std::optional<Action> action,
       handler.SignalError(IostatOpenBadRecl,
           "OPEN(UNIT=%d,ACCESS='DIRECT',RECL=%jd): record length is invalid",
           unitNumber(), static_cast<std::intmax_t>(*recordLength));
-    } else if (!totalBytes) {
-      handler.SignalError(IostatOpenUnknownSize,
-          "OPEN(UNIT=%d,ACCESS='DIRECT'): file size is not known");
-    } else if (*totalBytes % *recordLength != 0) {
+    } else if (totalBytes && (*totalBytes % *recordLength != 0)) {
       handler.SignalError(IostatOpenBadAppend,
           "OPEN(UNIT=%d,ACCESS='DIRECT',RECL=%jd): record length is not an "
           "even divisor of the file size %jd",
@@ -137,7 +133,7 @@ void ExternalFileUnit::OpenUnit(OpenStatus status, std::optional<Action> action,
     }
   }
   if (position == Position::Append) {
-    if (*totalBytes && recordLength && *recordLength) {
+    if (totalBytes && recordLength && *recordLength) {
       endfileRecordNumber = 1 + (*totalBytes / *recordLength);
     } else {
       // Fake it so that we can backspace relative from the end
@@ -149,6 +145,17 @@ void ExternalFileUnit::OpenUnit(OpenStatus status, std::optional<Action> action,
   }
 }
 
+void ExternalFileUnit::OpenAnonymousUnit(OpenStatus status,
+    std::optional<Action> action, Position position, Convert convert,
+    IoErrorHandler &handler) {
+  // I/O to an unconnected unit reads/creates a local file, e.g. fort.7
+  std::size_t pathMaxLen{32};
+  auto path{SizedNew<char>{handler}(pathMaxLen)};
+  std::snprintf(path.get(), pathMaxLen, "fort.%d", unitNumber_);
+  OpenUnit(status, action, position, std::move(path), std::strlen(path.get()),
+      convert, handler);
+}
+
 void ExternalFileUnit::CloseUnit(CloseStatus status, IoErrorHandler &handler) {
   DoImpliedEndfile(handler);
   Flush(handler);
diff --git a/flang/runtime/unit.h b/flang/runtime/unit.h
index f94e4229cd4c8..9d66d962bc56d 100644
--- a/flang/runtime/unit.h
+++ b/flang/runtime/unit.h
@@ -35,6 +35,7 @@ class ExternalFileUnit : public ConnectionState,
 public:
   explicit ExternalFileUnit(int unitNumber) : unitNumber_{unitNumber} {}
   int unitNumber() const { return unitNumber_; }
+  bool swapEndianness() const { return swapEndianness_; }
 
   static ExternalFileUnit *LookUp(int unit);
   static ExternalFileUnit &LookUpOrCrash(int unit, const Terminator &);
@@ -42,6 +43,7 @@ class ExternalFileUnit : public ConnectionState,
       int unit, const Terminator &, bool &wasExtant);
   static ExternalFileUnit &LookUpOrCreateAnonymous(
       int unit, Direction, bool isUnformatted, const Terminator &);
+  static ExternalFileUnit *LookUp(const char *path);
   static ExternalFileUnit &CreateNew(int unit, const Terminator &);
   static ExternalFileUnit *LookUpForClose(int unit);
   static int NewUnit(const Terminator &);
@@ -51,13 +53,15 @@ class ExternalFileUnit : public ConnectionState,
   void OpenUnit(OpenStatus, std::optional<Action>, Position,
       OwningPtr<char> &&path, std::size_t pathLength, Convert,
       IoErrorHandler &);
+  void OpenAnonymousUnit(
+      OpenStatus, std::optional<Action>, Position, Convert, IoErrorHandler &);
   void CloseUnit(CloseStatus, IoErrorHandler &);
   void DestroyClosed();
 
   bool SetDirection(Direction, IoErrorHandler &);
 
   template <typename A, typename... X>
-  IoStatementState &BeginIoStatement(X &&... xs) {
+  IoStatementState &BeginIoStatement(X &&...xs) {
     // TODO: Child data transfer statements vs. locking
     lock_.Take(); // dropped in EndIoStatement()
     A &state{u_.emplace<A>(std::forward<X>(xs)...)};
@@ -111,7 +115,7 @@ class ExternalFileUnit : public ConnectionState,
       ExternalListIoStatementState<Direction::Output>,
       ExternalListIoStatementState<Direction::Input>,
       UnformattedIoStatementState<Direction::Output>,
-      UnformattedIoStatementState<Direction::Input>,
+      UnformattedIoStatementState<Direction::Input>, InquireUnitState,
       ExternalMiscIoStatementState>
       u_;
 

From ba955397ac44e5df6135469d76c645fdcac256da Mon Sep 17 00:00:00 2001
From: Chen Zheng <czhengsz@cn.ibm.com>
Date: Mon, 3 Aug 2020 08:37:52 -0400
Subject: [PATCH 0979/1035] [SCEVExpander][PowerPC]clear scev rewriter before
 deleting instructions.

Reviewed By: lebedev.ri
Differential Revision: https://reviews.llvm.org/D85130
---
 llvm/lib/Target/PowerPC/PPCLoopInstrFormPrep.cpp | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/llvm/lib/Target/PowerPC/PPCLoopInstrFormPrep.cpp b/llvm/lib/Target/PowerPC/PPCLoopInstrFormPrep.cpp
index a7546d2be5d83..0068ad7174ee7 100644
--- a/llvm/lib/Target/PowerPC/PPCLoopInstrFormPrep.cpp
+++ b/llvm/lib/Target/PowerPC/PPCLoopInstrFormPrep.cpp
@@ -606,6 +606,10 @@ bool PPCLoopInstrFormPrep::rewriteLoadStores(Loop *L, Bucket &BucketChain,
       NewBasePtr = NewPHI;
   }
 
+  // Clear the rewriter cache, because values that are in the rewriter's cache
+  // can be deleted below, causing the AssertingVH in the cache to trigger.
+  SCEVE.clear();
+
   if (Instruction *IDel = dyn_cast<Instruction>(BasePtr))
     BBChanged.insert(IDel->getParent());
   BasePtr->replaceAllUsesWith(NewBasePtr);

From a06c28df3e8c85ceb665d3d9a1ebc2853dfd87a9 Mon Sep 17 00:00:00 2001
From: Jordan Rupprecht <rupprecht@google.com>
Date: Mon, 3 Aug 2020 18:37:50 -0700
Subject: [PATCH 0980/1035] Temporarily revert "[test] Exit with an error if no
 tests are run."

This reverts commit adb5c23f8c0d60eeec41dcbe21d1b26184e1c97d. It surprisingly fails on a windows build bot: http://lab.llvm.org:8011/builders/lldb-x64-windows-ninja/builds/18009

Will reland after some investigation and/or after adding some extra logging to help debug the issue.
---
 lldb/packages/Python/lldbsuite/test/dotest.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/lldb/packages/Python/lldbsuite/test/dotest.py b/lldb/packages/Python/lldbsuite/test/dotest.py
index 6607f52c49dbd..3fb802f1c1aa5 100644
--- a/lldb/packages/Python/lldbsuite/test/dotest.py
+++ b/lldb/packages/Python/lldbsuite/test/dotest.py
@@ -1039,10 +1039,6 @@ def run_suite():
             (configuration.suite.countTestCases(),
              configuration.suite.countTestCases() != 1 and "s" or ""))
 
-    if configuration.suite.countTestCases() == 0:
-        logging.error("did not discover any matching tests")
-        exitTestSuite(1)
-
     # Invoke the test runner.
     if configuration.count == 1:
         result = unittest2.TextTestRunner(

From d6a5cce0e7d65562f081569a61595e53cdb8d5d0 Mon Sep 17 00:00:00 2001
From: Jonas Devlieghere <jonas@devlieghere.com>
Date: Mon, 3 Aug 2020 19:56:10 -0700
Subject: [PATCH 0981/1035] [lldb/Test] Fix skipTestIfFn for fucntions that
 return a value

Sometimes the decorator is used on a common function rather than the
test method, which can return a value. This fails with decorators that
use skipTestIfFn under the hood.
---
 lldb/packages/Python/lldbsuite/test/decorators.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lldb/packages/Python/lldbsuite/test/decorators.py b/lldb/packages/Python/lldbsuite/test/decorators.py
index 8c8f2509a8639..873952e4c91ab 100644
--- a/lldb/packages/Python/lldbsuite/test/decorators.py
+++ b/lldb/packages/Python/lldbsuite/test/decorators.py
@@ -132,7 +132,7 @@ def wrapper(*args, **kwargs):
             if reason is not None:
                 self.skipTest(reason)
             else:
-                func(*args, **kwargs)
+                return func(*args, **kwargs)
         return wrapper
 
     # Some decorators can be called both with no arguments (e.g. @expectedFailureWindows)

From 7647c2716e383c091b7063e150d48d5821bcaa67 Mon Sep 17 00:00:00 2001
From: Max Kazantsev <mkazantsev@azul.com>
Date: Tue, 4 Aug 2020 10:16:40 +0700
Subject: [PATCH 0982/1035] [SimpleLoopUnswitch][NFC] Add option to always drop
 make.implicit metadata in non-trivial unswitching and save compile time

We might want this if we find out that using of MustExecute analysis is too expensive.
By default we do the analysis because its complexity does not exceed the complexity
of whole loop copying in unswitching. Follow-up for D84925.

Differential Revision: https://reviews.llvm.org/D85001
Reviewed By: asbirlea
---
 .../Transforms/Scalar/SimpleLoopUnswitch.cpp  | 21 ++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp b/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp
index b1cbc714ec8e1..ab1945a1aff11 100644
--- a/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp
+++ b/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp
@@ -94,6 +94,11 @@ static cl::opt<bool> UnswitchGuards(
     "simple-loop-unswitch-guards", cl::init(true), cl::Hidden,
     cl::desc("If enabled, simple loop unswitching will also consider "
              "llvm.experimental.guard intrinsics as unswitch candidates."));
+static cl::opt<bool> DropNonTrivialImplicitNullChecks(
+    "simple-loop-unswitch-drop-non-trivial-implicit-null-checks",
+    cl::init(false), cl::Hidden,
+    cl::desc("If enabled, drop make.implicit metadata in unswitched implicit "
+             "null checks to save time analyzing if we can keep it."));
 
 /// Collect all of the loop invariant input values transitively used by the
 /// homogeneous instruction graph from a given root.
@@ -2074,12 +2079,18 @@ static void unswitchNontrivialInvariants(
   // Drop metadata if we may break its semantics by moving this instr into the
   // split block.
   if (TI.getMetadata(LLVMContext::MD_make_implicit)) {
-    // It is only legal to preserve make.implicit metadata if we are guaranteed
-    // to reach implicit null check block after following this branch.
-    ICFLoopSafetyInfo SafetyInfo;
-    SafetyInfo.computeLoopSafetyInfo(&L);
-    if (!SafetyInfo.isGuaranteedToExecute(TI, &DT, &L))
+    if (DropNonTrivialImplicitNullChecks)
+      // Do not spend time trying to understand if we can keep it, just drop it
+      // to save compile time.
       TI.setMetadata(LLVMContext::MD_make_implicit, nullptr);
+    else {
+      // It is only legal to preserve make.implicit metadata if we are
+      // guaranteed no reach implicit null check after following this branch.
+      ICFLoopSafetyInfo SafetyInfo;
+      SafetyInfo.computeLoopSafetyInfo(&L);
+      if (!SafetyInfo.isGuaranteedToExecute(TI, &DT, &L))
+        TI.setMetadata(LLVMContext::MD_make_implicit, nullptr);
+    }
   }
 
   // The stitching of the branched code back together depends on whether we're

From e56626e43826c9d7c35113635d62b57c905ef3c0 Mon Sep 17 00:00:00 2001
From: Fangrui Song <maskray@google.com>
Date: Mon, 3 Aug 2020 20:35:50 -0700
Subject: [PATCH 0983/1035] [PGO] Move __profc_ and __profvp_ from their own
 comdat groups to __profd_'s comdat group

D68041 placed `__profc_`,  `__profd_` and (if exists) `__profvp_` in different comdat groups.
There are some issues:

* Cost: one or two additional section headers (`.group` section(s)): 64 or 128 bytes on ELF64.
* `__profc_`,  `__profd_` and (if exists) `__profvp_` should be retained or
  discarded. Placing them into separate comdat groups is conceptually inferior.
* If the prevailing group does not include `__profvp_` (value profiling not
  used) but a non-prevailing group from another translation unit has `__profvp_`
  (the function is inlined into another and triggers value profiling), there
  will be a stray `__profvp_` if --gc-sections is not enabled.
  This has been fixed by 3d6f53018f845e893ad34f64ff2851a2e5c3ba1d.

Actually, we can reuse an existing symbol (we choose `__profd_`) as the group
signature to avoid a string in the string table (the sole reason that D68041
could improve code size is that `__profv_` was an otherwise unused symbol which
wasted string table space). This saves one or two section headers.

For a -DCMAKE_BUILD_TYPE=Release -DLLVM_BUILD_INSTRUMENTED=IR build, `ninja
clang lld`, the patch has saved 10.5MiB (2.2%) for the total .o size.

Reviewed By: davidxl

Differential Revision: https://reviews.llvm.org/D84723
---
 .../Transforms/Instrumentation/InstrProfiling.cpp |  9 +++++----
 .../Instrumentation/InstrProfiling/PR23499.ll     |  4 ++--
 .../test/Instrumentation/InstrProfiling/comdat.ll |  8 ++++----
 llvm/test/Instrumentation/InstrProfiling/icall.ll | 15 ++++++++++++++-
 .../Instrumentation/InstrProfiling/linkage.ll     |  4 ++--
 .../test/Transforms/PGOProfile/comdat_internal.ll |  3 ++-
 6 files changed, 29 insertions(+), 14 deletions(-)

diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
index 75988893fdb85..623f463293cd9 100644
--- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
+++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
@@ -882,9 +882,10 @@ InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) {
       Visibility = GlobalValue::HiddenVisibility;
     }
   }
+  std::string DataVarName = getVarName(Inc, getInstrProfDataVarPrefix());
   auto MaybeSetComdat = [=](GlobalVariable *GV) {
     if (NeedComdat)
-      GV->setComdat(M->getOrInsertComdat(GV->getName()));
+      GV->setComdat(M->getOrInsertComdat(DataVarName));
   };
 
   uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
@@ -949,9 +950,9 @@ InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) {
 #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Init,
 #include "llvm/ProfileData/InstrProfData.inc"
   };
-  auto *Data = new GlobalVariable(*M, DataTy, false, Linkage,
-                                  ConstantStruct::get(DataTy, DataVals),
-                                  getVarName(Inc, getInstrProfDataVarPrefix()));
+  auto *Data =
+      new GlobalVariable(*M, DataTy, false, Linkage,
+                         ConstantStruct::get(DataTy, DataVals), DataVarName);
   Data->setVisibility(Visibility);
   Data->setSection(getInstrProfSectionName(IPSK_data, TT.getObjectFormat()));
   Data->setAlignment(Align(INSTR_PROF_DATA_ALIGNMENT));
diff --git a/llvm/test/Instrumentation/InstrProfiling/PR23499.ll b/llvm/test/Instrumentation/InstrProfiling/PR23499.ll
index 1a4c04947a2b1..098153f39e1db 100644
--- a/llvm/test/Instrumentation/InstrProfiling/PR23499.ll
+++ b/llvm/test/Instrumentation/InstrProfiling/PR23499.ll
@@ -14,13 +14,13 @@ $_Z3barIvEvv = comdat any
 @__profn__Z3barIvEvv = linkonce_odr hidden constant [11 x i8] c"_Z3barIvEvv", align 1
 
 ; CHECK-NOT: __profn__Z3barIvEvv
-; CHECK: @__profc__Z3barIvEvv = linkonce_odr hidden global [1 x i64] zeroinitializer, section "{{.*}}__llvm_prf_cnts", comdat, align 8
+; CHECK: @__profc__Z3barIvEvv = linkonce_odr hidden global [1 x i64] zeroinitializer, section "{{.*}}__llvm_prf_cnts", comdat($__profd__Z3barIvEvv), align 8
 ; CHECK: @__profd__Z3barIvEvv = linkonce_odr hidden global { i64, i64, i64*, i8*, i8*, i32, [2 x i16] } { i64 4947693190065689389, i64 0, i64* getelementptr inbounds ([1 x i64], [1 x i64]* @__profc__Z3barIvEvv, i32 0, i32 0), i8*{{.*}}, i8* null, i32 1, [2 x i16] zeroinitializer }, section "{{.*}}__llvm_prf_data{{.*}}", comdat, align 8
 ; CHECK: @__llvm_prf_nm = private constant [{{.*}} x i8] c"{{.*}}", section "{{.*}}__llvm_prf_names"
 
 
 ; COFF-NOT: __profn__Z3barIvEvv
-; COFF: @__profc__Z3barIvEvv = linkonce_odr hidden global [1 x i64] zeroinitializer, section "{{.*}}prfc$M", comdat, align 8
+; COFF: @__profc__Z3barIvEvv = linkonce_odr hidden global [1 x i64] zeroinitializer, section "{{.*}}prfc$M", comdat($__profd__Z3barIvEvv), align 8
 ; COFF: @__profd__Z3barIvEvv = linkonce_odr hidden global { i64, i64, i64*, i8*, i8*, i32, [2 x i16] } { i64 4947693190065689389, i64 0, i64* getelementptr inbounds ([1 x i64], [1 x i64]* @__profc__Z3barIvEvv, i32 0, i32 0), i8*{{.*}}, i8* null, i32 1, [2 x i16] zeroinitializer }, section "{{.*}}prfd{{.*}}", comdat, align 8
 
 
diff --git a/llvm/test/Instrumentation/InstrProfiling/comdat.ll b/llvm/test/Instrumentation/InstrProfiling/comdat.ll
index 3f169c4b73423..dfcd71fd06291 100644
--- a/llvm/test/Instrumentation/InstrProfiling/comdat.ll
+++ b/llvm/test/Instrumentation/InstrProfiling/comdat.ll
@@ -15,9 +15,9 @@ $foo_inline = comdat any
 
 @__profn_foo_inline = linkonce_odr hidden constant [10 x i8] c"foo_inline"
 
-; ELF: @__profc_foo_inline = linkonce_odr hidden global{{.*}}, section "__llvm_prf_cnts", comdat, align 8
+; ELF: @__profc_foo_inline = linkonce_odr hidden global{{.*}}, section "__llvm_prf_cnts", comdat($__profd_foo_inline), align 8
 ; ELF: @__profd_foo_inline = linkonce_odr hidden global{{.*}}, section "__llvm_prf_data", comdat, align 8
-; COFF: @__profc_foo_inline = linkonce_odr hidden global{{.*}}, section ".lprfc$M", comdat, align 8
+; COFF: @__profc_foo_inline = linkonce_odr hidden global{{.*}}, section ".lprfc$M", comdat($__profd_foo_inline), align 8
 ; COFF: @__profd_foo_inline = linkonce_odr hidden global{{.*}}, section ".lprfd$M", comdat, align 8
 define weak_odr void @foo_inline() comdat {
   call void @llvm.instrprof.increment(i8* getelementptr inbounds ([10 x i8], [10 x i8]* @__profn_foo_inline, i32 0, i32 0), i64 0, i32 1, i32 0)
@@ -28,9 +28,9 @@ $foo_extern = comdat any
 
 @__profn_foo_extern = linkonce_odr hidden constant [10 x i8] c"foo_extern"
 
-; ELF: @__profc_foo_extern = linkonce_odr hidden global{{.*}}, section "__llvm_prf_cnts", comdat, align 8
+; ELF: @__profc_foo_extern = linkonce_odr hidden global{{.*}}, section "__llvm_prf_cnts", comdat($__profd_foo_extern), align 8
 ; ELF: @__profd_foo_extern = linkonce_odr hidden global{{.*}}, section "__llvm_prf_data", comdat, align 8
-; COFF: @__profc_foo_extern = linkonce_odr hidden global{{.*}}, section ".lprfc$M", comdat, align 8
+; COFF: @__profc_foo_extern = linkonce_odr hidden global{{.*}}, section ".lprfc$M", comdat($__profd_foo_extern), align 8
 ; COFF: @__profd_foo_extern = linkonce_odr hidden global{{.*}}, section ".lprfd$M", comdat, align 8
 define available_externally void @foo_extern() {
   call void @llvm.instrprof.increment(i8* getelementptr inbounds ([10 x i8], [10 x i8]* @__profn_foo_extern, i32 0, i32 0), i64 0, i32 1, i32 0)
diff --git a/llvm/test/Instrumentation/InstrProfiling/icall.ll b/llvm/test/Instrumentation/InstrProfiling/icall.ll
index d92de47421d40..9d45d7cd56358 100644
--- a/llvm/test/Instrumentation/InstrProfiling/icall.ll
+++ b/llvm/test/Instrumentation/InstrProfiling/icall.ll
@@ -14,6 +14,7 @@
 
 
 @__profn_foo = private constant [3 x i8] c"foo"
+@__profn_bar = private constant [3 x i8] c"bar"
 
 define i32 @foo(i32 ()* ) {
   call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_foo, i32 0, i32 0), i64 12884901887, i32 1, i32 0)
@@ -23,6 +24,17 @@ define i32 @foo(i32 ()* ) {
   ret i32 %3
 }
 
+$bar = comdat any
+
+define i32 @bar(i32 ()* ) comdat {
+entry:
+  call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_bar, i32 0, i32 0), i64 12884901887, i32 1, i32 0)
+  %1 = ptrtoint i32 ()* %0 to i64
+  call void @llvm.instrprof.value.profile(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_bar, i32 0, i32 0), i64 12884901887, i64 %1, i32 0, i32 0)
+  %2 = tail call i32 %0()
+  ret i32 %2
+}
+
 ; Function Attrs: nounwind
 declare void @llvm.instrprof.increment(i8*, i64, i32, i32) #0
 
@@ -31,7 +43,8 @@ declare void @llvm.instrprof.value.profile(i8*, i64, i64, i32, i32) #0
 
 attributes #0 = { nounwind }
 
-; STATIC: @__profvp_foo
+; STATIC: @__profvp_foo = private global [1 x i64] zeroinitializer, section "{{[^"]+}}", align 8
+; STATIC: @__profvp_bar = private global [1 x i64] zeroinitializer, section "{{[^"]+}}", comdat($__profd_bar), align 8
 ; STATIC: @__llvm_prf_vnodes
 
 ; DYN-NOT: @__profvp_foo
diff --git a/llvm/test/Instrumentation/InstrProfiling/linkage.ll b/llvm/test/Instrumentation/InstrProfiling/linkage.ll
index be3af8e51a5f8..5e55ef716820e 100644
--- a/llvm/test/Instrumentation/InstrProfiling/linkage.ll
+++ b/llvm/test/Instrumentation/InstrProfiling/linkage.ll
@@ -55,11 +55,11 @@ define linkonce_odr void @foo_inline() {
   ret void
 }
 
-; LINUX: @__profc_foo_extern = linkonce_odr hidden global {{.*}}section "__llvm_prf_cnts", comdat, align 8
+; LINUX: @__profc_foo_extern = linkonce_odr hidden global {{.*}}section "__llvm_prf_cnts", comdat($__profd_foo_extern), align 8
 ; LINUX: @__profd_foo_extern = linkonce_odr hidden global {{.*}}section "__llvm_prf_data", comdat, align 8
 ; MACHO: @__profc_foo_extern = linkonce_odr hidden global
 ; MACHO: @__profd_foo_extern = linkonce_odr hidden global
-; COFF: @__profc_foo_extern = linkonce_odr hidden global {{.*}}section ".lprfc$M", comdat, align 8
+; COFF: @__profc_foo_extern = linkonce_odr hidden global {{.*}}section ".lprfc$M", comdat($__profd_foo_extern), align 8
 ; COFF: @__profd_foo_extern = linkonce_odr hidden global {{.*}}section ".lprfd$M", comdat, align 8
 define available_externally void @foo_extern() {
   call void @llvm.instrprof.increment(i8* getelementptr inbounds ([10 x i8], [10 x i8]* @__profn_foo_extern, i32 0, i32 0), i64 0, i32 1, i32 0)
diff --git a/llvm/test/Transforms/PGOProfile/comdat_internal.ll b/llvm/test/Transforms/PGOProfile/comdat_internal.ll
index f0906bf2b3720..e5915d906a086 100644
--- a/llvm/test/Transforms/PGOProfile/comdat_internal.ll
+++ b/llvm/test/Transforms/PGOProfile/comdat_internal.ll
@@ -7,12 +7,13 @@ $foo = comdat any
 ; CHECK: $foo = comdat any
 
 ; CHECK: $__llvm_profile_raw_version = comdat any
+; CHECK: $__profd__stdin__foo.[[FOO_HASH:[0-9]+]] = comdat any
 
 @bar = global i32 ()* @foo, align 8
 
 ; CHECK: @__llvm_profile_raw_version = constant i64 {{[0-9]+}}, comdat
 ; CHECK-NOT: __profn__stdin__foo
-; CHECK: @__profc__stdin__foo.[[FOO_HASH:[0-9]+]] = private global [1 x i64] zeroinitializer, section "__llvm_prf_cnts", comdat, align 8
+; CHECK: @__profc__stdin__foo.[[FOO_HASH]] = private global [1 x i64] zeroinitializer, section "__llvm_prf_cnts", comdat($__profd__stdin__foo.[[FOO_HASH]]), align 8
 ; CHECK: @__profd__stdin__foo.[[FOO_HASH]] = private global { i64, i64, i64*, i8*, i8*, i32, [2 x i16] } { i64 -5640069336071256030, i64 [[FOO_HASH]], i64* getelementptr inbounds ([1 x i64], [1 x i64]* @__profc__stdin__foo.[[FOO_HASH]], i32 0, i32 0), i8* null
 ; CHECK-NOT: bitcast (i32 ()* @foo to i8*)
 ; CHECK-SAME: , i8* null, i32 1, [2 x i16] zeroinitializer }, section "__llvm_prf_data", comdat, align 8

From bcea3a7a288e0b5ac977f90c46e4eef7946467e7 Mon Sep 17 00:00:00 2001
From: Fangrui Song <maskray@google.com>
Date: Mon, 3 Aug 2020 10:17:55 -0700
Subject: [PATCH 0984/1035] Add test utility 'split-file'

See https://lists.llvm.org/pipermail/llvm-dev/2020-July/143373.html
"[llvm-dev] Multiple documents in one test file" for some discussions.

This patch has explored several alternatives. The current semantics are similar to
what @dblaikie proposed.
`split-file filename output` splits the input file into multiple parts separated by
regex `^(.|//)--- filename` and write each part to the file `output/filename`
(`filename` can include path separators).

Use case A (organizing input of different formats (e.g. linker
script+assembly) in one file).

```
# RUN: split-file %s %t
# RUN: llvm-mc %t/asm -o %t.o
# RUN: ld.lld -T %t/lds %t.o -o %t
This is sometimes better than the %S/Inputs/ approach because the user
can see the auxiliary files immediately and don't have to open another file.

# asm
...
# lds
...
```

Use case B (for utilities which don't have built-in input splitting
feature):

```
// RUN: split-file %s %t
// RUN: llc < %t/1.ll | FileCheck %s --check-prefix=CASE1
// RUN: llc < %t/2.ll | FileCheck %s --check-prefix=CASE2
Combing tests prudently can improve readability.
For example, when testing parsing errors if the recovery mechanism isn't possible,
grouping the tests in one file can more readily see test coverage/strategy.

//--- 1.ll
...
//--- 2.ll
...
```

Since this is a new utility, there is no git history concerns for
UpperCase variable names. I use lowerCase variable names like mlir/lld.

Reviewed By: jhenderson, lattner

Differential Revision: https://reviews.llvm.org/D83834
---
 lld/test/CMakeLists.txt                       |   2 +-
 lld/test/ELF/linkerscript/noload.s            |  21 ++-
 llvm/docs/TestingGuide.rst                    |  23 ++-
 llvm/test/CMakeLists.txt                      |   1 +
 llvm/test/lit.cfg.py                          |   1 +
 llvm/test/tools/gold/X86/multiple-sections.ll |  14 +-
 llvm/test/tools/llvm-strings/radix.test       |  45 ++---
 .../test/tools/split-file/Inputs/basic-aa.txt |   2 +
 .../test/tools/split-file/Inputs/basic-bb.txt |   6 +
 .../test/tools/split-file/Inputs/basic-cc.txt |   8 +
 llvm/test/tools/split-file/basic.test         |  40 ++++
 llvm/test/tools/split-file/empty.test         |   4 +
 llvm/test/tools/split-file/error.test         |  16 ++
 llvm/test/tools/split-file/help.test          |   6 +
 .../tools/split-file/no-leading-lines.test    |  10 +
 .../tools/split-file/output-is-special.test   |   8 +
 llvm/tools/split-file/.clang-tidy             |  19 ++
 llvm/tools/split-file/CMakeLists.txt          |   7 +
 llvm/tools/split-file/split-file.cpp          | 172 ++++++++++++++++++
 llvm/utils/gn/secondary/lld/test/BUILD.gn     |   1 +
 llvm/utils/gn/secondary/llvm/test/BUILD.gn    |   1 +
 .../secondary/llvm/tools/split-file/BUILD.gn  |   4 +
 22 files changed, 374 insertions(+), 37 deletions(-)
 create mode 100644 llvm/test/tools/split-file/Inputs/basic-aa.txt
 create mode 100644 llvm/test/tools/split-file/Inputs/basic-bb.txt
 create mode 100644 llvm/test/tools/split-file/Inputs/basic-cc.txt
 create mode 100644 llvm/test/tools/split-file/basic.test
 create mode 100644 llvm/test/tools/split-file/empty.test
 create mode 100644 llvm/test/tools/split-file/error.test
 create mode 100644 llvm/test/tools/split-file/help.test
 create mode 100644 llvm/test/tools/split-file/no-leading-lines.test
 create mode 100644 llvm/test/tools/split-file/output-is-special.test
 create mode 100644 llvm/tools/split-file/.clang-tidy
 create mode 100644 llvm/tools/split-file/CMakeLists.txt
 create mode 100644 llvm/tools/split-file/split-file.cpp
 create mode 100644 llvm/utils/gn/secondary/llvm/tools/split-file/BUILD.gn

diff --git a/lld/test/CMakeLists.txt b/lld/test/CMakeLists.txt
index e7d1133307393..52e6118ba876b 100644
--- a/lld/test/CMakeLists.txt
+++ b/lld/test/CMakeLists.txt
@@ -28,7 +28,7 @@ if (NOT LLD_BUILT_STANDALONE)
     FileCheck count llc llvm-ar llvm-as llvm-bcanalyzer llvm-config llvm-cvtres
     llvm-dis llvm-dwarfdump llvm-lib llvm-lipo llvm-mc llvm-nm llvm-objcopy
     llvm-objdump llvm-pdbutil llvm-readelf llvm-readobj llvm-strip not obj2yaml
-    opt yaml2obj
+    opt split-file yaml2obj
     )
 endif()
 
diff --git a/lld/test/ELF/linkerscript/noload.s b/lld/test/ELF/linkerscript/noload.s
index 2f52b465854e2..20b07b2b185a2 100644
--- a/lld/test/ELF/linkerscript/noload.s
+++ b/lld/test/ELF/linkerscript/noload.s
@@ -1,12 +1,8 @@
 # REQUIRES: x86
-# RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %s -o %t.o
-# RUN: echo "SECTIONS { \
-# RUN:        .data_noload_a (NOLOAD) : { *(.data_noload_a) } \
-# RUN:        .data_noload_b (0x10000) (NOLOAD) : { *(.data_noload_b) } \
-# RUN:        .no_input_sec_noload (NOLOAD) : { . += 1; } \
-# RUN:        .text (0x20000) : { *(.text) } };" > %t.script
-# RUN: ld.lld -o %t --script %t.script %t.o
-# RUN: llvm-readelf -S -l %t | FileCheck %s
+# RUN: split-file %s %t
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %t/asm -o %t.o
+# RUN: ld.lld --script %t/lds %t.o -o %t/out
+# RUN: llvm-readelf -S -l %t/out | FileCheck %s
 
 # CHECK:      Name                 Type   Address          Off               Size
 # CHECK:      .data_noload_a       NOBITS 0000000000000000 [[OFF:[0-9a-f]+]] 001000
@@ -16,6 +12,7 @@
 # CHECK:      Type Offset   VirtAddr           PhysAddr
 # CHECK-NEXT: LOAD 0x001000 0x0000000000020000 0x0000000000020000
 
+#--- asm
 .section .text,"ax",@progbits
   nop
 
@@ -24,3 +21,11 @@
 
 .section .data_noload_b,"aw",@progbits
 .zero 4096
+
+#--- lds
+SECTIONS {
+  .data_noload_a (NOLOAD) : { *(.data_noload_a) }
+  .data_noload_b (0x10000) (NOLOAD) : { *(.data_noload_b) }
+  .no_input_sec_noload (NOLOAD) : { . += 1; }
+  .text (0x20000) : { *(.text) }
+}
diff --git a/llvm/docs/TestingGuide.rst b/llvm/docs/TestingGuide.rst
index 2e937f0006272..4ca1a359b64c9 100644
--- a/llvm/docs/TestingGuide.rst
+++ b/llvm/docs/TestingGuide.rst
@@ -271,8 +271,27 @@ adding your code there instead of creating a new file.
 Extra files
 -----------
 
-If your test requires extra files besides the file containing the ``RUN:``
-lines, the idiomatic place to put them is in a subdirectory ``Inputs``.
+If your test requires extra files besides the file containing the ``RUN:`` lines
+and the extra files are small, consider specifying them in the same file and
+using ``split-file`` to extract them. For example,
+
+.. code-block:: llvm
+
+  ; RUN: split-file %s %t
+  ; RUN: llvm-link -S %t/a.ll %t/b.ll | FileCheck %s
+
+  ; CHECK: ...
+
+  ;--- a.ll
+  ...
+  ;--- b.ll
+  ...
+
+The parts are separated by the regex ``^(.|//)--- <part>``. By default the
+extracted content has leading empty lines to preserve line numbers. Specify
+``--no-leading-lines`` to drop leading lines.
+
+If the extra files are large, the idiomatic place to put them is in a subdirectory ``Inputs``.
 You can then refer to the extra files as ``%S/Inputs/foo.bar``.
 
 For example, consider ``test/Linker/ident.ll``. The directory structure is
diff --git a/llvm/test/CMakeLists.txt b/llvm/test/CMakeLists.txt
index 91215b3ca0ef0..cde80035a09bf 100644
--- a/llvm/test/CMakeLists.txt
+++ b/llvm/test/CMakeLists.txt
@@ -119,6 +119,7 @@ set(LLVM_TEST_DEPENDS
           opt
           sancov
           sanstats
+          split-file
           verify-uselistorder
           yaml-bench
           yaml2obj
diff --git a/llvm/test/lit.cfg.py b/llvm/test/lit.cfg.py
index 0a3289fcc4ad4..4502ac58c45ac 100644
--- a/llvm/test/lit.cfg.py
+++ b/llvm/test/lit.cfg.py
@@ -141,6 +141,7 @@ def get_asan_rtlib():
     ToolSubst('%llvm-objcopy', FindTool('llvm-objcopy')),
     ToolSubst('%llvm-strip', FindTool('llvm-strip')),
     ToolSubst('%llvm-install-name-tool', FindTool('llvm-install-name-tool')),
+    ToolSubst('%split-file', FindTool('split-file')),
 ]
 
 # FIXME: Why do we have both `lli` and `%lli` that do slightly different things?
diff --git a/llvm/test/tools/gold/X86/multiple-sections.ll b/llvm/test/tools/gold/X86/multiple-sections.ll
index facbd8d992ed7..575fb81fcd6f4 100644
--- a/llvm/test/tools/gold/X86/multiple-sections.ll
+++ b/llvm/test/tools/gold/X86/multiple-sections.ll
@@ -1,10 +1,8 @@
-; RUN: echo ".text.tin" > %t_order_lto.txt
-; RUN: echo ".text._start" >> %t_order_lto.txt
-; RUN: echo ".text.pat" >> %t_order_lto.txt
-; RUN: llvm-as %s -o %t.o
+; RUN: split-file %s %t
+; RUN: llvm-as %t/a.ll -o %t.o
 ; RUN: %gold -plugin %llvmshlibdir/LLVMgold%shlibext \
 ; RUN:     -m elf_x86_64 -o %t.exe %t.o \
-; RUN:     --section-ordering-file=%t_order_lto.txt
+; RUN:     --section-ordering-file=%t/order
 ; RUN: llvm-readelf -s %t.exe | FileCheck %s
 
 ; Check that the order of the sections is tin -> _start -> pat.
@@ -13,6 +11,12 @@
 ; CHECK:      00000000004000b0     1 FUNC    LOCAL  DEFAULT    1 tin
 ; CHECK:      00000000004000c0    15 FUNC    GLOBAL DEFAULT    1 _start
 
+;--- order
+.text.tin
+.text._start
+.text.pat
+
+;--- a.ll
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
diff --git a/llvm/test/tools/llvm-strings/radix.test b/llvm/test/tools/llvm-strings/radix.test
index d23fb3cddc8f8..4dafbd1c84fc9 100644
--- a/llvm/test/tools/llvm-strings/radix.test
+++ b/llvm/test/tools/llvm-strings/radix.test
@@ -1,29 +1,32 @@
 ## Show that llvm-strings can handle the -t/--radix switch properly.
 
-RUN: echo one > %t
-RUN: echo two >> %t
-RUN: echo three >> %t
-RUN: echo four >> %t
-RUN: echo five >> %t
-RUN: echo six >> %t
-RUN: echo seven >> %t
-RUN: echo eight >> %t
-RUN: echo nine >> %t
-RUN: echo ten >> %t
-
-RUN: llvm-strings %t | FileCheck %s -check-prefix CHECK-NONE --implicit-check-not={{.}}
-RUN: llvm-strings -t d %t | FileCheck %s -check-prefix CHECK-DEC --strict-whitespace --implicit-check-not={{.}}
-RUN: llvm-strings -t o %t | FileCheck %s -check-prefix CHECK-OCT --strict-whitespace --implicit-check-not={{.}}
-RUN: llvm-strings -t x %t | FileCheck %s -check-prefix CHECK-HEX --strict-whitespace --implicit-check-not={{.}}
+RUN: split-file --no-leading-lines %s %t
+#--- a.txt
+one
+two
+three
+four
+five
+six
+seven
+eight
+nine
+ten
+#--- end
+
+RUN: llvm-strings %t/a.txt | FileCheck %s -check-prefix CHECK-NONE --implicit-check-not={{.}}
+RUN: llvm-strings -t d %t/a.txt | FileCheck %s -check-prefix CHECK-DEC --strict-whitespace --implicit-check-not={{.}}
+RUN: llvm-strings -t o %t/a.txt | FileCheck %s -check-prefix CHECK-OCT --strict-whitespace --implicit-check-not={{.}}
+RUN: llvm-strings -t x %t/a.txt | FileCheck %s -check-prefix CHECK-HEX --strict-whitespace --implicit-check-not={{.}}
 
 ## Show --radix works too.
-RUN: llvm-strings --radix d %t | FileCheck %s -check-prefix CHECK-DEC --strict-whitespace
-RUN: llvm-strings --radix o %t | FileCheck %s -check-prefix CHECK-OCT --strict-whitespace
-RUN: llvm-strings --radix x %t | FileCheck %s -check-prefix CHECK-HEX --strict-whitespace
+RUN: llvm-strings --radix d %t/a.txt | FileCheck %s -check-prefix CHECK-DEC --strict-whitespace
+RUN: llvm-strings --radix o %t/a.txt | FileCheck %s -check-prefix CHECK-OCT --strict-whitespace
+RUN: llvm-strings --radix x %t/a.txt | FileCheck %s -check-prefix CHECK-HEX --strict-whitespace
 
 ## Show different syntaxes work.
-RUN: llvm-strings --radix=d %t | FileCheck %s -check-prefix CHECK-DEC --strict-whitespace
-RUN: llvm-strings -t=d %t | FileCheck %s -check-prefix CHECK-DEC --strict-whitespace
+RUN: llvm-strings --radix=d %t/a.txt | FileCheck %s -check-prefix CHECK-DEC --strict-whitespace
+RUN: llvm-strings -t=d %t/a.txt | FileCheck %s -check-prefix CHECK-DEC --strict-whitespace
 
 CHECK-NONE: {{^}}three
 CHECK-NONE: {{^}}four
@@ -54,5 +57,5 @@ CHECK-HEX: {{^}}     22 eight
 CHECK-HEX: {{^}}     28 nine
 
 ## Show that an invalid value is rejected.
-RUN: not llvm-strings --radix z %t 2>&1 | FileCheck %s --check-prefix=INVALID
+RUN: not llvm-strings --radix z %t/a.txt 2>&1 | FileCheck %s --check-prefix=INVALID
 INVALID: llvm-strings{{.*}}: for the --radix option: Cannot find option named 'z'!
diff --git a/llvm/test/tools/split-file/Inputs/basic-aa.txt b/llvm/test/tools/split-file/Inputs/basic-aa.txt
new file mode 100644
index 0000000000000..0b9ddeb2fc12a
--- /dev/null
+++ b/llvm/test/tools/split-file/Inputs/basic-aa.txt
@@ -0,0 +1,2 @@
+
+aa
diff --git a/llvm/test/tools/split-file/Inputs/basic-bb.txt b/llvm/test/tools/split-file/Inputs/basic-bb.txt
new file mode 100644
index 0000000000000..0f20b8cf755bc
--- /dev/null
+++ b/llvm/test/tools/split-file/Inputs/basic-bb.txt
@@ -0,0 +1,6 @@
+
+
+
+; Comments are preserved.
+bb
+
diff --git a/llvm/test/tools/split-file/Inputs/basic-cc.txt b/llvm/test/tools/split-file/Inputs/basic-cc.txt
new file mode 100644
index 0000000000000..dc815bf4b7dc2
--- /dev/null
+++ b/llvm/test/tools/split-file/Inputs/basic-cc.txt
@@ -0,0 +1,8 @@
+
+
+
+
+
+
+
+cc
diff --git a/llvm/test/tools/split-file/basic.test b/llvm/test/tools/split-file/basic.test
new file mode 100644
index 0000000000000..5d32c3429ed37
--- /dev/null
+++ b/llvm/test/tools/split-file/basic.test
@@ -0,0 +1,40 @@
+#--- aa
+aa
+;--- bb
+; Comments are preserved.
+bb
+
+//--- subdir/cc
+cc
+//--- end
+
+# RUN: rm -rf %t
+# RUN: split-file %s %t
+# RUN: diff %S/Inputs/basic-aa.txt %t/aa
+# RUN: diff %S/Inputs/basic-bb.txt %t/bb
+# RUN: diff %S/Inputs/basic-cc.txt %t/subdir/cc
+# RUN: FileCheck %s --check-prefix=END < %t/end
+
+## Can be called on a non-empty directory.
+# RUN: split-file %s %t
+# RUN: diff %S/Inputs/basic-aa.txt %t/aa
+
+## Test that we will delete the output if it is a file, so that we can create
+## a directory.
+# RUN: rm -rf %t && touch %t
+# RUN: split-file %s %t
+# RUN: diff %S/Inputs/basic-aa.txt %t/aa
+
+# END: RUN: split-file %s %t
+
+# RUN: not %split-file 2>&1 | FileCheck %s --check-prefix=NO_INPUT
+
+# NO_INPUT: split-file: error: input filename is not specified
+
+# RUN: not %split-file %s '' 2>&1 | FileCheck %s --check-prefix=NO_OUTPUT
+
+# NO_OUTPUT: split-file: error: output directory is not specified
+
+# RUN: not %split-file %S/Inputs/basic-aa.txt %t 2>&1 | FileCheck %s --check-prefix=NOT_EXIST
+
+# NOT_EXIST: split-file: error: {{.*}}.txt: no part separator was found
diff --git a/llvm/test/tools/split-file/empty.test b/llvm/test/tools/split-file/empty.test
new file mode 100644
index 0000000000000..e76bea93bc2e1
--- /dev/null
+++ b/llvm/test/tools/split-file/empty.test
@@ -0,0 +1,4 @@
+# RUN: split-file --no-leading-lines %s %t
+# RUN: count 0 < %t/empty
+
+#--- empty
diff --git a/llvm/test/tools/split-file/error.test b/llvm/test/tools/split-file/error.test
new file mode 100644
index 0000000000000..9efa5adca49ac
--- /dev/null
+++ b/llvm/test/tools/split-file/error.test
@@ -0,0 +1,16 @@
+# RUN: not %split-file %s %t 2>&1 | FileCheck %s
+# RUN: not ls %t/dup
+
+# CHECK: {{.*}}.test:[[#@LINE+1]]: error: empty part name
+//--- 
+
+# CHECK: {{.*}}.test:[[#@LINE+1]]: error: part name cannot have leading or trailing space
+//---  leading_space
+
+# CHECK: {{.*}}.test:[[#@LINE+1]]: error: part name cannot have leading or trailing space
+//--- trailing_space 
+
+;--- dup
+
+# CHECK: {{.*}}.test:[[#@LINE+1]]: error: ';--- dup' occurs more than once
+;--- dup
diff --git a/llvm/test/tools/split-file/help.test b/llvm/test/tools/split-file/help.test
new file mode 100644
index 0000000000000..27c450aeac3a7
--- /dev/null
+++ b/llvm/test/tools/split-file/help.test
@@ -0,0 +1,6 @@
+RUN: split-file --help 2>&1 | FileCheck --implicit-check-not='General Options:' %s
+CHECK: OVERVIEW: Split input {{.*}}
+CHECK: USAGE: split-file [options] filename directory
+CHECK: Generic Options:
+CHECK: split-file Options:
+CHECK:   --no-leading-lines
diff --git a/llvm/test/tools/split-file/no-leading-lines.test b/llvm/test/tools/split-file/no-leading-lines.test
new file mode 100644
index 0000000000000..d4de34f33d1e8
--- /dev/null
+++ b/llvm/test/tools/split-file/no-leading-lines.test
@@ -0,0 +1,10 @@
+## With --no-leading-lines, don't add leading lines (which is used to preserve line numbers).
+
+# RUN: split-file --no-leading-lines %s %t
+# RUN: count 1 < %t/a.txt
+# RUN: FileCheck %s < %t/a.txt
+
+# CHECK: input
+
+#--- a.txt
+input
diff --git a/llvm/test/tools/split-file/output-is-special.test b/llvm/test/tools/split-file/output-is-special.test
new file mode 100644
index 0000000000000..98bb4d36a4ff3
--- /dev/null
+++ b/llvm/test/tools/split-file/output-is-special.test
@@ -0,0 +1,8 @@
+# UNSUPPORTED: system-windows
+# REQUIRES: shell
+
+## Don't delete the output if it is special, otherwise root may accidentally
+## remove important special files.
+# RUN: not split-file %s /dev/null 2>&1 | FileCheck %s
+
+# CHECK: error: /dev/null: output cannot be a special file
diff --git a/llvm/tools/split-file/.clang-tidy b/llvm/tools/split-file/.clang-tidy
new file mode 100644
index 0000000000000..87ec2ff53af6e
--- /dev/null
+++ b/llvm/tools/split-file/.clang-tidy
@@ -0,0 +1,19 @@
+# Almost identical to the top-level .clang-tidy, except that {Member,Parameter,Variable}Case use camelBack.
+Checks: '-*,clang-diagnostic-*,llvm-*,misc-*,-misc-unused-parameters,-misc-non-private-member-variables-in-classes,readability-identifier-naming'
+CheckOptions:
+  - key:             readability-identifier-naming.ClassCase
+    value:           CamelCase
+  - key:             readability-identifier-naming.EnumCase
+    value:           CamelCase
+  - key:             readability-identifier-naming.FunctionCase
+    value:           camelBack
+  - key:             readability-identifier-naming.MemberCase
+    value:           camelBack
+  - key:             readability-identifier-naming.ParameterCase
+    value:           camelBack
+  - key:             readability-identifier-naming.UnionCase
+    value:           CamelCase
+  - key:             readability-identifier-naming.VariableCase
+    value:           camelBack
+  - key:             readability-identifier-naming.IgnoreMainLikeFunctions
+    value:           1
diff --git a/llvm/tools/split-file/CMakeLists.txt b/llvm/tools/split-file/CMakeLists.txt
new file mode 100644
index 0000000000000..ba998483c22aa
--- /dev/null
+++ b/llvm/tools/split-file/CMakeLists.txt
@@ -0,0 +1,7 @@
+set(LLVM_LINK_COMPONENTS
+  Support
+  )
+
+add_llvm_tool(split-file
+  split-file.cpp
+  )
diff --git a/llvm/tools/split-file/split-file.cpp b/llvm/tools/split-file/split-file.cpp
new file mode 100644
index 0000000000000..772a19164dc48
--- /dev/null
+++ b/llvm/tools/split-file/split-file.cpp
@@ -0,0 +1,172 @@
+//===- split-file.cpp - Input splitting utility ---------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Split input into multipe parts separated by regex '^(.|//)--- ' and extract
+// the specified part.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/FileOutputBuffer.h"
+#include "llvm/Support/LineIterator.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/ToolOutputFile.h"
+#include "llvm/Support/WithColor.h"
+#include <string>
+#include <system_error>
+
+using namespace llvm;
+
+static cl::OptionCategory cat("split-file Options");
+
+static cl::opt<std::string> input(cl::Positional, cl::desc("filename"),
+                                  cl::cat(cat));
+
+static cl::opt<std::string> output(cl::Positional, cl::desc("directory"),
+                                   cl::value_desc("directory"), cl::cat(cat));
+
+static cl::opt<bool> noLeadingLines("no-leading-lines",
+                                    cl::desc("Don't preserve line numbers"),
+                                    cl::cat(cat));
+
+static StringRef toolName;
+static int errorCount;
+
+LLVM_ATTRIBUTE_NORETURN static void fatal(StringRef filename,
+                                          const Twine &message) {
+  if (filename.empty())
+    WithColor::error(errs(), toolName) << message << '\n';
+  else
+    WithColor::error(errs(), toolName) << filename << ": " << message << '\n';
+  exit(1);
+}
+
+static void error(StringRef filename, int64_t line, const Twine &message) {
+  ++errorCount;
+  errs() << filename << ':' << line << ": ";
+  WithColor::error(errs()) << message << '\n';
+}
+
+namespace {
+struct Part {
+  const char *begin = nullptr;
+  const char *end = nullptr;
+  int64_t leadingLines = 0;
+};
+} // namespace
+
+static int handle(MemoryBuffer &inputBuf, StringRef input) {
+  DenseMap<StringRef, Part> partToBegin;
+  StringRef lastPart, separator;
+  for (line_iterator i(inputBuf, /*SkipBlanks=*/false, '\0'); !i.is_at_eof();) {
+    const int64_t lineNo = i.line_number();
+    const StringRef line = *i++;
+    const size_t markerLen = line.startswith("//") ? 6 : 5;
+    if (!(line.size() >= markerLen &&
+          line.substr(markerLen - 4).startswith("--- ")))
+      continue;
+    separator = line.substr(0, markerLen);
+    const StringRef partName = line.substr(markerLen);
+    if (partName.empty()) {
+      error(input, lineNo, "empty part name");
+      continue;
+    }
+    if (isSpace(partName.front()) || isSpace(partName.back())) {
+      error(input, lineNo, "part name cannot have leading or trailing space");
+      continue;
+    }
+
+    auto res = partToBegin.try_emplace(partName);
+    if (!res.second) {
+      error(input, lineNo,
+            "'" + separator + partName + "' occurs more than once");
+      continue;
+    }
+    if (!lastPart.empty())
+      partToBegin[lastPart].end = line.data();
+    Part &cur = res.first->second;
+    if (!i.is_at_eof())
+      cur.begin = i->data();
+    // If --no-leading-lines is not specified, numEmptyLines is 0. Append
+    // newlines so that the extracted part preserves line numbers.
+    cur.leadingLines = noLeadingLines ? 0 : i.line_number() - 1;
+
+    lastPart = partName;
+  }
+  if (lastPart.empty())
+    fatal(input, "no part separator was found");
+  if (errorCount)
+    return 1;
+  partToBegin[lastPart].end = inputBuf.getBufferEnd();
+
+  std::vector<std::unique_ptr<ToolOutputFile>> outputFiles;
+  SmallString<256> partPath;
+  for (auto &keyValue : partToBegin) {
+    partPath.clear();
+    sys::path::append(partPath, output, keyValue.first);
+    std::error_code ec =
+        sys::fs::create_directories(sys::path::parent_path(partPath));
+    if (ec)
+      fatal(input, ec.message());
+    auto f = std::make_unique<ToolOutputFile>(partPath.str(), ec,
+                                              llvm::sys::fs::OF_None);
+    if (!f)
+      fatal(input, ec.message());
+
+    Part &part = keyValue.second;
+    for (int64_t i = 0; i != part.leadingLines; ++i)
+      (*f).os().write('\n');
+    if (part.begin)
+      (*f).os().write(part.begin, part.end - part.begin);
+    outputFiles.push_back(std::move(f));
+  }
+
+  for (std::unique_ptr<ToolOutputFile> &outputFile : outputFiles)
+    outputFile->keep();
+  return 0;
+}
+
+int main(int argc, const char **argv) {
+  toolName = sys::path::stem(argv[0]);
+  cl::HideUnrelatedOptions({&cat});
+  cl::ParseCommandLineOptions(
+      argc, argv,
+      "Split input into multiple parts separated by regex '^(.|//)--- ' and "
+      "extract the part specified by '^(.|//)--- <part>'\n",
+      nullptr,
+      /*EnvVar=*/nullptr,
+      /*LongOptionsUseDoubleDash=*/true);
+
+  if (input.empty())
+    fatal("", "input filename is not specified");
+  if (output.empty())
+    fatal("", "output directory is not specified");
+  ErrorOr<std::unique_ptr<MemoryBuffer>> bufferOrErr =
+      MemoryBuffer::getFileOrSTDIN(input);
+  if (std::error_code ec = bufferOrErr.getError())
+    fatal(input, ec.message());
+
+  // Delete output if it is a file or an empty directory, so that we can create
+  // a directory.
+  sys::fs::file_status status;
+  if (std::error_code ec = sys::fs::status(output, status))
+    if (ec.value() != static_cast<int>(std::errc::no_such_file_or_directory))
+      fatal(output, ec.message());
+  if (status.type() != sys::fs::file_type::file_not_found &&
+      status.type() != sys::fs::file_type::directory_file &&
+      status.type() != sys::fs::file_type::regular_file)
+    fatal(output, "output cannot be a special file");
+  if (std::error_code ec = sys::fs::remove(output, /*IgnoreNonExisting=*/true))
+    if (ec.value() != static_cast<int>(std::errc::directory_not_empty))
+      fatal(output, ec.message());
+  return handle(**bufferOrErr, input);
+}
diff --git a/llvm/utils/gn/secondary/lld/test/BUILD.gn b/llvm/utils/gn/secondary/lld/test/BUILD.gn
index 581cc5482578c..bfb63a39ba65a 100644
--- a/llvm/utils/gn/secondary/lld/test/BUILD.gn
+++ b/llvm/utils/gn/secondary/lld/test/BUILD.gn
@@ -94,6 +94,7 @@ group("test") {
     "//llvm/tools/llvm-readobj:symlinks",
     "//llvm/tools/obj2yaml",
     "//llvm/tools/opt",
+    "//llvm/tools/split-file",
     "//llvm/tools/yaml2obj",
     "//llvm/utils/FileCheck",
     "//llvm/utils/count",
diff --git a/llvm/utils/gn/secondary/llvm/test/BUILD.gn b/llvm/utils/gn/secondary/llvm/test/BUILD.gn
index 2c4a23ffbaacb..c714d9b5ba7b1 100644
--- a/llvm/utils/gn/secondary/llvm/test/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/test/BUILD.gn
@@ -260,6 +260,7 @@ group("test") {
     "//llvm/tools/opt",
     "//llvm/tools/sancov",
     "//llvm/tools/sanstats",
+    "//llvm/tools/split-file",
     "//llvm/tools/verify-uselistorder",
     "//llvm/tools/yaml2obj",
     "//llvm/unittests",
diff --git a/llvm/utils/gn/secondary/llvm/tools/split-file/BUILD.gn b/llvm/utils/gn/secondary/llvm/tools/split-file/BUILD.gn
new file mode 100644
index 0000000000000..4bf9269c3c38d
--- /dev/null
+++ b/llvm/utils/gn/secondary/llvm/tools/split-file/BUILD.gn
@@ -0,0 +1,4 @@
+executable("split-file") {
+  deps = [ "//llvm/lib/Support" ]
+  sources = [ "split-file.cpp" ]
+}

From 57899934eab18bbcab3482cc3ef862b0a1617ad0 Mon Sep 17 00:00:00 2001
From: Carl Ritson <carl.ritson@amd.com>
Date: Tue, 4 Aug 2020 12:20:12 +0900
Subject: [PATCH 0985/1035] [AMDGPU] Make GCNRegBankReassign assign based on
 subreg banks

When scavenging consider the sub-register of the source operand
to determine the bank of a candidate register (not just sub0).
Without this it is possible to introduce an infinite loop,
e.g. $sgpr15_sgpr16_sgpr17 can be assigned for a conflict between
$sgpr0 and SGPR_96:sub1.

Reviewed By: rampitec

Differential Revision: https://reviews.llvm.org/D84910
---
 llvm/lib/Target/AMDGPU/GCNRegBankReassign.cpp | 97 ++++++++++++-------
 .../AMDGPU/GlobalISel/insertelement.ll        | 74 +++++++-------
 .../AMDGPU/regbank-reassign-wave64.mir        | 69 +++++++++++++
 llvm/test/CodeGen/AMDGPU/regbank-reassign.mir | 78 +++++++++++++++
 4 files changed, 245 insertions(+), 73 deletions(-)
 create mode 100644 llvm/test/CodeGen/AMDGPU/regbank-reassign-wave64.mir

diff --git a/llvm/lib/Target/AMDGPU/GCNRegBankReassign.cpp b/llvm/lib/Target/AMDGPU/GCNRegBankReassign.cpp
index 98d971630ca4f..79b33e24c8302 100644
--- a/llvm/lib/Target/AMDGPU/GCNRegBankReassign.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNRegBankReassign.cpp
@@ -83,9 +83,10 @@ class GCNRegBankReassign : public MachineFunctionPass {
 
   class Candidate {
   public:
-    Candidate(MachineInstr *mi, unsigned reg, unsigned freebanks,
-              unsigned weight)
-      : MI(mi), Reg(reg), FreeBanks(freebanks), Weight(weight) {}
+    Candidate(MachineInstr *mi, unsigned reg, unsigned subreg,
+              unsigned freebanks, unsigned weight)
+        : MI(mi), Reg(reg), SubReg(subreg), FreeBanks(freebanks),
+          Weight(weight) {}
 
     bool operator< (const Candidate& RHS) const { return Weight < RHS.Weight; }
 
@@ -100,6 +101,7 @@ class GCNRegBankReassign : public MachineFunctionPass {
 
     MachineInstr *MI;
     unsigned Reg;
+    unsigned SubReg;
     unsigned FreeBanks;
     unsigned Weight;
   };
@@ -162,7 +164,7 @@ class GCNRegBankReassign : public MachineFunctionPass {
   const MCPhysReg *CSRegs;
 
   // Returns bank for a phys reg.
-  unsigned getPhysRegBank(unsigned Reg) const;
+  unsigned getPhysRegBank(unsigned Reg, unsigned SubReg) const;
 
   // Return a bit set for each register bank used. 4 banks for VGPRs and
   // 8 banks for SGPRs.
@@ -176,7 +178,7 @@ class GCNRegBankReassign : public MachineFunctionPass {
   // a register chosen from Bank.
   std::pair<unsigned, unsigned> analyzeInst(const MachineInstr &MI,
                                             unsigned Reg = AMDGPU::NoRegister,
-                                            int Bank = -1);
+                                            unsigned SubReg = 0, int Bank = -1);
 
   // Return true if register is regular VGPR or SGPR or their tuples.
   // Returns false for special registers like m0, vcc etc.
@@ -216,11 +218,12 @@ class GCNRegBankReassign : public MachineFunctionPass {
   // candidates are collected and added to work list.
   unsigned computeStallCycles(unsigned SrcReg,
                               unsigned Reg = AMDGPU::NoRegister,
-                              int Bank = -1, bool Collect = false);
+                              unsigned SubReg = 0, int Bank = -1,
+                              bool Collect = false);
 
   // Search for a register in Bank unused within LI.
   // Returns phys reg or NoRegister.
-  unsigned scavengeReg(LiveInterval& LI, unsigned Bank) const;
+  unsigned scavengeReg(LiveInterval &LI, unsigned Bank, unsigned SubReg) const;
 
   // Try to reassign candidate. Returns number or stall cycles saved.
   unsigned tryReassign(Candidate &C);
@@ -277,15 +280,24 @@ char GCNRegBankReassign::ID = 0;
 
 char &llvm::GCNRegBankReassignID = GCNRegBankReassign::ID;
 
-unsigned GCNRegBankReassign::getPhysRegBank(unsigned Reg) const {
+unsigned GCNRegBankReassign::getPhysRegBank(unsigned Reg,
+                                            unsigned SubReg) const {
   assert(Register::isPhysicalRegister(Reg));
 
   const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg);
   unsigned Size = TRI->getRegSizeInBits(*RC);
   if (Size == 16)
     Reg = TRI->get32BitRegister(Reg);
-  else if (Size > 32)
-    Reg = TRI->getSubReg(Reg, AMDGPU::sub0);
+  else if (Size > 32) {
+    if (SubReg) {
+      const TargetRegisterClass *SubRC = TRI->getSubRegClass(RC, SubReg);
+      Reg = TRI->getSubReg(Reg, SubReg);
+      if (TRI->getRegSizeInBits(*SubRC) > 32)
+        Reg = TRI->getSubReg(Reg, AMDGPU::sub0);
+    } else {
+      Reg = TRI->getSubReg(Reg, AMDGPU::sub0);
+    }
+  }
 
   if (TRI->hasVGPRs(RC)) {
     Reg -= AMDGPU::VGPR0;
@@ -360,7 +372,7 @@ uint32_t GCNRegBankReassign::getRegBankMask(unsigned Reg, unsigned SubReg,
 
 std::pair<unsigned, unsigned>
 GCNRegBankReassign::analyzeInst(const MachineInstr &MI, unsigned Reg,
-                                int Bank) {
+                                unsigned SubReg, int Bank) {
   unsigned StallCycles = 0;
   unsigned UsedBanks = 0;
 
@@ -375,26 +387,39 @@ GCNRegBankReassign::analyzeInst(const MachineInstr &MI, unsigned Reg,
     if (!Op.isReg() || Op.isUndef())
       continue;
 
-    Register R = Op.getReg();
-    if (TRI->hasAGPRs(TRI->getRegClassForReg(*MRI, R)))
-      continue;
+    const Register R = Op.getReg();
+    const TargetRegisterClass *RC = TRI->getRegClassForReg(*MRI, R);
 
-    unsigned ShiftedBank = Bank;
+    // Do not compute stalls for AGPRs
+    if (TRI->hasAGPRs(RC))
+      continue;
 
-    if (Bank != -1 && R == Reg && Op.getSubReg()) {
-      unsigned Offset = TRI->getChannelFromSubReg(Op.getSubReg());
+    // Do not compute stalls if sub-register covers all banks
+    if (Op.getSubReg()) {
       LaneBitmask LM = TRI->getSubRegIndexLaneMask(Op.getSubReg());
-      if (Offset && Bank < NUM_VGPR_BANKS) {
-        // If a register spans all banks we cannot shift it to avoid conflict.
+      if (TRI->hasVGPRs(RC)) {
         if (TRI->getNumCoveredRegs(LM) >= NUM_VGPR_BANKS)
           continue;
-        ShiftedBank = (Bank + Offset) % NUM_VGPR_BANKS;
-      } else if (Offset > 1 && Bank >= SGPR_BANK_OFFSET) {
-        // If a register spans all banks we cannot shift it to avoid conflict.
+      } else {
         if (TRI->getNumCoveredRegs(LM) / 2 >= NUM_SGPR_BANKS)
           continue;
+      }
+    }
+
+    unsigned ShiftedBank = Bank;
+
+    if (Bank != -1 && R == Reg && (Op.getSubReg() || SubReg)) {
+      unsigned RegOffset =
+          TRI->getChannelFromSubReg(SubReg ? SubReg : (unsigned)AMDGPU::sub0);
+      unsigned Offset = TRI->getChannelFromSubReg(
+          Op.getSubReg() ? Op.getSubReg() : (unsigned)AMDGPU::sub0);
+      if (Bank < NUM_VGPR_BANKS) {
+        unsigned Shift = ((NUM_VGPR_BANKS + Offset) - RegOffset);
+        ShiftedBank = (Bank + Shift) % NUM_VGPR_BANKS;
+      } else if (Bank >= SGPR_BANK_OFFSET) {
+        unsigned Shift = (NUM_SGPR_BANKS + (Offset >> 1)) - (RegOffset >> 1);
         ShiftedBank = SGPR_BANK_OFFSET +
-          (Bank - SGPR_BANK_OFFSET + (Offset >> 1)) % NUM_SGPR_BANKS;
+                      (Bank - SGPR_BANK_OFFSET + Shift) % NUM_SGPR_BANKS;
       }
     }
 
@@ -576,17 +601,17 @@ void GCNRegBankReassign::collectCandidates(MachineInstr& MI,
       unsigned FreeBanks1 = getFreeBanks(Reg1, SubReg1, Mask1, UsedBanks);
       unsigned FreeBanks2 = getFreeBanks(Reg2, SubReg2, Mask2, UsedBanks);
       if (FreeBanks1)
-        Candidates.push(Candidate(&MI, Reg1, FreeBanks1, Weight
-                                    + ((Size2 > Size1) ? 1 : 0)));
+        Candidates.push(Candidate(&MI, Reg1, SubReg1, FreeBanks1,
+                                  Weight + ((Size2 > Size1) ? 1 : 0)));
       if (FreeBanks2)
-        Candidates.push(Candidate(&MI, Reg2, FreeBanks2, Weight
-                                    + ((Size1 > Size2) ? 1 : 0)));
+        Candidates.push(Candidate(&MI, Reg2, SubReg2, FreeBanks2,
+                                  Weight + ((Size1 > Size2) ? 1 : 0)));
     }
   }
 }
 
-unsigned GCNRegBankReassign::computeStallCycles(unsigned SrcReg,
-                                                unsigned Reg, int Bank,
+unsigned GCNRegBankReassign::computeStallCycles(unsigned SrcReg, unsigned Reg,
+                                                unsigned SubReg, int Bank,
                                                 bool Collect) {
   unsigned TotalStallCycles = 0;
   SmallSet<const MachineInstr *, 16> Visited;
@@ -598,7 +623,7 @@ unsigned GCNRegBankReassign::computeStallCycles(unsigned SrcReg,
       continue;
     unsigned StallCycles;
     unsigned UsedBanks;
-    std::tie(StallCycles, UsedBanks) = analyzeInst(MI, Reg, Bank);
+    std::tie(StallCycles, UsedBanks) = analyzeInst(MI, Reg, SubReg, Bank);
     TotalStallCycles += StallCycles;
     if (Collect)
       collectCandidates(MI, UsedBanks, StallCycles);
@@ -607,8 +632,8 @@ unsigned GCNRegBankReassign::computeStallCycles(unsigned SrcReg,
   return TotalStallCycles;
 }
 
-unsigned GCNRegBankReassign::scavengeReg(LiveInterval& LI,
-                                         unsigned Bank) const {
+unsigned GCNRegBankReassign::scavengeReg(LiveInterval &LI, unsigned Bank,
+                                         unsigned SubReg) const {
   const TargetRegisterClass *RC = MRI->getRegClass(LI.reg);
   unsigned MaxNumRegs = (Bank < NUM_VGPR_BANKS) ? MaxNumVGPRs
                                                 : MaxNumSGPRs;
@@ -620,7 +645,7 @@ unsigned GCNRegBankReassign::scavengeReg(LiveInterval& LI,
     if (TRI->isSubRegisterEq(Reg, MaxReg))
       break;
 
-    if (!MRI->isAllocatable(Reg) || getPhysRegBank(Reg) != Bank)
+    if (!MRI->isAllocatable(Reg) || getPhysRegBank(Reg, SubReg) != Bank)
       continue;
 
     for (unsigned I = 0; CSRegs[I]; ++I)
@@ -669,7 +694,7 @@ unsigned GCNRegBankReassign::tryReassign(Candidate &C) {
   for (int Bank = 0; Bank < NUM_BANKS; ++Bank) {
     if (C.FreeBanks & (1 << Bank)) {
       LLVM_DEBUG(dbgs() << "Trying bank " << printBank(Bank) << '\n');
-      unsigned Stalls = computeStallCycles(C.Reg, C.Reg, Bank);
+      unsigned Stalls = computeStallCycles(C.Reg, C.Reg, C.SubReg, Bank);
       if (Stalls < OrigStalls) {
         LLVM_DEBUG(dbgs() << "With bank " << printBank(Bank) << " -> "
                      << Stalls << '\n');
@@ -683,7 +708,7 @@ unsigned GCNRegBankReassign::tryReassign(Candidate &C) {
   LRM->unassign(LI);
   while (!BankStalls.empty()) {
     BankStall BS = BankStalls.pop_back_val();
-    unsigned Reg = scavengeReg(LI, BS.Bank);
+    unsigned Reg = scavengeReg(LI, BS.Bank, C.SubReg);
     if (Reg == AMDGPU::NoRegister) {
       LLVM_DEBUG(dbgs() << "No free registers in bank " << printBank(BS.Bank)
                    << '\n');
@@ -801,7 +826,7 @@ bool GCNRegBankReassign::runOnMachineFunction(MachineFunction &MF) {
     Candidates.pop_back();
     if (LocalCyclesSaved) {
       removeCandidates(C.Reg);
-      computeStallCycles(C.Reg, AMDGPU::NoRegister, -1, true);
+      computeStallCycles(C.Reg, AMDGPU::NoRegister, 0, -1, true);
       Candidates.sort();
 
       LLVM_DEBUG(dbgs() << "\nCandidates:\n\n";
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.ll
index 8e4a071701b35..b4afc48f98ba8 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.ll
@@ -1492,7 +1492,7 @@ define amdgpu_ps void @dyn_insertelement_v8f64_v_v_v(<8 x double> %vec, double %
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 s4, 5, v18
 ; MOVREL-NEXT:    v_mov_b32_e32 v19, v0
 ; MOVREL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v18
-; MOVREL-NEXT:    v_mov_b32_e32 v20, v1
+; MOVREL-NEXT:    v_mov_b32_e32 v23, v1
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 s1, 2, v18
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 s2, 3, v18
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 s5, 7, v18
@@ -1501,7 +1501,7 @@ define amdgpu_ps void @dyn_insertelement_v8f64_v_v_v(<8 x double> %vec, double %
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v2, v2, v16, s0
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v8, v8, v16, s3
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v10, v10, v16, s4
-; MOVREL-NEXT:    v_cndmask_b32_e32 v1, v20, v17, vcc_lo
+; MOVREL-NEXT:    v_cndmask_b32_e32 v1, v23, v17, vcc_lo
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v3, v3, v17, s0
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v9, v9, v17, s3
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v11, v11, v17, s4
@@ -2123,7 +2123,7 @@ define amdgpu_ps void @dyn_insertelement_v8f64_v_v_v_add_1(<8 x double> %vec, do
 ; MOVREL:       ; %bb.0: ; %entry
 ; MOVREL-NEXT:    v_add_nc_u32_e32 v18, 1, v18
 ; MOVREL-NEXT:    v_mov_b32_e32 v19, v0
-; MOVREL-NEXT:    v_mov_b32_e32 v20, v1
+; MOVREL-NEXT:    v_mov_b32_e32 v23, v1
 ; MOVREL-NEXT:    ; implicit-def: $vcc_hi
 ; MOVREL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v18
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 s0, 1, v18
@@ -2137,7 +2137,7 @@ define amdgpu_ps void @dyn_insertelement_v8f64_v_v_v_add_1(<8 x double> %vec, do
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v2, v2, v16, s0
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v8, v8, v16, s3
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v10, v10, v16, s4
-; MOVREL-NEXT:    v_cndmask_b32_e32 v1, v20, v17, vcc_lo
+; MOVREL-NEXT:    v_cndmask_b32_e32 v1, v23, v17, vcc_lo
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v3, v3, v17, s0
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v9, v9, v17, s3
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v11, v11, v17, s4
@@ -4111,7 +4111,7 @@ define amdgpu_ps <7 x double> @dyn_insertelement_v7f64_v_v_v(<7 x double> %vec,
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 s3, 4, v16
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 s4, 5, v16
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 s5, 6, v16
-; MOVREL-NEXT:    v_mov_b32_e32 v17, v2
+; MOVREL-NEXT:    v_mov_b32_e32 v19, v2
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 s0, 1, v16
 ; MOVREL-NEXT:    v_mov_b32_e32 v18, v3
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v6, v6, v14, s2
@@ -4119,7 +4119,7 @@ define amdgpu_ps <7 x double> @dyn_insertelement_v7f64_v_v_v(<7 x double> %vec,
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v10, v10, v14, s4
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v12, v12, v14, s5
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v7, v7, v15, s2
-; MOVREL-NEXT:    v_cndmask_b32_e64 v2, v17, v14, s0
+; MOVREL-NEXT:    v_cndmask_b32_e64 v2, v19, v14, s0
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v9, v9, v15, s3
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v3, v18, v15, s0
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v0, v0, v14, vcc_lo
@@ -4251,42 +4251,42 @@ define amdgpu_ps <5 x double> @dyn_insertelement_v5f64_s_v_s(<5 x double> inreg
 ; MOVREL-NEXT:    s_mov_b32 s7, s9
 ; MOVREL-NEXT:    s_mov_b32 s8, s10
 ; MOVREL-NEXT:    s_mov_b32 s9, s11
-; MOVREL-NEXT:    v_mov_b32_e32 v18, s15
-; MOVREL-NEXT:    v_mov_b32_e32 v17, s14
-; MOVREL-NEXT:    v_mov_b32_e32 v16, s13
-; MOVREL-NEXT:    v_mov_b32_e32 v15, s12
-; MOVREL-NEXT:    v_mov_b32_e32 v14, s11
-; MOVREL-NEXT:    v_mov_b32_e32 v13, s10
-; MOVREL-NEXT:    v_mov_b32_e32 v12, s9
-; MOVREL-NEXT:    v_mov_b32_e32 v11, s8
-; MOVREL-NEXT:    v_mov_b32_e32 v10, s7
-; MOVREL-NEXT:    v_mov_b32_e32 v9, s6
-; MOVREL-NEXT:    v_mov_b32_e32 v8, s5
-; MOVREL-NEXT:    v_mov_b32_e32 v7, s4
-; MOVREL-NEXT:    v_mov_b32_e32 v6, s3
-; MOVREL-NEXT:    v_mov_b32_e32 v5, s2
-; MOVREL-NEXT:    v_mov_b32_e32 v4, s1
-; MOVREL-NEXT:    v_mov_b32_e32 v3, s0
+; MOVREL-NEXT:    v_mov_b32_e32 v20, s15
+; MOVREL-NEXT:    v_mov_b32_e32 v19, s14
+; MOVREL-NEXT:    v_mov_b32_e32 v18, s13
+; MOVREL-NEXT:    v_mov_b32_e32 v17, s12
+; MOVREL-NEXT:    v_mov_b32_e32 v16, s11
+; MOVREL-NEXT:    v_mov_b32_e32 v15, s10
+; MOVREL-NEXT:    v_mov_b32_e32 v14, s9
+; MOVREL-NEXT:    v_mov_b32_e32 v13, s8
+; MOVREL-NEXT:    v_mov_b32_e32 v12, s7
+; MOVREL-NEXT:    v_mov_b32_e32 v11, s6
+; MOVREL-NEXT:    v_mov_b32_e32 v10, s5
+; MOVREL-NEXT:    v_mov_b32_e32 v9, s4
+; MOVREL-NEXT:    v_mov_b32_e32 v8, s3
+; MOVREL-NEXT:    v_mov_b32_e32 v7, s2
+; MOVREL-NEXT:    v_mov_b32_e32 v6, s1
+; MOVREL-NEXT:    v_mov_b32_e32 v5, s0
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 vcc_lo, s12, 0
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 s0, s12, 1
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 s1, s12, 4
 ; MOVREL-NEXT:    ; implicit-def: $vcc_hi
-; MOVREL-NEXT:    v_cndmask_b32_e32 v2, v3, v0, vcc_lo
-; MOVREL-NEXT:    v_cndmask_b32_e32 v3, v4, v1, vcc_lo
-; MOVREL-NEXT:    v_cndmask_b32_e64 v4, v5, v0, s0
-; MOVREL-NEXT:    v_cndmask_b32_e64 v5, v6, v1, s0
+; MOVREL-NEXT:    v_cndmask_b32_e32 v2, v5, v0, vcc_lo
+; MOVREL-NEXT:    v_cndmask_b32_e32 v3, v6, v1, vcc_lo
+; MOVREL-NEXT:    v_cndmask_b32_e64 v4, v7, v0, s0
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 vcc_lo, s12, 2
+; MOVREL-NEXT:    v_cndmask_b32_e64 v5, v8, v1, s0
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 s0, s12, 3
 ; MOVREL-NEXT:    v_readfirstlane_b32 s2, v4
-; MOVREL-NEXT:    v_readfirstlane_b32 s3, v5
-; MOVREL-NEXT:    v_cndmask_b32_e32 v6, v7, v0, vcc_lo
-; MOVREL-NEXT:    v_cndmask_b32_e32 v7, v8, v1, vcc_lo
-; MOVREL-NEXT:    v_cndmask_b32_e64 v8, v9, v0, s0
-; MOVREL-NEXT:    v_cndmask_b32_e64 v9, v10, v1, s0
-; MOVREL-NEXT:    v_cndmask_b32_e64 v0, v11, v0, s1
-; MOVREL-NEXT:    v_cndmask_b32_e64 v1, v12, v1, s1
+; MOVREL-NEXT:    v_cndmask_b32_e32 v6, v9, v0, vcc_lo
+; MOVREL-NEXT:    v_cndmask_b32_e32 v7, v10, v1, vcc_lo
+; MOVREL-NEXT:    v_cndmask_b32_e64 v8, v11, v0, s0
+; MOVREL-NEXT:    v_cndmask_b32_e64 v9, v12, v1, s0
+; MOVREL-NEXT:    v_cndmask_b32_e64 v0, v13, v0, s1
+; MOVREL-NEXT:    v_cndmask_b32_e64 v1, v14, v1, s1
 ; MOVREL-NEXT:    v_readfirstlane_b32 s0, v2
 ; MOVREL-NEXT:    v_readfirstlane_b32 s1, v3
+; MOVREL-NEXT:    v_readfirstlane_b32 s3, v5
 ; MOVREL-NEXT:    v_readfirstlane_b32 s4, v6
 ; MOVREL-NEXT:    v_readfirstlane_b32 s5, v7
 ; MOVREL-NEXT:    v_readfirstlane_b32 s6, v8
@@ -4448,7 +4448,7 @@ define amdgpu_ps <5 x double> @dyn_insertelement_v5f64_v_v_s(<5 x double> %vec,
 ; MOVREL-LABEL: dyn_insertelement_v5f64_v_v_s:
 ; MOVREL:       ; %bb.0: ; %entry
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 vcc_lo, s2, 0
-; MOVREL-NEXT:    v_mov_b32_e32 v13, v2
+; MOVREL-NEXT:    v_mov_b32_e32 v15, v2
 ; MOVREL-NEXT:    v_mov_b32_e32 v14, v3
 ; MOVREL-NEXT:    ; implicit-def: $vcc_hi
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v0, v0, v10, vcc_lo
@@ -4457,7 +4457,7 @@ define amdgpu_ps <5 x double> @dyn_insertelement_v5f64_v_v_s(<5 x double> %vec,
 ; MOVREL-NEXT:    v_readfirstlane_b32 s0, v0
 ; MOVREL-NEXT:    v_readfirstlane_b32 s1, v1
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v3, v14, v11, vcc_lo
-; MOVREL-NEXT:    v_cndmask_b32_e32 v2, v13, v10, vcc_lo
+; MOVREL-NEXT:    v_cndmask_b32_e32 v2, v15, v10, vcc_lo
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 vcc_lo, s2, 2
 ; MOVREL-NEXT:    v_readfirstlane_b32 s3, v3
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v4, v4, v10, vcc_lo
@@ -4514,7 +4514,7 @@ define amdgpu_ps <5 x double> @dyn_insertelement_v5f64_v_v_v(<5 x double> %vec,
 ; MOVREL-LABEL: dyn_insertelement_v5f64_v_v_v:
 ; MOVREL:       ; %bb.0: ; %entry
 ; MOVREL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v12
-; MOVREL-NEXT:    v_mov_b32_e32 v13, v2
+; MOVREL-NEXT:    v_mov_b32_e32 v15, v2
 ; MOVREL-NEXT:    v_mov_b32_e32 v14, v3
 ; MOVREL-NEXT:    ; implicit-def: $vcc_hi
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v0, v0, v10, vcc_lo
@@ -4522,7 +4522,7 @@ define amdgpu_ps <5 x double> @dyn_insertelement_v5f64_v_v_v(<5 x double> %vec,
 ; MOVREL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v12
 ; MOVREL-NEXT:    v_readfirstlane_b32 s0, v0
 ; MOVREL-NEXT:    v_readfirstlane_b32 s1, v1
-; MOVREL-NEXT:    v_cndmask_b32_e32 v2, v13, v10, vcc_lo
+; MOVREL-NEXT:    v_cndmask_b32_e32 v2, v15, v10, vcc_lo
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v3, v14, v11, vcc_lo
 ; MOVREL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 2, v12
 ; MOVREL-NEXT:    v_readfirstlane_b32 s2, v2
diff --git a/llvm/test/CodeGen/AMDGPU/regbank-reassign-wave64.mir b/llvm/test/CodeGen/AMDGPU/regbank-reassign-wave64.mir
new file mode 100644
index 0000000000000..49d6a9ad19717
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/regbank-reassign-wave64.mir
@@ -0,0 +1,69 @@
+# RUN: llc -march=amdgcn -mcpu=gfx1010 -mattr=-WavefrontSize32,+WavefrontSize64 -verify-machineinstrs -run-pass greedy,amdgpu-regbanks-reassign,virtregrewriter -o - %s | FileCheck -check-prefix=GCN %s
+
+
+# Test that subreg reassignments are correctly handled when whole register also
+# conflicts.  If this is mishandled stall counts will be incorrect and cause an
+# infinite loop.
+# GCN-LABEL: vgpr64_mixed_use{{$}}
+# GCN: $vgpr0_vgpr1 = IMPLICIT_DEF
+# GCN: $vgpr4_vgpr5 = IMPLICIT_DEF
+# GCN: $vcc = IMPLICIT_DEF
+# GCN: $vgpr2_vgpr3 = IMPLICIT_DEF
+# GCN: $vgpr6_vgpr7 = IMPLICIT_DEF
+# GCN: $vgpr8_vgpr9_vgpr10_vgpr11 = IMPLICIT_DEF
+# GCN: $vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+# GCN: $vgpr16_vgpr17_vgpr18_vgpr19 = IMPLICIT_DEF
+# GCN: $vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+# GCN: $vgpr24_vgpr25_vgpr26_vgpr27 = IMPLICIT_DEF
+# GCN: $vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+# GCN: $vgpr32_vgpr33_vgpr34_vgpr35 = IMPLICIT_DEF
+# GCN: $vgpr36_vgpr37_vgpr38_vgpr39 = IMPLICIT_DEF
+# GCN: $vgpr40_vgpr41_vgpr42_vgpr43 = IMPLICIT_DEF
+# GCN: $vgpr44_vgpr45_vgpr46_vgpr47 = IMPLICIT_DEF
+# GCN: $vgpr2 = V_CNDMASK_B32_e64 0, $vgpr1, 0, $vgpr5, $vcc, implicit $exec
+# GCN: $vgpr2 = V_CNDMASK_B32_e64 0, $vgpr0, 0, $vgpr4, killed $vcc, implicit $exec
+# GCN: $sgpr0_sgpr1 = V_CMP_LT_U64_e64 $vgpr4_vgpr5, $vgpr0_vgpr1, implicit $exec
+---
+name:            vgpr64_mixed_use
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: vreg_64, preferred-register: '$vgpr0_vgpr1' }
+  - { id: 1, class: vreg_64, preferred-register: '$vgpr4_vgpr5' }
+  - { id: 2, class: sreg_64_xexec, preferred-register: '$vcc' }
+  - { id: 3, class: vgpr_32 }
+  - { id: 4, class: vgpr_32 }
+  - { id: 5, class: sreg_64_xexec }
+  - { id: 6, class: vreg_64, preferred-register: '$vgpr2_vgpr3' }
+  - { id: 7, class: vreg_64, preferred-register: '$vgpr6_vgpr7' }
+  - { id: 8, class: vreg_128, preferred-register: '$vgpr8_vgpr9_vgpr10_vgpr11' }
+  - { id: 9, class: vreg_128, preferred-register: '$vgpr12_vgpr13_vgpr14_vgpr15' }
+  - { id: 10, class: vreg_128, preferred-register: '$vgpr16_vgpr17_vgpr18_vgpr19' }
+  - { id: 11, class: vreg_128, preferred-register: '$vgpr20_vgpr21_vgpr22_vgpr23' }
+  - { id: 12, class: vreg_128, preferred-register: '$vgpr24_vgpr25_vgpr26_vgpr27' }
+  - { id: 13, class: vreg_128, preferred-register: '$vgpr28_vgpr29_vgpr30_vgpr31' }
+  - { id: 14, class: vreg_128, preferred-register: '$vgpr32_vgpr33_vgpr34_vgpr35' }
+  - { id: 15, class: vreg_128, preferred-register: '$vgpr36_vgpr37_vgpr38_vgpr39' }
+  - { id: 16, class: vreg_128, preferred-register: '$vgpr40_vgpr41_vgpr42_vgpr43' }
+  - { id: 17, class: vreg_128, preferred-register: '$vgpr44_vgpr45_vgpr46_vgpr47' }
+body: |
+  bb.0:
+    %0 = IMPLICIT_DEF
+    %1 = IMPLICIT_DEF
+    %2 = IMPLICIT_DEF
+    %6 = IMPLICIT_DEF
+    %7 = IMPLICIT_DEF
+    %8 = IMPLICIT_DEF
+    %9 = IMPLICIT_DEF
+    %10 = IMPLICIT_DEF
+    %11 = IMPLICIT_DEF
+    %12 = IMPLICIT_DEF
+    %13 = IMPLICIT_DEF
+    %14 = IMPLICIT_DEF
+    %15 = IMPLICIT_DEF
+    %16 = IMPLICIT_DEF
+    %17 = IMPLICIT_DEF
+    %3 = V_CNDMASK_B32_e64 0, %0.sub1, 0, %1.sub1, %2, implicit $exec
+    %4 = V_CNDMASK_B32_e64 0, %0.sub0, 0, %1.sub0, %2, implicit $exec
+    %5 = V_CMP_LT_U64_e64 %1, %0, implicit $exec
+    S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/regbank-reassign.mir b/llvm/test/CodeGen/AMDGPU/regbank-reassign.mir
index 0020e17a0b6fe..2078d8c22922b 100644
--- a/llvm/test/CodeGen/AMDGPU/regbank-reassign.mir
+++ b/llvm/test/CodeGen/AMDGPU/regbank-reassign.mir
@@ -494,3 +494,81 @@ body: |
     %2 = V_AND_B32_e32 %1, %0, implicit $exec
     S_ENDPGM 0
 ...
+
+# Test that bank of subreg is considered during scavenging.
+# If handled incorrectly an infinite loop occurs.
+# GCN-LABEL: s0_vs_s15_16_17_sub1{{$}}
+# GCN: S_AND_B32 renamable $sgpr13, $sgpr0,
+---
+name:            s0_vs_s15_16_17_sub1
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: sgpr_96, preferred-register: '$sgpr15_sgpr16_sgpr17' }
+  - { id: 1, class: sgpr_32 }
+body: |
+  bb.0:
+    %0 = IMPLICIT_DEF
+    $sgpr0 = IMPLICIT_DEF
+    %1 = S_AND_B32 %0.sub1, $sgpr0, implicit-def $scc
+    S_ENDPGM 0
+...
+
+# Test that the size of subreg is correctly handled in bank calculation.
+# If handled incorrectly an infinite loop occurs.
+# GCN-LABEL: vgpr_sub_dependence{{$}}
+# GCN: $vgpr9_vgpr10_vgpr11_vgpr12 = IMPLICIT_DEF
+# GCN: $vgpr16_vgpr17 = IMPLICIT_DEF
+# GCN: $vgpr14_vgpr15 = IMPLICIT_DEF
+# GCN: $vgpr0_vgpr1 = IMPLICIT_DEF
+# GCN: $vgpr7_vgpr8 = IMPLICIT_DEF
+# GCN: $vgpr3_vgpr4_vgpr5_vgpr6 = IMPLICIT_DEF
+# GCN: $vgpr18_vgpr19 = IMPLICIT_DEF
+# GCN: $vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+# GCN: $vgpr24_vgpr25_vgpr26_vgpr27 = IMPLICIT_DEF
+# GCN: $vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+# GCN: $vgpr32_vgpr33_vgpr34_vgpr35 = IMPLICIT_DEF
+# GCN: $vgpr36_vgpr37_vgpr38_vgpr39 = IMPLICIT_DEF
+# GCN: $vgpr40_vgpr41_vgpr42_vgpr43 = IMPLICIT_DEF
+# GCN: $vgpr44_vgpr45_vgpr46_vgpr47 = IMPLICIT_DEF
+# GCN: $vgpr0_vgpr1 = V_ADD_F64 0, $vgpr11_vgpr12, 0, killed $vgpr16_vgpr17, 0, 0, implicit $mode, implicit $exec
+# GCN: $vgpr0_vgpr1 = V_ADD_F64 0, $vgpr9_vgpr10, 0, killed $vgpr14_vgpr15, 0, 0, implicit $mode, implicit $exec
+---
+name:            vgpr_sub_dependence
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: vreg_128, preferred-register: '$vgpr10_vgpr11_vgpr12_vgpr13' }
+  - { id: 1, class: vreg_64, preferred-register: '$vgpr16_vgpr17' }
+  - { id: 2, class: vreg_64, preferred-register: '$vgpr14_vgpr15' }
+  - { id: 3, class: vreg_64 }
+  - { id: 4, class: vreg_64 }
+  - { id: 5, class: vreg_64, preferred-register: '$vgpr0_vgpr1' }
+  - { id: 6, class: vreg_64, preferred-register: '$vgpr7_vgpr8' }
+  - { id: 7, class: vreg_128, preferred-register: '$vgpr3_vgpr4_vgpr5_vgpr6' }
+  - { id: 8, class: vreg_64, preferred-register: '$vgpr18_vgpr19' }
+  - { id: 9, class: vreg_128, preferred-register: '$vgpr20_vgpr21_vgpr22_vgpr23' }
+  - { id: 10, class: vreg_128, preferred-register: '$vgpr24_vgpr25_vgpr26_vgpr27' }
+  - { id: 11, class: vreg_128, preferred-register: '$vgpr28_vgpr29_vgpr30_vgpr31' }
+  - { id: 12, class: vreg_128, preferred-register: '$vgpr32_vgpr33_vgpr34_vgpr35' }
+  - { id: 13, class: vreg_128, preferred-register: '$vgpr36_vgpr37_vgpr38_vgpr39' }
+  - { id: 14, class: vreg_128, preferred-register: '$vgpr40_vgpr41_vgpr42_vgpr43' }
+  - { id: 15, class: vreg_128, preferred-register: '$vgpr44_vgpr45_vgpr46_vgpr47' }
+body: |
+  bb.0:
+    %0 = IMPLICIT_DEF
+    %1 = IMPLICIT_DEF
+    %2 = IMPLICIT_DEF
+    %5 = IMPLICIT_DEF
+    %6 = IMPLICIT_DEF
+    %7 = IMPLICIT_DEF
+    %8 = IMPLICIT_DEF
+    %9 = IMPLICIT_DEF
+    %10 = IMPLICIT_DEF
+    %11 = IMPLICIT_DEF
+    %12 = IMPLICIT_DEF
+    %13 = IMPLICIT_DEF
+    %14 = IMPLICIT_DEF
+    %15 = IMPLICIT_DEF
+    %3 = V_ADD_F64 0, %0.sub2_sub3:vreg_128, 0, %1:vreg_64, 0, 0, implicit $mode, implicit $exec
+    %4 = V_ADD_F64 0, %0.sub0_sub1:vreg_128, 0, %2:vreg_64, 0, 0, implicit $mode, implicit $exec
+    S_ENDPGM 0
+...

From 45c46d180e15da9974fb0177f8e4b5a293ac241a Mon Sep 17 00:00:00 2001
From: Chen Zheng <czhengsz@cn.ibm.com>
Date: Mon, 3 Aug 2020 23:46:52 -0400
Subject: [PATCH 0986/1035] [PowerPC] mark r+i as legal address mode for vector
 type after pwr9

Reviewed By: steven.zhang

Differential Revision: https://reviews.llvm.org/D84735
---
 llvm/lib/Target/PowerPC/PPCISelLowering.cpp | 12 ++++++--
 llvm/test/CodeGen/PowerPC/prefer-dqform.ll  | 33 +++++++++++----------
 2 files changed, 26 insertions(+), 19 deletions(-)

diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index 85f1630d8e223..dfe0c2dd5c9cc 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -15128,9 +15128,15 @@ void PPCTargetLowering::LowerAsmOperandForConstraint(SDValue Op,
 // by AM is legal for this target, for a load/store of the specified type.
 bool PPCTargetLowering::isLegalAddressingMode(const DataLayout &DL,
                                               const AddrMode &AM, Type *Ty,
-                                              unsigned AS, Instruction *I) const {
-  // PPC does not allow r+i addressing modes for vectors!
-  if (Ty->isVectorTy() && AM.BaseOffs != 0)
+                                              unsigned AS,
+                                              Instruction *I) const {
+  // Vector type r+i form is supported since power9 as DQ form. We don't check
+  // the offset matching DQ form requirement(off % 16 == 0), because on PowerPC,
+  // imm form is preferred and the offset can be adjusted to use imm form later
+  // in pass PPCLoopInstrFormPrep. Also in LSR, for one LSRUse, it uses min and
+  // max offset to check legal addressing mode, we should be a little aggressive
+  // to contain other offsets for that LSRUse.
+  if (Ty->isVectorTy() && AM.BaseOffs != 0 && !Subtarget.hasP9Vector())
     return false;
 
   // PPC allows a sign-extended 16-bit immediate field.
diff --git a/llvm/test/CodeGen/PowerPC/prefer-dqform.ll b/llvm/test/CodeGen/PowerPC/prefer-dqform.ll
index 0d1992763d0ff..79e6026365ba7 100644
--- a/llvm/test/CodeGen/PowerPC/prefer-dqform.ll
+++ b/llvm/test/CodeGen/PowerPC/prefer-dqform.ll
@@ -12,26 +12,27 @@ target triple = "powerpc64le-unknown-linux-gnu"
 define void @test(i32* dereferenceable(4) %.ial, i32* noalias dereferenceable(4) %.m, i32* noalias dereferenceable(4) %.n, [0 x %_elem_type_of_a]*  %.a, i32* noalias dereferenceable(4) %.lda, [0 x %_elem_type_of_x]* noalias %.x, [0 x %_elem_type_of_y]* noalias %.y) {
 ; CHECK-P9-LABEL: test:
 ; CHECK-P9:       .LBB0_2: # %_loop_2_do_
-; CHECK-P9:         lxvx
-; CHECK-P9:         lxvx
-; CHECK-P9-DAG:     lxvx
-; CHECK-P9-DAG:     lxvx
-; CHECK-P9-DAG:     xvmaddadp
-; CHECK-P9-DAG:     xvmaddadp
-; CHECK-P9-DAG:     stxvx
-; CHECK-P9:         stxvx
+; CHECK-P9:         lxv vs1, -16(r4)
+; CHECK-P9:         lxv vs2, 0(r4)
+; CHECK-P9-DAG:     lxv vs3, -16(r3)
+; CHECK-P9-DAG:     lxv vs4, 0(r3)
+; CHECK-P9-DAG:     xvmaddadp vs1, vs3, vs1
+; CHECK-P9-DAG:     stxv vs1, -16(r4)
+; CHECK-P9-DAG:     xvmaddadp vs2, vs4, vs0
+; CHECK-P9:         stxv vs2, 0(r4)
 ; CHECK-P9:         bdnz .LBB0_2
 ;
+; FIXME: use pair load/store instructions lxvp/stxvp
 ; CHECK-P10-LABEL: test:
 ; CHECK-P10:       .LBB0_2: # %_loop_2_do_
-; CHECK-P10:         lxvx
-; CHECK-P10:         lxvx
-; CHECK-P10-DAG:     lxvx
-; CHECK-P10-DAG:     lxvx
-; CHECK-P10-DAG:     xvmaddadp
-; CHECK-P10-DAG:     xvmaddadp
-; CHECK-P10-DAG:     stxvx
-; CHECK-P10:         stxvx
+; CHECK-P10:         lxv vs1, -16(r4)
+; CHECK-P10:         lxv vs2, 0(r4)
+; CHECK-P10-DAG:     lxv vs3, -16(r3)
+; CHECK-P10-DAG:     lxv vs4, 0(r3)
+; CHECK-P10-DAG:     xvmaddadp vs1, vs3, vs1
+; CHECK-P10-DAG:     xvmaddadp vs2, vs4, vs0
+; CHECK-P10-DAG:     stxv vs1, -16(r4)
+; CHECK-P10:         stxv vs2, 0(r4)
 ; CHECK-P10:         bdnz .LBB0_2
 test_entry:
   %_conv5 = ptrtoint [0 x %_elem_type_of_a]* %.a to i64

From 6bc7ea2d8d8638f5bb753715e9023df514e814e9 Mon Sep 17 00:00:00 2001
From: "Wang, Pengfei" <pengfei.wang@intel.com>
Date: Tue, 4 Aug 2020 11:39:21 +0800
Subject: [PATCH 0987/1035] [X86][AVX512] Fix build fail after D81548

Test function mask_cmp_128 failed during ISEL
LLVM ERROR: Cannot select: t37: v8i1 = X86ISD::KSHIFTL t48, TargetConstant:i8<4>
due to v8i1 only available under AVX512DQ.

Reviewed By: RKSimon

Differential Revision: https://reviews.llvm.org/D84922
---
 llvm/lib/Target/X86/X86ISelLowering.cpp    |  4 ++
 llvm/test/CodeGen/X86/vector-shuffle-v1.ll | 58 ++++++++++++++++++++++
 2 files changed, 62 insertions(+)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index e9bb50aacec0e..b80a23f5a608e 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -38351,6 +38351,10 @@ static SDValue combineBitcastToBoolVector(EVT VT, SDValue V, const SDLoc &DL,
   case ISD::SHL: {
     // If we find a suitable source, a SHL becomes a KSHIFTL.
     SDValue Src0 = V.getOperand(0);
+    if ((VT == MVT::v8i1 && !Subtarget.hasDQI()) ||
+        ((VT == MVT::v32i1 || VT == MVT::v64i1) && !Subtarget.hasBWI()))
+      break;
+
     if (auto *Amt = dyn_cast<ConstantSDNode>(V.getOperand(1)))
       if (SDValue N0 = combineBitcastToBoolVector(VT, Src0, DL, DAG, Subtarget))
         return DAG.getNode(
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-v1.ll b/llvm/test/CodeGen/X86/vector-shuffle-v1.ll
index 8c746a3425f51..37ada5e66bbac 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-v1.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-v1.ll
@@ -977,5 +977,63 @@ define void @PR32547_swap(<8 x float> %a, <8 x float> %b, <8 x float> %c, <8 x f
    tail call void @llvm.masked.store.v16f32.p0v16f32(<16 x float> zeroinitializer, <16 x float>* %2, i32 64, <16 x i1> %3) #4
    ret void
 }
+
+define void @mask_cmp_128(<4 x float> %a, <4 x float> %b, <4 x float> %c, <4 x float> %d, float* %p) {
+; AVX512F-LABEL: mask_cmp_128:
+; AVX512F:       # %bb.0: # %entry
+; AVX512F-NEXT:    # kill: def $xmm3 killed $xmm3 def $zmm3
+; AVX512F-NEXT:    # kill: def $xmm2 killed $xmm2 def $zmm2
+; AVX512F-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vcmpltps %zmm1, %zmm0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    vcmpltps %zmm3, %zmm2, %k0
+; AVX512F-NEXT:    kshiftlw $12, %k0, %k0
+; AVX512F-NEXT:    kshiftrw $12, %k0, %k0
+; AVX512F-NEXT:    shlb $4, %al
+; AVX512F-NEXT:    kmovw %eax, %k1
+; AVX512F-NEXT:    korw %k1, %k0, %k0
+; AVX512F-NEXT:    kshiftlw $8, %k0, %k0
+; AVX512F-NEXT:    kshiftrw $8, %k0, %k1
+; AVX512F-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; AVX512F-NEXT:    vmovaps %zmm0, (%rdi) {%k1}
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: mask_cmp_128:
+; AVX512VL:       # %bb.0: # %entry
+; AVX512VL-NEXT:    vcmpltps %xmm1, %xmm0, %k0
+; AVX512VL-NEXT:    kmovw %k0, %eax
+; AVX512VL-NEXT:    vcmpltps %xmm3, %xmm2, %k0
+; AVX512VL-NEXT:    shlb $4, %al
+; AVX512VL-NEXT:    kmovw %eax, %k1
+; AVX512VL-NEXT:    korw %k1, %k0, %k1
+; AVX512VL-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; AVX512VL-NEXT:    vmovaps %ymm0, (%rdi) {%k1}
+; AVX512VL-NEXT:    vzeroupper
+; AVX512VL-NEXT:    retq
+;
+; VL_BW_DQ-LABEL: mask_cmp_128:
+; VL_BW_DQ:       # %bb.0: # %entry
+; VL_BW_DQ-NEXT:    vcmpltps %xmm1, %xmm0, %k0
+; VL_BW_DQ-NEXT:    vcmpltps %xmm3, %xmm2, %k1
+; VL_BW_DQ-NEXT:    kshiftlb $4, %k0, %k0
+; VL_BW_DQ-NEXT:    korb %k0, %k1, %k1
+; VL_BW_DQ-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; VL_BW_DQ-NEXT:    vmovaps %ymm0, (%rdi) {%k1}
+; VL_BW_DQ-NEXT:    vzeroupper
+; VL_BW_DQ-NEXT:    retq
+entry:
+  %0 = tail call i8 @llvm.x86.avx512.mask.cmp.ps.128(<4 x float> %a, <4 x float> %b, i32 1, i8 -1)
+  %1 = tail call i8 @llvm.x86.avx512.mask.cmp.ps.128(<4 x float> %c, <4 x float> %d, i32 1, i8 -1)
+  %shl = shl nuw i8 %0, 4
+  %or = or i8 %1, %shl
+  %2 = bitcast float* %p to <8 x float>*
+  %3 = bitcast i8 %or to <8 x i1>
+  tail call void @llvm.masked.store.v8f32.p0v8f32(<8 x float> zeroinitializer, <8 x float>* %2, i32 64, <8 x i1> %3)
+  ret void
+}
+declare i8 @llvm.x86.avx512.mask.cmp.ps.128(<4 x float>, <4 x float>, i32, i8)
 declare i8 @llvm.x86.avx512.mask.cmp.ps.256(<8 x float>, <8 x float>, i32, i8)
+declare void @llvm.masked.store.v8f32.p0v8f32(<8 x float>, <8 x float>*, i32, <8 x i1>)
 declare void @llvm.masked.store.v16f32.p0v16f32(<16 x float>, <16 x float>*, i32, <16 x i1>)

From b959906cb9e79f844ea4e34aa701f21bad007253 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Mon, 3 Aug 2020 21:32:44 -0700
Subject: [PATCH 0988/1035] [PGO] Use multiple comdat groups for COFF

D84723 caused multiple definition issues (related to comdat) on Windows:
http://lab.llvm.org:8011/builders/sanitizer-windows/builds/67465
---
 llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp | 3 ++-
 llvm/test/Instrumentation/InstrProfiling/PR23499.ll    | 2 +-
 llvm/test/Instrumentation/InstrProfiling/comdat.ll     | 4 ++--
 llvm/test/Instrumentation/InstrProfiling/linkage.ll    | 2 +-
 4 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
index 623f463293cd9..42939c8dbc017 100644
--- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
+++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
@@ -885,7 +885,8 @@ InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) {
   std::string DataVarName = getVarName(Inc, getInstrProfDataVarPrefix());
   auto MaybeSetComdat = [=](GlobalVariable *GV) {
     if (NeedComdat)
-      GV->setComdat(M->getOrInsertComdat(DataVarName));
+      GV->setComdat(M->getOrInsertComdat(TT.isOSBinFormatCOFF() ? GV->getName()
+                                                                : DataVarName));
   };
 
   uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
diff --git a/llvm/test/Instrumentation/InstrProfiling/PR23499.ll b/llvm/test/Instrumentation/InstrProfiling/PR23499.ll
index 098153f39e1db..88e8426ef6cb5 100644
--- a/llvm/test/Instrumentation/InstrProfiling/PR23499.ll
+++ b/llvm/test/Instrumentation/InstrProfiling/PR23499.ll
@@ -20,7 +20,7 @@ $_Z3barIvEvv = comdat any
 
 
 ; COFF-NOT: __profn__Z3barIvEvv
-; COFF: @__profc__Z3barIvEvv = linkonce_odr hidden global [1 x i64] zeroinitializer, section "{{.*}}prfc$M", comdat($__profd__Z3barIvEvv), align 8
+; COFF: @__profc__Z3barIvEvv = linkonce_odr hidden global [1 x i64] zeroinitializer, section "{{.*}}prfc$M", comdat, align 8
 ; COFF: @__profd__Z3barIvEvv = linkonce_odr hidden global { i64, i64, i64*, i8*, i8*, i32, [2 x i16] } { i64 4947693190065689389, i64 0, i64* getelementptr inbounds ([1 x i64], [1 x i64]* @__profc__Z3barIvEvv, i32 0, i32 0), i8*{{.*}}, i8* null, i32 1, [2 x i16] zeroinitializer }, section "{{.*}}prfd{{.*}}", comdat, align 8
 
 
diff --git a/llvm/test/Instrumentation/InstrProfiling/comdat.ll b/llvm/test/Instrumentation/InstrProfiling/comdat.ll
index dfcd71fd06291..81c6db7f730d0 100644
--- a/llvm/test/Instrumentation/InstrProfiling/comdat.ll
+++ b/llvm/test/Instrumentation/InstrProfiling/comdat.ll
@@ -17,7 +17,7 @@ $foo_inline = comdat any
 
 ; ELF: @__profc_foo_inline = linkonce_odr hidden global{{.*}}, section "__llvm_prf_cnts", comdat($__profd_foo_inline), align 8
 ; ELF: @__profd_foo_inline = linkonce_odr hidden global{{.*}}, section "__llvm_prf_data", comdat, align 8
-; COFF: @__profc_foo_inline = linkonce_odr hidden global{{.*}}, section ".lprfc$M", comdat($__profd_foo_inline), align 8
+; COFF: @__profc_foo_inline = linkonce_odr hidden global{{.*}}, section ".lprfc$M", comdat, align 8
 ; COFF: @__profd_foo_inline = linkonce_odr hidden global{{.*}}, section ".lprfd$M", comdat, align 8
 define weak_odr void @foo_inline() comdat {
   call void @llvm.instrprof.increment(i8* getelementptr inbounds ([10 x i8], [10 x i8]* @__profn_foo_inline, i32 0, i32 0), i64 0, i32 1, i32 0)
@@ -30,7 +30,7 @@ $foo_extern = comdat any
 
 ; ELF: @__profc_foo_extern = linkonce_odr hidden global{{.*}}, section "__llvm_prf_cnts", comdat($__profd_foo_extern), align 8
 ; ELF: @__profd_foo_extern = linkonce_odr hidden global{{.*}}, section "__llvm_prf_data", comdat, align 8
-; COFF: @__profc_foo_extern = linkonce_odr hidden global{{.*}}, section ".lprfc$M", comdat($__profd_foo_extern), align 8
+; COFF: @__profc_foo_extern = linkonce_odr hidden global{{.*}}, section ".lprfc$M", comdat, align 8
 ; COFF: @__profd_foo_extern = linkonce_odr hidden global{{.*}}, section ".lprfd$M", comdat, align 8
 define available_externally void @foo_extern() {
   call void @llvm.instrprof.increment(i8* getelementptr inbounds ([10 x i8], [10 x i8]* @__profn_foo_extern, i32 0, i32 0), i64 0, i32 1, i32 0)
diff --git a/llvm/test/Instrumentation/InstrProfiling/linkage.ll b/llvm/test/Instrumentation/InstrProfiling/linkage.ll
index 5e55ef716820e..6320060afe42d 100644
--- a/llvm/test/Instrumentation/InstrProfiling/linkage.ll
+++ b/llvm/test/Instrumentation/InstrProfiling/linkage.ll
@@ -59,7 +59,7 @@ define linkonce_odr void @foo_inline() {
 ; LINUX: @__profd_foo_extern = linkonce_odr hidden global {{.*}}section "__llvm_prf_data", comdat, align 8
 ; MACHO: @__profc_foo_extern = linkonce_odr hidden global
 ; MACHO: @__profd_foo_extern = linkonce_odr hidden global
-; COFF: @__profc_foo_extern = linkonce_odr hidden global {{.*}}section ".lprfc$M", comdat($__profd_foo_extern), align 8
+; COFF: @__profc_foo_extern = linkonce_odr hidden global {{.*}}section ".lprfc$M", comdat, align 8
 ; COFF: @__profd_foo_extern = linkonce_odr hidden global {{.*}}section ".lprfd$M", comdat, align 8
 define available_externally void @foo_extern() {
   call void @llvm.instrprof.increment(i8* getelementptr inbounds ([10 x i8], [10 x i8]* @__profn_foo_extern, i32 0, i32 0), i64 0, i32 1, i32 0)

From 25abd1994ed209c1bf4139946a42e36a42143a85 Mon Sep 17 00:00:00 2001
From: Xing GUO <higuoxing@gmail.com>
Date: Tue, 4 Aug 2020 12:42:09 +0800
Subject: [PATCH 0989/1035] [YAMLParser] Fix a typo: iff -> if. NFC.

---
 llvm/include/llvm/Support/YAMLParser.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/include/llvm/Support/YAMLParser.h b/llvm/include/llvm/Support/YAMLParser.h
index 53009d7ff4aad..44daf7850904a 100644
--- a/llvm/include/llvm/Support/YAMLParser.h
+++ b/llvm/include/llvm/Support/YAMLParser.h
@@ -222,7 +222,7 @@ class ScalarNode final : public Node {
 
   /// Gets the value of this node as a StringRef.
   ///
-  /// \param Storage is used to store the content of the returned StringRef iff
+  /// \param Storage is used to store the content of the returned StringRef if
   ///        it requires any modification from how it appeared in the source.
   ///        This happens with escaped characters and multi-line literals.
   StringRef getValue(SmallVectorImpl<char> &Storage) const;

From 79f7cfb435ed0c4836a245de0531751c1bfa83ea Mon Sep 17 00:00:00 2001
From: Soumi Manna <soumi.manna@intel.com>
Date: Mon, 3 Aug 2020 22:54:52 -0700
Subject: [PATCH 0990/1035] Update Patch and add tests based on review comments

Signed-off-by: Soumi Manna <soumi.manna@intel.com>
---
 clang/lib/Sema/SemaSYCL.cpp                   |  39 +----
 clang/test/CodeGenSYCL/union_kernel_param.cpp |  58 ++++++++
 .../test/CodeGenSYCL/union_kernel_param1.cpp  |  72 +++++++++
 clang/test/SemaSYCL/union-kernel-param.cpp    |  47 ++++--
 sycl/test/union_param/union_kernel_param.cpp  | 138 ++++++++++++++++++
 5 files changed, 307 insertions(+), 47 deletions(-)
 create mode 100644 clang/test/CodeGenSYCL/union_kernel_param.cpp
 create mode 100644 clang/test/CodeGenSYCL/union_kernel_param1.cpp
 create mode 100644 sycl/test/union_param/union_kernel_param.cpp

diff --git a/clang/lib/Sema/SemaSYCL.cpp b/clang/lib/Sema/SemaSYCL.cpp
index 444b63487e1a1..73b1802881414 100644
--- a/clang/lib/Sema/SemaSYCL.cpp
+++ b/clang/lib/Sema/SemaSYCL.cpp
@@ -825,9 +825,7 @@ class KernelObjVisitor {
     else if (ElementTy->isArrayType())
       VisitArrayElements(ArrayField, ElementTy, handlers...);
     else if (ElementTy->isScalarType())
-      KF_FOR_EACH(handleScalarType, ArrayField, ElementTy);
-    else if (ElementTy->isUnionType())
-      KF_FOR_EACH(handleUnionType, ArrayField, ElementTy);
+      KF_FOR_EACH(handleSimpleType, ArrayField, ElementTy);
   }
 
   template <typename... Handlers>
@@ -943,11 +941,9 @@ class KernelObjVisitor {
       else if (FieldTy->isArrayType()) {
         if (KF_FOR_EACH(handleArrayType, Field, FieldTy))
           VisitArrayElements(Field, FieldTy, handlers...);
-      } else if (FieldTy->isScalarType() || FieldTy->isVectorType()
-                 || FieldTy->isUnionType()) {
-        KF_FOR_EACH(handleScalarType, Field, FieldTy);
-      } else if (FieldTy->isUnionType()) {
-        KF_FOR_EACH(handleUnionType, Field, FieldTy);
+      } else if (FieldTy->isScalarType() || FieldTy->isUnionType()
+		 || FieldTy->isVectorType()) {
+        KF_FOR_EACH(handleSimpleType, Field, FieldTy);
       } else
         KF_FOR_EACH(handleOtherType, Field, FieldTy);
       (void)std::initializer_list<int>{
@@ -1005,8 +1001,7 @@ class SyclKernelFieldHandler {
   virtual bool handleReferenceType(FieldDecl *, QualType) { return true; }
   virtual bool handlePointerType(FieldDecl *, QualType) { return true; }
   virtual bool handleArrayType(FieldDecl *, QualType) { return true; }
-  virtual bool handleScalarType(FieldDecl *, QualType) { return true; }
-  virtual bool handleUnionType(FieldDecl *, QualType) { return true; }
+  virtual bool handleSimpleType(FieldDecl *, QualType) { return true; }
   // Most handlers shouldn't be handling this, just the field checker.
   virtual bool handleOtherType(FieldDecl *, QualType) { return true; }
 
@@ -1292,7 +1287,7 @@ class SyclKernelDeclCreator : public SyclKernelFieldHandler {
     return true;
   }
 
-  bool handleScalarType(FieldDecl *FD, QualType FieldTy) final {
+  bool handleSimpleType(FieldDecl *FD, QualType FieldTy) final {
     addParam(FD, FieldTy);
     return true;
   }
@@ -1307,11 +1302,6 @@ class SyclKernelDeclCreator : public SyclKernelFieldHandler {
     return true;
   }
 
-  bool handleUnionType(FieldDecl *FD, QualType FieldTy) final {
-    addParam(FD, FieldTy);
-    return true;
-  }
-
   bool handleSyclStreamType(const CXXBaseSpecifier &, QualType FieldTy) final {
     // FIXME SYCL stream should be usable as a base type
     // See https://github.com/intel/llvm/issues/1552
@@ -1616,15 +1606,7 @@ class SyclKernelBodyCreator : public SyclKernelFieldHandler {
     return true;
   }
 
-  bool handleScalarType(FieldDecl *FD, QualType FieldTy) final {
-    if (dyn_cast<ArraySubscriptExpr>(MemberExprBases.back()))
-      createExprForScalarElement(FD);
-    else
-      createExprForStructOrScalar(FD);
-    return true;
-  }
-
-  bool handleUnionType(FieldDecl *FD, QualType FieldTy) final {
+  bool handleSimpleType(FieldDecl *FD, QualType FieldTy) final {
     if (dyn_cast<ArraySubscriptExpr>(MemberExprBases.back()))
       createExprForScalarElement(FD);
     else
@@ -1831,7 +1813,7 @@ class SyclKernelIntHeaderCreator : public SyclKernelFieldHandler {
     return true;
   }
 
-  bool handleScalarType(FieldDecl *FD, QualType FieldTy) final {
+  bool handleSimpleType(FieldDecl *FD, QualType FieldTy) final {
     addParam(FD, FieldTy, SYCLIntegrationHeader::kind_std_layout);
     return true;
   }
@@ -1841,11 +1823,6 @@ class SyclKernelIntHeaderCreator : public SyclKernelFieldHandler {
     return true;
   }
 
-  bool handleUnionType(FieldDecl *FD, QualType FieldTy) final {
-    addParam(FD, FieldTy, SYCLIntegrationHeader::kind_std_layout);
-    return true;
-  }
-
   bool handleSyclStreamType(const CXXBaseSpecifier &BC,
                             QualType FieldTy) final {
     // FIXME SYCL stream should be usable as a base type
diff --git a/clang/test/CodeGenSYCL/union_kernel_param.cpp b/clang/test/CodeGenSYCL/union_kernel_param.cpp
new file mode 100644
index 0000000000000..5735ed20bdedd
--- /dev/null
+++ b/clang/test/CodeGenSYCL/union_kernel_param.cpp
@@ -0,0 +1,58 @@
+// RUN: %clang_cc1 -I %S/Inputs -fsycl -fsycl-is-device -triple spir64-unknown-unknown-sycldevice -fsycl-int-header=%t.h %s -o %t.out
+// RUN: FileCheck -input-file=%t.h %s
+
+// This test checks the integration header generated when
+// the kernel argument is union.
+
+// CHECK: #include <CL/sycl/detail/kernel_desc.hpp>
+
+// CHECK: class MyKernel;
+
+// CHECK: __SYCL_INLINE_NAMESPACE(cl) {
+// CHECK-NEXT: namespace sycl {
+// CHECK-NEXT: namespace detail {
+
+// CHECK: static constexpr
+// CHECK-NEXT: const char* const kernel_names[] = {
+// CHECK-NEXT:   "_ZTSZZ5test0vENK3$_0clERN2cl4sycl7handlerEE8MyKernel"
+// CHECK-NEXT: };
+
+// CHECK: static constexpr
+// CHECK-NEXT: const kernel_param_desc_t kernel_signatures[] = {
+// CHECK-NEXT:   //--- _ZTSZZ5test0vENK3$_0clERN2cl4sycl7handlerEE8MyKernel
+// CHECK-NEXT:   { kernel_param_kind_t::kind_accessor, 4062, 0 },
+// CHECK-NEXT:   { kernel_param_kind_t::kind_std_layout, 12, 12 },
+// CHECK-EMPTY:
+// CHECK-NEXT: };
+
+// CHECK: static constexpr
+// CHECK-NEXT: const unsigned kernel_signature_start[] = {
+// CHECK-NEXT:  0 // _ZTSZZ5test0vENK3$_0clERN2cl4sycl7handlerEE8MyKernel
+// CHECK-NEXT: };
+
+// CHECK: template <> struct KernelInfo<class MyKernel> {
+
+#include "sycl.hpp"
+
+using namespace cl::sycl;
+
+union MyUnion {
+  int FldInt;
+  int FldArr[3];
+};
+
+MyUnion GlobS;
+
+bool test0() {
+  MyUnion S = GlobS;
+  MyUnion S0 = { 0 };
+  {
+    buffer<MyUnion, 1> Buf(&S0, range<1>(1));
+    queue myQueue;
+    myQueue.submit([&](handler &cgh) {
+      auto B = Buf.get_access<access::mode::write>(cgh);
+      cgh.single_task<class MyKernel>([=] { B; S; });
+    });
+  }
+}
+
diff --git a/clang/test/CodeGenSYCL/union_kernel_param1.cpp b/clang/test/CodeGenSYCL/union_kernel_param1.cpp
new file mode 100644
index 0000000000000..e2a8159f4be01
--- /dev/null
+++ b/clang/test/CodeGenSYCL/union_kernel_param1.cpp
@@ -0,0 +1,72 @@
+// RUN: %clang_cc1 -fsycl -fsycl-is-device -I %S/Inputs -triple spir64-unknown-unknown-sycldevice -disable-llvm-passes -emit-llvm %s -o - | FileCheck %s
+
+// This test checks if compiler accepts union as kernel parameters.
+
+#include "sycl.hpp"
+
+using namespace cl::sycl;
+
+union MyUnion {
+  int FldInt;
+  int FldArr[3];
+};
+
+MyUnion GlobS;
+
+bool test0() {
+  MyUnion S = GlobS;
+  MyUnion S0 = { 0 };
+  {
+    buffer<MyUnion, 1> Buf(&S0, range<1>(1));
+    queue myQueue;
+    myQueue.submit([&](handler &cgh) {
+      auto B = Buf.get_access<access::mode::write>(cgh);
+      cgh.single_task<class MyKernel>([=] { B; S; });
+    });
+  }
+}
+
+// CHECK MyKernel parameters
+// CHECK: define spir_kernel void @{{.*}}MyKernel
+// CHECK-SAME: %union.{{.*}}.MyUnion addrspace(1)* [[MEM_ARG1:%[a-zA-Z0-9_]+]],
+// CHECK-SAME: %"struct.{{.*}}.cl::sycl::range"* byval({{.*}}) align 4 [[MEM_ARG2:%[a-zA-Z0-9_]+1]],
+// CHECK-SAME: %"struct.{{.*}}.cl::sycl::range"* byval({{.*}}) align 4 [[MEM_ARG3:%[a-zA-Z0-9_]+2]],
+// CHECK-SAME: %"struct.{{.*}}.cl::sycl::id"* byval({{.*}}) align 4 [[OFFSET1:%[a-zA-Z0-9_]+3]],
+// CHECK-SAME: %union.{{.*}}.MyUnion* byval({{.*}}) align 4 [[MEM_ARG4:%[a-zA-Z0-9_]+4]])
+
+// Check alloca for pointer arguments
+// CHECK: [[MEM_ARG1]].addr{{[0-9]*}} = alloca %union._ZTS7MyUnion.MyUnion addrspace(1)*, align 8
+
+// Check lambda object alloca
+// CHECK: [[LOCAL_OBJECT:%0]] = alloca %"class.{{.*}}.anon", align 4
+
+// Check allocas for ranges
+// CHECK: [[ACC_RANGE1:%[a-zA-Z0-9_.]+]] = alloca %"struct.{{.*}}.cl::sycl::range"
+// CHECK: [[ACC_RANGE2:%[a-zA-Z0-9_.]+]] = alloca %"struct.{{.*}}.cl::sycl::range"
+// CHECK: [[OFFSET2:%[a-zA-Z0-9_.]+]] = alloca %"struct.{{.*}}.cl::sycl::id"
+
+// CHECK: [[L_STRUCT_ADDR:%[a-zA-Z0-9_]+]] = getelementptr inbounds %"class.{{.*}}.anon", %"class.{{.*}}.anon"* [[LOCAL_OBJECT]], i32 0, i32 0
+// CHECK: [[ACC_CAST1:%[0-9]+]] = addrspacecast %"class{{.*}}accessor"* [[L_STRUCT_ADDR]] to %"class{{.*}}accessor" addrspace(4)*
+// CHECK: call spir_func void @{{.*}}MyUnion{{.*}}(%"class.{{.*}}.cl::sycl::accessor" addrspace(4)* [[ACC_CAST1]])
+// CHECK: [[Z0:%[a-zA-Z0-9_]*]]  = getelementptr inbounds %"class.{{.*}}.anon", %"class.{{.*}}.anon"* [[LOCAL_OBJECT]], i32 0, i32 1
+// CHECK: [[MEMCPY_DST:%[0-9a-zA-Z_]+]] = bitcast %union.{{.*}}MyUnion* [[Z0]] to i8*
+// CHECK: [[MEMCPY_SRC:%[0-9a-zA-Z_]+]] = bitcast %union.{{.*}}MyUnion* [[MEM_ARG4]] to i8*
+// CHECK: call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 [[MEMCPY_DST]], i8* align 4 [[MEMCPY_SRC]], i64 12, i1 false)
+
+// CHECK: [[Z1:%[a-zA-Z0-9_]*]]  = getelementptr inbounds %"class.{{.*}}.anon", %"class.{{.*}}.anon"* [[LOCAL_OBJECT]], i32 0, i32 0
+
+// Check load from kernel pointer argument alloca
+// CHECK: [[MEM_LOAD1:%[a-zA-Z0-9_]+]] = load %union._ZTS7MyUnion.MyUnion addrspace(1)*, %union._ZTS7MyUnion.MyUnion addrspace(1)** [[MEM_ARG1]].addr{{[0-9]*}}, align 8
+// CHECK: [[MEMCPY_DST1:%[0-9a-zA-Z_]+]] = bitcast %"struct.{{.*}}.cl::sycl::range"* [[ACC_RANGE1]] to i8*
+// CHECK: [[MEMCPY_SRC1:%[0-9a-zA-Z_]+]] = bitcast %"struct.{{.*}}.cl::sycl::range"* [[MEM_ARG2]] to i8*
+// call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 [[MEMCPY_DST1]], i8* align 4 [[MEMCPY_SRC1]], i64 4, i1 false), !tbaa.struct [[ACC_CAST2:%[0-9]+]]
+// CHECK: [[MEMCPY_DST2:%[0-9a-zA-Z_]+]] = bitcast %"struct.{{.*}}.cl::sycl::range"* [[ACC_RANGE2]] to i8*
+// CHECK: [[MEMCPY_SRC2:%[0-9a-zA-Z_]+]] = bitcast %"struct.{{.*}}.cl::sycl::range"* [[MEM_ARG3]] to i8*
+// call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 [[MEMCPY_DST2]], i8* align 4 [[MEMCPY_SRC2]], i64 4, i1 false), !tbaa.struct [[ACC_CAST2:%[0-9]+]]
+
+// Check __init method call
+// CHECK: [[ACC_CAST1:%[0-9]+]] = addrspacecast %"class{{.*}}accessor"* [[Z1]] to %"class{{.*}}accessor" addrspace(4)*
+// CHECK: call spir_func void @{{.*}}__init{{.*}}(%"class.{{.*}}.cl::sycl::accessor" addrspace(4)* [[ACC_CAST1]], %union._ZTS7MyUnion.MyUnion addrspace(1)* [[MEM_LOAD1]], %"struct.{{.*}}.cl::sycl::range"* byval({{.*}}) align 4 [[ACC_RANGE1]], %"struct.{{.*}}.cl::sycl::range"* byval({{.*}}) align 4 [[ACC_RANGE2]], %"struct.{{.*}}.cl::sycl::id"* byval({{.*}}) align 4 [[OFFSET2]])
+// CHECK: [[ACC_CAST2:%[0-9]+]] = addrspacecast %"class{{.*}}.anon"* [[LOCAL_OBJECT]] to %"class{{.*}}.anon" addrspace(4)*
+// CHECK: call spir_func void @{{.*}}(%"class.{{.*}}.anon" addrspace(4)* [[ACC_CAST2]])
+
diff --git a/clang/test/SemaSYCL/union-kernel-param.cpp b/clang/test/SemaSYCL/union-kernel-param.cpp
index 801be04b816e9..2e5f971c3828a 100644
--- a/clang/test/SemaSYCL/union-kernel-param.cpp
+++ b/clang/test/SemaSYCL/union-kernel-param.cpp
@@ -1,30 +1,45 @@
-// RUN: %clang_cc1 -I %S/Inputs -fsycl -fsyntax-only -verify %s
+// RUN: %clang_cc1 -I %S/Inputs -fsycl -fsycl-is-device -ast-dump %s | FileCheck %s
 // expected-no-diagnostics
 
+// This test checks that compiler generates correct kernel arguments for
+// union.
+
 #include <sycl.hpp>
 
 using namespace cl::sycl;
 
-namespace s = sycl;
+typedef float realw;
 
-struct P {
-    union S {
-        int32_t p;
-    } q;
-};
+typedef union dpct_type_54e08f {
+  float cuda;
+} gpu_realw_mem;
 
-static_assert(std::is_trivially_copyable<P>::value, "");
-static_assert(std::is_trivially_copyable<decltype(P{}.q)>::value, "");
+void call_some_dummy_kernel(float data) {
+  data = 2.0f;
+}
 
-int main() {
-    s::queue q;
+template <typename Name, typename Type>
+__attribute__((sycl_kernel)) void parallel_for(Type lambda) {
+  lambda();
+}
 
-    P p = {};
+int main() {
+  gpu_realw_mem accel;
 
-    q.submit([&](s::handler& h) {
-        h.single_task<class foo>([=](){
-            (void) p;
-        });
+  parallel_for<class kernel>(
+    [=]() {
+      call_some_dummy_kernel(accel.cuda);
     });
 }
 
+// Check kernel parameters
+// CHECK: FunctionDecl {{.*}}kernel{{.*}} 'void (gpu_realw_mem)'
+// CHECK-NEXT: ParmVarDecl {{.*}} used _arg_ 'gpu_realw_mem':'dpct_type_54e08f'
+// Check kernel inits
+// CHECK-NEXT: CompoundStmt
+// CHECK-NEXT: DeclStmt
+// CHECK-NEXT: VarDecl {{.*}} cinit
+// CHECK-NEXT: InitListExpr
+// CHECK-NEXT: CXXConstructExpr {{.*}} 'gpu_realw_mem':'dpct_type_54e08f' 'void (const dpct_type_54e08f &) noexcept'
+// CHECK: ImplicitCastExpr
+// CHECK: DeclRefExpr {{.*}} lvalue ParmVar {{.*}} '_arg_' 'gpu_realw_mem':'dpct_type_54e08f'
diff --git a/sycl/test/union_param/union_kernel_param.cpp b/sycl/test/union_param/union_kernel_param.cpp
new file mode 100644
index 0000000000000..09a4bc4786e30
--- /dev/null
+++ b/sycl/test/union_param/union_kernel_param.cpp
@@ -0,0 +1,138 @@
+// RUN: %clangxx -fsycl -fsycl-targets=%sycl_triple %s -o %t.out
+// RUN: env SYCL_DEVICE_TYPE=HOST %t.out
+// RUN: %CPU_RUN_PLACEHOLDER %t.out
+// TODO: Uncomment once test is fixed on GPU
+// RUNx: %GPU_RUN_PLACEHOLDER %t.out
+// RUN: %ACC_RUN_PLACEHOLDER %t.out
+
+//==-union_kernel_param.cpp-Checks passing unions as kernel params--------==//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <CL/sycl.hpp>
+#include <algorithm>
+#include <cstring>
+#include <iostream>
+#include <iterator>
+
+using namespace cl::sycl;
+
+union MyNestedUnion {
+  bool operator==(const MyNestedUnion &Rhs) {
+    return (FldArr[0] == Rhs.FldArr[0] && FldFloat == Rhs.FldFloat);
+  }
+  cl::sycl::cl_char FldArr[1];
+  cl::sycl::cl_float FldFloat;
+};
+
+union MyUnion {
+  bool operator==(const MyUnion &Rhs) {
+    return (FldChar == Rhs.FldChar && FldLong == Rhs.FldLong &&
+            FldShort == Rhs.FldShort && FldUint == Rhs.FldUint &&
+            FldUnion == Rhs.FldUnion &&
+            std::equal(std::begin(FldArr), std::end(FldArr),
+                       std::begin(Rhs.FldArr)) &&
+            FldInt == Rhs.FldInt);
+  }
+  cl::sycl::cl_char FldChar;
+  cl::sycl::cl_long FldLong;
+  cl::sycl::cl_short FldShort;
+  cl::sycl::cl_uint FldUint;
+  MyNestedUnion FldUnion;
+  cl::sycl::cl_short FldArr[3];
+  cl::sycl::cl_int FldInt;
+};
+
+MyUnion GlobS;
+
+static void printUnion(const MyUnion &S0) {
+  std::cout << "{ " << (int)S0.FldChar << ", " << S0.FldLong << ", "
+            << S0.FldShort << ", " << S0.FldUint << " { { "
+            << (int)S0.FldUnion.FldArr[0] << " }, " << S0.FldUnion.FldFloat
+            << " }, { " << S0.FldArr[0] << ", " << S0.FldArr[1] << ", "
+            << S0.FldArr[2] << " }, " << S0.FldInt << " }";
+}
+
+bool test0() {
+  MyUnion S = GlobS;
+  MyUnion S0 = {0};
+  {
+    buffer<MyUnion, 1> Buf(&S0, range<1>(1));
+    queue myQueue;
+    myQueue.submit([&](handler &cgh) {
+      auto B = Buf.get_access<access::mode::write>(cgh);
+      cgh.single_task<class MyKernel>([=] { B[0] = S; });
+    });
+  }
+  bool Passed = (S == S0);
+
+  if (!Passed) {
+    std::cout << "test0 failed" << std::endl;
+    std::cout << "test0 input:" << std::endl;
+    printStruct(S);
+    std::cout << std::endl;
+    std::cout << "test0 result:\n";
+    printStruct(S0);
+    std::cout << std::endl;
+  }
+  return Passed;
+}
+
+bool test1() {
+  range<3> ice(8, 9, 10);
+  uint ice2 = 888;
+  uint result[4] = {0};
+
+  {
+    buffer<unsigned int, 1> Buffer((unsigned int *)result, range<1>(4));
+    queue myQueue;
+    myQueue.submit([&](handler &cgh) {
+      auto B = Buffer.get_access<access::mode::write>(cgh);
+      cgh.parallel_for<class bufferByRange_cap>(range<1>{4}, [=](id<1> index) {
+        B[index.get(0)] = index.get(0) > 2 ? ice2 : ice.get(index.get(0));
+      });
+    });
+  }
+
+  bool Passed = true;
+
+  for (unsigned long i = 0; i < 4; ++i) {
+    if (i <= 2) {
+      if (result[i] != ice[i])
+        Passed = false;
+    } else {
+      if (result[i] != ice2)
+        Passed = false;
+    }
+  }
+  if (!Passed)
+    std::cout << "test1 failed" << std::endl;
+
+  return Passed;
+}
+
+int main(int argc, char **argv) {
+  cl::sycl::cl_char PartChar = argc;
+  cl::sycl::cl_short PartShort = argc << 8;
+  cl::sycl::cl_int PartInt = argc << 16;
+  cl::sycl::cl_uint PartUint = argc << 16;
+  cl::sycl::cl_long PartLong = ((cl::sycl::cl_long)argc) << 32;
+  cl::sycl::cl_float PartFloat = argc;
+
+  GlobS = {PartChar,
+           PartLong,
+           PartShort,
+           PartUint,
+           {{PartChar}, PartFloat},
+           {PartShort, PartShort, PartShort},
+           PartInt};
+
+  bool Pass = test0() & test1();
+
+  std::cout << "Test " << (Pass ? "passed" : "FAILED") << std::endl;
+  return Pass ? 0 : 1;
+}

From 4d504366794c9244c98f12fe3ca7b60371104336 Mon Sep 17 00:00:00 2001
From: Soumi Manna <soumi.manna@intel.com>
Date: Mon, 3 Aug 2020 23:03:51 -0700
Subject: [PATCH 0991/1035] Fix clang-format issues

Signed-off-by: Soumi Manna <soumi.manna@intel.com>
---
 clang/lib/Sema/SemaSYCL.cpp                    | 4 ++--
 clang/test/CodeGenSYCL/union_kernel_param.cpp  | 2 +-
 clang/test/CodeGenSYCL/union_kernel_param1.cpp | 4 +---
 clang/test/SemaSYCL/union-kernel-param.cpp     | 6 +++---
 4 files changed, 7 insertions(+), 9 deletions(-)

diff --git a/clang/lib/Sema/SemaSYCL.cpp b/clang/lib/Sema/SemaSYCL.cpp
index 73b1802881414..c870f9397c269 100644
--- a/clang/lib/Sema/SemaSYCL.cpp
+++ b/clang/lib/Sema/SemaSYCL.cpp
@@ -941,8 +941,8 @@ class KernelObjVisitor {
       else if (FieldTy->isArrayType()) {
         if (KF_FOR_EACH(handleArrayType, Field, FieldTy))
           VisitArrayElements(Field, FieldTy, handlers...);
-      } else if (FieldTy->isScalarType() || FieldTy->isUnionType()
-		 || FieldTy->isVectorType()) {
+      } else if (FieldTy->isScalarType() || FieldTy->isUnionType() ||
+		 FieldTy->isVectorType()) {
         KF_FOR_EACH(handleSimpleType, Field, FieldTy);
       } else
         KF_FOR_EACH(handleOtherType, Field, FieldTy);
diff --git a/clang/test/CodeGenSYCL/union_kernel_param.cpp b/clang/test/CodeGenSYCL/union_kernel_param.cpp
index 5735ed20bdedd..85a9e4f0c8c42 100644
--- a/clang/test/CodeGenSYCL/union_kernel_param.cpp
+++ b/clang/test/CodeGenSYCL/union_kernel_param.cpp
@@ -45,7 +45,7 @@ MyUnion GlobS;
 
 bool test0() {
   MyUnion S = GlobS;
-  MyUnion S0 = { 0 };
+  MyUnion S0 = {0};
   {
     buffer<MyUnion, 1> Buf(&S0, range<1>(1));
     queue myQueue;
diff --git a/clang/test/CodeGenSYCL/union_kernel_param1.cpp b/clang/test/CodeGenSYCL/union_kernel_param1.cpp
index e2a8159f4be01..34b4120a4f04d 100644
--- a/clang/test/CodeGenSYCL/union_kernel_param1.cpp
+++ b/clang/test/CodeGenSYCL/union_kernel_param1.cpp
@@ -15,7 +15,7 @@ MyUnion GlobS;
 
 bool test0() {
   MyUnion S = GlobS;
-  MyUnion S0 = { 0 };
+  MyUnion S0 = {0};
   {
     buffer<MyUnion, 1> Buf(&S0, range<1>(1));
     queue myQueue;
@@ -52,7 +52,6 @@ bool test0() {
 // CHECK: [[MEMCPY_DST:%[0-9a-zA-Z_]+]] = bitcast %union.{{.*}}MyUnion* [[Z0]] to i8*
 // CHECK: [[MEMCPY_SRC:%[0-9a-zA-Z_]+]] = bitcast %union.{{.*}}MyUnion* [[MEM_ARG4]] to i8*
 // CHECK: call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 [[MEMCPY_DST]], i8* align 4 [[MEMCPY_SRC]], i64 12, i1 false)
-
 // CHECK: [[Z1:%[a-zA-Z0-9_]*]]  = getelementptr inbounds %"class.{{.*}}.anon", %"class.{{.*}}.anon"* [[LOCAL_OBJECT]], i32 0, i32 0
 
 // Check load from kernel pointer argument alloca
@@ -69,4 +68,3 @@ bool test0() {
 // CHECK: call spir_func void @{{.*}}__init{{.*}}(%"class.{{.*}}.cl::sycl::accessor" addrspace(4)* [[ACC_CAST1]], %union._ZTS7MyUnion.MyUnion addrspace(1)* [[MEM_LOAD1]], %"struct.{{.*}}.cl::sycl::range"* byval({{.*}}) align 4 [[ACC_RANGE1]], %"struct.{{.*}}.cl::sycl::range"* byval({{.*}}) align 4 [[ACC_RANGE2]], %"struct.{{.*}}.cl::sycl::id"* byval({{.*}}) align 4 [[OFFSET2]])
 // CHECK: [[ACC_CAST2:%[0-9]+]] = addrspacecast %"class{{.*}}.anon"* [[LOCAL_OBJECT]] to %"class{{.*}}.anon" addrspace(4)*
 // CHECK: call spir_func void @{{.*}}(%"class.{{.*}}.anon" addrspace(4)* [[ACC_CAST2]])
-
diff --git a/clang/test/SemaSYCL/union-kernel-param.cpp b/clang/test/SemaSYCL/union-kernel-param.cpp
index 2e5f971c3828a..5267cf458b285 100644
--- a/clang/test/SemaSYCL/union-kernel-param.cpp
+++ b/clang/test/SemaSYCL/union-kernel-param.cpp
@@ -27,9 +27,9 @@ int main() {
   gpu_realw_mem accel;
 
   parallel_for<class kernel>(
-    [=]() {
-      call_some_dummy_kernel(accel.cuda);
-    });
+      [=]() {
+        call_some_dummy_kernel(accel.cuda);
+      });
 }
 
 // Check kernel parameters

From 503182aaddb6b499f71c9a7e32bd2187bf98c234 Mon Sep 17 00:00:00 2001
From: Soumi Manna <soumi.manna@intel.com>
Date: Mon, 3 Aug 2020 23:08:03 -0700
Subject: [PATCH 0992/1035] Fix clang-format issues

Signed-off-by: Soumi Manna <soumi.manna@intel.com>
---
 clang/lib/Sema/SemaSYCL.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/lib/Sema/SemaSYCL.cpp b/clang/lib/Sema/SemaSYCL.cpp
index c870f9397c269..69f9061f7c482 100644
--- a/clang/lib/Sema/SemaSYCL.cpp
+++ b/clang/lib/Sema/SemaSYCL.cpp
@@ -942,7 +942,7 @@ class KernelObjVisitor {
         if (KF_FOR_EACH(handleArrayType, Field, FieldTy))
           VisitArrayElements(Field, FieldTy, handlers...);
       } else if (FieldTy->isScalarType() || FieldTy->isUnionType() ||
-		 FieldTy->isVectorType()) {
+		  FieldTy->isVectorType()) {
         KF_FOR_EACH(handleSimpleType, Field, FieldTy);
       } else
         KF_FOR_EACH(handleOtherType, Field, FieldTy);

From 5313ca28278b3efd5ff5e153eed52546f40609b6 Mon Sep 17 00:00:00 2001
From: Soumi Manna <soumi.manna@intel.com>
Date: Mon, 3 Aug 2020 23:18:26 -0700
Subject: [PATCH 0993/1035] Fix clang-format issues

Signed-off-by: Soumi Manna <soumi.manna@intel.com>
---
 clang/lib/Sema/SemaSYCL.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/clang/lib/Sema/SemaSYCL.cpp b/clang/lib/Sema/SemaSYCL.cpp
index 69f9061f7c482..f7bd9481abb15 100644
--- a/clang/lib/Sema/SemaSYCL.cpp
+++ b/clang/lib/Sema/SemaSYCL.cpp
@@ -941,8 +941,8 @@ class KernelObjVisitor {
       else if (FieldTy->isArrayType()) {
         if (KF_FOR_EACH(handleArrayType, Field, FieldTy))
           VisitArrayElements(Field, FieldTy, handlers...);
-      } else if (FieldTy->isScalarType() || FieldTy->isUnionType() ||
-		  FieldTy->isVectorType()) {
+      } else if (FieldTy->isScalarType() || FieldTy->isVectorType() ||
+		  FieldTy->isUnionType()) {
         KF_FOR_EACH(handleSimpleType, Field, FieldTy);
       } else
         KF_FOR_EACH(handleOtherType, Field, FieldTy);

From f6f1d7c3c0d37657575032a49509ce358a214ad6 Mon Sep 17 00:00:00 2001
From: Soumi Manna <soumi.manna@intel.com>
Date: Mon, 3 Aug 2020 23:29:39 -0700
Subject: [PATCH 0994/1035] Fix clang-format issues

Signed-off-by: Soumi Manna <soumi.manna@intel.com>
---
 clang/lib/Sema/SemaSYCL.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/clang/lib/Sema/SemaSYCL.cpp b/clang/lib/Sema/SemaSYCL.cpp
index f7bd9481abb15..a1413c589da4e 100644
--- a/clang/lib/Sema/SemaSYCL.cpp
+++ b/clang/lib/Sema/SemaSYCL.cpp
@@ -941,8 +941,8 @@ class KernelObjVisitor {
       else if (FieldTy->isArrayType()) {
         if (KF_FOR_EACH(handleArrayType, Field, FieldTy))
           VisitArrayElements(Field, FieldTy, handlers...);
-      } else if (FieldTy->isScalarType() || FieldTy->isVectorType() ||
-		  FieldTy->isUnionType()) {
+      } else if (FieldTy->isScalarType() || FieldTy->isUnioType() ||
+		  FieldTy->isVectorType()) {
         KF_FOR_EACH(handleSimpleType, Field, FieldTy);
       } else
         KF_FOR_EACH(handleOtherType, Field, FieldTy);

From 6fb50b4c51dc07ee38c35d5eed3353984fbd8fa2 Mon Sep 17 00:00:00 2001
From: Soumi Manna <soumi.manna@intel.com>
Date: Mon, 3 Aug 2020 23:43:07 -0700
Subject: [PATCH 0995/1035] Fix clang-format issues

Signed-off-by: Soumi Manna <soumi.manna@intel.com>
---
 clang/lib/Sema/SemaSYCL.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/lib/Sema/SemaSYCL.cpp b/clang/lib/Sema/SemaSYCL.cpp
index a1413c589da4e..69f9061f7c482 100644
--- a/clang/lib/Sema/SemaSYCL.cpp
+++ b/clang/lib/Sema/SemaSYCL.cpp
@@ -941,7 +941,7 @@ class KernelObjVisitor {
       else if (FieldTy->isArrayType()) {
         if (KF_FOR_EACH(handleArrayType, Field, FieldTy))
           VisitArrayElements(Field, FieldTy, handlers...);
-      } else if (FieldTy->isScalarType() || FieldTy->isUnioType() ||
+      } else if (FieldTy->isScalarType() || FieldTy->isUnionType() ||
 		  FieldTy->isVectorType()) {
         KF_FOR_EACH(handleSimpleType, Field, FieldTy);
       } else

From 4ede3968498174f35f8456cd4bf95d14811d40d1 Mon Sep 17 00:00:00 2001
From: Nathan Ridge <zeratul976@hotmail.com>
Date: Mon, 3 Aug 2020 03:21:01 -0400
Subject: [PATCH 0996/1035] [clang] Include trailing-requires-clause in
 FunctionDecl's source range

Fixes https://github.com/clangd/clangd/issues/476

Differential Revision: https://reviews.llvm.org/D85108
---
 .../clangd/unittests/FindTargetTests.cpp      | 19 ++++++++++++++++---
 clang/include/clang/Sema/DeclSpec.h           |  2 ++
 clang/test/AST/ast-dump-concepts.cpp          |  9 +++++++++
 3 files changed, 27 insertions(+), 3 deletions(-)

diff --git a/clang-tools-extra/clangd/unittests/FindTargetTests.cpp b/clang-tools-extra/clangd/unittests/FindTargetTests.cpp
index 8b872d6314d45..3421b9cec2d30 100644
--- a/clang-tools-extra/clangd/unittests/FindTargetTests.cpp
+++ b/clang-tools-extra/clangd/unittests/FindTargetTests.cpp
@@ -405,6 +405,11 @@ TEST_F(TargetDeclTest, ClassTemplate) {
 }
 
 TEST_F(TargetDeclTest, Concept) {
+  Flags.push_back("-std=c++20");
+
+  // FIXME: Should we truncate the pretty-printed form of a concept decl
+  // somewhere?
+
   Code = R"cpp(
     template <typename T>
     concept Fooable = requires (T t) { t.foo(); };
@@ -414,12 +419,20 @@ TEST_F(TargetDeclTest, Concept) {
       t.foo();
     }
   )cpp";
-  Flags.push_back("-std=c++20");
   EXPECT_DECLS(
       "ConceptSpecializationExpr",
-      // FIXME: Should we truncate the pretty-printed form of a concept decl
-      // somewhere?
       {"template <typename T> concept Fooable = requires (T t) { t.foo(); };"});
+
+  // trailing requires clause
+  Code = R"cpp(
+      template <typename T>
+      concept Fooable = true;
+
+      template <typename T>
+      void foo() requires [[Fooable]]<T>;
+  )cpp";
+  EXPECT_DECLS("ConceptSpecializationExpr",
+               {"template <typename T> concept Fooable = true;"});
 }
 
 TEST_F(TargetDeclTest, FunctionTemplate) {
diff --git a/clang/include/clang/Sema/DeclSpec.h b/clang/include/clang/Sema/DeclSpec.h
index 0a22b5af7c644..93a9126096557 100644
--- a/clang/include/clang/Sema/DeclSpec.h
+++ b/clang/include/clang/Sema/DeclSpec.h
@@ -2448,6 +2448,8 @@ class Declarator {
   /// \brief Sets a trailing requires clause for this declarator.
   void setTrailingRequiresClause(Expr *TRC) {
     TrailingRequiresClause = TRC;
+
+    SetRangeEnd(TRC->getEndLoc());
   }
 
   /// \brief Sets a trailing requires clause for this declarator.
diff --git a/clang/test/AST/ast-dump-concepts.cpp b/clang/test/AST/ast-dump-concepts.cpp
index 3429fa6b46be5..7050ee0fb4492 100644
--- a/clang/test/AST/ast-dump-concepts.cpp
+++ b/clang/test/AST/ast-dump-concepts.cpp
@@ -24,4 +24,13 @@ struct Foo {
   // CHECK-NEXT: `-ConceptSpecializationExpr {{.*}} <col:13> 'bool'
   template <unary_concept R>
   Foo(R);
+
+  // CHECK:      FunctionTemplateDecl {{.*}} <line:29:3, line:30:39> {{.*}} Foo<T>
+  template <typename R>
+  Foo(R, int) requires unary_concept<R>;
+
+  // CHECK:      FunctionTemplateDecl {{.*}} <line:33:3, line:35:3> {{.*}} Foo<T>
+  template <typename R>
+  Foo(R, char) requires unary_concept<R> {
+  }
 };

From 689096965d9af1b00cd90f117c57fd9e7d07a479 Mon Sep 17 00:00:00 2001
From: Jakub Lichman <limo@google.com>
Date: Mon, 3 Aug 2020 07:57:06 +0000
Subject: [PATCH 0997/1035] [mlir][Linalg] Conv ops lowering to std calls
 added.

Lowering of newly defined Conv ops in TC syntax to standard
dialect is not supported and therefore this commit adds support
for it.

Differential Revision: https://reviews.llvm.org/D84840
---
 .../Conversion/LinalgToStandard/LinalgToStandard.cpp  | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/mlir/lib/Conversion/LinalgToStandard/LinalgToStandard.cpp b/mlir/lib/Conversion/LinalgToStandard/LinalgToStandard.cpp
index 55ffa3f8b6e61..d56dffdd0dc17 100644
--- a/mlir/lib/Conversion/LinalgToStandard/LinalgToStandard.cpp
+++ b/mlir/lib/Conversion/LinalgToStandard/LinalgToStandard.cpp
@@ -244,7 +244,16 @@ void mlir::populateLinalgToStandardConversionPatterns(
       LinalgOpConversion<DotOp>,
       LinalgOpConversion<BatchMatmulOp>,
       LinalgOpConversion<MatvecOp>,
-      LinalgOpConversion<MatmulOp>>(ctx);
+      LinalgOpConversion<MatmulOp>,
+      LinalgOpConversion<ConvWOp>,
+      LinalgOpConversion<ConvNWCOp>,
+      LinalgOpConversion<ConvNCWOp>,
+      LinalgOpConversion<ConvHWOp>,
+      LinalgOpConversion<ConvNHWCOp>,
+      LinalgOpConversion<ConvNCHWOp>,
+      LinalgOpConversion<ConvDHWOp>,
+      LinalgOpConversion<ConvNDHWCOp>,
+      LinalgOpConversion<ConvNCDHWOp>>(ctx);
   // clang-format on
 }
 

From 6f97103b561cb14e26aafa3b90ecec97f1d08944 Mon Sep 17 00:00:00 2001
From: Juneyoung Lee <aqjune@gmail.com>
Date: Thu, 30 Jul 2020 23:46:26 +0900
Subject: [PATCH 0998/1035] [JumpThreading] Don't limit the type of an operand

Compared to the optimized code with branch conditions never frozen,
limiting the type of freeze's operand causes generation of suboptimal code in
some cases.
I would like to suggest removing the constraint, as this patch does.
If the number of freeze instructions becomes significant, this can be revisited.

Differential Revision: https://reviews.llvm.org/D84949
---
 llvm/lib/Transforms/Scalar/JumpThreading.cpp |  4 ----
 llvm/test/Transforms/JumpThreading/freeze.ll | 16 +++++-----------
 2 files changed, 5 insertions(+), 15 deletions(-)

diff --git a/llvm/lib/Transforms/Scalar/JumpThreading.cpp b/llvm/lib/Transforms/Scalar/JumpThreading.cpp
index 2a5bbfff1e1e7..f42d4841f7939 100644
--- a/llvm/lib/Transforms/Scalar/JumpThreading.cpp
+++ b/llvm/lib/Transforms/Scalar/JumpThreading.cpp
@@ -693,12 +693,8 @@ bool JumpThreadingPass::ComputeValueKnownInPredecessorsImpl(
     return true;
   }
 
-  // Handle Freeze instructions, in a manner similar to Cast.
   if (FreezeInst *FI = dyn_cast<FreezeInst>(I)) {
     Value *Source = FI->getOperand(0);
-    if (!isa<PHINode>(Source) && !isa<CmpInst>(Source) &&
-        !isa<CastInst>(Source))
-      return false;
     ComputeValueKnownInPredecessorsImpl(Source, BB, Result, Preference,
                                         RecursionSet, CxtI);
 
diff --git a/llvm/test/Transforms/JumpThreading/freeze.ll b/llvm/test/Transforms/JumpThreading/freeze.ll
index 99df46f38a8ec..99fa058ade814 100644
--- a/llvm/test/Transforms/JumpThreading/freeze.ll
+++ b/llvm/test/Transforms/JumpThreading/freeze.ll
@@ -85,20 +85,14 @@ F2:
 
 define i32 @test1_cast2(i1 %cond) {
 ; CHECK-LABEL: @test1_cast2(
-; CHECK-NEXT:    br i1 [[COND:%.*]], label [[MERGE_THREAD:%.*]], label [[MERGE:%.*]]
-; CHECK:       Merge.thread:
-; CHECK-NEXT:    [[V1:%.*]] = call i32 @f1()
-; CHECK-NEXT:    br label [[T2:%.*]]
-; CHECK:       Merge:
-; CHECK-NEXT:    [[V2:%.*]] = call i32 @f2()
-; CHECK-NEXT:    [[A0_FR:%.*]] = freeze i32 0
-; CHECK-NEXT:    [[A_FR:%.*]] = trunc i32 [[A0_FR]] to i1
-; CHECK-NEXT:    br i1 [[A_FR]], label [[T2]], label [[F2:%.*]]
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[T2:%.*]], label [[F2:%.*]]
 ; CHECK:       T2:
-; CHECK-NEXT:    [[B5:%.*]] = phi i32 [ [[V1]], [[MERGE_THREAD]] ], [ [[V2]], [[MERGE]] ]
+; CHECK-NEXT:    [[V1:%.*]] = call i32 @f1()
 ; CHECK-NEXT:    call void @f3()
-; CHECK-NEXT:    ret i32 [[B5]]
+; CHECK-NEXT:    ret i32 [[V1]]
 ; CHECK:       F2:
+; CHECK-NEXT:    [[V2:%.*]] = call i32 @f2()
+; CHECK-NEXT:    [[A0_FR:%.*]] = freeze i32 0
 ; CHECK-NEXT:    ret i32 [[V2]]
 ;
   br i1 %cond, label %T1, label %F1

From 6a78a8dd378b77be002f0243d80322bbdd6513c5 Mon Sep 17 00:00:00 2001
From: Qiu Chaofan <qiucofan@cn.ibm.com>
Date: Tue, 4 Aug 2020 15:48:16 +0800
Subject: [PATCH 0999/1035] [NFC] [PowerPC] Refactor fp/int conversion lowering

For FP_TO_INT and INT_TO_FP lowering, we have direct-move and
non-direct-move methods. But they share some conversion logic, so we can
reduce redundant code by introducing new methods.

Reviewed By: steven.zhang

Differential Revision: https://reviews.llvm.org/D81818
---
 llvm/lib/Target/PowerPC/PPCISelLowering.cpp | 169 ++++++++------------
 1 file changed, 66 insertions(+), 103 deletions(-)

diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index dfe0c2dd5c9cc..ac7f4f9c34f92 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -7915,36 +7915,41 @@ SDValue PPCTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
   return Op;
 }
 
-void PPCTargetLowering::LowerFP_TO_INTForReuse(SDValue Op, ReuseLoadInfo &RLI,
-                                               SelectionDAG &DAG,
-                                               const SDLoc &dl) const {
-  assert(Op.getOperand(0).getValueType().isFloatingPoint());
+static SDValue convertFPToInt(SDValue Op, SelectionDAG &DAG,
+                              const PPCSubtarget &Subtarget) {
+  SDLoc dl(Op);
+  bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT;
   SDValue Src = Op.getOperand(0);
+  assert(Src.getValueType().isFloatingPoint());
   if (Src.getValueType() == MVT::f32)
     Src = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Src);
-
-  SDValue Tmp;
+  SDValue Conv;
   switch (Op.getSimpleValueType().SimpleTy) {
   default: llvm_unreachable("Unhandled FP_TO_INT type in custom expander!");
   case MVT::i32:
-    Tmp = DAG.getNode(
-        Op.getOpcode() == ISD::FP_TO_SINT
-            ? PPCISD::FCTIWZ
-            : (Subtarget.hasFPCVT() ? PPCISD::FCTIWUZ : PPCISD::FCTIDZ),
+    Conv = DAG.getNode(
+        IsSigned ? PPCISD::FCTIWZ
+                 : (Subtarget.hasFPCVT() ? PPCISD::FCTIWUZ : PPCISD::FCTIDZ),
         dl, MVT::f64, Src);
     break;
   case MVT::i64:
-    assert((Op.getOpcode() == ISD::FP_TO_SINT || Subtarget.hasFPCVT()) &&
+    assert((IsSigned || Subtarget.hasFPCVT()) &&
            "i64 FP_TO_UINT is supported only with FPCVT");
-    Tmp = DAG.getNode(Op.getOpcode()==ISD::FP_TO_SINT ? PPCISD::FCTIDZ :
-                                                        PPCISD::FCTIDUZ,
-                      dl, MVT::f64, Src);
-    break;
+    Conv = DAG.getNode(IsSigned ? PPCISD::FCTIDZ : PPCISD::FCTIDUZ, dl,
+                       MVT::f64, Src);
   }
+  return Conv;
+}
+
+void PPCTargetLowering::LowerFP_TO_INTForReuse(SDValue Op, ReuseLoadInfo &RLI,
+                                               SelectionDAG &DAG,
+                                               const SDLoc &dl) const {
+  SDValue Tmp = convertFPToInt(Op, DAG, Subtarget);
+  bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT;
 
   // Convert the FP value to an int value through memory.
   bool i32Stack = Op.getValueType() == MVT::i32 && Subtarget.hasSTFIWX() &&
-    (Op.getOpcode() == ISD::FP_TO_SINT || Subtarget.hasFPCVT());
+                  (IsSigned || Subtarget.hasFPCVT());
   SDValue FIPtr = DAG.CreateStackTemporary(i32Stack ? MVT::i32 : MVT::f64);
   int FI = cast<FrameIndexSDNode>(FIPtr)->getIndex();
   MachinePointerInfo MPI =
@@ -7985,51 +7990,25 @@ SDValue PPCTargetLowering::LowerFP_TO_INTDirectMove(SDValue Op,
                                                     SelectionDAG &DAG,
                                                     const SDLoc &dl) const {
   assert(Op.getOperand(0).getValueType().isFloatingPoint());
-  SDValue Src = Op.getOperand(0);
-
-  if (Src.getValueType() == MVT::f32)
-    Src = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Src);
-
-  SDValue Tmp;
-  switch (Op.getSimpleValueType().SimpleTy) {
-  default: llvm_unreachable("Unhandled FP_TO_INT type in custom expander!");
-  case MVT::i32:
-    Tmp = DAG.getNode(
-        Op.getOpcode() == ISD::FP_TO_SINT
-            ? PPCISD::FCTIWZ
-            : (Subtarget.hasFPCVT() ? PPCISD::FCTIWUZ : PPCISD::FCTIDZ),
-        dl, MVT::f64, Src);
-    Tmp = DAG.getNode(PPCISD::MFVSR, dl, MVT::i32, Tmp);
-    break;
-  case MVT::i64:
-    assert((Op.getOpcode() == ISD::FP_TO_SINT || Subtarget.hasFPCVT()) &&
-           "i64 FP_TO_UINT is supported only with FPCVT");
-    Tmp = DAG.getNode(Op.getOpcode()==ISD::FP_TO_SINT ? PPCISD::FCTIDZ :
-                                                        PPCISD::FCTIDUZ,
-                      dl, MVT::f64, Src);
-    Tmp = DAG.getNode(PPCISD::MFVSR, dl, MVT::i64, Tmp);
-    break;
-  }
-  return Tmp;
+  return DAG.getNode(PPCISD::MFVSR, dl, Op.getSimpleValueType().SimpleTy,
+                     convertFPToInt(Op, DAG, Subtarget));
 }
 
 SDValue PPCTargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG,
                                           const SDLoc &dl) const {
-
+  SDValue Src = Op.getOperand(0);
   // FP to INT conversions are legal for f128.
-  if (Op->getOperand(0).getValueType() == MVT::f128)
+  if (Src.getValueType() == MVT::f128)
     return Op;
 
   // Expand ppcf128 to i32 by hand for the benefit of llvm-gcc bootstrap on
   // PPC (the libcall is not available).
-  if (Op.getOperand(0).getValueType() == MVT::ppcf128) {
+  if (Src.getValueType() == MVT::ppcf128) {
     if (Op.getValueType() == MVT::i32) {
       if (Op.getOpcode() == ISD::FP_TO_SINT) {
-        SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl,
-                                 MVT::f64, Op.getOperand(0),
+        SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::f64, Src,
                                  DAG.getIntPtrConstant(0, dl));
-        SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl,
-                                 MVT::f64, Op.getOperand(0),
+        SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::f64, Src,
                                  DAG.getIntPtrConstant(1, dl));
 
         // Add the two halves of the long double in round-to-zero mode.
@@ -8045,15 +8024,12 @@ SDValue PPCTargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG,
         //  X>=2^31 ? (int)(X-2^31)+0x80000000 : (int)X
         // FIXME: generated code sucks.
         // TODO: Are there fast-math-flags to propagate to this FSUB?
-        SDValue True = DAG.getNode(ISD::FSUB, dl, MVT::ppcf128,
-                                   Op.getOperand(0), Tmp);
+        SDValue True = DAG.getNode(ISD::FSUB, dl, MVT::ppcf128, Src, Tmp);
         True = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, True);
         True = DAG.getNode(ISD::ADD, dl, MVT::i32, True,
                            DAG.getConstant(0x80000000, dl, MVT::i32));
-        SDValue False = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32,
-                                    Op.getOperand(0));
-        return DAG.getSelectCC(dl, Op.getOperand(0), Tmp, True, False,
-                               ISD::SETGE);
+        SDValue False = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, Src);
+        return DAG.getSelectCC(dl, Src, Tmp, True, False, ISD::SETGE);
       }
     }
 
@@ -8172,6 +8148,19 @@ bool PPCTargetLowering::directMoveIsProfitable(const SDValue &Op) const {
   return false;
 }
 
+static SDValue convertIntToFP(SDValue Op, SDValue Src, SelectionDAG &DAG,
+                              const PPCSubtarget &Subtarget) {
+  bool IsSigned = Op.getOpcode() == ISD::SINT_TO_FP;
+  SDLoc dl(Op);
+  // If we have FCFIDS, then use it when converting to single-precision.
+  // Otherwise, convert to double-precision and then round.
+  bool IsSingle = Op.getValueType() == MVT::f32 && Subtarget.hasFPCVT();
+  unsigned ConvOpc = IsSingle ? (IsSigned ? PPCISD::FCFIDS : PPCISD::FCFIDUS)
+                              : (IsSigned ? PPCISD::FCFID : PPCISD::FCFIDU);
+  EVT ConvTy = IsSingle ? MVT::f32 : MVT::f64;
+  return DAG.getNode(ConvOpc, dl, ConvTy, Src);
+}
+
 /// Custom lowers integer to floating point conversions to use
 /// the direct move instructions available in ISA 2.07 to avoid the
 /// need for load/store combinations.
@@ -8183,25 +8172,12 @@ SDValue PPCTargetLowering::LowerINT_TO_FPDirectMove(SDValue Op,
          "Invalid floating point type as target of conversion");
   assert(Subtarget.hasFPCVT() &&
          "Int to FP conversions with direct moves require FPCVT");
-  SDValue FP;
   SDValue Src = Op.getOperand(0);
-  bool SinglePrec = Op.getValueType() == MVT::f32;
   bool WordInt = Src.getSimpleValueType().SimpleTy == MVT::i32;
   bool Signed = Op.getOpcode() == ISD::SINT_TO_FP;
-  unsigned ConvOp = Signed ? (SinglePrec ? PPCISD::FCFIDS : PPCISD::FCFID) :
-                             (SinglePrec ? PPCISD::FCFIDUS : PPCISD::FCFIDU);
-
-  if (WordInt) {
-    FP = DAG.getNode(Signed ? PPCISD::MTVSRA : PPCISD::MTVSRZ,
-                     dl, MVT::f64, Src);
-    FP = DAG.getNode(ConvOp, dl, SinglePrec ? MVT::f32 : MVT::f64, FP);
-  }
-  else {
-    FP = DAG.getNode(PPCISD::MTVSRA, dl, MVT::f64, Src);
-    FP = DAG.getNode(ConvOp, dl, SinglePrec ? MVT::f32 : MVT::f64, FP);
-  }
-
-  return FP;
+  unsigned MovOpc = (WordInt && !Signed) ? PPCISD::MTVSRZ : PPCISD::MTVSRA;
+  SDValue Mov = DAG.getNode(MovOpc, dl, MVT::f64, Src);
+  return convertIntToFP(Op, Mov, DAG, Subtarget);
 }
 
 static SDValue widenVec(SelectionDAG &DAG, SDValue Vec, const SDLoc &dl) {
@@ -8277,8 +8253,10 @@ SDValue PPCTargetLowering::LowerINT_TO_FPVector(SDValue Op, SelectionDAG &DAG,
 SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op,
                                           SelectionDAG &DAG) const {
   SDLoc dl(Op);
+  SDValue Src = Op.getOperand(0);
+  bool IsSigned = Op.getOpcode() == ISD::SINT_TO_FP;
 
-  EVT InVT = Op.getOperand(0).getValueType();
+  EVT InVT = Src.getValueType();
   EVT OutVT = Op.getValueType();
   if (OutVT.isVector() && OutVT.isFloatingPoint() &&
       isOperationCustom(Op.getOpcode(), InVT))
@@ -8292,8 +8270,8 @@ SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op,
   if (Op.getValueType() != MVT::f32 && Op.getValueType() != MVT::f64)
     return SDValue();
 
-  if (Op.getOperand(0).getValueType() == MVT::i1)
-    return DAG.getNode(ISD::SELECT, dl, Op.getValueType(), Op.getOperand(0),
+  if (Src.getValueType() == MVT::i1)
+    return DAG.getNode(ISD::SELECT, dl, Op.getValueType(), Src,
                        DAG.getConstantFP(1.0, dl, Op.getValueType()),
                        DAG.getConstantFP(0.0, dl, Op.getValueType()));
 
@@ -8303,22 +8281,11 @@ SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op,
       Subtarget.isPPC64() && Subtarget.hasFPCVT())
     return LowerINT_TO_FPDirectMove(Op, DAG, dl);
 
-  assert((Op.getOpcode() == ISD::SINT_TO_FP || Subtarget.hasFPCVT()) &&
+  assert((IsSigned || Subtarget.hasFPCVT()) &&
          "UINT_TO_FP is supported only with FPCVT");
 
-  // If we have FCFIDS, then use it when converting to single-precision.
-  // Otherwise, convert to double-precision and then round.
-  unsigned FCFOp = (Subtarget.hasFPCVT() && Op.getValueType() == MVT::f32)
-                       ? (Op.getOpcode() == ISD::UINT_TO_FP ? PPCISD::FCFIDUS
-                                                            : PPCISD::FCFIDS)
-                       : (Op.getOpcode() == ISD::UINT_TO_FP ? PPCISD::FCFIDU
-                                                            : PPCISD::FCFID);
-  MVT FCFTy = (Subtarget.hasFPCVT() && Op.getValueType() == MVT::f32)
-                  ? MVT::f32
-                  : MVT::f64;
-
-  if (Op.getOperand(0).getValueType() == MVT::i64) {
-    SDValue SINT = Op.getOperand(0);
+  if (Src.getValueType() == MVT::i64) {
+    SDValue SINT = Src;
     // When converting to single-precision, we actually need to convert
     // to double-precision first and then round to single-precision.
     // To avoid double-rounding effects during that operation, we have
@@ -8431,7 +8398,7 @@ SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op,
     } else
       Bits = DAG.getNode(ISD::BITCAST, dl, MVT::f64, SINT);
 
-    SDValue FP = DAG.getNode(FCFOp, dl, FCFTy, Bits);
+    SDValue FP = convertIntToFP(Op, Bits, DAG, Subtarget);
 
     if (Op.getValueType() == MVT::f32 && !Subtarget.hasFPCVT())
       FP = DAG.getNode(ISD::FP_ROUND, dl,
@@ -8439,7 +8406,7 @@ SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op,
     return FP;
   }
 
-  assert(Op.getOperand(0).getValueType() == MVT::i32 &&
+  assert(Src.getValueType() == MVT::i32 &&
          "Unhandled INT_TO_FP type in custom expander!");
   // Since we only generate this in 64-bit mode, we can take advantage of
   // 64-bit registers.  In particular, sign extend the input value into the
@@ -8453,15 +8420,13 @@ SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op,
   if (Subtarget.hasLFIWAX() || Subtarget.hasFPCVT()) {
     ReuseLoadInfo RLI;
     bool ReusingLoad;
-    if (!(ReusingLoad = canReuseLoadAddress(Op.getOperand(0), MVT::i32, RLI,
-                                            DAG))) {
+    if (!(ReusingLoad = canReuseLoadAddress(Src, MVT::i32, RLI, DAG))) {
       int FrameIdx = MFI.CreateStackObject(4, Align(4), false);
       SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT);
 
-      SDValue Store =
-          DAG.getStore(DAG.getEntryNode(), dl, Op.getOperand(0), FIdx,
-                       MachinePointerInfo::getFixedStack(
-                           DAG.getMachineFunction(), FrameIdx));
+      SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Src, FIdx,
+                                   MachinePointerInfo::getFixedStack(
+                                       DAG.getMachineFunction(), FrameIdx));
 
       assert(cast<StoreSDNode>(Store)->getMemoryVT() == MVT::i32 &&
              "Expected an i32 store");
@@ -8477,10 +8442,9 @@ SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op,
       MF.getMachineMemOperand(RLI.MPI, MachineMemOperand::MOLoad, 4,
                               RLI.Alignment, RLI.AAInfo, RLI.Ranges);
     SDValue Ops[] = { RLI.Chain, RLI.Ptr };
-    Ld = DAG.getMemIntrinsicNode(Op.getOpcode() == ISD::UINT_TO_FP ?
-                                   PPCISD::LFIWZX : PPCISD::LFIWAX,
-                                 dl, DAG.getVTList(MVT::f64, MVT::Other),
-                                 Ops, MVT::i32, MMO);
+    Ld = DAG.getMemIntrinsicNode(IsSigned ? PPCISD::LFIWAX : PPCISD::LFIWZX, dl,
+                                 DAG.getVTList(MVT::f64, MVT::Other), Ops,
+                                 MVT::i32, MMO);
     if (ReusingLoad)
       spliceIntoChain(RLI.ResChain, Ld.getValue(1), DAG);
   } else {
@@ -8490,8 +8454,7 @@ SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op,
     int FrameIdx = MFI.CreateStackObject(8, Align(8), false);
     SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT);
 
-    SDValue Ext64 = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i64,
-                                Op.getOperand(0));
+    SDValue Ext64 = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i64, Src);
 
     // STD the extended value into the stack slot.
     SDValue Store = DAG.getStore(
@@ -8505,7 +8468,7 @@ SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op,
   }
 
   // FCFID it and return it.
-  SDValue FP = DAG.getNode(FCFOp, dl, FCFTy, Ld);
+  SDValue FP = convertIntToFP(Op, Ld, DAG, Subtarget);
   if (Op.getValueType() == MVT::f32 && !Subtarget.hasFPCVT())
     FP = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, FP,
                      DAG.getIntPtrConstant(0, dl));

From 1ea84653378132091b5b6d31d4f6bf3ec7da7b56 Mon Sep 17 00:00:00 2001
From: Juneyoung Lee <aqjune@gmail.com>
Date: Tue, 4 Aug 2020 17:07:28 +0900
Subject: [PATCH 1000/1035] [JumpThreading] Merge/rename thread-two-bbsN.ll
 tests; NFC

---
 ...ead-two-bbs5.ll => thread-two-bbs-cuda.ll} |   0
 ...ead-two-bbs3.ll => thread-two-bbs-msvc.ll} |   0
 .../JumpThreading/thread-two-bbs.ll           | 184 ++++++++++++++++++
 .../JumpThreading/thread-two-bbs1.ll          |  59 ------
 .../JumpThreading/thread-two-bbs2.ll          |  56 ------
 .../JumpThreading/thread-two-bbs4.ll          |  43 ----
 .../JumpThreading/thread-two-bbs6.ll          |  42 ----
 7 files changed, 184 insertions(+), 200 deletions(-)
 rename llvm/test/Transforms/JumpThreading/{thread-two-bbs5.ll => thread-two-bbs-cuda.ll} (100%)
 rename llvm/test/Transforms/JumpThreading/{thread-two-bbs3.ll => thread-two-bbs-msvc.ll} (100%)
 create mode 100644 llvm/test/Transforms/JumpThreading/thread-two-bbs.ll
 delete mode 100644 llvm/test/Transforms/JumpThreading/thread-two-bbs1.ll
 delete mode 100644 llvm/test/Transforms/JumpThreading/thread-two-bbs2.ll
 delete mode 100644 llvm/test/Transforms/JumpThreading/thread-two-bbs4.ll
 delete mode 100644 llvm/test/Transforms/JumpThreading/thread-two-bbs6.ll

diff --git a/llvm/test/Transforms/JumpThreading/thread-two-bbs5.ll b/llvm/test/Transforms/JumpThreading/thread-two-bbs-cuda.ll
similarity index 100%
rename from llvm/test/Transforms/JumpThreading/thread-two-bbs5.ll
rename to llvm/test/Transforms/JumpThreading/thread-two-bbs-cuda.ll
diff --git a/llvm/test/Transforms/JumpThreading/thread-two-bbs3.ll b/llvm/test/Transforms/JumpThreading/thread-two-bbs-msvc.ll
similarity index 100%
rename from llvm/test/Transforms/JumpThreading/thread-two-bbs3.ll
rename to llvm/test/Transforms/JumpThreading/thread-two-bbs-msvc.ll
diff --git a/llvm/test/Transforms/JumpThreading/thread-two-bbs.ll b/llvm/test/Transforms/JumpThreading/thread-two-bbs.ll
new file mode 100644
index 0000000000000..94832b6502ee9
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/thread-two-bbs.ll
@@ -0,0 +1,184 @@
+; RUN: opt < %s -jump-threading -S -verify | FileCheck %s
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@a = global i32 0, align 4
+
+define void @foo(i32 %cond1, i32 %cond2) {
+; CHECK-LABEL: @foo
+; CHECK-LABEL: entry
+entry:
+  %tobool = icmp eq i32 %cond1, 0
+  br i1 %tobool, label %bb.cond2, label %bb.f1
+
+bb.f1:
+  call void @f1()
+  br label %bb.cond2
+; Verify that we branch on cond2 without checking ptr.
+; CHECK:      call void @f1()
+; CHECK-NEXT: icmp eq i32 %cond2, 0
+; CHECK-NEXT: label %bb.f4, label %bb.f2
+
+bb.cond2:
+  %ptr = phi i32* [ null, %bb.f1 ], [ @a, %entry ]
+  %tobool1 = icmp eq i32 %cond2, 0
+  br i1 %tobool1, label %bb.file, label %bb.f2
+; Verify that we branch on cond2 without checking ptr.
+; CHECK:      icmp eq i32 %cond2, 0
+; CHECK-NEXT: label %bb.f3, label %bb.f2
+
+bb.f2:
+  call void @f2()
+  br label %exit
+
+; Verify that we eliminate this basic block.
+; CHECK-NOT: bb.file:
+bb.file:
+  %cmp = icmp eq i32* %ptr, null
+  br i1 %cmp, label %bb.f4, label %bb.f3
+
+bb.f3:
+  call void @f3()
+  br label %exit
+
+bb.f4:
+  call void @f4()
+  br label %exit
+
+exit:
+  ret void
+}
+
+declare void @f1()
+
+declare void @f2()
+
+declare void @f3()
+
+declare void @f4()
+
+
+define void @foo2(i32 %cond1, i32 %cond2) {
+; CHECK-LABEL: @foo2
+; CHECK-LABEL: entry
+entry:
+  %tobool = icmp ne i32 %cond1, 0
+  br i1 %tobool, label %bb.f1, label %bb.f2
+
+bb.f1:
+  call void @f1()
+  br label %bb.cond2
+; Verify that we branch on cond2 without checking tobool again.
+; CHECK:      call void @f1()
+; CHECK-NEXT: icmp eq i32 %cond2, 0
+; CHECK-NEXT: label %exit, label %bb.f3
+
+bb.f2:
+  call void @f2()
+  br label %bb.cond2
+; Verify that we branch on cond2 without checking tobool again.
+; CHECK:      call void @f2()
+; CHECK-NEXT: icmp eq i32 %cond2, 0
+; CHECK-NEXT: label %exit, label %bb.f4
+
+bb.cond2:
+  %tobool1 = icmp eq i32 %cond2, 0
+  br i1 %tobool1, label %exit, label %bb.cond1again
+
+; Verify that we eliminate this basic block.
+; CHECK-NOT: bb.cond1again:
+bb.cond1again:
+  br i1 %tobool, label %bb.f3, label %bb.f4
+
+bb.f3:
+  call void @f3()
+  br label %exit
+
+bb.f4:
+  call void @f4()
+  br label %exit
+
+exit:
+  ret void
+}
+
+
+; Verify that we do *not* thread any edge.  We used to evaluate
+; constant expressions like:
+;
+;   icmp ugt i8* null, inttoptr (i64 4 to i8*)
+;
+; as "true", causing jump threading to a wrong destination.
+define void @foo3(i8* %arg1, i8* %arg2) {
+; CHECK-LABEL: @foo
+; CHECK-NOT: bb_{{[^ ]*}}.thread:
+entry:
+  %cmp1 = icmp eq i8* %arg1, null
+  br i1 %cmp1, label %bb_bar1, label %bb_end
+
+bb_bar1:
+  call void @bar(i32 1)
+  br label %bb_end
+
+bb_end:
+  %cmp2 = icmp ne i8* %arg2, null
+  br i1 %cmp2, label %bb_cont, label %bb_bar2
+
+bb_bar2:
+  call void @bar(i32 2)
+  br label %bb_exit
+
+bb_cont:
+  %cmp3 = icmp ule i8* %arg1, inttoptr (i64 4 to i8*)
+  br i1 %cmp3, label %bb_exit, label %bb_bar3
+
+bb_bar3:
+  call void @bar(i32 3)
+  br label %bb_exit
+
+bb_exit:
+  ret void
+}
+
+declare void @bar(i32)
+
+
+;; Test that we skip unconditional PredBB when threading jumps through two
+;; successive basic blocks.
+
+define i32 @foo4(i32* %0) {
+; CHECK-LABEL: @f
+; CHECK: br i1 %good, label %pred.bb, label %pred.pred.bb
+entry:
+  %size = call i64 @get_size(i32* %0)
+  %good = icmp ugt i64 %size, 3
+  br i1 %good, label %pred.bb, label %pred.pred.bb
+
+; CHECK:      pred.pred.bb:
+; CHECK:       br label %pred.bb
+; CHECK:      pred.bb:
+; CHECK:       br label %bb
+; CHECK:      bb:
+pred.pred.bb:                                        ; preds = %entry
+  call void @effect()
+  br label %pred.bb
+pred.bb:                                             ; preds = %pred.pred.bb, %entry
+  %v = load i32, i32* %0
+  br label %bb
+
+bb:                                                  ; preds = %pred.bb
+  call void @effect1(i8* blockaddress(@foo4, %bb))
+  br i1 %good, label %cont2, label %cont1
+
+cont1:                                               ; preds = %bb
+  br i1 %good, label %exit, label %cont2
+cont2:                                               ; preds = %bb
+  br label %exit
+exit:                                                ; preds = %cont1, %cont2
+  ret i32 %v
+}
+
+declare i64 @get_size(i32*)
+declare void @effect()
+declare void @effect1(i8*)
diff --git a/llvm/test/Transforms/JumpThreading/thread-two-bbs1.ll b/llvm/test/Transforms/JumpThreading/thread-two-bbs1.ll
deleted file mode 100644
index 1b5f5cb14aeeb..0000000000000
--- a/llvm/test/Transforms/JumpThreading/thread-two-bbs1.ll
+++ /dev/null
@@ -1,59 +0,0 @@
-; RUN: opt < %s -jump-threading -S -verify | FileCheck %s
-
-target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
-target triple = "x86_64-unknown-linux-gnu"
-
-@a = global i32 0, align 4
-
-define void @foo(i32 %cond1, i32 %cond2) {
-; CHECK-LABEL: @foo
-; CHECK-LABEL: entry
-entry:
-  %tobool = icmp eq i32 %cond1, 0
-  br i1 %tobool, label %bb.cond2, label %bb.f1
-
-bb.f1:
-  call void @f1()
-  br label %bb.cond2
-; Verify that we branch on cond2 without checking ptr.
-; CHECK:      call void @f1()
-; CHECK-NEXT: icmp eq i32 %cond2, 0
-; CHECK-NEXT: label %bb.f4, label %bb.f2
-
-bb.cond2:
-  %ptr = phi i32* [ null, %bb.f1 ], [ @a, %entry ]
-  %tobool1 = icmp eq i32 %cond2, 0
-  br i1 %tobool1, label %bb.file, label %bb.f2
-; Verify that we branch on cond2 without checking ptr.
-; CHECK:      icmp eq i32 %cond2, 0
-; CHECK-NEXT: label %bb.f3, label %bb.f2
-
-bb.f2:
-  call void @f2()
-  br label %exit
-
-; Verify that we eliminate this basic block.
-; CHECK-NOT: bb.file:
-bb.file:
-  %cmp = icmp eq i32* %ptr, null
-  br i1 %cmp, label %bb.f4, label %bb.f3
-
-bb.f3:
-  call void @f3()
-  br label %exit
-
-bb.f4:
-  call void @f4()
-  br label %exit
-
-exit:
-  ret void
-}
-
-declare void @f1()
-
-declare void @f2()
-
-declare void @f3()
-
-declare void @f4()
diff --git a/llvm/test/Transforms/JumpThreading/thread-two-bbs2.ll b/llvm/test/Transforms/JumpThreading/thread-two-bbs2.ll
deleted file mode 100644
index ebb7ce013eb07..0000000000000
--- a/llvm/test/Transforms/JumpThreading/thread-two-bbs2.ll
+++ /dev/null
@@ -1,56 +0,0 @@
-; RUN: opt < %s -jump-threading -S -verify | FileCheck %s
-
-target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
-target triple = "x86_64-unknown-linux-gnu"
-
-define void @foo(i32 %cond1, i32 %cond2) {
-; CHECK-LABEL: @foo
-; CHECK-LABEL: entry
-entry:
-  %tobool = icmp ne i32 %cond1, 0
-  br i1 %tobool, label %bb.f1, label %bb.f2
-
-bb.f1:
-  call void @f1()
-  br label %bb.cond2
-; Verify that we branch on cond2 without checking tobool again.
-; CHECK:      call void @f1()
-; CHECK-NEXT: icmp eq i32 %cond2, 0
-; CHECK-NEXT: label %exit, label %bb.f3
-
-bb.f2:
-  call void @f2()
-  br label %bb.cond2
-; Verify that we branch on cond2 without checking tobool again.
-; CHECK:      call void @f2()
-; CHECK-NEXT: icmp eq i32 %cond2, 0
-; CHECK-NEXT: label %exit, label %bb.f4
-
-bb.cond2:
-  %tobool1 = icmp eq i32 %cond2, 0
-  br i1 %tobool1, label %exit, label %bb.cond1again
-
-; Verify that we eliminate this basic block.
-; CHECK-NOT: bb.cond1again:
-bb.cond1again:
-  br i1 %tobool, label %bb.f3, label %bb.f4
-
-bb.f3:
-  call void @f3()
-  br label %exit
-
-bb.f4:
-  call void @f4()
-  br label %exit
-
-exit:
-  ret void
-}
-
-declare void @f1() local_unnamed_addr
-
-declare void @f2() local_unnamed_addr
-
-declare void @f3() local_unnamed_addr
-
-declare void @f4() local_unnamed_addr
diff --git a/llvm/test/Transforms/JumpThreading/thread-two-bbs4.ll b/llvm/test/Transforms/JumpThreading/thread-two-bbs4.ll
deleted file mode 100644
index 6ab757fc191de..0000000000000
--- a/llvm/test/Transforms/JumpThreading/thread-two-bbs4.ll
+++ /dev/null
@@ -1,43 +0,0 @@
-; RUN: opt < %s -jump-threading -S -verify | FileCheck %s
-
-target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
-target triple = "x86_64-unknown-linux-gnu"
-
-; Verify that we do *not* thread any edge.  We used to evaluate
-; constant expressions like:
-;
-;   icmp ugt i8* null, inttoptr (i64 4 to i8*)
-;
-; as "true", causing jump threading to a wrong destination.
-define void @foo(i8* %arg1, i8* %arg2) {
-; CHECK-LABEL: @foo
-; CHECK-NOT: bb_{{[^ ]*}}.thread:
-entry:
-  %cmp1 = icmp eq i8* %arg1, null
-  br i1 %cmp1, label %bb_bar1, label %bb_end
-
-bb_bar1:
-  call void @bar(i32 1)
-  br label %bb_end
-
-bb_end:
-  %cmp2 = icmp ne i8* %arg2, null
-  br i1 %cmp2, label %bb_cont, label %bb_bar2
-
-bb_bar2:
-  call void @bar(i32 2)
-  br label %bb_exit
-
-bb_cont:
-  %cmp3 = icmp ule i8* %arg1, inttoptr (i64 4 to i8*)
-  br i1 %cmp3, label %bb_exit, label %bb_bar3
-
-bb_bar3:
-  call void @bar(i32 3)
-  br label %bb_exit
-
-bb_exit:
-  ret void
-}
-
-declare void @bar(i32)
diff --git a/llvm/test/Transforms/JumpThreading/thread-two-bbs6.ll b/llvm/test/Transforms/JumpThreading/thread-two-bbs6.ll
deleted file mode 100644
index 0d8d2f058b895..0000000000000
--- a/llvm/test/Transforms/JumpThreading/thread-two-bbs6.ll
+++ /dev/null
@@ -1,42 +0,0 @@
-;; Test that we skip unconditional PredBB when threading jumps through two
-;; successive basic blocks.
-; RUN: opt -S -passes='function(jump-threading)' < %s | FileCheck %s
-
-target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
-target triple = "x86_64-unknown-linux-gnu"
-
-define i32 @f(i32* %0) {
-; CHECK-LABEL: @f
-; CHECK: br i1 %good, label %pred.bb, label %pred.pred.bb
-entry:
-  %size = call i64 @get_size(i32* %0)
-  %good = icmp ugt i64 %size, 3
-  br i1 %good, label %pred.bb, label %pred.pred.bb
-
-; CHECK:      pred.pred.bb:
-; CHECK:       br label %pred.bb
-; CHECK:      pred.bb:
-; CHECK:       br label %bb
-; CHECK:      bb:
-pred.pred.bb:                                        ; preds = %entry
-  call void @effect()
-  br label %pred.bb
-pred.bb:                                             ; preds = %pred.pred.bb, %entry
-  %v = load i32, i32* %0
-  br label %bb
-
-bb:                                                  ; preds = %pred.bb
-  call void @effect1(i8* blockaddress(@f, %bb))
-  br i1 %good, label %cont2, label %cont1
-
-cont1:                                               ; preds = %bb
-  br i1 %good, label %exit, label %cont2
-cont2:                                               ; preds = %bb
-  br label %exit
-exit:                                                ; preds = %cont1, %cont2
-  ret i32 %v
-}
-
-declare i64 @get_size(i32*)
-declare void @effect()
-declare void @effect1(i8*)

From 8f576a75661668594ff0ac795e31cdb8df90780b Mon Sep 17 00:00:00 2001
From: Georgii Rymar <grimar@accesssoftek.com>
Date: Mon, 3 Aug 2020 17:21:58 +0300
Subject: [PATCH 1001/1035] [llvm-readobj] - Simplify findSectionByName().
 NFCI.

It turns out that findSectionByName can return
const Elf_Shdr * instead of Expected<>, because its
code never returns an error currently (it reports warnings instead).

Differential revision: https://reviews.llvm.org/D85135
---
 llvm/tools/llvm-readobj/ELFDumper.cpp | 44 ++++++++-------------------
 1 file changed, 13 insertions(+), 31 deletions(-)

diff --git a/llvm/tools/llvm-readobj/ELFDumper.cpp b/llvm/tools/llvm-readobj/ELFDumper.cpp
index 2edca5aaa0b9d..046ade9abf86d 100644
--- a/llvm/tools/llvm-readobj/ELFDumper.cpp
+++ b/llvm/tools/llvm-readobj/ELFDumper.cpp
@@ -353,7 +353,7 @@ template <typename ELFT> class ELFDumper : public ObjDumper {
   void printSymbolsHelper(bool IsDynamic) const;
   std::string getDynamicEntry(uint64_t Type, uint64_t Value) const;
 
-  Expected<const typename ELFT::Shdr *> findSectionByName(StringRef Name) const;
+  const Elf_Shdr *findSectionByName(StringRef Name) const;
 
   const Elf_Shdr *getDotSymtabSec() const { return DotSymtabSec; }
   const Elf_Shdr *getDotCGProfileSec() const { return DotCGProfileSec; }
@@ -2454,7 +2454,7 @@ void printFlags(T Value, ArrayRef<EnumEntry<TFlag>> Flags, raw_ostream &OS) {
 }
 
 template <class ELFT>
-Expected<const typename ELFT::Shdr *>
+const typename ELFT::Shdr *
 ELFDumper<ELFT>::findSectionByName(StringRef Name) const {
   const ELFFile<ELFT> *Obj = ObjF->getELFFile();
   for (const Elf_Shdr &Shdr : cantFail(Obj->sections())) {
@@ -3009,12 +3009,7 @@ Error MipsGOTParser<ELFT>::findGOT(Elf_Dyn_Range DynTable,
 
   // Find static GOT secton.
   if (IsStatic) {
-    Expected<const Elf_Shdr *> GotOrErr = Dumper.findSectionByName(".got");
-    if (!GotOrErr)
-      return GotOrErr.takeError();
-    else
-      GotSec = *GotOrErr;
-
+    GotSec = Dumper.findSectionByName(".got");
     if (!GotSec)
       return Error::success();
 
@@ -3333,19 +3328,14 @@ static void printMipsReginfoData(ScopedPrinter &W,
 
 template <class ELFT> void ELFDumper<ELFT>::printMipsReginfo() {
   const ELFFile<ELFT> *Obj = ObjF->getELFFile();
-  Expected<const Elf_Shdr *> RegInfoOrErr = findSectionByName(".reginfo");
-  if (!RegInfoOrErr) {
-    reportUniqueWarning(RegInfoOrErr.takeError());
-    return;
-  }
-
-  if ((*RegInfoOrErr) == nullptr) {
+  const Elf_Shdr *RegInfo = findSectionByName(".reginfo");
+  if (!RegInfo) {
     W.startLine() << "There is no .reginfo section in the file.\n";
     return;
   }
 
   ArrayRef<uint8_t> Sec = unwrapOrError(ObjF->getFileName(),
-                                        Obj->getSectionContents(*RegInfoOrErr));
+                                        Obj->getSectionContents(RegInfo));
   if (Sec.size() != sizeof(Elf_Mips_RegInfo<ELFT>)) {
     W.startLine() << "The .reginfo section has a wrong size.\n";
     return;
@@ -3358,21 +3348,16 @@ template <class ELFT> void ELFDumper<ELFT>::printMipsReginfo() {
 
 template <class ELFT> void ELFDumper<ELFT>::printMipsOptions() {
   const ELFFile<ELFT> *Obj = ObjF->getELFFile();
-  Expected<const Elf_Shdr *> MipsOptOrErr = findSectionByName(".MIPS.options");
-  if (!MipsOptOrErr) {
-    reportUniqueWarning(MipsOptOrErr.takeError());
-    return;
-  }
-
-  if ((*MipsOptOrErr) == nullptr) {
+  const Elf_Shdr *MipsOpts = findSectionByName(".MIPS.options");
+  if (!MipsOpts) {
     W.startLine() << "There is no .MIPS.options section in the file.\n";
     return;
   }
 
   DictScope GS(W, "MIPS Options");
 
-  ArrayRef<uint8_t> Sec = unwrapOrError(ObjF->getFileName(),
-                                        Obj->getSectionContents(*MipsOptOrErr));
+  ArrayRef<uint8_t> Sec =
+      unwrapOrError(ObjF->getFileName(), Obj->getSectionContents(MipsOpts));
   while (!Sec.empty()) {
     if (Sec.size() < sizeof(Elf_Mips_Options<ELFT>)) {
       W.startLine() << "The .MIPS.options section has a wrong size.\n";
@@ -5982,16 +5967,13 @@ template <class ELFT>
 Expected<const Elf_Mips_ABIFlags<ELFT> *>
 getMipsAbiFlagsSection(const ELFObjectFile<ELFT> *ObjF,
                        const ELFDumper<ELFT> &Dumper) {
-  Expected<const typename ELFT::Shdr *> SecOrErr =
-      Dumper.findSectionByName(".MIPS.abiflags");
-  if (!SecOrErr)
-    return SecOrErr.takeError();
-  if (*SecOrErr == nullptr)
+  const typename ELFT::Shdr *Sec = Dumper.findSectionByName(".MIPS.abiflags");
+  if (Sec == nullptr)
     return nullptr;
 
   const ELFFile<ELFT> *Obj = ObjF->getELFFile();
   constexpr StringRef ErrPrefix = "unable to read the .MIPS.abiflags section: ";
-  Expected<ArrayRef<uint8_t>> DataOrErr = Obj->getSectionContents(*SecOrErr);
+  Expected<ArrayRef<uint8_t>> DataOrErr = Obj->getSectionContents(Sec);
   if (!DataOrErr)
     return createError(ErrPrefix + toString(DataOrErr.takeError()));
 

From 7c4782ce91d66a8447a851362b99bb86a42b7c08 Mon Sep 17 00:00:00 2001
From: Nathan James <n.james93@hotmail.co.uk>
Date: Tue, 4 Aug 2020 09:27:01 +0100
Subject: [PATCH 1002/1035] [clang-tidy] Fix regression in RenamerClangTidy

See bug https://bugs.llvm.org/show_bug.cgi\?id\=46976
---
 clang-tools-extra/clang-tidy/utils/RenamerClangTidyCheck.cpp | 5 +++--
 .../clang-tidy/checkers/readability-identifier-naming.cpp    | 5 +++++
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/clang-tools-extra/clang-tidy/utils/RenamerClangTidyCheck.cpp b/clang-tools-extra/clang-tidy/utils/RenamerClangTidyCheck.cpp
index 040378d980f1a..2d67ca4a16180 100644
--- a/clang-tools-extra/clang-tidy/utils/RenamerClangTidyCheck.cpp
+++ b/clang-tools-extra/clang-tidy/utils/RenamerClangTidyCheck.cpp
@@ -157,6 +157,9 @@ void RenamerClangTidyCheck::addUsage(
   RenamerClangTidyCheck::NamingCheckFailure &Failure =
       NamingCheckFailures[Decl];
 
+  if (!Failure.RawUsageLocs.insert(FixLocation.getRawEncoding()).second)
+    return;
+
   if (!Failure.ShouldFix())
     return;
 
@@ -165,8 +168,6 @@ void RenamerClangTidyCheck::addUsage(
 
   if (!utils::rangeCanBeFixed(Range, SourceMgr))
     Failure.FixStatus = RenamerClangTidyCheck::ShouldFixStatus::InsideMacro;
-
-  Failure.RawUsageLocs.insert(FixLocation.getRawEncoding());
 }
 
 void RenamerClangTidyCheck::addUsage(const NamedDecl *Decl, SourceRange Range,
diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability-identifier-naming.cpp b/clang-tools-extra/test/clang-tidy/checkers/readability-identifier-naming.cpp
index 24c1c4270dec8..fed362bbecdec 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/readability-identifier-naming.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/readability-identifier-naming.cpp
@@ -578,3 +578,8 @@ void Foo() {
 #undef M1
 #undef DUP
 } // namespace scratchspace
+
+template<typename type_t>
+auto GetRes(type_t& Param) -> decltype(Param.res());
+// CHECK-MESSAGES: :[[@LINE-1]]:21: warning: invalid case style for parameter 'Param'
+// CHECK-FIXES: auto GetRes(type_t& a_param) -> decltype(a_param.res());

From 4bfbf74e57ad02c1b6dabe099148e079613cac54 Mon Sep 17 00:00:00 2001
From: Nicolas Vasilache <ntv@google.com>
Date: Tue, 4 Aug 2020 04:28:11 -0400
Subject: [PATCH 1003/1035] [MLIR] Add an integration test for 2 D
 vector.transfer_read

Added a "clone" of the 1D vector's test_transfer_read and added a second dimensionality. The test is not as generic as I would like it to be, because more generic versions appear to break the compiler or the runtime at this stage. As bug are fixed, I will be happy to add another more complete test.

Differential Revision: https://reviews.llvm.org/D83096
---
 .../Vector/CPU/test-transfer-read-2d.mlir     | 59 +++++++++++++++++++
 1 file changed, 59 insertions(+)
 create mode 100644 mlir/integration_test/Dialect/Vector/CPU/test-transfer-read-2d.mlir

diff --git a/mlir/integration_test/Dialect/Vector/CPU/test-transfer-read-2d.mlir b/mlir/integration_test/Dialect/Vector/CPU/test-transfer-read-2d.mlir
new file mode 100644
index 0000000000000..b6146137f4e1f
--- /dev/null
+++ b/mlir/integration_test/Dialect/Vector/CPU/test-transfer-read-2d.mlir
@@ -0,0 +1,59 @@
+// RUN: mlir-opt %s -convert-vector-to-scf -lower-affine -convert-scf-to-std -convert-vector-to-llvm | \
+// RUN: mlir-cpu-runner -e entry -entry-point-result=void  \
+// RUN:   -shared-libs=%mlir_integration_test_dir/libmlir_c_runner_utils%shlibext | \
+// RUN: FileCheck %s
+
+func @transfer_read_2d(%A : memref<?x?xf32>, %base1: index, %base2: index) {
+  %fm42 = constant -42.0: f32
+  %f = vector.transfer_read %A[%base1, %base2], %fm42
+      {permutation_map = affine_map<(d0, d1) -> (d0, d1)>} :
+    memref<?x?xf32>, vector<4x9xf32>
+  vector.print %f: vector<4x9xf32>
+  return
+}
+
+func @transfer_write_2d(%A : memref<?x?xf32>, %base1: index, %base2: index) {
+  %fn1 = constant -1.0 : f32
+  %vf0 = splat %fn1 : vector<1x4xf32>
+  vector.transfer_write %vf0, %A[%base1, %base2]
+    {permutation_map = affine_map<(d0, d1) -> (d0, d1)>} :
+    vector<1x4xf32>, memref<?x?xf32>
+  return
+}
+
+func @entry() {
+  %c0 = constant 0: index
+  %c1 = constant 1: index
+  %c2 = constant 2: index
+  %c3 = constant 3: index
+  %c4 = constant 4: index
+  %c5 = constant 5: index
+  %c8 = constant 5: index
+  %f10 = constant 10.0: f32
+  // work with dims of 4, not of 3
+  %first = constant 3: index
+  %second = constant 4: index
+  %A = alloc(%first, %second) : memref<?x?xf32>
+  scf.for %i = %c0 to %first step %c1 {
+    %i32 = index_cast %i : index to i32
+    %fi = sitofp %i32 : i32 to f32
+    %fi10 = mulf %fi, %f10 : f32
+    scf.for %j = %c0 to %second step %c1 {
+        %j32 = index_cast %j : index to i32
+        %fj = sitofp %j32 : i32 to f32
+        %fres = addf %fi10, %fj : f32
+        store %fres, %A[%i, %j] : memref<?x?xf32>
+    }
+  }
+  // On input, memory contains [[ 0, 1, 2, ...], [10, 11, 12, ...], ...]
+  // Read shifted by 2 and pad with -42:
+  call @transfer_read_2d(%A, %c1, %c2) : (memref<?x?xf32>, index, index) -> ()
+  // Write into memory shifted by 3
+  call @transfer_write_2d(%A, %c3, %c1) : (memref<?x?xf32>, index, index) -> ()
+  // Read shifted by 0 and pad with -42:
+  call @transfer_read_2d(%A, %c0, %c0) : (memref<?x?xf32>, index, index) -> ()
+  return
+}
+
+// CHECK: ( ( 12, 13, -42, -42, -42, -42, -42, -42, -42 ), ( 22, 23, -42, -42, -42, -42, -42, -42, -42 ), ( -42, -42, -42, -42, -42, -42, -42, -42, -42 ), ( -42, -42, -42, -42, -42, -42, -42, -42, -42 ) )
+// CHECK: ( ( 0, 1, 2, 3, -42, -42, -42, -42, -42 ), ( 10, 11, 12, 13, -42, -42, -42, -42, -42 ), ( 20, 21, 22, 23, -42, -42, -42, -42, -42 ), ( -42, -42, -42, -42, -42, -42, -42, -42, -42 ) )

From 5d597e20dfdb8eea4c11375f1ae5940e151246c4 Mon Sep 17 00:00:00 2001
From: Xing GUO <higuoxing@gmail.com>
Date: Tue, 4 Aug 2020 16:37:20 +0800
Subject: [PATCH 1004/1035] [DebugInfo][unittest] Use YAML to generate the
 .debug_loclists section.

DWARFYAML supports generating the .debug_loclists section. We can use it
to simplify tests.

Reviewed By: jhenderson, grimar

Differential Revision: https://reviews.llvm.org/D85179
---
 .../DebugInfo/DWARF/DWARFDieTest.cpp          | 47 +++++++------------
 1 file changed, 16 insertions(+), 31 deletions(-)

diff --git a/llvm/unittests/DebugInfo/DWARF/DWARFDieTest.cpp b/llvm/unittests/DebugInfo/DWARF/DWARFDieTest.cpp
index 9fb6a48157cfb..1d468a956e2b7 100644
--- a/llvm/unittests/DebugInfo/DWARF/DWARFDieTest.cpp
+++ b/llvm/unittests/DebugInfo/DWARF/DWARFDieTest.cpp
@@ -46,42 +46,27 @@ TEST(DWARFDie, getLocations) {
                 BlockData:       [ 0x47 ]
               - Value:           20
               - Value:           25
+    debug_loclists:
+      - AddressSize:      4
+        OffsetEntryCount: 0
+        Lists:
+          - Entries:
+              - Operator: DW_LLE_start_length
+                Values:   [ 0x01, 0x02 ]
+              - Operator: DW_LLE_end_of_list
+          - Entries:
+              - Operator: DW_LLE_startx_length
+                Values:   [ 0x01, 0x02 ]
+              - Operator: DW_LLE_end_of_list
+          - Entries:
+              - Operator: DW_LLE_start_length
+                Values:   [ 0x01, 0x02 ]
+              ## end_of_list intentionally missing.
   )";
   Expected<StringMap<std::unique_ptr<MemoryBuffer>>> Sections =
       DWARFYAML::emitDebugSections(StringRef(yamldata),
                                    /*IsLittleEndian=*/true);
   ASSERT_THAT_EXPECTED(Sections, Succeeded());
-  std::vector<uint8_t> Loclists{
-      // Header
-      0, 0, 0, 0, // Length
-      5, 0,       // Version
-      4,          // Address size
-      0,          // Segment selector size
-      0, 0, 0, 0, // Offset entry count
-      // First location list.
-      DW_LLE_start_length, // First entry
-      1, 0, 0, 0,          // Start offset
-      2,                   // Length
-      0,                   // Expression length
-      DW_LLE_end_of_list,
-      // Second location list.
-      DW_LLE_startx_length, // First entry
-      1,                    // Start index
-      2,                    // Length
-      0,                    // Expression length
-      DW_LLE_end_of_list,
-      // Third location list.
-      DW_LLE_start_length, // First entry
-      1, 0, 0, 0,          // Start offset
-      2,                   // Length
-      0,                   // Expression length
-                           // end_of_list intentionally missing
-  };
-  Loclists[0] = Loclists.size() - 4;
-  Sections->try_emplace(
-      "debug_loclists",
-      MemoryBuffer::getMemBuffer(toStringRef(Loclists), "debug_loclists",
-                                 /*RequiresNullTerminator=*/false));
   std::unique_ptr<DWARFContext> Ctx =
       DWARFContext::create(*Sections, 4, /*isLittleEndian=*/true);
   DWARFCompileUnit *CU = Ctx->getCompileUnitForOffset(0);

From 94862018375f2330b070cbf36085919b13a0c7a4 Mon Sep 17 00:00:00 2001
From: Georgii Rymar <grimar@accesssoftek.com>
Date: Wed, 29 Jul 2020 17:06:44 +0300
Subject: [PATCH 1005/1035] [llvm-readobj/readelf] - Refine the implementation
 of printMipsOptions().

`printMipsOptions()` and the test related has the following issues currently:

1) It does not check the value of Elf_Mips_Options<ELFT>::size field.
2) For ODK_REGINFO options it is possible to read past the end of buffer,
   because there is no check against the `sizeof(Elf_Mips_RegInfo<ELFT>)`.
3) The error about the broken size is just printed to the standard output.
4) The binary input is used for the test.
5) There is no testing for multiple options in the .MIPS.options section,
   though the code supports it.
6) Only llvm-readobj is tested, but not llvm-readelf.
7) "Unsupported MIPS options tag" message does not reveal the tag ID/name.

This patch fixes all of these points.

Differential revision: https://reviews.llvm.org/D84854
---
 .../ELF/Inputs/options.obj.elf-mipsel         | Bin 1720 -> 0 bytes
 .../llvm-readobj/ELF/mips-options-sec.test    |  96 +++++++++++++++++-
 llvm/tools/llvm-readobj/ELFDumper.cpp         |  72 ++++++++++---
 3 files changed, 149 insertions(+), 19 deletions(-)
 delete mode 100644 llvm/test/tools/llvm-readobj/ELF/Inputs/options.obj.elf-mipsel

diff --git a/llvm/test/tools/llvm-readobj/ELF/Inputs/options.obj.elf-mipsel b/llvm/test/tools/llvm-readobj/ELF/Inputs/options.obj.elf-mipsel
deleted file mode 100644
index 6309d87c592bd1db51a22b1ba5d863cd7ece3bbf..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 1720
zcmbtT-D(q25S~ri8vlbR1`)eLLQ|o~w3Rdgg`j8=e<+9`^d_5RvxaVV!|t}!8+`%a
zz?-~*FJQHJ<%N$RB$r;QSG|ifn;Ca|h&MWL&NuVTH)m((?7q4G_(8@nK+J#~B$2QH
zN13UA0-)K5-3FXT<#=-76yVb&4<9Ev_%JE`7-j*C+bg*`s;+kc>fZp04q<6*!pS2O
z^JI&~Ir#j(lpkkH-w(4`JHj(Yc-F{-11AS#)=kyv-NfJ;u0>q;EY~%cy1u`33ZU&;
zo@4IWLD=&>b6wR{#Vkj`uD4^ZRo6Ev)n&8nx_kYbDXzARrzB>Sr~YMtUkN+QYZ#eK
zGOAfKOINipn1i2_G)MjhE(36?Y-boZ5ibA_9sMTr+lYDg({Pc4cxdy5KL=h-^9u=Y
zGS5$taws0>FL4Mw7JP@;P75!{mz6ZE0^7AzWba3y+J3)pd+4;n5Y&^0ThEj~i13ai
zvRb{aYdIklRTu>@tsQg%z>5Xt`H`(QpFUp+BdfgwgP`X{T}r7okUp|nh_CuFcC5$(
z)$iD?T}R=$)mwO+Pb^7S)v9#-fA|v1XMLbAcmnUV<Mr=j0rSKWrmew2Gfkqe-#OLt
zT^v(>;Ro$Adk&FW%5hv^e2u90M5!TdGfdasMUBo;*^o9gGv1R;@AVF*bdJ;wo}ctd
z{m<jl`+tiZJ%0^X+73C8|44Gut03jcuZV#0D`QRLrB3h>=EYA-=Vp;Jv~)it)xV+q
zU+VSyryrAM@~`z*u=Xs#0LLw(MFzh!ettQQl1)U?9_4RL>k|=rH5SPINmA;CA7Fy7
SS|l-UaXrn4j6!-xz5X}SI(fAK

diff --git a/llvm/test/tools/llvm-readobj/ELF/mips-options-sec.test b/llvm/test/tools/llvm-readobj/ELF/mips-options-sec.test
index ef5ece472821e..19cae2627e750 100644
--- a/llvm/test/tools/llvm-readobj/ELF/mips-options-sec.test
+++ b/llvm/test/tools/llvm-readobj/ELF/mips-options-sec.test
@@ -1,11 +1,22 @@
 ## Check that we are able to dump the SHT_MIPS_OPTIONS section using -A properly.
 
-# RUN: llvm-readobj -A %p/Inputs/options.obj.elf-mipsel | FileCheck %s
+## Check we are able to dump multiple MIPS options properly.
+# RUN: yaml2obj %s -o %t1
+# RUN: llvm-readobj -A %t1 | FileCheck %s
+# RUN: llvm-readelf -A %t1 | FileCheck %s
 
 # CHECK:      MIPS Options {
 # CHECK-NEXT:   ODK_REGINFO {
+# CHECK-NEXT:     GP: 0x807060504030201
+# CHECK-NEXT:     General Mask: 0xD0C0B0A
+# CHECK-NEXT:     Co-Proc Mask0: 0x88776655
+# CHECK-NEXT:     Co-Proc Mask1: 0xCCBBAA99
+# CHECK-NEXT:     Co-Proc Mask2: 0x1EFFEEDD
+# CHECK-NEXT:     Co-Proc Mask3: 0x5E4E3E2E
+# CHECK-NEXT:   }
+# CHECK-NEXT:   ODK_REGINFO {
 # CHECK-NEXT:     GP: 0x0
-# CHECK-NEXT:     General Mask: 0xF2000017
+# CHECK-NEXT:     General Mask: 0x0
 # CHECK-NEXT:     Co-Proc Mask0: 0x0
 # CHECK-NEXT:     Co-Proc Mask1: 0x0
 # CHECK-NEXT:     Co-Proc Mask2: 0x0
@@ -14,7 +25,7 @@
 # CHECK-NEXT: }
 
 ## Check that we try to dump the .MIPS.options section when we are able to locate it by name.
-# RUN: yaml2obj --docnum=1 -DNAME=0xffff %s -o %t.err1
+# RUN: yaml2obj -DNAME=0xffff %s -o %t.err1
 # RUN: llvm-readelf -A %t.err1 2>&1 | \
 # RUN:   FileCheck %s -DFILE=%t.err1 --check-prefix=NAME-ERR-FOUND --implicit-check-not=warning:
 # RUN: llvm-readobj -A %t.err1 2>&1 | \
@@ -23,6 +34,22 @@
 # NAME-ERR-FOUND:      warning: '[[FILE]]': unable to read the name of SHT_PROGBITS section with index 1: a section [index 1] has an invalid sh_name (0xffff) offset which goes past the end of the section name string table
 # NAME-ERR-FOUND-NEXT: warning: '[[FILE]]': unable to read the name of SHT_PROGBITS section with index 3: a section [index 3] has an invalid sh_name (0xffff) offset which goes past the end of the section name string table
 # NAME-ERR-FOUND:      MIPS Options {
+# NAME-ERR-FOUND-NEXT:  ODK_REGINFO {
+# NAME-ERR-FOUND-NEXT:    GP: 0x807060504030201
+# NAME-ERR-FOUND-NEXT:    General Mask: 0xD0C0B0A
+# NAME-ERR-FOUND-NEXT:    Co-Proc Mask0: 0x88776655
+# NAME-ERR-FOUND-NEXT:    Co-Proc Mask1: 0xCCBBAA99
+# NAME-ERR-FOUND-NEXT:    Co-Proc Mask2: 0x1EFFEEDD
+# NAME-ERR-FOUND-NEXT:    Co-Proc Mask3: 0x5E4E3E2E
+# NAME-ERR-FOUND-NEXT:  }
+# NAME-ERR-FOUND-NEXT:  ODK_REGINFO {
+# NAME-ERR-FOUND-NEXT:    GP: 0x0
+# NAME-ERR-FOUND-NEXT:    General Mask: 0x0
+# NAME-ERR-FOUND-NEXT:    Co-Proc Mask0: 0x0
+# NAME-ERR-FOUND-NEXT:    Co-Proc Mask1: 0x0
+# NAME-ERR-FOUND-NEXT:    Co-Proc Mask2: 0x0
+# NAME-ERR-FOUND-NEXT:    Co-Proc Mask3: 0x0
+# NAME-ERR-FOUND-NEXT:  }
 # NAME-ERR-FOUND-NEXT: }
 
 --- !ELF
@@ -37,14 +64,75 @@ Sections:
   - Name:   .MIPS.options
     Type:   SHT_MIPS_OPTIONS
     ShName: [[OPTNAME=<none>]]
+    ShSize: [[SECSIZE=<none>]]
+    ContentArray: [ [[KIND=0x1]],           ## Kind. ODK_REGINFO == 1.
+                    [[DESCSIZE=0x28]],      ## Byte size of descriptor, including this header.
+                    0x0, 0x0,               ## Section header index of section affected or 0 for global options.
+                    0x0, 0x0, 0x0, 0x0,     ## Kind-specific information.
+                    0xA, 0xB, 0xC, 0xD,     ## ODK_REGINFO: bit-mask of used general registers.
+                    0x11, 0x22, 0x33, 0x44, ## ODK_REGINFO: unused padding field.
+                    0x55, 0x66, 0x77, 0x88, ## ODK_REGINFO: bit-mask of used co-processor registers (0).
+                    0x99, 0xAA, 0xBB, 0xCC, ## ODK_REGINFO: bit-mask of used co-processor registers (1).
+                    0xDD, 0xEE, 0xFF, 0x1E, ## ODK_REGINFO: bit-mask of used co-processor registers (2).
+                    0x2E, 0x3E, 0x4E, 0x5E, ## ODK_REGINFO: bit-mask of used co-processor registers (3).
+                    0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, ## ODK_REGINFO: gp register value.
+## A descriptor for one more arbirtary supported option.
+                    0x1, 0x28, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+                    0x0, 0x0,  0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+                    0x0, 0x0,  0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+                    0x0, 0x0,  0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ]
   - Type:   SHT_PROGBITS
     ShName: [[NAME=<none>]]
 
 ## Check we report a warning when we are unable to find the .MIPS.options section due to an error.
-# RUN: yaml2obj --docnum=1 -DOPTNAME=0xffff %s -o %t.err2
+# RUN: yaml2obj -DOPTNAME=0xffff %s -o %t.err2
 # RUN: llvm-readelf -A %t.err2 2>&1 | \
 # RUN:   FileCheck %s -DFILE=%t.err2 --check-prefix=NAME-ERR-NOTFOUND --implicit-check-not=warning: --implicit-check-not="MIPS Options"
 # RUN: llvm-readobj -A %t.err2 2>&1 | \
 # RUN:   FileCheck %s -DFILE=%t.err2 --check-prefix=NAME-ERR-NOTFOUND --implicit-check-not=warning: --implicit-check-not="MIPS Options"
 
 # NAME-ERR-NOTFOUND: warning: '[[FILE]]': unable to read the name of SHT_MIPS_OPTIONS section with index 2: a section [index 2] has an invalid sh_name (0xffff) offset which goes past the end of the section name string table
+
+## Check we report a warning when the .MIPS.options section has a size that is less than the
+## size of the .MIPS.options description header.
+
+# RUN: yaml2obj %s -DSECSIZE=0x1 -o %t2
+# RUN: llvm-readelf -A %t2 2>&1 | FileCheck %s -DFILE=%t2 --check-prefix=SEC-SIZE
+# RUN: llvm-readobj -A %t2 2>&1 | FileCheck %s -DFILE=%t2 --check-prefix=SEC-SIZE
+
+# SEC-SIZE:      MIPS Options {
+# SEC-SIZE-NEXT: warning: '[[FILE]]': the .MIPS.options section has an invalid size (0x1)
+# SEC-SIZE-NEXT: }
+
+## Check we report a warning when the .MIPS.options description header has a size
+## that goes past the end of the section.
+
+# RUN: yaml2obj %s -DDESCSIZE=0x51 -o %t3
+# RUN: llvm-readelf -A %t3 2>&1 | FileCheck %s -DFILE=%t3 --check-prefix=DESC-SIZE
+# RUN: llvm-readobj -A %t3 2>&1 | FileCheck %s -DFILE=%t3 --check-prefix=DESC-SIZE
+
+# DESC-SIZE:      IPS Options {
+# DESC-SIZE-NEXT: warning: '[[FILE]]':  a descriptor of size 0x51 at offset 0x0 goes past the end of the .MIPS.options section of size 0x50
+# DESC-SIZE-NEXT: }
+
+## Check we are able to skip unsupported options and continue dumping.
+
+# RUN: yaml2obj %s -DKIND=0x2 -o %t4
+# RUN: llvm-readelf -A %t4 2>&1 | FileCheck %s -DFILE=%t4 -DTAG="ODK_EXCEPTIONS (2)" --check-prefix=KIND
+# RUN: llvm-readobj -A %t4 2>&1 | FileCheck %s -DFILE=%t4 -DTAG="ODK_EXCEPTIONS (2)" --check-prefix=KIND
+
+# RUN: yaml2obj %s -DKIND=0xFF -o %t5
+# RUN: llvm-readelf -A %t5 2>&1 | FileCheck %s -DFILE=%t5 -DTAG="Unknown (255)" --check-prefix=KIND
+# RUN: llvm-readobj -A %t5 2>&1 | FileCheck %s -DFILE=%t5 -DTAG="Unknown (255)" --check-prefix=KIND
+
+# KIND:      MIPS Options {
+# KIND-NEXT:   Unsupported MIPS options tag: [[TAG]]
+# KIND-NEXT:   ODK_REGINFO {
+# KIND-NEXT:     GP: 0x0
+# KIND-NEXT:     General Mask: 0x0
+# KIND-NEXT:     Co-Proc Mask0: 0x0
+# KIND-NEXT:     Co-Proc Mask1: 0x0
+# KIND-NEXT:     Co-Proc Mask2: 0x0
+# KIND-NEXT:     Co-Proc Mask3: 0x0
+# KIND-NEXT:   }
+# KIND-NEXT: }
diff --git a/llvm/tools/llvm-readobj/ELFDumper.cpp b/llvm/tools/llvm-readobj/ELFDumper.cpp
index 046ade9abf86d..2b7a103df9752 100644
--- a/llvm/tools/llvm-readobj/ELFDumper.cpp
+++ b/llvm/tools/llvm-readobj/ELFDumper.cpp
@@ -3346,6 +3346,41 @@ template <class ELFT> void ELFDumper<ELFT>::printMipsReginfo() {
   printMipsReginfoData(W, *Reginfo);
 }
 
+template <class ELFT>
+static Expected<const Elf_Mips_Options<ELFT> *>
+readMipsOptions(const uint8_t *SecBegin, ArrayRef<uint8_t> &SecData,
+                bool &IsSupported) {
+  if (SecData.size() < sizeof(Elf_Mips_Options<ELFT>))
+    return createError("the .MIPS.options section has an invalid size (0x" +
+                       Twine::utohexstr(SecData.size()) + ")");
+
+  auto *O = reinterpret_cast<const Elf_Mips_Options<ELFT> *>(SecData.data());
+  if (O->size > SecData.size()) {
+    const uint64_t Offset = SecData.data() - SecBegin;
+    const uint64_t SecSize = Offset + SecData.size();
+    return createError("a descriptor of size 0x" + Twine::utohexstr(O->size) +
+                       " at offset 0x" + Twine::utohexstr(Offset) +
+                       " goes past the end of the .MIPS.options "
+                       "section of size 0x" +
+                       Twine::utohexstr(SecSize));
+  }
+
+  IsSupported = O->kind == ODK_REGINFO;
+  size_t ExpectedSize =
+      sizeof(Elf_Mips_Options<ELFT>) + sizeof(Elf_Mips_RegInfo<ELFT>);
+
+  if (IsSupported)
+    if (O->size < ExpectedSize)
+      return createError(
+          "a .MIPS.options entry of kind " +
+          Twine(getElfMipsOptionsOdkType(O->kind)) +
+          " has an invalid size (0x" + Twine::utohexstr(O->size) +
+          "), the expected size is 0x" + Twine::utohexstr(ExpectedSize));
+
+  SecData = SecData.drop_front(O->size);
+  return O;
+}
+
 template <class ELFT> void ELFDumper<ELFT>::printMipsOptions() {
   const ELFFile<ELFT> *Obj = ObjF->getELFFile();
   const Elf_Shdr *MipsOpts = findSectionByName(".MIPS.options");
@@ -3356,24 +3391,31 @@ template <class ELFT> void ELFDumper<ELFT>::printMipsOptions() {
 
   DictScope GS(W, "MIPS Options");
 
-  ArrayRef<uint8_t> Sec =
+  ArrayRef<uint8_t> Data =
       unwrapOrError(ObjF->getFileName(), Obj->getSectionContents(MipsOpts));
-  while (!Sec.empty()) {
-    if (Sec.size() < sizeof(Elf_Mips_Options<ELFT>)) {
-      W.startLine() << "The .MIPS.options section has a wrong size.\n";
-      return;
-    }
-    auto *O = reinterpret_cast<const Elf_Mips_Options<ELFT> *>(Sec.data());
-    DictScope GS(W, getElfMipsOptionsOdkType(O->kind));
-    switch (O->kind) {
-    case ODK_REGINFO:
-      printMipsReginfoData(W, O->getRegInfo());
-      break;
-    default:
-      W.startLine() << "Unsupported MIPS options tag.\n";
+  const uint8_t *const SecBegin = Data.begin();
+  while (!Data.empty()) {
+    bool IsSupported;
+    Expected<const Elf_Mips_Options<ELFT> *> OptsOrErr =
+        readMipsOptions<ELFT>(SecBegin, Data, IsSupported);
+    if (!OptsOrErr) {
+      reportUniqueWarning(OptsOrErr.takeError());
       break;
     }
-    Sec = Sec.slice(O->size);
+
+    unsigned Kind = (*OptsOrErr)->kind;
+    const char *Type = getElfMipsOptionsOdkType(Kind);
+    if (!IsSupported) {
+      W.startLine() << "Unsupported MIPS options tag: " << Type << " (" << Kind
+                    << ")\n";
+      continue;
+    }
+
+    DictScope GS(W, Type);
+    if (Kind == ODK_REGINFO)
+      printMipsReginfoData(W, (*OptsOrErr)->getRegInfo());
+    else
+      llvm_unreachable("unexpected .MIPS.options section descriptor kind");
   }
 }
 

From d9d22105792e08267583961a66ab6a53992fff89 Mon Sep 17 00:00:00 2001
From: Georgii Rymar <grimar@accesssoftek.com>
Date: Tue, 4 Aug 2020 11:56:58 +0300
Subject: [PATCH 1006/1035] [llvm-readobj] - An attempt to fix BB.
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

http://lab.llvm.org:8011/builders/clang-cmake-x86_64-avx2-linux/builds/15710/steps/build%20stage%201/logs/stdio
fails with:

/home/ssglocal/clang-cmake-x86_64-avx2-linux/clang-cmake-x86_64-avx2-linux/llvm/llvm/tools/llvm-readobj/ELFDumper.cpp: In function ‘llvm::Expected<const llvm::object::Elf_Mips_Options<ELFT>*> readMipsOptions(const uint8_t*, llvm::ArrayRef<unsigned char>&, bool&)’:
/home/ssglocal/clang-cmake-x86_64-avx2-linux/clang-cmake-x86_64-avx2-linux/llvm/llvm/tools/llvm-readobj/ELFDumper.cpp:3373:19: error: the value of ‘ExpectedSize’ is not usable in a constant expression
     if (O->size < ExpectedSize)
                   ^
/home/ssglocal/clang-cmake-x86_64-avx2-linux/clang-cmake-x86_64-avx2-linux/llvm/llvm/tools/llvm-readobj/ELFDumper.cpp:3369:10: note: ‘size_t ExpectedSize’ is not const
   size_t ExpectedSize =
          ^
/home/ssglocal/clang-cmake-x86_64-avx2-linux/clang-cmake-x86_64-avx2-linux/llvm/llvm/tools/llvm-readobj/ELFDumper.cpp:3373:12: error: parse error in template argument list
     if (O->size < ExpectedSize)
            ^
/home/ssglocal/clang-cmake-x86_64-avx2-linux/clang-cmake-x86_64-avx2-linux/llvm/llvm/tools/llvm-readobj/ELFDumper.cpp: In instantiation of ‘llvm::Expected<const llvm::object::Elf_Mips_Options<ELFT>*> readMipsOptions(const uint8_t*, llvm::ArrayRef<unsigned char>&, bool&) [with ELFT = llvm::object::ELFType<(llvm::support::endianness)0u, true>; uint8_t = unsigned char]’:
/home/ssglocal/clang-cmake-x86_64-avx2-linux/clang-cmake-x86_64-avx2-linux/llvm/llvm/tools/llvm-readobj/ELFDumper.cpp:3400:30:   required from ‘void {anonymous}::ELFDumper<ELFT>::printMipsOptions() [with ELFT = llvm::object::ELFType<(llvm::support::endianness)0u, true>]’
/home/ssglocal/clang-cmake-x86_64-avx2-linux/clang-cmake-x86_64-avx2-linux/llvm/llvm/tools/llvm-readobj/ELFDumper.cpp:2878:21:   required from ‘void {anonymous}::ELFDumper<ELFT>::printArchSpecificInfo() [with ELFT = llvm::object::ELFType<(llvm::support::endianness)0u, true>]’
/home/ssglocal/clang-cmake-x86_64-avx2-linux/clang-cmake-x86_64-avx2-linux/llvm/llvm/tools/llvm-readobj/ELFDumper.cpp:6999:1:   required from here
/home/ssglocal/clang-cmake-x86_64-avx2-linux/clang-cmake-x86_64-avx2-linux/llvm/llvm/tools/llvm-readobj/ELFDumper.cpp:3373:5: error: ‘size’ is not a member template function

Fix: add 2 `const` words to variables.
---
 llvm/tools/llvm-readobj/ELFDumper.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/llvm/tools/llvm-readobj/ELFDumper.cpp b/llvm/tools/llvm-readobj/ELFDumper.cpp
index 2b7a103df9752..3aee021748cb6 100644
--- a/llvm/tools/llvm-readobj/ELFDumper.cpp
+++ b/llvm/tools/llvm-readobj/ELFDumper.cpp
@@ -3354,7 +3354,7 @@ readMipsOptions(const uint8_t *SecBegin, ArrayRef<uint8_t> &SecData,
     return createError("the .MIPS.options section has an invalid size (0x" +
                        Twine::utohexstr(SecData.size()) + ")");
 
-  auto *O = reinterpret_cast<const Elf_Mips_Options<ELFT> *>(SecData.data());
+  const auto *O = reinterpret_cast<const Elf_Mips_Options<ELFT> *>(SecData.data());
   if (O->size > SecData.size()) {
     const uint64_t Offset = SecData.data() - SecBegin;
     const uint64_t SecSize = Offset + SecData.size();
@@ -3366,7 +3366,7 @@ readMipsOptions(const uint8_t *SecBegin, ArrayRef<uint8_t> &SecData,
   }
 
   IsSupported = O->kind == ODK_REGINFO;
-  size_t ExpectedSize =
+  const size_t ExpectedSize =
       sizeof(Elf_Mips_Options<ELFT>) + sizeof(Elf_Mips_RegInfo<ELFT>);
 
   if (IsSupported)

From 141cb8a1eecc0c843cdd4e788a28d2b6715e4dc5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Endre=20F=C3=BCl=C3=B6p?= <endre.fulop@sigmatechnology.se>
Date: Tue, 21 Jul 2020 09:31:42 +0200
Subject: [PATCH 1007/1035] [analyzer] Model iterator random incrementation
 symmetrically

Summary:
In case a pointer iterator is incremented in a binary plus expression
(operator+), where the iterator is on the RHS, IteratorModeling should
now detect, and track the resulting value.

Reviewers: Szelethus, baloghadamsoftware

Reviewed By: baloghadamsoftware

Subscribers: rnkovacs, whisperity, xazax.hun, baloghadamsoftware, szepet, a.sidorin, mikhail.ramalho, Szelethus, donat.nagy, dkrupp, Charusso, steakhal, martong, ASDenysPetrov, cfe-commits

Tags: #clang

Differential Revision: https://reviews.llvm.org/D83190
---
 .../Checkers/IteratorModeling.cpp             | 73 ++++++++++++-------
 clang/test/Analysis/iterator-modeling.cpp     | 42 ++++++++++-
 2 files changed, 87 insertions(+), 28 deletions(-)

diff --git a/clang/lib/StaticAnalyzer/Checkers/IteratorModeling.cpp b/clang/lib/StaticAnalyzer/Checkers/IteratorModeling.cpp
index 632de9e5dc832..ab5e6a1c9991f 100644
--- a/clang/lib/StaticAnalyzer/Checkers/IteratorModeling.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/IteratorModeling.cpp
@@ -109,7 +109,7 @@ class IteratorModeling
                        bool Postfix) const;
   void handleRandomIncrOrDecr(CheckerContext &C, const Expr *CE,
                               OverloadedOperatorKind Op, const SVal &RetVal,
-                              const SVal &LHS, const SVal &RHS) const;
+                              const SVal &Iterator, const SVal &Amount) const;
   void handlePtrIncrOrDecr(CheckerContext &C, const Expr *Iterator,
                            OverloadedOperatorKind OK, SVal Offset) const;
   void handleAdvance(CheckerContext &C, const Expr *CE, SVal RetVal, SVal Iter,
@@ -262,20 +262,30 @@ void IteratorModeling::checkPostStmt(const UnaryOperator *UO,
 
 void IteratorModeling::checkPostStmt(const BinaryOperator *BO,
                                      CheckerContext &C) const {
-  ProgramStateRef State = C.getState();
-  BinaryOperatorKind OK = BO->getOpcode();
-  SVal RVal = State->getSVal(BO->getRHS(), C.getLocationContext());
+  const ProgramStateRef State = C.getState();
+  const BinaryOperatorKind OK = BO->getOpcode();
+  const Expr *const LHS = BO->getLHS();
+  const Expr *const RHS = BO->getRHS();
+  const SVal LVal = State->getSVal(LHS, C.getLocationContext());
+  const SVal RVal = State->getSVal(RHS, C.getLocationContext());
 
   if (isSimpleComparisonOperator(BO->getOpcode())) {
-    SVal LVal = State->getSVal(BO->getLHS(), C.getLocationContext());
     SVal Result = State->getSVal(BO, C.getLocationContext());
     handleComparison(C, BO, Result, LVal, RVal,
                      BinaryOperator::getOverloadedOperator(OK));
   } else if (isRandomIncrOrDecrOperator(OK)) {
-    if (!BO->getRHS()->getType()->isIntegralOrEnumerationType())
+    // In case of operator+ the iterator can be either on the LHS (eg.: it + 1),
+    // or on the RHS (eg.: 1 + it). Both cases are modeled.
+    const bool IsIterOnLHS = BO->getLHS()->getType()->isPointerType();
+    const Expr *const &IterExpr = IsIterOnLHS ? LHS : RHS;
+    const Expr *const &AmountExpr = IsIterOnLHS ? RHS : LHS;
+
+    // The non-iterator side must have an integral or enumeration type.
+    if (!AmountExpr->getType()->isIntegralOrEnumerationType())
       return;
-    handlePtrIncrOrDecr(C, BO->getLHS(),
-                        BinaryOperator::getOverloadedOperator(OK), RVal);
+    const SVal &AmountVal = IsIterOnLHS ? RVal : LVal;
+    handlePtrIncrOrDecr(C, IterExpr, BinaryOperator::getOverloadedOperator(OK),
+                        AmountVal);
   }
 }
 
@@ -368,11 +378,24 @@ IteratorModeling::handleOverloadedOperator(CheckerContext &C,
                                  InstCall->getCXXThisVal(), Call.getArgSVal(0));
           return;
         }
-      } else {
-        if (Call.getNumArgs() >= 2 &&
-              Call.getArgExpr(1)->getType()->isIntegralOrEnumerationType()) {
+      } else if (Call.getNumArgs() >= 2) {
+        const Expr *FirstArg = Call.getArgExpr(0);
+        const Expr *SecondArg = Call.getArgExpr(1);
+        const QualType FirstType = FirstArg->getType();
+        const QualType SecondType = SecondArg->getType();
+
+        if (FirstType->isIntegralOrEnumerationType() ||
+            SecondType->isIntegralOrEnumerationType()) {
+          // In case of operator+ the iterator can be either on the LHS (eg.:
+          // it + 1), or on the RHS (eg.: 1 + it). Both cases are modeled.
+          const bool IsIterFirst = FirstType->isStructureOrClassType();
+          const SVal FirstArg = Call.getArgSVal(0);
+          const SVal SecondArg = Call.getArgSVal(1);
+          const SVal &Iterator = IsIterFirst ? FirstArg : SecondArg;
+          const SVal &Amount = IsIterFirst ? SecondArg : FirstArg;
+
           handleRandomIncrOrDecr(C, OrigExpr, Op, Call.getReturnValue(),
-                                 Call.getArgSVal(0), Call.getArgSVal(1));
+                                 Iterator, Amount);
           return;
         }
       }
@@ -564,35 +587,35 @@ void IteratorModeling::handleDecrement(CheckerContext &C, const SVal &RetVal,
   C.addTransition(State);
 }
 
-void IteratorModeling::handleRandomIncrOrDecr(CheckerContext &C,
-                                              const Expr *CE,
+void IteratorModeling::handleRandomIncrOrDecr(CheckerContext &C, const Expr *CE,
                                               OverloadedOperatorKind Op,
                                               const SVal &RetVal,
-                                              const SVal &LHS,
-                                              const SVal &RHS) const {
+                                              const SVal &Iterator,
+                                              const SVal &Amount) const {
   // Increment or decrement the symbolic expressions which represents the
   // position of the iterator
   auto State = C.getState();
 
-  const auto *Pos = getIteratorPosition(State, LHS);
+  const auto *Pos = getIteratorPosition(State, Iterator);
   if (!Pos)
     return;
 
-  const auto *value = &RHS;
-  SVal val;
-  if (auto loc = RHS.getAs<Loc>()) {
-    val = State->getRawSVal(*loc);
-    value = &val;
+  const auto *Value = &Amount;
+  SVal Val;
+  if (auto LocAmount = Amount.getAs<Loc>()) {
+    Val = State->getRawSVal(*LocAmount);
+    Value = &Val;
   }
 
-  auto &TgtVal = (Op == OO_PlusEqual || Op == OO_MinusEqual) ? LHS : RetVal;
+  const auto &TgtVal =
+      (Op == OO_PlusEqual || Op == OO_MinusEqual) ? Iterator : RetVal;
 
   // `AdvancedState` is a state where the position of `LHS` is advanced. We
   // only need this state to retrieve the new position, but we do not want
   // to change the position of `LHS` (in every case).
-  auto AdvancedState = advancePosition(State, LHS, Op, *value);
+  auto AdvancedState = advancePosition(State, Iterator, Op, *Value);
   if (AdvancedState) {
-    const auto *NewPos = getIteratorPosition(AdvancedState, LHS);
+    const auto *NewPos = getIteratorPosition(AdvancedState, Iterator);
     assert(NewPos &&
            "Iterator should have position after successful advancement");
 
diff --git a/clang/test/Analysis/iterator-modeling.cpp b/clang/test/Analysis/iterator-modeling.cpp
index 0b76b0bfa7232..f1538839d06c8 100644
--- a/clang/test/Analysis/iterator-modeling.cpp
+++ b/clang/test/Analysis/iterator-modeling.cpp
@@ -149,7 +149,7 @@ void copy(const std::vector<int> &v) {
   clang_analyzer_express(clang_analyzer_iterator_position(i2));     // expected-warning-re {{$v.end(){{$}}}}
 }
 
-void plus(const std::vector<int> &v) {
+void plus_lhs(const std::vector<int> &v) {
   auto i1 = v.begin();
 
   clang_analyzer_denote(clang_analyzer_container_begin(v), "$v.begin()");
@@ -161,7 +161,19 @@ void plus(const std::vector<int> &v) {
   clang_analyzer_express(clang_analyzer_iterator_position(i2));     // expected-warning-re{{$v.begin() + 2{{$}}}}
 }
 
-void plus_negative(const std::vector<int> &v) {
+void plus_rhs(const std::vector<int> &v) {
+  auto i1 = v.begin();
+
+  clang_analyzer_denote(clang_analyzer_container_begin(v), "$v.begin()");
+
+  auto i2 = 2 + i1;
+
+  clang_analyzer_eval(clang_analyzer_iterator_container(i2) == &v); // expected-warning{{TRUE}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i1));     // expected-warning-re{{$v.begin(){{$}}}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i2));     // expected-warning-re{{$v.begin() + 2{{$}}}}
+}
+
+void plus_lhs_negative(const std::vector<int> &v) {
   auto i1 = v.end();
 
   clang_analyzer_denote(clang_analyzer_container_end(v), "$v.end()");
@@ -173,6 +185,18 @@ void plus_negative(const std::vector<int> &v) {
   clang_analyzer_express(clang_analyzer_iterator_position(i2));     // expected-warning-re {{$v.end() - 2{{$}}}}
 }
 
+void plus_rhs_negative(const std::vector<int> &v) {
+  auto i1 = v.end();
+
+  clang_analyzer_denote(clang_analyzer_container_end(v), "$v.end()");
+
+  auto i2 = (-2) + i1;
+
+  clang_analyzer_eval(clang_analyzer_iterator_container(i2) == &v); // expected-warning{{TRUE}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i1));     // expected-warning-re {{$v.end(){{$}}}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i2));     // expected-warning-re {{$v.end() - 2{{$}}}}
+}
+
 void minus(const std::vector<int> &v) {
   auto i1 = v.end();
 
@@ -1955,7 +1979,7 @@ void minus_equal_ptr_iterator_variable(const cont_with_ptr_iterator<int> &c,
   i -= n; // no-crash
 }
 
-void plus_ptr_iterator(const cont_with_ptr_iterator<int> &c) {
+void plus_lhs_ptr_iterator(const cont_with_ptr_iterator<int> &c) {
   auto i1 = c.begin();
 
   clang_analyzer_denote(clang_analyzer_container_begin(c), "$c.begin()");
@@ -1967,6 +1991,18 @@ void plus_ptr_iterator(const cont_with_ptr_iterator<int> &c) {
   clang_analyzer_express(clang_analyzer_iterator_position(i2)); // expected-warning{{$c.begin() + 2}}
 }
 
+void plus_rhs_ptr_iterator(const cont_with_ptr_iterator<int> &c) {
+  auto i1 = c.begin();
+
+  clang_analyzer_denote(clang_analyzer_container_begin(c), "$c.begin()");
+
+  auto i2 = 2 + i1;
+
+  clang_analyzer_eval(clang_analyzer_iterator_container(i2) == &c); // expected-warning{{TRUE}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i1));     // expected-warning{{$c.begin()}}
+  clang_analyzer_express(clang_analyzer_iterator_position(i2));     // expected-warning{{$c.begin() + 2}}
+}
+
 void minus_ptr_iterator(const cont_with_ptr_iterator<int> &c) {
   auto i1 = c.end();
 

From bb303a17260108db42b6a567febffb1e281ffe70 Mon Sep 17 00:00:00 2001
From: Georgii Rymar <grimar@accesssoftek.com>
Date: Tue, 4 Aug 2020 12:13:43 +0300
Subject: [PATCH 1008/1035] [llvm-readobj] - A second attempt to fix BB.
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The failure is:
http://lab.llvm.org:8011/builders/clang-cmake-x86_64-avx2-linux/builds/15711/steps/build%20stage%201/logs/stdio

FAILED: /usr/bin/c++  -DGTEST_HAS_RTTI=0 -D_DEBUG -D_GNU_SOURCE -D__STDC_CONSTANT_MACROS -D__STDC_FORMAT_MACROS -D__STDC_LIMIT_MACROS -Itools/llvm-readobj -I/home/ssglocal/clang-cmake-x86_64-avx2-linux/clang-cmake-x86_64-avx2-linux/llvm/llvm/tools/llvm-readobj -Iinclude -I/home/ssglocal/clang-cmake-x86_64-avx2-linux/clang-cmake-x86_64-avx2-linux/llvm/llvm/include -march=broadwell -fPIC -fvisibility-inlines-hidden -Werror=date-time -Wall -Wextra -Wno-unused-parameter -Wwrite-strings -Wcast-qual -Wno-missing-field-initializers -pedantic -Wno-long-long -Wno-maybe-uninitialized -Wdelete-non-virtual-dtor -Wno-comment -fdiagnostics-color -ffunction-sections -fdata-sections -O3     -fno-exceptions -fno-rtti -UNDEBUG -std=c++14 -MD -MT tools/llvm-readobj/CMakeFiles/llvm-readobj.dir/ELFDumper.cpp.o -MF tools/llvm-readobj/CMakeFiles/llvm-readobj.dir/ELFDumper.cpp.o.d -o tools/llvm-readobj/CMakeFiles/llvm-readobj.dir/ELFDumper.cpp.o -c /home/ssglocal/clang-cmake-x86_64-avx2-linux/clang-cmake-x86_64-avx2-linux/llvm/llvm/tools/llvm-readobj/ELFDumper.cpp
/home/ssglocal/clang-cmake-x86_64-avx2-linux/clang-cmake-x86_64-avx2-linux/llvm/llvm/tools/llvm-readobj/ELFDumper.cpp: In function ‘llvm::Expected<const llvm::object::Elf_Mips_Options<ELFT>*> readMipsOptions(const uint8_t*, llvm::ArrayRef<unsigned char>&, bool&)’:
/home/ssglocal/clang-cmake-x86_64-avx2-linux/clang-cmake-x86_64-avx2-linux/llvm/llvm/tools/llvm-readobj/ELFDumper.cpp:3373:12: error: parse error in template argument list
     if (O->size < ExpectedSize)
            ^
/home/ssglocal/clang-cmake-x86_64-avx2-linux/clang-cmake-x86_64-avx2-linux/llvm/llvm/tools/llvm-readobj/ELFDumper.cpp: In instantiation of ‘llvm::Expected<const llvm::object::Elf_Mips_Options<ELFT>*> readMipsOptions(const uint8_t*, llvm::ArrayRef<unsigned char>&, bool&) [with ELFT = llvm::object::ELFType<(llvm::support::endianness)0u, true>; uint8_t = unsigned char]’:
/home/ssglocal/clang-cmake-x86_64-avx2-linux/clang-cmake-x86_64-avx2-linux/llvm/llvm/tools/llvm-readobj/ELFDumper.cpp:3400:30:   required from ‘void {anonymous}::ELFDumper<ELFT>::printMipsOptions() [with ELFT = llvm::object::ELFType<(llvm::support::endianness)0u, true>]’
/home/ssglocal/clang-cmake-x86_64-avx2-linux/clang-cmake-x86_64-avx2-linux/llvm/llvm/tools/llvm-readobj/ELFDumper.cpp:2878:21:   required from ‘void {anonymous}::ELFDumper<ELFT>::printArchSpecificInfo() [with ELFT = llvm::object::ELFType<(llvm::support::endianness)0u, true>]’
/home/ssglocal/clang-cmake-x86_64-avx2-linux/clang-cmake-x86_64-avx2-linux/llvm/llvm/tools/llvm-readobj/ELFDumper.cpp:6999:1:   required from here
/home/ssglocal/clang-cmake-x86_64-avx2-linux/clang-cmake-x86_64-avx2-linux/llvm/llvm/tools/llvm-readobj/ELFDumper.cpp:3373:5: error: ‘size’ is not a member template function
     if (O->size < ExpectedSize)
     ^
/home/ssglocal/clang-cmake-x86_64-avx2-linux/clang-cmake-x86_64-avx2-linux/llvm/llvm/tools/llvm-readobj/ELFDumper.cpp: In instantiation of ‘llvm::Expected<const llvm::object::Elf_Mips_Options<ELFT>*> readMipsOptions(const uint8_t*, llvm::ArrayRef<unsigned char>&, bool&) [with ELFT = llvm::object::ELFType<(llvm::support::endianness)1u, true>; uint8_t = unsigned char]’:
/home/ssglocal/clang-cmake-x86_64-avx2-linux/clang-cmake-x86_64-avx2-linux/llvm/llvm/tools/llvm-readobj/ELFDumper.cpp:3400:30:   required from ‘void {anonymous}::ELFDumper<ELFT>::printMipsOptions() [with ELFT = llvm::object::ELFType<(llvm::support::endianness)1u, true>]’
/home/ssglocal/clang-cmake-x86_64-avx2-linux/clang-cmake-x86_64-avx2-linux/llvm/llvm/tools/llvm-readobj/ELFDumper.cpp:2878:21:   required from ‘void {anonymous}::ELFDumper<ELFT>::printArchSpecificInfo() [with ELFT = llvm::object::ELFType<(llvm::support::endianness)1u, true>]’
/home/ssglocal/clang-cmake-x86_64-avx2-linux/clang-cmake-x86_64-avx2-linux/llvm/llvm/tools/llvm-readobj/ELFDumper.cpp:6999:1:   required from here
/home/ssglocal/clang-cmake-x86_64-avx2-linux/clang-cmake-x86_64-avx2-linux/llvm/llvm/tools/llvm-readobj/ELFDumper.cpp:3373:5: error: ‘size’ is not a member template function
/home/ssglocal/clang-cmake-x86_64-avx2-linux/clang-cmake-x86_64-avx2-linux/llvm/llvm/tools/llvm-readobj/ELFDumper.cpp: In instantiation of ‘llvm::Expected<const llvm::object::Elf_Mips_Options<ELFT>*> readMipsOptions(const uint8_t*, llvm::ArrayRef<unsigned char>&, bool&) [with ELFT = llvm::object::ELFType<(llvm::support::endianness)0u, false>; uint8_t = unsigned char]’:
/home/ssglocal/clang-cmake-x86_64-avx2-linux/clang-cmake-x86_64-avx2-linux/llvm/llvm/tools/llvm-readobj/ELFDumper.cpp:3400:30:   required from ‘void {anonymous}::ELFDumper<ELFT>::printMipsOptions() [with ELFT = llvm::object::ELFType<(llvm::support::endianness)0u, false>]’
/home/ssglocal/clang-cmake-x86_64-avx2-linux/clang-cmake-x86_64-avx2-linux/llvm/llvm/tools/llvm-readobj/ELFDumper.cpp:2878:21:   required from ‘void {anonymous}::ELFDumper<ELFT>::printArchSpecificInfo() [with ELFT = llvm::object::ELFType<(llvm::support::endianness)0u, false>]’
/home/ssglocal/clang-cmake-x86_64-avx2-linux/clang-cmake-x86_64-avx2-linux/llvm/llvm/tools/llvm-readobj/ELFDumper.cpp:6999:1:   required from here
/home/ssglocal/clang-cmake-x86_64-avx2-linux/clang-cmake-x86_64-avx2-linux/llvm/llvm/tools/llvm-readobj/ELFDumper.cpp:3373:5: error: ‘size’ is not a member template function
/home/ssglocal/clang-cmake-x86_64-avx2-linux/clang-cmake-x86_64-avx2-linux/llvm/llvm/tools/llvm-readobj/ELFDumper.cpp: In instantiation of ‘llvm::Expected<const llvm::object::Elf_Mips_Options<ELFT>*> readMipsOptions(const uint8_t*, llvm::ArrayRef<unsigned char>&, bool&) [with ELFT = llvm::object::ELFType<(llvm::support::endianness)1u, false>; uint8_t = unsigned char]’:
/home/ssglocal/clang-cmake-x86_64-avx2-linux/clang-cmake-x86_64-avx2-linux/llvm/llvm/tools/llvm-readobj/ELFDumper.cpp:3400:30:   required from ‘void {anonymous}::ELFDumper<ELFT>::printMipsOptions() [with ELFT = llvm::object::ELFType<(llvm::support::endianness)1u, false>]’
/home/ssglocal/clang-cmake-x86_64-avx2-linux/clang-cmake-x86_64-avx2-linux/llvm/llvm/tools/llvm-readobj/ELFDumper.cpp:2878:21:   required from ‘void {anonymous}::ELFDumper<ELFT>::printArchSpecificInfo() [with ELFT = llvm::object::ELFType<(llvm::support::endianness)1u, false>]’
/home/ssglocal/clang-cmake-x86_64-avx2-linux/clang-cmake-x86_64-avx2-linux/llvm/llvm/tools/llvm-readobj/ELFDumper.cpp:6999:1:   required from here
/home/ssglocal/clang-cmake-x86_64-avx2-linux/clang-cmake-x86_64-avx2-linux/llvm/llvm/tools/llvm-readobj/ELFDumper.cpp:3373:5: error: ‘size’ is not a member template function
ninja: build stopped: subcommand failed.

Fix: stop using `auto` for `O` variable.
---
 llvm/tools/llvm-readobj/ELFDumper.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/llvm/tools/llvm-readobj/ELFDumper.cpp b/llvm/tools/llvm-readobj/ELFDumper.cpp
index 3aee021748cb6..19aae413acec4 100644
--- a/llvm/tools/llvm-readobj/ELFDumper.cpp
+++ b/llvm/tools/llvm-readobj/ELFDumper.cpp
@@ -3354,7 +3354,8 @@ readMipsOptions(const uint8_t *SecBegin, ArrayRef<uint8_t> &SecData,
     return createError("the .MIPS.options section has an invalid size (0x" +
                        Twine::utohexstr(SecData.size()) + ")");
 
-  const auto *O = reinterpret_cast<const Elf_Mips_Options<ELFT> *>(SecData.data());
+  const Elf_Mips_Options<ELFT> *O =
+      reinterpret_cast<const Elf_Mips_Options<ELFT> *>(SecData.data());
   if (O->size > SecData.size()) {
     const uint64_t Offset = SecData.data() - SecBegin;
     const uint64_t SecSize = Offset + SecData.size();

From 0acef82480df05486b52618692ebcab1c9b1f77c Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Tue, 4 Aug 2020 10:16:20 +0100
Subject: [PATCH 1009/1035] [X86][AVX] Add v8f32 'reverse'
 HADD(SHUFFLE,SHUFFLE) test coverage

Shows missed opportunity for HADD on AVX1 targets with a relatively simple lane crossing post shuffle
---
 llvm/test/CodeGen/X86/haddsub-4.ll | 72 ++++++++++++++++++++++++++++++
 1 file changed, 72 insertions(+)

diff --git a/llvm/test/CodeGen/X86/haddsub-4.ll b/llvm/test/CodeGen/X86/haddsub-4.ll
index 31b6617b45a96..720b63431a24c 100644
--- a/llvm/test/CodeGen/X86/haddsub-4.ll
+++ b/llvm/test/CodeGen/X86/haddsub-4.ll
@@ -50,6 +50,78 @@ define <8 x i16> @hadd_reverse2_v8i16(<8 x i16> %a0, <8 x i16> %a1) nounwind {
   ret <8 x i16> %add
 }
 
+define <8 x float> @hadd_reverse_v8f32(<8 x float> %a0, <8 x float> %a1) {
+; SSE-LABEL: hadd_reverse_v8f32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movaps %xmm0, %xmm4
+; SSE-NEXT:    haddps %xmm3, %xmm1
+; SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,0,3,2]
+; SSE-NEXT:    haddps %xmm2, %xmm4
+; SSE-NEXT:    shufps {{.*#+}} xmm4 = xmm4[1,0,3,2]
+; SSE-NEXT:    movaps %xmm1, %xmm0
+; SSE-NEXT:    movaps %xmm4, %xmm1
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: hadd_reverse_v8f32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vshufps {{.*#+}} ymm2 = ymm0[3,1],ymm1[3,1],ymm0[7,5],ymm1[7,5]
+; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm2[2,3,0,1]
+; AVX1-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[2,0],ymm1[2,0],ymm0[6,4],ymm1[6,4]
+; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,0,1]
+; AVX1-NEXT:    vaddps %ymm0, %ymm2, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: hadd_reverse_v8f32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vhaddps %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpermilps {{.*#+}} ymm0 = ymm0[1,0,3,2,5,4,7,6]
+; AVX2-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[2,3,0,1]
+; AVX2-NEXT:    retq
+  %lhs = shufflevector <8 x float> %a0, <8 x float> %a1, <8 x i32> <i32 7, i32 5, i32 15, i32 13, i32 3, i32 1, i32 11, i32 9>
+  %rhs = shufflevector <8 x float> %a0, <8 x float> %a1, <8 x i32> <i32 6, i32 4, i32 14, i32 12, i32 2, i32 0, i32 10, i32 8>
+  %add = fadd <8 x float> %lhs, %rhs
+  ret <8 x float> %add
+}
+
+define <8 x float> @hadd_reverse2_v8f32(<8 x float> %a0, <8 x float> %a1) {
+; SSE-LABEL: hadd_reverse2_v8f32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movaps %xmm0, %xmm4
+; SSE-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,2],xmm0[1,0]
+; SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,2,1,0]
+; SSE-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,2,1,0]
+; SSE-NEXT:    haddps %xmm2, %xmm4
+; SSE-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,2,1,0]
+; SSE-NEXT:    haddps %xmm3, %xmm1
+; SSE-NEXT:    movaps %xmm1, %xmm0
+; SSE-NEXT:    movaps %xmm4, %xmm1
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: hadd_reverse2_v8f32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpermilps {{.*#+}} ymm0 = ymm0[3,2,1,0,7,6,5,4]
+; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,0,1]
+; AVX1-NEXT:    vpermilps {{.*#+}} ymm1 = ymm1[3,2,1,0,7,6,5,4]
+; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm1[2,3,0,1]
+; AVX1-NEXT:    vhaddps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: hadd_reverse2_v8f32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpermilps {{.*#+}} ymm0 = ymm0[3,2,1,0,7,6,5,4]
+; AVX2-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[2,3,0,1]
+; AVX2-NEXT:    vpermilps {{.*#+}} ymm1 = ymm1[3,2,1,0,7,6,5,4]
+; AVX2-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[2,3,0,1]
+; AVX2-NEXT:    vhaddps %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %shuf0 = shufflevector <8 x float> %a0, <8 x float> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  %shuf1 = shufflevector <8 x float> %a1, <8 x float> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  %lhs = shufflevector <8 x float> %shuf0, <8 x float> %shuf1, <8 x i32> <i32 0, i32 2, i32 8, i32 10, i32 4, i32 6, i32 12, i32 14>
+  %rhs = shufflevector <8 x float> %shuf0, <8 x float> %shuf1, <8 x i32> <i32 1, i32 3, i32 9, i32 11, i32 5, i32 7, i32 13, i32 15>
+  %add = fadd <8 x float> %lhs, %rhs
+  ret <8 x float> %add
+}
+
 define <16 x i16> @hadd_reverse_v16i16(<16 x i16> %a0, <16 x i16> %a1) nounwind {
 ; SSE-LABEL: hadd_reverse_v16i16:
 ; SSE:       # %bb.0:

From 4f5ccc72f6a6eba61117806f049ca869233a6864 Mon Sep 17 00:00:00 2001
From: Tatyana Krasnukha <tatyana@synopsys.com>
Date: Mon, 3 Aug 2020 14:30:12 +0300
Subject: [PATCH 1010/1035] Partially revert "[cmake] Make MSVC generate
 appropriate __cplusplus macro definition"

The /Zc:__cplusplus option fixes GTEST_LANG_CXX11 value but not GTEST_HAS_TR1_TUPLE,
so we still need to force the latter off.

Still pass the option since it is required by https://reviews.llvm.org/D78186 too.

Differential Revision: https://reviews.llvm.org/D84023
---
 llvm/cmake/modules/AddLLVM.cmake   | 4 ++++
 llvm/utils/unittest/CMakeLists.txt | 3 +++
 2 files changed, 7 insertions(+)

diff --git a/llvm/cmake/modules/AddLLVM.cmake b/llvm/cmake/modules/AddLLVM.cmake
index 5b5be953767b5..2ff5c29c6b1a5 100644
--- a/llvm/cmake/modules/AddLLVM.cmake
+++ b/llvm/cmake/modules/AddLLVM.cmake
@@ -1405,6 +1405,10 @@ function(add_unittest test_suite test_name)
     set(EXCLUDE_FROM_ALL ON)
   endif()
 
+  # Our current version of gtest uses tr1/tuple which is deprecated on MSVC.
+  # Since LLVM itself requires C++14, we can safely force it off.
+  add_definitions(-DGTEST_HAS_TR1_TUPLE=0)
+
   include_directories(${LLVM_MAIN_SRC_DIR}/utils/unittest/googletest/include)
   include_directories(${LLVM_MAIN_SRC_DIR}/utils/unittest/googlemock/include)
   if (NOT LLVM_ENABLE_THREADS)
diff --git a/llvm/utils/unittest/CMakeLists.txt b/llvm/utils/unittest/CMakeLists.txt
index 9127ebce7b4c3..e7caf37727fce 100644
--- a/llvm/utils/unittest/CMakeLists.txt
+++ b/llvm/utils/unittest/CMakeLists.txt
@@ -19,6 +19,9 @@ include_directories(
   googlemock
   )
 
+# Gtest 1.8.0 uses tr1/tuple which is deprecated on MSVC, so we force it off.
+add_definitions(-DGTEST_HAS_TR1_TUPLE=0)
+
 if(WIN32)
   add_definitions(-DGTEST_OS_WINDOWS=1)
 endif()

From b7856f9d8ded99045fcc35da8db71bf1e41285f6 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Tue, 4 Aug 2020 09:55:38 +0100
Subject: [PATCH 1011/1035] [SCEV] Consolidate some smin/smax folding tests
 into single test file.

This patch moves a few spread out smin/smax tests to smin-smax-folds.ll
and adds additional test cases that expose further potential for
folds.
---
 .../pr46939-trip-count-count-down.ll          |  38 ------
 llvm/test/Analysis/ScalarEvolution/smax.ll    |  15 ---
 .../ScalarEvolution/smin-smax-folds.ll        | 125 ++++++++++++++++++
 3 files changed, 125 insertions(+), 53 deletions(-)
 delete mode 100644 llvm/test/Analysis/ScalarEvolution/pr46939-trip-count-count-down.ll
 delete mode 100644 llvm/test/Analysis/ScalarEvolution/smax.ll
 create mode 100644 llvm/test/Analysis/ScalarEvolution/smin-smax-folds.ll

diff --git a/llvm/test/Analysis/ScalarEvolution/pr46939-trip-count-count-down.ll b/llvm/test/Analysis/ScalarEvolution/pr46939-trip-count-count-down.ll
deleted file mode 100644
index 803652e47cf62..0000000000000
--- a/llvm/test/Analysis/ScalarEvolution/pr46939-trip-count-count-down.ll
+++ /dev/null
@@ -1,38 +0,0 @@
-; RUN: opt -analyze -scalar-evolution %s 2>&1 | FileCheck %s
-
-declare void @iteration()
-
-define void @reverse_loop(i32 %n) {
-; CHECK-LABEL: 'reverse_loop'
-; CHECK-NEXT:  Classifying expressions for: @reverse_loop
-; CHECK-NEXT:    %i.011 = phi i32 [ %n, %for.body.lr.ph ], [ %dec, %for.body ]
-; CHECK-NEXT:    --> {%n,+,-1}<nsw><%for.body> U: full-set S: full-set Exits: 0 LoopDispositions: { %for.body: Computable }
-; CHECK-NEXT:    %dec = add nsw i32 %i.011, -1
-; CHECK-NEXT:    --> {(-1 + %n),+,-1}<nw><%for.body> U: full-set S: full-set Exits: -1 LoopDispositions: { %for.body: Computable }
-; CHECK-NEXT:  Determining loop execution counts for: @reverse_loop
-; CHECK-NEXT:  Loop %for.body: backedge-taken count is %n
-; CHECK-NEXT:  Loop %for.body: max backedge-taken count is 2147483647
-; CHECK-NEXT:  Loop %for.body: Predicated backedge-taken count is %n
-; CHECK-NEXT:   Predicates:
-; CHECK:       Loop %for.body: Trip multiple is 1
-;
-entry:
-  %cmp10 = icmp sgt i32 %n, -1
-  br i1 %cmp10, label %for.body.lr.ph, label %for.cond.cleanup
-
-for.body.lr.ph:
-  br label %for.body
-
-for.body:
-  %i.011 = phi i32 [ %n, %for.body.lr.ph ], [ %dec, %for.body ]
-  call void @iteration()
-  %dec = add nsw i32 %i.011, -1
-  %cmp = icmp sgt i32 %i.011, 0
-  br i1 %cmp, label %for.body, label %for.cond.cleanup.loopexit
-
-for.cond.cleanup.loopexit:
-  br label %for.cond.cleanup
-
-for.cond.cleanup:
-  ret void
-}
diff --git a/llvm/test/Analysis/ScalarEvolution/smax.ll b/llvm/test/Analysis/ScalarEvolution/smax.ll
deleted file mode 100644
index 2b2c81c8e90df..0000000000000
--- a/llvm/test/Analysis/ScalarEvolution/smax.ll
+++ /dev/null
@@ -1,15 +0,0 @@
-; RUN: opt < %s -analyze -enable-new-pm=0 -scalar-evolution | FileCheck %s
-; RUN: opt < %s -disable-output "-passes=print<scalar-evolution>" 2>&1 | FileCheck %s
-; PR1614
-
-; CHECK: -->  (%a smax %b)
-; CHECK: -->  (%a smax %b smax %c)
-; CHECK-NOT: smax
-
-define i32 @x(i32 %a, i32 %b, i32 %c) {
-  %A = icmp sgt i32 %a, %b
-  %B = select i1 %A, i32 %a, i32 %b
-  %C = icmp sle i32 %c, %B
-  %D = select i1 %C, i32 %B, i32 %c
-  ret i32 %D
-}
diff --git a/llvm/test/Analysis/ScalarEvolution/smin-smax-folds.ll b/llvm/test/Analysis/ScalarEvolution/smin-smax-folds.ll
new file mode 100644
index 0000000000000..15ec72317d9ab
--- /dev/null
+++ b/llvm/test/Analysis/ScalarEvolution/smin-smax-folds.ll
@@ -0,0 +1,125 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt -analyze -enable-new-pm=0 -scalar-evolution < %s | FileCheck %s
+; RUN: opt -disable-output "-passes=print<scalar-evolution>" < %s 2>&1 | FileCheck %s
+
+; Tests for smin & smax folds.
+
+; Test case from PR1614.
+define i32 @test_PR1614(i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: 'test_PR1614'
+; CHECK: -->  (%a smax %b)
+; CHECK: -->  (%a smax %b smax %c)
+; CHECK-NOT: smax
+
+  %A = icmp sgt i32 %a, %b
+  %B = select i1 %A, i32 %a, i32 %b
+  %C = icmp sle i32 %c, %B
+  %D = select i1 %C, i32 %B, i32 %c
+  ret i32 %D
+}
+
+declare void @iteration()
+
+; Test case from PR46939.
+; The information from the loop guard can be used to simplify the trip count expression.
+define void @smin_simplify_with_guard(i32 %n) {
+; CHECK-LABEL: 'smin_simplify_with_guard'
+; CHECK-NEXT:  Classifying expressions for: @smin_simplify_with_guard
+; CHECK-NEXT:    %i.011 = phi i32 [ %n, %for.body.lr.ph ], [ %dec, %for.body ]
+; CHECK-NEXT:    --> {%n,+,-1}<nsw><%for.body> U: full-set S: full-set Exits: 0 LoopDispositions: { %for.body: Computable }
+; CHECK-NEXT:    %dec = add nsw i32 %i.011, -1
+; CHECK-NEXT:    --> {(-1 + %n),+,-1}<nw><%for.body> U: full-set S: full-set Exits: -1 LoopDispositions: { %for.body: Computable }
+; CHECK-NEXT:  Determining loop execution counts for: @smin_simplify_with_guard
+; CHECK-NEXT:  Loop %for.body: backedge-taken count is %n
+; CHECK-NEXT:  Loop %for.body: max backedge-taken count is 2147483647
+; CHECK-NEXT:  Loop %for.body: Predicated backedge-taken count is %n
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %for.body: Trip multiple is 1
+;
+entry:
+  %cmp10 = icmp sgt i32 %n, -1
+  br i1 %cmp10, label %for.body.lr.ph, label %for.cond.cleanup
+
+for.body.lr.ph:
+  br label %for.body
+
+for.body:
+  %i.011 = phi i32 [ %n, %for.body.lr.ph ], [ %dec, %for.body ]
+  call void @iteration()
+  %dec = add nsw i32 %i.011, -1
+  %cmp = icmp sgt i32 %i.011, 0
+  br i1 %cmp, label %for.body, label %for.cond.cleanup.loopexit
+
+for.cond.cleanup.loopexit:
+  br label %for.cond.cleanup
+
+for.cond.cleanup:
+  ret void
+}
+
+define void @smin_to_smax(i32 %n) {
+; FIXME: ((-1 * (0 smin %n)) + %n)  is actually just  (0 smax %n)
+
+; CHECK-LABEL: 'smin_to_smax'
+; CHECK-NEXT:  Classifying expressions for: @smin_to_smax
+; CHECK-NEXT:    %i.011 = phi i32 [ %n, %for.body.lr.ph ], [ %dec, %for.body ]
+; CHECK-NEXT:    --> {%n,+,-1}<nsw><%for.body> U: full-set S: full-set Exits: (0 smin %n) LoopDispositions: { %for.body: Computable }
+; CHECK-NEXT:    %dec = add nsw i32 %i.011, -1
+; CHECK-NEXT:    --> {(-1 + %n),+,-1}<nw><%for.body> U: full-set S: full-set Exits: (-1 + (0 smin %n)) LoopDispositions: { %for.body: Computable }
+; CHECK-NEXT:  Determining loop execution counts for: @smin_to_smax
+; CHECK-NEXT:  Loop %for.body: backedge-taken count is ((-1 * (0 smin %n)) + %n)
+; CHECK-NEXT:  Loop %for.body: max backedge-taken count is 2147483647
+; CHECK-NEXT:  Loop %for.body: Predicated backedge-taken count is ((-1 * (0 smin %n)) + %n)
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %for.body: Trip multiple is 1
+;
+entry:
+  br label %for.body.lr.ph
+
+for.body.lr.ph:
+  br label %for.body
+
+for.body:
+  %i.011 = phi i32 [ %n, %for.body.lr.ph ], [ %dec, %for.body ]
+  call void @iteration()
+  %dec = add nsw i32 %i.011, -1
+  %cmp = icmp sgt i32 %i.011, 0
+  br i1 %cmp, label %for.body, label %for.cond.cleanup.loopexit
+
+for.cond.cleanup.loopexit:
+  br label %for.cond.cleanup
+
+for.cond.cleanup:
+  ret void
+}
+
+; The information from the loop guard can be used to simplify the trip count expression.
+define void @smax_simplify_with_guard(i32 %start, i32 %n) {
+; CHECK-LABEL:  'smax_simplify_with_guard'
+; CHECK-NEXT:  Classifying expressions for: @smax_simplify_with_guard
+; CHECK-NEXT:    %k.0.i26 = phi i32 [ %start, %loop.ph ], [ %inc.i, %loop ]
+; CHECK-NEXT:    -->  {%start,+,1}<nsw><%loop> U: full-set S: full-set      Exits: (%start smax %n)     LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %inc.i = add nsw i32 %k.0.i26, 1
+; CHECK-NEXT:    -->  {(1 + %start),+,1}<nw><%loop> U: full-set S: full-set     Exits: (1 + (%start smax %n))       LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:  Determining loop execution counts for: @smax_simplify_with_guard
+; CHECK-NEXT:  Loop %loop: backedge-taken count is ((-1 * %start) + (%start smax %n))
+; CHECK-NEXT:  Loop %loop: max backedge-taken count is -1
+; CHECK-NEXT:  Loop %loop: Predicated backedge-taken count is ((-1 * %start) + (%start smax %n))
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %loop: Trip multiple is 1
+entry:
+  %guard = icmp sge i32 %n, %start
+  br i1 %guard, label %loop.ph, label %exit
+
+loop.ph:
+  br label %loop
+
+loop:
+  %k.0.i26 = phi i32 [ %start, %loop.ph ], [ %inc.i, %loop ]
+  %inc.i = add nsw i32 %k.0.i26, 1
+  %cmp26.not.i.not = icmp slt i32 %k.0.i26, %n
+  br i1 %cmp26.not.i.not, label %loop, label %exit
+
+exit:
+  ret void
+}

From f7658241cb27491b4160a1f7060ef883bc535d09 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Tue, 4 Aug 2020 10:25:16 +0100
Subject: [PATCH 1012/1035] [AArch64] Consider instruction-level contract FMFs
 in combiner patterns.

Currently, instruction level fast math flags are not considered when
generating patterns for the machine combiner.

This currently leads to some missed opportunities to generate FMAs in
combination with `#pragma clang fp contract (fast)`.

For example, when building the example below with -O3 for AArch64, no
FMADD is generated. If built with -O2 and the DAGCombiner is used
instead of the MachineCombiner for FMAs, an FMADD is generated.

With this patch, the same code is generated in both cases.

    float madd_contract(float a, float b, float c) {
    #pragma clang fp contract (fast)
      return (a * b) + c;
    }

Reviewed By: dmgreen

Differential Revision: https://reviews.llvm.org/D84930
---
 llvm/lib/Target/AArch64/AArch64InstrInfo.cpp  | 10 ++++--
 .../AArch64/machine-combiner-instr-fmf.mir    | 32 ++++++++-----------
 llvm/test/CodeGen/AArch64/neon-fma-FMF.ll     |  1 +
 3 files changed, 21 insertions(+), 22 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index 323ac76e903fd..b6fda6b367bf2 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -3861,7 +3861,7 @@ static bool isCombineInstrCandidate64(unsigned Opc) {
   return false;
 }
 
-// FP Opcodes that can be combined with a FMUL
+// FP Opcodes that can be combined with a FMUL.
 static bool isCombineInstrCandidateFP(const MachineInstr &Inst) {
   switch (Inst.getOpcode()) {
   default:
@@ -3883,8 +3883,12 @@ static bool isCombineInstrCandidateFP(const MachineInstr &Inst) {
   case AArch64::FSUBv2f64:
   case AArch64::FSUBv4f32:
     TargetOptions Options = Inst.getParent()->getParent()->getTarget().Options;
-    return (Options.UnsafeFPMath ||
-            Options.AllowFPOpFusion == FPOpFusion::Fast);
+    // We can fuse FADD/FSUB with FMUL, if fusion is either allowed globally by
+    // the target options or if FADD/FSUB has the contract fast-math flag.
+    return Options.UnsafeFPMath ||
+           Options.AllowFPOpFusion == FPOpFusion::Fast ||
+           Inst.getFlag(MachineInstr::FmContract);
+    return true;
   }
   return false;
 }
diff --git a/llvm/test/CodeGen/AArch64/machine-combiner-instr-fmf.mir b/llvm/test/CodeGen/AArch64/machine-combiner-instr-fmf.mir
index 23ed96a42b513..992e636011b6b 100644
--- a/llvm/test/CodeGen/AArch64/machine-combiner-instr-fmf.mir
+++ b/llvm/test/CodeGen/AArch64/machine-combiner-instr-fmf.mir
@@ -6,8 +6,7 @@
 # CHECK:        [[C:%.*]]:fpr32 = COPY $s2
 # CHECK-NEXT:   [[B:%.*]]:fpr32 = COPY $s1
 # CHECK-NEXT:   [[A:%.*]]:fpr32 = COPY $s0
-# CHECK-NEXT:   [[MUL:%.*]]:fpr32 = nnan ninf nsz arcp contract afn reassoc FMULSrr [[B]], [[A]]
-# CHECK-NEXT:   fpr32 = nnan ninf nsz arcp contract afn reassoc FADDSrr killed [[MUL]], [[C]]
+# CHECK-NEXT:   :fpr32 = FMADDSrrr [[B]], [[A]], [[C]]
 ---
 name:            scalar_fmadd_fast
 alignment:       4
@@ -46,8 +45,7 @@ body:             |
 # CHECK:        [[C:%.*]]:fpr32 = COPY $s2
 # CHECK-NEXT:   [[B:%.*]]:fpr32 = COPY $s1
 # CHECK-NEXT:   [[A:%.*]]:fpr32 = COPY $s0
-# CHECK-NEXT:   [[MUL:%.*]]:fpr32 = contract FMULSrr [[B]], [[A]]
-# CHECK-NEXT:   fpr32 = contract FADDSrr killed [[MUL]], [[C]]
+# CHECK-NEXT:   :fpr32 = FMADDSrrr [[B]], [[A]], [[C]]
 
 ---
 name:            scalar_fmadd_contract
@@ -81,7 +79,7 @@ body:             |
 
 ...
 
-# Do not create FMADD, because we don't have the contract flag on moth instructions.
+# Do not create FMADD, because we don't have the contract flag on the FADD.
 
 # CHECK-LABEL: name: scalar_fmadd_contract_op0
 # CHECK:        [[C:%.*]]:fpr32 = COPY $s2
@@ -121,14 +119,13 @@ body:             |
 
 ...
 
-# Do not create FMADD, because we don't have the contract flag on moth instructions.
+# Do create FMADD, because we have the contract flag on the FADD.
 #
 # CHECK-LABEL: name: scalar_fmadd_contract_op1
 # CHECK:        [[C:%.*]]:fpr32 = COPY $s2
 # CHECK-NEXT:   [[B:%.*]]:fpr32 = COPY $s1
 # CHECK-NEXT:   [[A:%.*]]:fpr32 = COPY $s0
-# CHECK-NEXT:   [[MUL:%.*]]:fpr32 = FMULSrr [[B]], [[A]]
-# CHECK-NEXT:   fpr32 = contract FADDSrr killed [[MUL]], [[C]]
+# CHECK-NEXT:   :fpr32 = FMADDSrrr [[B]], [[A]], [[C]]
 
 ---
 name:            scalar_fmadd_contract_op1
@@ -203,14 +200,13 @@ body:             |
 
 ...
 
-# Can create FMADD, because both the fmul and fadd have all fast-math flags.
+# Can create FMLA, because both the fmul and fadd have all fast-math flags.
 #
 # CHECK-LABEL: name: vector_fmadd_fast
 # CHECK:       [[C:%.*]]:fpr128 = COPY $q2
 # CHECK-NEXT:  [[B:%.*]]:fpr128 = COPY $q1
 # CHECK-NEXT:  [[A:%.*]]:fpr128 = COPY $q0
-# CHECK-NEXT:  [[MUL:%.*]]:fpr128 = nnan ninf nsz arcp contract afn reassoc FMULv2f64 [[B]], [[A]]
-# CHECK-NEXT:  fpr128 = nnan ninf nsz arcp contract afn reassoc FADDv2f64 killed [[MUL]], [[C]]
+# CHECK-NEXT:  fpr128 = FMLAv2f64 [[C]], [[B]], [[A]]
 ---
 name:            vector_fmadd_fast
 alignment:       4
@@ -243,14 +239,13 @@ body:             |
 
 ...
 
-# Can create FMADD, because both the fmul and fadd have the contract fast-math flag.
+# Can create FMLA, because both the fmul and fadd have the contract fast-math flag.
 #
 # CHECK-LABEL: name: vector_fmadd_contract
 # CHECK:       [[C:%.*]]:fpr128 = COPY $q2
 # CHECK-NEXT:  [[B:%.*]]:fpr128 = COPY $q1
 # CHECK-NEXT:  [[A:%.*]]:fpr128 = COPY $q0
-# CHECK-NEXT:  [[MUL:%.*]]:fpr128 = contract FMULv2f64 [[B]], [[A]]
-# CHECK-NEXT:  fpr128 = contract FADDv2f64 killed [[MUL]], [[C]]
+# CHECK-NEXT:  fpr128 = FMLAv2f64 [[C]], [[B]], [[A]]
 ---
 name:            vector_fmadd_contract
 alignment:       4
@@ -283,7 +278,7 @@ body:             |
 
 ...
 
-# Do not create FMADD, because we don't have the contract flag on moth instructions.
+# Do not create FMLA, because we don't have the contract flag on the FADD.
 #
 # CHECK-LABEL: name: vector_fmadd_contract_op0
 # CHECK:       [[C:%.*]]:fpr128 = COPY $q2
@@ -323,14 +318,13 @@ body:             |
 
 ...
 
-# Do not create FMADD, because we don't have the contract flag on moth instructions.
+# Do create FMLA, because we have the contract flag on the FADD.
 #
 # CHECK-LABEL: name: vector_fmadd_contract_op1
 # CHECK:       [[C:%.*]]:fpr128 = COPY $q2
 # CHECK-NEXT:  [[B:%.*]]:fpr128 = COPY $q1
 # CHECK-NEXT:  [[A:%.*]]:fpr128 = COPY $q0
-# CHECK-NEXT:  [[MUL:%.*]]:fpr128 = FMULv2f64 [[B]], [[A]]
-# CHECK-NEXT:  fpr128 = contract FADDv2f64 killed [[MUL]], [[C]]
+# CHECK-NEXT:  fpr128 = FMLAv2f64 [[C]], [[B]], [[A]]
 
 ---
 name:            vector_fmadd_contract_op1
@@ -364,7 +358,7 @@ body:             |
 
 ...
 
-# Do not create FMADD, as nsz flag does not allow it.
+# Do not create FMLA, as nsz flag does not allow it.
 #
 # CHECK-LABEL: name: vector_fmadd_nsz
 # CHECK:       [[C:%.*]]:fpr128 = COPY $q2
diff --git a/llvm/test/CodeGen/AArch64/neon-fma-FMF.ll b/llvm/test/CodeGen/AArch64/neon-fma-FMF.ll
index 893d153801a71..0eb173396ce97 100644
--- a/llvm/test/CodeGen/AArch64/neon-fma-FMF.ll
+++ b/llvm/test/CodeGen/AArch64/neon-fma-FMF.ll
@@ -1,4 +1,5 @@
 ; RUN: llc < %s -verify-machineinstrs -mtriple=aarch64-none-linux-gnu -mattr=+neon | FileCheck %s
+; RUN: llc < %s -O3 -verify-machineinstrs -mtriple=aarch64-none-linux-gnu -mattr=+neon | FileCheck %s
 
 define <2 x float> @fma_1(<2 x float> %A, <2 x float> %B, <2 x float> %C) {
 ; CHECK-LABEL: fma_1:

From 47cea9e82dda941e49a2d9cc11f0414436336d46 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Tue, 4 Aug 2020 10:32:27 +0100
Subject: [PATCH 1013/1035] Revert rG66e7dce714fab "Revert "[X86][SSE] Shuffle
 combine blends to OR(X,Y) if the relevant elements are known zero.""

[X86][SSE] Shuffle combine blends to OR(X,Y) if the relevant elements are known zero (REAPPLIED)

This allows us to remove the (depth violating) code in getFauxShuffleMask where we were combining the OR(SHUFFLE,SHUFFLE) shuffle inputs as well, and not just the OR().

This is a minor step toward being able to shuffle combine from/to SELECT/BLENDV as a faux shuffle.

Reapplied with fixed signed/unsigned comparisons.
---
 llvm/lib/Target/X86/X86ISelLowering.cpp       | 65 ++++++++++++++-----
 llvm/test/CodeGen/X86/insertelement-ones.ll   | 12 ++--
 llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll |  8 +--
 .../test/CodeGen/X86/vector-shuffle-128-v8.ll |  5 +-
 .../CodeGen/X86/vector-shuffle-256-v32.ll     |  6 +-
 5 files changed, 61 insertions(+), 35 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index b80a23f5a608e..79047c90ff999 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -7401,8 +7401,8 @@ static bool getFauxShuffleMask(SDValue N, const APInt &DemandedElts,
 
     // Handle OR(SHUFFLE,SHUFFLE) case where one source is zero and the other
     // is a valid shuffle index.
-    SDValue N0 = peekThroughOneUseBitcasts(N.getOperand(0));
-    SDValue N1 = peekThroughOneUseBitcasts(N.getOperand(1));
+    SDValue N0 = peekThroughBitcasts(N.getOperand(0));
+    SDValue N1 = peekThroughBitcasts(N.getOperand(1));
     if (!N0.getValueType().isVector() || !N1.getValueType().isVector())
       return false;
     SmallVector<int, 64> SrcMask0, SrcMask1;
@@ -7413,34 +7413,24 @@ static bool getFauxShuffleMask(SDValue N, const APInt &DemandedElts,
                                 true))
       return false;
 
-    // Shuffle inputs must be the same size as the result.
-    if (llvm::any_of(SrcInputs0, [VT](SDValue Op) {
-          return VT.getSizeInBits() != Op.getValueSizeInBits();
-        }))
-      return false;
-    if (llvm::any_of(SrcInputs1, [VT](SDValue Op) {
-          return VT.getSizeInBits() != Op.getValueSizeInBits();
-        }))
-      return false;
-
     size_t MaskSize = std::max(SrcMask0.size(), SrcMask1.size());
     SmallVector<int, 64> Mask0, Mask1;
     narrowShuffleMaskElts(MaskSize / SrcMask0.size(), SrcMask0, Mask0);
     narrowShuffleMaskElts(MaskSize / SrcMask1.size(), SrcMask1, Mask1);
-    for (size_t i = 0; i != MaskSize; ++i) {
+    for (int i = 0; i != (int)MaskSize; ++i) {
       if (Mask0[i] == SM_SentinelUndef && Mask1[i] == SM_SentinelUndef)
         Mask.push_back(SM_SentinelUndef);
       else if (Mask0[i] == SM_SentinelZero && Mask1[i] == SM_SentinelZero)
         Mask.push_back(SM_SentinelZero);
       else if (Mask1[i] == SM_SentinelZero)
-        Mask.push_back(Mask0[i]);
+        Mask.push_back(i);
       else if (Mask0[i] == SM_SentinelZero)
-        Mask.push_back(Mask1[i] + (int)(MaskSize * SrcInputs0.size()));
+        Mask.push_back(i + MaskSize);
       else
         return false;
     }
-    Ops.append(SrcInputs0.begin(), SrcInputs0.end());
-    Ops.append(SrcInputs1.begin(), SrcInputs1.end());
+    Ops.push_back(N0);
+    Ops.push_back(N1);
     return true;
   }
   case ISD::INSERT_SUBVECTOR: {
@@ -34219,6 +34209,7 @@ static bool matchBinaryShuffle(MVT MaskVT, ArrayRef<int> Mask,
                                SelectionDAG &DAG, const X86Subtarget &Subtarget,
                                unsigned &Shuffle, MVT &SrcVT, MVT &DstVT,
                                bool IsUnary) {
+  unsigned NumMaskElts = Mask.size();
   unsigned EltSizeInBits = MaskVT.getScalarSizeInBits();
 
   if (MaskVT.is128BitVector()) {
@@ -34276,6 +34267,46 @@ static bool matchBinaryShuffle(MVT MaskVT, ArrayRef<int> Mask,
     }
   }
 
+  // Attempt to match against a OR if we're performing a blend shuffle and the
+  // non-blended source element is zero in each case.
+  if ((EltSizeInBits % V1.getScalarValueSizeInBits()) == 0 &&
+      (EltSizeInBits % V2.getScalarValueSizeInBits()) == 0) {
+    bool IsBlend = true;
+    unsigned NumV1Elts = V1.getValueType().getVectorNumElements();
+    unsigned NumV2Elts = V2.getValueType().getVectorNumElements();
+    unsigned Scale1 = NumV1Elts / NumMaskElts;
+    unsigned Scale2 = NumV2Elts / NumMaskElts;
+    APInt DemandedZeroV1 = APInt::getNullValue(NumV1Elts);
+    APInt DemandedZeroV2 = APInt::getNullValue(NumV2Elts);
+    for (unsigned i = 0; i != NumMaskElts; ++i) {
+      int M = Mask[i];
+      if (M == SM_SentinelUndef)
+        continue;
+      if (M == SM_SentinelZero) {
+        DemandedZeroV1.setBits(i * Scale1, (i + 1) * Scale1);
+        DemandedZeroV2.setBits(i * Scale2, (i + 1) * Scale2);
+        continue;
+      }
+      if (M == (int)i) {
+        DemandedZeroV2.setBits(i * Scale2, (i + 1) * Scale2);
+        continue;
+      }
+      if (M == (int)(i + NumMaskElts)) {
+        DemandedZeroV1.setBits(i * Scale1, (i + 1) * Scale1);
+        continue;
+      }
+      IsBlend = false;
+      break;
+    }
+    if (IsBlend &&
+        DAG.computeKnownBits(V1, DemandedZeroV1).isZero() &&
+        DAG.computeKnownBits(V2, DemandedZeroV2).isZero()) {
+      Shuffle = ISD::OR;
+      SrcVT = DstVT = EVT(MaskVT).changeTypeToInteger().getSimpleVT();
+      return true;
+    }
+  }
+
   return false;
 }
 
diff --git a/llvm/test/CodeGen/X86/insertelement-ones.ll b/llvm/test/CodeGen/X86/insertelement-ones.ll
index 3d8e42b9c07d0..6a9a401264c56 100644
--- a/llvm/test/CodeGen/X86/insertelement-ones.ll
+++ b/llvm/test/CodeGen/X86/insertelement-ones.ll
@@ -389,11 +389,9 @@ define <32 x i8> @insert_v32i8_x123456789ABCDEzGHIJKLMNOPQRSTxx(<32 x i8> %a) {
 ; SSE2-NEXT:    movdqa %xmm3, %xmm4
 ; SSE2-NEXT:    pslldq {{.*#+}} xmm4 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm4[0]
 ; SSE2-NEXT:    por %xmm4, %xmm0
-; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255]
-; SSE2-NEXT:    pand %xmm5, %xmm1
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
 ; SSE2-NEXT:    pslldq {{.*#+}} xmm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm3[0,1]
-; SSE2-NEXT:    pandn %xmm3, %xmm5
-; SSE2-NEXT:    por %xmm5, %xmm1
+; SSE2-NEXT:    por %xmm3, %xmm1
 ; SSE2-NEXT:    pand %xmm2, %xmm1
 ; SSE2-NEXT:    por %xmm4, %xmm1
 ; SSE2-NEXT:    retq
@@ -411,11 +409,9 @@ define <32 x i8> @insert_v32i8_x123456789ABCDEzGHIJKLMNOPQRSTxx(<32 x i8> %a) {
 ; SSE3-NEXT:    movdqa %xmm3, %xmm4
 ; SSE3-NEXT:    pslldq {{.*#+}} xmm4 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm4[0]
 ; SSE3-NEXT:    por %xmm4, %xmm0
-; SSE3-NEXT:    movdqa {{.*#+}} xmm5 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255]
-; SSE3-NEXT:    pand %xmm5, %xmm1
+; SSE3-NEXT:    pand {{.*}}(%rip), %xmm1
 ; SSE3-NEXT:    pslldq {{.*#+}} xmm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm3[0,1]
-; SSE3-NEXT:    pandn %xmm3, %xmm5
-; SSE3-NEXT:    por %xmm5, %xmm1
+; SSE3-NEXT:    por %xmm3, %xmm1
 ; SSE3-NEXT:    pand %xmm2, %xmm1
 ; SSE3-NEXT:    por %xmm4, %xmm1
 ; SSE3-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll b/llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll
index 6b49f22f21f1f..9256a43f8e339 100644
--- a/llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll
+++ b/llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll
@@ -1314,10 +1314,10 @@ define void @trunc_v4i64_to_v4i8(<32 x i8>* %L, <4 x i8>* %S) nounwind {
 define <16 x i8> @negative(<32 x i8> %v, <32 x i8> %w) nounwind {
 ; AVX1-LABEL: negative:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpshufb {{.*#+}} xmm2 = xmm0[u,2,4,6,8,10,12,14],zero,zero,zero,zero,zero,zero,zero,zero
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u],zero,zero,zero,zero,zero,zero,zero,xmm0[0,2,4,6,8,10,12,14]
-; AVX1-NEXT:    vpor %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,u,u,u,u,u,u,u,0,2,4,6,8,10,12,14]
+; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4,5,6,7]
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
 ; AVX1-NEXT:    vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
 ; AVX1-NEXT:    vzeroupper
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-128-v8.ll b/llvm/test/CodeGen/X86/vector-shuffle-128-v8.ll
index f448f41cf522e..86423ce76065b 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-128-v8.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-128-v8.ll
@@ -1713,9 +1713,8 @@ define <8 x i16> @shuffle_v8i16_XX4X8acX(<8 x i16> %a, <8 x i16> %b) {
 ;
 ; SSSE3-LABEL: shuffle_v8i16_XX4X8acX:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,8,9,u,u],zero,zero,zero,zero,zero,zero,xmm0[u,u]
-; SSSE3-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[u,u,u,u],zero,zero,xmm1[u,u,0,1,4,5,8,9,u,u]
-; SSSE3-NEXT:    por %xmm1, %xmm0
+; SSSE3-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[u,u,u,u,u,u,u,u,0,1,4,5,8,9,u,u]
+; SSSE3-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[2,3]
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: shuffle_v8i16_XX4X8acX:
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-256-v32.ll b/llvm/test/CodeGen/X86/vector-shuffle-256-v32.ll
index 82df05e5ae068..e5285aebda69e 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-256-v32.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-256-v32.ll
@@ -3358,9 +3358,9 @@ define <32 x i8> @shuffle_v32i8_42_45_12_13_35_35_60_40_17_22_29_44_33_12_48_51_
 ; AVX1-NEXT:    vpshufb {{.*#+}} xmm2 = zero,zero,xmm2[u,u],zero,zero,xmm2[12],zero,xmm2[u,u,u],zero,zero,xmm2[u,0,3]
 ; AVX1-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[10,13,u,u,3,3],zero,xmm1[8,u,u,u,12,1,u],zero,zero
 ; AVX1-NEXT:    vpor %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpshufb {{.*#+}} xmm2 = xmm4[u,u],zero,zero,xmm4[u,u,u,u,1,6,13,u,u],zero,xmm4[u,u]
-; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,12,13,u,u,u,u],zero,zero,zero,xmm0[u,u,12,u,u]
-; AVX1-NEXT:    vpor %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufb {{.*#+}} xmm2 = xmm4[u,u,u,u,u,u,u,u,1,6,13,u,u,u,u,u]
+; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,12,13,u,u,u,u,u,u,u,u,u,12,u,u]
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4,5],xmm0[6,7]
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [255,255,0,0,255,255,255,255,0,0,0,255,255,0,255,255]
 ; AVX1-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0

From 20283ff491a44b8d4526a0915afbe7643705b457 Mon Sep 17 00:00:00 2001
From: Meera Nakrani <meera.nakrani@arm.com>
Date: Tue, 4 Aug 2020 09:38:17 +0000
Subject: [PATCH 1014/1035] [ARM] Generated SSAT and USAT instructions with
 shift

Added patterns so that both SSAT and USAT instructions are generated with shifts. Added corresponding regression tests.

Differential Review: https://reviews.llvm.org/D85120
---
 llvm/lib/Target/ARM/ARMInstrInfo.td      | 16 ++++++++++++++
 llvm/lib/Target/ARM/ARMInstrThumb2.td    |  9 ++++++++
 llvm/test/CodeGen/ARM/ssat-with-shift.ll | 27 ++++++++++++++++++++++++
 llvm/test/CodeGen/ARM/usat-with-shift.ll | 27 ++++++++++++++++++++++++
 4 files changed, 79 insertions(+)
 create mode 100644 llvm/test/CodeGen/ARM/ssat-with-shift.ll
 create mode 100644 llvm/test/CodeGen/ARM/usat-with-shift.ll

diff --git a/llvm/lib/Target/ARM/ARMInstrInfo.td b/llvm/lib/Target/ARM/ARMInstrInfo.td
index d0c1e14989d32..675cf07328e0d 100644
--- a/llvm/lib/Target/ARM/ARMInstrInfo.td
+++ b/llvm/lib/Target/ARM/ARMInstrInfo.td
@@ -380,6 +380,11 @@ def imm_not_XFORM : SDNodeXForm<imm, [{
   return CurDAG->getTargetConstant(~(int)N->getZExtValue(), SDLoc(N), MVT::i32);
 }]>;
 
+// asr_imm_XFORM - Returns a shift immediate with bit {5} set to 1
+def asr_imm_XFORM : SDNodeXForm<imm, [{
+  return CurDAG->getTargetConstant(0x20 | N->getZExtValue(), SDLoc(N), MVT:: i32);
+}]>;
+
 /// imm16_31 predicate - True if the 32-bit immediate is in the range [16,31].
 def imm16_31 : ImmLeaf<i32, [{
   return (int32_t)Imm >= 16 && (int32_t)Imm < 32;
@@ -446,6 +451,8 @@ def fsub_mlx : PatFrag<(ops node:$lhs, node:$rhs),(fsub node:$lhs, node:$rhs),[{
 def imm_even : ImmLeaf<i32, [{ return (Imm & 1) == 0; }]>;
 def imm_odd : ImmLeaf<i32, [{ return (Imm & 1) == 1; }]>;
 
+def asr_imm : ImmLeaf<i32, [{ return Imm > 0 && Imm <= 32; }], asr_imm_XFORM>;
+
 //===----------------------------------------------------------------------===//
 // NEON/MVE pattern fragments
 //
@@ -4083,6 +4090,15 @@ def : ARMV6Pat<(int_arm_ssat16 GPRnopc:$a, imm1_16:$pos),
                (SSAT16 imm1_16:$pos, GPRnopc:$a)>;
 def : ARMV6Pat<(int_arm_usat16 GPRnopc:$a, imm0_15:$pos),
                (USAT16 imm0_15:$pos, GPRnopc:$a)>;
+def : ARMV6Pat<(int_arm_ssat (shl GPRnopc:$a, imm0_31:$shft), imm1_32:$pos),
+               (SSAT imm1_32:$pos, GPRnopc:$a, imm0_31:$shft)>;
+def : ARMV6Pat<(int_arm_ssat (sra GPRnopc:$a, asr_imm:$shft), imm1_32:$pos),
+               (SSAT imm1_32:$pos, GPRnopc:$a, asr_imm:$shft)>;
+def : ARMV6Pat<(int_arm_usat (shl GPRnopc:$a, imm0_31:$shft), imm0_31:$pos),
+               (USAT imm0_31:$pos, GPRnopc:$a, imm0_31:$shft)>;
+def : ARMV6Pat<(int_arm_usat (sra GPRnopc:$a, asr_imm:$shft), imm0_31:$pos),
+               (USAT imm0_31:$pos, GPRnopc:$a, asr_imm:$shft)>;
+
 
 //===----------------------------------------------------------------------===//
 //  Bitwise Instructions.
diff --git a/llvm/lib/Target/ARM/ARMInstrThumb2.td b/llvm/lib/Target/ARM/ARMInstrThumb2.td
index d5143adaac179..fc92230ac81df 100644
--- a/llvm/lib/Target/ARM/ARMInstrThumb2.td
+++ b/llvm/lib/Target/ARM/ARMInstrThumb2.td
@@ -2636,6 +2636,15 @@ def : T2Pat<(int_arm_ssat16 GPR:$a, imm1_16:$pos),
             (t2SSAT16 imm1_16:$pos, GPR:$a)>;
 def : T2Pat<(int_arm_usat16 GPR:$a, imm0_15:$pos),
             (t2USAT16 imm0_15:$pos, GPR:$a)>;
+def : T2Pat<(int_arm_ssat (shl GPRnopc:$a, imm0_31:$shft), imm1_32:$pos),
+            (t2SSAT imm1_32:$pos, GPRnopc:$a, imm0_31:$shft)>;
+def : T2Pat<(int_arm_ssat (sra GPRnopc:$a, asr_imm:$shft), imm1_32:$pos),
+            (t2SSAT imm1_32:$pos, GPRnopc:$a, asr_imm:$shft)>;
+def : T2Pat<(int_arm_usat (shl GPRnopc:$a, imm0_31:$shft), imm0_31:$pos),
+            (t2USAT imm0_31:$pos, GPRnopc:$a, imm0_31:$shft)>;
+def : T2Pat<(int_arm_usat (sra GPRnopc:$a, asr_imm:$shft), imm0_31:$pos),
+            (t2USAT imm0_31:$pos, GPRnopc:$a, asr_imm:$shft)>;
+
 
 //===----------------------------------------------------------------------===//
 //  Shift and rotate Instructions.
diff --git a/llvm/test/CodeGen/ARM/ssat-with-shift.ll b/llvm/test/CodeGen/ARM/ssat-with-shift.ll
new file mode 100644
index 0000000000000..03670eb7e0075
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/ssat-with-shift.ll
@@ -0,0 +1,27 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=armv6-eabi %s -o - | FileCheck %s 
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+dsp %s -o - | FileCheck %s
+
+define arm_aapcs_vfpcc i32 @ssat_lsl(i32 %num){
+; CHECK-LABEL:  ssat_lsl
+; CHECK:        @ %bb.0: @ %entry
+; CHECK-NEXT: 	ssat	r0, #8, r0, lsl #7
+; CHECK-NEXT:   bx	lr
+entry:
+  %shl = shl i32 %num, 7
+  %0 = tail call i32 @llvm.arm.ssat(i32 %shl, i32 8)
+  ret i32 %0
+}
+
+define arm_aapcs_vfpcc i32 @ssat_asr(i32 %num){
+; CHECK-LABEL:  ssat_asr
+; CHECK:        @ %bb.0: @ %entry
+; CHECK-NEXT:   ssat	r0, #8, r0, asr #7
+; CHECK-NEXT:   bx	lr
+entry:
+  %shr = ashr i32 %num, 7
+  %0 = tail call i32 @llvm.arm.ssat(i32 %shr, i32 8)
+  ret i32 %0
+}
+
+declare i32 @llvm.arm.ssat(i32, i32)
diff --git a/llvm/test/CodeGen/ARM/usat-with-shift.ll b/llvm/test/CodeGen/ARM/usat-with-shift.ll
new file mode 100644
index 0000000000000..56444394157a8
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/usat-with-shift.ll
@@ -0,0 +1,27 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=armv6-eabi %s -o - | FileCheck %s
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+dsp %s -o - | FileCheck %s
+
+define arm_aapcs_vfpcc i32 @usat_lsl(i32 %num){
+; CHECK-LABEL:  usat_lsl
+; CHECK:        @ %bb.0: @ %entry
+; CHECK-NEXT: 	usat	r0, #7, r0, lsl #2
+; CHECK-NEXT:   bx	lr
+entry:
+  %shl = shl i32 %num, 2
+  %0 = tail call i32 @llvm.arm.usat(i32 %shl, i32 7)
+  ret i32 %0
+}
+
+define arm_aapcs_vfpcc i32 @usat_asr(i32 %num){
+; CHECK-LABEL:  usat_asr
+; CHECK:        @ %bb.0: @ %entry
+; CHECK-NEXT:   usat	r0, #7, r0, asr #2
+; CHECK-NEXT:   bx	lr
+entry:
+  %shr = ashr i32 %num, 2
+  %0 = tail call i32 @llvm.arm.usat(i32 %shr, i32 7)
+  ret i32 %0
+}
+
+declare i32 @llvm.arm.usat(i32, i32)

From 3c7e7d40a996f998dfe3c36abb70accdf4f20d79 Mon Sep 17 00:00:00 2001
From: David Green <david.green@arm.com>
Date: Tue, 4 Aug 2020 10:43:42 +0100
Subject: [PATCH 1015/1035] [BasicAA] Enable -basic-aa-recphi by default

This option was added a while back, to help improve AA around pointer
phi loops. It looks for phi(gep(phi, const), x) loops, checking if x can
then prove more precise aliasing info.

Differential Revision: https://reviews.llvm.org/D82998
---
 llvm/lib/Analysis/BasicAliasAnalysis.cpp     | 2 +-
 llvm/test/Analysis/BasicAA/phi-loop.ll       | 2 +-
 llvm/test/Analysis/BasicAA/recphi.ll         | 2 +-
 llvm/test/Transforms/LoopIdiom/reuse-cast.ll | 5 +++--
 4 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/llvm/lib/Analysis/BasicAliasAnalysis.cpp b/llvm/lib/Analysis/BasicAliasAnalysis.cpp
index b1578f4b88e07..2ede4baaf6832 100644
--- a/llvm/lib/Analysis/BasicAliasAnalysis.cpp
+++ b/llvm/lib/Analysis/BasicAliasAnalysis.cpp
@@ -66,7 +66,7 @@ using namespace llvm;
 
 /// Enable analysis of recursive PHI nodes.
 static cl::opt<bool> EnableRecPhiAnalysis("basic-aa-recphi", cl::Hidden,
-                                          cl::init(false));
+                                          cl::init(true));
 
 /// By default, even on 32-bit architectures we use 64-bit integers for
 /// calculations. This will allow us to more-aggressively decompose indexing
diff --git a/llvm/test/Analysis/BasicAA/phi-loop.ll b/llvm/test/Analysis/BasicAA/phi-loop.ll
index db3023c6560d0..e54752a9223f7 100644
--- a/llvm/test/Analysis/BasicAA/phi-loop.ll
+++ b/llvm/test/Analysis/BasicAA/phi-loop.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basic-aa -basic-aa-recphi=1 -gvn -S | FileCheck %s
+; RUN: opt < %s -basic-aa -gvn -S | FileCheck %s
 ;
 ; Check that section->word_ofs doesn't get reloaded in every iteration of the
 ; for loop.
diff --git a/llvm/test/Analysis/BasicAA/recphi.ll b/llvm/test/Analysis/BasicAA/recphi.ll
index dfc88937bf699..a6157f22a4f97 100644
--- a/llvm/test/Analysis/BasicAA/recphi.ll
+++ b/llvm/test/Analysis/BasicAA/recphi.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -basic-aa -aa-eval -print-all-alias-modref-info -basic-aa-recphi -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -basic-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
 
 ; CHECK-LABEL: Function: simple: 5 pointers, 0 call sites
 ; CHECK:         NoAlias:      float* %src1, float* %src2
diff --git a/llvm/test/Transforms/LoopIdiom/reuse-cast.ll b/llvm/test/Transforms/LoopIdiom/reuse-cast.ll
index 0b1bfeaa27302..0dda351515647 100644
--- a/llvm/test/Transforms/LoopIdiom/reuse-cast.ll
+++ b/llvm/test/Transforms/LoopIdiom/reuse-cast.ll
@@ -83,21 +83,22 @@ declare void @use.i1(i1)
 define void @reuse_cast_2(i32 %x, i32* %ptr.1.start) {
 ; CHECK-LABEL: @reuse_cast_2(
 ; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[PTR_1_START1:%.*]] = bitcast i32* [[PTR_1_START:%.*]] to i8*
 ; CHECK-NEXT:    [[STACK:%.*]] = alloca [2 x i32], align 4
 ; CHECK-NEXT:    [[CAST_TO_REUSE:%.*]] = bitcast [2 x i32]* [[STACK]] to i8*
 ; CHECK-NEXT:    [[C_0:%.*]] = icmp sgt i32 [[X:%.*]], 0
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast [2 x i32]* [[STACK]] to i8*
 ; CHECK-NEXT:    [[PTR_2_START:%.*]] = getelementptr inbounds [2 x i32], [2 x i32]* [[STACK]], i64 0, i64 0
 ; CHECK-NEXT:    call void @use.i8(i8* [[CAST_TO_REUSE]])
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 [[CAST_TO_REUSE]], i8* align 4 [[PTR_1_START1]], i64 8, i1 false)
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[PTR_1:%.*]] = phi i32* [ [[PTR_1_START:%.*]], [[ENTRY]] ], [ [[PTR_1_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[PTR_1:%.*]] = phi i32* [ [[PTR_1_START]], [[ENTRY]] ], [ [[PTR_1_NEXT:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[PTR_2:%.*]] = phi i32* [ [[PTR_2_START]], [[ENTRY]] ], [ [[PTR_2_NEXT:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[PTR_1_NEXT]] = getelementptr inbounds i32, i32* [[PTR_1]], i64 1
 ; CHECK-NEXT:    [[LV:%.*]] = load i32, i32* [[PTR_1]], align 4
 ; CHECK-NEXT:    [[PTR_2_NEXT]] = getelementptr inbounds i32, i32* [[PTR_2]], i64 1
-; CHECK-NEXT:    store i32 [[LV]], i32* [[PTR_2]], align 4
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
 ; CHECK-NEXT:    [[C_1:%.*]] = icmp eq i32 [[IV]], 0
 ; CHECK-NEXT:    br i1 [[C_1]], label [[LOOP]], label [[EXIT:%.*]]

From 4e4243848ece311d982d3fe5550b555e26709f9a Mon Sep 17 00:00:00 2001
From: Georgii Rymar <grimar@accesssoftek.com>
Date: Tue, 4 Aug 2020 12:40:10 +0300
Subject: [PATCH 1016/1035] [llvm-readobj] - A third attempt to fix BB.
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

http://lab.llvm.org:8011/builders/clang-cmake-x86_64-avx2-linux/builds/15718/steps/build%20stage%201/logs/stdio:

FAILED: /usr/bin/c++  -DGTEST_HAS_RTTI=0 -D_DEBUG -D_GNU_SOURCE -D__STDC_CONSTANT_MACROS -D__STDC_FORMAT_MACROS -D__STDC_LIMIT_MACROS -Itools/llvm-readobj -I/home/ssglocal/clang-cmake-x86_64-avx2-linux/clang-cmake-x86_64-avx2-linux/llvm/llvm/tools/llvm-readobj -Iinclude -I/home/ssglocal/clang-cmake-x86_64-avx2-linux/clang-cmake-x86_64-avx2-linux/llvm/llvm/include -march=broadwell -fPIC -fvisibility-inlines-hidden -Werror=date-time -Wall -Wextra -Wno-unused-parameter -Wwrite-strings -Wcast-qual -Wno-missing-field-initializers -pedantic -Wno-long-long -Wno-maybe-uninitialized -Wdelete-non-virtual-dtor -Wno-comment -fdiagnostics-color -ffunction-sections -fdata-sections -O3     -fno-exceptions -fno-rtti -UNDEBUG -std=c++14 -MD -MT tools/llvm-readobj/CMakeFiles/llvm-readobj.dir/ELFDumper.cpp.o -MF tools/llvm-readobj/CMakeFiles/llvm-readobj.dir/ELFDumper.cpp.o.d -o tools/llvm-readobj/CMakeFiles/llvm-readobj.dir/ELFDumper.cpp.o -c /home/ssglocal/clang-cmake-x86_64-avx2-linux/clang-cmake-x86_64-avx2-linux/llvm/llvm/tools/llvm-readobj/ELFDumper.cpp
/home/ssglocal/clang-cmake-x86_64-avx2-linux/clang-cmake-x86_64-avx2-linux/llvm/llvm/tools/llvm-readobj/ELFDumper.cpp: In function ‘llvm::Expected<const llvm::object::Elf_Mips_Options<ELFT>*> readMipsOptions(const uint8_t*, llvm::ArrayRef<unsigned char>&, bool&)’:
/home/ssglocal/clang-cmake-x86_64-avx2-linux/clang-cmake-x86_64-avx2-linux/llvm/llvm/tools/llvm-readobj/ELFDumper.cpp:3374:12: error: parse error in template argument list
     if (O->size < ExpectedSize)

Note: I played with godbolt.org and was able to catch the similar "error in template argument list" error when used gcc 4.9.0 with this code.
Fix: try to introduce a variable to store `O->size`, it helped to me in godbolt.
---
 llvm/tools/llvm-readobj/ELFDumper.cpp | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/llvm/tools/llvm-readobj/ELFDumper.cpp b/llvm/tools/llvm-readobj/ELFDumper.cpp
index 19aae413acec4..53ebfd5663c94 100644
--- a/llvm/tools/llvm-readobj/ELFDumper.cpp
+++ b/llvm/tools/llvm-readobj/ELFDumper.cpp
@@ -3356,10 +3356,11 @@ readMipsOptions(const uint8_t *SecBegin, ArrayRef<uint8_t> &SecData,
 
   const Elf_Mips_Options<ELFT> *O =
       reinterpret_cast<const Elf_Mips_Options<ELFT> *>(SecData.data());
-  if (O->size > SecData.size()) {
+  const uint8_t Size = O->size;
+  if (Size > SecData.size()) {
     const uint64_t Offset = SecData.data() - SecBegin;
     const uint64_t SecSize = Offset + SecData.size();
-    return createError("a descriptor of size 0x" + Twine::utohexstr(O->size) +
+    return createError("a descriptor of size 0x" + Twine::utohexstr(Size) +
                        " at offset 0x" + Twine::utohexstr(Offset) +
                        " goes past the end of the .MIPS.options "
                        "section of size 0x" +
@@ -3371,14 +3372,14 @@ readMipsOptions(const uint8_t *SecBegin, ArrayRef<uint8_t> &SecData,
       sizeof(Elf_Mips_Options<ELFT>) + sizeof(Elf_Mips_RegInfo<ELFT>);
 
   if (IsSupported)
-    if (O->size < ExpectedSize)
+    if (Size < ExpectedSize)
       return createError(
           "a .MIPS.options entry of kind " +
           Twine(getElfMipsOptionsOdkType(O->kind)) +
-          " has an invalid size (0x" + Twine::utohexstr(O->size) +
+          " has an invalid size (0x" + Twine::utohexstr(Size) +
           "), the expected size is 0x" + Twine::utohexstr(ExpectedSize));
 
-  SecData = SecData.drop_front(O->size);
+  SecData = SecData.drop_front(Size);
   return O;
 }
 

From daa1c6d9d126ea1666463c44930f00df83f3acd6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Lubo=C5=A1=20Lu=C5=88=C3=A1k?= <l.lunak@centrum.cz>
Date: Tue, 4 Aug 2020 11:58:37 +0200
Subject: [PATCH 1017/1035] [lldb] fix typo

---
 lldb/source/Core/IOHandlerCursesGUI.cpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lldb/source/Core/IOHandlerCursesGUI.cpp b/lldb/source/Core/IOHandlerCursesGUI.cpp
index 262a19dc04b4c..144b2112183c2 100644
--- a/lldb/source/Core/IOHandlerCursesGUI.cpp
+++ b/lldb/source/Core/IOHandlerCursesGUI.cpp
@@ -916,9 +916,9 @@ void Menu::DrawMenuTitle(Window &window, bool highlight) {
   } else {
     const int shortcut_key = m_key_value;
     bool underlined_shortcut = false;
-    const attr_t hilgight_attr = A_REVERSE;
+    const attr_t highlight_attr = A_REVERSE;
     if (highlight)
-      window.AttributeOn(hilgight_attr);
+      window.AttributeOn(highlight_attr);
     if (llvm::isPrint(shortcut_key)) {
       size_t lower_pos = m_name.find(tolower(shortcut_key));
       size_t upper_pos = m_name.find(toupper(shortcut_key));
@@ -945,7 +945,7 @@ void Menu::DrawMenuTitle(Window &window, bool highlight) {
     }
 
     if (highlight)
-      window.AttributeOff(hilgight_attr);
+      window.AttributeOff(highlight_attr);
 
     if (m_key_name.empty()) {
       if (!underlined_shortcut && llvm::isPrint(m_key_value)) {

From e218da7ff39df3aeb7c923fbd75aaa8ace581e37 Mon Sep 17 00:00:00 2001
From: Juneyoung Lee <aqjune@gmail.com>
Date: Tue, 4 Aug 2020 18:15:00 +0900
Subject: [PATCH 1018/1035] [JumpThreading] Add a test for simplification of
 cast of any op; NFC

---
 .../JumpThreading/threadable-edge-cast.ll     | 101 ++++++++++++++++++
 1 file changed, 101 insertions(+)
 create mode 100644 llvm/test/Transforms/JumpThreading/threadable-edge-cast.ll

diff --git a/llvm/test/Transforms/JumpThreading/threadable-edge-cast.ll b/llvm/test/Transforms/JumpThreading/threadable-edge-cast.ll
new file mode 100644
index 0000000000000..71c34dea3a069
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/threadable-edge-cast.ll
@@ -0,0 +1,101 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -jump-threading -S < %s | FileCheck %s
+
+declare i32 @f1()
+declare i32 @f2()
+declare void @f3()
+
+; trunc(and) should be simplified
+define i32 @test(i1 %cond0) {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:    br i1 [[COND0:%.*]], label [[T1:%.*]], label [[F1:%.*]]
+; CHECK:       T1:
+; CHECK-NEXT:    [[V1:%.*]] = call i32 @f1()
+; CHECK-NEXT:    br label [[MERGE:%.*]]
+; CHECK:       F1:
+; CHECK-NEXT:    [[V2:%.*]] = call i32 @f2()
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       Merge:
+; CHECK-NEXT:    [[A:%.*]] = phi i32 [ 10, [[T1]] ], [ 0, [[F1]] ]
+; CHECK-NEXT:    [[B:%.*]] = phi i32 [ [[V1]], [[T1]] ], [ [[V2]], [[F1]] ]
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[A]], 1
+; CHECK-NEXT:    [[COND:%.*]] = trunc i32 [[AND]] to i1
+; CHECK-NEXT:    br i1 [[COND]], label [[T2:%.*]], label [[F2:%.*]]
+; CHECK:       T2:
+; CHECK-NEXT:    call void @f3()
+; CHECK-NEXT:    ret i32 [[B]]
+; CHECK:       F2:
+; CHECK-NEXT:    ret i32 [[B]]
+;
+  br i1 %cond0, label %T1, label %F1
+T1:
+  %v1 = call i32 @f1()
+  br label %Merge
+
+F1:
+  %v2 = call i32 @f2()
+  br label %Merge
+
+Merge:
+  %A = phi i32 [10, %T1], [0, %F1]
+  %B = phi i32 [%v1, %T1], [%v2, %F1]
+  %And = and i32 %A, 1
+  %cond = trunc i32 %And to i1
+  br i1 %cond, label %T2, label %F2
+
+T2:
+  call void @f3()
+  ret i32 %B
+
+F2:
+  ret i32 %B
+}
+
+; trunc(select) should be simplified
+define i32 @test2(i1 %cond0) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    br i1 [[COND0:%.*]], label [[T1:%.*]], label [[F1:%.*]]
+; CHECK:       T1:
+; CHECK-NEXT:    [[V1:%.*]] = call i32 @f1()
+; CHECK-NEXT:    br label [[MERGE:%.*]]
+; CHECK:       F1:
+; CHECK-NEXT:    [[V2:%.*]] = call i32 @f2()
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       Merge:
+; CHECK-NEXT:    [[A:%.*]] = phi i1 [ true, [[T1]] ], [ false, [[F1]] ]
+; CHECK-NEXT:    [[B:%.*]] = phi i32 [ [[V1]], [[T1]] ], [ [[V2]], [[F1]] ]
+; CHECK-NEXT:    [[A2:%.*]] = xor i1 [[A]], true
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[A2]], i32 10, i32 0
+; CHECK-NEXT:    [[COND:%.*]] = trunc i32 [[SEL]] to i1
+; CHECK-NEXT:    br i1 [[COND]], label [[T2:%.*]], label [[F2:%.*]]
+; CHECK:       T2:
+; CHECK-NEXT:    call void @f3()
+; CHECK-NEXT:    ret i32 [[B]]
+; CHECK:       F2:
+; CHECK-NEXT:    ret i32 [[B]]
+;
+  br i1 %cond0, label %T1, label %F1
+T1:
+  %v1 = call i32 @f1()
+  br label %Merge
+
+F1:
+  %v2 = call i32 @f2()
+  br label %Merge
+
+Merge:
+  %A = phi i1 [1, %T1], [0, %F1]
+  %B = phi i32 [%v1, %T1], [%v2, %F1]
+  %A2 = xor i1 %A, 1
+  %sel = select i1 %A2, i32 10, i32 0
+  %cond = trunc i32 %sel to i1
+  br i1 %cond, label %T2, label %F2
+
+T2:
+  call void @f3()
+  ret i32 %B
+
+F2:
+  ret i32 %B
+}
+

From e734e8286b4b521d829aaddb6d1cbbd264953625 Mon Sep 17 00:00:00 2001
From: Juneyoung Lee <aqjune@gmail.com>
Date: Tue, 4 Aug 2020 18:26:56 +0900
Subject: [PATCH 1019/1035] [JumpThreading] Remove cast's constraint

As discussed in D84949, this removes the constraint to cast since it does not
cause compile time degradation.

Reviewed By: lebedev.ri

Differential Revision: https://reviews.llvm.org/D85188
---
 llvm/lib/Transforms/Scalar/JumpThreading.cpp  |  6 +----
 .../JumpThreading/threadable-edge-cast.ll     | 24 +++++--------------
 2 files changed, 7 insertions(+), 23 deletions(-)

diff --git a/llvm/lib/Transforms/Scalar/JumpThreading.cpp b/llvm/lib/Transforms/Scalar/JumpThreading.cpp
index f42d4841f7939..e6d261fa9aff5 100644
--- a/llvm/lib/Transforms/Scalar/JumpThreading.cpp
+++ b/llvm/lib/Transforms/Scalar/JumpThreading.cpp
@@ -674,13 +674,9 @@ bool JumpThreadingPass::ComputeValueKnownInPredecessorsImpl(
     return !Result.empty();
   }
 
-  // Handle Cast instructions.  Only see through Cast when the source operand is
-  // PHI, Cmp, or Freeze to save the compilation time.
+  // Handle Cast instructions.
   if (CastInst *CI = dyn_cast<CastInst>(I)) {
     Value *Source = CI->getOperand(0);
-    if (!isa<PHINode>(Source) && !isa<CmpInst>(Source) &&
-        !isa<FreezeInst>(Source))
-      return false;
     ComputeValueKnownInPredecessorsImpl(Source, BB, Result, Preference,
                                         RecursionSet, CxtI);
     if (Result.empty())
diff --git a/llvm/test/Transforms/JumpThreading/threadable-edge-cast.ll b/llvm/test/Transforms/JumpThreading/threadable-edge-cast.ll
index 71c34dea3a069..ef1e8f28d14f1 100644
--- a/llvm/test/Transforms/JumpThreading/threadable-edge-cast.ll
+++ b/llvm/test/Transforms/JumpThreading/threadable-edge-cast.ll
@@ -11,20 +11,14 @@ define i32 @test(i1 %cond0) {
 ; CHECK-NEXT:    br i1 [[COND0:%.*]], label [[T1:%.*]], label [[F1:%.*]]
 ; CHECK:       T1:
 ; CHECK-NEXT:    [[V1:%.*]] = call i32 @f1()
-; CHECK-NEXT:    br label [[MERGE:%.*]]
+; CHECK-NEXT:    br label [[F2:%.*]]
 ; CHECK:       F1:
 ; CHECK-NEXT:    [[V2:%.*]] = call i32 @f2()
-; CHECK-NEXT:    br label [[MERGE]]
-; CHECK:       Merge:
+; CHECK-NEXT:    br label [[F2]]
+; CHECK:       F2:
 ; CHECK-NEXT:    [[A:%.*]] = phi i32 [ 10, [[T1]] ], [ 0, [[F1]] ]
 ; CHECK-NEXT:    [[B:%.*]] = phi i32 [ [[V1]], [[T1]] ], [ [[V2]], [[F1]] ]
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[A]], 1
-; CHECK-NEXT:    [[COND:%.*]] = trunc i32 [[AND]] to i1
-; CHECK-NEXT:    br i1 [[COND]], label [[T2:%.*]], label [[F2:%.*]]
-; CHECK:       T2:
-; CHECK-NEXT:    call void @f3()
-; CHECK-NEXT:    ret i32 [[B]]
-; CHECK:       F2:
 ; CHECK-NEXT:    ret i32 [[B]]
 ;
   br i1 %cond0, label %T1, label %F1
@@ -57,21 +51,15 @@ define i32 @test2(i1 %cond0) {
 ; CHECK-NEXT:    br i1 [[COND0:%.*]], label [[T1:%.*]], label [[F1:%.*]]
 ; CHECK:       T1:
 ; CHECK-NEXT:    [[V1:%.*]] = call i32 @f1()
-; CHECK-NEXT:    br label [[MERGE:%.*]]
+; CHECK-NEXT:    br label [[F2:%.*]]
 ; CHECK:       F1:
 ; CHECK-NEXT:    [[V2:%.*]] = call i32 @f2()
-; CHECK-NEXT:    br label [[MERGE]]
-; CHECK:       Merge:
+; CHECK-NEXT:    br label [[F2]]
+; CHECK:       F2:
 ; CHECK-NEXT:    [[A:%.*]] = phi i1 [ true, [[T1]] ], [ false, [[F1]] ]
 ; CHECK-NEXT:    [[B:%.*]] = phi i32 [ [[V1]], [[T1]] ], [ [[V2]], [[F1]] ]
 ; CHECK-NEXT:    [[A2:%.*]] = xor i1 [[A]], true
 ; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[A2]], i32 10, i32 0
-; CHECK-NEXT:    [[COND:%.*]] = trunc i32 [[SEL]] to i1
-; CHECK-NEXT:    br i1 [[COND]], label [[T2:%.*]], label [[F2:%.*]]
-; CHECK:       T2:
-; CHECK-NEXT:    call void @f3()
-; CHECK-NEXT:    ret i32 [[B]]
-; CHECK:       F2:
 ; CHECK-NEXT:    ret i32 [[B]]
 ;
   br i1 %cond0, label %T1, label %F1

From 4be13b15d69d9d70506277eb1900eadccd75f608 Mon Sep 17 00:00:00 2001
From: Paul Walker <paul.walker@arm.com>
Date: Tue, 4 Aug 2020 11:19:17 +0100
Subject: [PATCH 1020/1035] [SVE] Replace remaining _MERGE_OP1 nodes with _PRED
 variants.

This is the final bit of work to relax the register allocation
requirements when code generating normal LLVM IR, which rarely
care about the result of inactive lanes. By using _PRED nodes
we can make better use of SVE's reversed instructions.

Also removes a redundant parameter from the min/max tests.

Differential Revision: https://reviews.llvm.org/D85142
---
 .../Target/AArch64/AArch64ISelLowering.cpp    |  68 ++++---
 llvm/lib/Target/AArch64/AArch64ISelLowering.h |  16 +-
 .../lib/Target/AArch64/AArch64SVEInstrInfo.td |  63 +++---
 llvm/lib/Target/AArch64/SVEInstrFormats.td    |  18 +-
 .../CodeGen/AArch64/llvm-ir-to-intrinsic.ll   | 190 +++++++++++++++---
 5 files changed, 257 insertions(+), 98 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 8b533e97b928d..402d7656ca215 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -1397,14 +1397,14 @@ const char *AArch64TargetLowering::getTargetNodeName(unsigned Opcode) const {
     MAKE_CASE(AArch64ISD::TLSDESC_CALLSEQ)
     MAKE_CASE(AArch64ISD::ADD_PRED)
     MAKE_CASE(AArch64ISD::SDIV_PRED)
+    MAKE_CASE(AArch64ISD::SHL_PRED)
+    MAKE_CASE(AArch64ISD::SMAX_PRED)
+    MAKE_CASE(AArch64ISD::SMIN_PRED)
+    MAKE_CASE(AArch64ISD::SRA_PRED)
+    MAKE_CASE(AArch64ISD::SRL_PRED)
     MAKE_CASE(AArch64ISD::UDIV_PRED)
-    MAKE_CASE(AArch64ISD::SMIN_MERGE_OP1)
-    MAKE_CASE(AArch64ISD::UMIN_MERGE_OP1)
-    MAKE_CASE(AArch64ISD::SMAX_MERGE_OP1)
-    MAKE_CASE(AArch64ISD::UMAX_MERGE_OP1)
-    MAKE_CASE(AArch64ISD::SHL_MERGE_OP1)
-    MAKE_CASE(AArch64ISD::SRL_MERGE_OP1)
-    MAKE_CASE(AArch64ISD::SRA_MERGE_OP1)
+    MAKE_CASE(AArch64ISD::UMAX_PRED)
+    MAKE_CASE(AArch64ISD::UMIN_PRED)
     MAKE_CASE(AArch64ISD::SETCC_MERGE_ZERO)
     MAKE_CASE(AArch64ISD::ADC)
     MAKE_CASE(AArch64ISD::SBC)
@@ -3540,13 +3540,13 @@ SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
   case ISD::UDIV:
     return LowerToPredicatedOp(Op, DAG, AArch64ISD::UDIV_PRED);
   case ISD::SMIN:
-    return LowerToPredicatedOp(Op, DAG, AArch64ISD::SMIN_MERGE_OP1);
+    return LowerToPredicatedOp(Op, DAG, AArch64ISD::SMIN_PRED);
   case ISD::UMIN:
-    return LowerToPredicatedOp(Op, DAG, AArch64ISD::UMIN_MERGE_OP1);
+    return LowerToPredicatedOp(Op, DAG, AArch64ISD::UMIN_PRED);
   case ISD::SMAX:
-    return LowerToPredicatedOp(Op, DAG, AArch64ISD::SMAX_MERGE_OP1);
+    return LowerToPredicatedOp(Op, DAG, AArch64ISD::SMAX_PRED);
   case ISD::UMAX:
-    return LowerToPredicatedOp(Op, DAG, AArch64ISD::UMAX_MERGE_OP1);
+    return LowerToPredicatedOp(Op, DAG, AArch64ISD::UMAX_PRED);
   case ISD::SRA:
   case ISD::SRL:
   case ISD::SHL:
@@ -8914,7 +8914,7 @@ SDValue AArch64TargetLowering::LowerVectorSRA_SRL_SHL(SDValue Op,
 
   case ISD::SHL:
     if (VT.isScalableVector())
-      return LowerToPredicatedOp(Op, DAG, AArch64ISD::SHL_MERGE_OP1);
+      return LowerToPredicatedOp(Op, DAG, AArch64ISD::SHL_PRED);
 
     if (isVShiftLImm(Op.getOperand(1), VT, false, Cnt) && Cnt < EltSize)
       return DAG.getNode(AArch64ISD::VSHL, DL, VT, Op.getOperand(0),
@@ -8926,8 +8926,8 @@ SDValue AArch64TargetLowering::LowerVectorSRA_SRL_SHL(SDValue Op,
   case ISD::SRA:
   case ISD::SRL:
     if (VT.isScalableVector()) {
-      unsigned Opc = Op.getOpcode() == ISD::SRA ? AArch64ISD::SRA_MERGE_OP1
-                                                : AArch64ISD::SRL_MERGE_OP1;
+      unsigned Opc = Op.getOpcode() == ISD::SRA ? AArch64ISD::SRA_PRED
+                                                : AArch64ISD::SRL_PRED;
       return LowerToPredicatedOp(Op, DAG, Opc);
     }
 
@@ -11940,6 +11940,25 @@ static SDValue combineSVEReductionOrderedFP(SDNode *N, unsigned Opc,
                      Zero);
 }
 
+// If a merged operation has no inactive lanes we can relax it to a predicated
+// or unpredicated operation, which potentially allows better isel (perhaps
+// using immediate forms) or relaxing register reuse requirements.
+static SDValue convertMergedOpToPredOp(SDNode *N, unsigned PredOpc,
+                                       SelectionDAG &DAG) {
+  assert(N->getOpcode() == ISD::INTRINSIC_WO_CHAIN && "Expected intrinsic!");
+  assert(N->getNumOperands() == 4 && "Expected 3 operand intrinsic!");
+  SDValue Pg = N->getOperand(1);
+
+  // ISD way to specify an all active predicate.
+  if ((Pg.getOpcode() == AArch64ISD::PTRUE) &&
+      (Pg.getConstantOperandVal(0) == AArch64SVEPredPattern::all))
+    return DAG.getNode(PredOpc, SDLoc(N), N->getValueType(0), Pg,
+                       N->getOperand(2), N->getOperand(3));
+
+  // FUTURE: SplatVector(true)
+  return SDValue();
+}
+
 static SDValue performIntrinsicCombine(SDNode *N,
                                        TargetLowering::DAGCombinerInfo &DCI,
                                        const AArch64Subtarget *Subtarget) {
@@ -12018,26 +12037,19 @@ static SDValue performIntrinsicCombine(SDNode *N,
   case Intrinsic::aarch64_sve_ext:
     return LowerSVEIntrinsicEXT(N, DAG);
   case Intrinsic::aarch64_sve_smin:
-    return DAG.getNode(AArch64ISD::SMIN_MERGE_OP1, SDLoc(N), N->getValueType(0),
-                       N->getOperand(1), N->getOperand(2), N->getOperand(3));
+    return convertMergedOpToPredOp(N, AArch64ISD::SMIN_PRED, DAG);
   case Intrinsic::aarch64_sve_umin:
-    return DAG.getNode(AArch64ISD::UMIN_MERGE_OP1, SDLoc(N), N->getValueType(0),
-                       N->getOperand(1), N->getOperand(2), N->getOperand(3));
+    return convertMergedOpToPredOp(N, AArch64ISD::UMIN_PRED, DAG);
   case Intrinsic::aarch64_sve_smax:
-    return DAG.getNode(AArch64ISD::SMAX_MERGE_OP1, SDLoc(N), N->getValueType(0),
-                       N->getOperand(1), N->getOperand(2), N->getOperand(3));
+    return convertMergedOpToPredOp(N, AArch64ISD::SMAX_PRED, DAG);
   case Intrinsic::aarch64_sve_umax:
-    return DAG.getNode(AArch64ISD::UMAX_MERGE_OP1, SDLoc(N), N->getValueType(0),
-                       N->getOperand(1), N->getOperand(2), N->getOperand(3));
+    return convertMergedOpToPredOp(N, AArch64ISD::UMAX_PRED, DAG);
   case Intrinsic::aarch64_sve_lsl:
-    return DAG.getNode(AArch64ISD::SHL_MERGE_OP1, SDLoc(N), N->getValueType(0),
-                       N->getOperand(1), N->getOperand(2), N->getOperand(3));
+    return convertMergedOpToPredOp(N, AArch64ISD::SHL_PRED, DAG);
   case Intrinsic::aarch64_sve_lsr:
-    return DAG.getNode(AArch64ISD::SRL_MERGE_OP1, SDLoc(N), N->getValueType(0),
-                       N->getOperand(1), N->getOperand(2), N->getOperand(3));
+    return convertMergedOpToPredOp(N, AArch64ISD::SRL_PRED, DAG);
   case Intrinsic::aarch64_sve_asr:
-    return DAG.getNode(AArch64ISD::SRA_MERGE_OP1, SDLoc(N), N->getValueType(0),
-                       N->getOperand(1), N->getOperand(2), N->getOperand(3));
+    return convertMergedOpToPredOp(N, AArch64ISD::SRA_PRED, DAG);
   case Intrinsic::aarch64_sve_cmphs:
     if (!N->getOperand(2).getValueType().isFloatingPoint())
       return DAG.getNode(AArch64ISD::SETCC_MERGE_ZERO, SDLoc(N),
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index a793fb6bb4625..c346debb823aa 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -72,7 +72,7 @@ enum NodeType : unsigned {
   ADC,
   SBC, // adc, sbc instructions
 
-  // Arithmetic instructions
+  // Predicated instructions where inactive lanes produce undefined results.
   ADD_PRED,
   FADD_PRED,
   FDIV_PRED,
@@ -80,14 +80,14 @@ enum NodeType : unsigned {
   FMUL_PRED,
   FSUB_PRED,
   SDIV_PRED,
+  SHL_PRED,
+  SMAX_PRED,
+  SMIN_PRED,
+  SRA_PRED,
+  SRL_PRED,
   UDIV_PRED,
-  SMIN_MERGE_OP1,
-  UMIN_MERGE_OP1,
-  SMAX_MERGE_OP1,
-  UMAX_MERGE_OP1,
-  SHL_MERGE_OP1,
-  SRL_MERGE_OP1,
-  SRA_MERGE_OP1,
+  UMAX_PRED,
+  UMIN_PRED,
 
   SETCC_MERGE_ZERO,
 
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 7c39268a4441f..cf34e5f6470da 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -174,22 +174,20 @@ def SDT_AArch64FMA : SDTypeProfile<1, 4, [
 
 // Predicated operations with the result of inactive lanes being unspecified.
 def AArch64add_p  : SDNode<"AArch64ISD::ADD_PRED",  SDT_AArch64Arith>;
+def AArch64asr_p  : SDNode<"AArch64ISD::SRA_PRED",  SDT_AArch64Arith>;
 def AArch64fadd_p : SDNode<"AArch64ISD::FADD_PRED", SDT_AArch64Arith>;
 def AArch64fdiv_p : SDNode<"AArch64ISD::FDIV_PRED", SDT_AArch64Arith>;
 def AArch64fma_p  : SDNode<"AArch64ISD::FMA_PRED",  SDT_AArch64FMA>;
 def AArch64fmul_p : SDNode<"AArch64ISD::FMUL_PRED", SDT_AArch64Arith>;
 def AArch64fsub_p : SDNode<"AArch64ISD::FSUB_PRED", SDT_AArch64Arith>;
+def AArch64lsl_p  : SDNode<"AArch64ISD::SHL_PRED",  SDT_AArch64Arith>;
+def AArch64lsr_p  : SDNode<"AArch64ISD::SRL_PRED",  SDT_AArch64Arith>;
 def AArch64sdiv_p : SDNode<"AArch64ISD::SDIV_PRED", SDT_AArch64Arith>;
+def AArch64smax_p : SDNode<"AArch64ISD::SMAX_PRED", SDT_AArch64Arith>;
+def AArch64smin_p : SDNode<"AArch64ISD::SMIN_PRED", SDT_AArch64Arith>;
 def AArch64udiv_p : SDNode<"AArch64ISD::UDIV_PRED", SDT_AArch64Arith>;
-
-// Merging op1 into the inactive lanes.
-def AArch64smin_m1 :  SDNode<"AArch64ISD::SMIN_MERGE_OP1", SDT_AArch64Arith>;
-def AArch64umin_m1 :  SDNode<"AArch64ISD::UMIN_MERGE_OP1", SDT_AArch64Arith>;
-def AArch64smax_m1 :  SDNode<"AArch64ISD::SMAX_MERGE_OP1", SDT_AArch64Arith>;
-def AArch64umax_m1 :  SDNode<"AArch64ISD::UMAX_MERGE_OP1", SDT_AArch64Arith>;
-def AArch64lsl_m1  :  SDNode<"AArch64ISD::SHL_MERGE_OP1",  SDT_AArch64Arith>;
-def AArch64lsr_m1  :  SDNode<"AArch64ISD::SRL_MERGE_OP1",  SDT_AArch64Arith>;
-def AArch64asr_m1  :  SDNode<"AArch64ISD::SRA_MERGE_OP1",  SDT_AArch64Arith>;
+def AArch64umax_p : SDNode<"AArch64ISD::UMAX_PRED", SDT_AArch64Arith>;
+def AArch64umin_p : SDNode<"AArch64ISD::UMIN_PRED", SDT_AArch64Arith>;
 
 def SDT_AArch64ReduceWithInit : SDTypeProfile<1, 3, [SDTCisVec<1>, SDTCisVec<3>]>;
 def AArch64clasta_n   : SDNode<"AArch64ISD::CLASTA_N",   SDT_AArch64ReduceWithInit>;
@@ -287,10 +285,10 @@ let Predicates = [HasSVE] in {
   defm EOR_ZI : sve_int_log_imm<0b01, "eor", "eon", xor>;
   defm AND_ZI : sve_int_log_imm<0b10, "and", "bic", and>;
 
-  defm SMAX_ZI   : sve_int_arith_imm1<0b00, "smax", AArch64smax_m1>;
-  defm SMIN_ZI   : sve_int_arith_imm1<0b10, "smin", AArch64smin_m1>;
-  defm UMAX_ZI   : sve_int_arith_imm1_unsigned<0b01, "umax", AArch64umax_m1>;
-  defm UMIN_ZI   : sve_int_arith_imm1_unsigned<0b11, "umin", AArch64umin_m1>;
+  defm SMAX_ZI   : sve_int_arith_imm1<0b00, "smax", AArch64smax_p>;
+  defm SMIN_ZI   : sve_int_arith_imm1<0b10, "smin", AArch64smin_p>;
+  defm UMAX_ZI   : sve_int_arith_imm1_unsigned<0b01, "umax", AArch64umax_p>;
+  defm UMIN_ZI   : sve_int_arith_imm1_unsigned<0b11, "umin", AArch64umin_p>;
 
   defm MUL_ZI     : sve_int_arith_imm2<"mul", mul>;
   defm MUL_ZPmZ   : sve_int_bin_pred_arit_2<0b000, "mul",   int_aarch64_sve_mul>;
@@ -343,12 +341,17 @@ let Predicates = [HasSVE] in {
   defm FABS_ZPmZ : sve_int_un_pred_arit_1_fp<0b100, "fabs", int_aarch64_sve_fabs>;
   defm FNEG_ZPmZ : sve_int_un_pred_arit_1_fp<0b101, "fneg", int_aarch64_sve_fneg>;
 
-  defm SMAX_ZPmZ : sve_int_bin_pred_arit_1<0b000, "smax", AArch64smax_m1>;
-  defm UMAX_ZPmZ : sve_int_bin_pred_arit_1<0b001, "umax", AArch64umax_m1>;
-  defm SMIN_ZPmZ : sve_int_bin_pred_arit_1<0b010, "smin", AArch64smin_m1>;
-  defm UMIN_ZPmZ : sve_int_bin_pred_arit_1<0b011, "umin", AArch64umin_m1>;
-  defm SABD_ZPmZ : sve_int_bin_pred_arit_1<0b100, "sabd", int_aarch64_sve_sabd>;
-  defm UABD_ZPmZ : sve_int_bin_pred_arit_1<0b101, "uabd", int_aarch64_sve_uabd>;
+  defm SMAX_ZPmZ : sve_int_bin_pred_arit_1<0b000, "smax", "SMAX_ZPZZ", int_aarch64_sve_smax, DestructiveBinaryComm>;
+  defm UMAX_ZPmZ : sve_int_bin_pred_arit_1<0b001, "umax", "UMAX_ZPZZ", int_aarch64_sve_umax, DestructiveBinaryComm>;
+  defm SMIN_ZPmZ : sve_int_bin_pred_arit_1<0b010, "smin", "SMIN_ZPZZ", int_aarch64_sve_smin, DestructiveBinaryComm>;
+  defm UMIN_ZPmZ : sve_int_bin_pred_arit_1<0b011, "umin", "UMIN_ZPZZ", int_aarch64_sve_umin, DestructiveBinaryComm>;
+  defm SABD_ZPmZ : sve_int_bin_pred_arit_1<0b100, "sabd", "SABD_ZPZZ", int_aarch64_sve_sabd, DestructiveBinaryComm>;
+  defm UABD_ZPmZ : sve_int_bin_pred_arit_1<0b101, "uabd", "UABD_ZPZZ", int_aarch64_sve_uabd, DestructiveBinaryComm>;
+
+  defm SMAX_ZPZZ  : sve_int_bin_pred_bhsd<AArch64smax_p>;
+  defm UMAX_ZPZZ  : sve_int_bin_pred_bhsd<AArch64umax_p>;
+  defm SMIN_ZPZZ  : sve_int_bin_pred_bhsd<AArch64smin_p>;
+  defm UMIN_ZPZZ  : sve_int_bin_pred_bhsd<AArch64umin_p>;
 
   defm FRECPE_ZZ  : sve_fp_2op_u_zd<0b110, "frecpe",  int_aarch64_sve_frecpe_x>;
   defm FRSQRTE_ZZ : sve_fp_2op_u_zd<0b111, "frsqrte", int_aarch64_sve_frsqrte_x>;
@@ -1313,9 +1316,9 @@ multiclass sve_prefetch<SDPatternOperator prefetch, ValueType PredTy, Instructio
   defm INDEX_II : sve_int_index_ii<"index", index_vector>;
 
   // Unpredicated shifts
-  defm ASR_ZZI : sve_int_bin_cons_shift_imm_right<0b00, "asr", AArch64asr_m1>;
-  defm LSR_ZZI : sve_int_bin_cons_shift_imm_right<0b01, "lsr", AArch64lsr_m1>;
-  defm LSL_ZZI : sve_int_bin_cons_shift_imm_left< 0b11, "lsl", AArch64lsl_m1>;
+  defm ASR_ZZI : sve_int_bin_cons_shift_imm_right<0b00, "asr", AArch64asr_p>;
+  defm LSR_ZZI : sve_int_bin_cons_shift_imm_right<0b01, "lsr", AArch64lsr_p>;
+  defm LSL_ZZI : sve_int_bin_cons_shift_imm_left< 0b11, "lsl", AArch64lsl_p>;
 
   defm ASR_WIDE_ZZZ : sve_int_bin_cons_shift_wide<0b00, "asr">;
   defm LSR_WIDE_ZZZ : sve_int_bin_cons_shift_wide<0b01, "lsr">;
@@ -1328,19 +1331,23 @@ multiclass sve_prefetch<SDPatternOperator prefetch, ValueType PredTy, Instructio
   defm ASRD_ZPmI : sve_int_bin_pred_shift_imm_right<0b0100, "asrd", "ASRD_ZPZI", int_aarch64_sve_asrd>;
 
   let Predicates = [HasSVE, UseExperimentalZeroingPseudos] in {
-    defm ASR_ZPZZ    : sve_int_bin_pred_zeroing_bhsd<AArch64asr_m1>;
-    defm LSR_ZPZZ    : sve_int_bin_pred_zeroing_bhsd<AArch64lsr_m1>;
-    defm LSL_ZPZZ    : sve_int_bin_pred_zeroing_bhsd<AArch64lsl_m1>;
+    defm ASR_ZPZZ    : sve_int_bin_pred_zeroing_bhsd<int_aarch64_sve_asr>;
+    defm LSR_ZPZZ    : sve_int_bin_pred_zeroing_bhsd<int_aarch64_sve_lsr>;
+    defm LSL_ZPZZ    : sve_int_bin_pred_zeroing_bhsd<int_aarch64_sve_lsl>;
     defm ASRD_ZPZI   : sve_int_bin_pred_shift_imm_right_zeroing_bhsd<int_aarch64_sve_asrd>;
   }
 
-  defm ASR_ZPmZ  : sve_int_bin_pred_shift<0b000, "asr", "ASR_ZPZZ", AArch64asr_m1, "ASRR_ZPmZ">;
-  defm LSR_ZPmZ  : sve_int_bin_pred_shift<0b001, "lsr", "LSR_ZPZZ", AArch64lsr_m1, "LSRR_ZPmZ">;
-  defm LSL_ZPmZ  : sve_int_bin_pred_shift<0b011, "lsl", "LSL_ZPZZ", AArch64lsl_m1, "LSLR_ZPmZ">;
+  defm ASR_ZPmZ  : sve_int_bin_pred_shift<0b000, "asr", "ASR_ZPZZ", int_aarch64_sve_asr, "ASRR_ZPmZ">;
+  defm LSR_ZPmZ  : sve_int_bin_pred_shift<0b001, "lsr", "LSR_ZPZZ", int_aarch64_sve_lsr, "LSRR_ZPmZ">;
+  defm LSL_ZPmZ  : sve_int_bin_pred_shift<0b011, "lsl", "LSL_ZPZZ", int_aarch64_sve_lsl, "LSLR_ZPmZ">;
   defm ASRR_ZPmZ : sve_int_bin_pred_shift<0b100, "asrr", "ASRR_ZPZZ", null_frag, "ASR_ZPmZ", /*isReverseInstr*/ 1>;
   defm LSRR_ZPmZ : sve_int_bin_pred_shift<0b101, "lsrr", "LSRR_ZPZZ", null_frag, "LSR_ZPmZ", /*isReverseInstr*/ 1>;
   defm LSLR_ZPmZ : sve_int_bin_pred_shift<0b111, "lslr", "LSLR_ZPZZ", null_frag, "LSL_ZPmZ", /*isReverseInstr*/ 1>;
 
+  defm ASR_ZPZZ  : sve_int_bin_pred_bhsd<AArch64asr_p>;
+  defm LSR_ZPZZ  : sve_int_bin_pred_bhsd<AArch64lsr_p>;
+  defm LSL_ZPZZ  : sve_int_bin_pred_bhsd<AArch64lsl_p>;
+
   defm ASR_WIDE_ZPmZ : sve_int_bin_pred_shift_wide<0b000, "asr", int_aarch64_sve_asr_wide>;
   defm LSR_WIDE_ZPmZ : sve_int_bin_pred_shift_wide<0b001, "lsr", int_aarch64_sve_lsr_wide>;
   defm LSL_WIDE_ZPmZ : sve_int_bin_pred_shift_wide<0b011, "lsl", int_aarch64_sve_lsl_wide>;
diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td
index 1f067908de6b4..c5dd327d2aef4 100644
--- a/llvm/lib/Target/AArch64/SVEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td
@@ -2382,11 +2382,19 @@ multiclass sve_int_bin_pred_arit_0<bits<3> opc, string asm, string Ps,
   def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;
 }
 
-multiclass sve_int_bin_pred_arit_1<bits<3> opc, string asm, SDPatternOperator op> {
-  def _B : sve_int_bin_pred_arit_log<0b00, 0b01, opc, asm, ZPR8>;
-  def _H : sve_int_bin_pred_arit_log<0b01, 0b01, opc, asm, ZPR16>;
-  def _S : sve_int_bin_pred_arit_log<0b10, 0b01, opc, asm, ZPR32>;
-  def _D : sve_int_bin_pred_arit_log<0b11, 0b01, opc, asm, ZPR64>;
+multiclass sve_int_bin_pred_arit_1<bits<3> opc, string asm, string Ps,
+                                   SDPatternOperator op,
+                                   DestructiveInstTypeEnum flags> {
+  let DestructiveInstType = flags in {
+  def _B : sve_int_bin_pred_arit_log<0b00, 0b01, opc, asm, ZPR8>,
+             SVEPseudo2Instr<Ps # _B, 1>;
+  def _H : sve_int_bin_pred_arit_log<0b01, 0b01, opc, asm, ZPR16>,
+             SVEPseudo2Instr<Ps # _H, 1>;
+  def _S : sve_int_bin_pred_arit_log<0b10, 0b01, opc, asm, ZPR32>,
+             SVEPseudo2Instr<Ps # _S, 1>;
+  def _D : sve_int_bin_pred_arit_log<0b11, 0b01, opc, asm, ZPR64>,
+             SVEPseudo2Instr<Ps # _D, 1>;
+  }
 
   def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;
   def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>;
diff --git a/llvm/test/CodeGen/AArch64/llvm-ir-to-intrinsic.ll b/llvm/test/CodeGen/AArch64/llvm-ir-to-intrinsic.ll
index 816465f9eaa17..9f3a77c8fe92b 100644
--- a/llvm/test/CodeGen/AArch64/llvm-ir-to-intrinsic.ll
+++ b/llvm/test/CodeGen/AArch64/llvm-ir-to-intrinsic.ll
@@ -182,7 +182,7 @@ define <vscale x 2 x i64> @urem_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b
 ; SMIN
 ;
 
-define <vscale x 16 x i8> @smin_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %c) {
+define <vscale x 16 x i8> @smin_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
 ; CHECK-LABEL: smin_i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.b
@@ -193,7 +193,7 @@ define <vscale x 16 x i8> @smin_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b,
   ret <vscale x 16 x i8> %min
 }
 
-define <vscale x 8 x i16> @smin_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b, <vscale x 8 x i16> %c) {
+define <vscale x 8 x i16> @smin_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
 ; CHECK-LABEL: smin_i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.h
@@ -204,7 +204,7 @@ define <vscale x 8 x i16> @smin_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b
   ret <vscale x 8 x i16> %min
 }
 
-define <vscale x 4 x i32> @smin_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c) {
+define <vscale x 4 x i32> @smin_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
 ; CHECK-LABEL: smin_i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.s
@@ -215,7 +215,7 @@ define <vscale x 4 x i32> @smin_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b
   ret <vscale x 4 x i32> %min
 }
 
-define <vscale x 2 x i64> @smin_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b, <vscale x 2 x i64> %c) {
+define <vscale x 2 x i64> @smin_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
 ; CHECK-LABEL: smin_i64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d
@@ -226,7 +226,7 @@ define <vscale x 2 x i64> @smin_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b
   ret <vscale x 2 x i64> %min
 }
 
-define <vscale x 32 x i8> @smin_split_i8(<vscale x 32 x i8> %a, <vscale x 32 x i8> %b, <vscale x 32 x i8> %c) {
+define <vscale x 32 x i8> @smin_split_i8(<vscale x 32 x i8> %a, <vscale x 32 x i8> %b) {
 ; CHECK-LABEL: smin_split_i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.b
@@ -238,7 +238,7 @@ define <vscale x 32 x i8> @smin_split_i8(<vscale x 32 x i8> %a, <vscale x 32 x i
   ret <vscale x 32 x i8> %min
 }
 
-define <vscale x 32 x i16> @smin_split_i16(<vscale x 32 x i16> %a, <vscale x 32 x i16> %b, <vscale x 32 x i16> %c) {
+define <vscale x 32 x i16> @smin_split_i16(<vscale x 32 x i16> %a, <vscale x 32 x i16> %b) {
 ; CHECK-LABEL: smin_split_i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.h
@@ -252,7 +252,7 @@ define <vscale x 32 x i16> @smin_split_i16(<vscale x 32 x i16> %a, <vscale x 32
   ret <vscale x 32 x i16> %min
 }
 
-define <vscale x 8 x i32> @smin_split_i32(<vscale x 8 x i32> %a, <vscale x 8 x i32> %b, <vscale x 8 x i32> %c) {
+define <vscale x 8 x i32> @smin_split_i32(<vscale x 8 x i32> %a, <vscale x 8 x i32> %b) {
 ; CHECK-LABEL: smin_split_i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.s
@@ -264,7 +264,7 @@ define <vscale x 8 x i32> @smin_split_i32(<vscale x 8 x i32> %a, <vscale x 8 x i
   ret <vscale x 8 x i32> %min
 }
 
-define <vscale x 4 x i64> @smin_split_i64(<vscale x 4 x i64> %a, <vscale x 4 x i64> %b, <vscale x 4 x i64> %c) {
+define <vscale x 4 x i64> @smin_split_i64(<vscale x 4 x i64> %a, <vscale x 4 x i64> %b) {
 ; CHECK-LABEL: smin_split_i64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d
@@ -276,7 +276,7 @@ define <vscale x 4 x i64> @smin_split_i64(<vscale x 4 x i64> %a, <vscale x 4 x i
   ret <vscale x 4 x i64> %min
 }
 
-define <vscale x 8 x i8> @smin_promote_i8(<vscale x 8 x i8> %a, <vscale x 8 x i8> %b, <vscale x 8 x i8> %c) {
+define <vscale x 8 x i8> @smin_promote_i8(<vscale x 8 x i8> %a, <vscale x 8 x i8> %b) {
 ; CHECK-LABEL: smin_promote_i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.h
@@ -289,7 +289,7 @@ define <vscale x 8 x i8> @smin_promote_i8(<vscale x 8 x i8> %a, <vscale x 8 x i8
   ret <vscale x 8 x i8> %min
 }
 
-define <vscale x 4 x i16> @smin_promote_i16(<vscale x 4 x i16> %a, <vscale x 4 x i16> %b, <vscale x 4 x i16> %c) {
+define <vscale x 4 x i16> @smin_promote_i16(<vscale x 4 x i16> %a, <vscale x 4 x i16> %b) {
 ; CHECK-LABEL: smin_promote_i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.s
@@ -302,7 +302,7 @@ define <vscale x 4 x i16> @smin_promote_i16(<vscale x 4 x i16> %a, <vscale x 4 x
   ret <vscale x 4 x i16> %min
 }
 
-define <vscale x 2 x i32> @smin_promote_i32(<vscale x 2 x i32> %a, <vscale x 2 x i32> %b, <vscale x 2 x i32> %c) {
+define <vscale x 2 x i32> @smin_promote_i32(<vscale x 2 x i32> %a, <vscale x 2 x i32> %b) {
 ; CHECK-LABEL: smin_promote_i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d
@@ -319,7 +319,7 @@ define <vscale x 2 x i32> @smin_promote_i32(<vscale x 2 x i32> %a, <vscale x 2 x
 ; UMIN
 ;
 
-define <vscale x 16 x i8> @umin_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %c) {
+define <vscale x 16 x i8> @umin_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
 ; CHECK-LABEL: umin_i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.b
@@ -330,7 +330,7 @@ define <vscale x 16 x i8> @umin_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b,
   ret <vscale x 16 x i8> %min
 }
 
-define <vscale x 8 x i16> @umin_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b, <vscale x 8 x i16> %c) {
+define <vscale x 8 x i16> @umin_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
 ; CHECK-LABEL: umin_i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.h
@@ -341,7 +341,7 @@ define <vscale x 8 x i16> @umin_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b
   ret <vscale x 8 x i16> %min
 }
 
-define <vscale x 4 x i32> @umin_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c) {
+define <vscale x 4 x i32> @umin_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
 ; CHECK-LABEL: umin_i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.s
@@ -352,7 +352,7 @@ define <vscale x 4 x i32> @umin_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b
   ret <vscale x 4 x i32> %min
 }
 
-define <vscale x 2 x i64> @umin_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b, <vscale x 2 x i64> %c) {
+define <vscale x 2 x i64> @umin_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
 ; CHECK-LABEL: umin_i64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d
@@ -363,7 +363,7 @@ define <vscale x 2 x i64> @umin_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b
   ret <vscale x 2 x i64> %min
 }
 
-define <vscale x 4 x i64> @umin_split_i64(<vscale x 4 x i64> %a, <vscale x 4 x i64> %b, <vscale x 4 x i64> %c) {
+define <vscale x 4 x i64> @umin_split_i64(<vscale x 4 x i64> %a, <vscale x 4 x i64> %b) {
 ; CHECK-LABEL: umin_split_i64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d
@@ -375,7 +375,7 @@ define <vscale x 4 x i64> @umin_split_i64(<vscale x 4 x i64> %a, <vscale x 4 x i
   ret <vscale x 4 x i64> %min
 }
 
-define <vscale x 8 x i8> @umin_promote_i8(<vscale x 8 x i8> %a, <vscale x 8 x i8> %b, <vscale x 8 x i8> %c) {
+define <vscale x 8 x i8> @umin_promote_i8(<vscale x 8 x i8> %a, <vscale x 8 x i8> %b) {
 ; CHECK-LABEL: umin_promote_i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.h
@@ -392,7 +392,7 @@ define <vscale x 8 x i8> @umin_promote_i8(<vscale x 8 x i8> %a, <vscale x 8 x i8
 ; SMAX
 ;
 
-define <vscale x 16 x i8> @smax_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %c) {
+define <vscale x 16 x i8> @smax_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
 ; CHECK-LABEL: smax_i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.b
@@ -403,7 +403,7 @@ define <vscale x 16 x i8> @smax_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b,
   ret <vscale x 16 x i8> %max
 }
 
-define <vscale x 8 x i16> @smax_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b, <vscale x 8 x i16> %c) {
+define <vscale x 8 x i16> @smax_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
 ; CHECK-LABEL: smax_i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.h
@@ -414,7 +414,7 @@ define <vscale x 8 x i16> @smax_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b
   ret <vscale x 8 x i16> %max
 }
 
-define <vscale x 4 x i32> @smax_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c) {
+define <vscale x 4 x i32> @smax_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
 ; CHECK-LABEL: smax_i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.s
@@ -425,7 +425,7 @@ define <vscale x 4 x i32> @smax_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b
   ret <vscale x 4 x i32> %max
 }
 
-define <vscale x 2 x i64> @smax_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b, <vscale x 2 x i64> %c) {
+define <vscale x 2 x i64> @smax_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
 ; CHECK-LABEL: smax_i64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d
@@ -436,7 +436,7 @@ define <vscale x 2 x i64> @smax_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b
   ret <vscale x 2 x i64> %max
 }
 
-define <vscale x 8 x i32> @smax_split_i32(<vscale x 8 x i32> %a, <vscale x 8 x i32> %b, <vscale x 8 x i32> %c) {
+define <vscale x 8 x i32> @smax_split_i32(<vscale x 8 x i32> %a, <vscale x 8 x i32> %b) {
 ; CHECK-LABEL: smax_split_i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.s
@@ -448,7 +448,7 @@ define <vscale x 8 x i32> @smax_split_i32(<vscale x 8 x i32> %a, <vscale x 8 x i
   ret <vscale x 8 x i32> %max
 }
 
-define <vscale x 4 x i16> @smax_promote_i16(<vscale x 4 x i16> %a, <vscale x 4 x i16> %b, <vscale x 4 x i16> %c) {
+define <vscale x 4 x i16> @smax_promote_i16(<vscale x 4 x i16> %a, <vscale x 4 x i16> %b) {
 ; CHECK-LABEL: smax_promote_i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.s
@@ -465,7 +465,7 @@ define <vscale x 4 x i16> @smax_promote_i16(<vscale x 4 x i16> %a, <vscale x 4 x
 ; UMAX
 ;
 
-define <vscale x 16 x i8> @umax_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %c) {
+define <vscale x 16 x i8> @umax_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
 ; CHECK-LABEL: umax_i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.b
@@ -476,7 +476,7 @@ define <vscale x 16 x i8> @umax_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b,
   ret <vscale x 16 x i8> %max
 }
 
-define <vscale x 8 x i16> @umax_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b, <vscale x 8 x i16> %c) {
+define <vscale x 8 x i16> @umax_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
 ; CHECK-LABEL: umax_i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.h
@@ -487,7 +487,7 @@ define <vscale x 8 x i16> @umax_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b
   ret <vscale x 8 x i16> %max
 }
 
-define <vscale x 4 x i32> @umax_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c) {
+define <vscale x 4 x i32> @umax_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
 ; CHECK-LABEL: umax_i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.s
@@ -498,7 +498,7 @@ define <vscale x 4 x i32> @umax_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b
   ret <vscale x 4 x i32> %max
 }
 
-define <vscale x 2 x i64> @umax_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b, <vscale x 2 x i64> %c) {
+define <vscale x 2 x i64> @umax_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
 ; CHECK-LABEL: umax_i64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d
@@ -509,7 +509,7 @@ define <vscale x 2 x i64> @umax_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b
   ret <vscale x 2 x i64> %max
 }
 
-define <vscale x 16 x i16> @umax_split_i16(<vscale x 16 x i16> %a, <vscale x 16 x i16> %b, <vscale x 16 x i16> %c) {
+define <vscale x 16 x i16> @umax_split_i16(<vscale x 16 x i16> %a, <vscale x 16 x i16> %b) {
 ; CHECK-LABEL: umax_split_i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.h
@@ -521,7 +521,7 @@ define <vscale x 16 x i16> @umax_split_i16(<vscale x 16 x i16> %a, <vscale x 16
   ret <vscale x 16 x i16> %max
 }
 
-define <vscale x 2 x i32> @umax_promote_i32(<vscale x 2 x i32> %a, <vscale x 2 x i32> %b, <vscale x 2 x i32> %c) {
+define <vscale x 2 x i32> @umax_promote_i32(<vscale x 2 x i32> %a, <vscale x 2 x i32> %b) {
 ; CHECK-LABEL: umax_promote_i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d
@@ -601,6 +601,50 @@ define <vscale x 2 x i32> @asr_promote_i32(<vscale x 2 x i32> %a, <vscale x 2 x
   ret <vscale x 2 x i32> %shr
 }
 
+;
+; ASRR
+;
+
+define <vscale x 16 x i8> @asrr_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b){
+; CHECK-LABEL: asrr_i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.b
+; CHECK-NEXT:    asrr z0.b, p0/m, z0.b, z1.b
+; CHECK-NEXT:    ret
+  %shr = ashr <vscale x 16 x i8> %b, %a
+  ret <vscale x 16 x i8> %shr
+}
+
+define <vscale x 8 x i16> @asrr_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b){
+; CHECK-LABEL: asrr_i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    asrr z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT:    ret
+  %shr = ashr <vscale x 8 x i16> %b, %a
+  ret <vscale x 8 x i16> %shr
+}
+
+define <vscale x 4 x i32> @asrr_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b){
+; CHECK-LABEL: asrr_i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    asrr z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT:    ret
+  %shr = ashr <vscale x 4 x i32> %b, %a
+  ret <vscale x 4 x i32> %shr
+}
+
+define <vscale x 2 x i64> @asrr_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b){
+; CHECK-LABEL: asrr_i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    asrr z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT:    ret
+  %shr = ashr <vscale x 2 x i64> %b, %a
+  ret <vscale x 2 x i64> %shr
+}
+
 ;
 ; LSL
 ;
@@ -667,6 +711,50 @@ define <vscale x 4 x i16> @lsl_promote_i16(<vscale x 4 x i16> %a, <vscale x 4 x
   ret <vscale x 4 x i16> %shl
 }
 
+;
+; LSLR
+;
+
+define <vscale x 16 x i8> @lslr_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b){
+; CHECK-LABEL: lslr_i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.b
+; CHECK-NEXT:    lslr z0.b, p0/m, z0.b, z1.b
+; CHECK-NEXT:    ret
+  %shl = shl <vscale x 16 x i8> %b, %a
+  ret <vscale x 16 x i8> %shl
+}
+
+define <vscale x 8 x i16> @lslr_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b){
+; CHECK-LABEL: lslr_i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    lslr z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT:    ret
+  %shl = shl <vscale x 8 x i16> %b, %a
+  ret <vscale x 8 x i16> %shl
+}
+
+define <vscale x 4 x i32> @lslr_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b){
+; CHECK-LABEL: lslr_i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    lslr z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT:    ret
+  %shl = shl <vscale x 4 x i32> %b, %a
+  ret <vscale x 4 x i32> %shl
+}
+
+define <vscale x 2 x i64> @lslr_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b){
+; CHECK-LABEL: lslr_i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    lslr z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT:    ret
+  %shl = shl <vscale x 2 x i64> %b, %a
+  ret <vscale x 2 x i64> %shl
+}
+
 ;
 ; LSR
 ;
@@ -734,6 +822,50 @@ define <vscale x 8 x i32> @lsr_split_i32(<vscale x 8 x i32> %a, <vscale x 8 x i3
   ret <vscale x 8 x i32> %shr
 }
 
+;
+; LSRR
+;
+
+define <vscale x 16 x i8> @lsrr_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b){
+; CHECK-LABEL: lsrr_i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.b
+; CHECK-NEXT:    lsrr z0.b, p0/m, z0.b, z1.b
+; CHECK-NEXT:    ret
+  %shr = lshr <vscale x 16 x i8> %b, %a
+  ret <vscale x 16 x i8> %shr
+}
+
+define <vscale x 8 x i16> @lsrr_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b){
+; CHECK-LABEL: lsrr_i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    lsrr z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT:    ret
+  %shr = lshr <vscale x 8 x i16> %b, %a
+  ret <vscale x 8 x i16> %shr
+}
+
+define <vscale x 4 x i32> @lsrr_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b){
+; CHECK-LABEL: lsrr_i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    lsrr z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT:    ret
+  %shr = lshr <vscale x 4 x i32> %b, %a
+  ret <vscale x 4 x i32> %shr
+}
+
+define <vscale x 2 x i64> @lsrr_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b){
+; CHECK-LABEL: lsrr_i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    lsrr z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT:    ret
+  %shr = lshr <vscale x 2 x i64> %b, %a
+  ret <vscale x 2 x i64> %shr
+}
+
 ;
 ; CMP
 ;

From 79b44a4d470041acf202027054ba86e935d86aa1 Mon Sep 17 00:00:00 2001
From: Xing GUO <higuoxing@gmail.com>
Date: Tue, 4 Aug 2020 16:47:38 +0800
Subject: [PATCH 1021/1035] [YAMLTraits] Fix mapping <none> value that followed
 by comments.

When mapping an optional value, if the value is <none> and followed
by comments, there will be a parsing error. This patch helps fix this
issue.

e.g.,

When mapping the following YAML,

```
Sections:
  - Name:  blah
    Type:  SHT_foo
    Flags: [[FLAGS=<none>]] ## some comments.
```

the raw value of `ScalarNode` is "<none> " rather than "<none>". We need
to remove the spaces.

Differential Revision: https://reviews.llvm.org/D85180
---
 llvm/include/llvm/Support/YAMLTraits.h       | 4 +++-
 llvm/test/tools/yaml2obj/ELF/none-value.yaml | 1 +
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/llvm/include/llvm/Support/YAMLTraits.h b/llvm/include/llvm/Support/YAMLTraits.h
index e52bf7892d711..acb1d61cf569d 100644
--- a/llvm/include/llvm/Support/YAMLTraits.h
+++ b/llvm/include/llvm/Support/YAMLTraits.h
@@ -1629,7 +1629,9 @@ void IO::processKeyWithDefault(const char *Key, Optional<T> &Val,
     bool IsNone = false;
     if (!outputting())
       if (auto *Node = dyn_cast<ScalarNode>(((Input *)this)->getCurrentNode()))
-        IsNone = Node->getRawValue() == "<none>";
+        // We use rtrim to ignore possible white spaces that might exist when a
+        // comment is present on the same line.
+        IsNone = Node->getRawValue().rtrim(' ') == "<none>";
 
     if (IsNone)
       Val = DefaultValue;
diff --git a/llvm/test/tools/yaml2obj/ELF/none-value.yaml b/llvm/test/tools/yaml2obj/ELF/none-value.yaml
index 786a9b53aba78..7993e54c53cfc 100644
--- a/llvm/test/tools/yaml2obj/ELF/none-value.yaml
+++ b/llvm/test/tools/yaml2obj/ELF/none-value.yaml
@@ -21,6 +21,7 @@ FileHeader:
 Sections:
   - Name:         .bar
     Type:         SHT_PROGBITS
+    Flags:        [[TEST=<none>]] ## Comment
     Offset:       [[TEST=<none>]]
     Address:      [[TEST=<none>]]
     Content:      [[TEST=<none>]]

From fd6584a22043b254a323635c142b28ce80ae5b5b Mon Sep 17 00:00:00 2001
From: Sander de Smalen <sander.desmalen@arm.com>
Date: Tue, 4 Aug 2020 10:58:47 +0100
Subject: [PATCH 1022/1035] [AArch64][SVE] Fix CFA calculation in presence of
 SVE objects.

The CFA is calculated as (SP/FP + offset), but when there are
SVE objects on the stack the SP offset is partly scalable and
should instead be expressed as the DWARF expression:

     SP + offset + scalable_offset * VG

where VG is the Vector Granule register, containing the
number of 64bits 'granules' in a scalable vector.

Reviewed By: efriedma

Differential Revision: https://reviews.llvm.org/D84043
---
 llvm/include/llvm/MC/MCDwarf.h                |  15 +-
 .../CodeGen/AsmPrinter/AsmPrinterDwarf.cpp    |   1 +
 .../Target/AArch64/AArch64FrameLowering.cpp   |  74 ++++++-
 .../lib/Target/AArch64/AArch64FrameLowering.h |   7 +
 .../lib/Target/AArch64/AArch64RegisterInfo.td |   3 +
 llvm/lib/Target/AArch64/AArch64StackOffset.h  |  12 ++
 llvm/test/CodeGen/AArch64/framelayout-sve.mir | 186 +++++++++++++++---
 llvm/test/CodeGen/AArch64/sve-trunc.ll        |   2 +-
 8 files changed, 267 insertions(+), 33 deletions(-)

diff --git a/llvm/include/llvm/MC/MCDwarf.h b/llvm/include/llvm/MC/MCDwarf.h
index e3cea0ae64cf3..70da5f76e7665 100644
--- a/llvm/include/llvm/MC/MCDwarf.h
+++ b/llvm/include/llvm/MC/MCDwarf.h
@@ -467,10 +467,12 @@ class MCCFIInstruction {
     unsigned Register2;
   };
   std::vector<char> Values;
+  std::string Comment;
 
-  MCCFIInstruction(OpType Op, MCSymbol *L, unsigned R, int O, StringRef V)
+  MCCFIInstruction(OpType Op, MCSymbol *L, unsigned R, int O, StringRef V,
+                   StringRef Comment = "")
       : Operation(Op), Label(L), Register(R), Offset(O),
-        Values(V.begin(), V.end()) {
+        Values(V.begin(), V.end()), Comment(Comment) {
     assert(Op != OpRegister);
   }
 
@@ -570,8 +572,9 @@ class MCCFIInstruction {
 
   /// .cfi_escape Allows the user to add arbitrary bytes to the unwind
   /// info.
-  static MCCFIInstruction createEscape(MCSymbol *L, StringRef Vals) {
-    return MCCFIInstruction(OpEscape, L, 0, 0, Vals);
+  static MCCFIInstruction createEscape(MCSymbol *L, StringRef Vals,
+                                       StringRef Comment = "") {
+    return MCCFIInstruction(OpEscape, L, 0, 0, Vals, Comment);
   }
 
   /// A special wrapper for .cfi_escape that indicates GNU_ARGS_SIZE
@@ -606,6 +609,10 @@ class MCCFIInstruction {
     assert(Operation == OpEscape);
     return StringRef(&Values[0], Values.size());
   }
+
+  StringRef getComment() const {
+    return Comment;
+  }
 };
 
 struct MCDwarfFrameInfo {
diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp
index d81a9be26d39b..b6a9a95683603 100644
--- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp
@@ -241,6 +241,7 @@ void AsmPrinter::emitCFIInstruction(const MCCFIInstruction &Inst) const {
     OutStreamer->emitCFIGnuArgsSize(Inst.getOffset());
     break;
   case MCCFIInstruction::OpEscape:
+    OutStreamer->AddComment(Inst.getComment());
     OutStreamer->emitCFIEscape(Inst.getValues());
     break;
   case MCCFIInstruction::OpRestore:
diff --git a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
index 4789a9f02937a..177d5e24fdb3f 100644
--- a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
@@ -148,6 +148,7 @@
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/LEB128.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Target/TargetMachine.h"
@@ -399,6 +400,64 @@ static bool ShouldSignReturnAddress(MachineFunction &MF) {
   return false;
 }
 
+// Convenience function to create a DWARF expression for
+//   Expr + NumBytes + NumVGScaledBytes * AArch64::VG
+static void appendVGScaledOffsetExpr(SmallVectorImpl<char> &Expr,
+                                     int NumBytes, int NumVGScaledBytes, unsigned VG,
+                                     llvm::raw_string_ostream &Comment) {
+  uint8_t buffer[16];
+
+  if (NumBytes) {
+    Expr.push_back(dwarf::DW_OP_consts);
+    Expr.append(buffer, buffer + encodeSLEB128(NumBytes, buffer));
+    Expr.push_back((uint8_t)dwarf::DW_OP_plus);
+    Comment << (NumBytes < 0 ? " - " : " + ") << std::abs(NumBytes);
+  }
+
+  if (NumVGScaledBytes) {
+    Expr.push_back((uint8_t)dwarf::DW_OP_consts);
+    Expr.append(buffer, buffer + encodeSLEB128(NumVGScaledBytes, buffer));
+
+    Expr.push_back((uint8_t)dwarf::DW_OP_bregx);
+    Expr.append(buffer, buffer + encodeULEB128(VG, buffer));
+    Expr.push_back(0);
+
+    Expr.push_back((uint8_t)dwarf::DW_OP_mul);
+    Expr.push_back((uint8_t)dwarf::DW_OP_plus);
+
+    Comment << (NumVGScaledBytes < 0 ? " - " : " + ")
+            << std::abs(NumVGScaledBytes) << " * VG";
+  }
+}
+
+// Creates an MCCFIInstruction:
+//    { DW_CFA_def_cfa_expression, ULEB128 (sizeof expr), expr }
+MCCFIInstruction AArch64FrameLowering::createDefCFAExpressionFromSP(
+    const TargetRegisterInfo &TRI, const StackOffset &OffsetFromSP) const {
+  int64_t NumBytes, NumVGScaledBytes;
+  OffsetFromSP.getForDwarfOffset(NumBytes, NumVGScaledBytes);
+
+  std::string CommentBuffer = "sp";
+  llvm::raw_string_ostream Comment(CommentBuffer);
+
+  // Build up the expression (SP + NumBytes + NumVGScaledBytes * AArch64::VG)
+  SmallString<64> Expr;
+  Expr.push_back(dwarf::DW_OP_breg0 + /*SP*/ 31);
+  Expr.push_back(0);
+  appendVGScaledOffsetExpr(Expr, NumBytes, NumVGScaledBytes,
+                           TRI.getDwarfRegNum(AArch64::VG, true), Comment);
+
+  // Wrap this into DW_CFA_def_cfa.
+  SmallString<64> DefCfaExpr;
+  DefCfaExpr.push_back(dwarf::DW_CFA_def_cfa_expression);
+  uint8_t buffer[16];
+  DefCfaExpr.append(buffer,
+                    buffer + encodeULEB128(Expr.size(), buffer));
+  DefCfaExpr.append(Expr.str());
+  return MCCFIInstruction::createEscape(nullptr, DefCfaExpr.str(),
+                                        Comment.str());
+}
+
 void AArch64FrameLowering::emitCalleeSavedFrameMoves(
     MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const {
   MachineFunction &MF = *MBB.getParent();
@@ -1383,9 +1442,18 @@ void AArch64FrameLowering::emitPrologue(MachineFunction &MF,
           .addCFIIndex(CFIIndex)
           .setMIFlags(MachineInstr::FrameSetup);
     } else {
-      // Encode the stack size of the leaf function.
-      unsigned CFIIndex = MF.addFrameInst(
-          MCCFIInstruction::cfiDefCfaOffset(nullptr, MFI.getStackSize()));
+      unsigned CFIIndex;
+      if (SVEStackSize) {
+        const TargetSubtargetInfo &STI = MF.getSubtarget();
+        const TargetRegisterInfo &TRI = *STI.getRegisterInfo();
+        StackOffset TotalSize =
+            SVEStackSize + StackOffset((int64_t)MFI.getStackSize(), MVT::i8);
+        CFIIndex = MF.addFrameInst(createDefCFAExpressionFromSP(TRI, TotalSize));
+      } else {
+        // Encode the stack size of the leaf function.
+        CFIIndex = MF.addFrameInst(
+            MCCFIInstruction::cfiDefCfaOffset(nullptr, MFI.getStackSize()));
+      }
       BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
           .addCFIIndex(CFIIndex)
           .setMIFlags(MachineInstr::FrameSetup);
diff --git a/llvm/lib/Target/AArch64/AArch64FrameLowering.h b/llvm/lib/Target/AArch64/AArch64FrameLowering.h
index 444740cb50ab9..753593df2b4dc 100644
--- a/llvm/lib/Target/AArch64/AArch64FrameLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64FrameLowering.h
@@ -18,6 +18,8 @@
 
 namespace llvm {
 
+class MCCFIInstruction;
+
 class AArch64FrameLowering : public TargetFrameLowering {
 public:
   explicit AArch64FrameLowering()
@@ -119,6 +121,11 @@ class AArch64FrameLowering : public TargetFrameLowering {
   int64_t assignSVEStackObjectOffsets(MachineFrameInfo &MF,
                                       int &MinCSFrameIndex,
                                       int &MaxCSFrameIndex) const;
+  MCCFIInstruction
+  createDefCFAExpressionFromSP(const TargetRegisterInfo &TRI,
+                               const StackOffset &OffsetFromSP) const;
+  MCCFIInstruction createCfaOffset(const MCRegisterInfo &MRI, unsigned DwarfReg,
+                                   const StackOffset &OffsetFromDefCFA) const;
   bool shouldCombineCSRLocalStackBumpInEpilogue(MachineBasicBlock &MBB,
                                                 unsigned StackBumpBytes) const;
 };
diff --git a/llvm/lib/Target/AArch64/AArch64RegisterInfo.td b/llvm/lib/Target/AArch64/AArch64RegisterInfo.td
index bd05c56009a1d..54b351fda053b 100644
--- a/llvm/lib/Target/AArch64/AArch64RegisterInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64RegisterInfo.td
@@ -133,6 +133,9 @@ def NZCV  : AArch64Reg<0, "nzcv">;
 // First fault status register
 def FFR : AArch64Reg<0, "ffr">, DwarfRegNum<[47]>;
 
+// Purely virtual Vector Granule (VG) Dwarf register
+def VG : AArch64Reg<0, "vg">, DwarfRegNum<[46]>;
+
 // GPR register classes with the intersections of GPR32/GPR32sp and
 // GPR64/GPR64sp for use by the coalescer.
 def GPR32common : RegisterClass<"AArch64", [i32], 32, (sequence "W%u", 0, 30)> {
diff --git a/llvm/lib/Target/AArch64/AArch64StackOffset.h b/llvm/lib/Target/AArch64/AArch64StackOffset.h
index 6fa1c744f77e2..24751a81797d3 100644
--- a/llvm/lib/Target/AArch64/AArch64StackOffset.h
+++ b/llvm/lib/Target/AArch64/AArch64StackOffset.h
@@ -123,6 +123,18 @@ class StackOffset {
     }
   }
 
+  void getForDwarfOffset(int64_t &ByteSized, int64_t &VGSized) const {
+    assert(isValid() && "Invalid frame offset");
+
+    // VGSized offsets are divided by '2', because the VG register is the
+    // the number of 64bit granules as opposed to 128bit vector chunks,
+    // which is how the 'n' in e.g. MVT::nxv1i8 is modelled.
+    // So, for a stack offset of 16 MVT::nxv1i8's, the size is n x 16 bytes.
+    // VG = n * 2 and the dwarf offset must be VG * 8 bytes.
+    ByteSized = Bytes;
+    VGSized = ScalableBytes / 2;
+  }
+
   /// Returns whether the offset is known zero.
   explicit operator bool() const { return Bytes || ScalableBytes; }
 
diff --git a/llvm/test/CodeGen/AArch64/framelayout-sve.mir b/llvm/test/CodeGen/AArch64/framelayout-sve.mir
index 668b243dd79e0..9e2077855c11a 100644
--- a/llvm/test/CodeGen/AArch64/framelayout-sve.mir
+++ b/llvm/test/CodeGen/AArch64/framelayout-sve.mir
@@ -1,4 +1,8 @@
 # RUN: llc -mattr=+sve -mtriple=aarch64-none-linux-gnu -run-pass=prologepilog %s -o - | FileCheck %s
+# RUN: llc -mtriple=aarch64-none-linux-gnu -mattr=+sve -start-before=prologepilog %s -o - | FileCheck %s --check-prefix=ASM
+# RUN: llc -mtriple=aarch64-none-linux-gnu -mattr=+sve -start-before=prologepilog %s -filetype=obj -o %t
+# RUN: llvm-objdump --dwarf=frames %t | FileCheck %s --check-prefix=UNWINDINFO
+# RUN: rm -rf %t
 #
 # Test allocation and deallocation of SVE objects on the stack,
 # as well as using a combination of scalable and non-scalable
@@ -23,19 +27,19 @@
 #
 --- |
 
-  define void @test_allocate_sve() nounwind { entry: unreachable }
-  define void @test_allocate_sve_gpr_callee_saves() nounwind { entry: unreachable }
-  define void @test_allocate_sve_gpr_realigned() nounwind { entry: unreachable }
-  define void @test_address_sve() nounwind { entry: unreachable }
-  define void @test_address_sve_fp() nounwind { entry: unreachable }
-  define void @test_stack_arg_sve() nounwind { entry: unreachable }
-  define void @test_address_sve_out_of_range() nounwind { entry: unreachable }
-  define void @test_address_gpr_vla() nounwind { entry: unreachable }
-  define aarch64_sve_vector_pcs void @save_restore_pregs_sve() nounwind { entry: unreachable }
-  define aarch64_sve_vector_pcs void @save_restore_zregs_sve() nounwind { entry: unreachable }
-  define aarch64_sve_vector_pcs void @save_restore_sve() nounwind { entry: unreachable }
-  define aarch64_sve_vector_pcs void @save_restore_sve_realign() nounwind { entry: unreachable }
-  define aarch64_sve_vector_pcs void @frame_layout() nounwind { entry: unreachable }
+  define void @test_allocate_sve() { entry: unreachable }
+  define void @test_allocate_sve_gpr_callee_saves() { entry: unreachable }
+  define void @test_allocate_sve_gpr_realigned() { entry: unreachable }
+  define void @test_address_sve() { entry: unreachable }
+  define void @test_address_sve_fp() { entry: unreachable }
+  define void @test_stack_arg_sve() { entry: unreachable }
+  define void @test_address_sve_out_of_range() { entry: unreachable }
+  define void @test_address_gpr_vla() { entry: unreachable }
+  define aarch64_sve_vector_pcs void @save_restore_pregs_sve() { entry: unreachable }
+  define aarch64_sve_vector_pcs void @save_restore_zregs_sve() { entry: unreachable }
+  define aarch64_sve_vector_pcs void @save_restore_sve() { entry: unreachable }
+  define aarch64_sve_vector_pcs void @save_restore_sve_realign() { entry: unreachable }
+  define aarch64_sve_vector_pcs void @frame_layout() { entry: unreachable }
 
 ...
 # +----------+
@@ -54,11 +58,19 @@
 # CHECK-NEXT: $sp = frame-setup STRXpre killed $[[SCRATCH:[a-z0-9]+]], $sp, -16
 # CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -2
 # CHECK-NEXT: $sp = frame-setup SUBXri $sp, 16, 0
+# CHECK-COUNT-2: frame-setup CFI_INSTRUCTION
 
 # CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 2
 # CHECK-NEXT: $sp = frame-destroy ADDXri $sp, 16, 0
 # CHECK-NEXT: $sp, $[[SCRATCH]] = frame-destroy LDRXpost $sp, 16
 # CHECK-NEXT: RET_ReallyLR
+
+# ASM-LABEL: test_allocate_sve:
+# ASM:       .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x20, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 32 + 16 * VG
+# ASM-NEXT:  .cfi_offset w29, -16
+#
+# UNWINDINFO:       DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +32, DW_OP_plus, DW_OP_consts +16, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+# UNWINDINFO-NEXT:  DW_CFA_offset: reg29 -16
 name:            test_allocate_sve
 stack:
   - { id: 0, stack-id: sve-vec, size: 18, alignment: 2 }
@@ -85,6 +97,8 @@ body:             |
 # CHECK-NEXT: frame-setup STPXi killed $x21, killed $x20, $sp, 2
 # CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -2
 # CHECK-NEXT: $sp = frame-setup SUBXri $sp, 16, 0
+# CHECK-COUNT-4: frame-setup CFI_INSTRUCTION
+#
 # CHECK-NEXT: $x20 = IMPLICIT_DEF
 # CHECK-NEXT: $x21 = IMPLICIT_DEF
 # CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 2
@@ -92,6 +106,17 @@ body:             |
 # CHECK-NEXT: $x21, $x20 = frame-destroy LDPXi $sp, 2
 # CHECK-NEXT: $sp, $[[SCRATCH]] = frame-destroy LDRXpost $sp, 32
 # CHECK-NEXT: RET_ReallyLR
+#
+# ASM-LABEL: test_allocate_sve_gpr_callee_saves:
+# ASM:       .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x30, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 48 + 16 * VG
+# ASM-NEXT:  .cfi_offset w20, -8
+# ASM-NEXT:  .cfi_offset w21, -16
+# ASM-NEXT:  .cfi_offset w29, -32
+#
+# UNWINDINFO:       DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +48, DW_OP_plus, DW_OP_consts +16, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+# UNWINDINFO-NEXT:  DW_CFA_offset: reg20 -8
+# UNWINDINFO-NEXT:  DW_CFA_offset: reg21 -16
+# UNWINDINFO-NEXT:  DW_CFA_offset: reg29 -32
 name:            test_allocate_sve_gpr_callee_saves
 stack:
   - { id: 0, stack-id: sve-vec, size: 18, alignment: 2 }
@@ -120,9 +145,20 @@ body:             |
 # CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -2
 # CHECK-NEXT: $[[TMP:x[0-9]+]] = frame-setup SUBXri $sp, 16, 0
 # CHECK-NEXT: $sp = ANDXri killed $[[TMP]]
+# CHECK-COUNT-3: frame-setup CFI_INSTRUCTION
 # CHECK-NEXT: $sp = frame-destroy ADDXri $fp, 0, 0
 # CHECK-NEXT: $sp, $fp, $lr = frame-destroy LDPXpost $sp, 2
 # CHECK-NEXT: RET_ReallyLR
+#
+# ASM-LABEL: test_allocate_sve_gpr_realigned:
+# ASM:       .cfi_def_cfa w29, 16
+# ASM-NEXT:  .cfi_offset w30, -8
+# ASM-NEXT:  .cfi_offset w29, -16
+#
+# UNWINDINFO:       DW_CFA_def_cfa: reg29 +16
+# UNWINDINFO-NEXT:  DW_CFA_offset: reg30 -8
+# UNWINDINFO-NEXT:  DW_CFA_offset: reg29 -16
+
 name:            test_allocate_sve_gpr_realigned
 stack:
   - { id: 0, stack-id: sve-vec, size: 18, alignment: 2  }
@@ -149,6 +185,7 @@ body:             |
 # CHECK-NEXT: $sp = frame-setup STRXpre killed $[[SCRATCH:[a-z0-9]+]], $sp, -16
 # CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -3
 # CHECK-NEXT: $sp = frame-setup SUBXri $sp, 16, 0
+# CHECK-COUNT-2: frame-setup CFI_INSTRUCTION
 
 # CHECK-NEXT: $[[TMP:x[0-9]+]] = ADDXri $sp, 16
 # CHECK-NEXT: STR_ZXI $z0, killed $[[TMP]], 2
@@ -161,6 +198,14 @@ body:             |
 # CHECK-NEXT: $sp = frame-destroy ADDXri $sp, 16, 0
 # CHECK-NEXT: $sp, $[[SCRATCH]] = frame-destroy LDRXpost $sp, 16
 # CHECK-NEXT: RET_ReallyLR
+#
+# ASM-LABEL:  test_address_sve:
+# ASM:       .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x20, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 32 + 24 * VG
+# ASM-NEXT:  .cfi_offset w29, -16
+#
+# UNWINDINFO:       DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +32, DW_OP_plus, DW_OP_consts +24, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+# UNWINDINFO-NEXT:  DW_CFA_offset: reg29 -16
+
 name:            test_address_sve
 frameInfo:
   maxAlignment:  16
@@ -199,6 +244,7 @@ body:             |
 # CHECK-NEXT: $fp = frame-setup ADDXri $sp, 0, 0
 # CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -3
 # CHECK-NEXT: $sp = frame-setup SUBXri $sp, 16, 0
+# CHECK-COUNT-3: frame-setup CFI_INSTRUCTION
 
 # CHECK-NEXT: STR_ZXI $z0, $fp, -1
 # CHECK-NEXT: STR_ZXI $z1, $fp, -2
@@ -208,6 +254,15 @@ body:             |
 # CHECK:      $sp = frame-destroy ADDXri $sp, 16, 0
 # CHECK-NEXT: $sp, $fp, $lr = frame-destroy LDPXpost $sp, 2
 # CHECK-NEXT: RET_ReallyLR
+#
+# ASM-LABEL: test_address_sve_fp:
+# ASM:       .cfi_def_cfa w29, 16
+# ASM-NEXT:  .cfi_offset w30, -8
+# ASM-NEXT:  .cfi_offset w29, -16
+#
+# UNWINDINFO:      DW_CFA_def_cfa: reg29 +16
+# UNWINDINFO-NEXT: DW_CFA_offset: reg30 -8
+# UNWINDINFO-NEXT: DW_CFA_offset: reg29 -16
 name:            test_address_sve_fp
 frameInfo:
   maxAlignment:  16
@@ -244,6 +299,7 @@ body:             |
 # CHECK-NEXT: $sp = frame-setup STRXpre killed $[[SCRATCH:[a-z0-9]+]], $sp, -16
 # CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -1
 # CHECK-NEXT: $sp = frame-setup SUBXri $sp, 16, 0
+# CHECK-COUNT-2: frame-setup CFI_INSTRUCTION
 
 # CHECK:      $[[TMP:x[0-9]+]] = ADDVL_XXI $sp, 1
 # CHECK-NEXT: $x0 = LDRXui killed $[[TMP]], 4
@@ -252,6 +308,14 @@ body:             |
 # CHECK-NEXT: $sp = frame-destroy ADDXri $sp, 16, 0
 # CHECK-NEXT: $sp, $[[SCRATCH]] = frame-destroy LDRXpost $sp, 16
 # CHECK-NEXT: RET_ReallyLR
+#
+# ASM-LABEL: test_stack_arg_sve:
+# ASM:       .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x20, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 32 + 8 * VG
+# ASM-NEXT:  .cfi_offset w29, -16
+#
+# UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +32, DW_OP_plus, DW_OP_consts +8, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+# UNWINDINFO-NEXT: DW_CFA_offset: reg29 -16
+
 name:             test_stack_arg_sve
 fixedStack:
   - { id: 0, stack-id: default, size: 16, alignment: 16, offset: 0 }
@@ -292,6 +356,7 @@ body:             |
 # CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -32
 # CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -32
 # CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -1
+# CHECK-COUNT-2: frame-setup CFI_INSTRUCTION
 
 # CHECK-NEXT: $[[TMP2:x[0-9]+]] = ADDVL_XXI $sp, 1
 # CHECK-NEXT: STR_ZXI $z0, killed $[[TMP2]], 255
@@ -310,6 +375,13 @@ body:             |
 # CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 9
 # CHECK-NEXT: $sp, $[[SCRATCH]] = frame-destroy LDRXpost $sp, 16
 # CHECK-NEXT: RET_ReallyLR
+#
+# ASM-LABEL: test_address_sve_out_of_range:
+# ASM:       .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x88, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 2056 * VG
+# ASM-NEXT:  .cfi_offset w29, -16
+#
+# UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +2056, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+# UNWINDINFO-NEXT: DW_CFA_offset: reg29 -16
 name:            test_address_sve_out_of_range
 frameInfo:
   maxAlignment:  16
@@ -344,6 +416,17 @@ body:             |
 # CHECK: bb.0.entry:
 # CHECK: STRXui $xzr, $x19, 0
 # CHECK: RET_ReallyLR
+#
+# ASM-LABEL: test_address_gpr_vla:
+# ASM:       .cfi_def_cfa w29, 32
+# ASM-NEXT:  .cfi_offset w19, -16
+# ASM-NEXT:  .cfi_offset w30, -24
+# ASM-NEXT:  .cfi_offset w29, -32
+#
+# UNWINDINFO:      DW_CFA_def_cfa: reg29 +32
+# UNWINDINFO-NEXT: DW_CFA_offset: reg19 -16
+# UNWINDINFO-NEXT: DW_CFA_offset: reg30 -24
+# UNWINDINFO-NEXT: DW_CFA_offset: reg29 -32
 name:            test_address_gpr_vla
 frameInfo:
   maxAlignment:  16
@@ -366,6 +449,7 @@ body:             |
 # CHECK: frame-setup STR_PXI killed $p5, $sp, 6
 # CHECK: frame-setup STR_PXI killed $p4, $sp, 7
 # CHECK: $sp = frame-setup SUBXri $sp, 32, 0
+# CHECK-COUNT-5: frame-setup CFI_INSTRUCTION
 
 # CHECK: $sp = frame-destroy ADDXri $sp, 32, 0
 # CHECK: $p6 = frame-destroy LDR_PXI $sp, 5
@@ -373,6 +457,15 @@ body:             |
 # CHECK: $p4 = frame-destroy LDR_PXI $sp, 7
 # CHECK: $sp = frame-destroy ADDVL_XXI $sp, 1
 # CHECK: RET_ReallyLR
+#
+# ASM-LABEL: save_restore_pregs_sve:
+# ASM:       .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x30, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 48 + 8 * VG
+# ASM-COUNT-3: .cfi_offset
+# ASM-NEXT:  .cfi_offset w29, -16
+#
+# UNWINDINFO:         DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +48, DW_OP_plus, DW_OP_consts +8, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+# UNWINDINFO-COUNT-3: DW_CFA_offset
+# UNWINDINFO-NEXT:    DW_CFA_offset: reg29 -16
 name: save_restore_pregs_sve
 stack:
   - { id: 0, stack-id: default, size: 32, alignment: 16 }
@@ -387,18 +480,29 @@ body:             |
 ---
 ...
 # CHECK-LABEL: name: save_restore_zregs_sve
-# CHECK: $sp = frame-setup ADDVL_XXI $sp, -3
-# CHECK: frame-setup STR_ZXI killed $z10, $sp, 0
-# CHECK: frame-setup STR_ZXI killed $z9, $sp, 1
-# CHECK: frame-setup STR_ZXI killed $z8, $sp, 2
-# CHECK: $sp = frame-setup SUBXri $sp, 32, 0
-
-# CHECK: $sp  = frame-destroy ADDXri $sp, 32, 0
-# CHECK: $z10 = frame-destroy LDR_ZXI $sp, 0
-# CHECK: $z9  = frame-destroy LDR_ZXI $sp, 1
-# CHECK: $z8  = frame-destroy LDR_ZXI $sp, 2
-# CHECK: $sp  = frame-destroy ADDVL_XXI $sp, 3
-# CHECK: RET_ReallyLR
+# CHECK:      $sp = frame-setup STRXpre killed $fp, $sp, -16
+# CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -3
+# CHECK-NEXT: frame-setup STR_ZXI killed $z10, $sp, 0
+# CHECK-NEXT: frame-setup STR_ZXI killed $z9, $sp, 1
+# CHECK-NEXT: frame-setup STR_ZXI killed $z8, $sp, 2
+# CHECK-NEXT: $sp = frame-setup SUBXri $sp, 32, 0
+# CHECK-COUNT-5: frame-setup CFI_INSTRUCTION
+
+# CHECK:      $sp  = frame-destroy ADDXri $sp, 32, 0
+# CHECK-NEXT: $z10 = frame-destroy LDR_ZXI $sp, 0
+# CHECK-NEXT: $z9  = frame-destroy LDR_ZXI $sp, 1
+# CHECK-NEXT: $z8  = frame-destroy LDR_ZXI $sp, 2
+# CHECK-NEXT: $sp  = frame-destroy ADDVL_XXI $sp, 3
+# CHECK-NEXT: $sp, $fp = frame-destroy LDRXpost $sp, 16
+# CHECK-NEXT: RET_ReallyLR
+#
+# ASM-LABEL: save_restore_zregs_sve:
+# ASM:       .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x30, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 48 + 24 * VG
+# ASM-COUNT-3: .cfi_offset
+#
+# UNWINDINFO:         DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +48, DW_OP_plus, DW_OP_consts +24, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+# UNWINDINFO-COUNT-3: DW_CFA_offset
+# UNWINDINFO-NEXT:    DW_CFA_offset: reg29 -16
 name: save_restore_zregs_sve
 stack:
   - { id: 0, stack-id: default, size: 32, alignment: 16 }
@@ -432,6 +536,7 @@ body:             |
 # CHECK: frame-setup STR_ZXI killed $z8, $sp, 17
 # CHECK: $sp = frame-setup ADDVL_XXI $sp, -1
 # CHECK: $sp = frame-setup SUBXri $sp, 32, 0
+# CHECK-COUNT-33: frame-setup CFI_INSTRUCTION
 
 # CHECK: $sp = frame-destroy ADDXri $sp, 32, 0
 # CHECK: $sp = frame-destroy ADDVL_XXI $sp, 1
@@ -447,6 +552,22 @@ body:             |
 # CHECK: $x20, $x19 = frame-destroy LDPXi $sp, 2
 # CHECK: $sp, ${{[a-z0-9]+}}, $x21 = frame-destroy LDPXpost $sp, 4
 # CHECK: RET_ReallyLR
+#
+# ASM-LABEL: save_restore_sve:
+# ASM:       .cfi_escape 0x0f, 0x0e, 0x8f, 0x00, 0x11, 0xc0, 0x00, 0x22, 0x11, 0x98, 0x01, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 64 + 152 * VG
+# ASM-COUNT-28: .cfi_offset
+# ASM-NEXT:  .cfi_offset w19, -8
+# ASM-NEXT:  .cfi_offset w20, -16
+# ASM-NEXT:  .cfi_offset w21, -24
+# ASM-NEXT:  .cfi_offset w29, -32
+#
+# UNWINDINFO:          DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +64, DW_OP_plus, DW_OP_consts +152, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+# UNWINDINFO-COUNT-28: DW_CFA_offset
+# UNWINDINFO-NEXT:     DW_CFA_offset: reg19 -8
+# UNWINDINFO-NEXT:     DW_CFA_offset: reg20 -16
+# UNWINDINFO-NEXT:     DW_CFA_offset: reg21 -24
+# UNWINDINFO-NEXT:     DW_CFA_offset: reg29 -32
+
 name: save_restore_sve
 stack:
   - { id: 0, stack-id: sve-vec, size: 16, alignment: 16 }
@@ -499,6 +620,7 @@ body:             |
 # CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -1
 # CHECK-NEXT: $[[TMP:x[0-9]+]] = frame-setup SUBXri $sp, 16, 0
 # CHECK-NEXT: $sp = ANDXri killed $[[TMP]]
+# CHECK-COUNT-31: frame-setup CFI_INSTRUCTION
 
 # CHECK:      $sp = frame-destroy ADDVL_XXI $fp, -18
 # CHECK-NEXT: $p15 = frame-destroy LDR_PXI $sp, 4
@@ -512,6 +634,11 @@ body:             |
 # CHECK-NEXT: $sp = frame-destroy ADDXri $fp, 0, 0
 # CHECK-NEXT: $sp, $fp, $lr = frame-destroy LDPXpost $sp, 2
 # CHECK-NEXT: RET_ReallyLR
+#
+# UNWINDINFO:          DW_CFA_def_cfa: reg29 +16
+# UNWINDINFO-COUNT-28: DW_CFA_offset
+# UNWINDINFO-NEXT:     DW_CFA_offset: reg30 -8
+# UNWINDINFO-NEXT:     DW_CFA_offset: reg29 -16
 name: save_restore_sve_realign
 stack:
   - { id: 0, stack-id: sve-vec, size: 16, alignment: 16 }
@@ -586,6 +713,15 @@ body:             |
 # CHECK-NEXT: STR_ZXI killed $z23, $sp, 1
 # CHECK-NEXT: STR_ZXI killed $z8, $sp, 2
 # CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -7
+# CHECK-COUNT-6: frame-setup CFI_INSTRUCTION
+# ASM-LABEL: frame_layout:
+# ASM:       .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0xd0, 0x00, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 80 * VG
+# ASM-COUNT-4: .cfi_offset
+# ASM-NEXT:  .cfi_offset w29, -16
+#
+# UNWINDINFO:         DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +80, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+# UNWINDINFO-COUNT-4: DW_CFA_offset
+# UNWINDINFO-NEXT:    DW_CFA_offset: reg29 -16
 name: frame_layout
 stack:
   - { id: 0, type: default,    size:  32, alignment: 16, stack-id: sve-vec }
diff --git a/llvm/test/CodeGen/AArch64/sve-trunc.ll b/llvm/test/CodeGen/AArch64/sve-trunc.ll
index af50176f6b101..191df22eda506 100644
--- a/llvm/test/CodeGen/AArch64/sve-trunc.ll
+++ b/llvm/test/CodeGen/AArch64/sve-trunc.ll
@@ -116,7 +116,7 @@ define <vscale x 16 x i1> @trunc_i64toi1_split3(<vscale x 16 x i64> %in) {
 ; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-NEXT:    addvl sp, sp, #-1
 ; CHECK-NEXT:    str p4, [sp, #7, mul vl] // 2-byte Folded Spill
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG
 ; CHECK-NEXT:    .cfi_offset p4, -2
 ; CHECK-NEXT:    .cfi_offset w29, -16
 ; CHECK-NEXT:    ptrue p0.d

From bb3344c7d8c2703c910dd481ada43ecaf11536a6 Mon Sep 17 00:00:00 2001
From: Sander de Smalen <sander.desmalen@arm.com>
Date: Tue, 4 Aug 2020 11:10:32 +0100
Subject: [PATCH 1023/1035] [AArch64][SVE] Add missing unwind info for SVE
 registers.

This patch adds a CFI entry for each SVE callee saved register
that needs unwind info at an offset from the CFA. The offset is
a DWARF expression because the offset is partly scalable.

The CFI entries only cover a subset of the SVE callee-saves and
only encodes the lower 64-bits, thus implementing the lowest
common denominator ABI. Existing unwinders may support VG but
only restore the lower 64-bits.

Reviewed By: efriedma

Differential Revision: https://reviews.llvm.org/D84044
---
 .../Target/AArch64/AArch64FrameLowering.cpp   | 60 +++++++++++--
 .../lib/Target/AArch64/AArch64FrameLowering.h |  2 +-
 .../Target/AArch64/AArch64RegisterInfo.cpp    | 23 +++++
 llvm/lib/Target/AArch64/AArch64RegisterInfo.h |  1 +
 llvm/test/CodeGen/AArch64/framelayout-sve.mir | 89 +++++++++++++------
 llvm/test/CodeGen/AArch64/sve-trunc.ll        |  1 -
 6 files changed, 142 insertions(+), 34 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
index 177d5e24fdb3f..30666009801c5 100644
--- a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
@@ -458,12 +458,44 @@ MCCFIInstruction AArch64FrameLowering::createDefCFAExpressionFromSP(
                                         Comment.str());
 }
 
+MCCFIInstruction AArch64FrameLowering::createCfaOffset(
+    const TargetRegisterInfo &TRI, unsigned Reg,
+    const StackOffset &OffsetFromDefCFA) const {
+  int64_t NumBytes, NumVGScaledBytes;
+  OffsetFromDefCFA.getForDwarfOffset(NumBytes, NumVGScaledBytes);
+
+  unsigned DwarfReg = TRI.getDwarfRegNum(Reg, true);
+
+  // Non-scalable offsets can use DW_CFA_offset directly.
+  if (!NumVGScaledBytes)
+    return MCCFIInstruction::createOffset(nullptr, DwarfReg, NumBytes);
+
+  std::string CommentBuffer;
+  llvm::raw_string_ostream Comment(CommentBuffer);
+  Comment << printReg(Reg, &TRI) << "  @ cfa";
+
+  // Build up expression (NumBytes + NumVGScaledBytes * AArch64::VG)
+  SmallString<64> OffsetExpr;
+  appendVGScaledOffsetExpr(OffsetExpr, NumBytes, NumVGScaledBytes,
+                           TRI.getDwarfRegNum(AArch64::VG, true), Comment);
+
+  // Wrap this into DW_CFA_expression
+  SmallString<64> CfaExpr;
+  CfaExpr.push_back(dwarf::DW_CFA_expression);
+  uint8_t buffer[16];
+  CfaExpr.append(buffer, buffer + encodeULEB128(DwarfReg, buffer));
+  CfaExpr.append(buffer, buffer + encodeULEB128(OffsetExpr.size(), buffer));
+  CfaExpr.append(OffsetExpr.str());
+
+  return MCCFIInstruction::createEscape(nullptr, CfaExpr.str(), Comment.str());
+}
+
 void AArch64FrameLowering::emitCalleeSavedFrameMoves(
     MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const {
   MachineFunction &MF = *MBB.getParent();
   MachineFrameInfo &MFI = MF.getFrameInfo();
   const TargetSubtargetInfo &STI = MF.getSubtarget();
-  const MCRegisterInfo *MRI = STI.getRegisterInfo();
+  const TargetRegisterInfo *TRI = STI.getRegisterInfo();
   const TargetInstrInfo *TII = STI.getInstrInfo();
   DebugLoc DL = MBB.findDebugLoc(MBBI);
 
@@ -474,11 +506,26 @@ void AArch64FrameLowering::emitCalleeSavedFrameMoves(
 
   for (const auto &Info : CSI) {
     unsigned Reg = Info.getReg();
-    int64_t Offset =
-        MFI.getObjectOffset(Info.getFrameIdx()) - getOffsetOfLocalArea();
-    unsigned DwarfReg = MRI->getDwarfRegNum(Reg, true);
-    unsigned CFIIndex = MF.addFrameInst(
-        MCCFIInstruction::createOffset(nullptr, DwarfReg, Offset));
+
+    // Not all unwinders may know about SVE registers, so assume the lowest
+    // common demoninator.
+    unsigned NewReg;
+    if (static_cast<const AArch64RegisterInfo *>(TRI)->regNeedsCFI(Reg, NewReg))
+      Reg = NewReg;
+    else
+      continue;
+
+    StackOffset Offset;
+    if (MFI.getStackID(Info.getFrameIdx()) == TargetStackID::SVEVector) {
+      AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
+      Offset = StackOffset(MFI.getObjectOffset(Info.getFrameIdx()), MVT::nxv1i8) -
+               StackOffset(AFI->getCalleeSavedStackSize(MFI), MVT::i8);
+    } else {
+      Offset = {MFI.getObjectOffset(Info.getFrameIdx()) -
+                    getOffsetOfLocalArea(),
+                MVT::i8};
+    }
+    unsigned CFIIndex = MF.addFrameInst(createCfaOffset(*TRI, Reg, Offset));
     BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
         .addCFIIndex(CFIIndex)
         .setMIFlags(MachineInstr::FrameSetup);
@@ -2074,6 +2121,7 @@ static void computeCalleeSaveRegisterPairs(
   // available unwind codes.  This flag assures that the alignment fixup is done
   // only once, as intened.
   bool FixupDone = false;
+
   for (unsigned i = 0; i < Count; ++i) {
     RegPairInfo RPI;
     RPI.Reg1 = CSI[i].getReg();
diff --git a/llvm/lib/Target/AArch64/AArch64FrameLowering.h b/llvm/lib/Target/AArch64/AArch64FrameLowering.h
index 753593df2b4dc..1ca8c3e9e2bf6 100644
--- a/llvm/lib/Target/AArch64/AArch64FrameLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64FrameLowering.h
@@ -124,7 +124,7 @@ class AArch64FrameLowering : public TargetFrameLowering {
   MCCFIInstruction
   createDefCFAExpressionFromSP(const TargetRegisterInfo &TRI,
                                const StackOffset &OffsetFromSP) const;
-  MCCFIInstruction createCfaOffset(const MCRegisterInfo &MRI, unsigned DwarfReg,
+  MCCFIInstruction createCfaOffset(const TargetRegisterInfo &MRI, unsigned DwarfReg,
                                    const StackOffset &OffsetFromDefCFA) const;
   bool shouldCombineCSRLocalStackBumpInEpilogue(MachineBasicBlock &MBB,
                                                 unsigned StackBumpBytes) const;
diff --git a/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp b/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
index 83a488afc7972..62cc865fd1c36 100644
--- a/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
@@ -40,6 +40,29 @@ AArch64RegisterInfo::AArch64RegisterInfo(const Triple &TT)
   AArch64_MC::initLLVMToCVRegMapping(this);
 }
 
+/// Return whether the register needs a CFI entry. Not all unwinders may know
+/// about SVE registers, so we assume the lowest common denominator, i.e. the
+/// callee-saves required by the base ABI. For the SVE registers z8-z15 only the
+/// lower 64-bits (d8-d15) need to be saved. The lower 64-bits subreg is
+/// returned in \p RegToUseForCFI.
+bool AArch64RegisterInfo::regNeedsCFI(unsigned Reg,
+                                      unsigned &RegToUseForCFI) const {
+  if (AArch64::PPRRegClass.contains(Reg))
+    return false;
+
+  if (AArch64::ZPRRegClass.contains(Reg)) {
+    RegToUseForCFI = getSubReg(Reg, AArch64::dsub);
+    for (int I = 0; CSR_AArch64_AAPCS_SaveList[I]; ++I) {
+      if (CSR_AArch64_AAPCS_SaveList[I] == RegToUseForCFI)
+        return true;
+    }
+    return false;
+  }
+
+  RegToUseForCFI = Reg;
+  return true;
+}
+
 static bool hasSVEArgsOrReturn(const MachineFunction *MF) {
   const Function &F = MF->getFunction();
   return isa<ScalableVectorType>(F.getReturnType()) ||
diff --git a/llvm/lib/Target/AArch64/AArch64RegisterInfo.h b/llvm/lib/Target/AArch64/AArch64RegisterInfo.h
index 22a8ba76c6111..91064787d3dac 100644
--- a/llvm/lib/Target/AArch64/AArch64RegisterInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64RegisterInfo.h
@@ -122,6 +122,7 @@ class AArch64RegisterInfo final : public AArch64GenRegisterInfo {
                                MachineFunction &MF) const override;
 
   unsigned getLocalAddressRegister(const MachineFunction &MF) const;
+  bool regNeedsCFI(unsigned Reg, unsigned &RegToUseForCFI) const;
 };
 
 } // end namespace llvm
diff --git a/llvm/test/CodeGen/AArch64/framelayout-sve.mir b/llvm/test/CodeGen/AArch64/framelayout-sve.mir
index 9e2077855c11a..e6eb9e7a3d3ed 100644
--- a/llvm/test/CodeGen/AArch64/framelayout-sve.mir
+++ b/llvm/test/CodeGen/AArch64/framelayout-sve.mir
@@ -449,7 +449,7 @@ body:             |
 # CHECK: frame-setup STR_PXI killed $p5, $sp, 6
 # CHECK: frame-setup STR_PXI killed $p4, $sp, 7
 # CHECK: $sp = frame-setup SUBXri $sp, 32, 0
-# CHECK-COUNT-5: frame-setup CFI_INSTRUCTION
+# CHECK-COUNT-2: frame-setup CFI_INSTRUCTION
 
 # CHECK: $sp = frame-destroy ADDXri $sp, 32, 0
 # CHECK: $p6 = frame-destroy LDR_PXI $sp, 5
@@ -460,11 +460,9 @@ body:             |
 #
 # ASM-LABEL: save_restore_pregs_sve:
 # ASM:       .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x30, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 48 + 8 * VG
-# ASM-COUNT-3: .cfi_offset
 # ASM-NEXT:  .cfi_offset w29, -16
 #
 # UNWINDINFO:         DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +48, DW_OP_plus, DW_OP_consts +8, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
-# UNWINDINFO-COUNT-3: DW_CFA_offset
 # UNWINDINFO-NEXT:    DW_CFA_offset: reg29 -16
 name: save_restore_pregs_sve
 stack:
@@ -498,11 +496,16 @@ body:             |
 #
 # ASM-LABEL: save_restore_zregs_sve:
 # ASM:       .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x30, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 48 + 24 * VG
-# ASM-COUNT-3: .cfi_offset
-#
-# UNWINDINFO:         DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +48, DW_OP_plus, DW_OP_consts +24, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
-# UNWINDINFO-COUNT-3: DW_CFA_offset
-# UNWINDINFO-NEXT:    DW_CFA_offset: reg29 -16
+# ASM-NEXT:  .cfi_escape 0x10, 0x48, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d8  @ cfa - 16 - 8 * VG
+# ASM-NEXT:  .cfi_escape 0x10, 0x49, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x70, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d9  @ cfa - 16 - 16 * VG
+# ASM-NEXT:  .cfi_escape 0x10, 0x4a, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x68, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d10  @ cfa - 16 - 24 * VG
+
+# UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +48, DW_OP_plus, DW_OP_consts +24, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+# UNWINDINFO-NEXT: DW_CFA_expression: reg72 DW_OP_consts -16, DW_OP_plus, DW_OP_consts -8, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+# UNWINDINFO-NEXT: DW_CFA_expression: reg73 DW_OP_consts -16, DW_OP_plus, DW_OP_consts -16, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+# UNWINDINFO-NEXT: DW_CFA_expression: reg74 DW_OP_consts -16, DW_OP_plus, DW_OP_consts -24, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+# UNWINDINFO-NEXT: DW_CFA_offset: reg29 -16
+
 name: save_restore_zregs_sve
 stack:
   - { id: 0, stack-id: default, size: 32, alignment: 16 }
@@ -536,7 +539,7 @@ body:             |
 # CHECK: frame-setup STR_ZXI killed $z8, $sp, 17
 # CHECK: $sp = frame-setup ADDVL_XXI $sp, -1
 # CHECK: $sp = frame-setup SUBXri $sp, 32, 0
-# CHECK-COUNT-33: frame-setup CFI_INSTRUCTION
+# CHECK-COUNT-13: frame-setup CFI_INSTRUCTION
 
 # CHECK: $sp = frame-destroy ADDXri $sp, 32, 0
 # CHECK: $sp = frame-destroy ADDVL_XXI $sp, 1
@@ -555,18 +558,32 @@ body:             |
 #
 # ASM-LABEL: save_restore_sve:
 # ASM:       .cfi_escape 0x0f, 0x0e, 0x8f, 0x00, 0x11, 0xc0, 0x00, 0x22, 0x11, 0x98, 0x01, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 64 + 152 * VG
-# ASM-COUNT-28: .cfi_offset
+# ASM-NEXT:  .cfi_escape 0x10, 0x48, 0x0a, 0x11, 0x60, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d8 @ cfa - 32 - 8 * VG
+# ASM-NEXT:  .cfi_escape 0x10, 0x49, 0x0a, 0x11, 0x60, 0x22, 0x11, 0x70, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d9 @ cfa - 32 - 16 * VG
+# ASM-NEXT:  .cfi_escape 0x10, 0x4a, 0x0a, 0x11, 0x60, 0x22, 0x11, 0x68, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d10 @ cfa - 32 - 24 * VG
+# ASM-NEXT:  .cfi_escape 0x10, 0x4b, 0x0a, 0x11, 0x60, 0x22, 0x11, 0x60, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d11 @ cfa - 32 - 32 * VG
+# ASM-NEXT:  .cfi_escape 0x10, 0x4c, 0x0a, 0x11, 0x60, 0x22, 0x11, 0x58, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d12 @ cfa - 32 - 40 * VG
+# ASM-NEXT:  .cfi_escape 0x10, 0x4d, 0x0a, 0x11, 0x60, 0x22, 0x11, 0x50, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d13 @ cfa - 32 - 48 * VG
+# ASM-NEXT:  .cfi_escape 0x10, 0x4e, 0x0a, 0x11, 0x60, 0x22, 0x11, 0x48, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d14 @ cfa - 32 - 56 * VG
+# ASM-NEXT:  .cfi_escape 0x10, 0x4f, 0x0a, 0x11, 0x60, 0x22, 0x11, 0x40, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d15 @ cfa - 32 - 64 * VG
 # ASM-NEXT:  .cfi_offset w19, -8
 # ASM-NEXT:  .cfi_offset w20, -16
 # ASM-NEXT:  .cfi_offset w21, -24
 # ASM-NEXT:  .cfi_offset w29, -32
 #
-# UNWINDINFO:          DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +64, DW_OP_plus, DW_OP_consts +152, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
-# UNWINDINFO-COUNT-28: DW_CFA_offset
-# UNWINDINFO-NEXT:     DW_CFA_offset: reg19 -8
-# UNWINDINFO-NEXT:     DW_CFA_offset: reg20 -16
-# UNWINDINFO-NEXT:     DW_CFA_offset: reg21 -24
-# UNWINDINFO-NEXT:     DW_CFA_offset: reg29 -32
+# UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +64, DW_OP_plus, DW_OP_consts +152, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+# UNWINDINFO-NEXT: DW_CFA_expression: reg72 DW_OP_consts -32, DW_OP_plus, DW_OP_consts -8, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+# UNWINDINFO-NEXT: DW_CFA_expression: reg73 DW_OP_consts -32, DW_OP_plus, DW_OP_consts -16, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+# UNWINDINFO-NEXT: DW_CFA_expression: reg74 DW_OP_consts -32, DW_OP_plus, DW_OP_consts -24, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+# UNWINDINFO-NEXT: DW_CFA_expression: reg75 DW_OP_consts -32, DW_OP_plus, DW_OP_consts -32, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+# UNWINDINFO-NEXT: DW_CFA_expression: reg76 DW_OP_consts -32, DW_OP_plus, DW_OP_consts -40, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+# UNWINDINFO-NEXT: DW_CFA_expression: reg77 DW_OP_consts -32, DW_OP_plus, DW_OP_consts -48, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+# UNWINDINFO-NEXT: DW_CFA_expression: reg78 DW_OP_consts -32, DW_OP_plus, DW_OP_consts -56, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+# UNWINDINFO-NEXT: DW_CFA_expression: reg79 DW_OP_consts -32, DW_OP_plus, DW_OP_consts -64, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+# UNWINDINFO-NEXT: DW_CFA_offset: reg19 -8
+# UNWINDINFO-NEXT: DW_CFA_offset: reg20 -16
+# UNWINDINFO-NEXT: DW_CFA_offset: reg21 -24
+# UNWINDINFO-NEXT: DW_CFA_offset: reg29 -32
 
 name: save_restore_sve
 stack:
@@ -620,7 +637,7 @@ body:             |
 # CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -1
 # CHECK-NEXT: $[[TMP:x[0-9]+]] = frame-setup SUBXri $sp, 16, 0
 # CHECK-NEXT: $sp = ANDXri killed $[[TMP]]
-# CHECK-COUNT-31: frame-setup CFI_INSTRUCTION
+# CHECK-COUNT-11: frame-setup CFI_INSTRUCTION
 
 # CHECK:      $sp = frame-destroy ADDVL_XXI $fp, -18
 # CHECK-NEXT: $p15 = frame-destroy LDR_PXI $sp, 4
@@ -635,10 +652,30 @@ body:             |
 # CHECK-NEXT: $sp, $fp, $lr = frame-destroy LDPXpost $sp, 2
 # CHECK-NEXT: RET_ReallyLR
 #
-# UNWINDINFO:          DW_CFA_def_cfa: reg29 +16
-# UNWINDINFO-COUNT-28: DW_CFA_offset
-# UNWINDINFO-NEXT:     DW_CFA_offset: reg30 -8
-# UNWINDINFO-NEXT:     DW_CFA_offset: reg29 -16
+# ASM-LABEL: save_restore_sve_realign:
+# ASM:       .cfi_def_cfa w29, 16
+# ASM-NEXT:  .cfi_escape 0x10, 0x48, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d8  @ cfa - 16 - 8 * VG
+# ASM-NEXT:  .cfi_escape 0x10, 0x49, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x70, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d9  @ cfa - 16 - 16 * VG
+# ASM-NEXT:  .cfi_escape 0x10, 0x4a, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x68, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d10  @ cfa - 16 - 24 * VG
+# ASM-NEXT:  .cfi_escape 0x10, 0x4b, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x60, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d11  @ cfa - 16 - 32 * VG
+# ASM-NEXT:  .cfi_escape 0x10, 0x4c, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x58, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d12  @ cfa - 16 - 40 * VG
+# ASM-NEXT:  .cfi_escape 0x10, 0x4d, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x50, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d13  @ cfa - 16 - 48 * VG
+# ASM-NEXT:  .cfi_escape 0x10, 0x4e, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x48, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d14  @ cfa - 16 - 56 * VG
+# ASM-NEXT:  .cfi_escape 0x10, 0x4f, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x40, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d15  @ cfa - 16 - 64 * VG
+# ASM-NEXT:  .cfi_offset w30, -8
+# ASM-NEXT:  .cfi_offset w29, -16
+#
+# UNWINDINFO:      DW_CFA_def_cfa: reg29 +16
+# UNWINDINFO-NEXT: DW_CFA_expression: reg72 DW_OP_consts -16, DW_OP_plus, DW_OP_consts -8, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+# UNWINDINFO-NEXT: DW_CFA_expression: reg73 DW_OP_consts -16, DW_OP_plus, DW_OP_consts -16, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+# UNWINDINFO-NEXT: DW_CFA_expression: reg74 DW_OP_consts -16, DW_OP_plus, DW_OP_consts -24, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+# UNWINDINFO-NEXT: DW_CFA_expression: reg75 DW_OP_consts -16, DW_OP_plus, DW_OP_consts -32, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+# UNWINDINFO-NEXT: DW_CFA_expression: reg76 DW_OP_consts -16, DW_OP_plus, DW_OP_consts -40, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+# UNWINDINFO-NEXT: DW_CFA_expression: reg77 DW_OP_consts -16, DW_OP_plus, DW_OP_consts -48, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+# UNWINDINFO-NEXT: DW_CFA_expression: reg78 DW_OP_consts -16, DW_OP_plus, DW_OP_consts -56, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+# UNWINDINFO-NEXT: DW_CFA_expression: reg79 DW_OP_consts -16, DW_OP_plus, DW_OP_consts -64, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+# UNWINDINFO-NEXT: DW_CFA_offset: reg30 -8
+# UNWINDINFO-NEXT: DW_CFA_offset: reg29 -16
 name: save_restore_sve_realign
 stack:
   - { id: 0, stack-id: sve-vec, size: 16, alignment: 16 }
@@ -713,15 +750,15 @@ body:             |
 # CHECK-NEXT: STR_ZXI killed $z23, $sp, 1
 # CHECK-NEXT: STR_ZXI killed $z8, $sp, 2
 # CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -7
-# CHECK-COUNT-6: frame-setup CFI_INSTRUCTION
+# CHECK-COUNT-3: frame-setup CFI_INSTRUCTION
 # ASM-LABEL: frame_layout:
 # ASM:       .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0xd0, 0x00, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 80 * VG
-# ASM-COUNT-4: .cfi_offset
+# ASM-NEXT:  .cfi_escape 0x10, 0x48, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d8  @ cfa - 16 - 8 * VG
 # ASM-NEXT:  .cfi_offset w29, -16
 #
-# UNWINDINFO:         DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +80, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
-# UNWINDINFO-COUNT-4: DW_CFA_offset
-# UNWINDINFO-NEXT:    DW_CFA_offset: reg29 -16
+# UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +80, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+# UNWINDINFO-NEXT: DW_CFA_expression: reg72 DW_OP_consts -16, DW_OP_plus, DW_OP_consts -8, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+# UNWINDINFO-NEXT: DW_CFA_offset: reg29 -16
 name: frame_layout
 stack:
   - { id: 0, type: default,    size:  32, alignment: 16, stack-id: sve-vec }
diff --git a/llvm/test/CodeGen/AArch64/sve-trunc.ll b/llvm/test/CodeGen/AArch64/sve-trunc.ll
index 191df22eda506..7c0e9e9f4d9b3 100644
--- a/llvm/test/CodeGen/AArch64/sve-trunc.ll
+++ b/llvm/test/CodeGen/AArch64/sve-trunc.ll
@@ -117,7 +117,6 @@ define <vscale x 16 x i1> @trunc_i64toi1_split3(<vscale x 16 x i64> %in) {
 ; CHECK-NEXT:    addvl sp, sp, #-1
 ; CHECK-NEXT:    str p4, [sp, #7, mul vl] // 2-byte Folded Spill
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG
-; CHECK-NEXT:    .cfi_offset p4, -2
 ; CHECK-NEXT:    .cfi_offset w29, -16
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    and z7.d, z7.d, #0x1

From 998c0efee0e6d6909fdee579e6d2694babd99c0f Mon Sep 17 00:00:00 2001
From: Juneyoung Lee <aqjune@gmail.com>
Date: Tue, 4 Aug 2020 20:27:26 +0900
Subject: [PATCH 1024/1035] [JumpThreading] Update test freeze.ll; NFC

---
 llvm/test/Transforms/JumpThreading/freeze.ll | 16 +++++-----------
 1 file changed, 5 insertions(+), 11 deletions(-)

diff --git a/llvm/test/Transforms/JumpThreading/freeze.ll b/llvm/test/Transforms/JumpThreading/freeze.ll
index 99fa058ade814..650cbeb2205d5 100644
--- a/llvm/test/Transforms/JumpThreading/freeze.ll
+++ b/llvm/test/Transforms/JumpThreading/freeze.ll
@@ -42,20 +42,14 @@ F2:
 
 define i32 @test1_cast(i1 %cond) {
 ; CHECK-LABEL: @test1_cast(
-; CHECK-NEXT:    br i1 [[COND:%.*]], label [[MERGE_THREAD:%.*]], label [[MERGE:%.*]]
-; CHECK:       Merge.thread:
-; CHECK-NEXT:    [[V1:%.*]] = call i32 @f1()
-; CHECK-NEXT:    br label [[T2:%.*]]
-; CHECK:       Merge:
-; CHECK-NEXT:    [[V2:%.*]] = call i32 @f2()
-; CHECK-NEXT:    [[A:%.*]] = trunc i32 0 to i1
-; CHECK-NEXT:    [[A_FR:%.*]] = freeze i1 [[A]]
-; CHECK-NEXT:    br i1 [[A_FR]], label [[T2]], label [[F2:%.*]]
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[T2:%.*]], label [[F2:%.*]]
 ; CHECK:       T2:
-; CHECK-NEXT:    [[B5:%.*]] = phi i32 [ [[V1]], [[MERGE_THREAD]] ], [ [[V2]], [[MERGE]] ]
+; CHECK-NEXT:    [[V1:%.*]] = call i32 @f1()
 ; CHECK-NEXT:    call void @f3()
-; CHECK-NEXT:    ret i32 [[B5]]
+; CHECK-NEXT:    ret i32 [[V1]]
 ; CHECK:       F2:
+; CHECK-NEXT:    [[V2:%.*]] = call i32 @f2()
+; CHECK-NEXT:    [[A:%.*]] = trunc i32 0 to i1
 ; CHECK-NEXT:    ret i32 [[V2]]
 ;
   br i1 %cond, label %T1, label %F1

From f003b28363f9e09f3f68a5e08ebcca2635a4cdca Mon Sep 17 00:00:00 2001
From: George Mitenkov <georgemitenk0v@gmail.com>
Date: Tue, 4 Aug 2020 12:45:18 +0300
Subject: [PATCH 1025/1035] [MLIR][SPIRVToLLVM] Indentation and style fix in
 tests

This is a first patch that sweeps over tests to fix
indentation (tabs to spaces). It also adds label checks and
removes redundant matching of `%{{.*}} = `.

The following tests have been fixed:
- arithmetic-ops-to-llvm
- bitwise-ops-to-llvm
- cast-ops-to-llvm
- comparison-ops-to-llvm
- logical-ops-to-llvm (renamed to match the rest)

Reviewed By: ftynse

Differential Revision: https://reviews.llvm.org/D85181
---
 .../SPIRVToLLVM/arithmetic-ops-to-llvm.mlir   | 182 ++++----
 .../SPIRVToLLVM/bitwise-ops-to-llvm.mlir      | 388 +++++++++---------
 .../SPIRVToLLVM/cast-ops-to-llvm.mlir         | 164 ++++----
 .../SPIRVToLLVM/comparison-ops-to-llvm.mlir   | 304 ++++++++------
 ...-to-llvm.mlir => logical-ops-to-llvm.mlir} |  52 ++-
 5 files changed, 601 insertions(+), 489 deletions(-)
 rename mlir/test/Conversion/SPIRVToLLVM/{logical-to-llvm.mlir => logical-ops-to-llvm.mlir} (55%)

diff --git a/mlir/test/Conversion/SPIRVToLLVM/arithmetic-ops-to-llvm.mlir b/mlir/test/Conversion/SPIRVToLLVM/arithmetic-ops-to-llvm.mlir
index 2f5d18a6acf8f..8ae0d98407131 100644
--- a/mlir/test/Conversion/SPIRVToLLVM/arithmetic-ops-to-llvm.mlir
+++ b/mlir/test/Conversion/SPIRVToLLVM/arithmetic-ops-to-llvm.mlir
@@ -4,206 +4,232 @@
 // spv.IAdd
 //===----------------------------------------------------------------------===//
 
+// CHECK-LABEL: @iadd_scalar
 func @iadd_scalar(%arg0: i32, %arg1: i32) {
-	// CHECK: %{{.*}} = llvm.add %{{.*}}, %{{.*}} : !llvm.i32
-	%0 = spv.IAdd %arg0, %arg1 : i32
-	return
+  // CHECK: llvm.add %{{.*}}, %{{.*}} : !llvm.i32
+  %0 = spv.IAdd %arg0, %arg1 : i32
+  return
 }
 
+// CHECK-LABEL: @iadd_vector
 func @iadd_vector(%arg0: vector<4xi64>, %arg1: vector<4xi64>) {
-	// CHECK: %{{.*}} = llvm.add %{{.*}}, %{{.*}} : !llvm<"<4 x i64>">
-	%0 = spv.IAdd %arg0, %arg1 : vector<4xi64>
-	return
+  // CHECK: llvm.add %{{.*}}, %{{.*}} : !llvm<"<4 x i64>">
+  %0 = spv.IAdd %arg0, %arg1 : vector<4xi64>
+  return
 }
 
 //===----------------------------------------------------------------------===//
 // spv.ISub
 //===----------------------------------------------------------------------===//
 
+// CHECK-LABEL: @isub_scalar
 func @isub_scalar(%arg0: i8, %arg1: i8) {
-	// CHECK: %{{.*}} = llvm.sub %{{.*}}, %{{.*}} : !llvm.i8
-	%0 = spv.ISub %arg0, %arg1 : i8
-	return
+  // CHECK: llvm.sub %{{.*}}, %{{.*}} : !llvm.i8
+  %0 = spv.ISub %arg0, %arg1 : i8
+  return
 }
 
+// CHECK-LABEL: @isub_vector
 func @isub_vector(%arg0: vector<2xi16>, %arg1: vector<2xi16>) {
-	// CHECK: %{{.*}} = llvm.sub %{{.*}}, %{{.*}} : !llvm<"<2 x i16>">
-	%0 = spv.ISub %arg0, %arg1 : vector<2xi16>
-	return
+  // CHECK: llvm.sub %{{.*}}, %{{.*}} : !llvm<"<2 x i16>">
+  %0 = spv.ISub %arg0, %arg1 : vector<2xi16>
+  return
 }
 
 //===----------------------------------------------------------------------===//
 // spv.IMul
 //===----------------------------------------------------------------------===//
 
+// CHECK-LABEL: @imul_scalar
 func @imul_scalar(%arg0: i32, %arg1: i32) {
-	// CHECK: %{{.*}} = llvm.mul %{{.*}}, %{{.*}} : !llvm.i32
-	%0 = spv.IMul %arg0, %arg1 : i32
-	return
+  // CHECK: llvm.mul %{{.*}}, %{{.*}} : !llvm.i32
+  %0 = spv.IMul %arg0, %arg1 : i32
+  return
 }
 
+// CHECK-LABEL: @imul_vector
 func @imul_vector(%arg0: vector<3xi32>, %arg1: vector<3xi32>) {
-	// CHECK: %{{.*}} = llvm.mul %{{.*}}, %{{.*}} : !llvm<"<3 x i32>">
-	%0 = spv.IMul %arg0, %arg1 : vector<3xi32>
-	return
+  // CHECK: llvm.mul %{{.*}}, %{{.*}} : !llvm<"<3 x i32>">
+  %0 = spv.IMul %arg0, %arg1 : vector<3xi32>
+  return
 }
 
 //===----------------------------------------------------------------------===//
 // spv.FAdd
 //===----------------------------------------------------------------------===//
 
+// CHECK-LABEL: @fadd_scalar
 func @fadd_scalar(%arg0: f16, %arg1: f16) {
-	// CHECK: %{{.*}} = llvm.fadd %{{.*}}, %{{.*}} : !llvm.half
-	%0 = spv.FAdd %arg0, %arg1 : f16
-	return
+  // CHECK: llvm.fadd %{{.*}}, %{{.*}} : !llvm.half
+  %0 = spv.FAdd %arg0, %arg1 : f16
+  return
 }
 
+// CHECK-LABEL: @fadd_vector
 func @fadd_vector(%arg0: vector<4xf32>, %arg1: vector<4xf32>) {
-	// CHECK: %{{.*}} = llvm.fadd %{{.*}}, %{{.*}} : !llvm<"<4 x float>">
-	%0 = spv.FAdd %arg0, %arg1 : vector<4xf32>
-	return
+  // CHECK: llvm.fadd %{{.*}}, %{{.*}} : !llvm<"<4 x float>">
+  %0 = spv.FAdd %arg0, %arg1 : vector<4xf32>
+  return
 }
 
 //===----------------------------------------------------------------------===//
 // spv.FSub
 //===----------------------------------------------------------------------===//
 
+// CHECK-LABEL: @fsub_scalar
 func @fsub_scalar(%arg0: f32, %arg1: f32) {
-	// CHECK: %{{.*}} = llvm.fsub %{{.*}}, %{{.*}} : !llvm.float
-	%0 = spv.FSub %arg0, %arg1 : f32
-	return
+  // CHECK: llvm.fsub %{{.*}}, %{{.*}} : !llvm.float
+  %0 = spv.FSub %arg0, %arg1 : f32
+  return
 }
 
+// CHECK-LABEL: @fsub_vector
 func @fsub_vector(%arg0: vector<2xf32>, %arg1: vector<2xf32>) {
-	// CHECK: %{{.*}} = llvm.fsub %{{.*}}, %{{.*}} : !llvm<"<2 x float>">
-	%0 = spv.FSub %arg0, %arg1 : vector<2xf32>
-	return
+  // CHECK: llvm.fsub %{{.*}}, %{{.*}} : !llvm<"<2 x float>">
+  %0 = spv.FSub %arg0, %arg1 : vector<2xf32>
+  return
 }
 
 //===----------------------------------------------------------------------===//
 // spv.FDiv
 //===----------------------------------------------------------------------===//
 
+// CHECK-LABEL: @fdiv_scalar
 func @fdiv_scalar(%arg0: f32, %arg1: f32) {
-	// CHECK: %{{.*}} = llvm.fdiv %{{.*}}, %{{.*}} : !llvm.float
-	%0 = spv.FDiv %arg0, %arg1 : f32
-	return
+  // CHECK: llvm.fdiv %{{.*}}, %{{.*}} : !llvm.float
+  %0 = spv.FDiv %arg0, %arg1 : f32
+  return
 }
 
+// CHECK-LABEL: @fdiv_vector
 func @fdiv_vector(%arg0: vector<3xf64>, %arg1: vector<3xf64>) {
-	// CHECK: %{{.*}} = llvm.fdiv %{{.*}}, %{{.*}} : !llvm<"<3 x double>">
-	%0 = spv.FDiv %arg0, %arg1 : vector<3xf64>
-	return
+  // CHECK: llvm.fdiv %{{.*}}, %{{.*}} : !llvm<"<3 x double>">
+  %0 = spv.FDiv %arg0, %arg1 : vector<3xf64>
+  return
 }
 
 //===----------------------------------------------------------------------===//
 // spv.FMul
 //===----------------------------------------------------------------------===//
 
+// CHECK-LABEL: @fmul_scalar
 func @fmul_scalar(%arg0: f32, %arg1: f32) {
-	// CHECK: %{{.*}} = llvm.fmul %{{.*}}, %{{.*}} : !llvm.float
-	%0 = spv.FMul %arg0, %arg1 : f32
-	return
+  // CHECK: llvm.fmul %{{.*}}, %{{.*}} : !llvm.float
+  %0 = spv.FMul %arg0, %arg1 : f32
+  return
 }
 
+// CHECK-LABEL: @fmul_vector
 func @fmul_vector(%arg0: vector<2xf32>, %arg1: vector<2xf32>) {
-	// CHECK: %{{.*}} = llvm.fmul %{{.*}}, %{{.*}} : !llvm<"<2 x float>">
-	%0 = spv.FMul %arg0, %arg1 : vector<2xf32>
-	return
+  // CHECK: llvm.fmul %{{.*}}, %{{.*}} : !llvm<"<2 x float>">
+  %0 = spv.FMul %arg0, %arg1 : vector<2xf32>
+  return
 }
 
 //===----------------------------------------------------------------------===//
 // spv.FRem
 //===----------------------------------------------------------------------===//
 
+// CHECK-LABEL: @frem_scalar
 func @frem_scalar(%arg0: f32, %arg1: f32) {
-	// CHECK: %{{.*}} = llvm.frem %{{.*}}, %{{.*}} : !llvm.float
-	%0 = spv.FRem %arg0, %arg1 : f32
-	return
+  // CHECK: llvm.frem %{{.*}}, %{{.*}} : !llvm.float
+  %0 = spv.FRem %arg0, %arg1 : f32
+  return
 }
 
+// CHECK-LABEL: @frem_vector
 func @frem_vector(%arg0: vector<3xf64>, %arg1: vector<3xf64>) {
-	// CHECK: %{{.*}} = llvm.frem %{{.*}}, %{{.*}} : !llvm<"<3 x double>">
-	%0 = spv.FRem %arg0, %arg1 : vector<3xf64>
-	return
+  // CHECK: llvm.frem %{{.*}}, %{{.*}} : !llvm<"<3 x double>">
+  %0 = spv.FRem %arg0, %arg1 : vector<3xf64>
+  return
 }
 
 //===----------------------------------------------------------------------===//
 // spv.FNegate
 //===----------------------------------------------------------------------===//
 
+// CHECK-LABEL: @fneg_scalar
 func @fneg_scalar(%arg: f64) {
-	// CHECK: %{{.*}} = llvm.fneg %{{.*}} : !llvm.double
-	%0 = spv.FNegate %arg : f64
-	return
+  // CHECK: llvm.fneg %{{.*}} : !llvm.double
+  %0 = spv.FNegate %arg : f64
+  return
 }
 
+// CHECK-LABEL: @fneg_vector
 func @fneg_vector(%arg: vector<2xf32>) {
-	// CHECK: %{{.*}} = llvm.fneg %{{.*}} : !llvm<"<2 x float>">
-	%0 = spv.FNegate %arg : vector<2xf32>
-	return
+  // CHECK: llvm.fneg %{{.*}} : !llvm<"<2 x float>">
+  %0 = spv.FNegate %arg : vector<2xf32>
+  return
 }
 
 //===----------------------------------------------------------------------===//
 // spv.UDiv
 //===----------------------------------------------------------------------===//
 
+// CHECK-LABEL: @udiv_scalar
 func @udiv_scalar(%arg0: i32, %arg1: i32) {
-	// CHECK: %{{.*}} = llvm.udiv %{{.*}}, %{{.*}} : !llvm.i32
-	%0 = spv.UDiv %arg0, %arg1 : i32
-	return
+  // CHECK: llvm.udiv %{{.*}}, %{{.*}} : !llvm.i32
+  %0 = spv.UDiv %arg0, %arg1 : i32
+  return
 }
 
+// CHECK-LABEL: @udiv_vector
 func @udiv_vector(%arg0: vector<3xi64>, %arg1: vector<3xi64>) {
-	// CHECK: %{{.*}} = llvm.udiv %{{.*}}, %{{.*}} : !llvm<"<3 x i64>">
-	%0 = spv.UDiv %arg0, %arg1 : vector<3xi64>
-	return
+  // CHECK: llvm.udiv %{{.*}}, %{{.*}} : !llvm<"<3 x i64>">
+  %0 = spv.UDiv %arg0, %arg1 : vector<3xi64>
+  return
 }
 
 //===----------------------------------------------------------------------===//
 // spv.UMod
 //===----------------------------------------------------------------------===//
 
+// CHECK-LABEL: @umod_scalar
 func @umod_scalar(%arg0: i32, %arg1: i32) {
-	// CHECK: %{{.*}} = llvm.urem %{{.*}}, %{{.*}} : !llvm.i32
-	%0 = spv.UMod %arg0, %arg1 : i32
-	return
+  // CHECK: llvm.urem %{{.*}}, %{{.*}} : !llvm.i32
+  %0 = spv.UMod %arg0, %arg1 : i32
+  return
 }
 
+// CHECK-LABEL: @umod_vector
 func @umod_vector(%arg0: vector<3xi64>, %arg1: vector<3xi64>) {
-	// CHECK: %{{.*}} = llvm.urem %{{.*}}, %{{.*}} : !llvm<"<3 x i64>">
-	%0 = spv.UMod %arg0, %arg1 : vector<3xi64>
-	return
+  // CHECK: llvm.urem %{{.*}}, %{{.*}} : !llvm<"<3 x i64>">
+  %0 = spv.UMod %arg0, %arg1 : vector<3xi64>
+  return
 }
 
 //===----------------------------------------------------------------------===//
 // spv.SDiv
 //===----------------------------------------------------------------------===//
 
+// CHECK-LABEL: @sdiv_scalar
 func @sdiv_scalar(%arg0: i16, %arg1: i16) {
-	// CHECK: %{{.*}} = llvm.sdiv %{{.*}}, %{{.*}} : !llvm.i16
-	%0 = spv.SDiv %arg0, %arg1 : i16
-	return
+  // CHECK: llvm.sdiv %{{.*}}, %{{.*}} : !llvm.i16
+  %0 = spv.SDiv %arg0, %arg1 : i16
+  return
 }
 
+// CHECK-LABEL: @sdiv_vector
 func @sdiv_vector(%arg0: vector<2xi64>, %arg1: vector<2xi64>) {
-	// CHECK: %{{.*}} = llvm.sdiv %{{.*}}, %{{.*}} : !llvm<"<2 x i64>">
-	%0 = spv.SDiv %arg0, %arg1 : vector<2xi64>
-	return
+  // CHECK: llvm.sdiv %{{.*}}, %{{.*}} : !llvm<"<2 x i64>">
+  %0 = spv.SDiv %arg0, %arg1 : vector<2xi64>
+  return
 }
 
 //===----------------------------------------------------------------------===//
 // spv.SRem
 //===----------------------------------------------------------------------===//
 
+// CHECK-LABEL: @srem_scalar
 func @srem_scalar(%arg0: i32, %arg1: i32) {
-	// CHECK: %{{.*}} = llvm.srem %{{.*}}, %{{.*}} : !llvm.i32
-	%0 = spv.SRem %arg0, %arg1 : i32
-	return
+  // CHECK: llvm.srem %{{.*}}, %{{.*}} : !llvm.i32
+  %0 = spv.SRem %arg0, %arg1 : i32
+  return
 }
 
+// CHECK-LABEL: @srem_vector
 func @srem_vector(%arg0: vector<4xi32>, %arg1: vector<4xi32>) {
-	// CHECK: %{{.*}} = llvm.srem %{{.*}}, %{{.*}} : !llvm<"<4 x i32>">
-	%0 = spv.SRem %arg0, %arg1 : vector<4xi32>
-	return
+  // CHECK: llvm.srem %{{.*}}, %{{.*}} : !llvm<"<4 x i32>">
+  %0 = spv.SRem %arg0, %arg1 : vector<4xi32>
+  return
 }
diff --git a/mlir/test/Conversion/SPIRVToLLVM/bitwise-ops-to-llvm.mlir b/mlir/test/Conversion/SPIRVToLLVM/bitwise-ops-to-llvm.mlir
index 31ffc6dbf7dc9..aed82d218db21 100644
--- a/mlir/test/Conversion/SPIRVToLLVM/bitwise-ops-to-llvm.mlir
+++ b/mlir/test/Conversion/SPIRVToLLVM/bitwise-ops-to-llvm.mlir
@@ -4,309 +4,321 @@
 // spv.BitCount
 //===----------------------------------------------------------------------===//
 
+// CHECK-LABEL: @bitcount_scalar
 func @bitcount_scalar(%arg0: i16) {
-	// CHECK: %{{.*}} = "llvm.intr.ctpop"(%{{.*}}) : (!llvm.i16) -> !llvm.i16
-	%0 = spv.BitCount %arg0: i16
-	return
+  // CHECK: "llvm.intr.ctpop"(%{{.*}}) : (!llvm.i16) -> !llvm.i16
+  %0 = spv.BitCount %arg0: i16
+  return
 }
 
+// CHECK-LABEL: @bitcount_vector
 func @bitcount_vector(%arg0: vector<3xi32>) {
-	// CHECK: %{{.*}} = "llvm.intr.ctpop"(%{{.*}}) : (!llvm<"<3 x i32>">) -> !llvm<"<3 x i32>">
-	%0 = spv.BitCount %arg0: vector<3xi32>
-	return
+  // CHECK: "llvm.intr.ctpop"(%{{.*}}) : (!llvm<"<3 x i32>">) -> !llvm<"<3 x i32>">
+  %0 = spv.BitCount %arg0: vector<3xi32>
+  return
 }
 
 //===----------------------------------------------------------------------===//
 // spv.BitReverse
 //===----------------------------------------------------------------------===//
 
+// CHECK-LABEL: @bitreverse_scalar
 func @bitreverse_scalar(%arg0: i64) {
-	// CHECK: %{{.*}} = "llvm.intr.bitreverse"(%{{.*}}) : (!llvm.i64) -> !llvm.i64
-	%0 = spv.BitReverse %arg0: i64
-	return
+  // CHECK: "llvm.intr.bitreverse"(%{{.*}}) : (!llvm.i64) -> !llvm.i64
+  %0 = spv.BitReverse %arg0: i64
+  return
 }
 
+// CHECK-LABEL: @bitreverse_vector
 func @bitreverse_vector(%arg0: vector<4xi32>) {
-	// CHECK: %{{.*}} = "llvm.intr.bitreverse"(%{{.*}}) : (!llvm<"<4 x i32>">) -> !llvm<"<4 x i32>">
-	%0 = spv.BitReverse %arg0: vector<4xi32>
-	return
+  // CHECK: "llvm.intr.bitreverse"(%{{.*}}) : (!llvm<"<4 x i32>">) -> !llvm<"<4 x i32>">
+  %0 = spv.BitReverse %arg0: vector<4xi32>
+  return
 }
 
 //===----------------------------------------------------------------------===//
 // spv.BitFieldInsert
 //===----------------------------------------------------------------------===//
 
-// CHECK-LABEL: func @bitfield_insert_scalar_same_bit_width
+// CHECK-LABEL: @bitfield_insert_scalar_same_bit_width
 // CHECK-SAME: %[[BASE:.*]]: !llvm.i32, %[[INSERT:.*]]: !llvm.i32, %[[OFFSET:.*]]: !llvm.i32, %[[COUNT:.*]]: !llvm.i32
 func @bitfield_insert_scalar_same_bit_width(%base: i32, %insert: i32, %offset: i32, %count: i32) {
-    // CHECK: %[[MINUS_ONE:.*]] = llvm.mlir.constant(-1 : i32) : !llvm.i32
-    // CHECK: %[[T0:.*]] = llvm.shl %[[MINUS_ONE]], %[[COUNT]] : !llvm.i32
-    // CHECK: %[[T1:.*]] = llvm.xor %[[T0]], %[[MINUS_ONE]] : !llvm.i32
-    // CHECK: %[[T2:.*]] = llvm.shl %[[T1]], %[[OFFSET]] : !llvm.i32
-    // CHECK: %[[MASK:.*]] = llvm.xor %[[T2]], %[[MINUS_ONE]] : !llvm.i32
-    // CHECK: %[[NEW_BASE:.*]] = llvm.and %[[BASE]], %[[MASK]] : !llvm.i32
-    // CHECK: %[[SHIFTED_INSERT:.*]] = llvm.shl %[[INSERT]], %[[OFFSET]] : !llvm.i32
-    // CHECK: %{{.*}} = llvm.or %[[NEW_BASE]], %[[SHIFTED_INSERT]] : !llvm.i32
-    %0 = spv.BitFieldInsert %base, %insert, %offset, %count : i32, i32, i32
-    return
+  // CHECK: %[[MINUS_ONE:.*]] = llvm.mlir.constant(-1 : i32) : !llvm.i32
+  // CHECK: %[[T0:.*]] = llvm.shl %[[MINUS_ONE]], %[[COUNT]] : !llvm.i32
+  // CHECK: %[[T1:.*]] = llvm.xor %[[T0]], %[[MINUS_ONE]] : !llvm.i32
+  // CHECK: %[[T2:.*]] = llvm.shl %[[T1]], %[[OFFSET]] : !llvm.i32
+  // CHECK: %[[MASK:.*]] = llvm.xor %[[T2]], %[[MINUS_ONE]] : !llvm.i32
+  // CHECK: %[[NEW_BASE:.*]] = llvm.and %[[BASE]], %[[MASK]] : !llvm.i32
+  // CHECK: %[[SHIFTED_INSERT:.*]] = llvm.shl %[[INSERT]], %[[OFFSET]] : !llvm.i32
+  // CHECK: llvm.or %[[NEW_BASE]], %[[SHIFTED_INSERT]] : !llvm.i32
+  %0 = spv.BitFieldInsert %base, %insert, %offset, %count : i32, i32, i32
+  return
 }
 
-// CHECK-LABEL: func @bitfield_insert_scalar_smaller_bit_width
+// CHECK-LABEL: @bitfield_insert_scalar_smaller_bit_width
 // CHECK-SAME: %[[BASE:.*]]: !llvm.i64, %[[INSERT:.*]]: !llvm.i64, %[[OFFSET:.*]]: !llvm.i8, %[[COUNT:.*]]: !llvm.i8
 func @bitfield_insert_scalar_smaller_bit_width(%base: i64, %insert: i64, %offset: i8, %count: i8) {
-    // CHECK: %[[EXT_OFFSET:.*]] = llvm.zext %[[OFFSET]] : !llvm.i8 to !llvm.i64
-    // CHECK: %[[EXT_COUNT:.*]] = llvm.zext %[[COUNT]] : !llvm.i8 to !llvm.i64
-    // CHECK: %[[MINUS_ONE:.*]] = llvm.mlir.constant(-1 : i64) : !llvm.i64
-    // CHECK: %[[T0:.*]] = llvm.shl %[[MINUS_ONE]], %[[EXT_COUNT]] : !llvm.i64
-    // CHECK: %[[T1:.*]] = llvm.xor %[[T0]], %[[MINUS_ONE]] : !llvm.i64
-    // CHECK: %[[T2:.*]] = llvm.shl %[[T1]], %[[EXT_OFFSET]] : !llvm.i64
-    // CHECK: %[[MASK:.*]] = llvm.xor %[[T2]], %[[MINUS_ONE]] : !llvm.i64
-    // CHECK: %[[NEW_BASE:.*]] = llvm.and %[[BASE]], %[[MASK]] : !llvm.i64
-    // CHECK: %[[SHIFTED_INSERT:.*]] = llvm.shl %[[INSERT]], %[[EXT_OFFSET]] : !llvm.i64
-    // CHECK: %{{.*}} = llvm.or %[[NEW_BASE]], %[[SHIFTED_INSERT]] : !llvm.i64
-    %0 = spv.BitFieldInsert %base, %insert, %offset, %count : i64, i8, i8
-    return
+  // CHECK: %[[EXT_OFFSET:.*]] = llvm.zext %[[OFFSET]] : !llvm.i8 to !llvm.i64
+  // CHECK: %[[EXT_COUNT:.*]] = llvm.zext %[[COUNT]] : !llvm.i8 to !llvm.i64
+  // CHECK: %[[MINUS_ONE:.*]] = llvm.mlir.constant(-1 : i64) : !llvm.i64
+  // CHECK: %[[T0:.*]] = llvm.shl %[[MINUS_ONE]], %[[EXT_COUNT]] : !llvm.i64
+  // CHECK: %[[T1:.*]] = llvm.xor %[[T0]], %[[MINUS_ONE]] : !llvm.i64
+  // CHECK: %[[T2:.*]] = llvm.shl %[[T1]], %[[EXT_OFFSET]] : !llvm.i64
+  // CHECK: %[[MASK:.*]] = llvm.xor %[[T2]], %[[MINUS_ONE]] : !llvm.i64
+  // CHECK: %[[NEW_BASE:.*]] = llvm.and %[[BASE]], %[[MASK]] : !llvm.i64
+  // CHECK: %[[SHIFTED_INSERT:.*]] = llvm.shl %[[INSERT]], %[[EXT_OFFSET]] : !llvm.i64
+  // CHECK: llvm.or %[[NEW_BASE]], %[[SHIFTED_INSERT]] : !llvm.i64
+  %0 = spv.BitFieldInsert %base, %insert, %offset, %count : i64, i8, i8
+  return
 }
 
-// CHECK-LABEL: func @bitfield_insert_scalar_greater_bit_width
+// CHECK-LABEL: @bitfield_insert_scalar_greater_bit_width
 // CHECK-SAME: %[[BASE:.*]]: !llvm.i16, %[[INSERT:.*]]: !llvm.i16, %[[OFFSET:.*]]: !llvm.i32, %[[COUNT:.*]]: !llvm.i64
 func @bitfield_insert_scalar_greater_bit_width(%base: i16, %insert: i16, %offset: i32, %count: i64) {
-    // CHECK: %[[TRUNC_OFFSET:.*]] = llvm.trunc %[[OFFSET]] : !llvm.i32 to !llvm.i16
-    // CHECK: %[[TRUNC_COUNT:.*]] = llvm.trunc %[[COUNT]] : !llvm.i64 to !llvm.i16
-    // CHECK: %[[MINUS_ONE:.*]] = llvm.mlir.constant(-1 : i16) : !llvm.i16
-    // CHECK: %[[T0:.*]] = llvm.shl %[[MINUS_ONE]], %[[TRUNC_COUNT]] : !llvm.i16
-    // CHECK: %[[T1:.*]] = llvm.xor %[[T0]], %[[MINUS_ONE]] : !llvm.i16
-    // CHECK: %[[T2:.*]] = llvm.shl %[[T1]], %[[TRUNC_OFFSET]] : !llvm.i16
-    // CHECK: %[[MASK:.*]] = llvm.xor %[[T2]], %[[MINUS_ONE]] : !llvm.i16
-    // CHECK: %[[NEW_BASE:.*]] = llvm.and %[[BASE]], %[[MASK]] : !llvm.i16
-    // CHECK: %[[SHIFTED_INSERT:.*]] = llvm.shl %[[INSERT]], %[[TRUNC_OFFSET]] : !llvm.i16
-    // CHECK: %{{.*}} = llvm.or %[[NEW_BASE]], %[[SHIFTED_INSERT]] : !llvm.i16
-    %0 = spv.BitFieldInsert %base, %insert, %offset, %count : i16, i32, i64
-    return
+  // CHECK: %[[TRUNC_OFFSET:.*]] = llvm.trunc %[[OFFSET]] : !llvm.i32 to !llvm.i16
+  // CHECK: %[[TRUNC_COUNT:.*]] = llvm.trunc %[[COUNT]] : !llvm.i64 to !llvm.i16
+  // CHECK: %[[MINUS_ONE:.*]] = llvm.mlir.constant(-1 : i16) : !llvm.i16
+  // CHECK: %[[T0:.*]] = llvm.shl %[[MINUS_ONE]], %[[TRUNC_COUNT]] : !llvm.i16
+  // CHECK: %[[T1:.*]] = llvm.xor %[[T0]], %[[MINUS_ONE]] : !llvm.i16
+  // CHECK: %[[T2:.*]] = llvm.shl %[[T1]], %[[TRUNC_OFFSET]] : !llvm.i16
+  // CHECK: %[[MASK:.*]] = llvm.xor %[[T2]], %[[MINUS_ONE]] : !llvm.i16
+  // CHECK: %[[NEW_BASE:.*]] = llvm.and %[[BASE]], %[[MASK]] : !llvm.i16
+  // CHECK: %[[SHIFTED_INSERT:.*]] = llvm.shl %[[INSERT]], %[[TRUNC_OFFSET]] : !llvm.i16
+  // CHECK: llvm.or %[[NEW_BASE]], %[[SHIFTED_INSERT]] : !llvm.i16
+  %0 = spv.BitFieldInsert %base, %insert, %offset, %count : i16, i32, i64
+  return
 }
 
-// CHECK-LABEL: func @bitfield_insert_vector
+// CHECK-LABEL: @bitfield_insert_vector
 // CHECK-SAME: %[[BASE:.*]]: !llvm<"<2 x i32>">, %[[INSERT:.*]]: !llvm<"<2 x i32>">, %[[OFFSET:.*]]: !llvm.i32, %[[COUNT:.*]]: !llvm.i32
 func @bitfield_insert_vector(%base: vector<2xi32>, %insert: vector<2xi32>, %offset: i32, %count: i32) {
-    // CHECK: %[[OFFSET_V0:.*]] = llvm.mlir.undef : !llvm<"<2 x i32>">
-    // CHECK: %[[ZERO:.*]] = llvm.mlir.constant(0 : i32) : !llvm.i32
-    // CHECK: %[[OFFSET_V1:.*]] = llvm.insertelement %[[OFFSET]], %[[OFFSET_V0]][%[[ZERO]] : !llvm.i32] : !llvm<"<2 x i32>">
-    // CHECK: %[[ONE:.*]] = llvm.mlir.constant(1 : i32) : !llvm.i32
-    // CHECK: %[[OFFSET_V2:.*]] = llvm.insertelement  %[[OFFSET]], %[[OFFSET_V1]][%[[ONE]] : !llvm.i32] : !llvm<"<2 x i32>">
-    // CHECK: %[[COUNT_V0:.*]] = llvm.mlir.undef : !llvm<"<2 x i32>">
-    // CHECK: %[[ZERO:.*]] = llvm.mlir.constant(0 : i32) : !llvm.i32
-    // CHECK: %[[COUNT_V1:.*]] = llvm.insertelement %[[COUNT]], %[[COUNT_V0]][%[[ZERO]] : !llvm.i32] : !llvm<"<2 x i32>">
-    // CHECK: %[[ONE:.*]] = llvm.mlir.constant(1 : i32) : !llvm.i32
-    // CHECK: %[[COUNT_V2:.*]] = llvm.insertelement %[[COUNT]], %[[COUNT_V1]][%[[ONE]] : !llvm.i32] : !llvm<"<2 x i32>">
-    // CHECK: %[[MINUS_ONE:.*]] = llvm.mlir.constant(dense<-1> : vector<2xi32>) : !llvm<"<2 x i32>">
-    // CHECK: %[[T0:.*]] = llvm.shl %[[MINUS_ONE]], %[[COUNT_V2]] : !llvm<"<2 x i32>">
-    // CHECK: %[[T1:.*]] = llvm.xor %[[T0]], %[[MINUS_ONE]] : !llvm<"<2 x i32>">
-    // CHECK: %[[T2:.*]] = llvm.shl %[[T1]], %[[OFFSET_V2]] : !llvm<"<2 x i32>">
-    // CHECK: %[[MASK:.*]] = llvm.xor %[[T2]], %[[MINUS_ONE]] : !llvm<"<2 x i32>">
-    // CHECK: %[[NEW_BASE:.*]] = llvm.and %[[BASE]], %[[MASK]] : !llvm<"<2 x i32>">
-    // CHECK: %[[SHIFTED_INSERT:.*]] = llvm.shl %[[INSERT]], %[[OFFSET_V2]] : !llvm<"<2 x i32>">
-    // CHECK: %{{.*}} = llvm.or %[[NEW_BASE]], %[[SHIFTED_INSERT]] : !llvm<"<2 x i32>">
-    %0 = spv.BitFieldInsert %base, %insert, %offset, %count : vector<2xi32>, i32, i32
-    return
+  // CHECK: %[[OFFSET_V0:.*]] = llvm.mlir.undef : !llvm<"<2 x i32>">
+  // CHECK: %[[ZERO:.*]] = llvm.mlir.constant(0 : i32) : !llvm.i32
+  // CHECK: %[[OFFSET_V1:.*]] = llvm.insertelement %[[OFFSET]], %[[OFFSET_V0]][%[[ZERO]] : !llvm.i32] : !llvm<"<2 x i32>">
+  // CHECK: %[[ONE:.*]] = llvm.mlir.constant(1 : i32) : !llvm.i32
+  // CHECK: %[[OFFSET_V2:.*]] = llvm.insertelement  %[[OFFSET]], %[[OFFSET_V1]][%[[ONE]] : !llvm.i32] : !llvm<"<2 x i32>">
+  // CHECK: %[[COUNT_V0:.*]] = llvm.mlir.undef : !llvm<"<2 x i32>">
+  // CHECK: %[[ZERO:.*]] = llvm.mlir.constant(0 : i32) : !llvm.i32
+  // CHECK: %[[COUNT_V1:.*]] = llvm.insertelement %[[COUNT]], %[[COUNT_V0]][%[[ZERO]] : !llvm.i32] : !llvm<"<2 x i32>">
+  // CHECK: %[[ONE:.*]] = llvm.mlir.constant(1 : i32) : !llvm.i32
+  // CHECK: %[[COUNT_V2:.*]] = llvm.insertelement %[[COUNT]], %[[COUNT_V1]][%[[ONE]] : !llvm.i32] : !llvm<"<2 x i32>">
+  // CHECK: %[[MINUS_ONE:.*]] = llvm.mlir.constant(dense<-1> : vector<2xi32>) : !llvm<"<2 x i32>">
+  // CHECK: %[[T0:.*]] = llvm.shl %[[MINUS_ONE]], %[[COUNT_V2]] : !llvm<"<2 x i32>">
+  // CHECK: %[[T1:.*]] = llvm.xor %[[T0]], %[[MINUS_ONE]] : !llvm<"<2 x i32>">
+  // CHECK: %[[T2:.*]] = llvm.shl %[[T1]], %[[OFFSET_V2]] : !llvm<"<2 x i32>">
+  // CHECK: %[[MASK:.*]] = llvm.xor %[[T2]], %[[MINUS_ONE]] : !llvm<"<2 x i32>">
+  // CHECK: %[[NEW_BASE:.*]] = llvm.and %[[BASE]], %[[MASK]] : !llvm<"<2 x i32>">
+  // CHECK: %[[SHIFTED_INSERT:.*]] = llvm.shl %[[INSERT]], %[[OFFSET_V2]] : !llvm<"<2 x i32>">
+  // CHECK: llvm.or %[[NEW_BASE]], %[[SHIFTED_INSERT]] : !llvm<"<2 x i32>">
+  %0 = spv.BitFieldInsert %base, %insert, %offset, %count : vector<2xi32>, i32, i32
+  return
 }
 
 //===----------------------------------------------------------------------===//
 // spv.BitFieldSExtract
 //===----------------------------------------------------------------------===//
 
-// CHECK-LABEL: func @bitfield_sextract_scalar_same_bit_width
+// CHECK-LABEL: @bitfield_sextract_scalar_same_bit_width
 // CHECK-SAME: %[[BASE:.*]]: !llvm.i64, %[[OFFSET:.*]]: !llvm.i64, %[[COUNT:.*]]: !llvm.i64
 func @bitfield_sextract_scalar_same_bit_width(%base: i64, %offset: i64, %count: i64) {
-    // CHECK: %[[SIZE:.]] = llvm.mlir.constant(64 : i64) : !llvm.i64
-    // CHECK: %[[T0:.*]] = llvm.add %[[COUNT]], %[[OFFSET]] : !llvm.i64
-    // CHECK: %[[T1:.*]] = llvm.sub %[[SIZE]], %[[T0]] : !llvm.i64
-    // CHECK: %[[SHIFTED_LEFT:.*]] = llvm.shl %[[BASE]], %[[T1]] : !llvm.i64
-    // CHECK: %[[T2:.*]] = llvm.add %[[OFFSET]], %[[T1]] : !llvm.i64
-    // CHECK: %{{.*}} = llvm.ashr %[[SHIFTED_LEFT]], %[[T2]] : !llvm.i64
-    %0 = spv.BitFieldSExtract %base, %offset, %count : i64, i64, i64
-    return
+  // CHECK: %[[SIZE:.]] = llvm.mlir.constant(64 : i64) : !llvm.i64
+  // CHECK: %[[T0:.*]] = llvm.add %[[COUNT]], %[[OFFSET]] : !llvm.i64
+  // CHECK: %[[T1:.*]] = llvm.sub %[[SIZE]], %[[T0]] : !llvm.i64
+  // CHECK: %[[SHIFTED_LEFT:.*]] = llvm.shl %[[BASE]], %[[T1]] : !llvm.i64
+  // CHECK: %[[T2:.*]] = llvm.add %[[OFFSET]], %[[T1]] : !llvm.i64
+  // CHECK: llvm.ashr %[[SHIFTED_LEFT]], %[[T2]] : !llvm.i64
+  %0 = spv.BitFieldSExtract %base, %offset, %count : i64, i64, i64
+  return
 }
 
-// CHECK-LABEL: func @bitfield_sextract_scalar_smaller_bit_width
+// CHECK-LABEL: @bitfield_sextract_scalar_smaller_bit_width
 // CHECK-SAME: %[[BASE:.*]]: !llvm.i32, %[[OFFSET:.*]]: !llvm.i8, %[[COUNT:.*]]: !llvm.i8
 func @bitfield_sextract_scalar_smaller_bit_width(%base: i32, %offset: i8, %count: i8) {
-    // CHECK: %[[EXT_OFFSET:.*]] = llvm.zext %[[OFFSET]] : !llvm.i8 to !llvm.i32
-    // CHECK: %[[EXT_COUNT:.*]] = llvm.zext %[[COUNT]] : !llvm.i8 to !llvm.i32
-    // CHECK: %[[SIZE:.]] = llvm.mlir.constant(32 : i32) : !llvm.i32
-    // CHECK: %[[T0:.*]] = llvm.add %[[EXT_COUNT]], %[[EXT_OFFSET]] : !llvm.i32
-    // CHECK: %[[T1:.*]] = llvm.sub %[[SIZE]], %[[T0]] : !llvm.i32
-    // CHECK: %[[SHIFTED_LEFT:.*]] = llvm.shl %[[BASE]], %[[T1]] : !llvm.i32
-    // CHECK: %[[T2:.*]] = llvm.add %[[EXT_OFFSET]], %[[T1]] : !llvm.i32
-    // CHECK: %{{.*}} = llvm.ashr %[[SHIFTED_LEFT]], %[[T2]] : !llvm.i32
-    %0 = spv.BitFieldSExtract %base, %offset, %count : i32, i8, i8
-    return
+  // CHECK: %[[EXT_OFFSET:.*]] = llvm.zext %[[OFFSET]] : !llvm.i8 to !llvm.i32
+  // CHECK: %[[EXT_COUNT:.*]] = llvm.zext %[[COUNT]] : !llvm.i8 to !llvm.i32
+  // CHECK: %[[SIZE:.]] = llvm.mlir.constant(32 : i32) : !llvm.i32
+  // CHECK: %[[T0:.*]] = llvm.add %[[EXT_COUNT]], %[[EXT_OFFSET]] : !llvm.i32
+  // CHECK: %[[T1:.*]] = llvm.sub %[[SIZE]], %[[T0]] : !llvm.i32
+  // CHECK: %[[SHIFTED_LEFT:.*]] = llvm.shl %[[BASE]], %[[T1]] : !llvm.i32
+  // CHECK: %[[T2:.*]] = llvm.add %[[EXT_OFFSET]], %[[T1]] : !llvm.i32
+  // CHECK: llvm.ashr %[[SHIFTED_LEFT]], %[[T2]] : !llvm.i32
+  %0 = spv.BitFieldSExtract %base, %offset, %count : i32, i8, i8
+  return
 }
 
-// CHECK-LABEL: func @bitfield_sextract_scalar_greater_bit_width
+// CHECK-LABEL: @bitfield_sextract_scalar_greater_bit_width
 // CHECK-SAME: %[[BASE:.*]]: !llvm.i32, %[[OFFSET:.*]]: !llvm.i64, %[[COUNT:.*]]: !llvm.i64
 func @bitfield_sextract_scalar_greater_bit_width(%base: i32, %offset: i64, %count: i64) {
-    // CHECK: %[[TRUNC_OFFSET:.*]] = llvm.trunc %[[OFFSET]] : !llvm.i64 to !llvm.i32
-    // CHECK: %[[TRUNC_COUNT:.*]] = llvm.trunc %[[COUNT]] : !llvm.i64 to !llvm.i32
-    // CHECK: %[[SIZE:.]] = llvm.mlir.constant(32 : i32) : !llvm.i32
-    // CHECK: %[[T0:.*]] = llvm.add %[[TRUNC_COUNT]], %[[TRUNC_OFFSET]] : !llvm.i32
-    // CHECK: %[[T1:.*]] = llvm.sub %[[SIZE]], %[[T0]] : !llvm.i32
-    // CHECK: %[[SHIFTED_LEFT:.*]] = llvm.shl %[[BASE]], %[[T1]] : !llvm.i32
-    // CHECK: %[[T2:.*]] = llvm.add %[[TRUNC_OFFSET]], %[[T1]] : !llvm.i32
-    // CHECK: %{{.*}} = llvm.ashr %[[SHIFTED_LEFT]], %[[T2]] : !llvm.i32
-    %0 = spv.BitFieldSExtract %base, %offset, %count : i32, i64, i64
-    return
+  // CHECK: %[[TRUNC_OFFSET:.*]] = llvm.trunc %[[OFFSET]] : !llvm.i64 to !llvm.i32
+  // CHECK: %[[TRUNC_COUNT:.*]] = llvm.trunc %[[COUNT]] : !llvm.i64 to !llvm.i32
+  // CHECK: %[[SIZE:.]] = llvm.mlir.constant(32 : i32) : !llvm.i32
+  // CHECK: %[[T0:.*]] = llvm.add %[[TRUNC_COUNT]], %[[TRUNC_OFFSET]] : !llvm.i32
+  // CHECK: %[[T1:.*]] = llvm.sub %[[SIZE]], %[[T0]] : !llvm.i32
+  // CHECK: %[[SHIFTED_LEFT:.*]] = llvm.shl %[[BASE]], %[[T1]] : !llvm.i32
+  // CHECK: %[[T2:.*]] = llvm.add %[[TRUNC_OFFSET]], %[[T1]] : !llvm.i32
+  // CHECK: llvm.ashr %[[SHIFTED_LEFT]], %[[T2]] : !llvm.i32
+  %0 = spv.BitFieldSExtract %base, %offset, %count : i32, i64, i64
+  return
 }
 
-// CHECK-LABEL: func @bitfield_sextract_vector
+// CHECK-LABEL: @bitfield_sextract_vector
 // CHECK-SAME: %[[BASE:.*]]: !llvm<"<2 x i32>">, %[[OFFSET:.*]]: !llvm.i32, %[[COUNT:.*]]: !llvm.i32
 func @bitfield_sextract_vector(%base: vector<2xi32>, %offset: i32, %count: i32) {
-    // CHECK: %[[OFFSET_V0:.*]] = llvm.mlir.undef : !llvm<"<2 x i32>">
-    // CHECK: %[[ZERO:.*]] = llvm.mlir.constant(0 : i32) : !llvm.i32
-    // CHECK: %[[OFFSET_V1:.*]] = llvm.insertelement %[[OFFSET]], %[[OFFSET_V0]][%[[ZERO]] : !llvm.i32] : !llvm<"<2 x i32>">
-    // CHECK: %[[ONE:.*]] = llvm.mlir.constant(1 : i32) : !llvm.i32
-    // CHECK: %[[OFFSET_V2:.*]] = llvm.insertelement  %[[OFFSET]], %[[OFFSET_V1]][%[[ONE]] : !llvm.i32] : !llvm<"<2 x i32>">
-    // CHECK: %[[COUNT_V0:.*]] = llvm.mlir.undef : !llvm<"<2 x i32>">
-    // CHECK: %[[ZERO:.*]] = llvm.mlir.constant(0 : i32) : !llvm.i32
-    // CHECK: %[[COUNT_V1:.*]] = llvm.insertelement %[[COUNT]], %[[COUNT_V0]][%[[ZERO]] : !llvm.i32] : !llvm<"<2 x i32>">
-    // CHECK: %[[ONE:.*]] = llvm.mlir.constant(1 : i32) : !llvm.i32
-    // CHECK: %[[COUNT_V2:.*]] = llvm.insertelement %[[COUNT]], %[[COUNT_V1]][%[[ONE]] : !llvm.i32] : !llvm<"<2 x i32>">
-    // CHECK: %[[SIZE:.*]] = llvm.mlir.constant(dense<32> : vector<2xi32>) : !llvm<"<2 x i32>">
-    // CHECK: %[[T0:.*]] = llvm.add %[[COUNT_V2]], %[[OFFSET_V2]] : !llvm<"<2 x i32>">
-    // CHECK: %[[T1:.*]] = llvm.sub %[[SIZE]], %[[T0]] : !llvm<"<2 x i32>">
-    // CHECK: %[[SHIFTED_LEFT:.*]] = llvm.shl %[[BASE]], %[[T1]] : !llvm<"<2 x i32>">
-    // CHECK: %[[T2:.*]] = llvm.add %[[OFFSET_V2]], %[[T1]] : !llvm<"<2 x i32>">
-    // CHECK: %{{.*}} = llvm.ashr %[[SHIFTED_LEFT]], %[[T2]] : !llvm<"<2 x i32>">
-    %0 = spv.BitFieldSExtract %base, %offset, %count : vector<2xi32>, i32, i32
-    return
+  // CHECK: %[[OFFSET_V0:.*]] = llvm.mlir.undef : !llvm<"<2 x i32>">
+  // CHECK: %[[ZERO:.*]] = llvm.mlir.constant(0 : i32) : !llvm.i32
+  // CHECK: %[[OFFSET_V1:.*]] = llvm.insertelement %[[OFFSET]], %[[OFFSET_V0]][%[[ZERO]] : !llvm.i32] : !llvm<"<2 x i32>">
+  // CHECK: %[[ONE:.*]] = llvm.mlir.constant(1 : i32) : !llvm.i32
+  // CHECK: %[[OFFSET_V2:.*]] = llvm.insertelement  %[[OFFSET]], %[[OFFSET_V1]][%[[ONE]] : !llvm.i32] : !llvm<"<2 x i32>">
+  // CHECK: %[[COUNT_V0:.*]] = llvm.mlir.undef : !llvm<"<2 x i32>">
+  // CHECK: %[[ZERO:.*]] = llvm.mlir.constant(0 : i32) : !llvm.i32
+  // CHECK: %[[COUNT_V1:.*]] = llvm.insertelement %[[COUNT]], %[[COUNT_V0]][%[[ZERO]] : !llvm.i32] : !llvm<"<2 x i32>">
+  // CHECK: %[[ONE:.*]] = llvm.mlir.constant(1 : i32) : !llvm.i32
+  // CHECK: %[[COUNT_V2:.*]] = llvm.insertelement %[[COUNT]], %[[COUNT_V1]][%[[ONE]] : !llvm.i32] : !llvm<"<2 x i32>">
+  // CHECK: %[[SIZE:.*]] = llvm.mlir.constant(dense<32> : vector<2xi32>) : !llvm<"<2 x i32>">
+  // CHECK: %[[T0:.*]] = llvm.add %[[COUNT_V2]], %[[OFFSET_V2]] : !llvm<"<2 x i32>">
+  // CHECK: %[[T1:.*]] = llvm.sub %[[SIZE]], %[[T0]] : !llvm<"<2 x i32>">
+  // CHECK: %[[SHIFTED_LEFT:.*]] = llvm.shl %[[BASE]], %[[T1]] : !llvm<"<2 x i32>">
+  // CHECK: %[[T2:.*]] = llvm.add %[[OFFSET_V2]], %[[T1]] : !llvm<"<2 x i32>">
+  // CHECK: llvm.ashr %[[SHIFTED_LEFT]], %[[T2]] : !llvm<"<2 x i32>">
+  %0 = spv.BitFieldSExtract %base, %offset, %count : vector<2xi32>, i32, i32
+  return
 }
 
 //===----------------------------------------------------------------------===//
 // spv.BitFieldUExtract
 //===----------------------------------------------------------------------===//
 
-// CHECK-LABEL: func @bitfield_uextract_scalar_same_bit_width
+// CHECK-LABEL: @bitfield_uextract_scalar_same_bit_width
 // CHECK-SAME: %[[BASE:.*]]: !llvm.i32, %[[OFFSET:.*]]: !llvm.i32, %[[COUNT:.*]]: !llvm.i32
 func @bitfield_uextract_scalar_same_bit_width(%base: i32, %offset: i32, %count: i32) {
-    // CHECK: %[[MINUS_ONE:.*]] = llvm.mlir.constant(-1 : i32) : !llvm.i32
-    // CHECK: %[[T0:.*]] = llvm.shl %[[MINUS_ONE]], %[[COUNT]] : !llvm.i32
-    // CHECK: %[[MASK:.*]] = llvm.xor %[[T0]], %[[MINUS_ONE]] : !llvm.i32
-    // CHECK: %[[SHIFTED_BASE:.*]] = llvm.lshr %[[BASE]], %[[OFFSET]] : !llvm.i32
-    // CHECK: %{{.*}} = llvm.and %[[SHIFTED_BASE]], %[[MASK]] : !llvm.i32
-    %0 = spv.BitFieldUExtract %base, %offset, %count : i32, i32, i32
-    return
+  // CHECK: %[[MINUS_ONE:.*]] = llvm.mlir.constant(-1 : i32) : !llvm.i32
+  // CHECK: %[[T0:.*]] = llvm.shl %[[MINUS_ONE]], %[[COUNT]] : !llvm.i32
+  // CHECK: %[[MASK:.*]] = llvm.xor %[[T0]], %[[MINUS_ONE]] : !llvm.i32
+  // CHECK: %[[SHIFTED_BASE:.*]] = llvm.lshr %[[BASE]], %[[OFFSET]] : !llvm.i32
+  // CHECK: llvm.and %[[SHIFTED_BASE]], %[[MASK]] : !llvm.i32
+  %0 = spv.BitFieldUExtract %base, %offset, %count : i32, i32, i32
+  return
 }
 
-// CHECK-LABEL: func @bitfield_uextract_scalar_smaller_bit_width
+// CHECK-LABEL: @bitfield_uextract_scalar_smaller_bit_width
 // CHECK-SAME: %[[BASE:.*]]: !llvm.i32, %[[OFFSET:.*]]: !llvm.i16, %[[COUNT:.*]]: !llvm.i8
 func @bitfield_uextract_scalar_smaller_bit_width(%base: i32, %offset: i16, %count: i8) {
-    // CHECK: %[[EXT_OFFSET:.*]] = llvm.zext %[[OFFSET]] : !llvm.i16 to !llvm.i32
-    // CHECK: %[[EXT_COUNT:.*]] = llvm.zext %[[COUNT]] : !llvm.i8 to !llvm.i32
-    // CHECK: %[[MINUS_ONE:.*]] = llvm.mlir.constant(-1 : i32) : !llvm.i32
-    // CHECK: %[[T0:.*]] = llvm.shl %[[MINUS_ONE]], %[[EXT_COUNT]] : !llvm.i32
-    // CHECK: %[[MASK:.*]] = llvm.xor %[[T0]], %[[MINUS_ONE]] : !llvm.i32
-    // CHECK: %[[SHIFTED_BASE:.*]] = llvm.lshr %[[BASE]], %[[EXT_OFFSET]] : !llvm.i32
-    // CHECK: %{{.*}} = llvm.and %[[SHIFTED_BASE]], %[[MASK]] : !llvm.i32
-    %0 = spv.BitFieldUExtract %base, %offset, %count : i32, i16, i8
-    return
+  // CHECK: %[[EXT_OFFSET:.*]] = llvm.zext %[[OFFSET]] : !llvm.i16 to !llvm.i32
+  // CHECK: %[[EXT_COUNT:.*]] = llvm.zext %[[COUNT]] : !llvm.i8 to !llvm.i32
+  // CHECK: %[[MINUS_ONE:.*]] = llvm.mlir.constant(-1 : i32) : !llvm.i32
+  // CHECK: %[[T0:.*]] = llvm.shl %[[MINUS_ONE]], %[[EXT_COUNT]] : !llvm.i32
+  // CHECK: %[[MASK:.*]] = llvm.xor %[[T0]], %[[MINUS_ONE]] : !llvm.i32
+  // CHECK: %[[SHIFTED_BASE:.*]] = llvm.lshr %[[BASE]], %[[EXT_OFFSET]] : !llvm.i32
+  // CHECK: llvm.and %[[SHIFTED_BASE]], %[[MASK]] : !llvm.i32
+  %0 = spv.BitFieldUExtract %base, %offset, %count : i32, i16, i8
+  return
 }
 
-// CHECK-LABEL: func @bitfield_uextract_scalar_greater_bit_width
+// CHECK-LABEL: @bitfield_uextract_scalar_greater_bit_width
 // CHECK-SAME: %[[BASE:.*]]: !llvm.i8, %[[OFFSET:.*]]: !llvm.i16, %[[COUNT:.*]]: !llvm.i8
 func @bitfield_uextract_scalar_greater_bit_width(%base: i8, %offset: i16, %count: i8) {
-    // CHECK: %[[TRUNC_OFFSET:.*]] = llvm.trunc %[[OFFSET]] : !llvm.i16 to !llvm.i8
-    // CHECK: %[[MINUS_ONE:.*]] = llvm.mlir.constant(-1 : i8) : !llvm.i8
-    // CHECK: %[[T0:.*]] = llvm.shl %[[MINUS_ONE]], %[[COUNT]] : !llvm.i8
-    // CHECK: %[[MASK:.*]] = llvm.xor %[[T0]], %[[MINUS_ONE]] : !llvm.i8
-    // CHECK: %[[SHIFTED_BASE:.*]] = llvm.lshr %[[BASE]], %[[TRUNC_OFFSET]] : !llvm.i8
-    // CHECK: %{{.*}} = llvm.and %[[SHIFTED_BASE]], %[[MASK]] : !llvm.i8
-    %0 = spv.BitFieldUExtract %base, %offset, %count : i8, i16, i8
-    return
+  // CHECK: %[[TRUNC_OFFSET:.*]] = llvm.trunc %[[OFFSET]] : !llvm.i16 to !llvm.i8
+  // CHECK: %[[MINUS_ONE:.*]] = llvm.mlir.constant(-1 : i8) : !llvm.i8
+  // CHECK: %[[T0:.*]] = llvm.shl %[[MINUS_ONE]], %[[COUNT]] : !llvm.i8
+  // CHECK: %[[MASK:.*]] = llvm.xor %[[T0]], %[[MINUS_ONE]] : !llvm.i8
+  // CHECK: %[[SHIFTED_BASE:.*]] = llvm.lshr %[[BASE]], %[[TRUNC_OFFSET]] : !llvm.i8
+  // CHECK: llvm.and %[[SHIFTED_BASE]], %[[MASK]] : !llvm.i8
+  %0 = spv.BitFieldUExtract %base, %offset, %count : i8, i16, i8
+  return
 }
 
-// CHECK-LABEL: func @bitfield_uextract_vector
+// CHECK-LABEL: @bitfield_uextract_vector
 // CHECK-SAME: %[[BASE:.*]]: !llvm<"<2 x i32>">, %[[OFFSET:.*]]: !llvm.i32, %[[COUNT:.*]]: !llvm.i32
 func @bitfield_uextract_vector(%base: vector<2xi32>, %offset: i32, %count: i32) {
-    // CHECK: %[[OFFSET_V0:.*]] = llvm.mlir.undef : !llvm<"<2 x i32>">
-    // CHECK: %[[ZERO:.*]] = llvm.mlir.constant(0 : i32) : !llvm.i32
-    // CHECK: %[[OFFSET_V1:.*]] = llvm.insertelement %[[OFFSET]], %[[OFFSET_V0]][%[[ZERO]] : !llvm.i32] : !llvm<"<2 x i32>">
-    // CHECK: %[[ONE:.*]] = llvm.mlir.constant(1 : i32) : !llvm.i32
-    // CHECK: %[[OFFSET_V2:.*]] = llvm.insertelement  %[[OFFSET]], %[[OFFSET_V1]][%[[ONE]] : !llvm.i32] : !llvm<"<2 x i32>">
-    // CHECK: %[[COUNT_V0:.*]] = llvm.mlir.undef : !llvm<"<2 x i32>">
-    // CHECK: %[[ZERO:.*]] = llvm.mlir.constant(0 : i32) : !llvm.i32
-    // CHECK: %[[COUNT_V1:.*]] = llvm.insertelement %[[COUNT]], %[[COUNT_V0]][%[[ZERO]] : !llvm.i32] : !llvm<"<2 x i32>">
-    // CHECK: %[[ONE:.*]] = llvm.mlir.constant(1 : i32) : !llvm.i32
-    // CHECK: %[[COUNT_V2:.*]] = llvm.insertelement %[[COUNT]], %[[COUNT_V1]][%[[ONE]] : !llvm.i32] : !llvm<"<2 x i32>">
-    // CHECK: %[[MINUS_ONE:.*]] = llvm.mlir.constant(dense<-1> : vector<2xi32>) : !llvm<"<2 x i32>">
-    // CHECK: %[[T0:.*]] = llvm.shl %[[MINUS_ONE]], %[[COUNT_V2]] : !llvm<"<2 x i32>">
-    // CHECK: %[[MASK:.*]] = llvm.xor %[[T0]], %[[MINUS_ONE]] : !llvm<"<2 x i32>">
-    // CHECK: %[[SHIFTED_BASE:.*]] = llvm.lshr %[[BASE]], %[[OFFSET_V2]] : !llvm<"<2 x i32>">
-    // CHECK: %{{.*}} = llvm.and %[[SHIFTED_BASE]], %[[MASK]] : !llvm<"<2 x i32>">
-    %0 = spv.BitFieldUExtract %base, %offset, %count : vector<2xi32>, i32, i32
-    return
+  // CHECK: %[[OFFSET_V0:.*]] = llvm.mlir.undef : !llvm<"<2 x i32>">
+  // CHECK: %[[ZERO:.*]] = llvm.mlir.constant(0 : i32) : !llvm.i32
+  // CHECK: %[[OFFSET_V1:.*]] = llvm.insertelement %[[OFFSET]], %[[OFFSET_V0]][%[[ZERO]] : !llvm.i32] : !llvm<"<2 x i32>">
+  // CHECK: %[[ONE:.*]] = llvm.mlir.constant(1 : i32) : !llvm.i32
+  // CHECK: %[[OFFSET_V2:.*]] = llvm.insertelement  %[[OFFSET]], %[[OFFSET_V1]][%[[ONE]] : !llvm.i32] : !llvm<"<2 x i32>">
+  // CHECK: %[[COUNT_V0:.*]] = llvm.mlir.undef : !llvm<"<2 x i32>">
+  // CHECK: %[[ZERO:.*]] = llvm.mlir.constant(0 : i32) : !llvm.i32
+  // CHECK: %[[COUNT_V1:.*]] = llvm.insertelement %[[COUNT]], %[[COUNT_V0]][%[[ZERO]] : !llvm.i32] : !llvm<"<2 x i32>">
+  // CHECK: %[[ONE:.*]] = llvm.mlir.constant(1 : i32) : !llvm.i32
+  // CHECK: %[[COUNT_V2:.*]] = llvm.insertelement %[[COUNT]], %[[COUNT_V1]][%[[ONE]] : !llvm.i32] : !llvm<"<2 x i32>">
+  // CHECK: %[[MINUS_ONE:.*]] = llvm.mlir.constant(dense<-1> : vector<2xi32>) : !llvm<"<2 x i32>">
+  // CHECK: %[[T0:.*]] = llvm.shl %[[MINUS_ONE]], %[[COUNT_V2]] : !llvm<"<2 x i32>">
+  // CHECK: %[[MASK:.*]] = llvm.xor %[[T0]], %[[MINUS_ONE]] : !llvm<"<2 x i32>">
+  // CHECK: %[[SHIFTED_BASE:.*]] = llvm.lshr %[[BASE]], %[[OFFSET_V2]] : !llvm<"<2 x i32>">
+  // CHECK: llvm.and %[[SHIFTED_BASE]], %[[MASK]] : !llvm<"<2 x i32>">
+  %0 = spv.BitFieldUExtract %base, %offset, %count : vector<2xi32>, i32, i32
+  return
 }
 
 //===----------------------------------------------------------------------===//
 // spv.BitwiseAnd
 //===----------------------------------------------------------------------===//
 
+// CHECK-LABEL: @bitwise_and_scalar
 func @bitwise_and_scalar(%arg0: i32, %arg1: i32) {
-	// CHECK: %{{.*}} = llvm.and %{{.*}}, %{{.*}} : !llvm.i32
-	%0 = spv.BitwiseAnd %arg0, %arg1 : i32
-	return
+  // CHECK: llvm.and %{{.*}}, %{{.*}} : !llvm.i32
+  %0 = spv.BitwiseAnd %arg0, %arg1 : i32
+  return
 }
 
+// CHECK-LABEL: @bitwise_and_vector
 func @bitwise_and_vector(%arg0: vector<4xi64>, %arg1: vector<4xi64>) {
-	// CHECK: %{{.*}} = llvm.and %{{.*}}, %{{.*}} : !llvm<"<4 x i64>">
-	%0 = spv.BitwiseAnd %arg0, %arg1 : vector<4xi64>
-	return
+  // CHECK: llvm.and %{{.*}}, %{{.*}} : !llvm<"<4 x i64>">
+  %0 = spv.BitwiseAnd %arg0, %arg1 : vector<4xi64>
+  return
 }
 
 //===----------------------------------------------------------------------===//
 // spv.BitwiseOr
 //===----------------------------------------------------------------------===//
 
+// CHECK-LABEL: @bitwise_or_scalar
 func @bitwise_or_scalar(%arg0: i64, %arg1: i64) {
-	// CHECK: %{{.*}} = llvm.or %{{.*}}, %{{.*}} : !llvm.i64
-	%0 = spv.BitwiseOr %arg0, %arg1 : i64
-	return
+  // CHECK: llvm.or %{{.*}}, %{{.*}} : !llvm.i64
+  %0 = spv.BitwiseOr %arg0, %arg1 : i64
+  return
 }
 
+// CHECK-LABEL: @bitwise_or_vector
 func @bitwise_or_vector(%arg0: vector<3xi8>, %arg1: vector<3xi8>) {
-	// CHECK: %{{.*}} = llvm.or %{{.*}}, %{{.*}} : !llvm<"<3 x i8>">
-	%0 = spv.BitwiseOr %arg0, %arg1 : vector<3xi8>
-	return
+  // CHECK: llvm.or %{{.*}}, %{{.*}} : !llvm<"<3 x i8>">
+  %0 = spv.BitwiseOr %arg0, %arg1 : vector<3xi8>
+  return
 }
 
 //===----------------------------------------------------------------------===//
 // spv.BitwiseXor
 //===----------------------------------------------------------------------===//
 
+// CHECK-LABEL: @bitwise_xor_scalar
 func @bitwise_xor_scalar(%arg0: i32, %arg1: i32) {
-	// CHECK: %{{.*}} = llvm.xor %{{.*}}, %{{.*}} : !llvm.i32
-	%0 = spv.BitwiseXor %arg0, %arg1 : i32
-	return
+  // CHECK: llvm.xor %{{.*}}, %{{.*}} : !llvm.i32
+  %0 = spv.BitwiseXor %arg0, %arg1 : i32
+  return
 }
 
+// CHECK-LABEL: @bitwise_xor_vector
 func @bitwise_xor_vector(%arg0: vector<2xi16>, %arg1: vector<2xi16>) {
-	// CHECK: %{{.*}} = llvm.xor %{{.*}}, %{{.*}} : !llvm<"<2 x i16>">
-	%0 = spv.BitwiseXor %arg0, %arg1 : vector<2xi16>
-	return
+  // CHECK: llvm.xor %{{.*}}, %{{.*}} : !llvm<"<2 x i16>">
+  %0 = spv.BitwiseXor %arg0, %arg1 : vector<2xi16>
+  return
 }
 
 //===----------------------------------------------------------------------===//
 // spv.Not
 //===----------------------------------------------------------------------===//
 
-func @not__scalar(%arg0: i32) {
+// CHECK-LABEL: @not_scalar
+func @not_scalar(%arg0: i32) {
   // CHECK: %[[CONST:.*]] = llvm.mlir.constant(-1 : i32) : !llvm.i32
-  // CHECK: %{{.*}} = llvm.xor %{{.*}}, %[[CONST]] : !llvm.i32
-	%0 = spv.Not %arg0 : i32
+  // CHECK: llvm.xor %{{.*}}, %[[CONST]] : !llvm.i32
+  %0 = spv.Not %arg0 : i32
   return
 }
 
+// CHECK-LABEL: @not_vector
 func @not_vector(%arg0: vector<2xi16>) {
   // CHECK: %[[CONST:.*]] = llvm.mlir.constant(dense<-1> : vector<2xi16>) : !llvm<"<2 x i16>">
-  // CHECK: %{{.*}} = llvm.xor %{{.*}}, %[[CONST]] : !llvm<"<2 x i16>">
-	%0 = spv.Not %arg0 : vector<2xi16>
+  // CHECK: llvm.xor %{{.*}}, %[[CONST]] : !llvm<"<2 x i16>">
+  %0 = spv.Not %arg0 : vector<2xi16>
   return
 }
diff --git a/mlir/test/Conversion/SPIRVToLLVM/cast-ops-to-llvm.mlir b/mlir/test/Conversion/SPIRVToLLVM/cast-ops-to-llvm.mlir
index 29f43fadf933c..a0434aad3bea7 100644
--- a/mlir/test/Conversion/SPIRVToLLVM/cast-ops-to-llvm.mlir
+++ b/mlir/test/Conversion/SPIRVToLLVM/cast-ops-to-llvm.mlir
@@ -4,168 +4,188 @@
 // spv.Bitcast
 //===----------------------------------------------------------------------===//
 
+// CHECK-LABEL: @bitcast_float_to_integer_scalar
 func @bitcast_float_to_integer_scalar(%arg0 : f32) {
-	// CHECK: {{.*}} = llvm.bitcast {{.*}} : !llvm.float to !llvm.i32
-	%0 = spv.Bitcast %arg0: f32 to i32
-	return
+  // CHECK: llvm.bitcast {{.*}} : !llvm.float to !llvm.i32
+  %0 = spv.Bitcast %arg0: f32 to i32
+  return
 }
 
+// CHECK-LABEL: @bitcast_float_to_integer_vector
 func @bitcast_float_to_integer_vector(%arg0 : vector<3xf32>) {
-	// CHECK: {{.*}} = llvm.bitcast {{.*}} : !llvm<"<3 x float>"> to !llvm<"<3 x i32>">
-	%0 = spv.Bitcast %arg0: vector<3xf32> to vector<3xi32>
-	return
+  // CHECK: llvm.bitcast {{.*}} : !llvm<"<3 x float>"> to !llvm<"<3 x i32>">
+  %0 = spv.Bitcast %arg0: vector<3xf32> to vector<3xi32>
+  return
 }
 
+// CHECK-LABEL: @bitcast_vector_to_scalar
 func @bitcast_vector_to_scalar(%arg0 : vector<2xf32>) {
-	// CHECK: {{.*}} = llvm.bitcast {{.*}} : !llvm<"<2 x float>"> to !llvm.i64
-	%0 = spv.Bitcast %arg0: vector<2xf32> to i64
-	return
+  // CHECK: llvm.bitcast {{.*}} : !llvm<"<2 x float>"> to !llvm.i64
+  %0 = spv.Bitcast %arg0: vector<2xf32> to i64
+  return
 }
 
+// CHECK-LABEL: @bitcast_scalar_to_vector
 func @bitcast_scalar_to_vector(%arg0 : f64) {
-	// CHECK: {{.*}} = llvm.bitcast {{.*}} : !llvm.double to !llvm<"<2 x i32>">
-	%0 = spv.Bitcast %arg0: f64 to vector<2xi32>
-	return
+  // CHECK: llvm.bitcast {{.*}} : !llvm.double to !llvm<"<2 x i32>">
+  %0 = spv.Bitcast %arg0: f64 to vector<2xi32>
+  return
 }
 
+// CHECK-LABEL: @bitcast_vector_to_vector
 func @bitcast_vector_to_vector(%arg0 : vector<4xf32>) {
-	// CHECK: {{.*}} = llvm.bitcast {{.*}} : !llvm<"<4 x float>"> to !llvm<"<2 x i64>">
-	%0 = spv.Bitcast %arg0: vector<4xf32> to vector<2xi64>
-	return
+  // CHECK: llvm.bitcast {{.*}} : !llvm<"<4 x float>"> to !llvm<"<2 x i64>">
+  %0 = spv.Bitcast %arg0: vector<4xf32> to vector<2xi64>
+  return
 }
 
+// CHECK-LABEL: @bitcast_pointer
 func @bitcast_pointer(%arg0: !spv.ptr<f32, Function>) {
-	// CHECK: %{{.*}} = llvm.bitcast %{{.*}} : !llvm<"float*"> to !llvm<"i32*">
-	%0 = spv.Bitcast %arg0 : !spv.ptr<f32, Function> to !spv.ptr<i32, Function>
-	return
+  // CHECK: llvm.bitcast %{{.*}} : !llvm<"float*"> to !llvm<"i32*">
+  %0 = spv.Bitcast %arg0 : !spv.ptr<f32, Function> to !spv.ptr<i32, Function>
+  return
 }
 
 //===----------------------------------------------------------------------===//
 // spv.ConvertFToS
 //===----------------------------------------------------------------------===//
 
+// CHECK-LABEL: @convert_float_to_signed_scalar
 func @convert_float_to_signed_scalar(%arg0: f32) {
-	// CHECK: %{{.*}} = llvm.fptosi %{{.*}} : !llvm.float to !llvm.i32
-    %0 = spv.ConvertFToS %arg0: f32 to i32
-	return
+  // CHECK: llvm.fptosi %{{.*}} : !llvm.float to !llvm.i32
+  %0 = spv.ConvertFToS %arg0: f32 to i32
+  return
 }
 
+// CHECK-LABEL: @convert_float_to_signed_vector
 func @convert_float_to_signed_vector(%arg0: vector<2xf32>) {
-	// CHECK: %{{.*}} = llvm.fptosi %{{.*}} : !llvm<"<2 x float>"> to !llvm<"<2 x i32>">
-    %0 = spv.ConvertFToS %arg0: vector<2xf32> to vector<2xi32>
-	return
+  // CHECK: llvm.fptosi %{{.*}} : !llvm<"<2 x float>"> to !llvm<"<2 x i32>">
+  %0 = spv.ConvertFToS %arg0: vector<2xf32> to vector<2xi32>
+  return
 }
 
 //===----------------------------------------------------------------------===//
 // spv.ConvertFToU
 //===----------------------------------------------------------------------===//
 
+// CHECK-LABEL: @convert_float_to_unsigned_scalar
 func @convert_float_to_unsigned_scalar(%arg0: f32) {
-	// CHECK: %{{.*}} = llvm.fptoui %{{.*}} : !llvm.float to !llvm.i32
-    %0 = spv.ConvertFToU %arg0: f32 to i32
-	return
+  // CHECK: llvm.fptoui %{{.*}} : !llvm.float to !llvm.i32
+  %0 = spv.ConvertFToU %arg0: f32 to i32
+  return
 }
 
+// CHECK-LABEL: @convert_float_to_unsigned_vector
 func @convert_float_to_unsigned_vector(%arg0: vector<2xf32>) {
-	// CHECK: %{{.*}} = llvm.fptoui %{{.*}} : !llvm<"<2 x float>"> to !llvm<"<2 x i32>">
-    %0 = spv.ConvertFToU %arg0: vector<2xf32> to vector<2xi32>
-	return
+  // CHECK: llvm.fptoui %{{.*}} : !llvm<"<2 x float>"> to !llvm<"<2 x i32>">
+  %0 = spv.ConvertFToU %arg0: vector<2xf32> to vector<2xi32>
+  return
 }
 
 //===----------------------------------------------------------------------===//
 // spv.ConvertSToF
 //===----------------------------------------------------------------------===//
 
+// CHECK-LABEL: @convert_signed_to_float_scalar
 func @convert_signed_to_float_scalar(%arg0: i32) {
-	// CHECK: %{{.*}} = llvm.sitofp %{{.*}} : !llvm.i32 to !llvm.float
-    %0 = spv.ConvertSToF %arg0: i32 to f32
-	return
+  // CHECK: llvm.sitofp %{{.*}} : !llvm.i32 to !llvm.float
+  %0 = spv.ConvertSToF %arg0: i32 to f32
+  return
 }
 
+// CHECK-LABEL: @convert_signed_to_float_vector
 func @convert_signed_to_float_vector(%arg0: vector<3xi32>) {
-	// CHECK: %{{.*}} = llvm.sitofp %{{.*}} : !llvm<"<3 x i32>"> to !llvm<"<3 x float>">
-    %0 = spv.ConvertSToF %arg0: vector<3xi32> to vector<3xf32>
-	return
+  // CHECK: llvm.sitofp %{{.*}} : !llvm<"<3 x i32>"> to !llvm<"<3 x float>">
+  %0 = spv.ConvertSToF %arg0: vector<3xi32> to vector<3xf32>
+  return
 }
 
 //===----------------------------------------------------------------------===//
 // spv.ConvertUToF
 //===----------------------------------------------------------------------===//
 
+// CHECK-LABEL: @convert_unsigned_to_float_scalar
 func @convert_unsigned_to_float_scalar(%arg0: i32) {
-	// CHECK: %{{.*}} = llvm.uitofp %{{.*}} : !llvm.i32 to !llvm.float
-    %0 = spv.ConvertUToF %arg0: i32 to f32
-	return
+  // CHECK: llvm.uitofp %{{.*}} : !llvm.i32 to !llvm.float
+  %0 = spv.ConvertUToF %arg0: i32 to f32
+  return
 }
 
+// CHECK-LABEL: @convert_unsigned_to_float_vector
 func @convert_unsigned_to_float_vector(%arg0: vector<3xi32>) {
-	// CHECK: %{{.*}} = llvm.uitofp %{{.*}} : !llvm<"<3 x i32>"> to !llvm<"<3 x float>">
-    %0 = spv.ConvertUToF %arg0: vector<3xi32> to vector<3xf32>
-	return
+  // CHECK: llvm.uitofp %{{.*}} : !llvm<"<3 x i32>"> to !llvm<"<3 x float>">
+  %0 = spv.ConvertUToF %arg0: vector<3xi32> to vector<3xf32>
+  return
 }
 
 //===----------------------------------------------------------------------===//
 // spv.FConvert
 //===----------------------------------------------------------------------===//
 
+// CHECK-LABEL: @fconvert_scalar
 func @fconvert_scalar(%arg0: f32, %arg1: f64) {
-	// CHECK: %{{.*}} = llvm.fpext %{{.*}} : !llvm.float to !llvm.double
-    %0 = spv.FConvert %arg0: f32 to f64
+  // CHECK: llvm.fpext %{{.*}} : !llvm.float to !llvm.double
+  %0 = spv.FConvert %arg0: f32 to f64
 
-    // CHECK: %{{.*}} = llvm.fptrunc %{{.*}} : !llvm.double to !llvm.float
-    %1 = spv.FConvert %arg1: f64 to f32
-	return
+  // CHECK: llvm.fptrunc %{{.*}} : !llvm.double to !llvm.float
+  %1 = spv.FConvert %arg1: f64 to f32
+  return
 }
 
+// CHECK-LABEL: @fconvert_vector
 func @fconvert_vector(%arg0: vector<2xf32>, %arg1: vector<2xf64>) {
-	// CHECK: %{{.*}} = llvm.fpext %{{.*}} : !llvm<"<2 x float>"> to !llvm<"<2 x double>">
-    %0 = spv.FConvert %arg0: vector<2xf32> to vector<2xf64>
+  // CHECK: llvm.fpext %{{.*}} : !llvm<"<2 x float>"> to !llvm<"<2 x double>">
+  %0 = spv.FConvert %arg0: vector<2xf32> to vector<2xf64>
 
-    // CHECK: %{{.*}} = llvm.fptrunc %{{.*}} : !llvm<"<2 x double>"> to !llvm<"<2 x float>">
-    %1 = spv.FConvert %arg1: vector<2xf64> to vector<2xf32>
-	return
+  // CHECK: llvm.fptrunc %{{.*}} : !llvm<"<2 x double>"> to !llvm<"<2 x float>">
+  %1 = spv.FConvert %arg1: vector<2xf64> to vector<2xf32>
+  return
 }
 
 //===----------------------------------------------------------------------===//
 // spv.SConvert
 //===----------------------------------------------------------------------===//
 
+// CHECK-LABEL: @sconvert_scalar
 func @sconvert_scalar(%arg0: i32, %arg1: i64) {
-	// CHECK: %{{.*}} = llvm.sext %{{.*}} : !llvm.i32 to !llvm.i64
-    %0 = spv.SConvert %arg0: i32 to i64
+  // CHECK: llvm.sext %{{.*}} : !llvm.i32 to !llvm.i64
+  %0 = spv.SConvert %arg0: i32 to i64
 
-    // CHECK: %{{.*}} = llvm.trunc %{{.*}} : !llvm.i64 to !llvm.i32
-    %1 = spv.SConvert %arg1: i64 to i32
-	return
+  // CHECK: llvm.trunc %{{.*}} : !llvm.i64 to !llvm.i32
+  %1 = spv.SConvert %arg1: i64 to i32
+  return
 }
 
+// CHECK-LABEL: @sconvert_vector
 func @sconvert_vector(%arg0: vector<3xi32>, %arg1: vector<3xi64>) {
-	// CHECK: %{{.*}} = llvm.sext %{{.*}} : !llvm<"<3 x i32>"> to !llvm<"<3 x i64>">
-    %0 = spv.SConvert %arg0: vector<3xi32> to vector<3xi64>
+  // CHECK: llvm.sext %{{.*}} : !llvm<"<3 x i32>"> to !llvm<"<3 x i64>">
+  %0 = spv.SConvert %arg0: vector<3xi32> to vector<3xi64>
 
-    // CHECK: %{{.*}} = llvm.trunc %{{.*}} : !llvm<"<3 x i64>"> to !llvm<"<3 x i32>">
-    %1 = spv.SConvert %arg1: vector<3xi64> to vector<3xi32>
-	return
+  // CHECK: llvm.trunc %{{.*}} : !llvm<"<3 x i64>"> to !llvm<"<3 x i32>">
+  %1 = spv.SConvert %arg1: vector<3xi64> to vector<3xi32>
+  return
 }
 
 //===----------------------------------------------------------------------===//
 // spv.UConvert
 //===----------------------------------------------------------------------===//
 
+// CHECK-LABEL: @uconvert_scalar
 func @uconvert_scalar(%arg0: i32, %arg1: i64) {
-	// CHECK: %{{.*}} = llvm.zext %{{.*}} : !llvm.i32 to !llvm.i64
-    %0 = spv.UConvert %arg0: i32 to i64
+  // CHECK: llvm.zext %{{.*}} : !llvm.i32 to !llvm.i64
+  %0 = spv.UConvert %arg0: i32 to i64
 
-    // CHECK: %{{.*}} = llvm.trunc %{{.*}} : !llvm.i64 to !llvm.i32
-    %1 = spv.UConvert %arg1: i64 to i32
-	return
+  // CHECK: llvm.trunc %{{.*}} : !llvm.i64 to !llvm.i32
+  %1 = spv.UConvert %arg1: i64 to i32
+  return
 }
 
+// CHECK-LABEL: @uconvert_vector
 func @uconvert_vector(%arg0: vector<3xi32>, %arg1: vector<3xi64>) {
-	// CHECK: %{{.*}} = llvm.zext %{{.*}} : !llvm<"<3 x i32>"> to !llvm<"<3 x i64>">
-    %0 = spv.UConvert %arg0: vector<3xi32> to vector<3xi64>
+  // CHECK: llvm.zext %{{.*}} : !llvm<"<3 x i32>"> to !llvm<"<3 x i64>">
+  %0 = spv.UConvert %arg0: vector<3xi32> to vector<3xi64>
 
-    // CHECK: %{{.*}} = llvm.trunc %{{.*}} : !llvm<"<3 x i64>"> to !llvm<"<3 x i32>">
-    %1 = spv.UConvert %arg1: vector<3xi64> to vector<3xi32>
-	return
+  // CHECK: llvm.trunc %{{.*}} : !llvm<"<3 x i64>"> to !llvm<"<3 x i32>">
+  %1 = spv.UConvert %arg1: vector<3xi64> to vector<3xi32>
+  return
 }
diff --git a/mlir/test/Conversion/SPIRVToLLVM/comparison-ops-to-llvm.mlir b/mlir/test/Conversion/SPIRVToLLVM/comparison-ops-to-llvm.mlir
index c4241e226a1ba..bcfe234f7080d 100644
--- a/mlir/test/Conversion/SPIRVToLLVM/comparison-ops-to-llvm.mlir
+++ b/mlir/test/Conversion/SPIRVToLLVM/comparison-ops-to-llvm.mlir
@@ -4,190 +4,214 @@
 // spv.IEqual
 //===----------------------------------------------------------------------===//
 
+// CHECK-LABEL: @i_equal_scalar
 func @i_equal_scalar(%arg0: i32, %arg1: i32) {
-	// CHECK: %{{.*}} = llvm.icmp "eq" %{{.*}}, %{{.*}} : !llvm.i32
-	%0 = spv.IEqual %arg0, %arg1 : i32
-	return
+  // CHECK: llvm.icmp "eq" %{{.*}}, %{{.*}} : !llvm.i32
+  %0 = spv.IEqual %arg0, %arg1 : i32
+  return
 }
 
+// CHECK-LABEL: @i_equal_vector
 func @i_equal_vector(%arg0: vector<4xi64>, %arg1: vector<4xi64>) {
-	// CHECK: %{{.*}} = llvm.icmp "eq" %{{.*}}, %{{.*}} : !llvm<"<4 x i64>">
-	%0 = spv.IEqual %arg0, %arg1 : vector<4xi64>
-	return
+  // CHECK: llvm.icmp "eq" %{{.*}}, %{{.*}} : !llvm<"<4 x i64>">
+  %0 = spv.IEqual %arg0, %arg1 : vector<4xi64>
+  return
 }
 
 //===----------------------------------------------------------------------===//
 // spv.INotEqual
 //===----------------------------------------------------------------------===//
 
+// CHECK-LABEL: @i_not_equal_scalar
 func @i_not_equal_scalar(%arg0: i64, %arg1: i64) {
-	// CHECK: %{{.*}} = llvm.icmp "ne" %{{.*}}, %{{.*}} : !llvm.i64
-	%0 = spv.INotEqual %arg0, %arg1 : i64
-	return
+  // CHECK: llvm.icmp "ne" %{{.*}}, %{{.*}} : !llvm.i64
+  %0 = spv.INotEqual %arg0, %arg1 : i64
+  return
 }
 
+// CHECK-LABEL: @i_not_equal_vector
 func @i_not_equal_vector(%arg0: vector<2xi64>, %arg1: vector<2xi64>) {
-	// CHECK: %{{.*}} = llvm.icmp "ne" %{{.*}}, %{{.*}} : !llvm<"<2 x i64>">
-	%0 = spv.INotEqual %arg0, %arg1 : vector<2xi64>
-	return
+  // CHECK: llvm.icmp "ne" %{{.*}}, %{{.*}} : !llvm<"<2 x i64>">
+  %0 = spv.INotEqual %arg0, %arg1 : vector<2xi64>
+  return
 }
 
 //===----------------------------------------------------------------------===//
 // spv.SGreaterThanEqual
 //===----------------------------------------------------------------------===//
 
+// CHECK-LABEL: @s_greater_than_equal_scalar
 func @s_greater_than_equal_scalar(%arg0: i64, %arg1: i64) {
-	// CHECK: %{{.*}} = llvm.icmp "sge" %{{.*}}, %{{.*}} : !llvm.i64
-	%0 = spv.SGreaterThanEqual %arg0, %arg1 : i64
-	return
+  // CHECK: llvm.icmp "sge" %{{.*}}, %{{.*}} : !llvm.i64
+  %0 = spv.SGreaterThanEqual %arg0, %arg1 : i64
+  return
 }
 
+// CHECK-LABEL: @s_greater_than_equal_vector
 func @s_greater_than_equal_vector(%arg0: vector<2xi64>, %arg1: vector<2xi64>) {
-	// CHECK: %{{.*}} = llvm.icmp "sge" %{{.*}}, %{{.*}} : !llvm<"<2 x i64>">
-	%0 = spv.SGreaterThanEqual %arg0, %arg1 : vector<2xi64>
-	return
+  // CHECK: llvm.icmp "sge" %{{.*}}, %{{.*}} : !llvm<"<2 x i64>">
+  %0 = spv.SGreaterThanEqual %arg0, %arg1 : vector<2xi64>
+  return
 }
 
 //===----------------------------------------------------------------------===//
 // spv.SGreaterThan
 //===----------------------------------------------------------------------===//
 
+// CHECK-LABEL: @s_greater_than_scalar
 func @s_greater_than_scalar(%arg0: i64, %arg1: i64) {
-	// CHECK: %{{.*}} = llvm.icmp "sgt" %{{.*}}, %{{.*}} : !llvm.i64
-	%0 = spv.SGreaterThan %arg0, %arg1 : i64
-	return
+  // CHECK: llvm.icmp "sgt" %{{.*}}, %{{.*}} : !llvm.i64
+  %0 = spv.SGreaterThan %arg0, %arg1 : i64
+  return
 }
 
+// CHECK-LABEL: @s_greater_than_vector
 func @s_greater_than_vector(%arg0: vector<2xi64>, %arg1: vector<2xi64>) {
-	// CHECK: %{{.*}} = llvm.icmp "sgt" %{{.*}}, %{{.*}} : !llvm<"<2 x i64>">
-	%0 = spv.SGreaterThan %arg0, %arg1 : vector<2xi64>
-	return
+  // CHECK: llvm.icmp "sgt" %{{.*}}, %{{.*}} : !llvm<"<2 x i64>">
+  %0 = spv.SGreaterThan %arg0, %arg1 : vector<2xi64>
+  return
 }
 
 //===----------------------------------------------------------------------===//
 // spv.SLessThanEqual
 //===----------------------------------------------------------------------===//
 
+// CHECK-LABEL: @s_less_than_equal_scalar
 func @s_less_than_equal_scalar(%arg0: i64, %arg1: i64) {
-	// CHECK: %{{.*}} = llvm.icmp "sle" %{{.*}}, %{{.*}} : !llvm.i64
-	%0 = spv.SLessThanEqual %arg0, %arg1 : i64
-	return
+  // CHECK: llvm.icmp "sle" %{{.*}}, %{{.*}} : !llvm.i64
+  %0 = spv.SLessThanEqual %arg0, %arg1 : i64
+  return
 }
 
+// CHECK-LABEL: @s_less_than_equal_vector
 func @s_less_than_equal_vector(%arg0: vector<2xi64>, %arg1: vector<2xi64>) {
-	// CHECK: %{{.*}} = llvm.icmp "sle" %{{.*}}, %{{.*}} : !llvm<"<2 x i64>">
-	%0 = spv.SLessThanEqual %arg0, %arg1 : vector<2xi64>
-	return
+  // CHECK: llvm.icmp "sle" %{{.*}}, %{{.*}} : !llvm<"<2 x i64>">
+  %0 = spv.SLessThanEqual %arg0, %arg1 : vector<2xi64>
+  return
 }
 
 //===----------------------------------------------------------------------===//
 // spv.SLessThan
 //===----------------------------------------------------------------------===//
 
+// CHECK-LABEL: @s_less_than_scalar
 func @s_less_than_scalar(%arg0: i64, %arg1: i64) {
-	// CHECK: %{{.*}} = llvm.icmp "slt" %{{.*}}, %{{.*}} : !llvm.i64
-	%0 = spv.SLessThan %arg0, %arg1 : i64
-	return
+  // CHECK: llvm.icmp "slt" %{{.*}}, %{{.*}} : !llvm.i64
+  %0 = spv.SLessThan %arg0, %arg1 : i64
+  return
 }
 
+// CHECK-LABEL: @s_less_than_vector
 func @s_less_than_vector(%arg0: vector<2xi64>, %arg1: vector<2xi64>) {
-	// CHECK: %{{.*}} = llvm.icmp "slt" %{{.*}}, %{{.*}} : !llvm<"<2 x i64>">
-	%0 = spv.SLessThan %arg0, %arg1 : vector<2xi64>
-	return
+  // CHECK: llvm.icmp "slt" %{{.*}}, %{{.*}} : !llvm<"<2 x i64>">
+  %0 = spv.SLessThan %arg0, %arg1 : vector<2xi64>
+  return
 }
 
 //===----------------------------------------------------------------------===//
 // spv.UGreaterThanEqual
 //===----------------------------------------------------------------------===//
 
+// CHECK-LABEL: @u_greater_than_equal_scalar
 func @u_greater_than_equal_scalar(%arg0: i64, %arg1: i64) {
-	// CHECK: %{{.*}} = llvm.icmp "uge" %{{.*}}, %{{.*}} : !llvm.i64
-	%0 = spv.UGreaterThanEqual %arg0, %arg1 : i64
-	return
+  // CHECK: llvm.icmp "uge" %{{.*}}, %{{.*}} : !llvm.i64
+  %0 = spv.UGreaterThanEqual %arg0, %arg1 : i64
+  return
 }
 
+// CHECK-LABEL: @u_greater_than_equal_vector
 func @u_greater_than_equal_vector(%arg0: vector<2xi64>, %arg1: vector<2xi64>) {
-	// CHECK: %{{.*}} = llvm.icmp "uge" %{{.*}}, %{{.*}} : !llvm<"<2 x i64>">
-	%0 = spv.UGreaterThanEqual %arg0, %arg1 : vector<2xi64>
-	return
+  // CHECK: llvm.icmp "uge" %{{.*}}, %{{.*}} : !llvm<"<2 x i64>">
+  %0 = spv.UGreaterThanEqual %arg0, %arg1 : vector<2xi64>
+  return
 }
 
 //===----------------------------------------------------------------------===//
 // spv.UGreaterThan
 //===----------------------------------------------------------------------===//
 
+// CHECK-LABEL: @u_greater_than_scalar
 func @u_greater_than_scalar(%arg0: i64, %arg1: i64) {
-	// CHECK: %{{.*}} = llvm.icmp "ugt" %{{.*}}, %{{.*}} : !llvm.i64
-	%0 = spv.UGreaterThan %arg0, %arg1 : i64
-	return
+  // CHECK: llvm.icmp "ugt" %{{.*}}, %{{.*}} : !llvm.i64
+  %0 = spv.UGreaterThan %arg0, %arg1 : i64
+  return
 }
 
+// CHECK-LABEL: @u_greater_than_vector
 func @u_greater_than_vector(%arg0: vector<2xi64>, %arg1: vector<2xi64>) {
-	// CHECK: %{{.*}} = llvm.icmp "ugt" %{{.*}}, %{{.*}} : !llvm<"<2 x i64>">
-	%0 = spv.UGreaterThan %arg0, %arg1 : vector<2xi64>
-	return
+  // CHECK: llvm.icmp "ugt" %{{.*}}, %{{.*}} : !llvm<"<2 x i64>">
+  %0 = spv.UGreaterThan %arg0, %arg1 : vector<2xi64>
+  return
 }
 
 //===----------------------------------------------------------------------===//
 // spv.ULessThanEqual
 //===----------------------------------------------------------------------===//
 
+// CHECK-LABEL: @u_less_than_equal_scalar
 func @u_less_than_equal_scalar(%arg0: i64, %arg1: i64) {
-	// CHECK: %{{.*}} = llvm.icmp "ule" %{{.*}}, %{{.*}} : !llvm.i64
-	%0 = spv.ULessThanEqual %arg0, %arg1 : i64
-	return
+  // CHECK: llvm.icmp "ule" %{{.*}}, %{{.*}} : !llvm.i64
+  %0 = spv.ULessThanEqual %arg0, %arg1 : i64
+  return
 }
 
+// CHECK-LABEL: @u_less_than_equal_vector
 func @u_less_than_equal_vector(%arg0: vector<2xi64>, %arg1: vector<2xi64>) {
-	// CHECK: %{{.*}} = llvm.icmp "ule" %{{.*}}, %{{.*}} : !llvm<"<2 x i64>">
-	%0 = spv.ULessThanEqual %arg0, %arg1 : vector<2xi64>
-	return
+  // CHECK: llvm.icmp "ule" %{{.*}}, %{{.*}} : !llvm<"<2 x i64>">
+  %0 = spv.ULessThanEqual %arg0, %arg1 : vector<2xi64>
+  return
 }
 
 //===----------------------------------------------------------------------===//
 // spv.ULessThan
 //===----------------------------------------------------------------------===//
 
+// CHECK-LABEL: @u_less_than_scalar
 func @u_less_than_scalar(%arg0: i64, %arg1: i64) {
-	// CHECK: %{{.*}} = llvm.icmp "ult" %{{.*}}, %{{.*}} : !llvm.i64
-	%0 = spv.ULessThan %arg0, %arg1 : i64
-	return
+  // CHECK: llvm.icmp "ult" %{{.*}}, %{{.*}} : !llvm.i64
+  %0 = spv.ULessThan %arg0, %arg1 : i64
+  return
 }
 
+// CHECK-LABEL: @u_less_than_vector
 func @u_less_than_vector(%arg0: vector<2xi64>, %arg1: vector<2xi64>) {
-	// CHECK: %{{.*}} = llvm.icmp "ult" %{{.*}}, %{{.*}} : !llvm<"<2 x i64>">
-	%0 = spv.ULessThan %arg0, %arg1 : vector<2xi64>
-	return
+  // CHECK: llvm.icmp "ult" %{{.*}}, %{{.*}} : !llvm<"<2 x i64>">
+  %0 = spv.ULessThan %arg0, %arg1 : vector<2xi64>
+  return
 }
 
 //===----------------------------------------------------------------------===//
 // spv.FOrdEqual
 //===----------------------------------------------------------------------===//
 
+// CHECK-LABEL: @f_ord_equal_scalar
 func @f_ord_equal_scalar(%arg0: f32, %arg1: f32) {
-	// CHECK: %{{.*}} = llvm.fcmp "oeq" %{{.*}}, %{{.*}} : !llvm.float
-	%0 = spv.FOrdEqual %arg0, %arg1 : f32
-	return
+  // CHECK: llvm.fcmp "oeq" %{{.*}}, %{{.*}} : !llvm.float
+  %0 = spv.FOrdEqual %arg0, %arg1 : f32
+  return
 }
 
+// CHECK-LABEL: @f_ord_equal_vector
 func @f_ord_equal_vector(%arg0: vector<4xf64>, %arg1: vector<4xf64>) {
-	// CHECK: %{{.*}} = llvm.fcmp "oeq" %{{.*}}, %{{.*}} : !llvm<"<4 x double>">
-	%0 = spv.FOrdEqual %arg0, %arg1 : vector<4xf64>
-	return
+  // CHECK: llvm.fcmp "oeq" %{{.*}}, %{{.*}} : !llvm<"<4 x double>">
+  %0 = spv.FOrdEqual %arg0, %arg1 : vector<4xf64>
+  return
 }
 
 //===----------------------------------------------------------------------===//
 // spv.FOrdGreaterThanEqual
 //===----------------------------------------------------------------------===//
 
+// CHECK-LABEL: @f_ord_greater_than_equal_scalar
 func @f_ord_greater_than_equal_scalar(%arg0: f64, %arg1: f64) {
-	// CHECK: %{{.*}} = llvm.fcmp "oge" %{{.*}}, %{{.*}} : !llvm.double
-	%0 = spv.FOrdGreaterThanEqual %arg0, %arg1 : f64
-	return
+  // CHECK: llvm.fcmp "oge" %{{.*}}, %{{.*}} : !llvm.double
+  %0 = spv.FOrdGreaterThanEqual %arg0, %arg1 : f64
+  return
 }
 
+// CHECK-LABEL: @f_ord_greater_than_equal_vector
 func @f_ord_greater_than_equal_vector(%arg0: vector<2xf64>, %arg1: vector<2xf64>) {
-	// CHECK: %{{.*}} = llvm.fcmp "oge" %{{.*}}, %{{.*}} : !llvm<"<2 x double>">
+	// CHECK: llvm.fcmp "oge" %{{.*}}, %{{.*}} : !llvm<"<2 x double>">
 	%0 = spv.FOrdGreaterThanEqual %arg0, %arg1 : vector<2xf64>
 	return
 }
@@ -196,158 +220,178 @@ func @f_ord_greater_than_equal_vector(%arg0: vector<2xf64>, %arg1: vector<2xf64>
 // spv.FOrdGreaterThan
 //===----------------------------------------------------------------------===//
 
+// CHECK-LABEL: @f_ord_greater_than_scalar
 func @f_ord_greater_than_scalar(%arg0: f64, %arg1: f64) {
-	// CHECK: %{{.*}} = llvm.fcmp "ogt" %{{.*}}, %{{.*}} : !llvm.double
-	%0 = spv.FOrdGreaterThan %arg0, %arg1 : f64
-	return
+  // CHECK: llvm.fcmp "ogt" %{{.*}}, %{{.*}} : !llvm.double
+  %0 = spv.FOrdGreaterThan %arg0, %arg1 : f64
+  return
 }
 
+// CHECK-LABEL: @f_ord_greater_than_vector
 func @f_ord_greater_than_vector(%arg0: vector<2xf64>, %arg1: vector<2xf64>) {
-	// CHECK: %{{.*}} = llvm.fcmp "ogt" %{{.*}}, %{{.*}} : !llvm<"<2 x double>">
-	%0 = spv.FOrdGreaterThan %arg0, %arg1 : vector<2xf64>
-	return
+  // CHECK: llvm.fcmp "ogt" %{{.*}}, %{{.*}} : !llvm<"<2 x double>">
+  %0 = spv.FOrdGreaterThan %arg0, %arg1 : vector<2xf64>
+  return
 }
 
 //===----------------------------------------------------------------------===//
 // spv.FOrdLessThan
 //===----------------------------------------------------------------------===//
 
+// CHECK-LABEL: @f_ord_less_than_scalar
 func @f_ord_less_than_scalar(%arg0: f64, %arg1: f64) {
-	// CHECK: %{{.*}} = llvm.fcmp "olt" %{{.*}}, %{{.*}} : !llvm.double
-	%0 = spv.FOrdLessThan %arg0, %arg1 : f64
-	return
+  // CHECK: llvm.fcmp "olt" %{{.*}}, %{{.*}} : !llvm.double
+  %0 = spv.FOrdLessThan %arg0, %arg1 : f64
+  return
 }
 
+// CHECK-LABEL: @f_ord_less_than_vector
 func @f_ord_less_than_vector(%arg0: vector<2xf64>, %arg1: vector<2xf64>) {
-	// CHECK: %{{.*}} = llvm.fcmp "olt" %{{.*}}, %{{.*}} : !llvm<"<2 x double>">
-	%0 = spv.FOrdLessThan %arg0, %arg1 : vector<2xf64>
-	return
+  // CHECK: llvm.fcmp "olt" %{{.*}}, %{{.*}} : !llvm<"<2 x double>">
+  %0 = spv.FOrdLessThan %arg0, %arg1 : vector<2xf64>
+  return
 }
 
 //===----------------------------------------------------------------------===//
 // spv.FOrdLessThanEqual
 //===----------------------------------------------------------------------===//
 
+// CHECK-LABEL: @f_ord_less_than_equal_scalar
 func @f_ord_less_than_equal_scalar(%arg0: f64, %arg1: f64) {
-	// CHECK: %{{.*}} = llvm.fcmp "ole" %{{.*}}, %{{.*}} : !llvm.double
-	%0 = spv.FOrdLessThanEqual %arg0, %arg1 : f64
-	return
+  // CHECK: llvm.fcmp "ole" %{{.*}}, %{{.*}} : !llvm.double
+  %0 = spv.FOrdLessThanEqual %arg0, %arg1 : f64
+  return
 }
 
+// CHECK-LABEL: @f_ord_less_than_equal_vector
 func @f_ord_less_than_equal_vector(%arg0: vector<2xf64>, %arg1: vector<2xf64>) {
-	// CHECK: %{{.*}} = llvm.fcmp "ole" %{{.*}}, %{{.*}} : !llvm<"<2 x double>">
-	%0 = spv.FOrdLessThanEqual %arg0, %arg1 : vector<2xf64>
-	return
+  // CHECK: llvm.fcmp "ole" %{{.*}}, %{{.*}} : !llvm<"<2 x double>">
+  %0 = spv.FOrdLessThanEqual %arg0, %arg1 : vector<2xf64>
+  return
 }
 
 //===----------------------------------------------------------------------===//
 // spv.FOrdNotEqual
 //===----------------------------------------------------------------------===//
 
+// CHECK-LABEL: @f_ord_not_equal_scalar
 func @f_ord_not_equal_scalar(%arg0: f32, %arg1: f32) {
-	// CHECK: %{{.*}} = llvm.fcmp "one" %{{.*}}, %{{.*}} : !llvm.float
-	%0 = spv.FOrdNotEqual %arg0, %arg1 : f32
-	return
+  // CHECK: llvm.fcmp "one" %{{.*}}, %{{.*}} : !llvm.float
+  %0 = spv.FOrdNotEqual %arg0, %arg1 : f32
+  return
 }
 
+// CHECK-LABEL: @f_ord_not_equal_vector
 func @f_ord_not_equal_vector(%arg0: vector<4xf64>, %arg1: vector<4xf64>) {
-	// CHECK: %{{.*}} = llvm.fcmp "one" %{{.*}}, %{{.*}} : !llvm<"<4 x double>">
-	%0 = spv.FOrdNotEqual %arg0, %arg1 : vector<4xf64>
-	return
+  // CHECK: llvm.fcmp "one" %{{.*}}, %{{.*}} : !llvm<"<4 x double>">
+  %0 = spv.FOrdNotEqual %arg0, %arg1 : vector<4xf64>
+  return
 }
 
 //===----------------------------------------------------------------------===//
 // spv.FUnordEqual
 //===----------------------------------------------------------------------===//
 
+// CHECK-LABEL: @f_unord_equal_scalar
 func @f_unord_equal_scalar(%arg0: f32, %arg1: f32) {
-	// CHECK: %{{.*}} = llvm.fcmp "ueq" %{{.*}}, %{{.*}} : !llvm.float
-	%0 = spv.FUnordEqual %arg0, %arg1 : f32
-	return
+  // CHECK: llvm.fcmp "ueq" %{{.*}}, %{{.*}} : !llvm.float
+  %0 = spv.FUnordEqual %arg0, %arg1 : f32
+  return
 }
 
+// CHECK-LABEL: @f_unord_equal_vector
 func @f_unord_equal_vector(%arg0: vector<4xf64>, %arg1: vector<4xf64>) {
-	// CHECK: %{{.*}} = llvm.fcmp "ueq" %{{.*}}, %{{.*}} : !llvm<"<4 x double>">
-	%0 = spv.FUnordEqual %arg0, %arg1 : vector<4xf64>
-	return
+  // CHECK: llvm.fcmp "ueq" %{{.*}}, %{{.*}} : !llvm<"<4 x double>">
+  %0 = spv.FUnordEqual %arg0, %arg1 : vector<4xf64>
+  return
 }
 
 //===----------------------------------------------------------------------===//
 // spv.FUnordGreaterThanEqual
 //===----------------------------------------------------------------------===//
 
+// CHECK-LABEL: @f_unord_greater_than_equal_scalar
 func @f_unord_greater_than_equal_scalar(%arg0: f64, %arg1: f64) {
-	// CHECK: %{{.*}} = llvm.fcmp "uge" %{{.*}}, %{{.*}} : !llvm.double
-	%0 = spv.FUnordGreaterThanEqual %arg0, %arg1 : f64
-	return
+  // CHECK: llvm.fcmp "uge" %{{.*}}, %{{.*}} : !llvm.double
+  %0 = spv.FUnordGreaterThanEqual %arg0, %arg1 : f64
+  return
 }
 
+// CHECK-LABEL: @f_unord_greater_than_equal_vector
 func @f_unord_greater_than_equal_vector(%arg0: vector<2xf64>, %arg1: vector<2xf64>) {
-	// CHECK: %{{.*}} = llvm.fcmp "uge" %{{.*}}, %{{.*}} : !llvm<"<2 x double>">
-	%0 = spv.FUnordGreaterThanEqual %arg0, %arg1 : vector<2xf64>
-	return
+  // CHECK: llvm.fcmp "uge" %{{.*}}, %{{.*}} : !llvm<"<2 x double>">
+  %0 = spv.FUnordGreaterThanEqual %arg0, %arg1 : vector<2xf64>
+  return
 }
 
 //===----------------------------------------------------------------------===//
 // spv.FUnordGreaterThan
 //===----------------------------------------------------------------------===//
 
+// CHECK-LABEL: @f_unord_greater_than_scalar
 func @f_unord_greater_than_scalar(%arg0: f64, %arg1: f64) {
-	// CHECK: %{{.*}} = llvm.fcmp "ugt" %{{.*}}, %{{.*}} : !llvm.double
-	%0 = spv.FUnordGreaterThan %arg0, %arg1 : f64
-	return
+  // CHECK: llvm.fcmp "ugt" %{{.*}}, %{{.*}} : !llvm.double
+  %0 = spv.FUnordGreaterThan %arg0, %arg1 : f64
+  return
 }
 
+// CHECK-LABEL: @f_unord_greater_than_vector
 func @f_unord_greater_than_vector(%arg0: vector<2xf64>, %arg1: vector<2xf64>) {
-	// CHECK: %{{.*}} = llvm.fcmp "ugt" %{{.*}}, %{{.*}} : !llvm<"<2 x double>">
-	%0 = spv.FUnordGreaterThan %arg0, %arg1 : vector<2xf64>
-	return
+  // CHECK: llvm.fcmp "ugt" %{{.*}}, %{{.*}} : !llvm<"<2 x double>">
+  %0 = spv.FUnordGreaterThan %arg0, %arg1 : vector<2xf64>
+  return
 }
 
 //===----------------------------------------------------------------------===//
 // spv.FUnordLessThan
 //===----------------------------------------------------------------------===//
 
+// CHECK-LABEL: @f_unord_less_than_scalar
 func @f_unord_less_than_scalar(%arg0: f64, %arg1: f64) {
-	// CHECK: %{{.*}} = llvm.fcmp "ult" %{{.*}}, %{{.*}} : !llvm.double
-	%0 = spv.FUnordLessThan %arg0, %arg1 : f64
-	return
+  // CHECK: llvm.fcmp "ult" %{{.*}}, %{{.*}} : !llvm.double
+  %0 = spv.FUnordLessThan %arg0, %arg1 : f64
+  return
 }
 
+// CHECK-LABEL: @f_unord_less_than_vector
 func @f_unord_less_than_vector(%arg0: vector<2xf64>, %arg1: vector<2xf64>) {
-	// CHECK: %{{.*}} = llvm.fcmp "ult" %{{.*}}, %{{.*}} : !llvm<"<2 x double>">
-	%0 = spv.FUnordLessThan %arg0, %arg1 : vector<2xf64>
-	return
+  // CHECK: llvm.fcmp "ult" %{{.*}}, %{{.*}} : !llvm<"<2 x double>">
+  %0 = spv.FUnordLessThan %arg0, %arg1 : vector<2xf64>
+  return
 }
 
 //===----------------------------------------------------------------------===//
 // spv.FUnordLessThanEqual
 //===----------------------------------------------------------------------===//
 
+// CHECK-LABEL: @f_unord_less_than_equal_scalar
 func @f_unord_less_than_equal_scalar(%arg0: f64, %arg1: f64) {
-	// CHECK: %{{.*}} = llvm.fcmp "ule" %{{.*}}, %{{.*}} : !llvm.double
-	%0 = spv.FUnordLessThanEqual %arg0, %arg1 : f64
-	return
+  // CHECK: llvm.fcmp "ule" %{{.*}}, %{{.*}} : !llvm.double
+  %0 = spv.FUnordLessThanEqual %arg0, %arg1 : f64
+  return
 }
 
+// CHECK-LABEL: @f_unord_less_than_equal_vector
 func @f_unord_less_than_equal_vector(%arg0: vector<2xf64>, %arg1: vector<2xf64>) {
-	// CHECK: %{{.*}} = llvm.fcmp "ule" %{{.*}}, %{{.*}} : !llvm<"<2 x double>">
-	%0 = spv.FUnordLessThanEqual %arg0, %arg1 : vector<2xf64>
-	return
+  // CHECK: llvm.fcmp "ule" %{{.*}}, %{{.*}} : !llvm<"<2 x double>">
+  %0 = spv.FUnordLessThanEqual %arg0, %arg1 : vector<2xf64>
+  return
 }
 
 //===----------------------------------------------------------------------===//
 // spv.FUnordNotEqual
 //===----------------------------------------------------------------------===//
 
+// CHECK-LABEL: @f_unord_not_equal_scalar
 func @f_unord_not_equal_scalar(%arg0: f32, %arg1: f32) {
-	// CHECK: %{{.*}} = llvm.fcmp "une" %{{.*}}, %{{.*}} : !llvm.float
-	%0 = spv.FUnordNotEqual %arg0, %arg1 : f32
-	return
+  // CHECK: llvm.fcmp "une" %{{.*}}, %{{.*}} : !llvm.float
+  %0 = spv.FUnordNotEqual %arg0, %arg1 : f32
+  return
 }
 
+// CHECK-LABEL: @f_unord_not_equal_vector
 func @f_unord_not_equal_vector(%arg0: vector<4xf64>, %arg1: vector<4xf64>) {
-	// CHECK: %{{.*}} = llvm.fcmp "une" %{{.*}}, %{{.*}} : !llvm<"<4 x double>">
-	%0 = spv.FUnordNotEqual %arg0, %arg1 : vector<4xf64>
-	return
+  // CHECK: llvm.fcmp "une" %{{.*}}, %{{.*}} : !llvm<"<4 x double>">
+  %0 = spv.FUnordNotEqual %arg0, %arg1 : vector<4xf64>
+  return
 }
diff --git a/mlir/test/Conversion/SPIRVToLLVM/logical-to-llvm.mlir b/mlir/test/Conversion/SPIRVToLLVM/logical-ops-to-llvm.mlir
similarity index 55%
rename from mlir/test/Conversion/SPIRVToLLVM/logical-to-llvm.mlir
rename to mlir/test/Conversion/SPIRVToLLVM/logical-ops-to-llvm.mlir
index e6f2ec2433fa5..a816424d5b1c3 100644
--- a/mlir/test/Conversion/SPIRVToLLVM/logical-to-llvm.mlir
+++ b/mlir/test/Conversion/SPIRVToLLVM/logical-ops-to-llvm.mlir
@@ -4,15 +4,17 @@
 // spv.LogicalEqual
 //===----------------------------------------------------------------------===//
 
+// CHECK-LABEL: @logical_equal_scalar
 func @logical_equal_scalar(%arg0: i1, %arg1: i1) {
-  // CHECK: %{{.*}} = llvm.icmp "eq" %{{.*}}, %{{.*}} : !llvm.i1
-	%0 = spv.LogicalEqual %arg0, %arg0 : i1
+  // CHECK: llvm.icmp "eq" %{{.*}}, %{{.*}} : !llvm.i1
+  %0 = spv.LogicalEqual %arg0, %arg0 : i1
   return
 }
 
+// CHECK-LABEL: @logical_equal_vector
 func @logical_equal_vector(%arg0: vector<4xi1>, %arg1: vector<4xi1>) {
-  // CHECK: %{{.*}} = llvm.icmp "eq" %{{.*}}, %{{.*}} : !llvm<"<4 x i1>">
-	%0 = spv.LogicalEqual %arg0, %arg0 : vector<4xi1>
+  // CHECK: llvm.icmp "eq" %{{.*}}, %{{.*}} : !llvm<"<4 x i1>">
+  %0 = spv.LogicalEqual %arg0, %arg0 : vector<4xi1>
   return
 }
 
@@ -20,15 +22,17 @@ func @logical_equal_vector(%arg0: vector<4xi1>, %arg1: vector<4xi1>) {
 // spv.LogicalNotEqual
 //===----------------------------------------------------------------------===//
 
+// CHECK-LABEL: @logical_not_equal_scalar
 func @logical_not_equal_scalar(%arg0: i1, %arg1: i1) {
-  // CHECK: %{{.*}} = llvm.icmp "ne" %{{.*}}, %{{.*}} : !llvm.i1
-	%0 = spv.LogicalNotEqual %arg0, %arg0 : i1
+  // CHECK: llvm.icmp "ne" %{{.*}}, %{{.*}} : !llvm.i1
+  %0 = spv.LogicalNotEqual %arg0, %arg0 : i1
   return
 }
 
+// CHECK-LABEL: @logical_not_equal_vector
 func @logical_not_equal_vector(%arg0: vector<4xi1>, %arg1: vector<4xi1>) {
-  // CHECK: %{{.*}} = llvm.icmp "ne" %{{.*}}, %{{.*}} : !llvm<"<4 x i1>">
-	%0 = spv.LogicalNotEqual %arg0, %arg0 : vector<4xi1>
+  // CHECK: llvm.icmp "ne" %{{.*}}, %{{.*}} : !llvm<"<4 x i1>">
+  %0 = spv.LogicalNotEqual %arg0, %arg0 : vector<4xi1>
   return
 }
 
@@ -36,17 +40,19 @@ func @logical_not_equal_vector(%arg0: vector<4xi1>, %arg1: vector<4xi1>) {
 // spv.LogicalNot
 //===----------------------------------------------------------------------===//
 
-func @logical_not__scalar(%arg0: i1) {
+// CHECK-LABEL: @logical_not_scalar
+func @logical_not_scalar(%arg0: i1) {
   // CHECK: %[[CONST:.*]] = llvm.mlir.constant(true) : !llvm.i1
-  // CHECK: %{{.*}} = llvm.xor %{{.*}}, %[[CONST]] : !llvm.i1
-	%0 = spv.LogicalNot %arg0 : i1
+  // CHECK: llvm.xor %{{.*}}, %[[CONST]] : !llvm.i1
+  %0 = spv.LogicalNot %arg0 : i1
   return
 }
 
+// CHECK-LABEL: @logical_not_vector
 func @logical_not_vector(%arg0: vector<4xi1>) {
   // CHECK: %[[CONST:.*]] = llvm.mlir.constant(dense<true> : vector<4xi1>) : !llvm<"<4 x i1>">
-  // CHECK: %{{.*}} = llvm.xor %{{.*}}, %[[CONST]] : !llvm<"<4 x i1>">
-	%0 = spv.LogicalNot %arg0 : vector<4xi1>
+  // CHECK: llvm.xor %{{.*}}, %[[CONST]] : !llvm<"<4 x i1>">
+  %0 = spv.LogicalNot %arg0 : vector<4xi1>
   return
 }
 
@@ -54,15 +60,17 @@ func @logical_not_vector(%arg0: vector<4xi1>) {
 // spv.LogicalAnd
 //===----------------------------------------------------------------------===//
 
+// CHECK-LABEL: @logical_and_scalar
 func @logical_and_scalar(%arg0: i1, %arg1: i1) {
-  // CHECK: %{{.*}} = llvm.and %{{.*}}, %{{.*}} : !llvm.i1
-	%0 = spv.LogicalAnd %arg0, %arg0 : i1
+  // CHECK: llvm.and %{{.*}}, %{{.*}} : !llvm.i1
+  %0 = spv.LogicalAnd %arg0, %arg0 : i1
   return
 }
 
+// CHECK-LABEL: @logical_and_vector
 func @logical_and_vector(%arg0: vector<4xi1>, %arg1: vector<4xi1>) {
-  // CHECK: %{{.*}} = llvm.and %{{.*}}, %{{.*}} : !llvm<"<4 x i1>">
-	%0 = spv.LogicalAnd %arg0, %arg0 : vector<4xi1>
+  // CHECK: llvm.and %{{.*}}, %{{.*}} : !llvm<"<4 x i1>">
+  %0 = spv.LogicalAnd %arg0, %arg0 : vector<4xi1>
   return
 }
 
@@ -70,14 +78,16 @@ func @logical_and_vector(%arg0: vector<4xi1>, %arg1: vector<4xi1>) {
 // spv.LogicalOr
 //===----------------------------------------------------------------------===//
 
+// CHECK-LABEL: @logical_or_scalar
 func @logical_or_scalar(%arg0: i1, %arg1: i1) {
-  // CHECK: %{{.*}} = llvm.or %{{.*}}, %{{.*}} : !llvm.i1
-	%0 = spv.LogicalOr %arg0, %arg0 : i1
+  // CHECK: llvm.or %{{.*}}, %{{.*}} : !llvm.i1
+  %0 = spv.LogicalOr %arg0, %arg0 : i1
   return
 }
 
+// CHECK-LABEL: @logical_or_vector
 func @logical_or_vector(%arg0: vector<4xi1>, %arg1: vector<4xi1>) {
-  // CHECK: %{{.*}} = llvm.or %{{.*}}, %{{.*}} : !llvm<"<4 x i1>">
-	%0 = spv.LogicalOr %arg0, %arg0 : vector<4xi1>
+  // CHECK: llvm.or %{{.*}}, %{{.*}} : !llvm<"<4 x i1>">
+  %0 = spv.LogicalOr %arg0, %arg0 : vector<4xi1>
   return
 }

From 8979a9cdf226066196f1710903d13492e6929563 Mon Sep 17 00:00:00 2001
From: Alexander Belyaev <pifon@google.com>
Date: Tue, 4 Aug 2020 13:34:47 +0200
Subject: [PATCH 1026/1035] [mlir] Fix adding wrong operand value in
 `promoteMemRefDescriptors`.

The bug was not noticed because we didn't have a lot of custom type conversions
directly to LLVM dialect.

Differential Revision: https://reviews.llvm.org/D85192
---
 mlir/lib/Conversion/StandardToLLVM/StandardToLLVM.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mlir/lib/Conversion/StandardToLLVM/StandardToLLVM.cpp b/mlir/lib/Conversion/StandardToLLVM/StandardToLLVM.cpp
index 42c4d49b8941b..d0b49bb181955 100644
--- a/mlir/lib/Conversion/StandardToLLVM/StandardToLLVM.cpp
+++ b/mlir/lib/Conversion/StandardToLLVM/StandardToLLVM.cpp
@@ -3352,7 +3352,7 @@ LLVMTypeConverter::promoteMemRefDescriptors(Location loc, ValueRange opOperands,
       continue;
     }
 
-    promotedOperands.push_back(operand);
+    promotedOperands.push_back(llvmOperand);
   }
   return promotedOperands;
 }

From d4fbbab2e494a59480096a257136ed2b75d07e87 Mon Sep 17 00:00:00 2001
From: Alex Zinenko <zinenko@google.com>
Date: Tue, 4 Aug 2020 11:37:25 +0200
Subject: [PATCH 1027/1035] [mlir] translate types between MLIR LLVM dialect
 and LLVM IR

With new LLVM dialect type modeling, the dialect types no longer wrap LLVM IR
types. Therefore, they need to be translated to and from LLVM IR during export
and import. Introduce the relevant functionality for translating types. It is
currently exercised by an ad-hoc type translation roundtripping test that will
be subsumed by the actual translation test when the type system transition is
complete.

Depends On D84339

Reviewed By: herhut

Differential Revision: https://reviews.llvm.org/D85019
---
 .../mlir/Target/LLVMIR/TypeTranslation.h      |  36 ++
 mlir/lib/Target/CMakeLists.txt                |   1 +
 mlir/lib/Target/LLVMIR/TypeTranslation.cpp    | 309 ++++++++++++++++++
 mlir/test/Target/llvmir-types.mlir            | 228 +++++++++++++
 mlir/test/lib/CMakeLists.txt                  |   1 +
 mlir/test/lib/Target/CMakeLists.txt           |  13 +
 .../lib/Target/TestLLVMTypeTranslation.cpp    |  79 +++++
 mlir/tools/mlir-translate/CMakeLists.txt      |   4 +
 mlir/tools/mlir-translate/mlir-translate.cpp  |   4 +
 9 files changed, 675 insertions(+)
 create mode 100644 mlir/include/mlir/Target/LLVMIR/TypeTranslation.h
 create mode 100644 mlir/lib/Target/LLVMIR/TypeTranslation.cpp
 create mode 100644 mlir/test/Target/llvmir-types.mlir
 create mode 100644 mlir/test/lib/Target/CMakeLists.txt
 create mode 100644 mlir/test/lib/Target/TestLLVMTypeTranslation.cpp

diff --git a/mlir/include/mlir/Target/LLVMIR/TypeTranslation.h b/mlir/include/mlir/Target/LLVMIR/TypeTranslation.h
new file mode 100644
index 0000000000000..5a82f0a096dfc
--- /dev/null
+++ b/mlir/include/mlir/Target/LLVMIR/TypeTranslation.h
@@ -0,0 +1,36 @@
+//===- TypeTranslation.h - Translate types between MLIR & LLVM --*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file declares the type translation function going from MLIR LLVM dialect
+// to LLVM IR and back.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_TARGET_LLVMIR_TYPETRANSLATION_H
+#define MLIR_TARGET_LLVMIR_TYPETRANSLATION_H
+
+namespace llvm {
+class LLVMContext;
+class Type;
+} // namespace llvm
+
+namespace mlir {
+
+class MLIRContext;
+
+namespace LLVM {
+
+class LLVMTypeNew;
+
+llvm::Type *translateTypeToLLVMIR(LLVMTypeNew type, llvm::LLVMContext &context);
+LLVMTypeNew translateTypeFromLLVMIR(llvm::Type *type, MLIRContext &context);
+
+} // namespace LLVM
+} // namespace mlir
+
+#endif // MLIR_TARGET_LLVMIR_TYPETRANSLATION_H
diff --git a/mlir/lib/Target/CMakeLists.txt b/mlir/lib/Target/CMakeLists.txt
index 4a0af66a04b16..5ca335b4b4b50 100644
--- a/mlir/lib/Target/CMakeLists.txt
+++ b/mlir/lib/Target/CMakeLists.txt
@@ -1,6 +1,7 @@
 add_mlir_translation_library(MLIRTargetLLVMIRModuleTranslation
   LLVMIR/DebugTranslation.cpp
   LLVMIR/ModuleTranslation.cpp
+  LLVMIR/TypeTranslation.cpp
 
   ADDITIONAL_HEADER_DIRS
   ${MLIR_MAIN_INCLUDE_DIR}/mlir/Target/LLVMIR
diff --git a/mlir/lib/Target/LLVMIR/TypeTranslation.cpp b/mlir/lib/Target/LLVMIR/TypeTranslation.cpp
new file mode 100644
index 0000000000000..6163334d3b4ef
--- /dev/null
+++ b/mlir/lib/Target/LLVMIR/TypeTranslation.cpp
@@ -0,0 +1,309 @@
+//===- TypeTranslation.cpp - type translation between MLIR LLVM & LLVM IR -===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Target/LLVMIR/TypeTranslation.h"
+#include "mlir/Dialect/LLVMIR/LLVMTypes.h"
+#include "mlir/IR/MLIRContext.h"
+
+#include "llvm/ADT/TypeSwitch.h"
+#include "llvm/IR/DerivedTypes.h"
+#include "llvm/IR/Type.h"
+
+using namespace mlir;
+
+namespace {
+/// Support for translating MLIR LLVM dialect types to LLVM IR.
+class TypeToLLVMIRTranslator {
+public:
+  /// Constructs a class creating types in the given LLVM context.
+  TypeToLLVMIRTranslator(llvm::LLVMContext &context) : context(context) {}
+
+  /// Translates a single type.
+  llvm::Type *translateType(LLVM::LLVMTypeNew type) {
+    // If the conversion is already known, just return it.
+    if (knownTranslations.count(type))
+      return knownTranslations.lookup(type);
+
+    // Dispatch to an appropriate function.
+    llvm::Type *translated =
+        llvm::TypeSwitch<LLVM::LLVMTypeNew, llvm::Type *>(type)
+            .Case([this](LLVM::LLVMVoidType) {
+              return llvm::Type::getVoidTy(context);
+            })
+            .Case([this](LLVM::LLVMHalfType) {
+              return llvm::Type::getHalfTy(context);
+            })
+            .Case([this](LLVM::LLVMBFloatType) {
+              return llvm::Type::getBFloatTy(context);
+            })
+            .Case([this](LLVM::LLVMFloatType) {
+              return llvm::Type::getFloatTy(context);
+            })
+            .Case([this](LLVM::LLVMDoubleType) {
+              return llvm::Type::getDoubleTy(context);
+            })
+            .Case([this](LLVM::LLVMFP128Type) {
+              return llvm::Type::getFP128Ty(context);
+            })
+            .Case([this](LLVM::LLVMX86FP80Type) {
+              return llvm::Type::getX86_FP80Ty(context);
+            })
+            .Case([this](LLVM::LLVMPPCFP128Type) {
+              return llvm::Type::getPPC_FP128Ty(context);
+            })
+            .Case([this](LLVM::LLVMX86MMXType) {
+              return llvm::Type::getX86_MMXTy(context);
+            })
+            .Case([this](LLVM::LLVMTokenType) {
+              return llvm::Type::getTokenTy(context);
+            })
+            .Case([this](LLVM::LLVMLabelType) {
+              return llvm::Type::getLabelTy(context);
+            })
+            .Case([this](LLVM::LLVMMetadataType) {
+              return llvm::Type::getMetadataTy(context);
+            })
+            .Case<LLVM::LLVMArrayType, LLVM::LLVMIntegerType,
+                  LLVM::LLVMFunctionType, LLVM::LLVMPointerType,
+                  LLVM::LLVMStructType, LLVM::LLVMFixedVectorType,
+                  LLVM::LLVMScalableVectorType>(
+                [this](auto array) { return translate(array); })
+            .Default([](LLVM::LLVMTypeNew t) -> llvm::Type * {
+              llvm_unreachable("unknown LLVM dialect type");
+            });
+
+    // Cache the result of the conversion and return.
+    knownTranslations.try_emplace(type, translated);
+    return translated;
+  }
+
+private:
+  /// Translates the given array type.
+  llvm::Type *translate(LLVM::LLVMArrayType type) {
+    return llvm::ArrayType::get(translateType(type.getElementType()),
+                                type.getNumElements());
+  }
+
+  /// Translates the given function type.
+  llvm::Type *translate(LLVM::LLVMFunctionType type) {
+    SmallVector<llvm::Type *, 8> paramTypes;
+    translateTypes(type.getParams(), paramTypes);
+    return llvm::FunctionType::get(translateType(type.getReturnType()),
+                                   paramTypes, type.isVarArg());
+  }
+
+  /// Translates the given integer type.
+  llvm::Type *translate(LLVM::LLVMIntegerType type) {
+    return llvm::IntegerType::get(context, type.getBitWidth());
+  }
+
+  /// Translates the given pointer type.
+  llvm::Type *translate(LLVM::LLVMPointerType type) {
+    return llvm::PointerType::get(translateType(type.getElementType()),
+                                  type.getAddressSpace());
+  }
+
+  /// Translates the given structure type, supports both identified and literal
+  /// structs. This will _create_ a new identified structure every time, use
+  /// `convertType` if a structure with the same name must be looked up instead.
+  llvm::Type *translate(LLVM::LLVMStructType type) {
+    SmallVector<llvm::Type *, 8> subtypes;
+    if (!type.isIdentified()) {
+      translateTypes(type.getBody(), subtypes);
+      return llvm::StructType::get(context, subtypes, type.isPacked());
+    }
+
+    llvm::StructType *structType =
+        llvm::StructType::create(context, type.getName());
+    // Mark the type we just created as known so that recursive calls can pick
+    // it up and use directly.
+    knownTranslations.try_emplace(type, structType);
+    if (type.isOpaque())
+      return structType;
+
+    translateTypes(type.getBody(), subtypes);
+    structType->setBody(subtypes, type.isPacked());
+    return structType;
+  }
+
+  /// Translates the given fixed-vector type.
+  llvm::Type *translate(LLVM::LLVMFixedVectorType type) {
+    return llvm::FixedVectorType::get(translateType(type.getElementType()),
+                                      type.getNumElements());
+  }
+
+  /// Translates the given scalable-vector type.
+  llvm::Type *translate(LLVM::LLVMScalableVectorType type) {
+    return llvm::ScalableVectorType::get(translateType(type.getElementType()),
+                                         type.getMinNumElements());
+  }
+
+  /// Translates a list of types.
+  void translateTypes(ArrayRef<LLVM::LLVMTypeNew> types,
+                      SmallVectorImpl<llvm::Type *> &result) {
+    result.reserve(result.size() + types.size());
+    for (auto type : types)
+      result.push_back(translateType(type));
+  }
+
+  /// Reference to the context in which the LLVM IR types are created.
+  llvm::LLVMContext &context;
+
+  /// Map of known translation. This serves a double purpose: caches translation
+  /// results to avoid repeated recursive calls and makes sure identified
+  /// structs with the same name (that is, equal) are resolved to an existing
+  /// type instead of creating a new type.
+  llvm::DenseMap<LLVM::LLVMTypeNew, llvm::Type *> knownTranslations;
+};
+} // end namespace
+
+/// Translates a type from MLIR LLVM dialect to LLVM IR. This does not maintain
+/// the mapping for identified structs so new structs will be created with
+/// auto-renaming on each call. This is intended exclusively for testing.
+llvm::Type *mlir::LLVM::translateTypeToLLVMIR(LLVM::LLVMTypeNew type,
+                                              llvm::LLVMContext &context) {
+  return TypeToLLVMIRTranslator(context).translateType(type);
+}
+
+namespace {
+/// Support for translating LLVM IR types to MLIR LLVM dialect types.
+class TypeFromLLVMIRTranslator {
+public:
+  /// Constructs a class creating types in the given MLIR context.
+  TypeFromLLVMIRTranslator(MLIRContext &context) : context(context) {}
+
+  /// Translates the given type.
+  LLVM::LLVMTypeNew translateType(llvm::Type *type) {
+    if (knownTranslations.count(type))
+      return knownTranslations.lookup(type);
+
+    LLVM::LLVMTypeNew translated =
+        llvm::TypeSwitch<llvm::Type *, LLVM::LLVMTypeNew>(type)
+            .Case<llvm::ArrayType, llvm::FunctionType, llvm::IntegerType,
+                  llvm::PointerType, llvm::StructType, llvm::FixedVectorType,
+                  llvm::ScalableVectorType>(
+                [this](auto *type) { return translate(type); })
+            .Default([this](llvm::Type *type) {
+              return translatePrimitiveType(type);
+            });
+    knownTranslations.try_emplace(type, translated);
+    return translated;
+  }
+
+private:
+  /// Translates the given primitive, i.e. non-parametric in MLIR nomenclature,
+  /// type.
+  LLVM::LLVMTypeNew translatePrimitiveType(llvm::Type *type) {
+    if (type->isVoidTy())
+      return LLVM::LLVMVoidType::get(&context);
+    if (type->isHalfTy())
+      return LLVM::LLVMHalfType::get(&context);
+    if (type->isBFloatTy())
+      return LLVM::LLVMBFloatType::get(&context);
+    if (type->isFloatTy())
+      return LLVM::LLVMFloatType::get(&context);
+    if (type->isDoubleTy())
+      return LLVM::LLVMDoubleType::get(&context);
+    if (type->isFP128Ty())
+      return LLVM::LLVMFP128Type::get(&context);
+    if (type->isX86_FP80Ty())
+      return LLVM::LLVMX86FP80Type::get(&context);
+    if (type->isPPC_FP128Ty())
+      return LLVM::LLVMPPCFP128Type::get(&context);
+    if (type->isX86_MMXTy())
+      return LLVM::LLVMX86MMXType::get(&context);
+    if (type->isLabelTy())
+      return LLVM::LLVMLabelType::get(&context);
+    if (type->isMetadataTy())
+      return LLVM::LLVMMetadataType::get(&context);
+    llvm_unreachable("not a primitive type");
+  }
+
+  /// Translates the given array type.
+  LLVM::LLVMTypeNew translate(llvm::ArrayType *type) {
+    return LLVM::LLVMArrayType::get(translateType(type->getElementType()),
+                                    type->getNumElements());
+  }
+
+  /// Translates the given function type.
+  LLVM::LLVMTypeNew translate(llvm::FunctionType *type) {
+    SmallVector<LLVM::LLVMTypeNew, 8> paramTypes;
+    translateTypes(type->params(), paramTypes);
+    return LLVM::LLVMFunctionType::get(translateType(type->getReturnType()),
+                                       paramTypes, type->isVarArg());
+  }
+
+  /// Translates the given integer type.
+  LLVM::LLVMTypeNew translate(llvm::IntegerType *type) {
+    return LLVM::LLVMIntegerType::get(&context, type->getBitWidth());
+  }
+
+  /// Translates the given pointer type.
+  LLVM::LLVMTypeNew translate(llvm::PointerType *type) {
+    return LLVM::LLVMPointerType::get(translateType(type->getElementType()),
+                                      type->getAddressSpace());
+  }
+
+  /// Translates the given structure type.
+  LLVM::LLVMTypeNew translate(llvm::StructType *type) {
+    SmallVector<LLVM::LLVMTypeNew, 8> subtypes;
+    if (type->isLiteral()) {
+      translateTypes(type->subtypes(), subtypes);
+      return LLVM::LLVMStructType::getLiteral(&context, subtypes,
+                                              type->isPacked());
+    }
+
+    if (type->isOpaque())
+      return LLVM::LLVMStructType::getOpaque(type->getName(), &context);
+
+    LLVM::LLVMStructType translated =
+        LLVM::LLVMStructType::getIdentified(&context, type->getName());
+    knownTranslations.try_emplace(type, translated);
+    translateTypes(type->subtypes(), subtypes);
+    LogicalResult bodySet = translated.setBody(subtypes, type->isPacked());
+    assert(succeeded(bodySet) &&
+           "could not set the body of an identified struct");
+    (void)bodySet;
+    return translated;
+  }
+
+  /// Translates the given fixed-vector type.
+  LLVM::LLVMTypeNew translate(llvm::FixedVectorType *type) {
+    return LLVM::LLVMFixedVectorType::get(translateType(type->getElementType()),
+                                          type->getNumElements());
+  }
+
+  /// Translates the given scalable-vector type.
+  LLVM::LLVMTypeNew translate(llvm::ScalableVectorType *type) {
+    return LLVM::LLVMScalableVectorType::get(
+        translateType(type->getElementType()), type->getMinNumElements());
+  }
+
+  /// Translates a list of types.
+  void translateTypes(ArrayRef<llvm::Type *> types,
+                      SmallVectorImpl<LLVM::LLVMTypeNew> &result) {
+    result.reserve(result.size() + types.size());
+    for (llvm::Type *type : types)
+      result.push_back(translateType(type));
+  }
+
+  /// Map of known translations. Serves as a cache and as recursion stopper for
+  /// translating recursive structs.
+  llvm::DenseMap<llvm::Type *, LLVM::LLVMTypeNew> knownTranslations;
+
+  /// The context in which MLIR types are created.
+  MLIRContext &context;
+};
+} // end namespace
+
+/// Translates a type from LLVM IR to MLIR LLVM dialect. This is intended
+/// exclusively for testing.
+LLVM::LLVMTypeNew mlir::LLVM::translateTypeFromLLVMIR(llvm::Type *type,
+                                                      MLIRContext &context) {
+  return TypeFromLLVMIRTranslator(context).translateType(type);
+}
diff --git a/mlir/test/Target/llvmir-types.mlir b/mlir/test/Target/llvmir-types.mlir
new file mode 100644
index 0000000000000..d807562d1a2d0
--- /dev/null
+++ b/mlir/test/Target/llvmir-types.mlir
@@ -0,0 +1,228 @@
+// RUN: mlir-translate -test-mlir-to-llvmir -split-input-file %s | FileCheck %s
+
+llvm.func @primitives() {
+  // CHECK: declare void @return_void()
+  // CHECK: declare void @return_void_round()
+  "llvm.test_introduce_func"() { name = "return_void", type = !llvm2.void } : () -> ()
+  // CHECK: declare half @return_half()
+  // CHECK: declare half @return_half_round()
+  "llvm.test_introduce_func"() { name = "return_half", type = !llvm2.half } : () -> ()
+  // CHECK: declare bfloat @return_bfloat()
+  // CHECK: declare bfloat @return_bfloat_round()
+  "llvm.test_introduce_func"() { name = "return_bfloat", type = !llvm2.bfloat } : () -> ()
+  // CHECK: declare float @return_float()
+  // CHECK: declare float @return_float_round()
+  "llvm.test_introduce_func"() { name = "return_float", type = !llvm2.float } : () -> ()
+  // CHECK: declare double @return_double()
+  // CHECK: declare double @return_double_round()
+  "llvm.test_introduce_func"() { name = "return_double", type = !llvm2.double } : () -> ()
+  // CHECK: declare fp128 @return_fp128()
+  // CHECK: declare fp128 @return_fp128_round()
+  "llvm.test_introduce_func"() { name = "return_fp128", type = !llvm2.fp128 } : () -> ()
+  // CHECK: declare x86_fp80 @return_x86_fp80()
+  // CHECK: declare x86_fp80 @return_x86_fp80_round()
+  "llvm.test_introduce_func"() { name = "return_x86_fp80", type = !llvm2.x86_fp80 } : () -> ()
+  // CHECK: declare ppc_fp128 @return_ppc_fp128()
+  // CHECK: declare ppc_fp128 @return_ppc_fp128_round()
+  "llvm.test_introduce_func"() { name = "return_ppc_fp128", type = !llvm2.ppc_fp128 } : () -> ()
+  // CHECK: declare x86_mmx @return_x86_mmx()
+  // CHECK: declare x86_mmx @return_x86_mmx_round()
+  "llvm.test_introduce_func"() { name = "return_x86_mmx", type = !llvm2.x86_mmx } : () -> ()
+  llvm.return
+}
+
+llvm.func @funcs() {
+  // CHECK: declare void @f_void_i32(i32)
+  // CHECK: declare void @f_void_i32_round(i32)
+  "llvm.test_introduce_func"() { name ="f_void_i32", type = !llvm2.func<void (i32)> } : () -> ()
+  // CHECK: declare i32 @f_i32_empty()
+  // CHECK: declare i32 @f_i32_empty_round()
+  "llvm.test_introduce_func"() { name ="f_i32_empty", type = !llvm2.func<i32 ()> } : () -> ()
+  // CHECK: declare i32 @f_i32_half_bfloat_float_double(half, bfloat, float, double)
+  // CHECK: declare i32 @f_i32_half_bfloat_float_double_round(half, bfloat, float, double)
+  "llvm.test_introduce_func"() { name ="f_i32_half_bfloat_float_double", type = !llvm2.func<i32 (half, bfloat, float, double)> } : () -> ()
+  // CHECK: declare i32 @f_i32_i32_i32(i32, i32)
+  // CHECK: declare i32 @f_i32_i32_i32_round(i32, i32)
+  "llvm.test_introduce_func"() { name ="f_i32_i32_i32", type = !llvm2.func<i32 (i32, i32)> } : () -> ()
+  // CHECK: declare void @f_void_variadic(...)
+  // CHECK: declare void @f_void_variadic_round(...)
+  "llvm.test_introduce_func"() { name ="f_void_variadic", type = !llvm2.func<void (...)> } : () -> ()
+  // CHECK: declare void @f_void_i32_i32_variadic(i32, i32, ...)
+  // CHECK: declare void @f_void_i32_i32_variadic_round(i32, i32, ...)
+  "llvm.test_introduce_func"() { name ="f_void_i32_i32_variadic", type = !llvm2.func<void (i32, i32, ...)> } : () -> ()
+  llvm.return
+}
+
+llvm.func @ints() {
+  // CHECK: declare i1 @return_i1()
+  // CHECK: declare i1 @return_i1_round()
+  "llvm.test_introduce_func"() { name = "return_i1", type = !llvm2.i1 } : () -> ()
+  // CHECK: declare i8 @return_i8()
+  // CHECK: declare i8 @return_i8_round()
+  "llvm.test_introduce_func"() { name = "return_i8", type = !llvm2.i8 } : () -> ()
+  // CHECK: declare i16 @return_i16()
+  // CHECK: declare i16 @return_i16_round()
+  "llvm.test_introduce_func"() { name = "return_i16", type = !llvm2.i16 } : () -> ()
+  // CHECK: declare i32 @return_i32()
+  // CHECK: declare i32 @return_i32_round()
+  "llvm.test_introduce_func"() { name = "return_i32", type = !llvm2.i32 } : () -> ()
+  // CHECK: declare i64 @return_i64()
+  // CHECK: declare i64 @return_i64_round()
+  "llvm.test_introduce_func"() { name = "return_i64", type = !llvm2.i64 } : () -> ()
+  // CHECK: declare i57 @return_i57()
+  // CHECK: declare i57 @return_i57_round()
+  "llvm.test_introduce_func"() { name = "return_i57", type = !llvm2.i57 } : () -> ()
+  // CHECK: declare i129 @return_i129()
+  // CHECK: declare i129 @return_i129_round()
+  "llvm.test_introduce_func"() { name = "return_i129", type = !llvm2.i129 } : () -> ()
+  llvm.return
+}
+
+llvm.func @pointers() {
+  // CHECK: declare i8* @return_pi8()
+  // CHECK: declare i8* @return_pi8_round()
+  "llvm.test_introduce_func"() { name = "return_pi8", type = !llvm2.ptr<i8> } : () -> ()
+  // CHECK: declare float* @return_pfloat()
+  // CHECK: declare float* @return_pfloat_round()
+  "llvm.test_introduce_func"() { name = "return_pfloat", type = !llvm2.ptr<float> } : () -> ()
+  // CHECK: declare i8** @return_ppi8()
+  // CHECK: declare i8** @return_ppi8_round()
+  "llvm.test_introduce_func"() { name = "return_ppi8", type = !llvm2.ptr<ptr<i8>> } : () -> ()
+  // CHECK: declare i8***** @return_pppppi8()
+  // CHECK: declare i8***** @return_pppppi8_round()
+  "llvm.test_introduce_func"() { name = "return_pppppi8", type = !llvm2.ptr<ptr<ptr<ptr<ptr<i8>>>>> } : () -> ()
+  // CHECK: declare i8* @return_pi8_0()
+  // CHECK: declare i8* @return_pi8_0_round()
+  "llvm.test_introduce_func"() { name = "return_pi8_0", type = !llvm2.ptr<i8, 0> } : () -> ()
+  // CHECK: declare i8 addrspace(1)* @return_pi8_1()
+  // CHECK: declare i8 addrspace(1)* @return_pi8_1_round()
+  "llvm.test_introduce_func"() { name = "return_pi8_1", type = !llvm2.ptr<i8, 1> } : () -> ()
+  // CHECK: declare i8 addrspace(42)* @return_pi8_42()
+  // CHECK: declare i8 addrspace(42)* @return_pi8_42_round()
+  "llvm.test_introduce_func"() { name = "return_pi8_42", type = !llvm2.ptr<i8, 42> } : () -> ()
+  // CHECK: declare i8 addrspace(42)* addrspace(9)* @return_ppi8_42_9()
+  // CHECK: declare i8 addrspace(42)* addrspace(9)* @return_ppi8_42_9_round()
+  "llvm.test_introduce_func"() { name = "return_ppi8_42_9", type = !llvm2.ptr<ptr<i8, 42>, 9> } : () -> ()
+  llvm.return
+}
+
+llvm.func @vectors() {
+  // CHECK: declare <4 x i32> @return_v4_i32()
+  // CHECK: declare <4 x i32> @return_v4_i32_round()
+  "llvm.test_introduce_func"() { name = "return_v4_i32", type = !llvm2.vec<4 x i32> } : () -> ()
+  // CHECK: declare <4 x float> @return_v4_float()
+  // CHECK: declare <4 x float> @return_v4_float_round()
+  "llvm.test_introduce_func"() { name = "return_v4_float", type = !llvm2.vec<4 x float> } : () -> ()
+  // CHECK: declare <vscale x 4 x i32> @return_vs_4_i32()
+  // CHECK: declare <vscale x 4 x i32> @return_vs_4_i32_round()
+  "llvm.test_introduce_func"() { name = "return_vs_4_i32", type = !llvm2.vec<? x 4 x i32> } : () -> ()
+  // CHECK: declare <vscale x 8 x half> @return_vs_8_half()
+  // CHECK: declare <vscale x 8 x half> @return_vs_8_half_round()
+  "llvm.test_introduce_func"() { name = "return_vs_8_half", type = !llvm2.vec<? x 8 x half> } : () -> ()
+  // CHECK: declare <4 x i8*> @return_v_4_pi8()
+  // CHECK: declare <4 x i8*> @return_v_4_pi8_round()
+  "llvm.test_introduce_func"() { name = "return_v_4_pi8", type = !llvm2.vec<4 x ptr<i8>> } : () -> ()
+  llvm.return
+}
+
+llvm.func @arrays() {
+  // CHECK: declare [10 x i32] @return_a10_i32()
+  // CHECK: declare [10 x i32] @return_a10_i32_round()
+  "llvm.test_introduce_func"() { name = "return_a10_i32", type = !llvm2.array<10 x i32> } : () -> ()
+  // CHECK: declare [8 x float] @return_a8_float()
+  // CHECK: declare [8 x float] @return_a8_float_round()
+  "llvm.test_introduce_func"() { name = "return_a8_float", type = !llvm2.array<8 x float> } : () -> ()
+  // CHECK: declare [10 x i32 addrspace(4)*] @return_a10_pi32_4()
+  // CHECK: declare [10 x i32 addrspace(4)*] @return_a10_pi32_4_round()
+  "llvm.test_introduce_func"() { name = "return_a10_pi32_4", type = !llvm2.array<10 x ptr<i32, 4>> } : () -> ()
+  // CHECK: declare [10 x [4 x float]] @return_a10_a4_float()
+  // CHECK: declare [10 x [4 x float]] @return_a10_a4_float_round()
+  "llvm.test_introduce_func"() { name = "return_a10_a4_float", type = !llvm2.array<10 x array<4 x float>> } : () -> ()
+  llvm.return
+}
+
+llvm.func @literal_structs() {
+  // CHECK: declare {} @return_struct_empty()
+  // CHECK: declare {} @return_struct_empty_round()
+  "llvm.test_introduce_func"() { name = "return_struct_empty", type = !llvm2.struct<()> } : () -> ()
+  // CHECK: declare { i32 } @return_s_i32()
+  // CHECK: declare { i32 } @return_s_i32_round()
+  "llvm.test_introduce_func"() { name = "return_s_i32", type = !llvm2.struct<(i32)> } : () -> ()
+  // CHECK: declare { float, i32 } @return_s_float_i32()
+  // CHECK: declare { float, i32 } @return_s_float_i32_round()
+  "llvm.test_introduce_func"() { name = "return_s_float_i32", type = !llvm2.struct<(float, i32)> } : () -> ()
+  // CHECK: declare { { i32 } } @return_s_s_i32()
+  // CHECK: declare { { i32 } } @return_s_s_i32_round()
+  "llvm.test_introduce_func"() { name = "return_s_s_i32", type = !llvm2.struct<(struct<(i32)>)> } : () -> ()
+  // CHECK: declare { i32, { i32 }, float } @return_s_i32_s_i32_float()
+  // CHECK: declare { i32, { i32 }, float } @return_s_i32_s_i32_float_round()
+  "llvm.test_introduce_func"() { name = "return_s_i32_s_i32_float", type = !llvm2.struct<(i32, struct<(i32)>, float)> } : () -> ()
+
+  // CHECK: declare <{}> @return_sp_empty()
+  // CHECK: declare <{}> @return_sp_empty_round()
+  "llvm.test_introduce_func"() { name = "return_sp_empty", type = !llvm2.struct<packed ()> } : () -> ()
+  // CHECK: declare <{ i32 }> @return_sp_i32()
+  // CHECK: declare <{ i32 }> @return_sp_i32_round()
+  "llvm.test_introduce_func"() { name = "return_sp_i32", type = !llvm2.struct<packed (i32)> } : () -> ()
+  // CHECK: declare <{ float, i32 }> @return_sp_float_i32()
+  // CHECK: declare <{ float, i32 }> @return_sp_float_i32_round()
+  "llvm.test_introduce_func"() { name = "return_sp_float_i32", type = !llvm2.struct<packed (float, i32)> } : () -> ()
+  // CHECK: declare <{ i32, { i32, i1 }, float }> @return_sp_i32_s_i31_1_float()
+  // CHECK: declare <{ i32, { i32, i1 }, float }> @return_sp_i32_s_i31_1_float_round()
+  "llvm.test_introduce_func"() { name = "return_sp_i32_s_i31_1_float", type = !llvm2.struct<packed (i32, struct<(i32, i1)>, float)> } : () -> ()
+
+  // CHECK: declare { <{ i32 }> } @return_s_sp_i32()
+  // CHECK: declare { <{ i32 }> } @return_s_sp_i32_round()
+  "llvm.test_introduce_func"() { name = "return_s_sp_i32", type = !llvm2.struct<(struct<packed (i32)>)> } : () -> ()
+  // CHECK: declare <{ { i32 } }> @return_sp_s_i32()
+  // CHECK: declare <{ { i32 } }> @return_sp_s_i32_round()
+  "llvm.test_introduce_func"() { name = "return_sp_s_i32", type = !llvm2.struct<packed (struct<(i32)>)> } : () -> ()
+  llvm.return
+}
+
+// -----
+// Put structs into a separate split so that we can match their declarations
+// locally.
+
+// CHECK: %empty = type {}
+// CHECK: %opaque = type opaque
+// CHECK: %long = type { i32, { i32, i1 }, float, void ()* }
+// CHECK: %self-recursive = type { %self-recursive* }
+// CHECK: %unpacked = type { i32 }
+// CHECK: %packed = type <{ i32 }>
+// CHECK: %"name with spaces and !^$@$#" = type <{ i32 }>
+// CHECK: %mutually-a = type { %mutually-b* }
+// CHECK: %mutually-b = type { %mutually-a addrspace(3)* }
+// CHECK: %struct-of-arrays = type { [10 x i32] }
+// CHECK: %array-of-structs = type { i32 }
+// CHECK: %ptr-to-struct = type { i8 }
+
+llvm.func @identified_structs() {
+  // CHECK: declare %empty
+  "llvm.test_introduce_func"() { name = "return_s_empty", type = !llvm2.struct<"empty", ()> } : () -> ()
+  // CHECK: declare %opaque
+  "llvm.test_introduce_func"() { name = "return_s_opaque", type = !llvm2.struct<"opaque", opaque> } : () -> ()
+  // CHECK: declare %long
+  "llvm.test_introduce_func"() { name = "return_s_long", type = !llvm2.struct<"long", (i32, struct<(i32, i1)>, float, ptr<func<void ()>>)> } : () -> ()
+  // CHECK: declare %self-recursive
+  "llvm.test_introduce_func"() { name = "return_s_self_recurisve", type = !llvm2.struct<"self-recursive", (ptr<struct<"self-recursive">>)> } : () -> ()
+  // CHECK: declare %unpacked
+  "llvm.test_introduce_func"() { name = "return_s_unpacked", type = !llvm2.struct<"unpacked", (i32)> } : () -> ()
+  // CHECK: declare %packed
+  "llvm.test_introduce_func"() { name = "return_s_packed", type = !llvm2.struct<"packed", packed (i32)> } : () -> ()
+  // CHECK: declare %"name with spaces and !^$@$#"
+  "llvm.test_introduce_func"() { name = "return_s_symbols", type = !llvm2.struct<"name with spaces and !^$@$#", packed (i32)> } : () -> ()
+
+  // CHECK: declare %mutually-a
+  "llvm.test_introduce_func"() { name = "return_s_mutually_a", type = !llvm2.struct<"mutually-a", (ptr<struct<"mutually-b", (ptr<struct<"mutually-a">, 3>)>>)> } : () -> ()
+  // CHECK: declare %mutually-b
+  "llvm.test_introduce_func"() { name = "return_s_mutually_b", type = !llvm2.struct<"mutually-b", (ptr<struct<"mutually-a", (ptr<struct<"mutually-b">>)>, 3>)> } : () -> ()
+
+  // CHECK: declare %struct-of-arrays
+  "llvm.test_introduce_func"() { name = "return_s_struct_of_arrays", type = !llvm2.struct<"struct-of-arrays", (array<10 x i32>)> } : () -> ()
+  // CHECK: declare [10 x %array-of-structs]
+  "llvm.test_introduce_func"() { name = "return_s_array_of_structs", type = !llvm2.array<10 x struct<"array-of-structs", (i32)>> } : () -> ()
+  // CHECK: declare %ptr-to-struct*
+  "llvm.test_introduce_func"() { name = "return_s_ptr_to_struct", type = !llvm2.ptr<struct<"ptr-to-struct", (i8)>> } : () -> ()
+  llvm.return
+}
diff --git a/mlir/test/lib/CMakeLists.txt b/mlir/test/lib/CMakeLists.txt
index 0df357c8c355e..ec9e5cd998018 100644
--- a/mlir/test/lib/CMakeLists.txt
+++ b/mlir/test/lib/CMakeLists.txt
@@ -2,4 +2,5 @@ add_subdirectory(Dialect)
 add_subdirectory(IR)
 add_subdirectory(Pass)
 add_subdirectory(Reducer)
+add_subdirectory(Target)
 add_subdirectory(Transforms)
diff --git a/mlir/test/lib/Target/CMakeLists.txt b/mlir/test/lib/Target/CMakeLists.txt
new file mode 100644
index 0000000000000..cb8f206469aec
--- /dev/null
+++ b/mlir/test/lib/Target/CMakeLists.txt
@@ -0,0 +1,13 @@
+add_mlir_translation_library(MLIRTestLLVMTypeTranslation
+  TestLLVMTypeTranslation.cpp
+
+  LINK_COMPONENTS
+  Core
+  TransformUtils
+
+  LINK_LIBS PUBLIC
+  MLIRLLVMIR
+  MLIRTargetLLVMIRModuleTranslation
+  MLIRTestIR
+  MLIRTranslation
+  )
diff --git a/mlir/test/lib/Target/TestLLVMTypeTranslation.cpp b/mlir/test/lib/Target/TestLLVMTypeTranslation.cpp
new file mode 100644
index 0000000000000..b76ac2a13344b
--- /dev/null
+++ b/mlir/test/lib/Target/TestLLVMTypeTranslation.cpp
@@ -0,0 +1,79 @@
+//===- TestLLVMTypeTranslation.cpp - Test MLIR/LLVM IR type translation ---===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/LLVMIR/LLVMTypes.h"
+#include "mlir/Target/LLVMIR/ModuleTranslation.h"
+#include "mlir/Target/LLVMIR/TypeTranslation.h"
+#include "mlir/Translation.h"
+
+using namespace mlir;
+
+namespace {
+class TestLLVMTypeTranslation : public LLVM::ModuleTranslation {
+  // Allow access to the constructors under MSVC.
+  friend LLVM::ModuleTranslation;
+
+public:
+  using LLVM::ModuleTranslation::ModuleTranslation;
+
+protected:
+  /// Simple test facility for translating types from MLIR LLVM dialect to LLVM
+  /// IR. This converts the "llvm.test_introduce_func" operation into an LLVM IR
+  /// function with the name extracted from the `name` attribute that returns
+  /// the type contained in the `type` attribute if it is a non-function type or
+  /// that has the signature obtained by converting `type` if it is a function
+  /// type. This is a temporary check before type translation is substituted
+  /// into the main translation flow and exercised here.
+  LogicalResult convertOperation(Operation &op,
+                                 llvm::IRBuilder<> &builder) override {
+    if (op.getName().getStringRef() == "llvm.test_introduce_func") {
+      auto attr = op.getAttrOfType<TypeAttr>("type");
+      assert(attr && "expected 'type' attribute");
+      auto type = attr.getValue().cast<LLVM::LLVMTypeNew>();
+
+      auto nameAttr = op.getAttrOfType<StringAttr>("name");
+      assert(nameAttr && "expected 'name' attributes");
+
+      llvm::Type *translated =
+          LLVM::translateTypeToLLVMIR(type, builder.getContext());
+
+      llvm::Module *module = builder.GetInsertBlock()->getModule();
+      if (auto *funcType = dyn_cast<llvm::FunctionType>(translated))
+        module->getOrInsertFunction(nameAttr.getValue(), funcType);
+      else
+        module->getOrInsertFunction(nameAttr.getValue(), translated);
+
+      std::string roundtripName = (Twine(nameAttr.getValue()) + "_round").str();
+      LLVM::LLVMTypeNew translatedBack =
+          LLVM::translateTypeFromLLVMIR(translated, *op.getContext());
+      llvm::Type *translatedBackAndForth =
+          LLVM::translateTypeToLLVMIR(translatedBack, builder.getContext());
+      if (auto *funcType = dyn_cast<llvm::FunctionType>(translatedBackAndForth))
+        module->getOrInsertFunction(roundtripName, funcType);
+      else
+        module->getOrInsertFunction(roundtripName, translatedBackAndForth);
+      return success();
+    }
+
+    return LLVM::ModuleTranslation::convertOperation(op, builder);
+  }
+};
+} // namespace
+
+namespace mlir {
+void registerTestLLVMTypeTranslation() {
+  TranslateFromMLIRRegistration reg(
+      "test-mlir-to-llvmir", [](ModuleOp module, raw_ostream &output) {
+        std::unique_ptr<llvm::Module> llvmModule =
+            LLVM::ModuleTranslation::translateModule<TestLLVMTypeTranslation>(
+                module.getOperation());
+        llvmModule->print(output, nullptr);
+        return success();
+      });
+}
+} // namespace mlir
diff --git a/mlir/tools/mlir-translate/CMakeLists.txt b/mlir/tools/mlir-translate/CMakeLists.txt
index 897e7adc03bd1..1e6cdfe0f3b14 100644
--- a/mlir/tools/mlir-translate/CMakeLists.txt
+++ b/mlir/tools/mlir-translate/CMakeLists.txt
@@ -13,7 +13,11 @@ target_link_libraries(mlir-translate
   PRIVATE
   ${dialect_libs}
   ${translation_libs}
+  ${test_libs}
   MLIRIR
+  # TODO: remove after LLVM dialect transition is complete; translation uses a
+  # registration function defined in this library unconditionally.
+  MLIRLLVMTypeTestDialect
   MLIRParser
   MLIRPass
   MLIRSPIRV
diff --git a/mlir/tools/mlir-translate/mlir-translate.cpp b/mlir/tools/mlir-translate/mlir-translate.cpp
index 914bd340b3f56..70bf285112a4b 100644
--- a/mlir/tools/mlir-translate/mlir-translate.cpp
+++ b/mlir/tools/mlir-translate/mlir-translate.cpp
@@ -49,17 +49,21 @@ static llvm::cl::opt<bool> verifyDiagnostics(
 
 namespace mlir {
 // Defined in the test directory, no public header.
+void registerLLVMTypeTestDialect();
+void registerTestLLVMTypeTranslation();
 void registerTestRoundtripSPIRV();
 void registerTestRoundtripDebugSPIRV();
 } // namespace mlir
 
 static void registerTestTranslations() {
+  registerTestLLVMTypeTranslation();
   registerTestRoundtripSPIRV();
   registerTestRoundtripDebugSPIRV();
 }
 
 int main(int argc, char **argv) {
   registerAllDialects();
+  registerLLVMTypeTestDialect();
   registerAllTranslations();
   registerTestTranslations();
   llvm::InitLLVM y(argc, argv);

From 6abd7e2e622bc7eabdb673a7815f6673523a1e94 Mon Sep 17 00:00:00 2001
From: Alex Zinenko <zinenko@google.com>
Date: Tue, 4 Aug 2020 11:37:31 +0200
Subject: [PATCH 1028/1035] [mlir] provide same APIs as existing LLVMType in
 the new LLVM type modeling

These are intended to smoothen the transition and may be removed in the future
in favor of more MLIR-compatible APIs. They intentionally have the same
semantics as the existing functions, which must remain stable until the
transition is complete.

Depends On D85019

Reviewed By: nicolasvasilache

Differential Revision: https://reviews.llvm.org/D85020
---
 mlir/include/mlir/Dialect/LLVMIR/LLVMTypes.h | 155 ++++++++++++++
 mlir/lib/Dialect/LLVMIR/IR/LLVMTypes.cpp     | 209 +++++++++++++++++++
 2 files changed, 364 insertions(+)

diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMTypes.h b/mlir/include/mlir/Dialect/LLVMIR/LLVMTypes.h
index 6764f9815c3fb..e409d6880283f 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMTypes.h
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMTypes.h
@@ -26,6 +26,8 @@ class DialectAsmParser;
 class DialectAsmPrinter;
 
 namespace LLVM {
+class LLVMDialect;
+
 namespace detail {
 struct LLVMFunctionTypeStorage;
 struct LLVMIntegerTypeStorage;
@@ -34,6 +36,12 @@ struct LLVMStructTypeStorage;
 struct LLVMTypeAndSizeStorage;
 } // namespace detail
 
+class LLVMBFloatType;
+class LLVMHalfType;
+class LLVMFloatType;
+class LLVMDoubleType;
+class LLVMIntegerType;
+
 //===----------------------------------------------------------------------===//
 // LLVMTypeNew.
 //===----------------------------------------------------------------------===//
@@ -96,6 +104,150 @@ class LLVMTypeNew : public Type {
   static bool kindof(unsigned kind) {
     return FIRST_NEW_LLVM_TYPE <= kind && kind <= LAST_NEW_LLVM_TYPE;
   }
+
+  LLVMDialect &getDialect();
+
+  /// Floating-point type utilities.
+  bool isBFloatTy() { return isa<LLVMBFloatType>(); }
+  bool isHalfTy() { return isa<LLVMHalfType>(); }
+  bool isFloatTy() { return isa<LLVMFloatType>(); }
+  bool isDoubleTy() { return isa<LLVMDoubleType>(); }
+  bool isFloatingPointTy() {
+    return isa<LLVMHalfType>() || isa<LLVMBFloatType>() ||
+           isa<LLVMFloatType>() || isa<LLVMDoubleType>();
+  }
+
+  /// Array type utilities.
+  LLVMTypeNew getArrayElementType();
+  unsigned getArrayNumElements();
+  bool isArrayTy();
+
+  /// Integer type utilities.
+  bool isIntegerTy() { return isa<LLVMIntegerType>(); }
+  bool isIntegerTy(unsigned bitwidth);
+  unsigned getIntegerBitWidth();
+
+  /// Vector type utilities.
+  LLVMTypeNew getVectorElementType();
+  unsigned getVectorNumElements();
+  llvm::ElementCount getVectorElementCount();
+  bool isVectorTy();
+
+  /// Function type utilities.
+  LLVMTypeNew getFunctionParamType(unsigned argIdx);
+  unsigned getFunctionNumParams();
+  LLVMTypeNew getFunctionResultType();
+  bool isFunctionTy();
+  bool isFunctionVarArg();
+
+  /// Pointer type utilities.
+  LLVMTypeNew getPointerTo(unsigned addrSpace = 0);
+  LLVMTypeNew getPointerElementTy();
+  bool isPointerTy();
+  static bool isValidPointerElementType(LLVMTypeNew type);
+
+  /// Struct type utilities.
+  LLVMTypeNew getStructElementType(unsigned i);
+  unsigned getStructNumElements();
+  bool isStructTy();
+
+  /// Utilities used to generate floating point types.
+  static LLVMTypeNew getDoubleTy(LLVMDialect *dialect);
+  static LLVMTypeNew getFloatTy(LLVMDialect *dialect);
+  static LLVMTypeNew getBFloatTy(LLVMDialect *dialect);
+  static LLVMTypeNew getHalfTy(LLVMDialect *dialect);
+  static LLVMTypeNew getFP128Ty(LLVMDialect *dialect);
+  static LLVMTypeNew getX86_FP80Ty(LLVMDialect *dialect);
+
+  /// Utilities used to generate integer types.
+  static LLVMTypeNew getIntNTy(LLVMDialect *dialect, unsigned numBits);
+  static LLVMTypeNew getInt1Ty(LLVMDialect *dialect) {
+    return getIntNTy(dialect, /*numBits=*/1);
+  }
+  static LLVMTypeNew getInt8Ty(LLVMDialect *dialect) {
+    return getIntNTy(dialect, /*numBits=*/8);
+  }
+  static LLVMTypeNew getInt8PtrTy(LLVMDialect *dialect) {
+    return getInt8Ty(dialect).getPointerTo();
+  }
+  static LLVMTypeNew getInt16Ty(LLVMDialect *dialect) {
+    return getIntNTy(dialect, /*numBits=*/16);
+  }
+  static LLVMTypeNew getInt32Ty(LLVMDialect *dialect) {
+    return getIntNTy(dialect, /*numBits=*/32);
+  }
+  static LLVMTypeNew getInt64Ty(LLVMDialect *dialect) {
+    return getIntNTy(dialect, /*numBits=*/64);
+  }
+
+  /// Utilities used to generate other miscellaneous types.
+  static LLVMTypeNew getArrayTy(LLVMTypeNew elementType, uint64_t numElements);
+  static LLVMTypeNew getFunctionTy(LLVMTypeNew result,
+                                   ArrayRef<LLVMTypeNew> params, bool isVarArg);
+  static LLVMTypeNew getFunctionTy(LLVMTypeNew result, bool isVarArg) {
+    return getFunctionTy(result, llvm::None, isVarArg);
+  }
+  static LLVMTypeNew getStructTy(LLVMDialect *dialect,
+                                 ArrayRef<LLVMTypeNew> elements,
+                                 bool isPacked = false);
+  static LLVMTypeNew getStructTy(LLVMDialect *dialect, bool isPacked = false) {
+    return getStructTy(dialect, llvm::None, isPacked);
+  }
+  template <typename... Args>
+  static typename std::enable_if<llvm::are_base_of<LLVMTypeNew, Args...>::value,
+                                 LLVMTypeNew>::type
+  getStructTy(LLVMTypeNew elt1, Args... elts) {
+    SmallVector<LLVMTypeNew, 8> fields({elt1, elts...});
+    return getStructTy(&elt1.getDialect(), fields);
+  }
+  static LLVMTypeNew getVectorTy(LLVMTypeNew elementType, unsigned numElements);
+
+  /// Void type utilities.
+  static LLVMTypeNew getVoidTy(LLVMDialect *dialect);
+  bool isVoidTy();
+
+  // Creation and setting of LLVM's identified struct types
+  static LLVMTypeNew createStructTy(LLVMDialect *dialect,
+                                    ArrayRef<LLVMTypeNew> elements,
+                                    Optional<StringRef> name,
+                                    bool isPacked = false);
+
+  static LLVMTypeNew createStructTy(LLVMDialect *dialect,
+                                    Optional<StringRef> name) {
+    return createStructTy(dialect, llvm::None, name);
+  }
+
+  static LLVMTypeNew createStructTy(ArrayRef<LLVMTypeNew> elements,
+                                    Optional<StringRef> name,
+                                    bool isPacked = false) {
+    assert(!elements.empty() &&
+           "This method may not be invoked with an empty list");
+    LLVMTypeNew ele0 = elements.front();
+    return createStructTy(&ele0.getDialect(), elements, name, isPacked);
+  }
+
+  template <typename... Args>
+  static
+      typename std::enable_if_t<llvm::are_base_of<LLVMTypeNew, Args...>::value,
+                                LLVMTypeNew>
+      createStructTy(StringRef name, LLVMTypeNew elt1, Args... elts) {
+    SmallVector<LLVMTypeNew, 8> fields({elt1, elts...});
+    Optional<StringRef> opt_name(name);
+    return createStructTy(&elt1.getDialect(), fields, opt_name);
+  }
+
+  static LLVMTypeNew setStructTyBody(LLVMTypeNew structType,
+                                     ArrayRef<LLVMTypeNew> elements,
+                                     bool isPacked = false);
+
+  template <typename... Args>
+  static
+      typename std::enable_if_t<llvm::are_base_of<LLVMTypeNew, Args...>::value,
+                                LLVMTypeNew>
+      setStructTyBody(LLVMTypeNew structType, LLVMTypeNew elt1, Args... elts) {
+    SmallVector<LLVMTypeNew, 8> fields({elt1, elts...});
+    return setStructTyBody(structType, fields);
+  }
 };
 
 //===----------------------------------------------------------------------===//
@@ -323,6 +475,9 @@ class LLVMStructType : public Type::TypeBase<LLVMStructType, LLVMTypeNew,
   /// Checks if a struct is opaque.
   bool isOpaque();
 
+  /// Checks if a struct is initialized.
+  bool isInitialized();
+
   /// Returns the name of an identified struct.
   StringRef getName();
 
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMTypes.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMTypes.cpp
index 3540091e90a3e..abecbccb1d4aa 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMTypes.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMTypes.cpp
@@ -13,6 +13,7 @@
 
 #include "TypeDetail.h"
 
+#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/Dialect/LLVMIR/LLVMTypes.h"
 #include "mlir/IR/DialectImplementation.h"
 #include "mlir/IR/TypeSupport.h"
@@ -22,6 +23,213 @@
 using namespace mlir;
 using namespace mlir::LLVM;
 
+//===----------------------------------------------------------------------===//
+// LLVMTypeNew.
+//===----------------------------------------------------------------------===//
+
+// TODO: when these types are registered with the LLVMDialect, this method
+// should be removed and the regular Type::getDialect should just work.
+LLVMDialect &LLVMTypeNew::getDialect() {
+  return *getContext()->getRegisteredDialect<LLVM::LLVMDialect>();
+}
+
+//----------------------------------------------------------------------------//
+// Integer type utilities.
+
+bool LLVMTypeNew::isIntegerTy(unsigned bitwidth) {
+  if (auto intType = dyn_cast<LLVMIntegerType>())
+    return intType.getBitWidth() == bitwidth;
+  return false;
+}
+
+unsigned LLVMTypeNew::getIntegerBitWidth() {
+  return cast<LLVMIntegerType>().getBitWidth();
+}
+
+LLVMTypeNew LLVMTypeNew::getArrayElementType() {
+  return cast<LLVMArrayType>().getElementType();
+}
+
+//----------------------------------------------------------------------------//
+// Array type utilities.
+
+unsigned LLVMTypeNew::getArrayNumElements() {
+  return cast<LLVMArrayType>().getNumElements();
+}
+
+bool LLVMTypeNew::isArrayTy() { return isa<LLVMArrayType>(); }
+
+//----------------------------------------------------------------------------//
+// Vector type utilities.
+
+LLVMTypeNew LLVMTypeNew::getVectorElementType() {
+  return cast<LLVMVectorType>().getElementType();
+}
+
+unsigned LLVMTypeNew::getVectorNumElements() {
+  return cast<LLVMFixedVectorType>().getNumElements();
+}
+llvm::ElementCount LLVMTypeNew::getVectorElementCount() {
+  return cast<LLVMVectorType>().getElementCount();
+}
+
+bool LLVMTypeNew::isVectorTy() { return isa<LLVMVectorType>(); }
+
+//----------------------------------------------------------------------------//
+// Function type utilities.
+
+LLVMTypeNew LLVMTypeNew::getFunctionParamType(unsigned argIdx) {
+  return cast<LLVMFunctionType>().getParamType(argIdx);
+}
+
+unsigned LLVMTypeNew::getFunctionNumParams() {
+  return cast<LLVMFunctionType>().getNumParams();
+}
+
+LLVMTypeNew LLVMTypeNew::getFunctionResultType() {
+  return cast<LLVMFunctionType>().getReturnType();
+}
+
+bool LLVMTypeNew::isFunctionTy() { return isa<LLVMFunctionType>(); }
+
+bool LLVMTypeNew::isFunctionVarArg() {
+  return cast<LLVMFunctionType>().isVarArg();
+}
+
+//----------------------------------------------------------------------------//
+// Pointer type utilities.
+
+LLVMTypeNew LLVMTypeNew::getPointerTo(unsigned addrSpace) {
+  return LLVMPointerType::get(*this, addrSpace);
+}
+
+LLVMTypeNew LLVMTypeNew::getPointerElementTy() {
+  return cast<LLVMPointerType>().getElementType();
+}
+
+bool LLVMTypeNew::isPointerTy() { return isa<LLVMPointerType>(); }
+
+bool LLVMTypeNew::isValidPointerElementType(LLVMTypeNew type) {
+  return !type.isa<LLVMVoidType>() && !type.isa<LLVMTokenType>() &&
+         !type.isa<LLVMMetadataType>() && !type.isa<LLVMLabelType>();
+}
+
+//----------------------------------------------------------------------------//
+// Struct type utilities.
+
+LLVMTypeNew LLVMTypeNew::getStructElementType(unsigned i) {
+  return cast<LLVMStructType>().getBody()[i];
+}
+
+unsigned LLVMTypeNew::getStructNumElements() {
+  return cast<LLVMStructType>().getBody().size();
+}
+
+bool LLVMTypeNew::isStructTy() { return isa<LLVMStructType>(); }
+
+//----------------------------------------------------------------------------//
+// Utilities used to generate floating point types.
+
+LLVMTypeNew LLVMTypeNew::getDoubleTy(LLVMDialect *dialect) {
+  return LLVMDoubleType::get(dialect->getContext());
+}
+
+LLVMTypeNew LLVMTypeNew::getFloatTy(LLVMDialect *dialect) {
+  return LLVMFloatType::get(dialect->getContext());
+}
+
+LLVMTypeNew LLVMTypeNew::getBFloatTy(LLVMDialect *dialect) {
+  return LLVMBFloatType::get(dialect->getContext());
+}
+
+LLVMTypeNew LLVMTypeNew::getHalfTy(LLVMDialect *dialect) {
+  return LLVMHalfType::get(dialect->getContext());
+}
+
+LLVMTypeNew LLVMTypeNew::getFP128Ty(LLVMDialect *dialect) {
+  return LLVMFP128Type::get(dialect->getContext());
+}
+
+LLVMTypeNew LLVMTypeNew::getX86_FP80Ty(LLVMDialect *dialect) {
+  return LLVMX86FP80Type::get(dialect->getContext());
+}
+
+//----------------------------------------------------------------------------//
+// Utilities used to generate integer types.
+
+LLVMTypeNew LLVMTypeNew::getIntNTy(LLVMDialect *dialect, unsigned numBits) {
+  return LLVMIntegerType::get(dialect->getContext(), numBits);
+}
+
+//----------------------------------------------------------------------------//
+// Utilities used to generate other miscellaneous types.
+
+LLVMTypeNew LLVMTypeNew::getArrayTy(LLVMTypeNew elementType,
+                                    uint64_t numElements) {
+  return LLVMArrayType::get(elementType, numElements);
+}
+
+LLVMTypeNew LLVMTypeNew::getFunctionTy(LLVMTypeNew result,
+                                       ArrayRef<LLVMTypeNew> params,
+                                       bool isVarArg) {
+  return LLVMFunctionType::get(result, params, isVarArg);
+}
+
+LLVMTypeNew LLVMTypeNew::getStructTy(LLVMDialect *dialect,
+                                     ArrayRef<LLVMTypeNew> elements,
+                                     bool isPacked) {
+  return LLVMStructType::getLiteral(dialect->getContext(), elements, isPacked);
+}
+
+LLVMTypeNew LLVMTypeNew::getVectorTy(LLVMTypeNew elementType,
+                                     unsigned numElements) {
+  return LLVMFixedVectorType::get(elementType, numElements);
+}
+
+//----------------------------------------------------------------------------//
+// Void type utilities.
+
+LLVMTypeNew LLVMTypeNew::getVoidTy(LLVMDialect *dialect) {
+  return LLVMVoidType::get(dialect->getContext());
+}
+
+bool LLVMTypeNew::isVoidTy() { return isa<LLVMVoidType>(); }
+
+//----------------------------------------------------------------------------//
+// Creation and setting of LLVM's identified struct types
+
+LLVMTypeNew LLVMTypeNew::createStructTy(LLVMDialect *dialect,
+                                        ArrayRef<LLVMTypeNew> elements,
+                                        Optional<StringRef> name,
+                                        bool isPacked) {
+  assert(name.hasValue() &&
+         "identified structs with no identifier not supported");
+  StringRef stringNameBase = name.getValueOr("");
+  std::string stringName = stringNameBase.str();
+  unsigned counter = 0;
+  do {
+    auto type =
+        LLVMStructType::getIdentified(dialect->getContext(), stringName);
+    if (type.isInitialized() || failed(type.setBody(elements, isPacked))) {
+      counter += 1;
+      stringName =
+          (Twine(stringNameBase) + "." + std::to_string(counter)).str();
+      continue;
+    }
+    return type;
+  } while (true);
+}
+
+LLVMTypeNew LLVMTypeNew::setStructTyBody(LLVMTypeNew structType,
+                                         ArrayRef<LLVMTypeNew> elements,
+                                         bool isPacked) {
+  LogicalResult couldSet =
+      structType.cast<LLVMStructType>().setBody(elements, isPacked);
+  assert(succeeded(couldSet) && "failed to set the body");
+  (void)couldSet;
+  return structType;
+}
+
 //===----------------------------------------------------------------------===//
 // Array type.
 
@@ -117,6 +325,7 @@ bool LLVMStructType::isIdentified() { return getImpl()->isIdentified(); }
 bool LLVMStructType::isOpaque() {
   return getImpl()->isOpaque() || !getImpl()->isInitialized();
 }
+bool LLVMStructType::isInitialized() { return getImpl()->isInitialized(); }
 StringRef LLVMStructType::getName() { return getImpl()->getIdentifier(); }
 ArrayRef<LLVMTypeNew> LLVMStructType::getBody() {
   return isIdentified() ? getImpl()->getIdentifiedStructBody()

From 20c71e55aad5bf6008c7f5ed63c90ed98907fa99 Mon Sep 17 00:00:00 2001
From: Sanjay Patel <spatel@rotateright.com>
Date: Mon, 3 Aug 2020 15:31:24 -0400
Subject: [PATCH 1029/1035] [InstSimplify] reduce code for min/max analysis;
 NFC

This should probably be moved up to some common area eventually
when there's another user.
---
 llvm/lib/Analysis/InstructionSimplify.cpp | 53 ++++++++++-------------
 1 file changed, 24 insertions(+), 29 deletions(-)

diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp
index f827f0230a3e4..2119ddcc7649b 100644
--- a/llvm/lib/Analysis/InstructionSimplify.cpp
+++ b/llvm/lib/Analysis/InstructionSimplify.cpp
@@ -5208,6 +5208,16 @@ static Intrinsic::ID getMaxMinOpposite(Intrinsic::ID ID) {
   }
 }
 
+static APInt getMaxMinLimit(Intrinsic::ID ID, unsigned BitWidth) {
+  switch (ID) {
+  case Intrinsic::smax: return APInt::getSignedMaxValue(BitWidth);
+  case Intrinsic::smin: return APInt::getSignedMinValue(BitWidth);
+  case Intrinsic::umax: return APInt::getMaxValue(BitWidth);
+  case Intrinsic::umin: return APInt::getMinValue(BitWidth);
+  default: llvm_unreachable("Unexpected intrinsic");
+  }
+}
+
 static Value *simplifyBinaryIntrinsic(Function *F, Value *Op0, Value *Op1,
                                       const SimplifyQuery &Q) {
   Intrinsic::ID IID = F->getIntrinsicID();
@@ -5238,16 +5248,8 @@ static Value *simplifyBinaryIntrinsic(Function *F, Value *Op0, Value *Op1,
       std::swap(Op0, Op1);
 
     // Assume undef is the limit value.
-    if (isa<UndefValue>(Op1)) {
-      if (IID == Intrinsic::smax)
-        return ConstantInt::get(ReturnType, APInt::getSignedMaxValue(BitWidth));
-      if (IID == Intrinsic::smin)
-        return ConstantInt::get(ReturnType, APInt::getSignedMinValue(BitWidth));
-      if (IID == Intrinsic::umax)
-        return ConstantInt::get(ReturnType, APInt::getMaxValue(BitWidth));
-      if (IID == Intrinsic::umin)
-        return ConstantInt::get(ReturnType, APInt::getMinValue(BitWidth));
-    }
+    if (isa<UndefValue>(Op1))
+      return ConstantInt::get(ReturnType, getMaxMinLimit(IID, BitWidth));
 
     auto hasSpecificOperand = [](IntrinsicInst *II, Value *V) {
       return II->getOperand(0) == V || II->getOperand(1) == V;
@@ -5272,25 +5274,18 @@ static Value *simplifyBinaryIntrinsic(Function *F, Value *Op0, Value *Op1,
     }
 
     const APInt *C;
-    if (!match(Op1, m_APIntAllowUndef(C)))
-      break;
-
-    // Clamp to limit value. For example:
-    // umax(i8 %x, i8 255) --> 255
-    if ((IID == Intrinsic::smax && C->isMaxSignedValue()) ||
-        (IID == Intrinsic::smin && C->isMinSignedValue()) ||
-        (IID == Intrinsic::umax && C->isMaxValue()) ||
-        (IID == Intrinsic::umin && C->isMinValue()))
-      return ConstantInt::get(ReturnType, *C);
-
-    // If the constant op is the opposite of the limit value, the other must be
-    // larger/smaller or equal. For example:
-    // umin(i8 %x, i8 255) --> %x
-    if ((IID == Intrinsic::smax && C->isMinSignedValue()) ||
-        (IID == Intrinsic::smin && C->isMaxSignedValue()) ||
-        (IID == Intrinsic::umax && C->isMinValue()) ||
-        (IID == Intrinsic::umin && C->isMaxValue()))
-      return Op0;
+    if (match(Op1, m_APIntAllowUndef(C))) {
+      // Clamp to limit value. For example:
+      // umax(i8 %x, i8 255) --> 255
+      if (*C == getMaxMinLimit(IID, BitWidth))
+        return ConstantInt::get(ReturnType, *C);
+
+      // If the constant op is the opposite of the limit value, the other must
+      // be larger/smaller or equal. For example:
+      // umin(i8 %x, i8 255) --> %x
+      if (*C == getMaxMinLimit(getMaxMinOpposite(IID), BitWidth))
+        return Op0;
+    }
 
     break;
   }

From 011e15bea3456cf429a801e309c65bef14fa22ad Mon Sep 17 00:00:00 2001
From: Sanjay Patel <spatel@rotateright.com>
Date: Mon, 3 Aug 2020 16:51:05 -0400
Subject: [PATCH 1030/1035] [InstSimplify] add tests for min/max with
 constants; NFC

---
 .../InstSimplify/maxmin_intrinsics.ll         | 206 ++++++++++++++++++
 1 file changed, 206 insertions(+)

diff --git a/llvm/test/Transforms/InstSimplify/maxmin_intrinsics.ll b/llvm/test/Transforms/InstSimplify/maxmin_intrinsics.ll
index 7a31a4dcb9a3b..3aa19e91e0e38 100644
--- a/llvm/test/Transforms/InstSimplify/maxmin_intrinsics.ll
+++ b/llvm/test/Transforms/InstSimplify/maxmin_intrinsics.ll
@@ -572,6 +572,8 @@ define i8 @smin_smax_commute3(i8 %x, i8 %y) {
   ret i8 %m2
 }
 
+; Negative test - mismatched intrinsics.
+
 define i8 @smax_umin(i8 %x, i8 %y) {
 ; CHECK-LABEL: @smax_umin(
 ; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.smax.i8(i8 [[Y:%.*]], i8 [[X:%.*]])
@@ -583,6 +585,8 @@ define i8 @smax_umin(i8 %x, i8 %y) {
   ret i8 %m2
 }
 
+; Negative test - mismatched intrinsics.
+
 define i8 @smax_umax(i8 %x, i8 %y) {
 ; CHECK-LABEL: @smax_umax(
 ; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.smax.i8(i8 [[Y:%.*]], i8 [[X:%.*]])
@@ -594,6 +598,8 @@ define i8 @smax_umax(i8 %x, i8 %y) {
   ret i8 %m2
 }
 
+; Negative test - mismatched intrinsics.
+
 define i8 @umax_smin(i8 %x, i8 %y) {
 ; CHECK-LABEL: @umax_smin(
 ; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.umax.i8(i8 [[Y:%.*]], i8 [[X:%.*]])
@@ -605,6 +611,8 @@ define i8 @umax_smin(i8 %x, i8 %y) {
   ret i8 %m2
 }
 
+; Negative test - mismatched intrinsics.
+
 define i8 @umin_smin(i8 %x, i8 %y) {
 ; CHECK-LABEL: @umin_smin(
 ; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.umin.i8(i8 [[Y:%.*]], i8 [[X:%.*]])
@@ -615,3 +623,201 @@ define i8 @umin_smin(i8 %x, i8 %y) {
   %m2 = call i8 @llvm.smin.i8(i8 %m, i8 %x)
   ret i8 %m2
 }
+
+define i8 @umax_umax_constants(i8 %x) {
+; CHECK-LABEL: @umax_umax_constants(
+; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.umax.i8(i8 [[X:%.*]], i8 9)
+; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.umax.i8(i8 7, i8 [[M]])
+; CHECK-NEXT:    ret i8 [[M2]]
+;
+  %m = call i8 @llvm.umax.i8(i8 %x, i8 9)
+  %m2 = call i8 @llvm.umax.i8(i8 7, i8 %m)
+  ret i8 %m2
+}
+
+define i8 @umax_umax_constants_commute1(i8 %x) {
+; CHECK-LABEL: @umax_umax_constants_commute1(
+; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.umax.i8(i8 -128, i8 [[X:%.*]])
+; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.umax.i8(i8 7, i8 [[M]])
+; CHECK-NEXT:    ret i8 [[M2]]
+;
+  %m = call i8 @llvm.umax.i8(i8 128, i8 %x)
+  %m2 = call i8 @llvm.umax.i8(i8 7, i8 %m)
+  ret i8 %m2
+}
+
+define i8 @umax_umax_constants_commute2(i8 %x) {
+; CHECK-LABEL: @umax_umax_constants_commute2(
+; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.umax.i8(i8 [[X:%.*]], i8 -56)
+; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.umax.i8(i8 [[M]], i8 127)
+; CHECK-NEXT:    ret i8 [[M2]]
+;
+  %m = call i8 @llvm.umax.i8(i8 %x, i8 200)
+  %m2 = call i8 @llvm.umax.i8(i8 %m, i8 127)
+  ret i8 %m2
+}
+
+define <2 x i8> @umax_umax_constants_commute3(<2 x i8> %x) {
+; CHECK-LABEL: @umax_umax_constants_commute3(
+; CHECK-NEXT:    [[M:%.*]] = call <2 x i8> @llvm.umax.v2i8(<2 x i8> <i8 -2, i8 -2>, <2 x i8> [[X:%.*]])
+; CHECK-NEXT:    [[M2:%.*]] = call <2 x i8> @llvm.umax.v2i8(<2 x i8> [[M]], <2 x i8> <i8 -128, i8 -128>)
+; CHECK-NEXT:    ret <2 x i8> [[M2]]
+;
+  %m = call <2 x i8> @llvm.umax.v2i8(<2 x i8> <i8 254, i8 254>, <2 x i8> %x)
+  %m2 = call <2 x i8> @llvm.umax.v2i8(<2 x i8> %m, <2 x i8> <i8 128, i8 128>)
+  ret <2 x i8> %m2
+}
+
+define i8 @umin_umin_constants(i8 %x) {
+; CHECK-LABEL: @umin_umin_constants(
+; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.umin.i8(i8 [[X:%.*]], i8 7)
+; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.umin.i8(i8 9, i8 [[M]])
+; CHECK-NEXT:    ret i8 [[M2]]
+;
+  %m = call i8 @llvm.umin.i8(i8 %x, i8 7)
+  %m2 = call i8 @llvm.umin.i8(i8 9, i8 %m)
+  ret i8 %m2
+}
+
+define i8 @umin_umin_constants_commute1(i8 %x) {
+; CHECK-LABEL: @umin_umin_constants_commute1(
+; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.umin.i8(i8 7, i8 [[X:%.*]])
+; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.umin.i8(i8 -128, i8 [[M]])
+; CHECK-NEXT:    ret i8 [[M2]]
+;
+  %m = call i8 @llvm.umin.i8(i8 7, i8 %x)
+  %m2 = call i8 @llvm.umin.i8(i8 128, i8 %m)
+  ret i8 %m2
+}
+
+define <2 x i8> @umin_umin_constants_commute2(<2 x i8> %x) {
+; CHECK-LABEL: @umin_umin_constants_commute2(
+; CHECK-NEXT:    [[M:%.*]] = call <2 x i8> @llvm.umin.v2i8(<2 x i8> [[X:%.*]], <2 x i8> <i8 127, i8 127>)
+; CHECK-NEXT:    [[M2:%.*]] = call <2 x i8> @llvm.umin.v2i8(<2 x i8> [[M]], <2 x i8> <i8 -56, i8 undef>)
+; CHECK-NEXT:    ret <2 x i8> [[M2]]
+;
+  %m = call <2 x i8> @llvm.umin.v2i8(<2 x i8> %x, <2 x i8> <i8 127, i8 127>)
+  %m2 = call <2 x i8> @llvm.umin.v2i8(<2 x i8> %m, <2 x i8> <i8 200, i8 undef>)
+  ret <2 x i8> %m2
+}
+
+define i8 @umin_umin_constants_commute3(i8 %x) {
+; CHECK-LABEL: @umin_umin_constants_commute3(
+; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.umin.i8(i8 -128, i8 [[X:%.*]])
+; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.umin.i8(i8 [[M]], i8 -2)
+; CHECK-NEXT:    ret i8 [[M2]]
+;
+  %m = call i8 @llvm.umin.i8(i8 128, i8 %x)
+  %m2 = call i8 @llvm.umin.i8(i8 %m, i8 254)
+  ret i8 %m2
+}
+
+define i8 @smax_smax_constants(i8 %x) {
+; CHECK-LABEL: @smax_smax_constants(
+; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.smax.i8(i8 [[X:%.*]], i8 9)
+; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.smax.i8(i8 7, i8 [[M]])
+; CHECK-NEXT:    ret i8 [[M2]]
+;
+  %m = call i8 @llvm.smax.i8(i8 %x, i8 9)
+  %m2 = call i8 @llvm.smax.i8(i8 7, i8 %m)
+  ret i8 %m2
+}
+
+define <2 x i8> @smax_smax_constants_commute1(<2 x i8> %x) {
+; CHECK-LABEL: @smax_smax_constants_commute1(
+; CHECK-NEXT:    [[M:%.*]] = call <2 x i8> @llvm.smax.v2i8(<2 x i8> <i8 7, i8 7>, <2 x i8> [[X:%.*]])
+; CHECK-NEXT:    [[M2:%.*]] = call <2 x i8> @llvm.smax.v2i8(<2 x i8> <i8 -127, i8 -127>, <2 x i8> [[M]])
+; CHECK-NEXT:    ret <2 x i8> [[M2]]
+;
+  %m = call <2 x i8> @llvm.smax.v2i8(<2 x i8> <i8 7, i8 7>, <2 x i8> %x)
+  %m2 = call <2 x i8> @llvm.smax.v2i8(<2 x i8> <i8 -127, i8 -127>, <2 x i8> %m)
+  ret <2 x i8> %m2
+}
+
+define i8 @smax_smax_constants_commute2(i8 %x) {
+; CHECK-LABEL: @smax_smax_constants_commute2(
+; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.smax.i8(i8 [[X:%.*]], i8 0)
+; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.smax.i8(i8 [[M]], i8 -1)
+; CHECK-NEXT:    ret i8 [[M2]]
+;
+  %m = call i8 @llvm.smax.i8(i8 %x, i8 0)
+  %m2 = call i8 @llvm.smax.i8(i8 %m, i8 -1)
+  ret i8 %m2
+}
+
+define i8 @smax_smax_constants_commute3(i8 %x) {
+; CHECK-LABEL: @smax_smax_constants_commute3(
+; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.smax.i8(i8 -1, i8 [[X:%.*]])
+; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.smax.i8(i8 [[M]], i8 -127)
+; CHECK-NEXT:    ret i8 [[M2]]
+;
+  %m = call i8 @llvm.smax.i8(i8 -1, i8 %x)
+  %m2 = call i8 @llvm.smax.i8(i8 %m, i8 -127)
+  ret i8 %m2
+}
+
+define <2 x i8> @smin_smin_constants(<2 x i8> %x) {
+; CHECK-LABEL: @smin_smin_constants(
+; CHECK-NEXT:    [[M:%.*]] = call <2 x i8> @llvm.smin.v2i8(<2 x i8> [[X:%.*]], <2 x i8> <i8 7, i8 7>)
+; CHECK-NEXT:    [[M2:%.*]] = call <2 x i8> @llvm.smin.v2i8(<2 x i8> <i8 undef, i8 9>, <2 x i8> [[M]])
+; CHECK-NEXT:    ret <2 x i8> [[M2]]
+;
+  %m = call <2 x i8> @llvm.smin.v2i8(<2 x i8> %x, <2 x i8> <i8 7, i8 7>)
+  %m2 = call <2 x i8> @llvm.smin.v2i8(<2 x i8> <i8 undef, i8 9>, <2 x i8> %m)
+  ret <2 x i8> %m2
+}
+
+define i8 @smin_smin_constants_commute1(i8 %x) {
+; CHECK-LABEL: @smin_smin_constants_commute1(
+; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.smin.i8(i8 -127, i8 [[X:%.*]])
+; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.smin.i8(i8 7, i8 [[M]])
+; CHECK-NEXT:    ret i8 [[M2]]
+;
+  %m = call i8 @llvm.smin.i8(i8 -127, i8 %x)
+  %m2 = call i8 @llvm.smin.i8(i8 7, i8 %m)
+  ret i8 %m2
+}
+
+define i8 @smin_smin_constants_commute2(i8 %x) {
+; CHECK-LABEL: @smin_smin_constants_commute2(
+; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.smin.i8(i8 [[X:%.*]], i8 -1)
+; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.smin.i8(i8 [[M]], i8 0)
+; CHECK-NEXT:    ret i8 [[M2]]
+;
+  %m = call i8 @llvm.smin.i8(i8 %x, i8 -1)
+  %m2 = call i8 @llvm.smin.i8(i8 %m, i8 0)
+  ret i8 %m2
+}
+
+define i8 @smin_smin_constants_commute3(i8 %x) {
+; CHECK-LABEL: @smin_smin_constants_commute3(
+; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.smin.i8(i8 -127, i8 [[X:%.*]])
+; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.smin.i8(i8 [[M]], i8 -1)
+; CHECK-NEXT:    ret i8 [[M2]]
+;
+  %m = call i8 @llvm.smin.i8(i8 -127, i8 %x)
+  %m2 = call i8 @llvm.smin.i8(i8 %m, i8 -1)
+  ret i8 %m2
+}
+
+define <2 x i8> @umin_umin_constants_partial_undef(<2 x i8> %x) {
+; CHECK-LABEL: @umin_umin_constants_partial_undef(
+; CHECK-NEXT:    [[M:%.*]] = call <2 x i8> @llvm.umin.v2i8(<2 x i8> [[X:%.*]], <2 x i8> <i8 7, i8 undef>)
+; CHECK-NEXT:    [[M2:%.*]] = call <2 x i8> @llvm.umin.v2i8(<2 x i8> <i8 9, i8 9>, <2 x i8> [[M]])
+; CHECK-NEXT:    ret <2 x i8> [[M2]]
+;
+  %m = call <2 x i8> @llvm.umin.v2i8(<2 x i8> %x, <2 x i8> <i8 7, i8 undef> )
+  %m2 = call <2 x i8> @llvm.umin.v2i8(<2 x i8> <i8 9, i8 9>, <2 x i8> %m)
+  ret <2 x i8> %m2
+}
+
+define <2 x i8> @smax_smax_constants_partial_undef(<2 x i8> %x) {
+; CHECK-LABEL: @smax_smax_constants_partial_undef(
+; CHECK-NEXT:    [[M:%.*]] = call <2 x i8> @llvm.smax.v2i8(<2 x i8> [[X:%.*]], <2 x i8> <i8 undef, i8 10>)
+; CHECK-NEXT:    [[M2:%.*]] = call <2 x i8> @llvm.smax.v2i8(<2 x i8> <i8 9, i8 9>, <2 x i8> [[M]])
+; CHECK-NEXT:    ret <2 x i8> [[M2]]
+;
+  %m = call <2 x i8> @llvm.smax.v2i8(<2 x i8> %x, <2 x i8> <i8 undef, i8 10> )
+  %m2 = call <2 x i8> @llvm.smax.v2i8(<2 x i8> <i8 9, i8 9>, <2 x i8> %m)
+  ret <2 x i8> %m2
+}

From ec1f4e7c3b17656658c9cf49c33bc06c4bc747c2 Mon Sep 17 00:00:00 2001
From: Alex Zinenko <zinenko@google.com>
Date: Tue, 4 Aug 2020 11:37:50 +0200
Subject: [PATCH 1031/1035] [mlir] switch the modeling of LLVM types to use the
 new mechanism

A new first-party modeling for LLVM IR types in the LLVM dialect has been
developed in parallel to the existing modeling based on wrapping LLVM `Type *`
instances. It resolves the long-standing problem of modeling identified
structure types, including recursive structures, and enables future removal of
LLVMContext and related locking mechanisms from LLVMDialect.

This commit only switches the modeling by (a) renaming LLVMTypeNew to LLVMType,
(b) removing the old implementaiton of LLVMType, and (c) updating the tests. It
is intentionally minimal. Separate commits will remove the infrastructure built
for the transition and update API uses where appropriate.

Depends On D85020

Reviewed By: rriddle

Differential Revision: https://reviews.llvm.org/D85021
---
 .../include/mlir/Dialect/LLVMIR/LLVMDialect.h | 176 +----
 mlir/include/mlir/Dialect/LLVMIR/LLVMTypes.h  | 253 +++---
 mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td   |   3 +-
 .../mlir/Target/LLVMIR/TypeTranslation.h      |   6 +-
 .../LLVMIR/CPU/test-vector-reductions-fp.mlir |  32 +-
 .../CPU/test-vector-reductions-int.mlir       |  30 +-
 .../Conversion/VectorToLLVM/CMakeLists.txt    |   3 +-
 .../VectorToLLVM/ConvertVectorToLLVM.cpp      |   7 +-
 mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp    | 302 +------
 mlir/lib/Dialect/LLVMIR/IR/LLVMTypeSyntax.cpp |  98 ++-
 mlir/lib/Dialect/LLVMIR/IR/LLVMTypes.cpp      | 150 ++--
 mlir/lib/Dialect/LLVMIR/IR/TypeDetail.h       |  47 +-
 mlir/lib/Target/LLVMIR/ModuleTranslation.cpp  |   5 +-
 mlir/lib/Target/LLVMIR/TypeTranslation.cpp    |  46 +-
 ...ower-launch-func-to-gpu-runtime-calls.mlir |  12 +-
 .../GPUCommon/memory-attrbution.mlir          |  56 +-
 .../GPUToCUDA/lower-nvvm-kernel-to-cubin.mlir |   2 +-
 .../Conversion/GPUToNVVM/gpu-to-nvvm.mlir     |   6 +-
 .../lower-rocdl-kernel-to-hsaco.mlir          |   2 +-
 .../Conversion/GPUToVulkan/invoke-vulkan.mlir |  72 +-
 .../SPIRVToLLVM/arithmetic-ops-to-llvm.mlir   |  26 +-
 .../SPIRVToLLVM/bitwise-ops-to-llvm.mlir      | 100 +--
 .../SPIRVToLLVM/cast-ops-to-llvm.mlir         |  38 +-
 .../SPIRVToLLVM/comparison-ops-to-llvm.mlir   |  48 +-
 .../SPIRVToLLVM/constant-op-to-llvm.mlir      |  12 +-
 .../Conversion/SPIRVToLLVM/func-to-llvm.mlir  |   8 +-
 .../SPIRVToLLVM/glsl-ops-to-llvm.mlir         | 142 ++--
 .../SPIRVToLLVM/logical-ops-to-llvm.mlir      |  12 +-
 .../SPIRVToLLVM/memory-ops-to-llvm.mlir       |  72 +-
 .../SPIRVToLLVM/misc-ops-to-llvm.mlir         |   6 +-
 .../SPIRVToLLVM/shifts-to-llvm.mlir           |  36 +-
 .../SPIRVToLLVM/spirv-types-to-llvm.mlir      |  14 +-
 .../StandardToLLVM/calling-convention.mlir    |  52 +-
 .../convert-dynamic-memref-ops.mlir           | 246 +++---
 .../StandardToLLVM/convert-funcs.mlir         |  28 +-
 .../convert-static-memref-ops.mlir            | 278 +++----
 .../StandardToLLVM/convert-to-llvmir.mlir     | 542 ++++++-------
 .../Conversion/StandardToLLVM/invalid.mlir    |   6 +-
 .../StandardToLLVM/standard-to-llvm.mlir      |  24 +-
 .../vector-reduction-to-llvm.mlir             |  16 +-
 .../VectorToLLVM/vector-to-llvm.mlir          | 736 +++++++++---------
 .../VectorToROCDL/vector-to-rocdl.mlir        |   8 +-
 mlir/test/Dialect/GPU/invalid.mlir            |  22 +-
 .../test/Dialect/GPU/multiple-all-reduce.mlir |   4 +-
 mlir/test/Dialect/GPU/outlining.mlir          |   4 +-
 mlir/test/Dialect/LLVMIR/func.mlir            |  42 +-
 mlir/test/Dialect/LLVMIR/global.mlir          |  26 +-
 mlir/test/Dialect/LLVMIR/invalid.mlir         | 194 +++--
 mlir/test/Dialect/LLVMIR/nvvm.mlir            |  22 +-
 mlir/test/Dialect/LLVMIR/rocdl.mlir           | 164 ++--
 mlir/test/Dialect/LLVMIR/roundtrip.mlir       | 232 +++---
 mlir/test/Dialect/LLVMIR/types.mlir           | 276 +++----
 mlir/test/Dialect/Linalg/llvm.mlir            | 166 ++--
 mlir/test/Target/avx512.mlir                  |  20 +-
 mlir/test/Target/import.ll                    |  66 +-
 mlir/test/Target/llvmir-intrinsics.mlir       | 146 ++--
 mlir/test/Target/llvmir-invalid.mlir          |   8 +-
 mlir/test/Target/llvmir.mlir                  | 386 ++++-----
 mlir/test/Target/nvvmir.mlir                  |  16 +-
 mlir/test/Target/rocdl.mlir                   | 110 +--
 .../Dialect/LLVMIR/LLVMTypeTestDialect.cpp    |  40 +-
 .../lib/Target/TestLLVMTypeTranslation.cpp    |   4 +-
 .../mlir-cpu-runner/bare_ptr_call_conv.mlir   | 102 +--
 mlir/test/mlir-cpu-runner/simple.mlir         |  30 +-
 64 files changed, 2693 insertions(+), 3145 deletions(-)

diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.h b/mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.h
index 52acfbfa8e507..6b771f8e91239 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.h
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.h
@@ -14,6 +14,7 @@
 #ifndef MLIR_DIALECT_LLVMIR_LLVMDIALECT_H_
 #define MLIR_DIALECT_LLVMIR_LLVMDIALECT_H_
 
+#include "mlir/Dialect/LLVMIR/LLVMTypes.h"
 #include "mlir/IR/Dialect.h"
 #include "mlir/IR/Function.h"
 #include "mlir/IR/OpDefinition.h"
@@ -47,187 +48,12 @@ struct LLVMTypeStorage;
 struct LLVMDialectImpl;
 } // namespace detail
 
-class LLVMType;
-
 /// Converts an MLIR LLVM dialect type to LLVM IR type. Note that this function
 /// exists exclusively for the purpose of gradual transition to the first-party
 /// modeling of LLVM types. It should not be used outside MLIR-to-LLVM
 /// translation.
 llvm::Type *convertLLVMType(LLVMType type);
 
-class LLVMType : public mlir::Type::TypeBase<LLVMType, mlir::Type,
-                                             detail::LLVMTypeStorage> {
-public:
-  enum Kind {
-    LLVM_TYPE = FIRST_LLVM_TYPE,
-  };
-
-  using Base::Base;
-
-  static bool kindof(unsigned kind) { return kind == LLVM_TYPE; }
-
-  LLVMDialect &getDialect();
-
-  /// Utilities to identify types.
-  bool isBFloatTy() { return getUnderlyingType()->isBFloatTy(); }
-  bool isHalfTy() { return getUnderlyingType()->isHalfTy(); }
-  bool isFloatTy() { return getUnderlyingType()->isFloatTy(); }
-  bool isDoubleTy() { return getUnderlyingType()->isDoubleTy(); }
-  bool isFloatingPointTy() { return getUnderlyingType()->isFloatingPointTy(); }
-
-  /// Array type utilities.
-  LLVMType getArrayElementType();
-  unsigned getArrayNumElements();
-  bool isArrayTy();
-
-  /// Integer type utilities.
-  unsigned getIntegerBitWidth() {
-    return getUnderlyingType()->getIntegerBitWidth();
-  }
-  bool isIntegerTy() { return getUnderlyingType()->isIntegerTy(); }
-  bool isIntegerTy(unsigned bitwidth) {
-    return getUnderlyingType()->isIntegerTy(bitwidth);
-  }
-
-  /// Vector type utilities.
-  LLVMType getVectorElementType();
-  unsigned getVectorNumElements();
-  llvm::ElementCount getVectorElementCount();
-  bool isVectorTy();
-
-  /// Function type utilities.
-  LLVMType getFunctionParamType(unsigned argIdx);
-  unsigned getFunctionNumParams();
-  LLVMType getFunctionResultType();
-  bool isFunctionTy();
-  bool isFunctionVarArg();
-
-  /// Pointer type utilities.
-  LLVMType getPointerTo(unsigned addrSpace = 0);
-  LLVMType getPointerElementTy();
-  bool isPointerTy();
-  static bool isValidPointerElementType(LLVMType type);
-
-  /// Struct type utilities.
-  LLVMType getStructElementType(unsigned i);
-  unsigned getStructNumElements();
-  bool isStructTy();
-
-  /// Utilities used to generate floating point types.
-  static LLVMType getDoubleTy(LLVMDialect *dialect);
-  static LLVMType getFloatTy(LLVMDialect *dialect);
-  static LLVMType getBFloatTy(LLVMDialect *dialect);
-  static LLVMType getHalfTy(LLVMDialect *dialect);
-  static LLVMType getFP128Ty(LLVMDialect *dialect);
-  static LLVMType getX86_FP80Ty(LLVMDialect *dialect);
-
-  /// Utilities used to generate integer types.
-  static LLVMType getIntNTy(LLVMDialect *dialect, unsigned numBits);
-  static LLVMType getInt1Ty(LLVMDialect *dialect) {
-    return getIntNTy(dialect, /*numBits=*/1);
-  }
-  static LLVMType getInt8Ty(LLVMDialect *dialect) {
-    return getIntNTy(dialect, /*numBits=*/8);
-  }
-  static LLVMType getInt8PtrTy(LLVMDialect *dialect) {
-    return getInt8Ty(dialect).getPointerTo();
-  }
-  static LLVMType getInt16Ty(LLVMDialect *dialect) {
-    return getIntNTy(dialect, /*numBits=*/16);
-  }
-  static LLVMType getInt32Ty(LLVMDialect *dialect) {
-    return getIntNTy(dialect, /*numBits=*/32);
-  }
-  static LLVMType getInt64Ty(LLVMDialect *dialect) {
-    return getIntNTy(dialect, /*numBits=*/64);
-  }
-
-  /// Utilities used to generate other miscellaneous types.
-  static LLVMType getArrayTy(LLVMType elementType, uint64_t numElements);
-  static LLVMType getFunctionTy(LLVMType result, ArrayRef<LLVMType> params,
-                                bool isVarArg);
-  static LLVMType getFunctionTy(LLVMType result, bool isVarArg) {
-    return getFunctionTy(result, llvm::None, isVarArg);
-  }
-  static LLVMType getStructTy(LLVMDialect *dialect, ArrayRef<LLVMType> elements,
-                              bool isPacked = false);
-  static LLVMType getStructTy(LLVMDialect *dialect, bool isPacked = false) {
-    return getStructTy(dialect, llvm::None, isPacked);
-  }
-  template <typename... Args>
-  static typename std::enable_if<llvm::are_base_of<LLVMType, Args...>::value,
-                                 LLVMType>::type
-  getStructTy(LLVMType elt1, Args... elts) {
-    SmallVector<LLVMType, 8> fields({elt1, elts...});
-    return getStructTy(&elt1.getDialect(), fields);
-  }
-  static LLVMType getVectorTy(LLVMType elementType, unsigned numElements);
-
-  /// Void type utilities.
-  static LLVMType getVoidTy(LLVMDialect *dialect);
-  bool isVoidTy();
-
-  // Creation and setting of LLVM's identified struct types
-  static LLVMType createStructTy(LLVMDialect *dialect,
-                                 ArrayRef<LLVMType> elements,
-                                 Optional<StringRef> name,
-                                 bool isPacked = false);
-
-  static LLVMType createStructTy(LLVMDialect *dialect,
-                                 Optional<StringRef> name) {
-    return createStructTy(dialect, llvm::None, name);
-  }
-
-  static LLVMType createStructTy(ArrayRef<LLVMType> elements,
-                                 Optional<StringRef> name,
-                                 bool isPacked = false) {
-    assert(!elements.empty() &&
-           "This method may not be invoked with an empty list");
-    LLVMType ele0 = elements.front();
-    return createStructTy(&ele0.getDialect(), elements, name, isPacked);
-  }
-
-  template <typename... Args>
-  static typename std::enable_if_t<llvm::are_base_of<LLVMType, Args...>::value,
-                                   LLVMType>
-  createStructTy(StringRef name, LLVMType elt1, Args... elts) {
-    SmallVector<LLVMType, 8> fields({elt1, elts...});
-    Optional<StringRef> opt_name(name);
-    return createStructTy(&elt1.getDialect(), fields, opt_name);
-  }
-
-  static LLVMType setStructTyBody(LLVMType structType,
-                                  ArrayRef<LLVMType> elements,
-                                  bool isPacked = false);
-
-  template <typename... Args>
-  static typename std::enable_if_t<llvm::are_base_of<LLVMType, Args...>::value,
-                                   LLVMType>
-  setStructTyBody(LLVMType structType, LLVMType elt1, Args... elts) {
-    SmallVector<LLVMType, 8> fields({elt1, elts...});
-    return setStructTyBody(structType, fields);
-  }
-
-private:
-  friend LLVMDialect;
-  friend llvm::Type *convertLLVMType(LLVMType type);
-
-  /// Get the underlying LLVM IR type.
-  llvm::Type *getUnderlyingType() const;
-
-  /// Get the underlying LLVM IR types for the given array of types.
-  static void getUnderlyingTypes(ArrayRef<LLVMType> types,
-                                 SmallVectorImpl<llvm::Type *> &result);
-
-  /// Get an LLVMType with a pre-existing llvm type.
-  static LLVMType get(MLIRContext *context, llvm::Type *llvmType);
-
-  /// Get an LLVMType with an llvm type that may cause changes to the underlying
-  /// llvm context when constructed.
-  static LLVMType getLocked(LLVMDialect *dialect,
-                            function_ref<llvm::Type *()> typeBuilder);
-};
-
 ///// Ops /////
 #define GET_OP_CLASSES
 #include "mlir/Dialect/LLVMIR/LLVMOps.h.inc"
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMTypes.h b/mlir/include/mlir/Dialect/LLVMIR/LLVMTypes.h
index e409d6880283f..7d7839c166f78 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMTypes.h
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMTypes.h
@@ -43,7 +43,7 @@ class LLVMDoubleType;
 class LLVMIntegerType;
 
 //===----------------------------------------------------------------------===//
-// LLVMTypeNew.
+// LLVMType.
 //===----------------------------------------------------------------------===//
 
 /// Base class for LLVM dialect types.
@@ -61,7 +61,7 @@ class LLVMIntegerType;
 /// Similarly to other MLIR types, LLVM dialect types are owned by the MLIR
 /// context, have an immutable identifier (for most types except identified
 /// structs, the entire type is the identifier) and are thread-safe.
-class LLVMTypeNew : public Type {
+class LLVMType : public Type::TypeBase<LLVMType, Type, TypeStorage> {
 public:
   enum Kind {
     // Keep non-parametric types contiguous in the enum.
@@ -92,12 +92,12 @@ class LLVMTypeNew : public Type {
   };
 
   /// Inherit base constructors.
-  using Type::Type;
+  using Base::Base;
 
   /// Support for PointerLikeTypeTraits.
   using Type::getAsOpaquePointer;
-  static LLVMTypeNew getFromOpaquePointer(const void *ptr) {
-    return LLVMTypeNew(static_cast<ImplType *>(const_cast<void *>(ptr)));
+  static LLVMType getFromOpaquePointer(const void *ptr) {
+    return LLVMType(static_cast<ImplType *>(const_cast<void *>(ptr)));
   }
 
   /// Support for isa/cast.
@@ -118,7 +118,7 @@ class LLVMTypeNew : public Type {
   }
 
   /// Array type utilities.
-  LLVMTypeNew getArrayElementType();
+  LLVMType getArrayElementType();
   unsigned getArrayNumElements();
   bool isArrayTy();
 
@@ -128,124 +128,121 @@ class LLVMTypeNew : public Type {
   unsigned getIntegerBitWidth();
 
   /// Vector type utilities.
-  LLVMTypeNew getVectorElementType();
+  LLVMType getVectorElementType();
   unsigned getVectorNumElements();
   llvm::ElementCount getVectorElementCount();
   bool isVectorTy();
 
   /// Function type utilities.
-  LLVMTypeNew getFunctionParamType(unsigned argIdx);
+  LLVMType getFunctionParamType(unsigned argIdx);
   unsigned getFunctionNumParams();
-  LLVMTypeNew getFunctionResultType();
+  LLVMType getFunctionResultType();
   bool isFunctionTy();
   bool isFunctionVarArg();
 
   /// Pointer type utilities.
-  LLVMTypeNew getPointerTo(unsigned addrSpace = 0);
-  LLVMTypeNew getPointerElementTy();
+  LLVMType getPointerTo(unsigned addrSpace = 0);
+  LLVMType getPointerElementTy();
   bool isPointerTy();
-  static bool isValidPointerElementType(LLVMTypeNew type);
+  static bool isValidPointerElementType(LLVMType type);
 
   /// Struct type utilities.
-  LLVMTypeNew getStructElementType(unsigned i);
+  LLVMType getStructElementType(unsigned i);
   unsigned getStructNumElements();
   bool isStructTy();
 
   /// Utilities used to generate floating point types.
-  static LLVMTypeNew getDoubleTy(LLVMDialect *dialect);
-  static LLVMTypeNew getFloatTy(LLVMDialect *dialect);
-  static LLVMTypeNew getBFloatTy(LLVMDialect *dialect);
-  static LLVMTypeNew getHalfTy(LLVMDialect *dialect);
-  static LLVMTypeNew getFP128Ty(LLVMDialect *dialect);
-  static LLVMTypeNew getX86_FP80Ty(LLVMDialect *dialect);
+  static LLVMType getDoubleTy(LLVMDialect *dialect);
+  static LLVMType getFloatTy(LLVMDialect *dialect);
+  static LLVMType getBFloatTy(LLVMDialect *dialect);
+  static LLVMType getHalfTy(LLVMDialect *dialect);
+  static LLVMType getFP128Ty(LLVMDialect *dialect);
+  static LLVMType getX86_FP80Ty(LLVMDialect *dialect);
 
   /// Utilities used to generate integer types.
-  static LLVMTypeNew getIntNTy(LLVMDialect *dialect, unsigned numBits);
-  static LLVMTypeNew getInt1Ty(LLVMDialect *dialect) {
+  static LLVMType getIntNTy(LLVMDialect *dialect, unsigned numBits);
+  static LLVMType getInt1Ty(LLVMDialect *dialect) {
     return getIntNTy(dialect, /*numBits=*/1);
   }
-  static LLVMTypeNew getInt8Ty(LLVMDialect *dialect) {
+  static LLVMType getInt8Ty(LLVMDialect *dialect) {
     return getIntNTy(dialect, /*numBits=*/8);
   }
-  static LLVMTypeNew getInt8PtrTy(LLVMDialect *dialect) {
+  static LLVMType getInt8PtrTy(LLVMDialect *dialect) {
     return getInt8Ty(dialect).getPointerTo();
   }
-  static LLVMTypeNew getInt16Ty(LLVMDialect *dialect) {
+  static LLVMType getInt16Ty(LLVMDialect *dialect) {
     return getIntNTy(dialect, /*numBits=*/16);
   }
-  static LLVMTypeNew getInt32Ty(LLVMDialect *dialect) {
+  static LLVMType getInt32Ty(LLVMDialect *dialect) {
     return getIntNTy(dialect, /*numBits=*/32);
   }
-  static LLVMTypeNew getInt64Ty(LLVMDialect *dialect) {
+  static LLVMType getInt64Ty(LLVMDialect *dialect) {
     return getIntNTy(dialect, /*numBits=*/64);
   }
 
   /// Utilities used to generate other miscellaneous types.
-  static LLVMTypeNew getArrayTy(LLVMTypeNew elementType, uint64_t numElements);
-  static LLVMTypeNew getFunctionTy(LLVMTypeNew result,
-                                   ArrayRef<LLVMTypeNew> params, bool isVarArg);
-  static LLVMTypeNew getFunctionTy(LLVMTypeNew result, bool isVarArg) {
+  static LLVMType getArrayTy(LLVMType elementType, uint64_t numElements);
+  static LLVMType getFunctionTy(LLVMType result, ArrayRef<LLVMType> params,
+                                bool isVarArg);
+  static LLVMType getFunctionTy(LLVMType result, bool isVarArg) {
     return getFunctionTy(result, llvm::None, isVarArg);
   }
-  static LLVMTypeNew getStructTy(LLVMDialect *dialect,
-                                 ArrayRef<LLVMTypeNew> elements,
-                                 bool isPacked = false);
-  static LLVMTypeNew getStructTy(LLVMDialect *dialect, bool isPacked = false) {
+  static LLVMType getStructTy(LLVMDialect *dialect, ArrayRef<LLVMType> elements,
+                              bool isPacked = false);
+  static LLVMType getStructTy(LLVMDialect *dialect, bool isPacked = false) {
     return getStructTy(dialect, llvm::None, isPacked);
   }
   template <typename... Args>
-  static typename std::enable_if<llvm::are_base_of<LLVMTypeNew, Args...>::value,
-                                 LLVMTypeNew>::type
-  getStructTy(LLVMTypeNew elt1, Args... elts) {
-    SmallVector<LLVMTypeNew, 8> fields({elt1, elts...});
+  static typename std::enable_if<llvm::are_base_of<LLVMType, Args...>::value,
+                                 LLVMType>::type
+  getStructTy(LLVMType elt1, Args... elts) {
+    SmallVector<LLVMType, 8> fields({elt1, elts...});
     return getStructTy(&elt1.getDialect(), fields);
   }
-  static LLVMTypeNew getVectorTy(LLVMTypeNew elementType, unsigned numElements);
+  static LLVMType getVectorTy(LLVMType elementType, unsigned numElements);
 
   /// Void type utilities.
-  static LLVMTypeNew getVoidTy(LLVMDialect *dialect);
+  static LLVMType getVoidTy(LLVMDialect *dialect);
   bool isVoidTy();
 
   // Creation and setting of LLVM's identified struct types
-  static LLVMTypeNew createStructTy(LLVMDialect *dialect,
-                                    ArrayRef<LLVMTypeNew> elements,
-                                    Optional<StringRef> name,
-                                    bool isPacked = false);
+  static LLVMType createStructTy(LLVMDialect *dialect,
+                                 ArrayRef<LLVMType> elements,
+                                 Optional<StringRef> name,
+                                 bool isPacked = false);
 
-  static LLVMTypeNew createStructTy(LLVMDialect *dialect,
-                                    Optional<StringRef> name) {
+  static LLVMType createStructTy(LLVMDialect *dialect,
+                                 Optional<StringRef> name) {
     return createStructTy(dialect, llvm::None, name);
   }
 
-  static LLVMTypeNew createStructTy(ArrayRef<LLVMTypeNew> elements,
-                                    Optional<StringRef> name,
-                                    bool isPacked = false) {
+  static LLVMType createStructTy(ArrayRef<LLVMType> elements,
+                                 Optional<StringRef> name,
+                                 bool isPacked = false) {
     assert(!elements.empty() &&
            "This method may not be invoked with an empty list");
-    LLVMTypeNew ele0 = elements.front();
+    LLVMType ele0 = elements.front();
     return createStructTy(&ele0.getDialect(), elements, name, isPacked);
   }
 
   template <typename... Args>
-  static
-      typename std::enable_if_t<llvm::are_base_of<LLVMTypeNew, Args...>::value,
-                                LLVMTypeNew>
-      createStructTy(StringRef name, LLVMTypeNew elt1, Args... elts) {
-    SmallVector<LLVMTypeNew, 8> fields({elt1, elts...});
+  static typename std::enable_if_t<llvm::are_base_of<LLVMType, Args...>::value,
+                                   LLVMType>
+  createStructTy(StringRef name, LLVMType elt1, Args... elts) {
+    SmallVector<LLVMType, 8> fields({elt1, elts...});
     Optional<StringRef> opt_name(name);
     return createStructTy(&elt1.getDialect(), fields, opt_name);
   }
 
-  static LLVMTypeNew setStructTyBody(LLVMTypeNew structType,
-                                     ArrayRef<LLVMTypeNew> elements,
-                                     bool isPacked = false);
+  static LLVMType setStructTyBody(LLVMType structType,
+                                  ArrayRef<LLVMType> elements,
+                                  bool isPacked = false);
 
   template <typename... Args>
-  static
-      typename std::enable_if_t<llvm::are_base_of<LLVMTypeNew, Args...>::value,
-                                LLVMTypeNew>
-      setStructTyBody(LLVMTypeNew structType, LLVMTypeNew elt1, Args... elts) {
-    SmallVector<LLVMTypeNew, 8> fields({elt1, elts...});
+  static typename std::enable_if_t<llvm::are_base_of<LLVMType, Args...>::value,
+                                   LLVMType>
+  setStructTyBody(LLVMType structType, LLVMType elt1, Args... elts) {
+    SmallVector<LLVMType, 8> fields({elt1, elts...});
     return setStructTyBody(structType, fields);
   }
 };
@@ -256,8 +253,7 @@ class LLVMTypeNew : public Type {
 
 // Batch-define trivial types.
 #define DEFINE_TRIVIAL_LLVM_TYPE(ClassName, Kind)                              \
-  class ClassName                                                              \
-      : public Type::TypeBase<ClassName, LLVMTypeNew, TypeStorage> {           \
+  class ClassName : public Type::TypeBase<ClassName, LLVMType, TypeStorage> {  \
   public:                                                                      \
     using Base::Base;                                                          \
     static bool kindof(unsigned kind) { return kind == Kind; }                 \
@@ -266,18 +262,18 @@ class LLVMTypeNew : public Type {
     }                                                                          \
   }
 
-DEFINE_TRIVIAL_LLVM_TYPE(LLVMVoidType, LLVMTypeNew::VoidType);
-DEFINE_TRIVIAL_LLVM_TYPE(LLVMHalfType, LLVMTypeNew::HalfType);
-DEFINE_TRIVIAL_LLVM_TYPE(LLVMBFloatType, LLVMTypeNew::BFloatType);
-DEFINE_TRIVIAL_LLVM_TYPE(LLVMFloatType, LLVMTypeNew::FloatType);
-DEFINE_TRIVIAL_LLVM_TYPE(LLVMDoubleType, LLVMTypeNew::DoubleType);
-DEFINE_TRIVIAL_LLVM_TYPE(LLVMFP128Type, LLVMTypeNew::FP128Type);
-DEFINE_TRIVIAL_LLVM_TYPE(LLVMX86FP80Type, LLVMTypeNew::X86FP80Type);
-DEFINE_TRIVIAL_LLVM_TYPE(LLVMPPCFP128Type, LLVMTypeNew::PPCFP128Type);
-DEFINE_TRIVIAL_LLVM_TYPE(LLVMX86MMXType, LLVMTypeNew::X86MMXType);
-DEFINE_TRIVIAL_LLVM_TYPE(LLVMTokenType, LLVMTypeNew::TokenType);
-DEFINE_TRIVIAL_LLVM_TYPE(LLVMLabelType, LLVMTypeNew::LabelType);
-DEFINE_TRIVIAL_LLVM_TYPE(LLVMMetadataType, LLVMTypeNew::MetadataType);
+DEFINE_TRIVIAL_LLVM_TYPE(LLVMVoidType, LLVMType::VoidType);
+DEFINE_TRIVIAL_LLVM_TYPE(LLVMHalfType, LLVMType::HalfType);
+DEFINE_TRIVIAL_LLVM_TYPE(LLVMBFloatType, LLVMType::BFloatType);
+DEFINE_TRIVIAL_LLVM_TYPE(LLVMFloatType, LLVMType::FloatType);
+DEFINE_TRIVIAL_LLVM_TYPE(LLVMDoubleType, LLVMType::DoubleType);
+DEFINE_TRIVIAL_LLVM_TYPE(LLVMFP128Type, LLVMType::FP128Type);
+DEFINE_TRIVIAL_LLVM_TYPE(LLVMX86FP80Type, LLVMType::X86FP80Type);
+DEFINE_TRIVIAL_LLVM_TYPE(LLVMPPCFP128Type, LLVMType::PPCFP128Type);
+DEFINE_TRIVIAL_LLVM_TYPE(LLVMX86MMXType, LLVMType::X86MMXType);
+DEFINE_TRIVIAL_LLVM_TYPE(LLVMTokenType, LLVMType::TokenType);
+DEFINE_TRIVIAL_LLVM_TYPE(LLVMLabelType, LLVMType::LabelType);
+DEFINE_TRIVIAL_LLVM_TYPE(LLVMMetadataType, LLVMType::MetadataType);
 
 #undef DEFINE_TRIVIAL_LLVM_TYPE
 
@@ -288,21 +284,21 @@ DEFINE_TRIVIAL_LLVM_TYPE(LLVMMetadataType, LLVMTypeNew::MetadataType);
 /// LLVM dialect array type. It is an aggregate type representing consecutive
 /// elements in memory, parameterized by the number of elements and the element
 /// type.
-class LLVMArrayType : public Type::TypeBase<LLVMArrayType, LLVMTypeNew,
+class LLVMArrayType : public Type::TypeBase<LLVMArrayType, LLVMType,
                                             detail::LLVMTypeAndSizeStorage> {
 public:
   /// Inherit base constructors.
   using Base::Base;
 
   /// Support for isa/cast.
-  static bool kindof(unsigned kind) { return kind == LLVMTypeNew::ArrayType; }
+  static bool kindof(unsigned kind) { return kind == LLVMType::ArrayType; }
 
   /// Gets or creates an instance of LLVM dialect array type containing
   /// `numElements` of `elementType`, in the same context as `elementType`.
-  static LLVMArrayType get(LLVMTypeNew elementType, unsigned numElements);
+  static LLVMArrayType get(LLVMType elementType, unsigned numElements);
 
   /// Returns the element type of the array.
-  LLVMTypeNew getElementType();
+  LLVMType getElementType();
 
   /// Returns the number of elements in the array type.
   unsigned getNumElements();
@@ -316,38 +312,35 @@ class LLVMArrayType : public Type::TypeBase<LLVMArrayType, LLVMTypeNew,
 /// which can have multiple), a list of parameter types and can optionally be
 /// variadic.
 class LLVMFunctionType
-    : public Type::TypeBase<LLVMFunctionType, LLVMTypeNew,
+    : public Type::TypeBase<LLVMFunctionType, LLVMType,
                             detail::LLVMFunctionTypeStorage> {
 public:
   /// Inherit base constructors.
   using Base::Base;
 
   /// Support for isa/cast.
-  static bool kindof(unsigned kind) {
-    return kind == LLVMTypeNew::FunctionType;
-  }
+  static bool kindof(unsigned kind) { return kind == LLVMType::FunctionType; }
 
   /// Gets or creates an instance of LLVM dialect function in the same context
   /// as the `result` type.
-  static LLVMFunctionType get(LLVMTypeNew result,
-                              ArrayRef<LLVMTypeNew> arguments,
+  static LLVMFunctionType get(LLVMType result, ArrayRef<LLVMType> arguments,
                               bool isVarArg = false);
 
   /// Returns the result type of the function.
-  LLVMTypeNew getReturnType();
+  LLVMType getReturnType();
 
   /// Returns the number of arguments to the function.
   unsigned getNumParams();
 
   /// Returns `i`-th argument of the function. Asserts on out-of-bounds.
-  LLVMTypeNew getParamType(unsigned i);
+  LLVMType getParamType(unsigned i);
 
   /// Returns whether the function is variadic.
   bool isVarArg();
 
   /// Returns a list of argument types of the function.
-  ArrayRef<LLVMTypeNew> getParams();
-  ArrayRef<LLVMTypeNew> params() { return getParams(); }
+  ArrayRef<LLVMType> getParams();
+  ArrayRef<LLVMType> params() { return getParams(); }
 };
 
 //===----------------------------------------------------------------------===//
@@ -355,14 +348,14 @@ class LLVMFunctionType
 //===----------------------------------------------------------------------===//
 
 /// LLVM dialect signless integer type parameterized by bitwidth.
-class LLVMIntegerType : public Type::TypeBase<LLVMIntegerType, LLVMTypeNew,
+class LLVMIntegerType : public Type::TypeBase<LLVMIntegerType, LLVMType,
                                               detail::LLVMIntegerTypeStorage> {
 public:
   /// Inherit base constructor.
   using Base::Base;
 
   /// Support for isa/cast.
-  static bool kindof(unsigned kind) { return kind == LLVMTypeNew::IntegerType; }
+  static bool kindof(unsigned kind) { return kind == LLVMType::IntegerType; }
 
   /// Gets or creates an instance of the integer of the specified `bitwidth` in
   /// the given context.
@@ -379,22 +372,22 @@ class LLVMIntegerType : public Type::TypeBase<LLVMIntegerType, LLVMTypeNew,
 /// LLVM dialect pointer type. This type typically represents a reference to an
 /// object in memory. It is parameterized by the element type and the address
 /// space.
-class LLVMPointerType : public Type::TypeBase<LLVMPointerType, LLVMTypeNew,
+class LLVMPointerType : public Type::TypeBase<LLVMPointerType, LLVMType,
                                               detail::LLVMPointerTypeStorage> {
 public:
   /// Inherit base constructors.
   using Base::Base;
 
   /// Support for isa/cast.
-  static bool kindof(unsigned kind) { return kind == LLVMTypeNew::PointerType; }
+  static bool kindof(unsigned kind) { return kind == LLVMType::PointerType; }
 
   /// Gets or creates an instance of LLVM dialect pointer type pointing to an
   /// object of `pointee` type in the given address space. The pointer type is
   /// created in the same context as `pointee`.
-  static LLVMPointerType get(LLVMTypeNew pointee, unsigned addressSpace = 0);
+  static LLVMPointerType get(LLVMType pointee, unsigned addressSpace = 0);
 
   /// Returns the pointed-to type.
-  LLVMTypeNew getElementType();
+  LLVMType getElementType();
 
   /// Returns the address space of the pointer.
   unsigned getAddressSpace();
@@ -428,14 +421,14 @@ class LLVMPointerType : public Type::TypeBase<LLVMPointerType, LLVMTypeNew,
 ///
 /// Note that the packedness of the struct takes place in uniquing of literal
 /// structs, but does not in uniquing of identified structs.
-class LLVMStructType : public Type::TypeBase<LLVMStructType, LLVMTypeNew,
+class LLVMStructType : public Type::TypeBase<LLVMStructType, LLVMType,
                                              detail::LLVMStructTypeStorage> {
 public:
   /// Inherit base construtors.
   using Base::Base;
 
   /// Support for isa/cast.
-  static bool kindof(unsigned kind) { return kind == LLVMTypeNew::StructType; }
+  static bool kindof(unsigned kind) { return kind == LLVMType::StructType; }
 
   /// Gets or creates an identified struct with the given name in the provided
   /// context. Note that unlike llvm::StructType::create, this function will
@@ -447,7 +440,7 @@ class LLVMStructType : public Type::TypeBase<LLVMStructType, LLVMTypeNew,
   /// Gets or creates a literal struct with the given body in the provided
   /// context.
   static LLVMStructType getLiteral(MLIRContext *context,
-                                   ArrayRef<LLVMTypeNew> types,
+                                   ArrayRef<LLVMType> types,
                                    bool isPacked = false);
 
   /// Gets or creates an intentionally-opaque identified struct. Such a struct
@@ -464,7 +457,7 @@ class LLVMStructType : public Type::TypeBase<LLVMStructType, LLVMTypeNew,
   /// different thread modified the struct after it was created. Most callers
   /// are likely to assert this always succeeds, but it is possible to implement
   /// a local renaming scheme based on the result of this call.
-  LogicalResult setBody(ArrayRef<LLVMTypeNew> types, bool isPacked);
+  LogicalResult setBody(ArrayRef<LLVMType> types, bool isPacked);
 
   /// Checks if a struct is packed.
   bool isPacked();
@@ -482,7 +475,7 @@ class LLVMStructType : public Type::TypeBase<LLVMStructType, LLVMTypeNew,
   StringRef getName();
 
   /// Returns the list of element types contained in a non-opaque struct.
-  ArrayRef<LLVMTypeNew> getBody();
+  ArrayRef<LLVMType> getBody();
 };
 
 //===----------------------------------------------------------------------===//
@@ -492,19 +485,20 @@ class LLVMStructType : public Type::TypeBase<LLVMStructType, LLVMTypeNew,
 /// LLVM dialect vector type, represents a sequence of elements that can be
 /// processed as one, typically in SIMD context. This is a base class for fixed
 /// and scalable vectors.
-class LLVMVectorType : public LLVMTypeNew {
+class LLVMVectorType : public Type::TypeBase<LLVMVectorType, LLVMType,
+                                             detail::LLVMTypeAndSizeStorage> {
 public:
   /// Inherit base constructor.
-  using LLVMTypeNew::LLVMTypeNew;
+  using Base::Base;
 
   /// Support for isa/cast.
   static bool kindof(unsigned kind) {
-    return kind == LLVMTypeNew::FixedVectorType ||
-           kind == LLVMTypeNew::ScalableVectorType;
+    return kind == LLVMType::FixedVectorType ||
+           kind == LLVMType::ScalableVectorType;
   }
 
   /// Returns the element type of the vector.
-  LLVMTypeNew getElementType();
+  LLVMType getElementType();
 
   /// Returns the number of elements in the vector.
   llvm::ElementCount getElementCount();
@@ -525,12 +519,12 @@ class LLVMFixedVectorType
 
   /// Support for isa/cast.
   static bool kindof(unsigned kind) {
-    return kind == LLVMTypeNew::FixedVectorType;
+    return kind == LLVMType::FixedVectorType;
   }
 
   /// Gets or creates a fixed vector type containing `numElements` of
   /// `elementType` in the same context as `elementType`.
-  static LLVMFixedVectorType get(LLVMTypeNew elementType, unsigned numElements);
+  static LLVMFixedVectorType get(LLVMType elementType, unsigned numElements);
 
   /// Returns the number of elements in the fixed vector.
   unsigned getNumElements();
@@ -552,12 +546,12 @@ class LLVMScalableVectorType
 
   /// Support for isa/cast.
   static bool kindof(unsigned kind) {
-    return kind == LLVMTypeNew::ScalableVectorType;
+    return kind == LLVMType::ScalableVectorType;
   }
 
   /// Gets or creates a scalable vector type containing a non-zero multiple of
   /// `minNumElements` of `elementType` in the same context as `elementType`.
-  static LLVMScalableVectorType get(LLVMTypeNew elementType,
+  static LLVMScalableVectorType get(LLVMType elementType,
                                     unsigned minNumElements);
 
   /// Returns the scaling factor of the number of elements in the vector. The
@@ -572,10 +566,10 @@ class LLVMScalableVectorType
 
 namespace detail {
 /// Parses an LLVM dialect type.
-LLVMTypeNew parseType(DialectAsmParser &parser);
+LLVMType parseType(DialectAsmParser &parser);
 
 /// Prints an LLVM Dialect type.
-void printType(LLVMTypeNew type, DialectAsmPrinter &printer);
+void printType(LLVMType type, DialectAsmPrinter &printer);
 } // namespace detail
 
 } // namespace LLVM
@@ -587,34 +581,35 @@ void printType(LLVMTypeNew type, DialectAsmPrinter &printer);
 
 namespace llvm {
 
-// LLVMTypeNew instances hash just like pointers.
-template <> struct DenseMapInfo<mlir::LLVM::LLVMTypeNew> {
-  static mlir::LLVM::LLVMTypeNew getEmptyKey() {
+// LLVMType instances hash just like pointers.
+template <>
+struct DenseMapInfo<mlir::LLVM::LLVMType> {
+  static mlir::LLVM::LLVMType getEmptyKey() {
     void *pointer = llvm::DenseMapInfo<void *>::getEmptyKey();
-    return mlir::LLVM::LLVMTypeNew(
-        static_cast<mlir::LLVM::LLVMTypeNew::ImplType *>(pointer));
+    return mlir::LLVM::LLVMType(
+        static_cast<mlir::LLVM::LLVMType::ImplType *>(pointer));
   }
-  static mlir::LLVM::LLVMTypeNew getTombstoneKey() {
+  static mlir::LLVM::LLVMType getTombstoneKey() {
     void *pointer = llvm::DenseMapInfo<void *>::getTombstoneKey();
-    return mlir::LLVM::LLVMTypeNew(
-        static_cast<mlir::LLVM::LLVMTypeNew::ImplType *>(pointer));
+    return mlir::LLVM::LLVMType(
+        static_cast<mlir::LLVM::LLVMType::ImplType *>(pointer));
   }
-  static unsigned getHashValue(mlir::LLVM::LLVMTypeNew val) {
+  static unsigned getHashValue(mlir::LLVM::LLVMType val) {
     return mlir::hash_value(val);
   }
-  static bool isEqual(mlir::LLVM::LLVMTypeNew lhs,
-                      mlir::LLVM::LLVMTypeNew rhs) {
+  static bool isEqual(mlir::LLVM::LLVMType lhs, mlir::LLVM::LLVMType rhs) {
     return lhs == rhs;
   }
 };
 
-// LLVMTypeNew behaves like a pointer similarly to mlir::Type.
-template <> struct PointerLikeTypeTraits<mlir::LLVM::LLVMTypeNew> {
-  static inline void *getAsVoidPointer(mlir::LLVM::LLVMTypeNew type) {
+// LLVMType behaves like a pointer similarly to mlir::Type.
+template <>
+struct PointerLikeTypeTraits<mlir::LLVM::LLVMType> {
+  static inline void *getAsVoidPointer(mlir::LLVM::LLVMType type) {
     return const_cast<void *>(type.getAsOpaquePointer());
   }
-  static inline mlir::LLVM::LLVMTypeNew getFromVoidPointer(void *ptr) {
-    return mlir::LLVM::LLVMTypeNew::getFromOpaquePointer(ptr);
+  static inline mlir::LLVM::LLVMType getFromVoidPointer(void *ptr) {
+    return mlir::LLVM::LLVMType::getFromOpaquePointer(ptr);
   }
   static constexpr int NumLowBitsAvailable =
       PointerLikeTypeTraits<mlir::Type>::NumLowBitsAvailable;
diff --git a/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td
index 5c793f8547dab..5f022e32b801d 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td
@@ -112,7 +112,8 @@ def NVVM_ShflBflyOp :
     if (!type.isStructTy() || type.getStructNumElements() != 2 ||
         !type.getStructElementType(1).isIntegerTy(
             /*Bitwidth=*/1))
-      return emitError("expected return type !llvm<\"{ ?, i1 }\">");
+      return emitError("expected return type to be a two-element struct with "
+                       "i1 as the second element");
     return success();
   }];
 }
diff --git a/mlir/include/mlir/Target/LLVMIR/TypeTranslation.h b/mlir/include/mlir/Target/LLVMIR/TypeTranslation.h
index 5a82f0a096dfc..3ab962b9ab11b 100644
--- a/mlir/include/mlir/Target/LLVMIR/TypeTranslation.h
+++ b/mlir/include/mlir/Target/LLVMIR/TypeTranslation.h
@@ -25,10 +25,10 @@ class MLIRContext;
 
 namespace LLVM {
 
-class LLVMTypeNew;
+class LLVMType;
 
-llvm::Type *translateTypeToLLVMIR(LLVMTypeNew type, llvm::LLVMContext &context);
-LLVMTypeNew translateTypeFromLLVMIR(llvm::Type *type, MLIRContext &context);
+llvm::Type *translateTypeToLLVMIR(LLVMType type, llvm::LLVMContext &context);
+LLVMType translateTypeFromLLVMIR(llvm::Type *type, MLIRContext &context);
 
 } // namespace LLVM
 } // namespace mlir
diff --git a/mlir/integration_test/Dialect/LLVMIR/CPU/test-vector-reductions-fp.mlir b/mlir/integration_test/Dialect/LLVMIR/CPU/test-vector-reductions-fp.mlir
index 9b61414277b6a..2f17dbe4455f8 100644
--- a/mlir/integration_test/Dialect/LLVMIR/CPU/test-vector-reductions-fp.mlir
+++ b/mlir/integration_test/Dialect/LLVMIR/CPU/test-vector-reductions-fp.mlir
@@ -12,74 +12,74 @@ module {
     %1 = llvm.mlir.constant(2.000000e+00 : f32) : !llvm.float
     %2 = llvm.mlir.constant(3.000000e+00 : f32) : !llvm.float
     %3 = llvm.mlir.constant(4.000000e+00 : f32) : !llvm.float
-    %4 = llvm.mlir.undef : !llvm<"<4 x float>">
+    %4 = llvm.mlir.undef : !llvm.vec<4 x float>
     %5 = llvm.mlir.constant(0 : index) : !llvm.i64
-    %6 = llvm.insertelement %0, %4[%5 : !llvm.i64] : !llvm<"<4 x float>">
+    %6 = llvm.insertelement %0, %4[%5 : !llvm.i64] : !llvm.vec<4 x float>
     %7 = llvm.shufflevector %6, %4 [0 : i32, 0 : i32, 0 : i32, 0 : i32]
-        : !llvm<"<4 x float>">, !llvm<"<4 x float>">
+        : !llvm.vec<4 x float>, !llvm.vec<4 x float>
     %8 = llvm.mlir.constant(1 : i64) : !llvm.i64
-    %9 = llvm.insertelement %1, %7[%8 : !llvm.i64] : !llvm<"<4 x float>">
+    %9 = llvm.insertelement %1, %7[%8 : !llvm.i64] : !llvm.vec<4 x float>
     %10 = llvm.mlir.constant(2 : i64) : !llvm.i64
-    %11 = llvm.insertelement %2, %9[%10 : !llvm.i64] : !llvm<"<4 x float>">
+    %11 = llvm.insertelement %2, %9[%10 : !llvm.i64] : !llvm.vec<4 x float>
     %12 = llvm.mlir.constant(3 : i64) : !llvm.i64
-    %v = llvm.insertelement %3, %11[%12 : !llvm.i64] : !llvm<"<4 x float>">
+    %v = llvm.insertelement %3, %11[%12 : !llvm.i64] : !llvm.vec<4 x float>
 
     %max = "llvm.intr.experimental.vector.reduce.fmax"(%v)
-        : (!llvm<"<4 x float>">) -> !llvm.float
+        : (!llvm.vec<4 x float>) -> !llvm.float
     llvm.call @print_f32(%max) : (!llvm.float) -> ()
     llvm.call @print_newline() : () -> ()
     // CHECK: 4
 
     %min = "llvm.intr.experimental.vector.reduce.fmin"(%v)
-        : (!llvm<"<4 x float>">) -> !llvm.float
+        : (!llvm.vec<4 x float>) -> !llvm.float
     llvm.call @print_f32(%min) : (!llvm.float) -> ()
     llvm.call @print_newline() : () -> ()
     // CHECK: 1
 
     %add1 = "llvm.intr.experimental.vector.reduce.v2.fadd"(%0, %v)
-        : (!llvm.float, !llvm<"<4 x float>">) -> !llvm.float
+        : (!llvm.float, !llvm.vec<4 x float>) -> !llvm.float
     llvm.call @print_f32(%add1) : (!llvm.float) -> ()
     llvm.call @print_newline() : () -> ()
     // CHECK: 11
 
     %add1r = "llvm.intr.experimental.vector.reduce.v2.fadd"(%0, %v)
-        {reassoc = true} : (!llvm.float, !llvm<"<4 x float>">) -> !llvm.float
+        {reassoc = true} : (!llvm.float, !llvm.vec<4 x float>) -> !llvm.float
     llvm.call @print_f32(%add1r) : (!llvm.float) -> ()
     llvm.call @print_newline() : () -> ()
     // CHECK: 11
 
     %add2 = "llvm.intr.experimental.vector.reduce.v2.fadd"(%1, %v)
-        : (!llvm.float, !llvm<"<4 x float>">) -> !llvm.float
+        : (!llvm.float, !llvm.vec<4 x float>) -> !llvm.float
     llvm.call @print_f32(%add2) : (!llvm.float) -> ()
     llvm.call @print_newline() : () -> ()
     // CHECK: 12
 
     %add2r = "llvm.intr.experimental.vector.reduce.v2.fadd"(%1, %v)
-        {reassoc = true} : (!llvm.float, !llvm<"<4 x float>">) -> !llvm.float
+        {reassoc = true} : (!llvm.float, !llvm.vec<4 x float>) -> !llvm.float
     llvm.call @print_f32(%add2r) : (!llvm.float) -> ()
     llvm.call @print_newline() : () -> ()
     // CHECK: 12
 
     %mul1 = "llvm.intr.experimental.vector.reduce.v2.fmul"(%0, %v)
-        : (!llvm.float, !llvm<"<4 x float>">) -> !llvm.float
+        : (!llvm.float, !llvm.vec<4 x float>) -> !llvm.float
     llvm.call @print_f32(%mul1) : (!llvm.float) -> ()
     llvm.call @print_newline() : () -> ()
     // CHECK: 24
 
     %mul1r = "llvm.intr.experimental.vector.reduce.v2.fmul"(%0, %v)
-        {reassoc = true} : (!llvm.float, !llvm<"<4 x float>">) -> !llvm.float
+        {reassoc = true} : (!llvm.float, !llvm.vec<4 x float>) -> !llvm.float
     llvm.call @print_f32(%mul1r) : (!llvm.float) -> ()
     llvm.call @print_newline() : () -> ()
     // CHECK: 24
 
     %mul2 = "llvm.intr.experimental.vector.reduce.v2.fmul"(%1, %v)
-        : (!llvm.float, !llvm<"<4 x float>">) -> !llvm.float
+        : (!llvm.float, !llvm.vec<4 x float>) -> !llvm.float
     llvm.call @print_f32(%mul2) : (!llvm.float) -> ()
     llvm.call @print_newline() : () -> ()
     // CHECK: 48
 
     %mul2r = "llvm.intr.experimental.vector.reduce.v2.fmul"(%1, %v)
-        {reassoc = true} : (!llvm.float, !llvm<"<4 x float>">) -> !llvm.float
+        {reassoc = true} : (!llvm.float, !llvm.vec<4 x float>) -> !llvm.float
     llvm.call @print_f32(%mul2r) : (!llvm.float) -> ()
     llvm.call @print_newline() : () -> ()
     // CHECK: 48
diff --git a/mlir/integration_test/Dialect/LLVMIR/CPU/test-vector-reductions-int.mlir b/mlir/integration_test/Dialect/LLVMIR/CPU/test-vector-reductions-int.mlir
index 3fa556c62b31e..227c00a08ad82 100644
--- a/mlir/integration_test/Dialect/LLVMIR/CPU/test-vector-reductions-int.mlir
+++ b/mlir/integration_test/Dialect/LLVMIR/CPU/test-vector-reductions-int.mlir
@@ -12,68 +12,68 @@ module {
     %1 = llvm.mlir.constant(2 : i32) : !llvm.i32
     %2 = llvm.mlir.constant(3 : i32) : !llvm.i32
     %3 = llvm.mlir.constant(4 : i32) : !llvm.i32
-    %4 = llvm.mlir.undef : !llvm<"<4 x i32>">
+    %4 = llvm.mlir.undef : !llvm.vec<4 x i32>
     %5 = llvm.mlir.constant(0 : index) : !llvm.i64
-    %6 = llvm.insertelement %0, %4[%5 : !llvm.i64] : !llvm<"<4 x i32>">
+    %6 = llvm.insertelement %0, %4[%5 : !llvm.i64] : !llvm.vec<4 x i32>
     %7 = llvm.shufflevector %6, %4 [0 : i32, 0 : i32, 0 : i32, 0 : i32]
-        : !llvm<"<4 x i32>">, !llvm<"<4 x i32>">
+        : !llvm.vec<4 x i32>, !llvm.vec<4 x i32>
     %8 = llvm.mlir.constant(1 : i64) : !llvm.i64
-    %9 = llvm.insertelement %1, %7[%8 : !llvm.i64] : !llvm<"<4 x i32>">
+    %9 = llvm.insertelement %1, %7[%8 : !llvm.i64] : !llvm.vec<4 x i32>
     %10 = llvm.mlir.constant(2 : i64) : !llvm.i64
-    %11 = llvm.insertelement %2, %9[%10 : !llvm.i64] : !llvm<"<4 x i32>">
+    %11 = llvm.insertelement %2, %9[%10 : !llvm.i64] : !llvm.vec<4 x i32>
     %12 = llvm.mlir.constant(3 : i64) : !llvm.i64
-    %v = llvm.insertelement %3, %11[%12 : !llvm.i64] : !llvm<"<4 x i32>">
+    %v = llvm.insertelement %3, %11[%12 : !llvm.i64] : !llvm.vec<4 x i32>
 
     %add = "llvm.intr.experimental.vector.reduce.add"(%v)
-        : (!llvm<"<4 x i32>">) -> !llvm.i32
+        : (!llvm.vec<4 x i32>) -> !llvm.i32
     llvm.call @print_i32(%add) : (!llvm.i32) -> ()
     llvm.call @print_newline() : () -> ()
     // CHECK: 10
 
     %and = "llvm.intr.experimental.vector.reduce.and"(%v)
-        : (!llvm<"<4 x i32>">) -> !llvm.i32
+        : (!llvm.vec<4 x i32>) -> !llvm.i32
     llvm.call @print_i32(%and) : (!llvm.i32) -> ()
     llvm.call @print_newline() : () -> ()
     // CHECK: 0
 
     %mul = "llvm.intr.experimental.vector.reduce.mul"(%v)
-        : (!llvm<"<4 x i32>">) -> !llvm.i32
+        : (!llvm.vec<4 x i32>) -> !llvm.i32
     llvm.call @print_i32(%mul) : (!llvm.i32) -> ()
     llvm.call @print_newline() : () -> ()
     // CHECK: 24
 
     %or = "llvm.intr.experimental.vector.reduce.or"(%v)
-        : (!llvm<"<4 x i32>">) -> !llvm.i32
+        : (!llvm.vec<4 x i32>) -> !llvm.i32
     llvm.call @print_i32(%or) : (!llvm.i32) -> ()
     llvm.call @print_newline() : () -> ()
     // CHECK: 7
 
     %smax = "llvm.intr.experimental.vector.reduce.smax"(%v)
-        : (!llvm<"<4 x i32>">) -> !llvm.i32
+        : (!llvm.vec<4 x i32>) -> !llvm.i32
     llvm.call @print_i32(%smax) : (!llvm.i32) -> ()
     llvm.call @print_newline() : () -> ()
     // CHECK: 4
 
     %smin = "llvm.intr.experimental.vector.reduce.smin"(%v)
-        : (!llvm<"<4 x i32>">) -> !llvm.i32
+        : (!llvm.vec<4 x i32>) -> !llvm.i32
     llvm.call @print_i32(%smin) : (!llvm.i32) -> ()
     llvm.call @print_newline() : () -> ()
     // CHECK: 1
 
     %umax = "llvm.intr.experimental.vector.reduce.umax"(%v)
-        : (!llvm<"<4 x i32>">) -> !llvm.i32
+        : (!llvm.vec<4 x i32>) -> !llvm.i32
     llvm.call @print_i32(%umax) : (!llvm.i32) -> ()
     llvm.call @print_newline() : () -> ()
     // CHECK: 4
 
     %umin = "llvm.intr.experimental.vector.reduce.umin"(%v)
-        : (!llvm<"<4 x i32>">) -> !llvm.i32
+        : (!llvm.vec<4 x i32>) -> !llvm.i32
     llvm.call @print_i32(%umin) : (!llvm.i32) -> ()
     llvm.call @print_newline() : () -> ()
     // CHECK: 1
 
     %xor = "llvm.intr.experimental.vector.reduce.xor"(%v)
-        : (!llvm<"<4 x i32>">) -> !llvm.i32
+        : (!llvm.vec<4 x i32>) -> !llvm.i32
     llvm.call @print_i32(%xor) : (!llvm.i32) -> ()
     llvm.call @print_newline() : () -> ()
     // CHECK: 4
diff --git a/mlir/lib/Conversion/VectorToLLVM/CMakeLists.txt b/mlir/lib/Conversion/VectorToLLVM/CMakeLists.txt
index 569619f2bcef2..eeefd372f85f5 100644
--- a/mlir/lib/Conversion/VectorToLLVM/CMakeLists.txt
+++ b/mlir/lib/Conversion/VectorToLLVM/CMakeLists.txt
@@ -14,6 +14,7 @@ add_mlir_conversion_library(MLIRVectorToLLVM
   LINK_LIBS PUBLIC
   MLIRLLVMIR
   MLIRStandardToLLVM
-  MLIRVector
+  MLIRTargetLLVMIRModuleTranslation
   MLIRTransforms
+  MLIRVector
   )
diff --git a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
index 4fa7b573f84ef..3dbfaf88a443b 100644
--- a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
+++ b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
@@ -23,6 +23,7 @@
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/IR/StandardTypes.h"
 #include "mlir/IR/Types.h"
+#include "mlir/Target/LLVMIR/TypeTranslation.h"
 #include "mlir/Transforms/DialectConversion.h"
 #include "mlir/Transforms/Passes.h"
 #include "llvm/IR/DerivedTypes.h"
@@ -126,8 +127,10 @@ LogicalResult getMemRefAlignment(LLVMTypeConverter &typeConverter, T op,
     return failure();
 
   auto dataLayout = typeConverter.getDialect()->getLLVMModule().getDataLayout();
-  align = dataLayout.getPrefTypeAlignment(
-      LLVM::convertLLVMType(elementTy.cast<LLVM::LLVMType>()));
+  // TODO: this should be abstracted away to avoid depending on translation.
+  align = dataLayout.getPrefTypeAlignment(LLVM::translateTypeToLLVMIR(
+      elementTy.cast<LLVM::LLVMType>(),
+      typeConverter.getDialect()->getLLVMContext()));
   return success();
 }
 
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
index 17848c6bf3ee9..cc2200e84da57 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
@@ -11,6 +11,7 @@
 //
 //===----------------------------------------------------------------------===//
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
+#include "mlir/Dialect/LLVMIR/LLVMTypes.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/DialectImplementation.h"
 #include "mlir/IR/FunctionImplementation.h"
@@ -1691,12 +1692,6 @@ struct LLVMDialectImpl {
   llvm::LLVMContext llvmContext;
   llvm::Module module;
 
-  /// A set of LLVMTypes that are cached on construction to avoid any lookups or
-  /// locking.
-  LLVMType int1Ty, int8Ty, int16Ty, int32Ty, int64Ty, int128Ty;
-  LLVMType doubleTy, floatTy, bfloatTy, halfTy, fp128Ty, x86_fp80Ty;
-  LLVMType voidTy;
-
   /// A smart mutex to lock access to the llvm context. Unlike MLIR, LLVM is not
   /// multi-threaded and requires locked access to prevent race conditions.
   llvm::sys::SmartMutex<true> mutex;
@@ -1708,7 +1703,27 @@ struct LLVMDialectImpl {
 LLVMDialect::LLVMDialect(MLIRContext *context)
     : Dialect(getDialectNamespace(), context),
       impl(new detail::LLVMDialectImpl()) {
-  addTypes<LLVMType>();
+  // clang-format off
+  addTypes<LLVMVoidType,
+           LLVMHalfType,
+           LLVMBFloatType,
+           LLVMFloatType,
+           LLVMDoubleType,
+           LLVMFP128Type,
+           LLVMX86FP80Type,
+           LLVMPPCFP128Type,
+           LLVMX86MMXType,
+           LLVMTokenType,
+           LLVMLabelType,
+           LLVMMetadataType,
+           LLVMFunctionType,
+           LLVMIntegerType,
+           LLVMPointerType,
+           LLVMFixedVectorType,
+           LLVMScalableVectorType,
+           LLVMArrayType,
+           LLVMStructType>();
+  // clang-format on
   addOperations<
 #define GET_OP_LIST
 #include "mlir/Dialect/LLVMIR/LLVMOps.cpp.inc"
@@ -1716,26 +1731,6 @@ LLVMDialect::LLVMDialect(MLIRContext *context)
 
   // Support unknown operations because not all LLVM operations are registered.
   allowUnknownOperations();
-
-  // Cache some of the common LLVM types to avoid the need for lookups/locking.
-  auto &llvmContext = impl->llvmContext;
-  /// Integer Types.
-  impl->int1Ty = LLVMType::get(context, llvm::Type::getInt1Ty(llvmContext));
-  impl->int8Ty = LLVMType::get(context, llvm::Type::getInt8Ty(llvmContext));
-  impl->int16Ty = LLVMType::get(context, llvm::Type::getInt16Ty(llvmContext));
-  impl->int32Ty = LLVMType::get(context, llvm::Type::getInt32Ty(llvmContext));
-  impl->int64Ty = LLVMType::get(context, llvm::Type::getInt64Ty(llvmContext));
-  impl->int128Ty = LLVMType::get(context, llvm::Type::getInt128Ty(llvmContext));
-  /// Float Types.
-  impl->doubleTy = LLVMType::get(context, llvm::Type::getDoubleTy(llvmContext));
-  impl->floatTy = LLVMType::get(context, llvm::Type::getFloatTy(llvmContext));
-  impl->bfloatTy = LLVMType::get(context, llvm::Type::getBFloatTy(llvmContext));
-  impl->halfTy = LLVMType::get(context, llvm::Type::getHalfTy(llvmContext));
-  impl->fp128Ty = LLVMType::get(context, llvm::Type::getFP128Ty(llvmContext));
-  impl->x86_fp80Ty =
-      LLVMType::get(context, llvm::Type::getX86_FP80Ty(llvmContext));
-  /// Other Types.
-  impl->voidTy = LLVMType::get(context, llvm::Type::getVoidTy(llvmContext));
 }
 
 LLVMDialect::~LLVMDialect() {}
@@ -1751,25 +1746,12 @@ llvm::sys::SmartMutex<true> &LLVMDialect::getLLVMContextMutex() {
 
 /// Parse a type registered to this dialect.
 Type LLVMDialect::parseType(DialectAsmParser &parser) const {
-  StringRef tyData = parser.getFullSymbolSpec();
-
-  // LLVM is not thread-safe, so lock access to it.
-  llvm::sys::SmartScopedLock<true> lock(impl->mutex);
-
-  llvm::SMDiagnostic errorMessage;
-  llvm::Type *type = llvm::parseType(tyData, errorMessage, impl->module);
-  if (!type)
-    return (parser.emitError(parser.getNameLoc(), errorMessage.getMessage()),
-            nullptr);
-  return LLVMType::get(getContext(), type);
+  return detail::parseType(parser);
 }
 
 /// Print a type registered to this dialect.
 void LLVMDialect::printType(Type type, DialectAsmPrinter &os) const {
-  auto llvmType = type.dyn_cast<LLVMType>();
-  assert(llvmType && "printing wrong type");
-  assert(llvmType.getUnderlyingType() && "no underlying LLVM type");
-  llvmType.getUnderlyingType()->print(os.getStream());
+  return detail::printType(type.cast<LLVMType>(), os);
 }
 
 /// Verify LLVMIR function argument attributes.
@@ -1788,242 +1770,6 @@ LogicalResult LLVMDialect::verifyRegionArgAttribute(Operation *op,
   return success();
 }
 
-//===----------------------------------------------------------------------===//
-// LLVMType.
-//===----------------------------------------------------------------------===//
-
-namespace mlir {
-namespace LLVM {
-namespace detail {
-struct LLVMTypeStorage : public ::mlir::TypeStorage {
-  LLVMTypeStorage(llvm::Type *ty) : underlyingType(ty) {}
-
-  // LLVM types are pointer-unique.
-  using KeyTy = llvm::Type *;
-  bool operator==(const KeyTy &key) const { return key == underlyingType; }
-
-  static LLVMTypeStorage *construct(TypeStorageAllocator &allocator,
-                                    llvm::Type *ty) {
-    return new (allocator.allocate<LLVMTypeStorage>()) LLVMTypeStorage(ty);
-  }
-
-  llvm::Type *underlyingType;
-};
-} // end namespace detail
-} // end namespace LLVM
-} // end namespace mlir
-
-LLVMType LLVMType::get(MLIRContext *context, llvm::Type *llvmType) {
-  return Base::get(context, FIRST_LLVM_TYPE, llvmType);
-}
-
-/// Get an LLVMType with an llvm type that may cause changes to the underlying
-/// llvm context when constructed.
-LLVMType LLVMType::getLocked(LLVMDialect *dialect,
-                             function_ref<llvm::Type *()> typeBuilder) {
-  // Lock access to the llvm context and build the type.
-  llvm::sys::SmartScopedLock<true> lock(dialect->impl->mutex);
-  return get(dialect->getContext(), typeBuilder());
-}
-
-LLVMDialect &LLVMType::getDialect() {
-  return static_cast<LLVMDialect &>(Type::getDialect());
-}
-
-llvm::Type *LLVMType::getUnderlyingType() const {
-  return getImpl()->underlyingType;
-}
-
-void LLVMType::getUnderlyingTypes(ArrayRef<LLVMType> types,
-                                  SmallVectorImpl<llvm::Type *> &result) {
-  result.reserve(result.size() + types.size());
-  for (LLVMType ty : types)
-    result.push_back(ty.getUnderlyingType());
-}
-
-/// Array type utilities.
-LLVMType LLVMType::getArrayElementType() {
-  return get(getContext(), getUnderlyingType()->getArrayElementType());
-}
-unsigned LLVMType::getArrayNumElements() {
-  return getUnderlyingType()->getArrayNumElements();
-}
-bool LLVMType::isArrayTy() { return getUnderlyingType()->isArrayTy(); }
-
-/// Vector type utilities.
-LLVMType LLVMType::getVectorElementType() {
-  return get(
-      getContext(),
-      llvm::cast<llvm::VectorType>(getUnderlyingType())->getElementType());
-}
-unsigned LLVMType::getVectorNumElements() {
-  return llvm::cast<llvm::FixedVectorType>(getUnderlyingType())
-      ->getNumElements();
-}
-llvm::ElementCount LLVMType::getVectorElementCount() {
-  return llvm::cast<llvm::VectorType>(getUnderlyingType())->getElementCount();
-}
-bool LLVMType::isVectorTy() { return getUnderlyingType()->isVectorTy(); }
-
-/// Function type utilities.
-LLVMType LLVMType::getFunctionParamType(unsigned argIdx) {
-  return get(getContext(), getUnderlyingType()->getFunctionParamType(argIdx));
-}
-unsigned LLVMType::getFunctionNumParams() {
-  return getUnderlyingType()->getFunctionNumParams();
-}
-LLVMType LLVMType::getFunctionResultType() {
-  return get(
-      getContext(),
-      llvm::cast<llvm::FunctionType>(getUnderlyingType())->getReturnType());
-}
-bool LLVMType::isFunctionTy() { return getUnderlyingType()->isFunctionTy(); }
-bool LLVMType::isFunctionVarArg() {
-  return getUnderlyingType()->isFunctionVarArg();
-}
-
-/// Pointer type utilities.
-LLVMType LLVMType::getPointerTo(unsigned addrSpace) {
-  // Lock access to the dialect as this may modify the LLVM context.
-  return getLocked(&getDialect(), [=] {
-    return getUnderlyingType()->getPointerTo(addrSpace);
-  });
-}
-LLVMType LLVMType::getPointerElementTy() {
-  return get(getContext(), getUnderlyingType()->getPointerElementType());
-}
-bool LLVMType::isPointerTy() { return getUnderlyingType()->isPointerTy(); }
-bool LLVMType::isValidPointerElementType(LLVMType type) {
-  return llvm::PointerType::isValidElementType(type.getUnderlyingType());
-}
-
-/// Struct type utilities.
-LLVMType LLVMType::getStructElementType(unsigned i) {
-  return get(getContext(), getUnderlyingType()->getStructElementType(i));
-}
-unsigned LLVMType::getStructNumElements() {
-  return getUnderlyingType()->getStructNumElements();
-}
-bool LLVMType::isStructTy() { return getUnderlyingType()->isStructTy(); }
-
-/// Utilities used to generate floating point types.
-LLVMType LLVMType::getDoubleTy(LLVMDialect *dialect) {
-  return dialect->impl->doubleTy;
-}
-LLVMType LLVMType::getFloatTy(LLVMDialect *dialect) {
-  return dialect->impl->floatTy;
-}
-LLVMType LLVMType::getBFloatTy(LLVMDialect *dialect) {
-  return dialect->impl->bfloatTy;
-}
-LLVMType LLVMType::getHalfTy(LLVMDialect *dialect) {
-  return dialect->impl->halfTy;
-}
-LLVMType LLVMType::getFP128Ty(LLVMDialect *dialect) {
-  return dialect->impl->fp128Ty;
-}
-LLVMType LLVMType::getX86_FP80Ty(LLVMDialect *dialect) {
-  return dialect->impl->x86_fp80Ty;
-}
-
-/// Utilities used to generate integer types.
-LLVMType LLVMType::getIntNTy(LLVMDialect *dialect, unsigned numBits) {
-  switch (numBits) {
-  case 1:
-    return dialect->impl->int1Ty;
-  case 8:
-    return dialect->impl->int8Ty;
-  case 16:
-    return dialect->impl->int16Ty;
-  case 32:
-    return dialect->impl->int32Ty;
-  case 64:
-    return dialect->impl->int64Ty;
-  case 128:
-    return dialect->impl->int128Ty;
-  default:
-    break;
-  }
-
-  // Lock access to the dialect as this may modify the LLVM context.
-  return getLocked(dialect, [=] {
-    return llvm::Type::getIntNTy(dialect->getLLVMContext(), numBits);
-  });
-}
-
-/// Utilities used to generate other miscellaneous types.
-LLVMType LLVMType::getArrayTy(LLVMType elementType, uint64_t numElements) {
-  // Lock access to the dialect as this may modify the LLVM context.
-  return getLocked(&elementType.getDialect(), [=] {
-    return llvm::ArrayType::get(elementType.getUnderlyingType(), numElements);
-  });
-}
-LLVMType LLVMType::getFunctionTy(LLVMType result, ArrayRef<LLVMType> params,
-                                 bool isVarArg) {
-  SmallVector<llvm::Type *, 8> llvmParams;
-  for (auto param : params)
-    llvmParams.push_back(param.getUnderlyingType());
-
-  // Lock access to the dialect as this may modify the LLVM context.
-  return getLocked(&result.getDialect(), [=] {
-    return llvm::FunctionType::get(result.getUnderlyingType(), llvmParams,
-                                   isVarArg);
-  });
-}
-LLVMType LLVMType::getStructTy(LLVMDialect *dialect,
-                               ArrayRef<LLVMType> elements, bool isPacked) {
-  SmallVector<llvm::Type *, 8> llvmElements;
-  for (auto elt : elements)
-    llvmElements.push_back(elt.getUnderlyingType());
-
-  // Lock access to the dialect as this may modify the LLVM context.
-  return getLocked(dialect, [=] {
-    return llvm::StructType::get(dialect->getLLVMContext(), llvmElements,
-                                 isPacked);
-  });
-}
-LLVMType LLVMType::createStructTy(LLVMDialect *dialect,
-                                  ArrayRef<LLVMType> elements,
-                                  Optional<StringRef> name, bool isPacked) {
-  StringRef sr = name.hasValue() ? *name : "";
-  SmallVector<llvm::Type *, 8> llvmElements;
-  getUnderlyingTypes(elements, llvmElements);
-  return getLocked(dialect, [=] {
-    auto *rv = llvm::StructType::create(dialect->getLLVMContext(), sr);
-    if (!llvmElements.empty())
-      rv->setBody(llvmElements, isPacked);
-    return rv;
-  });
-}
-LLVMType LLVMType::setStructTyBody(LLVMType structType,
-                                   ArrayRef<LLVMType> elements, bool isPacked) {
-  llvm::StructType *st =
-      llvm::cast<llvm::StructType>(structType.getUnderlyingType());
-  SmallVector<llvm::Type *, 8> llvmElements;
-  getUnderlyingTypes(elements, llvmElements);
-  return getLocked(&structType.getDialect(), [=] {
-    st->setBody(llvmElements, isPacked);
-    return st;
-  });
-}
-LLVMType LLVMType::getVectorTy(LLVMType elementType, unsigned numElements) {
-  // Lock access to the dialect as this may modify the LLVM context.
-  return getLocked(&elementType.getDialect(), [=] {
-    return llvm::FixedVectorType::get(elementType.getUnderlyingType(),
-                                      numElements);
-  });
-}
-
-LLVMType LLVMType::getVoidTy(LLVMDialect *dialect) {
-  return dialect->impl->voidTy;
-}
-
-bool LLVMType::isVoidTy() { return getUnderlyingType()->isVoidTy(); }
-
-llvm::Type *mlir::LLVM::convertLLVMType(LLVMType type) {
-  return type.getUnderlyingType();
-}
-
 //===----------------------------------------------------------------------===//
 // Utility functions.
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMTypeSyntax.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMTypeSyntax.cpp
index d272297525c1e..7df3ebe7c5b4e 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMTypeSyntax.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMTypeSyntax.cpp
@@ -18,48 +18,48 @@ using namespace mlir::LLVM;
 // Printing.
 //===----------------------------------------------------------------------===//
 
-static void printTypeImpl(llvm::raw_ostream &os, LLVMTypeNew type,
+static void printTypeImpl(llvm::raw_ostream &os, LLVMType type,
                           llvm::SetVector<StringRef> &stack);
 
 /// Returns the keyword to use for the given type.
-static StringRef getTypeKeyword(LLVMTypeNew type) {
+static StringRef getTypeKeyword(LLVMType type) {
   switch (type.getKind()) {
-  case LLVMTypeNew::VoidType:
+  case LLVMType::VoidType:
     return "void";
-  case LLVMTypeNew::HalfType:
+  case LLVMType::HalfType:
     return "half";
-  case LLVMTypeNew::BFloatType:
+  case LLVMType::BFloatType:
     return "bfloat";
-  case LLVMTypeNew::FloatType:
+  case LLVMType::FloatType:
     return "float";
-  case LLVMTypeNew::DoubleType:
+  case LLVMType::DoubleType:
     return "double";
-  case LLVMTypeNew::FP128Type:
+  case LLVMType::FP128Type:
     return "fp128";
-  case LLVMTypeNew::X86FP80Type:
+  case LLVMType::X86FP80Type:
     return "x86_fp80";
-  case LLVMTypeNew::PPCFP128Type:
+  case LLVMType::PPCFP128Type:
     return "ppc_fp128";
-  case LLVMTypeNew::X86MMXType:
+  case LLVMType::X86MMXType:
     return "x86_mmx";
-  case LLVMTypeNew::TokenType:
+  case LLVMType::TokenType:
     return "token";
-  case LLVMTypeNew::LabelType:
+  case LLVMType::LabelType:
     return "label";
-  case LLVMTypeNew::MetadataType:
+  case LLVMType::MetadataType:
     return "metadata";
-  case LLVMTypeNew::FunctionType:
+  case LLVMType::FunctionType:
     return "func";
-  case LLVMTypeNew::IntegerType:
+  case LLVMType::IntegerType:
     return "i";
-  case LLVMTypeNew::PointerType:
+  case LLVMType::PointerType:
     return "ptr";
-  case LLVMTypeNew::FixedVectorType:
-  case LLVMTypeNew::ScalableVectorType:
+  case LLVMType::FixedVectorType:
+  case LLVMType::ScalableVectorType:
     return "vec";
-  case LLVMTypeNew::ArrayType:
+  case LLVMType::ArrayType:
     return "array";
-  case LLVMTypeNew::StructType:
+  case LLVMType::StructType:
     return "struct";
   }
   llvm_unreachable("unhandled type kind");
@@ -81,7 +81,7 @@ static void printStructTypeBody(llvm::raw_ostream &os, LLVMStructType type,
   os << '(';
   if (type.isIdentified())
     stack.insert(type.getName());
-  llvm::interleaveComma(type.getBody(), os, [&](LLVMTypeNew subtype) {
+  llvm::interleaveComma(type.getBody(), os, [&](LLVMType subtype) {
     printTypeImpl(os, subtype, stack);
   });
   if (type.isIdentified())
@@ -126,10 +126,9 @@ static void printFunctionType(llvm::raw_ostream &os, LLVMFunctionType funcType,
   os << '<';
   printTypeImpl(os, funcType.getReturnType(), stack);
   os << " (";
-  llvm::interleaveComma(funcType.getParams(), os,
-                        [&os, &stack](LLVMTypeNew subtype) {
-                          printTypeImpl(os, subtype, stack);
-                        });
+  llvm::interleaveComma(
+      funcType.getParams(), os,
+      [&os, &stack](LLVMType subtype) { printTypeImpl(os, subtype, stack); });
   if (funcType.isVarArg()) {
     if (funcType.getNumParams() != 0)
       os << ", ";
@@ -147,7 +146,7 @@ static void printFunctionType(llvm::raw_ostream &os, LLVMFunctionType funcType,
 ///   struct<"c", (ptr<struct<"b", (ptr<struct<"c">>)>>,
 ///                ptr<struct<"b", (ptr<struct<"c">>)>>)>
 /// note that "b" is printed twice.
-static void printTypeImpl(llvm::raw_ostream &os, LLVMTypeNew type,
+static void printTypeImpl(llvm::raw_ostream &os, LLVMType type,
                           llvm::SetVector<StringRef> &stack) {
   if (!type) {
     os << "<<NULL-TYPE>>";
@@ -158,8 +157,8 @@ static void printTypeImpl(llvm::raw_ostream &os, LLVMTypeNew type,
   os << getTypeKeyword(type);
 
   // Trivial types only consist of their keyword.
-  if (LLVMTypeNew::FIRST_TRIVIAL_TYPE <= kind &&
-      kind <= LLVMTypeNew::LAST_TRIVIAL_TYPE)
+  if (LLVMType::FIRST_TRIVIAL_TYPE <= kind &&
+      kind <= LLVMType::LAST_TRIVIAL_TYPE)
     return;
 
   if (auto intType = type.dyn_cast<LLVMIntegerType>()) {
@@ -194,8 +193,7 @@ static void printTypeImpl(llvm::raw_ostream &os, LLVMTypeNew type,
   printFunctionType(os, type.cast<LLVMFunctionType>(), stack);
 }
 
-void mlir::LLVM::detail::printType(LLVMTypeNew type,
-                                   DialectAsmPrinter &printer) {
+void mlir::LLVM::detail::printType(LLVMType type, DialectAsmPrinter &printer) {
   llvm::SetVector<StringRef> stack;
   return printTypeImpl(printer.getStream(), type, stack);
 }
@@ -204,13 +202,13 @@ void mlir::LLVM::detail::printType(LLVMTypeNew type,
 // Parsing.
 //===----------------------------------------------------------------------===//
 
-static LLVMTypeNew parseTypeImpl(DialectAsmParser &parser,
-                                 llvm::SetVector<StringRef> &stack);
+static LLVMType parseTypeImpl(DialectAsmParser &parser,
+                              llvm::SetVector<StringRef> &stack);
 
 /// Helper to be chained with other parsing functions.
 static ParseResult parseTypeImpl(DialectAsmParser &parser,
                                  llvm::SetVector<StringRef> &stack,
-                                 LLVMTypeNew &result) {
+                                 LLVMType &result) {
   result = parseTypeImpl(parser, stack);
   return success(result != nullptr);
 }
@@ -219,7 +217,7 @@ static ParseResult parseTypeImpl(DialectAsmParser &parser,
 ///   llvm-type :: = `func<` llvm-type `(` llvm-type-list `...`? `)>`
 static LLVMFunctionType parseFunctionType(DialectAsmParser &parser,
                                           llvm::SetVector<StringRef> &stack) {
-  LLVMTypeNew returnType;
+  LLVMType returnType;
   if (parser.parseLess() || parseTypeImpl(parser, stack, returnType) ||
       parser.parseLParen())
     return LLVMFunctionType();
@@ -232,7 +230,7 @@ static LLVMFunctionType parseFunctionType(DialectAsmParser &parser,
   }
 
   // Parse arguments.
-  SmallVector<LLVMTypeNew, 8> argTypes;
+  SmallVector<LLVMType, 8> argTypes;
   do {
     if (succeeded(parser.parseOptionalEllipsis())) {
       if (parser.parseOptionalRParen() || parser.parseOptionalGreater())
@@ -254,7 +252,7 @@ static LLVMFunctionType parseFunctionType(DialectAsmParser &parser,
 ///   llvm-type ::= `ptr<` llvm-type (`,` integer)? `>`
 static LLVMPointerType parsePointerType(DialectAsmParser &parser,
                                         llvm::SetVector<StringRef> &stack) {
-  LLVMTypeNew elementType;
+  LLVMType elementType;
   if (parser.parseLess() || parseTypeImpl(parser, stack, elementType))
     return LLVMPointerType();
 
@@ -274,7 +272,7 @@ static LLVMVectorType parseVectorType(DialectAsmParser &parser,
                                       llvm::SetVector<StringRef> &stack) {
   SmallVector<int64_t, 2> dims;
   llvm::SMLoc dimPos;
-  LLVMTypeNew elementType;
+  LLVMType elementType;
   if (parser.parseLess() || parser.getCurrentLocation(&dimPos) ||
       parser.parseDimensionList(dims, /*allowDynamic=*/true) ||
       parseTypeImpl(parser, stack, elementType) || parser.parseGreater())
@@ -304,7 +302,7 @@ static LLVMArrayType parseArrayType(DialectAsmParser &parser,
                                     llvm::SetVector<StringRef> &stack) {
   SmallVector<int64_t, 1> dims;
   llvm::SMLoc sizePos;
-  LLVMTypeNew elementType;
+  LLVMType elementType;
   if (parser.parseLess() || parser.getCurrentLocation(&sizePos) ||
       parser.parseDimensionList(dims, /*allowDynamic=*/false) ||
       parseTypeImpl(parser, stack, elementType) || parser.parseGreater())
@@ -322,7 +320,7 @@ static LLVMArrayType parseArrayType(DialectAsmParser &parser,
 /// error at `subtypesLoc` in case of failure, uses `stack` to make sure the
 /// types printed in the error message look like they did when parsed.
 static LLVMStructType trySetStructBody(LLVMStructType type,
-                                       ArrayRef<LLVMTypeNew> subtypes,
+                                       ArrayRef<LLVMType> subtypes,
                                        bool isPacked, DialectAsmParser &parser,
                                        llvm::SMLoc subtypesLoc,
                                        llvm::SetVector<StringRef> &stack) {
@@ -398,12 +396,12 @@ static LLVMStructType parseStructType(DialectAsmParser &parser,
 
   // Parse subtypes. For identified structs, put the identifier of the struct on
   // the stack to support self-references in the recursive calls.
-  SmallVector<LLVMTypeNew, 4> subtypes;
+  SmallVector<LLVMType, 4> subtypes;
   llvm::SMLoc subtypesLoc = parser.getCurrentLocation();
   do {
     if (isIdentified)
       stack.insert(name);
-    LLVMTypeNew type = parseTypeImpl(parser, stack);
+    LLVMType type = parseTypeImpl(parser, stack);
     if (!type)
       return LLVMStructType();
     subtypes.push_back(type);
@@ -422,8 +420,8 @@ static LLVMStructType parseStructType(DialectAsmParser &parser,
 }
 
 /// Parses one of the LLVM dialect types.
-static LLVMTypeNew parseTypeImpl(DialectAsmParser &parser,
-                                 llvm::SetVector<StringRef> &stack) {
+static LLVMType parseTypeImpl(DialectAsmParser &parser,
+                              llvm::SetVector<StringRef> &stack) {
   // Special case for integers (i[1-9][0-9]*) that are literals rather than
   // keywords for the parser, so they are not caught by the main dispatch below.
   // Try parsing it a built-in integer type instead.
@@ -433,11 +431,11 @@ static LLVMTypeNew parseTypeImpl(DialectAsmParser &parser,
   OptionalParseResult result = parser.parseOptionalType(maybeIntegerType);
   if (result.hasValue()) {
     if (failed(*result))
-      return LLVMTypeNew();
+      return LLVMType();
 
     if (!maybeIntegerType.isSignlessInteger()) {
       parser.emitError(keyLoc) << "unexpected type, expected i* or keyword";
-      return LLVMTypeNew();
+      return LLVMType();
     }
     return LLVMIntegerType::get(ctx, maybeIntegerType.getIntOrFloatBitWidth());
   }
@@ -445,9 +443,9 @@ static LLVMTypeNew parseTypeImpl(DialectAsmParser &parser,
   // Dispatch to concrete functions.
   StringRef key;
   if (failed(parser.parseKeyword(&key)))
-    return LLVMTypeNew();
+    return LLVMType();
 
-  return llvm::StringSwitch<function_ref<LLVMTypeNew()>>(key)
+  return llvm::StringSwitch<function_ref<LLVMType()>>(key)
       .Case("void", [&] { return LLVMVoidType::get(ctx); })
       .Case("half", [&] { return LLVMHalfType::get(ctx); })
       .Case("bfloat", [&] { return LLVMBFloatType::get(ctx); })
@@ -467,11 +465,11 @@ static LLVMTypeNew parseTypeImpl(DialectAsmParser &parser,
       .Case("struct", [&] { return parseStructType(parser, stack); })
       .Default([&] {
         parser.emitError(keyLoc) << "unknown LLVM type: " << key;
-        return LLVMTypeNew();
+        return LLVMType();
       })();
 }
 
-LLVMTypeNew mlir::LLVM::detail::parseType(DialectAsmParser &parser) {
+LLVMType mlir::LLVM::detail::parseType(DialectAsmParser &parser) {
   llvm::SetVector<StringRef> stack;
   return parseTypeImpl(parser, stack);
 }
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMTypes.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMTypes.cpp
index abecbccb1d4aa..fa25f2dcdad85 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMTypes.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMTypes.cpp
@@ -24,92 +24,89 @@ using namespace mlir;
 using namespace mlir::LLVM;
 
 //===----------------------------------------------------------------------===//
-// LLVMTypeNew.
+// LLVMType.
 //===----------------------------------------------------------------------===//
 
-// TODO: when these types are registered with the LLVMDialect, this method
-// should be removed and the regular Type::getDialect should just work.
-LLVMDialect &LLVMTypeNew::getDialect() {
-  return *getContext()->getRegisteredDialect<LLVM::LLVMDialect>();
+LLVMDialect &LLVMType::getDialect() {
+  return static_cast<LLVMDialect &>(Type::getDialect());
 }
 
 //----------------------------------------------------------------------------//
 // Integer type utilities.
 
-bool LLVMTypeNew::isIntegerTy(unsigned bitwidth) {
+bool LLVMType::isIntegerTy(unsigned bitwidth) {
   if (auto intType = dyn_cast<LLVMIntegerType>())
     return intType.getBitWidth() == bitwidth;
   return false;
 }
-
-unsigned LLVMTypeNew::getIntegerBitWidth() {
+unsigned LLVMType::getIntegerBitWidth() {
   return cast<LLVMIntegerType>().getBitWidth();
 }
 
-LLVMTypeNew LLVMTypeNew::getArrayElementType() {
+LLVMType LLVMType::getArrayElementType() {
   return cast<LLVMArrayType>().getElementType();
 }
 
 //----------------------------------------------------------------------------//
 // Array type utilities.
 
-unsigned LLVMTypeNew::getArrayNumElements() {
+unsigned LLVMType::getArrayNumElements() {
   return cast<LLVMArrayType>().getNumElements();
 }
 
-bool LLVMTypeNew::isArrayTy() { return isa<LLVMArrayType>(); }
+bool LLVMType::isArrayTy() { return isa<LLVMArrayType>(); }
 
 //----------------------------------------------------------------------------//
 // Vector type utilities.
 
-LLVMTypeNew LLVMTypeNew::getVectorElementType() {
+LLVMType LLVMType::getVectorElementType() {
   return cast<LLVMVectorType>().getElementType();
 }
 
-unsigned LLVMTypeNew::getVectorNumElements() {
+unsigned LLVMType::getVectorNumElements() {
   return cast<LLVMFixedVectorType>().getNumElements();
 }
-llvm::ElementCount LLVMTypeNew::getVectorElementCount() {
+llvm::ElementCount LLVMType::getVectorElementCount() {
   return cast<LLVMVectorType>().getElementCount();
 }
 
-bool LLVMTypeNew::isVectorTy() { return isa<LLVMVectorType>(); }
+bool LLVMType::isVectorTy() { return isa<LLVMVectorType>(); }
 
 //----------------------------------------------------------------------------//
 // Function type utilities.
 
-LLVMTypeNew LLVMTypeNew::getFunctionParamType(unsigned argIdx) {
+LLVMType LLVMType::getFunctionParamType(unsigned argIdx) {
   return cast<LLVMFunctionType>().getParamType(argIdx);
 }
 
-unsigned LLVMTypeNew::getFunctionNumParams() {
+unsigned LLVMType::getFunctionNumParams() {
   return cast<LLVMFunctionType>().getNumParams();
 }
 
-LLVMTypeNew LLVMTypeNew::getFunctionResultType() {
+LLVMType LLVMType::getFunctionResultType() {
   return cast<LLVMFunctionType>().getReturnType();
 }
 
-bool LLVMTypeNew::isFunctionTy() { return isa<LLVMFunctionType>(); }
+bool LLVMType::isFunctionTy() { return isa<LLVMFunctionType>(); }
 
-bool LLVMTypeNew::isFunctionVarArg() {
+bool LLVMType::isFunctionVarArg() {
   return cast<LLVMFunctionType>().isVarArg();
 }
 
 //----------------------------------------------------------------------------//
 // Pointer type utilities.
 
-LLVMTypeNew LLVMTypeNew::getPointerTo(unsigned addrSpace) {
+LLVMType LLVMType::getPointerTo(unsigned addrSpace) {
   return LLVMPointerType::get(*this, addrSpace);
 }
 
-LLVMTypeNew LLVMTypeNew::getPointerElementTy() {
+LLVMType LLVMType::getPointerElementTy() {
   return cast<LLVMPointerType>().getElementType();
 }
 
-bool LLVMTypeNew::isPointerTy() { return isa<LLVMPointerType>(); }
+bool LLVMType::isPointerTy() { return isa<LLVMPointerType>(); }
 
-bool LLVMTypeNew::isValidPointerElementType(LLVMTypeNew type) {
+bool LLVMType::isValidPointerElementType(LLVMType type) {
   return !type.isa<LLVMVoidType>() && !type.isa<LLVMTokenType>() &&
          !type.isa<LLVMMetadataType>() && !type.isa<LLVMLabelType>();
 }
@@ -117,91 +114,86 @@ bool LLVMTypeNew::isValidPointerElementType(LLVMTypeNew type) {
 //----------------------------------------------------------------------------//
 // Struct type utilities.
 
-LLVMTypeNew LLVMTypeNew::getStructElementType(unsigned i) {
+LLVMType LLVMType::getStructElementType(unsigned i) {
   return cast<LLVMStructType>().getBody()[i];
 }
 
-unsigned LLVMTypeNew::getStructNumElements() {
+unsigned LLVMType::getStructNumElements() {
   return cast<LLVMStructType>().getBody().size();
 }
 
-bool LLVMTypeNew::isStructTy() { return isa<LLVMStructType>(); }
+bool LLVMType::isStructTy() { return isa<LLVMStructType>(); }
 
 //----------------------------------------------------------------------------//
 // Utilities used to generate floating point types.
 
-LLVMTypeNew LLVMTypeNew::getDoubleTy(LLVMDialect *dialect) {
+LLVMType LLVMType::getDoubleTy(LLVMDialect *dialect) {
   return LLVMDoubleType::get(dialect->getContext());
 }
 
-LLVMTypeNew LLVMTypeNew::getFloatTy(LLVMDialect *dialect) {
+LLVMType LLVMType::getFloatTy(LLVMDialect *dialect) {
   return LLVMFloatType::get(dialect->getContext());
 }
 
-LLVMTypeNew LLVMTypeNew::getBFloatTy(LLVMDialect *dialect) {
+LLVMType LLVMType::getBFloatTy(LLVMDialect *dialect) {
   return LLVMBFloatType::get(dialect->getContext());
 }
 
-LLVMTypeNew LLVMTypeNew::getHalfTy(LLVMDialect *dialect) {
+LLVMType LLVMType::getHalfTy(LLVMDialect *dialect) {
   return LLVMHalfType::get(dialect->getContext());
 }
 
-LLVMTypeNew LLVMTypeNew::getFP128Ty(LLVMDialect *dialect) {
+LLVMType LLVMType::getFP128Ty(LLVMDialect *dialect) {
   return LLVMFP128Type::get(dialect->getContext());
 }
 
-LLVMTypeNew LLVMTypeNew::getX86_FP80Ty(LLVMDialect *dialect) {
+LLVMType LLVMType::getX86_FP80Ty(LLVMDialect *dialect) {
   return LLVMX86FP80Type::get(dialect->getContext());
 }
 
 //----------------------------------------------------------------------------//
 // Utilities used to generate integer types.
 
-LLVMTypeNew LLVMTypeNew::getIntNTy(LLVMDialect *dialect, unsigned numBits) {
+LLVMType LLVMType::getIntNTy(LLVMDialect *dialect, unsigned numBits) {
   return LLVMIntegerType::get(dialect->getContext(), numBits);
 }
 
 //----------------------------------------------------------------------------//
 // Utilities used to generate other miscellaneous types.
 
-LLVMTypeNew LLVMTypeNew::getArrayTy(LLVMTypeNew elementType,
-                                    uint64_t numElements) {
+LLVMType LLVMType::getArrayTy(LLVMType elementType, uint64_t numElements) {
   return LLVMArrayType::get(elementType, numElements);
 }
 
-LLVMTypeNew LLVMTypeNew::getFunctionTy(LLVMTypeNew result,
-                                       ArrayRef<LLVMTypeNew> params,
-                                       bool isVarArg) {
+LLVMType LLVMType::getFunctionTy(LLVMType result, ArrayRef<LLVMType> params,
+                                 bool isVarArg) {
   return LLVMFunctionType::get(result, params, isVarArg);
 }
 
-LLVMTypeNew LLVMTypeNew::getStructTy(LLVMDialect *dialect,
-                                     ArrayRef<LLVMTypeNew> elements,
-                                     bool isPacked) {
+LLVMType LLVMType::getStructTy(LLVMDialect *dialect,
+                               ArrayRef<LLVMType> elements, bool isPacked) {
   return LLVMStructType::getLiteral(dialect->getContext(), elements, isPacked);
 }
 
-LLVMTypeNew LLVMTypeNew::getVectorTy(LLVMTypeNew elementType,
-                                     unsigned numElements) {
+LLVMType LLVMType::getVectorTy(LLVMType elementType, unsigned numElements) {
   return LLVMFixedVectorType::get(elementType, numElements);
 }
 
 //----------------------------------------------------------------------------//
 // Void type utilities.
 
-LLVMTypeNew LLVMTypeNew::getVoidTy(LLVMDialect *dialect) {
+LLVMType LLVMType::getVoidTy(LLVMDialect *dialect) {
   return LLVMVoidType::get(dialect->getContext());
 }
 
-bool LLVMTypeNew::isVoidTy() { return isa<LLVMVoidType>(); }
+bool LLVMType::isVoidTy() { return isa<LLVMVoidType>(); }
 
 //----------------------------------------------------------------------------//
 // Creation and setting of LLVM's identified struct types
 
-LLVMTypeNew LLVMTypeNew::createStructTy(LLVMDialect *dialect,
-                                        ArrayRef<LLVMTypeNew> elements,
-                                        Optional<StringRef> name,
-                                        bool isPacked) {
+LLVMType LLVMType::createStructTy(LLVMDialect *dialect,
+                                  ArrayRef<LLVMType> elements,
+                                  Optional<StringRef> name, bool isPacked) {
   assert(name.hasValue() &&
          "identified structs with no identifier not supported");
   StringRef stringNameBase = name.getValueOr("");
@@ -220,9 +212,8 @@ LLVMTypeNew LLVMTypeNew::createStructTy(LLVMDialect *dialect,
   } while (true);
 }
 
-LLVMTypeNew LLVMTypeNew::setStructTyBody(LLVMTypeNew structType,
-                                         ArrayRef<LLVMTypeNew> elements,
-                                         bool isPacked) {
+LLVMType LLVMType::setStructTyBody(LLVMType structType,
+                                   ArrayRef<LLVMType> elements, bool isPacked) {
   LogicalResult couldSet =
       structType.cast<LLVMStructType>().setBody(elements, isPacked);
   assert(succeeded(couldSet) && "failed to set the body");
@@ -233,29 +224,28 @@ LLVMTypeNew LLVMTypeNew::setStructTyBody(LLVMTypeNew structType,
 //===----------------------------------------------------------------------===//
 // Array type.
 
-LLVMArrayType LLVMArrayType::get(LLVMTypeNew elementType,
-                                 unsigned numElements) {
+LLVMArrayType LLVMArrayType::get(LLVMType elementType, unsigned numElements) {
   assert(elementType && "expected non-null subtype");
-  return Base::get(elementType.getContext(), LLVMTypeNew::ArrayType,
-                   elementType, numElements);
+  return Base::get(elementType.getContext(), LLVMType::ArrayType, elementType,
+                   numElements);
 }
 
-LLVMTypeNew LLVMArrayType::getElementType() { return getImpl()->elementType; }
+LLVMType LLVMArrayType::getElementType() { return getImpl()->elementType; }
 
 unsigned LLVMArrayType::getNumElements() { return getImpl()->numElements; }
 
 //===----------------------------------------------------------------------===//
 // Function type.
 
-LLVMFunctionType LLVMFunctionType::get(LLVMTypeNew result,
-                                       ArrayRef<LLVMTypeNew> arguments,
+LLVMFunctionType LLVMFunctionType::get(LLVMType result,
+                                       ArrayRef<LLVMType> arguments,
                                        bool isVarArg) {
   assert(result && "expected non-null result");
-  return Base::get(result.getContext(), LLVMTypeNew::FunctionType, result,
+  return Base::get(result.getContext(), LLVMType::FunctionType, result,
                    arguments, isVarArg);
 }
 
-LLVMTypeNew LLVMFunctionType::getReturnType() {
+LLVMType LLVMFunctionType::getReturnType() {
   return getImpl()->getReturnType();
 }
 
@@ -263,13 +253,13 @@ unsigned LLVMFunctionType::getNumParams() {
   return getImpl()->getArgumentTypes().size();
 }
 
-LLVMTypeNew LLVMFunctionType::getParamType(unsigned i) {
+LLVMType LLVMFunctionType::getParamType(unsigned i) {
   return getImpl()->getArgumentTypes()[i];
 }
 
 bool LLVMFunctionType::isVarArg() { return getImpl()->isVariadic(); }
 
-ArrayRef<LLVMTypeNew> LLVMFunctionType::getParams() {
+ArrayRef<LLVMType> LLVMFunctionType::getParams() {
   return getImpl()->getArgumentTypes();
 }
 
@@ -277,7 +267,7 @@ ArrayRef<LLVMTypeNew> LLVMFunctionType::getParams() {
 // Integer type.
 
 LLVMIntegerType LLVMIntegerType::get(MLIRContext *ctx, unsigned bitwidth) {
-  return Base::get(ctx, LLVMTypeNew::IntegerType, bitwidth);
+  return Base::get(ctx, LLVMType::IntegerType, bitwidth);
 }
 
 unsigned LLVMIntegerType::getBitWidth() { return getImpl()->bitwidth; }
@@ -285,14 +275,13 @@ unsigned LLVMIntegerType::getBitWidth() { return getImpl()->bitwidth; }
 //===----------------------------------------------------------------------===//
 // Pointer type.
 
-LLVMPointerType LLVMPointerType::get(LLVMTypeNew pointee,
-                                     unsigned addressSpace) {
+LLVMPointerType LLVMPointerType::get(LLVMType pointee, unsigned addressSpace) {
   assert(pointee && "expected non-null subtype");
-  return Base::get(pointee.getContext(), LLVMTypeNew::PointerType, pointee,
+  return Base::get(pointee.getContext(), LLVMType::PointerType, pointee,
                    addressSpace);
 }
 
-LLVMTypeNew LLVMPointerType::getElementType() { return getImpl()->pointeeType; }
+LLVMType LLVMPointerType::getElementType() { return getImpl()->pointeeType; }
 
 unsigned LLVMPointerType::getAddressSpace() { return getImpl()->addressSpace; }
 
@@ -301,21 +290,20 @@ unsigned LLVMPointerType::getAddressSpace() { return getImpl()->addressSpace; }
 
 LLVMStructType LLVMStructType::getIdentified(MLIRContext *context,
                                              StringRef name) {
-  return Base::get(context, LLVMTypeNew::StructType, name, /*opaque=*/false);
+  return Base::get(context, LLVMType::StructType, name, /*opaque=*/false);
 }
 
 LLVMStructType LLVMStructType::getLiteral(MLIRContext *context,
-                                          ArrayRef<LLVMTypeNew> types,
+                                          ArrayRef<LLVMType> types,
                                           bool isPacked) {
-  return Base::get(context, LLVMTypeNew::StructType, types, isPacked);
+  return Base::get(context, LLVMType::StructType, types, isPacked);
 }
 
 LLVMStructType LLVMStructType::getOpaque(StringRef name, MLIRContext *context) {
-  return Base::get(context, LLVMTypeNew::StructType, name, /*opaque=*/true);
+  return Base::get(context, LLVMType::StructType, name, /*opaque=*/true);
 }
 
-LogicalResult LLVMStructType::setBody(ArrayRef<LLVMTypeNew> types,
-                                      bool isPacked) {
+LogicalResult LLVMStructType::setBody(ArrayRef<LLVMType> types, bool isPacked) {
   assert(isIdentified() && "can only set bodies of identified structs");
   return Base::mutate(types, isPacked);
 }
@@ -327,7 +315,7 @@ bool LLVMStructType::isOpaque() {
 }
 bool LLVMStructType::isInitialized() { return getImpl()->isInitialized(); }
 StringRef LLVMStructType::getName() { return getImpl()->getIdentifier(); }
-ArrayRef<LLVMTypeNew> LLVMStructType::getBody() {
+ArrayRef<LLVMType> LLVMStructType::getBody() {
   return isIdentified() ? getImpl()->getIdentifiedStructBody()
                         : getImpl()->getTypeList();
 }
@@ -335,7 +323,7 @@ ArrayRef<LLVMTypeNew> LLVMStructType::getBody() {
 //===----------------------------------------------------------------------===//
 // Vector types.
 
-LLVMTypeNew LLVMVectorType::getElementType() {
+LLVMType LLVMVectorType::getElementType() {
   // Both derived classes share the implementation type.
   return static_cast<detail::LLVMTypeAndSizeStorage *>(impl)->elementType;
 }
@@ -347,10 +335,10 @@ llvm::ElementCount LLVMVectorType::getElementCount() {
       this->isa<LLVMScalableVectorType>());
 }
 
-LLVMFixedVectorType LLVMFixedVectorType::get(LLVMTypeNew elementType,
+LLVMFixedVectorType LLVMFixedVectorType::get(LLVMType elementType,
                                              unsigned numElements) {
   assert(elementType && "expected non-null subtype");
-  return Base::get(elementType.getContext(), LLVMTypeNew::FixedVectorType,
+  return Base::get(elementType.getContext(), LLVMType::FixedVectorType,
                    elementType, numElements)
       .cast<LLVMFixedVectorType>();
 }
@@ -359,10 +347,10 @@ unsigned LLVMFixedVectorType::getNumElements() {
   return getImpl()->numElements;
 }
 
-LLVMScalableVectorType LLVMScalableVectorType::get(LLVMTypeNew elementType,
+LLVMScalableVectorType LLVMScalableVectorType::get(LLVMType elementType,
                                                    unsigned minNumElements) {
   assert(elementType && "expected non-null subtype");
-  return Base::get(elementType.getContext(), LLVMTypeNew::ScalableVectorType,
+  return Base::get(elementType.getContext(), LLVMType::ScalableVectorType,
                    elementType, minNumElements)
       .cast<LLVMScalableVectorType>();
 }
diff --git a/mlir/lib/Dialect/LLVMIR/IR/TypeDetail.h b/mlir/lib/Dialect/LLVMIR/IR/TypeDetail.h
index 2b72e43e51648..3f2cc13299a48 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/TypeDetail.h
+++ b/mlir/lib/Dialect/LLVMIR/IR/TypeDetail.h
@@ -72,7 +72,7 @@ struct LLVMStructTypeStorage : public TypeStorage {
     Key(StringRef name, bool opaque)
         : name(name), identified(true), packed(false), opaque(opaque) {}
     /// Constructs a key for a literal struct.
-    Key(ArrayRef<LLVMTypeNew> types, bool packed)
+    Key(ArrayRef<LLVMType> types, bool packed)
         : types(types), identified(false), packed(packed), opaque(false) {}
 
     /// Checks a specific property of the struct.
@@ -96,7 +96,7 @@ struct LLVMStructTypeStorage : public TypeStorage {
     }
 
     /// Returns the list of type contained in the key of a literal struct.
-    ArrayRef<LLVMTypeNew> getTypeList() const {
+    ArrayRef<LLVMType> getTypeList() const {
       assert(!isIdentified() &&
              "identified struct key cannot have a type list");
       return types;
@@ -138,7 +138,7 @@ struct LLVMStructTypeStorage : public TypeStorage {
     }
 
   private:
-    ArrayRef<LLVMTypeNew> types;
+    ArrayRef<LLVMType> types;
     StringRef name;
     bool identified;
     bool packed;
@@ -153,19 +153,18 @@ struct LLVMStructTypeStorage : public TypeStorage {
   }
 
   /// Returns the list of types (partially) identifying a literal struct.
-  ArrayRef<LLVMTypeNew> getTypeList() const {
+  ArrayRef<LLVMType> getTypeList() const {
     // If this triggers, use getIdentifiedStructBody() instead.
     assert(!isIdentified() && "requested typelist on an identified struct");
-    return ArrayRef<LLVMTypeNew>(static_cast<const LLVMTypeNew *>(keyPtr),
-                                 keySize());
+    return ArrayRef<LLVMType>(static_cast<const LLVMType *>(keyPtr), keySize());
   }
 
   /// Returns the list of types contained in an identified struct.
-  ArrayRef<LLVMTypeNew> getIdentifiedStructBody() const {
+  ArrayRef<LLVMType> getIdentifiedStructBody() const {
     // If this triggers, use getTypeList() instead.
     assert(isIdentified() &&
            "requested struct body on a non-identified struct");
-    return ArrayRef<LLVMTypeNew>(identifiedBodyArray, identifiedBodySize());
+    return ArrayRef<LLVMType>(identifiedBodyArray, identifiedBodySize());
   }
 
   /// Checks whether the struct is identified.
@@ -200,7 +199,7 @@ struct LLVMStructTypeStorage : public TypeStorage {
   /// as initalized and can no longer be mutated.
   LLVMStructTypeStorage(const KeyTy &key) {
     if (!key.isIdentified()) {
-      ArrayRef<LLVMTypeNew> types = key.getTypeList();
+      ArrayRef<LLVMType> types = key.getTypeList();
       keyPtr = static_cast<const void *>(types.data());
       setKeySize(types.size());
       llvm::Bitfield::set<KeyFlagPacked>(keySizeAndFlags, key.isPacked());
@@ -233,8 +232,8 @@ struct LLVMStructTypeStorage : public TypeStorage {
   /// initialized, succeeds only if the body is equal to the current body. Fails
   /// if the struct is marked as intentionally opaque. The struct will be marked
   /// as initialized as a result of this operation and can no longer be changed.
-  LogicalResult mutate(TypeStorageAllocator &allocator,
-                       ArrayRef<LLVMTypeNew> body, bool packed) {
+  LogicalResult mutate(TypeStorageAllocator &allocator, ArrayRef<LLVMType> body,
+                       bool packed) {
     if (!isIdentified())
       return failure();
     if (isInitialized())
@@ -245,7 +244,7 @@ struct LLVMStructTypeStorage : public TypeStorage {
                                                 true);
     llvm::Bitfield::set<MutableFlagPacked>(identifiedBodySizeAndFlags, packed);
 
-    ArrayRef<LLVMTypeNew> typesInAllocator = allocator.copyInto(body);
+    ArrayRef<LLVMType> typesInAllocator = allocator.copyInto(body);
     identifiedBodyArray = typesInAllocator.data();
     setIdentifiedBodySize(typesInAllocator.size());
 
@@ -311,7 +310,7 @@ struct LLVMStructTypeStorage : public TypeStorage {
   const void *keyPtr = nullptr;
 
   /// Pointer to the first type contained in an identified struct.
-  const LLVMTypeNew *identifiedBodyArray = nullptr;
+  const LLVMType *identifiedBodyArray = nullptr;
 
   /// Size of the uniquing key combined with identified/literal and
   /// packedness bits. Must only be used through the Key* bitfields.
@@ -329,11 +328,11 @@ struct LLVMStructTypeStorage : public TypeStorage {
 /// Type storage for LLVM dialect function types. These are uniqued using the
 /// list of types they contain and the vararg bit.
 struct LLVMFunctionTypeStorage : public TypeStorage {
-  using KeyTy = std::tuple<LLVMTypeNew, ArrayRef<LLVMTypeNew>, bool>;
+  using KeyTy = std::tuple<LLVMType, ArrayRef<LLVMType>, bool>;
 
   /// Construct a storage from the given components. The list is expected to be
   /// allocated in the context.
-  LLVMFunctionTypeStorage(LLVMTypeNew result, ArrayRef<LLVMTypeNew> arguments,
+  LLVMFunctionTypeStorage(LLVMType result, ArrayRef<LLVMType> arguments,
                           bool variadic)
       : argumentTypes(arguments) {
     returnTypeAndVariadic.setPointerAndInt(result, variadic);
@@ -360,21 +359,19 @@ struct LLVMFunctionTypeStorage : public TypeStorage {
   }
 
   /// Returns the list of function argument types.
-  ArrayRef<LLVMTypeNew> getArgumentTypes() const { return argumentTypes; }
+  ArrayRef<LLVMType> getArgumentTypes() const { return argumentTypes; }
 
   /// Checks whether the function type is variadic.
   bool isVariadic() const { return returnTypeAndVariadic.getInt(); }
 
   /// Returns the function result type.
-  LLVMTypeNew getReturnType() const {
-    return returnTypeAndVariadic.getPointer();
-  }
+  LLVMType getReturnType() const { return returnTypeAndVariadic.getPointer(); }
 
 private:
   /// Function result type packed with the variadic bit.
-  llvm::PointerIntPair<LLVMTypeNew, 1, bool> returnTypeAndVariadic;
+  llvm::PointerIntPair<LLVMType, 1, bool> returnTypeAndVariadic;
   /// Argument types.
-  ArrayRef<LLVMTypeNew> argumentTypes;
+  ArrayRef<LLVMType> argumentTypes;
 };
 
 //===----------------------------------------------------------------------===//
@@ -405,7 +402,7 @@ struct LLVMIntegerTypeStorage : public TypeStorage {
 /// Storage type for LLVM dialect pointer types. These are uniqued by a pair of
 /// element type and address space.
 struct LLVMPointerTypeStorage : public TypeStorage {
-  using KeyTy = std::tuple<LLVMTypeNew, unsigned>;
+  using KeyTy = std::tuple<LLVMType, unsigned>;
 
   LLVMPointerTypeStorage(const KeyTy &key)
       : pointeeType(std::get<0>(key)), addressSpace(std::get<1>(key)) {}
@@ -420,7 +417,7 @@ struct LLVMPointerTypeStorage : public TypeStorage {
     return std::make_tuple(pointeeType, addressSpace) == key;
   }
 
-  LLVMTypeNew pointeeType;
+  LLVMType pointeeType;
   unsigned addressSpace;
 };
 
@@ -432,7 +429,7 @@ struct LLVMPointerTypeStorage : public TypeStorage {
 /// number: arrays, fixed and scalable vectors. The actual semantics of the
 /// type is defined by its kind.
 struct LLVMTypeAndSizeStorage : public TypeStorage {
-  using KeyTy = std::tuple<LLVMTypeNew, unsigned>;
+  using KeyTy = std::tuple<LLVMType, unsigned>;
 
   LLVMTypeAndSizeStorage(const KeyTy &key)
       : elementType(std::get<0>(key)), numElements(std::get<1>(key)) {}
@@ -447,7 +444,7 @@ struct LLVMTypeAndSizeStorage : public TypeStorage {
     return std::make_tuple(elementType, numElements) == key;
   }
 
-  LLVMTypeNew elementType;
+  LLVMType elementType;
   unsigned numElements;
 };
 
diff --git a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
index 6b068660d98ff..5107efe28971c 100644
--- a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
@@ -20,6 +20,7 @@
 #include "mlir/IR/Module.h"
 #include "mlir/IR/StandardTypes.h"
 #include "mlir/Support/LLVM.h"
+#include "mlir/Target/LLVMIR/TypeTranslation.h"
 #include "llvm/ADT/TypeSwitch.h"
 
 #include "llvm/ADT/SetVector.h"
@@ -932,7 +933,9 @@ LogicalResult ModuleTranslation::convertFunctions() {
 }
 
 llvm::Type *ModuleTranslation::convertType(LLVMType type) {
-  return LLVM::convertLLVMType(type);
+  // Lock the LLVM context as we create types in it.
+  llvm::sys::SmartScopedLock<true> lock(llvmDialect->getLLVMContextMutex());
+  return LLVM::translateTypeToLLVMIR(type, llvmDialect->getLLVMContext());
 }
 
 /// A helper to look up remapped operands in the value remapping table.`
diff --git a/mlir/lib/Target/LLVMIR/TypeTranslation.cpp b/mlir/lib/Target/LLVMIR/TypeTranslation.cpp
index 6163334d3b4ef..15e0f158007a5 100644
--- a/mlir/lib/Target/LLVMIR/TypeTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/TypeTranslation.cpp
@@ -24,14 +24,14 @@ class TypeToLLVMIRTranslator {
   TypeToLLVMIRTranslator(llvm::LLVMContext &context) : context(context) {}
 
   /// Translates a single type.
-  llvm::Type *translateType(LLVM::LLVMTypeNew type) {
+  llvm::Type *translateType(LLVM::LLVMType type) {
     // If the conversion is already known, just return it.
     if (knownTranslations.count(type))
       return knownTranslations.lookup(type);
 
     // Dispatch to an appropriate function.
     llvm::Type *translated =
-        llvm::TypeSwitch<LLVM::LLVMTypeNew, llvm::Type *>(type)
+        llvm::TypeSwitch<LLVM::LLVMType, llvm::Type *>(type)
             .Case([this](LLVM::LLVMVoidType) {
               return llvm::Type::getVoidTy(context);
             })
@@ -73,7 +73,7 @@ class TypeToLLVMIRTranslator {
                   LLVM::LLVMStructType, LLVM::LLVMFixedVectorType,
                   LLVM::LLVMScalableVectorType>(
                 [this](auto array) { return translate(array); })
-            .Default([](LLVM::LLVMTypeNew t) -> llvm::Type * {
+            .Default([](LLVM::LLVMType t) -> llvm::Type * {
               llvm_unreachable("unknown LLVM dialect type");
             });
 
@@ -144,7 +144,7 @@ class TypeToLLVMIRTranslator {
   }
 
   /// Translates a list of types.
-  void translateTypes(ArrayRef<LLVM::LLVMTypeNew> types,
+  void translateTypes(ArrayRef<LLVM::LLVMType> types,
                       SmallVectorImpl<llvm::Type *> &result) {
     result.reserve(result.size() + types.size());
     for (auto type : types)
@@ -158,14 +158,14 @@ class TypeToLLVMIRTranslator {
   /// results to avoid repeated recursive calls and makes sure identified
   /// structs with the same name (that is, equal) are resolved to an existing
   /// type instead of creating a new type.
-  llvm::DenseMap<LLVM::LLVMTypeNew, llvm::Type *> knownTranslations;
+  llvm::DenseMap<LLVM::LLVMType, llvm::Type *> knownTranslations;
 };
 } // end namespace
 
 /// Translates a type from MLIR LLVM dialect to LLVM IR. This does not maintain
 /// the mapping for identified structs so new structs will be created with
 /// auto-renaming on each call. This is intended exclusively for testing.
-llvm::Type *mlir::LLVM::translateTypeToLLVMIR(LLVM::LLVMTypeNew type,
+llvm::Type *mlir::LLVM::translateTypeToLLVMIR(LLVM::LLVMType type,
                                               llvm::LLVMContext &context) {
   return TypeToLLVMIRTranslator(context).translateType(type);
 }
@@ -178,12 +178,12 @@ class TypeFromLLVMIRTranslator {
   TypeFromLLVMIRTranslator(MLIRContext &context) : context(context) {}
 
   /// Translates the given type.
-  LLVM::LLVMTypeNew translateType(llvm::Type *type) {
+  LLVM::LLVMType translateType(llvm::Type *type) {
     if (knownTranslations.count(type))
       return knownTranslations.lookup(type);
 
-    LLVM::LLVMTypeNew translated =
-        llvm::TypeSwitch<llvm::Type *, LLVM::LLVMTypeNew>(type)
+    LLVM::LLVMType translated =
+        llvm::TypeSwitch<llvm::Type *, LLVM::LLVMType>(type)
             .Case<llvm::ArrayType, llvm::FunctionType, llvm::IntegerType,
                   llvm::PointerType, llvm::StructType, llvm::FixedVectorType,
                   llvm::ScalableVectorType>(
@@ -198,7 +198,7 @@ class TypeFromLLVMIRTranslator {
 private:
   /// Translates the given primitive, i.e. non-parametric in MLIR nomenclature,
   /// type.
-  LLVM::LLVMTypeNew translatePrimitiveType(llvm::Type *type) {
+  LLVM::LLVMType translatePrimitiveType(llvm::Type *type) {
     if (type->isVoidTy())
       return LLVM::LLVMVoidType::get(&context);
     if (type->isHalfTy())
@@ -225,33 +225,33 @@ class TypeFromLLVMIRTranslator {
   }
 
   /// Translates the given array type.
-  LLVM::LLVMTypeNew translate(llvm::ArrayType *type) {
+  LLVM::LLVMType translate(llvm::ArrayType *type) {
     return LLVM::LLVMArrayType::get(translateType(type->getElementType()),
                                     type->getNumElements());
   }
 
   /// Translates the given function type.
-  LLVM::LLVMTypeNew translate(llvm::FunctionType *type) {
-    SmallVector<LLVM::LLVMTypeNew, 8> paramTypes;
+  LLVM::LLVMType translate(llvm::FunctionType *type) {
+    SmallVector<LLVM::LLVMType, 8> paramTypes;
     translateTypes(type->params(), paramTypes);
     return LLVM::LLVMFunctionType::get(translateType(type->getReturnType()),
                                        paramTypes, type->isVarArg());
   }
 
   /// Translates the given integer type.
-  LLVM::LLVMTypeNew translate(llvm::IntegerType *type) {
+  LLVM::LLVMType translate(llvm::IntegerType *type) {
     return LLVM::LLVMIntegerType::get(&context, type->getBitWidth());
   }
 
   /// Translates the given pointer type.
-  LLVM::LLVMTypeNew translate(llvm::PointerType *type) {
+  LLVM::LLVMType translate(llvm::PointerType *type) {
     return LLVM::LLVMPointerType::get(translateType(type->getElementType()),
                                       type->getAddressSpace());
   }
 
   /// Translates the given structure type.
-  LLVM::LLVMTypeNew translate(llvm::StructType *type) {
-    SmallVector<LLVM::LLVMTypeNew, 8> subtypes;
+  LLVM::LLVMType translate(llvm::StructType *type) {
+    SmallVector<LLVM::LLVMType, 8> subtypes;
     if (type->isLiteral()) {
       translateTypes(type->subtypes(), subtypes);
       return LLVM::LLVMStructType::getLiteral(&context, subtypes,
@@ -273,20 +273,20 @@ class TypeFromLLVMIRTranslator {
   }
 
   /// Translates the given fixed-vector type.
-  LLVM::LLVMTypeNew translate(llvm::FixedVectorType *type) {
+  LLVM::LLVMType translate(llvm::FixedVectorType *type) {
     return LLVM::LLVMFixedVectorType::get(translateType(type->getElementType()),
                                           type->getNumElements());
   }
 
   /// Translates the given scalable-vector type.
-  LLVM::LLVMTypeNew translate(llvm::ScalableVectorType *type) {
+  LLVM::LLVMType translate(llvm::ScalableVectorType *type) {
     return LLVM::LLVMScalableVectorType::get(
         translateType(type->getElementType()), type->getMinNumElements());
   }
 
   /// Translates a list of types.
   void translateTypes(ArrayRef<llvm::Type *> types,
-                      SmallVectorImpl<LLVM::LLVMTypeNew> &result) {
+                      SmallVectorImpl<LLVM::LLVMType> &result) {
     result.reserve(result.size() + types.size());
     for (llvm::Type *type : types)
       result.push_back(translateType(type));
@@ -294,7 +294,7 @@ class TypeFromLLVMIRTranslator {
 
   /// Map of known translations. Serves as a cache and as recursion stopper for
   /// translating recursive structs.
-  llvm::DenseMap<llvm::Type *, LLVM::LLVMTypeNew> knownTranslations;
+  llvm::DenseMap<llvm::Type *, LLVM::LLVMType> knownTranslations;
 
   /// The context in which MLIR types are created.
   MLIRContext &context;
@@ -303,7 +303,7 @@ class TypeFromLLVMIRTranslator {
 
 /// Translates a type from LLVM IR to MLIR LLVM dialect. This is intended
 /// exclusively for testing.
-LLVM::LLVMTypeNew mlir::LLVM::translateTypeFromLLVMIR(llvm::Type *type,
-                                                      MLIRContext &context) {
+LLVM::LLVMType mlir::LLVM::translateTypeFromLLVMIR(llvm::Type *type,
+                                                   MLIRContext &context) {
   return TypeFromLLVMIRTranslator(context).translateType(type);
 }
diff --git a/mlir/test/Conversion/GPUCommon/lower-launch-func-to-gpu-runtime-calls.mlir b/mlir/test/Conversion/GPUCommon/lower-launch-func-to-gpu-runtime-calls.mlir
index bdcde0be60c20..f5c085224f07a 100644
--- a/mlir/test/Conversion/GPUCommon/lower-launch-func-to-gpu-runtime-calls.mlir
+++ b/mlir/test/Conversion/GPUCommon/lower-launch-func-to-gpu-runtime-calls.mlir
@@ -8,27 +8,27 @@ module attributes {gpu.container_module} {
   // ROCDL: llvm.mlir.global internal constant @[[global:.*]]("HSACO")
 
   gpu.module @kernel_module attributes {nvvm.cubin = "CUBIN", rocdl.hsaco = "HSACO"} {
-    llvm.func @kernel(%arg0: !llvm.float, %arg1: !llvm<"float*">) attributes {gpu.kernel} {
+    llvm.func @kernel(%arg0: !llvm.float, %arg1: !llvm.ptr<float>) attributes {gpu.kernel} {
       llvm.return
     }
   }
 
   llvm.func @foo() {
     %0 = "op"() : () -> (!llvm.float)
-    %1 = "op"() : () -> (!llvm<"float*">)
+    %1 = "op"() : () -> (!llvm.ptr<float>)
     %cst = llvm.mlir.constant(8 : index) : !llvm.i64
 
     // CHECK: %[[addressof:.*]] = llvm.mlir.addressof @[[global]]
     // CHECK: %[[c0:.*]] = llvm.mlir.constant(0 : index)
     // CHECK: %[[binary:.*]] = llvm.getelementptr %[[addressof]][%[[c0]], %[[c0]]]
-    // CHECK-SAME: -> !llvm<"i8*">
-    // CHECK: %[[module:.*]] = llvm.call @mgpuModuleLoad(%[[binary]]) : (!llvm<"i8*">) -> !llvm<"i8*">
-    // CHECK: %[[func:.*]] = llvm.call @mgpuModuleGetFunction(%[[module]], {{.*}}) : (!llvm<"i8*">, !llvm<"i8*">) -> !llvm<"i8*">
+    // CHECK-SAME: -> !llvm.ptr<i8>
+    // CHECK: %[[module:.*]] = llvm.call @mgpuModuleLoad(%[[binary]]) : (!llvm.ptr<i8>) -> !llvm.ptr<i8>
+    // CHECK: %[[func:.*]] = llvm.call @mgpuModuleGetFunction(%[[module]], {{.*}}) : (!llvm.ptr<i8>, !llvm.ptr<i8>) -> !llvm.ptr<i8>
     // CHECK: llvm.call @mgpuStreamCreate
     // CHECK: llvm.call @mgpuLaunchKernel
     // CHECK: llvm.call @mgpuStreamSynchronize
     "gpu.launch_func"(%cst, %cst, %cst, %cst, %cst, %cst, %0, %1) { kernel = @kernel_module::@kernel }
-        : (!llvm.i64, !llvm.i64, !llvm.i64, !llvm.i64, !llvm.i64, !llvm.i64, !llvm.float, !llvm<"float*">) -> ()
+        : (!llvm.i64, !llvm.i64, !llvm.i64, !llvm.i64, !llvm.i64, !llvm.i64, !llvm.float, !llvm.ptr<float>) -> ()
 
     llvm.return
   }
diff --git a/mlir/test/Conversion/GPUCommon/memory-attrbution.mlir b/mlir/test/Conversion/GPUCommon/memory-attrbution.mlir
index c13afefdb78dc..a2764d4242058 100644
--- a/mlir/test/Conversion/GPUCommon/memory-attrbution.mlir
+++ b/mlir/test/Conversion/GPUCommon/memory-attrbution.mlir
@@ -6,13 +6,13 @@ gpu.module @kernel {
   gpu.func @private(%arg0: f32) private(%arg1: memref<4xf32, 5>) {
     // Allocate private memory inside the function.
     // NVVM: %[[size:.*]] = llvm.mlir.constant(4 : i64) : !llvm.i64
-    // NVVM: %[[raw:.*]] = llvm.alloca %[[size]] x !llvm.float : (!llvm.i64) -> !llvm<"float*">
+    // NVVM: %[[raw:.*]] = llvm.alloca %[[size]] x !llvm.float : (!llvm.i64) -> !llvm.ptr<float>
 
     // ROCDL: %[[size:.*]] = llvm.mlir.constant(4 : i64) : !llvm.i64
-    // ROCDL: %[[raw:.*]] = llvm.alloca %[[size]] x !llvm.float : (!llvm.i64) -> !llvm<"float addrspace(5)*">
+    // ROCDL: %[[raw:.*]] = llvm.alloca %[[size]] x !llvm.float : (!llvm.i64) -> !llvm.ptr<float, 5>
 
     // Populate the memref descriptor.
-    // NVVM: %[[descr1:.*]] = llvm.mlir.undef : !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }">
+    // NVVM: %[[descr1:.*]] = llvm.mlir.undef : !llvm.struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>
     // NVVM: %[[descr2:.*]] = llvm.insertvalue %[[raw]], %[[descr1]][0]
     // NVVM: %[[descr3:.*]] = llvm.insertvalue %[[raw]], %[[descr2]][1]
     // NVVM: %[[c0:.*]] = llvm.mlir.constant(0 : index) : !llvm.i64
@@ -22,7 +22,7 @@ gpu.module @kernel {
     // NVVM: %[[c1:.*]] = llvm.mlir.constant(1 : index) : !llvm.i64
     // NVVM: %[[descr6:.*]] = llvm.insertvalue %[[c1]], %[[descr5]][4, 0]
 
-    // ROCDL: %[[descr1:.*]] = llvm.mlir.undef : !llvm<"{ float addrspace(5)*, float addrspace(5)*, i64, [1 x i64], [1 x i64] }">
+    // ROCDL: %[[descr1:.*]] = llvm.mlir.undef : !llvm.struct<(ptr<float, 5>, ptr<float, 5>, i64, array<1 x i64>, array<1 x i64>)>
     // ROCDL: %[[descr2:.*]] = llvm.insertvalue %[[raw]], %[[descr1]][0]
     // ROCDL: %[[descr3:.*]] = llvm.insertvalue %[[raw]], %[[descr2]][1]
     // ROCDL: %[[c0:.*]] = llvm.mlir.constant(0 : index) : !llvm.i64
@@ -54,11 +54,11 @@ gpu.module @kernel {
   // Workgroup buffers are allocated as globals.
   // NVVM: llvm.mlir.global internal @[[$buffer:.*]]()
   // NVVM-SAME:  addr_space = 3
-  // NVVM-SAME:  !llvm<"[4 x float]">
+  // NVVM-SAME:  !llvm.array<4 x float>
 
   // ROCDL: llvm.mlir.global internal @[[$buffer:.*]]()
   // ROCDL-SAME:  addr_space = 3
-  // ROCDL-SAME:  !llvm<"[4 x float]">
+  // ROCDL-SAME:  !llvm.array<4 x float>
 
   // NVVM-LABEL: llvm.func @workgroup
   // NVVM-SAME: {
@@ -68,17 +68,17 @@ gpu.module @kernel {
   gpu.func @workgroup(%arg0: f32) workgroup(%arg1: memref<4xf32, 3>) {
     // Get the address of the first element in the global array.
     // NVVM: %[[c0:.*]] = llvm.mlir.constant(0 : i32) : !llvm.i32
-    // NVVM: %[[addr:.*]] = llvm.mlir.addressof @[[$buffer]] : !llvm<"[4 x float] addrspace(3)*">
+    // NVVM: %[[addr:.*]] = llvm.mlir.addressof @[[$buffer]] : !llvm.ptr<array<4 x float>, 3>
     // NVVM: %[[raw:.*]] = llvm.getelementptr %[[addr]][%[[c0]], %[[c0]]]
-    // NVVM-SAME: !llvm<"float addrspace(3)*">
+    // NVVM-SAME: !llvm.ptr<float, 3>
 
     // ROCDL: %[[c0:.*]] = llvm.mlir.constant(0 : i32) : !llvm.i32
-    // ROCDL: %[[addr:.*]] = llvm.mlir.addressof @[[$buffer]] : !llvm<"[4 x float] addrspace(3)*">
+    // ROCDL: %[[addr:.*]] = llvm.mlir.addressof @[[$buffer]] : !llvm.ptr<array<4 x float>, 3>
     // ROCDL: %[[raw:.*]] = llvm.getelementptr %[[addr]][%[[c0]], %[[c0]]]
-    // ROCDL-SAME: !llvm<"float addrspace(3)*">
+    // ROCDL-SAME: !llvm.ptr<float, 3>
 
     // Populate the memref descriptor.
-    // NVVM: %[[descr1:.*]] = llvm.mlir.undef : !llvm<"{ float addrspace(3)*, float addrspace(3)*, i64, [1 x i64], [1 x i64] }">
+    // NVVM: %[[descr1:.*]] = llvm.mlir.undef : !llvm.struct<(ptr<float, 3>, ptr<float, 3>, i64, array<1 x i64>, array<1 x i64>)>
     // NVVM: %[[descr2:.*]] = llvm.insertvalue %[[raw]], %[[descr1]][0]
     // NVVM: %[[descr3:.*]] = llvm.insertvalue %[[raw]], %[[descr2]][1]
     // NVVM: %[[c0:.*]] = llvm.mlir.constant(0 : index) : !llvm.i64
@@ -88,7 +88,7 @@ gpu.module @kernel {
     // NVVM: %[[c1:.*]] = llvm.mlir.constant(1 : index) : !llvm.i64
     // NVVM: %[[descr6:.*]] = llvm.insertvalue %[[c1]], %[[descr5]][4, 0]
 
-    // ROCDL: %[[descr1:.*]] = llvm.mlir.undef : !llvm<"{ float addrspace(3)*, float addrspace(3)*, i64, [1 x i64], [1 x i64] }">
+    // ROCDL: %[[descr1:.*]] = llvm.mlir.undef : !llvm.struct<(ptr<float, 3>, ptr<float, 3>, i64, array<1 x i64>, array<1 x i64>)>
     // ROCDL: %[[descr2:.*]] = llvm.insertvalue %[[raw]], %[[descr1]][0]
     // ROCDL: %[[descr3:.*]] = llvm.insertvalue %[[raw]], %[[descr2]][1]
     // ROCDL: %[[c0:.*]] = llvm.mlir.constant(0 : index) : !llvm.i64
@@ -120,28 +120,28 @@ gpu.module @kernel {
   // Check that the total size was computed correctly.
   // NVVM: llvm.mlir.global internal @[[$buffer:.*]]()
   // NVVM-SAME:  addr_space = 3
-  // NVVM-SAME:  !llvm<"[48 x float]">
+  // NVVM-SAME:  !llvm.array<48 x float>
 
   // ROCDL: llvm.mlir.global internal @[[$buffer:.*]]()
   // ROCDL-SAME:  addr_space = 3
-  // ROCDL-SAME:  !llvm<"[48 x float]">
+  // ROCDL-SAME:  !llvm.array<48 x float>
 
   // NVVM-LABEL: llvm.func @workgroup3d
   // ROCDL-LABEL: llvm.func @workgroup3d
   gpu.func @workgroup3d(%arg0: f32) workgroup(%arg1: memref<4x2x6xf32, 3>) {
     // Get the address of the first element in the global array.
     // NVVM: %[[c0:.*]] = llvm.mlir.constant(0 : i32) : !llvm.i32
-    // NVVM: %[[addr:.*]] = llvm.mlir.addressof @[[$buffer]] : !llvm<"[48 x float] addrspace(3)*">
+    // NVVM: %[[addr:.*]] = llvm.mlir.addressof @[[$buffer]] : !llvm.ptr<array<48 x float>, 3>
     // NVVM: %[[raw:.*]] = llvm.getelementptr %[[addr]][%[[c0]], %[[c0]]]
-    // NVVM-SAME: !llvm<"float addrspace(3)*">
+    // NVVM-SAME: !llvm.ptr<float, 3>
 
     // ROCDL: %[[c0:.*]] = llvm.mlir.constant(0 : i32) : !llvm.i32
-    // ROCDL: %[[addr:.*]] = llvm.mlir.addressof @[[$buffer]] : !llvm<"[48 x float] addrspace(3)*">
+    // ROCDL: %[[addr:.*]] = llvm.mlir.addressof @[[$buffer]] : !llvm.ptr<array<48 x float>, 3>
     // ROCDL: %[[raw:.*]] = llvm.getelementptr %[[addr]][%[[c0]], %[[c0]]]
-    // ROCDL-SAME: !llvm<"float addrspace(3)*">
+    // ROCDL-SAME: !llvm.ptr<float, 3>
 
     // Populate the memref descriptor.
-    // NVVM: %[[descr1:.*]] = llvm.mlir.undef : !llvm<"{ float addrspace(3)*, float addrspace(3)*, i64, [3 x i64], [3 x i64] }">
+    // NVVM: %[[descr1:.*]] = llvm.mlir.undef : !llvm.struct<(ptr<float, 3>, ptr<float, 3>, i64, array<3 x i64>, array<3 x i64>)>
     // NVVM: %[[descr2:.*]] = llvm.insertvalue %[[raw]], %[[descr1]][0]
     // NVVM: %[[descr3:.*]] = llvm.insertvalue %[[raw]], %[[descr2]][1]
     // NVVM: %[[c0:.*]] = llvm.mlir.constant(0 : index) : !llvm.i64
@@ -159,7 +159,7 @@ gpu.module @kernel {
     // NVVM: %[[c1:.*]] = llvm.mlir.constant(1 : index) : !llvm.i64
     // NVVM: %[[descr10:.*]] = llvm.insertvalue %[[c1]], %[[descr9]][4, 2]
 
-    // ROCDL: %[[descr1:.*]] = llvm.mlir.undef : !llvm<"{ float addrspace(3)*, float addrspace(3)*, i64, [3 x i64], [3 x i64] }">
+    // ROCDL: %[[descr1:.*]] = llvm.mlir.undef : !llvm.struct<(ptr<float, 3>, ptr<float, 3>, i64, array<3 x i64>, array<3 x i64>)>
     // ROCDL: %[[descr2:.*]] = llvm.insertvalue %[[raw]], %[[descr1]][0]
     // ROCDL: %[[descr3:.*]] = llvm.insertvalue %[[raw]], %[[descr2]][1]
     // ROCDL: %[[c0:.*]] = llvm.mlir.constant(0 : index) : !llvm.i64
@@ -188,14 +188,14 @@ gpu.module @kernel {
 gpu.module @kernel {
   // Check that several buffers are defined.
   // NVVM: llvm.mlir.global internal @[[$buffer1:.*]]()
-  // NVVM-SAME:  !llvm<"[1 x float]">
+  // NVVM-SAME:  !llvm.array<1 x float>
   // NVVM: llvm.mlir.global internal @[[$buffer2:.*]]()
-  // NVVM-SAME:  !llvm<"[2 x float]">
+  // NVVM-SAME:  !llvm.array<2 x float>
 
   // ROCDL: llvm.mlir.global internal @[[$buffer1:.*]]()
-  // ROCDL-SAME:  !llvm<"[1 x float]">
+  // ROCDL-SAME:  !llvm.array<1 x float>
   // ROCDL: llvm.mlir.global internal @[[$buffer2:.*]]()
-  // ROCDL-SAME:  !llvm<"[2 x float]">
+  // ROCDL-SAME:  !llvm.array<2 x float>
 
   // NVVM-LABEL: llvm.func @multiple
   // ROCDL-LABEL: llvm.func @multiple
@@ -212,14 +212,14 @@ gpu.module @kernel {
 
     // Private buffers.
     // NVVM: %[[c3:.*]] = llvm.mlir.constant(3 : i64)
-    // NVVM: llvm.alloca %[[c3]] x !llvm.float : (!llvm.i64) -> !llvm<"float*">
+    // NVVM: llvm.alloca %[[c3]] x !llvm.float : (!llvm.i64) -> !llvm.ptr<float>
     // NVVM: %[[c4:.*]] = llvm.mlir.constant(4 : i64)
-    // NVVM: llvm.alloca %[[c4]] x !llvm.float : (!llvm.i64) -> !llvm<"float*">
+    // NVVM: llvm.alloca %[[c4]] x !llvm.float : (!llvm.i64) -> !llvm.ptr<float>
 
     // ROCDL: %[[c3:.*]] = llvm.mlir.constant(3 : i64)
-    // ROCDL: llvm.alloca %[[c3]] x !llvm.float : (!llvm.i64) -> !llvm<"float addrspace(5)*">
+    // ROCDL: llvm.alloca %[[c3]] x !llvm.float : (!llvm.i64) -> !llvm.ptr<float, 5>
     // ROCDL: %[[c4:.*]] = llvm.mlir.constant(4 : i64)
-    // ROCDL: llvm.alloca %[[c4]] x !llvm.float : (!llvm.i64) -> !llvm<"float addrspace(5)*">
+    // ROCDL: llvm.alloca %[[c4]] x !llvm.float : (!llvm.i64) -> !llvm.ptr<float, 5>
 
     %c0 = constant 0 : index
     store %arg0, %arg1[%c0] : memref<1xf32, 3>
diff --git a/mlir/test/Conversion/GPUToCUDA/lower-nvvm-kernel-to-cubin.mlir b/mlir/test/Conversion/GPUToCUDA/lower-nvvm-kernel-to-cubin.mlir
index 78b9f56b6202c..d103031fbd54d 100644
--- a/mlir/test/Conversion/GPUToCUDA/lower-nvvm-kernel-to-cubin.mlir
+++ b/mlir/test/Conversion/GPUToCUDA/lower-nvvm-kernel-to-cubin.mlir
@@ -2,7 +2,7 @@
 
 // CHECK: attributes {nvvm.cubin = "CUBIN"}
 gpu.module @foo {
-  llvm.func @kernel(%arg0 : !llvm.float, %arg1 : !llvm<"float*">)
+  llvm.func @kernel(%arg0 : !llvm.float, %arg1 : !llvm.ptr<float>)
     // CHECK: attributes  {gpu.kernel}
     attributes  { gpu.kernel } {
     llvm.return
diff --git a/mlir/test/Conversion/GPUToNVVM/gpu-to-nvvm.mlir b/mlir/test/Conversion/GPUToNVVM/gpu-to-nvvm.mlir
index fa9a478c1d83b..df38df1749cbc 100644
--- a/mlir/test/Conversion/GPUToNVVM/gpu-to-nvvm.mlir
+++ b/mlir/test/Conversion/GPUToNVVM/gpu-to-nvvm.mlir
@@ -120,9 +120,9 @@ gpu.module @test_module {
     // CHECK: %[[#SHL:]] = llvm.shl %[[#ONE]], %[[#WIDTH]] : !llvm.i32
     // CHECK: %[[#MASK:]] = llvm.sub %[[#SHL]], %[[#ONE]] : !llvm.i32
     // CHECK: %[[#CLAMP:]] = llvm.sub %[[#WIDTH]], %[[#ONE]] : !llvm.i32
-    // CHECK: %[[#SHFL:]] = nvvm.shfl.sync.bfly %[[#MASK]], %[[#VALUE]], %[[#OFFSET]], %[[#CLAMP]] : !llvm<"{ float, i1 }">
-    // CHECK: llvm.extractvalue %[[#SHFL]][0 : index] : !llvm<"{ float, i1 }">
-    // CHECK: llvm.extractvalue %[[#SHFL]][1 : index] : !llvm<"{ float, i1 }">
+    // CHECK: %[[#SHFL:]] = nvvm.shfl.sync.bfly %[[#MASK]], %[[#VALUE]], %[[#OFFSET]], %[[#CLAMP]] : !llvm.struct<(float, i1)>
+    // CHECK: llvm.extractvalue %[[#SHFL]][0 : index] : !llvm.struct<(float, i1)>
+    // CHECK: llvm.extractvalue %[[#SHFL]][1 : index] : !llvm.struct<(float, i1)>
     %shfl, %pred = "gpu.shuffle"(%arg0, %arg1, %arg2) { mode = "xor" } : (f32, i32, i32) -> (f32, i1)
 
     std.return %shfl : f32
diff --git a/mlir/test/Conversion/GPUToROCm/lower-rocdl-kernel-to-hsaco.mlir b/mlir/test/Conversion/GPUToROCm/lower-rocdl-kernel-to-hsaco.mlir
index 5ee3bb21aa916..d88f842a2186f 100644
--- a/mlir/test/Conversion/GPUToROCm/lower-rocdl-kernel-to-hsaco.mlir
+++ b/mlir/test/Conversion/GPUToROCm/lower-rocdl-kernel-to-hsaco.mlir
@@ -2,7 +2,7 @@
 
 // CHECK: attributes {rocdl.hsaco = "HSACO"}
 gpu.module @foo {
-  llvm.func @kernel(%arg0 : !llvm.float, %arg1 : !llvm<"float*">)
+  llvm.func @kernel(%arg0 : !llvm.float, %arg1 : !llvm.ptr<float>)
     // CHECK: attributes  {gpu.kernel}
     attributes  { gpu.kernel } {
     llvm.return
diff --git a/mlir/test/Conversion/GPUToVulkan/invoke-vulkan.mlir b/mlir/test/Conversion/GPUToVulkan/invoke-vulkan.mlir
index f944d007ebaa9..e39bfa41e692b 100644
--- a/mlir/test/Conversion/GPUToVulkan/invoke-vulkan.mlir
+++ b/mlir/test/Conversion/GPUToVulkan/invoke-vulkan.mlir
@@ -2,60 +2,60 @@
 
 // CHECK: llvm.mlir.global internal constant @kernel_spv_entry_point_name
 // CHECK: llvm.mlir.global internal constant @SPIRV_BIN
-// CHECK: %[[Vulkan_Runtime_ptr:.*]] = llvm.call @initVulkan() : () -> !llvm<"i8*">
+// CHECK: %[[Vulkan_Runtime_ptr:.*]] = llvm.call @initVulkan() : () -> !llvm.ptr<i8>
 // CHECK: %[[addressof_SPIRV_BIN:.*]] = llvm.mlir.addressof @SPIRV_BIN
 // CHECK: %[[SPIRV_BIN_ptr:.*]] = llvm.getelementptr %[[addressof_SPIRV_BIN]]
 // CHECK: %[[SPIRV_BIN_size:.*]] = llvm.mlir.constant
-// CHECK: llvm.call @bindMemRef1DFloat(%[[Vulkan_Runtime_ptr]], %{{.*}}, %{{.*}}, %{{.*}}) : (!llvm<"i8*">, !llvm.i32, !llvm.i32, !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }*">) -> !llvm.void
-// CHECK: llvm.call @setBinaryShader(%[[Vulkan_Runtime_ptr]], %[[SPIRV_BIN_ptr]], %[[SPIRV_BIN_size]]) : (!llvm<"i8*">, !llvm<"i8*">, !llvm.i32) -> !llvm.void
+// CHECK: llvm.call @bindMemRef1DFloat(%[[Vulkan_Runtime_ptr]], %{{.*}}, %{{.*}}, %{{.*}}) : (!llvm.ptr<i8>, !llvm.i32, !llvm.i32, !llvm.ptr<struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>>) -> !llvm.void
+// CHECK: llvm.call @setBinaryShader(%[[Vulkan_Runtime_ptr]], %[[SPIRV_BIN_ptr]], %[[SPIRV_BIN_size]]) : (!llvm.ptr<i8>, !llvm.ptr<i8>, !llvm.i32) -> !llvm.void
 // CHECK: %[[addressof_entry_point:.*]] = llvm.mlir.addressof @kernel_spv_entry_point_name
 // CHECK: %[[entry_point_ptr:.*]] = llvm.getelementptr %[[addressof_entry_point]]
-// CHECK: llvm.call @setEntryPoint(%[[Vulkan_Runtime_ptr]], %[[entry_point_ptr]]) : (!llvm<"i8*">, !llvm<"i8*">) -> !llvm.void
-// CHECK: llvm.call @setNumWorkGroups(%[[Vulkan_Runtime_ptr]], %{{.*}}, %{{.*}}, %{{.*}}) : (!llvm<"i8*">, !llvm.i64, !llvm.i64, !llvm.i64) -> !llvm.void
-// CHECK: llvm.call @runOnVulkan(%[[Vulkan_Runtime_ptr]]) : (!llvm<"i8*">) -> !llvm.void
-// CHECK: llvm.call @deinitVulkan(%[[Vulkan_Runtime_ptr]]) : (!llvm<"i8*">) -> !llvm.void
+// CHECK: llvm.call @setEntryPoint(%[[Vulkan_Runtime_ptr]], %[[entry_point_ptr]]) : (!llvm.ptr<i8>, !llvm.ptr<i8>) -> !llvm.void
+// CHECK: llvm.call @setNumWorkGroups(%[[Vulkan_Runtime_ptr]], %{{.*}}, %{{.*}}, %{{.*}}) : (!llvm.ptr<i8>, !llvm.i64, !llvm.i64, !llvm.i64) -> !llvm.void
+// CHECK: llvm.call @runOnVulkan(%[[Vulkan_Runtime_ptr]]) : (!llvm.ptr<i8>) -> !llvm.void
+// CHECK: llvm.call @deinitVulkan(%[[Vulkan_Runtime_ptr]]) : (!llvm.ptr<i8>) -> !llvm.void
 
 module attributes {gpu.container_module} {
-  llvm.func @malloc(!llvm.i64) -> !llvm<"i8*">
+  llvm.func @malloc(!llvm.i64) -> !llvm.ptr<i8>
   llvm.func @foo() {
     %0 = llvm.mlir.constant(12 : index) : !llvm.i64
-    %1 = llvm.mlir.null : !llvm<"float*">
+    %1 = llvm.mlir.null : !llvm.ptr<float>
     %2 = llvm.mlir.constant(1 : index) : !llvm.i64
-    %3 = llvm.getelementptr %1[%2] : (!llvm<"float*">, !llvm.i64) -> !llvm<"float*">
-    %4 = llvm.ptrtoint %3 : !llvm<"float*"> to !llvm.i64
+    %3 = llvm.getelementptr %1[%2] : (!llvm.ptr<float>, !llvm.i64) -> !llvm.ptr<float>
+    %4 = llvm.ptrtoint %3 : !llvm.ptr<float> to !llvm.i64
     %5 = llvm.mul %0, %4 : !llvm.i64
-    %6 = llvm.call @malloc(%5) : (!llvm.i64) -> !llvm<"i8*">
-    %7 = llvm.bitcast %6 : !llvm<"i8*"> to !llvm<"float*">
-    %8 = llvm.mlir.undef : !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }">
-    %9 = llvm.insertvalue %7, %8[0] : !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }">
-    %10 = llvm.insertvalue %7, %9[1] : !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }">
+    %6 = llvm.call @malloc(%5) : (!llvm.i64) -> !llvm.ptr<i8>
+    %7 = llvm.bitcast %6 : !llvm.ptr<i8> to !llvm.ptr<float>
+    %8 = llvm.mlir.undef : !llvm.struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>
+    %9 = llvm.insertvalue %7, %8[0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>
+    %10 = llvm.insertvalue %7, %9[1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>
     %11 = llvm.mlir.constant(0 : index) : !llvm.i64
-    %12 = llvm.insertvalue %11, %10[2] : !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }">
+    %12 = llvm.insertvalue %11, %10[2] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>
     %13 = llvm.mlir.constant(1 : index) : !llvm.i64
-    %14 = llvm.insertvalue %0, %12[3, 0] : !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }">
-    %15 = llvm.insertvalue %13, %14[4, 0] : !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }">
+    %14 = llvm.insertvalue %0, %12[3, 0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>
+    %15 = llvm.insertvalue %13, %14[4, 0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>
     %16 = llvm.mlir.constant(1 : index) : !llvm.i64
-    %17 = llvm.extractvalue %15[0] : !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }">
-    %18 = llvm.extractvalue %15[1] : !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }">
-    %19 = llvm.extractvalue %15[2] : !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }">
-    %20 = llvm.extractvalue %15[3, 0] : !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }">
-    %21 = llvm.extractvalue %15[4, 0] : !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }">
+    %17 = llvm.extractvalue %15[0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>
+    %18 = llvm.extractvalue %15[1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>
+    %19 = llvm.extractvalue %15[2] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>
+    %20 = llvm.extractvalue %15[3, 0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>
+    %21 = llvm.extractvalue %15[4, 0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>
     llvm.call @vulkanLaunch(%16, %16, %16, %17, %18, %19, %20, %21) {spirv_blob = "\03\02#\07\00", spirv_entry_point = "kernel"}
-    : (!llvm.i64, !llvm.i64, !llvm.i64, !llvm<"float*">, !llvm<"float*">, !llvm.i64, !llvm.i64, !llvm.i64) -> ()
+    : (!llvm.i64, !llvm.i64, !llvm.i64, !llvm.ptr<float>, !llvm.ptr<float>, !llvm.i64, !llvm.i64, !llvm.i64) -> ()
     llvm.return
   }
-  llvm.func @vulkanLaunch(%arg0: !llvm.i64, %arg1: !llvm.i64, %arg2: !llvm.i64, %arg6: !llvm<"float*">, %arg7: !llvm<"float*">, %arg8: !llvm.i64, %arg9: !llvm.i64, %arg10: !llvm.i64) {
-    %0 = llvm.mlir.undef : !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }">
-    %1 = llvm.insertvalue %arg6, %0[0] : !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }">
-    %2 = llvm.insertvalue %arg7, %1[1] : !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }">
-    %3 = llvm.insertvalue %arg8, %2[2] : !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }">
-    %4 = llvm.insertvalue %arg9, %3[3, 0] : !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }">
-    %5 = llvm.insertvalue %arg10, %4[4, 0] : !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }">
+  llvm.func @vulkanLaunch(%arg0: !llvm.i64, %arg1: !llvm.i64, %arg2: !llvm.i64, %arg6: !llvm.ptr<float>, %arg7: !llvm.ptr<float>, %arg8: !llvm.i64, %arg9: !llvm.i64, %arg10: !llvm.i64) {
+    %0 = llvm.mlir.undef : !llvm.struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>
+    %1 = llvm.insertvalue %arg6, %0[0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>
+    %2 = llvm.insertvalue %arg7, %1[1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>
+    %3 = llvm.insertvalue %arg8, %2[2] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>
+    %4 = llvm.insertvalue %arg9, %3[3, 0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>
+    %5 = llvm.insertvalue %arg10, %4[4, 0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>
     %6 = llvm.mlir.constant(1 : index) : !llvm.i64
-    %7 = llvm.alloca %6 x !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }"> : (!llvm.i64) -> !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }*">
-    llvm.store %5, %7 : !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }*">
-    llvm.call @_mlir_ciface_vulkanLaunch(%arg0, %arg1, %arg2, %7) : (!llvm.i64, !llvm.i64, !llvm.i64, !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }*">) -> ()
+    %7 = llvm.alloca %6 x !llvm.struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)> : (!llvm.i64) -> !llvm.ptr<struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>>
+    llvm.store %5, %7 : !llvm.ptr<struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>>
+    llvm.call @_mlir_ciface_vulkanLaunch(%arg0, %arg1, %arg2, %7) : (!llvm.i64, !llvm.i64, !llvm.i64, !llvm.ptr<struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>>) -> ()
     llvm.return
   }
-  llvm.func @_mlir_ciface_vulkanLaunch(!llvm.i64, !llvm.i64, !llvm.i64, !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }*">)
+  llvm.func @_mlir_ciface_vulkanLaunch(!llvm.i64, !llvm.i64, !llvm.i64, !llvm.ptr<struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>>)
 }
diff --git a/mlir/test/Conversion/SPIRVToLLVM/arithmetic-ops-to-llvm.mlir b/mlir/test/Conversion/SPIRVToLLVM/arithmetic-ops-to-llvm.mlir
index 8ae0d98407131..a37e82bf59953 100644
--- a/mlir/test/Conversion/SPIRVToLLVM/arithmetic-ops-to-llvm.mlir
+++ b/mlir/test/Conversion/SPIRVToLLVM/arithmetic-ops-to-llvm.mlir
@@ -13,7 +13,7 @@ func @iadd_scalar(%arg0: i32, %arg1: i32) {
 
 // CHECK-LABEL: @iadd_vector
 func @iadd_vector(%arg0: vector<4xi64>, %arg1: vector<4xi64>) {
-  // CHECK: llvm.add %{{.*}}, %{{.*}} : !llvm<"<4 x i64>">
+  // CHECK: llvm.add %{{.*}}, %{{.*}} : !llvm.vec<4 x i64>
   %0 = spv.IAdd %arg0, %arg1 : vector<4xi64>
   return
 }
@@ -31,7 +31,7 @@ func @isub_scalar(%arg0: i8, %arg1: i8) {
 
 // CHECK-LABEL: @isub_vector
 func @isub_vector(%arg0: vector<2xi16>, %arg1: vector<2xi16>) {
-  // CHECK: llvm.sub %{{.*}}, %{{.*}} : !llvm<"<2 x i16>">
+  // CHECK: llvm.sub %{{.*}}, %{{.*}} : !llvm.vec<2 x i16>
   %0 = spv.ISub %arg0, %arg1 : vector<2xi16>
   return
 }
@@ -49,7 +49,7 @@ func @imul_scalar(%arg0: i32, %arg1: i32) {
 
 // CHECK-LABEL: @imul_vector
 func @imul_vector(%arg0: vector<3xi32>, %arg1: vector<3xi32>) {
-  // CHECK: llvm.mul %{{.*}}, %{{.*}} : !llvm<"<3 x i32>">
+  // CHECK: llvm.mul %{{.*}}, %{{.*}} : !llvm.vec<3 x i32>
   %0 = spv.IMul %arg0, %arg1 : vector<3xi32>
   return
 }
@@ -67,7 +67,7 @@ func @fadd_scalar(%arg0: f16, %arg1: f16) {
 
 // CHECK-LABEL: @fadd_vector
 func @fadd_vector(%arg0: vector<4xf32>, %arg1: vector<4xf32>) {
-  // CHECK: llvm.fadd %{{.*}}, %{{.*}} : !llvm<"<4 x float>">
+  // CHECK: llvm.fadd %{{.*}}, %{{.*}} : !llvm.vec<4 x float>
   %0 = spv.FAdd %arg0, %arg1 : vector<4xf32>
   return
 }
@@ -85,7 +85,7 @@ func @fsub_scalar(%arg0: f32, %arg1: f32) {
 
 // CHECK-LABEL: @fsub_vector
 func @fsub_vector(%arg0: vector<2xf32>, %arg1: vector<2xf32>) {
-  // CHECK: llvm.fsub %{{.*}}, %{{.*}} : !llvm<"<2 x float>">
+  // CHECK: llvm.fsub %{{.*}}, %{{.*}} : !llvm.vec<2 x float>
   %0 = spv.FSub %arg0, %arg1 : vector<2xf32>
   return
 }
@@ -103,7 +103,7 @@ func @fdiv_scalar(%arg0: f32, %arg1: f32) {
 
 // CHECK-LABEL: @fdiv_vector
 func @fdiv_vector(%arg0: vector<3xf64>, %arg1: vector<3xf64>) {
-  // CHECK: llvm.fdiv %{{.*}}, %{{.*}} : !llvm<"<3 x double>">
+  // CHECK: llvm.fdiv %{{.*}}, %{{.*}} : !llvm.vec<3 x double>
   %0 = spv.FDiv %arg0, %arg1 : vector<3xf64>
   return
 }
@@ -121,7 +121,7 @@ func @fmul_scalar(%arg0: f32, %arg1: f32) {
 
 // CHECK-LABEL: @fmul_vector
 func @fmul_vector(%arg0: vector<2xf32>, %arg1: vector<2xf32>) {
-  // CHECK: llvm.fmul %{{.*}}, %{{.*}} : !llvm<"<2 x float>">
+  // CHECK: llvm.fmul %{{.*}}, %{{.*}} : !llvm.vec<2 x float>
   %0 = spv.FMul %arg0, %arg1 : vector<2xf32>
   return
 }
@@ -139,7 +139,7 @@ func @frem_scalar(%arg0: f32, %arg1: f32) {
 
 // CHECK-LABEL: @frem_vector
 func @frem_vector(%arg0: vector<3xf64>, %arg1: vector<3xf64>) {
-  // CHECK: llvm.frem %{{.*}}, %{{.*}} : !llvm<"<3 x double>">
+  // CHECK: llvm.frem %{{.*}}, %{{.*}} : !llvm.vec<3 x double>
   %0 = spv.FRem %arg0, %arg1 : vector<3xf64>
   return
 }
@@ -157,7 +157,7 @@ func @fneg_scalar(%arg: f64) {
 
 // CHECK-LABEL: @fneg_vector
 func @fneg_vector(%arg: vector<2xf32>) {
-  // CHECK: llvm.fneg %{{.*}} : !llvm<"<2 x float>">
+  // CHECK: llvm.fneg %{{.*}} : !llvm.vec<2 x float>
   %0 = spv.FNegate %arg : vector<2xf32>
   return
 }
@@ -175,7 +175,7 @@ func @udiv_scalar(%arg0: i32, %arg1: i32) {
 
 // CHECK-LABEL: @udiv_vector
 func @udiv_vector(%arg0: vector<3xi64>, %arg1: vector<3xi64>) {
-  // CHECK: llvm.udiv %{{.*}}, %{{.*}} : !llvm<"<3 x i64>">
+  // CHECK: llvm.udiv %{{.*}}, %{{.*}} : !llvm.vec<3 x i64>
   %0 = spv.UDiv %arg0, %arg1 : vector<3xi64>
   return
 }
@@ -193,7 +193,7 @@ func @umod_scalar(%arg0: i32, %arg1: i32) {
 
 // CHECK-LABEL: @umod_vector
 func @umod_vector(%arg0: vector<3xi64>, %arg1: vector<3xi64>) {
-  // CHECK: llvm.urem %{{.*}}, %{{.*}} : !llvm<"<3 x i64>">
+  // CHECK: llvm.urem %{{.*}}, %{{.*}} : !llvm.vec<3 x i64>
   %0 = spv.UMod %arg0, %arg1 : vector<3xi64>
   return
 }
@@ -211,7 +211,7 @@ func @sdiv_scalar(%arg0: i16, %arg1: i16) {
 
 // CHECK-LABEL: @sdiv_vector
 func @sdiv_vector(%arg0: vector<2xi64>, %arg1: vector<2xi64>) {
-  // CHECK: llvm.sdiv %{{.*}}, %{{.*}} : !llvm<"<2 x i64>">
+  // CHECK: llvm.sdiv %{{.*}}, %{{.*}} : !llvm.vec<2 x i64>
   %0 = spv.SDiv %arg0, %arg1 : vector<2xi64>
   return
 }
@@ -229,7 +229,7 @@ func @srem_scalar(%arg0: i32, %arg1: i32) {
 
 // CHECK-LABEL: @srem_vector
 func @srem_vector(%arg0: vector<4xi32>, %arg1: vector<4xi32>) {
-  // CHECK: llvm.srem %{{.*}}, %{{.*}} : !llvm<"<4 x i32>">
+  // CHECK: llvm.srem %{{.*}}, %{{.*}} : !llvm.vec<4 x i32>
   %0 = spv.SRem %arg0, %arg1 : vector<4xi32>
   return
 }
diff --git a/mlir/test/Conversion/SPIRVToLLVM/bitwise-ops-to-llvm.mlir b/mlir/test/Conversion/SPIRVToLLVM/bitwise-ops-to-llvm.mlir
index aed82d218db21..e37072c7282da 100644
--- a/mlir/test/Conversion/SPIRVToLLVM/bitwise-ops-to-llvm.mlir
+++ b/mlir/test/Conversion/SPIRVToLLVM/bitwise-ops-to-llvm.mlir
@@ -13,7 +13,7 @@ func @bitcount_scalar(%arg0: i16) {
 
 // CHECK-LABEL: @bitcount_vector
 func @bitcount_vector(%arg0: vector<3xi32>) {
-  // CHECK: "llvm.intr.ctpop"(%{{.*}}) : (!llvm<"<3 x i32>">) -> !llvm<"<3 x i32>">
+  // CHECK: "llvm.intr.ctpop"(%{{.*}}) : (!llvm.vec<3 x i32>) -> !llvm.vec<3 x i32>
   %0 = spv.BitCount %arg0: vector<3xi32>
   return
 }
@@ -31,7 +31,7 @@ func @bitreverse_scalar(%arg0: i64) {
 
 // CHECK-LABEL: @bitreverse_vector
 func @bitreverse_vector(%arg0: vector<4xi32>) {
-  // CHECK: "llvm.intr.bitreverse"(%{{.*}}) : (!llvm<"<4 x i32>">) -> !llvm<"<4 x i32>">
+  // CHECK: "llvm.intr.bitreverse"(%{{.*}}) : (!llvm.vec<4 x i32>) -> !llvm.vec<4 x i32>
   %0 = spv.BitReverse %arg0: vector<4xi32>
   return
 }
@@ -89,27 +89,27 @@ func @bitfield_insert_scalar_greater_bit_width(%base: i16, %insert: i16, %offset
   return
 }
 
-// CHECK-LABEL: @bitfield_insert_vector
-// CHECK-SAME: %[[BASE:.*]]: !llvm<"<2 x i32>">, %[[INSERT:.*]]: !llvm<"<2 x i32>">, %[[OFFSET:.*]]: !llvm.i32, %[[COUNT:.*]]: !llvm.i32
+// CHECK-LABEL: func @bitfield_insert_vector
+// CHECK-SAME: %[[BASE:.*]]: !llvm.vec<2 x i32>, %[[INSERT:.*]]: !llvm.vec<2 x i32>, %[[OFFSET:.*]]: !llvm.i32, %[[COUNT:.*]]: !llvm.i32
 func @bitfield_insert_vector(%base: vector<2xi32>, %insert: vector<2xi32>, %offset: i32, %count: i32) {
-  // CHECK: %[[OFFSET_V0:.*]] = llvm.mlir.undef : !llvm<"<2 x i32>">
+  // CHECK: %[[OFFSET_V0:.*]] = llvm.mlir.undef : !llvm.vec<2 x i32>
   // CHECK: %[[ZERO:.*]] = llvm.mlir.constant(0 : i32) : !llvm.i32
-  // CHECK: %[[OFFSET_V1:.*]] = llvm.insertelement %[[OFFSET]], %[[OFFSET_V0]][%[[ZERO]] : !llvm.i32] : !llvm<"<2 x i32>">
+  // CHECK: %[[OFFSET_V1:.*]] = llvm.insertelement %[[OFFSET]], %[[OFFSET_V0]][%[[ZERO]] : !llvm.i32] : !llvm.vec<2 x i32>
   // CHECK: %[[ONE:.*]] = llvm.mlir.constant(1 : i32) : !llvm.i32
-  // CHECK: %[[OFFSET_V2:.*]] = llvm.insertelement  %[[OFFSET]], %[[OFFSET_V1]][%[[ONE]] : !llvm.i32] : !llvm<"<2 x i32>">
-  // CHECK: %[[COUNT_V0:.*]] = llvm.mlir.undef : !llvm<"<2 x i32>">
+  // CHECK: %[[OFFSET_V2:.*]] = llvm.insertelement  %[[OFFSET]], %[[OFFSET_V1]][%[[ONE]] : !llvm.i32] : !llvm.vec<2 x i32>
+  // CHECK: %[[COUNT_V0:.*]] = llvm.mlir.undef : !llvm.vec<2 x i32>
   // CHECK: %[[ZERO:.*]] = llvm.mlir.constant(0 : i32) : !llvm.i32
-  // CHECK: %[[COUNT_V1:.*]] = llvm.insertelement %[[COUNT]], %[[COUNT_V0]][%[[ZERO]] : !llvm.i32] : !llvm<"<2 x i32>">
+  // CHECK: %[[COUNT_V1:.*]] = llvm.insertelement %[[COUNT]], %[[COUNT_V0]][%[[ZERO]] : !llvm.i32] : !llvm.vec<2 x i32>
   // CHECK: %[[ONE:.*]] = llvm.mlir.constant(1 : i32) : !llvm.i32
-  // CHECK: %[[COUNT_V2:.*]] = llvm.insertelement %[[COUNT]], %[[COUNT_V1]][%[[ONE]] : !llvm.i32] : !llvm<"<2 x i32>">
-  // CHECK: %[[MINUS_ONE:.*]] = llvm.mlir.constant(dense<-1> : vector<2xi32>) : !llvm<"<2 x i32>">
-  // CHECK: %[[T0:.*]] = llvm.shl %[[MINUS_ONE]], %[[COUNT_V2]] : !llvm<"<2 x i32>">
-  // CHECK: %[[T1:.*]] = llvm.xor %[[T0]], %[[MINUS_ONE]] : !llvm<"<2 x i32>">
-  // CHECK: %[[T2:.*]] = llvm.shl %[[T1]], %[[OFFSET_V2]] : !llvm<"<2 x i32>">
-  // CHECK: %[[MASK:.*]] = llvm.xor %[[T2]], %[[MINUS_ONE]] : !llvm<"<2 x i32>">
-  // CHECK: %[[NEW_BASE:.*]] = llvm.and %[[BASE]], %[[MASK]] : !llvm<"<2 x i32>">
-  // CHECK: %[[SHIFTED_INSERT:.*]] = llvm.shl %[[INSERT]], %[[OFFSET_V2]] : !llvm<"<2 x i32>">
-  // CHECK: llvm.or %[[NEW_BASE]], %[[SHIFTED_INSERT]] : !llvm<"<2 x i32>">
+  // CHECK: %[[COUNT_V2:.*]] = llvm.insertelement %[[COUNT]], %[[COUNT_V1]][%[[ONE]] : !llvm.i32] : !llvm.vec<2 x i32>
+  // CHECK: %[[MINUS_ONE:.*]] = llvm.mlir.constant(dense<-1> : vector<2xi32>) : !llvm.vec<2 x i32>
+  // CHECK: %[[T0:.*]] = llvm.shl %[[MINUS_ONE]], %[[COUNT_V2]] : !llvm.vec<2 x i32>
+  // CHECK: %[[T1:.*]] = llvm.xor %[[T0]], %[[MINUS_ONE]] : !llvm.vec<2 x i32>
+  // CHECK: %[[T2:.*]] = llvm.shl %[[T1]], %[[OFFSET_V2]] : !llvm.vec<2 x i32>
+  // CHECK: %[[MASK:.*]] = llvm.xor %[[T2]], %[[MINUS_ONE]] : !llvm.vec<2 x i32>
+  // CHECK: %[[NEW_BASE:.*]] = llvm.and %[[BASE]], %[[MASK]] : !llvm.vec<2 x i32>
+  // CHECK: %[[SHIFTED_INSERT:.*]] = llvm.shl %[[INSERT]], %[[OFFSET_V2]] : !llvm.vec<2 x i32>
+  // CHECK: llvm.or %[[NEW_BASE]], %[[SHIFTED_INSERT]] : !llvm.vec<2 x i32>
   %0 = spv.BitFieldInsert %base, %insert, %offset, %count : vector<2xi32>, i32, i32
   return
 }
@@ -161,25 +161,25 @@ func @bitfield_sextract_scalar_greater_bit_width(%base: i32, %offset: i64, %coun
   return
 }
 
-// CHECK-LABEL: @bitfield_sextract_vector
-// CHECK-SAME: %[[BASE:.*]]: !llvm<"<2 x i32>">, %[[OFFSET:.*]]: !llvm.i32, %[[COUNT:.*]]: !llvm.i32
+// CHECK-LABEL: func @bitfield_sextract_vector
+// CHECK-SAME: %[[BASE:.*]]: !llvm.vec<2 x i32>, %[[OFFSET:.*]]: !llvm.i32, %[[COUNT:.*]]: !llvm.i32
 func @bitfield_sextract_vector(%base: vector<2xi32>, %offset: i32, %count: i32) {
-  // CHECK: %[[OFFSET_V0:.*]] = llvm.mlir.undef : !llvm<"<2 x i32>">
+  // CHECK: %[[OFFSET_V0:.*]] = llvm.mlir.undef : !llvm.vec<2 x i32>
   // CHECK: %[[ZERO:.*]] = llvm.mlir.constant(0 : i32) : !llvm.i32
-  // CHECK: %[[OFFSET_V1:.*]] = llvm.insertelement %[[OFFSET]], %[[OFFSET_V0]][%[[ZERO]] : !llvm.i32] : !llvm<"<2 x i32>">
+  // CHECK: %[[OFFSET_V1:.*]] = llvm.insertelement %[[OFFSET]], %[[OFFSET_V0]][%[[ZERO]] : !llvm.i32] : !llvm.vec<2 x i32>
   // CHECK: %[[ONE:.*]] = llvm.mlir.constant(1 : i32) : !llvm.i32
-  // CHECK: %[[OFFSET_V2:.*]] = llvm.insertelement  %[[OFFSET]], %[[OFFSET_V1]][%[[ONE]] : !llvm.i32] : !llvm<"<2 x i32>">
-  // CHECK: %[[COUNT_V0:.*]] = llvm.mlir.undef : !llvm<"<2 x i32>">
+  // CHECK: %[[OFFSET_V2:.*]] = llvm.insertelement  %[[OFFSET]], %[[OFFSET_V1]][%[[ONE]] : !llvm.i32] : !llvm.vec<2 x i32>
+  // CHECK: %[[COUNT_V0:.*]] = llvm.mlir.undef : !llvm.vec<2 x i32>
   // CHECK: %[[ZERO:.*]] = llvm.mlir.constant(0 : i32) : !llvm.i32
-  // CHECK: %[[COUNT_V1:.*]] = llvm.insertelement %[[COUNT]], %[[COUNT_V0]][%[[ZERO]] : !llvm.i32] : !llvm<"<2 x i32>">
+  // CHECK: %[[COUNT_V1:.*]] = llvm.insertelement %[[COUNT]], %[[COUNT_V0]][%[[ZERO]] : !llvm.i32] : !llvm.vec<2 x i32>
   // CHECK: %[[ONE:.*]] = llvm.mlir.constant(1 : i32) : !llvm.i32
-  // CHECK: %[[COUNT_V2:.*]] = llvm.insertelement %[[COUNT]], %[[COUNT_V1]][%[[ONE]] : !llvm.i32] : !llvm<"<2 x i32>">
-  // CHECK: %[[SIZE:.*]] = llvm.mlir.constant(dense<32> : vector<2xi32>) : !llvm<"<2 x i32>">
-  // CHECK: %[[T0:.*]] = llvm.add %[[COUNT_V2]], %[[OFFSET_V2]] : !llvm<"<2 x i32>">
-  // CHECK: %[[T1:.*]] = llvm.sub %[[SIZE]], %[[T0]] : !llvm<"<2 x i32>">
-  // CHECK: %[[SHIFTED_LEFT:.*]] = llvm.shl %[[BASE]], %[[T1]] : !llvm<"<2 x i32>">
-  // CHECK: %[[T2:.*]] = llvm.add %[[OFFSET_V2]], %[[T1]] : !llvm<"<2 x i32>">
-  // CHECK: llvm.ashr %[[SHIFTED_LEFT]], %[[T2]] : !llvm<"<2 x i32>">
+  // CHECK: %[[COUNT_V2:.*]] = llvm.insertelement %[[COUNT]], %[[COUNT_V1]][%[[ONE]] : !llvm.i32] : !llvm.vec<2 x i32>
+  // CHECK: %[[SIZE:.*]] = llvm.mlir.constant(dense<32> : vector<2xi32>) : !llvm.vec<2 x i32>
+  // CHECK: %[[T0:.*]] = llvm.add %[[COUNT_V2]], %[[OFFSET_V2]] : !llvm.vec<2 x i32>
+  // CHECK: %[[T1:.*]] = llvm.sub %[[SIZE]], %[[T0]] : !llvm.vec<2 x i32>
+  // CHECK: %[[SHIFTED_LEFT:.*]] = llvm.shl %[[BASE]], %[[T1]] : !llvm.vec<2 x i32>
+  // CHECK: %[[T2:.*]] = llvm.add %[[OFFSET_V2]], %[[T1]] : !llvm.vec<2 x i32>
+  // CHECK: llvm.ashr %[[SHIFTED_LEFT]], %[[T2]] : !llvm.vec<2 x i32>
   %0 = spv.BitFieldSExtract %base, %offset, %count : vector<2xi32>, i32, i32
   return
 }
@@ -227,24 +227,24 @@ func @bitfield_uextract_scalar_greater_bit_width(%base: i8, %offset: i16, %count
   return
 }
 
-// CHECK-LABEL: @bitfield_uextract_vector
-// CHECK-SAME: %[[BASE:.*]]: !llvm<"<2 x i32>">, %[[OFFSET:.*]]: !llvm.i32, %[[COUNT:.*]]: !llvm.i32
+// CHECK-LABEL: func @bitfield_uextract_vector
+// CHECK-SAME: %[[BASE:.*]]: !llvm.vec<2 x i32>, %[[OFFSET:.*]]: !llvm.i32, %[[COUNT:.*]]: !llvm.i32
 func @bitfield_uextract_vector(%base: vector<2xi32>, %offset: i32, %count: i32) {
-  // CHECK: %[[OFFSET_V0:.*]] = llvm.mlir.undef : !llvm<"<2 x i32>">
+  // CHECK: %[[OFFSET_V0:.*]] = llvm.mlir.undef : !llvm.vec<2 x i32>
   // CHECK: %[[ZERO:.*]] = llvm.mlir.constant(0 : i32) : !llvm.i32
-  // CHECK: %[[OFFSET_V1:.*]] = llvm.insertelement %[[OFFSET]], %[[OFFSET_V0]][%[[ZERO]] : !llvm.i32] : !llvm<"<2 x i32>">
+  // CHECK: %[[OFFSET_V1:.*]] = llvm.insertelement %[[OFFSET]], %[[OFFSET_V0]][%[[ZERO]] : !llvm.i32] : !llvm.vec<2 x i32>
   // CHECK: %[[ONE:.*]] = llvm.mlir.constant(1 : i32) : !llvm.i32
-  // CHECK: %[[OFFSET_V2:.*]] = llvm.insertelement  %[[OFFSET]], %[[OFFSET_V1]][%[[ONE]] : !llvm.i32] : !llvm<"<2 x i32>">
-  // CHECK: %[[COUNT_V0:.*]] = llvm.mlir.undef : !llvm<"<2 x i32>">
+  // CHECK: %[[OFFSET_V2:.*]] = llvm.insertelement  %[[OFFSET]], %[[OFFSET_V1]][%[[ONE]] : !llvm.i32] : !llvm.vec<2 x i32>
+  // CHECK: %[[COUNT_V0:.*]] = llvm.mlir.undef : !llvm.vec<2 x i32>
   // CHECK: %[[ZERO:.*]] = llvm.mlir.constant(0 : i32) : !llvm.i32
-  // CHECK: %[[COUNT_V1:.*]] = llvm.insertelement %[[COUNT]], %[[COUNT_V0]][%[[ZERO]] : !llvm.i32] : !llvm<"<2 x i32>">
+  // CHECK: %[[COUNT_V1:.*]] = llvm.insertelement %[[COUNT]], %[[COUNT_V0]][%[[ZERO]] : !llvm.i32] : !llvm.vec<2 x i32>
   // CHECK: %[[ONE:.*]] = llvm.mlir.constant(1 : i32) : !llvm.i32
-  // CHECK: %[[COUNT_V2:.*]] = llvm.insertelement %[[COUNT]], %[[COUNT_V1]][%[[ONE]] : !llvm.i32] : !llvm<"<2 x i32>">
-  // CHECK: %[[MINUS_ONE:.*]] = llvm.mlir.constant(dense<-1> : vector<2xi32>) : !llvm<"<2 x i32>">
-  // CHECK: %[[T0:.*]] = llvm.shl %[[MINUS_ONE]], %[[COUNT_V2]] : !llvm<"<2 x i32>">
-  // CHECK: %[[MASK:.*]] = llvm.xor %[[T0]], %[[MINUS_ONE]] : !llvm<"<2 x i32>">
-  // CHECK: %[[SHIFTED_BASE:.*]] = llvm.lshr %[[BASE]], %[[OFFSET_V2]] : !llvm<"<2 x i32>">
-  // CHECK: llvm.and %[[SHIFTED_BASE]], %[[MASK]] : !llvm<"<2 x i32>">
+  // CHECK: %[[COUNT_V2:.*]] = llvm.insertelement %[[COUNT]], %[[COUNT_V1]][%[[ONE]] : !llvm.i32] : !llvm.vec<2 x i32>
+  // CHECK: %[[MINUS_ONE:.*]] = llvm.mlir.constant(dense<-1> : vector<2xi32>) : !llvm.vec<2 x i32>
+  // CHECK: %[[T0:.*]] = llvm.shl %[[MINUS_ONE]], %[[COUNT_V2]] : !llvm.vec<2 x i32>
+  // CHECK: %[[MASK:.*]] = llvm.xor %[[T0]], %[[MINUS_ONE]] : !llvm.vec<2 x i32>
+  // CHECK: %[[SHIFTED_BASE:.*]] = llvm.lshr %[[BASE]], %[[OFFSET_V2]] : !llvm.vec<2 x i32>
+  // CHECK: llvm.and %[[SHIFTED_BASE]], %[[MASK]] : !llvm.vec<2 x i32>
   %0 = spv.BitFieldUExtract %base, %offset, %count : vector<2xi32>, i32, i32
   return
 }
@@ -262,7 +262,7 @@ func @bitwise_and_scalar(%arg0: i32, %arg1: i32) {
 
 // CHECK-LABEL: @bitwise_and_vector
 func @bitwise_and_vector(%arg0: vector<4xi64>, %arg1: vector<4xi64>) {
-  // CHECK: llvm.and %{{.*}}, %{{.*}} : !llvm<"<4 x i64>">
+  // CHECK: llvm.and %{{.*}}, %{{.*}} : !llvm.vec<4 x i64>
   %0 = spv.BitwiseAnd %arg0, %arg1 : vector<4xi64>
   return
 }
@@ -280,7 +280,7 @@ func @bitwise_or_scalar(%arg0: i64, %arg1: i64) {
 
 // CHECK-LABEL: @bitwise_or_vector
 func @bitwise_or_vector(%arg0: vector<3xi8>, %arg1: vector<3xi8>) {
-  // CHECK: llvm.or %{{.*}}, %{{.*}} : !llvm<"<3 x i8>">
+  // CHECK: llvm.or %{{.*}}, %{{.*}} : !llvm.vec<3 x i8>
   %0 = spv.BitwiseOr %arg0, %arg1 : vector<3xi8>
   return
 }
@@ -298,7 +298,7 @@ func @bitwise_xor_scalar(%arg0: i32, %arg1: i32) {
 
 // CHECK-LABEL: @bitwise_xor_vector
 func @bitwise_xor_vector(%arg0: vector<2xi16>, %arg1: vector<2xi16>) {
-  // CHECK: llvm.xor %{{.*}}, %{{.*}} : !llvm<"<2 x i16>">
+  // CHECK: llvm.xor %{{.*}}, %{{.*}} : !llvm.vec<2 x i16>
   %0 = spv.BitwiseXor %arg0, %arg1 : vector<2xi16>
   return
 }
@@ -317,8 +317,8 @@ func @not_scalar(%arg0: i32) {
 
 // CHECK-LABEL: @not_vector
 func @not_vector(%arg0: vector<2xi16>) {
-  // CHECK: %[[CONST:.*]] = llvm.mlir.constant(dense<-1> : vector<2xi16>) : !llvm<"<2 x i16>">
-  // CHECK: llvm.xor %{{.*}}, %[[CONST]] : !llvm<"<2 x i16>">
+  // CHECK: %[[CONST:.*]] = llvm.mlir.constant(dense<-1> : vector<2xi16>) : !llvm.vec<2 x i16>
+  // CHECK: llvm.xor %{{.*}}, %[[CONST]] : !llvm.vec<2 x i16>
   %0 = spv.Not %arg0 : vector<2xi16>
   return
 }
diff --git a/mlir/test/Conversion/SPIRVToLLVM/cast-ops-to-llvm.mlir b/mlir/test/Conversion/SPIRVToLLVM/cast-ops-to-llvm.mlir
index a0434aad3bea7..5071783073c27 100644
--- a/mlir/test/Conversion/SPIRVToLLVM/cast-ops-to-llvm.mlir
+++ b/mlir/test/Conversion/SPIRVToLLVM/cast-ops-to-llvm.mlir
@@ -13,35 +13,35 @@ func @bitcast_float_to_integer_scalar(%arg0 : f32) {
 
 // CHECK-LABEL: @bitcast_float_to_integer_vector
 func @bitcast_float_to_integer_vector(%arg0 : vector<3xf32>) {
-  // CHECK: llvm.bitcast {{.*}} : !llvm<"<3 x float>"> to !llvm<"<3 x i32>">
+  // CHECK: {{.*}} = llvm.bitcast {{.*}} : !llvm.vec<3 x float> to !llvm.vec<3 x i32>
   %0 = spv.Bitcast %arg0: vector<3xf32> to vector<3xi32>
   return
 }
 
 // CHECK-LABEL: @bitcast_vector_to_scalar
 func @bitcast_vector_to_scalar(%arg0 : vector<2xf32>) {
-  // CHECK: llvm.bitcast {{.*}} : !llvm<"<2 x float>"> to !llvm.i64
+  // CHECK: {{.*}} = llvm.bitcast {{.*}} : !llvm.vec<2 x float> to !llvm.i64
   %0 = spv.Bitcast %arg0: vector<2xf32> to i64
   return
 }
 
 // CHECK-LABEL: @bitcast_scalar_to_vector
 func @bitcast_scalar_to_vector(%arg0 : f64) {
-  // CHECK: llvm.bitcast {{.*}} : !llvm.double to !llvm<"<2 x i32>">
+  // CHECK: {{.*}} = llvm.bitcast {{.*}} : !llvm.double to !llvm.vec<2 x i32>
   %0 = spv.Bitcast %arg0: f64 to vector<2xi32>
   return
 }
 
 // CHECK-LABEL: @bitcast_vector_to_vector
 func @bitcast_vector_to_vector(%arg0 : vector<4xf32>) {
-  // CHECK: llvm.bitcast {{.*}} : !llvm<"<4 x float>"> to !llvm<"<2 x i64>">
+  // CHECK: {{.*}} = llvm.bitcast {{.*}} : !llvm.vec<4 x float> to !llvm.vec<2 x i64>
   %0 = spv.Bitcast %arg0: vector<4xf32> to vector<2xi64>
   return
 }
 
 // CHECK-LABEL: @bitcast_pointer
 func @bitcast_pointer(%arg0: !spv.ptr<f32, Function>) {
-  // CHECK: llvm.bitcast %{{.*}} : !llvm<"float*"> to !llvm<"i32*">
+  // CHECK: llvm.bitcast %{{.*}} : !llvm.ptr<float> to !llvm.ptr<i32>
   %0 = spv.Bitcast %arg0 : !spv.ptr<f32, Function> to !spv.ptr<i32, Function>
   return
 }
@@ -59,8 +59,8 @@ func @convert_float_to_signed_scalar(%arg0: f32) {
 
 // CHECK-LABEL: @convert_float_to_signed_vector
 func @convert_float_to_signed_vector(%arg0: vector<2xf32>) {
-  // CHECK: llvm.fptosi %{{.*}} : !llvm<"<2 x float>"> to !llvm<"<2 x i32>">
-  %0 = spv.ConvertFToS %arg0: vector<2xf32> to vector<2xi32>
+  // CHECK: llvm.fptosi %{{.*}} : !llvm.vec<2 x float> to !llvm.vec<2 x i32>
+    %0 = spv.ConvertFToS %arg0: vector<2xf32> to vector<2xi32>
   return
 }
 
@@ -77,8 +77,8 @@ func @convert_float_to_unsigned_scalar(%arg0: f32) {
 
 // CHECK-LABEL: @convert_float_to_unsigned_vector
 func @convert_float_to_unsigned_vector(%arg0: vector<2xf32>) {
-  // CHECK: llvm.fptoui %{{.*}} : !llvm<"<2 x float>"> to !llvm<"<2 x i32>">
-  %0 = spv.ConvertFToU %arg0: vector<2xf32> to vector<2xi32>
+  // CHECK: llvm.fptoui %{{.*}} : !llvm.vec<2 x float> to !llvm.vec<2 x i32>
+    %0 = spv.ConvertFToU %arg0: vector<2xf32> to vector<2xi32>
   return
 }
 
@@ -95,8 +95,8 @@ func @convert_signed_to_float_scalar(%arg0: i32) {
 
 // CHECK-LABEL: @convert_signed_to_float_vector
 func @convert_signed_to_float_vector(%arg0: vector<3xi32>) {
-  // CHECK: llvm.sitofp %{{.*}} : !llvm<"<3 x i32>"> to !llvm<"<3 x float>">
-  %0 = spv.ConvertSToF %arg0: vector<3xi32> to vector<3xf32>
+  // CHECK: llvm.sitofp %{{.*}} : !llvm.vec<3 x i32> to !llvm.vec<3 x float>
+    %0 = spv.ConvertSToF %arg0: vector<3xi32> to vector<3xf32>
   return
 }
 
@@ -113,8 +113,8 @@ func @convert_unsigned_to_float_scalar(%arg0: i32) {
 
 // CHECK-LABEL: @convert_unsigned_to_float_vector
 func @convert_unsigned_to_float_vector(%arg0: vector<3xi32>) {
-  // CHECK: llvm.uitofp %{{.*}} : !llvm<"<3 x i32>"> to !llvm<"<3 x float>">
-  %0 = spv.ConvertUToF %arg0: vector<3xi32> to vector<3xf32>
+  // CHECK: llvm.uitofp %{{.*}} : !llvm.vec<3 x i32> to !llvm.vec<3 x float>
+    %0 = spv.ConvertUToF %arg0: vector<3xi32> to vector<3xf32>
   return
 }
 
@@ -134,10 +134,10 @@ func @fconvert_scalar(%arg0: f32, %arg1: f64) {
 
 // CHECK-LABEL: @fconvert_vector
 func @fconvert_vector(%arg0: vector<2xf32>, %arg1: vector<2xf64>) {
-  // CHECK: llvm.fpext %{{.*}} : !llvm<"<2 x float>"> to !llvm<"<2 x double>">
+  // CHECK: llvm.fpext %{{.*}} : !llvm.vec<2 x float> to !llvm.vec<2 x double>
   %0 = spv.FConvert %arg0: vector<2xf32> to vector<2xf64>
 
-  // CHECK: llvm.fptrunc %{{.*}} : !llvm<"<2 x double>"> to !llvm<"<2 x float>">
+  // CHECK: llvm.fptrunc %{{.*}} : !llvm.vec<2 x double> to !llvm.vec<2 x float>
   %1 = spv.FConvert %arg1: vector<2xf64> to vector<2xf32>
   return
 }
@@ -158,10 +158,10 @@ func @sconvert_scalar(%arg0: i32, %arg1: i64) {
 
 // CHECK-LABEL: @sconvert_vector
 func @sconvert_vector(%arg0: vector<3xi32>, %arg1: vector<3xi64>) {
-  // CHECK: llvm.sext %{{.*}} : !llvm<"<3 x i32>"> to !llvm<"<3 x i64>">
+  // CHECK: llvm.sext %{{.*}} : !llvm.vec<3 x i32> to !llvm.vec<3 x i64>
   %0 = spv.SConvert %arg0: vector<3xi32> to vector<3xi64>
 
-  // CHECK: llvm.trunc %{{.*}} : !llvm<"<3 x i64>"> to !llvm<"<3 x i32>">
+  // CHECK: llvm.trunc %{{.*}} : !llvm.vec<3 x i64> to !llvm.vec<3 x i32>
   %1 = spv.SConvert %arg1: vector<3xi64> to vector<3xi32>
   return
 }
@@ -182,10 +182,10 @@ func @uconvert_scalar(%arg0: i32, %arg1: i64) {
 
 // CHECK-LABEL: @uconvert_vector
 func @uconvert_vector(%arg0: vector<3xi32>, %arg1: vector<3xi64>) {
-  // CHECK: llvm.zext %{{.*}} : !llvm<"<3 x i32>"> to !llvm<"<3 x i64>">
+  // CHECK: llvm.zext %{{.*}} : !llvm.vec<3 x i32> to !llvm.vec<3 x i64>
   %0 = spv.UConvert %arg0: vector<3xi32> to vector<3xi64>
 
-  // CHECK: llvm.trunc %{{.*}} : !llvm<"<3 x i64>"> to !llvm<"<3 x i32>">
+  // CHECK: llvm.trunc %{{.*}} : !llvm.vec<3 x i64> to !llvm.vec<3 x i32>
   %1 = spv.UConvert %arg1: vector<3xi64> to vector<3xi32>
   return
 }
diff --git a/mlir/test/Conversion/SPIRVToLLVM/comparison-ops-to-llvm.mlir b/mlir/test/Conversion/SPIRVToLLVM/comparison-ops-to-llvm.mlir
index bcfe234f7080d..0592d59ba6eef 100644
--- a/mlir/test/Conversion/SPIRVToLLVM/comparison-ops-to-llvm.mlir
+++ b/mlir/test/Conversion/SPIRVToLLVM/comparison-ops-to-llvm.mlir
@@ -13,7 +13,7 @@ func @i_equal_scalar(%arg0: i32, %arg1: i32) {
 
 // CHECK-LABEL: @i_equal_vector
 func @i_equal_vector(%arg0: vector<4xi64>, %arg1: vector<4xi64>) {
-  // CHECK: llvm.icmp "eq" %{{.*}}, %{{.*}} : !llvm<"<4 x i64>">
+  // CHECK: llvm.icmp "eq" %{{.*}}, %{{.*}} : !llvm.vec<4 x i64>
   %0 = spv.IEqual %arg0, %arg1 : vector<4xi64>
   return
 }
@@ -31,7 +31,7 @@ func @i_not_equal_scalar(%arg0: i64, %arg1: i64) {
 
 // CHECK-LABEL: @i_not_equal_vector
 func @i_not_equal_vector(%arg0: vector<2xi64>, %arg1: vector<2xi64>) {
-  // CHECK: llvm.icmp "ne" %{{.*}}, %{{.*}} : !llvm<"<2 x i64>">
+  // CHECK: llvm.icmp "ne" %{{.*}}, %{{.*}} : !llvm.vec<2 x i64>
   %0 = spv.INotEqual %arg0, %arg1 : vector<2xi64>
   return
 }
@@ -49,7 +49,7 @@ func @s_greater_than_equal_scalar(%arg0: i64, %arg1: i64) {
 
 // CHECK-LABEL: @s_greater_than_equal_vector
 func @s_greater_than_equal_vector(%arg0: vector<2xi64>, %arg1: vector<2xi64>) {
-  // CHECK: llvm.icmp "sge" %{{.*}}, %{{.*}} : !llvm<"<2 x i64>">
+  // CHECK: llvm.icmp "sge" %{{.*}}, %{{.*}} : !llvm.vec<2 x i64>
   %0 = spv.SGreaterThanEqual %arg0, %arg1 : vector<2xi64>
   return
 }
@@ -67,7 +67,7 @@ func @s_greater_than_scalar(%arg0: i64, %arg1: i64) {
 
 // CHECK-LABEL: @s_greater_than_vector
 func @s_greater_than_vector(%arg0: vector<2xi64>, %arg1: vector<2xi64>) {
-  // CHECK: llvm.icmp "sgt" %{{.*}}, %{{.*}} : !llvm<"<2 x i64>">
+  // CHECK: llvm.icmp "sgt" %{{.*}}, %{{.*}} : !llvm.vec<2 x i64>
   %0 = spv.SGreaterThan %arg0, %arg1 : vector<2xi64>
   return
 }
@@ -85,7 +85,7 @@ func @s_less_than_equal_scalar(%arg0: i64, %arg1: i64) {
 
 // CHECK-LABEL: @s_less_than_equal_vector
 func @s_less_than_equal_vector(%arg0: vector<2xi64>, %arg1: vector<2xi64>) {
-  // CHECK: llvm.icmp "sle" %{{.*}}, %{{.*}} : !llvm<"<2 x i64>">
+  // CHECK: llvm.icmp "sle" %{{.*}}, %{{.*}} : !llvm.vec<2 x i64>
   %0 = spv.SLessThanEqual %arg0, %arg1 : vector<2xi64>
   return
 }
@@ -103,7 +103,7 @@ func @s_less_than_scalar(%arg0: i64, %arg1: i64) {
 
 // CHECK-LABEL: @s_less_than_vector
 func @s_less_than_vector(%arg0: vector<2xi64>, %arg1: vector<2xi64>) {
-  // CHECK: llvm.icmp "slt" %{{.*}}, %{{.*}} : !llvm<"<2 x i64>">
+  // CHECK: llvm.icmp "slt" %{{.*}}, %{{.*}} : !llvm.vec<2 x i64>
   %0 = spv.SLessThan %arg0, %arg1 : vector<2xi64>
   return
 }
@@ -121,7 +121,7 @@ func @u_greater_than_equal_scalar(%arg0: i64, %arg1: i64) {
 
 // CHECK-LABEL: @u_greater_than_equal_vector
 func @u_greater_than_equal_vector(%arg0: vector<2xi64>, %arg1: vector<2xi64>) {
-  // CHECK: llvm.icmp "uge" %{{.*}}, %{{.*}} : !llvm<"<2 x i64>">
+  // CHECK: llvm.icmp "uge" %{{.*}}, %{{.*}} : !llvm.vec<2 x i64>
   %0 = spv.UGreaterThanEqual %arg0, %arg1 : vector<2xi64>
   return
 }
@@ -139,7 +139,7 @@ func @u_greater_than_scalar(%arg0: i64, %arg1: i64) {
 
 // CHECK-LABEL: @u_greater_than_vector
 func @u_greater_than_vector(%arg0: vector<2xi64>, %arg1: vector<2xi64>) {
-  // CHECK: llvm.icmp "ugt" %{{.*}}, %{{.*}} : !llvm<"<2 x i64>">
+  // CHECK: llvm.icmp "ugt" %{{.*}}, %{{.*}} : !llvm.vec<2 x i64>
   %0 = spv.UGreaterThan %arg0, %arg1 : vector<2xi64>
   return
 }
@@ -157,7 +157,7 @@ func @u_less_than_equal_scalar(%arg0: i64, %arg1: i64) {
 
 // CHECK-LABEL: @u_less_than_equal_vector
 func @u_less_than_equal_vector(%arg0: vector<2xi64>, %arg1: vector<2xi64>) {
-  // CHECK: llvm.icmp "ule" %{{.*}}, %{{.*}} : !llvm<"<2 x i64>">
+  // CHECK: llvm.icmp "ule" %{{.*}}, %{{.*}} : !llvm.vec<2 x i64>
   %0 = spv.ULessThanEqual %arg0, %arg1 : vector<2xi64>
   return
 }
@@ -175,7 +175,7 @@ func @u_less_than_scalar(%arg0: i64, %arg1: i64) {
 
 // CHECK-LABEL: @u_less_than_vector
 func @u_less_than_vector(%arg0: vector<2xi64>, %arg1: vector<2xi64>) {
-  // CHECK: llvm.icmp "ult" %{{.*}}, %{{.*}} : !llvm<"<2 x i64>">
+  // CHECK: llvm.icmp "ult" %{{.*}}, %{{.*}} : !llvm.vec<2 x i64>
   %0 = spv.ULessThan %arg0, %arg1 : vector<2xi64>
   return
 }
@@ -193,7 +193,7 @@ func @f_ord_equal_scalar(%arg0: f32, %arg1: f32) {
 
 // CHECK-LABEL: @f_ord_equal_vector
 func @f_ord_equal_vector(%arg0: vector<4xf64>, %arg1: vector<4xf64>) {
-  // CHECK: llvm.fcmp "oeq" %{{.*}}, %{{.*}} : !llvm<"<4 x double>">
+  // CHECK: llvm.fcmp "oeq" %{{.*}}, %{{.*}} : !llvm.vec<4 x double>
   %0 = spv.FOrdEqual %arg0, %arg1 : vector<4xf64>
   return
 }
@@ -211,9 +211,9 @@ func @f_ord_greater_than_equal_scalar(%arg0: f64, %arg1: f64) {
 
 // CHECK-LABEL: @f_ord_greater_than_equal_vector
 func @f_ord_greater_than_equal_vector(%arg0: vector<2xf64>, %arg1: vector<2xf64>) {
-	// CHECK: llvm.fcmp "oge" %{{.*}}, %{{.*}} : !llvm<"<2 x double>">
-	%0 = spv.FOrdGreaterThanEqual %arg0, %arg1 : vector<2xf64>
-	return
+  // CHECK: llvm.fcmp "oge" %{{.*}}, %{{.*}} : !llvm.vec<2 x double>
+  %0 = spv.FOrdGreaterThanEqual %arg0, %arg1 : vector<2xf64>
+  return
 }
 
 //===----------------------------------------------------------------------===//
@@ -229,7 +229,7 @@ func @f_ord_greater_than_scalar(%arg0: f64, %arg1: f64) {
 
 // CHECK-LABEL: @f_ord_greater_than_vector
 func @f_ord_greater_than_vector(%arg0: vector<2xf64>, %arg1: vector<2xf64>) {
-  // CHECK: llvm.fcmp "ogt" %{{.*}}, %{{.*}} : !llvm<"<2 x double>">
+  // CHECK: llvm.fcmp "ogt" %{{.*}}, %{{.*}} : !llvm.vec<2 x double>
   %0 = spv.FOrdGreaterThan %arg0, %arg1 : vector<2xf64>
   return
 }
@@ -247,7 +247,7 @@ func @f_ord_less_than_scalar(%arg0: f64, %arg1: f64) {
 
 // CHECK-LABEL: @f_ord_less_than_vector
 func @f_ord_less_than_vector(%arg0: vector<2xf64>, %arg1: vector<2xf64>) {
-  // CHECK: llvm.fcmp "olt" %{{.*}}, %{{.*}} : !llvm<"<2 x double>">
+  // CHECK: llvm.fcmp "olt" %{{.*}}, %{{.*}} : !llvm.vec<2 x double>
   %0 = spv.FOrdLessThan %arg0, %arg1 : vector<2xf64>
   return
 }
@@ -265,7 +265,7 @@ func @f_ord_less_than_equal_scalar(%arg0: f64, %arg1: f64) {
 
 // CHECK-LABEL: @f_ord_less_than_equal_vector
 func @f_ord_less_than_equal_vector(%arg0: vector<2xf64>, %arg1: vector<2xf64>) {
-  // CHECK: llvm.fcmp "ole" %{{.*}}, %{{.*}} : !llvm<"<2 x double>">
+  // CHECK: llvm.fcmp "ole" %{{.*}}, %{{.*}} : !llvm.vec<2 x double>
   %0 = spv.FOrdLessThanEqual %arg0, %arg1 : vector<2xf64>
   return
 }
@@ -283,7 +283,7 @@ func @f_ord_not_equal_scalar(%arg0: f32, %arg1: f32) {
 
 // CHECK-LABEL: @f_ord_not_equal_vector
 func @f_ord_not_equal_vector(%arg0: vector<4xf64>, %arg1: vector<4xf64>) {
-  // CHECK: llvm.fcmp "one" %{{.*}}, %{{.*}} : !llvm<"<4 x double>">
+  // CHECK: llvm.fcmp "one" %{{.*}}, %{{.*}} : !llvm.vec<4 x double>
   %0 = spv.FOrdNotEqual %arg0, %arg1 : vector<4xf64>
   return
 }
@@ -301,7 +301,7 @@ func @f_unord_equal_scalar(%arg0: f32, %arg1: f32) {
 
 // CHECK-LABEL: @f_unord_equal_vector
 func @f_unord_equal_vector(%arg0: vector<4xf64>, %arg1: vector<4xf64>) {
-  // CHECK: llvm.fcmp "ueq" %{{.*}}, %{{.*}} : !llvm<"<4 x double>">
+  // CHECK: llvm.fcmp "ueq" %{{.*}}, %{{.*}} : !llvm.vec<4 x double>
   %0 = spv.FUnordEqual %arg0, %arg1 : vector<4xf64>
   return
 }
@@ -319,7 +319,7 @@ func @f_unord_greater_than_equal_scalar(%arg0: f64, %arg1: f64) {
 
 // CHECK-LABEL: @f_unord_greater_than_equal_vector
 func @f_unord_greater_than_equal_vector(%arg0: vector<2xf64>, %arg1: vector<2xf64>) {
-  // CHECK: llvm.fcmp "uge" %{{.*}}, %{{.*}} : !llvm<"<2 x double>">
+  // CHECK: llvm.fcmp "uge" %{{.*}}, %{{.*}} : !llvm.vec<2 x double>
   %0 = spv.FUnordGreaterThanEqual %arg0, %arg1 : vector<2xf64>
   return
 }
@@ -337,7 +337,7 @@ func @f_unord_greater_than_scalar(%arg0: f64, %arg1: f64) {
 
 // CHECK-LABEL: @f_unord_greater_than_vector
 func @f_unord_greater_than_vector(%arg0: vector<2xf64>, %arg1: vector<2xf64>) {
-  // CHECK: llvm.fcmp "ugt" %{{.*}}, %{{.*}} : !llvm<"<2 x double>">
+  // CHECK: llvm.fcmp "ugt" %{{.*}}, %{{.*}} : !llvm.vec<2 x double>
   %0 = spv.FUnordGreaterThan %arg0, %arg1 : vector<2xf64>
   return
 }
@@ -355,7 +355,7 @@ func @f_unord_less_than_scalar(%arg0: f64, %arg1: f64) {
 
 // CHECK-LABEL: @f_unord_less_than_vector
 func @f_unord_less_than_vector(%arg0: vector<2xf64>, %arg1: vector<2xf64>) {
-  // CHECK: llvm.fcmp "ult" %{{.*}}, %{{.*}} : !llvm<"<2 x double>">
+  // CHECK: llvm.fcmp "ult" %{{.*}}, %{{.*}} : !llvm.vec<2 x double>
   %0 = spv.FUnordLessThan %arg0, %arg1 : vector<2xf64>
   return
 }
@@ -373,7 +373,7 @@ func @f_unord_less_than_equal_scalar(%arg0: f64, %arg1: f64) {
 
 // CHECK-LABEL: @f_unord_less_than_equal_vector
 func @f_unord_less_than_equal_vector(%arg0: vector<2xf64>, %arg1: vector<2xf64>) {
-  // CHECK: llvm.fcmp "ule" %{{.*}}, %{{.*}} : !llvm<"<2 x double>">
+  // CHECK: llvm.fcmp "ule" %{{.*}}, %{{.*}} : !llvm.vec<2 x double>
   %0 = spv.FUnordLessThanEqual %arg0, %arg1 : vector<2xf64>
   return
 }
@@ -391,7 +391,7 @@ func @f_unord_not_equal_scalar(%arg0: f32, %arg1: f32) {
 
 // CHECK-LABEL: @f_unord_not_equal_vector
 func @f_unord_not_equal_vector(%arg0: vector<4xf64>, %arg1: vector<4xf64>) {
-  // CHECK: llvm.fcmp "une" %{{.*}}, %{{.*}} : !llvm<"<4 x double>">
+  // CHECK: llvm.fcmp "une" %{{.*}}, %{{.*}} : !llvm.vec<4 x double>
   %0 = spv.FUnordNotEqual %arg0, %arg1 : vector<4xf64>
   return
 }
diff --git a/mlir/test/Conversion/SPIRVToLLVM/constant-op-to-llvm.mlir b/mlir/test/Conversion/SPIRVToLLVM/constant-op-to-llvm.mlir
index b9605e7918577..dc84f404906c7 100644
--- a/mlir/test/Conversion/SPIRVToLLVM/constant-op-to-llvm.mlir
+++ b/mlir/test/Conversion/SPIRVToLLVM/constant-op-to-llvm.mlir
@@ -13,9 +13,9 @@ func @bool_constant_scalar() {
 }
 
 func @bool_constant_vector() {
-  // CHECK: {{.*}} = llvm.mlir.constant(dense<[true, false]> : vector<2xi1>) : !llvm<"<2 x i1>">
+  // CHECK: {{.*}} = llvm.mlir.constant(dense<[true, false]> : vector<2xi1>) : !llvm.vec<2 x i1>
   %0 = constant dense<[true, false]> : vector<2xi1>
-  // CHECK: {{.*}} = llvm.mlir.constant(dense<false> : vector<3xi1>) : !llvm<"<3 x i1>">
+  // CHECK: {{.*}} = llvm.mlir.constant(dense<false> : vector<3xi1>) : !llvm.vec<3 x i1>
   %1 = constant dense<false> : vector<3xi1>
   return
 }
@@ -31,11 +31,11 @@ func @integer_constant_scalar() {
 }
 
 func @integer_constant_vector() {
-  // CHECK: {{.*}} = llvm.mlir.constant(dense<[2, 3]> : vector<2xi32>) : !llvm<"<2 x i32>">
+  // CHECK: {{.*}} = llvm.mlir.constant(dense<[2, 3]> : vector<2xi32>) : !llvm.vec<2 x i32>
   %0 = spv.constant dense<[2, 3]> : vector<2xi32>
-  // CHECK: {{.*}} = llvm.mlir.constant(dense<-4> : vector<2xi32>) : !llvm<"<2 x i32>">
+  // CHECK: {{.*}} = llvm.mlir.constant(dense<-4> : vector<2xi32>) : !llvm.vec<2 x i32>
   %1 = spv.constant dense<-4> : vector<2xsi32>
-  // CHECK: {{.*}} = llvm.mlir.constant(dense<[2, 3, 4]> : vector<3xi32>) : !llvm<"<3 x i32>">
+  // CHECK: {{.*}} = llvm.mlir.constant(dense<[2, 3, 4]> : vector<3xi32>) : !llvm.vec<3 x i32>
   %2 = spv.constant dense<[2, 3, 4]> : vector<3xui32>
   return
 }
@@ -49,7 +49,7 @@ func @float_constant_scalar() {
 }
 
 func @float_constant_vector() {
-  // CHECK: {{.*}} = llvm.mlir.constant(dense<[2.000000e+00, 3.000000e+00]> : vector<2xf32>) : !llvm<"<2 x float>">
+  // CHECK: {{.*}} = llvm.mlir.constant(dense<[2.000000e+00, 3.000000e+00]> : vector<2xf32>) : !llvm.vec<2 x float>
   %0 = spv.constant dense<[2.000000e+00, 3.000000e+00]> : vector<2xf32>
   return
 }
diff --git a/mlir/test/Conversion/SPIRVToLLVM/func-to-llvm.mlir b/mlir/test/Conversion/SPIRVToLLVM/func-to-llvm.mlir
index 23174f34fcecf..7e7820dee6bef 100644
--- a/mlir/test/Conversion/SPIRVToLLVM/func-to-llvm.mlir
+++ b/mlir/test/Conversion/SPIRVToLLVM/func-to-llvm.mlir
@@ -52,7 +52,7 @@ spv.func @scalar_types(%arg0: i32, %arg1: i1, %arg2: f64, %arg3: f32) -> () "Non
 	spv.Return
 }
 
-// CHECK-LABEL: llvm.func @vector_types(%arg0: !llvm<"<2 x i64>">, %arg1: !llvm<"<2 x i64>">) -> !llvm<"<2 x i64>">
+// CHECK-LABEL: llvm.func @vector_types(%arg0: !llvm.vec<2 x i64>, %arg1: !llvm.vec<2 x i64>) -> !llvm.vec<2 x i64>
 spv.func @vector_types(%arg0: vector<2xi64>, %arg1: vector<2xi64>) -> vector<2xi64> "None" {
 	%0 = spv.IAdd %arg0, %arg1 : vector<2xi64>
 	spv.ReturnValue %0 : vector<2xi64>
@@ -63,15 +63,15 @@ spv.func @vector_types(%arg0: vector<2xi64>, %arg1: vector<2xi64>) -> vector<2xi
 //===----------------------------------------------------------------------===//
 
 // CHECK-LABEL: llvm.func @function_calls
-// CHECK-SAME: %[[ARG0:.*]]: !llvm.i32, %[[ARG1:.*]]: !llvm.i1, %[[ARG2:.*]]: !llvm.double, %[[ARG3:.*]]: !llvm<"<2 x i64>">, %[[ARG4:.*]]: !llvm<"<2 x float>">
+// CHECK-SAME: %[[ARG0:.*]]: !llvm.i32, %[[ARG1:.*]]: !llvm.i1, %[[ARG2:.*]]: !llvm.double, %[[ARG3:.*]]: !llvm.vec<2 x i64>, %[[ARG4:.*]]: !llvm.vec<2 x float>
 spv.func @function_calls(%arg0: i32, %arg1: i1, %arg2: f64, %arg3: vector<2xi64>, %arg4: vector<2xf32>) -> () "None" {
 	// CHECK: llvm.call @void_1() : () -> ()
 	spv.FunctionCall @void_1() : () -> ()
-	// CHECK: llvm.call @void_2(%[[ARG3]]) : (!llvm<"<2 x i64>">) -> ()
+	// CHECK: llvm.call @void_2(%[[ARG3]]) : (!llvm.vec<2 x i64>) -> ()
 	spv.FunctionCall @void_2(%arg3) : (vector<2xi64>) -> ()
 	// CHECK: %{{.*}} = llvm.call @value_scalar(%[[ARG0]], %[[ARG1]], %[[ARG2]]) : (!llvm.i32, !llvm.i1, !llvm.double) -> !llvm.i32
 	%0 = spv.FunctionCall @value_scalar(%arg0, %arg1, %arg2) : (i32, i1, f64) -> i32
-	// CHECK: %{{.*}} = llvm.call @value_vector(%[[ARG3]], %[[ARG4]]) : (!llvm<"<2 x i64>">, !llvm<"<2 x float>">) -> !llvm<"<2 x float>">
+	// CHECK: %{{.*}} = llvm.call @value_vector(%[[ARG3]], %[[ARG4]]) : (!llvm.vec<2 x i64>, !llvm.vec<2 x float>) -> !llvm.vec<2 x float>
 	%1 = spv.FunctionCall @value_vector(%arg3, %arg4) : (vector<2xi64>, vector<2xf32>) -> vector<2xf32>
 	spv.Return
 }
diff --git a/mlir/test/Conversion/SPIRVToLLVM/glsl-ops-to-llvm.mlir b/mlir/test/Conversion/SPIRVToLLVM/glsl-ops-to-llvm.mlir
index ecb9843d64fdd..0cb975da56532 100644
--- a/mlir/test/Conversion/SPIRVToLLVM/glsl-ops-to-llvm.mlir
+++ b/mlir/test/Conversion/SPIRVToLLVM/glsl-ops-to-llvm.mlir
@@ -6,11 +6,11 @@
 
 // CHECK-LABEL: @ceil
 func @ceil(%arg0: f32, %arg1: vector<3xf16>) {
-	// CHECK: "llvm.intr.ceil"(%{{.*}}) : (!llvm.float) -> !llvm.float
-	%0 = spv.GLSL.Ceil %arg0 : f32
-	// CHECK: "llvm.intr.ceil"(%{{.*}}) : (!llvm<"<3 x half>">) -> !llvm<"<3 x half>">
-	%1 = spv.GLSL.Ceil %arg1 : vector<3xf16>
-	return
+  // CHECK: "llvm.intr.ceil"(%{{.*}}) : (!llvm.float) -> !llvm.float
+  %0 = spv.GLSL.Ceil %arg0 : f32
+  // CHECK: "llvm.intr.ceil"(%{{.*}}) : (!llvm.vec<3 x half>) -> !llvm.vec<3 x half>
+  %1 = spv.GLSL.Ceil %arg1 : vector<3xf16>
+  return
 }
 
 //===----------------------------------------------------------------------===//
@@ -19,11 +19,11 @@ func @ceil(%arg0: f32, %arg1: vector<3xf16>) {
 
 // CHECK-LABEL: @cos
 func @cos(%arg0: f32, %arg1: vector<3xf16>) {
-	// CHECK: "llvm.intr.cos"(%{{.*}}) : (!llvm.float) -> !llvm.float
-	%0 = spv.GLSL.Cos %arg0 : f32
-	// CHECK: "llvm.intr.cos"(%{{.*}}) : (!llvm<"<3 x half>">) -> !llvm<"<3 x half>">
-	%1 = spv.GLSL.Cos %arg1 : vector<3xf16>
-	return
+  // CHECK: "llvm.intr.cos"(%{{.*}}) : (!llvm.float) -> !llvm.float
+  %0 = spv.GLSL.Cos %arg0 : f32
+  // CHECK: "llvm.intr.cos"(%{{.*}}) : (!llvm.vec<3 x half>) -> !llvm.vec<3 x half>
+  %1 = spv.GLSL.Cos %arg1 : vector<3xf16>
+  return
 }
 
 //===----------------------------------------------------------------------===//
@@ -32,11 +32,11 @@ func @cos(%arg0: f32, %arg1: vector<3xf16>) {
 
 // CHECK-LABEL: @exp
 func @exp(%arg0: f32, %arg1: vector<3xf16>) {
-	// CHECK: "llvm.intr.exp"(%{{.*}}) : (!llvm.float) -> !llvm.float
-	%0 = spv.GLSL.Exp %arg0 : f32
-	// CHECK: "llvm.intr.exp"(%{{.*}}) : (!llvm<"<3 x half>">) -> !llvm<"<3 x half>">
-	%1 = spv.GLSL.Exp %arg1 : vector<3xf16>
-	return
+  // CHECK: "llvm.intr.exp"(%{{.*}}) : (!llvm.float) -> !llvm.float
+  %0 = spv.GLSL.Exp %arg0 : f32
+  // CHECK: "llvm.intr.exp"(%{{.*}}) : (!llvm.vec<3 x half>) -> !llvm.vec<3 x half>
+  %1 = spv.GLSL.Exp %arg1 : vector<3xf16>
+  return
 }
 
 //===----------------------------------------------------------------------===//
@@ -45,11 +45,11 @@ func @exp(%arg0: f32, %arg1: vector<3xf16>) {
 
 // CHECK-LABEL: @fabs
 func @fabs(%arg0: f32, %arg1: vector<3xf16>) {
-	// CHECK: "llvm.intr.fabs"(%{{.*}}) : (!llvm.float) -> !llvm.float
-	%0 = spv.GLSL.FAbs %arg0 : f32
-	// CHECK: "llvm.intr.fabs"(%{{.*}}) : (!llvm<"<3 x half>">) -> !llvm<"<3 x half>">
-	%1 = spv.GLSL.FAbs %arg1 : vector<3xf16>
-	return
+  // CHECK: "llvm.intr.fabs"(%{{.*}}) : (!llvm.float) -> !llvm.float
+  %0 = spv.GLSL.FAbs %arg0 : f32
+  // CHECK: "llvm.intr.fabs"(%{{.*}}) : (!llvm.vec<3 x half>) -> !llvm.vec<3 x half>
+  %1 = spv.GLSL.FAbs %arg1 : vector<3xf16>
+  return
 }
 
 //===----------------------------------------------------------------------===//
@@ -58,11 +58,11 @@ func @fabs(%arg0: f32, %arg1: vector<3xf16>) {
 
 // CHECK-LABEL: @floor
 func @floor(%arg0: f32, %arg1: vector<3xf16>) {
-	// CHECK: "llvm.intr.floor"(%{{.*}}) : (!llvm.float) -> !llvm.float
-	%0 = spv.GLSL.Floor %arg0 : f32
-	// CHECK: "llvm.intr.floor"(%{{.*}}) : (!llvm<"<3 x half>">) -> !llvm<"<3 x half>">
-	%1 = spv.GLSL.Floor %arg1 : vector<3xf16>
-	return
+  // CHECK: "llvm.intr.floor"(%{{.*}}) : (!llvm.float) -> !llvm.float
+  %0 = spv.GLSL.Floor %arg0 : f32
+  // CHECK: "llvm.intr.floor"(%{{.*}}) : (!llvm.vec<3 x half>) -> !llvm.vec<3 x half>
+  %1 = spv.GLSL.Floor %arg1 : vector<3xf16>
+  return
 }
 
 //===----------------------------------------------------------------------===//
@@ -71,11 +71,11 @@ func @floor(%arg0: f32, %arg1: vector<3xf16>) {
 
 // CHECK-LABEL: @fmax
 func @fmax(%arg0: f32, %arg1: vector<3xf16>) {
-	// CHECK: "llvm.intr.maxnum"(%{{.*}}, %{{.*}}) : (!llvm.float, !llvm.float) -> !llvm.float
-	%0 = spv.GLSL.FMax %arg0, %arg0 : f32
-	// CHECK: "llvm.intr.maxnum"(%{{.*}}, %{{.*}}) : (!llvm<"<3 x half>">, !llvm<"<3 x half>">) -> !llvm<"<3 x half>">
-	%1 = spv.GLSL.FMax %arg1, %arg1 : vector<3xf16>
-	return
+  // CHECK: "llvm.intr.maxnum"(%{{.*}}, %{{.*}}) : (!llvm.float, !llvm.float) -> !llvm.float
+  %0 = spv.GLSL.FMax %arg0, %arg0 : f32
+  // CHECK: "llvm.intr.maxnum"(%{{.*}}, %{{.*}}) : (!llvm.vec<3 x half>, !llvm.vec<3 x half>) -> !llvm.vec<3 x half>
+  %1 = spv.GLSL.FMax %arg1, %arg1 : vector<3xf16>
+  return
 }
 
 //===----------------------------------------------------------------------===//
@@ -84,11 +84,11 @@ func @fmax(%arg0: f32, %arg1: vector<3xf16>) {
 
 // CHECK-LABEL: @fmin
 func @fmin(%arg0: f32, %arg1: vector<3xf16>) {
-	// CHECK: "llvm.intr.minnum"(%{{.*}}, %{{.*}}) : (!llvm.float, !llvm.float) -> !llvm.float
-	%0 = spv.GLSL.FMin %arg0, %arg0 : f32
-	// CHECK: "llvm.intr.minnum"(%{{.*}}, %{{.*}}) : (!llvm<"<3 x half>">, !llvm<"<3 x half>">) -> !llvm<"<3 x half>">
-	%1 = spv.GLSL.FMin %arg1, %arg1 : vector<3xf16>
-	return
+  // CHECK: "llvm.intr.minnum"(%{{.*}}, %{{.*}}) : (!llvm.float, !llvm.float) -> !llvm.float
+  %0 = spv.GLSL.FMin %arg0, %arg0 : f32
+  // CHECK: "llvm.intr.minnum"(%{{.*}}, %{{.*}}) : (!llvm.vec<3 x half>, !llvm.vec<3 x half>) -> !llvm.vec<3 x half>
+  %1 = spv.GLSL.FMin %arg1, %arg1 : vector<3xf16>
+  return
 }
 
 //===----------------------------------------------------------------------===//
@@ -97,11 +97,11 @@ func @fmin(%arg0: f32, %arg1: vector<3xf16>) {
 
 // CHECK-LABEL: @log
 func @log(%arg0: f32, %arg1: vector<3xf16>) {
-	// CHECK: "llvm.intr.log"(%{{.*}}) : (!llvm.float) -> !llvm.float
-	%0 = spv.GLSL.Log %arg0 : f32
-	// CHECK: "llvm.intr.log"(%{{.*}}) : (!llvm<"<3 x half>">) -> !llvm<"<3 x half>">
-	%1 = spv.GLSL.Log %arg1 : vector<3xf16>
-	return
+  // CHECK: "llvm.intr.log"(%{{.*}}) : (!llvm.float) -> !llvm.float
+  %0 = spv.GLSL.Log %arg0 : f32
+  // CHECK: "llvm.intr.log"(%{{.*}}) : (!llvm.vec<3 x half>) -> !llvm.vec<3 x half>
+  %1 = spv.GLSL.Log %arg1 : vector<3xf16>
+  return
 }
 
 //===----------------------------------------------------------------------===//
@@ -110,11 +110,11 @@ func @log(%arg0: f32, %arg1: vector<3xf16>) {
 
 // CHECK-LABEL: @sin
 func @sin(%arg0: f32, %arg1: vector<3xf16>) {
-	// CHECK: "llvm.intr.sin"(%{{.*}}) : (!llvm.float) -> !llvm.float
-	%0 = spv.GLSL.Sin %arg0 : f32
-	// CHECK: "llvm.intr.sin"(%{{.*}}) : (!llvm<"<3 x half>">) -> !llvm<"<3 x half>">
-	%1 = spv.GLSL.Sin %arg1 : vector<3xf16>
-	return
+  // CHECK: "llvm.intr.sin"(%{{.*}}) : (!llvm.float) -> !llvm.float
+  %0 = spv.GLSL.Sin %arg0 : f32
+  // CHECK: "llvm.intr.sin"(%{{.*}}) : (!llvm.vec<3 x half>) -> !llvm.vec<3 x half>
+  %1 = spv.GLSL.Sin %arg1 : vector<3xf16>
+  return
 }
 
 //===----------------------------------------------------------------------===//
@@ -123,11 +123,11 @@ func @sin(%arg0: f32, %arg1: vector<3xf16>) {
 
 // CHECK-LABEL: @smax
 func @smax(%arg0: i16, %arg1: vector<3xi32>) {
-	// CHECK: "llvm.intr.smax"(%{{.*}}, %{{.*}}) : (!llvm.i16, !llvm.i16) -> !llvm.i16
-	%0 = spv.GLSL.SMax %arg0, %arg0 : i16
-	// CHECK: "llvm.intr.smax"(%{{.*}}, %{{.*}}) : (!llvm<"<3 x i32>">, !llvm<"<3 x i32>">) -> !llvm<"<3 x i32>">
-	%1 = spv.GLSL.SMax %arg1, %arg1 : vector<3xi32>
-	return
+  // CHECK: "llvm.intr.smax"(%{{.*}}, %{{.*}}) : (!llvm.i16, !llvm.i16) -> !llvm.i16
+  %0 = spv.GLSL.SMax %arg0, %arg0 : i16
+  // CHECK: "llvm.intr.smax"(%{{.*}}, %{{.*}}) : (!llvm.vec<3 x i32>, !llvm.vec<3 x i32>) -> !llvm.vec<3 x i32>
+  %1 = spv.GLSL.SMax %arg1, %arg1 : vector<3xi32>
+  return
 }
 
 //===----------------------------------------------------------------------===//
@@ -136,11 +136,11 @@ func @smax(%arg0: i16, %arg1: vector<3xi32>) {
 
 // CHECK-LABEL: @smin
 func @smin(%arg0: i16, %arg1: vector<3xi32>) {
-	// CHECK: "llvm.intr.smin"(%{{.*}}, %{{.*}}) : (!llvm.i16, !llvm.i16) -> !llvm.i16
-	%0 = spv.GLSL.SMin %arg0, %arg0 : i16
-	// CHECK: "llvm.intr.smin"(%{{.*}}, %{{.*}}) : (!llvm<"<3 x i32>">, !llvm<"<3 x i32>">) -> !llvm<"<3 x i32>">
-	%1 = spv.GLSL.SMin %arg1, %arg1 : vector<3xi32>
-	return
+  // CHECK: "llvm.intr.smin"(%{{.*}}, %{{.*}}) : (!llvm.i16, !llvm.i16) -> !llvm.i16
+  %0 = spv.GLSL.SMin %arg0, %arg0 : i16
+  // CHECK: "llvm.intr.smin"(%{{.*}}, %{{.*}}) : (!llvm.vec<3 x i32>, !llvm.vec<3 x i32>) -> !llvm.vec<3 x i32>
+  %1 = spv.GLSL.SMin %arg1, %arg1 : vector<3xi32>
+  return
 }
 
 //===----------------------------------------------------------------------===//
@@ -149,11 +149,11 @@ func @smin(%arg0: i16, %arg1: vector<3xi32>) {
 
 // CHECK-LABEL: @sqrt
 func @sqrt(%arg0: f32, %arg1: vector<3xf16>) {
-	// CHECK: "llvm.intr.sqrt"(%{{.*}}) : (!llvm.float) -> !llvm.float
-	%0 = spv.GLSL.Sqrt %arg0 : f32
-	// CHECK: "llvm.intr.sqrt"(%{{.*}}) : (!llvm<"<3 x half>">) -> !llvm<"<3 x half>">
-	%1 = spv.GLSL.Sqrt %arg1 : vector<3xf16>
-	return
+  // CHECK: "llvm.intr.sqrt"(%{{.*}}) : (!llvm.float) -> !llvm.float
+  %0 = spv.GLSL.Sqrt %arg0 : f32
+  // CHECK: "llvm.intr.sqrt"(%{{.*}}) : (!llvm.vec<3 x half>) -> !llvm.vec<3 x half>
+  %1 = spv.GLSL.Sqrt %arg1 : vector<3xf16>
+  return
 }
 
 //===----------------------------------------------------------------------===//
@@ -162,11 +162,11 @@ func @sqrt(%arg0: f32, %arg1: vector<3xf16>) {
 
 // CHECK-LABEL: @tan
 func @tan(%arg0: f32) {
-	// CHECK: %[[SIN:.*]] = "llvm.intr.sin"(%{{.*}}) : (!llvm.float) -> !llvm.float
+  // CHECK: %[[SIN:.*]] = "llvm.intr.sin"(%{{.*}}) : (!llvm.float) -> !llvm.float
   // CHECK: %[[COS:.*]] = "llvm.intr.cos"(%{{.*}}) : (!llvm.float) -> !llvm.float
-	// CHECK: llvm.fdiv %[[SIN]], %[[COS]] : !llvm.float
-	%0 = spv.GLSL.Tan %arg0 : f32
-	return
+  // CHECK: llvm.fdiv %[[SIN]], %[[COS]] : !llvm.float
+  %0 = spv.GLSL.Tan %arg0 : f32
+  return
 }
 
 //===----------------------------------------------------------------------===//
@@ -175,15 +175,15 @@ func @tan(%arg0: f32) {
 
 // CHECK-LABEL: @tanh
 func @tanh(%arg0: f32) {
-	// CHECK: %[[TWO:.*]] = llvm.mlir.constant(2.000000e+00 : f32) : !llvm.float
+  // CHECK: %[[TWO:.*]] = llvm.mlir.constant(2.000000e+00 : f32) : !llvm.float
   // CHECK: %[[X2:.*]] = llvm.fmul %[[TWO]], %{{.*}} : !llvm.float
   // CHECK: %[[EXP:.*]] = "llvm.intr.exp"(%[[X2]]) : (!llvm.float) -> !llvm.float
   // CHECK: %[[ONE:.*]] = llvm.mlir.constant(1.000000e+00 : f32) : !llvm.float
   // CHECK: %[[T0:.*]] = llvm.fsub %[[EXP]], %[[ONE]] : !llvm.float
   // CHECK: %[[T1:.*]] = llvm.fadd %[[EXP]], %[[ONE]] : !llvm.float
   // CHECK: llvm.fdiv %[[T0]], %[[T1]] : !llvm.float
-	%0 = spv.GLSL.Tanh %arg0 : f32
-	return
+  %0 = spv.GLSL.Tanh %arg0 : f32
+  return
 }
 
 //===----------------------------------------------------------------------===//
@@ -192,9 +192,9 @@ func @tanh(%arg0: f32) {
 
 // CHECK-LABEL: @inverse_sqrt
 func @inverse_sqrt(%arg0: f32) {
-	// CHECK: %[[ONE:.*]] = llvm.mlir.constant(1.000000e+00 : f32) : !llvm.float
+  // CHECK: %[[ONE:.*]] = llvm.mlir.constant(1.000000e+00 : f32) : !llvm.float
   // CHECK: %[[SQRT:.*]] = "llvm.intr.sqrt"(%{{.*}}) : (!llvm.float) -> !llvm.float
-	// CHECK: llvm.fdiv %[[ONE]], %[[SQRT]] : !llvm.float
-	%0 = spv.GLSL.InverseSqrt %arg0 : f32
-	return
+  // CHECK: llvm.fdiv %[[ONE]], %[[SQRT]] : !llvm.float
+  %0 = spv.GLSL.InverseSqrt %arg0 : f32
+  return
 }
diff --git a/mlir/test/Conversion/SPIRVToLLVM/logical-ops-to-llvm.mlir b/mlir/test/Conversion/SPIRVToLLVM/logical-ops-to-llvm.mlir
index a816424d5b1c3..9c9035ae89339 100644
--- a/mlir/test/Conversion/SPIRVToLLVM/logical-ops-to-llvm.mlir
+++ b/mlir/test/Conversion/SPIRVToLLVM/logical-ops-to-llvm.mlir
@@ -13,7 +13,7 @@ func @logical_equal_scalar(%arg0: i1, %arg1: i1) {
 
 // CHECK-LABEL: @logical_equal_vector
 func @logical_equal_vector(%arg0: vector<4xi1>, %arg1: vector<4xi1>) {
-  // CHECK: llvm.icmp "eq" %{{.*}}, %{{.*}} : !llvm<"<4 x i1>">
+  // CHECK: llvm.icmp "eq" %{{.*}}, %{{.*}} : !llvm.vec<4 x i1>
   %0 = spv.LogicalEqual %arg0, %arg0 : vector<4xi1>
   return
 }
@@ -31,7 +31,7 @@ func @logical_not_equal_scalar(%arg0: i1, %arg1: i1) {
 
 // CHECK-LABEL: @logical_not_equal_vector
 func @logical_not_equal_vector(%arg0: vector<4xi1>, %arg1: vector<4xi1>) {
-  // CHECK: llvm.icmp "ne" %{{.*}}, %{{.*}} : !llvm<"<4 x i1>">
+  // CHECK: llvm.icmp "ne" %{{.*}}, %{{.*}} : !llvm.vec<4 x i1>
   %0 = spv.LogicalNotEqual %arg0, %arg0 : vector<4xi1>
   return
 }
@@ -50,8 +50,8 @@ func @logical_not_scalar(%arg0: i1) {
 
 // CHECK-LABEL: @logical_not_vector
 func @logical_not_vector(%arg0: vector<4xi1>) {
-  // CHECK: %[[CONST:.*]] = llvm.mlir.constant(dense<true> : vector<4xi1>) : !llvm<"<4 x i1>">
-  // CHECK: llvm.xor %{{.*}}, %[[CONST]] : !llvm<"<4 x i1>">
+  // CHECK: %[[CONST:.*]] = llvm.mlir.constant(dense<true> : vector<4xi1>) : !llvm.vec<4 x i1>
+  // CHECK: llvm.xor %{{.*}}, %[[CONST]] : !llvm.vec<4 x i1>
   %0 = spv.LogicalNot %arg0 : vector<4xi1>
   return
 }
@@ -69,7 +69,7 @@ func @logical_and_scalar(%arg0: i1, %arg1: i1) {
 
 // CHECK-LABEL: @logical_and_vector
 func @logical_and_vector(%arg0: vector<4xi1>, %arg1: vector<4xi1>) {
-  // CHECK: llvm.and %{{.*}}, %{{.*}} : !llvm<"<4 x i1>">
+  // CHECK: llvm.and %{{.*}}, %{{.*}} : !llvm.vec<4 x i1>
   %0 = spv.LogicalAnd %arg0, %arg0 : vector<4xi1>
   return
 }
@@ -87,7 +87,7 @@ func @logical_or_scalar(%arg0: i1, %arg1: i1) {
 
 // CHECK-LABEL: @logical_or_vector
 func @logical_or_vector(%arg0: vector<4xi1>, %arg1: vector<4xi1>) {
-  // CHECK: llvm.or %{{.*}}, %{{.*}} : !llvm<"<4 x i1>">
+  // CHECK: llvm.or %{{.*}}, %{{.*}} : !llvm.vec<4 x i1>
   %0 = spv.LogicalOr %arg0, %arg0 : vector<4xi1>
   return
 }
diff --git a/mlir/test/Conversion/SPIRVToLLVM/memory-ops-to-llvm.mlir b/mlir/test/Conversion/SPIRVToLLVM/memory-ops-to-llvm.mlir
index f717948f8069d..6dafab982e35b 100644
--- a/mlir/test/Conversion/SPIRVToLLVM/memory-ops-to-llvm.mlir
+++ b/mlir/test/Conversion/SPIRVToLLVM/memory-ops-to-llvm.mlir
@@ -6,35 +6,35 @@
 
 func @load() {
   %0 = spv.Variable : !spv.ptr<f32, Function>
-  //  CHECK: %{{.*}} = llvm.load %{{.*}} : !llvm<"float*">
+  //  CHECK: %{{.*}} = llvm.load %{{.*}} : !llvm.ptr<float>
   %1 = spv.Load "Function" %0 : f32
   return
 }
 
 func @load_none() {
   %0 = spv.Variable : !spv.ptr<f32, Function>
-  //  CHECK: %{{.*}} = llvm.load %{{.*}} : !llvm<"float*">
+  //  CHECK: %{{.*}} = llvm.load %{{.*}} : !llvm.ptr<float>
   %1 = spv.Load "Function" %0 ["None"] : f32
   return
 }
 
 func @load_with_alignment() {
   %0 = spv.Variable : !spv.ptr<f32, Function>
-  // CHECK: %{{.*}} = llvm.load %{{.*}} {alignment = 4 : i64} : !llvm<"float*">
+  // CHECK: %{{.*}} = llvm.load %{{.*}} {alignment = 4 : i64} : !llvm.ptr<float>
   %1 = spv.Load "Function" %0 ["Aligned", 4] : f32
   return
 }
 
 func @load_volatile() {
   %0 = spv.Variable : !spv.ptr<f32, Function>
-  // CHECK: %{{.*}} = llvm.load volatile %{{.*}} : !llvm<"float*">
+  // CHECK: %{{.*}} = llvm.load volatile %{{.*}} : !llvm.ptr<float>
   %1 = spv.Load "Function" %0 ["Volatile"] : f32
   return
 }
 
 func @load_nontemporal() {
   %0 = spv.Variable : !spv.ptr<f32, Function>
-  // CHECK: %{{.*}} = llvm.load %{{.*}} {nontemporal} : !llvm<"float*">
+  // CHECK: %{{.*}} = llvm.load %{{.*}} {nontemporal} : !llvm.ptr<float>
   %1 = spv.Load "Function" %0 ["Nontemporal"] : f32
   return
 }
@@ -45,35 +45,35 @@ func @load_nontemporal() {
 
 func @store(%arg0 : f32) -> () {
   %0 = spv.Variable : !spv.ptr<f32, Function>
-  // CHECK: llvm.store %{{.*}}, %{{.*}} : !llvm<"float*">
+  // CHECK: llvm.store %{{.*}}, %{{.*}} : !llvm.ptr<float>
   spv.Store "Function" %0, %arg0 : f32
   return
 }
 
 func @store_composite(%arg0 : !spv.struct<f64>) -> () {
   %0 = spv.Variable : !spv.ptr<!spv.struct<f64>, Function>
-  // CHECK: llvm.store %{{.*}}, %{{.*}} : !llvm<"<{ double }>*">
+  // CHECK: llvm.store %{{.*}}, %{{.*}} : !llvm.ptr<struct<packed (double)>>
   spv.Store "Function" %0, %arg0 : !spv.struct<f64>
   return
 }
 
 func @store_with_alignment(%arg0 : f32) -> () {
   %0 = spv.Variable : !spv.ptr<f32, Function>
-  // CHECK: llvm.store %{{.*}}, %{{.*}} {alignment = 4 : i64} : !llvm<"float*">
+  // CHECK: llvm.store %{{.*}}, %{{.*}} {alignment = 4 : i64} : !llvm.ptr<float>
   spv.Store "Function" %0, %arg0 ["Aligned", 4] : f32
   return
 }
 
 func @store_volatile(%arg0 : f32) -> () {
   %0 = spv.Variable : !spv.ptr<f32, Function>
-  // CHECK: llvm.store volatile %{{.*}}, %{{.*}} : !llvm<"float*">
+  // CHECK: llvm.store volatile %{{.*}}, %{{.*}} : !llvm.ptr<float>
   spv.Store "Function" %0, %arg0 ["Volatile"] : f32
   return
 }
 
 func @store_nontemporal(%arg0 : f32) -> () {
   %0 = spv.Variable : !spv.ptr<f32, Function>
-  // CHECK: llvm.store %{{.*}}, %{{.*}} {nontemporal} : !llvm<"float*">
+  // CHECK: llvm.store %{{.*}}, %{{.*}} {nontemporal} : !llvm.ptr<float>
   spv.Store "Function" %0, %arg0 ["Nontemporal"] : f32
   return
 }
@@ -83,45 +83,45 @@ func @store_nontemporal(%arg0 : f32) -> () {
 //===----------------------------------------------------------------------===//
 
 func @variable_scalar() {
-	// CHECK: %[[SIZE1:.*]] = llvm.mlir.constant(1 : i32) : !llvm.i32
-  // CHECK: %{{.*}} = llvm.alloca %[[SIZE1]] x !llvm.float : (!llvm.i32) -> !llvm<"float*">
-	%0 = spv.Variable : !spv.ptr<f32, Function>
-	// CHECK: %[[SIZE2:.*]] = llvm.mlir.constant(1 : i32) : !llvm.i32
-  // CHECK: %{{.*}} = llvm.alloca %[[SIZE2]] x !llvm.i8 : (!llvm.i32) -> !llvm<"i8*">
-	%1 = spv.Variable : !spv.ptr<i8, Function>
+  // CHECK: %[[SIZE1:.*]] = llvm.mlir.constant(1 : i32) : !llvm.i32
+  // CHECK: %{{.*}} = llvm.alloca %[[SIZE1]] x !llvm.float : (!llvm.i32) -> !llvm.ptr<float>
+  %0 = spv.Variable : !spv.ptr<f32, Function>
+  // CHECK: %[[SIZE2:.*]] = llvm.mlir.constant(1 : i32) : !llvm.i32
+  // CHECK: %{{.*}} = llvm.alloca %[[SIZE2]] x !llvm.i8 : (!llvm.i32) -> !llvm.ptr<i8>
+  %1 = spv.Variable : !spv.ptr<i8, Function>
   return
 }
 
 func @variable_scalar_with_initialization() {
-	// CHECK: %[[VALUE:.*]] = llvm.mlir.constant(0 : i64) : !llvm.i64
-	// CHECK: %[[SIZE:.*]] = llvm.mlir.constant(1 : i32) : !llvm.i32
-  // CHECK: %[[ALLOCATED:.*]] = llvm.alloca %[[SIZE]] x !llvm.i64 : (!llvm.i32) -> !llvm<"i64*">
-	// CHECK: llvm.store %[[VALUE]], %[[ALLOCATED]] : !llvm<"i64*">
-	%c = spv.constant 0 : i64
-	%0 = spv.Variable init(%c) : !spv.ptr<i64, Function>
+  // CHECK: %[[VALUE:.*]] = llvm.mlir.constant(0 : i64) : !llvm.i64
+  // CHECK: %[[SIZE:.*]] = llvm.mlir.constant(1 : i32) : !llvm.i32
+  // CHECK: %[[ALLOCATED:.*]] = llvm.alloca %[[SIZE]] x !llvm.i64 : (!llvm.i32) -> !llvm.ptr<i64>
+  // CHECK: llvm.store %[[VALUE]], %[[ALLOCATED]] : !llvm.ptr<i64>
+  %c = spv.constant 0 : i64
+  %0 = spv.Variable init(%c) : !spv.ptr<i64, Function>
   return
 }
 
 func @variable_vector() {
-	// CHECK: %[[SIZE:.*]] = llvm.mlir.constant(1 : i32) : !llvm.i32
-	// CHECK: %{{.*}} = llvm.alloca  %[[SIZE]] x !llvm<"<3 x float>"> : (!llvm.i32) -> !llvm<"<3 x float>*">
-	%0 = spv.Variable : !spv.ptr<vector<3xf32>, Function>
-	return
+  // CHECK: %[[SIZE:.*]] = llvm.mlir.constant(1 : i32) : !llvm.i32
+  // CHECK: %{{.*}} = llvm.alloca  %[[SIZE]] x !llvm.vec<3 x float> : (!llvm.i32) -> !llvm.ptr<vec<3 x float>>
+  %0 = spv.Variable : !spv.ptr<vector<3xf32>, Function>
+  return
 }
 
 func @variable_vector_with_initialization() {
-	// CHECK: %[[VALUE:.*]] = llvm.mlir.constant(dense<false> : vector<3xi1>) : !llvm<"<3 x i1>">
-	// CHECK: %[[SIZE:.*]] = llvm.mlir.constant(1 : i32) : !llvm.i32
-  // CHECK: %[[ALLOCATED:.*]] = llvm.alloca %[[SIZE]] x !llvm<"<3 x i1>"> : (!llvm.i32) -> !llvm<"<3 x i1>*">
-	// CHECK: llvm.store %[[VALUE]], %[[ALLOCATED]] : !llvm<"<3 x i1>*">
-	%c = spv.constant dense<false> : vector<3xi1>
-	%0 = spv.Variable init(%c) : !spv.ptr<vector<3xi1>, Function>
+  // CHECK: %[[VALUE:.*]] = llvm.mlir.constant(dense<false> : vector<3xi1>) : !llvm.vec<3 x i1>
+  // CHECK: %[[SIZE:.*]] = llvm.mlir.constant(1 : i32) : !llvm.i32
+  // CHECK: %[[ALLOCATED:.*]] = llvm.alloca %[[SIZE]] x !llvm.vec<3 x i1> : (!llvm.i32) -> !llvm.ptr<vec<3 x i1>>
+  // CHECK: llvm.store %[[VALUE]], %[[ALLOCATED]] : !llvm.ptr<vec<3 x i1>>
+  %c = spv.constant dense<false> : vector<3xi1>
+  %0 = spv.Variable init(%c) : !spv.ptr<vector<3xi1>, Function>
   return
 }
 
 func @variable_array() {
-	// CHECK: %[[SIZE:.*]] = llvm.mlir.constant(1 : i32) : !llvm.i32
-	// CHECK: %{{.*}} = llvm.alloca %[[SIZE]] x !llvm<"[10 x i32]"> : (!llvm.i32) -> !llvm<"[10 x i32]*">
-	%0 = spv.Variable : !spv.ptr<!spv.array<10 x i32>, Function>
-	return
+  // CHECK: %[[SIZE:.*]] = llvm.mlir.constant(1 : i32) : !llvm.i32
+  // CHECK: %{{.*}} = llvm.alloca %[[SIZE]] x !llvm.array<10 x i32> : (!llvm.i32) -> !llvm.ptr<array<10 x i32>>
+  %0 = spv.Variable : !spv.ptr<!spv.array<10 x i32>, Function>
+  return
 }
diff --git a/mlir/test/Conversion/SPIRVToLLVM/misc-ops-to-llvm.mlir b/mlir/test/Conversion/SPIRVToLLVM/misc-ops-to-llvm.mlir
index 8f407ba8f596c..60fe79c0007e8 100644
--- a/mlir/test/Conversion/SPIRVToLLVM/misc-ops-to-llvm.mlir
+++ b/mlir/test/Conversion/SPIRVToLLVM/misc-ops-to-llvm.mlir
@@ -5,7 +5,7 @@
 //===----------------------------------------------------------------------===//
 
 func @select_scalar(%arg0: i1, %arg1: vector<3xi32>, %arg2: f32) {
-	// CHECK: %{{.*}} = llvm.select %{{.*}}, %{{.*}}, %{{.*}} : !llvm.i1, !llvm<"<3 x i32>">
+	// CHECK: %{{.*}} = llvm.select %{{.*}}, %{{.*}}, %{{.*}} : !llvm.i1, !llvm.vec<3 x i32>
 	%0 = spv.Select %arg0, %arg1, %arg1 : i1, vector<3xi32>
 	// CHECK: %{{.*}} = llvm.select %{{.*}}, %{{.*}}, %{{.*}} : !llvm.i1, !llvm.float
 	%1 = spv.Select %arg0, %arg2, %arg2 : i1, f32
@@ -13,7 +13,7 @@ func @select_scalar(%arg0: i1, %arg1: vector<3xi32>, %arg2: f32) {
 }
 
 func @select_vector(%arg0: vector<2xi1>, %arg1: vector<2xi32>) {
-	// CHECK: %{{.*}} = llvm.select %{{.*}}, %{{.*}}, %{{.*}} : !llvm<"<2 x i1>">, !llvm<"<2 x i32>">
+	// CHECK: %{{.*}} = llvm.select %{{.*}}, %{{.*}}, %{{.*}} : !llvm.vec<2 x i1>, !llvm.vec<2 x i32>
 	%0 = spv.Select %arg0, %arg1, %arg1 : vector<2xi1>, vector<2xi32>
   return
 }
@@ -29,7 +29,7 @@ func @undef_scalar() {
 }
 
 func @undef_vector() {
-	// CHECK: %{{.*}} = llvm.mlir.undef : !llvm<"<2 x i32>">
+	// CHECK: %{{.*}} = llvm.mlir.undef : !llvm.vec<2 x i32>
 	%0 = spv.undef : vector<2xi32>
 	return
 }
diff --git a/mlir/test/Conversion/SPIRVToLLVM/shifts-to-llvm.mlir b/mlir/test/Conversion/SPIRVToLLVM/shifts-to-llvm.mlir
index 09e396d6400fd..41431ea3b3b20 100644
--- a/mlir/test/Conversion/SPIRVToLLVM/shifts-to-llvm.mlir
+++ b/mlir/test/Conversion/SPIRVToLLVM/shifts-to-llvm.mlir
@@ -22,18 +22,18 @@ func @shift_right_arithmetic_scalar(%arg0: i32, %arg1: si32, %arg2 : i16, %arg3
 }
 
 func @shift_right_arithmetic_vector(%arg0: vector<4xi64>, %arg1: vector<4xui64>, %arg2: vector<4xi32>, %arg3: vector<4xui32>) {
-	// CHECK: %{{.*}} = llvm.ashr %{{.*}}, %{{.*}} : !llvm<"<4 x i64>">
+	// CHECK: %{{.*}} = llvm.ashr %{{.*}}, %{{.*}} : !llvm.vec<4 x i64>
 	%0 = spv.ShiftRightArithmetic %arg0, %arg0 : vector<4xi64>, vector<4xi64>
 
-    // CHECK: %{{.*}} = llvm.ashr %{{.*}}, %{{.*}} : !llvm<"<4 x i64>">
+    // CHECK: %{{.*}} = llvm.ashr %{{.*}}, %{{.*}} : !llvm.vec<4 x i64>
 	%1 = spv.ShiftRightArithmetic %arg0, %arg1 : vector<4xi64>, vector<4xui64>
 
-    // CHECK: %[[EXT1:.*]] = llvm.sext %{{.*}} : !llvm<"<4 x i32>"> to !llvm<"<4 x i64>">
-    // CHECK: %{{.*}} = llvm.ashr %{{.*}}, %[[EXT1:.*]]: !llvm<"<4 x i64>">
+    // CHECK: %[[EXT1:.*]] = llvm.sext %{{.*}} : !llvm.vec<4 x i32> to !llvm.vec<4 x i64>
+    // CHECK: %{{.*}} = llvm.ashr %{{.*}}, %[[EXT1:.*]]: !llvm.vec<4 x i64>
 	%2 = spv.ShiftRightArithmetic %arg0, %arg2 : vector<4xi64>,  vector<4xi32>
 
-    // CHECK: %[[EXT2:.*]] = llvm.zext %{{.*}} : !llvm<"<4 x i32>"> to !llvm<"<4 x i64>">
-    // CHECK: %{{.*}} = llvm.ashr %{{.*}}, %[[EXT2:.*]]: !llvm<"<4 x i64>">
+    // CHECK: %[[EXT2:.*]] = llvm.zext %{{.*}} : !llvm.vec<4 x i32> to !llvm.vec<4 x i64>
+    // CHECK: %{{.*}} = llvm.ashr %{{.*}}, %[[EXT2:.*]]: !llvm.vec<4 x i64>
 	%3 = spv.ShiftRightArithmetic %arg0, %arg3 : vector<4xi64>, vector<4xui32>
 	return
 }
@@ -60,18 +60,18 @@ func @shift_right_logical_scalar(%arg0: i32, %arg1: si32, %arg2 : si16, %arg3 :
 }
 
 func @shift_right_logical_vector(%arg0: vector<4xi64>, %arg1: vector<4xsi64>, %arg2: vector<4xi32>, %arg3: vector<4xui32>) {
-	// CHECK: %{{.*}} = llvm.lshr %{{.*}}, %{{.*}} : !llvm<"<4 x i64>">
+	// CHECK: %{{.*}} = llvm.lshr %{{.*}}, %{{.*}} : !llvm.vec<4 x i64>
 	%0 = spv.ShiftRightLogical %arg0, %arg0 : vector<4xi64>, vector<4xi64>
 
-    // CHECK: %{{.*}} = llvm.lshr %{{.*}}, %{{.*}} : !llvm<"<4 x i64>">
+    // CHECK: %{{.*}} = llvm.lshr %{{.*}}, %{{.*}} : !llvm.vec<4 x i64>
 	%1 = spv.ShiftRightLogical %arg0, %arg1 : vector<4xi64>, vector<4xsi64>
 
-    // CHECK: %[[EXT1:.*]] = llvm.sext %{{.*}} : !llvm<"<4 x i32>"> to !llvm<"<4 x i64>">
-    // CHECK: %{{.*}} = llvm.lshr %{{.*}}, %[[EXT1:.*]]: !llvm<"<4 x i64>">
+    // CHECK: %[[EXT1:.*]] = llvm.sext %{{.*}} : !llvm.vec<4 x i32> to !llvm.vec<4 x i64>
+    // CHECK: %{{.*}} = llvm.lshr %{{.*}}, %[[EXT1:.*]]: !llvm.vec<4 x i64>
 	%2 = spv.ShiftRightLogical %arg0, %arg2 : vector<4xi64>,  vector<4xi32>
 
-    // CHECK: %[[EXT2:.*]] = llvm.zext %{{.*}} : !llvm<"<4 x i32>"> to !llvm<"<4 x i64>">
-    // CHECK: %{{.*}} = llvm.lshr %{{.*}}, %[[EXT2:.*]]: !llvm<"<4 x i64>">
+    // CHECK: %[[EXT2:.*]] = llvm.zext %{{.*}} : !llvm.vec<4 x i32> to !llvm.vec<4 x i64>
+    // CHECK: %{{.*}} = llvm.lshr %{{.*}}, %[[EXT2:.*]]: !llvm.vec<4 x i64>
 	%3 = spv.ShiftRightLogical %arg0, %arg3 : vector<4xi64>, vector<4xui32>
 	return
 }
@@ -98,18 +98,18 @@ func @shift_left_logical_scalar(%arg0: i32, %arg1: si32, %arg2 : i16, %arg3 : ui
 }
 
 func @shift_left_logical_vector(%arg0: vector<4xi64>, %arg1: vector<4xsi64>, %arg2: vector<4xi32>, %arg3: vector<4xui32>) {
-	// CHECK: %{{.*}} = llvm.shl %{{.*}}, %{{.*}} : !llvm<"<4 x i64>">
+	// CHECK: %{{.*}} = llvm.shl %{{.*}}, %{{.*}} : !llvm.vec<4 x i64>
 	%0 = spv.ShiftLeftLogical %arg0, %arg0 : vector<4xi64>, vector<4xi64>
 
-    // CHECK: %{{.*}} = llvm.shl %{{.*}}, %{{.*}} : !llvm<"<4 x i64>">
+    // CHECK: %{{.*}} = llvm.shl %{{.*}}, %{{.*}} : !llvm.vec<4 x i64>
 	%1 = spv.ShiftLeftLogical %arg0, %arg1 : vector<4xi64>, vector<4xsi64>
 
-    // CHECK: %[[EXT1:.*]] = llvm.sext %{{.*}} : !llvm<"<4 x i32>"> to !llvm<"<4 x i64>">
-    // CHECK: %{{.*}} = llvm.shl %{{.*}}, %[[EXT1:.*]]: !llvm<"<4 x i64>">
+    // CHECK: %[[EXT1:.*]] = llvm.sext %{{.*}} : !llvm.vec<4 x i32> to !llvm.vec<4 x i64>
+    // CHECK: %{{.*}} = llvm.shl %{{.*}}, %[[EXT1:.*]]: !llvm.vec<4 x i64>
 	%2 = spv.ShiftLeftLogical %arg0, %arg2 : vector<4xi64>,  vector<4xi32>
 
-    // CHECK: %[[EXT2:.*]] = llvm.zext %{{.*}} : !llvm<"<4 x i32>"> to !llvm<"<4 x i64>">
-    // CHECK: %{{.*}} = llvm.shl %{{.*}}, %[[EXT2:.*]]: !llvm<"<4 x i64>">
+    // CHECK: %[[EXT2:.*]] = llvm.zext %{{.*}} : !llvm.vec<4 x i32> to !llvm.vec<4 x i64>
+    // CHECK: %{{.*}} = llvm.shl %{{.*}}, %[[EXT2:.*]]: !llvm.vec<4 x i64>
 	%3 = spv.ShiftLeftLogical %arg0, %arg3 : vector<4xi64>, vector<4xui32>
 	return
 }
diff --git a/mlir/test/Conversion/SPIRVToLLVM/spirv-types-to-llvm.mlir b/mlir/test/Conversion/SPIRVToLLVM/spirv-types-to-llvm.mlir
index 466f8ad4a9eac..d6618a7de7fed 100644
--- a/mlir/test/Conversion/SPIRVToLLVM/spirv-types-to-llvm.mlir
+++ b/mlir/test/Conversion/SPIRVToLLVM/spirv-types-to-llvm.mlir
@@ -4,35 +4,35 @@
 // Array type
 //===----------------------------------------------------------------------===//
 
-// CHECK-LABEL: @array(!llvm<"[16 x float]">, !llvm<"[32 x <4 x float>]">)
+// CHECK-LABEL: @array(!llvm.array<16 x float>, !llvm.array<32 x vec<4 x float>>)
 func @array(!spv.array<16xf32>, !spv.array< 32 x vector<4xf32> >) -> ()
 
 //===----------------------------------------------------------------------===//
 // Pointer type
 //===----------------------------------------------------------------------===//
 
-// CHECK-LABEL: @pointer_scalar(!llvm<"i1*">, !llvm<"float*">)
+// CHECK-LABEL: @pointer_scalar(!llvm.ptr<i1>, !llvm.ptr<float>)
 func @pointer_scalar(!spv.ptr<i1, Uniform>, !spv.ptr<f32, Private>) -> ()
 
-// CHECK-LABEL: @pointer_vector(!llvm<"<4 x i32>*">)
+// CHECK-LABEL: @pointer_vector(!llvm.ptr<vec<4 x i32>>)
 func @pointer_vector(!spv.ptr<vector<4xi32>, Function>) -> ()
 
 //===----------------------------------------------------------------------===//
 // Runtime array type
 //===----------------------------------------------------------------------===//
 
-// CHECK-LABEL: @runtime_array_vector(!llvm<"[0 x <4 x float>]">)
+// CHECK-LABEL: @runtime_array_vector(!llvm.array<0 x vec<4 x float>>)
 func @runtime_array_vector(!spv.rtarray< vector<4xf32> >) -> ()
 
-// CHECK-LABEL: @runtime_array_scalar(!llvm<"[0 x float]">)
+// CHECK-LABEL: @runtime_array_scalar(!llvm.array<0 x float>)
 func @runtime_array_scalar(!spv.rtarray<f32>) -> ()
 
 //===----------------------------------------------------------------------===//
 // Struct type
 //===----------------------------------------------------------------------===//
 
-// CHECK-LABEL: @struct(!llvm<"<{ double }>">)
+// CHECK-LABEL: @struct(!llvm.struct<packed (double)>)
 func @struct(!spv.struct<f64>) -> ()
 
-// CHECK-LABEL: @struct_nested(!llvm<"<{ i32, <{ i64, i32 }> }>">)
+// CHECK-LABEL: @struct_nested(!llvm.struct<packed (i32, struct<packed (i64, i32)>)>)
 func @struct_nested(!spv.struct<i32, !spv.struct<i64, i32>>)
diff --git a/mlir/test/Conversion/StandardToLLVM/calling-convention.mlir b/mlir/test/Conversion/StandardToLLVM/calling-convention.mlir
index e17bf3e24225a..4cdf9d6bace87 100644
--- a/mlir/test/Conversion/StandardToLLVM/calling-convention.mlir
+++ b/mlir/test/Conversion/StandardToLLVM/calling-convention.mlir
@@ -8,11 +8,11 @@
 
 // An external function is transformed into the glue around calling an interface function.
 // CHECK-LABEL: @external
-// CHECK: %[[ALLOC0:.*]]: !llvm<"float*">, %[[ALIGN0:.*]]: !llvm<"float*">, %[[OFFSET0:.*]]: !llvm.i64, %[[SIZE00:.*]]: !llvm.i64, %[[SIZE01:.*]]: !llvm.i64, %[[STRIDE00:.*]]: !llvm.i64, %[[STRIDE01:.*]]: !llvm.i64,
-// CHECK: %[[ALLOC1:.*]]: !llvm<"float*">, %[[ALIGN1:.*]]: !llvm<"float*">, %[[OFFSET1:.*]]: !llvm.i64)
+// CHECK: %[[ALLOC0:.*]]: !llvm.ptr<float>, %[[ALIGN0:.*]]: !llvm.ptr<float>, %[[OFFSET0:.*]]: !llvm.i64, %[[SIZE00:.*]]: !llvm.i64, %[[SIZE01:.*]]: !llvm.i64, %[[STRIDE00:.*]]: !llvm.i64, %[[STRIDE01:.*]]: !llvm.i64,
+// CHECK: %[[ALLOC1:.*]]: !llvm.ptr<float>, %[[ALIGN1:.*]]: !llvm.ptr<float>, %[[OFFSET1:.*]]: !llvm.i64)
 func @external(%arg0: memref<?x?xf32>, %arg1: memref<f32>)
   // Populate the descriptor for arg0.
-  // CHECK: %[[DESC00:.*]] = llvm.mlir.undef : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+  // CHECK: %[[DESC00:.*]] = llvm.mlir.undef : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: %[[DESC01:.*]] = llvm.insertvalue %arg0, %[[DESC00]][0]
   // CHECK: %[[DESC02:.*]] = llvm.insertvalue %arg1, %[[DESC01]][1]
   // CHECK: %[[DESC03:.*]] = llvm.insertvalue %arg2, %[[DESC02]][2]
@@ -23,18 +23,18 @@ func @external(%arg0: memref<?x?xf32>, %arg1: memref<f32>)
 
   // Allocate on stack and store to comply with C calling convention.
   // CHECK: %[[C1:.*]] = llvm.mlir.constant(1 : index)
-  // CHECK: %[[DESC0_ALLOCA:.*]] = llvm.alloca %[[C1]] x !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+  // CHECK: %[[DESC0_ALLOCA:.*]] = llvm.alloca %[[C1]] x !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: llvm.store %[[DESC07]], %[[DESC0_ALLOCA]]
 
   // Populate the descriptor for arg1.
-  // CHECK: %[[DESC10:.*]] = llvm.mlir.undef : !llvm<"{ float*, float*, i64 }">
-  // CHECK: %[[DESC11:.*]] = llvm.insertvalue %arg7, %[[DESC10]][0] : !llvm<"{ float*, float*, i64 }">
-  // CHECK: %[[DESC12:.*]] = llvm.insertvalue %arg8, %[[DESC11]][1] : !llvm<"{ float*, float*, i64 }">
-  // CHECK: %[[DESC13:.*]] = llvm.insertvalue %arg9, %[[DESC12]][2] : !llvm<"{ float*, float*, i64 }">
+  // CHECK: %[[DESC10:.*]] = llvm.mlir.undef : !llvm.struct<(ptr<float>, ptr<float>, i64)>
+  // CHECK: %[[DESC11:.*]] = llvm.insertvalue %arg7, %[[DESC10]][0] : !llvm.struct<(ptr<float>, ptr<float>, i64)>
+  // CHECK: %[[DESC12:.*]] = llvm.insertvalue %arg8, %[[DESC11]][1] : !llvm.struct<(ptr<float>, ptr<float>, i64)>
+  // CHECK: %[[DESC13:.*]] = llvm.insertvalue %arg9, %[[DESC12]][2] : !llvm.struct<(ptr<float>, ptr<float>, i64)>
 
   // Allocate on stack and store to comply with C calling convention.
   // CHECK: %[[C1:.*]] = llvm.mlir.constant(1 : index)
-  // CHECK: %[[DESC1_ALLOCA:.*]] = llvm.alloca %[[C1]] x !llvm<"{ float*, float*, i64 }">
+  // CHECK: %[[DESC1_ALLOCA:.*]] = llvm.alloca %[[C1]] x !llvm.struct<(ptr<float>, ptr<float>, i64)>
   // CHECK: llvm.store %[[DESC13]], %[[DESC1_ALLOCA]]
 
   // Call the interface function.
@@ -42,18 +42,18 @@ func @external(%arg0: memref<?x?xf32>, %arg1: memref<f32>)
 
 // Verify that an interface function is emitted.
 // CHECK-LABEL: llvm.func @_mlir_ciface_external
-// CHECK: (!llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }*">, !llvm<"{ float*, float*, i64 }*">)
+// CHECK: (!llvm.ptr<struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>>, !llvm.ptr<struct<(ptr<float>, ptr<float>, i64)>>)
 
 // Verify that the return value is not affected.
 // CHECK-LABEL: @returner
-// CHECK: -> !llvm<"{ { float*, float*, i64, [2 x i64], [2 x i64] }, { float*, float*, i64 } }">
+// CHECK: -> !llvm.struct<(struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>, struct<(ptr<float>, ptr<float>, i64)>)>
 func @returner() -> (memref<?x?xf32>, memref<f32>)
 
 // CHECK-LABEL: @caller
 func @caller() {
   %0:2 = call @returner() : () -> (memref<?x?xf32>, memref<f32>)
   // Extract individual values from the descriptor for the first memref.
-  // CHECK: %[[ALLOC0:.*]] = llvm.extractvalue %[[DESC0:.*]][0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+  // CHECK: %[[ALLOC0:.*]] = llvm.extractvalue %[[DESC0:.*]][0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: %[[ALIGN0:.*]] = llvm.extractvalue %[[DESC0]][1]
   // CHECK: %[[OFFSET0:.*]] = llvm.extractvalue %[[DESC0]][2]
   // CHECK: %[[SIZE00:.*]] = llvm.extractvalue %[[DESC0]][3, 0]
@@ -62,12 +62,12 @@ func @caller() {
   // CHECK: %[[STRIDE01:.*]] = llvm.extractvalue %[[DESC0]][4, 1]
 
   // Extract individual values from the descriptor for the second memref.
-  // CHECK: %[[ALLOC1:.*]] = llvm.extractvalue %[[DESC1:.*]][0] : !llvm<"{ float*, float*, i64 }">
+  // CHECK: %[[ALLOC1:.*]] = llvm.extractvalue %[[DESC1:.*]][0] : !llvm.struct<(ptr<float>, ptr<float>, i64)>
   // CHECK: %[[ALIGN1:.*]] = llvm.extractvalue %[[DESC1]][1]
   // CHECK: %[[OFFSET1:.*]] = llvm.extractvalue %[[DESC1]][2]
 
   // Forward the values to the call.
-  // CHECK: llvm.call @external(%[[ALLOC0]], %[[ALIGN0]], %[[OFFSET0]], %[[SIZE00]], %[[SIZE01]], %[[STRIDE00]], %[[STRIDE01]], %[[ALLOC1]], %[[ALIGN1]], %[[OFFSET1]]) : (!llvm<"float*">, !llvm<"float*">, !llvm.i64, !llvm.i64, !llvm.i64, !llvm.i64, !llvm.i64, !llvm<"float*">, !llvm<"float*">, !llvm.i64) -> ()
+  // CHECK: llvm.call @external(%[[ALLOC0]], %[[ALIGN0]], %[[OFFSET0]], %[[SIZE00]], %[[SIZE01]], %[[STRIDE00]], %[[STRIDE01]], %[[ALLOC1]], %[[ALIGN1]], %[[OFFSET1]]) : (!llvm.ptr<float>, !llvm.ptr<float>, !llvm.i64, !llvm.i64, !llvm.i64, !llvm.i64, !llvm.i64, !llvm.ptr<float>, !llvm.ptr<float>, !llvm.i64) -> ()
   call @external(%0#0, %0#1) : (memref<?x?xf32>, memref<f32>) -> ()
   return
 }
@@ -81,9 +81,9 @@ func @callee(%arg0: memref<?xf32>, %arg1: index) {
 
 // Verify that an interface function is emitted.
 // CHECK-LABEL: @_mlir_ciface_callee
-// CHECK: %[[ARG0:.*]]: !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }*">
+// CHECK: %[[ARG0:.*]]: !llvm.ptr<struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>>
   // Load the memref descriptor pointer.
-  // CHECK: %[[DESC:.*]] = llvm.load %[[ARG0]] : !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }*">
+  // CHECK: %[[DESC:.*]] = llvm.load %[[ARG0]] : !llvm.ptr<struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>>
 
   // Extract individual components of the descriptor.
   // CHECK: %[[ALLOC:.*]] = llvm.extractvalue %[[DESC]][0]
@@ -93,7 +93,7 @@ func @callee(%arg0: memref<?xf32>, %arg1: index) {
   // CHECK: %[[STRIDE:.*]] = llvm.extractvalue %[[DESC]][4, 0]
 
   // Forward the descriptor components to the call.
-  // CHECK: llvm.call @callee(%[[ALLOC]], %[[ALIGN]], %[[OFFSET]], %[[SIZE]], %[[STRIDE]], %{{.*}}) : (!llvm<"float*">, !llvm<"float*">, !llvm.i64, !llvm.i64, !llvm.i64, !llvm.i64) -> ()
+  // CHECK: llvm.call @callee(%[[ALLOC]], %[[ALIGN]], %[[OFFSET]], %[[SIZE]], %[[STRIDE]], %{{.*}}) : (!llvm.ptr<float>, !llvm.ptr<float>, !llvm.i64, !llvm.i64, !llvm.i64, !llvm.i64) -> ()
 
 //   EMIT_C_ATTRIBUTE-NOT: @mlir_ciface_callee
 
@@ -126,7 +126,7 @@ func @return_var_memref_caller(%arg0: memref<4x3xf32>) {
   // CHECK: %[[IDX_SIZE:.*]] = llvm.mlir.constant
 
   // CHECK: %[[DOUBLE_PTR_SIZE:.*]] = llvm.mul %[[TWO]], %[[PTR_SIZE]]
-  // CHECK: %[[RANK:.*]] = llvm.extractvalue %[[CALL_RES]][0] : !llvm<"{ i64, i8* }">
+  // CHECK: %[[RANK:.*]] = llvm.extractvalue %[[CALL_RES]][0] : !llvm.struct<(i64, ptr<i8>)>
   // CHECK: %[[DOUBLE_RANK:.*]] = llvm.mul %[[TWO]], %[[RANK]]
   // CHECK: %[[DOUBLE_RANK_INC:.*]] = llvm.add %[[DOUBLE_RANK]], %[[ONE]]
   // CHECK: %[[TABLES_SIZE:.*]] = llvm.mul %[[DOUBLE_RANK_INC]], %[[IDX_SIZE]]
@@ -136,8 +136,8 @@ func @return_var_memref_caller(%arg0: memref<4x3xf32>) {
   // CHECK: %[[SOURCE:.*]] = llvm.extractvalue %[[CALL_RES]][1]
   // CHECK: "llvm.intr.memcpy"(%[[ALLOCA]], %[[SOURCE]], %[[ALLOC_SIZE]], %[[FALSE]])
   // CHECK: llvm.call @free(%[[SOURCE]])
-  // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm<"{ i64, i8* }">
-  // CHECK: %[[RANK:.*]] = llvm.extractvalue %[[CALL_RES]][0] : !llvm<"{ i64, i8* }">
+  // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(i64, ptr<i8>)>
+  // CHECK: %[[RANK:.*]] = llvm.extractvalue %[[CALL_RES]][0] : !llvm.struct<(i64, ptr<i8>)>
   // CHECK: %[[DESC_1:.*]] = llvm.insertvalue %[[RANK]], %[[DESC]][0]
   // CHECK: llvm.insertvalue %[[ALLOCA]], %[[DESC_1]][1]
   return
@@ -148,7 +148,7 @@ func @return_var_memref(%arg0: memref<4x3xf32>) -> memref<*xf32> {
   // Match the construction of the unranked descriptor.
   // CHECK: %[[ALLOCA:.*]] = llvm.alloca
   // CHECK: %[[MEMORY:.*]] = llvm.bitcast %[[ALLOCA]]
-  // CHECK: %[[DESC_0:.*]] = llvm.mlir.undef : !llvm<"{ i64, i8* }">
+  // CHECK: %[[DESC_0:.*]] = llvm.mlir.undef : !llvm.struct<(i64, ptr<i8>)>
   // CHECK: %[[DESC_1:.*]] = llvm.insertvalue %{{.*}}, %[[DESC_0]][0]
   // CHECK: %[[DESC_2:.*]] = llvm.insertvalue %[[MEMORY]], %[[DESC_1]][1]
   %0 = memref_cast %arg0: memref<4x3xf32> to memref<*xf32>
@@ -160,7 +160,7 @@ func @return_var_memref(%arg0: memref<4x3xf32>) -> memref<*xf32> {
   // CHECK: %[[IDX_SIZE:.*]] = llvm.mlir.constant
 
   // CHECK: %[[DOUBLE_PTR_SIZE:.*]] = llvm.mul %[[TWO]], %[[PTR_SIZE]]
-  // CHECK: %[[RANK:.*]] = llvm.extractvalue %[[DESC_2]][0] : !llvm<"{ i64, i8* }">
+  // CHECK: %[[RANK:.*]] = llvm.extractvalue %[[DESC_2]][0] : !llvm.struct<(i64, ptr<i8>)>
   // CHECK: %[[DOUBLE_RANK:.*]] = llvm.mul %[[TWO]], %[[RANK]]
   // CHECK: %[[DOUBLE_RANK_INC:.*]] = llvm.add %[[DOUBLE_RANK]], %[[ONE]]
   // CHECK: %[[TABLES_SIZE:.*]] = llvm.mul %[[DOUBLE_RANK_INC]], %[[IDX_SIZE]]
@@ -169,8 +169,8 @@ func @return_var_memref(%arg0: memref<4x3xf32>) -> memref<*xf32> {
   // CHECK: %[[ALLOCATED:.*]] = llvm.call @malloc(%[[ALLOC_SIZE]])
   // CHECK: %[[SOURCE:.*]] = llvm.extractvalue %[[DESC_2]][1]
   // CHECK: "llvm.intr.memcpy"(%[[ALLOCATED]], %[[SOURCE]], %[[ALLOC_SIZE]], %[[FALSE]])
-  // CHECK: %[[NEW_DESC:.*]] = llvm.mlir.undef : !llvm<"{ i64, i8* }">
-  // CHECK: %[[RANK:.*]] = llvm.extractvalue %[[DESC_2]][0] : !llvm<"{ i64, i8* }">
+  // CHECK: %[[NEW_DESC:.*]] = llvm.mlir.undef : !llvm.struct<(i64, ptr<i8>)>
+  // CHECK: %[[RANK:.*]] = llvm.extractvalue %[[DESC_2]][0] : !llvm.struct<(i64, ptr<i8>)>
   // CHECK: %[[NEW_DESC_1:.*]] = llvm.insertvalue %[[RANK]], %[[NEW_DESC]][0]
   // CHECK: %[[NEW_DESC_2:.*]] = llvm.insertvalue %[[ALLOCATED]], %[[NEW_DESC_1]][1]
   // CHECL: llvm.return %[[NEW_DESC_2]]
@@ -210,7 +210,7 @@ func @return_two_var_memref(%arg0: memref<4x3xf32>) -> (memref<*xf32>, memref<*x
   // Match the construction of the unranked descriptor.
   // CHECK: %[[ALLOCA:.*]] = llvm.alloca
   // CHECK: %[[MEMORY:.*]] = llvm.bitcast %[[ALLOCA]]
-  // CHECK: %[[DESC_0:.*]] = llvm.mlir.undef : !llvm<"{ i64, i8* }">
+  // CHECK: %[[DESC_0:.*]] = llvm.mlir.undef : !llvm.struct<(i64, ptr<i8>)>
   // CHECK: %[[DESC_1:.*]] = llvm.insertvalue %{{.*}}, %[[DESC_0]][0]
   // CHECK: %[[DESC_2:.*]] = llvm.insertvalue %[[MEMORY]], %[[DESC_1]][1]
   %0 = memref_cast %arg0 : memref<4x3xf32> to memref<*xf32>
@@ -233,7 +233,7 @@ func @return_two_var_memref(%arg0: memref<4x3xf32>) -> (memref<*xf32>, memref<*x
   // CHECK: %[[RES_21:.*]] = llvm.insertvalue %{{.*}}, %[[RES_2]][0]
   // CHECK: %[[RES_22:.*]] = llvm.insertvalue %[[ALLOCATED_2]], %[[RES_21]][1]
 
-  // CHECK: %[[RESULTS:.*]] = llvm.mlir.undef : !llvm<"{ { i64, i8* }, { i64, i8* } }">
+  // CHECK: %[[RESULTS:.*]] = llvm.mlir.undef : !llvm.struct<(struct<(i64, ptr<i8>)>, struct<(i64, ptr<i8>)>)>
   // CHECK: %[[RESULTS_1:.*]] = llvm.insertvalue %[[RES_12]], %[[RESULTS]]
   // CHECK: %[[RESULTS_2:.*]] = llvm.insertvalue %[[RES_22]], %[[RESULTS_1]]
   // CHECK: llvm.return %[[RESULTS_2]]
diff --git a/mlir/test/Conversion/StandardToLLVM/convert-dynamic-memref-ops.mlir b/mlir/test/Conversion/StandardToLLVM/convert-dynamic-memref-ops.mlir
index 32365f362d047..e5d8ca72d9ad6 100644
--- a/mlir/test/Conversion/StandardToLLVM/convert-dynamic-memref-ops.mlir
+++ b/mlir/test/Conversion/StandardToLLVM/convert-dynamic-memref-ops.mlir
@@ -2,11 +2,11 @@
 // RUN: mlir-opt -convert-std-to-llvm='use-aligned-alloc=1' %s | FileCheck %s --check-prefix=ALIGNED-ALLOC
 
 // CHECK-LABEL: func @check_strided_memref_arguments(
-// CHECK-COUNT-2: !llvm<"float*">
+// CHECK-COUNT-2: !llvm.ptr<float>
 // CHECK-COUNT-5: !llvm.i64
-// CHECK-COUNT-2: !llvm<"float*">
+// CHECK-COUNT-2: !llvm.ptr<float>
 // CHECK-COUNT-5: !llvm.i64
-// CHECK-COUNT-2: !llvm<"float*">
+// CHECK-COUNT-2: !llvm.ptr<float>
 // CHECK-COUNT-5: !llvm.i64
 func @check_strided_memref_arguments(%static: memref<10x20xf32, affine_map<(i,j)->(20 * i + j + 1)>>,
                                      %dynamic : memref<?x?xf32, affine_map<(i,j)[M]->(M * i + j + 1)>>,
@@ -15,148 +15,148 @@ func @check_strided_memref_arguments(%static: memref<10x20xf32, affine_map<(i,j)
 }
 
 // CHECK-LABEL: func @check_arguments
-// CHECK-COUNT-2: !llvm<"float*">
+// CHECK-COUNT-2: !llvm.ptr<float>
 // CHECK-COUNT-5: !llvm.i64
-// CHECK-COUNT-2: !llvm<"float*">
+// CHECK-COUNT-2: !llvm.ptr<float>
 // CHECK-COUNT-5: !llvm.i64
-// CHECK-COUNT-2: !llvm<"float*">
+// CHECK-COUNT-2: !llvm.ptr<float>
 // CHECK-COUNT-5: !llvm.i64
 func @check_arguments(%static: memref<10x20xf32>, %dynamic : memref<?x?xf32>, %mixed : memref<10x?xf32>) {
   return
 }
 
 // CHECK-LABEL: func @mixed_alloc(
-//       CHECK:   %[[M:.*]]: !llvm.i64, %[[N:.*]]: !llvm.i64) -> !llvm<"{ float*, float*, i64, [3 x i64], [3 x i64] }"> {
+//       CHECK:   %[[M:.*]]: !llvm.i64, %[[N:.*]]: !llvm.i64) -> !llvm.struct<(ptr<float>, ptr<float>, i64, array<3 x i64>, array<3 x i64>)> {
 func @mixed_alloc(%arg0: index, %arg1: index) -> memref<?x42x?xf32> {
 //       CHECK:  %[[c42:.*]] = llvm.mlir.constant(42 : index) : !llvm.i64
 //  CHECK-NEXT:  llvm.mul %[[M]], %[[c42]] : !llvm.i64
 //  CHECK-NEXT:  %[[sz:.*]] = llvm.mul %{{.*}}, %[[N]] : !llvm.i64
-//  CHECK-NEXT:  %[[null:.*]] = llvm.mlir.null : !llvm<"float*">
+//  CHECK-NEXT:  %[[null:.*]] = llvm.mlir.null : !llvm.ptr<float>
 //  CHECK-NEXT:  %[[one:.*]] = llvm.mlir.constant(1 : index) : !llvm.i64
-//  CHECK-NEXT:  %[[gep:.*]] = llvm.getelementptr %[[null]][%[[one]]] : (!llvm<"float*">, !llvm.i64) -> !llvm<"float*">
-//  CHECK-NEXT:  %[[sizeof:.*]] = llvm.ptrtoint %[[gep]] : !llvm<"float*"> to !llvm.i64
+//  CHECK-NEXT:  %[[gep:.*]] = llvm.getelementptr %[[null]][%[[one]]] : (!llvm.ptr<float>, !llvm.i64) -> !llvm.ptr<float>
+//  CHECK-NEXT:  %[[sizeof:.*]] = llvm.ptrtoint %[[gep]] : !llvm.ptr<float> to !llvm.i64
 //  CHECK-NEXT:  %[[sz_bytes:.*]] = llvm.mul %[[sz]], %[[sizeof]] : !llvm.i64
 //  CHECK-NEXT:  %[[one_1:.*]] = llvm.mlir.constant(1 : index) : !llvm.i64
-//  CHECK-NEXT:  llvm.call @malloc(%[[sz_bytes]]) : (!llvm.i64) -> !llvm<"i8*">
-//  CHECK-NEXT:  llvm.bitcast %{{.*}} : !llvm<"i8*"> to !llvm<"float*">
-//  CHECK-NEXT:  llvm.mlir.undef : !llvm<"{ float*, float*, i64, [3 x i64], [3 x i64] }">
-//  CHECK-NEXT:  llvm.insertvalue %{{.*}}, %{{.*}}[0] : !llvm<"{ float*, float*, i64, [3 x i64], [3 x i64] }">
-//  CHECK-NEXT:  llvm.insertvalue %{{.*}}, %{{.*}}[1] : !llvm<"{ float*, float*, i64, [3 x i64], [3 x i64] }">
+//  CHECK-NEXT:  llvm.call @malloc(%[[sz_bytes]]) : (!llvm.i64) -> !llvm.ptr<i8>
+//  CHECK-NEXT:  llvm.bitcast %{{.*}} : !llvm.ptr<i8> to !llvm.ptr<float>
+//  CHECK-NEXT:  llvm.mlir.undef : !llvm.struct<(ptr<float>, ptr<float>, i64, array<3 x i64>, array<3 x i64>)>
+//  CHECK-NEXT:  llvm.insertvalue %{{.*}}, %{{.*}}[0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<3 x i64>, array<3 x i64>)>
+//  CHECK-NEXT:  llvm.insertvalue %{{.*}}, %{{.*}}[1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<3 x i64>, array<3 x i64>)>
 //  CHECK-NEXT:  %[[off:.*]] = llvm.mlir.constant(0 : index) : !llvm.i64
-//  CHECK-NEXT:  llvm.insertvalue %[[off]], %{{.*}}[2] : !llvm<"{ float*, float*, i64, [3 x i64], [3 x i64] }">
+//  CHECK-NEXT:  llvm.insertvalue %[[off]], %{{.*}}[2] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<3 x i64>, array<3 x i64>)>
 //  CHECK-NEXT:  %[[st2:.*]] = llvm.mlir.constant(1 : index) : !llvm.i64
 //  CHECK-NEXT:  %[[st1:.*]] = llvm.mul %{{.*}}, %[[N]] : !llvm.i64
 //  CHECK-NEXT:  %[[st0:.*]] = llvm.mul %{{.*}}, %[[c42]] : !llvm.i64
-//  CHECK-NEXT:  llvm.insertvalue %[[M]], %{{.*}}[3, 0] : !llvm<"{ float*, float*, i64, [3 x i64], [3 x i64] }">
-//  CHECK-NEXT:  llvm.insertvalue %[[st0]], %{{.*}}[4, 0] : !llvm<"{ float*, float*, i64, [3 x i64], [3 x i64] }">
-//  CHECK-NEXT:  llvm.insertvalue %[[c42]], %{{.*}}[3, 1] : !llvm<"{ float*, float*, i64, [3 x i64], [3 x i64] }">
-//  CHECK-NEXT:  llvm.insertvalue %[[st1]], %{{.*}}[4, 1] : !llvm<"{ float*, float*, i64, [3 x i64], [3 x i64] }">
-//  CHECK-NEXT:  llvm.insertvalue %[[N]], %{{.*}}[3, 2] : !llvm<"{ float*, float*, i64, [3 x i64], [3 x i64] }">
-//  CHECK-NEXT:  llvm.insertvalue %[[st2]], %{{.*}}[4, 2] : !llvm<"{ float*, float*, i64, [3 x i64], [3 x i64] }">
+//  CHECK-NEXT:  llvm.insertvalue %[[M]], %{{.*}}[3, 0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<3 x i64>, array<3 x i64>)>
+//  CHECK-NEXT:  llvm.insertvalue %[[st0]], %{{.*}}[4, 0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<3 x i64>, array<3 x i64>)>
+//  CHECK-NEXT:  llvm.insertvalue %[[c42]], %{{.*}}[3, 1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<3 x i64>, array<3 x i64>)>
+//  CHECK-NEXT:  llvm.insertvalue %[[st1]], %{{.*}}[4, 1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<3 x i64>, array<3 x i64>)>
+//  CHECK-NEXT:  llvm.insertvalue %[[N]], %{{.*}}[3, 2] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<3 x i64>, array<3 x i64>)>
+//  CHECK-NEXT:  llvm.insertvalue %[[st2]], %{{.*}}[4, 2] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<3 x i64>, array<3 x i64>)>
   %0 = alloc(%arg0, %arg1) : memref<?x42x?xf32>
-//  CHECK-NEXT:  llvm.return %{{.*}} : !llvm<"{ float*, float*, i64, [3 x i64], [3 x i64] }">
+//  CHECK-NEXT:  llvm.return %{{.*}} : !llvm.struct<(ptr<float>, ptr<float>, i64, array<3 x i64>, array<3 x i64>)>
   return %0 : memref<?x42x?xf32>
 }
 
 // CHECK-LABEL: func @mixed_dealloc
 func @mixed_dealloc(%arg0: memref<?x42x?xf32>) {
-//      CHECK:  %[[ptr:.*]] = llvm.extractvalue %{{.*}}[0] : !llvm<"{ float*, float*, i64, [3 x i64], [3 x i64] }">
-// CHECK-NEXT:  %[[ptri8:.*]] = llvm.bitcast %[[ptr]] : !llvm<"float*"> to !llvm<"i8*">
-// CHECK-NEXT:  llvm.call @free(%[[ptri8]]) : (!llvm<"i8*">) -> ()
+//      CHECK:  %[[ptr:.*]] = llvm.extractvalue %{{.*}}[0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<3 x i64>, array<3 x i64>)>
+// CHECK-NEXT:  %[[ptri8:.*]] = llvm.bitcast %[[ptr]] : !llvm.ptr<float> to !llvm.ptr<i8>
+// CHECK-NEXT:  llvm.call @free(%[[ptri8]]) : (!llvm.ptr<i8>) -> ()
   dealloc %arg0 : memref<?x42x?xf32>
 // CHECK-NEXT:  llvm.return
   return
 }
 
 // CHECK-LABEL: func @dynamic_alloc(
-//       CHECK:   %[[M:.*]]: !llvm.i64, %[[N:.*]]: !llvm.i64) -> !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }"> {
+//       CHECK:   %[[M:.*]]: !llvm.i64, %[[N:.*]]: !llvm.i64) -> !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)> {
 func @dynamic_alloc(%arg0: index, %arg1: index) -> memref<?x?xf32> {
 //       CHECK:  %[[sz:.*]] = llvm.mul %[[M]], %[[N]] : !llvm.i64
-//  CHECK-NEXT:  %[[null:.*]] = llvm.mlir.null : !llvm<"float*">
+//  CHECK-NEXT:  %[[null:.*]] = llvm.mlir.null : !llvm.ptr<float>
 //  CHECK-NEXT:  %[[one:.*]] = llvm.mlir.constant(1 : index) : !llvm.i64
-//  CHECK-NEXT:  %[[gep:.*]] = llvm.getelementptr %[[null]][%[[one]]] : (!llvm<"float*">, !llvm.i64) -> !llvm<"float*">
-//  CHECK-NEXT:  %[[sizeof:.*]] = llvm.ptrtoint %[[gep]] : !llvm<"float*"> to !llvm.i64
+//  CHECK-NEXT:  %[[gep:.*]] = llvm.getelementptr %[[null]][%[[one]]] : (!llvm.ptr<float>, !llvm.i64) -> !llvm.ptr<float>
+//  CHECK-NEXT:  %[[sizeof:.*]] = llvm.ptrtoint %[[gep]] : !llvm.ptr<float> to !llvm.i64
 //  CHECK-NEXT:  %[[sz_bytes:.*]] = llvm.mul %[[sz]], %[[sizeof]] : !llvm.i64
 //  CHECK-NEXT:  %[[one_1:.*]] = llvm.mlir.constant(1 : index) : !llvm.i64
-//  CHECK-NEXT:  llvm.call @malloc(%[[sz_bytes]]) : (!llvm.i64) -> !llvm<"i8*">
-//  CHECK-NEXT:  llvm.bitcast %{{.*}} : !llvm<"i8*"> to !llvm<"float*">
-//  CHECK-NEXT:  llvm.mlir.undef : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
-//  CHECK-NEXT:  llvm.insertvalue %{{.*}}, %{{.*}}[0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
-//  CHECK-NEXT:  llvm.insertvalue %{{.*}}, %{{.*}}[1] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+//  CHECK-NEXT:  llvm.call @malloc(%[[sz_bytes]]) : (!llvm.i64) -> !llvm.ptr<i8>
+//  CHECK-NEXT:  llvm.bitcast %{{.*}} : !llvm.ptr<i8> to !llvm.ptr<float>
+//  CHECK-NEXT:  llvm.mlir.undef : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
+//  CHECK-NEXT:  llvm.insertvalue %{{.*}}, %{{.*}}[0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
+//  CHECK-NEXT:  llvm.insertvalue %{{.*}}, %{{.*}}[1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
 //  CHECK-NEXT:  %[[off:.*]] = llvm.mlir.constant(0 : index) : !llvm.i64
-//  CHECK-NEXT:  llvm.insertvalue %[[off]], %{{.*}}[2] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+//  CHECK-NEXT:  llvm.insertvalue %[[off]], %{{.*}}[2] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
 //  CHECK-NEXT:  %[[st1:.*]] = llvm.mlir.constant(1 : index) : !llvm.i64
 //  CHECK-NEXT:  %[[st0:.*]] = llvm.mul %{{.*}}, %[[N]] : !llvm.i64
-//  CHECK-NEXT:  llvm.insertvalue %[[M]], %{{.*}}[3, 0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
-//  CHECK-NEXT:  llvm.insertvalue %[[st0]], %{{.*}}[4, 0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
-//  CHECK-NEXT:  llvm.insertvalue %[[N]], %{{.*}}[3, 1] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
-//  CHECK-NEXT:  llvm.insertvalue %[[st1]], %{{.*}}[4, 1] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+//  CHECK-NEXT:  llvm.insertvalue %[[M]], %{{.*}}[3, 0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
+//  CHECK-NEXT:  llvm.insertvalue %[[st0]], %{{.*}}[4, 0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
+//  CHECK-NEXT:  llvm.insertvalue %[[N]], %{{.*}}[3, 1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
+//  CHECK-NEXT:  llvm.insertvalue %[[st1]], %{{.*}}[4, 1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
   %0 = alloc(%arg0, %arg1) : memref<?x?xf32>
-//  CHECK-NEXT:  llvm.return %{{.*}} : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+//  CHECK-NEXT:  llvm.return %{{.*}} : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
   return %0 : memref<?x?xf32>
 }
 
 // -----
 
 // CHECK-LABEL: func @dynamic_alloca
-// CHECK: %[[M:.*]]: !llvm.i64, %[[N:.*]]: !llvm.i64) -> !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }"> {
+// CHECK: %[[M:.*]]: !llvm.i64, %[[N:.*]]: !llvm.i64) -> !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)> {
 func @dynamic_alloca(%arg0: index, %arg1: index) -> memref<?x?xf32> {
 //       CHECK:  %[[num_elems:.*]] = llvm.mul %[[M]], %[[N]] : !llvm.i64
-//  CHECK-NEXT:  %[[null:.*]] = llvm.mlir.null : !llvm<"float*">
+//  CHECK-NEXT:  %[[null:.*]] = llvm.mlir.null : !llvm.ptr<float>
 //  CHECK-NEXT:  %[[one:.*]] = llvm.mlir.constant(1 : index) : !llvm.i64
-//  CHECK-NEXT:  %[[gep:.*]] = llvm.getelementptr %[[null]][%[[one]]] : (!llvm<"float*">, !llvm.i64) -> !llvm<"float*">
-//  CHECK-NEXT:  %[[sizeof:.*]] = llvm.ptrtoint %[[gep]] : !llvm<"float*"> to !llvm.i64
+//  CHECK-NEXT:  %[[gep:.*]] = llvm.getelementptr %[[null]][%[[one]]] : (!llvm.ptr<float>, !llvm.i64) -> !llvm.ptr<float>
+//  CHECK-NEXT:  %[[sizeof:.*]] = llvm.ptrtoint %[[gep]] : !llvm.ptr<float> to !llvm.i64
 //  CHECK-NEXT:  %[[sz_bytes:.*]] = llvm.mul %[[num_elems]], %[[sizeof]] : !llvm.i64
 //  CHECK-NEXT:  %[[one_1:.*]] = llvm.mlir.constant(1 : index) : !llvm.i64
-//  CHECK-NEXT:  %[[allocated:.*]] = llvm.alloca %[[sz_bytes]] x !llvm.float : (!llvm.i64) -> !llvm<"float*">
-//  CHECK-NEXT:  llvm.mlir.undef : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
-//  CHECK-NEXT:  llvm.insertvalue %[[allocated]], %{{.*}}[0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
-//  CHECK-NEXT:  llvm.insertvalue %[[allocated]], %{{.*}}[1] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+//  CHECK-NEXT:  %[[allocated:.*]] = llvm.alloca %[[sz_bytes]] x !llvm.float : (!llvm.i64) -> !llvm.ptr<float>
+//  CHECK-NEXT:  llvm.mlir.undef : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
+//  CHECK-NEXT:  llvm.insertvalue %[[allocated]], %{{.*}}[0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
+//  CHECK-NEXT:  llvm.insertvalue %[[allocated]], %{{.*}}[1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
 //  CHECK-NEXT:  %[[off:.*]] = llvm.mlir.constant(0 : index) : !llvm.i64
-//  CHECK-NEXT:  llvm.insertvalue %[[off]], %{{.*}}[2] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+//  CHECK-NEXT:  llvm.insertvalue %[[off]], %{{.*}}[2] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
 //  CHECK-NEXT:  %[[st1:.*]] = llvm.mlir.constant(1 : index) : !llvm.i64
 //  CHECK-NEXT:  %[[st0:.*]] = llvm.mul %{{.*}}, %[[N]] : !llvm.i64
-//  CHECK-NEXT:  llvm.insertvalue %[[M]], %{{.*}}[3, 0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
-//  CHECK-NEXT:  llvm.insertvalue %[[st0]], %{{.*}}[4, 0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
-//  CHECK-NEXT:  llvm.insertvalue %[[N]], %{{.*}}[3, 1] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
-//  CHECK-NEXT:  llvm.insertvalue %[[st1]], %{{.*}}[4, 1] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+//  CHECK-NEXT:  llvm.insertvalue %[[M]], %{{.*}}[3, 0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
+//  CHECK-NEXT:  llvm.insertvalue %[[st0]], %{{.*}}[4, 0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
+//  CHECK-NEXT:  llvm.insertvalue %[[N]], %{{.*}}[3, 1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
+//  CHECK-NEXT:  llvm.insertvalue %[[st1]], %{{.*}}[4, 1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
   %0 = alloca(%arg0, %arg1) : memref<?x?xf32>
 
 // Test with explicitly specified alignment. llvm.alloca takes care of the
 // alignment. The same pointer is thus used for allocation and aligned
 // accesses.
-// CHECK: %[[alloca_aligned:.*]] = llvm.alloca %{{.*}} x !llvm.float {alignment = 32 : i64} : (!llvm.i64) -> !llvm<"float*">
-// CHECK: %[[desc:.*]] = llvm.mlir.undef : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
-// CHECK: %[[desc1:.*]] = llvm.insertvalue %[[alloca_aligned]], %[[desc]][0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
-// CHECK: llvm.insertvalue %[[alloca_aligned]], %[[desc1]][1] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+// CHECK: %[[alloca_aligned:.*]] = llvm.alloca %{{.*}} x !llvm.float {alignment = 32 : i64} : (!llvm.i64) -> !llvm.ptr<float>
+// CHECK: %[[desc:.*]] = llvm.mlir.undef : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
+// CHECK: %[[desc1:.*]] = llvm.insertvalue %[[alloca_aligned]], %[[desc]][0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
+// CHECK: llvm.insertvalue %[[alloca_aligned]], %[[desc1]][1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
   alloca(%arg0, %arg1) {alignment = 32} : memref<?x?xf32>
   return %0 : memref<?x?xf32>
 }
 
 // CHECK-LABEL: func @dynamic_dealloc
 func @dynamic_dealloc(%arg0: memref<?x?xf32>) {
-//      CHECK:  %[[ptr:.*]] = llvm.extractvalue %{{.*}}[0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
-// CHECK-NEXT:  %[[ptri8:.*]] = llvm.bitcast %[[ptr]] : !llvm<"float*"> to !llvm<"i8*">
-// CHECK-NEXT:  llvm.call @free(%[[ptri8]]) : (!llvm<"i8*">) -> ()
+//      CHECK:  %[[ptr:.*]] = llvm.extractvalue %{{.*}}[0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
+// CHECK-NEXT:  %[[ptri8:.*]] = llvm.bitcast %[[ptr]] : !llvm.ptr<float> to !llvm.ptr<i8>
+// CHECK-NEXT:  llvm.call @free(%[[ptri8]]) : (!llvm.ptr<i8>) -> ()
   dealloc %arg0 : memref<?x?xf32>
   return
 }
 
-// CHECK-LABEL: func @stdlib_aligned_alloc({{.*}}) -> !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }"> {
-// ALIGNED-ALLOC-LABEL: func @stdlib_aligned_alloc({{.*}}) -> !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }"> {
+// CHECK-LABEL: func @stdlib_aligned_alloc({{.*}}) -> !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)> {
+// ALIGNED-ALLOC-LABEL: func @stdlib_aligned_alloc({{.*}}) -> !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)> {
 func @stdlib_aligned_alloc(%N : index) -> memref<32x18xf32> {
 // ALIGNED-ALLOC-NEXT:  %[[sz1:.*]] = llvm.mlir.constant(32 : index) : !llvm.i64
 // ALIGNED-ALLOC-NEXT:  %[[sz2:.*]] = llvm.mlir.constant(18 : index) : !llvm.i64
 // ALIGNED-ALLOC-NEXT:  %[[num_elems:.*]] = llvm.mul %0, %1 : !llvm.i64
-// ALIGNED-ALLOC-NEXT:  %[[null:.*]] = llvm.mlir.null : !llvm<"float*">
+// ALIGNED-ALLOC-NEXT:  %[[null:.*]] = llvm.mlir.null : !llvm.ptr<float>
 // ALIGNED-ALLOC-NEXT:  %[[one:.*]] = llvm.mlir.constant(1 : index) : !llvm.i64
-// ALIGNED-ALLOC-NEXT:  %[[gep:.*]] = llvm.getelementptr %[[null]][%[[one]]] : (!llvm<"float*">, !llvm.i64) -> !llvm<"float*">
-// ALIGNED-ALLOC-NEXT:  %[[sizeof:.*]] = llvm.ptrtoint %[[gep]] : !llvm<"float*"> to !llvm.i64
+// ALIGNED-ALLOC-NEXT:  %[[gep:.*]] = llvm.getelementptr %[[null]][%[[one]]] : (!llvm.ptr<float>, !llvm.i64) -> !llvm.ptr<float>
+// ALIGNED-ALLOC-NEXT:  %[[sizeof:.*]] = llvm.ptrtoint %[[gep]] : !llvm.ptr<float> to !llvm.i64
 // ALIGNED-ALLOC-NEXT:  %[[bytes:.*]] = llvm.mul %[[num_elems]], %[[sizeof]] : !llvm.i64
 // ALIGNED-ALLOC-NEXT:  %[[one_1:.*]] = llvm.mlir.constant(1 : index) : !llvm.i64
 // ALIGNED-ALLOC-NEXT:  %[[alignment:.*]] = llvm.mlir.constant(32 : i64) : !llvm.i64
-// ALIGNED-ALLOC-NEXT:  %[[allocated:.*]] = llvm.call @aligned_alloc(%[[alignment]], %[[bytes]]) : (!llvm.i64, !llvm.i64) -> !llvm<"i8*">
-// ALIGNED-ALLOC-NEXT:  llvm.bitcast %[[allocated]] : !llvm<"i8*"> to !llvm<"float*">
+// ALIGNED-ALLOC-NEXT:  %[[allocated:.*]] = llvm.call @aligned_alloc(%[[alignment]], %[[bytes]]) : (!llvm.i64, !llvm.i64) -> !llvm.ptr<i8>
+// ALIGNED-ALLOC-NEXT:  llvm.bitcast %[[allocated]] : !llvm.ptr<i8> to !llvm.ptr<float>
   %0 = alloc() {alignment = 32} : memref<32x18xf32>
   // Do another alloc just to test that we have a unique declaration for
   // aligned_alloc.
@@ -190,28 +190,28 @@ func @stdlib_aligned_alloc(%N : index) -> memref<32x18xf32> {
 }
 
 // CHECK-LABEL: func @mixed_load(
-// CHECK-COUNT-2: !llvm<"float*">,
+// CHECK-COUNT-2: !llvm.ptr<float>,
 // CHECK-COUNT-5: {{%[a-zA-Z0-9]*}}: !llvm.i64
 // CHECK:         %[[I:.*]]: !llvm.i64,
 // CHECK:         %[[J:.*]]: !llvm.i64)
 func @mixed_load(%mixed : memref<42x?xf32>, %i : index, %j : index) {
-//       CHECK:  %[[ptr:.*]] = llvm.extractvalue %[[ld:.*]][1] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+//       CHECK:  %[[ptr:.*]] = llvm.extractvalue %[[ld:.*]][1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
 //  CHECK-NEXT:  %[[off:.*]] = llvm.mlir.constant(0 : index) : !llvm.i64
-//  CHECK-NEXT:  %[[st0:.*]] = llvm.extractvalue %[[ld]][4, 0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+//  CHECK-NEXT:  %[[st0:.*]] = llvm.extractvalue %[[ld]][4, 0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
 //  CHECK-NEXT:  %[[offI:.*]] = llvm.mul %[[I]], %[[st0]] : !llvm.i64
 //  CHECK-NEXT:  %[[off0:.*]] = llvm.add %[[off]], %[[offI]] : !llvm.i64
 //  CHECK-NEXT:  %[[st1:.*]] = llvm.mlir.constant(1 : index) : !llvm.i64
 //  CHECK-NEXT:  %[[offJ:.*]] = llvm.mul %[[J]], %[[st1]] : !llvm.i64
 //  CHECK-NEXT:  %[[off1:.*]] = llvm.add %[[off0]], %[[offJ]] : !llvm.i64
-//  CHECK-NEXT:  %[[addr:.*]] = llvm.getelementptr %[[ptr]][%[[off1]]] : (!llvm<"float*">, !llvm.i64) -> !llvm<"float*">
-//  CHECK-NEXT:  llvm.load %[[addr]] : !llvm<"float*">
+//  CHECK-NEXT:  %[[addr:.*]] = llvm.getelementptr %[[ptr]][%[[off1]]] : (!llvm.ptr<float>, !llvm.i64) -> !llvm.ptr<float>
+//  CHECK-NEXT:  llvm.load %[[addr]] : !llvm.ptr<float>
   %0 = load %mixed[%i, %j] : memref<42x?xf32>
   return
 }
 
 // CHECK-LABEL: func @dynamic_load(
-// CHECK-SAME:         %[[ARG0:[a-zA-Z0-9]*]]: !llvm<"float*">
-// CHECK-SAME:         %[[ARG1:[a-zA-Z0-9]*]]: !llvm<"float*">
+// CHECK-SAME:         %[[ARG0:[a-zA-Z0-9]*]]: !llvm.ptr<float>
+// CHECK-SAME:         %[[ARG1:[a-zA-Z0-9]*]]: !llvm.ptr<float>
 // CHECK-SAME:         %[[ARG2:[a-zA-Z0-9]*]]: !llvm.i64
 // CHECK-SAME:         %[[ARG3:[a-zA-Z0-9]*]]: !llvm.i64
 // CHECK-SAME:         %[[ARG4:[a-zA-Z0-9]*]]: !llvm.i64
@@ -220,23 +220,23 @@ func @mixed_load(%mixed : memref<42x?xf32>, %i : index, %j : index) {
 // CHECK-SAME:         %[[I:[a-zA-Z0-9]*]]: !llvm.i64
 // CHECK-SAME:         %[[J:[a-zA-Z0-9]*]]: !llvm.i64
 func @dynamic_load(%dynamic : memref<?x?xf32>, %i : index, %j : index) {
-//       CHECK:  %[[ptr:.*]] = llvm.extractvalue %[[ld:.*]][1] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+//       CHECK:  %[[ptr:.*]] = llvm.extractvalue %[[ld:.*]][1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
 //  CHECK-NEXT:  %[[off:.*]] = llvm.mlir.constant(0 : index) : !llvm.i64
-//  CHECK-NEXT:  %[[st0:.*]] = llvm.extractvalue %[[ld]][4, 0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+//  CHECK-NEXT:  %[[st0:.*]] = llvm.extractvalue %[[ld]][4, 0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
 //  CHECK-NEXT:  %[[offI:.*]] = llvm.mul %[[I]], %[[st0]] : !llvm.i64
 //  CHECK-NEXT:  %[[off0:.*]] = llvm.add %[[off]], %[[offI]] : !llvm.i64
 //  CHECK-NEXT:  %[[st1:.*]] = llvm.mlir.constant(1 : index) : !llvm.i64
 //  CHECK-NEXT:  %[[offJ:.*]] = llvm.mul %[[J]], %[[st1]] : !llvm.i64
 //  CHECK-NEXT:  %[[off1:.*]] = llvm.add %[[off0]], %[[offJ]] : !llvm.i64
-//  CHECK-NEXT:  %[[addr:.*]] = llvm.getelementptr %[[ptr]][%[[off1]]] : (!llvm<"float*">, !llvm.i64) -> !llvm<"float*">
-//  CHECK-NEXT:  llvm.load %[[addr]] : !llvm<"float*">
+//  CHECK-NEXT:  %[[addr:.*]] = llvm.getelementptr %[[ptr]][%[[off1]]] : (!llvm.ptr<float>, !llvm.i64) -> !llvm.ptr<float>
+//  CHECK-NEXT:  llvm.load %[[addr]] : !llvm.ptr<float>
   %0 = load %dynamic[%i, %j] : memref<?x?xf32>
   return
 }
 
 // CHECK-LABEL: func @prefetch
-// CHECK-SAME:         %[[ARG0:[a-zA-Z0-9]*]]: !llvm<"float*">
-// CHECK-SAME:         %[[ARG1:[a-zA-Z0-9]*]]: !llvm<"float*">
+// CHECK-SAME:         %[[ARG0:[a-zA-Z0-9]*]]: !llvm.ptr<float>
+// CHECK-SAME:         %[[ARG1:[a-zA-Z0-9]*]]: !llvm.ptr<float>
 // CHECK-SAME:         %[[ARG2:[a-zA-Z0-9]*]]: !llvm.i64
 // CHECK-SAME:         %[[ARG3:[a-zA-Z0-9]*]]: !llvm.i64
 // CHECK-SAME:         %[[ARG4:[a-zA-Z0-9]*]]: !llvm.i64
@@ -245,36 +245,36 @@ func @dynamic_load(%dynamic : memref<?x?xf32>, %i : index, %j : index) {
 // CHECK-SAME:         %[[I:[a-zA-Z0-9]*]]: !llvm.i64
 // CHECK-SAME:         %[[J:[a-zA-Z0-9]*]]: !llvm.i64
 func @prefetch(%A : memref<?x?xf32>, %i : index, %j : index) {
-//      CHECK:  %[[ptr:.*]] = llvm.extractvalue %[[ld:.*]][1] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+//      CHECK:  %[[ptr:.*]] = llvm.extractvalue %[[ld:.*]][1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
 // CHECK-NEXT:  %[[off:.*]] = llvm.mlir.constant(0 : index) : !llvm.i64
-// CHECK-NEXT:  %[[st0:.*]] = llvm.extractvalue %[[ld]][4, 0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+// CHECK-NEXT:  %[[st0:.*]] = llvm.extractvalue %[[ld]][4, 0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
 // CHECK-NEXT:  %[[offI:.*]] = llvm.mul %[[I]], %[[st0]] : !llvm.i64
 // CHECK-NEXT:  %[[off0:.*]] = llvm.add %[[off]], %[[offI]] : !llvm.i64
 // CHECK-NEXT:  %[[st1:.*]] = llvm.mlir.constant(1 : index) : !llvm.i64
 // CHECK-NEXT:  %[[offJ:.*]] = llvm.mul %[[J]], %[[st1]] : !llvm.i64
 // CHECK-NEXT:  %[[off1:.*]] = llvm.add %[[off0]], %[[offJ]] : !llvm.i64
-// CHECK-NEXT:  %[[addr:.*]] = llvm.getelementptr %[[ptr]][%[[off1]]] : (!llvm<"float*">, !llvm.i64) -> !llvm<"float*">
+// CHECK-NEXT:  %[[addr:.*]] = llvm.getelementptr %[[ptr]][%[[off1]]] : (!llvm.ptr<float>, !llvm.i64) -> !llvm.ptr<float>
 // CHECK-NEXT:  [[C1:%.*]] = llvm.mlir.constant(1 : i32) : !llvm.i32
 // CHECK-NEXT:  [[C3:%.*]] = llvm.mlir.constant(3 : i32) : !llvm.i32
 // CHECK-NEXT:  [[C1_1:%.*]] = llvm.mlir.constant(1 : i32) : !llvm.i32
-// CHECK-NEXT:  "llvm.intr.prefetch"(%[[addr]], [[C1]], [[C3]], [[C1_1]]) : (!llvm<"float*">, !llvm.i32, !llvm.i32, !llvm.i32) -> ()
+// CHECK-NEXT:  "llvm.intr.prefetch"(%[[addr]], [[C1]], [[C3]], [[C1_1]]) : (!llvm.ptr<float>, !llvm.i32, !llvm.i32, !llvm.i32) -> ()
   prefetch %A[%i, %j], write, locality<3>, data : memref<?x?xf32>
 // CHECK:  [[C0:%.*]] = llvm.mlir.constant(0 : i32) : !llvm.i32
 // CHECK:  [[C0_1:%.*]] = llvm.mlir.constant(0 : i32) : !llvm.i32
 // CHECK:  [[C1_2:%.*]] = llvm.mlir.constant(1 : i32) : !llvm.i32
-// CHECK:  "llvm.intr.prefetch"(%{{.*}}, [[C0]], [[C0_1]], [[C1_2]]) : (!llvm<"float*">, !llvm.i32, !llvm.i32, !llvm.i32) -> ()
+// CHECK:  "llvm.intr.prefetch"(%{{.*}}, [[C0]], [[C0_1]], [[C1_2]]) : (!llvm.ptr<float>, !llvm.i32, !llvm.i32, !llvm.i32) -> ()
   prefetch %A[%i, %j], read, locality<0>, data : memref<?x?xf32>
 // CHECK:  [[C0_2:%.*]] = llvm.mlir.constant(0 : i32) : !llvm.i32
 // CHECK:  [[C2:%.*]] = llvm.mlir.constant(2 : i32) : !llvm.i32
 // CHECK:  [[C0_3:%.*]] = llvm.mlir.constant(0 : i32) : !llvm.i32
-// CHECK:  "llvm.intr.prefetch"(%{{.*}}, [[C0_2]], [[C2]], [[C0_3]]) : (!llvm<"float*">, !llvm.i32, !llvm.i32, !llvm.i32) -> ()
+// CHECK:  "llvm.intr.prefetch"(%{{.*}}, [[C0_2]], [[C2]], [[C0_3]]) : (!llvm.ptr<float>, !llvm.i32, !llvm.i32, !llvm.i32) -> ()
   prefetch %A[%i, %j], read, locality<2>, instr : memref<?x?xf32>
   return
 }
 
 // CHECK-LABEL: func @dynamic_store
-// CHECK-SAME:         %[[ARG0:[a-zA-Z0-9]*]]: !llvm<"float*">
-// CHECK-SAME:         %[[ARG1:[a-zA-Z0-9]*]]: !llvm<"float*">
+// CHECK-SAME:         %[[ARG0:[a-zA-Z0-9]*]]: !llvm.ptr<float>
+// CHECK-SAME:         %[[ARG1:[a-zA-Z0-9]*]]: !llvm.ptr<float>
 // CHECK-SAME:         %[[ARG2:[a-zA-Z0-9]*]]: !llvm.i64
 // CHECK-SAME:         %[[ARG3:[a-zA-Z0-9]*]]: !llvm.i64
 // CHECK-SAME:         %[[ARG4:[a-zA-Z0-9]*]]: !llvm.i64
@@ -283,23 +283,23 @@ func @prefetch(%A : memref<?x?xf32>, %i : index, %j : index) {
 // CHECK-SAME:         %[[I:[a-zA-Z0-9]*]]: !llvm.i64
 // CHECK-SAME:         %[[J:[a-zA-Z0-9]*]]: !llvm.i64
 func @dynamic_store(%dynamic : memref<?x?xf32>, %i : index, %j : index, %val : f32) {
-//       CHECK:  %[[ptr:.*]] = llvm.extractvalue %[[ld:.*]][1] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+//       CHECK:  %[[ptr:.*]] = llvm.extractvalue %[[ld:.*]][1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
 //  CHECK-NEXT:  %[[off:.*]] = llvm.mlir.constant(0 : index) : !llvm.i64
-//  CHECK-NEXT:  %[[st0:.*]] = llvm.extractvalue %[[ld]][4, 0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+//  CHECK-NEXT:  %[[st0:.*]] = llvm.extractvalue %[[ld]][4, 0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
 //  CHECK-NEXT:  %[[offI:.*]] = llvm.mul %[[I]], %[[st0]] : !llvm.i64
 //  CHECK-NEXT:  %[[off0:.*]] = llvm.add %[[off]], %[[offI]] : !llvm.i64
 //  CHECK-NEXT:  %[[st1:.*]] = llvm.mlir.constant(1 : index) : !llvm.i64
 //  CHECK-NEXT:  %[[offJ:.*]] = llvm.mul %[[J]], %[[st1]] : !llvm.i64
 //  CHECK-NEXT:  %[[off1:.*]] = llvm.add %[[off0]], %[[offJ]] : !llvm.i64
-//  CHECK-NEXT:  %[[addr:.*]] = llvm.getelementptr %[[ptr]][%[[off1]]] : (!llvm<"float*">, !llvm.i64) -> !llvm<"float*">
-//  CHECK-NEXT:  llvm.store %{{.*}}, %[[addr]] : !llvm<"float*">
+//  CHECK-NEXT:  %[[addr:.*]] = llvm.getelementptr %[[ptr]][%[[off1]]] : (!llvm.ptr<float>, !llvm.i64) -> !llvm.ptr<float>
+//  CHECK-NEXT:  llvm.store %{{.*}}, %[[addr]] : !llvm.ptr<float>
   store %val, %dynamic[%i, %j] : memref<?x?xf32>
   return
 }
 
 // CHECK-LABEL: func @mixed_store
-// CHECK-SAME:         %[[ARG0:[a-zA-Z0-9]*]]: !llvm<"float*">
-// CHECK-SAME:         %[[ARG1:[a-zA-Z0-9]*]]: !llvm<"float*">
+// CHECK-SAME:         %[[ARG0:[a-zA-Z0-9]*]]: !llvm.ptr<float>
+// CHECK-SAME:         %[[ARG1:[a-zA-Z0-9]*]]: !llvm.ptr<float>
 // CHECK-SAME:         %[[ARG2:[a-zA-Z0-9]*]]: !llvm.i64
 // CHECK-SAME:         %[[ARG3:[a-zA-Z0-9]*]]: !llvm.i64
 // CHECK-SAME:         %[[ARG4:[a-zA-Z0-9]*]]: !llvm.i64
@@ -308,65 +308,65 @@ func @dynamic_store(%dynamic : memref<?x?xf32>, %i : index, %j : index, %val : f
 // CHECK-SAME:         %[[I:[a-zA-Z0-9]*]]: !llvm.i64
 // CHECK-SAME:         %[[J:[a-zA-Z0-9]*]]: !llvm.i64
 func @mixed_store(%mixed : memref<42x?xf32>, %i : index, %j : index, %val : f32) {
-//       CHECK:  %[[ptr:.*]] = llvm.extractvalue %[[ld:.*]][1] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+//       CHECK:  %[[ptr:.*]] = llvm.extractvalue %[[ld:.*]][1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
 //  CHECK-NEXT:  %[[off:.*]] = llvm.mlir.constant(0 : index) : !llvm.i64
-//  CHECK-NEXT:  %[[st0:.*]] = llvm.extractvalue %[[ld]][4, 0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+//  CHECK-NEXT:  %[[st0:.*]] = llvm.extractvalue %[[ld]][4, 0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
 //  CHECK-NEXT:  %[[offI:.*]] = llvm.mul %[[I]], %[[st0]] : !llvm.i64
 //  CHECK-NEXT:  %[[off0:.*]] = llvm.add %[[off]], %[[offI]] : !llvm.i64
 //  CHECK-NEXT:  %[[st1:.*]] = llvm.mlir.constant(1 : index) : !llvm.i64
 //  CHECK-NEXT:  %[[offJ:.*]] = llvm.mul %[[J]], %[[st1]] : !llvm.i64
 //  CHECK-NEXT:  %[[off1:.*]] = llvm.add %[[off0]], %[[offJ]] : !llvm.i64
-//  CHECK-NEXT:  %[[addr:.*]] = llvm.getelementptr %[[ptr]][%[[off1]]] : (!llvm<"float*">, !llvm.i64) -> !llvm<"float*">
-//  CHECK-NEXT:  llvm.store %{{.*}}, %[[addr]] : !llvm<"float*">
+//  CHECK-NEXT:  %[[addr:.*]] = llvm.getelementptr %[[ptr]][%[[off1]]] : (!llvm.ptr<float>, !llvm.i64) -> !llvm.ptr<float>
+//  CHECK-NEXT:  llvm.store %{{.*}}, %[[addr]] : !llvm.ptr<float>
   store %val, %mixed[%i, %j] : memref<42x?xf32>
   return
 }
 
 // CHECK-LABEL: func @memref_cast_static_to_dynamic
 func @memref_cast_static_to_dynamic(%static : memref<10x42xf32>) {
-// CHECK: llvm.bitcast %{{.*}} : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }"> to !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+// CHECK: llvm.bitcast %{{.*}} : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)> to !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
   %0 = memref_cast %static : memref<10x42xf32> to memref<?x?xf32>
   return
 }
 
 // CHECK-LABEL: func @memref_cast_static_to_mixed
 func @memref_cast_static_to_mixed(%static : memref<10x42xf32>) {
-// CHECK: llvm.bitcast %{{.*}} : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }"> to !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+// CHECK: llvm.bitcast %{{.*}} : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)> to !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
   %0 = memref_cast %static : memref<10x42xf32> to memref<?x42xf32>
   return
 }
 
 // CHECK-LABEL: func @memref_cast_dynamic_to_static
 func @memref_cast_dynamic_to_static(%dynamic : memref<?x?xf32>) {
-// CHECK: llvm.bitcast %{{.*}} : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }"> to !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+// CHECK: llvm.bitcast %{{.*}} : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)> to !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
   %0 = memref_cast %dynamic : memref<?x?xf32> to memref<10x12xf32>
   return
 }
 
 // CHECK-LABEL: func @memref_cast_dynamic_to_mixed
 func @memref_cast_dynamic_to_mixed(%dynamic : memref<?x?xf32>) {
-// CHECK: llvm.bitcast %{{.*}} : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }"> to !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+// CHECK: llvm.bitcast %{{.*}} : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)> to !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
   %0 = memref_cast %dynamic : memref<?x?xf32> to memref<?x12xf32>
   return
 }
 
 // CHECK-LABEL: func @memref_cast_mixed_to_dynamic
 func @memref_cast_mixed_to_dynamic(%mixed : memref<42x?xf32>) {
-// CHECK: llvm.bitcast %{{.*}} : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }"> to !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+// CHECK: llvm.bitcast %{{.*}} : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)> to !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
   %0 = memref_cast %mixed : memref<42x?xf32> to memref<?x?xf32>
   return
 }
 
 // CHECK-LABEL: func @memref_cast_mixed_to_static
 func @memref_cast_mixed_to_static(%mixed : memref<42x?xf32>) {
-// CHECK: llvm.bitcast %{{.*}} : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }"> to !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+// CHECK: llvm.bitcast %{{.*}} : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)> to !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
   %0 = memref_cast %mixed : memref<42x?xf32> to memref<42x1xf32>
   return
 }
 
 // CHECK-LABEL: func @memref_cast_mixed_to_mixed
 func @memref_cast_mixed_to_mixed(%mixed : memref<42x?xf32>) {
-// CHECK: llvm.bitcast %{{.*}} : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }"> to !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+// CHECK: llvm.bitcast %{{.*}} : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)> to !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
   %0 = memref_cast %mixed : memref<42x?xf32> to memref<?x1xf32>
   return
 }
@@ -374,21 +374,21 @@ func @memref_cast_mixed_to_mixed(%mixed : memref<42x?xf32>) {
 // CHECK-LABEL: func @memref_cast_ranked_to_unranked
 func @memref_cast_ranked_to_unranked(%arg : memref<42x2x?xf32>) {
 // CHECK-DAG:  %[[c:.*]] = llvm.mlir.constant(1 : index) : !llvm.i64
-// CHECK-DAG:  %[[p:.*]] = llvm.alloca %[[c]] x !llvm<"{ float*, float*, i64, [3 x i64], [3 x i64] }"> : (!llvm.i64) -> !llvm<"{ float*, float*, i64, [3 x i64], [3 x i64] }*">
-// CHECK-DAG:  llvm.store %{{.*}}, %[[p]] : !llvm<"{ float*, float*, i64, [3 x i64], [3 x i64] }*">
-// CHECK-DAG:  %[[p2:.*]] = llvm.bitcast %[[p]] : !llvm<"{ float*, float*, i64, [3 x i64], [3 x i64] }*"> to !llvm<"i8*">
+// CHECK-DAG:  %[[p:.*]] = llvm.alloca %[[c]] x !llvm.struct<(ptr<float>, ptr<float>, i64, array<3 x i64>, array<3 x i64>)> : (!llvm.i64) -> !llvm.ptr<struct<(ptr<float>, ptr<float>, i64, array<3 x i64>, array<3 x i64>)>>
+// CHECK-DAG:  llvm.store %{{.*}}, %[[p]] : !llvm.ptr<struct<(ptr<float>, ptr<float>, i64, array<3 x i64>, array<3 x i64>)>>
+// CHECK-DAG:  %[[p2:.*]] = llvm.bitcast %[[p]] : !llvm.ptr<struct<(ptr<float>, ptr<float>, i64, array<3 x i64>, array<3 x i64>)>> to !llvm.ptr<i8>
 // CHECK-DAG:  %[[r:.*]] = llvm.mlir.constant(3 : i64) : !llvm.i64
-// CHECK    :  llvm.mlir.undef : !llvm<"{ i64, i8* }">
-// CHECK-DAG:  llvm.insertvalue %[[r]], %{{.*}}[0] : !llvm<"{ i64, i8* }">
-// CHECK-DAG:  llvm.insertvalue %[[p2]], %{{.*}}[1] : !llvm<"{ i64, i8* }">
+// CHECK    :  llvm.mlir.undef : !llvm.struct<(i64, ptr<i8>)>
+// CHECK-DAG:  llvm.insertvalue %[[r]], %{{.*}}[0] : !llvm.struct<(i64, ptr<i8>)>
+// CHECK-DAG:  llvm.insertvalue %[[p2]], %{{.*}}[1] : !llvm.struct<(i64, ptr<i8>)>
   %0 = memref_cast %arg : memref<42x2x?xf32> to memref<*xf32>
   return
 }
 
 // CHECK-LABEL: func @memref_cast_unranked_to_ranked
 func @memref_cast_unranked_to_ranked(%arg : memref<*xf32>) {
-//      CHECK: %[[p:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm<"{ i64, i8* }">
-// CHECK-NEXT: llvm.bitcast %[[p]] : !llvm<"i8*"> to !llvm<"{ float*, float*, i64, [4 x i64], [4 x i64] }*">
+//      CHECK: %[[p:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(i64, ptr<i8>)>
+// CHECK-NEXT: llvm.bitcast %[[p]] : !llvm.ptr<i8> to !llvm.ptr<struct<(ptr<float>, ptr<float>, i64, array<4 x i64>, array<4 x i64>)>>
   %0 = memref_cast %arg : memref<*xf32> to memref<?x?x10x2xf32>
   return
 }
@@ -398,25 +398,25 @@ func @mixed_memref_dim(%mixed : memref<42x?x?x13x?xf32>) {
 // CHECK: llvm.mlir.constant(42 : index) : !llvm.i64
   %c0 = constant 0 : index
   %0 = dim %mixed, %c0 : memref<42x?x?x13x?xf32>
-// CHECK: llvm.extractvalue %[[ld:.*]][3, 1] : !llvm<"{ float*, float*, i64, [5 x i64], [5 x i64] }">
+// CHECK: llvm.extractvalue %[[ld:.*]][3, 1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<5 x i64>, array<5 x i64>)>
   %c1 = constant 1 : index
   %1 = dim %mixed, %c1 : memref<42x?x?x13x?xf32>
-// CHECK: llvm.extractvalue %[[ld]][3, 2] : !llvm<"{ float*, float*, i64, [5 x i64], [5 x i64] }">
+// CHECK: llvm.extractvalue %[[ld]][3, 2] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<5 x i64>, array<5 x i64>)>
   %c2 = constant 2 : index
   %2 = dim %mixed, %c2 : memref<42x?x?x13x?xf32>
 // CHECK: llvm.mlir.constant(13 : index) : !llvm.i64
   %c3 = constant 3 : index
   %3 = dim %mixed, %c3 : memref<42x?x?x13x?xf32>
-// CHECK: llvm.extractvalue %[[ld]][3, 4] : !llvm<"{ float*, float*, i64, [5 x i64], [5 x i64] }">
+// CHECK: llvm.extractvalue %[[ld]][3, 4] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<5 x i64>, array<5 x i64>)>
   %c4 = constant 4 : index
   %4 = dim %mixed, %c4 : memref<42x?x?x13x?xf32>
   return
 }
 
 // CHECK-LABEL: @memref_dim_with_dyn_index
-// CHECK-SAME: %[[ALLOC_PTR:.*]]: !llvm<"float*">, %[[ALIGN_PTR:.*]]: !llvm<"float*">, %[[OFFSET:.*]]: !llvm.i64, %[[SIZE0:.*]]: !llvm.i64, %[[SIZE1:.*]]: !llvm.i64, %[[STRIDE0:.*]]: !llvm.i64, %[[STRIDE1:.*]]: !llvm.i64, %[[IDX:.*]]: !llvm.i64) -> !llvm.i64
+// CHECK-SAME: %[[ALLOC_PTR:.*]]: !llvm.ptr<float>, %[[ALIGN_PTR:.*]]: !llvm.ptr<float>, %[[OFFSET:.*]]: !llvm.i64, %[[SIZE0:.*]]: !llvm.i64, %[[SIZE1:.*]]: !llvm.i64, %[[STRIDE0:.*]]: !llvm.i64, %[[STRIDE1:.*]]: !llvm.i64, %[[IDX:.*]]: !llvm.i64) -> !llvm.i64
 func @memref_dim_with_dyn_index(%arg : memref<3x?xf32>, %idx : index) -> index {
-  // CHECK-NEXT: %[[DESCR0:.*]] = llvm.mlir.undef : [[DESCR_TY:!llvm<"{ float\*, float\*, i64, \[2 x i64\], \[2 x i64\] }">]]
+  // CHECK-NEXT: %[[DESCR0:.*]] = llvm.mlir.undef : [[DESCR_TY:!llvm.struct<\(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>\)>]]
   // CHECK-NEXT: %[[DESCR1:.*]] = llvm.insertvalue %[[ALLOC_PTR]], %[[DESCR0]][0] : [[DESCR_TY]]
   // CHECK-NEXT: %[[DESCR2:.*]] = llvm.insertvalue %[[ALIGN_PTR]], %[[DESCR1]][1] : [[DESCR_TY]]
   // CHECK-NEXT: %[[DESCR3:.*]] = llvm.insertvalue %[[OFFSET]],    %[[DESCR2]][2] : [[DESCR_TY]]
@@ -427,10 +427,10 @@ func @memref_dim_with_dyn_index(%arg : memref<3x?xf32>, %idx : index) -> index {
   // CHECK-DAG: %[[C0:.*]] = llvm.mlir.constant(0 : index) : !llvm.i64
   // CHECK-DAG: %[[C1:.*]] = llvm.mlir.constant(1 : index) : !llvm.i64
   // CHECK-DAG: %[[SIZES:.*]] = llvm.extractvalue %[[DESCR7]][3] : [[DESCR_TY]]
-  // CHECK-DAG: %[[SIZES_PTR:.*]] = llvm.alloca %[[C1]] x !llvm<"[2 x i64]"> : (!llvm.i64) -> !llvm<"[2 x i64]*">
-  // CHECK-DAG: llvm.store %[[SIZES]], %[[SIZES_PTR]] : !llvm<"[2 x i64]*">
-  // CHECK-DAG: %[[RESULT_PTR:.*]] = llvm.getelementptr %[[SIZES_PTR]][%[[C0]], %[[IDX]]] : (!llvm<"[2 x i64]*">, !llvm.i64, !llvm.i64) -> !llvm<"i64*">
-  // CHECK-DAG: %[[RESULT:.*]] = llvm.load %[[RESULT_PTR]] : !llvm<"i64*">
+  // CHECK-DAG: %[[SIZES_PTR:.*]] = llvm.alloca %[[C1]] x !llvm.array<2 x i64> : (!llvm.i64) -> !llvm.ptr<array<2 x i64>>
+  // CHECK-DAG: llvm.store %[[SIZES]], %[[SIZES_PTR]] : !llvm.ptr<array<2 x i64>>
+  // CHECK-DAG: %[[RESULT_PTR:.*]] = llvm.getelementptr %[[SIZES_PTR]][%[[C0]], %[[IDX]]] : (!llvm.ptr<array<2 x i64>>, !llvm.i64, !llvm.i64) -> !llvm.ptr<i64>
+  // CHECK-DAG: %[[RESULT:.*]] = llvm.load %[[RESULT_PTR]] : !llvm.ptr<i64>
   // CHECK-DAG: llvm.return %[[RESULT]] : !llvm.i64
   %result = dim %arg, %idx : memref<3x?xf32>
   return %result : index
diff --git a/mlir/test/Conversion/StandardToLLVM/convert-funcs.mlir b/mlir/test/Conversion/StandardToLLVM/convert-funcs.mlir
index 562918b71ac83..d2a88ed952f7c 100644
--- a/mlir/test/Conversion/StandardToLLVM/convert-funcs.mlir
+++ b/mlir/test/Conversion/StandardToLLVM/convert-funcs.mlir
@@ -1,39 +1,39 @@
 // RUN: mlir-opt -convert-std-to-llvm %s | FileCheck %s
 
-//CHECK: llvm.func @second_order_arg(!llvm<"void ()*">)
+//CHECK: llvm.func @second_order_arg(!llvm.ptr<func<void ()>>)
 func @second_order_arg(%arg0 : () -> ())
 
-//CHECK: llvm.func @second_order_result() -> !llvm<"void ()*">
+//CHECK: llvm.func @second_order_result() -> !llvm.ptr<func<void ()>>
 func @second_order_result() -> (() -> ())
 
-//CHECK: llvm.func @second_order_multi_result() -> !llvm<"{ i32 ()*, i64 ()*, float ()* }">
+//CHECK: llvm.func @second_order_multi_result() -> !llvm.struct<(ptr<func<i32 ()>>, ptr<func<i64 ()>>, ptr<func<float ()>>)>
 func @second_order_multi_result() -> (() -> (i32), () -> (i64), () -> (f32))
 
-//CHECK: llvm.func @third_order(!llvm<"void ()* (void ()*)*">) -> !llvm<"void ()* (void ()*)*">
+//CHECK: llvm.func @third_order(!llvm.ptr<func<ptr<func<void ()>> (ptr<func<void ()>>)>>) -> !llvm.ptr<func<ptr<func<void ()>> (ptr<func<void ()>>)>>
 func @third_order(%arg0 : (() -> ()) -> (() -> ())) -> ((() -> ()) -> (() -> ()))
 
-//CHECK: llvm.func @fifth_order_left(!llvm<"void (void (void (void ()*)*)*)*">)
+//CHECK: llvm.func @fifth_order_left(!llvm.ptr<func<void (ptr<func<void (ptr<func<void (ptr<func<void ()>>)>>)>>)>>)
 func @fifth_order_left(%arg0: (((() -> ()) -> ()) -> ()) -> ())
 
-//CHECK: llvm.func @fifth_order_right(!llvm<"void ()* ()* ()* ()*">)
+//CHECK: llvm.func @fifth_order_right(!llvm.ptr<func<ptr<func<ptr<func<ptr<func<void ()>> ()>> ()>> ()>>)
 func @fifth_order_right(%arg0: () -> (() -> (() -> (() -> ()))))
 
 // Check that memrefs are converted to argument packs if appear as function arguments.
-// CHECK: llvm.func @memref_call_conv(!llvm<"float*">, !llvm<"float*">, !llvm.i64, !llvm.i64, !llvm.i64)
+// CHECK: llvm.func @memref_call_conv(!llvm.ptr<float>, !llvm.ptr<float>, !llvm.i64, !llvm.i64, !llvm.i64)
 func @memref_call_conv(%arg0: memref<?xf32>)
 
 // Same in nested functions.
-// CHECK: llvm.func @memref_call_conv_nested(!llvm<"void (float*, float*, i64, i64, i64)*">)
+// CHECK: llvm.func @memref_call_conv_nested(!llvm.ptr<func<void (ptr<float>, ptr<float>, i64, i64, i64)>>)
 func @memref_call_conv_nested(%arg0: (memref<?xf32>) -> ())
 
-//CHECK-LABEL: llvm.func @pass_through(%arg0: !llvm<"void ()*">) -> !llvm<"void ()*"> {
+//CHECK-LABEL: llvm.func @pass_through(%arg0: !llvm.ptr<func<void ()>>) -> !llvm.ptr<func<void ()>> {
 func @pass_through(%arg0: () -> ()) -> (() -> ()) {
-// CHECK-NEXT:  llvm.br ^bb1(%arg0 : !llvm<"void ()*">)
+// CHECK-NEXT:  llvm.br ^bb1(%arg0 : !llvm.ptr<func<void ()>>)
   br ^bb1(%arg0 : () -> ())
 
-//CHECK-NEXT: ^bb1(%0: !llvm<"void ()*">):
+//CHECK-NEXT: ^bb1(%0: !llvm.ptr<func<void ()>>):
 ^bb1(%bbarg: () -> ()):
-// CHECK-NEXT:  llvm.return %0 : !llvm<"void ()*">
+// CHECK-NEXT:  llvm.return %0 : !llvm.ptr<func<void ()>>
   return %bbarg : () -> ()
 }
 
@@ -43,7 +43,7 @@ func @body(i32)
 // CHECK-LABEL: llvm.func @indirect_const_call
 // CHECK-SAME: (%[[ARG0:.*]]: !llvm.i32) {
 func @indirect_const_call(%arg0: i32) {
-// CHECK-NEXT: %[[ADDR:.*]] = llvm.mlir.addressof @body : !llvm<"void (i32)*">
+// CHECK-NEXT: %[[ADDR:.*]] = llvm.mlir.addressof @body : !llvm.ptr<func<void (i32)>>
   %0 = constant @body : (i32) -> ()
 // CHECK-NEXT:  llvm.call %[[ADDR]](%[[ARG0:.*]]) : (!llvm.i32) -> ()
   call_indirect %0(%arg0) : (i32) -> ()
@@ -51,7 +51,7 @@ func @indirect_const_call(%arg0: i32) {
   return
 }
 
-// CHECK-LABEL: llvm.func @indirect_call(%arg0: !llvm<"i32 (float)*">, %arg1: !llvm.float) -> !llvm.i32 {
+// CHECK-LABEL: llvm.func @indirect_call(%arg0: !llvm.ptr<func<i32 (float)>>, %arg1: !llvm.float) -> !llvm.i32 {
 func @indirect_call(%arg0: (f32) -> i32, %arg1: f32) -> i32 {
 // CHECK-NEXT:  %0 = llvm.call %arg0(%arg1) : (!llvm.float) -> !llvm.i32
   %0 = call_indirect %arg0(%arg1) : (f32) -> i32
diff --git a/mlir/test/Conversion/StandardToLLVM/convert-static-memref-ops.mlir b/mlir/test/Conversion/StandardToLLVM/convert-static-memref-ops.mlir
index dbf8fe6884289..b428d37a36167 100644
--- a/mlir/test/Conversion/StandardToLLVM/convert-static-memref-ops.mlir
+++ b/mlir/test/Conversion/StandardToLLVM/convert-static-memref-ops.mlir
@@ -2,7 +2,7 @@
 // RUN: mlir-opt -convert-std-to-llvm='use-bare-ptr-memref-call-conv=1' -split-input-file %s | FileCheck %s --check-prefix=BAREPTR
 
 // BAREPTR-LABEL: func @check_noalias
-// BAREPTR-SAME: %{{.*}}: !llvm<"float*"> {llvm.noalias = true}, %{{.*}}: !llvm<"float*"> {llvm.noalias = true}
+// BAREPTR-SAME: %{{.*}}: !llvm.ptr<float> {llvm.noalias = true}, %{{.*}}: !llvm.ptr<float> {llvm.noalias = true}
 func @check_noalias(%static : memref<2xf32> {llvm.noalias = true}, %other : memref<2xf32> {llvm.noalias = true}) {
     return
 }
@@ -10,93 +10,93 @@ func @check_noalias(%static : memref<2xf32> {llvm.noalias = true}, %other : memr
 // -----
 
 // CHECK-LABEL: func @check_static_return
-// CHECK-COUNT-2: !llvm<"float*">
+// CHECK-COUNT-2: !llvm.ptr<float>
 // CHECK-COUNT-5: !llvm.i64
-// CHECK-SAME: -> !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+// CHECK-SAME: -> !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
 // BAREPTR-LABEL: func @check_static_return
-// BAREPTR-SAME: (%[[arg:.*]]: !llvm<"float*">) -> !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }"> {
+// BAREPTR-SAME: (%[[arg:.*]]: !llvm.ptr<float>) -> !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)> {
 func @check_static_return(%static : memref<32x18xf32>) -> memref<32x18xf32> {
-// CHECK:  llvm.return %{{.*}} : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+// CHECK:  llvm.return %{{.*}} : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
 
-// BAREPTR: %[[udf:.*]] = llvm.mlir.undef : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
-// BAREPTR-NEXT: %[[base:.*]] = llvm.insertvalue %[[arg]], %[[udf]][0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
-// BAREPTR-NEXT: %[[aligned:.*]] = llvm.insertvalue %[[arg]], %[[base]][1] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+// BAREPTR: %[[udf:.*]] = llvm.mlir.undef : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
+// BAREPTR-NEXT: %[[base:.*]] = llvm.insertvalue %[[arg]], %[[udf]][0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
+// BAREPTR-NEXT: %[[aligned:.*]] = llvm.insertvalue %[[arg]], %[[base]][1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
 // BAREPTR-NEXT: %[[val0:.*]] = llvm.mlir.constant(0 : index) : !llvm.i64
-// BAREPTR-NEXT: %[[ins0:.*]] = llvm.insertvalue %[[val0]], %[[aligned]][2] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+// BAREPTR-NEXT: %[[ins0:.*]] = llvm.insertvalue %[[val0]], %[[aligned]][2] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
 // BAREPTR-NEXT: %[[val1:.*]] = llvm.mlir.constant(32 : index) : !llvm.i64
-// BAREPTR-NEXT: %[[ins1:.*]] = llvm.insertvalue %[[val1]], %[[ins0]][3, 0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+// BAREPTR-NEXT: %[[ins1:.*]] = llvm.insertvalue %[[val1]], %[[ins0]][3, 0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
 // BAREPTR-NEXT: %[[val2:.*]] = llvm.mlir.constant(18 : index) : !llvm.i64
-// BAREPTR-NEXT: %[[ins2:.*]] = llvm.insertvalue %[[val2]], %[[ins1]][4, 0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+// BAREPTR-NEXT: %[[ins2:.*]] = llvm.insertvalue %[[val2]], %[[ins1]][4, 0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
 // BAREPTR-NEXT: %[[val3:.*]] = llvm.mlir.constant(18 : index) : !llvm.i64
-// BAREPTR-NEXT: %[[ins3:.*]] = llvm.insertvalue %[[val3]], %[[ins2]][3, 1] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+// BAREPTR-NEXT: %[[ins3:.*]] = llvm.insertvalue %[[val3]], %[[ins2]][3, 1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
 // BAREPTR-NEXT: %[[val4:.*]] = llvm.mlir.constant(1 : index) : !llvm.i64
-// BAREPTR-NEXT: %[[ins4:.*]] = llvm.insertvalue %[[val4]], %[[ins3]][4, 1] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
-// BAREPTR-NEXT: llvm.return %[[ins4]] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+// BAREPTR-NEXT: %[[ins4:.*]] = llvm.insertvalue %[[val4]], %[[ins3]][4, 1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
+// BAREPTR-NEXT: llvm.return %[[ins4]] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
   return %static : memref<32x18xf32>
 }
 
 // -----
 
 // CHECK-LABEL: func @check_static_return_with_offset
-// CHECK-COUNT-2: !llvm<"float*">
+// CHECK-COUNT-2: !llvm.ptr<float>
 // CHECK-COUNT-5: !llvm.i64
-// CHECK-SAME: -> !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+// CHECK-SAME: -> !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
 // BAREPTR-LABEL: func @check_static_return_with_offset
-// BAREPTR-SAME: (%[[arg:.*]]: !llvm<"float*">) -> !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }"> {
+// BAREPTR-SAME: (%[[arg:.*]]: !llvm.ptr<float>) -> !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)> {
 func @check_static_return_with_offset(%static : memref<32x18xf32, offset:7, strides:[22,1]>) -> memref<32x18xf32, offset:7, strides:[22,1]> {
-// CHECK:  llvm.return %{{.*}} : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+// CHECK:  llvm.return %{{.*}} : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
 
-// BAREPTR: %[[udf:.*]] = llvm.mlir.undef : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
-// BAREPTR-NEXT: %[[base:.*]] = llvm.insertvalue %[[arg]], %[[udf]][0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
-// BAREPTR-NEXT: %[[aligned:.*]] = llvm.insertvalue %[[arg]], %[[base]][1] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+// BAREPTR: %[[udf:.*]] = llvm.mlir.undef : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
+// BAREPTR-NEXT: %[[base:.*]] = llvm.insertvalue %[[arg]], %[[udf]][0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
+// BAREPTR-NEXT: %[[aligned:.*]] = llvm.insertvalue %[[arg]], %[[base]][1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
 // BAREPTR-NEXT: %[[val0:.*]] = llvm.mlir.constant(7 : index) : !llvm.i64
-// BAREPTR-NEXT: %[[ins0:.*]] = llvm.insertvalue %[[val0]], %[[aligned]][2] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+// BAREPTR-NEXT: %[[ins0:.*]] = llvm.insertvalue %[[val0]], %[[aligned]][2] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
 // BAREPTR-NEXT: %[[val1:.*]] = llvm.mlir.constant(32 : index) : !llvm.i64
-// BAREPTR-NEXT: %[[ins1:.*]] = llvm.insertvalue %[[val1]], %[[ins0]][3, 0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+// BAREPTR-NEXT: %[[ins1:.*]] = llvm.insertvalue %[[val1]], %[[ins0]][3, 0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
 // BAREPTR-NEXT: %[[val2:.*]] = llvm.mlir.constant(22 : index) : !llvm.i64
-// BAREPTR-NEXT: %[[ins2:.*]] = llvm.insertvalue %[[val2]], %[[ins1]][4, 0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+// BAREPTR-NEXT: %[[ins2:.*]] = llvm.insertvalue %[[val2]], %[[ins1]][4, 0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
 // BAREPTR-NEXT: %[[val3:.*]] = llvm.mlir.constant(18 : index) : !llvm.i64
-// BAREPTR-NEXT: %[[ins3:.*]] = llvm.insertvalue %[[val3]], %[[ins2]][3, 1] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+// BAREPTR-NEXT: %[[ins3:.*]] = llvm.insertvalue %[[val3]], %[[ins2]][3, 1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
 // BAREPTR-NEXT: %[[val4:.*]] = llvm.mlir.constant(1 : index) : !llvm.i64
-// BAREPTR-NEXT: %[[ins4:.*]] = llvm.insertvalue %[[val4]], %[[ins3]][4, 1] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
-// BAREPTR-NEXT: llvm.return %[[ins4]] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+// BAREPTR-NEXT: %[[ins4:.*]] = llvm.insertvalue %[[val4]], %[[ins3]][4, 1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
+// BAREPTR-NEXT: llvm.return %[[ins4]] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
   return %static : memref<32x18xf32, offset:7, strides:[22,1]>
 }
 
 // -----
 
-// CHECK-LABEL: func @zero_d_alloc() -> !llvm<"{ float*, float*, i64 }"> {
-// BAREPTR-LABEL: func @zero_d_alloc() -> !llvm<"{ float*, float*, i64 }"> {
+// CHECK-LABEL: func @zero_d_alloc() -> !llvm.struct<(ptr<float>, ptr<float>, i64)> {
+// BAREPTR-LABEL: func @zero_d_alloc() -> !llvm.struct<(ptr<float>, ptr<float>, i64)> {
 func @zero_d_alloc() -> memref<f32> {
 // CHECK-NEXT:  llvm.mlir.constant(1 : index) : !llvm.i64
-// CHECK-NEXT:  %[[null:.*]] = llvm.mlir.null : !llvm<"float*">
+// CHECK-NEXT:  %[[null:.*]] = llvm.mlir.null : !llvm.ptr<float>
 // CHECK-NEXT:  %[[one:.*]] = llvm.mlir.constant(1 : index) : !llvm.i64
-// CHECK-NEXT:  %[[gep:.*]] = llvm.getelementptr %[[null]][%[[one]]] : (!llvm<"float*">, !llvm.i64) -> !llvm<"float*">
-// CHECK-NEXT:  %[[sizeof:.*]] = llvm.ptrtoint %[[gep]] : !llvm<"float*"> to !llvm.i64
+// CHECK-NEXT:  %[[gep:.*]] = llvm.getelementptr %[[null]][%[[one]]] : (!llvm.ptr<float>, !llvm.i64) -> !llvm.ptr<float>
+// CHECK-NEXT:  %[[sizeof:.*]] = llvm.ptrtoint %[[gep]] : !llvm.ptr<float> to !llvm.i64
 // CHECK-NEXT:  llvm.mul %{{.*}}, %[[sizeof]] : !llvm.i64
 // CHECK-NEXT:  %[[one_1:.*]] = llvm.mlir.constant(1 : index) : !llvm.i64
-// CHECK-NEXT:  llvm.call @malloc(%{{.*}}) : (!llvm.i64) -> !llvm<"i8*">
-// CHECK-NEXT:  %[[ptr:.*]] = llvm.bitcast %{{.*}} : !llvm<"i8*"> to !llvm<"float*">
-// CHECK-NEXT:  llvm.mlir.undef : !llvm<"{ float*, float*, i64 }">
-// CHECK-NEXT:  llvm.insertvalue %[[ptr]], %{{.*}}[0] : !llvm<"{ float*, float*, i64 }">
-// CHECK-NEXT:  llvm.insertvalue %[[ptr]], %{{.*}}[1] : !llvm<"{ float*, float*, i64 }">
+// CHECK-NEXT:  llvm.call @malloc(%{{.*}}) : (!llvm.i64) -> !llvm.ptr<i8>
+// CHECK-NEXT:  %[[ptr:.*]] = llvm.bitcast %{{.*}} : !llvm.ptr<i8> to !llvm.ptr<float>
+// CHECK-NEXT:  llvm.mlir.undef : !llvm.struct<(ptr<float>, ptr<float>, i64)>
+// CHECK-NEXT:  llvm.insertvalue %[[ptr]], %{{.*}}[0] : !llvm.struct<(ptr<float>, ptr<float>, i64)>
+// CHECK-NEXT:  llvm.insertvalue %[[ptr]], %{{.*}}[1] : !llvm.struct<(ptr<float>, ptr<float>, i64)>
 // CHECK-NEXT:  %[[c0:.*]] = llvm.mlir.constant(0 : index) : !llvm.i64
-// CHECK-NEXT:  llvm.insertvalue %[[c0]], %{{.*}}[2] : !llvm<"{ float*, float*, i64 }">
+// CHECK-NEXT:  llvm.insertvalue %[[c0]], %{{.*}}[2] : !llvm.struct<(ptr<float>, ptr<float>, i64)>
 
 // BAREPTR-NEXT:  llvm.mlir.constant(1 : index) : !llvm.i64
-// BAREPTR-NEXT:  %[[null:.*]] = llvm.mlir.null : !llvm<"float*">
+// BAREPTR-NEXT:  %[[null:.*]] = llvm.mlir.null : !llvm.ptr<float>
 // BAREPTR-NEXT:  %[[one:.*]] = llvm.mlir.constant(1 : index) : !llvm.i64
-// BAREPTR-NEXT:  %[[gep:.*]] = llvm.getelementptr %[[null]][%[[one]]] : (!llvm<"float*">, !llvm.i64) -> !llvm<"float*">
-// BAREPTR-NEXT:  %[[sizeof:.*]] = llvm.ptrtoint %[[gep]] : !llvm<"float*"> to !llvm.i64
+// BAREPTR-NEXT:  %[[gep:.*]] = llvm.getelementptr %[[null]][%[[one]]] : (!llvm.ptr<float>, !llvm.i64) -> !llvm.ptr<float>
+// BAREPTR-NEXT:  %[[sizeof:.*]] = llvm.ptrtoint %[[gep]] : !llvm.ptr<float> to !llvm.i64
 // BAREPTR-NEXT:  llvm.mul %{{.*}}, %[[sizeof]] : !llvm.i64
 // BAREPTR-NEXT:  %[[one_1:.*]] = llvm.mlir.constant(1 : index) : !llvm.i64
-// BAREPTR-NEXT:  llvm.call @malloc(%{{.*}}) : (!llvm.i64) -> !llvm<"i8*">
-// BAREPTR-NEXT:  %[[ptr:.*]] = llvm.bitcast %{{.*}} : !llvm<"i8*"> to !llvm<"float*">
-// BAREPTR-NEXT:  llvm.mlir.undef : !llvm<"{ float*, float*, i64 }">
-// BAREPTR-NEXT:  llvm.insertvalue %[[ptr]], %{{.*}}[0] : !llvm<"{ float*, float*, i64 }">
-// BAREPTR-NEXT:  llvm.insertvalue %[[ptr]], %{{.*}}[1] : !llvm<"{ float*, float*, i64 }">
+// BAREPTR-NEXT:  llvm.call @malloc(%{{.*}}) : (!llvm.i64) -> !llvm.ptr<i8>
+// BAREPTR-NEXT:  %[[ptr:.*]] = llvm.bitcast %{{.*}} : !llvm.ptr<i8> to !llvm.ptr<float>
+// BAREPTR-NEXT:  llvm.mlir.undef : !llvm.struct<(ptr<float>, ptr<float>, i64)>
+// BAREPTR-NEXT:  llvm.insertvalue %[[ptr]], %{{.*}}[0] : !llvm.struct<(ptr<float>, ptr<float>, i64)>
+// BAREPTR-NEXT:  llvm.insertvalue %[[ptr]], %{{.*}}[1] : !llvm.struct<(ptr<float>, ptr<float>, i64)>
 // BAREPTR-NEXT:  %[[c0:.*]] = llvm.mlir.constant(0 : index) : !llvm.i64
-// BAREPTR-NEXT:  llvm.insertvalue %[[c0]], %{{.*}}[2] : !llvm<"{ float*, float*, i64 }">
+// BAREPTR-NEXT:  llvm.insertvalue %[[c0]], %{{.*}}[2] : !llvm.struct<(ptr<float>, ptr<float>, i64)>
   %0 = alloc() : memref<f32>
   return %0 : memref<f32>
 }
@@ -104,15 +104,15 @@ func @zero_d_alloc() -> memref<f32> {
 // -----
 
 // CHECK-LABEL: func @zero_d_dealloc
-// BAREPTR-LABEL: func @zero_d_dealloc(%{{.*}}: !llvm<"float*">) {
+// BAREPTR-LABEL: func @zero_d_dealloc(%{{.*}}: !llvm.ptr<float>) {
 func @zero_d_dealloc(%arg0: memref<f32>) {
-//      CHECK:  %[[ptr:.*]] = llvm.extractvalue %{{.*}}[0] : !llvm<"{ float*, float*, i64 }">
-// CHECK-NEXT:  %[[bc:.*]] = llvm.bitcast %[[ptr]] : !llvm<"float*"> to !llvm<"i8*">
-// CHECK-NEXT:  llvm.call @free(%[[bc]]) : (!llvm<"i8*">) -> ()
+//      CHECK:  %[[ptr:.*]] = llvm.extractvalue %{{.*}}[0] : !llvm.struct<(ptr<float>, ptr<float>, i64)>
+// CHECK-NEXT:  %[[bc:.*]] = llvm.bitcast %[[ptr]] : !llvm.ptr<float> to !llvm.ptr<i8>
+// CHECK-NEXT:  llvm.call @free(%[[bc]]) : (!llvm.ptr<i8>) -> ()
 
-// BAREPTR: %[[ptr:.*]] = llvm.extractvalue %{{.*}}[0] : !llvm<"{ float*, float*, i64 }">
-// BAREPTR-NEXT: %[[bc:.*]] = llvm.bitcast %[[ptr]] : !llvm<"float*"> to !llvm<"i8*">
-// BAREPTR-NEXT: llvm.call @free(%[[bc]]) : (!llvm<"i8*">) -> ()
+// BAREPTR: %[[ptr:.*]] = llvm.extractvalue %{{.*}}[0] : !llvm.struct<(ptr<float>, ptr<float>, i64)>
+// BAREPTR-NEXT: %[[bc:.*]] = llvm.bitcast %[[ptr]] : !llvm.ptr<float> to !llvm.ptr<i8>
+// BAREPTR-NEXT: llvm.call @free(%[[bc]]) : (!llvm.ptr<i8>) -> ()
   dealloc %arg0 : memref<f32>
   return
 }
@@ -123,111 +123,111 @@ func @zero_d_dealloc(%arg0: memref<f32>) {
 // BAREPTR-LABEL: func @aligned_1d_alloc(
 func @aligned_1d_alloc() -> memref<42xf32> {
 //      CHECK:  llvm.mlir.constant(42 : index) : !llvm.i64
-// CHECK-NEXT:  %[[null:.*]] = llvm.mlir.null : !llvm<"float*">
+// CHECK-NEXT:  %[[null:.*]] = llvm.mlir.null : !llvm.ptr<float>
 // CHECK-NEXT:  %[[one:.*]] = llvm.mlir.constant(1 : index) : !llvm.i64
-// CHECK-NEXT:  %[[gep:.*]] = llvm.getelementptr %[[null]][%[[one]]] : (!llvm<"float*">, !llvm.i64) -> !llvm<"float*">
-// CHECK-NEXT:  %[[sizeof:.*]] = llvm.ptrtoint %[[gep]] : !llvm<"float*"> to !llvm.i64
+// CHECK-NEXT:  %[[gep:.*]] = llvm.getelementptr %[[null]][%[[one]]] : (!llvm.ptr<float>, !llvm.i64) -> !llvm.ptr<float>
+// CHECK-NEXT:  %[[sizeof:.*]] = llvm.ptrtoint %[[gep]] : !llvm.ptr<float> to !llvm.i64
 // CHECK-NEXT:  llvm.mul %{{.*}}, %[[sizeof]] : !llvm.i64
 // CHECK-NEXT:  %[[one_1:.*]] = llvm.mlir.constant(1 : index) : !llvm.i64
 // CHECK-NEXT:  %[[alignment:.*]] = llvm.mlir.constant(8 : index) : !llvm.i64
 // CHECK-NEXT:  %[[alignmentMinus1:.*]] = llvm.add {{.*}}, %[[alignment]] : !llvm.i64
 // CHECK-NEXT:  %[[allocsize:.*]] = llvm.sub %[[alignmentMinus1]], %[[one_1]] : !llvm.i64
-// CHECK-NEXT:  %[[allocated:.*]] = llvm.call @malloc(%[[allocsize]]) : (!llvm.i64) -> !llvm<"i8*">
-// CHECK-NEXT:  %[[ptr:.*]] = llvm.bitcast %{{.*}} : !llvm<"i8*"> to !llvm<"float*">
-// CHECK-NEXT:  llvm.mlir.undef : !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }">
-// CHECK-NEXT:  llvm.insertvalue %[[ptr]], %{{.*}}[0] : !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }">
-// CHECK-NEXT:  %[[allocatedAsInt:.*]] = llvm.ptrtoint %[[allocated]] : !llvm<"i8*"> to !llvm.i64
+// CHECK-NEXT:  %[[allocated:.*]] = llvm.call @malloc(%[[allocsize]]) : (!llvm.i64) -> !llvm.ptr<i8>
+// CHECK-NEXT:  %[[ptr:.*]] = llvm.bitcast %{{.*}} : !llvm.ptr<i8> to !llvm.ptr<float>
+// CHECK-NEXT:  llvm.mlir.undef : !llvm.struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>
+// CHECK-NEXT:  llvm.insertvalue %[[ptr]], %{{.*}}[0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>
+// CHECK-NEXT:  %[[allocatedAsInt:.*]] = llvm.ptrtoint %[[allocated]] : !llvm.ptr<i8> to !llvm.i64
 // CHECK-NEXT:  %[[alignAdj1:.*]] = llvm.urem %[[allocatedAsInt]], %[[alignment]] : !llvm.i64
 // CHECK-NEXT:  %[[alignAdj2:.*]] = llvm.sub %[[alignment]], %[[alignAdj1]] : !llvm.i64
 // CHECK-NEXT:  %[[alignAdj3:.*]] = llvm.urem %[[alignAdj2]], %[[alignment]] : !llvm.i64
-// CHECK-NEXT:  %[[aligned:.*]] = llvm.getelementptr %[[allocated]][%[[alignAdj3]]] : (!llvm<"i8*">, !llvm.i64) -> !llvm<"i8*">
-// CHECK-NEXT:  %[[alignedBitCast:.*]] = llvm.bitcast %[[aligned]] : !llvm<"i8*"> to !llvm<"float*">
-// CHECK-NEXT:  llvm.insertvalue %[[alignedBitCast]], %{{.*}}[1] : !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }">
+// CHECK-NEXT:  %[[aligned:.*]] = llvm.getelementptr %[[allocated]][%[[alignAdj3]]] : (!llvm.ptr<i8>, !llvm.i64) -> !llvm.ptr<i8>
+// CHECK-NEXT:  %[[alignedBitCast:.*]] = llvm.bitcast %[[aligned]] : !llvm.ptr<i8> to !llvm.ptr<float>
+// CHECK-NEXT:  llvm.insertvalue %[[alignedBitCast]], %{{.*}}[1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>
 // CHECK-NEXT:  %[[c0:.*]] = llvm.mlir.constant(0 : index) : !llvm.i64
-// CHECK-NEXT:  llvm.insertvalue %[[c0]], %{{.*}}[2] : !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }">
+// CHECK-NEXT:  llvm.insertvalue %[[c0]], %{{.*}}[2] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>
 
 // BAREPTR-NEXT:  llvm.mlir.constant(42 : index) : !llvm.i64
-// BAREPTR-NEXT:  %[[null:.*]] = llvm.mlir.null : !llvm<"float*">
+// BAREPTR-NEXT:  %[[null:.*]] = llvm.mlir.null : !llvm.ptr<float>
 // BAREPTR-NEXT:  %[[one:.*]] = llvm.mlir.constant(1 : index) : !llvm.i64
-// BAREPTR-NEXT:  %[[gep:.*]] = llvm.getelementptr %[[null]][%[[one]]] : (!llvm<"float*">, !llvm.i64) -> !llvm<"float*">
-// BAREPTR-NEXT:  %[[sizeof:.*]] = llvm.ptrtoint %[[gep]] : !llvm<"float*"> to !llvm.i64
+// BAREPTR-NEXT:  %[[gep:.*]] = llvm.getelementptr %[[null]][%[[one]]] : (!llvm.ptr<float>, !llvm.i64) -> !llvm.ptr<float>
+// BAREPTR-NEXT:  %[[sizeof:.*]] = llvm.ptrtoint %[[gep]] : !llvm.ptr<float> to !llvm.i64
 // BAREPTR-NEXT:  llvm.mul %{{.*}}, %[[sizeof]] : !llvm.i64
 // BAREPTR-NEXT:  %[[one_1:.*]] = llvm.mlir.constant(1 : index) : !llvm.i64
 // BAREPTR-NEXT:  %[[alignment:.*]] = llvm.mlir.constant(8 : index) : !llvm.i64
 // BAREPTR-NEXT:  %[[alignmentMinus1:.*]] = llvm.add {{.*}}, %[[alignment]] : !llvm.i64
 // BAREPTR-NEXT:  %[[allocsize:.*]] = llvm.sub %[[alignmentMinus1]], %[[one_1]] : !llvm.i64
-// BAREPTR-NEXT:  %[[allocated:.*]] = llvm.call @malloc(%[[allocsize]]) : (!llvm.i64) -> !llvm<"i8*">
-// BAREPTR-NEXT:  %[[ptr:.*]] = llvm.bitcast %{{.*}} : !llvm<"i8*"> to !llvm<"float*">
-// BAREPTR-NEXT:  llvm.mlir.undef : !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }">
-// BAREPTR-NEXT:  llvm.insertvalue %[[ptr]], %{{.*}}[0] : !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }">
-// BAREPTR-NEXT:  %[[allocatedAsInt:.*]] = llvm.ptrtoint %[[allocated]] : !llvm<"i8*"> to !llvm.i64
+// BAREPTR-NEXT:  %[[allocated:.*]] = llvm.call @malloc(%[[allocsize]]) : (!llvm.i64) -> !llvm.ptr<i8>
+// BAREPTR-NEXT:  %[[ptr:.*]] = llvm.bitcast %{{.*}} : !llvm.ptr<i8> to !llvm.ptr<float>
+// BAREPTR-NEXT:  llvm.mlir.undef : !llvm.struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>
+// BAREPTR-NEXT:  llvm.insertvalue %[[ptr]], %{{.*}}[0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>
+// BAREPTR-NEXT:  %[[allocatedAsInt:.*]] = llvm.ptrtoint %[[allocated]] : !llvm.ptr<i8> to !llvm.i64
 // BAREPTR-NEXT:  %[[alignAdj1:.*]] = llvm.urem %[[allocatedAsInt]], %[[alignment]] : !llvm.i64
 // BAREPTR-NEXT:  %[[alignAdj2:.*]] = llvm.sub %[[alignment]], %[[alignAdj1]] : !llvm.i64
 // BAREPTR-NEXT:  %[[alignAdj3:.*]] = llvm.urem %[[alignAdj2]], %[[alignment]] : !llvm.i64
-// BAREPTR-NEXT:  %[[aligned:.*]] = llvm.getelementptr %[[allocated]][%[[alignAdj3]]] : (!llvm<"i8*">, !llvm.i64) -> !llvm<"i8*">
-// BAREPTR-NEXT:  %[[alignedBitCast:.*]] = llvm.bitcast %[[aligned]] : !llvm<"i8*"> to !llvm<"float*">
-// BAREPTR-NEXT:  llvm.insertvalue %[[alignedBitCast]], %{{.*}}[1] : !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }">
+// BAREPTR-NEXT:  %[[aligned:.*]] = llvm.getelementptr %[[allocated]][%[[alignAdj3]]] : (!llvm.ptr<i8>, !llvm.i64) -> !llvm.ptr<i8>
+// BAREPTR-NEXT:  %[[alignedBitCast:.*]] = llvm.bitcast %[[aligned]] : !llvm.ptr<i8> to !llvm.ptr<float>
+// BAREPTR-NEXT:  llvm.insertvalue %[[alignedBitCast]], %{{.*}}[1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>
 // BAREPTR-NEXT:  %[[c0:.*]] = llvm.mlir.constant(0 : index) : !llvm.i64
-// BAREPTR-NEXT:  llvm.insertvalue %[[c0]], %{{.*}}[2] : !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }">
+// BAREPTR-NEXT:  llvm.insertvalue %[[c0]], %{{.*}}[2] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>
   %0 = alloc() {alignment = 8} : memref<42xf32>
   return %0 : memref<42xf32>
 }
 
 // -----
 
-// CHECK-LABEL: func @static_alloc() -> !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }"> {
-// BAREPTR-LABEL: func @static_alloc() -> !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }"> {
+// CHECK-LABEL: func @static_alloc() -> !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)> {
+// BAREPTR-LABEL: func @static_alloc() -> !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)> {
 func @static_alloc() -> memref<32x18xf32> {
 //      CHECK:  %[[sz1:.*]] = llvm.mlir.constant(32 : index) : !llvm.i64
 // CHECK-NEXT:  %[[sz2:.*]] = llvm.mlir.constant(18 : index) : !llvm.i64
 // CHECK-NEXT:  %[[num_elems:.*]] = llvm.mul %0, %1 : !llvm.i64
-// CHECK-NEXT:  %[[null:.*]] = llvm.mlir.null : !llvm<"float*">
+// CHECK-NEXT:  %[[null:.*]] = llvm.mlir.null : !llvm.ptr<float>
 // CHECK-NEXT:  %[[one:.*]] = llvm.mlir.constant(1 : index) : !llvm.i64
-// CHECK-NEXT:  %[[gep:.*]] = llvm.getelementptr %[[null]][%[[one]]] : (!llvm<"float*">, !llvm.i64) -> !llvm<"float*">
-//  CHECK-NEXT:  %[[sizeof:.*]] = llvm.ptrtoint %[[gep]] : !llvm<"float*"> to !llvm.i64
+// CHECK-NEXT:  %[[gep:.*]] = llvm.getelementptr %[[null]][%[[one]]] : (!llvm.ptr<float>, !llvm.i64) -> !llvm.ptr<float>
+// CHECK-NEXT:  %[[sizeof:.*]] = llvm.ptrtoint %[[gep]] : !llvm.ptr<float> to !llvm.i64
 // CHECK-NEXT:  %[[bytes:.*]] = llvm.mul %[[num_elems]], %[[sizeof]] : !llvm.i64
 // CHECK-NEXT:  %[[one_1:.*]] = llvm.mlir.constant(1 : index) : !llvm.i64
-// CHECK-NEXT:  %[[allocated:.*]] = llvm.call @malloc(%[[bytes]]) : (!llvm.i64) -> !llvm<"i8*">
-// CHECK-NEXT:  llvm.bitcast %[[allocated]] : !llvm<"i8*"> to !llvm<"float*">
+// CHECK-NEXT:  %[[allocated:.*]] = llvm.call @malloc(%[[bytes]]) : (!llvm.i64) -> !llvm.ptr<i8>
+// CHECK-NEXT:  llvm.bitcast %[[allocated]] : !llvm.ptr<i8> to !llvm.ptr<float>
 
 // BAREPTR-NEXT: %[[sz1:.*]] = llvm.mlir.constant(32 : index) : !llvm.i64
 // BAREPTR-NEXT: %[[sz2:.*]] = llvm.mlir.constant(18 : index) : !llvm.i64
 // BAREPTR-NEXT: %[[num_elems:.*]] = llvm.mul %[[sz1]], %[[sz2]] : !llvm.i64
-// BAREPTR-NEXT: %[[null:.*]] = llvm.mlir.null : !llvm<"float*">
+// BAREPTR-NEXT: %[[null:.*]] = llvm.mlir.null : !llvm.ptr<float>
 // BAREPTR-NEXT: %[[one:.*]] = llvm.mlir.constant(1 : index) : !llvm.i64
-// BAREPTR-NEXT: %[[gep:.*]] = llvm.getelementptr %[[null]][%[[one]]] : (!llvm<"float*">, !llvm.i64) -> !llvm<"float*">
-// BAREPTR-NEXT: %[[sizeof:.*]] = llvm.ptrtoint %[[gep]] : !llvm<"float*"> to !llvm.i64
+// BAREPTR-NEXT: %[[gep:.*]] = llvm.getelementptr %[[null]][%[[one]]] : (!llvm.ptr<float>, !llvm.i64) -> !llvm.ptr<float>
+// BAREPTR-NEXT: %[[sizeof:.*]] = llvm.ptrtoint %[[gep]] : !llvm.ptr<float> to !llvm.i64
 // BAREPTR-NEXT: %[[bytes:.*]] = llvm.mul %[[num_elems]], %[[sizeof]] : !llvm.i64
 // BAREPTR-NEXT:  %[[one_1:.*]] = llvm.mlir.constant(1 : index) : !llvm.i64
-// BAREPTR-NEXT: %[[allocated:.*]] = llvm.call @malloc(%[[bytes]]) : (!llvm.i64) -> !llvm<"i8*">
-// BAREPTR-NEXT: llvm.bitcast %[[allocated]] : !llvm<"i8*"> to !llvm<"float*">
+// BAREPTR-NEXT: %[[allocated:.*]] = llvm.call @malloc(%[[bytes]]) : (!llvm.i64) -> !llvm.ptr<i8>
+// BAREPTR-NEXT: llvm.bitcast %[[allocated]] : !llvm.ptr<i8> to !llvm.ptr<float>
  %0 = alloc() : memref<32x18xf32>
  return %0 : memref<32x18xf32>
 }
 
 // -----
 
-// CHECK-LABEL: func @static_alloca() -> !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }"> {
+// CHECK-LABEL: func @static_alloca() -> !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)> {
 func @static_alloca() -> memref<32x18xf32> {
 // CHECK-NEXT:  %[[sz1:.*]] = llvm.mlir.constant(32 : index) : !llvm.i64
 // CHECK-NEXT:  %[[sz2:.*]] = llvm.mlir.constant(18 : index) : !llvm.i64
 // CHECK-NEXT:  %[[num_elems:.*]] = llvm.mul %0, %1 : !llvm.i64
-// CHECK-NEXT:  %[[null:.*]] = llvm.mlir.null : !llvm<"float*">
+// CHECK-NEXT:  %[[null:.*]] = llvm.mlir.null : !llvm.ptr<float>
 // CHECK-NEXT:  %[[one:.*]] = llvm.mlir.constant(1 : index) : !llvm.i64
-// CHECK-NEXT:  %[[gep:.*]] = llvm.getelementptr %[[null]][%[[one]]] : (!llvm<"float*">, !llvm.i64) -> !llvm<"float*">
-// CHECK-NEXT:  %[[sizeof:.*]] = llvm.ptrtoint %[[gep]] : !llvm<"float*"> to !llvm.i64
+// CHECK-NEXT:  %[[gep:.*]] = llvm.getelementptr %[[null]][%[[one]]] : (!llvm.ptr<float>, !llvm.i64) -> !llvm.ptr<float>
+// CHECK-NEXT:  %[[sizeof:.*]] = llvm.ptrtoint %[[gep]] : !llvm.ptr<float> to !llvm.i64
 // CHECK-NEXT:  %[[bytes:.*]] = llvm.mul %[[num_elems]], %[[sizeof]] : !llvm.i64
 // CHECK-NEXT:  %[[one_1:.*]] = llvm.mlir.constant(1 : index) : !llvm.i64
-// CHECK-NEXT:  %[[allocated:.*]] = llvm.alloca %[[bytes]] x !llvm.float : (!llvm.i64) -> !llvm<"float*">
+// CHECK-NEXT:  %[[allocated:.*]] = llvm.alloca %[[bytes]] x !llvm.float : (!llvm.i64) -> !llvm.ptr<float>
  %0 = alloca() : memref<32x18xf32>
 
  // Test with explicitly specified alignment. llvm.alloca takes care of the
  // alignment. The same pointer is thus used for allocation and aligned
  // accesses.
- // CHECK: %[[alloca_aligned:.*]] = llvm.alloca %{{.*}} x !llvm.float {alignment = 32 : i64} : (!llvm.i64) -> !llvm<"float*">
- // CHECK: %[[desc:.*]] = llvm.mlir.undef : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
- // CHECK: %[[desc1:.*]] = llvm.insertvalue %[[alloca_aligned]], %[[desc]][0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
- // CHECK: llvm.insertvalue %[[alloca_aligned]], %[[desc1]][1] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+ // CHECK: %[[alloca_aligned:.*]] = llvm.alloca %{{.*}} x !llvm.float {alignment = 32 : i64} : (!llvm.i64) -> !llvm.ptr<float>
+ // CHECK: %[[desc:.*]] = llvm.mlir.undef : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
+ // CHECK: %[[desc1:.*]] = llvm.insertvalue %[[alloca_aligned]], %[[desc]][0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
+ // CHECK: llvm.insertvalue %[[alloca_aligned]], %[[desc1]][1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
  alloca() {alignment = 32} : memref<32x18xf32>
  return %0 : memref<32x18xf32>
 }
@@ -235,15 +235,15 @@ func @static_alloca() -> memref<32x18xf32> {
 // -----
 
 // CHECK-LABEL: func @static_dealloc
-// BAREPTR-LABEL: func @static_dealloc(%{{.*}}: !llvm<"float*">) {
+// BAREPTR-LABEL: func @static_dealloc(%{{.*}}: !llvm.ptr<float>) {
 func @static_dealloc(%static: memref<10x8xf32>) {
-//      CHECK:  %[[ptr:.*]] = llvm.extractvalue %{{.*}}[0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
-// CHECK-NEXT:  %[[bc:.*]] = llvm.bitcast %[[ptr]] : !llvm<"float*"> to !llvm<"i8*">
-// CHECK-NEXT:  llvm.call @free(%[[bc]]) : (!llvm<"i8*">) -> ()
+//      CHECK:  %[[ptr:.*]] = llvm.extractvalue %{{.*}}[0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
+// CHECK-NEXT:  %[[bc:.*]] = llvm.bitcast %[[ptr]] : !llvm.ptr<float> to !llvm.ptr<i8>
+// CHECK-NEXT:  llvm.call @free(%[[bc]]) : (!llvm.ptr<i8>) -> ()
 
-// BAREPTR:      %[[ptr:.*]] = llvm.extractvalue %{{.*}}[0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
-// BAREPTR-NEXT: %[[bc:.*]] = llvm.bitcast %[[ptr]] : !llvm<"float*"> to !llvm<"i8*">
-// BAREPTR-NEXT: llvm.call @free(%[[bc]]) : (!llvm<"i8*">) -> ()
+// BAREPTR:      %[[ptr:.*]] = llvm.extractvalue %{{.*}}[0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
+// BAREPTR-NEXT: %[[bc:.*]] = llvm.bitcast %[[ptr]] : !llvm.ptr<float> to !llvm.ptr<i8>
+// BAREPTR-NEXT: llvm.call @free(%[[bc]]) : (!llvm.ptr<i8>) -> ()
   dealloc %static : memref<10x8xf32>
   return
 }
@@ -251,17 +251,17 @@ func @static_dealloc(%static: memref<10x8xf32>) {
 // -----
 
 // CHECK-LABEL: func @zero_d_load
-// BAREPTR-LABEL: func @zero_d_load(%{{.*}}: !llvm<"float*">) -> !llvm.float
+// BAREPTR-LABEL: func @zero_d_load(%{{.*}}: !llvm.ptr<float>) -> !llvm.float
 func @zero_d_load(%arg0: memref<f32>) -> f32 {
-//      CHECK:  %[[ptr:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm<"{ float*, float*, i64 }">
+//      CHECK:  %[[ptr:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr<float>, ptr<float>, i64)>
 // CHECK-NEXT:  %[[c0:.*]] = llvm.mlir.constant(0 : index) : !llvm.i64
-// CHECK-NEXT:  %[[addr:.*]] = llvm.getelementptr %[[ptr]][%[[c0]]] : (!llvm<"float*">, !llvm.i64) -> !llvm<"float*">
-// CHECK-NEXT:  %{{.*}} = llvm.load %[[addr]] : !llvm<"float*">
+// CHECK-NEXT:  %[[addr:.*]] = llvm.getelementptr %[[ptr]][%[[c0]]] : (!llvm.ptr<float>, !llvm.i64) -> !llvm.ptr<float>
+// CHECK-NEXT:  %{{.*}} = llvm.load %[[addr]] : !llvm.ptr<float>
 
-// BAREPTR:      %[[ptr:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm<"{ float*, float*, i64 }">
+// BAREPTR:      %[[ptr:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr<float>, ptr<float>, i64)>
 // BAREPTR-NEXT: %[[c0:.*]] = llvm.mlir.constant(0 : index) : !llvm.i64
-// BAREPTR-NEXT: %[[addr:.*]] = llvm.getelementptr %[[ptr]][%[[c0]]] : (!llvm<"float*">, !llvm.i64) -> !llvm<"float*">
-// BAREPTR-NEXT: llvm.load %[[addr:.*]] : !llvm<"float*">
+// BAREPTR-NEXT: %[[addr:.*]] = llvm.getelementptr %[[ptr]][%[[c0]]] : (!llvm.ptr<float>, !llvm.i64) -> !llvm.ptr<float>
+// BAREPTR-NEXT: llvm.load %[[addr:.*]] : !llvm.ptr<float>
   %0 = load %arg0[] : memref<f32>
   return %0 : f32
 }
@@ -269,14 +269,14 @@ func @zero_d_load(%arg0: memref<f32>) -> f32 {
 // -----
 
 // CHECK-LABEL: func @static_load(
-// CHECK-COUNT-2: !llvm<"float*">,
+// CHECK-COUNT-2: !llvm.ptr<float>,
 // CHECK-COUNT-5: {{%[a-zA-Z0-9]*}}: !llvm.i64
 // CHECK:         %[[I:.*]]: !llvm.i64,
 // CHECK:         %[[J:.*]]: !llvm.i64)
 // BAREPTR-LABEL: func @static_load
-// BAREPTR-SAME: (%[[A:.*]]: !llvm<"float*">, %[[I:.*]]: !llvm.i64, %[[J:.*]]: !llvm.i64) {
+// BAREPTR-SAME: (%[[A:.*]]: !llvm.ptr<float>, %[[I:.*]]: !llvm.i64, %[[J:.*]]: !llvm.i64) {
 func @static_load(%static : memref<10x42xf32>, %i : index, %j : index) {
-//       CHECK:  %[[ptr:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+//       CHECK:  %[[ptr:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
 //  CHECK-NEXT:  %[[off:.*]] = llvm.mlir.constant(0 : index) : !llvm.i64
 //  CHECK-NEXT:  %[[st0:.*]] = llvm.mlir.constant(42 : index) : !llvm.i64
 //  CHECK-NEXT:  %[[offI:.*]] = llvm.mul %[[I]], %[[st0]] : !llvm.i64
@@ -284,10 +284,10 @@ func @static_load(%static : memref<10x42xf32>, %i : index, %j : index) {
 //  CHECK-NEXT:  %[[st1:.*]] = llvm.mlir.constant(1 : index) : !llvm.i64
 //  CHECK-NEXT:  %[[offJ:.*]] = llvm.mul %[[J]], %[[st1]] : !llvm.i64
 //  CHECK-NEXT:  %[[off1:.*]] = llvm.add %[[off0]], %[[offJ]] : !llvm.i64
-//  CHECK-NEXT:  %[[addr:.*]] = llvm.getelementptr %[[ptr]][%[[off1]]] : (!llvm<"float*">, !llvm.i64) -> !llvm<"float*">
-// CHECK-NEXT:  llvm.load %[[addr]] : !llvm<"float*">
+//  CHECK-NEXT:  %[[addr:.*]] = llvm.getelementptr %[[ptr]][%[[off1]]] : (!llvm.ptr<float>, !llvm.i64) -> !llvm.ptr<float>
+// CHECK-NEXT:  llvm.load %[[addr]] : !llvm.ptr<float>
 
-// BAREPTR:      %[[ptr:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+// BAREPTR:      %[[ptr:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
 // BAREPTR-NEXT: %[[off:.*]] = llvm.mlir.constant(0 : index) : !llvm.i64
 // BAREPTR-NEXT: %[[st0:.*]] = llvm.mlir.constant(42 : index) : !llvm.i64
 // BAREPTR-NEXT: %[[offI:.*]] = llvm.mul %[[I]], %[[st0]] : !llvm.i64
@@ -295,8 +295,8 @@ func @static_load(%static : memref<10x42xf32>, %i : index, %j : index) {
 // BAREPTR-NEXT: %[[st1:.*]] = llvm.mlir.constant(1 : index) : !llvm.i64
 // BAREPTR-NEXT: %[[offJ:.*]] = llvm.mul %[[J]], %[[st1]] : !llvm.i64
 // BAREPTR-NEXT: %[[off1:.*]] = llvm.add %[[off0]], %[[offJ]] : !llvm.i64
-// BAREPTR-NEXT: %[[addr:.*]] = llvm.getelementptr %[[ptr]][%[[off1]]] : (!llvm<"float*">, !llvm.i64) -> !llvm<"float*">
-// BAREPTR-NEXT: llvm.load %[[addr]] : !llvm<"float*">
+// BAREPTR-NEXT: %[[addr:.*]] = llvm.getelementptr %[[ptr]][%[[off1]]] : (!llvm.ptr<float>, !llvm.i64) -> !llvm.ptr<float>
+// BAREPTR-NEXT: llvm.load %[[addr]] : !llvm.ptr<float>
   %0 = load %static[%i, %j] : memref<10x42xf32>
   return
 }
@@ -305,17 +305,17 @@ func @static_load(%static : memref<10x42xf32>, %i : index, %j : index) {
 
 // CHECK-LABEL: func @zero_d_store
 // BAREPTR-LABEL: func @zero_d_store
-// BAREPTR-SAME: (%[[A:.*]]: !llvm<"float*">, %[[val:.*]]: !llvm.float)
+// BAREPTR-SAME: (%[[A:.*]]: !llvm.ptr<float>, %[[val:.*]]: !llvm.float)
 func @zero_d_store(%arg0: memref<f32>, %arg1: f32) {
-//      CHECK:  %[[ptr:.*]] = llvm.extractvalue %[[ld:.*]][1] : !llvm<"{ float*, float*, i64 }">
+//      CHECK:  %[[ptr:.*]] = llvm.extractvalue %[[ld:.*]][1] : !llvm.struct<(ptr<float>, ptr<float>, i64)>
 // CHECK-NEXT:  %[[off:.*]] = llvm.mlir.constant(0 : index) : !llvm.i64
-// CHECK-NEXT:  %[[addr:.*]] = llvm.getelementptr %[[ptr]][%[[off]]] : (!llvm<"float*">, !llvm.i64) -> !llvm<"float*">
-// CHECK-NEXT:  llvm.store %{{.*}}, %[[addr]] : !llvm<"float*">
+// CHECK-NEXT:  %[[addr:.*]] = llvm.getelementptr %[[ptr]][%[[off]]] : (!llvm.ptr<float>, !llvm.i64) -> !llvm.ptr<float>
+// CHECK-NEXT:  llvm.store %{{.*}}, %[[addr]] : !llvm.ptr<float>
 
-// BAREPTR:      %[[ptr:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm<"{ float*, float*, i64 }">
+// BAREPTR:      %[[ptr:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr<float>, ptr<float>, i64)>
 // BAREPTR-NEXT: %[[off:.*]] = llvm.mlir.constant(0 : index) : !llvm.i64
-// BAREPTR-NEXT: %[[addr:.*]] = llvm.getelementptr %[[ptr]][%[[off]]] : (!llvm<"float*">, !llvm.i64) -> !llvm<"float*">
-// BAREPTR-NEXT: llvm.store %[[val]], %[[addr]] : !llvm<"float*">
+// BAREPTR-NEXT: %[[addr:.*]] = llvm.getelementptr %[[ptr]][%[[off]]] : (!llvm.ptr<float>, !llvm.i64) -> !llvm.ptr<float>
+// BAREPTR-NEXT: llvm.store %[[val]], %[[addr]] : !llvm.ptr<float>
   store %arg1, %arg0[] : memref<f32>
   return
 }
@@ -323,8 +323,8 @@ func @zero_d_store(%arg0: memref<f32>, %arg1: f32) {
 // -----
 
 // CHECK-LABEL: func @static_store
-// CHECK-SAME:         %[[ARG0:[a-zA-Z0-9]*]]: !llvm<"float*">
-// CHECK-SAME:         %[[ARG1:[a-zA-Z0-9]*]]: !llvm<"float*">
+// CHECK-SAME:         %[[ARG0:[a-zA-Z0-9]*]]: !llvm.ptr<float>
+// CHECK-SAME:         %[[ARG1:[a-zA-Z0-9]*]]: !llvm.ptr<float>
 // CHECK-SAME:         %[[ARG2:[a-zA-Z0-9]*]]: !llvm.i64
 // CHECK-SAME:         %[[ARG3:[a-zA-Z0-9]*]]: !llvm.i64
 // CHECK-SAME:         %[[ARG4:[a-zA-Z0-9]*]]: !llvm.i64
@@ -333,11 +333,11 @@ func @zero_d_store(%arg0: memref<f32>, %arg1: f32) {
 // CHECK-SAME:         %[[I:[a-zA-Z0-9]*]]: !llvm.i64
 // CHECK-SAME:         %[[J:[a-zA-Z0-9]*]]: !llvm.i64
 // BAREPTR-LABEL: func @static_store
-// BAREPTR-SAME: %[[A:.*]]: !llvm<"float*">
+// BAREPTR-SAME: %[[A:.*]]: !llvm.ptr<float>
 // BAREPTR-SAME:         %[[I:[a-zA-Z0-9]*]]: !llvm.i64
 // BAREPTR-SAME:         %[[J:[a-zA-Z0-9]*]]: !llvm.i64
 func @static_store(%static : memref<10x42xf32>, %i : index, %j : index, %val : f32) {
-//       CHECK:  %[[ptr:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+//       CHECK:  %[[ptr:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
 //  CHECK-NEXT:  %[[off:.*]] = llvm.mlir.constant(0 : index) : !llvm.i64
 //  CHECK-NEXT:  %[[st0:.*]] = llvm.mlir.constant(42 : index) : !llvm.i64
 //  CHECK-NEXT:  %[[offI:.*]] = llvm.mul %[[I]], %[[st0]] : !llvm.i64
@@ -345,10 +345,10 @@ func @static_store(%static : memref<10x42xf32>, %i : index, %j : index, %val : f
 //  CHECK-NEXT:  %[[st1:.*]] = llvm.mlir.constant(1 : index) : !llvm.i64
 //  CHECK-NEXT:  %[[offJ:.*]] = llvm.mul %[[J]], %[[st1]] : !llvm.i64
 //  CHECK-NEXT:  %[[off1:.*]] = llvm.add %[[off0]], %[[offJ]] : !llvm.i64
-//  CHECK-NEXT:  %[[addr:.*]] = llvm.getelementptr %[[ptr]][%[[off1]]] : (!llvm<"float*">, !llvm.i64) -> !llvm<"float*">
-//  CHECK-NEXT:  llvm.store %{{.*}}, %[[addr]] : !llvm<"float*">
+//  CHECK-NEXT:  %[[addr:.*]] = llvm.getelementptr %[[ptr]][%[[off1]]] : (!llvm.ptr<float>, !llvm.i64) -> !llvm.ptr<float>
+//  CHECK-NEXT:  llvm.store %{{.*}}, %[[addr]] : !llvm.ptr<float>
 
-// BAREPTR:      %[[ptr:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+// BAREPTR:      %[[ptr:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
 // BAREPTR-NEXT: %[[off:.*]] = llvm.mlir.constant(0 : index) : !llvm.i64
 // BAREPTR-NEXT: %[[st0:.*]] = llvm.mlir.constant(42 : index) : !llvm.i64
 // BAREPTR-NEXT: %[[offI:.*]] = llvm.mul %[[I]], %[[st0]] : !llvm.i64
@@ -356,8 +356,8 @@ func @static_store(%static : memref<10x42xf32>, %i : index, %j : index, %val : f
 // BAREPTR-NEXT: %[[st1:.*]] = llvm.mlir.constant(1 : index) : !llvm.i64
 // BAREPTR-NEXT: %[[offJ:.*]] = llvm.mul %[[J]], %[[st1]] : !llvm.i64
 // BAREPTR-NEXT: %[[off1:.*]] = llvm.add %[[off0]], %[[offJ]] : !llvm.i64
-// BAREPTR-NEXT: %[[addr:.*]] = llvm.getelementptr %[[ptr]][%[[off1]]] : (!llvm<"float*">, !llvm.i64) -> !llvm<"float*">
-// BAREPTR-NEXT: llvm.store %{{.*}}, %[[addr]] : !llvm<"float*">
+// BAREPTR-NEXT: %[[addr:.*]] = llvm.getelementptr %[[ptr]][%[[off1]]] : (!llvm.ptr<float>, !llvm.i64) -> !llvm.ptr<float>
+// BAREPTR-NEXT: llvm.store %{{.*}}, %[[addr]] : !llvm.ptr<float>
   store %val, %static[%i, %j] : memref<10x42xf32>
   return
 }
@@ -365,10 +365,10 @@ func @static_store(%static : memref<10x42xf32>, %i : index, %j : index, %val : f
 // -----
 
 // CHECK-LABEL: func @static_memref_dim
-// BAREPTR-LABEL: func @static_memref_dim(%{{.*}}: !llvm<"float*">) {
+// BAREPTR-LABEL: func @static_memref_dim(%{{.*}}: !llvm.ptr<float>) {
 func @static_memref_dim(%static : memref<42x32x15x13x27xf32>) {
 // CHECK:        llvm.mlir.constant(42 : index) : !llvm.i64
-// BAREPTR:      llvm.insertvalue %{{.*}}, %{{.*}}[4, 4] : !llvm<"{ float*, float*, i64, [5 x i64], [5 x i64] }">
+// BAREPTR:      llvm.insertvalue %{{.*}}, %{{.*}}[4, 4] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<5 x i64>, array<5 x i64>)>
 // BAREPTR: llvm.mlir.constant(42 : index) : !llvm.i64
   %c0 = constant 0 : index
   %0 = dim %static, %c0 : memref<42x32x15x13x27xf32>
diff --git a/mlir/test/Conversion/StandardToLLVM/convert-to-llvmir.mlir b/mlir/test/Conversion/StandardToLLVM/convert-to-llvmir.mlir
index c1ec558da86f1..d0e883a10bee3 100644
--- a/mlir/test/Conversion/StandardToLLVM/convert-to-llvmir.mlir
+++ b/mlir/test/Conversion/StandardToLLVM/convert-to-llvmir.mlir
@@ -68,11 +68,11 @@ func @simple_loop() {
 // CHECK-LABEL: llvm.func @complex_numbers()
 // CHECK-NEXT:    %[[REAL0:.*]] = llvm.mlir.constant(1.200000e+00 : f32) : !llvm.float
 // CHECK-NEXT:    %[[IMAG0:.*]] = llvm.mlir.constant(3.400000e+00 : f32) : !llvm.float
-// CHECK-NEXT:    %[[CPLX0:.*]] = llvm.mlir.undef : !llvm<"{ float, float }">
-// CHECK-NEXT:    %[[CPLX1:.*]] = llvm.insertvalue %[[REAL0]], %[[CPLX0]][0] : !llvm<"{ float, float }">
-// CHECK-NEXT:    %[[CPLX2:.*]] = llvm.insertvalue %[[IMAG0]], %[[CPLX1]][1] : !llvm<"{ float, float }">
-// CHECK-NEXT:    %[[REAL1:.*]] = llvm.extractvalue %[[CPLX2:.*]][0] : !llvm<"{ float, float }">
-// CHECK-NEXT:    %[[IMAG1:.*]] = llvm.extractvalue %[[CPLX2:.*]][1] : !llvm<"{ float, float }">
+// CHECK-NEXT:    %[[CPLX0:.*]] = llvm.mlir.undef : !llvm.struct<(float, float)>
+// CHECK-NEXT:    %[[CPLX1:.*]] = llvm.insertvalue %[[REAL0]], %[[CPLX0]][0] : !llvm.struct<(float, float)>
+// CHECK-NEXT:    %[[CPLX2:.*]] = llvm.insertvalue %[[IMAG0]], %[[CPLX1]][1] : !llvm.struct<(float, float)>
+// CHECK-NEXT:    %[[REAL1:.*]] = llvm.extractvalue %[[CPLX2:.*]][0] : !llvm.struct<(float, float)>
+// CHECK-NEXT:    %[[IMAG1:.*]] = llvm.extractvalue %[[CPLX2:.*]][1] : !llvm.struct<(float, float)>
 // CHECK-NEXT:    llvm.return
 func @complex_numbers() {
   %real0 = constant 1.2 : f32
@@ -84,15 +84,15 @@ func @complex_numbers() {
 }
 
 // CHECK-LABEL: llvm.func @complex_addition()
-// CHECK-DAG:     %[[A_REAL:.*]] = llvm.extractvalue %[[A:.*]][0] : !llvm<"{ double, double }">
-// CHECK-DAG:     %[[B_REAL:.*]] = llvm.extractvalue %[[B:.*]][0] : !llvm<"{ double, double }">
-// CHECK-DAG:     %[[A_IMAG:.*]] = llvm.extractvalue %[[A]][1] : !llvm<"{ double, double }">
-// CHECK-DAG:     %[[B_IMAG:.*]] = llvm.extractvalue %[[B]][1] : !llvm<"{ double, double }">
-// CHECK:         %[[C0:.*]] = llvm.mlir.undef : !llvm<"{ double, double }">
+// CHECK-DAG:     %[[A_REAL:.*]] = llvm.extractvalue %[[A:.*]][0] : !llvm.struct<(double, double)>
+// CHECK-DAG:     %[[B_REAL:.*]] = llvm.extractvalue %[[B:.*]][0] : !llvm.struct<(double, double)>
+// CHECK-DAG:     %[[A_IMAG:.*]] = llvm.extractvalue %[[A]][1] : !llvm.struct<(double, double)>
+// CHECK-DAG:     %[[B_IMAG:.*]] = llvm.extractvalue %[[B]][1] : !llvm.struct<(double, double)>
+// CHECK:         %[[C0:.*]] = llvm.mlir.undef : !llvm.struct<(double, double)>
 // CHECK-DAG:     %[[C_REAL:.*]] = llvm.fadd %[[A_REAL]], %[[B_REAL]] : !llvm.double
 // CHECK-DAG:     %[[C_IMAG:.*]] = llvm.fadd %[[A_IMAG]], %[[B_IMAG]] : !llvm.double
-// CHECK:         %[[C1:.*]] = llvm.insertvalue %[[C_REAL]], %[[C0]][0] : !llvm<"{ double, double }">
-// CHECK:         %[[C2:.*]] = llvm.insertvalue %[[C_IMAG]], %[[C1]][1] : !llvm<"{ double, double }">
+// CHECK:         %[[C1:.*]] = llvm.insertvalue %[[C_REAL]], %[[C0]][0] : !llvm.struct<(double, double)>
+// CHECK:         %[[C2:.*]] = llvm.insertvalue %[[C_IMAG]], %[[C1]][1] : !llvm.struct<(double, double)>
 func @complex_addition() {
   %a_re = constant 1.2 : f64
   %a_im = constant 3.4 : f64
@@ -105,15 +105,15 @@ func @complex_addition() {
 }
 
 // CHECK-LABEL: llvm.func @complex_substraction()
-// CHECK-DAG:     %[[A_REAL:.*]] = llvm.extractvalue %[[A:.*]][0] : !llvm<"{ double, double }">
-// CHECK-DAG:     %[[B_REAL:.*]] = llvm.extractvalue %[[B:.*]][0] : !llvm<"{ double, double }">
-// CHECK-DAG:     %[[A_IMAG:.*]] = llvm.extractvalue %[[A]][1] : !llvm<"{ double, double }">
-// CHECK-DAG:     %[[B_IMAG:.*]] = llvm.extractvalue %[[B]][1] : !llvm<"{ double, double }">
-// CHECK:         %[[C0:.*]] = llvm.mlir.undef : !llvm<"{ double, double }">
+// CHECK-DAG:     %[[A_REAL:.*]] = llvm.extractvalue %[[A:.*]][0] : !llvm.struct<(double, double)>
+// CHECK-DAG:     %[[B_REAL:.*]] = llvm.extractvalue %[[B:.*]][0] : !llvm.struct<(double, double)>
+// CHECK-DAG:     %[[A_IMAG:.*]] = llvm.extractvalue %[[A]][1] : !llvm.struct<(double, double)>
+// CHECK-DAG:     %[[B_IMAG:.*]] = llvm.extractvalue %[[B]][1] : !llvm.struct<(double, double)>
+// CHECK:         %[[C0:.*]] = llvm.mlir.undef : !llvm.struct<(double, double)>
 // CHECK-DAG:     %[[C_REAL:.*]] = llvm.fsub %[[A_REAL]], %[[B_REAL]] : !llvm.double
 // CHECK-DAG:     %[[C_IMAG:.*]] = llvm.fsub %[[A_IMAG]], %[[B_IMAG]] : !llvm.double
-// CHECK:         %[[C1:.*]] = llvm.insertvalue %[[C_REAL]], %[[C0]][0] : !llvm<"{ double, double }">
-// CHECK:         %[[C2:.*]] = llvm.insertvalue %[[C_IMAG]], %[[C1]][1] : !llvm<"{ double, double }">
+// CHECK:         %[[C1:.*]] = llvm.insertvalue %[[C_REAL]], %[[C0]][0] : !llvm.struct<(double, double)>
+// CHECK:         %[[C2:.*]] = llvm.insertvalue %[[C_IMAG]], %[[C1]][1] : !llvm.struct<(double, double)>
 func @complex_substraction() {
   %a_re = constant 1.2 : f64
   %a_im = constant 3.4 : f64
@@ -427,39 +427,39 @@ func @more_imperfectly_nested_loops() {
 func @get_i64() -> (i64)
 // CHECK-LABEL: func @get_f32() -> !llvm.float
 func @get_f32() -> (f32)
-// CHECK-LABEL: func @get_c16() -> !llvm<"{ half, half }">
+// CHECK-LABEL: func @get_c16() -> !llvm.struct<(half, half)>
 func @get_c16() -> (complex<f16>)
-// CHECK-LABEL: func @get_c32() -> !llvm<"{ float, float }">
+// CHECK-LABEL: func @get_c32() -> !llvm.struct<(float, float)>
 func @get_c32() -> (complex<f32>)
-// CHECK-LABEL: func @get_c64() -> !llvm<"{ double, double }">
+// CHECK-LABEL: func @get_c64() -> !llvm.struct<(double, double)>
 func @get_c64() -> (complex<f64>)
-// CHECK-LABEL: func @get_memref() -> !llvm<"{ float*, float*, i64, [4 x i64], [4 x i64] }">
-// CHECK32-LABEL: func @get_memref() -> !llvm<"{ float*, float*, i32, [4 x i32], [4 x i32] }">
+// CHECK-LABEL: func @get_memref() -> !llvm.struct<(ptr<float>, ptr<float>, i64, array<4 x i64>, array<4 x i64>)>
+// CHECK32-LABEL: func @get_memref() -> !llvm.struct<(ptr<float>, ptr<float>, i32, array<4 x i32>, array<4 x i32>)>
 func @get_memref() -> (memref<42x?x10x?xf32>)
 
-// CHECK-LABEL: func @multireturn() -> !llvm<"{ i64, float, { float*, float*, i64, [4 x i64], [4 x i64] } }"> {
-// CHECK32-LABEL: func @multireturn() -> !llvm<"{ i64, float, { float*, float*, i32, [4 x i32], [4 x i32] } }"> {
+// CHECK-LABEL: func @multireturn() -> !llvm.struct<(i64, float, struct<(ptr<float>, ptr<float>, i64, array<4 x i64>, array<4 x i64>)>)> {
+// CHECK32-LABEL: func @multireturn() -> !llvm.struct<(i64, float, struct<(ptr<float>, ptr<float>, i32, array<4 x i32>, array<4 x i32>)>)> {
 func @multireturn() -> (i64, f32, memref<42x?x10x?xf32>) {
 ^bb0:
 // CHECK-NEXT:  {{.*}} = llvm.call @get_i64() : () -> !llvm.i64
 // CHECK-NEXT:  {{.*}} = llvm.call @get_f32() : () -> !llvm.float
-// CHECK-NEXT:  {{.*}} = llvm.call @get_memref() : () -> !llvm<"{ float*, float*, i64, [4 x i64], [4 x i64] }">
+// CHECK-NEXT:  {{.*}} = llvm.call @get_memref() : () -> !llvm.struct<(ptr<float>, ptr<float>, i64, array<4 x i64>, array<4 x i64>)>
 // CHECK32-NEXT:  {{.*}} = llvm.call @get_i64() : () -> !llvm.i64
 // CHECK32-NEXT:  {{.*}} = llvm.call @get_f32() : () -> !llvm.float
-// CHECK32-NEXT:  {{.*}} = llvm.call @get_memref() : () -> !llvm<"{ float*, float*, i32, [4 x i32], [4 x i32] }">
+// CHECK32-NEXT:  {{.*}} = llvm.call @get_memref() : () -> !llvm.struct<(ptr<float>, ptr<float>, i32, array<4 x i32>, array<4 x i32>)>
   %0 = call @get_i64() : () -> (i64)
   %1 = call @get_f32() : () -> (f32)
   %2 = call @get_memref() : () -> (memref<42x?x10x?xf32>)
-// CHECK-NEXT:  {{.*}} = llvm.mlir.undef : !llvm<"{ i64, float, { float*, float*, i64, [4 x i64], [4 x i64] } }">
-// CHECK-NEXT:  {{.*}} = llvm.insertvalue {{.*}}, {{.*}}[0] : !llvm<"{ i64, float, { float*, float*, i64, [4 x i64], [4 x i64] } }">
-// CHECK-NEXT:  {{.*}} = llvm.insertvalue {{.*}}, {{.*}}[1] : !llvm<"{ i64, float, { float*, float*, i64, [4 x i64], [4 x i64] } }">
-// CHECK-NEXT:  {{.*}} = llvm.insertvalue {{.*}}, {{.*}}[2] : !llvm<"{ i64, float, { float*, float*, i64, [4 x i64], [4 x i64] } }">
-// CHECK-NEXT:  llvm.return {{.*}} : !llvm<"{ i64, float, { float*, float*, i64, [4 x i64], [4 x i64] } }">
-// CHECK32-NEXT:  {{.*}} = llvm.mlir.undef : !llvm<"{ i64, float, { float*, float*, i32, [4 x i32], [4 x i32] } }">
-// CHECK32-NEXT:  {{.*}} = llvm.insertvalue {{.*}}, {{.*}}[0] : !llvm<"{ i64, float, { float*, float*, i32, [4 x i32], [4 x i32] } }">
-// CHECK32-NEXT:  {{.*}} = llvm.insertvalue {{.*}}, {{.*}}[1] : !llvm<"{ i64, float, { float*, float*, i32, [4 x i32], [4 x i32] } }">
-// CHECK32-NEXT:  {{.*}} = llvm.insertvalue {{.*}}, {{.*}}[2] : !llvm<"{ i64, float, { float*, float*, i32, [4 x i32], [4 x i32] } }">
-// CHECK32-NEXT:  llvm.return {{.*}} : !llvm<"{ i64, float, { float*, float*, i32, [4 x i32], [4 x i32] } }">
+// CHECK-NEXT:  {{.*}} = llvm.mlir.undef : !llvm.struct<(i64, float, struct<(ptr<float>, ptr<float>, i64, array<4 x i64>, array<4 x i64>)>)>
+// CHECK-NEXT:  {{.*}} = llvm.insertvalue {{.*}}, {{.*}}[0] : !llvm.struct<(i64, float, struct<(ptr<float>, ptr<float>, i64, array<4 x i64>, array<4 x i64>)>)>
+// CHECK-NEXT:  {{.*}} = llvm.insertvalue {{.*}}, {{.*}}[1] : !llvm.struct<(i64, float, struct<(ptr<float>, ptr<float>, i64, array<4 x i64>, array<4 x i64>)>)>
+// CHECK-NEXT:  {{.*}} = llvm.insertvalue {{.*}}, {{.*}}[2] : !llvm.struct<(i64, float, struct<(ptr<float>, ptr<float>, i64, array<4 x i64>, array<4 x i64>)>)>
+// CHECK-NEXT:  llvm.return {{.*}} : !llvm.struct<(i64, float, struct<(ptr<float>, ptr<float>, i64, array<4 x i64>, array<4 x i64>)>)>
+// CHECK32-NEXT:  {{.*}} = llvm.mlir.undef : !llvm.struct<(i64, float, struct<(ptr<float>, ptr<float>, i32, array<4 x i32>, array<4 x i32>)>)>
+// CHECK32-NEXT:  {{.*}} = llvm.insertvalue {{.*}}, {{.*}}[0] : !llvm.struct<(i64, float, struct<(ptr<float>, ptr<float>, i32, array<4 x i32>, array<4 x i32>)>)>
+// CHECK32-NEXT:  {{.*}} = llvm.insertvalue {{.*}}, {{.*}}[1] : !llvm.struct<(i64, float, struct<(ptr<float>, ptr<float>, i32, array<4 x i32>, array<4 x i32>)>)>
+// CHECK32-NEXT:  {{.*}} = llvm.insertvalue {{.*}}, {{.*}}[2] : !llvm.struct<(i64, float, struct<(ptr<float>, ptr<float>, i32, array<4 x i32>, array<4 x i32>)>)>
+// CHECK32-NEXT:  llvm.return {{.*}} : !llvm.struct<(i64, float, struct<(ptr<float>, ptr<float>, i32, array<4 x i32>, array<4 x i32>)>)>
   return %0, %1, %2 : i64, f32, memref<42x?x10x?xf32>
 }
 
@@ -468,14 +468,14 @@ func @multireturn() -> (i64, f32, memref<42x?x10x?xf32>) {
 // CHECK32-LABEL: func @multireturn_caller() {
 func @multireturn_caller() {
 ^bb0:
-// CHECK-NEXT:  {{.*}} = llvm.call @multireturn() : () -> !llvm<"{ i64, float, { float*, float*, i64, [4 x i64], [4 x i64] } }">
-// CHECK-NEXT:  {{.*}} = llvm.extractvalue {{.*}}[0] : !llvm<"{ i64, float, { float*, float*, i64, [4 x i64], [4 x i64] } }">
-// CHECK-NEXT:  {{.*}} = llvm.extractvalue {{.*}}[1] : !llvm<"{ i64, float, { float*, float*, i64, [4 x i64], [4 x i64] } }">
-// CHECK-NEXT:  {{.*}} = llvm.extractvalue {{.*}}[2] : !llvm<"{ i64, float, { float*, float*, i64, [4 x i64], [4 x i64] } }">
-// CHECK32-NEXT:  {{.*}} = llvm.call @multireturn() : () -> !llvm<"{ i64, float, { float*, float*, i32, [4 x i32], [4 x i32] } }">
-// CHECK32-NEXT:  {{.*}} = llvm.extractvalue {{.*}}[0] : !llvm<"{ i64, float, { float*, float*, i32, [4 x i32], [4 x i32] } }">
-// CHECK32-NEXT:  {{.*}} = llvm.extractvalue {{.*}}[1] : !llvm<"{ i64, float, { float*, float*, i32, [4 x i32], [4 x i32] } }">
-// CHECK32-NEXT:  {{.*}} = llvm.extractvalue {{.*}}[2] : !llvm<"{ i64, float, { float*, float*, i32, [4 x i32], [4 x i32] } }">
+// CHECK-NEXT:  {{.*}} = llvm.call @multireturn() : () -> !llvm.struct<(i64, float, struct<(ptr<float>, ptr<float>, i64, array<4 x i64>, array<4 x i64>)>)>
+// CHECK-NEXT:  {{.*}} = llvm.extractvalue {{.*}}[0] : !llvm.struct<(i64, float, struct<(ptr<float>, ptr<float>, i64, array<4 x i64>, array<4 x i64>)>)>
+// CHECK-NEXT:  {{.*}} = llvm.extractvalue {{.*}}[1] : !llvm.struct<(i64, float, struct<(ptr<float>, ptr<float>, i64, array<4 x i64>, array<4 x i64>)>)>
+// CHECK-NEXT:  {{.*}} = llvm.extractvalue {{.*}}[2] : !llvm.struct<(i64, float, struct<(ptr<float>, ptr<float>, i64, array<4 x i64>, array<4 x i64>)>)>
+// CHECK32-NEXT:  {{.*}} = llvm.call @multireturn() : () -> !llvm.struct<(i64, float, struct<(ptr<float>, ptr<float>, i32, array<4 x i32>, array<4 x i32>)>)>
+// CHECK32-NEXT:  {{.*}} = llvm.extractvalue {{.*}}[0] : !llvm.struct<(i64, float, struct<(ptr<float>, ptr<float>, i32, array<4 x i32>, array<4 x i32>)>)>
+// CHECK32-NEXT:  {{.*}} = llvm.extractvalue {{.*}}[1] : !llvm.struct<(i64, float, struct<(ptr<float>, ptr<float>, i32, array<4 x i32>, array<4 x i32>)>)>
+// CHECK32-NEXT:  {{.*}} = llvm.extractvalue {{.*}}[2] : !llvm.struct<(i64, float, struct<(ptr<float>, ptr<float>, i32, array<4 x i32>, array<4 x i32>)>)>
   %0:3 = call @multireturn() : () -> (i64, f32, memref<42x?x10x?xf32>)
   %1 = constant 42 : i64
 // CHECK:       {{.*}} = llvm.add {{.*}}, {{.*}} : !llvm.i64
@@ -487,35 +487,35 @@ func @multireturn_caller() {
   return
 }
 
-// CHECK-LABEL: func @vector_ops(%arg0: !llvm<"<4 x float>">, %arg1: !llvm<"<4 x i1>">, %arg2: !llvm<"<4 x i64>">, %arg3: !llvm<"<4 x i64>">) -> !llvm<"<4 x float>"> {
+// CHECK-LABEL: func @vector_ops(%arg0: !llvm.vec<4 x float>, %arg1: !llvm.vec<4 x i1>, %arg2: !llvm.vec<4 x i64>, %arg3: !llvm.vec<4 x i64>) -> !llvm.vec<4 x float> {
 func @vector_ops(%arg0: vector<4xf32>, %arg1: vector<4xi1>, %arg2: vector<4xi64>, %arg3: vector<4xi64>) -> vector<4xf32> {
-// CHECK-NEXT:  %0 = llvm.mlir.constant(dense<4.200000e+01> : vector<4xf32>) : !llvm<"<4 x float>">
+// CHECK-NEXT:  %0 = llvm.mlir.constant(dense<4.200000e+01> : vector<4xf32>) : !llvm.vec<4 x float>
   %0 = constant dense<42.> : vector<4xf32>
-// CHECK-NEXT:  %1 = llvm.fadd %arg0, %0 : !llvm<"<4 x float>">
+// CHECK-NEXT:  %1 = llvm.fadd %arg0, %0 : !llvm.vec<4 x float>
   %1 = addf %arg0, %0 : vector<4xf32>
-// CHECK-NEXT:  %2 = llvm.sdiv %arg2, %arg2 : !llvm<"<4 x i64>">
+// CHECK-NEXT:  %2 = llvm.sdiv %arg2, %arg2 : !llvm.vec<4 x i64>
   %3 = divi_signed %arg2, %arg2 : vector<4xi64>
-// CHECK-NEXT:  %3 = llvm.udiv %arg2, %arg2 : !llvm<"<4 x i64>">
+// CHECK-NEXT:  %3 = llvm.udiv %arg2, %arg2 : !llvm.vec<4 x i64>
   %4 = divi_unsigned %arg2, %arg2 : vector<4xi64>
-// CHECK-NEXT:  %4 = llvm.srem %arg2, %arg2 : !llvm<"<4 x i64>">
+// CHECK-NEXT:  %4 = llvm.srem %arg2, %arg2 : !llvm.vec<4 x i64>
   %5 = remi_signed %arg2, %arg2 : vector<4xi64>
-// CHECK-NEXT:  %5 = llvm.urem %arg2, %arg2 : !llvm<"<4 x i64>">
+// CHECK-NEXT:  %5 = llvm.urem %arg2, %arg2 : !llvm.vec<4 x i64>
   %6 = remi_unsigned %arg2, %arg2 : vector<4xi64>
-// CHECK-NEXT:  %6 = llvm.fdiv %arg0, %0 : !llvm<"<4 x float>">
+// CHECK-NEXT:  %6 = llvm.fdiv %arg0, %0 : !llvm.vec<4 x float>
   %7 = divf %arg0, %0 : vector<4xf32>
-// CHECK-NEXT:  %7 = llvm.frem %arg0, %0 : !llvm<"<4 x float>">
+// CHECK-NEXT:  %7 = llvm.frem %arg0, %0 : !llvm.vec<4 x float>
   %8 = remf %arg0, %0 : vector<4xf32>
-// CHECK-NEXT:  %8 = llvm.and %arg2, %arg3 : !llvm<"<4 x i64>">
+// CHECK-NEXT:  %8 = llvm.and %arg2, %arg3 : !llvm.vec<4 x i64>
   %9 = and %arg2, %arg3 : vector<4xi64>
-// CHECK-NEXT:  %9 = llvm.or %arg2, %arg3 : !llvm<"<4 x i64>">
+// CHECK-NEXT:  %9 = llvm.or %arg2, %arg3 : !llvm.vec<4 x i64>
   %10 = or %arg2, %arg3 : vector<4xi64>
-// CHECK-NEXT:  %10 = llvm.xor %arg2, %arg3 : !llvm<"<4 x i64>">
+// CHECK-NEXT:  %10 = llvm.xor %arg2, %arg3 : !llvm.vec<4 x i64>
   %11 = xor %arg2, %arg3 : vector<4xi64>
-// CHECK-NEXT:  %11 = llvm.shl %arg2, %arg2 : !llvm<"<4 x i64>">
+// CHECK-NEXT:  %11 = llvm.shl %arg2, %arg2 : !llvm.vec<4 x i64>
   %12 = shift_left %arg2, %arg2 : vector<4xi64>
-// CHECK-NEXT:  %12 = llvm.ashr %arg2, %arg2 : !llvm<"<4 x i64>">
+// CHECK-NEXT:  %12 = llvm.ashr %arg2, %arg2 : !llvm.vec<4 x i64>
   %13 = shift_right_signed %arg2, %arg2 : vector<4xi64>
-// CHECK-NEXT:  %13 = llvm.lshr %arg2, %arg2 : !llvm<"<4 x i64>">
+// CHECK-NEXT:  %13 = llvm.lshr %arg2, %arg2 : !llvm.vec<4 x i64>
   %14 = shift_right_unsigned %arg2, %arg2 : vector<4xi64>
   return %1 : vector<4xf32>
 }
@@ -583,13 +583,13 @@ func @index_cast(%arg0: index, %arg1: i1) {
 // Checking conversion of integer types to floating point.
 // CHECK-LABEL: @sitofp
 func @sitofp(%arg0 : i32, %arg1 : i64) {
-// CHECK-NEXT: = llvm.sitofp {{.*}} : !llvm.i{{.*}} to !llvm.float
+// CHECK-NEXT: = llvm.sitofp {{.*}} : !llvm.i32 to !llvm.float
   %0 = sitofp %arg0: i32 to f32
-// CHECK-NEXT: = llvm.sitofp {{.*}} : !llvm.i{{.*}} to !llvm.double
+// CHECK-NEXT: = llvm.sitofp {{.*}} : !llvm.i32 to !llvm.double
   %1 = sitofp %arg0: i32 to f64
-// CHECK-NEXT: = llvm.sitofp {{.*}} : !llvm.i{{.*}} to !llvm.float
+// CHECK-NEXT: = llvm.sitofp {{.*}} : !llvm.i64 to !llvm.float
   %2 = sitofp %arg1: i64 to f32
-// CHECK-NEXT: = llvm.sitofp {{.*}} : !llvm.i{{.*}} to !llvm.double
+// CHECK-NEXT: = llvm.sitofp {{.*}} : !llvm.i64 to !llvm.double
   %3 = sitofp %arg1: i64 to f64
   return
 }
@@ -609,11 +609,11 @@ func @fpext(%arg0 : f16, %arg1 : f32) {
 // Checking conversion of integer types to floating point.
 // CHECK-LABEL: @fpext
 func @fpext_vector(%arg0 : vector<2xf16>, %arg1 : vector<2xf32>) {
-// CHECK-NEXT: = llvm.fpext {{.*}} : !llvm<"<2 x half>"> to !llvm<"<2 x float>">
+// CHECK-NEXT: = llvm.fpext {{.*}} : !llvm.vec<2 x half> to !llvm.vec<2 x float>
   %0 = fpext %arg0: vector<2xf16> to vector<2xf32>
-// CHECK-NEXT: = llvm.fpext {{.*}} : !llvm<"<2 x half>"> to !llvm<"<2 x double>">
+// CHECK-NEXT: = llvm.fpext {{.*}} : !llvm.vec<2 x half> to !llvm.vec<2 x double>
   %1 = fpext %arg0: vector<2xf16> to vector<2xf64>
-// CHECK-NEXT: = llvm.fpext {{.*}} : !llvm<"<2 x float>"> to !llvm<"<2 x double>">
+// CHECK-NEXT: = llvm.fpext {{.*}} : !llvm.vec<2 x float> to !llvm.vec<2 x double>
   %2 = fpext %arg1: vector<2xf32> to vector<2xf64>
   return
 }
@@ -647,11 +647,11 @@ func @fptrunc(%arg0 : f32, %arg1 : f64) {
 // Checking conversion of integer types to floating point.
 // CHECK-LABEL: @fptrunc
 func @fptrunc_vector(%arg0 : vector<2xf32>, %arg1 : vector<2xf64>) {
-// CHECK-NEXT: = llvm.fptrunc {{.*}} : !llvm<"<2 x float>"> to !llvm<"<2 x half>">
+// CHECK-NEXT: = llvm.fptrunc {{.*}} : !llvm.vec<2 x float> to !llvm.vec<2 x half>
   %0 = fptrunc %arg0: vector<2xf32> to vector<2xf16>
-// CHECK-NEXT: = llvm.fptrunc {{.*}} : !llvm<"<2 x double>"> to !llvm<"<2 x half>">
+// CHECK-NEXT: = llvm.fptrunc {{.*}} : !llvm.vec<2 x double> to !llvm.vec<2 x half>
   %1 = fptrunc %arg1: vector<2xf64> to vector<2xf16>
-// CHECK-NEXT: = llvm.fptrunc {{.*}} : !llvm<"<2 x double>"> to !llvm<"<2 x float>">
+// CHECK-NEXT: = llvm.fptrunc {{.*}} : !llvm.vec<2 x double> to !llvm.vec<2 x float>
   %2 = fptrunc %arg1: vector<2xf64> to vector<2xf32>
   return
 }
@@ -731,110 +731,110 @@ func @vec_bin(%arg0: vector<2x2x2xf32>) -> vector<2x2x2xf32> {
   %0 = addf %arg0, %arg0 : vector<2x2x2xf32>
   return %0 : vector<2x2x2xf32>
 
-//  CHECK-NEXT: llvm.mlir.undef : !llvm<"[2 x [2 x <2 x float>]]">
+//  CHECK-NEXT: llvm.mlir.undef : !llvm.array<2 x array<2 x vec<2 x float>>>
 
 // This block appears 2x2 times
-//  CHECK-NEXT: llvm.extractvalue %{{.*}}[0, 0] : !llvm<"[2 x [2 x <2 x float>]]">
-//  CHECK-NEXT: llvm.extractvalue %{{.*}}[0, 0] : !llvm<"[2 x [2 x <2 x float>]]">
-//  CHECK-NEXT: llvm.fadd %{{.*}} : !llvm<"<2 x float>">
-//  CHECK-NEXT: llvm.insertvalue %{{.*}}[0, 0] : !llvm<"[2 x [2 x <2 x float>]]">
+//  CHECK-NEXT: llvm.extractvalue %{{.*}}[0, 0] : !llvm.array<2 x array<2 x vec<2 x float>>>
+//  CHECK-NEXT: llvm.extractvalue %{{.*}}[0, 0] : !llvm.array<2 x array<2 x vec<2 x float>>>
+//  CHECK-NEXT: llvm.fadd %{{.*}} : !llvm.vec<2 x float>
+//  CHECK-NEXT: llvm.insertvalue %{{.*}}[0, 0] : !llvm.array<2 x array<2 x vec<2 x float>>>
 
 // We check the proper indexing of extract/insert in the remaining 3 positions.
-//       CHECK: llvm.extractvalue %{{.*}}[0, 1] : !llvm<"[2 x [2 x <2 x float>]]">
-//       CHECK: llvm.insertvalue %{{.*}}[0, 1] : !llvm<"[2 x [2 x <2 x float>]]">
-//       CHECK: llvm.extractvalue %{{.*}}[1, 0] : !llvm<"[2 x [2 x <2 x float>]]">
-//       CHECK: llvm.insertvalue %{{.*}}[1, 0] : !llvm<"[2 x [2 x <2 x float>]]">
-//       CHECK: llvm.extractvalue %{{.*}}[1, 1] : !llvm<"[2 x [2 x <2 x float>]]">
-//       CHECK: llvm.insertvalue %{{.*}}[1, 1] : !llvm<"[2 x [2 x <2 x float>]]">
+//       CHECK: llvm.extractvalue %{{.*}}[0, 1] : !llvm.array<2 x array<2 x vec<2 x float>>>
+//       CHECK: llvm.insertvalue %{{.*}}[0, 1] : !llvm.array<2 x array<2 x vec<2 x float>>>
+//       CHECK: llvm.extractvalue %{{.*}}[1, 0] : !llvm.array<2 x array<2 x vec<2 x float>>>
+//       CHECK: llvm.insertvalue %{{.*}}[1, 0] : !llvm.array<2 x array<2 x vec<2 x float>>>
+//       CHECK: llvm.extractvalue %{{.*}}[1, 1] : !llvm.array<2 x array<2 x vec<2 x float>>>
+//       CHECK: llvm.insertvalue %{{.*}}[1, 1] : !llvm.array<2 x array<2 x vec<2 x float>>>
 
 // And we're done
 //   CHECK-NEXT: return
 }
 
 // CHECK-LABEL: @splat
-// CHECK-SAME: %[[A:arg[0-9]+]]: !llvm<"<4 x float>">
+// CHECK-SAME: %[[A:arg[0-9]+]]: !llvm.vec<4 x float>
 // CHECK-SAME: %[[ELT:arg[0-9]+]]: !llvm.float
 func @splat(%a: vector<4xf32>, %b: f32) -> vector<4xf32> {
   %vb = splat %b : vector<4xf32>
   %r = mulf %a, %vb : vector<4xf32>
   return %r : vector<4xf32>
 }
-// CHECK-NEXT: %[[UNDEF:[0-9]+]] = llvm.mlir.undef : !llvm<"<4 x float>">
+// CHECK-NEXT: %[[UNDEF:[0-9]+]] = llvm.mlir.undef : !llvm.vec<4 x float>
 // CHECK-NEXT: %[[ZERO:[0-9]+]] = llvm.mlir.constant(0 : i32) : !llvm.i32
-// CHECK-NEXT: %[[V:[0-9]+]] = llvm.insertelement %[[ELT]], %[[UNDEF]][%[[ZERO]] : !llvm.i32] : !llvm<"<4 x float>">
+// CHECK-NEXT: %[[V:[0-9]+]] = llvm.insertelement %[[ELT]], %[[UNDEF]][%[[ZERO]] : !llvm.i32] : !llvm.vec<4 x float>
 // CHECK-NEXT: %[[SPLAT:[0-9]+]] = llvm.shufflevector %[[V]], %[[UNDEF]] [0 : i32, 0 : i32, 0 : i32, 0 : i32]
-// CHECK-NEXT: %[[SCALE:[0-9]+]] = llvm.fmul %[[A]], %[[SPLAT]] : !llvm<"<4 x float>">
-// CHECK-NEXT: llvm.return %[[SCALE]] : !llvm<"<4 x float>">
+// CHECK-NEXT: %[[SCALE:[0-9]+]] = llvm.fmul %[[A]], %[[SPLAT]] : !llvm.vec<4 x float>
+// CHECK-NEXT: llvm.return %[[SCALE]] : !llvm.vec<4 x float>
 
 // CHECK-LABEL: func @view(
 // CHECK: %[[ARG0:.*]]: !llvm.i64, %[[ARG1:.*]]: !llvm.i64, %[[ARG2:.*]]: !llvm.i64
 func @view(%arg0 : index, %arg1 : index, %arg2 : index) {
   // CHECK: llvm.mlir.constant(2048 : index) : !llvm.i64
-  // CHECK: llvm.mlir.undef : !llvm<"{ i8*, i8*, i64, [1 x i64], [1 x i64] }">
+  // CHECK: llvm.mlir.undef : !llvm.struct<(ptr<i8>, ptr<i8>, i64, array<1 x i64>, array<1 x i64>)>
   %0 = alloc() : memref<2048xi8>
 
   // Test two dynamic sizes.
-  // CHECK: llvm.mlir.undef : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
-  // CHECK: %[[BASE_PTR:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm<"{ i8*, i8*, i64, [1 x i64], [1 x i64] }">
-  // CHECK: %[[SHIFTED_BASE_PTR:.*]] = llvm.getelementptr %[[BASE_PTR]][%[[ARG2]]] : (!llvm<"i8*">, !llvm.i64) -> !llvm<"i8*">
-  // CHECK: %[[CAST_SHIFTED_BASE_PTR:.*]] = llvm.bitcast %[[SHIFTED_BASE_PTR]] : !llvm<"i8*"> to !llvm<"float*">
-  // CHECK: llvm.insertvalue %[[CAST_SHIFTED_BASE_PTR]], %{{.*}}[1] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+  // CHECK: llvm.mlir.undef : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
+  // CHECK: %[[BASE_PTR:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr<i8>, ptr<i8>, i64, array<1 x i64>, array<1 x i64>)>
+  // CHECK: %[[SHIFTED_BASE_PTR:.*]] = llvm.getelementptr %[[BASE_PTR]][%[[ARG2]]] : (!llvm.ptr<i8>, !llvm.i64) -> !llvm.ptr<i8>
+  // CHECK: %[[CAST_SHIFTED_BASE_PTR:.*]] = llvm.bitcast %[[SHIFTED_BASE_PTR]] : !llvm.ptr<i8> to !llvm.ptr<float>
+  // CHECK: llvm.insertvalue %[[CAST_SHIFTED_BASE_PTR]], %{{.*}}[1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: %[[C0:.*]] = llvm.mlir.constant(0 : index) : !llvm.i64
-  // CHECK: llvm.insertvalue %[[C0]], %{{.*}}[2] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
-  // CHECK: llvm.insertvalue %[[ARG1]], %{{.*}}[3, 1] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+  // CHECK: llvm.insertvalue %[[C0]], %{{.*}}[2] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
+  // CHECK: llvm.insertvalue %[[ARG1]], %{{.*}}[3, 1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: llvm.mlir.constant(1 : index) : !llvm.i64
-  // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[4, 1] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
-  // CHECK: llvm.insertvalue %[[ARG0]], %{{.*}}[3, 0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+  // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[4, 1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
+  // CHECK: llvm.insertvalue %[[ARG0]], %{{.*}}[3, 0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: llvm.mul %{{.*}}, %[[ARG1]]
-  // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[4, 0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+  // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[4, 0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
   %1 = view %0[%arg2][%arg0, %arg1] : memref<2048xi8> to memref<?x?xf32>
 
   // Test one dynamic size.
-  // CHECK: llvm.mlir.undef : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
-  // CHECK: %[[BASE_PTR_2:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm<"{ i8*, i8*, i64, [1 x i64], [1 x i64] }">
-  // CHECK: %[[SHIFTED_BASE_PTR_2:.*]] = llvm.getelementptr %[[BASE_PTR_2]][%[[ARG2]]] : (!llvm<"i8*">, !llvm.i64) -> !llvm<"i8*">
-  // CHECK: %[[CAST_SHIFTED_BASE_PTR_2:.*]] = llvm.bitcast %[[SHIFTED_BASE_PTR_2]] : !llvm<"i8*"> to !llvm<"float*">
-  // CHECK: llvm.insertvalue %[[CAST_SHIFTED_BASE_PTR_2]], %{{.*}}[1] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+  // CHECK: llvm.mlir.undef : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
+  // CHECK: %[[BASE_PTR_2:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr<i8>, ptr<i8>, i64, array<1 x i64>, array<1 x i64>)>
+  // CHECK: %[[SHIFTED_BASE_PTR_2:.*]] = llvm.getelementptr %[[BASE_PTR_2]][%[[ARG2]]] : (!llvm.ptr<i8>, !llvm.i64) -> !llvm.ptr<i8>
+  // CHECK: %[[CAST_SHIFTED_BASE_PTR_2:.*]] = llvm.bitcast %[[SHIFTED_BASE_PTR_2]] : !llvm.ptr<i8> to !llvm.ptr<float>
+  // CHECK: llvm.insertvalue %[[CAST_SHIFTED_BASE_PTR_2]], %{{.*}}[1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: %[[C0_2:.*]] = llvm.mlir.constant(0 : index) : !llvm.i64
-  // CHECK: llvm.insertvalue %[[C0_2]], %{{.*}}[2] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
-  // CHECK: llvm.insertvalue %[[ARG1]], %{{.*}}[3, 1] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+  // CHECK: llvm.insertvalue %[[C0_2]], %{{.*}}[2] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
+  // CHECK: llvm.insertvalue %[[ARG1]], %{{.*}}[3, 1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: llvm.mlir.constant(1 : index) : !llvm.i64
-  // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[4, 1] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+  // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[4, 1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: llvm.mlir.constant(4 : index) : !llvm.i64
-  // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[3, 0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+  // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[3, 0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: llvm.mul %{{.*}}, %[[ARG1]]
-  // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[4, 0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+  // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[4, 0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
   %3 = view %0[%arg2][%arg1] : memref<2048xi8> to memref<4x?xf32>
 
   // Test static sizes.
-  // CHECK: llvm.mlir.undef : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
-  // CHECK: %[[BASE_PTR_3:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm<"{ i8*, i8*, i64, [1 x i64], [1 x i64] }">
-  // CHECK: %[[SHIFTED_BASE_PTR_3:.*]] = llvm.getelementptr %[[BASE_PTR_3]][%[[ARG2]]] : (!llvm<"i8*">, !llvm.i64) -> !llvm<"i8*">
-  // CHECK: %[[CAST_SHIFTED_BASE_PTR_3:.*]] = llvm.bitcast %[[SHIFTED_BASE_PTR_3]] : !llvm<"i8*"> to !llvm<"float*">
-  // CHECK: llvm.insertvalue %[[CAST_SHIFTED_BASE_PTR_3]], %{{.*}}[1] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+  // CHECK: llvm.mlir.undef : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
+  // CHECK: %[[BASE_PTR_3:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr<i8>, ptr<i8>, i64, array<1 x i64>, array<1 x i64>)>
+  // CHECK: %[[SHIFTED_BASE_PTR_3:.*]] = llvm.getelementptr %[[BASE_PTR_3]][%[[ARG2]]] : (!llvm.ptr<i8>, !llvm.i64) -> !llvm.ptr<i8>
+  // CHECK: %[[CAST_SHIFTED_BASE_PTR_3:.*]] = llvm.bitcast %[[SHIFTED_BASE_PTR_3]] : !llvm.ptr<i8> to !llvm.ptr<float>
+  // CHECK: llvm.insertvalue %[[CAST_SHIFTED_BASE_PTR_3]], %{{.*}}[1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: %[[C0_3:.*]] = llvm.mlir.constant(0 : index) : !llvm.i64
-  // CHECK: llvm.insertvalue %[[C0_3]], %{{.*}}[2] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+  // CHECK: llvm.insertvalue %[[C0_3]], %{{.*}}[2] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: llvm.mlir.constant(4 : index) : !llvm.i64
-  // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[3, 1] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+  // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[3, 1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: llvm.mlir.constant(1 : index) : !llvm.i64
-  // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[4, 1] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+  // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[4, 1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: llvm.mlir.constant(64 : index) : !llvm.i64
-  // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[3, 0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+  // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[3, 0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: llvm.mlir.constant(4 : index) : !llvm.i64
-  // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[4, 0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+  // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[4, 0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
   %5 = view %0[%arg2][] : memref<2048xi8> to memref<64x4xf32>
 
   return
 }
 
 // CHECK-LABEL: func @subview(
-// CHECK-COUNT-2: !llvm<"float*">,
+// CHECK-COUNT-2: !llvm.ptr<float>,
 // CHECK-COUNT-5: {{%[a-zA-Z0-9]*}}: !llvm.i64,
 // CHECK:         %[[ARG0:[a-zA-Z0-9]*]]: !llvm.i64,
 // CHECK:         %[[ARG1:[a-zA-Z0-9]*]]: !llvm.i64,
 // CHECK:         %[[ARG2:.*]]: !llvm.i64)
 // CHECK32-LABEL: func @subview(
-// CHECK32-COUNT-2: !llvm<"float*">,
+// CHECK32-COUNT-2: !llvm.ptr<float>,
 // CHECK32-COUNT-5: {{%[a-zA-Z0-9]*}}: !llvm.i32,
 // CHECK32:         %[[ARG0:[a-zA-Z0-9]*]]: !llvm.i32,
 // CHECK32:         %[[ARG1:[a-zA-Z0-9]*]]: !llvm.i32,
@@ -844,42 +844,42 @@ func @subview(%0 : memref<64x4xf32, offset: 0, strides: [4, 1]>, %arg0 : index,
   // CHECK: %[[MEMREF:.*]] = llvm.insertvalue %{{.*}}, %{{.*}}[4, 1]
   // CHECK32: %[[MEMREF:.*]] = llvm.insertvalue %{{.*}}, %{{.*}}[4, 1]
 
-  // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
-  // CHECK: %[[BITCAST0:.*]] = llvm.bitcast %{{.*}} : !llvm<"float*"> to !llvm<"float*">
-  // CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BITCAST0]], %[[DESC]][0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
-  // CHECK: %[[BITCAST1:.*]] = llvm.bitcast %{{.*}} : !llvm<"float*"> to !llvm<"float*">
-  // CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[BITCAST1]], %[[DESC0]][1] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
-  // CHECK: %[[STRIDE0:.*]] = llvm.extractvalue %[[MEMREF]][4, 0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
-  // CHECK: %[[STRIDE1:.*]] = llvm.extractvalue %[[MEMREF]][4, 1] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
-  // CHECK: %[[OFF:.*]] = llvm.extractvalue %[[MEMREF]][2] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+  // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
+  // CHECK: %[[BITCAST0:.*]] = llvm.bitcast %{{.*}} : !llvm.ptr<float> to !llvm.ptr<float>
+  // CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BITCAST0]], %[[DESC]][0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
+  // CHECK: %[[BITCAST1:.*]] = llvm.bitcast %{{.*}} : !llvm.ptr<float> to !llvm.ptr<float>
+  // CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[BITCAST1]], %[[DESC0]][1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
+  // CHECK: %[[STRIDE0:.*]] = llvm.extractvalue %[[MEMREF]][4, 0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
+  // CHECK: %[[STRIDE1:.*]] = llvm.extractvalue %[[MEMREF]][4, 1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
+  // CHECK: %[[OFF:.*]] = llvm.extractvalue %[[MEMREF]][2] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: %[[OFFINC:.*]] = llvm.mul %[[ARG0]], %[[STRIDE0]] : !llvm.i64
   // CHECK: %[[OFF1:.*]] = llvm.add %[[OFF]], %[[OFFINC]] : !llvm.i64
   // CHECK: %[[OFFINC1:.*]] = llvm.mul %[[ARG1]], %[[STRIDE1]] : !llvm.i64
   // CHECK: %[[OFF2:.*]] = llvm.add %[[OFF1]], %[[OFFINC1]] : !llvm.i64
-  // CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[OFF2]], %[[DESC1]][2] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
-  // CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[ARG1]], %[[DESC2]][3, 1] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+  // CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[OFF2]], %[[DESC1]][2] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
+  // CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[ARG1]], %[[DESC2]][3, 1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: %[[DESCSTRIDE1:.*]] = llvm.mul %[[ARG1]], %[[STRIDE1]] : !llvm.i64
-  // CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[DESCSTRIDE1]], %[[DESC3]][4, 1] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
-  // CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[ARG0]], %[[DESC4]][3, 0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+  // CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[DESCSTRIDE1]], %[[DESC3]][4, 1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
+  // CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[ARG0]], %[[DESC4]][3, 0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: %[[DESCSTRIDE0:.*]] = llvm.mul %[[ARG0]], %[[STRIDE0]] : !llvm.i64
-  // CHECK: llvm.insertvalue %[[DESCSTRIDE0]], %[[DESC5]][4, 0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
-  // CHECK32: %[[DESC:.*]] = llvm.mlir.undef : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
-  // CHECK32: %[[BITCAST0:.*]] = llvm.bitcast %{{.*}} : !llvm<"float*"> to !llvm<"float*">
-  // CHECK32: %[[DESC0:.*]] = llvm.insertvalue %[[BITCAST0]], %[[DESC]][0] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
-  // CHECK32: %[[BITCAST1:.*]] = llvm.bitcast %{{.*}} : !llvm<"float*"> to !llvm<"float*">
-  // CHECK32: %[[DESC1:.*]] = llvm.insertvalue %[[BITCAST1]], %[[DESC0]][1] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
-  // CHECK32: %[[STRIDE0:.*]] = llvm.extractvalue %[[MEMREF]][4, 0] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
-  // CHECK32: %[[STRIDE1:.*]] = llvm.extractvalue %[[MEMREF]][4, 1] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
-  // CHECK32: %[[OFF:.*]] = llvm.extractvalue %[[MEMREF]][2] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK: llvm.insertvalue %[[DESCSTRIDE0]], %[[DESC5]][4, 0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
+  // CHECK32: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr<float>, ptr<float>, i32, array<2 x i32>, array<2 x i32>)>
+  // CHECK32: %[[BITCAST0:.*]] = llvm.bitcast %{{.*}} : !llvm.ptr<float> to !llvm.ptr<float>
+  // CHECK32: %[[DESC0:.*]] = llvm.insertvalue %[[BITCAST0]], %[[DESC]][0] : !llvm.struct<(ptr<float>, ptr<float>, i32, array<2 x i32>, array<2 x i32>)>
+  // CHECK32: %[[BITCAST1:.*]] = llvm.bitcast %{{.*}} : !llvm.ptr<float> to !llvm.ptr<float>
+  // CHECK32: %[[DESC1:.*]] = llvm.insertvalue %[[BITCAST1]], %[[DESC0]][1] : !llvm.struct<(ptr<float>, ptr<float>, i32, array<2 x i32>, array<2 x i32>)>
+  // CHECK32: %[[STRIDE0:.*]] = llvm.extractvalue %[[MEMREF]][4, 0] : !llvm.struct<(ptr<float>, ptr<float>, i32, array<2 x i32>, array<2 x i32>)>
+  // CHECK32: %[[STRIDE1:.*]] = llvm.extractvalue %[[MEMREF]][4, 1] : !llvm.struct<(ptr<float>, ptr<float>, i32, array<2 x i32>, array<2 x i32>)>
+  // CHECK32: %[[OFF:.*]] = llvm.extractvalue %[[MEMREF]][2] : !llvm.struct<(ptr<float>, ptr<float>, i32, array<2 x i32>, array<2 x i32>)>
   // CHECK32: %[[OFFINC:.*]] = llvm.mul %[[ARG0]], %[[STRIDE0]] : !llvm.i32
   // CHECK32: %[[OFF1:.*]] = llvm.add %[[OFF]], %[[OFFINC]] : !llvm.i32
   // CHECK32: %[[OFFINC1:.*]] = llvm.mul %[[ARG1]], %[[STRIDE1]] : !llvm.i32
   // CHECK32: %[[OFF2:.*]] = llvm.add %[[OFF1]], %[[OFFINC1]] : !llvm.i32
-  // CHECK32: %[[DESC2:.*]] = llvm.insertvalue %[[OFF2]], %[[DESC1]][2] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
-  // CHECK32: %[[DESC3:.*]] = llvm.insertvalue %[[ARG1]], %[[DESC2]][3, 1] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: %[[DESC2:.*]] = llvm.insertvalue %[[OFF2]], %[[DESC1]][2] : !llvm.struct<(ptr<float>, ptr<float>, i32, array<2 x i32>, array<2 x i32>)>
+  // CHECK32: %[[DESC3:.*]] = llvm.insertvalue %[[ARG1]], %[[DESC2]][3, 1] : !llvm.struct<(ptr<float>, ptr<float>, i32, array<2 x i32>, array<2 x i32>)>
   // CHECK32: %[[DESCSTRIDE1:.*]] = llvm.mul %[[ARG1]], %[[STRIDE1]] : !llvm.i32
-  // CHECK32: %[[DESC4:.*]] = llvm.insertvalue %[[DESCSTRIDE1]], %[[DESC3]][4, 1] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
-  // CHECK32: %[[DESC5:.*]] = llvm.insertvalue %[[ARG0]], %[[DESC4]][3, 0] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: %[[DESC4:.*]] = llvm.insertvalue %[[DESCSTRIDE1]], %[[DESC3]][4, 1] : !llvm.struct<(ptr<float>, ptr<float>, i32, array<2 x i32>, array<2 x i32>)>
+  // CHECK32: %[[DESC5:.*]] = llvm.insertvalue %[[ARG0]], %[[DESC4]][3, 0] : !llvm.struct<(ptr<float>, ptr<float>, i32, array<2 x i32>, array<2 x i32>)>
   // CHECK32: %[[DESCSTRIDE0:.*]] = llvm.mul %[[ARG0]], %[[STRIDE0]] : !llvm.i32
 
   %1 = subview %0[%arg0, %arg1][%arg0, %arg1][%arg0, %arg1] :
@@ -889,13 +889,13 @@ func @subview(%0 : memref<64x4xf32, offset: 0, strides: [4, 1]>, %arg0 : index,
 }
 
 // CHECK-LABEL: func @subview_non_zero_addrspace(
-// CHECK-COUNT-2: !llvm<"float addrspace(3)*">,
+// CHECK-COUNT-2: !llvm.ptr<float, 3>,
 // CHECK-COUNT-5: {{%[a-zA-Z0-9]*}}: !llvm.i64,
 // CHECK:         %[[ARG0:[a-zA-Z0-9]*]]: !llvm.i64,
 // CHECK:         %[[ARG1:[a-zA-Z0-9]*]]: !llvm.i64,
 // CHECK:         %[[ARG2:.*]]: !llvm.i64)
 // CHECK32-LABEL: func @subview_non_zero_addrspace(
-// CHECK32-COUNT-2: !llvm<"float addrspace(3)*">,
+// CHECK32-COUNT-2: !llvm.ptr<float, 3>,
 // CHECK32-COUNT-5: {{%[a-zA-Z0-9]*}}: !llvm.i32,
 // CHECK32:         %[[ARG0:[a-zA-Z0-9]*]]: !llvm.i32,
 // CHECK32:         %[[ARG1:[a-zA-Z0-9]*]]: !llvm.i32,
@@ -905,42 +905,42 @@ func @subview_non_zero_addrspace(%0 : memref<64x4xf32, offset: 0, strides: [4, 1
   // CHECK: %[[MEMREF:.*]] = llvm.insertvalue %{{.*}}, %{{.*}}[4, 1]
   // CHECK32: %[[MEMREF:.*]] = llvm.insertvalue %{{.*}}, %{{.*}}[4, 1]
 
-  // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm<"{ float addrspace(3)*, float addrspace(3)*, i64, [2 x i64], [2 x i64] }">
-  // CHECK: %[[BITCAST0:.*]] = llvm.bitcast %{{.*}} : !llvm<"float addrspace(3)*"> to !llvm<"float addrspace(3)*">
-  // CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BITCAST0]], %[[DESC]][0] : !llvm<"{ float addrspace(3)*, float addrspace(3)*, i64, [2 x i64], [2 x i64] }">
-  // CHECK: %[[BITCAST1:.*]] = llvm.bitcast %{{.*}} : !llvm<"float addrspace(3)*"> to !llvm<"float addrspace(3)*">
-  // CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[BITCAST1]], %[[DESC0]][1] : !llvm<"{ float addrspace(3)*, float addrspace(3)*, i64, [2 x i64], [2 x i64] }">
-  // CHECK: %[[STRIDE0:.*]] = llvm.extractvalue %[[MEMREF]][4, 0] : !llvm<"{ float addrspace(3)*, float addrspace(3)*, i64, [2 x i64], [2 x i64] }">
-  // CHECK: %[[STRIDE1:.*]] = llvm.extractvalue %[[MEMREF]][4, 1] : !llvm<"{ float addrspace(3)*, float addrspace(3)*, i64, [2 x i64], [2 x i64] }">
-  // CHECK: %[[OFF:.*]] = llvm.extractvalue %[[MEMREF]][2] : !llvm<"{ float addrspace(3)*, float addrspace(3)*, i64, [2 x i64], [2 x i64] }">
+  // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr<float, 3>, ptr<float, 3>, i64, array<2 x i64>, array<2 x i64>)>
+  // CHECK: %[[BITCAST0:.*]] = llvm.bitcast %{{.*}} : !llvm.ptr<float, 3> to !llvm.ptr<float, 3>
+  // CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BITCAST0]], %[[DESC]][0] : !llvm.struct<(ptr<float, 3>, ptr<float, 3>, i64, array<2 x i64>, array<2 x i64>)>
+  // CHECK: %[[BITCAST1:.*]] = llvm.bitcast %{{.*}} : !llvm.ptr<float, 3> to !llvm.ptr<float, 3>
+  // CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[BITCAST1]], %[[DESC0]][1] : !llvm.struct<(ptr<float, 3>, ptr<float, 3>, i64, array<2 x i64>, array<2 x i64>)>
+  // CHECK: %[[STRIDE0:.*]] = llvm.extractvalue %[[MEMREF]][4, 0] : !llvm.struct<(ptr<float, 3>, ptr<float, 3>, i64, array<2 x i64>, array<2 x i64>)>
+  // CHECK: %[[STRIDE1:.*]] = llvm.extractvalue %[[MEMREF]][4, 1] : !llvm.struct<(ptr<float, 3>, ptr<float, 3>, i64, array<2 x i64>, array<2 x i64>)>
+  // CHECK: %[[OFF:.*]] = llvm.extractvalue %[[MEMREF]][2] : !llvm.struct<(ptr<float, 3>, ptr<float, 3>, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: %[[OFFINC:.*]] = llvm.mul %[[ARG0]], %[[STRIDE0]] : !llvm.i64
   // CHECK: %[[OFF1:.*]] = llvm.add %[[OFF]], %[[OFFINC]] : !llvm.i64
   // CHECK: %[[OFFINC1:.*]] = llvm.mul %[[ARG1]], %[[STRIDE1]] : !llvm.i64
   // CHECK: %[[OFF2:.*]] = llvm.add %[[OFF1]], %[[OFFINC1]] : !llvm.i64
-  // CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[OFF2]], %[[DESC1]][2] : !llvm<"{ float addrspace(3)*, float addrspace(3)*, i64, [2 x i64], [2 x i64] }">
-  // CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[ARG1]], %[[DESC2]][3, 1] : !llvm<"{ float addrspace(3)*, float addrspace(3)*, i64, [2 x i64], [2 x i64] }">
+  // CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[OFF2]], %[[DESC1]][2] : !llvm.struct<(ptr<float, 3>, ptr<float, 3>, i64, array<2 x i64>, array<2 x i64>)>
+  // CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[ARG1]], %[[DESC2]][3, 1] : !llvm.struct<(ptr<float, 3>, ptr<float, 3>, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: %[[DESCSTRIDE1:.*]] = llvm.mul %[[ARG1]], %[[STRIDE1]] : !llvm.i64
-  // CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[DESCSTRIDE1]], %[[DESC3]][4, 1] : !llvm<"{ float addrspace(3)*, float addrspace(3)*, i64, [2 x i64], [2 x i64] }">
-  // CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[ARG0]], %[[DESC4]][3, 0] : !llvm<"{ float addrspace(3)*, float addrspace(3)*, i64, [2 x i64], [2 x i64] }">
+  // CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[DESCSTRIDE1]], %[[DESC3]][4, 1] : !llvm.struct<(ptr<float, 3>, ptr<float, 3>, i64, array<2 x i64>, array<2 x i64>)>
+  // CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[ARG0]], %[[DESC4]][3, 0] : !llvm.struct<(ptr<float, 3>, ptr<float, 3>, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: %[[DESCSTRIDE0:.*]] = llvm.mul %[[ARG0]], %[[STRIDE0]] : !llvm.i64
-  // CHECK: llvm.insertvalue %[[DESCSTRIDE0]], %[[DESC5]][4, 0] : !llvm<"{ float addrspace(3)*, float addrspace(3)*, i64, [2 x i64], [2 x i64] }">
-  // CHECK32: %[[DESC:.*]] = llvm.mlir.undef : !llvm<"{ float addrspace(3)*, float addrspace(3)*, i32, [2 x i32], [2 x i32] }">
-  // CHECK32: %[[BITCAST0:.*]] = llvm.bitcast %{{.*}} : !llvm<"float addrspace(3)*"> to !llvm<"float addrspace(3)*">
-  // CHECK32: %[[DESC0:.*]] = llvm.insertvalue %[[BITCAST0]], %[[DESC]][0] : !llvm<"{ float addrspace(3)*, float addrspace(3)*, i32, [2 x i32], [2 x i32] }">
-  // CHECK32: %[[BITCAST1:.*]] = llvm.bitcast %{{.*}} : !llvm<"float addrspace(3)*"> to !llvm<"float addrspace(3)*">
-  // CHECK32: %[[DESC1:.*]] = llvm.insertvalue %[[BITCAST1]], %[[DESC0]][1] : !llvm<"{ float addrspace(3)*, float addrspace(3)*, i32, [2 x i32], [2 x i32] }">
-  // CHECK32: %[[STRIDE0:.*]] = llvm.extractvalue %[[MEMREF]][4, 0] : !llvm<"{ float addrspace(3)*, float addrspace(3)*, i32, [2 x i32], [2 x i32] }">
-  // CHECK32: %[[STRIDE1:.*]] = llvm.extractvalue %[[MEMREF]][4, 1] : !llvm<"{ float addrspace(3)*, float addrspace(3)*, i32, [2 x i32], [2 x i32] }">
-  // CHECK32: %[[OFF:.*]] = llvm.extractvalue %[[MEMREF]][2] : !llvm<"{ float addrspace(3)*, float addrspace(3)*, i32, [2 x i32], [2 x i32] }">
+  // CHECK: llvm.insertvalue %[[DESCSTRIDE0]], %[[DESC5]][4, 0] : !llvm.struct<(ptr<float, 3>, ptr<float, 3>, i64, array<2 x i64>, array<2 x i64>)>
+  // CHECK32: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr<float, 3>, ptr<float, 3>, i32, array<2 x i32>, array<2 x i32>)>
+  // CHECK32: %[[BITCAST0:.*]] = llvm.bitcast %{{.*}} : !llvm.ptr<float, 3> to !llvm.ptr<float, 3>
+  // CHECK32: %[[DESC0:.*]] = llvm.insertvalue %[[BITCAST0]], %[[DESC]][0] : !llvm.struct<(ptr<float, 3>, ptr<float, 3>, i32, array<2 x i32>, array<2 x i32>)>
+  // CHECK32: %[[BITCAST1:.*]] = llvm.bitcast %{{.*}} : !llvm.ptr<float, 3> to !llvm.ptr<float, 3>
+  // CHECK32: %[[DESC1:.*]] = llvm.insertvalue %[[BITCAST1]], %[[DESC0]][1] : !llvm.struct<(ptr<float, 3>, ptr<float, 3>, i32, array<2 x i32>, array<2 x i32>)>
+  // CHECK32: %[[STRIDE0:.*]] = llvm.extractvalue %[[MEMREF]][4, 0] : !llvm.struct<(ptr<float, 3>, ptr<float, 3>, i32, array<2 x i32>, array<2 x i32>)>
+  // CHECK32: %[[STRIDE1:.*]] = llvm.extractvalue %[[MEMREF]][4, 1] : !llvm.struct<(ptr<float, 3>, ptr<float, 3>, i32, array<2 x i32>, array<2 x i32>)>
+  // CHECK32: %[[OFF:.*]] = llvm.extractvalue %[[MEMREF]][2] : !llvm.struct<(ptr<float, 3>, ptr<float, 3>, i32, array<2 x i32>, array<2 x i32>)>
   // CHECK32: %[[OFFINC:.*]] = llvm.mul %[[ARG0]], %[[STRIDE0]] : !llvm.i32
   // CHECK32: %[[OFF1:.*]] = llvm.add %[[OFF]], %[[OFFINC]] : !llvm.i32
   // CHECK32: %[[OFFINC1:.*]] = llvm.mul %[[ARG1]], %[[STRIDE1]] : !llvm.i32
   // CHECK32: %[[OFF2:.*]] = llvm.add %[[OFF1]], %[[OFFINC1]] : !llvm.i32
-  // CHECK32: %[[DESC2:.*]] = llvm.insertvalue %[[OFF2]], %[[DESC1]][2] : !llvm<"{ float addrspace(3)*, float addrspace(3)*, i32, [2 x i32], [2 x i32] }">
-  // CHECK32: %[[DESC3:.*]] = llvm.insertvalue %[[ARG1]], %[[DESC2]][3, 1] : !llvm<"{ float addrspace(3)*, float addrspace(3)*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: %[[DESC2:.*]] = llvm.insertvalue %[[OFF2]], %[[DESC1]][2] : !llvm.struct<(ptr<float, 3>, ptr<float, 3>, i32, array<2 x i32>, array<2 x i32>)>
+  // CHECK32: %[[DESC3:.*]] = llvm.insertvalue %[[ARG1]], %[[DESC2]][3, 1] : !llvm.struct<(ptr<float, 3>, ptr<float, 3>, i32, array<2 x i32>, array<2 x i32>)>
   // CHECK32: %[[DESCSTRIDE1:.*]] = llvm.mul %[[ARG1]], %[[STRIDE1]] : !llvm.i32
-  // CHECK32: %[[DESC4:.*]] = llvm.insertvalue %[[DESCSTRIDE1]], %[[DESC3]][4, 1] : !llvm<"{ float addrspace(3)*, float addrspace(3)*, i32, [2 x i32], [2 x i32] }">
-  // CHECK32: %[[DESC5:.*]] = llvm.insertvalue %[[ARG0]], %[[DESC4]][3, 0] : !llvm<"{ float addrspace(3)*, float addrspace(3)*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: %[[DESC4:.*]] = llvm.insertvalue %[[DESCSTRIDE1]], %[[DESC3]][4, 1] : !llvm.struct<(ptr<float, 3>, ptr<float, 3>, i32, array<2 x i32>, array<2 x i32>)>
+  // CHECK32: %[[DESC5:.*]] = llvm.insertvalue %[[ARG0]], %[[DESC4]][3, 0] : !llvm.struct<(ptr<float, 3>, ptr<float, 3>, i32, array<2 x i32>, array<2 x i32>)>
   // CHECK32: %[[DESCSTRIDE0:.*]] = llvm.mul %[[ARG0]], %[[STRIDE0]] : !llvm.i32
 
   %1 = subview %0[%arg0, %arg1][%arg0, %arg1][%arg0, %arg1] :
@@ -950,8 +950,8 @@ func @subview_non_zero_addrspace(%0 : memref<64x4xf32, offset: 0, strides: [4, 1
 }
 
 // CHECK-LABEL: func @subview_const_size(
-// CHECK-SAME:         %[[ARG0:[a-zA-Z0-9]*]]: !llvm<"float*">,
-// CHECK-SAME:         %[[ARG1:[a-zA-Z0-9]*]]: !llvm<"float*">,
+// CHECK-SAME:         %[[ARG0:[a-zA-Z0-9]*]]: !llvm.ptr<float>,
+// CHECK-SAME:         %[[ARG1:[a-zA-Z0-9]*]]: !llvm.ptr<float>,
 // CHECK-SAME:         %[[ARG2:[a-zA-Z0-9]*]]: !llvm.i64
 // CHECK-SAME:         %[[ARG3:[a-zA-Z0-9]*]]: !llvm.i64
 // CHECK-SAME:         %[[ARG4:[a-zA-Z0-9]*]]: !llvm.i64
@@ -961,8 +961,8 @@ func @subview_non_zero_addrspace(%0 : memref<64x4xf32, offset: 0, strides: [4, 1
 // CHECK-SAME:         %[[ARG8:[a-zA-Z0-9]*]]: !llvm.i64
 // CHECK-SAME:         %[[ARG9:[a-zA-Z0-9]*]]: !llvm.i64
 // CHECK32-LABEL: func @subview_const_size(
-// CHECK32-SAME:         %[[ARG0:[a-zA-Z0-9]*]]: !llvm<"float*">,
-// CHECK32-SAME:         %[[ARG1:[a-zA-Z0-9]*]]: !llvm<"float*">,
+// CHECK32-SAME:         %[[ARG0:[a-zA-Z0-9]*]]: !llvm.ptr<float>,
+// CHECK32-SAME:         %[[ARG1:[a-zA-Z0-9]*]]: !llvm.ptr<float>,
 // CHECK32-SAME:         %[[ARG2:[a-zA-Z0-9]*]]: !llvm.i32
 // CHECK32-SAME:         %[[ARG3:[a-zA-Z0-9]*]]: !llvm.i32
 // CHECK32-SAME:         %[[ARG4:[a-zA-Z0-9]*]]: !llvm.i32
@@ -976,48 +976,48 @@ func @subview_const_size(%0 : memref<64x4xf32, offset: 0, strides: [4, 1]>, %arg
   // CHECK: %[[MEMREF:.*]] = llvm.insertvalue %{{.*}}, %{{.*}}[4, 1]
   // CHECK32: %[[MEMREF:.*]] = llvm.insertvalue %{{.*}}, %{{.*}}[4, 1]
 
-  // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
-  // CHECK: %[[BITCAST0:.*]] = llvm.bitcast %{{.*}} : !llvm<"float*"> to !llvm<"float*">
-  // CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BITCAST0]], %[[DESC]][0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
-  // CHECK: %[[BITCAST1:.*]] = llvm.bitcast %{{.*}} : !llvm<"float*"> to !llvm<"float*">
-  // CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[BITCAST1]], %[[DESC0]][1] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
-  // CHECK: %[[STRIDE0:.*]] = llvm.extractvalue %[[MEMREF]][4, 0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
-  // CHECK: %[[STRIDE1:.*]] = llvm.extractvalue %[[MEMREF]][4, 1] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
-  // CHECK: %[[OFF:.*]] = llvm.extractvalue %[[MEMREF]][2] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+  // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
+  // CHECK: %[[BITCAST0:.*]] = llvm.bitcast %{{.*}} : !llvm.ptr<float> to !llvm.ptr<float>
+  // CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BITCAST0]], %[[DESC]][0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
+  // CHECK: %[[BITCAST1:.*]] = llvm.bitcast %{{.*}} : !llvm.ptr<float> to !llvm.ptr<float>
+  // CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[BITCAST1]], %[[DESC0]][1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
+  // CHECK: %[[STRIDE0:.*]] = llvm.extractvalue %[[MEMREF]][4, 0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
+  // CHECK: %[[STRIDE1:.*]] = llvm.extractvalue %[[MEMREF]][4, 1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
+  // CHECK: %[[OFF:.*]] = llvm.extractvalue %[[MEMREF]][2] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: %[[OFFINC:.*]] = llvm.mul %[[ARG7]], %[[STRIDE0]] : !llvm.i64
   // CHECK: %[[OFF1:.*]] = llvm.add %[[OFF]], %[[OFFINC]] : !llvm.i64
   // CHECK: %[[OFFINC1:.*]] = llvm.mul %[[ARG8]], %[[STRIDE1]] : !llvm.i64
   // CHECK: %[[OFF2:.*]] = llvm.add %[[OFF1]], %[[OFFINC1]] : !llvm.i64
-  // CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[OFF2]], %[[DESC1]][2] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+  // CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[OFF2]], %[[DESC1]][2] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: %[[CST2:.*]] = llvm.mlir.constant(2 : i64)
-  // CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[CST2]], %[[DESC2]][3, 1] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+  // CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[CST2]], %[[DESC2]][3, 1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: %[[DESCSTRIDE1:.*]] = llvm.mul %[[ARG8]], %[[STRIDE1]] : !llvm.i64
-  // CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[DESCSTRIDE1]], %[[DESC3]][4, 1] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+  // CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[DESCSTRIDE1]], %[[DESC3]][4, 1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: %[[CST4:.*]] = llvm.mlir.constant(4 : i64)
-  // CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[CST4]], %[[DESC4]][3, 0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+  // CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[CST4]], %[[DESC4]][3, 0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: %[[DESCSTRIDE0:.*]] = llvm.mul %[[ARG7]], %[[STRIDE0]] : !llvm.i64
-  // CHECK: llvm.insertvalue %[[DESCSTRIDE0]], %[[DESC5]][4, 0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
-  // CHECK32: %[[DESC:.*]] = llvm.mlir.undef : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
-  // CHECK32: %[[BITCAST0:.*]] = llvm.bitcast %{{.*}} : !llvm<"float*"> to !llvm<"float*">
-  // CHECK32: %[[DESC0:.*]] = llvm.insertvalue %[[BITCAST0]], %[[DESC]][0] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
-  // CHECK32: %[[BITCAST1:.*]] = llvm.bitcast %{{.*}} : !llvm<"float*"> to !llvm<"float*">
-  // CHECK32: %[[DESC1:.*]] = llvm.insertvalue %[[BITCAST1]], %[[DESC0]][1] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
-  // CHECK32: %[[STRIDE0:.*]] = llvm.extractvalue %[[MEMREF]][4, 0] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
-  // CHECK32: %[[STRIDE1:.*]] = llvm.extractvalue %[[MEMREF]][4, 1] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
-  // CHECK32: %[[OFF:.*]] = llvm.extractvalue %[[MEMREF]][2] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK: llvm.insertvalue %[[DESCSTRIDE0]], %[[DESC5]][4, 0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
+  // CHECK32: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr<float>, ptr<float>, i32, array<2 x i32>, array<2 x i32>)>
+  // CHECK32: %[[BITCAST0:.*]] = llvm.bitcast %{{.*}} : !llvm.ptr<float> to !llvm.ptr<float>
+  // CHECK32: %[[DESC0:.*]] = llvm.insertvalue %[[BITCAST0]], %[[DESC]][0] : !llvm.struct<(ptr<float>, ptr<float>, i32, array<2 x i32>, array<2 x i32>)>
+  // CHECK32: %[[BITCAST1:.*]] = llvm.bitcast %{{.*}} : !llvm.ptr<float> to !llvm.ptr<float>
+  // CHECK32: %[[DESC1:.*]] = llvm.insertvalue %[[BITCAST1]], %[[DESC0]][1] : !llvm.struct<(ptr<float>, ptr<float>, i32, array<2 x i32>, array<2 x i32>)>
+  // CHECK32: %[[STRIDE0:.*]] = llvm.extractvalue %[[MEMREF]][4, 0] : !llvm.struct<(ptr<float>, ptr<float>, i32, array<2 x i32>, array<2 x i32>)>
+  // CHECK32: %[[STRIDE1:.*]] = llvm.extractvalue %[[MEMREF]][4, 1] : !llvm.struct<(ptr<float>, ptr<float>, i32, array<2 x i32>, array<2 x i32>)>
+  // CHECK32: %[[OFF:.*]] = llvm.extractvalue %[[MEMREF]][2] : !llvm.struct<(ptr<float>, ptr<float>, i32, array<2 x i32>, array<2 x i32>)>
   // CHECK32: %[[OFFINC:.*]] = llvm.mul %[[ARG7]], %[[STRIDE0]] : !llvm.i32
   // CHECK32: %[[OFF1:.*]] = llvm.add %[[OFF]], %[[OFFINC]] : !llvm.i32
   // CHECK32: %[[OFFINC1:.*]] = llvm.mul %[[ARG8]], %[[STRIDE1]] : !llvm.i32
   // CHECK32: %[[OFF2:.*]] = llvm.add %[[OFF1]], %[[OFFINC1]] : !llvm.i32
-  // CHECK32: %[[DESC2:.*]] = llvm.insertvalue %[[OFF2]], %[[DESC1]][2] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: %[[DESC2:.*]] = llvm.insertvalue %[[OFF2]], %[[DESC1]][2] : !llvm.struct<(ptr<float>, ptr<float>, i32, array<2 x i32>, array<2 x i32>)>
   // CHECK32: %[[CST2:.*]] = llvm.mlir.constant(2 : i64)
-  // CHECK32: %[[DESC3:.*]] = llvm.insertvalue %[[CST2]], %[[DESC2]][3, 1] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: %[[DESC3:.*]] = llvm.insertvalue %[[CST2]], %[[DESC2]][3, 1] : !llvm.struct<(ptr<float>, ptr<float>, i32, array<2 x i32>, array<2 x i32>)>
   // CHECK32: %[[DESCSTRIDE1:.*]] = llvm.mul %[[ARG8]], %[[STRIDE1]] : !llvm.i32
-  // CHECK32: %[[DESC4:.*]] = llvm.insertvalue %[[DESCSTRIDE1]], %[[DESC3]][4, 1] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: %[[DESC4:.*]] = llvm.insertvalue %[[DESCSTRIDE1]], %[[DESC3]][4, 1] : !llvm.struct<(ptr<float>, ptr<float>, i32, array<2 x i32>, array<2 x i32>)>
   // CHECK32: %[[CST4:.*]] = llvm.mlir.constant(4 : i64)
-  // CHECK32: %[[DESC5:.*]] = llvm.insertvalue %[[CST4]], %[[DESC4]][3, 0] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: %[[DESC5:.*]] = llvm.insertvalue %[[CST4]], %[[DESC4]][3, 0] : !llvm.struct<(ptr<float>, ptr<float>, i32, array<2 x i32>, array<2 x i32>)>
   // CHECK32: %[[DESCSTRIDE0:.*]] = llvm.mul %[[ARG7]], %[[STRIDE0]] : !llvm.i32
-  // CHECK32: llvm.insertvalue %[[DESCSTRIDE0]], %[[DESC5]][4, 0] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: llvm.insertvalue %[[DESCSTRIDE0]], %[[DESC5]][4, 0] : !llvm.struct<(ptr<float>, ptr<float>, i32, array<2 x i32>, array<2 x i32>)>
   %1 = subview %0[%arg0, %arg1][4, 2][%arg0, %arg1] :
     memref<64x4xf32, offset: 0, strides: [4, 1]>
     to memref<4x2xf32, offset: ?, strides: [?, ?]>
@@ -1025,8 +1025,8 @@ func @subview_const_size(%0 : memref<64x4xf32, offset: 0, strides: [4, 1]>, %arg
 }
 
 // CHECK-LABEL: func @subview_const_stride(
-// CHECK-SAME:         %[[ARG0:[a-zA-Z0-9]*]]: !llvm<"float*">,
-// CHECK-SAME:         %[[ARG1:[a-zA-Z0-9]*]]: !llvm<"float*">,
+// CHECK-SAME:         %[[ARG0:[a-zA-Z0-9]*]]: !llvm.ptr<float>,
+// CHECK-SAME:         %[[ARG1:[a-zA-Z0-9]*]]: !llvm.ptr<float>,
 // CHECK-SAME:         %[[ARG2:[a-zA-Z0-9]*]]: !llvm.i64
 // CHECK-SAME:         %[[ARG3:[a-zA-Z0-9]*]]: !llvm.i64
 // CHECK-SAME:         %[[ARG4:[a-zA-Z0-9]*]]: !llvm.i64
@@ -1036,8 +1036,8 @@ func @subview_const_size(%0 : memref<64x4xf32, offset: 0, strides: [4, 1]>, %arg
 // CHECK-SAME:         %[[ARG8:[a-zA-Z0-9]*]]: !llvm.i64
 // CHECK-SAME:         %[[ARG9:[a-zA-Z0-9]*]]: !llvm.i64
 // CHECK32-LABEL: func @subview_const_stride(
-// CHECK32-SAME:         %[[ARG0:[a-zA-Z0-9]*]]: !llvm<"float*">,
-// CHECK32-SAME:         %[[ARG1:[a-zA-Z0-9]*]]: !llvm<"float*">,
+// CHECK32-SAME:         %[[ARG0:[a-zA-Z0-9]*]]: !llvm.ptr<float>,
+// CHECK32-SAME:         %[[ARG1:[a-zA-Z0-9]*]]: !llvm.ptr<float>,
 // CHECK32-SAME:         %[[ARG2:[a-zA-Z0-9]*]]: !llvm.i32
 // CHECK32-SAME:         %[[ARG3:[a-zA-Z0-9]*]]: !llvm.i32
 // CHECK32-SAME:         %[[ARG4:[a-zA-Z0-9]*]]: !llvm.i32
@@ -1051,44 +1051,44 @@ func @subview_const_stride(%0 : memref<64x4xf32, offset: 0, strides: [4, 1]>, %a
   // CHECK: %[[MEMREF:.*]] = llvm.insertvalue %{{.*}}, %{{.*}}[4, 1]
   // CHECK32: %[[MEMREF:.*]] = llvm.insertvalue %{{.*}}, %{{.*}}[4, 1]
 
-  // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
-  // CHECK: %[[BITCAST0:.*]] = llvm.bitcast %{{.*}} : !llvm<"float*"> to !llvm<"float*">
-  // CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BITCAST0]], %[[DESC]][0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
-  // CHECK: %[[BITCAST1:.*]] = llvm.bitcast %{{.*}} : !llvm<"float*"> to !llvm<"float*">
-  // CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[BITCAST1]], %[[DESC0]][1] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
-  // CHECK: %[[STRIDE0:.*]] = llvm.extractvalue %[[MEMREF]][4, 0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
-  // CHECK: %[[STRIDE1:.*]] = llvm.extractvalue %[[MEMREF]][4, 1] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
-  // CHECK: %[[OFF:.*]] = llvm.extractvalue %[[MEMREF]][2] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+  // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
+  // CHECK: %[[BITCAST0:.*]] = llvm.bitcast %{{.*}} : !llvm.ptr<float> to !llvm.ptr<float>
+  // CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BITCAST0]], %[[DESC]][0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
+  // CHECK: %[[BITCAST1:.*]] = llvm.bitcast %{{.*}} : !llvm.ptr<float> to !llvm.ptr<float>
+  // CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[BITCAST1]], %[[DESC0]][1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
+  // CHECK: %[[STRIDE0:.*]] = llvm.extractvalue %[[MEMREF]][4, 0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
+  // CHECK: %[[STRIDE1:.*]] = llvm.extractvalue %[[MEMREF]][4, 1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
+  // CHECK: %[[OFF:.*]] = llvm.extractvalue %[[MEMREF]][2] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: %[[OFFINC:.*]] = llvm.mul %[[ARG7]], %[[STRIDE0]] : !llvm.i64
   // CHECK: %[[OFF1:.*]] = llvm.add %[[OFF]], %[[OFFINC]] : !llvm.i64
   // CHECK: %[[OFFINC1:.*]] = llvm.mul %[[ARG8]], %[[STRIDE1]] : !llvm.i64
   // CHECK: %[[OFF2:.*]] = llvm.add %[[OFF1]], %[[OFFINC1]] : !llvm.i64
-  // CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[OFF2]], %[[DESC1]][2] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
-  // CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[ARG8]], %[[DESC2]][3, 1] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+  // CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[OFF2]], %[[DESC1]][2] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
+  // CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[ARG8]], %[[DESC2]][3, 1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: %[[CST2:.*]] = llvm.mlir.constant(2 : i64)
-  // CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[CST2]], %[[DESC3]][4, 1] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
-  // CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[ARG7]], %[[DESC4]][3, 0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+  // CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[CST2]], %[[DESC3]][4, 1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
+  // CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[ARG7]], %[[DESC4]][3, 0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK: %[[CST4:.*]] = llvm.mlir.constant(4 : i64)
-  // CHECK: llvm.insertvalue %[[CST4]], %[[DESC5]][4, 0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
-  // CHECK32: %[[DESC:.*]] = llvm.mlir.undef : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
-  // CHECK32: %[[BITCAST0:.*]] = llvm.bitcast %{{.*}} : !llvm<"float*"> to !llvm<"float*">
-  // CHECK32: %[[DESC0:.*]] = llvm.insertvalue %[[BITCAST0]], %[[DESC]][0] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
-  // CHECK32: %[[BITCAST1:.*]] = llvm.bitcast %{{.*}} : !llvm<"float*"> to !llvm<"float*">
-  // CHECK32: %[[DESC1:.*]] = llvm.insertvalue %[[BITCAST1]], %[[DESC0]][1] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
-  // CHECK32: %[[STRIDE0:.*]] = llvm.extractvalue %[[MEMREF]][4, 0] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
-  // CHECK32: %[[STRIDE1:.*]] = llvm.extractvalue %[[MEMREF]][4, 1] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
-  // CHECK32: %[[OFF:.*]] = llvm.extractvalue %[[MEMREF]][2] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK: llvm.insertvalue %[[CST4]], %[[DESC5]][4, 0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
+  // CHECK32: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr<float>, ptr<float>, i32, array<2 x i32>, array<2 x i32>)>
+  // CHECK32: %[[BITCAST0:.*]] = llvm.bitcast %{{.*}} : !llvm.ptr<float> to !llvm.ptr<float>
+  // CHECK32: %[[DESC0:.*]] = llvm.insertvalue %[[BITCAST0]], %[[DESC]][0] : !llvm.struct<(ptr<float>, ptr<float>, i32, array<2 x i32>, array<2 x i32>)>
+  // CHECK32: %[[BITCAST1:.*]] = llvm.bitcast %{{.*}} : !llvm.ptr<float> to !llvm.ptr<float>
+  // CHECK32: %[[DESC1:.*]] = llvm.insertvalue %[[BITCAST1]], %[[DESC0]][1] : !llvm.struct<(ptr<float>, ptr<float>, i32, array<2 x i32>, array<2 x i32>)>
+  // CHECK32: %[[STRIDE0:.*]] = llvm.extractvalue %[[MEMREF]][4, 0] : !llvm.struct<(ptr<float>, ptr<float>, i32, array<2 x i32>, array<2 x i32>)>
+  // CHECK32: %[[STRIDE1:.*]] = llvm.extractvalue %[[MEMREF]][4, 1] : !llvm.struct<(ptr<float>, ptr<float>, i32, array<2 x i32>, array<2 x i32>)>
+  // CHECK32: %[[OFF:.*]] = llvm.extractvalue %[[MEMREF]][2] : !llvm.struct<(ptr<float>, ptr<float>, i32, array<2 x i32>, array<2 x i32>)>
   // CHECK32: %[[OFFINC:.*]] = llvm.mul %[[ARG7]], %[[STRIDE0]] : !llvm.i32
   // CHECK32: %[[OFF1:.*]] = llvm.add %[[OFF]], %[[OFFINC]] : !llvm.i32
   // CHECK32: %[[OFFINC1:.*]] = llvm.mul %[[ARG8]], %[[STRIDE1]] : !llvm.i32
   // CHECK32: %[[OFF2:.*]] = llvm.add %[[OFF1]], %[[OFFINC1]] : !llvm.i32
-  // CHECK32: %[[DESC2:.*]] = llvm.insertvalue %[[OFF2]], %[[DESC1]][2] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
-  // CHECK32: %[[DESC3:.*]] = llvm.insertvalue %[[ARG8]], %[[DESC2]][3, 1] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: %[[DESC2:.*]] = llvm.insertvalue %[[OFF2]], %[[DESC1]][2] : !llvm.struct<(ptr<float>, ptr<float>, i32, array<2 x i32>, array<2 x i32>)>
+  // CHECK32: %[[DESC3:.*]] = llvm.insertvalue %[[ARG8]], %[[DESC2]][3, 1] : !llvm.struct<(ptr<float>, ptr<float>, i32, array<2 x i32>, array<2 x i32>)>
   // CHECK32: %[[CST2:.*]] = llvm.mlir.constant(2 : i64)
-  // CHECK32: %[[DESC4:.*]] = llvm.insertvalue %[[CST2]], %[[DESC3]][4, 1] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
-  // CHECK32: %[[DESC5:.*]] = llvm.insertvalue %[[ARG7]], %[[DESC4]][3, 0] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: %[[DESC4:.*]] = llvm.insertvalue %[[CST2]], %[[DESC3]][4, 1] : !llvm.struct<(ptr<float>, ptr<float>, i32, array<2 x i32>, array<2 x i32>)>
+  // CHECK32: %[[DESC5:.*]] = llvm.insertvalue %[[ARG7]], %[[DESC4]][3, 0] : !llvm.struct<(ptr<float>, ptr<float>, i32, array<2 x i32>, array<2 x i32>)>
   // CHECK32: %[[CST4:.*]] = llvm.mlir.constant(4 : i64)
-  // CHECK32: llvm.insertvalue %[[CST4]], %[[DESC5]][4, 0] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: llvm.insertvalue %[[CST4]], %[[DESC5]][4, 0] : !llvm.struct<(ptr<float>, ptr<float>, i32, array<2 x i32>, array<2 x i32>)>
   %1 = subview %0[%arg0, %arg1][%arg0, %arg1][1, 2] :
     memref<64x4xf32, offset: 0, strides: [4, 1]>
     to memref<?x?xf32, offset: ?, strides: [4, 2]>
@@ -1102,23 +1102,23 @@ func @subview_const_stride_and_offset(%0 : memref<64x4xf32, offset: 0, strides:
   // CHECK: %[[MEMREF:.*]] = llvm.insertvalue %{{.*}}, %{{.*}}[4, 1]
   // CHECK32: %[[MEMREF:.*]] = llvm.insertvalue %{{.*}}, %{{.*}}[4, 1]
 
-  // CHECK32: %[[DESC:.*]] = llvm.mlir.undef : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
-  // CHECK32: %[[BITCAST0:.*]] = llvm.bitcast %{{.*}} : !llvm<"float*"> to !llvm<"float*">
-  // CHECK32: %[[DESC0:.*]] = llvm.insertvalue %[[BITCAST0]], %[[DESC]][0] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
-  // CHECK32: %[[BITCAST1:.*]] = llvm.bitcast %{{.*}} : !llvm<"float*"> to !llvm<"float*">
-  // CHECK32: %[[DESC1:.*]] = llvm.insertvalue %[[BITCAST1]], %[[DESC0]][1] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
-  // CHECK32: %[[STRIDE0:.*]] = llvm.extractvalue %[[MEMREF]][4, 0] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
-  // CHECK32: %[[STRIDE1:.*]] = llvm.extractvalue %[[MEMREF]][4, 1] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr<float>, ptr<float>, i32, array<2 x i32>, array<2 x i32>)>
+  // CHECK32: %[[BITCAST0:.*]] = llvm.bitcast %{{.*}} : !llvm.ptr<float> to !llvm.ptr<float>
+  // CHECK32: %[[DESC0:.*]] = llvm.insertvalue %[[BITCAST0]], %[[DESC]][0] : !llvm.struct<(ptr<float>, ptr<float>, i32, array<2 x i32>, array<2 x i32>)>
+  // CHECK32: %[[BITCAST1:.*]] = llvm.bitcast %{{.*}} : !llvm.ptr<float> to !llvm.ptr<float>
+  // CHECK32: %[[DESC1:.*]] = llvm.insertvalue %[[BITCAST1]], %[[DESC0]][1] : !llvm.struct<(ptr<float>, ptr<float>, i32, array<2 x i32>, array<2 x i32>)>
+  // CHECK32: %[[STRIDE0:.*]] = llvm.extractvalue %[[MEMREF]][4, 0] : !llvm.struct<(ptr<float>, ptr<float>, i32, array<2 x i32>, array<2 x i32>)>
+  // CHECK32: %[[STRIDE1:.*]] = llvm.extractvalue %[[MEMREF]][4, 1] : !llvm.struct<(ptr<float>, ptr<float>, i32, array<2 x i32>, array<2 x i32>)>
   // CHECK32: %[[CST8:.*]] = llvm.mlir.constant(8 : index)
-  // CHECK32: %[[DESC2:.*]] = llvm.insertvalue %[[CST8]], %[[DESC1]][2] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: %[[DESC2:.*]] = llvm.insertvalue %[[CST8]], %[[DESC1]][2] : !llvm.struct<(ptr<float>, ptr<float>, i32, array<2 x i32>, array<2 x i32>)>
   // CHECK32: %[[CST3:.*]] = llvm.mlir.constant(3 : i64)
-  // CHECK32: %[[DESC3:.*]] = llvm.insertvalue %[[CST3]], %[[DESC2]][3, 1] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: %[[DESC3:.*]] = llvm.insertvalue %[[CST3]], %[[DESC2]][3, 1] : !llvm.struct<(ptr<float>, ptr<float>, i32, array<2 x i32>, array<2 x i32>)>
   // CHECK32: %[[CST1:.*]] = llvm.mlir.constant(1 : i64)
-  // CHECK32: %[[DESC4:.*]] = llvm.insertvalue %[[CST1]], %[[DESC3]][4, 1] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: %[[DESC4:.*]] = llvm.insertvalue %[[CST1]], %[[DESC3]][4, 1] : !llvm.struct<(ptr<float>, ptr<float>, i32, array<2 x i32>, array<2 x i32>)>
   // CHECK32: %[[CST62:.*]] = llvm.mlir.constant(62 : i64)
-  // CHECK32: %[[DESC5:.*]] = llvm.insertvalue %[[CST62]], %[[DESC4]][3, 0] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: %[[DESC5:.*]] = llvm.insertvalue %[[CST62]], %[[DESC4]][3, 0] : !llvm.struct<(ptr<float>, ptr<float>, i32, array<2 x i32>, array<2 x i32>)>
   // CHECK32: %[[CST4:.*]] = llvm.mlir.constant(4 : i64)
-  // CHECK32: llvm.insertvalue %[[CST4]], %[[DESC5]][4, 0] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: llvm.insertvalue %[[CST4]], %[[DESC5]][4, 0] : !llvm.struct<(ptr<float>, ptr<float>, i32, array<2 x i32>, array<2 x i32>)>
   %1 = subview %0[0, 8][62, 3][1, 1] :
     memref<64x4xf32, offset: 0, strides: [4, 1]>
     to memref<62x3xf32, offset: 8, strides: [4, 1]>
@@ -1126,13 +1126,13 @@ func @subview_const_stride_and_offset(%0 : memref<64x4xf32, offset: 0, strides:
 }
 
 // CHECK-LABEL: func @subview_mixed_static_dynamic(
-// CHECK-COUNT-2: !llvm<"float*">,
+// CHECK-COUNT-2: !llvm.ptr<float>,
 // CHECK-COUNT-5: {{%[a-zA-Z0-9]*}}: !llvm.i64,
 // CHECK:         %[[ARG0:[a-zA-Z0-9]*]]: !llvm.i64,
 // CHECK:         %[[ARG1:[a-zA-Z0-9]*]]: !llvm.i64,
 // CHECK:         %[[ARG2:.*]]: !llvm.i64)
 // CHECK32-LABEL: func @subview_mixed_static_dynamic(
-// CHECK32-COUNT-2: !llvm<"float*">,
+// CHECK32-COUNT-2: !llvm.ptr<float>,
 // CHECK32-COUNT-5: {{%[a-zA-Z0-9]*}}: !llvm.i32,
 // CHECK32:         %[[ARG0:[a-zA-Z0-9]*]]: !llvm.i32,
 // CHECK32:         %[[ARG1:[a-zA-Z0-9]*]]: !llvm.i32,
@@ -1142,28 +1142,28 @@ func @subview_mixed_static_dynamic(%0 : memref<64x4xf32, offset: 0, strides: [4,
   // CHECK: %[[MEMREF:.*]] = llvm.insertvalue %{{.*}}, %{{.*}}[4, 1]
   // CHECK32: %[[MEMREF:.*]] = llvm.insertvalue %{{.*}}, %{{.*}}[4, 1]
 
-  // CHECK32: %[[DESC:.*]] = llvm.mlir.undef : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
-  // CHECK32: %[[BITCAST0:.*]] = llvm.bitcast %{{.*}} : !llvm<"float*"> to !llvm<"float*">
-  // CHECK32: %[[DESC0:.*]] = llvm.insertvalue %[[BITCAST0]], %[[DESC]][0] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
-  // CHECK32: %[[BITCAST1:.*]] = llvm.bitcast %{{.*}} : !llvm<"float*"> to !llvm<"float*">
-  // CHECK32: %[[DESC1:.*]] = llvm.insertvalue %[[BITCAST1]], %[[DESC0]][1] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
-  // CHECK32: %[[STRIDE0:.*]] = llvm.extractvalue %[[MEMREF]][4, 0] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
-  // CHECK32: %[[STRIDE1:.*]] = llvm.extractvalue %[[MEMREF]][4, 1] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
-  // CHECK32: %[[OFF:.*]] = llvm.extractvalue %[[MEMREF]][2] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr<float>, ptr<float>, i32, array<2 x i32>, array<2 x i32>)>
+  // CHECK32: %[[BITCAST0:.*]] = llvm.bitcast %{{.*}} : !llvm.ptr<float> to !llvm.ptr<float>
+  // CHECK32: %[[DESC0:.*]] = llvm.insertvalue %[[BITCAST0]], %[[DESC]][0] : !llvm.struct<(ptr<float>, ptr<float>, i32, array<2 x i32>, array<2 x i32>)>
+  // CHECK32: %[[BITCAST1:.*]] = llvm.bitcast %{{.*}} : !llvm.ptr<float> to !llvm.ptr<float>
+  // CHECK32: %[[DESC1:.*]] = llvm.insertvalue %[[BITCAST1]], %[[DESC0]][1] : !llvm.struct<(ptr<float>, ptr<float>, i32, array<2 x i32>, array<2 x i32>)>
+  // CHECK32: %[[STRIDE0:.*]] = llvm.extractvalue %[[MEMREF]][4, 0] : !llvm.struct<(ptr<float>, ptr<float>, i32, array<2 x i32>, array<2 x i32>)>
+  // CHECK32: %[[STRIDE1:.*]] = llvm.extractvalue %[[MEMREF]][4, 1] : !llvm.struct<(ptr<float>, ptr<float>, i32, array<2 x i32>, array<2 x i32>)>
+  // CHECK32: %[[OFF:.*]] = llvm.extractvalue %[[MEMREF]][2] : !llvm.struct<(ptr<float>, ptr<float>, i32, array<2 x i32>, array<2 x i32>)>
   // CHECK32: %[[OFFM1:.*]] = llvm.mul %[[ARG1]], %[[STRIDE0]] : !llvm.i32
   // CHECK32: %[[OFFA1:.*]] = llvm.add %[[OFF]], %[[OFFM1]] : !llvm.i32
   // CHECK32: %[[CST8:.*]] = llvm.mlir.constant(8 : i64) : !llvm.i32
   // CHECK32: %[[OFFM2:.*]] = llvm.mul %[[CST8]], %[[STRIDE1]] : !llvm.i32
   // CHECK32: %[[OFFA2:.*]] = llvm.add %[[OFFA1]], %[[OFFM2]] : !llvm.i32
-  // CHECK32: %[[DESC2:.*]] = llvm.insertvalue %[[OFFA2]], %[[DESC1]][2] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: %[[DESC2:.*]] = llvm.insertvalue %[[OFFA2]], %[[DESC1]][2] : !llvm.struct<(ptr<float>, ptr<float>, i32, array<2 x i32>, array<2 x i32>)>
 
-  // CHECK32: %[[DESC3:.*]] = llvm.insertvalue %[[ARG2]], %[[DESC2]][3, 1] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: %[[DESC3:.*]] = llvm.insertvalue %[[ARG2]], %[[DESC2]][3, 1] : !llvm.struct<(ptr<float>, ptr<float>, i32, array<2 x i32>, array<2 x i32>)>
   // CHECK32: %[[CST1:.*]] = llvm.mlir.constant(1 : i64) : !llvm.i32
-  // CHECK32: %[[DESC4:.*]] = llvm.insertvalue %[[CST1]], %[[DESC3]][4, 1] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: %[[DESC4:.*]] = llvm.insertvalue %[[CST1]], %[[DESC3]][4, 1] : !llvm.struct<(ptr<float>, ptr<float>, i32, array<2 x i32>, array<2 x i32>)>
   // CHECK32: %[[CST62:.*]] = llvm.mlir.constant(62 : i64) : !llvm.i32
-  // CHECK32: %[[DESC5:.*]] = llvm.insertvalue %[[CST62]], %[[DESC4]][3, 0] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: %[[DESC5:.*]] = llvm.insertvalue %[[CST62]], %[[DESC4]][3, 0] : !llvm.struct<(ptr<float>, ptr<float>, i32, array<2 x i32>, array<2 x i32>)>
   // CHECK32: %[[DESCSTRIDE0:.*]] = llvm.mul %[[ARG0]], %[[STRIDE0]] : !llvm.i32
-  // CHECK32: llvm.insertvalue %[[DESCSTRIDE0]], %[[DESC5]][4, 0] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: llvm.insertvalue %[[DESCSTRIDE0]], %[[DESC5]][4, 0] : !llvm.struct<(ptr<float>, ptr<float>, i32, array<2 x i32>, array<2 x i32>)>
   %1 = subview %0[%arg1, 8][62, %arg2][%arg0, 1] :
     memref<64x4xf32, offset: 0, strides: [4, 1]>
     to memref<62x?xf32, offset: ?, strides: [?, 1]>
@@ -1200,7 +1200,7 @@ func @generic_atomic_rmw(%I : memref<10xf32>, %i : index) -> f32 {
       %c1 = constant 1.0 : f32
       atomic_yield %c1 : f32
   }
-  // CHECK: [[init:%.*]] = llvm.load %{{.*}} : !llvm<"float*">
+  // CHECK: [[init:%.*]] = llvm.load %{{.*}} : !llvm.ptr<float>
   // CHECK-NEXT: llvm.br ^bb1([[init]] : !llvm.float)
   // CHECK-NEXT: ^bb1([[loaded:%.*]]: !llvm.float):
   // CHECK-NEXT: [[c1:%.*]] = llvm.mlir.constant(1.000000e+00 : f32)
@@ -1222,10 +1222,10 @@ func @generic_atomic_rmw(%I : memref<10xf32>, %i : index) -> f32 {
 
 // CHECK-LABEL: func @assume_alignment
 func @assume_alignment(%0 : memref<4x4xf16>) {
-  // CHECK: %[[PTR:.*]] = llvm.extractvalue %[[MEMREF:.*]][1] : !llvm<"{ half*, half*, i64, [2 x i64], [2 x i64] }">
+  // CHECK: %[[PTR:.*]] = llvm.extractvalue %[[MEMREF:.*]][1] : !llvm.struct<(ptr<half>, ptr<half>, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK-NEXT: %[[ZERO:.*]] = llvm.mlir.constant(0 : index) : !llvm.i64
   // CHECK-NEXT: %[[MASK:.*]] = llvm.mlir.constant(15 : index) : !llvm.i64
-  // CHECK-NEXT: %[[INT:.*]] = llvm.ptrtoint %[[PTR]] : !llvm<"half*"> to !llvm.i64
+  // CHECK-NEXT: %[[INT:.*]] = llvm.ptrtoint %[[PTR]] : !llvm.ptr<half> to !llvm.i64
   // CHECK-NEXT: %[[MASKED_PTR:.*]] = llvm.and %[[INT]], %[[MASK:.*]] : !llvm.i64
   // CHECK-NEXT: %[[CONDITION:.*]] = llvm.icmp "eq" %[[MASKED_PTR]], %[[ZERO]] : !llvm.i64
   // CHECK-NEXT: "llvm.intr.assume"(%[[CONDITION]]) : (!llvm.i1) -> ()
@@ -1237,16 +1237,16 @@ func @assume_alignment(%0 : memref<4x4xf16>) {
 
 // CHECK-LABEL: func @mlir_cast_to_llvm
 // CHECK-SAME: %[[ARG:.*]]:
-func @mlir_cast_to_llvm(%0 : vector<2xf16>) -> !llvm<"<2 x half>"> {
-  %1 = llvm.mlir.cast %0 : vector<2xf16> to !llvm<"<2 x half>">
+func @mlir_cast_to_llvm(%0 : vector<2xf16>) -> !llvm.vec<2 x half> {
+  %1 = llvm.mlir.cast %0 : vector<2xf16> to !llvm.vec<2 x half>
   // CHECK-NEXT: llvm.return %[[ARG]]
-  return %1 : !llvm<"<2 x half>">
+  return %1 : !llvm.vec<2 x half>
 }
 
 // CHECK-LABEL: func @mlir_cast_from_llvm
 // CHECK-SAME: %[[ARG:.*]]:
-func @mlir_cast_from_llvm(%0 : !llvm<"<2 x half>">) -> vector<2xf16> {
-  %1 = llvm.mlir.cast %0 : !llvm<"<2 x half>"> to vector<2xf16>
+func @mlir_cast_from_llvm(%0 : !llvm.vec<2 x half>) -> vector<2xf16> {
+  %1 = llvm.mlir.cast %0 : !llvm.vec<2 x half> to vector<2xf16>
   // CHECK-NEXT: llvm.return %[[ARG]]
   return %1 : vector<2xf16>
 }
@@ -1281,13 +1281,13 @@ func @bfloat(%arg0: bf16) -> bf16 {
 // -----
 
 // CHECK-LABEL: func @memref_index
-// CHECK-SAME: %arg0: !llvm<"i64*">, %arg1: !llvm<"i64*">,
+// CHECK-SAME: %arg0: !llvm.ptr<i64>, %arg1: !llvm.ptr<i64>,
 // CHECK-SAME: %arg2: !llvm.i64, %arg3: !llvm.i64, %arg4: !llvm.i64)
-// CHECK-SAME: -> !llvm<"{ i64*, i64*, i64, [1 x i64], [1 x i64] }">
+// CHECK-SAME: -> !llvm.struct<(ptr<i64>, ptr<i64>, i64, array<1 x i64>, array<1 x i64>)>
 // CHECK32-LABEL: func @memref_index
-// CHECK32-SAME: %arg0: !llvm<"i32*">, %arg1: !llvm<"i32*">,
+// CHECK32-SAME: %arg0: !llvm.ptr<i32>, %arg1: !llvm.ptr<i32>,
 // CHECK32-SAME: %arg2: !llvm.i32, %arg3: !llvm.i32, %arg4: !llvm.i32)
-// CHECK32-SAME: -> !llvm<"{ i32*, i32*, i32, [1 x i32], [1 x i32] }">
+// CHECK32-SAME: -> !llvm.struct<(ptr<i32>, ptr<i32>, i32, array<1 x i32>, array<1 x i32>)>
 func @memref_index(%arg0: memref<32xindex>) -> memref<32xindex> {
   return %arg0 : memref<32xindex>
 }
diff --git a/mlir/test/Conversion/StandardToLLVM/invalid.mlir b/mlir/test/Conversion/StandardToLLVM/invalid.mlir
index 56e6612423360..469bb9753ec49 100644
--- a/mlir/test/Conversion/StandardToLLVM/invalid.mlir
+++ b/mlir/test/Conversion/StandardToLLVM/invalid.mlir
@@ -24,8 +24,8 @@ func @mlir_cast_to_llvm_int(%0 : i32) -> !llvm.i64 {
 
 // -----
 
-func @mlir_cast_to_llvm_vec(%0 : vector<1x1xf32>) -> !llvm<"<1 x float>"> {
+func @mlir_cast_to_llvm_vec(%0 : vector<1x1xf32>) -> !llvm.vec<1 x float> {
   // expected-error@+1 {{'llvm.mlir.cast' op only 1-d vector is allowed}}
-  %1 = llvm.mlir.cast %0 : vector<1x1xf32> to !llvm<"<1 x float>">
-  return %1 : !llvm<"<1 x float>">
+  %1 = llvm.mlir.cast %0 : vector<1x1xf32> to !llvm.vec<1 x float>
+  return %1 : !llvm.vec<1 x float>
 }
diff --git a/mlir/test/Conversion/StandardToLLVM/standard-to-llvm.mlir b/mlir/test/Conversion/StandardToLLVM/standard-to-llvm.mlir
index a6006023a5092..c55950a556344 100644
--- a/mlir/test/Conversion/StandardToLLVM/standard-to-llvm.mlir
+++ b/mlir/test/Conversion/StandardToLLVM/standard-to-llvm.mlir
@@ -1,11 +1,11 @@
 // RUN: mlir-opt -allow-unregistered-dialect %s -convert-std-to-llvm -split-input-file -verify-diagnostics | FileCheck %s
 
 // CHECK-LABEL: func @address_space(
-// CHECK-SAME:    !llvm<"float addrspace(7)*">
+// CHECK-SAME:    !llvm.ptr<float, 7>
 func @address_space(%arg0 : memref<32xf32, affine_map<(d0) -> (d0)>, 7>) {
   %0 = alloc() : memref<32xf32, affine_map<(d0) -> (d0)>, 5>
   %1 = constant 7 : index
-  // CHECK: llvm.load %{{.*}} : !llvm<"float addrspace(5)*">
+  // CHECK: llvm.load %{{.*}} : !llvm.ptr<float, 5>
   %2 = load %0[%1] : memref<32xf32, affine_map<(d0) -> (d0)>, 5>
   std.return
 }
@@ -53,11 +53,11 @@ func @rsqrt_double(%arg0 : f64) {
 // -----
 
 // CHECK-LABEL: func @rsqrt_vector(
-// CHECK-SAME: !llvm<"<4 x float>">
+// CHECK-SAME: !llvm.vec<4 x float>
 func @rsqrt_vector(%arg0 : vector<4xf32>) {
-  // CHECK: %[[ONE:.*]] = llvm.mlir.constant(dense<1.000000e+00> : vector<4xf32>) : !llvm<"<4 x float>">
-  // CHECK: %[[SQRT:.*]] = "llvm.intr.sqrt"(%arg0) : (!llvm<"<4 x float>">) -> !llvm<"<4 x float>">
-  // CHECK: %[[DIV:.*]] = llvm.fdiv %[[ONE]], %[[SQRT]] : !llvm<"<4 x float>">
+  // CHECK: %[[ONE:.*]] = llvm.mlir.constant(dense<1.000000e+00> : vector<4xf32>) : !llvm.vec<4 x float>
+  // CHECK: %[[SQRT:.*]] = "llvm.intr.sqrt"(%arg0) : (!llvm.vec<4 x float>) -> !llvm.vec<4 x float>
+  // CHECK: %[[DIV:.*]] = llvm.fdiv %[[ONE]], %[[SQRT]] : !llvm.vec<4 x float>
   %0 = rsqrt %arg0 : vector<4xf32>
   std.return
 }
@@ -65,13 +65,13 @@ func @rsqrt_vector(%arg0 : vector<4xf32>) {
 // -----
 
 // CHECK-LABEL: func @rsqrt_multidim_vector(
-// CHECK-SAME: !llvm<"[4 x <3 x float>]">
+// CHECK-SAME: !llvm.array<4 x vec<3 x float>>
 func @rsqrt_multidim_vector(%arg0 : vector<4x3xf32>) {
-  // CHECK: %[[EXTRACT:.*]] = llvm.extractvalue %arg0[0] : !llvm<"[4 x <3 x float>]">
-  // CHECK: %[[ONE:.*]] = llvm.mlir.constant(dense<1.000000e+00> : vector<3xf32>) : !llvm<"<3 x float>">
-  // CHECK: %[[SQRT:.*]] = "llvm.intr.sqrt"(%[[EXTRACT]]) : (!llvm<"<3 x float>">) -> !llvm<"<3 x float>">
-  // CHECK: %[[DIV:.*]] = llvm.fdiv %[[ONE]], %[[SQRT]] : !llvm<"<3 x float>">
-  // CHECK: %[[INSERT:.*]] = llvm.insertvalue %[[DIV]], %0[0] : !llvm<"[4 x <3 x float>]">
+  // CHECK: %[[EXTRACT:.*]] = llvm.extractvalue %arg0[0] : !llvm.array<4 x vec<3 x float>>
+  // CHECK: %[[ONE:.*]] = llvm.mlir.constant(dense<1.000000e+00> : vector<3xf32>) : !llvm.vec<3 x float>
+  // CHECK: %[[SQRT:.*]] = "llvm.intr.sqrt"(%[[EXTRACT]]) : (!llvm.vec<3 x float>) -> !llvm.vec<3 x float>
+  // CHECK: %[[DIV:.*]] = llvm.fdiv %[[ONE]], %[[SQRT]] : !llvm.vec<3 x float>
+  // CHECK: %[[INSERT:.*]] = llvm.insertvalue %[[DIV]], %0[0] : !llvm.array<4 x vec<3 x float>>
   %0 = rsqrt %arg0 : vector<4x3xf32>
   std.return
 }
diff --git a/mlir/test/Conversion/VectorToLLVM/vector-reduction-to-llvm.mlir b/mlir/test/Conversion/VectorToLLVM/vector-reduction-to-llvm.mlir
index eab211ffaaf4d..a2a47b0eff267 100644
--- a/mlir/test/Conversion/VectorToLLVM/vector-reduction-to-llvm.mlir
+++ b/mlir/test/Conversion/VectorToLLVM/vector-reduction-to-llvm.mlir
@@ -3,17 +3,17 @@
 
 //
 // CHECK-LABEL: llvm.func @reduce_add_f32(
-// CHECK-SAME: %[[A:.*]]: !llvm<"<16 x float>">)
+// CHECK-SAME: %[[A:.*]]: !llvm.vec<16 x float>)
 //      CHECK: %[[C:.*]] = llvm.mlir.constant(0.000000e+00 : f32) : !llvm.float
 //      CHECK: %[[V:.*]] = "llvm.intr.experimental.vector.reduce.v2.fadd"(%[[C]], %[[A]])
-// CHECK-SAME: {reassoc = false} : (!llvm.float, !llvm<"<16 x float>">) -> !llvm.float
+// CHECK-SAME: {reassoc = false} : (!llvm.float, !llvm.vec<16 x float>) -> !llvm.float
 //      CHECK: llvm.return %[[V]] : !llvm.float
 //
 // REASSOC-LABEL: llvm.func @reduce_add_f32(
-// REASSOC-SAME: %[[A:.*]]: !llvm<"<16 x float>">)
+// REASSOC-SAME: %[[A:.*]]: !llvm.vec<16 x float>)
 //      REASSOC: %[[C:.*]] = llvm.mlir.constant(0.000000e+00 : f32) : !llvm.float
 //      REASSOC: %[[V:.*]] = "llvm.intr.experimental.vector.reduce.v2.fadd"(%[[C]], %[[A]])
-// REASSOC-SAME: {reassoc = true} : (!llvm.float, !llvm<"<16 x float>">) -> !llvm.float
+// REASSOC-SAME: {reassoc = true} : (!llvm.float, !llvm.vec<16 x float>) -> !llvm.float
 //      REASSOC: llvm.return %[[V]] : !llvm.float
 //
 func @reduce_add_f32(%arg0: vector<16xf32>) -> f32 {
@@ -23,17 +23,17 @@ func @reduce_add_f32(%arg0: vector<16xf32>) -> f32 {
 
 //
 // CHECK-LABEL: llvm.func @reduce_mul_f32(
-// CHECK-SAME: %[[A:.*]]: !llvm<"<16 x float>">)
+// CHECK-SAME: %[[A:.*]]: !llvm.vec<16 x float>)
 //      CHECK: %[[C:.*]] = llvm.mlir.constant(1.000000e+00 : f32) : !llvm.float
 //      CHECK: %[[V:.*]] = "llvm.intr.experimental.vector.reduce.v2.fmul"(%[[C]], %[[A]])
-// CHECK-SAME: {reassoc = false} : (!llvm.float, !llvm<"<16 x float>">) -> !llvm.float
+// CHECK-SAME: {reassoc = false} : (!llvm.float, !llvm.vec<16 x float>) -> !llvm.float
 //      CHECK: llvm.return %[[V]] : !llvm.float
 //
 // REASSOC-LABEL: llvm.func @reduce_mul_f32(
-// REASSOC-SAME: %[[A:.*]]: !llvm<"<16 x float>">)
+// REASSOC-SAME: %[[A:.*]]: !llvm.vec<16 x float>)
 //      REASSOC: %[[C:.*]] = llvm.mlir.constant(1.000000e+00 : f32) : !llvm.float
 //      REASSOC: %[[V:.*]] = "llvm.intr.experimental.vector.reduce.v2.fmul"(%[[C]], %[[A]])
-// REASSOC-SAME: {reassoc = true} : (!llvm.float, !llvm<"<16 x float>">) -> !llvm.float
+// REASSOC-SAME: {reassoc = true} : (!llvm.float, !llvm.vec<16 x float>) -> !llvm.float
 //      REASSOC: llvm.return %[[V]] : !llvm.float
 //
 func @reduce_mul_f32(%arg0: vector<16xf32>) -> f32 {
diff --git a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir
index c5259a17fcef0..2e5aae886c380 100644
--- a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir
+++ b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir
@@ -6,11 +6,11 @@ func @broadcast_vec1d_from_scalar(%arg0: f32) -> vector<2xf32> {
 }
 // CHECK-LABEL: llvm.func @broadcast_vec1d_from_scalar(
 // CHECK-SAME:  %[[A:.*]]: !llvm.float)
-// CHECK:       %[[T0:.*]] = llvm.mlir.undef : !llvm<"<2 x float>">
+// CHECK:       %[[T0:.*]] = llvm.mlir.undef : !llvm.vec<2 x float>
 // CHECK:       %[[T1:.*]] = llvm.mlir.constant(0 : i32) : !llvm.i32
-// CHECK:       %[[T2:.*]] = llvm.insertelement %[[A]], %[[T0]][%[[T1]] : !llvm.i32] : !llvm<"<2 x float>">
-// CHECK:       %[[T3:.*]] = llvm.shufflevector %[[T2]], %[[T0]] [0 : i32, 0 : i32] : !llvm<"<2 x float>">, !llvm<"<2 x float>">
-// CHECK:       llvm.return %[[T3]] : !llvm<"<2 x float>">
+// CHECK:       %[[T2:.*]] = llvm.insertelement %[[A]], %[[T0]][%[[T1]] : !llvm.i32] : !llvm.vec<2 x float>
+// CHECK:       %[[T3:.*]] = llvm.shufflevector %[[T2]], %[[T0]] [0 : i32, 0 : i32] : !llvm.vec<2 x float>, !llvm.vec<2 x float>
+// CHECK:       llvm.return %[[T3]] : !llvm.vec<2 x float>
 
 func @broadcast_vec2d_from_scalar(%arg0: f32) -> vector<2x3xf32> {
   %0 = vector.broadcast %arg0 : f32 to vector<2x3xf32>
@@ -18,14 +18,14 @@ func @broadcast_vec2d_from_scalar(%arg0: f32) -> vector<2x3xf32> {
 }
 // CHECK-LABEL: llvm.func @broadcast_vec2d_from_scalar(
 // CHECK-SAME:  %[[A:.*]]: !llvm.float)
-// CHECK:       %[[T0:.*]] = llvm.mlir.undef : !llvm<"[2 x <3 x float>]">
-// CHECK:       %[[T1:.*]] = llvm.mlir.undef : !llvm<"<3 x float>">
+// CHECK:       %[[T0:.*]] = llvm.mlir.undef : !llvm.array<2 x vec<3 x float>>
+// CHECK:       %[[T1:.*]] = llvm.mlir.undef : !llvm.vec<3 x float>
 // CHECK:       %[[T2:.*]] = llvm.mlir.constant(0 : i32) : !llvm.i32
-// CHECK:       %[[T3:.*]] = llvm.insertelement %[[A]], %[[T1]][%[[T2]] : !llvm.i32] : !llvm<"<3 x float>">
-// CHECK:       %[[T4:.*]] = llvm.shufflevector %[[T3]], %[[T3]] [0 : i32, 0 : i32, 0 : i32] : !llvm<"<3 x float>">, !llvm<"<3 x float>">
-// CHECK:       %[[T5:.*]] = llvm.insertvalue %[[T4]], %[[T0]][0] : !llvm<"[2 x <3 x float>]">
-// CHECK:       %[[T6:.*]] = llvm.insertvalue %[[T4]], %[[T5]][1] : !llvm<"[2 x <3 x float>]">
-// CHECK:       llvm.return %[[T6]] : !llvm<"[2 x <3 x float>]">
+// CHECK:       %[[T3:.*]] = llvm.insertelement %[[A]], %[[T1]][%[[T2]] : !llvm.i32] : !llvm.vec<3 x float>
+// CHECK:       %[[T4:.*]] = llvm.shufflevector %[[T3]], %[[T3]] [0 : i32, 0 : i32, 0 : i32] : !llvm.vec<3 x float>, !llvm.vec<3 x float>
+// CHECK:       %[[T5:.*]] = llvm.insertvalue %[[T4]], %[[T0]][0] : !llvm.array<2 x vec<3 x float>>
+// CHECK:       %[[T6:.*]] = llvm.insertvalue %[[T4]], %[[T5]][1] : !llvm.array<2 x vec<3 x float>>
+// CHECK:       llvm.return %[[T6]] : !llvm.array<2 x vec<3 x float>>
 
 func @broadcast_vec3d_from_scalar(%arg0: f32) -> vector<2x3x4xf32> {
   %0 = vector.broadcast %arg0 : f32 to vector<2x3x4xf32>
@@ -33,277 +33,277 @@ func @broadcast_vec3d_from_scalar(%arg0: f32) -> vector<2x3x4xf32> {
 }
 // CHECK-LABEL: llvm.func @broadcast_vec3d_from_scalar(
 // CHECK-SAME:  %[[A:.*]]: !llvm.float)
-// CHECK:       %[[T0:.*]] = llvm.mlir.undef : !llvm<"[2 x [3 x <4 x float>]]">
-// CHECK:       %[[T1:.*]] = llvm.mlir.undef : !llvm<"<4 x float>">
+// CHECK:       %[[T0:.*]] = llvm.mlir.undef : !llvm.array<2 x array<3 x vec<4 x float>>>
+// CHECK:       %[[T1:.*]] = llvm.mlir.undef : !llvm.vec<4 x float>
 // CHECK:       %[[T2:.*]] = llvm.mlir.constant(0 : i32) : !llvm.i32
-// CHECK:       %[[T3:.*]] = llvm.insertelement %[[A]], %[[T1]][%[[T2]] : !llvm.i32] : !llvm<"<4 x float>">
-// CHECK:       %[[T4:.*]] = llvm.shufflevector %[[T3]], %[[T3]] [0 : i32, 0 : i32, 0 : i32, 0 : i32] : !llvm<"<4 x float>">, !llvm<"<4 x float>">
-// CHECK:       %[[T5:.*]] = llvm.insertvalue %[[T4]], %[[T0]][0, 0] : !llvm<"[2 x [3 x <4 x float>]]">
-// CHECK:       %[[T6:.*]] = llvm.insertvalue %[[T4]], %[[T5]][0, 1] : !llvm<"[2 x [3 x <4 x float>]]">
-// CHECK:       %[[T7:.*]] = llvm.insertvalue %[[T4]], %[[T6]][0, 2] : !llvm<"[2 x [3 x <4 x float>]]">
-// CHECK:       %[[T8:.*]] = llvm.insertvalue %[[T4]], %[[T7]][1, 0] : !llvm<"[2 x [3 x <4 x float>]]">
-// CHECK:       %[[T9:.*]] = llvm.insertvalue %[[T4]], %[[T8]][1, 1] : !llvm<"[2 x [3 x <4 x float>]]">
-// CHECK:       %[[T10:.*]] = llvm.insertvalue %[[T4]], %[[T9]][1, 2] : !llvm<"[2 x [3 x <4 x float>]]">
-// CHECK:       llvm.return %[[T10]] : !llvm<"[2 x [3 x <4 x float>]]">
+// CHECK:       %[[T3:.*]] = llvm.insertelement %[[A]], %[[T1]][%[[T2]] : !llvm.i32] : !llvm.vec<4 x float>
+// CHECK:       %[[T4:.*]] = llvm.shufflevector %[[T3]], %[[T3]] [0 : i32, 0 : i32, 0 : i32, 0 : i32] : !llvm.vec<4 x float>, !llvm.vec<4 x float>
+// CHECK:       %[[T5:.*]] = llvm.insertvalue %[[T4]], %[[T0]][0, 0] : !llvm.array<2 x array<3 x vec<4 x float>>>
+// CHECK:       %[[T6:.*]] = llvm.insertvalue %[[T4]], %[[T5]][0, 1] : !llvm.array<2 x array<3 x vec<4 x float>>>
+// CHECK:       %[[T7:.*]] = llvm.insertvalue %[[T4]], %[[T6]][0, 2] : !llvm.array<2 x array<3 x vec<4 x float>>>
+// CHECK:       %[[T8:.*]] = llvm.insertvalue %[[T4]], %[[T7]][1, 0] : !llvm.array<2 x array<3 x vec<4 x float>>>
+// CHECK:       %[[T9:.*]] = llvm.insertvalue %[[T4]], %[[T8]][1, 1] : !llvm.array<2 x array<3 x vec<4 x float>>>
+// CHECK:       %[[T10:.*]] = llvm.insertvalue %[[T4]], %[[T9]][1, 2] : !llvm.array<2 x array<3 x vec<4 x float>>>
+// CHECK:       llvm.return %[[T10]] : !llvm.array<2 x array<3 x vec<4 x float>>>
 
 func @broadcast_vec1d_from_vec1d(%arg0: vector<2xf32>) -> vector<2xf32> {
   %0 = vector.broadcast %arg0 : vector<2xf32> to vector<2xf32>
   return %0 : vector<2xf32>
 }
 // CHECK-LABEL: llvm.func @broadcast_vec1d_from_vec1d(
-// CHECK-SAME:  %[[A:.*]]: !llvm<"<2 x float>">)
-// CHECK:       llvm.return %[[A]] : !llvm<"<2 x float>">
+// CHECK-SAME:  %[[A:.*]]: !llvm.vec<2 x float>)
+// CHECK:       llvm.return %[[A]] : !llvm.vec<2 x float>
 
 func @broadcast_vec2d_from_vec1d(%arg0: vector<2xf32>) -> vector<3x2xf32> {
   %0 = vector.broadcast %arg0 : vector<2xf32> to vector<3x2xf32>
   return %0 : vector<3x2xf32>
 }
 // CHECK-LABEL: llvm.func @broadcast_vec2d_from_vec1d(
-// CHECK-SAME:  %[[A:.*]]: !llvm<"<2 x float>">)
-// CHECK:       %[[T0:.*]] = llvm.mlir.constant(dense<0.000000e+00> : vector<3x2xf32>) : !llvm<"[3 x <2 x float>]">
-// CHECK:       %[[T1:.*]] = llvm.insertvalue %[[A]], %[[T0]][0] : !llvm<"[3 x <2 x float>]">
-// CHECK:       %[[T2:.*]] = llvm.insertvalue %[[A]], %[[T1]][1] : !llvm<"[3 x <2 x float>]">
-// CHECK:       %[[T3:.*]] = llvm.insertvalue %[[A]], %[[T2]][2] : !llvm<"[3 x <2 x float>]">
-// CHECK:       llvm.return %[[T3]] : !llvm<"[3 x <2 x float>]">
+// CHECK-SAME:  %[[A:.*]]: !llvm.vec<2 x float>)
+// CHECK:       %[[T0:.*]] = llvm.mlir.constant(dense<0.000000e+00> : vector<3x2xf32>) : !llvm.array<3 x vec<2 x float>>
+// CHECK:       %[[T1:.*]] = llvm.insertvalue %[[A]], %[[T0]][0] : !llvm.array<3 x vec<2 x float>>
+// CHECK:       %[[T2:.*]] = llvm.insertvalue %[[A]], %[[T1]][1] : !llvm.array<3 x vec<2 x float>>
+// CHECK:       %[[T3:.*]] = llvm.insertvalue %[[A]], %[[T2]][2] : !llvm.array<3 x vec<2 x float>>
+// CHECK:       llvm.return %[[T3]] : !llvm.array<3 x vec<2 x float>>
 
 func @broadcast_vec3d_from_vec1d(%arg0: vector<2xf32>) -> vector<4x3x2xf32> {
   %0 = vector.broadcast %arg0 : vector<2xf32> to vector<4x3x2xf32>
   return %0 : vector<4x3x2xf32>
 }
 // CHECK-LABEL: llvm.func @broadcast_vec3d_from_vec1d(
-// CHECK-SAME:  %[[A:.*]]: !llvm<"<2 x float>">)
-// CHECK:       %[[T0:.*]] = llvm.mlir.constant(dense<0.000000e+00> : vector<3x2xf32>) : !llvm<"[3 x <2 x float>]">
-// CHECK:       %[[T1:.*]] = llvm.mlir.constant(dense<0.000000e+00> : vector<4x3x2xf32>) : !llvm<"[4 x [3 x <2 x float>]]">
-// CHECK:       %[[T2:.*]] = llvm.insertvalue %[[A]], %[[T0]][0] : !llvm<"[3 x <2 x float>]">
-// CHECK:       %[[T3:.*]] = llvm.insertvalue %[[A]], %[[T2]][1] : !llvm<"[3 x <2 x float>]">
-// CHECK:       %[[T4:.*]] = llvm.insertvalue %[[A]], %[[T3]][2] : !llvm<"[3 x <2 x float>]">
-// CHECK:       %[[T5:.*]] = llvm.insertvalue %[[T4]], %[[T1]][0] : !llvm<"[4 x [3 x <2 x float>]]">
-// CHECK:       %[[T6:.*]] = llvm.insertvalue %[[T4]], %[[T5]][1] : !llvm<"[4 x [3 x <2 x float>]]">
-// CHECK:       %[[T7:.*]] = llvm.insertvalue %[[T4]], %[[T6]][2] : !llvm<"[4 x [3 x <2 x float>]]">
-// CHECK:       %[[T8:.*]] = llvm.insertvalue %[[T4]], %[[T7]][3] : !llvm<"[4 x [3 x <2 x float>]]">
-// CHECK:       llvm.return %[[T8]] : !llvm<"[4 x [3 x <2 x float>]]">
+// CHECK-SAME:  %[[A:.*]]: !llvm.vec<2 x float>)
+// CHECK:       %[[T0:.*]] = llvm.mlir.constant(dense<0.000000e+00> : vector<3x2xf32>) : !llvm.array<3 x vec<2 x float>>
+// CHECK:       %[[T1:.*]] = llvm.mlir.constant(dense<0.000000e+00> : vector<4x3x2xf32>) : !llvm.array<4 x array<3 x vec<2 x float>>>
+// CHECK:       %[[T2:.*]] = llvm.insertvalue %[[A]], %[[T0]][0] : !llvm.array<3 x vec<2 x float>>
+// CHECK:       %[[T3:.*]] = llvm.insertvalue %[[A]], %[[T2]][1] : !llvm.array<3 x vec<2 x float>>
+// CHECK:       %[[T4:.*]] = llvm.insertvalue %[[A]], %[[T3]][2] : !llvm.array<3 x vec<2 x float>>
+// CHECK:       %[[T5:.*]] = llvm.insertvalue %[[T4]], %[[T1]][0] : !llvm.array<4 x array<3 x vec<2 x float>>>
+// CHECK:       %[[T6:.*]] = llvm.insertvalue %[[T4]], %[[T5]][1] : !llvm.array<4 x array<3 x vec<2 x float>>>
+// CHECK:       %[[T7:.*]] = llvm.insertvalue %[[T4]], %[[T6]][2] : !llvm.array<4 x array<3 x vec<2 x float>>>
+// CHECK:       %[[T8:.*]] = llvm.insertvalue %[[T4]], %[[T7]][3] : !llvm.array<4 x array<3 x vec<2 x float>>>
+// CHECK:       llvm.return %[[T8]] : !llvm.array<4 x array<3 x vec<2 x float>>>
 
 func @broadcast_vec3d_from_vec2d(%arg0: vector<3x2xf32>) -> vector<4x3x2xf32> {
   %0 = vector.broadcast %arg0 : vector<3x2xf32> to vector<4x3x2xf32>
   return %0 : vector<4x3x2xf32>
 }
 // CHECK-LABEL: llvm.func @broadcast_vec3d_from_vec2d(
-// CHECK-SAME:  %[[A:.*]]: !llvm<"[3 x <2 x float>]">)
-// CHECK:       %[[T0:.*]] = llvm.mlir.constant(dense<0.000000e+00> : vector<4x3x2xf32>) : !llvm<"[4 x [3 x <2 x float>]]">
-// CHECK:       %[[T1:.*]] = llvm.insertvalue %[[A]], %[[T0]][0] : !llvm<"[4 x [3 x <2 x float>]]">
-// CHECK:       %[[T2:.*]] = llvm.insertvalue %[[A]], %[[T1]][1] : !llvm<"[4 x [3 x <2 x float>]]">
-// CHECK:       %[[T3:.*]] = llvm.insertvalue %[[A]], %[[T2]][2] : !llvm<"[4 x [3 x <2 x float>]]">
-// CHECK:       %[[T4:.*]] = llvm.insertvalue %[[A]], %[[T3]][3] : !llvm<"[4 x [3 x <2 x float>]]">
-// CHECK:       llvm.return %[[T4]] : !llvm<"[4 x [3 x <2 x float>]]">
+// CHECK-SAME:  %[[A:.*]]: !llvm.array<3 x vec<2 x float>>)
+// CHECK:       %[[T0:.*]] = llvm.mlir.constant(dense<0.000000e+00> : vector<4x3x2xf32>) : !llvm.array<4 x array<3 x vec<2 x float>>>
+// CHECK:       %[[T1:.*]] = llvm.insertvalue %[[A]], %[[T0]][0] : !llvm.array<4 x array<3 x vec<2 x float>>>
+// CHECK:       %[[T2:.*]] = llvm.insertvalue %[[A]], %[[T1]][1] : !llvm.array<4 x array<3 x vec<2 x float>>>
+// CHECK:       %[[T3:.*]] = llvm.insertvalue %[[A]], %[[T2]][2] : !llvm.array<4 x array<3 x vec<2 x float>>>
+// CHECK:       %[[T4:.*]] = llvm.insertvalue %[[A]], %[[T3]][3] : !llvm.array<4 x array<3 x vec<2 x float>>>
+// CHECK:       llvm.return %[[T4]] : !llvm.array<4 x array<3 x vec<2 x float>>>
 
 func @broadcast_stretch(%arg0: vector<1xf32>) -> vector<4xf32> {
   %0 = vector.broadcast %arg0 : vector<1xf32> to vector<4xf32>
   return %0 : vector<4xf32>
 }
 // CHECK-LABEL: llvm.func @broadcast_stretch(
-// CHECK-SAME:  %[[A:.*]]: !llvm<"<1 x float>">)
+// CHECK-SAME:  %[[A:.*]]: !llvm.vec<1 x float>)
 // CHECK:       %[[T0:.*]] = llvm.mlir.constant(0 : i64) : !llvm.i64
-// CHECK:       %[[T1:.*]] = llvm.extractelement %[[A]][%[[T0]] : !llvm.i64] : !llvm<"<1 x float>">
-// CHECK:       %[[T2:.*]] = llvm.mlir.undef : !llvm<"<4 x float>">
+// CHECK:       %[[T1:.*]] = llvm.extractelement %[[A]][%[[T0]] : !llvm.i64] : !llvm.vec<1 x float>
+// CHECK:       %[[T2:.*]] = llvm.mlir.undef : !llvm.vec<4 x float>
 // CHECK:       %[[T3:.*]] = llvm.mlir.constant(0 : i32) : !llvm.i32
-// CHECK:       %[[T4:.*]] = llvm.insertelement %[[T1]], %[[T2]][%3 : !llvm.i32] : !llvm<"<4 x float>">
-// CHECK:       %[[T5:.*]] = llvm.shufflevector %[[T4]], %[[T2]] [0 : i32, 0 : i32, 0 : i32, 0 : i32] : !llvm<"<4 x float>">, !llvm<"<4 x float>">
-// CHECK:       llvm.return %[[T5]] : !llvm<"<4 x float>">
+// CHECK:       %[[T4:.*]] = llvm.insertelement %[[T1]], %[[T2]][%3 : !llvm.i32] : !llvm.vec<4 x float>
+// CHECK:       %[[T5:.*]] = llvm.shufflevector %[[T4]], %[[T2]] [0 : i32, 0 : i32, 0 : i32, 0 : i32] : !llvm.vec<4 x float>, !llvm.vec<4 x float>
+// CHECK:       llvm.return %[[T5]] : !llvm.vec<4 x float>
 
 func @broadcast_stretch_at_start(%arg0: vector<1x4xf32>) -> vector<3x4xf32> {
   %0 = vector.broadcast %arg0 : vector<1x4xf32> to vector<3x4xf32>
   return %0 : vector<3x4xf32>
 }
 // CHECK-LABEL: llvm.func @broadcast_stretch_at_start(
-// CHECK-SAME:  %[[A:.*]]: !llvm<"[1 x <4 x float>]">)
-// CHECK:       %[[T0:.*]] = llvm.mlir.constant(dense<0.000000e+00> : vector<3x4xf32>) : !llvm<"[3 x <4 x float>]">
-// CHECK:       %[[T1:.*]] = llvm.extractvalue %[[A]][0] : !llvm<"[1 x <4 x float>]">
-// CHECK:       %[[T2:.*]] = llvm.insertvalue %[[T1]], %[[T0]][0] : !llvm<"[3 x <4 x float>]">
-// CHECK:       %[[T3:.*]] = llvm.insertvalue %[[T1]], %[[T2]][1] : !llvm<"[3 x <4 x float>]">
-// CHECK:       %[[T4:.*]] = llvm.insertvalue %[[T1]], %[[T3]][2] : !llvm<"[3 x <4 x float>]">
-// CHECK:       llvm.return %[[T4]] : !llvm<"[3 x <4 x float>]">
+// CHECK-SAME:  %[[A:.*]]: !llvm.array<1 x vec<4 x float>>)
+// CHECK:       %[[T0:.*]] = llvm.mlir.constant(dense<0.000000e+00> : vector<3x4xf32>) : !llvm.array<3 x vec<4 x float>>
+// CHECK:       %[[T1:.*]] = llvm.extractvalue %[[A]][0] : !llvm.array<1 x vec<4 x float>>
+// CHECK:       %[[T2:.*]] = llvm.insertvalue %[[T1]], %[[T0]][0] : !llvm.array<3 x vec<4 x float>>
+// CHECK:       %[[T3:.*]] = llvm.insertvalue %[[T1]], %[[T2]][1] : !llvm.array<3 x vec<4 x float>>
+// CHECK:       %[[T4:.*]] = llvm.insertvalue %[[T1]], %[[T3]][2] : !llvm.array<3 x vec<4 x float>>
+// CHECK:       llvm.return %[[T4]] : !llvm.array<3 x vec<4 x float>>
 
 func @broadcast_stretch_at_end(%arg0: vector<4x1xf32>) -> vector<4x3xf32> {
   %0 = vector.broadcast %arg0 : vector<4x1xf32> to vector<4x3xf32>
   return %0 : vector<4x3xf32>
 }
 // CHECK-LABEL: llvm.func @broadcast_stretch_at_end(
-// CHECK-SAME:  %[[A:.*]]: !llvm<"[4 x <1 x float>]">)
-// CHECK:       %[[T0:.*]] = llvm.mlir.constant(dense<0.000000e+00> : vector<4x3xf32>) : !llvm<"[4 x <3 x float>]">
-// CHECK:       %[[T1:.*]] = llvm.extractvalue %[[A]][0] : !llvm<"[4 x <1 x float>]">
+// CHECK-SAME:  %[[A:.*]]: !llvm.array<4 x vec<1 x float>>)
+// CHECK:       %[[T0:.*]] = llvm.mlir.constant(dense<0.000000e+00> : vector<4x3xf32>) : !llvm.array<4 x vec<3 x float>>
+// CHECK:       %[[T1:.*]] = llvm.extractvalue %[[A]][0] : !llvm.array<4 x vec<1 x float>>
 // CHECK:       %[[T2:.*]] = llvm.mlir.constant(0 : i64) : !llvm.i64
-// CHECK:       %[[T3:.*]] = llvm.extractelement %[[T1]][%[[T2]] : !llvm.i64] : !llvm<"<1 x float>">
-// CHECK:       %[[T4:.*]] = llvm.mlir.undef : !llvm<"<3 x float>">
+// CHECK:       %[[T3:.*]] = llvm.extractelement %[[T1]][%[[T2]] : !llvm.i64] : !llvm.vec<1 x float>
+// CHECK:       %[[T4:.*]] = llvm.mlir.undef : !llvm.vec<3 x float>
 // CHECK:       %[[T5:.*]] = llvm.mlir.constant(0 : i32) : !llvm.i32
-// CHECK:       %[[T6:.*]] = llvm.insertelement %[[T3]], %[[T4]][%[[T5]] : !llvm.i32] : !llvm<"<3 x float>">
-// CHECK:       %[[T7:.*]] = llvm.shufflevector %[[T6]], %[[T4]] [0 : i32, 0 : i32, 0 : i32] : !llvm<"<3 x float>">, !llvm<"<3 x float>">
-// CHECK:       %[[T8:.*]] = llvm.insertvalue %[[T7]], %[[T0]][0] : !llvm<"[4 x <3 x float>]">
-// CHECK:       %[[T9:.*]] = llvm.extractvalue %[[A]][1] : !llvm<"[4 x <1 x float>]">
+// CHECK:       %[[T6:.*]] = llvm.insertelement %[[T3]], %[[T4]][%[[T5]] : !llvm.i32] : !llvm.vec<3 x float>
+// CHECK:       %[[T7:.*]] = llvm.shufflevector %[[T6]], %[[T4]] [0 : i32, 0 : i32, 0 : i32] : !llvm.vec<3 x float>, !llvm.vec<3 x float>
+// CHECK:       %[[T8:.*]] = llvm.insertvalue %[[T7]], %[[T0]][0] : !llvm.array<4 x vec<3 x float>>
+// CHECK:       %[[T9:.*]] = llvm.extractvalue %[[A]][1] : !llvm.array<4 x vec<1 x float>>
 // CHECK:       %[[T10:.*]] = llvm.mlir.constant(0 : i64) : !llvm.i64
-// CHECK:       %[[T11:.*]] = llvm.extractelement %[[T9]][%[[T10]] : !llvm.i64] : !llvm<"<1 x float>">
-// CHECK:       %[[T12:.*]] = llvm.mlir.undef : !llvm<"<3 x float>">
+// CHECK:       %[[T11:.*]] = llvm.extractelement %[[T9]][%[[T10]] : !llvm.i64] : !llvm.vec<1 x float>
+// CHECK:       %[[T12:.*]] = llvm.mlir.undef : !llvm.vec<3 x float>
 // CHECK:       %[[T13:.*]] = llvm.mlir.constant(0 : i32) : !llvm.i32
-// CHECK:       %[[T14:.*]] = llvm.insertelement %[[T11]], %[[T12]][%[[T13]] : !llvm.i32] : !llvm<"<3 x float>">
-// CHECK:       %[[T15:.*]] = llvm.shufflevector %[[T14]], %[[T12]] [0 : i32, 0 : i32, 0 : i32] : !llvm<"<3 x float>">, !llvm<"<3 x float>">
-// CHECK:       %[[T16:.*]] = llvm.insertvalue %[[T15]], %[[T8]][1] : !llvm<"[4 x <3 x float>]">
-// CHECK:       %[[T17:.*]] = llvm.extractvalue %[[A]][2] : !llvm<"[4 x <1 x float>]">
+// CHECK:       %[[T14:.*]] = llvm.insertelement %[[T11]], %[[T12]][%[[T13]] : !llvm.i32] : !llvm.vec<3 x float>
+// CHECK:       %[[T15:.*]] = llvm.shufflevector %[[T14]], %[[T12]] [0 : i32, 0 : i32, 0 : i32] : !llvm.vec<3 x float>, !llvm.vec<3 x float>
+// CHECK:       %[[T16:.*]] = llvm.insertvalue %[[T15]], %[[T8]][1] : !llvm.array<4 x vec<3 x float>>
+// CHECK:       %[[T17:.*]] = llvm.extractvalue %[[A]][2] : !llvm.array<4 x vec<1 x float>>
 // CHECK:       %[[T18:.*]] = llvm.mlir.constant(0 : i64) : !llvm.i64
-// CHECK:       %[[T19:.*]] = llvm.extractelement %[[T17]][%[[T18]] : !llvm.i64] : !llvm<"<1 x float>">
-// CHECK:       %[[T20:.*]] = llvm.mlir.undef : !llvm<"<3 x float>">
+// CHECK:       %[[T19:.*]] = llvm.extractelement %[[T17]][%[[T18]] : !llvm.i64] : !llvm.vec<1 x float>
+// CHECK:       %[[T20:.*]] = llvm.mlir.undef : !llvm.vec<3 x float>
 // CHECK:       %[[T21:.*]] = llvm.mlir.constant(0 : i32) : !llvm.i32
-// CHECK:       %[[T22:.*]] = llvm.insertelement %[[T19]], %[[T20]][%[[T21]] : !llvm.i32] : !llvm<"<3 x float>">
-// CHECK:       %[[T23:.*]] = llvm.shufflevector %[[T22]], %[[T20]] [0 : i32, 0 : i32, 0 : i32] : !llvm<"<3 x float>">, !llvm<"<3 x float>">
-// CHECK:       %[[T24:.*]] = llvm.insertvalue %[[T23]], %[[T16]][2] : !llvm<"[4 x <3 x float>]">
-// CHECK:       %[[T25:.*]] = llvm.extractvalue %[[A]][3] : !llvm<"[4 x <1 x float>]">
+// CHECK:       %[[T22:.*]] = llvm.insertelement %[[T19]], %[[T20]][%[[T21]] : !llvm.i32] : !llvm.vec<3 x float>
+// CHECK:       %[[T23:.*]] = llvm.shufflevector %[[T22]], %[[T20]] [0 : i32, 0 : i32, 0 : i32] : !llvm.vec<3 x float>, !llvm.vec<3 x float>
+// CHECK:       %[[T24:.*]] = llvm.insertvalue %[[T23]], %[[T16]][2] : !llvm.array<4 x vec<3 x float>>
+// CHECK:       %[[T25:.*]] = llvm.extractvalue %[[A]][3] : !llvm.array<4 x vec<1 x float>>
 // CHECK:       %[[T26:.*]] = llvm.mlir.constant(0 : i64) : !llvm.i64
-// CHECK:       %[[T27:.*]] = llvm.extractelement %[[T25]][%[[T26]] : !llvm.i64] : !llvm<"<1 x float>">
-// CHECK:       %[[T28:.*]] = llvm.mlir.undef : !llvm<"<3 x float>">
+// CHECK:       %[[T27:.*]] = llvm.extractelement %[[T25]][%[[T26]] : !llvm.i64] : !llvm.vec<1 x float>
+// CHECK:       %[[T28:.*]] = llvm.mlir.undef : !llvm.vec<3 x float>
 // CHECK:       %[[T29:.*]] = llvm.mlir.constant(0 : i32) : !llvm.i32
-// CHECK:       %[[T30:.*]] = llvm.insertelement %[[T27]], %[[T28]][%[[T29]] : !llvm.i32] : !llvm<"<3 x float>">
-// CHECK:       %[[T31:.*]] = llvm.shufflevector %[[T30]], %[[T28]] [0 : i32, 0 : i32, 0 : i32] : !llvm<"<3 x float>">, !llvm<"<3 x float>">
-// CHECK:       %[[T32:.*]] = llvm.insertvalue %[[T31]], %[[T24]][3] : !llvm<"[4 x <3 x float>]">
-// CHECK:       llvm.return %[[T32]] : !llvm<"[4 x <3 x float>]">
+// CHECK:       %[[T30:.*]] = llvm.insertelement %[[T27]], %[[T28]][%[[T29]] : !llvm.i32] : !llvm.vec<3 x float>
+// CHECK:       %[[T31:.*]] = llvm.shufflevector %[[T30]], %[[T28]] [0 : i32, 0 : i32, 0 : i32] : !llvm.vec<3 x float>, !llvm.vec<3 x float>
+// CHECK:       %[[T32:.*]] = llvm.insertvalue %[[T31]], %[[T24]][3] : !llvm.array<4 x vec<3 x float>>
+// CHECK:       llvm.return %[[T32]] : !llvm.array<4 x vec<3 x float>>
 
 func @broadcast_stretch_in_middle(%arg0: vector<4x1x2xf32>) -> vector<4x3x2xf32> {
   %0 = vector.broadcast %arg0 : vector<4x1x2xf32> to vector<4x3x2xf32>
   return %0 : vector<4x3x2xf32>
 }
 // CHECK-LABEL: llvm.func @broadcast_stretch_in_middle(
-// CHECK-SAME:  %[[A:.*]]: !llvm<"[4 x [1 x <2 x float>]]">)
-// CHECK:       %[[T0:.*]] = llvm.mlir.constant(dense<0.000000e+00> : vector<4x3x2xf32>) : !llvm<"[4 x [3 x <2 x float>]]">
-// CHECK:       %[[T1:.*]] = llvm.mlir.constant(dense<0.000000e+00> : vector<3x2xf32>) : !llvm<"[3 x <2 x float>]">
-// CHECK:       %[[T2:.*]] = llvm.extractvalue %[[A]][0, 0] : !llvm<"[4 x [1 x <2 x float>]]">
-// CHECK:       %[[T4:.*]] = llvm.insertvalue %[[T2]], %[[T1]][0] : !llvm<"[3 x <2 x float>]">
-// CHECK:       %[[T5:.*]] = llvm.insertvalue %[[T2]], %[[T4]][1] : !llvm<"[3 x <2 x float>]">
-// CHECK:       %[[T6:.*]] = llvm.insertvalue %[[T2]], %[[T5]][2] : !llvm<"[3 x <2 x float>]">
-// CHECK:       %[[T7:.*]] = llvm.insertvalue %[[T6]], %[[T0]][0] : !llvm<"[4 x [3 x <2 x float>]]">
-// CHECK:       %[[T8:.*]] = llvm.extractvalue %[[A]][1, 0] : !llvm<"[4 x [1 x <2 x float>]]">
-// CHECK:       %[[T10:.*]] = llvm.insertvalue %[[T8]], %[[T1]][0] : !llvm<"[3 x <2 x float>]">
-// CHECK:       %[[T11:.*]] = llvm.insertvalue %[[T8]], %[[T10]][1] : !llvm<"[3 x <2 x float>]">
-// CHECK:       %[[T12:.*]] = llvm.insertvalue %[[T8]], %[[T11]][2] : !llvm<"[3 x <2 x float>]">
-// CHECK:       %[[T13:.*]] = llvm.insertvalue %[[T12]], %[[T7]][1] : !llvm<"[4 x [3 x <2 x float>]]">
-// CHECK:       %[[T14:.*]] = llvm.extractvalue %[[A]][2, 0] : !llvm<"[4 x [1 x <2 x float>]]">
-// CHECK:       %[[T16:.*]] = llvm.insertvalue %[[T14]], %[[T1]][0] : !llvm<"[3 x <2 x float>]">
-// CHECK:       %[[T17:.*]] = llvm.insertvalue %[[T14]], %[[T16]][1] : !llvm<"[3 x <2 x float>]">
-// CHECK:       %[[T18:.*]] = llvm.insertvalue %[[T14]], %[[T17]][2] : !llvm<"[3 x <2 x float>]">
-// CHECK:       %[[T19:.*]] = llvm.insertvalue %[[T18]], %[[T13]][2] : !llvm<"[4 x [3 x <2 x float>]]">
-// CHECK:       %[[T20:.*]] = llvm.extractvalue %[[A]][3, 0] : !llvm<"[4 x [1 x <2 x float>]]">
-// CHECK:       %[[T22:.*]] = llvm.insertvalue %[[T20]], %[[T1]][0] : !llvm<"[3 x <2 x float>]">
-// CHECK:       %[[T23:.*]] = llvm.insertvalue %[[T20]], %[[T22]][1] : !llvm<"[3 x <2 x float>]">
-// CHECK:       %[[T24:.*]] = llvm.insertvalue %[[T20]], %[[T23]][2] : !llvm<"[3 x <2 x float>]">
-// CHECK:       %[[T25:.*]] = llvm.insertvalue %[[T24]], %[[T19]][3] : !llvm<"[4 x [3 x <2 x float>]]">
-// CHECK:       llvm.return %[[T25]] : !llvm<"[4 x [3 x <2 x float>]]">
+// CHECK-SAME:  %[[A:.*]]: !llvm.array<4 x array<1 x vec<2 x float>>>)
+// CHECK:       %[[T0:.*]] = llvm.mlir.constant(dense<0.000000e+00> : vector<4x3x2xf32>) : !llvm.array<4 x array<3 x vec<2 x float>>>
+// CHECK:       %[[T1:.*]] = llvm.mlir.constant(dense<0.000000e+00> : vector<3x2xf32>) : !llvm.array<3 x vec<2 x float>>
+// CHECK:       %[[T2:.*]] = llvm.extractvalue %[[A]][0, 0] : !llvm.array<4 x array<1 x vec<2 x float>>>
+// CHECK:       %[[T4:.*]] = llvm.insertvalue %[[T2]], %[[T1]][0] : !llvm.array<3 x vec<2 x float>>
+// CHECK:       %[[T5:.*]] = llvm.insertvalue %[[T2]], %[[T4]][1] : !llvm.array<3 x vec<2 x float>>
+// CHECK:       %[[T6:.*]] = llvm.insertvalue %[[T2]], %[[T5]][2] : !llvm.array<3 x vec<2 x float>>
+// CHECK:       %[[T7:.*]] = llvm.insertvalue %[[T6]], %[[T0]][0] : !llvm.array<4 x array<3 x vec<2 x float>>>
+// CHECK:       %[[T8:.*]] = llvm.extractvalue %[[A]][1, 0] : !llvm.array<4 x array<1 x vec<2 x float>>>
+// CHECK:       %[[T10:.*]] = llvm.insertvalue %[[T8]], %[[T1]][0] : !llvm.array<3 x vec<2 x float>>
+// CHECK:       %[[T11:.*]] = llvm.insertvalue %[[T8]], %[[T10]][1] : !llvm.array<3 x vec<2 x float>>
+// CHECK:       %[[T12:.*]] = llvm.insertvalue %[[T8]], %[[T11]][2] : !llvm.array<3 x vec<2 x float>>
+// CHECK:       %[[T13:.*]] = llvm.insertvalue %[[T12]], %[[T7]][1] : !llvm.array<4 x array<3 x vec<2 x float>>>
+// CHECK:       %[[T14:.*]] = llvm.extractvalue %[[A]][2, 0] : !llvm.array<4 x array<1 x vec<2 x float>>>
+// CHECK:       %[[T16:.*]] = llvm.insertvalue %[[T14]], %[[T1]][0] : !llvm.array<3 x vec<2 x float>>
+// CHECK:       %[[T17:.*]] = llvm.insertvalue %[[T14]], %[[T16]][1] : !llvm.array<3 x vec<2 x float>>
+// CHECK:       %[[T18:.*]] = llvm.insertvalue %[[T14]], %[[T17]][2] : !llvm.array<3 x vec<2 x float>>
+// CHECK:       %[[T19:.*]] = llvm.insertvalue %[[T18]], %[[T13]][2] : !llvm.array<4 x array<3 x vec<2 x float>>>
+// CHECK:       %[[T20:.*]] = llvm.extractvalue %[[A]][3, 0] : !llvm.array<4 x array<1 x vec<2 x float>>>
+// CHECK:       %[[T22:.*]] = llvm.insertvalue %[[T20]], %[[T1]][0] : !llvm.array<3 x vec<2 x float>>
+// CHECK:       %[[T23:.*]] = llvm.insertvalue %[[T20]], %[[T22]][1] : !llvm.array<3 x vec<2 x float>>
+// CHECK:       %[[T24:.*]] = llvm.insertvalue %[[T20]], %[[T23]][2] : !llvm.array<3 x vec<2 x float>>
+// CHECK:       %[[T25:.*]] = llvm.insertvalue %[[T24]], %[[T19]][3] : !llvm.array<4 x array<3 x vec<2 x float>>>
+// CHECK:       llvm.return %[[T25]] : !llvm.array<4 x array<3 x vec<2 x float>>>
 
 func @outerproduct(%arg0: vector<2xf32>, %arg1: vector<3xf32>) -> vector<2x3xf32> {
   %2 = vector.outerproduct %arg0, %arg1 : vector<2xf32>, vector<3xf32>
   return %2 : vector<2x3xf32>
 }
 // CHECK-LABEL: llvm.func @outerproduct(
-// CHECK-SAME: %[[A:.*]]: !llvm<"<2 x float>">,
-// CHECK-SAME: %[[B:.*]]: !llvm<"<3 x float>">)
+// CHECK-SAME: %[[A:.*]]: !llvm.vec<2 x float>,
+// CHECK-SAME: %[[B:.*]]: !llvm.vec<3 x float>)
 //      CHECK: %[[T0:.*]] = llvm.mlir.constant(dense<0.000000e+00> : vector<2x3xf32>)
 //      CHECK: %[[T1:.*]] = llvm.mlir.constant(0 : i64) : !llvm.i64
-//      CHECK: %[[T2:.*]] = llvm.extractelement %[[A]][%[[T1]] : !llvm.i64] : !llvm<"<2 x float>">
-//      CHECK: %[[T3:.*]] = llvm.mlir.undef : !llvm<"<3 x float>">
+//      CHECK: %[[T2:.*]] = llvm.extractelement %[[A]][%[[T1]] : !llvm.i64] : !llvm.vec<2 x float>
+//      CHECK: %[[T3:.*]] = llvm.mlir.undef : !llvm.vec<3 x float>
 //      CHECK: %[[T4:.*]] = llvm.mlir.constant(0 : i32) : !llvm.i32
-//      CHECK: %[[T5:.*]] = llvm.insertelement %[[T2]], %[[T3]][%4 : !llvm.i32] : !llvm<"<3 x float>">
-//      CHECK: %[[T6:.*]] = llvm.shufflevector %[[T5]], %[[T3]] [0 : i32, 0 : i32, 0 : i32] : !llvm<"<3 x float>">, !llvm<"<3 x float>">
-//      CHECK: %[[T7:.*]] = llvm.fmul %[[T6]], %[[B]] : !llvm<"<3 x float>">
-//      CHECK: %[[T8:.*]] = llvm.insertvalue %[[T7]], %[[T0]][0] : !llvm<"[2 x <3 x float>]">
+//      CHECK: %[[T5:.*]] = llvm.insertelement %[[T2]], %[[T3]][%4 : !llvm.i32] : !llvm.vec<3 x float>
+//      CHECK: %[[T6:.*]] = llvm.shufflevector %[[T5]], %[[T3]] [0 : i32, 0 : i32, 0 : i32] : !llvm.vec<3 x float>, !llvm.vec<3 x float>
+//      CHECK: %[[T7:.*]] = llvm.fmul %[[T6]], %[[B]] : !llvm.vec<3 x float>
+//      CHECK: %[[T8:.*]] = llvm.insertvalue %[[T7]], %[[T0]][0] : !llvm.array<2 x vec<3 x float>>
 //      CHECK: %[[T9:.*]] = llvm.mlir.constant(1 : i64) : !llvm.i64
-//      CHECK: %[[T10:.*]] = llvm.extractelement %[[A]][%9 : !llvm.i64] : !llvm<"<2 x float>">
-//      CHECK: %[[T11:.*]] = llvm.mlir.undef : !llvm<"<3 x float>">
+//      CHECK: %[[T10:.*]] = llvm.extractelement %[[A]][%9 : !llvm.i64] : !llvm.vec<2 x float>
+//      CHECK: %[[T11:.*]] = llvm.mlir.undef : !llvm.vec<3 x float>
 //      CHECK: %[[T12:.*]] = llvm.mlir.constant(0 : i32) : !llvm.i32
-//      CHECK: %[[T13:.*]] = llvm.insertelement %[[T10]], %[[T11]][%12 : !llvm.i32] : !llvm<"<3 x float>">
-//      CHECK: %[[T14:.*]] = llvm.shufflevector %[[T13]], %[[T11]] [0 : i32, 0 : i32, 0 : i32] : !llvm<"<3 x float>">, !llvm<"<3 x float>">
-//      CHECK: %[[T15:.*]] = llvm.fmul %[[T14]], %[[B]] : !llvm<"<3 x float>">
-//      CHECK: %[[T16:.*]] = llvm.insertvalue %[[T15]], %[[T8]][1] : !llvm<"[2 x <3 x float>]">
-//      CHECK: llvm.return %[[T16]] : !llvm<"[2 x <3 x float>]">
+//      CHECK: %[[T13:.*]] = llvm.insertelement %[[T10]], %[[T11]][%12 : !llvm.i32] : !llvm.vec<3 x float>
+//      CHECK: %[[T14:.*]] = llvm.shufflevector %[[T13]], %[[T11]] [0 : i32, 0 : i32, 0 : i32] : !llvm.vec<3 x float>, !llvm.vec<3 x float>
+//      CHECK: %[[T15:.*]] = llvm.fmul %[[T14]], %[[B]] : !llvm.vec<3 x float>
+//      CHECK: %[[T16:.*]] = llvm.insertvalue %[[T15]], %[[T8]][1] : !llvm.array<2 x vec<3 x float>>
+//      CHECK: llvm.return %[[T16]] : !llvm.array<2 x vec<3 x float>>
 
 func @outerproduct_add(%arg0: vector<2xf32>, %arg1: vector<3xf32>, %arg2: vector<2x3xf32>) -> vector<2x3xf32> {
   %2 = vector.outerproduct %arg0, %arg1, %arg2 : vector<2xf32>, vector<3xf32>
   return %2 : vector<2x3xf32>
 }
 // CHECK-LABEL: llvm.func @outerproduct_add(
-// CHECK-SAME: %[[A:.*]]: !llvm<"<2 x float>">,
-// CHECK-SAME: %[[B:.*]]: !llvm<"<3 x float>">,
-// CHECK-SAME: %[[C:.*]]: !llvm<"[2 x <3 x float>]">)
+// CHECK-SAME: %[[A:.*]]: !llvm.vec<2 x float>,
+// CHECK-SAME: %[[B:.*]]: !llvm.vec<3 x float>,
+// CHECK-SAME: %[[C:.*]]: !llvm.array<2 x vec<3 x float>>)
 //      CHECK: %[[T0:.*]] = llvm.mlir.constant(dense<0.000000e+00> : vector<2x3xf32>)
 //      CHECK: %[[T1:.*]] = llvm.mlir.constant(0 : i64) : !llvm.i64
-//      CHECK: %[[T2:.*]] = llvm.extractelement %[[A]][%[[T1]] : !llvm.i64] : !llvm<"<2 x float>">
-//      CHECK: %[[T3:.*]] = llvm.mlir.undef : !llvm<"<3 x float>">
+//      CHECK: %[[T2:.*]] = llvm.extractelement %[[A]][%[[T1]] : !llvm.i64] : !llvm.vec<2 x float>
+//      CHECK: %[[T3:.*]] = llvm.mlir.undef : !llvm.vec<3 x float>
 //      CHECK: %[[T4:.*]] = llvm.mlir.constant(0 : i32) : !llvm.i32
-//      CHECK: %[[T5:.*]] = llvm.insertelement %[[T2]], %[[T3]][%[[T4]] : !llvm.i32] : !llvm<"<3 x float>">
-//      CHECK: %[[T6:.*]] = llvm.shufflevector %[[T5]], %[[T3]] [0 : i32, 0 : i32, 0 : i32] : !llvm<"<3 x float>">, !llvm<"<3 x float>">
-//      CHECK: %[[T7:.*]] = llvm.extractvalue %[[C]][0] : !llvm<"[2 x <3 x float>]">
-//      CHECK: %[[T8:.*]] = "llvm.intr.fmuladd"(%[[T6]], %[[B]], %[[T7]]) : (!llvm<"<3 x float>">, !llvm<"<3 x float>">, !llvm<"<3 x float>">)
-//      CHECK: %[[T9:.*]] = llvm.insertvalue %[[T8]], %[[T0]][0] : !llvm<"[2 x <3 x float>]">
+//      CHECK: %[[T5:.*]] = llvm.insertelement %[[T2]], %[[T3]][%[[T4]] : !llvm.i32] : !llvm.vec<3 x float>
+//      CHECK: %[[T6:.*]] = llvm.shufflevector %[[T5]], %[[T3]] [0 : i32, 0 : i32, 0 : i32] : !llvm.vec<3 x float>, !llvm.vec<3 x float>
+//      CHECK: %[[T7:.*]] = llvm.extractvalue %[[C]][0] : !llvm.array<2 x vec<3 x float>>
+//      CHECK: %[[T8:.*]] = "llvm.intr.fmuladd"(%[[T6]], %[[B]], %[[T7]]) : (!llvm.vec<3 x float>, !llvm.vec<3 x float>, !llvm.vec<3 x float>)
+//      CHECK: %[[T9:.*]] = llvm.insertvalue %[[T8]], %[[T0]][0] : !llvm.array<2 x vec<3 x float>>
 //      CHECK: %[[T10:.*]] = llvm.mlir.constant(1 : i64) : !llvm.i64
-//      CHECK: %[[T11:.*]] = llvm.extractelement %[[A]][%[[T10]] : !llvm.i64] : !llvm<"<2 x float>">
-//      CHECK: %[[T12:.*]] = llvm.mlir.undef : !llvm<"<3 x float>">
+//      CHECK: %[[T11:.*]] = llvm.extractelement %[[A]][%[[T10]] : !llvm.i64] : !llvm.vec<2 x float>
+//      CHECK: %[[T12:.*]] = llvm.mlir.undef : !llvm.vec<3 x float>
 //      CHECK: %[[T13:.*]] = llvm.mlir.constant(0 : i32) : !llvm.i32
-//      CHECK: %[[T14:.*]] = llvm.insertelement %[[T11]], %[[T12]][%[[T13]] : !llvm.i32] : !llvm<"<3 x float>">
-//      CHECK: %[[T15:.*]] = llvm.shufflevector %[[T14]], %[[T12]] [0 : i32, 0 : i32, 0 : i32] : !llvm<"<3 x float>">, !llvm<"<3 x float>">
-//      CHECK: %[[T16:.*]] = llvm.extractvalue %[[C]][1] : !llvm<"[2 x <3 x float>]">
-//      CHECK: %[[T17:.*]] = "llvm.intr.fmuladd"(%[[T15]], %[[B]], %[[T16]]) : (!llvm<"<3 x float>">, !llvm<"<3 x float>">, !llvm<"<3 x float>">)
-//      CHECK: %[[T18:.*]] = llvm.insertvalue %[[T17]], %[[T9]][1] : !llvm<"[2 x <3 x float>]">
-//      CHECK: llvm.return %[[T18]] : !llvm<"[2 x <3 x float>]">
+//      CHECK: %[[T14:.*]] = llvm.insertelement %[[T11]], %[[T12]][%[[T13]] : !llvm.i32] : !llvm.vec<3 x float>
+//      CHECK: %[[T15:.*]] = llvm.shufflevector %[[T14]], %[[T12]] [0 : i32, 0 : i32, 0 : i32] : !llvm.vec<3 x float>, !llvm.vec<3 x float>
+//      CHECK: %[[T16:.*]] = llvm.extractvalue %[[C]][1] : !llvm.array<2 x vec<3 x float>>
+//      CHECK: %[[T17:.*]] = "llvm.intr.fmuladd"(%[[T15]], %[[B]], %[[T16]]) : (!llvm.vec<3 x float>, !llvm.vec<3 x float>, !llvm.vec<3 x float>)
+//      CHECK: %[[T18:.*]] = llvm.insertvalue %[[T17]], %[[T9]][1] : !llvm.array<2 x vec<3 x float>>
+//      CHECK: llvm.return %[[T18]] : !llvm.array<2 x vec<3 x float>>
 
 func @shuffle_1D_direct(%arg0: vector<2xf32>, %arg1: vector<2xf32>) -> vector<2xf32> {
   %1 = vector.shuffle %arg0, %arg1 [0, 1] : vector<2xf32>, vector<2xf32>
   return %1 : vector<2xf32>
 }
 // CHECK-LABEL: llvm.func @shuffle_1D_direct(
-// CHECK-SAME: %[[A:.*]]: !llvm<"<2 x float>">,
-// CHECK-SAME: %[[B:.*]]: !llvm<"<2 x float>">)
-//       CHECK:   %[[s:.*]] = llvm.shufflevector %[[A]], %[[B]] [0, 1] : !llvm<"<2 x float>">, !llvm<"<2 x float>">
-//       CHECK:   llvm.return %[[s]] : !llvm<"<2 x float>">
+// CHECK-SAME: %[[A:.*]]: !llvm.vec<2 x float>,
+// CHECK-SAME: %[[B:.*]]: !llvm.vec<2 x float>)
+//       CHECK:   %[[s:.*]] = llvm.shufflevector %[[A]], %[[B]] [0, 1] : !llvm.vec<2 x float>, !llvm.vec<2 x float>
+//       CHECK:   llvm.return %[[s]] : !llvm.vec<2 x float>
 
 func @shuffle_1D(%arg0: vector<2xf32>, %arg1: vector<3xf32>) -> vector<5xf32> {
   %1 = vector.shuffle %arg0, %arg1 [4, 3, 2, 1, 0] : vector<2xf32>, vector<3xf32>
   return %1 : vector<5xf32>
 }
 // CHECK-LABEL: llvm.func @shuffle_1D(
-// CHECK-SAME: %[[A:.*]]: !llvm<"<2 x float>">,
-// CHECK-SAME: %[[B:.*]]: !llvm<"<3 x float>">)
-//       CHECK:   %[[u0:.*]] = llvm.mlir.undef : !llvm<"<5 x float>">
+// CHECK-SAME: %[[A:.*]]: !llvm.vec<2 x float>,
+// CHECK-SAME: %[[B:.*]]: !llvm.vec<3 x float>)
+//       CHECK:   %[[u0:.*]] = llvm.mlir.undef : !llvm.vec<5 x float>
 //       CHECK:   %[[c2:.*]] = llvm.mlir.constant(2 : index) : !llvm.i64
-//       CHECK:   %[[e1:.*]] = llvm.extractelement %[[B]][%[[c2]] : !llvm.i64] : !llvm<"<3 x float>">
+//       CHECK:   %[[e1:.*]] = llvm.extractelement %[[B]][%[[c2]] : !llvm.i64] : !llvm.vec<3 x float>
 //       CHECK:   %[[c0:.*]] = llvm.mlir.constant(0 : index) : !llvm.i64
-//       CHECK:   %[[i1:.*]] = llvm.insertelement %[[e1]], %[[u0]][%[[c0]] : !llvm.i64] : !llvm<"<5 x float>">
+//       CHECK:   %[[i1:.*]] = llvm.insertelement %[[e1]], %[[u0]][%[[c0]] : !llvm.i64] : !llvm.vec<5 x float>
 //       CHECK:   %[[c1:.*]] = llvm.mlir.constant(1 : index) : !llvm.i64
-//       CHECK:   %[[e2:.*]] = llvm.extractelement %[[B]][%[[c1]] : !llvm.i64] : !llvm<"<3 x float>">
+//       CHECK:   %[[e2:.*]] = llvm.extractelement %[[B]][%[[c1]] : !llvm.i64] : !llvm.vec<3 x float>
 //       CHECK:   %[[c1:.*]] = llvm.mlir.constant(1 : index) : !llvm.i64
-//       CHECK:   %[[i2:.*]] = llvm.insertelement %[[e2]], %[[i1]][%[[c1]] : !llvm.i64] : !llvm<"<5 x float>">
+//       CHECK:   %[[i2:.*]] = llvm.insertelement %[[e2]], %[[i1]][%[[c1]] : !llvm.i64] : !llvm.vec<5 x float>
 //       CHECK:   %[[c0:.*]] = llvm.mlir.constant(0 : index) : !llvm.i64
-//       CHECK:   %[[e3:.*]] = llvm.extractelement %[[B]][%[[c0]] : !llvm.i64] : !llvm<"<3 x float>">
+//       CHECK:   %[[e3:.*]] = llvm.extractelement %[[B]][%[[c0]] : !llvm.i64] : !llvm.vec<3 x float>
 //       CHECK:   %[[c2:.*]] = llvm.mlir.constant(2 : index) : !llvm.i64
-//       CHECK:   %[[i3:.*]] = llvm.insertelement %[[e3]], %[[i2]][%[[c2]] : !llvm.i64] : !llvm<"<5 x float>">
+//       CHECK:   %[[i3:.*]] = llvm.insertelement %[[e3]], %[[i2]][%[[c2]] : !llvm.i64] : !llvm.vec<5 x float>
 //       CHECK:   %[[c1:.*]] = llvm.mlir.constant(1 : index) : !llvm.i64
-//       CHECK:   %[[e4:.*]] = llvm.extractelement %[[A]][%[[c1]] : !llvm.i64] : !llvm<"<2 x float>">
+//       CHECK:   %[[e4:.*]] = llvm.extractelement %[[A]][%[[c1]] : !llvm.i64] : !llvm.vec<2 x float>
 //       CHECK:   %[[c3:.*]] = llvm.mlir.constant(3 : index) : !llvm.i64
-//       CHECK:   %[[i4:.*]] = llvm.insertelement %[[e4]], %[[i3]][%[[c3]] : !llvm.i64] : !llvm<"<5 x float>">
+//       CHECK:   %[[i4:.*]] = llvm.insertelement %[[e4]], %[[i3]][%[[c3]] : !llvm.i64] : !llvm.vec<5 x float>
 //       CHECK:   %[[c0:.*]] = llvm.mlir.constant(0 : index) : !llvm.i64
-//       CHECK:   %[[e5:.*]] = llvm.extractelement %[[A]][%[[c0]] : !llvm.i64] : !llvm<"<2 x float>">
+//       CHECK:   %[[e5:.*]] = llvm.extractelement %[[A]][%[[c0]] : !llvm.i64] : !llvm.vec<2 x float>
 //       CHECK:   %[[c4:.*]] = llvm.mlir.constant(4 : index) : !llvm.i64
-//       CHECK:   %[[i5:.*]] = llvm.insertelement %[[e5]], %[[i4]][%[[c4]] : !llvm.i64] : !llvm<"<5 x float>">
-//       CHECK:   llvm.return %[[i5]] : !llvm<"<5 x float>">
+//       CHECK:   %[[i5:.*]] = llvm.insertelement %[[e5]], %[[i4]][%[[c4]] : !llvm.i64] : !llvm.vec<5 x float>
+//       CHECK:   llvm.return %[[i5]] : !llvm.vec<5 x float>
 
 func @shuffle_2D(%a: vector<1x4xf32>, %b: vector<2x4xf32>) -> vector<3x4xf32> {
   %1 = vector.shuffle %a, %b[1, 0, 2] : vector<1x4xf32>, vector<2x4xf32>
   return %1 : vector<3x4xf32>
 }
 // CHECK-LABEL: llvm.func @shuffle_2D(
-// CHECK-SAME: %[[A:.*]]: !llvm<"[1 x <4 x float>]">,
-// CHECK-SAME: %[[B:.*]]: !llvm<"[2 x <4 x float>]">)
-//       CHECK:   %[[u0:.*]] = llvm.mlir.undef : !llvm<"[3 x <4 x float>]">
-//       CHECK:   %[[e1:.*]] = llvm.extractvalue %[[B]][0] : !llvm<"[2 x <4 x float>]">
-//       CHECK:   %[[i1:.*]] = llvm.insertvalue %[[e1]], %[[u0]][0] : !llvm<"[3 x <4 x float>]">
-//       CHECK:   %[[e2:.*]] = llvm.extractvalue %[[A]][0] : !llvm<"[1 x <4 x float>]">
-//       CHECK:   %[[i2:.*]] = llvm.insertvalue %[[e2]], %[[i1]][1] : !llvm<"[3 x <4 x float>]">
-//       CHECK:   %[[e3:.*]] = llvm.extractvalue %[[B]][1] : !llvm<"[2 x <4 x float>]">
-//       CHECK:   %[[i3:.*]] = llvm.insertvalue %[[e3]], %[[i2]][2] : !llvm<"[3 x <4 x float>]">
-//       CHECK:   llvm.return %[[i3]] : !llvm<"[3 x <4 x float>]">
+// CHECK-SAME: %[[A:.*]]: !llvm.array<1 x vec<4 x float>>,
+// CHECK-SAME: %[[B:.*]]: !llvm.array<2 x vec<4 x float>>)
+//       CHECK:   %[[u0:.*]] = llvm.mlir.undef : !llvm.array<3 x vec<4 x float>>
+//       CHECK:   %[[e1:.*]] = llvm.extractvalue %[[B]][0] : !llvm.array<2 x vec<4 x float>>
+//       CHECK:   %[[i1:.*]] = llvm.insertvalue %[[e1]], %[[u0]][0] : !llvm.array<3 x vec<4 x float>>
+//       CHECK:   %[[e2:.*]] = llvm.extractvalue %[[A]][0] : !llvm.array<1 x vec<4 x float>>
+//       CHECK:   %[[i2:.*]] = llvm.insertvalue %[[e2]], %[[i1]][1] : !llvm.array<3 x vec<4 x float>>
+//       CHECK:   %[[e3:.*]] = llvm.extractvalue %[[B]][1] : !llvm.array<2 x vec<4 x float>>
+//       CHECK:   %[[i3:.*]] = llvm.insertvalue %[[e3]], %[[i2]][2] : !llvm.array<3 x vec<4 x float>>
+//       CHECK:   llvm.return %[[i3]] : !llvm.array<3 x vec<4 x float>>
 
 func @extract_element(%arg0: vector<16xf32>) -> f32 {
   %0 = constant 15 : i32
@@ -311,9 +311,9 @@ func @extract_element(%arg0: vector<16xf32>) -> f32 {
   return %1 : f32
 }
 // CHECK-LABEL: llvm.func @extract_element(
-// CHECK-SAME: %[[A:.*]]: !llvm<"<16 x float>">)
+// CHECK-SAME: %[[A:.*]]: !llvm.vec<16 x float>)
 //       CHECK:   %[[c:.*]] = llvm.mlir.constant(15 : i32) : !llvm.i32
-//       CHECK:   %[[x:.*]] = llvm.extractelement %[[A]][%[[c]] : !llvm.i32] : !llvm<"<16 x float>">
+//       CHECK:   %[[x:.*]] = llvm.extractelement %[[A]][%[[c]] : !llvm.i32] : !llvm.vec<16 x float>
 //       CHECK:   llvm.return %[[x]] : !llvm.float
 
 func @extract_element_from_vec_1d(%arg0: vector<16xf32>) -> f32 {
@@ -322,7 +322,7 @@ func @extract_element_from_vec_1d(%arg0: vector<16xf32>) -> f32 {
 }
 // CHECK-LABEL: llvm.func @extract_element_from_vec_1d
 //       CHECK:   llvm.mlir.constant(15 : i64) : !llvm.i64
-//       CHECK:   llvm.extractelement {{.*}}[{{.*}} : !llvm.i64] : !llvm<"<16 x float>">
+//       CHECK:   llvm.extractelement {{.*}}[{{.*}} : !llvm.i64] : !llvm.vec<16 x float>
 //       CHECK:   llvm.return {{.*}} : !llvm.float
 
 func @extract_vec_2d_from_vec_3d(%arg0: vector<4x3x16xf32>) -> vector<3x16xf32> {
@@ -330,25 +330,25 @@ func @extract_vec_2d_from_vec_3d(%arg0: vector<4x3x16xf32>) -> vector<3x16xf32>
   return %0 : vector<3x16xf32>
 }
 // CHECK-LABEL: llvm.func @extract_vec_2d_from_vec_3d
-//       CHECK:   llvm.extractvalue {{.*}}[0] : !llvm<"[4 x [3 x <16 x float>]]">
-//       CHECK:   llvm.return {{.*}} : !llvm<"[3 x <16 x float>]">
+//       CHECK:   llvm.extractvalue {{.*}}[0] : !llvm.array<4 x array<3 x vec<16 x float>>>
+//       CHECK:   llvm.return {{.*}} : !llvm.array<3 x vec<16 x float>>
 
 func @extract_vec_1d_from_vec_3d(%arg0: vector<4x3x16xf32>) -> vector<16xf32> {
   %0 = vector.extract %arg0[0, 0]: vector<4x3x16xf32>
   return %0 : vector<16xf32>
 }
 // CHECK-LABEL: llvm.func @extract_vec_1d_from_vec_3d
-//       CHECK:   llvm.extractvalue {{.*}}[0, 0] : !llvm<"[4 x [3 x <16 x float>]]">
-//       CHECK:   llvm.return {{.*}} : !llvm<"<16 x float>">
+//       CHECK:   llvm.extractvalue {{.*}}[0, 0] : !llvm.array<4 x array<3 x vec<16 x float>>>
+//       CHECK:   llvm.return {{.*}} : !llvm.vec<16 x float>
 
 func @extract_element_from_vec_3d(%arg0: vector<4x3x16xf32>) -> f32 {
   %0 = vector.extract %arg0[0, 0, 0]: vector<4x3x16xf32>
   return %0 : f32
 }
 // CHECK-LABEL: llvm.func @extract_element_from_vec_3d
-//       CHECK:   llvm.extractvalue {{.*}}[0, 0] : !llvm<"[4 x [3 x <16 x float>]]">
+//       CHECK:   llvm.extractvalue {{.*}}[0, 0] : !llvm.array<4 x array<3 x vec<16 x float>>>
 //       CHECK:   llvm.mlir.constant(0 : i64) : !llvm.i64
-//       CHECK:   llvm.extractelement {{.*}}[{{.*}} : !llvm.i64] : !llvm<"<16 x float>">
+//       CHECK:   llvm.extractelement {{.*}}[{{.*}} : !llvm.i64] : !llvm.vec<16 x float>
 //       CHECK:   llvm.return {{.*}} : !llvm.float
 
 func @insert_element(%arg0: f32, %arg1: vector<4xf32>) -> vector<4xf32> {
@@ -358,10 +358,10 @@ func @insert_element(%arg0: f32, %arg1: vector<4xf32>) -> vector<4xf32> {
 }
 // CHECK-LABEL: llvm.func @insert_element(
 // CHECK-SAME: %[[A:.*]]: !llvm.float,
-// CHECK-SAME: %[[B:.*]]: !llvm<"<4 x float>">)
+// CHECK-SAME: %[[B:.*]]: !llvm.vec<4 x float>)
 //       CHECK:   %[[c:.*]] = llvm.mlir.constant(3 : i32) : !llvm.i32
-//       CHECK:   %[[x:.*]] = llvm.insertelement %[[A]], %[[B]][%[[c]] : !llvm.i32] : !llvm<"<4 x float>">
-//       CHECK:   llvm.return %[[x]] : !llvm<"<4 x float>">
+//       CHECK:   %[[x:.*]] = llvm.insertelement %[[A]], %[[B]][%[[c]] : !llvm.i32] : !llvm.vec<4 x float>
+//       CHECK:   llvm.return %[[x]] : !llvm.vec<4 x float>
 
 func @insert_element_into_vec_1d(%arg0: f32, %arg1: vector<4xf32>) -> vector<4xf32> {
   %0 = vector.insert %arg0, %arg1[3] : f32 into vector<4xf32>
@@ -369,65 +369,65 @@ func @insert_element_into_vec_1d(%arg0: f32, %arg1: vector<4xf32>) -> vector<4xf
 }
 // CHECK-LABEL: llvm.func @insert_element_into_vec_1d
 //       CHECK:   llvm.mlir.constant(3 : i64) : !llvm.i64
-//       CHECK:   llvm.insertelement {{.*}}, {{.*}}[{{.*}} : !llvm.i64] : !llvm<"<4 x float>">
-//       CHECK:   llvm.return {{.*}} : !llvm<"<4 x float>">
+//       CHECK:   llvm.insertelement {{.*}}, {{.*}}[{{.*}} : !llvm.i64] : !llvm.vec<4 x float>
+//       CHECK:   llvm.return {{.*}} : !llvm.vec<4 x float>
 
 func @insert_vec_2d_into_vec_3d(%arg0: vector<8x16xf32>, %arg1: vector<4x8x16xf32>) -> vector<4x8x16xf32> {
   %0 = vector.insert %arg0, %arg1[3] : vector<8x16xf32> into vector<4x8x16xf32>
   return %0 : vector<4x8x16xf32>
 }
 // CHECK-LABEL: llvm.func @insert_vec_2d_into_vec_3d
-//       CHECK:   llvm.insertvalue {{.*}}, {{.*}}[3] : !llvm<"[4 x [8 x <16 x float>]]">
-//       CHECK:   llvm.return {{.*}} : !llvm<"[4 x [8 x <16 x float>]]">
+//       CHECK:   llvm.insertvalue {{.*}}, {{.*}}[3] : !llvm.array<4 x array<8 x vec<16 x float>>>
+//       CHECK:   llvm.return {{.*}} : !llvm.array<4 x array<8 x vec<16 x float>>>
 
 func @insert_vec_1d_into_vec_3d(%arg0: vector<16xf32>, %arg1: vector<4x8x16xf32>) -> vector<4x8x16xf32> {
   %0 = vector.insert %arg0, %arg1[3, 7] : vector<16xf32> into vector<4x8x16xf32>
   return %0 : vector<4x8x16xf32>
 }
 // CHECK-LABEL: llvm.func @insert_vec_1d_into_vec_3d
-//       CHECK:   llvm.insertvalue {{.*}}, {{.*}}[3, 7] : !llvm<"[4 x [8 x <16 x float>]]">
-//       CHECK:   llvm.return {{.*}} : !llvm<"[4 x [8 x <16 x float>]]">
+//       CHECK:   llvm.insertvalue {{.*}}, {{.*}}[3, 7] : !llvm.array<4 x array<8 x vec<16 x float>>>
+//       CHECK:   llvm.return {{.*}} : !llvm.array<4 x array<8 x vec<16 x float>>>
 
 func @insert_element_into_vec_3d(%arg0: f32, %arg1: vector<4x8x16xf32>) -> vector<4x8x16xf32> {
   %0 = vector.insert %arg0, %arg1[3, 7, 15] : f32 into vector<4x8x16xf32>
   return %0 : vector<4x8x16xf32>
 }
 // CHECK-LABEL: llvm.func @insert_element_into_vec_3d
-//       CHECK:   llvm.extractvalue {{.*}}[3, 7] : !llvm<"[4 x [8 x <16 x float>]]">
+//       CHECK:   llvm.extractvalue {{.*}}[3, 7] : !llvm.array<4 x array<8 x vec<16 x float>>>
 //       CHECK:   llvm.mlir.constant(15 : i64) : !llvm.i64
-//       CHECK:   llvm.insertelement {{.*}}, {{.*}}[{{.*}} : !llvm.i64] : !llvm<"<16 x float>">
-//       CHECK:   llvm.insertvalue {{.*}}, {{.*}}[3, 7] : !llvm<"[4 x [8 x <16 x float>]]">
-//       CHECK:   llvm.return {{.*}} : !llvm<"[4 x [8 x <16 x float>]]">
+//       CHECK:   llvm.insertelement {{.*}}, {{.*}}[{{.*}} : !llvm.i64] : !llvm.vec<16 x float>
+//       CHECK:   llvm.insertvalue {{.*}}, {{.*}}[3, 7] : !llvm.array<4 x array<8 x vec<16 x float>>>
+//       CHECK:   llvm.return {{.*}} : !llvm.array<4 x array<8 x vec<16 x float>>>
 
 func @vector_type_cast(%arg0: memref<8x8x8xf32>) -> memref<vector<8x8x8xf32>> {
   %0 = vector.type_cast %arg0: memref<8x8x8xf32> to memref<vector<8x8x8xf32>>
   return %0 : memref<vector<8x8x8xf32>>
 }
 // CHECK-LABEL: llvm.func @vector_type_cast
-//       CHECK:   llvm.mlir.undef : !llvm<"{ [8 x [8 x <8 x float>]]*, [8 x [8 x <8 x float>]]*, i64 }">
-//       CHECK:   %[[allocated:.*]] = llvm.extractvalue {{.*}}[0] : !llvm<"{ float*, float*, i64, [3 x i64], [3 x i64] }">
-//       CHECK:   %[[allocatedBit:.*]] = llvm.bitcast %[[allocated]] : !llvm<"float*"> to !llvm<"[8 x [8 x <8 x float>]]*">
-//       CHECK:   llvm.insertvalue %[[allocatedBit]], {{.*}}[0] : !llvm<"{ [8 x [8 x <8 x float>]]*, [8 x [8 x <8 x float>]]*, i64 }">
-//       CHECK:   %[[aligned:.*]] = llvm.extractvalue {{.*}}[1] : !llvm<"{ float*, float*, i64, [3 x i64], [3 x i64] }">
-//       CHECK:   %[[alignedBit:.*]] = llvm.bitcast %[[aligned]] : !llvm<"float*"> to !llvm<"[8 x [8 x <8 x float>]]*">
-//       CHECK:   llvm.insertvalue %[[alignedBit]], {{.*}}[1] : !llvm<"{ [8 x [8 x <8 x float>]]*, [8 x [8 x <8 x float>]]*, i64 }">
+//       CHECK:   llvm.mlir.undef : !llvm.struct<(ptr<array<8 x array<8 x vec<8 x float>>>>, ptr<array<8 x array<8 x vec<8 x float>>>>, i64)>
+//       CHECK:   %[[allocated:.*]] = llvm.extractvalue {{.*}}[0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<3 x i64>, array<3 x i64>)>
+//       CHECK:   %[[allocatedBit:.*]] = llvm.bitcast %[[allocated]] : !llvm.ptr<float> to !llvm.ptr<array<8 x array<8 x vec<8 x float>>>>
+//       CHECK:   llvm.insertvalue %[[allocatedBit]], {{.*}}[0] : !llvm.struct<(ptr<array<8 x array<8 x vec<8 x float>>>>, ptr<array<8 x array<8 x vec<8 x float>>>>, i64)>
+//       CHECK:   %[[aligned:.*]] = llvm.extractvalue {{.*}}[1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<3 x i64>, array<3 x i64>)>
+//       CHECK:   %[[alignedBit:.*]] = llvm.bitcast %[[aligned]] : !llvm.ptr<float> to !llvm.ptr<array<8 x array<8 x vec<8 x float>>>>
+//       CHECK:   llvm.insertvalue %[[alignedBit]], {{.*}}[1] : !llvm.struct<(ptr<array<8 x array<8 x vec<8 x float>>>>, ptr<array<8 x array<8 x vec<8 x float>>>>, i64)>
 //       CHECK:   llvm.mlir.constant(0 : index
-//       CHECK:   llvm.insertvalue {{.*}}[2] : !llvm<"{ [8 x [8 x <8 x float>]]*, [8 x [8 x <8 x float>]]*, i64 }">
+//       CHECK:   llvm.insertvalue {{.*}}[2] : !llvm.struct<(ptr<array<8 x array<8 x vec<8 x float>>>>, ptr<array<8 x array<8 x vec<8 x float>>>>, i64)>
 
 func @vector_type_cast_non_zero_addrspace(%arg0: memref<8x8x8xf32, 3>) -> memref<vector<8x8x8xf32>, 3> {
   %0 = vector.type_cast %arg0: memref<8x8x8xf32, 3> to memref<vector<8x8x8xf32>, 3>
   return %0 : memref<vector<8x8x8xf32>, 3>
 }
 // CHECK-LABEL: llvm.func @vector_type_cast_non_zero_addrspace
-//       CHECK:   llvm.mlir.undef : !llvm<"{ [8 x [8 x <8 x float>]] addrspace(3)*, [8 x [8 x <8 x float>]] addrspace(3)*, i64 }">
-//       CHECK:   %[[allocated:.*]] = llvm.extractvalue {{.*}}[0] : !llvm<"{ float addrspace(3)*, float addrspace(3)*, i64, [3 x i64], [3 x i64] }">
-//       CHECK:   %[[allocatedBit:.*]] = llvm.bitcast %[[allocated]] : !llvm<"float addrspace(3)*"> to !llvm<"[8 x [8 x <8 x float>]] addrspace(3)*">
-//       CHECK:   llvm.insertvalue %[[allocatedBit]], {{.*}}[0] : !llvm<"{ [8 x [8 x <8 x float>]] addrspace(3)*, [8 x [8 x <8 x float>]] addrspace(3)*, i64 }">
-//       CHECK:   %[[aligned:.*]] = llvm.extractvalue {{.*}}[1] : !llvm<"{ float addrspace(3)*, float addrspace(3)*, i64, [3 x i64], [3 x i64] }">
-//       CHECK:   %[[alignedBit:.*]] = llvm.bitcast %[[aligned]] : !llvm<"float addrspace(3)*"> to !llvm<"[8 x [8 x <8 x float>]] addrspace(3)*">
-//       CHECK:   llvm.insertvalue %[[alignedBit]], {{.*}}[1] : !llvm<"{ [8 x [8 x <8 x float>]] addrspace(3)*, [8 x [8 x <8 x float>]] addrspace(3)*, i64 }">
+//       CHECK:   llvm.mlir.undef : !llvm.struct<(ptr<array<8 x array<8 x vec<8 x float>>>, 3>, ptr<array<8 x array<8 x vec<8 x float>>>, 3>, i64)>
+//       CHECK:   %[[allocated:.*]] = llvm.extractvalue {{.*}}[0] : !llvm.struct<(ptr<float, 3>, ptr<float, 3>, i64, array<3 x i64>, array<3 x i64>)>
+//       CHECK:   %[[allocatedBit:.*]] = llvm.bitcast %[[allocated]] : !llvm.ptr<float, 3> to !llvm.ptr<array<8 x array<8 x vec<8 x float>>>, 3>
+//       CHECK:   llvm.insertvalue %[[allocatedBit]], {{.*}}[0] : !llvm.struct<(ptr<array<8 x array<8 x vec<8 x float>>>, 3>, ptr<array<8 x array<8 x vec<8 x float>>>, 3>, i64)>
+//       CHECK:   %[[aligned:.*]] = llvm.extractvalue {{.*}}[1] : !llvm.struct<(ptr<float, 3>, ptr<float, 3>, i64, array<3 x i64>, array<3 x i64>)>
+//       CHECK:   %[[alignedBit:.*]] = llvm.bitcast %[[aligned]] : !llvm.ptr<float, 3> to !llvm.ptr<array<8 x array<8 x vec<8 x float>>>, 3>
+//       CHECK:   llvm.insertvalue %[[alignedBit]], {{.*}}[1] : !llvm.struct<(ptr<array<8 x array<8 x vec<8 x float>>>, 3>, ptr<array<8 x array<8 x vec<8 x float>>>, 3>, i64)>
 //       CHECK:   llvm.mlir.constant(0 : index
-//       CHECK:   llvm.insertvalue {{.*}}[2] : !llvm<"{ [8 x [8 x <8 x float>]] addrspace(3)*, [8 x [8 x <8 x float>]] addrspace(3)*, i64 }">
+//       CHECK:   llvm.insertvalue {{.*}}[2] : !llvm.struct<(ptr<array<8 x array<8 x vec<8 x float>>>, 3>, ptr<array<8 x array<8 x vec<8 x float>>>, 3>, i64)>
 
 func @vector_print_scalar_i1(%arg0: i1) {
   vector.print %arg0 : i1
@@ -482,27 +482,27 @@ func @vector_print_vector(%arg0: vector<2x2xf32>) {
   return
 }
 // CHECK-LABEL: llvm.func @vector_print_vector(
-// CHECK-SAME: %[[A:.*]]: !llvm<"[2 x <2 x float>]">)
+// CHECK-SAME: %[[A:.*]]: !llvm.array<2 x vec<2 x float>>)
 //       CHECK:    llvm.call @print_open() : () -> ()
-//       CHECK:    %[[x0:.*]] = llvm.extractvalue %[[A]][0] : !llvm<"[2 x <2 x float>]">
+//       CHECK:    %[[x0:.*]] = llvm.extractvalue %[[A]][0] : !llvm.array<2 x vec<2 x float>>
 //       CHECK:    llvm.call @print_open() : () -> ()
 //       CHECK:    %[[x1:.*]] = llvm.mlir.constant(0 : index) : !llvm.i64
-//       CHECK:    %[[x2:.*]] = llvm.extractelement %[[x0]][%[[x1]] : !llvm.i64] : !llvm<"<2 x float>">
+//       CHECK:    %[[x2:.*]] = llvm.extractelement %[[x0]][%[[x1]] : !llvm.i64] : !llvm.vec<2 x float>
 //       CHECK:    llvm.call @print_f32(%[[x2]]) : (!llvm.float) -> ()
 //       CHECK:    llvm.call @print_comma() : () -> ()
 //       CHECK:    %[[x3:.*]] = llvm.mlir.constant(1 : index) : !llvm.i64
-//       CHECK:    %[[x4:.*]] = llvm.extractelement %[[x0]][%[[x3]] : !llvm.i64] : !llvm<"<2 x float>">
+//       CHECK:    %[[x4:.*]] = llvm.extractelement %[[x0]][%[[x3]] : !llvm.i64] : !llvm.vec<2 x float>
 //       CHECK:    llvm.call @print_f32(%[[x4]]) : (!llvm.float) -> ()
 //       CHECK:    llvm.call @print_close() : () -> ()
 //       CHECK:    llvm.call @print_comma() : () -> ()
-//       CHECK:    %[[x5:.*]] = llvm.extractvalue %[[A]][1] : !llvm<"[2 x <2 x float>]">
+//       CHECK:    %[[x5:.*]] = llvm.extractvalue %[[A]][1] : !llvm.array<2 x vec<2 x float>>
 //       CHECK:    llvm.call @print_open() : () -> ()
 //       CHECK:    %[[x6:.*]] = llvm.mlir.constant(0 : index) : !llvm.i64
-//       CHECK:    %[[x7:.*]] = llvm.extractelement %[[x5]][%[[x6]] : !llvm.i64] : !llvm<"<2 x float>">
+//       CHECK:    %[[x7:.*]] = llvm.extractelement %[[x5]][%[[x6]] : !llvm.i64] : !llvm.vec<2 x float>
 //       CHECK:    llvm.call @print_f32(%[[x7]]) : (!llvm.float) -> ()
 //       CHECK:    llvm.call @print_comma() : () -> ()
 //       CHECK:    %[[x8:.*]] = llvm.mlir.constant(1 : index) : !llvm.i64
-//       CHECK:    %[[x9:.*]] = llvm.extractelement %[[x5]][%[[x8]] : !llvm.i64] : !llvm<"<2 x float>">
+//       CHECK:    %[[x9:.*]] = llvm.extractelement %[[x5]][%[[x8]] : !llvm.i64] : !llvm.vec<2 x float>
 //       CHECK:    llvm.call @print_f32(%[[x9]]) : (!llvm.float) -> ()
 //       CHECK:    llvm.call @print_close() : () -> ()
 //       CHECK:    llvm.call @print_close() : () -> ()
@@ -514,15 +514,15 @@ func @extract_strided_slice1(%arg0: vector<4xf32>) -> vector<2xf32> {
 }
 // CHECK-LABEL: llvm.func @extract_strided_slice1
 //       CHECK:    llvm.mlir.constant(0.000000e+00 : f32) : !llvm.float
-//       CHECK:    llvm.mlir.constant(dense<0.000000e+00> : vector<2xf32>) : !llvm<"<2 x float>">
+//       CHECK:    llvm.mlir.constant(dense<0.000000e+00> : vector<2xf32>) : !llvm.vec<2 x float>
 //       CHECK:    llvm.mlir.constant(2 : index) : !llvm.i64
-//       CHECK:    llvm.extractelement %{{.*}}[%{{.*}} : !llvm.i64] : !llvm<"<4 x float>">
+//       CHECK:    llvm.extractelement %{{.*}}[%{{.*}} : !llvm.i64] : !llvm.vec<4 x float>
 //       CHECK:    llvm.mlir.constant(0 : index) : !llvm.i64
-//       CHECK:    llvm.insertelement %{{.*}}, %{{.*}}[%{{.*}} : !llvm.i64] : !llvm<"<2 x float>">
+//       CHECK:    llvm.insertelement %{{.*}}, %{{.*}}[%{{.*}} : !llvm.i64] : !llvm.vec<2 x float>
 //       CHECK:    llvm.mlir.constant(3 : index) : !llvm.i64
-//       CHECK:    llvm.extractelement %{{.*}}[%{{.*}} : !llvm.i64] : !llvm<"<4 x float>">
+//       CHECK:    llvm.extractelement %{{.*}}[%{{.*}} : !llvm.i64] : !llvm.vec<4 x float>
 //       CHECK:    llvm.mlir.constant(1 : index) : !llvm.i64
-//       CHECK:    llvm.insertelement %{{.*}}, %{{.*}}[%{{.*}} : !llvm.i64] : !llvm<"<2 x float>">
+//       CHECK:    llvm.insertelement %{{.*}}, %{{.*}}[%{{.*}} : !llvm.i64] : !llvm.vec<2 x float>
 
 func @extract_strided_slice2(%arg0: vector<4x8xf32>) -> vector<2x8xf32> {
   %0 = vector.extract_strided_slice %arg0 {offsets = [2], sizes = [2], strides = [1]} : vector<4x8xf32> to vector<2x8xf32>
@@ -530,11 +530,11 @@ func @extract_strided_slice2(%arg0: vector<4x8xf32>) -> vector<2x8xf32> {
 }
 // CHECK-LABEL: llvm.func @extract_strided_slice2
 //       CHECK:    llvm.mlir.constant(0.000000e+00 : f32) : !llvm.float
-//       CHECK:    llvm.mlir.constant(dense<0.000000e+00> : vector<2x8xf32>) : !llvm<"[2 x <8 x float>]">
-//       CHECK:    llvm.extractvalue %{{.*}}[2] : !llvm<"[4 x <8 x float>]">
-//       CHECK:    llvm.insertvalue %{{.*}}, %{{.*}}[0] : !llvm<"[2 x <8 x float>]">
-//       CHECK:    llvm.extractvalue %{{.*}}[3] : !llvm<"[4 x <8 x float>]">
-//       CHECK:    llvm.insertvalue %{{.*}}, %{{.*}}[1] : !llvm<"[2 x <8 x float>]">
+//       CHECK:    llvm.mlir.constant(dense<0.000000e+00> : vector<2x8xf32>) : !llvm.array<2 x vec<8 x float>>
+//       CHECK:    llvm.extractvalue %{{.*}}[2] : !llvm.array<4 x vec<8 x float>>
+//       CHECK:    llvm.insertvalue %{{.*}}, %{{.*}}[0] : !llvm.array<2 x vec<8 x float>>
+//       CHECK:    llvm.extractvalue %{{.*}}[3] : !llvm.array<4 x vec<8 x float>>
+//       CHECK:    llvm.insertvalue %{{.*}}, %{{.*}}[1] : !llvm.array<2 x vec<8 x float>>
 
 func @extract_strided_slice3(%arg0: vector<4x8xf32>) -> vector<2x2xf32> {
   %0 = vector.extract_strided_slice %arg0 {offsets = [2, 2], sizes = [2, 2], strides = [1, 1]} : vector<4x8xf32> to vector<2x2xf32>
@@ -542,43 +542,43 @@ func @extract_strided_slice3(%arg0: vector<4x8xf32>) -> vector<2x2xf32> {
 }
 // CHECK-LABEL: llvm.func @extract_strided_slice3
 //       CHECK:    llvm.mlir.constant(0.000000e+00 : f32) : !llvm.float
-//       CHECK:    llvm.mlir.constant(dense<0.000000e+00> : vector<2x2xf32>) : !llvm<"[2 x <2 x float>]">
+//       CHECK:    llvm.mlir.constant(dense<0.000000e+00> : vector<2x2xf32>) : !llvm.array<2 x vec<2 x float>>
 //
 // Subvector vector<8xf32> @2
-//       CHECK:    llvm.extractvalue {{.*}}[2] : !llvm<"[4 x <8 x float>]">
+//       CHECK:    llvm.extractvalue {{.*}}[2] : !llvm.array<4 x vec<8 x float>>
 //       CHECK:    llvm.mlir.constant(0.000000e+00 : f32) : !llvm.float
-//       CHECK:    llvm.mlir.constant(dense<0.000000e+00> : vector<2xf32>) : !llvm<"<2 x float>">
+//       CHECK:    llvm.mlir.constant(dense<0.000000e+00> : vector<2xf32>) : !llvm.vec<2 x float>
 //       CHECK:    llvm.mlir.constant(2 : index) : !llvm.i64
-//       CHECK:    llvm.extractelement {{.*}}[{{.*}} : !llvm.i64] : !llvm<"<8 x float>">
+//       CHECK:    llvm.extractelement {{.*}}[{{.*}} : !llvm.i64] : !llvm.vec<8 x float>
 //       CHECK:    llvm.mlir.constant(0 : index) : !llvm.i64
-//       CHECK:    llvm.insertelement {{.*}}, {{.*}}[{{.*}} : !llvm.i64] : !llvm<"<2 x float>">
+//       CHECK:    llvm.insertelement {{.*}}, {{.*}}[{{.*}} : !llvm.i64] : !llvm.vec<2 x float>
 //       CHECK:    llvm.mlir.constant(3 : index) : !llvm.i64
-//       CHECK:    llvm.extractelement {{.*}}[{{.*}} : !llvm.i64] : !llvm<"<8 x float>">
+//       CHECK:    llvm.extractelement {{.*}}[{{.*}} : !llvm.i64] : !llvm.vec<8 x float>
 //       CHECK:    llvm.mlir.constant(1 : index) : !llvm.i64
-//       CHECK:    llvm.insertelement {{.*}}, {{.*}}[{{.*}} : !llvm.i64] : !llvm<"<2 x float>">
-//       CHECK:    llvm.insertvalue {{.*}}, {{.*}}[0] : !llvm<"[2 x <2 x float>]">
+//       CHECK:    llvm.insertelement {{.*}}, {{.*}}[{{.*}} : !llvm.i64] : !llvm.vec<2 x float>
+//       CHECK:    llvm.insertvalue {{.*}}, {{.*}}[0] : !llvm.array<2 x vec<2 x float>>
 //
 // Subvector vector<8xf32> @3
-//       CHECK:    llvm.extractvalue {{.*}}[3] : !llvm<"[4 x <8 x float>]">
+//       CHECK:    llvm.extractvalue {{.*}}[3] : !llvm.array<4 x vec<8 x float>>
 //       CHECK:    llvm.mlir.constant(0.000000e+00 : f32) : !llvm.float
-//       CHECK:    llvm.mlir.constant(dense<0.000000e+00> : vector<2xf32>) : !llvm<"<2 x float>">
+//       CHECK:    llvm.mlir.constant(dense<0.000000e+00> : vector<2xf32>) : !llvm.vec<2 x float>
 //       CHECK:    llvm.mlir.constant(2 : index) : !llvm.i64
-//       CHECK:    llvm.extractelement {{.*}}[{{.*}} : !llvm.i64] : !llvm<"<8 x float>">
+//       CHECK:    llvm.extractelement {{.*}}[{{.*}} : !llvm.i64] : !llvm.vec<8 x float>
 //       CHECK:    llvm.mlir.constant(0 : index) : !llvm.i64
-//       CHECK:    llvm.insertelement {{.*}}, {{.*}}[{{.*}} : !llvm.i64] : !llvm<"<2 x float>">
+//       CHECK:    llvm.insertelement {{.*}}, {{.*}}[{{.*}} : !llvm.i64] : !llvm.vec<2 x float>
 //       CHECK:    llvm.mlir.constant(3 : index) : !llvm.i64
-//       CHECK:    llvm.extractelement {{.*}}[{{.*}} : !llvm.i64] : !llvm<"<8 x float>">
+//       CHECK:    llvm.extractelement {{.*}}[{{.*}} : !llvm.i64] : !llvm.vec<8 x float>
 //       CHECK:    llvm.mlir.constant(1 : index) : !llvm.i64
-//       CHECK:    llvm.insertelement {{.*}}, {{.*}}[{{.*}} : !llvm.i64] : !llvm<"<2 x float>">
-//       CHECK:    llvm.insertvalue {{.*}}, {{.*}}[1] : !llvm<"[2 x <2 x float>]">
+//       CHECK:    llvm.insertelement {{.*}}, {{.*}}[{{.*}} : !llvm.i64] : !llvm.vec<2 x float>
+//       CHECK:    llvm.insertvalue {{.*}}, {{.*}}[1] : !llvm.array<2 x vec<2 x float>>
 
 func @insert_strided_slice1(%b: vector<4x4xf32>, %c: vector<4x4x4xf32>) -> vector<4x4x4xf32> {
   %0 = vector.insert_strided_slice %b, %c {offsets = [2, 0, 0], strides = [1, 1]} : vector<4x4xf32> into vector<4x4x4xf32>
   return %0 : vector<4x4x4xf32>
 }
 // CHECK-LABEL: llvm.func @insert_strided_slice1
-//       CHECK:    llvm.extractvalue {{.*}}[2] : !llvm<"[4 x [4 x <4 x float>]]">
-//  CHECK-NEXT:    llvm.insertvalue {{.*}}, {{.*}}[2] : !llvm<"[4 x [4 x <4 x float>]]">
+//       CHECK:    llvm.extractvalue {{.*}}[2] : !llvm.array<4 x array<4 x vec<4 x float>>>
+//  CHECK-NEXT:    llvm.insertvalue {{.*}}, {{.*}}[2] : !llvm.array<4 x array<4 x vec<4 x float>>>
 
 func @insert_strided_slice2(%a: vector<2x2xf32>, %b: vector<4x4xf32>) -> vector<4x4xf32> {
   %0 = vector.insert_strided_slice %a, %b {offsets = [2, 2], strides = [1, 1]} : vector<2x2xf32> into vector<4x4xf32>
@@ -587,34 +587,34 @@ func @insert_strided_slice2(%a: vector<2x2xf32>, %b: vector<4x4xf32>) -> vector<
 // CHECK-LABEL: llvm.func @insert_strided_slice2
 //
 // Subvector vector<2xf32> @0 into vector<4xf32> @2
-//       CHECK:    llvm.extractvalue {{.*}}[0] : !llvm<"[2 x <2 x float>]">
-//  CHECK-NEXT:    llvm.extractvalue {{.*}}[2] : !llvm<"[4 x <4 x float>]">
+//       CHECK:    llvm.extractvalue {{.*}}[0] : !llvm.array<2 x vec<2 x float>>
+//  CHECK-NEXT:    llvm.extractvalue {{.*}}[2] : !llvm.array<4 x vec<4 x float>>
 // Element @0 -> element @2
 //  CHECK-NEXT:    llvm.mlir.constant(0 : index) : !llvm.i64
-//  CHECK-NEXT:    llvm.extractelement {{.*}}[{{.*}} : !llvm.i64] : !llvm<"<2 x float>">
+//  CHECK-NEXT:    llvm.extractelement {{.*}}[{{.*}} : !llvm.i64] : !llvm.vec<2 x float>
 //  CHECK-NEXT:    llvm.mlir.constant(2 : index) : !llvm.i64
-//  CHECK-NEXT:    llvm.insertelement {{.*}}, {{.*}}[{{.*}} : !llvm.i64] : !llvm<"<4 x float>">
+//  CHECK-NEXT:    llvm.insertelement {{.*}}, {{.*}}[{{.*}} : !llvm.i64] : !llvm.vec<4 x float>
 // Element @1 -> element @3
 //  CHECK-NEXT:    llvm.mlir.constant(1 : index) : !llvm.i64
-//  CHECK-NEXT:    llvm.extractelement {{.*}}[{{.*}} : !llvm.i64] : !llvm<"<2 x float>">
+//  CHECK-NEXT:    llvm.extractelement {{.*}}[{{.*}} : !llvm.i64] : !llvm.vec<2 x float>
 //  CHECK-NEXT:    llvm.mlir.constant(3 : index) : !llvm.i64
-//  CHECK-NEXT:    llvm.insertelement {{.*}}, {{.*}}[{{.*}} : !llvm.i64] : !llvm<"<4 x float>">
-//  CHECK-NEXT:    llvm.insertvalue {{.*}}, {{.*}}[2] : !llvm<"[4 x <4 x float>]">
+//  CHECK-NEXT:    llvm.insertelement {{.*}}, {{.*}}[{{.*}} : !llvm.i64] : !llvm.vec<4 x float>
+//  CHECK-NEXT:    llvm.insertvalue {{.*}}, {{.*}}[2] : !llvm.array<4 x vec<4 x float>>
 //
 // Subvector vector<2xf32> @1 into vector<4xf32> @3
-//       CHECK:    llvm.extractvalue {{.*}}[1] : !llvm<"[2 x <2 x float>]">
-//  CHECK-NEXT:    llvm.extractvalue {{.*}}[3] : !llvm<"[4 x <4 x float>]">
+//       CHECK:    llvm.extractvalue {{.*}}[1] : !llvm.array<2 x vec<2 x float>>
+//  CHECK-NEXT:    llvm.extractvalue {{.*}}[3] : !llvm.array<4 x vec<4 x float>>
 // Element @0 -> element @2
 //  CHECK-NEXT:    llvm.mlir.constant(0 : index) : !llvm.i64
-//  CHECK-NEXT:    llvm.extractelement {{.*}}[{{.*}} : !llvm.i64] : !llvm<"<2 x float>">
+//  CHECK-NEXT:    llvm.extractelement {{.*}}[{{.*}} : !llvm.i64] : !llvm.vec<2 x float>
 //  CHECK-NEXT:    llvm.mlir.constant(2 : index) : !llvm.i64
-//  CHECK-NEXT:    llvm.insertelement {{.*}}, {{.*}}[{{.*}} : !llvm.i64] : !llvm<"<4 x float>">
+//  CHECK-NEXT:    llvm.insertelement {{.*}}, {{.*}}[{{.*}} : !llvm.i64] : !llvm.vec<4 x float>
 // Element @1 -> element @3
 //  CHECK-NEXT:    llvm.mlir.constant(1 : index) : !llvm.i64
-//  CHECK-NEXT:    llvm.extractelement {{.*}}[{{.*}} : !llvm.i64] : !llvm<"<2 x float>">
+//  CHECK-NEXT:    llvm.extractelement {{.*}}[{{.*}} : !llvm.i64] : !llvm.vec<2 x float>
 //  CHECK-NEXT:    llvm.mlir.constant(3 : index) : !llvm.i64
-//  CHECK-NEXT:    llvm.insertelement {{.*}}, {{.*}}[{{.*}} : !llvm.i64] : !llvm<"<4 x float>">
-//  CHECK-NEXT:    llvm.insertvalue {{.*}}, {{.*}}[3] : !llvm<"[4 x <4 x float>]">
+//  CHECK-NEXT:    llvm.insertelement {{.*}}, {{.*}}[{{.*}} : !llvm.i64] : !llvm.vec<4 x float>
+//  CHECK-NEXT:    llvm.insertvalue {{.*}}, {{.*}}[3] : !llvm.array<4 x vec<4 x float>>
 
 func @insert_strided_slice3(%arg0: vector<2x4xf32>, %arg1: vector<16x4x8xf32>) -> vector<16x4x8xf32> {
   %0 = vector.insert_strided_slice %arg0, %arg1 {offsets = [0, 0, 2], strides = [1, 1]}:
@@ -622,49 +622,49 @@ func @insert_strided_slice3(%arg0: vector<2x4xf32>, %arg1: vector<16x4x8xf32>) -
   return %0 : vector<16x4x8xf32>
 }
 // CHECK-LABEL: llvm.func @insert_strided_slice3(
-// CHECK-SAME: %[[A:.*]]: !llvm<"[2 x <4 x float>]">,
-// CHECK-SAME: %[[B:.*]]: !llvm<"[16 x [4 x <8 x float>]]">)
-//      CHECK: %[[s0:.*]] = llvm.extractvalue %[[B]][0] : !llvm<"[16 x [4 x <8 x float>]]">
-//      CHECK: %[[s1:.*]] = llvm.extractvalue %[[A]][0] : !llvm<"[2 x <4 x float>]">
-//      CHECK: %[[s2:.*]] = llvm.extractvalue %[[B]][0, 0] : !llvm<"[16 x [4 x <8 x float>]]">
+// CHECK-SAME: %[[A:.*]]: !llvm.array<2 x vec<4 x float>>,
+// CHECK-SAME: %[[B:.*]]: !llvm.array<16 x array<4 x vec<8 x float>>>)
+//      CHECK: %[[s0:.*]] = llvm.extractvalue %[[B]][0] : !llvm.array<16 x array<4 x vec<8 x float>>>
+//      CHECK: %[[s1:.*]] = llvm.extractvalue %[[A]][0] : !llvm.array<2 x vec<4 x float>>
+//      CHECK: %[[s2:.*]] = llvm.extractvalue %[[B]][0, 0] : !llvm.array<16 x array<4 x vec<8 x float>>>
 //      CHECK: %[[s3:.*]] = llvm.mlir.constant(0 : index) : !llvm.i64
-//      CHECK: %[[s4:.*]] = llvm.extractelement %[[s1]][%[[s3]] : !llvm.i64] : !llvm<"<4 x float>">
+//      CHECK: %[[s4:.*]] = llvm.extractelement %[[s1]][%[[s3]] : !llvm.i64] : !llvm.vec<4 x float>
 //      CHECK: %[[s5:.*]] = llvm.mlir.constant(2 : index) : !llvm.i64
-//      CHECK: %[[s6:.*]] = llvm.insertelement %[[s4]], %[[s2]][%[[s5]] : !llvm.i64] : !llvm<"<8 x float>">
+//      CHECK: %[[s6:.*]] = llvm.insertelement %[[s4]], %[[s2]][%[[s5]] : !llvm.i64] : !llvm.vec<8 x float>
 //      CHECK: %[[s7:.*]] = llvm.mlir.constant(1 : index) : !llvm.i64
-//      CHECK: %[[s8:.*]] = llvm.extractelement %[[s1]][%[[s7]] : !llvm.i64] : !llvm<"<4 x float>">
+//      CHECK: %[[s8:.*]] = llvm.extractelement %[[s1]][%[[s7]] : !llvm.i64] : !llvm.vec<4 x float>
 //      CHECK: %[[s9:.*]] = llvm.mlir.constant(3 : index) : !llvm.i64
-//      CHECK: %[[s10:.*]] = llvm.insertelement %[[s8]], %[[s6]][%[[s9]] : !llvm.i64] : !llvm<"<8 x float>">
+//      CHECK: %[[s10:.*]] = llvm.insertelement %[[s8]], %[[s6]][%[[s9]] : !llvm.i64] : !llvm.vec<8 x float>
 //      CHECK: %[[s11:.*]] = llvm.mlir.constant(2 : index) : !llvm.i64
-//      CHECK: %[[s12:.*]] = llvm.extractelement %[[s1]][%[[s11]] : !llvm.i64] : !llvm<"<4 x float>">
+//      CHECK: %[[s12:.*]] = llvm.extractelement %[[s1]][%[[s11]] : !llvm.i64] : !llvm.vec<4 x float>
 //      CHECK: %[[s13:.*]] = llvm.mlir.constant(4 : index) : !llvm.i64
-//      CHECK: %[[s14:.*]] = llvm.insertelement %[[s12]], %[[s10]][%[[s13]] : !llvm.i64] : !llvm<"<8 x float>">
+//      CHECK: %[[s14:.*]] = llvm.insertelement %[[s12]], %[[s10]][%[[s13]] : !llvm.i64] : !llvm.vec<8 x float>
 //      CHECK: %[[s15:.*]] = llvm.mlir.constant(3 : index) : !llvm.i64
-//      CHECK: %[[s16:.*]] = llvm.extractelement %[[s1]][%[[s15]] : !llvm.i64] : !llvm<"<4 x float>">
+//      CHECK: %[[s16:.*]] = llvm.extractelement %[[s1]][%[[s15]] : !llvm.i64] : !llvm.vec<4 x float>
 //      CHECK: %[[s17:.*]] = llvm.mlir.constant(5 : index) : !llvm.i64
-//      CHECK: %[[s18:.*]] = llvm.insertelement %[[s16]], %[[s14]][%[[s17]] : !llvm.i64] : !llvm<"<8 x float>">
-//      CHECK: %[[s19:.*]] = llvm.insertvalue %[[s18]], %[[s0]][0] : !llvm<"[4 x <8 x float>]">
-//      CHECK: %[[s20:.*]] = llvm.extractvalue %[[A]][1] : !llvm<"[2 x <4 x float>]">
-//      CHECK: %[[s21:.*]] = llvm.extractvalue %[[B]][0, 1] : !llvm<"[16 x [4 x <8 x float>]]">
+//      CHECK: %[[s18:.*]] = llvm.insertelement %[[s16]], %[[s14]][%[[s17]] : !llvm.i64] : !llvm.vec<8 x float>
+//      CHECK: %[[s19:.*]] = llvm.insertvalue %[[s18]], %[[s0]][0] : !llvm.array<4 x vec<8 x float>>
+//      CHECK: %[[s20:.*]] = llvm.extractvalue %[[A]][1] : !llvm.array<2 x vec<4 x float>>
+//      CHECK: %[[s21:.*]] = llvm.extractvalue %[[B]][0, 1] : !llvm.array<16 x array<4 x vec<8 x float>>>
 //      CHECK: %[[s22:.*]] = llvm.mlir.constant(0 : index) : !llvm.i64
-//      CHECK: %[[s23:.*]] = llvm.extractelement %[[s20]][%[[s22]] : !llvm.i64] : !llvm<"<4 x float>">
+//      CHECK: %[[s23:.*]] = llvm.extractelement %[[s20]][%[[s22]] : !llvm.i64] : !llvm.vec<4 x float>
 //      CHECK: %[[s24:.*]] = llvm.mlir.constant(2 : index) : !llvm.i64
-//      CHECK: %[[s25:.*]] = llvm.insertelement %[[s23]], %[[s21]][%[[s24]] : !llvm.i64] : !llvm<"<8 x float>">
+//      CHECK: %[[s25:.*]] = llvm.insertelement %[[s23]], %[[s21]][%[[s24]] : !llvm.i64] : !llvm.vec<8 x float>
 //      CHECK: %[[s26:.*]] = llvm.mlir.constant(1 : index) : !llvm.i64
-//      CHECK: %[[s27:.*]] = llvm.extractelement %[[s20]][%[[s26]] : !llvm.i64] : !llvm<"<4 x float>">
+//      CHECK: %[[s27:.*]] = llvm.extractelement %[[s20]][%[[s26]] : !llvm.i64] : !llvm.vec<4 x float>
 //      CHECK: %[[s28:.*]] = llvm.mlir.constant(3 : index) : !llvm.i64
-//      CHECK: %[[s29:.*]] = llvm.insertelement %[[s27]], %[[s25]][%[[s28]] : !llvm.i64] : !llvm<"<8 x float>">
+//      CHECK: %[[s29:.*]] = llvm.insertelement %[[s27]], %[[s25]][%[[s28]] : !llvm.i64] : !llvm.vec<8 x float>
 //      CHECK: %[[s30:.*]] = llvm.mlir.constant(2 : index) : !llvm.i64
-//      CHECK: %[[s31:.*]] = llvm.extractelement %[[s20]][%[[s30]] : !llvm.i64] : !llvm<"<4 x float>">
+//      CHECK: %[[s31:.*]] = llvm.extractelement %[[s20]][%[[s30]] : !llvm.i64] : !llvm.vec<4 x float>
 //      CHECK: %[[s32:.*]] = llvm.mlir.constant(4 : index) : !llvm.i64
-//      CHECK: %[[s33:.*]] = llvm.insertelement %[[s31]], %[[s29]][%[[s32]] : !llvm.i64] : !llvm<"<8 x float>">
+//      CHECK: %[[s33:.*]] = llvm.insertelement %[[s31]], %[[s29]][%[[s32]] : !llvm.i64] : !llvm.vec<8 x float>
 //      CHECK: %[[s34:.*]] = llvm.mlir.constant(3 : index) : !llvm.i64
-//      CHECK: %[[s35:.*]] = llvm.extractelement %[[s20]][%[[s34]] : !llvm.i64] : !llvm<"<4 x float>">
+//      CHECK: %[[s35:.*]] = llvm.extractelement %[[s20]][%[[s34]] : !llvm.i64] : !llvm.vec<4 x float>
 //      CHECK: %[[s36:.*]] = llvm.mlir.constant(5 : index) : !llvm.i64
-//      CHECK: %[[s37:.*]] = llvm.insertelement %[[s35]], %[[s33]][%[[s36]] : !llvm.i64] : !llvm<"<8 x float>">
-//      CHECK: %[[s38:.*]] = llvm.insertvalue %[[s37]], %[[s19]][1] : !llvm<"[4 x <8 x float>]">
-//      CHECK: %[[s39:.*]] = llvm.insertvalue %[[s38]], %[[B]][0] : !llvm<"[16 x [4 x <8 x float>]]">
-//      CHECK: llvm.return %[[s39]] : !llvm<"[16 x [4 x <8 x float>]]">
+//      CHECK: %[[s37:.*]] = llvm.insertelement %[[s35]], %[[s33]][%[[s36]] : !llvm.i64] : !llvm.vec<8 x float>
+//      CHECK: %[[s38:.*]] = llvm.insertvalue %[[s37]], %[[s19]][1] : !llvm.array<4 x vec<8 x float>>
+//      CHECK: %[[s39:.*]] = llvm.insertvalue %[[s38]], %[[B]][0] : !llvm.array<16 x array<4 x vec<8 x float>>>
+//      CHECK: llvm.return %[[s39]] : !llvm.array<16 x array<4 x vec<8 x float>>>
 
 func @extract_strides(%arg0: vector<3x3xf32>) -> vector<1x1xf32> {
   %0 = vector.extract_slices %arg0, [2, 2], [1, 1]
@@ -673,37 +673,37 @@ func @extract_strides(%arg0: vector<3x3xf32>) -> vector<1x1xf32> {
   return %1 : vector<1x1xf32>
 }
 // CHECK-LABEL: llvm.func @extract_strides(
-// CHECK-SAME: %[[A:.*]]: !llvm<"[3 x <3 x float>]">)
-//      CHECK: %[[s0:.*]] = llvm.mlir.constant(dense<0.000000e+00> : vector<1x1xf32>) : !llvm<"[1 x <1 x float>]">
-//      CHECK: %[[s1:.*]] = llvm.extractvalue %[[A]][2] : !llvm<"[3 x <3 x float>]">
-//      CHECK: %[[s3:.*]] = llvm.mlir.constant(dense<0.000000e+00> : vector<1xf32>) : !llvm<"<1 x float>">
+// CHECK-SAME: %[[A:.*]]: !llvm.array<3 x vec<3 x float>>)
+//      CHECK: %[[s0:.*]] = llvm.mlir.constant(dense<0.000000e+00> : vector<1x1xf32>) : !llvm.array<1 x vec<1 x float>>
+//      CHECK: %[[s1:.*]] = llvm.extractvalue %[[A]][2] : !llvm.array<3 x vec<3 x float>>
+//      CHECK: %[[s3:.*]] = llvm.mlir.constant(dense<0.000000e+00> : vector<1xf32>) : !llvm.vec<1 x float>
 //      CHECK: %[[s4:.*]] = llvm.mlir.constant(2 : index) : !llvm.i64
-//      CHECK: %[[s5:.*]] = llvm.extractelement %[[s1]][%[[s4]] : !llvm.i64] : !llvm<"<3 x float>">
+//      CHECK: %[[s5:.*]] = llvm.extractelement %[[s1]][%[[s4]] : !llvm.i64] : !llvm.vec<3 x float>
 //      CHECK: %[[s6:.*]] = llvm.mlir.constant(0 : index) : !llvm.i64
-//      CHECK: %[[s7:.*]] = llvm.insertelement %[[s5]], %[[s3]][%[[s6]] : !llvm.i64] : !llvm<"<1 x float>">
-//      CHECK: %[[s8:.*]] = llvm.insertvalue %[[s7]], %[[s0]][0] : !llvm<"[1 x <1 x float>]">
-//      CHECK: llvm.return %[[s8]] : !llvm<"[1 x <1 x float>]">
+//      CHECK: %[[s7:.*]] = llvm.insertelement %[[s5]], %[[s3]][%[[s6]] : !llvm.i64] : !llvm.vec<1 x float>
+//      CHECK: %[[s8:.*]] = llvm.insertvalue %[[s7]], %[[s0]][0] : !llvm.array<1 x vec<1 x float>>
+//      CHECK: llvm.return %[[s8]] : !llvm.array<1 x vec<1 x float>>
 
 // CHECK-LABEL: llvm.func @vector_fma(
-//  CHECK-SAME: %[[A:.*]]: !llvm<"<8 x float>">, %[[B:.*]]: !llvm<"[2 x <4 x float>]">)
-//  CHECK-SAME: -> !llvm<"{ <8 x float>, [2 x <4 x float>] }"> {
+//  CHECK-SAME: %[[A:.*]]: !llvm.vec<8 x float>, %[[B:.*]]: !llvm.array<2 x vec<4 x float>>)
+//  CHECK-SAME: -> !llvm.struct<(vec<8 x float>, array<2 x vec<4 x float>>)> {
 func @vector_fma(%a: vector<8xf32>, %b: vector<2x4xf32>) -> (vector<8xf32>, vector<2x4xf32>) {
   //         CHECK: "llvm.intr.fmuladd"(%[[A]], %[[A]], %[[A]]) :
-  //    CHECK-SAME:   (!llvm<"<8 x float>">, !llvm<"<8 x float>">, !llvm<"<8 x float>">) -> !llvm<"<8 x float>">
+  //    CHECK-SAME:   (!llvm.vec<8 x float>, !llvm.vec<8 x float>, !llvm.vec<8 x float>) -> !llvm.vec<8 x float>
   %0 = vector.fma %a, %a, %a : vector<8xf32>
 
-  //       CHECK: %[[b00:.*]] = llvm.extractvalue %[[B]][0] : !llvm<"[2 x <4 x float>]">
-  //       CHECK: %[[b01:.*]] = llvm.extractvalue %[[B]][0] : !llvm<"[2 x <4 x float>]">
-  //       CHECK: %[[b02:.*]] = llvm.extractvalue %[[B]][0] : !llvm<"[2 x <4 x float>]">
+  //       CHECK: %[[b00:.*]] = llvm.extractvalue %[[B]][0] : !llvm.array<2 x vec<4 x float>>
+  //       CHECK: %[[b01:.*]] = llvm.extractvalue %[[B]][0] : !llvm.array<2 x vec<4 x float>>
+  //       CHECK: %[[b02:.*]] = llvm.extractvalue %[[B]][0] : !llvm.array<2 x vec<4 x float>>
   //       CHECK: %[[B0:.*]] = "llvm.intr.fmuladd"(%[[b00]], %[[b01]], %[[b02]]) :
-  //  CHECK-SAME: (!llvm<"<4 x float>">, !llvm<"<4 x float>">, !llvm<"<4 x float>">) -> !llvm<"<4 x float>">
-  //       CHECK: llvm.insertvalue %[[B0]], {{.*}}[0] : !llvm<"[2 x <4 x float>]">
-  //       CHECK: %[[b10:.*]] = llvm.extractvalue %[[B]][1] : !llvm<"[2 x <4 x float>]">
-  //       CHECK: %[[b11:.*]] = llvm.extractvalue %[[B]][1] : !llvm<"[2 x <4 x float>]">
-  //       CHECK: %[[b12:.*]] = llvm.extractvalue %[[B]][1] : !llvm<"[2 x <4 x float>]">
+  //  CHECK-SAME: (!llvm.vec<4 x float>, !llvm.vec<4 x float>, !llvm.vec<4 x float>) -> !llvm.vec<4 x float>
+  //       CHECK: llvm.insertvalue %[[B0]], {{.*}}[0] : !llvm.array<2 x vec<4 x float>>
+  //       CHECK: %[[b10:.*]] = llvm.extractvalue %[[B]][1] : !llvm.array<2 x vec<4 x float>>
+  //       CHECK: %[[b11:.*]] = llvm.extractvalue %[[B]][1] : !llvm.array<2 x vec<4 x float>>
+  //       CHECK: %[[b12:.*]] = llvm.extractvalue %[[B]][1] : !llvm.array<2 x vec<4 x float>>
   //       CHECK: %[[B1:.*]] = "llvm.intr.fmuladd"(%[[b10]], %[[b11]], %[[b12]]) :
-  //  CHECK-SAME: (!llvm<"<4 x float>">, !llvm<"<4 x float>">, !llvm<"<4 x float>">) -> !llvm<"<4 x float>">
-  //       CHECK: llvm.insertvalue %[[B1]], {{.*}}[1] : !llvm<"[2 x <4 x float>]">
+  //  CHECK-SAME: (!llvm.vec<4 x float>, !llvm.vec<4 x float>, !llvm.vec<4 x float>) -> !llvm.vec<4 x float>
+  //       CHECK: llvm.insertvalue %[[B1]], {{.*}}[1] : !llvm.array<2 x vec<4 x float>>
   %1 = vector.fma %b, %b, %b : vector<2x4xf32>
 
   return %0, %1: vector<8xf32>, vector<2x4xf32>
@@ -714,10 +714,10 @@ func @reduce_f32(%arg0: vector<16xf32>) -> f32 {
   return %0 : f32
 }
 // CHECK-LABEL: llvm.func @reduce_f32(
-// CHECK-SAME: %[[A:.*]]: !llvm<"<16 x float>">)
+// CHECK-SAME: %[[A:.*]]: !llvm.vec<16 x float>)
 //      CHECK: %[[C:.*]] = llvm.mlir.constant(0.000000e+00 : f32) : !llvm.float
 //      CHECK: %[[V:.*]] = "llvm.intr.experimental.vector.reduce.v2.fadd"(%[[C]], %[[A]])
-// CHECK-SAME: {reassoc = false} : (!llvm.float, !llvm<"<16 x float>">) -> !llvm.float
+// CHECK-SAME: {reassoc = false} : (!llvm.float, !llvm.vec<16 x float>) -> !llvm.float
 //      CHECK: llvm.return %[[V]] : !llvm.float
 
 func @reduce_f64(%arg0: vector<16xf64>) -> f64 {
@@ -725,10 +725,10 @@ func @reduce_f64(%arg0: vector<16xf64>) -> f64 {
   return %0 : f64
 }
 // CHECK-LABEL: llvm.func @reduce_f64(
-// CHECK-SAME: %[[A:.*]]: !llvm<"<16 x double>">)
+// CHECK-SAME: %[[A:.*]]: !llvm.vec<16 x double>)
 //      CHECK: %[[C:.*]] = llvm.mlir.constant(0.000000e+00 : f64) : !llvm.double
 //      CHECK: %[[V:.*]] = "llvm.intr.experimental.vector.reduce.v2.fadd"(%[[C]], %[[A]])
-// CHECK-SAME: {reassoc = false} : (!llvm.double, !llvm<"<16 x double>">) -> !llvm.double
+// CHECK-SAME: {reassoc = false} : (!llvm.double, !llvm.vec<16 x double>) -> !llvm.double
 //      CHECK: llvm.return %[[V]] : !llvm.double
 
 func @reduce_i32(%arg0: vector<16xi32>) -> i32 {
@@ -736,7 +736,7 @@ func @reduce_i32(%arg0: vector<16xi32>) -> i32 {
   return %0 : i32
 }
 // CHECK-LABEL: llvm.func @reduce_i32(
-// CHECK-SAME: %[[A:.*]]: !llvm<"<16 x i32>">)
+// CHECK-SAME: %[[A:.*]]: !llvm.vec<16 x i32>)
 //      CHECK: %[[V:.*]] = "llvm.intr.experimental.vector.reduce.add"(%[[A]])
 //      CHECK: llvm.return %[[V]] : !llvm.i32
 
@@ -745,7 +745,7 @@ func @reduce_i64(%arg0: vector<16xi64>) -> i64 {
   return %0 : i64
 }
 // CHECK-LABEL: llvm.func @reduce_i64(
-// CHECK-SAME: %[[A:.*]]: !llvm<"<16 x i64>">)
+// CHECK-SAME: %[[A:.*]]: !llvm.vec<16 x i64>)
 //      CHECK: %[[V:.*]] = "llvm.intr.experimental.vector.reduce.add"(%[[A]])
 //      CHECK: llvm.return %[[V]] : !llvm.i64
 
@@ -760,7 +760,7 @@ func @matrix_ops(%A: vector<64xf64>, %B: vector<48xf64>) -> vector<12xf64> {
 // CHECK-LABEL: llvm.func @matrix_ops
 //       CHECK:   llvm.intr.matrix.multiply %{{.*}}, %{{.*}} {
 //  CHECK-SAME: lhs_columns = 16 : i32, lhs_rows = 4 : i32, rhs_columns = 3 : i32
-//  CHECK-SAME: } : (!llvm<"<64 x double>">, !llvm<"<48 x double>">) -> !llvm<"<12 x double>">
+//  CHECK-SAME: } : (!llvm.vec<64 x double>, !llvm.vec<48 x double>) -> !llvm.vec<12 x double>
 
 func @transfer_read_1d(%A : memref<?xf32>, %base: index) -> vector<17xf32> {
   %f7 = constant 7.0: f32
@@ -773,72 +773,72 @@ func @transfer_read_1d(%A : memref<?xf32>, %base: index) -> vector<17xf32> {
   return %f: vector<17xf32>
 }
 // CHECK-LABEL: func @transfer_read_1d
-//  CHECK-SAME: %[[BASE:[a-zA-Z0-9]*]]: !llvm.i64) -> !llvm<"<17 x float>">
+//  CHECK-SAME: %[[BASE:[a-zA-Z0-9]*]]: !llvm.i64) -> !llvm.vec<17 x float>
 //
 // 1. Bitcast to vector form.
 //       CHECK: %[[gep:.*]] = llvm.getelementptr {{.*}} :
-//  CHECK-SAME: (!llvm<"float*">, !llvm.i64) -> !llvm<"float*">
+//  CHECK-SAME: (!llvm.ptr<float>, !llvm.i64) -> !llvm.ptr<float>
 //       CHECK: %[[vecPtr:.*]] = llvm.bitcast %[[gep]] :
-//  CHECK-SAME: !llvm<"float*"> to !llvm<"<17 x float>*">
+//  CHECK-SAME: !llvm.ptr<float> to !llvm.ptr<vec<17 x float>>
 //
 // 2. Create a vector with linear indices [ 0 .. vector_length - 1 ].
 //       CHECK: %[[linearIndex:.*]] = llvm.mlir.constant(
 //  CHECK-SAME: dense<[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]> :
-//  CHECK-SAME: vector<17xi64>) : !llvm<"<17 x i64>">
+//  CHECK-SAME: vector<17xi64>) : !llvm.vec<17 x i64>
 //
 // 3. Create offsetVector = [ offset + 0 .. offset + vector_length - 1 ].
-//       CHECK: %[[offsetVec:.*]] = llvm.mlir.undef : !llvm<"<17 x i64>">
+//       CHECK: %[[offsetVec:.*]] = llvm.mlir.undef : !llvm.vec<17 x i64>
 //       CHECK: %[[c0:.*]] = llvm.mlir.constant(0 : i32) : !llvm.i32
 //       CHECK: %[[offsetVec2:.*]] = llvm.insertelement %[[BASE]], %[[offsetVec]][%[[c0]] :
-//  CHECK-SAME: !llvm.i32] : !llvm<"<17 x i64>">
+//  CHECK-SAME: !llvm.i32] : !llvm.vec<17 x i64>
 //       CHECK: %[[offsetVec3:.*]] = llvm.shufflevector %[[offsetVec2]], %{{.*}} [
 //  CHECK-SAME:  0 : i32, 0 : i32, 0 : i32, 0 : i32, 0 : i32, 0 : i32, 0 : i32,
 //  CHECK-SAME:  0 : i32, 0 : i32, 0 : i32, 0 : i32, 0 : i32, 0 : i32, 0 : i32,
 //  CHECK-SAME:  0 : i32, 0 : i32, 0 : i32] :
-//  CHECK-SAME: !llvm<"<17 x i64>">, !llvm<"<17 x i64>">
+//  CHECK-SAME: !llvm.vec<17 x i64>, !llvm.vec<17 x i64>
 //       CHECK: %[[offsetVec4:.*]] = llvm.add %[[offsetVec3]], %[[linearIndex]] :
-//  CHECK-SAME: !llvm<"<17 x i64>">
+//  CHECK-SAME: !llvm.vec<17 x i64>
 //
 // 4. Let dim the memref dimension, compute the vector comparison mask:
 //    [ offset + 0 .. offset + vector_length - 1 ] < [ dim .. dim ]
 //       CHECK: %[[DIM:.*]] = llvm.extractvalue %{{.*}}[3, 0] :
-//  CHECK-SAME: !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }">
-//       CHECK: %[[dimVec:.*]] = llvm.mlir.undef : !llvm<"<17 x i64>">
+//  CHECK-SAME: !llvm.struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>
+//       CHECK: %[[dimVec:.*]] = llvm.mlir.undef : !llvm.vec<17 x i64>
 //       CHECK: %[[c01:.*]] = llvm.mlir.constant(0 : i32) : !llvm.i32
 //       CHECK: %[[dimVec2:.*]] = llvm.insertelement %[[DIM]], %[[dimVec]][%[[c01]] :
-//  CHECK-SAME:  !llvm.i32] : !llvm<"<17 x i64>">
+//  CHECK-SAME:  !llvm.i32] : !llvm.vec<17 x i64>
 //       CHECK: %[[dimVec3:.*]] = llvm.shufflevector %[[dimVec2]], %{{.*}} [
 //  CHECK-SAME:  0 : i32, 0 : i32, 0 : i32, 0 : i32, 0 : i32, 0 : i32, 0 : i32,
 //  CHECK-SAME:  0 : i32, 0 : i32, 0 : i32, 0 : i32, 0 : i32, 0 : i32, 0 : i32,
 //  CHECK-SAME:  0 : i32, 0 : i32, 0 : i32] :
-//  CHECK-SAME: !llvm<"<17 x i64>">, !llvm<"<17 x i64>">
+//  CHECK-SAME: !llvm.vec<17 x i64>, !llvm.vec<17 x i64>
 //       CHECK: %[[mask:.*]] = llvm.icmp "slt" %[[offsetVec4]], %[[dimVec3]] :
-//  CHECK-SAME: !llvm<"<17 x i64>">
+//  CHECK-SAME: !llvm.vec<17 x i64>
 //
 // 5. Rewrite as a masked read.
 //       CHECK: %[[PASS_THROUGH:.*]] =  llvm.mlir.constant(dense<7.000000e+00> :
-//  CHECK-SAME:  vector<17xf32>) : !llvm<"<17 x float>">
+//  CHECK-SAME:  vector<17xf32>) : !llvm.vec<17 x float>
 //       CHECK: %[[loaded:.*]] = llvm.intr.masked.load %[[vecPtr]], %[[mask]],
 //  CHECK-SAME: %[[PASS_THROUGH]] {alignment = 4 : i32} :
-//  CHECK-SAME: (!llvm<"<17 x float>*">, !llvm<"<17 x i1>">, !llvm<"<17 x float>">) -> !llvm<"<17 x float>">
+//  CHECK-SAME: (!llvm.ptr<vec<17 x float>>, !llvm.vec<17 x i1>, !llvm.vec<17 x float>) -> !llvm.vec<17 x float>
 
 //
 // 1. Bitcast to vector form.
 //       CHECK: %[[gep_b:.*]] = llvm.getelementptr {{.*}} :
-//  CHECK-SAME: (!llvm<"float*">, !llvm.i64) -> !llvm<"float*">
+//  CHECK-SAME: (!llvm.ptr<float>, !llvm.i64) -> !llvm.ptr<float>
 //       CHECK: %[[vecPtr_b:.*]] = llvm.bitcast %[[gep_b]] :
-//  CHECK-SAME: !llvm<"float*"> to !llvm<"<17 x float>*">
+//  CHECK-SAME: !llvm.ptr<float> to !llvm.ptr<vec<17 x float>>
 //
 // 2. Create a vector with linear indices [ 0 .. vector_length - 1 ].
 //       CHECK: %[[linearIndex_b:.*]] = llvm.mlir.constant(
 //  CHECK-SAME: dense<[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]> :
-//  CHECK-SAME:  vector<17xi64>) : !llvm<"<17 x i64>">
+//  CHECK-SAME:  vector<17xi64>) : !llvm.vec<17 x i64>
 //
 // 3. Create offsetVector = [ offset + 0 .. offset + vector_length - 1 ].
 //       CHECK: llvm.shufflevector {{.*}} [0 : i32, 0 : i32, 0 : i32, 0 : i32,
 //  CHECK-SAME:  0 : i32, 0 : i32, 0 : i32, 0 : i32, 0 : i32, 0 : i32, 0 : i32,
 //  CHECK-SAME:  0 : i32, 0 : i32, 0 : i32, 0 : i32, 0 : i32, 0 : i32] :
-//  CHECK-SAME: !llvm<"<17 x i64>">, !llvm<"<17 x i64>">
+//  CHECK-SAME: !llvm.vec<17 x i64>, !llvm.vec<17 x i64>
 //       CHECK: llvm.add
 //
 // 4. Let dim the memref dimension, compute the vector comparison mask:
@@ -846,13 +846,13 @@ func @transfer_read_1d(%A : memref<?xf32>, %base: index) -> vector<17xf32> {
 //       CHECK: llvm.shufflevector {{.*}} [0 : i32, 0 : i32, 0 : i32, 0 : i32,
 //  CHECK-SAME:  0 : i32, 0 : i32, 0 : i32, 0 : i32, 0 : i32, 0 : i32, 0 : i32,
 //  CHECK-SAME:  0 : i32, 0 : i32, 0 : i32, 0 : i32, 0 : i32, 0 : i32] :
-//  CHECK-SAME: !llvm<"<17 x i64>">, !llvm<"<17 x i64>">
-//       CHECK: %[[mask_b:.*]] = llvm.icmp "slt" {{.*}} : !llvm<"<17 x i64>">
+//  CHECK-SAME: !llvm.vec<17 x i64>, !llvm.vec<17 x i64>
+//       CHECK: %[[mask_b:.*]] = llvm.icmp "slt" {{.*}} : !llvm.vec<17 x i64>
 //
 // 5. Rewrite as a masked write.
 //       CHECK: llvm.intr.masked.store %[[loaded]], %[[vecPtr_b]], %[[mask_b]]
 //  CHECK-SAME: {alignment = 4 : i32} :
-//  CHECK-SAME: !llvm<"<17 x float>">, !llvm<"<17 x i1>"> into !llvm<"<17 x float>*">
+//  CHECK-SAME: !llvm.vec<17 x float>, !llvm.vec<17 x i1> into !llvm.ptr<vec<17 x float>>
 
 func @transfer_read_2d_to_1d(%A : memref<?x?xf32>, %base0: index, %base1: index) -> vector<17xf32> {
   %f7 = constant 7.0: f32
@@ -862,14 +862,14 @@ func @transfer_read_2d_to_1d(%A : memref<?x?xf32>, %base0: index, %base1: index)
   return %f: vector<17xf32>
 }
 // CHECK-LABEL: func @transfer_read_2d_to_1d
-//  CHECK-SAME: %[[BASE_0:[a-zA-Z0-9]*]]: !llvm.i64, %[[BASE_1:[a-zA-Z0-9]*]]: !llvm.i64) -> !llvm<"<17 x float>">
+//  CHECK-SAME: %[[BASE_0:[a-zA-Z0-9]*]]: !llvm.i64, %[[BASE_1:[a-zA-Z0-9]*]]: !llvm.i64) -> !llvm.vec<17 x float>
 //
 // Create offsetVector = [ offset + 0 .. offset + vector_length - 1 ].
-//       CHECK: %[[offsetVec:.*]] = llvm.mlir.undef : !llvm<"<17 x i64>">
+//       CHECK: %[[offsetVec:.*]] = llvm.mlir.undef : !llvm.vec<17 x i64>
 //       CHECK: %[[c0:.*]] = llvm.mlir.constant(0 : i32) : !llvm.i32
 // Here we check we properly use %BASE_1
 //       CHECK: %[[offsetVec2:.*]] = llvm.insertelement %[[BASE_1]], %[[offsetVec]][%[[c0]] :
-//  CHECK-SAME: !llvm.i32] : !llvm<"<17 x i64>">
+//  CHECK-SAME: !llvm.i32] : !llvm.vec<17 x i64>
 //       CHECK: %[[offsetVec3:.*]] = llvm.shufflevector %[[offsetVec2]], %{{.*}} [
 //  CHECK-SAME:  0 : i32, 0 : i32, 0 : i32, 0 : i32, 0 : i32, 0 : i32, 0 : i32,
 //  CHECK-SAME:  0 : i32, 0 : i32, 0 : i32, 0 : i32, 0 : i32, 0 : i32, 0 : i32,
@@ -879,16 +879,16 @@ func @transfer_read_2d_to_1d(%A : memref<?x?xf32>, %base0: index, %base1: index)
 //    [ offset + 0 .. offset + vector_length - 1 ] < [ dim .. dim ]
 // Here we check we properly use %DIM[1]
 //       CHECK: %[[DIM:.*]] = llvm.extractvalue %{{.*}}[3, 1] :
-//  CHECK-SAME: !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
-//       CHECK: %[[dimVec:.*]] = llvm.mlir.undef : !llvm<"<17 x i64>">
+//  CHECK-SAME: !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
+//       CHECK: %[[dimVec:.*]] = llvm.mlir.undef : !llvm.vec<17 x i64>
 //       CHECK: %[[c01:.*]] = llvm.mlir.constant(0 : i32) : !llvm.i32
 //       CHECK: %[[dimVec2:.*]] = llvm.insertelement %[[DIM]], %[[dimVec]][%[[c01]] :
-//  CHECK-SAME:  !llvm.i32] : !llvm<"<17 x i64>">
+//  CHECK-SAME:  !llvm.i32] : !llvm.vec<17 x i64>
 //       CHECK: %[[dimVec3:.*]] = llvm.shufflevector %[[dimVec2]], %{{.*}} [
 //  CHECK-SAME:  0 : i32, 0 : i32, 0 : i32, 0 : i32, 0 : i32, 0 : i32, 0 : i32,
 //  CHECK-SAME:  0 : i32, 0 : i32, 0 : i32, 0 : i32, 0 : i32, 0 : i32, 0 : i32,
 //  CHECK-SAME:  0 : i32, 0 : i32, 0 : i32] :
-//  CHECK-SAME: !llvm<"<17 x i64>">, !llvm<"<17 x i64>">
+//  CHECK-SAME: !llvm.vec<17 x i64>, !llvm.vec<17 x i64>
 
 func @transfer_read_1d_non_zero_addrspace(%A : memref<?xf32, 3>, %base: index) -> vector<17xf32> {
   %f7 = constant 7.0: f32
@@ -901,23 +901,23 @@ func @transfer_read_1d_non_zero_addrspace(%A : memref<?xf32, 3>, %base: index) -
   return %f: vector<17xf32>
 }
 // CHECK-LABEL: func @transfer_read_1d_non_zero_addrspace
-//  CHECK-SAME: %[[BASE:[a-zA-Z0-9]*]]: !llvm.i64) -> !llvm<"<17 x float>">
+//  CHECK-SAME: %[[BASE:[a-zA-Z0-9]*]]: !llvm.i64) -> !llvm.vec<17 x float>
 //
 // 1. Check address space for GEP is correct.
 //       CHECK: %[[gep:.*]] = llvm.getelementptr {{.*}} :
-//  CHECK-SAME: (!llvm<"float addrspace(3)*">, !llvm.i64) -> !llvm<"float addrspace(3)*">
+//  CHECK-SAME: (!llvm.ptr<float, 3>, !llvm.i64) -> !llvm.ptr<float, 3>
 //       CHECK: %[[vecPtr:.*]] = llvm.addrspacecast %[[gep]] :
-//  CHECK-SAME: !llvm<"float addrspace(3)*"> to !llvm<"<17 x float>*">
+//  CHECK-SAME: !llvm.ptr<float, 3> to !llvm.ptr<vec<17 x float>>
 //
 // 2. Check address space of the memref is correct.
 //       CHECK: %[[DIM:.*]] = llvm.extractvalue %{{.*}}[3, 0] :
-//  CHECK-SAME: !llvm<"{ float addrspace(3)*, float addrspace(3)*, i64, [1 x i64], [1 x i64] }">
+//  CHECK-SAME: !llvm.struct<(ptr<float, 3>, ptr<float, 3>, i64, array<1 x i64>, array<1 x i64>)>
 //
 // 3. Check address apce for GEP is correct.
 //       CHECK: %[[gep_b:.*]] = llvm.getelementptr {{.*}} :
-//  CHECK-SAME: (!llvm<"float addrspace(3)*">, !llvm.i64) -> !llvm<"float addrspace(3)*">
+//  CHECK-SAME: (!llvm.ptr<float, 3>, !llvm.i64) -> !llvm.ptr<float, 3>
 //       CHECK: %[[vecPtr_b:.*]] = llvm.addrspacecast %[[gep_b]] :
-//  CHECK-SAME: !llvm<"float addrspace(3)*"> to !llvm<"<17 x float>*">
+//  CHECK-SAME: !llvm.ptr<float, 3> to !llvm.ptr<vec<17 x float>>
 
 func @transfer_read_1d_not_masked(%A : memref<?xf32>, %base: index) -> vector<17xf32> {
   %f7 = constant 7.0: f32
@@ -926,24 +926,24 @@ func @transfer_read_1d_not_masked(%A : memref<?xf32>, %base: index) -> vector<17
   return %f: vector<17xf32>
 }
 // CHECK-LABEL: func @transfer_read_1d_not_masked
-//  CHECK-SAME: %[[BASE:[a-zA-Z0-9]*]]: !llvm.i64) -> !llvm<"<17 x float>">
+//  CHECK-SAME: %[[BASE:[a-zA-Z0-9]*]]: !llvm.i64) -> !llvm.vec<17 x float>
 //
 // 1. Bitcast to vector form.
 //       CHECK: %[[gep:.*]] = llvm.getelementptr {{.*}} :
-//  CHECK-SAME: (!llvm<"float*">, !llvm.i64) -> !llvm<"float*">
+//  CHECK-SAME: (!llvm.ptr<float>, !llvm.i64) -> !llvm.ptr<float>
 //       CHECK: %[[vecPtr:.*]] = llvm.bitcast %[[gep]] :
-//  CHECK-SAME: !llvm<"float*"> to !llvm<"<17 x float>*">
+//  CHECK-SAME: !llvm.ptr<float> to !llvm.ptr<vec<17 x float>>
 //
 // 2. Rewrite as a load.
-//       CHECK: %[[loaded:.*]] = llvm.load %[[vecPtr]] {alignment = 4 : i64} : !llvm<"<17 x float>*">
+//       CHECK: %[[loaded:.*]] = llvm.load %[[vecPtr]] {alignment = 4 : i64} : !llvm.ptr<vec<17 x float>>
 
 func @genbool_1d() -> vector<8xi1> {
   %0 = vector.constant_mask [4] : vector<8xi1>
   return %0 : vector<8xi1>
 }
 // CHECK-LABEL: func @genbool_1d
-// CHECK: %[[C1:.*]] = llvm.mlir.constant(dense<[true, true, true, true, false, false, false, false]> : vector<8xi1>) : !llvm<"<8 x i1>">
-// CHECK: llvm.return %[[C1]] : !llvm<"<8 x i1>">
+// CHECK: %[[C1:.*]] = llvm.mlir.constant(dense<[true, true, true, true, false, false, false, false]> : vector<8xi1>) : !llvm.vec<8 x i1>
+// CHECK: llvm.return %[[C1]] : !llvm.vec<8 x i1>
 
 func @genbool_2d() -> vector<4x4xi1> {
   %v = vector.constant_mask [2, 2] : vector<4x4xi1>
@@ -951,11 +951,11 @@ func @genbool_2d() -> vector<4x4xi1> {
 }
 
 // CHECK-LABEL: func @genbool_2d
-// CHECK: %[[C1:.*]] = llvm.mlir.constant(dense<[true, true, false, false]> : vector<4xi1>) : !llvm<"<4 x i1>">
-// CHECK: %[[C2:.*]] = llvm.mlir.constant(dense<false> : vector<4x4xi1>) : !llvm<"[4 x <4 x i1>]">
-// CHECK: %[[T0:.*]] = llvm.insertvalue %[[C1]], %[[C2]][0] : !llvm<"[4 x <4 x i1>]">
-// CHECK: %[[T1:.*]] = llvm.insertvalue %[[C1]], %[[T0]][1] : !llvm<"[4 x <4 x i1>]">
-// CHECK: llvm.return %[[T1]] : !llvm<"[4 x <4 x i1>]">
+// CHECK: %[[C1:.*]] = llvm.mlir.constant(dense<[true, true, false, false]> : vector<4xi1>) : !llvm.vec<4 x i1>
+// CHECK: %[[C2:.*]] = llvm.mlir.constant(dense<false> : vector<4x4xi1>) : !llvm.array<4 x vec<4 x i1>>
+// CHECK: %[[T0:.*]] = llvm.insertvalue %[[C1]], %[[C2]][0] : !llvm.array<4 x vec<4 x i1>>
+// CHECK: %[[T1:.*]] = llvm.insertvalue %[[C1]], %[[T0]][1] : !llvm.array<4 x vec<4 x i1>>
+// CHECK: llvm.return %[[T1]] : !llvm.array<4 x vec<4 x i1>>
 
 func @flat_transpose(%arg0: vector<16xf32>) -> vector<16xf32> {
   %0 = vector.flat_transpose %arg0 { rows = 4: i32, columns = 4: i32 }
@@ -964,11 +964,11 @@ func @flat_transpose(%arg0: vector<16xf32>) -> vector<16xf32> {
 }
 
 // CHECK-LABEL: func @flat_transpose
-// CHECK-SAME:  %[[A:.*]]: !llvm<"<16 x float>">
+// CHECK-SAME:  %[[A:.*]]: !llvm.vec<16 x float>
 // CHECK:       %[[T:.*]] = llvm.intr.matrix.transpose %[[A]]
 // CHECK-SAME:      {columns = 4 : i32, rows = 4 : i32} :
-// CHECK-SAME:      !llvm<"<16 x float>"> into !llvm<"<16 x float>">
-// CHECK:       llvm.return %[[T]] : !llvm<"<16 x float>">
+// CHECK-SAME:      !llvm.vec<16 x float> into !llvm.vec<16 x float>
+// CHECK:       llvm.return %[[T]] : !llvm.vec<16 x float>
 
 func @gather_op(%arg0: memref<?xf32>, %arg1: vector<3xi32>, %arg2: vector<3xi1>, %arg3: vector<3xf32>) -> vector<3xf32> {
   %0 = vector.gather %arg0, %arg1, %arg2, %arg3 : (memref<?xf32>, vector<3xi32>, vector<3xi1>, vector<3xf32>) -> vector<3xf32>
@@ -976,9 +976,9 @@ func @gather_op(%arg0: memref<?xf32>, %arg1: vector<3xi32>, %arg2: vector<3xi1>,
 }
 
 // CHECK-LABEL: func @gather_op
-// CHECK: %[[P:.*]] = llvm.getelementptr {{.*}}[%{{.*}}] : (!llvm<"float*">, !llvm<"<3 x i32>">) -> !llvm<"<3 x float*>">
-// CHECK: %[[G:.*]] = llvm.intr.masked.gather %[[P]], %{{.*}}, %{{.*}} {alignment = 4 : i32} : (!llvm<"<3 x float*>">, !llvm<"<3 x i1>">, !llvm<"<3 x float>">) -> !llvm<"<3 x float>">
-// CHECK: llvm.return %[[G]] : !llvm<"<3 x float>">
+// CHECK: %[[P:.*]] = llvm.getelementptr {{.*}}[%{{.*}}] : (!llvm.ptr<float>, !llvm.vec<3 x i32>) -> !llvm.vec<3 x ptr<float>>
+// CHECK: %[[G:.*]] = llvm.intr.masked.gather %[[P]], %{{.*}}, %{{.*}} {alignment = 4 : i32} : (!llvm.vec<3 x ptr<float>>, !llvm.vec<3 x i1>, !llvm.vec<3 x float>) -> !llvm.vec<3 x float>
+// CHECK: llvm.return %[[G]] : !llvm.vec<3 x float>
 
 func @scatter_op(%arg0: memref<?xf32>, %arg1: vector<3xi32>, %arg2: vector<3xi1>, %arg3: vector<3xf32>) {
   vector.scatter %arg0, %arg1, %arg2, %arg3 : vector<3xi32>, vector<3xi1>, vector<3xf32> into memref<?xf32>
@@ -986,6 +986,6 @@ func @scatter_op(%arg0: memref<?xf32>, %arg1: vector<3xi32>, %arg2: vector<3xi1>
 }
 
 // CHECK-LABEL: func @scatter_op
-// CHECK: %[[P:.*]] = llvm.getelementptr {{.*}}[%{{.*}}] : (!llvm<"float*">, !llvm<"<3 x i32>">) -> !llvm<"<3 x float*>">
-// CHECK: llvm.intr.masked.scatter %{{.*}}, %[[P]], %{{.*}} {alignment = 4 : i32} : !llvm<"<3 x float>">, !llvm<"<3 x i1>"> into !llvm<"<3 x float*>">
+// CHECK: %[[P:.*]] = llvm.getelementptr {{.*}}[%{{.*}}] : (!llvm.ptr<float>, !llvm.vec<3 x i32>) -> !llvm.vec<3 x ptr<float>>
+// CHECK: llvm.intr.masked.scatter %{{.*}}, %[[P]], %{{.*}} {alignment = 4 : i32} : !llvm.vec<3 x float>, !llvm.vec<3 x i1> into !llvm.vec<3 x ptr<float>>
 // CHECK: llvm.return
diff --git a/mlir/test/Conversion/VectorToROCDL/vector-to-rocdl.mlir b/mlir/test/Conversion/VectorToROCDL/vector-to-rocdl.mlir
index 1113197aa589b..f8522c8960080 100644
--- a/mlir/test/Conversion/VectorToROCDL/vector-to-rocdl.mlir
+++ b/mlir/test/Conversion/VectorToROCDL/vector-to-rocdl.mlir
@@ -9,7 +9,7 @@ func @transfer_readx2(%A : memref<?xf32>, %base: index) -> vector<2xf32> {
   return %f: vector<2xf32>
 }
 // CHECK-LABEL: @transfer_readx2
-// CHECK: rocdl.buffer.load {{.*}} !llvm<"<2 x float>">
+// CHECK: rocdl.buffer.load {{.*}} !llvm.vec<2 x float>
 
 func @transfer_readx4(%A : memref<?xf32>, %base: index) -> vector<4xf32> {
   %f0 = constant 0.0: f32
@@ -19,7 +19,7 @@ func @transfer_readx4(%A : memref<?xf32>, %base: index) -> vector<4xf32> {
   return %f: vector<4xf32>
 }
 // CHECK-LABEL: @transfer_readx4
-// CHECK: rocdl.buffer.load {{.*}} !llvm<"<4 x float>">
+// CHECK: rocdl.buffer.load {{.*}} !llvm.vec<4 x float>
 
 func @transfer_read_dwordConfig(%A : memref<?xf32>, %base: index) -> vector<4xf32> {
   %f0 = constant 0.0: f32
@@ -43,7 +43,7 @@ func @transfer_writex2(%A : memref<?xf32>, %B : vector<2xf32>, %base: index) {
   return
 }
 // CHECK-LABEL: @transfer_writex2
-// CHECK: rocdl.buffer.store {{.*}} !llvm<"<2 x float>">
+// CHECK: rocdl.buffer.store {{.*}} !llvm.vec<2 x float>
 
 func @transfer_writex4(%A : memref<?xf32>, %B : vector<4xf32>, %base: index) {
   vector.transfer_write %B, %A[%base]
@@ -52,7 +52,7 @@ func @transfer_writex4(%A : memref<?xf32>, %B : vector<4xf32>, %base: index) {
   return
 }
 // CHECK-LABEL: @transfer_writex4
-// CHECK: rocdl.buffer.store {{.*}} !llvm<"<4 x float>">
+// CHECK: rocdl.buffer.store {{.*}} !llvm.vec<4 x float>
 
 func @transfer_write_dwordConfig(%A : memref<?xf32>, %B : vector<2xf32>, %base: index) {
   vector.transfer_write %B, %A[%base]
diff --git a/mlir/test/Dialect/GPU/invalid.mlir b/mlir/test/Dialect/GPU/invalid.mlir
index 43b2434a2cdf9..739d23a59f058 100644
--- a/mlir/test/Dialect/GPU/invalid.mlir
+++ b/mlir/test/Dialect/GPU/invalid.mlir
@@ -89,7 +89,7 @@ module attributes {gpu.container_module} {
 module attributes {gpu.container_module} {
   module @kernels {
     // expected-error@+1 {{'gpu.func' op expects parent op 'gpu.module'}}
-    gpu.func @kernel_1(%arg1 : !llvm<"float*">) {
+    gpu.func @kernel_1(%arg1 : !llvm.ptr<float>) {
       gpu.return
     }
   }
@@ -128,16 +128,16 @@ module attributes {gpu.container_module} {
 
 module attributes {gpu.container_module} {
   module @kernels {
-    gpu.func @kernel_1(%arg1 : !llvm<"float*">) kernel {
+    gpu.func @kernel_1(%arg1 : !llvm.ptr<float>) kernel {
       gpu.return
     }
   }
 
-  func @launch_func_missing_kernel_attr(%sz : index, %arg : !llvm<"float*">) {
+  func @launch_func_missing_kernel_attr(%sz : index, %arg : !llvm.ptr<float>) {
     // expected-error@+1 {{kernel module 'kernels' is undefined}}
     "gpu.launch_func"(%sz, %sz, %sz, %sz, %sz, %sz, %arg)
     {kernel = @kernels::@kernel_1}
-        : (index, index, index, index, index, index, !llvm<"float*">) -> ()
+        : (index, index, index, index, index, index, !llvm.ptr<float>) -> ()
     return
   }
 }
@@ -146,16 +146,16 @@ module attributes {gpu.container_module} {
 
 module attributes {gpu.container_module} {
   gpu.module @kernels {
-    gpu.func @kernel_1(%arg1 : !llvm<"float*">) {
+    gpu.func @kernel_1(%arg1 : !llvm.ptr<float>) {
       gpu.return
     }
   }
 
-  func @launch_func_missing_kernel_attr(%sz : index, %arg : !llvm<"float*">) {
+  func @launch_func_missing_kernel_attr(%sz : index, %arg : !llvm.ptr<float>) {
     // expected-error@+1 {{kernel function is missing the 'gpu.kernel' attribute}}
     "gpu.launch_func"(%sz, %sz, %sz, %sz, %sz, %sz, %arg)
     {kernel = @kernels::@kernel_1}
-        : (index, index, index, index, index, index, !llvm<"float*">) -> ()
+        : (index, index, index, index, index, index, !llvm.ptr<float>) -> ()
     return
   }
 }
@@ -164,17 +164,17 @@ module attributes {gpu.container_module} {
 
 module attributes {gpu.container_module} {
   gpu.module @kernels {
-    gpu.func @kernel_1(%arg1 : !llvm<"float*">) kernel {
+    gpu.func @kernel_1(%arg1 : !llvm.ptr<float>) kernel {
       gpu.return
     }
   }
 
-  func @launch_func_kernel_operand_size(%sz : index, %arg : !llvm<"float*">) {
+  func @launch_func_kernel_operand_size(%sz : index, %arg : !llvm.ptr<float>) {
     // expected-error@+1 {{got 2 kernel operands but expected 1}}
     "gpu.launch_func"(%sz, %sz, %sz, %sz, %sz, %sz, %arg, %arg)
         {kernel = @kernels::@kernel_1}
-        : (index, index, index, index, index, index, !llvm<"float*">,
-           !llvm<"float*">) -> ()
+        : (index, index, index, index, index, index, !llvm.ptr<float>,
+           !llvm.ptr<float>) -> ()
     return
   }
 }
diff --git a/mlir/test/Dialect/GPU/multiple-all-reduce.mlir b/mlir/test/Dialect/GPU/multiple-all-reduce.mlir
index f1437dbb1adb2..a084faec29c88 100644
--- a/mlir/test/Dialect/GPU/multiple-all-reduce.mlir
+++ b/mlir/test/Dialect/GPU/multiple-all-reduce.mlir
@@ -18,8 +18,8 @@ func @main() {
   }
 
 // CHECK:      gpu.module @main_kernel {
-// CHECK-NEXT:   llvm.mlir.global internal @{{.*}}() {addr_space = 3 : i32} : !llvm<"[32 x float]">
-// CHECK-NEXT:   llvm.mlir.global internal @{{.*}}() {addr_space = 3 : i32} : !llvm<"[32 x float]">
+// CHECK-NEXT:   llvm.mlir.global internal @{{.*}}() {addr_space = 3 : i32} : !llvm.array<32 x float>
+// CHECK-NEXT:   llvm.mlir.global internal @{{.*}}() {addr_space = 3 : i32} : !llvm.array<32 x float>
 
   return
 }
diff --git a/mlir/test/Dialect/GPU/outlining.mlir b/mlir/test/Dialect/GPU/outlining.mlir
index 9a3206f313459..23a8b9d98881f 100644
--- a/mlir/test/Dialect/GPU/outlining.mlir
+++ b/mlir/test/Dialect/GPU/outlining.mlir
@@ -143,7 +143,7 @@ func @function_call(%arg0 : memref<?xf32>) {
                                         %block_z = %cst) {
     call @device_function() : () -> ()
     call @device_function() : () -> ()
-    %0 = llvm.mlir.addressof @global : !llvm<"i64*">
+    %0 = llvm.mlir.addressof @global : !llvm.ptr<i64>
     gpu.terminator
   }
   return
@@ -163,7 +163,7 @@ func @recursive_device_function() {
 // CHECK:   gpu.func @function_call_kernel()
 // CHECK:     call @device_function() : () -> ()
 // CHECK:     call @device_function() : () -> ()
-// CHECK:     llvm.mlir.addressof @global : !llvm<"i64*">
+// CHECK:     llvm.mlir.addressof @global : !llvm.ptr<i64>
 // CHECK:     gpu.return
 //
 // CHECK:   llvm.mlir.global internal @global(42 : i64) : !llvm.i64
diff --git a/mlir/test/Dialect/LLVMIR/func.mlir b/mlir/test/Dialect/LLVMIR/func.mlir
index 689e6db540650..65dc33cc1c4f9 100644
--- a/mlir/test/Dialect/LLVMIR/func.mlir
+++ b/mlir/test/Dialect/LLVMIR/func.mlir
@@ -4,19 +4,19 @@
 module {
   // GENERIC: "llvm.func"
   // GENERIC: sym_name = "foo"
-  // GENERIC-SAME: type = !llvm<"void ()">
+  // GENERIC-SAME: type = !llvm.func<void ()>
   // GENERIC-SAME: () -> ()
   // CHECK: llvm.func @foo()
   "llvm.func"() ({
-  }) {sym_name = "foo", type = !llvm<"void ()">} : () -> ()
+  }) {sym_name = "foo", type = !llvm.func<void ()>} : () -> ()
 
   // GENERIC: "llvm.func"
   // GENERIC: sym_name = "bar"
-  // GENERIC-SAME: type = !llvm<"i64 (i64, i64)">
+  // GENERIC-SAME: type = !llvm.func<i64 (i64, i64)>
   // GENERIC-SAME: () -> ()
   // CHECK: llvm.func @bar(!llvm.i64, !llvm.i64) -> !llvm.i64
   "llvm.func"() ({
-  }) {sym_name = "bar", type = !llvm<"i64 (i64, i64)">} : () -> ()
+  }) {sym_name = "bar", type = !llvm.func<i64 (i64, i64)>} : () -> ()
 
   // GENERIC: "llvm.func"
   // CHECK: llvm.func @baz(%{{.*}}: !llvm.i64) -> !llvm.i64
@@ -27,14 +27,14 @@ module {
     llvm.return %arg0 : !llvm.i64
 
   // GENERIC: sym_name = "baz"
-  // GENERIC-SAME: type = !llvm<"i64 (i64)">
+  // GENERIC-SAME: type = !llvm.func<i64 (i64)>
   // GENERIC-SAME: () -> ()
-  }) {sym_name = "baz", type = !llvm<"i64 (i64)">} : () -> ()
+  }) {sym_name = "baz", type = !llvm.func<i64 (i64)>} : () -> ()
 
-  // CHECK: llvm.func @qux(!llvm<"i64*"> {llvm.noalias = true}, !llvm.i64)
+  // CHECK: llvm.func @qux(!llvm.ptr<i64> {llvm.noalias = true}, !llvm.i64)
   // CHECK: attributes {xxx = {yyy = 42 : i64}}
   "llvm.func"() ({
-  }) {sym_name = "qux", type = !llvm<"void (i64*, i64)">,
+  }) {sym_name = "qux", type = !llvm.func<void (ptr<i64>, i64)>,
       arg0 = {llvm.noalias = true}, xxx = {yyy = 42}} : () -> ()
 
   // CHECK: llvm.func @roundtrip1()
@@ -69,20 +69,20 @@ module {
   // CHECK: llvm.func @roundtrip8() -> !llvm.i32
   llvm.func @roundtrip8() -> !llvm.i32 attributes {}
 
-  // CHECK: llvm.func @roundtrip9(!llvm<"i32*"> {llvm.noalias = true})
-  llvm.func @roundtrip9(!llvm<"i32*"> {llvm.noalias = true})
+  // CHECK: llvm.func @roundtrip9(!llvm.ptr<i32> {llvm.noalias = true})
+  llvm.func @roundtrip9(!llvm.ptr<i32> {llvm.noalias = true})
 
-  // CHECK: llvm.func @roundtrip10(!llvm<"i32*"> {llvm.noalias = true})
-  llvm.func @roundtrip10(%arg0: !llvm<"i32*"> {llvm.noalias = true})
+  // CHECK: llvm.func @roundtrip10(!llvm.ptr<i32> {llvm.noalias = true})
+  llvm.func @roundtrip10(%arg0: !llvm.ptr<i32> {llvm.noalias = true})
 
-  // CHECK: llvm.func @roundtrip11(%{{.*}}: !llvm<"i32*"> {llvm.noalias = true}) {
-  llvm.func @roundtrip11(%arg0: !llvm<"i32*"> {llvm.noalias = true}) {
+  // CHECK: llvm.func @roundtrip11(%{{.*}}: !llvm.ptr<i32> {llvm.noalias = true}) {
+  llvm.func @roundtrip11(%arg0: !llvm.ptr<i32> {llvm.noalias = true}) {
     llvm.return
   }
 
-  // CHECK: llvm.func @roundtrip12(%{{.*}}: !llvm<"i32*"> {llvm.noalias = true})
+  // CHECK: llvm.func @roundtrip12(%{{.*}}: !llvm.ptr<i32> {llvm.noalias = true})
   // CHECK: attributes {foo = 42 : i32}
-  llvm.func @roundtrip12(%arg0: !llvm<"i32*"> {llvm.noalias = true})
+  llvm.func @roundtrip12(%arg0: !llvm.ptr<i32> {llvm.noalias = true})
   attributes {foo = 42 : i32} {
     llvm.return
   }
@@ -119,7 +119,7 @@ module {
 
 module {
   // expected-error@+1 {{requires one region}}
-  "llvm.func"() {sym_name = "no_region", type = !llvm<"void ()">} : () -> ()
+  "llvm.func"() {sym_name = "no_region", type = !llvm.func<void ()>} : () -> ()
 }
 
 // -----
@@ -140,7 +140,7 @@ module {
 
 module {
   // expected-error@+1 {{requires 'type' attribute of wrapped LLVM function type}}
-  "llvm.func"() ({}) {sym_name = "non_function_type", type = !llvm<"i64">} : () -> ()
+  "llvm.func"() ({}) {sym_name = "non_function_type", type = !llvm.i64} : () -> ()
 }
 
 // -----
@@ -150,7 +150,7 @@ module {
   "llvm.func"() ({
   ^bb0(%arg0: !llvm.i64):
     llvm.return
-  }) {sym_name = "wrong_arg_number", type = !llvm<"void ()">} : () -> ()
+  }) {sym_name = "wrong_arg_number", type = !llvm.func<void ()>} : () -> ()
 }
 
 // -----
@@ -160,7 +160,7 @@ module {
   "llvm.func"() ({
   ^bb0(%arg0: i64):
     llvm.return
-  }) {sym_name = "wrong_arg_number", type = !llvm<"void (i64)">} : () -> ()
+  }) {sym_name = "wrong_arg_number", type = !llvm.func<void (i64)>} : () -> ()
 }
 
 // -----
@@ -170,7 +170,7 @@ module {
   "llvm.func"() ({
   ^bb0(%arg0: !llvm.i32):
     llvm.return
-  }) {sym_name = "wrong_arg_number", type = !llvm<"void (i64)">} : () -> ()
+  }) {sym_name = "wrong_arg_number", type = !llvm.func<void (i64)>} : () -> ()
 }
 
 // -----
diff --git a/mlir/test/Dialect/LLVMIR/global.mlir b/mlir/test/Dialect/LLVMIR/global.mlir
index e3faec63f00a8..7d7860645ee9b 100644
--- a/mlir/test/Dialect/LLVMIR/global.mlir
+++ b/mlir/test/Dialect/LLVMIR/global.mlir
@@ -13,7 +13,7 @@ llvm.mlir.global internal @global(42 : i64) : !llvm.i64
 llvm.mlir.global internal constant @constant(37.0) : !llvm.float
 
 // CHECK: llvm.mlir.global internal constant @".string"("foobar")
-llvm.mlir.global internal constant @".string"("foobar") : !llvm<"[6 x i8]">
+llvm.mlir.global internal constant @".string"("foobar") : !llvm.array<6 x i8>
 
 // CHECK: llvm.mlir.global internal @string_notype("1234567")
 llvm.mlir.global internal @string_notype("1234567")
@@ -54,11 +54,11 @@ llvm.mlir.global weak_odr @weak_odr() : !llvm.i64
 
 // CHECK-LABEL: references
 func @references() {
-  // CHECK: llvm.mlir.addressof @global : !llvm<"i64*">
-  %0 = llvm.mlir.addressof @global : !llvm<"i64*">
+  // CHECK: llvm.mlir.addressof @global : !llvm.ptr<i64>
+  %0 = llvm.mlir.addressof @global : !llvm.ptr<i64>
 
-  // CHECK: llvm.mlir.addressof @".string" : !llvm<"[6 x i8]*">
-  %1 = llvm.mlir.addressof @".string" : !llvm<"[6 x i8]*">
+  // CHECK: llvm.mlir.addressof @".string" : !llvm.ptr<array<6 x i8>>
+  %1 = llvm.mlir.addressof @".string" : !llvm.ptr<array<6 x i8>>
 
   llvm.return
 }
@@ -76,7 +76,7 @@ func @references() {
 // -----
 
 // expected-error @+1 {{expects type to be a valid element type for an LLVM pointer}}
-llvm.mlir.global internal constant @constant(37.0) : !llvm<"label">
+llvm.mlir.global internal constant @constant(37.0) : !llvm.label
 
 // -----
 
@@ -98,7 +98,7 @@ func @foo() {
 // -----
 
 // expected-error @+1 {{requires an i8 array type of the length equal to that of the string}}
-llvm.mlir.global internal constant @string("foobar") : !llvm<"[42 x i8]">
+llvm.mlir.global internal constant @string("foobar") : !llvm.array<42 x i8>
 
 // -----
 
@@ -125,14 +125,14 @@ func @foo() {
   // The attribute parser will consume the first colon-type, so we put two of
   // them to trigger the attribute type mismatch error.
   // expected-error @+1 {{invalid kind of attribute specified}}
-  llvm.mlir.addressof "foo" : i64 : !llvm<"void ()*">
+  llvm.mlir.addressof "foo" : i64 : !llvm.ptr<func<void ()>>
 }
 
 // -----
 
 func @foo() {
   // expected-error @+1 {{must reference a global defined by 'llvm.mlir.global'}}
-  llvm.mlir.addressof @foo : !llvm<"void ()*">
+  llvm.mlir.addressof @foo : !llvm.ptr<func<void ()>>
 }
 
 // -----
@@ -141,7 +141,7 @@ llvm.mlir.global internal @foo(0: i32) : !llvm.i32
 
 func @bar() {
   // expected-error @+1 {{the type must be a pointer to the type of the referenced global}}
-  llvm.mlir.addressof @foo : !llvm<"i64*">
+  llvm.mlir.addressof @foo : !llvm.ptr<i64>
 }
 
 // -----
@@ -150,7 +150,7 @@ llvm.func @foo()
 
 llvm.func @bar() {
   // expected-error @+1 {{the type must be a pointer to the type of the referenced function}}
-  llvm.mlir.addressof @foo : !llvm<"i8*">
+  llvm.mlir.addressof @foo : !llvm.ptr<i8>
 }
 
 // -----
@@ -182,7 +182,7 @@ llvm.mlir.global internal @g(43 : i64) : !llvm.i64 {
 llvm.mlir.global internal @g(32 : i64) {addr_space = 3: i32} : !llvm.i64
 func @mismatch_addr_space_implicit_global() {
   // expected-error @+1 {{op the type must be a pointer to the type of the referenced global}}
-  llvm.mlir.addressof @g : !llvm<"i64*">
+  llvm.mlir.addressof @g : !llvm.ptr<i64>
 }
 
 // -----
@@ -190,5 +190,5 @@ func @mismatch_addr_space_implicit_global() {
 llvm.mlir.global internal @g(32 : i64) {addr_space = 3: i32} : !llvm.i64
 func @mismatch_addr_space() {
   // expected-error @+1 {{op the type must be a pointer to the type of the referenced global}}
-  llvm.mlir.addressof @g : !llvm<"i64 addrspace(4)*">
+  llvm.mlir.addressof @g : !llvm.ptr<i64, 4>
 }
diff --git a/mlir/test/Dialect/LLVMIR/invalid.mlir b/mlir/test/Dialect/LLVMIR/invalid.mlir
index 6ef25f85a5060..b4475df66fd1d 100644
--- a/mlir/test/Dialect/LLVMIR/invalid.mlir
+++ b/mlir/test/Dialect/LLVMIR/invalid.mlir
@@ -18,7 +18,7 @@ func @invalid_align(%arg0: !llvm.i32 {llvm.align = "foo"}) {
 
 // -----
 
-func @icmp_non_string(%arg0 : !llvm.i32, %arg1 : !llvm<"i16">) {
+func @icmp_non_string(%arg0 : !llvm.i32, %arg1 : !llvm.i16) {
   // expected-error@+1 {{invalid kind of attribute specified}}
   llvm.icmp 42 %arg0, %arg0 : !llvm.i32
   return
@@ -26,7 +26,7 @@ func @icmp_non_string(%arg0 : !llvm.i32, %arg1 : !llvm<"i16">) {
 
 // -----
 
-func @icmp_wrong_string(%arg0 : !llvm.i32, %arg1 : !llvm<"i16">) {
+func @icmp_wrong_string(%arg0 : !llvm.i32, %arg1 : !llvm.i16) {
   // expected-error@+1 {{'foo' is an incorrect value of the 'predicate' attribute}}
   llvm.icmp "foo" %arg0, %arg0 : !llvm.i32
   return
@@ -43,7 +43,7 @@ func @alloca_missing_input_result_type(%size : !llvm.i64) {
 
 func @alloca_missing_input_type() {
   // expected-error@+1 {{expected trailing function type with one argument and one result}}
-  llvm.alloca %size x !llvm.i32 : () -> (!llvm<"i32*">)
+  llvm.alloca %size x !llvm.i32 : () -> (!llvm.ptr<i32>)
 }
 
 // -----
@@ -57,42 +57,42 @@ func @alloca_mising_result_type() {
 
 func @alloca_non_function_type() {
   // expected-error@+1 {{expected trailing function type with one argument and one result}}
-  llvm.alloca %size x !llvm.i32 : !llvm<"i32*">
+  llvm.alloca %size x !llvm.i32 : !llvm.ptr<i32>
 }
 
 // -----
 
 func @alloca_nonpositive_alignment(%size : !llvm.i64) {
   // expected-error@+1 {{expected positive alignment}}
-  llvm.alloca %size x !llvm.i32 {alignment = -1} : (!llvm.i64) -> (!llvm<"i32*">)
+  llvm.alloca %size x !llvm.i32 {alignment = -1} : (!llvm.i64) -> (!llvm.ptr<i32>)
 }
 
 // -----
 
-func @gep_missing_input_result_type(%pos : !llvm.i64, %base : !llvm<"float*">) {
+func @gep_missing_input_result_type(%pos : !llvm.i64, %base : !llvm.ptr<float>) {
   // expected-error@+1 {{2 operands present, but expected 0}}
   llvm.getelementptr %base[%pos] : () -> ()
 }
 
 // -----
 
-func @gep_missing_input_type(%pos : !llvm.i64, %base : !llvm<"float*">) {
+func @gep_missing_input_type(%pos : !llvm.i64, %base : !llvm.ptr<float>) {
   // expected-error@+1 {{2 operands present, but expected 0}}
-  llvm.getelementptr %base[%pos] : () -> (!llvm<"float*">)
+  llvm.getelementptr %base[%pos] : () -> (!llvm.ptr<float>)
 }
 
 // -----
 
-func @gep_missing_result_type(%pos : !llvm.i64, %base : !llvm<"float*">) {
+func @gep_missing_result_type(%pos : !llvm.i64, %base : !llvm.ptr<float>) {
   // expected-error@+1 {{op requires one result}}
-  llvm.getelementptr %base[%pos] : (!llvm<"float *">, !llvm.i64) -> ()
+  llvm.getelementptr %base[%pos] : (!llvm.ptr<float>, !llvm.i64) -> ()
 }
 
 // -----
 
-func @gep_non_function_type(%pos : !llvm.i64, %base : !llvm<"float*">) {
+func @gep_non_function_type(%pos : !llvm.i64, %base : !llvm.ptr<float>) {
   // expected-error@+1 {{invalid kind of type specified}}
-  llvm.getelementptr %base[%pos] : !llvm<"float*">
+  llvm.getelementptr %base[%pos] : !llvm.ptr<float>
 }
 
 // -----
@@ -125,9 +125,9 @@ func @store_non_ptr_type(%foo : !llvm.float, %bar : !llvm.float) {
 
 // -----
 
-func @call_non_function_type(%callee : !llvm<"i8(i8)">, %arg : !llvm<"i8">) {
+func @call_non_function_type(%callee : !llvm.func<i8 (i8)>, %arg : !llvm.i8) {
   // expected-error@+1 {{expected function type}}
-  llvm.call %callee(%arg) : !llvm<"i8(i8)">
+  llvm.call %callee(%arg) : !llvm.func<i8 (i8)>
 }
 
 // -----
@@ -155,7 +155,7 @@ func @call_non_llvm_input(%callee : (i32) -> (), %arg : i32) {
 
 func @constant_wrong_type() {
   // expected-error@+1 {{only supports integer, float, string or elements attributes}}
-  llvm.mlir.constant(@constant_wrong_type) : !llvm<"void ()*">
+  llvm.mlir.constant(@constant_wrong_type) : !llvm.ptr<func<void ()>>
 }
 
 // -----
@@ -171,35 +171,35 @@ func @insertvalue_non_array_position() {
   // Note the double-type, otherwise attribute parsing consumes the trailing
   // type of the op as the (wrong) attribute type.
   // expected-error@+1 {{invalid kind of attribute specified}}
-  llvm.insertvalue %a, %b 0 : i32 : !llvm<"{i32}">
+  llvm.insertvalue %a, %b 0 : i32 : !llvm.struct<(i32)>
 }
 
 // -----
 
 func @insertvalue_non_integer_position() {
   // expected-error@+1 {{expected an array of integer literals}}
-  llvm.insertvalue %a, %b[0.0] : !llvm<"{i32}">
+  llvm.insertvalue %a, %b[0.0] : !llvm.struct<(i32)>
 }
 
 // -----
 
 func @insertvalue_struct_out_of_bounds() {
   // expected-error@+1 {{position out of bounds}}
-  llvm.insertvalue %a, %b[1] : !llvm<"{i32}">
+  llvm.insertvalue %a, %b[1] : !llvm.struct<(i32)>
 }
 
 // -----
 
 func @insertvalue_array_out_of_bounds() {
   // expected-error@+1 {{position out of bounds}}
-  llvm.insertvalue %a, %b[1] : !llvm<"[1 x i32]">
+  llvm.insertvalue %a, %b[1] : !llvm.array<1 x i32>
 }
 
 // -----
 
 func @insertvalue_wrong_nesting() {
   // expected-error@+1 {{expected wrapped LLVM IR structure/array type}}
-  llvm.insertvalue %a, %b[0,0] : !llvm<"{i32}">
+  llvm.insertvalue %a, %b[0,0] : !llvm.struct<(i32)>
 }
 
 // -----
@@ -215,41 +215,41 @@ func @extractvalue_non_array_position() {
   // Note the double-type, otherwise attribute parsing consumes the trailing
   // type of the op as the (wrong) attribute type.
   // expected-error@+1 {{invalid kind of attribute specified}}
-  llvm.extractvalue %b 0 : i32 : !llvm<"{i32}">
+  llvm.extractvalue %b 0 : i32 : !llvm.struct<(i32)>
 }
 
 // -----
 
 func @extractvalue_non_integer_position() {
   // expected-error@+1 {{expected an array of integer literals}}
-  llvm.extractvalue %b[0.0] : !llvm<"{i32}">
+  llvm.extractvalue %b[0.0] : !llvm.struct<(i32)>
 }
 
 // -----
 
 func @extractvalue_struct_out_of_bounds() {
   // expected-error@+1 {{position out of bounds}}
-  llvm.extractvalue %b[1] : !llvm<"{i32}">
+  llvm.extractvalue %b[1] : !llvm.struct<(i32)>
 }
 
 // -----
 
 func @extractvalue_array_out_of_bounds() {
   // expected-error@+1 {{position out of bounds}}
-  llvm.extractvalue %b[1] : !llvm<"[1 x i32]">
+  llvm.extractvalue %b[1] : !llvm.array<1 x i32>
 }
 
 // -----
 
 func @extractvalue_wrong_nesting() {
   // expected-error@+1 {{expected wrapped LLVM IR structure/array type}}
-  llvm.extractvalue %b[0,0] : !llvm<"{i32}">
+  llvm.extractvalue %b[0,0] : !llvm.struct<(i32)>
 }
 
 // -----
 
 // CHECK-LABEL: @invalid_vector_type_1
-func @invalid_vector_type_1(%arg0: !llvm<"<4 x float>">, %arg1: !llvm.i32, %arg2: !llvm.float) {
+func @invalid_vector_type_1(%arg0: !llvm.vec<4 x float>, %arg1: !llvm.i32, %arg2: !llvm.float) {
   // expected-error@+1 {{expected LLVM IR dialect vector type for operand #1}}
   %0 = llvm.extractelement %arg2[%arg1 : !llvm.i32] : !llvm.float
 }
@@ -257,7 +257,7 @@ func @invalid_vector_type_1(%arg0: !llvm<"<4 x float>">, %arg1: !llvm.i32, %arg2
 // -----
 
 // CHECK-LABEL: @invalid_vector_type_2
-func @invalid_vector_type_2(%arg0: !llvm<"<4 x float>">, %arg1: !llvm.i32, %arg2: !llvm.float) {
+func @invalid_vector_type_2(%arg0: !llvm.vec<4 x float>, %arg1: !llvm.i32, %arg2: !llvm.float) {
   // expected-error@+1 {{expected LLVM IR dialect vector type for operand #1}}
   %0 = llvm.insertelement %arg2, %arg2[%arg1 : !llvm.i32] : !llvm.float
 }
@@ -265,7 +265,7 @@ func @invalid_vector_type_2(%arg0: !llvm<"<4 x float>">, %arg1: !llvm.i32, %arg2
 // -----
 
 // CHECK-LABEL: @invalid_vector_type_3
-func @invalid_vector_type_3(%arg0: !llvm<"<4 x float>">, %arg1: !llvm.i32, %arg2: !llvm.float) {
+func @invalid_vector_type_3(%arg0: !llvm.vec<4 x float>, %arg1: !llvm.i32, %arg2: !llvm.float) {
   // expected-error@+1 {{expected LLVM IR dialect vector type for operand #1}}
   %0 = llvm.shufflevector %arg2, %arg2 [0 : i32, 0 : i32, 0 : i32, 0 : i32, 7 : i32] : !llvm.float, !llvm.float
 }
@@ -281,7 +281,7 @@ func @null_non_llvm_type() {
 
 // CHECK-LABEL: @nvvm_invalid_shfl_pred_1
 func @nvvm_invalid_shfl_pred_1(%arg0 : !llvm.i32, %arg1 : !llvm.i32, %arg2 : !llvm.i32, %arg3 : !llvm.i32) {
-  // expected-error@+1 {{expected return type !llvm<"{ ?, i1 }">}}
+  // expected-error@+1 {{expected return type to be a two-element struct with i1 as the second element}}
   %0 = nvvm.shfl.sync.bfly %arg0, %arg3, %arg1, %arg2 {return_value_and_is_valid} : !llvm.i32
 }
 
@@ -289,122 +289,112 @@ func @nvvm_invalid_shfl_pred_1(%arg0 : !llvm.i32, %arg1 : !llvm.i32, %arg2 : !ll
 
 // CHECK-LABEL: @nvvm_invalid_shfl_pred_2
 func @nvvm_invalid_shfl_pred_2(%arg0 : !llvm.i32, %arg1 : !llvm.i32, %arg2 : !llvm.i32, %arg3 : !llvm.i32) {
-  // expected-error@+1 {{expected return type !llvm<"{ ?, i1 }">}}
-  %0 = nvvm.shfl.sync.bfly %arg0, %arg3, %arg1, %arg2 {return_value_and_is_valid} : !llvm<"{ i32 }">
+  // expected-error@+1 {{expected return type to be a two-element struct with i1 as the second element}}
+  %0 = nvvm.shfl.sync.bfly %arg0, %arg3, %arg1, %arg2 {return_value_and_is_valid} : !llvm.struct<(i32)>
 }
 
 // -----
 
 // CHECK-LABEL: @nvvm_invalid_shfl_pred_3
 func @nvvm_invalid_shfl_pred_3(%arg0 : !llvm.i32, %arg1 : !llvm.i32, %arg2 : !llvm.i32, %arg3 : !llvm.i32) {
-  // expected-error@+1 {{expected return type !llvm<"{ ?, i1 }">}}
-  %0 = nvvm.shfl.sync.bfly %arg0, %arg3, %arg1, %arg2 {return_value_and_is_valid} : !llvm<"{ i32, i32 }">
+  // expected-error@+1 {{expected return type to be a two-element struct with i1 as the second element}}
+  %0 = nvvm.shfl.sync.bfly %arg0, %arg3, %arg1, %arg2 {return_value_and_is_valid} : !llvm.struct<(i32, i32)>
 }
 
 // -----
 
 // CHECK-LABEL: @nvvm_invalid_mma_0
-func @nvvm_invalid_mma_0(%a0 : !llvm.half, %a1 : !llvm<"<2 x half>">,
-                         %b0 : !llvm<"<2 x half>">, %b1 : !llvm<"<2 x half>">,
+func @nvvm_invalid_mma_0(%a0 : !llvm.half, %a1 : !llvm.vec<2 x half>,
+                         %b0 : !llvm.vec<2 x half>, %b1 : !llvm.vec<2 x half>,
                          %c0 : !llvm.float, %c1 : !llvm.float, %c2 : !llvm.float, %c3 : !llvm.float,
                          %c4 : !llvm.float, %c5 : !llvm.float, %c6 : !llvm.float, %c7 : !llvm.float) {
   // expected-error@+1 {{expected operands to be 4 <halfx2>s followed by either 4 <halfx2>s or 8 floats}}
-  %0 = nvvm.mma.sync %a0, %a1, %b0, %b1, %c0, %c1, %c2, %c3, %c4, %c5, %c6, %c7 {alayout="row", blayout="col"} : (!llvm.half, !llvm<"<2 x half>">, !llvm<"<2 x half>">, !llvm<"<2 x half>">, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float) -> !llvm<"{ float, float, float, float, float, float, float, float }">
-  llvm.return %0 : !llvm<"{ float, float, float, float, float, float, float, float }">
+  %0 = nvvm.mma.sync %a0, %a1, %b0, %b1, %c0, %c1, %c2, %c3, %c4, %c5, %c6, %c7 {alayout="row", blayout="col"} : (!llvm.half, !llvm.vec<2 x half>, !llvm.vec<2 x half>, !llvm.vec<2 x half>, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float) -> !llvm.struct<(float, float, float, float, float, float, float, float)>
+  llvm.return %0 : !llvm.struct<(float, float, float, float, float, float, float, float)>
 }
 
 // -----
 
 // CHECK-LABEL: @nvvm_invalid_mma_1
-func @nvvm_invalid_mma_1(%a0 : !llvm<"<2 x half>">, %a1 : !llvm<"<2 x half>">,
-                         %b0 : !llvm<"<2 x half>">, %b1 : !llvm<"<2 x half>">,
+func @nvvm_invalid_mma_1(%a0 : !llvm.vec<2 x half>, %a1 : !llvm.vec<2 x half>,
+                         %b0 : !llvm.vec<2 x half>, %b1 : !llvm.vec<2 x half>,
                          %c0 : !llvm.float, %c1 : !llvm.float, %c2 : !llvm.float, %c3 : !llvm.float,
                          %c4 : !llvm.float, %c5 : !llvm.float, %c6 : !llvm.float, %c7 : !llvm.float) {
   // expected-error@+1 {{expected result type to be a struct of either 4 <halfx2>s or 8 floats}}
-  %0 = nvvm.mma.sync %a0, %a1, %b0, %b1, %c0, %c1, %c2, %c3, %c4, %c5, %c6, %c7 {alayout="row", blayout="col"} : (!llvm<"<2 x half>">, !llvm<"<2 x half>">, !llvm<"<2 x half>">, !llvm<"<2 x half>">, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float) -> !llvm<"{ float, float, float, float, float, float, float, half }">
-  llvm.return %0 : !llvm<"{ float, float, float, float, float, float, float, half }">
+  %0 = nvvm.mma.sync %a0, %a1, %b0, %b1, %c0, %c1, %c2, %c3, %c4, %c5, %c6, %c7 {alayout="row", blayout="col"} : (!llvm.vec<2 x half>, !llvm.vec<2 x half>, !llvm.vec<2 x half>, !llvm.vec<2 x half>, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float) -> !llvm.struct<(float, float, float, float, float, float, float, half)>
+  llvm.return %0 : !llvm.struct<(float, float, float, float, float, float, float, half)>
 }
 
 // -----
 
 // CHECK-LABEL: @nvvm_invalid_mma_2
-func @nvvm_invalid_mma_2(%a0 : !llvm<"<2 x half>">, %a1 : !llvm<"<2 x half>">,
-                         %b0 : !llvm<"<2 x half>">, %b1 : !llvm<"<2 x half>">,
+func @nvvm_invalid_mma_2(%a0 : !llvm.vec<2 x half>, %a1 : !llvm.vec<2 x half>,
+                         %b0 : !llvm.vec<2 x half>, %b1 : !llvm.vec<2 x half>,
                          %c0 : !llvm.float, %c1 : !llvm.float, %c2 : !llvm.float, %c3 : !llvm.float,
                          %c4 : !llvm.float, %c5 : !llvm.float, %c6 : !llvm.float, %c7 : !llvm.float) {
   // expected-error@+1 {{alayout and blayout attributes must be set to either "row" or "col"}}
-  %0 = nvvm.mma.sync %a0, %a1, %b0, %b1, %c0, %c1, %c2, %c3, %c4, %c5, %c6, %c7 : (!llvm<"<2 x half>">, !llvm<"<2 x half>">, !llvm<"<2 x half>">, !llvm<"<2 x half>">, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float) -> !llvm<"{ float, float, float, float, float, float, float, float }">
-  llvm.return %0 : !llvm<"{ float, float, float, float, float, float, float, float }">
+  %0 = nvvm.mma.sync %a0, %a1, %b0, %b1, %c0, %c1, %c2, %c3, %c4, %c5, %c6, %c7 : (!llvm.vec<2 x half>, !llvm.vec<2 x half>, !llvm.vec<2 x half>, !llvm.vec<2 x half>, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float) -> !llvm.struct<(float, float, float, float, float, float, float, float)>
+  llvm.return %0 : !llvm.struct<(float, float, float, float, float, float, float, float)>
 }
 
 // -----
 
 // CHECK-LABEL: @nvvm_invalid_mma_3
-func @nvvm_invalid_mma_3(%a0 : !llvm<"<2 x half>">, %a1 : !llvm<"<2 x half>">,
-                         %b0 : !llvm<"<2 x half>">, %b1 : !llvm<"<2 x half>">,
-                         %c0 : !llvm<"<2 x half>">, %c1 : !llvm<"<2 x half>">,
-                         %c2 : !llvm<"<2 x half>">, %c3 : !llvm<"<2 x half>">) {
+func @nvvm_invalid_mma_3(%a0 : !llvm.vec<2 x half>, %a1 : !llvm.vec<2 x half>,
+                         %b0 : !llvm.vec<2 x half>, %b1 : !llvm.vec<2 x half>,
+                         %c0 : !llvm.vec<2 x half>, %c1 : !llvm.vec<2 x half>,
+                         %c2 : !llvm.vec<2 x half>, %c3 : !llvm.vec<2 x half>) {
   // expected-error@+1 {{unimplemented mma.sync variant}}
-  %0 = nvvm.mma.sync %a0, %a1, %b0, %b1, %c0, %c1, %c2, %c3 {alayout="row", blayout="col"} : (!llvm<"<2 x half>">, !llvm<"<2 x half>">, !llvm<"<2 x half>">, !llvm<"<2 x half>">, !llvm<"<2 x half>">, !llvm<"<2 x half>">, !llvm<"<2 x half>">, !llvm<"<2 x half>">) -> !llvm<"{ float, float, float, float, float, float, float, float }">
-  llvm.return %0 : !llvm<"{ float, float, float, float, float, float, float, float }">
+  %0 = nvvm.mma.sync %a0, %a1, %b0, %b1, %c0, %c1, %c2, %c3 {alayout="row", blayout="col"} : (!llvm.vec<2 x half>, !llvm.vec<2 x half>, !llvm.vec<2 x half>, !llvm.vec<2 x half>, !llvm.vec<2 x half>, !llvm.vec<2 x half>, !llvm.vec<2 x half>, !llvm.vec<2 x half>) -> !llvm.struct<(float, float, float, float, float, float, float, float)>
+  llvm.return %0 : !llvm.struct<(float, float, float, float, float, float, float, float)>
 }
 
 // -----
 
 // CHECK-LABEL: @nvvm_invalid_mma_4
-func @nvvm_invalid_mma_4(%a0 : !llvm<"<2 x half>">, %a1 : !llvm<"<2 x half>">,
-                         %b0 : !llvm<"<2 x half>">, %b1 : !llvm<"<2 x half>">,
+func @nvvm_invalid_mma_4(%a0 : !llvm.vec<2 x half>, %a1 : !llvm.vec<2 x half>,
+                         %b0 : !llvm.vec<2 x half>, %b1 : !llvm.vec<2 x half>,
                          %c0 : !llvm.float, %c1 : !llvm.float, %c2 : !llvm.float, %c3 : !llvm.float,
                          %c4 : !llvm.float, %c5 : !llvm.float, %c6 : !llvm.float, %c7 : !llvm.float) {
   // expected-error@+1 {{unimplemented mma.sync variant}}
-  %0 = nvvm.mma.sync %a0, %a1, %b0, %b1, %c0, %c1, %c2, %c3, %c4, %c5, %c6, %c7 {alayout="row", blayout="col"} : (!llvm<"<2 x half>">, !llvm<"<2 x half>">, !llvm<"<2 x half>">, !llvm<"<2 x half>">, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float) -> !llvm<"{<2 x half>, <2 x half>, <2 x half>, <2 x half>}">
-  llvm.return %0 : !llvm<"{<2 x half>, <2 x half>, <2 x half>, <2 x half>}">
+  %0 = nvvm.mma.sync %a0, %a1, %b0, %b1, %c0, %c1, %c2, %c3, %c4, %c5, %c6, %c7 {alayout="row", blayout="col"} : (!llvm.vec<2 x half>, !llvm.vec<2 x half>, !llvm.vec<2 x half>, !llvm.vec<2 x half>, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float) -> !llvm.struct<(vec<2 x half>, vec<2 x half>, vec<2 x half>, vec<2 x half>)>
+  llvm.return %0 : !llvm.struct<(vec<2 x half>, vec<2 x half>, vec<2 x half>, vec<2 x half>)>
 }
 
 // -----
 
 // CHECK-LABEL: @nvvm_invalid_mma_5
-func @nvvm_invalid_mma_5(%a0 : !llvm<"<2 x half>">, %a1 : !llvm<"<2 x half>">,
-                         %b0 : !llvm<"<2 x half>">, %b1 : !llvm<"<2 x half>">,
+func @nvvm_invalid_mma_5(%a0 : !llvm.vec<2 x half>, %a1 : !llvm.vec<2 x half>,
+                         %b0 : !llvm.vec<2 x half>, %b1 : !llvm.vec<2 x half>,
                          %c0 : !llvm.float, %c1 : !llvm.float, %c2 : !llvm.float, %c3 : !llvm.float,
                          %c4 : !llvm.float, %c5 : !llvm.float, %c6 : !llvm.float, %c7 : !llvm.float) {
   // expected-error@+1 {{unimplemented mma.sync variant}}
-  %0 = nvvm.mma.sync %a0, %a1, %b0, %b1, %c0, %c1, %c2, %c3, %c4, %c5, %c6, %c7 {alayout="col", blayout="row"} : (!llvm<"<2 x half>">, !llvm<"<2 x half>">, !llvm<"<2 x half>">, !llvm<"<2 x half>">, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float) -> !llvm<"{ float, float, float, float, float, float, float, float }">
-  llvm.return %0 : !llvm<"{ float, float, float, float, float, float, float, float }">
+  %0 = nvvm.mma.sync %a0, %a1, %b0, %b1, %c0, %c1, %c2, %c3, %c4, %c5, %c6, %c7 {alayout="col", blayout="row"} : (!llvm.vec<2 x half>, !llvm.vec<2 x half>, !llvm.vec<2 x half>, !llvm.vec<2 x half>, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float) -> !llvm.struct<(float, float, float, float, float, float, float, float)>
+  llvm.return %0 : !llvm.struct<(float, float, float, float, float, float, float, float)>
 }
 
 // -----
 
 // CHECK-LABEL: @nvvm_invalid_mma_6
-func @nvvm_invalid_mma_6(%a0 : !llvm<"<2 x half>">, %a1 : !llvm<"<2 x half>">,
-                         %b0 : !llvm<"<2 x half>">, %b1 : !llvm<"<2 x half>">,
+func @nvvm_invalid_mma_6(%a0 : !llvm.vec<2 x half>, %a1 : !llvm.vec<2 x half>,
+                         %b0 : !llvm.vec<2 x half>, %b1 : !llvm.vec<2 x half>,
                          %c0 : !llvm.float, %c1 : !llvm.float, %c2 : !llvm.float, %c3 : !llvm.float,
                          %c4 : !llvm.float, %c5 : !llvm.float, %c6 : !llvm.float, %c7 : !llvm.float) {
   // expected-error@+1 {{invalid kind of type specified}}
-  %0 = nvvm.mma.sync %a0, %a1, %b0, %b1, %c0, %c1, %c2, %c3, %c4, %c5, %c6, %c7 {alayout="col", blayout="row"} : !llvm<"{ float, float, float, float, float, float, float, float }">
-  llvm.return %0 : !llvm<"{ float, float, float, float, float, float, float, float }">
+  %0 = nvvm.mma.sync %a0, %a1, %b0, %b1, %c0, %c1, %c2, %c3, %c4, %c5, %c6, %c7 {alayout="col", blayout="row"} : !llvm.struct<(float, float, float, float, float, float, float, float)>
+  llvm.return %0 : !llvm.struct<(float, float, float, float, float, float, float, float)>
 }
 
 // -----
 
 // CHECK-LABEL: @nvvm_invalid_mma_7
-func @nvvm_invalid_mma_7(%a0 : !llvm<"<2 x half>">, %a1 : !llvm<"<2 x half>">,
-                         %b0 : !llvm<"<2 x half>">, %b1 : !llvm<"<2 x half>">,
+func @nvvm_invalid_mma_7(%a0 : !llvm.vec<2 x half>, %a1 : !llvm.vec<2 x half>,
+                         %b0 : !llvm.vec<2 x half>, %b1 : !llvm.vec<2 x half>,
                          %c0 : !llvm.float, %c1 : !llvm.float, %c2 : !llvm.float, %c3 : !llvm.float,
                          %c4 : !llvm.float, %c5 : !llvm.float, %c6 : !llvm.float, %c7 : !llvm.float) {
   // expected-error@+1 {{op requires one result}}
-  %0:2 = nvvm.mma.sync %a0, %a1, %b0, %b1, %c0, %c1, %c2, %c3, %c4, %c5, %c6, %c7 {alayout="col", blayout="row"} : (!llvm<"<2 x half>">, !llvm<"<2 x half>">, !llvm<"<2 x half>">, !llvm<"<2 x half>">, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float) -> (!llvm<"{ float, float, float, float, float, float, float, float }">, !llvm.i32)
-  llvm.return %0#0 : !llvm<"{ float, float, float, float, float, float, float, float }">
-}
-
-// -----
-
-// FIXME: the LLVM-IR dialect should parse mutually recursive types
-// CHECK-LABEL: @recursive_type
-// expected-error@+1 {{expected end of string}}
-llvm.func @recursive_type(%a : !llvm<"%a = type { %a* }">) ->
-                          !llvm<"%a = type { %a* }"> {
-  llvm.return %a : !llvm<"%a = type { %a* }">
+  %0:2 = nvvm.mma.sync %a0, %a1, %b0, %b1, %c0, %c1, %c2, %c3, %c4, %c5, %c6, %c7 {alayout="col", blayout="row"} : (!llvm.vec<2 x half>, !llvm.vec<2 x half>, !llvm.vec<2 x half>, !llvm.vec<2 x half>, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float) -> (!llvm.struct<(float, float, float, float, float, float, float, float)>, !llvm.i32)
+  llvm.return %0#0 : !llvm.struct<(float, float, float, float, float, float, float, float)>
 }
 
 // -----
@@ -419,25 +409,25 @@ func @atomicrmw_expected_ptr(%f32 : !llvm.float) {
 // -----
 
 // CHECK-LABEL: @atomicrmw_mismatched_operands
-func @atomicrmw_mismatched_operands(%f32_ptr : !llvm<"float*">, %i32 : !llvm.i32) {
+func @atomicrmw_mismatched_operands(%f32_ptr : !llvm.ptr<float>, %i32 : !llvm.i32) {
   // expected-error@+1 {{expected LLVM IR element type for operand #0 to match type for operand #1}}
-  %0 = "llvm.atomicrmw"(%f32_ptr, %i32) {bin_op=11, ordering=1} : (!llvm<"float*">, !llvm.i32) -> !llvm.float
+  %0 = "llvm.atomicrmw"(%f32_ptr, %i32) {bin_op=11, ordering=1} : (!llvm.ptr<float>, !llvm.i32) -> !llvm.float
   llvm.return
 }
 
 // -----
 
 // CHECK-LABEL: @atomicrmw_mismatched_result
-func @atomicrmw_mismatched_operands(%f32_ptr : !llvm<"float*">, %f32 : !llvm.float) {
+func @atomicrmw_mismatched_operands(%f32_ptr : !llvm.ptr<float>, %f32 : !llvm.float) {
   // expected-error@+1 {{expected LLVM IR result type to match type for operand #1}}
-  %0 = "llvm.atomicrmw"(%f32_ptr, %f32) {bin_op=11, ordering=1} : (!llvm<"float*">, !llvm.float) -> !llvm.i32
+  %0 = "llvm.atomicrmw"(%f32_ptr, %f32) {bin_op=11, ordering=1} : (!llvm.ptr<float>, !llvm.float) -> !llvm.i32
   llvm.return
 }
 
 // -----
 
 // CHECK-LABEL: @atomicrmw_expected_float
-func @atomicrmw_expected_float(%i32_ptr : !llvm<"i32*">, %i32 : !llvm.i32) {
+func @atomicrmw_expected_float(%i32_ptr : !llvm.ptr<i32>, %i32 : !llvm.i32) {
   // expected-error@+1 {{expected LLVM IR floating point type}}
   %0 = llvm.atomicrmw fadd %i32_ptr, %i32 unordered : !llvm.i32
   llvm.return
@@ -446,7 +436,7 @@ func @atomicrmw_expected_float(%i32_ptr : !llvm<"i32*">, %i32 : !llvm.i32) {
 // -----
 
 // CHECK-LABEL: @atomicrmw_unexpected_xchg_type
-func @atomicrmw_unexpected_xchg_type(%i1_ptr : !llvm<"i1*">, %i1 : !llvm.i1) {
+func @atomicrmw_unexpected_xchg_type(%i1_ptr : !llvm.ptr<i1>, %i1 : !llvm.i1) {
   // expected-error@+1 {{unexpected LLVM IR type for 'xchg' bin_op}}
   %0 = llvm.atomicrmw xchg %i1_ptr, %i1 unordered : !llvm.i1
   llvm.return
@@ -455,7 +445,7 @@ func @atomicrmw_unexpected_xchg_type(%i1_ptr : !llvm<"i1*">, %i1 : !llvm.i1) {
 // -----
 
 // CHECK-LABEL: @atomicrmw_expected_int
-func @atomicrmw_expected_int(%f32_ptr : !llvm<"float*">, %f32 : !llvm.float) {
+func @atomicrmw_expected_int(%f32_ptr : !llvm.ptr<float>, %f32 : !llvm.float) {
   // expected-error@+1 {{expected LLVM IR integer type}}
   %0 = llvm.atomicrmw max %f32_ptr, %f32 unordered : !llvm.float
   llvm.return
@@ -464,25 +454,25 @@ func @atomicrmw_expected_int(%f32_ptr : !llvm<"float*">, %f32 : !llvm.float) {
 // -----
 
 // CHECK-LABEL: @cmpxchg_expected_ptr
-func @cmpxchg_expected_ptr(%f32_ptr : !llvm<"float*">, %f32 : !llvm.float) {
+func @cmpxchg_expected_ptr(%f32_ptr : !llvm.ptr<float>, %f32 : !llvm.float) {
   // expected-error@+1 {{expected LLVM IR pointer type for operand #0}}
-  %0 = "llvm.cmpxchg"(%f32, %f32, %f32) {success_ordering=2,failure_ordering=2} : (!llvm.float, !llvm.float, !llvm.float) -> !llvm<"{ float, i1 }">
+  %0 = "llvm.cmpxchg"(%f32, %f32, %f32) {success_ordering=2,failure_ordering=2} : (!llvm.float, !llvm.float, !llvm.float) -> !llvm.struct<(float, i1)>
   llvm.return
 }
 
 // -----
 
 // CHECK-LABEL: @cmpxchg_mismatched_operands
-func @cmpxchg_mismatched_operands(%f32_ptr : !llvm<"float*">, %i32 : !llvm.i32) {
+func @cmpxchg_mismatched_operands(%f32_ptr : !llvm.ptr<float>, %i32 : !llvm.i32) {
   // expected-error@+1 {{expected LLVM IR element type for operand #0 to match type for all other operands}}
-  %0 = "llvm.cmpxchg"(%f32_ptr, %i32, %i32) {success_ordering=2,failure_ordering=2} : (!llvm<"float*">, !llvm.i32, !llvm.i32) -> !llvm<"{ i32, i1 }">
+  %0 = "llvm.cmpxchg"(%f32_ptr, %i32, %i32) {success_ordering=2,failure_ordering=2} : (!llvm.ptr<float>, !llvm.i32, !llvm.i32) -> !llvm.struct<(i32, i1)>
   llvm.return
 }
 
 // -----
 
 // CHECK-LABEL: @cmpxchg_unexpected_type
-func @cmpxchg_unexpected_type(%i1_ptr : !llvm<"i1*">, %i1 : !llvm.i1) {
+func @cmpxchg_unexpected_type(%i1_ptr : !llvm.ptr<i1>, %i1 : !llvm.i1) {
   // expected-error@+1 {{unexpected LLVM IR type}}
   %0 = llvm.cmpxchg %i1_ptr, %i1, %i1 monotonic monotonic : !llvm.i1
   llvm.return
@@ -491,7 +481,7 @@ func @cmpxchg_unexpected_type(%i1_ptr : !llvm<"i1*">, %i1 : !llvm.i1) {
 // -----
 
 // CHECK-LABEL: @cmpxchg_at_least_monotonic_success
-func @cmpxchg_at_least_monotonic_success(%i32_ptr : !llvm<"i32*">, %i32 : !llvm.i32) {
+func @cmpxchg_at_least_monotonic_success(%i32_ptr : !llvm.ptr<i32>, %i32 : !llvm.i32) {
   // expected-error@+1 {{ordering must be at least 'monotonic'}}
   %0 = llvm.cmpxchg %i32_ptr, %i32, %i32 unordered monotonic : !llvm.i32
   llvm.return
@@ -500,7 +490,7 @@ func @cmpxchg_at_least_monotonic_success(%i32_ptr : !llvm<"i32*">, %i32 : !llvm.
 // -----
 
 // CHECK-LABEL: @cmpxchg_at_least_monotonic_failure
-func @cmpxchg_at_least_monotonic_failure(%i32_ptr : !llvm<"i32*">, %i32 : !llvm.i32) {
+func @cmpxchg_at_least_monotonic_failure(%i32_ptr : !llvm.ptr<i32>, %i32 : !llvm.i32) {
   // expected-error@+1 {{ordering must be at least 'monotonic'}}
   %0 = llvm.cmpxchg %i32_ptr, %i32, %i32 monotonic unordered : !llvm.i32
   llvm.return
@@ -509,7 +499,7 @@ func @cmpxchg_at_least_monotonic_failure(%i32_ptr : !llvm<"i32*">, %i32 : !llvm.
 // -----
 
 // CHECK-LABEL: @cmpxchg_failure_release
-func @cmpxchg_failure_release(%i32_ptr : !llvm<"i32*">, %i32 : !llvm.i32) {
+func @cmpxchg_failure_release(%i32_ptr : !llvm.ptr<i32>, %i32 : !llvm.i32) {
   // expected-error@+1 {{failure ordering cannot be 'release' or 'acq_rel'}}
   %0 = llvm.cmpxchg %i32_ptr, %i32, %i32 acq_rel release : !llvm.i32
   llvm.return
@@ -518,7 +508,7 @@ func @cmpxchg_failure_release(%i32_ptr : !llvm<"i32*">, %i32 : !llvm.i32) {
 // -----
 
 // CHECK-LABEL: @cmpxchg_failure_acq_rel
-func @cmpxchg_failure_acq_rel(%i32_ptr : !llvm<"i32*">, %i32 : !llvm.i32) {
+func @cmpxchg_failure_acq_rel(%i32_ptr : !llvm.ptr<i32>, %i32 : !llvm.i32) {
   // expected-error@+1 {{failure ordering cannot be 'release' or 'acq_rel'}}
   %0 = llvm.cmpxchg %i32_ptr, %i32, %i32 acq_rel acq_rel : !llvm.i32
   llvm.return
@@ -529,7 +519,7 @@ func @cmpxchg_failure_acq_rel(%i32_ptr : !llvm<"i32*">, %i32 : !llvm.i32) {
 llvm.func @foo(!llvm.i32) -> !llvm.i32
 llvm.func @__gxx_personality_v0(...) -> !llvm.i32
 
-llvm.func @bad_landingpad(%arg0: !llvm<"i8**">) attributes { personality = @__gxx_personality_v0} {
+llvm.func @bad_landingpad(%arg0: !llvm.ptr<ptr<i8>>) attributes { personality = @__gxx_personality_v0} {
   %0 = llvm.mlir.constant(3 : i32) : !llvm.i32
   %1 = llvm.mlir.constant(2 : i32) : !llvm.i32
   %2 = llvm.invoke @foo(%1) to ^bb1 unwind ^bb2 : (!llvm.i32) -> !llvm.i32
@@ -537,7 +527,7 @@ llvm.func @bad_landingpad(%arg0: !llvm<"i8**">) attributes { personality = @__gx
   llvm.return %1 : !llvm.i32
 ^bb2:  // pred: ^bb0
   // expected-error@+1 {{clause #0 is not a known constant - null, addressof, bitcast}}
-  %3 = llvm.landingpad cleanup (catch %1 : !llvm.i32) (catch %arg0 : !llvm<"i8**">) : !llvm<"{ i8*, i32 }">
+  %3 = llvm.landingpad cleanup (catch %1 : !llvm.i32) (catch %arg0 : !llvm.ptr<ptr<i8>>) : !llvm.struct<(ptr<i8>, i32)>
   llvm.return %0 : !llvm.i32
 }
 
@@ -548,15 +538,15 @@ llvm.func @__gxx_personality_v0(...) -> !llvm.i32
 
 llvm.func @caller(%arg0: !llvm.i32) -> !llvm.i32 attributes { personality = @__gxx_personality_v0} {
   %0 = llvm.mlir.constant(1 : i32) : !llvm.i32
-  %1 = llvm.alloca %0 x !llvm<"i8*"> : (!llvm.i32) -> !llvm<"i8**">
+  %1 = llvm.alloca %0 x !llvm.ptr<i8> : (!llvm.i32) -> !llvm.ptr<ptr<i8>>
   // expected-note@+1 {{global addresses expected as operand to bitcast used in clauses for landingpad}}
-  %2 = llvm.bitcast %1 : !llvm<"i8**"> to !llvm<"i8*">
+  %2 = llvm.bitcast %1 : !llvm.ptr<ptr<i8>> to !llvm.ptr<i8>
   %3 = llvm.invoke @foo(%0) to ^bb1 unwind ^bb2 : (!llvm.i32) -> !llvm.i32
 ^bb1: // pred: ^bb0
   llvm.return %0 : !llvm.i32
 ^bb2: // pred: ^bb0
   // expected-error@+1 {{constant clauses expected}}
-  %5 = llvm.landingpad (catch %2 : !llvm<"i8*">) : !llvm<"{ i8*, i32 }">
+  %5 = llvm.landingpad (catch %2 : !llvm.ptr<i8>) : !llvm.struct<(ptr<i8>, i32)>
   llvm.return %0 : !llvm.i32
 }
 
@@ -572,7 +562,7 @@ llvm.func @caller(%arg0: !llvm.i32) -> !llvm.i32 attributes { personality = @__g
   llvm.return %0 : !llvm.i32
 ^bb2: // pred: ^bb0
   // expected-error@+1 {{landingpad instruction expects at least one clause or cleanup attribute}}
-  %2 = llvm.landingpad : !llvm<"{ i8*, i32 }">
+  %2 = llvm.landingpad : !llvm.struct<(ptr<i8>, i32)>
   llvm.return %0 : !llvm.i32
 }
 
@@ -587,7 +577,7 @@ llvm.func @caller(%arg0: !llvm.i32) -> !llvm.i32 attributes { personality = @__g
 ^bb1: // pred: ^bb0
   llvm.return %0 : !llvm.i32
 ^bb2: // pred: ^bb0
-  %2 = llvm.landingpad cleanup : !llvm<"{ i8*, i32 }">
+  %2 = llvm.landingpad cleanup : !llvm.struct<(ptr<i8>, i32)>
   // expected-error@+1 {{'llvm.resume' op expects landingpad value as operand}}
   llvm.resume %0 : !llvm.i32
 }
@@ -603,8 +593,8 @@ llvm.func @caller(%arg0: !llvm.i32) -> !llvm.i32 {
   llvm.return %0 : !llvm.i32
 ^bb2: // pred: ^bb0
   // expected-error@+1 {{llvm.landingpad needs to be in a function with a personality}}
-  %2 = llvm.landingpad cleanup : !llvm<"{ i8*, i32 }">
-  llvm.resume %2 : !llvm<"{ i8*, i32 }">
+  %2 = llvm.landingpad cleanup : !llvm.struct<(ptr<i8>, i32)>
+  llvm.resume %2 : !llvm.struct<(ptr<i8>, i32)>
 }
 
 // -----
diff --git a/mlir/test/Dialect/LLVMIR/nvvm.mlir b/mlir/test/Dialect/LLVMIR/nvvm.mlir
index a55b35907db03..67d43f7146d5f 100644
--- a/mlir/test/Dialect/LLVMIR/nvvm.mlir
+++ b/mlir/test/Dialect/LLVMIR/nvvm.mlir
@@ -46,12 +46,12 @@ func @nvvm_shfl(
 
 func @nvvm_shfl_pred(
     %arg0 : !llvm.i32, %arg1 : !llvm.i32, %arg2 : !llvm.i32,
-    %arg3 : !llvm.i32, %arg4 : !llvm.float) -> !llvm<"{ i32, i1 }"> {
-  // CHECK: nvvm.shfl.sync.bfly %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} : !llvm<"{ i32, i1 }">
-  %0 = nvvm.shfl.sync.bfly %arg0, %arg3, %arg1, %arg2 {return_value_and_is_valid} : !llvm<"{ i32, i1 }">
-  // CHECK: nvvm.shfl.sync.bfly %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} : !llvm<"{ float, i1 }">
-  %1 = nvvm.shfl.sync.bfly %arg0, %arg4, %arg1, %arg2 {return_value_and_is_valid} : !llvm<"{ float, i1 }">
-  llvm.return %0 : !llvm<"{ i32, i1 }">
+    %arg3 : !llvm.i32, %arg4 : !llvm.float) -> !llvm.struct<(i32, i1)> {
+  // CHECK: nvvm.shfl.sync.bfly %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} : !llvm.struct<(i32, i1)>
+  %0 = nvvm.shfl.sync.bfly %arg0, %arg3, %arg1, %arg2 {return_value_and_is_valid} : !llvm.struct<(i32, i1)>
+  // CHECK: nvvm.shfl.sync.bfly %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} : !llvm.struct<(float, i1)>
+  %1 = nvvm.shfl.sync.bfly %arg0, %arg4, %arg1, %arg2 {return_value_and_is_valid} : !llvm.struct<(float, i1)>
+  llvm.return %0 : !llvm.struct<(i32, i1)>
 }
 
 func @nvvm_vote(%arg0 : !llvm.i32, %arg1 : !llvm.i1) -> !llvm.i32 {
@@ -60,11 +60,11 @@ func @nvvm_vote(%arg0 : !llvm.i32, %arg1 : !llvm.i1) -> !llvm.i32 {
   llvm.return %0 : !llvm.i32
 }
 
-func @nvvm_mma(%a0 : !llvm<"<2 x half>">, %a1 : !llvm<"<2 x half>">,
-               %b0 : !llvm<"<2 x half>">, %b1 : !llvm<"<2 x half>">,
+func @nvvm_mma(%a0 : !llvm.vec<2 x half>, %a1 : !llvm.vec<2 x half>,
+               %b0 : !llvm.vec<2 x half>, %b1 : !llvm.vec<2 x half>,
                %c0 : !llvm.float, %c1 : !llvm.float, %c2 : !llvm.float, %c3 : !llvm.float,
                %c4 : !llvm.float, %c5 : !llvm.float, %c6 : !llvm.float, %c7 : !llvm.float) {
-  // CHECK: nvvm.mma.sync {{.*}} {alayout = "row", blayout = "col"} : (!llvm<"<2 x half>">, !llvm<"<2 x half>">, !llvm<"<2 x half>">, !llvm<"<2 x half>">, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float) -> !llvm<"{ float, float, float, float, float, float, float, float }">
-  %0 = nvvm.mma.sync %a0, %a1, %b0, %b1, %c0, %c1, %c2, %c3, %c4, %c5, %c6, %c7 {alayout="row", blayout="col"} : (!llvm<"<2 x half>">, !llvm<"<2 x half>">, !llvm<"<2 x half>">, !llvm<"<2 x half>">, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float) -> !llvm<"{ float, float, float, float, float, float, float, float }">
-  llvm.return %0 : !llvm<"{ float, float, float, float, float, float, float, float }">
+  // CHECK: nvvm.mma.sync {{.*}} {alayout = "row", blayout = "col"} : (!llvm.vec<2 x half>, !llvm.vec<2 x half>, !llvm.vec<2 x half>, !llvm.vec<2 x half>, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float) -> !llvm.struct<(float, float, float, float, float, float, float, float)>
+  %0 = nvvm.mma.sync %a0, %a1, %b0, %b1, %c0, %c1, %c2, %c3, %c4, %c5, %c6, %c7 {alayout="row", blayout="col"} : (!llvm.vec<2 x half>, !llvm.vec<2 x half>, !llvm.vec<2 x half>, !llvm.vec<2 x half>, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float) -> !llvm.struct<(float, float, float, float, float, float, float, float)>
+  llvm.return %0 : !llvm.struct<(float, float, float, float, float, float, float, float)>
 }
diff --git a/mlir/test/Dialect/LLVMIR/rocdl.mlir b/mlir/test/Dialect/LLVMIR/rocdl.mlir
index 5fb28c45805c4..3f640810c543c 100644
--- a/mlir/test/Dialect/LLVMIR/rocdl.mlir
+++ b/mlir/test/Dialect/LLVMIR/rocdl.mlir
@@ -36,133 +36,133 @@ func @rocdl.barrier() {
 }
 
 func @rocdl.xdlops(%arg0 : !llvm.float, %arg1 : !llvm.float,
-                   %arg2 : !llvm<"<32 x float>">, %arg3 : !llvm.i32,
-                   %arg4 : !llvm<"<16 x float>">, %arg5 : !llvm<"<4 x float>">,
-                   %arg6 : !llvm<"<4 x half>">, %arg7 : !llvm<"<32 x i32>">,
-                   %arg8 : !llvm<"<16 x i32>">, %arg9 : !llvm<"<4 x i32>">,
-                   %arg10 : !llvm<"<2 x i16>">) -> !llvm<"<32 x float>"> {
+                   %arg2 : !llvm.vec<32 x float>, %arg3 : !llvm.i32,
+                   %arg4 : !llvm.vec<16 x float>, %arg5 : !llvm.vec<4 x float>,
+                   %arg6 : !llvm.vec<4 x half>, %arg7 : !llvm.vec<32 x i32>,
+                   %arg8 : !llvm.vec<16 x i32>, %arg9 : !llvm.vec<4 x i32>,
+                   %arg10 : !llvm.vec<2 x i16>) -> !llvm.vec<32 x float> {
   // CHECK-LABEL: rocdl.xdlops
-  // CHECK: rocdl.mfma.f32.32x32x1f32 {{.*}} : (!llvm.float, !llvm.float, !llvm<"<32 x float>">, !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm<"<32 x float>">
+  // CHECK: rocdl.mfma.f32.32x32x1f32 {{.*}} : (!llvm.float, !llvm.float, !llvm.vec<32 x float>, !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm.vec<32 x float>
   %r0 = rocdl.mfma.f32.32x32x1f32 %arg0, %arg1, %arg2, %arg3, %arg3, %arg3 :
-                            (!llvm.float, !llvm.float, !llvm<"<32 x float>">,
-                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm<"<32 x float>">
+                            (!llvm.float, !llvm.float, !llvm.vec<32 x float>,
+                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm.vec<32 x float>
 
-  // CHECK: rocdl.mfma.f32.16x16x1f32 {{.*}} : (!llvm.float, !llvm.float, !llvm<"<16 x float>">, !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm<"<16 x float>">
+  // CHECK: rocdl.mfma.f32.16x16x1f32 {{.*}} : (!llvm.float, !llvm.float, !llvm.vec<16 x float>, !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm.vec<16 x float>
   %r1 = rocdl.mfma.f32.16x16x1f32 %arg0, %arg1, %arg4, %arg3, %arg3, %arg3 :
-                            (!llvm.float, !llvm.float, !llvm<"<16 x float>">,
-                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm<"<16 x float>">
+                            (!llvm.float, !llvm.float, !llvm.vec<16 x float>,
+                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm.vec<16 x float>
 
-  // CHECK: rocdl.mfma.f32.16x16x4f32 {{.*}} : (!llvm.float, !llvm.float, !llvm<"<4 x float>">, !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm<"<4 x float>">
+  // CHECK: rocdl.mfma.f32.16x16x4f32 {{.*}} : (!llvm.float, !llvm.float, !llvm.vec<4 x float>, !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm.vec<4 x float>
   %r2 = rocdl.mfma.f32.16x16x4f32 %arg0, %arg1, %arg5, %arg3, %arg3, %arg3 :
-                            (!llvm.float, !llvm.float, !llvm<"<4 x float>">,
-                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm<"<4 x float>">
+                            (!llvm.float, !llvm.float, !llvm.vec<4 x float>,
+                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm.vec<4 x float>
 
-  // CHECK: rocdl.mfma.f32.4x4x1f32 {{.*}} : (!llvm.float, !llvm.float, !llvm<"<4 x float>">, !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm<"<4 x float>">
+  // CHECK: rocdl.mfma.f32.4x4x1f32 {{.*}} : (!llvm.float, !llvm.float, !llvm.vec<4 x float>, !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm.vec<4 x float>
   %r3 = rocdl.mfma.f32.4x4x1f32 %arg0, %arg1, %arg5, %arg3, %arg3, %arg3 :
-                            (!llvm.float, !llvm.float, !llvm<"<4 x float>">,
-                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm<"<4 x float>">
+                            (!llvm.float, !llvm.float, !llvm.vec<4 x float>,
+                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm.vec<4 x float>
 
-  // CHECK: rocdl.mfma.f32.32x32x2f32 {{.*}} : (!llvm.float, !llvm.float, !llvm<"<16 x float>">, !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm<"<16 x float>">
+  // CHECK: rocdl.mfma.f32.32x32x2f32 {{.*}} : (!llvm.float, !llvm.float, !llvm.vec<16 x float>, !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm.vec<16 x float>
   %r4= rocdl.mfma.f32.32x32x2f32 %arg0, %arg1, %arg4, %arg3, %arg3, %arg3 :
-                            (!llvm.float, !llvm.float, !llvm<"<16 x float>">,
-                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm<"<16 x float>">
+                            (!llvm.float, !llvm.float, !llvm.vec<16 x float>,
+                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm.vec<16 x float>
 
-  // CHECK: rocdl.mfma.f32.32x32x4f16 {{.*}} : (!llvm<"<4 x half>">, !llvm<"<4 x half>">, !llvm<"<32 x float>">, !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm<"<32 x float>">
+  // CHECK: rocdl.mfma.f32.32x32x4f16 {{.*}} : (!llvm.vec<4 x half>, !llvm.vec<4 x half>, !llvm.vec<32 x float>, !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm.vec<32 x float>
   %r5 = rocdl.mfma.f32.32x32x4f16 %arg6, %arg6, %arg2, %arg3, %arg3, %arg3 :
-                            (!llvm<"<4 x half>">, !llvm<"<4 x half>">, !llvm<"<32 x float>">,
-                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm<"<32 x float>">
+                            (!llvm.vec<4 x half>, !llvm.vec<4 x half>, !llvm.vec<32 x float>,
+                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm.vec<32 x float>
 
-  // CHECK: rocdl.mfma.f32.16x16x4f16 {{.*}} : (!llvm<"<4 x half>">, !llvm<"<4 x half>">, !llvm<"<16 x float>">, !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm<"<16 x float>">
+  // CHECK: rocdl.mfma.f32.16x16x4f16 {{.*}} : (!llvm.vec<4 x half>, !llvm.vec<4 x half>, !llvm.vec<16 x float>, !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm.vec<16 x float>
   %r6 = rocdl.mfma.f32.16x16x4f16 %arg6, %arg6, %arg4, %arg3, %arg3, %arg3 :
-                            (!llvm<"<4 x half>">, !llvm<"<4 x half>">, !llvm<"<16 x float>">,
-                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm<"<16 x float>">
+                            (!llvm.vec<4 x half>, !llvm.vec<4 x half>, !llvm.vec<16 x float>,
+                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm.vec<16 x float>
 
-  // CHECK: rocdl.mfma.f32.4x4x4f16 {{.*}} : (!llvm<"<4 x half>">, !llvm<"<4 x half>">, !llvm<"<4 x float>">, !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm<"<4 x float>">
+  // CHECK: rocdl.mfma.f32.4x4x4f16 {{.*}} : (!llvm.vec<4 x half>, !llvm.vec<4 x half>, !llvm.vec<4 x float>, !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm.vec<4 x float>
   %r7 = rocdl.mfma.f32.4x4x4f16 %arg6, %arg6, %arg5, %arg3, %arg3, %arg3 :
-                            (!llvm<"<4 x half>">, !llvm<"<4 x half>">, !llvm<"<4 x float>">,
-                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm<"<4 x float>">
+                            (!llvm.vec<4 x half>, !llvm.vec<4 x half>, !llvm.vec<4 x float>,
+                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm.vec<4 x float>
 
-  // CHECK: rocdl.mfma.f32.32x32x8f16 {{.*}} : (!llvm<"<4 x half>">, !llvm<"<4 x half>">, !llvm<"<16 x float>">, !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm<"<16 x float>">
+  // CHECK: rocdl.mfma.f32.32x32x8f16 {{.*}} : (!llvm.vec<4 x half>, !llvm.vec<4 x half>, !llvm.vec<16 x float>, !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm.vec<16 x float>
   %r8 = rocdl.mfma.f32.32x32x8f16 %arg6, %arg6, %arg4, %arg3, %arg3, %arg3 :
-                            (!llvm<"<4 x half>">, !llvm<"<4 x half>">, !llvm<"<16 x float>">,
-                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm<"<16 x float>">
+                            (!llvm.vec<4 x half>, !llvm.vec<4 x half>, !llvm.vec<16 x float>,
+                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm.vec<16 x float>
 
-  // CHECK: rocdl.mfma.f32.16x16x16f16 {{.*}} : (!llvm<"<4 x half>">, !llvm<"<4 x half>">, !llvm<"<4 x float>">, !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm<"<4 x float>">
+  // CHECK: rocdl.mfma.f32.16x16x16f16 {{.*}} : (!llvm.vec<4 x half>, !llvm.vec<4 x half>, !llvm.vec<4 x float>, !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm.vec<4 x float>
   %r9 = rocdl.mfma.f32.16x16x16f16 %arg6, %arg6, %arg5, %arg3, %arg3, %arg3 :
-                            (!llvm<"<4 x half>">, !llvm<"<4 x half>">, !llvm<"<4 x float>">,
-                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm<"<4 x float>">
+                            (!llvm.vec<4 x half>, !llvm.vec<4 x half>, !llvm.vec<4 x float>,
+                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm.vec<4 x float>
 
-  // CHECK: rocdl.mfma.i32.32x32x4i8 {{.*}} : (!llvm.i32, !llvm.i32, !llvm<"<32 x i32>">, !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm<"<32 x i32>">
+  // CHECK: rocdl.mfma.i32.32x32x4i8 {{.*}} : (!llvm.i32, !llvm.i32, !llvm.vec<32 x i32>, !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm.vec<32 x i32>
   %r10 = rocdl.mfma.i32.32x32x4i8 %arg3, %arg3, %arg7, %arg3, %arg3, %arg3 :
-                            (!llvm.i32, !llvm.i32, !llvm<"<32 x i32>">,
-                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm<"<32 x i32>">
+                            (!llvm.i32, !llvm.i32, !llvm.vec<32 x i32>,
+                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm.vec<32 x i32>
 
-  // CHECK: rocdl.mfma.i32.16x16x4i8 {{.*}} : (!llvm.i32, !llvm.i32, !llvm<"<16 x i32>">, !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm<"<16 x i32>">
+  // CHECK: rocdl.mfma.i32.16x16x4i8 {{.*}} : (!llvm.i32, !llvm.i32, !llvm.vec<16 x i32>, !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm.vec<16 x i32>
   %r11 = rocdl.mfma.i32.16x16x4i8 %arg3, %arg3, %arg8, %arg3, %arg3, %arg3 :
-                            (!llvm.i32, !llvm.i32, !llvm<"<16 x i32>">,
-                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm<"<16 x i32>">
+                            (!llvm.i32, !llvm.i32, !llvm.vec<16 x i32>,
+                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm.vec<16 x i32>
 
-  // CHECK: rocdl.mfma.i32.4x4x4i8 {{.*}} : (!llvm.i32, !llvm.i32, !llvm<"<4 x i32>">, !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm<"<4 x i32>">
+  // CHECK: rocdl.mfma.i32.4x4x4i8 {{.*}} : (!llvm.i32, !llvm.i32, !llvm.vec<4 x i32>, !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm.vec<4 x i32>
   %r12 = rocdl.mfma.i32.4x4x4i8 %arg3, %arg3, %arg9, %arg3, %arg3, %arg3 :
-                            (!llvm.i32, !llvm.i32, !llvm<"<4 x i32>">,
-                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm<"<4 x i32>">
+                            (!llvm.i32, !llvm.i32, !llvm.vec<4 x i32>,
+                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm.vec<4 x i32>
 
-  // CHECK: rocdl.mfma.i32.32x32x8i8 {{.*}} : (!llvm.i32, !llvm.i32, !llvm<"<16 x i32>">, !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm<"<16 x i32>">
+  // CHECK: rocdl.mfma.i32.32x32x8i8 {{.*}} : (!llvm.i32, !llvm.i32, !llvm.vec<16 x i32>, !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm.vec<16 x i32>
   %r13 = rocdl.mfma.i32.32x32x8i8 %arg3, %arg3, %arg8, %arg3, %arg3, %arg3 :
-                            (!llvm.i32, !llvm.i32, !llvm<"<16 x i32>">,
-                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm<"<16 x i32>">
+                            (!llvm.i32, !llvm.i32, !llvm.vec<16 x i32>,
+                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm.vec<16 x i32>
 
-  // CHECK: rocdl.mfma.i32.16x16x16i8 {{.*}} : (!llvm.i32, !llvm.i32, !llvm<"<4 x i32>">, !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm<"<4 x i32>">
+  // CHECK: rocdl.mfma.i32.16x16x16i8 {{.*}} : (!llvm.i32, !llvm.i32, !llvm.vec<4 x i32>, !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm.vec<4 x i32>
   %r14 = rocdl.mfma.i32.16x16x16i8 %arg3, %arg3, %arg9, %arg3, %arg3, %arg3 :
-                            (!llvm.i32, !llvm.i32, !llvm<"<4 x i32>">,
-                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm<"<4 x i32>">
+                            (!llvm.i32, !llvm.i32, !llvm.vec<4 x i32>,
+                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm.vec<4 x i32>
 
-  // CHECK: rocdl.mfma.f32.32x32x2bf16 {{.*}} : (!llvm<"<2 x i16>">, !llvm<"<2 x i16>">, !llvm<"<32 x float>">, !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm<"<32 x float>">
+  // CHECK: rocdl.mfma.f32.32x32x2bf16 {{.*}} : (!llvm.vec<2 x i16>, !llvm.vec<2 x i16>, !llvm.vec<32 x float>, !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm.vec<32 x float>
   %r15 = rocdl.mfma.f32.32x32x2bf16 %arg10, %arg10, %arg2, %arg3, %arg3, %arg3 :
-                            (!llvm<"<2 x i16>">, !llvm<"<2 x i16>">, !llvm<"<32 x float>">,
-                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm<"<32 x float>">
+                            (!llvm.vec<2 x i16>, !llvm.vec<2 x i16>, !llvm.vec<32 x float>,
+                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm.vec<32 x float>
 
-  // CHECK: rocdl.mfma.f32.16x16x2bf16 {{.*}} : (!llvm<"<2 x i16>">, !llvm<"<2 x i16>">, !llvm<"<16 x float>">, !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm<"<16 x float>">
+  // CHECK: rocdl.mfma.f32.16x16x2bf16 {{.*}} : (!llvm.vec<2 x i16>, !llvm.vec<2 x i16>, !llvm.vec<16 x float>, !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm.vec<16 x float>
   %r16 = rocdl.mfma.f32.16x16x2bf16 %arg10, %arg10, %arg4, %arg3, %arg3, %arg3 :
-                            (!llvm<"<2 x i16>">, !llvm<"<2 x i16>">, !llvm<"<16 x float>">,
-                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm<"<16 x float>">
+                            (!llvm.vec<2 x i16>, !llvm.vec<2 x i16>, !llvm.vec<16 x float>,
+                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm.vec<16 x float>
 
-  // CHECK: rocdl.mfma.f32.4x4x2bf16 {{.*}} : (!llvm<"<2 x i16>">, !llvm<"<2 x i16>">, !llvm<"<4 x float>">, !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm<"<4 x float>">
+  // CHECK: rocdl.mfma.f32.4x4x2bf16 {{.*}} : (!llvm.vec<2 x i16>, !llvm.vec<2 x i16>, !llvm.vec<4 x float>, !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm.vec<4 x float>
   %r17 = rocdl.mfma.f32.4x4x2bf16 %arg10, %arg10, %arg5, %arg3, %arg3, %arg3 :
-                            (!llvm<"<2 x i16>">, !llvm<"<2 x i16>">, !llvm<"<4 x float>">,
-                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm<"<4 x float>">
+                            (!llvm.vec<2 x i16>, !llvm.vec<2 x i16>, !llvm.vec<4 x float>,
+                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm.vec<4 x float>
 
-  // CHECK: rocdl.mfma.f32.32x32x4bf16 {{.*}} : (!llvm<"<2 x i16>">, !llvm<"<2 x i16>">, !llvm<"<16 x float>">, !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm<"<16 x float>">
+  // CHECK: rocdl.mfma.f32.32x32x4bf16 {{.*}} : (!llvm.vec<2 x i16>, !llvm.vec<2 x i16>, !llvm.vec<16 x float>, !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm.vec<16 x float>
   %r18 = rocdl.mfma.f32.32x32x4bf16 %arg10, %arg10, %arg4, %arg3, %arg3, %arg3 :
-                            (!llvm<"<2 x i16>">, !llvm<"<2 x i16>">, !llvm<"<16 x float>">,
-                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm<"<16 x float>">
+                            (!llvm.vec<2 x i16>, !llvm.vec<2 x i16>, !llvm.vec<16 x float>,
+                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm.vec<16 x float>
 
-  // CHECK: rocdl.mfma.f32.16x16x8bf16 {{.*}} : (!llvm<"<2 x i16>">, !llvm<"<2 x i16>">, !llvm<"<4 x float>">, !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm<"<4 x float>">
+  // CHECK: rocdl.mfma.f32.16x16x8bf16 {{.*}} : (!llvm.vec<2 x i16>, !llvm.vec<2 x i16>, !llvm.vec<4 x float>, !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm.vec<4 x float>
   %r19 = rocdl.mfma.f32.16x16x8bf16 %arg10, %arg10, %arg5, %arg3, %arg3, %arg3 :
-                            (!llvm<"<2 x i16>">, !llvm<"<2 x i16>">, !llvm<"<4 x float>">,
-                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm<"<4 x float>">
+                            (!llvm.vec<2 x i16>, !llvm.vec<2 x i16>, !llvm.vec<4 x float>,
+                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm.vec<4 x float>
 
-  llvm.return %r0 : !llvm<"<32 x float>">
+  llvm.return %r0 : !llvm.vec<32 x float>
 }
 
-llvm.func @rocdl.mubuf(%rsrc : !llvm<"<4 x i32>">, %vindex : !llvm.i32,
+llvm.func @rocdl.mubuf(%rsrc : !llvm.vec<4 x i32>, %vindex : !llvm.i32,
                        %offset : !llvm.i32, %glc : !llvm.i1,
-                       %slc : !llvm.i1, %vdata1 : !llvm<"<1 x float>">,
-                       %vdata2 : !llvm<"<2 x float>">, %vdata4 : !llvm<"<4 x float>">) {
+                       %slc : !llvm.i1, %vdata1 : !llvm.vec<1 x float>,
+                       %vdata2 : !llvm.vec<2 x float>, %vdata4 : !llvm.vec<4 x float>) {
   // CHECK-LABEL: rocdl.mubuf
-  // CHECK: %{{.*}} = rocdl.buffer.load %{{.*}} %{{.*}} %{{.*}} %{{.*}} %{{.*}} : !llvm<"<1 x float>">
-  %r1 = rocdl.buffer.load %rsrc, %vindex, %offset, %glc, %slc : !llvm<"<1 x float>">
-  // CHECK: %{{.*}} = rocdl.buffer.load %{{.*}} %{{.*}} %{{.*}} %{{.*}} %{{.*}} : !llvm<"<2 x float>">
-  %r2 = rocdl.buffer.load %rsrc, %vindex, %offset, %glc, %slc : !llvm<"<2 x float>">
-  // CHECK: %{{.*}} = rocdl.buffer.load %{{.*}} %{{.*}} %{{.*}} %{{.*}} %{{.*}} : !llvm<"<4 x float>">
-  %r4 = rocdl.buffer.load %rsrc, %vindex, %offset, %glc, %slc : !llvm<"<4 x float>">
-
-  // CHECK: rocdl.buffer.store %{{.*}} %{{.*}} %{{.*}} %{{.*}} %{{.*}} %{{.*}} : !llvm<"<1 x float>">
-  rocdl.buffer.store %vdata1, %rsrc, %vindex, %offset, %glc, %slc : !llvm<"<1 x float>">
-  // CHECK: rocdl.buffer.store %{{.*}} %{{.*}} %{{.*}} %{{.*}} %{{.*}} %{{.*}} : !llvm<"<2 x float>">
-  rocdl.buffer.store %vdata2, %rsrc, %vindex, %offset, %glc, %slc : !llvm<"<2 x float>">
-  // CHECK: rocdl.buffer.store %{{.*}} %{{.*}} %{{.*}} %{{.*}} %{{.*}} %{{.*}} : !llvm<"<4 x float>">
-  rocdl.buffer.store %vdata4, %rsrc, %vindex, %offset, %glc, %slc : !llvm<"<4 x float>">
+  // CHECK: %{{.*}} = rocdl.buffer.load %{{.*}} %{{.*}} %{{.*}} %{{.*}} %{{.*}} : !llvm.vec<1 x float>
+  %r1 = rocdl.buffer.load %rsrc, %vindex, %offset, %glc, %slc : !llvm.vec<1 x float>
+  // CHECK: %{{.*}} = rocdl.buffer.load %{{.*}} %{{.*}} %{{.*}} %{{.*}} %{{.*}} : !llvm.vec<2 x float>
+  %r2 = rocdl.buffer.load %rsrc, %vindex, %offset, %glc, %slc : !llvm.vec<2 x float>
+  // CHECK: %{{.*}} = rocdl.buffer.load %{{.*}} %{{.*}} %{{.*}} %{{.*}} %{{.*}} : !llvm.vec<4 x float>
+  %r4 = rocdl.buffer.load %rsrc, %vindex, %offset, %glc, %slc : !llvm.vec<4 x float>
+
+  // CHECK: rocdl.buffer.store %{{.*}} %{{.*}} %{{.*}} %{{.*}} %{{.*}} %{{.*}} : !llvm.vec<1 x float>
+  rocdl.buffer.store %vdata1, %rsrc, %vindex, %offset, %glc, %slc : !llvm.vec<1 x float>
+  // CHECK: rocdl.buffer.store %{{.*}} %{{.*}} %{{.*}} %{{.*}} %{{.*}} %{{.*}} : !llvm.vec<2 x float>
+  rocdl.buffer.store %vdata2, %rsrc, %vindex, %offset, %glc, %slc : !llvm.vec<2 x float>
+  // CHECK: rocdl.buffer.store %{{.*}} %{{.*}} %{{.*}} %{{.*}} %{{.*}} %{{.*}} : !llvm.vec<4 x float>
+  rocdl.buffer.store %vdata4, %rsrc, %vindex, %offset, %glc, %slc : !llvm.vec<4 x float>
 
   llvm.return
 }
diff --git a/mlir/test/Dialect/LLVMIR/roundtrip.mlir b/mlir/test/Dialect/LLVMIR/roundtrip.mlir
index 1180561cf77b2..ef89d76387d77 100644
--- a/mlir/test/Dialect/LLVMIR/roundtrip.mlir
+++ b/mlir/test/Dialect/LLVMIR/roundtrip.mlir
@@ -1,9 +1,9 @@
 // RUN: mlir-opt %s | mlir-opt | FileCheck %s
 
 // CHECK-LABEL: func @ops
-// CHECK-SAME: (%[[I32:.*]]: !llvm.i32, %[[FLOAT:.*]]: !llvm.float, %[[I8PTR1:.*]]: !llvm<"i8*">, %[[I8PTR2:.*]]: !llvm<"i8*">, %[[BOOL:.*]]: !llvm.i1)
+// CHECK-SAME: (%[[I32:.*]]: !llvm.i32, %[[FLOAT:.*]]: !llvm.float, %[[I8PTR1:.*]]: !llvm.ptr<i8>, %[[I8PTR2:.*]]: !llvm.ptr<i8>, %[[BOOL:.*]]: !llvm.i1)
 func @ops(%arg0: !llvm.i32, %arg1: !llvm.float,
-          %arg2: !llvm<"i8*">, %arg3: !llvm<"i8*">,
+          %arg2: !llvm.ptr<i8>, %arg3: !llvm.ptr<i8>,
           %arg4: !llvm.i1) {
 // Integer arithmetic binary operations.
 //
@@ -39,29 +39,29 @@ func @ops(%arg0: !llvm.i32, %arg1: !llvm.float,
 
 // Memory-related operations.
 //
-// CHECK-NEXT:  %[[ALLOCA:.*]] = llvm.alloca %[[I32]] x !llvm.double : (!llvm.i32) -> !llvm<"double*">
-// CHECK-NEXT:  %[[GEP:.*]] = llvm.getelementptr %[[ALLOCA]][%[[I32]], %[[I32]]] : (!llvm<"double*">, !llvm.i32, !llvm.i32) -> !llvm<"double*">
-// CHECK-NEXT:  %[[VALUE:.*]] = llvm.load %[[GEP]] : !llvm<"double*">
-// CHECK-NEXT:  llvm.store %[[VALUE]], %[[ALLOCA]] : !llvm<"double*">
-// CHECK-NEXT:  %{{.*}} = llvm.bitcast %[[ALLOCA]] : !llvm<"double*"> to !llvm<"i64*">
-  %13 = llvm.alloca %arg0 x !llvm.double : (!llvm.i32) -> !llvm<"double*">
-  %14 = llvm.getelementptr %13[%arg0, %arg0] : (!llvm<"double*">, !llvm.i32, !llvm.i32) -> !llvm<"double*">
-  %15 = llvm.load %14 : !llvm<"double*">
-  llvm.store %15, %13 : !llvm<"double*">
-  %16 = llvm.bitcast %13 : !llvm<"double*"> to !llvm<"i64*">
+// CHECK-NEXT:  %[[ALLOCA:.*]] = llvm.alloca %[[I32]] x !llvm.double : (!llvm.i32) -> !llvm.ptr<double>
+// CHECK-NEXT:  %[[GEP:.*]] = llvm.getelementptr %[[ALLOCA]][%[[I32]], %[[I32]]] : (!llvm.ptr<double>, !llvm.i32, !llvm.i32) -> !llvm.ptr<double>
+// CHECK-NEXT:  %[[VALUE:.*]] = llvm.load %[[GEP]] : !llvm.ptr<double>
+// CHECK-NEXT:  llvm.store %[[VALUE]], %[[ALLOCA]] : !llvm.ptr<double>
+// CHECK-NEXT:  %{{.*}} = llvm.bitcast %[[ALLOCA]] : !llvm.ptr<double> to !llvm.ptr<i64>
+  %13 = llvm.alloca %arg0 x !llvm.double : (!llvm.i32) -> !llvm.ptr<double>
+  %14 = llvm.getelementptr %13[%arg0, %arg0] : (!llvm.ptr<double>, !llvm.i32, !llvm.i32) -> !llvm.ptr<double>
+  %15 = llvm.load %14 : !llvm.ptr<double>
+  llvm.store %15, %13 : !llvm.ptr<double>
+  %16 = llvm.bitcast %13 : !llvm.ptr<double> to !llvm.ptr<i64>
 
 // Function call-related operations.
 //
-// CHECK: %[[STRUCT:.*]] = llvm.call @foo(%[[I32]]) : (!llvm.i32) -> !llvm<"{ i32, double, i32 }">
-// CHECK: %[[VALUE:.*]] = llvm.extractvalue %[[STRUCT]][0] : !llvm<"{ i32, double, i32 }">
-// CHECK: %[[NEW_STRUCT:.*]] = llvm.insertvalue %[[VALUE]], %[[STRUCT]][2] : !llvm<"{ i32, double, i32 }">
-// CHECK: %[[FUNC:.*]] = llvm.mlir.addressof @foo : !llvm<"{ i32, double, i32 } (i32)*">
-// CHECK: %{{.*}} = llvm.call %[[FUNC]](%[[I32]]) : (!llvm.i32) -> !llvm<"{ i32, double, i32 }">
-  %17 = llvm.call @foo(%arg0) : (!llvm.i32) -> !llvm<"{ i32, double, i32 }">
-  %18 = llvm.extractvalue %17[0] : !llvm<"{ i32, double, i32 }">
-  %19 = llvm.insertvalue %18, %17[2] : !llvm<"{ i32, double, i32 }">
-  %20 = llvm.mlir.addressof @foo : !llvm<"{ i32, double, i32 } (i32)*">
-  %21 = llvm.call %20(%arg0) : (!llvm.i32) -> !llvm<"{ i32, double, i32 }">
+// CHECK: %[[STRUCT:.*]] = llvm.call @foo(%[[I32]]) : (!llvm.i32) -> !llvm.struct<(i32, double, i32)>
+// CHECK: %[[VALUE:.*]] = llvm.extractvalue %[[STRUCT]][0] : !llvm.struct<(i32, double, i32)>
+// CHECK: %[[NEW_STRUCT:.*]] = llvm.insertvalue %[[VALUE]], %[[STRUCT]][2] : !llvm.struct<(i32, double, i32)>
+// CHECK: %[[FUNC:.*]] = llvm.mlir.addressof @foo : !llvm.ptr<func<struct<(i32, double, i32)> (i32)>>
+// CHECK: %{{.*}} = llvm.call %[[FUNC]](%[[I32]]) : (!llvm.i32) -> !llvm.struct<(i32, double, i32)>
+  %17 = llvm.call @foo(%arg0) : (!llvm.i32) -> !llvm.struct<(i32, double, i32)>
+  %18 = llvm.extractvalue %17[0] : !llvm.struct<(i32, double, i32)>
+  %19 = llvm.insertvalue %18, %17[2] : !llvm.struct<(i32, double, i32)>
+  %20 = llvm.mlir.addressof @foo : !llvm.ptr<func<struct<(i32, double, i32)> (i32)>>
+  %21 = llvm.call %20(%arg0) : (!llvm.i32) -> !llvm.struct<(i32, double, i32)>
 
 
 // Terminator operations and their successors.
@@ -76,9 +76,9 @@ func @ops(%arg0: !llvm.i32, %arg1: !llvm.float,
 
 // CHECK: ^[[BB2]]
 ^bb2:
-// CHECK: %{{.*}} = llvm.mlir.undef : !llvm<"{ i32, double, i32 }">
+// CHECK: %{{.*}} = llvm.mlir.undef : !llvm.struct<(i32, double, i32)>
 // CHECK: %{{.*}} = llvm.mlir.constant(42 : i64) : !llvm.i47
-  %22 = llvm.mlir.undef : !llvm<"{ i32, double, i32 }">
+  %22 = llvm.mlir.undef : !llvm.struct<(i32, double, i32)>
   %23 = llvm.mlir.constant(42) : !llvm.i47
 
 // Misc operations.
@@ -87,10 +87,10 @@ func @ops(%arg0: !llvm.i32, %arg1: !llvm.float,
 
 // Integer to pointer and pointer to integer conversions.
 //
-// CHECK: %[[PTR:.*]] = llvm.inttoptr %[[I32]] : !llvm.i32 to !llvm<"i32*">
-// CHECK: %{{.*}} = llvm.ptrtoint %[[PTR]] : !llvm<"i32*"> to !llvm.i32
-  %25 = llvm.inttoptr %arg0 : !llvm.i32 to !llvm<"i32*">
-  %26 = llvm.ptrtoint %25 : !llvm<"i32*"> to !llvm.i32
+// CHECK: %[[PTR:.*]] = llvm.inttoptr %[[I32]] : !llvm.i32 to !llvm.ptr<i32>
+// CHECK: %{{.*}} = llvm.ptrtoint %[[PTR]] : !llvm.ptr<i32> to !llvm.i32
+  %25 = llvm.inttoptr %arg0 : !llvm.i32 to !llvm.ptr<i32>
+  %26 = llvm.ptrtoint %25 : !llvm.ptr<i32> to !llvm.i32
 
 // Extended and Quad floating point
 //
@@ -114,24 +114,24 @@ func @ops(%arg0: !llvm.i32, %arg1: !llvm.float,
 // CHECK: "llvm.intr.ctpop"(%{{.*}}) : (!llvm.i32) -> !llvm.i32
   %33 = "llvm.intr.ctpop"(%arg0) : (!llvm.i32) -> !llvm.i32
 
-// CHECK: "llvm.intr.memcpy"(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (!llvm<"i8*">, !llvm<"i8*">, !llvm.i32, !llvm.i1) -> ()
-  "llvm.intr.memcpy"(%arg2, %arg3, %arg0, %arg4) : (!llvm<"i8*">, !llvm<"i8*">, !llvm.i32, !llvm.i1) -> ()
+// CHECK: "llvm.intr.memcpy"(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (!llvm.ptr<i8>, !llvm.ptr<i8>, !llvm.i32, !llvm.i1) -> ()
+  "llvm.intr.memcpy"(%arg2, %arg3, %arg0, %arg4) : (!llvm.ptr<i8>, !llvm.ptr<i8>, !llvm.i32, !llvm.i1) -> ()
 
-// CHECK: "llvm.intr.memcpy"(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (!llvm<"i8*">, !llvm<"i8*">, !llvm.i32, !llvm.i1) -> ()
-  "llvm.intr.memcpy"(%arg2, %arg3, %arg0, %arg4) : (!llvm<"i8*">, !llvm<"i8*">, !llvm.i32, !llvm.i1) -> ()
+// CHECK: "llvm.intr.memcpy"(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (!llvm.ptr<i8>, !llvm.ptr<i8>, !llvm.i32, !llvm.i1) -> ()
+  "llvm.intr.memcpy"(%arg2, %arg3, %arg0, %arg4) : (!llvm.ptr<i8>, !llvm.ptr<i8>, !llvm.i32, !llvm.i1) -> ()
 
 // CHECK: %[[SZ:.*]] = llvm.mlir.constant
   %sz = llvm.mlir.constant(10: i64) : !llvm.i64
-// CHECK: "llvm.intr.memcpy.inline"(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (!llvm<"i8*">, !llvm<"i8*">, !llvm.i64, !llvm.i1) -> ()
-  "llvm.intr.memcpy.inline"(%arg2, %arg3, %sz, %arg4) : (!llvm<"i8*">, !llvm<"i8*">, !llvm.i64, !llvm.i1) -> ()
+// CHECK: "llvm.intr.memcpy.inline"(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (!llvm.ptr<i8>, !llvm.ptr<i8>, !llvm.i64, !llvm.i1) -> ()
+  "llvm.intr.memcpy.inline"(%arg2, %arg3, %sz, %arg4) : (!llvm.ptr<i8>, !llvm.ptr<i8>, !llvm.i64, !llvm.i1) -> ()
 
 // CHECK:  llvm.return
   llvm.return
 }
 
 // An larger self-contained function.
-// CHECK-LABEL: llvm.func @foo(%{{.*}}: !llvm.i32) -> !llvm<"{ i32, double, i32 }"> {
-llvm.func @foo(%arg0: !llvm.i32) -> !llvm<"{ i32, double, i32 }"> {
+// CHECK-LABEL: llvm.func @foo(%{{.*}}: !llvm.i32) -> !llvm.struct<(i32, double, i32)> {
+llvm.func @foo(%arg0: !llvm.i32) -> !llvm.struct<(i32, double, i32)> {
 // CHECK:  %[[V0:.*]] = llvm.mlir.constant(3 : i64) : !llvm.i32
 // CHECK:  %[[V1:.*]] = llvm.mlir.constant(3 : i64) : !llvm.i32
 // CHECK:  %[[V2:.*]] = llvm.mlir.constant(4.200000e+01 : f64) : !llvm.double
@@ -154,56 +154,56 @@ llvm.func @foo(%arg0: !llvm.i32) -> !llvm<"{ i32, double, i32 }"> {
   llvm.cond_br %8, ^bb1(%4 : !llvm.i32), ^bb2(%4 : !llvm.i32)
 
 // CHECK:^[[BB1]](%[[V9:.*]]: !llvm.i32):
-// CHECK:  %[[V10:.*]] = llvm.call @foo(%[[V9]]) : (!llvm.i32) -> !llvm<"{ i32, double, i32 }">
-// CHECK:  %[[V11:.*]] = llvm.extractvalue %[[V10]][0] : !llvm<"{ i32, double, i32 }">
-// CHECK:  %[[V12:.*]] = llvm.extractvalue %[[V10]][1] : !llvm<"{ i32, double, i32 }">
-// CHECK:  %[[V13:.*]] = llvm.extractvalue %[[V10]][2] : !llvm<"{ i32, double, i32 }">
-// CHECK:  %[[V14:.*]] = llvm.mlir.undef : !llvm<"{ i32, double, i32 }">
-// CHECK:  %[[V15:.*]] = llvm.insertvalue %[[V5]], %[[V14]][0] : !llvm<"{ i32, double, i32 }">
-// CHECK:  %[[V16:.*]] = llvm.insertvalue %[[V7]], %[[V15]][1] : !llvm<"{ i32, double, i32 }">
-// CHECK:  %[[V17:.*]] = llvm.insertvalue %[[V11]], %[[V16]][2] : !llvm<"{ i32, double, i32 }">
-// CHECK:  llvm.return %[[V17]] : !llvm<"{ i32, double, i32 }">
+// CHECK:  %[[V10:.*]] = llvm.call @foo(%[[V9]]) : (!llvm.i32) -> !llvm.struct<(i32, double, i32)>
+// CHECK:  %[[V11:.*]] = llvm.extractvalue %[[V10]][0] : !llvm.struct<(i32, double, i32)>
+// CHECK:  %[[V12:.*]] = llvm.extractvalue %[[V10]][1] : !llvm.struct<(i32, double, i32)>
+// CHECK:  %[[V13:.*]] = llvm.extractvalue %[[V10]][2] : !llvm.struct<(i32, double, i32)>
+// CHECK:  %[[V14:.*]] = llvm.mlir.undef : !llvm.struct<(i32, double, i32)>
+// CHECK:  %[[V15:.*]] = llvm.insertvalue %[[V5]], %[[V14]][0] : !llvm.struct<(i32, double, i32)>
+// CHECK:  %[[V16:.*]] = llvm.insertvalue %[[V7]], %[[V15]][1] : !llvm.struct<(i32, double, i32)>
+// CHECK:  %[[V17:.*]] = llvm.insertvalue %[[V11]], %[[V16]][2] : !llvm.struct<(i32, double, i32)>
+// CHECK:  llvm.return %[[V17]] : !llvm.struct<(i32, double, i32)>
 ^bb1(%9: !llvm.i32):
-  %10 = llvm.call @foo(%9) : (!llvm.i32) -> !llvm<"{ i32, double, i32 }">
-  %11 = llvm.extractvalue %10[0] : !llvm<"{ i32, double, i32 }">
-  %12 = llvm.extractvalue %10[1] : !llvm<"{ i32, double, i32 }">
-  %13 = llvm.extractvalue %10[2] : !llvm<"{ i32, double, i32 }">
-  %14 = llvm.mlir.undef : !llvm<"{ i32, double, i32 }">
-  %15 = llvm.insertvalue %5, %14[0] : !llvm<"{ i32, double, i32 }">
-  %16 = llvm.insertvalue %7, %15[1] : !llvm<"{ i32, double, i32 }">
-  %17 = llvm.insertvalue %11, %16[2] : !llvm<"{ i32, double, i32 }">
-  llvm.return %17 : !llvm<"{ i32, double, i32 }">
+  %10 = llvm.call @foo(%9) : (!llvm.i32) -> !llvm.struct<(i32, double, i32)>
+  %11 = llvm.extractvalue %10[0] : !llvm.struct<(i32, double, i32)>
+  %12 = llvm.extractvalue %10[1] : !llvm.struct<(i32, double, i32)>
+  %13 = llvm.extractvalue %10[2] : !llvm.struct<(i32, double, i32)>
+  %14 = llvm.mlir.undef : !llvm.struct<(i32, double, i32)>
+  %15 = llvm.insertvalue %5, %14[0] : !llvm.struct<(i32, double, i32)>
+  %16 = llvm.insertvalue %7, %15[1] : !llvm.struct<(i32, double, i32)>
+  %17 = llvm.insertvalue %11, %16[2] : !llvm.struct<(i32, double, i32)>
+  llvm.return %17 : !llvm.struct<(i32, double, i32)>
 
 // CHECK:^[[BB2]](%[[V18:.*]]: !llvm.i32):
-// CHECK:  %[[V19:.*]] = llvm.mlir.undef : !llvm<"{ i32, double, i32 }">
-// CHECK:  %[[V20:.*]] = llvm.insertvalue %[[V18]], %[[V19]][0] : !llvm<"{ i32, double, i32 }">
-// CHECK:  %[[V21:.*]] = llvm.insertvalue %[[V7]], %[[V20]][1] : !llvm<"{ i32, double, i32 }">
-// CHECK:  %[[V22:.*]] = llvm.insertvalue %[[V5]], %[[V21]][2] : !llvm<"{ i32, double, i32 }">
-// CHECK:  llvm.return %[[V22]] : !llvm<"{ i32, double, i32 }">
+// CHECK:  %[[V19:.*]] = llvm.mlir.undef : !llvm.struct<(i32, double, i32)>
+// CHECK:  %[[V20:.*]] = llvm.insertvalue %[[V18]], %[[V19]][0] : !llvm.struct<(i32, double, i32)>
+// CHECK:  %[[V21:.*]] = llvm.insertvalue %[[V7]], %[[V20]][1] : !llvm.struct<(i32, double, i32)>
+// CHECK:  %[[V22:.*]] = llvm.insertvalue %[[V5]], %[[V21]][2] : !llvm.struct<(i32, double, i32)>
+// CHECK:  llvm.return %[[V22]] : !llvm.struct<(i32, double, i32)>
 ^bb2(%18: !llvm.i32):
-  %19 = llvm.mlir.undef : !llvm<"{ i32, double, i32 }">
-  %20 = llvm.insertvalue %18, %19[0] : !llvm<"{ i32, double, i32 }">
-  %21 = llvm.insertvalue %7, %20[1] : !llvm<"{ i32, double, i32 }">
-  %22 = llvm.insertvalue %5, %21[2] : !llvm<"{ i32, double, i32 }">
-  llvm.return %22 : !llvm<"{ i32, double, i32 }">
+  %19 = llvm.mlir.undef : !llvm.struct<(i32, double, i32)>
+  %20 = llvm.insertvalue %18, %19[0] : !llvm.struct<(i32, double, i32)>
+  %21 = llvm.insertvalue %7, %20[1] : !llvm.struct<(i32, double, i32)>
+  %22 = llvm.insertvalue %5, %21[2] : !llvm.struct<(i32, double, i32)>
+  llvm.return %22 : !llvm.struct<(i32, double, i32)>
 }
 
 // CHECK-LABEL: @casts
-// CHECK-SAME: (%[[I32:.*]]: !llvm.i32, %[[I64:.*]]: !llvm.i64, %[[V4I32:.*]]: !llvm<"<4 x i32>">, %[[V4I64:.*]]: !llvm<"<4 x i64>">, %[[I32PTR:.*]]: !llvm<"i32*">)
-func @casts(%arg0: !llvm.i32, %arg1: !llvm.i64, %arg2: !llvm<"<4 x i32>">,
-            %arg3: !llvm<"<4 x i64>">, %arg4: !llvm<"i32*">) {
+// CHECK-SAME: (%[[I32:.*]]: !llvm.i32, %[[I64:.*]]: !llvm.i64, %[[V4I32:.*]]: !llvm.vec<4 x i32>, %[[V4I64:.*]]: !llvm.vec<4 x i64>, %[[I32PTR:.*]]: !llvm.ptr<i32>)
+func @casts(%arg0: !llvm.i32, %arg1: !llvm.i64, %arg2: !llvm.vec<4 x i32>,
+            %arg3: !llvm.vec<4 x i64>, %arg4: !llvm.ptr<i32>) {
 // CHECK:  = llvm.sext %[[I32]] : !llvm.i32 to !llvm.i56
   %0 = llvm.sext %arg0 : !llvm.i32 to !llvm.i56
 // CHECK:  = llvm.zext %[[I32]] : !llvm.i32 to !llvm.i64
   %1 = llvm.zext %arg0 : !llvm.i32 to !llvm.i64
 // CHECK:  = llvm.trunc %[[I64]] : !llvm.i64 to !llvm.i56
   %2 = llvm.trunc %arg1 : !llvm.i64 to !llvm.i56
-// CHECK:  = llvm.sext %[[V4I32]] : !llvm<"<4 x i32>"> to !llvm<"<4 x i56>">
-  %3 = llvm.sext %arg2 : !llvm<"<4 x i32>"> to !llvm<"<4 x i56>">
-// CHECK:  = llvm.zext %[[V4I32]] : !llvm<"<4 x i32>"> to !llvm<"<4 x i64>">
-  %4 = llvm.zext %arg2 : !llvm<"<4 x i32>"> to !llvm<"<4 x i64>">
-// CHECK:  = llvm.trunc %[[V4I64]] : !llvm<"<4 x i64>"> to !llvm<"<4 x i56>">
-  %5 = llvm.trunc %arg3 : !llvm<"<4 x i64>"> to !llvm<"<4 x i56>">
+// CHECK:  = llvm.sext %[[V4I32]] : !llvm.vec<4 x i32> to !llvm.vec<4 x i56>
+  %3 = llvm.sext %arg2 : !llvm.vec<4 x i32> to !llvm.vec<4 x i56>
+// CHECK:  = llvm.zext %[[V4I32]] : !llvm.vec<4 x i32> to !llvm.vec<4 x i64>
+  %4 = llvm.zext %arg2 : !llvm.vec<4 x i32> to !llvm.vec<4 x i64>
+// CHECK:  = llvm.trunc %[[V4I64]] : !llvm.vec<4 x i64> to !llvm.vec<4 x i56>
+  %5 = llvm.trunc %arg3 : !llvm.vec<4 x i64> to !llvm.vec<4 x i56>
 // CHECK:  = llvm.sitofp %[[I32]] : !llvm.i32 to !llvm.float
   %6 = llvm.sitofp %arg0 : !llvm.i32 to !llvm.float
 // CHECK: %[[FLOAT:.*]] = llvm.uitofp %[[I32]] : !llvm.i32 to !llvm.float
@@ -212,89 +212,89 @@ func @casts(%arg0: !llvm.i32, %arg1: !llvm.i64, %arg2: !llvm<"<4 x i32>">,
   %8 = llvm.fptosi %7 : !llvm.float to !llvm.i32
 // CHECK:  = llvm.fptoui %[[FLOAT]] : !llvm.float to !llvm.i32
   %9 = llvm.fptoui %7 : !llvm.float to !llvm.i32
-// CHECK:  = llvm.addrspacecast %[[I32PTR]] : !llvm<"i32*"> to !llvm<"i32 addrspace(2)*">
-  %10 = llvm.addrspacecast %arg4 : !llvm<"i32*"> to !llvm<"i32 addrspace(2)*">
+// CHECK:  = llvm.addrspacecast %[[I32PTR]] : !llvm.ptr<i32> to !llvm.ptr<i32, 2>
+  %10 = llvm.addrspacecast %arg4 : !llvm.ptr<i32> to !llvm.ptr<i32, 2>
   llvm.return
 }
 
 // CHECK-LABEL: @vect
-func @vect(%arg0: !llvm<"<4 x float>">, %arg1: !llvm.i32, %arg2: !llvm.float) {
-// CHECK:  = llvm.extractelement {{.*}} : !llvm<"<4 x float>">
-  %0 = llvm.extractelement %arg0[%arg1 : !llvm.i32] : !llvm<"<4 x float>">
-// CHECK:  = llvm.insertelement {{.*}} : !llvm<"<4 x float>">
-  %1 = llvm.insertelement %arg2, %arg0[%arg1 : !llvm.i32] : !llvm<"<4 x float>">
-// CHECK:  = llvm.shufflevector {{.*}} [0 : i32, 0 : i32, 0 : i32, 0 : i32, 7 : i32] : !llvm<"<4 x float>">, !llvm<"<4 x float>">
-  %2 = llvm.shufflevector %arg0, %arg0 [0 : i32, 0 : i32, 0 : i32, 0 : i32, 7 : i32] : !llvm<"<4 x float>">, !llvm<"<4 x float>">
-// CHECK:  = llvm.mlir.constant(dense<1.000000e+00> : vector<4xf32>) : !llvm<"<4 x float>">
-  %3 = llvm.mlir.constant(dense<1.0> : vector<4xf32>) : !llvm<"<4 x float>">
+func @vect(%arg0: !llvm.vec<4 x float>, %arg1: !llvm.i32, %arg2: !llvm.float) {
+// CHECK:  = llvm.extractelement {{.*}} : !llvm.vec<4 x float>
+  %0 = llvm.extractelement %arg0[%arg1 : !llvm.i32] : !llvm.vec<4 x float>
+// CHECK:  = llvm.insertelement {{.*}} : !llvm.vec<4 x float>
+  %1 = llvm.insertelement %arg2, %arg0[%arg1 : !llvm.i32] : !llvm.vec<4 x float>
+// CHECK:  = llvm.shufflevector {{.*}} [0 : i32, 0 : i32, 0 : i32, 0 : i32, 7 : i32] : !llvm.vec<4 x float>, !llvm.vec<4 x float>
+  %2 = llvm.shufflevector %arg0, %arg0 [0 : i32, 0 : i32, 0 : i32, 0 : i32, 7 : i32] : !llvm.vec<4 x float>, !llvm.vec<4 x float>
+// CHECK:  = llvm.mlir.constant(dense<1.000000e+00> : vector<4xf32>) : !llvm.vec<4 x float>
+  %3 = llvm.mlir.constant(dense<1.0> : vector<4xf32>) : !llvm.vec<4 x float>
   return
 }
 
 // CHECK-LABEL: @alloca
 func @alloca(%size : !llvm.i64) {
-  // CHECK: llvm.alloca %{{.*}} x !llvm.i32 : (!llvm.i64) -> !llvm<"i32*">
-  llvm.alloca %size x !llvm.i32 {alignment = 0} : (!llvm.i64) -> (!llvm<"i32*">)
-  // CHECK: llvm.alloca %{{.*}} x !llvm.i32 {alignment = 8 : i64} : (!llvm.i64) -> !llvm<"i32*">
-  llvm.alloca %size x !llvm.i32 {alignment = 8} : (!llvm.i64) -> (!llvm<"i32*">)
+  // CHECK: llvm.alloca %{{.*}} x !llvm.i32 : (!llvm.i64) -> !llvm.ptr<i32>
+  llvm.alloca %size x !llvm.i32 {alignment = 0} : (!llvm.i64) -> (!llvm.ptr<i32>)
+  // CHECK: llvm.alloca %{{.*}} x !llvm.i32 {alignment = 8 : i64} : (!llvm.i64) -> !llvm.ptr<i32>
+  llvm.alloca %size x !llvm.i32 {alignment = 8} : (!llvm.i64) -> (!llvm.ptr<i32>)
   llvm.return
 }
 
 // CHECK-LABEL: @null
 func @null() {
-  // CHECK: llvm.mlir.null : !llvm<"i8*">
-  %0 = llvm.mlir.null : !llvm<"i8*">
-  // CHECK: llvm.mlir.null : !llvm<"{ void (i32, void ()*)*, i64 }*">
-  %1 = llvm.mlir.null : !llvm<"{void(i32, void()*)*, i64}*">
+  // CHECK: llvm.mlir.null : !llvm.ptr<i8>
+  %0 = llvm.mlir.null : !llvm.ptr<i8>
+  // CHECK: llvm.mlir.null : !llvm.ptr<struct<(ptr<func<void (i32, ptr<func<void ()>>)>>, i64)>>
+  %1 = llvm.mlir.null : !llvm.ptr<struct<(ptr<func<void (i32, ptr<func<void ()>>)>>, i64)>>
   llvm.return
 }
 
 // CHECK-LABEL: @atomicrmw
-func @atomicrmw(%ptr : !llvm<"float*">, %val : !llvm.float) {
+func @atomicrmw(%ptr : !llvm.ptr<float>, %val : !llvm.float) {
   // CHECK: llvm.atomicrmw fadd %{{.*}}, %{{.*}} unordered : !llvm.float
   %0 = llvm.atomicrmw fadd %ptr, %val unordered : !llvm.float
   llvm.return
 }
 
 // CHECK-LABEL: @cmpxchg
-func @cmpxchg(%ptr : !llvm<"float*">, %cmp : !llvm.float, %new : !llvm.float) {
+func @cmpxchg(%ptr : !llvm.ptr<float>, %cmp : !llvm.float, %new : !llvm.float) {
   // CHECK: llvm.cmpxchg %{{.*}}, %{{.*}}, %{{.*}} acq_rel monotonic : !llvm.float
   %0 = llvm.cmpxchg %ptr, %cmp, %new acq_rel monotonic : !llvm.float
   llvm.return
 }
 
-llvm.mlir.global external constant @_ZTIi() : !llvm<"i8*">
-llvm.func @bar(!llvm<"i8*">, !llvm<"i8*">, !llvm<"i8*">)
+llvm.mlir.global external constant @_ZTIi() : !llvm.ptr<i8>
+llvm.func @bar(!llvm.ptr<i8>, !llvm.ptr<i8>, !llvm.ptr<i8>)
 llvm.func @__gxx_personality_v0(...) -> !llvm.i32
 
 // CHECK-LABEL: @invokeLandingpad
 llvm.func @invokeLandingpad() -> !llvm.i32 attributes { personality = @__gxx_personality_v0 } {
 // CHECK: %[[a0:.*]] = llvm.mlir.constant(0 : i32) : !llvm.i32
 // CHECK: %{{.*}} = llvm.mlir.constant(3 : i32) : !llvm.i32
-// CHECK: %[[a2:.*]] = llvm.mlir.constant("\01") : !llvm<"[1 x i8]">
-// CHECK: %[[a3:.*]] = llvm.mlir.null : !llvm<"i8**">
-// CHECK: %[[a4:.*]] = llvm.mlir.null : !llvm<"i8*">
-// CHECK: %[[a5:.*]] = llvm.mlir.addressof @_ZTIi : !llvm<"i8**">
-// CHECK: %[[a6:.*]] = llvm.bitcast %[[a5]] : !llvm<"i8**"> to !llvm<"i8*">
+// CHECK: %[[a2:.*]] = llvm.mlir.constant("\01") : !llvm.array<1 x i8>
+// CHECK: %[[a3:.*]] = llvm.mlir.null : !llvm.ptr<ptr<i8>>
+// CHECK: %[[a4:.*]] = llvm.mlir.null : !llvm.ptr<i8>
+// CHECK: %[[a5:.*]] = llvm.mlir.addressof @_ZTIi : !llvm.ptr<ptr<i8>>
+// CHECK: %[[a6:.*]] = llvm.bitcast %[[a5]] : !llvm.ptr<ptr<i8>> to !llvm.ptr<i8>
 // CHECK: %[[a7:.*]] = llvm.mlir.constant(1 : i32) : !llvm.i32
-// CHECK: %[[a8:.*]] = llvm.alloca %[[a7]] x !llvm.i8 : (!llvm.i32) -> !llvm<"i8*">
-// CHECK: %{{.*}} = llvm.invoke @foo(%[[a7]]) to ^[[BB2:.*]] unwind ^[[BB1:.*]] : (!llvm.i32) -> !llvm<"{ i32, double, i32 }">
+// CHECK: %[[a8:.*]] = llvm.alloca %[[a7]] x !llvm.i8 : (!llvm.i32) -> !llvm.ptr<i8>
+// CHECK: %{{.*}} = llvm.invoke @foo(%[[a7]]) to ^[[BB2:.*]] unwind ^[[BB1:.*]] : (!llvm.i32) -> !llvm.struct<(i32, double, i32)>
   %0 = llvm.mlir.constant(0 : i32) : !llvm.i32
   %1 = llvm.mlir.constant(3 : i32) : !llvm.i32
-  %2 = llvm.mlir.constant("\01") : !llvm<"[1 x i8]">
-  %3 = llvm.mlir.null : !llvm<"i8**">
-  %4 = llvm.mlir.null : !llvm<"i8*">
-  %5 = llvm.mlir.addressof @_ZTIi : !llvm<"i8**">
-  %6 = llvm.bitcast %5 : !llvm<"i8**"> to !llvm<"i8*">
+  %2 = llvm.mlir.constant("\01") : !llvm.array<1 x i8>
+  %3 = llvm.mlir.null : !llvm.ptr<ptr<i8>>
+  %4 = llvm.mlir.null : !llvm.ptr<i8>
+  %5 = llvm.mlir.addressof @_ZTIi : !llvm.ptr<ptr<i8>>
+  %6 = llvm.bitcast %5 : !llvm.ptr<ptr<i8>> to !llvm.ptr<i8>
   %7 = llvm.mlir.constant(1 : i32) : !llvm.i32
-  %8 = llvm.alloca %7 x !llvm.i8 : (!llvm.i32) -> !llvm<"i8*">
-  %9 = llvm.invoke @foo(%7) to ^bb2 unwind ^bb1 : (!llvm.i32) -> !llvm<"{ i32, double, i32 }">
+  %8 = llvm.alloca %7 x !llvm.i8 : (!llvm.i32) -> !llvm.ptr<i8>
+  %9 = llvm.invoke @foo(%7) to ^bb2 unwind ^bb1 : (!llvm.i32) -> !llvm.struct<(i32, double, i32)>
 
 // CHECK: ^[[BB1]]:
-// CHECK:   %[[lp:.*]] = llvm.landingpad cleanup (catch %[[a3]] : !llvm<"i8**">) (catch %[[a6]] : !llvm<"i8*">) (filter %[[a2]] : !llvm<"[1 x i8]">) : !llvm<"{ i8*, i32 }">
-// CHECK:   llvm.resume %[[lp]] : !llvm<"{ i8*, i32 }">
+// CHECK:   %[[lp:.*]] = llvm.landingpad cleanup (catch %[[a3]] : !llvm.ptr<ptr<i8>>) (catch %[[a6]] : !llvm.ptr<i8>) (filter %[[a2]] : !llvm.array<1 x i8>) : !llvm.struct<(ptr<i8>, i32)>
+// CHECK:   llvm.resume %[[lp]] : !llvm.struct<(ptr<i8>, i32)>
 ^bb1:
-  %10 = llvm.landingpad cleanup (catch %3 : !llvm<"i8**">) (catch %6 : !llvm<"i8*">) (filter %2 : !llvm<"[1 x i8]">) : !llvm<"{ i8*, i32 }">
-  llvm.resume %10 : !llvm<"{ i8*, i32 }">
+  %10 = llvm.landingpad cleanup (catch %3 : !llvm.ptr<ptr<i8>>) (catch %6 : !llvm.ptr<i8>) (filter %2 : !llvm.array<1 x i8>) : !llvm.struct<(ptr<i8>, i32)>
+  llvm.resume %10 : !llvm.struct<(ptr<i8>, i32)>
 
 // CHECK: ^[[BB2]]:
 // CHECK:   llvm.return %[[a7]] : !llvm.i32
@@ -302,9 +302,9 @@ llvm.func @invokeLandingpad() -> !llvm.i32 attributes { personality = @__gxx_per
   llvm.return %7 : !llvm.i32
 
 // CHECK: ^[[BB3:.*]]:
-// CHECK:   llvm.invoke @bar(%[[a8]], %[[a6]], %[[a4]]) to ^[[BB2]] unwind ^[[BB1]] : (!llvm<"i8*">, !llvm<"i8*">, !llvm<"i8*">) -> ()
+// CHECK:   llvm.invoke @bar(%[[a8]], %[[a6]], %[[a4]]) to ^[[BB2]] unwind ^[[BB1]] : (!llvm.ptr<i8>, !llvm.ptr<i8>, !llvm.ptr<i8>) -> ()
 ^bb3:
-  llvm.invoke @bar(%8, %6, %4) to ^bb2 unwind ^bb1 : (!llvm<"i8*">, !llvm<"i8*">, !llvm<"i8*">) -> ()
+  llvm.invoke @bar(%8, %6, %4) to ^bb2 unwind ^bb1 : (!llvm.ptr<i8>, !llvm.ptr<i8>, !llvm.ptr<i8>) -> ()
 
 // CHECK: ^[[BB4:.*]]:
 // CHECK:   llvm.return %[[a0]] : !llvm.i32
diff --git a/mlir/test/Dialect/LLVMIR/types.mlir b/mlir/test/Dialect/LLVMIR/types.mlir
index 7ce606fe8c6a8..bd24c68b8883a 100644
--- a/mlir/test/Dialect/LLVMIR/types.mlir
+++ b/mlir/test/Dialect/LLVMIR/types.mlir
@@ -2,183 +2,183 @@
 
 // CHECK-LABEL: @primitive
 func @primitive() {
-  // CHECK: !llvm2.void
-  "some.op"() : () -> !llvm2.void
-  // CHECK: !llvm2.half
-  "some.op"() : () -> !llvm2.half
-  // CHECK: !llvm2.bfloat
-  "some.op"() : () -> !llvm2.bfloat
-  // CHECK: !llvm2.float
-  "some.op"() : () -> !llvm2.float
-  // CHECK: !llvm2.double
-  "some.op"() : () -> !llvm2.double
-  // CHECK: !llvm2.fp128
-  "some.op"() : () -> !llvm2.fp128
-  // CHECK: !llvm2.x86_fp80
-  "some.op"() : () -> !llvm2.x86_fp80
-  // CHECK: !llvm2.ppc_fp128
-  "some.op"() : () -> !llvm2.ppc_fp128
-  // CHECK: !llvm2.x86_mmx
-  "some.op"() : () -> !llvm2.x86_mmx
-  // CHECK: !llvm2.token
-  "some.op"() : () -> !llvm2.token
-  // CHECK: !llvm2.label
-  "some.op"() : () -> !llvm2.label
-  // CHECK: !llvm2.metadata
-  "some.op"() : () -> !llvm2.metadata
+  // CHECK: !llvm.void
+  "some.op"() : () -> !llvm.void
+  // CHECK: !llvm.half
+  "some.op"() : () -> !llvm.half
+  // CHECK: !llvm.bfloat
+  "some.op"() : () -> !llvm.bfloat
+  // CHECK: !llvm.float
+  "some.op"() : () -> !llvm.float
+  // CHECK: !llvm.double
+  "some.op"() : () -> !llvm.double
+  // CHECK: !llvm.fp128
+  "some.op"() : () -> !llvm.fp128
+  // CHECK: !llvm.x86_fp80
+  "some.op"() : () -> !llvm.x86_fp80
+  // CHECK: !llvm.ppc_fp128
+  "some.op"() : () -> !llvm.ppc_fp128
+  // CHECK: !llvm.x86_mmx
+  "some.op"() : () -> !llvm.x86_mmx
+  // CHECK: !llvm.token
+  "some.op"() : () -> !llvm.token
+  // CHECK: !llvm.label
+  "some.op"() : () -> !llvm.label
+  // CHECK: !llvm.metadata
+  "some.op"() : () -> !llvm.metadata
   return
 }
 
 // CHECK-LABEL: @func
 func @func() {
-  // CHECK: !llvm2.func<void ()>
-  "some.op"() : () -> !llvm2.func<void ()>
-  // CHECK: !llvm2.func<void (i32)>
-  "some.op"() : () -> !llvm2.func<void (i32)>
-  // CHECK: !llvm2.func<i32 ()>
-  "some.op"() : () -> !llvm2.func<i32 ()>
-  // CHECK: !llvm2.func<i32 (half, bfloat, float, double)>
-  "some.op"() : () -> !llvm2.func<i32 (half, bfloat, float, double)>
-  // CHECK: !llvm2.func<i32 (i32, i32)>
-  "some.op"() : () -> !llvm2.func<i32 (i32, i32)>
-  // CHECK: !llvm2.func<void (...)>
-  "some.op"() : () -> !llvm2.func<void (...)>
-  // CHECK: !llvm2.func<void (i32, i32, ...)>
-  "some.op"() : () -> !llvm2.func<void (i32, i32, ...)>
+  // CHECK: !llvm.func<void ()>
+  "some.op"() : () -> !llvm.func<void ()>
+  // CHECK: !llvm.func<void (i32)>
+  "some.op"() : () -> !llvm.func<void (i32)>
+  // CHECK: !llvm.func<i32 ()>
+  "some.op"() : () -> !llvm.func<i32 ()>
+  // CHECK: !llvm.func<i32 (half, bfloat, float, double)>
+  "some.op"() : () -> !llvm.func<i32 (half, bfloat, float, double)>
+  // CHECK: !llvm.func<i32 (i32, i32)>
+  "some.op"() : () -> !llvm.func<i32 (i32, i32)>
+  // CHECK: !llvm.func<void (...)>
+  "some.op"() : () -> !llvm.func<void (...)>
+  // CHECK: !llvm.func<void (i32, i32, ...)>
+  "some.op"() : () -> !llvm.func<void (i32, i32, ...)>
   return
 }
 
 // CHECK-LABEL: @integer
 func @integer() {
-  // CHECK: !llvm2.i1
-  "some.op"() : () -> !llvm2.i1
-  // CHECK: !llvm2.i8
-  "some.op"() : () -> !llvm2.i8
-  // CHECK: !llvm2.i16
-  "some.op"() : () -> !llvm2.i16
-  // CHECK: !llvm2.i32
-  "some.op"() : () -> !llvm2.i32
-  // CHECK: !llvm2.i64
-  "some.op"() : () -> !llvm2.i64
-  // CHECK: !llvm2.i57
-  "some.op"() : () -> !llvm2.i57
-  // CHECK: !llvm2.i129
-  "some.op"() : () -> !llvm2.i129
+  // CHECK: !llvm.i1
+  "some.op"() : () -> !llvm.i1
+  // CHECK: !llvm.i8
+  "some.op"() : () -> !llvm.i8
+  // CHECK: !llvm.i16
+  "some.op"() : () -> !llvm.i16
+  // CHECK: !llvm.i32
+  "some.op"() : () -> !llvm.i32
+  // CHECK: !llvm.i64
+  "some.op"() : () -> !llvm.i64
+  // CHECK: !llvm.i57
+  "some.op"() : () -> !llvm.i57
+  // CHECK: !llvm.i129
+  "some.op"() : () -> !llvm.i129
   return
 }
 
 // CHECK-LABEL: @ptr
 func @ptr() {
-  // CHECK: !llvm2.ptr<i8>
-  "some.op"() : () -> !llvm2.ptr<i8>
-  // CHECK: !llvm2.ptr<float>
-  "some.op"() : () -> !llvm2.ptr<float>
-  // CHECK: !llvm2.ptr<ptr<i8>>
-  "some.op"() : () -> !llvm2.ptr<ptr<i8>>
-  // CHECK: !llvm2.ptr<ptr<ptr<ptr<ptr<i8>>>>>
-  "some.op"() : () -> !llvm2.ptr<ptr<ptr<ptr<ptr<i8>>>>>
-  // CHECK: !llvm2.ptr<i8>
-  "some.op"() : () -> !llvm2.ptr<i8, 0>
-  // CHECK: !llvm2.ptr<i8, 1>
-  "some.op"() : () -> !llvm2.ptr<i8, 1>
-  // CHECK: !llvm2.ptr<i8, 42>
-  "some.op"() : () -> !llvm2.ptr<i8, 42>
-  // CHECK: !llvm2.ptr<ptr<i8, 42>, 9>
-  "some.op"() : () -> !llvm2.ptr<ptr<i8, 42>, 9>
+  // CHECK: !llvm.ptr<i8>
+  "some.op"() : () -> !llvm.ptr<i8>
+  // CHECK: !llvm.ptr<float>
+  "some.op"() : () -> !llvm.ptr<float>
+  // CHECK: !llvm.ptr<ptr<i8>>
+  "some.op"() : () -> !llvm.ptr<ptr<i8>>
+  // CHECK: !llvm.ptr<ptr<ptr<ptr<ptr<i8>>>>>
+  "some.op"() : () -> !llvm.ptr<ptr<ptr<ptr<ptr<i8>>>>>
+  // CHECK: !llvm.ptr<i8>
+  "some.op"() : () -> !llvm.ptr<i8, 0>
+  // CHECK: !llvm.ptr<i8, 1>
+  "some.op"() : () -> !llvm.ptr<i8, 1>
+  // CHECK: !llvm.ptr<i8, 42>
+  "some.op"() : () -> !llvm.ptr<i8, 42>
+  // CHECK: !llvm.ptr<ptr<i8, 42>, 9>
+  "some.op"() : () -> !llvm.ptr<ptr<i8, 42>, 9>
   return
 }
 
 // CHECK-LABEL: @vec
 func @vec() {
-  // CHECK: !llvm2.vec<4 x i32>
-  "some.op"() : () -> !llvm2.vec<4 x i32>
-  // CHECK: !llvm2.vec<4 x float>
-  "some.op"() : () -> !llvm2.vec<4 x float>
-  // CHECK: !llvm2.vec<? x 4 x i32>
-  "some.op"() : () -> !llvm2.vec<? x 4 x i32>
-  // CHECK: !llvm2.vec<? x 8 x half>
-  "some.op"() : () -> !llvm2.vec<? x 8 x half>
-  // CHECK: !llvm2.vec<4 x ptr<i8>>
-  "some.op"() : () -> !llvm2.vec<4 x ptr<i8>>
+  // CHECK: !llvm.vec<4 x i32>
+  "some.op"() : () -> !llvm.vec<4 x i32>
+  // CHECK: !llvm.vec<4 x float>
+  "some.op"() : () -> !llvm.vec<4 x float>
+  // CHECK: !llvm.vec<? x 4 x i32>
+  "some.op"() : () -> !llvm.vec<? x 4 x i32>
+  // CHECK: !llvm.vec<? x 8 x half>
+  "some.op"() : () -> !llvm.vec<? x 8 x half>
+  // CHECK: !llvm.vec<4 x ptr<i8>>
+  "some.op"() : () -> !llvm.vec<4 x ptr<i8>>
   return
 }
 
 // CHECK-LABEL: @array
 func @array() {
-  // CHECK: !llvm2.array<10 x i32>
-  "some.op"() : () -> !llvm2.array<10 x i32>
-  // CHECK: !llvm2.array<8 x float>
-  "some.op"() : () -> !llvm2.array<8 x float>
-  // CHECK: !llvm2.array<10 x ptr<i32, 4>>
-  "some.op"() : () -> !llvm2.array<10 x ptr<i32, 4>>
-  // CHECK: !llvm2.array<10 x array<4 x float>>
-  "some.op"() : () -> !llvm2.array<10 x array<4 x float>>
+  // CHECK: !llvm.array<10 x i32>
+  "some.op"() : () -> !llvm.array<10 x i32>
+  // CHECK: !llvm.array<8 x float>
+  "some.op"() : () -> !llvm.array<8 x float>
+  // CHECK: !llvm.array<10 x ptr<i32, 4>>
+  "some.op"() : () -> !llvm.array<10 x ptr<i32, 4>>
+  // CHECK: !llvm.array<10 x array<4 x float>>
+  "some.op"() : () -> !llvm.array<10 x array<4 x float>>
   return
 }
 
 // CHECK-LABEL: @literal_struct
 func @literal_struct() {
-  // CHECK: !llvm2.struct<()>
-  "some.op"() : () -> !llvm2.struct<()>
-  // CHECK: !llvm2.struct<(i32)>
-  "some.op"() : () -> !llvm2.struct<(i32)>
-  // CHECK: !llvm2.struct<(float, i32)>
-  "some.op"() : () -> !llvm2.struct<(float, i32)>
-  // CHECK: !llvm2.struct<(struct<(i32)>)>
-  "some.op"() : () -> !llvm2.struct<(struct<(i32)>)>
-  // CHECK: !llvm2.struct<(i32, struct<(i32)>, float)>
-  "some.op"() : () -> !llvm2.struct<(i32, struct<(i32)>, float)>
+  // CHECK: !llvm.struct<()>
+  "some.op"() : () -> !llvm.struct<()>
+  // CHECK: !llvm.struct<(i32)>
+  "some.op"() : () -> !llvm.struct<(i32)>
+  // CHECK: !llvm.struct<(float, i32)>
+  "some.op"() : () -> !llvm.struct<(float, i32)>
+  // CHECK: !llvm.struct<(struct<(i32)>)>
+  "some.op"() : () -> !llvm.struct<(struct<(i32)>)>
+  // CHECK: !llvm.struct<(i32, struct<(i32)>, float)>
+  "some.op"() : () -> !llvm.struct<(i32, struct<(i32)>, float)>
 
-  // CHECK: !llvm2.struct<packed ()>
-  "some.op"() : () -> !llvm2.struct<packed ()>
-  // CHECK: !llvm2.struct<packed (i32)>
-  "some.op"() : () -> !llvm2.struct<packed (i32)>
-  // CHECK: !llvm2.struct<packed (float, i32)>
-  "some.op"() : () -> !llvm2.struct<packed (float, i32)>
-  // CHECK: !llvm2.struct<packed (float, i32)>
-  "some.op"() : () -> !llvm2.struct<packed (float, i32)>
-  // CHECK: !llvm2.struct<packed (struct<(i32)>)>
-  "some.op"() : () -> !llvm2.struct<packed (struct<(i32)>)>
-  // CHECK: !llvm2.struct<packed (i32, struct<(i32, i1)>, float)>
-  "some.op"() : () -> !llvm2.struct<packed (i32, struct<(i32, i1)>, float)>
+  // CHECK: !llvm.struct<packed ()>
+  "some.op"() : () -> !llvm.struct<packed ()>
+  // CHECK: !llvm.struct<packed (i32)>
+  "some.op"() : () -> !llvm.struct<packed (i32)>
+  // CHECK: !llvm.struct<packed (float, i32)>
+  "some.op"() : () -> !llvm.struct<packed (float, i32)>
+  // CHECK: !llvm.struct<packed (float, i32)>
+  "some.op"() : () -> !llvm.struct<packed (float, i32)>
+  // CHECK: !llvm.struct<packed (struct<(i32)>)>
+  "some.op"() : () -> !llvm.struct<packed (struct<(i32)>)>
+  // CHECK: !llvm.struct<packed (i32, struct<(i32, i1)>, float)>
+  "some.op"() : () -> !llvm.struct<packed (i32, struct<(i32, i1)>, float)>
 
-  // CHECK: !llvm2.struct<(struct<packed (i32)>)>
-  "some.op"() : () -> !llvm2.struct<(struct<packed (i32)>)>
-  // CHECK: !llvm2.struct<packed (struct<(i32)>)>
-  "some.op"() : () -> !llvm2.struct<packed (struct<(i32)>)>
+  // CHECK: !llvm.struct<(struct<packed (i32)>)>
+  "some.op"() : () -> !llvm.struct<(struct<packed (i32)>)>
+  // CHECK: !llvm.struct<packed (struct<(i32)>)>
+  "some.op"() : () -> !llvm.struct<packed (struct<(i32)>)>
   return
 }
 
 // CHECK-LABEL: @identified_struct
 func @identified_struct() {
-  // CHECK: !llvm2.struct<"empty", ()>
-  "some.op"() : () -> !llvm2.struct<"empty", ()>
-  // CHECK: !llvm2.struct<"opaque", opaque>
-  "some.op"() : () -> !llvm2.struct<"opaque", opaque>
-  // CHECK: !llvm2.struct<"long", (i32, struct<(i32, i1)>, float, ptr<func<void ()>>)>
-  "some.op"() : () -> !llvm2.struct<"long", (i32, struct<(i32, i1)>, float, ptr<func<void ()>>)>
-  // CHECK: !llvm2.struct<"self-recursive", (ptr<struct<"self-recursive">>)>
-  "some.op"() : () -> !llvm2.struct<"self-recursive", (ptr<struct<"self-recursive">>)>
-  // CHECK: !llvm2.struct<"unpacked", (i32)>
-  "some.op"() : () -> !llvm2.struct<"unpacked", (i32)>
-  // CHECK: !llvm2.struct<"packed", packed (i32)>
-  "some.op"() : () -> !llvm2.struct<"packed", packed (i32)>
-  // CHECK: !llvm2.struct<"name with spaces and !^$@$#", packed (i32)>
-  "some.op"() : () -> !llvm2.struct<"name with spaces and !^$@$#", packed (i32)>
+  // CHECK: !llvm.struct<"empty", ()>
+  "some.op"() : () -> !llvm.struct<"empty", ()>
+  // CHECK: !llvm.struct<"opaque", opaque>
+  "some.op"() : () -> !llvm.struct<"opaque", opaque>
+  // CHECK: !llvm.struct<"long", (i32, struct<(i32, i1)>, float, ptr<func<void ()>>)>
+  "some.op"() : () -> !llvm.struct<"long", (i32, struct<(i32, i1)>, float, ptr<func<void ()>>)>
+  // CHECK: !llvm.struct<"self-recursive", (ptr<struct<"self-recursive">>)>
+  "some.op"() : () -> !llvm.struct<"self-recursive", (ptr<struct<"self-recursive">>)>
+  // CHECK: !llvm.struct<"unpacked", (i32)>
+  "some.op"() : () -> !llvm.struct<"unpacked", (i32)>
+  // CHECK: !llvm.struct<"packed", packed (i32)>
+  "some.op"() : () -> !llvm.struct<"packed", packed (i32)>
+  // CHECK: !llvm.struct<"name with spaces and !^$@$#", packed (i32)>
+  "some.op"() : () -> !llvm.struct<"name with spaces and !^$@$#", packed (i32)>
 
-  // CHECK: !llvm2.struct<"mutually-a", (ptr<struct<"mutually-b", (ptr<struct<"mutually-a">, 3>)>>)>
-  "some.op"() : () -> !llvm2.struct<"mutually-a", (ptr<struct<"mutually-b", (ptr<struct<"mutually-a">, 3>)>>)>
-  // CHECK: !llvm2.struct<"mutually-b", (ptr<struct<"mutually-a", (ptr<struct<"mutually-b">>)>, 3>)>
-  "some.op"() : () -> !llvm2.struct<"mutually-b", (ptr<struct<"mutually-a", (ptr<struct<"mutually-b">>)>, 3>)>
-  // CHECK: !llvm2.struct<"referring-another", (ptr<struct<"unpacked", (i32)>>)>
-  "some.op"() : () -> !llvm2.struct<"referring-another", (ptr<struct<"unpacked", (i32)>>)>
+  // CHECK: !llvm.struct<"mutually-a", (ptr<struct<"mutually-b", (ptr<struct<"mutually-a">, 3>)>>)>
+  "some.op"() : () -> !llvm.struct<"mutually-a", (ptr<struct<"mutually-b", (ptr<struct<"mutually-a">, 3>)>>)>
+  // CHECK: !llvm.struct<"mutually-b", (ptr<struct<"mutually-a", (ptr<struct<"mutually-b">>)>, 3>)>
+  "some.op"() : () -> !llvm.struct<"mutually-b", (ptr<struct<"mutually-a", (ptr<struct<"mutually-b">>)>, 3>)>
+  // CHECK: !llvm.struct<"referring-another", (ptr<struct<"unpacked", (i32)>>)>
+  "some.op"() : () -> !llvm.struct<"referring-another", (ptr<struct<"unpacked", (i32)>>)>
 
-  // CHECK: !llvm2.struct<"struct-of-arrays", (array<10 x i32>)>
-  "some.op"() : () -> !llvm2.struct<"struct-of-arrays", (array<10 x i32>)>
-  // CHECK: !llvm2.array<10 x struct<"array-of-structs", (i32)>>
-  "some.op"() : () -> !llvm2.array<10 x struct<"array-of-structs", (i32)>>
-  // CHECK: !llvm2.ptr<struct<"ptr-to-struct", (i8)>>
-  "some.op"() : () -> !llvm2.ptr<struct<"ptr-to-struct", (i8)>>
+  // CHECK: !llvm.struct<"struct-of-arrays", (array<10 x i32>)>
+  "some.op"() : () -> !llvm.struct<"struct-of-arrays", (array<10 x i32>)>
+  // CHECK: !llvm.array<10 x struct<"array-of-structs", (i32)>>
+  "some.op"() : () -> !llvm.array<10 x struct<"array-of-structs", (i32)>>
+  // CHECK: !llvm.ptr<struct<"ptr-to-struct", (i8)>>
+  "some.op"() : () -> !llvm.ptr<struct<"ptr-to-struct", (i8)>>
   return
 }
 
diff --git a/mlir/test/Dialect/Linalg/llvm.mlir b/mlir/test/Dialect/Linalg/llvm.mlir
index 9b052fd2fab47..02693e5d1be46 100644
--- a/mlir/test/Dialect/Linalg/llvm.mlir
+++ b/mlir/test/Dialect/Linalg/llvm.mlir
@@ -9,10 +9,10 @@ func @range(%arg0: index) {
 // CHECK-LABEL: func @range(%{{.*}}: !llvm.i64) {
 //       CHECK:   llvm.mlir.constant(0 : index) : !llvm.i64
 //  CHECK-NEXT:   llvm.mlir.constant(1 : index) : !llvm.i64
-//  CHECK-NEXT:   llvm.mlir.undef : !llvm<"{ i64, i64, i64 }">
-//  CHECK-NEXT:   llvm.insertvalue %{{.*}}, %{{.*}}[0] : !llvm<"{ i64, i64, i64 }">
-//  CHECK-NEXT:   llvm.insertvalue %{{.*}}, %{{.*}}[1] : !llvm<"{ i64, i64, i64 }">
-//  CHECK-NEXT:   llvm.insertvalue %{{.*}}, %{{.*}}[2] : !llvm<"{ i64, i64, i64 }">
+//  CHECK-NEXT:   llvm.mlir.undef : !llvm.struct<(i64, i64, i64)>
+//  CHECK-NEXT:   llvm.insertvalue %{{.*}}, %{{.*}}[0] : !llvm.struct<(i64, i64, i64)>
+//  CHECK-NEXT:   llvm.insertvalue %{{.*}}, %{{.*}}[1] : !llvm.struct<(i64, i64, i64)>
+//  CHECK-NEXT:   llvm.insertvalue %{{.*}}, %{{.*}}[2] : !llvm.struct<(i64, i64, i64)>
 
 func @slice(%arg0: memref<?xf32, offset: ?, strides: [1]>, %arg1: !linalg.range) {
   %1 = linalg.slice %arg0[%arg1] : memref<?xf32, offset: ?, strides: [1]>, !linalg.range, memref<?xf32, offset: ?, strides: [1]>
@@ -20,20 +20,20 @@ func @slice(%arg0: memref<?xf32, offset: ?, strides: [1]>, %arg1: !linalg.range)
 }
 // CHECK-LABEL: func @slice
 //   insert data ptr for slice op
-//       CHECK:   llvm.extractvalue %{{.*}}[4, 0] : !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }">
-//  CHECK-NEXT:   llvm.extractvalue %{{.*}}[2] : !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }">
-//  CHECK-NEXT:   llvm.extractvalue %{{.*}}[0] : !llvm<"{ i64, i64, i64 }">
+//       CHECK:   llvm.extractvalue %{{.*}}[4, 0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>
+//  CHECK-NEXT:   llvm.extractvalue %{{.*}}[2] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>
+//  CHECK-NEXT:   llvm.extractvalue %{{.*}}[0] : !llvm.struct<(i64, i64, i64)>
 //  CHECK-NEXT:   llvm.mul %{{.*}}, %{{.*}} : !llvm.i64
 //  CHECK-NEXT:   llvm.add %{{.*}}, %{{.*}} : !llvm.i64
 //    insert offset
-//       CHECK:   llvm.insertvalue %{{.*}}, %{{.*}}[1] : !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }">
-//  CHECK-NEXT:   llvm.insertvalue %{{.*}}, %{{.*}}[2] : !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }">
+//       CHECK:   llvm.insertvalue %{{.*}}, %{{.*}}[1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>
+//  CHECK-NEXT:   llvm.insertvalue %{{.*}}, %{{.*}}[2] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>
 //  CHECK-NEXT:   llvm.mlir.constant(0 : index)
-//  CHECK-NEXT:   llvm.extractvalue %{{.*}}[0] : !llvm<"{ i64, i64, i64 }">
-//  CHECK-NEXT:   llvm.extractvalue %{{.*}}[1] : !llvm<"{ i64, i64, i64 }">
-//  CHECK-NEXT:   llvm.extractvalue %{{.*}}[2] : !llvm<"{ i64, i64, i64 }">
+//  CHECK-NEXT:   llvm.extractvalue %{{.*}}[0] : !llvm.struct<(i64, i64, i64)>
+//  CHECK-NEXT:   llvm.extractvalue %{{.*}}[1] : !llvm.struct<(i64, i64, i64)>
+//  CHECK-NEXT:   llvm.extractvalue %{{.*}}[2] : !llvm.struct<(i64, i64, i64)>
 //    get size[0] from parent view
-//  CHECK-NEXT:   llvm.extractvalue %{{.*}}[3, 0] : !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }">
+//  CHECK-NEXT:   llvm.extractvalue %{{.*}}[3, 0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>
 //  CHECK-NEXT:   llvm.icmp "slt" %{{.*}}, %{{.*}} : !llvm.i64
 //  CHECK-NEXT:   llvm.select %{{.*}}, %{{.*}}, %{{.*}} : !llvm.i1, !llvm.i64
 //    compute size[0] bounded by parent view's size[0]
@@ -44,8 +44,8 @@ func @slice(%arg0: memref<?xf32, offset: ?, strides: [1]>, %arg1: !linalg.range)
 //    compute stride[0] using bounded size
 //  CHECK-NEXT:   llvm.mul %{{.*}}, %{{.*}} : !llvm.i64
 //    insert size and stride
-//  CHECK-NEXT:   llvm.insertvalue %{{.*}}, %{{.*}}[3, 0] : !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }">
-//  CHECK-NEXT:   llvm.insertvalue %{{.*}}, %{{.*}}[4, 0] : !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }">
+//  CHECK-NEXT:   llvm.insertvalue %{{.*}}, %{{.*}}[3, 0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>
+//  CHECK-NEXT:   llvm.insertvalue %{{.*}}, %{{.*}}[4, 0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>
 
 func @slice_with_range_and_index(%arg0: memref<?x?xf64, offset: ?, strides: [?, 1]>) {
   %c0 = constant 0 : index
@@ -58,32 +58,32 @@ func @slice_with_range_and_index(%arg0: memref<?x?xf64, offset: ?, strides: [?,
 }
 // CHECK-LABEL: func @slice_with_range_and_index
 // loop-body.
-//       CHECK:   llvm.mlir.undef : !llvm<"{ double*, double*, i64, [1 x i64], [1 x i64] }">
-//       CHECK:   llvm.extractvalue %{{.*}}[4, 0] : !llvm<"{ double*, double*, i64, [2 x i64], [2 x i64] }">
-//       CHECK:   llvm.extractvalue %{{.*}}[4, 1] : !llvm<"{ double*, double*, i64, [2 x i64], [2 x i64] }">
-//       CHECK:   llvm.extractvalue %{{.*}}[2] : !llvm<"{ double*, double*, i64, [2 x i64], [2 x i64] }">
-//       CHECK:   llvm.insertvalue %{{.*}}, %{{.*}}[0] : !llvm<"{ double*, double*, i64, [1 x i64], [1 x i64] }">
-//       CHECK:   llvm.insertvalue %{{.*}}[2] : !llvm<"{ double*, double*, i64, [1 x i64], [1 x i64] }">
-//       CHECK:   llvm.extractvalue %{{.*}}[0] : !llvm<"{ i64, i64, i64 }">
-//       CHECK:   llvm.extractvalue %{{.*}}[1] : !llvm<"{ i64, i64, i64 }">
-//       CHECK:   llvm.insertvalue %{{.*}}[3, 0] : !llvm<"{ double*, double*, i64, [1 x i64], [1 x i64] }">
-//       CHECK:   llvm.insertvalue %{{.*}}[4, 0] : !llvm<"{ double*, double*, i64, [1 x i64], [1 x i64] }">
+//       CHECK:   llvm.mlir.undef : !llvm.struct<(ptr<double>, ptr<double>, i64, array<1 x i64>, array<1 x i64>)>
+//       CHECK:   llvm.extractvalue %{{.*}}[4, 0] : !llvm.struct<(ptr<double>, ptr<double>, i64, array<2 x i64>, array<2 x i64>)>
+//       CHECK:   llvm.extractvalue %{{.*}}[4, 1] : !llvm.struct<(ptr<double>, ptr<double>, i64, array<2 x i64>, array<2 x i64>)>
+//       CHECK:   llvm.extractvalue %{{.*}}[2] : !llvm.struct<(ptr<double>, ptr<double>, i64, array<2 x i64>, array<2 x i64>)>
+//       CHECK:   llvm.insertvalue %{{.*}}, %{{.*}}[0] : !llvm.struct<(ptr<double>, ptr<double>, i64, array<1 x i64>, array<1 x i64>)>
+//       CHECK:   llvm.insertvalue %{{.*}}[2] : !llvm.struct<(ptr<double>, ptr<double>, i64, array<1 x i64>, array<1 x i64>)>
+//       CHECK:   llvm.extractvalue %{{.*}}[0] : !llvm.struct<(i64, i64, i64)>
+//       CHECK:   llvm.extractvalue %{{.*}}[1] : !llvm.struct<(i64, i64, i64)>
+//       CHECK:   llvm.insertvalue %{{.*}}[3, 0] : !llvm.struct<(ptr<double>, ptr<double>, i64, array<1 x i64>, array<1 x i64>)>
+//       CHECK:   llvm.insertvalue %{{.*}}[4, 0] : !llvm.struct<(ptr<double>, ptr<double>, i64, array<1 x i64>, array<1 x i64>)>
 
 func @transpose(%arg0: memref<?x?x?xf32, offset: ?, strides: [?, ?, 1]>) {
   %0 = linalg.transpose %arg0 (i, j, k) -> (k, i, j) : memref<?x?x?xf32, offset: ?, strides: [?, ?, 1]>
   return
 }
 // CHECK-LABEL: func @transpose
-//       CHECK:   llvm.mlir.undef : !llvm<"{ float*, float*, i64, [3 x i64], [3 x i64] }">
-//       CHECK:   llvm.insertvalue {{.*}}[0] : !llvm<"{ float*, float*, i64, [3 x i64], [3 x i64] }">
-//       CHECK:    llvm.insertvalue {{.*}}[1] : !llvm<"{ float*, float*, i64, [3 x i64], [3 x i64] }">
-//       CHECK:    llvm.insertvalue {{.*}}[2] : !llvm<"{ float*, float*, i64, [3 x i64], [3 x i64] }">
-//       CHECK:   llvm.extractvalue {{.*}}[3, 0] : !llvm<"{ float*, float*, i64, [3 x i64], [3 x i64] }">
-//       CHECK:    llvm.insertvalue {{.*}}[3, 2] : !llvm<"{ float*, float*, i64, [3 x i64], [3 x i64] }">
-//       CHECK:   llvm.extractvalue {{.*}}[3, 1] : !llvm<"{ float*, float*, i64, [3 x i64], [3 x i64] }">
-//       CHECK:    llvm.insertvalue {{.*}}[3, 0] : !llvm<"{ float*, float*, i64, [3 x i64], [3 x i64] }">
-//       CHECK:   llvm.extractvalue {{.*}}[3, 2] : !llvm<"{ float*, float*, i64, [3 x i64], [3 x i64] }">
-//       CHECK:    llvm.insertvalue {{.*}}[3, 1] : !llvm<"{ float*, float*, i64, [3 x i64], [3 x i64] }">
+//       CHECK:   llvm.mlir.undef : !llvm.struct<(ptr<float>, ptr<float>, i64, array<3 x i64>, array<3 x i64>)>
+//       CHECK:   llvm.insertvalue {{.*}}[0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<3 x i64>, array<3 x i64>)>
+//       CHECK:    llvm.insertvalue {{.*}}[1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<3 x i64>, array<3 x i64>)>
+//       CHECK:    llvm.insertvalue {{.*}}[2] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<3 x i64>, array<3 x i64>)>
+//       CHECK:   llvm.extractvalue {{.*}}[3, 0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<3 x i64>, array<3 x i64>)>
+//       CHECK:    llvm.insertvalue {{.*}}[3, 2] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<3 x i64>, array<3 x i64>)>
+//       CHECK:   llvm.extractvalue {{.*}}[3, 1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<3 x i64>, array<3 x i64>)>
+//       CHECK:    llvm.insertvalue {{.*}}[3, 0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<3 x i64>, array<3 x i64>)>
+//       CHECK:   llvm.extractvalue {{.*}}[3, 2] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<3 x i64>, array<3 x i64>)>
+//       CHECK:    llvm.insertvalue {{.*}}[3, 1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<3 x i64>, array<3 x i64>)>
 
 func @reshape_static_expand(%arg0: memref<3x4x5xf32>) -> memref<1x3x4x1x5xf32> {
   // Reshapes that expand a contiguous tensor with some 1's.
@@ -94,33 +94,33 @@ func @reshape_static_expand(%arg0: memref<3x4x5xf32>) -> memref<1x3x4x1x5xf32> {
   return %0 : memref<1x3x4x1x5xf32>
 }
 // CHECK-LABEL: func @reshape_static_expand
-//       CHECK:    llvm.mlir.undef : !llvm<"{ float*, float*, i64, [5 x i64], [5 x i64] }">
-//       CHECK:    llvm.extractvalue %{{.*}}[0] : !llvm<"{ float*, float*, i64, [3 x i64], [3 x i64] }">
-//       CHECK:    llvm.insertvalue %{{.*}}, %{{.*}}[0] : !llvm<"{ float*, float*, i64, [5 x i64], [5 x i64] }">
-//       CHECK:    llvm.extractvalue %{{.*}}[1] : !llvm<"{ float*, float*, i64, [3 x i64], [3 x i64] }">
-//       CHECK:    llvm.insertvalue %{{.*}}, %{{.*}}[1] : !llvm<"{ float*, float*, i64, [5 x i64], [5 x i64] }">
-//       CHECK:    llvm.extractvalue %{{.*}}[2] : !llvm<"{ float*, float*, i64, [3 x i64], [3 x i64] }">
-//       CHECK:    llvm.insertvalue %{{.*}}, %{{.*}}[2] : !llvm<"{ float*, float*, i64, [5 x i64], [5 x i64] }">
+//       CHECK:    llvm.mlir.undef : !llvm.struct<(ptr<float>, ptr<float>, i64, array<5 x i64>, array<5 x i64>)>
+//       CHECK:    llvm.extractvalue %{{.*}}[0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<3 x i64>, array<3 x i64>)>
+//       CHECK:    llvm.insertvalue %{{.*}}, %{{.*}}[0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<5 x i64>, array<5 x i64>)>
+//       CHECK:    llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<3 x i64>, array<3 x i64>)>
+//       CHECK:    llvm.insertvalue %{{.*}}, %{{.*}}[1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<5 x i64>, array<5 x i64>)>
+//       CHECK:    llvm.extractvalue %{{.*}}[2] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<3 x i64>, array<3 x i64>)>
+//       CHECK:    llvm.insertvalue %{{.*}}, %{{.*}}[2] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<5 x i64>, array<5 x i64>)>
 //       CHECK:    llvm.mlir.constant(1 : index) : !llvm.i64
-//       CHECK:    llvm.insertvalue %{{.*}}, %{{.*}}[3, 0] : !llvm<"{ float*, float*, i64, [5 x i64], [5 x i64] }">
+//       CHECK:    llvm.insertvalue %{{.*}}, %{{.*}}[3, 0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<5 x i64>, array<5 x i64>)>
 //       CHECK:    llvm.mlir.constant(3 : index) : !llvm.i64
-//       CHECK:    llvm.insertvalue %{{.*}}, %{{.*}}[3, 1] : !llvm<"{ float*, float*, i64, [5 x i64], [5 x i64] }">
+//       CHECK:    llvm.insertvalue %{{.*}}, %{{.*}}[3, 1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<5 x i64>, array<5 x i64>)>
 //       CHECK:    llvm.mlir.constant(4 : index) : !llvm.i64
-//       CHECK:    llvm.insertvalue %{{.*}}, %{{.*}}[3, 2] : !llvm<"{ float*, float*, i64, [5 x i64], [5 x i64] }">
+//       CHECK:    llvm.insertvalue %{{.*}}, %{{.*}}[3, 2] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<5 x i64>, array<5 x i64>)>
 //       CHECK:    llvm.mlir.constant(1 : index) : !llvm.i64
-//       CHECK:    llvm.insertvalue %{{.*}}, %{{.*}}[3, 3] : !llvm<"{ float*, float*, i64, [5 x i64], [5 x i64] }">
+//       CHECK:    llvm.insertvalue %{{.*}}, %{{.*}}[3, 3] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<5 x i64>, array<5 x i64>)>
 //       CHECK:    llvm.mlir.constant(5 : index) : !llvm.i64
-//       CHECK:    llvm.insertvalue %{{.*}}, %{{.*}}[3, 4] : !llvm<"{ float*, float*, i64, [5 x i64], [5 x i64] }">
+//       CHECK:    llvm.insertvalue %{{.*}}, %{{.*}}[3, 4] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<5 x i64>, array<5 x i64>)>
 //       CHECK:    llvm.mlir.constant(60 : index) : !llvm.i64
-//       CHECK:    llvm.insertvalue %{{.*}}, %{{.*}}[4, 0] : !llvm<"{ float*, float*, i64, [5 x i64], [5 x i64] }">
+//       CHECK:    llvm.insertvalue %{{.*}}, %{{.*}}[4, 0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<5 x i64>, array<5 x i64>)>
 //       CHECK:    llvm.mlir.constant(20 : index) : !llvm.i64
-//       CHECK:    llvm.insertvalue %{{.*}}, %{{.*}}[4, 1] : !llvm<"{ float*, float*, i64, [5 x i64], [5 x i64] }">
+//       CHECK:    llvm.insertvalue %{{.*}}, %{{.*}}[4, 1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<5 x i64>, array<5 x i64>)>
 //       CHECK:    llvm.mlir.constant(5 : index) : !llvm.i64
-//       CHECK:    llvm.insertvalue %{{.*}}, %{{.*}}[4, 2] : !llvm<"{ float*, float*, i64, [5 x i64], [5 x i64] }">
+//       CHECK:    llvm.insertvalue %{{.*}}, %{{.*}}[4, 2] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<5 x i64>, array<5 x i64>)>
 //       CHECK:    llvm.mlir.constant(5 : index) : !llvm.i64
-//       CHECK:    llvm.insertvalue %{{.*}}, %{{.*}}[4, 3] : !llvm<"{ float*, float*, i64, [5 x i64], [5 x i64] }">
+//       CHECK:    llvm.insertvalue %{{.*}}, %{{.*}}[4, 3] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<5 x i64>, array<5 x i64>)>
 //       CHECK:    llvm.mlir.constant(1 : index) : !llvm.i64
-//       CHECK:    llvm.insertvalue %{{.*}}, %{{.*}}[4, 4] : !llvm<"{ float*, float*, i64, [5 x i64], [5 x i64] }">
+//       CHECK:    llvm.insertvalue %{{.*}}, %{{.*}}[4, 4] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<5 x i64>, array<5 x i64>)>
 
 func @reshape_static_collapse(%arg0: memref<1x3x4x1x5xf32>) -> memref<3x4x5xf32> {
   %0 = linalg.reshape %arg0 [affine_map<(i, j, k, l, m) -> (i, j)>,
@@ -130,56 +130,56 @@ func @reshape_static_collapse(%arg0: memref<1x3x4x1x5xf32>) -> memref<3x4x5xf32>
   return %0 : memref<3x4x5xf32>
 }
 // CHECK-LABEL: func @reshape_static_collapse
-//       CHECK:    llvm.mlir.undef : !llvm<"{ float*, float*, i64, [3 x i64], [3 x i64] }">
-//       CHECK:    llvm.extractvalue %{{.*}}[0] : !llvm<"{ float*, float*, i64, [5 x i64], [5 x i64] }">
-//       CHECK:    llvm.insertvalue %{{.*}}, %{{.*}}[0] : !llvm<"{ float*, float*, i64, [3 x i64], [3 x i64] }">
-//       CHECK:    llvm.extractvalue %{{.*}}[1] : !llvm<"{ float*, float*, i64, [5 x i64], [5 x i64] }">
-//       CHECK:    llvm.insertvalue %{{.*}}, %{{.*}}[1] : !llvm<"{ float*, float*, i64, [3 x i64], [3 x i64] }">
-//       CHECK:    llvm.extractvalue %{{.*}}[2] : !llvm<"{ float*, float*, i64, [5 x i64], [5 x i64] }">
-//       CHECK:    llvm.insertvalue %{{.*}}, %{{.*}}[2] : !llvm<"{ float*, float*, i64, [3 x i64], [3 x i64] }">
+//       CHECK:    llvm.mlir.undef : !llvm.struct<(ptr<float>, ptr<float>, i64, array<3 x i64>, array<3 x i64>)>
+//       CHECK:    llvm.extractvalue %{{.*}}[0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<5 x i64>, array<5 x i64>)>
+//       CHECK:    llvm.insertvalue %{{.*}}, %{{.*}}[0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<3 x i64>, array<3 x i64>)>
+//       CHECK:    llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<5 x i64>, array<5 x i64>)>
+//       CHECK:    llvm.insertvalue %{{.*}}, %{{.*}}[1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<3 x i64>, array<3 x i64>)>
+//       CHECK:    llvm.extractvalue %{{.*}}[2] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<5 x i64>, array<5 x i64>)>
+//       CHECK:    llvm.insertvalue %{{.*}}, %{{.*}}[2] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<3 x i64>, array<3 x i64>)>
 //       CHECK:    llvm.mlir.constant(3 : index) : !llvm.i64
-//       CHECK:    llvm.insertvalue %{{.*}}, %{{.*}}[3, 0] : !llvm<"{ float*, float*, i64, [3 x i64], [3 x i64] }">
+//       CHECK:    llvm.insertvalue %{{.*}}, %{{.*}}[3, 0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<3 x i64>, array<3 x i64>)>
 //       CHECK:    llvm.mlir.constant(4 : index) : !llvm.i64
-//       CHECK:    llvm.insertvalue %{{.*}}, %{{.*}}[3, 1] : !llvm<"{ float*, float*, i64, [3 x i64], [3 x i64] }">
+//       CHECK:    llvm.insertvalue %{{.*}}, %{{.*}}[3, 1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<3 x i64>, array<3 x i64>)>
 //       CHECK:    llvm.mlir.constant(5 : index) : !llvm.i64
-//       CHECK:    llvm.insertvalue %{{.*}}, %{{.*}}[3, 2] : !llvm<"{ float*, float*, i64, [3 x i64], [3 x i64] }">
+//       CHECK:    llvm.insertvalue %{{.*}}, %{{.*}}[3, 2] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<3 x i64>, array<3 x i64>)>
 //       CHECK:    llvm.mlir.constant(20 : index) : !llvm.i64
-//       CHECK:    llvm.insertvalue %{{.*}}, %{{.*}}[4, 0] : !llvm<"{ float*, float*, i64, [3 x i64], [3 x i64] }">
+//       CHECK:    llvm.insertvalue %{{.*}}, %{{.*}}[4, 0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<3 x i64>, array<3 x i64>)>
 //       CHECK:    llvm.mlir.constant(5 : index) : !llvm.i64
-//       CHECK:    llvm.insertvalue %{{.*}}, %{{.*}}[4, 1] : !llvm<"{ float*, float*, i64, [3 x i64], [3 x i64] }">
+//       CHECK:    llvm.insertvalue %{{.*}}, %{{.*}}[4, 1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<3 x i64>, array<3 x i64>)>
 //       CHECK:    llvm.mlir.constant(1 : index) : !llvm.i64
-//       CHECK:    llvm.insertvalue %{{.*}}, %{{.*}}[4, 2] : !llvm<"{ float*, float*, i64, [3 x i64], [3 x i64] }">
+//       CHECK:    llvm.insertvalue %{{.*}}, %{{.*}}[4, 2] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<3 x i64>, array<3 x i64>)>
 
 func @reshape_fold_zero_dim(%arg0 : memref<1x1xf32>) -> memref<f32> {
   %0 = linalg.reshape %arg0 [] : memref<1x1xf32> into memref<f32>
   return %0 : memref<f32>
 }
 // CHECK-LABEL: func @reshape_fold_zero_dim
-//       CHECK:   llvm.mlir.undef : !llvm<"{ float*, float*, i64 }">
-//       CHECK:   llvm.extractvalue %{{.*}}[0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
-//       CHECK:   llvm.insertvalue %{{.*}}, %{{.*}}[0] : !llvm<"{ float*, float*, i64 }">
-//       CHECK:   llvm.extractvalue %{{.*}}[1] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
-//       CHECK:   llvm.insertvalue %{{.*}}, %{{.*}}[1] : !llvm<"{ float*, float*, i64 }">
-//       CHECK:   llvm.extractvalue %{{.*}}[2] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
-//       CHECK:   llvm.insertvalue %{{.*}}, %{{.*}}[2] : !llvm<"{ float*, float*, i64 }">
+//       CHECK:   llvm.mlir.undef : !llvm.struct<(ptr<float>, ptr<float>, i64)>
+//       CHECK:   llvm.extractvalue %{{.*}}[0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
+//       CHECK:   llvm.insertvalue %{{.*}}, %{{.*}}[0] : !llvm.struct<(ptr<float>, ptr<float>, i64)>
+//       CHECK:   llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
+//       CHECK:   llvm.insertvalue %{{.*}}, %{{.*}}[1] : !llvm.struct<(ptr<float>, ptr<float>, i64)>
+//       CHECK:   llvm.extractvalue %{{.*}}[2] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
+//       CHECK:   llvm.insertvalue %{{.*}}, %{{.*}}[2] : !llvm.struct<(ptr<float>, ptr<float>, i64)>
 
 func @reshape_expand_zero_dim(%arg0 : memref<f32>) -> memref<1x1xf32> {
   %0 = linalg.reshape %arg0 [] : memref<f32> into memref<1x1xf32>
   return %0 : memref<1x1xf32>
 }
 // CHECK-LABEL: func @reshape_expand_zero_dim
-//       CHECK:   llvm.mlir.undef : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
-//       CHECK:   llvm.extractvalue %{{.*}}[0] : !llvm<"{ float*, float*, i64 }">
-//       CHECK:   llvm.insertvalue %{{.*}}, %{{.*}}[0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
-//       CHECK:   llvm.extractvalue %{{.*}}[1] : !llvm<"{ float*, float*, i64 }">
-//       CHECK:   llvm.insertvalue %{{.*}}, %{{.*}}[1] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
-//       CHECK:   llvm.extractvalue %{{.*}}[2] : !llvm<"{ float*, float*, i64 }">
-//       CHECK:   llvm.insertvalue %{{.*}}, %{{.*}}[2] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+//       CHECK:   llvm.mlir.undef : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
+//       CHECK:   llvm.extractvalue %{{.*}}[0] : !llvm.struct<(ptr<float>, ptr<float>, i64)>
+//       CHECK:   llvm.insertvalue %{{.*}}, %{{.*}}[0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
+//       CHECK:   llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr<float>, ptr<float>, i64)>
+//       CHECK:   llvm.insertvalue %{{.*}}, %{{.*}}[1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
+//       CHECK:   llvm.extractvalue %{{.*}}[2] : !llvm.struct<(ptr<float>, ptr<float>, i64)>
+//       CHECK:   llvm.insertvalue %{{.*}}, %{{.*}}[2] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
 //       CHECK:   llvm.mlir.constant(1 : index) : !llvm.i64
-//       CHECK:   llvm.insertvalue %{{.*}}, %{{.*}}[3, 0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+//       CHECK:   llvm.insertvalue %{{.*}}, %{{.*}}[3, 0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
 //       CHECK:   llvm.mlir.constant(1 : index) : !llvm.i64
-//       CHECK:   llvm.insertvalue %{{.*}}, %{{.*}}[3, 1] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+//       CHECK:   llvm.insertvalue %{{.*}}, %{{.*}}[3, 1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
 //       CHECK:   llvm.mlir.constant(1 : index) : !llvm.i64
-//       CHECK:   llvm.insertvalue %{{.*}}, %{{.*}}[4, 0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+//       CHECK:   llvm.insertvalue %{{.*}}, %{{.*}}[4, 0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
 //       CHECK:   llvm.mlir.constant(1 : index) : !llvm.i64
-//       CHECK:   llvm.insertvalue %{{.*}}, %{{.*}}[4, 1] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+//       CHECK:   llvm.insertvalue %{{.*}}, %{{.*}}[4, 1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<2 x i64>, array<2 x i64>)>
diff --git a/mlir/test/Target/avx512.mlir b/mlir/test/Target/avx512.mlir
index 5e75a98dc4ef8..0cc336d29df0c 100644
--- a/mlir/test/Target/avx512.mlir
+++ b/mlir/test/Target/avx512.mlir
@@ -1,31 +1,31 @@
 // RUN: mlir-opt -verify-diagnostics %s | mlir-opt | mlir-translate --avx512-mlir-to-llvmir | FileCheck %s
 
 // CHECK-LABEL: define <16 x float> @LLVM_x86_avx512_mask_ps_512
-llvm.func @LLVM_x86_avx512_mask_ps_512(%a: !llvm<"<16 x float>">,
+llvm.func @LLVM_x86_avx512_mask_ps_512(%a: !llvm.vec<16 x float>,
                                        %b: !llvm.i32,
                                        %c: !llvm.i16)
-  -> (!llvm<"<16 x float>">)
+  -> (!llvm.vec<16 x float>)
 {
   // CHECK: call <16 x float> @llvm.x86.avx512.mask.rndscale.ps.512(<16 x float>
   %0 = "llvm_avx512.mask.rndscale.ps.512"(%a, %b, %a, %c, %b) :
-    (!llvm<"<16 x float>">, !llvm.i32, !llvm<"<16 x float>">, !llvm.i16, !llvm.i32) -> !llvm<"<16 x float>">
+    (!llvm.vec<16 x float>, !llvm.i32, !llvm.vec<16 x float>, !llvm.i16, !llvm.i32) -> !llvm.vec<16 x float>
   // CHECK: call <16 x float> @llvm.x86.avx512.mask.scalef.ps.512(<16 x float>
   %1 = "llvm_avx512.mask.scalef.ps.512"(%a, %a, %a, %c, %b) :
-    (!llvm<"<16 x float>">, !llvm<"<16 x float>">, !llvm<"<16 x float>">, !llvm.i16, !llvm.i32) -> !llvm<"<16 x float>">
-  llvm.return %1: !llvm<"<16 x float>">
+    (!llvm.vec<16 x float>, !llvm.vec<16 x float>, !llvm.vec<16 x float>, !llvm.i16, !llvm.i32) -> !llvm.vec<16 x float>
+  llvm.return %1: !llvm.vec<16 x float>
 }
 
 // CHECK-LABEL: define <8 x double> @LLVM_x86_avx512_mask_pd_512
-llvm.func @LLVM_x86_avx512_mask_pd_512(%a: !llvm<"<8 x double>">,
+llvm.func @LLVM_x86_avx512_mask_pd_512(%a: !llvm.vec<8 x double>,
                                        %b: !llvm.i32,
                                        %c: !llvm.i8)
-  -> (!llvm<"<8 x double>">)
+  -> (!llvm.vec<8 x double>)
 {
   // CHECK: call <8 x double> @llvm.x86.avx512.mask.rndscale.pd.512(<8 x double>
   %0 = "llvm_avx512.mask.rndscale.pd.512"(%a, %b, %a, %c, %b) :
-    (!llvm<"<8 x double>">, !llvm.i32, !llvm<"<8 x double>">, !llvm.i8, !llvm.i32) -> !llvm<"<8 x double>">
+    (!llvm.vec<8 x double>, !llvm.i32, !llvm.vec<8 x double>, !llvm.i8, !llvm.i32) -> !llvm.vec<8 x double>
   // CHECK: call <8 x double> @llvm.x86.avx512.mask.scalef.pd.512(<8 x double>
   %1 = "llvm_avx512.mask.scalef.pd.512"(%a, %a, %a, %c, %b) :
-    (!llvm<"<8 x double>">, !llvm<"<8 x double>">, !llvm<"<8 x double>">, !llvm.i8, !llvm.i32) -> !llvm<"<8 x double>">
-  llvm.return %1: !llvm<"<8 x double>">
+    (!llvm.vec<8 x double>, !llvm.vec<8 x double>, !llvm.vec<8 x double>, !llvm.i8, !llvm.i32) -> !llvm.vec<8 x double>
+  llvm.return %1: !llvm.vec<8 x double>
 }
diff --git a/mlir/test/Target/import.ll b/mlir/test/Target/import.ll
index 24b4a0b392b0f..d67bbb029f8ac 100644
--- a/mlir/test/Target/import.ll
+++ b/mlir/test/Target/import.ll
@@ -3,22 +3,22 @@
 %struct.t = type {}
 %struct.s = type { %struct.t, i64 }
 
-; CHECK: llvm.mlir.global external @g1() : !llvm<"{ {}, i64 }">
+; CHECK: llvm.mlir.global external @g1() : !llvm.struct<(struct<()>, i64)>
 @g1 = external global %struct.s, align 8
 ; CHECK: llvm.mlir.global external @g2() : !llvm.double
 @g2 = external global double, align 8
 ; CHECK: llvm.mlir.global internal @g3("string")
 @g3 = internal global [6 x i8] c"string"
 
-; CHECK: llvm.mlir.global external @g5() : !llvm<"<8 x i32>">
+; CHECK: llvm.mlir.global external @g5() : !llvm.vec<8 x i32>
 @g5 = external global <8 x i32>
 
 @g4 = external global i32, align 8
-; CHECK: llvm.mlir.global internal constant @int_gep() : !llvm<"i32*"> {
-; CHECK-DAG:   %[[addr:[0-9]+]] = llvm.mlir.addressof @g4 : !llvm<"i32*">
+; CHECK: llvm.mlir.global internal constant @int_gep() : !llvm.ptr<i32> {
+; CHECK-DAG:   %[[addr:[0-9]+]] = llvm.mlir.addressof @g4 : !llvm.ptr<i32>
 ; CHECK-DAG:   %[[c2:[0-9]+]] = llvm.mlir.constant(2 : i32) : !llvm.i32
-; CHECK-NEXT:  %[[gepinit:[0-9]+]] = llvm.getelementptr %[[addr]][%[[c2]]] : (!llvm<"i32*">, !llvm.i32) -> !llvm<"i32*">
-; CHECK-NEXT:  llvm.return %[[gepinit]] : !llvm<"i32*">
+; CHECK-NEXT:  %[[gepinit:[0-9]+]] = llvm.getelementptr %[[addr]][%[[c2]]] : (!llvm.ptr<i32>, !llvm.i32) -> !llvm.ptr<i32>
+; CHECK-NEXT:  llvm.return %[[gepinit]] : !llvm.ptr<i32>
 ; CHECK-NEXT: }
 @int_gep = internal constant i32* getelementptr (i32, i32* @g4, i32 2)
 
@@ -53,15 +53,15 @@
 ; Sequential constants.
 ;
 
-; CHECK: llvm.mlir.global internal constant @vector_constant(dense<[1, 2]> : vector<2xi32>) : !llvm<"<2 x i32>">
+; CHECK: llvm.mlir.global internal constant @vector_constant(dense<[1, 2]> : vector<2xi32>) : !llvm.vec<2 x i32>
 @vector_constant = internal constant <2 x i32> <i32 1, i32 2>
-; CHECK: llvm.mlir.global internal constant @array_constant(dense<[1.000000e+00, 2.000000e+00]> : tensor<2xf32>) : !llvm<"[2 x float]">
+; CHECK: llvm.mlir.global internal constant @array_constant(dense<[1.000000e+00, 2.000000e+00]> : tensor<2xf32>) : !llvm.array<2 x float>
 @array_constant = internal constant [2 x float] [float 1., float 2.]
-; CHECK: llvm.mlir.global internal constant @nested_array_constant(dense<[{{\[}}1, 2], [3, 4]]> : tensor<2x2xi32>) : !llvm<"[2 x [2 x i32]]">
+; CHECK: llvm.mlir.global internal constant @nested_array_constant(dense<[{{\[}}1, 2], [3, 4]]> : tensor<2x2xi32>) : !llvm.array<2 x array<2 x i32>>
 @nested_array_constant = internal constant [2 x [2 x i32]] [[2 x i32] [i32 1, i32 2], [2 x i32] [i32 3, i32 4]]
-; CHECK: llvm.mlir.global internal constant @nested_array_constant3(dense<[{{\[}}[1, 2], [3, 4]]]> : tensor<1x2x2xi32>) : !llvm<"[1 x [2 x [2 x i32]]]">
+; CHECK: llvm.mlir.global internal constant @nested_array_constant3(dense<[{{\[}}[1, 2], [3, 4]]]> : tensor<1x2x2xi32>) : !llvm.array<1 x array<2 x array<2 x i32>>>
 @nested_array_constant3 = internal constant [1 x [2 x [2 x i32]]] [[2 x [2 x i32]] [[2 x i32] [i32 1, i32 2], [2 x i32] [i32 3, i32 4]]]
-; CHECK: llvm.mlir.global internal constant @nested_array_vector(dense<[{{\[}}[1, 2], [3, 4]]]> : vector<1x2x2xi32>) : !llvm<"[1 x [2 x <2 x i32>]]">
+; CHECK: llvm.mlir.global internal constant @nested_array_vector(dense<[{{\[}}[1, 2], [3, 4]]]> : vector<1x2x2xi32>) : !llvm.array<1 x array<2 x vec<2 x i32>>>
 @nested_array_vector = internal constant [1 x [2 x <2 x i32>]] [[2 x <2 x i32>] [<2 x i32> <i32 1, i32 2>, <2 x i32> <i32 3, i32 4>]]
 
 ;
@@ -84,13 +84,13 @@ declare float @fe(i32)
 ; CHECK-DAG: %[[c43:[0-9]+]] = llvm.mlir.constant(43 : i32) : !llvm.i32
 define internal dso_local i32 @f1(i64 %a) norecurse {
 entry:
-; CHECK: %{{[0-9]+}} = llvm.inttoptr %arg0 : !llvm.i64 to !llvm<"i64*">
+; CHECK: %{{[0-9]+}} = llvm.inttoptr %arg0 : !llvm.i64 to !llvm.ptr<i64>
   %aa = inttoptr i64 %a to i64*
-; %[[addrof:[0-9]+]] = llvm.mlir.addressof @g2 : !llvm<"double*">
-; %[[addrof2:[0-9]+]] = llvm.mlir.addressof @g2 : !llvm<"double*">
-; %{{[0-9]+}} = llvm.inttoptr %arg0 : !llvm.i64 to !llvm<"i64*">
-; %{{[0-9]+}} = llvm.ptrtoint %[[addrof2]] : !llvm<"double*"> to !llvm.i64
-; %{{[0-9]+}} = llvm.getelementptr %[[addrof]][%3] : (!llvm<"double*">, !llvm.i32) -> !llvm<"double*">
+; %[[addrof:[0-9]+]] = llvm.mlir.addressof @g2 : !llvm.ptr<double>
+; %[[addrof2:[0-9]+]] = llvm.mlir.addressof @g2 : !llvm.ptr<double>
+; %{{[0-9]+}} = llvm.inttoptr %arg0 : !llvm.i64 to !llvm.ptr<i64>
+; %{{[0-9]+}} = llvm.ptrtoint %[[addrof2]] : !llvm.ptr<double> to !llvm.i64
+; %{{[0-9]+}} = llvm.getelementptr %[[addrof]][%3] : (!llvm.ptr<double>, !llvm.i32) -> !llvm.ptr<double>
   %bb = ptrtoint double* @g2 to i64
   %cc = getelementptr double, double* @g2, i32 2
 ; CHECK: %[[b:[0-9]+]] = llvm.trunc %arg0 : !llvm.i64 to !llvm.i32
@@ -161,18 +161,18 @@ next:
   br label %end
 }
 
-; CHECK-LABEL: llvm.func @f3() -> !llvm<"i32*">
+; CHECK-LABEL: llvm.func @f3() -> !llvm.ptr<i32>
 define i32* @f3() {
-; CHECK: %[[c:[0-9]+]] = llvm.mlir.addressof @g2 : !llvm<"double*">
-; CHECK: %[[b:[0-9]+]] = llvm.bitcast %[[c]] : !llvm<"double*"> to !llvm<"i32*">
-; CHECK: llvm.return %[[b]] : !llvm<"i32*">
+; CHECK: %[[c:[0-9]+]] = llvm.mlir.addressof @g2 : !llvm.ptr<double>
+; CHECK: %[[b:[0-9]+]] = llvm.bitcast %[[c]] : !llvm.ptr<double> to !llvm.ptr<i32>
+; CHECK: llvm.return %[[b]] : !llvm.ptr<i32>
   ret i32* bitcast (double* @g2 to i32*)
 }
 
-; CHECK-LABEL: llvm.func @f4() -> !llvm<"i32*">
+; CHECK-LABEL: llvm.func @f4() -> !llvm.ptr<i32>
 define i32* @f4() {
-; CHECK: %[[b:[0-9]+]] = llvm.mlir.null : !llvm<"i32*">
-; CHECK: llvm.return %[[b]] : !llvm<"i32*">
+; CHECK: %[[b:[0-9]+]] = llvm.mlir.null : !llvm.ptr<i32>
+; CHECK: llvm.return %[[b]] : !llvm.ptr<i32>
   ret i32* bitcast (double* null to i32*)
 }
 
@@ -198,7 +198,7 @@ define void @f5(i32 %d) {
   ret void
 }
 
-; CHECK-LABEL: llvm.func @f6(%arg0: !llvm<"void (i16)*">)
+; CHECK-LABEL: llvm.func @f6(%arg0: !llvm.ptr<func<void (i16)>>)
 define void @f6(void (i16) *%fn) {
 ; CHECK: %[[c:[0-9]+]] = llvm.mlir.constant(0 : i16) : !llvm.i16
 ; CHECK: llvm.call %arg0(%[[c]])
@@ -243,7 +243,7 @@ define void @FPArithmetic(float %a, float %b, double %c, double %d) {
 ; CHECK-LABEL: @precaller
 define i32 @precaller() {
   %1 = alloca i32 ()*
-  ; CHECK: %[[func:.*]] = llvm.mlir.addressof @callee : !llvm<"i32 ()*">
+  ; CHECK: %[[func:.*]] = llvm.mlir.addressof @callee : !llvm.ptr<func<i32 ()>>
   ; CHECK: llvm.store %[[func]], %[[loc:.*]]
   store i32 ()* @callee, i32 ()** %1
   ; CHECK: %[[indir:.*]] = llvm.load %[[loc]]
@@ -261,7 +261,7 @@ define i32 @callee() {
 ; CHECK-LABEL: @postcaller
 define i32 @postcaller() {
   %1 = alloca i32 ()*
-  ; CHECK: %[[func:.*]] = llvm.mlir.addressof @callee : !llvm<"i32 ()*">
+  ; CHECK: %[[func:.*]] = llvm.mlir.addressof @callee : !llvm.ptr<func<i32 ()>>
   ; CHECK: llvm.store %[[func]], %[[loc:.*]]
   store i32 ()* @callee, i32 ()** %1
   ; CHECK: %[[indir:.*]] = llvm.load %[[loc]]
@@ -279,16 +279,16 @@ declare i32 @__gxx_personality_v0(...)
 
 ; CHECK-LABEL: @invokeLandingpad
 define i32 @invokeLandingpad() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
-  ; CHECK: %[[a1:[0-9]+]] = llvm.bitcast %{{[0-9]+}} : !llvm<"i8***"> to !llvm<"i8*">
-  ; CHECK: %[[a3:[0-9]+]] = llvm.alloca %{{[0-9]+}} x !llvm.i8 : (!llvm.i32) -> !llvm<"i8*">
+  ; CHECK: %[[a1:[0-9]+]] = llvm.bitcast %{{[0-9]+}} : !llvm.ptr<ptr<ptr<i8>>> to !llvm.ptr<i8>
+  ; CHECK: %[[a3:[0-9]+]] = llvm.alloca %{{[0-9]+}} x !llvm.i8 : (!llvm.i32) -> !llvm.ptr<i8>
   %1 = alloca i8
-  ; CHECK: llvm.invoke @foo(%[[a3]]) to ^bb2 unwind ^bb1 : (!llvm<"i8*">) -> ()
+  ; CHECK: llvm.invoke @foo(%[[a3]]) to ^bb2 unwind ^bb1 : (!llvm.ptr<i8>) -> ()
   invoke void @foo(i8* %1) to label %4 unwind label %2
 
 ; CHECK: ^bb1:
-  ; CHECK: %{{[0-9]+}} = llvm.landingpad (catch %{{[0-9]+}} : !llvm<"i8**">) (catch %[[a1]] : !llvm<"i8*">) (filter %{{[0-9]+}} : !llvm<"[1 x i8]">) : !llvm<"{ i8*, i32 }">
+  ; CHECK: %{{[0-9]+}} = llvm.landingpad (catch %{{[0-9]+}} : !llvm.ptr<ptr<i8>>) (catch %[[a1]] : !llvm.ptr<i8>) (filter %{{[0-9]+}} : !llvm.array<1 x i8>) : !llvm.struct<(ptr<i8>, i32)>
   %3 = landingpad { i8*, i32 } catch i8** @_ZTIi catch i8* bitcast (i8*** @_ZTIii to i8*)
-  ; FIXME: Change filter to a constant array once they are handled. 
+  ; FIXME: Change filter to a constant array once they are handled.
   ; Currently, even though it parses this, LLVM module is broken
           filter [1 x i8] [i8 1]
   resume { i8*, i32 } %3
@@ -298,7 +298,7 @@ define i32 @invokeLandingpad() personality i8* bitcast (i32 (...)* @__gxx_person
   ret i32 1
 
 ; CHECK: ^bb3:
-  ; CHECK: %{{[0-9]+}} = llvm.invoke @bar(%[[a3]]) to ^bb2 unwind ^bb1 : (!llvm<"i8*">) -> !llvm<"i8*">
+  ; CHECK: %{{[0-9]+}} = llvm.invoke @bar(%[[a3]]) to ^bb2 unwind ^bb1 : (!llvm.ptr<i8>) -> !llvm.ptr<i8>
   %6 = invoke i8* @bar(i8* %1) to label %4 unwind label %2
 
 ; CHECK: ^bb4:
diff --git a/mlir/test/Target/llvmir-intrinsics.mlir b/mlir/test/Target/llvmir-intrinsics.mlir
index 286d8fbe5f68b..fc286599ee955 100644
--- a/mlir/test/Target/llvmir-intrinsics.mlir
+++ b/mlir/test/Target/llvmir-intrinsics.mlir
@@ -1,277 +1,277 @@
 // RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
 
 // CHECK-LABEL: @intrinsics
-llvm.func @intrinsics(%arg0: !llvm.float, %arg1: !llvm.float, %arg2: !llvm<"<8 x float>">, %arg3: !llvm<"i8*">) {
+llvm.func @intrinsics(%arg0: !llvm.float, %arg1: !llvm.float, %arg2: !llvm.vec<8 x float>, %arg3: !llvm.ptr<i8>) {
   %c3 = llvm.mlir.constant(3 : i32) : !llvm.i32
   %c1 = llvm.mlir.constant(1 : i32) : !llvm.i32
   %c0 = llvm.mlir.constant(0 : i32) : !llvm.i32
   // CHECK: call float @llvm.fmuladd.f32
   "llvm.intr.fmuladd"(%arg0, %arg1, %arg0) : (!llvm.float, !llvm.float, !llvm.float) -> !llvm.float
   // CHECK: call <8 x float> @llvm.fmuladd.v8f32
-  "llvm.intr.fmuladd"(%arg2, %arg2, %arg2) : (!llvm<"<8 x float>">, !llvm<"<8 x float>">, !llvm<"<8 x float>">) -> !llvm<"<8 x float>">
+  "llvm.intr.fmuladd"(%arg2, %arg2, %arg2) : (!llvm.vec<8 x float>, !llvm.vec<8 x float>, !llvm.vec<8 x float>) -> !llvm.vec<8 x float>
   // CHECK: call float @llvm.fma.f32
   "llvm.intr.fma"(%arg0, %arg1, %arg0) : (!llvm.float, !llvm.float, !llvm.float) -> !llvm.float
   // CHECK: call <8 x float> @llvm.fma.v8f32
-  "llvm.intr.fma"(%arg2, %arg2, %arg2) : (!llvm<"<8 x float>">, !llvm<"<8 x float>">, !llvm<"<8 x float>">) -> !llvm<"<8 x float>">
+  "llvm.intr.fma"(%arg2, %arg2, %arg2) : (!llvm.vec<8 x float>, !llvm.vec<8 x float>, !llvm.vec<8 x float>) -> !llvm.vec<8 x float>
   // CHECK: call void @llvm.prefetch.p0i8(i8* %3, i32 0, i32 3, i32 1)
-  "llvm.intr.prefetch"(%arg3, %c0, %c3, %c1) : (!llvm<"i8*">, !llvm.i32, !llvm.i32, !llvm.i32) -> ()
+  "llvm.intr.prefetch"(%arg3, %c0, %c3, %c1) : (!llvm.ptr<i8>, !llvm.i32, !llvm.i32, !llvm.i32) -> ()
   llvm.return
 }
 
 // CHECK-LABEL: @exp_test
-llvm.func @exp_test(%arg0: !llvm.float, %arg1: !llvm<"<8 x float>">) {
+llvm.func @exp_test(%arg0: !llvm.float, %arg1: !llvm.vec<8 x float>) {
   // CHECK: call float @llvm.exp.f32
   "llvm.intr.exp"(%arg0) : (!llvm.float) -> !llvm.float
   // CHECK: call <8 x float> @llvm.exp.v8f32
-  "llvm.intr.exp"(%arg1) : (!llvm<"<8 x float>">) -> !llvm<"<8 x float>">
+  "llvm.intr.exp"(%arg1) : (!llvm.vec<8 x float>) -> !llvm.vec<8 x float>
   llvm.return
 }
 
 // CHECK-LABEL: @exp2_test
-llvm.func @exp2_test(%arg0: !llvm.float, %arg1: !llvm<"<8 x float>">) {
+llvm.func @exp2_test(%arg0: !llvm.float, %arg1: !llvm.vec<8 x float>) {
   // CHECK: call float @llvm.exp2.f32
   "llvm.intr.exp2"(%arg0) : (!llvm.float) -> !llvm.float
   // CHECK: call <8 x float> @llvm.exp2.v8f32
-  "llvm.intr.exp2"(%arg1) : (!llvm<"<8 x float>">) -> !llvm<"<8 x float>">
+  "llvm.intr.exp2"(%arg1) : (!llvm.vec<8 x float>) -> !llvm.vec<8 x float>
   llvm.return
 }
 
 // CHECK-LABEL: @log_test
-llvm.func @log_test(%arg0: !llvm.float, %arg1: !llvm<"<8 x float>">) {
+llvm.func @log_test(%arg0: !llvm.float, %arg1: !llvm.vec<8 x float>) {
   // CHECK: call float @llvm.log.f32
   "llvm.intr.log"(%arg0) : (!llvm.float) -> !llvm.float
   // CHECK: call <8 x float> @llvm.log.v8f32
-  "llvm.intr.log"(%arg1) : (!llvm<"<8 x float>">) -> !llvm<"<8 x float>">
+  "llvm.intr.log"(%arg1) : (!llvm.vec<8 x float>) -> !llvm.vec<8 x float>
   llvm.return
 }
 
 // CHECK-LABEL: @log10_test
-llvm.func @log10_test(%arg0: !llvm.float, %arg1: !llvm<"<8 x float>">) {
+llvm.func @log10_test(%arg0: !llvm.float, %arg1: !llvm.vec<8 x float>) {
   // CHECK: call float @llvm.log10.f32
   "llvm.intr.log10"(%arg0) : (!llvm.float) -> !llvm.float
   // CHECK: call <8 x float> @llvm.log10.v8f32
-  "llvm.intr.log10"(%arg1) : (!llvm<"<8 x float>">) -> !llvm<"<8 x float>">
+  "llvm.intr.log10"(%arg1) : (!llvm.vec<8 x float>) -> !llvm.vec<8 x float>
   llvm.return
 }
 
 // CHECK-LABEL: @log2_test
-llvm.func @log2_test(%arg0: !llvm.float, %arg1: !llvm<"<8 x float>">) {
+llvm.func @log2_test(%arg0: !llvm.float, %arg1: !llvm.vec<8 x float>) {
   // CHECK: call float @llvm.log2.f32
   "llvm.intr.log2"(%arg0) : (!llvm.float) -> !llvm.float
   // CHECK: call <8 x float> @llvm.log2.v8f32
-  "llvm.intr.log2"(%arg1) : (!llvm<"<8 x float>">) -> !llvm<"<8 x float>">
+  "llvm.intr.log2"(%arg1) : (!llvm.vec<8 x float>) -> !llvm.vec<8 x float>
   llvm.return
 }
 
 // CHECK-LABEL: @fabs_test
-llvm.func @fabs_test(%arg0: !llvm.float, %arg1: !llvm<"<8 x float>">) {
+llvm.func @fabs_test(%arg0: !llvm.float, %arg1: !llvm.vec<8 x float>) {
   // CHECK: call float @llvm.fabs.f32
   "llvm.intr.fabs"(%arg0) : (!llvm.float) -> !llvm.float
   // CHECK: call <8 x float> @llvm.fabs.v8f32
-  "llvm.intr.fabs"(%arg1) : (!llvm<"<8 x float>">) -> !llvm<"<8 x float>">
+  "llvm.intr.fabs"(%arg1) : (!llvm.vec<8 x float>) -> !llvm.vec<8 x float>
   llvm.return
 }
 
 // CHECK-LABEL: @sqrt_test
-llvm.func @sqrt_test(%arg0: !llvm.float, %arg1: !llvm<"<8 x float>">) {
+llvm.func @sqrt_test(%arg0: !llvm.float, %arg1: !llvm.vec<8 x float>) {
   // CHECK: call float @llvm.sqrt.f32
   "llvm.intr.sqrt"(%arg0) : (!llvm.float) -> !llvm.float
   // CHECK: call <8 x float> @llvm.sqrt.v8f32
-  "llvm.intr.sqrt"(%arg1) : (!llvm<"<8 x float>">) -> !llvm<"<8 x float>">
+  "llvm.intr.sqrt"(%arg1) : (!llvm.vec<8 x float>) -> !llvm.vec<8 x float>
   llvm.return
 }
 
 // CHECK-LABEL: @ceil_test
-llvm.func @ceil_test(%arg0: !llvm.float, %arg1: !llvm<"<8 x float>">) {
+llvm.func @ceil_test(%arg0: !llvm.float, %arg1: !llvm.vec<8 x float>) {
   // CHECK: call float @llvm.ceil.f32
   "llvm.intr.ceil"(%arg0) : (!llvm.float) -> !llvm.float
   // CHECK: call <8 x float> @llvm.ceil.v8f32
-  "llvm.intr.ceil"(%arg1) : (!llvm<"<8 x float>">) -> !llvm<"<8 x float>">
+  "llvm.intr.ceil"(%arg1) : (!llvm.vec<8 x float>) -> !llvm.vec<8 x float>
   llvm.return
 }
 
 // CHECK-LABEL: @floor_test
-llvm.func @floor_test(%arg0: !llvm.float, %arg1: !llvm<"<8 x float>">) {
+llvm.func @floor_test(%arg0: !llvm.float, %arg1: !llvm.vec<8 x float>) {
   // CHECK: call float @llvm.floor.f32
   "llvm.intr.floor"(%arg0) : (!llvm.float) -> !llvm.float
   // CHECK: call <8 x float> @llvm.floor.v8f32
-  "llvm.intr.floor"(%arg1) : (!llvm<"<8 x float>">) -> !llvm<"<8 x float>">
+  "llvm.intr.floor"(%arg1) : (!llvm.vec<8 x float>) -> !llvm.vec<8 x float>
   llvm.return
 }
 
 // CHECK-LABEL: @cos_test
-llvm.func @cos_test(%arg0: !llvm.float, %arg1: !llvm<"<8 x float>">) {
+llvm.func @cos_test(%arg0: !llvm.float, %arg1: !llvm.vec<8 x float>) {
   // CHECK: call float @llvm.cos.f32
   "llvm.intr.cos"(%arg0) : (!llvm.float) -> !llvm.float
   // CHECK: call <8 x float> @llvm.cos.v8f32
-  "llvm.intr.cos"(%arg1) : (!llvm<"<8 x float>">) -> !llvm<"<8 x float>">
+  "llvm.intr.cos"(%arg1) : (!llvm.vec<8 x float>) -> !llvm.vec<8 x float>
   llvm.return
 }
 
 // CHECK-LABEL: @copysign_test
-llvm.func @copysign_test(%arg0: !llvm.float, %arg1: !llvm.float, %arg2: !llvm<"<8 x float>">, %arg3: !llvm<"<8 x float>">) {
+llvm.func @copysign_test(%arg0: !llvm.float, %arg1: !llvm.float, %arg2: !llvm.vec<8 x float>, %arg3: !llvm.vec<8 x float>) {
   // CHECK: call float @llvm.copysign.f32
   "llvm.intr.copysign"(%arg0, %arg1) : (!llvm.float, !llvm.float) -> !llvm.float
   // CHECK: call <8 x float> @llvm.copysign.v8f32
-  "llvm.intr.copysign"(%arg2, %arg3) : (!llvm<"<8 x float>">, !llvm<"<8 x float>">) -> !llvm<"<8 x float>">
+  "llvm.intr.copysign"(%arg2, %arg3) : (!llvm.vec<8 x float>, !llvm.vec<8 x float>) -> !llvm.vec<8 x float>
   llvm.return
 }
 
 // CHECK-LABEL: @pow_test
-llvm.func @pow_test(%arg0: !llvm.float, %arg1: !llvm.float, %arg2: !llvm<"<8 x float>">, %arg3: !llvm<"<8 x float>">) {
+llvm.func @pow_test(%arg0: !llvm.float, %arg1: !llvm.float, %arg2: !llvm.vec<8 x float>, %arg3: !llvm.vec<8 x float>) {
   // CHECK: call float @llvm.pow.f32
   "llvm.intr.pow"(%arg0, %arg1) : (!llvm.float, !llvm.float) -> !llvm.float
   // CHECK: call <8 x float> @llvm.pow.v8f32
-  "llvm.intr.pow"(%arg2, %arg3) : (!llvm<"<8 x float>">, !llvm<"<8 x float>">) -> !llvm<"<8 x float>">
+  "llvm.intr.pow"(%arg2, %arg3) : (!llvm.vec<8 x float>, !llvm.vec<8 x float>) -> !llvm.vec<8 x float>
   llvm.return
 }
 
 // CHECK-LABEL: @bitreverse_test
-llvm.func @bitreverse_test(%arg0: !llvm.i32, %arg1: !llvm<"<8 x i32>">) {
+llvm.func @bitreverse_test(%arg0: !llvm.i32, %arg1: !llvm.vec<8 x i32>) {
   // CHECK: call i32 @llvm.bitreverse.i32
   "llvm.intr.bitreverse"(%arg0) : (!llvm.i32) -> !llvm.i32
   // CHECK: call <8 x i32> @llvm.bitreverse.v8i32
-  "llvm.intr.bitreverse"(%arg1) : (!llvm<"<8 x i32>">) -> !llvm<"<8 x i32>">
+  "llvm.intr.bitreverse"(%arg1) : (!llvm.vec<8 x i32>) -> !llvm.vec<8 x i32>
   llvm.return
 }
 
 // CHECK-LABEL: @ctpop_test
-llvm.func @ctpop_test(%arg0: !llvm.i32, %arg1: !llvm<"<8 x i32>">) {
+llvm.func @ctpop_test(%arg0: !llvm.i32, %arg1: !llvm.vec<8 x i32>) {
   // CHECK: call i32 @llvm.ctpop.i32
   "llvm.intr.ctpop"(%arg0) : (!llvm.i32) -> !llvm.i32
   // CHECK: call <8 x i32> @llvm.ctpop.v8i32
-  "llvm.intr.ctpop"(%arg1) : (!llvm<"<8 x i32>">) -> !llvm<"<8 x i32>">
+  "llvm.intr.ctpop"(%arg1) : (!llvm.vec<8 x i32>) -> !llvm.vec<8 x i32>
   llvm.return
 }
 
 // CHECK-LABEL: @maxnum_test
-llvm.func @maxnum_test(%arg0: !llvm.float, %arg1: !llvm.float, %arg2: !llvm<"<8 x float>">, %arg3: !llvm<"<8 x float>">) {
+llvm.func @maxnum_test(%arg0: !llvm.float, %arg1: !llvm.float, %arg2: !llvm.vec<8 x float>, %arg3: !llvm.vec<8 x float>) {
   // CHECK: call float @llvm.maxnum.f32
   "llvm.intr.maxnum"(%arg0, %arg1) : (!llvm.float, !llvm.float) -> !llvm.float
   // CHECK: call <8 x float> @llvm.maxnum.v8f32
-  "llvm.intr.maxnum"(%arg2, %arg3) : (!llvm<"<8 x float>">, !llvm<"<8 x float>">) -> !llvm<"<8 x float>">
+  "llvm.intr.maxnum"(%arg2, %arg3) : (!llvm.vec<8 x float>, !llvm.vec<8 x float>) -> !llvm.vec<8 x float>
   llvm.return
 }
 
 // CHECK-LABEL: @minnum_test
-llvm.func @minnum_test(%arg0: !llvm.float, %arg1: !llvm.float, %arg2: !llvm<"<8 x float>">, %arg3: !llvm<"<8 x float>">) {
+llvm.func @minnum_test(%arg0: !llvm.float, %arg1: !llvm.float, %arg2: !llvm.vec<8 x float>, %arg3: !llvm.vec<8 x float>) {
   // CHECK: call float @llvm.minnum.f32
   "llvm.intr.minnum"(%arg0, %arg1) : (!llvm.float, !llvm.float) -> !llvm.float
   // CHECK: call <8 x float> @llvm.minnum.v8f32
-  "llvm.intr.minnum"(%arg2, %arg3) : (!llvm<"<8 x float>">, !llvm<"<8 x float>">) -> !llvm<"<8 x float>">
+  "llvm.intr.minnum"(%arg2, %arg3) : (!llvm.vec<8 x float>, !llvm.vec<8 x float>) -> !llvm.vec<8 x float>
   llvm.return
 }
 
 // CHECK-LABEL: @smax_test
-llvm.func @smax_test(%arg0: !llvm.i32, %arg1: !llvm.i32, %arg2: !llvm<"<8 x i32>">, %arg3: !llvm<"<8 x i32>">) {
+llvm.func @smax_test(%arg0: !llvm.i32, %arg1: !llvm.i32, %arg2: !llvm.vec<8 x i32>, %arg3: !llvm.vec<8 x i32>) {
   // CHECK: call i32 @llvm.smax.i32
   "llvm.intr.smax"(%arg0, %arg1) : (!llvm.i32, !llvm.i32) -> !llvm.i32
   // CHECK: call <8 x i32> @llvm.smax.v8i32
-  "llvm.intr.smax"(%arg2, %arg3) : (!llvm<"<8 x i32>">, !llvm<"<8 x i32>">) -> !llvm<"<8 x i32>">
+  "llvm.intr.smax"(%arg2, %arg3) : (!llvm.vec<8 x i32>, !llvm.vec<8 x i32>) -> !llvm.vec<8 x i32>
   llvm.return
 }
 
 // CHECK-LABEL: @smin_test
-llvm.func @smin_test(%arg0: !llvm.i32, %arg1: !llvm.i32, %arg2: !llvm<"<8 x i32>">, %arg3: !llvm<"<8 x i32>">) {
+llvm.func @smin_test(%arg0: !llvm.i32, %arg1: !llvm.i32, %arg2: !llvm.vec<8 x i32>, %arg3: !llvm.vec<8 x i32>) {
   // CHECK: call i32 @llvm.smin.i32
   "llvm.intr.smin"(%arg0, %arg1) : (!llvm.i32, !llvm.i32) -> !llvm.i32
   // CHECK: call <8 x i32> @llvm.smin.v8i32
-  "llvm.intr.smin"(%arg2, %arg3) : (!llvm<"<8 x i32>">, !llvm<"<8 x i32>">) -> !llvm<"<8 x i32>">
+  "llvm.intr.smin"(%arg2, %arg3) : (!llvm.vec<8 x i32>, !llvm.vec<8 x i32>) -> !llvm.vec<8 x i32>
   llvm.return
 }
 
 // CHECK-LABEL: @vector_reductions
-llvm.func @vector_reductions(%arg0: !llvm.float, %arg1: !llvm<"<8 x float>">, %arg2: !llvm<"<8 x i32>">) {
+llvm.func @vector_reductions(%arg0: !llvm.float, %arg1: !llvm.vec<8 x float>, %arg2: !llvm.vec<8 x i32>) {
   // CHECK: call i32 @llvm.experimental.vector.reduce.add.v8i32
-  "llvm.intr.experimental.vector.reduce.add"(%arg2) : (!llvm<"<8 x i32>">) -> !llvm.i32
+  "llvm.intr.experimental.vector.reduce.add"(%arg2) : (!llvm.vec<8 x i32>) -> !llvm.i32
   // CHECK: call i32 @llvm.experimental.vector.reduce.and.v8i32
-  "llvm.intr.experimental.vector.reduce.and"(%arg2) : (!llvm<"<8 x i32>">) -> !llvm.i32
+  "llvm.intr.experimental.vector.reduce.and"(%arg2) : (!llvm.vec<8 x i32>) -> !llvm.i32
   // CHECK: call float @llvm.experimental.vector.reduce.fmax.v8f32
-  "llvm.intr.experimental.vector.reduce.fmax"(%arg1) : (!llvm<"<8 x float>">) -> !llvm.float
+  "llvm.intr.experimental.vector.reduce.fmax"(%arg1) : (!llvm.vec<8 x float>) -> !llvm.float
   // CHECK: call float @llvm.experimental.vector.reduce.fmin.v8f32
-  "llvm.intr.experimental.vector.reduce.fmin"(%arg1) : (!llvm<"<8 x float>">) -> !llvm.float
+  "llvm.intr.experimental.vector.reduce.fmin"(%arg1) : (!llvm.vec<8 x float>) -> !llvm.float
   // CHECK: call i32 @llvm.experimental.vector.reduce.mul.v8i32
-  "llvm.intr.experimental.vector.reduce.mul"(%arg2) : (!llvm<"<8 x i32>">) -> !llvm.i32
+  "llvm.intr.experimental.vector.reduce.mul"(%arg2) : (!llvm.vec<8 x i32>) -> !llvm.i32
   // CHECK: call i32 @llvm.experimental.vector.reduce.or.v8i32
-  "llvm.intr.experimental.vector.reduce.or"(%arg2) : (!llvm<"<8 x i32>">) -> !llvm.i32
+  "llvm.intr.experimental.vector.reduce.or"(%arg2) : (!llvm.vec<8 x i32>) -> !llvm.i32
   // CHECK: call i32 @llvm.experimental.vector.reduce.smax.v8i32
-  "llvm.intr.experimental.vector.reduce.smax"(%arg2) : (!llvm<"<8 x i32>">) -> !llvm.i32
+  "llvm.intr.experimental.vector.reduce.smax"(%arg2) : (!llvm.vec<8 x i32>) -> !llvm.i32
   // CHECK: call i32 @llvm.experimental.vector.reduce.smin.v8i32
-  "llvm.intr.experimental.vector.reduce.smin"(%arg2) : (!llvm<"<8 x i32>">) -> !llvm.i32
+  "llvm.intr.experimental.vector.reduce.smin"(%arg2) : (!llvm.vec<8 x i32>) -> !llvm.i32
   // CHECK: call i32 @llvm.experimental.vector.reduce.umax.v8i32
-  "llvm.intr.experimental.vector.reduce.umax"(%arg2) : (!llvm<"<8 x i32>">) -> !llvm.i32
+  "llvm.intr.experimental.vector.reduce.umax"(%arg2) : (!llvm.vec<8 x i32>) -> !llvm.i32
   // CHECK: call i32 @llvm.experimental.vector.reduce.umin.v8i32
-  "llvm.intr.experimental.vector.reduce.umin"(%arg2) : (!llvm<"<8 x i32>">) -> !llvm.i32
+  "llvm.intr.experimental.vector.reduce.umin"(%arg2) : (!llvm.vec<8 x i32>) -> !llvm.i32
   // CHECK: call float @llvm.experimental.vector.reduce.v2.fadd.f32.v8f32
-  "llvm.intr.experimental.vector.reduce.v2.fadd"(%arg0, %arg1) : (!llvm.float, !llvm<"<8 x float>">) -> !llvm.float
+  "llvm.intr.experimental.vector.reduce.v2.fadd"(%arg0, %arg1) : (!llvm.float, !llvm.vec<8 x float>) -> !llvm.float
   // CHECK: call float @llvm.experimental.vector.reduce.v2.fmul.f32.v8f32
-  "llvm.intr.experimental.vector.reduce.v2.fmul"(%arg0, %arg1) : (!llvm.float, !llvm<"<8 x float>">) -> !llvm.float
+  "llvm.intr.experimental.vector.reduce.v2.fmul"(%arg0, %arg1) : (!llvm.float, !llvm.vec<8 x float>) -> !llvm.float
   // CHECK: call reassoc float @llvm.experimental.vector.reduce.v2.fadd.f32.v8f32
-  "llvm.intr.experimental.vector.reduce.v2.fadd"(%arg0, %arg1) {reassoc = true} : (!llvm.float, !llvm<"<8 x float>">) -> !llvm.float
+  "llvm.intr.experimental.vector.reduce.v2.fadd"(%arg0, %arg1) {reassoc = true} : (!llvm.float, !llvm.vec<8 x float>) -> !llvm.float
   // CHECK: call reassoc float @llvm.experimental.vector.reduce.v2.fmul.f32.v8f32
-  "llvm.intr.experimental.vector.reduce.v2.fmul"(%arg0, %arg1) {reassoc = true} : (!llvm.float, !llvm<"<8 x float>">) -> !llvm.float
+  "llvm.intr.experimental.vector.reduce.v2.fmul"(%arg0, %arg1) {reassoc = true} : (!llvm.float, !llvm.vec<8 x float>) -> !llvm.float
   // CHECK: call i32 @llvm.experimental.vector.reduce.xor.v8i32
-  "llvm.intr.experimental.vector.reduce.xor"(%arg2) : (!llvm<"<8 x i32>">) -> !llvm.i32
+  "llvm.intr.experimental.vector.reduce.xor"(%arg2) : (!llvm.vec<8 x i32>) -> !llvm.i32
   llvm.return
 }
 
 // CHECK-LABEL: @matrix_intrinsics
 //                                       4x16                       16x3
-llvm.func @matrix_intrinsics(%A: !llvm<"<64 x float>">, %B: !llvm<"<48 x float>">,
-                             %ptr: !llvm<"float*">, %stride: !llvm.i64) {
+llvm.func @matrix_intrinsics(%A: !llvm.vec<64 x float>, %B: !llvm.vec<48 x float>,
+                             %ptr: !llvm.ptr<float>, %stride: !llvm.i64) {
   // CHECK: call <12 x float> @llvm.matrix.multiply.v12f32.v64f32.v48f32(<64 x float> %0, <48 x float> %1, i32 4, i32 16, i32 3)
   %C = llvm.intr.matrix.multiply %A, %B
     { lhs_rows = 4: i32, lhs_columns = 16: i32 , rhs_columns = 3: i32} :
-    (!llvm<"<64 x float>">, !llvm<"<48 x float>">) -> !llvm<"<12 x float>">
+    (!llvm.vec<64 x float>, !llvm.vec<48 x float>) -> !llvm.vec<12 x float>
   // CHECK: call <48 x float> @llvm.matrix.transpose.v48f32(<48 x float> %1, i32 3, i32 16)
   %D = llvm.intr.matrix.transpose %B { rows = 3: i32, columns = 16: i32} :
-    !llvm<"<48 x float>"> into !llvm<"<48 x float>">
+    !llvm.vec<48 x float> into !llvm.vec<48 x float>
   // CHECK: call <48 x float> @llvm.matrix.column.major.load.v48f32(float* align 4 %2, i64 %3, i1 false, i32 3, i32 16)
   %E = llvm.intr.matrix.column.major.load %ptr, <stride=%stride>
     { isVolatile = 0: i1, rows = 3: i32, columns = 16: i32} :
-    !llvm<"<48 x float>"> from !llvm<"float*"> stride !llvm.i64
+    !llvm.vec<48 x float> from !llvm.ptr<float> stride !llvm.i64
   // CHECK: call void @llvm.matrix.column.major.store.v48f32(<48 x float> %7, float* align 4 %2, i64 %3, i1 false, i32 3, i32 16)
   llvm.intr.matrix.column.major.store %E, %ptr, <stride=%stride>
     { isVolatile = 0: i1, rows = 3: i32, columns = 16: i32} :
-    !llvm<"<48 x float>"> to !llvm<"float*"> stride !llvm.i64
+    !llvm.vec<48 x float> to !llvm.ptr<float> stride !llvm.i64
   llvm.return
 }
 
 // CHECK-LABEL: @masked_intrinsics
-llvm.func @masked_intrinsics(%A: !llvm<"<7 x float>*">, %mask: !llvm<"<7 x i1>">) {
+llvm.func @masked_intrinsics(%A: !llvm.ptr<vec<7 x float>>, %mask: !llvm.vec<7 x i1>) {
   // CHECK: call <7 x float> @llvm.masked.load.v7f32.p0v7f32(<7 x float>* %{{.*}}, i32 1, <7 x i1> %{{.*}}, <7 x float> undef)
   %a = llvm.intr.masked.load %A, %mask { alignment = 1: i32} :
-    (!llvm<"<7 x float>*">, !llvm<"<7 x i1>">) -> !llvm<"<7 x float>">
+    (!llvm.ptr<vec<7 x float>>, !llvm.vec<7 x i1>) -> !llvm.vec<7 x float>
   // CHECK: call <7 x float> @llvm.masked.load.v7f32.p0v7f32(<7 x float>* %{{.*}}, i32 1, <7 x i1> %{{.*}}, <7 x float> %{{.*}})
   %b = llvm.intr.masked.load %A, %mask, %a { alignment = 1: i32} :
-    (!llvm<"<7 x float>*">, !llvm<"<7 x i1>">, !llvm<"<7 x float>">) -> !llvm<"<7 x float>">
+    (!llvm.ptr<vec<7 x float>>, !llvm.vec<7 x i1>, !llvm.vec<7 x float>) -> !llvm.vec<7 x float>
   // CHECK: call void @llvm.masked.store.v7f32.p0v7f32(<7 x float> %{{.*}}, <7 x float>* %0, i32 {{.*}}, <7 x i1> %{{.*}})
   llvm.intr.masked.store %b, %A, %mask { alignment = 1: i32} :
-    !llvm<"<7 x float>">, !llvm<"<7 x i1>"> into !llvm<"<7 x float>*">
+    !llvm.vec<7 x float>, !llvm.vec<7 x i1> into !llvm.ptr<vec<7 x float>>
   llvm.return
 }
 
 // CHECK-LABEL: @masked_gather_scatter_intrinsics
-llvm.func @masked_gather_scatter_intrinsics(%M: !llvm<"<7 x float*>">, %mask: !llvm<"<7 x i1>">) {
+llvm.func @masked_gather_scatter_intrinsics(%M: !llvm.vec<7 x ptr<float>>, %mask: !llvm.vec<7 x i1>) {
   // CHECK: call <7 x float> @llvm.masked.gather.v7f32.v7p0f32(<7 x float*> %{{.*}}, i32 1, <7 x i1> %{{.*}}, <7 x float> undef)
   %a = llvm.intr.masked.gather %M, %mask { alignment = 1: i32} :
-      (!llvm<"<7 x float*>">, !llvm<"<7 x i1>">) -> !llvm<"<7 x float>">
+      (!llvm.vec<7 x ptr<float>>, !llvm.vec<7 x i1>) -> !llvm.vec<7 x float>
   // CHECK: call <7 x float> @llvm.masked.gather.v7f32.v7p0f32(<7 x float*> %{{.*}}, i32 1, <7 x i1> %{{.*}}, <7 x float> %{{.*}})
   %b = llvm.intr.masked.gather %M, %mask, %a { alignment = 1: i32} :
-      (!llvm<"<7 x float*>">, !llvm<"<7 x i1>">, !llvm<"<7 x float>">) -> !llvm<"<7 x float>">
+      (!llvm.vec<7 x ptr<float>>, !llvm.vec<7 x i1>, !llvm.vec<7 x float>) -> !llvm.vec<7 x float>
   // CHECK: call void @llvm.masked.scatter.v7f32.v7p0f32(<7 x float> %{{.*}}, <7 x float*> %{{.*}}, i32 1, <7 x i1> %{{.*}})
   llvm.intr.masked.scatter %b, %M, %mask { alignment = 1: i32} :
-      !llvm<"<7 x float>">, !llvm<"<7 x i1>"> into !llvm<"<7 x float*>">
+      !llvm.vec<7 x float>, !llvm.vec<7 x i1> into !llvm.vec<7 x ptr<float>>
   llvm.return
 }
 
 // CHECK-LABEL: @memcpy_test
-llvm.func @memcpy_test(%arg0: !llvm.i32, %arg1: !llvm.i1, %arg2: !llvm<"i8*">, %arg3: !llvm<"i8*">) {
+llvm.func @memcpy_test(%arg0: !llvm.i32, %arg1: !llvm.i1, %arg2: !llvm.ptr<i8>, %arg3: !llvm.ptr<i8>) {
   // CHECK: call void @llvm.memcpy.p0i8.p0i8.i32(i8* %{{.*}}, i8* %{{.*}}, i32 %{{.*}}, i1 %{{.*}})
-  "llvm.intr.memcpy"(%arg2, %arg3, %arg0, %arg1) : (!llvm<"i8*">, !llvm<"i8*">, !llvm.i32, !llvm.i1) -> ()
+  "llvm.intr.memcpy"(%arg2, %arg3, %arg0, %arg1) : (!llvm.ptr<i8>, !llvm.ptr<i8>, !llvm.i32, !llvm.i1) -> ()
   %sz = llvm.mlir.constant(10: i64) : !llvm.i64
   // CHECK: call void @llvm.memcpy.inline.p0i8.p0i8.i64(i8* %{{.*}}, i8* %{{.*}}, i64 10, i1 %{{.*}})
-  "llvm.intr.memcpy.inline"(%arg2, %arg3, %sz, %arg1) : (!llvm<"i8*">, !llvm<"i8*">, !llvm.i64, !llvm.i1) -> ()
+  "llvm.intr.memcpy.inline"(%arg2, %arg3, %sz, %arg1) : (!llvm.ptr<i8>, !llvm.ptr<i8>, !llvm.i64, !llvm.i1) -> ()
   llvm.return
 }
 
diff --git a/mlir/test/Target/llvmir-invalid.mlir b/mlir/test/Target/llvmir-invalid.mlir
index 5effcdc60d374..14117594e2f89 100644
--- a/mlir/test/Target/llvmir-invalid.mlir
+++ b/mlir/test/Target/llvmir-invalid.mlir
@@ -21,16 +21,16 @@ llvm.func @invalid_align(%arg0 : !llvm.float {llvm.align = 4}) -> !llvm.float {
 
 // -----
 
-llvm.func @no_nested_struct() -> !llvm<"[2 x [2 x [2 x {i32}]]]"> {
+llvm.func @no_nested_struct() -> !llvm.array<2 x array<2 x array<2 x struct<(i32)>>>> {
   // expected-error @+1 {{struct types are not supported in constants}}
-  %0 = llvm.mlir.constant(dense<[[[1, 2], [3, 4]], [[42, 43], [44, 45]]]> : tensor<2x2x2xi32>) : !llvm<"[2 x [2 x [2 x {i32}]]]">
-  llvm.return %0 : !llvm<"[2 x [2 x [2 x {i32}]]]">
+  %0 = llvm.mlir.constant(dense<[[[1, 2], [3, 4]], [[42, 43], [44, 45]]]> : tensor<2x2x2xi32>) : !llvm.array<2 x array<2 x array<2 x struct<(i32)>>>>
+  llvm.return %0 : !llvm.array<2 x array<2 x array<2 x struct<(i32)>>>>
 }
 
 // -----
 
 // expected-error @+1 {{unsupported constant value}}
-llvm.mlir.global internal constant @test([2.5, 7.4]) : !llvm<"[2 x double]">
+llvm.mlir.global internal constant @test([2.5, 7.4]) : !llvm.array<2 x double>
 
 // -----
 
diff --git a/mlir/test/Target/llvmir.mlir b/mlir/test/Target/llvmir.mlir
index d6180cbf18494..5e57f1c7c6986 100644
--- a/mlir/test/Target/llvmir.mlir
+++ b/mlir/test/Target/llvmir.mlir
@@ -7,7 +7,7 @@ llvm.mlir.global internal @i32_global(42: i32) : !llvm.i32
 llvm.mlir.global internal constant @i32_const(52: i53) : !llvm.i53
 
 // CHECK: @int_global_array = internal global [3 x i32] [i32 62, i32 62, i32 62]
-llvm.mlir.global internal @int_global_array(dense<62> : vector<3xi32>) : !llvm<"[3 x i32]">
+llvm.mlir.global internal @int_global_array(dense<62> : vector<3xi32>) : !llvm.array<3 x i32>
 
 // CHECK: @i32_global_addr_space = internal addrspace(7) global i32 62
 llvm.mlir.global internal @i32_global_addr_space(62: i32) {addr_space = 7 : i32} : !llvm.i32
@@ -16,20 +16,20 @@ llvm.mlir.global internal @i32_global_addr_space(62: i32) {addr_space = 7 : i32}
 llvm.mlir.global internal @float_global(0.0: f32) : !llvm.float
 
 // CHECK: @float_global_array = internal global [1 x float] [float -5.000000e+00]
-llvm.mlir.global internal @float_global_array(dense<[-5.0]> : vector<1xf32>) : !llvm<"[1 x float]">
+llvm.mlir.global internal @float_global_array(dense<[-5.0]> : vector<1xf32>) : !llvm.array<1 x float>
 
 // CHECK: @string_const = internal constant [6 x i8] c"foobar"
-llvm.mlir.global internal constant @string_const("foobar") : !llvm<"[6 x i8]">
+llvm.mlir.global internal constant @string_const("foobar") : !llvm.array<6 x i8>
 
 // CHECK: @int_global_undef = internal global i64 undef
 llvm.mlir.global internal @int_global_undef() : !llvm.i64
 
 // CHECK: @int_gep = internal constant i32* getelementptr (i32, i32* @i32_global, i32 2)
-llvm.mlir.global internal constant @int_gep() : !llvm<"i32*"> {
-  %addr = llvm.mlir.addressof @i32_global : !llvm<"i32*">
+llvm.mlir.global internal constant @int_gep() : !llvm.ptr<i32> {
+  %addr = llvm.mlir.addressof @i32_global : !llvm.ptr<i32>
   %_c0 = llvm.mlir.constant(2: i32) :!llvm.i32
-  %gepinit = llvm.getelementptr %addr[%_c0] : (!llvm<"i32*">, !llvm.i32) -> !llvm<"i32*">
-  llvm.return %gepinit : !llvm<"i32*">
+  %gepinit = llvm.getelementptr %addr[%_c0] : (!llvm.ptr<i32>, !llvm.i32) -> !llvm.ptr<i32>
+  llvm.return %gepinit : !llvm.ptr<i32>
 }
 
 //
@@ -66,7 +66,7 @@ llvm.mlir.global external @external() : !llvm.i32
 //
 
 // CHECK: declare i8* @malloc(i64)
-llvm.func @malloc(!llvm.i64) -> !llvm<"i8*">
+llvm.func @malloc(!llvm.i64) -> !llvm.ptr<i8>
 // CHECK: declare void @free(i8*)
 
 
@@ -86,15 +86,15 @@ llvm.func @empty() {
 llvm.func @global_refs() {
   // Check load from globals.
   // CHECK: load i32, i32* @i32_global
-  %0 = llvm.mlir.addressof @i32_global : !llvm<"i32*">
-  %1 = llvm.load %0 : !llvm<"i32*">
+  %0 = llvm.mlir.addressof @i32_global : !llvm.ptr<i32>
+  %1 = llvm.load %0 : !llvm.ptr<i32>
 
   // Check the contracted form of load from array constants.
   // CHECK: load i8, i8* getelementptr inbounds ([6 x i8], [6 x i8]* @string_const, i64 0, i64 0)
-  %2 = llvm.mlir.addressof @string_const : !llvm<"[6 x i8]*">
+  %2 = llvm.mlir.addressof @string_const : !llvm.ptr<array<6 x i8>>
   %c0 = llvm.mlir.constant(0 : index) : !llvm.i64
-  %3 = llvm.getelementptr %2[%c0, %c0] : (!llvm<"[6 x i8]*">, !llvm.i64, !llvm.i64) -> !llvm<"i8*">
-  %4 = llvm.load %3 : !llvm<"i8*">
+  %3 = llvm.getelementptr %2[%c0, %c0] : (!llvm.ptr<array<6 x i8>>, !llvm.i64, !llvm.i64) -> !llvm.ptr<i8>
+  %4 = llvm.load %3 : !llvm.ptr<i8>
 
   llvm.return
 }
@@ -414,12 +414,12 @@ llvm.func @memref_alloc() {
   %0 = llvm.mlir.constant(10 : index) : !llvm.i64
   %1 = llvm.mlir.constant(10 : index) : !llvm.i64
   %2 = llvm.mul %0, %1 : !llvm.i64
-  %3 = llvm.mlir.undef : !llvm<"{ float* }">
+  %3 = llvm.mlir.undef : !llvm.struct<(ptr<float>)>
   %4 = llvm.mlir.constant(4 : index) : !llvm.i64
   %5 = llvm.mul %2, %4 : !llvm.i64
-  %6 = llvm.call @malloc(%5) : (!llvm.i64) -> !llvm<"i8*">
-  %7 = llvm.bitcast %6 : !llvm<"i8*"> to !llvm<"float*">
-  %8 = llvm.insertvalue %7, %3[0] : !llvm<"{ float* }">
+  %6 = llvm.call @malloc(%5) : (!llvm.i64) -> !llvm.ptr<i8>
+  %7 = llvm.bitcast %6 : !llvm.ptr<i8> to !llvm.ptr<float>
+  %8 = llvm.insertvalue %7, %3[0] : !llvm.struct<(ptr<float>)>
 // CHECK-NEXT: ret void
   llvm.return
 }
@@ -434,12 +434,12 @@ llvm.func @store_load_static() {
 // CHECK-NEXT: %{{[0-9]+}} = bitcast i8* %{{[0-9]+}} to float*
 // CHECK-NEXT: %{{[0-9]+}} = insertvalue { float* } undef, float* %{{[0-9]+}}, 0
   %0 = llvm.mlir.constant(10 : index) : !llvm.i64
-  %1 = llvm.mlir.undef : !llvm<"{ float* }">
+  %1 = llvm.mlir.undef : !llvm.struct<(ptr<float>)>
   %2 = llvm.mlir.constant(4 : index) : !llvm.i64
   %3 = llvm.mul %0, %2 : !llvm.i64
-  %4 = llvm.call @malloc(%3) : (!llvm.i64) -> !llvm<"i8*">
-  %5 = llvm.bitcast %4 : !llvm<"i8*"> to !llvm<"float*">
-  %6 = llvm.insertvalue %5, %1[0] : !llvm<"{ float* }">
+  %4 = llvm.call @malloc(%3) : (!llvm.i64) -> !llvm.ptr<i8>
+  %5 = llvm.bitcast %4 : !llvm.ptr<i8> to !llvm.ptr<float>
+  %6 = llvm.insertvalue %5, %1[0] : !llvm.struct<(ptr<float>)>
   %7 = llvm.mlir.constant(1.000000e+00 : f32) : !llvm.float
   llvm.br ^bb1
 ^bb1:   // pred: ^bb0
@@ -457,9 +457,9 @@ llvm.func @store_load_static() {
 // CHECK-NEXT: %{{[0-9]+}} = getelementptr float, float* %{{[0-9]+}}, i64 %{{[0-9]+}}
 // CHECK-NEXT: store float 1.000000e+00, float* %{{[0-9]+}}
   %12 = llvm.mlir.constant(10 : index) : !llvm.i64
-  %13 = llvm.extractvalue %6[0] : !llvm<"{ float* }">
-  %14 = llvm.getelementptr %13[%10] : (!llvm<"float*">, !llvm.i64) -> !llvm<"float*">
-  llvm.store %7, %14 : !llvm<"float*">
+  %13 = llvm.extractvalue %6[0] : !llvm.struct<(ptr<float>)>
+  %14 = llvm.getelementptr %13[%10] : (!llvm.ptr<float>, !llvm.i64) -> !llvm.ptr<float>
+  llvm.store %7, %14 : !llvm.ptr<float>
   %15 = llvm.mlir.constant(1 : index) : !llvm.i64
 // CHECK-NEXT: %{{[0-9]+}} = add i64 %{{[0-9]+}}, 1
   %16 = llvm.add %10, %15 : !llvm.i64
@@ -482,9 +482,9 @@ llvm.func @store_load_static() {
 // CHECK-NEXT: %{{[0-9]+}} = getelementptr float, float* %{{[0-9]+}}, i64 %{{[0-9]+}}
 // CHECK-NEXT: %{{[0-9]+}} = load float, float* %{{[0-9]+}}
   %21 = llvm.mlir.constant(10 : index) : !llvm.i64
-  %22 = llvm.extractvalue %6[0] : !llvm<"{ float* }">
-  %23 = llvm.getelementptr %22[%19] : (!llvm<"float*">, !llvm.i64) -> !llvm<"float*">
-  %24 = llvm.load %23 : !llvm<"float*">
+  %22 = llvm.extractvalue %6[0] : !llvm.struct<(ptr<float>)>
+  %23 = llvm.getelementptr %22[%19] : (!llvm.ptr<float>, !llvm.i64) -> !llvm.ptr<float>
+  %24 = llvm.load %23 : !llvm.ptr<float>
   %25 = llvm.mlir.constant(1 : index) : !llvm.i64
 // CHECK-NEXT: %{{[0-9]+}} = add i64 %{{[0-9]+}}, 1
   %26 = llvm.add %19, %25 : !llvm.i64
@@ -502,13 +502,13 @@ llvm.func @store_load_dynamic(%arg0: !llvm.i64) {
 // CHECK-NEXT: %{{[0-9]+}} = bitcast i8* %{{[0-9]+}} to float*
 // CHECK-NEXT: %{{[0-9]+}} = insertvalue { float*, i64 } undef, float* %{{[0-9]+}}, 0
 // CHECK-NEXT: %{{[0-9]+}} = insertvalue { float*, i64 } %{{[0-9]+}}, i64 %{{[0-9]+}}, 1
-  %0 = llvm.mlir.undef : !llvm<"{ float*, i64 }">
+  %0 = llvm.mlir.undef : !llvm.struct<(ptr<float>, i64)>
   %1 = llvm.mlir.constant(4 : index) : !llvm.i64
   %2 = llvm.mul %arg0, %1 : !llvm.i64
-  %3 = llvm.call @malloc(%2) : (!llvm.i64) -> !llvm<"i8*">
-  %4 = llvm.bitcast %3 : !llvm<"i8*"> to !llvm<"float*">
-  %5 = llvm.insertvalue %4, %0[0] : !llvm<"{ float*, i64 }">
-  %6 = llvm.insertvalue %arg0, %5[1] : !llvm<"{ float*, i64 }">
+  %3 = llvm.call @malloc(%2) : (!llvm.i64) -> !llvm.ptr<i8>
+  %4 = llvm.bitcast %3 : !llvm.ptr<i8> to !llvm.ptr<float>
+  %5 = llvm.insertvalue %4, %0[0] : !llvm.struct<(ptr<float>, i64)>
+  %6 = llvm.insertvalue %arg0, %5[1] : !llvm.struct<(ptr<float>, i64)>
   %7 = llvm.mlir.constant(1.000000e+00 : f32) : !llvm.float
 // CHECK-NEXT: br label %{{[0-9]+}}
   llvm.br ^bb1
@@ -526,10 +526,10 @@ llvm.func @store_load_dynamic(%arg0: !llvm.i64) {
 // CHECK-NEXT: %{{[0-9]+}} = extractvalue { float*, i64 } %{{[0-9]+}}, 0
 // CHECK-NEXT: %{{[0-9]+}} = getelementptr float, float* %{{[0-9]+}}, i64 %{{[0-9]+}}
 // CHECK-NEXT: store float 1.000000e+00, float* %{{[0-9]+}}
-  %11 = llvm.extractvalue %6[1] : !llvm<"{ float*, i64 }">
-  %12 = llvm.extractvalue %6[0] : !llvm<"{ float*, i64 }">
-  %13 = llvm.getelementptr %12[%9] : (!llvm<"float*">, !llvm.i64) -> !llvm<"float*">
-  llvm.store %7, %13 : !llvm<"float*">
+  %11 = llvm.extractvalue %6[1] : !llvm.struct<(ptr<float>, i64)>
+  %12 = llvm.extractvalue %6[0] : !llvm.struct<(ptr<float>, i64)>
+  %13 = llvm.getelementptr %12[%9] : (!llvm.ptr<float>, !llvm.i64) -> !llvm.ptr<float>
+  llvm.store %7, %13 : !llvm.ptr<float>
   %14 = llvm.mlir.constant(1 : index) : !llvm.i64
 // CHECK-NEXT: %{{[0-9]+}} = add i64 %{{[0-9]+}}, 1
   %15 = llvm.add %9, %14 : !llvm.i64
@@ -551,10 +551,10 @@ llvm.func @store_load_dynamic(%arg0: !llvm.i64) {
 // CHECK-NEXT: %{{[0-9]+}} = extractvalue { float*, i64 } %{{[0-9]+}}, 0
 // CHECK-NEXT: %{{[0-9]+}} = getelementptr float, float* %{{[0-9]+}}, i64 %{{[0-9]+}}
 // CHECK-NEXT: %{{[0-9]+}} = load float, float* %{{[0-9]+}}
-  %19 = llvm.extractvalue %6[1] : !llvm<"{ float*, i64 }">
-  %20 = llvm.extractvalue %6[0] : !llvm<"{ float*, i64 }">
-  %21 = llvm.getelementptr %20[%17] : (!llvm<"float*">, !llvm.i64) -> !llvm<"float*">
-  %22 = llvm.load %21 : !llvm<"float*">
+  %19 = llvm.extractvalue %6[1] : !llvm.struct<(ptr<float>, i64)>
+  %20 = llvm.extractvalue %6[0] : !llvm.struct<(ptr<float>, i64)>
+  %21 = llvm.getelementptr %20[%17] : (!llvm.ptr<float>, !llvm.i64) -> !llvm.ptr<float>
+  %22 = llvm.load %21 : !llvm.ptr<float>
   %23 = llvm.mlir.constant(1 : index) : !llvm.i64
 // CHECK-NEXT: %{{[0-9]+}} = add i64 %{{[0-9]+}}, 1
   %24 = llvm.add %17, %23 : !llvm.i64
@@ -582,14 +582,14 @@ llvm.func @store_load_mixed(%arg0: !llvm.i64) {
   %3 = llvm.mul %1, %arg0 : !llvm.i64
   %4 = llvm.mul %3, %2 : !llvm.i64
   %5 = llvm.mul %4, %0 : !llvm.i64
-  %6 = llvm.mlir.undef : !llvm<"{ float*, i64, i64 }">
+  %6 = llvm.mlir.undef : !llvm.struct<(ptr<float>, i64, i64)>
   %7 = llvm.mlir.constant(4 : index) : !llvm.i64
   %8 = llvm.mul %5, %7 : !llvm.i64
-  %9 = llvm.call @malloc(%8) : (!llvm.i64) -> !llvm<"i8*">
-  %10 = llvm.bitcast %9 : !llvm<"i8*"> to !llvm<"float*">
-  %11 = llvm.insertvalue %10, %6[0] : !llvm<"{ float*, i64, i64 }">
-  %12 = llvm.insertvalue %arg0, %11[1] : !llvm<"{ float*, i64, i64 }">
-  %13 = llvm.insertvalue %0, %12[2] : !llvm<"{ float*, i64, i64 }">
+  %9 = llvm.call @malloc(%8) : (!llvm.i64) -> !llvm.ptr<i8>
+  %10 = llvm.bitcast %9 : !llvm.ptr<i8> to !llvm.ptr<float>
+  %11 = llvm.insertvalue %10, %6[0] : !llvm.struct<(ptr<float>, i64, i64)>
+  %12 = llvm.insertvalue %arg0, %11[1] : !llvm.struct<(ptr<float>, i64, i64)>
+  %13 = llvm.insertvalue %0, %12[2] : !llvm.struct<(ptr<float>, i64, i64)>
 
 // CHECK-NEXT: %{{[0-9]+}} = call i64 @get_index()
 // CHECK-NEXT: %{{[0-9]+}} = call i64 @get_index()
@@ -610,18 +610,18 @@ llvm.func @store_load_mixed(%arg0: !llvm.i64) {
 // CHECK-NEXT: %{{[0-9]+}} = extractvalue { float*, i64, i64 } %{{[0-9]+}}, 0
 // CHECK-NEXT: %{{[0-9]+}} = getelementptr float, float* %{{[0-9]+}}, i64 %{{[0-9]+}}
 // CHECK-NEXT: store float 4.200000e+01, float* %{{[0-9]+}}
-  %20 = llvm.extractvalue %13[1] : !llvm<"{ float*, i64, i64 }">
+  %20 = llvm.extractvalue %13[1] : !llvm.struct<(ptr<float>, i64, i64)>
   %21 = llvm.mlir.constant(4 : index) : !llvm.i64
-  %22 = llvm.extractvalue %13[2] : !llvm<"{ float*, i64, i64 }">
+  %22 = llvm.extractvalue %13[2] : !llvm.struct<(ptr<float>, i64, i64)>
   %23 = llvm.mul %14, %20 : !llvm.i64
   %24 = llvm.add %23, %15 : !llvm.i64
   %25 = llvm.mul %24, %21 : !llvm.i64
   %26 = llvm.add %25, %16 : !llvm.i64
   %27 = llvm.mul %26, %22 : !llvm.i64
   %28 = llvm.add %27, %17 : !llvm.i64
-  %29 = llvm.extractvalue %13[0] : !llvm<"{ float*, i64, i64 }">
-  %30 = llvm.getelementptr %29[%28] : (!llvm<"float*">, !llvm.i64) -> !llvm<"float*">
-  llvm.store %18, %30 : !llvm<"float*">
+  %29 = llvm.extractvalue %13[0] : !llvm.struct<(ptr<float>, i64, i64)>
+  %30 = llvm.getelementptr %29[%28] : (!llvm.ptr<float>, !llvm.i64) -> !llvm.ptr<float>
+  llvm.store %18, %30 : !llvm.ptr<float>
 // CHECK-NEXT: %{{[0-9]+}} = extractvalue { float*, i64, i64 } %{{[0-9]+}}, 1
 // CHECK-NEXT: %{{[0-9]+}} = extractvalue { float*, i64, i64 } %{{[0-9]+}}, 2
 // CHECK-NEXT: %{{[0-9]+}} = mul i64 %{{[0-9]+}}, %{{[0-9]+}}
@@ -634,24 +634,24 @@ llvm.func @store_load_mixed(%arg0: !llvm.i64) {
 // CHECK-NEXT: %{{[0-9]+}} = getelementptr float, float* %{{[0-9]+}}, i64 %{{[0-9]+}}
 // CHECK-NEXT: %{{[0-9]+}} = load float, float* %{{[0-9]+}}
   %31 = llvm.mlir.constant(2 : index) : !llvm.i64
-  %32 = llvm.extractvalue %13[1] : !llvm<"{ float*, i64, i64 }">
+  %32 = llvm.extractvalue %13[1] : !llvm.struct<(ptr<float>, i64, i64)>
   %33 = llvm.mlir.constant(4 : index) : !llvm.i64
-  %34 = llvm.extractvalue %13[2] : !llvm<"{ float*, i64, i64 }">
+  %34 = llvm.extractvalue %13[2] : !llvm.struct<(ptr<float>, i64, i64)>
   %35 = llvm.mul %17, %32 : !llvm.i64
   %36 = llvm.add %35, %16 : !llvm.i64
   %37 = llvm.mul %36, %33 : !llvm.i64
   %38 = llvm.add %37, %15 : !llvm.i64
   %39 = llvm.mul %38, %34 : !llvm.i64
   %40 = llvm.add %39, %14 : !llvm.i64
-  %41 = llvm.extractvalue %13[0] : !llvm<"{ float*, i64, i64 }">
-  %42 = llvm.getelementptr %41[%40] : (!llvm<"float*">, !llvm.i64) -> !llvm<"float*">
-  %43 = llvm.load %42 : !llvm<"float*">
+  %41 = llvm.extractvalue %13[0] : !llvm.struct<(ptr<float>, i64, i64)>
+  %42 = llvm.getelementptr %41[%40] : (!llvm.ptr<float>, !llvm.i64) -> !llvm.ptr<float>
+  %43 = llvm.load %42 : !llvm.ptr<float>
 // CHECK-NEXT: ret void
   llvm.return
 }
 
 // CHECK-LABEL: define { float*, i64 } @memref_args_rets({ float* } {{%.*}}, { float*, i64 } {{%.*}}, { float*, i64 } {{%.*}})
-llvm.func @memref_args_rets(%arg0: !llvm<"{ float* }">, %arg1: !llvm<"{ float*, i64 }">, %arg2: !llvm<"{ float*, i64 }">) -> !llvm<"{ float*, i64 }"> {
+llvm.func @memref_args_rets(%arg0: !llvm.struct<(ptr<float>)>, %arg1: !llvm.struct<(ptr<float>, i64)>, %arg2: !llvm.struct<(ptr<float>, i64)>) -> !llvm.struct<(ptr<float>, i64)> {
   %0 = llvm.mlir.constant(7 : index) : !llvm.i64
 // CHECK-NEXT: %{{[0-9]+}} = call i64 @get_index()
   %1 = llvm.call @get_index() : () -> !llvm.i64
@@ -660,17 +660,17 @@ llvm.func @memref_args_rets(%arg0: !llvm<"{ float* }">, %arg1: !llvm<"{ float*,
 // CHECK-NEXT: %{{[0-9]+}} = getelementptr float, float* %{{[0-9]+}}, i64 7
 // CHECK-NEXT: store float 4.200000e+01, float* %{{[0-9]+}}
   %3 = llvm.mlir.constant(10 : index) : !llvm.i64
-  %4 = llvm.extractvalue %arg0[0] : !llvm<"{ float* }">
-  %5 = llvm.getelementptr %4[%0] : (!llvm<"float*">, !llvm.i64) -> !llvm<"float*">
-  llvm.store %2, %5 : !llvm<"float*">
+  %4 = llvm.extractvalue %arg0[0] : !llvm.struct<(ptr<float>)>
+  %5 = llvm.getelementptr %4[%0] : (!llvm.ptr<float>, !llvm.i64) -> !llvm.ptr<float>
+  llvm.store %2, %5 : !llvm.ptr<float>
 // CHECK-NEXT: %{{[0-9]+}} = extractvalue { float*, i64 } %{{[0-9]+}}, 1
 // CHECK-NEXT: %{{[0-9]+}} = extractvalue { float*, i64 } %{{[0-9]+}}, 0
 // CHECK-NEXT: %{{[0-9]+}} = getelementptr float, float* %{{[0-9]+}}, i64 7
 // CHECK-NEXT: store float 4.200000e+01, float* %{{[0-9]+}}
-  %6 = llvm.extractvalue %arg1[1] : !llvm<"{ float*, i64 }">
-  %7 = llvm.extractvalue %arg1[0] : !llvm<"{ float*, i64 }">
-  %8 = llvm.getelementptr %7[%0] : (!llvm<"float*">, !llvm.i64) -> !llvm<"float*">
-  llvm.store %2, %8 : !llvm<"float*">
+  %6 = llvm.extractvalue %arg1[1] : !llvm.struct<(ptr<float>, i64)>
+  %7 = llvm.extractvalue %arg1[0] : !llvm.struct<(ptr<float>, i64)>
+  %8 = llvm.getelementptr %7[%0] : (!llvm.ptr<float>, !llvm.i64) -> !llvm.ptr<float>
+  llvm.store %2, %8 : !llvm.ptr<float>
 // CHECK-NEXT: %{{[0-9]+}} = extractvalue { float*, i64 } %{{[0-9]+}}, 1
 // CHECK-NEXT: %{{[0-9]+}} = mul i64 7, %{{[0-9]+}}
 // CHECK-NEXT: %{{[0-9]+}} = add i64 %{{[0-9]+}}, %{{[0-9]+}}
@@ -678,12 +678,12 @@ llvm.func @memref_args_rets(%arg0: !llvm<"{ float* }">, %arg1: !llvm<"{ float*,
 // CHECK-NEXT: %{{[0-9]+}} = getelementptr float, float* %{{[0-9]+}}, i64 %{{[0-9]+}}
 // CHECK-NEXT: store float 4.200000e+01, float* %{{[0-9]+}}
   %9 = llvm.mlir.constant(10 : index) : !llvm.i64
-  %10 = llvm.extractvalue %arg2[1] : !llvm<"{ float*, i64 }">
+  %10 = llvm.extractvalue %arg2[1] : !llvm.struct<(ptr<float>, i64)>
   %11 = llvm.mul %0, %10 : !llvm.i64
   %12 = llvm.add %11, %1 : !llvm.i64
-  %13 = llvm.extractvalue %arg2[0] : !llvm<"{ float*, i64 }">
-  %14 = llvm.getelementptr %13[%12] : (!llvm<"float*">, !llvm.i64) -> !llvm<"float*">
-  llvm.store %2, %14 : !llvm<"float*">
+  %13 = llvm.extractvalue %arg2[0] : !llvm.struct<(ptr<float>, i64)>
+  %14 = llvm.getelementptr %13[%12] : (!llvm.ptr<float>, !llvm.i64) -> !llvm.ptr<float>
+  llvm.store %2, %14 : !llvm.ptr<float>
 // CHECK-NEXT: %{{[0-9]+}} = mul i64 10, %{{[0-9]+}}
 // CHECK-NEXT: %{{[0-9]+}} = mul i64 %{{[0-9]+}}, 4
 // CHECK-NEXT: %{{[0-9]+}} = call i8* @malloc(i64 %{{[0-9]+}})
@@ -692,28 +692,28 @@ llvm.func @memref_args_rets(%arg0: !llvm<"{ float* }">, %arg1: !llvm<"{ float*,
 // CHECK-NEXT: %{{[0-9]+}} = insertvalue { float*, i64 } %{{[0-9]+}}, i64 %{{[0-9]+}}, 1
   %15 = llvm.mlir.constant(10 : index) : !llvm.i64
   %16 = llvm.mul %15, %1 : !llvm.i64
-  %17 = llvm.mlir.undef : !llvm<"{ float*, i64 }">
+  %17 = llvm.mlir.undef : !llvm.struct<(ptr<float>, i64)>
   %18 = llvm.mlir.constant(4 : index) : !llvm.i64
   %19 = llvm.mul %16, %18 : !llvm.i64
-  %20 = llvm.call @malloc(%19) : (!llvm.i64) -> !llvm<"i8*">
-  %21 = llvm.bitcast %20 : !llvm<"i8*"> to !llvm<"float*">
-  %22 = llvm.insertvalue %21, %17[0] : !llvm<"{ float*, i64 }">
-  %23 = llvm.insertvalue %1, %22[1] : !llvm<"{ float*, i64 }">
+  %20 = llvm.call @malloc(%19) : (!llvm.i64) -> !llvm.ptr<i8>
+  %21 = llvm.bitcast %20 : !llvm.ptr<i8> to !llvm.ptr<float>
+  %22 = llvm.insertvalue %21, %17[0] : !llvm.struct<(ptr<float>, i64)>
+  %23 = llvm.insertvalue %1, %22[1] : !llvm.struct<(ptr<float>, i64)>
 // CHECK-NEXT: ret { float*, i64 } %{{[0-9]+}}
-  llvm.return %23 : !llvm<"{ float*, i64 }">
+  llvm.return %23 : !llvm.struct<(ptr<float>, i64)>
 }
 
 
 // CHECK-LABEL: define i64 @memref_dim({ float*, i64, i64 } {{%.*}})
-llvm.func @memref_dim(%arg0: !llvm<"{ float*, i64, i64 }">) -> !llvm.i64 {
+llvm.func @memref_dim(%arg0: !llvm.struct<(ptr<float>, i64, i64)>) -> !llvm.i64 {
 // Expecting this to create an LLVM constant.
   %0 = llvm.mlir.constant(42 : index) : !llvm.i64
 // CHECK-NEXT: %2 = extractvalue { float*, i64, i64 } %0, 1
-  %1 = llvm.extractvalue %arg0[1] : !llvm<"{ float*, i64, i64 }">
+  %1 = llvm.extractvalue %arg0[1] : !llvm.struct<(ptr<float>, i64, i64)>
 // Expecting this to create an LLVM constant.
   %2 = llvm.mlir.constant(10 : index) : !llvm.i64
 // CHECK-NEXT: %3 = extractvalue { float*, i64, i64 } %0, 2
-  %3 = llvm.extractvalue %arg0[2] : !llvm<"{ float*, i64, i64 }">
+  %3 = llvm.extractvalue %arg0[2] : !llvm.struct<(ptr<float>, i64, i64)>
 // Checking that the constant for d0 has been created.
 // CHECK-NEXT: %4 = add i64 42, %2
   %4 = llvm.add %0, %1 : !llvm.i64
@@ -728,22 +728,22 @@ llvm.func @memref_dim(%arg0: !llvm<"{ float*, i64, i64 }">) -> !llvm.i64 {
 
 llvm.func @get_i64() -> !llvm.i64
 llvm.func @get_f32() -> !llvm.float
-llvm.func @get_memref() -> !llvm<"{ float*, i64, i64 }">
+llvm.func @get_memref() -> !llvm.struct<(ptr<float>, i64, i64)>
 
 // CHECK-LABEL: define { i64, float, { float*, i64, i64 } } @multireturn()
-llvm.func @multireturn() -> !llvm<"{ i64, float, { float*, i64, i64 } }"> {
+llvm.func @multireturn() -> !llvm.struct<(i64, float, struct<(ptr<float>, i64, i64)>)> {
   %0 = llvm.call @get_i64() : () -> !llvm.i64
   %1 = llvm.call @get_f32() : () -> !llvm.float
-  %2 = llvm.call @get_memref() : () -> !llvm<"{ float*, i64, i64 }">
+  %2 = llvm.call @get_memref() : () -> !llvm.struct<(ptr<float>, i64, i64)>
 // CHECK:        %{{[0-9]+}} = insertvalue { i64, float, { float*, i64, i64 } } undef, i64 %{{[0-9]+}}, 0
 // CHECK-NEXT:   %{{[0-9]+}} = insertvalue { i64, float, { float*, i64, i64 } } %{{[0-9]+}}, float %{{[0-9]+}}, 1
 // CHECK-NEXT:   %{{[0-9]+}} = insertvalue { i64, float, { float*, i64, i64 } } %{{[0-9]+}}, { float*, i64, i64 } %{{[0-9]+}}, 2
 // CHECK-NEXT:   ret { i64, float, { float*, i64, i64 } } %{{[0-9]+}}
-  %3 = llvm.mlir.undef : !llvm<"{ i64, float, { float*, i64, i64 } }">
-  %4 = llvm.insertvalue %0, %3[0] : !llvm<"{ i64, float, { float*, i64, i64 } }">
-  %5 = llvm.insertvalue %1, %4[1] : !llvm<"{ i64, float, { float*, i64, i64 } }">
-  %6 = llvm.insertvalue %2, %5[2] : !llvm<"{ i64, float, { float*, i64, i64 } }">
-  llvm.return %6 : !llvm<"{ i64, float, { float*, i64, i64 } }">
+  %3 = llvm.mlir.undef : !llvm.struct<(i64, float, struct<(ptr<float>, i64, i64)>)>
+  %4 = llvm.insertvalue %0, %3[0] : !llvm.struct<(i64, float, struct<(ptr<float>, i64, i64)>)>
+  %5 = llvm.insertvalue %1, %4[1] : !llvm.struct<(i64, float, struct<(ptr<float>, i64, i64)>)>
+  %6 = llvm.insertvalue %2, %5[2] : !llvm.struct<(i64, float, struct<(ptr<float>, i64, i64)>)>
+  llvm.return %6 : !llvm.struct<(i64, float, struct<(ptr<float>, i64, i64)>)>
 }
 
 
@@ -753,10 +753,10 @@ llvm.func @multireturn_caller() {
 // CHECK-NEXT:   [[ret0:%[0-9]+]] = extractvalue { i64, float, { float*, i64, i64 } } %1, 0
 // CHECK-NEXT:   [[ret1:%[0-9]+]] = extractvalue { i64, float, { float*, i64, i64 } } %1, 1
 // CHECK-NEXT:   [[ret2:%[0-9]+]] = extractvalue { i64, float, { float*, i64, i64 } } %1, 2
-  %0 = llvm.call @multireturn() : () -> !llvm<"{ i64, float, { float*, i64, i64 } }">
-  %1 = llvm.extractvalue %0[0] : !llvm<"{ i64, float, { float*, i64, i64 } }">
-  %2 = llvm.extractvalue %0[1] : !llvm<"{ i64, float, { float*, i64, i64 } }">
-  %3 = llvm.extractvalue %0[2] : !llvm<"{ i64, float, { float*, i64, i64 } }">
+  %0 = llvm.call @multireturn() : () -> !llvm.struct<(i64, float, struct<(ptr<float>, i64, i64)>)>
+  %1 = llvm.extractvalue %0[0] : !llvm.struct<(i64, float, struct<(ptr<float>, i64, i64)>)>
+  %2 = llvm.extractvalue %0[1] : !llvm.struct<(i64, float, struct<(ptr<float>, i64, i64)>)>
+  %3 = llvm.extractvalue %0[2] : !llvm.struct<(i64, float, struct<(ptr<float>, i64, i64)>)>
   %4 = llvm.mlir.constant(42) : !llvm.i64
 // CHECK:   add i64 [[ret0]], 42
   %5 = llvm.add %1, %4 : !llvm.i64
@@ -766,86 +766,86 @@ llvm.func @multireturn_caller() {
   %8 = llvm.mlir.constant(0 : index) : !llvm.i64
   %9 = llvm.mlir.constant(42 : index) : !llvm.i64
 // CHECK:   extractvalue { float*, i64, i64 } [[ret2]], 0
-  %10 = llvm.extractvalue %3[1] : !llvm<"{ float*, i64, i64 }">
+  %10 = llvm.extractvalue %3[1] : !llvm.struct<(ptr<float>, i64, i64)>
   %11 = llvm.mlir.constant(10 : index) : !llvm.i64
-  %12 = llvm.extractvalue %3[2] : !llvm<"{ float*, i64, i64 }">
+  %12 = llvm.extractvalue %3[2] : !llvm.struct<(ptr<float>, i64, i64)>
   %13 = llvm.mul %8, %10 : !llvm.i64
   %14 = llvm.add %13, %8 : !llvm.i64
   %15 = llvm.mul %14, %11 : !llvm.i64
   %16 = llvm.add %15, %8 : !llvm.i64
   %17 = llvm.mul %16, %12 : !llvm.i64
   %18 = llvm.add %17, %8 : !llvm.i64
-  %19 = llvm.extractvalue %3[0] : !llvm<"{ float*, i64, i64 }">
-  %20 = llvm.getelementptr %19[%18] : (!llvm<"float*">, !llvm.i64) -> !llvm<"float*">
-  %21 = llvm.load %20 : !llvm<"float*">
+  %19 = llvm.extractvalue %3[0] : !llvm.struct<(ptr<float>, i64, i64)>
+  %20 = llvm.getelementptr %19[%18] : (!llvm.ptr<float>, !llvm.i64) -> !llvm.ptr<float>
+  %21 = llvm.load %20 : !llvm.ptr<float>
   llvm.return
 }
 
 // CHECK-LABEL: define <4 x float> @vector_ops(<4 x float> {{%.*}}, <4 x i1> {{%.*}}, <4 x i64> {{%.*}})
-llvm.func @vector_ops(%arg0: !llvm<"<4 x float>">, %arg1: !llvm<"<4 x i1>">, %arg2: !llvm<"<4 x i64>">) -> !llvm<"<4 x float>"> {
-  %0 = llvm.mlir.constant(dense<4.200000e+01> : vector<4xf32>) : !llvm<"<4 x float>">
+llvm.func @vector_ops(%arg0: !llvm.vec<4 x float>, %arg1: !llvm.vec<4 x i1>, %arg2: !llvm.vec<4 x i64>) -> !llvm.vec<4 x float> {
+  %0 = llvm.mlir.constant(dense<4.200000e+01> : vector<4xf32>) : !llvm.vec<4 x float>
 // CHECK-NEXT: %4 = fadd <4 x float> %0, <float 4.200000e+01, float 4.200000e+01, float 4.200000e+01, float 4.200000e+01>
-  %1 = llvm.fadd %arg0, %0 : !llvm<"<4 x float>">
+  %1 = llvm.fadd %arg0, %0 : !llvm.vec<4 x float>
 // CHECK-NEXT: %5 = select <4 x i1> %1, <4 x float> %4, <4 x float> %0
-  %2 = llvm.select %arg1, %1, %arg0 : !llvm<"<4 x i1>">, !llvm<"<4 x float>">
+  %2 = llvm.select %arg1, %1, %arg0 : !llvm.vec<4 x i1>, !llvm.vec<4 x float>
 // CHECK-NEXT: %6 = sdiv <4 x i64> %2, %2
-  %3 = llvm.sdiv %arg2, %arg2 : !llvm<"<4 x i64>">
+  %3 = llvm.sdiv %arg2, %arg2 : !llvm.vec<4 x i64>
 // CHECK-NEXT: %7 = udiv <4 x i64> %2, %2
-  %4 = llvm.udiv %arg2, %arg2 : !llvm<"<4 x i64>">
+  %4 = llvm.udiv %arg2, %arg2 : !llvm.vec<4 x i64>
 // CHECK-NEXT: %8 = srem <4 x i64> %2, %2
-  %5 = llvm.srem %arg2, %arg2 : !llvm<"<4 x i64>">
+  %5 = llvm.srem %arg2, %arg2 : !llvm.vec<4 x i64>
 // CHECK-NEXT: %9 = urem <4 x i64> %2, %2
-  %6 = llvm.urem %arg2, %arg2 : !llvm<"<4 x i64>">
+  %6 = llvm.urem %arg2, %arg2 : !llvm.vec<4 x i64>
 // CHECK-NEXT: %10 = fdiv <4 x float> %0, <float 4.200000e+01, float 4.200000e+01, float 4.200000e+01, float 4.200000e+01>
-  %7 = llvm.fdiv %arg0, %0 : !llvm<"<4 x float>">
+  %7 = llvm.fdiv %arg0, %0 : !llvm.vec<4 x float>
 // CHECK-NEXT: %11 = frem <4 x float> %0, <float 4.200000e+01, float 4.200000e+01, float 4.200000e+01, float 4.200000e+01>
-  %8 = llvm.frem %arg0, %0 : !llvm<"<4 x float>">
+  %8 = llvm.frem %arg0, %0 : !llvm.vec<4 x float>
 // CHECK-NEXT: %12 = and <4 x i64> %2, %2
-  %9 = llvm.and %arg2, %arg2 : !llvm<"<4 x i64>">
+  %9 = llvm.and %arg2, %arg2 : !llvm.vec<4 x i64>
 // CHECK-NEXT: %13 = or <4 x i64> %2, %2
-  %10 = llvm.or %arg2, %arg2 : !llvm<"<4 x i64>">
+  %10 = llvm.or %arg2, %arg2 : !llvm.vec<4 x i64>
 // CHECK-NEXT: %14 = xor <4 x i64> %2, %2
-  %11 = llvm.xor %arg2, %arg2 : !llvm<"<4 x i64>">
+  %11 = llvm.xor %arg2, %arg2 : !llvm.vec<4 x i64>
 // CHECK-NEXT: %15 = shl <4 x i64> %2, %2
-  %12 = llvm.shl %arg2, %arg2 : !llvm<"<4 x i64>">
+  %12 = llvm.shl %arg2, %arg2 : !llvm.vec<4 x i64>
 // CHECK-NEXT: %16 = lshr <4 x i64> %2, %2
-  %13 = llvm.lshr %arg2, %arg2 : !llvm<"<4 x i64>">
+  %13 = llvm.lshr %arg2, %arg2 : !llvm.vec<4 x i64>
 // CHECK-NEXT: %17 = ashr <4 x i64> %2, %2
-  %14 = llvm.ashr %arg2, %arg2 : !llvm<"<4 x i64>">
+  %14 = llvm.ashr %arg2, %arg2 : !llvm.vec<4 x i64>
 // CHECK-NEXT:    ret <4 x float> %4
-  llvm.return %1 : !llvm<"<4 x float>">
+  llvm.return %1 : !llvm.vec<4 x float>
 }
 
 // CHECK-LABEL: @vector_splat_1d
-llvm.func @vector_splat_1d() -> !llvm<"<4 x float>"> {
+llvm.func @vector_splat_1d() -> !llvm.vec<4 x float> {
   // CHECK: ret <4 x float> zeroinitializer
-  %0 = llvm.mlir.constant(dense<0.000000e+00> : vector<4xf32>) : !llvm<"<4 x float>">
-  llvm.return %0 : !llvm<"<4 x float>">
+  %0 = llvm.mlir.constant(dense<0.000000e+00> : vector<4xf32>) : !llvm.vec<4 x float>
+  llvm.return %0 : !llvm.vec<4 x float>
 }
 
 // CHECK-LABEL: @vector_splat_2d
-llvm.func @vector_splat_2d() -> !llvm<"[4 x <16 x float>]"> {
+llvm.func @vector_splat_2d() -> !llvm.array<4 x vec<16 x float>> {
   // CHECK: ret [4 x <16 x float>] zeroinitializer
-  %0 = llvm.mlir.constant(dense<0.000000e+00> : vector<4x16xf32>) : !llvm<"[4 x <16 x float>]">
-  llvm.return %0 : !llvm<"[4 x <16 x float>]">
+  %0 = llvm.mlir.constant(dense<0.000000e+00> : vector<4x16xf32>) : !llvm.array<4 x vec<16 x float>>
+  llvm.return %0 : !llvm.array<4 x vec<16 x float>>
 }
 
 // CHECK-LABEL: @vector_splat_3d
-llvm.func @vector_splat_3d() -> !llvm<"[4 x [16 x <4 x float>]]"> {
+llvm.func @vector_splat_3d() -> !llvm.array<4 x array<16 x vec<4 x float>>> {
   // CHECK: ret [4 x [16 x <4 x float>]] zeroinitializer
-  %0 = llvm.mlir.constant(dense<0.000000e+00> : vector<4x16x4xf32>) : !llvm<"[4 x [16 x <4 x float>]]">
-  llvm.return %0 : !llvm<"[4 x [16 x <4 x float>]]">
+  %0 = llvm.mlir.constant(dense<0.000000e+00> : vector<4x16x4xf32>) : !llvm.array<4 x array<16 x vec<4 x float>>>
+  llvm.return %0 : !llvm.array<4 x array<16 x vec<4 x float>>>
 }
 
 // CHECK-LABEL: @vector_splat_nonzero
-llvm.func @vector_splat_nonzero() -> !llvm<"<4 x float>"> {
+llvm.func @vector_splat_nonzero() -> !llvm.vec<4 x float> {
   // CHECK: ret <4 x float> <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>
-  %0 = llvm.mlir.constant(dense<1.000000e+00> : vector<4xf32>) : !llvm<"<4 x float>">
-  llvm.return %0 : !llvm<"<4 x float>">
+  %0 = llvm.mlir.constant(dense<1.000000e+00> : vector<4xf32>) : !llvm.vec<4 x float>
+  llvm.return %0 : !llvm.vec<4 x float>
 }
 
 // CHECK-LABEL: @ops
-llvm.func @ops(%arg0: !llvm.float, %arg1: !llvm.float, %arg2: !llvm.i32, %arg3: !llvm.i32) -> !llvm<"{ float, i32 }"> {
+llvm.func @ops(%arg0: !llvm.float, %arg1: !llvm.float, %arg2: !llvm.i32, %arg3: !llvm.i32) -> !llvm.struct<(float, i32)> {
 // CHECK-NEXT: fsub float %0, %1
   %0 = llvm.fsub %arg0, %arg1 : !llvm.float
 // CHECK-NEXT: %6 = sub i32 %2, %3
@@ -863,9 +863,9 @@ llvm.func @ops(%arg0: !llvm.float, %arg1: !llvm.float, %arg2: !llvm.i32, %arg3:
 // CHECK-NEXT: %12 = urem i32 %2, %3
   %7 = llvm.urem %arg2, %arg3 : !llvm.i32
 
-  %8 = llvm.mlir.undef : !llvm<"{ float, i32 }">
-  %9 = llvm.insertvalue %0, %8[0] : !llvm<"{ float, i32 }">
-  %10 = llvm.insertvalue %3, %9[1] : !llvm<"{ float, i32 }">
+  %8 = llvm.mlir.undef : !llvm.struct<(float, i32)>
+  %9 = llvm.insertvalue %0, %8[0] : !llvm.struct<(float, i32)>
+  %10 = llvm.insertvalue %3, %9[1] : !llvm.struct<(float, i32)>
 
 // CHECK: %15 = fdiv float %0, %1
   %11 = llvm.fdiv %arg0, %arg1 : !llvm.float
@@ -888,7 +888,7 @@ llvm.func @ops(%arg0: !llvm.float, %arg1: !llvm.float, %arg2: !llvm.i32, %arg3:
 // CHECK-NEXT: fneg float %0
   %19 = llvm.fneg %arg0 : !llvm.float
 
-  llvm.return %10 : !llvm<"{ float, i32 }">
+  llvm.return %10 : !llvm.struct<(float, i32)>
 }
 
 //
@@ -898,14 +898,14 @@ llvm.func @ops(%arg0: !llvm.float, %arg1: !llvm.float, %arg2: !llvm.i32, %arg3:
 // CHECK-LABEL: define void @indirect_const_call(i64 {{%.*}})
 llvm.func @indirect_const_call(%arg0: !llvm.i64) {
 // CHECK-NEXT:  call void @body(i64 %0)
-  %0 = llvm.mlir.addressof @body : !llvm<"void (i64)*">
+  %0 = llvm.mlir.addressof @body : !llvm.ptr<func<void (i64)>>
   llvm.call %0(%arg0) : (!llvm.i64) -> ()
 // CHECK-NEXT:  ret void
   llvm.return
 }
 
 // CHECK-LABEL: define i32 @indirect_call(i32 (float)* {{%.*}}, float {{%.*}})
-llvm.func @indirect_call(%arg0: !llvm<"i32 (float)*">, %arg1: !llvm.float) -> !llvm.i32 {
+llvm.func @indirect_call(%arg0: !llvm.ptr<func<i32 (float)>>, %arg1: !llvm.float) -> !llvm.i32 {
 // CHECK-NEXT:  %3 = call i32 %0(float %1)
   %0 = llvm.call %arg0(%arg1) : (!llvm.float) -> !llvm.i32
 // CHECK-NEXT:  ret i32 %3
@@ -935,12 +935,12 @@ llvm.func @cond_br_arguments(%arg0: !llvm.i1, %arg1: !llvm.i1) {
 }
 
 // CHECK-LABEL: define void @llvm_noalias(float* noalias {{%*.}})
-llvm.func @llvm_noalias(%arg0: !llvm<"float*"> {llvm.noalias = true}) {
+llvm.func @llvm_noalias(%arg0: !llvm.ptr<float> {llvm.noalias = true}) {
   llvm.return
 }
 
 // CHECK-LABEL: define void @llvm_align(float* align 4 {{%*.}})
-llvm.func @llvm_align(%arg0: !llvm<"float*"> {llvm.align = 4}) {
+llvm.func @llvm_align(%arg0: !llvm.ptr<float> {llvm.align = 4}) {
   llvm.return
 }
 
@@ -950,8 +950,8 @@ llvm.func @llvm_varargs(...)
 llvm.func @intpointerconversion(%arg0 : !llvm.i32) -> !llvm.i32 {
 // CHECK:      %2 = inttoptr i32 %0 to i32*
 // CHECK-NEXT: %3 = ptrtoint i32* %2 to i32
-  %1 = llvm.inttoptr %arg0 : !llvm.i32 to !llvm<"i32*">
-  %2 = llvm.ptrtoint %1 : !llvm<"i32*"> to !llvm.i32
+  %1 = llvm.inttoptr %arg0 : !llvm.i32 to !llvm.ptr<i32>
+  %2 = llvm.ptrtoint %1 : !llvm.ptr<i32> to !llvm.i32
   llvm.return %2 : !llvm.i32
 }
 
@@ -968,16 +968,16 @@ llvm.func @fpconversion(%arg0 : !llvm.i32) -> !llvm.i32 {
 }
 
 // CHECK-LABEL: @addrspace
-llvm.func @addrspace(%arg0 : !llvm<"i32*">) -> !llvm<"i32 addrspace(2)*"> {
+llvm.func @addrspace(%arg0 : !llvm.ptr<i32>) -> !llvm.ptr<i32, 2> {
 // CHECK: %2 = addrspacecast i32* %0 to i32 addrspace(2)*
-  %1 = llvm.addrspacecast %arg0 : !llvm<"i32*"> to !llvm<"i32 addrspace(2)*">
-  llvm.return %1 : !llvm<"i32 addrspace(2)*">
+  %1 = llvm.addrspacecast %arg0 : !llvm.ptr<i32> to !llvm.ptr<i32, 2>
+  llvm.return %1 : !llvm.ptr<i32, 2>
 }
 
-llvm.func @stringconstant() -> !llvm<"i8*"> {
-  %1 = llvm.mlir.constant("Hello world!") : !llvm<"i8*">
+llvm.func @stringconstant() -> !llvm.ptr<i8> {
+  %1 = llvm.mlir.constant("Hello world!") : !llvm.ptr<i8>
   // CHECK: ret [12 x i8] c"Hello world!"
-  llvm.return %1 : !llvm<"i8*">
+  llvm.return %1 : !llvm.ptr<i8>
 }
 
 llvm.func @noreach() {
@@ -1019,22 +1019,22 @@ llvm.func @fcmp(%arg0: !llvm.float, %arg1: !llvm.float) {
 }
 
 // CHECK-LABEL: @vect
-llvm.func @vect(%arg0: !llvm<"<4 x float>">, %arg1: !llvm.i32, %arg2: !llvm.float) {
+llvm.func @vect(%arg0: !llvm.vec<4 x float>, %arg1: !llvm.i32, %arg2: !llvm.float) {
   // CHECK-NEXT: extractelement <4 x float> {{.*}}, i32
   // CHECK-NEXT: insertelement <4 x float> {{.*}}, float %2, i32
   // CHECK-NEXT: shufflevector <4 x float> {{.*}}, <4 x float> {{.*}}, <5 x i32> <i32 0, i32 0, i32 0, i32 0, i32 7>
-  %0 = llvm.extractelement %arg0[%arg1 : !llvm.i32] : !llvm<"<4 x float>">
-  %1 = llvm.insertelement %arg2, %arg0[%arg1 : !llvm.i32] : !llvm<"<4 x float>">
-  %2 = llvm.shufflevector %arg0, %arg0 [0 : i32, 0 : i32, 0 : i32, 0 : i32, 7 : i32] : !llvm<"<4 x float>">, !llvm<"<4 x float>">
+  %0 = llvm.extractelement %arg0[%arg1 : !llvm.i32] : !llvm.vec<4 x float>
+  %1 = llvm.insertelement %arg2, %arg0[%arg1 : !llvm.i32] : !llvm.vec<4 x float>
+  %2 = llvm.shufflevector %arg0, %arg0 [0 : i32, 0 : i32, 0 : i32, 0 : i32, 7 : i32] : !llvm.vec<4 x float>, !llvm.vec<4 x float>
   llvm.return
 }
 
 // CHECK-LABEL: @vect_i64idx
-llvm.func @vect_i64idx(%arg0: !llvm<"<4 x float>">, %arg1: !llvm.i64, %arg2: !llvm.float) {
+llvm.func @vect_i64idx(%arg0: !llvm.vec<4 x float>, %arg1: !llvm.i64, %arg2: !llvm.float) {
   // CHECK-NEXT: extractelement <4 x float> {{.*}}, i64
   // CHECK-NEXT: insertelement <4 x float> {{.*}}, float %2, i64
-  %0 = llvm.extractelement %arg0[%arg1 : !llvm.i64] : !llvm<"<4 x float>">
-  %1 = llvm.insertelement %arg2, %arg0[%arg1 : !llvm.i64] : !llvm<"<4 x float>">
+  %0 = llvm.extractelement %arg0[%arg1 : !llvm.i64] : !llvm.vec<4 x float>
+  %1 = llvm.insertelement %arg2, %arg0[%arg1 : !llvm.i64] : !llvm.vec<4 x float>
   llvm.return
 }
 
@@ -1043,27 +1043,27 @@ llvm.func @alloca(%size : !llvm.i64) {
   // Alignment automatically set by the LLVM IR builder when alignment attribute
   // is 0.
   //  CHECK: alloca {{.*}} align 4
-  llvm.alloca %size x !llvm.i32 {alignment = 0} : (!llvm.i64) -> (!llvm<"i32*">)
+  llvm.alloca %size x !llvm.i32 {alignment = 0} : (!llvm.i64) -> (!llvm.ptr<i32>)
   // CHECK-NEXT: alloca {{.*}} align 8
-  llvm.alloca %size x !llvm.i32 {alignment = 8} : (!llvm.i64) -> (!llvm<"i32*">)
+  llvm.alloca %size x !llvm.i32 {alignment = 8} : (!llvm.i64) -> (!llvm.ptr<i32>)
   llvm.return
 }
 
 // CHECK-LABEL: @constants
-llvm.func @constants() -> !llvm<"<4 x float>"> {
+llvm.func @constants() -> !llvm.vec<4 x float> {
   // CHECK: ret <4 x float> <float 4.2{{0*}}e+01, float 0.{{0*}}e+00, float 0.{{0*}}e+00, float 0.{{0*}}e+00>
-  %0 = llvm.mlir.constant(sparse<[[0]], [4.2e+01]> : vector<4xf32>) : !llvm<"<4 x float>">
-  llvm.return %0 : !llvm<"<4 x float>">
+  %0 = llvm.mlir.constant(sparse<[[0]], [4.2e+01]> : vector<4xf32>) : !llvm.vec<4 x float>
+  llvm.return %0 : !llvm.vec<4 x float>
 }
 
 // CHECK-LABEL: @fp_casts
-llvm.func @fp_casts(%fp1 : !llvm<"float">, %fp2 : !llvm<"double">) -> !llvm.i16 {
+llvm.func @fp_casts(%fp1 : !llvm.float, %fp2 : !llvm.double) -> !llvm.i16 {
 // CHECK:    fptrunc double {{.*}} to float
-  %a = llvm.fptrunc %fp2 : !llvm<"double"> to !llvm<"float">
+  %a = llvm.fptrunc %fp2 : !llvm.double to !llvm.float
 // CHECK:    fpext float {{.*}} to double
-  %b = llvm.fpext %fp1 : !llvm<"float"> to !llvm<"double">
+  %b = llvm.fpext %fp1 : !llvm.float to !llvm.double
 // CHECK:    fptosi double {{.*}} to i16
-  %c = llvm.fptosi %b : !llvm<"double"> to !llvm.i16
+  %c = llvm.fptosi %b : !llvm.double to !llvm.i16
   llvm.return %c : !llvm.i16
 }
 
@@ -1080,35 +1080,35 @@ llvm.func @integer_extension_and_truncation(%a : !llvm.i32) {
 
 // Check that the auxiliary `null` operation is converted into a `null` value.
 // CHECK-LABEL: @null
-llvm.func @null() -> !llvm<"i32*"> {
-  %0 = llvm.mlir.null : !llvm<"i32*">
+llvm.func @null() -> !llvm.ptr<i32> {
+  %0 = llvm.mlir.null : !llvm.ptr<i32>
   // CHECK: ret i32* null
-  llvm.return %0 : !llvm<"i32*">
+  llvm.return %0 : !llvm.ptr<i32>
 }
 
 // Check that dense elements attributes are exported properly in constants.
 // CHECK-LABEL: @elements_constant_3d_vector
-llvm.func @elements_constant_3d_vector() -> !llvm<"[2 x [2 x <2 x i32>]]"> {
+llvm.func @elements_constant_3d_vector() -> !llvm.array<2 x array<2 x vec<2 x i32>>> {
   // CHECK: ret [2 x [2 x <2 x i32>]]
   // CHECK-SAME: {{\[}}[2 x <2 x i32>] [<2 x i32> <i32 1, i32 2>, <2 x i32> <i32 3, i32 4>],
   // CHECK-SAME:       [2 x <2 x i32>] [<2 x i32> <i32 42, i32 43>, <2 x i32> <i32 44, i32 45>]]
-  %0 = llvm.mlir.constant(dense<[[[1, 2], [3, 4]], [[42, 43], [44, 45]]]> : vector<2x2x2xi32>) : !llvm<"[2 x [2 x <2 x i32>]]">
-  llvm.return %0 : !llvm<"[2 x [2 x <2 x i32>]]">
+  %0 = llvm.mlir.constant(dense<[[[1, 2], [3, 4]], [[42, 43], [44, 45]]]> : vector<2x2x2xi32>) : !llvm.array<2 x array<2 x vec<2 x i32>>>
+  llvm.return %0 : !llvm.array<2 x array<2 x vec<2 x i32>>>
 }
 
 // CHECK-LABEL: @elements_constant_3d_array
-llvm.func @elements_constant_3d_array() -> !llvm<"[2 x [2 x [2 x i32]]]"> {
+llvm.func @elements_constant_3d_array() -> !llvm.array<2 x array<2 x array<2 x i32>>> {
   // CHECK: ret [2 x [2 x [2 x i32]]]
   // CHECK-SAME: {{\[}}[2 x [2 x i32]] {{\[}}[2 x i32] [i32 1, i32 2], [2 x i32] [i32 3, i32 4]],
   // CHECK-SAME:       [2 x [2 x i32]] {{\[}}[2 x i32] [i32 42, i32 43], [2 x i32] [i32 44, i32 45]]]
-  %0 = llvm.mlir.constant(dense<[[[1, 2], [3, 4]], [[42, 43], [44, 45]]]> : tensor<2x2x2xi32>) : !llvm<"[2 x [2 x [2 x i32]]]">
-  llvm.return %0 : !llvm<"[2 x [2 x [2 x i32]]]">
+  %0 = llvm.mlir.constant(dense<[[[1, 2], [3, 4]], [[42, 43], [44, 45]]]> : tensor<2x2x2xi32>) : !llvm.array<2 x array<2 x array<2 x i32>>>
+  llvm.return %0 : !llvm.array<2 x array<2 x array<2 x i32>>>
 }
 
 // CHECK-LABEL: @atomicrmw
 llvm.func @atomicrmw(
-    %f32_ptr : !llvm<"float*">, %f32 : !llvm.float,
-    %i32_ptr : !llvm<"i32*">, %i32 : !llvm.i32) {
+    %f32_ptr : !llvm.ptr<float>, %f32 : !llvm.float,
+    %i32_ptr : !llvm.ptr<i32>, %i32 : !llvm.i32) {
   // CHECK: atomicrmw fadd float* %{{.*}}, float %{{.*}} unordered
   %0 = llvm.atomicrmw fadd %f32_ptr, %f32 unordered : !llvm.float
   // CHECK: atomicrmw fsub float* %{{.*}}, float %{{.*}} unordered
@@ -1139,34 +1139,34 @@ llvm.func @atomicrmw(
 }
 
 // CHECK-LABEL: @cmpxchg
-llvm.func @cmpxchg(%ptr : !llvm<"float*">, %cmp : !llvm.float, %val: !llvm.float) {
+llvm.func @cmpxchg(%ptr : !llvm.ptr<float>, %cmp : !llvm.float, %val: !llvm.float) {
   // CHECK: cmpxchg float* %{{.*}}, float %{{.*}}, float %{{.*}} acq_rel monotonic
   %0 = llvm.cmpxchg %ptr, %cmp, %val acq_rel monotonic : !llvm.float
   // CHECK: %{{[0-9]+}} = extractvalue { float, i1 } %{{[0-9]+}}, 0
-  %1 = llvm.extractvalue %0[0] : !llvm<"{ float, i1 }">
+  %1 = llvm.extractvalue %0[0] : !llvm.struct<(float, i1)>
   // CHECK: %{{[0-9]+}} = extractvalue { float, i1 } %{{[0-9]+}}, 1
-  %2 = llvm.extractvalue %0[1] : !llvm<"{ float, i1 }">
+  %2 = llvm.extractvalue %0[1] : !llvm.struct<(float, i1)>
   llvm.return
 }
 
-llvm.mlir.global external constant @_ZTIi() : !llvm<"i8*">
-llvm.func @foo(!llvm<"i8*">)
-llvm.func @bar(!llvm<"i8*">) -> !llvm<"i8*">
+llvm.mlir.global external constant @_ZTIi() : !llvm.ptr<i8>
+llvm.func @foo(!llvm.ptr<i8>)
+llvm.func @bar(!llvm.ptr<i8>) -> !llvm.ptr<i8>
 llvm.func @__gxx_personality_v0(...) -> !llvm.i32
 
 // CHECK-LABEL: @invokeLandingpad
 llvm.func @invokeLandingpad() -> !llvm.i32 attributes { personality = @__gxx_personality_v0 } {
 // CHECK: %[[a1:[0-9]+]] = alloca i8
   %0 = llvm.mlir.constant(0 : i32) : !llvm.i32
-  %1 = llvm.mlir.constant("\01") : !llvm<"[1 x i8]">
-  %2 = llvm.mlir.addressof @_ZTIi : !llvm<"i8**">
-  %3 = llvm.bitcast %2 : !llvm<"i8**"> to !llvm<"i8*">
-  %4 = llvm.mlir.null : !llvm<"i8**">
+  %1 = llvm.mlir.constant("\01") : !llvm.array<1 x i8>
+  %2 = llvm.mlir.addressof @_ZTIi : !llvm.ptr<ptr<i8>>
+  %3 = llvm.bitcast %2 : !llvm.ptr<ptr<i8>> to !llvm.ptr<i8>
+  %4 = llvm.mlir.null : !llvm.ptr<ptr<i8>>
   %5 = llvm.mlir.constant(1 : i32) : !llvm.i32
-  %6 = llvm.alloca %5 x !llvm.i8 : (!llvm.i32) -> !llvm<"i8*">
+  %6 = llvm.alloca %5 x !llvm.i8 : (!llvm.i32) -> !llvm.ptr<i8>
 // CHECK: invoke void @foo(i8* %[[a1]])
 // CHECK-NEXT: to label %[[normal:[0-9]+]] unwind label %[[unwind:[0-9]+]]
-  llvm.invoke @foo(%6) to ^bb2 unwind ^bb1 : (!llvm<"i8*">) -> ()
+  llvm.invoke @foo(%6) to ^bb2 unwind ^bb1 : (!llvm.ptr<i8>) -> ()
 
 // CHECK: [[unwind]]:
 ^bb1:
@@ -1174,7 +1174,7 @@ llvm.func @invokeLandingpad() -> !llvm.i32 attributes { personality = @__gxx_per
 // CHECK-NEXT:             catch i8** null
 // CHECK-NEXT:             catch i8* bitcast (i8** @_ZTIi to i8*)
 // CHECK-NEXT:             filter [1 x i8] c"\01"
-  %7 = llvm.landingpad (catch %4 : !llvm<"i8**">) (catch %3 : !llvm<"i8*">) (filter %1 : !llvm<"[1 x i8]">) : !llvm<"{ i8*, i32 }">
+  %7 = llvm.landingpad (catch %4 : !llvm.ptr<ptr<i8>>) (catch %3 : !llvm.ptr<i8>) (filter %1 : !llvm.array<1 x i8>) : !llvm.struct<(ptr<i8>, i32)>
 // CHECK: br label %[[final:[0-9]+]]
   llvm.br ^bb3
 
@@ -1187,7 +1187,7 @@ llvm.func @invokeLandingpad() -> !llvm.i32 attributes { personality = @__gxx_per
 // CHECK-NEXT: %{{[0-9]+}} = invoke i8* @bar(i8* %[[a1]])
 // CHECK-NEXT:          to label %[[normal]] unwind label %[[unwind]]
 ^bb3:	// pred: ^bb1
-  %8 = llvm.invoke @bar(%6) to ^bb2 unwind ^bb1 : (!llvm<"i8*">) -> !llvm<"i8*">
+  %8 = llvm.invoke @bar(%6) to ^bb2 unwind ^bb1 : (!llvm.ptr<i8>) -> !llvm.ptr<i8>
 }
 
 // CHECK-LABEL: @callFreezeOp
@@ -1235,9 +1235,9 @@ llvm.func @passthrough() attributes {passthrough = ["noinline", ["alignstack", "
 // -----
 
 // CHECK-LABEL: @constant_bf16
-llvm.func @constant_bf16() -> !llvm<"bfloat"> {
-  %0 = llvm.mlir.constant(1.000000e+01 : bf16) : !llvm<"bfloat">
-  llvm.return %0 : !llvm<"bfloat">
+llvm.func @constant_bf16() -> !llvm.bfloat {
+  %0 = llvm.mlir.constant(1.000000e+01 : bf16) : !llvm.bfloat
+  llvm.return %0 : !llvm.bfloat
 }
 
 // CHECK: ret bfloat 0xR4120
@@ -1248,9 +1248,9 @@ llvm.func @address_taken() {
   llvm.return
 }
 
-llvm.mlir.global internal constant @taker_of_address() : !llvm<"void()*"> {
-  %0 = llvm.mlir.addressof @address_taken : !llvm<"void()*">
-  llvm.return %0 : !llvm<"void()*">
+llvm.mlir.global internal constant @taker_of_address() : !llvm.ptr<func<void ()>> {
+  %0 = llvm.mlir.addressof @address_taken : !llvm.ptr<func<void ()>>
+  llvm.return %0 : !llvm.ptr<func<void ()>>
 }
 
 // -----
@@ -1272,11 +1272,11 @@ llvm.func @cond_br_weights(%cond : !llvm.i1, %arg0 : !llvm.i32,  %arg1 : !llvm.i
 llvm.func @volatile_store_and_load() {
   %val = llvm.mlir.constant(5 : i32) : !llvm.i32
   %size = llvm.mlir.constant(1 : i64) : !llvm.i64
-  %0 = llvm.alloca %size x !llvm.i32 : (!llvm.i64) -> (!llvm<"i32*">)
+  %0 = llvm.alloca %size x !llvm.i32 : (!llvm.i64) -> (!llvm.ptr<i32>)
   // CHECK: store volatile i32 5, i32* %{{.*}}
-  llvm.store volatile %val, %0 : !llvm<"i32*">
+  llvm.store volatile %val, %0 : !llvm.ptr<i32>
   // CHECK: %{{.*}} = load volatile i32, i32* %{{.*}}
-  %1 = llvm.load volatile %0: !llvm<"i32*">
+  %1 = llvm.load volatile %0: !llvm.ptr<i32>
   llvm.return
 }
 
@@ -1286,11 +1286,11 @@ llvm.func @volatile_store_and_load() {
 llvm.func @nontemoral_store_and_load() {
   %val = llvm.mlir.constant(5 : i32) : !llvm.i32
   %size = llvm.mlir.constant(1 : i64) : !llvm.i64
-  %0 = llvm.alloca %size x !llvm.i32 : (!llvm.i64) -> (!llvm<"i32*">)
+  %0 = llvm.alloca %size x !llvm.i32 : (!llvm.i64) -> (!llvm.ptr<i32>)
   // CHECK: !nontemporal ![[NODE:[0-9]+]]
-  llvm.store %val, %0 {nontemporal} : !llvm<"i32*">
+  llvm.store %val, %0 {nontemporal} : !llvm.ptr<i32>
   // CHECK: !nontemporal ![[NODE]]
-  %1 = llvm.load %0 {nontemporal} : !llvm<"i32*">
+  %1 = llvm.load %0 {nontemporal} : !llvm.ptr<i32>
   llvm.return
 }
 
diff --git a/mlir/test/Target/nvvmir.mlir b/mlir/test/Target/nvvmir.mlir
index 7e8cfb6c0a389..fdb438b4e6a96 100644
--- a/mlir/test/Target/nvvmir.mlir
+++ b/mlir/test/Target/nvvmir.mlir
@@ -50,12 +50,12 @@ llvm.func @nvvm_shfl(
 
 llvm.func @nvvm_shfl_pred(
     %0 : !llvm.i32, %1 : !llvm.i32, %2 : !llvm.i32,
-    %3 : !llvm.i32, %4 : !llvm.float) -> !llvm<"{ i32, i1 }"> {
+    %3 : !llvm.i32, %4 : !llvm.float) -> !llvm.struct<(i32, i1)> {
   // CHECK: call { i32, i1 } @llvm.nvvm.shfl.sync.bfly.i32p(i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}})
-  %6 = nvvm.shfl.sync.bfly %0, %3, %1, %2 {return_value_and_is_valid} : !llvm<"{ i32, i1 }">
+  %6 = nvvm.shfl.sync.bfly %0, %3, %1, %2 {return_value_and_is_valid} : !llvm.struct<(i32, i1)>
   // CHECK: call { float, i1 } @llvm.nvvm.shfl.sync.bfly.f32p(i32 %{{.*}}, float %{{.*}}, i32 %{{.*}}, i32 %{{.*}})
-  %7 = nvvm.shfl.sync.bfly %0, %4, %1, %2 {return_value_and_is_valid} : !llvm<"{ float, i1 }">
-  llvm.return %6 : !llvm<"{ i32, i1 }">
+  %7 = nvvm.shfl.sync.bfly %0, %4, %1, %2 {return_value_and_is_valid} : !llvm.struct<(float, i1)>
+  llvm.return %6 : !llvm.struct<(i32, i1)>
 }
 
 llvm.func @nvvm_vote(%0 : !llvm.i32, %1 : !llvm.i1) -> !llvm.i32 {
@@ -64,13 +64,13 @@ llvm.func @nvvm_vote(%0 : !llvm.i32, %1 : !llvm.i1) -> !llvm.i32 {
   llvm.return %3 : !llvm.i32
 }
 
-llvm.func @nvvm_mma(%a0 : !llvm<"<2 x half>">, %a1 : !llvm<"<2 x half>">,
-                    %b0 : !llvm<"<2 x half>">, %b1 : !llvm<"<2 x half>">,
+llvm.func @nvvm_mma(%a0 : !llvm.vec<2 x half>, %a1 : !llvm.vec<2 x half>,
+                    %b0 : !llvm.vec<2 x half>, %b1 : !llvm.vec<2 x half>,
                     %c0 : !llvm.float, %c1 : !llvm.float, %c2 : !llvm.float, %c3 : !llvm.float,
                     %c4 : !llvm.float, %c5 : !llvm.float, %c6 : !llvm.float, %c7 : !llvm.float) {
   // CHECK: call { float, float, float, float, float, float, float, float } @llvm.nvvm.mma.m8n8k4.row.col.f32.f32
-  %0 = nvvm.mma.sync %a0, %a1, %b0, %b1, %c0, %c1, %c2, %c3, %c4, %c5, %c6, %c7 {alayout="row", blayout="col"} : (!llvm<"<2 x half>">, !llvm<"<2 x half>">, !llvm<"<2 x half>">, !llvm<"<2 x half>">, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float) -> !llvm<"{ float, float, float, float, float, float, float, float }">
-  llvm.return %0 : !llvm<"{ float, float, float, float, float, float, float, float }">
+  %0 = nvvm.mma.sync %a0, %a1, %b0, %b1, %c0, %c1, %c2, %c3, %c4, %c5, %c6, %c7 {alayout="row", blayout="col"} : (!llvm.vec<2 x half>, !llvm.vec<2 x half>, !llvm.vec<2 x half>, !llvm.vec<2 x half>, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float, !llvm.float) -> !llvm.struct<(float, float, float, float, float, float, float, float)>
+  llvm.return %0 : !llvm.struct<(float, float, float, float, float, float, float, float)>
 }
 
 // This function has the "kernel" attribute attached and should appear in the
diff --git a/mlir/test/Target/rocdl.mlir b/mlir/test/Target/rocdl.mlir
index 127f654893519..949b1e74ae29a 100644
--- a/mlir/test/Target/rocdl.mlir
+++ b/mlir/test/Target/rocdl.mlir
@@ -43,133 +43,133 @@ llvm.func @rocdl.barrier() {
 }
 
 llvm.func @rocdl.xdlops(%arg0 : !llvm.float, %arg1 : !llvm.float,
-                   %arg2 : !llvm<"<32 x float>">, %arg3 : !llvm.i32,
-                   %arg4 : !llvm<"<16 x float>">, %arg5 : !llvm<"<4 x float>">,
-                   %arg6 : !llvm<"<4 x half>">, %arg7 : !llvm<"<32 x i32>">,
-                   %arg8 : !llvm<"<16 x i32>">, %arg9 : !llvm<"<4 x i32>">,
-                   %arg10 : !llvm<"<2 x i16>">) -> !llvm<"<32 x float>"> {
+                   %arg2 : !llvm.vec<32 x float>, %arg3 : !llvm.i32,
+                   %arg4 : !llvm.vec<16 x float>, %arg5 : !llvm.vec<4 x float>,
+                   %arg6 : !llvm.vec<4 x half>, %arg7 : !llvm.vec<32 x i32>,
+                   %arg8 : !llvm.vec<16 x i32>, %arg9 : !llvm.vec<4 x i32>,
+                   %arg10 : !llvm.vec<2 x i16>) -> !llvm.vec<32 x float> {
   // CHECK-LABEL: rocdl.xdlops
   // CHECK: call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float %{{.*}}, float %{{.*}}, <32 x float> %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}})
   %r0 = rocdl.mfma.f32.32x32x1f32 %arg0, %arg1, %arg2, %arg3, %arg3, %arg3 :
-                            (!llvm.float, !llvm.float, !llvm<"<32 x float>">,
-                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm<"<32 x float>">
+                            (!llvm.float, !llvm.float, !llvm.vec<32 x float>,
+                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm.vec<32 x float>
 
   // CHECK: call <16 x float> @llvm.amdgcn.mfma.f32.16x16x1f32(float %{{.*}}, float %{{.*}}, <16 x float> %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}})
   %r1 = rocdl.mfma.f32.16x16x1f32 %arg0, %arg1, %arg4, %arg3, %arg3, %arg3 :
-                            (!llvm.float, !llvm.float, !llvm<"<16 x float>">,
-                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm<"<16 x float>">
+                            (!llvm.float, !llvm.float, !llvm.vec<16 x float>,
+                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm.vec<16 x float>
 
   // CHECK: call <4 x float> @llvm.amdgcn.mfma.f32.16x16x4f32(float %{{.*}}, float %{{.*}}, <4 x float> %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}})
   %r2 = rocdl.mfma.f32.16x16x4f32 %arg0, %arg1, %arg5, %arg3, %arg3, %arg3 :
-                            (!llvm.float, !llvm.float, !llvm<"<4 x float>">,
-                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm<"<4 x float>">
+                            (!llvm.float, !llvm.float, !llvm.vec<4 x float>,
+                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm.vec<4 x float>
 
   // CHECK: call <4 x float> @llvm.amdgcn.mfma.f32.4x4x1f32(float %{{.*}}, float %{{.*}}, <4 x float> %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}})
   %r3 = rocdl.mfma.f32.4x4x1f32 %arg0, %arg1, %arg5, %arg3, %arg3, %arg3 :
-                            (!llvm.float, !llvm.float, !llvm<"<4 x float>">,
-                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm<"<4 x float>">
+                            (!llvm.float, !llvm.float, !llvm.vec<4 x float>,
+                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm.vec<4 x float>
 
   // CHECK: call <16 x float> @llvm.amdgcn.mfma.f32.32x32x2f32(float %{{.*}}, float %{{.*}}, <16 x float> %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}})
   %r4= rocdl.mfma.f32.32x32x2f32 %arg0, %arg1, %arg4, %arg3, %arg3, %arg3 :
-                            (!llvm.float, !llvm.float, !llvm<"<16 x float>">,
-                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm<"<16 x float>">
+                            (!llvm.float, !llvm.float, !llvm.vec<16 x float>,
+                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm.vec<16 x float>
 
   // CHECK: call <32 x float> @llvm.amdgcn.mfma.f32.32x32x4f16(<4 x half> %{{.*}}, <4 x half> %{{.*}}, <32 x float> %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}})
   %r5 = rocdl.mfma.f32.32x32x4f16 %arg6, %arg6, %arg2, %arg3, %arg3, %arg3 :
-                            (!llvm<"<4 x half>">, !llvm<"<4 x half>">, !llvm<"<32 x float>">,
-                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm<"<32 x float>">
+                            (!llvm.vec<4 x half>, !llvm.vec<4 x half>, !llvm.vec<32 x float>,
+                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm.vec<32 x float>
 
   // CHECK: call <16 x float> @llvm.amdgcn.mfma.f32.16x16x4f16(<4 x half> %{{.*}}, <4 x half> %{{.*}}, <16 x float> %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}})
   %r6 = rocdl.mfma.f32.16x16x4f16 %arg6, %arg6, %arg4, %arg3, %arg3, %arg3 :
-                            (!llvm<"<4 x half>">, !llvm<"<4 x half>">, !llvm<"<16 x float>">,
-                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm<"<16 x float>">
+                            (!llvm.vec<4 x half>, !llvm.vec<4 x half>, !llvm.vec<16 x float>,
+                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm.vec<16 x float>
 
   // CHECK: call <4 x float> @llvm.amdgcn.mfma.f32.4x4x4f16(<4 x half> %{{.*}}, <4 x half> %{{.*}}, <4 x float> %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}})
   %r7 = rocdl.mfma.f32.4x4x4f16 %arg6, %arg6, %arg5, %arg3, %arg3, %arg3 :
-                            (!llvm<"<4 x half>">, !llvm<"<4 x half>">, !llvm<"<4 x float>">,
-                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm<"<4 x float>">
+                            (!llvm.vec<4 x half>, !llvm.vec<4 x half>, !llvm.vec<4 x float>,
+                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm.vec<4 x float>
 
   // CHECK: call <16 x float> @llvm.amdgcn.mfma.f32.32x32x8f16(<4 x half> %{{.*}}, <4 x half> %{{.*}}, <16 x float> %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}})
   %r8 = rocdl.mfma.f32.32x32x8f16 %arg6, %arg6, %arg4, %arg3, %arg3, %arg3 :
-                            (!llvm<"<4 x half>">, !llvm<"<4 x half>">, !llvm<"<16 x float>">,
-                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm<"<16 x float>">
+                            (!llvm.vec<4 x half>, !llvm.vec<4 x half>, !llvm.vec<16 x float>,
+                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm.vec<16 x float>
 
   // CHECK: call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16f16(<4 x half> %{{.*}}, <4 x half> %{{.*}}, <4 x float> %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}})
   %r9 = rocdl.mfma.f32.16x16x16f16 %arg6, %arg6, %arg5, %arg3, %arg3, %arg3 :
-                            (!llvm<"<4 x half>">, !llvm<"<4 x half>">, !llvm<"<4 x float>">,
-                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm<"<4 x float>">
+                            (!llvm.vec<4 x half>, !llvm.vec<4 x half>, !llvm.vec<4 x float>,
+                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm.vec<4 x float>
 
   // CHECK: call <32 x i32> @llvm.amdgcn.mfma.i32.32x32x4i8(i32 %{{.*}}, i32 %{{.*}}, <32 x i32> %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}})
   %r10 = rocdl.mfma.i32.32x32x4i8 %arg3, %arg3, %arg7, %arg3, %arg3, %arg3 :
-                            (!llvm.i32, !llvm.i32, !llvm<"<32 x i32>">,
-                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm<"<32 x i32>">
+                            (!llvm.i32, !llvm.i32, !llvm.vec<32 x i32>,
+                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm.vec<32 x i32>
 
   // CHECK: call <16 x i32> @llvm.amdgcn.mfma.i32.16x16x4i8(i32 %{{.*}}, i32 %{{.*}}, <16 x i32> %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}})
   %r11 = rocdl.mfma.i32.16x16x4i8 %arg3, %arg3, %arg8, %arg3, %arg3, %arg3 :
-                            (!llvm.i32, !llvm.i32, !llvm<"<16 x i32>">,
-                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm<"<16 x i32>">
+                            (!llvm.i32, !llvm.i32, !llvm.vec<16 x i32>,
+                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm.vec<16 x i32>
 
   // CHECK: call <4 x i32> @llvm.amdgcn.mfma.i32.4x4x4i8(i32 %{{.*}}, i32 %{{.*}}, <4 x i32> %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}})
   %r12 = rocdl.mfma.i32.4x4x4i8 %arg3, %arg3, %arg9, %arg3, %arg3, %arg3 :
-                            (!llvm.i32, !llvm.i32, !llvm<"<4 x i32>">,
-                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm<"<4 x i32>">
+                            (!llvm.i32, !llvm.i32, !llvm.vec<4 x i32>,
+                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm.vec<4 x i32>
 
   // CHECK: call <16 x i32> @llvm.amdgcn.mfma.i32.32x32x8i8(i32 %{{.*}}, i32 %{{.*}}, <16 x i32> %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}})
   %r13 = rocdl.mfma.i32.32x32x8i8 %arg3, %arg3, %arg8, %arg3, %arg3, %arg3 :
-                            (!llvm.i32, !llvm.i32, !llvm<"<16 x i32>">,
-                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm<"<16 x i32>">
+                            (!llvm.i32, !llvm.i32, !llvm.vec<16 x i32>,
+                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm.vec<16 x i32>
 
   // CHECK: call <4 x i32> @llvm.amdgcn.mfma.i32.16x16x16i8(i32 %{{.*}}, i32 %{{.*}}, <4 x i32> %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}})
   %r14 = rocdl.mfma.i32.16x16x16i8 %arg3, %arg3, %arg9, %arg3, %arg3, %arg3 :
-                            (!llvm.i32, !llvm.i32, !llvm<"<4 x i32>">,
-                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm<"<4 x i32>">
+                            (!llvm.i32, !llvm.i32, !llvm.vec<4 x i32>,
+                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm.vec<4 x i32>
 
   // CHECK: call <32 x float> @llvm.amdgcn.mfma.f32.32x32x2bf16(<2 x i16> %{{.*}}, <2 x i16> %{{.*}}, <32 x float> %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}})
   %r15 = rocdl.mfma.f32.32x32x2bf16 %arg10, %arg10, %arg2, %arg3, %arg3, %arg3 :
-                            (!llvm<"<2 x i16>">, !llvm<"<2 x i16>">, !llvm<"<32 x float>">,
-                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm<"<32 x float>">
+                            (!llvm.vec<2 x i16>, !llvm.vec<2 x i16>, !llvm.vec<32 x float>,
+                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm.vec<32 x float>
 
   // CHECK: call <16 x float> @llvm.amdgcn.mfma.f32.16x16x2bf16(<2 x i16> %{{.*}}, <2 x i16> %{{.*}}, <16 x float> %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}})
   %r16 = rocdl.mfma.f32.16x16x2bf16 %arg10, %arg10, %arg4, %arg3, %arg3, %arg3 :
-                            (!llvm<"<2 x i16>">, !llvm<"<2 x i16>">, !llvm<"<16 x float>">,
-                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm<"<16 x float>">
+                            (!llvm.vec<2 x i16>, !llvm.vec<2 x i16>, !llvm.vec<16 x float>,
+                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm.vec<16 x float>
 
   // CHECK: call <4 x float> @llvm.amdgcn.mfma.f32.4x4x2bf16(<2 x i16> %{{.*}}, <2 x i16> %{{.*}}, <4 x float> %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}})
   %r17 = rocdl.mfma.f32.4x4x2bf16 %arg10, %arg10, %arg5, %arg3, %arg3, %arg3 :
-                            (!llvm<"<2 x i16>">, !llvm<"<2 x i16>">, !llvm<"<4 x float>">,
-                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm<"<4 x float>">
+                            (!llvm.vec<2 x i16>, !llvm.vec<2 x i16>, !llvm.vec<4 x float>,
+                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm.vec<4 x float>
 
   // CHECK: call <16 x float> @llvm.amdgcn.mfma.f32.32x32x4bf16(<2 x i16> %{{.*}}, <2 x i16> %{{.*}}, <16 x float> %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}})
   %r18 = rocdl.mfma.f32.32x32x4bf16 %arg10, %arg10, %arg4, %arg3, %arg3, %arg3 :
-                            (!llvm<"<2 x i16>">, !llvm<"<2 x i16>">, !llvm<"<16 x float>">,
-                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm<"<16 x float>">
+                            (!llvm.vec<2 x i16>, !llvm.vec<2 x i16>, !llvm.vec<16 x float>,
+                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm.vec<16 x float>
 
   // CHECK: call <4 x float> @llvm.amdgcn.mfma.f32.16x16x8bf16(<2 x i16> %{{.*}}, <2 x i16> %{{.*}}, <4 x float> %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}})
   %r19 = rocdl.mfma.f32.16x16x8bf16 %arg10, %arg10, %arg5, %arg3, %arg3, %arg3 :
-                            (!llvm<"<2 x i16>">, !llvm<"<2 x i16>">, !llvm<"<4 x float>">,
-                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm<"<4 x float>">
+                            (!llvm.vec<2 x i16>, !llvm.vec<2 x i16>, !llvm.vec<4 x float>,
+                            !llvm.i32, !llvm.i32, !llvm.i32) -> !llvm.vec<4 x float>
 
-  llvm.return %r0 : !llvm<"<32 x float>">
+  llvm.return %r0 : !llvm.vec<32 x float>
 }
 
-llvm.func @rocdl.mubuf(%rsrc : !llvm<"<4 x i32>">, %vindex : !llvm.i32,
+llvm.func @rocdl.mubuf(%rsrc : !llvm.vec<4 x i32>, %vindex : !llvm.i32,
                        %offset : !llvm.i32, %glc : !llvm.i1,
-                       %slc : !llvm.i1, %vdata1 : !llvm<"<1 x float>">,
-                       %vdata2 : !llvm<"<2 x float>">, %vdata4 : !llvm<"<4 x float>">) {
+                       %slc : !llvm.i1, %vdata1 : !llvm.vec<1 x float>,
+                       %vdata2 : !llvm.vec<2 x float>, %vdata4 : !llvm.vec<4 x float>) {
   // CHECK-LABEL: rocdl.mubuf
   // CHECK: call <1 x float> @llvm.amdgcn.buffer.load.v1f32(<4 x i32> %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i1 %{{.*}}, i1 %{{.*}})
-  %r1 = rocdl.buffer.load %rsrc, %vindex, %offset, %glc, %slc : !llvm<"<1 x float>">
+  %r1 = rocdl.buffer.load %rsrc, %vindex, %offset, %glc, %slc : !llvm.vec<1 x float>
   // CHECK: call <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32> %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i1 %{{.*}}, i1 %{{.*}})
-  %r2 = rocdl.buffer.load %rsrc, %vindex, %offset, %glc, %slc : !llvm<"<2 x float>">
+  %r2 = rocdl.buffer.load %rsrc, %vindex, %offset, %glc, %slc : !llvm.vec<2 x float>
   // CHECK: call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i1 %{{.*}}, i1 %{{.*}})
-  %r4 = rocdl.buffer.load %rsrc, %vindex, %offset, %glc, %slc : !llvm<"<4 x float>">
+  %r4 = rocdl.buffer.load %rsrc, %vindex, %offset, %glc, %slc : !llvm.vec<4 x float>
 
   // CHECK: call void @llvm.amdgcn.buffer.store.v1f32(<1 x float> %{{.*}}, <4 x i32> %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i1 %{{.*}}, i1 %{{.*}})
-  rocdl.buffer.store %vdata1, %rsrc, %vindex, %offset, %glc, %slc : !llvm<"<1 x float>">
+  rocdl.buffer.store %vdata1, %rsrc, %vindex, %offset, %glc, %slc : !llvm.vec<1 x float>
   // CHECK: call void @llvm.amdgcn.buffer.store.v2f32(<2 x float> %{{.*}}, <4 x i32> %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i1 %{{.*}}, i1 %{{.*}})
-  rocdl.buffer.store %vdata2, %rsrc, %vindex, %offset, %glc, %slc : !llvm<"<2 x float>">
+  rocdl.buffer.store %vdata2, %rsrc, %vindex, %offset, %glc, %slc : !llvm.vec<2 x float>
   // CHECK: call void @llvm.amdgcn.buffer.store.v4f32(<4 x float> %{{.*}}, <4 x i32> %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i1 %{{.*}}, i1 %{{.*}})
-  rocdl.buffer.store %vdata4, %rsrc, %vindex, %offset, %glc, %slc : !llvm<"<4 x float>">
+  rocdl.buffer.store %vdata4, %rsrc, %vindex, %offset, %glc, %slc : !llvm.vec<4 x float>
 
   llvm.return
 }
diff --git a/mlir/test/lib/Dialect/LLVMIR/LLVMTypeTestDialect.cpp b/mlir/test/lib/Dialect/LLVMIR/LLVMTypeTestDialect.cpp
index 8ac1ef0a8c170..873ed16169093 100644
--- a/mlir/test/lib/Dialect/LLVMIR/LLVMTypeTestDialect.cpp
+++ b/mlir/test/lib/Dialect/LLVMIR/LLVMTypeTestDialect.cpp
@@ -11,25 +11,25 @@ class LLVMDialectNewTypes : public Dialect {
 public:
   LLVMDialectNewTypes(MLIRContext *ctx) : Dialect(getDialectNamespace(), ctx) {
     // clang-format off
-    addTypes<LLVMVoidType,
-             LLVMHalfType,
-             LLVMBFloatType,
-             LLVMFloatType,
-             LLVMDoubleType,
-             LLVMFP128Type,
-             LLVMX86FP80Type,
-             LLVMPPCFP128Type,
-             LLVMX86MMXType,
-             LLVMTokenType,
-             LLVMLabelType,
-             LLVMMetadataType,
-             LLVMFunctionType,
-             LLVMIntegerType,
-             LLVMPointerType,
-             LLVMFixedVectorType,
-             LLVMScalableVectorType,
-             LLVMArrayType,
-             LLVMStructType>();
+    // addTypes<LLVMVoidType,
+    //          LLVMHalfType,
+    //          LLVMBFloatType,
+    //          LLVMFloatType,
+    //          LLVMDoubleType,
+    //          LLVMFP128Type,
+    //          LLVMX86FP80Type,
+    //          LLVMPPCFP128Type,
+    //          LLVMX86MMXType,
+    //          LLVMTokenType,
+    //          LLVMLabelType,
+    //          LLVMMetadataType,
+    //          LLVMFunctionType,
+    //          LLVMIntegerType,
+    //          LLVMPointerType,
+    //          LLVMFixedVectorType,
+    //          LLVMScalableVectorType,
+    //          LLVMArrayType,
+    //          LLVMStructType>();
     // clang-format on
   }
   static StringRef getDialectNamespace() { return "llvm2"; }
@@ -38,7 +38,7 @@ class LLVMDialectNewTypes : public Dialect {
     return detail::parseType(parser);
   }
   void printType(Type type, DialectAsmPrinter &printer) const override {
-    detail::printType(type.cast<LLVMTypeNew>(), printer);
+    detail::printType(type.cast<LLVMType>(), printer);
   }
 };
 } // namespace
diff --git a/mlir/test/lib/Target/TestLLVMTypeTranslation.cpp b/mlir/test/lib/Target/TestLLVMTypeTranslation.cpp
index b76ac2a13344b..e51734308aa66 100644
--- a/mlir/test/lib/Target/TestLLVMTypeTranslation.cpp
+++ b/mlir/test/lib/Target/TestLLVMTypeTranslation.cpp
@@ -34,7 +34,7 @@ class TestLLVMTypeTranslation : public LLVM::ModuleTranslation {
     if (op.getName().getStringRef() == "llvm.test_introduce_func") {
       auto attr = op.getAttrOfType<TypeAttr>("type");
       assert(attr && "expected 'type' attribute");
-      auto type = attr.getValue().cast<LLVM::LLVMTypeNew>();
+      auto type = attr.getValue().cast<LLVM::LLVMType>();
 
       auto nameAttr = op.getAttrOfType<StringAttr>("name");
       assert(nameAttr && "expected 'name' attributes");
@@ -49,7 +49,7 @@ class TestLLVMTypeTranslation : public LLVM::ModuleTranslation {
         module->getOrInsertFunction(nameAttr.getValue(), translated);
 
       std::string roundtripName = (Twine(nameAttr.getValue()) + "_round").str();
-      LLVM::LLVMTypeNew translatedBack =
+      LLVM::LLVMType translatedBack =
           LLVM::translateTypeFromLLVMIR(translated, *op.getContext());
       llvm::Type *translatedBackAndForth =
           LLVM::translateTypeToLLVMIR(translatedBack, builder.getContext());
diff --git a/mlir/test/mlir-cpu-runner/bare_ptr_call_conv.mlir b/mlir/test/mlir-cpu-runner/bare_ptr_call_conv.mlir
index b48b523165bd7..5f5ff6f1fbb9f 100644
--- a/mlir/test/mlir-cpu-runner/bare_ptr_call_conv.mlir
+++ b/mlir/test/mlir-cpu-runner/bare_ptr_call_conv.mlir
@@ -26,8 +26,8 @@ func @simple_add1_add2_test(%arg0: memref<2xf32>, %arg1: memref<2xf32>) {
 }
 
 // External declarations.
-llvm.func @malloc(!llvm.i64) -> !llvm<"i8*">
-llvm.func @free(!llvm<"i8*">)
+llvm.func @malloc(!llvm.i64) -> !llvm.ptr<i8>
+llvm.func @free(!llvm.ptr<i8>)
 func @print_f32(%arg0: f32)
 func @print_comma()
 func @print_newline()
@@ -79,105 +79,105 @@ llvm.func @main() {
   %3 = llvm.mlir.constant(1.000000e+00 : f32) : !llvm.float
   %4 = llvm.mlir.constant(2.000000e+00 : f32) : !llvm.float
   %5 = llvm.mlir.constant(2 : index) : !llvm.i64
-  %6 = llvm.mlir.null : !llvm<"float*">
+  %6 = llvm.mlir.null : !llvm.ptr<float>
   %7 = llvm.mlir.constant(1 : index) : !llvm.i64
-  %8 = llvm.getelementptr %6[%7] : (!llvm<"float*">, !llvm.i64) -> !llvm<"float*">
-  %9 = llvm.ptrtoint %8 : !llvm<"float*"> to !llvm.i64
+  %8 = llvm.getelementptr %6[%7] : (!llvm.ptr<float>, !llvm.i64) -> !llvm.ptr<float>
+  %9 = llvm.ptrtoint %8 : !llvm.ptr<float> to !llvm.i64
   %10 = llvm.mul %5, %9 : !llvm.i64
-  %11 = llvm.call @malloc(%10) : (!llvm.i64) -> !llvm<"i8*">
-  %12 = llvm.bitcast %11 : !llvm<"i8*"> to !llvm<"float*">
-  %13 = llvm.mlir.undef : !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }">
-  %14 = llvm.insertvalue %12, %13[0] : !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }">
-  %15 = llvm.insertvalue %12, %14[1] : !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }">
+  %11 = llvm.call @malloc(%10) : (!llvm.i64) -> !llvm.ptr<i8>
+  %12 = llvm.bitcast %11 : !llvm.ptr<i8> to !llvm.ptr<float>
+  %13 = llvm.mlir.undef : !llvm.struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>
+  %14 = llvm.insertvalue %12, %13[0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>
+  %15 = llvm.insertvalue %12, %14[1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>
   %16 = llvm.mlir.constant(0 : index) : !llvm.i64
-  %17 = llvm.insertvalue %16, %15[2] : !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }">
+  %17 = llvm.insertvalue %16, %15[2] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>
   %18 = llvm.mlir.constant(1 : index) : !llvm.i64
-  %19 = llvm.insertvalue %5, %17[3, 0] : !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }">
-  %20 = llvm.insertvalue %18, %19[4, 0] : !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }">
+  %19 = llvm.insertvalue %5, %17[3, 0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>
+  %20 = llvm.insertvalue %18, %19[4, 0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>
   %21 = llvm.mlir.constant(2 : index) : !llvm.i64
-  %22 = llvm.mlir.null : !llvm<"float*">
+  %22 = llvm.mlir.null : !llvm.ptr<float>
   %23 = llvm.mlir.constant(1 : index) : !llvm.i64
-  %24 = llvm.getelementptr %22[%23] : (!llvm<"float*">, !llvm.i64) -> !llvm<"float*">
-  %25 = llvm.ptrtoint %24 : !llvm<"float*"> to !llvm.i64
+  %24 = llvm.getelementptr %22[%23] : (!llvm.ptr<float>, !llvm.i64) -> !llvm.ptr<float>
+  %25 = llvm.ptrtoint %24 : !llvm.ptr<float> to !llvm.i64
   %26 = llvm.mul %21, %25 : !llvm.i64
-  %27 = llvm.call @malloc(%26) : (!llvm.i64) -> !llvm<"i8*">
-  %28 = llvm.bitcast %27 : !llvm<"i8*"> to !llvm<"float*">
-  %29 = llvm.mlir.undef : !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }">
-  %30 = llvm.insertvalue %28, %29[0] : !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }">
-  %31 = llvm.insertvalue %28, %30[1] : !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }">
+  %27 = llvm.call @malloc(%26) : (!llvm.i64) -> !llvm.ptr<i8>
+  %28 = llvm.bitcast %27 : !llvm.ptr<i8> to !llvm.ptr<float>
+  %29 = llvm.mlir.undef : !llvm.struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>
+  %30 = llvm.insertvalue %28, %29[0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>
+  %31 = llvm.insertvalue %28, %30[1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>
   %32 = llvm.mlir.constant(0 : index) : !llvm.i64
-  %33 = llvm.insertvalue %32, %31[2] : !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }">
+  %33 = llvm.insertvalue %32, %31[2] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>
   %34 = llvm.mlir.constant(1 : index) : !llvm.i64
-  %35 = llvm.insertvalue %21, %33[3, 0] : !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }">
-  %36 = llvm.insertvalue %34, %35[4, 0] : !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }">
+  %35 = llvm.insertvalue %21, %33[3, 0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>
+  %36 = llvm.insertvalue %34, %35[4, 0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>
   llvm.br ^bb1(%1 : !llvm.i64)
 ^bb1(%37: !llvm.i64):	// 2 preds: ^bb0, ^bb2
   %38 = llvm.icmp "slt" %37, %0 : !llvm.i64
   llvm.cond_br %38, ^bb2, ^bb3
 ^bb2:	// pred: ^bb1
-  %39 = llvm.extractvalue %20[1] : !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }">
+  %39 = llvm.extractvalue %20[1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>
   %40 = llvm.mlir.constant(0 : index) : !llvm.i64
   %41 = llvm.mlir.constant(1 : index) : !llvm.i64
   %42 = llvm.mul %37, %41 : !llvm.i64
   %43 = llvm.add %40, %42 : !llvm.i64
-  %44 = llvm.getelementptr %39[%43] : (!llvm<"float*">, !llvm.i64) -> !llvm<"float*">
-  llvm.store %3, %44 : !llvm<"float*">
-  %45 = llvm.extractvalue %36[1] : !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }">
+  %44 = llvm.getelementptr %39[%43] : (!llvm.ptr<float>, !llvm.i64) -> !llvm.ptr<float>
+  llvm.store %3, %44 : !llvm.ptr<float>
+  %45 = llvm.extractvalue %36[1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>
   %46 = llvm.mlir.constant(0 : index) : !llvm.i64
   %47 = llvm.mlir.constant(1 : index) : !llvm.i64
   %48 = llvm.mul %37, %47 : !llvm.i64
   %49 = llvm.add %46, %48 : !llvm.i64
-  %50 = llvm.getelementptr %45[%49] : (!llvm<"float*">, !llvm.i64) -> !llvm<"float*">
-  llvm.store %3, %50 : !llvm<"float*">
+  %50 = llvm.getelementptr %45[%49] : (!llvm.ptr<float>, !llvm.i64) -> !llvm.ptr<float>
+  llvm.store %3, %50 : !llvm.ptr<float>
   %51 = llvm.add %37, %2 : !llvm.i64
   llvm.br ^bb1(%51 : !llvm.i64)
 ^bb3:	// pred: ^bb1
   %52 = llvm.mlir.constant(1 : index) : !llvm.i64
   %53 = llvm.mlir.constant(1 : index) : !llvm.i64
-  %54 = llvm.extractvalue %20[1] : !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }">
-  %55 = llvm.extractvalue %36[1] : !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }">
-  llvm.call @simple_add1_add2_test(%54, %55) : (!llvm<"float*">, !llvm<"float*">) -> ()
-  %56 = llvm.extractvalue %20[1] : !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }">
+  %54 = llvm.extractvalue %20[1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>
+  %55 = llvm.extractvalue %36[1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>
+  llvm.call @simple_add1_add2_test(%54, %55) : (!llvm.ptr<float>, !llvm.ptr<float>) -> ()
+  %56 = llvm.extractvalue %20[1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>
   %57 = llvm.mlir.constant(0 : index) : !llvm.i64
   %58 = llvm.mlir.constant(1 : index) : !llvm.i64
   %59 = llvm.mul %1, %58 : !llvm.i64
   %60 = llvm.add %57, %59 : !llvm.i64
-  %61 = llvm.getelementptr %56[%60] : (!llvm<"float*">, !llvm.i64) -> !llvm<"float*">
-  %62 = llvm.load %61 : !llvm<"float*">
+  %61 = llvm.getelementptr %56[%60] : (!llvm.ptr<float>, !llvm.i64) -> !llvm.ptr<float>
+  %62 = llvm.load %61 : !llvm.ptr<float>
   llvm.call @print_f32(%62) : (!llvm.float) -> ()
   llvm.call @print_comma() : () -> ()
-  %63 = llvm.extractvalue %20[1] : !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }">
+  %63 = llvm.extractvalue %20[1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>
   %64 = llvm.mlir.constant(0 : index) : !llvm.i64
   %65 = llvm.mlir.constant(1 : index) : !llvm.i64
   %66 = llvm.mul %2, %65 : !llvm.i64
   %67 = llvm.add %64, %66 : !llvm.i64
-  %68 = llvm.getelementptr %63[%67] : (!llvm<"float*">, !llvm.i64) -> !llvm<"float*">
-  %69 = llvm.load %68 : !llvm<"float*">
+  %68 = llvm.getelementptr %63[%67] : (!llvm.ptr<float>, !llvm.i64) -> !llvm.ptr<float>
+  %69 = llvm.load %68 : !llvm.ptr<float>
   llvm.call @print_f32(%69) : (!llvm.float) -> ()
   llvm.call @print_newline() : () -> ()
-  %70 = llvm.extractvalue %36[1] : !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }">
+  %70 = llvm.extractvalue %36[1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>
   %71 = llvm.mlir.constant(0 : index) : !llvm.i64
   %72 = llvm.mlir.constant(1 : index) : !llvm.i64
   %73 = llvm.mul %1, %72 : !llvm.i64
   %74 = llvm.add %71, %73 : !llvm.i64
-  %75 = llvm.getelementptr %70[%74] : (!llvm<"float*">, !llvm.i64) -> !llvm<"float*">
-  %76 = llvm.load %75 : !llvm<"float*">
+  %75 = llvm.getelementptr %70[%74] : (!llvm.ptr<float>, !llvm.i64) -> !llvm.ptr<float>
+  %76 = llvm.load %75 : !llvm.ptr<float>
   llvm.call @print_f32(%76) : (!llvm.float) -> ()
   llvm.call @print_comma() : () -> ()
-  %77 = llvm.extractvalue %36[1] : !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }">
+  %77 = llvm.extractvalue %36[1] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>
   %78 = llvm.mlir.constant(0 : index) : !llvm.i64
   %79 = llvm.mlir.constant(1 : index) : !llvm.i64
   %80 = llvm.mul %2, %79 : !llvm.i64
   %81 = llvm.add %78, %80 : !llvm.i64
-  %82 = llvm.getelementptr %77[%81] : (!llvm<"float*">, !llvm.i64) -> !llvm<"float*">
-  %83 = llvm.load %82 : !llvm<"float*">
+  %82 = llvm.getelementptr %77[%81] : (!llvm.ptr<float>, !llvm.i64) -> !llvm.ptr<float>
+  %83 = llvm.load %82 : !llvm.ptr<float>
   llvm.call @print_f32(%83) : (!llvm.float) -> ()
   llvm.call @print_newline() : () -> ()
-  %84 = llvm.extractvalue %20[0] : !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }">
-  %85 = llvm.bitcast %84 : !llvm<"float*"> to !llvm<"i8*">
-  llvm.call @free(%85) : (!llvm<"i8*">) -> ()
-  %86 = llvm.extractvalue %36[0] : !llvm<"{ float*, float*, i64, [1 x i64], [1 x i64] }">
-  %87 = llvm.bitcast %86 : !llvm<"float*"> to !llvm<"i8*">
-  llvm.call @free(%87) : (!llvm<"i8*">) -> ()
+  %84 = llvm.extractvalue %20[0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>
+  %85 = llvm.bitcast %84 : !llvm.ptr<float> to !llvm.ptr<i8>
+  llvm.call @free(%85) : (!llvm.ptr<i8>) -> ()
+  %86 = llvm.extractvalue %36[0] : !llvm.struct<(ptr<float>, ptr<float>, i64, array<1 x i64>, array<1 x i64>)>
+  %87 = llvm.bitcast %86 : !llvm.ptr<float> to !llvm.ptr<i8>
+  llvm.call @free(%87) : (!llvm.ptr<i8>) -> ()
   llvm.return
 }
diff --git a/mlir/test/mlir-cpu-runner/simple.mlir b/mlir/test/mlir-cpu-runner/simple.mlir
index a1b6cf62dab67..e75d98aa8bd3c 100644
--- a/mlir/test/mlir-cpu-runner/simple.mlir
+++ b/mlir/test/mlir-cpu-runner/simple.mlir
@@ -15,8 +15,8 @@
 
 // Declarations of C library functions.
 llvm.func @fabsf(!llvm.float) -> !llvm.float
-llvm.func @malloc(!llvm.i64) -> !llvm<"i8*">
-llvm.func @free(!llvm<"i8*">)
+llvm.func @malloc(!llvm.i64) -> !llvm.ptr<i8>
+llvm.func @free(!llvm.ptr<i8>)
 
 // Check that a simple function with a nested call works.
 llvm.func @main() -> !llvm.float {
@@ -27,29 +27,29 @@ llvm.func @main() -> !llvm.float {
 // CHECK: 4.200000e+02
 
 // Helper typed functions wrapping calls to "malloc" and "free".
-llvm.func @allocation() -> !llvm<"float*"> {
+llvm.func @allocation() -> !llvm.ptr<float> {
   %0 = llvm.mlir.constant(4 : index) : !llvm.i64
-  %1 = llvm.call @malloc(%0) : (!llvm.i64) -> !llvm<"i8*">
-  %2 = llvm.bitcast %1 : !llvm<"i8*"> to !llvm<"float*">
-  llvm.return %2 : !llvm<"float*">
+  %1 = llvm.call @malloc(%0) : (!llvm.i64) -> !llvm.ptr<i8>
+  %2 = llvm.bitcast %1 : !llvm.ptr<i8> to !llvm.ptr<float>
+  llvm.return %2 : !llvm.ptr<float>
 }
-llvm.func @deallocation(%arg0: !llvm<"float*">) {
-  %0 = llvm.bitcast %arg0 : !llvm<"float*"> to !llvm<"i8*">
-  llvm.call @free(%0) : (!llvm<"i8*">) -> ()
+llvm.func @deallocation(%arg0: !llvm.ptr<float>) {
+  %0 = llvm.bitcast %arg0 : !llvm.ptr<float> to !llvm.ptr<i8>
+  llvm.call @free(%0) : (!llvm.ptr<i8>) -> ()
   llvm.return
 }
 
 // Check that allocation and deallocation works, and that a custom entry point
 // works.
 llvm.func @foo() -> !llvm.float {
-  %0 = llvm.call @allocation() : () -> !llvm<"float*">
+  %0 = llvm.call @allocation() : () -> !llvm.ptr<float>
   %1 = llvm.mlir.constant(0 : index) : !llvm.i64
   %2 = llvm.mlir.constant(1.234000e+03 : f32) : !llvm.float
-  %3 = llvm.getelementptr %0[%1] : (!llvm<"float*">, !llvm.i64) -> !llvm<"float*">
-  llvm.store %2, %3 : !llvm<"float*">
-  %4 = llvm.getelementptr %0[%1] : (!llvm<"float*">, !llvm.i64) -> !llvm<"float*">
-  %5 = llvm.load %4 : !llvm<"float*">
-  llvm.call @deallocation(%0) : (!llvm<"float*">) -> ()
+  %3 = llvm.getelementptr %0[%1] : (!llvm.ptr<float>, !llvm.i64) -> !llvm.ptr<float>
+  llvm.store %2, %3 : !llvm.ptr<float>
+  %4 = llvm.getelementptr %0[%1] : (!llvm.ptr<float>, !llvm.i64) -> !llvm.ptr<float>
+  %5 = llvm.load %4 : !llvm.ptr<float>
+  llvm.call @deallocation(%0) : (!llvm.ptr<float>) -> ()
   llvm.return %5 : !llvm.float
 }
 // NOMAIN: 1.234000e+03

From cb9f9df5f8239e291a62934b0f64eb795b26d84a Mon Sep 17 00:00:00 2001
From: Alex Zinenko <zinenko@google.com>
Date: Tue, 4 Aug 2020 14:42:17 +0200
Subject: [PATCH 1032/1035] [mlir] Fix GCC5 compilation problem in MLIR->LLVM
 type translation

GCC5 seems to dislike generic lambdas calling a method of the class
containing the lambda without explicit `this`.
---
 mlir/lib/Target/LLVMIR/TypeTranslation.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/mlir/lib/Target/LLVMIR/TypeTranslation.cpp b/mlir/lib/Target/LLVMIR/TypeTranslation.cpp
index 15e0f158007a5..b327e9ed8d2c4 100644
--- a/mlir/lib/Target/LLVMIR/TypeTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/TypeTranslation.cpp
@@ -72,7 +72,7 @@ class TypeToLLVMIRTranslator {
                   LLVM::LLVMFunctionType, LLVM::LLVMPointerType,
                   LLVM::LLVMStructType, LLVM::LLVMFixedVectorType,
                   LLVM::LLVMScalableVectorType>(
-                [this](auto array) { return translate(array); })
+                [this](auto type) { return this->translate(type); })
             .Default([](LLVM::LLVMType t) -> llvm::Type * {
               llvm_unreachable("unknown LLVM dialect type");
             });
@@ -187,7 +187,7 @@ class TypeFromLLVMIRTranslator {
             .Case<llvm::ArrayType, llvm::FunctionType, llvm::IntegerType,
                   llvm::PointerType, llvm::StructType, llvm::FixedVectorType,
                   llvm::ScalableVectorType>(
-                [this](auto *type) { return translate(type); })
+                [this](auto *type) { return this->translate(type); })
             .Default([this](llvm::Type *type) {
               return translatePrimitiveType(type);
             });

From 04e45ae1c6d2fdbf3fd4242df69d1511df757d48 Mon Sep 17 00:00:00 2001
From: Sanjay Patel <spatel@rotateright.com>
Date: Tue, 4 Aug 2020 08:28:44 -0400
Subject: [PATCH 1033/1035] [InstSimplify] fold nested min/max intrinsics with
 constant operands

This is based on the existing code for the non-intrinsic idioms
in InstCombine.

The vector constant constraint is non-obvious: undefs should be
ok in the outer call, but they can't propagate safely from the
inner call in all cases. Example:

https://alive2.llvm.org/ce/z/-2bVbM
  define <2 x i8> @src(<2 x i8> %x) {
  %0:
    %m = umin <2 x i8> %x, { 7, undef }
    %m2 = umin <2 x i8> { 9, 9 }, %m
    ret <2 x i8> %m2
  }
  =>
  define <2 x i8> @tgt(<2 x i8> %x) {
  %0:
    %m = umin <2 x i8> %x, { 7, undef }
    ret <2 x i8> %m
  }
  Transformation doesn't verify!
  ERROR: Value mismatch

  Example:
  <2 x i8> %x = < undef, undef >

  Source:
  <2 x i8> %m = < #x00 (0)	[based on undef value], #x00 (0) >
  <2 x i8> %m2 = < #x00 (0), #x00 (0) >

  Target:
  <2 x i8> %m = < #x07 (7), #x10 (16) >
  Source value: < #x00 (0), #x00 (0) >
  Target value: < #x07 (7), #x10 (16) >
---
 llvm/lib/Analysis/InstructionSimplify.cpp     | 15 ++++++
 .../InstSimplify/maxmin_intrinsics.ll         | 52 +++++++------------
 2 files changed, 35 insertions(+), 32 deletions(-)

diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp
index 2119ddcc7649b..6b8f8e3acc179 100644
--- a/llvm/lib/Analysis/InstructionSimplify.cpp
+++ b/llvm/lib/Analysis/InstructionSimplify.cpp
@@ -5285,6 +5285,21 @@ static Value *simplifyBinaryIntrinsic(Function *F, Value *Op0, Value *Op1,
       // umin(i8 %x, i8 255) --> %x
       if (*C == getMaxMinLimit(getMaxMinOpposite(IID), BitWidth))
         return Op0;
+
+      // Remove nested call if constant operands allow it. Example:
+      // max (max X, 7), 5 -> max X, 7
+      auto *MinMax0 = dyn_cast<IntrinsicInst>(Op0);
+      if (MinMax0 && MinMax0->getIntrinsicID() == IID) {
+        // TODO: loosen undef/splat restrictions for vector constants.
+        Value *M00 = MinMax0->getOperand(0), *M01 = MinMax0->getOperand(1);
+        const APInt *InnerC;
+        if ((match(M00, m_APInt(InnerC)) || match(M01, m_APInt(InnerC))) &&
+            ((IID == Intrinsic::smax && InnerC->sge(*C)) ||
+             (IID == Intrinsic::smin && InnerC->sle(*C)) ||
+             (IID == Intrinsic::umax && InnerC->uge(*C)) ||
+             (IID == Intrinsic::umin && InnerC->ule(*C))))
+          return Op0;
+      }
     }
 
     break;
diff --git a/llvm/test/Transforms/InstSimplify/maxmin_intrinsics.ll b/llvm/test/Transforms/InstSimplify/maxmin_intrinsics.ll
index 3aa19e91e0e38..6b10853dd78f2 100644
--- a/llvm/test/Transforms/InstSimplify/maxmin_intrinsics.ll
+++ b/llvm/test/Transforms/InstSimplify/maxmin_intrinsics.ll
@@ -627,8 +627,7 @@ define i8 @umin_smin(i8 %x, i8 %y) {
 define i8 @umax_umax_constants(i8 %x) {
 ; CHECK-LABEL: @umax_umax_constants(
 ; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.umax.i8(i8 [[X:%.*]], i8 9)
-; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.umax.i8(i8 7, i8 [[M]])
-; CHECK-NEXT:    ret i8 [[M2]]
+; CHECK-NEXT:    ret i8 [[M]]
 ;
   %m = call i8 @llvm.umax.i8(i8 %x, i8 9)
   %m2 = call i8 @llvm.umax.i8(i8 7, i8 %m)
@@ -638,8 +637,7 @@ define i8 @umax_umax_constants(i8 %x) {
 define i8 @umax_umax_constants_commute1(i8 %x) {
 ; CHECK-LABEL: @umax_umax_constants_commute1(
 ; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.umax.i8(i8 -128, i8 [[X:%.*]])
-; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.umax.i8(i8 7, i8 [[M]])
-; CHECK-NEXT:    ret i8 [[M2]]
+; CHECK-NEXT:    ret i8 [[M]]
 ;
   %m = call i8 @llvm.umax.i8(i8 128, i8 %x)
   %m2 = call i8 @llvm.umax.i8(i8 7, i8 %m)
@@ -649,8 +647,7 @@ define i8 @umax_umax_constants_commute1(i8 %x) {
 define i8 @umax_umax_constants_commute2(i8 %x) {
 ; CHECK-LABEL: @umax_umax_constants_commute2(
 ; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.umax.i8(i8 [[X:%.*]], i8 -56)
-; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.umax.i8(i8 [[M]], i8 127)
-; CHECK-NEXT:    ret i8 [[M2]]
+; CHECK-NEXT:    ret i8 [[M]]
 ;
   %m = call i8 @llvm.umax.i8(i8 %x, i8 200)
   %m2 = call i8 @llvm.umax.i8(i8 %m, i8 127)
@@ -660,8 +657,7 @@ define i8 @umax_umax_constants_commute2(i8 %x) {
 define <2 x i8> @umax_umax_constants_commute3(<2 x i8> %x) {
 ; CHECK-LABEL: @umax_umax_constants_commute3(
 ; CHECK-NEXT:    [[M:%.*]] = call <2 x i8> @llvm.umax.v2i8(<2 x i8> <i8 -2, i8 -2>, <2 x i8> [[X:%.*]])
-; CHECK-NEXT:    [[M2:%.*]] = call <2 x i8> @llvm.umax.v2i8(<2 x i8> [[M]], <2 x i8> <i8 -128, i8 -128>)
-; CHECK-NEXT:    ret <2 x i8> [[M2]]
+; CHECK-NEXT:    ret <2 x i8> [[M]]
 ;
   %m = call <2 x i8> @llvm.umax.v2i8(<2 x i8> <i8 254, i8 254>, <2 x i8> %x)
   %m2 = call <2 x i8> @llvm.umax.v2i8(<2 x i8> %m, <2 x i8> <i8 128, i8 128>)
@@ -671,8 +667,7 @@ define <2 x i8> @umax_umax_constants_commute3(<2 x i8> %x) {
 define i8 @umin_umin_constants(i8 %x) {
 ; CHECK-LABEL: @umin_umin_constants(
 ; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.umin.i8(i8 [[X:%.*]], i8 7)
-; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.umin.i8(i8 9, i8 [[M]])
-; CHECK-NEXT:    ret i8 [[M2]]
+; CHECK-NEXT:    ret i8 [[M]]
 ;
   %m = call i8 @llvm.umin.i8(i8 %x, i8 7)
   %m2 = call i8 @llvm.umin.i8(i8 9, i8 %m)
@@ -682,8 +677,7 @@ define i8 @umin_umin_constants(i8 %x) {
 define i8 @umin_umin_constants_commute1(i8 %x) {
 ; CHECK-LABEL: @umin_umin_constants_commute1(
 ; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.umin.i8(i8 7, i8 [[X:%.*]])
-; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.umin.i8(i8 -128, i8 [[M]])
-; CHECK-NEXT:    ret i8 [[M2]]
+; CHECK-NEXT:    ret i8 [[M]]
 ;
   %m = call i8 @llvm.umin.i8(i8 7, i8 %x)
   %m2 = call i8 @llvm.umin.i8(i8 128, i8 %m)
@@ -693,8 +687,7 @@ define i8 @umin_umin_constants_commute1(i8 %x) {
 define <2 x i8> @umin_umin_constants_commute2(<2 x i8> %x) {
 ; CHECK-LABEL: @umin_umin_constants_commute2(
 ; CHECK-NEXT:    [[M:%.*]] = call <2 x i8> @llvm.umin.v2i8(<2 x i8> [[X:%.*]], <2 x i8> <i8 127, i8 127>)
-; CHECK-NEXT:    [[M2:%.*]] = call <2 x i8> @llvm.umin.v2i8(<2 x i8> [[M]], <2 x i8> <i8 -56, i8 undef>)
-; CHECK-NEXT:    ret <2 x i8> [[M2]]
+; CHECK-NEXT:    ret <2 x i8> [[M]]
 ;
   %m = call <2 x i8> @llvm.umin.v2i8(<2 x i8> %x, <2 x i8> <i8 127, i8 127>)
   %m2 = call <2 x i8> @llvm.umin.v2i8(<2 x i8> %m, <2 x i8> <i8 200, i8 undef>)
@@ -704,8 +697,7 @@ define <2 x i8> @umin_umin_constants_commute2(<2 x i8> %x) {
 define i8 @umin_umin_constants_commute3(i8 %x) {
 ; CHECK-LABEL: @umin_umin_constants_commute3(
 ; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.umin.i8(i8 -128, i8 [[X:%.*]])
-; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.umin.i8(i8 [[M]], i8 -2)
-; CHECK-NEXT:    ret i8 [[M2]]
+; CHECK-NEXT:    ret i8 [[M]]
 ;
   %m = call i8 @llvm.umin.i8(i8 128, i8 %x)
   %m2 = call i8 @llvm.umin.i8(i8 %m, i8 254)
@@ -715,8 +707,7 @@ define i8 @umin_umin_constants_commute3(i8 %x) {
 define i8 @smax_smax_constants(i8 %x) {
 ; CHECK-LABEL: @smax_smax_constants(
 ; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.smax.i8(i8 [[X:%.*]], i8 9)
-; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.smax.i8(i8 7, i8 [[M]])
-; CHECK-NEXT:    ret i8 [[M2]]
+; CHECK-NEXT:    ret i8 [[M]]
 ;
   %m = call i8 @llvm.smax.i8(i8 %x, i8 9)
   %m2 = call i8 @llvm.smax.i8(i8 7, i8 %m)
@@ -726,8 +717,7 @@ define i8 @smax_smax_constants(i8 %x) {
 define <2 x i8> @smax_smax_constants_commute1(<2 x i8> %x) {
 ; CHECK-LABEL: @smax_smax_constants_commute1(
 ; CHECK-NEXT:    [[M:%.*]] = call <2 x i8> @llvm.smax.v2i8(<2 x i8> <i8 7, i8 7>, <2 x i8> [[X:%.*]])
-; CHECK-NEXT:    [[M2:%.*]] = call <2 x i8> @llvm.smax.v2i8(<2 x i8> <i8 -127, i8 -127>, <2 x i8> [[M]])
-; CHECK-NEXT:    ret <2 x i8> [[M2]]
+; CHECK-NEXT:    ret <2 x i8> [[M]]
 ;
   %m = call <2 x i8> @llvm.smax.v2i8(<2 x i8> <i8 7, i8 7>, <2 x i8> %x)
   %m2 = call <2 x i8> @llvm.smax.v2i8(<2 x i8> <i8 -127, i8 -127>, <2 x i8> %m)
@@ -737,8 +727,7 @@ define <2 x i8> @smax_smax_constants_commute1(<2 x i8> %x) {
 define i8 @smax_smax_constants_commute2(i8 %x) {
 ; CHECK-LABEL: @smax_smax_constants_commute2(
 ; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.smax.i8(i8 [[X:%.*]], i8 0)
-; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.smax.i8(i8 [[M]], i8 -1)
-; CHECK-NEXT:    ret i8 [[M2]]
+; CHECK-NEXT:    ret i8 [[M]]
 ;
   %m = call i8 @llvm.smax.i8(i8 %x, i8 0)
   %m2 = call i8 @llvm.smax.i8(i8 %m, i8 -1)
@@ -748,8 +737,7 @@ define i8 @smax_smax_constants_commute2(i8 %x) {
 define i8 @smax_smax_constants_commute3(i8 %x) {
 ; CHECK-LABEL: @smax_smax_constants_commute3(
 ; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.smax.i8(i8 -1, i8 [[X:%.*]])
-; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.smax.i8(i8 [[M]], i8 -127)
-; CHECK-NEXT:    ret i8 [[M2]]
+; CHECK-NEXT:    ret i8 [[M]]
 ;
   %m = call i8 @llvm.smax.i8(i8 -1, i8 %x)
   %m2 = call i8 @llvm.smax.i8(i8 %m, i8 -127)
@@ -759,8 +747,7 @@ define i8 @smax_smax_constants_commute3(i8 %x) {
 define <2 x i8> @smin_smin_constants(<2 x i8> %x) {
 ; CHECK-LABEL: @smin_smin_constants(
 ; CHECK-NEXT:    [[M:%.*]] = call <2 x i8> @llvm.smin.v2i8(<2 x i8> [[X:%.*]], <2 x i8> <i8 7, i8 7>)
-; CHECK-NEXT:    [[M2:%.*]] = call <2 x i8> @llvm.smin.v2i8(<2 x i8> <i8 undef, i8 9>, <2 x i8> [[M]])
-; CHECK-NEXT:    ret <2 x i8> [[M2]]
+; CHECK-NEXT:    ret <2 x i8> [[M]]
 ;
   %m = call <2 x i8> @llvm.smin.v2i8(<2 x i8> %x, <2 x i8> <i8 7, i8 7>)
   %m2 = call <2 x i8> @llvm.smin.v2i8(<2 x i8> <i8 undef, i8 9>, <2 x i8> %m)
@@ -770,8 +757,7 @@ define <2 x i8> @smin_smin_constants(<2 x i8> %x) {
 define i8 @smin_smin_constants_commute1(i8 %x) {
 ; CHECK-LABEL: @smin_smin_constants_commute1(
 ; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.smin.i8(i8 -127, i8 [[X:%.*]])
-; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.smin.i8(i8 7, i8 [[M]])
-; CHECK-NEXT:    ret i8 [[M2]]
+; CHECK-NEXT:    ret i8 [[M]]
 ;
   %m = call i8 @llvm.smin.i8(i8 -127, i8 %x)
   %m2 = call i8 @llvm.smin.i8(i8 7, i8 %m)
@@ -781,8 +767,7 @@ define i8 @smin_smin_constants_commute1(i8 %x) {
 define i8 @smin_smin_constants_commute2(i8 %x) {
 ; CHECK-LABEL: @smin_smin_constants_commute2(
 ; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.smin.i8(i8 [[X:%.*]], i8 -1)
-; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.smin.i8(i8 [[M]], i8 0)
-; CHECK-NEXT:    ret i8 [[M2]]
+; CHECK-NEXT:    ret i8 [[M]]
 ;
   %m = call i8 @llvm.smin.i8(i8 %x, i8 -1)
   %m2 = call i8 @llvm.smin.i8(i8 %m, i8 0)
@@ -792,14 +777,15 @@ define i8 @smin_smin_constants_commute2(i8 %x) {
 define i8 @smin_smin_constants_commute3(i8 %x) {
 ; CHECK-LABEL: @smin_smin_constants_commute3(
 ; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.smin.i8(i8 -127, i8 [[X:%.*]])
-; CHECK-NEXT:    [[M2:%.*]] = call i8 @llvm.smin.i8(i8 [[M]], i8 -1)
-; CHECK-NEXT:    ret i8 [[M2]]
+; CHECK-NEXT:    ret i8 [[M]]
 ;
   %m = call i8 @llvm.smin.i8(i8 -127, i8 %x)
   %m2 = call i8 @llvm.smin.i8(i8 %m, i8 -1)
   ret i8 %m2
 }
 
+; Negative test - undef in inner constant must not propagate.
+
 define <2 x i8> @umin_umin_constants_partial_undef(<2 x i8> %x) {
 ; CHECK-LABEL: @umin_umin_constants_partial_undef(
 ; CHECK-NEXT:    [[M:%.*]] = call <2 x i8> @llvm.umin.v2i8(<2 x i8> [[X:%.*]], <2 x i8> <i8 7, i8 undef>)
@@ -811,6 +797,8 @@ define <2 x i8> @umin_umin_constants_partial_undef(<2 x i8> %x) {
   ret <2 x i8> %m2
 }
 
+; Negative test - undef in inner constant must not propagate.
+
 define <2 x i8> @smax_smax_constants_partial_undef(<2 x i8> %x) {
 ; CHECK-LABEL: @smax_smax_constants_partial_undef(
 ; CHECK-NEXT:    [[M:%.*]] = call <2 x i8> @llvm.smax.v2i8(<2 x i8> [[X:%.*]], <2 x i8> <i8 undef, i8 10>)

From 1a4263d394c1a93757613bde4b1c2cf8d6a7bbb9 Mon Sep 17 00:00:00 2001
From: Nicolas Vasilache <ntv@google.com>
Date: Mon, 3 Aug 2020 05:34:07 -0400
Subject: [PATCH 1034/1035] [mlir][Vector] Add linalg.copy-based pattern for
 splitting vector.transfer_read into full and partial copies.

This revision adds a transformation and a pattern that rewrites a "maybe masked" `vector.transfer_read %view[...], %pad `into a pattern resembling:

```
   %1:3 = scf.if (%inBounds) {
      scf.yield %view : memref<A...>, index, index
    } else {
      %2 = linalg.fill(%extra_alloc, %pad)
      %3 = subview %view [...][...][...]
      linalg.copy(%3, %alloc)
      memref_cast %extra_alloc: memref<B...> to memref<A...>
      scf.yield %4 : memref<A...>, index, index
   }
   %res= vector.transfer_read %1#0[%1#1, %1#2] {masked = [false ... false]}
```
where `extra_alloc` is a top of the function alloca'ed buffer of one vector.

This rewrite makes it possible to realize the "always full tile" abstraction where vector.transfer_read operations are guaranteed to read from a padded full buffer.
The extra work only occurs on the boundary tiles.
---
 mlir/include/mlir/Dialect/Vector/VectorOps.h  |  32 ++-
 .../mlir/Dialect/Vector/VectorTransforms.h    |  42 ++--
 mlir/lib/Dialect/Vector/CMakeLists.txt        |   1 +
 mlir/lib/Dialect/Vector/VectorTransforms.cpp  | 238 ++++++++++++++----
 .../vector-transfer-full-partial-split.mlir   |  94 ++++++-
 .../lib/Transforms/TestVectorTransforms.cpp   |  15 +-
 6 files changed, 348 insertions(+), 74 deletions(-)

diff --git a/mlir/include/mlir/Dialect/Vector/VectorOps.h b/mlir/include/mlir/Dialect/Vector/VectorOps.h
index edf9557df389b..562e07f98774d 100644
--- a/mlir/include/mlir/Dialect/Vector/VectorOps.h
+++ b/mlir/include/mlir/Dialect/Vector/VectorOps.h
@@ -56,22 +56,48 @@ enum class VectorContractLowering {
 };
 /// Enum to control the lowering of `vector.transpose` operations.
 enum class VectorTransposeLowering {
-  // Lower transpose into element-wise extract and inserts.
+  /// Lower transpose into element-wise extract and inserts.
   EltWise = 0,
   /// Lower 2-D transpose to `vector.flat_transpose`, maps 1-1 to LLVM matrix
   /// intrinsics.
   Flat = 1,
 };
+/// Enum to control the splitting of `vector.transfer` operations into masked
+/// and unmasked variants.
+enum class VectorTransferSplit {
+  /// Do not split vector transfer operations.
+  None = 0,
+  /// Split using masked + unmasked vector.transfer operations.
+  VectorTransfer = 1,
+  /// Split using a unmasked vector.transfer + linalg.fill + linalg.copy
+  /// operations.
+  LinalgCopy = 2,
+  /// Do not split vector transfer operation but instead mark it as "unmasked".
+  ForceUnmasked = 3
+};
 /// Structure to control the behavior of vector transform patterns.
 struct VectorTransformsOptions {
+  /// Option to control the lowering of vector.contract.
   VectorContractLowering vectorContractLowering = VectorContractLowering::Dot;
-  VectorTransposeLowering vectorTransposeLowering =
-      VectorTransposeLowering::EltWise;
   VectorTransformsOptions &
   setVectorTransformsOptions(VectorContractLowering opt) {
     vectorContractLowering = opt;
     return *this;
   }
+  /// Option to control the lowering of vector.transpose.
+  VectorTransposeLowering vectorTransposeLowering =
+      VectorTransposeLowering::EltWise;
+  VectorTransformsOptions &
+  setVectorTransposeLowering(VectorTransposeLowering opt) {
+    vectorTransposeLowering = opt;
+    return *this;
+  }
+  /// Option to control the splitting of vector transfers.
+  VectorTransferSplit vectorTransferSplit = VectorTransferSplit::None;
+  VectorTransformsOptions &setVectorTransferSplit(VectorTransferSplit opt) {
+    vectorTransferSplit = opt;
+    return *this;
+  }
 };
 
 /// Collect a set of transformation patterns that are related to contracting
diff --git a/mlir/include/mlir/Dialect/Vector/VectorTransforms.h b/mlir/include/mlir/Dialect/Vector/VectorTransforms.h
index 835ad18a79ad2..e6c7b7abebd53 100644
--- a/mlir/include/mlir/Dialect/Vector/VectorTransforms.h
+++ b/mlir/include/mlir/Dialect/Vector/VectorTransforms.h
@@ -109,13 +109,13 @@ struct UnrollVectorPattern : public OpRewritePattern<OpTy> {
   FilterConstraintType filter;
 };
 
-/// Split a vector.transfer operation into an unmasked fastpath vector.transfer
-/// and a slowpath masked vector.transfer. If `ifOp` is not null and the result
-/// is `success, the `ifOp` points to the newly created conditional upon
-/// function return. To accomodate for the fact that the original
-/// vector.transfer indexing may be arbitrary and the slow path indexes @[0...0]
-/// in the temporary buffer, the scf.if op returns a view and values of type
-/// index. At this time, only vector.transfer_read is implemented.
+/// Split a vector.transfer operation into an unmasked fastpath and a slowpath.
+/// If `ifOp` is not null and the result is `success, the `ifOp` points to the
+/// newly created conditional upon function return.
+/// To accomodate for the fact that the original vector.transfer indexing may be
+/// arbitrary and the slow path indexes @[0...0] in the temporary buffer, the
+/// scf.if op returns a view and values of type index.
+/// At this time, only vector.transfer_read case is implemented.
 ///
 /// Example (a 2-D vector.transfer_read):
 /// ```
@@ -124,17 +124,17 @@ struct UnrollVectorPattern : public OpRewritePattern<OpTy> {
 /// is transformed into:
 /// ```
 ///    %1:3 = scf.if (%inBounds) {
-///       scf.yield %0 : memref<A...>, index, index
-///     } else {
-///       %2 = vector.transfer_read %0[...], %pad : memref<A...>, vector<...>
-///       %3 = vector.type_cast %extra_alloc : memref<...> to
-///       memref<vector<...>> store %2, %3[] : memref<vector<...>> %4 =
-///       memref_cast %extra_alloc: memref<B...> to memref<A...> scf.yield %4 :
-///       memref<A...>, index, index
+///      // fastpath, direct cast
+///      memref_cast %A: memref<A...> to compatibleMemRefType
+///      scf.yield %view : compatibleMemRefType, index, index
+///    } else {
+///      // slowpath, masked vector.transfer or linalg.copy.
+///      memref_cast %alloc: memref<B...> to compatibleMemRefType
+///      scf.yield %4 : compatibleMemRefType, index, index
 //     }
 ///    %0 = vector.transfer_read %1#0[%1#1, %1#2] {masked = [false ... false]}
 /// ```
-/// where `extra_alloc` is a top of the function alloca'ed buffer of one vector.
+/// where `alloc` is a top of the function alloca'ed buffer of one vector.
 ///
 /// Preconditions:
 ///  1. `xferOp.permutation_map()` must be a minor identity map
@@ -143,9 +143,10 @@ struct UnrollVectorPattern : public OpRewritePattern<OpTy> {
 ///  rank-reducing subviews.
 LogicalResult
 splitFullAndPartialTransferPrecondition(VectorTransferOpInterface xferOp);
-LogicalResult splitFullAndPartialTransfer(OpBuilder &b,
-                                          VectorTransferOpInterface xferOp,
-                                          scf::IfOp *ifOp = nullptr);
+LogicalResult splitFullAndPartialTransfer(
+    OpBuilder &b, VectorTransferOpInterface xferOp,
+    VectorTransformsOptions options = VectorTransformsOptions(),
+    scf::IfOp *ifOp = nullptr);
 
 /// Apply `splitFullAndPartialTransfer` selectively via a pattern. This pattern
 /// may take an extra filter to perform selection at a finer granularity.
@@ -155,16 +156,19 @@ struct VectorTransferFullPartialRewriter : public RewritePattern {
 
   explicit VectorTransferFullPartialRewriter(
       MLIRContext *context,
+      VectorTransformsOptions options = VectorTransformsOptions(),
       FilterConstraintType filter =
           [](VectorTransferOpInterface op) { return success(); },
       PatternBenefit benefit = 1)
-      : RewritePattern(benefit, MatchAnyOpTypeTag()), filter(filter) {}
+      : RewritePattern(benefit, MatchAnyOpTypeTag()), options(options),
+        filter(filter) {}
 
   /// Performs the rewrite.
   LogicalResult matchAndRewrite(Operation *op,
                                 PatternRewriter &rewriter) const override;
 
 private:
+  VectorTransformsOptions options;
   FilterConstraintType filter;
 };
 
diff --git a/mlir/lib/Dialect/Vector/CMakeLists.txt b/mlir/lib/Dialect/Vector/CMakeLists.txt
index 13dbf6da73fa2..1087feba7fbdb 100644
--- a/mlir/lib/Dialect/Vector/CMakeLists.txt
+++ b/mlir/lib/Dialect/Vector/CMakeLists.txt
@@ -16,6 +16,7 @@ add_mlir_dialect_library(MLIRVector
   MLIRIR
   MLIRStandardOps
   MLIRAffineOps
+  MLIRLinalgOps
   MLIRSCF
   MLIRLoopAnalysis
   MLIRSideEffectInterfaces
diff --git a/mlir/lib/Dialect/Vector/VectorTransforms.cpp b/mlir/lib/Dialect/Vector/VectorTransforms.cpp
index 573b822503f3a..3c23c5a6d869d 100644
--- a/mlir/lib/Dialect/Vector/VectorTransforms.cpp
+++ b/mlir/lib/Dialect/Vector/VectorTransforms.cpp
@@ -14,6 +14,7 @@
 
 #include "mlir/Dialect/Affine/EDSC/Intrinsics.h"
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
+#include "mlir/Dialect/Linalg/EDSC/Intrinsics.h"
 #include "mlir/Dialect/SCF/EDSC/Intrinsics.h"
 #include "mlir/Dialect/StandardOps/EDSC/Intrinsics.h"
 #include "mlir/Dialect/StandardOps/IR/Ops.h"
@@ -2056,7 +2057,16 @@ LogicalResult mlir::vector::splitFullAndPartialTransferPrecondition(
   return success();
 }
 
-MemRefType getCastCompatibleMemRefType(MemRefType aT, MemRefType bT) {
+/// Given two MemRefTypes `aT` and `bT`, return a MemRefType to which both can
+/// be cast. If the MemRefTypes don't have the same rank or are not strided,
+/// return null; otherwise:
+///   1. if `aT` and `bT` are cast-compatible, return `aT`.
+///   2. else return a new MemRefType obtained by iterating over the shape and
+///   strides and:
+///     a. keeping the ones that are static and equal across `aT` and `bT`.
+///     b. using a dynamic shape and/or stride for the dimeniosns that don't
+///        agree.
+static MemRefType getCastCompatibleMemRefType(MemRefType aT, MemRefType bT) {
   if (MemRefCastOp::areCastCompatible(aT, bT))
     return aT;
   if (aT.getRank() != bT.getRank())
@@ -2086,13 +2096,154 @@ MemRefType getCastCompatibleMemRefType(MemRefType aT, MemRefType bT) {
       makeStridedLinearLayoutMap(resStrides, resOffset, aT.getContext()));
 }
 
-/// Split a vector.transfer operation into an unmasked fastpath vector.transfer
-/// and a slowpath masked vector.transfer. If `ifOp` is not null and the result
-/// is `success, the `ifOp` points to the newly created conditional upon
-/// function return. To accomodate for the fact that the original
-/// vector.transfer indexing may be arbitrary and the slow path indexes @[0...0]
-/// in the temporary buffer, the scf.if op returns a view and values of type
-/// index. At this time, only vector.transfer_read is implemented.
+/// Operates under a scoped context to build the intersection between the
+/// view `xferOp.memref()` @ `xferOp.indices()` and the view `alloc`.
+// TODO: view intersection/union/differences should be a proper std op.
+static Value createScopedSubViewIntersection(VectorTransferOpInterface xferOp,
+                                             Value alloc) {
+  using namespace edsc::intrinsics;
+  int64_t memrefRank = xferOp.getMemRefType().getRank();
+  // TODO: relax this precondition, will require rank-reducing subviews.
+  assert(memrefRank == alloc.getType().cast<MemRefType>().getRank() &&
+         "Expected memref rank to match the alloc rank");
+  Value one = std_constant_index(1);
+  ValueRange leadingIndices =
+      xferOp.indices().take_front(xferOp.getLeadingMemRefRank());
+  SmallVector<Value, 4> sizes;
+  sizes.append(leadingIndices.begin(), leadingIndices.end());
+  xferOp.zipResultAndIndexing([&](int64_t resultIdx, int64_t indicesIdx) {
+    using MapList = ArrayRef<ArrayRef<AffineExpr>>;
+    Value dimMemRef = std_dim(xferOp.memref(), indicesIdx);
+    Value dimAlloc = std_dim(alloc, resultIdx);
+    Value index = xferOp.indices()[indicesIdx];
+    AffineExpr i, j, k;
+    bindDims(xferOp.getContext(), i, j, k);
+    SmallVector<AffineMap, 4> maps =
+        AffineMap::inferFromExprList(MapList{{i - j, k}});
+    // affine_min(%dimMemRef - %index, %dimAlloc)
+    Value affineMin = affine_min(index.getType(), maps[0],
+                                 ValueRange{dimMemRef, index, dimAlloc});
+    sizes.push_back(affineMin);
+  });
+  return std_sub_view(xferOp.memref(), xferOp.indices(), sizes,
+                      SmallVector<Value, 4>(memrefRank, one));
+}
+
+/// Given an `xferOp` for which:
+///   1. `inBoundsCond` and a `compatibleMemRefType` have been computed.
+///   2. a memref of single vector `alloc` has been allocated.
+/// Produce IR resembling:
+/// ```
+///    %1:3 = scf.if (%inBounds) {
+///      memref_cast %A: memref<A...> to compatibleMemRefType
+///      scf.yield %view, ... : compatibleMemRefType, index, index
+///    } else {
+///      %2 = linalg.fill(%alloc, %pad)
+///      %3 = subview %view [...][...][...]
+///      linalg.copy(%3, %alloc)
+///      memref_cast %alloc: memref<B...> to compatibleMemRefType
+///      scf.yield %4, ... : compatibleMemRefType, index, index
+///   }
+/// ```
+/// Return the produced scf::IfOp.
+static scf::IfOp createScopedFullPartialLinalgCopy(
+    vector::TransferReadOp xferOp, TypeRange returnTypes, Value inBoundsCond,
+    MemRefType compatibleMemRefType, Value alloc) {
+  using namespace edsc;
+  using namespace edsc::intrinsics;
+  scf::IfOp fullPartialIfOp;
+  Value zero = std_constant_index(0);
+  Value memref = xferOp.memref();
+  conditionBuilder(
+      returnTypes, inBoundsCond,
+      [&]() -> scf::ValueVector {
+        Value res = memref;
+        if (compatibleMemRefType != xferOp.getMemRefType())
+          res = std_memref_cast(memref, compatibleMemRefType);
+        scf::ValueVector viewAndIndices{res};
+        viewAndIndices.insert(viewAndIndices.end(), xferOp.indices().begin(),
+                              xferOp.indices().end());
+        return viewAndIndices;
+      },
+      [&]() -> scf::ValueVector {
+        linalg_fill(alloc, xferOp.padding());
+        // Take partial subview of memref which guarantees no dimension
+        // overflows.
+        Value memRefSubView = createScopedSubViewIntersection(
+            cast<VectorTransferOpInterface>(xferOp.getOperation()), alloc);
+        linalg_copy(memRefSubView, alloc);
+        Value casted = std_memref_cast(alloc, compatibleMemRefType);
+        scf::ValueVector viewAndIndices{casted};
+        viewAndIndices.insert(viewAndIndices.end(), xferOp.getTransferRank(),
+                              zero);
+        return viewAndIndices;
+      },
+      &fullPartialIfOp);
+  return fullPartialIfOp;
+}
+
+/// Given an `xferOp` for which:
+///   1. `inBoundsCond` and a `compatibleMemRefType` have been computed.
+///   2. a memref of single vector `alloc` has been allocated.
+/// Produce IR resembling:
+/// ```
+///    %1:3 = scf.if (%inBounds) {
+///      memref_cast %A: memref<A...> to compatibleMemRefType
+///      scf.yield %view, ... : compatibleMemRefType, index, index
+///    } else {
+///      %2 = vector.transfer_read %view[...], %pad : memref<A...>, vector<...>
+///      %3 = vector.type_cast %extra_alloc :
+///        memref<...> to memref<vector<...>>
+///      store %2, %3[] : memref<vector<...>>
+///      %4 = memref_cast %alloc: memref<B...> to compatibleMemRefType
+///      scf.yield %4, ... : compatibleMemRefType, index, index
+///   }
+/// ```
+/// Return the produced scf::IfOp.
+static scf::IfOp createScopedFullPartialVectorTransferRead(
+    vector::TransferReadOp xferOp, TypeRange returnTypes, Value inBoundsCond,
+    MemRefType compatibleMemRefType, Value alloc) {
+  using namespace edsc;
+  using namespace edsc::intrinsics;
+  scf::IfOp fullPartialIfOp;
+  Value zero = std_constant_index(0);
+  Value memref = xferOp.memref();
+  conditionBuilder(
+      returnTypes, inBoundsCond,
+      [&]() -> scf::ValueVector {
+        Value res = memref;
+        if (compatibleMemRefType != xferOp.getMemRefType())
+          res = std_memref_cast(memref, compatibleMemRefType);
+        scf::ValueVector viewAndIndices{res};
+        viewAndIndices.insert(viewAndIndices.end(), xferOp.indices().begin(),
+                              xferOp.indices().end());
+        return viewAndIndices;
+      },
+      [&]() -> scf::ValueVector {
+        Operation *newXfer =
+            ScopedContext::getBuilderRef().clone(*xferOp.getOperation());
+        Value vector = cast<VectorTransferOpInterface>(newXfer).vector();
+        std_store(vector, vector_type_cast(
+                              MemRefType::get({}, vector.getType()), alloc));
+
+        Value casted = std_memref_cast(alloc, compatibleMemRefType);
+        scf::ValueVector viewAndIndices{casted};
+        viewAndIndices.insert(viewAndIndices.end(), xferOp.getTransferRank(),
+                              zero);
+
+        return viewAndIndices;
+      },
+      &fullPartialIfOp);
+  return fullPartialIfOp;
+}
+
+/// Split a vector.transfer operation into an unmasked fastpath and a slowpath.
+/// If `ifOp` is not null and the result is `success, the `ifOp` points to the
+/// newly created conditional upon function return.
+/// To accomodate for the fact that the original vector.transfer indexing may be
+/// arbitrary and the slow path indexes @[0...0] in the temporary buffer, the
+/// scf.if op returns a view and values of type index.
+/// At this time, only vector.transfer_read case is implemented.
 ///
 /// Example (a 2-D vector.transfer_read):
 /// ```
@@ -2101,17 +2252,17 @@ MemRefType getCastCompatibleMemRefType(MemRefType aT, MemRefType bT) {
 /// is transformed into:
 /// ```
 ///    %1:3 = scf.if (%inBounds) {
-///       scf.yield %0 : memref<A...>, index, index
-///     } else {
-///       %2 = vector.transfer_read %0[...], %pad : memref<A...>, vector<...>
-///       %3 = vector.type_cast %extra_alloc : memref<...> to
-///       memref<vector<...>> store %2, %3[] : memref<vector<...>> %4 =
-///       memref_cast %extra_alloc: memref<B...> to memref<A...> scf.yield %4 :
-///       memref<A...>, index, index
+///      // fastpath, direct cast
+///      memref_cast %A: memref<A...> to compatibleMemRefType
+///      scf.yield %view : compatibleMemRefType, index, index
+///    } else {
+///      // slowpath, masked vector.transfer or linalg.copy.
+///      memref_cast %alloc: memref<B...> to compatibleMemRefType
+///      scf.yield %4 : compatibleMemRefType, index, index
 //     }
 ///    %0 = vector.transfer_read %1#0[%1#1, %1#2] {masked = [false ... false]}
 /// ```
-/// where `extra_alloc` is a top of the function alloca'ed buffer of one vector.
+/// where `alloc` is a top of the function alloca'ed buffer of one vector.
 ///
 /// Preconditions:
 ///  1. `xferOp.permutation_map()` must be a minor identity map
@@ -2119,10 +2270,21 @@ MemRefType getCastCompatibleMemRefType(MemRefType aT, MemRefType bT) {
 ///  must be equal. This will be relaxed in the future but requires
 ///  rank-reducing subviews.
 LogicalResult mlir::vector::splitFullAndPartialTransfer(
-    OpBuilder &b, VectorTransferOpInterface xferOp, scf::IfOp *ifOp) {
+    OpBuilder &b, VectorTransferOpInterface xferOp,
+    VectorTransformsOptions options, scf::IfOp *ifOp) {
   using namespace edsc;
   using namespace edsc::intrinsics;
 
+  if (options.vectorTransferSplit == VectorTransferSplit::None)
+    return failure();
+
+  SmallVector<bool, 4> bools(xferOp.getTransferRank(), false);
+  auto unmaskedAttr = b.getBoolArrayAttr(bools);
+  if (options.vectorTransferSplit == VectorTransferSplit::ForceUnmasked) {
+    xferOp.setAttr(vector::TransferReadOp::getMaskedAttrName(), unmaskedAttr);
+    return success();
+  }
+
   assert(succeeded(splitFullAndPartialTransferPrecondition(xferOp)) &&
          "Expected splitFullAndPartialTransferPrecondition to hold");
   auto xferReadOp = dyn_cast<vector::TransferReadOp>(xferOp.getOperation());
@@ -2154,45 +2316,21 @@ LogicalResult mlir::vector::splitFullAndPartialTransfer(
                        b.getI64IntegerAttr(32));
   }
 
-  Value memref = xferOp.memref();
-  SmallVector<bool, 4> bools(xferOp.getTransferRank(), false);
-  auto unmaskedAttr = b.getBoolArrayAttr(bools);
-
   MemRefType compatibleMemRefType = getCastCompatibleMemRefType(
       xferOp.getMemRefType(), alloc.getType().cast<MemRefType>());
 
   // Read case: full fill + partial copy -> unmasked vector.xfer_read.
-  Value zero = std_constant_index(0);
   SmallVector<Type, 4> returnTypes(1 + xferOp.getTransferRank(),
                                    b.getIndexType());
   returnTypes[0] = compatibleMemRefType;
-  scf::IfOp fullPartialIfOp;
-  conditionBuilder(
-      returnTypes, inBoundsCond,
-      [&]() -> scf::ValueVector {
-        Value res = memref;
-        if (compatibleMemRefType != xferOp.getMemRefType())
-          res = std_memref_cast(memref, compatibleMemRefType);
-        scf::ValueVector viewAndIndices{res};
-        viewAndIndices.insert(viewAndIndices.end(), xferOp.indices().begin(),
-                              xferOp.indices().end());
-        return viewAndIndices;
-      },
-      [&]() -> scf::ValueVector {
-        Operation *newXfer =
-            ScopedContext::getBuilderRef().clone(*xferOp.getOperation());
-        Value vector = cast<VectorTransferOpInterface>(newXfer).vector();
-        std_store(vector, vector_type_cast(
-                              MemRefType::get({}, vector.getType()), alloc));
-
-        Value casted = std_memref_cast(alloc, compatibleMemRefType);
-        scf::ValueVector viewAndIndices{casted};
-        viewAndIndices.insert(viewAndIndices.end(), xferOp.getTransferRank(),
-                              zero);
-
-        return viewAndIndices;
-      },
-      &fullPartialIfOp);
+  scf::IfOp fullPartialIfOp =
+      options.vectorTransferSplit == VectorTransferSplit::VectorTransfer
+          ? createScopedFullPartialVectorTransferRead(
+                xferReadOp, returnTypes, inBoundsCond, compatibleMemRefType,
+                alloc)
+          : createScopedFullPartialLinalgCopy(xferReadOp, returnTypes,
+                                              inBoundsCond,
+                                              compatibleMemRefType, alloc);
   if (ifOp)
     *ifOp = fullPartialIfOp;
 
@@ -2211,7 +2349,7 @@ LogicalResult mlir::vector::VectorTransferFullPartialRewriter::matchAndRewrite(
       failed(filter(xferOp)))
     return failure();
   rewriter.startRootUpdate(xferOp);
-  if (succeeded(splitFullAndPartialTransfer(rewriter, xferOp))) {
+  if (succeeded(splitFullAndPartialTransfer(rewriter, xferOp, options))) {
     rewriter.finalizeRootUpdate(xferOp);
     return success();
   }
diff --git a/mlir/test/Dialect/Vector/vector-transfer-full-partial-split.mlir b/mlir/test/Dialect/Vector/vector-transfer-full-partial-split.mlir
index ef76247ee9d4b..e364542039828 100644
--- a/mlir/test/Dialect/Vector/vector-transfer-full-partial-split.mlir
+++ b/mlir/test/Dialect/Vector/vector-transfer-full-partial-split.mlir
@@ -1,13 +1,26 @@
 // RUN: mlir-opt %s -test-vector-transfer-full-partial-split | FileCheck %s
+// RUN: mlir-opt %s -test-vector-transfer-full-partial-split=use-linalg-copy | FileCheck %s --check-prefix=LINALG
 
 // CHECK-DAG: #[[$map_p4:.*]] = affine_map<()[s0] -> (s0 + 4)>
 // CHECK-DAG: #[[$map_p8:.*]] = affine_map<()[s0] -> (s0 + 8)>
 // CHECK-DAG: #[[$map_2d_stride_1:.*]] = affine_map<(d0, d1)[s0, s1] -> (d0 * s1 + s0 + d1)>
 
+// LINALG-DAG: #[[$map_p4:.*]] = affine_map<()[s0] -> (s0 + 4)>
+// LINALG-DAG: #[[$map_p8:.*]] = affine_map<()[s0] -> (s0 + 8)>
+// LINALG-DAG: #[[$map_2d_stride_1:.*]] = affine_map<(d0, d1)[s0, s1] -> (d0 * s1 + s0 + d1)>
+// LINALG-DAG: #[[$map_2d_dynamic:.*]] = affine_map<(d0, d1)[s0, s1, s2] -> (d0 * s1 + s0 + d1 * s2)>
+// LINALG-DAG: #[[$bounds_map_4:.*]] = affine_map<(d0, d1, d2) -> (d0 - d1, 4)>
+// LINALG-DAG: #[[$bounds_map_8:.*]] = affine_map<(d0, d1, d2) -> (d0 - d1, 8)>
+
 // CHECK-LABEL: split_vector_transfer_read_2d(
 //  CHECK-SAME: %[[A:[a-zA-Z0-9]*]]: memref
 //  CHECK-SAME: %[[i:[a-zA-Z0-9]*]]: index
 //  CHECK-SAME: %[[j:[a-zA-Z0-9]*]]: index
+
+// LINALG-LABEL: split_vector_transfer_read_2d(
+//  LINALG-SAME: %[[A:[a-zA-Z0-9]*]]: memref
+//  LINALG-SAME: %[[i:[a-zA-Z0-9]*]]: index
+//  LINALG-SAME: %[[j:[a-zA-Z0-9]*]]: index
 func @split_vector_transfer_read_2d(%A: memref<?x8xf32>, %i: index, %j: index) -> vector<4x8xf32> {
   %c0 = constant 0 : index
   %f0 = constant 0.0 : f32
@@ -43,9 +56,45 @@ func @split_vector_transfer_read_2d(%A: memref<?x8xf32>, %i: index, %j: index) -
   //      CHECK: }
   //      CHECK: %[[res:.*]] = vector.transfer_read %[[ifres]]#0[%[[ifres]]#1, %[[ifres]]#2], %[[cst]]
   // CHECK_SAME:   {masked = [false, false]} : memref<?x8xf32>, vector<4x8xf32>
+
+  //  LINALG-DAG: %[[c0:.*]] = constant 0 : index
+  //  LINALG-DAG: %[[c1:.*]] = constant 1 : index
+  //  LINALG-DAG: %[[c4:.*]] = constant 4 : index
+  //  LINALG-DAG: %[[c8:.*]] = constant 8 : index
+  //  LINALG-DAG: %[[cst:.*]] = constant 0.000000e+00 : f32
+  // alloca for boundary full tile
+  //      LINALG: %[[alloc:.*]] = alloca() {alignment = 32 : i64} : memref<4x8xf32>
+  // %i + 4 <= dim(%A, 0)
+  //      LINALG: %[[idx0:.*]] = affine.apply #[[$map_p4]]()[%[[i]]]
+  //      LINALG: %[[d0:.*]] = dim %[[A]], %[[c0]] : memref<?x8xf32>
+  //      LINALG: %[[cmp0:.*]] = cmpi "sle", %[[idx0]], %[[d0]] : index
+  // %j + 8 <= dim(%A, 1)
+  //      LINALG: %[[idx1:.*]] = affine.apply #[[$map_p8]]()[%[[j]]]
+  //      LINALG: %[[cmp1:.*]] = cmpi "sle", %[[idx1]], %[[c8]] : index
+  // are both conds true
+  //      LINALG: %[[cond:.*]] = and %[[cmp0]], %[[cmp1]] : i1
+  //      LINALG: %[[ifres:.*]]:3 = scf.if %[[cond]] -> (memref<?x8xf32>, index, index) {
+  //               inBounds, just yield %A
+  //      LINALG:   scf.yield %[[A]], %[[i]], %[[j]] : memref<?x8xf32>, index, index
+  //      LINALG: } else {
+  //               slow path, fill tmp alloc and yield a memref_casted version of it
+  //      LINALG:   linalg.fill(%[[alloc]], %[[cst]]) : memref<4x8xf32>, f32
+  //      LINALG:   %[[d0:.*]] = dim %[[A]], %[[c0]] : memref<?x8xf32>
+  //      LINALG:   %[[sv0:.*]] = affine.min #[[$bounds_map_4]](%[[d0]], %[[i]], %[[c4]])
+  //      LINALG:   %[[sv1:.*]] = affine.min #[[$bounds_map_8]](%[[c8]], %[[j]], %[[c8]])
+  //      LINALG:   %[[sv:.*]] = subview %[[A]][%[[i]], %[[j]]] [%[[sv0]], %[[sv1]]] [%[[c1]], %[[c1]]]
+  // LINALG-SAME:     memref<?x8xf32> to memref<?x?xf32, #[[$map_2d_dynamic]]>
+  //      LINALG:   linalg.copy(%[[sv]], %[[alloc]]) : memref<?x?xf32, #[[$map_2d_dynamic]]>, memref<4x8xf32>
+  //      LINALG:   %[[yielded:.*]] = memref_cast %[[alloc]] :
+  // LINALG-SAME:     memref<4x8xf32> to memref<?x8xf32>
+  //      LINALG:   scf.yield %[[yielded]], %[[c0]], %[[c0]] :
+  // LINALG-SAME:     memref<?x8xf32>, index, index
+  //      LINALG: }
+  //      LINALG: %[[res:.*]] = vector.transfer_read %[[ifres]]#0[%[[ifres]]#1, %[[ifres]]#2], %[[cst]]
+  // LINALG_SAME:   {masked = [false, false]} : memref<?x8xf32>, vector<4x8xf32>
   %1 = vector.transfer_read %A[%i, %j], %f0 : memref<?x8xf32>, vector<4x8xf32>
 
-  // CHECK: return %[[res]] : vector<4x8xf32>
+  // LINALG: return %[[res]] : vector<4x8xf32>
   return %1: vector<4x8xf32>
 }
 
@@ -53,6 +102,11 @@ func @split_vector_transfer_read_2d(%A: memref<?x8xf32>, %i: index, %j: index) -
 //  CHECK-SAME: %[[A:[a-zA-Z0-9]*]]: memref
 //  CHECK-SAME: %[[i:[a-zA-Z0-9]*]]: index
 //  CHECK-SAME: %[[j:[a-zA-Z0-9]*]]: index
+
+// LINALG-LABEL: split_vector_transfer_read_strided_2d(
+//  LINALG-SAME: %[[A:[a-zA-Z0-9]*]]: memref
+//  LINALG-SAME: %[[i:[a-zA-Z0-9]*]]: index
+//  LINALG-SAME: %[[j:[a-zA-Z0-9]*]]: index
 func @split_vector_transfer_read_strided_2d(
     %A: memref<7x8xf32, offset:?, strides:[?, 1]>,
     %i: index, %j: index) -> vector<4x8xf32> {
@@ -94,6 +148,44 @@ func @split_vector_transfer_read_strided_2d(
   //      CHECK: }
   //      CHECK: %[[res:.*]] = vector.transfer_read {{.*}} {masked = [false, false]} :
   // CHECK-SAME:   memref<?x8xf32, #[[$map_2d_stride_1]]>, vector<4x8xf32>
+
+  //  LINALG-DAG: %[[c0:.*]] = constant 0 : index
+  //  LINALG-DAG: %[[c1:.*]] = constant 1 : index
+  //  LINALG-DAG: %[[c4:.*]] = constant 4 : index
+  //  LINALG-DAG: %[[c7:.*]] = constant 7 : index
+  //  LINALG-DAG: %[[c8:.*]] = constant 8 : index
+  //  LINALG-DAG: %[[cst:.*]] = constant 0.000000e+00 : f32
+  // alloca for boundary full tile
+  //      LINALG: %[[alloc:.*]] = alloca() {alignment = 32 : i64} : memref<4x8xf32>
+  // %i + 4 <= dim(%A, 0)
+  //      LINALG: %[[idx0:.*]] = affine.apply #[[$map_p4]]()[%[[i]]]
+  //      LINALG: %[[cmp0:.*]] = cmpi "sle", %[[idx0]], %[[c7]] : index
+  // %j + 8 <= dim(%A, 1)
+  //      LINALG: %[[idx1:.*]] = affine.apply #[[$map_p8]]()[%[[j]]]
+  //      LINALG: %[[cmp1:.*]] = cmpi "sle", %[[idx1]], %[[c8]] : index
+  // are both conds true
+  //      LINALG: %[[cond:.*]] = and %[[cmp0]], %[[cmp1]] : i1
+  //      LINALG: %[[ifres:.*]]:3 = scf.if %[[cond]] -> (memref<?x8xf32, #[[$map_2d_stride_1]]>, index, index) {
+  //               inBounds but not cast-compatible: yield a memref_casted form of %A
+  //      LINALG:   %[[casted:.*]] = memref_cast %arg0 :
+  // LINALG-SAME:     memref<7x8xf32, #[[$map_2d_stride_1]]> to memref<?x8xf32, #[[$map_2d_stride_1]]>
+  //      LINALG:   scf.yield %[[casted]], %[[i]], %[[j]] :
+  // LINALG-SAME:     memref<?x8xf32, #[[$map_2d_stride_1]]>, index, index
+  //      LINALG: } else {
+  //               slow path, fill tmp alloc and yield a memref_casted version of it
+  //      LINALG:   linalg.fill(%[[alloc]], %[[cst]]) : memref<4x8xf32>, f32
+  //      LINALG:   %[[sv0:.*]] = affine.min #[[$bounds_map_4]](%[[c7]], %[[i]], %[[c4]])
+  //      LINALG:   %[[sv1:.*]] = affine.min #[[$bounds_map_8]](%[[c8]], %[[j]], %[[c8]])
+  //      LINALG:   %[[sv:.*]] = subview %[[A]][%[[i]], %[[j]]] [%[[sv0]], %[[sv1]]] [%[[c1]], %[[c1]]]
+  // LINALG-SAME:     memref<7x8xf32, #[[$map_2d_stride_1]]> to memref<?x?xf32, #[[$map_2d_dynamic]]>
+  //      LINALG:   linalg.copy(%[[sv]], %[[alloc]]) : memref<?x?xf32, #[[$map_2d_dynamic]]>, memref<4x8xf32>
+  //      LINALG:   %[[yielded:.*]] = memref_cast %[[alloc]] :
+  // LINALG-SAME:     memref<4x8xf32> to memref<?x8xf32, #[[$map_2d_stride_1]]>
+  //      LINALG:   scf.yield %[[yielded]], %[[c0]], %[[c0]] :
+  // LINALG-SAME:     memref<?x8xf32, #[[$map_2d_stride_1]]>, index, index
+  //      LINALG: }
+  //      LINALG: %[[res:.*]] = vector.transfer_read {{.*}} {masked = [false, false]} :
+  // LINALG-SAME:   memref<?x8xf32, #[[$map_2d_stride_1]]>, vector<4x8xf32>
   %1 = vector.transfer_read %A[%i, %j], %f0 :
     memref<7x8xf32, offset:?, strides:[?, 1]>, vector<4x8xf32>
 
diff --git a/mlir/test/lib/Transforms/TestVectorTransforms.cpp b/mlir/test/lib/Transforms/TestVectorTransforms.cpp
index 0bba74e76385e..9da3156d53593 100644
--- a/mlir/test/lib/Transforms/TestVectorTransforms.cpp
+++ b/mlir/test/lib/Transforms/TestVectorTransforms.cpp
@@ -125,10 +125,23 @@ struct TestVectorUnrollingPatterns
 struct TestVectorTransferFullPartialSplitPatterns
     : public PassWrapper<TestVectorTransferFullPartialSplitPatterns,
                          FunctionPass> {
+  TestVectorTransferFullPartialSplitPatterns() = default;
+  TestVectorTransferFullPartialSplitPatterns(
+      const TestVectorTransferFullPartialSplitPatterns &pass) {}
+  Option<bool> useLinalgOps{
+      *this, "use-linalg-copy",
+      llvm::cl::desc("Split using a unmasked vector.transfer + linalg.fill + "
+                     "linalg.copy operations."),
+      llvm::cl::init(false)};
   void runOnFunction() override {
     MLIRContext *ctx = &getContext();
     OwningRewritePatternList patterns;
-    patterns.insert<VectorTransferFullPartialRewriter>(ctx);
+    VectorTransformsOptions options;
+    if (useLinalgOps)
+      options.setVectorTransferSplit(VectorTransferSplit::LinalgCopy);
+    else
+      options.setVectorTransferSplit(VectorTransferSplit::VectorTransfer);
+    patterns.insert<VectorTransferFullPartialRewriter>(ctx, options);
     applyPatternsAndFoldGreedily(getFunction(), patterns);
   }
 };

From 586f020841b3971d4e4c49d9dcc7ad19c4af6fb3 Mon Sep 17 00:00:00 2001
From: Soumi Manna <soumi.manna@intel.com>
Date: Tue, 4 Aug 2020 06:34:10 -0700
Subject: [PATCH 1035/1035] Fix test failure

Signed-off-by: Soumi Manna <soumi.manna@intel.com>
---
 sycl/test/union_param/union_kernel_param.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/sycl/test/union_param/union_kernel_param.cpp b/sycl/test/union_param/union_kernel_param.cpp
index 09a4bc4786e30..a05bf95a97b33 100644
--- a/sycl/test/union_param/union_kernel_param.cpp
+++ b/sycl/test/union_param/union_kernel_param.cpp
@@ -73,10 +73,10 @@ bool test0() {
   if (!Passed) {
     std::cout << "test0 failed" << std::endl;
     std::cout << "test0 input:" << std::endl;
-    printStruct(S);
+    printUnion(S);
     std::cout << std::endl;
     std::cout << "test0 result:\n";
-    printStruct(S0);
+    printUnion(S0);
     std::cout << std::endl;
   }
   return Passed;